diff --git a/.get_maintainer.ignore b/.get_maintainer.ignore index a64d219137455f407a7b1f2c6b156c5575852e9e..c298bab3d3207fc5c6dd81d843b8a145ea8c655d 100644 --- a/.get_maintainer.ignore +++ b/.get_maintainer.ignore @@ -1,2 +1,4 @@ +Alan Cox +Alan Cox Christoph Hellwig Marc Gonzalez diff --git a/.gitignore b/.gitignore index 265959544978a795516c321eec0659725a970a90..5da004814678d09c9b836651ba53425aae0df993 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ *.o *.o.* *.patch +*.rmeta +*.rsi *.s *.so *.so.dbg @@ -97,6 +99,7 @@ modules.order !.gitattributes !.gitignore !.mailmap +!.rustfmt.toml # # Generated include files @@ -162,3 +165,6 @@ x509.genkey # Documentation toolchain sphinx_*/ + +# Rust analyzer configuration +/rust-project.json diff --git a/.mailmap b/.mailmap index 38255d412f0b3a82f26d63b4c52f84e3c50d4cc0..380378e2db368fec6d15235f6e76b103228f994f 100644 --- a/.mailmap +++ b/.mailmap @@ -71,6 +71,9 @@ Ben M Cahill Ben Widawsky Ben Widawsky Ben Widawsky +Bjorn Andersson +Bjorn Andersson +Bjorn Andersson Björn Steinbrink Björn Töpel Björn Töpel @@ -98,8 +101,7 @@ Christian Brauner Christian Marangi Christophe Ricard Christoph Hellwig -Colin Ian King -Colin Ian King +Colin Ian King Corey Minyard Damian Hobson-Garcia Daniel Borkmann @@ -135,6 +137,7 @@ Filipe Lautert Finn Thain Franck Bui-Huu Frank Rowand +Frank Rowand Frank Rowand Frank Rowand Frank Zago @@ -150,6 +153,8 @@ Greg Kroah-Hartman Greg Kroah-Hartman Greg Kurz Gregory CLEMENT +Guilherme G. Piccoli +Guilherme G. Piccoli Guo Ren Guo Ren Gustavo Padovan @@ -253,6 +258,7 @@ Linus Lüssing Li Yang Li Yang Lorenzo Pieralisi +Luca Ceresoli Lukasz Luba Maciej W. Rozycki Maciej W. Rozycki @@ -313,6 +319,7 @@ Morten Welinder Mythri P K Nadia Yvette Chambers William Lee Irwin III Nathan Chancellor +Neil Armstrong Nguyen Anh Quynh Nicholas Piggin Nicholas Piggin @@ -330,6 +337,7 @@ Oleksij Rempel Oleksij Rempel Oleksij Rempel Oleksij Rempel +Oliver Upton Pali Rohár Paolo 'Blaisorblade' Giarrusso Patrick Mochel diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000000000000000000000000000000000000..3de5cc497465c2722e160cb76188a8b287e48b7f --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,12 @@ +edition = "2021" +newline_style = "Unix" + +# Unstable options that help catching some mistakes in formatting and that we may want to enable +# when they become stable. +# +# They are kept here since they are useful to run from time to time. +#format_code_in_doc_comments = true +#reorder_impl_items = true +#comment_width = 100 +#wrap_comments = true +#normalize_comments = true diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 0c2b613f2373608511fb286bb34cbd8d7184881a..8e2c2c405db22d411b7f86f8c0af634b09a3f36a 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -227,6 +227,17 @@ Contact: dmaengine@vger.kernel.org Description: Indicate the number of retires for an enqcmds submission on a sharedwq. A max value to set attribute is capped at 64. +What: /sys/bus/dsa/devices/wq./op_config +Date: Sept 14, 2022 +KernelVersion: 6.0.0 +Contact: dmaengine@vger.kernel.org +Description: Shows the operation capability bits displayed in bitmap format + presented by %*pb printk() output format specifier. + The attribute can be configured when the WQ is disabled in + order to configure the WQ to accept specific bits that + correlates to the operations allowed. It's visible only + on platforms that support the capability. + What: /sys/bus/dsa/devices/engine./group_id Date: Oct 25, 2019 KernelVersion: 5.6.0 @@ -255,3 +266,27 @@ Contact: dmaengine@vger.kernel.org Description: Indicates the number of Read Buffers reserved for the use of engines in the group. See DSA spec v1.2 9.2.18 GRPCFG Read Buffers Reserved. + +What: /sys/bus/dsa/devices/group./desc_progress_limit +Date: Sept 14, 2022 +KernelVersion: 6.0.0 +Contact: dmaengine@vger.kernel.org +Description: Allows control of the number of work descriptors that can be + concurrently processed by an engine in the group as a fraction + of the Maximum Work Descriptors in Progress value specified in + the ENGCAP register. The acceptable values are 0 (default), + 1 (1/2 of max value), 2 (1/4 of the max value), and 3 (1/8 of + the max value). It's visible only on platforms that support + the capability. + +What: /sys/bus/dsa/devices/group./batch_progress_limit +Date: Sept 14, 2022 +KernelVersion: 6.0.0 +Contact: dmaengine@vger.kernel.org +Description: Allows control of the number of batch descriptors that can be + concurrently processed by an engine in the group as a fraction + of the Maximum Batch Descriptors in Progress value specified in + the ENGCAP register. The acceptable values are 0 (default), + 1 (1/2 of max value), 2 (1/4 of the max value), and 3 (1/8 of + the max value). It's visible only on platforms that support + the capability. diff --git a/Documentation/ABI/testing/debugfs-cros-ec b/Documentation/ABI/testing/debugfs-cros-ec index 1fe0add99a2a991f4d5b1824ded0526c59be65a6..9a040c6f5e03083456eb86feb6ba887ff7e5ab55 100644 --- a/Documentation/ABI/testing/debugfs-cros-ec +++ b/Documentation/ABI/testing/debugfs-cros-ec @@ -54,3 +54,25 @@ Description: this feature. Output will be in the format: "0x%08x\n". + +What: /sys/kernel/debug//suspend_timeout_ms +Date: August 2022 +KernelVersion: 6.1 +Description: + Some ECs have a feature where they will track transitions of + a hardware-controlled sleep line, such as Intel's SLP_S0 line, + in order to detect cases where a system failed to go into deep + sleep states. The suspend_timeout_ms file controls the amount of + time in milliseconds the EC will wait before declaring a sleep + timeout event and attempting to wake the system. + + Supply 0 to use the default value coded into EC firmware. Supply + 65535 (EC_HOST_SLEEP_TIMEOUT_INFINITE) to disable the EC sleep + failure detection mechanism. Values in between 0 and 65535 + indicate the number of milliseconds the EC should wait after a + sleep transition before declaring a timeout. This includes both + the duration after a sleep command was received but before the + hardware line changed, as well as the duration between when the + hardware line changed and the kernel sent an EC resume command. + + Output will be in the format: "%u\n". diff --git a/Documentation/ABI/testing/sysfs-amd-pmc b/Documentation/ABI/testing/sysfs-amd-pmc new file mode 100644 index 0000000000000000000000000000000000000000..c421b72844f1e660b75ba2704031662d753c8c97 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-amd-pmc @@ -0,0 +1,13 @@ +What: /sys/bus/platform/drivers/amd_pmc/*/smu_fw_version +Date: October 2022 +Contact: Mario Limonciello +Description: Reading this file reports the version of the firmware loaded to + System Management Unit (SMU) contained in AMD CPUs and + APUs. + +What: /sys/bus/platform/drivers/amd_pmc/*/smu_program +Date: October 2022 +Contact: Mario Limonciello +Description: Reading this file reports the program corresponding to the SMU + firmware version. The program field is used to disambiguate two + APU/CPU models that can share the same firmware binary. diff --git a/Documentation/ABI/testing/sysfs-amd-pmf b/Documentation/ABI/testing/sysfs-amd-pmf new file mode 100644 index 0000000000000000000000000000000000000000..7fc0e1c2b76b43c1d338507abc11d76b6c95639b --- /dev/null +++ b/Documentation/ABI/testing/sysfs-amd-pmf @@ -0,0 +1,13 @@ +What: /sys/devices/platform/*/cnqf_enable +Date: September 2022 +Contact: Shyam Sundar S K +Description: Reading this file tells if the AMD Platform Management(PMF) + Cool n Quiet Framework(CnQF) feature is enabled or not. + + This feature is not enabled by default and gets only turned on + if OEM BIOS passes a "flag" to PMF ACPI function (index 11 or 12) + or in case the user writes "on". + + To turn off CnQF user can write "off" to the sysfs node. + Note: Systems that support auto mode will not have this sysfs file + available. diff --git a/Documentation/ABI/testing/sysfs-bus-bcma b/Documentation/ABI/testing/sysfs-bus-bcma index 721b4aea3020a43f4722e61c6cff9cede284e664..e93d3ddca844e81288642bc8f5ae395084136afc 100644 --- a/Documentation/ABI/testing/sysfs-bus-bcma +++ b/Documentation/ABI/testing/sysfs-bus-bcma @@ -3,7 +3,7 @@ Date: May 2011 KernelVersion: 3.0 Contact: Rafał Miłecki Description: - Each BCMA core has it's manufacturer id. See + Each BCMA core has its manufacturer id. See include/linux/bcma/bcma.h for possible values. What: /sys/bus/bcma/devices/.../id diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x index 8e53a32f815056f97220b2092249128c12a848d0..08b1964f27d3e147c0c59d57f7f21ce5c1aa74c1 100644 --- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x @@ -516,3 +516,11 @@ Contact: Mathieu Poirier Description: (Read) Returns the number of special conditional P1 right-hand keys that the trace unit can use (0x194). The value is taken directly from the HW. + +What: /sys/bus/coresight/devices/etm/ts_source +Date: October 2022 +KernelVersion: 6.1 +Contact: Mathieu Poirier or Suzuki K Poulose +Description: (Read) When FEAT_TRF is implemented, value of TRFCR_ELx.TS used for + trace session. Otherwise -1 indicates an unknown time source. Check + trcidr0.tssize to see if a global timestamp is available. diff --git a/Documentation/ABI/testing/sysfs-bus-counter b/Documentation/ABI/testing/sysfs-bus-counter index 06c2b3e27e0bf5a4a0dc3a3bf51482f8473ea00c..ff83320b42552a4290dcad7d8ec9bb9ab6a82a1e 100644 --- a/Documentation/ABI/testing/sysfs-bus-counter +++ b/Documentation/ABI/testing/sysfs-bus-counter @@ -4,6 +4,12 @@ Contact: linux-iio@vger.kernel.org Description: Count data of Count Y represented as a string. +What: /sys/bus/counter/devices/counterX/countY/capture +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Historical capture of the Count Y count data. + What: /sys/bus/counter/devices/counterX/countY/ceiling KernelVersion: 5.2 Contact: linux-iio@vger.kernel.org @@ -203,6 +209,13 @@ Description: both edges: Any state transition. +What: /sys/bus/counter/devices/counterX/countY/num_overflows +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + This attribute indicates the number of overflows of count Y. + +What: /sys/bus/counter/devices/counterX/countY/capture_component_id What: /sys/bus/counter/devices/counterX/countY/ceiling_component_id What: /sys/bus/counter/devices/counterX/countY/floor_component_id What: /sys/bus/counter/devices/counterX/countY/count_mode_component_id @@ -213,11 +226,14 @@ What: /sys/bus/counter/devices/counterX/countY/prescaler_component_id What: /sys/bus/counter/devices/counterX/countY/preset_component_id What: /sys/bus/counter/devices/counterX/countY/preset_enable_component_id What: /sys/bus/counter/devices/counterX/countY/signalZ_action_component_id +What: /sys/bus/counter/devices/counterX/countY/num_overflows_component_id What: /sys/bus/counter/devices/counterX/signalY/cable_fault_component_id What: /sys/bus/counter/devices/counterX/signalY/cable_fault_enable_component_id What: /sys/bus/counter/devices/counterX/signalY/filter_clock_prescaler_component_id What: /sys/bus/counter/devices/counterX/signalY/index_polarity_component_id +What: /sys/bus/counter/devices/counterX/signalY/polarity_component_id What: /sys/bus/counter/devices/counterX/signalY/synchronous_mode_component_id +What: /sys/bus/counter/devices/counterX/signalY/frequency_component_id KernelVersion: 5.16 Contact: linux-iio@vger.kernel.org Description: @@ -303,6 +319,19 @@ Description: Discrete set of available values for the respective Signal Y configuration are listed in this file. +What: /sys/bus/counter/devices/counterX/signalY/polarity +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Active level of Signal Y. The following polarity values are + available: + + positive: + Signal high state considered active level (rising edge). + + negative: + Signal low state considered active level (falling edge). + What: /sys/bus/counter/devices/counterX/signalY/name KernelVersion: 5.2 Contact: linux-iio@vger.kernel.org @@ -345,3 +374,9 @@ Description: via index_polarity. The index function (as enabled via preset_enable) is performed synchronously with the quadrature clock on the active level of the index input. + +What: /sys/bus/counter/devices/counterX/signalY/frequency +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Read-only attribute that indicates the signal Y frequency, in Hz. diff --git a/Documentation/ABI/testing/sysfs-bus-fcoe b/Documentation/ABI/testing/sysfs-bus-fcoe index 8fe787cc4ab70a4c0fe05949f7db84f640dddfb5..5a4f2091ac370379e5e08a65a66b50bf28826337 100644 --- a/Documentation/ABI/testing/sysfs-bus-fcoe +++ b/Documentation/ABI/testing/sysfs-bus-fcoe @@ -31,7 +31,7 @@ Description: 'FCoE Controller' instances on the fcoe bus. 1) Write interface name to ctlr_create 2) Configure the FCoE Controller (ctlr_X) 3) Enable the FCoE Controller to begin discovery and login. The FCoE Controller is destroyed by - writing it's name, i.e. ctlr_X to the ctlr_delete file. + writing its name, i.e. ctlr_X to the ctlr_delete file. Attributes: diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio index e81ba6f5e1c8732fa9a1c168d493ec684de05e82..6ba34c0d97897e7f1b8770329bbc4bba38dfc67b 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio +++ b/Documentation/ABI/testing/sysfs-bus-iio @@ -196,7 +196,7 @@ Description: Raw capacitance measurement from channel Y. Units after application of scale and offset are nanofarads. -What: /sys/.../iio:deviceX/in_capacitanceY-in_capacitanceZ_raw +What: /sys/.../iio:deviceX/in_capacitanceY-capacitanceZ_raw KernelVersion: 3.2 Contact: linux-iio@vger.kernel.org Description: @@ -207,6 +207,25 @@ Description: is required is a consistent labeling. Units after application of scale and offset are nanofarads. +What: /sys/.../iio:deviceX/in_capacitanceY-capacitanceZ_zeropoint +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + For differential channels, this an offset that is applied + equally to both inputs. As the reading is of the difference + between the two inputs, this should not be applied to the _raw + reading by userspace (unlike _offset) and unlike calibbias + it does not affect the differential value measured because + the effect of _zeropoint cancels out across the two inputs + that make up the differential pair. It's purpose is to bring + the individual signals, before the differential is measured, + within the measurement range of the device. The naming is + chosen because if the separate inputs that make the + differential pair are drawn on a graph in their + _raw units, this is the value that the zero point on the + measurement axis represents. It is expressed with the + same scaling as _raw. + What: /sys/bus/iio/devices/iio:deviceX/in_temp_raw What: /sys/bus/iio/devices/iio:deviceX/in_tempX_raw What: /sys/bus/iio/devices/iio:deviceX/in_temp_x_raw @@ -241,6 +260,15 @@ Description: Has all of the equivalent parameters as per voltageY. Units after application of scale and offset are m/s^2. +What: /sys/bus/iio/devices/iio:deviceX/in_accel_linear_x_raw +What: /sys/bus/iio/devices/iio:deviceX/in_accel_linear_y_raw +What: /sys/bus/iio/devices/iio:deviceX/in_accel_linear_z_raw +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + As per in_accel_X_raw attributes, but minus the + acceleration due to gravity. + What: /sys/bus/iio/devices/iio:deviceX/in_gravity_x_raw What: /sys/bus/iio/devices/iio:deviceX/in_gravity_y_raw What: /sys/bus/iio/devices/iio:deviceX/in_gravity_z_raw @@ -2038,3 +2066,99 @@ Description: Available range for the forced calibration value, expressed as: - a range specified as "[min step max]" + +What: /sys/bus/iio/devices/iio:deviceX/in_voltageX_sampling_frequency +What: /sys/bus/iio/devices/iio:deviceX/in_powerY_sampling_frequency +What: /sys/bus/iio/devices/iio:deviceX/in_currentZ_sampling_frequency +KernelVersion: 5.20 +Contact: linux-iio@vger.kernel.org +Description: + Some devices have separate controls of sampling frequency for + individual channels. If multiple channels are enabled in a scan, + then the sampling_frequency of the scan may be computed from the + per channel sampling frequencies. + +What: /sys/.../events/in_accel_gesture_singletap_en +What: /sys/.../events/in_accel_gesture_doubletap_en +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Device generates an event on a single or double tap. + +What: /sys/.../events/in_accel_gesture_singletap_value +What: /sys/.../events/in_accel_gesture_doubletap_value +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Specifies the threshold value that the device is comparing + against to generate the tap gesture event. The lower + threshold value increases the sensitivity of tap detection. + Units and the exact meaning of value are device-specific. + +What: /sys/.../events/in_accel_gesture_tap_value_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Lists all available threshold values which can be used to + modify the sensitivity of the tap detection. + +What: /sys/.../events/in_accel_gesture_singletap_reset_timeout +What: /sys/.../events/in_accel_gesture_doubletap_reset_timeout +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Specifies the timeout value in seconds for the tap detector + to not to look for another tap event after the event as + occurred. Basically the minimum quiet time between the two + single-tap's or two double-tap's. + +What: /sys/.../events/in_accel_gesture_tap_reset_timeout_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Lists all available tap reset timeout values. Units in seconds. + +What: /sys/.../events/in_accel_gesture_doubletap_tap2_min_delay +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Specifies the minimum quiet time in seconds between the two + taps of a double tap. + +What: /sys/.../events/in_accel_gesture_doubletap_tap2_min_delay_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Lists all available delay values between two taps in the double + tap. Units in seconds. + +What: /sys/.../events/in_accel_gesture_tap_maxtomin_time +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Specifies the maximum time difference allowed between upper + and lower peak of tap to consider it as the valid tap event. + Units in seconds. + +What: /sys/.../events/in_accel_gesture_tap_maxtomin_time_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Lists all available time values between upper peak to lower + peak. Units in seconds. + +What: /sys/bus/iio/devices/iio:deviceX/in_rot_yaw_raw +What: /sys/bus/iio/devices/iio:deviceX/in_rot_pitch_raw +What: /sys/bus/iio/devices/iio:deviceX/in_rot_roll_raw +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Raw (unscaled) euler angles readings. Units after + application of scale are deg. + +What: /sys/bus/iio/devices/iio:deviceX/serialnumber +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + An example format is 16-bytes, 2-digits-per-byte, HEX-string + representing the sensor unique ID number. diff --git a/Documentation/ABI/testing/sysfs-bus-iio-bno055 b/Documentation/ABI/testing/sysfs-bus-iio-bno055 new file mode 100644 index 0000000000000000000000000000000000000000..f32b1644e98689f4977bec5d6ccce4fbdac54e48 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-iio-bno055 @@ -0,0 +1,81 @@ +What: /sys/bus/iio/devices/iio:deviceX/in_accel_raw_range +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Raw (unscaled) range for acceleration readings. Unit after + application of scale is m/s^2. Note that this doesn't affects + the scale (which should be used when changing the maximum and + minimum readable value affects also the reading scaling factor). + +What: /sys/bus/iio/devices/iio:deviceX/in_anglvel_raw_range +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Range for angular velocity readings in radians per second. Note + that this does not affects the scale (which should be used when + changing the maximum and minimum readable value affects also the + reading scaling factor). + +What: /sys/bus/iio/devices/iio:deviceX/in_accel_raw_range_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + List of allowed values for in_accel_raw_range attribute + +What: /sys/bus/iio/devices/iio:deviceX/in_anglvel_raw_range_available +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + List of allowed values for in_anglvel_raw_range attribute + +What: /sys/bus/iio/devices/iio:deviceX/in_magn_calibration_fast_enable +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Can be 1 or 0. Enables/disables the "Fast Magnetometer + Calibration" HW function. + +What: /sys/bus/iio/devices/iio:deviceX/fusion_enable +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Can be 1 or 0. Enables/disables the "sensor fusion" (a.k.a. + NDOF) HW function. + +What: /sys/bus/iio/devices/iio:deviceX/calibration_data +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Reports the binary calibration data blob for the IMU sensors. + +What: /sys/bus/iio/devices/iio:deviceX/in_accel_calibration_auto_status +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Reports the autocalibration status for the accelerometer sensor. + Can be 0 (calibration non even enabled) or 1 to 5 where the greater + the number, the better the calibration status. + +What: /sys/bus/iio/devices/iio:deviceX/in_gyro_calibration_auto_status +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Reports the autocalibration status for the gyroscope sensor. + Can be 0 (calibration non even enabled) or 1 to 5 where the greater + the number, the better the calibration status. + +What: /sys/bus/iio/devices/iio:deviceX/in_magn_calibration_auto_status +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Reports the autocalibration status for the magnetometer sensor. + Can be 0 (calibration non even enabled) or 1 to 5 where the greater + the number, the better the calibration status. + +What: /sys/bus/iio/devices/iio:deviceX/sys_calibration_auto_status +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Reports the status for the IMU overall autocalibration. + Can be 0 (calibration non even enabled) or 1 to 5 where the greater + the number, the better the calibration status. diff --git a/Documentation/ABI/testing/sysfs-bus-iio-cdc-ad7746 b/Documentation/ABI/testing/sysfs-bus-iio-cdc-ad7746 new file mode 100644 index 0000000000000000000000000000000000000000..02ca8941dce13c18bfc500ea9cb77739c8ffeab1 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-iio-cdc-ad7746 @@ -0,0 +1,11 @@ +What: /sys/.../iio:deviceX/in_capacitableY_calibbias_calibration +What: /sys/.../iio:deviceX/in_capacitableY_calibscale_calibration +KernelVersion: 6.1 +Contact: linux-iio@vger.kernel.org +Description: + Write 1 to trigger a calibration of the calibbias or + calibscale. For calibscale, a full scale capacitance should + be connected to the capacitance input and a + calibscale_calibration then started. For calibbias see + the device datasheet section on "capacitive system offset + calibration". diff --git a/Documentation/ABI/testing/sysfs-bus-iio-proximity b/Documentation/ABI/testing/sysfs-bus-iio-proximity index 3aac6dab8775d7715b9d7b33e2ee96d4e21b2b1b..9b9d1cc9b7036ba218fcd8162bcac0391e0ef53f 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio-proximity +++ b/Documentation/ABI/testing/sysfs-bus-iio-proximity @@ -18,7 +18,7 @@ Description: on the signal from which time of flight measurements are taken. The appropriate values to take is dependent on both the - sensor and it's operating environment: + sensor and its operating environment: * as3935 (0-31 range) 18 = indoors (default) 14 = outdoors diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index 6fc2c2efe8ab2655c9ce697865a45a0aaf1ec798..840727fc75dcf90347455225ca1df58dc80400ea 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -457,3 +457,36 @@ Description: The file is writable if the PF is bound to a driver that implements ->sriov_set_msix_vec_count(). + +What: /sys/bus/pci/devices/.../resourceN_resize +Date: September 2022 +Contact: Alex Williamson +Description: + These files provide an interface to PCIe Resizable BAR support. + A file is created for each BAR resource (N) supported by the + PCIe Resizable BAR extended capability of the device. Reading + each file exposes the bitmap of available resource sizes: + + # cat resource1_resize + 00000000000001c0 + + The bitmap represents supported resource sizes for the BAR, + where bit0 = 1MB, bit1 = 2MB, bit2 = 4MB, etc. In the above + example the device supports 64MB, 128MB, and 256MB BAR sizes. + + When writing the file, the user provides the bit position of + the desired resource size, for example: + + # echo 7 > resource1_resize + + This indicates to set the size value corresponding to bit 7, + 128MB. The resulting size is 2 ^ (bit# + 20). This definition + matches the PCIe specification of this capability. + + In order to make use of resource resizing, all PCI drivers must + be unbound from the device and peer devices under the same + parent bridge may need to be soft removed. In the case of + VGA devices, writing a resize value will remove low level + console drivers from the device. Raw users of pci-sysfs + resourceN attributes must be terminated prior to resizing. + Success of the resizing operation is not guaranteed. diff --git a/Documentation/ABI/testing/sysfs-bus-thunderbolt b/Documentation/ABI/testing/sysfs-bus-thunderbolt index f7570c240ce8ec3a4fcf87bc1332cc99b344773c..76ab3e1fe374c4d0d42998daeb63f03fe0a164b0 100644 --- a/Documentation/ABI/testing/sysfs-bus-thunderbolt +++ b/Documentation/ABI/testing/sysfs-bus-thunderbolt @@ -153,7 +153,7 @@ Date: Jan 2020 KernelVersion: 5.5 Contact: Mika Westerberg Description: This attribute reports number of RX lanes the device is - using simultaneusly through its upstream port. + using simultaneously through its upstream port. What: /sys/bus/thunderbolt/devices/.../tx_speed Date: Jan 2020 @@ -167,7 +167,7 @@ Date: Jan 2020 KernelVersion: 5.5 Contact: Mika Westerberg Description: This attribute reports number of TX lanes the device is - using simultaneusly through its upstream port. + using simultaneously through its upstream port. What: /sys/bus/thunderbolt/devices/.../vendor Date: Sep 2017 diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power index a9ce63cfbe87c79c98bc34718d79817bbeb5645d..e434fc523291d85b10503c45d394032d7359d086 100644 --- a/Documentation/ABI/testing/sysfs-class-power +++ b/Documentation/ABI/testing/sysfs-class-power @@ -364,7 +364,10 @@ Date: April 2019 Contact: linux-pm@vger.kernel.org Description: Represents a battery percentage level, above which charging will - stop. + stop. Not all hardware is capable of setting this to an arbitrary + percentage. Drivers will round written values to the nearest + supported value. Reading back the value will show the actual + threshold set by the driver. Access: Read, Write diff --git a/Documentation/ABI/testing/sysfs-devices-hisi_ptt b/Documentation/ABI/testing/sysfs-devices-hisi_ptt new file mode 100644 index 0000000000000000000000000000000000000000..82de6d7102660273e941857ea4cce26024117cc1 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-hisi_ptt @@ -0,0 +1,61 @@ +What: /sys/devices/hisi_ptt_/tune +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: This directory contains files for tuning the PCIe link + parameters(events). Each file is named after the event + of the PCIe link. + + See Documentation/trace/hisi-ptt.rst for more information. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_cpl +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx completion TLPs, which influence + the proportion of outbound completion TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_np +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx non-posted TLPs, which influence + the proportion of outbound non-posted TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/qos_tx_p +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Controls the weight of Tx posted TLPs, which influence the + proportion of outbound posted TLPs on the PCIe link. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/rx_alloc_buf_level +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Control the allocated buffer watermark for inbound packets. + The packets will be stored in the buffer first and then transmitted + either when the watermark reached or when timed out. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. + +What: /sys/devices/hisi_ptt_/tune/tx_alloc_buf_level +Date: October 2022 +KernelVersion: 6.1 +Contact: Yicong Yang +Description: (RW) Control the allocated buffer watermark of outbound packets. + The packets will be stored in the buffer first and then transmitted + either when the watermark reached or when timed out. + The available tune data is [0, 1, 2]. Writing a negative value + will return an error, and out of range values will be converted + to 2. The value indicates a probable level of the event. diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index 5bf61881f01269196fa97a4d5ec325cd330cdb1a..f54867cadb0f68a80151cf6a59706119b6119047 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -296,7 +296,7 @@ Description: Processor frequency boosting control This switch controls the boost setting for the whole system. Boosting allows the CPU and the firmware to run at a frequency - beyond it's nominal limit. + beyond its nominal limit. More details can be found in Documentation/admin-guide/pm/cpufreq.rst @@ -523,6 +523,7 @@ What: /sys/devices/system/cpu/vulnerabilities /sys/devices/system/cpu/vulnerabilities/tsx_async_abort /sys/devices/system/cpu/vulnerabilities/itlb_multihit /sys/devices/system/cpu/vulnerabilities/mmio_stale_data + /sys/devices/system/cpu/vulnerabilities/retbleed Date: January 2018 Contact: Linux kernel mailing list Description: Information about CPU vulnerabilities diff --git a/Documentation/ABI/testing/sysfs-devices-vfio-dev b/Documentation/ABI/testing/sysfs-devices-vfio-dev new file mode 100644 index 0000000000000000000000000000000000000000..e21424fd9666296caf63a4e7887b95b8bad8aa33 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-vfio-dev @@ -0,0 +1,8 @@ +What: /sys/...//vfio-dev/vfioX/ +Date: September 2022 +Contact: Yi Liu +Description: + This directory is created when the device is bound to a + vfio driver. The layout under this directory matches what + exists for a standard 'struct device'. 'X' is a unique + index marking this device in vfio. diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs index 96646fb2e7a1df7b3e2684784955ae238d65282b..13b5b2ec3be7698678beba7be0c1b8654be10f73 100644 --- a/Documentation/ABI/testing/sysfs-driver-habanalabs +++ b/Documentation/ABI/testing/sysfs-driver-habanalabs @@ -16,7 +16,7 @@ Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/clk_max_freq_mhz Date: Jun 2019 -KernelVersion: not yet upstreamed +KernelVersion: 5.7 Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in MHz. The device clock might be set to lower value than the maximum. @@ -26,7 +26,7 @@ Description: Allows the user to set the maximum clock frequency, in MHz. What: /sys/class/habanalabs/hl/clk_cur_freq_mhz Date: Jun 2019 -KernelVersion: not yet upstreamed +KernelVersion: 5.7 Contact: ogabbay@kernel.org Description: Displays the current frequency, in MHz, of the device clock. This property is valid only for the Gaudi ASIC family @@ -176,6 +176,12 @@ KernelVersion: 5.1 Contact: ogabbay@kernel.org Description: Version of the device's preboot F/W code +What: /sys/class/habanalabs/hl/security_enabled +Date: Oct 2022 +KernelVersion: 6.1 +Contact: obitton@habana.ai +Description: Displays the device's security status + What: /sys/class/habanalabs/hl/soft_reset Date: Jan 2019 KernelVersion: 5.1 @@ -230,6 +236,6 @@ Description: Version of the u-boot running on the device's CPU What: /sys/class/habanalabs/hl/vrm_ver Date: Jan 2022 -KernelVersion: not yet upstreamed +KernelVersion: 5.17 Contact: ogabbay@kernel.org Description: Version of the Device's Voltage Regulator Monitor F/W code. N/A to GOYA and GAUDI diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index 6b248abb1bd71df0e80f5185e7f54c38212ce5dd..228aa43e14ed5d38cf01af24c42d821dbbe15c94 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -1417,6 +1417,15 @@ Description: This node is used to set or display whether UFS WriteBooster is platform that doesn't support UFSHCD_CAP_CLK_SCALING, we can disable/enable WriteBooster through this sysfs node. +What: /sys/bus/platform/drivers/ufshcd/*/enable_wb_buf_flush +What: /sys/bus/platform/devices/*.ufs/enable_wb_buf_flush +Date: July 2022 +Contact: Jinyoung Choi +Description: This entry shows the status of WriteBooster buffer flushing + and it can be used to enable or disable the flushing. + If flushing is enabled, the device executes the flush + operation when the command queue is empty. + What: /sys/bus/platform/drivers/ufshcd/*/device_descriptor/hpb_version What: /sys/bus/platform/devices/*.ufs/device_descriptor/hpb_version Date: June 2021 @@ -1591,6 +1600,43 @@ Description: This entry shows the status of HPB. The file is read only. +Contact: Daniil Lunev +What: /sys/bus/platform/drivers/ufshcd/*/capabilities/ +What: /sys/bus/platform/devices/*.ufs/capabilities/ +Date: August 2022 +Description: The group represents the effective capabilities of the + host-device pair. i.e. the capabilities which are enabled in the + driver for the specific host controller, supported by the host + controller and are supported and/or have compatible + configuration on the device side. + +Contact: Daniil Lunev +What: /sys/bus/platform/drivers/ufshcd/*/capabilities/clock_scaling +What: /sys/bus/platform/devices/*.ufs/capabilities/clock_scaling +Date: August 2022 +Contact: Daniil Lunev +Description: Indicates status of clock scaling. + + == ============================ + 0 Clock scaling is not supported. + 1 Clock scaling is supported. + == ============================ + + The file is read only. + +What: /sys/bus/platform/drivers/ufshcd/*/capabilities/write_booster +What: /sys/bus/platform/devices/*.ufs/capabilities/write_booster +Date: August 2022 +Contact: Daniil Lunev +Description: Indicates status of Write Booster. + + == ============================ + 0 Write Booster can not be enabled. + 1 Write Booster can be enabled. + == ============================ + + The file is read only. + What: /sys/class/scsi_device/*/device/hpb_param_sysfs/activation_thld Date: February 2021 Contact: Avri Altman diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 083ac2d63eefd1cdda8970800ad787979989b10c..483639fb727b203e405b125a21611aaca2cfd487 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -466,6 +466,30 @@ Description: Show status of f2fs superblock in real time. 0x4000 SBI_IS_FREEZING freefs is in process ====== ===================== ================================= +What: /sys/fs/f2fs//stat/cp_status +Date: September 2022 +Contact: "Chao Yu" +Description: Show status of f2fs checkpoint in real time. + + =============================== ============================== + cp flag value + CP_UMOUNT_FLAG 0x00000001 + CP_ORPHAN_PRESENT_FLAG 0x00000002 + CP_COMPACT_SUM_FLAG 0x00000004 + CP_ERROR_FLAG 0x00000008 + CP_FSCK_FLAG 0x00000010 + CP_FASTBOOT_FLAG 0x00000020 + CP_CRC_RECOVERY_FLAG 0x00000040 + CP_NAT_BITS_FLAG 0x00000080 + CP_TRIMMED_FLAG 0x00000100 + CP_NOCRC_RECOVERY_FLAG 0x00000200 + CP_LARGE_NAT_BITMAP_FLAG 0x00000400 + CP_QUOTA_NEED_FSCK_FLAG 0x00000800 + CP_DISABLED_FLAG 0x00001000 + CP_DISABLED_QUICK_FLAG 0x00002000 + CP_RESIZEFS_FLAG 0x00004000 + =============================== ============================== + What: /sys/fs/f2fs//ckpt_thread_ioprio Date: January 2021 Contact: "Daeho Jeong" diff --git a/Documentation/ABI/testing/sysfs-kernel-livepatch b/Documentation/ABI/testing/sysfs-kernel-livepatch index bea7bd5a1d5fc01124c090ca457eac089f6a3c57..a5df9b4910dc45fc528735004f49a02b20619e6f 100644 --- a/Documentation/ABI/testing/sysfs-kernel-livepatch +++ b/Documentation/ABI/testing/sysfs-kernel-livepatch @@ -55,6 +55,14 @@ Description: The object directory contains subdirectories for each function that is patched within the object. +What: /sys/kernel/livepatch///patched +Date: August 2022 +KernelVersion: 6.1.0 +Contact: live-patching@vger.kernel.org +Description: + An attribute which indicates whether the object is currently + patched. + What: /sys/kernel/livepatch/// Date: Nov 2014 KernelVersion: 3.19.0 diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers new file mode 100644 index 0000000000000000000000000000000000000000..45985e411f13d69cef048eda2d8665960636c158 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers @@ -0,0 +1,25 @@ +What: /sys/devices/virtual/memory_tiering/ +Date: August 2022 +Contact: Linux memory management mailing list +Description: A collection of all the memory tiers allocated. + + Individual memory tier details are contained in subdirectories + named by the abstract distance of the memory tier. + + /sys/devices/virtual/memory_tiering/memory_tierN/ + + +What: /sys/devices/virtual/memory_tiering/memory_tierN/ + /sys/devices/virtual/memory_tiering/memory_tierN/nodes +Date: August 2022 +Contact: Linux memory management mailing list +Description: Directory with details of a specific memory tier + + This is the directory containing information about a particular + memory tier, memtierN, where N is derived based on abstract distance. + + A smaller value of N implies a higher (faster) memory tier in the + hierarchy. + + nodes: NUMA nodes that are part of this memory tier. + diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi index 04885738cf156987afd64c12b035d1b7736cf772..a77a004a1baa46158e9ac719eb96092101a50d82 100644 --- a/Documentation/ABI/testing/sysfs-platform-asus-wmi +++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi @@ -57,3 +57,44 @@ Description: * 0 - default, * 1 - overboost, * 2 - silent + +What: /sys/devices/platform//gpu_mux_mode +Date: Aug 2022 +KernelVersion: 6.1 +Contact: "Luke Jones" +Description: + Switch the GPU hardware MUX mode. Laptops with this feature can + can be toggled to boot with only the dGPU (discrete mode) or in + standard Optimus/Hybrid mode. On switch a reboot is required: + + * 0 - Discrete GPU, + * 1 - Optimus/Hybrid, + +What: /sys/devices/platform//dgpu_disable +Date: Aug 2022 +KernelVersion: 5.17 +Contact: "Luke Jones" +Description: + Disable discrete GPU: + * 0 - Enable dGPU, + * 1 - Disable dGPU + +What: /sys/devices/platform//egpu_enable +Date: Aug 2022 +KernelVersion: 5.17 +Contact: "Luke Jones" +Description: + Enable the external GPU paired with ROG X-Flow laptops. + Toggling this setting will also trigger ACPI to disable the dGPU: + + * 0 - Disable, + * 1 - Enable + +What: /sys/devices/platform//panel_od +Date: Aug 2022 +KernelVersion: 5.17 +Contact: "Luke Jones" +Description: + Enable an LCD response-time boost to reduce or remove ghosting: + * 0 - Disable, + * 1 - Enable diff --git a/Documentation/ABI/testing/sysfs-platform-brcmstb-memc b/Documentation/ABI/testing/sysfs-platform-brcmstb-memc new file mode 100644 index 0000000000000000000000000000000000000000..2f2b750ac2fd66aca4102311e1dc2885a72e46a9 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-platform-brcmstb-memc @@ -0,0 +1,15 @@ +What: /sys/bus/platform/devices/*/srpd +Date: July 2022 +KernelVersion: 5.21 +Contact: Florian Fainelli +Description: + Self Refresh Power Down (SRPD) inactivity timeout counted in + internal DDR controller clock cycles. Possible values range + from 0 (disable inactivity timeout) to 65535 (0xffff). + +What: /sys/bus/platform/devices/*/frequency +Date: July 2022 +KernelVersion: 5.21 +Contact: Florian Fainelli +Description: + DDR PHY frequency in Hz. diff --git a/Documentation/ABI/testing/sysfs-platform-chipidea-usb2 b/Documentation/ABI/testing/sysfs-platform-chipidea-usb2 index b0f4684a83fe7ab6d72a9ff780565933833c64a0..b9f7d924f28a31c5ffe15a0a2ebbba7d38c41d4f 100644 --- a/Documentation/ABI/testing/sysfs-platform-chipidea-usb2 +++ b/Documentation/ABI/testing/sysfs-platform-chipidea-usb2 @@ -2,8 +2,8 @@ What: /sys/bus/platform/devices/ci_hdrc.0/role Date: Mar 2017 Contact: Peter Chen Description: - It returns string "gadget" or "host" when read it, it indicates - current controller role. + When read, it returns string "gadget" or "host", indicating + the current controller role. - It will do role switch when write "gadget" or "host" to it. + It will do role switch when "gadget" or "host" is written to it. Only controller at dual-role configuration supports writing. diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power index 90ec4987074b2bb4846d0dc17f13166c3105dbef..f99d433ff3117ce16f57b101afe48202759a32c9 100644 --- a/Documentation/ABI/testing/sysfs-power +++ b/Documentation/ABI/testing/sysfs-power @@ -152,7 +152,7 @@ Description: case further investigation is required to determine which device is causing the problem. Note that genuine RTC clock values (such as when pm_trace has not been used), can still - match a device and output it's name here. + match a device and output its name here. What: /sys/power/pm_async Date: January 2009 diff --git a/Documentation/RCU/checklist.rst b/Documentation/RCU/checklist.rst index 42cc5d891bd26e4d3841c04e1bad52d01fcb62b7..048c5bc1f813e473e4477f1f2e19bfe0095d6dbe 100644 --- a/Documentation/RCU/checklist.rst +++ b/Documentation/RCU/checklist.rst @@ -66,8 +66,13 @@ over a rather long period of time, but improvements are always welcome! As a rough rule of thumb, any dereference of an RCU-protected pointer must be covered by rcu_read_lock(), rcu_read_lock_bh(), rcu_read_lock_sched(), or by the appropriate update-side lock. - Disabling of preemption can serve as rcu_read_lock_sched(), but - is less readable and prevents lockdep from detecting locking issues. + Explicit disabling of preemption (preempt_disable(), for example) + can serve as rcu_read_lock_sched(), but is less readable and + prevents lockdep from detecting locking issues. + + Please not that you *cannot* rely on code known to be built + only in non-preemptible kernels. Such code can and will break, + especially in kernels built with CONFIG_PREEMPT_COUNT=y. Letting RCU-protected pointers "leak" out of an RCU read-side critical section is every bit as bad as letting them leak out @@ -185,6 +190,9 @@ over a rather long period of time, but improvements are always welcome! 5. If call_rcu() or call_srcu() is used, the callback function will be called from softirq context. In particular, it cannot block. + If you need the callback to block, run that code in a workqueue + handler scheduled from the callback. The queue_rcu_work() + function does this for you in the case of call_rcu(). 6. Since synchronize_rcu() can block, it cannot be called from any sort of irq context. The same rule applies @@ -297,7 +305,8 @@ over a rather long period of time, but improvements are always welcome! the machine. d. Periodically invoke synchronize_rcu(), permitting a limited - number of updates per grace period. + number of updates per grace period. Better yet, periodically + invoke rcu_barrier() to wait for all outstanding callbacks. The same cautions apply to call_srcu() and kfree_rcu(). @@ -477,6 +486,6 @@ over a rather long period of time, but improvements are always welcome! So if you need to wait for both an RCU grace period and for all pre-existing call_rcu() callbacks, you will need to execute both rcu_barrier() and synchronize_rcu(), if necessary, using - something like workqueues to to execute them concurrently. + something like workqueues to execute them concurrently. See rcubarrier.rst for more information. diff --git a/Documentation/RCU/lockdep.rst b/Documentation/RCU/lockdep.rst index cc860a0c296be1bdb3f57ac79fa10dbe30fd99db..a94f55991a71cbe9a87a227590329a21caf4be3e 100644 --- a/Documentation/RCU/lockdep.rst +++ b/Documentation/RCU/lockdep.rst @@ -61,7 +61,7 @@ checking of rcu_dereference() primitives: rcu_access_pointer(p): Return the value of the pointer and omit all barriers, but retain the compiler constraints that prevent duplicating - or coalescsing. This is useful when when testing the + or coalescsing. This is useful when testing the value of the pointer itself, for example, against NULL. The rcu_dereference_check() check expression can be any boolean diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst index 0b418a5b243c535b5de8949b493c4a2892870258..81e828c8313b8eb2effa026366476a73e29aae5c 100644 --- a/Documentation/RCU/rcu_dereference.rst +++ b/Documentation/RCU/rcu_dereference.rst @@ -128,10 +128,16 @@ Follow these rules to keep your RCU code working properly: This sort of comparison occurs frequently when scanning RCU-protected circular linked lists. - Note that if checks for being within an RCU read-side - critical section are not required and the pointer is never - dereferenced, rcu_access_pointer() should be used in place - of rcu_dereference(). + Note that if the pointer comparison is done outside + of an RCU read-side critical section, and the pointer + is never dereferenced, rcu_access_pointer() should be + used in place of rcu_dereference(). In most cases, + it is best to avoid accidental dereferences by testing + the rcu_access_pointer() return value directly, without + assigning it to a variable. + + Within an RCU read-side critical section, there is little + reason to use rcu_access_pointer(). - The comparison is against a pointer that references memory that was initialized "a long time ago." The reason diff --git a/Documentation/RCU/whatisRCU.rst b/Documentation/RCU/whatisRCU.rst index 77ea260efd1207e797d84bf895c6c7d26063467a..1c747ac3f2c8e0db247382490adefdfb7d88bba3 100644 --- a/Documentation/RCU/whatisRCU.rst +++ b/Documentation/RCU/whatisRCU.rst @@ -6,13 +6,15 @@ What is RCU? -- "Read, Copy, Update" Please note that the "What is RCU?" LWN series is an excellent place to start learning about RCU: -| 1. What is RCU, Fundamentally? http://lwn.net/Articles/262464/ -| 2. What is RCU? Part 2: Usage http://lwn.net/Articles/263130/ -| 3. RCU part 3: the RCU API http://lwn.net/Articles/264090/ -| 4. The RCU API, 2010 Edition http://lwn.net/Articles/418853/ -| 2010 Big API Table http://lwn.net/Articles/419086/ -| 5. The RCU API, 2014 Edition http://lwn.net/Articles/609904/ -| 2014 Big API Table http://lwn.net/Articles/609973/ +| 1. What is RCU, Fundamentally? https://lwn.net/Articles/262464/ +| 2. What is RCU? Part 2: Usage https://lwn.net/Articles/263130/ +| 3. RCU part 3: the RCU API https://lwn.net/Articles/264090/ +| 4. The RCU API, 2010 Edition https://lwn.net/Articles/418853/ +| 2010 Big API Table https://lwn.net/Articles/419086/ +| 5. The RCU API, 2014 Edition https://lwn.net/Articles/609904/ +| 2014 Big API Table https://lwn.net/Articles/609973/ +| 6. The RCU API, 2019 Edition https://lwn.net/Articles/777036/ +| 2019 Big API Table https://lwn.net/Articles/777165/ What is RCU? @@ -915,13 +917,18 @@ which an RCU reference is held include: The understanding that RCU provides a reference that only prevents a change of type is particularly visible with objects allocated from a slab cache marked ``SLAB_TYPESAFE_BY_RCU``. RCU operations may yield a -reference to an object from such a cache that has been concurrently -freed and the memory reallocated to a completely different object, -though of the same type. In this case RCU doesn't even protect the -identity of the object from changing, only its type. So the object -found may not be the one expected, but it will be one where it is safe -to take a reference or spinlock and then confirm that the identity -matches the expectations. +reference to an object from such a cache that has been concurrently freed +and the memory reallocated to a completely different object, though of +the same type. In this case RCU doesn't even protect the identity of the +object from changing, only its type. So the object found may not be the +one expected, but it will be one where it is safe to take a reference +(and then potentially acquiring a spinlock), allowing subsequent code +to check whether the identity matches expectations. It is tempting +to simply acquire the spinlock without first taking the reference, but +unfortunately any spinlock in a ``SLAB_TYPESAFE_BY_RCU`` object must be +initialized after each and every call to kmem_cache_alloc(), which renders +reference-free spinlock acquisition completely unsafe. Therefore, when +using ``SLAB_TYPESAFE_BY_RCU``, make proper use of a reference counter. With traditional reference counting -- such as that implemented by the kref library in Linux -- there is typically code that runs when the last @@ -1057,14 +1064,20 @@ SRCU: Initialization/cleanup:: init_srcu_struct cleanup_srcu_struct -All: lockdep-checked RCU-protected pointer access:: +All: lockdep-checked RCU utility APIs:: - rcu_access_pointer - rcu_dereference_raw RCU_LOCKDEP_WARN rcu_sleep_check RCU_NONIDLE +All: Unchecked RCU-protected pointer access:: + + rcu_dereference_raw + +All: Unchecked RCU-protected pointer access with dereferencing prohibited:: + + rcu_access_pointer + See the comment headers in the source code (or the docbook generated from them) for more information. diff --git a/Documentation/accounting/delay-accounting.rst b/Documentation/accounting/delay-accounting.rst index 241d1a87f2cdb488f1cac46553f437f134fce271..7103b62ba6d7eaa1359abcdef3c2e15ee5356d86 100644 --- a/Documentation/accounting/delay-accounting.rst +++ b/Documentation/accounting/delay-accounting.rst @@ -13,7 +13,7 @@ a) waiting for a CPU (while being runnable) b) completion of synchronous block I/O initiated by the task c) swapping in pages d) memory reclaim -e) thrashing page cache +e) thrashing f) direct compact g) write-protect copy diff --git a/Documentation/admin-guide/README.rst b/Documentation/admin-guide/README.rst index caa3c09a5c3f91f73fd386370eb415c5e71ee9eb..9a969c0157f1e52b603941a4dfbcf8f0cd0e3f4d 100644 --- a/Documentation/admin-guide/README.rst +++ b/Documentation/admin-guide/README.rst @@ -1,9 +1,9 @@ .. _readme: -Linux kernel release 5.x +Linux kernel release 6.x ============================================= -These are the release notes for Linux version 5. Read them carefully, +These are the release notes for Linux version 6. Read them carefully, as they tell you what this is all about, explain how to install the kernel, and what to do if something goes wrong. @@ -63,7 +63,7 @@ Installing the kernel source directory where you have permissions (e.g. your home directory) and unpack it:: - xz -cd linux-5.x.tar.xz | tar xvf - + xz -cd linux-6.x.tar.xz | tar xvf - Replace "X" with the version number of the latest kernel. @@ -72,12 +72,12 @@ Installing the kernel source files. They should match the library, and not get messed up by whatever the kernel-du-jour happens to be. - - You can also upgrade between 5.x releases by patching. Patches are + - You can also upgrade between 6.x releases by patching. Patches are distributed in the xz format. To install by patching, get all the newer patch files, enter the top level directory of the kernel source - (linux-5.x) and execute:: + (linux-6.x) and execute:: - xz -cd ../patch-5.x.xz | patch -p1 + xz -cd ../patch-6.x.xz | patch -p1 Replace "x" for all versions bigger than the version "x" of your current source tree, **in_order**, and you should be ok. You may want to remove @@ -85,13 +85,13 @@ Installing the kernel source that there are no failed patches (some-file-name# or some-file-name.rej). If there are, either you or I have made a mistake. - Unlike patches for the 5.x kernels, patches for the 5.x.y kernels + Unlike patches for the 6.x kernels, patches for the 6.x.y kernels (also known as the -stable kernels) are not incremental but instead apply - directly to the base 5.x kernel. For example, if your base kernel is 5.0 - and you want to apply the 5.0.3 patch, you must not first apply the 5.0.1 - and 5.0.2 patches. Similarly, if you are running kernel version 5.0.2 and - want to jump to 5.0.3, you must first reverse the 5.0.2 patch (that is, - patch -R) **before** applying the 5.0.3 patch. You can read more on this in + directly to the base 6.x kernel. For example, if your base kernel is 6.0 + and you want to apply the 6.0.3 patch, you must not first apply the 6.0.1 + and 6.0.2 patches. Similarly, if you are running kernel version 6.0.2 and + want to jump to 6.0.3, you must first reverse the 6.0.2 patch (that is, + patch -R) **before** applying the 6.0.3 patch. You can read more on this in :ref:`Documentation/process/applying-patches.rst `. Alternatively, the script patch-kernel can be used to automate this @@ -114,7 +114,7 @@ Installing the kernel source Software requirements --------------------- - Compiling and running the 5.x kernels requires up-to-date + Compiling and running the 6.x kernels requires up-to-date versions of various software packages. Consult :ref:`Documentation/process/changes.rst ` for the minimum version numbers required and how to get updates for these packages. Beware that using @@ -132,12 +132,12 @@ Build directory for the kernel place for the output files (including .config). Example:: - kernel source code: /usr/src/linux-5.x + kernel source code: /usr/src/linux-6.x build directory: /home/name/build/kernel To configure and build the kernel, use:: - cd /usr/src/linux-5.x + cd /usr/src/linux-6.x make O=/home/name/build/kernel menuconfig make O=/home/name/build/kernel sudo make O=/home/name/build/kernel modules_install install @@ -262,8 +262,6 @@ Compiling the kernel - Make sure you have at least gcc 5.1 available. For more information, refer to :ref:`Documentation/process/changes.rst `. - Please note that you can still run a.out user programs with this kernel. - - Do a ``make`` to create a compressed kernel image. It is also possible to do ``make install`` if you have lilo installed to suit the kernel makefiles, but you may want to check your particular lilo setup first. @@ -332,85 +330,10 @@ Compiling the kernel If something goes wrong ----------------------- - - If you have problems that seem to be due to kernel bugs, please check - the file MAINTAINERS to see if there is a particular person associated - with the part of the kernel that you are having trouble with. If there - isn't anyone listed there, then the second best thing is to mail - them to me (torvalds@linux-foundation.org), and possibly to any other - relevant mailing-list or to the newsgroup. - - - In all bug-reports, *please* tell what kernel you are talking about, - how to duplicate the problem, and what your setup is (use your common - sense). If the problem is new, tell me so, and if the problem is - old, please try to tell me when you first noticed it. - - - If the bug results in a message like:: - - unable to handle kernel paging request at address C0000010 - Oops: 0002 - EIP: 0010:XXXXXXXX - eax: xxxxxxxx ebx: xxxxxxxx ecx: xxxxxxxx edx: xxxxxxxx - esi: xxxxxxxx edi: xxxxxxxx ebp: xxxxxxxx - ds: xxxx es: xxxx fs: xxxx gs: xxxx - Pid: xx, process nr: xx - xx xx xx xx xx xx xx xx xx xx - - or similar kernel debugging information on your screen or in your - system log, please duplicate it *exactly*. The dump may look - incomprehensible to you, but it does contain information that may - help debugging the problem. The text above the dump is also - important: it tells something about why the kernel dumped code (in - the above example, it's due to a bad kernel pointer). More information - on making sense of the dump is in Documentation/admin-guide/bug-hunting.rst - - - If you compiled the kernel with CONFIG_KALLSYMS you can send the dump - as is, otherwise you will have to use the ``ksymoops`` program to make - sense of the dump (but compiling with CONFIG_KALLSYMS is usually preferred). - This utility can be downloaded from - https://www.kernel.org/pub/linux/utils/kernel/ksymoops/ . - Alternatively, you can do the dump lookup by hand: - - - In debugging dumps like the above, it helps enormously if you can - look up what the EIP value means. The hex value as such doesn't help - me or anybody else very much: it will depend on your particular - kernel setup. What you should do is take the hex value from the EIP - line (ignore the ``0010:``), and look it up in the kernel namelist to - see which kernel function contains the offending address. - - To find out the kernel function name, you'll need to find the system - binary associated with the kernel that exhibited the symptom. This is - the file 'linux/vmlinux'. To extract the namelist and match it against - the EIP from the kernel crash, do:: - - nm vmlinux | sort | less - - This will give you a list of kernel addresses sorted in ascending - order, from which it is simple to find the function that contains the - offending address. Note that the address given by the kernel - debugging messages will not necessarily match exactly with the - function addresses (in fact, that is very unlikely), so you can't - just 'grep' the list: the list will, however, give you the starting - point of each kernel function, so by looking for the function that - has a starting address lower than the one you are searching for but - is followed by a function with a higher address you will find the one - you want. In fact, it may be a good idea to include a bit of - "context" in your problem report, giving a few lines around the - interesting one. - - If you for some reason cannot do the above (you have a pre-compiled - kernel image or similar), telling me as much about your setup as - possible will help. Please read - 'Documentation/admin-guide/reporting-issues.rst' for details. - - - Alternatively, you can use gdb on a running kernel. (read-only; i.e. you - cannot change values or set break points.) To do this, first compile the - kernel with -g; edit arch/x86/Makefile appropriately, then do a ``make - clean``. You'll also need to enable CONFIG_PROC_FS (via ``make config``). - - After you've rebooted with the new kernel, do ``gdb vmlinux /proc/kcore``. - You can now use all the usual gdb commands. The command to look up the - point where your system crashed is ``l *0xXXXXXXXX``. (Replace the XXXes - with the EIP value.) - - gdb'ing a non-running kernel currently fails because ``gdb`` (wrongly) - disregards the starting offset for which the kernel is compiled. +If you have problems that seem to be due to kernel bugs, please follow the +instructions at 'Documentation/admin-guide/reporting-issues.rst'. + +Hints on understanding kernel bug reports are in +'Documentation/admin-guide/bug-hunting.rst'. More on debugging the kernel +with gdb is in 'Documentation/dev-tools/gdb-kernel-debugging.rst' and +'Documentation/dev-tools/kgdb.rst'. diff --git a/Documentation/admin-guide/acpi/dsdt-override.rst b/Documentation/admin-guide/acpi/dsdt-override.rst deleted file mode 100644 index 50bd7f194bf440e40ff86f4b98670eff1c715c3c..0000000000000000000000000000000000000000 --- a/Documentation/admin-guide/acpi/dsdt-override.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -=============== -Overriding DSDT -=============== - -Linux supports a method of overriding the BIOS DSDT: - -CONFIG_ACPI_CUSTOM_DSDT - builds the image into the kernel. - -When to use this method is described in detail on the -Linux/ACPI home page: -https://01.org/linux-acpi/documentation/overriding-dsdt diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 2cc502a75ef64078c30794af1223634bf6a86ce3..5b86245450bdc4e695770ab728949617e87d5ec6 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -299,7 +299,7 @@ Per-node-per-memcgroup LRU (cgroup's private LRU) is guarded by lruvec->lru_lock; PG_lru bit of page->flags is cleared before isolating a page from its LRU under lruvec->lru_lock. -2.7 Kernel Memory Extension (CONFIG_MEMCG_KMEM) +2.7 Kernel Memory Extension ----------------------------------------------- With the Kernel memory extension, the Memory Controller is able to limit @@ -386,8 +386,6 @@ U != 0, K >= U: a. Enable CONFIG_CGROUPS b. Enable CONFIG_MEMCG -c. Enable CONFIG_MEMCG_SWAP (to use swap extension) -d. Enable CONFIG_MEMCG_KMEM (to use kmem extension) 3.1. Prepare the cgroups (see cgroups.txt, Why are cgroups needed?) ------------------------------------------------------------------- diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index be4a77baf78414c8e0cfeabd6ed8dac78e58d059..dc254a3cb95686e67a7335bad3101313e97eedd9 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -976,6 +976,29 @@ All cgroup core files are prefixed with "cgroup." killing cgroups is a process directed operation, i.e. it affects the whole thread-group. + cgroup.pressure + A read-write single value file that allowed values are "0" and "1". + The default is "1". + + Writing "0" to the file will disable the cgroup PSI accounting. + Writing "1" to the file will re-enable the cgroup PSI accounting. + + This control attribute is not hierarchical, so disable or enable PSI + accounting in a cgroup does not affect PSI accounting in descendants + and doesn't need pass enablement via ancestors from root. + + The reason this control attribute exists is that PSI accounts stalls for + each cgroup separately and aggregates it at each level of the hierarchy. + This may cause non-negligible overhead for some workloads when under + deep level of the hierarchy, in which case this control attribute can + be used to disable PSI accounting in the non-leaf cgroups. + + irq.pressure + A read-write nested-keyed file. + + Shows pressure stall information for IRQ/SOFTIRQ. See + :ref:`Documentation/accounting/psi.rst ` for details. + Controllers =========== @@ -1355,6 +1378,11 @@ PAGE_SIZE multiple when read back. pagetables Amount of memory allocated for page tables. + sec_pagetables + Amount of memory allocated for secondary page tables, + this currently includes KVM mmu allocations on x86 + and arm64. + percpu (npn) Amount of memory used for storing per-cpu kernel data structures. @@ -2185,75 +2213,93 @@ Cpuset Interface Files It accepts only the following input values when written to. - ======== ================================ - "root" a partition root - "member" a non-root member of a partition - ======== ================================ - - When set to be a partition root, the current cgroup is the - root of a new partition or scheduling domain that comprises - itself and all its descendants except those that are separate - partition roots themselves and their descendants. The root - cgroup is always a partition root. - - There are constraints on where a partition root can be set. - It can only be set in a cgroup if all the following conditions - are true. - - 1) The "cpuset.cpus" is not empty and the list of CPUs are - exclusive, i.e. they are not shared by any of its siblings. - 2) The parent cgroup is a partition root. - 3) The "cpuset.cpus" is also a proper subset of the parent's - "cpuset.cpus.effective". - 4) There is no child cgroups with cpuset enabled. This is for - eliminating corner cases that have to be handled if such a - condition is allowed. - - Setting it to partition root will take the CPUs away from the - effective CPUs of the parent cgroup. Once it is set, this - file cannot be reverted back to "member" if there are any child - cgroups with cpuset enabled. - - A parent partition cannot distribute all its CPUs to its - child partitions. There must be at least one cpu left in the - parent partition. - - Once becoming a partition root, changes to "cpuset.cpus" is - generally allowed as long as the first condition above is true, - the change will not take away all the CPUs from the parent - partition and the new "cpuset.cpus" value is a superset of its - children's "cpuset.cpus" values. - - Sometimes, external factors like changes to ancestors' - "cpuset.cpus" or cpu hotplug can cause the state of the partition - root to change. On read, the "cpuset.sched.partition" file - can show the following values. - - ============== ============================== - "member" Non-root member of a partition - "root" Partition root - "root invalid" Invalid partition root - ============== ============================== - - It is a partition root if the first 2 partition root conditions - above are true and at least one CPU from "cpuset.cpus" is - granted by the parent cgroup. - - A partition root can become invalid if none of CPUs requested - in "cpuset.cpus" can be granted by the parent cgroup or the - parent cgroup is no longer a partition root itself. In this - case, it is not a real partition even though the restriction - of the first partition root condition above will still apply. - The cpu affinity of all the tasks in the cgroup will then be - associated with CPUs in the nearest ancestor partition. - - An invalid partition root can be transitioned back to a - real partition root if at least one of the requested CPUs - can now be granted by its parent. In this case, the cpu - affinity of all the tasks in the formerly invalid partition - will be associated to the CPUs of the newly formed partition. - Changing the partition state of an invalid partition root to - "member" is always allowed even if child cpusets are present. + ========== ===================================== + "member" Non-root member of a partition + "root" Partition root + "isolated" Partition root without load balancing + ========== ===================================== + + The root cgroup is always a partition root and its state + cannot be changed. All other non-root cgroups start out as + "member". + + When set to "root", the current cgroup is the root of a new + partition or scheduling domain that comprises itself and all + its descendants except those that are separate partition roots + themselves and their descendants. + + When set to "isolated", the CPUs in that partition root will + be in an isolated state without any load balancing from the + scheduler. Tasks placed in such a partition with multiple + CPUs should be carefully distributed and bound to each of the + individual CPUs for optimal performance. + + The value shown in "cpuset.cpus.effective" of a partition root + is the CPUs that the partition root can dedicate to a potential + new child partition root. The new child subtracts available + CPUs from its parent "cpuset.cpus.effective". + + A partition root ("root" or "isolated") can be in one of the + two possible states - valid or invalid. An invalid partition + root is in a degraded state where some state information may + be retained, but behaves more like a "member". + + All possible state transitions among "member", "root" and + "isolated" are allowed. + + On read, the "cpuset.cpus.partition" file can show the following + values. + + ============================= ===================================== + "member" Non-root member of a partition + "root" Partition root + "isolated" Partition root without load balancing + "root invalid ()" Invalid partition root + "isolated invalid ()" Invalid isolated partition root + ============================= ===================================== + + In the case of an invalid partition root, a descriptive string on + why the partition is invalid is included within parentheses. + + For a partition root to become valid, the following conditions + must be met. + + 1) The "cpuset.cpus" is exclusive with its siblings , i.e. they + are not shared by any of its siblings (exclusivity rule). + 2) The parent cgroup is a valid partition root. + 3) The "cpuset.cpus" is not empty and must contain at least + one of the CPUs from parent's "cpuset.cpus", i.e. they overlap. + 4) The "cpuset.cpus.effective" cannot be empty unless there is + no task associated with this partition. + + External events like hotplug or changes to "cpuset.cpus" can + cause a valid partition root to become invalid and vice versa. + Note that a task cannot be moved to a cgroup with empty + "cpuset.cpus.effective". + + For a valid partition root with the sibling cpu exclusivity + rule enabled, changes made to "cpuset.cpus" that violate the + exclusivity rule will invalidate the partition as well as its + sibiling partitions with conflicting cpuset.cpus values. So + care must be taking in changing "cpuset.cpus". + + A valid non-root parent partition may distribute out all its CPUs + to its child partitions when there is no task associated with it. + + Care must be taken to change a valid partition root to + "member" as all its child partitions, if present, will become + invalid causing disruption to tasks running in those child + partitions. These inactivated partitions could be recovered if + their parent is switched back to a partition root with a proper + set of "cpuset.cpus". + + Poll and inotify events are triggered whenever the state of + "cpuset.cpus.partition" changes. That includes changes caused + by write to "cpuset.cpus.partition", cpu hotplug or other + changes that modify the validity status of the partition. + This will allow user space agents to monitor unexpected changes + to "cpuset.cpus.partition" without the need to do continuous + polling. Device controller diff --git a/Documentation/admin-guide/dynamic-debug-howto.rst b/Documentation/admin-guide/dynamic-debug-howto.rst index a89cfa0831551ff40151ec6b8fc6d253e69942d2..faa22f77847a42b8cfbba3f7030ca0cca95e9ebd 100644 --- a/Documentation/admin-guide/dynamic-debug-howto.rst +++ b/Documentation/admin-guide/dynamic-debug-howto.rst @@ -5,143 +5,115 @@ Dynamic debug Introduction ============ -This document describes how to use the dynamic debug (dyndbg) feature. +Dynamic debug allows you to dynamically enable/disable kernel +debug-print code to obtain additional kernel information. -Dynamic debug is designed to allow you to dynamically enable/disable -kernel code to obtain additional kernel information. Currently, if -``CONFIG_DYNAMIC_DEBUG`` is set, then all ``pr_debug()``/``dev_dbg()`` and -``print_hex_dump_debug()``/``print_hex_dump_bytes()`` calls can be dynamically -enabled per-callsite. +If ``/proc/dynamic_debug/control`` exists, your kernel has dynamic +debug. You'll need root access (sudo su) to use this. -If you do not want to enable dynamic debug globally (i.e. in some embedded -system), you may set ``CONFIG_DYNAMIC_DEBUG_CORE`` as basic support of dynamic -debug and add ``ccflags := -DDYNAMIC_DEBUG_MODULE`` into the Makefile of any -modules which you'd like to dynamically debug later. - -If ``CONFIG_DYNAMIC_DEBUG`` is not set, ``print_hex_dump_debug()`` is just -shortcut for ``print_hex_dump(KERN_DEBUG)``. - -For ``print_hex_dump_debug()``/``print_hex_dump_bytes()``, format string is -its ``prefix_str`` argument, if it is constant string; or ``hexdump`` -in case ``prefix_str`` is built dynamically. +Dynamic debug provides: -Dynamic debug has even more useful features: + * a Catalog of all *prdbgs* in your kernel. + ``cat /proc/dynamic_debug/control`` to see them. - * Simple query language allows turning on and off debugging - statements by matching any combination of 0 or 1 of: + * a Simple query/command language to alter *prdbgs* by selecting on + any combination of 0 or 1 of: - source filename - function name - line number (including ranges of line numbers) - module name - format string - - * Provides a debugfs control file: ``/dynamic_debug/control`` - which can be read to display the complete list of known debug - statements, to help guide you - -Controlling dynamic debug Behaviour -=================================== - -The behaviour of ``pr_debug()``/``dev_dbg()`` are controlled via writing to a -control file in the 'debugfs' filesystem. Thus, you must first mount -the debugfs filesystem, in order to make use of this feature. -Subsequently, we refer to the control file as: -``/dynamic_debug/control``. For example, if you want to enable -printing from source file ``svcsock.c``, line 1603 you simply do:: - - nullarbor:~ # echo 'file svcsock.c line 1603 +p' > - /dynamic_debug/control - -If you make a mistake with the syntax, the write will fail thus:: - - nullarbor:~ # echo 'file svcsock.c wtf 1 +p' > - /dynamic_debug/control - -bash: echo: write error: Invalid argument - -Note, for systems without 'debugfs' enabled, the control file can be -found in ``/proc/dynamic_debug/control``. + - class name (as known/declared by each module) Viewing Dynamic Debug Behaviour =============================== -You can view the currently configured behaviour of all the debug -statements via:: +You can view the currently configured behaviour in the *prdbg* catalog:: - nullarbor:~ # cat /dynamic_debug/control + :#> head -n7 /proc/dynamic_debug/control # filename:lineno [module]function flags format - net/sunrpc/svc_rdma.c:323 [svcxprt_rdma]svc_rdma_cleanup =_ "SVCRDMA Module Removed, deregister RPC RDMA transport\012" - net/sunrpc/svc_rdma.c:341 [svcxprt_rdma]svc_rdma_init =_ "\011max_inline : %d\012" - net/sunrpc/svc_rdma.c:340 [svcxprt_rdma]svc_rdma_init =_ "\011sq_depth : %d\012" - net/sunrpc/svc_rdma.c:338 [svcxprt_rdma]svc_rdma_init =_ "\011max_requests : %d\012" - ... + init/main.c:1179 [main]initcall_blacklist =_ "blacklisting initcall %s\012 + init/main.c:1218 [main]initcall_blacklisted =_ "initcall %s blacklisted\012" + init/main.c:1424 [main]run_init_process =_ " with arguments:\012" + init/main.c:1426 [main]run_init_process =_ " %s\012" + init/main.c:1427 [main]run_init_process =_ " with environment:\012" + init/main.c:1429 [main]run_init_process =_ " %s\012" +The 3rd space-delimited column shows the current flags, preceded by +a ``=`` for easy use with grep/cut. ``=p`` shows enabled callsites. -You can also apply standard Unix text manipulation filters to this -data, e.g.:: +Controlling dynamic debug Behaviour +=================================== - nullarbor:~ # grep -i rdma /dynamic_debug/control | wc -l - 62 +The behaviour of *prdbg* sites are controlled by writing +query/commands to the control file. Example:: - nullarbor:~ # grep -i tcp /dynamic_debug/control | wc -l - 42 + # grease the interface + :#> alias ddcmd='echo $* > /proc/dynamic_debug/control' -The third column shows the currently enabled flags for each debug -statement callsite (see below for definitions of the flags). The -default value, with no flags enabled, is ``=_``. So you can view all -the debug statement callsites with any non-default flags:: + :#> ddcmd '-p; module main func run* +p' + :#> grep =p /proc/dynamic_debug/control + init/main.c:1424 [main]run_init_process =p " with arguments:\012" + init/main.c:1426 [main]run_init_process =p " %s\012" + init/main.c:1427 [main]run_init_process =p " with environment:\012" + init/main.c:1429 [main]run_init_process =p " %s\012" - nullarbor:~ # awk '$3 != "=_"' /dynamic_debug/control - # filename:lineno [module]function flags format - net/sunrpc/svcsock.c:1603 [sunrpc]svc_send p "svc_process: st_sendto returned %d\012" +Error messages go to console/syslog:: + + :#> ddcmd mode foo +p + dyndbg: unknown keyword "mode" + dyndbg: query parse failed + bash: echo: write error: Invalid argument + +If debugfs is also enabled and mounted, ``dynamic_debug/control`` is +also under the mount-dir, typically ``/sys/kernel/debug/``. Command Language Reference ========================== -At the lexical level, a command comprises a sequence of words separated +At the basic lexical level, a command is a sequence of words separated by spaces or tabs. So these are all equivalent:: - nullarbor:~ # echo -n 'file svcsock.c line 1603 +p' > - /dynamic_debug/control - nullarbor:~ # echo -n ' file svcsock.c line 1603 +p ' > - /dynamic_debug/control - nullarbor:~ # echo -n 'file svcsock.c line 1603 +p' > - /dynamic_debug/control + :#> ddcmd file svcsock.c line 1603 +p + :#> ddcmd "file svcsock.c line 1603 +p" + :#> ddcmd ' file svcsock.c line 1603 +p ' Command submissions are bounded by a write() system call. Multiple commands can be written together, separated by ``;`` or ``\n``:: - ~# echo "func pnpacpi_get_resources +p; func pnp_assign_mem +p" \ - > /dynamic_debug/control + :#> ddcmd "func pnpacpi_get_resources +p; func pnp_assign_mem +p" + :#> ddcmd <<"EOC" + func pnpacpi_get_resources +p + func pnp_assign_mem +p + EOC + :#> cat query-batch-file > /proc/dynamic_debug/control -If your query set is big, you can batch them too:: +You can also use wildcards in each query term. The match rule supports +``*`` (matches zero or more characters) and ``?`` (matches exactly one +character). For example, you can match all usb drivers:: - ~# cat query-batch-file > /dynamic_debug/control + :#> ddcmd file "drivers/usb/*" +p # "" to suppress shell expansion -Another way is to use wildcards. The match rule supports ``*`` (matches -zero or more characters) and ``?`` (matches exactly one character). For -example, you can match all usb drivers:: - - ~# echo "file drivers/usb/* +p" > /dynamic_debug/control - -At the syntactical level, a command comprises a sequence of match -specifications, followed by a flags change specification:: +Syntactically, a command is pairs of keyword values, followed by a +flags change or setting:: command ::= match-spec* flags-spec -The match-spec's are used to choose a subset of the known pr_debug() -callsites to which to apply the flags-spec. Think of them as a query -with implicit ANDs between each pair. Note that an empty list of -match-specs will select all debug statement callsites. +The match-spec's select *prdbgs* from the catalog, upon which to apply +the flags-spec, all constraints are ANDed together. An absent keyword +is the same as keyword "*". + -A match specification comprises a keyword, which controls the -attribute of the callsite to be compared, and a value to compare -against. Possible keywords are::: +A match specification is a keyword, which selects the attribute of +the callsite to be compared, and a value to compare against. Possible +keywords are::: match-spec ::= 'func' string | 'file' string | 'module' string | 'format' string | + 'class' string | 'line' line-range line-range ::= lineno | @@ -203,6 +175,16 @@ format format "nfsd: SETATTR" // a neater way to match a format with whitespace format 'nfsd: SETATTR' // yet another way to match a format with whitespace +class + The given class_name is validated against each module, which may + have declared a list of known class_names. If the class_name is + found for a module, callsite & class matching and adjustment + proceeds. Examples:: + + class DRM_UT_KMS # a DRM.debug category + class JUNK # silent non-match + // class TLD_* # NOTICE: no wildcard in class names + line The given line number or range of line numbers is compared against the line number of each ``pr_debug()`` callsite. A single @@ -228,17 +210,16 @@ of the characters:: The flags are:: p enables the pr_debug() callsite. - f Include the function name in the printed message - l Include line number in the printed message - m Include module name in the printed message - t Include thread ID in messages not generated from interrupt context - _ No flags are set. (Or'd with others on input) + _ enables no flags. -For ``print_hex_dump_debug()`` and ``print_hex_dump_bytes()``, only ``p`` flag -have meaning, other flags ignored. + Decorator flags add to the message-prefix, in order: + t Include thread ID, or + m Include module name + f Include the function name + l Include line number -For display, the flags are preceded by ``=`` -(mnemonic: what the flags are currently equal to). +For ``print_hex_dump_debug()`` and ``print_hex_dump_bytes()``, only +the ``p`` flag has meaning, other flags are ignored. Note the regexp ``^[-+=][flmpt_]+$`` matches a flags specification. To clear all flags at once, use ``=_`` or ``-flmpt``. @@ -313,7 +294,7 @@ For ``CONFIG_DYNAMIC_DEBUG`` kernels, any settings given at boot-time (or enabled by ``-DDEBUG`` flag during compilation) can be disabled later via the debugfs interface if the debug messages are no longer needed:: - echo "module module_name -p" > /dynamic_debug/control + echo "module module_name -p" > /proc/dynamic_debug/control Examples ======== @@ -321,37 +302,31 @@ Examples :: // enable the message at line 1603 of file svcsock.c - nullarbor:~ # echo -n 'file svcsock.c line 1603 +p' > - /dynamic_debug/control + :#> ddcmd 'file svcsock.c line 1603 +p' // enable all the messages in file svcsock.c - nullarbor:~ # echo -n 'file svcsock.c +p' > - /dynamic_debug/control + :#> ddcmd 'file svcsock.c +p' // enable all the messages in the NFS server module - nullarbor:~ # echo -n 'module nfsd +p' > - /dynamic_debug/control + :#> ddcmd 'module nfsd +p' // enable all 12 messages in the function svc_process() - nullarbor:~ # echo -n 'func svc_process +p' > - /dynamic_debug/control + :#> ddcmd 'func svc_process +p' // disable all 12 messages in the function svc_process() - nullarbor:~ # echo -n 'func svc_process -p' > - /dynamic_debug/control + :#> ddcmd 'func svc_process -p' // enable messages for NFS calls READ, READLINK, READDIR and READDIR+. - nullarbor:~ # echo -n 'format "nfsd: READ" +p' > - /dynamic_debug/control + :#> ddcmd 'format "nfsd: READ" +p' // enable messages in files of which the paths include string "usb" - nullarbor:~ # echo -n 'file *usb* +p' > /dynamic_debug/control + :#> ddcmd 'file *usb* +p' > /proc/dynamic_debug/control // enable all messages - nullarbor:~ # echo -n '+p' > /dynamic_debug/control + :#> ddcmd '+p' > /proc/dynamic_debug/control // add module, function to all enabled messages - nullarbor:~ # echo -n '+mf' > /dynamic_debug/control + :#> ddcmd '+mf' > /proc/dynamic_debug/control // boot-args example, with newlines and comments for readability Kernel command line: ... @@ -364,3 +339,38 @@ Examples dyndbg="file init/* +p #cmt ; func parse_one +p" // enable pr_debugs in 2 functions in a module loaded later pc87360.dyndbg="func pc87360_init_device +p; func pc87360_find +p" + +Kernel Configuration +==================== + +Dynamic Debug is enabled via kernel config items:: + + CONFIG_DYNAMIC_DEBUG=y # build catalog, enables CORE + CONFIG_DYNAMIC_DEBUG_CORE=y # enable mechanics only, skip catalog + +If you do not want to enable dynamic debug globally (i.e. in some embedded +system), you may set ``CONFIG_DYNAMIC_DEBUG_CORE`` as basic support of dynamic +debug and add ``ccflags := -DDYNAMIC_DEBUG_MODULE`` into the Makefile of any +modules which you'd like to dynamically debug later. + + +Kernel *prdbg* API +================== + +The following functions are cataloged and controllable when dynamic +debug is enabled:: + + pr_debug() + dev_dbg() + print_hex_dump_debug() + print_hex_dump_bytes() + +Otherwise, they are off by default; ``ccflags += -DDEBUG`` or +``#define DEBUG`` in a source file will enable them appropriately. + +If ``CONFIG_DYNAMIC_DEBUG`` is not set, ``print_hex_dump_debug()`` is +just a shortcut for ``print_hex_dump(KERN_DEBUG)``. + +For ``print_hex_dump_debug()``/``print_hex_dump_bytes()``, format string is +its ``prefix_str`` argument, if it is constant string; or ``hexdump`` +in case ``prefix_str`` is built dynamically. diff --git a/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst index 9393c50b5afc9c9fe8b9ac90ed9fe4774e1d1550..c98fd11907cc8754878ea89f371f5e753d9e0269 100644 --- a/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst +++ b/Documentation/admin-guide/hw-vuln/processor_mmio_stale_data.rst @@ -230,6 +230,20 @@ The possible values in this file are: * - 'Mitigation: Clear CPU buffers' - The processor is vulnerable and the CPU buffer clearing mitigation is enabled. + * - 'Unknown: No mitigations' + - The processor vulnerability status is unknown because it is + out of Servicing period. Mitigation is not attempted. + +Definitions: +------------ + +Servicing period: The process of providing functional and security updates to +Intel processors or platforms, utilizing the Intel Platform Update (IPU) +process or other similar mechanisms. + +End of Servicing Updates (ESU): ESU is the date at which Intel will no +longer provide Servicing, such as through IPU or other similar update +processes. ESU dates will typically be aligned to end of quarter. If the processor is vulnerable then the following information is appended to the above information: diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst index 2ce2a38cdd556c7c52b37c5b1e79092a551653d1..c4dcdb3d0d45109f79ed08287394df35e01afce7 100644 --- a/Documentation/admin-guide/hw-vuln/spectre.rst +++ b/Documentation/admin-guide/hw-vuln/spectre.rst @@ -613,6 +613,7 @@ kernel command line. eibrs enhanced IBRS eibrs,retpoline enhanced IBRS + Retpolines eibrs,lfence enhanced IBRS + LFENCE + ibrs use IBRS to protect kernel Not specifying this option is equivalent to spectre_v2=auto. diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst index 8419019b6a88f2136904bfb820e6fa0038d97556..6726f439958ca0bad0d0fe26a3fe8034ccbdbaef 100644 --- a/Documentation/admin-guide/kdump/vmcoreinfo.rst +++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst @@ -200,7 +200,7 @@ prb A pointer to the printk ringbuffer (struct printk_ringbuffer). This may be pointing to the static boot ringbuffer or the dynamically -allocated ringbuffer, depending on when the the core dump occurred. +allocated ringbuffer, depending on when the core dump occurred. Used by user-space tools to read the active kernel log buffer. printk_rb_static diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index d7f30902fda02fe09ea9eaa6563065b1655b0c2c..a465d5242774af8f89779121b5acca4439b7f5f0 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -321,6 +321,8 @@ force_enable - Force enable the IOMMU on platforms known to be buggy with IOMMU enabled. Use this option with care. + pgtbl_v1 - Use v1 page table for DMA-API (Default). + pgtbl_v2 - Use v2 page table for DMA-API. amd_iommu_dump= [HW,X86-64] Enable AMD IOMMU driver option to dump the ACPI table @@ -966,10 +968,6 @@ debugpat [X86] Enable PAT debugging - decnet.addr= [HW,NET] - Format: [,] - See also Documentation/networking/decnet.rst. - default_hugepagesz= [HW] The size of the default HugeTLB page. This is the size represented by the legacy /proc/ hugepages @@ -1471,6 +1469,14 @@ Permit 'security.evm' to be updated regardless of current integrity status. + early_page_ext [KNL] Enforces page_ext initialization to earlier + stages so cover more early boot allocations. + Please note that as side effect some optimizations + might be disabled to achieve that (e.g. parallelized + memory initialization is disabled) so the boot process + might take longer, especially on systems with a lot of + memory. Available with CONFIG_PAGE_EXTENSION=y. + failslab= fail_usercopy= fail_page_alloc= @@ -2436,6 +2442,12 @@ 0: force disabled 1: force enabled + kunit.enable= [KUNIT] Enable executing KUnit tests. Requires + CONFIG_KUNIT to be set to be fully enabled. The + default value can be overridden via + KUNIT_DEFAULT_ENABLED. + Default is 1 (enabled) + kvm.ignore_msrs=[KVM] Ignore guest accesses to unhandled MSRs. Default is 0 (don't ignore, but inject #GP) @@ -3207,6 +3219,7 @@ spectre_v2_user=off [X86] spec_store_bypass_disable=off [X86,PPC] ssbd=force-off [ARM64] + nospectre_bhb [ARM64] l1tf=off [X86] mds=off [X86] tsx_async_abort=off [X86] @@ -3613,7 +3626,7 @@ nohugeiomap [KNL,X86,PPC,ARM64] Disable kernel huge I/O mappings. - nohugevmalloc [PPC] Disable kernel huge vmalloc mappings. + nohugevmalloc [KNL,X86,PPC,ARM64] Disable kernel huge vmalloc mappings. nosmt [KNL,S390] Disable symmetric multithreading (SMT). Equivalent to smt=1. @@ -3626,11 +3639,15 @@ (bounds check bypass). With this option data leaks are possible in the system. - nospectre_v2 [X86,PPC_FSL_BOOK3E,ARM64] Disable all mitigations for + nospectre_v2 [X86,PPC_E500,ARM64] Disable all mitigations for the Spectre variant 2 (indirect branch prediction) vulnerability. System may allow data leaks with this option. + nospectre_bhb [ARM64] Disable all mitigations for Spectre-BHB (branch + history injection) vulnerability. System may allow data leaks + with this option. + nospec_store_bypass_disable [HW] Disable all mitigations for the Speculative Store Bypass vulnerability @@ -3741,9 +3758,9 @@ [X86,PV_OPS] Disable paravirtualized VMware scheduler clock and use the default one. - no-steal-acc [X86,PV_OPS,ARM64] Disable paravirtualized steal time - accounting. steal time is computed, but won't - influence scheduler behaviour + no-steal-acc [X86,PV_OPS,ARM64,PPC/PSERIES] Disable paravirtualized + steal time accounting. steal time is computed, but + won't influence scheduler behaviour nolapic [X86-32,APIC] Do not enable or use the local APIC. @@ -3805,6 +3822,10 @@ nox2apic [X86-64,APIC] Do not enable x2APIC mode. + NOTE: this parameter will be ignored on systems with the + LEGACY_XAPIC_DISABLED bit set in the + IA32_XAPIC_DISABLE_STATUS MSR. + nps_mtm_hs_ctr= [KNL,ARC] This parameter sets the maximum duration, in cycles, each HW thread of the CTOP can run @@ -5331,6 +5352,8 @@ rodata= [KNL] on Mark read-only kernel memory as read-only (default). off Leave read-only kernel memory writable for debugging. + full Mark read-only kernel memory and aliases as read-only + [arm64] rockchip.usb_uart Enable the uart passthrough on the designated usb port @@ -6026,12 +6049,6 @@ This parameter controls use of the Protected Execution Facility on pSeries. - swapaccount= [KNL] - Format: [0|1] - Enable accounting of swap in memory resource - controller if no parameter or 1 is given or disable - it if 0 is given (See Documentation/admin-guide/cgroup-v1/memory.rst) - swiotlb= [ARM,IA-64,PPC,MIPS,X86] Format: { [,] | force | noforce } -- Number of I/O TLB slabs @@ -6834,6 +6851,12 @@ Crash from Xen panic notifier, without executing late panic() code such as dumping handler. + xen_msr_safe= [X86,XEN] + Format: + Select whether to always use non-faulting (safe) MSR + access functions when running as Xen PV guest. The + default value is controlled by CONFIG_XEN_PV_MSR_SAFE. + xen_nopvspin [X86,XEN] Disables the qspinlock slowpath using Xen PV optimizations. This parameter is obsoleted by "nopvspin" parameter, which diff --git a/Documentation/admin-guide/mm/cma_debugfs.rst b/Documentation/admin-guide/mm/cma_debugfs.rst index 4e06ffabd78ae370c6534e1a71a419226afa9b6f..7367e6294ef6ee18564bca158ac99faeace03a5e 100644 --- a/Documentation/admin-guide/mm/cma_debugfs.rst +++ b/Documentation/admin-guide/mm/cma_debugfs.rst @@ -5,10 +5,10 @@ CMA Debugfs Interface The CMA debugfs interface is useful to retrieve basic information out of the different CMA areas and to test allocation/release in each of the areas. -Each CMA zone represents a directory under /cma/, indexed by the -kernel's CMA index. So the first CMA zone would be: +Each CMA area represents a directory under /cma/, represented by +its CMA name like below: - /cma/cma-0 + /cma/ The structure of the files created under that directory is as follows: @@ -18,8 +18,8 @@ The structure of the files created under that directory is as follows: - [RO] bitmap: The bitmap of page states in the zone. - [WO] alloc: Allocate N pages from that CMA area. For example:: - echo 5 > /cma/cma-2/alloc + echo 5 > /cma//alloc -would try to allocate 5 pages from the cma-2 area. +would try to allocate 5 pages from the 'cma_name' area. - [WO] free: Free N pages from that CMA area, similar to the above. diff --git a/Documentation/admin-guide/mm/damon/index.rst b/Documentation/admin-guide/mm/damon/index.rst index 05500042f77767a186535770d4e653cb96cea84d..33d37bb2fb4e53937b497e3fc05de14e3b2925e3 100644 --- a/Documentation/admin-guide/mm/damon/index.rst +++ b/Documentation/admin-guide/mm/damon/index.rst @@ -1,8 +1,8 @@ .. SPDX-License-Identifier: GPL-2.0 -======================== -Monitoring Data Accesses -======================== +========================== +DAMON: Data Access MONitor +========================== :doc:`DAMON ` allows light-weight data access monitoring. Using DAMON, users can analyze the memory access patterns of their systems and diff --git a/Documentation/admin-guide/mm/damon/start.rst b/Documentation/admin-guide/mm/damon/start.rst index 4d5ca2c46288a13c587f6feccc40ee843044656c..9f88afc734da47d6073aba43fe4939d5d61c5793 100644 --- a/Documentation/admin-guide/mm/damon/start.rst +++ b/Documentation/admin-guide/mm/damon/start.rst @@ -29,16 +29,9 @@ called DAMON Operator (DAMO). It is available at https://github.com/awslabs/damo. The examples below assume that ``damo`` is on your ``$PATH``. It's not mandatory, though. -Because DAMO is using the debugfs interface (refer to :doc:`usage` for the -detail) of DAMON, you should ensure debugfs is mounted. Mount it manually as -below:: - - # mount -t debugfs none /sys/kernel/debug/ - -or append the following line to your ``/etc/fstab`` file so that your system -can automatically mount debugfs upon booting:: - - debugfs /sys/kernel/debug debugfs defaults 0 0 +Because DAMO is using the sysfs interface (refer to :doc:`usage` for the +detail) of DAMON, you should ensure :doc:`sysfs ` is +mounted. Recording Data Access Patterns diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst index d52f572a902989db2e0ba28b69a62fd69aab14c4..b47b0cbbd491d35b13df1a4806509672d15a7130 100644 --- a/Documentation/admin-guide/mm/damon/usage.rst +++ b/Documentation/admin-guide/mm/damon/usage.rst @@ -50,10 +50,10 @@ For a short example, users can monitor the virtual address space of a given workload as below. :: # cd /sys/kernel/mm/damon/admin/ - # echo 1 > kdamonds/nr && echo 1 > kdamonds/0/contexts/nr + # echo 1 > kdamonds/nr_kdamonds && echo 1 > kdamonds/0/contexts/nr_contexts # echo vaddr > kdamonds/0/contexts/0/operations - # echo 1 > kdamonds/0/contexts/0/targets/nr - # echo $(pidof ) > kdamonds/0/contexts/0/targets/0/pid + # echo 1 > kdamonds/0/contexts/0/targets/nr_targets + # echo $(pidof ) > kdamonds/0/contexts/0/targets/0/pid_target # echo on > kdamonds/0/state Files Hierarchy @@ -366,12 +366,12 @@ memory rate becomes larger than 60%, or lower than 30%". :: # echo 1 > kdamonds/0/contexts/0/schemes/nr_schemes # cd kdamonds/0/contexts/0/schemes/0 # # set the basic access pattern and the action - # echo 4096 > access_patterns/sz/min - # echo 8192 > access_patterns/sz/max - # echo 0 > access_patterns/nr_accesses/min - # echo 5 > access_patterns/nr_accesses/max - # echo 10 > access_patterns/age/min - # echo 20 > access_patterns/age/max + # echo 4096 > access_pattern/sz/min + # echo 8192 > access_pattern/sz/max + # echo 0 > access_pattern/nr_accesses/min + # echo 5 > access_pattern/nr_accesses/max + # echo 10 > access_pattern/age/min + # echo 20 > access_pattern/age/max # echo pageout > action # # set quotas # echo 10 > quotas/ms @@ -393,6 +393,11 @@ the files as above. Above is only for an example. debugfs Interface ================= +.. note:: + + DAMON debugfs interface will be removed after next LTS kernel is released, so + users should move to the :ref:`sysfs interface `. + DAMON exports eight files, ``attrs``, ``target_ids``, ``init_regions``, ``schemes``, ``monitor_on``, ``kdamond_pid``, ``mk_contexts`` and ``rm_contexts`` under its debugfs directory, ``/damon/``. diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst index 8e2727dc18d4d3a458e1d931abe889e46dd7ca8c..19f27c0d92e074a7e829ed0eb1d403fe9bb65709 100644 --- a/Documentation/admin-guide/mm/hugetlbpage.rst +++ b/Documentation/admin-guide/mm/hugetlbpage.rst @@ -65,7 +65,7 @@ HugePages_Surp may be temporarily larger than the maximum number of surplus huge pages when the system is under memory pressure. Hugepagesize - is the default hugepage size (in Kb). + is the default hugepage size (in kB). Hugetlb is the total amount of memory (in kB), consumed by huge pages of all sizes. diff --git a/Documentation/admin-guide/mm/index.rst b/Documentation/admin-guide/mm/index.rst index 1bd11118dfb1c6c1fd5575005685b25bdf0bb620..d1064e0ba34a295cd678ab0e00f021445af3220d 100644 --- a/Documentation/admin-guide/mm/index.rst +++ b/Documentation/admin-guide/mm/index.rst @@ -32,6 +32,7 @@ the Linux memory management. idle_page_tracking ksm memory-hotplug + multigen_lru nommu-mmap numa_memory_policy numaperf diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index b244f0202a0360120e8ddaf271df8d72dfa97532..fb6ba2002a4b2aaa73e20b7203578366ff8d472a 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -184,6 +184,42 @@ The maximum possible ``pages_sharing/pages_shared`` ratio is limited by the ``max_page_sharing`` tunable. To increase the ratio ``max_page_sharing`` must be increased accordingly. +Monitoring KSM profit +===================== + +KSM can save memory by merging identical pages, but also can consume +additional memory, because it needs to generate a number of rmap_items to +save each scanned page's brief rmap information. Some of these pages may +be merged, but some may not be abled to be merged after being checked +several times, which are unprofitable memory consumed. + +1) How to determine whether KSM save memory or consume memory in system-wide + range? Here is a simple approximate calculation for reference:: + + general_profit =~ pages_sharing * sizeof(page) - (all_rmap_items) * + sizeof(rmap_item); + + where all_rmap_items can be easily obtained by summing ``pages_sharing``, + ``pages_shared``, ``pages_unshared`` and ``pages_volatile``. + +2) The KSM profit inner a single process can be similarly obtained by the + following approximate calculation:: + + process_profit =~ ksm_merging_pages * sizeof(page) - + ksm_rmap_items * sizeof(rmap_item). + + where ksm_merging_pages is shown under the directory ``/proc//``, + and ksm_rmap_items is shown in ``/proc//ksm_stat``. + +From the perspective of application, a high ratio of ``ksm_rmap_items`` to +``ksm_merging_pages`` means a bad madvise-applied policy, so developers or +administrators have to rethink how to change madvise policy. Giving an example +for reference, a page's size is usually 4K, and the rmap_item's size is +separately 32B on 32-bit CPU architecture and 64B on 64-bit CPU architecture. +so if the ``ksm_rmap_items/ksm_merging_pages`` ratio exceeds 64 on 64-bit CPU +or exceeds 128 on 32-bit CPU, then the app's madvise policy should be dropped, +because the ksm profit is approximately zero or negative. + Monitoring KSM events ===================== diff --git a/Documentation/admin-guide/mm/multigen_lru.rst b/Documentation/admin-guide/mm/multigen_lru.rst new file mode 100644 index 0000000000000000000000000000000000000000..33e068830497e742a0d91081fc84dcf8661d9fde --- /dev/null +++ b/Documentation/admin-guide/mm/multigen_lru.rst @@ -0,0 +1,162 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +Multi-Gen LRU +============= +The multi-gen LRU is an alternative LRU implementation that optimizes +page reclaim and improves performance under memory pressure. Page +reclaim decides the kernel's caching policy and ability to overcommit +memory. It directly impacts the kswapd CPU usage and RAM efficiency. + +Quick start +=========== +Build the kernel with the following configurations. + +* ``CONFIG_LRU_GEN=y`` +* ``CONFIG_LRU_GEN_ENABLED=y`` + +All set! + +Runtime options +=============== +``/sys/kernel/mm/lru_gen/`` contains stable ABIs described in the +following subsections. + +Kill switch +----------- +``enabled`` accepts different values to enable or disable the +following components. Its default value depends on +``CONFIG_LRU_GEN_ENABLED``. All the components should be enabled +unless some of them have unforeseen side effects. Writing to +``enabled`` has no effect when a component is not supported by the +hardware, and valid values will be accepted even when the main switch +is off. + +====== =============================================================== +Values Components +====== =============================================================== +0x0001 The main switch for the multi-gen LRU. +0x0002 Clearing the accessed bit in leaf page table entries in large + batches, when MMU sets it (e.g., on x86). This behavior can + theoretically worsen lock contention (mmap_lock). If it is + disabled, the multi-gen LRU will suffer a minor performance + degradation for workloads that contiguously map hot pages, + whose accessed bits can be otherwise cleared by fewer larger + batches. +0x0004 Clearing the accessed bit in non-leaf page table entries as + well, when MMU sets it (e.g., on x86). This behavior was not + verified on x86 varieties other than Intel and AMD. If it is + disabled, the multi-gen LRU will suffer a negligible + performance degradation. +[yYnN] Apply to all the components above. +====== =============================================================== + +E.g., +:: + + echo y >/sys/kernel/mm/lru_gen/enabled + cat /sys/kernel/mm/lru_gen/enabled + 0x0007 + echo 5 >/sys/kernel/mm/lru_gen/enabled + cat /sys/kernel/mm/lru_gen/enabled + 0x0005 + +Thrashing prevention +-------------------- +Personal computers are more sensitive to thrashing because it can +cause janks (lags when rendering UI) and negatively impact user +experience. The multi-gen LRU offers thrashing prevention to the +majority of laptop and desktop users who do not have ``oomd``. + +Users can write ``N`` to ``min_ttl_ms`` to prevent the working set of +``N`` milliseconds from getting evicted. The OOM killer is triggered +if this working set cannot be kept in memory. In other words, this +option works as an adjustable pressure relief valve, and when open, it +terminates applications that are hopefully not being used. + +Based on the average human detectable lag (~100ms), ``N=1000`` usually +eliminates intolerable janks due to thrashing. Larger values like +``N=3000`` make janks less noticeable at the risk of premature OOM +kills. + +The default value ``0`` means disabled. + +Experimental features +===================== +``/sys/kernel/debug/lru_gen`` accepts commands described in the +following subsections. Multiple command lines are supported, so does +concatenation with delimiters ``,`` and ``;``. + +``/sys/kernel/debug/lru_gen_full`` provides additional stats for +debugging. ``CONFIG_LRU_GEN_STATS=y`` keeps historical stats from +evicted generations in this file. + +Working set estimation +---------------------- +Working set estimation measures how much memory an application needs +in a given time interval, and it is usually done with little impact on +the performance of the application. E.g., data centers want to +optimize job scheduling (bin packing) to improve memory utilizations. +When a new job comes in, the job scheduler needs to find out whether +each server it manages can allocate a certain amount of memory for +this new job before it can pick a candidate. To do so, the job +scheduler needs to estimate the working sets of the existing jobs. + +When it is read, ``lru_gen`` returns a histogram of numbers of pages +accessed over different time intervals for each memcg and node. +``MAX_NR_GENS`` decides the number of bins for each histogram. The +histograms are noncumulative. +:: + + memcg memcg_id memcg_path + node node_id + min_gen_nr age_in_ms nr_anon_pages nr_file_pages + ... + max_gen_nr age_in_ms nr_anon_pages nr_file_pages + +Each bin contains an estimated number of pages that have been accessed +within ``age_in_ms``. E.g., ``min_gen_nr`` contains the coldest pages +and ``max_gen_nr`` contains the hottest pages, since ``age_in_ms`` of +the former is the largest and that of the latter is the smallest. + +Users can write the following command to ``lru_gen`` to create a new +generation ``max_gen_nr+1``: + + ``+ memcg_id node_id max_gen_nr [can_swap [force_scan]]`` + +``can_swap`` defaults to the swap setting and, if it is set to ``1``, +it forces the scan of anon pages when swap is off, and vice versa. +``force_scan`` defaults to ``1`` and, if it is set to ``0``, it +employs heuristics to reduce the overhead, which is likely to reduce +the coverage as well. + +A typical use case is that a job scheduler runs this command at a +certain time interval to create new generations, and it ranks the +servers it manages based on the sizes of their cold pages defined by +this time interval. + +Proactive reclaim +----------------- +Proactive reclaim induces page reclaim when there is no memory +pressure. It usually targets cold pages only. E.g., when a new job +comes in, the job scheduler wants to proactively reclaim cold pages on +the server it selected, to improve the chance of successfully landing +this new job. + +Users can write the following command to ``lru_gen`` to evict +generations less than or equal to ``min_gen_nr``. + + ``- memcg_id node_id min_gen_nr [swappiness [nr_to_reclaim]]`` + +``min_gen_nr`` should be less than ``max_gen_nr-1``, since +``max_gen_nr`` and ``max_gen_nr-1`` are not fully aged (equivalent to +the active list) and therefore cannot be evicted. ``swappiness`` +overrides the default value in ``/proc/sys/vm/swappiness``. +``nr_to_reclaim`` limits the number of pages to evict. + +A typical use case is that a job scheduler runs this command before it +tries to land a new job on a server. If it fails to materialize enough +cold pages because of the overestimation, it retries on the next +server according to the ranking result obtained from the working set +estimation step. This less forceful approach limits the impacts on the +existing jobs. diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index c9c37f16eef881aa49ab3c35d79ac42904ec7dc8..8ee78ec232ebcf73228abb95c308c0eaac3fc3b0 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -191,7 +191,14 @@ allocation failure to throttle the next allocation attempt:: /sys/kernel/mm/transparent_hugepage/khugepaged/alloc_sleep_millisecs -The khugepaged progress can be seen in the number of pages collapsed:: +The khugepaged progress can be seen in the number of pages collapsed (note +that this counter may not be an exact count of the number of pages +collapsed, since "collapsed" could mean multiple things: (1) A PTE mapping +being replaced by a PMD mapping, or (2) All 4K physical pages replaced by +one 2M hugepage. Each may happen independently, or together, depending on +the type of memory and the failures that occur. As such, this value should +be interpreted roughly as a sign of progress, and counters in /proc/vmstat +consulted for more accurate accounting):: /sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed @@ -366,10 +373,9 @@ thp_split_pmd page table entry. thp_zero_page_alloc - is incremented every time a huge zero page is - successfully allocated. It includes allocations which where - dropped due race with other allocation. Note, it doesn't count - every map of the huge zero page, only its allocation. + is incremented every time a huge zero page used for thp is + successfully allocated. Note, it doesn't count every map of + the huge zero page, only its allocation. thp_zero_page_alloc_failed is incremented if kernel fails to allocate diff --git a/Documentation/admin-guide/mm/userfaultfd.rst b/Documentation/admin-guide/mm/userfaultfd.rst index 6528036093e1fb5a56cf3a7a8fdf0c43fa7e6fe2..83f31919ebb3cd298aeb7e7dd94858579626778f 100644 --- a/Documentation/admin-guide/mm/userfaultfd.rst +++ b/Documentation/admin-guide/mm/userfaultfd.rst @@ -17,7 +17,10 @@ of the ``PROT_NONE+SIGSEGV`` trick. Design ====== -Userfaults are delivered and resolved through the ``userfaultfd`` syscall. +Userspace creates a new userfaultfd, initializes it, and registers one or more +regions of virtual memory with it. Then, any page faults which occur within the +region(s) result in a message being delivered to the userfaultfd, notifying +userspace of the fault. The ``userfaultfd`` (aside from registering and unregistering virtual memory ranges) provides two primary functionalities: @@ -34,12 +37,11 @@ The real advantage of userfaults if compared to regular virtual memory management of mremap/mprotect is that the userfaults in all their operations never involve heavyweight structures like vmas (in fact the ``userfaultfd`` runtime load never takes the mmap_lock for writing). - Vmas are not suitable for page- (or hugepage) granular fault tracking when dealing with virtual address spaces that could span Terabytes. Too many vmas would be needed for that. -The ``userfaultfd`` once opened by invoking the syscall, can also be +The ``userfaultfd``, once created, can also be passed using unix domain sockets to a manager process, so the same manager process could handle the userfaults of a multitude of different processes without them being aware about what is going on @@ -50,6 +52,39 @@ is a corner case that would currently return ``-EBUSY``). API === +Creating a userfaultfd +---------------------- + +There are two ways to create a new userfaultfd, each of which provide ways to +restrict access to this functionality (since historically userfaultfds which +handle kernel page faults have been a useful tool for exploiting the kernel). + +The first way, supported since userfaultfd was introduced, is the +userfaultfd(2) syscall. Access to this is controlled in several ways: + +- Any user can always create a userfaultfd which traps userspace page faults + only. Such a userfaultfd can be created using the userfaultfd(2) syscall + with the flag UFFD_USER_MODE_ONLY. + +- In order to also trap kernel page faults for the address space, either the + process needs the CAP_SYS_PTRACE capability, or the system must have + vm.unprivileged_userfaultfd set to 1. By default, vm.unprivileged_userfaultfd + is set to 0. + +The second way, added to the kernel more recently, is by opening +/dev/userfaultfd and issuing a USERFAULTFD_IOC_NEW ioctl to it. This method +yields equivalent userfaultfds to the userfaultfd(2) syscall. + +Unlike userfaultfd(2), access to /dev/userfaultfd is controlled via normal +filesystem permissions (user/group/mode), which gives fine grained access to +userfaultfd specifically, without also granting other unrelated privileges at +the same time (as e.g. granting CAP_SYS_PTRACE would do). Users who have access +to /dev/userfaultfd can always create userfaultfds that trap kernel page faults; +vm.unprivileged_userfaultfd is not considered. + +Initializing a userfaultfd +-------------------------- + When first opened the ``userfaultfd`` must be enabled invoking the ``UFFDIO_API`` ioctl specifying a ``uffdio_api.api`` value set to ``UFFD_API`` (or a later API version) which will specify the ``read/POLLIN`` protocol diff --git a/Documentation/admin-guide/perf/alibaba_pmu.rst b/Documentation/admin-guide/perf/alibaba_pmu.rst new file mode 100644 index 0000000000000000000000000000000000000000..11de998bb4806ac8071db104564a2620b9c77d28 --- /dev/null +++ b/Documentation/admin-guide/perf/alibaba_pmu.rst @@ -0,0 +1,100 @@ +============================================================= +Alibaba's T-Head SoC Uncore Performance Monitoring Unit (PMU) +============================================================= + +The Yitian 710, custom-built by Alibaba Group's chip development business, +T-Head, implements uncore PMU for performance and functional debugging to +facilitate system maintenance. + +DDR Sub-System Driveway (DRW) PMU Driver +========================================= + +Yitian 710 employs eight DDR5/4 channels, four on each die. Each DDR5 channel +is independent of others to service system memory requests. And one DDR5 +channel is split into two independent sub-channels. The DDR Sub-System Driveway +implements separate PMUs for each sub-channel to monitor various performance +metrics. + +The Driveway PMU devices are named as ali_drw_ with perf. +For example, ali_drw_21000 and ali_drw_21080 are two PMU devices for two +sub-channels of the same channel in die 0. And the PMU device of die 1 is +prefixed with ali_drw_400XXXXX, e.g. ali_drw_40021000. + +Each sub-channel has 36 PMU counters in total, which is classified into +four groups: + +- Group 0: PMU Cycle Counter. This group has one pair of counters + pmu_cycle_cnt_low and pmu_cycle_cnt_high, that is used as the cycle count + based on DDRC core clock. + +- Group 1: PMU Bandwidth Counters. This group has 8 counters that are used + to count the total access number of either the eight bank groups in a + selected rank, or four ranks separately in the first 4 counters. The base + transfer unit is 64B. + +- Group 2: PMU Retry Counters. This group has 10 counters, that intend to + count the total retry number of each type of uncorrectable error. + +- Group 3: PMU Common Counters. This group has 16 counters, that are used + to count the common events. + +For now, the Driveway PMU driver only uses counters in group 0 and group 3. + +The DDR Controller (DDRCTL) and DDR PHY combine to create a complete solution +for connecting an SoC application bus to DDR memory devices. The DDRCTL +receives transactions Host Interface (HIF) which is custom-defined by Synopsys. +These transactions are queued internally and scheduled for access while +satisfying the SDRAM protocol timing requirements, transaction priorities, and +dependencies between the transactions. The DDRCTL in turn issues commands on +the DDR PHY Interface (DFI) to the PHY module, which launches and captures data +to and from the SDRAM. The driveway PMUs have hardware logic to gather +statistics and performance logging signals on HIF, DFI, etc. + +By counting the READ, WRITE and RMW commands sent to the DDRC through the HIF +interface, we could calculate the bandwidth. Example usage of counting memory +data bandwidth:: + + perf stat \ + -e ali_drw_21000/hif_wr/ \ + -e ali_drw_21000/hif_rd/ \ + -e ali_drw_21000/hif_rmw/ \ + -e ali_drw_21000/cycle/ \ + -e ali_drw_21080/hif_wr/ \ + -e ali_drw_21080/hif_rd/ \ + -e ali_drw_21080/hif_rmw/ \ + -e ali_drw_21080/cycle/ \ + -e ali_drw_23000/hif_wr/ \ + -e ali_drw_23000/hif_rd/ \ + -e ali_drw_23000/hif_rmw/ \ + -e ali_drw_23000/cycle/ \ + -e ali_drw_23080/hif_wr/ \ + -e ali_drw_23080/hif_rd/ \ + -e ali_drw_23080/hif_rmw/ \ + -e ali_drw_23080/cycle/ \ + -e ali_drw_25000/hif_wr/ \ + -e ali_drw_25000/hif_rd/ \ + -e ali_drw_25000/hif_rmw/ \ + -e ali_drw_25000/cycle/ \ + -e ali_drw_25080/hif_wr/ \ + -e ali_drw_25080/hif_rd/ \ + -e ali_drw_25080/hif_rmw/ \ + -e ali_drw_25080/cycle/ \ + -e ali_drw_27000/hif_wr/ \ + -e ali_drw_27000/hif_rd/ \ + -e ali_drw_27000/hif_rmw/ \ + -e ali_drw_27000/cycle/ \ + -e ali_drw_27080/hif_wr/ \ + -e ali_drw_27080/hif_rd/ \ + -e ali_drw_27080/hif_rmw/ \ + -e ali_drw_27080/cycle/ -- sleep 10 + +The average DRAM bandwidth can be calculated as follows: + +- Read Bandwidth = perf_hif_rd * DDRC_WIDTH * DDRC_Freq / DDRC_Cycle +- Write Bandwidth = (perf_hif_wr + perf_hif_rmw) * DDRC_WIDTH * DDRC_Freq / DDRC_Cycle + +Here, DDRC_WIDTH = 64 bytes. + +The current driver does not support sampling. So "perf record" is +unsupported. Also attach to a task is unsupported as the events are all +uncore. diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst index 9c9ece88ce53a10ce1525d959d853f72e1d6b572..793e1970bc0502ec9e29f4a870a1ae55a83cf571 100644 --- a/Documentation/admin-guide/perf/index.rst +++ b/Documentation/admin-guide/perf/index.rst @@ -18,3 +18,4 @@ Performance monitor support xgene-pmu arm_dsu_pmu thunderx2-pmu + alibaba_pmu diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index 83b58eb4ab4d0fd67c2ced984418fda6af9532cf..8f3d30c5a0d8fffe5820d66d1190dcc484d46efd 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -182,6 +182,7 @@ to the ``struct sugov_cpu`` that the utilization update belongs to. Then, ``amd-pstate`` updates the desired performance according to the CPU scheduler assigned. +.. _processor_support: Processor Support ======================= @@ -282,6 +283,8 @@ efficiency frequency management method on AMD processors. Kernel Module Options for ``amd-pstate`` ========================================= +.. _shared_mem: + ``shared_mem`` Use a module param (shared_mem) to enable related processors manually with **amd_pstate.shared_mem=1**. @@ -393,6 +396,76 @@ about part of the output. :: CPU_005 712 116384 39 49 166 0.7565 9645075 2214891 38431470 25.1 11.646 469 2.496 kworker/5:0-40 CPU_006 712 116408 39 49 166 0.6769 8950227 1839034 37192089 24.06 11.272 470 2.496 kworker/6:0-1264 +Unit Tests for amd-pstate +------------------------- + +``amd-pstate-ut`` is a test module for testing the ``amd-pstate`` driver. + + * It can help all users to verify their processor support (SBIOS/Firmware or Hardware). + + * Kernel can have a basic function test to avoid the kernel regression during the update. + + * We can introduce more functional or performance tests to align the result together, it will benefit power and performance scale optimization. + +1. Test case decriptions + + +---------+--------------------------------+------------------------------------------------------------------------------------+ + | Index | Functions | Description | + +=========+================================+====================================================================================+ + | 0 | amd_pstate_ut_acpi_cpc_valid || Check whether the _CPC object is present in SBIOS. | + | | || | + | | || The detail refer to `Processor Support `_. | + +---------+--------------------------------+------------------------------------------------------------------------------------+ + | 1 | amd_pstate_ut_check_enabled || Check whether AMD P-State is enabled. | + | | || | + | | || AMD P-States and ACPI hardware P-States always can be supported in one processor. | + | | | But AMD P-States has the higher priority and if it is enabled with | + | | | :c:macro:`MSR_AMD_CPPC_ENABLE` or ``cppc_set_enable``, it will respond to the | + | | | request from AMD P-States. | + +---------+--------------------------------+------------------------------------------------------------------------------------+ + | 2 | amd_pstate_ut_check_perf || Check if the each performance values are reasonable. | + | | || highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0. | + +---------+--------------------------------+------------------------------------------------------------------------------------+ + | 3 | amd_pstate_ut_check_freq || Check if the each frequency values and max freq when set support boost mode | + | | | are reasonable. | + | | || max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0 | + | | || If boost is not active but supported, this maximum frequency will be larger than | + | | | the one in ``cpuinfo``. | + +---------+--------------------------------+------------------------------------------------------------------------------------+ + +#. How to execute the tests + + We use test module in the kselftest frameworks to implement it. + We create amd-pstate-ut module and tie it into kselftest.(for + details refer to Linux Kernel Selftests [4]_). + + 1. Build + + + open the :c:macro:`CONFIG_X86_AMD_PSTATE` configuration option. + + set the :c:macro:`CONFIG_X86_AMD_PSTATE_UT` configuration option to M. + + make project + + make selftest :: + + $ cd linux + $ make -C tools/testing/selftests + + #. Installation & Steps :: + + $ make -C tools/testing/selftests install INSTALL_PATH=~/kselftest + $ sudo ./kselftest/run_kselftest.sh -c amd-pstate + TAP version 13 + 1..1 + # selftests: amd-pstate: amd-pstate-ut.sh + # amd-pstate-ut: ok + ok 1 selftests: amd-pstate: amd-pstate-ut.sh + + #. Results :: + + $ dmesg | grep "amd_pstate_ut" | tee log.txt + [12977.570663] amd_pstate_ut: 1 amd_pstate_ut_acpi_cpc_valid success! + [12977.570673] amd_pstate_ut: 2 amd_pstate_ut_check_enabled success! + [12977.571207] amd_pstate_ut: 3 amd_pstate_ut_check_perf success! + [12977.571212] amd_pstate_ut: 4 amd_pstate_ut_check_freq success! Reference =========== @@ -405,3 +478,6 @@ Reference .. [3] Processor Programming Reference (PPR) for AMD Family 19h Model 51h, Revision A1 Processors https://www.amd.com/system/files/TechDocs/56569-A1-PUB.zip + +.. [4] Linux Kernel Selftests, + https://www.kernel.org/doc/html/latest/dev-tools/kselftest.html diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index ee6572b1edadaffb3968047f2d568f777203c2ca..98d1b198b2b4c1040172761c88c7ee956c1edb12 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -65,6 +65,11 @@ combining the following values: 4 s3_beep = ======= +arch +==== + +The machine hardware name, the same output as ``uname -m`` +(e.g. ``x86_64`` or ``aarch64``). auto_msgmni =========== @@ -635,6 +640,17 @@ different types of memory (represented as different NUMA nodes) to place the hot pages in the fast memory. This is implemented based on unmapping and page fault too. +numa_balancing_promote_rate_limit_MBps +====================================== + +Too high promotion/demotion throughput between different memory types +may hurt application latency. This can be used to rate limit the +promotion throughput. The per-node max promotion throughput in MB/s +will be limited to be no more than the set value. + +A rule of thumb is to set this to less than 1/10 of the PMEM node +write bandwidth. + oops_all_cpu_backtrace ====================== diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst index 805f2281e000ae0677c3f399b12f27208de6cfb6..6394f5dc2303d4dc0fdb42acafdfce50fc01cef1 100644 --- a/Documentation/admin-guide/sysctl/net.rst +++ b/Documentation/admin-guide/sysctl/net.rst @@ -31,17 +31,18 @@ see only some of them, depending on your kernel's configuration. Table : Subdirectories in /proc/sys/net - ========= =================== = ========== ================== + ========= =================== = ========== =================== Directory Content Directory Content - ========= =================== = ========== ================== - core General parameter appletalk Appletalk protocol - unix Unix domain sockets netrom NET/ROM - 802 E802 protocol ax25 AX25 - ethernet Ethernet protocol rose X.25 PLP layer + ========= =================== = ========== =================== + 802 E802 protocol mptcp Multipath TCP + appletalk Appletalk protocol netfilter Network Filter + ax25 AX25 netrom NET/ROM + bridge Bridging rose X.25 PLP layer + core General parameter tipc TIPC + ethernet Ethernet protocol unix Unix domain sockets ipv4 IP version 4 x25 X.25 protocol - bridge Bridging decnet DEC net - ipv6 IP version 6 tipc TIPC - ========= =================== = ========== ================== + ipv6 IP version 6 + ========= =================== = ========== =================== 1. /proc/sys/net/core - Network core options ============================================ @@ -101,6 +102,9 @@ Values: - 1 - enable JIT hardening for unprivileged users only - 2 - enable JIT hardening for all users +where "privileged user" in this context means a process having +CAP_BPF or CAP_SYS_ADMIN in the root user name space. + bpf_jit_kallsyms ---------------- @@ -271,7 +275,7 @@ poll cycle or the number of packets processed reaches netdev_budget. netdev_max_backlog ------------------ -Maximum number of packets, queued on the INPUT side, when the interface +Maximum number of packets, queued on the INPUT side, when the interface receives packets faster than kernel can process them. netdev_rss_key diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 9b833e439f097520fcd31f11960f43145c3bc4c2..988f6a4c8084fbd3a7952bb7ab820c59218cc0b3 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -926,6 +926,9 @@ calls without any restrictions. The default value is 0. +Another way to control permissions for userfaultfd is to use +/dev/userfaultfd instead of userfaultfd(2). See +Documentation/admin-guide/mm/userfaultfd.rst. user_reserve_kbytes =================== diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst index 7d80e8c307d14009e76404317680d3e72c841517..92a8a07f5c438c48a8df6fdef51050284074820d 100644 --- a/Documentation/admin-guide/tainted-kernels.rst +++ b/Documentation/admin-guide/tainted-kernels.rst @@ -134,6 +134,12 @@ More detailed explanation for tainting scsi/snic on something else than x86_64, scsi/ips on non x86/x86_64/itanium, have broken firmware settings for the irqchip/irq-gic on arm64 ...). + - x86/x86_64: Microcode late loading is dangerous and will result in + tainting the kernel. It requires that all CPUs rendezvous to make sure + the update happens when the system is as quiescent as possible. However, + a higher priority MCE/SMI/NMI can move control flow away from that + rendezvous and interrupt the update, which can be detrimental to the + machine. 3) ``R`` if a module was force unloaded by ``rmmod -f``, ``' '`` if all modules were unloaded normally. diff --git a/Documentation/arm/index.rst b/Documentation/arm/index.rst index 495ada7915e1e3860a42beed537b7c538a3080b4..8c636d4a061f571b520e0bcef1a471ecfacada60 100644 --- a/Documentation/arm/index.rst +++ b/Documentation/arm/index.rst @@ -59,6 +59,7 @@ SoC-specific documents stm32/stm32f429-overview stm32/stm32mp13-overview stm32/stm32mp157-overview + stm32/stm32-dma-mdma-chaining sunxi diff --git a/Documentation/arm/stm32/stm32-dma-mdma-chaining.rst b/Documentation/arm/stm32/stm32-dma-mdma-chaining.rst new file mode 100644 index 0000000000000000000000000000000000000000..2945e0e331047a1aa050da72e69f15563d222083 --- /dev/null +++ b/Documentation/arm/stm32/stm32-dma-mdma-chaining.rst @@ -0,0 +1,415 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================= +STM32 DMA-MDMA chaining +======================= + + +Introduction +------------ + + This document describes the STM32 DMA-MDMA chaining feature. But before going + further, let's introduce the peripherals involved. + + To offload data transfers from the CPU, STM32 microprocessors (MPUs) embed + direct memory access controllers (DMA). + + STM32MP1 SoCs embed both STM32 DMA and STM32 MDMA controllers. STM32 DMA + request routing capabilities are enhanced by a DMA request multiplexer + (STM32 DMAMUX). + + **STM32 DMAMUX** + + STM32 DMAMUX routes any DMA request from a given peripheral to any STM32 DMA + controller (STM32MP1 counts two STM32 DMA controllers) channels. + + **STM32 DMA** + + STM32 DMA is mainly used to implement central data buffer storage (usually in + the system SRAM) for different peripheral. It can access external RAMs but + without the ability to generate convenient burst transfer ensuring the best + load of the AXI. + + **STM32 MDMA** + + STM32 MDMA (Master DMA) is mainly used to manage direct data transfers between + RAM data buffers without CPU intervention. It can also be used in a + hierarchical structure that uses STM32 DMA as first level data buffer + interfaces for AHB peripherals, while the STM32 MDMA acts as a second level + DMA with better performance. As a AXI/AHB master, STM32 MDMA can take control + of the AXI/AHB bus. + + +Principles +---------- + + STM32 DMA-MDMA chaining feature relies on the strengths of STM32 DMA and + STM32 MDMA controllers. + + STM32 DMA has a circular Double Buffer Mode (DBM). At each end of transaction + (when DMA data counter - DMA_SxNDTR - reaches 0), the memory pointers + (configured with DMA_SxSM0AR and DMA_SxM1AR) are swapped and the DMA data + counter is automatically reloaded. This allows the SW or the STM32 MDMA to + process one memory area while the second memory area is being filled/used by + the STM32 DMA transfer. + + With STM32 MDMA linked-list mode, a single request initiates the data array + (collection of nodes) to be transferred until the linked-list pointer for the + channel is null. The channel transfer complete of the last node is the end of + transfer, unless first and last nodes are linked to each other, in such a + case, the linked-list loops on to create a circular MDMA transfer. + + STM32 MDMA has direct connections with STM32 DMA. This enables autonomous + communication and synchronization between peripherals, thus saving CPU + resources and bus congestion. Transfer Complete signal of STM32 DMA channel + can triggers STM32 MDMA transfer. STM32 MDMA can clear the request generated + by the STM32 DMA by writing to its Interrupt Clear register (whose address is + stored in MDMA_CxMAR, and bit mask in MDMA_CxMDR). + + .. table:: STM32 MDMA interconnect table with STM32 DMA + + +--------------+----------------+-----------+------------+ + | STM32 DMAMUX | STM32 DMA | STM32 DMA | STM32 MDMA | + | channels | channels | Transfer | request | + | | | complete | | + | | | signal | | + +==============+================+===========+============+ + | Channel *0* | DMA1 channel 0 | dma1_tcf0 | *0x00* | + +--------------+----------------+-----------+------------+ + | Channel *1* | DMA1 channel 1 | dma1_tcf1 | *0x01* | + +--------------+----------------+-----------+------------+ + | Channel *2* | DMA1 channel 2 | dma1_tcf2 | *0x02* | + +--------------+----------------+-----------+------------+ + | Channel *3* | DMA1 channel 3 | dma1_tcf3 | *0x03* | + +--------------+----------------+-----------+------------+ + | Channel *4* | DMA1 channel 4 | dma1_tcf4 | *0x04* | + +--------------+----------------+-----------+------------+ + | Channel *5* | DMA1 channel 5 | dma1_tcf5 | *0x05* | + +--------------+----------------+-----------+------------+ + | Channel *6* | DMA1 channel 6 | dma1_tcf6 | *0x06* | + +--------------+----------------+-----------+------------+ + | Channel *7* | DMA1 channel 7 | dma1_tcf7 | *0x07* | + +--------------+----------------+-----------+------------+ + | Channel *8* | DMA2 channel 0 | dma2_tcf0 | *0x08* | + +--------------+----------------+-----------+------------+ + | Channel *9* | DMA2 channel 1 | dma2_tcf1 | *0x09* | + +--------------+----------------+-----------+------------+ + | Channel *10* | DMA2 channel 2 | dma2_tcf2 | *0x0A* | + +--------------+----------------+-----------+------------+ + | Channel *11* | DMA2 channel 3 | dma2_tcf3 | *0x0B* | + +--------------+----------------+-----------+------------+ + | Channel *12* | DMA2 channel 4 | dma2_tcf4 | *0x0C* | + +--------------+----------------+-----------+------------+ + | Channel *13* | DMA2 channel 5 | dma2_tcf5 | *0x0D* | + +--------------+----------------+-----------+------------+ + | Channel *14* | DMA2 channel 6 | dma2_tcf6 | *0x0E* | + +--------------+----------------+-----------+------------+ + | Channel *15* | DMA2 channel 7 | dma2_tcf7 | *0x0F* | + +--------------+----------------+-----------+------------+ + + STM32 DMA-MDMA chaining feature then uses a SRAM buffer. STM32MP1 SoCs embed + three fast access static internal RAMs of various size, used for data storage. + Due to STM32 DMA legacy (within microcontrollers), STM32 DMA performances are + bad with DDR, while they are optimal with SRAM. Hence the SRAM buffer used + between STM32 DMA and STM32 MDMA. This buffer is split in two equal periods + and STM32 DMA uses one period while STM32 MDMA uses the other period + simultaneously. + :: + + dma[1:2]-tcf[0:7] + .----------------. + ____________ ' _________ V____________ + | STM32 DMA | / __|>_ \ | STM32 MDMA | + |------------| | / \ | |------------| + | DMA_SxM0AR |<=>| | SRAM | |<=>| []-[]...[] | + | DMA_SxM1AR | | \_____/ | | | + |____________| \___<|____/ |____________| + + STM32 DMA-MDMA chaining uses (struct dma_slave_config).peripheral_config to + exchange the parameters needed to configure MDMA. These parameters are + gathered into a u32 array with three values: + + * the STM32 MDMA request (which is actually the DMAMUX channel ID), + * the address of the STM32 DMA register to clear the Transfer Complete + interrupt flag, + * the mask of the Transfer Complete interrupt flag of the STM32 DMA channel. + +Device Tree updates for STM32 DMA-MDMA chaining support +------------------------------------------------------- + + **1. Allocate a SRAM buffer** + + SRAM device tree node is defined in SoC device tree. You can refer to it in + your board device tree to define your SRAM pool. + :: + + &sram { + my_foo_device_dma_pool: dma-sram@0 { + reg = <0x0 0x1000>; + }; + }; + + Be careful of the start index, in case there are other SRAM consumers. + Define your pool size strategically: to optimise chaining, the idea is that + STM32 DMA and STM32 MDMA can work simultaneously, on each buffer of the + SRAM. + If the SRAM period is greater than the expected DMA transfer, then STM32 DMA + and STM32 MDMA will work sequentially instead of simultaneously. It is not a + functional issue but it is not optimal. + + Don't forget to refer to your SRAM pool in your device node. You need to + define a new property. + :: + + &my_foo_device { + ... + my_dma_pool = &my_foo_device_dma_pool; + }; + + Then get this SRAM pool in your foo driver and allocate your SRAM buffer. + + **2. Allocate a STM32 DMA channel and a STM32 MDMA channel** + + You need to define an extra channel in your device tree node, in addition to + the one you should already have for "classic" DMA operation. + + This new channel must be taken from STM32 MDMA channels, so, the phandle of + the DMA controller to use is the MDMA controller's one. + :: + + &my_foo_device { + [...] + my_dma_pool = &my_foo_device_dma_pool; + dmas = <&dmamux1 ...>, // STM32 DMA channel + <&mdma1 0 0x3 0x1200000a 0 0>; // + STM32 MDMA channel + }; + + Concerning STM32 MDMA bindings: + + 1. The request line number : whatever the value here, it will be overwritten + by MDMA driver with the STM32 DMAMUX channel ID passed through + (struct dma_slave_config).peripheral_config + + 2. The priority level : choose Very High (0x3) so that your channel will + take priority other the other during request arbitration + + 3. A 32bit mask specifying the DMA channel configuration : source and + destination address increment, block transfer with 128 bytes per single + transfer + + 4. The 32bit value specifying the register to be used to acknowledge the + request: it will be overwritten by MDMA driver, with the DMA channel + interrupt flag clear register address passed through + (struct dma_slave_config).peripheral_config + + 5. The 32bit mask specifying the value to be written to acknowledge the + request: it will be overwritten by MDMA driver, with the DMA channel + Transfer Complete flag passed through + (struct dma_slave_config).peripheral_config + +Driver updates for STM32 DMA-MDMA chaining support in foo driver +---------------------------------------------------------------- + + **0. (optional) Refactor the original sg_table if dmaengine_prep_slave_sg()** + + In case of dmaengine_prep_slave_sg(), the original sg_table can't be used as + is. Two new sg_tables must be created from the original one. One for + STM32 DMA transfer (where memory address targets now the SRAM buffer instead + of DDR buffer) and one for STM32 MDMA transfer (where memory address targets + the DDR buffer). + + The new sg_list items must fit SRAM period length. Here is an example for + DMA_DEV_TO_MEM: + :: + + /* + * Assuming sgl and nents, respectively the initial scatterlist and its + * length. + * Assuming sram_dma_buf and sram_period, respectively the memory + * allocated from the pool for DMA usage, and the length of the period, + * which is half of the sram_buf size. + */ + struct sg_table new_dma_sgt, new_mdma_sgt; + struct scatterlist *s, *_sgl; + dma_addr_t ddr_dma_buf; + u32 new_nents = 0, len; + int i; + + /* Count the number of entries needed */ + for_each_sg(sgl, s, nents, i) + if (sg_dma_len(s) > sram_period) + new_nents += DIV_ROUND_UP(sg_dma_len(s), sram_period); + else + new_nents++; + + /* Create sg table for STM32 DMA channel */ + ret = sg_alloc_table(&new_dma_sgt, new_nents, GFP_ATOMIC); + if (ret) + dev_err(dev, "DMA sg table alloc failed\n"); + + for_each_sg(new_dma_sgt.sgl, s, new_dma_sgt.nents, i) { + _sgl = sgl; + sg_dma_len(s) = min(sg_dma_len(_sgl), sram_period); + /* Targets the beginning = first half of the sram_buf */ + s->dma_address = sram_buf; + /* + * Targets the second half of the sram_buf + * for odd indexes of the item of the sg_list + */ + if (i & 1) + s->dma_address += sram_period; + } + + /* Create sg table for STM32 MDMA channel */ + ret = sg_alloc_table(&new_mdma_sgt, new_nents, GFP_ATOMIC); + if (ret) + dev_err(dev, "MDMA sg_table alloc failed\n"); + + _sgl = sgl; + len = sg_dma_len(sgl); + ddr_dma_buf = sg_dma_address(sgl); + for_each_sg(mdma_sgt.sgl, s, mdma_sgt.nents, i) { + size_t bytes = min_t(size_t, len, sram_period); + + sg_dma_len(s) = bytes; + sg_dma_address(s) = ddr_dma_buf; + len -= bytes; + + if (!len && sg_next(_sgl)) { + _sgl = sg_next(_sgl); + len = sg_dma_len(_sgl); + ddr_dma_buf = sg_dma_address(_sgl); + } else { + ddr_dma_buf += bytes; + } + } + + Don't forget to release these new sg_tables after getting the descriptors + with dmaengine_prep_slave_sg(). + + **1. Set controller specific parameters** + + First, use dmaengine_slave_config() with a struct dma_slave_config to + configure STM32 DMA channel. You just have to take care of DMA addresses, + the memory address (depending on the transfer direction) must point on your + SRAM buffer, and set (struct dma_slave_config).peripheral_size != 0. + + STM32 DMA driver will check (struct dma_slave_config).peripheral_size to + determine if chaining is being used or not. If it is used, then STM32 DMA + driver fills (struct dma_slave_config).peripheral_config with an array of + three u32 : the first one containing STM32 DMAMUX channel ID, the second one + the channel interrupt flag clear register address, and the third one the + channel Transfer Complete flag mask. + + Then, use dmaengine_slave_config with another struct dma_slave_config to + configure STM32 MDMA channel. Take care of DMA addresses, the device address + (depending on the transfer direction) must point on your SRAM buffer, and + the memory address must point to the buffer originally used for "classic" + DMA operation. Use the previous (struct dma_slave_config).peripheral_size + and .peripheral_config that have been updated by STM32 DMA driver, to set + (struct dma_slave_config).peripheral_size and .peripheral_config of the + struct dma_slave_config to configure STM32 MDMA channel. + :: + + struct dma_slave_config dma_conf; + struct dma_slave_config mdma_conf; + + memset(&dma_conf, 0, sizeof(dma_conf)); + [...] + config.direction = DMA_DEV_TO_MEM; + config.dst_addr = sram_dma_buf; // SRAM buffer + config.peripheral_size = 1; // peripheral_size != 0 => chaining + + dmaengine_slave_config(dma_chan, &dma_config); + + memset(&mdma_conf, 0, sizeof(mdma_conf)); + config.direction = DMA_DEV_TO_MEM; + mdma_conf.src_addr = sram_dma_buf; // SRAM buffer + mdma_conf.dst_addr = rx_dma_buf; // original memory buffer + mdma_conf.peripheral_size = dma_conf.peripheral_size; // <- dma_conf + mdma_conf.peripheral_config = dma_config.peripheral_config; // <- dma_conf + + dmaengine_slave_config(mdma_chan, &mdma_conf); + + **2. Get a descriptor for STM32 DMA channel transaction** + + In the same way you get your descriptor for your "classic" DMA operation, + you just have to replace the original sg_list (in case of + dmaengine_prep_slave_sg()) with the new sg_list using SRAM buffer, or to + replace the original buffer address, length and period (in case of + dmaengine_prep_dma_cyclic()) with the new SRAM buffer. + + **3. Get a descriptor for STM32 MDMA channel transaction** + + If you previously get descriptor (for STM32 DMA) with + + * dmaengine_prep_slave_sg(), then use dmaengine_prep_slave_sg() for + STM32 MDMA; + * dmaengine_prep_dma_cyclic(), then use dmaengine_prep_dma_cyclic() for + STM32 MDMA. + + Use the new sg_list using SRAM buffer (in case of dmaengine_prep_slave_sg()) + or, depending on the transfer direction, either the original DDR buffer (in + case of DMA_DEV_TO_MEM) or the SRAM buffer (in case of DMA_MEM_TO_DEV), the + source address being previously set with dmaengine_slave_config(). + + **4. Submit both transactions** + + Before submitting your transactions, you may need to define on which + descriptor you want a callback to be called at the end of the transfer + (dmaengine_prep_slave_sg()) or the period (dmaengine_prep_dma_cyclic()). + Depending on the direction, set the callback on the descriptor that finishes + the overal transfer: + + * DMA_DEV_TO_MEM: set the callback on the "MDMA" descriptor + * DMA_MEM_TO_DEV: set the callback on the "DMA" descriptor + + Then, submit the descriptors whatever the order, with dmaengine_tx_submit(). + + **5. Issue pending requests (and wait for callback notification)** + + As STM32 MDMA channel transfer is triggered by STM32 DMA, you must issue + STM32 MDMA channel before STM32 DMA channel. + + If any, your callback will be called to warn you about the end of the overal + transfer or the period completion. + + Don't forget to terminate both channels. STM32 DMA channel is configured in + cyclic Double-Buffer mode so it won't be disabled by HW, you need to terminate + it. STM32 MDMA channel will be stopped by HW in case of sg transfer, but not + in case of cyclic transfer. You can terminate it whatever the kind of transfer. + + **STM32 DMA-MDMA chaining DMA_MEM_TO_DEV special case** + + STM32 DMA-MDMA chaining in DMA_MEM_TO_DEV is a special case. Indeed, the + STM32 MDMA feeds the SRAM buffer with the DDR data, and the STM32 DMA reads + data from SRAM buffer. So some data (the first period) have to be copied in + SRAM buffer when the STM32 DMA starts to read. + + A trick could be pausing the STM32 DMA channel (that will raise a Transfer + Complete signal, triggering the STM32 MDMA channel), but the first data read + by the STM32 DMA could be "wrong". The proper way is to prepare the first SRAM + period with dmaengine_prep_dma_memcpy(). Then this first period should be + "removed" from the sg or the cyclic transfer. + + Due to this complexity, rather use the STM32 DMA-MDMA chaining for + DMA_DEV_TO_MEM and keep the "classic" DMA usage for DMA_MEM_TO_DEV, unless + you're not afraid. + +Resources +--------- + + Application note, datasheet and reference manual are available on ST website + (STM32MP1_). + + Dedicated focus on three application notes (AN5224_, AN4031_ & AN5001_) + dealing with STM32 DMAMUX, STM32 DMA and STM32 MDMA. + +.. _STM32MP1: https://www.st.com/en/microcontrollers-microprocessors/stm32mp1-series.html +.. _AN5224: https://www.st.com/resource/en/application_note/an5224-stm32-dmamux-the-dma-request-router-stmicroelectronics.pdf +.. _AN4031: https://www.st.com/resource/en/application_note/dm00046011-using-the-stm32f2-stm32f4-and-stm32f7-series-dma-controller-stmicroelectronics.pdf +.. _AN5001: https://www.st.com/resource/en/application_note/an5001-stm32cube-expansion-package-for-stm32h7-series-mdma-stmicroelectronics.pdf + +:Authors: + +- Amelie Delaunay \ No newline at end of file diff --git a/Documentation/arm/uefi.rst b/Documentation/arm/uefi.rst index 9b0b5e458a1e609ea2083c76137d850e31fe5f8d..baebe688a0064114253f2d1a33acae1ac417078a 100644 --- a/Documentation/arm/uefi.rst +++ b/Documentation/arm/uefi.rst @@ -65,10 +65,6 @@ linux,uefi-mmap-desc-size 32-bit Size in bytes of each entry in the UEFI linux,uefi-mmap-desc-ver 32-bit Version of the mmap descriptor format. -linux,initrd-start 64-bit Physical start address of an initrd - -linux,initrd-end 64-bit Physical end address of an initrd - kaslr-seed 64-bit Entropy used to randomize the kernel image base address location. ========================== ====== =========================================== diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index 52b75a25c20541bd056d83872d25cbcffbf9606c..bb34287c8e012891ee4bf04f153e5be5dc93a8a5 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -242,46 +242,39 @@ HWCAP2_MTE3 by Documentation/arm64/memory-tagging-extension.rst. HWCAP2_SME - Functionality implied by ID_AA64PFR1_EL1.SME == 0b0001, as described by Documentation/arm64/sme.rst. HWCAP2_SME_I16I64 - Functionality implied by ID_AA64SMFR0_EL1.I16I64 == 0b1111. HWCAP2_SME_F64F64 - Functionality implied by ID_AA64SMFR0_EL1.F64F64 == 0b1. HWCAP2_SME_I8I32 - Functionality implied by ID_AA64SMFR0_EL1.I8I32 == 0b1111. HWCAP2_SME_F16F32 - Functionality implied by ID_AA64SMFR0_EL1.F16F32 == 0b1. HWCAP2_SME_B16F32 - Functionality implied by ID_AA64SMFR0_EL1.B16F32 == 0b1. HWCAP2_SME_F32F32 - Functionality implied by ID_AA64SMFR0_EL1.F32F32 == 0b1. HWCAP2_SME_FA64 - Functionality implied by ID_AA64SMFR0_EL1.FA64 == 0b1. HWCAP2_WFXT - Functionality implied by ID_AA64ISAR2_EL1.WFXT == 0b0010. HWCAP2_EBF16 - Functionality implied by ID_AA64ISAR1_EL1.BF16 == 0b0010. +HWCAP2_SVE_EBF16 + Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst index 33b04db8408f98dce58be3e71cbef70fcdb2f66e..808ade4cc008ac7c41b0a13e5685fe5afae1dfe3 100644 --- a/Documentation/arm64/silicon-errata.rst +++ b/Documentation/arm64/silicon-errata.rst @@ -52,6 +52,8 @@ stable kernels. | Allwinner | A64/R18 | UNKNOWN1 | SUN50I_ERRATUM_UNKNOWN1 | +----------------+-----------------+-----------------+-----------------------------+ +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A510 | #2457168 | ARM64_ERRATUM_2457168 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A510 | #2064142 | ARM64_ERRATUM_2064142 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A510 | #2038923 | ARM64_ERRATUM_2038923 | @@ -74,6 +76,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A55 | #1530923 | ARM64_ERRATUM_1530923 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A55 | #2441007 | ARM64_ERRATUM_2441007 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A57 | #832075 | ARM64_ERRATUM_832075 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A57 | #852523 | N/A | @@ -108,6 +112,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A510 | #2441009 | ARM64_ERRATUM_2441009 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | Cortex-A510 | #2658417 | ARM64_ERRATUM_2658417 | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2119858 | ARM64_ERRATUM_2119858 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | Cortex-A710 | #2054223 | ARM64_ERRATUM_2054223 | diff --git a/Documentation/arm64/sme.rst b/Documentation/arm64/sme.rst index 937147f58cc5429471ce527bb38de5dd0b226366..16d2db4c2e2e9b08c5a1005dbfd1ab9d8be7c7d0 100644 --- a/Documentation/arm64/sme.rst +++ b/Documentation/arm64/sme.rst @@ -331,6 +331,9 @@ The regset data starts with struct user_za_header, containing: been read if a PTRACE_GETREGSET of NT_ARM_ZA were executed for each thread when the coredump was generated. +* The NT_ARM_TLS note will be extended to two registers, the second register + will contain TPIDR2_EL0 on systems that support SME and will be read as + zero with writes ignored otherwise. 9. System runtime configuration -------------------------------- diff --git a/Documentation/arm64/sve.rst b/Documentation/arm64/sve.rst index 93c2c299058497c2a2dee3a14fcd7056e47d9134..f338ee2df46d145b959b430a32246b8839fd60c0 100644 --- a/Documentation/arm64/sve.rst +++ b/Documentation/arm64/sve.rst @@ -111,7 +111,7 @@ the SVE instruction set architecture. * On syscall, V0..V31 are preserved (as without SVE). Thus, bits [127:0] of Z0..Z31 are preserved. All other bits of Z0..Z31, and all of P0..P15 and FFR - become unspecified on return from a syscall. + become zero on return from a syscall. * The SVE registers are not used to pass arguments to or receive results from any syscall. @@ -452,6 +452,24 @@ The regset data starts with struct user_sve_header, containing: * Modifying the system default vector length does not affect the vector length of any existing process or thread that does not make an execve() call. +10. Perf extensions +-------------------------------- + +* The arm64 specific DWARF standard [5] added the VG (Vector Granule) register + at index 46. This register is used for DWARF unwinding when variable length + SVE registers are pushed onto the stack. + +* Its value is equivalent to the current SVE vector length (VL) in bits divided + by 64. + +* The value is included in Perf samples in the regs[46] field if + PERF_SAMPLE_REGS_USER is set and the sample_regs_user mask has bit 46 set. + +* The value is the current value at the time the sample was taken, and it can + change over time. + +* If the system doesn't support SVE when perf_event_open is called with these + settings, the event will fail to open. Appendix A. SVE programmer's model (informative) ================================================= @@ -593,3 +611,5 @@ References http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055c/IHI0055C_beta_aapcs64.pdf http://infocenter.arm.com/help/topic/com.arm.doc.subset.swdev.abi/index.html Procedure Call Standard for the ARM 64-bit Architecture (AArch64) + +[5] https://github.com/ARM-software/abi-aa/blob/main/aadwarf64/aadwarf64.rst diff --git a/Documentation/atomic_bitops.txt b/Documentation/atomic_bitops.txt index d8b101c97031b0870d6431505fbd12f9b7d82351..edea4656c5c05f3da3f6ef46a8e8faf2b2a3f1a1 100644 --- a/Documentation/atomic_bitops.txt +++ b/Documentation/atomic_bitops.txt @@ -58,13 +58,11 @@ Like with atomic_t, the rule of thumb is: - RMW operations that have a return value are fully ordered. - - RMW operations that are conditional are unordered on FAILURE, - otherwise the above rules apply. In the case of test_and_set_bit_lock(), - if the bit in memory is unchanged by the operation then it is deemed to have - failed. + - RMW operations that are conditional are fully ordered. -Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics and -clear_bit_unlock() which has RELEASE semantics. +Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics, +clear_bit_unlock() which has RELEASE semantics and test_bit_acquire which has +ACQUIRE semantics. Since a platform only has a single means of achieving atomic operations the same barriers as for atomic_t are used, see atomic_t.txt. diff --git a/Documentation/block/index.rst b/Documentation/block/index.rst index 68f115f2b1c6bf8ec4e40654917e635cfa250216..c4c73db748a81f8c782db1e60dbe7b59480a3883 100644 --- a/Documentation/block/index.rst +++ b/Documentation/block/index.rst @@ -23,3 +23,4 @@ Block stat switching-sched writeback_cache_control + ublk diff --git a/Documentation/block/ublk.rst b/Documentation/block/ublk.rst new file mode 100644 index 0000000000000000000000000000000000000000..2122d1a4a541963450791a86320a7d0169b6e667 --- /dev/null +++ b/Documentation/block/ublk.rst @@ -0,0 +1,253 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=========================================== +Userspace block device driver (ublk driver) +=========================================== + +Overview +======== + +ublk is a generic framework for implementing block device logic from userspace. +The motivation behind it is that moving virtual block drivers into userspace, +such as loop, nbd and similar can be very helpful. It can help to implement +new virtual block device such as ublk-qcow2 (there are several attempts of +implementing qcow2 driver in kernel). + +Userspace block devices are attractive because: + +- They can be written many programming languages. +- They can use libraries that are not available in the kernel. +- They can be debugged with tools familiar to application developers. +- Crashes do not kernel panic the machine. +- Bugs are likely to have a lower security impact than bugs in kernel + code. +- They can be installed and updated independently of the kernel. +- They can be used to simulate block device easily with user specified + parameters/setting for test/debug purpose + +ublk block device (``/dev/ublkb*``) is added by ublk driver. Any IO request +on the device will be forwarded to ublk userspace program. For convenience, +in this document, ``ublk server`` refers to generic ublk userspace +program. ``ublksrv`` [#userspace]_ is one of such implementation. It +provides ``libublksrv`` [#userspace_lib]_ library for developing specific +user block device conveniently, while also generic type block device is +included, such as loop and null. Richard W.M. Jones wrote userspace nbd device +``nbdublk`` [#userspace_nbdublk]_ based on ``libublksrv`` [#userspace_lib]_. + +After the IO is handled by userspace, the result is committed back to the +driver, thus completing the request cycle. This way, any specific IO handling +logic is totally done by userspace, such as loop's IO handling, NBD's IO +communication, or qcow2's IO mapping. + +``/dev/ublkb*`` is driven by blk-mq request-based driver. Each request is +assigned by one queue wide unique tag. ublk server assigns unique tag to each +IO too, which is 1:1 mapped with IO of ``/dev/ublkb*``. + +Both the IO request forward and IO handling result committing are done via +``io_uring`` passthrough command; that is why ublk is also one io_uring based +block driver. It has been observed that using io_uring passthrough command can +give better IOPS than block IO; which is why ublk is one of high performance +implementation of userspace block device: not only IO request communication is +done by io_uring, but also the preferred IO handling in ublk server is io_uring +based approach too. + +ublk provides control interface to set/get ublk block device parameters. +The interface is extendable and kabi compatible: basically any ublk request +queue's parameter or ublk generic feature parameters can be set/get via the +interface. Thus, ublk is generic userspace block device framework. +For example, it is easy to setup a ublk device with specified block +parameters from userspace. + +Using ublk +========== + +ublk requires userspace ublk server to handle real block device logic. + +Below is example of using ``ublksrv`` to provide ublk-based loop device. + +- add a device:: + + ublk add -t loop -f ublk-loop.img + +- format with xfs, then use it:: + + mkfs.xfs /dev/ublkb0 + mount /dev/ublkb0 /mnt + # do anything. all IOs are handled by io_uring + ... + umount /mnt + +- list the devices with their info:: + + ublk list + +- delete the device:: + + ublk del -a + ublk del -n $ublk_dev_id + +See usage details in README of ``ublksrv`` [#userspace_readme]_. + +Design +====== + +Control plane +------------- + +ublk driver provides global misc device node (``/dev/ublk-control``) for +managing and controlling ublk devices with help of several control commands: + +- ``UBLK_CMD_ADD_DEV`` + + Add a ublk char device (``/dev/ublkc*``) which is talked with ublk server + WRT IO command communication. Basic device info is sent together with this + command. It sets UAPI structure of ``ublksrv_ctrl_dev_info``, + such as ``nr_hw_queues``, ``queue_depth``, and max IO request buffer size, + for which the info is negotiated with the driver and sent back to the server. + When this command is completed, the basic device info is immutable. + +- ``UBLK_CMD_SET_PARAMS`` / ``UBLK_CMD_GET_PARAMS`` + + Set or get parameters of the device, which can be either generic feature + related, or request queue limit related, but can't be IO logic specific, + because the driver does not handle any IO logic. This command has to be + sent before sending ``UBLK_CMD_START_DEV``. + +- ``UBLK_CMD_START_DEV`` + + After the server prepares userspace resources (such as creating per-queue + pthread & io_uring for handling ublk IO), this command is sent to the + driver for allocating & exposing ``/dev/ublkb*``. Parameters set via + ``UBLK_CMD_SET_PARAMS`` are applied for creating the device. + +- ``UBLK_CMD_STOP_DEV`` + + Halt IO on ``/dev/ublkb*`` and remove the device. When this command returns, + ublk server will release resources (such as destroying per-queue pthread & + io_uring). + +- ``UBLK_CMD_DEL_DEV`` + + Remove ``/dev/ublkc*``. When this command returns, the allocated ublk device + number can be reused. + +- ``UBLK_CMD_GET_QUEUE_AFFINITY`` + + When ``/dev/ublkc`` is added, the driver creates block layer tagset, so + that each queue's affinity info is available. The server sends + ``UBLK_CMD_GET_QUEUE_AFFINITY`` to retrieve queue affinity info. It can + set up the per-queue context efficiently, such as bind affine CPUs with IO + pthread and try to allocate buffers in IO thread context. + +- ``UBLK_CMD_GET_DEV_INFO`` + + For retrieving device info via ``ublksrv_ctrl_dev_info``. It is the server's + responsibility to save IO target specific info in userspace. + +Data plane +---------- + +ublk server needs to create per-queue IO pthread & io_uring for handling IO +commands via io_uring passthrough. The per-queue IO pthread +focuses on IO handling and shouldn't handle any control & management +tasks. + +The's IO is assigned by a unique tag, which is 1:1 mapping with IO +request of ``/dev/ublkb*``. + +UAPI structure of ``ublksrv_io_desc`` is defined for describing each IO from +the driver. A fixed mmaped area (array) on ``/dev/ublkc*`` is provided for +exporting IO info to the server; such as IO offset, length, OP/flags and +buffer address. Each ``ublksrv_io_desc`` instance can be indexed via queue id +and IO tag directly. + +The following IO commands are communicated via io_uring passthrough command, +and each command is only for forwarding the IO and committing the result +with specified IO tag in the command data: + +- ``UBLK_IO_FETCH_REQ`` + + Sent from the server IO pthread for fetching future incoming IO requests + destined to ``/dev/ublkb*``. This command is sent only once from the server + IO pthread for ublk driver to setup IO forward environment. + +- ``UBLK_IO_COMMIT_AND_FETCH_REQ`` + + When an IO request is destined to ``/dev/ublkb*``, the driver stores + the IO's ``ublksrv_io_desc`` to the specified mapped area; then the + previous received IO command of this IO tag (either ``UBLK_IO_FETCH_REQ`` + or ``UBLK_IO_COMMIT_AND_FETCH_REQ)`` is completed, so the server gets + the IO notification via io_uring. + + After the server handles the IO, its result is committed back to the + driver by sending ``UBLK_IO_COMMIT_AND_FETCH_REQ`` back. Once ublkdrv + received this command, it parses the result and complete the request to + ``/dev/ublkb*``. In the meantime setup environment for fetching future + requests with the same IO tag. That is, ``UBLK_IO_COMMIT_AND_FETCH_REQ`` + is reused for both fetching request and committing back IO result. + +- ``UBLK_IO_NEED_GET_DATA`` + + With ``UBLK_F_NEED_GET_DATA`` enabled, the WRITE request will be firstly + issued to ublk server without data copy. Then, IO backend of ublk server + receives the request and it can allocate data buffer and embed its addr + inside this new io command. After the kernel driver gets the command, + data copy is done from request pages to this backend's buffer. Finally, + backend receives the request again with data to be written and it can + truly handle the request. + + ``UBLK_IO_NEED_GET_DATA`` adds one additional round-trip and one + io_uring_enter() syscall. Any user thinks that it may lower performance + should not enable UBLK_F_NEED_GET_DATA. ublk server pre-allocates IO + buffer for each IO by default. Any new project should try to use this + buffer to communicate with ublk driver. However, existing project may + break or not able to consume the new buffer interface; that's why this + command is added for backwards compatibility so that existing projects + can still consume existing buffers. + +- data copy between ublk server IO buffer and ublk block IO request + + The driver needs to copy the block IO request pages into the server buffer + (pages) first for WRITE before notifying the server of the coming IO, so + that the server can handle WRITE request. + + When the server handles READ request and sends + ``UBLK_IO_COMMIT_AND_FETCH_REQ`` to the server, ublkdrv needs to copy + the server buffer (pages) read to the IO request pages. + +Future development +================== + +Container-aware ublk deivice +---------------------------- + +ublk driver doesn't handle any IO logic. Its function is well defined +for now and very limited userspace interfaces are needed, which is also +well defined too. It is possible to make ublk devices container-aware block +devices in future as Stefan Hajnoczi suggested [#stefan]_, by removing +ADMIN privilege. + +Zero copy +--------- + +Zero copy is a generic requirement for nbd, fuse or similar drivers. A +problem [#xiaoguang]_ Xiaoguang mentioned is that pages mapped to userspace +can't be remapped any more in kernel with existing mm interfaces. This can +occurs when destining direct IO to ``/dev/ublkb*``. Also, he reported that +big requests (IO size >= 256 KB) may benefit a lot from zero copy. + + +References +========== + +.. [#userspace] https://github.com/ming1/ubdsrv + +.. [#userspace_lib] https://github.com/ming1/ubdsrv/tree/master/lib + +.. [#userspace_nbdublk] https://gitlab.com/rwmjones/libnbd/-/tree/nbdublk + +.. [#userspace_readme] https://github.com/ming1/ubdsrv/blob/master/README + +.. [#stefan] https://lore.kernel.org/linux-block/YoOr6jBfgVm8GvWg@stefanha-x1.localdomain/ + +.. [#xiaoguang] https://lore.kernel.org/linux-block/YoOr6jBfgVm8GvWg@stefanha-x1.localdomain/ diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst new file mode 100644 index 0000000000000000000000000000000000000000..528feddf2db90a097e3960691a68b502d42ce71d --- /dev/null +++ b/Documentation/bpf/clang-notes.rst @@ -0,0 +1,30 @@ +.. contents:: +.. sectnum:: + +========================== +Clang implementation notes +========================== + +This document provides more details specific to the Clang/LLVM implementation of the eBPF instruction set. + +Versions +======== + +Clang defined "CPU" versions, where a CPU version of 3 corresponds to the current eBPF ISA. + +Clang can select the eBPF ISA version using ``-mcpu=v3`` for example to select version 3. + +Arithmetic instructions +======================= + +For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support with +``-Xclang -target-feature -Xclang +alu32``. In CPU version 3, support is automatically included. + +Atomic operations +================= + +Clang can generate atomic instructions by default when ``-mcpu=v3`` is +enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction +Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable +the atomics features, while keeping a lower ``-mcpu`` version, you can use +``-Xclang -target-feature -Xclang +alu32``. diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst index 1bc2c5c58bdbdc8b5c2d94dcc6c87b8754abb235..1b50de1983ee2c6be9f33b8e63930e3f1a871e5c 100644 --- a/Documentation/bpf/index.rst +++ b/Documentation/bpf/index.rst @@ -26,6 +26,8 @@ that goes into great technical depth about the BPF Architecture. classic_vs_extended.rst bpf_licensing test_debug + clang-notes + linux-notes other .. only:: subproject and html diff --git a/Documentation/bpf/instruction-set.rst b/Documentation/bpf/instruction-set.rst index 1b0e6711dec971fd9bf9550a3762142e167445ac..5d798437dad47f99e4086fe7b4dad9eaa2a08433 100644 --- a/Documentation/bpf/instruction-set.rst +++ b/Documentation/bpf/instruction-set.rst @@ -1,7 +1,12 @@ +.. contents:: +.. sectnum:: + +======================================== +eBPF Instruction Set Specification, v1.0 +======================================== + +This document specifies version 1.0 of the eBPF instruction set. -==================== -eBPF Instruction Set -==================== Registers and calling convention ================================ @@ -11,10 +16,10 @@ all of which are 64-bits wide. The eBPF calling convention is defined as: - * R0: return value from function calls, and exit value for eBPF programs - * R1 - R5: arguments for function calls - * R6 - R9: callee saved registers that function calls will preserve - * R10: read-only frame pointer to access stack +* R0: return value from function calls, and exit value for eBPF programs +* R1 - R5: arguments for function calls +* R6 - R9: callee saved registers that function calls will preserve +* R10: read-only frame pointer to access stack R0 - R5 are scratch registers and eBPF programs needs to spill/fill them if necessary across calls. @@ -24,17 +29,17 @@ Instruction encoding eBPF has two instruction encodings: - * the basic instruction encoding, which uses 64 bits to encode an instruction - * the wide instruction encoding, which appends a second 64-bit immediate value - (imm64) after the basic instruction for a total of 128 bits. +* the basic instruction encoding, which uses 64 bits to encode an instruction +* the wide instruction encoding, which appends a second 64-bit immediate value + (imm64) after the basic instruction for a total of 128 bits. The basic instruction encoding looks as follows: - ============= ======= =============== ==================== ============ - 32 bits (MSB) 16 bits 4 bits 4 bits 8 bits (LSB) - ============= ======= =============== ==================== ============ - immediate offset source register destination register opcode - ============= ======= =============== ==================== ============ +============= ======= =============== ==================== ============ +32 bits (MSB) 16 bits 4 bits 4 bits 8 bits (LSB) +============= ======= =============== ==================== ============ +immediate offset source register destination register opcode +============= ======= =============== ==================== ============ Note that most instructions do not use all of the fields. Unused fields shall be cleared to zero. @@ -44,30 +49,30 @@ Instruction classes The three LSB bits of the 'opcode' field store the instruction class: - ========= ===== =============================== - class value description - ========= ===== =============================== - BPF_LD 0x00 non-standard load operations - BPF_LDX 0x01 load into register operations - BPF_ST 0x02 store from immediate operations - BPF_STX 0x03 store from register operations - BPF_ALU 0x04 32-bit arithmetic operations - BPF_JMP 0x05 64-bit jump operations - BPF_JMP32 0x06 32-bit jump operations - BPF_ALU64 0x07 64-bit arithmetic operations - ========= ===== =============================== +========= ===== =============================== =================================== +class value description reference +========= ===== =============================== =================================== +BPF_LD 0x00 non-standard load operations `Load and store instructions`_ +BPF_LDX 0x01 load into register operations `Load and store instructions`_ +BPF_ST 0x02 store from immediate operations `Load and store instructions`_ +BPF_STX 0x03 store from register operations `Load and store instructions`_ +BPF_ALU 0x04 32-bit arithmetic operations `Arithmetic and jump instructions`_ +BPF_JMP 0x05 64-bit jump operations `Arithmetic and jump instructions`_ +BPF_JMP32 0x06 32-bit jump operations `Arithmetic and jump instructions`_ +BPF_ALU64 0x07 64-bit arithmetic operations `Arithmetic and jump instructions`_ +========= ===== =============================== =================================== Arithmetic and jump instructions ================================ -For arithmetic and jump instructions (BPF_ALU, BPF_ALU64, BPF_JMP and -BPF_JMP32), the 8-bit 'opcode' field is divided into three parts: +For arithmetic and jump instructions (``BPF_ALU``, ``BPF_ALU64``, ``BPF_JMP`` and +``BPF_JMP32``), the 8-bit 'opcode' field is divided into three parts: - ============== ====== ================= - 4 bits (MSB) 1 bit 3 bits (LSB) - ============== ====== ================= - operation code source instruction class - ============== ====== ================= +============== ====== ================= +4 bits (MSB) 1 bit 3 bits (LSB) +============== ====== ================= +operation code source instruction class +============== ====== ================= The 4th bit encodes the source operand: @@ -84,66 +89,66 @@ The four MSB bits store the operation code. Arithmetic instructions ----------------------- -BPF_ALU uses 32-bit wide operands while BPF_ALU64 uses 64-bit wide operands for +``BPF_ALU`` uses 32-bit wide operands while ``BPF_ALU64`` uses 64-bit wide operands for otherwise identical operations. -The code field encodes the operation as below: - - ======== ===== ================================================= - code value description - ======== ===== ================================================= - BPF_ADD 0x00 dst += src - BPF_SUB 0x10 dst -= src - BPF_MUL 0x20 dst \*= src - BPF_DIV 0x30 dst /= src - BPF_OR 0x40 dst \|= src - BPF_AND 0x50 dst &= src - BPF_LSH 0x60 dst <<= src - BPF_RSH 0x70 dst >>= src - BPF_NEG 0x80 dst = ~src - BPF_MOD 0x90 dst %= src - BPF_XOR 0xa0 dst ^= src - BPF_MOV 0xb0 dst = src - BPF_ARSH 0xc0 sign extending shift right - BPF_END 0xd0 byte swap operations (see separate section below) - ======== ===== ================================================= - -BPF_ADD | BPF_X | BPF_ALU means:: +The 'code' field encodes the operation as below: + +======== ===== ========================================================== +code value description +======== ===== ========================================================== +BPF_ADD 0x00 dst += src +BPF_SUB 0x10 dst -= src +BPF_MUL 0x20 dst \*= src +BPF_DIV 0x30 dst /= src +BPF_OR 0x40 dst \|= src +BPF_AND 0x50 dst &= src +BPF_LSH 0x60 dst <<= src +BPF_RSH 0x70 dst >>= src +BPF_NEG 0x80 dst = ~src +BPF_MOD 0x90 dst %= src +BPF_XOR 0xa0 dst ^= src +BPF_MOV 0xb0 dst = src +BPF_ARSH 0xc0 sign extending shift right +BPF_END 0xd0 byte swap operations (see `Byte swap instructions`_ below) +======== ===== ========================================================== + +``BPF_ADD | BPF_X | BPF_ALU`` means:: dst_reg = (u32) dst_reg + (u32) src_reg; -BPF_ADD | BPF_X | BPF_ALU64 means:: +``BPF_ADD | BPF_X | BPF_ALU64`` means:: dst_reg = dst_reg + src_reg -BPF_XOR | BPF_K | BPF_ALU means:: +``BPF_XOR | BPF_K | BPF_ALU`` means:: src_reg = (u32) src_reg ^ (u32) imm32 -BPF_XOR | BPF_K | BPF_ALU64 means:: +``BPF_XOR | BPF_K | BPF_ALU64`` means:: src_reg = src_reg ^ imm32 Byte swap instructions ----------------------- +~~~~~~~~~~~~~~~~~~~~~~ The byte swap instructions use an instruction class of ``BPF_ALU`` and a 4-bit -code field of ``BPF_END``. +'code' field of ``BPF_END``. The byte swap instructions operate on the destination register only and do not use a separate source register or immediate value. -The 1-bit source operand field in the opcode is used to to select what byte +The 1-bit source operand field in the opcode is used to select what byte order the operation convert from or to: - ========= ===== ================================================= - source value description - ========= ===== ================================================= - BPF_TO_LE 0x00 convert between host byte order and little endian - BPF_TO_BE 0x08 convert between host byte order and big endian - ========= ===== ================================================= +========= ===== ================================================= +source value description +========= ===== ================================================= +BPF_TO_LE 0x00 convert between host byte order and little endian +BPF_TO_BE 0x08 convert between host byte order and big endian +========= ===== ================================================= -The imm field encodes the width of the swap operations. The following widths +The 'imm' field encodes the width of the swap operations. The following widths are supported: 16, 32 and 64. Examples: @@ -156,35 +161,31 @@ Examples: dst_reg = htobe64(dst_reg) -``BPF_FROM_LE`` and ``BPF_FROM_BE`` exist as aliases for ``BPF_TO_LE`` and -``BPF_TO_BE`` respectively. - - Jump instructions ----------------- -BPF_JMP32 uses 32-bit wide operands while BPF_JMP uses 64-bit wide operands for +``BPF_JMP32`` uses 32-bit wide operands while ``BPF_JMP`` uses 64-bit wide operands for otherwise identical operations. -The code field encodes the operation as below: - - ======== ===== ========================= ============ - code value description notes - ======== ===== ========================= ============ - BPF_JA 0x00 PC += off BPF_JMP only - BPF_JEQ 0x10 PC += off if dst == src - BPF_JGT 0x20 PC += off if dst > src unsigned - BPF_JGE 0x30 PC += off if dst >= src unsigned - BPF_JSET 0x40 PC += off if dst & src - BPF_JNE 0x50 PC += off if dst != src - BPF_JSGT 0x60 PC += off if dst > src signed - BPF_JSGE 0x70 PC += off if dst >= src signed - BPF_CALL 0x80 function call - BPF_EXIT 0x90 function / program return BPF_JMP only - BPF_JLT 0xa0 PC += off if dst < src unsigned - BPF_JLE 0xb0 PC += off if dst <= src unsigned - BPF_JSLT 0xc0 PC += off if dst < src signed - BPF_JSLE 0xd0 PC += off if dst <= src signed - ======== ===== ========================= ============ +The 'code' field encodes the operation as below: + +======== ===== ========================= ============ +code value description notes +======== ===== ========================= ============ +BPF_JA 0x00 PC += off BPF_JMP only +BPF_JEQ 0x10 PC += off if dst == src +BPF_JGT 0x20 PC += off if dst > src unsigned +BPF_JGE 0x30 PC += off if dst >= src unsigned +BPF_JSET 0x40 PC += off if dst & src +BPF_JNE 0x50 PC += off if dst != src +BPF_JSGT 0x60 PC += off if dst > src signed +BPF_JSGE 0x70 PC += off if dst >= src signed +BPF_CALL 0x80 function call +BPF_EXIT 0x90 function / program return BPF_JMP only +BPF_JLT 0xa0 PC += off if dst < src unsigned +BPF_JLE 0xb0 PC += off if dst <= src unsigned +BPF_JSLT 0xc0 PC += off if dst < src signed +BPF_JSLE 0xd0 PC += off if dst <= src signed +======== ===== ========================= ============ The eBPF program needs to store the return value into register R0 before doing a BPF_EXIT. @@ -193,14 +194,26 @@ BPF_EXIT. Load and store instructions =========================== -For load and store instructions (BPF_LD, BPF_LDX, BPF_ST and BPF_STX), the +For load and store instructions (``BPF_LD``, ``BPF_LDX``, ``BPF_ST``, and ``BPF_STX``), the 8-bit 'opcode' field is divided as: - ============ ====== ================= - 3 bits (MSB) 2 bits 3 bits (LSB) - ============ ====== ================= - mode size instruction class - ============ ====== ================= +============ ====== ================= +3 bits (MSB) 2 bits 3 bits (LSB) +============ ====== ================= +mode size instruction class +============ ====== ================= + +The mode modifier is one of: + + ============= ===== ==================================== ============= + mode modifier value description reference + ============= ===== ==================================== ============= + BPF_IMM 0x00 64-bit immediate instructions `64-bit immediate instructions`_ + BPF_ABS 0x20 legacy BPF packet access (absolute) `Legacy BPF Packet access instructions`_ + BPF_IND 0x40 legacy BPF packet access (indirect) `Legacy BPF Packet access instructions`_ + BPF_MEM 0x60 regular load and store operations `Regular load and store operations`_ + BPF_ATOMIC 0xc0 atomic operations `Atomic operations`_ + ============= ===== ==================================== ============= The size modifier is one of: @@ -213,19 +226,6 @@ The size modifier is one of: BPF_DW 0x18 double word (8 bytes) ============= ===== ===================== -The mode modifier is one of: - - ============= ===== ==================================== - mode modifier value description - ============= ===== ==================================== - BPF_IMM 0x00 64-bit immediate instructions - BPF_ABS 0x20 legacy BPF packet access (absolute) - BPF_IND 0x40 legacy BPF packet access (indirect) - BPF_MEM 0x60 regular load and store operations - BPF_ATOMIC 0xc0 atomic operations - ============= ===== ==================================== - - Regular load and store operations --------------------------------- @@ -256,44 +256,42 @@ by other eBPF programs or means outside of this specification. All atomic operations supported by eBPF are encoded as store operations that use the ``BPF_ATOMIC`` mode modifier as follows: - * ``BPF_ATOMIC | BPF_W | BPF_STX`` for 32-bit operations - * ``BPF_ATOMIC | BPF_DW | BPF_STX`` for 64-bit operations - * 8-bit and 16-bit wide atomic operations are not supported. +* ``BPF_ATOMIC | BPF_W | BPF_STX`` for 32-bit operations +* ``BPF_ATOMIC | BPF_DW | BPF_STX`` for 64-bit operations +* 8-bit and 16-bit wide atomic operations are not supported. -The imm field is used to encode the actual atomic operation. +The 'imm' field is used to encode the actual atomic operation. Simple atomic operation use a subset of the values defined to encode -arithmetic operations in the imm field to encode the atomic operation: +arithmetic operations in the 'imm' field to encode the atomic operation: - ======== ===== =========== - imm value description - ======== ===== =========== - BPF_ADD 0x00 atomic add - BPF_OR 0x40 atomic or - BPF_AND 0x50 atomic and - BPF_XOR 0xa0 atomic xor - ======== ===== =========== +======== ===== =========== +imm value description +======== ===== =========== +BPF_ADD 0x00 atomic add +BPF_OR 0x40 atomic or +BPF_AND 0x50 atomic and +BPF_XOR 0xa0 atomic xor +======== ===== =========== -``BPF_ATOMIC | BPF_W | BPF_STX`` with imm = BPF_ADD means:: +``BPF_ATOMIC | BPF_W | BPF_STX`` with 'imm' = BPF_ADD means:: *(u32 *)(dst_reg + off16) += src_reg -``BPF_ATOMIC | BPF_DW | BPF_STX`` with imm = BPF ADD means:: +``BPF_ATOMIC | BPF_DW | BPF_STX`` with 'imm' = BPF ADD means:: *(u64 *)(dst_reg + off16) += src_reg -``BPF_XADD`` is a deprecated name for ``BPF_ATOMIC | BPF_ADD``. - In addition to the simple atomic operations, there also is a modifier and two complex atomic operations: - =========== ================ =========================== - imm value description - =========== ================ =========================== - BPF_FETCH 0x01 modifier: return old value - BPF_XCHG 0xe0 | BPF_FETCH atomic exchange - BPF_CMPXCHG 0xf0 | BPF_FETCH atomic compare and exchange - =========== ================ =========================== +=========== ================ =========================== +imm value description +=========== ================ =========================== +BPF_FETCH 0x01 modifier: return old value +BPF_XCHG 0xe0 | BPF_FETCH atomic exchange +BPF_CMPXCHG 0xf0 | BPF_FETCH atomic compare and exchange +=========== ================ =========================== The ``BPF_FETCH`` modifier is optional for simple atomic operations, and always set for the complex atomic operations. If the ``BPF_FETCH`` flag @@ -309,16 +307,10 @@ The ``BPF_CMPXCHG`` operation atomically compares the value addressed by value that was at ``dst_reg + off`` before the operation is zero-extended and loaded back to ``R0``. -Clang can generate atomic instructions by default when ``-mcpu=v3`` is -enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction -Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable -the atomics features, while keeping a lower ``-mcpu`` version, you can use -``-Xclang -target-feature -Xclang +alu32``. - 64-bit immediate instructions ----------------------------- -Instructions with the ``BPF_IMM`` mode modifier use the wide instruction +Instructions with the ``BPF_IMM`` 'mode' modifier use the wide instruction encoding for an extra imm64 value. There is currently only one such instruction. @@ -331,36 +323,6 @@ There is currently only one such instruction. Legacy BPF Packet access instructions ------------------------------------- -eBPF has special instructions for access to packet data that have been -carried over from classic BPF to retain the performance of legacy socket -filters running in the eBPF interpreter. - -The instructions come in two forms: ``BPF_ABS | | BPF_LD`` and -``BPF_IND | | BPF_LD``. - -These instructions are used to access packet data and can only be used when -the program context is a pointer to networking packet. ``BPF_ABS`` -accesses packet data at an absolute offset specified by the immediate data -and ``BPF_IND`` access packet data at an offset that includes the value of -a register in addition to the immediate data. - -These instructions have seven implicit operands: - - * Register R6 is an implicit input that must contain pointer to a - struct sk_buff. - * Register R0 is an implicit output which contains the data fetched from - the packet. - * Registers R1-R5 are scratch registers that are clobbered after a call to - ``BPF_ABS | BPF_LD`` or ``BPF_IND | BPF_LD`` instructions. - -These instructions have an implicit program exit condition as well. When an -eBPF program is trying to access the data beyond the packet boundary, the -program execution will be aborted. - -``BPF_ABS | BPF_W | BPF_LD`` means:: - - R0 = ntohl(*(u32 *) (((struct sk_buff *) R6)->data + imm32)) - -``BPF_IND | BPF_W | BPF_LD`` means:: - - R0 = ntohl(*(u32 *) (((struct sk_buff *) R6)->data + src_reg + imm32)) +eBPF previously introduced special instructions for access to packet data that were +carried over from classic BPF. However, these instructions are +deprecated and should no longer be used. diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index c0b7dae6dbf549d1fd34b77e2d945521d320bde0..0f858156371de15d84631612dd812a339ac7d12a 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -137,14 +137,37 @@ KF_ACQUIRE and KF_RET_NULL flags. -------------------------- The KF_TRUSTED_ARGS flag is used for kfuncs taking pointer arguments. It -indicates that the all pointer arguments will always be refcounted, and have -their offset set to 0. It can be used to enforce that a pointer to a refcounted -object acquired from a kfunc or BPF helper is passed as an argument to this -kfunc without any modifications (e.g. pointer arithmetic) such that it is -trusted and points to the original object. This flag is often used for kfuncs -that operate (change some property, perform some operation) on an object that -was obtained using an acquire kfunc. Such kfuncs need an unchanged pointer to -ensure the integrity of the operation being performed on the expected object. +indicates that the all pointer arguments will always have a guaranteed lifetime, +and pointers to kernel objects are always passed to helpers in their unmodified +form (as obtained from acquire kfuncs). + +It can be used to enforce that a pointer to a refcounted object acquired from a +kfunc or BPF helper is passed as an argument to this kfunc without any +modifications (e.g. pointer arithmetic) such that it is trusted and points to +the original object. + +Meanwhile, it is also allowed pass pointers to normal memory to such kfuncs, +but those can have a non-zero offset. + +This flag is often used for kfuncs that operate (change some property, perform +some operation) on an object that was obtained using an acquire kfunc. Such +kfuncs need an unchanged pointer to ensure the integrity of the operation being +performed on the expected object. + +2.4.6 KF_SLEEPABLE flag +----------------------- + +The KF_SLEEPABLE flag is used for kfuncs that may sleep. Such kfuncs can only +be called by sleepable BPF programs (BPF_F_SLEEPABLE). + +2.4.7 KF_DESTRUCTIVE flag +-------------------------- + +The KF_DESTRUCTIVE flag is used to indicate functions calling which is +destructive to the system. For example such a call can result in system +rebooting or panicking. Due to this additional restrictions apply to these +calls. At the moment they only require CAP_SYS_BOOT capability, but more can be +added later. 2.5 Registering the kfuncs -------------------------- diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst new file mode 100644 index 0000000000000000000000000000000000000000..956b0c86699d07222ca9d718cbbdc46dc9dfda48 --- /dev/null +++ b/Documentation/bpf/linux-notes.rst @@ -0,0 +1,53 @@ +.. contents:: +.. sectnum:: + +========================== +Linux implementation notes +========================== + +This document provides more details specific to the Linux kernel implementation of the eBPF instruction set. + +Byte swap instructions +====================== + +``BPF_FROM_LE`` and ``BPF_FROM_BE`` exist as aliases for ``BPF_TO_LE`` and ``BPF_TO_BE`` respectively. + +Legacy BPF Packet access instructions +===================================== + +As mentioned in the `ISA standard documentation `_, +Linux has special eBPF instructions for access to packet data that have been +carried over from classic BPF to retain the performance of legacy socket +filters running in the eBPF interpreter. + +The instructions come in two forms: ``BPF_ABS | | BPF_LD`` and +``BPF_IND | | BPF_LD``. + +These instructions are used to access packet data and can only be used when +the program context is a pointer to a networking packet. ``BPF_ABS`` +accesses packet data at an absolute offset specified by the immediate data +and ``BPF_IND`` access packet data at an offset that includes the value of +a register in addition to the immediate data. + +These instructions have seven implicit operands: + +* Register R6 is an implicit input that must contain a pointer to a + struct sk_buff. +* Register R0 is an implicit output which contains the data fetched from + the packet. +* Registers R1-R5 are scratch registers that are clobbered by the + instruction. + +These instructions have an implicit program exit condition as well. If an +eBPF program attempts access data beyond the packet boundary, the +program execution will be aborted. + +``BPF_ABS | BPF_W | BPF_LD`` (0x20) means:: + + R0 = ntohl(*(u32 *) ((struct sk_buff *) R6->data + imm)) + +where ``ntohl()`` converts a 32-bit value from network byte order to host byte order. + +``BPF_IND | BPF_W | BPF_LD`` (0x40) means:: + + R0 = ntohl(*(u32 *) ((struct sk_buff *) R6->data + src + imm)) diff --git a/Documentation/bpf/map_cgroup_storage.rst b/Documentation/bpf/map_cgroup_storage.rst index cab9543017bfafce70d3b5ca19359c05050ddf01..8e5fe532c07ea6a39c6b0cc9cac7dfd526358546 100644 --- a/Documentation/bpf/map_cgroup_storage.rst +++ b/Documentation/bpf/map_cgroup_storage.rst @@ -31,7 +31,7 @@ The map uses key of type of either ``__u64 cgroup_inode_id`` or }; ``cgroup_inode_id`` is the inode id of the cgroup directory. -``attach_type`` is the the program's attach type. +``attach_type`` is the program's attach type. Linux 5.9 added support for type ``__u64 cgroup_inode_id`` as the key type. When this key type is used, then all attach types of the particular cgroup and @@ -155,7 +155,7 @@ However, the BPF program can still only associate with one map of each type ``BPF_MAP_TYPE_CGROUP_STORAGE`` or more than one ``BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE``. -In all versions, userspace may use the the attach parameters of cgroup and +In all versions, userspace may use the attach parameters of cgroup and attach type pair in ``struct bpf_cgroup_storage_key`` as the key to the BPF map APIs to read or update the storage for a given attachment. For Linux 5.9 attach type shared storages, only the first value in the struct, cgroup inode diff --git a/Documentation/conf.py b/Documentation/conf.py index 934727e23e0eb4d57c8dcf7d6191896c0faaf1fd..b50c85083149d474fae64725c2e717e171f8d6bc 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -15,6 +15,18 @@ import sys import os import sphinx +import shutil + +# helper +# ------ + +def have_command(cmd): + """Search ``cmd`` in the ``PATH`` environment. + + If found, return True. + If not found, return False. + """ + return shutil.which(cmd) is not None # Get Sphinx version major, minor, patch = sphinx.version_info[:3] @@ -86,6 +98,7 @@ if major >= 3: "__used", "__weak", "noinline", + "__fix_address", # include/linux/memblock.h: "__init_memblock", @@ -106,7 +119,32 @@ else: autosectionlabel_prefix_document = True autosectionlabel_maxdepth = 2 -extensions.append("sphinx.ext.imgmath") +# Load math renderer: +# For html builder, load imgmath only when its dependencies are met. +# mathjax is the default math renderer since Sphinx 1.8. +have_latex = have_command('latex') +have_dvipng = have_command('dvipng') +load_imgmath = have_latex and have_dvipng + +# Respect SPHINX_IMGMATH (for html docs only) +if 'SPHINX_IMGMATH' in os.environ: + env_sphinx_imgmath = os.environ['SPHINX_IMGMATH'] + if 'yes' in env_sphinx_imgmath: + load_imgmath = True + elif 'no' in env_sphinx_imgmath: + load_imgmath = False + else: + sys.stderr.write("Unknown env SPHINX_IMGMATH=%s ignored.\n" % env_sphinx_imgmath) + +# Always load imgmath for Sphinx <1.8 or for epub docs +load_imgmath = (load_imgmath or (major == 1 and minor < 8) + or 'epub' in sys.argv) + +if load_imgmath: + extensions.append("sphinx.ext.imgmath") + math_renderer = 'imgmath' +else: + math_renderer = 'mathjax' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -332,7 +370,8 @@ html_static_path = ['sphinx-static'] html_use_smartypants = False # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# Note that the RTD theme ignores this. +html_sidebars = { '**': ['searchbox.html', 'localtoc.html', 'sourcelink.html']} # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/Documentation/asm-annotations.rst b/Documentation/core-api/asm-annotations.rst similarity index 97% rename from Documentation/asm-annotations.rst rename to Documentation/core-api/asm-annotations.rst index a64f2ca469d4579eb4d88e46852181be0f53b04d..bc514ed598870c69406663b505642987052bb632 100644 --- a/Documentation/asm-annotations.rst +++ b/Documentation/core-api/asm-annotations.rst @@ -43,10 +43,11 @@ annotated objects like this, tools can be run on them to generate more useful information. In particular, on properly annotated objects, ``objtool`` can be run to check and fix the object if needed. Currently, ``objtool`` can report missing frame pointer setup/destruction in functions. It can also -automatically generate annotations for :doc:`ORC unwinder ` +automatically generate annotations for the ORC unwinder +(Documentation/x86/orc-unwinder.rst) for most code. Both of these are especially important to support reliable -stack traces which are in turn necessary for :doc:`Kernel live patching -`. +stack traces which are in turn necessary for kernel live patching +(Documentation/livepatch/livepatch.rst). Caveat and Discussion --------------------- diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst index c6f4ba2fb32d1137485746005cfc78303bda3073..f75778d374884810eaa1aae0bceb02351976dedb 100644 --- a/Documentation/core-api/cpu_hotplug.rst +++ b/Documentation/core-api/cpu_hotplug.rst @@ -560,7 +560,7 @@ available: * cpuhp_state_remove_instance(state, node) * cpuhp_state_remove_instance_nocalls(state, node) -The arguments are the same as for the the cpuhp_state_add_instance*() +The arguments are the same as for the cpuhp_state_add_instance*() variants above. The functions differ in the way how the installed callbacks are treated: diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index dc95df462eea293446e0c76db26073026b9e0623..77eb775b8b4273d200de4033d06e73d7e7ba1bbe 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -23,6 +23,7 @@ it. printk-formats printk-index symbol-namespaces + asm-annotations Data structures and low-level utilities ======================================= @@ -36,6 +37,7 @@ Library functionality that is used throughout the kernel. kref assoc_array xarray + maple_tree idr circular-buffers rbtree @@ -44,6 +46,8 @@ Library functionality that is used throughout the kernel. this_cpu_ops timekeeping errseq + wrappers/atomic_t + wrappers/atomic_bitops Low level entry and exit ======================== @@ -67,6 +71,7 @@ Documentation/locking/index.rst for more related documentation. local_ops padata ../RCU/index + wrappers/memory-barriers.rst Low-level hardware management ============================= diff --git a/Documentation/core-api/irq/irq-domain.rst b/Documentation/core-api/irq/irq-domain.rst index d30b4d0a9769c968fcd5f741d1d834f78c45d2fc..f88a6ee67a350f5b480e894023ca29b960ba31be 100644 --- a/Documentation/core-api/irq/irq-domain.rst +++ b/Documentation/core-api/irq/irq-domain.rst @@ -71,7 +71,7 @@ variety of methods: Note that irq domain lookups must happen in contexts that are compatible with a RCU read-side critical section. -The irq_create_mapping() function must be called *atleast once* +The irq_create_mapping() function must be called *at least once* before any call to irq_find_mapping(), lest the descriptor will not be allocated. diff --git a/Documentation/core-api/maple_tree.rst b/Documentation/core-api/maple_tree.rst new file mode 100644 index 0000000000000000000000000000000000000000..45defcf15da71a23a64985e1dbb17a1ef6c2b3ea --- /dev/null +++ b/Documentation/core-api/maple_tree.rst @@ -0,0 +1,217 @@ +.. SPDX-License-Identifier: GPL-2.0+ + + +========== +Maple Tree +========== + +:Author: Liam R. Howlett + +Overview +======== + +The Maple Tree is a B-Tree data type which is optimized for storing +non-overlapping ranges, including ranges of size 1. The tree was designed to +be simple to use and does not require a user written search method. It +supports iterating over a range of entries and going to the previous or next +entry in a cache-efficient manner. The tree can also be put into an RCU-safe +mode of operation which allows reading and writing concurrently. Writers must +synchronize on a lock, which can be the default spinlock, or the user can set +the lock to an external lock of a different type. + +The Maple Tree maintains a small memory footprint and was designed to use +modern processor cache efficiently. The majority of the users will be able to +use the normal API. An :ref:`maple-tree-advanced-api` exists for more complex +scenarios. The most important usage of the Maple Tree is the tracking of the +virtual memory areas. + +The Maple Tree can store values between ``0`` and ``ULONG_MAX``. The Maple +Tree reserves values with the bottom two bits set to '10' which are below 4096 +(ie 2, 6, 10 .. 4094) for internal use. If the entries may use reserved +entries then the users can convert the entries using xa_mk_value() and convert +them back by calling xa_to_value(). If the user needs to use a reserved +value, then the user can convert the value when using the +:ref:`maple-tree-advanced-api`, but are blocked by the normal API. + +The Maple Tree can also be configured to support searching for a gap of a given +size (or larger). + +Pre-allocating of nodes is also supported using the +:ref:`maple-tree-advanced-api`. This is useful for users who must guarantee a +successful store operation within a given +code segment when allocating cannot be done. Allocations of nodes are +relatively small at around 256 bytes. + +.. _maple-tree-normal-api: + +Normal API +========== + +Start by initialising a maple tree, either with DEFINE_MTREE() for statically +allocated maple trees or mt_init() for dynamically allocated ones. A +freshly-initialised maple tree contains a ``NULL`` pointer for the range ``0`` +- ``ULONG_MAX``. There are currently two types of maple trees supported: the +allocation tree and the regular tree. The regular tree has a higher branching +factor for internal nodes. The allocation tree has a lower branching factor +but allows the user to search for a gap of a given size or larger from either +``0`` upwards or ``ULONG_MAX`` down. An allocation tree can be used by +passing in the ``MT_FLAGS_ALLOC_RANGE`` flag when initialising the tree. + +You can then set entries using mtree_store() or mtree_store_range(). +mtree_store() will overwrite any entry with the new entry and return 0 on +success or an error code otherwise. mtree_store_range() works in the same way +but takes a range. mtree_load() is used to retrieve the entry stored at a +given index. You can use mtree_erase() to erase an entire range by only +knowing one value within that range, or mtree_store() call with an entry of +NULL may be used to partially erase a range or many ranges at once. + +If you want to only store a new entry to a range (or index) if that range is +currently ``NULL``, you can use mtree_insert_range() or mtree_insert() which +return -EEXIST if the range is not empty. + +You can search for an entry from an index upwards by using mt_find(). + +You can walk each entry within a range by calling mt_for_each(). You must +provide a temporary variable to store a cursor. If you want to walk each +element of the tree then ``0`` and ``ULONG_MAX`` may be used as the range. If +the caller is going to hold the lock for the duration of the walk then it is +worth looking at the mas_for_each() API in the :ref:`maple-tree-advanced-api` +section. + +Sometimes it is necessary to ensure the next call to store to a maple tree does +not allocate memory, please see :ref:`maple-tree-advanced-api` for this use case. + +Finally, you can remove all entries from a maple tree by calling +mtree_destroy(). If the maple tree entries are pointers, you may wish to free +the entries first. + +Allocating Nodes +---------------- + +The allocations are handled by the internal tree code. See +:ref:`maple-tree-advanced-alloc` for other options. + +Locking +------- + +You do not have to worry about locking. See :ref:`maple-tree-advanced-locks` +for other options. + +The Maple Tree uses RCU and an internal spinlock to synchronise access: + +Takes RCU read lock: + * mtree_load() + * mt_find() + * mt_for_each() + * mt_next() + * mt_prev() + +Takes ma_lock internally: + * mtree_store() + * mtree_store_range() + * mtree_insert() + * mtree_insert_range() + * mtree_erase() + * mtree_destroy() + * mt_set_in_rcu() + * mt_clear_in_rcu() + +If you want to take advantage of the internal lock to protect the data +structures that you are storing in the Maple Tree, you can call mtree_lock() +before calling mtree_load(), then take a reference count on the object you +have found before calling mtree_unlock(). This will prevent stores from +removing the object from the tree between looking up the object and +incrementing the refcount. You can also use RCU to avoid dereferencing +freed memory, but an explanation of that is beyond the scope of this +document. + +.. _maple-tree-advanced-api: + +Advanced API +============ + +The advanced API offers more flexibility and better performance at the +cost of an interface which can be harder to use and has fewer safeguards. +You must take care of your own locking while using the advanced API. +You can use the ma_lock, RCU or an external lock for protection. +You can mix advanced and normal operations on the same array, as long +as the locking is compatible. The :ref:`maple-tree-normal-api` is implemented +in terms of the advanced API. + +The advanced API is based around the ma_state, this is where the 'mas' +prefix originates. The ma_state struct keeps track of tree operations to make +life easier for both internal and external tree users. + +Initialising the maple tree is the same as in the :ref:`maple-tree-normal-api`. +Please see above. + +The maple state keeps track of the range start and end in mas->index and +mas->last, respectively. + +mas_walk() will walk the tree to the location of mas->index and set the +mas->index and mas->last according to the range for the entry. + +You can set entries using mas_store(). mas_store() will overwrite any entry +with the new entry and return the first existing entry that is overwritten. +The range is passed in as members of the maple state: index and last. + +You can use mas_erase() to erase an entire range by setting index and +last of the maple state to the desired range to erase. This will erase +the first range that is found in that range, set the maple state index +and last as the range that was erased and return the entry that existed +at that location. + +You can walk each entry within a range by using mas_for_each(). If you want +to walk each element of the tree then ``0`` and ``ULONG_MAX`` may be used as +the range. If the lock needs to be periodically dropped, see the locking +section mas_pause(). + +Using a maple state allows mas_next() and mas_prev() to function as if the +tree was a linked list. With such a high branching factor the amortized +performance penalty is outweighed by cache optimization. mas_next() will +return the next entry which occurs after the entry at index. mas_prev() +will return the previous entry which occurs before the entry at index. + +mas_find() will find the first entry which exists at or above index on +the first call, and the next entry from every subsequent calls. + +mas_find_rev() will find the fist entry which exists at or below the last on +the first call, and the previous entry from every subsequent calls. + +If the user needs to yield the lock during an operation, then the maple state +must be paused using mas_pause(). + +There are a few extra interfaces provided when using an allocation tree. +If you wish to search for a gap within a range, then mas_empty_area() +or mas_empty_area_rev() can be used. mas_empty_area() searches for a gap +starting at the lowest index given up to the maximum of the range. +mas_empty_area_rev() searches for a gap starting at the highest index given +and continues downward to the lower bound of the range. + +.. _maple-tree-advanced-alloc: + +Advanced Allocating Nodes +------------------------- + +Allocations are usually handled internally to the tree, however if allocations +need to occur before a write occurs then calling mas_expected_entries() will +allocate the worst-case number of needed nodes to insert the provided number of +ranges. This also causes the tree to enter mass insertion mode. Once +insertions are complete calling mas_destroy() on the maple state will free the +unused allocations. + +.. _maple-tree-advanced-locks: + +Advanced Locking +---------------- + +The maple tree uses a spinlock by default, but external locks can be used for +tree updates as well. To use an external lock, the tree must be initialized +with the ``MT_FLAGS_LOCK_EXTERN flag``, this is usually done with the +MTREE_INIT_EXT() #define, which takes an external lock as an argument. + +Functions and structures +======================== + +.. kernel-doc:: include/linux/maple_tree.h +.. kernel-doc:: lib/maple_tree.c diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 1ebcc6c3fafe7ebd06f5a195b1d428a9d9ead52a..f5dde5bceaeaf0b60f9ca87ce4e3eb96efc1047c 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -19,9 +19,6 @@ User Space Memory Access Memory Allocation Controls ========================== -.. kernel-doc:: include/linux/gfp.h - :internal: - .. kernel-doc:: include/linux/gfp_types.h :doc: Page mobility and placement hints diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index 5e89497ba314e7630c747ab37cda247f37a62ddb..dbe1aacc79d0fc44c7235a97959f69d64854a9ec 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -625,6 +625,16 @@ Examples:: %p4cc Y10 little-endian (0x20303159) %p4cc NV12 big-endian (0xb231564e) +Rust +---- + +:: + + %pA + +Only intended to be used from Rust code to format ``core::fmt::Arguments``. +Do *not* use it from C. + Thanks ====== diff --git a/Documentation/core-api/wrappers/atomic_bitops.rst b/Documentation/core-api/wrappers/atomic_bitops.rst new file mode 100644 index 0000000000000000000000000000000000000000..bf24e4081a8f4c5fb3e2846e123b20adb37caf46 --- /dev/null +++ b/Documentation/core-api/wrappers/atomic_bitops.rst @@ -0,0 +1,18 @@ +.. SPDX-License-Identifier: GPL-2.0 + This is a simple wrapper to bring atomic_bitops.txt into the RST world + until such a time as that file can be converted directly. + +============= +Atomic bitops +============= + +.. raw:: latex + + \footnotesize + +.. include:: ../../atomic_bitops.txt + :literal: + +.. raw:: latex + + \normalsize diff --git a/Documentation/core-api/wrappers/atomic_t.rst b/Documentation/core-api/wrappers/atomic_t.rst new file mode 100644 index 0000000000000000000000000000000000000000..ed109a964c778914d0b1641d201f65cb6e94ea8c --- /dev/null +++ b/Documentation/core-api/wrappers/atomic_t.rst @@ -0,0 +1,19 @@ +.. SPDX-License-Identifier: GPL-2.0 + This is a simple wrapper to bring atomic_t.txt into the RST world + until such a time as that file can be converted directly. + +============ +Atomic types +============ + +.. raw:: latex + + \footnotesize + +.. include:: ../../atomic_t.txt + :literal: + +.. raw:: latex + + \normalsize + diff --git a/Documentation/core-api/wrappers/memory-barriers.rst b/Documentation/core-api/wrappers/memory-barriers.rst new file mode 100644 index 0000000000000000000000000000000000000000..532460b5e3eb88b62107e714f1442a909faf7f04 --- /dev/null +++ b/Documentation/core-api/wrappers/memory-barriers.rst @@ -0,0 +1,18 @@ +.. SPDX-License-Identifier: GPL-2.0 + This is a simple wrapper to bring memory-barriers.txt into the RST world + until such a time as that file can be converted directly. + +============================ +Linux kernel memory barriers +============================ + +.. raw:: latex + + \footnotesize + +.. include:: ../../memory-barriers.txt + :literal: + +.. raw:: latex + + \normalsize diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index b52452bc29636c592b0e6755c8da5e2a30179724..c3389c6f38381f038ed5d9a884f2d333a749f8a2 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -612,6 +612,13 @@ Commit message See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + **BAD_FIXES_TAG** + The Fixes: tag is malformed or does not follow the community conventions. + This can occur if the tag have been split into multiple lines (e.g., when + pasted in an email program with word wrapping enabled). + + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + Comparison style ---------------- diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst index 4621eac290f46d6266ee359fae636ae3983fe823..6b0663075dc04e5d8baed21b6f6a6f36b930beab 100644 --- a/Documentation/dev-tools/index.rst +++ b/Documentation/dev-tools/index.rst @@ -24,6 +24,7 @@ Documentation/dev-tools/testing-overview.rst kcov gcov kasan + kmsan ubsan kmemleak kcsan diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst index 1772fd457fed9aa3b1f8ffb6fbded542398eb632..5c93ab91504942c8fa33d2cd0dee0efba18005c1 100644 --- a/Documentation/dev-tools/kasan.rst +++ b/Documentation/dev-tools/kasan.rst @@ -111,9 +111,17 @@ parameter can be used to control panic and reporting behaviour: report or also panic the kernel (default: ``report``). The panic happens even if ``kasan_multi_shot`` is enabled. -Hardware Tag-Based KASAN mode (see the section about various modes below) is -intended for use in production as a security mitigation. Therefore, it supports -additional boot parameters that allow disabling KASAN or controlling features: +Software and Hardware Tag-Based KASAN modes (see the section about various +modes below) support altering stack trace collection behavior: + +- ``kasan.stacktrace=off`` or ``=on`` disables or enables alloc and free stack + traces collection (default: ``on``). +- ``kasan.stack_ring_size=`` specifies the number of entries + in the stack ring (default: ``32768``). + +Hardware Tag-Based KASAN mode is intended for use in production as a security +mitigation. Therefore, it supports additional boot parameters that allow +disabling KASAN altogether or controlling its features: - ``kasan=off`` or ``=on`` controls whether KASAN is enabled (default: ``on``). @@ -132,9 +140,6 @@ additional boot parameters that allow disabling KASAN or controlling features: - ``kasan.vmalloc=off`` or ``=on`` disables or enables tagging of vmalloc allocations (default: ``on``). -- ``kasan.stacktrace=off`` or ``=on`` disables or enables alloc and free stack - traces collection (default: ``on``). - Error reports ~~~~~~~~~~~~~ diff --git a/Documentation/dev-tools/kmsan.rst b/Documentation/dev-tools/kmsan.rst new file mode 100644 index 0000000000000000000000000000000000000000..2a53a801198cbf5fe8f132cd6293b707b3d5ad16 --- /dev/null +++ b/Documentation/dev-tools/kmsan.rst @@ -0,0 +1,427 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. Copyright (C) 2022, Google LLC. + +=================================== +The Kernel Memory Sanitizer (KMSAN) +=================================== + +KMSAN is a dynamic error detector aimed at finding uses of uninitialized +values. It is based on compiler instrumentation, and is quite similar to the +userspace `MemorySanitizer tool`_. + +An important note is that KMSAN is not intended for production use, because it +drastically increases kernel memory footprint and slows the whole system down. + +Usage +===== + +Building the kernel +------------------- + +In order to build a kernel with KMSAN you will need a fresh Clang (14.0.6+). +Please refer to `LLVM documentation`_ for the instructions on how to build Clang. + +Now configure and build the kernel with CONFIG_KMSAN enabled. + +Example report +-------------- + +Here is an example of a KMSAN report:: + + ===================================================== + BUG: KMSAN: uninit-value in test_uninit_kmsan_check_memory+0x1be/0x380 [kmsan_test] + test_uninit_kmsan_check_memory+0x1be/0x380 mm/kmsan/kmsan_test.c:273 + kunit_run_case_internal lib/kunit/test.c:333 + kunit_try_run_case+0x206/0x420 lib/kunit/test.c:374 + kunit_generic_run_threadfn_adapter+0x6d/0xc0 lib/kunit/try-catch.c:28 + kthread+0x721/0x850 kernel/kthread.c:327 + ret_from_fork+0x1f/0x30 ??:? + + Uninit was stored to memory at: + do_uninit_local_array+0xfa/0x110 mm/kmsan/kmsan_test.c:260 + test_uninit_kmsan_check_memory+0x1a2/0x380 mm/kmsan/kmsan_test.c:271 + kunit_run_case_internal lib/kunit/test.c:333 + kunit_try_run_case+0x206/0x420 lib/kunit/test.c:374 + kunit_generic_run_threadfn_adapter+0x6d/0xc0 lib/kunit/try-catch.c:28 + kthread+0x721/0x850 kernel/kthread.c:327 + ret_from_fork+0x1f/0x30 ??:? + + Local variable uninit created at: + do_uninit_local_array+0x4a/0x110 mm/kmsan/kmsan_test.c:256 + test_uninit_kmsan_check_memory+0x1a2/0x380 mm/kmsan/kmsan_test.c:271 + + Bytes 4-7 of 8 are uninitialized + Memory access of size 8 starts at ffff888083fe3da0 + + CPU: 0 PID: 6731 Comm: kunit_try_catch Tainted: G B E 5.16.0-rc3+ #104 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-2 04/01/2014 + ===================================================== + +The report says that the local variable ``uninit`` was created uninitialized in +``do_uninit_local_array()``. The third stack trace corresponds to the place +where this variable was created. + +The first stack trace shows where the uninit value was used (in +``test_uninit_kmsan_check_memory()``). The tool shows the bytes which were left +uninitialized in the local variable, as well as the stack where the value was +copied to another memory location before use. + +A use of uninitialized value ``v`` is reported by KMSAN in the following cases: + - in a condition, e.g. ``if (v) { ... }``; + - in an indexing or pointer dereferencing, e.g. ``array[v]`` or ``*v``; + - when it is copied to userspace or hardware, e.g. ``copy_to_user(..., &v, ...)``; + - when it is passed as an argument to a function, and + ``CONFIG_KMSAN_CHECK_PARAM_RETVAL`` is enabled (see below). + +The mentioned cases (apart from copying data to userspace or hardware, which is +a security issue) are considered undefined behavior from the C11 Standard point +of view. + +Disabling the instrumentation +----------------------------- + +A function can be marked with ``__no_kmsan_checks``. Doing so makes KMSAN +ignore uninitialized values in that function and mark its output as initialized. +As a result, the user will not get KMSAN reports related to that function. + +Another function attribute supported by KMSAN is ``__no_sanitize_memory``. +Applying this attribute to a function will result in KMSAN not instrumenting +it, which can be helpful if we do not want the compiler to interfere with some +low-level code (e.g. that marked with ``noinstr`` which implicitly adds +``__no_sanitize_memory``). + +This however comes at a cost: stack allocations from such functions will have +incorrect shadow/origin values, likely leading to false positives. Functions +called from non-instrumented code may also receive incorrect metadata for their +parameters. + +As a rule of thumb, avoid using ``__no_sanitize_memory`` explicitly. + +It is also possible to disable KMSAN for a single file (e.g. main.o):: + + KMSAN_SANITIZE_main.o := n + +or for the whole directory:: + + KMSAN_SANITIZE := n + +in the Makefile. Think of this as applying ``__no_sanitize_memory`` to every +function in the file or directory. Most users won't need KMSAN_SANITIZE, unless +their code gets broken by KMSAN (e.g. runs at early boot time). + +Support +======= + +In order for KMSAN to work the kernel must be built with Clang, which so far is +the only compiler that has KMSAN support. The kernel instrumentation pass is +based on the userspace `MemorySanitizer tool`_. + +The runtime library only supports x86_64 at the moment. + +How KMSAN works +=============== + +KMSAN shadow memory +------------------- + +KMSAN associates a metadata byte (also called shadow byte) with every byte of +kernel memory. A bit in the shadow byte is set iff the corresponding bit of the +kernel memory byte is uninitialized. Marking the memory uninitialized (i.e. +setting its shadow bytes to ``0xff``) is called poisoning, marking it +initialized (setting the shadow bytes to ``0x00``) is called unpoisoning. + +When a new variable is allocated on the stack, it is poisoned by default by +instrumentation code inserted by the compiler (unless it is a stack variable +that is immediately initialized). Any new heap allocation done without +``__GFP_ZERO`` is also poisoned. + +Compiler instrumentation also tracks the shadow values as they are used along +the code. When needed, instrumentation code invokes the runtime library in +``mm/kmsan/`` to persist shadow values. + +The shadow value of a basic or compound type is an array of bytes of the same +length. When a constant value is written into memory, that memory is unpoisoned. +When a value is read from memory, its shadow memory is also obtained and +propagated into all the operations which use that value. For every instruction +that takes one or more values the compiler generates code that calculates the +shadow of the result depending on those values and their shadows. + +Example:: + + int a = 0xff; // i.e. 0x000000ff + int b; + int c = a | b; + +In this case the shadow of ``a`` is ``0``, shadow of ``b`` is ``0xffffffff``, +shadow of ``c`` is ``0xffffff00``. This means that the upper three bytes of +``c`` are uninitialized, while the lower byte is initialized. + +Origin tracking +--------------- + +Every four bytes of kernel memory also have a so-called origin mapped to them. +This origin describes the point in program execution at which the uninitialized +value was created. Every origin is associated with either the full allocation +stack (for heap-allocated memory), or the function containing the uninitialized +variable (for locals). + +When an uninitialized variable is allocated on stack or heap, a new origin +value is created, and that variable's origin is filled with that value. When a +value is read from memory, its origin is also read and kept together with the +shadow. For every instruction that takes one or more values, the origin of the +result is one of the origins corresponding to any of the uninitialized inputs. +If a poisoned value is written into memory, its origin is written to the +corresponding storage as well. + +Example 1:: + + int a = 42; + int b; + int c = a + b; + +In this case the origin of ``b`` is generated upon function entry, and is +stored to the origin of ``c`` right before the addition result is written into +memory. + +Several variables may share the same origin address, if they are stored in the +same four-byte chunk. In this case every write to either variable updates the +origin for all of them. We have to sacrifice precision in this case, because +storing origins for individual bits (and even bytes) would be too costly. + +Example 2:: + + int combine(short a, short b) { + union ret_t { + int i; + short s[2]; + } ret; + ret.s[0] = a; + ret.s[1] = b; + return ret.i; + } + +If ``a`` is initialized and ``b`` is not, the shadow of the result would be +0xffff0000, and the origin of the result would be the origin of ``b``. +``ret.s[0]`` would have the same origin, but it will never be used, because +that variable is initialized. + +If both function arguments are uninitialized, only the origin of the second +argument is preserved. + +Origin chaining +~~~~~~~~~~~~~~~ + +To ease debugging, KMSAN creates a new origin for every store of an +uninitialized value to memory. The new origin references both its creation stack +and the previous origin the value had. This may cause increased memory +consumption, so we limit the length of origin chains in the runtime. + +Clang instrumentation API +------------------------- + +Clang instrumentation pass inserts calls to functions defined in +``mm/kmsan/nstrumentation.c`` into the kernel code. + +Shadow manipulation +~~~~~~~~~~~~~~~~~~~ + +For every memory access the compiler emits a call to a function that returns a +pair of pointers to the shadow and origin addresses of the given memory:: + + typedef struct { + void *shadow, *origin; + } shadow_origin_ptr_t + + shadow_origin_ptr_t __msan_metadata_ptr_for_load_{1,2,4,8}(void *addr) + shadow_origin_ptr_t __msan_metadata_ptr_for_store_{1,2,4,8}(void *addr) + shadow_origin_ptr_t __msan_metadata_ptr_for_load_n(void *addr, uintptr_t size) + shadow_origin_ptr_t __msan_metadata_ptr_for_store_n(void *addr, uintptr_t size) + +The function name depends on the memory access size. + +The compiler makes sure that for every loaded value its shadow and origin +values are read from memory. When a value is stored to memory, its shadow and +origin are also stored using the metadata pointers. + +Handling locals +~~~~~~~~~~~~~~~ + +A special function is used to create a new origin value for a local variable and +set the origin of that variable to that value:: + + void __msan_poison_alloca(void *addr, uintptr_t size, char *descr) + +Access to per-task data +~~~~~~~~~~~~~~~~~~~~~~~ + +At the beginning of every instrumented function KMSAN inserts a call to +``__msan_get_context_state()``:: + + kmsan_context_state *__msan_get_context_state(void) + +``kmsan_context_state`` is declared in ``include/linux/kmsan.h``:: + + struct kmsan_context_state { + char param_tls[KMSAN_PARAM_SIZE]; + char retval_tls[KMSAN_RETVAL_SIZE]; + char va_arg_tls[KMSAN_PARAM_SIZE]; + char va_arg_origin_tls[KMSAN_PARAM_SIZE]; + u64 va_arg_overflow_size_tls; + char param_origin_tls[KMSAN_PARAM_SIZE]; + depot_stack_handle_t retval_origin_tls; + }; + +This structure is used by KMSAN to pass parameter shadows and origins between +instrumented functions (unless the parameters are checked immediately by +``CONFIG_KMSAN_CHECK_PARAM_RETVAL``). + +Passing uninitialized values to functions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clang's MemorySanitizer instrumentation has an option, +``-fsanitize-memory-param-retval``, which makes the compiler check function +parameters passed by value, as well as function return values. + +The option is controlled by ``CONFIG_KMSAN_CHECK_PARAM_RETVAL``, which is +enabled by default to let KMSAN report uninitialized values earlier. +Please refer to the `LKML discussion`_ for more details. + +Because of the way the checks are implemented in LLVM (they are only applied to +parameters marked as ``noundef``), not all parameters are guaranteed to be +checked, so we cannot give up the metadata storage in ``kmsan_context_state``. + +String functions +~~~~~~~~~~~~~~~~ + +The compiler replaces calls to ``memcpy()``/``memmove()``/``memset()`` with the +following functions. These functions are also called when data structures are +initialized or copied, making sure shadow and origin values are copied alongside +with the data:: + + void *__msan_memcpy(void *dst, void *src, uintptr_t n) + void *__msan_memmove(void *dst, void *src, uintptr_t n) + void *__msan_memset(void *dst, int c, uintptr_t n) + +Error reporting +~~~~~~~~~~~~~~~ + +For each use of a value the compiler emits a shadow check that calls +``__msan_warning()`` in the case that value is poisoned:: + + void __msan_warning(u32 origin) + +``__msan_warning()`` causes KMSAN runtime to print an error report. + +Inline assembly instrumentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +KMSAN instruments every inline assembly output with a call to:: + + void __msan_instrument_asm_store(void *addr, uintptr_t size) + +, which unpoisons the memory region. + +This approach may mask certain errors, but it also helps to avoid a lot of +false positives in bitwise operations, atomics etc. + +Sometimes the pointers passed into inline assembly do not point to valid memory. +In such cases they are ignored at runtime. + + +Runtime library +--------------- + +The code is located in ``mm/kmsan/``. + +Per-task KMSAN state +~~~~~~~~~~~~~~~~~~~~ + +Every task_struct has an associated KMSAN task state that holds the KMSAN +context (see above) and a per-task flag disallowing KMSAN reports:: + + struct kmsan_context { + ... + bool allow_reporting; + struct kmsan_context_state cstate; + ... + } + + struct task_struct { + ... + struct kmsan_context kmsan; + ... + } + +KMSAN contexts +~~~~~~~~~~~~~~ + +When running in a kernel task context, KMSAN uses ``current->kmsan.cstate`` to +hold the metadata for function parameters and return values. + +But in the case the kernel is running in the interrupt, softirq or NMI context, +where ``current`` is unavailable, KMSAN switches to per-cpu interrupt state:: + + DEFINE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); + +Metadata allocation +~~~~~~~~~~~~~~~~~~~ + +There are several places in the kernel for which the metadata is stored. + +1. Each ``struct page`` instance contains two pointers to its shadow and +origin pages:: + + struct page { + ... + struct page *shadow, *origin; + ... + }; + +At boot-time, the kernel allocates shadow and origin pages for every available +kernel page. This is done quite late, when the kernel address space is already +fragmented, so normal data pages may arbitrarily interleave with the metadata +pages. + +This means that in general for two contiguous memory pages their shadow/origin +pages may not be contiguous. Consequently, if a memory access crosses the +boundary of a memory block, accesses to shadow/origin memory may potentially +corrupt other pages or read incorrect values from them. + +In practice, contiguous memory pages returned by the same ``alloc_pages()`` +call will have contiguous metadata, whereas if these pages belong to two +different allocations their metadata pages can be fragmented. + +For the kernel data (``.data``, ``.bss`` etc.) and percpu memory regions +there also are no guarantees on metadata contiguity. + +In the case ``__msan_metadata_ptr_for_XXX_YYY()`` hits the border between two +pages with non-contiguous metadata, it returns pointers to fake shadow/origin regions:: + + char dummy_load_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); + char dummy_store_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); + +``dummy_load_page`` is zero-initialized, so reads from it always yield zeroes. +All stores to ``dummy_store_page`` are ignored. + +2. For vmalloc memory and modules, there is a direct mapping between the memory +range, its shadow and origin. KMSAN reduces the vmalloc area by 3/4, making only +the first quarter available to ``vmalloc()``. The second quarter of the vmalloc +area contains shadow memory for the first quarter, the third one holds the +origins. A small part of the fourth quarter contains shadow and origins for the +kernel modules. Please refer to ``arch/x86/include/asm/pgtable_64_types.h`` for +more details. + +When an array of pages is mapped into a contiguous virtual memory space, their +shadow and origin pages are similarly mapped into contiguous regions. + +References +========== + +E. Stepanov, K. Serebryany. `MemorySanitizer: fast detector of uninitialized +memory use in C++ +`_. +In Proceedings of CGO 2015. + +.. _MemorySanitizer tool: https://clang.llvm.org/docs/MemorySanitizer.html +.. _LLVM documentation: https://llvm.org/docs/GettingStarted.html +.. _LKML discussion: https://lore.kernel.org/all/20220614144853.3693273-1-glider@google.com/ diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst index e87973763b919ce0da2cea0396dd70c2ce848fc8..12b575b76b20b1ded2f33b28746f32244a16ac44 100644 --- a/Documentation/dev-tools/kselftest.rst +++ b/Documentation/dev-tools/kselftest.rst @@ -320,7 +320,7 @@ A bare bones test module might look like this: #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - #include "../tools/testing/selftests/kselftest/module.h" + #include "../tools/testing/selftests/kselftest_module.h" KSTM_MODULE_GLOBALS(); diff --git a/Documentation/dev-tools/kunit/architecture.rst b/Documentation/dev-tools/kunit/architecture.rst index cf9e6e3eeae4c2014b0208f7e86612509796674b..8efe792bdcb90573dcd5000318e0fd0c2be89814 100644 --- a/Documentation/dev-tools/kunit/architecture.rst +++ b/Documentation/dev-tools/kunit/architecture.rst @@ -6,8 +6,8 @@ KUnit Architecture The KUnit architecture can be divided into two parts: -- Kernel testing library -- kunit_tool (Command line test harness) +- `In-Kernel Testing Framework`_ +- `kunit_tool (Command Line Test Harness)`_ In-Kernel Testing Framework =========================== diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst index 172e239791a8cfd1ef036c50fe69126e23cb9847..fae426f2634a1f6c5148ed2274c2901c043b3431 100644 --- a/Documentation/dev-tools/kunit/faq.rst +++ b/Documentation/dev-tools/kunit/faq.rst @@ -31,13 +31,16 @@ For the most part, the KUnit core framework (what we use to write the tests) can compile to any architecture. It compiles like just another part of the kernel and runs when the kernel boots, or when built as a module, when the module is loaded. However, there is infrastructure, like the KUnit Wrapper -(``tools/testing/kunit/kunit.py``) that does not support other architectures. +(``tools/testing/kunit/kunit.py``) that might not support some architectures +(see :ref:`kunit-on-qemu`). In short, yes, you can run KUnit on other architectures, but it might require more work than using KUnit on UML. For more information, see :ref:`kunit-on-non-uml`. +.. _kinds-of-tests: + What is the difference between a unit test and other kinds of tests? ==================================================================== Most existing tests for the Linux kernel would be categorized as an integration @@ -95,8 +98,7 @@ things to try. seeing. When tests are built-in, they will execute when the kernel boots, and modules will automatically execute associated tests when loaded. Test results can be collected from ``/sys/kernel/debug/kunit//results``, and - can be parsed with ``kunit.py parse``. For more details, see "KUnit on - non-UML architectures" in Documentation/dev-tools/kunit/usage.rst. + can be parsed with ``kunit.py parse``. For more details, see :ref:`kunit-on-qemu`. If none of the above tricks help, you are always welcome to email any issues to kunit-dev@googlegroups.com. diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst index 595205348d2d9cb33045eec94ff68d9cabc344da..f5d13f1d37be1d67abb3c10cf85d8bbb0d24706c 100644 --- a/Documentation/dev-tools/kunit/index.rst +++ b/Documentation/dev-tools/kunit/index.rst @@ -13,7 +13,6 @@ KUnit - Linux Kernel Unit Testing run_wrapper run_manual usage - kunit-tool api/index style faq @@ -29,10 +28,10 @@ KUnit (Kernel unit testing framework) provides a common framework for unit tests within the Linux kernel. Using KUnit, you can define groups of test cases called test suites. The tests either run on kernel boot if built-in, or load as a module. KUnit automatically flags and reports -failed test cases in the kernel log. The test results appear in `TAP -(Test Anything Protocol) format `_. It is inspired by -JUnit, Python’s unittest.mock, and GoogleTest/GoogleMock (C++ unit testing -framework). +failed test cases in the kernel log. The test results appear in +:doc:`KTAP (Kernel - Test Anything Protocol) format`. +It is inspired by JUnit, Python’s unittest.mock, and GoogleTest/GoogleMock +(C++ unit testing framework). KUnit tests are part of the kernel, written in the C (programming) language, and test parts of the Kernel implementation (example: a C @@ -46,8 +45,9 @@ internal system functionality. KUnit runs in kernel space and is not restricted to things exposed to user-space. In addition, KUnit has kunit_tool, a script (``tools/testing/kunit/kunit.py``) -that configures the Linux kernel, runs KUnit tests under QEMU or UML (`User Mode -Linux `_), parses the test results and +that configures the Linux kernel, runs KUnit tests under QEMU or UML +(:doc:`User Mode Linux `), +parses the test results and displays them in a user friendly manner. Features @@ -95,6 +95,8 @@ Unit Testing Advantages - Improves code quality. - Encourages writing testable code. +Read also :ref:`kinds-of-tests`. + How do I use it? ================ @@ -107,7 +109,5 @@ How do I use it? examples. * Documentation/dev-tools/kunit/api/index.rst - KUnit APIs used for testing. -* Documentation/dev-tools/kunit/kunit-tool.rst - kunit_tool helper - script. * Documentation/dev-tools/kunit/faq.rst - KUnit common questions and answers. diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst deleted file mode 100644 index ae52e0f489f915d5242b25f43105774df1a9c0d4..0000000000000000000000000000000000000000 --- a/Documentation/dev-tools/kunit/kunit-tool.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -================= -kunit_tool How-To -================= - -What is kunit_tool? -=================== - -kunit_tool is a script (``tools/testing/kunit/kunit.py``) that aids in building -the Linux kernel as UML (`User Mode Linux -`_), running KUnit tests, parsing -the test results and displaying them in a user friendly manner. - -kunit_tool addresses the problem of being able to run tests without needing a -virtual machine or actual hardware with User Mode Linux. User Mode Linux is a -Linux architecture, like ARM or x86; however, unlike other architectures it -compiles the kernel as a standalone Linux executable that can be run like any -other program directly inside of a host operating system. To be clear, it does -not require any virtualization support: it is just a regular program. - -What is a .kunitconfig? -======================= - -It's just a defconfig that kunit_tool looks for in the build directory -(``.kunit`` by default). kunit_tool uses it to generate a .config as you might -expect. In addition, it verifies that the generated .config contains the CONFIG -options in the .kunitconfig; the reason it does this is so that it is easy to -be sure that a CONFIG that enables a test actually ends up in the .config. - -It's also possible to pass a separate .kunitconfig fragment to kunit_tool, -which is useful if you have several different groups of tests you wish -to run independently, or if you want to use pre-defined test configs for -certain subsystems. - -Getting Started with kunit_tool -=============================== - -If a kunitconfig is present at the root directory, all you have to do is: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run - -However, you most likely want to use it with the following options: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` - -- ``--timeout`` sets a maximum amount of time to allow tests to run. -- ``--jobs`` sets the number of threads to use to build the kernel. - -.. note:: - This command will work even without a .kunitconfig file: if no - .kunitconfig is present, a default one will be used instead. - -If you wish to use a different .kunitconfig file (such as one provided for -testing a particular subsystem), you can pass it as an option. - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run --kunitconfig=fs/ext4/.kunitconfig - -For a list of all the flags supported by kunit_tool, you can run: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run --help - -Configuring, Building, and Running Tests -======================================== - -It's also possible to run just parts of the KUnit build process independently, -which is useful if you want to make manual changes to part of the process. - -A .config can be generated from a .kunitconfig by using the ``config`` argument -when running kunit_tool: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py config - -Similarly, if you just want to build a KUnit kernel from the current .config, -you can use the ``build`` argument: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py build - -And, if you already have a built UML kernel with built-in KUnit tests, you can -run the kernel and display the test results with the ``exec`` argument: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py exec - -The ``run`` command which is discussed above is equivalent to running all three -of these in sequence. - -All of these commands accept a number of optional command-line arguments. The -``--help`` flag will give a complete list of these, or keep reading this page -for a guide to some of the more useful ones. - -Parsing Test Results -==================== - -KUnit tests output their results in TAP (Test Anything Protocol) format. -kunit_tool will, when running tests, parse this output and print a summary -which is much more pleasant to read. If you wish to look at the raw test -results in TAP format, you can pass the ``--raw_output`` argument. - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run --raw_output - -The raw output from test runs may contain other, non-KUnit kernel log -lines. You can see just KUnit output with ``--raw_output=kunit``: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run --raw_output=kunit - -If you have KUnit results in their raw TAP format, you can parse them and print -the human-readable summary with the ``parse`` command for kunit_tool. This -accepts a filename for an argument, or will read from standard input. - -.. code-block:: bash - - # Reading from a file - ./tools/testing/kunit/kunit.py parse /var/log/dmesg - # Reading from stdin - dmesg | ./tools/testing/kunit/kunit.py parse - -This is very useful if you wish to run tests in a configuration not supported -by kunit_tool (such as on real hardware, or an unsupported architecture). - -Filtering Tests -=============== - -It's possible to run only a subset of the tests built into a kernel by passing -a filter to the ``exec`` or ``run`` commands. For example, if you only wanted -to run KUnit resource tests, you could use: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run 'kunit-resource*' - -This uses the standard glob format for wildcards. - -Running Tests on QEMU -===================== - -kunit_tool supports running tests on QEMU as well as via UML (as mentioned -elsewhere). The default way of running tests on QEMU requires two flags: - -``--arch`` - Selects a collection of configs (Kconfig as well as QEMU configs - options, etc) that allow KUnit tests to be run on the specified - architecture in a minimal way; this is usually not much slower than - using UML. The architecture argument is the same as the name of the - option passed to the ``ARCH`` variable used by Kbuild. Not all - architectures are currently supported by this flag, but can be handled - by the ``--qemu_config`` discussed later. If ``um`` is passed (or this - this flag is ignored) the tests will run via UML. Non-UML architectures, - e.g. i386, x86_64, arm, um, etc. Non-UML run on QEMU. - -``--cross_compile`` - Specifies the use of a toolchain by Kbuild. The argument passed here is - the same passed to the ``CROSS_COMPILE`` variable used by Kbuild. As a - reminder this will be the prefix for the toolchain binaries such as gcc - for example ``sparc64-linux-gnu-`` if you have the sparc toolchain - installed on your system, or - ``$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-`` - if you have downloaded the microblaze toolchain from the 0-day website - to a directory in your home directory called ``toolchains``. - -In many cases it is likely that you may want to run an architecture which is -not supported by the ``--arch`` flag, or you may want to just run KUnit tests -on QEMU using a non-default configuration. For this use case, you can write -your own QemuConfig. These QemuConfigs are written in Python. They must have an -import line ``from ..qemu_config import QemuArchParams`` at the top of the file -and the file must contain a variable called ``QEMU_ARCH`` that has an instance -of ``QemuArchParams`` assigned to it. An example can be seen in -``tools/testing/kunit/qemu_configs/x86_64.py``. - -Once you have a QemuConfig you can pass it into kunit_tool using the -``--qemu_config`` flag; when used this flag replaces the ``--arch`` flag. If we -were to do this with the ``x86_64.py`` example from above, the invocation would -look something like this: - -.. code-block:: bash - - ./tools/testing/kunit/kunit.py run \ - --timeout=60 \ - --jobs=12 \ - --qemu_config=./tools/testing/kunit/qemu_configs/x86_64.py - -Other Useful Options -==================== - -kunit_tool has a number of other command-line arguments which can be useful -when adapting it to fit your environment or needs. - -Some of the more useful ones are: - -``--help`` - Lists all of the available options. Note that different commands - (``config``, ``build``, ``run``, etc) will have different supported - options. Place ``--help`` before the command to list common options, - and after the command for options specific to that command. - -``--build_dir`` - Specifies the build directory that kunit_tool will use. This is where - the .kunitconfig file is located, as well as where the .config and - compiled kernel will be placed. Defaults to ``.kunit``. - -``--make_options`` - Specifies additional options to pass to ``make`` when compiling a - kernel (with the ``build`` or ``run`` commands). For example, to enable - compiler warnings, you can pass ``--make_options W=1``. - -``--alltests`` - Builds a UML kernel with all config options enabled using ``make - allyesconfig``. This allows you to run as many tests as is possible, - but is very slow and prone to breakage as new options are added or - modified. In most cases, enabling all tests which have satisfied - dependencies by adding ``CONFIG_KUNIT_ALL_TESTS=1`` to your - .kunitconfig is preferable. - -There are several other options (and new ones are often added), so do check -``--help`` if you're looking for something not mentioned here. diff --git a/Documentation/dev-tools/kunit/run_wrapper.rst b/Documentation/dev-tools/kunit/run_wrapper.rst index cce203138fb7a2f63e7336c546b6b15414e06625..dafe8eb28d301597f71d7568575ef005a02b7723 100644 --- a/Documentation/dev-tools/kunit/run_wrapper.rst +++ b/Documentation/dev-tools/kunit/run_wrapper.rst @@ -1,8 +1,8 @@ .. SPDX-License-Identifier: GPL-2.0 -========================= -Run Tests with kunit_tool -========================= +============================= +Running tests with kunit_tool +============================= We can either run KUnit tests using kunit_tool or can run tests manually, and then use kunit_tool to parse the results. To run tests @@ -22,7 +22,7 @@ We should see the following: .. code-block:: - Generating .config... + Configuring KUnit Kernel ... Building KUnit kernel... Starting KUnit kernel... @@ -30,7 +30,7 @@ We may want to use the following options: .. code-block:: - ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all + ./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all` - ``--timeout`` sets a maximum amount of time for tests to run. - ``--jobs`` sets the number of threads to build the kernel. @@ -58,8 +58,8 @@ To view kunit_tool flags (optional command-line arguments), run: ./tools/testing/kunit/kunit.py run --help -Create a ``.kunitconfig`` File -=============================== +Creating a ``.kunitconfig`` file +================================ If we want to run a specific set of tests (rather than those listed in the KUnit ``defconfig``), we can provide Kconfig options in the @@ -98,8 +98,8 @@ have not included the options dependencies. The build dir needs to be set for ``make menuconfig`` to work, therefore by default use ``make O=.kunit menuconfig``. -Configure, Build, and Run Tests -=============================== +Configuring, building, and running tests +======================================== If we want to make manual changes to the KUnit build process, we can run part of the KUnit build process independently. @@ -125,11 +125,11 @@ argument: ./tools/testing/kunit/kunit.py exec -The ``run`` command discussed in section: **Run Tests with kunit_tool**, +The ``run`` command discussed in section: **Running tests with kunit_tool**, is equivalent to running the above three commands in sequence. -Parse Test Results -================== +Parsing test results +==================== KUnit tests output displays results in TAP (Test Anything Protocol) format. When running tests, kunit_tool parses this output and prints @@ -152,8 +152,8 @@ standard input. # Reading from stdin dmesg | ./tools/testing/kunit/kunit.py parse -Run Selected Test Suites -======================== +Filtering tests +=============== By passing a bash style glob filter to the ``exec`` or ``run`` commands, we can run a subset of the tests built into a kernel . For @@ -165,8 +165,10 @@ example: if we only want to run KUnit resource tests, use: This uses the standard glob format with wildcard characters. -Run Tests on qemu -================= +.. _kunit-on-qemu: + +Running tests on QEMU +===================== kunit_tool supports running tests on qemu as well as via UML. To run tests on qemu, by default it requires two flags: @@ -229,8 +231,8 @@ as --jobs=12 \ --qemu_config=./tools/testing/kunit/qemu_configs/x86_64.py -Command-Line Arguments -====================== +Running command-line arguments +============================== kunit_tool has a number of other command-line arguments which can be useful for our test environment. Below are the most commonly used @@ -249,14 +251,15 @@ command line arguments: compiling a kernel (using ``build`` or ``run`` commands). For example: to enable compiler warnings, we can pass ``--make_options W=1``. -- ``--alltests``: Builds a UML kernel with all config options enabled - using ``make allyesconfig``. This allows us to run as many tests as - possible. +- ``--alltests``: Enable a predefined set of options in order to build + as many tests as possible. + + .. note:: The list of enabled options can be found in + ``tools/testing/kunit/configs/all_tests.config``. - .. note:: It is slow and prone to breakage as new options are - added or modified. Instead, enable all tests - which have satisfied dependencies by adding - ``CONFIG_KUNIT_ALL_TESTS=y`` to your ``.kunitconfig``. + If you only want to enable all tests with otherwise satisfied + dependencies, instead add ``CONFIG_KUNIT_ALL_TESTS=y`` to your + ``.kunitconfig``. - ``--kunitconfig``: Specifies the path or the directory of the ``.kunitconfig`` file. For example: diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst index 867a4bba6bf69c7ccd4999129e595d3cb51f10a0..f4f504f1fb154f090564d39f286bee7a23c9966f 100644 --- a/Documentation/dev-tools/kunit/start.rst +++ b/Documentation/dev-tools/kunit/start.rst @@ -4,6 +4,10 @@ Getting Started =============== +This page contains an overview of the kunit_tool and KUnit framework, +teaching how to run existing tests and then how to write a simple test case, +and covers common problems users face when using KUnit for the first time. + Installing Dependencies ======================= KUnit has the same dependencies as the Linux kernel. As long as you can @@ -19,30 +23,53 @@ can run kunit_tool: ./tools/testing/kunit/kunit.py run -For more information on this wrapper, see: -Documentation/dev-tools/kunit/run_wrapper.rst. +.. note :: + You may see the following error: + "The source tree is not clean, please run 'make ARCH=um mrproper'" -Creating a ``.kunitconfig`` ---------------------------- + This happens because internally kunit.py specifies ``.kunit`` + (default option) as the build directory in the command ``make O=output/dir`` + through the argument ``--build_dir``. Hence, before starting an + out-of-tree build, the source tree must be clean. -By default, kunit_tool runs a selection of tests. However, you can specify which -unit tests to run by creating a ``.kunitconfig`` file with kernel config options -that enable only a specific set of tests and their dependencies. -The ``.kunitconfig`` file contains a list of kconfig options which are required -to run the desired targets. The ``.kunitconfig`` also contains any other test -specific config options, such as test dependencies. For example: the -``FAT_FS`` tests - ``FAT_KUNIT_TEST``, depends on -``FAT_FS``. ``FAT_FS`` can be enabled by selecting either ``MSDOS_FS`` -or ``VFAT_FS``. To run ``FAT_KUNIT_TEST``, the ``.kunitconfig`` has: + There is also the same caveat mentioned in the "Build directory for + the kernel" section of the :doc:`admin-guide `, + that is, its use, it must be used for all invocations of ``make``. + The good news is that it can indeed be solved by running + ``make ARCH=um mrproper``, just be aware that this will delete the + current configuration and all generated files. -.. code-block:: none +If everything worked correctly, you should see the following: - CONFIG_KUNIT=y - CONFIG_MSDOS_FS=y - CONFIG_FAT_KUNIT_TEST=y +.. code-block:: -1. A good starting point for the ``.kunitconfig`` is the KUnit default config. - You can generate it by running: + Configuring KUnit Kernel ... + Building KUnit Kernel ... + Starting KUnit Kernel ... + +The tests will pass or fail. + +.. note :: + Because it is building a lot of sources for the first time, + the ``Building KUnit Kernel`` step may take a while. + +For detailed information on this wrapper, see: +Documentation/dev-tools/kunit/run_wrapper.rst. + +Selecting which tests to run +---------------------------- + +By default, kunit_tool runs all tests reachable with minimal configuration, +that is, using default values for most of the kconfig options. However, +you can select which tests to run by: + +- `Customizing Kconfig`_ used to compile the kernel, or +- `Filtering tests by name`_ to select specifically which compiled tests to run. + +Customizing Kconfig +~~~~~~~~~~~~~~~~~~~ +A good starting point for the ``.kunitconfig`` is the KUnit default config. +If you didn't run ``kunit.py run`` yet, you can generate it by running: .. code-block:: bash @@ -54,48 +81,69 @@ or ``VFAT_FS``. To run ``FAT_KUNIT_TEST``, the ``.kunitconfig`` has: ``.kunitconfig`` lives in the ``--build_dir`` used by kunit.py, which is ``.kunit`` by default. -.. note :: +Before running the tests, kunit_tool ensures that all config options +set in ``.kunitconfig`` are set in the kernel ``.config``. It will warn +you if you have not included dependencies for the options used. + +There are many ways to customize the configurations: + +a. Edit ``.kunit/.kunitconfig``. The file should contain the list of kconfig + options required to run the desired tests, including their dependencies. You may want to remove CONFIG_KUNIT_ALL_TESTS from the ``.kunitconfig`` as it will enable a number of additional tests that you may not want. + If you need to run on an architecture other than UML see :ref:`kunit-on-qemu`. -2. You can then add any other Kconfig options, for example: +b. Enable additional kconfig options on top of ``.kunit/.kunitconfig``. + For example, to include the kernel's linked-list test you can run:: -.. code-block:: none + ./tools/testing/kunit/kunit.py run \ + --kconfig_add CONFIG_LIST_KUNIT_TEST=y - CONFIG_LIST_KUNIT_TEST=y +c. Provide the path of one or more .kunitconfig files from the tree. + For example, to run only ``FAT_FS`` and ``EXT4`` tests you can run:: -Before running the tests, kunit_tool ensures that all config options -set in ``.kunitconfig`` are set in the kernel ``.config``. It will warn -you if you have not included dependencies for the options used. + ./tools/testing/kunit/kunit.py run \ + --kunitconfig ./fs/fat/.kunitconfig \ + --kunitconfig ./fs/ext4/.kunitconfig -.. note :: - If you change the ``.kunitconfig``, kunit.py will trigger a rebuild of the +d. If you change the ``.kunitconfig``, kunit.py will trigger a rebuild of the ``.config`` file. But you can edit the ``.config`` file directly or with tools like ``make menuconfig O=.kunit``. As long as its a superset of ``.kunitconfig``, kunit.py won't overwrite your changes. -Running Tests (KUnit Wrapper) ------------------------------ -1. To make sure that everything is set up correctly, invoke the Python - wrapper from your kernel repository: -.. code-block:: bash +.. note :: - ./tools/testing/kunit/kunit.py run + To save a .kunitconfig after finding a satisfactory configuration:: -If everything worked correctly, you should see the following: + make savedefconfig O=.kunit + cp .kunit/defconfig .kunit/.kunitconfig -.. code-block:: +Filtering tests by name +~~~~~~~~~~~~~~~~~~~~~~~ +If you want to be more specific than Kconfig can provide, it is also possible +to select which tests to execute at boot-time by passing a glob filter +(read instructions regarding the pattern in the manpage :manpage:`glob(7)`). +If there is a ``"."`` (period) in the filter, it will be interpreted as a +separator between the name of the test suite and the test case, +otherwise, it will be interpreted as the name of the test suite. +For example, let's assume we are using the default config: - Generating .config ... - Building KUnit Kernel ... - Starting KUnit Kernel ... +a. inform the name of a test suite, like ``"kunit_executor_test"``, + to run every test case it contains:: -The tests will pass or fail. + ./tools/testing/kunit/kunit.py run "kunit_executor_test" -.. note :: - Because it is building a lot of sources for the first time, the - ``Building KUnit kernel`` may take a while. +b. inform the name of a test case prefixed by its test suite, + like ``"example.example_simple_test"``, to run specifically that test case:: + + ./tools/testing/kunit/kunit.py run "example.example_simple_test" + +c. use wildcard characters (``*?[``) to run any test case that matches the pattern, + like ``"*.*64*"`` to run test cases containing ``"64"`` in the name inside + any test suite:: + + ./tools/testing/kunit/kunit.py run "*.*64*" Running Tests without the KUnit Wrapper ======================================= @@ -217,7 +265,7 @@ Now we are ready to write the test cases. obj-$(CONFIG_MISC_EXAMPLE_TEST) += example_test.o -4. Add the following lines to ``.kunitconfig``: +4. Add the following lines to ``.kunit/.kunitconfig``: .. code-block:: none @@ -254,7 +302,5 @@ Next Steps examples. * Documentation/dev-tools/kunit/api/index.rst - KUnit APIs used for testing. -* Documentation/dev-tools/kunit/kunit-tool.rst - kunit_tool helper - script. * Documentation/dev-tools/kunit/faq.rst - KUnit common questions and answers. diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst index 44158eecb51e69ac9a23931031545644a968607f..2737863ef365321a8b9c8db6494be37fcf58fa95 100644 --- a/Documentation/dev-tools/kunit/usage.rst +++ b/Documentation/dev-tools/kunit/usage.rst @@ -165,6 +165,8 @@ built as a module). For more information, see Documentation/dev-tools/kunit/api/test.rst. +.. _kunit-on-non-uml: + Writing Tests For Other Architectures ------------------------------------- @@ -544,8 +546,6 @@ By reusing the same ``cases`` array from above, we can write the test as a {} }; -.. _kunit-on-non-uml: - Exiting Early on Failed Expectations ------------------------------------ diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 1eaccf135b3018c900f5ac1f943bcdc7aa1542a3..bf2d8a8ced77f2474c9e8163dcdc835b1629dbed 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -75,3 +75,6 @@ always-$(CHECK_DT_BINDING) += $(patsubst $(srctree)/$(src)/%.yaml,%.example.dtb, # build artifacts here before they are processed by scripts/Makefile.clean clean-files = $(shell find $(obj) \( -name '*.example.dts' -o \ -name '*.example.dtb' \) -delete 2>/dev/null) + +dt_compatible_check: $(obj)/processed-schema.json + $(Q)$(srctree)/scripts/dtc/dt-extract-compatibles $(srctree) | xargs dt-check-compatible -v -s $< diff --git a/Documentation/devicetree/bindings/arm/actions.yaml b/Documentation/devicetree/bindings/arm/actions.yaml index 02dc72c976456bb790d5267ec469d1cf49a82967..e012f612f0390ba306665c99c7648d0d532c035d 100644 --- a/Documentation/devicetree/bindings/arm/actions.yaml +++ b/Documentation/devicetree/bindings/arm/actions.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/actions.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Actions Semi platforms device tree bindings +title: Actions Semi platforms maintainers: - Andreas Färber diff --git a/Documentation/devicetree/bindings/arm/airoha.yaml b/Documentation/devicetree/bindings/arm/airoha.yaml index fc19b1a6f37baca5c161a83fe8918fe61c11f4f4..3292c669ee11d5e143408206bec20151857f4bf8 100644 --- a/Documentation/devicetree/bindings/arm/airoha.yaml +++ b/Documentation/devicetree/bindings/arm/airoha.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/airoha.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Airoha SoC based Platforms Device Tree Bindings +title: Airoha SoC based Platforms maintainers: - Felix Fietkau diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml index e6de1d7f516c38a752d3dc6a365fac56c6f8bbae..3eee03aa935c7e83479e14a83cdc4e174557fe08 100644 --- a/Documentation/devicetree/bindings/arm/altera.yaml +++ b/Documentation/devicetree/bindings/arm/altera.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/altera.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Altera's SoCFPGA platform device tree bindings +title: Altera's SoCFPGA platform maintainers: - Dinh Nguyen diff --git a/Documentation/devicetree/bindings/arm/amazon,al.yaml b/Documentation/devicetree/bindings/arm/amazon,al.yaml index 0f03135d91b6165888a5a94f78fec6b5be57f919..37dbb4768e5b164bb265308d8acec85dab793acc 100644 --- a/Documentation/devicetree/bindings/arm/amazon,al.yaml +++ b/Documentation/devicetree/bindings/arm/amazon,al.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/amazon,al.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Amazon's Annapurna Labs Alpine Platform Device Tree Bindings +title: Amazon's Annapurna Labs Alpine Platform maintainers: - Hanna Hawa diff --git a/Documentation/devicetree/bindings/arm/amlogic.yaml b/Documentation/devicetree/bindings/arm/amlogic.yaml index 61a6cabb375be6a241bd29255123644b423c4a02..9fda2436c618485fa23aa44841ec925d388966e4 100644 --- a/Documentation/devicetree/bindings/arm/amlogic.yaml +++ b/Documentation/devicetree/bindings/arm/amlogic.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/amlogic.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Amlogic MesonX device tree bindings +title: Amlogic MesonX maintainers: - Kevin Hilman @@ -120,6 +120,7 @@ properties: - enum: - amlogic,q200 - amlogic,q201 + - azw,gt1-ultimate - khadas,vim2 - kingnovel,r-box-pro - libretech,aml-s912-pc @@ -136,6 +137,7 @@ properties: - enum: - amlogic,s400 - jethome,jethub-j100 + - jethome,jethub-j110 - const: amlogic,a113d - const: amlogic,meson-axg diff --git a/Documentation/devicetree/bindings/arm/amlogic/amlogic,meson-gx-ao-secure.yaml b/Documentation/devicetree/bindings/arm/amlogic/amlogic,meson-gx-ao-secure.yaml index 6cc74523ebfd3355cc2d76ca1c7cacac10819007..1748f1605cc701b55ac97687c27c23e0e6716465 100644 --- a/Documentation/devicetree/bindings/arm/amlogic/amlogic,meson-gx-ao-secure.yaml +++ b/Documentation/devicetree/bindings/arm/amlogic/amlogic,meson-gx-ao-secure.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson Firmware registers Interface maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Meson SoCs have a register bank with status and data shared with the diff --git a/Documentation/devicetree/bindings/arm/apple.yaml b/Documentation/devicetree/bindings/arm/apple.yaml index 8d93e8a6cc1849e20a0c9e7d8a97e5df7921dfe4..7262f3c098678cdd61dd6a07a180474903550822 100644 --- a/Documentation/devicetree/bindings/arm/apple.yaml +++ b/Documentation/devicetree/bindings/arm/apple.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/apple.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Apple ARM Machine Device Tree Bindings +title: Apple ARM Machine maintainers: - Hector Martin diff --git a/Documentation/devicetree/bindings/arm/arm,cci-400.yaml b/Documentation/devicetree/bindings/arm/arm,cci-400.yaml index 1706134b75a3733faca830ef323ac0ec4194566c..d28303d909e11ba92dab05933c3207f4d3f7fe0d 100644 --- a/Documentation/devicetree/bindings/arm/arm,cci-400.yaml +++ b/Documentation/devicetree/bindings/arm/arm,cci-400.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,cci-400.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM CCI Cache Coherent Interconnect Device Tree Binding +title: ARM CCI Cache Coherent Interconnect maintainers: - Lorenzo Pieralisi diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-catu.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-catu.yaml index d783d9276124862773820cab304de0aa845f363e..2bae06eed693a03d8a58645f63f08e1b28b62da7 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-catu.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-catu.yaml @@ -61,6 +61,9 @@ properties: maxItems: 1 description: Address translation error interrupt + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml index 72ffe4d1e94884a2531a9bd2ac484dde63061ee6..0c5b875cb654bc286edd75981d46946e7b6db421 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-cti.yaml @@ -98,6 +98,9 @@ properties: base cti node if compatible string arm,coresight-cti-v8-arch is used, or may appear in a trig-conns child node when appropriate. + power-domains: + maxItems: 1 + arm,cti-ctm-id: $ref: /schemas/types.yaml#/definitions/uint32 description: diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml index 1eeedc22857c0c785ca1d7d9b9d34ee0217b00d7..44a1041cb0fc958d2eb86219c339bcc76953cf72 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-funnel.yaml @@ -54,6 +54,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml index a26ed9214e0083467e8b793845c112e9481f7e56..03792e9bd97a03f9ba14f6882aea040af2bf3afe 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-dynamic-replicator.yaml @@ -54,6 +54,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + qcom,replicator-loses-context: type: boolean description: diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-etb10.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-etb10.yaml index fd06ede26cebaf9508fec7e5007978c31f5a163d..90679788e0bf37e2458d0c18aff65405019ecd51 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-etb10.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-etb10.yaml @@ -54,6 +54,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml index e0377ce48537d49525db2162066469bfc25164c2..01200f67504a53c8c501dff7edfc0dd9a3a24b7d 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-etm.yaml @@ -73,6 +73,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + arm,coresight-loses-context-with-cpu: type: boolean description: diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-static-funnel.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-static-funnel.yaml index 374083956b20d81cb07d87af275773cdaaba8049..cc8c3baa79b457280b3f7907f888dfee03ec8192 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-static-funnel.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-static-funnel.yaml @@ -27,6 +27,9 @@ properties: compatible: const: arm,coresight-static-funnel + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-static-replicator.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-static-replicator.yaml index a34d8583830cb2b70d4ed0dd4b27eddd57ae0128..1892a091ac358a7b5432e6c9cfc17f835d09ca12 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-static-replicator.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-static-replicator.yaml @@ -27,6 +27,9 @@ properties: compatible: const: arm,coresight-static-replicator + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml index 905008faa012fd3174c63b675ff1555ef41280e6..378380c3f5aab50eff498e3b5c5e95a55bf41140 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-stm.yaml @@ -61,6 +61,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + out-ports: $ref: /schemas/graph.yaml#/properties/ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml index 3463b6e53aef3bb748b3abbef7be9de5c76b2bd0..cb8dceaca70efc341a3471b2debb045d702ee797 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-tmc.yaml @@ -55,6 +55,12 @@ properties: - const: apb_pclk - const: atclk + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + arm,buffer-size: $ref: /schemas/types.yaml#/definitions/uint32 deprecated: true diff --git a/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml b/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml index e80d48200c3788795ab2049b31dc7578862f912a..61a0cdc27745fc0f5c718da1fee434c9c5c6d6df 100644 --- a/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml +++ b/Documentation/devicetree/bindings/arm/arm,coresight-tpiu.yaml @@ -54,6 +54,9 @@ properties: - const: apb_pclk - const: atclk + power-domains: + maxItems: 1 + in-ports: $ref: /schemas/graph.yaml#/properties/ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/arm/arm,corstone1000.yaml b/Documentation/devicetree/bindings/arm/arm,corstone1000.yaml index a77f882238019fd0c6ef91fa08c4eab2d6e4967e..693f3fe7be60ff1df0f907918a7a5176fe82ba6b 100644 --- a/Documentation/devicetree/bindings/arm/arm,corstone1000.yaml +++ b/Documentation/devicetree/bindings/arm/arm,corstone1000.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,corstone1000.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM Corstone1000 Device Tree Bindings +title: ARM Corstone1000 maintainers: - Vishnu Banavath diff --git a/Documentation/devicetree/bindings/arm/arm,embedded-trace-extension.yaml b/Documentation/devicetree/bindings/arm/arm,embedded-trace-extension.yaml index 5f07fb166c56b289a982c6a1bd158bb4367adb18..108460627d9ac660fe205c6ca0475b6e317c03d0 100644 --- a/Documentation/devicetree/bindings/arm/arm,embedded-trace-extension.yaml +++ b/Documentation/devicetree/bindings/arm/arm,embedded-trace-extension.yaml @@ -33,6 +33,9 @@ properties: Handle to the cpu this ETE is bound to. $ref: /schemas/types.yaml#/definitions/phandle + power-domains: + maxItems: 1 + out-ports: description: | Output connections from the ETE to legacy CoreSight trace bus. diff --git a/Documentation/devicetree/bindings/arm/arm,integrator.yaml b/Documentation/devicetree/bindings/arm/arm,integrator.yaml index 528eee64290ab514fe5baa356ef1af86c8cd012b..98ff5698ae1f1f9c78840e20f73bef081609e50c 100644 --- a/Documentation/devicetree/bindings/arm/arm,integrator.yaml +++ b/Documentation/devicetree/bindings/arm/arm,integrator.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,integrator.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM Integrator Boards Device Tree Bindings +title: ARM Integrator Boards maintainers: - Linus Walleij diff --git a/Documentation/devicetree/bindings/arm/arm,realview.yaml b/Documentation/devicetree/bindings/arm/arm,realview.yaml index 4f9b21f49e84619b59fd8c4e8ebd8d4a11eca6d4..8d3ed2e4ed315ca5359756fb39c7673ec749886a 100644 --- a/Documentation/devicetree/bindings/arm/arm,realview.yaml +++ b/Documentation/devicetree/bindings/arm/arm,realview.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,realview.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM RealView Boards Device Tree Bindings +title: ARM RealView Boards maintainers: - Linus Walleij diff --git a/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml b/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..491eef1e1b10da532f20f615482694ff59b0399c --- /dev/null +++ b/Documentation/devicetree/bindings/arm/arm,versatile-sysreg.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/arm,versatile-sysreg.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Arm Versatile system registers + +maintainers: + - Linus Walleij + +description: + This is a system control registers block, providing multiple low level + platform functions like board detection and identification, software + interrupt generation, MMC and NOR Flash control, etc. + +properties: + compatible: + items: + - const: arm,versatile-sysreg + - const: syscon + - const: simple-mfd + + reg: + maxItems: 1 + + panel: + type: object + +required: + - compatible + - reg + +additionalProperties: false +... diff --git a/Documentation/devicetree/bindings/arm/arm,versatile.yaml b/Documentation/devicetree/bindings/arm/arm,versatile.yaml index 34b437c727517ec14bf357177b79112327439fdb..13e52ba920601854bdae06dd5fb2c778a8c866bb 100644 --- a/Documentation/devicetree/bindings/arm/arm,versatile.yaml +++ b/Documentation/devicetree/bindings/arm/arm,versatile.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,versatile.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM Versatile Boards Device Tree Bindings +title: ARM Versatile Boards maintainers: - Linus Walleij diff --git a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml index a4b4452afc1dc520a2dfc1b8951f4ae1fe3969a7..eec190a96225741e9f33be0379c3a1446b294b97 100644 --- a/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml +++ b/Documentation/devicetree/bindings/arm/arm,vexpress-juno.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/arm,vexpress-juno.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM Versatile Express and Juno Boards Device Tree Bindings +title: ARM Versatile Express and Juno Boards maintainers: - Sudeep Holla diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml index 1895ce9de461dfffee8ca8588e13c7d940b3cd05..217a1d674863fbf076f41996e9915298f36617d6 100644 --- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml +++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml @@ -29,6 +29,7 @@ properties: - description: AST2500 based boards items: - enum: + - amd,daytonax-bmc - amd,ethanolx-bmc - ampere,mtjade-bmc - aspeed,ast2500-evb @@ -69,6 +70,7 @@ properties: - description: AST2600 based boards items: - enum: + - ampere,mtmitchell-bmc - aspeed,ast2600-evb - aspeed,ast2600-evb-a1 - facebook,bletchley-bmc diff --git a/Documentation/devicetree/bindings/arm/atmel-at91.yaml b/Documentation/devicetree/bindings/arm/atmel-at91.yaml index 2b7848bb7769547f3a2ab788ca38b8b47531c4c9..2224b18801a1993a2607e2cd9aafea78d750635e 100644 --- a/Documentation/devicetree/bindings/arm/atmel-at91.yaml +++ b/Documentation/devicetree/bindings/arm/atmel-at91.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/atmel-at91.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Atmel AT91 device tree bindings. +title: Atmel AT91. maintainers: - Alexandre Belloni @@ -127,6 +127,13 @@ properties: - const: atmel,sama5d3 - const: atmel,sama5 + - description: Microchip SAMA5D3 Ethernet Development System Board + items: + - const: microchip,sama5d3-eds + - const: atmel,sama5d36 + - const: atmel,sama5d3 + - const: atmel,sama5 + - description: CalAmp LMU5000 board items: - const: calamp,lmu5000 diff --git a/Documentation/devicetree/bindings/arm/axxia.yaml b/Documentation/devicetree/bindings/arm/axxia.yaml index e0d2bb71cf502b1e1697a0b9db6a95660df79b44..d60907e43efcb50bf7a9ed67409faac50b602f18 100644 --- a/Documentation/devicetree/bindings/arm/axxia.yaml +++ b/Documentation/devicetree/bindings/arm/axxia.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/axxia.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Axxia AXM55xx device tree bindings +title: Axxia AXM55xx maintainers: - Anders Berg diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml b/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml deleted file mode 100644 index 9b745531ff04948a024901c06b3a39e321db52c1..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/arm/bcm/brcm,bcm4908.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Broadcom BCM4908 device tree bindings - -description: - Broadcom BCM4906 / BCM4908 / BCM49408 Wi-Fi/network SoCs with Brahma CPUs. - -maintainers: - - Rafał Miłecki - -properties: - $nodename: - const: '/' - compatible: - oneOf: - - description: BCM4906 based boards - items: - - enum: - - netgear,r8000p - - tplink,archer-c2300-v1 - - const: brcm,bcm4906 - - const: brcm,bcm4908 - - - description: BCM4908 based boards - items: - - enum: - - asus,gt-ac5300 - - netgear,raxe500 - - const: brcm,bcm4908 - - - description: BCM49408 based boards - items: - - const: brcm,bcm49408 - - const: brcm,bcm4908 - -additionalProperties: true - -... diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml b/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml index 324e591043609d6942636664bf86c0fca757ea33..84866e29cab0e6b50c959b37a5f225f107434a55 100644 --- a/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml +++ b/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml @@ -15,6 +15,7 @@ maintainers: - William Zhang - Anand Gore - Kursad Oney + - Rafał Miłecki properties: $nodename: @@ -28,6 +29,30 @@ properties: - const: brcm,bcm47622 - const: brcm,bcmbca + - description: BCM4906 based boards + items: + - enum: + - netgear,r8000p + - tplink,archer-c2300-v1 + - const: brcm,bcm4906 + - const: brcm,bcm4908 + - const: brcm,bcmbca + + - description: BCM4908 based boards + items: + - enum: + - asus,gt-ac5300 + - brcm,bcm94908 + - netgear,raxe500 + - const: brcm,bcm4908 + - const: brcm,bcmbca + + - description: BCM49408 based boards + items: + - const: brcm,bcm49408 + - const: brcm,bcm4908 + - const: brcm,bcmbca + - description: BCM4912 based boards items: - enum: diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt index 104cc9b41df46bebb2d8db5cdd4fb7843a677f9c..071421dbc4d0f4bf6707f1eb477fda7fc4aed28b 100644 --- a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt +++ b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt @@ -187,15 +187,8 @@ Required properties: Sequencer DRAM parameters and control registers. Used for Self-Refresh Power-Down (SRPD), among other things. -Required properties: -- compatible : should contain one of these - "brcm,brcmstb-memc-ddr-rev-b.2.1" - "brcm,brcmstb-memc-ddr-rev-b.2.2" - "brcm,brcmstb-memc-ddr-rev-b.2.3" - "brcm,brcmstb-memc-ddr-rev-b.3.0" - "brcm,brcmstb-memc-ddr-rev-b.3.1" - "brcm,brcmstb-memc-ddr" -- reg : the MEMC DDR register range +See Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml for a +full list of supported compatible strings and properties. Example: diff --git a/Documentation/devicetree/bindings/arm/bitmain.yaml b/Documentation/devicetree/bindings/arm/bitmain.yaml index 90ba02be48ce5b9ce13a2ae5dc681ab79660b0f2..55a5a570b5bc30456d6d06fd2b0f248209fed9b0 100644 --- a/Documentation/devicetree/bindings/arm/bitmain.yaml +++ b/Documentation/devicetree/bindings/arm/bitmain.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/bitmain.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Bitmain platform device tree bindings +title: Bitmain platform maintainers: - Manivannan Sadhasivam diff --git a/Documentation/devicetree/bindings/arm/calxeda.yaml b/Documentation/devicetree/bindings/arm/calxeda.yaml index 46f78addebb0f35445df83c6c3b37dc726b1a0d0..3e9f5e1d862ea3e3c0e7455c8ebdb440a179215a 100644 --- a/Documentation/devicetree/bindings/arm/calxeda.yaml +++ b/Documentation/devicetree/bindings/arm/calxeda.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/calxeda.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Calxeda Platforms Device Tree Bindings +title: Calxeda Platforms maintainers: - Rob Herring diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml index a07c5bac7c4647568c39cbbbee70bb2bcd2395a9..5c13b73e4d57693a55338d4e18549f72e9407e97 100644 --- a/Documentation/devicetree/bindings/arm/cpus.yaml +++ b/Documentation/devicetree/bindings/arm/cpus.yaml @@ -174,6 +174,7 @@ properties: - nvidia,tegra194-carmel - qcom,krait - qcom,kryo + - qcom,kryo240 - qcom,kryo250 - qcom,kryo260 - qcom,kryo280 diff --git a/Documentation/devicetree/bindings/arm/digicolor.yaml b/Documentation/devicetree/bindings/arm/digicolor.yaml index a35de3c9e28474113f3f4f7addf466c6babc27d5..0cf9ddaa527e9e652736f78d03024c12556d20e3 100644 --- a/Documentation/devicetree/bindings/arm/digicolor.yaml +++ b/Documentation/devicetree/bindings/arm/digicolor.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/digicolor.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Conexant Digicolor Platforms Device Tree Bindings +title: Conexant Digicolor Platforms maintainers: - Baruch Siach diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml index 7431579ab0e8545dbbfc59c8bb00882ca52c2677..fbfc4f99c01e97bbf1187b83437411166cdaa9e5 100644 --- a/Documentation/devicetree/bindings/arm/fsl.yaml +++ b/Documentation/devicetree/bindings/arm/fsl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/fsl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Freescale i.MX Platforms Device Tree Bindings +title: Freescale i.MX Platforms maintainers: - Shawn Guo @@ -554,8 +554,7 @@ properties: - engicam,imx6ul-isiot # Engicam Is.IoT MX6UL eMMC/NAND Starter kit - fsl,imx6ul-14x14-evk # i.MX6 UltraLite 14x14 EVK Board - karo,imx6ul-tx6ul # Ka-Ro electronics TXUL-0010 Module - - kontron,imx6ul-n6310-som # Kontron N6310 SOM - - kontron,imx6ul-n6311-som # Kontron N6311 SOM + - kontron,sl-imx6ul # Kontron SL i.MX6UL SoM - prt,prti6g # Protonic PRTI6G Board - technexion,imx6ul-pico-dwarf # TechNexion i.MX6UL Pico-Dwarf - technexion,imx6ul-pico-hobbit # TechNexion i.MX6UL Pico-Hobbit @@ -591,23 +590,17 @@ properties: - const: phytec,imx6ul-pcl063 # PHYTEC phyCORE-i.MX 6UL - const: fsl,imx6ul - - description: Kontron N6310 S Board + - description: Kontron BL i.MX6UL (N631X S) Board items: - - const: kontron,imx6ul-n6310-s - - const: kontron,imx6ul-n6310-som + - const: kontron,bl-imx6ul # Kontron BL i.MX6UL Carrier Board + - const: kontron,sl-imx6ul # Kontron SL i.MX6UL SoM - const: fsl,imx6ul - - description: Kontron N6311 S Board + - description: Kontron BL i.MX6UL 43 (N631X S 43) Board items: - - const: kontron,imx6ul-n6311-s - - const: kontron,imx6ul-n6311-som - - const: fsl,imx6ul - - - description: Kontron N6310 S 43 Board - items: - - const: kontron,imx6ul-n6310-s-43 - - const: kontron,imx6ul-n6310-s - - const: kontron,imx6ul-n6310-som + - const: kontron,bl-imx6ul-43 # Kontron BL i.MX6UL Carrier Board with 4.3" Display + - const: kontron,bl-imx6ul # Kontron BL i.MX6UL Carrier Board + - const: kontron,sl-imx6ul # Kontron SL i.MX6UL SoM - const: fsl,imx6ul - description: TQ-Systems TQMa6UL1 SoM on MBa6ULx board @@ -637,7 +630,7 @@ properties: - enum: - fsl,imx6ull-14x14-evk # i.MX6 UltraLiteLite 14x14 EVK Board - joz,jozacp # JOZ Access Point - - kontron,imx6ull-n6411-som # Kontron N6411 SOM + - kontron,sl-imx6ull # Kontron SL i.MX6ULL SoM - myir,imx6ull-mys-6ulx-eval # MYiR Tech iMX6ULL Evaluation Board - toradex,colibri-imx6ull # Colibri iMX6ULL Modules - toradex,colibri-imx6ull-emmc # Colibri iMX6ULL 1GB (eMMC) Module @@ -698,10 +691,10 @@ properties: - const: toradex,colibri-imx6ull-wifi # Colibri iMX6ULL Wi-Fi / BT Module - const: fsl,imx6ull - - description: Kontron N6411 S Board + - description: Kontron BL i.MX6ULL (N6411 S) Board items: - - const: kontron,imx6ull-n6411-s - - const: kontron,imx6ull-n6411-som + - const: kontron,bl-imx6ull # Kontron BL i.MX6ULL Carrier Board + - const: kontron,sl-imx6ull # Kontron SL i.MX6ULL SoM - const: fsl,imx6ull - description: TQ Systems TQMa6ULLx SoM on MBa6ULx board @@ -825,13 +818,15 @@ properties: - emtrion,emcon-mx8mm-avari # emCON-MX8MM SoM on Avari Base - fsl,imx8mm-ddr4-evk # i.MX8MM DDR4 EVK Board - fsl,imx8mm-evk # i.MX8MM EVK Board + - gateworks,imx8mm-gw7904 - gw,imx8mm-gw71xx-0x # i.MX8MM Gateworks Development Kit - gw,imx8mm-gw72xx-0x # i.MX8MM Gateworks Development Kit - gw,imx8mm-gw73xx-0x # i.MX8MM Gateworks Development Kit - gw,imx8mm-gw7901 # i.MX8MM Gateworks Board - gw,imx8mm-gw7902 # i.MX8MM Gateworks Board - gw,imx8mm-gw7903 # i.MX8MM Gateworks Board - - kontron,imx8mm-n801x-som # i.MX8MM Kontron SL (N801X) SOM + - kontron,imx8mm-sl # i.MX8MM Kontron SL (N801X) SOM + - kontron,imx8mm-osm-s # i.MX8MM Kontron OSM-S (N802X) SOM - menlo,mx8menlo # i.MX8MM Menlo board with Verdin SoM - toradex,verdin-imx8mm # Verdin iMX8M Mini Modules - toradex,verdin-imx8mm-nonwifi # Verdin iMX8M Mini Modules without Wi-Fi / BT @@ -850,8 +845,14 @@ properties: - description: Kontron BL i.MX8MM (N801X S) Board items: - - const: kontron,imx8mm-n801x-s - - const: kontron,imx8mm-n801x-som + - const: kontron,imx8mm-bl + - const: kontron,imx8mm-sl + - const: fsl,imx8mm + + - description: Kontron BL i.MX8MM OSM-S (N802X S) Board + items: + - const: kontron,imx8mm-bl-osm-s + - const: kontron,imx8mm-osm-s - const: fsl,imx8mm - description: Toradex Boards with Verdin iMX8M Mini Modules @@ -936,6 +937,13 @@ properties: - toradex,verdin-imx8mp-wifi # Verdin iMX8M Plus Wi-Fi / BT Modules - const: fsl,imx8mp + - description: Avnet (MSC Branded) Boards with SM2S i.MX8M Plus Modules + items: + - const: avnet,sm2s-imx8mp-14N0600E-ep1 # SM2S-IMX8PLUS-14N0600E on SM2-MB-EP1 Carrier Board + - const: avnet,sm2s-imx8mp-14N0600E # 14N0600E variant of SM2S-IMX8PLUS SoM + - const: avnet,sm2s-imx8mp # SM2S-IMX8PLUS SoM + - const: fsl,imx8mp + - description: Engicam i.Core MX8M Plus SoM based boards items: - enum: @@ -1034,6 +1042,12 @@ properties: - toradex,colibri-imx8x # Colibri iMX8X Modules - const: fsl,imx8qxp + - description: i.MX8DXL based Boards + items: + - enum: + - fsl,imx8dxl-evk # i.MX8DXL EVK Board + - const: fsl,imx8dxl + - description: i.MX8QXP Boards with Toradex Coilbri iMX8X Modules items: - enum: diff --git a/Documentation/devicetree/bindings/arm/intel,keembay.yaml b/Documentation/devicetree/bindings/arm/intel,keembay.yaml index 107e686ab207fe1070cf4c178aa1b3a3b577147c..53d2ce02b207755bd813fac8d9c2db5da2b3cbd2 100644 --- a/Documentation/devicetree/bindings/arm/intel,keembay.yaml +++ b/Documentation/devicetree/bindings/arm/intel,keembay.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/intel,keembay.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Keem Bay platform device tree bindings +title: Keem Bay platform maintainers: - Paul J. Murphy diff --git a/Documentation/devicetree/bindings/arm/intel,socfpga.yaml b/Documentation/devicetree/bindings/arm/intel,socfpga.yaml index 61a454a40e87a373997a68df7e40e5aa672171c2..4b4dcf551eb66854c04f85a96cf15f030612e104 100644 --- a/Documentation/devicetree/bindings/arm/intel,socfpga.yaml +++ b/Documentation/devicetree/bindings/arm/intel,socfpga.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/intel,socfpga.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel SoCFPGA platform device tree bindings +title: Intel SoCFPGA platform maintainers: - Dinh Nguyen diff --git a/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml b/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml index 230bffeec0e50a1eccdc7bce069472ad7a19268b..553dcbc70e35f9b6012c6e19feb76a066edc7866 100644 --- a/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml +++ b/Documentation/devicetree/bindings/arm/intel-ixp4xx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/intel-ixp4xx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel IXP4xx Device Tree Bindings +title: Intel IXP4xx maintainers: - Linus Walleij diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt index f6d6642d81c0758eb61e5c8c9433665194950da8..29fa93dad52bac2ba1ea978e6091da1e4fc87494 100644 --- a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt +++ b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt @@ -1,21 +1,3 @@ -Marvell Armada 37xx Platforms Device Tree Bindings --------------------------------------------------- - -Boards using a SoC of the Marvell Armada 37xx family must carry the -following root node property: - - - compatible: must contain "marvell,armada3710" - -In addition, boards using the Marvell Armada 3720 SoC shall have the -following property before the previous one: - - - compatible: must contain "marvell,armada3720" - -Example: - -compatible = "marvell,armada-3720-db", "marvell,armada3720", "marvell,armada3710"; - - Power management ---------------- @@ -48,11 +30,3 @@ avs: avs@11500 { compatible = "marvell,armada-3700-avs", "syscon"; reg = <0x11500 0x40>; } - - -CZ.NIC's Turris Mox SOHO router Device Tree Bindings ----------------------------------------------------- - -Required root node property: - - - compatible: must contain "cznic,turris-mox" diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6905d29f31088a887602e3a2d3e41be5802a5097 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/arm/marvell/armada-37xx.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Marvell Armada 37xx Platforms + +maintainers: + - Robert Marko + +properties: + $nodename: + const: '/' + compatible: + oneOf: + - description: Armada 3720 SoC boards + items: + - enum: + - cznic,turris-mox + - globalscale,espressobin + - marvell,armada-3720-db + - methode,edpu + - methode,udpu + - const: marvell,armada3720 + - const: marvell,armada3710 + + - description: Globalscale Espressobin boards + items: + - enum: + - globalscale,espressobin-emmc + - globalscale,espressobin-ultra + - globalscale,espressobin-v7 + - const: globalscale,espressobin + - const: marvell,armada3720 + - const: marvell,armada3710 + + - description: Globalscale Espressobin V7 boards + items: + - enum: + - globalscale,espressobin-v7-emmc + - const: globalscale,espressobin-v7 + - const: globalscale,espressobin + - const: marvell,armada3720 + - const: marvell,armada3710 + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/arm/mediatek.yaml b/Documentation/devicetree/bindings/arm/mediatek.yaml index 07c0ea94e85080210de3a0bd3f6a9df37a870261..d76ce4c3819db8b6196cc05057ae49d7314cd75d 100644 --- a/Documentation/devicetree/bindings/arm/mediatek.yaml +++ b/Documentation/devicetree/bindings/arm/mediatek.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/mediatek.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek SoC based Platforms Device Tree Bindings +title: MediaTek SoC based Platforms maintainers: - Sean Wang diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.yaml index 8681b785ed6d1f4cdeb69d91536cd764eb8125e9..1d7c837d93788f7b44083f409e6e6e4d03d86bbf 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.yaml +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.yaml @@ -23,6 +23,7 @@ properties: - mediatek,mt2701-infracfg - mediatek,mt2712-infracfg - mediatek,mt6765-infracfg + - mediatek,mt6795-infracfg - mediatek,mt6779-infracfg_ao - mediatek,mt6797-infracfg - mediatek,mt7622-infracfg @@ -60,6 +61,7 @@ if: enum: - mediatek,mt2701-infracfg - mediatek,mt2712-infracfg + - mediatek,mt6795-infracfg - mediatek,mt7622-infracfg - mediatek,mt7986-infracfg - mediatek,mt8135-infracfg diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml index 6ad023eec193a6d69cdafa56e055dfaab90326e6..eb451bec23d3d4f511ed50ba5bc5936bc7783192 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml @@ -25,6 +25,7 @@ properties: - mediatek,mt2712-mmsys - mediatek,mt6765-mmsys - mediatek,mt6779-mmsys + - mediatek,mt6795-mmsys - mediatek,mt6797-mmsys - mediatek,mt8167-mmsys - mediatek,mt8173-mmsys @@ -52,7 +53,8 @@ properties: description: Using mailbox to communicate with GCE, it should have this property and list of phandle, mailbox specifiers. See - Documentation/devicetree/bindings/mailbox/mtk-gce.txt for details. + Documentation/devicetree/bindings/mailbox/mediatek,gce-mailbox.yaml + for details. $ref: /schemas/types.yaml#/definitions/phandle-array mediatek,gce-client-reg: diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml index 787d6673f952bf170355da56c45639888db93e4d..84fb0a146b6e1b1186f2a5618854cb63e9c4dff6 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml @@ -20,6 +20,7 @@ properties: items: - enum: - mediatek,mt7622-wed + - mediatek,mt7986-wed - const: syscon reg: diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96221f51c1c33f8b0045c5ba85982a3426aea989 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/arm/mediatek/mediatek,mt7986-wed-pcie.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: MediaTek PCIE WED Controller for MT7986 + +maintainers: + - Lorenzo Bianconi + - Felix Fietkau + +description: + The mediatek WED PCIE provides a configuration interface for PCIE + controller on MT7986 soc. + +properties: + compatible: + items: + - enum: + - mediatek,mt7986-wed-pcie + - const: syscon + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + wed_pcie: wed-pcie@10003000 { + compatible = "mediatek,mt7986-wed-pcie", + "syscon"; + reg = <0 0x10003000 0 0x10>; + }; + }; diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,pericfg.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,pericfg.yaml index 8585f6f18f691b44f74cf18ed3743a6bd553bc9b..ef62cbb13590402354a19bdb8da22702ae6bf977 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,pericfg.yaml +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,pericfg.yaml @@ -21,6 +21,7 @@ properties: - mediatek,mt2701-pericfg - mediatek,mt2712-pericfg - mediatek,mt6765-pericfg + - mediatek,mt6795-pericfg - mediatek,mt7622-pericfg - mediatek,mt7629-pericfg - mediatek,mt8135-pericfg diff --git a/Documentation/devicetree/bindings/arm/microchip,sparx5.yaml b/Documentation/devicetree/bindings/arm/microchip,sparx5.yaml index 6193388c63182a7069b45cb21ff84640c85448d2..9a0d54e9799c4f6767856490c788d2f43ebaf2d1 100644 --- a/Documentation/devicetree/bindings/arm/microchip,sparx5.yaml +++ b/Documentation/devicetree/bindings/arm/microchip,sparx5.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/microchip,sparx5.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip Sparx5 Boards Device Tree Bindings +title: Microchip Sparx5 Boards maintainers: - Lars Povlsen diff --git a/Documentation/devicetree/bindings/arm/moxart.yaml b/Documentation/devicetree/bindings/arm/moxart.yaml index 670d24ce8ec5e57d8f075dd70d90f461226ced73..42565280914c3f4fcb5947522c058233e67634f7 100644 --- a/Documentation/devicetree/bindings/arm/moxart.yaml +++ b/Documentation/devicetree/bindings/arm/moxart.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/moxart.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MOXA ART device tree bindings +title: MOXA ART maintainers: - Jonas Jensen diff --git a/Documentation/devicetree/bindings/arm/nvidia,tegra194-ccplex.yaml b/Documentation/devicetree/bindings/arm/nvidia,tegra194-ccplex.yaml index c9675c4cdc1bd35b73cf46865c3b2fdafbf50122..b6f57d79a7530339af72412709b93bfc34d05441 100644 --- a/Documentation/devicetree/bindings/arm/nvidia,tegra194-ccplex.yaml +++ b/Documentation/devicetree/bindings/arm/nvidia,tegra194-ccplex.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/arm/nvidia,tegra194-ccplex.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: NVIDIA Tegra194 CPU Complex device tree bindings +title: NVIDIA Tegra194 CPU Complex maintainers: - Thierry Reding diff --git a/Documentation/devicetree/bindings/arm/psci.yaml b/Documentation/devicetree/bindings/arm/psci.yaml index dd83ef278af03b7c4bb4c29d075c901324debb2c..3a2c908ff28200f004a392238951319ed7ae8595 100644 --- a/Documentation/devicetree/bindings/arm/psci.yaml +++ b/Documentation/devicetree/bindings/arm/psci.yaml @@ -41,31 +41,26 @@ properties: For implementations complying to PSCI versions prior to 0.2. const: arm,psci - - description: - For implementations complying to PSCI 0.2. - const: arm,psci-0.2 - - description: For implementations complying to PSCI 0.2. Function IDs are not required and should be ignored by an OS with PSCI 0.2 support, but are permitted to be present for compatibility with existing software when "arm,psci" is later in the compatible list. + minItems: 1 items: - const: arm,psci-0.2 - const: arm,psci - - description: - For implementations complying to PSCI 1.0. - const: arm,psci-1.0 - - description: For implementations complying to PSCI 1.0. PSCI 1.0 is backward compatible with PSCI 0.2 with minor specification updates, as defined in the PSCI specification[2]. + minItems: 1 items: - const: arm,psci-1.0 - const: arm,psci-0.2 + - const: arm,psci method: description: The method of calling the PSCI firmware. diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml index fb1d00bcc847c14f1a2bda9d2d49bd82773a128e..1b5ac6b02bc5aeaed4c5cf779e971cad48b517bd 100644 --- a/Documentation/devicetree/bindings/arm/qcom.yaml +++ b/Documentation/devicetree/bindings/arm/qcom.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/qcom.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: QCOM device tree bindings +title: QCOM maintainers: - Bjorn Andersson @@ -176,6 +176,9 @@ properties: - longcheer,l8910 - samsung,a3u-eur - samsung,a5u-eur + - samsung,e5 + - samsung,e7 + - samsung,grandmax - samsung,j5 - samsung,serranove - wingtech,wt88047 @@ -450,6 +453,7 @@ properties: - description: Google Pazquel with LTE and Parade (newest rev) items: + - const: google,pazquel-sku6 - const: google,pazquel-sku4 - const: qcom,sc7180 @@ -550,6 +554,7 @@ properties: - description: Qualcomm Technologies, Inc. sc7280 CRD platform (newest rev) items: + - const: google,zoglin - const: google,hoglin - const: qcom,sc7280 @@ -565,16 +570,31 @@ properties: - const: google,piglin - const: qcom,sc7280 + - description: Google Evoker (newest rev) + items: + - const: google,evoker + - const: qcom,sc7280 + - description: Google Herobrine (newest rev) items: - const: google,herobrine - const: qcom,sc7280 + - description: Google Villager (rev0) + items: + - const: google,villager-rev0 + - const: qcom,sc7280 + - description: Google Villager (newest rev) items: - const: google,villager - const: qcom,sc7280 + - description: Google Villager with LTE (newest rev) + items: + - const: google,villager-sku512 + - const: qcom,sc7280 + - items: - enum: - lenovo,flex-5g @@ -716,6 +736,7 @@ properties: - enum: - qcom,sm8450-hdk - qcom,sm8450-qrd + - sony,pdx223 - const: qcom,sm8450 additionalProperties: true diff --git a/Documentation/devicetree/bindings/arm/rda.yaml b/Documentation/devicetree/bindings/arm/rda.yaml index a5c0444aa2b47b7281f7ff095ce7f6944195372e..09241ea1d2280394ed4731bc8432c98b8e66b22f 100644 --- a/Documentation/devicetree/bindings/arm/rda.yaml +++ b/Documentation/devicetree/bindings/arm/rda.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/rda.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: RDA Micro platforms device tree bindings +title: RDA Micro platforms maintainers: - Manivannan Sadhasivam diff --git a/Documentation/devicetree/bindings/arm/realtek.yaml b/Documentation/devicetree/bindings/arm/realtek.yaml index 9fb0297fe1ced79c9b342f953fa8972a58c95007..ddd9a85099e91c899fe207b3a792d3242ca3c276 100644 --- a/Documentation/devicetree/bindings/arm/realtek.yaml +++ b/Documentation/devicetree/bindings/arm/realtek.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/realtek.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Realtek platforms device tree bindings +title: Realtek platforms maintainers: - Andreas Färber diff --git a/Documentation/devicetree/bindings/arm/renesas.yaml b/Documentation/devicetree/bindings/arm/renesas.yaml index ff80152f092fc10eed8b2e9f92256f9b5f8847e6..f51464a08aff83564c1b67d7e81df6f6ebb57e99 100644 --- a/Documentation/devicetree/bindings/arm/renesas.yaml +++ b/Documentation/devicetree/bindings/arm/renesas.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/renesas.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Renesas SH-Mobile, R-Mobile, and R-Car Platform Device Tree Bindings +title: Renesas SH-Mobile, R-Mobile, and R-Car Platform maintainers: - Geert Uytterhoeven @@ -264,6 +264,7 @@ properties: - renesas,r8a779m4 - renesas,r8a779m5 - renesas,r8a779m8 + - renesas,r8a779mb - enum: - renesas,r8a7795 - renesas,r8a77961 @@ -291,6 +292,13 @@ properties: - renesas,v3hsk # V3HSK (Y-ASK-RCAR-V3H-WS10) - const: renesas,r8a77980 + - description: R-Car V3H2 (R8A77980A) + items: + - enum: + - renesas,condor-i # Condor-I (RTP0RC77980SEBS012SA01) + - const: renesas,r8a77980a + - const: renesas,r8a77980 + - description: R-Car E3 (R8A77990) items: - enum: @@ -409,6 +417,14 @@ properties: - const: renesas,r8a779m8 - const: renesas,r8a7795 + - description: R-Car H3Ne-1.7G (R8A779MB) + items: + - enum: + - renesas,h3ulcb # H3ULCB (R-Car Starter Kit Premier) + - renesas,salvator-xs # Salvator-XS (Salvator-X 2nd version) + - const: renesas,r8a779mb + - const: renesas,r8a7795 + - description: RZ/N1D (R9A06G032) items: - enum: diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml index 7811ba64149cbb8271d3dd90d2c0124e993fccbe..c6c69a4e3777b5cd3f16e2372500dd954587e114 100644 --- a/Documentation/devicetree/bindings/arm/rockchip.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/rockchip.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip platforms device tree bindings +title: Rockchip platforms maintainers: - Heiko Stuebner @@ -30,6 +30,16 @@ properties: - const: amarula,vyasa-rk3288 - const: rockchip,rk3288 + - description: Anbernic RG353P + items: + - const: anbernic,rg353p + - const: rockchip,rk3566 + + - description: Anbernic RG503 + items: + - const: anbernic,rg503 + - const: rockchip,rk3566 + - description: Asus Tinker board items: - const: asus,rk3288-tinker @@ -151,6 +161,7 @@ properties: - friendlyarm,nanopi-m4b - friendlyarm,nanopi-neo4 - friendlyarm,nanopi-r4s + - friendlyarm,nanopi-r4s-enterprise - const: rockchip,rk3399 - description: GeekBuying GeekBox @@ -363,30 +374,55 @@ properties: - const: google,gru - const: rockchip,rk3399 - - description: Google Scarlet - Innolux display (Acer Chromebook Tab 10) + - description: | + Google Scarlet - Innolux display (Acer Chromebook Tab 10 and more) items: + - const: google,scarlet-rev15-sku2 + - const: google,scarlet-rev15-sku4 - const: google,scarlet-rev15-sku6 - const: google,scarlet-rev15 + - const: google,scarlet-rev14-sku2 + - const: google,scarlet-rev14-sku4 - const: google,scarlet-rev14-sku6 - const: google,scarlet-rev14 + - const: google,scarlet-rev13-sku2 + - const: google,scarlet-rev13-sku4 - const: google,scarlet-rev13-sku6 - const: google,scarlet-rev13 + - const: google,scarlet-rev12-sku2 + - const: google,scarlet-rev12-sku4 - const: google,scarlet-rev12-sku6 - const: google,scarlet-rev12 + - const: google,scarlet-rev11-sku2 + - const: google,scarlet-rev11-sku4 - const: google,scarlet-rev11-sku6 - const: google,scarlet-rev11 + - const: google,scarlet-rev10-sku2 + - const: google,scarlet-rev10-sku4 - const: google,scarlet-rev10-sku6 - const: google,scarlet-rev10 + - const: google,scarlet-rev9-sku2 + - const: google,scarlet-rev9-sku4 - const: google,scarlet-rev9-sku6 - const: google,scarlet-rev9 + - const: google,scarlet-rev8-sku2 + - const: google,scarlet-rev8-sku4 - const: google,scarlet-rev8-sku6 - const: google,scarlet-rev8 + - const: google,scarlet-rev7-sku2 + - const: google,scarlet-rev7-sku4 - const: google,scarlet-rev7-sku6 - const: google,scarlet-rev7 + - const: google,scarlet-rev6-sku2 + - const: google,scarlet-rev6-sku4 - const: google,scarlet-rev6-sku6 - const: google,scarlet-rev6 + - const: google,scarlet-rev5-sku2 + - const: google,scarlet-rev5-sku4 - const: google,scarlet-rev5-sku6 - const: google,scarlet-rev5 + - const: google,scarlet-rev4-sku2 + - const: google,scarlet-rev4-sku4 - const: google,scarlet-rev4-sku6 - const: google,scarlet-rev4 - const: google,scarlet @@ -470,6 +506,11 @@ properties: - const: netxeon,r89 - const: rockchip,rk3288 + - description: OPEN AI LAB EAIDK-610 + items: + - const: openailab,eaidk-610 + - const: rockchip,rk3399 + - description: Orange Pi RK3399 board items: - const: rockchip,rk3399-orangepi @@ -494,6 +535,11 @@ properties: - const: pine64,pinenote - const: rockchip,rk3566 + - description: Pine64 PinePhonePro + items: + - const: pine64,pinephone-pro + - const: rockchip,rk3399 + - description: Pine64 Rock64 items: - const: pine64,rock64 @@ -537,6 +583,11 @@ properties: - const: radxa,rockpi4 - const: rockchip,rk3399 + - description: Radxa ROCK 4C+ + items: + - const: radxa,rock-4c-plus + - const: rockchip,rk3399 + - description: Radxa ROCK Pi E items: - const: radxa,rockpi-e diff --git a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml index 5ece38065e54ea59e66bdd4709b8c76cb562f281..8c73bc7f4009fc3a3241a783f1d81cbbf724680c 100644 --- a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml @@ -21,10 +21,12 @@ select: enum: - rockchip,px30-pmu - rockchip,rk3066-pmu + - rockchip,rk3128-pmu - rockchip,rk3288-pmu - rockchip,rk3368-pmu - rockchip,rk3399-pmu - rockchip,rk3568-pmu + - rockchip,rk3588-pmu required: - compatible @@ -35,10 +37,12 @@ properties: - enum: - rockchip,px30-pmu - rockchip,rk3066-pmu + - rockchip,rk3128-pmu - rockchip,rk3288-pmu - rockchip,rk3368-pmu - rockchip,rk3399-pmu - rockchip,rk3568-pmu + - rockchip,rk3588-pmu - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml index 7ca5375f278f35e8035334481f5e61dbe3764c15..6096c082d56d0b55efa7b190c80d636aaa990419 100644 --- a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml +++ b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml @@ -22,7 +22,6 @@ properties: description: | should contain 3 regions: control register, revision register, operation register, in this order. - minItems: 3 maxItems: 3 interrupts: diff --git a/Documentation/devicetree/bindings/arm/spear.yaml b/Documentation/devicetree/bindings/arm/spear.yaml index 605ad3f882efcfe8de256b5dad429eefb0afb056..a465c9eca76eb7cea4872ebbbe95ddbbb01616d7 100644 --- a/Documentation/devicetree/bindings/arm/spear.yaml +++ b/Documentation/devicetree/bindings/arm/spear.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/spear.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ST SPEAr Platforms Device Tree Bindings +title: ST SPEAr Platforms maintainers: - Viresh Kumar diff --git a/Documentation/devicetree/bindings/arm/sti.yaml b/Documentation/devicetree/bindings/arm/sti.yaml index a41cd8764885a7b6bb7aca7ad531f9a5ef19bff6..3ca054c643773237d52f6870866783710812dda0 100644 --- a/Documentation/devicetree/bindings/arm/sti.yaml +++ b/Documentation/devicetree/bindings/arm/sti.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/sti.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ST STi Platforms Device Tree Bindings +title: ST STi Platforms maintainers: - Patrice Chotard diff --git a/Documentation/devicetree/bindings/arm/sunxi.yaml b/Documentation/devicetree/bindings/arm/sunxi.yaml index 0c2356778208afadfdf4f61f4569cad90d15a3cf..3ad1cd50e3fe0bad70c5f37490d237be7eb23f7a 100644 --- a/Documentation/devicetree/bindings/arm/sunxi.yaml +++ b/Documentation/devicetree/bindings/arm/sunxi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/sunxi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner platforms device tree bindings +title: Allwinner platforms maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/arm/tegra.yaml b/Documentation/devicetree/bindings/arm/tegra.yaml index 49841ca272ee98a4050a1e3cfaedfb2cd92ef70b..1f62253f941008cc2aeab3e42ebfa4cfa753801c 100644 --- a/Documentation/devicetree/bindings/arm/tegra.yaml +++ b/Documentation/devicetree/bindings/arm/tegra.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/tegra.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NVIDIA Tegra device tree bindings +title: NVIDIA Tegra maintainers: - Thierry Reding diff --git a/Documentation/devicetree/bindings/arm/tesla.yaml b/Documentation/devicetree/bindings/arm/tesla.yaml index 09856da657dcc9ffa79178df9f4b562793505c4e..d670a0d562226e5ed8c9a2b6973a8420de2aab5e 100644 --- a/Documentation/devicetree/bindings/arm/tesla.yaml +++ b/Documentation/devicetree/bindings/arm/tesla.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/tesla.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tesla Full Self Driving(FSD) platforms device tree bindings +title: Tesla Full Self Driving(FSD) platforms maintainers: - Alim Akhtar diff --git a/Documentation/devicetree/bindings/arm/ti/k3.yaml b/Documentation/devicetree/bindings/arm/ti/k3.yaml index 61c6ab4f52e26f29ccba98e628acedfc47743bef..28b8232e1c5ba56339799df94837124529b4569f 100644 --- a/Documentation/devicetree/bindings/arm/ti/k3.yaml +++ b/Documentation/devicetree/bindings/arm/ti/k3.yaml @@ -19,32 +19,11 @@ properties: compatible: oneOf: - - description: K3 AM654 SoC + - description: K3 AM62A7 SoC items: - enum: - - ti,am654-evm - - siemens,iot2050-basic - - siemens,iot2050-basic-pg2 - - siemens,iot2050-advanced - - siemens,iot2050-advanced-pg2 - - const: ti,am654 - - - description: K3 J721E SoC - oneOf: - - const: ti,j721e - - items: - - enum: - - ti,j721e-evm - - ti,j721e-sk - - const: ti,j721e - - - description: K3 J7200 SoC - oneOf: - - const: ti,j7200 - - items: - - enum: - - ti,j7200-evm - - const: ti,j7200 + - ti,am62a7-sk + - const: ti,am62a7 - description: K3 AM625 SoC items: @@ -59,6 +38,33 @@ properties: - ti,am642-sk - const: ti,am642 + - description: K3 AM654 SoC + items: + - enum: + - siemens,iot2050-advanced + - siemens,iot2050-advanced-pg2 + - siemens,iot2050-basic + - siemens,iot2050-basic-pg2 + - ti,am654-evm + - const: ti,am654 + + - description: K3 J7200 SoC + oneOf: + - const: ti,j7200 + - items: + - enum: + - ti,j7200-evm + - const: ti,j7200 + + - description: K3 J721E SoC + oneOf: + - const: ti,j721e + - items: + - enum: + - ti,j721e-evm + - ti,j721e-sk + - const: ti,j721e + - description: K3 J721s2 SoC items: - enum: diff --git a/Documentation/devicetree/bindings/arm/toshiba.yaml b/Documentation/devicetree/bindings/arm/toshiba.yaml index 9c1cacbdc9166a0899b7e718804b08e31f455f6b..716ba4a3cab4e9f94f946e36157cb21fe6e2efc1 100644 --- a/Documentation/devicetree/bindings/arm/toshiba.yaml +++ b/Documentation/devicetree/bindings/arm/toshiba.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/toshiba.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Toshiba Visconti Platform Device Tree Bindings +title: Toshiba Visconti Platform maintainers: - Nobuhiro Iwamatsu diff --git a/Documentation/devicetree/bindings/arm/ux500.yaml b/Documentation/devicetree/bindings/arm/ux500.yaml index 17accb31bca0512b184e7e794a7ebff8760e4f63..b42d20fa43596868ce9a61f07dda9343d61ce9a2 100644 --- a/Documentation/devicetree/bindings/arm/ux500.yaml +++ b/Documentation/devicetree/bindings/arm/ux500.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/ux500.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Ux500 platforms device tree bindings +title: Ux500 platforms maintainers: - Linus Walleij diff --git a/Documentation/devicetree/bindings/arm/versatile-sysreg.txt b/Documentation/devicetree/bindings/arm/versatile-sysreg.txt deleted file mode 100644 index a4f15262d717a96b86b17640ee49915e3fe57d02..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/arm/versatile-sysreg.txt +++ /dev/null @@ -1,10 +0,0 @@ -ARM Versatile system registers --------------------------------------- - -This is a system control registers block, providing multiple low level -platform functions like board detection and identification, software -interrupt generation, MMC and NOR Flash control etc. - -Required node properties: -- compatible value : = "arm,versatile-sysreg", "syscon" -- reg : physical base address and the size of the registers window diff --git a/Documentation/devicetree/bindings/arm/vt8500.yaml b/Documentation/devicetree/bindings/arm/vt8500.yaml index 7b762bfc11e765566500ae39d1cf50baaae2b947..5d5ad5a60451f569e6ef30c924a1964d02e1aa82 100644 --- a/Documentation/devicetree/bindings/arm/vt8500.yaml +++ b/Documentation/devicetree/bindings/arm/vt8500.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/vt8500.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: VIA/Wondermedia VT8500 Platforms Device Tree Bindings +title: VIA/Wondermedia VT8500 Platforms maintainers: - Tony Prisk diff --git a/Documentation/devicetree/bindings/arm/xilinx.yaml b/Documentation/devicetree/bindings/arm/xilinx.yaml index 4dc0e0195974f39d5936a960caa6a787c0348221..969cfe6dc43480cfd741ba78560d67fb4bc4c140 100644 --- a/Documentation/devicetree/bindings/arm/xilinx.yaml +++ b/Documentation/devicetree/bindings/arm/xilinx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/arm/xilinx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx Zynq Platforms Device Tree Bindings +title: Xilinx Zynq Platforms maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/ata/ahci-common.yaml b/Documentation/devicetree/bindings/ata/ahci-common.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94d72aeaad0f7b926c28e6fe441f32c6ba8da3f9 --- /dev/null +++ b/Documentation/devicetree/bindings/ata/ahci-common.yaml @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/ahci-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Common Properties for Serial ATA AHCI controllers + +maintainers: + - Hans de Goede + - Damien Le Moal + +description: + This document defines device tree properties for a common AHCI SATA + controller implementation. It's hardware interface is supposed to + conform to the technical standard defined by Intel (see Serial ATA + Advanced Host Controller Interface specification for details). The + document doesn't constitute a DT-node binding by itself but merely + defines a set of common properties for the AHCI-compatible devices. + +select: false + +allOf: + - $ref: sata-common.yaml# + +properties: + reg: + description: + Generic AHCI registers space conforming to the Serial ATA AHCI + specification. + + reg-names: + description: CSR space IDs + contains: + const: ahci + + interrupts: + description: + Generic AHCI state change interrupt. Can be implemented either as a + single line attached to the controller or as a set of the signals + indicating the particular port events. + minItems: 1 + maxItems: 32 + + ahci-supply: + description: Power regulator for AHCI controller + + target-supply: + description: Power regulator for SATA target device + + phy-supply: + description: Power regulator for SATA PHY + + phys: + description: Reference to the SATA PHY node + maxItems: 1 + + phy-names: + const: sata-phy + + hba-cap: + $ref: '/schemas/types.yaml#/definitions/uint32' + description: + Bitfield of the HBA generic platform capabilities like Staggered + Spin-up or Mechanical Presence Switch support. It can be used to + appropriately initialize the HWinit fields of the HBA CAP register + in case if the system firmware hasn't done it. + + ports-implemented: + $ref: '/schemas/types.yaml#/definitions/uint32' + description: + Mask that indicates which ports the HBA supports. Useful if PI is not + programmed by the BIOS, which is true for some embedded SoC's. + +patternProperties: + "^sata-port@[0-9a-f]+$": + $ref: '#/$defs/ahci-port' + description: + It is optionally possible to describe the ports as sub-nodes so + to enable each port independently when dealing with multiple PHYs. + +required: + - reg + - interrupts + +additionalProperties: true + +$defs: + ahci-port: + $ref: /schemas/ata/sata-common.yaml#/$defs/sata-port + + properties: + reg: + description: + AHCI SATA port identifier. By design AHCI controller can't have + more than 32 ports due to the CAP.NP fields and PI register size + constraints. + minimum: 0 + maximum: 31 + + phys: + description: Individual AHCI SATA port PHY + maxItems: 1 + + phy-names: + description: AHCI SATA port PHY ID + const: sata-phy + + target-supply: + description: Power regulator for SATA port target device + + hba-port-cap: + $ref: '/schemas/types.yaml#/definitions/uint32' + description: + Bitfield of the HBA port-specific platform capabilities like Hot + plugging, eSATA, FIS-based Switching, etc (see AHCI specification + for details). It can be used to initialize the HWinit fields of + the PxCMD register in case if the system firmware hasn't done it. + + required: + - reg + +... diff --git a/Documentation/devicetree/bindings/ata/ahci-platform.yaml b/Documentation/devicetree/bindings/ata/ahci-platform.yaml index c146ab8e14e5ffecc23251235ca4a0a55b7fa7ce..7dc2a2e8f598b2919650cd2842ccbf6ad1b01ee5 100644 --- a/Documentation/devicetree/bindings/ata/ahci-platform.yaml +++ b/Documentation/devicetree/bindings/ata/ahci-platform.yaml @@ -30,14 +30,11 @@ select: - marvell,armada-3700-ahci - marvell,armada-8k-ahci - marvell,berlin2q-ahci - - snps,dwc-ahci - - snps,spear-ahci required: - compatible allOf: - - $ref: "sata-common.yaml#" - + - $ref: "ahci-common.yaml#" properties: compatible: @@ -49,17 +46,11 @@ properties: - marvell,berlin2-ahci - marvell,berlin2q-ahci - const: generic-ahci - - items: - - enum: - - rockchip,rk3568-dwc-ahci - - const: snps,dwc-ahci - enum: - cavium,octeon-7130-ahci - hisilicon,hisi-ahci - ibm,476gtr-ahci - marvell,armada-3700-ahci - - snps,dwc-ahci - - snps,spear-ahci reg: minItems: 1 @@ -69,92 +60,37 @@ properties: maxItems: 1 clocks: - description: - Clock IDs array as required by the controller. minItems: 1 maxItems: 3 clock-names: - description: - Names of clocks corresponding to IDs in the clock property. minItems: 1 maxItems: 3 interrupts: maxItems: 1 - ahci-supply: - description: - regulator for AHCI controller - - dma-coherent: true - - phy-supply: - description: - regulator for PHY power - - phys: - description: - List of all PHYs on this controller - maxItems: 1 - - phy-names: - description: - Name specifier for the PHYs - maxItems: 1 - - ports-implemented: - $ref: '/schemas/types.yaml#/definitions/uint32' - description: | - Mask that indicates which ports that the HBA supports - are available for software to use. Useful if PORTS_IMPL - is not programmed by the BIOS, which is true with - some embedded SoCs. - maximum: 0x1f - power-domains: maxItems: 1 resets: maxItems: 1 - target-supply: - description: - regulator for SATA target power - -required: - - compatible - - reg - - interrupts - patternProperties: "^sata-port@[0-9a-f]+$": - type: object - additionalProperties: false - description: - Subnode with configuration of the Ports. - - properties: - reg: - maxItems: 1 - - phys: - maxItems: 1 - - phy-names: - maxItems: 1 - - target-supply: - description: - regulator for SATA target power - - required: - - reg + $ref: /schemas/ata/ahci-common.yaml#/$defs/ahci-port anyOf: - required: [ phys ] - required: [ target-supply ] + unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + unevaluatedProperties: false examples: @@ -167,6 +103,8 @@ examples: - | #include #include + #include + sata@f7e90000 { compatible = "marvell,berlin2q-ahci", "generic-ahci"; reg = <0xf7e90000 0x1000>; @@ -175,15 +113,23 @@ examples: #address-cells = <1>; #size-cells = <0>; + hba-cap = ; + sata0: sata-port@0 { reg = <0>; + phys = <&sata_phy 0>; target-supply = <®_sata0>; + + hba-port-cap = <(HBA_PORT_FBSCP | HBA_PORT_ESP)>; }; sata1: sata-port@1 { reg = <1>; + phys = <&sata_phy 1>; target-supply = <®_sata1>; + + hba-port-cap = <(HBA_PORT_HPCP | HBA_PORT_MPSP | HBA_PORT_FBSCP)>; }; }; diff --git a/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml b/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b7ca4759bd72b29f1993569fd1c6ae9190f2175 --- /dev/null +++ b/Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/baikal,bt1-ahci.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Baikal-T1 SoC AHCI SATA controller + +maintainers: + - Serge Semin + +description: + AHCI SATA controller embedded into the Baikal-T1 SoC is based on the + DWC AHCI SATA v4.10a IP-core. + +allOf: + - $ref: snps,dwc-ahci-common.yaml# + +properties: + compatible: + const: baikal,bt1-ahci + + clocks: + items: + - description: Peripheral APB bus clock + - description: Application AXI BIU clock + - description: SATA Ports reference clock + + clock-names: + items: + - const: pclk + - const: aclk + - const: ref + + resets: + items: + - description: Application AXI BIU domain reset + - description: SATA Ports clock domain reset + + reset-names: + items: + - const: arst + - const: ref + + ports-implemented: + maximum: 0x3 + +patternProperties: + "^sata-port@[0-1]$": + $ref: /schemas/ata/snps,dwc-ahci-common.yaml#/$defs/dwc-ahci-port + + properties: + reg: + minimum: 0 + maximum: 1 + + snps,tx-ts-max: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Due to having AXI3 bus interface utilized the maximum Tx DMA + transaction size can't exceed 16 beats (AxLEN[3:0]). + enum: [ 1, 2, 4, 8, 16 ] + + snps,rx-ts-max: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Due to having AXI3 bus interface utilized the maximum Rx DMA + transaction size can't exceed 16 beats (AxLEN[3:0]). + enum: [ 1, 2, 4, 8, 16 ] + + unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - resets + +unevaluatedProperties: false + +examples: + - | + sata@1f050000 { + compatible = "baikal,bt1-ahci"; + reg = <0x1f050000 0x2000>; + #address-cells = <1>; + #size-cells = <0>; + + interrupts = <0 64 4>; + + clocks = <&ccu_sys 1>, <&ccu_axi 2>, <&sata_ref_clk>; + clock-names = "pclk", "aclk", "ref"; + + resets = <&ccu_axi 2>, <&ccu_sys 0>; + reset-names = "arst", "ref"; + + ports-implemented = <0x3>; + + sata-port@0 { + reg = <0>; + + snps,tx-ts-max = <4>; + snps,rx-ts-max = <4>; + }; + + sata-port@1 { + reg = <1>; + + snps,tx-ts-max = <4>; + snps,rx-ts-max = <4>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/ata/brcm,sata-brcm.yaml b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.yaml index 235a93ac86b05b1ba7ad5acb9af5c896f4f27438..fe7f091e744f3619f6b7969da940648d5e17d3ce 100644 --- a/Documentation/devicetree/bindings/ata/brcm,sata-brcm.yaml +++ b/Documentation/devicetree/bindings/ata/brcm,sata-brcm.yaml @@ -14,7 +14,7 @@ maintainers: - Florian Fainelli allOf: - - $ref: sata-common.yaml# + - $ref: ahci-common.yaml# properties: compatible: @@ -30,7 +30,6 @@ properties: - const: brcm,bcm-nsp-ahci reg: - minItems: 2 maxItems: 2 reg-names: @@ -41,8 +40,6 @@ properties: interrupts: maxItems: 1 - dma-coherent: true - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml b/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml index 21a90975593ba5da09cc7bff5030168132ca0da2..5290936665084a5d1176fcd0059a398a051caa0c 100644 --- a/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml +++ b/Documentation/devicetree/bindings/ata/cortina,gemini-sata-bridge.yaml @@ -22,7 +22,6 @@ properties: maxItems: 1 resets: - minItems: 2 maxItems: 2 description: phandles to the reset lines for both SATA bridges @@ -32,7 +31,6 @@ properties: - const: sata1 clocks: - minItems: 2 maxItems: 2 description: phandles to the compulsory peripheral clocks diff --git a/Documentation/devicetree/bindings/ata/sata-common.yaml b/Documentation/devicetree/bindings/ata/sata-common.yaml index 7ac77b1c5850d85062024d974bb871ce95e168ec..58c9342b9925588e002f55042de238c5fa954958 100644 --- a/Documentation/devicetree/bindings/ata/sata-common.yaml +++ b/Documentation/devicetree/bindings/ata/sata-common.yaml @@ -31,22 +31,27 @@ properties: "#size-cells": const: 0 + dma-coherent: true + patternProperties: "^sata-port@[0-9a-e]$": + $ref: '#/$defs/sata-port' description: | DT nodes for ports connected on the SATA host. The SATA port nodes will be named "sata-port". + +additionalProperties: true + +$defs: + sata-port: type: object properties: reg: minimum: 0 - maximum: 14 description: - The ID number of the drive port SATA can potentially use a port - multiplier making it possible to connect up to 15 disks to a single - SATA port. - -additionalProperties: true + The ID number of the SATA port. Aside with being directly used, + each port can have a Port Multiplier attached thus allowing to + access more than one drive by means of a single SATA port. ... diff --git a/Documentation/devicetree/bindings/ata/sata_highbank.yaml b/Documentation/devicetree/bindings/ata/sata_highbank.yaml index 49679b58041ca999191f0965ef445604342352b0..f23f26a8f21c6efc0bf7f4af3d3eeda517c8d85b 100644 --- a/Documentation/devicetree/bindings/ata/sata_highbank.yaml +++ b/Documentation/devicetree/bindings/ata/sata_highbank.yaml @@ -52,7 +52,6 @@ properties: minItems: 1 maxItems: 8 items: - minItems: 2 maxItems: 2 calxeda,tx-atten: diff --git a/Documentation/devicetree/bindings/ata/snps,dwc-ahci-common.yaml b/Documentation/devicetree/bindings/ata/snps,dwc-ahci-common.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1457910520bf952784ddaf040f4790c6faece3d --- /dev/null +++ b/Documentation/devicetree/bindings/ata/snps,dwc-ahci-common.yaml @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/snps,dwc-ahci-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DWC AHCI SATA controller properties + +maintainers: + - Serge Semin + +description: + This document defines device tree schema for the generic Synopsys DWC + AHCI controller properties. + +select: false + +allOf: + - $ref: ahci-common.yaml# + +properties: + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + description: + Basic DWC AHCI SATA clock sources like application AXI/AHB BIU clock, + PM-alive clock, RxOOB detection clock, embedded PHYs reference (Rx/Tx) + clock, etc. + minItems: 1 + maxItems: 4 + + clock-names: + minItems: 1 + maxItems: 4 + items: + oneOf: + - description: Application APB/AHB/AXI BIU clock + enum: + - pclk + - aclk + - hclk + - sata + - description: Power Module keep-alive clock + const: pmalive + - description: RxOOB detection clock + const: rxoob + - description: SATA Ports reference clock + const: ref + + resets: + description: + At least basic application and reference clock domains resets are + normally supported by the DWC AHCI SATA controller. + minItems: 1 + maxItems: 4 + + reset-names: + minItems: 1 + maxItems: 4 + items: + oneOf: + - description: Application AHB/AXI BIU clock domain reset control + enum: + - arst + - hrst + - description: Power Module keep-alive clock domain reset control + const: pmalive + - description: RxOOB detection clock domain reset control + const: rxoob + - description: Reference clock domain reset control + const: ref + +patternProperties: + "^sata-port@[0-9a-e]$": + $ref: '#/$defs/dwc-ahci-port' + +additionalProperties: true + +$defs: + dwc-ahci-port: + $ref: /schemas/ata/ahci-common.yaml#/$defs/ahci-port + + properties: + reg: + minimum: 0 + maximum: 7 + + snps,tx-ts-max: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Maximal size of Tx DMA transactions in FIFO words + enum: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] + + snps,rx-ts-max: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Maximal size of Rx DMA transactions in FIFO words + enum: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] + +... diff --git a/Documentation/devicetree/bindings/ata/snps,dwc-ahci.yaml b/Documentation/devicetree/bindings/ata/snps,dwc-ahci.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5afa4b57ce20cb5fc82f408857ac9a40ff9e85e8 --- /dev/null +++ b/Documentation/devicetree/bindings/ata/snps,dwc-ahci.yaml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/ata/snps,dwc-ahci.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DWC AHCI SATA controller + +maintainers: + - Serge Semin + +description: + This document defines device tree bindings for the generic Synopsys DWC + implementation of the AHCI SATA controller. + +allOf: + - $ref: snps,dwc-ahci-common.yaml# + +properties: + compatible: + oneOf: + - description: Synopsys AHCI SATA-compatible devices + const: snps,dwc-ahci + - description: SPEAr1340 AHCI SATA device + const: snps,spear-ahci + - description: Rockhip RK3568 AHCI controller + items: + - const: rockchip,rk3568-dwc-ahci + - const: snps,dwc-ahci + +patternProperties: + "^sata-port@[0-9a-e]$": + $ref: /schemas/ata/snps,dwc-ahci-common.yaml#/$defs/dwc-ahci-port + + unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + sata@122f0000 { + compatible = "snps,dwc-ahci"; + reg = <0x122F0000 0x1ff>; + #address-cells = <1>; + #size-cells = <0>; + + interrupts = ; + + clocks = <&clock1>, <&clock2>; + clock-names = "aclk", "ref"; + + phys = <&sata_phy>; + phy-names = "sata-phy"; + + ports-implemented = <0x1>; + + sata-port@0 { + reg = <0>; + + hba-port-cap = ; + + snps,tx-ts-max = <512>; + snps,rx-ts-max = <512>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/bus/allwinner,sun50i-a64-de2.yaml b/Documentation/devicetree/bindings/bus/allwinner,sun50i-a64-de2.yaml index ad313ccaaaefc3ce57868b937ba96e9f0e6e6c44..85c4a979aec4631b2a61c8118a18685c393fba8a 100644 --- a/Documentation/devicetree/bindings/bus/allwinner,sun50i-a64-de2.yaml +++ b/Documentation/devicetree/bindings/bus/allwinner,sun50i-a64-de2.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/bus/allwinner,sun50i-a64-de2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A64 Display Engine Bus Device Tree Bindings +title: Allwinner A64 Display Engine Bus maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/bus/allwinner,sun8i-a23-rsb.yaml b/Documentation/devicetree/bindings/bus/allwinner,sun8i-a23-rsb.yaml index 3d719f468a5bdb9d7e536ad7e75e9e86ecee0c70..bee5f53f837fea5aa1804886d6facce036d1aae5 100644 --- a/Documentation/devicetree/bindings/bus/allwinner,sun8i-a23-rsb.yaml +++ b/Documentation/devicetree/bindings/bus/allwinner,sun8i-a23-rsb.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/bus/allwinner,sun8i-a23-rsb.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A23 RSB Device Tree Bindings +title: Allwinner A23 RSB maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/bus/palmbus.yaml b/Documentation/devicetree/bindings/bus/palmbus.yaml index f5cbfaf52d53ccc704ac7a53fee431ff7fef5ce4..30fa6526cfc22ccfc21438552976f75b3bdead3e 100644 --- a/Documentation/devicetree/bindings/bus/palmbus.yaml +++ b/Documentation/devicetree/bindings/bus/palmbus.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/bus/palmbus.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Ralink PalmBus Device Tree Bindings +title: Ralink PalmBus maintainers: - Sergio Paracuellos diff --git a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml index d60e746548097cf29de7d9d70b2956e02b6aad42..79b0752faa9179eace055b92800a4cb337813de2 100644 --- a/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml +++ b/Documentation/devicetree/bindings/clock/airoha,en7523-scu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/airoha,en7523-scu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: EN7523 Clock Device Tree Bindings +title: EN7523 Clock maintainers: - Felix Fietkau diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ahb-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ahb-clk.yaml index 558db4b6ed173e0b533ae18acea98733b3b4e1af..93587b7004764d139fc581e913dfe9fd40e41327 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ahb-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ahb-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-ahb-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 AHB Clock Device Tree Bindings +title: Allwinner A10 AHB Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb0-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb0-clk.yaml index b1e3d739beb251bb87cefad00a8cf4aa9faad8a0..e14e1aad9fd60bd3d0dfb69c6b0dd13a554fb06d 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb0-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb0-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-apb0-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 APB0 Bus Clock Device Tree Bindings +title: Allwinner A10 APB0 Bus Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb1-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb1-clk.yaml index 51b7a6d4ea54781b54dd4ca975b20c7fc0fd2f4c..8a4747ebe0ba0cb44dd3b22a47874546e683608c 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb1-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-apb1-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-apb1-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 APB1 Bus Clock Device Tree Bindings +title: Allwinner A10 APB1 Bus Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-axi-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-axi-clk.yaml index d801158e15dec8cad5fba98b0bb72435d0c7c53f..aa08dd49dd61f85b84c4983303eaf2057bdce9b1 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-axi-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-axi-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-axi-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 AXI Clock Device Tree Bindings +title: Allwinner A10 AXI Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml index 15ed64d35261b2e8e30658b9c7bf7b56fa81ebbf..1690b9d99c3d4fee5f6b007fe1d5b011614e8bd9 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-ccu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner Clock Control Unit Device Tree Bindings +title: Allwinner Clock Control Unit maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-cpu-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-cpu-clk.yaml index 0dfafba1a168b0a4cc1d12d8dc90485a457f2bc6..08d073520cfa3db9618c8c5e28f0e3c478ee20d1 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-cpu-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-cpu-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-cpu-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 CPU Clock Device Tree Bindings +title: Allwinner A10 CPU Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-display-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-display-clk.yaml index 7484a7ab7deafe49cab8e1aec90d7ae7edf12569..e665e50c17855d0d55a4ffa8e4bcf3184898046c 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-display-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-display-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-display-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Display Clock Device Tree Bindings +title: Allwinner A10 Display Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-gates-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-gates-clk.yaml index 9a37a357cb4e7cbfeeee64f1677c4b2a7f6017c9..c4714d0fbe077aac7cc499b5611c72daef5490a9 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-gates-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-gates-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-gates-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Bus Gates Clock Device Tree Bindings +title: Allwinner A10 Bus Gates Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mbus-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mbus-clk.yaml index 18f131e262b44ed975e500eb8228a36e71744244..e824e33489b620bd1e14a5cb62df246859b92337 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mbus-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mbus-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-mbus-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 MBUS Clock Device Tree Bindings +title: Allwinner A10 MBUS Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mmc-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mmc-clk.yaml index 5199285a661ae0f38dedb776cc8dfc7867c5bc47..c612f94befb940104ef7f1a472b2f53f33398d57 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mmc-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mmc-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-mmc-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Module 1 Clock Device Tree Bindings +title: Allwinner A10 Module 1 Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod0-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod0-clk.yaml index 3e2abe3e67c15eaa9de6f5d3c44b783523a62abc..80ae3a7a588c12c129cc11051b6692c68aa66d26 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod0-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod0-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-mod0-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Module 0 Clock Device Tree Bindings +title: Allwinner A10 Module 0 Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod1-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod1-clk.yaml index 7ddb55c75cff603e0233124de385c61092ca2a1a..4f9a8d44d42a04f48876cbea3359274335fa617a 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod1-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-mod1-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-mod1-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Module 1 Clock Device Tree Bindings +title: Allwinner A10 Module 1 Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-osc-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-osc-clk.yaml index c604822cda073a3d97dc2a4625136aab9b2b40df..52a7b6e7124c466df91a8d7abffb5c68270a1482 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-osc-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-osc-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-osc-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Gatable Oscillator Clock Device Tree Bindings +title: Allwinner A10 Gatable Oscillator Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll1-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll1-clk.yaml index e5d9d45dab8ad03c5fe05a3bc3267b6e09948c8c..b13a1f21d5da89a40040df25bd293b3dfa2e1823 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll1-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll1-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-pll1-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 CPU PLL Device Tree Bindings +title: Allwinner A10 CPU PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll3-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll3-clk.yaml index 4b80a42fb3da18a9c11c880e5c51cc76b2690e6a..418d207d23b8bd8c0e2ba665d46c077c70887061 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll3-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll3-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-pll3-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Video PLL Device Tree Bindings +title: Allwinner A10 Video PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll5-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll5-clk.yaml index 415bd77de53d132266438f5c13542190b2142f49..76ef3f0c7f2c53d3017d8f300da951b6f12bc172 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll5-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll5-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-pll5-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 DRAM PLL Device Tree Bindings +title: Allwinner A10 DRAM PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll6-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll6-clk.yaml index ec5652f76027aa465965d4837fccaab31a6aa827..a94c93c90ecec9fe8b49eaf081b1a3c56b01007e 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll6-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-pll6-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-pll6-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Peripheral PLL Device Tree Bindings +title: Allwinner A10 Peripheral PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-tcon-ch0-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-tcon-ch0-clk.yaml index 0a335c615efd20af2622b153b478a403aa74233b..6646b2a99fc12072377a1d406d0f89ac49f2c35d 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-tcon-ch0-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-tcon-ch0-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-tcon-ch0-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 TCON Channel 0 Clock Device Tree Bindings +title: Allwinner A10 TCON Channel 0 Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-usb-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-usb-clk.yaml index cd95d25bfe7c5fd61e60c4c9f9f6aeffd36e2d33..5103b675e488886d873556016329fafb86b2a905 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-usb-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-usb-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-usb-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 USB Clock Device Tree Bindings +title: Allwinner A10 USB Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ve-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ve-clk.yaml index 5dfd0c1c27b4107d2e44cfd6fc60514fa2a0124d..80337e38d6e5c001b64b8152fd72a5c86b6e054b 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ve-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ve-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun4i-a10-ve-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Video Engine Clock Device Tree Bindings +title: Allwinner A10 Video Engine Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun5i-a13-ahb-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun5i-a13-ahb-clk.yaml index 99add7991c488d5c671ca7b0fb314d236f4bceed..c6a6fbb6863ba02d2f1ddb75f0251c0c2e8bcaa2 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun5i-a13-ahb-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun5i-a13-ahb-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun5i-a13-ahb-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A13 AHB Clock Device Tree Bindings +title: Allwinner A13 AHB Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun6i-a31-pll6-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun6i-a31-pll6-clk.yaml index 5f377205af718b9ea122b5068ebb0984b38bde98..7d6a6a34d20c0f0bab08801703ebee7198d5328f 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun6i-a31-pll6-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun6i-a31-pll6-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun6i-a31-pll6-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 Peripheral PLL Device Tree Bindings +title: Allwinner A31 Peripheral PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-gmac-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-gmac-clk.yaml index 59e5dce1b65affc344f64b9e0d6d84a024fedad8..b6202de35707f348a3bff0eaa24a480aaa4e39d0 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-gmac-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-gmac-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun7i-a20-gmac-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A20 GMAC TX Clock Device Tree Bindings +title: Allwinner A20 GMAC TX Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-out-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-out-clk.yaml index c745733bcf04485ecc30f33483b837dba22a85ab..fde7f7dc3d340b518b66d5716856e55099fdc4d5 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-out-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun7i-a20-out-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun7i-a20-out-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A20 Output Clock Device Tree Bindings +title: Allwinner A20 Output Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun8i-a83t-de2-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun8i-a83t-de2-clk.yaml index 17caf78f0ccf553ce65a27b2a84d41b4490d189c..70369bd633e40c07c4dae16fc96b2412a9a08111 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun8i-a83t-de2-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun8i-a83t-de2-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun8i-a83t-de2-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83t Display Engine 2/3 Clock Controller Device Tree Bindings +title: Allwinner A83t Display Engine 2/3 Clock Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun8i-h3-bus-gates-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun8i-h3-bus-gates-clk.yaml index 3eb2bf65b230c972daa953e69f079ba2b383cb39..45b9e2c7c1d141361ef8574af64aeb2ec22110f2 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun8i-h3-bus-gates-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun8i-h3-bus-gates-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun8i-h3-bus-gates-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Bus Gates Clock Device Tree Bindings +title: Allwinner A10 Bus Gates Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-ahb-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-ahb-clk.yaml index d178da90aaec6b1311d94fb5181dc17b2abf12fd..f0f65af8ae22dd7df40570bf9b66deeda7eaa173 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-ahb-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-ahb-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-ahb-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 AHB Clock Device Tree Bindings +title: Allwinner A80 AHB Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-apb0-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-apb0-clk.yaml index 0351c79bd221d24e683cad41ceb128349fe62e4e..e9f9bc8f579499a7cd2113ba850aa3c638e35919 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-apb0-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-apb0-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-apb0-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 APB0 Bus Clock Device Tree Bindings +title: Allwinner A80 APB0 Bus Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-cpus-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-cpus-clk.yaml index 24d5b2f1a31455e6a6f3bca25c5fb88415d15a53..c48db2d49340a9a4cab33af9f2062e1aeae0677c 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-cpus-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-cpus-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-cpus-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 CPUS Clock Device Tree Bindings +title: Allwinner A80 CPUS Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-de-clks.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-de-clks.yaml index a82c7c7e942bfe4beb954296c9af06a39aa03214..e9f81a343be16e5ef3b251415cb02a2331f5a922 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-de-clks.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-de-clks.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-de-clks.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 Display Engine Clock Controller Device Tree Bindings +title: Allwinner A80 Display Engine Clock Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-gt-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-gt-clk.yaml index 43963c3062c81a650de3c4f553724a9b1eb0dacb..d3ce5eb18d4e8abb850d3e987009c2aba748d3b5 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-gt-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-gt-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-gt-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 GT Bus Clock Device Tree Bindings +title: Allwinner A80 GT Bus Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-mmc-config-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-mmc-config-clk.yaml index 20dc115fa2110a4b19b49e0dd64da0b233758daa..65ee5afe83ccfc9198631fdbd0ad0be600f7fd82 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-mmc-config-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-mmc-config-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-mmc-config-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 MMC Configuration Clock Device Tree Bindings +title: Allwinner A80 MMC Configuration Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-pll4-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-pll4-clk.yaml index b76bab6a30e94e993d77288afc5a9550ad0da9d0..261264a8aef662e9e58f5272e606a5c711073172 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-pll4-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-pll4-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-pll4-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 Peripheral PLL Device Tree Bindings +title: Allwinner A80 Peripheral PLL maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml index 6532fb6821bc1dc4a7a7a42628bbf5e2be7a9305..515c15d5f661b1e9f41b6d6725c735baf291efcf 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-clks.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 USB Clock Controller Device Tree Bindings +title: Allwinner A80 USB Clock Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-mod-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-mod-clk.yaml index 15218d10e78e8b27a34379a35eaa7ccee9f3c202..3f7b8d9511f1c50dd11dcf8cadb4c5d7e6022a3e 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-mod-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-mod-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-mod-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 USB Module Clock Device Tree Bindings +title: Allwinner A80 USB Module Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-phy-clk.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-phy-clk.yaml index 2569041684e655e664db5da94c0f1e436c06a5aa..0d49072d47ca1ef9d2fbd3304d824f895a049823 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-phy-clk.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-phy-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-phy-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 USB PHY Clock Device Tree Bindings +title: Allwinner A80 USB PHY Clock maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/clock/amlogic,meson8-ddr-clkc.yaml b/Documentation/devicetree/bindings/clock/amlogic,meson8-ddr-clkc.yaml index 4b8669f870ecbad47b0d962e252ba679bfda80e8..d98d95d8e8c9e10a4dbe55b25b894adf3a3ea7f9 100644 --- a/Documentation/devicetree/bindings/clock/amlogic,meson8-ddr-clkc.yaml +++ b/Documentation/devicetree/bindings/clock/amlogic,meson8-ddr-clkc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/amlogic,meson8-ddr-clkc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Amlogic DDR Clock Controller Device Tree Bindings +title: Amlogic DDR Clock Controller maintainers: - Martin Blumenstingl diff --git a/Documentation/devicetree/bindings/clock/brcm,bcm2711-dvp.yaml b/Documentation/devicetree/bindings/clock/brcm,bcm2711-dvp.yaml index 08543ecbe35b294c07d2c6b48d1681244b083d76..2d40df2d34df16b0bdc4159198db296cb0da3e0c 100644 --- a/Documentation/devicetree/bindings/clock/brcm,bcm2711-dvp.yaml +++ b/Documentation/devicetree/bindings/clock/brcm,bcm2711-dvp.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/brcm,bcm2711-dvp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM2711 HDMI DVP Device Tree Bindings +title: Broadcom BCM2711 HDMI DVP maintainers: - Maxime Ripard diff --git a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml index 7f5cf4001f762baaa538cd654c541e6cd29e0a38..998e5cce652f32085e5e75797d5d7df1874a92e2 100644 --- a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml +++ b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/canaan,k210-clk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Canaan Kendryte K210 Clock Device Tree Bindings +title: Canaan Kendryte K210 Clock maintainers: - Damien Le Moal diff --git a/Documentation/devicetree/bindings/clock/cirrus,cs2000-cp.yaml b/Documentation/devicetree/bindings/clock/cirrus,cs2000-cp.yaml index 0abd6ba82dfd9fe51e6b8f36c51139859afd1a2a..82836086cac1492e808a9043d8d2aa40bb7c1780 100644 --- a/Documentation/devicetree/bindings/clock/cirrus,cs2000-cp.yaml +++ b/Documentation/devicetree/bindings/clock/cirrus,cs2000-cp.yaml @@ -23,7 +23,6 @@ properties: clocks: description: Common clock binding for CLK_IN, XTI/REF_CLK - minItems: 2 maxItems: 2 clock-names: diff --git a/Documentation/devicetree/bindings/clock/gpio-gate-clock.txt b/Documentation/devicetree/bindings/clock/gpio-gate-clock.txt deleted file mode 100644 index d3379ff9b84b120c9a2668e7a0ec8827ecc97965..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/gpio-gate-clock.txt +++ /dev/null @@ -1,21 +0,0 @@ -Binding for simple gpio gated clock. - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be "gpio-gate-clock". -- #clock-cells : from common clock binding; shall be set to 0. -- enable-gpios : GPIO reference for enabling and disabling the clock. - -Optional properties: -- clocks: Maximum of one parent clock is supported. - -Example: - clock { - compatible = "gpio-gate-clock"; - clocks = <&parentclk>; - #clock-cells = <0>; - enable-gpios = <&gpio 1 GPIO_ACTIVE_HIGH>; - }; diff --git a/Documentation/devicetree/bindings/clock/gpio-gate-clock.yaml b/Documentation/devicetree/bindings/clock/gpio-gate-clock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d09d0e3f0c6ecb17e3f8880e14790fece660416e --- /dev/null +++ b/Documentation/devicetree/bindings/clock/gpio-gate-clock.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/gpio-gate-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Simple GPIO clock gate + +maintainers: + - Jyri Sarha + +properties: + compatible: + const: gpio-gate-clock + + clocks: + maxItems: 1 + + '#clock-cells': + const: 0 + + enable-gpios: + description: GPIO reference for enabling and disabling the clock. + maxItems: 1 + +required: + - compatible + - '#clock-cells' + - enable-gpios + +additionalProperties: false + +examples: + - | + #include + + clock { + compatible = "gpio-gate-clock"; + clocks = <&parentclk>; + #clock-cells = <0>; + enable-gpios = <&gpio 1 GPIO_ACTIVE_HIGH>; + }; diff --git a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml index 7c331bfbe370eb5c61c0ef6a11afd022039c9dc4..f9ba9864d8b5282bab591e0c5b62b62cf6360804 100644 --- a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml +++ b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml @@ -56,6 +56,7 @@ properties: - idt,5p49v5935 - idt,5p49v6901 - idt,5p49v6965 + - idt,5p49v6975 reg: description: I2C device address @@ -108,7 +109,7 @@ patternProperties: properties: idt,mode: description: - The output drive mode. Values defined in dt-bindings/clk/versaclock.h + The output drive mode. Values defined in dt-bindings/clock/versaclock.h $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 6 @@ -134,6 +135,7 @@ allOf: enum: - idt,5p49v5933 - idt,5p49v5935 + - idt,5p49v6975 then: # Devices with builtin crystal + optional external input properties: @@ -151,7 +153,7 @@ additionalProperties: false examples: - | - #include + #include /* 25MHz reference crystal */ ref25: ref25m { diff --git a/Documentation/devicetree/bindings/clock/mediatek,apmixedsys.yaml b/Documentation/devicetree/bindings/clock/mediatek,apmixedsys.yaml index 770546195fb5a23f0d5e5e637895f1998f4e4663..731bfe0408c206fa9e9107924a7956cb0c9424cb 100644 --- a/Documentation/devicetree/bindings/clock/mediatek,apmixedsys.yaml +++ b/Documentation/devicetree/bindings/clock/mediatek,apmixedsys.yaml @@ -34,6 +34,7 @@ properties: - mediatek,mt2712-apmixedsys - mediatek,mt6765-apmixedsys - mediatek,mt6779-apmixedsys + - mediatek,mt6795-apmixedsys - mediatek,mt7629-apmixedsys - mediatek,mt8167-apmixedsys - mediatek,mt8183-apmixedsys diff --git a/Documentation/devicetree/bindings/clock/mediatek,mt6795-clock.yaml b/Documentation/devicetree/bindings/clock/mediatek,mt6795-clock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04469eabc8fa44ac17a52331cc4d11dbbedbb559 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mediatek,mt6795-clock.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mediatek,mt6795-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Functional Clock Controller for MT6795 + +maintainers: + - AngeloGioacchino Del Regno + - Chun-Jie Chen + +description: | + The clock architecture in MediaTek like below + PLLs --> + dividers --> + muxes + --> + clock gate + + The devices provide clock gate control in different IP blocks. + +properties: + compatible: + enum: + - mediatek,mt6795-mfgcfg + - mediatek,mt6795-vdecsys + - mediatek,mt6795-vencsys + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + mfgcfg: clock-controller@13000000 { + compatible = "mediatek,mt6795-mfgcfg"; + reg = <0 0x13000000 0 0x1000>; + #clock-cells = <1>; + }; + + vdecsys: clock-controller@16000000 { + compatible = "mediatek,mt6795-vdecsys"; + reg = <0 0x16000000 0 0x1000>; + #clock-cells = <1>; + }; + + vencsys: clock-controller@18000000 { + compatible = "mediatek,mt6795-vencsys"; + reg = <0 0x18000000 0 0x1000>; + #clock-cells = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/mediatek,mt6795-sys-clock.yaml b/Documentation/devicetree/bindings/clock/mediatek,mt6795-sys-clock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..378b761237d3a29d8dd1e749da0420814d233a83 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mediatek,mt6795-sys-clock.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mediatek,mt6795-sys-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek System Clock Controller for MT6795 + +maintainers: + - AngeloGioacchino Del Regno + - Chun-Jie Chen + +description: + The Mediatek system clock controller provides various clocks and system + configuration like reset and bus protection on MT6795. + +properties: + compatible: + items: + - enum: + - mediatek,mt6795-apmixedsys + - mediatek,mt6795-infracfg + - mediatek,mt6795-pericfg + - mediatek,mt6795-topckgen + - const: syscon + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + topckgen: clock-controller@10000000 { + compatible = "mediatek,mt6795-topckgen", "syscon"; + reg = <0 0x10000000 0 0x1000>; + #clock-cells = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/mediatek,mt7621-sysc.yaml b/Documentation/devicetree/bindings/clock/mediatek,mt7621-sysc.yaml index 0c0b0ae5e2acc3813f2a94b98f7d5035df62bf7e..b42f0f5c11b7ac362ec2c0be9f8a2ec8db5e1133 100644 --- a/Documentation/devicetree/bindings/clock/mediatek,mt7621-sysc.yaml +++ b/Documentation/devicetree/bindings/clock/mediatek,mt7621-sysc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/mediatek,mt7621-sysc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MT7621 Clock Device Tree Bindings +title: MT7621 Clock maintainers: - Sergio Paracuellos diff --git a/Documentation/devicetree/bindings/clock/mediatek,mt8365-clock.yaml b/Documentation/devicetree/bindings/clock/mediatek,mt8365-clock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b327ecb4e5246fc7030d7fd10bd973d6a4d88bd1 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mediatek,mt8365-clock.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mediatek,mt8365-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Functional Clock Controller for MT8365 + +maintainers: + - Markus Schneider-Pargmann + +properties: + compatible: + items: + - enum: + - mediatek,mt8365-apu + - mediatek,mt8365-imgsys + - mediatek,mt8365-mfgcfg + - mediatek,mt8365-vdecsys + - mediatek,mt8365-vencsys + - const: syscon + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + apu: clock-controller@19020000 { + compatible = "mediatek,mt8365-apu", "syscon"; + reg = <0x19020000 0x1000>; + #clock-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/clock/mediatek,mt8365-sys-clock.yaml b/Documentation/devicetree/bindings/clock/mediatek,mt8365-sys-clock.yaml new file mode 100644 index 0000000000000000000000000000000000000000..643f84660c8e6d10847d6d23dd4523dc6eb31e12 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mediatek,mt8365-sys-clock.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mediatek,mt8365-sys-clock.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek System Clock Controller for MT8365 + +maintainers: + - Markus Schneider-Pargmann + +description: + The apmixedsys module provides most of PLLs which generated from SoC 26m. + The topckgen provides dividers and muxes which provides the clock source to other IP blocks. + The infracfg_ao and pericfg_ao provides clock gate in peripheral and infrastructure IP blocks. + +properties: + compatible: + items: + - enum: + - mediatek,mt8365-topckgen + - mediatek,mt8365-infracfg + - mediatek,mt8365-apmixedsys + - mediatek,mt8365-pericfg + - mediatek,mt8365-mcucfg + - const: syscon + + reg: + maxItems: 1 + + '#clock-cells': + const: 1 + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + topckgen: clock-controller@10000000 { + compatible = "mediatek,mt8365-topckgen", "syscon"; + reg = <0x10000000 0x1000>; + #clock-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/clock/mediatek,topckgen.yaml b/Documentation/devicetree/bindings/clock/mediatek,topckgen.yaml index 5b8b37a2e594b341b6dbab05936e8eced88a8db3..81531b5b0db79c9cc3a026f0197a1e2574d9ceae 100644 --- a/Documentation/devicetree/bindings/clock/mediatek,topckgen.yaml +++ b/Documentation/devicetree/bindings/clock/mediatek,topckgen.yaml @@ -33,6 +33,7 @@ properties: - mediatek,mt2712-topckgen - mediatek,mt6765-topckgen - mediatek,mt6779-topckgen + - mediatek,mt6795-topckgen - mediatek,mt7629-topckgen - mediatek,mt7986-topckgen - mediatek,mt8167-topckgen diff --git a/Documentation/devicetree/bindings/clock/microchip,mpfs-ccc.yaml b/Documentation/devicetree/bindings/clock/microchip,mpfs-ccc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1770360798ffab620aac1c2542cfb9f9ac01444 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/microchip,mpfs-ccc.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/microchip,mpfs-ccc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip PolarFire SoC Fabric Clock Conditioning Circuitry + +maintainers: + - Conor Dooley + +description: | + Microchip PolarFire SoC has 4 Clock Conditioning Circuitry blocks. Each of + these blocks contains two PLLs and 2 DLLs & are located in the four corners of + the FPGA. For more information see "PolarFire SoC FPGA Clocking Resources" at: + https://onlinedocs.microchip.com/pr/GUID-8F0CC4C0-0317-4262-89CA-CE7773ED1931-en-US-1/index.html + +properties: + compatible: + const: microchip,mpfs-ccc + + reg: + items: + - description: PLL0's control registers + - description: PLL1's control registers + - description: DLL0's control registers + - description: DLL1's control registers + + clocks: + description: + The CCC PLL's have two input clocks. It is required that even if the input + clocks are identical that both are provided. + minItems: 2 + items: + - description: PLL0's refclk0 + - description: PLL0's refclk1 + - description: PLL1's refclk0 + - description: PLL1's refclk1 + - description: DLL0's refclk + - description: DLL1's refclk + + clock-names: + minItems: 2 + items: + - const: pll0_ref0 + - const: pll0_ref1 + - const: pll1_ref0 + - const: pll1_ref1 + - const: dll0_ref + - const: dll1_ref + + '#clock-cells': + const: 1 + description: | + The clock consumer should specify the desired clock by having the clock + ID in its "clocks" phandle cell. + See include/dt-bindings/clock/microchip,mpfs-clock.h for the full list of + PolarFire clock IDs. + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + +additionalProperties: false + +examples: + - | + clock-controller@38100000 { + compatible = "microchip,mpfs-ccc"; + reg = <0x38010000 0x1000>, <0x38020000 0x1000>, + <0x39010000 0x1000>, <0x39020000 0x1000>; + #clock-cells = <1>; + clocks = <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, <&refclk_ccc>, + <&refclk_ccc>, <&refclk_ccc>; + clock-names = "pll0_ref0", "pll0_ref1", "pll1_ref0", "pll1_ref1", + "dll0_ref", "dll1_ref"; + }; diff --git a/Documentation/devicetree/bindings/clock/microchip,mpfs.yaml b/Documentation/devicetree/bindings/clock/microchip,mpfs-clkcfg.yaml similarity index 73% rename from Documentation/devicetree/bindings/clock/microchip,mpfs.yaml rename to Documentation/devicetree/bindings/clock/microchip,mpfs-clkcfg.yaml index 016a4f378b9b67ffefedc871257c44b86990b88a..b2ce78722247cd4b6a8fef78ac2abd6dc1bc8233 100644 --- a/Documentation/devicetree/bindings/clock/microchip,mpfs.yaml +++ b/Documentation/devicetree/bindings/clock/microchip,mpfs-clkcfg.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/clock/microchip,mpfs.yaml# +$id: http://devicetree.org/schemas/clock/microchip,mpfs-clkcfg.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: Microchip PolarFire Clock Control Module Binding @@ -40,8 +40,21 @@ properties: const: 1 description: | The clock consumer should specify the desired clock by having the clock - ID in its "clocks" phandle cell. See include/dt-bindings/clock/microchip,mpfs-clock.h - for the full list of PolarFire clock IDs. + ID in its "clocks" phandle cell. + See include/dt-bindings/clock/microchip,mpfs-clock.h for the full list of + PolarFire clock IDs. + + resets: + maxItems: 1 + + '#reset-cells': + description: + The AHB/AXI peripherals on the PolarFire SoC have reset support, so from + CLK_ENVM to CLK_CFM. The reset consumer should specify the desired + peripheral via the clock ID in its "resets" phandle cell. + See include/dt-bindings/clock/microchip,mpfs-clock.h for the full list of + PolarFire clock IDs. + const: 1 required: - compatible diff --git a/Documentation/devicetree/bindings/clock/qcom,a53pll.yaml b/Documentation/devicetree/bindings/clock/qcom,a53pll.yaml index fbd758470b8849a181948d75a07d6133815a3c4e..fe6ca4f68bbe6fd63120e94062cee7f2ce47b225 100644 --- a/Documentation/devicetree/bindings/clock/qcom,a53pll.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,a53pll.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Qualcomm A53 PLL Binding maintainers: - - Sivaprakash Murugesan + - Bjorn Andersson description: The A53 PLL on few Qualcomm platforms is the main CPU PLL used used for @@ -17,6 +17,7 @@ properties: compatible: enum: - qcom,ipq6018-a53pll + - qcom,ipq8074-a53pll - qcom,msm8916-a53pll - qcom,msm8939-a53pll diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml index 3cf404c9325a944252aad4ac608da8cc70739c25..6b4efd64c15481c1d97d0db69d63fdc3eab981d7 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-apq8064.yaml @@ -38,6 +38,15 @@ properties: description: child tsens device $ref: /schemas/thermal/qcom-tsens.yaml# + clocks: + maxItems: 3 + + clock-names: + items: + - const: cxo + - const: pxo + - const: pll4 + nvmem-cells: minItems: 1 maxItems: 2 diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09b2ea60d356b46ad33bf5ab81102e1a18546103 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8660.yaml @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-msm8660.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for MSM8660 + +maintainers: + - Stephen Boyd + - Taniya Das + +description: | + Qualcomm global clock control module which supports the clocks and resets on + MSM8660 + + See also: + - dt-bindings/clock/qcom,gcc-msm8660.h + - dt-bindings/reset/qcom,gcc-msm8660.h + +allOf: + - $ref: "qcom,gcc.yaml#" + +properties: + compatible: + enum: + - qcom,gcc-msm8660 + + clocks: + maxItems: 2 + + clock-names: + items: + - const: pxo + - const: cxo + +required: + - compatible + +unevaluatedProperties: false + +examples: + # Example for GCC for MSM8974: + - | + clock-controller@900000 { + compatible = "qcom,gcc-msm8660"; + reg = <0x900000 0x4000>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + clocks = <&pxo_board>, <&cxo_board>; + clock-names = "pxo", "cxo"; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2272ea5f78d0aa9d6750fc2f9080e31a9deb8653 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8909.yaml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-msm8909.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for MSM8909 + +maintainers: + - Stephan Gerhold + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on MSM8909. + + See also: + - dt-bindings/clock/qcom,gcc-msm8909.h + +properties: + compatible: + const: qcom,gcc-msm8909 + + clocks: + items: + - description: XO source + - description: Sleep clock source + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + + clock-names: + items: + - const: xo + - const: sleep_clk + - const: dsi0pll + - const: dsi0pllbyte + +required: + - compatible + - clocks + - clock-names + +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false + +examples: + - | + gcc: clock-controller@1800000 { + compatible = "qcom,gcc-msm8909"; + reg = <0x01800000 0x80000>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + clocks = <&xo_board>, <&sleep_clk>, <&dsi0_phy 1>, <&dsi0_phy 0>; + clock-names = "xo", "sleep_clk", "dsi0pll", "dsi0pllbyte"; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ceb1e501ef99152f58102169150b89ac5bf5146 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8916.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-msm8916.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for MSM8916 and MSM8939 + +maintainers: + - Stephen Boyd + - Taniya Das + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on MSM8916 or MSM8939. + + See also: + - dt-bindings/clock/qcom,gcc-msm8916.h + - dt-bindings/clock/qcom,gcc-msm8939.h + - dt-bindings/reset/qcom,gcc-msm8916.h + - dt-bindings/reset/qcom,gcc-msm8939.h + +properties: + compatible: + enum: + - qcom,gcc-msm8916 + - qcom,gcc-msm8939 + + clocks: + items: + - description: XO source + - description: Sleep clock source + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + - description: External MCLK clock + - description: External Primary I2S clock + - description: External Secondary I2S clock + + clock-names: + items: + - const: xo + - const: sleep_clk + - const: dsi0pll + - const: dsi0pllbyte + - const: ext_mclk + - const: ext_pri_i2s + - const: ext_sec_i2s + +required: + - compatible + +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false + +examples: + - | + clock-controller@300000 { + compatible = "qcom,gcc-msm8916"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + reg = <0x300000 0x90000>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml index f3430b159caabf65c3a01ed559224d920afc7bfa..4b7d6951837146065166b1797b3cdb79f47bbfaf 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8976.yaml @@ -45,29 +45,16 @@ properties: description: Phandle to voltage regulator providing power to the GX domain. - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - - reg - clocks - clock-names - vdd_gfx-supply - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8994.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8994.yaml index 22e67b238bb608baa49056de302d2464451b8fb7..7b9fef6d9b239a54b9a73341b7e1b4bbf259f93c 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8994.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8994.yaml @@ -32,28 +32,15 @@ properties: - const: xo - const: sleep - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8996.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8996.yaml index 005e0edd4609ace460eb306b1c379adc27612abe..dfc5165db9f198debc1dd1290f2c2ccc1e38a2e6 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8996.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8996.yaml @@ -49,30 +49,13 @@ properties: - const: ufs_rx_symbol_1_clk_src - const: ufs_tx_symbol_0_clk_src - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8998.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8998.yaml index 8151c0a056493fdeec00edb9890e3d1a4f8dd9fc..544a2335cf05d3afda6e649fe1d41b95e6391777 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-msm8998.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-msm8998.yaml @@ -37,32 +37,15 @@ properties: - const: core_bi_pll_test_se # Optional clock minItems: 2 - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-other.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-other.yaml index 6c78df0c46a9043afaf791da8cc0c72ef2ed4c81..76988e04c7db98cbbff1c96f34e1b001169ebfa1 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-other.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-other.yaml @@ -18,11 +18,7 @@ description: | - dt-bindings/clock/qcom,gcc-ipq4019.h - dt-bindings/clock/qcom,gcc-ipq6018.h - dt-bindings/reset/qcom,gcc-ipq6018.h - - dt-bindings/clock/qcom,gcc-msm8939.h - dt-bindings/clock/qcom,gcc-msm8953.h - - dt-bindings/reset/qcom,gcc-msm8939.h - - dt-bindings/clock/qcom,gcc-msm8660.h - - dt-bindings/reset/qcom,gcc-msm8660.h - dt-bindings/clock/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974) - dt-bindings/reset/qcom,gcc-msm8974.h (qcom,gcc-msm8226 and qcom,gcc-msm8974) - dt-bindings/clock/qcom,gcc-mdm9607.h @@ -40,9 +36,6 @@ properties: - qcom,gcc-ipq6018 - qcom,gcc-mdm9607 - qcom,gcc-msm8226 - - qcom,gcc-msm8660 - - qcom,gcc-msm8916 - - qcom,gcc-msm8939 - qcom,gcc-msm8953 - qcom,gcc-msm8974 - qcom,gcc-msm8974pro diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-qcm2290.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-qcm2290.yaml index 5de9c826313868292235a9e33b6b882d81223727..aec37e3f5e3057a9904aa5f51bcd4524967c1cc2 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-qcm2290.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-qcm2290.yaml @@ -30,32 +30,15 @@ properties: - const: bi_tcxo - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7180.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7180.yaml index a404c8fbee675703f552c9731c6a659b0f5e0eb6..e4d490e65d141c1902d2df1a95174d4e7d3c68e3 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7180.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7180.yaml @@ -33,32 +33,15 @@ properties: - const: bi_tcxo_ao - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml index 5693b8997570bd4adb9b4c68be710aea2402c9d1..ea61367e5abc6c8502390515ded51be553e06aa0 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml @@ -44,28 +44,15 @@ properties: - const: ufs_phy_tx_symbol_0_clk - const: usb3_phy_wrapper_gcc_usb30_pipe_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml index f03ef96e57fa8d8a5b832a070d29db601877fec2..30b5d1215fa8fa71f10899a815319d1011245b9b 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml @@ -32,32 +32,15 @@ properties: - const: bi_tcxo_ao - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8280xp.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8280xp.yaml index 0bcdc69c6f8964811206b90d1727fcfc32ef7306..b1bf768530a341d29b86a43c4ab6fb85473018e2 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8280xp.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8280xp.yaml @@ -33,7 +33,7 @@ properties: - description: Primary USB SuperSpeed pipe clock - description: USB4 PHY pipegmux clock source - description: USB4 PHY DP gmux clock source - - description: USB4 PHY sys piegmux clock source + - description: USB4 PHY sys pipegmux clock source - description: USB4 PHY PCIe pipe clock - description: USB4 PHY router max pipe clock - description: Primary USB4 RX0 clock @@ -46,7 +46,7 @@ properties: - description: Second USB4 PHY router max pipe clock - description: Secondary USB4 RX0 clock - description: Secondary USB4 RX1 clock - - description: Multiport USB first SupserSpeed pipe clock + - description: Multiport USB first SuperSpeed pipe clock - description: Multiport USB second SuperSpeed pipe clock - description: PCIe 2a pipe clock - description: PCIe 2b pipe clock @@ -56,30 +56,17 @@ properties: - description: First EMAC controller reference clock - description: Second EMAC controller reference clock - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - protected-clocks: maxItems: 389 required: - compatible - clocks - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml index daf7906ebc409cec0e4e0266bf1ebb47d9d46119..e169d46c78f87069b214b998a79d2f6287f52f6e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdm845.yaml @@ -19,51 +19,67 @@ description: | properties: compatible: - const: qcom,gcc-sdm845 + enum: + - qcom,gcc-sdm670 + - qcom,gcc-sdm845 clocks: - items: - - description: Board XO source - - description: Board active XO source - - description: Sleep clock source - - description: PCIE 0 Pipe clock source - - description: PCIE 1 Pipe clock source + minItems: 3 + maxItems: 5 clock-names: - items: - - const: bi_tcxo - - const: bi_tcxo_ao - - const: sleep_clk - - const: pcie_0_pipe_clk - - const: pcie_1_pipe_clk - - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 + minItems: 3 + maxItems: 5 power-domains: maxItems: 1 - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + - if: + properties: + compatible: + contains: + const: qcom,gcc-sdm670 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + + - if: + properties: + compatible: + contains: + const: qcom,gcc-sdm845 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + - description: PCIE 0 Pipe clock source + - description: PCIE 1 Pipe clock source + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + - const: pcie_0_pipe_clk + - const: pcie_1_pipe_clk + +unevaluatedProperties: false examples: # Example for GCC for SDM845: diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdx55.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdx55.yaml index b0d1c65aa35484218d4525e083b58c365b84cc5b..13ffa16e0833164ad48132420f55b63c48c21b1a 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdx55.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdx55.yaml @@ -35,28 +35,15 @@ properties: - const: core_bi_pll_test_se # Optional clock minItems: 2 - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sdx65.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sdx65.yaml index 16c4cdc7b4d63d2eff18171a79803839057c66af..8a1419c4d465d3530c9ecf6003c3c244f8f536aa 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sdx65.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sdx65.yaml @@ -20,9 +20,6 @@ properties: compatible: const: qcom,gcc-sdx65 - reg: - maxItems: 1 - clocks: items: - description: Board XO source @@ -43,25 +40,15 @@ properties: - const: core_bi_pll_test_se # Optional clock minItems: 5 - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - required: - compatible - - reg - clocks - clock-names - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6115.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6115.yaml index 26050da844d54449f8a3fc351d865dcb65cce1bd..bb81a27a1b163d91fe5b41f5335686e4cc434b0e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6115.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6115.yaml @@ -30,32 +30,15 @@ properties: - const: bi_tcxo - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6125.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6125.yaml index ab12b391effc1fdd69028944f166164f2bfc4c07..03e84e15815ca3f59a1aa8e6a35649feba285940 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6125.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6125.yaml @@ -30,32 +30,15 @@ properties: - const: bi_tcxo - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml index 20926cd8293e9de5601bc4e1845321c58b57e37d..cbe98c01c085f5cec7ca5a246489ded93b798ce5 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm6350.yaml @@ -32,32 +32,15 @@ properties: - const: bi_tcxo_ao - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml index 12766a8666253c209ba6887b2c9962ca16b622ee..0333ccb07d8ded78b4ec00ef68301ca49738a56a 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8150.yaml @@ -31,32 +31,15 @@ properties: - const: bi_tcxo - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml index 80bd6caf5bc9b6a59671421dfea8773c7321f02e..4e2a9cac0a91ec2427cc63c77d4a79179e2e8961 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8250.yaml @@ -31,32 +31,15 @@ properties: - const: bi_tcxo - const: sleep_clk - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - - protected-clocks: - description: - Protected clock specifier list as per common clock binding. - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml index 1122700dcc2b3b7ef20809732e48f9ff45f658d8..3edbeca70a9ce47ad2127bcd42ff71275ad912c3 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml @@ -54,28 +54,15 @@ properties: - const: usb3_uni_phy_sec_gcc_usb30_pipe_clk # Optional clock minItems: 2 - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - clocks - clock-names - - reg - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml index 58d98a766de6cec7bf3a757d9cbe4cff0c88b797..102ce6862e2414a64764f5c7147f8e8e96da82ef 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8450.yaml @@ -46,28 +46,15 @@ properties: - const: usb3_phy_wrapper_gcc_usb30_pipe_clk # Optional clock minItems: 2 - '#clock-cells': - const: 1 - - '#reset-cells': - const: 1 - - '#power-domain-cells': - const: 1 - - reg: - maxItems: 1 - required: - compatible - - reg - clocks - clock-names - - '#clock-cells' - - '#reset-cells' - - '#power-domain-cells' -additionalProperties: false +allOf: + - $ref: qcom,gcc.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml index 9ebcb1943b0a5a99bcd29d66e8a6604fc2a773e0..a7d0af1bd9e0750b9baa8ae2d9c63d4795ed3b6e 100644 --- a/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,gpucc.yaml @@ -17,6 +17,7 @@ description: | dt-bindings/clock/qcom,gpucc-sdm845.h dt-bindings/clock/qcom,gpucc-sc7180.h dt-bindings/clock/qcom,gpucc-sc7280.h + dt-bindings/clock/qcom,gpucc-sc8280xp.h dt-bindings/clock/qcom,gpucc-sm6350.h dt-bindings/clock/qcom,gpucc-sm8150.h dt-bindings/clock/qcom,gpucc-sm8250.h @@ -28,6 +29,7 @@ properties: - qcom,sc7180-gpucc - qcom,sc7280-gpucc - qcom,sc8180x-gpucc + - qcom,sc8280xp-gpucc - qcom,sm6350-gpucc - qcom,sm8150-gpucc - qcom,sm8250-gpucc diff --git a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml index 32e87014bb553705ea4dc0ec6e3a2a67b983de54..03faab5b6a41d63d770801e75acf45fcf8d76597 100644 --- a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml @@ -31,30 +31,12 @@ properties: - qcom,mmcc-sdm660 clocks: - items: - - description: Board XO source - - description: Board sleep source - - description: Global PLL 0 clock - - description: DSI phy instance 0 dsi clock - - description: DSI phy instance 0 byte clock - - description: DSI phy instance 1 dsi clock - - description: DSI phy instance 1 byte clock - - description: HDMI phy PLL clock - - description: DisplayPort phy PLL vco clock - - description: DisplayPort phy PLL link clock + minItems: 8 + maxItems: 10 clock-names: - items: - - const: xo - - const: sleep - - const: gpll0 - - const: dsi0dsi - - const: dsi0byte - - const: dsi1dsi - - const: dsi1byte - - const: hdmipll - - const: dpvco - - const: dplink + minItems: 8 + maxItems: 10 '#clock-cells': const: 1 @@ -85,16 +67,179 @@ required: additionalProperties: false -if: - properties: - compatible: - contains: - const: qcom,mmcc-msm8998 - -then: - required: - - clocks - - clock-names +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,mmcc-apq8064 + - qcom,mmcc-msm8960 + then: + properties: + clocks: + items: + - description: Board PXO source + - description: PLL 3 clock + - description: PLL 3 Vote clock + - description: DSI phy instance 1 dsi clock + - description: DSI phy instance 1 byte clock + - description: DSI phy instance 2 dsi clock + - description: DSI phy instance 2 byte clock + - description: HDMI phy PLL clock + + clock-names: + items: + - const: pxo + - const: pll3 + - const: pll8_vote + - const: dsi1pll + - const: dsi1pllbyte + - const: dsi2pll + - const: dsi2pllbyte + - const: hdmipll + + - if: + properties: + compatible: + contains: + enum: + - qcom,mmcc-msm8994 + - qcom,mmcc-msm8998 + - qcom,mmcc-sdm630 + - qcom,mmcc-sdm660 + then: + required: + - clocks + - clock-names + + - if: + properties: + compatible: + contains: + const: qcom,mmcc-msm8994 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Global PLL 0 clock + - description: MMSS NoC AHB clock + - description: GFX3D clock + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + - description: DSI phy instance 1 dsi clock + - description: DSI phy instance 1 byte clock + - description: HDMI phy PLL clock + + clock-names: + items: + - const: xo + - const: gpll0 + - const: mmssnoc_ahb + - const: oxili_gfx3d_clk_src + - const: dsi0pll + - const: dsi0pllbyte + - const: dsi1pll + - const: dsi1pllbyte + - const: hdmipll + + - if: + properties: + compatible: + contains: + const: qcom,mmcc-msm8996 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Global PLL 0 clock + - description: MMSS NoC AHB clock + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + - description: DSI phy instance 1 dsi clock + - description: DSI phy instance 1 byte clock + - description: HDMI phy PLL clock + + clock-names: + items: + - const: xo + - const: gpll0 + - const: gcc_mmss_noc_cfg_ahb_clk + - const: dsi0pll + - const: dsi0pllbyte + - const: dsi1pll + - const: dsi1pllbyte + - const: hdmipll + + - if: + properties: + compatible: + contains: + const: qcom,mmcc-msm8998 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Global PLL 0 clock + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + - description: DSI phy instance 1 dsi clock + - description: DSI phy instance 1 byte clock + - description: HDMI phy PLL clock + - description: DisplayPort phy PLL link clock + - description: DisplayPort phy PLL vco clock + - description: Test clock + + clock-names: + items: + - const: xo + - const: gpll0 + - const: dsi0dsi + - const: dsi0byte + - const: dsi1dsi + - const: dsi1byte + - const: hdmipll + - const: dplink + - const: dpvco + - const: core_bi_pll_test_se + + - if: + properties: + compatible: + contains: + enum: + - qcom,mmcc-sdm630 + - qcom,mmcc-sdm660 + then: + properties: + clocks: + items: + - description: Board XO source + - description: Board sleep source + - description: Global PLL 0 clock + - description: Global PLL 0 DIV clock + - description: DSI phy instance 0 dsi clock + - description: DSI phy instance 0 byte clock + - description: DSI phy instance 1 dsi clock + - description: DSI phy instance 1 byte clock + - description: DisplayPort phy PLL link clock + - description: DisplayPort phy PLL vco clock + + clock-names: + items: + - const: xo + - const: sleep_clk + - const: gpll0 + - const: gpll0_div + - const: dsi0pll + - const: dsi0pllbyte + - const: dsi1pll + - const: dsi1pllbyte + - const: dp_link_2x_clk_divsel_five + - const: dp_vco_divided_clk_src_mux examples: # Example for MMCC for MSM8960: diff --git a/Documentation/devicetree/bindings/clock/qcom,msm8996-apcc.yaml b/Documentation/devicetree/bindings/clock/qcom,msm8996-apcc.yaml index a20cb10636dd45bb731c0b11e5bead0804eca35a..c4971234fef8b4c70ffd9b5be95f72038b8782fe 100644 --- a/Documentation/devicetree/bindings/clock/qcom,msm8996-apcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,msm8996-apcc.yaml @@ -26,22 +26,18 @@ properties: clocks: items: - - description: Primary PLL clock for power cluster (little) - - description: Primary PLL clock for perf cluster (big) - - description: Alternate PLL clock for power cluster (little) - - description: Alternate PLL clock for perf cluster (big) + - description: XO source clock-names: items: - - const: pwrcl_pll - - const: perfcl_pll - - const: pwrcl_alt_pll - - const: perfcl_alt_pll + - const: xo required: - compatible - reg - '#clock-cells' + - clocks + - clock-names additionalProperties: false @@ -51,4 +47,7 @@ examples: compatible = "qcom,msm8996-apcc"; reg = <0x6400000 0x90000>; #clock-cells = <1>; + + clocks = <&xo_board>; + clock-names = "xo"; }; diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml index d63b45ad06e8a3f03be793257456e5d8e59a134a..2a95bf8664f9fa8c6dd661ae94590aa11a86948c 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmcc.yaml @@ -29,6 +29,7 @@ properties: - qcom,rpmcc-mdm9607 - qcom,rpmcc-msm8226 - qcom,rpmcc-msm8660 + - qcom,rpmcc-msm8909 - qcom,rpmcc-msm8916 - qcom,rpmcc-msm8936 - qcom,rpmcc-msm8953 @@ -43,6 +44,7 @@ properties: - qcom,rpmcc-sdm660 - qcom,rpmcc-sm6115 - qcom,rpmcc-sm6125 + - qcom,rpmcc-sm6375 - const: qcom,rpmcc '#clock-cells': diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index 8fcaf418f84af3283e9e8ef30421ad6ccf3d64e5..437a34b930e37552a10e739ad0468c995f0a0c28 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -21,6 +21,7 @@ properties: - qcom,sc7280-rpmh-clk - qcom,sc8180x-rpmh-clk - qcom,sc8280xp-rpmh-clk + - qcom,sdm670-rpmh-clk - qcom,sdm845-rpmh-clk - qcom,sdx55-rpmh-clk - qcom,sdx65-rpmh-clk diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml index 47028d7b98e4d0b22b4ca9829c8536dda6ce3f16..633887dc2f8a8786a3242bea6ed7d85344cdc4b9 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml @@ -36,13 +36,11 @@ properties: items: - description: LPASS qdsp6ss register - description: LPASS top-cc register - - description: LPASS cc register reg-names: items: - const: qdsp6ss - const: top_cc - - const: cc required: - compatible @@ -59,8 +57,8 @@ examples: #include clock-controller@3000000 { compatible = "qcom,sc7280-lpasscc"; - reg = <0x03000000 0x40>, <0x03c04000 0x4>, <0x03389000 0x24>; - reg-names = "qdsp6ss", "top_cc", "cc"; + reg = <0x03000000 0x40>, <0x03c04000 0x4>; + reg-names = "qdsp6ss", "top_cc"; clocks = <&gcc GCC_CFG_NOC_LPASS_CLK>; clock-names = "iface"; #clock-cells = <1>; diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml index bad9135489de5538d4043ac2d45bcff61a43486e..f50e284e5f4636b15df21d7cffcd3f74207eacdd 100644 --- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml @@ -22,6 +22,8 @@ properties: clock-names: true + reg: true + compatible: enum: - qcom,sc7280-lpassaoncc @@ -38,8 +40,14 @@ properties: '#power-domain-cells': const: 1 - reg: - maxItems: 1 + '#reset-cells': + const: 1 + + qcom,adsp-pil-mode: + description: + Indicates if the LPASS would be brought out of reset using + peripheral loader. + type: boolean required: - compatible @@ -69,6 +77,11 @@ allOf: items: - const: bi_tcxo - const: lpass_aon_cc_main_rcg_clk_src + + reg: + items: + - description: lpass core cc register + - description: lpass audio csr register - if: properties: compatible: @@ -90,6 +103,8 @@ allOf: - const: bi_tcxo_ao - const: iface + reg: + maxItems: 1 - if: properties: compatible: @@ -108,6 +123,8 @@ allOf: items: - const: bi_tcxo + reg: + maxItems: 1 examples: - | #include @@ -116,13 +133,15 @@ examples: #include lpass_audiocc: clock-controller@3300000 { compatible = "qcom,sc7280-lpassaudiocc"; - reg = <0x3300000 0x30000>; + reg = <0x3300000 0x30000>, + <0x32a9000 0x1000>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&lpass_aon LPASS_AON_CC_MAIN_RCG_CLK_SRC>; clock-names = "bi_tcxo", "lpass_aon_cc_main_rcg_clk_src"; power-domains = <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>; #clock-cells = <1>; #power-domain-cells = <1>; + #reset-cells = <1>; }; - | @@ -165,6 +184,7 @@ examples: clocks = <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>, <&lpasscore LPASS_CORE_CC_CORE_CLK>; clock-names = "bi_tcxo", "bi_tcxo_ao","iface"; + qcom,adsp-pil-mode; #clock-cells = <1>; #power-domain-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/clock/qcom,sm6115-dispcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm6115-dispcc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6660ff16ad1bb6c5f348359cf3a2e15c67d6c6ca --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,sm6115-dispcc.yaml @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,sm6115-dispcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Display Clock Controller for SM6115 + +maintainers: + - Bjorn Andersson + +description: | + Qualcomm display clock control module which supports the clocks and + power domains on SM6115. + + See also: + include/dt-bindings/clock/qcom,sm6115-dispcc.h + +properties: + compatible: + enum: + - qcom,sm6115-dispcc + + clocks: + items: + - description: Board XO source + - description: Board sleep clock + - description: Byte clock from DSI PHY0 + - description: Pixel clock from DSI PHY0 + - description: GPLL0 DISP DIV clock from GCC + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + #include + clock-controller@5f00000 { + compatible = "qcom,sm6115-dispcc"; + reg = <0x5f00000 0x20000>; + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, + <&sleep_clk>, + <&dsi0_phy 0>, + <&dsi0_phy 1>, + <&gcc GCC_DISP_GPLL0_DIV_CLK_SRC>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c573e1a1257b8deb19941710dd4590937317faf --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,sm6375-gcc.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,sm6375-gcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SM6375 + +maintainers: + - Konrad Dybcio + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SM6375 + + See also: + - dt-bindings/clock/qcom,sm6375-gcc.h + +allOf: + - $ref: qcom,gcc.yaml# + +properties: + compatible: + const: qcom,sm6375-gcc + + clocks: + items: + - description: Board XO source + - description: Board XO Active-Only source + - description: Sleep clock source + +required: + - compatible + - clocks + +unevaluatedProperties: false + +examples: + - | + #include + clock-controller@1400000 { + compatible = "qcom,sm6375-gcc"; + reg = <0x01400000 0x1f0000>; + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, + <&rpmcc RPM_SMD_XO_A_CLK_SRC>, + <&sleep_clk>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8450-dispcc.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8450-dispcc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cc2457f82082eacfc155413e75943c0e9861c91 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,sm8450-dispcc.yaml @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,sm8450-dispcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Display Clock & Reset Controller for SM8450 + +maintainers: + - Dmitry Baryshkov + +description: | + Qualcomm display clock control module which supports the clocks, resets and + power domains on SM8450. + + See also: + include/dt-bindings/clock/qcom,sm8450-dispcc.h + +properties: + compatible: + enum: + - qcom,sm8450-dispcc + + clocks: + minItems: 3 + items: + - description: Board XO source + - description: Board Always On XO source + - description: Display's AHB clock + - description: sleep clock + - description: Byte clock from DSI PHY0 + - description: Pixel clock from DSI PHY0 + - description: Byte clock from DSI PHY1 + - description: Pixel clock from DSI PHY1 + - description: Link clock from DP PHY0 + - description: VCO DIV clock from DP PHY0 + - description: Link clock from DP PHY1 + - description: VCO DIV clock from DP PHY1 + - description: Link clock from DP PHY2 + - description: VCO DIV clock from DP PHY2 + - description: Link clock from DP PHY3 + - description: VCO DIV clock from DP PHY3 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + + power-domains: + description: + A phandle and PM domain specifier for the MMCX power domain. + maxItems: 1 + + required-opps: + description: + A phandle to an OPP node describing required MMCX performance point. + maxItems: 1 + +required: + - compatible + - reg + - clocks + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + #include + #include + clock-controller@af00000 { + compatible = "qcom,sm8450-dispcc"; + reg = <0x0af00000 0x10000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&gcc GCC_DISP_AHB_CLK>, + <&sleep_clk>, + <&dsi0_phy 0>, + <&dsi0_phy 1>, + <&dsi1_phy 0>, + <&dsi1_phy 1>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + power-domains = <&rpmhpd SM8450_MMCX>; + required-opps = <&rpmhpd_opp_low_svs>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml index 6eaabb4d82ec21cbb9bd939dc32a2c1b24d54e7f..81f09df7147ec31c7dd0941f70b4e0146b2d9bd8 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml @@ -47,7 +47,6 @@ properties: maxItems: 1 clocks: - minItems: 4 maxItems: 4 clock-names: @@ -64,7 +63,6 @@ properties: maxItems: 1 resets: - minItems: 2 maxItems: 2 reset-names: diff --git a/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml b/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml index d036675e0779addcead5aff983b8de003c98b6d1..487f74cdc749fcd48f51b19cec10f7a1f471374c 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rzg2l-cpg.yaml @@ -24,7 +24,7 @@ description: | properties: compatible: enum: - - renesas,r9a07g043-cpg # RZ/G2UL{Type-1,Type-2} + - renesas,r9a07g043-cpg # RZ/G2UL{Type-1,Type-2} and RZ/Five - renesas,r9a07g044-cpg # RZ/G2{L,LC} - renesas,r9a07g054-cpg # RZ/V2L - renesas,r9a09g011-cpg # RZ/V2M diff --git a/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d4eb4475fc8b27ee0059e5fd41b3fedd7c2771b --- /dev/null +++ b/Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/renesas,versaclock7.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas Versaclock7 Programmable Clock Device Tree Bindings + +maintainers: + - Alex Helms + +description: | + Renesas Versaclock7 is a family of configurable clock generator and + jitter attenuator ICs with fractional and integer dividers. + +properties: + '#clock-cells': + const: 1 + + compatible: + enum: + - renesas,rc21008a + + reg: + maxItems: 1 + + clocks: + items: + - description: External crystal or oscillator + + clock-names: + items: + - const: xin + +required: + - '#clock-cells' + - compatible + - reg + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + vc7_xin: clock { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <49152000>; + }; + + i2c@0 { + reg = <0x0 0x100>; + #address-cells = <1>; + #size-cells = <0>; + + vc7: clock-controller@9 { + compatible = "renesas,rc21008a"; + reg = <0x9>; + #clock-cells = <1>; + clocks = <&vc7_xin>; + clock-names = "xin"; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/rockchip,px30-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,px30-cru.yaml index 3eec381c7cf54c45065238548793747fdbc967be..0f0f64b6f8cb9a0c01da0c3f3e865d73477d4f0f 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,px30-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,px30-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,px30-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3036-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3036-cru.yaml index 1376230fede60f83bc1ce61b03694e9c9fcd735e..ba5b45464315675d5eb9686a9faf7162afd53598 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3036-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3036-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3036-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.txt b/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.txt deleted file mode 100644 index 6f8744fd301b23609f924873f325b9fc4e0c98a0..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.txt +++ /dev/null @@ -1,58 +0,0 @@ -* Rockchip RK3126/RK3128 Clock and Reset Unit - -The RK3126/RK3128 clock controller generates and supplies clock to various -controllers within the SoC and also implements a reset controller for SoC -peripherals. - -Required Properties: - -- compatible: should be "rockchip,rk3126-cru" or "rockchip,rk3128-cru" - "rockchip,rk3126-cru" - controller compatible with RK3126 SoC. - "rockchip,rk3128-cru" - controller compatible with RK3128 SoC. -- reg: physical base address of the controller and length of memory mapped - region. -- #clock-cells: should be 1. -- #reset-cells: should be 1. - -Optional Properties: - -- rockchip,grf: phandle to the syscon managing the "general register files" - If missing pll rates are not changeable, due to the missing pll lock status. - -Each clock is assigned an identifier and client nodes can use this identifier -to specify the clock which they consume. All available clocks are defined as -preprocessor macros in the dt-bindings/clock/rk3128-cru.h headers and can be -used in device tree sources. Similar macros exist for the reset sources in -these files. - -External clocks: - -There are several clocks that are generated outside the SoC. It is expected -that they are defined using standard clock bindings with following -clock-output-names: - - "xin24m" - crystal input - required, - - "ext_i2s" - external I2S clock - optional, - - "gmac_clkin" - external GMAC clock - optional - -Example: Clock controller node: - - cru: cru@20000000 { - compatible = "rockchip,rk3128-cru"; - reg = <0x20000000 0x1000>; - rockchip,grf = <&grf>; - - #clock-cells = <1>; - #reset-cells = <1>; - }; - -Example: UART controller node that consumes the clock generated by the clock - controller: - - uart2: serial@20068000 { - compatible = "rockchip,serial"; - reg = <0x20068000 0x100>; - interrupts = ; - clock-frequency = <24000000>; - clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; - clock-names = "sclk_uart", "pclk_uart"; - }; diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3d9c8eca989f0d0213bbe219071cd28f10b7257 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3128-cru.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/rockchip,rk3128-cru.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip RK3126/RK3128 Clock and Reset Unit (CRU) + +maintainers: + - Elaine Zhang + - Heiko Stuebner + +description: | + The RK3126/RK3128 clock controller generates and supplies clock to various + controllers within the SoC and also implements a reset controller for SoC + peripherals. + Each clock is assigned an identifier and client nodes can use this identifier + to specify the clock which they consume. All available clocks are defined as + preprocessor macros in the dt-bindings/clock/rk3128-cru.h headers and can be + used in device tree sources. Similar macros exist for the reset sources in + these files. + +properties: + compatible: + enum: + - rockchip,rk3126-cru + - rockchip,rk3128-cru + + reg: + maxItems: 1 + + "#clock-cells": + const: 1 + + "#reset-cells": + const: 1 + + clocks: + minItems: 1 + maxItems: 3 + + clock-names: + minItems: 1 + items: + - const: xin24m + - enum: + - ext_i2s + - gmac_clkin + - enum: + - ext_i2s + - gmac_clkin + + rockchip,grf: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the syscon managing the "general register files" (GRF), + if missing pll rates are not changeable, due to the missing pll + lock status. + +required: + - compatible + - reg + - "#clock-cells" + - "#reset-cells" + +additionalProperties: false + +examples: + - | + cru: clock-controller@20000000 { + compatible = "rockchip,rk3128-cru"; + reg = <0x20000000 0x1000>; + rockchip,grf = <&grf>; + #clock-cells = <1>; + #reset-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3228-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3228-cru.yaml index cf7dc01d9478500a8355131e6be30897a1673b87..1050fff72ade18d64a9b7c453ed79c63791f98a5 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3228-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3228-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3228-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3288-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3288-cru.yaml index 96bc05749e1a4f46ce39272d1a014a26c389f53d..6655e97d52e43ded2f527da42c1d26cf34d50bdf 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3288-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3288-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3288-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3308-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3308-cru.yaml index 523ee578a586a955ca3a66d901ce3aedef82a35e..fec37f5b80f6d549f7ef9c9bc0444a702cdb3841 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3308-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3308-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3308-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3368-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3368-cru.yaml index adb67877720da46ae1f8c84dbc75d26368194c0d..90af242b41c1d228caa70505360e10ecc6d518c7 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3368-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3368-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3368-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rk3399-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rk3399-cru.yaml index 54da1e31ea7386081fc599e811308efff9569aa9..0b758e015ee319f567a12fa4527174feb5c566b5 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rk3399-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rk3399-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rk3399-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rv1108-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rv1108-cru.yaml index 20421c22f1846dd8868d6189d7b15dc86fa998a4..4611d920b8df46eb6f240f9866459f293a589aa1 100644 --- a/Documentation/devicetree/bindings/clock/rockchip,rv1108-cru.yaml +++ b/Documentation/devicetree/bindings/clock/rockchip,rv1108-cru.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: (GPL-2.0+ OR MIT) %YAML 1.2 --- $id: http://devicetree.org/schemas/clock/rockchip,rv1108-cru.yaml# diff --git a/Documentation/devicetree/bindings/clock/rockchip,rv1126-cru.yaml b/Documentation/devicetree/bindings/clock/rockchip,rv1126-cru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0998f8b922bdf581638e8a08b868005dd31f5576 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/rockchip,rv1126-cru.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/rockchip,rv1126-cru.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip RV1126 Clock and Reset Unit + +maintainers: + - Jagan Teki + - Finley Xiao + - Heiko Stuebner + +description: + The RV1126 clock controller generates the clock and also implements a + reset controller for SoC peripherals. + +properties: + compatible: + enum: + - rockchip,rv1126-cru + - rockchip,rv1126-pmucru + + reg: + maxItems: 1 + + "#clock-cells": + const: 1 + + "#reset-cells": + const: 1 + + clocks: + maxItems: 1 + + clock-names: + const: xin24m + + rockchip,grf: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the syscon managing the "general register files" (GRF), + if missing pll rates are not changeable, due to the missing pll + lock status. + +required: + - compatible + - reg + - "#clock-cells" + - "#reset-cells" + +additionalProperties: false + +examples: + - | + cru: clock-controller@ff490000 { + compatible = "rockchip,rv1126-cru"; + reg = <0xff490000 0x1000>; + rockchip,grf = <&grf>; + #clock-cells = <1>; + #reset-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml index aa11815ad3a308fbbba4686177f1e4acdc9909c9..141cf173f87d48f4ee9bcc7d42ef468ce14402f2 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml @@ -33,10 +33,13 @@ properties: enum: - samsung,exynos850-cmu-top - samsung,exynos850-cmu-apm + - samsung,exynos850-cmu-aud - samsung,exynos850-cmu-cmgp - samsung,exynos850-cmu-core - samsung,exynos850-cmu-dpu - samsung,exynos850-cmu-hsi + - samsung,exynos850-cmu-is + - samsung,exynos850-cmu-mfcmscl - samsung,exynos850-cmu-peri clocks: @@ -88,6 +91,24 @@ allOf: - const: oscclk - const: dout_clkcmu_apm_bus + - if: + properties: + compatible: + contains: + const: samsung,exynos850-cmu-aud + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: AUD clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_aud + - if: properties: compatible: @@ -172,6 +193,54 @@ allOf: - const: dout_hsi_mmc_card - const: dout_hsi_usb20drd + - if: + properties: + compatible: + contains: + const: samsung,exynos850-cmu-is + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_IS bus clock (from CMU_TOP) + - description: Image Texture Processing core clock (from CMU_TOP) + - description: Visual Recognition Accelerator clock (from CMU_TOP) + - description: Geometric Distortion Correction clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_is_bus + - const: dout_is_itp + - const: dout_is_vra + - const: dout_is_gdc + + - if: + properties: + compatible: + contains: + const: samsung,exynos850-cmu-mfcmscl + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: Multi-Format Codec clock (from CMU_TOP) + - description: Memory to Memory Scaler clock (from CMU_TOP) + - description: Multi-Channel Scaler clock (from CMU_TOP) + - description: JPEG codec clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_mfcmscl_mfc + - const: dout_mfcmscl_m2m + - const: dout_mfcmscl_mcsc + - const: dout_mfcmscl_jpeg + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml index eafc715d2d02cb9d083d5709d353ab77a37af056..2ab4642679c06720ec31c31dfda6d219f5f195d6 100644 --- a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml @@ -35,6 +35,8 @@ properties: - samsung,exynosautov9-cmu-top - samsung,exynosautov9-cmu-busmc - samsung,exynosautov9-cmu-core + - samsung,exynosautov9-cmu-fsys0 + - samsung,exynosautov9-cmu-fsys1 - samsung,exynosautov9-cmu-fsys2 - samsung,exynosautov9-cmu-peric0 - samsung,exynosautov9-cmu-peric1 @@ -107,6 +109,48 @@ allOf: - const: oscclk - const: dout_clkcmu_core_bus + - if: + properties: + compatible: + contains: + const: samsung,exynosautov9-cmu-fsys0 + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_FSYS0 bus clock (from CMU_TOP) + - description: CMU_FSYS0 pcie clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_clkcmu_fsys0_bus + - const: dout_clkcmu_fsys0_pcie + + - if: + properties: + compatible: + contains: + const: samsung,exynosautov9-cmu-fsys1 + + then: + properties: + clocks: + items: + - description: External reference clock (26 MHz) + - description: CMU_FSYS1 bus clock (from CMU_TOP) + - description: CMU_FSYS1 mmc card clock (from CMU_TOP) + - description: CMU_FSYS1 usb clock (from CMU_TOP) + + clock-names: + items: + - const: oscclk + - const: dout_clkcmu_fsys1_bus + - const: dout_clkcmu_fsys1_mmc_card + - const: dout_clkcmu_fsys1_usbdrd + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/clock/samsung,s2mps11.yaml b/Documentation/devicetree/bindings/clock/samsung,s2mps11.yaml index 9248bfc16d484a12b2a8fb37ab87f11a15833a42..d5296e6053a1881650b8e8ff2524ea01689b7395 100644 --- a/Documentation/devicetree/bindings/clock/samsung,s2mps11.yaml +++ b/Documentation/devicetree/bindings/clock/samsung,s2mps11.yaml @@ -34,7 +34,6 @@ properties: const: 1 clock-output-names: - minItems: 3 maxItems: 3 description: Names for AP, CP and BT clocks. diff --git a/Documentation/devicetree/bindings/clock/sprd,sc9863a-clk.yaml b/Documentation/devicetree/bindings/clock/sprd,sc9863a-clk.yaml index 47e1ab08c95d18f2909c7af81be42ba87a7a1884..785a12797a42d1f764a4535adb9e2438b466d1da 100644 --- a/Documentation/devicetree/bindings/clock/sprd,sc9863a-clk.yaml +++ b/Documentation/devicetree/bindings/clock/sprd,sc9863a-clk.yaml @@ -5,7 +5,7 @@ $id: "http://devicetree.org/schemas/clock/sprd,sc9863a-clk.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: SC9863A Clock Control Unit Device Tree Bindings +title: SC9863A Clock Control Unit maintainers: - Orson Zhai diff --git a/Documentation/devicetree/bindings/clock/ti/gate.txt b/Documentation/devicetree/bindings/clock/ti/gate.txt index b4820b1de4f01be8640c6c33ee387b0d95c2549e..4982615c01b9cb7fd187828e2c1d03a6c1d59255 100644 --- a/Documentation/devicetree/bindings/clock/ti/gate.txt +++ b/Documentation/devicetree/bindings/clock/ti/gate.txt @@ -10,7 +10,7 @@ will be controlled instead and the corresponding hw-ops for that is used. [1] Documentation/devicetree/bindings/clock/clock-bindings.txt -[2] Documentation/devicetree/bindings/clock/gpio-gate-clock.txt +[2] Documentation/devicetree/bindings/clock/gpio-gate-clock.yaml [3] Documentation/devicetree/bindings/clock/ti/clockdomain.txt Required properties: diff --git a/Documentation/devicetree/bindings/clock/ti/interface.txt b/Documentation/devicetree/bindings/clock/ti/interface.txt index 94ec77dc3c59aabbe889f13a9b10e05d9f54dede..d3eb5ca92a7fe6e349f974a97f9eeb4c721e5304 100644 --- a/Documentation/devicetree/bindings/clock/ti/interface.txt +++ b/Documentation/devicetree/bindings/clock/ti/interface.txt @@ -9,7 +9,7 @@ companion clock finding (match corresponding functional gate clock) and hardware autoidle enable / disable. [1] Documentation/devicetree/bindings/clock/clock-bindings.txt -[2] Documentation/devicetree/bindings/clock/gpio-gate-clock.txt +[2] Documentation/devicetree/bindings/clock/gpio-gate-clock.yaml Required properties: - compatible : shall be one of: diff --git a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml index 7b7300ce96d637ff3fd4ba65b24ff40ff820e0cc..d36558aa39f335338270dd7358c58e12b1e7788c 100644 --- a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml +++ b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pipllct.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/toshiba,tmpv770x-pipllct.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Toshiba Visconti5 TMPV770X PLL Controller Device Tree Bindings +title: Toshiba Visconti5 TMPV770X PLL Controller maintainers: - Nobuhiro Iwamatsu diff --git a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml index ed79f16fe6bcb17ef419117ae913d2617f4b4ece..081f85b1eb88abee426148f9b1d19b66a0466529 100644 --- a/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml +++ b/Documentation/devicetree/bindings/clock/toshiba,tmpv770x-pismu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/clock/toshiba,tmpv770x-pismu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Toshiba Visconti5 TMPV770x SMU controller Device Tree Bindings +title: Toshiba Visconti5 TMPV770x SMU controller maintainers: - Nobuhiro Iwamatsu diff --git a/Documentation/devicetree/bindings/clock/xlnx,clocking-wizard.yaml b/Documentation/devicetree/bindings/clock/xlnx,clocking-wizard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..634b7b964606cc7b7de2d9d88a858586f383c456 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/xlnx,clocking-wizard.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/clock/xlnx,clocking-wizard.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Xilinx clocking wizard + +maintainers: + - Shubhrajyoti Datta + +description: + The clocking wizard is a soft ip clocking block of Xilinx versal. It + reads required input clock frequencies from the devicetree and acts as clock + clock output. + +properties: + compatible: + enum: + - xlnx,clocking-wizard + - xlnx,clocking-wizard-v5.2 + - xlnx,clocking-wizard-v6.0 + + + reg: + maxItems: 1 + + "#clock-cells": + const: 1 + + clocks: + items: + - description: clock input + - description: axi clock + + clock-names: + items: + - const: clk_in1 + - const: s_axi_aclk + + + xlnx,speed-grade: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2, 3] + description: + Speed grade of the device. Higher the speed grade faster is the FPGA device. + + xlnx,nr-outputs: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 8 + description: + Number of outputs. + +required: + - compatible + - reg + - "#clock-cells" + - clocks + - clock-names + - xlnx,speed-grade + - xlnx,nr-outputs + +additionalProperties: false + +examples: + - | + clock-controller@b0000000 { + compatible = "xlnx,clocking-wizard"; + reg = <0xb0000000 0x10000>; + #clock-cells = <1>; + xlnx,speed-grade = <1>; + xlnx,nr-outputs = <6>; + clock-names = "clk_in1", "s_axi_aclk"; + clocks = <&clkc 15>, <&clkc 15>; + }; +... diff --git a/Documentation/devicetree/bindings/counter/ti,am62-ecap-capture.yaml b/Documentation/devicetree/bindings/counter/ti,am62-ecap-capture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e0b2d2b303e4f34df05041c973c1c09b26cdeed --- /dev/null +++ b/Documentation/devicetree/bindings/counter/ti,am62-ecap-capture.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/counter/ti,am62-ecap-capture.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments Enhanced Capture (eCAP) Module + +maintainers: + - Julien Panis + +description: | + The eCAP module resources can be used to capture timestamps + on input signal events (falling/rising edges). + +properties: + compatible: + const: ti,am62-ecap-capture + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: fck + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + capture@23100000 { /* eCAP in capture mode on am62x */ + compatible = "ti,am62-ecap-capture"; + reg = <0x00 0x23100000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 51 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 51 0>; + clock-names = "fck"; + }; + }; diff --git a/Documentation/devicetree/bindings/crypto/allwinner,sun4i-a10-crypto.yaml b/Documentation/devicetree/bindings/crypto/allwinner,sun4i-a10-crypto.yaml index dedc99e34ebcf9bbac4c8ec5dda503fdc20db1e7..0401c11da8d94f93670632e8895f14df5d177ce6 100644 --- a/Documentation/devicetree/bindings/crypto/allwinner,sun4i-a10-crypto.yaml +++ b/Documentation/devicetree/bindings/crypto/allwinner,sun4i-a10-crypto.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/crypto/allwinner,sun4i-a10-crypto.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Security System Device Tree Bindings +title: Allwinner A10 Security System maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml b/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a772d232de09baa94cfd2768aaa69f115a1af5dc --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/aspeed,ast2500-hace.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ASPEED HACE hash and crypto Hardware Accelerator Engines + +maintainers: + - Neal Liu + +description: | + The Hash and Crypto Engine (HACE) is designed to accelerate the throughput + of hash data digest, encryption, and decryption. Basically, HACE can be + divided into two independently engines - Hash Engine and Crypto Engine. + +properties: + compatible: + enum: + - aspeed,ast2500-hace + - aspeed,ast2600-hace + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + - resets + +additionalProperties: false + +examples: + - | + #include + hace: crypto@1e6d0000 { + compatible = "aspeed,ast2600-hace"; + reg = <0x1e6d0000 0x200>; + interrupts = <4>; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml index ee2c099981b28d58ac2e4e4ac7404cc218f39686..fedd8be56ad6f447a05cd9ec00241602abb98c2a 100644 --- a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-aes.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Keem Bay OCS AES Device Tree Bindings +title: Intel Keem Bay OCS AES maintainers: - Daniele Alessandrelli diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml index a3c16451b1ad71fba33a600956309e89f0f77f1e..2bb95247b64f060dcdaf2cad3ebda83fa7a35beb 100644 --- a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-ecc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Keem Bay OCS ECC Device Tree Bindings +title: Intel Keem Bay OCS ECC maintainers: - Daniele Alessandrelli diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml index acb92706d28096b5e5b3e8ecd34cf07c1ebe88f1..46e2853ab8f4976075bb79f57211e94445e70f59 100644 --- a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-hcu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Keem Bay OCS HCU Device Tree Bindings +title: Intel Keem Bay OCS HCU maintainers: - Declan Murphy diff --git a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml index 676950bb7b37df340ff29e8b937b4ed8c8bb3690..5b31891c97fe247910cab24150e395e9c26d93a4 100644 --- a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml +++ b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml @@ -24,7 +24,6 @@ properties: maxItems: 1 clocks: - minItems: 2 maxItems: 2 clock-names: diff --git a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml index 02f47c2e79987f1975580c34a5f1f3f2235d96bb..0c15fefb6671baa69e1018ab9ff1290e02b4528d 100644 --- a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml +++ b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml @@ -35,8 +35,6 @@ properties: - const: rx1 - const: rx2 - dma-coherent: true - "#address-cells": const: 2 @@ -72,16 +70,6 @@ required: - dmas - dma-names -if: - properties: - compatible: - enum: - - ti,j721e-sa2ul - - ti,am654-sa2ul -then: - required: - - dma-coherent - additionalProperties: false examples: @@ -95,5 +83,4 @@ examples: dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>, <&main_udmap 0x4001>; dma-names = "tx", "rx1", "rx2"; - dma-coherent; }; diff --git a/Documentation/devicetree/bindings/crypto/xlnx,zynqmp-aes.yaml b/Documentation/devicetree/bindings/crypto/xlnx,zynqmp-aes.yaml index 55dd6e3d270de7f7bada473a3bc9b71e8cb3059e..9e8fbd02b150e6665ff26ee989f9c6938b34d764 100644 --- a/Documentation/devicetree/bindings/crypto/xlnx,zynqmp-aes.yaml +++ b/Documentation/devicetree/bindings/crypto/xlnx,zynqmp-aes.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/crypto/xlnx,zynqmp-aes.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx ZynqMP AES-GCM Hardware Accelerator Device Tree Bindings +title: Xilinx ZynqMP AES-GCM Hardware Accelerator maintainers: - Kalyani Akula diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml index 3d8ea3c2d8ddd57d39b9a624c1af4e63d8699ea8..ba06d1857b7d2e633f24ea2b186fe417a0af2231 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-display-backend.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Display Engine Backend Device Tree Bindings +title: Allwinner A10 Display Engine Backend maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml index c9c346e6228e9a9bdf9e9ae835f088d136182ac2..e6088f379f7042146cd2c4b15cca7824a71f799c 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-engine.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-display-engine.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Display Engine Pipeline Device Tree Bindings +title: Allwinner A10 Display Engine Pipeline maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml index 055157fbf3bf935fdceb1cebf90191f86d1595c7..98e8240a05bd9a71df7da5588131735dc6e30267 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-display-frontend.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Display Engine Frontend Device Tree Bindings +title: Allwinner A10 Display Engine Frontend maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml index 7f11452539f413e17627d5c098cef77ecc4388df..55703caacb9c482980b5d48ffdab944e84eb2907 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-hdmi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 HDMI Controller Device Tree Bindings +title: Allwinner A10 HDMI Controller description: | The HDMI Encoder supports the HDMI video and audio outputs, and does diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml index f8168986a0a9eaf02b33fce13a8eddaed57c087d..724d93b9193bc788b180c5306553fd40fd8fb99d 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-tcon.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Timings Controller (TCON) Device Tree Bindings +title: Allwinner A10 Timings Controller (TCON) maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml index afc0ed799e0ed20200c87735769a27c827c27f3c..c39e90a5945f9a64b5fccf063457bfa82eb07c2f 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun4i-a10-tv-encoder.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 TV Encoder Device Tree Bindings +title: Allwinner A10 TV Encoder maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml index 71cce5687580612c9cc8a79470b64754a125533d..895506d93f4cf18b26347f1434d001d4fcb53b50 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun6i-a31-drc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 Dynamic Range Controller Device Tree Bindings +title: Allwinner A31 Dynamic Range Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml index bf0bdf54e5f9e487f75486aab6a9a4b0a7139791..7910831fa4b82b0391af4cbd87b600fac45537cf 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun6i-a31-mipi-dsi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 MIPI-DSI Controller Device Tree Bindings +title: Allwinner A31 MIPI-DSI Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml index cb243bc58ef7510220afbfc89fadaa7966692130..b75c1ec686ad2b64791d2eff12980e392b1b559d 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun8i-a83t-de2-mixer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner Display Engine 2.0 Mixer Device Tree Bindings +title: Allwinner Display Engine 2.0 Mixer maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml index 4951b5ef5c6af41d9040e7867ca21214ad425555..60fd927b5a0664240b1c07def7f7ecbe79ee0118 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun8i-a83t-dw-hdmi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83t DWC HDMI TX Encoder Device Tree Bindings +title: Allwinner A83t DWC HDMI TX Encoder description: | The HDMI transmitter is a Synopsys DesignWare HDMI 1.4 TX controller diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-hdmi-phy.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-hdmi-phy.yaml index a97366aaf924f0701c56ccb8b5df8be329405fd4..1b47f3d99a78e5c79ac68bc768f395a9ccc1b240 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-hdmi-phy.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-hdmi-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun8i-a83t-hdmi-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83t HDMI PHY Device Tree Bindings +title: Allwinner A83t HDMI PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml index 845e226d7afffab81b3562906b1412cb4dc69724..7d849c4095a36c3859268299c84881ad29cff0b8 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun8i-r40-tcon-top.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner R40 TCON TOP Device Tree Bindings +title: Allwinner R40 TCON TOP maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml index 637372ec4614d42a1211532c628f35d8754f5a88..193afee2c3c16ab3ec0f5d4da63f46643100502e 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/allwinner,sun9i-a80-deu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 Detail Enhancement Unit Device Tree Bindings +title: Allwinner A80 Detail Enhancement Unit maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml index 2e208d2fc98f22e6c60716863e1ba30d8d500495..7cdffdb131ac4ca0b8a93d7133b53d595a2ec0b6 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic specific extensions to the Synopsys Designware HDMI Controller maintainers: - - Neil Armstrong + - Neil Armstrong allOf: - $ref: /schemas/sound/name-prefix.yaml# diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml index 047fd69e0377006da7c112d41ddae8aad19ef171..6655a93b187402d4809a47bb00db22290e954ba8 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson Display Controller maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Amlogic Meson Display controller is composed of several components diff --git a/Documentation/devicetree/bindings/display/arm,komeda.yaml b/Documentation/devicetree/bindings/display/arm,komeda.yaml index 9f4aade97f10a10a98acc08e8d808ff30c283361..3ad3eef89ca8d7c2b0b0b67fbaf4c5ad63801c73 100644 --- a/Documentation/devicetree/bindings/display/arm,komeda.yaml +++ b/Documentation/devicetree/bindings/display/arm,komeda.yaml @@ -58,6 +58,7 @@ properties: patternProperties: '^pipeline@[01]$': type: object + additionalProperties: false description: clocks diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml index a9d34dd7bbc53906ccbad5825a975f6b2e16185c..5b35adf34c7bd0c993ced68a72aae30f8293526d 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/brcm,bcm2711-hdmi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM2711 HDMI Controller Device Tree Bindings +title: Broadcom BCM2711 HDMI Controller maintainers: - Eric Anholt diff --git a/Documentation/devicetree/bindings/display/bridge/adi,adv7511.yaml b/Documentation/devicetree/bindings/display/bridge/adi,adv7511.yaml index f08a01dfedf3fc80af9679eaa0676ada88e53128..5bbe81862c8f981891155574b3b5705b15861e87 100644 --- a/Documentation/devicetree/bindings/display/bridge/adi,adv7511.yaml +++ b/Documentation/devicetree/bindings/display/bridge/adi,adv7511.yaml @@ -117,23 +117,21 @@ properties: ports: description: - The ADV7511(W)/13 has two video ports and one audio port. This node - models their connections as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt - type: object + The ADV7511(W)/13 has two video ports and one audio port. + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: description: Video port for the RGB or YUV input. - type: object + $ref: /schemas/graph.yaml#/properties/port port@1: description: Video port for the HDMI output. - type: object + $ref: /schemas/graph.yaml#/properties/port port@2: description: Audio port for the HDMI output. - type: object + $ref: /schemas/graph.yaml#/properties/port # adi,input-colorspace and adi,input-clock are required except in # "rgb 1x" and "yuv444 1x" modes, in which case they must not be diff --git a/Documentation/devicetree/bindings/display/bridge/adi,adv7533.yaml b/Documentation/devicetree/bindings/display/bridge/adi,adv7533.yaml index f36209137c8a4536592c0cd74a858bdceb4c77a7..987aa83c2649436f23ccfaf2a3e38ab77a8b6f7d 100644 --- a/Documentation/devicetree/bindings/display/bridge/adi,adv7533.yaml +++ b/Documentation/devicetree/bindings/display/bridge/adi,adv7533.yaml @@ -91,25 +91,23 @@ properties: ports: description: - The ADV7533/35 has two video ports and one audio port. This node - models their connections as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt - type: object + The ADV7533/35 has two video ports and one audio port. + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: description: Video port for the DSI input. The remote endpoint phandle should be a reference to a valid mipi_dsi_host_device. - type: object + $ref: /schemas/graph.yaml#/properties/port port@1: description: Video port for the HDMI output. - type: object + $ref: /schemas/graph.yaml#/properties/port port@2: description: Audio port for the HDMI output. - type: object + $ref: /schemas/graph.yaml#/properties/port required: - compatible diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml index bce96b5b0db083463eed3dba24dc6b26b99b025f..4a5e5d9d6f909064ea15e4ba7f8b29827e6b20fb 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml +++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml @@ -8,7 +8,7 @@ title: Analogix ANX7814 SlimPort (Full-HD Transmitter) maintainers: - Andrzej Hajda - - Neil Armstrong + - Neil Armstrong - Robert Foss properties: diff --git a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml index 2ebaa43eb62e9bfae47514481a0ebac911e4ece2..b19be0804abe59eaf1f5e216ab2d7054e6dcb6fa 100644 --- a/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml +++ b/Documentation/devicetree/bindings/display/bridge/fsl,ldb.yaml @@ -25,7 +25,6 @@ properties: const: ldb reg: - minItems: 2 maxItems: 2 reg-names: diff --git a/Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml b/Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml index c6e81f5322152b84c57cc63496d50b37b0326d1c..1b2185be92cdd32ce67f447f021f55862006a552 100644 --- a/Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml @@ -8,7 +8,7 @@ title: ITE it66121 HDMI bridge Device Tree Bindings maintainers: - Phong LE - - Neil Armstrong + - Neil Armstrong description: | The IT66121 is a high-performance and low-power single channel HDMI diff --git a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml index 3a8614e0f62732ed4efa9b9b9ef5a18f9cd49006..84aafcbf09190a33e852bcff7c542125b7b1743c 100644 --- a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml @@ -51,6 +51,7 @@ properties: properties: port@0: $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: | For LVDS encoders, port 0 is the parallel input For LVDS decoders, port 0 is the LVDS input diff --git a/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml b/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4bf543974737b5ce6a23eff854b7ed81e307d77 --- /dev/null +++ b/Documentation/devicetree/bindings/display/bridge/nxp,tda998x.yaml @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/bridge/nxp,tda998x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP TDA998x HDMI transmitter + +maintainers: + - Russell King + +properties: + compatible: + const: nxp,tda998x + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + video-ports: + default: 0x230145 + maximum: 0xffffff + description: + 24 bits value which defines how the video controller output is wired to + the TDA998x input. + + audio-ports: + description: + Array of 8-bit values, 2 values per DAI (Documentation/sound/soc/dai.rst). + The implementation allows one or two DAIs. + If two DAIs are defined, they must be of different type. + $ref: /schemas/types.yaml#/definitions/uint32-matrix + items: + minItems: 1 + items: + - description: | + The first value defines the DAI type: TDA998x_SPDIF or TDA998x_I2S + (see include/dt-bindings/display/tda998x.h). + - description: + The second value defines the tda998x AP_ENA reg content when the + DAI in question is used. + + '#sound-dai-cells': + enum: [ 0, 1 ] + + nxp,calib-gpios: + maxItems: 1 + description: + Calibration GPIO, which must correspond with the gpio used for the + TDA998x interrupt pin. + + port: + $ref: /schemas/graph.yaml#/properties/port + description: Parallel input port + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port@0: + type: object + description: Parallel input port + + port@1: + type: object + description: HDMI output port + +required: + - compatible + - reg + +oneOf: + - required: + - port + - required: + - ports + +additionalProperties: false + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + tda998x: hdmi-encoder@70 { + compatible = "nxp,tda998x"; + reg = <0x70>; + interrupt-parent = <&gpio0>; + interrupts = <27 IRQ_TYPE_EDGE_FALLING>; + video-ports = <0x230145>; + + #sound-dai-cells = <1>; + /* DAI-format / AP_ENA reg value */ + audio-ports = , + ; + + port { + tda998x_in: endpoint { + remote-endpoint = <&lcdc_0>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml index 0c9785c8db515a8ba37a6c99b8db47516ce40e33..e3ec697f89e7165812200ec5b7a47cd6e229fdb0 100644 --- a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml @@ -38,6 +38,9 @@ properties: clock-names: maxItems: 2 + resets: + maxItems: 1 + ports: $ref: /schemas/graph.yaml#/properties/ports @@ -67,6 +70,7 @@ required: - reg - clocks - clock-names + - resets - interrupts - ports @@ -85,6 +89,7 @@ examples: clocks = <&cpg CPG_CORE R8A7795_CLK_S0D4>, <&cpg CPG_MOD 729>; clock-names = "iahb", "isfr"; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; + resets = <&cpg 729>; ports { #address-cells = <1>; diff --git a/Documentation/devicetree/bindings/display/bridge/synopsys,dw-hdmi.yaml b/Documentation/devicetree/bindings/display/bridge/synopsys,dw-hdmi.yaml index b00246faea57c8efcb7821a02bed0133bac84391..4b7e54a8f037fb0fc81750d674594a4345091adb 100644 --- a/Documentation/devicetree/bindings/display/bridge/synopsys,dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/synopsys,dw-hdmi.yaml @@ -26,7 +26,6 @@ properties: reg-io-width: description: Width (in bytes) of the registers specified by the reg property. - $ref: /schemas/types.yaml#/definitions/uint32 enum: [1, 4] default: 1 diff --git a/Documentation/devicetree/bindings/display/bridge/tda998x.txt b/Documentation/devicetree/bindings/display/bridge/tda998x.txt deleted file mode 100644 index f5a02f61dd36f1c68acc89f15507a144188db8c0..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/display/bridge/tda998x.txt +++ /dev/null @@ -1,54 +0,0 @@ -Device-Tree bindings for the NXP TDA998x HDMI transmitter - -Required properties; - - compatible: must be "nxp,tda998x" - - - reg: I2C address - -Required node: - - port: Input port node with endpoint definition, as described - in Documentation/devicetree/bindings/graph.txt - -Optional properties: - - interrupts: interrupt number and trigger type - default: polling - - - pinctrl-0: pin control group to be used for - screen plug/unplug interrupt. - - - pinctrl-names: must contain a "default" entry. - - - video-ports: 24 bits value which defines how the video controller - output is wired to the TDA998x input - default: <0x230145> - - - audio-ports: array of 8-bit values, 2 values per one DAI[1]. - The first value defines the DAI type: TDA998x_SPDIF or TDA998x_I2S[2]. - The second value defines the tda998x AP_ENA reg content when the DAI - in question is used. The implementation allows one or two DAIs. If two - DAIs are defined, they must be of different type. - - - nxp,calib-gpios: calibration GPIO, which must correspond with the - gpio used for the TDA998x interrupt pin. - -[1] Documentation/sound/soc/dai.rst -[2] include/dt-bindings/display/tda998x.h - -Example: - -#include - - tda998x: hdmi-encoder { - compatible = "nxp,tda998x"; - reg = <0x70>; - interrupt-parent = <&gpio0>; - interrupts = <27 2>; /* falling edge */ - pinctrl-0 = <&pmx_camera>; - pinctrl-names = "default"; - video-ports = <0x230145>; - - #sound-dai-cells = <2>; - /* DAI-format AP_ENA reg value */ - audio-ports = < TDA998x_SPDIF 0x04 - TDA998x_I2S 0x03>; - - }; diff --git a/Documentation/devicetree/bindings/display/ilitek,ili9486.yaml b/Documentation/devicetree/bindings/display/ilitek,ili9486.yaml index aecff34f505d706328562fac15555268ab4262d2..1f8f2182e2f1f7d2db024de57b7f0a9f49762305 100644 --- a/Documentation/devicetree/bindings/display/ilitek,ili9486.yaml +++ b/Documentation/devicetree/bindings/display/ilitek,ili9486.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/ilitek,ili9486.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Ilitek ILI9486 display panels device tree bindings +title: Ilitek ILI9486 display panels maintainers: - Kamlesh Gurudasani diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml index 94bc6e1b64518a23a052fa20d1e40957d6ce9afb..f2515af8256f0f9734a54efc46d9c4c3586a719a 100644 --- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml +++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml @@ -24,6 +24,7 @@ properties: - qcom,sm8350-dp reg: + minItems: 4 items: - description: ahb register block - description: aux register block @@ -70,14 +71,28 @@ properties: operating-points-v2: maxItems: 1 + opp-table: true + power-domains: maxItems: 1 + aux-bus: + $ref: /schemas/display/dp-aux-bus.yaml# + + data-lanes: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + items: + maximum: 3 + "#sound-dai-cells": const: 0 - vdda-0p9-supply: true - vdda-1p2-supply: true + vdda-0p9-supply: + deprecated: true + vdda-1p2-supply: + deprecated: true ports: $ref: /schemas/graph.yaml#/properties/ports @@ -98,10 +113,33 @@ required: - clock-names - phys - phy-names - - "#sound-dai-cells" - power-domains - ports +allOf: + # AUX BUS does not exist on DP controllers + # Audio output also is present only on DP output + # p1 regions is present on DP, but not on eDP + - if: + properties: + compatible: + contains: + enum: + - qcom,sc7280-edp + - qcom,sc8180x-edp + then: + properties: + "#sound-dai-cells": false + reg: + maxItems: 4 + else: + properties: + aux-bus: false + reg: + minItems: 5 + required: + - "#sound-dai-cells" + additionalProperties: false examples: @@ -140,9 +178,6 @@ examples: power-domains = <&rpmhpd SC7180_CX>; - vdda-0p9-supply = <&vdda_usb_ss_dp_core>; - vdda-1p2-supply = <&vdda_usb_ss_dp_1p2>; - ports { #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml b/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml index 2df64afb76e6e3a134fe2f9ea266a2b0e6ddc80a..253665c693e6b91370dadba4134c3c2011649a34 100644 --- a/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml +++ b/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml @@ -62,6 +62,7 @@ patternProperties: "^display-controller@[0-9a-f]+$": type: object description: Node containing the properties of DPU. + additionalProperties: false properties: compatible: @@ -105,6 +106,9 @@ patternProperties: maxItems: 1 operating-points-v2: true + opp-table: + type: object + ports: $ref: /schemas/graph.yaml#/properties/ports description: | diff --git a/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml b/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml index 734d14de966d004fb0c49919cc8171844b84314d..c5824e1d23824d59601bc8f5bf3fc3ba58b19c70 100644 --- a/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml +++ b/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml @@ -74,6 +74,7 @@ patternProperties: "^display-controller@[0-9a-f]+$": type: object description: Node containing the properties of DPU. + additionalProperties: false properties: compatible: @@ -113,6 +114,8 @@ patternProperties: maxItems: 1 operating-points-v2: true + opp-table: + type: object ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml index d3c3e4b078971d9fd41f04b16ca507a34eb7827b..4890bc25f3fdc5e68f526427a13da92c7c21da2f 100644 --- a/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml +++ b/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml @@ -73,6 +73,7 @@ patternProperties: "^display-controller@[0-9a-f]+$": type: object description: Node containing the properties of DPU. + additionalProperties: false properties: compatible: @@ -114,6 +115,8 @@ patternProperties: maxItems: 1 operating-points-v2: true + opp-table: + type: object ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml index f427eec3d3a45fdd617d638b7ef7dd9f34c4f1b0..584d646021d518cafd68704b4b229bc972b1dffe 100644 --- a/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml +++ b/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml @@ -72,6 +72,7 @@ patternProperties: "^display-controller@[0-9a-f]+$": type: object description: Node containing the properties of DPU. + additionalProperties: false properties: compatible: @@ -112,6 +113,8 @@ patternProperties: maxItems: 1 operating-points-v2: true + opp-table: + type: object ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml index 2bb8896beffc69611d8102c23c029dc26641bb1d..7d10373731752b912b0950e393e1d807f2459068 100644 --- a/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml +++ b/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml @@ -65,6 +65,7 @@ patternProperties: "^display-controller@[0-9a-f]+$": type: object description: Node containing the properties of DPU. + additionalProperties: false properties: compatible: @@ -102,6 +103,9 @@ patternProperties: maxItems: 1 operating-points-v2: true + opp-table: + type: object + ports: $ref: /schemas/graph.yaml#/properties/ports description: | diff --git a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml index 880bfe930830c4cf754f8697b33bcca6b8db84ea..3b609c19e0bc4b39f41ba7a04e254883a8def590 100644 --- a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml +++ b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml @@ -66,13 +66,11 @@ properties: 2 DSI links. assigned-clocks: - minItems: 2 maxItems: 2 description: | Parents of "byte" and "pixel" for the given platform. assigned-clock-parents: - minItems: 2 maxItems: 2 description: | The Byte clock and Pixel clock PLL outputs provided by a DSI PHY block. diff --git a/Documentation/devicetree/bindings/display/msm/dsi-phy-10nm.yaml b/Documentation/devicetree/bindings/display/msm/dsi-phy-10nm.yaml index 716f921e3532ecd79216479a29c4f9e61de59dfd..d9ad8b659f58ee0f5bf3b64b696beb64914c9609 100644 --- a/Documentation/devicetree/bindings/display/msm/dsi-phy-10nm.yaml +++ b/Documentation/devicetree/bindings/display/msm/dsi-phy-10nm.yaml @@ -37,7 +37,6 @@ properties: qcom,phy-rescode-offset-top: $ref: /schemas/types.yaml#/definitions/int8-array - minItems: 5 maxItems: 5 description: Integer array of offset for pull-up legs rescode for all five lanes. @@ -49,7 +48,6 @@ properties: qcom,phy-rescode-offset-bot: $ref: /schemas/types.yaml#/definitions/int8-array - minItems: 5 maxItems: 5 description: Integer array of offset for pull-down legs rescode for all five lanes. diff --git a/Documentation/devicetree/bindings/display/msm/gmu.yaml b/Documentation/devicetree/bindings/display/msm/gmu.yaml index fe55611d2603c5a2619183d4cffee88e06016d4d..67fdeeabae0cc398e1a9e66898df8f6b550290a6 100644 --- a/Documentation/devicetree/bindings/display/msm/gmu.yaml +++ b/Documentation/devicetree/bindings/display/msm/gmu.yaml @@ -20,35 +20,24 @@ description: | properties: compatible: items: - - enum: - - qcom,adreno-gmu-630.2 + - pattern: '^qcom,adreno-gmu-6[0-9][0-9]\.[0-9]$' - const: qcom,adreno-gmu reg: - items: - - description: Core GMU registers - - description: GMU PDC registers - - description: GMU PDC sequence registers + minItems: 3 + maxItems: 4 reg-names: - items: - - const: gmu - - const: gmu_pdc - - const: gmu_pdc_seq + minItems: 3 + maxItems: 4 clocks: - items: - - description: GMU clock - - description: GPU CX clock - - description: GPU AXI clock - - description: GPU MEMNOC clock + minItems: 4 + maxItems: 7 clock-names: - items: - - const: gmu - - const: cxo - - const: axi - - const: memnoc + minItems: 4 + maxItems: 7 interrupts: items: @@ -76,6 +65,9 @@ properties: operating-points-v2: true + opp-table: + type: object + required: - compatible - reg @@ -91,6 +83,140 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-gmu-618.0 + - qcom,adreno-gmu-630.2 + then: + properties: + reg: + items: + - description: Core GMU registers + - description: GMU PDC registers + - description: GMU PDC sequence registers + reg-names: + items: + - const: gmu + - const: gmu_pdc + - const: gmu_pdc_seq + clocks: + items: + - description: GMU clock + - description: GPU CX clock + - description: GPU AXI clock + - description: GPU MEMNOC clock + clock-names: + items: + - const: gmu + - const: cxo + - const: axi + - const: memnoc + + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-gmu-635.0 + then: + properties: + reg: + items: + - description: Core GMU registers + - description: Resource controller registers + - description: GMU PDC registers + reg-names: + items: + - const: gmu + - const: rscc + - const: gmu_pdc + clocks: + items: + - description: GMU clock + - description: GPU CX clock + - description: GPU AXI clock + - description: GPU MEMNOC clock + - description: GPU AHB clock + - description: GPU HUB CX clock + - description: GPU SMMU vote clock + clock-names: + items: + - const: gmu + - const: cxo + - const: axi + - const: memnoc + - const: ahb + - const: hub + - const: smmu_vote + + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-gmu-640.1 + then: + properties: + reg: + items: + - description: Core GMU registers + - description: GMU PDC registers + - description: GMU PDC sequence registers + reg-names: + items: + - const: gmu + - const: gmu_pdc + - const: gmu_pdc_seq + + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-gmu-650.2 + then: + properties: + reg: + items: + - description: Core GMU registers + - description: Resource controller registers + - description: GMU PDC registers + - description: GMU PDC sequence registers + reg-names: + items: + - const: gmu + - const: rscc + - const: gmu_pdc + - const: gmu_pdc_seq + + - if: + properties: + compatible: + contains: + enum: + - qcom,adreno-gmu-640.1 + - qcom,adreno-gmu-650.2 + then: + properties: + clocks: + items: + - description: GPU AHB clock + - description: GMU clock + - description: GPU CX clock + - description: GPU AXI clock + - description: GPU MEMNOC clock + clock-names: + items: + - const: ahb + - const: gmu + - const: cxo + - const: axi + - const: memnoc + examples: - | #include diff --git a/Documentation/devicetree/bindings/display/msm/gpu.yaml b/Documentation/devicetree/bindings/display/msm/gpu.yaml index 3397bc31d0875dae8283ed72df9f1988671e7ed9..ec4b1a75f46ac4843db94d965830ec8614b8d230 100644 --- a/Documentation/devicetree/bindings/display/msm/gpu.yaml +++ b/Documentation/devicetree/bindings/display/msm/gpu.yaml @@ -58,7 +58,8 @@ properties: - const: ocmem iommus: - maxItems: 1 + minItems: 1 + maxItems: 64 sram: $ref: /schemas/types.yaml#/definitions/phandle-array @@ -81,6 +82,7 @@ properties: zap-shader: type: object + additionalProperties: false description: | For a5xx and a6xx devices this node contains a memory-region that points to reserved memory to store the zap shader that can be used to diff --git a/Documentation/devicetree/bindings/display/msm/mdp4.yaml b/Documentation/devicetree/bindings/display/msm/mdp4.yaml index f63f60fea27c8a843c7f2386aa0661f14fd18e3a..58c13f5277b6108629b18e2c6df8da7195a24cad 100644 --- a/Documentation/devicetree/bindings/display/msm/mdp4.yaml +++ b/Documentation/devicetree/bindings/display/msm/mdp4.yaml @@ -36,7 +36,7 @@ properties: maxItems: 1 iommus: - maxItems: 1 + maxItems: 4 ports: $ref: /schemas/graph.yaml#/properties/ports diff --git a/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml b/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml index be69e0cc50fcd08235666ce045889aca05fc7c24..c9958f824d9ab313d79742d0e19fae27753a0ecd 100644 --- a/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml +++ b/Documentation/devicetree/bindings/display/panel/arm,versatile-tft-panel.yaml @@ -37,9 +37,6 @@ examples: compatible = "arm,versatile-sysreg", "syscon", "simple-mfd"; reg = <0x00000 0x1000>; - #address-cells = <1>; - #size-cells = <0>; - panel { compatible = "arm,versatile-tft-panel"; diff --git a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml index 2a2756d196810e6489ef2cd912d0fc361ee61fd2..b4be9bd8ddde7c681895894689d03c3e4f855ed9 100644 --- a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml +++ b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml @@ -23,6 +23,8 @@ properties: reg: true reset-gpios: true + spi-3wire: true + required: - compatible - power-supply diff --git a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml index 817a9bed7d5a88e5586b60eb9824d62b5c8484a1..ebdca5f5a00117299833abf8e535ce965139c19a 100644 --- a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml +++ b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml @@ -24,6 +24,8 @@ properties: reg: true reset-gpios: true + spi-3wire: true + required: - compatible - power-supply diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml index 940f7f88526ff9796a9540f566609781e6248722..6f1fc7469f076f68432b16c39341612c16326261 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml @@ -24,6 +24,10 @@ properties: default-brightness: true max-brightness: true + spi-3wire: true + spi-cpha: true + spi-cpol: true + vdd3-supply: description: VDD regulator diff --git a/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml index 44e02decdf3a003bc17bfd201d6c332ca3a207df..2e75e3738ff094317e2c8bba97757b5712f43c8d 100644 --- a/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml +++ b/Documentation/devicetree/bindings/display/panel/sgd,gktw70sdae4se.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Solomon Goldentek Display GKTW70SDAE4SE 7" WVGA LVDS Display Panel maintainers: - - Neil Armstrong + - Neil Armstrong - Thierry Reding allOf: diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-decon.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-decon.yaml index 921bfe925cd64668a975228c060c0b5a6120085b..6380eeebb0738558277ceceb56e646112fec20af 100644 --- a/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-decon.yaml +++ b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-decon.yaml @@ -24,7 +24,6 @@ properties: - samsung,exynos5433-decon-tv clocks: - minItems: 11 maxItems: 11 clock-names: @@ -59,7 +58,6 @@ properties: - const: te iommus: - minItems: 2 maxItems: 2 iommu-names: diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-mic.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-mic.yaml index 7d405f2febcd266d795e2e32eab58166d7839f80..26e5017737a354721b1f700250ff573902cdbeff 100644 --- a/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-mic.yaml +++ b/Documentation/devicetree/bindings/display/samsung/samsung,exynos5433-mic.yaml @@ -24,7 +24,6 @@ properties: const: samsung,exynos5433-mic clocks: - minItems: 2 maxItems: 2 clock-names: diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,exynos7-decon.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,exynos7-decon.yaml index 969bd8c563a50e7e5a1841ce76e130c94ae38fbf..992c23ca7a4ee84cf5b5e4e90b42149e362cc9b2 100644 --- a/Documentation/devicetree/bindings/display/samsung/samsung,exynos7-decon.yaml +++ b/Documentation/devicetree/bindings/display/samsung/samsung,exynos7-decon.yaml @@ -22,7 +22,6 @@ properties: const: samsung,exynos7-decon clocks: - minItems: 4 maxItems: 4 clock-names: @@ -37,6 +36,7 @@ properties: i80-if-timings: type: object + additionalProperties: false description: timing configuration for lcd i80 interface support properties: cs-setup: diff --git a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml index 5d5cc220f78a91c853f4cfad4adf6fe87d3cf67b..075231716b2ffd33eecb173af9dd01adc3822c7c 100644 --- a/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml +++ b/Documentation/devicetree/bindings/display/samsung/samsung,fimd.yaml @@ -27,7 +27,6 @@ properties: const: 1 clocks: - minItems: 2 maxItems: 2 clock-names: @@ -40,6 +39,7 @@ properties: i80-if-timings: type: object + additionalProperties: false description: | Timing configuration for lcd i80 interface support. The parameters are defined as:: diff --git a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml index 1f905d85dd9cbd7c7b4e778807a18d5c697fb00b..dd64f70b5014d5d8c24a83cbdb36c5b01794af0d 100644 --- a/Documentation/devicetree/bindings/display/simple-framebuffer.yaml +++ b/Documentation/devicetree/bindings/display/simple-framebuffer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/simple-framebuffer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Simple Framebuffer Device Tree Bindings +title: Simple Framebuffer maintainers: - Hans de Goede diff --git a/Documentation/devicetree/bindings/display/sitronix,st7735r.yaml b/Documentation/devicetree/bindings/display/sitronix,st7735r.yaml index 53f181ef3670152bd34fa87bc1d76592080f4350..621f2714841990a90ea6cdda04d9aa04fd51c394 100644 --- a/Documentation/devicetree/bindings/display/sitronix,st7735r.yaml +++ b/Documentation/devicetree/bindings/display/sitronix,st7735r.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/display/sitronix,st7735r.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Sitronix ST7735R Display Panels Device Tree Bindings +title: Sitronix ST7735R Display Panels maintainers: - David Lechner diff --git a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml index 54f67cb5104015b46849eded7f374e8bd4b65850..c488308d7be132ecb7acdb384cf559858b2103f9 100644 --- a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml +++ b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml @@ -58,9 +58,20 @@ properties: DSI input port node, connected to the ltdc rgb output port. port@1: - $ref: /schemas/graph.yaml#/properties/port - description: - DSI output port node, connected to a panel or a bridge input port" + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + description: | + DSI output port node, connected to a panel or a bridge input port. + properties: + endpoint: + $ref: /schemas/media/video-interfaces.yaml# + unevaluatedProperties: false + properties: + data-lanes: + minItems: 1 + items: + - const: 1 + - const: 2 required: - "#address-cells" diff --git a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-gr3d.yaml b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-gr3d.yaml index dbdf0229d9f658235ed43147f225e6ee28b59ad3..4755a73473c74c18dd8a503b412aa5012e253c04 100644 --- a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-gr3d.yaml +++ b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-gr3d.yaml @@ -59,7 +59,6 @@ properties: maxItems: 2 power-domain-names: - minItems: 2 maxItems: 2 allOf: diff --git a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-mpe.yaml b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-mpe.yaml index 4154ae01ad13305b57f0cec1a996d1e151775fed..5f4f0fb4b69293bbe16bbca6c737fe620226ceb1 100644 --- a/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-mpe.yaml +++ b/Documentation/devicetree/bindings/display/tegra/nvidia,tegra20-mpe.yaml @@ -42,11 +42,9 @@ properties: maxItems: 1 interconnects: - minItems: 6 maxItems: 6 interconnect-names: - minItems: 6 maxItems: 6 operating-points-v2: diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml index 83808199657b731544c5fd0059b9f6639bcdf92b..26d0d8ab7984bb98a754b8a543c74519dd970b02 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun4i-a10-dma.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/dma/allwinner,sun4i-a10-dma.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 DMA Controller Device Tree Bindings +title: Allwinner A10 DMA Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml index e712444abff17776e9138908560551bedec4dd11..bd599bda2653566cf5aaaa4d7ee2a369fdf93e95 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun50i-a64-dma.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/dma/allwinner,sun50i-a64-dma.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A64 DMA Controller Device Tree Bindings +title: Allwinner A64 DMA Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/dma/allwinner,sun6i-a31-dma.yaml b/Documentation/devicetree/bindings/dma/allwinner,sun6i-a31-dma.yaml index a6df6f8b54db592b68ed59d718764b39b8e081e7..344dc7e04931d84acf2503db7eb22435767e2a46 100644 --- a/Documentation/devicetree/bindings/dma/allwinner,sun6i-a31-dma.yaml +++ b/Documentation/devicetree/bindings/dma/allwinner,sun6i-a31-dma.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/dma/allwinner,sun6i-a31-dma.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 DMA Controller Device Tree Bindings +title: Allwinner A31 DMA Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/dma/apple,admac.yaml b/Documentation/devicetree/bindings/dma/apple,admac.yaml index bdc8c129c4f5bc6159bba6d214f529b1e6ec8326..3b1e667f7ea01ff80b501d365deaccc75a99ab7b 100644 --- a/Documentation/devicetree/bindings/dma/apple,admac.yaml +++ b/Documentation/devicetree/bindings/dma/apple,admac.yaml @@ -49,6 +49,13 @@ properties: in an interrupts-extended list the disconnected positions will contain an empty phandle reference <0>. + iommus: + minItems: 1 + maxItems: 2 + + power-domains: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/dma/arm,pl330.yaml b/Documentation/devicetree/bindings/dma/arm,pl330.yaml index 2bec69b308f8665d64d86af4d9a28557022cfda0..4a3dd6f5309b1fa4fca1b0dbe57122b9aa8fabf2 100644 --- a/Documentation/devicetree/bindings/dma/arm,pl330.yaml +++ b/Documentation/devicetree/bindings/dma/arm,pl330.yaml @@ -55,6 +55,12 @@ properties: dma-coherent: true + iommus: + minItems: 1 + maxItems: 9 + description: Up to 1 IOMMU entry per DMA channel for writes and 1 + IOMMU entry for reads. + power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml index 19ea8dcbcbced074535624bc969e798975970f89..9ab4d81ead35f4d50eb42c3bcfabb779b9593ab7 100644 --- a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml +++ b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml @@ -22,6 +22,7 @@ properties: - items: - enum: - mediatek,mt2712-uart-dma + - mediatek,mt6795-uart-dma - mediatek,mt8365-uart-dma - mediatek,mt8516-uart-dma - const: mediatek,mt6577-uart-dma diff --git a/Documentation/devicetree/bindings/dma/moxa,moxart-dma.txt b/Documentation/devicetree/bindings/dma/moxa,moxart-dma.txt index 8a9f3559335b50f0c663bcfe49ca6cc336745e6f..7e14e26676ec9216f610198fb7571ab0e28a558b 100644 --- a/Documentation/devicetree/bindings/dma/moxa,moxart-dma.txt +++ b/Documentation/devicetree/bindings/dma/moxa,moxart-dma.txt @@ -34,8 +34,8 @@ Example: Use specific request line passing from dma For example, MMC request line is 5 - sdhci: sdhci@98e00000 { - compatible = "moxa,moxart-sdhci"; + mmc: mmc@98e00000 { + compatible = "moxa,moxart-mmc"; reg = <0x98e00000 0x5C>; interrupts = <5 0>; clocks = <&clk_apb>; diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml index 7e575296df0c4abaccd04663a784c04eda1206d0..c8894476b6abf95a6183c1fb0dfbc78fb80ba6cb 100644 --- a/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml +++ b/Documentation/devicetree/bindings/dma/nvidia,tegra186-gpc-dma.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/dma/nvidia,tegra186-gpc-dma.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NVIDIA Tegra GPC DMA Controller Device Tree Bindings +title: NVIDIA Tegra GPC DMA Controller description: | The Tegra General Purpose Central (GPC) DMA controller is used for faster diff --git a/Documentation/devicetree/bindings/dma/qcom,adm.yaml b/Documentation/devicetree/bindings/dma/qcom,adm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a9d7bc74affb4589119978d47261243ad8bbd58 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/qcom,adm.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/qcom,adm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm ADM DMA Controller + +maintainers: + - Christian Marangi + - Bjorn Andersson + +description: | + QCOM ADM DMA controller provides DMA capabilities for + peripheral buses such as NAND and SPI. + +properties: + compatible: + const: qcom,adm + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + "#dma-cells": + const: 1 + + clocks: + items: + - description: phandle to the core clock + - description: phandle to the iface clock + + clock-names: + items: + - const: core + - const: iface + + resets: + items: + - description: phandle to the clk reset + - description: phandle to the pbus reset + - description: phandle to the c0 reset + - description: phandle to the c1 reset + - description: phandle to the c2 reset + + reset-names: + items: + - const: clk + - const: pbus + - const: c0 + - const: c1 + - const: c2 + + qcom,ee: + $ref: /schemas/types.yaml#/definitions/uint32 + description: indicates the security domain identifier used in the secure world. + minimum: 0 + maximum: 255 + +required: + - compatible + - reg + - interrupts + - "#dma-cells" + - clocks + - clock-names + - resets + - reset-names + - qcom,ee + +additionalProperties: false + +examples: + - | + #include + #include + + adm_dma: dma-controller@18300000 { + compatible = "qcom,adm"; + reg = <0x18300000 0x100000>; + interrupts = <0 170 0>; + #dma-cells = <1>; + + clocks = <&gcc ADM0_CLK>, + <&gcc ADM0_PBUS_CLK>; + clock-names = "core", "iface"; + + resets = <&gcc ADM0_RESET>, + <&gcc ADM0_PBUS_RESET>, + <&gcc ADM0_C0_RESET>, + <&gcc ADM0_C1_RESET>, + <&gcc ADM0_C2_RESET>; + reset-names = "clk", "pbus", "c0", "c1", "c2"; + qcom,ee = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml index 9bf3a1b164f18e58831941ce8cbda78d6fafb6f4..003098caf7095f40a6ebd8df618355e48f889774 100644 --- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml @@ -8,7 +8,7 @@ title: Qualcomm Technologies Inc BAM DMA controller maintainers: - Andy Gross - - Bjorn Andersson + - Bjorn Andersson allOf: - $ref: "dma-controller.yaml#" @@ -20,7 +20,7 @@ properties: - qcom,bam-v1.3.0 # MSM8974, APQ8074 and APQ8084 - qcom,bam-v1.4.0 - # MSM8916 + # MSM8916 and SDM845 - qcom,bam-v1.7.0 clocks: @@ -90,8 +90,8 @@ examples: dma-controller@f9944000 { compatible = "qcom,bam-v1.4.0"; - reg = <0xf9944000 0x15000>; - interrupts = ; + reg = <0xf9944000 0x19000>; + interrupts = ; clocks = <&gcc GCC_BLSP2_AHB_CLK>; clock-names = "bam_clk"; #dma-cells = <1>; diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml index 7d2fc4eb55305eb683008f342f7dbb6e4cd01f35..eabf8a76d3a05a6a88a6fecefdae5c4b00ada7a4 100644 --- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml @@ -21,6 +21,7 @@ properties: enum: - qcom,sc7280-gpi-dma - qcom,sdm845-gpi-dma + - qcom,sm6350-gpi-dma - qcom,sm8150-gpi-dma - qcom,sm8250-gpi-dma - qcom,sm8350-gpi-dma diff --git a/Documentation/devicetree/bindings/dma/qcom_adm.txt b/Documentation/devicetree/bindings/dma/qcom_adm.txt deleted file mode 100644 index 9d3b2f917b7b31ea0c9ac6ff7e545b5324b77005..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/dma/qcom_adm.txt +++ /dev/null @@ -1,61 +0,0 @@ -QCOM ADM DMA Controller - -Required properties: -- compatible: must contain "qcom,adm" for IPQ/APQ8064 and MSM8960 -- reg: Address range for DMA registers -- interrupts: Should contain one interrupt shared by all channels -- #dma-cells: must be <2>. First cell denotes the channel number. Second cell - denotes CRCI (client rate control interface) flow control assignment. -- clocks: Should contain the core clock and interface clock. -- clock-names: Must contain "core" for the core clock and "iface" for the - interface clock. -- resets: Must contain an entry for each entry in reset names. -- reset-names: Must include the following entries: - - clk - - c0 - - c1 - - c2 -- qcom,ee: indicates the security domain identifier used in the secure world. - -Example: - adm_dma: dma@18300000 { - compatible = "qcom,adm"; - reg = <0x18300000 0x100000>; - interrupts = <0 170 0>; - #dma-cells = <2>; - - clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>; - clock-names = "core", "iface"; - - resets = <&gcc ADM0_RESET>, - <&gcc ADM0_C0_RESET>, - <&gcc ADM0_C1_RESET>, - <&gcc ADM0_C2_RESET>; - reset-names = "clk", "c0", "c1", "c2"; - qcom,ee = <0>; - }; - -DMA clients must use the format descripted in the dma.txt file, using a three -cell specifier for each channel. - -Each dmas request consists of 3 cells: - 1. phandle pointing to the DMA controller - 2. channel number - 3. CRCI assignment, if applicable. If no CRCI flow control is required, use 0. - The CRCI is used for flow control. It identifies the peripheral device that - is the source/destination for the transferred data. - -Example: - - spi4: spi@1a280000 { - spi-max-frequency = <50000000>; - - pinctrl-0 = <&spi_pins>; - pinctrl-names = "default"; - - cs-gpios = <&qcom_pinmux 20 0>; - - dmas = <&adm_dma 6 9>, - <&adm_dma 5 10>; - dma-names = "rx", "tx"; - }; diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml index 7202cd68e7597dc23fffdafdf5537963e0b3fbba..89b591a05bce5fe5e61d5c1e9b3d6d68b3755bf3 100644 --- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml @@ -45,6 +45,7 @@ properties: - enum: - renesas,dmac-r8a779a0 # R-Car V3U - renesas,dmac-r8a779f0 # R-Car S4-8 + - renesas,dmac-r8a779g0 # R-Car V4H - const: renesas,rcar-gen4-dmac # R-Car Gen4 reg: true diff --git a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt b/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt index b849a1ed389d55df9662ca5708f8b93424b9ce88..47e477cce6d24d41430fbd192a64cbd6ef2593c0 100644 --- a/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt +++ b/Documentation/devicetree/bindings/dma/ti-dma-crossbar.txt @@ -4,7 +4,7 @@ Required properties: - compatible: "ti,dra7-dma-crossbar" for DRA7xx DMA crossbar "ti,am335x-edma-crossbar" for AM335x and AM437x - reg: Memory map for accessing module -- #dma-cells: Should be set to to match with the DMA controller's dma-cells +- #dma-cells: Should be set to match with the DMA controller's dma-cells for ti,dra7-dma-crossbar and <3> for ti,am335x-edma-crossbar. - dma-requests: Number of DMA requests the crossbar can receive - dma-masters: phandle pointing to the DMA controller diff --git a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml index e66ef2da78799f4ef1b3dbca49931663c96c16e0..9af40da5688efe430a3ecadf8eaea0f18e132e7a 100644 --- a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml +++ b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml @@ -20,6 +20,7 @@ properties: - fsl,imx8qxp-dsp - fsl,imx8qm-dsp - fsl,imx8mp-dsp + - fsl,imx8ulp-dsp - fsl,imx8qxp-hifi4 - fsl,imx8qm-hifi4 - fsl,imx8mp-hifi4 diff --git a/Documentation/devicetree/bindings/firmware/fsl,scu.yaml b/Documentation/devicetree/bindings/firmware/fsl,scu.yaml index b40b0ef569789f03ea7d4f0bfd4363df0a399c14..557e524786c2224e1af8478c2f3ba84f9d0eb342 100644 --- a/Documentation/devicetree/bindings/firmware/fsl,scu.yaml +++ b/Documentation/devicetree/bindings/firmware/fsl,scu.yaml @@ -30,6 +30,11 @@ properties: Clock controller node that provides the clocks controlled by the SCU $ref: /schemas/clock/fsl,scu-clk.yaml + gpio: + description: + Control the GPIO PINs on SCU domain over the firmware APIs + $ref: /schemas/gpio/fsl,imx8qxp-sc-gpio.yaml + ocotp: description: OCOTP controller node provided by the SCU diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.txt b/Documentation/devicetree/bindings/firmware/qcom,scm.txt deleted file mode 100644 index b3f702cbed87872be3998fc0b3d43e8f4cfe7288..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/firmware/qcom,scm.txt +++ /dev/null @@ -1,61 +0,0 @@ -QCOM Secure Channel Manager (SCM) - -Qualcomm processors include an interface to communicate to the secure firmware. -This interface allows for clients to request different types of actions. These -can include CPU power up/down, HDCP requests, loading of firmware, and other -assorted actions. - -Required properties: -- compatible: must contain one of the following: - * "qcom,scm-apq8064" - * "qcom,scm-apq8084" - * "qcom,scm-ipq4019" - * "qcom,scm-ipq806x" - * "qcom,scm-ipq8074" - * "qcom,scm-mdm9607" - * "qcom,scm-msm8226" - * "qcom,scm-msm8660" - * "qcom,scm-msm8916" - * "qcom,scm-msm8953" - * "qcom,scm-msm8960" - * "qcom,scm-msm8974" - * "qcom,scm-msm8976" - * "qcom,scm-msm8994" - * "qcom,scm-msm8996" - * "qcom,scm-msm8998" - * "qcom,scm-qcs404" - * "qcom,scm-sc7180" - * "qcom,scm-sc7280" - * "qcom,scm-sm6125" - * "qcom,scm-sdm845" - * "qcom,scm-sdx55" - * "qcom,scm-sdx65" - * "qcom,scm-sm6350" - * "qcom,scm-sm8150" - * "qcom,scm-sm8250" - * "qcom,scm-sm8350" - * "qcom,scm-sm8450" - and: - * "qcom,scm" -- clocks: Specifies clocks needed by the SCM interface, if any: - * core clock required for "qcom,scm-apq8064", "qcom,scm-msm8660" and - "qcom,scm-msm8960" - * core, iface and bus clocks required for "qcom,scm-apq8084", - "qcom,scm-msm8916", "qcom,scm-msm8953", "qcom,scm-msm8974" and "qcom,scm-msm8976" -- clock-names: Must contain "core" for the core clock, "iface" for the interface - clock and "bus" for the bus clock per the requirements of the compatible. -- qcom,dload-mode: phandle to the TCSR hardware block and offset of the - download mode control register (optional) -- interconnects: Specifies the bandwidth requirements of the SCM interface (optional) - -Example for MSM8916: - - firmware { - scm { - compatible = "qcom,msm8916", "qcom,scm"; - clocks = <&gcc GCC_CRYPTO_CLK> , - <&gcc GCC_CRYPTO_AXI_CLK>, - <&gcc GCC_CRYPTO_AHB_CLK>; - clock-names = "core", "bus", "iface"; - }; - }; diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5b76c9f7ad041b76a7e603ad3ea29533149762d --- /dev/null +++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/firmware/qcom,scm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: QCOM Secure Channel Manager (SCM) + +description: | + Qualcomm processors include an interface to communicate to the secure firmware. + This interface allows for clients to request different types of actions. + These can include CPU power up/down, HDCP requests, loading of firmware, + and other assorted actions. + +maintainers: + - Bjorn Andersson + - Robert Marko + - Guru Das Srinagesh + +properties: + compatible: + items: + - enum: + - qcom,scm-apq8064 + - qcom,scm-apq8084 + - qcom,scm-ipq4019 + - qcom,scm-ipq6018 + - qcom,scm-ipq806x + - qcom,scm-ipq8074 + - qcom,scm-mdm9607 + - qcom,scm-msm8226 + - qcom,scm-msm8660 + - qcom,scm-msm8916 + - qcom,scm-msm8953 + - qcom,scm-msm8960 + - qcom,scm-msm8974 + - qcom,scm-msm8976 + - qcom,scm-msm8994 + - qcom,scm-msm8996 + - qcom,scm-msm8998 + - qcom,scm-sc7180 + - qcom,scm-sc7280 + - qcom,scm-sc8280xp + - qcom,scm-sdm845 + - qcom,scm-sdx55 + - qcom,scm-sdx65 + - qcom,scm-sm6115 + - qcom,scm-sm6125 + - qcom,scm-sm6350 + - qcom,scm-sm8150 + - qcom,scm-sm8250 + - qcom,scm-sm8350 + - qcom,scm-sm8450 + - qcom,scm-qcs404 + - const: qcom,scm + + clocks: + minItems: 1 + maxItems: 3 + + clock-names: + minItems: 1 + maxItems: 3 + + interconnects: + maxItems: 1 + + interconnect-names: + maxItems: 1 + + '#reset-cells': + const: 1 + + qcom,dload-mode: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: phandle to TCSR hardware block + - description: offset of the download mode control register + description: TCSR hardware block + +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,scm-apq8064 + - qcom,scm-msm8660 + - qcom,scm-msm8960 + then: + properties: + clock-names: + items: + - const: core + + clocks: + maxItems: 1 + + required: + - clocks + - clock-names + + - if: + properties: + compatible: + contains: + enum: + - qcom,scm-apq8084 + - qcom,scm-mdm9607 + - qcom,scm-msm8916 + - qcom,scm-msm8953 + - qcom,scm-msm8974 + - qcom,scm-msm8976 + then: + properties: + clock-names: + items: + - const: core + - const: bus + - const: iface + + clocks: + minItems: 3 + maxItems: 3 + + required: + - clocks + - clock-names + +required: + - compatible + +additionalProperties: false + +examples: + - | + #include + + firmware { + scm { + compatible = "qcom,scm-msm8916", "qcom,scm"; + clocks = <&gcc GCC_CRYPTO_CLK>, + <&gcc GCC_CRYPTO_AXI_CLK>, + <&gcc GCC_CRYPTO_AHB_CLK>; + clock-names = "core", "bus", "iface"; + }; + }; diff --git a/Documentation/devicetree/bindings/fpga/microchip,mpf-spi-fpga-mgr.yaml b/Documentation/devicetree/bindings/fpga/microchip,mpf-spi-fpga-mgr.yaml index aee45cb15592dfe6277c92bf2c4b3495156304e1..527532f039cecad9b39248b74fa9839804da6345 100644 --- a/Documentation/devicetree/bindings/fpga/microchip,mpf-spi-fpga-mgr.yaml +++ b/Documentation/devicetree/bindings/fpga/microchip,mpf-spi-fpga-mgr.yaml @@ -22,13 +22,14 @@ properties: description: SPI chip select maxItems: 1 - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/fpga/xilinx-zynq-fpga-mgr.yaml b/Documentation/devicetree/bindings/fpga/xilinx-zynq-fpga-mgr.yaml index 29daca4be47fcd440e8bfd5bbfaa24bb801580cb..f47b6140a7429797a01b13bafb22e6d6589febbd 100644 --- a/Documentation/devicetree/bindings/fpga/xilinx-zynq-fpga-mgr.yaml +++ b/Documentation/devicetree/bindings/fpga/xilinx-zynq-fpga-mgr.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/fpga/xilinx-zynq-fpga-mgr.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx Zynq FPGA Manager Device Tree Bindings +title: Xilinx Zynq FPGA Manager maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/fpga/xlnx,zynqmp-pcap-fpga.yaml b/Documentation/devicetree/bindings/fpga/xlnx,zynqmp-pcap-fpga.yaml index 6cd2bdc06b5fefb59a600b597088b76cddafd7bd..00a8d92ff73680a45755d84b60414273b57cf929 100644 --- a/Documentation/devicetree/bindings/fpga/xlnx,zynqmp-pcap-fpga.yaml +++ b/Documentation/devicetree/bindings/fpga/xlnx,zynqmp-pcap-fpga.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/fpga/xlnx,zynqmp-pcap-fpga.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx Zynq Ultrascale MPSoC FPGA Manager Device Tree Bindings +title: Xilinx Zynq Ultrascale MPSoC FPGA Manager maintainers: - Nava kishore Manne diff --git a/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml b/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml index e62b30386ac25ed64c5a7f47f5bc6cc973ff7acb..c21549e0fba6421b2c49a141e0d1d3c787716f13 100644 --- a/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml +++ b/Documentation/devicetree/bindings/gnss/brcm,bcm4751.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gnss/brcm,bcm4751.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM4751 family GNSS Receiver Device Tree Bindings +title: Broadcom BCM4751 family GNSS Receiver maintainers: - Johan Hovold diff --git a/Documentation/devicetree/bindings/gnss/mediatek.yaml b/Documentation/devicetree/bindings/gnss/mediatek.yaml index 45cf01b277006aeab7711d263028004a1423e9dd..c0eb35beb2ef39a45cc778dabe31afd1a0ddb1d4 100644 --- a/Documentation/devicetree/bindings/gnss/mediatek.yaml +++ b/Documentation/devicetree/bindings/gnss/mediatek.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gnss/mediatek.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek GNSS Receiver Device Tree Bindings +title: Mediatek GNSS Receiver maintainers: - Johan Hovold diff --git a/Documentation/devicetree/bindings/gnss/sirfstar.yaml b/Documentation/devicetree/bindings/gnss/sirfstar.yaml index 991599cdaa6b0061700d29f14f7129862354f2eb..0bbe684d82e105a642ecf2a517f4d8b99ebf437c 100644 --- a/Documentation/devicetree/bindings/gnss/sirfstar.yaml +++ b/Documentation/devicetree/bindings/gnss/sirfstar.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gnss/sirfstar.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: SiRFstar GNSS Receiver Device Tree Bindings +title: SiRFstar GNSS Receiver maintainers: - Johan Hovold diff --git a/Documentation/devicetree/bindings/gnss/u-blox,neo-6m.yaml b/Documentation/devicetree/bindings/gnss/u-blox,neo-6m.yaml index 35a760cfd3430a1ce48c201abaf1e4e4c08b6742..4835a280b3bff543d03a6fdc0dfd33483661db4d 100644 --- a/Documentation/devicetree/bindings/gnss/u-blox,neo-6m.yaml +++ b/Documentation/devicetree/bindings/gnss/u-blox,neo-6m.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gnss/u-blox,neo-6m.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: U-blox GNSS Receiver Device Tree Bindings +title: U-blox GNSS Receiver allOf: - $ref: gnss-common.yaml# diff --git a/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml b/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml index a99e7842ca1739e2507134dee1aa66166334a7fd..c0ad70e66f760c30dbdb25f06935d246685b8f2b 100644 --- a/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml +++ b/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml @@ -33,8 +33,6 @@ properties: description: GPIO connected to the OE (Output Enable) pin. maxItems: 1 - spi-max-frequency: true - patternProperties: "^(hog-[0-9]+|.+-hog(-[0-9]+)?)$": type: object @@ -59,7 +57,10 @@ required: - '#gpio-cells' - registers-number -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/gpio/fsl,imx8qxp-sc-gpio.yaml b/Documentation/devicetree/bindings/gpio/fsl,imx8qxp-sc-gpio.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7b32220935d7b12932ecd165521020c4d814f3e --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/fsl,imx8qxp-sc-gpio.yaml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/fsl,imx8qxp-sc-gpio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: GPIO driver over IMX SCU firmware API + +maintainers: + - Shenwei Wang + +description: | + This module provides the standard interface to control the + resource pins in SCU domain on i.MX8 platforms. + +properties: + compatible: + enum: + - fsl,imx8qxp-sc-gpio + + "#gpio-cells": + const: 2 + + gpio-controller: true + +required: + - compatible + - "#gpio-cells" + - gpio-controller + +additionalProperties: false + +examples: + - | + gpio0: gpio { + compatible = "fsl,imx8qxp-sc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml index 977b14db09b04b382a416fb394ac2316bd37e671..1b70e9f308f3bed71f2e69fcabc015816cc584a6 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml @@ -15,52 +15,59 @@ description: |+ properties: compatible: - enum: - - exar,xra1202 - - maxim,max7310 - - maxim,max7312 - - maxim,max7313 - - maxim,max7315 - - maxim,max7319 - - maxim,max7320 - - maxim,max7321 - - maxim,max7322 - - maxim,max7323 - - maxim,max7324 - - maxim,max7325 - - maxim,max7326 - - maxim,max7327 - - nxp,pca6408 - - nxp,pca6416 - - nxp,pca9505 - - nxp,pca9506 - - nxp,pca9534 - - nxp,pca9535 - - nxp,pca9536 - - nxp,pca9537 - - nxp,pca9538 - - nxp,pca9539 - - nxp,pca9554 - - nxp,pca9555 - - nxp,pca9556 - - nxp,pca9557 - - nxp,pca9574 - - nxp,pca9575 - - nxp,pca9698 - - nxp,pcal6416 - - nxp,pcal6524 - - nxp,pcal9535 - - nxp,pcal9554b - - nxp,pcal9555a - - onnn,cat9554 - - onnn,pca9654 - - ti,pca6107 - - ti,pca9536 - - ti,tca6408 - - ti,tca6416 - - ti,tca6424 - - ti,tca9539 - - ti,tca9554 + oneOf: + - items: + - const: diodes,pi4ioe5v6534q + - const: nxp,pcal6534 + - items: + - enum: + - exar,xra1202 + - maxim,max7310 + - maxim,max7312 + - maxim,max7313 + - maxim,max7315 + - maxim,max7319 + - maxim,max7320 + - maxim,max7321 + - maxim,max7322 + - maxim,max7323 + - maxim,max7324 + - maxim,max7325 + - maxim,max7326 + - maxim,max7327 + - nxp,pca6408 + - nxp,pca6416 + - nxp,pca9505 + - nxp,pca9506 + - nxp,pca9534 + - nxp,pca9535 + - nxp,pca9536 + - nxp,pca9537 + - nxp,pca9538 + - nxp,pca9539 + - nxp,pca9554 + - nxp,pca9555 + - nxp,pca9556 + - nxp,pca9557 + - nxp,pca9574 + - nxp,pca9575 + - nxp,pca9698 + - nxp,pcal6408 + - nxp,pcal6416 + - nxp,pcal6524 + - nxp,pcal6534 + - nxp,pcal9535 + - nxp,pcal9554b + - nxp,pcal9555a + - onnn,cat9554 + - onnn,pca9654 + - ti,pca6107 + - ti,pca9536 + - ti,tca6408 + - ti,tca6416 + - ti,tca6424 + - ti,tca9539 + - ti,tca9554 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/gpio/gpio-stmpe.txt b/Documentation/devicetree/bindings/gpio/gpio-stmpe.txt index a0e4cf8852139eea53f193c9e6f961544d1f026c..b33f8f02c0d729c5871c4e5790f18e621d6ac95b 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-stmpe.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-stmpe.txt @@ -8,8 +8,7 @@ Optional properties: - st,norequest-mask: bitmask specifying which GPIOs should _not_ be requestable due to different usage (e.g. touch, keypad) -Node name must be stmpe_gpio and should be child node of stmpe node to which it -belongs. +Node should be child node of stmpe node to which it belongs. Example: stmpe_gpio { diff --git a/Documentation/devicetree/bindings/gpio/gpio-xilinx.txt b/Documentation/devicetree/bindings/gpio/gpio-xilinx.txt deleted file mode 100644 index e506f30e1a95072afc180f4c4a2f15bea7fc934c..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-xilinx.txt +++ /dev/null @@ -1,48 +0,0 @@ -Xilinx plb/axi GPIO controller - -Dual channel GPIO controller with configurable number of pins -(from 1 to 32 per channel). Every pin can be configured as -input/output/tristate. Both channels share the same global IRQ but -local interrupts can be enabled on channel basis. - -Required properties: -- compatible : Should be "xlnx,xps-gpio-1.00.a" -- reg : Address and length of the register set for the device -- #gpio-cells : Should be two. The first cell is the pin number and the - second cell is used to specify optional parameters (currently unused). -- gpio-controller : Marks the device node as a GPIO controller. - -Optional properties: -- clocks : Input clock specifier. Refer to common clock bindings. -- interrupts : Interrupt mapping for GPIO IRQ. -- xlnx,all-inputs : if n-th bit is setup, GPIO-n is input -- xlnx,dout-default : if n-th bit is 1, GPIO-n default value is 1 -- xlnx,gpio-width : gpio width -- xlnx,tri-default : if n-th bit is 1, GPIO-n is in tristate mode -- xlnx,is-dual : if 1, controller also uses the second channel -- xlnx,all-inputs-2 : as above but for the second channel -- xlnx,dout-default-2 : as above but the second channel -- xlnx,gpio2-width : as above but for the second channel -- xlnx,tri-default-2 : as above but for the second channel - - -Example: -gpio: gpio@40000000 { - #gpio-cells = <2>; - compatible = "xlnx,xps-gpio-1.00.a"; - clocks = <&clkc25>; - gpio-controller ; - interrupt-parent = <µblaze_0_intc>; - interrupts = < 6 2 >; - reg = < 0x40000000 0x10000 >; - xlnx,all-inputs = <0x0>; - xlnx,all-inputs-2 = <0x0>; - xlnx,dout-default = <0x0>; - xlnx,dout-default-2 = <0x0>; - xlnx,gpio-width = <0x2>; - xlnx,gpio2-width = <0x2>; - xlnx,interrupt-present = <0x1>; - xlnx,is-dual = <0x1>; - xlnx,tri-default = <0xffffffff>; - xlnx,tri-default-2 = <0xffffffff>; -} ; diff --git a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml index 29c27eadbac8151225b551f2dff5da08799d0f28..572e1718f5015243ead10ad282b20a0ce38c29ef 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gpio/gpio-zynq.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx Zynq GPIO controller Device Tree Bindings +title: Xilinx Zynq GPIO controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml index 110651eafa70f74eaa371af4bea8f5bf0c394528..d481e78958a74ae9f62f61c83d12eaf9caf51a3f 100644 --- a/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/gpio/microchip,mpfs-gpio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip MPFS GPIO Controller Device Tree Bindings +title: Microchip MPFS GPIO Controller maintainers: - Conor Dooley @@ -44,6 +44,24 @@ properties: gpio-controller: true +patternProperties: + "^.+-hog(-[0-9]+)?$": + type: object + + additionalProperties: false + + properties: + gpio-hog: true + gpios: true + input: true + output-high: true + output-low: true + line-name: true + + required: + - gpio-hog + - gpios + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index 75e5da6a7cc04bbd9768540e012615b474d1f946..aa424e2b95f87a51eea1597d42eb60ee45de5a81 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -52,6 +52,7 @@ properties: - enum: - renesas,gpio-r8a779a0 # R-Car V3U - renesas,gpio-r8a779f0 # R-Car S4-8 + - renesas,gpio-r8a779g0 # R-Car V4H - const: renesas,rcar-gen4-gpio # R-Car Gen4 reg: diff --git a/Documentation/devicetree/bindings/gpio/x-powers,axp209-gpio.yaml b/Documentation/devicetree/bindings/gpio/x-powers,axp209-gpio.yaml index 14486aee97b43639c4b93f76a3e2c727b07b4343..7f26f6b1eea1105127d204f5e01967df661ad1df 100644 --- a/Documentation/devicetree/bindings/gpio/x-powers,axp209-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/x-powers,axp209-gpio.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/gpio/x-powers,axp209-gpio.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: X-Powers AXP209 GPIO Device Tree Bindings +title: X-Powers AXP209 GPIO maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/gpio/xlnx,gpio-xilinx.yaml b/Documentation/devicetree/bindings/gpio/xlnx,gpio-xilinx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f333ee2288e7683f00a379dfe7618aa8682bc2cf --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/xlnx,gpio-xilinx.yaml @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/xlnx,gpio-xilinx.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xilinx AXI GPIO controller + +maintainers: + - Neeli Srinivas + +description: + The AXI GPIO design provides a general purpose input/output interface + to an AXI4-Lite interface. The AXI GPIO can be configured as either + a single or a dual-channel device. The width of each channel is + independently configurable. The channels can be configured to + generate an interrupt when a transition on any of their inputs occurs. + +properties: + compatible: + enum: + - xlnx,xps-gpio-1.00.a + + reg: + maxItems: 1 + + "#gpio-cells": + const: 2 + + interrupts: + maxItems: 1 + + gpio-controller: true + + gpio-line-names: + description: strings describing the names of each gpio line + minItems: 1 + maxItems: 64 + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + clocks: + maxItems: 1 + + interrupt-names: true + + xlnx,all-inputs: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This option sets this GPIO channel1 bits in input mode. + + xlnx,all-inputs-2: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This option sets this GPIO channel2 bits in input mode. + + xlnx,all-outputs: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This option sets this GPIO channel1 bits in output mode. + + xlnx,all-outputs-2: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This option sets this GPIO channel2 bits in output mode. + + xlnx,dout-default: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Sets the default value of all the enabled bits of + channel1. + default: 0 + + xlnx,dout-default-2: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Sets the default value of all the enabled bits of + channel2. + default: 0 + + xlnx,gpio-width: + $ref: /schemas/types.yaml#/definitions/uint32 + description: The value defines the bit width of the GPIO channel1. + minimum: 1 + maximum: 32 + default: 32 + + xlnx,gpio2-width: + $ref: /schemas/types.yaml#/definitions/uint32 + description: The value defines the bit width of the GPIO channel2. + minimum: 1 + maximum: 32 + default: 32 + + xlnx,interrupt-present: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This parameter enables interrupt control logic + and interrupt registers in GPIO module. + minimum: 0 + maximum: 1 + default: 0 + + xlnx,is-dual: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This parameter enables a second GPIO channel (GPIO2). + minimum: 0 + maximum: 1 + default: 0 + + xlnx,tri-default: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This value configures the input or output mode + of each bit of GPIO channel1. + + xlnx,tri-default-2: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This value configures the input or output mode + of each bit of GPIO channel2. + +required: + - reg + - compatible + - gpio-controller + - "#gpio-cells" + +unevaluatedProperties: false + +examples: + - | + #include + + gpio@e000a000 { + compatible = "xlnx,xps-gpio-1.00.a"; + reg = <0xa0020000 0x10000>; + #gpio-cells = <2>; + #interrupt-cells = <0x2>; + clocks = <&zynqmp_clk 71>; + gpio-controller; + interrupt-controller; + interrupt-names = "ip2intc_irpt"; + interrupt-parent = <&gic>; + interrupts = <0 89 4>; + xlnx,all-inputs = <0x0>; + xlnx,all-inputs-2 = <0x0>; + xlnx,all-outputs = <0x0>; + xlnx,all-outputs-2 = <0x0>; + xlnx,dout-default = <0x0>; + xlnx,dout-default-2 = <0x0>; + xlnx,gpio-width = <0x20>; + xlnx,gpio2-width = <0x20>; + xlnx,interrupt-present = <0x1>; + xlnx,is-dual = <0x1>; + xlnx,tri-default = <0xFFFFFFFF>; + xlnx,tri-default-2 = <0xFFFFFFFF>; + }; + +... diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml index d209f272625d067e1b19a50b8034ce663ae01019..2a25384ca3ef1e5fba1ee03d801e996f850c1adb 100644 --- a/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml +++ b/Documentation/devicetree/bindings/gpu/arm,mali-midgard.yaml @@ -74,7 +74,8 @@ properties: - const: bus mali-supply: true - opp-table: true + opp-table: + type: object power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml index eceaa176bd57e29c0ea2d01ac8869bc00090d5a7..318122d95eb5bde41083a48cf4a8d8bfd498bc59 100644 --- a/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml +++ b/Documentation/devicetree/bindings/gpu/arm,mali-utgard.yaml @@ -101,7 +101,8 @@ properties: mali-supply: true - opp-table: true + opp-table: + type: object power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml b/Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml index 6747b870f29783edf3faf140a752c00b4c48c191..f2f99afb3a3ba7798fef1bcbe56d942ebc455c07 100644 --- a/Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml +++ b/Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/hwmon/adi,axi-fan-control.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices AXI FAN Control Device Tree Bindings +title: Analog Devices AXI FAN Control maintainers: - Nuno Sá diff --git a/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml b/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f2d08d7b978dc0a0ebad1cde0c8ea2cb4ee96fb --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/adi,max31760.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices MAX31760 Fan-Speed Controller + +maintainers: + - Ibrahim Tilki + +description: | + Analog Devices MAX31760 Fan-Speed Controller + https://datasheets.maximintegrated.com/en/ds/MAX31760.pdf + +properties: + compatible: + enum: + - adi,max31760 + + reg: + description: I2C address of slave device. + minimum: 0x50 + maximum: 0x57 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + fan-controller@50 { + reg = <0x50>; + compatible = "adi,max31760"; + }; + }; diff --git a/Documentation/devicetree/bindings/hwmon/ibm,occ-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/ibm,occ-hwmon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dbdc5af2804afeaa320d88ea3d162db8993735a --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/ibm,occ-hwmon.yaml @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/ibm,occ-hwmon.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: IBM On-Chip Controller (OCC) accessed from a service processor + +maintainers: + - Eddie James + +description: | + The POWER processor On-Chip Controller (OCC) helps manage power and + thermals for the system. A service processor or baseboard management + controller can query the OCC for it's power and thermal data to report + through hwmon. + +properties: + compatible: + enum: + - ibm,p9-occ-hwmon + - ibm,p10-occ-hwmon + + ibm,no-poll-on-init: + description: This property describes whether or not the OCC should + be polled during driver initialization. + type: boolean + +required: + - compatible + +additionalProperties: false + +examples: + - | + hwmon { + compatible = "ibm,p10-occ-hwmon"; + ibm,no-poll-on-init; + }; diff --git a/Documentation/devicetree/bindings/hwmon/iio-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/iio-hwmon.yaml index f5a6cc3efd33242c4cb61571204211ec8785eb61..e1ccbd30e0ebd6c4de54cd029531f730f28d1867 100644 --- a/Documentation/devicetree/bindings/hwmon/iio-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/iio-hwmon.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/hwmon/iio-hwmon.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: ADC-attached Hardware Sensor Device Tree Bindings +title: ADC-attached Hardware Sensor maintainers: - Jonathan Cameron diff --git a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml index 76be625d56460d8150f84cd3091a84e500c9afc1..51e8619dbf3c6207f002c8f7b483d97e56ccbaf0 100644 --- a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml +++ b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml @@ -22,7 +22,7 @@ properties: clocks: items: - - description: AHB reference clock + - description: System reference clock '#thermal-sensor-cells': const: 0 @@ -40,5 +40,5 @@ examples: compatible = "microchip,sparx5-temp"; reg = <0x10508110 0xc>; #thermal-sensor-cells = <0>; - clocks = <&ahb_clk>; + clocks = <&sys_clk>; }; diff --git a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml index b79f069a04c2ad17462c0d8bff173ebbb54e8dba..d0d54974920875aadac57f133be8417aa498bef7 100644 --- a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml +++ b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml @@ -9,6 +9,32 @@ title: Moortec Semiconductor MR75203 PVT Controller bindings maintainers: - Rahul Tanwar +description: | + A Moortec PVT (Process, Voltage, Temperature) monitoring logic design can + include many different units. + Such a design will usually consists of several Moortec's embedded analog IPs, + and a single Moortec controller (mr75203) to configure and control the IPs. + + Some of the Moortec's analog hard IPs that can be used in a design: + *) Temperature Sensor (TS) - used to monitor core temperature (e.g. mr74137). + *) Voltage Monitor (VM) - used to monitor voltage levels (e.g. mr74138). + *) Process Detector (PD) - used to assess silicon speed (e.g. mr74139). + *) Delay Chain - ring oscillator connected to the PD, used to measure IO + based transistors (e.g. mr76008 ring oscillator at 1.1V, mr76007 ring + oscillator at 1.8V). + *) Pre Scaler - provides divide-by-X scaling of input voltage, which can then + be presented for VM for measurement within its range (e.g. mr76006 - + divide by 2 pre-scaler). + + TS, VM & PD also include a digital interface, which consists of configuration + inputs and measurement outputs. + + Some of the units have number of series, each series can have slightly + different characteristics. + + The mr75203 binding describes configuration for the controller unit, but also + for some of the analog IPs. + properties: compatible: const: moortec,mr75203 @@ -44,13 +70,76 @@ properties: "#thermal-sensor-cells": const: 1 + moortec,vm-active-channels: + description: + Defines the number of channels per VM that are actually used and are + connected to some input source. + Maximum number of items - number of VMs. + Maximum value of each item - number of channels. + Minimum value of each item - 0 (which means entire VM sensor is not used). + $ref: /schemas/types.yaml#/definitions/uint8-array + + moortec,vm-pre-scaler-x2: + description: + Defines the channels that use a mr76006 pre-scaler to divide the input + source by 2. + The pre-scaler is used for input sources that exceed the VM input range. + The driver uses this information to present to the user with the actual + value of the voltage source. + For channels that are not listed, no pre-scaler is assumed. + Maximum number of items - total number of channels in all VMs. + Each channel should not appear more than once. + $ref: /schemas/types.yaml#/definitions/uint8-array + + moortec,ts-series: + description: + Definition of the temperature equation and coefficients that shall be + used to convert the digital output to value in milli-Celsius. + minimum: 5 + maximum: 6 + default: 5 + $ref: /schemas/types.yaml#/definitions/uint32 + + moortec,ts-coeff-g: + description: + G coefficient for temperature equation. + Default for series 5 = 60000 + Default for series 6 = 57400 + multipleOf: 1000 + minimum: 1000 + $ref: /schemas/types.yaml#/definitions/uint32 + + moortec,ts-coeff-h: + description: + H coefficient for temperature equation. + Default for series 5 = 200000 + Default for series 6 = 249400 + multipleOf: 1000 + minimum: 1000 + $ref: /schemas/types.yaml#/definitions/uint32 + + moortec,ts-coeff-cal5: + description: + cal5 coefficient for temperature equation. + Default for series 5 = 4094 + Default for series 6 = 4096 + minimum: 1 + $ref: /schemas/types.yaml#/definitions/uint32 + + moortec,ts-coeff-j: + description: + J coefficient for temperature equation. + Default for series 5 = -100 + Default for series 6 = 0 + multipleOf: 1000 + maximum: 0 + $ref: /schemas/types.yaml#/definitions/int32 + required: - compatible - reg - reg-names - - intel,vm-map - clocks - - resets - "#thermal-sensor-cells" additionalProperties: false @@ -67,5 +156,9 @@ examples: intel,vm-map = [03 01 04 ff ff]; clocks = <&osc0>; resets = <&rcu0 0x40 7>; + moortec,vm-active-channels = /bits/ 8 <0x10 0x05>; + moortec,vm-pre-scaler-x2 = /bits/ 8 <5 6 20>; + moortec,ts-coeff-g = <61400>; + moortec,ts-coeff-h = <253700>; #thermal-sensor-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml b/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml index 7d49478d9668ecef500ac51a062d5ee854ec67ae..159238efa9ed26f1333a1d5f94342b553ac1ba94 100644 --- a/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml +++ b/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml @@ -10,7 +10,7 @@ maintainers: - Christopher Ruehl chris.ruehl@gtsys.com.hk description: | - The SHTC1, SHTW1 and SHTC3 are digital humidity and temperature sensor + The SHTC1, SHTW1 and SHTC3 are digital humidity and temperature sensors designed especially for battery-driven high-volume consumer electronics applications. For further information refere to Documentation/hwmon/shtc1.rst @@ -31,13 +31,13 @@ properties: sensirion,blocking-io: $ref: /schemas/types.yaml#/definitions/flag description: - If set, the driver hold the i2c bus until measurement is finished. + If set, the driver holds the i2c bus until the measurement is finished. sensirion,low-precision: $ref: /schemas/types.yaml#/definitions/flag description: - If set, the sensor aquire data with low precision (not recommended). - The driver aquire data with high precision by default. + If set, the sensor acquires data with low precision (not recommended). + The driver acquires data with high precision by default. required: - compatible diff --git a/Documentation/devicetree/bindings/i2c/allwinner,sun6i-a31-p2wi.yaml b/Documentation/devicetree/bindings/i2c/allwinner,sun6i-a31-p2wi.yaml index 1b03810d4b4d0cd64a78cd06dfa4f61eff3fe954..5a799246a373c558cbccc5b6ffef56d10edb0ae7 100644 --- a/Documentation/devicetree/bindings/i2c/allwinner,sun6i-a31-p2wi.yaml +++ b/Documentation/devicetree/bindings/i2c/allwinner,sun6i-a31-p2wi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/i2c/allwinner,sun6i-a31-p2wi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 P2WI (Push/Pull 2 Wires Interface) Device Tree Bindings +title: Allwinner A31 P2WI (Push/Pull 2 Wires Interface) maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml b/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml index 6ecb0270d88d84e2a5b3a873f940bde0011298cc..199a354ccb9701ecd692481244e1bc7522f87172 100644 --- a/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/amlogic,meson6-i2c.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson I2C Controller maintainers: - - Neil Armstrong + - Neil Armstrong - Beniamino Galvani allOf: diff --git a/Documentation/devicetree/bindings/i2c/aspeed,i2c.yaml b/Documentation/devicetree/bindings/i2c/aspeed,i2c.yaml index f597f73ccd8748e213b4839357131e3e02acfc40..869b4d633353dcaf516ea51667978d63e9117e14 100644 --- a/Documentation/devicetree/bindings/i2c/aspeed,i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/aspeed,i2c.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/i2c/aspeed,i2c.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ASPEED I2C on the AST24XX, AST25XX, and AST26XX SoCs Device Tree Bindings +title: ASPEED I2C on the AST24XX, AST25XX, and AST26XX SoCs maintainers: - Rayn Chen diff --git a/Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml b/Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml index 1ca1cd19bd1dd8ba5c618d1eeb484c5e6fd180c4..2e95cda7262ad11d0669221c9bf11abe1488f2f1 100644 --- a/Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml +++ b/Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/i2c/cdns,i2c-r1p10.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: Cadence I2C controller Device Tree Bindings +title: Cadence I2C controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml index 529bea56d3247a2c0c3ce1a7eb425551f12bce2a..4656f5112b84e97d7642e8bec4eb2fe26c5ac8da 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml +++ b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml @@ -23,6 +23,7 @@ properties: - fsl,imx8dxl-lpi2c - fsl,imx8qm-lpi2c - fsl,imx8ulp-lpi2c + - fsl,imx93-lpi2c - const: fsl,imx7ulp-lpi2c reg: @@ -37,10 +38,22 @@ properties: clock-frequency: true clock-names: - maxItems: 1 + items: + - const: per + - const: ipg clocks: - maxItems: 1 + maxItems: 2 + + dmas: + items: + - description: DMA controller phandle and request line for TX + - description: DMA controller phandle and request line for RX + + dma-names: + items: + - const: tx + - const: rx power-domains: maxItems: 1 @@ -63,5 +76,6 @@ examples: reg = <0x40A50000 0x10000>; interrupt-parent = <&intc>; interrupts = ; - clocks = <&clks IMX7ULP_CLK_LPI2C7>; + clocks = <&clks IMX7ULP_CLK_LPI2C7>, + <&clks IMX7ULP_CLK_NIC1_BUS_DIV>; }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx.yaml b/Documentation/devicetree/bindings/i2c/i2c-imx.yaml index 01720e338b4c35ad30aa9e72fd4b6027a278cfce..85ee1282d6d2d28ed7d75fe235bec1adeabacfa6 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-imx.yaml +++ b/Documentation/devicetree/bindings/i2c/i2c-imx.yaml @@ -18,6 +18,17 @@ properties: - const: fsl,imx1-i2c - const: fsl,imx21-i2c - const: fsl,vf610-i2c + - items: + - enum: + - fsl,ls1012a-i2c + - fsl,ls1021a-i2c + - fsl,ls1028a-i2c + - fsl,ls1043a-i2c + - fsl,ls1046a-i2c + - fsl,ls1088a-i2c + - fsl,ls208xa-i2c + - fsl,lx2160a-i2c + - const: fsl,vf610-i2c - items: - const: fsl,imx35-i2c - const: fsl,imx1-i2c diff --git a/Documentation/devicetree/bindings/i2c/i2c-mt7621.txt b/Documentation/devicetree/bindings/i2c/i2c-mt7621.txt deleted file mode 100644 index bc36f0eb94cd01278fde62052998dcffeeaa59b4..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-mt7621.txt +++ /dev/null @@ -1,25 +0,0 @@ -MediaTek MT7621/MT7628 I2C master controller - -Required properties: - -- compatible: Should be one of the following: - - "mediatek,mt7621-i2c": for MT7621/MT7628/MT7688 platforms -- #address-cells: should be 1. -- #size-cells: should be 0. -- reg: Address and length of the register set for the device -- resets: phandle to the reset controller asserting this device in - reset - See ../reset/reset.txt for details. - -Optional properties : - -Example: - -i2c: i2c@900 { - compatible = "mediatek,mt7621-i2c"; - reg = <0x900 0x100>; - #address-cells = <1>; - #size-cells = <0>; - resets = <&rstctrl 16>; - reset-names = "i2c"; -}; diff --git a/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml b/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml index ee9f8b91d2e21917768ea08ba5dab398ce0d2077..82b9d6682297b941c23c2c6f51495bccd5fc4652 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml +++ b/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml @@ -29,6 +29,7 @@ properties: - items: - enum: - rockchip,rk3036-i2c + - rockchip,rk3128-i2c - rockchip,rk3368-i2c - const: rockchip,rk3288-i2c - items: diff --git a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml index 0ec033e488304ab8d35cd0ea80473e01ad7dcbcb..93c164aa00dafc43b558c223bd06461b08c34cc9 100644 --- a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/i2c/marvell,mv64xxx-i2c.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Marvell MV64XXX I2C Controller Device Tree Bindings +title: Marvell MV64XXX I2C Controller maintainers: - Gregory CLEMENT @@ -66,6 +66,19 @@ properties: resets: maxItems: 1 + dmas: + items: + - description: RX DMA Channel + - description: TX DMA Channel + + dma-names: + items: + - const: rx + - const: tx + +dependencies: + dmas: [ dma-names ] + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/i2c/mediatek,mt7621-i2c.yaml b/Documentation/devicetree/bindings/i2c/mediatek,mt7621-i2c.yaml new file mode 100644 index 0000000000000000000000000000000000000000..118ec00fc1905f3e7e04518f0f492c71f18501b8 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/mediatek,mt7621-i2c.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/mediatek,mt7621-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +maintainers: + - Stefan Roese + +title: Mediatek MT7621/MT7628 I2C master controller + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +properties: + compatible: + const: mediatek,mt7621-i2c + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: i2c + + resets: + maxItems: 1 + + reset-names: + const: i2c + +required: + - compatible + - reg + - resets + - "#address-cells" + - "#size-cells" + +unevaluatedProperties: false + +examples: + - | + #include + #include + + i2c: i2c@900 { + compatible = "mediatek,mt7621-i2c"; + reg = <0x900 0x100>; + clocks = <&sysc MT7621_CLK_I2C>; + clock-names = "i2c"; + resets = <&sysc MT7621_RST_I2C>; + reset-names = "i2c"; + + #address-cells = <1>; + #size-cells = <0>; + + pinctrl-names = "default"; + pinctrl-0 = <&i2c_pins>; + }; diff --git a/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml b/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml deleted file mode 100644 index 93198d5d43a6c902a6fea3e4940eb30d61f219ec..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/i2c/mellanox,i2c-mlxbf.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Mellanox I2C SMBus on BlueField SoCs - -maintainers: - - Khalil Blaiech - -allOf: - - $ref: /schemas/i2c/i2c-controller.yaml# - -properties: - compatible: - enum: - - mellanox,i2c-mlxbf1 - - mellanox,i2c-mlxbf2 - - reg: - minItems: 3 - items: - - description: Smbus block registers - - description: Cause master registers - - description: Cause slave registers - - description: Cause coalesce registers - - interrupts: - maxItems: 1 - - clock-frequency: - enum: [ 100000, 400000, 1000000 ] - description: - bus frequency used to configure timing registers; - The frequency is expressed in Hz. Default is 100000. - -required: - - compatible - - reg - - interrupts - -unevaluatedProperties: false - -if: - properties: - compatible: - contains: - enum: - - mellanox,i2c-mlxbf1 - -then: - properties: - reg: - maxItems: 3 - -examples: - - | - i2c@2804000 { - compatible = "mellanox,i2c-mlxbf1"; - reg = <0x02804000 0x800>, - <0x02801200 0x020>, - <0x02801260 0x020>; - interrupts = <57>; - clock-frequency = <100000>; - }; - - - | - i2c@2808800 { - compatible = "mellanox,i2c-mlxbf2"; - reg = <0x02808800 0x600>, - <0x02808e00 0x020>, - <0x02808e20 0x020>, - <0x02808e40 0x010>; - interrupts = <57>; - clock-frequency = <400000>; - }; diff --git a/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml b/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml index 7bad4b946a349c8b21c252c2cdd2b95646cefe02..afa3db7262298e149bc762c080073fe8b980cd02 100644 --- a/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml +++ b/Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/i2c/microchip,corei2c.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip MPFS I2C Controller Device Tree Bindings +title: Microchip MPFS I2C Controller maintainers: - Daire McNamara diff --git a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml index 09d2591e1fa3dbd4620e89ffc80fd7c1dd9f76db..00eb6ff6f5b18d384d54beb15ab4c8257477f2f5 100644 --- a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/i2c/nuvoton,npcm7xx-i2c.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: nuvoton NPCM7XX I2C Controller Device Tree Bindings +title: nuvoton NPCM7XX I2C Controller description: | I2C bus controllers of the NPCM series support both master and diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml index 90c9e401229ea8824c588792aca8ee42b9744bd4..cf9f8fda595fc326dd91857e6ec7929ebe599aad 100644 --- a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml +++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml @@ -13,6 +13,7 @@ maintainers: properties: compatible: enum: + - qcom,msm8226-cci - qcom,msm8916-cci - qcom,msm8974-cci - qcom,msm8996-cci @@ -27,11 +28,11 @@ properties: const: 0 clocks: - minItems: 4 + minItems: 3 maxItems: 6 clock-names: - minItems: 4 + minItems: 3 maxItems: 6 interrupts: @@ -78,11 +79,29 @@ allOf: compatible: contains: enum: + - qcom,msm8226-cci - qcom,msm8916-cci then: properties: i2c-bus@1: false + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8226-cci + - qcom,msm8974-cci + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: camss_top_ahb + - const: cci_ahb + - const: cci + - if: properties: compatible: @@ -126,6 +145,7 @@ allOf: contains: enum: - qcom,sm8250-cci + - qcom,sm8450-cci then: properties: clocks: diff --git a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml index f9929578c7613f0799d774ed28a00a5b745c01a2..c4ace5585e1e22d32b17abc5a859f5c3b298b258 100644 --- a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml @@ -52,6 +52,7 @@ properties: - enum: - renesas,i2c-r8a779a0 # R-Car V3U - renesas,i2c-r8a779f0 # R-Car S4-8 + - renesas,i2c-r8a779g0 # R-Car V4H - const: renesas,rcar-gen4-i2c # R-Car Gen4 reg: diff --git a/Documentation/devicetree/bindings/i2c/renesas,riic.yaml b/Documentation/devicetree/bindings/i2c/renesas,riic.yaml index 2f315489aaaecc7fee12c359eed1ebe68cb06e29..d3c0d5c427acbcbf39fd6ad07fb525980dac3bd8 100644 --- a/Documentation/devicetree/bindings/i2c/renesas,riic.yaml +++ b/Documentation/devicetree/bindings/i2c/renesas,riic.yaml @@ -60,6 +60,9 @@ properties: power-domains: maxItems: 1 + resets: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml index a415887637862059540914cb4b75af2ed2e16328..bf396e9466aaf7ca1dbc218a2a6585708406deed 100644 --- a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml @@ -57,6 +57,11 @@ properties: - description: interrupt ID for I2C event - description: interrupt ID for I2C error + interrupt-names: + items: + - const: event + - const: error + resets: maxItems: 1 @@ -92,6 +97,8 @@ properties: - description: register offset within syscfg - description: register bitmask for FMP bit + wakeup-source: true + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml b/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml index 715dcfa5a92229060730db883bccae8c77305148..8d241a703d855d658ea44f0424b47e55d4183e19 100644 --- a/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml +++ b/Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/i2c/xlnx,xps-iic-2.00.a.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: Xilinx IIC controller Device Tree Bindings +title: Xilinx IIC controller maintainers: - info@mocean-labs.com diff --git a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml index 04da001fc6ec952786b09470cfeba3e06e80685c..c002afdbfc7ce051e42d9510ac667c3ac4934764 100644 --- a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml +++ b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/i3c/mipi-i3c-hci.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: MIPI I3C HCI Device Tree Bindings +title: MIPI I3C HCI maintainers: - Nicolas Pitre diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml index 6f8f8a6258fe41e7666039530698a5b7fdf06af9..7332442e5661d25f5f5546b0766226a0f653eebb 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adis16201.yaml @@ -27,15 +27,16 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - vdd-supply: true required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adis16240.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adis16240.yaml index 8d829ef878bc260fe6628a1b5dfef915be15623b..f6f97164c2cac8f25c967b631418ec8d9e8156fd 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adis16240.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adis16240.yaml @@ -25,14 +25,15 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - required: - compatible - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl313.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl313.yaml index d6afc1b8c272879e8e98fdb49db8d06108d30ede..185b68ffb536a3287d9bcaa80f73fe746b314d7c 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adxl313.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl313.yaml @@ -4,28 +4,30 @@ $id: http://devicetree.org/schemas/iio/accel/adi,adxl313.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices ADXL313 3-Axis Digital Accelerometer +title: Analog Devices ADXL312, ADXL313, and ADXL314 3-Axis Digital Accelerometers maintainers: - Lucas Stankus description: | - Analog Devices ADXL313 3-Axis Digital Accelerometer that supports - both I2C & SPI interfaces. + Analog Devices ADXL312, ADXL313, and ADXL314 3-Axis Digital Accelerometer that + support both I2C & SPI interfaces. + https://www.analog.com/en/products/adxl312.html https://www.analog.com/en/products/adxl313.html + https://www.analog.com/en/products/adxl314.html properties: compatible: enum: + - adi,adxl312 - adi,adxl313 + - adi,adxl314 reg: maxItems: 1 spi-3wire: true - spi-max-frequency: true - vs-supply: description: Regulator that supplies power to the accelerometer @@ -48,7 +50,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl345.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl345.yaml index 9bb039e2f53316b5f5c51250ba545211379ca2f1..346abfb13a3a916a5f277519005405c238ad6b00 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adxl345.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl345.yaml @@ -32,8 +32,6 @@ properties: spi-cpol: true - spi-max-frequency: true - interrupts: maxItems: 1 @@ -42,7 +40,10 @@ required: - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl355.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl355.yaml index ba54d6998f2ee78040eee005249bbc3f4f2a8bc2..14b487088ab49e902dcb36b6902300eb906f2956 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adxl355.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl355.yaml @@ -45,13 +45,14 @@ properties: vddio-supply: description: Regulator that provides power to the bus - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml index d259e796c1d6881aa2722d9dc9d13ad2315787a0..f10d98d34cb83ebc2de227563f79181cf319d699 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml @@ -35,8 +35,6 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - vdd-supply: true vddio-supply: true @@ -45,7 +43,10 @@ required: - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/adi,adxl372.yaml b/Documentation/devicetree/bindings/iio/accel/adi,adxl372.yaml index 38b59b6454ce7dcba958977ad5cdc25d59bfdf50..73a5c8f814cc22f94ea484bcb39405b08e11906f 100644 --- a/Documentation/devicetree/bindings/iio/accel/adi,adxl372.yaml +++ b/Documentation/devicetree/bindings/iio/accel/adi,adxl372.yaml @@ -25,14 +25,15 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - required: - compatible - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/bosch,bma220.yaml b/Documentation/devicetree/bindings/iio/accel/bosch,bma220.yaml index 942b23ad07120a5f98f4e7ae6de12dcb3e99897e..5dd06f5905b4b876978d15087d15b2df14f5402c 100644 --- a/Documentation/devicetree/bindings/iio/accel/bosch,bma220.yaml +++ b/Documentation/devicetree/bindings/iio/accel/bosch,bma220.yaml @@ -20,8 +20,6 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - vdda-supply: true vddd-supply: true vddio-supply: true @@ -30,7 +28,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml index 478e75ae0885e7fb5b94345ea58e9a1910cbef4a..457a709b583c95744a3298e9fac43492f06b9d76 100644 --- a/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml +++ b/Documentation/devicetree/bindings/iio/accel/bosch,bma255.yaml @@ -72,7 +72,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/bosch,bmi088.yaml b/Documentation/devicetree/bindings/iio/accel/bosch,bmi088.yaml index 272eb48eef5a5335ceb42b6e0635b0ae6acb5e40..3cb82576d7586427b709d32f568211760d0e417f 100644 --- a/Documentation/devicetree/bindings/iio/accel/bosch,bmi088.yaml +++ b/Documentation/devicetree/bindings/iio/accel/bosch,bmi088.yaml @@ -24,8 +24,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vdd-supply: true vddio-supply: true @@ -50,7 +48,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/fsl,mma7455.yaml b/Documentation/devicetree/bindings/iio/accel/fsl,mma7455.yaml index 9c7c66feeffca4f0a157273aaf61251d6a969236..c8659c5eba2a34c0ab30c7fea4c5b896c3ae0ae0 100644 --- a/Documentation/devicetree/bindings/iio/accel/fsl,mma7455.yaml +++ b/Documentation/devicetree/bindings/iio/accel/fsl,mma7455.yaml @@ -39,13 +39,14 @@ properties: - "INT1" - "INT2" - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/kionix,kxsd9.yaml b/Documentation/devicetree/bindings/iio/accel/kionix,kxsd9.yaml index 390b87242fcbfa905abf266b3c18f86a4f81102e..f64d99b35492eb0feae5796c51f17c2c72051bca 100644 --- a/Documentation/devicetree/bindings/iio/accel/kionix,kxsd9.yaml +++ b/Documentation/devicetree/bindings/iio/accel/kionix,kxsd9.yaml @@ -29,13 +29,14 @@ properties: mount-matrix: description: an optional 3x3 mounting rotation matrix. - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/memsensing,msa311.yaml b/Documentation/devicetree/bindings/iio/accel/memsensing,msa311.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23528dcaa0734b14208ba2eeceeef3d04f4bbedb --- /dev/null +++ b/Documentation/devicetree/bindings/iio/accel/memsensing,msa311.yaml @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause + +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/iio/accel/memsensing,msa311.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: MEMSensing digital 3-Axis accelerometer + +maintainers: + - Dmitry Rokosov + +description: | + MSA311 is a tri-axial, low-g accelerometer with I2C digital output for + sensitivity consumer applications. It has dynamical user selectable full + scales range of +-2g/+-4g/+-8g/+-16g and allows acceleration measurements + with output data rates from 1Hz to 1000Hz. + Datasheet can be found at following URL + https://cdn-shop.adafruit.com/product-files/5309/MSA311-V1.1-ENG.pdf + +properties: + compatible: + const: memsensing,msa311 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + accelerometer@62 { + compatible = "memsensing,msa311"; + reg = <0x62>; + interrupt-parent = <&gpio_intc>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + vdd-supply = <&vcc_5v>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/accel/murata,sca3300.yaml b/Documentation/devicetree/bindings/iio/accel/murata,sca3300.yaml index f6e2a16a710b16dd51a9f12c1d7f4c61f828e395..00c990caa1e4a5a71993712c47a211e1285b532a 100644 --- a/Documentation/devicetree/bindings/iio/accel/murata,sca3300.yaml +++ b/Documentation/devicetree/bindings/iio/accel/murata,sca3300.yaml @@ -29,7 +29,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/accel/nxp,fxls8962af.yaml b/Documentation/devicetree/bindings/iio/accel/nxp,fxls8962af.yaml index ad529ab2c6e2373702cc6f41aeabd547866abd88..65ce8ea14b52f12d1aba53259d0133494ca696e0 100644 --- a/Documentation/devicetree/bindings/iio/accel/nxp,fxls8962af.yaml +++ b/Documentation/devicetree/bindings/iio/accel/nxp,fxls8962af.yaml @@ -27,8 +27,6 @@ properties: vdd-supply: description: phandle to the regulator that provides power to the accelerometer - spi-max-frequency: true - interrupts: maxItems: 1 @@ -44,7 +42,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7124.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7124.yaml index fb3d0dae9bae95cf2fbad37710a5d083812b88e8..75a7184a47352d0b95bbb2db380292fca9c5ca06 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7124.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7124.yaml @@ -52,8 +52,6 @@ properties: avdd-supply: description: avdd supply can be used as reference for conversion. - spi-max-frequency: true - required: - compatible - reg @@ -106,7 +104,10 @@ patternProperties: additionalProperties: false -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7192.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7192.yaml index 22b7ed3723f6735360f7515b722607979e13e3aa..cc347dade4ef5f099ff30405fad5c7a00c7c2790 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7192.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7192.yaml @@ -30,8 +30,6 @@ properties: spi-cpha: true - spi-max-frequency: true - clocks: maxItems: 1 description: phandle to the master clock (mclk) @@ -94,7 +92,10 @@ required: - spi-cpol - spi-cpha -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7280a.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7280a.yaml index a694d5794d4a07e17f9576f1d0222853c66c1bec..dfb8f305e2f0bf8a5107d39e4faa435b9d9a3fbb 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7280a.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7280a.yaml @@ -28,8 +28,6 @@ properties: description: IRQ line for the ADC maxItems: 1 - spi-max-frequency: true - adi,voltage-alert-last-chan: $ref: /schemas/types.yaml#/definitions/uint32 description: @@ -55,7 +53,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7292.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7292.yaml index a3e39a40c9b3ea5df8d5b06ba00790aa69293e57..1bfbeed6f2999a091f1582c2fa55a2fb6e35fe77 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7292.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7292.yaml @@ -30,8 +30,6 @@ properties: spi-cpha: true - spi-max-frequency: true - '#address-cells': const: 1 @@ -65,7 +63,10 @@ patternProperties: additionalProperties: true -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7298.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7298.yaml index ca414bb396c5957828183c356df59dc498294956..cd8ac5162d2759013c1762e030b82974464d27f9 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7298.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7298.yaml @@ -24,13 +24,15 @@ properties: vref-supply: true vdd-supply: true - spi-max-frequency: true required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7476.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7476.yaml index 666414a9c0defd831e4dc561e267790ddeb61f67..44c671eeda734d6120ff53f2d22f27aa03dc9116 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7476.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7476.yaml @@ -66,8 +66,6 @@ properties: to the other supplies. Needed to be able to establish channel scaling unless there is also an internal reference available (e.g. ad7091r) - spi-max-frequency: true - adi,conversion-start-gpios: description: A GPIO used to trigger the start of a conversion maxItems: 1 @@ -76,9 +74,9 @@ required: - compatible - reg -additionalProperties: false - allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + # Devices where reference is vcc - if: properties: @@ -158,6 +156,8 @@ allOf: properties: adi,conversion-start-gpios: false +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7606.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7606.yaml index 516fc24d33468a813986d97a6d09100b7c16b6f6..ac5a47c8f070b04428766cb8615b0c69c0767897 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7606.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7606.yaml @@ -32,8 +32,6 @@ properties: spi-cpol: true - spi-max-frequency: true - avcc-supply: true interrupts: @@ -105,7 +103,10 @@ required: - interrupts - adi,conversion-start-gpios -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml index a85a28145ef6b8f3f673524a52a0aa2483d7239c..3ce59d4d065f5d657404f9709df40beee3ae78a3 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml @@ -50,8 +50,6 @@ properties: reset-gpios: maxItems: 1 - spi-max-frequency: true - spi-cpol: true spi-cpha: true @@ -88,7 +86,10 @@ patternProperties: - reg additionalProperties: false -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7923.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7923.yaml index e82194974eea6119c71c82878dca6da114971f5c..07f9d1c09c7da98cc3bf3f88b447c910c52cf85b 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7923.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7923.yaml @@ -8,7 +8,6 @@ title: Analog Devices AD7923 and similars with 4 and 8 Channel ADCs. maintainers: - Michael Hennerich - - Patrick Vasseur description: | Analog Devices AD7904, AD7914, AD7923, AD7924 4 Channel ADCs, and AD7908, @@ -37,19 +36,24 @@ properties: description: | The regulator supply for ADC reference voltage. + adi,range-double: + description: Sets the analog input range from 0 to 2xVREF. + type: boolean + '#address-cells': const: 1 '#size-cells': const: 0 - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7949.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7949.yaml index 0b10ed5f74aeb52ddde457fd1c78e4d5159593cb..9ee4d977c5edfa4d07ac5954869f530f42d61406 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7949.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7949.yaml @@ -49,8 +49,6 @@ properties: default: 4096000 - spi-max-frequency: true - '#io-channel-cells': const: 1 @@ -64,7 +62,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/holt,hi8435.yaml b/Documentation/devicetree/bindings/iio/adc/holt,hi8435.yaml index 52490cbb0af08755427adc08403bd986eaafe133..56bcbe5dcd798ac4aa6be0d6d8b0f6381f9db344 100644 --- a/Documentation/devicetree/bindings/iio/adc/holt,hi8435.yaml +++ b/Documentation/devicetree/bindings/iio/adc/holt,hi8435.yaml @@ -24,8 +24,6 @@ properties: GPIO used for controlling the reset pin maxItems: 1 - spi-max-frequency: true - "#io-channel-cells": const: 1 @@ -33,7 +31,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml index 0bd2fc0356c86476eb12dbab07244ca57bd94e83..5207c919abe0e54ab8a46eeb74d082a4bbc412f0 100644 --- a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml +++ b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml @@ -22,15 +22,15 @@ properties: reg: maxItems: 1 - spi-max-frequency: - description: maximal spi bus frequency supported - required: - compatible - vref-supply - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml index c1772b568cd1b10f7af038babb98f65094a72a43..875f394576c21c7cbf992e5cc6468db80104a0fd 100644 --- a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml +++ b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml @@ -13,10 +13,14 @@ description: | 16bit ADC supporting up to 16 single ended or 8 differential inputs. I2C interface. + https://www.analog.com/media/en/technical-documentation/data-sheets/2497fb.pdf + https://www.analog.com/media/en/technical-documentation/data-sheets/2499fe.pdf + properties: compatible: - const: - lltc,ltc2497 + enum: + - lltc,ltc2497 + - lltc,ltc2499 reg: true vref-supply: true diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max1027.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max1027.yaml index 46b7747076b9b65682f6a5bceddfd8486eed1aec..d0a7ed26d9ea78c511bc82f4e8857f7f8bbe596c 100644 --- a/Documentation/devicetree/bindings/iio/adc/maxim,max1027.yaml +++ b/Documentation/devicetree/bindings/iio/adc/maxim,max1027.yaml @@ -45,7 +45,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max11100.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max11100.yaml index 0cf87556ef823c97d1ba6d387a4774ff0ed6b9f1..4f74cb33383a4f68bec347d7fb2c031220697647 100644 --- a/Documentation/devicetree/bindings/iio/adc/maxim,max11100.yaml +++ b/Documentation/devicetree/bindings/iio/adc/maxim,max11100.yaml @@ -26,13 +26,16 @@ properties: minimum: 100000 maximum: 4800000 -additionalProperties: false - required: - compatible - reg - vref-supply +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max1118.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max1118.yaml index e948b3e37b0c66f16a37bfa01e67b2916fe5492e..bb336e33ebe25715b7b0c9a9bb0aafb2f6341295 100644 --- a/Documentation/devicetree/bindings/iio/adc/maxim,max1118.yaml +++ b/Documentation/devicetree/bindings/iio/adc/maxim,max1118.yaml @@ -28,23 +28,25 @@ properties: vref-supply: description: External reference, needed to establish input scaling -if: - properties: - compatible: - contains: - const: maxim,max1118 -then: - required: - - vref-supply -else: - properties: - vref-supply: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + - if: + properties: + compatible: + contains: + const: maxim,max1118 + then: + required: + - vref-supply + else: + properties: + vref-supply: false required: - compatible - reg -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max11205.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max11205.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f9e043cf5cd391898924e5007c0cb921c9a0c66 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/adc/maxim,max11205.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/adc/maxim,max11205.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Maxim MAX11205 ADC + +maintainers: + - Ramona Bolboaca + +description: | + The MAX11205 is an ultra-low-power (< 300FA max active current), + high-resolution, serial-output ADC. + + https://datasheets.maximintegrated.com/en/ds/MAX11205.pdf + +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + enum: + - maxim,max11205a + - maxim,max11205b + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + spi-max-frequency: + maximum: 5000000 + + spi-cpha: true + + vref-supply: + description: + The regulator supply for the ADC reference voltage. This is a differential + reference. It is equal to the V_REFP - V_REFN. The maximum value is 3.6V. + +required: + - compatible + - reg + - interrupts + - spi-max-frequency + - spi-cpha + - vref-supply + +unevaluatedProperties: false + +examples: + - | + #include + spi { + #address-cells = <1>; + #size-cells = <0>; + adc@0 { + compatible = "maxim,max11205a"; + reg = <0>; + spi-max-frequency = <5000000>; + spi-cpha; + interrupt-parent = <&gpio>; + interrupts = <19 IRQ_TYPE_EDGE_FALLING>; + vref-supply = <&max11205_vref>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/iio/adc/maxim,max1241.yaml b/Documentation/devicetree/bindings/iio/adc/maxim,max1241.yaml index 4c7e0d94bff1426d68017a5b3047da08eb9477a9..58b12fe8070c9b7ad6cede0e893ef84880ade014 100644 --- a/Documentation/devicetree/bindings/iio/adc/maxim,max1241.yaml +++ b/Documentation/devicetree/bindings/iio/adc/maxim,max1241.yaml @@ -39,15 +39,16 @@ properties: thus enabling power-down mode. maxItems: 1 - spi-max-frequency: true - required: - compatible - reg - vdd-supply - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/microchip,mcp3201.yaml b/Documentation/devicetree/bindings/iio/adc/microchip,mcp3201.yaml index fcc1ba53b20d3ed4da508db521370a95eb3b2e95..18108f0f37319fe40c1be7c499a1f2727e9b858e 100644 --- a/Documentation/devicetree/bindings/iio/adc/microchip,mcp3201.yaml +++ b/Documentation/devicetree/bindings/iio/adc/microchip,mcp3201.yaml @@ -32,7 +32,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true spi-cpha: true spi-cpol: true @@ -51,7 +50,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/microchip,mcp3911.yaml b/Documentation/devicetree/bindings/iio/adc/microchip,mcp3911.yaml index 95ab285f4eba94515efbb503a90edf02509a4a31..2c93fb41f1728a3f61186b356ccb07de28f7b6a4 100644 --- a/Documentation/devicetree/bindings/iio/adc/microchip,mcp3911.yaml +++ b/Documentation/devicetree/bindings/iio/adc/microchip,mcp3911.yaml @@ -36,6 +36,13 @@ properties: description: IRQ line of the ADC maxItems: 1 + microchip,data-ready-hiz: + description: + Data Ready Pin Inactive State Control + true = The DR pin state is high-impedance + false = The DR pin state is logic high + type: boolean + microchip,device-addr: description: Device address when multiple MCP3911 chips are present on the same SPI bus. $ref: /schemas/types.yaml#/definitions/uint32 @@ -51,7 +58,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml index 74a4a9d95798c8d771d553ea4ead04c17a8c728c..8bac0c4120dd09b893ce08ee572807bc0dbf3b86 100644 --- a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml @@ -58,6 +58,7 @@ required: patternProperties: "^.*@[0-9a-f]+$": type: object + additionalProperties: false description: | Represents the external channels which are connected to the ADC. For compatible property "qcom,spmi-vadc" following channels, also known as diff --git a/Documentation/devicetree/bindings/iio/adc/richtek,rtq6056.yaml b/Documentation/devicetree/bindings/iio/adc/richtek,rtq6056.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88e008629ea89a494a2d994b5d3a9dbaf7deb71e --- /dev/null +++ b/Documentation/devicetree/bindings/iio/adc/richtek,rtq6056.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/adc/richtek,rtq6056.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: RTQ6056 Bi-Directional Current and Power Monitor with 16-bit ADC + +maintainers: + - ChiYuan Huang + +description: | + The RTQ6056 is a high accuracy current-sense monitor with I2C and SMBus + interface, and the device provides full information for system by reading + out the loading current and power. + + The device monitors both of the drops across sense resistor and the BUS + voltage, converts into the current in amperes, and power in watts through + internal analog-to-digital converter ADC. The programmable calibration, + adjustable conversion time, and averaging function are also built in for + more design flexibility. + + Datasheet is available at + https://www.richtek.com/assets/product_file/RTQ6056/DSQ6056-00.pdf + +properties: + compatible: + const: richtek,rtq6056 + + reg: + maxItems: 1 + + "#io-channel-cells": + const: 1 + + shunt-resistor-micro-ohms: + description: Shunt IN+/IN- sensing node resistor + +required: + - compatible + - reg + - "#io-channel-cells" + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + adc@40 { + compatible = "richtek,rtq6056"; + reg = <0x40>; + #io-channel-cells = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.yaml b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.yaml index 92f9472a77ae1e5b46acfd2def91a5ec41bb24c1..1970503389aa9aacf88c356477b4ad6aff5e0bbc 100644 --- a/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-dfsdm-adc.yaml @@ -72,6 +72,7 @@ additionalProperties: false patternProperties: "^filter@[0-9]+$": type: object + unevaluatedProperties: false description: child node properties: @@ -225,6 +226,7 @@ patternProperties: patternProperties: "^dfsdm-dai+$": type: object + additionalProperties: false description: child node properties: diff --git a/Documentation/devicetree/bindings/iio/adc/st,stmpe-adc.yaml b/Documentation/devicetree/bindings/iio/adc/st,stmpe-adc.yaml index 9049c699152f7e41f0e87a16d5128724b5997766..333744a2159c67eceadef6838f59d70d74e57c32 100644 --- a/Documentation/devicetree/bindings/iio/adc/st,stmpe-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/st,stmpe-adc.yaml @@ -13,8 +13,7 @@ description: This ADC forms part of an ST microelectronics STMPE multifunction device . The ADC is shared with the STMPE touchscreen. As a result some ADC related settings are specified in the parent node. - The node name myst be stmpe_adc and should be a child node of the stmpe node - to which it belongs. + The node should be a child node of the stmpe node to which it belongs. properties: compatible: diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc0832.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc0832.yaml index f5a923cc847f46c11fc7d9b10c5042498a35672f..686721176a58c391d7ee2cc09bb7bdfaff62d7fa 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc0832.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc0832.yaml @@ -24,8 +24,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: description: External reference, needed to establish input scaling @@ -37,7 +35,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc084s021.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc084s021.yaml index 1a113b30a4146a4b0452700f1cbc6d7d930a0619..726d2cbfa368b0eed218dfb8c207fbdfa591cd61 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc084s021.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc084s021.yaml @@ -19,8 +19,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: description: External reference, needed to establish input scaling @@ -37,7 +35,10 @@ required: - spi-cpol - spi-cpha -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc108s102.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc108s102.yaml index ae5ce60987fec2ac5372d1fd620281281d2b1da8..9b072b057f164e0a7cdb47e793da14ba45739b1f 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc108s102.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc108s102.yaml @@ -19,7 +19,6 @@ properties: reg: true vref-supply: true - spi-max-frequency: true "#io-channel-cells": const: 1 @@ -28,7 +27,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc12138.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc12138.yaml index ec3b2edf1fb7d95220efefa8903b91a395fe331e..076088a328c3c6d009ccd71aaa30bdc10d603a24 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc12138.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc12138.yaml @@ -31,8 +31,6 @@ properties: maxItems: 1 description: Conversion clock input. - spi-max-frequency: true - vref-p-supply: description: The regulator supply for positive analog voltage reference @@ -62,7 +60,10 @@ required: - clocks - vref-p-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc128s052.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc128s052.yaml index d54a0183f024d6f8c1dfdac2e4f674650a2bbf03..775eee972b12b15d3673fef02ab3d3d157e2f433 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc128s052.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc128s052.yaml @@ -27,8 +27,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: true "#io-channel-cells": @@ -39,7 +37,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,adc161s626.yaml b/Documentation/devicetree/bindings/iio/adc/ti,adc161s626.yaml index 3f4f334d6f73294fa9cc9f87cc870b9cb636932a..afe7825229049f3dbd552fe04b61f2c893839cf7 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,adc161s626.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,adc161s626.yaml @@ -21,8 +21,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vdda-supply: true "#io-channel-cells": @@ -32,7 +30,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml index a3b79438a13a507e09f6af518b4e4c4c38bf046c..2c3c2cf2145c638ce84889d8f81b7ae16e4c8292 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,ads1015.yaml @@ -43,6 +43,7 @@ additionalProperties: false patternProperties: "^channel@[0-7]+$": type: object + additionalProperties: false description: Child nodes needed for each channel that the platform uses. diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads124s08.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads124s08.yaml index 2e6abc9d746ac325b62ee971b6d5de0ca53d1f49..56a3f1766aabe5dbf2167a875ebc42da5e0b8b67 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,ads124s08.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,ads124s08.yaml @@ -18,8 +18,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - spi-cpha: true reset-gpios: @@ -32,7 +30,10 @@ required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads131e08.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads131e08.yaml index e0670e3fbb726095eeef8dd1e25ce4fc9fa9fc89..55c2c73626f48c042d17f124f007472d9619f17d 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,ads131e08.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,ads131e08.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Texas Instruments ADS131E0x 4-, 6- and 8-Channel ADCs maintainers: - - Tomislav Denis + - Jonathan Cameron description: | The ADS131E0x are a family of multichannel, simultaneous sampling, @@ -28,8 +28,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - spi-cpha: true clocks: @@ -120,7 +118,10 @@ patternProperties: additionalProperties: false -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads8344.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads8344.yaml index b8c398187d5cb45a4fad26fb5741f8d7bc73b228..f75b2c702986dd8afdbadb6f7caf6f6d7a8052be 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,ads8344.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,ads8344.yaml @@ -19,8 +19,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: description: Supply the 2.5V or 5V reference voltage @@ -32,7 +30,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,ads8688.yaml b/Documentation/devicetree/bindings/iio/adc/ti,ads8688.yaml index a0af4b24877f2af224158fb9979c23147876b6aa..f26fdbc15f84f84c90b28760b71a4df6de98fd52 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,ads8688.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,ads8688.yaml @@ -25,13 +25,14 @@ properties: description: Optional external reference. If not supplied, assume REFSEL input tied low to enable the internal reference. - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,am3359-adc.yaml b/Documentation/devicetree/bindings/iio/adc/ti,am3359-adc.yaml index d6f21d5cccd732b355208ae9043da46ffcd94580..b32be24a9f980a43d9caaca5885d25718fea7676 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,am3359-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,am3359-adc.yaml @@ -11,9 +11,14 @@ maintainers: properties: compatible: - enum: - - ti,am3359-adc - - ti,am4372-adc + oneOf: + - enum: + - ti,am3359-adc + - ti,am4372-adc + - items: + - enum: + - ti,am654-adc + - const: ti,am3359-adc '#io-channel-cells': const: 1 diff --git a/Documentation/devicetree/bindings/iio/adc/ti,tlc4541.yaml b/Documentation/devicetree/bindings/iio/adc/ti,tlc4541.yaml index 6c2539b3d7077a67d486497253827fec1e27217a..314d1d99bf738aafe9cc73faf1d105127bea95d2 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,tlc4541.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,tlc4541.yaml @@ -21,8 +21,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: true "#io-channel-cells": @@ -33,7 +31,10 @@ required: - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml b/Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml index 601d69971d84a97e49b51ef21f5d6ebec94074f4..bdf3bba2d7507eacde9e96a9cfebd403ed510158 100644 --- a/Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml +++ b/Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml @@ -23,7 +23,8 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true + vref-supply: + description: Optional supply of the reference voltage "#io-channel-cells": const: 1 @@ -59,7 +60,10 @@ patternProperties: additionalProperties: false -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/adc/xlnx,zynqmp-ams.yaml b/Documentation/devicetree/bindings/iio/adc/xlnx,zynqmp-ams.yaml index 3698b4b0900f5ad2df202a6d2b5692faee60e69a..be93c109d6ac6427cd1b70dd0ecbc75fc31017c3 100644 --- a/Documentation/devicetree/bindings/iio/adc/xlnx,zynqmp-ams.yaml +++ b/Documentation/devicetree/bindings/iio/adc/xlnx,zynqmp-ams.yaml @@ -134,6 +134,7 @@ properties: ams-pl@400: type: object + additionalProperties: false description: PL-SYSMON is capable of monitoring off chip voltage and temperature. PL-SYSMON block has DRP, JTAG and I2C interface to enable monitoring diff --git a/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml b/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml index baa65a521bad5b2cf533061ee331e023eb62bdd5..03bb90a7f4f888e13a0bf556d79948de7930b5f8 100644 --- a/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml +++ b/Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml @@ -63,6 +63,7 @@ additionalProperties: false patternProperties: "^channel@[0-3]$": type: object + additionalProperties: false description: Represents the external channels which are connected to the device. properties: diff --git a/Documentation/devicetree/bindings/iio/amplifiers/adi,ada4250.yaml b/Documentation/devicetree/bindings/iio/amplifiers/adi,ada4250.yaml index 5277479be38225581908217592b18462602b7c45..c15da155d3003c40e45ed6d5462f6e9d5f5f6946 100644 --- a/Documentation/devicetree/bindings/iio/amplifiers/adi,ada4250.yaml +++ b/Documentation/devicetree/bindings/iio/amplifiers/adi,ada4250.yaml @@ -27,14 +27,15 @@ properties: Enable internal buffer to drive the reference pin. type: boolean - spi-max-frequency: true - required: - compatible - reg - avdd-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad3552r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad3552r.yaml index 9c48c76993fec24f7ae91ae30c112c379aced290..fee0f023a8c81667988c513831252f32a7007c8f 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad3552r.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad3552r.yaml @@ -78,6 +78,7 @@ patternProperties: custom-output-range-config: type: object + additionalProperties: false description: Configuration of custom range when adi,output-range-microvolt is not present. The formulas for calculation the output voltages are diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5064.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5064.yaml index 05ed4e0ec364144cd056fc3efe9b062759935d4b..c04165fa9259bbf1450d693720741caf4d5cb156 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5064.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5064.yaml @@ -95,15 +95,12 @@ properties: vrefD-supply: true vref-supply: true - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - # Shared external vref, no internal reference if: properties: @@ -232,6 +229,8 @@ allOf: - vrefA-supply - vrefB-supply +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5360.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5360.yaml index 65f86f26947cd5d68d24c16bd2957d08169ea719..86e2884cdfb1ea0105e273ea86a0d93167900b7c 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5360.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5360.yaml @@ -28,10 +28,6 @@ properties: vref1-supply: true vref2-supply: true - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg @@ -39,6 +35,7 @@ required: - vref1-supply allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -63,6 +60,8 @@ allOf: required: - vref2-supply +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5380.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5380.yaml index d599b418a020facc68dc1a072ddb8a7c4bd36eb2..ff50c72c62b553b855ce6d118dafeafe871cbe17 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5380.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5380.yaml @@ -39,14 +39,15 @@ properties: description: If not supplied devices will use internal regulators. - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5421.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5421.yaml index 188f656617e31c46c5e06fc2ed85fe4fd82c5af7..52d089ebde9530a6ab62af8beb37dd072929f224 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5421.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5421.yaml @@ -26,13 +26,14 @@ properties: maxItems: 1 description: Fault signal. - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5449.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5449.yaml index 044332c97743a772e1538346e64240cd0c4a27dd..d2af2d491986d913bc88485d0c22cebbccfe0302 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5449.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5449.yaml @@ -27,19 +27,16 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - VREF-supply: true VREFA-supply: true VREFB-supply: true -additionalProperties: false - required: - compatible - reg allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -72,6 +69,8 @@ allOf: - VREFA-supply - VREFB-supply +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5624r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5624r.yaml index 330383b85eebfe02551b81ed85e8795b21955f55..4d5111a5f9bd87179ca51b90acb94dad9847e160 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5624r.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5624r.yaml @@ -22,17 +22,18 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: description: If not present, internal reference will be used. -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml index 5c26441eae9f6ecea9dfe1822b1f81635686b78c..13f214234b8ed74dcdf92aa444706f73c60d39de 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml @@ -53,14 +53,15 @@ properties: vcc-supply: description: If not supplied the internal reference is used. - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5755.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5755.yaml index f866b88e14409c193e9228eecf7bf20bbde52d66..9a3c2926bf851e450b5f64a3cf9f1f07bfe2d087 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5755.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5755.yaml @@ -25,8 +25,6 @@ properties: description: Either this or spi-cpol but not both. spi-cpol: true - spi-max-frequency: true - adi,ext-dc-dc-compenstation-resistor: $ref: /schemas/types.yaml#/definitions/flag description: @@ -67,8 +65,6 @@ required: - compatible - reg -additionalProperties: false - patternProperties: "^channel@[0-7]$": type: object @@ -123,6 +119,11 @@ oneOf: - required: - spi-cpol +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5758.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5758.yaml index fd4edca34a2851bc5d4f5df944e5c403dc04fad3..e49e7556175df26469304c1e6baac3bada49175a 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5758.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5758.yaml @@ -16,7 +16,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true spi-cpha: true adi,dc-dc-mode: @@ -99,6 +98,7 @@ required: - adi,dc-dc-mode allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: adi,dc-dc-mode: @@ -115,7 +115,7 @@ allOf: required: - adi,range-microvolt -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5761.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5761.yaml index 7f95a9ed55fec5628ad82328dc32fb026554b498..df550b5af2f72b558f570cacea0da223eb592cf9 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5761.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5761.yaml @@ -22,18 +22,15 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: description: If not supplied, internal reference will be used. -additionalProperties: false - required: - compatible - reg allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -45,6 +42,8 @@ allOf: required: - vref-supply +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5764.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5764.yaml index 8e893d52bfb1b3b2cd22e3837a58dbea31da3052..0b409a727a4388ada68ba0a4f9d840ea79d0dff5 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5764.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5764.yaml @@ -22,18 +22,15 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vrefAB-supply: true vrefCD-supply: true -additionalProperties: false - required: - compatible - reg allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -46,6 +43,8 @@ allOf: - vrefAB-supply - vrefCD-supply +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5770r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5770r.yaml index 24ac40180ac12c8daba5ea4f8ab3fc687ed11186..8e7da0de918ff88528018fef1cf425809fdb98ea 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5770r.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5770r.yaml @@ -49,8 +49,6 @@ properties: asserted during driver probe. maxItems: 1 - spi-max-frequency: true - '#address-cells': const: 1 @@ -62,6 +60,7 @@ properties: connected to the DAC. Channel 0 can act both as a current source and sink. type: object + additionalProperties: false properties: reg: @@ -85,6 +84,7 @@ properties: description: Represents an external channel which are connected to the DAC. type: object + additionalProperties: false properties: reg: @@ -101,6 +101,7 @@ properties: description: Represents an external channel which are connected to the DAC. type: object + additionalProperties: false properties: reg: @@ -116,6 +117,7 @@ properties: patternProperties: "^channel@([3-5])$": type: object + additionalProperties: false description: Represents the external channels which are connected to the DAC. properties: reg: @@ -138,7 +140,10 @@ required: - channel@4 - channel@5 -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5791.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5791.yaml index 650d1ebdcec37f0033012fdae361cd98fd8409bd..3a84739736f620cf62a399e21bc50b7d0c03bf69 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5791.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5791.yaml @@ -23,19 +23,20 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vdd-supply: true vss-supply: true -additionalProperties: false - required: - compatible - reg - vdd-supply - vss-supply +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad8801.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad8801.yaml index 6a3990a8d0adb1fb627f5c77e6a155a63b2077c6..1849a2ff05c7aacc8792a493b715e633dfeac7ce 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad8801.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad8801.yaml @@ -19,19 +19,16 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vrefh-supply: true vrefl-supply: true -additionalProperties: false - required: - compatible - reg - vrefh-supply allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -44,6 +41,8 @@ allOf: properties: vrefl-supply: false +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ltc2688.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ltc2688.yaml index 48f9e7d29423571f0c3ec2fcc6ac63875dae5b6c..15cc6bf59b1309c71b5b276d93c63a98bc69e2ed 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ltc2688.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ltc2688.yaml @@ -48,6 +48,7 @@ properties: patternProperties: "^channel@([0-9]|1[0-5])$": type: object + additionalProperties: false properties: reg: diff --git a/Documentation/devicetree/bindings/iio/dac/microchip,mcp4922.yaml b/Documentation/devicetree/bindings/iio/dac/microchip,mcp4922.yaml index 4c430abcdbf9c43c3eb5b77d5e361c79f28083de..19374401e5093b225044d0518553abea5aafac39 100644 --- a/Documentation/devicetree/bindings/iio/dac/microchip,mcp4922.yaml +++ b/Documentation/devicetree/bindings/iio/dac/microchip,mcp4922.yaml @@ -21,17 +21,18 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - vref-supply: true -additionalProperties: false - required: - compatible - reg - vref-supply +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/dac/ti,dac082s085.yaml b/Documentation/devicetree/bindings/iio/dac/ti,dac082s085.yaml index b0157050f1eebf27d5f38441befdad037a03be79..201b04af2b22ef186c61a1d0314a6c037751552f 100644 --- a/Documentation/devicetree/bindings/iio/dac/ti,dac082s085.yaml +++ b/Documentation/devicetree/bindings/iio/dac/ti,dac082s085.yaml @@ -33,21 +33,22 @@ properties: vref-supply: description: Needed to provide output scaling. - spi-max-frequency: true - required: - compatible - reg - vref-supply -additionalProperties: false - oneOf: - required: - spi-cpha - required: - spi-cpol +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | vref_2v5_reg: regulator-vref { diff --git a/Documentation/devicetree/bindings/iio/dac/ti,dac7311.yaml b/Documentation/devicetree/bindings/iio/dac/ti,dac7311.yaml index 10be98d1f19ca29081ce24e0fee5f30d71c35a26..a6814587dbc48495236ba0b79e5f7351583ef934 100644 --- a/Documentation/devicetree/bindings/iio/dac/ti,dac7311.yaml +++ b/Documentation/devicetree/bindings/iio/dac/ti,dac7311.yaml @@ -24,14 +24,15 @@ properties: Reference voltage must be supplied to establish the scaling of the output voltage. - spi-max-frequency: true - required: - compatible - reg - vref-supply -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml b/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml index d172b142f6ed9b6e69b229e457c7ee82dcf2e94d..20dd1370660d5a7bd5d82d88c42f29e270507541 100644 --- a/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml +++ b/Documentation/devicetree/bindings/iio/dac/ti,dac7612.yaml @@ -29,13 +29,14 @@ properties: DACs are loaded when the pin connected to this GPIO is pulled low. maxItems: 1 - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/frequency/adf4371.yaml b/Documentation/devicetree/bindings/iio/frequency/adf4371.yaml index 6b3a611e1cf1a8ed34a11ca9c842656c1e8c5dd5..0144f74a4768a76dd16af4abd312807360d56dee 100644 --- a/Documentation/devicetree/bindings/iio/frequency/adf4371.yaml +++ b/Documentation/devicetree/bindings/iio/frequency/adf4371.yaml @@ -40,15 +40,16 @@ properties: output stage will shut down until the ADF4371/ADF4372 achieves lock as measured by the digital lock detect circuitry. - spi-max-frequency: true - required: - compatible - reg - clocks - clock-names -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml index b6bbc312a7cf7e3d4c6299cae38714b5a8d8267a..1414ba9977c1634a8636f93d965a0852942acaac 100644 --- a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml +++ b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml @@ -24,8 +24,10 @@ properties: interrupts: minItems: 1 + maxItems: 2 description: Should be configured with type IRQ_TYPE_EDGE_RISING. + If two interrupts are provided, expected order is INT1 and INT2. required: - compatible diff --git a/Documentation/devicetree/bindings/iio/health/ti,afe4403.yaml b/Documentation/devicetree/bindings/iio/health/ti,afe4403.yaml index d861526c5c428cab4f6423f08b8fffed3922d08c..6c5ad426a0165a52e0763f823261df0d2f75480d 100644 --- a/Documentation/devicetree/bindings/iio/health/ti,afe4403.yaml +++ b/Documentation/devicetree/bindings/iio/health/ti,afe4403.yaml @@ -25,14 +25,15 @@ properties: reset-gpios: true - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml b/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml index 340be256f2831db73334b08c4a667c27984e8f5b..d166dbca18c34faaa048a5b6ade679b31ead60fd 100644 --- a/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml +++ b/Documentation/devicetree/bindings/iio/imu/adi,adis16460.yaml @@ -25,8 +25,6 @@ properties: spi-cpol: true - spi-max-frequency: true - interrupts: maxItems: 1 @@ -35,7 +33,10 @@ required: - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/imu/adi,adis16480.yaml b/Documentation/devicetree/bindings/iio/imu/adi,adis16480.yaml index dd29dc6c4c196ea0b3c87636ee6653a0ae5cc3c4..56e0dc20f5e4f3dda7c039a62690d0367b695b17 100644 --- a/Documentation/devicetree/bindings/iio/imu/adi,adis16480.yaml +++ b/Documentation/devicetree/bindings/iio/imu/adi,adis16480.yaml @@ -47,8 +47,6 @@ properties: - DIO3 - DIO4 - spi-max-frequency: true - spi-cpha: true spi-cpol: true @@ -96,8 +94,6 @@ properties: - DIO3 - DIO4 -additionalProperties: false - required: - compatible - reg @@ -106,6 +102,11 @@ required: - spi-cpol - spi-max-frequency +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/iio/imu/bosch,bmi160.yaml b/Documentation/devicetree/bindings/iio/imu/bosch,bmi160.yaml index 6e73cd889b5c717984ce1bf8dc026c2bedc221f6..a0760382548daa8d9c78b82c7b2b5674657a5c17 100644 --- a/Documentation/devicetree/bindings/iio/imu/bosch,bmi160.yaml +++ b/Documentation/devicetree/bindings/iio/imu/bosch,bmi160.yaml @@ -46,13 +46,14 @@ properties: mount-matrix: description: an optional 3x3 mounting rotation matrix - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/imu/bosch,bno055.yaml b/Documentation/devicetree/bindings/iio/imu/bosch,bno055.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0d06db161a9f7956480c26d08359e5ef6bfac99 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/imu/bosch,bno055.yaml @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/imu/bosch,bno055.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Bosch BNO055 + +maintainers: + - Andrea Merello + +description: | + Inertial Measurement Unit with Accelerometer, Gyroscope, Magnetometer and + internal MCU for sensor fusion + https://www.bosch-sensortec.com/products/smart-sensors/bno055/ + +properties: + compatible: + enum: + - bosch,bno055 + + reg: + maxItems: 1 + + reset-gpios: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + +additionalProperties: false + +examples: + - | + #include + serial { + imu { + compatible = "bosch,bno055"; + reset-gpios = <&gpio0 54 GPIO_ACTIVE_LOW>; + clocks = <&imu_clk>; + }; + }; + + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + imu@28 { + compatible = "bosch,bno055"; + reg = <0x28>; + reset-gpios = <&gpio0 54 GPIO_ACTIVE_LOW>; + clocks = <&imu_clk>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml b/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml index 4c1c083d0e9249762cfee62cbca28ff6e0592041..488349755c99d8084e5309a742ee3d966af166a0 100644 --- a/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml +++ b/Documentation/devicetree/bindings/iio/imu/invensense,icm42600.yaml @@ -47,7 +47,6 @@ properties: vddio-supply: description: Regulator that provides power to the bus - spi-max-frequency: true spi-cpha: true spi-cpol: true @@ -56,7 +55,10 @@ required: - reg - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/imu/invensense,mpu6050.yaml b/Documentation/devicetree/bindings/iio/imu/invensense,mpu6050.yaml index 3ebc6526d82d7a7b0c9fb2267eb1b0ca85bc394e..ec64d7877fe5b39ba6586a75fe9dcb890dd30d79 100644 --- a/Documentation/devicetree/bindings/iio/imu/invensense,mpu6050.yaml +++ b/Documentation/devicetree/bindings/iio/imu/invensense,mpu6050.yaml @@ -40,8 +40,6 @@ properties: interrupts: maxItems: 1 - spi-max-frequency: true - vdd-supply: true vddio-supply: true @@ -54,6 +52,7 @@ properties: These devices also support an auxiliary i2c bus via an i2c-gate. allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: not: properties: @@ -67,7 +66,7 @@ allOf: properties: i2c-gate: false -additionalProperties: false +unevaluatedProperties: false required: - compatible diff --git a/Documentation/devicetree/bindings/iio/imu/nxp,fxos8700.yaml b/Documentation/devicetree/bindings/iio/imu/nxp,fxos8700.yaml index 0203b83b858770e24c39d807ae4d98f26f3429d8..24416b59b782cc60a3f61f684bbcf27d86e8f613 100644 --- a/Documentation/devicetree/bindings/iio/imu/nxp,fxos8700.yaml +++ b/Documentation/devicetree/bindings/iio/imu/nxp,fxos8700.yaml @@ -36,13 +36,14 @@ properties: drive-open-drain: type: boolean - spi-max-frequency: true - required: - compatible - reg -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/imu/st,lsm6dsx.yaml b/Documentation/devicetree/bindings/iio/imu/st,lsm6dsx.yaml index 5d4839f0089880a0889046f079c0cfa66e377a5c..fe1e02e5d7b39d6fa1b57f0bf0654b1b1d595148 100644 --- a/Documentation/devicetree/bindings/iio/imu/st,lsm6dsx.yaml +++ b/Documentation/devicetree/bindings/iio/imu/st,lsm6dsx.yaml @@ -35,6 +35,9 @@ properties: - items: - const: st,asm330lhhx - const: st,lsm6dsr + - items: + - const: st,lsm6dstx + - const: st,lsm6dst reg: maxItems: 1 @@ -45,8 +48,6 @@ properties: description: Supports up to 2 interrupt lines via the INT1 and INT2 pins. - spi-max-frequency: true - vdd-supply: description: if defined provides VDD power to the sensor. @@ -81,12 +82,15 @@ properties: wakeup-source: $ref: /schemas/types.yaml#/definitions/flag -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/iio/light/liteon,ltrf216a.yaml b/Documentation/devicetree/bindings/iio/light/liteon,ltrf216a.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7de1b0e721ca294b04f7ee803a03aed8a9249640 --- /dev/null +++ b/Documentation/devicetree/bindings/iio/light/liteon,ltrf216a.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iio/light/liteon,ltrf216a.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: LTRF216A Ambient Light Sensor + +maintainers: + - Shreeya Patel + +description: + Ambient light sensing with an i2c interface. + +properties: + compatible: + const: liteon,ltrf216a + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + vdd-supply: + description: Regulator that provides power to the sensor. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + light-sensor@53 { + compatible = "liteon,ltrf216a"; + reg = <0x53>; + vdd-supply = <&vdd_regulator>; + interrupt-parent = <&gpio0>; + interrupts = <5 IRQ_TYPE_LEVEL_LOW>; + }; + }; diff --git a/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp41010.yaml b/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp41010.yaml index 567697d996ec6ddb8b3cce917e75489d1b528fda..87e88f2a99082be8ffae188f83bbfb64c526eff9 100644 --- a/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp41010.yaml +++ b/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp41010.yaml @@ -25,14 +25,15 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp4131.yaml b/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp4131.yaml index 32e92bced81fd0cfe8bb3872390181c0c3d4a7f0..896fe0b5edcc606f15e87bd8ecfc049abb76722c 100644 --- a/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp4131.yaml +++ b/Documentation/devicetree/bindings/iio/potentiometer/microchip,mcp4131.yaml @@ -80,14 +80,15 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/pressure/asc,dlhl60d.yaml b/Documentation/devicetree/bindings/iio/pressure/asc,dlhl60d.yaml index be2be4b556dbbd3b520d95374b0875edc57160e7..1f9fe15b4b3c1dae572c3f41ccc07450ad12536c 100644 --- a/Documentation/devicetree/bindings/iio/pressure/asc,dlhl60d.yaml +++ b/Documentation/devicetree/bindings/iio/pressure/asc,dlhl60d.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: All Sensors DLH series low voltage digital pressure sensors maintainers: - - Tomislav Denis + - Jonathan Cameron description: | Bindings for the All Sensors DLH series pressure sensors. diff --git a/Documentation/devicetree/bindings/iio/pressure/bmp085.yaml b/Documentation/devicetree/bindings/iio/pressure/bmp085.yaml index 49257f9251e87e85b8bf0038c98a0ac19abba153..72cd2c2d3f174cfe4bf2f10d5cb1becb465c3ddb 100644 --- a/Documentation/devicetree/bindings/iio/pressure/bmp085.yaml +++ b/Documentation/devicetree/bindings/iio/pressure/bmp085.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/iio/pressure/bmp085.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: BMP085/BMP180/BMP280/BME280 pressure iio sensors +title: BMP085/BMP180/BMP280/BME280/BMP380 pressure iio sensors maintainers: - Andreas Klinger @@ -16,6 +16,7 @@ description: | https://www.bosch-sensortec.com/bst/products/all_products/bmp180 https://www.bosch-sensortec.com/bst/products/all_products/bmp280 https://www.bosch-sensortec.com/bst/products/all_products/bme280 + https://www.bosch-sensortec.com/bst/products/all_products/bmp380 properties: compatible: @@ -24,6 +25,7 @@ properties: - bosch,bmp180 - bosch,bmp280 - bosch,bme280 + - bosch,bmp380 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/iio/samsung,sensorhub-rinato.yaml b/Documentation/devicetree/bindings/iio/samsung,sensorhub-rinato.yaml index a88b3b14d6bdcd773735aa4895ddbc9e7c4c1b7e..dd2ae2bd1ad77c9bf95ff02c3852b0d47d2edcea 100644 --- a/Documentation/devicetree/bindings/iio/samsung,sensorhub-rinato.yaml +++ b/Documentation/devicetree/bindings/iio/samsung,sensorhub-rinato.yaml @@ -40,10 +40,6 @@ properties: description: Reset the sensorhub. - spi-max-frequency: true - -additionalProperties: false - required: - compatible - reg @@ -52,6 +48,11 @@ required: - mcu-ap-gpios - mcu-reset-gpios +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | spi { diff --git a/Documentation/devicetree/bindings/iio/st,st-sensors.yaml b/Documentation/devicetree/bindings/iio/st,st-sensors.yaml index fcb2902683c70c2c9d2c45b7dfd79d1d5b546869..250439b13152573bfc8a99b934d31d36b9e2ba6d 100644 --- a/Documentation/devicetree/bindings/iio/st,st-sensors.yaml +++ b/Documentation/devicetree/bindings/iio/st,st-sensors.yaml @@ -73,6 +73,7 @@ properties: - description: STMicroelectronics Pressure Sensors enum: - st,lps001wp-press + - st,lps22df - st,lps22hb-press - st,lps22hh - st,lps25h-press @@ -141,6 +142,7 @@ allOf: - st,lis2mdl - st,lis3l02dq - st,lis3lv02dl-accel + - st,lps22df - st,lps22hb-press - st,lps22hh - st,lps25h-press diff --git a/Documentation/devicetree/bindings/iio/temperature/maxim,max31855k.yaml b/Documentation/devicetree/bindings/iio/temperature/maxim,max31855k.yaml index 9969bac66aa1be17b67f808cf7a3799d4c373739..0805ed7e2113a325c315431a30ee32f9a23c03b6 100644 --- a/Documentation/devicetree/bindings/iio/temperature/maxim,max31855k.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/maxim,max31855k.yaml @@ -32,7 +32,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true spi-cpha: true required: @@ -40,6 +39,7 @@ required: - reg allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# - if: properties: compatible: @@ -53,7 +53,7 @@ allOf: properties: spi-cpha: false -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/temperature/maxim,max31856.yaml b/Documentation/devicetree/bindings/iio/temperature/maxim,max31856.yaml index 873b347666762aa71e00bb6418375421ccc731f9..228a941654872c0d606800b135bd2d84072b7ef4 100644 --- a/Documentation/devicetree/bindings/iio/temperature/maxim,max31856.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/maxim,max31856.yaml @@ -19,7 +19,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true spi-cpha: true thermocouple-type: @@ -34,7 +33,10 @@ required: - reg - spi-cpha -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/temperature/maxim,max31865.yaml b/Documentation/devicetree/bindings/iio/temperature/maxim,max31865.yaml index aafb33b165499e8a38e251997631f0ba6ebf3d7b..a2823ed6867be407ad9f5440eb67cd3f0a9bed38 100644 --- a/Documentation/devicetree/bindings/iio/temperature/maxim,max31865.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/maxim,max31865.yaml @@ -25,7 +25,6 @@ properties: enables 3-wire RTD connection. Else 2-wire or 4-wire RTD connection. type: boolean - spi-max-frequency: true spi-cpha: true required: @@ -33,7 +32,10 @@ required: - reg - spi-cpha -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/iio/temperature/melexis,mlx90632.yaml b/Documentation/devicetree/bindings/iio/temperature/melexis,mlx90632.yaml index b547ddcd544ad8e1eff1fe99d9d3e0018fb30b15..4a55e7f25ae7cb5bdf163ee7720be100d043be88 100644 --- a/Documentation/devicetree/bindings/iio/temperature/melexis,mlx90632.yaml +++ b/Documentation/devicetree/bindings/iio/temperature/melexis,mlx90632.yaml @@ -35,6 +35,9 @@ properties: maxItems: 1 description: Default is 0x3a, but can be reprogrammed. + vdd-supply: + description: provide VDD power to the sensor. + required: - compatible - reg @@ -50,6 +53,7 @@ examples: temp-sensor@3a { compatible = "melexis,mlx90632"; reg = <0x3a>; + vdd-supply = <&ldo4_reg>; }; }; ... diff --git a/Documentation/devicetree/bindings/input/adc-joystick.yaml b/Documentation/devicetree/bindings/input/adc-joystick.yaml index 64d961458ac7cfc4b67e0c99a6bac794007a719e..da0f8dfca8bfd951ea2ee8ebf0e491ac2e5ba701 100644 --- a/Documentation/devicetree/bindings/input/adc-joystick.yaml +++ b/Documentation/devicetree/bindings/input/adc-joystick.yaml @@ -14,6 +14,9 @@ description: > Bindings for joystick devices connected to ADC controllers supporting the Industrial I/O subsystem. +allOf: + - $ref: input.yaml# + properties: compatible: const: adc-joystick @@ -28,6 +31,8 @@ properties: https://github.com/devicetree-org/dt-schema/blob/master/schemas/iio/iio-consumer.yaml for details. + poll-interval: true + '#address-cells': const: 1 diff --git a/Documentation/devicetree/bindings/input/adi,adp5588.yaml b/Documentation/devicetree/bindings/input/adi,adp5588.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ea66834ae24922cf828caec440a071c867e41e --- /dev/null +++ b/Documentation/devicetree/bindings/input/adi,adp5588.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/adi,adp5588.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Analog Devices ADP5588 Keypad Controller + +maintainers: + - Nuno Sá + +description: | + Analog Devices Mobile I/O Expander and QWERTY Keypad Controller + https://www.analog.com/media/en/technical-documentation/data-sheets/ADP5588.pdf + +allOf: + - $ref: matrix-keymap.yaml# + - $ref: input.yaml# + +properties: + compatible: + enum: + - adi,adp5587 + - adi,adp5588 + + reg: + maxItems: 1 + + vcc-supply: + description: Supply Voltage Input + + reset-gpios: + description: + If specified, it will be asserted during driver probe. As the line is + active low, it should be marked GPIO_ACTIVE_LOW. + maxItems: 1 + + interrupts: + maxItems: 1 + + gpio-controller: + description: + This property applies if either keypad,num-rows lower than 8 or + keypad,num-columns lower than 10. + + '#gpio-cells': + const: 2 + + interrupt-controller: + description: + This property applies if either keypad,num-rows lower than 8 or + keypad,num-columns lower than 10. + + '#interrupt-cells': + const: 2 + + adi,unlock-keys: + description: + Specifies a maximum of 2 keys that can be used to unlock the keypad. + If this property is set, the keyboard will be locked and only unlocked + after these keys are pressed. If only one key is set, a double click is + needed to unlock the keypad. The value of this property cannot be bigger + or equal than keypad,num-rows * keypad,num-columns. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 2 + +required: + - compatible + - reg + - interrupts + - keypad,num-rows + - keypad,num-columns + - linux,keymap + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + keys@34 { + compatible = "adi,adp5588"; + reg = <0x34>; + + vcc-supply = <&vcc>; + interrupts = <21 IRQ_TYPE_EDGE_FALLING>; + interrupt-parent = <&gpio>; + reset-gpios = <&gpio 20 GPIO_ACTIVE_LOW>; + + keypad,num-rows = <1>; + keypad,num-columns = <9>; + linux,keymap = < + MATRIX_KEY(0x00, 0x00, KEY_1) + MATRIX_KEY(0x00, 0x01, KEY_2) + MATRIX_KEY(0x00, 0x02, KEY_3) + MATRIX_KEY(0x00, 0x03, KEY_4) + MATRIX_KEY(0x00, 0x04, KEY_5) + MATRIX_KEY(0x00, 0x05, KEY_6) + MATRIX_KEY(0x00, 0x06, KEY_7) + MATRIX_KEY(0x00, 0x07, KEY_8) + MATRIX_KEY(0x00, 0x08, KEY_9) + >; + }; + }; +... diff --git a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml index 9700dc468b252baa2ffb1476104242e0c1dc6f29..5d631f7137e788e09c456840afe7b0b40db3b74f 100644 --- a/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml +++ b/Documentation/devicetree/bindings/input/allwinner,sun4i-a10-lradc-keys.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/input/allwinner,sun4i-a10-lradc-keys.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 LRADC Device Tree Bindings +title: Allwinner A10 LRADC maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/input/hid-over-i2c.txt b/Documentation/devicetree/bindings/input/hid-over-i2c.txt deleted file mode 100644 index 34c43d3bddfd197c9688f94d86bd316be801dfc5..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/input/hid-over-i2c.txt +++ /dev/null @@ -1,46 +0,0 @@ -* HID over I2C Device-Tree bindings - -HID over I2C provides support for various Human Interface Devices over the -I2C bus. These devices can be for example touchpads, keyboards, touch screens -or sensors. - -The specification has been written by Microsoft and is currently available here: -http://msdn.microsoft.com/en-us/library/windows/hardware/hh852380.aspx - -If this binding is used, the kernel module i2c-hid will handle the communication -with the device and the generic hid core layer will handle the protocol. - -Required properties: -- compatible: must be "hid-over-i2c" -- reg: i2c slave address -- hid-descr-addr: HID descriptor address -- interrupts: interrupt line - -Additional optional properties: - -Some devices may support additional optional properties to help with, e.g., -power sequencing. The following properties can be supported by one or more -device-specific compatible properties, which should be used in addition to the -"hid-over-i2c" string. - -- compatible: - * "wacom,w9013" (Wacom W9013 digitizer). Supports: - - vdd-supply (3.3V) - - vddl-supply (1.8V) - - post-power-on-delay-ms - -- vdd-supply: phandle of the regulator that provides the supply voltage. -- post-power-on-delay-ms: time required by the device after enabling its regulators - or powering it on, before it is ready for communication. -- touchscreen-inverted-x: See touchscreen.txt -- touchscreen-inverted-y: See touchscreen.txt - -Example: - - i2c-hid-dev@2c { - compatible = "hid-over-i2c"; - reg = <0x2c>; - hid-descr-addr = <0x0020>; - interrupt-parent = <&gpx3>; - interrupts = <3 2>; - }; diff --git a/Documentation/devicetree/bindings/input/hid-over-i2c.yaml b/Documentation/devicetree/bindings/input/hid-over-i2c.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7156b08f76453b49040ad47e53c30bcc0fde90f7 --- /dev/null +++ b/Documentation/devicetree/bindings/input/hid-over-i2c.yaml @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/hid-over-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: HID over I2C Devices + +maintainers: + - Benjamin Tissoires + - Jiri Kosina + +description: |+ + HID over I2C provides support for various Human Interface Devices over the + I2C bus. These devices can be for example touchpads, keyboards, touch screens + or sensors. + + The specification has been written by Microsoft and is currently available here: + https://msdn.microsoft.com/en-us/library/windows/hardware/hh852380.aspx + + If this binding is used, the kernel module i2c-hid will handle the communication + with the device and the generic hid core layer will handle the protocol. + +allOf: + - $ref: /schemas/input/touchscreen/touchscreen.yaml# + +properties: + compatible: + oneOf: + - items: + - enum: + - wacom,w9013 + - const: hid-over-i2c + - description: Just "hid-over-i2c" alone is allowed, but not recommended. + const: hid-over-i2c + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + hid-descr-addr: + description: HID descriptor address + $ref: /schemas/types.yaml#/definitions/uint32 + + post-power-on-delay-ms: + description: Time required by the device after enabling its regulators + or powering it on, before it is ready for communication. + + touchscreen-inverted-x: true + + touchscreen-inverted-y: true + + vdd-supply: + description: 3.3V supply + + vddl-supply: + description: 1.8V supply + + wakeup-source: true + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + hid@2c { + compatible = "hid-over-i2c"; + reg = <0x2c>; + hid-descr-addr = <0x0020>; + interrupts = <3 2>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/input/ibm,op-panel.yaml b/Documentation/devicetree/bindings/input/ibm,op-panel.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29a1879e356d81a08137507fc960e1edf48ac33b --- /dev/null +++ b/Documentation/devicetree/bindings/input/ibm,op-panel.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/ibm,op-panel.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: IBM Operation Panel + +maintainers: + - Eddie James + +allOf: + - $ref: input.yaml# + +description: | + The IBM Operation Panel provides a simple interface to control the connected + server. It has a display and three buttons: two directional arrows and one + 'Enter' button. + +properties: + compatible: + const: ibm,op-panel + + reg: + maxItems: 1 + + linux,keycodes: + minItems: 1 + maxItems: 3 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + ibm-op-panel@62 { + compatible = "ibm,op-panel"; + reg = <(0x62 | I2C_OWN_SLAVE_ADDRESS)>; + linux,keycodes = , , ; + }; + }; diff --git a/Documentation/devicetree/bindings/input/imx-keypad.yaml b/Documentation/devicetree/bindings/input/imx-keypad.yaml index f21db81206b4632345f9e23027e773cfd7d08b1b..7514df62b5926a8dbaedf91a5ec390eac355f5e9 100644 --- a/Documentation/devicetree/bindings/input/imx-keypad.yaml +++ b/Documentation/devicetree/bindings/input/imx-keypad.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/input/imx-keypad.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Freescale i.MX Keypad Port(KPP) device tree bindings +title: Freescale i.MX Keypad Port(KPP) maintainers: - Liu Ying diff --git a/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml index 03ebd2665d0781e69ab388fc11e08bd63803f07f..d768c30f48fb172fa6d48d81a4d17df8808550c2 100644 --- a/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml +++ b/Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/input/mediatek,mt6779-keypad.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek's Keypad Controller device tree bindings +title: Mediatek's Keypad Controller maintainers: - Mattijs Korpershoek @@ -49,6 +49,12 @@ properties: maximum: 256 default: 16 + mediatek,keys-per-group: + description: each (row, column) group has multiple keys + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1 + maximum: 2 + required: - compatible - reg @@ -56,7 +62,7 @@ required: - clocks - clock-names -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/input/mediatek,pmic-keys.yaml b/Documentation/devicetree/bindings/input/mediatek,pmic-keys.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f72ec41841574588b3b0df15c656da547214dbf --- /dev/null +++ b/Documentation/devicetree/bindings/input/mediatek,pmic-keys.yaml @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/mediatek,pmic-keys.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek PMIC Keys + +maintainers: + - Chen Zhong + +allOf: + - $ref: input.yaml# + +description: | + There are two key functions provided by MT6397, MT6323 and other MediaTek + PMICs: pwrkey and homekey. + The key functions are defined as the subnode of the function node provided + by the PMIC that is defined as a Multi-Function Device (MFD). + + For MediaTek MT6323/MT6397 PMIC bindings see + Documentation/devicetree/bindings/mfd/mt6397.txt + +properties: + compatible: + enum: + - mediatek,mt6323-keys + - mediatek,mt6331-keys + - mediatek,mt6358-keys + - mediatek,mt6397-keys + + power-off-time-sec: true + + mediatek,long-press-mode: + description: | + Key long-press force shutdown setting + 0 - disabled + 1 - pwrkey + 2 - pwrkey+homekey + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0 + maximum: 2 + +patternProperties: + "^((power|home)|(key-[a-z0-9-]+|[a-z0-9-]+-key))$": + $ref: input.yaml# + + properties: + interrupts: + minItems: 1 + items: + - description: Key press interrupt + - description: Key release interrupt + + interrupt-names: true + + linux-keycodes: + maxItems: 1 + + wakeup-source: true + + required: + - linux,keycodes + + if: + properties: + interrupt-names: + contains: + const: powerkey + then: + properties: + interrupt-names: + minItems: 1 + items: + - const: powerkey + - const: powerkey_r + else: + properties: + interrupt-names: + minItems: 1 + items: + - const: homekey + - const: homekey_r + + unevaluatedProperties: false + +required: + - compatible + +unevaluatedProperties: false + +examples: + - | + #include + #include + + pmic { + compatible = "mediatek,mt6397"; + + keys { + compatible = "mediatek,mt6397-keys"; + mediatek,long-press-mode = <1>; + power-off-time-sec = <0>; + + key-power { + linux,keycodes = ; + wakeup-source; + }; + + key-home { + linux,keycodes = ; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt b/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt deleted file mode 100644 index 9d00f2a8e13a4ece0e8573ff6df7b1ec6fc17b39..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/input/mtk-pmic-keys.txt +++ /dev/null @@ -1,46 +0,0 @@ -MediaTek MT6397/MT6323 PMIC Keys Device Driver - -There are two key functions provided by MT6397/MT6323 PMIC, pwrkey -and homekey. The key functions are defined as the subnode of the function -node provided by MT6397/MT6323 PMIC that is being defined as one kind -of Muti-Function Device (MFD) - -For MT6397/MT6323 MFD bindings see: -Documentation/devicetree/bindings/mfd/mt6397.txt - -Required properties: -- compatible: Should be one of: - - "mediatek,mt6397-keys" - - "mediatek,mt6323-keys" - - "mediatek,mt6358-keys" -- linux,keycodes: See Documentation/devicetree/bindings/input/input.yaml - -Optional Properties: -- wakeup-source: See Documentation/devicetree/bindings/power/wakeup-source.txt -- mediatek,long-press-mode: Long press key shutdown setting, 1 for - pwrkey only, 2 for pwrkey/homekey together, others for disabled. -- power-off-time-sec: See Documentation/devicetree/bindings/input/input.yaml - -Example: - - pmic: mt6397 { - compatible = "mediatek,mt6397"; - - ... - - mt6397keys: mt6397keys { - compatible = "mediatek,mt6397-keys"; - mediatek,long-press-mode = <1>; - power-off-time-sec = <0>; - - power { - linux,keycodes = <116>; - wakeup-source; - }; - - home { - linux,keycodes = <114>; - }; - }; - - }; diff --git a/Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml b/Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e4a0ac0fff9a7c35a796e2be1e04805384745599 --- /dev/null +++ b/Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/pine64,pinephone-keyboard.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Pine64 PinePhone keyboard device tree bindings + +maintainers: + - Samuel Holland + +description: + A keyboard accessory is available for the Pine64 PinePhone and PinePhone Pro. + It connects via I2C, providing a raw scan matrix, a flashing interface, and a + subordinate I2C bus for communication with a battery charger IC. + +properties: + compatible: + const: pine64,pinephone-keyboard + + reg: + const: 0x15 + + interrupts: + maxItems: 1 + + vbat-supply: + description: Supply for the keyboard MCU + + wakeup-source: true + + i2c: + $ref: /schemas/i2c/i2c-controller.yaml# + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + keyboard@15 { + compatible = "pine64,pinephone-keyboard"; + reg = <0x15>; + interrupt-parent = <&r_pio>; + interrupts = <0 12 IRQ_TYPE_EDGE_FALLING>; /* PL12 */ + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + charger@75 { + reg = <0x75>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.txt b/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.txt deleted file mode 100644 index 64bb990075c31c3c51a1b1b1aabd4b8a3083fc30..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.txt +++ /dev/null @@ -1,23 +0,0 @@ -Qualcomm PM8xxx PMIC Vibrator - -PROPERTIES - -- compatible: - Usage: required - Value type: - Definition: must be one of: - "qcom,pm8058-vib" - "qcom,pm8916-vib" - "qcom,pm8921-vib" - -- reg: - Usage: required - Value type: - Definition: address of vibration control register - -EXAMPLE - - vibrator@4a { - compatible = "qcom,pm8058-vib"; - reg = <0x4a>; - }; diff --git a/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.yaml b/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8832cd0d7da2f9c1a85fdbb86f6ebb33a24e448 --- /dev/null +++ b/Documentation/devicetree/bindings/input/qcom,pm8xxx-vib.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/qcom,pm8xxx-vib.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm PM8xxx PMIC Vibrator + +maintainers: + - Bjorn Andersson + +properties: + compatible: + enum: + - qcom,pm8058-vib + - qcom,pm8916-vib + - qcom,pm8921-vib + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + pmic { + #address-cells = <1>; + #size-cells = <0>; + + vibrator@4a { + compatible = "qcom,pm8058-vib"; + reg = <0x4a>; + }; + }; diff --git a/Documentation/devicetree/bindings/input/regulator-haptic.yaml b/Documentation/devicetree/bindings/input/regulator-haptic.yaml index b1ae72f9cd2dfb991390bdc5a08757876cd940e4..627891e1ef55f9e3c618652c846c6a30083dfba2 100644 --- a/Documentation/devicetree/bindings/input/regulator-haptic.yaml +++ b/Documentation/devicetree/bindings/input/regulator-haptic.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/input/regulator-haptic.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: Regulator Haptic Device Tree Bindings +title: Regulator Haptic maintainers: - Jaewon Kim diff --git a/Documentation/devicetree/bindings/input/sprd,sc27xx-vibrator.yaml b/Documentation/devicetree/bindings/input/sprd,sc27xx-vibrator.yaml index 5d67fc8ebc180d0cbfc6abb3b82b4a5d3f72e2d2..a401a0bfcbec21e098e4fe7bf9b85410d6ea83a8 100644 --- a/Documentation/devicetree/bindings/input/sprd,sc27xx-vibrator.yaml +++ b/Documentation/devicetree/bindings/input/sprd,sc27xx-vibrator.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/input/sprd,sc27xx-vibrator.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Spreadtrum SC27xx PMIC Vibrator Device Tree Bindings +title: Spreadtrum SC27xx PMIC Vibrator maintainers: - Orson Zhai diff --git a/Documentation/devicetree/bindings/input/touchscreen/auo_pixcir_ts.txt b/Documentation/devicetree/bindings/input/touchscreen/auo_pixcir_ts.txt index f40f21c642b96f90d9447d5984a0119d6e58cba4..b8db975e9f7788179af6fff916170c690d8dd202 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/auo_pixcir_ts.txt +++ b/Documentation/devicetree/bindings/input/touchscreen/auo_pixcir_ts.txt @@ -17,10 +17,10 @@ Example: auo_pixcir_ts@5c { compatible = "auo,auo_pixcir_ts"; reg = <0x5c>; - interrupts = <2 0>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>; - gpios = <&gpf 2 0 2>, /* INT */ - <&gpf 5 1 0>; /* RST */ + gpios = <&gpf 2 0 GPIO_LEVEL_HIGH>, /* INT */ + <&gpf 5 1 GPIO_LEVEL_LOW>; /* RST */ x-size = <800>; y-size = <600>; diff --git a/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt b/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt index 2e1490a8fe74ed8ca36f3b655d62f4e4ca3458d6..ca304357c374adbbe7079ddd6693d2416291f8a3 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt +++ b/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt @@ -3,15 +3,16 @@ Required Properties: - compatible must be toradex,vf50-touchscreen - io-channels: adc channels being used by the Colibri VF50 module + IIO ADC for Y-, X-, Y+, X+ connections - xp-gpios: FET gate driver for input of X+ - xm-gpios: FET gate driver for input of X- - yp-gpios: FET gate driver for input of Y+ - ym-gpios: FET gate driver for input of Y- -- interrupts: pen irq interrupt for touch detection -- pinctrl-names: "idle", "default", "gpios" -- pinctrl-0: pinctrl node for pen/touch detection state pinmux +- interrupts: pen irq interrupt for touch detection, signal from X plate +- pinctrl-names: "idle", "default" +- pinctrl-0: pinctrl node for pen/touch detection, pinctrl must provide + pull-up resistor on X+, X-. - pinctrl-1: pinctrl node for X/Y and pressure measurement (ADC) state pinmux -- pinctrl-2: pinctrl node for gpios functioning as FET gate drivers - vf50-ts-min-pressure: pressure level at which to stop measuring X/Y values Example: @@ -26,9 +27,8 @@ Example: ym-gpios = <&gpio0 4 GPIO_ACTIVE_HIGH>; interrupt-parent = <&gpio0>; interrupts = <8 IRQ_TYPE_LEVEL_LOW>; - pinctrl-names = "idle","default","gpios"; - pinctrl-0 = <&pinctrl_touchctrl_idle>; - pinctrl-1 = <&pinctrl_touchctrl_default>; - pinctrl-2 = <&pinctrl_touchctrl_gpios>; + pinctrl-names = "idle","default"; + pinctrl-0 = <&pinctrl_touchctrl_idle>, <&pinctrl_touchctrl_gpios>; + pinctrl-1 = <&pinctrl_touchctrl_default>, <&pinctrl_touchctrl_gpios>; vf50-ts-min-pressure = <200>; }; diff --git a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml index a9b53c2e6f0abc8e8bc495981995105322e7bfaa..f9053e5e9b240e95426b47070a5f423d31c709ad 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml @@ -14,9 +14,13 @@ allOf: properties: compatible: - enum: - - elan,ektf3624 - - elan,ekth3500 + oneOf: + - enum: + - elan,ektf3624 + - elan,ekth3500 + - items: + - const: elan,ekth3915 + - const: elan,ekth3500 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml index 93f2ce3130ae7e9c55806162d994a0414c3e94b4..19ac9da421df3356ef19ffaa7bbf53f5d4126e63 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/goodix.yaml @@ -16,6 +16,7 @@ properties: compatible: enum: - goodix,gt1151 + - goodix,gt1158 - goodix,gt5663 - goodix,gt5688 - goodix,gt911 diff --git a/Documentation/devicetree/bindings/input/touchscreen/stmpe.txt b/Documentation/devicetree/bindings/input/touchscreen/stmpe.txt index c549924603d2691e5e7ceac14517c41cbf922165..238b51555c0473879424a3fb0903c814681e3e43 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/stmpe.txt +++ b/Documentation/devicetree/bindings/input/touchscreen/stmpe.txt @@ -54,8 +54,7 @@ Optional properties common with MFD (deprecated): 1 -> 3.25 MHz 2 || 3 -> 6.5 MHz -Node name must be stmpe_touchscreen and should be child node of stmpe node to -which it belongs. +Node should be child node of stmpe node to which it belongs. Note that common ADC settings of stmpe_touchscreen (child) will take precedence over the settings done in MFD. diff --git a/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml b/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml index 09c8948b5e251ee2d1983d378f31e00e77f8d084..f7a5e31c506ee439283b380a5599e3575dbdeeec 100644 --- a/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml +++ b/Documentation/devicetree/bindings/interconnect/fsl,imx8m-noc.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Generic i.MX bus frequency device maintainers: - - Leonard Crestez + - Peng Fan description: | The i.MX SoC family has multiple buses for which clock frequency (and @@ -47,7 +47,8 @@ properties: maxItems: 1 operating-points-v2: true - opp-table: true + opp-table: + type: object fsl,ddrc: $ref: "/schemas/types.yaml#/definitions/phandle" diff --git a/Documentation/devicetree/bindings/interconnect/mediatek,cci.yaml b/Documentation/devicetree/bindings/interconnect/mediatek,cci.yaml index 449c7c9882296a7ac3d3e21856029e473ec69b17..58611ba2a0f440b25aa8aa37c36ca339eecbbc85 100644 --- a/Documentation/devicetree/bindings/interconnect/mediatek,cci.yaml +++ b/Documentation/devicetree/bindings/interconnect/mediatek,cci.yaml @@ -36,7 +36,8 @@ properties: - const: intermediate operating-points-v2: true - opp-table: true + opp-table: + type: object proc-supply: description: diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml index c2e697f6e6cf8186a7b9cc2e640132182a7a48b0..2684562df4d91115c674022481cb2eb22e6fd01b 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml @@ -24,9 +24,12 @@ properties: oneOf: - items: - enum: + - qcom,sc7280-bwmon - qcom,sdm845-bwmon - const: qcom,msm8998-bwmon - const: qcom,msm8998-bwmon # BWMON v4 + - const: qcom,sc7280-llcc-bwmon # BWMON v5 + - const: qcom,sdm845-llcc-bwmon # BWMON v5 interconnects: maxItems: 1 @@ -35,7 +38,8 @@ properties: maxItems: 1 operating-points-v2: true - opp-table: true + opp-table: + type: object reg: # BWMON v4 (currently described) and BWMON v5 use one register address diff --git a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun4i-a10-ic.yaml b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun4i-a10-ic.yaml index 953d875b5e74aab183787caf6aec267b9a9e7322..a713633be73347fdfe558e9141c4bfb8dac79b5f 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun4i-a10-ic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun4i-a10-ic.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/interrupt-controller/allwinner,sun4i-a10-ic.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Interrupt Controller Device Tree Bindings +title: Allwinner A10 Interrupt Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun6i-a31-r-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun6i-a31-r-intc.yaml index 4db24b8a9ffe225ba9db50f1fc53b25273927b16..4fa6fd400eefac6403efc0d42f6e970ee0d89e23 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun6i-a31-r-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun6i-a31-r-intc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/interrupt-controller/allwinner,sun6i-a31-r-intc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 NMI/Wakeup Interrupt Controller Device Tree Bindings +title: Allwinner A31 NMI/Wakeup Interrupt Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml index 7fc9ad5ef38cc90a794237e12529dbbc6ed12b91..83603180d8d9cd6b64602dcfd415db7a4c65cf58 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/interrupt-controller/allwinner,sun7i-a20-sc-nmi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A20 Non-Maskable Interrupt Controller Device Tree Bindings +title: Allwinner A20 Non-Maskable Interrupt Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml b/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml index 85c85b694217c16d157ee0c68509b95f9baa322c..e18107eafe7cc4038fb2f4b16fe47f0741129175 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/apple,aic.yaml @@ -96,7 +96,7 @@ properties: Documentation/devicetree/bindings/arm/cpus.yaml). required: - - fiq-index + - apple,fiq-index - cpus required: diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml index 3912a89162f02b79daf687e9b4927104420251f2..9f7d3e11aacb8aa377f05a7a08666a12a81b54e2 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml @@ -170,7 +170,6 @@ dependencies: required: - compatible - - interrupts - reg patternProperties: diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml b/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml index 62219a5c21c5b423892b7e9c28a43934fe71fdbc..220256907461c12e1b16e9bfa9bb2eddc00d5a0f 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic.yaml @@ -64,9 +64,9 @@ properties: interrupt-controller: true "#address-cells": - enum: [ 0, 1 ] + enum: [ 0, 1, 2 ] "#size-cells": - const: 1 + enum: [ 1, 2 ] "#interrupt-cells": const: 3 diff --git a/Documentation/devicetree/bindings/interrupt-controller/fsl,mu-msi.yaml b/Documentation/devicetree/bindings/interrupt-controller/fsl,mu-msi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..799ae5c3e32ae705cc680a2cdcb6a390349f98e9 --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/fsl,mu-msi.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interrupt-controller/fsl,mu-msi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale/NXP i.MX Messaging Unit (MU) work as msi controller + +maintainers: + - Frank Li + +description: | + The Messaging Unit module enables two processors within the SoC to + communicate and coordinate by passing messages (e.g. data, status + and control) through the MU interface. The MU also provides the ability + for one processor (A side) to signal the other processor (B side) using + interrupts. + + Because the MU manages the messaging between processors, the MU uses + different clocks (from each side of the different peripheral buses). + Therefore, the MU must synchronize the accesses from one side to the + other. The MU accomplishes synchronization using two sets of matching + registers (Processor A-side, Processor B-side). + + MU can work as msi interrupt controller to do doorbell + +allOf: + - $ref: /schemas/interrupt-controller/msi-controller.yaml# + +properties: + compatible: + enum: + - fsl,imx6sx-mu-msi + - fsl,imx7ulp-mu-msi + - fsl,imx8ulp-mu-msi + - fsl,imx8ulp-mu-msi-s4 + + reg: + items: + - description: a side register base address + - description: b side register base address + + reg-names: + items: + - const: processor-a-side + - const: processor-b-side + + interrupts: + description: a side interrupt number. + maxItems: 1 + + clocks: + maxItems: 1 + + power-domains: + items: + - description: a side power domain + - description: b side power domain + + power-domain-names: + items: + - const: processor-a-side + - const: processor-b-side + + interrupt-controller: true + + msi-controller: true + + "#msi-cells": + const: 0 + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - msi-controller + - "#msi-cells" + +additionalProperties: false + +examples: + - | + #include + #include + + msi-controller@5d270000 { + compatible = "fsl,imx6sx-mu-msi"; + msi-controller; + #msi-cells = <0>; + interrupt-controller; + reg = <0x5d270000 0x10000>, /* A side */ + <0x5d300000 0x10000>; /* B side */ + reg-names = "processor-a-side", "processor-b-side"; + interrupts = ; + power-domains = <&pd IMX_SC_R_MU_12A>, + <&pd IMX_SC_R_MU_12B>; + power-domain-names = "processor-a-side", "processor-b-side"; + }; diff --git a/Documentation/devicetree/bindings/interrupt-controller/idt,32434-pic.yaml b/Documentation/devicetree/bindings/interrupt-controller/idt,32434-pic.yaml index 160ff4b07cac904f35ab50bcbfff80eedb99156d..afb3dd80b643bbaaa95b9a66109a98755473469c 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/idt,32434-pic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/idt,32434-pic.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/interrupt-controller/idt,32434-pic.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: IDT 79RC32434 Interrupt Controller Device Tree Bindings +title: IDT 79RC32434 Interrupt Controller maintainers: - Thomas Bogendoerfer diff --git a/Documentation/devicetree/bindings/interrupt-controller/mti,cpu-interrupt-controller.yaml b/Documentation/devicetree/bindings/interrupt-controller/mti,cpu-interrupt-controller.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46a1f5f54b746cd6cee73ed6da7e1b600b97c11a --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/mti,cpu-interrupt-controller.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interrupt-controller/mti,cpu-interrupt-controller.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MIPS CPU Interrupt Controller + +description: > + On MIPS the mips_cpu_irq_of_init() helper can be used to initialize the 8 CPU + IRQs from a devicetree file and create a irq_domain for IRQ controller. + + With the irq_domain in place we can describe how the 8 IRQs are wired to the + platforms internal interrupt controller cascade. + +maintainers: + - Thomas Bogendoerfer + +properties: + compatible: + const: mti,cpu-interrupt-controller + + '#interrupt-cells': + const: 1 + + '#address-cells': + const: 0 + + interrupt-controller: true + +additionalProperties: false + +required: + - compatible + - '#interrupt-cells' + - '#address-cells' + - interrupt-controller + +examples: + - | + interrupt-controller { + compatible = "mti,cpu-interrupt-controller"; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.txt b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.txt deleted file mode 100644 index 159a423e558690b9f09f1c16e07f2aad7b7b63d0..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.txt +++ /dev/null @@ -1,78 +0,0 @@ -PDC interrupt controller - -Qualcomm Technologies Inc. SoCs based on the RPM Hardened architecture have a -Power Domain Controller (PDC) that is on always-on domain. In addition to -providing power control for the power domains, the hardware also has an -interrupt controller that can be used to help detect edge low interrupts as -well detect interrupts when the GIC is non-operational. - -GIC is parent interrupt controller at the highest level. Platform interrupt -controller PDC is next in hierarchy, followed by others. Drivers requiring -wakeup capabilities of their device interrupts routed through the PDC, must -specify PDC as their interrupt controller and request the PDC port associated -with the GIC interrupt. See example below. - -Properties: - -- compatible: - Usage: required - Value type: - Definition: Should contain "qcom,-pdc" and "qcom,pdc" - - "qcom,sc7180-pdc": For SC7180 - - "qcom,sc7280-pdc": For SC7280 - - "qcom,sdm845-pdc": For SDM845 - - "qcom,sm6350-pdc": For SM6350 - - "qcom,sm8150-pdc": For SM8150 - - "qcom,sm8250-pdc": For SM8250 - - "qcom,sm8350-pdc": For SM8350 - -- reg: - Usage: required - Value type: - Definition: Specifies the base physical address for PDC hardware. - -- interrupt-cells: - Usage: required - Value type: - Definition: Specifies the number of cells needed to encode an interrupt - source. - Must be 2. - The first element of the tuple is the PDC pin for the - interrupt. - The second element is the trigger type. - -- interrupt-controller: - Usage: required - Value type: - Definition: Identifies the node as an interrupt controller. - -- qcom,pdc-ranges: - Usage: required - Value type: - Definition: Specifies the PDC pin offset and the number of PDC ports. - The tuples indicates the valid mapping of valid PDC ports - and their hwirq mapping. - The first element of the tuple is the starting PDC port. - The second element is the GIC hwirq number for the PDC port. - The third element is the number of interrupts in sequence. - -Example: - - pdc: interrupt-controller@b220000 { - compatible = "qcom,sdm845-pdc"; - reg = <0xb220000 0x30000>; - qcom,pdc-ranges = <0 512 94>, <94 641 15>, <115 662 7>; - #interrupt-cells = <2>; - interrupt-parent = <&intc>; - interrupt-controller; - }; - -DT binding of a device that wants to use the GIC SPI 514 as a wakeup -interrupt, must do - - - wake-device { - interrupts-extended = <&pdc 2 IRQ_TYPE_LEVEL_HIGH>; - }; - -In this case interrupt 514 would be mapped to port 2 on the PDC as defined by -the qcom,pdc-ranges property. diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6f56cf5fbe3cb8ae937b27cfb7122473b15705a --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/interrupt-controller/qcom,pdc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: PDC interrupt controller + +maintainers: + - Bjorn Andersson + +description: | + Qualcomm Technologies Inc. SoCs based on the RPM Hardened architecture have a + Power Domain Controller (PDC) that is on always-on domain. In addition to + providing power control for the power domains, the hardware also has an + interrupt controller that can be used to help detect edge low interrupts as + well detect interrupts when the GIC is non-operational. + + GIC is parent interrupt controller at the highest level. Platform interrupt + controller PDC is next in hierarchy, followed by others. Drivers requiring + wakeup capabilities of their device interrupts routed through the PDC, must + specify PDC as their interrupt controller and request the PDC port associated + with the GIC interrupt. See example below. + +properties: + compatible: + items: + - enum: + - qcom,sc7180-pdc + - qcom,sc7280-pdc + - qcom,sdm845-pdc + - qcom,sm6350-pdc + - qcom,sm8150-pdc + - qcom,sm8250-pdc + - qcom,sm8350-pdc + - const: qcom,pdc + + reg: + minItems: 1 + items: + - description: PDC base register region + - description: Edge or Level config register for SPI interrupts + + '#interrupt-cells': + const: 2 + + interrupt-controller: true + + qcom,pdc-ranges: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + minItems: 1 + maxItems: 32 # no hard limit + items: + items: + - description: starting PDC port + - description: GIC hwirq number for the PDC port + - description: number of interrupts in sequence + description: | + Specifies the PDC pin offset and the number of PDC ports. + The tuples indicates the valid mapping of valid PDC ports + and their hwirq mapping. + +required: + - compatible + - reg + - '#interrupt-cells' + - interrupt-controller + - qcom,pdc-ranges + +additionalProperties: false + +examples: + - | + #include + + pdc: interrupt-controller@b220000 { + compatible = "qcom,sdm845-pdc", "qcom,pdc"; + reg = <0xb220000 0x30000>; + qcom,pdc-ranges = <0 512 94>, <94 641 15>, <115 662 7>; + #interrupt-cells = <2>; + interrupt-parent = <&intc>; + interrupt-controller; + }; + + wake-device { + interrupts-extended = <&pdc 2 IRQ_TYPE_LEVEL_HIGH>; + }; diff --git a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml index 9e76fff20323c6f1508c5109925b752ecdf168d4..13a893b18fb64aa9d15458ac93b01809b9aa07a8 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/realtek,rtl-intc.yaml @@ -6,6 +6,14 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Realtek RTL SoC interrupt controller devicetree bindings +description: + Interrupt controller and router for Realtek MIPS SoCs, allowing each SoC + interrupt to be routed to one parent CPU (hardware) interrupt, or left + disconnected. + All connected input lines from SoC peripherals can be masked individually, + and an interrupt status register is present to indicate which interrupts are + pending. + maintainers: - Birger Koblitz - Bert Vermeulen @@ -13,23 +21,33 @@ maintainers: properties: compatible: - const: realtek,rtl-intc + oneOf: + - items: + - enum: + - realtek,rtl8380-intc + - const: realtek,rtl-intc + - const: realtek,rtl-intc + deprecated: true "#interrupt-cells": + description: + SoC interrupt line index. const: 1 reg: maxItems: 1 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 15 + description: + List of parent interrupts, in the order that they are connected to this + interrupt router's outputs, starting at the first output. interrupt-controller: true - "#address-cells": - const: 0 - interrupt-map: + deprecated: true description: Describes mapping from SoC interrupts to CPU interrupts required: @@ -37,21 +55,33 @@ required: - reg - "#interrupt-cells" - interrupt-controller - - "#address-cells" - - interrupt-map + +allOf: + - if: + properties: + compatible: + const: realtek,rtl-intc + then: + properties: + "#address-cells": + const: 0 + required: + - "#address-cells" + - interrupt-map + else: + required: + - interrupts additionalProperties: false examples: - | - intc: interrupt-controller@3000 { - compatible = "realtek,rtl-intc"; + interrupt-controller@3000 { + compatible = "realtek,rtl8380-intc", "realtek,rtl-intc"; #interrupt-cells = <1>; interrupt-controller; - reg = <0x3000 0x20>; - #address-cells = <0>; - interrupt-map = - <31 &cpuintc 2>, - <30 &cpuintc 1>, - <29 &cpuintc 5>; + reg = <0x3000 0x18>; + + interrupt-parent = <&cpuintc>; + interrupts = <2>, <3>, <4>, <5>, <6>; }; diff --git a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml index 620f01775e429c3e4b76ccd1ae51f813da41b31d..62fd47c88275d58722df4f093f0479059ae22063 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/renesas,irqc.yaml @@ -37,6 +37,7 @@ properties: - renesas,intc-ex-r8a77990 # R-Car E3 - renesas,intc-ex-r8a77995 # R-Car D3 - renesas,intc-ex-r8a779a0 # R-Car V3U + - renesas,intc-ex-r8a779g0 # R-Car V4H - const: renesas,irqc '#interrupt-cells': diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml index 92e0f8c3eff2da6b8e69cbcdedc9dc3888468989..99e01f4d0a6933acfce4f403554475fba0dbf60a 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml @@ -66,6 +66,11 @@ properties: - enum: - allwinner,sun20i-d1-plic - const: thead,c900-plic + - items: + - const: sifive,plic-1.0.0 + - const: riscv,plic0 + deprecated: true + description: For the QEMU virt machine only reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml index e44daa09b137c6f9b870bb974078d196463e1cd2..00c10a8258f138e94e089f464b8d5b293e74cf6c 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/interrupt-controller/st,stm32-exti.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: STM32 External Interrupt Controller Device Tree Bindings +title: STM32 External Interrupt Controller maintainers: - Alexandre Torgue diff --git a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.yaml b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.yaml index 88c46e61732e18a998e54bf2b9055185bac10c43..1151518859bd0a8ff8b032c0708141747a4869c6 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-inta.yaml @@ -59,6 +59,9 @@ properties: interrupt-controller: true + '#interrupt-cells': + const: 0 + msi-controller: true ti,interrupt-ranges: diff --git a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.yaml b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.yaml index e12aee42b12685fe6a26233bc79a73a8d30e6137..c99cc7323c711de6bada10ed3adb8078d88d003a 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/ti,sci-intr.yaml @@ -58,6 +58,9 @@ properties: 1 = If intr supports edge triggered interrupts. 4 = If intr supports level triggered interrupts. + reg: + maxItems: 1 + interrupt-controller: true '#interrupt-cells': diff --git a/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml b/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml index 5e125cf2a88b458cd5c3aa7c1b2079bd474d73fc..e20016f120175170a7754192ee658462d9ba3fc9 100644 --- a/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml +++ b/Documentation/devicetree/bindings/iommu/allwinner,sun50i-h6-iommu.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/iommu/allwinner,sun50i-h6-iommu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner H6 IOMMU Device Tree Bindings +title: Allwinner H6 IOMMU maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml index c57a53d87e4e6dfaba2da9c1f676f4d915e699e6..75fcf4cb52d9f6449238578f20288966af35cab3 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml @@ -39,16 +39,11 @@ properties: any others. - minItems: 1 items: - - enum: - - eventq # Event Queue not empty - - gerror # Global Error activated - - const: gerror - - enum: - - cmdq-sync # CMD_SYNC complete - - priq # PRI Queue not empty - - enum: - - cmdq-sync - - priq + enum: + - eventq # Event Queue not empty + - gerror # Global Error activated + - cmdq-sync # CMD_SYNC complete + - priq # PRI Queue not empty '#iommu-cells': const: 1 diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml index fee0241b509880c84e9e0f92001a3e7215e35e0e..839e3be0bf3ca7f7f302cff51c144dd3c9b56b18 100644 --- a/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml +++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml @@ -73,6 +73,7 @@ properties: - mediatek,mt2701-m4u # generation one - mediatek,mt2712-m4u # generation two - mediatek,mt6779-m4u # generation two + - mediatek,mt6795-m4u # generation two - mediatek,mt8167-m4u # generation two - mediatek,mt8173-m4u # generation two - mediatek,mt8183-m4u # generation two @@ -124,6 +125,7 @@ properties: dt-binding/memory/mt2701-larb-port.h for mt2701 and mt7623, dt-binding/memory/mt2712-larb-port.h for mt2712, dt-binding/memory/mt6779-larb-port.h for mt6779, + dt-binding/memory/mt6795-larb-port.h for mt6795, dt-binding/memory/mt8167-larb-port.h for mt8167, dt-binding/memory/mt8173-larb-port.h for mt8173, dt-binding/memory/mt8183-larb-port.h for mt8183, @@ -148,6 +150,7 @@ allOf: enum: - mediatek,mt2701-m4u - mediatek,mt2712-m4u + - mediatek,mt6795-m4u - mediatek,mt8173-m4u - mediatek,mt8186-iommu-mm - mediatek,mt8192-m4u @@ -177,6 +180,7 @@ allOf: contains: enum: - mediatek,mt2712-m4u + - mediatek,mt6795-m4u - mediatek,mt8173-m4u then: diff --git a/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt b/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt index 352f5e9c759bc3f5da8d7b4a1d6f7c754a794da5..4fda76e63396a0bb22faa5bf0dbc3258e395509e 100644 --- a/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt +++ b/Documentation/devicetree/bindings/ipmi/npcm7xx-kcs-bmc.txt @@ -1,12 +1,13 @@ -* Nuvoton NPCM7xx KCS (Keyboard Controller Style) IPMI interface +* Nuvoton NPCM KCS (Keyboard Controller Style) IPMI interface -The Nuvoton SOCs (NPCM7xx) are commonly used as BMCs +The Nuvoton SOCs (NPCM) are commonly used as BMCs (Baseboard Management Controllers) and the KCS interface can be used to perform in-band IPMI communication with their host. Required properties: - compatible : should be one of "nuvoton,npcm750-kcs-bmc" + "nuvoton,npcm845-kcs-bmc", "nuvoton,npcm750-kcs-bmc" - interrupts : interrupt generated by the controller - kcs_chan : The KCS channel number in the controller diff --git a/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml b/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5533b6562d92414838bff25d8f9f3156608483c6 --- /dev/null +++ b/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/backlight/mediatek,mt6370-backlight.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MT6370 Backlight + +maintainers: + - ChiaEn Wu + +description: | + This module is part of the MT6370 MFD device. + The MT6370 Backlight WLED driver supports up to a 29V output voltage for + 4 channels of 8 series WLEDs. Each channel supports up to 30mA of current + capability with 2048 current steps (11 bits, only for MT6370/MT6371) or + 16384 current steps (14 bits, only for MT6372) in exponential or linear + mapping curves. + +allOf: + - $ref: common.yaml# + +properties: + compatible: + enum: + - mediatek,mt6370-backlight + - mediatek,mt6372-backlight + + default-brightness: + minimum: 0 + + max-brightness: + minimum: 0 + + enable-gpios: + description: External backlight 'enable' pin + maxItems: 1 + + mediatek,bled-pwm-enable: + description: | + Enable external PWM input for backlight dimming + type: boolean + + mediatek,bled-pwm-hys-enable: + description: | + Enable the backlight input-hysteresis for PWM mode + type: boolean + + mediatek,bled-pwm-hys-input-th-steps: + $ref: /schemas/types.yaml#/definitions/uint8 + enum: [1, 4, 16, 64] + description: | + The selection of the upper and lower bounds threshold of backlight + PWM resolution. If we choose selection 64, the variation of PWM + resolution needs more than 64 steps. + + mediatek,bled-ovp-shutdown: + description: | + Enable the backlight shutdown when OVP level triggered + type: boolean + + mediatek,bled-ovp-microvolt: + enum: [17000000, 21000000, 25000000, 29000000] + description: | + Backlight OVP level selection. + + mediatek,bled-ocp-shutdown: + description: | + Enable the backlight shutdown when OCP level triggerred. + type: boolean + + mediatek,bled-ocp-microamp: + enum: [900000, 1200000, 1500000, 1800000] + description: | + Backlight OC level selection. + + mediatek,bled-exponential-mode-enable: + description: | + Enable the exponential mode of backlight brightness. If this property + is not enabled, the default is to use linear mode. + type: boolean + + mediatek,bled-channel-use: + $ref: /schemas/types.yaml#/definitions/uint8 + description: | + Backlight LED channel to be used. + Each bit mapping to: + - 0: CH4 + - 1: CH3 + - 2: CH2 + - 3: CH1 + minimum: 1 + maximum: 15 + +if: + properties: + compatible: + contains: + const: mediatek,mt6372-backlight + +then: + properties: + default-brightness: + maximum: 16384 + + max-brightness: + maximum: 16384 + +else: + properties: + default-brightness: + maximum: 2048 + + max-brightness: + maximum: 2048 + +required: + - compatible + - mediatek,bled-channel-use + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml b/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml index 5d66c3e4def50e8c11771cb5c6ff42f2f44d3c9e..4c15693f7a013e648fecb834d475811d93de8781 100644 --- a/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml +++ b/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml @@ -26,7 +26,8 @@ properties: - qcom,pm8150l-wled reg: - maxItems: 1 + minItems: 1 + maxItems: 2 default-brightness: description: | @@ -171,6 +172,9 @@ allOf: then: properties: + reg: + maxItems: 1 + qcom,current-boost-limit: enum: [ 105, 385, 525, 805, 980, 1260, 1400, 1680 ] default: 805 @@ -189,6 +193,9 @@ allOf: else: properties: + reg: + minItems: 2 + qcom,current-boost-limit: enum: [ 105, 280, 450, 620, 970, 1150, 1300, 1500 ] default: 970 diff --git a/Documentation/devicetree/bindings/leds/common.yaml b/Documentation/devicetree/bindings/leds/common.yaml index 328952d7acbbc39fcdfd3fd9afe504175c56b070..3c14a98430e190afa765419a27032652f7cc01bf 100644 --- a/Documentation/devicetree/bindings/leds/common.yaml +++ b/Documentation/devicetree/bindings/leds/common.yaml @@ -79,24 +79,27 @@ properties: the LED. $ref: /schemas/types.yaml#/definitions/string - enum: - # LED will act as a back-light, controlled by the framebuffer system - - backlight - # LED will turn on (but for leds-gpio see "default-state" property in - # Documentation/devicetree/bindings/leds/leds-gpio.yaml) - - default-on - # LED "double" flashes at a load average based rate - - heartbeat - # LED indicates disk activity - - disk-activity - # LED indicates IDE disk activity (deprecated), in new implementations - # use "disk-activity" - - ide-disk - # LED flashes at a fixed, configurable rate - - timer - # LED alters the brightness for the specified duration with one software - # timer (requires "led-pattern" property) - - pattern + oneOf: + - enum: + # LED will act as a back-light, controlled by the framebuffer system + - backlight + # LED will turn on (but for leds-gpio see "default-state" property in + # Documentation/devicetree/bindings/leds/leds-gpio.yaml) + - default-on + # LED "double" flashes at a load average based rate + - heartbeat + # LED indicates disk activity + - disk-activity + # LED indicates IDE disk activity (deprecated), in new implementations + # use "disk-activity" + - ide-disk + # LED flashes at a fixed, configurable rate + - timer + # LED alters the brightness for the specified duration with one software + # timer (requires "led-pattern" property) + - pattern + # LED is triggered by SD/MMC activity + - pattern: "^mmc[0-9]+$" led-pattern: description: | diff --git a/Documentation/devicetree/bindings/leds/mediatek,mt6370-flashlight.yaml b/Documentation/devicetree/bindings/leds/mediatek,mt6370-flashlight.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9d02ed6a590c9de6fe4d17ef190ba9a6c9843dd --- /dev/null +++ b/Documentation/devicetree/bindings/leds/mediatek,mt6370-flashlight.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/mediatek,mt6370-flashlight.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Flash LED driver for MT6370 PMIC from MediaTek Integrated. + +maintainers: + - Alice Chen + +description: | + This module is part of the MT6370 MFD device. + Add MT6370 flash LED driver include 2-channel flash LED support Torch/Strobe Mode. + +properties: + compatible: + const: mediatek,mt6370-flashlight + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^led@[0-1]$": + type: object + $ref: common.yaml# + unevaluatedProperties: false + + properties: + reg: + enum: [0, 1] + +required: + - compatible + - "#address-cells" + - "#size-cells" + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/leds/mediatek,mt6370-indicator.yaml b/Documentation/devicetree/bindings/leds/mediatek,mt6370-indicator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16b3abc2af3a779d1e687402c1f4a590866b1003 --- /dev/null +++ b/Documentation/devicetree/bindings/leds/mediatek,mt6370-indicator.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/mediatek,mt6370-indicator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: LED driver for MT6370 PMIC from MediaTek Integrated. + +maintainers: + - Alice Chen + +description: | + This module is part of the MT6370 MFD device. + Add MT6370 LED driver include 4-channel RGB LED support Register/PWM/Breath Mode + +properties: + compatible: + const: mediatek,mt6370-indicator + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^multi-led@[0-3]$": + type: object + $ref: leds-class-multicolor.yaml# + unevaluatedProperties: false + + properties: + reg: + enum: [0, 1, 2, 3] + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + patternProperties: + "^led@[0-2]$": + type: object + $ref: common.yaml# + unevaluatedProperties: false + + properties: + reg: + enum: [0, 1, 2] + + required: + - reg + - color + + required: + - reg + - color + - "#address-cells" + - "#size-cells" + + "^led@[0-3]$": + type: object + $ref: common.yaml# + unevaluatedProperties: false + + properties: + reg: + enum: [0, 1, 2, 3] + + required: + - reg + - color + +required: + - compatible + - "#address-cells" + - "#size-cells" + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml b/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml index ea06976fbbc7ad809797391c19fcb57f12937336..dfd26b998189a10906dc043fc337586e5ee06227 100644 --- a/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml +++ b/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson Message-Handling-Unit Controller maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Amlogic's Meson SoCs Message-Handling-Unit (MHU) is a mailbox controller diff --git a/Documentation/devicetree/bindings/mailbox/mediatek,gce-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/mediatek,gce-mailbox.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c579ac074ca7dc7db5516b74aefb0d7738e17826 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/mediatek,gce-mailbox.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/mediatek,gce-mailbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mediatek Global Command Engine Mailbox + +maintainers: + - Houlong Wei + +description: + The Global Command Engine (GCE) is used to help read/write registers with + critical time limitation, such as updating display configuration during the + vblank. The GCE can be used to implement the Command Queue (CMDQ) driver. + +properties: + compatible: + enum: + - mediatek,mt6779-gce + - mediatek,mt8173-gce + - mediatek,mt8183-gce + - mediatek,mt8186-gce + - mediatek,mt8192-gce + - mediatek,mt8195-gce + + "#mbox-cells": + const: 2 + description: + The first cell describes the Thread ID of the GCE, + the second cell describes the priority of the GCE thread + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: Global Command Engine clock + + clock-names: + items: + - const: gce + +required: + - compatible + - "#mbox-cells" + - reg + - interrupts + - clocks + +allOf: + - if: + not: + properties: + compatible: + contains: + const: mediatek,mt8195-gce + then: + required: + - clock-names + +additionalProperties: false + +examples: + - | + #include + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + gce: mailbox@10212000 { + compatible = "mediatek,mt8173-gce"; + reg = <0 0x10212000 0 0x1000>; + interrupts = ; + #mbox-cells = <2>; + clocks = <&infracfg CLK_INFRA_GCE>; + clock-names = "gce"; + }; + }; diff --git a/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml index 082d397d3e89e1a2e22065b5f92f1e2cedc85d21..935937c67133d526e402cd5cbc5659ad8c983b46 100644 --- a/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml @@ -14,9 +14,15 @@ properties: const: microchip,mpfs-mailbox reg: - items: - - description: mailbox data registers - - description: mailbox interrupt registers + oneOf: + - items: + - description: mailbox control & data registers + - description: mailbox interrupt registers + deprecated: true + - items: + - description: mailbox control registers + - description: mailbox interrupt registers + - description: mailbox data registers interrupts: maxItems: 1 @@ -39,7 +45,8 @@ examples: #size-cells = <2>; mbox: mailbox@37020000 { compatible = "microchip,mpfs-mailbox"; - reg = <0x0 0x37020000 0x0 0x1000>, <0x0 0x2000318c 0x0 0x40>; + reg = <0x0 0x37020000 0x0 0x58>, <0x0 0x2000318C 0x0 0x40>, + <0x0 0x37020800 0x0 0x100>; interrupt-parent = <&L1>; interrupts = <96>; #mbox-cells = <1>; diff --git a/Documentation/devicetree/bindings/mailbox/mtk-gce.txt b/Documentation/devicetree/bindings/mailbox/mtk-gce.txt deleted file mode 100644 index c2aeba63bd47dd13232122cf23525101d9b291b3..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mailbox/mtk-gce.txt +++ /dev/null @@ -1,82 +0,0 @@ -MediaTek GCE -=============== - -The Global Command Engine (GCE) is used to help read/write registers with -critical time limitation, such as updating display configuration during the -vblank. The GCE can be used to implement the Command Queue (CMDQ) driver. - -CMDQ driver uses mailbox framework for communication. Please refer to -mailbox.txt for generic information about mailbox device-tree bindings. - -Required properties: -- compatible: can be "mediatek,mt8173-gce", "mediatek,mt8183-gce", - "mediatek,mt8186-gce", "mediatek,mt8192-gce", "mediatek,mt8195-gce" or - "mediatek,mt6779-gce". -- reg: Address range of the GCE unit -- interrupts: The interrupt signal from the GCE block -- clock: Clocks according to the common clock binding -- clock-names: Must be "gce" to stand for GCE clock -- #mbox-cells: Should be 2. - <&phandle channel priority> - phandle: Label name of a gce node. - channel: Channel of mailbox. Be equal to the thread id of GCE. - priority: Priority of GCE thread. - -Required properties for a client device: -- mboxes: Client use mailbox to communicate with GCE, it should have this - property and list of phandle, mailbox specifiers. -Optional properties for a client device: -- mediatek,gce-client-reg: Specify the sub-system id which is corresponding - to the register address, it should have this property and list of phandle, - sub-system specifiers. - <&phandle subsys_number start_offset size> - phandle: Label name of a gce node. - subsys_number: specify the sub-system id which is corresponding - to the register address. - start_offset: the start offset of register address that GCE can access. - size: the total size of register address that GCE can access. - -Optional properties for a client mutex node: -- mediatek,gce-events: GCE events used by clients. The event numbers are - defined in 'dt-bindings/gce/-gce.h'. - -Some vaules of properties are defined in 'dt-bindings/gce/mt8173-gce.h', -'dt-bindings/gce/mt8183-gce.h', 'dt-bindings/gce/mt8186-gce.h' -'dt-bindings/gce/mt8192-gce.h', 'dt-bindings/gce/mt8195-gce.h' or -'dt-bindings/gce/mt6779-gce.h'. -Such as sub-system ids, thread priority, event ids. - -Example: - - gce: gce@10212000 { - compatible = "mediatek,mt8173-gce"; - reg = <0 0x10212000 0 0x1000>; - interrupts = ; - clocks = <&infracfg CLK_INFRA_GCE>; - clock-names = "gce"; - #mbox-cells = <2>; - }; - -Example for a client device: - - mmsys: clock-controller@14000000 { - compatible = "mediatek,mt8173-mmsys"; - mboxes = <&gce 0 CMDQ_THR_PRIO_LOWEST>, - <&gce 1 CMDQ_THR_PRIO_LOWEST>; - mutex-event-eof = ; - mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x3000 0x1000>, - <&gce SUBSYS_1401XXXX 0x2000 0x100>; - ... - }; - -Example for a client mutex node: - mutex: mutex@14020000 { - compatible = "mediatek,mt8173-disp-mutex"; - reg = <0 0x14020000 0 0x1000>; - interrupts = ; - power-domains = <&scpsys MT8173_POWER_DOMAIN_MM>; - clocks = <&mmsys CLK_MM_MUTEX_32K>; - mediatek,gce-events = , - ; - }; diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml index f504652fc0ea2cb449a75f141f861ebee72de49f..f24fd84b4b05c21510d66f4a9cf256a6e0b7da3b 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -45,23 +45,17 @@ properties: clocks: description: phandles to the parent clocks of the clock driver minItems: 2 - items: - - description: primary pll parent of the clock driver - - description: auxiliary parent - - description: reference clock + maxItems: 3 '#mbox-cells': const: 1 '#clock-cells': - const: 0 + enum: [0, 1] clock-names: minItems: 2 - items: - - const: pll - - const: aux - - const: ref + maxItems: 3 required: - compatible @@ -75,8 +69,6 @@ allOf: properties: compatible: enum: - - qcom,ipq6018-apcs-apps-global - - qcom,ipq8074-apcs-apps-global - qcom,msm8916-apcs-kpss-global - qcom,msm8994-apcs-kpss-global - qcom,msm8996-apcs-hmss-global @@ -90,7 +82,13 @@ allOf: then: properties: clocks: - maxItems: 2 + items: + - description: primary pll parent of the clock driver + - description: auxiliary parent + clock-names: + items: + - const: pll + - const: aux - if: properties: compatible: @@ -99,7 +97,46 @@ allOf: then: properties: clocks: - maxItems: 3 + items: + - description: primary pll parent of the clock driver + - description: auxiliary parent + - description: reference clock + clock-names: + items: + - const: pll + - const: aux + - const: ref + - if: + properties: + compatible: + enum: + - qcom,ipq6018-apcs-apps-global + - qcom,ipq8074-apcs-apps-global + then: + properties: + clocks: + items: + - description: primary pll parent of the clock driver + - description: XO clock + clock-names: + items: + - const: pll + - const: xo + - if: + properties: + compatible: + enum: + - qcom,ipq6018-apcs-apps-global + - qcom,ipq8074-apcs-apps-global + then: + properties: + '#clock-cells': + const: 1 + else: + properties: + '#clock-cells': + const: 0 + examples: # Example apcs with msm8996 diff --git a/Documentation/devicetree/bindings/media/allegro,al5e.yaml b/Documentation/devicetree/bindings/media/allegro,al5e.yaml index 135bea94b587d5cfa7cfb1713e76e1b9f1fd3351..2899d26d690e623d196bc8241547e91faff11de9 100644 --- a/Documentation/devicetree/bindings/media/allegro,al5e.yaml +++ b/Documentation/devicetree/bindings/media/allegro,al5e.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allegro,al5e.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allegro DVT Video IP Codecs Device Tree Bindings +title: Allegro DVT Video IP Codecs maintainers: - Michael Tretter diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-csi.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-csi.yaml index 6ced9406421551c4637460a85008867d03ebd90d..617264ce477dff4c07f71f8decaf6245ee83f152 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-csi.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-csi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun4i-a10-csi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 CMOS Sensor Interface (CSI) Device Tree Bindings +title: Allwinner A10 CMOS Sensor Interface (CSI) maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-ir.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-ir.yaml index 6d8395d6bca0b914f24f0de01fe251593b2ce351..704033e21ee80242d8ba255491a0abff2db86cb8 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-ir.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-ir.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun4i-a10-ir.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Infrared Controller Device Tree Bindings +title: Allwinner A10 Infrared Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml index ee7fc3515d890f3146087f7a7b34005dba37ff70..541325f900a1d38ceed49ebea19b8792160bdc4b 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun4i-a10-video-engine.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Video Engine Device Tree Bindings +title: Allwinner A10 Video Engine maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml b/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml index 8551c4a711dccaf6293825609c50c4b8c5cd9594..f1ccca35a79034d9b7c794dc1ff6f5eed3ce9d19 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-csi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-csi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 CMOS Sensor Interface (CSI) Device Tree Bindings +title: Allwinner A31 CMOS Sensor Interface (CSI) maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-mipi-csi2.yaml index 09725ca955f63c074eb4b85c5172071deb50594a..54e15ab8a7f545a298536e53c446cb0d6f92a202 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-mipi-csi2.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun6i-a31-mipi-csi2.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun6i-a31-mipi-csi2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 MIPI CSI-2 Device Tree Bindings +title: Allwinner A31 MIPI CSI-2 maintainers: - Paul Kocialkowski diff --git a/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-de2-rotate.yaml b/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-de2-rotate.yaml index a258832d520cb0ca3bc242370b08559732b785b9..c2f292dd01ede9e1ef08540c94a3c6a578ba245c 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-de2-rotate.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-de2-rotate.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun8i-a83t-de2-rotate.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83T DE2 Rotate Device Tree Bindings +title: Allwinner A83T DE2 Rotate maintainers: - Jernej Skrabec diff --git a/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-mipi-csi2.yaml index 5b27482b5687290c002b6b0019bd62df9752a9fa..3cac68a87ad7ad8b0734933c8e82971da06519fe 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-mipi-csi2.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun8i-a83t-mipi-csi2.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun8i-a83t-mipi-csi2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83T MIPI CSI-2 Device Tree Bindings +title: Allwinner A83T MIPI CSI-2 maintainers: - Paul Kocialkowski diff --git a/Documentation/devicetree/bindings/media/allwinner,sun8i-h3-deinterlace.yaml b/Documentation/devicetree/bindings/media/allwinner,sun8i-h3-deinterlace.yaml index b80980b1908ede7048beaa651aa3aeb929f6e2cd..3ccd52164f5bbb6c5ecc3cffa9ec37673f2871d9 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun8i-h3-deinterlace.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun8i-h3-deinterlace.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/allwinner,sun8i-h3-deinterlace.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner H3 Deinterlace Device Tree Bindings +title: Allwinner H3 Deinterlace maintainers: - Jernej Skrabec diff --git a/Documentation/devicetree/bindings/media/amlogic,axg-ge2d.yaml b/Documentation/devicetree/bindings/media/amlogic,axg-ge2d.yaml index bee93bd8477152f216998d31ba400099a8cc1e95..e551be5e680e36cb06a96a73fe1fea3c32694499 100644 --- a/Documentation/devicetree/bindings/media/amlogic,axg-ge2d.yaml +++ b/Documentation/devicetree/bindings/media/amlogic,axg-ge2d.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic GE2D Acceleration Unit maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/media/amlogic,gx-vdec.yaml b/Documentation/devicetree/bindings/media/amlogic,gx-vdec.yaml index 5044c4bb94e099750912b974f0a9c7a4d4b05e25..b827edabcafaa475437b88d3d2b66e350f3a119c 100644 --- a/Documentation/devicetree/bindings/media/amlogic,gx-vdec.yaml +++ b/Documentation/devicetree/bindings/media/amlogic,gx-vdec.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Video Decoder maintainers: - - Neil Armstrong + - Neil Armstrong - Maxime Jourdan description: | diff --git a/Documentation/devicetree/bindings/media/amlogic,meson-gx-ao-cec.yaml b/Documentation/devicetree/bindings/media/amlogic,meson-gx-ao-cec.yaml index d93aea6a0258e98dc753020dfe336b27219f8f5f..8d844f4312d157e088dc5b840f094edeae1f0f4d 100644 --- a/Documentation/devicetree/bindings/media/amlogic,meson-gx-ao-cec.yaml +++ b/Documentation/devicetree/bindings/media/amlogic,meson-gx-ao-cec.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson AO-CEC Controller maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Amlogic Meson AO-CEC module is present is Amlogic SoCs and its purpose is diff --git a/Documentation/devicetree/bindings/media/exynos5-gsc.txt b/Documentation/devicetree/bindings/media/exynos5-gsc.txt deleted file mode 100644 index 1872688fa4089000be544962cfaca10e31463c35..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/media/exynos5-gsc.txt +++ /dev/null @@ -1,38 +0,0 @@ -* Samsung Exynos5 G-Scaler device - -G-Scaler is used for scaling and color space conversion on Exynos5 SoCs. - -Required properties: -- compatible: should be one of - "samsung,exynos5250-gsc" - "samsung,exynos5420-gsc" - "samsung,exynos5433-gsc" - "samsung,exynos5-gsc" (deprecated) -- reg: should contain G-Scaler physical address location and length. -- interrupts: should contain G-Scaler interrupt number - -Optional properties: -- samsung,sysreg: handle to syscon used to control the system registers to - set writeback input and destination - -Example: - -gsc_0: gsc@13e00000 { - compatible = "samsung,exynos5250-gsc"; - reg = <0x13e00000 0x1000>; - interrupts = <0 85 0>; -}; - -Aliases: -Each G-Scaler node should have a numbered alias in the aliases node, -in the form of gscN, N = 0...3. G-Scaler driver uses these aliases -to retrieve the device IDs using "of_alias_get_id()" call. - -Example: - -aliases { - gsc0 =&gsc_0; - gsc1 =&gsc_1; - gsc2 =&gsc_2; - gsc3 =&gsc_3; -}; diff --git a/Documentation/devicetree/bindings/media/mediatek,mdp3-rdma.yaml b/Documentation/devicetree/bindings/media/mediatek,mdp3-rdma.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cfc0c7d23e0688937f5e4f53a4688c2e595eaba --- /dev/null +++ b/Documentation/devicetree/bindings/media/mediatek,mdp3-rdma.yaml @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/mediatek,mdp3-rdma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Read Direct Memory Access + +maintainers: + - Matthias Brugger + - Moudy Ho + +description: | + MediaTek Read Direct Memory Access(RDMA) component used to do read DMA. + It contains one line buffer to store the sufficient pixel data, and + must be siblings to the central MMSYS_CONFIG node. + For a description of the MMSYS_CONFIG binding, see + Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.yaml + for details. + +properties: + compatible: + items: + - const: mediatek,mt8183-mdp3-rdma + + reg: + maxItems: 1 + + mediatek,gce-client-reg: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + items: + items: + - description: phandle of GCE + - description: GCE subsys id + - description: register offset + - description: register size + description: The register of client driver can be configured by gce with + 4 arguments defined in this property. Each GCE subsys id is mapping to + a client defined in the header include/dt-bindings/gce/-gce.h. + + mediatek,gce-events: + description: + The event id which is mapping to the specific hardware event signal + to gce. The event id is defined in the gce header + include/dt-bindings/gce/-gce.h of each chips. + $ref: /schemas/types.yaml#/definitions/uint32-array + + power-domains: + maxItems: 1 + + clocks: + items: + - description: RDMA clock + - description: RSZ clock + + iommus: + maxItems: 1 + + mboxes: + items: + - description: used for 1st data pipe from RDMA + - description: used for 2nd data pipe from RDMA + +required: + - compatible + - reg + - mediatek,gce-client-reg + - mediatek,gce-events + - power-domains + - clocks + - iommus + - mboxes + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + mdp3_rdma0: mdp3-rdma0@14001000 { + compatible = "mediatek,mt8183-mdp3-rdma"; + reg = <0x14001000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x1000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_RDMA0>, + <&mmsys CLK_MM_MDP_RSZ1>; + iommus = <&iommu>; + mboxes = <&gce 20 CMDQ_THR_PRIO_LOWEST>, + <&gce 21 CMDQ_THR_PRIO_LOWEST>; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek,mdp3-rsz.yaml b/Documentation/devicetree/bindings/media/mediatek,mdp3-rsz.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78f9de6192ef474297ba484c30d2028314a307b0 --- /dev/null +++ b/Documentation/devicetree/bindings/media/mediatek,mdp3-rsz.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/mediatek,mdp3-rsz.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Resizer + +maintainers: + - Matthias Brugger + - Moudy Ho + +description: | + One of Media Data Path 3 (MDP3) components used to do frame resizing. + +properties: + compatible: + items: + - enum: + - mediatek,mt8183-mdp3-rsz + + reg: + maxItems: 1 + + mediatek,gce-client-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + items: + - description: phandle of GCE + - description: GCE subsys id + - description: register offset + - description: register size + description: The register of client driver can be configured by gce with + 4 arguments defined in this property. Each GCE subsys id is mapping to + a client defined in the header include/dt-bindings/gce/-gce.h. + + mediatek,gce-events: + description: + The event id which is mapping to the specific hardware event signal + to gce. The event id is defined in the gce header + include/dt-bindings/gce/-gce.h of each chips. + $ref: /schemas/types.yaml#/definitions/uint32-array + + clocks: + minItems: 1 + +required: + - compatible + - reg + - mediatek,gce-client-reg + - mediatek,gce-events + - clocks + +additionalProperties: false + +examples: + - | + #include + #include + + mdp3_rsz0: mdp3-rsz0@14003000 { + compatible = "mediatek,mt8183-mdp3-rsz"; + reg = <0x14003000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x3000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_RSZ0>; + }; + + mdp3_rsz1: mdp3-rsz1@14004000 { + compatible = "mediatek,mt8183-mdp3-rsz"; + reg = <0x14004000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x4000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_RSZ1>; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek,mdp3-wrot.yaml b/Documentation/devicetree/bindings/media/mediatek,mdp3-wrot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0baa77198fa21777af1fc59067cbf773145bce1b --- /dev/null +++ b/Documentation/devicetree/bindings/media/mediatek,mdp3-wrot.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/mediatek,mdp3-wrot.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Write DMA with Rotation + +maintainers: + - Matthias Brugger + - Moudy Ho + +description: | + One of Media Data Path 3 (MDP3) components used to write DMA with frame rotation. + +properties: + compatible: + items: + - enum: + - mediatek,mt8183-mdp3-wrot + + reg: + maxItems: 1 + + mediatek,gce-client-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + items: + - description: phandle of GCE + - description: GCE subsys id + - description: register offset + - description: register size + description: The register of client driver can be configured by gce with + 4 arguments defined in this property. Each GCE subsys id is mapping to + a client defined in the header include/dt-bindings/gce/-gce.h. + + mediatek,gce-events: + description: + The event id which is mapping to the specific hardware event signal + to gce. The event id is defined in the gce header + include/dt-bindings/gce/-gce.h of each chips. + $ref: /schemas/types.yaml#/definitions/uint32-array + + power-domains: + maxItems: 1 + + clocks: + minItems: 1 + + iommus: + maxItems: 1 + +required: + - compatible + - reg + - mediatek,gce-client-reg + - mediatek,gce-events + - power-domains + - clocks + - iommus + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + mdp3_wrot0: mdp3-wrot0@14005000 { + compatible = "mediatek,mt8183-mdp3-wrot"; + reg = <0x14005000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x5000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_WROT0>; + iommus = <&iommu>; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml index d36fcca04cbc7249f92e4ec7c132e8e0d69a51ca..32aee09aea333af273e4c0f95e12dd0f14180271 100644 --- a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml +++ b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml @@ -20,6 +20,7 @@ properties: - mediatek,mt8173-vcodec-enc-vp8 - mediatek,mt8173-vcodec-enc - mediatek,mt8183-vcodec-enc + - mediatek,mt8188-vcodec-enc - mediatek,mt8192-vcodec-enc - mediatek,mt8195-vcodec-enc diff --git a/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml b/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml index d4e2051beeb6daedae0010a64c42456202e85f03..c4f20acdc1f8cf8fe4c14d130e1f2b49b2db2d81 100644 --- a/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml +++ b/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml @@ -57,6 +57,7 @@ properties: enum: - mediatek,mt8192-vcodec-dec - mediatek,mt8186-vcodec-dec + - mediatek,mt8188-vcodec-dec - mediatek,mt8195-vcodec-dec reg: diff --git a/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml b/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml index 052e752157b443b2e6ecfdb9d461c3d1fc0c8d34..5e8d001492cca2ebb9b2a3779756c9af2d172b50 100644 --- a/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml +++ b/Documentation/devicetree/bindings/media/mediatek-jpeg-decoder.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/mediatek-jpeg-decoder.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek JPEG Decoder Device Tree Bindings +title: MediaTek JPEG Decoder maintainers: - Xia Jiang diff --git a/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml b/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml index 4fd390c042a99dcc6eb0f5711196ed6f8f4aef7c..fc727300b4931fd90c35d6b42ced7462f4b08d68 100644 --- a/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml +++ b/Documentation/devicetree/bindings/media/mediatek-jpeg-encoder.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/mediatek-jpeg-encoder.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek JPEG Encoder Device Tree Bindings +title: MediaTek JPEG Encoder maintainers: - Xia Jiang diff --git a/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml b/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml index e8544fb2d0345275f808d8d57e27c3eb6aa715ca..b3a345fc646432724f98c7a1edd3cc30429a98fd 100644 --- a/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml +++ b/Documentation/devicetree/bindings/media/microchip,csi2dc.yaml @@ -75,6 +75,7 @@ properties: properties: port@0: $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: Input port node, single endpoint describing the input port. @@ -103,6 +104,7 @@ properties: port@1: $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: Output port node, single endpoint describing the output port. diff --git a/Documentation/devicetree/bindings/media/nxp,dw100.yaml b/Documentation/devicetree/bindings/media/nxp,dw100.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21910ff0e1c372a8123fc8348db6534b473635a6 --- /dev/null +++ b/Documentation/devicetree/bindings/media/nxp,dw100.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/nxp,dw100.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP i.MX8MP DW100 Dewarper core + +maintainers: + - Xavier Roumegue + +description: |- + The Dewarp Engine provides high-performance dewarp processing for the + correction of the distortion that is introduced in images produced by fisheye + and wide angle lenses. It is implemented with a line/tile-cache based + architecture. With configurable address mapping look up tables and per tile + processing, it successfully generates a corrected output image. + The engine can be used to perform scaling, cropping and pixel format + conversion. + +properties: + compatible: + enum: + - nxp,imx8mp-dw100 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: The AXI clock + - description: The AHB clock + + clock-names: + items: + - const: axi + - const: ahb + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - power-domains + +additionalProperties: false + +examples: + - | + #include + #include + #include + + dewarp: dwe@32e30000 { + compatible = "nxp,imx8mp-dw100"; + reg = <0x32e30000 0x10000>; + interrupts = ; + clocks = <&clk IMX8MP_CLK_MEDIA_AXI_ROOT>, + <&clk IMX8MP_CLK_MEDIA_APB_ROOT>; + clock-names = "axi", "ahb"; + power-domains = <&media_blk_ctrl IMX8MP_MEDIABLK_PD_DWE>; + }; diff --git a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml index 3cc6f42aeb76269551dd306659957d95fbe76fb6..3d9d1db37040de06edc38bc26c0983d38e8c6731 100644 --- a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml +++ b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/nxp,imx8-jpeg.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: i.MX8QXP/QM JPEG decoder/encoder Device Tree Bindings +title: i.MX8QXP/QM JPEG decoder/encoder maintainers: - Mirela Rabulea diff --git a/Documentation/devicetree/bindings/media/qcom,msm8916-venus.yaml b/Documentation/devicetree/bindings/media/qcom,msm8916-venus.yaml index 59ab16ad12f1fff0005ef7d509bfdb5743a6139d..2abb7d21c0d1d49814eaba46feafa14c72d07e52 100644 --- a/Documentation/devicetree/bindings/media/qcom,msm8916-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,msm8916-venus.yaml @@ -68,6 +68,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml b/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml index 199f45217b4a0df78b69df49836cf8fce13a1238..29d0cb6c6ebe0423db86a9fbef74f61b4b22d756 100644 --- a/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,msm8996-venus.yaml @@ -95,6 +95,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml index 90b4af2c972491c4c6b581c43109d0d9b2d9f469..42ee3f06c6bec844aa414e3e8a821e3e8377563c 100644 --- a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml @@ -86,6 +86,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml index e2874683b4d5faf395046b4bc8abfb056a47d0b9..cf361dd9de08614b885a27347682749b27972e86 100644 --- a/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml @@ -86,6 +86,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sdm660-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sdm660-venus.yaml index 33da7d3cfd3838ae96dc0b6638c6649b70b9f121..45e3f58f52bde49108d0c468561714b29b33cb44 100644 --- a/Documentation/devicetree/bindings/media/qcom,sdm660-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sdm660-venus.yaml @@ -104,6 +104,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sdm845-venus-v2.yaml b/Documentation/devicetree/bindings/media/qcom,sdm845-venus-v2.yaml index 177bf81544b1aa8a244bdca035d1f3020ee15174..8edc8a2f43a5cb01524efba0ceffe60ca7a60cda 100644 --- a/Documentation/devicetree/bindings/media/qcom,sdm845-venus-v2.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sdm845-venus-v2.yaml @@ -81,6 +81,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sdm845-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sdm845-venus.yaml index 680f37726fdf9e708843ab51f89f3d8b981ad9a7..57d503373efe8cc0f8fd18466c6695664d649fad 100644 --- a/Documentation/devicetree/bindings/media/qcom,sdm845-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sdm845-venus.yaml @@ -96,6 +96,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml index ebf8f3d866a591381068d6262c312eb2056199e3..4b7a12523dcff521a1a0d17b890dc4f3aca8cb3a 100644 --- a/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml +++ b/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml @@ -92,6 +92,7 @@ properties: video-firmware: type: object + additionalProperties: false description: | Firmware subnode is needed when the platform does not diff --git a/Documentation/devicetree/bindings/media/rc.yaml b/Documentation/devicetree/bindings/media/rc.yaml index b11d14ab89c4d5d7174cc31f13343f4bacf11f44..e732b7f3a635ce721668ee50a5dc4d7e7ebab9f8 100644 --- a/Documentation/devicetree/bindings/media/rc.yaml +++ b/Documentation/devicetree/bindings/media/rc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/rc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic Infrared Remote Controller Device Tree Bindings +title: Generic Infrared Remote Controller maintainers: - Mauro Carvalho Chehab diff --git a/Documentation/devicetree/bindings/media/renesas,vsp1.yaml b/Documentation/devicetree/bindings/media/renesas,vsp1.yaml index 990e9c1dbc4312098f5c39509dc25ebfc4efdf3e..7a8f32473852b4955b6625bc69e1f8844a8e430d 100644 --- a/Documentation/devicetree/bindings/media/renesas,vsp1.yaml +++ b/Documentation/devicetree/bindings/media/renesas,vsp1.yaml @@ -17,6 +17,7 @@ description: properties: compatible: enum: + - renesas,r9a07g044-vsp2 # RZ/G2L - renesas,vsp1 # R-Car Gen2 and RZ/G1 - renesas,vsp2 # R-Car Gen3 and RZ/G2 @@ -26,8 +27,8 @@ properties: interrupts: maxItems: 1 - clocks: - maxItems: 1 + clocks: true + clock-names: true power-domains: maxItems: 1 @@ -50,17 +51,43 @@ required: additionalProperties: false -if: - properties: - compatible: - items: - - const: renesas,vsp1 -then: - properties: - renesas,fcp: false -else: - required: - - renesas,fcp +allOf: + - if: + properties: + compatible: + contains: + const: renesas,vsp1 + then: + properties: + renesas,fcp: false + else: + required: + - renesas,fcp + + - if: + properties: + compatible: + contains: + const: renesas,r9a07g044-vsp2 + then: + properties: + clocks: + items: + - description: Main clock + - description: Register access clock + - description: Video clock + clock-names: + items: + - const: aclk + - const: pclk + - const: vclk + required: + - clock-names + else: + properties: + clocks: + maxItems: 1 + clock-names: false examples: # R8A7790 (R-Car H2) VSP1-S diff --git a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml index 3bcfb8e12333f3de59ba2a60eb0fb8bb94bcc7eb..08b02ec1675575e0cbf305fa9118c622696f83e5 100644 --- a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml +++ b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/rockchip,vdec.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip Video Decoder (VDec) Device Tree Bindings +title: Rockchip Video Decoder (VDec) maintainers: - Heiko Stuebner diff --git a/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml b/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..878397830a4ddf76c7a549c435c4593e0b8af753 --- /dev/null +++ b/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/samsung,exynos5250-gsc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Samsung Exynos SoC G-Scaler + +maintainers: + - Inki Dae + - Krzysztof Kozlowski + - Seung-Woo Kim + #include + + video-scaler@13e00000 { + compatible = "samsung,exynos5250-gsc", "samsung,exynos5-gsc"; + reg = <0x13e00000 0x1000>; + interrupts = ; + power-domains = <&pd_gsc>; + clocks = <&clock CLK_GSCL0>; + clock-names = "gscl"; + iommus = <&sysmmu_gsc0>; + }; diff --git a/Documentation/devicetree/bindings/media/ti,cal.yaml b/Documentation/devicetree/bindings/media/ti,cal.yaml index 7e078424ca4de3907be859a08713586d5875e7bb..f8e4d260d10a03a8a4251c4ad4f4c79a827360e1 100644 --- a/Documentation/devicetree/bindings/media/ti,cal.yaml +++ b/Documentation/devicetree/bindings/media/ti,cal.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/ti,cal.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Texas Instruments DRA72x CAMERA ADAPTATION LAYER (CAL) Device Tree Bindings +title: Texas Instruments DRA72x CAMERA ADAPTATION LAYER (CAL) maintainers: - Benoit Parrot diff --git a/Documentation/devicetree/bindings/media/ti,vpe.yaml b/Documentation/devicetree/bindings/media/ti,vpe.yaml index ef473f2873990620ce1d2277f969f1d9e38973d7..7fa8a367ed226335e7814e8019b75a0cdf48da23 100644 --- a/Documentation/devicetree/bindings/media/ti,vpe.yaml +++ b/Documentation/devicetree/bindings/media/ti,vpe.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/media/ti,vpe.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Texas Instruments DRA7x Video Processing Engine (VPE) Device Tree Bindings +title: Texas Instruments DRA7x Video Processing Engine (VPE) maintainers: - Benoit Parrot diff --git a/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml b/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b072c879b02de6b4c8a7c9c251320bdebeb7dbb --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/memory-controllers/brcm,brcmstb-memc-ddr.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Memory controller (MEMC) for Broadcom STB + +maintainers: + - Florian Fainelli + +properties: + compatible: + items: + - enum: + - brcm,brcmstb-memc-ddr-rev-b.1.x + - brcm,brcmstb-memc-ddr-rev-b.2.0 + - brcm,brcmstb-memc-ddr-rev-b.2.1 + - brcm,brcmstb-memc-ddr-rev-b.2.2 + - brcm,brcmstb-memc-ddr-rev-b.2.3 + - brcm,brcmstb-memc-ddr-rev-b.2.5 + - brcm,brcmstb-memc-ddr-rev-b.2.6 + - brcm,brcmstb-memc-ddr-rev-b.2.7 + - brcm,brcmstb-memc-ddr-rev-b.2.8 + - brcm,brcmstb-memc-ddr-rev-b.3.0 + - brcm,brcmstb-memc-ddr-rev-b.3.1 + - brcm,brcmstb-memc-ddr-rev-c.1.0 + - brcm,brcmstb-memc-ddr-rev-c.1.1 + - brcm,brcmstb-memc-ddr-rev-c.1.2 + - brcm,brcmstb-memc-ddr-rev-c.1.3 + - brcm,brcmstb-memc-ddr-rev-c.1.4 + - const: brcm,brcmstb-memc-ddr + + reg: + maxItems: 1 + + clock-frequency: + description: DDR PHY frequency in Hz + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + memory-controller@9902000 { + compatible = "brcm,brcmstb-memc-ddr-rev-c.1.1", "brcm,brcmstb-memc-ddr"; + reg = <0x9902000 0x600>; + clock-frequency = <2133000000>; + }; diff --git a/Documentation/devicetree/bindings/memory-controllers/fsl/imx8m-ddrc.yaml b/Documentation/devicetree/bindings/memory-controllers/fsl/imx8m-ddrc.yaml index 445e46feda69200ee05c86014501909174e13838..519b123116dc10409b912bb79adbcb2e8378aaf6 100644 --- a/Documentation/devicetree/bindings/memory-controllers/fsl/imx8m-ddrc.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/fsl/imx8m-ddrc.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: i.MX8M DDR Controller maintainers: - - Leonard Crestez + - Peng Fan description: The DDRC block is integrated in i.MX8M for interfacing with DDR based @@ -47,7 +47,8 @@ properties: - const: apb operating-points-v2: true - opp-table: true + opp-table: + type: object required: - reg diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml index 85e02854f0834740b3a6f0878bbcb1c5a4158ec5..6ccdaf99c7787e735a30f49821011973163776db 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml @@ -11,7 +11,9 @@ maintainers: properties: compatible: - const: mediatek,mt7621-memc + items: + - const: mediatek,mt7621-memc + - const: syscon reg: maxItems: 1 @@ -25,6 +27,6 @@ additionalProperties: false examples: - | memory-controller@5000 { - compatible = "mediatek,mt7621-memc"; + compatible = "mediatek,mt7621-memc", "syscon"; reg = <0x5000 0x1000>; }; diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml index 71bc5cefb49cf8742cc35cafe5ca22ab21047b44..a8fda30cccbb5b605f929230715be90021d77505 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml @@ -16,7 +16,7 @@ description: | MediaTek SMI have two generations of HW architecture, here is the list which generation the SoCs use: generation 1: mt2701 and mt7623. - generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8192 and mt8195. + generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8188, mt8192 and mt8195. There's slight differences between the two SMI, for generation 2, the register which control the iommu port is at each larb's register base. But @@ -37,6 +37,8 @@ properties: - mediatek,mt8173-smi-common - mediatek,mt8183-smi-common - mediatek,mt8186-smi-common + - mediatek,mt8188-smi-common-vdo + - mediatek,mt8188-smi-common-vpp - mediatek,mt8192-smi-common - mediatek,mt8195-smi-common-vdo - mediatek,mt8195-smi-common-vpp @@ -144,7 +146,16 @@ allOf: - const: gals0 - const: gals1 - else: # for gen2 HW that don't have gals + - if: # for gen2 HW that don't have gals + properties: + compatible: + enum: + - mediatek,mt2712-smi-common + - mediatek,mt6795-smi-common + - mediatek,mt8167-smi-common + - mediatek,mt8173-smi-common + + then: properties: clocks: minItems: 2 diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml index 59dcd163668f39edec220108954fec781fe15976..5f4ac3609887f2975b5ed8cf0d6fb73da114aed7 100644 --- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml @@ -25,6 +25,7 @@ properties: - mediatek,mt8173-smi-larb - mediatek,mt8183-smi-larb - mediatek,mt8186-smi-larb + - mediatek,mt8188-smi-larb - mediatek,mt8192-smi-larb - mediatek,mt8195-smi-larb @@ -78,6 +79,7 @@ allOf: enum: - mediatek,mt8183-smi-larb - mediatek,mt8186-smi-larb + - mediatek,mt8188-smi-larb - mediatek,mt8195-smi-larb then: @@ -111,6 +113,7 @@ allOf: - mediatek,mt2712-smi-larb - mediatek,mt6779-smi-larb - mediatek,mt8186-smi-larb + - mediatek,mt8188-smi-larb - mediatek,mt8192-smi-larb - mediatek,mt8195-smi-larb diff --git a/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml b/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e68c4306025a17a2100447c814fb66b913c68b35 --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/memory-controllers/snps,dw-umctl2-ddrc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DesignWare Universal Multi-Protocol Memory Controller + +maintainers: + - Krzysztof Kozlowski + - Manish Narani + - Michal Simek + +description: | + Synopsys DesignWare Enhanced uMCTL2 DDR Memory Controller is capable of + working with the memory devices supporting up to (LP)DDR4 protocol. It can + be equipped with SEC/DEC ECC feature if DRAM data bus width is either + 16-bits or 32-bits or 64-bits wide. + + For instance the ZynqMP DDR controller is based on the DW uMCTL2 v2.40a + controller. It has an optional SEC/DEC ECC support in 64- and 32-bits + bus width configurations. + +properties: + compatible: + oneOf: + - deprecated: true + description: Synopsys DW uMCTL2 DDR controller v3.80a + const: snps,ddrc-3.80a + - description: Synopsys DW uMCTL2 DDR controller + const: snps,dw-umctl2-ddrc + - description: Xilinx ZynqMP DDR controller v2.40a + const: xlnx,zynqmp-ddrc-2.40a + + interrupts: + description: + DW uMCTL2 DDRC IP-core provides individual IRQ signal for each event":" + ECC Corrected Error, ECC Uncorrected Error, ECC Address Protection, + Scrubber-Done signal, DFI Parity/CRC Error. Some platforms may have the + signals merged before they reach the IRQ controller or have some of them + absent in case if the corresponding feature is unavailable/disabled. + minItems: 1 + maxItems: 5 + + interrupt-names: + minItems: 1 + maxItems: 5 + oneOf: + - description: Common ECC CE/UE/Scrubber/DFI Errors IRQ + items: + - const: ecc + - description: Individual ECC CE/UE/Scrubber/DFI Errors IRQs + items: + enum: [ ecc_ce, ecc_ue, ecc_ap, ecc_sbr, dfi_e ] + + reg: + maxItems: 1 + + clocks: + description: + A standard set of the clock sources contains CSRs bus clock, AXI-ports + reference clock, DDRC core clock, Scrubber standalone clock + (synchronous to the DDRC clock). + minItems: 1 + maxItems: 4 + + clock-names: + minItems: 1 + maxItems: 4 + items: + enum: [ pclk, aclk, core, sbr ] + + resets: + description: + Each clock domain can have separate reset signal. + minItems: 1 + maxItems: 4 + + reset-names: + minItems: 1 + maxItems: 4 + items: + enum: [ prst, arst, core, sbr ] + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + + memory-controller@fd070000 { + compatible = "xlnx,zynqmp-ddrc-2.40a"; + reg = <0xfd070000 0x30000>; + + interrupt-parent = <&gic>; + interrupts = ; + interrupt-names = "ecc"; + }; + - | + #include + + memory-controller@3d400000 { + compatible = "snps,dw-umctl2-ddrc"; + reg = <0x3d400000 0x400000>; + + interrupts = <147 IRQ_TYPE_LEVEL_HIGH>, <148 IRQ_TYPE_LEVEL_HIGH>, + <149 IRQ_TYPE_LEVEL_HIGH>, <150 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "ecc_ce", "ecc_ue", "ecc_sbr", "dfi_e"; + + clocks = <&pclk>, <&aclk>, <&core_clk>, <&sbr_clk>; + clock-names = "pclk", "aclk", "core", "sbr"; + }; +... diff --git a/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml b/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml deleted file mode 100644 index f46e95704f532a62991dfe2f8ef14b483cdced2b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/memory-controllers/synopsys,ddrc-ecc.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Synopsys IntelliDDR Multi Protocol memory controller - -maintainers: - - Krzysztof Kozlowski - - Manish Narani - - Michal Simek - -description: | - The ZynqMP DDR ECC controller has an optional ECC support in 64-bit and - 32-bit bus width configurations. - - The Zynq DDR ECC controller has an optional ECC support in half-bus width - (16-bit) configuration. - - These both ECC controllers correct single bit ECC errors and detect double bit - ECC errors. - -properties: - compatible: - enum: - - snps,ddrc-3.80a - - xlnx,zynq-ddrc-a05 - - xlnx,zynqmp-ddrc-2.40a - - interrupts: - maxItems: 1 - - reg: - maxItems: 1 - -required: - - compatible - - reg - -allOf: - - if: - properties: - compatible: - contains: - enum: - - snps,ddrc-3.80a - - xlnx,zynqmp-ddrc-2.40a - then: - required: - - interrupts - else: - properties: - interrupts: false - -additionalProperties: false - -examples: - - | - memory-controller@f8006000 { - compatible = "xlnx,zynq-ddrc-a05"; - reg = <0xf8006000 0x1000>; - }; - - - | - axi { - #address-cells = <2>; - #size-cells = <2>; - - memory-controller@fd070000 { - compatible = "xlnx,zynqmp-ddrc-2.40a"; - reg = <0x0 0xfd070000 0x0 0x30000>; - interrupt-parent = <&gic>; - interrupts = <0 112 4>; - }; - }; diff --git a/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f72e2f8588abdc02dddf36eaa79a8bf51724d96 --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/memory-controllers/xlnx,zynq-ddrc-a05.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Zynq A05 DDR Memory Controller + +maintainers: + - Krzysztof Kozlowski + - Manish Narani + - Michal Simek + +description: + The Zynq DDR ECC controller has an optional ECC support in half-bus width + (16-bit) configuration. It is cappable of correcting single bit ECC errors + and detecting double bit ECC errors. + +properties: + compatible: + const: xlnx,zynq-ddrc-a05 + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + memory-controller@f8006000 { + compatible = "xlnx,zynq-ddrc-a05"; + reg = <0xf8006000 0x1000>; + }; +... diff --git a/Documentation/devicetree/bindings/mfd/allwinner,sun4i-a10-ts.yaml b/Documentation/devicetree/bindings/mfd/allwinner,sun4i-a10-ts.yaml index f591332fc4629941049ac22bd49a45b7ca631b1c..93f5065a62801a4bdee993e193a746f816bf86cd 100644 --- a/Documentation/devicetree/bindings/mfd/allwinner,sun4i-a10-ts.yaml +++ b/Documentation/devicetree/bindings/mfd/allwinner,sun4i-a10-ts.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/allwinner,sun4i-a10-ts.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Resistive Touchscreen Controller Device Tree Bindings +title: Allwinner A10 Resistive Touchscreen Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/mfd/allwinner,sun6i-a31-prcm.yaml b/Documentation/devicetree/bindings/mfd/allwinner,sun6i-a31-prcm.yaml index d131759ccaf36b2c53e3dfe2234891afda4e98a2..cf94176fe1eb60d3f42392246ea4c568d9dd7a63 100644 --- a/Documentation/devicetree/bindings/mfd/allwinner,sun6i-a31-prcm.yaml +++ b/Documentation/devicetree/bindings/mfd/allwinner,sun6i-a31-prcm.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/allwinner,sun6i-a31-prcm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 PRCM Device Tree Bindings +title: Allwinner A31 PRCM maintainers: - Chen-Yu Tsai @@ -22,6 +22,7 @@ properties: patternProperties: "^.*_(clk|rst)$": type: object + unevaluatedProperties: false properties: compatible: @@ -34,6 +35,45 @@ patternProperties: - fixed-factor-clock allOf: + - if: + properties: + compatible: + contains: + const: fixed-factor-clock + + then: + $ref: /schemas/clock/fixed-factor-clock.yaml# + + - if: + properties: + compatible: + contains: + const: allwinner,sun4i-a10-mod0-clk + + then: + properties: + "#clock-cells": + const: 0 + + # Already checked in the main schema + compatible: true + + clocks: + maxItems: 2 + + clock-output-names: + maxItems: 1 + + phandle: true + + required: + - "#clock-cells" + - compatible + - clocks + - clock-output-names + + additionalProperties: false + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/mfd/allwinner,sun8i-a23-prcm.yaml b/Documentation/devicetree/bindings/mfd/allwinner,sun8i-a23-prcm.yaml index aa5e683b236c32fe0ae43311cdd590f7f8d52fed..16c80a7eec49c065568616a2d41321316db5963c 100644 --- a/Documentation/devicetree/bindings/mfd/allwinner,sun8i-a23-prcm.yaml +++ b/Documentation/devicetree/bindings/mfd/allwinner,sun8i-a23-prcm.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/allwinner,sun8i-a23-prcm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A23 PRCM Device Tree Bindings +title: Allwinner A23 PRCM maintainers: - Chen-Yu Tsai @@ -22,6 +22,7 @@ properties: patternProperties: "^.*(clk|rst|codec).*$": type: object + unevaluatedProperties: false properties: compatible: @@ -36,6 +37,15 @@ patternProperties: - compatible allOf: + - if: + properties: + compatible: + contains: + const: fixed-factor-clock + + then: + $ref: /schemas/clock/fixed-factor-clock.yaml# + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml b/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1689b986f4417ac9e226e885dd2d56c8284259f6 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: GPL-2.0 +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/aspeed,ast2x00-scu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Aspeed System Control Unit + +description: + The Aspeed System Control Unit manages the global behaviour of the SoC, + configuring elements such as clocks, pinmux, and reset. + +maintainers: + - Joel Stanley + - Andrew Jeffery + +properties: + compatible: + items: + - enum: + - aspeed,ast2400-scu + - aspeed,ast2500-scu + - aspeed,ast2600-scu + - const: syscon + - const: simple-mfd + + reg: + maxItems: 1 + + ranges: true + + '#address-cells': + const: 1 + + '#size-cells': + const: 1 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + +patternProperties: + '^p2a-control@[0-9a-f]+$': + description: See Documentation/devicetree/bindings/misc/aspeed-p2a-ctrl.txt + type: object + + '^pinctrl(@[0-9a-f]+)?$': + oneOf: + - $ref: /schemas/pinctrl/aspeed,ast2400-pinctrl.yaml + - $ref: /schemas/pinctrl/aspeed,ast2500-pinctrl.yaml + - $ref: /schemas/pinctrl/aspeed,ast2600-pinctrl.yaml + + '^interrupt-controller@[0-9a-f]+$': + description: See Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic.txt + type: object + + '^silicon-id@[0-9a-f]+$': + description: Unique hardware silicon identifiers within the SoC + type: object + additionalProperties: false + + properties: + compatible: + items: + - enum: + - aspeed,ast2400-silicon-id + - aspeed,ast2500-silicon-id + - aspeed,ast2600-silicon-id + - const: aspeed,silicon-id + + reg: + description: + The reg should be the unique silicon id register, and not backwards + compatible one in eg. the 2600. + minItems: 1 + items: + - description: silicon id information registers + - description: unique chip id registers + +required: + - compatible + - reg + - ranges + - '#address-cells' + - '#size-cells' + - '#clock-cells' + - '#reset-cells' + +additionalProperties: false + +examples: + - | + syscon@1e6e2000 { + compatible = "aspeed,ast2400-scu", "syscon", "simple-mfd"; + reg = <0x1e6e2000 0x1a8>; + #clock-cells = <1>; + #reset-cells = <1>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x1e6e2000 0x1000>; + + silicon-id@7c { + compatible = "aspeed,ast2500-silicon-id", "aspeed,silicon-id"; + reg = <0x7c 0x4>, <0x150 0x8>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/mfd/aspeed-scu.txt b/Documentation/devicetree/bindings/mfd/aspeed-scu.txt deleted file mode 100644 index 857ee33f732962b8bd4cb6c49c96c1d9297ae34a..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mfd/aspeed-scu.txt +++ /dev/null @@ -1,48 +0,0 @@ -The Aspeed System Control Unit manages the global behaviour of the SoC, -configuring elements such as clocks, pinmux, and reset. - -Required properties: -- compatible: One of: - "aspeed,ast2400-scu", "syscon", "simple-mfd" - "aspeed,ast2500-scu", "syscon", "simple-mfd" - -- reg: contains the offset and length of the SCU memory region -- #clock-cells: should be set to <1> - the system controller is also a - clock provider -- #reset-cells: should be set to <1> - the system controller is also a - reset line provider - -Example: - -syscon: syscon@1e6e2000 { - compatible = "aspeed,ast2400-scu", "syscon", "simple-mfd"; - reg = <0x1e6e2000 0x1a8>; - #clock-cells = <1>; - #reset-cells = <1>; -}; - -Silicon ID ------------------ - -Families have unique hardware silicon identifiers within the SoC. - -Required properties: - - - compatible: "aspeed,silicon-id" or: - "aspeed,ast2400-silicon-id" or - "aspeed,ast2500-silicon-id" or - "aspeed,ast2600-silicon-id" - - - reg: offset and length of the silicon id information - optionally, a second offset and length describes the unique chip id - - The reg should be the unique silicon id register, and - not backwards compatible one in eg. the 2600. - -Example: - - -silicon-id@7c { - compatible = "aspeed,ast2500-silicon-id", "aspeed,silicon-id"; - reg = <0x7c 0x4 0x150 0x8>; -}; diff --git a/Documentation/devicetree/bindings/mfd/atmel-usart.txt b/Documentation/devicetree/bindings/mfd/atmel-usart.txt deleted file mode 100644 index a09133066affd085f4e20448127b280529aef381..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mfd/atmel-usart.txt +++ /dev/null @@ -1,98 +0,0 @@ -* Atmel Universal Synchronous Asynchronous Receiver/Transmitter (USART) - -Required properties for USART: -- compatible: Should be one of the following: - - "atmel,at91rm9200-usart" - - "atmel,at91sam9260-usart" - - "microchip,sam9x60-usart" - - "atmel,at91rm9200-dbgu", "atmel,at91rm9200-usart" - - "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart" - - "microchip,sam9x60-dbgu", "microchip,sam9x60-usart" -- reg: Should contain registers location and length -- interrupts: Should contain interrupt -- clock-names: tuple listing input clock names. - Required elements: "usart" -- clocks: phandles to input clocks. - -Required properties for USART in SPI mode: -- #size-cells : Must be <0> -- #address-cells : Must be <1> -- cs-gpios: chipselects (internal cs not supported) -- atmel,usart-mode : Must be (found in dt-bindings/mfd/at91-usart.h) - -Optional properties in serial and SPI mode: -- dma bindings for dma transfer: - - dmas: DMA specifier, consisting of a phandle to DMA controller node, - memory peripheral interface and USART DMA channel ID, FIFO configuration. - The order of DMA channels is fixed. The first DMA channel must be TX - associated channel and the second one must be RX associated channel. - Refer to dma.txt and atmel-dma.txt for details. - - dma-names: "tx" for TX channel. - "rx" for RX channel. - The order of dma-names is also fixed. The first name must be "tx" - and the second one must be "rx" as in the examples below. - -Optional properties in serial mode: -- atmel,use-dma-rx: use of PDC or DMA for receiving data -- atmel,use-dma-tx: use of PDC or DMA for transmitting data -- {rts,cts,dtr,dsr,rng,dcd}-gpios: specify a GPIO for RTS/CTS/DTR/DSR/RI/DCD line respectively. - It will use specified PIO instead of the peripheral function pin for the USART feature. - If unsure, don't specify this property. -- atmel,fifo-size: maximum number of data the RX and TX FIFOs can store for FIFO - capable USARTs. -- rs485-rts-delay, rs485-rx-during-tx, linux,rs485-enabled-at-boot-time: see rs485.txt - - compatible description: -- at91rm9200: legacy USART support -- at91sam9260: generic USART implementation for SAM9 SoCs - -Example: -- use PDC: - usart0: serial@fff8c000 { - compatible = "atmel,at91sam9260-usart"; - reg = <0xfff8c000 0x4000>; - interrupts = <7>; - clocks = <&usart0_clk>; - clock-names = "usart"; - atmel,use-dma-rx; - atmel,use-dma-tx; - rts-gpios = <&pioD 15 GPIO_ACTIVE_LOW>; - cts-gpios = <&pioD 16 GPIO_ACTIVE_LOW>; - dtr-gpios = <&pioD 17 GPIO_ACTIVE_LOW>; - dsr-gpios = <&pioD 18 GPIO_ACTIVE_LOW>; - dcd-gpios = <&pioD 20 GPIO_ACTIVE_LOW>; - rng-gpios = <&pioD 19 GPIO_ACTIVE_LOW>; - }; - -- use DMA: - usart0: serial@f001c000 { - compatible = "atmel,at91sam9260-usart"; - reg = <0xf001c000 0x100>; - interrupts = <12 4 5>; - clocks = <&usart0_clk>; - clock-names = "usart"; - atmel,use-dma-rx; - atmel,use-dma-tx; - dmas = <&dma0 2 0x3>, - <&dma0 2 0x204>; - dma-names = "tx", "rx"; - atmel,fifo-size = <32>; - }; - -- SPI mode: - #include - - spi0: spi@f001c000 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "atmel,at91rm9200-usart", "atmel,at91sam9260-usart"; - atmel,usart-mode = ; - reg = <0xf001c000 0x100>; - interrupts = <12 IRQ_TYPE_LEVEL_HIGH 5>; - clocks = <&usart0_clk>; - clock-names = "usart"; - dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(3)>, - <&dma0 2 (AT91_DMA_CFG_PER_ID(4) | AT91_DMA_CFG_FIFOCFG_ASAP)>; - dma-names = "tx", "rx"; - cs-gpios = <&pioB 3 0>; - }; diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm6318-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm6318-gpio-sysctl.yaml index afc569bc15cf8c9d9c524ae9c93c388a908f97b6..148f1da476031f540a673d7d760b00143e9f6497 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm6318-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm6318-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm6318-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM6318 GPIO System Controller Device Tree Bindings +title: Broadcom BCM6318 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm63268-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm63268-gpio-sysctl.yaml index c7771c86d7c1451686227e9e6cf8492da3985694..7e582243ea7659152912540616509d5eff6c226b 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm63268-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm63268-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm63268-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM63268 GPIO System Controller Device Tree Bindings +title: Broadcom BCM63268 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm6328-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm6328-gpio-sysctl.yaml index 33963c11ae62bc3fe7f8d47294f2bff36b6c9c4e..2230848e11c3a177dbd5f775fe03b69f02eab8fe 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm6328-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm6328-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm6328-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM6328 GPIO System Controller Device Tree Bindings +title: Broadcom BCM6328 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm6358-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm6358-gpio-sysctl.yaml index 3e44bea78b03ea1f0e9a09867d007cb5ad5c3600..c06693b6f7aa025fa1771477261067eac870323c 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm6358-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm6358-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm6358-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM6358 GPIO System Controller Device Tree Bindings +title: Broadcom BCM6358 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm6362-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm6362-gpio-sysctl.yaml index 48d14a5fe0d58d87c58a4236fb2ca448c40425dc..c560bede0e37a36d84995b2c0a3132b1f6c63acc 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm6362-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm6362-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm6362-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM6362 GPIO System Controller Device Tree Bindings +title: Broadcom BCM6362 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/brcm,bcm6368-gpio-sysctl.yaml b/Documentation/devicetree/bindings/mfd/brcm,bcm6368-gpio-sysctl.yaml index 307270b0cfed60d192943504a1d0b86c28255fae..c534f5f2404e3ed369d9bec689800f282b124a7c 100644 --- a/Documentation/devicetree/bindings/mfd/brcm,bcm6368-gpio-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/brcm,bcm6368-gpio-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/brcm,bcm6368-gpio-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM6368 GPIO System Controller Device Tree Bindings +title: Broadcom BCM6368 GPIO System Controller maintainers: - Álvaro Fernández Rojas diff --git a/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml index c24ad45cabb506aa8a1acca4aeb4812a7d0d7854..e2046f07a40e58740ed65a80382f761478bad777 100644 --- a/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml +++ b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/canaan,k210-sysctl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Canaan Kendryte K210 System Controller Device Tree Bindings +title: Canaan Kendryte K210 System Controller maintainers: - Damien Le Moal diff --git a/Documentation/devicetree/bindings/mfd/cirrus,lochnagar.yaml b/Documentation/devicetree/bindings/mfd/cirrus,lochnagar.yaml index ad285cb480c9e678e3a098c7740ec47b3dd56050..ebbedabb2bd433edf41ff2c302c756453e45ae46 100644 --- a/Documentation/devicetree/bindings/mfd/cirrus,lochnagar.yaml +++ b/Documentation/devicetree/bindings/mfd/cirrus,lochnagar.yaml @@ -144,6 +144,7 @@ properties: CODECs digital core if not being provided by an internal regulator. type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false properties: compatible: enum: @@ -161,6 +162,7 @@ properties: CODECs MICVDD. type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false properties: compatible: enum: @@ -177,6 +179,7 @@ properties: Initialisation data for the MIC1VDD supplies. type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false properties: compatible: enum: @@ -202,6 +205,7 @@ properties: Initialisation data for the MIC2VDD supplies. type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false properties: compatible: enum: @@ -228,6 +232,7 @@ properties: the CODECs analog and 1.8V digital supplies. type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false properties: compatible: enum: @@ -261,7 +266,7 @@ additionalProperties: false examples: - | - #include + #include #include i2c@e0004000 { #address-cells = <1>; diff --git a/Documentation/devicetree/bindings/mfd/dlg,da9063.yaml b/Documentation/devicetree/bindings/mfd/dlg,da9063.yaml index d71933460e9071267f9e90ad5b868ee90e106125..e8e74e91070cb9ee116cf02cc452524ac7375273 100644 --- a/Documentation/devicetree/bindings/mfd/dlg,da9063.yaml +++ b/Documentation/devicetree/bindings/mfd/dlg,da9063.yaml @@ -71,8 +71,9 @@ properties: regulators: type: object + additionalProperties: false patternProperties: - "^(ldo[1-11]|bcore[1-2]|bpro|bmem|bio|bperi)$": + "^(ldo([1-9]|1[01])|bcore([1-2]|s-merged)|b(pro|mem|io|peri)|bmem-bio-merged)$": $ref: /schemas/regulator/regulator.yaml unevaluatedProperties: false @@ -112,7 +113,7 @@ examples: }; regulators { - regulator-bcore1 { + bcore1 { regulator-name = "BCORE1"; regulator-min-microvolt = <300000>; regulator-max-microvolt = <1570000>; @@ -120,7 +121,7 @@ examples: regulator-max-microamp = <2000000>; regulator-boot-on; }; - regulator-ldo11 { + ldo11 { regulator-name = "LDO_11"; regulator-min-microvolt = <900000>; regulator-max-microvolt = <3600000>; diff --git a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml index 5e0fe3ebe1d29d0aff209770e42fc54d15e12271..acb9c54942d9cc4b6fc70e61fda4c28ee0fb5430 100644 --- a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml +++ b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml @@ -46,6 +46,7 @@ properties: adc: type: object + additionalProperties: false description: Optional hardware monitoring module properties: @@ -59,8 +60,9 @@ properties: const: 0 patternProperties: - "^channel@[0-9]+$": + "^channel@[0-9a-f]+$": type: object + additionalProperties: false description: | Properties for a single ADC which can report cooked values (i.e. temperature sensor based on thermister), raw values @@ -113,6 +115,7 @@ properties: patternProperties: "^fan-controller@[0-9a-f]+$": type: object + additionalProperties: false description: Optional fan controller properties: diff --git a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml index a3b976f101e8c637bfec7122746420bf36f04a9d..084960fd5a1fdd6c1162235369690c019ecfe1aa 100644 --- a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml +++ b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml @@ -4,10 +4,10 @@ $id: http://devicetree.org/schemas/mfd/khadas,mcu.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Khadas on-board Microcontroller Device Tree Bindings +title: Khadas on-board Microcontroller maintainers: - - Neil Armstrong + - Neil Armstrong description: | Khadas embeds a microcontroller on their VIM and Edge boards adding some diff --git a/Documentation/devicetree/bindings/mfd/maxim,max14577.yaml b/Documentation/devicetree/bindings/mfd/maxim,max14577.yaml index 52edd1bf549f6d863f5662e77ac906f23f6f515f..995e96ee7445cff7e90e41d25caf3d1737643854 100644 --- a/Documentation/devicetree/bindings/mfd/maxim,max14577.yaml +++ b/Documentation/devicetree/bindings/mfd/maxim,max14577.yaml @@ -39,6 +39,7 @@ properties: extcon: type: object + additionalProperties: false properties: compatible: enum: diff --git a/Documentation/devicetree/bindings/mfd/maxim,max77843.yaml b/Documentation/devicetree/bindings/mfd/maxim,max77843.yaml index f30f96bbff43ab85c4b5bd273ae9dcbb9227d733..2e2a2a86b57d36dbe47d9dc3edf84589e76ef16a 100644 --- a/Documentation/devicetree/bindings/mfd/maxim,max77843.yaml +++ b/Documentation/devicetree/bindings/mfd/maxim,max77843.yaml @@ -32,6 +32,7 @@ properties: motor-driver: type: object + additionalProperties: false properties: compatible: const: maxim,max77843-haptic diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt6370.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt6370.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5644882db2e8a84761b886c4c9fae777447d1213 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/mediatek,mt6370.yaml @@ -0,0 +1,280 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/mediatek,mt6370.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MT6370 SubPMIC + +maintainers: + - ChiYuan Huang + +description: | + MT6370 is a highly-integrated smart power management IC, which includes a + single cell Li-Ion/Li-Polymer switching battery charger, a USB Type-C & + Power Delivery (PD) controller, dual flash LED current sources, a RGB LED + driver, a backlight WLED driver, a display bias driver and a general LDO for + portable devices. + +properties: + compatible: + const: mediatek,mt6370 + + reg: + maxItems: 1 + + wakeup-source: true + + interrupts: + maxItems: 1 + + interrupt-controller: true + + "#interrupt-cells": + const: 1 + + adc: + type: object + description: | + Provides 9 channels for system monitoring, including VBUSDIV5 (lower + accuracy, higher measure range), VBUSDIV2 (higher accuracy, lower + measure range), VBAT, VSYS, CHG_VDDP, TS_BAT, IBUS, IBAT, and TEMP_JC. + + properties: + compatible: + const: mediatek,mt6370-adc + + "#io-channel-cells": + const: 1 + + required: + - compatible + - "#io-channel-cells" + + backlight: + type: object + $ref: /schemas/leds/backlight/mediatek,mt6370-backlight.yaml# + + charger: + type: object + $ref: /schemas/power/supply/mediatek,mt6370-charger.yaml# + + tcpc: + type: object + $ref: /schemas/usb/mediatek,mt6370-tcpc.yaml# + + indicator: + type: object + $ref: /schemas/leds/mediatek,mt6370-indicator.yaml# + + flashlight: + type: object + $ref: /schemas/leds/mediatek,mt6370-flashlight.yaml# + + regulators: + type: object + description: | + List all supported regulators, which support the control for DisplayBias + voltages and one general purpose LDO which commonly used to drive the + vibrator. + + patternProperties: + "^(dsvbst|vibldo)$": + $ref: /schemas/regulator/regulator.yaml# + type: object + unevaluatedProperties: false + + "^(dsvpos|dsvneg)$": + $ref: /schemas/regulator/regulator.yaml# + type: object + unevaluatedProperties: false + + properties: + enable-gpios: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - "#interrupt-cells" + - regulators + - adc + - backlight + - indicator + - tcpc + - charger + - flashlight + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + pmic@34 { + compatible = "mediatek,mt6370"; + reg = <0x34>; + wakeup-source; + interrupts-extended = <&gpio26 3 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + #interrupt-cells = <1>; + + mt6370_adc: adc { + compatible = "mediatek,mt6370-adc"; + #io-channel-cells = <1>; + }; + + backlight { + compatible = "mediatek,mt6370-backlight"; + mediatek,bled-channel-use = /bits/ 8 <15>; + }; + + charger { + compatible = "mediatek,mt6370-charger"; + interrupts = <68>, <48>, <6>; + interrupt-names = "uvp_d_evt", "attach_i", "mivr"; + io-channels = <&mt6370_adc MT6370_CHAN_IBUS>; + + mt6370_otg_vbus: usb-otg-vbus-regulator { + regulator-name = "mt6370-usb-otg-vbus"; + regulator-min-microvolt = <4350000>; + regulator-max-microvolt = <5800000>; + regulator-min-microamp = <500000>; + regulator-max-microamp = <3000000>; + }; + }; + + indicator { + compatible = "mediatek,mt6370-indicator"; + #address-cells = <1>; + #size-cells = <0>; + + multi-led@0 { + reg = <0>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <24000>; + #address-cells = <1>; + #size-cells = <0>; + led@0 { + reg = <0>; + color = ; + }; + led@1 { + reg = <1>; + color = ; + }; + led@2 { + reg = <2>; + color = ; + }; + }; + led@3 { + reg = <3>; + function = LED_FUNCTION_INDICATOR; + color = ; + led-max-microamp = <6000>; + }; + }; + + flashlight { + compatible = "mediatek,mt6370-flashlight"; + #address-cells = <1>; + #size-cells = <0>; + led@0 { + reg = <0>; + led-sources = <0>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <1>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1248000>; + }; + led@1 { + reg = <1>; + led-sources = <1>; + function = LED_FUNCTION_FLASH; + color = ; + function-enumerator = <2>; + led-max-microamp = <200000>; + flash-max-microamp = <500000>; + flash-max-timeout-us = <1248000>; + }; + }; + + tcpc { + compatible = "mediatek,mt6370-tcpc"; + interrupts-extended = <&gpio26 4 IRQ_TYPE_LEVEL_LOW>; + + connector { + compatible = "usb-c-connector"; + label = "USB-C"; + vbus-supply = <&mt6370_otg_vbus>; + data-role = "dual"; + power-role = "dual"; + try-power-role = "sink"; + source-pdos = ; + sink-pdos = ; + op-sink-microwatt = <10000000>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + endpoint { + remote-endpoint = <&usb_hs>; + }; + }; + port@1 { + reg = <1>; + endpoint { + remote-endpoint = <&usb_ss>; + }; + }; + port@2 { + reg = <2>; + endpoint { + remote-endpoint = <&dp_aux>; + }; + }; + }; + }; + }; + + regulators { + dsvbst { + regulator-name = "mt6370-dsv-vbst"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <6200000>; + }; + dsvpos { + regulator-name = "mt6370-dsv-vpos"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <6000000>; + regulator-boot-on; + }; + dsvneg { + regulator-name = "mt6370-dsv-vneg"; + regulator-min-microvolt = <4000000>; + regulator-max-microvolt = <6000000>; + regulator-boot-on; + }; + vibldo { + regulator-name = "mt6370-vib-ldo"; + regulator-min-microvolt = <1600000>; + regulator-max-microvolt = <4000000>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8c4812fffe26381acc0ed957114ff4aa66d78d8 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/mediatek,mt8195-scpsys.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/mediatek,mt8195-scpsys.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek System Control Processor System + +maintainers: + - MandyJH Liu + +description: + MediaTek System Control Processor System (SCPSYS) has several + power management tasks. The tasks include MTCMOS power + domain control, thermal measurement, DVFS, etc. + +properties: + compatible: + items: + - enum: + - mediatek,mt8167-scpsys + - mediatek,mt8173-scpsys + - mediatek,mt8183-scpsys + - mediatek,mt8186-scpsys + - mediatek,mt8192-scpsys + - mediatek,mt8195-scpsys + - const: syscon + - const: simple-mfd + + reg: + maxItems: 1 + + power-controller: + $ref: /schemas/power/mediatek,power-controller.yaml# + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + #include + + syscon@10006000 { + compatible = "mediatek,mt8195-scpsys", "syscon", "simple-mfd"; + reg = <0x10006000 0x100>; + + spm: power-controller { + compatible = "mediatek,mt8195-power-controller"; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + /* sample of power domain nodes */ + power-domain@MT8195_POWER_DOMAIN_PCIE_PHY { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_SSUSB_PCIE_PHY { + reg = ; + #power-domain-cells = <0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml b/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bf45a5673a4744cc21262e12ff13b940963167e --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/mscc,ocelot.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ocelot Externally-Controlled Ethernet Switch + +maintainers: + - Colin Foster + +description: | + The Ocelot ethernet switch family contains chips that have an internal CPU + (VSC7513, VSC7514) and chips that don't (VSC7511, VSC7512). All switches have + the option to be controlled externally, which is the purpose of this driver. + + The switch family is a multi-port networking switch that supports many + interfaces. Additionally, the device can perform pin control, MDIO buses, and + external GPIO expanders. + +properties: + compatible: + enum: + - mscc,vsc7512 + + reg: + maxItems: 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 1 + + spi-max-frequency: + maxItems: 1 + +patternProperties: + "^pinctrl@[0-9a-f]+$": + type: object + $ref: /schemas/pinctrl/mscc,ocelot-pinctrl.yaml + + "^gpio@[0-9a-f]+$": + type: object + $ref: /schemas/pinctrl/microchip,sparx5-sgpio.yaml + properties: + compatible: + enum: + - mscc,ocelot-sgpio + + "^mdio@[0-9a-f]+$": + type: object + $ref: /schemas/net/mscc,miim.yaml + properties: + compatible: + enum: + - mscc,ocelot-miim + +required: + - compatible + - reg + - '#address-cells' + - '#size-cells' + - spi-max-frequency + +additionalProperties: false + +examples: + - | + ocelot_clock: ocelot-clock { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + spi { + #address-cells = <1>; + #size-cells = <0>; + + soc@0 { + compatible = "mscc,vsc7512"; + spi-max-frequency = <2500000>; + reg = <0>; + #address-cells = <1>; + #size-cells = <1>; + + mdio@7107009c { + compatible = "mscc,ocelot-miim"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x7107009c 0x24>; + + sw_phy0: ethernet-phy@0 { + reg = <0x0>; + }; + }; + + mdio@710700c0 { + compatible = "mscc,ocelot-miim"; + pinctrl-names = "default"; + pinctrl-0 = <&miim1_pins>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x710700c0 0x24>; + + sw_phy4: ethernet-phy@4 { + reg = <0x4>; + }; + }; + + gpio: pinctrl@71070034 { + compatible = "mscc,ocelot-pinctrl"; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&gpio 0 0 22>; + reg = <0x71070034 0x6c>; + + sgpio_pins: sgpio-pins { + pins = "GPIO_0", "GPIO_1", "GPIO_2", "GPIO_3"; + function = "sg0"; + }; + + miim1_pins: miim1-pins { + pins = "GPIO_14", "GPIO_15"; + function = "miim"; + }; + }; + + gpio@710700f8 { + compatible = "mscc,ocelot-sgpio"; + #address-cells = <1>; + #size-cells = <0>; + bus-frequency = <12500000>; + clocks = <&ocelot_clock>; + microchip,sgpio-port-ranges = <0 15>; + pinctrl-names = "default"; + pinctrl-0 = <&sgpio_pins>; + reg = <0x710700f8 0x100>; + + sgpio_in0: gpio@0 { + compatible = "microchip,sparx5-sgpio-bank"; + reg = <0>; + gpio-controller; + #gpio-cells = <3>; + ngpios = <64>; + }; + + sgpio_out1: gpio@1 { + compatible = "microchip,sparx5-sgpio-bank"; + reg = <1>; + gpio-controller; + #gpio-cells = <3>; + ngpios = <64>; + }; + }; + }; + }; + +... + diff --git a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml index 65cbc6dee545e6f2ca9bd824011f402ed5cbe071..6a3e3ede1ede7508943f34cd9a209cd048125076 100644 --- a/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml +++ b/Documentation/devicetree/bindings/mfd/qcom,spmi-pmic.yaml @@ -33,19 +33,22 @@ properties: compatible: items: - enum: - - qcom,pm660 - - qcom,pm660l - qcom,pm6150 - qcom,pm6150l - qcom,pm6350 + - qcom,pm660 + - qcom,pm660l + - qcom,pm7250b - qcom,pm7325 - qcom,pm8004 - qcom,pm8005 - qcom,pm8009 - qcom,pm8019 + - qcom,pm8028 - qcom,pm8110 - qcom,pm8150 - qcom,pm8150b + - qcom,pm8150c - qcom,pm8150l - qcom,pm8226 - qcom,pm8350 @@ -56,6 +59,7 @@ properties: - qcom,pm8916 - qcom,pm8941 - qcom,pm8950 + - qcom,pm8953 - qcom,pm8994 - qcom,pm8998 - qcom,pma8084 @@ -64,8 +68,10 @@ properties: - qcom,pmi8962 - qcom,pmi8994 - qcom,pmi8998 + - qcom,pmk8002 - qcom,pmk8350 - qcom,pmm8155au + - qcom,pmp8074 - qcom,pmr735a - qcom,pmr735b - qcom,pms405 @@ -90,7 +96,7 @@ properties: regulators: type: object - $ref: /schemas/regulator/regulator.yaml# + $ref: /schemas/regulator/qcom,spmi-regulator.yaml# patternProperties: "^adc@[0-9a-f]+$": @@ -99,7 +105,7 @@ patternProperties: "^adc-tm@[0-9a-f]+$": type: object - $ref: /schemas/thermal/qcom-spmi-adc-tm5.yaml# + # ref depends on compatible, see allOf below "^audio-codec@[0-9a-f]+$": type: object @@ -146,6 +152,22 @@ required: - compatible - reg +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,pm8998 + then: + patternProperties: + "^adc-tm@[0-9a-f]+$": + $ref: /schemas/thermal/qcom-spmi-adc-tm-hc.yaml# + else: + patternProperties: + "^adc-tm@[0-9a-f]+$": + $ref: /schemas/thermal/qcom-spmi-adc-tm5.yaml# + additionalProperties: false examples: @@ -188,3 +210,87 @@ examples: }; }; }; + + - | + #include + #include + #include + #include + #include + + pmic@0 { + compatible = "qcom,pm6150", "qcom,spmi-pmic"; + reg = <0x0 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pon@800 { + compatible = "qcom,pm8998-pon"; + reg = <0x800>; + mode-bootloader = <0x2>; + mode-recovery = <0x1>; + + pwrkey { + compatible = "qcom,pm8941-pwrkey"; + interrupts = <0x0 0x8 0 IRQ_TYPE_EDGE_BOTH>; + debounce = <15625>; + bias-pull-up; + linux,code = ; + }; + }; + + temp-alarm@2400 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0x2400>; + interrupts = <0x0 0x24 0x0 IRQ_TYPE_EDGE_RISING>; + io-channels = <&pm6150_adc ADC5_DIE_TEMP>; + io-channel-names = "thermal"; + #thermal-sensor-cells = <0>; + }; + + pm6150_adc: adc@3100 { + compatible = "qcom,spmi-adc5"; + reg = <0x3100>; + interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>; + #address-cells = <1>; + #size-cells = <0>; + #io-channel-cells = <1>; + + adc-chan@6 { + reg = ; + label = "die_temp"; + }; + + adc-chan@4f { + reg = ; + qcom,ratiometric; + qcom,hw-settle-time = <200>; + }; + }; + + adc-tm@3500 { + compatible = "qcom,spmi-adc-tm5"; + reg = <0x3500>; + interrupts = <0x0 0x35 0x0 IRQ_TYPE_EDGE_RISING>; + #thermal-sensor-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + + charger-thermistor@0 { + reg = <0>; + io-channels = <&pm6150_adc ADC5_AMUX_THM3_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + }; + + pm6150_gpio: gpios@c000 { + compatible = "qcom,pm6150-gpio", "qcom,spmi-gpio"; + reg = <0xc000>; + gpio-controller; + gpio-ranges = <&pm6150_gpio 0 0 10>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml index 2f816fd0c9ecd71a8e05fb0f8b5afd43d361f798..b12809b5cc22efcf2757ed953b352ccb0854b897 100644 --- a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml +++ b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml @@ -15,23 +15,27 @@ description: properties: compatible: - oneOf: - - items: - - enum: - - qcom,tcsr-apq8064 - - qcom,tcsr-apq8084 - - qcom,tcsr-ipq8064 - - qcom,tcsr-mdm9615 - - qcom,tcsr-msm8660 - - qcom,tcsr-msm8916 - - qcom,tcsr-msm8953 - - qcom,tcsr-msm8960 - - qcom,tcsr-msm8974 - - const: syscon - - items: - - const: qcom,tcsr-ipq6018 - - const: syscon - - const: simple-mfd + items: + - enum: + - qcom,msm8998-tcsr + - qcom,qcs404-tcsr + - qcom,sc7180-tcsr + - qcom,sc7280-tcsr + - qcom,sdm630-tcsr + - qcom,sdm845-tcsr + - qcom,sm8150-tcsr + - qcom,tcsr-apq8064 + - qcom,tcsr-apq8084 + - qcom,tcsr-ipq6018 + - qcom,tcsr-ipq8064 + - qcom,tcsr-mdm9615 + - qcom,tcsr-msm8660 + - qcom,tcsr-msm8916 + - qcom,tcsr-msm8953 + - qcom,tcsr-msm8960 + - qcom,tcsr-msm8974 + - qcom,tcsr-msm8996 + - const: syscon reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mfd/richtek,rt5120.yaml b/Documentation/devicetree/bindings/mfd/richtek,rt5120.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f73b8b25d7d50303c8cfed043babcdc8305ec2ca --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/richtek,rt5120.yaml @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/richtek,rt5120.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Richtek RT5120 PMIC + +maintainers: + - ChiYuan Huang + +description: | + The RT5120 provides four high-efficiency buck converters and one LDO voltage + regulator. The device is targeted at providingthe processor voltage, memory, + I/O, and peripheral rails in home entertainment devices. The I2C interface is + used for dynamic voltage scaling of the processor voltage, power rails on/off + sequence control, operation mode selection. + +properties: + compatible: + enum: + - richtek,rt5120 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + "#interrupt-cells": + const: 1 + + wakeup-source: true + + richtek,enable-undervolt-hiccup: + type: boolean + description: | + If used, under voltage protection trigger hiccup behavior, else latchup as + default + + richtek,enable-overvolt-hiccup: + type: boolean + description: + Like as 'enable-uv-hiccup', it configures over voltage protection to + hiccup, else latchup as default + + vin1-supply: + description: phandle for buck1 input power source + + vin2-supply: + description: phandle for buck2 input power source + + vin3-supply: + description: phandle for buck3 input power source + + vin4-supply: + description: phandle for buck4 input power source + + vinldo-supply: + description: phandle for ldo input power source + + regulators: + type: object + + patternProperties: + "^buck[1-4]$": + type: object + $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false + + properties: + regulator-allowed-modes: + description: | + Used to specify the allowed buck converter operating mode + mode mapping: + 0: auto mode + 1: force pwm mode + items: + enum: [0, 1] + + "^(ldo|exten)$": + type: object + $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false + + additionalProperties: false + + powerkey: + type: object + description: + PON key that connected to RT5120 PMIC. + + properties: + compatible: + enum: + - richtek,rt5120-pwrkey + + required: + - compatible + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + - '#interrupt-cells' + - interrupt-controller + - regulators + - powerkey + +additionalProperties: false + +examples: + - | + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + pmic@62 { + compatible = "richtek,rt5120"; + reg = <0x62>; + interrupts-extended = <&gpio_intc 32 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + #interrupt-cells = <1>; + wakeup-source; + + regulators { + buck1 { + regulator-name = "rt5120-buck1"; + regulator-min-microvolt = <600000>; + regulator-max-microvolt = <1393750>; + regulator-allowed-modes = <0 1>; + regulator-boot-on; + }; + buck2 { + regulator-name = "rt5120-buck2"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + regulator-allowed-modes = <0 1>; + regulator-always-on; + }; + buck3 { + regulator-name = "rt5120-buck3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-allowed-modes = <0 1>; + regulator-always-on; + }; + buck4 { + regulator-name = "rt5120-buck4"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-allowed-modes = <0 1>; + regulator-always-on; + }; + ldo { + regulator-name = "rt5120-ldo"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + exten { + regulator-name = "rt5120-exten"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + }; + }; + powerkey { + compatible = "richtek,rt5120-pwrkey"; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/rockchip,rk817.yaml b/Documentation/devicetree/bindings/mfd/rockchip,rk817.yaml index bfc1720adc43b17bfe79d4769c13b06092f5e1fb..935e17099213febbe712abb1bc480a3ce493dc27 100644 --- a/Documentation/devicetree/bindings/mfd/rockchip,rk817.yaml +++ b/Documentation/devicetree/bindings/mfd/rockchip,rk817.yaml @@ -87,6 +87,7 @@ properties: patternProperties: "^(LDO_REG[1-9]|DCDC_REG[1-4]|BOOST|OTG_SWITCH)$": type: object + unevaluatedProperties: false $ref: ../regulator/regulator.yaml# unevaluatedProperties: false @@ -111,12 +112,56 @@ properties: additional properties are required for the codec, this node can be omitted. type: object + additionalProperties: false properties: rockchip,mic-in-differential: type: boolean description: Describes if the microphone uses differential mode. + charger: + description: | + The child node for the charger to hold additional properties. If a + battery is not in use, this node can be omitted. + type: object + properties: + monitored-battery: + description: | + A phandle to a monitored battery node that contains a valid + value for: + charge-full-design-microamp-hours, + charge-term-current-microamp, + constant-charge-current-max-microamp, + constant-charge-voltage-max-microvolt, + voltage-max-design-microvolt, + voltage-min-design-microvolt, + and a valid ocv-capacity table. + + rockchip,resistor-sense-micro-ohms: + description: | + Value in microohms of the battery sense resistor. This value is + used by the driver to set the correct divisor value to translate + ADC readings into the proper units of measure. + enum: [10000, 20000] + + rockchip,sleep-enter-current-microamp: + description: | + Value in microamps of the sleep enter current for the charger. + Value is used by the driver to calibrate the relax threshold. + + rockchip,sleep-filter-current-microamp: + description: + Value in microamps of the sleep filter current for the charger. + Value is used by the driver to derive the sleep sample current. + + required: + - monitored-battery + - rockchip,resistor-sense-micro-ohms + - rockchip,sleep-enter-current-microamp + - rockchip,sleep-filter-current-microamp + + additionalProperties: false + allOf: - if: properties: @@ -323,6 +368,13 @@ examples: }; }; + rk817_charger: charger { + monitored-battery = <&battery>; + rockchip,resistor-sense-micro-ohms = <10000>; + rockchip,sleep-enter-current-microamp = <300000>; + rockchip,sleep-filter-current-microamp = <100000>; + }; + rk817_codec: codec { rockchip,mic-in-differential; }; diff --git a/Documentation/devicetree/bindings/mfd/silergy,sy7636a.yaml b/Documentation/devicetree/bindings/mfd/silergy,sy7636a.yaml index 6de74c70163569497d931f65a53d4b72d71de5c6..ee0be32ac0204d5dd76d2d31358b5501a3976b92 100644 --- a/Documentation/devicetree/bindings/mfd/silergy,sy7636a.yaml +++ b/Documentation/devicetree/bindings/mfd/silergy,sy7636a.yaml @@ -42,6 +42,7 @@ properties: vcom: type: object $ref: /schemas/regulator/regulator.yaml# + unevaluatedProperties: false description: The regulator for the compenstation voltage. Enabling/disabling this enables/disables the entire device. diff --git a/Documentation/devicetree/bindings/mfd/sprd,ums512-glbreg.yaml b/Documentation/devicetree/bindings/mfd/sprd,ums512-glbreg.yaml new file mode 100644 index 0000000000000000000000000000000000000000..996bd4a17ca357ee5b6b8f8e740bf89eb955c9eb --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/sprd,ums512-glbreg.yaml @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright 2022 Unisoc Inc. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/sprd,ums512-glbreg.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Unisoc System Global Register + +maintainers: + - Orson Zhai + - Baolin Wang + - Chunyan Zhang + +description: + Unisoc system global registers provide register map + for clocks and some multimedia modules of the SoC. + +properties: + compatible: + items: + - const: sprd,ums512-glbregs + - const: syscon + - const: simple-mfd + + "#address-cells": + const: 1 + + "#size-cells": + const: 1 + + ranges: + maxItems: 1 + + reg: + maxItems: 1 + +patternProperties: + "^clock-controller@[0-9a-f]+$": + type: object + $ref: /schemas/clock/sprd,ums512-clk.yaml# + description: + Clock controller for the SoC clocks. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + ap_apb_regs: syscon@71000000 { + compatible = "sprd,ums512-glbregs", "syscon", "simple-mfd"; + reg = <0x71000000 0x3000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x71000000 0x3000>; + + clock-controller@0 { + compatible = "sprd,ums512-apahb-gate"; + reg = <0x0 0x2000>; + #clock-cells = <1>; + }; + }; + + - | + ap_intc5_regs: syscon@32360000 { + compatible = "sprd,ums512-glbregs", "syscon", "simple-mfd"; + reg = <0x32360000 0x1000>; + }; diff --git a/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml b/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml index a58f08aa430d5412254a9cd11b7b799737421eae..d950dd5d48bd632129991be2134cd328b92ecca8 100644 --- a/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml +++ b/Documentation/devicetree/bindings/mfd/st,stm32-lptimer.yaml @@ -46,6 +46,7 @@ properties: pwm: type: object + additionalProperties: false properties: compatible: @@ -60,6 +61,7 @@ properties: counter: type: object + additionalProperties: false properties: compatible: @@ -70,6 +72,7 @@ properties: timer: type: object + additionalProperties: false properties: compatible: @@ -81,6 +84,7 @@ properties: patternProperties: "^trigger@[0-9]+$": type: object + additionalProperties: false properties: compatible: diff --git a/Documentation/devicetree/bindings/mfd/st,stm32-timers.yaml b/Documentation/devicetree/bindings/mfd/st,stm32-timers.yaml index 5db00af8e116659b8956360ccef4c03a1fd66ef9..e2c3c3b44abba45bd1d8843e70499aa6ac3a6a50 100644 --- a/Documentation/devicetree/bindings/mfd/st,stm32-timers.yaml +++ b/Documentation/devicetree/bindings/mfd/st,stm32-timers.yaml @@ -69,6 +69,7 @@ properties: pwm: type: object + additionalProperties: false properties: compatible: @@ -104,6 +105,7 @@ properties: counter: type: object + additionalProperties: false properties: compatible: @@ -115,6 +117,7 @@ properties: patternProperties: "^timer@[0-9]+$": type: object + additionalProperties: false properties: compatible: diff --git a/Documentation/devicetree/bindings/mfd/st,stmfx.yaml b/Documentation/devicetree/bindings/mfd/st,stmfx.yaml index b2a4e4aa7ff6537fd1de503d963384b9cb460220..b4d54302582fbe17eff79930f0cb57ff9c4000b3 100644 --- a/Documentation/devicetree/bindings/mfd/st,stmfx.yaml +++ b/Documentation/devicetree/bindings/mfd/st,stmfx.yaml @@ -57,6 +57,7 @@ properties: patternProperties: "^[a-zA-Z]*-pins$": type: object + additionalProperties: false allOf: - $ref: ../pinctrl/pinmux-node.yaml diff --git a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml index 623a4b5cd27a86ce9af07593ae271154327ea048..6c8d42f27fe8a05dc485f9c1fb99459ee06f6385 100644 --- a/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml +++ b/Documentation/devicetree/bindings/mfd/stericsson,ab8500.yaml @@ -51,6 +51,7 @@ properties: provides the reference clock for the entire U8500 system and the DB8500 counterpart. type: object + additionalProperties: false properties: compatible: @@ -63,6 +64,7 @@ properties: description: Node describing the AB8500 GPIO controller. A few GPIO pins available for misc usage. type: object + additionalProperties: false properties: compatible: @@ -78,6 +80,7 @@ properties: rtc: description: Node describing the AB8500 battery-backed RTC. type: object + additionalProperties: false properties: compatible: @@ -337,34 +340,40 @@ properties: description: The voltage for the auxilary LDO regulator 1 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_aux2: description: The voltage for the auxilary LDO regulator 2 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_aux3: description: The voltage for the auxilary LDO regulator 3 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_aux4: description: The voltage for the auxilary LDO regulator 4 only present on AB8505 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_aux5: description: The voltage for the auxilary LDO regulator 5 only present on AB8505 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_aux6: description: The voltage for the auxilary LDO regulator 6 only present on AB8505 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false # There is never any AUX7 regulator which is confusing @@ -373,18 +382,21 @@ properties: only present on AB8505 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_intcore: description: The LDO regulator for the internal core voltage of the AB8500 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_adc: description: Analog power regulator for the analog to digital converter ADC, only present on AB8505 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_tvout: description: The voltage for the TV output regulator, incidentally @@ -393,33 +405,39 @@ properties: Only present on AB8500. type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_audio: description: The LDO regulator for the audio codec output type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_anamic1: description: The LDO regulator for the analog microphone 1 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_anamic2: description: The LDO regulator for the analog microphone 2 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_dmic: description: The LDO regulator for the digital microphone only present on AB8500 type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ldo_ana: description: Analog power regulator for CSI and DSI interfaces, Camera Serial Interface CSI and Display Serial Interface DSI. type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false required: - compatible @@ -442,16 +460,19 @@ properties: description: The voltage for the VSMPS1 external regulator type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ext2: description: The voltage for the VSMPS2 external regulator type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false ab8500_ext3: description: The voltage for the VSMPS3 external regulator type: object $ref: ../regulator/regulator.yaml# + unevaluatedProperties: false required: - compatible @@ -462,6 +483,7 @@ patternProperties: "^pwm@[1-9]+?$": type: object $ref: ../pwm/pwm.yaml# + unevaluatedProperties: false description: Represents each of the PWM blocks in the AB8500 properties: diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml index c10f0b577268e9c2510210fa676798085e95ab8e..4e4baf53796dea0b21981890667d65268a8da31f 100644 --- a/Documentation/devicetree/bindings/mfd/syscon.yaml +++ b/Documentation/devicetree/bindings/mfd/syscon.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/syscon.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: System Controller Registers R/W Device Tree Bindings +title: System Controller Registers R/W description: | System controller node represents a register region containing a set @@ -40,6 +40,8 @@ properties: - allwinner,sun50i-a64-system-controller - brcm,cru-clkset - freecom,fsg-cs2-system-controller + - fsl,imx93-aonmix-ns-syscfg + - fsl,imx93-wakeupmix-syscfg - hisilicon,dsa-subctrl - hisilicon,hi6220-sramctrl - hisilicon,pcie-sas-subctrl @@ -59,6 +61,8 @@ properties: - rockchip,rk3368-qos - rockchip,rk3399-qos - rockchip,rk3568-qos + - rockchip,rk3588-qos + - rockchip,rv1126-qos - samsung,exynos3-sysreg - samsung,exynos4-sysreg - samsung,exynos5-sysreg @@ -71,7 +75,7 @@ properties: - contains: const: syscon minItems: 2 - maxItems: 4 # Should be enough + maxItems: 5 # Should be enough reg: maxItems: 1 @@ -80,7 +84,6 @@ properties: description: | The size (in bytes) of the IO accesses that should be performed on the device. - $ref: /schemas/types.yaml#/definitions/uint32 enum: [1, 2, 4, 8] hwlocks: @@ -92,6 +95,18 @@ required: - compatible - reg +allOf: + - if: + properties: + compatible: + contains: + const: simple-mfd + then: + properties: + compatible: + minItems: 3 + maxItems: 5 + additionalProperties: true examples: diff --git a/Documentation/devicetree/bindings/mfd/ti,j721e-system-controller.yaml b/Documentation/devicetree/bindings/mfd/ti,j721e-system-controller.yaml index 73cffc45e0563d44a4140e5e786680adada03697..873ee0c0973f1195691c99af6e8fc30ab5d44802 100644 --- a/Documentation/devicetree/bindings/mfd/ti,j721e-system-controller.yaml +++ b/Documentation/devicetree/bindings/mfd/ti,j721e-system-controller.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/mfd/ti,j721e-system-controller.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: TI J721e System Controller Registers R/W Device Tree Bindings +title: TI J721e System Controller Registers R/W description: | This represents the Control Module registers (CTRL_MMR0) on the SoC. @@ -54,6 +54,12 @@ patternProperties: description: Clock provider for TI EHRPWM nodes. + "phy@[0-9a-f]+$": + type: object + $ref: /schemas/phy/ti,phy-gmii-sel.yaml# + description: + The phy node corresponding to the ethernet MAC. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/mfd/ti,tps65086.yaml b/Documentation/devicetree/bindings/mfd/ti,tps65086.yaml index 6aeedda3be157ac15c6dbaffef2c9a5c984c032a..3fdd9cb5b3477643e3ac828f0a248434216e9657 100644 --- a/Documentation/devicetree/bindings/mfd/ti,tps65086.yaml +++ b/Documentation/devicetree/bindings/mfd/ti,tps65086.yaml @@ -38,6 +38,7 @@ properties: regulators: type: object + additionalProperties: false description: | List of child nodes that specify the regulator initialization data. Child nodes must be named after their hardware counterparts: diff --git a/Documentation/devicetree/bindings/mfd/x-powers,ac100.yaml b/Documentation/devicetree/bindings/mfd/x-powers,ac100.yaml index de330c9869ffbb97b4e0df2b1ae67b775873a8c1..309606d2d8067b40095cfcccfaca69f0d672da48 100644 --- a/Documentation/devicetree/bindings/mfd/x-powers,ac100.yaml +++ b/Documentation/devicetree/bindings/mfd/x-powers,ac100.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/mfd/x-powers,ac100.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: X-Powers AC100 Device Tree Bindings +title: X-Powers AC100 maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/mfd/x-powers,axp152.yaml b/Documentation/devicetree/bindings/mfd/x-powers,axp152.yaml index 3a53bae611bcd1566604da8b659ddc9c0d494cfd..b7a8747d5fa0e00a25ecf4b484032a3f02c36010 100644 --- a/Documentation/devicetree/bindings/mfd/x-powers,axp152.yaml +++ b/Documentation/devicetree/bindings/mfd/x-powers,axp152.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mfd/x-powers,axp152.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: X-Powers AXP PMIC Device Tree Bindings +title: X-Powers AXP PMIC maintainers: - Chen-Yu Tsai @@ -92,6 +92,9 @@ properties: - x-powers,axp806 - x-powers,axp809 - x-powers,axp813 + - items: + - const: x-powers,axp228 + - const: x-powers,axp221 - items: - const: x-powers,axp805 - const: x-powers,axp806 @@ -260,6 +263,7 @@ properties: "^(([a-f])?ldo[0-9]|dcdc[0-7a-e]|ldo(_|-)io(0|1)|(dc1)?sw|rtc(_|-)ldo|drivevbus|dc5ldo)$": $ref: /schemas/regulator/regulator.yaml# type: object + unevaluatedProperties: false properties: regulator-ramp-delay: diff --git a/Documentation/devicetree/bindings/mips/cpu_irq.txt b/Documentation/devicetree/bindings/mips/cpu_irq.txt deleted file mode 100644 index f080f06da6d89d8dc4c4751b5e8be4a392f9c7cd..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mips/cpu_irq.txt +++ /dev/null @@ -1,47 +0,0 @@ -MIPS CPU interrupt controller - -On MIPS the mips_cpu_irq_of_init() helper can be used to initialize the 8 CPU -IRQs from a devicetree file and create a irq_domain for IRQ controller. - -With the irq_domain in place we can describe how the 8 IRQs are wired to the -platforms internal interrupt controller cascade. - -Below is an example of a platform describing the cascade inside the devicetree -and the code used to load it inside arch_init_irq(). - -Required properties: -- compatible : Should be "mti,cpu-interrupt-controller" - -Example devicetree: - cpu-irq: cpu-irq { - #address-cells = <0>; - - interrupt-controller; - #interrupt-cells = <1>; - - compatible = "mti,cpu-interrupt-controller"; - }; - - intc: intc@200 { - compatible = "ralink,rt2880-intc"; - reg = <0x200 0x100>; - - interrupt-controller; - #interrupt-cells = <1>; - - interrupt-parent = <&cpu-irq>; - interrupts = <2>; - }; - - -Example platform irq.c: -static struct of_device_id __initdata of_irq_ids[] = { - { .compatible = "mti,cpu-interrupt-controller", .data = mips_cpu_irq_of_init }, - { .compatible = "ralink,rt2880-intc", .data = intc_of_init }, - {}, -}; - -void __init arch_init_irq(void) -{ - of_irq_init(of_irq_ids); -} diff --git a/Documentation/devicetree/bindings/mips/ralink.yaml b/Documentation/devicetree/bindings/mips/ralink.yaml index 0588cee25ae9f99e88f69d0fce84413b4826266b..704b5b59512718ba9076b583e6de876e5159e92e 100644 --- a/Documentation/devicetree/bindings/mips/ralink.yaml +++ b/Documentation/devicetree/bindings/mips/ralink.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mips/ralink.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Ralink SoC based Platforms Device Tree Bindings +title: Ralink SoC based Platforms maintainers: - Sergio Paracuellos diff --git a/Documentation/devicetree/bindings/mips/realtek-rtl.yaml b/Documentation/devicetree/bindings/mips/realtek-rtl.yaml index aadff8ce0f49c7e59b310ea83eb5ff93834efcaf..f8ac309d2994c91240ef390544f5ab173dc1056c 100644 --- a/Documentation/devicetree/bindings/mips/realtek-rtl.yaml +++ b/Documentation/devicetree/bindings/mips/realtek-rtl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mips/realtek-rtl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Realtek RTL83xx/93xx SoC series device tree bindings +title: Realtek RTL83xx/93xx SoC series maintainers: - Bert Vermeulen diff --git a/Documentation/devicetree/bindings/misc/qcom,fastrpc.txt b/Documentation/devicetree/bindings/misc/qcom,fastrpc.txt deleted file mode 100644 index 5ec124b138a68f8b20873e11a0bd3dbff93a092b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/misc/qcom,fastrpc.txt +++ /dev/null @@ -1,88 +0,0 @@ -Qualcomm Technologies, Inc. FastRPC Driver - -The FastRPC implements an IPC (Inter-Processor Communication) -mechanism that allows for clients to transparently make remote method -invocations across DSP and APPS boundaries. This enables developers -to offload tasks to the DSP and free up the application processor for -other tasks. - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,fastrpc" - -- label - Usage: required - Value type: - Definition: should specify the dsp domain name this fastrpc - corresponds to. must be one of this: "adsp", "mdsp", "sdsp", "cdsp" - -- qcom,non-secure-domain: - Usage: required - Value type: - Definition: Property to specify that dsp domain is non-secure. - -- qcom,vmids: - Usage: optional - Value type: - Definition: Virtual machine IDs for remote processor. - -- #address-cells - Usage: required - Value type: - Definition: Must be 1 - -- #size-cells - Usage: required - Value type: - Definition: Must be 0 - -= COMPUTE BANKS -Each subnode of the Fastrpc represents compute context banks available -on the dsp. -- All Compute context banks MUST contain the following properties: - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,fastrpc-compute-cb" - -- reg - Usage: required - Value type: - Definition: Context Bank ID. - -- qcom,nsessions: - Usage: Optional - Value type: - Defination: A value indicating how many sessions can share this - context bank. Defaults to 1 when this property - is not specified. - -Example: - -adsp-pil { - compatible = "qcom,msm8996-adsp-pil"; - ... - smd-edge { - label = "lpass"; - fastrpc { - compatible = "qcom,fastrpc"; - qcom,smd-channels = "fastrpcsmd-apps-dsp"; - label = "adsp"; - #address-cells = <1>; - #size-cells = <0>; - - cb@1 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <1>; - }; - - cb@2 { - compatible = "qcom,fastrpc-compute-cb"; - reg = <2>; - }; - ... - }; - }; -}; diff --git a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7576f8ac94b436df0670097a467d0c7c5a7bc0b --- /dev/null +++ b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/misc/qcom,fastrpc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm FastRPC Driver + +maintainers: + - Srinivas Kandagatla + +description: | + The FastRPC implements an IPC (Inter-Processor Communication) + mechanism that allows for clients to transparently make remote method + invocations across DSP and APPS boundaries. This enables developers + to offload tasks to the DSP and free up the application processor for + other tasks. + +properties: + compatible: + const: qcom,fastrpc + + label: + enum: + - adsp + - mdsp + - sdsp + - cdsp + + memory-region: + maxItems: 1 + description: + Phandle to a node describing memory to be used for remote heap CMA. + + qcom,glink-channels: + description: + A list of channels tied to this function, used for matching + the function to a set of virtual channels. + $ref: "/schemas/types.yaml#/definitions/string-array" + items: + - const: fastrpcglink-apps-dsp + + qcom,non-secure-domain: + description: + Used to mark the current domain as non-secure. + type: boolean + + qcom,smd-channels: + description: + Channel name used for the RPM communication + $ref: "/schemas/types.yaml#/definitions/string-array" + items: + - const: fastrpcsmd-apps-dsp + + qcom,vmids: + description: + Virtual machine IDs for remote processor. + $ref: "/schemas/types.yaml#/definitions/uint32-array" + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "(compute-)?cb@[0-9]*$": + type: object + + description: > + Each subnode of the Fastrpc represents compute context banks available on the dsp. + + properties: + compatible: + const: qcom,fastrpc-compute-cb + + reg: + maxItems: 1 + + iommus: + minItems: 1 + maxItems: 2 + + qcom,nsessions: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 1 + description: > + A value indicating how many sessions can share this context bank. + + required: + - compatible + - reg + + additionalProperties: false + +required: + - compatible + - label + - "#address-cells" + - "#size-cells" + +additionalProperties: false + +examples: + - | + #include + #include + + glink-edge { + interrupts-extended = <&ipcc IPCC_CLIENT_LPASS + IPCC_MPROC_SIGNAL_GLINK_QMP + IRQ_TYPE_EDGE_RISING>; + mboxes = <&ipcc IPCC_CLIENT_LPASS + IPCC_MPROC_SIGNAL_GLINK_QMP>; + label = "lpass"; + qcom,remote-pid = <2>; + + fastrpc { + compatible = "qcom,fastrpc"; + qcom,glink-channels = "fastrpcglink-apps-dsp"; + label = "sdsp"; + qcom,non-secure-domain; + #address-cells = <1>; + #size-cells = <0>; + + compute-cb@1 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <1>; + iommus = <&apps_smmu 0x0541 0x0>; + }; + + compute-cb@2 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <2>; + iommus = <&apps_smmu 0x0542 0x0>; + }; + + compute-cb@3 { + compatible = "qcom,fastrpc-compute-cb"; + reg = <3>; + iommus = <&apps_smmu 0x0543 0x0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml index 7803597b63665f012f1cdd131c2bb452980e4a15..02ecc93417efaa5dc07a0f6c811f1b4b0940ee8a 100644 --- a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml +++ b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mmc/allwinner,sun4i-a10-mmc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 MMC Controller Device Tree Bindings +title: Allwinner A10 MMC Controller allOf: - $ref: "mmc-controller.yaml" diff --git a/Documentation/devicetree/bindings/mmc/amlogic,meson-mx-sdhc.yaml b/Documentation/devicetree/bindings/mmc/amlogic,meson-mx-sdhc.yaml index 60955acb8e578469ed2fb7cc65fbd3498cbde659..1c391bec43dcd3aee25d7437a44bdb806abfda8e 100644 --- a/Documentation/devicetree/bindings/mmc/amlogic,meson-mx-sdhc.yaml +++ b/Documentation/devicetree/bindings/mmc/amlogic,meson-mx-sdhc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mmc/amlogic,meson-mx-sdhc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Amlogic Meson SDHC controller Device Tree Bindings +title: Amlogic Meson SDHC controller allOf: - $ref: "mmc-controller.yaml" diff --git a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml index 4207fed62dfee5fcf862db1e965e84cd2fcde8d4..8b1a0fdcb5e3e2e8b87d8d7678e37f3dad447fc1 100644 --- a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml @@ -8,7 +8,6 @@ title: Cadence SD/SDIO/eMMC Host Controller (SD4HC) maintainers: - Masahiro Yamada - - Piotr Sroka allOf: - $ref: mmc-controller.yaml diff --git a/Documentation/devicetree/bindings/mmc/mmc-card.yaml b/Documentation/devicetree/bindings/mmc/mmc-card.yaml index b17d454442b363eae118899eaffdf9116f5b5f37..fd347126449ac0119344d1312af5f9c7135c9c82 100644 --- a/Documentation/devicetree/bindings/mmc/mmc-card.yaml +++ b/Documentation/devicetree/bindings/mmc/mmc-card.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mmc/mmc-card.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MMC Card / eMMC Generic Device Tree Bindings +title: MMC Card / eMMC Generic maintainers: - Ulf Hansson diff --git a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml index ff5ce89e511149d9efc95b0fbd0540feec9c3b78..802e3ca8be4df0bcb6f563d9b99b6ffed61b3fe8 100644 --- a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml +++ b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml @@ -88,11 +88,18 @@ properties: default: 1 max-frequency: - description: - Maximum operating frequency of the bus. + description: | + Maximum operating frequency of the bus: + - for eMMC, the maximum supported frequency is 200MHz, + - for SD/SDIO cards the SDR104 mode has a max supported + frequency of 208MHz, + - some mmc host controllers do support a max frequency upto + 384MHz. + So, lets keep the maximum supported value here. + $ref: /schemas/types.yaml#/definitions/uint32 minimum: 400000 - maximum: 200000000 + maximum: 384000000 disable-wp: $ref: /schemas/types.yaml#/definitions/flag diff --git a/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml b/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml index c45b91099325a325f40e72bc21fa158274bd98b4..c0662ce9946dc06935f4f9752b1bb1c2ee7f9b77 100644 --- a/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml +++ b/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml @@ -23,8 +23,6 @@ properties: reg: maxItems: 1 - spi-max-frequency: true - interrupts: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml index 083d1ec2f661cf7f26fff73b54dc2d6f7b9b9beb..d8e1e2e9adf20cc2a3fd9bbb034be85c1f2bb6b7 100644 --- a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml +++ b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml @@ -20,6 +20,7 @@ properties: - mediatek,mt2701-mmc - mediatek,mt2712-mmc - mediatek,mt6779-mmc + - mediatek,mt6795-mmc - mediatek,mt7620-mmc - mediatek,mt7622-mmc - mediatek,mt8135-mmc diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml index 14945ebc31d26292917c84cc42ddce9a84c542dd..0424b06cb6551e0042a830a8ae3b044e477381dc 100644 --- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml +++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml @@ -89,6 +89,9 @@ properties: - tx - rx + iommus: + maxItems: 1 + power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml index 8d888b435817be3add29e133a515a57df5269bdb..95f59a5e3576f2a4bf29d6145ba1cefbafe4af52 100644 --- a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml +++ b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mmc/rockchip-dw-mshc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip designware mobile storage host controller device tree bindings +title: Rockchip designware mobile storage host controller description: Rockchip uses the Synopsys designware mobile storage host controller @@ -32,6 +32,7 @@ properties: - rockchip,px30-dw-mshc - rockchip,rk1808-dw-mshc - rockchip,rk3036-dw-mshc + - rockchip,rk3128-dw-mshc - rockchip,rk3228-dw-mshc - rockchip,rk3308-dw-mshc - rockchip,rk3328-dw-mshc diff --git a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml index 0ab07759b47289c5950364b83cc2e38a310a0cdd..ea9121fb188d03da44d983798d3cd9c1c46ec5e2 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml +++ b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml @@ -51,6 +51,9 @@ properties: sdhci-caps-mask: true + dma-coherent: + type: boolean + # PHY output tap delays: # Used to delay the data valid window and align it to the sampling clock. # Binding needs to be provided for each supported speed mode otherwise the diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml index fc0e81c2066c8bd620d2109cf6e80052a9b4e00c..a96f143479c7935e61264499d8b3f97bcf89536c 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml +++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml @@ -38,9 +38,11 @@ properties: - qcom,sc7180-sdhci - qcom,sc7280-sdhci - qcom,sdm630-sdhci + - qcom,sdm670-sdhci - qcom,sdm845-sdhci - qcom,sdx55-sdhci - qcom,sdx65-sdhci + - qcom,sm6115-sdhci - qcom,sm6125-sdhci - qcom,sm6350-sdhci - qcom,sm8150-sdhci @@ -96,6 +98,10 @@ properties: description: Should specify pin control groups used for this controller. + pinctrl-1: + description: + Should specify sleep pin control groups used for this controller. + resets: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mtd/allwinner,sun4i-a10-nand.yaml b/Documentation/devicetree/bindings/mtd/allwinner,sun4i-a10-nand.yaml index c033ac3f147d2a48b2ea885150597e536ed0e1c5..4741864da48eec2ca82299e6ce5d3e348b4e73d4 100644 --- a/Documentation/devicetree/bindings/mtd/allwinner,sun4i-a10-nand.yaml +++ b/Documentation/devicetree/bindings/mtd/allwinner,sun4i-a10-nand.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/allwinner,sun4i-a10-nand.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 NAND Controller Device Tree Bindings +title: Allwinner A10 NAND Controller allOf: - $ref: "nand-controller.yaml" diff --git a/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt b/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt deleted file mode 100644 index 5794ab1147c1e41794e09808c44aa1a837458a8f..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt +++ /dev/null @@ -1,60 +0,0 @@ -Amlogic NAND Flash Controller (NFC) for GXBB/GXL/AXG family SoCs - -This file documents the properties in addition to those available in -the MTD NAND bindings. - -Required properties: -- compatible : contains one of: - - "amlogic,meson-gxl-nfc" - - "amlogic,meson-axg-nfc" -- clocks : - A list of phandle + clock-specifier pairs for the clocks listed - in clock-names. - -- clock-names: Should contain the following: - "core" - NFC module gate clock - "device" - device clock from eMMC sub clock controller - "rx" - rx clock phase - "tx" - tx clock phase - -- amlogic,mmc-syscon : Required for NAND clocks, it's shared with SD/eMMC - controller port C - -Optional children nodes: -Children nodes represent the available nand chips. - -Other properties: -see Documentation/devicetree/bindings/mtd/nand-controller.yaml for generic bindings. - -Example demonstrate on AXG SoC: - - sd_emmc_c_clkc: mmc@7000 { - compatible = "amlogic,meson-axg-mmc-clkc", "syscon"; - reg = <0x0 0x7000 0x0 0x800>; - }; - - nand-controller@7800 { - compatible = "amlogic,meson-axg-nfc"; - reg = <0x0 0x7800 0x0 0x100>; - #address-cells = <1>; - #size-cells = <0>; - interrupts = ; - - clocks = <&clkc CLKID_SD_EMMC_C>, - <&sd_emmc_c_clkc CLKID_MMC_DIV>, - <&sd_emmc_c_clkc CLKID_MMC_PHASE_RX>, - <&sd_emmc_c_clkc CLKID_MMC_PHASE_TX>; - clock-names = "core", "device", "rx", "tx"; - amlogic,mmc-syscon = <&sd_emmc_c_clkc>; - - pinctrl-names = "default"; - pinctrl-0 = <&nand_pins>; - - nand@0 { - reg = <0>; - #address-cells = <1>; - #size-cells = <1>; - - nand-on-flash-bbt; - }; - }; diff --git a/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.yaml b/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28fb9a7dd70f1ec6707378dc8dba51a4f706f8fd --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/amlogic,meson-nand.yaml @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mtd/amlogic,meson-nand.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Amlogic NAND Flash Controller (NFC) for GXBB/GXL/AXG family SoCs + +allOf: + - $ref: nand-controller.yaml + +maintainers: + - liang.yang@amlogic.com + +properties: + compatible: + enum: + - amlogic,meson-gxl-nfc + - amlogic,meson-axg-nfc + + reg: + maxItems: 2 + + reg-names: + items: + - const: nfc + - const: emmc + + interrupts: + maxItems: 1 + + clocks: + minItems: 2 + + clock-names: + items: + - const: core + - const: device + +patternProperties: + "^nand@[0-7]$": + type: object + properties: + reg: + minimum: 0 + maximum: 1 + + nand-ecc-mode: + const: hw + + nand-ecc-step-size: + const: 1024 + + nand-ecc-strength: + enum: [8, 16, 24, 30, 40, 50, 60] + description: | + The ECC configurations that can be supported are as follows. + meson-gxl-nfc 8, 16, 24, 30, 40, 50, 60 + meson-axg-nfc 8 + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +unevaluatedProperties: false + +examples: + - | + #include + #include + nand-controller@ffe07800 { + compatible = "amlogic,meson-axg-nfc"; + reg = <0xffe07800 0x100>, <0xffe07000 0x800>; + reg-names = "nfc", "emmc"; + interrupts = ; + clocks = <&clkc CLKID_SD_EMMC_C>, <&clkc CLKID_FCLK_DIV2>; + clock-names = "core", "device"; + + pinctrl-0 = <&nand_pins>; + pinctrl-names = "default"; + + #address-cells = <1>; + #size-cells = <0>; + + nand@0 { + reg = <0>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml b/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml index b32876933269d78c3b45a489e8329ab87ce27a71..f013fb976d95c9626216ed2eea450789e41370b7 100644 --- a/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml +++ b/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/arasan,nand-controller.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Arasan NAND Flash Controller with ONFI 3.1 support device tree bindings +title: Arasan NAND Flash Controller with ONFI 3.1 support allOf: - $ref: "nand-controller.yaml" diff --git a/Documentation/devicetree/bindings/mtd/arm,pl353-nand-r2p1.yaml b/Documentation/devicetree/bindings/mtd/arm,pl353-nand-r2p1.yaml index 5f126bb9b202f2dd42b6734588e4b2d1545e062d..023f3ef0fa133597b45b891a1b2659dd925001bb 100644 --- a/Documentation/devicetree/bindings/mtd/arm,pl353-nand-r2p1.yaml +++ b/Documentation/devicetree/bindings/mtd/arm,pl353-nand-r2p1.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/arm,pl353-nand-r2p1.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: PL353 NAND Controller device tree bindings +title: PL353 NAND Controller allOf: - $ref: "nand-controller.yaml" diff --git a/Documentation/devicetree/bindings/mtd/intel,lgm-nand.yaml b/Documentation/devicetree/bindings/mtd/intel,lgm-ebunand.yaml similarity index 88% rename from Documentation/devicetree/bindings/mtd/intel,lgm-nand.yaml rename to Documentation/devicetree/bindings/mtd/intel,lgm-ebunand.yaml index 30e0c66ab0eb70e9b661f44988053b8da705b73c..741c66ee06c3733575b5a07805e375d074775c28 100644 --- a/Documentation/devicetree/bindings/mtd/intel,lgm-nand.yaml +++ b/Documentation/devicetree/bindings/mtd/intel,lgm-ebunand.yaml @@ -1,10 +1,10 @@ # SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/mtd/intel,lgm-nand.yaml# +$id: http://devicetree.org/schemas/mtd/intel,lgm-ebunand.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel LGM SoC NAND Controller Device Tree Bindings +title: Intel LGM SoC NAND Controller allOf: - $ref: "nand-controller.yaml" @@ -14,7 +14,7 @@ maintainers: properties: compatible: - const: intel,lgm-nand + const: intel,lgm-ebunand reg: maxItems: 6 @@ -51,7 +51,7 @@ patternProperties: properties: reg: minimum: 0 - maximum: 7 + maximum: 1 nand-ecc-mode: true @@ -75,7 +75,7 @@ additionalProperties: false examples: - | nand-controller@e0f00000 { - compatible = "intel,lgm-nand"; + compatible = "intel,lgm-ebunand"; reg = <0xe0f00000 0x100>, <0xe1000000 0x300>, <0xe1400000 0x8000>, diff --git a/Documentation/devicetree/bindings/mtd/mtd.yaml b/Documentation/devicetree/bindings/mtd/mtd.yaml index 376b679cfc701b3e2ff355ed5bd8096f230f3045..3498e485679b1e447b87336aabf37873ff65bde7 100644 --- a/Documentation/devicetree/bindings/mtd/mtd.yaml +++ b/Documentation/devicetree/bindings/mtd/mtd.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/mtd.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MTD (Memory Technology Device) Device Tree Bindings +title: MTD (Memory Technology Device) maintainers: - Miquel Raynal diff --git a/Documentation/devicetree/bindings/mtd/mxicy,nand-ecc-engine.yaml b/Documentation/devicetree/bindings/mtd/mxicy,nand-ecc-engine.yaml index 804479999ccb590f64d31f089541d8cc169d6236..f92e7234deabbaa856053aa8250b3cb87ea8ae89 100644 --- a/Documentation/devicetree/bindings/mtd/mxicy,nand-ecc-engine.yaml +++ b/Documentation/devicetree/bindings/mtd/mxicy,nand-ecc-engine.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/mxicy,nand-ecc-engine.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Macronix NAND ECC engine device tree bindings +title: Macronix NAND ECC engine maintainers: - Miquel Raynal diff --git a/Documentation/devicetree/bindings/mtd/partitions/u-boot.yaml b/Documentation/devicetree/bindings/mtd/partitions/u-boot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a88e7d16524848ccf7301ba13e468a09fec4bef --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/partitions/u-boot.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mtd/partitions/u-boot.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: U-Boot bootloader partition + +description: | + U-Boot is a bootlodaer commonly used in embedded devices. It's almost always + located on some kind of flash device. + + Device configuration is stored as a set of environment variables that are + located in a (usually standalone) block of data. + +maintainers: + - Rafał Miłecki + +allOf: + - $ref: partition.yaml# + +properties: + compatible: + oneOf: + - const: brcm,u-boot + description: | + Broadcom stores environment variables inside a U-Boot partition. They + can be identified by a custom header with magic value. + +unevaluatedProperties: false + +examples: + - | + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + compatible = "brcm,u-boot"; + reg = <0x0 0x100000>; + label = "u-boot"; + }; + + partition@100000 { + reg = <0x100000 0x1ff00000>; + label = "firmware"; + }; + }; diff --git a/Documentation/devicetree/bindings/mtd/renesas-nandc.yaml b/Documentation/devicetree/bindings/mtd/renesas-nandc.yaml index 7b18bc5cc8b3d73bd0b113239de8da37025b777a..f0dc78bb0515ac7c5f0f3e837b94dd62f87ef97e 100644 --- a/Documentation/devicetree/bindings/mtd/renesas-nandc.yaml +++ b/Documentation/devicetree/bindings/mtd/renesas-nandc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/renesas-nandc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Renesas R-Car Gen3 & RZ/N1x NAND flash controller device tree bindings +title: Renesas R-Car Gen3 & RZ/N1x NAND flash controller maintainers: - Miquel Raynal diff --git a/Documentation/devicetree/bindings/mtd/spi-nand.yaml b/Documentation/devicetree/bindings/mtd/spi-nand.yaml index dd3cd1d530095a7f6afa49c6a5eeb3ddef0a5bee..4d095e613204c79716f19468b7dbf1f3076f1189 100644 --- a/Documentation/devicetree/bindings/mtd/spi-nand.yaml +++ b/Documentation/devicetree/bindings/mtd/spi-nand.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/mtd/spi-nand.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: SPI-NAND flash device tree bindings +title: SPI-NAND flash maintainers: - Miquel Raynal diff --git a/Documentation/devicetree/bindings/net/adi,adin1110.yaml b/Documentation/devicetree/bindings/net/adi,adin1110.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6bd8ee38a18644cc656650450e0c87c5818852e --- /dev/null +++ b/Documentation/devicetree/bindings/net/adi,adin1110.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/adi,adin1110.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ADI ADIN1110 MAC-PHY + +maintainers: + - Alexandru Tachici + +description: | + The ADIN1110 is a low power single port 10BASE-T1L MAC- + PHY designed for industrial Ethernet applications. It integrates + an Ethernet PHY core with a MAC and all the associated analog + circuitry, input and output clock buffering. + + The ADIN2111 is a low power, low complexity, two-Ethernet ports + switch with integrated 10BASE-T1L PHYs and one serial peripheral + interface (SPI) port. The device is designed for industrial Ethernet + applications using low power constrained nodes and is compliant + with the IEEE 802.3cg-2019 Ethernet standard for long reach + 10 Mbps single pair Ethernet (SPE). + + The device has a 4-wire SPI interface for communication + between the MAC and host processor. + +allOf: + - $ref: ethernet-controller.yaml# + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +properties: + compatible: + enum: + - adi,adin1110 + - adi,adin2111 + + reg: + maxItems: 1 + + adi,spi-crc: + description: | + Enable CRC8 checks on SPI read/writes. + type: boolean + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + + spi { + + #address-cells = <1>; + #size-cells = <0>; + + ethernet@0 { + compatible = "adi,adin2111"; + reg = <0>; + spi-max-frequency = <24500000>; + + adi,spi-crc; + + interrupt-parent = <&gpio>; + interrupts = <25 IRQ_TYPE_LEVEL_LOW>; + + local-mac-address = [ 00 11 22 33 44 55 ]; + }; + }; diff --git a/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-emac.yaml b/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-emac.yaml index 098b2bf7d97632d38420b5713dedc42e3e89d780..987b91b9afe99b4b7c537055f3b03c8b8b78973c 100644 --- a/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-emac.yaml +++ b/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-emac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/allwinner,sun4i-a10-emac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 EMAC Ethernet Controller Device Tree Bindings +title: Allwinner A10 EMAC Ethernet Controller allOf: - $ref: "ethernet-controller.yaml#" diff --git a/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-mdio.yaml b/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-mdio.yaml index 767193ec1d3285ec0ebe71a505c543c92c876de9..ede977cdfb8d8eec583439fd4a7019747529207c 100644 --- a/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-mdio.yaml +++ b/Documentation/devicetree/bindings/net/allwinner,sun4i-a10-mdio.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/allwinner,sun4i-a10-mdio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 MDIO Controller Device Tree Bindings +title: Allwinner A10 MDIO Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/net/allwinner,sun7i-a20-gmac.yaml b/Documentation/devicetree/bindings/net/allwinner,sun7i-a20-gmac.yaml index 703d0d8868846dde2ce14a0fafef6acd264bce28..3bd912ed7c7ead993346eadf3523156cc04d3cd2 100644 --- a/Documentation/devicetree/bindings/net/allwinner,sun7i-a20-gmac.yaml +++ b/Documentation/devicetree/bindings/net/allwinner,sun7i-a20-gmac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/allwinner,sun7i-a20-gmac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A20 GMAC Device Tree Bindings +title: Allwinner A20 GMAC allOf: - $ref: "snps,dwmac.yaml#" diff --git a/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml b/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml index 55fc620c72cd9dcfb192afae017ff4e92ecc613b..1432fda3b603f74e11d287f8a29d7bf03a042e79 100644 --- a/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml +++ b/Documentation/devicetree/bindings/net/allwinner,sun8i-a83t-emac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/allwinner,sun8i-a83t-emac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83t EMAC Device Tree Bindings +title: Allwinner A83t EMAC maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/net/altera_tse.txt b/Documentation/devicetree/bindings/net/altera_tse.txt deleted file mode 100644 index 1d9148ff51309d742deb868d40421b6e264d8ba9..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/net/altera_tse.txt +++ /dev/null @@ -1,113 +0,0 @@ -* Altera Triple-Speed Ethernet MAC driver (TSE) - -Required properties: -- compatible: Should be "altr,tse-1.0" for legacy SGDMA based TSE, and should - be "altr,tse-msgdma-1.0" for the preferred MSGDMA based TSE. - ALTR is supported for legacy device trees, but is deprecated. - altr should be used for all new designs. -- reg: Address and length of the register set for the device. It contains - the information of registers in the same order as described by reg-names -- reg-names: Should contain the reg names - "control_port": MAC configuration space region - "tx_csr": xDMA Tx dispatcher control and status space region - "tx_desc": MSGDMA Tx dispatcher descriptor space region - "rx_csr" : xDMA Rx dispatcher control and status space region - "rx_desc": MSGDMA Rx dispatcher descriptor space region - "rx_resp": MSGDMA Rx dispatcher response space region - "s1": SGDMA descriptor memory -- interrupts: Should contain the TSE interrupts and its mode. -- interrupt-names: Should contain the interrupt names - "rx_irq": xDMA Rx dispatcher interrupt - "tx_irq": xDMA Tx dispatcher interrupt -- rx-fifo-depth: MAC receive FIFO buffer depth in bytes -- tx-fifo-depth: MAC transmit FIFO buffer depth in bytes -- phy-mode: See ethernet.txt in the same directory. -- phy-handle: See ethernet.txt in the same directory. -- phy-addr: See ethernet.txt in the same directory. A configuration should - include phy-handle or phy-addr. -- altr,has-supplementary-unicast: - If present, TSE supports additional unicast addresses. - Otherwise additional unicast addresses are not supported. -- altr,has-hash-multicast-filter: - If present, TSE supports a hash based multicast filter. - Otherwise, hash-based multicast filtering is not supported. - -- mdio device tree subnode: When the TSE has a phy connected to its local - mdio, there must be device tree subnode with the following - required properties: - - - compatible: Must be "altr,tse-mdio". - - #address-cells: Must be <1>. - - #size-cells: Must be <0>. - - For each phy on the mdio bus, there must be a node with the following - fields: - - - reg: phy id used to communicate to phy. - - device_type: Must be "ethernet-phy". - -The MAC address will be determined using the optional properties defined in -ethernet.txt. - -Example: - - tse_sub_0_eth_tse_0: ethernet@1,00000000 { - compatible = "altr,tse-msgdma-1.0"; - reg = <0x00000001 0x00000000 0x00000400>, - <0x00000001 0x00000460 0x00000020>, - <0x00000001 0x00000480 0x00000020>, - <0x00000001 0x000004A0 0x00000008>, - <0x00000001 0x00000400 0x00000020>, - <0x00000001 0x00000420 0x00000020>; - reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc"; - interrupt-parent = <&hps_0_arm_gic_0>; - interrupts = <0 41 4>, <0 40 4>; - interrupt-names = "rx_irq", "tx_irq"; - rx-fifo-depth = <2048>; - tx-fifo-depth = <2048>; - address-bits = <48>; - max-frame-size = <1500>; - local-mac-address = [ 00 00 00 00 00 00 ]; - phy-mode = "gmii"; - altr,has-supplementary-unicast; - altr,has-hash-multicast-filter; - phy-handle = <&phy0>; - mdio { - compatible = "altr,tse-mdio"; - #address-cells = <1>; - #size-cells = <0>; - phy0: ethernet-phy@0 { - reg = <0x0>; - device_type = "ethernet-phy"; - }; - - phy1: ethernet-phy@1 { - reg = <0x1>; - device_type = "ethernet-phy"; - }; - - }; - }; - - tse_sub_1_eth_tse_0: ethernet@1,00001000 { - compatible = "altr,tse-msgdma-1.0"; - reg = <0x00000001 0x00001000 0x00000400>, - <0x00000001 0x00001460 0x00000020>, - <0x00000001 0x00001480 0x00000020>, - <0x00000001 0x000014A0 0x00000008>, - <0x00000001 0x00001400 0x00000020>, - <0x00000001 0x00001420 0x00000020>; - reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc"; - interrupt-parent = <&hps_0_arm_gic_0>; - interrupts = <0 43 4>, <0 42 4>; - interrupt-names = "rx_irq", "tx_irq"; - rx-fifo-depth = <2048>; - tx-fifo-depth = <2048>; - address-bits = <48>; - max-frame-size = <1500>; - local-mac-address = [ 00 00 00 00 00 00 ]; - phy-mode = "gmii"; - altr,has-supplementary-unicast; - altr,has-hash-multicast-filter; - phy-handle = <&phy1>; - }; diff --git a/Documentation/devicetree/bindings/net/altr,tse.yaml b/Documentation/devicetree/bindings/net/altr,tse.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1d94494349e840101ddad5da60ca5900781229 --- /dev/null +++ b/Documentation/devicetree/bindings/net/altr,tse.yaml @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/altr,tse.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Altera Triple Speed Ethernet MAC driver (TSE) + +maintainers: + - Maxime Chevallier + +properties: + compatible: + oneOf: + - const: altr,tse-1.0 + - const: ALTR,tse-1.0 + deprecated: true + - const: altr,tse-msgdma-1.0 + + interrupts: + minItems: 2 + + interrupt-names: + items: + - const: rx_irq + - const: tx_irq + + rx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Depth in bytes of the RX FIFO + + tx-fifo-depth: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Depth in bytes of the TX FIFO + + altr,has-supplementary-unicast: + type: boolean + description: + If present, TSE supports additional unicast addresses. + + altr,has-hash-multicast-filter: + type: boolean + description: + If present, TSE supports hash based multicast filter. + + mdio: + $ref: mdio.yaml# + unevaluatedProperties: false + description: + Creates and registers an MDIO bus. + + properties: + compatible: + const: altr,tse-mdio + + required: + - compatible + +required: + - compatible + - reg + - interrupts + - rx-fifo-depth + - tx-fifo-depth + +allOf: + - $ref: "ethernet-controller.yaml#" + - if: + properties: + compatible: + contains: + enum: + - const: altr,tse-1.0 + - const: ALTR,tse-1.0 + then: + properties: + reg: + minItems: 4 + reg-names: + items: + - const: control_port + - const: rx_csr + - const: tx_csr + - const: s1 + + - if: + properties: + compatible: + contains: + enum: + - altr,tse-msgdma-1.0 + then: + properties: + reg: + minItems: 6 + maxItems: 7 + reg-names: + minItems: 6 + items: + - const: control_port + - const: rx_csr + - const: rx_desc + - const: rx_resp + - const: tx_csr + - const: tx_desc + - const: pcs + +unevaluatedProperties: false + +examples: + - | + tse_sub_0: ethernet@c0100000 { + compatible = "altr,tse-msgdma-1.0"; + reg = <0xc0100000 0x00000400>, + <0xc0101000 0x00000020>, + <0xc0102000 0x00000020>, + <0xc0103000 0x00000008>, + <0xc0104000 0x00000020>, + <0xc0105000 0x00000020>, + <0xc0106000 0x00000100>; + reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc", "pcs"; + interrupt-parent = <&intc>; + interrupts = <0 44 4>,<0 45 4>; + interrupt-names = "rx_irq","tx_irq"; + rx-fifo-depth = <2048>; + tx-fifo-depth = <2048>; + max-frame-size = <1500>; + local-mac-address = [ 00 00 00 00 00 00 ]; + altr,has-supplementary-unicast; + altr,has-hash-multicast-filter; + sfp = <&sfp0>; + phy-mode = "sgmii"; + managed = "in-band-status"; + }; + - | + tse_sub_1_eth_tse_0: ethernet@1,00001000 { + compatible = "altr,tse-msgdma-1.0"; + reg = <0x00001000 0x00000400>, + <0x00001460 0x00000020>, + <0x00001480 0x00000020>, + <0x000014A0 0x00000008>, + <0x00001400 0x00000020>, + <0x00001420 0x00000020>; + reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc"; + interrupt-parent = <&hps_0_arm_gic_0>; + interrupts = <0 43 4>, <0 42 4>; + interrupt-names = "rx_irq", "tx_irq"; + rx-fifo-depth = <2048>; + tx-fifo-depth = <2048>; + max-frame-size = <1500>; + local-mac-address = [ 00 00 00 00 00 00 ]; + phy-mode = "gmii"; + altr,has-supplementary-unicast; + altr,has-hash-multicast-filter; + phy-handle = <&phy1>; + mdio { + compatible = "altr,tse-mdio"; + #address-cells = <1>; + #size-cells = <0>; + phy1: ethernet-phy@1 { + reg = <0x1>; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/net/amlogic,meson-dwmac.yaml b/Documentation/devicetree/bindings/net/amlogic,meson-dwmac.yaml index 608e1d62bed5ca44b2dcc0d83339ccefe621059f..ddd5a073c3a89183156a32c940ac7a4a829cb333 100644 --- a/Documentation/devicetree/bindings/net/amlogic,meson-dwmac.yaml +++ b/Documentation/devicetree/bindings/net/amlogic,meson-dwmac.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson DWMAC Ethernet controller maintainers: - - Neil Armstrong + - Neil Armstrong - Martin Blumenstingl # We need a select here so we don't match all nodes with 'snps,dwmac' diff --git a/Documentation/devicetree/bindings/net/brcm,amac.yaml b/Documentation/devicetree/bindings/net/brcm,amac.yaml index 8f031932c8afba26fcbfc69c63dd4134ba70fddb..ee2eac8f5710130f1a10a63be90ce23ba62b08d8 100644 --- a/Documentation/devicetree/bindings/net/brcm,amac.yaml +++ b/Documentation/devicetree/bindings/net/brcm,amac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/brcm,amac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom AMAC Ethernet Controller Device Tree Bindings +title: Broadcom AMAC Ethernet Controller maintainers: - Florian Fainelli diff --git a/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml index b1327c5b86cfbef71608ae33bcd39b126aff6f07..144a3785132c31a23253dc5b11b4ea215deaf9fc 100644 --- a/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml +++ b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml @@ -30,8 +30,10 @@ properties: clocks: maxItems: 1 + power-domains: + maxItems: 1 + reg-io-width: - $ref: /schemas/types.yaml#/definitions/uint32 description: I/O register width (in bytes) implemented by this device default: 1 enum: [ 1, 2, 4 ] @@ -105,6 +107,7 @@ allOf: then: required: - clocks + - power-domains unevaluatedProperties: false @@ -129,4 +132,5 @@ examples: reg-io-width = <4>; interrupts = ; clocks = <&sysctrl R9A06G032_HCLK_CAN0>; + power-domains = <&sysctrl>; }; diff --git a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml index cc01b9b5752a74ad7a71a3639c126a01794ece45..253b5d1407ee97c8d160bb458312c301691848bb 100644 --- a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml +++ b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml @@ -37,6 +37,7 @@ properties: patternProperties: "^ethernet-port@[0-9]+$": type: object + unevaluatedProperties: false description: contains the resources for ethernet port allOf: - $ref: ethernet-controller.yaml# diff --git a/Documentation/devicetree/bindings/net/dsa/ar9331.txt b/Documentation/devicetree/bindings/net/dsa/ar9331.txt index 320607cbbb1766765028ed4050b7fcb352d7c812..f824fdae0da20afc8976d0fa1f8063a5bd5894da 100644 --- a/Documentation/devicetree/bindings/net/dsa/ar9331.txt +++ b/Documentation/devicetree/bindings/net/dsa/ar9331.txt @@ -76,7 +76,6 @@ eth1: ethernet@1a000000 { switch_port0: port@0 { reg = <0x0>; - label = "cpu"; ethernet = <ð1>; phy-mode = "gmii"; diff --git a/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml b/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml index 3f01b65f3b22705a31d7aee582431b7eb004f52f..259a0c6547f3073fa5eae83d3282a4717d132eb1 100644 --- a/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml +++ b/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml @@ -61,8 +61,9 @@ examples: }; ethernet-port@3 { reg = <3>; - label = "cpu"; ethernet = <&fec1>; + phy-mode = "rgmii-id"; + fixed-link { speed = <1000>; full-duplex; diff --git a/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml b/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml index 23114d691d2a17b0797ac01add974f05e40b4146..1219b830b1a46591c9e96e213bed7ae22a77d376 100644 --- a/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml +++ b/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml @@ -169,7 +169,6 @@ examples: port@8 { reg = <8>; - label = "cpu"; phy-mode = "rgmii-txid"; ethernet = <ð0>; fixed-link { @@ -252,8 +251,9 @@ examples: port@8 { ethernet = <&amac2>; - label = "cpu"; reg = <8>; + phy-mode = "internal"; + fixed-link { speed = <1000>; full-duplex; diff --git a/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml index 09317e16cb5dbde4e792a17ef90c37e60c9bcd5c..10ad7e71097b57d7fa08bc58004b35423ff4b640 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml +++ b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml @@ -76,6 +76,23 @@ properties: required: - reg +# CPU and DSA ports must have phylink-compatible link descriptions +if: + oneOf: + - required: [ ethernet ] + - required: [ link ] +then: + allOf: + - required: + - phy-mode + - oneOf: + - required: + - fixed-link + - required: + - phy-handle + - required: + - managed + additionalProperties: true ... diff --git a/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml b/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml index 228683773151c04a1105d06d0a11226f87b488cd..73b774eadd0b1bcaec1796d8de1054e2d839a582 100644 --- a/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml +++ b/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml @@ -91,8 +91,13 @@ examples: port@0 { reg = <0>; - label = "cpu"; ethernet = <&gmac0>; + phy-mode = "mii"; + + fixed-link { + speed = <100>; + full-duplex; + }; }; port@2 { diff --git a/Documentation/devicetree/bindings/net/dsa/lan9303.txt b/Documentation/devicetree/bindings/net/dsa/lan9303.txt index 464d6bf87605392a5302349abf7b54e673ed5dc1..46a732087f5cab7c5dafa289ca6548c0b4246fd8 100644 --- a/Documentation/devicetree/bindings/net/dsa/lan9303.txt +++ b/Documentation/devicetree/bindings/net/dsa/lan9303.txt @@ -46,7 +46,6 @@ I2C managed mode: port@0 { /* RMII fixed link to master */ reg = <0>; - label = "cpu"; ethernet = <&master>; }; @@ -83,7 +82,6 @@ MDIO managed mode: port@0 { reg = <0>; - label = "cpu"; ethernet = <&master>; }; diff --git a/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt b/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt index e3829d3e480e772fcfff85d3a32ed1c0397db15e..8bb1eff21cb1a9fc84ba36da3fd03dd17410c79e 100644 --- a/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt +++ b/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt @@ -96,7 +96,6 @@ switch@e108000 { port@6 { reg = <0x6>; - label = "cpu"; ethernet = <ð0>; }; }; diff --git a/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml b/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml index 17ab6c69ecc7e49c8414dbf3ee21855016a87ecb..f2e9ff3f580bfe6deafafeddf0352794c72f935b 100644 --- a/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml +++ b/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml @@ -4,67 +4,92 @@ $id: http://devicetree.org/schemas/net/dsa/mediatek,mt7530.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT7530 Ethernet switch +title: Mediatek MT7530 and MT7531 Ethernet Switches maintainers: - - Sean Wang + - Arınç ÜNAL - Landen Chao - DENG Qingfang + - Sean Wang description: | - Port 5 of mt7530 and mt7621 switch is muxed between: - 1. GMAC5: GMAC5 can interface with another external MAC or PHY. - 2. PHY of port 0 or port 4: PHY interfaces with an external MAC like 2nd GMAC - of the SOC. Used in many setups where port 0/4 becomes the WAN port. - Note: On a MT7621 SOC with integrated switch: 2nd GMAC can only connected to - GMAC5 when the gpios for RGMII2 (GPIO 22-33) are not used and not - connected to external component! - - Port 5 modes/configurations: - 1. Port 5 is disabled and isolated: An external phy can interface to the 2nd - GMAC of the SOC. - In the case of a build-in MT7530 switch, port 5 shares the RGMII bus with 2nd - GMAC and an optional external phy. Mind the GPIO/pinctl settings of the SOC! - 2. Port 5 is muxed to PHY of port 0/4: Port 0/4 interfaces with 2nd GMAC. - It is a simple MAC to PHY interface, port 5 needs to be setup for xMII mode - and RGMII delay. - 3. Port 5 is muxed to GMAC5 and can interface to an external phy. - Port 5 becomes an extra switch port. - Only works on platform where external phy TX<->RX lines are swapped. - Like in the Ubiquiti ER-X-SFP. - 4. Port 5 is muxed to GMAC5 and interfaces with the 2nd GAMC as 2nd CPU port. - Currently a 2nd CPU port is not supported by DSA code. - - Depending on how the external PHY is wired: - 1. normal: The PHY can only connect to 2nd GMAC but not to the switch - 2. swapped: RGMII TX, RX are swapped; external phy interface with the switch as - a ethernet port. But can't interface to the 2nd GMAC. - - Based on the DT the port 5 mode is configured. - - Driver tries to lookup the phy-handle of the 2nd GMAC of the master device. - When phy-handle matches PHY of port 0 or 4 then port 5 set-up as mode 2. - phy-mode must be set, see also example 2 below! - * mt7621: phy-mode = "rgmii-txid"; - * mt7623: phy-mode = "rgmii"; - - CPU-Ports need a phy-mode property: - Allowed values on mt7530 and mt7621: - - "rgmii" - - "trgmii" - On mt7531: - - "1000base-x" - - "2500base-x" - - "rgmii" - - "sgmii" + There are two versions of MT7530, standalone and in a multi-chip module. + + MT7530 is a part of the multi-chip module in MT7620AN, MT7620DA, MT7620DAN, + MT7620NN, MT7621AT, MT7621DAT, MT7621ST and MT7623AI SoCs. + + MT7530 in MT7620AN, MT7620DA, MT7620DAN and MT7620NN SoCs has got 10/100 PHYs + and the switch registers are directly mapped into SoC's memory map rather than + using MDIO. The DSA driver currently doesn't support this. + + There is only the standalone version of MT7531. + + Port 5 on MT7530 has got various ways of configuration. + + For standalone MT7530: + + - Port 5 can be used as a CPU port. + + - PHY 0 or 4 of the switch can be muxed to connect to the gmac of the SoC + which port 5 is wired to. Usually used for connecting the wan port + directly to the CPU to achieve 2 Gbps routing in total. + + The driver looks up the reg on the ethernet-phy node which the phy-handle + property refers to on the gmac node to mux the specified phy. + + The driver requires the gmac of the SoC to have "mediatek,eth-mac" as the + compatible string and the reg must be 1. So, for now, only gmac1 of an + MediaTek SoC can benefit this. Banana Pi BPI-R2 suits this. + Check out example 5 for a similar configuration. + + - Port 5 can be wired to an external phy. Port 5 becomes a DSA slave. + Check out example 7 for a similar configuration. + + For multi-chip module MT7530: + + - Port 5 can be used as a CPU port. + + - PHY 0 or 4 of the switch can be muxed to connect to gmac1 of the SoC. + Usually used for connecting the wan port directly to the CPU to achieve 2 + Gbps routing in total. + + The driver looks up the reg on the ethernet-phy node which the phy-handle + property refers to on the gmac node to mux the specified phy. + + For the MT7621 SoCs, rgmii2 group must be claimed with rgmii2 function. + Check out example 5. + + - In case of an external phy wired to gmac1 of the SoC, port 5 must not be + enabled. + + In case of muxing PHY 0 or 4, the external phy must not be enabled. + + For the MT7621 SoCs, rgmii2 group must be claimed with rgmii2 function. + Check out example 6. + - Port 5 can be muxed to an external phy. Port 5 becomes a DSA slave. + The external phy must be wired TX to TX to gmac1 of the SoC for this to + work. Ubiquiti EdgeRouter X SFP is wired this way. + + Muxing PHY 0 or 4 won't work when the external phy is connected TX to TX. + + For the MT7621 SoCs, rgmii2 group must be claimed with gpio function. + Check out example 7. properties: compatible: - enum: - - mediatek,mt7530 - - mediatek,mt7531 - - mediatek,mt7621 + oneOf: + - description: + Standalone MT7530 and multi-chip module MT7530 in MT7623AI SoC + const: mediatek,mt7530 + + - description: + Standalone MT7531 + const: mediatek,mt7531 + + - description: + Multi-chip module MT7530 in MT7621AT, MT7621DAT and MT7621ST SoCs + const: mediatek,mt7621 reg: maxItems: 1 @@ -79,7 +104,14 @@ properties: gpio-controller: type: boolean description: - if defined, MT7530's LED controller will run on GPIO mode. + If defined, LED controller of the MT7530 switch will run on GPIO mode. + + There are 15 controllable pins. + port 0 LED 0..2 as GPIO 0..2 + port 1 LED 0..2 as GPIO 3..5 + port 2 LED 0..2 as GPIO 6..8 + port 3 LED 0..2 as GPIO 9..11 + port 4 LED 0..2 as GPIO 12..14 "#interrupt-cells": const: 1 @@ -92,17 +124,21 @@ properties: io-supply: description: Phandle to the regulator node necessary for the I/O power. - See Documentation/devicetree/bindings/regulator/mt6323-regulator.txt - for details for the regulator setup on these boards. + See Documentation/devicetree/bindings/regulator/mt6323-regulator.txt for + details for the regulator setup on these boards. mediatek,mcm: type: boolean description: - if defined, indicates that either MT7530 is the part on multi-chip - module belong to MT7623A has or the remotely standalone chip as the - function MT7623N reference board provided for. + Used for MT7621AT, MT7621DAT, MT7621ST and MT7623AI SoCs which the MT7530 + switch is a part of the multi-chip module. reset-gpios: + description: + GPIO to reset the switch. Use this if mediatek,mcm is not used. + This property is optional because some boards share the reset line with + other components which makes it impossible to probe the switch if the + reset line is used. maxItems: 1 reset-names: @@ -110,8 +146,8 @@ properties: resets: description: - Phandle pointing to the system reset controller with line index for - the ethsys. + Phandle pointing to the system reset controller with line index for the + ethsys. maxItems: 1 patternProperties: @@ -128,31 +164,88 @@ patternProperties: properties: reg: description: - Port address described must be 5 or 6 for CPU port and from 0 - to 5 for user ports. + Port address described must be 5 or 6 for CPU port and from 0 to 5 + for user ports. allOf: - $ref: dsa-port.yaml# - if: - properties: - label: - items: - - const: cpu + required: [ ethernet ] then: - required: - - reg - - phy-mode + properties: + reg: + enum: + - 5 + - 6 required: - compatible - reg +$defs: + mt7530-dsa-port: + patternProperties: + "^(ethernet-)?ports$": + patternProperties: + "^(ethernet-)?port@[0-9]+$": + if: + required: [ ethernet ] + then: + if: + properties: + reg: + const: 5 + then: + properties: + phy-mode: + enum: + - gmii + - mii + - rgmii + else: + properties: + phy-mode: + enum: + - rgmii + - trgmii + + mt7531-dsa-port: + patternProperties: + "^(ethernet-)?ports$": + patternProperties: + "^(ethernet-)?port@[0-9]+$": + if: + required: [ ethernet ] + then: + if: + properties: + reg: + const: 5 + then: + properties: + phy-mode: + enum: + - 1000base-x + - 2500base-x + - rgmii + - sgmii + else: + properties: + phy-mode: + enum: + - 1000base-x + - 2500base-x + - sgmii + allOf: - - $ref: "dsa.yaml#" + - $ref: dsa.yaml# - if: required: - mediatek,mcm then: + properties: + reset-gpios: false + required: - resets - reset-names @@ -163,52 +256,139 @@ allOf: - if: properties: compatible: - items: - - const: mediatek,mt7530 + const: mediatek,mt7530 then: + $ref: "#/$defs/mt7530-dsa-port" required: - core-supply - io-supply + - if: + properties: + compatible: + const: mediatek,mt7531 + then: + $ref: "#/$defs/mt7531-dsa-port" + properties: + gpio-controller: false + mediatek,mcm: false + + - if: + properties: + compatible: + const: mediatek,mt7621 + then: + $ref: "#/$defs/mt7530-dsa-port" + required: + - mediatek,mcm + unevaluatedProperties: false examples: + # Example 1: Standalone MT7530 - | #include + mdio { #address-cells = <1>; #size-cells = <0>; - switch@0 { + + switch@1f { compatible = "mediatek,mt7530"; - reg = <0>; + reg = <0x1f>; + + reset-gpios = <&pio 33 0>; core-supply = <&mt6323_vpa_reg>; io-supply = <&mt6323_vemc3v3_reg>; - reset-gpios = <&pio 33 GPIO_ACTIVE_HIGH>; ethernet-ports { #address-cells = <1>; #size-cells = <0>; + port@0 { reg = <0>; - label = "lan0"; + label = "lan1"; }; port@1 { reg = <1>; - label = "lan1"; + label = "lan2"; }; port@2 { reg = <2>; - label = "lan2"; + label = "lan3"; }; port@3 { reg = <3>; + label = "lan4"; + }; + + port@4 { + reg = <4>; + label = "wan"; + }; + + port@6 { + reg = <6>; + ethernet = <&gmac0>; + phy-mode = "rgmii"; + + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + }; + }; + }; + + # Example 2: MT7530 in MT7623AI SoC + - | + #include + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + switch@1f { + compatible = "mediatek,mt7530"; + reg = <0x1f>; + + mediatek,mcm; + resets = <ðsys MT2701_ETHSYS_MCM_RST>; + reset-names = "mcm"; + + core-supply = <&mt6323_vpa_reg>; + io-supply = <&mt6323_vemc3v3_reg>; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + label = "lan1"; + }; + + port@1 { + reg = <1>; + label = "lan2"; + }; + + port@2 { + reg = <2>; label = "lan3"; }; + port@3 { + reg = <3>; + label = "lan4"; + }; + port@4 { reg = <4>; label = "wan"; @@ -216,96 +396,226 @@ examples: port@6 { reg = <6>; - label = "cpu"; ethernet = <&gmac0>; phy-mode = "trgmii"; + fixed-link { speed = <1000>; full-duplex; + pause; }; }; }; }; }; + # Example 3: Standalone MT7531 - | - //Example 2: MT7621: Port 4 is WAN port: 2nd GMAC -> Port 5 -> PHY port 4. + #include + #include - ethernet { + mdio { #address-cells = <1>; #size-cells = <0>; - gmac0: mac@0 { - compatible = "mediatek,eth-mac"; + + switch@0 { + compatible = "mediatek,mt7531"; reg = <0>; - phy-mode = "rgmii"; - fixed-link { - speed = <1000>; - full-duplex; - pause; + reset-gpios = <&pio 54 0>; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&pio>; + interrupts = <53 IRQ_TYPE_LEVEL_HIGH>; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + label = "lan1"; + }; + + port@1 { + reg = <1>; + label = "lan2"; + }; + + port@2 { + reg = <2>; + label = "lan3"; + }; + + port@3 { + reg = <3>; + label = "lan4"; + }; + + port@4 { + reg = <4>; + label = "wan"; + }; + + port@6 { + reg = <6>; + ethernet = <&gmac0>; + phy-mode = "2500base-x"; + + fixed-link { + speed = <2500>; + full-duplex; + pause; + }; + }; + }; + }; + }; + + # Example 4: MT7530 in MT7621AT, MT7621DAT and MT7621ST SoCs + - | + #include + #include + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + switch@1f { + compatible = "mediatek,mt7621"; + reg = <0x1f>; + + mediatek,mcm; + resets = <&sysc MT7621_RST_MCM>; + reset-names = "mcm"; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + label = "lan1"; + }; + + port@1 { + reg = <1>; + label = "lan2"; + }; + + port@2 { + reg = <2>; + label = "lan3"; + }; + + port@3 { + reg = <3>; + label = "lan4"; + }; + + port@4 { + reg = <4>; + label = "wan"; + }; + + port@6 { + reg = <6>; + ethernet = <&gmac0>; + phy-mode = "trgmii"; + + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; }; }; + }; - gmac1: mac@1 { + # Example 5: MT7621: mux MT7530's phy4 to SoC's gmac1 + - | + #include + #include + + ethernet { + #address-cells = <1>; + #size-cells = <0>; + + pinctrl-names = "default"; + pinctrl-0 = <&rgmii2_pins>; + + mac@1 { compatible = "mediatek,eth-mac"; reg = <1>; - phy-mode = "rgmii-txid"; - phy-handle = <&phy4>; + + phy-mode = "rgmii"; + phy-handle = <&example5_ethphy4>; }; - mdio: mdio-bus { + mdio { #address-cells = <1>; #size-cells = <0>; - /* Internal phy */ - phy4: ethernet-phy@4 { + /* MT7530's phy4 */ + example5_ethphy4: ethernet-phy@4 { reg = <4>; }; - mt7530: switch@1f { + switch@1f { compatible = "mediatek,mt7621"; reg = <0x1f>; - mediatek,mcm; - resets = <&rstctrl 2>; + mediatek,mcm; + resets = <&sysc MT7621_RST_MCM>; reset-names = "mcm"; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + ethernet-ports { #address-cells = <1>; #size-cells = <0>; port@0 { reg = <0>; - label = "lan0"; + label = "lan1"; }; port@1 { reg = <1>; - label = "lan1"; + label = "lan2"; }; port@2 { reg = <2>; - label = "lan2"; + label = "lan3"; }; port@3 { reg = <3>; - label = "lan3"; + label = "lan4"; }; - /* Commented out. Port 4 is handled by 2nd GMAC. + /* Commented out, phy4 is muxed to gmac1. port@4 { reg = <4>; - label = "lan4"; + label = "wan"; }; */ port@6 { reg = <6>; - label = "cpu"; ethernet = <&gmac0>; - phy-mode = "rgmii"; + phy-mode = "trgmii"; fixed-link { speed = <1000>; @@ -318,82 +628,169 @@ examples: }; }; + # Example 6: MT7621: mux external phy to SoC's gmac1 - | - //Example 3: MT7621: Port 5 is connected to external PHY: Port 5 -> external PHY. + #include + #include ethernet { #address-cells = <1>; #size-cells = <0>; - gmac_0: mac@0 { + + pinctrl-names = "default"; + pinctrl-0 = <&rgmii2_pins>; + + mac@1 { compatible = "mediatek,eth-mac"; - reg = <0>; + reg = <1>; + phy-mode = "rgmii"; + phy-handle = <&example6_ethphy7>; + }; + + mdio { + #address-cells = <1>; + #size-cells = <0>; - fixed-link { - speed = <1000>; - full-duplex; - pause; + /* External PHY */ + example6_ethphy7: ethernet-phy@7 { + reg = <7>; + phy-mode = "rgmii"; + }; + + switch@1f { + compatible = "mediatek,mt7621"; + reg = <0x1f>; + + mediatek,mcm; + resets = <&sysc MT7621_RST_MCM>; + reset-names = "mcm"; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + label = "lan1"; + }; + + port@1 { + reg = <1>; + label = "lan2"; + }; + + port@2 { + reg = <2>; + label = "lan3"; + }; + + port@3 { + reg = <3>; + label = "lan4"; + }; + + port@4 { + reg = <4>; + label = "wan"; + }; + + port@6 { + reg = <6>; + ethernet = <&gmac0>; + phy-mode = "trgmii"; + + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + }; }; }; + }; + + # Example 7: MT7621: mux external phy to MT7530's port 5 + - | + #include + #include + + ethernet { + #address-cells = <1>; + #size-cells = <0>; - mdio0: mdio-bus { + pinctrl-names = "default"; + pinctrl-0 = <&rgmii2_pins>; + + mdio { #address-cells = <1>; #size-cells = <0>; - /* External phy */ - ephy5: ethernet-phy@7 { + /* External PHY */ + example7_ethphy7: ethernet-phy@7 { reg = <7>; + phy-mode = "rgmii"; }; switch@1f { compatible = "mediatek,mt7621"; reg = <0x1f>; - mediatek,mcm; - resets = <&rstctrl 2>; + mediatek,mcm; + resets = <&sysc MT7621_RST_MCM>; reset-names = "mcm"; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + ethernet-ports { #address-cells = <1>; #size-cells = <0>; port@0 { reg = <0>; - label = "lan0"; + label = "lan1"; }; port@1 { reg = <1>; - label = "lan1"; + label = "lan2"; }; port@2 { reg = <2>; - label = "lan2"; + label = "lan3"; }; port@3 { reg = <3>; - label = "lan3"; + label = "lan4"; }; port@4 { reg = <4>; - label = "lan4"; + label = "wan"; }; port@5 { reg = <5>; - label = "lan5"; - phy-mode = "rgmii"; - phy-handle = <&ephy5>; + label = "extphy"; + phy-mode = "rgmii-txid"; + phy-handle = <&example7_ethphy7>; }; - cpu_port0: port@6 { + port@6 { reg = <6>; - label = "cpu"; - ethernet = <&gmac_0>; - phy-mode = "rgmii"; + ethernet = <&gmac0>; + phy-mode = "trgmii"; fixed-link { speed = <1000>; diff --git a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml index 6bbd8145b6c1aba45663db05eda8a4465f52448d..4da75b1f9533945c0f8875031f91966403710107 100644 --- a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml +++ b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml @@ -107,8 +107,9 @@ examples: }; port@5 { reg = <5>; - label = "cpu"; ethernet = <ð0>; + phy-mode = "rgmii"; + fixed-link { speed = <1000>; full-duplex; @@ -144,8 +145,9 @@ examples: }; port@6 { reg = <6>; - label = "cpu"; ethernet = <ð0>; + phy-mode = "rgmii"; + fixed-link { speed = <1000>; full-duplex; diff --git a/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml b/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d93ed9c172cb7a81b26afa069be080ec13845c6 --- /dev/null +++ b/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: (GPL-2.0 OR MIT) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/dsa/mscc,ocelot.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip Ocelot Switch Family Device Tree Bindings + +maintainers: + - Vladimir Oltean + - Claudiu Manoil + - Alexandre Belloni + - UNGLinuxDriver@microchip.com + +description: | + There are multiple switches which are either part of the Ocelot-1 family, or + derivatives of this architecture. These switches can be found embedded in + various SoCs and accessed using MMIO, or as discrete chips and accessed over + SPI or PCIe. The present DSA binding shall be used when the host controlling + them performs packet I/O primarily through an Ethernet port of the switch + (which is attached to an Ethernet port of the host), rather than through + Frame DMA or register-based I/O. + + VSC9953 (Seville): + + This is found in the NXP T1040, where it is a memory-mapped platform + device. + + The following PHY interface types are supported: + + - phy-mode = "internal": on ports 8 and 9 + - phy-mode = "sgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7 + - phy-mode = "qsgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7 + - phy-mode = "1000base-x": on ports 0, 1, 2, 3, 4, 5, 6, 7 + + VSC9959 (Felix): + + This is found in the NXP LS1028A. It is a PCI device, part of the larger + enetc root complex. As a result, the ethernet-switch node is a sub-node of + the PCIe root complex node and its "reg" property conforms to the parent + node bindings, describing it as PF 5 of device 0, bus 0. + + If any external switch port is enabled, the enetc PF2 (enetc_port2) should + be enabled as well. This is because the internal MDIO bus (exposed through + EA BAR 0) used to access the MAC PCS registers truly belongs to the enetc + port 2 and not to Felix. + + The following PHY interface types are supported: + + - phy-mode = "internal": on ports 4 and 5 + - phy-mode = "sgmii": on ports 0, 1, 2, 3 + - phy-mode = "qsgmii": on ports 0, 1, 2, 3 + - phy-mode = "usxgmii": on ports 0, 1, 2, 3 + - phy-mode = "1000base-x": on ports 0, 1, 2, 3 + - phy-mode = "2500base-x": on ports 0, 1, 2, 3 + +properties: + compatible: + enum: + - mscc,vsc9953-switch + - pci1957,eef0 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + description: + Used to signal availability of PTP TX timestamps, and state changes of + the MAC merge layer of ports that support Frame Preemption. + + little-endian: true + big-endian: true + +required: + - compatible + - reg + +allOf: + - $ref: dsa.yaml# + - if: + properties: + compatible: + const: pci1957,eef0 + then: + required: + - interrupts + +unevaluatedProperties: false + +examples: + # Felix VSC9959 (NXP LS1028A) + - | + #include + + pcie { /* Integrated Endpoint Root Complex */ + #address-cells = <3>; + #size-cells = <2>; + + ethernet-switch@0,5 { + compatible = "pci1957,eef0"; + reg = <0x000500 0 0 0 0>; + interrupts = ; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + phy-mode = "qsgmii"; + phy-handle = <&phy0>; + managed = "in-band-status"; + }; + + port@1 { + reg = <1>; + phy-mode = "qsgmii"; + phy-handle = <&phy1>; + managed = "in-band-status"; + }; + + port@2 { + reg = <2>; + phy-mode = "qsgmii"; + phy-handle = <&phy2>; + managed = "in-band-status"; + }; + + port@3 { + reg = <3>; + phy-mode = "qsgmii"; + phy-handle = <&phy3>; + managed = "in-band-status"; + }; + + port@4 { + reg = <4>; + ethernet = <&enetc_port2>; + phy-mode = "internal"; + + fixed-link { + speed = <2500>; + full-duplex; + pause; + }; + }; + + port@5 { + reg = <5>; + ethernet = <&enetc_port3>; + phy-mode = "internal"; + + fixed-link { + speed = <1000>; + full-duplex; + pause; + }; + }; + }; + }; + }; + # Seville VSC9953 (NXP T1040) + - | + soc { + #address-cells = <1>; + #size-cells = <1>; + + ethernet-switch@800000 { + compatible = "mscc,vsc9953-switch"; + reg = <0x800000 0x290000>; + little-endian; + + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + phy-mode = "qsgmii"; + phy-handle = <&phy0>; + managed = "in-band-status"; + }; + + port@1 { + reg = <1>; + phy-mode = "qsgmii"; + phy-handle = <&phy1>; + managed = "in-band-status"; + }; + + port@2 { + reg = <2>; + phy-mode = "qsgmii"; + phy-handle = <&phy2>; + managed = "in-band-status"; + }; + + port@3 { + reg = <3>; + phy-mode = "qsgmii"; + phy-handle = <&phy3>; + managed = "in-band-status"; + }; + + port@4 { + reg = <4>; + phy-mode = "qsgmii"; + phy-handle = <&phy4>; + managed = "in-band-status"; + }; + + port@5 { + reg = <5>; + phy-mode = "qsgmii"; + phy-handle = <&phy5>; + managed = "in-band-status"; + }; + + port@6 { + reg = <6>; + phy-mode = "qsgmii"; + phy-handle = <&phy6>; + managed = "in-band-status"; + }; + + port@7 { + reg = <7>; + phy-mode = "qsgmii"; + phy-handle = <&phy7>; + managed = "in-band-status"; + }; + + port@8 { + reg = <8>; + phy-mode = "internal"; + ethernet = <&enet0>; + + fixed-link { + speed = <2500>; + full-duplex; + pause; + }; + }; + + port@9 { + reg = <9>; + phy-mode = "internal"; + ethernet = <&enet1>; + + fixed-link { + speed = <2500>; + full-duplex; + pause; + }; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/dsa/ocelot.txt b/Documentation/devicetree/bindings/net/dsa/ocelot.txt deleted file mode 100644 index 7a271d070b72430d69ab4261a417d672a33402fa..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/net/dsa/ocelot.txt +++ /dev/null @@ -1,213 +0,0 @@ -Microchip Ocelot switch driver family -===================================== - -Felix ------ - -Currently the switches supported by the felix driver are: - -- VSC9959 (Felix) -- VSC9953 (Seville) - -The VSC9959 switch is found in the NXP LS1028A. It is a PCI device, part of the -larger ENETC root complex. As a result, the ethernet-switch node is a sub-node -of the PCIe root complex node and its "reg" property conforms to the parent -node bindings: - -* reg: Specifies PCIe Device Number and Function Number of the endpoint device, - in this case for the Ethernet L2Switch it is PF5 (of device 0, bus 0). - -It does not require a "compatible" string. - -The interrupt line is used to signal availability of PTP TX timestamps and for -TSN frame preemption. - -For the external switch ports, depending on board configuration, "phy-mode" and -"phy-handle" are populated by board specific device tree instances. Ports 4 and -5 are fixed as internal ports in the NXP LS1028A instantiation. - -The CPU port property ("ethernet") configures the feature called "NPI port" in -the Ocelot hardware core. The CPU port in Ocelot is a set of queues, which are -connected, in the Node Processor Interface (NPI) mode, to an Ethernet port. -By default, in fsl-ls1028a.dtsi, the NPI port is assigned to the internal -2.5Gbps port@4, but can be moved to the 1Gbps port@5, depending on the specific -use case. Moving the NPI port to an external switch port is hardware possible, -but there is no platform support for the Linux system on the LS1028A chip to -operate as an entire slave DSA chip. NPI functionality (and therefore DSA -tagging) is supported on a single port at a time. - -Any port can be disabled (and in fsl-ls1028a.dtsi, they are indeed all disabled -by default, and should be enabled on a per-board basis). But if any external -switch port is enabled at all, the ENETC PF2 (enetc_port2) should be enabled as -well, regardless of whether it is configured as the DSA master or not. This is -because the Felix PHYLINK implementation accesses the MAC PCS registers, which -in hardware truly belong to the ENETC port #2 and not to Felix. - -Supported PHY interface types (appropriate SerDes protocol setting changes are -needed in the RCW binary): - -* phy_mode = "internal": on ports 4 and 5 -* phy_mode = "sgmii": on ports 0, 1, 2, 3 -* phy_mode = "qsgmii": on ports 0, 1, 2, 3 -* phy_mode = "usxgmii": on ports 0, 1, 2, 3 -* phy_mode = "2500base-x": on ports 0, 1, 2, 3 - -For the rest of the device tree binding definitions, which are standard DSA and -PCI, refer to the following documents: - -Documentation/devicetree/bindings/net/dsa/dsa.txt -Documentation/devicetree/bindings/pci/pci.txt - -Example: - -&soc { - pcie@1f0000000 { /* Integrated Endpoint Root Complex */ - ethernet-switch@0,5 { - reg = <0x000500 0 0 0 0>; - /* IEP INT_B */ - interrupts = ; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - /* External ports */ - port@0 { - reg = <0>; - label = "swp0"; - }; - - port@1 { - reg = <1>; - label = "swp1"; - }; - - port@2 { - reg = <2>; - label = "swp2"; - }; - - port@3 { - reg = <3>; - label = "swp3"; - }; - - /* Tagging CPU port */ - port@4 { - reg = <4>; - ethernet = <&enetc_port2>; - phy-mode = "internal"; - - fixed-link { - speed = <2500>; - full-duplex; - }; - }; - - /* Non-tagging CPU port */ - port@5 { - reg = <5>; - phy-mode = "internal"; - status = "disabled"; - - fixed-link { - speed = <1000>; - full-duplex; - }; - }; - }; - }; - }; -}; - -The VSC9953 switch is found inside NXP T1040. It is a platform device with the -following required properties: - -- compatible: - Must be "mscc,vsc9953-switch". - -Supported PHY interface types (appropriate SerDes protocol setting changes are -needed in the RCW binary): - -* phy_mode = "internal": on ports 8 and 9 -* phy_mode = "sgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7 -* phy_mode = "qsgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7 - -Example: - -&soc { - ethernet-switch@800000 { - #address-cells = <0x1>; - #size-cells = <0x0>; - compatible = "mscc,vsc9953-switch"; - little-endian; - reg = <0x800000 0x290000>; - - ports { - #address-cells = <0x1>; - #size-cells = <0x0>; - - port@0 { - reg = <0x0>; - label = "swp0"; - }; - - port@1 { - reg = <0x1>; - label = "swp1"; - }; - - port@2 { - reg = <0x2>; - label = "swp2"; - }; - - port@3 { - reg = <0x3>; - label = "swp3"; - }; - - port@4 { - reg = <0x4>; - label = "swp4"; - }; - - port@5 { - reg = <0x5>; - label = "swp5"; - }; - - port@6 { - reg = <0x6>; - label = "swp6"; - }; - - port@7 { - reg = <0x7>; - label = "swp7"; - }; - - port@8 { - reg = <0x8>; - phy-mode = "internal"; - ethernet = <&enet0>; - - fixed-link { - speed = <2500>; - full-duplex; - }; - }; - - port@9 { - reg = <0x9>; - phy-mode = "internal"; - status = "disabled"; - - fixed-link { - speed = <2500>; - full-duplex; - }; - }; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml index f3c88371d76ceb5c59ff682ccec790bc60e31562..978162df51f73429adf9bdf0bce9a510023a2f1c 100644 --- a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml +++ b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml @@ -159,7 +159,6 @@ examples: port@0 { reg = <0>; - label = "cpu"; ethernet = <&gmac1>; phy-mode = "rgmii"; @@ -221,7 +220,6 @@ examples: port@0 { reg = <0>; - label = "cpu"; ethernet = <&gmac1>; phy-mode = "rgmii"; @@ -268,7 +266,6 @@ examples: port@6 { reg = <0>; - label = "cpu"; ethernet = <&gmac1>; phy-mode = "sgmii"; diff --git a/Documentation/devicetree/bindings/net/dsa/realtek.yaml b/Documentation/devicetree/bindings/net/dsa/realtek.yaml index 4f99aff029dc9d8cbf2dcff3fc1a0cfe4ee35678..1a7d45a8ad66a557caab9a4639d2d5874a6a745d 100644 --- a/Documentation/devicetree/bindings/net/dsa/realtek.yaml +++ b/Documentation/devicetree/bindings/net/dsa/realtek.yaml @@ -189,7 +189,6 @@ examples: }; port@5 { reg = <5>; - label = "cpu"; ethernet = <&gmac0>; phy-mode = "rgmii"; fixed-link { @@ -277,7 +276,6 @@ examples: }; port@6 { reg = <6>; - label = "cpu"; ethernet = <&fec1>; phy-mode = "rgmii"; tx-internal-delay-ps = <2000>; diff --git a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml index 4d428f5ad044ba05ade0d5df9de6a5ff5e14c4c4..7ca9c19a157c62b7361e28640f82693430ca0efb 100644 --- a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml +++ b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml @@ -130,7 +130,8 @@ examples: port@4 { reg = <4>; ethernet = <&gmac2>; - label = "cpu"; + phy-mode = "internal"; + fixed-link { speed = <1000>; full-duplex; diff --git a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt index bbf4a13f6d755809a39ff110ad9b7d2d290c7f4f..258bef483673f45d3f1c2993bf321b903a086d9c 100644 --- a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt +++ b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt @@ -75,7 +75,6 @@ switch@0 { }; vsc: port@6 { reg = <6>; - label = "cpu"; ethernet = <&gmac1>; phy-mode = "rgmii"; fixed-link { @@ -117,7 +116,6 @@ switch@2,0 { }; vsc: port@6 { reg = <6>; - label = "cpu"; ethernet = <&enet0>; phy-mode = "rgmii"; fixed-link { diff --git a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml index d0e1476e15b50729f6651001a2cfa55ffafc99da..5bd964a46a9d0324b92a7fbfd3d0ec0e925a6d7d 100644 --- a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml +++ b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml @@ -20,7 +20,26 @@ properties: maxItems: 1 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 8 + + interrupt-names: + minItems: 1 + items: + - const: mac + - const: txrx-1 + - const: txrx-2 + - const: txrx-3 + - const: txrx-4 + - const: txrx-5 + - const: txrx-6 + - const: txrx-7 + description: + The main interrupt for basic MAC features and the first TX/RX queue pair + is named "mac". "txrx-[1-7]" are the interrupts for additional TX/RX + queue pairs. + + dma-coherent: true local-mac-address: true @@ -58,7 +77,7 @@ examples: axi { #address-cells = <2>; #size-cells = <2>; - tnsep0: ethernet@a0000000 { + tsnep0: ethernet@a0000000 { compatible = "engleder,tsnep"; reg = <0x0 0xa0000000 0x0 0x10000>; interrupts = <0 89 1>; @@ -76,4 +95,24 @@ examples: }; }; }; + + tsnep1: ethernet@a0010000 { + compatible = "engleder,tsnep"; + reg = <0x0 0xa0010000 0x0 0x10000>; + interrupts = <0 93 1>, <0 94 1>, <0 95 1>, <0 96 1>; + interrupt-names = "mac", "txrx-1", "txrx-2", "txrx-3"; + interrupt-parent = <&gic>; + local-mac-address = [00 00 00 00 00 00]; + phy-mode = "rgmii"; + phy-handle = <&phy1>; + mdio { + #address-cells = <1>; + #size-cells = <0>; + suppress-preamble; + phy1: ethernet-phy@1 { + reg = <1>; + rxc-skew-ps = <1080>; + }; + }; + }; }; diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml index c138a1022879764af2c7e813b1a3688f67c24285..4b3c590fcebf06d1417245b1343a8ae73077c66b 100644 --- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml @@ -67,6 +67,7 @@ properties: - gmii - sgmii - qsgmii + - qusgmii - tbi - rev-mii - rmii diff --git a/Documentation/devicetree/bindings/net/ethernet-phy.yaml b/Documentation/devicetree/bindings/net/ethernet-phy.yaml index ed1415a4381f207b8d27324222a2347a4c51ad7a..ad808e9ce5b9ed5cecc9f8fc3c673cf9c9886542 100644 --- a/Documentation/devicetree/bindings/net/ethernet-phy.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-phy.yaml @@ -144,6 +144,12 @@ properties: Mark the corresponding energy efficient ethernet mode as broken and request the ethernet to stop advertising it. + pses: + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 + description: + Specifies a reference to a node representing a Power Sourcing Equipment. + phy-is-integrated: $ref: /schemas/types.yaml#/definitions/flag description: diff --git a/Documentation/devicetree/bindings/net/fsl,fec.yaml b/Documentation/devicetree/bindings/net/fsl,fec.yaml index 5cfb661be12408e1889244864d43702eb3e14727..e0f376f7e274c617677ea63895e80d92fd3216d7 100644 --- a/Documentation/devicetree/bindings/net/fsl,fec.yaml +++ b/Documentation/devicetree/bindings/net/fsl,fec.yaml @@ -21,6 +21,7 @@ properties: - fsl,imx28-fec - fsl,imx6q-fec - fsl,mvf600-fec + - fsl,s32v234-fec - items: - enum: - fsl,imx53-fec diff --git a/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a35ac1c260d7a4b4a6a711aadd93d634f556126 --- /dev/null +++ b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/fsl,fman-dtsec.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP FMan MAC + +maintainers: + - Madalin Bucur + +description: | + Each FMan has several MACs, each implementing an Ethernet interface. Earlier + versions of FMan used the Datapath Three Speed Ethernet Controller (dTSEC) for + 10/100/1000 MBit/s speeds, and the 10-Gigabit Ethernet Media Access Controller + (10GEC) for 10 Gbit/s speeds. Later versions of FMan use the Multirate + Ethernet Media Access Controller (mEMAC) to handle all speeds. + +properties: + compatible: + enum: + - fsl,fman-dtsec + - fsl,fman-xgec + - fsl,fman-memac + + cell-index: + maximum: 64 + description: | + FManV2: + register[bit] MAC cell-index + ============================================================ + FM_EPI[16] XGEC 8 + FM_EPI[16+n] dTSECn n-1 + FM_NPI[11+n] dTSECn n-1 + n = 1,..,5 + + FManV3: + register[bit] MAC cell-index + ============================================================ + FM_EPI[16+n] mEMACn n-1 + FM_EPI[25] mEMAC10 9 + + FM_NPI[11+n] mEMACn n-1 + FM_NPI[10] mEMAC10 9 + FM_NPI[11] mEMAC9 8 + n = 1,..8 + + FM_EPI and FM_NPI are located in the FMan memory map. + + 2. SoC registers: + + - P2041, P3041, P4080 P5020, P5040: + register[bit] FMan MAC cell + Unit index + ============================================================ + DCFG_DEVDISR2[7] 1 XGEC 8 + DCFG_DEVDISR2[7+n] 1 dTSECn n-1 + DCFG_DEVDISR2[15] 2 XGEC 8 + DCFG_DEVDISR2[15+n] 2 dTSECn n-1 + n = 1,..5 + + - T1040, T2080, T4240, B4860: + register[bit] FMan MAC cell + Unit index + ============================================================ + DCFG_CCSR_DEVDISR2[n-1] 1 mEMACn n-1 + DCFG_CCSR_DEVDISR2[11+n] 2 mEMACn n-1 + n = 1,..6,9,10 + + EVDISR, DCFG_DEVDISR2 and DCFG_CCSR_DEVDISR2 are located in + the specific SoC "Device Configuration/Pin Control" Memory + Map. + + reg: + maxItems: 1 + + fsl,fman-ports: + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 2 + description: | + An array of two references: the first is the FMan RX port and the second + is the TX port used by this MAC. + + ptp-timer: + $ref: /schemas/types.yaml#/definitions/phandle + description: A reference to the IEEE1588 timer + + pcsphy-handle: + $ref: /schemas/types.yaml#/definitions/phandle + description: A reference to the PCS (typically found on the SerDes) + + tbi-handle: + $ref: /schemas/types.yaml#/definitions/phandle + description: A reference to the (TBI-based) PCS + +required: + - compatible + - cell-index + - reg + - fsl,fman-ports + - ptp-timer + +allOf: + - $ref: ethernet-controller.yaml# + - if: + properties: + compatible: + contains: + const: fsl,fman-dtsec + then: + required: + - tbi-handle + - if: + properties: + compatible: + contains: + const: fsl,fman-memac + then: + required: + - pcsphy-handle + +unevaluatedProperties: false + +examples: + - | + ethernet@e0000 { + compatible = "fsl,fman-dtsec"; + cell-index = <0>; + reg = <0xe0000 0x1000>; + fsl,fman-ports = <&fman1_rx8 &fman1_tx28>; + ptp-timer = <&ptp_timer>; + tbi-handle = <&tbi0>; + }; + - | + ethernet@e8000 { + cell-index = <4>; + compatible = "fsl,fman-memac"; + reg = <0xe8000 0x1000>; + fsl,fman-ports = <&fman0_rx_0x0c &fman0_tx_0x2c>; + ptp-timer = <&ptp_timer0>; + pcsphy-handle = <&pcsphy4>; + phy-handle = <&sgmii_phy1>; + phy-connection-type = "sgmii"; + }; +... diff --git a/Documentation/devicetree/bindings/net/fsl-fman.txt b/Documentation/devicetree/bindings/net/fsl-fman.txt index 801efc7d681811ee519288002ae9ede6978f8742..b9055335db3bc74732833e598ff65144a38f845c 100644 --- a/Documentation/devicetree/bindings/net/fsl-fman.txt +++ b/Documentation/devicetree/bindings/net/fsl-fman.txt @@ -232,133 +232,7 @@ port@81000 { ============================================================================= FMan dTSEC/XGEC/mEMAC Node -DESCRIPTION - -mEMAC/dTSEC/XGEC are the Ethernet network interfaces - -PROPERTIES - -- compatible - Usage: required - Value type: - Definition: A standard property. - Must include one of the following: - - "fsl,fman-dtsec" for dTSEC MAC - - "fsl,fman-xgec" for XGEC MAC - - "fsl,fman-memac" for mEMAC MAC - -- cell-index - Usage: required - Value type: - Definition: Specifies the MAC id. - - The cell-index value may be used by the FMan or the SoC, to - identify the MAC unit in the FMan (or SoC) memory map. - In the tables below there's a description of the cell-index - use, there are two tables, one describes the use of cell-index - by the FMan, the second describes the use by the SoC: - - 1. FMan Registers - - FManV2: - register[bit] MAC cell-index - ============================================================ - FM_EPI[16] XGEC 8 - FM_EPI[16+n] dTSECn n-1 - FM_NPI[11+n] dTSECn n-1 - n = 1,..,5 - - FManV3: - register[bit] MAC cell-index - ============================================================ - FM_EPI[16+n] mEMACn n-1 - FM_EPI[25] mEMAC10 9 - - FM_NPI[11+n] mEMACn n-1 - FM_NPI[10] mEMAC10 9 - FM_NPI[11] mEMAC9 8 - n = 1,..8 - - FM_EPI and FM_NPI are located in the FMan memory map. - - 2. SoC registers: - - - P2041, P3041, P4080 P5020, P5040: - register[bit] FMan MAC cell - Unit index - ============================================================ - DCFG_DEVDISR2[7] 1 XGEC 8 - DCFG_DEVDISR2[7+n] 1 dTSECn n-1 - DCFG_DEVDISR2[15] 2 XGEC 8 - DCFG_DEVDISR2[15+n] 2 dTSECn n-1 - n = 1,..5 - - - T1040, T2080, T4240, B4860: - register[bit] FMan MAC cell - Unit index - ============================================================ - DCFG_CCSR_DEVDISR2[n-1] 1 mEMACn n-1 - DCFG_CCSR_DEVDISR2[11+n] 2 mEMACn n-1 - n = 1,..6,9,10 - - EVDISR, DCFG_DEVDISR2 and DCFG_CCSR_DEVDISR2 are located in - the specific SoC "Device Configuration/Pin Control" Memory - Map. - -- reg - Usage: required - Value type: - Definition: A standard property. - -- fsl,fman-ports - Usage: required - Value type: - Definition: An array of two phandles - the first references is - the FMan RX port and the second is the TX port used by this - MAC. - -- ptp-timer - Usage required - Value type: - Definition: A phandle for 1EEE1588 timer. - -- pcsphy-handle - Usage required for "fsl,fman-memac" MACs - Value type: - Definition: A phandle for pcsphy. - -- tbi-handle - Usage required for "fsl,fman-dtsec" MACs - Value type: - Definition: A phandle for tbiphy. - -EXAMPLE - -fman1_tx28: port@a8000 { - cell-index = <0x28>; - compatible = "fsl,fman-v2-port-tx"; - reg = <0xa8000 0x1000>; -}; - -fman1_rx8: port@88000 { - cell-index = <0x8>; - compatible = "fsl,fman-v2-port-rx"; - reg = <0x88000 0x1000>; -}; - -ptp-timer: ptp_timer@fe000 { - compatible = "fsl,fman-ptp-timer"; - reg = <0xfe000 0x1000>; -}; - -ethernet@e0000 { - compatible = "fsl,fman-dtsec"; - cell-index = <0>; - reg = <0xe0000 0x1000>; - fsl,fman-ports = <&fman1_rx8 &fman1_tx28>; - ptp-timer = <&ptp-timer>; - tbi-handle = <&tbi0>; -}; +Refer to Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml ============================================================================ FMan IEEE 1588 Node diff --git a/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml b/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml index 52a7fa4f49a48359cfe2ea30e6c3af945c5d0f69..d23fa3771210a0b5475bc190769c815aadbf14b2 100644 --- a/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml +++ b/Documentation/devicetree/bindings/net/intel,dwmac-plat.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/intel,dwmac-plat.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel DWMAC glue layer Device Tree Bindings +title: Intel DWMAC glue layer maintainers: - Vineetha G. Jaya Kumaran diff --git a/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt b/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt deleted file mode 100644 index 358fed2fab434ff77834e04e28ef0b89c00cf9d8..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt +++ /dev/null @@ -1,24 +0,0 @@ -Mediatek Gigabit Switch -======================= - -The mediatek gigabit switch can be found on Mediatek SoCs (mt7620, mt7621). - -Required properties: -- compatible: Should be "mediatek,mt7620-gsw" or "mediatek,mt7621-gsw" -- reg: Address and length of the register set for the device -- interrupts: Should contain the gigabit switches interrupt -- resets: Should contain the gigabit switches resets -- reset-names: Should contain the reset names "gsw" - -Example: - -gsw@10110000 { - compatible = "ralink,mt7620-gsw"; - reg = <0x10110000 8000>; - - resets = <&rstctrl 23>; - reset-names = "gsw"; - - interrupt-parent = <&intc>; - interrupts = <17>; -}; diff --git a/Documentation/devicetree/bindings/net/mediatek,net.yaml b/Documentation/devicetree/bindings/net/mediatek,net.yaml index f5564ecddb6261a22c246edbdd6308b1b898f4e7..7ef696204c5a76c25fc836fbb359539748cd69ca 100644 --- a/Documentation/devicetree/bindings/net/mediatek,net.yaml +++ b/Documentation/devicetree/bindings/net/mediatek,net.yaml @@ -69,6 +69,15 @@ properties: A list of phandle to the syscon node that handles the SGMII setup which is required for those SoCs equipped with SGMII. + mediatek,wed: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 2 + maxItems: 2 + items: + maxItems: 1 + description: + List of phandles to wireless ethernet dispatch nodes. + dma-coherent: true mdio-bus: @@ -112,6 +121,8 @@ allOf: Phandle to the syscon node that handles the ports slew rate and driver current. + mediatek,wed: false + - if: properties: compatible: @@ -144,15 +155,6 @@ allOf: minItems: 1 maxItems: 1 - mediatek,wed: - $ref: /schemas/types.yaml#/definitions/phandle-array - minItems: 2 - maxItems: 2 - items: - maxItems: 1 - description: - List of phandles to wireless ethernet dispatch nodes. - mediatek,pcie-mirror: $ref: /schemas/types.yaml#/definitions/phandle description: @@ -202,6 +204,8 @@ allOf: minItems: 2 maxItems: 2 + mediatek,wed: false + - if: properties: compatible: @@ -238,6 +242,11 @@ allOf: minItems: 2 maxItems: 2 + mediatek,wed-pcie: + $ref: /schemas/types.yaml#/definitions/phandle + description: + Phandle to the mediatek wed-pcie controller. + patternProperties: "^mac@[0-1]$": type: object diff --git a/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml b/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml index 61b2fb9e141bf6e32c1c0707b17fbdf57e037f83..0fa2132fa4f4d16207f290153b4af6c5236b79d3 100644 --- a/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml +++ b/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml @@ -19,6 +19,7 @@ select: contains: enum: - mediatek,mt2712-gmac + - mediatek,mt8188-gmac - mediatek,mt8195-gmac required: - compatible @@ -37,6 +38,11 @@ properties: - enum: - mediatek,mt8195-gmac - const: snps,dwmac-5.10a + - items: + - enum: + - mediatek,mt8188-gmac + - const: mediatek,mt8195-gmac + - const: snps,dwmac-5.10a clocks: minItems: 5 @@ -74,7 +80,7 @@ properties: or will round down. Range 0~31*170. For MT2712 RMII/MII interface, Allowed value need to be a multiple of 550, or will round down. Range 0~31*550. - For MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple of 290, + For MT8188/MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple of 290, or will round down. Range 0~31*290. mediatek,rx-delay-ps: @@ -84,7 +90,7 @@ properties: or will round down. Range 0~31*170. For MT2712 RMII/MII interface, Allowed value need to be a multiple of 550, or will round down. Range 0~31*550. - For MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple + For MT8188/MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple of 290, or will round down. Range 0~31*290. mediatek,rmii-rxc: diff --git a/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml b/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml index 6c86d3d85e9910969b2b3fdcae5108072df7f7ca..57ffeb8fc876661e6f86cbf39c133e25e606f7f1 100644 --- a/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml +++ b/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml @@ -74,16 +74,20 @@ properties: ethernet-ports: type: object + additionalProperties: false + + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + patternProperties: "^port@[0-9a-f]+$": - type: object + $ref: /schemas/net/ethernet-controller.yaml# + unevaluatedProperties: false properties: - '#address-cells': - const: 1 - '#size-cells': - const: 0 - reg: description: Switch port number @@ -93,29 +97,11 @@ properties: phandle of a Ethernet SerDes PHY. This defines which SerDes instance will handle the Ethernet traffic. - phy-mode: - description: - This specifies the interface used by the Ethernet SerDes towards - the PHY or SFP. - microchip,bandwidth: description: Specifies bandwidth in Mbit/s allocated to the port. $ref: "/schemas/types.yaml#/definitions/uint32" maximum: 25000 - phy-handle: - description: - phandle of a Ethernet PHY. This is optional and if provided it - points to the cuPHY used by the Ethernet SerDes. - - sfp: - description: - phandle of an SFP. This is optional and used when not specifying - a cuPHY. It points to the SFP node that describes the SFP used by - the Ethernet SerDes. - - managed: true - microchip,sd-sgpio: description: Index of the ports Signal Detect SGPIO in the set of 384 SGPIOs @@ -144,8 +130,6 @@ required: - reg-names - interrupts - interrupt-names - - resets - - reset-names - ethernet-ports additionalProperties: false diff --git a/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml b/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml index a191a04e681c46d89a4e9cd9fa7a61494036261b..308485a8ee6ca6d34a17f251171729cc21f761f0 100644 --- a/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml +++ b/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml @@ -128,7 +128,7 @@ examples: i2c-int-rising; - reset-n-io = <&gpio3 19 GPIO_ACTIVE_HIGH>; + reset-n-io = <&gpio3 19 GPIO_ACTIVE_LOW>; }; }; @@ -151,7 +151,7 @@ examples: interrupt-parent = <&gpio1>; interrupts = <17 IRQ_TYPE_EDGE_RISING>; - reset-n-io = <&gpio3 19 GPIO_ACTIVE_HIGH>; + reset-n-io = <&gpio3 19 GPIO_ACTIVE_LOW>; }; }; @@ -162,7 +162,7 @@ examples: nfc { compatible = "marvell,nfc-uart"; - reset-n-io = <&gpio3 16 GPIO_ACTIVE_HIGH>; + reset-n-io = <&gpio3 16 GPIO_ACTIVE_LOW>; hci-muxed; flow-control; diff --git a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bd3efff2485ef69f0321501adf815943aa37d06 --- /dev/null +++ b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/nvidia,tegra234-mgbe.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Tegra234 MGBE Multi-Gigabit Ethernet Controller + +maintainers: + - Thierry Reding + - Jon Hunter + +properties: + compatible: + const: nvidia,tegra234-mgbe + + reg: + maxItems: 3 + + reg-names: + items: + - const: hypervisor + - const: mac + - const: xpcs + + interrupts: + minItems: 1 + maxItems: 3 + + interrupt-names: + minItems: 1 + items: + - const: common + - const: macsec-ns + - const: macsec + + clocks: + maxItems: 12 + + clock-names: + items: + - const: mgbe + - const: mac + - const: mac-divider + - const: ptp-ref + - const: rx-input-m + - const: rx-input + - const: tx + - const: eee-pcs + - const: rx-pcs-input + - const: rx-pcs-m + - const: rx-pcs + - const: tx-pcs + + resets: + maxItems: 2 + + reset-names: + items: + - const: mac + - const: pcs + + interconnects: + items: + - description: memory read client + - description: memory write client + + interconnect-names: + items: + - const: dma-mem + - const: write + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + phy-handle: true + + phy-mode: + contains: + enum: + - usxgmii + - 10gbase-kr + + mdio: + $ref: mdio.yaml# + unevaluatedProperties: false + description: + Optional node for embedded MDIO controller. + +required: + - compatible + - reg + - interrupts + - interrupt-names + - clocks + - clock-names + - resets + - reset-names + - power-domains + - phy-handle + - phy-mode + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + #include + + ethernet@6800000 { + compatible = "nvidia,tegra234-mgbe"; + reg = <0x06800000 0x10000>, + <0x06810000 0x10000>, + <0x068a0000 0x10000>; + reg-names = "hypervisor", "mac", "xpcs"; + interrupts = ; + interrupt-names = "common"; + clocks = <&bpmp TEGRA234_CLK_MGBE0_APP>, + <&bpmp TEGRA234_CLK_MGBE0_MAC>, + <&bpmp TEGRA234_CLK_MGBE0_MAC_DIVIDER>, + <&bpmp TEGRA234_CLK_MGBE0_PTP_REF>, + <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT_M>, + <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT>, + <&bpmp TEGRA234_CLK_MGBE0_TX>, + <&bpmp TEGRA234_CLK_MGBE0_EEE_PCS>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_INPUT>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>, + <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>; + clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", + "rx-pcs", "tx-pcs"; + resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>, + <&bpmp TEGRA234_RESET_MGBE0_PCS>; + reset-names = "mac", "pcs"; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEARD &emc>, + <&mc TEGRA234_MEMORY_CLIENT_MGBEAWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_MGBE>; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEA>; + + phy-handle = <&mgbe0_phy>; + phy-mode = "usxgmii"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + mgbe0_phy: phy@0 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0x0>; + + #phy-cells = <0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/nxp,dwmac-imx.yaml b/Documentation/devicetree/bindings/net/nxp,dwmac-imx.yaml index 0113631667896fa93ada89b637367d6b66b2918f..4c155441acbf013c2e93c50db39e6bc0ec1c0362 100644 --- a/Documentation/devicetree/bindings/net/nxp,dwmac-imx.yaml +++ b/Documentation/devicetree/bindings/net/nxp,dwmac-imx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/nxp,dwmac-imx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NXP i.MX8 DWMAC glue layer Device Tree Bindings +title: NXP i.MX8 DWMAC glue layer maintainers: - Joakim Zhang diff --git a/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml b/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml index d51da24f350544dd7def75e583508870ad755a98..ab8867e6939bf144e826f19d7829eb5ecf77b467 100644 --- a/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml +++ b/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml @@ -31,6 +31,22 @@ patternProperties: description: The ID number for the child PHY. Should be +1 of parent PHY. + nxp,rmii-refclk-in: + type: boolean + description: | + The REF_CLK is provided for both transmitted and received data + in RMII mode. This clock signal is provided by the PHY and is + typically derived from an external 25MHz crystal. Alternatively, + a 50MHz clock signal generated by an external oscillator can be + connected to pin REF_CLK. A third option is to connect a 25MHz + clock to pin CLK_IN_OUT. So, the REF_CLK should be configured + as input or output according to the actual circuit connection. + If present, indicates that the REF_CLK will be configured as + interface reference clock input when RMII mode enabled. + If not present, the REF_CLK will be configured as interface + reference clock output when RMII mode enabled. + Only supported on TJA1100 and TJA1101. + required: - reg @@ -44,6 +60,7 @@ examples: tja1101_phy0: ethernet-phy@4 { reg = <0x4>; + nxp,rmii-refclk-in; }; }; - | diff --git a/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml b/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b1c188abf7e1031b68d4fb10719cc01c5f3280 --- /dev/null +++ b/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/pse-pd/podl-pse-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Regulator based Power Sourcing Equipment + +maintainers: + - Oleksij Rempel + +description: Regulator based PoDL PSE controller. The device must be referenced + by the PHY node to control power injection to the Ethernet cable. + +allOf: + - $ref: "pse-controller.yaml#" + +properties: + compatible: + const: podl-pse-regulator + + '#pse-cells': + const: 0 + + pse-supply: + description: Power supply for the PSE controller + +additionalProperties: false + +required: + - compatible + - pse-supply + +examples: + - | + ethernet-pse { + compatible = "podl-pse-regulator"; + pse-supply = <®_t1l1>; + #pse-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml b/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b110abb42597e64a43584f1c9c3c0a39a96f6999 --- /dev/null +++ b/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/pse-pd/pse-controller.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Power Sourcing Equipment (PSE). + +description: Binding for the Power Sourcing Equipment (PSE) as defined in the + IEEE 802.3 specification. It is designed for hardware which is delivering + power over twisted pair/ethernet cable. The ethernet-pse nodes should be + used to describe PSE controller and referenced by the ethernet-phy node. + +maintainers: + - Oleksij Rempel + +properties: + $nodename: + pattern: "^ethernet-pse(@.*)?$" + + "#pse-cells": + description: + Used to uniquely identify a PSE instance within an IC. Will be + 0 on PSE nodes with only a single output and at least 1 on nodes + controlling several outputs. + enum: [0, 1] + +required: + - "#pse-cells" + +additionalProperties: true + +... diff --git a/Documentation/devicetree/bindings/net/qca,ar803x.yaml b/Documentation/devicetree/bindings/net/qca,ar803x.yaml index b3d4013b7ca6cae38e467427e9ae7a39056c5fcf..161d2891931652e3a2a128a250f49c18b57a95eb 100644 --- a/Documentation/devicetree/bindings/net/qca,ar803x.yaml +++ b/Documentation/devicetree/bindings/net/qca,ar803x.yaml @@ -40,6 +40,14 @@ properties: Only supported on the AR8031. type: boolean + qca,disable-hibernation-mode: + description: | + Disable Atheros AR803X PHYs hibernation mode. If present, indicates + that the hardware of PHY will not enter power saving mode when the + cable is disconnected. And the RX_CLK always keeps outputting a + valid clock. + type: boolean + qca,smarteee-tw-us-100m: description: EEE Tw parameter for 100M links. $ref: /schemas/types.yaml#/definitions/uint32 diff --git a/Documentation/devicetree/bindings/net/qcom,ipq4019-mdio.yaml b/Documentation/devicetree/bindings/net/qcom,ipq4019-mdio.yaml index 2af3043417725cbaf3361a32277d138128ea22cc..ad8b2b41c140370b80a2924caac2b21eb1bd2096 100644 --- a/Documentation/devicetree/bindings/net/qcom,ipq4019-mdio.yaml +++ b/Documentation/devicetree/bindings/net/qcom,ipq4019-mdio.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/qcom,ipq4019-mdio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Qualcomm IPQ40xx MDIO Controller Device Tree Bindings +title: Qualcomm IPQ40xx MDIO Controller maintainers: - Robert Marko diff --git a/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt b/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt deleted file mode 100644 index 9fe1a0a22e441c2720e8950b2e22df17ef12389a..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt +++ /dev/null @@ -1,59 +0,0 @@ -Ralink Frame Engine Ethernet controller -======================================= - -The Ralink frame engine ethernet controller can be found on Ralink and -Mediatek SoCs (RT288x, RT3x5x, RT366x, RT388x, rt5350, mt7620, mt7621, mt76x8). - -Depending on the SoC, there is a number of ports connected to the CPU port -directly and/or via a (gigabit-)switch. - -* Ethernet controller node - -Required properties: -- compatible: Should be one of "ralink,rt2880-eth", "ralink,rt3050-eth", - "ralink,rt3050-eth", "ralink,rt3883-eth", "ralink,rt5350-eth", - "mediatek,mt7620-eth", "mediatek,mt7621-eth" -- reg: Address and length of the register set for the device -- interrupts: Should contain the frame engines interrupt -- resets: Should contain the frame engines resets -- reset-names: Should contain the reset names "fe". If a switch is present - "esw" is also required. - - -* Ethernet port node - -Required properties: -- compatible: Should be "ralink,eth-port" -- reg: The number of the physical port -- phy-handle: reference to the node describing the phy - -Example: - -mdio-bus { - ... - phy0: ethernet-phy@0 { - phy-mode = "mii"; - reg = <0>; - }; -}; - -ethernet@400000 { - compatible = "ralink,rt2880-eth"; - reg = <0x00400000 10000>; - - #address-cells = <1>; - #size-cells = <0>; - - resets = <&rstctrl 18>; - reset-names = "fe"; - - interrupt-parent = <&cpuintc>; - interrupts = <5>; - - port@0 { - compatible = "ralink,eth-port"; - reg = <0>; - phy-handle = <&phy0>; - }; - -}; diff --git a/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt b/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt deleted file mode 100644 index 87e315856efa0c998f43ddb7df83292cd581f87b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt +++ /dev/null @@ -1,30 +0,0 @@ -Ralink Fast Ethernet Embedded Switch -==================================== - -The ralink fast ethernet embedded switch can be found on Ralink and Mediatek -SoCs (RT3x5x, RT5350, MT76x8). - -Required properties: -- compatible: Should be "ralink,rt3050-esw" -- reg: Address and length of the register set for the device -- interrupts: Should contain the embedded switches interrupt -- resets: Should contain the embedded switches resets -- reset-names: Should contain the reset names "esw" - -Optional properties: -- ralink,portmap: can be used to choose if the default switch setup is - llllw or wllll -- ralink,led_polarity: override the active high/low settings of the leds - -Example: - -esw@10110000 { - compatible = "ralink,rt3050-esw"; - reg = <0x10110000 8000>; - - resets = <&rstctrl 23>; - reset-names = "esw"; - - interrupt-parent = <&intc>; - interrupts = <17>; -}; diff --git a/Documentation/devicetree/bindings/net/realtek-bluetooth.yaml b/Documentation/devicetree/bindings/net/realtek-bluetooth.yaml index 157d606bf9cb6fb3924b0818c492ab86b47c774e..e329ef06e10f85a999c762276c9eed1f6cf435bf 100644 --- a/Documentation/devicetree/bindings/net/realtek-bluetooth.yaml +++ b/Documentation/devicetree/bindings/net/realtek-bluetooth.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/realtek-bluetooth.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: RTL8723BS/RTL8723CS/RTL8822CS Bluetooth Device Tree Bindings +title: RTL8723BS/RTL8723CS/RTL8822CS Bluetooth maintainers: - Vasily Khoruzhick diff --git a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml index acf347f3cdbe22847361dfa15d4be74b462f5eb3..3f41294f5997722f6827387dbbff2bc9891b89c7 100644 --- a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml +++ b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml @@ -40,9 +40,14 @@ properties: - renesas,etheravb-r8a77980 # R-Car V3H - renesas,etheravb-r8a77990 # R-Car E3 - renesas,etheravb-r8a77995 # R-Car D3 - - renesas,etheravb-r8a779a0 # R-Car V3U - const: renesas,etheravb-rcar-gen3 # R-Car Gen3 and RZ/G2 + - items: + - enum: + - renesas,etheravb-r8a779a0 # R-Car V3U + - renesas,etheravb-r8a779g0 # R-Car V4H + - const: renesas,etheravb-rcar-gen4 # R-Car Gen4 + - items: - enum: - renesas,etheravb-r9a09g011 # RZ/V2M @@ -207,7 +212,7 @@ allOf: - renesas,etheravb-r8a77965 - renesas,etheravb-r8a77970 - renesas,etheravb-r8a77980 - - renesas,etheravb-r8a779a0 + - renesas,etheravb-rcar-gen4 then: required: - tx-internal-delay-ps diff --git a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml index 083623c8d718111df59bf2c0f5bdb3b97dd557ff..42fb72b6909d1adb16e8a0c78054f5e09c8983f6 100644 --- a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml +++ b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml @@ -25,7 +25,9 @@ select: - rockchip,rk3368-gmac - rockchip,rk3399-gmac - rockchip,rk3568-gmac + - rockchip,rk3588-gmac - rockchip,rv1108-gmac + - rockchip,rv1126-gmac required: - compatible @@ -47,9 +49,11 @@ properties: - rockchip,rk3368-gmac - rockchip,rk3399-gmac - rockchip,rv1108-gmac + - rockchip,rv1126-gmac - items: - enum: - rockchip,rk3568-gmac + - rockchip,rk3588-gmac - const: snps,dwmac-4.20a clocks: @@ -81,6 +85,11 @@ properties: description: The phandle of the syscon node for the general register file. $ref: /schemas/types.yaml#/definitions/phandle + rockchip,php-grf: + description: + The phandle of the syscon node for the peripheral general register file. + $ref: /schemas/types.yaml#/definitions/phandle + tx_delay: description: Delay value for TXD timing. Range value is 0~0x7F, 0x30 as default. $ref: /schemas/types.yaml#/definitions/uint32 diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml index 491597c02edf2f441e681823feb26d1f6413ad68..13b984076af5381e25d39a2ff4585316d3442bc3 100644 --- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml +++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/snps,dwmac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Synopsys DesignWare MAC Device Tree Bindings +title: Synopsys DesignWare MAC maintainers: - Alexandre Torgue @@ -74,6 +74,7 @@ properties: - rockchip,rk3328-gmac - rockchip,rk3366-gmac - rockchip,rk3368-gmac + - rockchip,rk3588-gmac - rockchip,rk3399-gmac - rockchip,rv1108-gmac - snps,dwmac @@ -288,6 +289,11 @@ properties: is supported. For example, this is used in case of SGMII and MAC2MAC connection. + snps,clk-csr: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Frequency division factor for MDC clock. + mdio: $ref: mdio.yaml# unevaluatedProperties: false @@ -301,6 +307,60 @@ properties: required: - compatible + stmmac-axi-config: + type: object + unevaluatedProperties: false + description: + AXI BUS Mode parameters. + + properties: + snps,lpi_en: + $ref: /schemas/types.yaml#/definitions/flag + description: + enable Low Power Interface + + snps,xit_frm: + $ref: /schemas/types.yaml#/definitions/flag + description: + unlock on WoL + + snps,wr_osr_lmt: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + max write outstanding req. limit + + snps,rd_osr_lmt: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + max read outstanding req. limit + + snps,kbbe: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + do not cross 1KiB boundary. + + snps,blen: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + this is a vector of supported burst length. + minItems: 7 + maxItems: 7 + + snps,fb: + $ref: /schemas/types.yaml#/definitions/flag + description: + fixed-burst + + snps,mb: + $ref: /schemas/types.yaml#/definitions/flag + description: + mixed-burst + + snps,rb: + $ref: /schemas/types.yaml#/definitions/flag + description: + rebuild INCRx Burst + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml b/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml index 62dffee27c3d81089c301529a12d17a7715c8f5d..5b130097449b09b0ae2c6cecc1e0dfc9822065be 100644 --- a/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml +++ b/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/net/sunplus,sp7021-emac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Sunplus SP7021 Dual Ethernet MAC Device Tree Bindings +title: Sunplus SP7021 Dual Ethernet MAC maintainers: - Wells Lu @@ -32,6 +32,7 @@ properties: ethernet-ports: type: object + additionalProperties: false description: Ethernet ports to PHY properties: @@ -44,6 +45,7 @@ properties: patternProperties: "^port@[0-1]$": type: object + additionalProperties: false description: Port to PHY properties: diff --git a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml index 31bf825c65987c55186e56a4a1a9f50b9d313b34..e36c7817be69ef953138b6be3dc84b4f35f15b64 100644 --- a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml +++ b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/ti,cpsw-switch.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: TI SoC Ethernet Switch Controller (CPSW) Device Tree Bindings +title: TI SoC Ethernet Switch Controller (CPSW) maintainers: - Grygorii Strashko @@ -77,6 +77,8 @@ properties: ethernet-ports: type: object + additionalProperties: false + properties: '#address-cells': const: 1 @@ -89,6 +91,7 @@ properties: description: CPSW external ports $ref: ethernet-controller.yaml# + unevaluatedProperties: false properties: reg: @@ -117,6 +120,7 @@ properties: cpts: type: object + unevaluatedProperties: false description: The Common Platform Time Sync (CPTS) module diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml index b8281d8be9400fdccccbff88880e5f0114abc3f8..821974815dec9766fdb068115da440c73a556a5c 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/ti,k3-am654-cpsw-nuss.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: The TI AM654x/J721E/AM642x SoC Gigabit Ethernet MAC (Media Access Controller) Device Tree Bindings +title: The TI AM654x/J721E/AM642x SoC Gigabit Ethernet MAC (Media Access Controller) maintainers: - Grygorii Strashko @@ -55,6 +55,7 @@ properties: compatible: enum: - ti,am654-cpsw-nuss + - ti,j7200-cpswxg-nuss - ti,j721e-cpsw-nuss - ti,am642-cpsw-nuss @@ -110,16 +111,17 @@ properties: const: 0 patternProperties: - port@[1-2]: + "^port@[1-4]$": type: object description: CPSWxG NUSS external ports $ref: ethernet-controller.yaml# + unevaluatedProperties: false properties: reg: minimum: 1 - maximum: 2 + maximum: 4 description: CPSW port number phys: @@ -178,6 +180,19 @@ required: - '#address-cells' - '#size-cells' +allOf: + - if: + not: + properties: + compatible: + contains: + const: ti,j7200-cpswxg-nuss + then: + properties: + ethernet-ports: + patternProperties: + "^port@[3-4]$": false + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml index b783ad0d1f53f7699185a403d423607deefe7cfc..6230f576134bb9edfb9ace20f5b75290d9da5924 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/ti,k3-am654-cpts.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: The TI AM654x/J721E Common Platform Time Sync (CPTS) module Device Tree Bindings +title: The TI AM654x/J721E Common Platform Time Sync (CPTS) module maintainers: - Grygorii Strashko @@ -95,6 +95,7 @@ properties: refclk-mux: type: object + additionalProperties: false description: CPTS reference clock multiplexer clock properties: '#clock-cells': diff --git a/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml b/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml index 8156a9aeb58940a1b75a5922631714d96b925dc9..6a71f694cb554ceaabcb411ec1d14e9668ba95e2 100644 --- a/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml +++ b/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml @@ -4,10 +4,10 @@ $id: "http://devicetree.org/schemas/net/vertexcom-mse102x.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: The Vertexcom MSE102x (SPI) Device Tree Bindings +title: The Vertexcom MSE102x (SPI) maintainers: - - Stefan Wahren + - Stefan Wahren description: Vertexcom's MSE102x are a family of HomePlug GreenPHY chips. diff --git a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml index 53b4153d9bfcc78d0cb7d9eca1620baf427b3c37..fec1cc9b9a085184cf4200e0489fea0c6bae489e 100644 --- a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml +++ b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/net/wireless/brcm,bcm4329-fmac.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom BCM4329 family fullmac wireless SDIO devices +title: Broadcom BCM4329 family fullmac wireless SDIO/PCIE devices maintainers: - Arend van Spriel @@ -41,11 +41,17 @@ properties: - cypress,cyw4373-fmac - cypress,cyw43012-fmac - const: brcm,bcm4329-fmac - - const: brcm,bcm4329-fmac + - enum: + - brcm,bcm4329-fmac + - pci14e4,43dc # BCM4355 + - pci14e4,4464 # BCM4364 + - pci14e4,4488 # BCM4377 + - pci14e4,4425 # BCM4378 + - pci14e4,4433 # BCM4387 reg: - description: SDIO function number for the device, for most cases - this will be 1. + description: SDIO function number for the device (for most cases + this will be 1) or PCI device identifier. interrupts: maxItems: 1 @@ -85,6 +91,31 @@ properties: takes precedence. type: boolean + brcm,cal-blob: + $ref: /schemas/types.yaml#/definitions/uint8-array + description: A per-device calibration blob for the Wi-Fi radio. This + should be filled in by the bootloader from platform configuration + data, if necessary, and will be uploaded to the device if present. + + brcm,board-type: + $ref: /schemas/types.yaml#/definitions/string + description: Overrides the board type, which is normally the compatible of + the root node. This can be used to decouple the overall system board or + device name from the board type for WiFi purposes, which is used to + construct firmware and NVRAM configuration filenames, allowing for + multiple devices that share the same module or characteristics for the + WiFi subsystem to share the same firmware/NVRAM files. On Apple platforms, + this should be the Apple module-instance codename prefixed by "apple,", + e.g. "apple,honshu". + + apple,antenna-sku: + $ref: /schemas/types.yaml#/definitions/string + description: Antenna SKU used to identify a specific antenna configuration + on Apple platforms. This is use to build firmware filenames, to allow + platforms with different antenna configs to have different firmware and/or + NVRAM. This would normally be filled in by the bootloader from platform + configuration data. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml b/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml index 60de78f1bc7b9c4a8885d4fc4ade73ddc5f6dcbe..b3405f284580abe61fe7eaa0c1d0ae3015ed4d69 100644 --- a/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml +++ b/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml @@ -20,8 +20,6 @@ properties: reg: true - spi-max-frequency: true - interrupts: maxItems: 1 @@ -51,7 +49,10 @@ required: - compatible - interrupts -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml index a677b056f11206f8140972a6f9ab02cd92422399..f7cf135aa37fff9b2370edf1b03af3d16602dfcf 100644 --- a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml +++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml @@ -66,6 +66,18 @@ properties: required: - iommus + qcom,smem-states: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: State bits used by the AP to signal the WLAN Q6. + items: + - description: Signal bits used to enable/disable low power mode + on WCN6750 in the case of WoW (Wake on Wireless). + + qcom,smem-state-names: + description: The names of the state bits used for SMP2P output. + items: + - const: wlan-smp2p-out + required: - compatible - reg @@ -448,6 +460,8 @@ examples: ; qcom,rproc = <&remoteproc_wpss>; memory-region = <&wlan_fw_mem>, <&wlan_ce_mem>; + qcom,smem-states = <&wlan_smp2p_out 0>; + qcom,smem-state-names = "wlan-smp2p-out"; wifi-firmware { iommus = <&apps_smmu 0x1c02 0x1>; }; diff --git a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml index 76199a67d6282b4dc5e7ce7a26388280aa04d66e..b35d2f3ad1adcc1a703aaa9c69d66670269206bc 100644 --- a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml +++ b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml @@ -29,12 +29,6 @@ description: > Documentation/devicetree/bindings/mmc/mmc-pwrseq-simple.yaml for more information. - For SPI: - - In add of the properties below, please consult - Documentation/devicetree/bindings/spi/spi-controller.yaml for optional SPI - related properties. - properties: compatible: items: @@ -52,8 +46,6 @@ properties: bindings. maxItems: 1 - spi-max-frequency: true - interrupts: description: The interrupt line. Should be IRQ_TYPE_EDGE_RISING. When SPI is used, this property is required. When SDIO is used, the "in-band" @@ -84,12 +76,15 @@ properties: mac-address: true -additionalProperties: false - required: - compatible - reg +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + +unevaluatedProperties: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml b/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml index d68bb2ec1f7ef30a6c204bf9e0c7faa8ab137884..e31456730e9fa3531e650f0c2a131bf189134c9e 100644 --- a/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml +++ b/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml @@ -36,8 +36,6 @@ properties: This is required when connected via SPI, and optional when connected via SDIO. - spi-max-frequency: true - interrupts: minItems: 1 maxItems: 2 @@ -69,20 +67,22 @@ required: - compatible - interrupts -if: - properties: - compatible: - contains: - enum: - - ti,wl1271 - - ti,wl1273 - - ti,wl1281 - - ti,wl1283 -then: - required: - - ref-clock-frequency - -additionalProperties: false +allOf: + - $ref: /schemas/spi/spi-peripheral-props.yaml# + - if: + properties: + compatible: + contains: + enum: + - ti,wl1271 + - ti,wl1273 + - ti,wl1281 + - ti,wl1283 + then: + required: + - ref-clock-frequency + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/nvmem/allwinner,sun4i-a10-sid.yaml b/Documentation/devicetree/bindings/nvmem/allwinner,sun4i-a10-sid.yaml index e558587ff88540e2ae3ae6127a9f0b39c0425fc0..14c170c6a86ed76d9f2529adb69dc0551d25656f 100644 --- a/Documentation/devicetree/bindings/nvmem/allwinner,sun4i-a10-sid.yaml +++ b/Documentation/devicetree/bindings/nvmem/allwinner,sun4i-a10-sid.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/allwinner,sun4i-a10-sid.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Security ID Device Tree Bindings +title: Allwinner A10 Security ID maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/nvmem/imx-iim.yaml b/Documentation/devicetree/bindings/nvmem/imx-iim.yaml index 9cc43e7a4b38177d61ecf0758473d77797b329e2..7aac1995cfaf153a42655be2efffcd73e9652d8d 100644 --- a/Documentation/devicetree/bindings/nvmem/imx-iim.yaml +++ b/Documentation/devicetree/bindings/nvmem/imx-iim.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/imx-iim.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Freescale i.MX IC Identification Module (IIM) device tree bindings +title: Freescale i.MX IC Identification Module (IIM) maintainers: - Anson Huang diff --git a/Documentation/devicetree/bindings/nvmem/imx-ocotp.yaml b/Documentation/devicetree/bindings/nvmem/imx-ocotp.yaml index 8a43dc1283fe5b5c12f0ccaa5b86fc764705d048..d0a239d7e199e3c161294eac1184c9aecc52fddd 100644 --- a/Documentation/devicetree/bindings/nvmem/imx-ocotp.yaml +++ b/Documentation/devicetree/bindings/nvmem/imx-ocotp.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/imx-ocotp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Freescale i.MX6 On-Chip OTP Controller (OCOTP) device tree bindings +title: Freescale i.MX6 On-Chip OTP Controller (OCOTP) maintainers: - Anson Huang diff --git a/Documentation/devicetree/bindings/nvmem/mediatek,efuse.yaml b/Documentation/devicetree/bindings/nvmem/mediatek,efuse.yaml index b5a1109f2ee122f83bf67e6996fc7275af5069ed..75e0a516e59a28c06908616e184ab03362a5d150 100644 --- a/Documentation/devicetree/bindings/nvmem/mediatek,efuse.yaml +++ b/Documentation/devicetree/bindings/nvmem/mediatek,efuse.yaml @@ -30,6 +30,7 @@ properties: - mediatek,mt8173-efuse - mediatek,mt8183-efuse - mediatek,mt8186-efuse + - mediatek,mt8188-efuse - mediatek,mt8192-efuse - mediatek,mt8195-efuse - mediatek,mt8516-efuse diff --git a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f97c6beb4766a3496be6dc3f800c0c0017896685 --- /dev/null +++ b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvmem/microchip,lan9662-otpc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip LAN9662 OTP Controller (OTPC) + +maintainers: + - Horatiu Vultur + +description: | + OTP controller drives a NVMEM memory where system specific data + (e.g. hardware configuration settings, chip identifiers) or + user specific data could be stored. + +allOf: + - $ref: nvmem.yaml# + +properties: + compatible: + oneOf: + - items: + - const: microchip,lan9668-otpc + - const: microchip,lan9662-otpc + - enum: + - microchip,lan9662-otpc + + reg: + maxItems: 1 + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + otpc: otp@e0021000 { + compatible = "microchip,lan9662-otpc"; + reg = <0xe0021000 0x300>; + }; + +... diff --git a/Documentation/devicetree/bindings/nvmem/nintendo-otp.yaml b/Documentation/devicetree/bindings/nvmem/nintendo-otp.yaml index dbe4ffdd644c5f59de6f635a93030cd44ae8dc9e..f93bc50c40d70d4e16eaaa3102bd70e858e98914 100644 --- a/Documentation/devicetree/bindings/nvmem/nintendo-otp.yaml +++ b/Documentation/devicetree/bindings/nvmem/nintendo-otp.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/nintendo-otp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Nintendo Wii and Wii U OTP Device Tree Bindings +title: Nintendo Wii and Wii U OTP description: | This binding represents the OTP memory as found on a Nintendo Wii or Wii U, diff --git a/Documentation/devicetree/bindings/nvmem/nvmem-consumer.yaml b/Documentation/devicetree/bindings/nvmem/nvmem-consumer.yaml index b1da238c8bcb3b7ee8c9a984d75235662db6516e..a26633bf52dbf7c6c8c059a368e27fc1a2bdd81a 100644 --- a/Documentation/devicetree/bindings/nvmem/nvmem-consumer.yaml +++ b/Documentation/devicetree/bindings/nvmem/nvmem-consumer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/nvmem-consumer.yaml# $schema: http://devicetree.org/meta-schemas/base.yaml# -title: NVMEM (Non Volatile Memory) Consumer Device Tree Bindings +title: NVMEM (Non Volatile Memory) Consumer maintainers: - Srinivas Kandagatla diff --git a/Documentation/devicetree/bindings/nvmem/nvmem.yaml b/Documentation/devicetree/bindings/nvmem/nvmem.yaml index 3bb349c634cbccb61ad77e7b1e825713e2c677ff..1eb22dba364c75f82c4368e0727c23a3f208489f 100644 --- a/Documentation/devicetree/bindings/nvmem/nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/nvmem.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/nvmem.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NVMEM (Non Volatile Memory) Device Tree Bindings +title: NVMEM (Non Volatile Memory) maintainers: - Srinivas Kandagatla diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml index dede8892ee0133e412474611dd6a2efd8ee0f682..2eab2f46cb65ad397aa7cb7133b347bc56bcbd47 100644 --- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml +++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml @@ -18,6 +18,7 @@ properties: - enum: - qcom,apq8064-qfprom - qcom,apq8084-qfprom + - qcom,ipq8064-qfprom - qcom,msm8974-qfprom - qcom,msm8916-qfprom - qcom,msm8996-qfprom @@ -25,7 +26,9 @@ properties: - qcom,qcs404-qfprom - qcom,sc7180-qfprom - qcom,sc7280-qfprom + - qcom,sdm630-qfprom - qcom,sdm845-qfprom + - qcom,sm6115-qfprom - const: qcom,qfprom reg: diff --git a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml index 104dd508565e62a6066b1a7ba237d4d89bc53837..febee8129aa9b17a3978b3b3b25d235d9477d840 100644 --- a/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml +++ b/Documentation/devicetree/bindings/nvmem/rockchip-efuse.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/nvmem/rockchip-efuse.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip eFuse device tree bindings +title: Rockchip eFuse maintainers: - Heiko Stuebner diff --git a/Documentation/devicetree/bindings/nvmem/u-boot,env.yaml b/Documentation/devicetree/bindings/nvmem/u-boot,env.yaml index e70b2a60cb9a688844fc317cea3656c282f5ebc4..e96bca99f2d94346a29da3f2bdde8bd21be92bd6 100644 --- a/Documentation/devicetree/bindings/nvmem/u-boot,env.yaml +++ b/Documentation/devicetree/bindings/nvmem/u-boot,env.yaml @@ -24,6 +24,8 @@ description: | Right now only flash partition case is covered but it may be extended to e.g. UBI volumes in the future. + Variables can be defined as NVMEM device subnodes. + maintainers: - Rafał Miłecki @@ -40,6 +42,14 @@ properties: reg: maxItems: 1 + bootcmd: + type: object + description: Command to use for automatic booting + + ethaddr: + type: object + description: Ethernet interface's MAC address + additionalProperties: false examples: @@ -58,5 +68,8 @@ examples: env: partition@40000 { compatible = "u-boot,env"; reg = <0x40000 0x10000>; + + mac: ethaddr { + }; }; }; diff --git a/Documentation/devicetree/bindings/opp/allwinner,sun50i-h6-operating-points.yaml b/Documentation/devicetree/bindings/opp/allwinner,sun50i-h6-operating-points.yaml index 729ae97b63d9c0fce4c1acc5b39020368a53c2e8..385b0692261cb54d0df598c27c7a3f0fce61f91d 100644 --- a/Documentation/devicetree/bindings/opp/allwinner,sun50i-h6-operating-points.yaml +++ b/Documentation/devicetree/bindings/opp/allwinner,sun50i-h6-operating-points.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/opp/allwinner,sun50i-h6-operating-points.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner H6 CPU OPP Device Tree Bindings +title: Allwinner H6 CPU OPP maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml index 59663e897dae951202cc3cba9b5a79c7bfb208c9..a202b6c6561d8a59c47daeeacec0d5814177fdff 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-kryo-cpu.yaml @@ -40,6 +40,7 @@ properties: patternProperties: '^opp-?[0-9]+$': type: object + additionalProperties: false properties: opp-hz: true diff --git a/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml b/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml index 14a7a689ad6d0ae47f72cb9eb0ee856444e13615..df8442fb11f0c992d1beb61cb482db5409c35edb 100644 --- a/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml +++ b/Documentation/devicetree/bindings/opp/opp-v2-qcom-level.yaml @@ -19,6 +19,7 @@ properties: patternProperties: '^opp-?[0-9]+$': type: object + additionalProperties: false properties: opp-level: true diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 0f064e4222f3ebd1fef99478c030d4b21fa2bd85..7e15aae7d69e9780dc2fc7adfa28b0ebec0acaaa 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pci/brcm,stb-pcie.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Brcmstb PCIe Host Controller Device Tree Bindings +title: Brcmstb PCIe Host Controller maintainers: - Nicolas Saenz Julienne diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml index 0499b94627aea62088a3f481ebefd541a1167735..c00be39af64e53e801a5d4d551235b5e16f504fe 100644 --- a/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml +++ b/Documentation/devicetree/bindings/pci/mediatek-pcie-gen3.yaml @@ -48,7 +48,13 @@ allOf: properties: compatible: - const: mediatek,mt8192-pcie + oneOf: + - items: + - enum: + - mediatek,mt8188-pcie + - mediatek,mt8195-pcie + - const: mediatek,mt8192-pcie + - const: mediatek,mt8192-pcie reg: maxItems: 1 @@ -84,7 +90,9 @@ properties: - const: tl_96m - const: tl_32k - const: peri_26m - - const: top_133m + - enum: + - top_133m # for MT8192 + - peri_mem # for MT8188/MT8195 assigned-clocks: maxItems: 1 @@ -126,6 +134,7 @@ required: - interrupts - ranges - clocks + - clock-names - '#interrupt-cells' - interrupt-controller diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml index edb4f81253c8eaf3bd9df500a234fe13133b0de0..f7a3c26363556d4daf3050db4da74c221c7711db 100644 --- a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pci/microchip,pcie-host.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip PCIe Root Port Bridge Controller Device Tree Bindings +title: Microchip PCIe Root Port Bridge Controller maintainers: - Daire McNamara @@ -25,6 +25,33 @@ properties: - const: cfg - const: apb + clocks: + description: + Fabric Interface Controllers, FICs, are the interface between the FPGA + fabric and the core complex on PolarFire SoC. The FICs require two clocks, + one from each side of the interface. The "FIC clocks" described by this + property are on the core complex side & communication through a FIC is not + possible unless it's corresponding clock is enabled. A clock must be + enabled for each of the interfaces the root port is connected through. + This could in theory be all 4 interfaces, one interface or any combination + in between. + minItems: 1 + items: + - description: FIC0's clock + - description: FIC1's clock + - description: FIC2's clock + - description: FIC3's clock + + clock-names: + description: + As any FIC connection combination is possible, the names should match the + order in the clocks property and take the form "ficN" where N is a number + 0-3 + minItems: 1 + maxItems: 4 + items: + pattern: '^fic[0-3]$' + interrupts: minItems: 1 items: @@ -40,6 +67,10 @@ properties: ranges: maxItems: 1 + dma-ranges: + minItems: 1 + maxItems: 6 + msi-controller: description: Identifies the node as an MSI controller. diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml index 3d23599e5e915854bd2376229c2e8f0f015f0fcb..977c976ea799467810843d2eef76aa72dcf1758f 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie-ep.yaml @@ -9,12 +9,11 @@ title: Qualcomm PCIe Endpoint Controller binding maintainers: - Manivannan Sadhasivam -allOf: - - $ref: "pci-ep.yaml#" - properties: compatible: - const: qcom,sdx55-pcie-ep + enum: + - qcom,sdx55-pcie-ep + - qcom,sm8450-pcie-ep reg: items: @@ -35,24 +34,12 @@ properties: - const: mmio clocks: - items: - - description: PCIe Auxiliary clock - - description: PCIe CFG AHB clock - - description: PCIe Master AXI clock - - description: PCIe Slave AXI clock - - description: PCIe Slave Q2A AXI clock - - description: PCIe Sleep clock - - description: PCIe Reference clock + minItems: 7 + maxItems: 8 clock-names: - items: - - const: aux - - const: cfg - - const: bus_master - - const: bus_slave - - const: slave_q2a - - const: sleep - - const: ref + minItems: 7 + maxItems: 8 qcom,perst-regs: description: Reference to a syscon representing TCSR followed by the two @@ -105,7 +92,6 @@ required: - reg-names - clocks - clock-names - - qcom,perst-regs - interrupts - interrupt-names - reset-gpios @@ -113,6 +99,64 @@ required: - reset-names - power-domains +allOf: + - $ref: pci-ep.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,sdx55-pcie-ep + then: + properties: + clocks: + items: + - description: PCIe Auxiliary clock + - description: PCIe CFG AHB clock + - description: PCIe Master AXI clock + - description: PCIe Slave AXI clock + - description: PCIe Slave Q2A AXI clock + - description: PCIe Sleep clock + - description: PCIe Reference clock + clock-names: + items: + - const: aux + - const: cfg + - const: bus_master + - const: bus_slave + - const: slave_q2a + - const: sleep + - const: ref + + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8450-pcie-ep + then: + properties: + clocks: + items: + - description: PCIe Auxiliary clock + - description: PCIe CFG AHB clock + - description: PCIe Master AXI clock + - description: PCIe Slave AXI clock + - description: PCIe Slave Q2A AXI clock + - description: PCIe Reference clock + - description: PCIe DDRSS SF TBU clock + - description: PCIe AGGRE NOC AXI clock + clock-names: + items: + - const: aux + - const: cfg + - const: bus_master + - const: bus_slave + - const: slave_q2a + - const: ref + - const: ddrss_sf_tbu + - const: aggre_noc_axi + unevaluatedProperties: false examples: diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml index 7d29e2a45183e05d1a7dd4a0b6f4c7f2ae85593d..54f07852d279e7bcdf28660935c6baada64cd4ed 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml @@ -25,8 +25,10 @@ properties: - qcom,pcie-ipq4019 - qcom,pcie-ipq8074 - qcom,pcie-qcs404 + - qcom,pcie-sa8540p - qcom,pcie-sc7280 - qcom,pcie-sc8180x + - qcom,pcie-sc8280xp - qcom,pcie-sdm845 - qcom,pcie-sm8150 - qcom,pcie-sm8250 @@ -54,11 +56,11 @@ properties: # Platform constraints are described later. clocks: minItems: 3 - maxItems: 12 + maxItems: 13 clock-names: minItems: 3 - maxItems: 12 + maxItems: 13 resets: minItems: 1 @@ -181,6 +183,7 @@ allOf: enum: - qcom,pcie-sc7280 - qcom,pcie-sc8180x + - qcom,pcie-sc8280xp - qcom,pcie-sm8250 - qcom,pcie-sm8450-pcie0 - qcom,pcie-sm8450-pcie1 @@ -424,8 +427,8 @@ allOf: then: properties: clocks: - minItems: 11 - maxItems: 11 + minItems: 13 + maxItems: 13 clock-names: items: - const: pipe # PIPE clock @@ -439,6 +442,8 @@ allOf: - const: slave_q2a # Slave Q2A clock - const: tbu # PCIe TBU clock - const: ddrss_sf_tbu # PCIe SF TBU clock + - const: aggre0 # Aggre NoC PCIe CENTER SF AXI clock + - const: aggre1 # Aggre NoC PCIe1 AXI clock resets: maxItems: 1 reset-names: @@ -596,6 +601,36 @@ allOf: items: - const: pci # PCIe core reset + - if: + properties: + compatible: + contains: + enum: + - qcom,pcie-sa8540p + - qcom,pcie-sc8280xp + then: + properties: + clocks: + minItems: 8 + maxItems: 9 + clock-names: + minItems: 8 + items: + - const: aux # Auxiliary clock + - const: cfg # Configuration clock + - const: bus_master # Master AXI clock + - const: bus_slave # Slave AXI clock + - const: slave_q2a # Slave Q2A clock + - const: ddrss_sf_tbu # PCIe SF TBU clock + - const: noc_aggr_4 # NoC aggregate 4 clock + - const: noc_aggr_south_sf # NoC aggregate South SF clock + - const: cnoc_qx # Configuration NoC QX clock + resets: + maxItems: 1 + reset-names: + items: + - const: pci # PCIe core reset + - if: not: properties: @@ -624,8 +659,6 @@ allOf: - resets - reset-names - # Newer chipsets support either 1 or 8 MSI vectors - # On older chipsets it's always 1 MSI vector - if: properties: compatible: @@ -660,7 +693,40 @@ allOf: - const: msi5 - const: msi6 - const: msi7 - else: + + - if: + properties: + compatible: + contains: + enum: + - qcom,pcie-sc8280xp + then: + properties: + interrupts: + minItems: 4 + maxItems: 4 + interrupt-names: + items: + - const: msi0 + - const: msi1 + - const: msi2 + - const: msi3 + + - if: + properties: + compatible: + contains: + enum: + - qcom,pcie-apq8064 + - qcom,pcie-apq8084 + - qcom,pcie-ipq4019 + - qcom,pcie-ipq6018 + - qcom,pcie-ipq8064 + - qcom,pcie-ipq8064-v2 + - qcom,pcie-ipq8074 + - qcom,pcie-qcs404 + - qcom,pcie-sa8540p + then: properties: interrupts: maxItems: 1 diff --git a/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml b/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml index 445eed94b53f136ec417162f6721920df5542022..f20ed7e709f7ca7205786b1e3396962885c4ed15 100644 --- a/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pci/samsung,exynos-pcie.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Samsung SoC series PCIe Host Controller Device Tree Bindings +title: Samsung SoC series PCIe Host Controller maintainers: - Marek Szyprowski diff --git a/Documentation/devicetree/bindings/pci/sifive,fu740-pcie.yaml b/Documentation/devicetree/bindings/pci/sifive,fu740-pcie.yaml index 195e6afeb16944feab1d366c6cc3d98c3d85d00f..844fc71423020b1e31e6497047a72f0068052ae8 100644 --- a/Documentation/devicetree/bindings/pci/sifive,fu740-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/sifive,fu740-pcie.yaml @@ -51,6 +51,12 @@ properties: description: A phandle to the PCIe power up reset line. maxItems: 1 + clocks: + maxItems: 1 + + clock-names: + const: pcie_aux + pwren-gpios: description: Should specify the GPIO for controlling the PCI bus device power on. maxItems: 1 @@ -66,6 +72,7 @@ required: - interrupt-map-mask - interrupt-map - clocks + - clock-names - resets - pwren-gpios - reset-gpios @@ -104,6 +111,7 @@ examples: <0x0 0x0 0x0 0x2 &plic0 58>, <0x0 0x0 0x0 0x3 &plic0 59>, <0x0 0x0 0x0 0x4 &plic0 60>; + clock-names = "pcie_aux"; clocks = <&prci FU740_PRCI_CLK_PCIE_AUX>; resets = <&prci 4>; pwren-gpios = <&gpio 5 0>; diff --git a/Documentation/devicetree/bindings/pci/toshiba,visconti-pcie.yaml b/Documentation/devicetree/bindings/pci/toshiba,visconti-pcie.yaml index 30b6396d83c83e4263fe7991fd2f26e0267b3a94..48ed227fc5b9e888781a9df39d52b90b1cf5f6b7 100644 --- a/Documentation/devicetree/bindings/pci/toshiba,visconti-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/toshiba,visconti-pcie.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pci/toshiba,visconti-pcie.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Toshiba Visconti5 SoC PCIe Host Controller Device Tree Bindings +title: Toshiba Visconti5 SoC PCIe Host Controller maintainers: - Nobuhiro Iwamatsu diff --git a/Documentation/devicetree/bindings/peci/peci-aspeed.yaml b/Documentation/devicetree/bindings/peci/peci-aspeed.yaml index 1e68a801a92a2236b2da76dd7193a3e9196c778b..b01e0c8b963da906242dab92bb4eaada802a24ab 100644 --- a/Documentation/devicetree/bindings/peci/peci-aspeed.yaml +++ b/Documentation/devicetree/bindings/peci/peci-aspeed.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/peci/peci-aspeed.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Aspeed PECI Bus Device Tree Bindings +title: Aspeed PECI Bus maintainers: - Iwona Winiarska diff --git a/Documentation/devicetree/bindings/peci/peci-controller.yaml b/Documentation/devicetree/bindings/peci/peci-controller.yaml index bbc3d3f3a9296cffd8eb85c7f1976757d7b63ab3..d7853291b0601d24bafbb6ac52b05d9dfc2b31c2 100644 --- a/Documentation/devicetree/bindings/peci/peci-controller.yaml +++ b/Documentation/devicetree/bindings/peci/peci-controller.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/peci/peci-controller.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic Device Tree Bindings for PECI +title: Generic for PECI maintainers: - Iwona Winiarska diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml index 77606c899fe2bf622b2cf791db2f6ebc3ed81f26..4fff091bd534bcabc2cf81daf8a127333dac7509 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun4i-a10-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 USB PHY Device Tree Bindings +title: Allwinner A10 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml index 0fa4b32b097e8d86a7d3e635f642a0b07d386f63..f557feca97630ccc5cd0cc4cbdf47190296fb7e5 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun50i-a64-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A64 USB PHY Device Tree Bindings +title: Allwinner A64 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml index e632140722a27c5a0a7f90319f4ccfd40693af64..4480fdedd669084d03c6a08fcafeb0aa63f8f64a 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun50i-h6-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner H6 USB PHY Device Tree Bindings +title: Allwinner H6 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml index 5bad9b06e2e71a6f428e8ed86d5b35cd5a1ef696..1563e3df305241615d439f5a3f0daf1e94b5d673 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun5i-a13-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A13 USB PHY Device Tree Bindings +title: Allwinner A13 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-mipi-dphy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-mipi-dphy.yaml index 22636c9fdab8e7068af3924757117c182c143fa4..dfb6a89935351d14e8dd7e4409e63ef85468859e 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-mipi-dphy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-mipi-dphy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun6i-a31-mipi-dphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 MIPI D-PHY Controller Device Tree Bindings +title: Allwinner A31 MIPI D-PHY Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml index 922b4665e00d09be4eb84db7580f323db74ce83b..4897a3d2fbc3ce3cb9377d307744acdff12469bb 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun6i-a31-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 USB PHY Device Tree Bindings +title: Allwinner A31 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml index a94019efc2f3a7c0357eedb05531782c1bd258c7..99de9c6889a07934d1af6b0dc81b7941aa314017 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun8i-a23-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A23 USB PHY Device Tree Bindings +title: Allwinner A23 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml index 33f3ddc0492d7855155e1570789ff298219cc348..73438d84de50607b111624b28b11e43bcf6be5c6 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun8i-a83t-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A83t USB PHY Device Tree Bindings +title: Allwinner A83t USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml index e288450e0844d65233746fa652240c908d200b7a..77539b4601c24d794930fa465d0c1bbf0c16b178 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun8i-h3-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner H3 USB PHY Device Tree Bindings +title: Allwinner H3 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml index d947e50a49d26ac2970963707eb8649b11b60a48..2dd31630c13e05ab447413192976d0c3b480df00 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun8i-r40-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner R40 USB PHY Device Tree Bindings +title: Allwinner R40 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml index a2836c296cc47033c55e2ef7f48e776a44c4ef04..395d33855d484ac5e65135ac44bdbc4417011b3c 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun8i-v3s-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner V3s USB PHY Device Tree Bindings +title: Allwinner V3s USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml index 2eb493fa64fd62eff7b713cdc089c1bb8752fde8..bd9445f6f130f422c2581521ce2af0d29618fd7a 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/allwinner,sun9i-a80-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A80 USB PHY Device Tree Bindings +title: Allwinner A80 USB PHY maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/phy/amlogic,axg-mipi-dphy.yaml b/Documentation/devicetree/bindings/phy/amlogic,axg-mipi-dphy.yaml index be485f5008870d81b3055d32a373bf5dc94e1725..5eddaed3d8535fc7020240d6f3a52d0a921510ce 100644 --- a/Documentation/devicetree/bindings/phy/amlogic,axg-mipi-dphy.yaml +++ b/Documentation/devicetree/bindings/phy/amlogic,axg-mipi-dphy.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic AXG MIPI D-PHY maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb2-phy.yaml b/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb2-phy.yaml index 399ebde454095fa19519b9ad0718ca3fd8db9a3c..f3a5fbabbbb5971b343ea6f892a49033dddbf5d0 100644 --- a/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb2-phy.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic G12A USB2 PHY maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb3-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb3-pcie-phy.yaml index 453c083cf44cb170e5e19038a1c34e6f8ff9d5c0..868b4e6fde71f1fb68d49b9ce8f73fc5176a12e6 100644 --- a/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb3-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/amlogic,meson-g12a-usb3-pcie-phy.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic G12A USB3 + PCIE Combo PHY maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/brcm,cygnus-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,cygnus-pcie-phy.yaml index 045699c657797795061ee6cc5d3aaa63625a43bf..808e90b2465d9d1eb351dfad6e5ffd9613dde2ab 100644 --- a/Documentation/devicetree/bindings/phy/brcm,cygnus-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/brcm,cygnus-pcie-phy.yaml @@ -32,6 +32,7 @@ properties: patternProperties: "^pcie-phy@[0-9]+$": type: object + additionalProperties: false description: > PCIe PHY child nodes diff --git a/Documentation/devicetree/bindings/phy/cdns,dphy-rx.yaml b/Documentation/devicetree/bindings/phy/cdns,dphy-rx.yaml index 07be031d82e64ea52935cf85bf66ae45c23577bb..6fdd46ad5a780fa3ce9628ddaf7181f5270824bb 100644 --- a/Documentation/devicetree/bindings/phy/cdns,dphy-rx.yaml +++ b/Documentation/devicetree/bindings/phy/cdns,dphy-rx.yaml @@ -4,10 +4,10 @@ $id: http://devicetree.org/schemas/phy/cdns,dphy-rx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Cadence DPHY Rx Device Tree Bindings +title: Cadence DPHY Rx maintainers: - - Pratyush Yadav + - Pratyush Yadav properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/cdns,dphy.yaml b/Documentation/devicetree/bindings/phy/cdns,dphy.yaml index f0e9ca8427bb9668c2d4b4eaad062af6d324cee2..6cd9b0f388a43a0b23f997de36a04f6b57a9869c 100644 --- a/Documentation/devicetree/bindings/phy/cdns,dphy.yaml +++ b/Documentation/devicetree/bindings/phy/cdns,dphy.yaml @@ -4,10 +4,10 @@ $id: http://devicetree.org/schemas/phy/cdns,dphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Cadence DPHY Device Tree Bindings +title: Cadence DPHY maintainers: - - Pratyush Yadav + - Pratyush Yadav properties: compatible: diff --git a/Documentation/devicetree/bindings/phy/fsl,imx8-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/fsl,imx8-pcie-phy.yaml index b6421eedece3a19d23395fd76317bb62b37e1617..0af765ba2793262a9284cdf52dda61dfc7f97177 100644 --- a/Documentation/devicetree/bindings/phy/fsl,imx8-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/fsl,imx8-pcie-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/fsl,imx8-pcie-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Freescale i.MX8 SoC series PCIe PHY Device Tree Bindings +title: Freescale i.MX8 SoC series PCIe PHY maintainers: - Richard Zhu diff --git a/Documentation/devicetree/bindings/phy/hisilicon,hi3660-usb3.yaml b/Documentation/devicetree/bindings/phy/hisilicon,hi3660-usb3.yaml index c2e073e261909db4400f29c9ae1b5d060890ea3d..20b79e2e8b8202ce3d0a44b57730222cea736a18 100644 --- a/Documentation/devicetree/bindings/phy/hisilicon,hi3660-usb3.yaml +++ b/Documentation/devicetree/bindings/phy/hisilicon,hi3660-usb3.yaml @@ -41,20 +41,10 @@ additionalProperties: false examples: - | - bus { - #address-cells = <2>; - #size-cells = <2>; - - usb3_otg_bc: usb3_otg_bc@ff200000 { - compatible = "syscon", "simple-mfd"; - reg = <0x0 0xff200000 0x0 0x1000>; - - usb-phy { - compatible = "hisilicon,hi3660-usb-phy"; - #phy-cells = <0>; - hisilicon,pericrg-syscon = <&crg_ctrl>; - hisilicon,pctrl-syscon = <&pctrl>; - hisilicon,eye-diagram-param = <0x22466e4>; - }; - }; + usb-phy { + compatible = "hisilicon,hi3660-usb-phy"; + #phy-cells = <0>; + hisilicon,pericrg-syscon = <&crg_ctrl>; + hisilicon,pctrl-syscon = <&pctrl>; + hisilicon,eye-diagram-param = <0x22466e4>; }; diff --git a/Documentation/devicetree/bindings/phy/hisilicon,hi3670-usb3.yaml b/Documentation/devicetree/bindings/phy/hisilicon,hi3670-usb3.yaml index ebd78acfe2de546b3c52aae5040c65b493d18de2..1cb00dbcd4c505035152180e407e363f38552f6d 100644 --- a/Documentation/devicetree/bindings/phy/hisilicon,hi3670-usb3.yaml +++ b/Documentation/devicetree/bindings/phy/hisilicon,hi3670-usb3.yaml @@ -52,22 +52,12 @@ additionalProperties: false examples: - | - bus { - #address-cells = <2>; - #size-cells = <2>; - - usb3_otg_bc: usb3_otg_bc@ff200000 { - compatible = "syscon", "simple-mfd"; - reg = <0x0 0xff200000 0x0 0x1000>; - - usb_phy { - compatible = "hisilicon,hi3670-usb-phy"; - #phy-cells = <0>; - hisilicon,pericrg-syscon = <&crg_ctrl>; - hisilicon,pctrl-syscon = <&pctrl>; - hisilicon,sctrl-syscon = <&sctrl>; - hisilicon,eye-diagram-param = <0xfdfee4>; - hisilicon,tx-vboost-lvl = <0x5>; - }; - }; + usb-phy { + compatible = "hisilicon,hi3670-usb-phy"; + #phy-cells = <0>; + hisilicon,pericrg-syscon = <&crg_ctrl>; + hisilicon,pctrl-syscon = <&pctrl>; + hisilicon,sctrl-syscon = <&sctrl>; + hisilicon,eye-diagram-param = <0xfdfee4>; + hisilicon,tx-vboost-lvl = <0x5>; }; diff --git a/Documentation/devicetree/bindings/phy/intel,lgm-emmc-phy.yaml b/Documentation/devicetree/bindings/phy/intel,lgm-emmc-phy.yaml index 954e67571dfdf5987a1c473b00635a16fe31754a..ca818f83579ba98d14f388bb2d00ab755de81397 100644 --- a/Documentation/devicetree/bindings/phy/intel,lgm-emmc-phy.yaml +++ b/Documentation/devicetree/bindings/phy/intel,lgm-emmc-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/intel,lgm-emmc-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Lightning Mountain(LGM) eMMC PHY Device Tree Bindings +title: Intel Lightning Mountain(LGM) eMMC PHY maintainers: - Ramuthevar Vadivel Murugan diff --git a/Documentation/devicetree/bindings/phy/intel,lgm-usb-phy.yaml b/Documentation/devicetree/bindings/phy/intel,lgm-usb-phy.yaml index ce62c0b94daf06666d07dfaa910a0dcc5be92291..653a12286637e43533a6235432386aabefb9421a 100644 --- a/Documentation/devicetree/bindings/phy/intel,lgm-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/intel,lgm-usb-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/intel,lgm-usb-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel LGM USB PHY Device Tree Bindings +title: Intel LGM USB PHY maintainers: - Vadivel Murugan Ramuthevar diff --git a/Documentation/devicetree/bindings/phy/lantiq,vrx200-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/lantiq,vrx200-pcie-phy.yaml index a97482179cf5a37c0dfc001fbb71b2f74496ee76..711de06bb0fcc8b8e7748d5817d2b2c358ed0903 100644 --- a/Documentation/devicetree/bindings/phy/lantiq,vrx200-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/lantiq,vrx200-pcie-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/lantiq,vrx200-pcie-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Lantiq VRX200 and ARX300 PCIe PHY Device Tree Bindings +title: Lantiq VRX200 and ARX300 PCIe PHY maintainers: - Martin Blumenstingl diff --git a/Documentation/devicetree/bindings/phy/mediatek,mt7621-pci-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,mt7621-pci-phy.yaml index 29d4123323c2a066d0501ef8eda1beb089784ba6..c2f4cb0b254a085ed4a40eaa7e5342c02583a815 100644 --- a/Documentation/devicetree/bindings/phy/mediatek,mt7621-pci-phy.yaml +++ b/Documentation/devicetree/bindings/phy/mediatek,mt7621-pci-phy.yaml @@ -4,7 +4,7 @@ $id: "http://devicetree.org/schemas/phy/mediatek,mt7621-pci-phy.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: Mediatek Mt7621 PCIe PHY Device Tree Bindings +title: Mediatek Mt7621 PCIe PHY maintainers: - Sergio Paracuellos diff --git a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml index b3e409988c1718f3c588fe36dbb2a31f2efb4901..5613cc5106e32f7f5be26aed73f0617801dc41ac 100644 --- a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml +++ b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/phy/mediatek,tphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek T-PHY Controller Device Tree Bindings +title: MediaTek T-PHY Controller maintainers: - Chunfeng Yun @@ -163,6 +163,7 @@ patternProperties: - PHY_TYPE_USB3 - PHY_TYPE_PCIE - PHY_TYPE_SATA + - PHY_TYPE_SGMII nvmem-cells: items: @@ -218,6 +219,16 @@ patternProperties: minimum: 1 maximum: 15 + mediatek,pre-emphasis: + description: + The level of pre-emphasis which used to widen the eye opening and + boost eye swing, the unit step is about 4.16% increment; e.g. the + level 1 means amplitude increases about 4.16%, the level 2 is about + 8.3% etc. (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 3 + mediatek,bc12: description: Specify the flag to enable BC1.2 if support it diff --git a/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml index 598fd2b95c298811ff0fe19342b60504277b1f81..a9e3139fd421de4bf6c159fee05dc0db6a7f8d07 100644 --- a/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml +++ b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/phy/mediatek,xsphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek XS-PHY Controller Device Tree Bindings +title: MediaTek XS-PHY Controller maintainers: - Chunfeng Yun diff --git a/Documentation/devicetree/bindings/phy/mscc,vsc7514-serdes.yaml b/Documentation/devicetree/bindings/phy/mscc,vsc7514-serdes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3169b873231e877860d274c819c979836286a673 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mscc,vsc7514-serdes.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mscc,vsc7514-serdes.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microsemi Ocelot SerDes muxing + +maintainers: + - Alexandre Belloni + - UNGLinuxDriver@microchip.com + +description: | + On Microsemi Ocelot, there is a handful of registers in HSIO address + space for setting up the SerDes to switch port muxing. + + A SerDes X can be "muxed" to work with switch port Y or Z for example. + One specific SerDes can also be used as a PCIe interface. + + Hence, a SerDes represents an interface, be it an Ethernet or a PCIe one. + + There are two kinds of SerDes: SERDES1G supports 10/100Mbps in + half/full-duplex and 1000Mbps in full-duplex mode while SERDES6G supports + 10/100Mbps in half/full-duplex and 1000/2500Mbps in full-duplex mode. + + Also, SERDES6G number (aka "macro") 0 is the only interface supporting + QSGMII. + + This is a child of the HSIO syscon ("mscc,ocelot-hsio", see + Documentation/devicetree/bindings/mips/mscc.txt) on the Microsemi Ocelot. + +properties: + compatible: + enum: + - mscc,vsc7514-serdes + + "#phy-cells": + const: 2 + description: | + The first number defines the input port to use for a given SerDes macro. + The second defines the macro to use. They are defined in + dt-bindings/phy/phy-ocelot-serdes.h + +required: + - compatible + - "#phy-cells" + +additionalProperties: + false + +examples: + - | + serdes: serdes { + compatible = "mscc,vsc7514-serdes"; + #phy-cells = <2>; + }; diff --git a/Documentation/devicetree/bindings/phy/mxs-usb-phy.txt b/Documentation/devicetree/bindings/phy/mxs-usb-phy.txt index c9e392c64a7c8c7b42176e49773e36db54061020..70c813b0755f971b5eecffeecfa084255d7f860e 100644 --- a/Documentation/devicetree/bindings/phy/mxs-usb-phy.txt +++ b/Documentation/devicetree/bindings/phy/mxs-usb-phy.txt @@ -15,10 +15,10 @@ Required properties: - fsl,anatop: phandle for anatop register, it is only for imx6 SoC series Optional properties: -- fsl,tx-cal-45-dn-ohms: Integer [30-55]. Resistance (in ohms) of switchable +- fsl,tx-cal-45-dn-ohms: Integer [35-54]. Resistance (in ohms) of switchable high-speed trimming resistor connected in parallel with the 45 ohm resistor that terminates the DN output signal. Default: 45 -- fsl,tx-cal-45-dp-ohms: Integer [30-55]. Resistance (in ohms) of switchable +- fsl,tx-cal-45-dp-ohms: Integer [35-54]. Resistance (in ohms) of switchable high-speed trimming resistor connected in parallel with the 45 ohm resistor that terminates the DP output signal. Default: 45 - fsl,tx-d-cal: Integer [79-119]. Current trimming value (as a percentage) of diff --git a/Documentation/devicetree/bindings/phy/phy-ocelot-serdes.txt b/Documentation/devicetree/bindings/phy/phy-ocelot-serdes.txt deleted file mode 100644 index 332219860187737a3dc41bba5c1ffa85cc7e1eb5..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/phy-ocelot-serdes.txt +++ /dev/null @@ -1,43 +0,0 @@ -Microsemi Ocelot SerDes muxing driver -------------------------------------- - -On Microsemi Ocelot, there is a handful of registers in HSIO address -space for setting up the SerDes to switch port muxing. - -A SerDes X can be "muxed" to work with switch port Y or Z for example. -One specific SerDes can also be used as a PCIe interface. - -Hence, a SerDes represents an interface, be it an Ethernet or a PCIe one. - -There are two kinds of SerDes: SERDES1G supports 10/100Mbps in -half/full-duplex and 1000Mbps in full-duplex mode while SERDES6G supports -10/100Mbps in half/full-duplex and 1000/2500Mbps in full-duplex mode. - -Also, SERDES6G number (aka "macro") 0 is the only interface supporting -QSGMII. - -This is a child of the HSIO syscon ("mscc,ocelot-hsio", see -Documentation/devicetree/bindings/mips/mscc.txt) on the Microsemi Ocelot. - -Required properties: - -- compatible: should be "mscc,vsc7514-serdes" -- #phy-cells : from the generic phy bindings, must be 2. - The first number defines the input port to use for a given - SerDes macro. The second defines the macro to use. They are - defined in dt-bindings/phy/phy-ocelot-serdes.h - -Example: - - serdes: serdes { - compatible = "mscc,vsc7514-serdes"; - #phy-cells = <2>; - }; - - ethernet { - port1 { - phy-handle = <&phy_foo>; - /* Link SERDES1G_5 to port1 */ - phys = <&serdes 1 SERDES1G_5>; - }; - }; diff --git a/Documentation/devicetree/bindings/phy/phy-rockchip-inno-usb2.yaml b/Documentation/devicetree/bindings/phy/phy-rockchip-inno-usb2.yaml index 4b75289735ebd750c667f2f00992357a35e6ee1b..f71920082fa3e6a09673ba08ad761c9bf5ea0ebf 100644 --- a/Documentation/devicetree/bindings/phy/phy-rockchip-inno-usb2.yaml +++ b/Documentation/devicetree/bindings/phy/phy-rockchip-inno-usb2.yaml @@ -13,6 +13,7 @@ properties: compatible: enum: - rockchip,px30-usb2phy + - rockchip,rk3128-usb2phy - rockchip,rk3228-usb2phy - rockchip,rk3308-usb2phy - rockchip,rk3328-usb2phy diff --git a/Documentation/devicetree/bindings/phy/phy-rockchip-naneng-combphy.yaml b/Documentation/devicetree/bindings/phy/phy-rockchip-naneng-combphy.yaml index f1445440141986745a3b99c8acd4b3d0658255a1..8d8698412de01283167f8c8ddf4307f3d3c16b06 100644 --- a/Documentation/devicetree/bindings/phy/phy-rockchip-naneng-combphy.yaml +++ b/Documentation/devicetree/bindings/phy/phy-rockchip-naneng-combphy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/phy-rockchip-naneng-combphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip SoC Naneng Combo Phy Device Tree Bindings +title: Rockchip SoC Naneng Combo Phy maintainers: - Heiko Stuebner diff --git a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml index dc287d428e49d4977e9f90b20a41be08e45afc62..801993813b18520989123387d4ecc87de506c223 100644 --- a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml +++ b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml @@ -77,6 +77,8 @@ patternProperties: connector: type: object $ref: /schemas/connector/usb-connector.yaml + unevaluatedProperties: false + properties: vbus-supply: true diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml index cf9e9b8011cb33b5737a5c7c53f89d4f01873d32..1e104ae76ee60f3c68344ca5e532615450069870 100644 --- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml @@ -19,6 +19,8 @@ properties: enum: - qcom,sc7280-edp-phy - qcom,sc8180x-edp-phy + - qcom,sc8280xp-dp-phy + - qcom,sc8280xp-edp-phy reg: items: diff --git a/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml b/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml index eea2e02678edc696b45074e6a0bec18d25960962..83fe4b39b56f450fd3f45a37b05557e636488966 100644 --- a/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml @@ -28,12 +28,15 @@ properties: - const: hdmi_phy clocks: - maxItems: 2 + minItems: 2 + maxItems: 3 clock-names: + minItems: 2 items: - const: iface - const: ref + - const: xo power-domains: maxItems: 1 @@ -44,6 +47,9 @@ properties: vddio-supply: description: phandle to VDD I/O supply regulator + '#clock-cells': + const: 0 + '#phy-cells': const: 0 @@ -75,9 +81,12 @@ examples: "hdmi_phy"; clocks = <&mmcc 116>, - <&gcc 214>; + <&gcc 214>, + <&xo_board>; clock-names = "iface", - "ref"; + "ref", + "xo"; + #clock-cells = <0>; #phy-cells = <0>; vddio-supply = <&vreg_l12a_1p8>; diff --git a/Documentation/devicetree/bindings/phy/qcom,msm8996-qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,msm8996-qmp-pcie-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e710ef75523a65fa31415cb12dbaa8c290060e8 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,msm8996-qmp-pcie-phy.yaml @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,msm8996-qmp-pcie-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMP PHY controller (MSM8996 PCIe) + +maintainers: + - Vinod Koul + +description: + QMP PHY controller supports physical layer functionality for a number of + controllers on Qualcomm chipsets, such as, PCIe, UFS, and USB. + +properties: + compatible: + const: qcom,msm8996-qmp-pcie-phy + + reg: + items: + - description: serdes + + "#address-cells": + enum: [ 1, 2 ] + + "#size-cells": + enum: [ 1, 2 ] + + ranges: true + + clocks: + maxItems: 3 + + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + + resets: + maxItems: 3 + + reset-names: + items: + - const: phy + - const: common + - const: cfg + + vdda-phy-supply: true + + vdda-pll-supply: true + + vddp-ref-clk-supply: true + +patternProperties: + "^phy@[0-9a-f]+$": + type: object + description: one child node per PHY provided by this block + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + + clocks: + items: + - description: PIPE clock + + clock-names: + deprecated: true + items: + - enum: + - pipe0 + - pipe1 + - pipe2 + + resets: + items: + - description: PHY reset + + reset-names: + deprecated: true + items: + - enum: + - lane0 + - lane1 + - lane2 + + "#clock-cells": + const: 0 + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + required: + - reg + - clocks + - resets + - "#clock-cells" + - clock-output-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + - ranges + - clocks + - clock-names + - resets + - reset-names + - vdda-phy-supply + - vdda-pll-supply + +additionalProperties: false + +examples: + - | + #include + pcie_phy: phy-wrapper@34000 { + compatible = "qcom,msm8996-qmp-pcie-phy"; + reg = <0x34000 0x488>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x34000 0x4000>; + + clocks = <&gcc GCC_PCIE_PHY_AUX_CLK>, + <&gcc GCC_PCIE_PHY_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_CLK>; + clock-names = "aux", "cfg_ahb", "ref"; + + resets = <&gcc GCC_PCIE_PHY_BCR>, + <&gcc GCC_PCIE_PHY_COM_BCR>, + <&gcc GCC_PCIE_PHY_COM_NOCSR_BCR>; + reset-names = "phy", "common", "cfg"; + + vdda-phy-supply = <&vreg_l28a_0p925>; + vdda-pll-supply = <&vreg_l12a_1p8>; + + pciephy_0: phy@1000 { + reg = <0x1000 0x130>, + <0x1200 0x200>, + <0x1400 0x1dc>; + + clocks = <&gcc GCC_PCIE_0_PIPE_CLK>; + resets = <&gcc GCC_PCIE_0_PHY_BCR>; + + #clock-cells = <0>; + clock-output-names = "pcie_0_pipe_clk_src"; + + #phy-cells = <0>; + }; + + pciephy_1: phy@2000 { + reg = <0x2000 0x130>, + <0x2200 0x200>, + <0x2400 0x1dc>; + + clocks = <&gcc GCC_PCIE_1_PIPE_CLK>; + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + + #clock-cells = <0>; + clock-output-names = "pcie_1_pipe_clk_src"; + + #phy-cells = <0>; + }; + + pciephy_2: phy@3000 { + reg = <0x3000 0x130>, + <0x3200 0x200>, + <0x3400 0x1dc>; + + clocks = <&gcc GCC_PCIE_2_PIPE_CLK>; + resets = <&gcc GCC_PCIE_2_PHY_BCR>; + + #clock-cells = <0>; + clock-output-names = "pcie_2_pipe_clk_src"; + + #phy-cells = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-pcie-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..324ad7d03a383a6f63138efc939a0b399113e4de --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-pcie-phy.yaml @@ -0,0 +1,296 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,qmp-pcie-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMP PHY controller (PCIe) + +maintainers: + - Vinod Koul + +description: + QMP PHY controller supports physical layer functionality for a number of + controllers on Qualcomm chipsets, such as, PCIe, UFS, and USB. + +properties: + compatible: + enum: + - qcom,ipq6018-qmp-pcie-phy + - qcom,ipq8074-qmp-gen3-pcie-phy + - qcom,ipq8074-qmp-pcie-phy + - qcom,msm8998-qmp-pcie-phy + - qcom,sc8180x-qmp-pcie-phy + - qcom,sdm845-qhp-pcie-phy + - qcom,sdm845-qmp-pcie-phy + - qcom,sdx55-qmp-pcie-phy + - qcom,sm8250-qmp-gen3x1-pcie-phy + - qcom,sm8250-qmp-gen3x2-pcie-phy + - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8450-qmp-gen3x1-pcie-phy + - qcom,sm8450-qmp-gen4x2-pcie-phy + + reg: + items: + - description: serdes + + "#address-cells": + enum: [ 1, 2 ] + + "#size-cells": + enum: [ 1, 2 ] + + ranges: true + + clocks: + minItems: 2 + maxItems: 4 + + clock-names: + minItems: 2 + maxItems: 4 + + resets: + minItems: 1 + maxItems: 2 + + reset-names: + minItems: 1 + maxItems: 2 + + vdda-phy-supply: true + + vdda-pll-supply: true + + vddp-ref-clk-supply: true + +patternProperties: + "^phy@[0-9a-f]+$": + type: object + description: single PHY-provider child node + properties: + reg: + minItems: 3 + maxItems: 6 + + clocks: + items: + - description: PIPE clock + + clock-names: + deprecated: true + items: + - const: pipe0 + + "#clock-cells": + const: 0 + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + required: + - reg + - clocks + - "#clock-cells" + - clock-output-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + - ranges + - clocks + - clock-names + - resets + - reset-names + +additionalProperties: false + +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8998-qmp-pcie-phy + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + required: + - vdda-phy-supply + - vdda-pll-supply + + - if: + properties: + compatible: + contains: + enum: + - qcom,ipq6018-qmp-pcie-phy + - qcom,ipq8074-qmp-gen3-pcie-phy + - qcom,ipq8074-qmp-pcie-phy + then: + properties: + clocks: + maxItems: 2 + clock-names: + items: + - const: aux + - const: cfg_ahb + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8180x-qmp-pcie-phy + - qcom,sdm845-qhp-pcie-phy + - qcom,sdm845-qmp-pcie-phy + - qcom,sdx55-qmp-pcie-phy + - qcom,sm8250-qmp-gen3x1-pcie-phy + - qcom,sm8250-qmp-gen3x2-pcie-phy + - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8450-qmp-gen3x1-pcie-phy + - qcom,sm8450-qmp-gen4x2-pcie-phy + then: + properties: + clocks: + maxItems: 4 + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + - const: refgen + resets: + maxItems: 1 + reset-names: + items: + - const: phy + required: + - vdda-phy-supply + - vdda-pll-supply + + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8250-qmp-gen3x2-pcie-phy + - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8450-qmp-gen4x2-pcie-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX lane 1 + - description: RX lane 1 + - description: PCS + - description: TX lane 2 + - description: RX lane 2 + - description: PCS_MISC + + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8180x-qmp-pcie-phy + - qcom,sdm845-qmp-pcie-phy + - qcom,sdx55-qmp-pcie-phy + - qcom,sm8250-qmp-gen3x1-pcie-phy + - qcom,sm8450-qmp-gen3x1-pcie-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + - description: PCS_MISC + + - if: + properties: + compatible: + contains: + enum: + - qcom,ipq6018-qmp-pcie-phy + - qcom,ipq8074-qmp-pcie-phy + - qcom,msm8998-qmp-pcie-phy + - qcom,sdm845-qhp-pcie-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + +examples: + - | + #include + phy-wrapper@1c0e000 { + compatible = "qcom,sm8250-qmp-gen3x2-pcie-phy"; + reg = <0x01c0e000 0x1c0>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x01c0e000 0x1000>; + + clocks = <&gcc GCC_PCIE_PHY_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_WIGIG_CLKREF_EN>, + <&gcc GCC_PCIE1_PHY_REFGEN_CLK>; + clock-names = "aux", "cfg_ahb", "ref", "refgen"; + + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + reset-names = "phy"; + + vdda-phy-supply = <&vreg_l10c_0p88>; + vdda-pll-supply = <&vreg_l6b_1p2>; + + phy@200 { + reg = <0x200 0x170>, + <0x400 0x200>, + <0xa00 0x1f0>, + <0x600 0x170>, + <0x800 0x200>, + <0xe00 0xf4>; + + clocks = <&gcc GCC_PCIE_1_PIPE_CLK>; + + #clock-cells = <0>; + clock-output-names = "pcie_1_pipe_clk"; + + #phy-cells = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml deleted file mode 100644 index 220788ce215f477ee79b8e186bb7cf91b4650cb5..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ /dev/null @@ -1,502 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) - -%YAML 1.2 ---- -$id: "http://devicetree.org/schemas/phy/qcom,qmp-phy.yaml#" -$schema: "http://devicetree.org/meta-schemas/core.yaml#" - -title: Qualcomm QMP PHY controller - -maintainers: - - Vinod Koul - -description: - QMP phy controller supports physical layer functionality for a number of - controllers on Qualcomm chipsets, such as, PCIe, UFS, and USB. - -properties: - compatible: - enum: - - qcom,ipq6018-qmp-pcie-phy - - qcom,ipq6018-qmp-usb3-phy - - qcom,ipq8074-qmp-gen3-pcie-phy - - qcom,ipq8074-qmp-pcie-phy - - qcom,ipq8074-qmp-usb3-phy - - qcom,msm8996-qmp-pcie-phy - - qcom,msm8996-qmp-ufs-phy - - qcom,msm8996-qmp-usb3-phy - - qcom,msm8998-qmp-pcie-phy - - qcom,msm8998-qmp-ufs-phy - - qcom,msm8998-qmp-usb3-phy - - qcom,qcm2290-qmp-usb3-phy - - qcom,sc7180-qmp-usb3-phy - - qcom,sc8180x-qmp-pcie-phy - - qcom,sc8180x-qmp-ufs-phy - - qcom,sc8180x-qmp-usb3-phy - - qcom,sc8280xp-qmp-ufs-phy - - qcom,sdm845-qhp-pcie-phy - - qcom,sdm845-qmp-pcie-phy - - qcom,sdm845-qmp-ufs-phy - - qcom,sdm845-qmp-usb3-phy - - qcom,sdm845-qmp-usb3-uni-phy - - qcom,sm6115-qmp-ufs-phy - - qcom,sm6350-qmp-ufs-phy - - qcom,sm8150-qmp-ufs-phy - - qcom,sm8150-qmp-usb3-phy - - qcom,sm8150-qmp-usb3-uni-phy - - qcom,sm8250-qmp-ufs-phy - - qcom,sm8250-qmp-gen3x1-pcie-phy - - qcom,sm8250-qmp-gen3x2-pcie-phy - - qcom,sm8250-qmp-modem-pcie-phy - - qcom,sm8250-qmp-usb3-phy - - qcom,sm8250-qmp-usb3-uni-phy - - qcom,sm8350-qmp-ufs-phy - - qcom,sm8350-qmp-usb3-phy - - qcom,sm8350-qmp-usb3-uni-phy - - qcom,sm8450-qmp-gen3x1-pcie-phy - - qcom,sm8450-qmp-gen4x2-pcie-phy - - qcom,sm8450-qmp-ufs-phy - - qcom,sm8450-qmp-usb3-phy - - qcom,sdx55-qmp-pcie-phy - - qcom,sdx55-qmp-usb3-uni-phy - - qcom,sdx65-qmp-usb3-uni-phy - - reg: - minItems: 1 - items: - - description: Address and length of PHY's common serdes block. - - description: Address and length of PHY's DP_COM control block. - - "#clock-cells": - enum: [ 1, 2 ] - - "#address-cells": - enum: [ 1, 2 ] - - "#size-cells": - enum: [ 1, 2 ] - - ranges: true - - clocks: - minItems: 1 - maxItems: 4 - - clock-names: - minItems: 1 - maxItems: 4 - - resets: - minItems: 1 - maxItems: 3 - - reset-names: - minItems: 1 - maxItems: 3 - - vdda-phy-supply: - description: - Phandle to a regulator supply to PHY core block. - - vdda-pll-supply: - description: - Phandle to 1.8V regulator supply to PHY refclk pll block. - - vddp-ref-clk-supply: - description: - Phandle to a regulator supply to any specific refclk pll block. - -#Required nodes: -patternProperties: - "^phy@[0-9a-f]+$": - type: object - description: - Each device node of QMP phy is required to have as many child nodes as - the number of lanes the PHY has. - -required: - - compatible - - reg - - "#clock-cells" - - "#address-cells" - - "#size-cells" - - ranges - - clocks - - clock-names - - resets - - reset-names - -additionalProperties: false - -allOf: - - if: - properties: - compatible: - contains: - enum: - - qcom,sdm845-qmp-usb3-uni-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - - description: 19.2 MHz ref clk. - - description: Phy common block aux clock. - clock-names: - items: - - const: aux - - const: cfg_ahb - - const: ref - - const: com_aux - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,sdx55-qmp-usb3-uni-phy - - qcom,sdx65-qmp-usb3-uni-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - - description: 19.2 MHz ref clk. - clock-names: - items: - - const: aux - - const: cfg_ahb - - const: ref - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,msm8996-qmp-pcie-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - - description: 19.2 MHz ref clk. - clock-names: - items: - - const: aux - - const: cfg_ahb - - const: ref - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - - description: phy's ahb cfg block reset. - reset-names: - items: - - const: phy - - const: common - - const: cfg - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,ipq8074-qmp-usb3-phy - - qcom,msm8996-qmp-usb3-phy - - qcom,msm8998-qmp-pcie-phy - - qcom,msm8998-qmp-usb3-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - - description: 19.2 MHz ref clk. - clock-names: - items: - - const: aux - - const: cfg_ahb - - const: ref - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,msm8996-qmp-ufs-phy - then: - properties: - clocks: - items: - - description: 19.2 MHz ref clk. - clock-names: - items: - - const: ref - resets: - items: - - description: PHY reset in the UFS controller. - reset-names: - items: - - const: ufsphy - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,msm8998-qmp-ufs-phy - - qcom,sdm845-qmp-ufs-phy - - qcom,sm6350-qmp-ufs-phy - - qcom,sm8150-qmp-ufs-phy - - qcom,sm8250-qmp-ufs-phy - - qcom,sc8180x-qmp-ufs-phy - - qcom,sc8280xp-qmp-ufs-phy - then: - properties: - clocks: - items: - - description: 19.2 MHz ref clk. - - description: Phy reference aux clock. - clock-names: - items: - - const: ref - - const: ref_aux - resets: - items: - - description: PHY reset in the UFS controller. - reset-names: - items: - - const: ufsphy - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,ipq6018-qmp-pcie-phy - - qcom,ipq8074-qmp-gen3-pcie-phy - - qcom,ipq8074-qmp-pcie-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - clock-names: - items: - - const: aux - - const: cfg_ahb - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - - if: - properties: - compatible: - contains: - enum: - - qcom,sc8180x-qmp-pcie-phy - - qcom,sdm845-qhp-pcie-phy - - qcom,sdm845-qmp-pcie-phy - - qcom,sdx55-qmp-pcie-phy - - qcom,sm8250-qmp-gen3x1-pcie-phy - - qcom,sm8250-qmp-gen3x2-pcie-phy - - qcom,sm8250-qmp-modem-pcie-phy - - qcom,sm8450-qmp-gen3x1-pcie-phy - - qcom,sm8450-qmp-gen4x2-pcie-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: Phy config clock. - - description: 19.2 MHz ref clk. - - description: Phy refgen clk. - clock-names: - items: - - const: aux - - const: cfg_ahb - - const: ref - - const: refgen - resets: - items: - - description: reset of phy block. - reset-names: - items: - - const: phy - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,sm8150-qmp-usb3-phy - - qcom,sm8150-qmp-usb3-uni-phy - - qcom,sm8250-qmp-usb3-uni-phy - - qcom,sm8350-qmp-usb3-uni-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: 19.2 MHz ref clk source. - - description: 19.2 MHz ref clk. - - description: Phy common block aux clock. - clock-names: - items: - - const: aux - - const: ref_clk_src - - const: ref - - const: com_aux - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,sm8250-qmp-usb3-phy - - qcom,sm8350-qmp-usb3-phy - then: - properties: - clocks: - items: - - description: Phy aux clock. - - description: 19.2 MHz ref clk. - - description: Phy common block aux clock. - clock-names: - items: - - const: aux - - const: ref_clk_src - - const: com_aux - resets: - items: - - description: reset of phy block. - - description: phy common block reset. - reset-names: - items: - - const: phy - - const: common - required: - - vdda-phy-supply - - vdda-pll-supply - - if: - properties: - compatible: - contains: - enum: - - qcom,qcm2290-qmp-usb3-phy - then: - properties: - clocks: - items: - - description: Phy config clock. - - description: 19.2 MHz ref clk. - - description: Phy common block aux clock. - clock-names: - items: - - const: cfg_ahb - - const: ref - - const: com_aux - resets: - items: - - description: phy_phy reset. - - description: reset of phy block. - reset-names: - items: - - const: phy_phy - - const: phy - required: - - vdda-phy-supply - - vdda-pll-supply - -examples: - - | - #include - usb_2_qmpphy: phy-wrapper@88eb000 { - compatible = "qcom,sdm845-qmp-usb3-uni-phy"; - reg = <0x088eb000 0x18c>; - #clock-cells = <1>; - #address-cells = <1>; - #size-cells = <1>; - ranges = <0x0 0x088eb000 0x2000>; - - clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK >, - <&gcc GCC_USB_PHY_CFG_AHB2PHY_CLK>, - <&gcc GCC_USB3_SEC_CLKREF_CLK>, - <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>; - clock-names = "aux", "cfg_ahb", "ref", "com_aux"; - - resets = <&gcc GCC_USB3PHY_PHY_SEC_BCR>, - <&gcc GCC_USB3_PHY_SEC_BCR>; - reset-names = "phy", "common"; - - vdda-phy-supply = <&vdda_usb2_ss_1p2>; - vdda-pll-supply = <&vdda_usb2_ss_core>; - - usb_2_ssphy: phy@200 { - reg = <0x200 0x128>, - <0x400 0x1fc>, - <0x800 0x218>, - <0x600 0x70>; - #clock-cells = <0>; - #phy-cells = <0>; - clocks = <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; - clock-names = "pipe0"; - clock-output-names = "usb3_uni_phy_pipe_clk_src"; - }; - }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-ufs-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..815c375d0f7bce89a62beafd8e2d92ff30dee3c6 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-ufs-phy.yaml @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,qmp-ufs-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMP PHY controller (UFS) + +maintainers: + - Vinod Koul + +description: + QMP PHY controller supports physical layer functionality for a number of + controllers on Qualcomm chipsets, such as, PCIe, UFS, and USB. + +properties: + compatible: + enum: + - qcom,msm8996-qmp-ufs-phy + - qcom,msm8998-qmp-ufs-phy + - qcom,sc8180x-qmp-ufs-phy + - qcom,sc8280xp-qmp-ufs-phy + - qcom,sdm845-qmp-ufs-phy + - qcom,sm6115-qmp-ufs-phy + - qcom,sm6350-qmp-ufs-phy + - qcom,sm8150-qmp-ufs-phy + - qcom,sm8250-qmp-ufs-phy + - qcom,sm8350-qmp-ufs-phy + - qcom,sm8450-qmp-ufs-phy + + reg: + items: + - description: serdes + + "#address-cells": + enum: [ 1, 2 ] + + "#size-cells": + enum: [ 1, 2 ] + + ranges: true + + clocks: + minItems: 1 + maxItems: 3 + + clock-names: + minItems: 1 + maxItems: 3 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + reset-names: + items: + - const: ufsphy + + vdda-phy-supply: true + + vdda-pll-supply: true + + vddp-ref-clk-supply: true + +patternProperties: + "^phy@[0-9a-f]+$": + type: object + description: single PHY-provider child node + properties: + reg: + minItems: 3 + maxItems: 6 + + "#phy-cells": + const: 0 + + required: + - reg + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + - ranges + - clocks + - clock-names + - resets + - reset-names + - vdda-phy-supply + - vdda-pll-supply + +additionalProperties: false + +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8996-qmp-ufs-phy + then: + properties: + clocks: + maxItems: 1 + clock-names: + items: + - const: ref + + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8998-qmp-ufs-phy + - qcom,sc8180x-qmp-ufs-phy + - qcom,sc8280xp-qmp-ufs-phy + - qcom,sdm845-qmp-ufs-phy + - qcom,sm6115-qmp-ufs-phy + - qcom,sm6350-qmp-ufs-phy + - qcom,sm8150-qmp-ufs-phy + - qcom,sm8250-qmp-ufs-phy + then: + properties: + clocks: + maxItems: 2 + clock-names: + items: + - const: ref + - const: ref_aux + + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8450-qmp-ufs-phy + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: ref + - const: ref_aux + - const: qref + + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8998-qmp-ufs-phy + - qcom,sc8280xp-qmp-ufs-phy + - qcom,sdm845-qmp-ufs-phy + - qcom,sm6350-qmp-ufs-phy + - qcom,sm8150-qmp-ufs-phy + - qcom,sm8250-qmp-ufs-phy + - qcom,sm8350-qmp-ufs-phy + - qcom,sm8450-qmp-ufs-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX lane 1 + - description: RX lane 1 + - description: PCS + - description: TX lane 2 + - description: RX lane 2 + + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8180x-qmp-ufs-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + - description: PCS_MISC + + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8996-qmp-ufs-phy + - qcom,sm6115-qmp-ufs-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + +examples: + - | + #include + #include + phy-wrapper@1d87000 { + compatible = "qcom,sc8280xp-qmp-ufs-phy"; + reg = <0x01d87000 0xe10>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x01d87000 0x1000>; + + clocks = <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_UFS_PHY_PHY_AUX_CLK>; + clock-names = "ref", "ref_aux"; + + resets = <&ufs_mem_hc 0>; + reset-names = "ufsphy"; + + vdda-phy-supply = <&vreg_l6b>; + vdda-pll-supply = <&vreg_l3b>; + + phy@400 { + reg = <0x400 0x108>, + <0x600 0x1e0>, + <0xc00 0x1dc>, + <0x800 0x108>, + <0xa00 0x1e0>; + #phy-cells = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-usb-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-usb-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7acb4b7de7f940747efabaa05a9fd9bb35045a12 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-usb-phy.yaml @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/qcom,qmp-usb-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm QMP PHY controller (USB) + +maintainers: + - Vinod Koul + +description: + QMP PHY controller supports physical layer functionality for a number of + controllers on Qualcomm chipsets, such as, PCIe, UFS, and USB. + +properties: + compatible: + enum: + - qcom,ipq6018-qmp-usb3-phy + - qcom,ipq8074-qmp-usb3-phy + - qcom,msm8996-qmp-usb3-phy + - qcom,msm8998-qmp-usb3-phy + - qcom,qcm2290-qmp-usb3-phy + - qcom,sc7180-qmp-usb3-phy + - qcom,sc8180x-qmp-usb3-phy + - qcom,sc8280xp-qmp-usb3-uni-phy + - qcom,sdm845-qmp-usb3-phy + - qcom,sdm845-qmp-usb3-uni-phy + - qcom,sdx55-qmp-usb3-uni-phy + - qcom,sdx65-qmp-usb3-uni-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-uni-phy + - qcom,sm8450-qmp-usb3-phy + + reg: + minItems: 1 + items: + - description: serdes + - description: DP_COM + + "#address-cells": + enum: [ 1, 2 ] + + "#size-cells": + enum: [ 1, 2 ] + + ranges: true + + clocks: + minItems: 3 + maxItems: 4 + + clock-names: + minItems: 3 + maxItems: 4 + + power-domains: + maxItems: 1 + + resets: + maxItems: 2 + + reset-names: + maxItems: 2 + + vdda-phy-supply: true + + vdda-pll-supply: true + + vddp-ref-clk-supply: true + +patternProperties: + "^phy@[0-9a-f]+$": + type: object + description: single PHY-provider child node + properties: + reg: + minItems: 3 + maxItems: 6 + + clocks: + items: + - description: PIPE clock + + clock-names: + deprecated: true + items: + - const: pipe0 + + "#clock-cells": + const: 0 + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + required: + - reg + - clocks + - "#clock-cells" + - clock-output-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + - ranges + - clocks + - clock-names + - resets + - reset-names + - vdda-phy-supply + - vdda-pll-supply + +additionalProperties: false + +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,sc7180-qmp-usb3-phy + then: + properties: + clocks: + maxItems: 4 + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + - const: com_aux + resets: + maxItems: 1 + reset-names: + items: + - const: phy + + - if: + properties: + compatible: + contains: + enum: + - qcom,sdm845-qmp-usb3-uni-phy + then: + properties: + clocks: + maxItems: 4 + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + - const: com_aux + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + + - if: + properties: + compatible: + contains: + enum: + - qcom,ipq8074-qmp-usb3-phy + - qcom,msm8996-qmp-usb3-phy + - qcom,msm8998-qmp-usb3-phy + - qcom,sdx55-qmp-usb3-uni-phy + - qcom,sdx65-qmp-usb3-uni-phy + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-qmp-usb3-uni-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + then: + properties: + clocks: + maxItems: 4 + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: ref + - const: com_aux + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-phy + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: com_aux + resets: + maxItems: 2 + reset-names: + items: + - const: phy + - const: common + + - if: + properties: + compatible: + contains: + enum: + - qcom,qcm2290-qmp-usb3-phy + then: + properties: + clocks: + maxItems: 3 + clock-names: + items: + - const: cfg_ahb + - const: ref + - const: com_aux + resets: + maxItems: 2 + reset-names: + items: + - const: phy_phy + - const: phy + + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-qmp-usb3-uni-phy + then: + required: + - power-domains + + - if: + properties: + compatible: + contains: + enum: + - qcom,sdm845-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-phy + - qcom,sm8450-qmp-usb3-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX lane 1 + - description: RX lane 1 + - description: PCS + - description: TX lane 2 + - description: RX lane 2 + - description: PCS_MISC + + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8998-qmp-usb3-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX lane 1 + - description: RX lane 1 + - description: PCS + - description: TX lane 2 + - description: RX lane 2 + + - if: + properties: + compatible: + contains: + enum: + - qcom,ipq6018-qmp-usb3-phy + - qcom,ipq8074-qmp-usb3-phy + - qcom,qcm2290-qmp-usb3-phy + - qcom,sc7180-qmp-usb3-phy + - qcom,sc8180x-qmp-usb3-phy + - qcom,sdx55-qmp-usb3-uni-phy + - qcom,sdx65-qmp-usb3-uni-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + - description: PCS_MISC + + - if: + properties: + compatible: + contains: + enum: + - qcom,msm8996-qmp-usb3-phy + - qcom,sc8280xp-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + then: + patternProperties: + "^phy@[0-9a-f]+$": + properties: + reg: + items: + - description: TX + - description: RX + - description: PCS + +examples: + - | + #include + usb_2_qmpphy: phy-wrapper@88eb000 { + compatible = "qcom,sdm845-qmp-usb3-uni-phy"; + reg = <0x088eb000 0x18c>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x088eb000 0x2000>; + + clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK >, + <&gcc GCC_USB_PHY_CFG_AHB2PHY_CLK>, + <&gcc GCC_USB3_SEC_CLKREF_CLK>, + <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>; + clock-names = "aux", "cfg_ahb", "ref", "com_aux"; + + resets = <&gcc GCC_USB3PHY_PHY_SEC_BCR>, + <&gcc GCC_USB3_PHY_SEC_BCR>; + reset-names = "phy", "common"; + + vdda-phy-supply = <&vdda_usb2_ss_1p2>; + vdda-pll-supply = <&vdda_usb2_ss_core>; + + usb_2_ssphy: phy@200 { + reg = <0x200 0x128>, + <0x400 0x1fc>, + <0x800 0x218>, + <0x600 0x70>; + + clocks = <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; + + #clock-cells = <0>; + clock-output-names = "usb3_uni_phy_pipe_clk_src"; + + #phy-cells = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml index b078009ed509dfda7a69aa24c03e4bcfb7ac8d71..97a7ecafbf852e371558b01c8754b07459fe338c 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml @@ -16,6 +16,7 @@ properties: - qcom,sc7180-qmp-usb3-dp-phy - qcom,sc7280-qmp-usb3-dp-phy - qcom,sc8180x-qmp-usb3-dp-phy + - qcom,sc8280xp-qmp-usb43dp-phy - qcom,sdm845-qmp-usb3-dp-phy - qcom,sm8250-qmp-usb3-dp-phy reg: @@ -30,9 +31,6 @@ properties: - const: dp_com - const: dp - "#clock-cells": - enum: [ 1, 2 ] - "#address-cells": enum: [ 1, 2 ] @@ -55,6 +53,9 @@ properties: - const: ref - const: com_aux + power-domains: + maxItems: 1 + resets: items: - description: reset of phy block. @@ -81,6 +82,7 @@ properties: patternProperties: "^usb3-phy@[0-9a-f]+$": type: object + additionalProperties: false description: The USB3 PHY. @@ -99,6 +101,7 @@ patternProperties: - description: pipe clock clock-names: + deprecated: true items: - const: pipe0 @@ -115,12 +118,12 @@ patternProperties: required: - reg - clocks - - clock-names - '#clock-cells' - '#phy-cells' "^dp-phy@[0-9a-f]+$": type: object + additionalProperties: false description: The DP PHY. @@ -147,7 +150,6 @@ patternProperties: required: - compatible - reg - - "#clock-cells" - "#address-cells" - "#size-cells" - ranges @@ -160,6 +162,17 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,sc8280xp-qmp-usb43dp-phy + then: + required: + - power-domains + examples: - | #include @@ -169,7 +182,6 @@ examples: <0x088e8000 0x10>, <0x088ea000 0x40>; reg-names = "usb", "dp_com", "dp"; - #clock-cells = <1>; #address-cells = <1>; #size-cells = <1>; ranges = <0x0 0x088e9000 0x2000>; @@ -197,7 +209,6 @@ examples: #clock-cells = <0>; #phy-cells = <0>; clocks = <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>; - clock-names = "pipe0"; clock-output-names = "usb3_phy_pipe_clk_src"; }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index d68ab49345b8fd77ceb71100d50ac508b511c575..636ea430fbffaf0edfa246e8f51571c878c5dd4c 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -30,6 +30,7 @@ properties: - items: - enum: - qcom,sc7180-qusb2-phy + - qcom,sdm670-qusb2-phy - qcom,sdm845-qusb2-phy - qcom,sm6350-qusb2-phy - const: qcom,qusb2-v2-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml index 7a0e6a9854da2360dc87922468becc33301573da..68e70961beb26b41b71747b2316501a692c90b66 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml @@ -20,6 +20,7 @@ properties: - qcom,sc7280-usb-hs-phy - qcom,sc8180x-usb-hs-phy - qcom,sc8280xp-usb-hs-phy + - qcom,sm6375-usb-hs-phy - qcom,sm8150-usb-hs-phy - qcom,sm8250-usb-hs-phy - qcom,sm8350-usb-hs-phy @@ -53,6 +54,94 @@ properties: vdda33-supply: description: phandle to the regulator 3.3V supply node. + qcom,hs-disconnect-bp: + description: + This adjusts the voltage level for the threshold used to + detect a disconnect event at the host. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -272 + maximum: 2156 + + qcom,squelch-detector-bp: + description: + This adjusts the voltage level for the threshold used to + detect valid high-speed data. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -2090 + maximum: 1590 + + qcom,hs-amplitude-bp: + description: + This adjusts the high-speed DC level voltage. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -660 + maximum: 2670 + + qcom,pre-emphasis-duration-bp: + description: + This signal controls the duration for which the + HS pre-emphasis current is sourced onto DP<#> or DM<#>. + The HS Transmitter pre-emphasis duration is defined in terms of + unit amounts. One unit of pre-emphasis duration is approximately + 650 ps and is defined as 1X pre-emphasis duration. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: 10000 + maximum: 20000 + + qcom,pre-emphasis-amplitude-bp: + description: + This signal controls the amount of current sourced to + DP<#> and DM<#> after a J-to-K or K-to-J transition. + The HS Transmitter pre-emphasis current is defined in terms of unit + amounts. One unit amount is approximately 2 mA and is defined as + 1X pre-emphasis current. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: 10000 + maximum: 40000 + + qcom,hs-rise-fall-time-bp: + description: + This adjusts the rise/fall times of the high-speed waveform. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -4100 + maximum: 5430 + + qcom,hs-crossover-voltage-microvolt: + description: + This adjusts the voltage at which the DP<#> and DM<#> + signals cross while transmitting in HS mode. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -31000 + maximum: 28000 + + qcom,hs-output-impedance-micro-ohms: + description: + In some applications, there can be significant series resistance + on the D+ and D- paths between the transceiver and cable. This adjusts + the driver source impedance to compensate for added series + resistance on the USB. The hardware accepts only discrete values. The + value closest to the provided input will be chosen as the override value + for this param. + minimum: -2300000 + maximum: 6100000 + + qcom,ls-fs-output-impedance-bp: + description: + This adjusts the low- and full-speed single-ended source + impedance while driving high. The following adjustment values are based + on nominal process, voltage, and temperature. + The hardware accepts only discrete values. The value closest to the + provided input will be chosen as the override value for this param. + minimum: -1053 + maximum: 1310 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt b/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt deleted file mode 100644 index a3bd1c4499b75bdb8edc604bd756cb74c8f1a90e..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt +++ /dev/null @@ -1,112 +0,0 @@ -* Renesas R-Car generation 2 USB PHY - -This file provides information on what the device node for the R-Car generation -2 USB PHY contains. - -Required properties: -- compatible: "renesas,usb-phy-r8a7742" if the device is a part of R8A7742 SoC. - "renesas,usb-phy-r8a7743" if the device is a part of R8A7743 SoC. - "renesas,usb-phy-r8a7744" if the device is a part of R8A7744 SoC. - "renesas,usb-phy-r8a7745" if the device is a part of R8A7745 SoC. - "renesas,usb-phy-r8a77470" if the device is a part of R8A77470 SoC. - "renesas,usb-phy-r8a7790" if the device is a part of R8A7790 SoC. - "renesas,usb-phy-r8a7791" if the device is a part of R8A7791 SoC. - "renesas,usb-phy-r8a7794" if the device is a part of R8A7794 SoC. - "renesas,rcar-gen2-usb-phy" for a generic R-Car Gen2 or - RZ/G1 compatible device. - - When compatible with the generic version, nodes must list the - SoC-specific version corresponding to the platform first - followed by the generic version. - -- reg: offset and length of the register block. -- #address-cells: number of address cells for the USB channel subnodes, must - be <1>. -- #size-cells: number of size cells for the USB channel subnodes, must be <0>. -- clocks: clock phandle and specifier pair. -- clock-names: string, clock input name, must be "usbhs". - -The USB PHY device tree node should have the subnodes corresponding to the USB -channels. These subnodes must contain the following properties: -- reg: the USB controller selector; see the table below for the values. -- #phy-cells: see phy-bindings.txt in the same directory, must be <1>. - -The phandle's argument in the PHY specifier is the USB controller selector for -the USB channel other than r8a77470 SoC; see the selector meanings below: - -+-----------+---------------+---------------+ -|\ Selector | | | -+ --------- + 0 | 1 | -| Channel \| | | -+-----------+---------------+---------------+ -| 0 | PCI EHCI/OHCI | HS-USB | -| 2 | PCI EHCI/OHCI | xHCI | -+-----------+---------------+---------------+ - -For r8a77470 SoC;see the selector meaning below: - -+-----------+---------------+---------------+ -|\ Selector | | | -+ --------- + 0 | 1 | -| Channel \| | | -+-----------+---------------+---------------+ -| 0 | EHCI/OHCI | HS-USB | -+-----------+---------------+---------------+ - -Example (Lager board): - - usb-phy@e6590100 { - compatible = "renesas,usb-phy-r8a7790", "renesas,rcar-gen2-usb-phy"; - reg = <0 0xe6590100 0 0x100>; - #address-cells = <1>; - #size-cells = <0>; - clocks = <&cpg CPG_MOD 704>; - clock-names = "usbhs"; - power-domains = <&sysc R8A7790_PD_ALWAYS_ON>; - resets = <&cpg 704>; - - usb0: usb-channel@0 { - reg = <0>; - #phy-cells = <1>; - }; - usb2: usb-channel@2 { - reg = <2>; - #phy-cells = <1>; - }; - }; - -Example (iWave RZ/G1C sbc): - - usbphy0: usb-phy0@e6590100 { - compatible = "renesas,usb-phy-r8a77470", - "renesas,rcar-gen2-usb-phy"; - reg = <0 0xe6590100 0 0x100>; - #address-cells = <1>; - #size-cells = <0>; - clocks = <&cpg CPG_MOD 704>; - clock-names = "usbhs"; - power-domains = <&sysc R8A77470_PD_ALWAYS_ON>; - resets = <&cpg 704>; - - usb0: usb-channel@0 { - reg = <0>; - #phy-cells = <1>; - }; - }; - - usbphy1: usb-phy@e6598100 { - compatible = "renesas,usb-phy-r8a77470", - "renesas,rcar-gen2-usb-phy"; - reg = <0 0xe6598100 0 0x100>; - #address-cells = <1>; - #size-cells = <0>; - clocks = <&cpg CPG_MOD 706>; - clock-names = "usbhs"; - power-domains = <&sysc R8A77470_PD_ALWAYS_ON>; - resets = <&cpg 706>; - - usb1: usb-channel@0 { - reg = <0>; - #phy-cells = <1>; - }; - }; diff --git a/Documentation/devicetree/bindings/phy/renesas,rcar-gen2-usb-phy.yaml b/Documentation/devicetree/bindings/phy/renesas,rcar-gen2-usb-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afc09f39b02bd2c41ddcae9973a4dfe8e1663f36 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/renesas,rcar-gen2-usb-phy.yaml @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/renesas,rcar-gen2-usb-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas R-Car Gen2 USB PHY + +maintainers: + - Yoshihiro Shimoda + +properties: + compatible: + items: + - enum: + - renesas,usb-phy-r8a7742 # RZ/G1H + - renesas,usb-phy-r8a7743 # RZ/G1M + - renesas,usb-phy-r8a7744 # RZ/G1N + - renesas,usb-phy-r8a7745 # RZ/G1E + - renesas,usb-phy-r8a77470 # RZ/G1C + - renesas,usb-phy-r8a7790 # R-Car H2 + - renesas,usb-phy-r8a7791 # R-Car M2-W + - renesas,usb-phy-r8a7794 # R-Car E2 + - const: renesas,rcar-gen2-usb-phy # R-Car Gen2 or RZ/G1 + + reg: + maxItems: 1 + + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: usbhs + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + +patternProperties: + "^usb-phy@[02]$": + type: object + description: Subnode corresponding to a USB channel. + + properties: + reg: + description: FIXME RZ/G1C supports channel 0 only + enum: [0, 2] + + '#phy-cells': + description: | + The phandle's argument in the PHY specifier is the USB controller + selector for the USB channel. + For RZ/G1C: + - 0 for EHCI/OHCI + - 1 for HS-USB + For all other SoCS: + - 0 for PCI EHCI/OHCI + - 1 for HS-USB (channel 0) or xHCI (channel 2) + const: 1 + + required: + - reg + - '#phy-cells' + + additionalProperties: false + +required: + - compatible + - reg + - '#address-cells' + - '#size-cells' + - clocks + - clock-names + - resets + - power-domains + - usb-phy@0 + +if: + properties: + compatible: + contains: + const: renesas,usb-phy-r8a77470 +then: + properties: + usb-phy@2: false +else: + required: + - usb-phy@2 + +additionalProperties: false + +examples: + - | + #include + #include + usb-phy-controller@e6590100 { + compatible = "renesas,usb-phy-r8a7790", "renesas,rcar-gen2-usb-phy"; + reg = <0xe6590100 0x100>; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&cpg CPG_MOD 704>; + clock-names = "usbhs"; + power-domains = <&sysc R8A7790_PD_ALWAYS_ON>; + resets = <&cpg 704>; + + usb0: usb-phy@0 { + reg = <0>; + #phy-cells = <1>; + }; + usb2: usb-phy@2 { + reg = <2>; + #phy-cells = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/phy/rockchip,pcie3-phy.yaml b/Documentation/devicetree/bindings/phy/rockchip,pcie3-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f2d8d2cc7a5468bb42208a92f6724e250711c7a --- /dev/null +++ b/Documentation/devicetree/bindings/phy/rockchip,pcie3-phy.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/rockchip,pcie3-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Rockchip PCIe v3 phy + +maintainers: + - Heiko Stuebner + +properties: + compatible: + enum: + - rockchip,rk3568-pcie3-phy + + reg: + maxItems: 1 + + clocks: + minItems: 3 + maxItems: 3 + + clock-names: + items: + - const: refclk_m + - const: refclk_n + - const: pclk + + data-lanes: + description: which lanes (by position) should be mapped to which + controller (value). 0 means lane disabled, higher value means used. + (controller-number +1 ) + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 2 + maxItems: 16 + items: + minimum: 0 + maximum: 16 + + "#phy-cells": + const: 0 + + resets: + maxItems: 1 + + reset-names: + const: phy + + rockchip,phy-grf: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle to the syscon managing the phy "general register files" + + rockchip,pipe-grf: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle to the syscon managing the pipe "general register files" + +required: + - compatible + - reg + - rockchip,phy-grf + - "#phy-cells" + +additionalProperties: false + +examples: + - | + #include + pcie30phy: phy@fe8c0000 { + compatible = "rockchip,rk3568-pcie3-phy"; + reg = <0xfe8c0000 0x20000>; + #phy-cells = <0>; + clocks = <&pmucru CLK_PCIE30PHY_REF_M>, + <&pmucru CLK_PCIE30PHY_REF_N>, + <&cru PCLK_PCIE30PHY>; + clock-names = "refclk_m", "refclk_n", "pclk"; + resets = <&cru SRST_PCIE30PHY>; + reset-names = "phy"; + rockchip,phy-grf = <&pcie30_phy_grf>; + }; diff --git a/Documentation/devicetree/bindings/phy/rockchip,px30-dsi-dphy.yaml b/Documentation/devicetree/bindings/phy/rockchip,px30-dsi-dphy.yaml index 8a3032a3bd736ccf8e35e13d6a30adf7ed5c2418..5c35e5ceec0bbb9a5ffdfbde65c4707a857286f9 100644 --- a/Documentation/devicetree/bindings/phy/rockchip,px30-dsi-dphy.yaml +++ b/Documentation/devicetree/bindings/phy/rockchip,px30-dsi-dphy.yaml @@ -18,6 +18,7 @@ properties: - rockchip,px30-dsi-dphy - rockchip,rk3128-dsi-dphy - rockchip,rk3368-dsi-dphy + - rockchip,rk3568-dsi-dphy reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml b/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml index bb4a2e4b8ab0e1a01c3203bec6acd8ca18bf8b6a..0e6505e9da50504a4530ff082ab2220d51b163b1 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml +++ b/Documentation/devicetree/bindings/phy/rockchip-inno-csi-dphy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/rockchip-inno-csi-dphy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip SoC MIPI RX0 D-PHY Device Tree Bindings +title: Rockchip SoC MIPI RX0 D-PHY maintainers: - Heiko Stuebner @@ -20,6 +20,7 @@ properties: - rockchip,rk1808-csi-dphy - rockchip,rk3326-csi-dphy - rockchip,rk3368-csi-dphy + - rockchip,rk3568-csi-dphy reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml b/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml index 7d888d3588239fa39306be5c6e1029113c0b5c6c..5114e99b0035af1fde51b79de350b941a4b0bdf2 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml +++ b/Documentation/devicetree/bindings/phy/rockchip-mipi-dphy-rx0.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/rockchip-mipi-dphy-rx0.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip SoC MIPI RX0 D-PHY Device Tree Bindings +title: Rockchip SoC MIPI RX0 D-PHY maintainers: - Helen Koike diff --git a/Documentation/devicetree/bindings/phy/samsung,exynos-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,exynos-pcie-phy.yaml index ac0af40be52d3db3e4f7d56a5fdb53b6f793163b..28e299a9609d2831398ff092c700015d28dee467 100644 --- a/Documentation/devicetree/bindings/phy/samsung,exynos-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/samsung,exynos-pcie-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/samsung,exynos-pcie-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Samsung SoC series PCIe PHY Device Tree Bindings +title: Samsung SoC series PCIe PHY maintainers: - Marek Szyprowski diff --git a/Documentation/devicetree/bindings/phy/samsung,ufs-phy.yaml b/Documentation/devicetree/bindings/phy/samsung,ufs-phy.yaml index 8da99461e817fdfbfdef0db11cf68904194717c5..c5dbb91ac40235c47a4d4d656e432330bff2fba4 100644 --- a/Documentation/devicetree/bindings/phy/samsung,ufs-phy.yaml +++ b/Documentation/devicetree/bindings/phy/samsung,ufs-phy.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/samsung,ufs-phy.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Samsung SoC series UFS PHY Device Tree Bindings +title: Samsung SoC series UFS PHY maintainers: - Alim Akhtar @@ -27,18 +27,12 @@ properties: - const: phy-pma clocks: - items: - - description: PLL reference clock - - description: symbol clock for input symbol ( rx0-ch0 symbol clock) - - description: symbol clock for input symbol ( rx1-ch1 symbol clock) - - description: symbol clock for output symbol ( tx0 symbol clock) + minItems: 1 + maxItems: 4 clock-names: - items: - - const: ref_clk - - const: rx1_symbol_clk - - const: rx0_symbol_clk - - const: tx0_symbol_clk + minItems: 1 + maxItems: 4 samsung,pmu-syscon: $ref: '/schemas/types.yaml#/definitions/phandle-array' @@ -62,6 +56,39 @@ required: - clock-names - samsung,pmu-syscon +allOf: + - if: + properties: + compatible: + contains: + const: samsung,exynos7-ufs-phy + + then: + properties: + clocks: + items: + - description: PLL reference clock + - description: symbol clock for input symbol (rx0-ch0 symbol clock) + - description: symbol clock for input symbol (rx1-ch1 symbol clock) + - description: symbol clock for output symbol (tx0 symbol clock) + + clock-names: + items: + - const: ref_clk + - const: rx1_symbol_clk + - const: rx0_symbol_clk + - const: tx0_symbol_clk + + else: + properties: + clocks: + items: + - description: PLL reference clock + + clock-names: + items: + - const: ref_clk + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/phy/sunplus,sp7021-usb2-phy.yaml b/Documentation/devicetree/bindings/phy/sunplus,sp7021-usb2-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..069d422775bb06be61ae95ab72cfa1b50d288b4b --- /dev/null +++ b/Documentation/devicetree/bindings/phy/sunplus,sp7021-usb2-phy.yaml @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) Sunplus Co., Ltd. 2021 +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/phy/sunplus,sp7021-usb2-phy.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Sunplus SP7021 USB 2.0 PHY Controller + +maintainers: + - Vincent Shih + +properties: + compatible: + const: sunplus,sp7021-usb2-phy + + reg: + items: + - description: UPHY register region + - description: MOON4 register region + + reg-names: + items: + - const: phy + - const: moon4 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + "#phy-cells": + const: 0 + + nvmem-cell-names: + description: names corresponding to the nvmem cells of disconnect voltage + const: disc_vol + + nvmem-cells: + description: nvmem cell address of disconnect voltage + maxItems: 1 + + sunplus,disc-vol-addr-off: + $ref: /schemas/types.yaml#/definitions/uint32 + description: the otp address offset of disconnect voltage + +required: + - compatible + - reg + - reg-names + - clocks + - resets + - "#phy-cells" + - nvmem-cell-names + - nvmem-cells + - sunplus,disc-vol-addr-off + +additionalProperties: false + +examples: + - | + sp_uphy0: usb-phy@9c004a80 { + compatible = "sunplus,sp7021-usb2-phy"; + reg = <0x9c004a80 0x80>, <0x9c000248 0x10>; + reg-names = "phy", "moon4"; + clocks = <&clkc 0x3d>; + resets = <&rstc 0x2d>; + #phy-cells = <0>; + nvmem-cell-names = "disc_vol"; + nvmem-cells = <&disc_vol>; + sunplus,disc-vol-addr-off = <0>; + }; diff --git a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml index ff8a6d9eb15314897a76b9b81ebe472ab7f62189..da7cac537e15c30a81ac6e30930b84d996803029 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml @@ -53,12 +53,25 @@ properties: - ti,am43xx-phy-gmii-sel - ti,dm814-phy-gmii-sel - ti,am654-phy-gmii-sel + - ti,j7200-cpsw5g-phy-gmii-sel reg: maxItems: 1 '#phy-cells': true + ti,qsgmii-main-ports: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Required only for QSGMII mode. Array to select the port for + QSGMII main mode. Rest of the ports are selected as QSGMII_SUB + ports automatically. Any one of the 4 CPSW5G ports can act as the + main port with the rest of them being the QSGMII_SUB ports. + maxItems: 1 + items: + minimum: 1 + maximum: 4 + allOf: - if: properties: @@ -73,6 +86,18 @@ allOf: '#phy-cells': const: 1 description: CPSW port number (starting from 1) + + - if: + not: + properties: + compatible: + contains: + enum: + - ti,j7200-cpsw5g-phy-gmii-sel + then: + properties: + ti,qsgmii-main-ports: false + - if: properties: compatible: @@ -97,7 +122,7 @@ additionalProperties: false examples: - | - phy_gmii_sel: phy-gmii-sel@650 { + phy_gmii_sel: phy@650 { compatible = "ti,am3352-phy-gmii-sel"; reg = <0x650 0x4>; #phy-cells = <2>; diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml index dcd63908aeaefe6e5fe7db312f3453b75c8bedff..2225925b6dad898b8cb653245a36df910770e29e 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml @@ -16,19 +16,23 @@ properties: - ti,j721e-wiz-16g - ti,j721e-wiz-10g - ti,am64-wiz-10g + - ti,j7200-wiz-10g power-domains: maxItems: 1 clocks: - maxItems: 3 + minItems: 3 + maxItems: 4 description: clock-specifier to represent input to the WIZ clock-names: + minItems: 3 items: - const: fck - const: core_ref_clk - const: ext_ref_clk + - const: core_ref1_clk num-lanes: minimum: 1 @@ -79,10 +83,12 @@ properties: refclk-dig: type: object + additionalProperties: false description: | WIZ node should have subnode for refclk_dig to select the reference clock source for the reference clock used in the PHY and PMA digital logic. + deprecated: true properties: clocks: minItems: 2 @@ -105,12 +111,19 @@ properties: - assigned-clocks - assigned-clock-parents + ti,scm: + $ref: /schemas/types.yaml#/definitions/phandle + description: | + phandle to System Control Module for syscon regmap access. + patternProperties: "^pll[0|1]-refclk$": type: object + additionalProperties: false description: | WIZ node should have subnodes for each of the PLLs present in the SERDES. + deprecated: true properties: clocks: maxItems: 2 @@ -133,9 +146,11 @@ patternProperties: "^cmn-refclk1?-dig-div$": type: object + additionalProperties: false description: WIZ node should have subnodes for each of the PMA common refclock provided by the SERDES. + deprecated: true properties: clocks: maxItems: 1 @@ -170,6 +185,16 @@ required: - "#reset-cells" - ranges +allOf: + - if: + properties: + compatible: + contains: + const: ti,j7200-wiz-10g + then: + required: + - ti,scm + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/phy/xlnx,zynqmp-psgtr.yaml b/Documentation/devicetree/bindings/phy/xlnx,zynqmp-psgtr.yaml index 79906519c6522c34aa93d1710fb386371ac78327..cefbc8b53a834ab18d556ad9e7419dc58d6a6093 100644 --- a/Documentation/devicetree/bindings/phy/xlnx,zynqmp-psgtr.yaml +++ b/Documentation/devicetree/bindings/phy/xlnx,zynqmp-psgtr.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/phy/xlnx,zynqmp-psgtr.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx ZynqMP Gigabit Transceiver PHY Device Tree Bindings +title: Xilinx ZynqMP Gigabit Transceiver PHY maintainers: - Laurent Pinchart diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml index d19d65c870aa24bcc1b2131cff02b2f87f2144ac..1e3c8de6cae1348f8cb92b33f175499c51e7b35e 100644 --- a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/allwinner,sun4i-a10-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Pin Controller Device Tree Bindings +title: Allwinner A10 Pin Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml index d3a8911728d033d63b93058c984e13d8360be10f..f4f1ee6b116e854a6ca37f4db8b6320169ab2e6a 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml @@ -63,6 +63,12 @@ examples: syscon: scu@1e6e2000 { compatible = "aspeed,ast2400-scu", "syscon", "simple-mfd"; reg = <0x1e6e2000 0x1a8>; + #clock-cells = <1>; + #reset-cells = <1>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x1e6e2000 0x1000>; pinctrl: pinctrl { compatible = "aspeed,ast2400-pinctrl"; diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml index 5d2c1b1fb7fd058ec2d1939faf27d98226deacdf..8168f00884710e4c7c08a14a11ae709e1a70c15c 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml @@ -82,6 +82,10 @@ examples: #clock-cells = <1>; #reset-cells = <1>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x1e6e2000 0x1000>; + pinctrl: pinctrl { compatible = "aspeed,ast2500-pinctrl"; aspeed,external-nodes = <&gfx>, <&lhc>; diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml index e92686d2f062021f11225a68f5d3a9e99d1a3a07..62424c42c981944afbb091244659c6abdd20464a 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml @@ -96,6 +96,12 @@ examples: syscon: scu@1e6e2000 { compatible = "aspeed,ast2600-scu", "syscon", "simple-mfd"; reg = <0x1e6e2000 0xf6c>; + #clock-cells = <1>; + #reset-cells = <1>; + + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x1e6e2000 0x1000>; pinctrl: pinctrl { compatible = "aspeed,ast2600-pinctrl"; diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,bcm4908-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/brcm,bcm4908-pinctrl.yaml index 175a992f15e16f363069edf5dd86e38dc2dbe211..8a9fb9b433ca1ef1d0d3e306cbce05390c35f3af 100644 --- a/Documentation/devicetree/bindings/pinctrl/brcm,bcm4908-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/brcm,bcm4908-pinctrl.yaml @@ -23,6 +23,7 @@ patternProperties: '-pins$': type: object $ref: pinmux-node.yaml# + additionalProperties: false properties: function: diff --git a/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml index 533b4cfe33d2d72a4a407af8307868936b7da58e..a78cb2796001b8380b9e01f5ff57bf9e85dc9006 100644 --- a/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml +++ b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/canaan,k210-fpioa.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Canaan Kendryte K210 FPIOA Device Tree Bindings +title: Canaan Kendryte K210 FPIOA maintainers: - Damien Le Moal diff --git a/Documentation/devicetree/bindings/pinctrl/cypress,cy8c95x0.yaml b/Documentation/devicetree/bindings/pinctrl/cypress,cy8c95x0.yaml new file mode 100644 index 0000000000000000000000000000000000000000..915cbbcc3555027db32fb79a006f0370ee2a4384 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/cypress,cy8c95x0.yaml @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/cypress,cy8c95x0.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Cypress CY8C95X0 I2C GPIO expander + +maintainers: + - Patrick Rudolph + +description: | + This supports the 20/40/60 pin Cypress CYC95x0 GPIO I2C expanders. + Pin function configuration is performed on a per-pin basis. + +properties: + compatible: + enum: + - cypress,cy8c9520 + - cypress,cy8c9540 + - cypress,cy8c9560 + + reg: + maxItems: 1 + + gpio-controller: true + + '#gpio-cells': + description: + The first cell is the GPIO number and the second cell specifies GPIO + flags, as defined in . + const: 2 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + '#interrupt-cells': + const: 2 + + gpio-line-names: true + + gpio-ranges: + maxItems: 1 + + gpio-reserved-ranges: + maxItems: 1 + + vdd-supply: + description: + Optional power supply. + +patternProperties: + '-pins$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: pincfg-node.yaml# + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + pattern: '^gp([0-7][0-7])$' + minItems: 1 + maxItems: 60 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + enum: [ gpio, pwm ] + + bias-pull-down: true + + bias-pull-up: true + + bias-disable: true + + output-high: true + + output-low: true + + drive-push-pull: true + + drive-open-drain: true + + drive-open-source: true + + required: + - pins + - function + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - '#interrupt-cells' + - gpio-controller + - '#gpio-cells' + +additionalProperties: false + +allOf: + - $ref: "pinctrl.yaml#" + +examples: + - | + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + pinctrl@20 { + compatible = "cypress,cy8c9520"; + reg = <0x20>; + gpio-controller; + #gpio-cells = <2>; + #interrupt-cells = <2>; + interrupts = ; + interrupt-controller; + vdd-supply = <&p3v3>; + gpio-reserved-ranges = <5 1>; + }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-keembay.yaml b/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-keembay.yaml index 5e99d79499b491d343b1cbe7c74e5600634e3852..005d95a9e4d6a01990c2b94a0f9dd149169d0703 100644 --- a/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-keembay.yaml +++ b/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-keembay.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/intel,pinctrl-keembay.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Keem Bay pin controller Device Tree Bindings +title: Intel Keem Bay pin controller maintainers: - Lakshmi Sowjanya D @@ -44,6 +44,7 @@ properties: patternProperties: '^gpio@[0-9a-f]*$': type: object + additionalProperties: false description: Child nodes can be specified to contain pin configuration information, diff --git a/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-thunderbay.yaml b/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-thunderbay.yaml index 0ec476248f216ea971e5ff3c066d189309e38e09..f001add1681414ea29485c3101f62214ed98d75f 100644 --- a/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-thunderbay.yaml +++ b/Documentation/devicetree/bindings/pinctrl/intel,pinctrl-thunderbay.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/intel,pinctrl-thunderbay.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Thunder Bay pin controller Device Tree Bindings +title: Intel Thunder Bay pin controller maintainers: - Lakshmi Sowjanya D @@ -42,6 +42,7 @@ properties: patternProperties: '^gpio@[0-9a-f]*$': type: object + additionalProperties: false description: Child nodes can be specified to contain pin configuration information, diff --git a/Documentation/devicetree/bindings/pinctrl/marvell,ac5-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/marvell,ac5-pinctrl.yaml index a651b2744caf35b01592afa997e0cebaca68c88f..491f67e7cc4fa42db9b87bf862d7cafa9e8b51f4 100644 --- a/Documentation/devicetree/bindings/pinctrl/marvell,ac5-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/marvell,ac5-pinctrl.yaml @@ -24,6 +24,7 @@ patternProperties: '-pins$': type: object $ref: pinmux-node.yaml# + additionalProperties: false properties: marvell,function: diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml index 161088a8be335c36f33b65f4e8565e94c5d6bbbc..33b5f79e741ab6fab9cdf1a360940af8ca0e3927 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt65xx-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT65xx Pin Controller Device Tree Bindings +title: Mediatek MT65xx Pin Controller maintainers: - Sean Wang diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt6779-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt6779-pinctrl.yaml index e7601c0f5a695a5de27fb6650b1b8ea64c0301c7..8c79fcef7c5215254d01e80b553a4d875cb86fca 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt6779-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt6779-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt6779-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT6779 Pin Controller Device Tree Bindings +title: Mediatek MT6779 Pin Controller maintainers: - Andy Teng @@ -76,6 +76,8 @@ required: patternProperties: '-[0-9]*$': type: object + additionalProperties: false + patternProperties: '-pins*$': type: object diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt6797-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt6797-pinctrl.yaml index d42215f59afd4f7abdf293db1b5b2ba2dc182a0c..637a8386e23ebdc9c463a08f2c1d78b71de3540b 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt6797-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt6797-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt6797-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT6797 Pin Controller Device Tree Bindings +title: Mediatek MT6797 Pin Controller maintainers: - Sean Wang diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml index 78a0175cecc75dee222260222aea0abdf2f83147..c9ea0cad489b71c3edcbd65b4a197b382702b3a4 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7622-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt7622-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT7622 Pin Controller Device Tree Bindings +title: Mediatek MT7622 Pin Controller maintainers: - Sean Wang diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7986-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7986-pinctrl.yaml index 4eadea55df10f1bce483147d43e40b91096dfbec..89b8f3dd67a19bfc5b9d8f2c047a4565c3dfb590 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt7986-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt7986-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt7986-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT7986 Pin Controller Device Tree Bindings +title: Mediatek MT7986 Pin Controller maintainers: - Sean Wang @@ -117,6 +117,10 @@ patternProperties: "i2s" "audio" 62, 63, 64, 65 "switch_int" "eth" 66 "mdc_mdio" "eth" 67 + "wf_2g" "wifi" 74, 75, 76, 77, 78, 79, 80, 81, 82, 83 + "wf_5g" "wifi" 91, 92, 93, 94, 95, 96, 97, 98, 99, 100 + "wf_dbdc" "wifi" 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85 $ref: "/schemas/pinctrl/pinmux-node.yaml" properties: @@ -234,7 +238,9 @@ patternProperties: then: properties: groups: - enum: [wf_2g, wf_5g, wf_dbdc] + items: + enum: [wf_2g, wf_5g, wf_dbdc] + maxItems: 3 '.*conf.*': type: object additionalProperties: false @@ -248,25 +254,27 @@ patternProperties: An array of strings. Each string contains the name of a pin. There is no PIN 41 to PIN 65 above on mt7686b, you can only use those pins on mt7986a. - enum: [SYS_WATCHDOG, WF2G_LED, WF5G_LED, I2C_SCL, I2C_SDA, GPIO_0, - GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, GPIO_6, GPIO_7, - GPIO_8, GPIO_9, GPIO_10, GPIO_11, GPIO_12, GPIO_13, GPIO_14, - GPIO_15, PWM0, PWM1, SPI0_CLK, SPI0_MOSI, SPI0_MISO, SPI0_CS, - SPI0_HOLD, SPI0_WP, SPI1_CLK, SPI1_MOSI, SPI1_MISO, SPI1_CS, - SPI2_CLK, SPI2_MOSI, SPI2_MISO, SPI2_CS, SPI2_HOLD, SPI2_WP, - UART0_RXD, UART0_TXD, PCIE_PERESET_N, UART1_RXD, UART1_TXD, - UART1_CTS, UART1_RTS, UART2_RXD, UART2_TXD, UART2_CTS, - UART2_RTS, EMMC_DATA_0, EMMC_DATA_1, EMMC_DATA_2, - EMMC_DATA_3, EMMC_DATA_4, EMMC_DATA_5, EMMC_DATA_6, - EMMC_DATA_7, EMMC_CMD, EMMC_CK, EMMC_DSL, EMMC_RSTB, PCM_DTX, - PCM_DRX, PCM_CLK, PCM_FS, MT7531_INT, SMI_MDC, SMI_MDIO, - WF0_DIG_RESETB, WF0_CBA_RESETB, WF0_XO_REQ, WF0_TOP_CLK, - WF0_TOP_DATA, WF0_HB1, WF0_HB2, WF0_HB3, WF0_HB4, WF0_HB0, - WF0_HB0_B, WF0_HB5, WF0_HB6, WF0_HB7, WF0_HB8, WF0_HB9, - WF0_HB10, WF1_DIG_RESETB, WF1_CBA_RESETB, WF1_XO_REQ, - WF1_TOP_CLK, WF1_TOP_DATA, WF1_HB1, WF1_HB2, WF1_HB3, - WF1_HB4, WF1_HB0, WF1_HB0_B, WF1_HB5, WF1_HB6, WF1_HB7, - WF1_HB8] + items: + enum: [SYS_WATCHDOG, WF2G_LED, WF5G_LED, I2C_SCL, I2C_SDA, GPIO_0, + GPIO_1, GPIO_2, GPIO_3, GPIO_4, GPIO_5, GPIO_6, GPIO_7, + GPIO_8, GPIO_9, GPIO_10, GPIO_11, GPIO_12, GPIO_13, GPIO_14, + GPIO_15, PWM0, PWM1, SPI0_CLK, SPI0_MOSI, SPI0_MISO, SPI0_CS, + SPI0_HOLD, SPI0_WP, SPI1_CLK, SPI1_MOSI, SPI1_MISO, SPI1_CS, + SPI2_CLK, SPI2_MOSI, SPI2_MISO, SPI2_CS, SPI2_HOLD, SPI2_WP, + UART0_RXD, UART0_TXD, PCIE_PERESET_N, UART1_RXD, UART1_TXD, + UART1_CTS, UART1_RTS, UART2_RXD, UART2_TXD, UART2_CTS, + UART2_RTS, EMMC_DATA_0, EMMC_DATA_1, EMMC_DATA_2, + EMMC_DATA_3, EMMC_DATA_4, EMMC_DATA_5, EMMC_DATA_6, + EMMC_DATA_7, EMMC_CMD, EMMC_CK, EMMC_DSL, EMMC_RSTB, PCM_DTX, + PCM_DRX, PCM_CLK, PCM_FS, MT7531_INT, SMI_MDC, SMI_MDIO, + WF0_DIG_RESETB, WF0_CBA_RESETB, WF0_XO_REQ, WF0_TOP_CLK, + WF0_TOP_DATA, WF0_HB1, WF0_HB2, WF0_HB3, WF0_HB4, WF0_HB0, + WF0_HB0_B, WF0_HB5, WF0_HB6, WF0_HB7, WF0_HB8, WF0_HB9, + WF0_HB10, WF1_DIG_RESETB, WF1_CBA_RESETB, WF1_XO_REQ, + WF1_TOP_CLK, WF1_TOP_DATA, WF1_HB1, WF1_HB2, WF1_HB3, + WF1_HB4, WF1_HB0, WF1_HB0_B, WF1_HB5, WF1_HB6, WF1_HB7, + WF1_HB8] + maxItems: 101 bias-disable: true diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml index 2d13a57b89615aacc9721b1b0f529518a66c0fb6..0d2484056a0fefed30355dbc02a3aeaaa91c4882 100644 --- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8183-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/mediatek,mt8183-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT8183 Pin Controller Device Tree Bindings +title: Mediatek MT8183 Pin Controller maintainers: - Sean Wang diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e750f1e643d02afb8ad46ca989c9ef3177ee8fb --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/mediatek,mt8188-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MT8188 Pin Controller + +maintainers: + - Hui Liu + +description: | + The MediaTek's MT8188 Pin controller is used to control SoC pins. + +properties: + compatible: + const: mediatek,mt8188-pinctrl + + gpio-controller: true + + '#gpio-cells': + description: | + Number of cells in GPIO specifier, should be two. The first cell + is the pin number, the second cell is used to specify optional + parameters which are defined in . + const: 2 + + gpio-ranges: + maxItems: 1 + + gpio-line-names: true + + reg: + items: + - description: gpio registers base address + - description: rm group io configuration registers base address + - description: lt group io configuration registers base address + - description: lm group io configuration registers base address + - description: rt group io configuration registers base address + - description: eint registers base address + + reg-names: + items: + - const: iocfg0 + - const: iocfg_rm + - const: iocfg_lt + - const: iocfg_lm + - const: iocfg_rt + - const: eint + + interrupt-controller: true + + '#interrupt-cells': + const: 2 + + interrupts: + description: The interrupt outputs to sysirq. + maxItems: 1 + + mediatek,rsel-resistance-in-si-unit: + type: boolean + description: | + We provide two methods to select the resistance for I2C when pull up or pull down. + The first is by RSEL definition value, another one is by resistance value(ohm). + This flag is used to identify if the method is resistance(si unit) value. + +# PIN CONFIGURATION NODES +patternProperties: + '-pins$': + type: object + additionalProperties: false + + patternProperties: + '^pins': + type: object + $ref: "/schemas/pinctrl/pincfg-node.yaml" + additionalProperties: false + description: | + A pinctrl node should contain at least one subnode representing the + pinctrl groups available on the machine. Each subnode will list the + pins it needs, and how they should be configured, with regard to muxer + configuration, pullups, drive strength, input enable/disable and + input schmitt. + + properties: + pinmux: + description: | + Integer array, represents gpio pin number and mux setting. + Supported pin number and mux varies for different SoCs, and are + defined as macros in dt-bindings/pinctrl/mediatek,-pinfunc.h + directly. + + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + + drive-strength-microamp: + enum: [125, 250, 500, 1000] + + bias-pull-down: + oneOf: + - type: boolean + - enum: [100, 101, 102, 103] + description: mt8188 pull down PUPD/R0/R1 type define value. + - enum: [200, 201, 202, 203, 204, 205, 206, 207] + description: mt8188 pull down RSEL type define value. + - enum: [75000, 5000] + description: mt8188 pull down RSEL type si unit value(ohm). + description: | + For pull down type is normal, it doesn't need add RSEL & R1R0 define + and resistance value. + For pull down type is PUPD/R0/R1 type, it can add R1R0 define to + set different resistance. It can support "MTK_PUPD_SET_R1R0_00" & + "MTK_PUPD_SET_R1R0_01" & "MTK_PUPD_SET_R1R0_10" & "MTK_PUPD_SET_R1R0_11" + define in mt8188. + For pull down type is RSEL, it can add RSEL define & resistance value(ohm) + to set different resistance by identifying property "mediatek,rsel-resistance-in-si-unit". + It can support "MTK_PULL_SET_RSEL_000" & "MTK_PULL_SET_RSEL_001" + & "MTK_PULL_SET_RSEL_010" & "MTK_PULL_SET_RSEL_011" & "MTK_PULL_SET_RSEL_100" + & "MTK_PULL_SET_RSEL_101" & "MTK_PULL_SET_RSEL_110" & "MTK_PULL_SET_RSEL_111" + define in mt8188. It can also support resistance value(ohm) "75000" & "5000" in mt8188. + + bias-pull-up: + oneOf: + - type: boolean + - enum: [100, 101, 102, 103] + description: mt8188 pull up PUPD/R0/R1 type define value. + - enum: [200, 201, 202, 203, 204, 205, 206, 207] + description: mt8188 pull up RSEL type define value. + - enum: [1000, 1500, 2000, 3000, 4000, 5000, 10000, 75000] + description: mt8188 pull up RSEL type si unit value(ohm). + description: | + For pull up type is normal, it don't need add RSEL & R1R0 define + and resistance value. + For pull up type is PUPD/R0/R1 type, it can add R1R0 define to + set different resistance. It can support "MTK_PUPD_SET_R1R0_00" & + "MTK_PUPD_SET_R1R0_01" & "MTK_PUPD_SET_R1R0_10" & "MTK_PUPD_SET_R1R0_11" + define in mt8188. + For pull up type is RSEL, it can add RSEL define & resistance value(ohm) + to set different resistance by identifying property "mediatek,rsel-resistance-in-si-unit". + It can support "MTK_PULL_SET_RSEL_000" & "MTK_PULL_SET_RSEL_001" + & "MTK_PULL_SET_RSEL_010" & "MTK_PULL_SET_RSEL_011" & "MTK_PULL_SET_RSEL_100" + & "MTK_PULL_SET_RSEL_101" & "MTK_PULL_SET_RSEL_110" & "MTK_PULL_SET_RSEL_111" + define in mt8188. It can also support resistance value(ohm) + "1000" & "1500" & "2000" & "3000" & "4000" & "5000" & "10000" & "75000" in mt8188. + + bias-disable: true + + output-high: true + + output-low: true + + input-enable: true + + input-disable: true + + input-schmitt-enable: true + + input-schmitt-disable: true + + required: + - pinmux + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - '#interrupt-cells' + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: false + +examples: + - | + #include + #include + + pio: pinctrl@10005000 { + compatible = "mediatek,mt8188-pinctrl"; + reg = <0x10005000 0x1000>, + <0x11c00000 0x1000>, + <0x11e10000 0x1000>, + <0x11e20000 0x1000>, + <0x11ea0000 0x1000>, + <0x1000b000 0x1000>; + reg-names = "iocfg0", "iocfg_rm", + "iocfg_lt", "iocfg_lm", "iocfg_rt", + "eint"; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pio 0 0 176>; + interrupt-controller; + interrupts = ; + #interrupt-cells = <2>; + + pio-pins { + pins { + pinmux = ; + output-low; + }; + }; + + spi0-pins { + pins-spi { + pinmux = , + , + ; + drive-strength = <6>; + }; + pins-spi-mi { + pinmux = ; + bias-pull-down = ; + }; + }; + + i2c0-pins { + pins { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/nuvoton,wpcm450-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/nuvoton,wpcm450-pinctrl.yaml index 7a11beb8f222dd7d77d38c78c7caa561b9cb54f1..7b7f840ffc4cf1eaef61affdb8c4bfeb1c9ed3af 100644 --- a/Documentation/devicetree/bindings/pinctrl/nuvoton,wpcm450-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/nuvoton,wpcm450-pinctrl.yaml @@ -30,6 +30,7 @@ patternProperties: "^gpio@[0-7]$": type: object + additionalProperties: false description: Eight GPIO banks (gpio@0 to gpio@7), that each contain between 14 and 18 diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8186.yaml b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8186.yaml index 1eeb885ce0c6b5dbffd21be675ecb3c5b3458ef3..26573a793b576e21692387b866779eb2b91b1e7c 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8186.yaml +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8186.yaml @@ -41,12 +41,12 @@ properties: Gpio base register names. items: - const: iocfg0 - - const: iocfg_bm - - const: iocfg_bl - - const: iocfg_br + - const: iocfg_lt - const: iocfg_lm + - const: iocfg_lb + - const: iocfg_bl - const: iocfg_rb - - const: iocfg_tl + - const: iocfg_rt - const: eint interrupt-controller: true @@ -235,9 +235,9 @@ examples: <0x10002A00 0x0200>, <0x10002c00 0x0200>, <0x1000b000 0x1000>; - reg-names = "iocfg0", "iocfg_bm", "iocfg_bl", - "iocfg_br", "iocfg_lm", "iocfg_rb", - "iocfg_tl", "eint"; + reg-names = "iocfg0", "iocfg_lt", "iocfg_lm", + "iocfg_lb", "iocfg_bl", "iocfg_rb", + "iocfg_rt", "eint"; gpio-controller; #gpio-cells = <2>; gpio-ranges = <&pio 0 0 185>; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml index 694898f382be5f1ecf86b0fa2918450bfdb82032..29dd503f95221fb6851719f53455c7f2b2c82f4e 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-gpio.yaml @@ -24,6 +24,7 @@ properties: - qcom,pm6150-gpio - qcom,pm6150l-gpio - qcom,pm6350-gpio + - qcom,pm7250b-gpio - qcom,pm7325-gpio - qcom,pm8005-gpio - qcom,pm8008-gpio @@ -231,6 +232,7 @@ allOf: enum: - qcom,pm660l-gpio - qcom,pm6150l-gpio + - qcom,pm7250b-gpio - qcom,pm8038-gpio - qcom,pm8150b-gpio - qcom,pm8150l-gpio @@ -392,6 +394,7 @@ $defs: - gpio1-gpio10 for pm6150 - gpio1-gpio12 for pm6150l - gpio1-gpio9 for pm6350 + - gpio1-gpio12 for pm7250b - gpio1-gpio10 for pm7325 - gpio1-gpio4 for pm8005 - gpio1-gpio2 for pm8008 @@ -407,6 +410,7 @@ $defs: - gpio1-gpio10 for pm8350 - gpio1-gpio8 for pm8350b - gpio1-gpio9 for pm8350c + - gpio1-gpio4 for pm8450 - gpio1-gpio38 for pm8917 - gpio1-gpio44 for pm8921 - gpio1-gpio36 for pm8941 diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-lpass-lpi-pinctrl.yaml index 33d1d37fdf6db1496f15bacde42e145f2575c51e..624e14f007900473711054e44a9d5c023f85734c 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-lpass-lpi-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-lpass-lpi-pinctrl.yaml @@ -8,7 +8,6 @@ title: Qualcomm Technologies, Inc. Low Power Audio SubSystem (LPASS) Low Power Island (LPI) TLMM block maintainers: - - Srinivasa Rao Mandadapu - Srinivas Kandagatla description: | diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-pinctrl.yaml index 2d228164357c2cffbccc3a685791c82b5205c16d..ad3496784678b7fa51a73513ef97af69e56dd1eb 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc7280-pinctrl.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Qualcomm Technologies, Inc. SC7280 TLMM block maintainers: - - Rajendra Nayak + - Bjorn Andersson description: | This binding describes the Top Level Mode Multiplexer block found in the @@ -42,6 +42,9 @@ properties: gpio-ranges: maxItems: 1 + gpio-line-names: + maxItems: 174 + wakeup-parent: true #PIN CONFIGURATION NODES @@ -51,7 +54,6 @@ patternProperties: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "/schemas/pinctrl/pincfg-node.yaml" properties: pins: @@ -60,7 +62,7 @@ patternProperties: subnode. items: oneOf: - - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-4])$" + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|18[0-2])$" - enum: [ sdc1_rclk, sdc1_clk, sdc1_cmd, sdc1_data, sdc2_clk, sdc2_cmd, sdc2_data, ufs_reset ] minItems: 1 @@ -118,12 +120,21 @@ patternProperties: required: - pins - - function + + allOf: + - $ref: /schemas/pinctrl/pincfg-node.yaml + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|18[0-2])$" + then: + required: + - function additionalProperties: false allOf: - - $ref: "pinctrl.yaml#" + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# required: - compatible @@ -139,22 +150,22 @@ additionalProperties: false examples: - | - #include - tlmm: pinctrl@f000000 { - compatible = "qcom,sc7280-pinctrl"; - reg = <0xf000000 0x1000000>; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 175>; - wakeup-parent = <&pdc>; - - qup_uart5_default: qup-uart5-pins { - pins = "gpio46", "gpio47"; - function = "qup13"; - drive-strength = <2>; - bias-disable; - }; + #include + tlmm: pinctrl@f000000 { + compatible = "qcom,sc7280-pinctrl"; + reg = <0xf000000 0x1000000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 175>; + wakeup-parent = <&pdc>; + + qup_uart5_default: qup-uart5-pins { + pins = "gpio46", "gpio47"; + function = "qup13"; + drive-strength = <2>; + bias-disable; }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml index 86509172603d39464d7c1be54d5c93928e0961d8..b98eeba2c530b46e743b7535a646ce69ad284311 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml @@ -51,8 +51,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sc8180x-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sc8180x-tlmm-state" + additionalProperties: false '$defs': qcom-sc8180x-tlmm-state: @@ -60,7 +61,6 @@ patternProperties: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -111,43 +111,52 @@ patternProperties: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-8][0-9])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@3100000 { - compatible = "qcom,sc8180x-tlmm"; - reg = <0x03100000 0x300000>, - <0x03500000 0x700000>, - <0x03d00000 0x300000>; - reg-names = "west", "east", "south"; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 190>; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio4"; - function = "qup6"; - bias-pull-up; - }; - - tx { - pins = "gpio5"; - function = "qup6"; - bias-disable; - }; - }; + #include + pinctrl@3100000 { + compatible = "qcom,sc8180x-tlmm"; + reg = <0x03100000 0x300000>, + <0x03500000 0x700000>, + <0x03d00000 0x300000>; + reg-names = "west", "east", "south"; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 190>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx-pins { + pins = "gpio4"; + function = "qup6"; + bias-pull-up; + }; + + tx-pins { + pins = "gpio5"; + function = "qup6"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-lpass-lpi-pinctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f468303bb08be57a85e1a36c386819c9849a74c --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-lpass-lpi-pinctrl.yaml @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sc8280xp-lpass-lpi-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Low Power Audio SubSystem (LPASS) + Low Power Island (LPI) TLMM block + +maintainers: + - Srinivas Kandagatla + +description: | + This binding describes the Top Level Mode Multiplexer block found in the + LPASS LPI IP on most Qualcomm SoCs + +properties: + compatible: + const: qcom,sc8280xp-lpass-lpi-pinctrl + + reg: + items: + - description: LPASS LPI TLMM Control and Status registers + - description: LPASS LPI pins SLEW registers + + clocks: + items: + - description: LPASS Core voting clock + - description: LPASS Audio voting clock + + clock-names: + items: + - const: core + - const: audio + + gpio-controller: true + + '#gpio-cells': + description: Specifying the pin number and flags, as defined in + include/dt-bindings/gpio/gpio.h + const: 2 + + gpio-ranges: + maxItems: 1 + +#PIN CONFIGURATION NODES +patternProperties: + '-pins$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: /schemas/pinctrl/pincfg-node.yaml + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + pattern: "^gpio([0-1]|1[0-8]])$" + + function: + enum: [ swr_tx_clk, swr_tx_data, swr_rx_clk, swr_rx_data, + dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic4_clk, + dmic4_data, i2s2_clk, i2s2_ws, dmic3_clk, dmic3_data, + qua_mi2s_sclk, qua_mi2s_ws, qua_mi2s_data, i2s1_clk, i2s1_ws, + i2s1_data, wsa_swr_clk, wsa_swr_data, wsa2_swr_clk, + wsa2_swr_data, i2s2_data, i2s3_clk, i2s3_ws, i2s3_data, + ext_mclk1_c, ext_mclk1_b, ext_mclk1_a ] + description: + Specify the alternative function to be configured for the specified + pins. + + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + default: 2 + description: + Selects the drive strength for the specified pins, in mA. + + slew-rate: + enum: [0, 1, 2, 3] + default: 0 + description: | + 0: No adjustments + 1: Higher Slew rate (faster edges) + 2: Lower Slew rate (slower edges) + 3: Reserved (No adjustments) + + bias-pull-down: true + + bias-pull-up: true + + bias-disable: true + + output-high: true + + output-low: true + + required: + - pins + - function + + additionalProperties: false + +allOf: + - $ref: pinctrl.yaml# + +required: + - compatible + - reg + - clocks + - clock-names + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: false + +examples: + - | + #include + pinctrl@33c0000 { + compatible = "qcom,sc8280xp-lpass-lpi-pinctrl"; + reg = <0x33c0000 0x20000>, + <0x3550000 0x10000>; + clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>; + clock-names = "core", "audio"; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&lpi_tlmm 0 0 18>; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-pinctrl.yaml index 87a381c9a19dc1ae5736e539a8a7d4cf9a423796..b9ab130cd558d4127dc43f7b8e6aee4bed691cc8 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8280xp-pinctrl.yaml @@ -43,8 +43,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sc8280xp-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sc8280xp-tlmm-state" + additionalProperties: false '$defs': qcom-sc8280xp-tlmm-state: @@ -52,7 +53,6 @@ patternProperties: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -112,40 +112,49 @@ patternProperties: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-1][0-9]|22[0-7])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@f100000 { - compatible = "qcom,sc8280xp-tlmm"; - reg = <0x0f100000 0x300000>; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 230>; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio4"; - function = "qup14"; - bias-pull-up; - }; - - tx { - pins = "gpio5"; - function = "qup14"; - bias-disable; - }; - }; + #include + pinctrl@f100000 { + compatible = "qcom,sc8280xp-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 230>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx-pins { + pins = "gpio4"; + function = "qup14"; + bias-pull-up; + }; + + tx-pins { + pins = "gpio5"; + function = "qup14"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm6115-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm6115-pinctrl.yaml index a7a2bb8bff467cb5fdf24d50a67e22f76424f259..e39fbb36d8c1c8ccabaa640cae39f58fc58a80dd 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm6115-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm6115-pinctrl.yaml @@ -49,6 +49,8 @@ properties: gpio-ranges: maxItems: 1 + gpio-reserved-ranges: true + wakeup-parent: true #PIN CONFIGURATION NODES @@ -57,8 +59,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm6115-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm6115-tlmm-state" + additionalProperties: false '$defs': qcom-sm6115-tlmm-state: @@ -66,7 +69,6 @@ patternProperties: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -118,6 +120,16 @@ patternProperties: required: - pins + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|10[0-9]|11[0-2])$" + then: + required: + - function + additionalProperties: false allOf: @@ -138,44 +150,44 @@ additionalProperties: false examples: - | - #include - tlmm: pinctrl@500000 { - compatible = "qcom,sm6115-tlmm"; - reg = <0x500000 0x400000>, - <0x900000 0x400000>, - <0xd00000 0x400000>; - reg-names = "west", "south", "east"; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 114>; - - sdc2_on_state: sdc2-on-state { - clk { - pins = "sdc2_clk"; - bias-disable; - drive-strength = <16>; - }; - - cmd { - pins = "sdc2_cmd"; - bias-pull-up; - drive-strength = <10>; - }; - - data { - pins = "sdc2_data"; - bias-pull-up; - drive-strength = <10>; - }; - - sd-cd { - pins = "gpio88"; - function = "gpio"; - bias-pull-up; - drive-strength = <2>; - }; - }; + #include + tlmm: pinctrl@500000 { + compatible = "qcom,sm6115-tlmm"; + reg = <0x500000 0x400000>, + <0x900000 0x400000>, + <0xd00000 0x400000>; + reg-names = "west", "south", "east"; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 114>; + + sdc2_on_state: sdc2-on-state { + clk-pins { + pins = "sdc2_clk"; + bias-disable; + drive-strength = <16>; + }; + + cmd-pins { + pins = "sdc2_cmd"; + bias-pull-up; + drive-strength = <10>; + }; + + data-pins { + pins = "sdc2_data"; + bias-pull-up; + drive-strength = <10>; + }; + + sd-cd-pins { + pins = "gpio88"; + function = "gpio"; + bias-pull-up; + drive-strength = <2>; + }; }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm6125-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm6125-pinctrl.yaml index c8eec845ade92411ce51beaa6ac1be4e4b135acc..5cb8b272cb7d895e9781ddd4dcab7eaefab0dae2 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm6125-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm6125-pinctrl.yaml @@ -51,8 +51,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm6125-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm6125-tlmm-state" + additionalProperties: false $defs: qcom-sm6125-tlmm-state: @@ -60,7 +61,6 @@ $defs: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -111,23 +111,52 @@ $defs: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio[0-9]|[1-9][0-9]|1[0-2][0-9]|13[0-2]$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@500000 { - compatible = "qcom,sm6125-tlmm"; - reg = <0x00500000 0x400000>, - <0x00900000 0x400000>, - <0x00d00000 0x400000>; - reg-names = "west", "south", "east"; - interrupts = ; - gpio-controller; - gpio-ranges = <&tlmm 0 0 134>; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; + #include + pinctrl@500000 { + compatible = "qcom,sm6125-tlmm"; + reg = <0x00500000 0x400000>, + <0x00900000 0x400000>, + <0x00d00000 0x400000>; + reg-names = "west", "south", "east"; + interrupts = ; + gpio-controller; + gpio-ranges = <&tlmm 0 0 134>; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + + sdc2-off-state { + clk-pins { + pins = "sdc2_clk"; + drive-strength = <2>; + bias-disable; + }; + + cmd-pins { + pins = "sdc2_cmd"; + drive-strength = <2>; + bias-pull-up; + }; + + data-pins { + pins = "sdc2_data"; + drive-strength = <2>; + bias-pull-up; + }; }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-pinctrl.yaml index 898608671c4be820fe903183f122fbe7a618441f..856b9c567ecb92aacaaf173d161386c8ee262d7f 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-pinctrl.yaml @@ -44,8 +44,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm6350-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm6350-tlmm-state" + additionalProperties: false $defs: qcom-sm6350-tlmm-state: @@ -53,7 +54,6 @@ $defs: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -110,40 +110,49 @@ $defs: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-4][0-9]|15[0-7])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@f100000 { - compatible = "qcom,sm6350-tlmm"; - reg = <0x0f100000 0x300000>; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 157>; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio25"; - function = "qup13_f2"; - bias-disable; - }; - - tx { - pins = "gpio26"; - function = "qup13_f2"; - bias-disable; - }; - }; + #include + pinctrl@f100000 { + compatible = "qcom,sm6350-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 157>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx-pins { + pins = "gpio25"; + function = "qup13_f2"; + bias-disable; + }; + + tx-pins { + pins = "gpio26"; + function = "qup13_f2"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm6375-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm6375-tlmm.yaml index 3908807a8339e6dc3b4a911be93344dc8165082c..025faf87d147a160dcc7360be0a8a29ace82e642 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm6375-tlmm.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm6375-tlmm.yaml @@ -44,8 +44,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm6375-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm6375-tlmm-state" + additionalProperties: false $defs: qcom-sm6375-tlmm-state: @@ -53,7 +54,6 @@ $defs: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -119,40 +119,49 @@ $defs: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-4][0-9]|15[0-6])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@500000 { - compatible = "qcom,sm6375-tlmm"; - reg = <0x00500000 0x800000>; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 157>; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio18"; - function = "qup13_f2"; - bias-pull-up; - }; - - tx { - pins = "gpio19"; - function = "qup13_f2"; - bias-disable; - }; - }; + #include + pinctrl@500000 { + compatible = "qcom,sm6375-tlmm"; + reg = <0x00500000 0x800000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 157>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx-pins { + pins = "gpio18"; + function = "qup13_f2"; + bias-pull-up; + }; + + tx-pins { + pins = "gpio19"; + function = "qup13_f2"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8250-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8250-pinctrl.yaml index 15bb1018cf21a80dde06d042ba19bdb2d5e18d92..c44d02d28bc9f1196803beed2a3f641d2a295fad 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm8250-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8250-pinctrl.yaml @@ -110,7 +110,16 @@ patternProperties: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9])$" + then: + required: + - function additionalProperties: false @@ -132,18 +141,18 @@ additionalProperties: false examples: - | - #include - pinctrl@1f00000 { - compatible = "qcom,sm8250-pinctrl"; - reg = <0x0f100000 0x300000>, - <0x0f500000 0x300000>, - <0x0f900000 0x300000>; - reg-names = "west", "south", "north"; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 180>; - wakeup-parent = <&pdc>; - }; + #include + pinctrl@1f00000 { + compatible = "qcom,sm8250-pinctrl"; + reg = <0x0f100000 0x300000>, + <0x0f500000 0x300000>, + <0x0f900000 0x300000>; + reg-names = "west", "south", "north"; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 180>; + wakeup-parent = <&pdc>; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml index 6b7789db2f756365db49450ec9db5224e35fb507..6ae5571f60da0a262a015034e360e115d8aeba62 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml @@ -44,8 +44,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm8350-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm8350-tlmm-state" + additionalProperties: false $defs: qcom-sm8350-tlmm-state: @@ -53,7 +54,6 @@ $defs: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -107,40 +107,49 @@ $defs: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-3])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@f100000 { - compatible = "qcom,sm8350-tlmm"; - reg = <0x0f100000 0x300000>; - interrupts = ; - gpio-controller; - #gpio-cells = <2>; - interrupt-controller; - #interrupt-cells = <2>; - gpio-ranges = <&tlmm 0 0 203>; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio18"; - function = "qup3"; - bias-pull-up; - }; - - tx { - pins = "gpio19"; - function = "qup3"; - bias-disable; - }; - }; + #include + pinctrl@f100000 { + compatible = "qcom,sm8350-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 203>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx-pins { + pins = "gpio18"; + function = "qup3"; + bias-pull-up; + }; + + tx-pins { + pins = "gpio19"; + function = "qup3"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3694795ec7938df14b9e185d35aef57974d1e854 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sm8450-lpass-lpi-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Low Power Audio SubSystem (LPASS) + Low Power Island (LPI) TLMM block + +maintainers: + - Srinivas Kandagatla + +description: | + This binding describes the Top Level Mode Multiplexer block found in the + LPASS LPI IP on most Qualcomm SoCs + +properties: + compatible: + const: qcom,sm8450-lpass-lpi-pinctrl + + reg: + items: + - description: LPASS LPI TLMM Control and Status registers + - description: LPASS LPI pins SLEW registers + + clocks: + items: + - description: LPASS Core voting clock + - description: LPASS Audio voting clock + + clock-names: + items: + - const: core + - const: audio + + gpio-controller: true + + '#gpio-cells': + description: Specifying the pin number and flags, as defined in + include/dt-bindings/gpio/gpio.h + const: 2 + + gpio-ranges: + maxItems: 1 + +#PIN CONFIGURATION NODES +patternProperties: + '-pins$': + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: /schemas/pinctrl/pincfg-node.yaml + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + pattern: "^gpio([0-9]|[1-2][0-9]])$" + + function: + enum: [ swr_tx_clk, swr_tx_data, swr_rx_clk, swr_rx_data, + dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic4_clk, + dmic4_data, i2s2_clk, i2s2_ws, dmic3_clk, dmic3_data, + qua_mi2s_sclk, qua_mi2s_ws, qua_mi2s_data, i2s1_clk, i2s1_ws, + i2s1_data, wsa_swr_clk, wsa_swr_data, wsa2_swr_clk, + wsa2_swr_data, i2s2_data, i2s4_ws, i2s4_clk, i2s4_data, + slimbus_clk, i2s3_clk, i2s3_ws, i2s3_data, slimbus_data, + ext_mclk1_c, ext_mclk1_b, ext_mclk1_a, ext_mclk1_d, + ext_mclk1_e ] + description: + Specify the alternative function to be configured for the specified + pins. + + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + default: 2 + description: + Selects the drive strength for the specified pins, in mA. + + slew-rate: + enum: [0, 1, 2, 3] + default: 0 + description: | + 0: No adjustments + 1: Higher Slew rate (faster edges) + 2: Lower Slew rate (slower edges) + 3: Reserved (No adjustments) + + bias-pull-down: true + + bias-pull-up: true + + bias-disable: true + + output-high: true + + output-low: true + + required: + - pins + - function + + additionalProperties: false + +allOf: + - $ref: pinctrl.yaml# + +required: + - compatible + - reg + - clocks + - clock-names + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: false + +examples: + - | + #include + pinctrl@3440000 { + compatible = "qcom,sm8450-lpass-lpi-pinctrl"; + reg = <0x3440000 0x20000>, + <0x34d0000 0x10000>; + clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>, + <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>; + clock-names = "core", "audio"; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&lpi_tlmm 0 0 23>; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-pinctrl.yaml index 9c891246245b76d72cf6268c8de1abd18074823f..9cd97a467648ae0f871eae557ff14a225f1d4a7d 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8450-pinctrl.yaml @@ -27,7 +27,14 @@ properties: interrupt-controller: true '#interrupt-cells': true gpio-controller: true - gpio-reserved-ranges: true + + gpio-reserved-ranges: + minItems: 1 + maxItems: 105 + + gpio-line-names: + maxItems: 209 + '#gpio-cells': true gpio-ranges: true wakeup-parent: true @@ -43,8 +50,9 @@ patternProperties: oneOf: - $ref: "#/$defs/qcom-sm8450-tlmm-state" - patternProperties: - ".*": + "-pins$": $ref: "#/$defs/qcom-sm8450-tlmm-state" + additionalProperties: false $defs: qcom-sm8450-tlmm-state: @@ -52,7 +60,6 @@ $defs: description: Pinctrl node's client devices use subnodes for desired pin configuration. Client device subnodes use below standard properties. - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" properties: pins: @@ -104,40 +111,49 @@ $defs: required: - pins - - function + + allOf: + - $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + - if: + properties: + pins: + pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-9])$" + then: + required: + - function additionalProperties: false examples: - | - #include - pinctrl@f100000 { - compatible = "qcom,sm8450-tlmm"; - reg = <0x0f100000 0x300000>; - gpio-controller; - #gpio-cells = <2>; - gpio-ranges = <&tlmm 0 0 211>; - interrupt-controller; - #interrupt-cells = <2>; - interrupts = ; - - gpio-wo-subnode-state { - pins = "gpio1"; - function = "gpio"; - }; - - uart-w-subnodes-state { - rx { - pins = "gpio26"; - function = "qup7"; - bias-pull-up; - }; - - tx { - pins = "gpio27"; - function = "qup7"; - bias-disable; - }; - }; + #include + pinctrl@f100000 { + compatible = "qcom,sm8450-tlmm"; + reg = <0x0f100000 0x300000>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&tlmm 0 0 211>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = ; + + gpio-wo-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-state { + rx-pins { + pins = "gpio26"; + function = "qup7"; + bias-pull-up; + }; + + tx-pins { + pins = "gpio27"; + function = "qup7"; + bias-disable; + }; }; + }; ... diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,rza1-ports.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,rza1-ports.yaml index 8ed4b98a1628970ba867e674e70fe067a5b6c27c..9083040c996ab06feccd6ecdbd8468dea5f81641 100644 --- a/Documentation/devicetree/bindings/pinctrl/renesas,rza1-ports.yaml +++ b/Documentation/devicetree/bindings/pinctrl/renesas,rza1-ports.yaml @@ -41,6 +41,7 @@ required: patternProperties: "^gpio-[0-9]*$": type: object + additionalProperties: false description: Each port of the r7s72100 pin controller hardware is itself a GPIO diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml index 997b746391120551f1e060b966aa27afc07ee59b..f081acb7ba049819f1eba3c4810183f62ebde3c0 100644 --- a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml @@ -23,7 +23,7 @@ properties: oneOf: - items: - enum: - - renesas,r9a07g043-pinctrl # RZ/G2UL{Type-1,Type-2} + - renesas,r9a07g043-pinctrl # RZ/G2UL{Type-1,Type-2} and RZ/Five - renesas,r9a07g044-pinctrl # RZ/G2{L,LC} - items: diff --git a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml index 677a285ca416985c539d7341b2750396434d926b..b486f41df65f1716b654662123770b6186f9fe11 100644 --- a/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/rockchip,pinctrl.yaml @@ -47,6 +47,7 @@ properties: - rockchip,rk3568-pinctrl - rockchip,rk3588-pinctrl - rockchip,rv1108-pinctrl + - rockchip,rv1126-pinctrl rockchip,grf: $ref: "/schemas/types.yaml#/definitions/phandle" diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-pins-cfg.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-pins-cfg.yaml index 9869d4dceddbbb71e572bdb65161efb0b2866209..f796f27bf0e648196a17013873893a7bae2deac6 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-pins-cfg.yaml +++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl-pins-cfg.yaml @@ -20,7 +20,6 @@ description: | The values used for config properties should be derived from the hardware manual and these values are programmed as-is into the pin pull up/down and driver strength register of the pin-controller. - See also include/dt-bindings/pinctrl/samsung.h with useful constants. See also Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml for additional information and example. diff --git a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml index 3a65c66ca71d226691befcb2a7b0f5fbf2e8726c..eb2b2692607d936ea71870dfaf224caa761f2075 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/samsung,pinctrl.yaml @@ -15,9 +15,6 @@ description: | This is a part of device tree bindings for Samsung S3C/S5P/Exynos SoC pin controller. - Pin group settings (like drive strength, pull up/down) are available as - macros in include/dt-bindings/pinctrl/samsung.h. - All the pin controller nodes should be represented in the aliases node using the following format 'pinctrl{n}' where n is a unique number for the alias. @@ -97,6 +94,9 @@ patternProperties: additionalProperties: false "^(initial|sleep)-state$": + type: object + additionalProperties: false + patternProperties: "^(pin-[a-z0-9-]+|[a-z0-9-]+-pin)$": $ref: samsung,pinctrl-pins-cfg.yaml @@ -138,8 +138,6 @@ additionalProperties: false examples: - | - #include - pinctrl@7f008000 { compatible = "samsung,s3c64xx-pinctrl"; reg = <0x7f008000 0x1000>; @@ -166,8 +164,8 @@ examples: uart0-data-pins { samsung,pins = "gpa-0", "gpa-1"; - samsung,pin-function = ; - samsung,pin-pud = ; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; }; // ... @@ -175,7 +173,6 @@ examples: - | #include - #include pinctrl@11400000 { compatible = "samsung,exynos4210-pinctrl"; @@ -197,9 +194,9 @@ examples: uart0-data-pins { samsung,pins = "gpa0-0", "gpa0-1"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; }; // ... @@ -207,14 +204,14 @@ examples: sleep0: sleep-state { gpa0-0-pin { samsung,pins = "gpa0-0"; - samsung,pin-con-pdn = ; - samsung,pin-pud-pdn = ; + samsung,pin-con-pdn = <2>; + samsung,pin-pud-pdn = <0>; }; gpa0-1-pin { samsung,pins = "gpa0-1"; - samsung,pin-con-pdn = ; - samsung,pin-pud-pdn = ; + samsung,pin-con-pdn = <0>; + samsung,pin-pud-pdn = <0>; }; // ... @@ -223,7 +220,6 @@ examples: - | #include - #include pinctrl@11000000 { compatible = "samsung,exynos4210-pinctrl"; @@ -272,26 +268,26 @@ examples: sd0-clk-pins { samsung,pins = "gpk0-0"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <3>; }; sd4-bus-width8-pins { part-1-pins { samsung,pins = "gpk0-3", "gpk0-4", "gpk0-5", "gpk0-6"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <3>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; }; part-2-pins { samsung,pins = "gpk1-3", "gpk1-4", "gpk1-5", "gpk1-6"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <4>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; }; }; @@ -299,16 +295,15 @@ examples: otg-gp-pins { samsung,pins = "gpx3-3"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <1>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; samsung,pin-val = <0>; }; }; - | #include - #include pinctrl@10580000 { compatible = "samsung,exynos5433-pinctrl"; @@ -352,9 +347,9 @@ examples: initial_alive: initial-state { gpa0-0-pin { samsung,pins = "gpa0-0"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <0>; + samsung,pin-pud = <1>; + samsung,pin-drv = <0>; }; // ... @@ -363,7 +358,6 @@ examples: - | #include - #include pinctrl@114b0000 { compatible = "samsung,exynos5433-pinctrl"; @@ -384,9 +378,9 @@ examples: i2s0-bus-pins { samsung,pins = "gpz0-0", "gpz0-1", "gpz0-2", "gpz0-3", "gpz0-4", "gpz0-5", "gpz0-6"; - samsung,pin-function = ; - samsung,pin-pud = ; - samsung,pin-drv = ; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; }; // ... diff --git a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.yaml index d35dcc4f024219ba869d72ce4506f7ab8553a502..9d59208d83c1852328bc396fe8a758742a3e7edd 100644 --- a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.yaml @@ -64,6 +64,9 @@ patternProperties: gpio-controller: true '#gpio-cells': const: 2 + interrupt-controller: true + '#interrupt-cells': + const: 2 reg: maxItems: 1 @@ -71,6 +74,7 @@ patternProperties: maxItems: 1 resets: maxItems: 1 + gpio-line-names: true gpio-ranges: minItems: 1 maxItems: 16 @@ -106,6 +110,12 @@ patternProperties: minimum: 0 maximum: 11 + patternProperties: + "^(.+-hog(-[0-9]+)?)$": + type: object + required: + - gpio-hog + required: - gpio-controller - '#gpio-cells' @@ -115,9 +125,12 @@ patternProperties: '-[0-9]*$': type: object + additionalProperties: false + patternProperties: '^pins': type: object + additionalProperties: false description: | A pinctrl node should contain at least one subnode representing the pinctrl group available on the machine. Each subnode will list the diff --git a/Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml index 92963604422f462128eac211ff58e353eab9ee18..69c0dd9998ea525ee591d2aa4a525c03ce35aa9d 100644 --- a/Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pinctrl/starfive,jh7100-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: StarFive JH7100 Pin Controller Device Tree Bindings +title: StarFive JH7100 Pin Controller description: | Bindings for the JH7100 RISC-V SoC from StarFive Ltd. @@ -165,7 +165,7 @@ examples: - | #include #include - #include + #include soc { #address-cells = <2>; diff --git a/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml index 15092fdd4b5bbbaa84e48388f8cebd6ac4c1ac30..347061eece9ea41a9868cc815f3cfd52fbff469d 100644 --- a/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/sunplus,sp7021-pinctrl.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/pinctrl/sunplus,sp7021-pinctrl.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Sunplus SP7021 Pin Controller Device Tree Bindings +title: Sunplus SP7021 Pin Controller maintainers: - Dvorkin Dmitry diff --git a/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml index 306524885a2b869aebf9da80e7c4f76f909a3838..98b4663f9766c696b4c222ffb9d3f66819163e8c 100644 --- a/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/toshiba,visconti-pinctrl.yaml @@ -36,6 +36,7 @@ patternProperties: pins it needs, and how they should be configured, with regard to muxer configuration, pullups, drive strength. $ref: "pinmux-node.yaml" + additionalProperties: false properties: function: diff --git a/Documentation/devicetree/bindings/power/amlogic,meson-ee-pwrc.yaml b/Documentation/devicetree/bindings/power/amlogic,meson-ee-pwrc.yaml index 5390e988a9347a739ff3ae88ae067d07d1528732..43a932237a92c5d8697689fa8a3b7606025713e6 100644 --- a/Documentation/devicetree/bindings/power/amlogic,meson-ee-pwrc.yaml +++ b/Documentation/devicetree/bindings/power/amlogic,meson-ee-pwrc.yaml @@ -8,7 +8,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Amlogic Meson Everything-Else Power Domains maintainers: - - Neil Armstrong + - Neil Armstrong description: |+ The Everything-Else Power Domains node should be the child of a syscon diff --git a/Documentation/devicetree/bindings/power/domain-idle-state.yaml b/Documentation/devicetree/bindings/power/domain-idle-state.yaml index 6a12efdf436af624ef1603bdf397502f8dd7d7cf..4ee920a1de69e8466935bfceece13fb6c772dc47 100644 --- a/Documentation/devicetree/bindings/power/domain-idle-state.yaml +++ b/Documentation/devicetree/bindings/power/domain-idle-state.yaml @@ -20,6 +20,7 @@ properties: patternProperties: "^(cpu|cluster|domain)-": type: object + additionalProperties: false description: Each state node represents a domain idle state description. @@ -44,6 +45,15 @@ patternProperties: state will yield power benefits, after overcoming the overhead while entering the idle state. + arm,psci-suspend-param: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + power_state parameter to pass to the ARM PSCI suspend call. + + Device tree nodes that require usage of PSCI CPU_SUSPEND function + (i.e. idle states node with entry-method property is set to "psci") + must specify this property. + required: - compatible - entry-latency-us diff --git a/Documentation/devicetree/bindings/power/fsl,imx-gpc.yaml b/Documentation/devicetree/bindings/power/fsl,imx-gpc.yaml index a055b3e819d81128905c20cd1581f2f5aa441b7c..777e1d852dddb525b9fce3415349a5f8d88a8f94 100644 --- a/Documentation/devicetree/bindings/power/fsl,imx-gpc.yaml +++ b/Documentation/devicetree/bindings/power/fsl,imx-gpc.yaml @@ -43,11 +43,21 @@ properties: pgc: type: object + additionalProperties: false description: list of power domains provided by this controller. + properties: + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + patternProperties: "power-domain@[0-9]$": type: object + additionalProperties: false + properties: '#power-domain-cells': @@ -78,6 +88,10 @@ properties: - '#power-domain-cells' - reg + required: + - '#address-cells' + - '#size-cells' + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml index 747622bdc57be95297e57b1a2d5d7c62402cdb47..58022ae7d5ddcc0dee8d08b9cedd49516d7930dd 100644 --- a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml +++ b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml @@ -42,11 +42,21 @@ properties: pgc: type: object + additionalProperties: false description: list of power domains provided by this controller. + properties: + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + patternProperties: - "power-domain@[0-9]$": + "power-domain@[0-9a-f]+$": type: object + additionalProperties: false + properties: '#power-domain-cells': @@ -85,6 +95,10 @@ properties: - '#power-domain-cells' - reg + required: + - '#address-cells' + - '#size-cells' + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml index b448101fac43e28ac50811dc8975525c0f1caf44..605ec7ab5f6331847f53c0a91da533fcfd376d01 100644 --- a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml +++ b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Mediatek Power Domains Controller maintainers: - - Weiyi Lu + - MandyJH Liu - Matthias Brugger description: | @@ -19,7 +19,7 @@ description: | properties: $nodename: - const: power-controller + pattern: '^power-controller(@[0-9a-f]+)?$' compatible: enum: @@ -42,6 +42,23 @@ properties: patternProperties: "^power-domain@[0-9a-f]+$": + $ref: "#/$defs/power-domain-node" + patternProperties: + "^power-domain@[0-9a-f]+$": + $ref: "#/$defs/power-domain-node" + patternProperties: + "^power-domain@[0-9a-f]+$": + $ref: "#/$defs/power-domain-node" + patternProperties: + "^power-domain@[0-9a-f]+$": + $ref: "#/$defs/power-domain-node" + unevaluatedProperties: false + unevaluatedProperties: false + unevaluatedProperties: false + unevaluatedProperties: false + +$defs: + power-domain-node: type: object description: | Represents the power domains within the power controller node as documented @@ -100,123 +117,9 @@ patternProperties: $ref: /schemas/types.yaml#/definitions/phandle description: phandle to the device containing the SMI register range. - patternProperties: - "^power-domain@[0-9a-f]+$": - type: object - description: | - Represents a power domain child within a power domain parent node. - - properties: - - '#power-domain-cells': - description: - Must be 0 for nodes representing a single PM domain and 1 for nodes - providing multiple PM domains. - - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - - reg: - maxItems: 1 - - clocks: - description: | - A number of phandles to clocks that need to be enabled during domain - power-up sequencing. - - clock-names: - description: | - List of names of clocks, in order to match the power-up sequencing - for each power domain we need to group the clocks by name. BASIC - clocks need to be enabled before enabling the corresponding power - domain, and should not have a '-' in their name (i.e mm, mfg, venc). - SUSBYS clocks need to be enabled before releasing the bus protection, - and should contain a '-' in their name (i.e mm-0, isp-0, cam-0). - - In order to follow properly the power-up sequencing, the clocks must - be specified by order, adding first the BASIC clocks followed by the - SUSBSYS clocks. - - domain-supply: - description: domain regulator supply. - - mediatek,infracfg: - $ref: /schemas/types.yaml#/definitions/phandle - description: phandle to the device containing the INFRACFG register range. - - mediatek,smi: - $ref: /schemas/types.yaml#/definitions/phandle - description: phandle to the device containing the SMI register range. - - patternProperties: - "^power-domain@[0-9a-f]+$": - type: object - description: | - Represents a power domain child within a power domain parent node. - - properties: - - '#power-domain-cells': - description: - Must be 0 for nodes representing a single PM domain and 1 for nodes - providing multiple PM domains. - - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - - reg: - maxItems: 1 - - clocks: - description: | - A number of phandles to clocks that need to be enabled during domain - power-up sequencing. - - clock-names: - description: | - List of names of clocks, in order to match the power-up sequencing - for each power domain we need to group the clocks by name. BASIC - clocks need to be enabled before enabling the corresponding power - domain, and should not have a '-' in their name (i.e mm, mfg, venc). - SUSBYS clocks need to be enabled before releasing the bus protection, - and should contain a '-' in their name (i.e mm-0, isp-0, cam-0). - - In order to follow properly the power-up sequencing, the clocks must - be specified by order, adding first the BASIC clocks followed by the - SUSBSYS clocks. - - domain-supply: - description: domain regulator supply. - - mediatek,infracfg: - $ref: /schemas/types.yaml#/definitions/phandle - description: phandle to the device containing the INFRACFG register range. - - mediatek,smi: - $ref: /schemas/types.yaml#/definitions/phandle - description: phandle to the device containing the SMI register range. - - required: - - reg - - additionalProperties: false - - required: - - reg - - additionalProperties: false - required: - reg - additionalProperties: false - required: - compatible @@ -232,7 +135,7 @@ examples: #size-cells = <2>; scpsys: syscon@10006000 { - compatible = "syscon", "simple-mfd"; + compatible = "mediatek,mt8173-scpsys", "syscon", "simple-mfd"; reg = <0 0x10006000 0 0x1000>; spm: power-controller { diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml index 0ccca493251acf1bacc1bebe63b0688580fec389..5b4eda919911a0f461b0088ef1d437f50c556372 100644 --- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml +++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Qualcomm RPM/RPMh Power domains maintainers: - - Rajendra Nayak + - Bjorn Andersson description: For RPM/RPMh Power domains, we communicate a performance state to RPM/RPMh @@ -40,6 +40,7 @@ properties: - qcom,sm6115-rpmpd - qcom,sm6125-rpmpd - qcom,sm6350-rpmhpd + - qcom,sm6375-rpmpd - qcom,sm8150-rpmhpd - qcom,sm8250-rpmhpd - qcom,sm8350-rpmhpd diff --git a/Documentation/devicetree/bindings/power/reset/qcom,pon.yaml b/Documentation/devicetree/bindings/power/reset/qcom,pon.yaml index e7b436d2e757f1fe0349dcefd8c41ea0ec6ae17a..d96170eecbd228bd95e6957bb8187af2a4b2e57a 100644 --- a/Documentation/devicetree/bindings/power/reset/qcom,pon.yaml +++ b/Documentation/devicetree/bindings/power/reset/qcom,pon.yaml @@ -15,18 +15,27 @@ description: | This DT node has pwrkey and resin as sub nodes. -allOf: - - $ref: reboot-mode.yaml# - properties: compatible: enum: - qcom,pm8916-pon - qcom,pms405-pon - qcom,pm8998-pon + - qcom,pmk8350-pon reg: - maxItems: 1 + description: | + Specifies the SPMI base address for the PON (power-on) peripheral. For + PMICs that have the PON peripheral (GEN3) split into PON_HLOS and PON_PBS + (e.g. PMK8350), this can hold addresses of both PON_HLOS and PON_PBS + peripherals. In that case, the PON_PBS address needs to be specified to + facilitate software debouncing on some PMIC. + minItems: 1 + maxItems: 2 + + reg-names: + minItems: 1 + maxItems: 2 pwrkey: type: object @@ -46,6 +55,39 @@ required: unevaluatedProperties: false +allOf: + - $ref: reboot-mode.yaml# + - if: + properties: + compatible: + contains: + enum: + - qcom,pm8916-pon + - qcom,pms405-pon + - qcom,pm8998-pon + then: + properties: + reg: + maxItems: 1 + reg-names: + items: + - const: pon + - if: + properties: + compatible: + contains: + const: qcom,pmk8350-pon + then: + properties: + reg: + minItems: 1 + maxItems: 2 + reg-names: + minItems: 1 + items: + - const: hlos + - const: pbs + examples: - | #include diff --git a/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml b/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml index 3deb0fc8dfd35d304116396db6c9d96db421659e..0d5e999a58f1b0c58d01f191d310b219e5441cb5 100644 --- a/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml +++ b/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml @@ -41,6 +41,8 @@ properties: - rockchip,rk3368-power-controller - rockchip,rk3399-power-controller - rockchip,rk3568-power-controller + - rockchip,rk3588-power-controller + - rockchip,rv1126-power-controller "#power-domain-cells": const: 1 @@ -119,6 +121,8 @@ $defs: "include/dt-bindings/power/rk3368-power.h" "include/dt-bindings/power/rk3399-power.h" "include/dt-bindings/power/rk3568-power.h" + "include/dt-bindings/power/rk3588-power.h" + "include/dt-bindings/power/rockchip,rv1126-power.h" clocks: minItems: 1 diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml b/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml index 1727bf10897932ea0613d69307b06fd83e119fbb..d71fc72d446444947f06c3aea228e7e329b6b7f2 100644 --- a/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml +++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml @@ -58,6 +58,7 @@ properties: - rockchip,rk3568-pmu-io-voltage-domain - rockchip,rv1108-io-voltage-domain - rockchip,rv1108-pmu-io-voltage-domain + - rockchip,rv1126-pmu-io-voltage-domain required: - compatible @@ -78,6 +79,7 @@ allOf: - $ref: "#/$defs/rk3568-pmu" - $ref: "#/$defs/rv1108" - $ref: "#/$defs/rv1108-pmu" + - $ref: "#/$defs/rv1126-pmu" $defs: px30: @@ -344,6 +346,34 @@ $defs: pmu-supply: description: The supply connected to PMUIO_VDD. + rv1126-pmu: + if: + properties: + compatible: + contains: + const: rockchip,rv1126-pmu-io-voltage-domain + + then: + properties: + vccio1-supply: + description: The supply connected to VCCIO1. + vccio2-supply: + description: The supply connected to VCCIO2. + vccio3-supply: + description: The supply connected to VCCIO3. + vccio4-supply: + description: The supply connected to VCCIO4. + vccio5-supply: + description: The supply connected to VCCIO5. + vccio6-supply: + description: The supply connected to VCCIO6. + vccio7-supply: + description: The supply connected to VCCIO7. + pmuio0-supply: + description: The supply connected to PMUIO0. + pmuio1-supply: + description: The supply connected to PMUIO1. + examples: - | io-domains { diff --git a/Documentation/devicetree/bindings/power/supply/mediatek,mt6370-charger.yaml b/Documentation/devicetree/bindings/power/supply/mediatek,mt6370-charger.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd491c598a0050334ab403f5b73c32cd3af0d7ba --- /dev/null +++ b/Documentation/devicetree/bindings/power/supply/mediatek,mt6370-charger.yaml @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/supply/mediatek,mt6370-charger.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MT6370 Battery Charger + +maintainers: + - ChiaEn Wu + +description: | + This module is part of the MT6370 MFD device. + Provides Battery Charger, Boost for OTG devices and BC1.2 detection. + +properties: + compatible: + const: mediatek,mt6370-charger + + interrupts: + description: | + Specify what irqs are needed to be handled by MT6370 Charger driver. + We need to use the IRQ "MT6370_IRQ_OVPCTRL_UVP_D" to know when USB + is plugged in, and then the driver will enable BC1.2 detection. + After the hardware of MT6370 completes the BC1.2 detection, + IRQ "MT6370_IRQ_ATTACH" will be triggered, and the driver will know + the result of BC1.2 detection. + When the IRQ "MT6370_IRQ_CHG_MIVR" is triggered, it means that the + hardware enters the "Minimum Input Voltage Regulation loop" and + a workaround needs to be applied at this time. + In summary, "MT6370_IRQ_OVPCTRL_UVP_D", "MT6370_IRQ_ATTACH" and + "MT6370_IRQ_CHG_MIVR" are required in this charger driver. + items: + - description: irq of "USB is plugged in" + - description: irq of "BC1.2 is done" + - description: irq of "Minimum Input Voltage Regulation loop is active" + + interrupt-names: + items: + - const: uvp_d_evt + - const: attach_i + - const: mivr + + io-channels: + description: | + Use ADC channel to read VBUS, IBUS, IBAT, etc., info. + minItems: 1 + items: + - description: | + VBUS voltage with lower accuracy (+-75mV) but higher measure + range (1~22V) + - description: | + VBUS voltage with higher accuracy (+-30mV) but lower measure + range (1~9.76V) + - description: the main system input voltage + - description: battery voltage + - description: battery temperature-sense input voltage + - description: IBUS current (required) + - description: battery current + - description: | + regulated output voltage to supply for the PWM low-side gate driver + and the bootstrap capacitor + - description: IC junction temperature + + io-channel-names: + minItems: 1 + items: + - const: vbusdiv5 + - const: vbusdiv2 + - const: vsys + - const: vbat + - const: ts_bat + - const: ibus + - const: ibat + - const: chg_vddp + - const: temp_jc + + usb-otg-vbus-regulator: + type: object + description: OTG boost regulator. + unevaluatedProperties: false + $ref: /schemas/regulator/regulator.yaml# + + properties: + enable-gpios: + maxItems: 1 + +required: + - compatible + - interrupts + - interrupt-names + - io-channels + +additionalProperties: false + +... diff --git a/Documentation/devicetree/bindings/ptp/ptp-idt82p33.yaml b/Documentation/devicetree/bindings/ptp/ptp-idt82p33.yaml index 9bc664f414a1681a361d310266a14e95cf4d0792..51381e4cbb1f16b5b134b63423f880b30b0d0f09 100644 --- a/Documentation/devicetree/bindings/ptp/ptp-idt82p33.yaml +++ b/Documentation/devicetree/bindings/ptp/ptp-idt82p33.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/ptp/ptp-idt82p33.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: IDT 82P33 PTP Clock Device Tree Bindings +title: IDT 82P33 PTP Clock description: | IDT 82P33XXX Synchronization Management Unit (SMU) based PTP clock diff --git a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml index 658cec67743e1220888db2e6b3ae2e63651145d6..7cf32663c806949938c9dfe01edd73fa11bc087f 100644 --- a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml +++ b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/ptp/ptp-idtcm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: IDT ClockMatrix (TM) PTP Clock Device Tree Bindings +title: IDT ClockMatrix (TM) PTP Clock maintainers: - Vincent Cheng diff --git a/Documentation/devicetree/bindings/pwm/allwinner,sun4i-a10-pwm.yaml b/Documentation/devicetree/bindings/pwm/allwinner,sun4i-a10-pwm.yaml index e93e935564fba6815518863e9ff8ef1a8c382d2b..4cc3cc7c50bec97d646490050f809e2f2ba87d89 100644 --- a/Documentation/devicetree/bindings/pwm/allwinner,sun4i-a10-pwm.yaml +++ b/Documentation/devicetree/bindings/pwm/allwinner,sun4i-a10-pwm.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pwm/allwinner,sun4i-a10-pwm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 PWM Device Tree Bindings +title: Allwinner A10 PWM maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/pwm/intel,keembay-pwm.yaml b/Documentation/devicetree/bindings/pwm/intel,keembay-pwm.yaml index ec9f6bab798ce5db90abc9c4730b6a014ee3837a..5b18a5913ed119e185a838f6bf16401b0086e361 100644 --- a/Documentation/devicetree/bindings/pwm/intel,keembay-pwm.yaml +++ b/Documentation/devicetree/bindings/pwm/intel,keembay-pwm.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/pwm/intel,keembay-pwm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel Keem Bay PWM Device Tree Bindings +title: Intel Keem Bay PWM maintainers: - Vijayakannan Ayyathurai diff --git a/Documentation/devicetree/bindings/pwm/mediatek,pwm-disp.yaml b/Documentation/devicetree/bindings/pwm/mediatek,pwm-disp.yaml index e4fe2d1bfef58bc44628db39aa5a9372bc3c4df9..0088bc8e7c541a3c6ba7001c06e93bb88a6586ac 100644 --- a/Documentation/devicetree/bindings/pwm/mediatek,pwm-disp.yaml +++ b/Documentation/devicetree/bindings/pwm/mediatek,pwm-disp.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/pwm/mediatek,pwm-disp.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek DISP_PWM Controller Device Tree Bindings +title: MediaTek DISP_PWM Controller maintainers: - Jitao Shi @@ -27,6 +27,7 @@ properties: - items: - enum: - mediatek,mt8186-disp-pwm + - mediatek,mt8188-disp-pwm - mediatek,mt8192-disp-pwm - mediatek,mt8195-disp-pwm - const: mediatek,mt8183-disp-pwm diff --git a/Documentation/devicetree/bindings/pwm/pwm-rockchip.yaml b/Documentation/devicetree/bindings/pwm/pwm-rockchip.yaml index a336ff9364a93cd05ac787b9e4f27f052afe1f12..f2d1dc7e7b3fb5cf1b84b352e150137be5f3fb82 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-rockchip.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-rockchip.yaml @@ -21,6 +21,7 @@ properties: - const: rockchip,rk2928-pwm - items: - enum: + - rockchip,rk3128-pwm - rockchip,rk3368-pwm - rockchip,rk3399-pwm - rockchip,rv1108-pwm @@ -30,6 +31,7 @@ properties: - rockchip,px30-pwm - rockchip,rk3308-pwm - rockchip,rk3568-pwm + - rockchip,rk3588-pwm - const: rockchip,rk3328-pwm reg: diff --git a/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml b/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml index 24ace6e1e5ec6f0d7bff605a8d828e72707218b2..63e1161a87dee09c9c08fb1d8d9f24842f24fd1e 100644 --- a/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml +++ b/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml @@ -83,6 +83,7 @@ properties: regulators: type: object + additionalProperties: false description: | List of regulators provided by the device diff --git a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml index f7e3d8fd3bf35a6df4298a8fd104f420c0a15e00..6c3371d706bbe5be2e27bc6a5eb5dfd7334edb72 100644 --- a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml @@ -85,6 +85,9 @@ properties: - current default: voltage + vin-supply: + description: Input supply phandle. + required: - compatible - regulator-name diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml index 236348c4710c9d9d3122ee843d3726ae1fef96e6..71138c611b6cf26958b1f8c3ac586e706a79772c 100644 --- a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml +++ b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml @@ -79,6 +79,7 @@ patternProperties: patternProperties: regulator-state-(standby|mem|disk): type: object + additionalProperties: true properties: regulator-mode: false diff --git a/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml b/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml index 4321f061a7f6249c407db73c1091d1a7b13cffc0..2b266ea43716471e7761689c52799817778ed554 100644 --- a/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml +++ b/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml @@ -111,6 +111,7 @@ properties: regulators: type: object + additionalProperties: false description: List of child nodes that specify the regulators. diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..771cc134393cedd5f935e5a31f0da53f16ef7647 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml @@ -0,0 +1,273 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/mediatek,mt6331-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MT6331 Regulator from MediaTek Integrated + +maintainers: + - AngeloGioacchino Del Regno + +description: | + The MT6331 PMIC provides 6 BUCK and 21 LDO (Low Dropout) regulators + and nodes are named according to the regulator type: + buck- and ldo-. + MT6331 regulators node should be sub node of the MT6397 MFD node. + +patternProperties: + "^buck-v(core2|io18|dvfs11|dvfs12|dvfs13|dvfs14)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^v(core2|io18|dvfs11|dvfs12|dvfs13|dvfs14)$" + + unevaluatedProperties: false + + "^ldo-v(avdd32aud|auxa32)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^v(avdd32aud|auxa32)$" + + unevaluatedProperties: false + + "^ldo-v(dig18|emc33|ibr|mc|mch|mipi|rtc|sram|usb10)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^v(dig18|emc33|ibr|mc|mch|mipi|rtc|sram|usb10)$" + + unevaluatedProperties: false + + "^ldo-vcam(a|af|d|io)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^vcam(a|af|d|io)$" + + unevaluatedProperties: false + + "^ldo-vtcxo[12]$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^vtcxo[12]$" + + required: + - regulator-name + + unevaluatedProperties: false + + "^ldo-vgp[1234]$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^vgp[12]$" + + required: + - regulator-name + + unevaluatedProperties: false + +additionalProperties: false + +examples: + - | + pmic { + regulators { + mt6331_vdvfs11_reg: buck-vdvfs11 { + regulator-name = "vdvfs11"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vdvfs12_reg: buck-vdvfs12 { + regulator-name = "vdvfs12"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vdvfs13_reg: buck-vdvfs13 { + regulator-name = "vdvfs13"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vdvfs14_reg: buck-vdvfs14 { + regulator-name = "vdvfs14"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vcore2_reg: buck-vcore2 { + regulator-name = "vcore2"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vio18_reg: buck-vio18 { + regulator-name = "vio18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <0>; + regulator-allowed-modes = <0 1>; + }; + mt6331_vtcxo1_reg: ldo-vtcxo1 { + regulator-name = "vtcxo1"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + regulator-boot-on; + }; + mt6331_vtcxo2_reg: ldo-vtcxo2 { + regulator-name = "vtcxo2"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + regulator-boot-on; + }; + mt6331_avdd32_aud_reg: ldo-avdd32aud { + regulator-name = "avdd32_aud"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3200000>; + }; + mt6331_vauxa32_reg: ldo-vauxa32 { + regulator-name = "vauxa32"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3200000>; + }; + mt6331_vcama_reg: ldo-vcama { + regulator-name = "vcama"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + }; + mt6331_vio28_reg: ldo-vio28 { + regulator-name = "vio28"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + regulator-boot-on; + }; + mt6331_vcamaf_reg: ldo-vcamaf { + regulator-name = "vcam_af"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vmc_reg: ldo-vmc { + regulator-name = "vmc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vmch_reg: ldo-vmch { + regulator-name = "vmch"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vemc33_reg: ldo-vemc33 { + regulator-name = "vemc33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vgp1_reg: ldo-vgp1 { + regulator-name = "vgp1"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vsim1_reg: ldo-vsim1 { + regulator-name = "vsim1"; + regulator-min-microvolt = <1700000>; + regulator-max-microvolt = <3100000>; + }; + mt6331_vsim2_reg: ldo-vsim2 { + regulator-name = "vsim2"; + regulator-min-microvolt = <1700000>; + regulator-max-microvolt = <3100000>; + }; + mt6331_vmipi_reg: ldo-vmipi { + regulator-name = "vmipi"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vibr_reg: ldo-vibr { + regulator-name = "vibr"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + }; + mt6331_vgp4_reg: ldo-vgp4 { + regulator-name = "vgp4"; + regulator-min-microvolt = <1600000>; + regulator-max-microvolt = <2200000>; + }; + mt6331_vcamd_reg: ldo-vcamd { + regulator-name = "vcamd"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1500000>; + }; + mt6331_vusb10_reg: ldo-vusb10 { + regulator-name = "vusb"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + }; + mt6331_vcamio_reg: ldo-vcamio { + regulator-name = "vcam_io"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + }; + mt6331_vsram_reg: ldo-vsram { + regulator-name = "vsram"; + regulator-min-microvolt = <1012500>; + regulator-max-microvolt = <1012500>; + regulator-always-on; + regulator-boot-on; + }; + mt6331_vgp2_reg: ldo-vgp2 { + regulator-name = "vgp2"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1500000>; + regulator-boot-on; + }; + mt6331_vgp3_reg: ldo-vgp3 { + regulator-name = "vgp3"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + }; + mt6331_vrtc_reg: ldo-vrtc { + regulator-name = "vrtc"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + }; + mt6331_vdig18_reg: ldo-vdig18 { + regulator-name = "dvdd18_dig"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3218f43e6957b8dc150e059f30b8539fd9f56dc6 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/mediatek,mt6332-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MT6332 Regulator from MediaTek Integrated + +maintainers: + - AngeloGioacchino Del Regno + +description: | + The MT6332 Companion PMIC provides 6 BUCK and 4 LDO (Low Dropout) + regulators and nodes are named according to the regulator type: + buck- and ldo-. + MT6332 regulators node should be sub node of the MT6397 MFD node. + +patternProperties: + "^buck-v(dram|dvfs2|pa|rf18a|rf18b|sbst)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^v(dram|dvfs2|pa|rf18a|rf18b|sbst)$" + + unevaluatedProperties: false + + "^ldo-v(bif28|dig18|sram|usb33)$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^v(bif28|dig18|sram|usb33)$" + + unevaluatedProperties: false + +additionalProperties: false + +examples: + - | + pmic { + regulators { + mt6332_vdram_reg: buck-vdram { + regulator-name = "vdram"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-ramp-delay = <12500>; + regulator-allowed-modes = <0 1>; + regulator-always-on; + }; + mt6332_vdvfs2_reg: buck-vdvfs2 { + regulator-name = "vdvfs2"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1312500>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <1>; + regulator-allowed-modes = <0 1>; + }; + mt6332_vpa_reg: buck-vpa { + regulator-name = "vpa"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <3400000>; + }; + mt6332_vrf18a_reg: buck-vrf18a { + regulator-name = "vrf18a"; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <2240625>; + regulator-allowed-modes = <0 1>; + }; + mt6332_vrf18b_reg: buck-vrf18b { + regulator-name = "vrf18b"; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <2240625>; + regulator-allowed-modes = <0 1>; + }; + mt6332_vsbst_reg: buck-vsbst { + regulator-name = "vsbst"; + regulator-min-microvolt = <3500000>; + regulator-max-microvolt = <7468750>; + }; + mt6332_vauxb32_reg: ldo-vauxb32 { + regulator-name = "vauxb32"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3200000>; + }; + mt6332_vbif28_reg: ldo-vbif28 { + regulator-name = "vbif28"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + }; + mt6332_vdig18_reg: ldo-vdig18 { + regulator-name = "vdig18"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + mt6332_vsram_reg: ldo-vsram { + regulator-name = "vauxa32"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1493750>; + regulator-always-on; + }; + mt6332_vusb33_reg: ldo-vusb33 { + regulator-name = "vusb33"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml index 37402c370fbbc87963126f8b67aad4f26164c360..364b58730be2ba39523f8702abd5da3e026d6f8a 100644 --- a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml @@ -29,6 +29,7 @@ properties: "^vbuck[1-4]$": type: object $ref: "regulator.yaml#" + unevaluatedProperties: false properties: regulator-compatible: diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml index 9a36bee750af080eff6e59af62572b67f5fda49e..90c3bda31c233826136764b7da71d821c3929f81 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml @@ -37,6 +37,8 @@ description: | For PM6150, smps1 - smps5, ldo1 - ldo19 For PM6150L, smps1 - smps8, ldo1 - ldo11, bob For PM6350, smps1 - smps5, ldo1 - ldo22 + For PM660, smps1 - smps6, ldo1 - ldo3, ldo5 - ldo19 + For PM660L, smps1 - smps3, smps5, ldo1 - ldo8, bob For PM7325, smps1 - smps8, ldo1 - ldo19 For PM8005, smps1 - smps4 For PM8009, smps1 - smps2, ldo1 - ldo7 @@ -57,6 +59,8 @@ properties: - qcom,pm6150-rpmh-regulators - qcom,pm6150l-rpmh-regulators - qcom,pm6350-rpmh-regulators + - qcom,pm660-rpmh-regulators + - qcom,pm660l-rpmh-regulators - qcom,pm7325-rpmh-regulators - qcom,pm8005-rpmh-regulators - qcom,pm8009-rpmh-regulators @@ -99,12 +103,16 @@ properties: type: object $ref: "regulator.yaml#" description: BOB regulator node. + dependencies: + regulator-allow-set-load: ["regulator-allowed-modes"] patternProperties: "^(smps|ldo|lvs)[0-9]+$": type: object $ref: "regulator.yaml#" description: smps/ldo regulator nodes(s). + dependencies: + regulator-allow-set-load: ["regulator-allowed-modes"] required: - compatible @@ -144,6 +152,38 @@ allOf: patternProperties: "^vdd-s[1-8]-supply$": true + - if: + properties: + compatible: + enum: + - qcom,pm660-rpmh-regulators + then: + properties: + vdd-l1-l6-l7-supply: true + vdd-l2-l3-supply: true + vdd-l5-supply: true + vdd-l8-l9-l10-l11-l12-l13-l14-supply: true + vdd-l15-l16-l17-l18-l19-supply: true + patternProperties: + "^vdd-s[1-6]-supply$": true + + - if: + properties: + compatible: + enum: + - qcom,pm660l-rpmh-regulators + then: + properties: + vdd-bob-supply: + description: BOB regulator parent supply phandle. + vdd-l1-l9-l10-supply: true + vdd-l2-supply: true + vdd-l3-l5-l7-l8-supply: true + vdd-l4-l6-supply: true + vdd-s3-s4-supply: true + patternProperties: + "^vdd-s[125]-supply$": true + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml index c233461cc980d37e65ae9943172d6693b9eeb8d3..961eed51912ce2cbee555e0f4c0d742af6aee699 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml @@ -24,6 +24,17 @@ description: For mp5496, s2 + For pm2250, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, + l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22 + + For pm6125 s1, s2, s3, s4, s5, s6, s7, s8, l1, l2, l3, l5, l6, l7, l8, l9, + l10, l22, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24 + + For pm660, s1, s2, s3, s4, s5, s6, l1, l2, l3, l5, l6, l7, l8, l9, l10, l22, + l12, l13, l14, l15, l16, l17, l18, l19 + + For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob + For pm8226, s1, s2, s3, s4, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, l25, l26, l27, l28, lvs1 @@ -52,11 +63,6 @@ description: l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, l25, l26, l27, l28, lvs1, lvs2 - For pm660, s1, s2, s3, s4, s5, s6, l1, l2, l3, l5, l6, l7, l8, l9, l10, l22, - l12, l13, l14, l15, l16, l17, l18, l19 - - For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob - For pma8084, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, l25, l26, l27, lvs1, lvs2, lvs3, lvs4, 5vs1 @@ -68,9 +74,6 @@ description: For pms405, s1, s2, s3, s4, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13 - For pm2250, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, - l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22 - maintainers: - Andy Gross - Bjorn Andersson @@ -79,6 +82,10 @@ properties: compatible: enum: - qcom,rpm-mp5496-regulators + - qcom,rpm-pm2250-regulators + - qcom,rpm-pm6125-regulators + - qcom,rpm-pm660-regulators + - qcom,rpm-pm660l-regulators - qcom,rpm-pm8226-regulators - qcom,rpm-pm8841-regulators - qcom,rpm-pm8909-regulators @@ -88,13 +95,10 @@ properties: - qcom,rpm-pm8953-regulators - qcom,rpm-pm8994-regulators - qcom,rpm-pm8998-regulators - - qcom,rpm-pm660-regulators - - qcom,rpm-pm660l-regulators - qcom,rpm-pma8084-regulators - qcom,rpm-pmi8994-regulators - qcom,rpm-pmi8998-regulators - qcom,rpm-pms405-regulators - - qcom,rpm-pm2250-regulators patternProperties: ".*-supply$": diff --git a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml index 8b7c4af4b5517e06a95b87765a6a5ed04638a31f..bdf34c2de96b67e1c505f40320a3c42f213555d8 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml @@ -12,6 +12,7 @@ maintainers: properties: compatible: enum: + - qcom,pm6125-regulators - qcom,pm660-regulators - qcom,pm660l-regulators - qcom,pm8004-regulators @@ -35,6 +36,7 @@ patternProperties: description: List of regulators and its properties type: object $ref: regulator.yaml# + unevaluatedProperties: false properties: qcom,ocp-max-retries: @@ -100,12 +102,29 @@ patternProperties: SAW controlled gang leader. Will be configured as SAW regulator. type: boolean - unevaluatedProperties: false - required: - compatible allOf: + - if: + properties: + compatible: + contains: + enum: + - qcom,pm6125-regulators + then: + properties: + vdd_l1_l7_l17_l18-supply: true + vdd_l2_l3_l4-supply: true + vdd_l5_l15_l19_l20_l21_l22-supply: true + vdd_l6_l8-supply: true + vdd_l9_l11-supply: true + vdd_l10_l13_l14-supply: true + vdd_l12_l16-supply: true + vdd_l23_l24-supply: true + patternProperties: + "^vdd_s[1-8]-supply$": true + - if: properties: compatible: @@ -121,6 +140,7 @@ allOf: vdd_l8_l9_l10_l11_l12_l13_l14-supply: true patternProperties: "^vdd_s[1-6]-supply$": true + - if: properties: compatible: @@ -135,6 +155,7 @@ allOf: vdd_l4_l6-supply: true patternProperties: "^vdd_s[1-5]-supply$": true + - if: properties: compatible: @@ -144,6 +165,7 @@ allOf: then: patternProperties: "^vdd_s[25]-supply$": true + - if: properties: compatible: @@ -153,6 +175,7 @@ allOf: then: patternProperties: "^vdd_s[1-4]-supply$": true + - if: properties: compatible: @@ -172,6 +195,7 @@ allOf: vdd_lvs1-supply: true patternProperties: "^vdd_s[1-5]-supply$": true + - if: properties: compatible: @@ -181,6 +205,7 @@ allOf: then: patternProperties: "^vdd_s[1-8]-supply$": true + - if: properties: compatible: @@ -196,6 +221,7 @@ allOf: patternProperties: "^vdd_l[27]-supply$": true "^vdd_s[1-4]-supply$": true + - if: properties: compatible: @@ -224,6 +250,7 @@ allOf: vin_5vs-supply: true patternProperties: "^vdd_s[1-3]-supply$": true + - if: properties: compatible: @@ -242,6 +269,7 @@ allOf: vdd_l9_l10_l13_l14_l15_l18-supply: true patternProperties: "^vdd_s[1-6]-supply$": true + - if: properties: compatible: @@ -266,6 +294,7 @@ allOf: vdd_lvs_1_2-supply: true patternProperties: "^vdd_s[1-9][0-2]?-supply$": true + - if: properties: compatible: @@ -277,6 +306,7 @@ allOf: vdd_l1-supply: true patternProperties: "^vdd_s[1-3]-supply$": true + - if: properties: compatible: @@ -292,6 +322,7 @@ allOf: patternProperties: "^vdd_l[479]-supply$": true "^vdd_s[1-5]-supply$": true + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml index 1ddc1efd19e29ea17cfd0ee5985b313bca70830c..f97b8083678f22a44a1758810a470cba8c180ffc 100644 --- a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml @@ -20,6 +20,7 @@ properties: lab: type: object + additionalProperties: false properties: qcom,soft-start-us: @@ -33,11 +34,19 @@ properties: description: Short-circuit and over-current interrupts for lab. + interrupt-names: + minItems: 1 + items: + - const: sc-err + - const: ocp + required: - interrupts + - interrupt-names ibb: type: object + additionalProperties: false properties: qcom,discharge-resistor-kohms: @@ -52,8 +61,15 @@ properties: description: Short-circuit and over-current interrupts for ibb. + interrupt-names: + minItems: 1 + items: + - const: sc-err + - const: ocp + required: - interrupts + - interrupt-names required: - compatible diff --git a/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml b/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml index 091150c4e5795b401ddc02d70779277bb2e25f22..4a8a221bc9027af18220b3d0124dd818e3a1d304 100644 --- a/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml @@ -42,6 +42,7 @@ patternProperties: "^DSV(P|N)$": type: object $ref: regulator.yaml# + unevaluatedProperties: false description: Properties for single display bias regulator. diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml index 7d0adb74a396a39493e25bb0d940eff5a22cd8f8..d61e8675f0672af651835b44ef378d5c93818a60 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml @@ -27,6 +27,7 @@ properties: description: properties for wled regulator $ref: regulator.yaml# + unevaluatedProperties: false properties: regulator-name: diff --git a/Documentation/devicetree/bindings/regulator/silergy,sy8106a.yaml b/Documentation/devicetree/bindings/regulator/silergy,sy8106a.yaml index a52a67c869b5ccd2b11e110db8feb975dd518ad3..c3a902e11b5d3764ab0ae300d0e6ae4da449ede0 100644 --- a/Documentation/devicetree/bindings/regulator/silergy,sy8106a.yaml +++ b/Documentation/devicetree/bindings/regulator/silergy,sy8106a.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/regulator/silergy,sy8106a.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Silergy SY8106A Voltage Regulator Device Tree Bindings +title: Silergy SY8106A Voltage Regulator maintainers: - Ondrej Jirman diff --git a/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml b/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78be79930fda4fad960e2bb76c7089a14c250bd3 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/ti,tps65219.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI tps65219 Power Management Integrated Circuit regulators + +maintainers: + - Jerome Neanne + +description: | + Regulator nodes should be named to buck and ldo. + +properties: + compatible: + enum: + - ti,tps65219 + + reg: + maxItems: 1 + + system-power-controller: + type: boolean + description: Optional property that indicates that this device is + controlling system power. + + interrupts: + description: Short-circuit, over-current, under-voltage for regulators, PB interrupts. + maxItems: 1 + + interrupt-controller: true + + '#interrupt-cells': + description: Specifies the PIN numbers and Flags, as defined in + include/dt-bindings/interrupt-controller/irq.h + const: 1 + + ti,power-button: + type: boolean + description: | + Optional property that sets the EN/PB/VSENSE pin to be a + power-button. + TPS65219 has a multipurpose pin called EN/PB/VSENSE that can be either + 1. EN in which case it functions as an enable pin. + 2. VSENSE which compares the voltages and triggers an automatic + on/off request. + 3. PB in which case it can be configured to trigger an interrupt + to the SoC. + ti,power-button reflects the last one of those options + where the board has a button wired to the pin and triggers + an interrupt on pressing it. + +patternProperties: + "^buck[1-3]-supply$": + description: Input supply phandle of one regulator. + + "^ldo[1-4]-supply$": + description: Input supply phandle of one regulator. + + regulators: + type: object + description: | + list of regulators provided by this controller + + patternProperties: + "^ldo[1-4]$": + type: object + $ref: regulator.yaml# + description: + Properties for single LDO regulator. + + unevaluatedProperties: false + + "^buck[1-3]$": + type: object + $ref: regulator.yaml# + description: + Properties for single BUCK regulator. + + unevaluatedProperties: false + + additionalProperties: false + +required: + - compatible + - reg + - interrupts + - regulators + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + tps65219: pmic@30 { + compatible = "ti,tps65219"; + reg = <0x30>; + buck1-supply = <&vcc_3v3_sys>; + buck2-supply = <&vcc_3v3_sys>; + buck3-supply = <&vcc_3v3_sys>; + ldo1-supply = <&vcc_3v3_sys>; + ldo2-supply = <&buck2_reg>; + ldo3-supply = <&vcc_3v3_sys>; + ldo4-supply = <&vcc_3v3_sys>; + + pinctrl-0 = <&pmic_irq_pins_default>; + + interrupt-parent = <&gic500>; + interrupts = ; + ti,power-button; + + regulators { + buck1_reg: buck1 { + regulator-name = "VDD_CORE"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <750000>; + regulator-boot-on; + regulator-always-on; + }; + + buck2_reg: buck2 { + regulator-name = "VCC1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + buck3_reg: buck3 { + regulator-name = "VDD_LPDDR4"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: ldo1 { + regulator-name = "VDDSHV_SD_IO_PMIC"; + regulator-min-microvolt = <33000000>; + regulator-max-microvolt = <33000000>; + }; + + ldo2_reg: ldo2 { + regulator-name = "VDDAR_CORE"; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <850000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo3_reg: ldo3 { + regulator-name = "VDDA_1V8"; + regulator-min-microvolt = <18000000>; + regulator-max-microvolt = <18000000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo4_reg: ldo4 { + regulator-name = "VDD_PHY_2V5"; + regulator-min-microvolt = <25000000>; + regulator-max-microvolt = <25000000>; + regulator-boot-on; + regulator-always-on; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml index 3072af5f9d79bab1e7914148ef091916870c829c..db9e0f0c2bea02334d2d708928abb3111a67dbe8 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml @@ -152,6 +152,7 @@ properties: description: Qualcomm Shared Memory subnode which represents communication edge, channels and devices related to the ADSP. + unevaluatedProperties: false glink-edge: $ref: /schemas/remoteproc/qcom,glink-edge.yaml# diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,glink-edge.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,glink-edge.yaml index fa69f7b21eed48a9ef9dcd864bb40e3ed1943ff3..25c27464ef25fbdeff9aeeff01e5fc6981cb34e0 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,glink-edge.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,glink-edge.yaml @@ -19,13 +19,17 @@ properties: apr: $ref: /schemas/soc/qcom/qcom,apr.yaml# + required: + - qcom,glink-channels description: Qualcomm APR/GPR (Asynchronous/Generic Packet Router) fastrpc: - type: object + $ref: /schemas/misc/qcom,fastrpc.yaml# + required: + - qcom,glink-channels description: - See Documentation/devicetree/bindings/misc/qcom,fastrpc.txt + Qualcomm FastRPC interrupts: maxItems: 1 diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pil-info.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pil-info.yaml index 9282837d64ba59e8d6e1729a29b81a966662cd4b..a7711e3c998cc62a7266d059a2e48e907e69b958 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,pil-info.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,pil-info.yaml @@ -30,7 +30,7 @@ additionalProperties: false examples: - | imem@146bf000 { - compatible = "syscon", "simple-mfd"; + compatible = "qcom,sdm630-imem", "syscon", "simple-mfd"; reg = <0x146bf000 0x1000>; #address-cells = <1>; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml index e76c861165dd004af1c2aeb132e1fb3ca9ab4550..e4a7da8020f4714f1c1c227a5758de591a007944 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml @@ -140,6 +140,7 @@ properties: glink-edge: $ref: qcom,glink-edge.yaml# + unevaluatedProperties: false description: Qualcomm G-Link subnode which represents communication edge, channels and devices related to the DSP. diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml index da1a5de3d38b4bab399bf41004e5b8ce9658da55..b4de0521a89dea9069d1e55b6cae9ff0c97bd9da 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml @@ -154,6 +154,7 @@ properties: glink-edge: $ref: qcom,glink-edge.yaml# + unevaluatedProperties: false description: Qualcomm G-Link subnode which represents communication edge, channels and devices related to the DSP. diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml index 3f06d66cbe47680c785f2f60cca9b38e7b9e6c84..b6bd3343858450db5da8aa9ba51f6f01c97fd411 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml @@ -107,6 +107,7 @@ properties: glink-edge: $ref: qcom,glink-edge.yaml# + unevaluatedProperties: false description: Qualcomm G-Link subnode which represents communication edge, channels and devices related to the ADSP. diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,smd-edge.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,smd-edge.yaml index 06eebf791e328da417f153363cf049e27d5071dd..7ec8a6b6682c573e94281972775fba79e87832e6 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,smd-edge.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,smd-edge.yaml @@ -13,12 +13,30 @@ description: Qualcomm SMD subnode represents a remote subsystem or a remote processor of some sort - or in SMD language an "edge". The name of the edges are not important. + + In turn, subnodes of the "edges" represent devices tied to SMD channels on + that "edge". The names of the devices are not important. The properties of + these nodes are defined by the individual bindings for the SMD devices. See also Documentation/devicetree/bindings/soc/qcom/qcom,smd.yaml properties: $nodename: const: "smd-edge" + apr: + $ref: /schemas/soc/qcom/qcom,apr.yaml# + required: + - qcom,smd-channels + description: + Qualcomm APR/GPR (Asynchronous/Generic Packet Router) + + fastrpc: + $ref: /schemas/misc/qcom,fastrpc.yaml# + required: + - qcom,smd-channels + description: + Qualcomm FastRPC + interrupts: maxItems: 1 @@ -56,6 +74,20 @@ properties: The identifier for the remote processor as known by the rest of the system. + rpm-requests: + $ref: /schemas/soc/qcom/qcom,smd-rpm.yaml# + required: + - qcom,smd-channels + description: + Qualcomm Resource Power Manager (RPM) over SMD. + + wcnss: + $ref: /schemas/soc/qcom/qcom,wcnss.yaml + required: + - qcom,smd-channels + description: + Qualcomm WCNSS for Bluetooth, WiFi and FM radio. + required: - interrupts - qcom,smd-edge @@ -66,7 +98,7 @@ oneOf: - required: - qcom,ipc -additionalProperties: true +additionalProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/reserved-memory/google,open-dice.yaml b/Documentation/devicetree/bindings/reserved-memory/google,open-dice.yaml index 257a0b51994a5477a9494395a319de0608a7f154..a924fcfca085ecbb940002dbd6636d8d3ecae21f 100644 --- a/Documentation/devicetree/bindings/reserved-memory/google,open-dice.yaml +++ b/Documentation/devicetree/bindings/reserved-memory/google,open-dice.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reserved-memory/google,open-dice.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Open Profile for DICE Device Tree Bindings +title: Open Profile for DICE description: | This binding represents a reserved memory region containing data diff --git a/Documentation/devicetree/bindings/reserved-memory/memory-region.yaml b/Documentation/devicetree/bindings/reserved-memory/memory-region.yaml index 83dfe499a25913312ae3e34045d418a0ffde64cc..592f180e6b0d98e2db4947d03b1417488e9a6867 100644 --- a/Documentation/devicetree/bindings/reserved-memory/memory-region.yaml +++ b/Documentation/devicetree/bindings/reserved-memory/memory-region.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reserved-memory/memory-region.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Reserved Memory Region Device Tree Binding +title: Reserved Memory Region maintainers: - devicetree-spec@vger.kernel.org diff --git a/Documentation/devicetree/bindings/reserved-memory/nvidia,tegra210-emc-table.yaml b/Documentation/devicetree/bindings/reserved-memory/nvidia,tegra210-emc-table.yaml index 035a50fe3ee443624b51e1162a3abe417c2405ce..b1b0421a425511f4f7f526d9aaabdf4671d4a912 100644 --- a/Documentation/devicetree/bindings/reserved-memory/nvidia,tegra210-emc-table.yaml +++ b/Documentation/devicetree/bindings/reserved-memory/nvidia,tegra210-emc-table.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reserved-memory/nvidia,tegra210-emc-table.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NVIDIA Tegra210 EMC Frequency Table Device Tree Bindings +title: NVIDIA Tegra210 EMC Frequency Table maintainers: - Thierry Reding diff --git a/Documentation/devicetree/bindings/reserved-memory/reserved-memory.yaml b/Documentation/devicetree/bindings/reserved-memory/reserved-memory.yaml index 7a0744052ff618974bb7ff404494477dfb25eda5..44f72bcf17825e13cd46d92f27fb28eaa441e271 100644 --- a/Documentation/devicetree/bindings/reserved-memory/reserved-memory.yaml +++ b/Documentation/devicetree/bindings/reserved-memory/reserved-memory.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reserved-memory/reserved-memory.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: /reserved-memory Child Node Common Device Tree Bindings +title: /reserved-memory Child Node Common maintainers: - devicetree-spec@vger.kernel.org diff --git a/Documentation/devicetree/bindings/reset/allwinner,sun6i-a31-clock-reset.yaml b/Documentation/devicetree/bindings/reset/allwinner,sun6i-a31-clock-reset.yaml index 001c0d2a8c1fd45828d80c97a50a33b7dae26257..2a248e511c184034572683335c7aa757a79558a6 100644 --- a/Documentation/devicetree/bindings/reset/allwinner,sun6i-a31-clock-reset.yaml +++ b/Documentation/devicetree/bindings/reset/allwinner,sun6i-a31-clock-reset.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reset/allwinner,sun6i-a31-clock-reset.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 Peripheral Reset Controller Device Tree Bindings +title: Allwinner A31 Peripheral Reset Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/reset/amlogic,meson-reset.yaml b/Documentation/devicetree/bindings/reset/amlogic,meson-reset.yaml index 494a454928ce62dc85b119ec9e1d25bae44bb9a5..98db2aa74dc88f9e6d603ef1df74726f73b1eec9 100644 --- a/Documentation/devicetree/bindings/reset/amlogic,meson-reset.yaml +++ b/Documentation/devicetree/bindings/reset/amlogic,meson-reset.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson SoC Reset Controller maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml index 53e4ede9c0bde2230ea2bf395defe9a0848cdb0b..ee8a2dcf5dfae9dea839339d8a75e8a68ac8188a 100644 --- a/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml +++ b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reset/canaan,k210-rst.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Canaan Kendryte K210 Reset Controller Device Tree Bindings +title: Canaan Kendryte K210 Reset Controller maintainers: - Damien Le Moal diff --git a/Documentation/devicetree/bindings/reset/starfive,jh7100-reset.yaml b/Documentation/devicetree/bindings/reset/starfive,jh7100-reset.yaml index 300359a5e14b41e77ff3c81db821c3f6ebd89f3c..2f5bd616b7ae4b69d8d211b067406c3aa09e3b89 100644 --- a/Documentation/devicetree/bindings/reset/starfive,jh7100-reset.yaml +++ b/Documentation/devicetree/bindings/reset/starfive,jh7100-reset.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/reset/starfive,jh7100-reset.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: StarFive JH7100 SoC Reset Controller Device Tree Bindings +title: StarFive JH7100 SoC Reset Controller maintainers: - Emil Renner Berthing diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index 873dd12f6e896d720fb3fe46fc27fb6139344d4d..90a7cabf58feb057730a72261ec11eb72d35ee22 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -9,6 +9,7 @@ title: RISC-V bindings for 'cpus' DT nodes maintainers: - Paul Walmsley - Palmer Dabbelt + - Conor Dooley description: | This document uses some terminology common to the RISC-V community @@ -79,9 +80,7 @@ properties: insensitive, letters in the riscv,isa string must be all lowercase to simplify parsing. $ref: "/schemas/types.yaml#/definitions/string" - enum: - - rv64imac - - rv64imafdc + pattern: ^rv(?:64|32)imaf?d?q?c?b?v?k?h?(?:_[hsxz](?:[a-z])+)*$ # RISC-V requires 'timebase-frequency' in /cpus, so disallow it here timebase-frequency: false diff --git a/Documentation/devicetree/bindings/riscv/microchip.yaml b/Documentation/devicetree/bindings/riscv/microchip.yaml index 1aa7336a9672f6c1c4248ee724cba1f4bc2042c5..714d0fcab39964d1ed7cad0756c8d41eb091adfa 100644 --- a/Documentation/devicetree/bindings/riscv/microchip.yaml +++ b/Documentation/devicetree/bindings/riscv/microchip.yaml @@ -4,11 +4,11 @@ $id: http://devicetree.org/schemas/riscv/microchip.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip PolarFire SoC-based boards device tree bindings +title: Microchip PolarFire SoC-based boards maintainers: - - Cyril Jean - - Lewis Hanly + - Conor Dooley + - Daire McNamara description: Microchip PolarFire SoC-based boards @@ -17,12 +17,20 @@ properties: $nodename: const: '/' compatible: - items: - - enum: - - microchip,mpfs-icicle-kit - - microchip,mpfs-icicle-reference-rtlv2203 - - sundance,polarberry - - const: microchip,mpfs + oneOf: + - items: + - enum: + - microchip,mpfs-icicle-reference-rtlv2203 + - microchip,mpfs-icicle-reference-rtlv2210 + - const: microchip,mpfs-icicle-kit + - const: microchip,mpfs + + - items: + - enum: + - aries,m100pfsevp + - microchip,mpfs-sev-kit + - sundance,polarberry + - const: microchip,mpfs additionalProperties: true diff --git a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml b/Documentation/devicetree/bindings/riscv/sifive,ccache0.yaml similarity index 54% rename from Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml rename to Documentation/devicetree/bindings/riscv/sifive,ccache0.yaml index 69cdab18d6294935e22f3ef0b5e78bf2bb2cde1f..bf3f07421f7e5df3e7ca90c88c06290074321650 100644 --- a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml +++ b/Documentation/devicetree/bindings/riscv/sifive,ccache0.yaml @@ -2,29 +2,27 @@ # Copyright (C) 2020 SiFive, Inc. %YAML 1.2 --- -$id: http://devicetree.org/schemas/riscv/sifive-l2-cache.yaml# +$id: http://devicetree.org/schemas/riscv/sifive,ccache0.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: SiFive L2 Cache Controller +title: SiFive Composable Cache Controller maintainers: - Sagar Kadam - Paul Walmsley description: - The SiFive Level 2 Cache Controller is used to provide access to fast copies - of memory for masters in a Core Complex. The Level 2 Cache Controller also + The SiFive Composable Cache Controller is used to provide access to fast copies + of memory for masters in a Core Complex. The Composable Cache Controller also acts as directory-based coherency manager. All the properties in ePAPR/DeviceTree specification applies for this platform. -allOf: - - $ref: /schemas/cache-controller.yaml# - select: properties: compatible: contains: enum: + - sifive,ccache0 - sifive,fu540-c000-ccache - sifive,fu740-c000-ccache @@ -33,17 +31,23 @@ select: properties: compatible: - items: - - enum: - - sifive,fu540-c000-ccache - - sifive,fu740-c000-ccache - - const: cache + oneOf: + - items: + - enum: + - sifive,ccache0 + - sifive,fu540-c000-ccache + - sifive,fu740-c000-ccache + - const: cache + - items: + - const: microchip,mpfs-ccache + - const: sifive,fu540-c000-ccache + - const: cache cache-block-size: const: 64 cache-level: - const: 2 + enum: [2, 3] cache-sets: enum: [1024, 2048] @@ -72,29 +76,62 @@ properties: The reference to the reserved-memory for the L2 Loosely Integrated Memory region. The reserved memory node should be defined as per the bindings in reserved-memory.txt. -if: - properties: - compatible: - contains: - const: sifive,fu540-c000-ccache +allOf: + - $ref: /schemas/cache-controller.yaml# -then: - properties: - interrupts: - description: | - Must contain entries for DirError, DataError and DataFail signals. - maxItems: 3 - cache-sets: - const: 1024 - -else: - properties: - interrupts: - description: | - Must contain entries for DirError, DataError, DataFail, DirFail signals. - minItems: 4 - cache-sets: - const: 2048 + - if: + properties: + compatible: + contains: + enum: + - sifive,fu740-c000-ccache + - microchip,mpfs-ccache + + then: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError, DataFail, DirFail signals. + minItems: 4 + + else: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError and DataFail signals. + maxItems: 3 + + - if: + properties: + compatible: + contains: + const: sifive,fu740-c000-ccache + + then: + properties: + cache-sets: + const: 2048 + + else: + properties: + cache-sets: + const: 1024 + + - if: + properties: + compatible: + contains: + const: sifive,ccache0 + + then: + properties: + cache-level: + enum: [2, 3] + + else: + properties: + cache-level: + const: 2 additionalProperties: false diff --git a/Documentation/devicetree/bindings/rng/amlogic,meson-rng.yaml b/Documentation/devicetree/bindings/rng/amlogic,meson-rng.yaml index 444be32a8a295ea43e1f806914b39a68fdce59ee..09c6c906b1f9733bbd3d3d66f90f7123e48eb28f 100644 --- a/Documentation/devicetree/bindings/rng/amlogic,meson-rng.yaml +++ b/Documentation/devicetree/bindings/rng/amlogic,meson-rng.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson Random number generator maintainers: - - Neil Armstrong + - Neil Armstrong properties: compatible: diff --git a/Documentation/devicetree/bindings/rng/omap_rng.yaml b/Documentation/devicetree/bindings/rng/omap_rng.yaml index 010188cdbec86f403a927f4412c78c9a5b1314b1..ccf54fae8302817e199fb1f7ed2f77311fc35590 100644 --- a/Documentation/devicetree/bindings/rng/omap_rng.yaml +++ b/Documentation/devicetree/bindings/rng/omap_rng.yaml @@ -53,17 +53,6 @@ allOf: required: - interrupts - - if: - properties: - compatible: - contains: - enum: - - inside-secure,safexcel-eip76 - - then: - required: - - clocks - required: - compatible diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml index 478b0234e8fa97416db572c50ce58fed4c5b88c7..dede49431733921c16b68c9aefe6603bcf438eac 100644 --- a/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/allwinner,sun4i-a10-rtc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/rtc/allwinner,sun4i-a10-rtc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 RTC Device Tree Bindings +title: Allwinner A10 RTC allOf: - $ref: "rtc.yaml#" diff --git a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml index 6b38bd7eb3b45ac0cdd32bebe81ee0d644bb63a8..04947e166cef3228896a77716ee21236bb41c4ad 100644 --- a/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/allwinner,sun6i-a31-rtc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/rtc/allwinner,sun6i-a31-rtc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 RTC Device Tree Bindings +title: Allwinner A31 RTC maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml index 994de43d17fadef9fcf4a8a4195cdff4f77e8e78..0e5f0fcc26b018036d02bfd06136d45256c0e693 100644 --- a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/rtc/atmel,at91rm9200-rtc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Atmel AT91 RTC Device Tree Bindings +title: Atmel AT91 RTC allOf: - $ref: "rtc.yaml#" diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml b/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml index 0ef1b7ff4a77f07f570b54b744b26c770fd7852e..b5cd20e89daf240f8276557f5b91aa98aef5e988 100644 --- a/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml +++ b/Documentation/devicetree/bindings/rtc/atmel,at91sam9260-rtt.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/rtc/atmel,at91sam9260-rtt.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Atmel AT91 RTT Device Tree Bindings +title: Atmel AT91 RTT allOf: - $ref: "rtc.yaml#" diff --git a/Documentation/devicetree/bindings/rtc/microchip,mfps-rtc.yaml b/Documentation/devicetree/bindings/rtc/microchip,mfps-rtc.yaml index 500c62becd6bc60215fbb1303fd33fbc7152d362..7742465b938398d7636b1de186f9954d777d3d90 100644 --- a/Documentation/devicetree/bindings/rtc/microchip,mfps-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/microchip,mfps-rtc.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/rtc/microchip,mfps-rtc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip PolarFire Soc (MPFS) RTC Device Tree Bindings +title: Microchip PolarFire Soc (MPFS) RTC allOf: - $ref: rtc.yaml# diff --git a/Documentation/devicetree/bindings/rtc/microcrystal,rv3032.yaml b/Documentation/devicetree/bindings/rtc/microcrystal,rv3032.yaml index 60f9027e8299845bb26c6b29a08341ec621d3e07..dd6eebf06ea6076111a3cf36785cf90040eeaa99 100644 --- a/Documentation/devicetree/bindings/rtc/microcrystal,rv3032.yaml +++ b/Documentation/devicetree/bindings/rtc/microcrystal,rv3032.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/rtc/microcrystal,rv3032.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip RV-3032 RTC Device Tree Bindings +title: Microchip RV-3032 RTC allOf: - $ref: "rtc.yaml#" diff --git a/Documentation/devicetree/bindings/rtc/mstar,msc313-rtc.yaml b/Documentation/devicetree/bindings/rtc/mstar,msc313-rtc.yaml index 114199cf4d28f0cfe9677f051ffcd7e115f5d789..585c185d1eb36ceabe1f060a12ff037fbdf1c831 100644 --- a/Documentation/devicetree/bindings/rtc/mstar,msc313-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/mstar,msc313-rtc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/rtc/mstar,msc313-rtc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mstar MSC313e RTC Device Tree Bindings +title: Mstar MSC313e RTC allOf: - $ref: "rtc.yaml#" diff --git a/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml b/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml index 72e8868db3e01770d1b4f3bdecd8b36a43619b59..7822705ad16c6baa77d323f3bc2aa5595a3610a4 100644 --- a/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml +++ b/Documentation/devicetree/bindings/serial/amlogic,meson-uart.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson SoC UART Serial Interface maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Amlogic Meson SoC UART Serial Interface is present on a large range diff --git a/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml b/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30b2131b586018f663e17750bb428e2a48798633 --- /dev/null +++ b/Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/serial/atmel,at91-usart.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel Universal Synchronous Asynchronous Receiver/Transmitter (USART) + +maintainers: + - Richard Genoud + +properties: + compatible: + oneOf: + - enum: + - atmel,at91rm9200-usart + - atmel,at91sam9260-usart + - microchip,sam9x60-usart + - items: + - const: atmel,at91rm9200-dbgu + - const: atmel,at91rm9200-usart + - items: + - const: atmel,at91sam9260-dbgu + - const: atmel,at91sam9260-usart + - items: + - const: microchip,sam9x60-dbgu + - const: microchip,sam9x60-usart + - const: atmel,at91sam9260-dbgu + - const: atmel,at91sam9260-usart + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clock-names: + minItems: 1 + items: + - const: usart + - const: gclk + + clocks: + minItems: 1 + items: + - description: USART Peripheral Clock + - description: USART Generic Clock + + dmas: + items: + - description: TX DMA Channel + - description: RX DMA Channel + + dma-names: + items: + - const: tx + - const: rx + + atmel,usart-mode: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Must be either for SPI or + for USART (found in dt-bindings/mfd/at91-usart.h). + enum: [ 0, 1 ] + + atmel,use-dma-rx: + type: boolean + description: use of PDC or DMA for receiving data + + atmel,use-dma-tx: + type: boolean + description: use of PDC or DMA for transmitting data + + atmel,fifo-size: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + Maximum number of data the RX and TX FIFOs can store for FIFO + capable USARTS. + enum: [ 16, 32 ] + +required: + - compatible + - reg + - interrupts + - clock-names + - clocks + - atmel,usart-mode + +allOf: + - if: + properties: + atmel,usart-mode: + const: 1 + then: + allOf: + - $ref: /schemas/spi/spi-controller.yaml# + + properties: + atmel,use-dma-rx: false + + atmel,use-dma-tx: false + + atmel,fifo-size: false + + "#size-cells": + const: 0 + + "#address-cells": + const: 1 + + required: + - "#size-cells" + - "#address-cells" + + else: + allOf: + - $ref: /schemas/serial/serial.yaml# + - $ref: /schemas/serial/rs485.yaml# + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + #include + + /* use PDC */ + usart0: serial@fff8c000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xfff8c000 0x4000>; + atmel,usart-mode = ; + interrupts = <7>; + clocks = <&usart0_clk>; + clock-names = "usart"; + atmel,use-dma-rx; + atmel,use-dma-tx; + rts-gpios = <&pioD 15 GPIO_ACTIVE_LOW>; + cts-gpios = <&pioD 16 GPIO_ACTIVE_LOW>; + dtr-gpios = <&pioD 17 GPIO_ACTIVE_LOW>; + dsr-gpios = <&pioD 18 GPIO_ACTIVE_LOW>; + dcd-gpios = <&pioD 20 GPIO_ACTIVE_LOW>; + rng-gpios = <&pioD 19 GPIO_ACTIVE_LOW>; + }; + + - | + #include + #include + #include + #include + + /* use DMA */ + usart1: serial@f001c000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xf001c000 0x100>; + atmel,usart-mode = ; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH 5>; + clocks = <&usart0_clk>; + clock-names = "usart"; + atmel,use-dma-rx; + atmel,use-dma-tx; + dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(3)>, + <&dma0 2 (AT91_DMA_CFG_PER_ID(4) | AT91_DMA_CFG_FIFOCFG_ASAP)>; + dma-names = "tx", "rx"; + atmel,fifo-size = <32>; + }; + + - | + #include + #include + #include + #include + + /* SPI mode */ + spi0: spi@f001c000 { + compatible = "atmel,at91sam9260-usart"; + reg = <0xf001c000 0x100>; + #address-cells = <1>; + #size-cells = <0>; + atmel,usart-mode = ; + interrupts = <12 IRQ_TYPE_LEVEL_HIGH 5>; + clocks = <&usart0_clk>; + clock-names = "usart"; + dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(3)>, + <&dma0 2 (AT91_DMA_CFG_PER_ID(4) | AT91_DMA_CFG_FIFOCFG_ASAP)>; + dma-names = "tx", "rx"; + cs-gpios = <&pioB 3 GPIO_ACTIVE_HIGH>; + }; diff --git a/Documentation/devicetree/bindings/serial/cdns,uart.yaml b/Documentation/devicetree/bindings/serial/cdns,uart.yaml index fcdbe6f87e98085702a829ca18dc49a14d17f2ab..876b8cf1cafbe83a6b7d2b23b5ed10a5f6fbc5c9 100644 --- a/Documentation/devicetree/bindings/serial/cdns,uart.yaml +++ b/Documentation/devicetree/bindings/serial/cdns,uart.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/serial/cdns,uart.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Cadence UART Controller Device Tree Bindings +title: Cadence UART Controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/serial/mediatek,uart.yaml b/Documentation/devicetree/bindings/serial/mediatek,uart.yaml index 4ff27d6d4d5b5913889ee462af8bdeb4cfe28e83..fe098d98af6ee12e48ed926eb220bf430dc70c23 100644 --- a/Documentation/devicetree/bindings/serial/mediatek,uart.yaml +++ b/Documentation/devicetree/bindings/serial/mediatek,uart.yaml @@ -42,6 +42,7 @@ properties: - mediatek,mt8173-uart - mediatek,mt8183-uart - mediatek,mt8186-uart + - mediatek,mt8188-uart - mediatek,mt8192-uart - mediatek,mt8195-uart - mediatek,mt8516-uart diff --git a/Documentation/devicetree/bindings/serial/pl011.yaml b/Documentation/devicetree/bindings/serial/pl011.yaml index d8aed84abcd32f4cd7a0d4c17f530e39af091448..80af728598764a006e82a0523815ea88fc2b225a 100644 --- a/Documentation/devicetree/bindings/serial/pl011.yaml +++ b/Documentation/devicetree/bindings/serial/pl011.yaml @@ -94,6 +94,12 @@ properties: resets: maxItems: 1 + reg-io-width: + description: + The size (in bytes) of the IO accesses that should be performed + on the device. + enum: [1, 4] + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/serial/renesas,scif.yaml b/Documentation/devicetree/bindings/serial/renesas,scif.yaml index 90fe45265fbc64cc203bba384329f70df8b79cdb..f930e7f1349fcea8e906dccd1cad0af7c883ba00 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scif.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scif.yaml @@ -76,7 +76,7 @@ properties: - items: - enum: - - renesas,scif-r9a07g043 # RZ/G2UL + - renesas,scif-r9a07g043 # RZ/G2UL and RZ/Five - renesas,scif-r9a07g054 # RZ/V2L - const: renesas,scif-r9a07g044 # RZ/G2{L,LC} fallback diff --git a/Documentation/devicetree/bindings/serial/samsung_uart.yaml b/Documentation/devicetree/bindings/serial/samsung_uart.yaml index 901c1e2cea28cbcbd7b2c94497ecf3c205b5394e..8bd88d5cbb1186bedb86ad9499990a6fc3135e7e 100644 --- a/Documentation/devicetree/bindings/serial/samsung_uart.yaml +++ b/Documentation/devicetree/bindings/serial/samsung_uart.yaml @@ -17,7 +17,10 @@ description: |+ properties: compatible: - items: + oneOf: + - items: + - const: samsung,exynosautov9-uart + - const: samsung,exynos850-uart - enum: - apple,s5l-uart - axis,artpec8-uart @@ -37,7 +40,6 @@ properties: description: | The size (in bytes) of the IO accesses that should be performed on the device. - $ref: /schemas/types.yaml#/definitions/uint32 enum: [ 1, 4 ] clocks: @@ -69,6 +71,9 @@ properties: minItems: 1 maxItems: 2 + power-domains: + maxItems: 1 + samsung,uart-fifosize: description: The fifo size supported by the UART channel. $ref: /schemas/types.yaml#/definitions/uint32 diff --git a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml index dc74643ae72ecc14e6812964430ddd2364ad5210..b9c2287c5d1e7c05c0a8cf75b51003ddbcd32e25 100644 --- a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml +++ b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml @@ -26,6 +26,7 @@ properties: - rockchip,rk1808-uart - rockchip,rk3036-uart - rockchip,rk3066-uart + - rockchip,rk3128-uart - rockchip,rk3188-uart - rockchip,rk3288-uart - rockchip,rk3308-uart diff --git a/Documentation/devicetree/bindings/serial/sunplus,sp7021-uart.yaml b/Documentation/devicetree/bindings/serial/sunplus,sp7021-uart.yaml index 2e9b64abde70963404d8b1bc83f18afaa1028601..ea1e637661c7b17e08fe90536df1bafa1a11d04f 100644 --- a/Documentation/devicetree/bindings/serial/sunplus,sp7021-uart.yaml +++ b/Documentation/devicetree/bindings/serial/sunplus,sp7021-uart.yaml @@ -5,7 +5,7 @@ $id: "http://devicetree.org/schemas/serial/sunplus,sp7021-uart.yaml#" $schema: "http://devicetree.org/meta-schemas/core.yaml#" -title: Sunplus SoC SP7021 UART Controller Device Tree Bindings +title: Sunplus SoC SP7021 UART Controller maintainers: - Hammer Hsieh diff --git a/Documentation/devicetree/bindings/serio/allwinner,sun4i-a10-ps2.yaml b/Documentation/devicetree/bindings/serio/allwinner,sun4i-a10-ps2.yaml index 2ecab8ed702a20abfa5d1e44b89e267852df1c06..7fa70fd1f2916b94cdf7adc7d2155f8254625d30 100644 --- a/Documentation/devicetree/bindings/serio/allwinner,sun4i-a10-ps2.yaml +++ b/Documentation/devicetree/bindings/serio/allwinner,sun4i-a10-ps2.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/serio/allwinner,sun4i-a10-ps2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 PS2 Host Controller Device Tree Bindings +title: Allwinner A10 PS2 Host Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/soc/amlogic/amlogic,canvas.yaml b/Documentation/devicetree/bindings/soc/amlogic/amlogic,canvas.yaml index 17db87cb9dabc95a027d1fdbad2b8e0ebc9f10d8..c3c59909635310d6a72a09b57f26c23412c10a0e 100644 --- a/Documentation/devicetree/bindings/soc/amlogic/amlogic,canvas.yaml +++ b/Documentation/devicetree/bindings/soc/amlogic/amlogic,canvas.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Canvas Video Lookup Table maintainers: - - Neil Armstrong + - Neil Armstrong - Maxime Jourdan description: | diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml index 26487daa64d9ade6ae27bd71c61740db40752012..d71bb20d49073b63ca7634eec4600d4fd6668109 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml @@ -27,25 +27,22 @@ properties: const: 1 power-domains: - minItems: 4 maxItems: 4 power-domain-names: - items: - - const: bus - - const: g1 - - const: g2 - - const: h1 + maxItems: 4 clocks: - minItems: 3 maxItems: 3 clock-names: - items: - - const: g1 - - const: g2 - - const: h1 + maxItems: 3 + + interconnects: + maxItems: 3 + + interconnect-names: + maxItems: 3 required: - compatible @@ -55,6 +52,97 @@ required: - clocks - clock-names +allOf: + - if: + properties: + compatible: + contains: + const: fsl,imx8mm-vpu-blk-ctrl + then: + properties: + power-domains: + items: + - description: bus power domain + - description: G1 decoder power domain + - description: G2 decoder power domain + - description: H1 encoder power domain + + power-domain-names: + items: + - const: bus + - const: g1 + - const: g2 + - const: h1 + + clocks: + items: + - description: G1 decoder clk + - description: G2 decoder clk + - description: H1 encoder clk + + clock-names: + items: + - const: g1 + - const: g2 + - const: h1 + + interconnects: + items: + - description: G1 decoder interconnect + - description: G2 decoder interconnect + - description: H1 encoder power domain + + interconnect-names: + items: + - const: g1 + - const: g2 + - const: h1 + + - if: + properties: + compatible: + contains: + const: fsl,imx8mp-vpu-blk-ctrl + then: + properties: + power-domains: + items: + - description: bus power domain + - description: G1 decoder power domain + - description: G2 decoder power domain + - description: VC8000E encoder power domain + + power-domain-names: + items: + - const: bus + - const: g1 + - const: g2 + - const: vc8000e + + clocks: + items: + - description: G1 decoder clk + - description: G2 decoder clk + - description: VC8000E encoder clk + + clock-names: + items: + - const: g1 + - const: g2 + - const: vc8000e + + interconnects: + items: + - description: G1 decoder interconnect + - description: G2 decoder interconnect + - description: VC8000E encoder interconnect + + interconnect-names: + items: + - const: g1 + - const: g2 + - const: vc8000e + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml index 563e1d0e327f8b0dad29cc98083fe593b214779e..1be4ce2a45e8e678accbb2194ce9c3fb896f3102 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml @@ -52,6 +52,15 @@ properties: - const: ref_266m - const: ref_24m + interconnects: + maxItems: 3 + + interconnect-names: + items: + - const: hrv + - const: lcdif-hdmi + - const: hdcp + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml index c1e29d94f40e85689b7deb2591de29d2373c4107..c29181a9745b96ac4fc3ed8282d41aeb6ba9bcd1 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml @@ -48,6 +48,16 @@ properties: - const: usb - const: pcie + interconnects: + maxItems: 4 + + interconnect-names: + items: + - const: noc-pcie + - const: usb1 + - const: usb2 + - const: pcie + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml index b246d8386ba4ad1ad01bf75cff23dd9fc3803a82..dadb6108e321368adef2329a6127a5b826846fbe 100644 --- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml @@ -64,6 +64,20 @@ properties: - const: isp - const: phy + interconnects: + maxItems: 8 + + interconnect-names: + items: + - const: lcdif-rd + - const: lcdif-wr + - const: isi0 + - const: isi1 + - const: isi2 + - const: isp0 + - const: isp1 + - const: dwe + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..792ebecec22d035a0ee708b5b5743baed46c78df --- /dev/null +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/imx/fsl,imx93-media-blk-ctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP i.MX93 Media blk-ctrl + +maintainers: + - Peng Fan + +description: + The i.MX93 MEDIAMIX domain contains control and status registers known + as MEDIAMIX Block Control (MEDIAMIX BLK_CTRL). These registers include + clocking, reset, and miscellaneous top-level controls for peripherals + within the MEDIAMIX domain + +properties: + compatible: + items: + - const: fsl,imx93-media-blk-ctrl + - const: syscon + + reg: + maxItems: 1 + + '#power-domain-cells': + const: 1 + + power-domains: + maxItems: 1 + + clocks: + maxItems: 10 + + clock-names: + items: + - const: apb + - const: axi + - const: nic + - const: disp + - const: cam + - const: pxp + - const: lcdif + - const: isi + - const: csi + - const: dsi + +required: + - compatible + - reg + - power-domains + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + #include + + media_blk_ctrl: system-controller@4ac10000 { + compatible = "fsl,imx93-media-blk-ctrl", "syscon"; + reg = <0x4ac10000 0x10000>; + power-domains = <&mediamix>; + clocks = <&clk IMX93_CLK_MEDIA_APB>, + <&clk IMX93_CLK_MEDIA_AXI>, + <&clk IMX93_CLK_NIC_MEDIA_GATE>, + <&clk IMX93_CLK_MEDIA_DISP_PIX>, + <&clk IMX93_CLK_CAM_PIX>, + <&clk IMX93_CLK_PXP_GATE>, + <&clk IMX93_CLK_LCDIF_GATE>, + <&clk IMX93_CLK_ISI_GATE>, + <&clk IMX93_CLK_MIPI_CSI_GATE>, + <&clk IMX93_CLK_MIPI_DSI_GATE>; + clock-names = "apb", "axi", "nic", "disp", "cam", + "pxp", "lcdif", "isi", "csi", "dsi"; + #power-domain-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1cc69b5198101c1acb0a4ce5f20c066b3c44f7a --- /dev/null +++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/imx/fsl,imx93-src.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP i.MX93 System Reset Controller + +maintainers: + - Peng Fan + +description: | + The System Reset Controller (SRC) is responsible for the generation of + all the system reset signals and boot argument latching. + + Its main functions are as follows, + - Deals with all global system reset sources from other modules, + and generates global system reset. + - Responsible for power gating of MIXs (Slices) and their memory + low power control. + +properties: + compatible: + items: + - const: fsl,imx93-src + - const: syscon + + reg: + maxItems: 1 + + ranges: true + + '#address-cells': + const: 1 + + '#size-cells': + const: 1 + +patternProperties: + "power-domain@[0-9a-f]+$": + + type: object + properties: + compatible: + items: + - const: fsl,imx93-src-slice + + '#power-domain-cells': + const: 0 + + reg: + items: + - description: mix slice register region + - description: mem slice register region + + clocks: + description: | + A number of phandles to clocks that need to be enabled + during domain power-up sequencing to ensure reset + propagation into devices located inside this power domain. + minItems: 1 + maxItems: 5 + + required: + - compatible + - '#power-domain-cells' + - reg + +required: + - compatible + - reg + - ranges + - '#address-cells' + - '#size-cells' + +additionalProperties: false + +examples: + - | + #include + + system-controller@44460000 { + compatible = "fsl,imx93-src", "syscon"; + reg = <0x44460000 0x10000>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mediamix: power-domain@0 { + compatible = "fsl,imx93-src-slice"; + reg = <0x44462400 0x400>, <0x44465800 0x400>; + #power-domain-cells = <0>; + clocks = <&clk IMX93_CLK_MEDIA_AXI>, + <&clk IMX93_CLK_MEDIA_APB>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,ccorr.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,ccorr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4380b98b0dfe05ecf1da4a25cb44895baf2ce221 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,ccorr.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/mediatek/mediatek,ccorr.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek color correction + +maintainers: + - Matthias Brugger + - Moudy Ho + +description: | + MediaTek color correction with 3X3 matrix. + +properties: + compatible: + items: + - enum: + - mediatek,mt8183-mdp3-ccorr + + reg: + maxItems: 1 + + mediatek,gce-client-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + items: + - description: phandle of GCE + - description: GCE subsys id + - description: register offset + - description: register size + description: The register of client driver can be configured by gce with + 4 arguments defined in this property. Each GCE subsys id is mapping to + a client defined in the header include/dt-bindings/gce/-gce.h. + + mediatek,gce-events: + description: + The event id which is mapping to the specific hardware event signal + to gce. The event id is defined in the gce header + include/dt-bindings/gce/-gce.h of each chips. + $ref: /schemas/types.yaml#/definitions/uint32-array + + clocks: + minItems: 1 + +required: + - compatible + - reg + - mediatek,gce-client-reg + - mediatek,gce-events + - clocks + +additionalProperties: false + +examples: + - | + #include + #include + + mdp3_ccorr: mdp3-ccorr@1401c000 { + compatible = "mediatek,mt8183-mdp3-ccorr"; + reg = <0x1401c000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0xc000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_CCORR>; + }; diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml index 627dcc3e8b323396852c173526e89061940a0171..9241e5fc7cff52a1f3ce5477b446c23dd027c5a4 100644 --- a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml +++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml @@ -26,10 +26,12 @@ properties: enum: - mediatek,mt2701-disp-mutex - mediatek,mt2712-disp-mutex + - mediatek,mt6795-disp-mutex - mediatek,mt8167-disp-mutex - mediatek,mt8173-disp-mutex - mediatek,mt8183-disp-mutex - mediatek,mt8186-disp-mutex + - mediatek,mt8186-mdp3-mutex - mediatek,mt8192-disp-mutex - mediatek,mt8195-disp-mutex diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,wdma.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,wdma.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69afb329e5f4e810edacd949334c0ce65b74808e --- /dev/null +++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,wdma.yaml @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/mediatek/mediatek,wdma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Write Direct Memory Access + +maintainers: + - Matthias Brugger + - Moudy Ho + +description: | + MediaTek Write Direct Memory Access(WDMA) component used to write + the data into DMA. + +properties: + compatible: + items: + - enum: + - mediatek,mt8183-mdp3-wdma + + reg: + maxItems: 1 + + mediatek,gce-client-reg: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + items: + - description: phandle of GCE + - description: GCE subsys id + - description: register offset + - description: register size + description: The register of client driver can be configured by gce with + 4 arguments defined in this property. Each GCE subsys id is mapping to + a client defined in the header include/dt-bindings/gce/-gce.h. + + mediatek,gce-events: + description: + The event id which is mapping to the specific hardware event signal + to gce. The event id is defined in the gce header + include/dt-bindings/gce/-gce.h of each chips. + $ref: /schemas/types.yaml#/definitions/uint32-array + + power-domains: + maxItems: 1 + + clocks: + minItems: 1 + + iommus: + maxItems: 1 + +required: + - compatible + - reg + - mediatek,gce-client-reg + - mediatek,gce-events + - power-domains + - clocks + - iommus + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + mdp3_wdma: mdp3-wdma@14006000 { + compatible = "mediatek,mt8183-mdp3-wdma"; + reg = <0x14006000 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x6000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_WDMA0>; + iommus = <&iommu>; + }; diff --git a/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt b/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt index 0581dbda4828d0ea5078bb88a146601e4a6683fd..d24e2bc444be3e9e3a477e683aeacd5be98c494b 100644 --- a/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt +++ b/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt @@ -28,6 +28,7 @@ Required properties in pwrap device node. "mediatek,mt8173-pwrap" for MT8173 SoCs "mediatek,mt8183-pwrap" for MT8183 SoCs "mediatek,mt8186-pwrap" for MT8186 SoCs + "mediatek,mt8188-pwrap", "mediatek,mt8195-pwrap" for MT8188 SoCs "mediatek,mt8195-pwrap" for MT8195 SoCs "mediatek,mt8516-pwrap" for MT8516 SoCs - interrupts: IRQ for pwrap in SOC diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml index 028c5d105adb3bf67dae18cacc3f162e4e5da32b..f47491aab3b1e2e997ffe46205e2180a43a29f60 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml @@ -20,6 +20,9 @@ properties: - qcom,apr-v2 - qcom,gpr + power-domains: + maxItems: 1 + qcom,apr-domain: $ref: /schemas/types.yaml#/definitions/uint32 enum: [1, 2, 3, 4, 5, 6, 7] @@ -52,6 +55,26 @@ properties: 2 = Audio DSP Domain 3 = Application Processor Domain + qcom,glink-channels: + $ref: /schemas/types.yaml#/definitions/string-array + description: Channel name used for the communication + items: + - const: apr_audio_svc + + qcom,intents: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + List of (size, amount) pairs describing what intents should be + preallocated for this virtual channel. This can be used to tweak the + default intents available for the channel to meet expectations of the + remote. + + qcom,smd-channels: + $ref: /schemas/types.yaml#/definitions/string-array + description: Channel name used for the communication + items: + - const: apr_audio_svc + '#address-cells': const: 1 @@ -97,6 +120,26 @@ patternProperties: 3 = AMDB Service. 4 = Voice processing manager. + clock-controller: + $ref: /schemas/sound/qcom,q6dsp-lpass-clocks.yaml# + description: Qualcomm DSP LPASS clock controller + unevaluatedProperties: false + + dais: + type: object + oneOf: + - $ref: /schemas/sound/qcom,q6apm-dai.yaml# + - $ref: /schemas/sound/qcom,q6dsp-lpass-ports.yaml# + - $ref: /schemas/sound/qcom,q6asm-dais.yaml# + unevaluatedProperties: false + description: Qualcomm DSP audio ports + + routing: + type: object + $ref: /schemas/sound/qcom,q6adm-routing.yaml# + unevaluatedProperties: false + description: Qualcomm DSP LPASS audio routing + qcom,protection-domain: $ref: /schemas/types.yaml#/definitions/string-array description: protection domain service name and path for apr service @@ -107,17 +150,44 @@ patternProperties: "tms/servreg", "msm/modem/wlan_pd". "tms/servreg", "msm/slpi/sensor_pd". - '#address-cells': - const: 1 + allOf: + - if: + properties: + compatible: + enum: + - qcom,q6afe + then: + properties: + dais: + properties: + compatible: + const: qcom,q6afe-dais - '#size-cells': - const: 0 + - if: + properties: + compatible: + enum: + - qcom,q6apm + then: + properties: + dais: + properties: + compatible: + enum: + - qcom,q6apm-dais + - qcom,q6apm-lpass-dais - patternProperties: - "^.*@[0-9a-f]+$": - type: object - description: - Service based devices like clock controllers or digital audio interfaces. + - if: + properties: + compatible: + enum: + - qcom,q6asm + then: + properties: + dais: + properties: + compatible: + const: qcom,q6asm-dais additionalProperties: false @@ -125,6 +195,30 @@ required: - compatible - qcom,domain +allOf: + - if: + properties: + compatible: + enum: + - qcom,gpr + then: + properties: + power-domains: false + + - if: + required: + - qcom,glink-channels + then: + properties: + qcom,smd-channels: false + + - if: + required: + - qcom,smd-channels + then: + properties: + qcom,glink-channels: false + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.yaml index 9b3efe97f47c36b0041781052d1ef10ceacfbfc3..063e595c12f77ba185cea6d85ed797d7b7263deb 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.yaml @@ -20,39 +20,14 @@ properties: const: qcom,smd patternProperties: - "^.*-edge|rpm$": + "^smd-edge|rpm$": $ref: /schemas/remoteproc/qcom,smd-edge.yaml# + unevaluatedProperties: false description: Each subnode of the SMD node represents a remote subsystem or a remote processor of some sort - or in SMD language an "edge". The name of the edges are not important. - properties: - rpm-requests: - type: object - description: - In turn, subnodes of the "edges" represent devices tied to SMD - channels on that "edge". The names of the devices are not - important. The properties of these nodes are defined by the - individual bindings for the SMD devices. - - properties: - qcom,smd-channels: - $ref: /schemas/types.yaml#/definitions/string-array - minItems: 1 - maxItems: 32 - description: - A list of channels tied to this device, used for matching the - device to channels. - - required: - - compatible - - qcom,smd-channels - - additionalProperties: true - - unevaluatedProperties: false - required: - compatible diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml index 473adca4e973c99f67cbbf5b94eaf0416e48b0d4..48eda4d0d391d73de70e862cc4b07fa1108fad39 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml @@ -20,6 +20,7 @@ properties: compatible: enum: - qcom,rpmh-stats + - qcom,sdm845-rpmh-stats - qcom,rpm-stats # For older RPM firmware versions with fixed offset for the sleep stats - qcom,apq8084-rpm-stats diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml index ce2875c8932998bab4ae2270a9e8351ea2c03606..398663d21ab12fc4f08e24ceb05656d8d1874e9b 100644 --- a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml +++ b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml @@ -20,7 +20,7 @@ description: properties: compatible: enum: - - renesas,r9a07g043-sysc # RZ/G2UL + - renesas,r9a07g043-sysc # RZ/G2UL and RZ/Five - renesas,r9a07g044-sysc # RZ/G2{L,LC} - renesas,r9a07g054-sysc # RZ/V2L @@ -44,8 +44,6 @@ properties: required: - compatible - reg - - interrupts - - interrupt-names additionalProperties: false diff --git a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml index 75a2b8bb25fb606d81aeef2f7dcbddc9fc4a5f2f..2ed8cca79b59cb89dc5050ceb6897ac0dddbf2ea 100644 --- a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml +++ b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml @@ -16,9 +16,12 @@ properties: - enum: - rockchip,rk3288-sgrf - rockchip,rk3566-pipe-grf + - rockchip,rk3568-pcie3-phy-grf - rockchip,rk3568-pipe-grf - rockchip,rk3568-pipe-phy-grf - rockchip,rk3568-usb2phy-grf + - rockchip,rk3588-pcie3-phy-grf + - rockchip,rk3588-pcie3-pipe-grf - rockchip,rv1108-usbgrf - const: syscon - items: @@ -28,6 +31,7 @@ properties: - rockchip,px30-usb2phy-grf - rockchip,rk3036-grf - rockchip,rk3066-grf + - rockchip,rk3128-grf - rockchip,rk3188-grf - rockchip,rk3228-grf - rockchip,rk3288-grf @@ -45,6 +49,8 @@ properties: - rockchip,rk3568-pmugrf - rockchip,rv1108-grf - rockchip,rv1108-pmugrf + - rockchip,rv1126-grf + - rockchip,rv1126-pmugrf - const: syscon - const: simple-mfd @@ -178,6 +184,7 @@ allOf: contains: enum: - rockchip,px30-usb2phy-grf + - rockchip,rk3128-grf - rockchip,rk3228-grf - rockchip,rk3308-usb2phy-grf - rockchip,rk3328-usb2phy-grf diff --git a/Documentation/devicetree/bindings/sound/adi,max98396.yaml b/Documentation/devicetree/bindings/sound/adi,max98396.yaml index 8d2ef991db40fbf22c787df27e272c69c6bf4536..fd5aa61b467f813ed28775d3ebffe4cf3b6c7809 100644 --- a/Documentation/devicetree/bindings/sound/adi,max98396.yaml +++ b/Documentation/devicetree/bindings/sound/adi,max98396.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/adi,max98396.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices MAX98396 Speaker Amplifier Device Tree Bindings +title: Analog Devices MAX98396 Speaker Amplifier maintainers: - Ryan Lee @@ -78,6 +78,40 @@ properties: interleaved on a single output channel. type: boolean + adi,dmon-stuck-enable: + description: + Enables the "data monitor stuck" feature. Once the data monitor is + enabled, it actively monitors the selected input data (from DIN) to the + speaker amplifier. Once a data error is detected, the data monitor + automatically places the device into software shutdown. + type: boolean + + adi,dmon-stuck-threshold-bits: + description: + Sets the threshold for the "data monitor stuck" feature, in bits. + enum: [9, 11, 13, 15] + default: 15 + + adi,dmon-magnitude-enable: + description: + Enables the "data monitor magnitude" feature. Once the data monitor is + enabled, it actively monitors the selected input data (from DIN) to the + speaker amplifier. Once a data error is detected, the data monitor + automatically places the device into software shutdown. + type: boolean + + adi,dmon-magnitude-threshold-bits: + description: + Sets the threshold for the "data monitor magnitude" feature, in bits. + enum: [2, 3, 4, 5] + default: 5 + + adi,dmon-duration-ms: + description: + Sets the duration for the "data monitor" feature, in milliseconds. + enum: [64, 256, 1024, 4096] + default: 64 + reset-gpios: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/ak4375.yaml b/Documentation/devicetree/bindings/sound/ak4375.yaml index f1d5074a024d9c152061b5d8e6826f47e5f792c6..5f0fc584bb38dca5c63611ae7e81686c76977c10 100644 --- a/Documentation/devicetree/bindings/sound/ak4375.yaml +++ b/Documentation/devicetree/bindings/sound/ak4375.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/ak4375.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: AK4375 DAC and headphones amplifier Device Tree Bindings +title: AK4375 DAC and headphones amplifier maintainers: - Vincent Knecht diff --git a/Documentation/devicetree/bindings/sound/ak4613.yaml b/Documentation/devicetree/bindings/sound/ak4613.yaml index ef4055ef0ccdc6e1dc29407fe560d1626e738c37..aa8a258a9f1c9b04d924a32a81a840624ee79ffe 100644 --- a/Documentation/devicetree/bindings/sound/ak4613.yaml +++ b/Documentation/devicetree/bindings/sound/ak4613.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/ak4613.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: AK4613 I2C transmitter Device Tree Bindings +title: AK4613 I2C transmitter maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/ak4642.yaml b/Documentation/devicetree/bindings/sound/ak4642.yaml index 1e2caa29790edd62dbd472a0757a5588d7dbbb66..48a5b2c3934e84a3b016c250524e6b2419d63e26 100644 --- a/Documentation/devicetree/bindings/sound/ak4642.yaml +++ b/Documentation/devicetree/bindings/sound/ak4642.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/ak4642.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: AK4642 I2C transmitter Device Tree Bindings +title: AK4642 I2C transmitter maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml index 559aff13ae232c785f12fec2c3f84de75fa2a9fd..292fcb64399908ef3453fc8833ea3e093fde553d 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun4i-a10-codec.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Codec Device Tree Bindings +title: Allwinner A10 Codec maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-i2s.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-i2s.yaml index 34f6ee9de392250598d4f8f6e80677cc59cff395..dd30881ad2f59cb481f5b397b7c69c80a5597115 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-i2s.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-i2s.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun4i-a10-i2s.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 I2S Controller Device Tree Bindings +title: Allwinner A10 I2S Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-spdif.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-spdif.yaml index 444a432912bb4ec7970a2a61e26e33b2b09fac24..68c84e29ce5742b2072f4e91329a8d8ccf04b941 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-spdif.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-spdif.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun4i-a10-spdif.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 S/PDIF Controller Device Tree Bindings +title: Allwinner A10 S/PDIF Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun50i-a64-codec-analog.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun50i-a64-codec-analog.yaml index 66859eb8f79ab1df66b486579a2273d030f771f0..5800de63fc842d0f20aa0ba0591261e25751304f 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun50i-a64-codec-analog.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun50i-a64-codec-analog.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun50i-a64-codec-analog.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A64 Analog Codec Device Tree Bindings +title: Allwinner A64 Analog Codec maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f12cabe4c7181065b346527bacca9607375c9d5 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/allwinner,sun50i-h6-dmic.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Allwinner H6 DMIC + +maintainers: + - Ban Tao + +properties: + compatible: + const: allwinner,sun50i-h6-dmic + + "#sound-dai-cells": + const: 0 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: Bus Clock + - description: Module Clock + + clock-names: + items: + - const: bus + - const: mod + + dmas: + items: + - description: RX DMA Channel + + dma-names: + items: + - const: rx + + resets: + maxItems: 1 + +required: + - "#sound-dai-cells" + - compatible + - reg + - interrupts + - clocks + - clock-names + - dmas + - dma-names + - resets + +additionalProperties: false + +examples: + - | + #include + #include + + #include + #include + + dmic: dmic@5095000 { + #sound-dai-cells = <0>; + compatible = "allwinner,sun50i-h6-dmic"; + reg = <0x05095000 0x400>; + interrupts = ; + clocks = <&ccu CLK_BUS_DMIC>, <&ccu CLK_DMIC>; + clock-names = "bus", "mod"; + dmas = <&dma 7>; + dma-names = "rx"; + resets = <&ccu RST_BUS_DMIC>; + }; + +... diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun8i-a23-codec-analog.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun8i-a23-codec-analog.yaml index 26eca21e1f0f5daf8a6cb63016db7496251ded1e..1c21a1b390c7bcd1a504fe731f24fe24ff7990fb 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun8i-a23-codec-analog.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun8i-a23-codec-analog.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun8i-a23-codec-analog.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A23 Analog Codec Device Tree Bindings +title: Allwinner A23 Analog Codec maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun8i-a33-codec.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun8i-a33-codec.yaml index 19f111f402257208d48f7fccbe9ceae5ee11fd5d..4eb11a8e622bd85a4be68f22c1e5bd3f60faec9b 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun8i-a33-codec.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun8i-a33-codec.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/allwinner,sun8i-a33-codec.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A33 Codec Device Tree Bindings +title: Allwinner A33 Codec maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml index b4b35edcb493336c4aeaaf0d7dab1be9f22ccf6c..5b8d59245f82fbcd261c8827732c77a5079d0a63 100644 --- a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml +++ b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml @@ -40,6 +40,7 @@ properties: patternProperties: "^dai-link-[0-9]+$": type: object + additionalProperties: false description: |- dai-link child nodes: Container for dai-link level properties and the CODEC sub-nodes. @@ -63,6 +64,7 @@ patternProperties: patternProperties: "^codec-[0-9]+$": type: object + additionalProperties: false description: |- Codecs: dai-link representing backend links should have at least one subnode. diff --git a/Documentation/devicetree/bindings/sound/apple,mca.yaml b/Documentation/devicetree/bindings/sound/apple,mca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5dc92b5b6542ab30aec9605fbda6bb8d4606782 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,mca.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,mca.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple MCA I2S transceiver + +description: | + MCA is an I2S transceiver peripheral found on M1 and other Apple chips. It is + composed of a number of identical clusters which can operate independently + or in an interlinked fashion. Up to 6 clusters have been seen on an MCA. + +maintainers: + - Martin Povišer + +properties: + compatible: + items: + - enum: + - apple,t6000-mca + - apple,t8103-mca + - const: apple,mca + + reg: + items: + - description: Register region of the MCA clusters proper + - description: Register region of the DMA glue and its FIFOs + + interrupts: + minItems: 4 + maxItems: 6 + description: + One interrupt per each cluster + + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + + dmas: + minItems: 16 + maxItems: 24 + description: + DMA channels corresponding to the SERDES units in the peripheral. They are + listed in groups of four per cluster, and within the group they are given + as associated to the TXA, RXA, TXB, RXB units. + + dma-names: + minItems: 16 + items: + - const: tx0a + - const: rx0a + - const: tx0b + - const: rx0b + - const: tx1a + - const: rx1a + - const: tx1b + - const: rx1b + - const: tx2a + - const: rx2a + - const: tx2b + - const: rx2b + - const: tx3a + - const: rx3a + - const: tx3b + - const: rx3b + - const: tx4a + - const: rx4a + - const: tx4b + - const: rx4b + - const: tx5a + - const: rx5a + - const: tx5b + - const: rx5b + description: | + Names for the DMA channels: 'tx'/'rx', then cluster number, then 'a'/'b' + based on the associated SERDES unit. + + clocks: + minItems: 4 + maxItems: 6 + description: + Clusters' input reference clock. + + resets: + maxItems: 1 + + power-domains: + minItems: 5 + maxItems: 7 + description: + First a general power domain for register access, then the power + domains of individual clusters for their operation. + + '#sound-dai-cells': + const: 1 + +required: + - compatible + - reg + - dmas + - dma-names + - clocks + - power-domains + - '#sound-dai-cells' + +additionalProperties: false + +examples: + - | + mca: i2s@9b600000 { + compatible = "apple,t6000-mca", "apple,mca"; + reg = <0x9b600000 0x10000>, + <0x9b200000 0x20000>; + + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + + #sound-dai-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/sound/audio-graph-card.yaml b/Documentation/devicetree/bindings/sound/audio-graph-card.yaml index 109e55f9e5971b0c493fea0aaf9e1ab654a8319e..274092ef36c5462edca2ebb49761b4e1707ef46b 100644 --- a/Documentation/devicetree/bindings/sound/audio-graph-card.yaml +++ b/Documentation/devicetree/bindings/sound/audio-graph-card.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/audio-graph-card.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Audio Graph Card Device Tree Bindings +title: Audio Graph Card maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/audio-graph-card2.yaml b/Documentation/devicetree/bindings/sound/audio-graph-card2.yaml index 7416067c945e3d5c672e6b9fae7c74d204326986..3de7b36829da7606b2188a751b867bcbdf5c3286 100644 --- a/Documentation/devicetree/bindings/sound/audio-graph-card2.yaml +++ b/Documentation/devicetree/bindings/sound/audio-graph-card2.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/audio-graph-card2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Audio Graph Card2 Device Tree Bindings +title: Audio Graph Card2 maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml index 5c368674d11a66c54059bd01a104edae6db1f6a5..64654ceef20899d66bde9b27f120fd864d4013bc 100644 --- a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml +++ b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml @@ -19,14 +19,17 @@ properties: description: "device name prefix" $ref: /schemas/types.yaml#/definitions/string convert-rate: - description: CPU to Codec rate convert. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate" convert-channels: - description: CPU to Codec rate channels. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels" + convert-sample-format: + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format" + patternProperties: "^endpoint(@[0-9a-f]+)?": $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false + properties: mclk-fs: description: | @@ -65,12 +68,18 @@ patternProperties: - msb - lsb convert-rate: - description: CPU to Codec rate convert. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate" convert-channels: - description: CPU to Codec rate channels. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels" + convert-sample-format: + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format" + dai-tdm-slot-num: + description: Number of slots in use. + $ref: /schemas/types.yaml#/definitions/uint32 + dai-tdm-slot-width: + description: Width in bits for each slot. + $ref: /schemas/types.yaml#/definitions/uint32 dai-tdm-slot-width-map: description: Mapping of sample widths to slot widths. For hardware that cannot support a fixed slot width or a slot width always diff --git a/Documentation/devicetree/bindings/sound/audio-graph.yaml b/Documentation/devicetree/bindings/sound/audio-graph.yaml index 4b46794e5153895abb910bd4605f9b96294a1c2c..d59baedee180d43fc53ecb36053ef5e74a76c179 100644 --- a/Documentation/devicetree/bindings/sound/audio-graph.yaml +++ b/Documentation/devicetree/bindings/sound/audio-graph.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/audio-graph.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Audio Graph Device Tree Bindings +title: Audio Graph maintainers: - Kuninori Morimoto @@ -27,11 +27,12 @@ properties: description: User specified audio sound widgets. $ref: /schemas/types.yaml#/definitions/non-unique-string-array convert-rate: - description: CPU to Codec rate convert. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate" convert-channels: - description: CPU to Codec rate channels. - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels" + convert-sample-format: + $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format" + pa-gpios: maxItems: 1 hp-det-gpio: diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 31800f70e9d91a06065f7bb4e7657dda294f90df..7356084a2ca2b7ff67fee9002b5ded2d564f24a2 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -19,6 +19,7 @@ properties: compatible: enum: - cirrus,cs42l42 + - cirrus,cs42l83 reg: description: diff --git a/Documentation/devicetree/bindings/sound/dai-params.yaml b/Documentation/devicetree/bindings/sound/dai-params.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f5fb71f9b603f1e9e8c16a5e65111000fb21c64a --- /dev/null +++ b/Documentation/devicetree/bindings/sound/dai-params.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/dai-params.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Digital Audio Interface (DAI) Stream Parameters + +maintainers: + - Kuninori Morimoto + +select: false + +$defs: + + dai-channels: + description: Number of audio channels used by DAI + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 32 + + dai-sample-format: + description: Audio sample format used by DAI + $ref: /schemas/types.yaml#/definitions/string + enum: + - s8 + - s16_le + - s24_le + - s24_3le + - s32_le + + dai-sample-rate: + description: Audio sample rate used by DAI + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 8000 + maximum: 192000 + +properties: {} + +additionalProperties: true diff --git a/Documentation/devicetree/bindings/sound/everest,es8326.yaml b/Documentation/devicetree/bindings/sound/everest,es8326.yaml new file mode 100755 index 0000000000000000000000000000000000000000..07781408e78820ad49d093d4bddcfd14eeadfa39 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/everest,es8326.yaml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/everest,es8326.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Everest ES8326 audio CODEC + +maintainers: + - David Yang + +properties: + compatible: + const: everest,es8326 + + reg: + maxItems: 1 + + clocks: + items: + - description: clock for master clock (MCLK) + + clock-names: + items: + - const: mclk + + "#sound-dai-cells": + const: 0 + + everest,jack-pol: + $ref: /schemas/types.yaml#/definitions/uint8 + description: | + just the value of reg 57. Bit(3) decides whether the jack polarity is inverted. + Bit(2) decides whether the button on the headset is inverted. + Bit(1)/(0) decides the mic properity to be OMTP/CTIA or auto. + minimum: 0x00 + maximum: 0x0f + default: 0x0f + + everest,mic1-src: + $ref: /schemas/types.yaml#/definitions/uint8 + description: + the value of reg 2A when headset plugged. + minimum: 0x00 + maximum: 0x77 + default: 0x22 + + everest,mic2-src: + $ref: /schemas/types.yaml#/definitions/uint8 + description: + the value of reg 2A when headset unplugged. + minimum: 0x00 + maximum: 0x77 + default: 0x44 + + everest,jack-detect-inverted: + $ref: /schemas/types.yaml#/definitions/flag + description: + Defined to invert the jack detection. + + everest,interrupt-src: + $ref: /schemas/types.yaml#/definitions/uint8 + description: | + value of reg 0x58, Defines the interrupt source. + Bit(2) 1 means button press triggers irq, 0 means not. + Bit(3) 1 means PIN9 is the irq source for jack detection. When set to 0, + bias change on PIN9 do not triggers irq. + Bit(4) 1 means PIN27 is the irq source for jack detection. + Bit(5) 1 means PIN9 is the irq source after MIC detect. + Bit(6) 1 means PIN27 is the irq source after MIC detect. + minimum: 0 + maximum: 0x3c + default: 0x08 + + everest,interrupt-clk: + $ref: /schemas/types.yaml#/definitions/uint8 + description: | + value of reg 0x59, Defines the interrupt output behavior. + Bit(0-3) 0 means irq pulse equals 512*internal clock + 1 means irq pulse equals 1024*internal clock + 2 means ... + 7 means irq pulse equals 65536*internal clock + 8 means irq mutes PA + 9 means irq mutes PA and DAC output + Bit(4) 1 means we invert the interrupt output. + Bit(6) 1 means the chip do not detect jack type after button released. + 0 means the chip detect jack type again after button released. + minimum: 0 + maximum: 0x7f + default: 0x45 + +required: + - compatible + - reg + - "#sound-dai-cells" + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + es8326: codec@19 { + compatible = "everest,es8326"; + reg = <0x19>; + clocks = <&clks 10>; + clock-names = "mclk"; + #sound-dai-cells = <0>; + everest,mic1-src = [22]; + everest,mic2-src = [44]; + everest,jack-pol = [0e]; + everest,interrupt-src = [08]; + everest,interrupt-clk = [45]; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/fsl,sai.yaml b/Documentation/devicetree/bindings/sound/fsl,sai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70c4111d59c710087d02ade6882797089a059122 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/fsl,sai.yaml @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/fsl,sai.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Freescale Synchronous Audio Interface (SAI). + +maintainers: + - Shengjiu Wang + +description: | + The SAI is based on I2S module that used communicating with audio codecs, + which provides a synchronous audio interface that supports fullduplex + serial interfaces with frame synchronization such as I2S, AC97, TDM, and + codec/DSP interfaces. + +properties: + compatible: + oneOf: + - enum: + - fsl,vf610-sai + - fsl,imx6sx-sai + - fsl,imx6ul-sai + - fsl,imx7ulp-sai + - fsl,imx8mq-sai + - fsl,imx8qm-sai + - fsl,imx8ulp-sai + - items: + - enum: + - fsl,imx8mm-sai + - fsl,imx8mn-sai + - fsl,imx8mp-sai + - const: fsl,imx8mq-sai + + reg: + maxItems: 1 + + interrupts: + items: + - description: receive and transmit interrupt + + dmas: + maxItems: 2 + + dma-names: + maxItems: 2 + + clocks: + items: + - description: The ipg clock for register access + - description: master clock source 0 (obsoleted) + - description: master clock source 1 + - description: master clock source 2 + - description: master clock source 3 + - description: PLL clock source for 8kHz series + - description: PLL clock source for 11kHz series + minItems: 4 + + clock-names: + oneOf: + - items: + - const: bus + - const: mclk0 + - const: mclk1 + - const: mclk2 + - const: mclk3 + - const: pll8k + - const: pll11k + minItems: 4 + - items: + - const: bus + - const: mclk1 + - const: mclk2 + - const: mclk3 + - const: pll8k + - const: pll11k + minItems: 4 + + lsb-first: + description: | + Configures whether the LSB or the MSB is transmitted + first for the fifo data. If this property is absent, + the MSB is transmitted first as default, or the LSB + is transmitted first. + type: boolean + + big-endian: + description: | + required if all the SAI registers are big-endian rather than little-endian. + type: boolean + + fsl,sai-synchronous-rx: + description: | + SAI will work in the synchronous mode (sync Tx with Rx) which means + both the transmitter and the receiver will send and receive data by + following receiver's bit clocks and frame sync clocks. + type: boolean + + fsl,sai-asynchronous: + description: | + SAI will work in the asynchronous mode, which means both transmitter + and receiver will send and receive data by following their own bit clocks + and frame sync clocks separately. + If both fsl,sai-asynchronous and fsl,sai-synchronous-rx are absent, the + default synchronous mode (sync Rx with Tx) will be used, which means both + transmitter and receiver will send and receive data by following clocks + of transmitter. + type: boolean + + fsl,dataline: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + description: | + Configure the dataline. It has 3 value for each configuration + maxItems: 16 + items: + items: + - description: format Default(0), I2S(1) or PDM(2) + enum: [0, 1, 2] + - description: dataline mask for 'rx' + - description: dataline mask for 'tx' + + fsl,sai-mclk-direction-output: + description: SAI will output the SAI MCLK clock. + type: boolean + + fsl,shared-interrupt: + description: Interrupt is shared with other modules. + type: boolean + + "#sound-dai-cells": + const: 0 + description: optional, some dts node didn't add it. + +allOf: + - if: + properties: + compatible: + contains: + const: fsl,vf610-sai + then: + properties: + dmas: + items: + - description: DMA controller phandle and request line for TX + - description: DMA controller phandle and request line for RX + dma-names: + items: + - const: tx + - const: rx + else: + properties: + dmas: + items: + - description: DMA controller phandle and request line for RX + - description: DMA controller phandle and request line for TX + dma-names: + items: + - const: rx + - const: tx + - if: + required: + - fsl,sai-asynchronous + then: + properties: + fsl,sai-synchronous-rx: false + +required: + - compatible + - reg + - interrupts + - dmas + - dma-names + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + #include + sai2: sai@40031000 { + compatible = "fsl,vf610-sai"; + reg = <0x40031000 0x1000>; + interrupts = <86 IRQ_TYPE_LEVEL_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_sai2_1>; + clocks = <&clks VF610_CLK_PLATFORM_BUS>, + <&clks VF610_CLK_SAI2>, + <&clks 0>, <&clks 0>; + clock-names = "bus", "mclk1", "mclk2", "mclk3"; + dma-names = "tx", "rx"; + dmas = <&edma0 0 21>, + <&edma0 0 20>; + big-endian; + lsb-first; + }; + + - | + #include + #include + sai1: sai@30010000 { + compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai"; + reg = <0x30010000 0x10000>; + interrupts = ; + clocks = <&clk IMX8MM_CLK_SAI1_IPG>, + <&clk IMX8MM_CLK_DUMMY>, + <&clk IMX8MM_CLK_SAI1_ROOT>, + <&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>; + clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3"; + dmas = <&sdma2 0 2 0>, <&sdma2 1 2 0>; + dma-names = "rx", "tx"; + fsl,dataline = <1 0xff 0xff 2 0xff 0x11>; + #sound-dai-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/sound/fsl-sai.txt b/Documentation/devicetree/bindings/sound/fsl-sai.txt deleted file mode 100644 index fbdefc3fade79ed33f4a9ff2090a71f18774b408..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/fsl-sai.txt +++ /dev/null @@ -1,95 +0,0 @@ -Freescale Synchronous Audio Interface (SAI). - -The SAI is based on I2S module that used communicating with audio codecs, -which provides a synchronous audio interface that supports fullduplex -serial interfaces with frame synchronization such as I2S, AC97, TDM, and -codec/DSP interfaces. - -Required properties: - - - compatible : Compatible list, contains "fsl,vf610-sai", - "fsl,imx6sx-sai", "fsl,imx6ul-sai", - "fsl,imx7ulp-sai", "fsl,imx8mq-sai", - "fsl,imx8qm-sai", "fsl,imx8mm-sai", - "fsl,imx8mn-sai", "fsl,imx8mp-sai", or - "fsl,imx8ulp-sai". - - - reg : Offset and length of the register set for the device. - - - clocks : Must contain an entry for each entry in clock-names. - - - clock-names : Must include the "bus" for register access and - "mclk1", "mclk2", "mclk3" for bit clock and frame - clock providing. - "pll8k", "pll11k" are optional, they are the clock - source for root clock, one is for 8kHz series rates - another one is for 11kHz series rates. - - dmas : Generic dma devicetree binding as described in - Documentation/devicetree/bindings/dma/dma.txt. - - - dma-names : Two dmas have to be defined, "tx" and "rx". - - - pinctrl-names : Must contain a "default" entry. - - - pinctrl-NNN : One property must exist for each entry in - pinctrl-names. See ../pinctrl/pinctrl-bindings.txt - for details of the property values. - - - lsb-first : Configures whether the LSB or the MSB is transmitted - first for the fifo data. If this property is absent, - the MSB is transmitted first as default, or the LSB - is transmitted first. - - - fsl,sai-synchronous-rx: This is a boolean property. If present, indicating - that SAI will work in the synchronous mode (sync Tx - with Rx) which means both the transmitter and the - receiver will send and receive data by following - receiver's bit clocks and frame sync clocks. - - - fsl,sai-asynchronous: This is a boolean property. If present, indicating - that SAI will work in the asynchronous mode, which - means both transmitter and receiver will send and - receive data by following their own bit clocks and - frame sync clocks separately. - - - fsl,dataline : configure the dataline. it has 3 value for each configuration - first one means the type: I2S(1) or PDM(2) - second one is dataline mask for 'rx' - third one is dataline mask for 'tx'. - for example: fsl,dataline = <1 0xff 0xff 2 0xff 0x11>; - it means I2S type rx mask is 0xff, tx mask is 0xff, PDM type - rx mask is 0xff, tx mask is 0x11 (dataline 1 and 4 enabled). - -Optional properties: - - - big-endian : Boolean property, required if all the SAI - registers are big-endian rather than little-endian. - -Optional properties (for mx6ul): - - - fsl,sai-mclk-direction-output: This is a boolean property. If present, - indicates that SAI will output the SAI MCLK clock. - -Note: -- If both fsl,sai-asynchronous and fsl,sai-synchronous-rx are absent, the - default synchronous mode (sync Rx with Tx) will be used, which means both - transmitter and receiver will send and receive data by following clocks - of transmitter. -- fsl,sai-asynchronous and fsl,sai-synchronous-rx are exclusive. - -Example: -sai2: sai@40031000 { - compatible = "fsl,vf610-sai"; - reg = <0x40031000 0x1000>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_sai2_1>; - clocks = <&clks VF610_CLK_PLATFORM_BUS>, - <&clks VF610_CLK_SAI2>, - <&clks 0>, <&clks 0>; - clock-names = "bus", "mclk1", "mclk2", "mclk3"; - dma-names = "tx", "rx"; - dmas = <&edma0 0 VF610_EDMA_MUXID0_SAI2_TX>, - <&edma0 0 VF610_EDMA_MUXID0_SAI2_RX>; - big-endian; - lsb-first; -}; diff --git a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml index 233caa0ade87809940f615573642287ef74f333a..67ccddd44489b323be952be4da215dce8a9e5e51 100644 --- a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml +++ b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml @@ -61,6 +61,8 @@ patternProperties: cpu: description: Holds subnode which indicates cpu dai. type: object + additionalProperties: false + properties: sound-dai: maxItems: 1 @@ -68,6 +70,8 @@ patternProperties: codec: description: Holds subnode which indicates codec dai. type: object + additionalProperties: false + properties: sound-dai: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml index bb3a435722c72b4cd3d67c1efe49efcb16c0f14f..b6f5d486600ed4c22e062b5bee33986912cb3af7 100644 --- a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml +++ b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml @@ -58,6 +58,7 @@ patternProperties: cpu: description: Holds subnode which indicates cpu dai. type: object + additionalProperties: false properties: sound-dai: maxItems: 1 @@ -65,6 +66,7 @@ patternProperties: codec: description: Holds subnode which indicates codec dai. type: object + additionalProperties: false properties: sound-dai: minItems: 1 diff --git a/Documentation/devicetree/bindings/sound/intel,keembay-i2s.yaml b/Documentation/devicetree/bindings/sound/intel,keembay-i2s.yaml index 803627e984f6089438d95ca7751b78dda9779d7c..b2603f611af9051440598e4eaa2ebcc9d032c3e7 100644 --- a/Documentation/devicetree/bindings/sound/intel,keembay-i2s.yaml +++ b/Documentation/devicetree/bindings/sound/intel,keembay-i2s.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/sound/intel,keembay-i2s.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Intel KeemBay I2S Device Tree Bindings +title: Intel KeemBay I2S maintainers: - Sia, Jee Heng diff --git a/Documentation/devicetree/bindings/sound/linux,bt-sco.yaml b/Documentation/devicetree/bindings/sound/linux,bt-sco.yaml index e3a1f485f6642f92e60b74a29cd76804ce3da28b..b97e0fcbdba37acade2f2bf9857d00a951282a8b 100644 --- a/Documentation/devicetree/bindings/sound/linux,bt-sco.yaml +++ b/Documentation/devicetree/bindings/sound/linux,bt-sco.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/linux,bt-sco.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Bluetooth SCO Audio Codec Device Tree Bindings +title: Bluetooth SCO Audio Codec maintainers: - Mark Brown diff --git a/Documentation/devicetree/bindings/sound/linux,spdif-dit.yaml b/Documentation/devicetree/bindings/sound/linux,spdif-dit.yaml index a4f9257e313daff15bea7b32ea0bc54b2fb1cc23..808f6d2736c7812dda3473526c5fb3fc906edeb0 100644 --- a/Documentation/devicetree/bindings/sound/linux,spdif-dit.yaml +++ b/Documentation/devicetree/bindings/sound/linux,spdif-dit.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/linux,spdif-dit.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Dummy SPDIF Transmitter Device Tree Bindings +title: Dummy SPDIF Transmitter maintainers: - Mark Brown diff --git a/Documentation/devicetree/bindings/sound/mchp,spdifrx.yaml b/Documentation/devicetree/bindings/sound/mchp,spdifrx.yaml index 9703111432532d35f53c22ebe5283628ae293e99..70a47c6823b1adae6e82142d2676fda9feae5d47 100644 --- a/Documentation/devicetree/bindings/sound/mchp,spdifrx.yaml +++ b/Documentation/devicetree/bindings/sound/mchp,spdifrx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/mchp,spdifrx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip S/PDIF Rx Controller Device Tree Bindings +title: Microchip S/PDIF Rx Controller maintainers: - Codrin Ciubotariu diff --git a/Documentation/devicetree/bindings/sound/mchp,spdiftx.yaml b/Documentation/devicetree/bindings/sound/mchp,spdiftx.yaml index d5c022e495266b5638a9bbb1ab035e66ae4a7d70..d218e4ab9a7a491731e4cc185e91e961d64e08ef 100644 --- a/Documentation/devicetree/bindings/sound/mchp,spdiftx.yaml +++ b/Documentation/devicetree/bindings/sound/mchp,spdiftx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/mchp,spdiftx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip S/PDIF Tx Controller Device Tree Bindings +title: Microchip S/PDIF Tx Controller maintainers: - Codrin Ciubotariu diff --git a/Documentation/devicetree/bindings/sound/mt6359.yaml b/Documentation/devicetree/bindings/sound/mt6359.yaml index a54f466f769d49c6e56354f201650f82c493f69a..23d411fc4200e67b4ae9e2bb8761be8e221d2dbe 100644 --- a/Documentation/devicetree/bindings/sound/mt6359.yaml +++ b/Documentation/devicetree/bindings/sound/mt6359.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/mt6359.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek MT6359 Codec Device Tree Bindings +title: Mediatek MT6359 Codec maintainers: - Eason Yen diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml index 513cd28b202765107f38c90da377a6822f70593f..d427f7f623db5999cbcb541092a07a6cdcc3c3ef 100644 --- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml +++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml @@ -43,6 +43,16 @@ properties: required: - sound-dai + mediatek,adsp: + $ref: /schemas/types.yaml#/definitions/phandle + description: The phandle of MT8186 ADSP platform. + + mediatek,dai-link: + $ref: /schemas/types.yaml#/definitions/string-array + description: + A list of the desired dai-links in the sound card. Each entry is a + name defined in the machine driver. + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml index 059a7629b2d37b7e4c1a270f2d9355a2d58b2b5f..4fc5b045d3cf776d448a6798cbb99ed06cc1b52c 100644 --- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml +++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml @@ -43,6 +43,16 @@ properties: required: - sound-dai + mediatek,adsp: + $ref: /schemas/types.yaml#/definitions/phandle + description: The phandle of MT8186 ADSP platform. + + mediatek,dai-link: + $ref: /schemas/types.yaml#/definitions/string-array + description: + A list of the desired dai-links in the sound card. Each entry is a + name defined in the machine driver. + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml index 4fa179909c629028647e08dca7860d3e05289bbd..478be7e3fa296ab6281806b161820d4021dac6ed 100644 --- a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml +++ b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml @@ -30,6 +30,8 @@ properties: headset-codec: type: object + additionalProperties: false + properties: sound-dai: $ref: /schemas/types.yaml#/definitions/phandle @@ -38,6 +40,8 @@ properties: speaker-codecs: type: object + additionalProperties: false + properties: sound-dai: minItems: 1 diff --git a/Documentation/devicetree/bindings/sound/nau8825.txt b/Documentation/devicetree/bindings/sound/nau8825.txt index 388a7bc60b1f4d77c7e9dd2e4cf1792830070049..cb861aca8d40f716659a6b0ad5e3cb61520601da 100644 --- a/Documentation/devicetree/bindings/sound/nau8825.txt +++ b/Documentation/devicetree/bindings/sound/nau8825.txt @@ -71,6 +71,9 @@ Optional properties: - nuvoton,crosstalk-enable: make crosstalk function enable if set. + - nuvoton,adcout-drive-strong: make the drive strength of ADCOUT IO PIN strong if set. + Otherwise, the drive keeps normal strength. + - clocks: list of phandle and clock specifier pairs according to common clock bindings for the clocks described in clock-names - clock-names: should include "mclk" for the MCLK master clock diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra186-asrc.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra186-asrc.yaml index 520d0d063d1a6ef04b0b02c231f54a580a4c3919..d82415c212719efc488d9cecdd96f83d6b006cdf 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra186-asrc.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra186-asrc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra186-asrc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra186 ASRC Device Tree Bindings +title: Tegra186 ASRC description: | Asynchronous Sample Rate Converter (ASRC) converts the sampling frequency diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra186-dspk.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra186-dspk.yaml index 73b98b2f3543ad3d19def6ac16f8eb83d529ea08..3d538df878ead7c522e3141d9c122550a46cd56c 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra186-dspk.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra186-dspk.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra186-dspk.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra186 DSPK Controller Device Tree Bindings +title: Tegra186 DSPK Controller description: | The Digital Speaker Controller (DSPK) can be viewed as a Pulse diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-admaif.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-admaif.yaml index 372043edd98f43d193a4b8fbd30d283ea9872b56..15ab40aeab1e0ebbb0ede46257730147913d4d86 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-admaif.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-admaif.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-admaif.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 ADMAIF Device Tree Bindings +title: Tegra210 ADMAIF description: | ADMAIF is the interface between ADMA and AHUB. Each ADMA channel diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-adx.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-adx.yaml index 8d8dc7fb3f0c93a46540592b1bd4b1520b186462..ea0dc0ece1bca19ee8b9f304c00ac359ee930149 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-adx.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-adx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-adx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 ADX Device Tree Bindings +title: Tegra210 ADX description: | The Audio Demultiplexer (ADX) block takes an input stream with up to diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-ahub.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-ahub.yaml index 47b6e712e4fbcf4e5c2c39c2fb33b6b5d32bea43..89f7805de2741c3238f02a0ca7f0b6f61f55a927 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-ahub.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-ahub.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-ahub.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 AHUB Device Tree Bindings +title: Tegra210 AHUB description: | The Audio Hub (AHUB) comprises a collection of hardware accelerators diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-amx.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-amx.yaml index f9e4fc6e0c472acfe82298a3fc8ef1e4ec2381c7..1aff61f072bbf2eb254febc68f42e0926352e488 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-amx.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-amx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-amx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 AMX Device Tree Bindings +title: Tegra210 AMX description: | The Audio Multiplexer (AMX) block can multiplex up to four input streams diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-dmic.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-dmic.yaml index bcb496d3ace5a5723b17278b738784b31022d5b4..0f9d2b461e02012277b1b05a91718cbcd982bdbb 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-dmic.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-dmic.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-dmic.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 DMIC Controller Device Tree Bindings +title: Tegra210 DMIC Controller description: | The Digital MIC (DMIC) Controller is used to interface with Pulse diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-i2s.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-i2s.yaml index 6188f561f878ad73706b601b3ab652e6c3991204..12cd17eede9977577772ea9399ea9f8c6871d9fd 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-i2s.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-i2s.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-i2s.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 I2S Controller Device Tree Bindings +title: Tegra210 I2S Controller description: | The Inter-IC Sound (I2S) controller implements full-duplex, diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-mixer.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-mixer.yaml index ee1e1d2da79a0462b467b169f2cb4673c6bbc0da..570b03282aeb1c6d78d36fc55af53debb5988e14 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-mixer.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-mixer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-mixer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 Mixer Device Tree Bindings +title: Tegra210 Mixer description: | The Mixer supports mixing of up to ten 7.1 audio input streams and diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-mvc.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-mvc.yaml index c9888c553e78119464b598c541106759ee33c99e..4aecbc847b989028411c8bdff0168635760bb790 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-mvc.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-mvc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-mvc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 MVC Device Tree Bindings +title: Tegra210 MVC description: | The Master Volume Control (MVC) provides gain or attenuation to a digital diff --git a/Documentation/devicetree/bindings/sound/nvidia,tegra210-sfc.yaml b/Documentation/devicetree/bindings/sound/nvidia,tegra210-sfc.yaml index 8579306fc56fc2c6cfb3c109ac421402d3b46b15..694f890d630594edfbf3c3eb70b336ceafe79787 100644 --- a/Documentation/devicetree/bindings/sound/nvidia,tegra210-sfc.yaml +++ b/Documentation/devicetree/bindings/sound/nvidia,tegra210-sfc.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/nvidia,tegra210-sfc.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Tegra210 SFC Device Tree Bindings +title: Tegra210 SFC description: | The Sampling Frequency Converter (SFC) converts the sampling frequency diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml index a6905bcf89d2daa54413873bfee93b95d11d134c..1de11e7f33bbc5345835c5f96f28b0e40a8dc17d 100644 --- a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml @@ -14,6 +14,8 @@ properties: enum: - qcom,sc7280-lpass-rx-macro - qcom,sm8250-lpass-rx-macro + - qcom,sm8450-lpass-rx-macro + - qcom,sc8280xp-lpass-rx-macro reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml index 324595a62ae87966e7fe9204f5aec835200c250b..de8297b358e891a8a816d0806f0e044b6b6443a9 100644 --- a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml @@ -14,6 +14,8 @@ properties: enum: - qcom,sc7280-lpass-tx-macro - qcom,sm8250-lpass-tx-macro + - qcom,sm8450-lpass-tx-macro + - qcom,sc8280xp-lpass-tx-macro reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml index 7b4cc84eda8c4ae678d0f591db7f5014a2749980..9f473c08cb2ef43fec110571320978f5df2ffe3d 100644 --- a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml @@ -14,6 +14,8 @@ properties: enum: - qcom,sc7280-lpass-va-macro - qcom,sm8250-lpass-va-macro + - qcom,sm8450-lpass-va-macro + - qcom,sc8280xp-lpass-va-macro reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml index 13cdb8a106877d7e3d610e8ace88d9a5847275ce..4959ad658eac3b05acdb0c3345f8a8952ab2833b 100644 --- a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml @@ -14,6 +14,8 @@ properties: enum: - qcom,sc7280-lpass-wsa-macro - qcom,sm8250-lpass-wsa-macro + - qcom,sm8450-lpass-wsa-macro + - qcom,sc8280xp-lpass-wsa-macro reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml b/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0f7a79e240a3f073fdf9d1033d6506266cb59d1 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/qcom,q6adm-routing.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Audio Device Manager (Q6ADM) routing + +maintainers: + - Krzysztof Kozlowski + - Srinivas Kandagatla + +description: + Qualcomm Audio Device Manager (Q6ADM) routing node represents routing + specific configuration. + +properties: + compatible: + enum: + - qcom,q6adm-routing + + "#sound-dai-cells": + const: 0 + +required: + - compatible + - "#sound-dai-cells" + +additionalProperties: false + +examples: + - | + #include + #include + + apr { + compatible = "qcom,apr-v2"; + qcom,domain = ; + #address-cells = <1>; + #size-cells = <0>; + + service@8 { + compatible = "qcom,q6adm"; + reg = ; + qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd"; + + routing { + compatible = "qcom,q6adm-routing"; + #sound-dai-cells = <0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6adm.txt b/Documentation/devicetree/bindings/sound/qcom,q6adm.txt deleted file mode 100644 index 15c353a20de8b8b9d963d57bd4c225057f69242d..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/qcom,q6adm.txt +++ /dev/null @@ -1,39 +0,0 @@ -Qualcomm Audio Device Manager (Q6ADM) binding - -Q6ADM is one of the APR audio service on Q6DSP. -Please refer to qcom,apr.txt for details of the coommon apr service bindings -used by the apr service device. - -- but must contain the following property: - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6adm-v.". - Or "qcom,q6adm" where the version number can be queried - from DSP. - example "qcom,q6adm-v2.0" - - -= ADM routing -"routing" subnode of the ADM node represents adm routing specific configuration - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6adm-routing". - -- #sound-dai-cells - Usage: required - Value type: - Definition: Must be 0 - -= EXAMPLE -apr-service@8 { - compatible = "qcom,q6adm"; - reg = ; - q6routing: routing { - compatible = "qcom,q6adm-routing"; - #sound-dai-cells = <0>; - }; -}; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6afe.txt b/Documentation/devicetree/bindings/sound/qcom,q6afe.txt deleted file mode 100644 index bc6b5f1fe4f199a9a8876d1854fcbbaf9efcaa9b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/qcom,q6afe.txt +++ /dev/null @@ -1,20 +0,0 @@ -Qualcomm Audio Front End (Q6AFE) binding - -AFE is one of the APR audio service on Q6DSP -Please refer to qcom,apr.txt for details of the common apr service bindings -used by all apr services. Must contain the following properties. - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6afe-v." - Or "qcom,q6afe" where the version number can be queried - from DSP. - example "qcom,q6afe" - -= EXAMPLE - -apr-service@4 { - compatible = "qcom,q6afe"; - reg = ; -}; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml b/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml index 5d972784321d72ceecb01f3b55786e751e3f2fb2..24f7bf2bfd95572d6b3ea11e48276d0cdac62e1f 100644 --- a/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml @@ -16,16 +16,12 @@ properties: compatible: const: qcom,q6apm-dais - reg: - maxItems: 1 - iommus: maxItems: 1 required: - compatible - iommus - - reg additionalProperties: false @@ -37,17 +33,14 @@ examples: #address-cells = <1>; #size-cells = <0>; qcom,domain = ; - service@1 { - compatible = "qcom,q6apm"; - reg = <1>; - #address-cells = <1>; - #size-cells = <0>; - - apm-dai@1 { - compatible = "qcom,q6apm-dais"; - iommus = <&apps_smmu 0x1801 0x0>; + service@1 { + compatible = "qcom,q6apm"; reg = <1>; - }; + + dais { + compatible = "qcom,q6apm-dais"; + iommus = <&apps_smmu 0x1801 0x0>; + }; }; }; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8deb8ffb143b2c8b16da05f4502b0f17718e3f96 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/qcom,q6asm-dais.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Audio Stream Manager (Q6ASM) + +maintainers: + - Krzysztof Kozlowski + - Srinivas Kandagatla + +description: + Q6ASM is one of the APR audio services on Q6DSP. Each of its subnodes + represent a dai with board specific configuration. + +properties: + compatible: + enum: + - qcom,q6asm-dais + + iommus: + maxItems: 1 + + "#sound-dai-cells": + const: 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + "^dai@[0-9]+$": + type: object + description: + Q6ASM Digital Audio Interface + + properties: + reg: + maxItems: 1 + + direction: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2] + description: | + The direction of the dai stream:: + - Q6ASM_DAI_TX_RX (0) for both tx and rx + - Q6ASM_DAI_TX (1) for only tx (Capture/Encode) + - Q6ASM_DAI_RX (2) for only rx (Playback/Decode) + + is-compress-dai: + type: boolean + description: + Compress offload dai. + + dependencies: + is-compress-dai: ["direction"] + + required: + - reg + + additionalProperties: false + +required: + - compatible + - "#sound-dai-cells" + - "#address-cells" + - "#size-cells" + +additionalProperties: false + +examples: + - | + #include + #include + + apr { + compatible = "qcom,apr-v2"; + qcom,domain = ; + #address-cells = <1>; + #size-cells = <0>; + + service@7 { + compatible = "qcom,q6asm"; + reg = ; + qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd"; + + dais { + compatible = "qcom,q6asm-dais"; + iommus = <&apps_smmu 0x1821 0x0>; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <1>; + + dai@0 { + reg = <0>; + }; + + dai@1 { + reg = <1>; + }; + + dai@2 { + reg = <2>; + is-compress-dai; + direction = <1>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm.txt b/Documentation/devicetree/bindings/sound/qcom,q6asm.txt deleted file mode 100644 index 0d0075125243ec22c201cb6c7c6e0206d6d0e9f6..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/qcom,q6asm.txt +++ /dev/null @@ -1,70 +0,0 @@ -Qualcomm Audio Stream Manager (Q6ASM) binding - -Q6ASM is one of the APR audio service on Q6DSP. -Please refer to qcom,apr.txt for details of the common apr service bindings -used by the apr service device. - -- but must contain the following property: - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6asm-v.". - Or "qcom,q6asm" where the version number can be queried - from DSP. - example "qcom,q6asm-v2.0" - -= ASM DAIs (Digital Audio Interface) -"dais" subnode of the ASM node represents dai specific configuration - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6asm-dais". - -- #sound-dai-cells - Usage: required - Value type: - Definition: Must be 1 - -== ASM DAI is subnode of "dais" and represent a dai, it includes board specific -configuration of each dai. Must contain the following properties. - -- reg - Usage: required - Value type: - Definition: Must be dai id - -- direction: - Usage: Required for Compress offload dais - Value type: - Definition: Specifies the direction of the dai stream - Q6ASM_DAI_TX_RX (0) for both tx and rx - Q6ASM_DAI_TX (1) for only tx (Capture/Encode) - Q6ASM_DAI_RX (2) for only rx (Playback/Decode) - -- is-compress-dai: - Usage: Required for Compress offload dais - Value type: - Definition: present for Compress offload dais - - -= EXAMPLE -#include - -apr-service@7 { - compatible = "qcom,q6asm"; - reg = ; - q6asmdai: dais { - compatible = "qcom,q6asm-dais"; - #address-cells = <1>; - #size-cells = <0>; - #sound-dai-cells = <1>; - - dai@0 { - reg = <0>; - direction = ; - is-compress-dai; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6core.txt b/Documentation/devicetree/bindings/sound/qcom,q6core.txt deleted file mode 100644 index 5cd4cc9b1fde889bfad03045867d3281f372fd20..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/qcom,q6core.txt +++ /dev/null @@ -1,21 +0,0 @@ -Qualcomm ADSP Core service binding - -Q6CORE is one of the APR audio service on Q6DSP. -Please refer to qcom,apr.txt for details of the common apr service bindings -used by the apr service device. - -- but must contain the following property: - -- compatible: - Usage: required - Value type: - Definition: must be "qcom,q6core-v.". - Or "qcom,q6core" where the version number can be queried - from DSP. - example "qcom,q6core-v2.0" - -= EXAMPLE -apr-service@3 { - compatible = "qcom,q6core"; - reg = ; -}; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml index f83f00737a2f0dec50874e2df87c503b2b4d32ed..fd567d20417dd9e38fefe69ff3b93a46142d1a7a 100644 --- a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml @@ -18,9 +18,6 @@ properties: - qcom,q6afe-clocks - qcom,q6prm-lpass-clocks - reg: - maxItems: 1 - '#clock-cells': const: 2 description: @@ -32,7 +29,6 @@ properties: required: - compatible - - reg - "#clock-cells" additionalProperties: false @@ -42,19 +38,22 @@ examples: #include #include apr { + compatible = "qcom,apr-v2"; + qcom,domain = ; #address-cells = <1>; #size-cells = <0>; - apr-service@4 { + + service@4 { + compatible = "qcom,q6afe"; reg = ; - #address-cells = <1>; - #size-cells = <0>; - clock-controller@2 { - compatible = "qcom,q6afe-clocks"; - reg = <2>; - #clock-cells = <2>; + qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd"; + + clock-controller { + compatible = "qcom,q6afe-clocks"; + #clock-cells = <2>; }; }; - }; + }; - | #include @@ -63,15 +62,14 @@ examples: qcom,domain = ; #address-cells = <1>; #size-cells = <0>; + service@2 { reg = ; compatible = "qcom,q6prm"; - #address-cells = <1>; - #size-cells = <0>; - clock-controller@2 { - compatible = "qcom,q6prm-lpass-clocks"; - reg = <2>; - #clock-cells = <2>; + + clock-controller { + compatible = "qcom,q6prm-lpass-clocks"; + #clock-cells = <2>; }; }; - }; + }; diff --git a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml index dc7fba7b92d501524010ac84293f2c6a49e795c2..e53fc0960a14c882d8122f1dadf1a9cdd6146b13 100644 --- a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml @@ -18,9 +18,6 @@ properties: - qcom,q6afe-dais - qcom,q6apm-lpass-dais - reg: - maxItems: 1 - '#sound-dai-cells': const: 1 @@ -145,7 +142,6 @@ patternProperties: required: - compatible - - reg - "#sound-dai-cells" - "#address-cells" - "#size-cells" @@ -157,26 +153,29 @@ examples: #include #include apr { + compatible = "qcom,apr-v2"; #address-cells = <1>; #size-cells = <0>; - apr-service@4 { + qcom,domain = ; + + service@4 { + compatible = "qcom,q6afe"; reg = ; - #address-cells = <1>; - #size-cells = <0>; - q6afedai@1 { - compatible = "qcom,q6afe-dais"; - reg = <1>; - #address-cells = <1>; - #size-cells = <0>; - #sound-dai-cells = <1>; - - dai@22 { - reg = ; - qcom,sd-lines = <0 1 2 3>; - }; + qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd"; + + dais { + compatible = "qcom,q6afe-dais"; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <1>; + + dai@22 { + reg = ; + qcom,sd-lines = <0 1 2 3>; + }; }; }; - }; + }; - | #include gpr { @@ -184,22 +183,21 @@ examples: #address-cells = <1>; #size-cells = <0>; qcom,domain = ; + service@1 { compatible = "qcom,q6apm"; reg = ; - #address-cells = <1>; - #size-cells = <0>; - q6apmdai@1 { - compatible = "qcom,q6apm-lpass-dais"; - reg = <1>; - #address-cells = <1>; - #size-cells = <0>; - #sound-dai-cells = <1>; - - dai@22 { - reg = ; - qcom,sd-lines = <0 1 2 3>; - }; + + dais { + compatible = "qcom,q6apm-lpass-dais"; + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <1>; + + dai@22 { + reg = ; + qcom,sd-lines = <0 1 2 3>; + }; }; }; - }; + }; diff --git a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml index e6e27d09783eca2f8a1f7410c421abb72c1c20cd..70080d04ddc93345ee9b90143a40508a100478d4 100644 --- a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml +++ b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml @@ -20,9 +20,11 @@ properties: - qcom,apq8016-sbc-sndcard - qcom,db845c-sndcard - qcom,msm8916-qdsp6-sndcard + - qcom,qrb5165-rb5-sndcard + - qcom,sc8280xp-sndcard - qcom,sdm845-sndcard - qcom,sm8250-sndcard - - qcom,qrb5165-rb5-sndcard + - qcom,sm8450-sndcard audio-routing: $ref: /schemas/types.yaml#/definitions/non-unique-string-array @@ -71,6 +73,8 @@ patternProperties: cpu: description: Holds subnode which indicates cpu dai. type: object + additionalProperties: false + properties: sound-dai: maxItems: 1 @@ -78,6 +82,8 @@ patternProperties: platform: description: Holds subnode which indicates platform dai. type: object + additionalProperties: false + properties: sound-dai: maxItems: 1 @@ -85,6 +91,8 @@ patternProperties: codec: description: Holds subnode which indicates codec dai. type: object + additionalProperties: false + properties: sound-dai: minItems: 1 diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml index e17c0245f77ab3faeb249b35483447427e2e7de8..679a246dd666d3f8a2da8906f2e2a989d3692b12 100644 --- a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml +++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/renesas,rsnd.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Renesas R-Car Sound Driver Device Tree Bindings +title: Renesas R-Car Sound Driver maintainers: - Kuninori Morimoto @@ -129,6 +129,8 @@ properties: patternProperties: "^dvc-[0-1]$": type: object + additionalProperties: false + properties: dmas: maxItems: 1 @@ -145,7 +147,7 @@ properties: patternProperties: "^mix-[0-1]$": type: object - # no properties + additionalProperties: false additionalProperties: false rcar_sound,ctu: @@ -154,7 +156,7 @@ properties: patternProperties: "^ctu-[0-7]$": type: object - # no properties + additionalProperties: false additionalProperties: false rcar_sound,src: @@ -163,6 +165,8 @@ properties: patternProperties: "^src-[0-9]$": type: object + additionalProperties: false + properties: interrupts: maxItems: 1 @@ -186,6 +190,8 @@ properties: patternProperties: "^ssiu-[0-9]+$": type: object + additionalProperties: false + properties: dmas: maxItems: 2 @@ -206,6 +212,8 @@ properties: patternProperties: "^ssi-[0-9]$": type: object + additionalProperties: false + properties: interrupts: maxItems: 1 @@ -243,6 +251,8 @@ properties: patternProperties: "^dai([0-9]+)?$": type: object + additionalProperties: false + properties: playback: $ref: /schemas/types.yaml#/definitions/phandle-array diff --git a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml index a01c4ad929b8e198882ad99ba23d027df934a239..447e013f6e1718cbbee8f914d173aed62c49b75b 100644 --- a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml @@ -23,6 +23,7 @@ properties: cpu: type: object + additionalProperties: false properties: sound-dai: minItems: 2 @@ -34,6 +35,7 @@ properties: - sound-dai codec: + additionalProperties: false type: object properties: sound-dai: diff --git a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml index ec50bcb4af5fc53986c28823da250c94d83020d9..31095913e330dd1c00f34118fc80bd37902db22a 100644 --- a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml @@ -19,6 +19,7 @@ properties: cpu: type: object + additionalProperties: false properties: sound-dai: maxItems: 1 @@ -28,6 +29,7 @@ properties: codec: type: object + additionalProperties: false properties: sound-dai: maxItems: 1 diff --git a/Documentation/devicetree/bindings/sound/samsung,snow.yaml b/Documentation/devicetree/bindings/sound/samsung,snow.yaml index 51a83d3c72742e6e57e9474c53411b00d56ee8cb..3d49aa4c9be29be9b829ed3410988eacf81b257b 100644 --- a/Documentation/devicetree/bindings/sound/samsung,snow.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,snow.yaml @@ -19,6 +19,7 @@ properties: codec: type: object + additionalProperties: false properties: sound-dai: description: List of phandles to the CODEC and HDMI IP nodes. @@ -30,6 +31,7 @@ properties: cpu: type: object + additionalProperties: false properties: sound-dai: description: Phandle to the Samsung I2S controller. diff --git a/Documentation/devicetree/bindings/sound/simple-audio-amplifier.yaml b/Documentation/devicetree/bindings/sound/simple-audio-amplifier.yaml index 8327846356d309f130e226808b8442951733261c..5428ba9e23a60696dc5076163dd7f7309655fc67 100644 --- a/Documentation/devicetree/bindings/sound/simple-audio-amplifier.yaml +++ b/Documentation/devicetree/bindings/sound/simple-audio-amplifier.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/simple-audio-amplifier.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Simple Audio Amplifier Device Tree Bindings +title: Simple Audio Amplifier maintainers: - Jerome Brunet diff --git a/Documentation/devicetree/bindings/sound/simple-card.yaml b/Documentation/devicetree/bindings/sound/simple-card.yaml index b261d49b9ddb9d59ba8d71aad984a4b4b99a8c68..ed19899bc94bacc108c8b764ff2a99162eb9c46c 100644 --- a/Documentation/devicetree/bindings/sound/simple-card.yaml +++ b/Documentation/devicetree/bindings/sound/simple-card.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/simple-card.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Simple Audio Card Driver Device Tree Bindings +title: Simple Audio Card Driver maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/sound-dai.yaml b/Documentation/devicetree/bindings/sound/sound-dai.yaml index 61c6f7abc4e70ad05c6d2e96f388495492d15c44..ff9036e43c441b31e16d340470cce199a9840107 100644 --- a/Documentation/devicetree/bindings/sound/sound-dai.yaml +++ b/Documentation/devicetree/bindings/sound/sound-dai.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/sound-dai.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Digital Audio Interface consumer Device Tree Bindings +title: Digital Audio Interface consumer maintainers: - Rob Herring diff --git a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml index fe2e15504ebc41119feefe1d57ccc743e7a78f71..56d206f97a96cc0a33e17dd3d5417c959d226838 100644 --- a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml +++ b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml @@ -60,6 +60,7 @@ required: patternProperties: "^audio-controller@[0-9a-f]+$": type: object + additionalProperties: false description: Two subnodes corresponding to SAI sub-block instances A et B can be defined. Subnode can be omitted for unsused sub-block. @@ -121,6 +122,10 @@ patternProperties: description: Configure the SAI device as master clock provider. const: 0 + port: + $ref: audio-graph-port.yaml# + unevaluatedProperties: false + required: - compatible - "#sound-dai-cells" diff --git a/Documentation/devicetree/bindings/sound/test-component.yaml b/Documentation/devicetree/bindings/sound/test-component.yaml index 17fdb43172391890941e85a346e233967354fd5c..9c40a2122dfdc74b130ab52c08b7f0d4aad3840e 100644 --- a/Documentation/devicetree/bindings/sound/test-component.yaml +++ b/Documentation/devicetree/bindings/sound/test-component.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/test-component.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Test Component Device Tree Bindings +title: Test Component maintainers: - Kuninori Morimoto diff --git a/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml b/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9681b72b491858cbab0eaa43d609a058b36c5db3 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/ti,src4xxx.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments SRC4392 Device Tree Bindings + +description: | + The SRC4392 is a digital audio codec that can be connected via + I2C or SPI. Currently, only I2C bus is supported. + +maintainers: + - Matt Flax + +allOf: + - $ref: name-prefix.yaml# + +properties: + compatible: + const: ti,src4392 + + "#sound-dai-cells": + const: 0 + + reg: + maxItems: 1 + +required: + - "#sound-dai-cells" + - compatible + - reg + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + audio-codec@70 { + #sound-dai-cells = <0>; + compatible = "ti,src4392"; + reg = <0x70>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml b/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml new file mode 100644 index 0000000000000000000000000000000000000000..785930658029531c4abcdc975e112f818bf6a52e --- /dev/null +++ b/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/ti,ts3a227e.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments TS3A227E + Autonomous Audio Accessory Detection and Configuration Switch + +maintainers: + - Dylan Reid + +description: | + The TS3A227E detect headsets of 3-ring and 4-ring standards and + switches automatically to route the microphone correctly. It also + handles key press detection in accordance with the Android audio + headset specification v1.0. + +properties: + compatible: + enum: + - ti,ts3a227e + + reg: + const: 0x3b + + interrupts: + maxItems: 1 + + ti,micbias: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Intended MICBIAS voltage (datasheet section 9.6.7). + enum: + - 0 # 2.1 V + - 1 # 2.2 V + - 2 # 2.3 V + - 3 # 2.4 V + - 4 # 2.5 V + - 5 # 2.6 V + - 6 # 2.7 V + - 7 # 2.8 V + default: 1 + + ti,debounce-release-ms: + description: key release debounce time in ms (datasheet section 9.6.7). + enum: + - 0 + - 20 + default: 20 + + ti,debounce-press-ms: + description: key press debounce time in ms (datasheet section 9.6.7). + enum: + - 2 + - 40 + - 80 + - 120 + default: 80 + + ti,debounce-insertion-ms: + description: headset insertion debounce time in ms (datasheet section 9.6.5). + enum: + - 2 + - 30 + - 60 + - 90 + - 120 + - 150 + - 1000 + - 2000 + default: 90 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + codec: audio-controller@3b { + compatible = "ti,ts3a227e"; + reg = <0x3b>; + interrupt-parent = <&gpio1>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/sound/ts3a227e.txt b/Documentation/devicetree/bindings/sound/ts3a227e.txt deleted file mode 100644 index 21ab45bc7e8f5b8db407a7518eaa49a017d45fda..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/sound/ts3a227e.txt +++ /dev/null @@ -1,30 +0,0 @@ -Texas Instruments TS3A227E -Autonomous Audio Accessory Detection and Configuration Switch - -The TS3A227E detect headsets of 3-ring and 4-ring standards and -switches automatically to route the microphone correctly. It also -handles key press detection in accordance with the Android audio -headset specification v1.0. - -Required properties: - - - compatible: Should contain "ti,ts3a227e". - - reg: The i2c address. Should contain <0x3b>. - - interrupts: Interrupt number for /INT pin from the 227e - -Optional properies: - - ti,micbias: Intended MICBIAS voltage (datasheet section 9.6.7). - Select 0/1/2/3/4/5/6/7 to specify MICBIAS voltage - 2.1V/2.2V/2.3V/2.4V/2.5V/2.6V/2.7V/2.8V - Default value is "1" (2.2V). - -Examples: - - i2c { - ts3a227e@3b { - compatible = "ti,ts3a227e"; - reg = <0x3b>; - interrupt-parent = <&gpio>; - interrupts = <3 IRQ_TYPE_LEVEL_LOW>; - }; - }; diff --git a/Documentation/devicetree/bindings/sound/wlf,wm8940.yaml b/Documentation/devicetree/bindings/sound/wlf,wm8940.yaml index 8aadcbeed5023b2f67612f5e75722de343d2070c..7386abb3a250b46fc8a572dba9f46920161ad570 100644 --- a/Documentation/devicetree/bindings/sound/wlf,wm8940.yaml +++ b/Documentation/devicetree/bindings/sound/wlf,wm8940.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/wlf,wm8940.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Wolfson WM8940 Codec Device Tree Bindings +title: Wolfson WM8940 Codec maintainers: - patches@opensource.cirrus.com diff --git a/Documentation/devicetree/bindings/sound/wlf,wm8978.yaml b/Documentation/devicetree/bindings/sound/wlf,wm8978.yaml index 96cf9fc9c8b0098bc3b7e22b4a06ad20a8dce973..1c8985d4dd5ab2b03caae9b54826f4ae47a6cac5 100644 --- a/Documentation/devicetree/bindings/sound/wlf,wm8978.yaml +++ b/Documentation/devicetree/bindings/sound/wlf,wm8978.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sound/wlf,wm8978.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Wolfson WM8978 Codec Device Tree Bindings +title: Wolfson WM8978 Codec maintainers: - patches@opensource.cirrus.com diff --git a/Documentation/devicetree/bindings/spi/allwinner,sun4i-a10-spi.yaml b/Documentation/devicetree/bindings/spi/allwinner,sun4i-a10-spi.yaml index 8036499112f5a858e00f8edf0fa425eacf7db76f..f1176a28fd87e98c6b1f11fa945dd7de2078cb34 100644 --- a/Documentation/devicetree/bindings/spi/allwinner,sun4i-a10-spi.yaml +++ b/Documentation/devicetree/bindings/spi/allwinner,sun4i-a10-spi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/allwinner,sun4i-a10-spi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 SPI Controller Device Tree Bindings +title: Allwinner A10 SPI Controller allOf: - $ref: "spi-controller.yaml" diff --git a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml index ca4c95345a49b24311802097741219d146dc80a4..58b7056f4a70970e832786f7df256b8e1f279463 100644 --- a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml +++ b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/allwinner,sun6i-a31-spi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A31 SPI Controller Device Tree Bindings +title: Allwinner A31 SPI Controller allOf: - $ref: "spi-controller.yaml" diff --git a/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml b/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml index 50de0da42c138ea650e2f8d40a46315e3319081f..0c10f7678178ae0c7a4ffdc4cb2d2ae99118298d 100644 --- a/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml +++ b/Documentation/devicetree/bindings/spi/amlogic,meson-gx-spicc.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson SPI Communication Controller maintainers: - - Neil Armstrong + - Neil Armstrong allOf: - $ref: "spi-controller.yaml#" diff --git a/Documentation/devicetree/bindings/spi/amlogic,meson6-spifc.yaml b/Documentation/devicetree/bindings/spi/amlogic,meson6-spifc.yaml index 8a9d526d06ebf393840a3546bbe661677b8c139c..ac3b2ec300acf0d502c14b2be5193fe281e7f417 100644 --- a/Documentation/devicetree/bindings/spi/amlogic,meson6-spifc.yaml +++ b/Documentation/devicetree/bindings/spi/amlogic,meson6-spifc.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson SPI Flash Controller maintainers: - - Neil Armstrong + - Neil Armstrong allOf: - $ref: "spi-controller.yaml#" diff --git a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml index d85d54024b2ee1d5e48b75d5e31b651594690812..4dd973e341e6c772a2f6f1d945ce31eb6aab325e 100644 --- a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml +++ b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml @@ -34,6 +34,16 @@ properties: clocks: maxItems: 1 + dmas: + items: + - description: TX DMA Channel + - description: RX DMA Channel + + dma-names: + items: + - const: tx + - const: rx + atmel,fifo-size: $ref: /schemas/types.yaml#/definitions/uint32 description: | diff --git a/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml b/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml index 970b1119898be27567ca8aae6d9f10f237c7794e..a453996c13f23dd6e8ce6473738169bbb44394ba 100644 --- a/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml +++ b/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml @@ -85,8 +85,9 @@ examples: compatible = "mediatek,mt8173-nor"; reg = <0 0x1100d000 0 0xe0>; interrupts = <1>; - clocks = <&pericfg CLK_PERI_SPI>, <&topckgen CLK_TOP_SPINFI_IFR_SEL>; - clock-names = "spi", "sf"; + clocks = <&pericfg CLK_PERI_SPI>, <&topckgen CLK_TOP_SPINFI_IFR_SEL>, + <&pericfg CLK_PERI_NFI>; + clock-names = "spi", "sf", "axi"; #address-cells = <1>; #size-cells = <0>; diff --git a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml index 7326c0a28d160965407efa36357e9c7e291a0ddb..1051690e37537172351857c4cc6807272c603f48 100644 --- a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml +++ b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml @@ -4,7 +4,11 @@ $id: http://devicetree.org/schemas/spi/microchip,mpfs-spi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip MPFS {Q,}SPI Controller Device Tree Bindings +title: Microchip FPGA {Q,}SPI Controllers + +description: + SPI and QSPI controllers on Microchip PolarFire SoC and the "soft"/ + fabric IP cores they are based on maintainers: - Conor Dooley @@ -14,9 +18,12 @@ allOf: properties: compatible: - enum: - - microchip,mpfs-spi - - microchip,mpfs-qspi + oneOf: + - items: + - const: microchip,mpfs-qspi + - const: microchip,coreqspi-rtl-v2 + - const: microchip,coreqspi-rtl-v2 #FPGA QSPI + - const: microchip,mpfs-spi reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/spi/mxicy,mx25f0a-spi.yaml b/Documentation/devicetree/bindings/spi/mxicy,mx25f0a-spi.yaml index 9202c44b44787ac6dbb478c9519d82c581cf5fb7..a3aa5e07c0e434ed4a549c23d99948cbbbfac58d 100644 --- a/Documentation/devicetree/bindings/spi/mxicy,mx25f0a-spi.yaml +++ b/Documentation/devicetree/bindings/spi/mxicy,mx25f0a-spi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/mxicy,mx25f0a-spi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Macronix SPI controller device tree bindings +title: Macronix SPI controller maintainers: - Miquel Raynal diff --git a/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt b/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt index b98203ca656d3a7ec655750d60958313648e399f..a4e72e52af592ce5e998d68bc1c1d55985db0b44 100644 --- a/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt +++ b/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt @@ -3,7 +3,8 @@ Nuvoton NPCM Peripheral Serial Peripheral Interface(PSPI) controller driver Nuvoton NPCM7xx SOC support two PSPI channels. Required properties: - - compatible : "nuvoton,npcm750-pspi" for NPCM7XX BMC + - compatible : "nuvoton,npcm750-pspi" for Poleg NPCM7XX. + "nuvoton,npcm845-pspi" for Arbel NPCM8XX. - #address-cells : should be 1. see spi-bus.txt - #size-cells : should be 0. see spi-bus.txt - specifies physical base address and size of the register. diff --git a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml index 24e0c2181d25db0166a84726f4741756b365033e..2c3cada75339279883d679cca42e51ff3e32daf9 100644 --- a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml +++ b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml @@ -29,5 +29,4 @@ properties: minimum: 0 maximum: 255 -unevaluatedProperties: true - +additionalProperties: true diff --git a/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml b/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22879f7dcb775e5bcaa46b7d30540ca3f7f3532c --- /dev/null +++ b/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/ralink,mt7621-spi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +maintainers: + - Sergio Paracuellos + +title: Mediatek MT7621/MT7628 SPI controller + +allOf: + - $ref: /schemas/spi/spi-controller.yaml# + +properties: + compatible: + const: ralink,mt7621-spi + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: spi + + resets: + maxItems: 1 + + reset-names: + const: spi + +required: + - compatible + - reg + - resets + - "#address-cells" + - "#size-cells" + +unevaluatedProperties: false + +examples: + - | + #include + #include + + spi@b00 { + compatible = "ralink,mt7621-spi"; + reg = <0xb00 0x100>; + clocks = <&sysc MT7621_CLK_SPI>; + clock-names = "spi"; + resets = <&sysc MT7621_RST_SPI>; + reset-names = "spi"; + + #address-cells = <1>; + #size-cells = <0>; + + pinctrl-names = "default"; + pinctrl-0 = <&spi_pins>; + }; diff --git a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml index 5de710adfa63cb12311aff4209b40a0f49d39d99..491a695a2deb3b834595855f1ff51b5fee302860 100644 --- a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml +++ b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml @@ -47,9 +47,15 @@ properties: - renesas,msiof-r8a77980 # R-Car V3H - renesas,msiof-r8a77990 # R-Car E3 - renesas,msiof-r8a77995 # R-Car D3 - - renesas,msiof-r8a779a0 # R-Car V3U - const: renesas,rcar-gen3-msiof # generic R-Car Gen3 and RZ/G2 # compatible device + - items: + - enum: + - renesas,msiof-r8a779a0 # R-Car V3U + - renesas,msiof-r8a779f0 # R-Car S4-8 + - renesas,msiof-r8a779g0 # R-Car V4H + - const: renesas,rcar-gen4-msiof # generic R-Car Gen4 + # compatible device - items: - const: renesas,sh-msiof # deprecated @@ -69,6 +75,12 @@ properties: clocks: maxItems: 1 + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + num-cs: description: | Total number of chip selects (default is 1). diff --git a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml index 37c3c272407d06394bf937053a53935703386510..d33b72fabc5d8c2f6baec5232ed4fae43b88373e 100644 --- a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml +++ b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml @@ -104,7 +104,6 @@ properties: const: spi reg-io-width: - $ref: /schemas/types.yaml#/definitions/uint32 description: I/O register width (in bytes) implemented by this device default: 4 enum: [ 2, 4 ] diff --git a/Documentation/devicetree/bindings/spi/spi-cadence.yaml b/Documentation/devicetree/bindings/spi/spi-cadence.yaml index 82d0ca5c00f3b0fb09aa084b9c8fb88ec0560142..64bf4e621142fcb20bacd9533cc6a17d929d418b 100644 --- a/Documentation/devicetree/bindings/spi/spi-cadence.yaml +++ b/Documentation/devicetree/bindings/spi/spi-cadence.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/spi-cadence.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Cadence SPI controller Device Tree Bindings +title: Cadence SPI controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml index 655713fba7e252354eb6c1b67f9018b916d439c6..01042a7f382e2b6f22d995333e161aa81ff5d6f2 100644 --- a/Documentation/devicetree/bindings/spi/spi-controller.yaml +++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml @@ -96,6 +96,11 @@ patternProperties: $ref: spi-peripheral-props.yaml properties: + spi-3wire: + $ref: /schemas/types.yaml#/definitions/flag + description: + The device requires 3-wire mode. + spi-cpha: $ref: /schemas/types.yaml#/definitions/flag description: diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml index 1d46877fe46a8b4a335f4e6237bf5395ca6ab2d7..8b44284d30c6c7066c9e9981a64f8873a3bf2ac0 100644 --- a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml @@ -19,7 +19,9 @@ properties: - fsl,imx7ulp-spi - fsl,imx8qxp-spi - items: - - const: fsl,imx8ulp-spi + - enum: + - fsl,imx8ulp-spi + - fsl,imx93-spi - const: fsl,imx7ulp-spi reg: maxItems: 1 @@ -37,6 +39,16 @@ properties: - const: per - const: ipg + dmas: + items: + - description: TX DMA Channel + - description: RX DMA Channel + + dma-names: + items: + - const: tx + - const: rx + fsl,spi-only-use-cs1-sel: description: spi common code does not support use of CS signals discontinuously. diff --git a/Documentation/devicetree/bindings/spi/spi-mt7621.txt b/Documentation/devicetree/bindings/spi/spi-mt7621.txt deleted file mode 100644 index d5baec0fa56e629e9cccf48741a1eb691daa7cb9..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/spi/spi-mt7621.txt +++ /dev/null @@ -1,26 +0,0 @@ -Binding for MTK SPI controller (MT7621 MIPS) - -Required properties: -- compatible: Should be one of the following: - - "ralink,mt7621-spi": for mt7621/mt7628/mt7688 platforms -- #address-cells: should be 1. -- #size-cells: should be 0. -- reg: Address and length of the register set for the device -- resets: phandle to the reset controller asserting this device in - reset - See ../reset/reset.txt for details. - -Optional properties: -- cs-gpios: see spi-bus.txt. - -Example: - -- SoC Specific Portion: -spi0: spi@b00 { - compatible = "ralink,mt7621-spi"; - reg = <0xb00 0x100>; - #address-cells = <1>; - #size-cells = <0>; - resets = <&rstctrl 18>; - reset-names = "spi"; -}; diff --git a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml index a4abe15880053419cb0926330c5955d4cbf8e529..dca677f9e1b98f44b2525deddc04d1488d5f7797 100644 --- a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml +++ b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml @@ -29,11 +29,6 @@ properties: description: Chip select used by the device. - spi-3wire: - $ref: /schemas/types.yaml#/definitions/flag - description: - The device requires 3-wire mode. - spi-cs-high: $ref: /schemas/types.yaml#/definitions/flag description: diff --git a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml index 52a78a2e362e0f2e83418bf1c881020140171b37..66e49947b7035ed71bf1628413297499eb3e4763 100644 --- a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml +++ b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml @@ -27,6 +27,7 @@ properties: - items: - enum: - rockchip,px30-spi + - rockchip,rk3128-spi - rockchip,rk3188-spi - rockchip,rk3288-spi - rockchip,rk3308-spi @@ -34,6 +35,7 @@ properties: - rockchip,rk3368-spi - rockchip,rk3399-spi - rockchip,rk3568-spi + - rockchip,rk3588-spi - rockchip,rv1126-spi - const: rockchip,rk3066-spi @@ -80,6 +82,9 @@ properties: where the "sleep" configuration may describe the state the pins should be in during system suspend. + power-domains: + maxItems: 1 + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/spi/spi-xilinx.yaml b/Documentation/devicetree/bindings/spi/spi-xilinx.yaml index 03e5dca7e933c3874f2ec4720fe81a87c2b436a0..bbb735603f298a6ac65a446713c225cd2cf9e12e 100644 --- a/Documentation/devicetree/bindings/spi/spi-xilinx.yaml +++ b/Documentation/devicetree/bindings/spi/spi-xilinx.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/spi-xilinx.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx SPI controller Device Tree Bindings +title: Xilinx SPI controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/spi/spi-zynqmp-qspi.yaml b/Documentation/devicetree/bindings/spi/spi-zynqmp-qspi.yaml index fafde1c06be67d403ff2da5d66ce097e32790f2c..6bf0edc57f4afc66bc857655977329c92065cbe6 100644 --- a/Documentation/devicetree/bindings/spi/spi-zynqmp-qspi.yaml +++ b/Documentation/devicetree/bindings/spi/spi-zynqmp-qspi.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spi/spi-zynqmp-qspi.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx Zynq UltraScale+ MPSoC GQSPI controller Device Tree Bindings +title: Xilinx Zynq UltraScale+ MPSoC GQSPI controller maintainers: - Michal Simek diff --git a/Documentation/devicetree/bindings/spmi/mtk,spmi-mtk-pmif.yaml b/Documentation/devicetree/bindings/spmi/mtk,spmi-mtk-pmif.yaml index 2445c5e0b0efb0b54a6b458166ff279a37b80d0e..abcbbe13723fb9dd7aad384045b647300720dcc5 100644 --- a/Documentation/devicetree/bindings/spmi/mtk,spmi-mtk-pmif.yaml +++ b/Documentation/devicetree/bindings/spmi/mtk,spmi-mtk-pmif.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/spmi/mtk,spmi-mtk-pmif.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mediatek SPMI Controller Device Tree Bindings +title: Mediatek SPMI Controller maintainers: - Hsin-Hsiung Wang diff --git a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml index 1c426c211e3668baf3a44077026b07c3e0a7bc06..98a7dc7f467d91fde72eab705ddb7edc2ea8d425 100644 --- a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml +++ b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/sram/allwinner,sun4i-a10-system-control.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 System Control Device Tree Bindings +title: Allwinner A10 System Control maintainers: - Chen-Yu Tsai @@ -24,32 +24,31 @@ properties: compatible: oneOf: - - const: allwinner,sun4i-a10-sram-controller + - enum: + - allwinner,sun4i-a10-sram-controller + - allwinner,sun50i-a64-sram-controller deprecated: true - - const: allwinner,sun4i-a10-system-control - - const: allwinner,sun5i-a13-system-control + - enum: + - allwinner,sun4i-a10-system-control + - allwinner,sun5i-a13-system-control + - allwinner,sun8i-a23-system-control + - allwinner,sun8i-h3-system-control + - allwinner,sun20i-d1-system-control + - allwinner,sun50i-a64-system-control + - allwinner,sun50i-h5-system-control + - allwinner,sun50i-h616-system-control - items: - - const: allwinner,sun7i-a20-system-control + - enum: + - allwinner,suniv-f1c100s-system-control + - allwinner,sun7i-a20-system-control + - allwinner,sun8i-r40-system-control - const: allwinner,sun4i-a10-system-control - - const: allwinner,sun8i-a23-system-control - - const: allwinner,sun8i-h3-system-control - items: - const: allwinner,sun8i-v3s-system-control - const: allwinner,sun8i-h3-system-control - - items: - - const: allwinner,sun8i-r40-system-control - - const: allwinner,sun4i-a10-system-control - - const: allwinner,sun50i-a64-sram-controller - deprecated: true - - const: allwinner,sun50i-a64-system-control - - const: allwinner,sun50i-h5-system-control - items: - const: allwinner,sun50i-h6-system-control - const: allwinner,sun50i-a64-system-control - - items: - - const: allwinner,suniv-f1c100s-system-control - - const: allwinner,sun4i-a10-system-control - - const: allwinner,sun50i-h616-system-control reg: maxItems: 1 @@ -76,43 +75,26 @@ patternProperties: - const: allwinner,sun4i-a10-sram-d - const: allwinner,sun50i-a64-sram-c - items: - - const: allwinner,sun5i-a13-sram-a3-a4 - - const: allwinner,sun4i-a10-sram-a3-a4 - - items: - - const: allwinner,sun7i-a20-sram-a3-a4 + - enum: + - allwinner,sun5i-a13-sram-a3-a4 + - allwinner,sun7i-a20-sram-a3-a4 - const: allwinner,sun4i-a10-sram-a3-a4 - items: - - const: allwinner,sun5i-a13-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun7i-a20-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun8i-a23-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun8i-h3-sram-c1 + - enum: + - allwinner,sun5i-a13-sram-c1 + - allwinner,sun7i-a20-sram-c1 + - allwinner,sun8i-a23-sram-c1 + - allwinner,sun8i-h3-sram-c1 + - allwinner,sun8i-r40-sram-c1 + - allwinner,sun50i-a64-sram-c1 + - allwinner,sun50i-h5-sram-c1 + - allwinner,sun50i-h6-sram-c1 - const: allwinner,sun4i-a10-sram-c1 - items: - - const: allwinner,sun8i-r40-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun50i-a64-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun50i-h5-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun50i-h6-sram-c1 - - const: allwinner,sun4i-a10-sram-c1 - - items: - - const: allwinner,sun5i-a13-sram-d - - const: allwinner,sun4i-a10-sram-d - - items: - - const: allwinner,sun7i-a20-sram-d - - const: allwinner,sun4i-a10-sram-d - - items: - - const: allwinner,suniv-f1c100s-sram-d + - enum: + - allwinner,suniv-f1c100s-sram-d + - allwinner,sun5i-a13-sram-d + - allwinner,sun7i-a20-sram-d - const: allwinner,sun4i-a10-sram-d - items: - const: allwinner,sun50i-h6-sram-c diff --git a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml index 6e0b110153b01faa73c3043937f278e3fd282223..fbd4212285e285efc751ac1f209d2671eaa0b3be 100644 --- a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml +++ b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/thermal/allwinner,sun8i-a83t-ths.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner SUN8I Thermal Controller Device Tree Bindings +title: Allwinner SUN8I Thermal Controller maintainers: - Vasily Khoruzhick diff --git a/Documentation/devicetree/bindings/thermal/thermal-zones.yaml b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml index 2d34f3ccb2572ddb9caafd2f290d6c7e2779b4ed..8d2c6d74b605a1ebde7d2e3d3b9cb78610d40dec 100644 --- a/Documentation/devicetree/bindings/thermal/thermal-zones.yaml +++ b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml @@ -214,6 +214,7 @@ patternProperties: - polling-delay - polling-delay-passive - thermal-sensors + - trips additionalProperties: false diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml index 3711872b6b99d2e99f8b9748d0b59b36fb7037c4..b3538fac1ad2ef645d58c61849328f16df06e5e4 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timer/allwinner,sun4i-a10-timer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Timer Device Tree Bindings +title: Allwinner A10 Timer maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml index 2ecac754e1cd009ec7e36f81c9ede85de83b6d6a..f1853daec2f9ad95081909a32b9ee39c9caee6d6 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timer/allwinner,sun5i-a13-hstimer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A13 High-Speed Timer Device Tree Bindings +title: Allwinner A13 High-Speed Timer maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/timer/arm,arch_timer.yaml b/Documentation/devicetree/bindings/timer/arm,arch_timer.yaml index df8ce87fd54b7f3c72e6ffc2eb663bfc7c45455d..c5fc3b6c8bd0bfeaf06867ab2d58b0be07729236 100644 --- a/Documentation/devicetree/bindings/timer/arm,arch_timer.yaml +++ b/Documentation/devicetree/bindings/timer/arm,arch_timer.yaml @@ -22,16 +22,15 @@ properties: compatible: oneOf: - items: - - enum: - - arm,cortex-a15-timer - - enum: - - arm,armv7-timer + - const: arm,cortex-a15-timer + - const: arm,armv7-timer - items: - enum: - arm,armv7-timer - - items: - - enum: - arm,armv8-timer + - items: + - const: arm,armv8-timer + - const: arm,armv7-timer interrupts: minItems: 1 diff --git a/Documentation/devicetree/bindings/timer/arm,arch_timer_mmio.yaml b/Documentation/devicetree/bindings/timer/arm,arch_timer_mmio.yaml index cd2176cad53aad6ecc981e9fc90a6b86a4b75393..f6efa48c4256cbc780f0d268166759f8f6305502 100644 --- a/Documentation/devicetree/bindings/timer/arm,arch_timer_mmio.yaml +++ b/Documentation/devicetree/bindings/timer/arm,arch_timer_mmio.yaml @@ -62,6 +62,7 @@ properties: patternProperties: '^frame@[0-9a-z]*$': type: object + additionalProperties: false description: A timer node has up to 8 frame sub-nodes, each with the following properties. properties: frame-number: diff --git a/Documentation/devicetree/bindings/timer/ingenic,tcu.yaml b/Documentation/devicetree/bindings/timer/ingenic,tcu.yaml index 0a01e4f5eddb0822a072d9fbe191e97e2b899ff7..a84fef0fe628f47a3140003e0b319e8dc9051f32 100644 --- a/Documentation/devicetree/bindings/timer/ingenic,tcu.yaml +++ b/Documentation/devicetree/bindings/timer/ingenic,tcu.yaml @@ -114,6 +114,8 @@ patternProperties: "^watchdog@[a-f0-9]+$": type: object $ref: /schemas/watchdog/watchdog.yaml# + unevaluatedProperties: false + properties: compatible: oneOf: @@ -146,6 +148,8 @@ patternProperties: "^pwm@[a-f0-9]+$": type: object $ref: /schemas/pwm/pwm.yaml# + unevaluatedProperties: false + properties: compatible: oneOf: diff --git a/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt index f1c848af91d3a7935049125262e3c6c0193daacc..8bbb6e94508b25c5773f9e8436cac9575215bf6a 100644 --- a/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt +++ b/Documentation/devicetree/bindings/timer/mediatek,mtk-timer.txt @@ -25,6 +25,7 @@ Required properties: For those SoCs that use SYST * "mediatek,mt8183-timer" for MT8183 compatible timers (SYST) * "mediatek,mt8186-timer" for MT8186 compatible timers (SYST) + * "mediatek,mt8188-timer" for MT8188 compatible timers (SYST) * "mediatek,mt8192-timer" for MT8192 compatible timers (SYST) * "mediatek,mt8195-timer" for MT8195 compatible timers (SYST) * "mediatek,mt7629-timer" for MT7629 compatible timers (SYST) diff --git a/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml b/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml index 03d5dba5d5b30a575164de415154eb1e3a70c062..f118ca423e382152a83aac1c0a9d1cda39090e46 100644 --- a/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml +++ b/Documentation/devicetree/bindings/timer/mstar,msc313e-timer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timer/mstar,msc313e-timer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Mstar MSC313e Timer Device Tree Bindings +title: Mstar MSC313e Timer maintainers: - Daniel Palmer diff --git a/Documentation/devicetree/bindings/timer/nxp,sysctr-timer.yaml b/Documentation/devicetree/bindings/timer/nxp,sysctr-timer.yaml index 830211c55b4aa6a1aaf65c41dd9dff3727137629..2b9653dafab8f3380d60f942ef04f29a14e34bb8 100644 --- a/Documentation/devicetree/bindings/timer/nxp,sysctr-timer.yaml +++ b/Documentation/devicetree/bindings/timer/nxp,sysctr-timer.yaml @@ -32,6 +32,10 @@ properties: clock-names: const: per + nxp,no-divider: + description: if present, means there is no internal base clk divider. + type: boolean + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/timer/renesas,tmu.yaml b/Documentation/devicetree/bindings/timer/renesas,tmu.yaml index c57169118b68def1265c3ff68dddf96b4603be1c..60f4c059bcffde4bfed2f4e3001bfc26d492ca7a 100644 --- a/Documentation/devicetree/bindings/timer/renesas,tmu.yaml +++ b/Documentation/devicetree/bindings/timer/renesas,tmu.yaml @@ -37,6 +37,7 @@ properties: - renesas,tmu-r8a77990 # R-Car E3 - renesas,tmu-r8a77995 # R-Car D3 - renesas,tmu-r8a779a0 # R-Car V3U + - renesas,tmu-r8a779f0 # R-Car S4-8 - const: renesas,tmu reg: diff --git a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml index 5d157d87dad50f62b8f64ff06a6a0555bb7af070..dc3bc1e62fe9ca9ddaa205c564a18197cb590f9b 100644 --- a/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml +++ b/Documentation/devicetree/bindings/timer/rockchip,rk-timer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timer/rockchip,rk-timer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Rockchip Timer Device Tree Bindings +title: Rockchip Timer maintainers: - Daniel Lezcano diff --git a/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.yaml b/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.yaml index 9c81d00b12e093b1669b345ce55dffa05022eec7..829bd2227f7c9c64a86417af3dc5c6057bba9234 100644 --- a/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.yaml +++ b/Documentation/devicetree/bindings/timer/samsung,exynos4210-mct.yaml @@ -25,6 +25,7 @@ properties: - samsung,exynos4412-mct - items: - enum: + - axis,artpec8-mct - samsung,exynos3250-mct - samsung,exynos5250-mct - samsung,exynos5260-mct @@ -45,6 +46,19 @@ properties: reg: maxItems: 1 + samsung,frc-shared: + type: boolean + description: | + Indicates that the hardware requires that this processor share the + free-running counter with a different (main) processor. + + samsung,local-timers: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 16 + description: | + List of indices of local timers usable from this processor. + interrupts: description: | Interrupts should be put in specific order. This is, the local timer @@ -74,6 +88,17 @@ required: - reg allOf: + - if: + not: + properties: + compatible: + contains: + enum: + - axis,artpec8-mct + then: + properties: + samsung,local-timers: false + samsung,frc-shared: false - if: properties: compatible: @@ -101,6 +126,7 @@ allOf: compatible: contains: enum: + - axis,artpec8-mct - samsung,exynos5260-mct - samsung,exynos5420-mct - samsung,exynos5433-mct diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index e64f46339079fa3ffe202daeb9f35c2c10dcdb8f..bbad241658374a954111c7fc14c1b74ade970581 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -22,12 +22,18 @@ description: properties: compatible: - items: - - enum: - - sifive,fu540-c000-clint - - starfive,jh7100-clint - - canaan,k210-clint - - const: sifive,clint0 + oneOf: + - items: + - enum: + - sifive,fu540-c000-clint + - starfive,jh7100-clint + - canaan,k210-clint + - const: sifive,clint0 + - items: + - const: sifive,clint0 + - const: riscv,clint0 + deprecated: true + description: For the QEMU virt machine only description: Should be ",-clint" and "sifive,clint". diff --git a/Documentation/devicetree/bindings/timer/ti,timer-dm.yaml b/Documentation/devicetree/bindings/timer/ti,timer-dm.yaml index e32df21e63a0481a40c599c77cc345c6395e66cb..acbb6f8997ee4001e46d8b0ef84e49ef7b33d109 100644 --- a/Documentation/devicetree/bindings/timer/ti,timer-dm.yaml +++ b/Documentation/devicetree/bindings/timer/ti,timer-dm.yaml @@ -51,6 +51,11 @@ properties: - const: timer_sys_ck minItems: 1 + power-domains: + description: + Power domain if available + maxItems: 1 + interrupts: description: Interrupt if available. The timer PWM features may be usable @@ -94,12 +99,14 @@ additionalProperties: false allOf: - if: - not: - properties: - compatible: - contains: - const: ti,am654-timer + properties: + compatible: + contains: + const: ti,am654-timer then: + required: + - power-domains + else: required: - interrupts diff --git a/Documentation/devicetree/bindings/timer/xlnx,xps-timer.yaml b/Documentation/devicetree/bindings/timer/xlnx,xps-timer.yaml index dd168d41d2e059a6781012824771785cb5b7e257..b1597db0426358cc2465195cd5dafc852849ccf0 100644 --- a/Documentation/devicetree/bindings/timer/xlnx,xps-timer.yaml +++ b/Documentation/devicetree/bindings/timer/xlnx,xps-timer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timer/xlnx,xps-timer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Xilinx LogiCORE IP AXI Timer Device Tree Binding +title: Xilinx LogiCORE IP AXI Timer maintainers: - Sean Anderson diff --git a/Documentation/devicetree/bindings/timestamp/hte-consumer.yaml b/Documentation/devicetree/bindings/timestamp/hte-consumer.yaml index 6456515c3d262a812b858bb29165869fa56f1a50..5142d6d4fc0ad92863c2ac933ea34983489d2aa5 100644 --- a/Documentation/devicetree/bindings/timestamp/hte-consumer.yaml +++ b/Documentation/devicetree/bindings/timestamp/hte-consumer.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/timestamp/hte-consumer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: HTE Consumer Device Tree Bindings +title: HTE Consumer maintainers: - Dipen Patel diff --git a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml index 2c715eec48b86ef1cc20005c3a78664cead036b9..a9988798898de73471bfb0f13482be476adecc1e 100644 --- a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml +++ b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/ufs/samsung,exynos-ufs.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Samsung SoC series UFS host controller Device Tree Bindings +title: Samsung SoC series UFS host controller maintainers: - Alim Akhtar diff --git a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml index e5dbf4169bc9e384195b4b45d4abf1330a180137..8992eff6ce3877169aecda6cbe8a8d82c09a7763 100644 --- a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml +++ b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/allwinner,sun4i-a10-musb.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 mUSB OTG Controller Device Tree Bindings +title: Allwinner A10 mUSB OTG Controller maintainers: - Chen-Yu Tsai diff --git a/Documentation/devicetree/bindings/usb/amlogic,meson-g12a-usb-ctrl.yaml b/Documentation/devicetree/bindings/usb/amlogic,meson-g12a-usb-ctrl.yaml index e349fa5de60611e155333c8f9cec378b139139ba..daf2a859418d40563f5c9b800074173e83816f7f 100644 --- a/Documentation/devicetree/bindings/usb/amlogic,meson-g12a-usb-ctrl.yaml +++ b/Documentation/devicetree/bindings/usb/amlogic,meson-g12a-usb-ctrl.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Amlogic Meson G12A DWC3 USB SoC Controller Glue maintainers: - - Neil Armstrong + - Neil Armstrong description: | The Amlogic G12A embeds a DWC3 USB IP Core configured for USB2 and USB3 diff --git a/Documentation/devicetree/bindings/usb/analogix,anx7411.yaml b/Documentation/devicetree/bindings/usb/analogix,anx7411.yaml index ee436308e5dc2c0d64861d8dd5a3eb60bbefea92..0e72c08e65665cc608e135d428f9cea6cf605651 100644 --- a/Documentation/devicetree/bindings/usb/analogix,anx7411.yaml +++ b/Documentation/devicetree/bindings/usb/analogix,anx7411.yaml @@ -23,6 +23,8 @@ properties: connector: type: object $ref: ../connector/usb-connector.yaml + unevaluatedProperties: false + description: Properties for usb c connector. diff --git a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml index 8b019ac05bbe0dfb1096fa60b857ed848682d14f..a86bcd95100eb3e3968df0dd042069e120ad455c 100644 --- a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml +++ b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml @@ -67,6 +67,7 @@ properties: vhub-strings: type: object + additionalProperties: false properties: '#address-cells': @@ -78,6 +79,7 @@ properties: patternProperties: '^string@[0-9a-f]+$': type: object + additionalProperties: false description: string descriptors of the specific language properties: diff --git a/Documentation/devicetree/bindings/usb/brcm,bcm7445-ehci.yaml b/Documentation/devicetree/bindings/usb/brcm,bcm7445-ehci.yaml index 2a9acf2b5a6472f36bdec208fa8ffe78a40c41cc..ad075407d85e8f03060f141a74d417081a8af228 100644 --- a/Documentation/devicetree/bindings/usb/brcm,bcm7445-ehci.yaml +++ b/Documentation/devicetree/bindings/usb/brcm,bcm7445-ehci.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/brcm,bcm7445-ehci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom STB USB EHCI Controller Device Tree Bindings +title: Broadcom STB USB EHCI Controller allOf: - $ref: "usb-hcd.yaml" diff --git a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml index d4618d15ecc1f791d1353d2d6b7c9724e7400a9f..5cc81ced589d50da71c91bed9f4d2dfa232e78dc 100644 --- a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml +++ b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/brcm,usb-pinmap.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Broadcom USB pin map Controller Device Tree Bindings +title: Broadcom USB pin map Controller maintainers: - Al Cooper diff --git a/Documentation/devicetree/bindings/usb/dwc2.yaml b/Documentation/devicetree/bindings/usb/dwc2.yaml index 1bfbc6ef16eb59354b778eb04b913439e62c6241..dc4988c0009cf25593d2d0055f13f375e2cee5c9 100644 --- a/Documentation/devicetree/bindings/usb/dwc2.yaml +++ b/Documentation/devicetree/bindings/usb/dwc2.yaml @@ -32,6 +32,7 @@ properties: - enum: - rockchip,px30-usb - rockchip,rk3036-usb + - rockchip,rk3128-usb - rockchip,rk3188-usb - rockchip,rk3228-usb - rockchip,rk3288-usb diff --git a/Documentation/devicetree/bindings/usb/faraday,fotg210.txt b/Documentation/devicetree/bindings/usb/faraday,fotg210.txt deleted file mode 100644 index 06a2286e205468cbb6a872a6255e5c2825f04123..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/usb/faraday,fotg210.txt +++ /dev/null @@ -1,35 +0,0 @@ -Faraday FOTG Host controller - -This OTG-capable USB host controller is found in Cortina Systems -Gemini and other SoC products. - -Required properties: -- compatible: should be one of: - "faraday,fotg210" - "cortina,gemini-usb", "faraday,fotg210" -- reg: should contain one register range i.e. start and length -- interrupts: description of the interrupt line - -Optional properties: -- clocks: should contain the IP block clock -- clock-names: should be "PCLK" for the IP block clock - -Required properties for "cortina,gemini-usb" compatible: -- syscon: a phandle to the system controller to access PHY registers - -Optional properties for "cortina,gemini-usb" compatible: -- cortina,gemini-mini-b: boolean property that indicates that a Mini-B - OTG connector is in use -- wakeup-source: see power/wakeup-source.txt - -Example for Gemini: - -usb@68000000 { - compatible = "cortina,gemini-usb", "faraday,fotg210"; - reg = <0x68000000 0x1000>; - interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; - clocks = <&cc 12>; - clock-names = "PCLK"; - syscon = <&syscon>; - wakeup-source; -}; diff --git a/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml b/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c69bbfbcf733d74c15d5c6582f704c56f005d508 --- /dev/null +++ b/Documentation/devicetree/bindings/usb/faraday,fotg210.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright 2022 Linaro Ltd. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/usb/faraday,fotg210.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Faraday Technology FOTG210 HS OTG USB 2.0 controller Bindings + +maintainers: + - Linus Walleij + +allOf: + - $ref: usb-drd.yaml# + - $ref: usb-hcd.yaml# + +properties: + compatible: + oneOf: + - const: faraday,fotg210 + - items: + - const: cortina,gemini-usb + - const: faraday,fotg210 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: PCLK + + resets: + maxItems: 1 + + syscon: + $ref: /schemas/types.yaml#/definitions/phandle + description: a phandle to the global Gemini system controller on + Gemini systems + + dr_mode: true + + phys: + maxItems: 1 + + phy-names: + const: usb2-phy + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + #include + #include + usb0: usb@68000000 { + compatible = "cortina,gemini-usb", "faraday,fotg210"; + reg = <0x68000000 0x1000>; + interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; + resets = <&syscon GEMINI_RESET_USB0>; + clocks = <&syscon GEMINI_CLK_GATE_USB0>; + clock-names = "PCLK"; + syscon = <&syscon>; + dr_mode = "host"; + }; diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml index 079f7cff0c243e475557ced9222502ffcbf4f39c..c5f629c5bc6171261b2a59dd01c599719dce0e8b 100644 --- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/generic-ehci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: USB EHCI Controller Device Tree Bindings +title: USB EHCI Controller maintainers: - Greg Kroah-Hartman diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml index 180361b79f526bb5a4b08e395d7d476d96b90a1a..f838f78d6164a41992cc31269f63f32bad6b5132 100644 --- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/generic-ohci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: USB OHCI Controller Device Tree Bindings +title: USB OHCI Controller allOf: - $ref: "usb-hcd.yaml" diff --git a/Documentation/devicetree/bindings/usb/generic-xhci.yaml b/Documentation/devicetree/bindings/usb/generic-xhci.yaml index 23d73df96ea3ad90cc9294e6fac8b97a377bb88e..db841589fc334f602e5032adbf258eb7f4d6ecba 100644 --- a/Documentation/devicetree/bindings/usb/generic-xhci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-xhci.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/generic-xhci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: USB xHCI Controller Device Tree Bindings +title: USB xHCI Controller maintainers: - Mathias Nyman diff --git a/Documentation/devicetree/bindings/usb/mediatek,mt6370-tcpc.yaml b/Documentation/devicetree/bindings/usb/mediatek,mt6370-tcpc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72f56cc88457436f60a00c85b7c0624c5886f44f --- /dev/null +++ b/Documentation/devicetree/bindings/usb/mediatek,mt6370-tcpc.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/usb/mediatek,mt6370-tcpc.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: MediatTek MT6370 Type-C Port Switch and Power Delivery controller + +maintainers: + - ChiYuan Huang + +description: | + MediaTek MT6370 is a multi-functional device. + It integrates charger, ADC, flash, RGB indicators, + regulators (DSV/VIBLDO), and TypeC Port Switch with Power Delivery controller. + This document only describes MT6370 Type-C Port Switch and + Power Delivery controller. + +properties: + compatible: + enum: + - mediatek,mt6370-tcpc + + interrupts: + maxItems: 1 + + connector: + type: object + $ref: /schemas/connector/usb-connector.yaml# + unevaluatedProperties: false + +additionalProperties: false + +required: + - compatible + - interrupts diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml index b0e58b15b9ae762b3595d126cf02714fb059895e..939623867a646d5cff3a622c48e822061b3fe135 100644 --- a/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml +++ b/Documentation/devicetree/bindings/usb/mediatek,mtk-xhci.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/usb/mediatek,mtk-xhci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek USB3 xHCI Device Tree Bindings +title: MediaTek USB3 xHCI maintainers: - Chunfeng Yun diff --git a/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml b/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml index e63b665453176cf6302b55cdcb37648a7cb3176d..80750b0f458a82b48856920c99eb43487be73fa3 100644 --- a/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml +++ b/Documentation/devicetree/bindings/usb/mediatek,mtu3.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/usb/mediatek,mtu3.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek USB3 DRD Controller Device Tree Bindings +title: MediaTek USB3 DRD Controller maintainers: - Chunfeng Yun @@ -24,6 +24,7 @@ properties: - mediatek,mt2712-mtu3 - mediatek,mt8173-mtu3 - mediatek,mt8183-mtu3 + - mediatek,mt8188-mtu3 - mediatek,mt8192-mtu3 - mediatek,mt8195-mtu3 - const: mediatek,mtu3 diff --git a/Documentation/devicetree/bindings/usb/mediatek,musb.yaml b/Documentation/devicetree/bindings/usb/mediatek,musb.yaml index 11a33f9b1f17096169b6a1f8b0b9dc205d97bab0..f16ab30a95d274083597faf55dda3ee89c98de9a 100644 --- a/Documentation/devicetree/bindings/usb/mediatek,musb.yaml +++ b/Documentation/devicetree/bindings/usb/mediatek,musb.yaml @@ -5,7 +5,7 @@ $id: http://devicetree.org/schemas/usb/mediatek,musb.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MediaTek MUSB DRD/OTG Controller Device Tree Bindings +title: MediaTek MUSB DRD/OTG Controller maintainers: - Min Guo diff --git a/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml b/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml index 48c458c65848d889b71718b9db127629630df8bb..c5e9ce2e7bc2a3a8a98216e7e633ed995c0645a6 100644 --- a/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml +++ b/Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/microchip,mpfs-musb.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Microchip MPFS USB Controller Device Tree Bindings +title: Microchip MPFS USB Controller allOf: - $ref: usb-drd.yaml# diff --git a/Documentation/devicetree/bindings/usb/npcm7xx-usb.txt b/Documentation/devicetree/bindings/usb/npcm7xx-usb.txt index 5a0f1f14fbfae1afd809fb0451fd9055e7b0a43a..352a0a1e2f76f7c22feda6edf19a3d03207da2c3 100644 --- a/Documentation/devicetree/bindings/usb/npcm7xx-usb.txt +++ b/Documentation/devicetree/bindings/usb/npcm7xx-usb.txt @@ -5,7 +5,9 @@ EHCI: ----- Required properties: -- compatible: "nuvoton,npcm750-ehci" +- compatible: should be one of + "nuvoton,npcm750-ehci" + "nuvoton,npcm845-ehci" - interrupts: Should contain the EHCI interrupt - reg: Physical address and length of the register set for the device diff --git a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml index fea3e7092aceab6b8212ea23039bbc95ec2012c8..a6e6abb4dfa9b7e5026b43ad59dc86f1701335a6 100644 --- a/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/qcom,dwc3.yaml @@ -26,6 +26,7 @@ properties: - qcom,sc7280-dwc3 - qcom,sc8280xp-dwc3 - qcom,sdm660-dwc3 + - qcom,sdm670-dwc3 - qcom,sdm845-dwc3 - qcom,sdx55-dwc3 - qcom,sdx65-dwc3 @@ -33,6 +34,7 @@ properties: - qcom,sm6115-dwc3 - qcom,sm6125-dwc3 - qcom,sm6350-dwc3 + - qcom,sm6375-dwc3 - qcom,sm8150-dwc3 - qcom,sm8250-dwc3 - qcom,sm8350-dwc3 @@ -108,12 +110,17 @@ properties: HS/FS/LS modes are supported. type: boolean + wakeup-source: true + # Required child node: patternProperties: "^usb@[0-9a-f]+$": $ref: snps,dwc3.yaml# + properties: + wakeup-source: false + required: - compatible - reg @@ -169,6 +176,7 @@ allOf: - qcom,msm8998-dwc3 - qcom,sc7180-dwc3 - qcom,sc7280-dwc3 + - qcom,sdm670-dwc3 - qcom,sdm845-dwc3 - qcom,sdx55-dwc3 - qcom,sm6350-dwc3 @@ -288,6 +296,7 @@ allOf: compatible: contains: enum: + - qcom,sm6115-dwc3 - qcom,sm6125-dwc3 - qcom,sm8150-dwc3 - qcom,sm8250-dwc3 @@ -338,11 +347,11 @@ allOf: - qcom,msm8994-dwc3 - qcom,qcs404-dwc3 - qcom,sc7180-dwc3 + - qcom,sdm670-dwc3 - qcom,sdm845-dwc3 - qcom,sdx55-dwc3 - qcom,sdx65-dwc3 - qcom,sm4250-dwc3 - - qcom,sm6115-dwc3 - qcom,sm6125-dwc3 - qcom,sm6350-dwc3 - qcom,sm8150-dwc3 @@ -374,6 +383,7 @@ allOf: - qcom,msm8953-dwc3 - qcom,msm8996-dwc3 - qcom,msm8998-dwc3 + - qcom,sm6115-dwc3 then: properties: interrupts: diff --git a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml index 9fcf54b10b0744469e851be035bc780425d36058..55dfd121b55543237c1322f70280687423fbc7c5 100644 --- a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml +++ b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml @@ -11,27 +11,55 @@ maintainers: properties: compatible: - items: - - enum: - - renesas,r8a774a1-usb3-peri # RZ/G2M - - renesas,r8a774b1-usb3-peri # RZ/G2N - - renesas,r8a774c0-usb3-peri # RZ/G2E - - renesas,r8a774e1-usb3-peri # RZ/G2H - - renesas,r8a7795-usb3-peri # R-Car H3 - - renesas,r8a7796-usb3-peri # R-Car M3-W - - renesas,r8a77961-usb3-peri # R-Car M3-W+ - - renesas,r8a77965-usb3-peri # R-Car M3-N - - renesas,r8a77990-usb3-peri # R-Car E3 - - const: renesas,rcar-gen3-usb3-peri + oneOf: + - items: + - enum: + - renesas,r8a774a1-usb3-peri # RZ/G2M + - renesas,r8a774b1-usb3-peri # RZ/G2N + - renesas,r8a774c0-usb3-peri # RZ/G2E + - renesas,r8a774e1-usb3-peri # RZ/G2H + - renesas,r8a7795-usb3-peri # R-Car H3 + - renesas,r8a7796-usb3-peri # R-Car M3-W + - renesas,r8a77961-usb3-peri # R-Car M3-W+ + - renesas,r8a77965-usb3-peri # R-Car M3-N + - renesas,r8a77990-usb3-peri # R-Car E3 + - const: renesas,rcar-gen3-usb3-peri + + - items: + - enum: + - renesas,r9a09g011-usb3-peri # RZ/V2M + - const: renesas,rzv2m-usb3-peri reg: maxItems: 1 interrupts: - maxItems: 1 + minItems: 1 + items: + - description: Combined interrupt for DMA, SYS and ERR + - description: Dual Role Device (DRD) + - description: Battery Charging + - description: Global Purpose Input + + interrupt-names: + minItems: 1 + items: + - const: all_p + - const: drd + - const: bc + - const: gpi clocks: - maxItems: 1 + minItems: 1 + items: + - description: Main clock + - description: Register access clock + + clock-names: + minItems: 1 + items: + - const: aclk + - const: reg phys: maxItems: 1 @@ -43,7 +71,15 @@ properties: maxItems: 1 resets: - maxItems: 1 + minItems: 1 + items: + - description: Peripheral reset + - description: DRD reset + + reset-names: + items: + - const: aresetn_p + - const: drd_reset usb-role-switch: $ref: /schemas/types.yaml#/definitions/flag @@ -78,6 +114,39 @@ required: - interrupts - clocks +allOf: + - if: + properties: + compatible: + contains: + enum: + - renesas,rzv2m-usb3-peri + then: + properties: + clocks: + minItems: 2 + clock-names: + minItems: 2 + interrupts: + minItems: 4 + interrupt-names: + minItems: 4 + resets: + minItems: 2 + required: + - clock-names + - interrupt-names + - resets + - reset-names + else: + properties: + clocks: + maxItems: 1 + interrupts: + maxItems: 1 + resets: + maxItems: 1 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1999f614c89b29c7f6151df195a6a51cd4ca25b3 --- /dev/null +++ b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/usb/richtek,rt1711h.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Richtek RT1711H Type-C Port Switch and Power Delivery controller + +maintainers: + - Gene Chen + +description: | + The RT1711H is a USB Type-C controller that complies with the latest + USB Type-C and PD standards. It does the USB Type-C detection including attach + and orientation. It integrates the physical layer of the USB BMC power + delivery protocol to allow up to 100W of power. The BMC PD block enables full + support for alternative interfaces of the Type-C specification. + +properties: + compatible: + enum: + - richtek,rt1711h + - richtek,rt1715 + description: + RT1711H support PD20, RT1715 support PD30 except Fast Role Swap. + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + wakeup-source: + type: boolean + + connector: + type: object + $ref: /schemas/connector/usb-connector.yaml# + description: + Properties for usb c connector. + +additionalProperties: false + +required: + - compatible + - reg + - connector + - interrupts + +examples: + - | + #include + #include + i2c0 { + #address-cells = <1>; + #size-cells = <0>; + + rt1711h@4e { + compatible = "richtek,rt1711h"; + reg = <0x4e>; + interrupts-extended = <&gpio26 3 IRQ_TYPE_LEVEL_LOW>; + wakeup-source; + + connector { + compatible = "usb-c-connector"; + label = "USB-C"; + data-role = "dual"; + power-role = "dual"; + try-power-role = "sink"; + source-pdos = ; + sink-pdos = ; + op-sink-microwatt = <10000000>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + endpoint { + remote-endpoint = <&usb_hs>; + }; + }; + port@1 { + reg = <1>; + endpoint { + remote-endpoint = <&usb_ss>; + }; + }; + port@2 { + reg = <2>; + endpoint { + remote-endpoint = <&dp_aux>; + }; + }; + }; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/usb/smsc,usb3503.yaml b/Documentation/devicetree/bindings/usb/smsc,usb3503.yaml index 321b6f166197344475fa8da45f3581434cda58d3..a09f4528aea3c257042e7a9eb5c0ab52ca6cdea2 100644 --- a/Documentation/devicetree/bindings/usb/smsc,usb3503.yaml +++ b/Documentation/devicetree/bindings/usb/smsc,usb3503.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/smsc,usb3503.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: SMSC USB3503 High-Speed Hub Controller Device Tree Bindings +title: SMSC USB3503 High-Speed Hub Controller maintainers: - Dongjin Kim diff --git a/Documentation/devicetree/bindings/usb/snps,dwc3.yaml b/Documentation/devicetree/bindings/usb/snps,dwc3.yaml index 1779d08ba1c009f52eddf151f4fd3935df05b69e..6d78048c4613e0c2259c6e4e12453f73b51f358d 100644 --- a/Documentation/devicetree/bindings/usb/snps,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/snps,dwc3.yaml @@ -234,6 +234,18 @@ properties: avoid -EPROTO errors with usbhid on some devices (Hikey 970). type: boolean + snps,gfladj-refclk-lpm-sel-quirk: + description: + When set, run the SOF/ITP counter based on ref_clk. + type: boolean + + snps,resume-hs-terminations: + description: + Fix the issue of HS terminations CRC error on resume by enabling this + quirk. When set, all the termsel, xcvrsel, opmode becomes 0 during end + of resume. This option is to support certain legacy ULPI PHYs. + type: boolean + snps,is-utmi-l1-suspend: description: True when DWC3 asserts output signal utmi_l1_suspend_n, false when diff --git a/Documentation/devicetree/bindings/usb/st,stusb160x.yaml b/Documentation/devicetree/bindings/usb/st,stusb160x.yaml index b5a8c9814dd3e865b059ccc95584439533fcb69f..b8974807b666de2092531883538504e52da2603b 100644 --- a/Documentation/devicetree/bindings/usb/st,stusb160x.yaml +++ b/Documentation/devicetree/bindings/usb/st,stusb160x.yaml @@ -33,6 +33,7 @@ properties: connector: type: object $ref: /schemas/connector/usb-connector.yaml# + unevaluatedProperties: false properties: compatible: @@ -74,9 +75,14 @@ examples: data-role = "dual"; typec-power-opmode = "default"; - port { - typec_con_ep: endpoint { - remote-endpoint = <&usbotg_hs_ep>; + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec_con_ep: endpoint { + remote-endpoint = <&usbotg_hs_ep>; + }; }; }; }; diff --git a/Documentation/devicetree/bindings/usb/usb-drd.yaml b/Documentation/devicetree/bindings/usb/usb-drd.yaml index f229fc8068d999ca6bc58309e9a7ec6189c41aff..1567549b05ceca125c7b0bcbf5dab35bcbf44d9e 100644 --- a/Documentation/devicetree/bindings/usb/usb-drd.yaml +++ b/Documentation/devicetree/bindings/usb/usb-drd.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/usb-drd.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic USB OTG Controller Device Tree Bindings +title: Generic USB OTG Controller maintainers: - Greg Kroah-Hartman diff --git a/Documentation/devicetree/bindings/usb/usb-hcd.yaml b/Documentation/devicetree/bindings/usb/usb-hcd.yaml index 1dc3d5d7b44fe750276518dcb23adb11e2f4eddd..692dd60e3f73f321a20761f81e54d512953f9dba 100644 --- a/Documentation/devicetree/bindings/usb/usb-hcd.yaml +++ b/Documentation/devicetree/bindings/usb/usb-hcd.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/usb-hcd.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic USB Host Controller Device Tree Bindings +title: Generic USB Host Controller maintainers: - Greg Kroah-Hartman diff --git a/Documentation/devicetree/bindings/usb/usb-xhci.yaml b/Documentation/devicetree/bindings/usb/usb-xhci.yaml index 965f87fef70205abc41aacd3ac726ccca034be83..f2139a9f35fb8e944b5feff936be3db10f0d7e21 100644 --- a/Documentation/devicetree/bindings/usb/usb-xhci.yaml +++ b/Documentation/devicetree/bindings/usb/usb-xhci.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/usb-xhci.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic USB xHCI Controller Device Tree Bindings +title: Generic USB xHCI Controller maintainers: - Mathias Nyman diff --git a/Documentation/devicetree/bindings/usb/usb.yaml b/Documentation/devicetree/bindings/usb/usb.yaml index 939f217b8c7bd7f35ddcddcd37e6ab3fc0b5c552..326b14f05d1c419dc39bb5d95de5150c99c135f3 100644 --- a/Documentation/devicetree/bindings/usb/usb.yaml +++ b/Documentation/devicetree/bindings/usb/usb.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/usb/usb.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Generic USB Controller Device Tree Bindings +title: Generic USB Controller maintainers: - Greg Kroah-Hartman diff --git a/Documentation/devicetree/bindings/usb/willsemi,wusb3801.yaml b/Documentation/devicetree/bindings/usb/willsemi,wusb3801.yaml index c2b2243c789208397b200e19d1fe74823be2494f..5aa4ffd67119136ed5b2509a21f1e8b8a0b273b8 100644 --- a/Documentation/devicetree/bindings/usb/willsemi,wusb3801.yaml +++ b/Documentation/devicetree/bindings/usb/willsemi,wusb3801.yaml @@ -28,6 +28,7 @@ properties: connector: type: object $ref: ../connector/usb-connector.yaml# + unevaluatedProperties: false description: The managed USB Type-C connector. Since WUSB3801 does not support Power Delivery, the node should have the "pd-disable" property. diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 2f0151e9f6bef2c0e799229d11de27d07028fd5c..6e323a380294adef8592cdb4b4dfd956f87b3eb5 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -105,6 +105,8 @@ patternProperties: description: AMS-Taos Inc. "^analogix,.*": description: Analogix Semiconductor, Inc. + "^anbernic,.*": + description: Anbernic "^andestech,.*": description: Andes Technology Corporation "^anvo,.*": @@ -328,6 +330,8 @@ patternProperties: description: Digi International Inc. "^digilent,.*": description: Diglent, Inc. + "^diodes,.*": + description: Diodes, Inc. "^dioo,.*": description: Dioo Microcircuit Co., Ltd "^dlc,.*": @@ -777,6 +781,8 @@ patternProperties: description: MELFAS Inc. "^mellanox,.*": description: Mellanox Technologies + "^memsensing,.*": + description: MEMSensing Microsystems Co., Ltd. "^memsic,.*": description: MEMSIC Inc. "^menlo,.*": @@ -787,6 +793,8 @@ patternProperties: description: Cisco Meraki, LLC "^merrii,.*": description: Merrii Technology Co., Ltd. + "^methode,.*": + description: Methode Electronics, Inc. "^micrel,.*": description: Micrel Inc. "^microchip,.*": @@ -819,6 +827,8 @@ patternProperties: description: MNT Research GmbH "^modtronix,.*": description: Modtronix Engineering + "^moortec,.*": + description: Moortec Semiconductor Ltd. "^mosaixtech,.*": description: Mosaix Technologies, Inc. "^motorola,.*": @@ -925,6 +935,8 @@ patternProperties: description: On Tat Industrial Company "^opalkelly,.*": description: Opal Kelly Incorporated + "^openailab,.*": + description: openailab.com "^opencores,.*": description: OpenCores.org "^openembed,.*": diff --git a/Documentation/devicetree/bindings/virtio/iommu.txt b/Documentation/devicetree/bindings/virtio/iommu.txt deleted file mode 100644 index 2407fea0651c9232646506894582e205e761d91b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/virtio/iommu.txt +++ /dev/null @@ -1,66 +0,0 @@ -* virtio IOMMU PCI device - -When virtio-iommu uses the PCI transport, its programming interface is -discovered dynamically by the PCI probing infrastructure. However the -device tree statically describes the relation between IOMMU and DMA -masters. Therefore, the PCI root complex that hosts the virtio-iommu -contains a child node representing the IOMMU device explicitly. - -Required properties: - -- compatible: Should be "virtio,pci-iommu" -- reg: PCI address of the IOMMU. As defined in the PCI Bus - Binding reference [1], the reg property is a five-cell - address encoded as (phys.hi phys.mid phys.lo size.hi - size.lo). phys.hi should contain the device's BDF as - 0b00000000 bbbbbbbb dddddfff 00000000. The other cells - should be zero. -- #iommu-cells: Each platform DMA master managed by the IOMMU is assigned - an endpoint ID, described by the "iommus" property [2]. - For virtio-iommu, #iommu-cells must be 1. - -Notes: - -- DMA from the IOMMU device isn't managed by another IOMMU. Therefore the - virtio-iommu node doesn't have an "iommus" property, and is omitted from - the iommu-map property of the root complex. - -Example: - -pcie@10000000 { - compatible = "pci-host-ecam-generic"; - ... - - /* The IOMMU programming interface uses slot 00:01.0 */ - iommu0: iommu@0008 { - compatible = "virtio,pci-iommu"; - reg = <0x00000800 0 0 0 0>; - #iommu-cells = <1>; - }; - - /* - * The IOMMU manages all functions in this PCI domain except - * itself. Omit BDF 00:01.0. - */ - iommu-map = <0x0 &iommu0 0x0 0x8> - <0x9 &iommu0 0x9 0xfff7>; -}; - -pcie@20000000 { - compatible = "pci-host-ecam-generic"; - ... - /* - * The IOMMU also manages all functions from this domain, - * with endpoint IDs 0x10000 - 0x1ffff - */ - iommu-map = <0x0 &iommu0 0x10000 0x10000>; -}; - -ethernet@fe001000 { - ... - /* The IOMMU manages this platform device with endpoint ID 0x20000 */ - iommus = <&iommu0 0x20000>; -}; - -[1] Documentation/devicetree/bindings/pci/pci.txt -[2] Documentation/devicetree/bindings/iommu/iommu.txt diff --git a/Documentation/devicetree/bindings/virtio/pci-iommu.yaml b/Documentation/devicetree/bindings/virtio/pci-iommu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..972a785a42de50104cf16e996a5db797beb840e2 --- /dev/null +++ b/Documentation/devicetree/bindings/virtio/pci-iommu.yaml @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/virtio/pci-iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: virtio-iommu device using the virtio-pci transport + +maintainers: + - Jean-Philippe Brucker + +description: | + When virtio-iommu uses the PCI transport, its programming interface is + discovered dynamically by the PCI probing infrastructure. However the + device tree statically describes the relation between IOMMU and DMA + masters. Therefore, the PCI root complex that hosts the virtio-iommu + contains a child node representing the IOMMU device explicitly. + + DMA from the IOMMU device isn't managed by another IOMMU. Therefore the + virtio-iommu node doesn't have an "iommus" property, and is omitted from + the iommu-map property of the root complex. + +properties: + # If compatible is present, it should contain the vendor and device ID + # according to the PCI Bus Binding specification. Since PCI provides + # built-in identification methods, compatible is not actually required. + compatible: + oneOf: + - items: + - const: virtio,pci-iommu + - const: pci1af4,1057 + - items: + - const: pci1af4,1057 + + reg: + description: | + PCI address of the IOMMU. As defined in the PCI Bus Binding + reference, the reg property is a five-cell address encoded as (phys.hi + phys.mid phys.lo size.hi size.lo). phys.hi should contain the device's + BDF as 0b00000000 bbbbbbbb dddddfff 00000000. The other cells should be + zero. See Documentation/devicetree/bindings/pci/pci.txt + + '#iommu-cells': + const: 1 + +required: + - compatible + - reg + - '#iommu-cells' + +additionalProperties: false + +examples: + - | + bus { + #address-cells = <2>; + #size-cells = <2>; + + pcie@40000000 { + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + reg = <0x0 0x40000000 0x0 0x1000000>; + ranges = <0x02000000 0x0 0x41000000 0x0 0x41000000 0x0 0x0f000000>; + + /* + * The IOMMU manages all functions in this PCI domain except + * itself. Omit BDF 00:01.0. + */ + iommu-map = <0x0 &iommu0 0x0 0x8 + 0x9 &iommu0 0x9 0xfff7>; + + /* The IOMMU programming interface uses slot 00:01.0 */ + iommu0: iommu@1,0 { + compatible = "pci1af4,1057"; + reg = <0x800 0 0 0 0>; + #iommu-cells = <1>; + }; + }; + + pcie@50000000 { + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + reg = <0x0 0x50000000 0x0 0x1000000>; + ranges = <0x02000000 0x0 0x51000000 0x0 0x51000000 0x0 0x0f000000>; + + /* + * The IOMMU also manages all functions from this domain, + * with endpoint IDs 0x10000 - 0x1ffff + */ + iommu-map = <0x0 &iommu0 0x10000 0x10000>; + }; + + ethernet { + /* The IOMMU manages this platform device with endpoint ID 0x20000 */ + iommus = <&iommu0 0x20000>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/w1/w1-gpio.yaml b/Documentation/devicetree/bindings/w1/w1-gpio.yaml index 8eef2380161bac8e334d387c124dd633d174d3a5..b02b02237082b991c90dcf4c832ab0309baf12db 100644 --- a/Documentation/devicetree/bindings/w1/w1-gpio.yaml +++ b/Documentation/devicetree/bindings/w1/w1-gpio.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/w1/w1-gpio.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Bitbanged GPIO 1-Wire Bus Device Tree Bindings +title: Bitbanged GPIO 1-Wire Bus maintainers: - Daniel Mack diff --git a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml index ed6c1ca80dcc4ed7f37518c9b1aef5dab6e4a67c..026c2e5e77aaccef471692c7058b0b7e9e1a2c0c 100644 --- a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/watchdog/allwinner,sun4i-a10-wdt.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Allwinner A10 Watchdog Device Tree Bindings +title: Allwinner A10 Watchdog allOf: - $ref: "watchdog.yaml#" diff --git a/Documentation/devicetree/bindings/watchdog/amlogic,meson-gxbb-wdt.yaml b/Documentation/devicetree/bindings/watchdog/amlogic,meson-gxbb-wdt.yaml index c7459cf70e30341153e14baeedbaf468f980a6a9..497d60408ea04d7ba8ca49796790ef82f2926de2 100644 --- a/Documentation/devicetree/bindings/watchdog/amlogic,meson-gxbb-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/amlogic,meson-gxbb-wdt.yaml @@ -8,7 +8,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#" title: Meson GXBB SoCs Watchdog timer maintainers: - - Neil Armstrong + - Neil Armstrong allOf: - $ref: watchdog.yaml# diff --git a/Documentation/devicetree/bindings/watchdog/atmel,at91sam9-wdt.yaml b/Documentation/devicetree/bindings/watchdog/atmel,at91sam9-wdt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad27bc518670a4dbe27e88f974b1c8a2a4840aa9 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/atmel,at91sam9-wdt.yaml @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/atmel,at91sam9-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel Watchdog Timers + +maintainers: + - Eugen Hristev + +properties: + compatible: + const: atmel,at91sam9260-wdt + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + atmel,max-heartbeat-sec: + description: + Should contain the maximum heartbeat value in seconds. This value + should be less or equal to 16. It is used to compute the WDV field. + maximum: 16 + + atmel,min-heartbeat-sec: + description: + Should contain the minimum heartbeat value in seconds. This value + must be smaller than the max-heartbeat-sec value. It is used to + compute the WDD field. + maximum: 16 + + atmel,watchdog-type: + $ref: /schemas/types.yaml#/definitions/string + description: | + Should be hardware or software. + oneOf: + - description: + Hardware watchdog uses the at91 watchdog reset. + const: hardware + - description: | + Software watchdog uses the watchdog interrupt + to trigger a software reset. + const: software + default: hardware + + atmel,reset-type: + $ref: /schemas/types.yaml#/definitions/string + description: | + Should be proc or all. This is valid only when using hardware watchdog. + oneOf: + - description: + Assert peripherals and processor reset signals. + const: all + - description: + Assert the processor reset signal. + const: proc + default: all + + atmel,disable: + $ref: /schemas/types.yaml#/definitions/flag + description: + Should be present if you want to stop the watchdog. + + atmel,idle-halt: + $ref: /schemas/types.yaml#/definitions/flag + description: | + Should be present if you want to stop the watchdog when + entering idle state. + CAUTION: This property should be used with care, it actually makes the + watchdog not counting when the CPU is in idle state, therefore the + watchdog reset time depends on mean CPU usage and will not reset at all + if the CPU stops working while it is in idle state, which is probably + not what you want. + + atmel,dbg-halt: + $ref: /schemas/types.yaml#/definitions/flag + description: | + Should be present if you want to stop the watchdog when + entering debug state. + +required: + - compatible + - reg + - clocks + +allOf: + - $ref: watchdog.yaml# + - if: + properties: + atmel,reset-type: + enum: + - all + - proc + then: + properties: + atmel,watchdog-type: + const: hardware + +dependencies: + atmel,reset-type: ['atmel,watchdog-type'] + +unevaluatedProperties: false + +examples: + - | + #include + + watchdog@fffffd40 { + compatible = "atmel,at91sam9260-wdt"; + reg = <0xfffffd40 0x10>; + interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; + clocks = <&clk32k>; + timeout-sec = <15>; + atmel,watchdog-type = "hardware"; + atmel,reset-type = "all"; + atmel,dbg-halt; + atmel,idle-halt; + atmel,max-heartbeat-sec = <16>; + atmel,min-heartbeat-sec = <0>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/atmel-wdt.txt b/Documentation/devicetree/bindings/watchdog/atmel-wdt.txt deleted file mode 100644 index 711a880b3d3bf46ed5b5e67dc98ceed3f10d80b0..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/atmel-wdt.txt +++ /dev/null @@ -1,51 +0,0 @@ -* Atmel Watchdog Timers - -** at91sam9-wdt - -Required properties: -- compatible: must be "atmel,at91sam9260-wdt". -- reg: physical base address of the controller and length of memory mapped - region. -- clocks: phandle to input clock. - -Optional properties: -- timeout-sec: contains the watchdog timeout in seconds. -- interrupts : Should contain WDT interrupt. -- atmel,max-heartbeat-sec : Should contain the maximum heartbeat value in - seconds. This value should be less or equal to 16. It is used to - compute the WDV field. -- atmel,min-heartbeat-sec : Should contain the minimum heartbeat value in - seconds. This value must be smaller than the max-heartbeat-sec value. - It is used to compute the WDD field. -- atmel,watchdog-type : Should be "hardware" or "software". Hardware watchdog - use the at91 watchdog reset. Software watchdog use the watchdog - interrupt to trigger a software reset. -- atmel,reset-type : Should be "proc" or "all". - "all" : assert peripherals and processor reset signals - "proc" : assert the processor reset signal - This is valid only when using "hardware" watchdog. -- atmel,disable : Should be present if you want to disable the watchdog. -- atmel,idle-halt : Should be present if you want to stop the watchdog when - entering idle state. - CAUTION: This property should be used with care, it actually makes the - watchdog not counting when the CPU is in idle state, therefore the - watchdog reset time depends on mean CPU usage and will not reset at all - if the CPU stop working while it is in idle state, which is probably - not what you want. -- atmel,dbg-halt : Should be present if you want to stop the watchdog when - entering debug state. - -Example: - watchdog@fffffd40 { - compatible = "atmel,at91sam9260-wdt"; - reg = <0xfffffd40 0x10>; - interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; - clocks = <&clk32k>; - timeout-sec = <15>; - atmel,watchdog-type = "hardware"; - atmel,reset-type = "all"; - atmel,dbg-halt; - atmel,idle-halt; - atmel,max-heartbeat-sec = <16>; - atmel,min-heartbeat-sec = <0>; - }; diff --git a/Documentation/devicetree/bindings/watchdog/mediatek,mt7621-wdt.yaml b/Documentation/devicetree/bindings/watchdog/mediatek,mt7621-wdt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2b17fdf4e3981400d1ff724c3c577888e52e9e5 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/mediatek,mt7621-wdt.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/mediatek,mt7621-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ralink Watchdog Timers + +maintainers: + - Sergio Paracuellos + +allOf: + - $ref: watchdog.yaml# + +properties: + compatible: + const: mediatek,mt7621-wdt + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + watchdog@100 { + compatible = "mediatek,mt7621-wdt"; + reg = <0x100 0x100>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/mstar,msc313e-wdt.yaml b/Documentation/devicetree/bindings/watchdog/mstar,msc313e-wdt.yaml index e3e8b86dbf6379678e61628498e453b9963b5f74..33794711c594542e31c17d498ae0816690e15647 100644 --- a/Documentation/devicetree/bindings/watchdog/mstar,msc313e-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/mstar,msc313e-wdt.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/watchdog/mstar,msc313e-wdt.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: MStar Watchdog Device Tree Bindings +title: MStar Watchdog maintainers: - Daniel Palmer diff --git a/Documentation/devicetree/bindings/watchdog/mt7621-wdt.txt b/Documentation/devicetree/bindings/watchdog/mt7621-wdt.txt deleted file mode 100644 index c15ef0ef609f3f26e6060e91f7c3ff0ca8631466..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/mt7621-wdt.txt +++ /dev/null @@ -1,12 +0,0 @@ -Ralink Watchdog Timers - -Required properties: -- compatible: must be "mediatek,mt7621-wdt" -- reg: physical base address of the controller and length of the register range - -Example: - - watchdog@100 { - compatible = "mediatek,mt7621-wdt"; - reg = <0x100 0x10>; - }; diff --git a/Documentation/devicetree/bindings/watchdog/of-xilinx-wdt.txt b/Documentation/devicetree/bindings/watchdog/of-xilinx-wdt.txt deleted file mode 100644 index c6ae9c9d5e3e2c13a640d48d2af3d79b8aa90388..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/of-xilinx-wdt.txt +++ /dev/null @@ -1,26 +0,0 @@ -Xilinx AXI/PLB soft-core watchdog Device Tree Bindings ---------------------------------------------------------- - -Required properties: -- compatible : Should be "xlnx,xps-timebase-wdt-1.00.a" or - "xlnx,xps-timebase-wdt-1.01.a". -- reg : Physical base address and size - -Optional properties: -- clocks : Input clock specifier. Refer to common clock - bindings. -- clock-frequency : Frequency of clock in Hz -- xlnx,wdt-enable-once : 0 - Watchdog can be restarted - 1 - Watchdog can be enabled just once -- xlnx,wdt-interval : Watchdog timeout interval in 2^ clock cycles, - is integer from 8 to 31. - -Example: -axi-timebase-wdt@40100000 { - clock-frequency = <50000000>; - compatible = "xlnx,xps-timebase-wdt-1.00.a"; - clocks = <&clkc 15>; - reg = <0x40100000 0x10000>; - xlnx,wdt-enable-once = <0x0>; - xlnx,wdt-interval = <0x1b>; -} ; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml index a8d7dde5271b8f2fba4a159bb4db0f42756c1463..26b1815a6753a81a1de6ab194f22002d0eca0f29 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml @@ -31,6 +31,11 @@ properties: - renesas,r9a07g054-wdt # RZ/V2L - const: renesas,rzg2l-wdt + - items: + - enum: + - renesas,r9a09g011-wdt # RZ/V2M + - const: renesas,rzv2m-wdt # RZ/V2M + - items: - enum: - renesas,r8a7742-wdt # RZ/G1H @@ -65,18 +70,35 @@ properties: - enum: - renesas,r8a779a0-wdt # R-Car V3U - renesas,r8a779f0-wdt # R-Car S4-8 + - renesas,r8a779g0-wdt # R-Car V4H - const: renesas,rcar-gen4-wdt # R-Car Gen4 reg: maxItems: 1 - interrupts: true - - interrupt-names: true - - clocks: true - - clock-names: true + interrupts: + minItems: 1 + items: + - description: Timeout + - description: Parity error + + interrupt-names: + minItems: 1 + items: + - const: wdt + - const: perrout + + clocks: + minItems: 1 + items: + - description: Register access clock + - description: Main clock + + clock-names: + minItems: 1 + items: + - const: pclk + - const: oscclk power-domains: maxItems: 1 @@ -89,6 +111,7 @@ properties: required: - compatible - reg + - interrupts - clocks allOf: @@ -113,31 +136,38 @@ allOf: contains: enum: - renesas,rzg2l-wdt + - renesas,rzv2m-wdt then: properties: - interrupts: - maxItems: 2 - interrupt-names: - items: - - const: wdt - - const: perrout clocks: - items: - - description: Register access clock - - description: Main clock + minItems: 2 clock-names: - items: - - const: pclk - - const: oscclk + minItems: 2 required: - clock-names + else: + properties: + clocks: + maxItems: 1 + + - if: + properties: + compatible: + contains: + enum: + - renesas,rzg2l-wdt + then: + properties: + interrupts: + minItems: 2 + interrupt-names: + minItems: 2 + required: - interrupt-names else: properties: interrupts: maxItems: 1 - clocks: - maxItems: 1 additionalProperties: false @@ -145,9 +175,11 @@ examples: - | #include #include + #include wdt0: watchdog@e6020000 { compatible = "renesas,r8a7795-wdt", "renesas,rcar-gen3-wdt"; reg = <0xe6020000 0x0c>; + interrupts = ; clocks = <&cpg CPG_MOD 402>; power-domains = <&sysc R8A7795_PD_ALWAYS_ON>; resets = <&cpg 402>; diff --git a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml index b08373336b161f630df056a447dd81231596357b..8fb6656ba0c28d233a63b161c027f3e52fc80113 100644 --- a/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/samsung-wdt.yaml @@ -23,6 +23,7 @@ properties: - samsung,exynos5420-wdt # for Exynos5420 - samsung,exynos7-wdt # for Exynos7 - samsung,exynos850-wdt # for Exynos850 + - samsung,exynosautov9-wdt # for Exynosautov9 reg: maxItems: 1 @@ -67,6 +68,7 @@ allOf: - samsung,exynos5420-wdt - samsung,exynos7-wdt - samsung,exynos850-wdt + - samsung,exynosautov9-wdt then: required: - samsung,syscon-phandle @@ -76,6 +78,7 @@ allOf: contains: enum: - samsung,exynos850-wdt + - samsung,exynosautov9-wdt then: properties: clocks: diff --git a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml index 6461eb4f4a278f498be2905583e3c29ea4296f7e..92df6e453f64b626d9d456adb3cc5ef9e9b6f2a0 100644 --- a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml @@ -20,6 +20,7 @@ properties: - enum: - rockchip,px30-wdt - rockchip,rk3066-wdt + - rockchip,rk3128-wdt - rockchip,rk3188-wdt - rockchip,rk3228-wdt - rockchip,rk3288-wdt diff --git a/Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml b/Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml index 690e19ce4b87857b509934847738153f631f4149..eba083822d1fbe48e3d7628a3813594c5b624a02 100644 --- a/Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml @@ -35,20 +35,16 @@ additionalProperties: false examples: - | + #include + soc { #address-cells = <2>; #size-cells = <2>; - wdt_clk: wdt-clk { - compatible = "fixed-clock"; - clock-frequency = <150000000>; - #clock-cells = <0>; - }; - - watchdog@28330000 { + wdt: watchdog@28330000 { compatible = "toshiba,visconti-wdt"; reg = <0 0x28330000 0 0x1000>; - clocks = <&wdt_clk>; timeout-sec = <20>; + clocks = <&pismu TMPV770X_CLK_WDTCLK>; }; }; diff --git a/Documentation/devicetree/bindings/watchdog/xlnx,xps-timebase-wdt.yaml b/Documentation/devicetree/bindings/watchdog/xlnx,xps-timebase-wdt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..493a1c954707764dc5cb11dcf11ee9a26d53a102 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/xlnx,xps-timebase-wdt.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: GPL-2.0-or-later OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/xlnx,xps-timebase-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Xilinx AXI/PLB softcore and window Watchdog Timer + +maintainers: + - Shubhrajyoti Datta + - Srinivas Neeli + +description: + The Timebase watchdog timer(WDT) is a free-running 32 bit counter. + WDT uses a dual-expiration architecture. After one expiration of + the timeout interval, an interrupt is generated and the WDT state + bit is set to one in the status register. If the state bit is not + cleared (by writing a one to the state bit) before the next + expiration of the timeout interval, a WDT reset is generated. + +allOf: + - $ref: watchdog.yaml# + +properties: + compatible: + enum: + - xlnx,xps-timebase-wdt-1.01.a + - xlnx,xps-timebase-wdt-1.00.a + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-frequency: + description: Frequency of clock in Hz + + xlnx,wdt-interval: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Watchdog timeout interval + minimum: 8 + maximum: 32 + + xlnx,wdt-enable-once: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1] + description: If watchdog is configured as enable once, + then the watchdog cannot be disabled after + it has been enabled. + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + watchdog@40100000 { + compatible = "xlnx,xps-timebase-wdt-1.00.a"; + reg = <0x40100000 0x1000>; + clock-frequency = <50000000>; + clocks = <&clkc 15>; + xlnx,wdt-enable-once = <0x0>; + xlnx,wdt-interval = <0x1b>; + }; +... diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst index 9c779bd7a751d37fb9c739bb86a80c81059155b5..1dcbd7332476f404359633d3db560f37736df3d5 100644 --- a/Documentation/doc-guide/kernel-doc.rst +++ b/Documentation/doc-guide/kernel-doc.rst @@ -14,6 +14,9 @@ when it is embedded in source files. reasons. The kernel source contains tens of thousands of kernel-doc comments. Please stick to the style described here. +.. note:: kernel-doc does not cover Rust code: please see + Documentation/rust/general-information.rst instead. + The kernel-doc structure is extracted from the comments, and proper `Sphinx C Domain`_ function and type descriptions with anchors are generated from them. The descriptions are filtered for special kernel-doc diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index 1228b85f6f7718d3fd6a57d91d28bc5769957e88..c708cec889afd9697e7f297ce27ceddb4ed07462 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -48,10 +48,6 @@ or ``virtualenv``, depending on how your distribution packaged Python 3. on the Sphinx version, it should be installed separately, with ``pip install sphinx_rtd_theme``. - #) Some ReST pages contain math expressions. Due to the way Sphinx works, - those expressions are written using LaTeX notation. It needs texlive - installed with amsfonts and amsmath in order to evaluate them. - In summary, if you want to install Sphinx version 2.4.4, you should do:: $ virtualenv sphinx_2.4.4 @@ -86,6 +82,27 @@ Depending on the distribution, you may also need to install a series of ``texlive`` packages that provide the minimal set of functionalities required for ``XeLaTeX`` to work. +Math Expressions in HTML +------------------------ + +Some ReST pages contain math expressions. Due to the way Sphinx works, +those expressions are written using LaTeX notation. +There are two options for Sphinx to render math expressions in html output. +One is an extension called `imgmath`_ which converts math expressions into +images and embeds them in html pages. +The other is an extension called `mathjax`_ which delegates math rendering +to JavaScript capable web browsers. +The former was the only option for pre-6.1 kernel documentation and it +requires quite a few texlive packages including amsfonts and amsmath among +others. + +Since kernel release 6.1, html pages with math expressions can be built +without installing any texlive packages. See `Choice of Math Renderer`_ for +further info. + +.. _imgmath: https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.imgmath +.. _mathjax: https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax + .. _sphinx-pre-install: Checking for Sphinx dependencies @@ -164,6 +181,38 @@ To remove the generated documentation, run ``make cleandocs``. as well would improve the quality of images embedded in PDF documents, especially for kernel releases 5.18 and later. +Choice of Math Renderer +----------------------- + +Since kernel release 6.1, mathjax works as a fallback math renderer for +html output.\ [#sph1_8]_ + +Math renderer is chosen depending on available commands as shown below: + +.. table:: Math Renderer Choices for HTML + + ============= ================= ============ + Math renderer Required commands Image format + ============= ================= ============ + imgmath latex, dvipng PNG (raster) + mathjax + ============= ================= ============ + +The choice can be overridden by setting an environment variable +``SPHINX_IMGMATH`` as shown below: + +.. table:: Effect of Setting ``SPHINX_IMGMATH`` + + ====================== ======== + Setting Renderer + ====================== ======== + ``SPHINX_IMGMATH=yes`` imgmath + ``SPHINX_IMGMATH=no`` mathjax + ====================== ======== + +.. [#sph1_8] Fallback of math renderer requires Sphinx >=1.8. + + Writing Documentation ===================== diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 55272942e721a8a70b80cc0598f16e33f9ca186b..687adb58048ec203239069bef32475634d2f7a88 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -301,6 +301,7 @@ IO region devm_release_region() devm_release_resource() devm_request_mem_region() + devm_request_free_mem_region() devm_request_region() devm_request_resource() @@ -310,7 +311,6 @@ IOMAP devm_ioremap() devm_ioremap_uc() devm_ioremap_wc() - devm_ioremap_np() devm_ioremap_resource() : checks resource, requests memory region, ioremaps devm_ioremap_resource_wc() devm_platform_ioremap_resource() : calls devm_ioremap_resource() for platform device @@ -334,7 +334,7 @@ IRQ devm_irq_alloc_descs_from() devm_irq_alloc_generic_chip() devm_irq_setup_generic_chip() - devm_irq_sim_init() + devm_irq_domain_create_sim() LED devm_led_classdev_register() @@ -392,7 +392,9 @@ PHY PINCTRL devm_pinctrl_get() devm_pinctrl_put() + devm_pinctrl_get_select() devm_pinctrl_register() + devm_pinctrl_register_and_init() devm_pinctrl_unregister() POWER @@ -401,14 +403,24 @@ POWER PWM devm_pwm_get() - devm_of_pwm_get() devm_fwnode_pwm_get() REGULATOR + devm_regulator_bulk_register_supply_alias() devm_regulator_bulk_get() + devm_regulator_bulk_get_enable() + devm_regulator_bulk_put() devm_regulator_get() + devm_regulator_get_enable() + devm_regulator_get_enable_optional() + devm_regulator_get_exclusive() + devm_regulator_get_optional() + devm_regulator_irq_helper() devm_regulator_put() devm_regulator_register() + devm_regulator_register_notifier() + devm_regulator_register_supply_alias() + devm_regulator_unregister_notifier() RESET devm_reset_control_get() @@ -427,6 +439,8 @@ SLAVE DMA ENGINE devm_acpi_dma_controller_register() SPI + devm_spi_alloc_master() + devm_spi_alloc_slave() devm_spi_register_master() WATCHDOG diff --git a/Documentation/driver-api/isa.rst b/Documentation/driver-api/isa.rst index def4a7b690b56cc21e4c1657b6b8006de5197b2b..3df1b16965241265e5f3a49a6c316858eadf70af 100644 --- a/Documentation/driver-api/isa.rst +++ b/Documentation/driver-api/isa.rst @@ -100,7 +100,7 @@ I believe platform_data is available for this, but if rather not, moving the isa_driver pointer to the private struct isa_dev is ofcourse fine as well. -Then, if the the driver did not provide a .match, it matches. If it did, +Then, if the driver did not provide a .match, it matches. If it did, the driver match() method is called to determine a match. If it did **not** match, dev->platform_data is reset to indicate this to diff --git a/Documentation/driver-api/media/drivers/pxa_camera.rst b/Documentation/driver-api/media/drivers/pxa_camera.rst index ee1bd96b66dd96975764dd6e2c2fd98539bd3687..46919919baac02395dec9e322a050fedbe31ed71 100644 --- a/Documentation/driver-api/media/drivers/pxa_camera.rst +++ b/Documentation/driver-api/media/drivers/pxa_camera.rst @@ -19,7 +19,7 @@ Global video workflow a) QCI stopped Initially, the QCI interface is stopped. - When a buffer is queued (pxa_videobuf_ops->buf_queue), the QCI starts. + When a buffer is queued, start_streaming is called and the QCI starts. b) QCI started More buffers can be queued while the QCI is started without halting the diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst index fd26c3d895b67619e13e1d6ddb79458ef4931f9d..8c71a2055d2781da4b011522d89a1fdec247e1f7 100644 --- a/Documentation/driver-api/pwm.rst +++ b/Documentation/driver-api/pwm.rst @@ -40,8 +40,7 @@ after usage with pwm_free(). New users should use the pwm_get() function and pass to it the consumer device or a consumer name. pwm_put() is used to free the PWM device. Managed -variants of the getter, devm_pwm_get(), devm_of_pwm_get(), -devm_fwnode_pwm_get(), also exist. +variants of the getter, devm_pwm_get() and devm_fwnode_pwm_get(), also exist. After being requested, a PWM has to be configured using:: diff --git a/Documentation/driver-api/vfio-mediated-device.rst b/Documentation/driver-api/vfio-mediated-device.rst index f47dca6645aae6aa483a87ce03914ab0c340d049..fdf7d69378ec4ad75aa3329d6e909be345673e65 100644 --- a/Documentation/driver-api/vfio-mediated-device.rst +++ b/Documentation/driver-api/vfio-mediated-device.rst @@ -58,19 +58,19 @@ devices as examples, as these devices are the first devices to use this module:: | MDEV CORE | | MODULE | | mdev.ko | - | +-----------+ | mdev_register_device() +--------------+ + | +-----------+ | mdev_register_parent() +--------------+ | | | +<------------------------+ | | | | | | nvidia.ko |<-> physical | | | +------------------------>+ | device | | | | callbacks +--------------+ | | Physical | | - | | device | | mdev_register_device() +--------------+ + | | device | | mdev_register_parent() +--------------+ | | interface | |<------------------------+ | | | | | | i915.ko |<-> physical | | | +------------------------>+ | device | | | | callbacks +--------------+ | | | | - | | | | mdev_register_device() +--------------+ + | | | | mdev_register_parent() +--------------+ | | | +<------------------------+ | | | | | | ccw_device.ko|<-> physical | | | +------------------------>+ | device @@ -103,7 +103,8 @@ structure to represent a mediated device's driver:: struct mdev_driver { int (*probe) (struct mdev_device *dev); void (*remove) (struct mdev_device *dev); - struct attribute_group **supported_type_groups; + unsigned int (*get_available)(struct mdev_type *mtype); + ssize_t (*show_description)(struct mdev_type *mtype, char *buf); struct device_driver driver; }; @@ -125,8 +126,8 @@ vfio_device_ops. When a driver wants to add the GUID creation sysfs to an existing device it has probe'd to then it should call:: - int mdev_register_device(struct device *dev, - struct mdev_driver *mdev_driver); + int mdev_register_parent(struct mdev_parent *parent, struct device *dev, + struct mdev_driver *mdev_driver); This will provide the 'mdev_supported_types/XX/create' files which can then be used to trigger the creation of a mdev_device. The created mdev_device will be @@ -134,7 +135,7 @@ attached to the specified driver. When the driver needs to remove itself it calls:: - void mdev_unregister_device(struct device *dev); + void mdev_unregister_parent(struct mdev_parent *parent); Which will unbind and destroy all the created mdevs and remove the sysfs files. @@ -200,17 +201,14 @@ Directories and files under the sysfs for Each Physical Device sprintf(buf, "%s-%s", dev_driver_string(parent->dev), group->name); - (or using mdev_parent_dev(mdev) to arrive at the parent device outside - of the core mdev code) - * device_api - This attribute should show which device API is being created, for example, + This attribute shows which device API is being created, for example, "vfio-pci" for a PCI device. * available_instances - This attribute should show the number of devices of type that can be + This attribute shows the number of devices of type that can be created. * [device] @@ -220,11 +218,11 @@ Directories and files under the sysfs for Each Physical Device * name - This attribute should show human readable name. This is optional attribute. + This attribute shows a human readable name. * description - This attribute should show brief features/description of the type. This is + This attribute can show brief features/description of the type. This is an optional attribute. Directories and Files Under the sysfs for Each mdev Device diff --git a/Documentation/fault-injection/notifier-error-inject.rst b/Documentation/fault-injection/notifier-error-inject.rst index 1668b6e48d3a229b3e896433b9cc7ec0f26bcd9d..fdf2dc433eadf11ca8754318f839be8ae72f9b47 100644 --- a/Documentation/fault-injection/notifier-error-inject.rst +++ b/Documentation/fault-injection/notifier-error-inject.rst @@ -91,8 +91,8 @@ For more usage examples There are tools/testing/selftests using the notifier error injection features for CPU and memory notifiers. - * tools/testing/selftests/cpu-hotplug/on-off-test.sh - * tools/testing/selftests/memory-hotplug/on-off-test.sh + * tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh + * tools/testing/selftests/memory-hotplug/mem-on-off-test.sh These scripts first do simple online and offline tests and then do fault injection tests if notifier error injection module is available. diff --git a/Documentation/fb/udlfb.rst b/Documentation/fb/udlfb.rst index 732b37db350483cf01f43a0ea24a341fadb863f1..99cfbb7a1922387111a6e5141167212d6d5bd195 100644 --- a/Documentation/fb/udlfb.rst +++ b/Documentation/fb/udlfb.rst @@ -86,17 +86,24 @@ Module Options Special configuration for udlfb is usually unnecessary. There are a few options, however. -From the command line, pass options to modprobe -modprobe udlfb fb_defio=0 console=1 shadow=1 +From the command line, pass options to modprobe:: -Or modify options on the fly at /sys/module/udlfb/parameters directory via -sudo nano fb_defio -change the parameter in place, and save the file. + modprobe udlfb fb_defio=0 console=1 shadow=1 -Unplug/replug USB device to apply with new settings +Or change options on the fly by editing +/sys/module/udlfb/parameters/PARAMETER_NAME :: -Or for permanent option, create file like /etc/modprobe.d/udlfb.conf with text -options udlfb fb_defio=0 console=1 shadow=1 + cd /sys/module/udlfb/parameters + ls # to see a list of parameter names + sudo nano PARAMETER_NAME + # change the parameter in place, and save the file. + +Unplug/replug USB device to apply with new settings. + +Or to apply options permanently, create a modprobe configuration file +like /etc/modprobe.d/udlfb.conf with text:: + + options udlfb fb_defio=0 console=1 shadow=1 Accepted boolean options: diff --git a/Documentation/filesystems/caching/backend-api.rst b/Documentation/filesystems/caching/backend-api.rst index d7507becf67478da56deb15c60bf306e1a40e8df..3a199fc50828b77a390a8e9173f9304991feccfb 100644 --- a/Documentation/filesystems/caching/backend-api.rst +++ b/Documentation/filesystems/caching/backend-api.rst @@ -122,7 +122,7 @@ volumes, calling:: to tell fscache that a volume has been withdrawn. This waits for all outstanding accesses on the volume to complete before returning. -When the the cache is completely withdrawn, fscache should be notified by +When the cache is completely withdrawn, fscache should be notified by calling:: void fscache_relinquish_cache(struct fscache_cache *cache); diff --git a/Documentation/filesystems/ceph.rst b/Documentation/filesystems/ceph.rst index 4942e018db855e0733a8bb9445bb5c9e2bc2df2c..76ce938e702445ac43aadbc2ef65d99771dc8c28 100644 --- a/Documentation/filesystems/ceph.rst +++ b/Documentation/filesystems/ceph.rst @@ -203,7 +203,6 @@ For more information on Ceph, see the home page at The Linux kernel client source tree is available at - https://github.com/ceph/ceph-client.git - - git://git.kernel.org/pub/scm/linux/kernel/git/sage/ceph-client.git and the source for the full system is at https://github.com/ceph/ceph.git diff --git a/Documentation/filesystems/cifs/ksmbd.rst b/Documentation/filesystems/cifs/ksmbd.rst index 1af600db2e708eeb229d90efbaa04fa060c6b345..7bed96d794fc2656d3bb9d69cbca3feef97ea6c8 100644 --- a/Documentation/filesystems/cifs/ksmbd.rst +++ b/Documentation/filesystems/cifs/ksmbd.rst @@ -118,26 +118,44 @@ ksmbd/nfsd interoperability Planned for future. The features that ksmbd How to run ========== -1. Download ksmbd-tools and compile them. - - https://github.com/cifsd-team/ksmbd-tools +1. Download ksmbd-tools(https://github.com/cifsd-team/ksmbd-tools/releases) and + compile them. -2. Create user/password for SMB share. + - Refer README(https://github.com/cifsd-team/ksmbd-tools/blob/master/README.md) + to know how to use ksmbd.mountd/adduser/addshare/control utils - # mkdir /etc/ksmbd/ - # ksmbd.adduser -a + $ ./autogen.sh + $ ./configure --with-rundir=/run + $ make && sudo make install -3. Create /etc/ksmbd/smb.conf file, add SMB share in smb.conf file - - Refer smb.conf.example and - https://github.com/cifsd-team/ksmbd-tools/blob/master/Documentation/configuration.txt +2. Create /usr/local/etc/ksmbd/ksmbd.conf file, add SMB share in ksmbd.conf file. -4. Insert ksmbd.ko module + - Refer ksmbd.conf.example in ksmbd-utils, See ksmbd.conf manpage + for details to configure shares. - # insmod ksmbd.ko + $ man ksmbd.conf + +3. Create user/password for SMB share. + + - See ksmbd.adduser manpage. + + $ man ksmbd.adduser + $ sudo ksmbd.adduser -a + +4. Insert ksmbd.ko module after build your kernel. No need to load module + if ksmbd is built into the kernel. + + - Set ksmbd in menuconfig(e.g. $ make menuconfig) + [*] Network File Systems ---> + SMB3 server support (EXPERIMENTAL) + + $ sudo modprobe ksmbd.ko 5. Start ksmbd user space daemon - # ksmbd.mountd -6. Access share from Windows or Linux using CIFS + $ sudo ksmbd.mountd + +6. Access share from Windows or Linux using SMB3 client (cifs.ko or smbclient of samba) Shutdown KSMBD ============== diff --git a/Documentation/filesystems/ext4/super.rst b/Documentation/filesystems/ext4/super.rst index 268888522e35cbc5e39c4eecb7923d96c0ca02e0..0152888cac29b0bc0f9dc80fc3f63ae4f1bebe6e 100644 --- a/Documentation/filesystems/ext4/super.rst +++ b/Documentation/filesystems/ext4/super.rst @@ -456,15 +456,15 @@ The ext4 superblock is laid out as follows in * - 0x277 - __u8 - s_lastcheck_hi - - Upper 8 bits of the s_lastcheck_hi field. + - Upper 8 bits of the s_lastcheck field. * - 0x278 - __u8 - s_first_error_time_hi - - Upper 8 bits of the s_first_error_time_hi field. + - Upper 8 bits of the s_first_error_time field. * - 0x279 - __u8 - s_last_error_time_hi - - Upper 8 bits of the s_last_error_time_hi field. + - Upper 8 bits of the s_last_error_time field. * - 0x27A - __u8 - s_pad[2] diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst index d0c09663dae859c04a50eceb89e786b2cc0e3329..17df9a02ccff14a17be195730ff5ce71ffeb79ae 100644 --- a/Documentation/filesystems/f2fs.rst +++ b/Documentation/filesystems/f2fs.rst @@ -286,9 +286,8 @@ compress_algorithm=%s:%d Control compress algorithm and its compress level, now, algorithm level range lz4 3 - 16 zstd 1 - 22 -compress_log_size=%u Support configuring compress cluster size, the size will - be 4KB * (1 << %u), 16KB is minimum size, also it's - default size. +compress_log_size=%u Support configuring compress cluster size. The size will + be 4KB * (1 << %u). The default and minimum sizes are 16KB. compress_extension=%s Support adding specified extension, so that f2fs can enable compression on those corresponding files, e.g. if all files with '.ext' has high compression rate, we can set the '.ext' diff --git a/Documentation/filesystems/idmappings.rst b/Documentation/filesystems/idmappings.rst index c1db8748389c2674369ad3f529147cc95f592c67..b9b31066aef2ce828f0b3288c78ced482a23b85b 100644 --- a/Documentation/filesystems/idmappings.rst +++ b/Documentation/filesystems/idmappings.rst @@ -661,7 +661,7 @@ idmappings:: mount idmapping: u0:k10000:r10000 Assume a file owned by ``u1000`` is read from disk. The filesystem maps this id -to ``k21000`` according to it's idmapping. This is what is stored in the +to ``k21000`` according to its idmapping. This is what is stored in the inode's ``i_uid`` and ``i_gid`` fields. When the caller queries the ownership of this file via ``stat()`` the kernel diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 4bb2627026ec8342d25c69a64d32e0f5f68f7f50..8f737e76935ce341972fac30eecfd81e90fc43cb 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -79,7 +79,8 @@ prototypes:: int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct inode *, struct dentry *, umode_t); + int (*tmpfile) (struct user_namespace *, struct inode *, + struct file *, umode_t); int (*fileattr_set)(struct user_namespace *mnt_userns, struct dentry *dentry, struct fileattr *fa); int (*fileattr_get)(struct dentry *dentry, struct fileattr *fa); diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index aee9aaf9f3df78bd578e87c8ce845da74abc9a86..df0dc37e6f582d6c5c11f0de22871419cc40f765 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -922,3 +922,24 @@ is provided - file_open_root_mnt(). In-tree users adjusted. no_llseek is gone; don't set .llseek to that - just leave it NULL instead. Checks for "does that file have llseek(2), or should it fail with ESPIPE" should be done by looking at FMODE_LSEEK in file->f_mode. + +--- + +*mandatory* + +filldir_t (readdir callbacks) calling conventions have changed. Instead of +returning 0 or -E... it returns bool now. false means "no more" (as -E... used +to) and true - "keep going" (as 0 in old calling conventions). Rationale: +callers never looked at specific -E... values anyway. ->iterate() and +->iterate_shared() instance require no changes at all, all filldir_t ones in +the tree converted. + +--- + +**mandatory** + +Calling conventions for ->tmpfile() have changed. It now takes a struct +file pointer instead of struct dentry pointer. d_tmpfile() is similarly +changed to simplify callers. The passed file is in a non-open state and on +success must be opened before returning (e.g. by calling +finish_open_simple()). diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index e7aafc82be99917748b1cae12cc58f8bdeb07ab6..898c99eae8e44630b459a2adca8b84ebb1423d8f 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -982,6 +982,7 @@ Example output. You may not have all of these fields. SUnreclaim: 142336 kB KernelStack: 11168 kB PageTables: 20540 kB + SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB @@ -1090,6 +1091,9 @@ KernelStack Memory consumed by the kernel stacks of all tasks PageTables Memory consumed by userspace page tables +SecPageTables + Memory consumed by secondary page tables, this currently + currently includes KVM mmu allocations on x86 and arm64. NFS_Unstable Always zero. Previous counted pages which had been written to the server, but has not been committed to stable storage. diff --git a/Documentation/filesystems/qnx6.rst b/Documentation/filesystems/qnx6.rst index fd13433d362c94ac46792d3363525248387abfa6..523b798f04e75f3084cfc28b2d635e697b5aeb93 100644 --- a/Documentation/filesystems/qnx6.rst +++ b/Documentation/filesystems/qnx6.rst @@ -176,7 +176,7 @@ Then userspace. The requirement for a static, fixed preallocated system area comes from how qnx6fs deals with writes. -Each superblock got it's own half of the system area. So superblock #1 +Each superblock got its own half of the system area. So superblock #1 always uses blocks from the lower half while superblock #2 just writes to blocks represented by the upper half bitmap system area bits. diff --git a/Documentation/filesystems/spufs/spufs.rst b/Documentation/filesystems/spufs/spufs.rst index 8a42859bb10083fc471f394b8c150f114a35e6b2..ca0441cbe37e7a9df510bd2134f7883a9303bcf6 100644 --- a/Documentation/filesystems/spufs/spufs.rst +++ b/Documentation/filesystems/spufs/spufs.rst @@ -227,7 +227,7 @@ Files from the data buffer, updating the value of the specified signal notification register. The signal notification register will either be replaced with the input data or will be updated to the - bitwise OR or the old value and the input data, depending on the + bitwise OR of the old value and the input data, depending on the contents of the signal1_type, or signal2_type respectively, file. diff --git a/Documentation/filesystems/sysfs.rst b/Documentation/filesystems/sysfs.rst index 004d490179f32ee5d0fe43183ec50bc4a684a145..8bba676b1365d72ad2af0ee3ed13710e5e92db6c 100644 --- a/Documentation/filesystems/sysfs.rst +++ b/Documentation/filesystems/sysfs.rst @@ -263,7 +263,7 @@ A very simple (and naive) implementation of a device attribute is:: static ssize_t show_name(struct device *dev, struct device_attribute *attr, char *buf) { - return scnprintf(buf, PAGE_SIZE, "%s\n", dev->name); + return sysfs_emit(buf, "%s\n", dev->name); } static ssize_t store_name(struct device *dev, struct device_attribute *attr, diff --git a/Documentation/filesystems/ubifs.rst b/Documentation/filesystems/ubifs.rst index e6ee99762534545b9009a7683e16bc4004c96074..ced2f7679ddb5432d6c5bf99868194ee8ce59c8d 100644 --- a/Documentation/filesystems/ubifs.rst +++ b/Documentation/filesystems/ubifs.rst @@ -59,7 +59,7 @@ differences. * JFFS2 is a write-through file-system, while UBIFS supports write-back, which makes UBIFS much faster on writes. -Similarly to JFFS2, UBIFS supports on-the-flight compression which makes +Similarly to JFFS2, UBIFS supports on-the-fly compression which makes it possible to fit quite a lot of data to the flash. Similarly to JFFS2, UBIFS is tolerant of unclean reboots and power-cuts. diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 6cd6953e175b360c9ee7c70cb0a45d5b2b531516..2b55f71e2ae1955b876db342cc7d1225a2afa18a 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -274,6 +274,9 @@ or bottom half). This is specifically for the inode itself being marked dirty, not its data. If the update needs to be persisted by fdatasync(), then I_DIRTY_DATASYNC will be set in the flags argument. + I_DIRTY_TIME will be set in the flags in case lazytime is enabled + and struct inode has times updated since the last ->dirty_inode + call. ``write_inode`` this method is called when the VFS needs to write an inode to @@ -439,7 +442,7 @@ As of kernel 2.6.22, the following members are defined: void (*update_time)(struct inode *, struct timespec *, int); int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct user_namespace *, struct inode *, struct dentry *, umode_t); + int (*tmpfile) (struct user_namespace *, struct inode *, struct file *, umode_t); int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int); int (*fileattr_set)(struct user_namespace *mnt_userns, struct dentry *dentry, struct fileattr *fa); @@ -589,7 +592,9 @@ otherwise noted. ``tmpfile`` called in the end of O_TMPFILE open(). Optional, equivalent to atomically creating, opening and unlinking a file in given - directory. + directory. On success needs to return with the file already + open; this can be done by calling finish_open_simple() right at + the end. ``fileattr_get`` called on ioctl(FS_IOC_GETFLAGS) and ioctl(FS_IOC_FSGETXATTR) to diff --git a/Documentation/filesystems/xfs-delayed-logging-design.rst b/Documentation/filesystems/xfs-delayed-logging-design.rst index 4ef419f546639dfac38d8df00108584f9854c088..6402ab8e370c81c4fae5e21f1a86850d8e2a61c5 100644 --- a/Documentation/filesystems/xfs-delayed-logging-design.rst +++ b/Documentation/filesystems/xfs-delayed-logging-design.rst @@ -100,7 +100,7 @@ transactions together:: ntp = xfs_trans_dup(tp); xfs_trans_commit(tp); - xfs_log_reserve(ntp); + xfs_trans_reserve(ntp); This results in a series of "rolling transactions" where the inode is locked across the entire chain of transactions. Hence while this series of rolling @@ -191,7 +191,7 @@ transaction rolling mechanism to re-reserve space on every transaction roll. We know from the implementation of the permanent transactions how many transaction rolls are likely for the common modifications that need to be made. -For example, and inode allocation is typically two transactions - one to +For example, an inode allocation is typically two transactions - one to physically allocate a free inode chunk on disk, and another to allocate an inode from an inode chunk that has free inodes in it. Hence for an inode allocation transaction, we might set the reservation log count to a value of 2 to indicate @@ -200,7 +200,7 @@ chain. Each time a permanent transaction rolls, it consumes an entire unit reservation. Hence when the permanent transaction is first allocated, the log space -reservation is increases from a single unit reservation to multiple unit +reservation is increased from a single unit reservation to multiple unit reservations. That multiple is defined by the reservation log count, and this means we can roll the transaction multiple times before we have to re-reserve log space when we roll the transaction. This ensures that the common @@ -259,7 +259,7 @@ the next transaction in the sequeunce, but we have none remaining. We cannot sleep during the transaction commit process waiting for new log space to become available, as we may end up on the end of the FIFO queue and the items we have locked while we sleep could end up pinning the tail of the log before there is -enough free space in the log to fulfil all of the pending reservations and +enough free space in the log to fulfill all of the pending reservations and then wake up transaction commit in progress. To take a new reservation without sleeping requires us to be able to take a @@ -551,14 +551,14 @@ Essentially, this shows that an item that is in the AIL can still be modified and relogged, so any tracking must be separate to the AIL infrastructure. As such, we cannot reuse the AIL list pointers for tracking committed items, nor can we store state in any field that is protected by the AIL lock. Hence the -committed item tracking needs it's own locks, lists and state fields in the log +committed item tracking needs its own locks, lists and state fields in the log item. Similar to the AIL, tracking of committed items is done through a new list called the Committed Item List (CIL). The list tracks log items that have been committed and have formatted memory buffers attached to them. It tracks objects in transaction commit order, so when an object is relogged it is removed from -it's place in the list and re-inserted at the tail. This is entirely arbitrary +its place in the list and re-inserted at the tail. This is entirely arbitrary and done to make it easy for debugging - the last items in the list are the ones that are most recently modified. Ordering of the CIL is not necessary for transactional integrity (as discussed in the next section) so the ordering is @@ -615,7 +615,7 @@ those changes into the current checkpoint context. We then initialise a new context and attach that to the CIL for aggregation of new transactions. This allows us to unlock the CIL immediately after transfer of all the -committed items and effectively allow new transactions to be issued while we +committed items and effectively allows new transactions to be issued while we are formatting the checkpoint into the log. It also allows concurrent checkpoints to be written into the log buffers in the case of log force heavy workloads, just like the existing transaction commit code does. This, however, @@ -884,9 +884,9 @@ pin the object the first time it is inserted into the CIL - if it is already in the CIL during a transaction commit, then we do not pin it again. Because there can be multiple outstanding checkpoint contexts, we can still see elevated pin counts, but as each checkpoint completes the pin count will retain the correct -value according to it's context. +value according to its context. -Just to make matters more slightly more complex, this checkpoint level context +Just to make matters slightly more complex, this checkpoint level context for the pin count means that the pinning of an item must take place under the CIL commit/flush lock. If we pin the object outside this lock, we cannot guarantee which context the pin count is associated with. This is because of diff --git a/Documentation/firmware-guide/acpi/enumeration.rst b/Documentation/firmware-guide/acpi/enumeration.rst index dbb03022b12711881f8391c3f2a0767d163dc762..b9dc0c603f3672789a462945fad841d95d5b8307 100644 --- a/Documentation/firmware-guide/acpi/enumeration.rst +++ b/Documentation/firmware-guide/acpi/enumeration.rst @@ -21,7 +21,7 @@ possible we decided to do following: - Devices behind real busses where there is a connector resource are represented as struct spi_device or struct i2c_device. Note that standard UARTs are not busses so there is no struct uart_device, - although some of them may be represented by sturct serdev_device. + although some of them may be represented by struct serdev_device. As both ACPI and Device Tree represent a tree of devices (and their resources) this implementation follows the Device Tree way as much as @@ -205,7 +205,7 @@ Here is what the ACPI namespace for a SPI slave might look like:: } ... -The SPI device drivers only need to add ACPI IDs in a similar way than with +The SPI device drivers only need to add ACPI IDs in a similar way to the platform device drivers. Below is an example where we add ACPI support to at25 SPI eeprom driver (this is meant for the above ACPI snippet):: @@ -362,7 +362,7 @@ These GPIO numbers are controller relative and path "\\_SB.PCI0.GPI0" specifies the path to the controller. In order to use these GPIOs in Linux we need to translate them to the corresponding Linux GPIO descriptors. -There is a standard GPIO API for that and is documented in +There is a standard GPIO API for that and it is documented in Documentation/admin-guide/gpio/. In the above example we can get the corresponding two GPIO descriptors with @@ -538,8 +538,8 @@ information. PCI hierarchy representation ============================ -Sometimes could be useful to enumerate a PCI device, knowing its position on the -PCI bus. +Sometimes it could be useful to enumerate a PCI device, knowing its position on +the PCI bus. For example, some systems use PCI devices soldered directly on the mother board, in a fixed position (ethernet, Wi-Fi, serial ports, etc.). In this conditions it @@ -550,7 +550,7 @@ To identify a PCI device, a complete hierarchical description is required, from the chipset root port to the final device, through all the intermediate bridges/switches of the board. -For example, let us assume to have a system with a PCIe serial port, an +For example, let's assume we have a system with a PCIe serial port, an Exar XR17V3521, soldered on the main board. This UART chip also includes 16 GPIOs and we want to add the property ``gpio-line-names`` [1] to these pins. In this case, the ``lspci`` output for this component is:: @@ -593,8 +593,8 @@ of the chipset bridge (also called "root port") with address:: Bus: 0 - Device: 14 - Function: 1 -To find this information is necessary disassemble the BIOS ACPI tables, in -particular the DSDT (see also [2]):: +To find this information, it is necessary to disassemble the BIOS ACPI tables, +in particular the DSDT (see also [2]):: mkdir ~/tables/ cd ~/tables/ diff --git a/Documentation/firmware-guide/acpi/osi.rst b/Documentation/firmware-guide/acpi/osi.rst index 05869c0045d747c954aaffe246994f7bd3fcf53b..784850adfcb674bb5b5e5b09b2b86107d1b5dd50 100644 --- a/Documentation/firmware-guide/acpi/osi.rst +++ b/Documentation/firmware-guide/acpi/osi.rst @@ -41,26 +41,23 @@ But it is likely that they will all eventually be added. What should an OEM do if they want to support Linux and Windows using the same BIOS image? Often they need to do something different for Linux to deal with how Linux is different from Windows. -Here the BIOS should ask exactly what it wants to know: +In this case, the OEM should create custom ASL to be executed by the +Linux kernel and changes to Linux kernel drivers to execute this custom +ASL. The easiest way to accomplish this is to introduce a device specific +method (_DSM) that is called from the Linux kernel. + +In the past the kernel used to support something like: _OSI("Linux-OEM-my_interface_name") where 'OEM' is needed if this is an OEM-specific hook, and 'my_interface_name' describes the hook, which could be a quirk, a bug, or a bug-fix. -In addition, the OEM should send a patch to upstream Linux -via the linux-acpi@vger.kernel.org mailing list. When that patch -is checked into Linux, the OS will answer "YES" when the BIOS -on the OEM's system uses _OSI to ask if the interface is supported -by the OS. Linux distributors can back-port that patch for Linux -pre-installs, and it will be included by all distributions that -re-base to upstream. If the distribution can not update the kernel binary, -they can also add an acpi_osi=Linux-OEM-my_interface_name -cmdline parameter to the boot loader, as needed. - -If the string refers to a feature where the upstream kernel -eventually grows support, a patch should be sent to remove -the string when that support is added to the kernel. +However this was discovered to be abused by other BIOS vendors to change +completely unrelated code on completely unrelated systems. This prompted +an evaluation of all of it's uses. This uncovered that they aren't needed +for any of the original reasons. As such, the kernel will not respond to +any custom Linux-* strings by default. That was easy. Read on, to find out how to do it wrong. diff --git a/Documentation/hwmon/aquacomputer_d5next.rst b/Documentation/hwmon/aquacomputer_d5next.rst index 33649a1e3a05f2d5c453f16478a55fa2240a64d1..e238533b5fe01ed640e29e5f74656f74dac63e08 100644 --- a/Documentation/hwmon/aquacomputer_d5next.rst +++ b/Documentation/hwmon/aquacomputer_d5next.rst @@ -10,6 +10,7 @@ Supported devices: * Aquacomputer Farbwerk 360 RGB controller * Aquacomputer Octo fan controller * Aquacomputer Quadro fan controller +* Aquacomputer High Flow Next sensor Author: Aleksa Savic @@ -20,10 +21,11 @@ This driver exposes hardware sensors of listed Aquacomputer devices, which communicate through proprietary USB HID protocols. For the D5 Next pump, available sensors are pump and fan speed, power, voltage -and current, as well as coolant temperature. Also available through debugfs are -the serial number, firmware version and power-on count. Attaching a fan to it is -optional and allows it to be controlled using temperature curves directly from the -pump. If it's not connected, the fan-related sensors will report zeroes. +and current, as well as coolant temperature and eight virtual temp sensors. Also +available through debugfs are the serial number, firmware version and power-on +count. Attaching a fan to it is optional and allows it to be controlled using +temperature curves directly from the pump. If it's not connected, the fan-related +sensors will report zeroes. The pump can be configured either through software or via its physical interface. Configuring the pump through this driver is not implemented, as it @@ -31,14 +33,23 @@ seems to require sending it a complete configuration. That includes addressable RGB LEDs, for which there is no standard sysfs interface. Thus, that task is better suited for userspace tools. -The Octo exposes four temperature sensors and eight PWM controllable fans, along -with their speed (in RPM), power, voltage and current. +The Octo exposes four physical and sixteen virtual temperature sensors, as well as +eight PWM controllable fans, along with their speed (in RPM), power, voltage and +current. -The Quadro exposes four temperature sensors, a flow sensor and four PWM controllable -fans, along with their speed (in RPM), power, voltage and current. +The Quadro exposes four physical and sixteen virtual temperature sensors, a flow +sensor and four PWM controllable fans, along with their speed (in RPM), power, +voltage and current. -The Farbwerk and Farbwerk 360 expose four temperature sensors. Depending on the device, -not all sysfs and debugfs entries will be available. +The Farbwerk and Farbwerk 360 expose four temperature sensors. Additionally, +sixteen virtual temperature sensors of the Farbwerk 360 are exposed. + +The High Flow Next exposes +5V voltages, water quality, conductivity and flow readings. +A temperature sensor can be connected to it, in which case it provides its reading +and an estimation of the dissipated/absorbed power in the liquid cooling loop. + +Depending on the device, not all sysfs and debugfs entries will be available. +Writing to virtual temperature sensors is not currently supported. Usage notes ----------- @@ -49,14 +60,14 @@ the kernel and supports hotswapping. Sysfs entries ------------- -================ ============================================== -temp[1-4]_input Temperature sensors (in millidegrees Celsius) +================ ============================================================== +temp[1-20]_input Physical/virtual temperature sensors (in millidegrees Celsius) fan[1-8]_input Pump/fan speed (in RPM) / Flow speed (in dL/h) power[1-8]_input Pump/fan power (in micro Watts) in[0-7]_input Pump/fan voltage (in milli Volts) curr[1-8]_input Pump/fan current (in milli Amperes) pwm[1-8] Fan PWM (0 - 255) -================ ============================================== +================ ============================================================== Debugfs entries --------------- diff --git a/Documentation/hwmon/asus_wmi_ec_sensors.rst b/Documentation/hwmon/asus_wmi_ec_sensors.rst deleted file mode 100644 index 1b287f229e86cedbc79f54bbdbe9dd75e29fd0c7..0000000000000000000000000000000000000000 --- a/Documentation/hwmon/asus_wmi_ec_sensors.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0-or-later - -Kernel driver asus_wmi_ec_sensors -================================= - -Supported boards: - * PRIME X570-PRO, - * Pro WS X570-ACE, - * ROG CROSSHAIR VIII DARK HERO, - * ROG CROSSHAIR VIII FORMULA, - * ROG CROSSHAIR VIII HERO, - * ROG STRIX B550-E GAMING, - * ROG STRIX B550-I GAMING, - * ROG STRIX X570-E GAMING. - -Authors: - - Eugene Shalygin - -Description: ------------- -ASUS mainboards publish hardware monitoring information via Super I/O -chip and the ACPI embedded controller (EC) registers. Some of the sensors -are only available via the EC. - -ASUS WMI interface provides a method (BREC) to read data from EC registers, -which is utilized by this driver to publish those sensor readings to the -HWMON system. The driver is aware of and reads the following sensors: - -1. Chipset (PCH) temperature -2. CPU package temperature -3. Motherboard temperature -4. Readings from the T_Sensor header -5. VRM temperature -6. CPU_Opt fan RPM -7. Chipset fan RPM -8. Readings from the "Water flow meter" header (RPM) -9. Readings from the "Water In" and "Water Out" temperature headers -10. CPU current diff --git a/Documentation/hwmon/corsair-psu.rst b/Documentation/hwmon/corsair-psu.rst index e8378e7a1d8c089c5dc311e6743fc5511ce9889b..3c1b164eb3c06f9739abdceeaccc2ce3de075195 100644 --- a/Documentation/hwmon/corsair-psu.rst +++ b/Documentation/hwmon/corsair-psu.rst @@ -15,7 +15,7 @@ Supported devices: Corsair HX850i - Corsair HX1000i + Corsair HX1000i (revision 1 and 2) Corsair HX1200i @@ -86,8 +86,9 @@ Debugfs entries --------------- ======================= ======================================================== -uptime Current uptime of the psu +ocpmode Single or multi rail mode of the PCIe power connectors +product Product name of the psu +uptime Session uptime of the psu uptime_total Total uptime of the psu vendor Vendor name of the psu -product Product name of the psu ======================= ======================================================== diff --git a/Documentation/hwmon/emc2305.rst b/Documentation/hwmon/emc2305.rst new file mode 100644 index 0000000000000000000000000000000000000000..2403dbaf2728917184bbf66f94685bc625f49dce --- /dev/null +++ b/Documentation/hwmon/emc2305.rst @@ -0,0 +1,37 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver emc2305 +===================== + +Supported chips: + Microchip EMC2305, EMC2303, EMC2302, EMC2301 + + Addresses scanned: I2C 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d + Prefixes: 'emc2305' + + Datasheet: Publicly available at the Microchip website : + https://www.microchip.com/en-us/product/EMC2305 + +Description: +------------ +This driver implements support for Microchip EMC2301/2/3/5 RPM-based PWM Fan Controller. +The EMC2305 Fan Controller supports up to 5 independently controlled PWM fan drives. +Fan rotation speeds are reported in RPM. +The driver supports the RPM-based PWM control to keep a fan at the desired speed. +The driver provides the possibility to have one common PWM interface for all FANs +or up to the maximum available or configured independent PWMs. + +The driver provides the following sysfs interfaces in hwmon subsystem: + +================= == =================================================== +fan[1-5]_fault RO files for tachometers TACH1-TACH5 fault indication +fan[1-5]_input RO files for tachometers TACH1-TACH5 input (in RPM) +pwm[1-5] RW file for fan[1-5] target duty cycle (0..255) +================= == =================================================== + +sysfs interfaces in thermal subsystem: + +================= == ======================================================================== +cur_state RW file for the current cooling state of the cooling device (0..max_state) +max_state RO file for the maximum cooling state of the cooling device +================= == ======================================================================== diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index f7113b0f8b2a66fb23227db7b12083d79272da43..c1d11cf13eef167326daeb1f0b69f688fdfb9044 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -44,7 +44,6 @@ Hardware Monitoring Kernel Drivers asc7621 aspeed-pwm-tacho asus_ec_sensors - asus_wmi_ec_sensors asus_wmi_sensors bcm54140 bel-pfe @@ -63,6 +62,7 @@ Hardware Monitoring Kernel Drivers ds620 emc1403 emc2103 + emc2305 emc6w201 f71805f f71882fg @@ -133,6 +133,7 @@ Hardware Monitoring Kernel Drivers max20751 max31722 max31730 + max31760 max31785 max31790 max34440 @@ -205,6 +206,7 @@ Hardware Monitoring Kernel Drivers tps23861 tps40422 tps53679 + tps546d24 twl4030-madc-hwmon ucd9000 ucd9200 diff --git a/Documentation/hwmon/max31760.rst b/Documentation/hwmon/max31760.rst new file mode 100644 index 0000000000000000000000000000000000000000..b1b55fb843c186f7655c814a7d3a8191d84af9f5 --- /dev/null +++ b/Documentation/hwmon/max31760.rst @@ -0,0 +1,77 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +Kernel driver max31760 +====================== + +Supported chips: + * Analog Devices MAX31760 + + Prefix: 'max31760' + + Addresses scanned: none + + Datasheet: https://datasheets.maximintegrated.com/en/ds/MAX31760.pdf + + +Author: Ibrahim Tilki + +Description +----------- + +The MAX31760 integrates temperature sensing along with precision PWM fan +control. It accurately measures its local die temperature and the remote +temperature of a discrete diode-connected transistor, such as a 2N3906, +or a thermal diode commonly found on CPUs, graphics processor units (GPUs), +and other ASICs. Multiple temperature thresholds, such as local +high/overtemperature (OT) and remote high/overtemperature, can be set by an +I2C-compatible interface. Fan speed is controlled based on the temperature +reading as an index to a 48-byte lookup table (LUT) containing +user-programmed PWM values. The flexible LUT-based architecture enables +the user to program a smooth nonlinear fan speed vs. temperature transfer +function to minimize acoustic fan noise. Two tachometer inputs allow +measuring the speeds of two fans independently. When the local or remote +OT threshold is exceeded, the SHDN pin is asserted low and can be used to +shut down the system. A dedicated ALERT pin reports that either a local or +remote high-temperature threshold has been exceeded. + +Temperature measurement range: from -55°C to 125°C + +Temperature Resolution: 11 Bits, ±0.125°C + +Please refer how to instantiate this driver: Documentation/i2c/instantiating-devices.rst + +Lookup table for auto fan control +--------------------------------- + +========= ================================= +LUT Index Name +========= ================================= +1 PWM value for T < +18°C +2 PWM value for +18°C ≤ T < +20°C +3 PWM value for +20°C ≤ T < +22°C +... ... +47 PWM value for +108°C ≤ T < +110°C +48 PWM value for T ≥ +110°C +========= ================================= + +Sysfs entries +------------- + +=============================== ================================================================================= +fan[1-2]_input Fan speed (in RPM) +fan[1-2]_enable Enable fan readings and fan fault alarms +fan[1-2]_fault Fan fault status +temp[1-2]_label "Remote" and "Local" temperature channel labels +temp[1-2]_input Temperature sensor readings (in millidegrees Celsius) +temp1_fault Remote temperature sensor fault status +temp[1-2]_max Temperature max value. Asserts "ALERT" pin when exceeded +temp[1-2]_max_alarm Temperature max alarm status +temp[1-2]_crit Temperature critical value. Asserts "SHDN" pin when exceeded +temp[1-2]_crit_alarm Temperature critical alarm status +pwm1 PWM value for direct fan control +pwm1_enable 1: direct fan control, 2: temperature based auto fan control +pwm1_freq PWM frequency in hertz +pwm1_auto_channels_temp Temperature source for auto fan control. 1: temp1, 2: temp2, 3: max(temp1, temp2) +pwm1_auto_point[1-48]_pwm PWM value for LUT point +pwm1_auto_point_temp_hyst Temperature hysteresis for auto fan control. Can be either 2000mC or 4000mC +=============================== ================================================================================= diff --git a/Documentation/hwmon/max31790.rst b/Documentation/hwmon/max31790.rst index 7b097c3b9b908afc45f45e53e4ee6f0509309379..33c5c7330efc90b5ea8adaf9c681671f35d6d579 100644 --- a/Documentation/hwmon/max31790.rst +++ b/Documentation/hwmon/max31790.rst @@ -38,6 +38,7 @@ Sysfs entries fan[1-12]_input RO fan tachometer speed in RPM fan[1-12]_fault RO fan experienced fault fan[1-6]_target RW desired fan speed in RPM +fan[1-6]_enable RW enable or disable the tachometer input pwm[1-6]_enable RW regulator mode, 0=disabled (duty cycle=0%), 1=manual mode, 2=rpm mode pwm[1-6] RW read: current pwm duty cycle, write: target pwm duty cycle (0-255) diff --git a/Documentation/hwmon/pwm-fan.rst b/Documentation/hwmon/pwm-fan.rst index 82fe96742feea339372b40c7be4cf06497bd8dd5..f77998b204ef27c20f45826f23373d3dcb2cdd30 100644 --- a/Documentation/hwmon/pwm-fan.rst +++ b/Documentation/hwmon/pwm-fan.rst @@ -18,3 +18,15 @@ the hwmon's sysfs interface. The fan rotation speed returned via the optional 'fan1_input' is extrapolated from the sampled interrupts from the tachometer signal within 1 second. + +The driver provides the following sensor accesses in sysfs: + +=============== ======= ======================================================= +fan1_input ro fan tachometer speed in RPM +pwm1_enable rw keep enable mode, defines behaviour when pwm1=0 + 0 -> disable pwm and regulator + 1 -> enable pwm; if pwm==0, disable pwm, keep regulator enabled + 2 -> enable pwm; if pwm==0, keep pwm and regulator enabled + 3 -> enable pwm; if pwm==0, disable pwm and regulator +pwm1 rw relative speed (0-255), 255=max. speed. +=============== ======= ======================================================= diff --git a/Documentation/hwmon/tps546d24.rst b/Documentation/hwmon/tps546d24.rst new file mode 100644 index 0000000000000000000000000000000000000000..97adb8a30fc09001502bc65683c154de1ca698d2 --- /dev/null +++ b/Documentation/hwmon/tps546d24.rst @@ -0,0 +1,35 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel driver tps546d24 +======================= + +Supported chips: + + * TI TPS546D24 + + Prefix: 'tps546d24' + + Addresses scanned: - + + Datasheet: https://www.ti.com/lit/gpn/tps546d24 + +Author: Duke Du + + +Description +----------- + +The TPS546D24A is a highly integrated, non-isolated DC/DC converter capable +of high frequency operation and 40-A current output from a 7-mm x 5-mm +package. + +Two, three, and four TPS546D24A devices can be interconnected +to provide up to 160 A on a single output. The device has an option to +overdrive the internal 5-V LDO with an external 5-V supply via the VDD5 +pin to improve efficiency and reduce power dissipation of the converter. + + +Platform data support +--------------------- + +The driver supports standard PMBus driver platform data. diff --git a/Documentation/i2c/busses/i2c-piix4.rst b/Documentation/i2c/busses/i2c-piix4.rst index cc900025922309f0bd86658e99b021f7b2bc8c38..07fe6f6f4b186a3115623c0b04eadeb1e8aaefcd 100644 --- a/Documentation/i2c/busses/i2c-piix4.rst +++ b/Documentation/i2c/busses/i2c-piix4.rst @@ -64,7 +64,7 @@ correct address for this module, you could get in big trouble (read: crashes, data corruption, etc.). Try this only as a last resort (try BIOS updates first, for example), and backup first! An even more dangerous option is 'force_addr='. This will not only enable the PIIX4 like -'force' foes, but it will also set a new base I/O port address. The SMBus +'force' does, but it will also set a new base I/O port address. The SMBus parts of the PIIX4 needs a range of 8 of these addresses to function correctly. If these addresses are already reserved by some other device, you will get into big trouble! DON'T USE THIS IF YOU ARE NOT VERY SURE @@ -86,15 +86,15 @@ If you own Force CPCI735 motherboard or other OSB4 based systems you may need to change the SMBus Interrupt Select register so the SMBus controller uses the SMI mode. -1) Use lspci command and locate the PCI device with the SMBus controller: +1) Use ``lspci`` command and locate the PCI device with the SMBus controller: 00:0f.0 ISA bridge: ServerWorks OSB4 South Bridge (rev 4f) The line may vary for different chipsets. Please consult the driver source - for all possible PCI ids (and lspci -n to match them). Lets assume the + for all possible PCI ids (and ``lspci -n`` to match them). Let's assume the device is located at 00:0f.0. 2) Now you just need to change the value in 0xD2 register. Get it first with - command: lspci -xxx -s 00:0f.0 + command: ``lspci -xxx -s 00:0f.0`` If the value is 0x3 then you need to change it to 0x1: - setpci -s 00:0f.0 d2.b=1 + ``setpci -s 00:0f.0 d2.b=1`` Please note that you don't need to do that in all cases, just when the SMBus is not working properly. @@ -109,6 +109,3 @@ which can easily get corrupted due to a state machine bug. These are mostly Thinkpad laptops, but desktop systems may also be affected. We have no list of all affected systems, so the only safe solution was to prevent access to the SMBus on all IBM systems (detected using DMI data.) - -For additional information, read: -http://www.lm-sensors.org/browser/lm-sensors/trunk/README diff --git a/Documentation/i2c/dev-interface.rst b/Documentation/i2c/dev-interface.rst index 73ad34849f993ef74d241f1f70107e1bfb83ee56..c277a8e1202b51403a8d00d6c92fca13da1afc58 100644 --- a/Documentation/i2c/dev-interface.rst +++ b/Documentation/i2c/dev-interface.rst @@ -148,7 +148,7 @@ You can do plain I2C transactions by using read(2) and write(2) calls. You do not need to pass the address byte; instead, set it through ioctl I2C_SLAVE before you try to access the device. -You can do SMBus level transactions (see documentation file smbus-protocol +You can do SMBus level transactions (see documentation file smbus-protocol.rst for details) through the following functions:: __s32 i2c_smbus_write_quick(int file, __u8 value); diff --git a/Documentation/i2c/i2c-topology.rst b/Documentation/i2c/i2c-topology.rst index 7cb53819778e6cb7d62787a62f4c124e28872d33..48fce0f7491bf1bcd4a851f685b010386c9ce0d2 100644 --- a/Documentation/i2c/i2c-topology.rst +++ b/Documentation/i2c/i2c-topology.rst @@ -5,6 +5,8 @@ I2C muxes and complex topologies There are a couple of reasons for building more complex I2C topologies than a straight-forward I2C bus with one adapter and one or more devices. +Some example use cases are: + 1. A mux may be needed on the bus to prevent address collisions. 2. The bus may be accessible from some external bus master, and arbitration @@ -14,10 +16,10 @@ than a straight-forward I2C bus with one adapter and one or more devices. from the I2C bus, at least most of the time, and sits behind a gate that has to be operated before the device can be accessed. -Etc -=== +Several types of hardware components such as I2C muxes, I2C gates and I2C +arbitrators allow to handle such needs. -These constructs are represented as I2C adapter trees by Linux, where +These components are represented as I2C adapter trees by Linux, where each adapter has a parent adapter (except the root adapter) and zero or more child adapters. The root adapter is the actual adapter that issues I2C transfers, and all adapters with a parent are part of an "i2c-mux" @@ -35,46 +37,7 @@ Locking ======= There are two variants of locking available to I2C muxes, they can be -mux-locked or parent-locked muxes. As is evident from below, it can be -useful to know if a mux is mux-locked or if it is parent-locked. The -following list was correct at the time of writing: - -In drivers/i2c/muxes/: - -====================== ============================================= -i2c-arb-gpio-challenge Parent-locked -i2c-mux-gpio Normally parent-locked, mux-locked iff - all involved gpio pins are controlled by the - same I2C root adapter that they mux. -i2c-mux-gpmux Normally parent-locked, mux-locked iff - specified in device-tree. -i2c-mux-ltc4306 Mux-locked -i2c-mux-mlxcpld Parent-locked -i2c-mux-pca9541 Parent-locked -i2c-mux-pca954x Parent-locked -i2c-mux-pinctrl Normally parent-locked, mux-locked iff - all involved pinctrl devices are controlled - by the same I2C root adapter that they mux. -i2c-mux-reg Parent-locked -====================== ============================================= - -In drivers/iio/: - -====================== ============================================= -gyro/mpu3050 Mux-locked -imu/inv_mpu6050/ Mux-locked -====================== ============================================= - -In drivers/media/: - -======================= ============================================= -dvb-frontends/lgdt3306a Mux-locked -dvb-frontends/m88ds3103 Parent-locked -dvb-frontends/rtl2830 Parent-locked -dvb-frontends/rtl2832 Mux-locked -dvb-frontends/si2168 Mux-locked -usb/cx231xx/ Parent-locked -======================= ============================================= +mux-locked or parent-locked muxes. Mux-locked muxes @@ -89,40 +52,8 @@ full transaction, unrelated I2C transfers may interleave the different stages of the transaction. This has the benefit that the mux driver may be easier and cleaner to implement, but it has some caveats. -==== ===================================================================== -ML1. If you build a topology with a mux-locked mux being the parent - of a parent-locked mux, this might break the expectation from the - parent-locked mux that the root adapter is locked during the - transaction. - -ML2. It is not safe to build arbitrary topologies with two (or more) - mux-locked muxes that are not siblings, when there are address - collisions between the devices on the child adapters of these - non-sibling muxes. - - I.e. the select-transfer-deselect transaction targeting e.g. device - address 0x42 behind mux-one may be interleaved with a similar - operation targeting device address 0x42 behind mux-two. The - intension with such a topology would in this hypothetical example - be that mux-one and mux-two should not be selected simultaneously, - but mux-locked muxes do not guarantee that in all topologies. - -ML3. A mux-locked mux cannot be used by a driver for auto-closing - gates/muxes, i.e. something that closes automatically after a given - number (one, in most cases) of I2C transfers. Unrelated I2C transfers - may creep in and close prematurely. - -ML4. If any non-I2C operation in the mux driver changes the I2C mux state, - the driver has to lock the root adapter during that operation. - Otherwise garbage may appear on the bus as seen from devices - behind the mux, when an unrelated I2C transfer is in flight during - the non-I2C mux-changing operation. -==== ===================================================================== - - Mux-locked Example ------------------- - +~~~~~~~~~~~~~~~~~~ :: @@ -153,6 +84,43 @@ This means that accesses to D2 are lockout out for the full duration of the entire operation. But accesses to D3 are possibly interleaved at any point. +Mux-locked caveats +~~~~~~~~~~~~~~~~~~ + +When using a mux-locked mux, be aware of the following restrictions: + +[ML1] + If you build a topology with a mux-locked mux being the parent + of a parent-locked mux, this might break the expectation from the + parent-locked mux that the root adapter is locked during the + transaction. + +[ML2] + It is not safe to build arbitrary topologies with two (or more) + mux-locked muxes that are not siblings, when there are address + collisions between the devices on the child adapters of these + non-sibling muxes. + + I.e. the select-transfer-deselect transaction targeting e.g. device + address 0x42 behind mux-one may be interleaved with a similar + operation targeting device address 0x42 behind mux-two. The + intent with such a topology would in this hypothetical example + be that mux-one and mux-two should not be selected simultaneously, + but mux-locked muxes do not guarantee that in all topologies. + +[ML3] + A mux-locked mux cannot be used by a driver for auto-closing + gates/muxes, i.e. something that closes automatically after a given + number (one, in most cases) of I2C transfers. Unrelated I2C transfers + may creep in and close prematurely. + +[ML4] + If any non-I2C operation in the mux driver changes the I2C mux state, + the driver has to lock the root adapter during that operation. + Otherwise garbage may appear on the bus as seen from devices + behind the mux, when an unrelated I2C transfer is in flight during + the non-I2C mux-changing operation. + Parent-locked muxes ------------------- @@ -161,28 +129,10 @@ Parent-locked muxes lock the parent adapter during the full select- transfer-deselect transaction. The implication is that the mux driver has to ensure that any and all I2C transfers through that parent adapter during the transaction are unlocked I2C transfers (using e.g. -__i2c_transfer), or a deadlock will follow. There are a couple of -caveats. - -==== ==================================================================== -PL1. If you build a topology with a parent-locked mux being the child - of another mux, this might break a possible assumption from the - child mux that the root adapter is unused between its select op - and the actual transfer (e.g. if the child mux is auto-closing - and the parent mux issues I2C transfers as part of its select). - This is especially the case if the parent mux is mux-locked, but - it may also happen if the parent mux is parent-locked. - -PL2. If select/deselect calls out to other subsystems such as gpio, - pinctrl, regmap or iio, it is essential that any I2C transfers - caused by these subsystems are unlocked. This can be convoluted to - accomplish, maybe even impossible if an acceptably clean solution - is sought. -==== ==================================================================== - +__i2c_transfer), or a deadlock will follow. Parent-locked Example ---------------------- +~~~~~~~~~~~~~~~~~~~~~ :: @@ -212,10 +162,30 @@ When there is an access to D1, this happens: 9. M1 unlocks its parent adapter. 10. M1 unlocks muxes on its parent. - This means that accesses to both D2 and D3 are locked out for the full duration of the entire operation. +Parent-locked Caveats +~~~~~~~~~~~~~~~~~~~~~ + +When using a parent-locked mux, be aware of the following restrictions: + +[PL1] + If you build a topology with a parent-locked mux being the child + of another mux, this might break a possible assumption from the + child mux that the root adapter is unused between its select op + and the actual transfer (e.g. if the child mux is auto-closing + and the parent mux issues I2C transfers as part of its select). + This is especially the case if the parent mux is mux-locked, but + it may also happen if the parent mux is parent-locked. + +[PL2] + If select/deselect calls out to other subsystems such as gpio, + pinctrl, regmap or iio, it is essential that any I2C transfers + caused by these subsystems are unlocked. This can be convoluted to + accomplish, maybe even impossible if an acceptably clean solution + is sought. + Complex Examples ================ @@ -261,8 +231,10 @@ This is a good topology:: When device D1 is accessed, accesses to D2 are locked out for the full duration of the operation (muxes on the top child adapter of M1 are locked). But accesses to D3 and D4 are possibly interleaved at -any point. Accesses to D3 locks out D1 and D2, but accesses to D4 -are still possibly interleaved. +any point. + +Accesses to D3 locks out D1 and D2, but accesses to D4 are still possibly +interleaved. Mux-locked mux as parent of parent-locked mux @@ -394,3 +366,47 @@ This is a good topology:: When D1 or D2 are accessed, accesses to D3 and D4 are locked out while accesses to D5 may interleave. When D3 or D4 are accessed, accesses to all other devices are locked out. + + +Mux type of existing device drivers +=================================== + +Whether a device is mux-locked or parent-locked depends on its +implementation. The following list was correct at the time of writing: + +In drivers/i2c/muxes/: + +====================== ============================================= +i2c-arb-gpio-challenge Parent-locked +i2c-mux-gpio Normally parent-locked, mux-locked iff + all involved gpio pins are controlled by the + same I2C root adapter that they mux. +i2c-mux-gpmux Normally parent-locked, mux-locked iff + specified in device-tree. +i2c-mux-ltc4306 Mux-locked +i2c-mux-mlxcpld Parent-locked +i2c-mux-pca9541 Parent-locked +i2c-mux-pca954x Parent-locked +i2c-mux-pinctrl Normally parent-locked, mux-locked iff + all involved pinctrl devices are controlled + by the same I2C root adapter that they mux. +i2c-mux-reg Parent-locked +====================== ============================================= + +In drivers/iio/: + +====================== ============================================= +gyro/mpu3050 Mux-locked +imu/inv_mpu6050/ Mux-locked +====================== ============================================= + +In drivers/media/: + +======================= ============================================= +dvb-frontends/lgdt3306a Mux-locked +dvb-frontends/m88ds3103 Parent-locked +dvb-frontends/rtl2830 Parent-locked +dvb-frontends/rtl2832 Mux-locked +dvb-frontends/si2168 Mux-locked +usb/cx231xx/ Parent-locked +======================= ============================================= diff --git a/Documentation/i2c/slave-interface.rst b/Documentation/i2c/slave-interface.rst index 82ea3e1d6fe43a02a15990bfa416e60ff2e08cdb..3f0d320bc80a7751baf6c3496a621a4eedce6d82 100644 --- a/Documentation/i2c/slave-interface.rst +++ b/Documentation/i2c/slave-interface.rst @@ -32,9 +32,9 @@ User manual =========== I2C slave backends behave like standard I2C clients. So, you can instantiate -them as described in the document 'instantiating-devices'. The only difference -is that i2c slave backends have their own address space. So, you have to add -0x1000 to the address you would originally request. An example for +them as described in the document instantiating-devices.rst. The only +difference is that i2c slave backends have their own address space. So, you +have to add 0x1000 to the address you would originally request. An example for instantiating the slave-eeprom driver from userspace at the 7 bit address 0x64 on bus 1:: @@ -72,12 +72,15 @@ Event types: 'val': unused - 'ret': always 0 + 'ret': 0 if the backend is ready, otherwise some errno Another I2C master wants to write data to us. This event should be sent once our own address and the write bit was detected. The data did not arrive yet, so -there is nothing to process or return. Wakeup or initialization probably needs -to be done, though. +there is nothing to process or return. After returning, the bus driver must +always ack the address phase. If 'ret' is zero, backend initialization or +wakeup is done and further data may be received. If 'ret' is an errno, the bus +driver should nack all incoming bytes until the next stop condition to enforce +a retry of the transmission. * I2C_SLAVE_READ_REQUESTED (mandatory) diff --git a/Documentation/i2c/writing-clients.rst b/Documentation/i2c/writing-clients.rst index e3b126cf4a3bb5045c5e9b53ec707d52c978240c..b7d3ae7458f8686ac8dd194eb44b76fe4af65b73 100644 --- a/Documentation/i2c/writing-clients.rst +++ b/Documentation/i2c/writing-clients.rst @@ -156,7 +156,7 @@ those devices, and a remove() method to unbind. :: static int foo_probe(struct i2c_client *client); - static int foo_remove(struct i2c_client *client); + static void foo_remove(struct i2c_client *client); Remember that the i2c_driver does not create those client handles. The handle may be used during foo_probe(). If foo_probe() reports success @@ -364,7 +364,7 @@ stop condition is issued between transaction. The i2c_msg structure contains for each message the client address, the number of bytes of the message and the message data itself. -You can read the file ``i2c-protocol`` for more information about the +You can read the file i2c-protocol.rst for more information about the actual I2C protocol. @@ -414,7 +414,7 @@ transactions return 0 on success; the 'read' transactions return the read value, except for block transactions, which return the number of values read. The block buffers need not be longer than 32 bytes. -You can read the file ``smbus-protocol`` for more information about the +You can read the file smbus-protocol.rst for more information about the actual SMBus protocol. diff --git a/Documentation/iio/bno055.rst b/Documentation/iio/bno055.rst new file mode 100644 index 0000000000000000000000000000000000000000..9a489a79d8f5a8682334266ab8c444eeddd51773 --- /dev/null +++ b/Documentation/iio/bno055.rst @@ -0,0 +1,51 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================== +BNO055 driver +============================== + +1. Overview +=========== + +This driver supports Bosch BNO055 IMUs (on both serial and I2C busses). + +Accelerometer, magnetometer and gyroscope measures are always provided. +When "fusion_enable" sysfs attribute is set to 1, orientation (both Euler +angles and quaternion), linear velocity and gravity vector are also +provided, but some sensor settings (e.g. low pass filtering and range) +became locked (the IMU firmware controls them). + +This driver supports also IIO buffers. + +2. Calibration +============== + +The IMU continuously performs an autocalibration procedure if (and only if) +operating in fusion mode. The magnetometer autocalibration can however be +disabled writing 0 in the sysfs in_magn_calibration_fast_enable attribute. + +The driver provides access to autocalibration flags (i.e. you can known if +the IMU has successfully autocalibrated) and to the calibration data blob. + +The user can save this blob in a firmware file (i.e. in /lib/firmware) that +the driver looks for at probe time. If found, then the IMU is initialized +with this calibration data. This saves the user from performing the +calibration procedure every time (which consist of moving the IMU in +various way). + +The driver looks for calibration data file using two different names: first +a file whose name is suffixed with the IMU unique ID (exposed in sysfs as +serial_number) is searched for; this is useful when there is more than one +IMU instance. If this file is not found, then a "generic" calibration file +is searched for (which can be used when only one IMU is present, without +struggling with fancy names, that change on each device). + +Valid calibration file names would be e.g. + bno055-caldata-0e7c26a33541515120204a35342b04ff.dat + bno055-caldata.dat + +In non-fusion mode the IIO 'offset' attributes provide access to the +offsets from calibration data (if any), so that the user can apply them to +the accel, angvel and magn IIO attributes. In fusion mode they are not +needed (the IMU firmware internally applies those corrections) and they +read as zero. diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst index 58b7a4ebac515de0f4c5aa2d0bef1c37ef14ff96..1b7292c58cd07a4cd33a897377335bb24af8703d 100644 --- a/Documentation/iio/index.rst +++ b/Documentation/iio/index.rst @@ -10,3 +10,5 @@ Industrial I/O iio_configfs ep93xx_adc + + bno055 diff --git a/Documentation/index.rst b/Documentation/index.rst index 4737c18c97ff0ce1e947b46074f265e0a0c2af44..bf6aa681c960863c2e61db4253e2447d7eecf792 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -1,11 +1,5 @@ .. SPDX-License-Identifier: GPL-2.0 - -.. The Linux Kernel documentation master file, created by - sphinx-quickstart on Fri Feb 12 13:51:46 2016. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - .. _linux_doc: The Linux Kernel documentation @@ -18,133 +12,85 @@ documents into a coherent whole. Please note that improvements to the documentation are welcome; join the linux-doc list at vger.kernel.org if you want to help out. -Licensing documentation ------------------------ +Working with the development community +-------------------------------------- -The following describes the license of the Linux kernel source code -(GPLv2), how to properly mark the license of individual files in the source -tree, as well as links to the full license text. - -* :ref:`kernel_licensing` - -User-oriented documentation ---------------------------- - -The following manuals are written for *users* of the kernel — those who are -trying to get it to work optimally on a given system. +The essential guides for interacting with the kernel's development +community and getting your work upstream. .. toctree:: - :maxdepth: 2 - - admin-guide/index - kbuild/index - -Firmware-related documentation ------------------------------- -The following holds information on the kernel's expectations regarding the -platform firmwares. + :maxdepth: 1 -.. toctree:: - :maxdepth: 2 + process/development-process + process/submitting-patches + Code of conduct + maintainer/index + All development-process docs - firmware-guide/index - devicetree/index -Application-developer documentation ------------------------------------ +Internal API manuals +-------------------- -The user-space API manual gathers together documents describing aspects of -the kernel interface as seen by application developers. +Manuals for use by developers working to interface with the rest of the +kernel. .. toctree:: - :maxdepth: 2 - - userspace-api/index + :maxdepth: 1 + core-api/index + driver-api/index + subsystem-apis + Locking in the kernel -Introduction to kernel development ----------------------------------- +Development tools and processes +------------------------------- -These manuals contain overall information about how to develop the kernel. -The kernel community is quite large, with thousands of developers -contributing over the course of a year. As with any large community, -knowing how things are done will make the process of getting your changes -merged much easier. +Various other manuals with useful information for all kernel developers. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - process/index - dev-tools/index + process/license-rules doc-guide/index + dev-tools/index + dev-tools/testing-overview kernel-hacking/index trace/index - maintainer/index fault-injection/index livepatch/index + rust/index -Kernel API documentation ------------------------- +User-oriented documentation +--------------------------- -These books get into the details of how specific kernel subsystems work -from the point of view of a kernel developer. Much of the information here -is taken directly from the kernel source, with supplemental material added -as needed (or at least as we managed to add it — probably *not* all that is -needed). +The following manuals are written for *users* of the kernel — those who are +trying to get it to work optimally on a given system and application +developers seeking information on the kernel's user-space APIs. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - driver-api/index - core-api/index - locking/index - accounting/index - block/index - cdrom/index - cpu-freq/index - fb/index - fpga/index - hid/index - i2c/index - iio/index - isdn/index - infiniband/index - leds/index - netlabel/index - networking/index - pcmcia/index - power/index - target/index - timers/index - spi/index - w1/index - watchdog/index - virt/index - input/index - hwmon/index - gpu/index - security/index - sound/index - crypto/index - filesystems/index - mm/index - bpf/index - usb/index - PCI/index - scsi/index - misc-devices/index - scheduler/index - mhi/index - peci/index - -Architecture-agnostic documentation ------------------------------------ + admin-guide/index + The kernel build system + admin-guide/reporting-issues.rst + User-space tools + userspace-api/index + +See also: the `Linux man pages `_, +which are kept separately from the kernel's own documentation. + +Firmware-related documentation +------------------------------ +The following holds information on the kernel's expectations regarding the +platform firmwares. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + + firmware-guide/index + devicetree/index - asm-annotations Architecture-specific documentation ----------------------------------- @@ -163,9 +109,8 @@ of the documentation body, or may require some adjustments and/or conversion to ReStructured Text format, or are simply too old. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - tools/index staging/index diff --git a/Documentation/input/event-codes.rst b/Documentation/input/event-codes.rst index 8741d390b1843b92070f8e2f918760fd102a4b79..b4557462edd7b3fef9e9cd6c2c3cb2d05bb531ab 100644 --- a/Documentation/input/event-codes.rst +++ b/Documentation/input/event-codes.rst @@ -235,6 +235,12 @@ A few EV_ABS codes have special meanings: BTN_TOOL_ signals the type of tool that is currently detected by the hardware and is otherwise independent of ABS_DISTANCE and/or BTN_TOUCH. +* ABS_PROFILE: + + - Used to describe the state of a multi-value profile switch. An event is + emitted only when the selected profile changes, indicating the newly + selected profile value. + * ABS_MT_: - Used to describe multitouch input events. Please see diff --git a/Documentation/input/gamepad.rst b/Documentation/input/gamepad.rst index 4d5e7fb80a8456599608817dc8ca00478e2ee1be..71019de4603679ea618dff92b0ee692121145963 100644 --- a/Documentation/input/gamepad.rst +++ b/Documentation/input/gamepad.rst @@ -189,3 +189,9 @@ Gamepads report the following events: - Rumble: Rumble is advertised as FF_RUMBLE. + +- Profile: + + Some pads provide a multi-value profile selection switch. An example is the + XBox Adaptive and the XBox Elite 2 controllers. When the active profile is + switched, its newly selected value is emitted as an ABS_PROFILE event. diff --git a/Documentation/input/joydev/joystick.rst b/Documentation/input/joydev/joystick.rst index f615906a0821b86eca1debfbb6134cd9af737ecf..6d721396717a22ae2382906934e179ae21818855 100644 --- a/Documentation/input/joydev/joystick.rst +++ b/Documentation/input/joydev/joystick.rst @@ -517,6 +517,7 @@ All I-Force devices are supported by the iforce module. This includes: * AVB Mag Turbo Force * AVB Top Shot Pegasus * AVB Top Shot Force Feedback Racing Wheel +* Boeder Force Feedback Wheel * Logitech WingMan Force * Logitech WingMan Force Wheel * Guillemot Race Leader Force Feedback diff --git a/Documentation/kbuild/gcc-plugins.rst b/Documentation/kbuild/gcc-plugins.rst index 0ba76719f1b936490ab361af5110c746f3487f53..c578c6ba3eb6ad26780dc3ff40492e1b1b81c61f 100644 --- a/Documentation/kbuild/gcc-plugins.rst +++ b/Documentation/kbuild/gcc-plugins.rst @@ -90,7 +90,11 @@ e.g., on Ubuntu for gcc-10:: Or on Fedora:: - dnf install gcc-plugin-devel + dnf install gcc-plugin-devel libmpc-devel + +Or on Fedora when using cross-compilers that include plugins:: + + dnf install libmpc-devel Enable the GCC plugin infrastructure and some plugin(s) you want to use in the kernel config:: @@ -99,6 +103,19 @@ in the kernel config:: CONFIG_GCC_PLUGIN_LATENT_ENTROPY=y ... +Run gcc (native or cross-compiler) to ensure plugin headers are detected:: + + gcc -print-file-name=plugin + CROSS_COMPILE=arm-linux-gnu- ${CROSS_COMPILE}gcc -print-file-name=plugin + +The word "plugin" means they are not detected:: + + plugin + +A full path means they are detected:: + + /usr/lib/gcc/x86_64-redhat-linux/12/plugin + To compile the minimum tool set including the plugin(s):: make scripts diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst index ef19b9c1352362e7d7c01773650b20d9fd7a5bd2..08f575e6236c714d7cba2de0cb278b26db24d62c 100644 --- a/Documentation/kbuild/kbuild.rst +++ b/Documentation/kbuild/kbuild.rst @@ -48,6 +48,10 @@ KCFLAGS ------- Additional options to the C compiler (for built-in and modules). +KRUSTFLAGS +---------- +Additional options to the Rust compiler (for built-in and modules). + CFLAGS_KERNEL ------------- Additional options for $(CC) when used to compile @@ -57,6 +61,15 @@ CFLAGS_MODULE ------------- Additional module specific options to use for $(CC). +RUSTFLAGS_KERNEL +---------------- +Additional options for $(RUSTC) when used to compile +code that is compiled as built-in. + +RUSTFLAGS_MODULE +---------------- +Additional module specific options to use for $(RUSTC). + LDFLAGS_MODULE -------------- Additional options used for $(LD) when linking modules. @@ -69,6 +82,10 @@ HOSTCXXFLAGS ------------ Additional flags to be passed to $(HOSTCXX) when building host programs. +HOSTRUSTFLAGS +------------- +Additional flags to be passed to $(HOSTRUSTC) when building host programs. + HOSTLDFLAGS ----------- Additional flags to be passed when linking host programs. diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index 11a296e52d680014f131e0e57fb9bba60fefb3aa..6b7368d1f51639c016e4be5f4c3c4f51b0f29991 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -29,8 +29,9 @@ This document describes the Linux kernel Makefiles. --- 4.1 Simple Host Program --- 4.2 Composite Host Programs --- 4.3 Using C++ for host programs - --- 4.4 Controlling compiler options for host programs - --- 4.5 When host programs are actually built + --- 4.4 Using Rust for host programs + --- 4.5 Controlling compiler options for host programs + --- 4.6 When host programs are actually built === 5 Userspace Program support --- 5.1 Simple Userspace Program @@ -340,19 +341,7 @@ more details, with real examples. Examples are: - 1) head objects - - Some objects must be placed at the head of vmlinux. They are - directly linked to vmlinux without going through built-in.a - A typical use-case is an object that contains the entry point. - - arch/$(SRCARCH)/Makefile should specify such objects as head-y. - - Discussion: - Given that we can control the section order in the linker script, - why do we need head-y? - - 2) vmlinux linker script + 1) vmlinux linker script The linker script for vmlinux is located at arch/$(SRCARCH)/kernel/vmlinux.lds @@ -360,10 +349,6 @@ more details, with real examples. Example:: # arch/x86/kernel/Makefile - extra-y := head_$(BITS).o - extra-y += head$(BITS).o - extra-y += ebda.o - extra-y += platform-quirks.o extra-y += vmlinux.lds $(extra-y) should only contain targets needed for vmlinux. @@ -682,22 +667,27 @@ more details, with real examples. In the above example, -Wno-unused-but-set-variable will be added to KBUILD_CFLAGS only if gcc really accepts it. - cc-ifversion - cc-ifversion tests the version of $(CC) and equals the fourth parameter - if version expression is true, or the fifth (if given) if the version - expression is false. + gcc-min-version + gcc-min-version tests if the value of $(CONFIG_GCC_VERSION) is greater than + or equal to the provided value and evaluates to y if so. + + Example:: + + cflags-$(call gcc-min-version, 70100) := -foo + + In this example, cflags-y will be assigned the value -foo if $(CC) is gcc and + $(CONFIG_GCC_VERSION) is >= 7.1. + + clang-min-version + clang-min-version tests if the value of $(CONFIG_CLANG_VERSION) is greater + than or equal to the provided value and evaluates to y if so. Example:: - #fs/reiserfs/Makefile - ccflags-y := $(call cc-ifversion, -lt, 0402, -O1) + cflags-$(call clang-min-version, 110000) := -foo - In this example, ccflags-y will be assigned the value -O1 if the - $(CC) version is less than 4.2. - cc-ifversion takes all the shell operators: - -eq, -ne, -lt, -le, -gt, and -ge - The third parameter may be a text as in this example, but it may also - be an expanded variable or a macro. + In this example, cflags-y will be assigned the value -foo if $(CC) is clang + and $(CONFIG_CLANG_VERSION) is >= 11.0.0. cc-cross-prefix cc-cross-prefix is used to check if there exists a $(CC) in path with @@ -835,7 +825,24 @@ Both possibilities are described in the following. qconf-cxxobjs := qconf.o qconf-objs := check.o -4.4 Controlling compiler options for host programs +4.4 Using Rust for host programs +-------------------------------- + + Kbuild offers support for host programs written in Rust. However, + since a Rust toolchain is not mandatory for kernel compilation, + it may only be used in scenarios where Rust is required to be + available (e.g. when ``CONFIG_RUST`` is enabled). + + Example:: + + hostprogs := target + target-rust := y + + Kbuild will compile ``target`` using ``target.rs`` as the crate root, + located in the same directory as the ``Makefile``. The crate may + consist of several source files (see ``samples/rust/hostprogs``). + +4.5 Controlling compiler options for host programs -------------------------------------------------- When compiling host programs, it is possible to set specific flags. @@ -867,7 +874,7 @@ Both possibilities are described in the following. When linking qconf, it will be passed the extra option "-L$(QTDIR)/lib". -4.5 When host programs are actually built +4.6 When host programs are actually built ----------------------------------------- Kbuild will only build host-programs when they are referenced @@ -1081,8 +1088,7 @@ When kbuild executes, the following steps are followed (roughly): - The values of the above variables are expanded in arch/$(SRCARCH)/Makefile. 5) All object files are then linked and the resulting file vmlinux is located at the root of the obj tree. - The very first objects linked are listed in head-y, assigned by - arch/$(SRCARCH)/Makefile. + The very first objects linked are listed in scripts/head-object-list.txt. 6) Finally, the architecture-specific part does any required post processing and builds the final bootimage. - This includes building boot records @@ -1181,6 +1187,17 @@ When kbuild executes, the following steps are followed (roughly): The first example utilises the trick that a config option expands to 'y' when selected. + KBUILD_RUSTFLAGS + $(RUSTC) compiler flags + + Default value - see top level Makefile + Append or modify as required per architecture. + + Often, the KBUILD_RUSTFLAGS variable depends on the configuration. + + Note that target specification file generation (for ``--target``) + is handled in ``scripts/generate_rust_target.rs``. + KBUILD_AFLAGS_KERNEL Assembler options specific for built-in @@ -1208,6 +1225,19 @@ When kbuild executes, the following steps are followed (roughly): are used for $(CC). From commandline CFLAGS_MODULE shall be used (see kbuild.rst). + KBUILD_RUSTFLAGS_KERNEL + $(RUSTC) options specific for built-in + + $(KBUILD_RUSTFLAGS_KERNEL) contains extra Rust compiler flags used to + compile resident kernel code. + + KBUILD_RUSTFLAGS_MODULE + Options for $(RUSTC) when building modules + + $(KBUILD_RUSTFLAGS_MODULE) is used to add arch-specific options that + are used for $(RUSTC). + From commandline RUSTFLAGS_MODULE shall be used (see kbuild.rst). + KBUILD_LDFLAGS_MODULE Options for $(LD) when linking modules @@ -1230,6 +1260,9 @@ When kbuild executes, the following steps are followed (roughly): All object files for vmlinux. They are linked to vmlinux in the same order as listed in KBUILD_VMLINUX_OBJS. + The objects listed in scripts/head-object-list.txt are exceptions; + they are placed before the other objects. + KBUILD_VMLINUX_LIBS All .a "lib" files for vmlinux. KBUILD_VMLINUX_OBJS and @@ -1273,8 +1306,7 @@ When kbuild executes, the following steps are followed (roughly): machinery is all architecture-independent. - head-y, core-y, libs-y, drivers-y - $(head-y) lists objects to be linked first in vmlinux. + core-y, libs-y, drivers-y $(libs-y) lists directories where a lib.a archive can be located. diff --git a/Documentation/locking/seqlock.rst b/Documentation/locking/seqlock.rst index 64405e5da63e4379df29ea77b800b1473a075a25..bfda1a5fecadc60005156f2c448d408edfce8cd0 100644 --- a/Documentation/locking/seqlock.rst +++ b/Documentation/locking/seqlock.rst @@ -39,7 +39,7 @@ as the writer can invalidate a pointer that the reader is following. Sequence counters (``seqcount_t``) ================================== -This is the the raw counting mechanism, which does not protect against +This is the raw counting mechanism, which does not protect against multiple writers. Write side critical sections must thus be serialized by an external lock. diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index 832b5d36e279cd4a89a6f3447c15c7ea3a3a6fef..06f80e3785c5de0b616f8ededfce57c9965a28f4 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -52,7 +52,7 @@ CONTENTS - Varieties of memory barrier. - What may not be assumed about memory barriers? - - Data dependency barriers (historical). + - Address-dependency barriers (historical). - Control dependencies. - SMP barrier pairing. - Examples of memory barrier sequences. @@ -187,9 +187,9 @@ As a further example, consider this sequence of events: B = 4; Q = P; P = &B; D = *Q; -There is an obvious data dependency here, as the value loaded into D depends on -the address retrieved from P by CPU 2. At the end of the sequence, any of the -following results are possible: +There is an obvious address dependency here, as the value loaded into D depends +on the address retrieved from P by CPU 2. At the end of the sequence, any of +the following results are possible: (Q == &A) and (D == 1) (Q == &B) and (D == 2) @@ -391,58 +391,62 @@ Memory barriers come in four basic varieties: memory system as time progresses. All stores _before_ a write barrier will occur _before_ all the stores after the write barrier. - [!] Note that write barriers should normally be paired with read or data - dependency barriers; see the "SMP barrier pairing" subsection. + [!] Note that write barriers should normally be paired with read or + address-dependency barriers; see the "SMP barrier pairing" subsection. - (2) Data dependency barriers. + (2) Address-dependency barriers (historical). - A data dependency barrier is a weaker form of read barrier. In the case - where two loads are performed such that the second depends on the result - of the first (eg: the first load retrieves the address to which the second - load will be directed), a data dependency barrier would be required to - make sure that the target of the second load is updated after the address - obtained by the first load is accessed. + An address-dependency barrier is a weaker form of read barrier. In the + case where two loads are performed such that the second depends on the + result of the first (eg: the first load retrieves the address to which + the second load will be directed), an address-dependency barrier would + be required to make sure that the target of the second load is updated + after the address obtained by the first load is accessed. - A data dependency barrier is a partial ordering on interdependent loads - only; it is not required to have any effect on stores, independent loads - or overlapping loads. + An address-dependency barrier is a partial ordering on interdependent + loads only; it is not required to have any effect on stores, independent + loads or overlapping loads. As mentioned in (1), the other CPUs in the system can be viewed as committing sequences of stores to the memory system that the CPU being - considered can then perceive. A data dependency barrier issued by the CPU - under consideration guarantees that for any load preceding it, if that - load touches one of a sequence of stores from another CPU, then by the - time the barrier completes, the effects of all the stores prior to that - touched by the load will be perceptible to any loads issued after the data - dependency barrier. + considered can then perceive. An address-dependency barrier issued by + the CPU under consideration guarantees that for any load preceding it, + if that load touches one of a sequence of stores from another CPU, then + by the time the barrier completes, the effects of all the stores prior to + that touched by the load will be perceptible to any loads issued after + the address-dependency barrier. See the "Examples of memory barrier sequences" subsection for diagrams showing the ordering constraints. - [!] Note that the first load really has to have a _data_ dependency and + [!] Note that the first load really has to have an _address_ dependency and not a control dependency. If the address for the second load is dependent on the first load, but the dependency is through a conditional rather than actually loading the address itself, then it's a _control_ dependency and a full read barrier or better is required. See the "Control dependencies" subsection for more information. - [!] Note that data dependency barriers should normally be paired with + [!] Note that address-dependency barriers should normally be paired with write barriers; see the "SMP barrier pairing" subsection. + [!] Kernel release v5.9 removed kernel APIs for explicit address- + dependency barriers. Nowadays, APIs for marking loads from shared + variables such as READ_ONCE() and rcu_dereference() provide implicit + address-dependency barriers. (3) Read (or load) memory barriers. - A read barrier is a data dependency barrier plus a guarantee that all the - LOAD operations specified before the barrier will appear to happen before - all the LOAD operations specified after the barrier with respect to the - other components of the system. + A read barrier is an address-dependency barrier plus a guarantee that all + the LOAD operations specified before the barrier will appear to happen + before all the LOAD operations specified after the barrier with respect to + the other components of the system. A read barrier is a partial ordering on loads only; it is not required to have any effect on stores. - Read memory barriers imply data dependency barriers, and so can substitute - for them. + Read memory barriers imply address-dependency barriers, and so can + substitute for them. [!] Note that read barriers should normally be paired with write barriers; see the "SMP barrier pairing" subsection. @@ -550,17 +554,21 @@ There are certain things that the Linux kernel memory barriers do not guarantee: Documentation/core-api/dma-api.rst -DATA DEPENDENCY BARRIERS (HISTORICAL) -------------------------------------- +ADDRESS-DEPENDENCY BARRIERS (HISTORICAL) +---------------------------------------- As of v4.15 of the Linux kernel, an smp_mb() was added to READ_ONCE() for DEC Alpha, which means that about the only people who need to pay attention to this section are those working on DEC Alpha architecture-specific code and those working on READ_ONCE() itself. For those who need it, and for those who are interested in the history, here is the story of -data-dependency barriers. +address-dependency barriers. + +[!] While address dependencies are observed in both load-to-load and +load-to-store relations, address-dependency barriers are not necessary +for load-to-store situations. -The usage requirements of data dependency barriers are a little subtle, and +The requirement of address-dependency barriers is a little subtle, and it's not always obvious that they're needed. To illustrate, consider the following sequence of events: @@ -570,11 +578,14 @@ following sequence of events: B = 4; WRITE_ONCE(P, &B); - Q = READ_ONCE(P); + Q = READ_ONCE_OLD(P); D = *Q; -There's a clear data dependency here, and it would seem that by the end of the -sequence, Q must be either &A or &B, and that: +[!] READ_ONCE_OLD() corresponds to READ_ONCE() of pre-4.15 kernel, which +doesn't imply an address-dependency barrier. + +There's a clear address dependency here, and it would seem that by the end of +the sequence, Q must be either &A or &B, and that: (Q == &A) implies (D == 1) (Q == &B) implies (D == 4) @@ -588,8 +599,8 @@ While this may seem like a failure of coherency or causality maintenance, it isn't, and this behaviour can be observed on certain real CPUs (such as the DEC Alpha). -To deal with this, a data dependency barrier or better must be inserted -between the address load and the data load: +To deal with this, READ_ONCE() provides an implicit address-dependency barrier +since kernel release v4.15: CPU 1 CPU 2 =============== =============== @@ -598,7 +609,7 @@ between the address load and the data load: WRITE_ONCE(P, &B); Q = READ_ONCE(P); - + D = *Q; This enforces the occurrence of one of the two implications, and prevents the @@ -615,13 +626,13 @@ odd-numbered bank is idle, one can see the new value of the pointer P (&B), but the old value of the variable B (2). -A data-dependency barrier is not required to order dependent writes -because the CPUs that the Linux kernel supports don't do writes -until they are certain (1) that the write will actually happen, (2) -of the location of the write, and (3) of the value to be written. +An address-dependency barrier is not required to order dependent writes +because the CPUs that the Linux kernel supports don't do writes until they +are certain (1) that the write will actually happen, (2) of the location of +the write, and (3) of the value to be written. But please carefully read the "CONTROL DEPENDENCIES" section and the -Documentation/RCU/rcu_dereference.rst file: The compiler can and does -break dependencies in a great many highly creative ways. +Documentation/RCU/rcu_dereference.rst file: The compiler can and does break +dependencies in a great many highly creative ways. CPU 1 CPU 2 =============== =============== @@ -629,12 +640,12 @@ break dependencies in a great many highly creative ways. B = 4; WRITE_ONCE(P, &B); - Q = READ_ONCE(P); + Q = READ_ONCE_OLD(P); WRITE_ONCE(*Q, 5); -Therefore, no data-dependency barrier is required to order the read into +Therefore, no address-dependency barrier is required to order the read into Q with the store into *Q. In other words, this outcome is prohibited, -even without a data-dependency barrier: +even without an implicit address-dependency barrier of modern READ_ONCE(): (Q == &B) && (B == 4) @@ -645,12 +656,12 @@ can be used to record rare error conditions and the like, and the CPUs' naturally occurring ordering prevents such records from being lost. -Note well that the ordering provided by a data dependency is local to +Note well that the ordering provided by an address dependency is local to the CPU containing it. See the section on "Multicopy atomicity" for more information. -The data dependency barrier is very important to the RCU system, +The address-dependency barrier is very important to the RCU system, for example. See rcu_assign_pointer() and rcu_dereference() in include/linux/rcupdate.h. This permits the current target of an RCU'd pointer to be replaced with a new modified target, without the replacement @@ -667,20 +678,21 @@ not understand them. The purpose of this section is to help you prevent the compiler's ignorance from breaking your code. A load-load control dependency requires a full read memory barrier, not -simply a data dependency barrier to make it work correctly. Consider the -following bit of code: +simply an (implicit) address-dependency barrier to make it work correctly. +Consider the following bit of code: q = READ_ONCE(a); + if (q) { - /* BUG: No data dependency!!! */ + /* BUG: No address dependency!!! */ p = READ_ONCE(b); } -This will not have the desired effect because there is no actual data +This will not have the desired effect because there is no actual address dependency, but rather a control dependency that the CPU may short-circuit by attempting to predict the outcome in advance, so that other CPUs see -the load from b as having happened before the load from a. In such a -case what's actually required is: +the load from b as having happened before the load from a. In such a case +what's actually required is: q = READ_ONCE(a); if (q) { @@ -927,9 +939,9 @@ General barriers pair with each other, though they also pair with most other types of barriers, albeit without multicopy atomicity. An acquire barrier pairs with a release barrier, but both may also pair with other barriers, including of course general barriers. A write barrier pairs -with a data dependency barrier, a control dependency, an acquire barrier, +with an address-dependency barrier, a control dependency, an acquire barrier, a release barrier, a read barrier, or a general barrier. Similarly a -read barrier, control dependency, or a data dependency barrier pairs +read barrier, control dependency, or an address-dependency barrier pairs with a write barrier, an acquire barrier, a release barrier, or a general barrier: @@ -948,7 +960,7 @@ Or: a = 1; WRITE_ONCE(b, &a); x = READ_ONCE(b); - + y = *x; Or even: @@ -968,8 +980,8 @@ Basically, the read barrier always has to be there, even though it can be of the "weaker" type. [!] Note that the stores before the write barrier would normally be expected to -match the loads after the read barrier or the data dependency barrier, and vice -versa: +match the loads after the read barrier or the address-dependency barrier, and +vice versa: CPU 1 CPU 2 =================== =================== @@ -1021,8 +1033,8 @@ STORE B, STORE C } all occurring before the unordered set of { STORE D, STORE E V -Secondly, data dependency barriers act as partial orderings on data-dependent -loads. Consider the following sequence of events: +Secondly, address-dependency barriers act as partial orderings on address- +dependent loads. Consider the following sequence of events: CPU 1 CPU 2 ======================= ======================= @@ -1067,8 +1079,8 @@ effectively random order, despite the write barrier issued by CPU 1: In the above example, CPU 2 perceives that B is 7, despite the load of *C (which would be B) coming after the LOAD of C. -If, however, a data dependency barrier were to be placed between the load of C -and the load of *C (ie: B) on CPU 2: +If, however, an address-dependency barrier were to be placed between the load +of C and the load of *C (ie: B) on CPU 2: CPU 1 CPU 2 ======================= ======================= @@ -1078,7 +1090,7 @@ and the load of *C (ie: B) on CPU 2: STORE C = &B LOAD X STORE D = 4 LOAD C (gets &B) - + LOAD *C (reads B) then the following will occur: @@ -1101,7 +1113,7 @@ then the following will occur: | +-------+ | | | | X->9 |------>| | | +-------+ | | - Makes sure all effects ---> \ ddddddddddddddddd | | + Makes sure all effects ---> \ aaaaaaaaaaaaaaaaa | | prior to the store of C \ +-------+ | | are perceptible to ----->| B->2 |------>| | subsequent loads +-------+ | | @@ -1292,7 +1304,7 @@ Which might appear as this: LOAD with immediate effect : : +-------+ -Placing a read barrier or a data dependency barrier just before the second +Placing a read barrier or an address-dependency barrier just before the second load: CPU 1 CPU 2 @@ -1816,20 +1828,20 @@ which may then reorder things however it wishes. CPU MEMORY BARRIERS ------------------- -The Linux kernel has eight basic CPU memory barriers: +The Linux kernel has seven basic CPU memory barriers: - TYPE MANDATORY SMP CONDITIONAL - =============== ======================= =========================== - GENERAL mb() smp_mb() - WRITE wmb() smp_wmb() - READ rmb() smp_rmb() - DATA DEPENDENCY READ_ONCE() + TYPE MANDATORY SMP CONDITIONAL + ======================= =============== =============== + GENERAL mb() smp_mb() + WRITE wmb() smp_wmb() + READ rmb() smp_rmb() + ADDRESS DEPENDENCY READ_ONCE() -All memory barriers except the data dependency barriers imply a compiler -barrier. Data dependencies do not impose any additional compiler ordering. +All memory barriers except the address-dependency barriers imply a compiler +barrier. Address dependencies do not impose any additional compiler ordering. -Aside: In the case of data dependencies, the compiler would be expected +Aside: In the case of address dependencies, the compiler would be expected to issue the loads in the correct order (eg. `a[b]` would have to load the value of b before loading a[b]), however there is no guarantee in the C specification that the compiler may not speculate the value of b @@ -2749,7 +2761,8 @@ is discarded from the CPU's cache and reloaded. To deal with this, the appropriate part of the kernel must invalidate the overlapping bits of the cache on each CPU. -See Documentation/core-api/cachetlb.rst for more information on cache management. +See Documentation/core-api/cachetlb.rst for more information on cache +management. CACHE COHERENCY VS MMIO @@ -2889,8 +2902,8 @@ AND THEN THERE'S THE ALPHA The DEC Alpha CPU is one of the most relaxed CPUs there is. Not only that, some versions of the Alpha CPU have a split data cache, permitting them to have two semantically-related cache lines updated at separate times. This is where -the data dependency barrier really becomes necessary as this synchronises both -caches with the memory coherence system, thus making it seem like pointer +the address-dependency barrier really becomes necessary as this synchronises +both caches with the memory coherence system, thus making it seem like pointer changes vs new data occur in the right order. The Alpha defines the Linux kernel's memory model, although as of v4.15 diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 575ccd40e30cfa706b266023bc20c43042398a79..4aa12b8be278d324744fc50aaf5058554362fe55 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -51,6 +51,7 @@ above structured documentation, or deleted if it has served its purpose. ksm memory-model mmu_notifier + multigen_lru numa overcommit-accounting page_migration diff --git a/Documentation/mm/ksm.rst b/Documentation/mm/ksm.rst index 9e37add068e64ccb7201e89cc93c6c6fd8f4b571..f83cfbc12f4ca9be68958fe7b76b5b41996a05e4 100644 --- a/Documentation/mm/ksm.rst +++ b/Documentation/mm/ksm.rst @@ -26,7 +26,7 @@ tree. If a KSM page is shared between less than ``max_page_sharing`` VMAs, the node of the stable tree that represents such KSM page points to a -list of struct rmap_item and the ``page->mapping`` of the +list of struct ksm_rmap_item and the ``page->mapping`` of the KSM page points to the stable tree node. When the sharing passes this threshold, KSM adds a second dimension to diff --git a/Documentation/mm/multigen_lru.rst b/Documentation/mm/multigen_lru.rst new file mode 100644 index 0000000000000000000000000000000000000000..d7062c6a8946466a06b4b5f88acf430ab4adba72 --- /dev/null +++ b/Documentation/mm/multigen_lru.rst @@ -0,0 +1,159 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +Multi-Gen LRU +============= +The multi-gen LRU is an alternative LRU implementation that optimizes +page reclaim and improves performance under memory pressure. Page +reclaim decides the kernel's caching policy and ability to overcommit +memory. It directly impacts the kswapd CPU usage and RAM efficiency. + +Design overview +=============== +Objectives +---------- +The design objectives are: + +* Good representation of access recency +* Try to profit from spatial locality +* Fast paths to make obvious choices +* Simple self-correcting heuristics + +The representation of access recency is at the core of all LRU +implementations. In the multi-gen LRU, each generation represents a +group of pages with similar access recency. Generations establish a +(time-based) common frame of reference and therefore help make better +choices, e.g., between different memcgs on a computer or different +computers in a data center (for job scheduling). + +Exploiting spatial locality improves efficiency when gathering the +accessed bit. A rmap walk targets a single page and does not try to +profit from discovering a young PTE. A page table walk can sweep all +the young PTEs in an address space, but the address space can be too +sparse to make a profit. The key is to optimize both methods and use +them in combination. + +Fast paths reduce code complexity and runtime overhead. Unmapped pages +do not require TLB flushes; clean pages do not require writeback. +These facts are only helpful when other conditions, e.g., access +recency, are similar. With generations as a common frame of reference, +additional factors stand out. But obvious choices might not be good +choices; thus self-correction is necessary. + +The benefits of simple self-correcting heuristics are self-evident. +Again, with generations as a common frame of reference, this becomes +attainable. Specifically, pages in the same generation can be +categorized based on additional factors, and a feedback loop can +statistically compare the refault percentages across those categories +and infer which of them are better choices. + +Assumptions +----------- +The protection of hot pages and the selection of cold pages are based +on page access channels and patterns. There are two access channels: + +* Accesses through page tables +* Accesses through file descriptors + +The protection of the former channel is by design stronger because: + +1. The uncertainty in determining the access patterns of the former + channel is higher due to the approximation of the accessed bit. +2. The cost of evicting the former channel is higher due to the TLB + flushes required and the likelihood of encountering the dirty bit. +3. The penalty of underprotecting the former channel is higher because + applications usually do not prepare themselves for major page + faults like they do for blocked I/O. E.g., GUI applications + commonly use dedicated I/O threads to avoid blocking rendering + threads. + +There are also two access patterns: + +* Accesses exhibiting temporal locality +* Accesses not exhibiting temporal locality + +For the reasons listed above, the former channel is assumed to follow +the former pattern unless ``VM_SEQ_READ`` or ``VM_RAND_READ`` is +present, and the latter channel is assumed to follow the latter +pattern unless outlying refaults have been observed. + +Workflow overview +================= +Evictable pages are divided into multiple generations for each +``lruvec``. The youngest generation number is stored in +``lrugen->max_seq`` for both anon and file types as they are aged on +an equal footing. The oldest generation numbers are stored in +``lrugen->min_seq[]`` separately for anon and file types as clean file +pages can be evicted regardless of swap constraints. These three +variables are monotonically increasing. + +Generation numbers are truncated into ``order_base_2(MAX_NR_GENS+1)`` +bits in order to fit into the gen counter in ``folio->flags``. Each +truncated generation number is an index to ``lrugen->lists[]``. The +sliding window technique is used to track at least ``MIN_NR_GENS`` and +at most ``MAX_NR_GENS`` generations. The gen counter stores a value +within ``[1, MAX_NR_GENS]`` while a page is on one of +``lrugen->lists[]``; otherwise it stores zero. + +Each generation is divided into multiple tiers. A page accessed ``N`` +times through file descriptors is in tier ``order_base_2(N)``. Unlike +generations, tiers do not have dedicated ``lrugen->lists[]``. In +contrast to moving across generations, which requires the LRU lock, +moving across tiers only involves atomic operations on +``folio->flags`` and therefore has a negligible cost. A feedback loop +modeled after the PID controller monitors refaults over all the tiers +from anon and file types and decides which tiers from which types to +evict or protect. + +There are two conceptually independent procedures: the aging and the +eviction. They form a closed-loop system, i.e., the page reclaim. + +Aging +----- +The aging produces young generations. Given an ``lruvec``, it +increments ``max_seq`` when ``max_seq-min_seq+1`` approaches +``MIN_NR_GENS``. The aging promotes hot pages to the youngest +generation when it finds them accessed through page tables; the +demotion of cold pages happens consequently when it increments +``max_seq``. The aging uses page table walks and rmap walks to find +young PTEs. For the former, it iterates ``lruvec_memcg()->mm_list`` +and calls ``walk_page_range()`` with each ``mm_struct`` on this list +to scan PTEs, and after each iteration, it increments ``max_seq``. For +the latter, when the eviction walks the rmap and finds a young PTE, +the aging scans the adjacent PTEs. For both, on finding a young PTE, +the aging clears the accessed bit and updates the gen counter of the +page mapped by this PTE to ``(max_seq%MAX_NR_GENS)+1``. + +Eviction +-------- +The eviction consumes old generations. Given an ``lruvec``, it +increments ``min_seq`` when ``lrugen->lists[]`` indexed by +``min_seq%MAX_NR_GENS`` becomes empty. To select a type and a tier to +evict from, it first compares ``min_seq[]`` to select the older type. +If both types are equally old, it selects the one whose first tier has +a lower refault percentage. The first tier contains single-use +unmapped clean pages, which are the best bet. The eviction sorts a +page according to its gen counter if the aging has found this page +accessed through page tables and updated its gen counter. It also +moves a page to the next generation, i.e., ``min_seq+1``, if this page +was accessed multiple times through file descriptors and the feedback +loop has detected outlying refaults from the tier this page is in. To +this end, the feedback loop uses the first tier as the baseline, for +the reason stated earlier. + +Summary +------- +The multi-gen LRU can be disassembled into the following parts: + +* Generations +* Rmap walks +* Page table walks +* Bloom filters +* PID controller + +The aging and the eviction form a producer-consumer model; +specifically, the latter drives the former by the sliding window over +generations. Within the aging, rmap walks drive page table walks by +inserting hot densely populated page tables to the Bloom filters. +Within the eviction, the PID controller uses refaults as the feedback +to select types to evict and tiers to protect. diff --git a/Documentation/mm/page_owner.rst b/Documentation/mm/page_owner.rst index f5c954afe97c7473418a0e617b55e00cc4d7b697..127514955a5e8021b1fe5ed8d1fe63b80c03679b 100644 --- a/Documentation/mm/page_owner.rst +++ b/Documentation/mm/page_owner.rst @@ -38,22 +38,10 @@ not affect to allocation performance, especially if the static keys jump label patching functionality is available. Following is the kernel's code size change due to this facility. -- Without page owner:: - - text data bss dec hex filename - 48392 2333 644 51369 c8a9 mm/page_alloc.o - -- With page owner:: - - text data bss dec hex filename - 48800 2445 644 51889 cab1 mm/page_alloc.o - 6662 108 29 6799 1a8f mm/page_owner.o - 1025 8 8 1041 411 mm/page_ext.o - -Although, roughly, 8 KB code is added in total, page_alloc.o increase by -520 bytes and less than half of it is in hotpath. Building the kernel with -page owner and turning it on if needed would be great option to debug -kernel memory problem. +Although enabling page owner increases kernel size by several kilobytes, +most of this code is outside page allocator and its hot path. Building +the kernel with page owner and turning it on if needed would be great +option to debug kernel memory problem. There is one notice that is caused by implementation detail. page owner stores information into the memory from struct page extension. This memory @@ -94,6 +82,11 @@ Usage Page allocated via order XXX, ... PFN XXX ... // Detailed stack + By default, it will do full pfn dump, to start with a given pfn, + page_owner supports fseek. + + FILE *fp = fopen("/sys/kernel/debug/page_owner", "r"); + fseek(fp, pfn_start, SEEK_SET); The ``page_owner_sort`` tool ignores ``PFN`` rows, puts the remaining rows in buf, uses regexp to extract the page order value, counts the times diff --git a/Documentation/mm/slub.rst b/Documentation/mm/slub.rst index 43063ade737af246b0dd13b9f0f9fb567cff479b..4e1578186b4f0c5a9634747b05788f2bbbd4ae8c 100644 --- a/Documentation/mm/slub.rst +++ b/Documentation/mm/slub.rst @@ -400,21 +400,30 @@ information: allocated objects. The output is sorted by frequency of each trace. Information in the output: - Number of objects, allocating function, minimal/average/maximal jiffies since alloc, - pid range of the allocating processes, cpu mask of allocating cpus, and stack trace. + Number of objects, allocating function, possible memory wastage of + kmalloc objects(total/per-object), minimal/average/maximal jiffies + since alloc, pid range of the allocating processes, cpu mask of + allocating cpus, numa node mask of origins of memory, and stack trace. Example::: - 1085 populate_error_injection_list+0x97/0x110 age=166678/166680/166682 pid=1 cpus=1:: - __slab_alloc+0x6d/0x90 - kmem_cache_alloc_trace+0x2eb/0x300 - populate_error_injection_list+0x97/0x110 - init_error_injection+0x1b/0x71 - do_one_initcall+0x5f/0x2d0 - kernel_init_freeable+0x26f/0x2d7 - kernel_init+0xe/0x118 - ret_from_fork+0x22/0x30 - + 338 pci_alloc_dev+0x2c/0xa0 waste=521872/1544 age=290837/291891/293509 pid=1 cpus=106 nodes=0-1 + __kmem_cache_alloc_node+0x11f/0x4e0 + kmalloc_trace+0x26/0xa0 + pci_alloc_dev+0x2c/0xa0 + pci_scan_single_device+0xd2/0x150 + pci_scan_slot+0xf7/0x2d0 + pci_scan_child_bus_extend+0x4e/0x360 + acpi_pci_root_create+0x32e/0x3b0 + pci_acpi_scan_root+0x2b9/0x2d0 + acpi_pci_root_add.cold.11+0x110/0xb0a + acpi_bus_attach+0x262/0x3f0 + device_for_each_child+0xb7/0x110 + acpi_dev_for_each_child+0x77/0xa0 + acpi_bus_attach+0x108/0x3f0 + device_for_each_child+0xb7/0x110 + acpi_dev_for_each_child+0x77/0xa0 + acpi_bus_attach+0x108/0x3f0 2. free_traces:: diff --git a/Documentation/mm/unevictable-lru.rst b/Documentation/mm/unevictable-lru.rst index b280367d6a44c321e76f7396b6e7e520cda50aa1..4a0e158aa9ce155320dbedad559c79575b7470c0 100644 --- a/Documentation/mm/unevictable-lru.rst +++ b/Documentation/mm/unevictable-lru.rst @@ -197,7 +197,7 @@ unevictable list for the memory cgroup and node being scanned. There may be situations where a page is mapped into a VM_LOCKED VMA, but the page is not marked as PG_mlocked. Such pages will make it all the way to shrink_active_list() or shrink_page_list() where they will be detected when -vmscan walks the reverse map in page_referenced() or try_to_unmap(). The page +vmscan walks the reverse map in folio_referenced() or try_to_unmap(). The page is culled to the unevictable list when it is released by the shrinker. To "cull" an unevictable page, vmscan simply puts the page back on the LRU list @@ -267,7 +267,7 @@ the LRU. Such pages can be "noticed" by memory management in several places: (4) in the fault path and when a VM_LOCKED stack segment is expanded; or (5) as mentioned above, in vmscan:shrink_page_list() when attempting to - reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap(). + reclaim a page in a VM_LOCKED VMA by folio_referenced() or try_to_unmap(). mlocked pages become unlocked and rescued from the unevictable list when: @@ -547,7 +547,7 @@ vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously unevictable pages found on the inactive lists to the appropriate memory cgroup and node unevictable list. -rmap's page_referenced_one(), called via vmscan's shrink_active_list() or +rmap's folio_referenced_one(), called via vmscan's shrink_active_list() or shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(), check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page() to correct them. Such pages are culled to the unevictable list when released diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index 7823a069a903170faa0648b9a6b5b6c6216da9c2..96cd7a26f3d918344841a2f9081f39ce518fe2d7 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -846,7 +846,7 @@ primary_reselect tlb_dynamic_lb Specifies if dynamic shuffling of flows is enabled in tlb - mode. The value has no effect on any other modes. + or alb mode. The value has no effect on any other modes. The default behavior of tlb mode is to shuffle active flows across slaves based on the load in that interval. This gives nice lb diff --git a/Documentation/networking/decnet.rst b/Documentation/networking/decnet.rst deleted file mode 100644 index b8bc11ff8370dd5be7d33055b2cffaa29bbde00a..0000000000000000000000000000000000000000 --- a/Documentation/networking/decnet.rst +++ /dev/null @@ -1,243 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -========================================= -Linux DECnet Networking Layer Information -========================================= - -1. Other documentation.... -========================== - - - Project Home Pages - - http://www.chygwyn.com/ - Kernel info - - http://linux-decnet.sourceforge.net/ - Userland tools - - http://www.sourceforge.net/projects/linux-decnet/ - Status page - -2. Configuring the kernel -========================= - -Be sure to turn on the following options: - - - CONFIG_DECNET (obviously) - - CONFIG_PROC_FS (to see what's going on) - - CONFIG_SYSCTL (for easy configuration) - -if you want to try out router support (not properly debugged yet) -you'll need the following options as well... - - - CONFIG_DECNET_ROUTER (to be able to add/delete routes) - - CONFIG_NETFILTER (will be required for the DECnet routing daemon) - -Don't turn on SIOCGIFCONF support for DECnet unless you are really sure -that you need it, in general you won't and it can cause ifconfig to -malfunction. - -Run time configuration has changed slightly from the 2.4 system. If you -want to configure an endnode, then the simplified procedure is as follows: - - - Set the MAC address on your ethernet card before starting _any_ other - network protocols. - -As soon as your network card is brought into the UP state, DECnet should -start working. If you need something more complicated or are unsure how -to set the MAC address, see the next section. Also all configurations which -worked with 2.4 will work under 2.5 with no change. - -3. Command line options -======================= - -You can set a DECnet address on the kernel command line for compatibility -with the 2.4 configuration procedure, but in general it's not needed any more. -If you do st a DECnet address on the command line, it has only one purpose -which is that its added to the addresses on the loopback device. - -With 2.4 kernels, DECnet would only recognise addresses as local if they -were added to the loopback device. In 2.5, any local interface address -can be used to loop back to the local machine. Of course this does not -prevent you adding further addresses to the loopback device if you -want to. - -N.B. Since the address list of an interface determines the addresses for -which "hello" messages are sent, if you don't set an address on the loopback -interface then you won't see any entries in /proc/net/neigh for the local -host until such time as you start a connection. This doesn't affect the -operation of the local communications in any other way though. - -The kernel command line takes options looking like the following:: - - decnet.addr=1,2 - -the two numbers are the node address 1,2 = 1.2 For 2.2.xx kernels -and early 2.3.xx kernels, you must use a comma when specifying the -DECnet address like this. For more recent 2.3.xx kernels, you may -use almost any character except space, although a `.` would be the most -obvious choice :-) - -There used to be a third number specifying the node type. This option -has gone away in favour of a per interface node type. This is now set -using /proc/sys/net/decnet/conf//forwarding. This file can be -set with a single digit, 0=EndNode, 1=L1 Router and 2=L2 Router. - -There are also equivalent options for modules. The node address can -also be set through the /proc/sys/net/decnet/ files, as can other system -parameters. - -Currently the only supported devices are ethernet and ip_gre. The -ethernet address of your ethernet card has to be set according to the DECnet -address of the node in order for it to be autoconfigured (and then appear in -/proc/net/decnet_dev). There is a utility available at the above -FTP sites called dn2ethaddr which can compute the correct ethernet -address to use. The address can be set by ifconfig either before or -at the time the device is brought up. If you are using RedHat you can -add the line:: - - MACADDR=AA:00:04:00:03:04 - -or something similar, to /etc/sysconfig/network-scripts/ifcfg-eth0 or -wherever your network card's configuration lives. Setting the MAC address -of your ethernet card to an address starting with "hi-ord" will cause a -DECnet address which matches to be added to the interface (which you can -verify with iproute2). - -The default device for routing can be set through the /proc filesystem -by setting /proc/sys/net/decnet/default_device to the -device you want DECnet to route packets out of when no specific route -is available. Usually this will be eth0, for example:: - - echo -n "eth0" >/proc/sys/net/decnet/default_device - -If you don't set the default device, then it will default to the first -ethernet card which has been autoconfigured as described above. You can -confirm that by looking in the default_device file of course. - -There is a list of what the other files under /proc/sys/net/decnet/ do -on the kernel patch web site (shown above). - -4. Run time kernel configuration -================================ - - -This is either done through the sysctl/proc interface (see the kernel web -pages for details on what the various options do) or through the iproute2 -package in the same way as IPv4/6 configuration is performed. - -Documentation for iproute2 is included with the package, although there is -as yet no specific section on DECnet, most of the features apply to both -IP and DECnet, albeit with DECnet addresses instead of IP addresses and -a reduced functionality. - -If you want to configure a DECnet router you'll need the iproute2 package -since its the _only_ way to add and delete routes currently. Eventually -there will be a routing daemon to send and receive routing messages for -each interface and update the kernel routing tables accordingly. The -routing daemon will use netfilter to listen to routing packets, and -rtnetlink to update the kernels routing tables. - -The DECnet raw socket layer has been removed since it was there purely -for use by the routing daemon which will now use netfilter (a much cleaner -and more generic solution) instead. - -5. How can I tell if its working? -================================= - -Here is a quick guide of what to look for in order to know if your DECnet -kernel subsystem is working. - - - Is the node address set (see /proc/sys/net/decnet/node_address) - - Is the node of the correct type - (see /proc/sys/net/decnet/conf//forwarding) - - Is the Ethernet MAC address of each Ethernet card set to match - the DECnet address. If in doubt use the dn2ethaddr utility available - at the ftp archive. - - If the previous two steps are satisfied, and the Ethernet card is up, - you should find that it is listed in /proc/net/decnet_dev and also - that it appears as a directory in /proc/sys/net/decnet/conf/. The - loopback device (lo) should also appear and is required to communicate - within a node. - - If you have any DECnet routers on your network, they should appear - in /proc/net/decnet_neigh, otherwise this file will only contain the - entry for the node itself (if it doesn't check to see if lo is up). - - If you want to send to any node which is not listed in the - /proc/net/decnet_neigh file, you'll need to set the default device - to point to an Ethernet card with connection to a router. This is - again done with the /proc/sys/net/decnet/default_device file. - - Try starting a simple server and client, like the dnping/dnmirror - over the loopback interface. With luck they should communicate. - For this step and those after, you'll need the DECnet library - which can be obtained from the above ftp sites as well as the - actual utilities themselves. - - If this seems to work, then try talking to a node on your local - network, and see if you can obtain the same results. - - At this point you are on your own... :-) - -6. How to send a bug report -=========================== - -If you've found a bug and want to report it, then there are several things -you can do to help me work out exactly what it is that is wrong. Useful -information (_most_ of which _is_ _essential_) includes: - - - What kernel version are you running ? - - What version of the patch are you running ? - - How far though the above set of tests can you get ? - - What is in the /proc/decnet* files and /proc/sys/net/decnet/* files ? - - Which services are you running ? - - Which client caused the problem ? - - How much data was being transferred ? - - Was the network congested ? - - How can the problem be reproduced ? - - Can you use tcpdump to get a trace ? (N.B. Most (all?) versions of - tcpdump don't understand how to dump DECnet properly, so including - the hex listing of the packet contents is _essential_, usually the -x flag. - You may also need to increase the length grabbed with the -s flag. The - -e flag also provides very useful information (ethernet MAC addresses)) - -7. MAC FAQ -========== - -A quick FAQ on ethernet MAC addresses to explain how Linux and DECnet -interact and how to get the best performance from your hardware. - -Ethernet cards are designed to normally only pass received network frames -to a host computer when they are addressed to it, or to the broadcast address. - -Linux has an interface which allows the setting of extra addresses for -an ethernet card to listen to. If the ethernet card supports it, the -filtering operation will be done in hardware, if not the extra unwanted packets -received will be discarded by the host computer. In the latter case, -significant processor time and bus bandwidth can be used up on a busy -network (see the NAPI documentation for a longer explanation of these -effects). - -DECnet makes use of this interface to allow running DECnet on an ethernet -card which has already been configured using TCP/IP (presumably using the -built in MAC address of the card, as usual) and/or to allow multiple DECnet -addresses on each physical interface. If you do this, be aware that if your -ethernet card doesn't support perfect hashing in its MAC address filter -then your computer will be doing more work than required. Some cards -will simply set themselves into promiscuous mode in order to receive -packets from the DECnet specified addresses. So if you have one of these -cards its better to set the MAC address of the card as described above -to gain the best efficiency. Better still is to use a card which supports -NAPI as well. - - -8. Mailing list -=============== - -If you are keen to get involved in development, or want to ask questions -about configuration, or even just report bugs, then there is a mailing -list that you can join, details are at: - -http://sourceforge.net/mail/?group_id=4993 - -9. Legal Info -============= - -The Linux DECnet project team have placed their code under the GPL. The -software is provided "as is" and without warranty express or implied. -DECnet is a trademark of Compaq. This software is not a product of -Compaq. We acknowledge the help of people at Compaq in providing extra -documentation above and beyond what was previously publicly available. - -Steve Whitehouse - diff --git a/Documentation/networking/device_drivers/can/freescale/flexcan.rst b/Documentation/networking/device_drivers/can/freescale/flexcan.rst index 4e3eec6cecd2a725348dc138c9793fca7eaf6815..106cd28901354f6c816270bf6b67a8e7f98351ce 100644 --- a/Documentation/networking/device_drivers/can/freescale/flexcan.rst +++ b/Documentation/networking/device_drivers/can/freescale/flexcan.rst @@ -5,7 +5,7 @@ Flexcan CAN Controller driver ============================= Authors: Marc Kleine-Budde , -Dario Binacchi +Dario Binacchi On/off RTR frames reception =========================== diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst index 7f1777173abb46f6eee296fd3f794abc0cd7bec2..5196905582c5b34e5d6ad2b8e6d6709f031ffba5 100644 --- a/Documentation/networking/device_drivers/ethernet/index.rst +++ b/Documentation/networking/device_drivers/ethernet/index.rst @@ -52,6 +52,7 @@ Contents: ti/tlan toshiba/spider_net wangxun/txgbe + wangxun/ngbe .. only:: subproject and html diff --git a/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst b/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst new file mode 100644 index 0000000000000000000000000000000000000000..43a02f9943e14b7ba572781c7ab97e3930fe7c98 --- /dev/null +++ b/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================================================= +Linux Base Driver for WangXun(R) Gigabit PCI Express Adapters +============================================================= + +WangXun Gigabit Linux driver. +Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. + +Support +======= + If you have problems with the software or hardware, please contact our + customer support team via email at nic-support@net-swift.com or check our website + at https://www.net-swift.com diff --git a/Documentation/networking/devlink/ice.rst b/Documentation/networking/devlink/ice.rst index 8c082b139bbdb9b036aee876aa8afe8649663b4f..0c89ceb8986d282df8ceaf78b8701978979cc99e 100644 --- a/Documentation/networking/devlink/ice.rst +++ b/Documentation/networking/devlink/ice.rst @@ -139,6 +139,42 @@ EMP firmware image. The driver does not currently support reloading the driver via ``DEVLINK_RELOAD_ACTION_DRIVER_REINIT``. +Port split +========== + +The ``ice`` driver supports port splitting only for port 0, as the FW has +a predefined set of available port split options for the whole device. + +A system reboot is required for port split to be applied. + +The following command will select the port split option with 4 ports: + +.. code:: shell + + $ devlink port split pci/0000:16:00.0/0 count 4 + +The list of all available port options will be printed to dynamic debug after +each ``split`` and ``unsplit`` command. The first option is the default. + +.. code:: shell + + ice 0000:16:00.0: Available port split options and max port speeds (Gbps): + ice 0000:16:00.0: Status Split Quad 0 Quad 1 + ice 0000:16:00.0: count L0 L1 L2 L3 L4 L5 L6 L7 + ice 0000:16:00.0: Active 2 100 - - - 100 - - - + ice 0000:16:00.0: 2 50 - 50 - - - - - + ice 0000:16:00.0: Pending 4 25 25 25 25 - - - - + ice 0000:16:00.0: 4 25 25 - - 25 25 - - + ice 0000:16:00.0: 8 10 10 10 10 10 10 10 10 + ice 0000:16:00.0: 1 100 - - - - - - - + +There could be multiple FW port options with the same port split count. When +the same port split count request is issued again, the next FW port option with +the same port split count will be selected. + +``devlink port unsplit`` will select the option with a split count of 1. If +there is no FW option available with split count 1, you will receive an error. + Regions ======= diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst index e3a5f985673efdc5fe83f25a55615a35560cd04d..4b653d0406276d966b6949aa6050e3318991d363 100644 --- a/Documentation/networking/devlink/index.rst +++ b/Documentation/networking/devlink/index.rst @@ -13,10 +13,8 @@ new APIs prefixed by ``devl_*``. The older APIs handle all the locking in devlink core, but don't allow registration of most sub-objects once the main devlink object is itself registered. The newer ``devl_*`` APIs assume the devlink instance lock is already held. Drivers can take the instance -lock by calling ``devl_lock()``. It is also held in most of the callbacks. -Eventually all callbacks will be invoked under the devlink instance lock, -refer to the use of the ``DEVLINK_NL_FLAG_NO_LOCK`` flag in devlink core -to find out which callbacks are not converted, yet. +lock by calling ``devl_lock()``. It is also held all callbacks of devlink +netlink commands. Drivers are encouraged to use the devlink instance lock for their own needs. diff --git a/Documentation/networking/devlink/netdevsim.rst b/Documentation/networking/devlink/netdevsim.rst index 8a292fb5aaea319738acc76462a5559f3ba8c72a..ec5e6d79b2e2438b029a51b920272b234e7961c5 100644 --- a/Documentation/networking/devlink/netdevsim.rst +++ b/Documentation/networking/devlink/netdevsim.rst @@ -67,7 +67,7 @@ The ``netdevsim`` driver supports rate objects management, which includes: - setting tx_share and tx_max rate values for any rate object type; - setting parent node for any rate object type. -Rate nodes and it's parameters are exposed in ``netdevsim`` debugfs in RO mode. +Rate nodes and their parameters are exposed in ``netdevsim`` debugfs in RO mode. For example created rate node with name ``some_group``: .. code:: shell diff --git a/Documentation/networking/driver.rst b/Documentation/networking/driver.rst index c8f59dbda46f77df6378f1afc7128b0deadcaf49..64f7236ff10be49833af6eeceeceff1357a83555 100644 --- a/Documentation/networking/driver.rst +++ b/Documentation/networking/driver.rst @@ -8,7 +8,7 @@ Transmit path guidelines: 1) The ndo_start_xmit method must not return NETDEV_TX_BUSY under any normal circumstances. It is considered a hard error unless - there is no way your device can tell ahead of time when it's + there is no way your device can tell ahead of time when its transmit function will become busy. Instead it must maintain the queue properly. For example, diff --git a/Documentation/networking/dsa/configuration.rst b/Documentation/networking/dsa/configuration.rst index 2b08f1a772d3e6ea502162f19d3fcfcbe81013ba..827701f8cbfe6fed317e96130593c9027dde7c89 100644 --- a/Documentation/networking/dsa/configuration.rst +++ b/Documentation/networking/dsa/configuration.rst @@ -49,6 +49,9 @@ In this documentation the following Ethernet interfaces are used: *eth0* the master interface +*eth1* + another master interface + *lan1* a slave interface @@ -360,3 +363,96 @@ the ``self`` flag) has been removed. This results in the following changes: Script writers are therefore encouraged to use the ``master static`` set of flags when working with bridge FDB entries on DSA switch interfaces. + +Affinity of user ports to CPU ports +----------------------------------- + +Typically, DSA switches are attached to the host via a single Ethernet +interface, but in cases where the switch chip is discrete, the hardware design +may permit the use of 2 or more ports connected to the host, for an increase in +termination throughput. + +DSA can make use of multiple CPU ports in two ways. First, it is possible to +statically assign the termination traffic associated with a certain user port +to be processed by a certain CPU port. This way, user space can implement +custom policies of static load balancing between user ports, by spreading the +affinities according to the available CPU ports. + +Secondly, it is possible to perform load balancing between CPU ports on a per +packet basis, rather than statically assigning user ports to CPU ports. +This can be achieved by placing the DSA masters under a LAG interface (bonding +or team). DSA monitors this operation and creates a mirror of this software LAG +on the CPU ports facing the physical DSA masters that constitute the LAG slave +devices. + +To make use of multiple CPU ports, the firmware (device tree) description of +the switch must mark all the links between CPU ports and their DSA masters +using the ``ethernet`` reference/phandle. At startup, only a single CPU port +and DSA master will be used - the numerically first port from the firmware +description which has an ``ethernet`` property. It is up to the user to +configure the system for the switch to use other masters. + +DSA uses the ``rtnl_link_ops`` mechanism (with a "dsa" ``kind``) to allow +changing the DSA master of a user port. The ``IFLA_DSA_MASTER`` u32 netlink +attribute contains the ifindex of the master device that handles each slave +device. The DSA master must be a valid candidate based on firmware node +information, or a LAG interface which contains only slaves which are valid +candidates. + +Using iproute2, the following manipulations are possible: + + .. code-block:: sh + + # See the DSA master in current use + ip -d link show dev swp0 + (...) + dsa master eth0 + + # Static CPU port distribution + ip link set swp0 type dsa master eth1 + ip link set swp1 type dsa master eth0 + ip link set swp2 type dsa master eth1 + ip link set swp3 type dsa master eth0 + + # CPU ports in LAG, using explicit assignment of the DSA master + ip link add bond0 type bond mode balance-xor && ip link set bond0 up + ip link set eth1 down && ip link set eth1 master bond0 + ip link set swp0 type dsa master bond0 + ip link set swp1 type dsa master bond0 + ip link set swp2 type dsa master bond0 + ip link set swp3 type dsa master bond0 + ip link set eth0 down && ip link set eth0 master bond0 + ip -d link show dev swp0 + (...) + dsa master bond0 + + # CPU ports in LAG, relying on implicit migration of the DSA master + ip link add bond0 type bond mode balance-xor && ip link set bond0 up + ip link set eth0 down && ip link set eth0 master bond0 + ip link set eth1 down && ip link set eth1 master bond0 + ip -d link show dev swp0 + (...) + dsa master bond0 + +Notice that in the case of CPU ports under a LAG, the use of the +``IFLA_DSA_MASTER`` netlink attribute is not strictly needed, but rather, DSA +reacts to the ``IFLA_MASTER`` attribute change of its present master (``eth0``) +and migrates all user ports to the new upper of ``eth0``, ``bond0``. Similarly, +when ``bond0`` is destroyed using ``RTM_DELLINK``, DSA migrates the user ports +that were assigned to this interface to the first physical DSA master which is +eligible, based on the firmware description (it effectively reverts to the +startup configuration). + +In a setup with more than 2 physical CPU ports, it is therefore possible to mix +static user to CPU port assignment with LAG between DSA masters. It is not +possible to statically assign a user port towards a DSA master that has any +upper interfaces (this includes LAG devices - the master must always be the LAG +in this case). + +Live changing of the DSA master (and thus CPU port) affinity of a user port is +permitted, in order to allow dynamic redistribution in response to traffic. + +Physical DSA masters are allowed to join and leave at any time a LAG interface +used as a DSA master; however, DSA will reject a LAG interface as a valid +candidate for being a DSA master unless it has at least one physical DSA master +as a slave device. diff --git a/Documentation/networking/dsa/dsa.rst b/Documentation/networking/dsa/dsa.rst index d742ba6bd21195de7ba1e04aab6cc915f8f333cc..a94ddf83348aa0c8c48532e27291fdb9870c051b 100644 --- a/Documentation/networking/dsa/dsa.rst +++ b/Documentation/networking/dsa/dsa.rst @@ -303,6 +303,20 @@ These frames are then queued for transmission using the master network device Ethernet switch will be able to process these incoming frames from the management interface and deliver them to the physical switch port. +When using multiple CPU ports, it is possible to stack a LAG (bonding/team) +device between the DSA slave devices and the physical DSA masters. The LAG +device is thus also a DSA master, but the LAG slave devices continue to be DSA +masters as well (just with no user port assigned to them; this is needed for +recovery in case the LAG DSA master disappears). Thus, the data path of the LAG +DSA master is used asymmetrically. On RX, the ``ETH_P_XDSA`` handler, which +calls ``dsa_switch_rcv()``, is invoked early (on the physical DSA master; +LAG slave). Therefore, the RX data path of the LAG DSA master is not used. +On the other hand, TX takes place linearly: ``dsa_slave_xmit`` calls +``dsa_enqueue_skb``, which calls ``dev_queue_xmit`` towards the LAG DSA master. +The latter calls ``dev_queue_xmit`` towards one physical DSA master or the +other, and in both cases, the packet exits the system through a hardware path +towards the switch. + Graphical representation ------------------------ @@ -629,6 +643,24 @@ Switch configuration PHY cannot be found. In this case, probing of the DSA switch continues without that particular port. +- ``port_change_master``: method through which the affinity (association used + for traffic termination purposes) between a user port and a CPU port can be + changed. By default all user ports from a tree are assigned to the first + available CPU port that makes sense for them (most of the times this means + the user ports of a tree are all assigned to the same CPU port, except for H + topologies as described in commit 2c0b03258b8b). The ``port`` argument + represents the index of the user port, and the ``master`` argument represents + the new DSA master ``net_device``. The CPU port associated with the new + master can be retrieved by looking at ``struct dsa_port *cpu_dp = + master->dsa_ptr``. Additionally, the master can also be a LAG device where + all the slave devices are physical DSA masters. LAG DSA masters also have a + valid ``master->dsa_ptr`` pointer, however this is not unique, but rather a + duplicate of the first physical DSA master's (LAG slave) ``dsa_ptr``. In case + of a LAG DSA master, a further call to ``port_lag_join`` will be emitted + separately for the physical CPU ports associated with the physical DSA + masters, requesting them to create a hardware LAG associated with the LAG + interface. + PHY devices and link management ------------------------------- @@ -1095,9 +1127,3 @@ capable hardware, but does not enforce a strict switch device driver model. On the other DSA enforces a fairly strict device driver model, and deals with most of the switch specific. At some point we should envision a merger between these two subsystems and get the best of both worlds. - -Other hanging fruits --------------------- - -- allowing more than one CPU/management interface: - http://comments.gmane.org/gmane.linux.network/365657 diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index dbca3e9ec782f677e3699c888ec51877d5bbff33..d578b8bcd8a40a3577f1f3b3713157d68752f4c4 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -220,6 +220,8 @@ Userspace to kernel: ``ETHTOOL_MSG_PHC_VCLOCKS_GET`` get PHC virtual clocks info ``ETHTOOL_MSG_MODULE_SET`` set transceiver module parameters ``ETHTOOL_MSG_MODULE_GET`` get transceiver module parameters + ``ETHTOOL_MSG_PSE_SET`` set PSE parameters + ``ETHTOOL_MSG_PSE_GET`` get PSE parameters ===================================== ================================= Kernel to userspace: @@ -260,6 +262,7 @@ Kernel to userspace: ``ETHTOOL_MSG_STATS_GET_REPLY`` standard statistics ``ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY`` PHC virtual clocks info ``ETHTOOL_MSG_MODULE_GET_REPLY`` transceiver module parameters + ``ETHTOOL_MSG_PSE_GET_REPLY`` PSE parameters ======================================== ================================= ``GET`` requests are sent by userspace applications to retrieve device @@ -426,6 +429,7 @@ Kernel response contents: ``ETHTOOL_A_LINKMODES_DUPLEX`` u8 duplex mode ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG`` u8 Master/slave port mode ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE`` u8 Master/slave port state + ``ETHTOOL_A_LINKMODES_RATE_MATCHING`` u8 PHY rate matching ========================================== ====== ========================== For ``ETHTOOL_A_LINKMODES_OURS``, value represents advertised modes and mask @@ -449,6 +453,7 @@ Request contents: ``ETHTOOL_A_LINKMODES_SPEED`` u32 link speed (Mb/s) ``ETHTOOL_A_LINKMODES_DUPLEX`` u8 duplex mode ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG`` u8 Master/slave port mode + ``ETHTOOL_A_LINKMODES_RATE_MATCHING`` u8 PHY rate matching ``ETHTOOL_A_LINKMODES_LANES`` u32 lanes ========================================== ====== ========================== @@ -1625,6 +1630,62 @@ For SFF-8636 modules, low power mode is forced by the host according to table For CMIS modules, low power mode is forced by the host according to table 6-12 in revision 5.0 of the specification. +PSE_GET +======= + +Gets PSE attributes. + +Request contents: + + ===================================== ====== ========================== + ``ETHTOOL_A_PSE_HEADER`` nested request header + ===================================== ====== ========================== + +Kernel response contents: + + ====================================== ====== ============================= + ``ETHTOOL_A_PSE_HEADER`` nested reply header + ``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` u32 Operational state of the PoDL + PSE functions + ``ETHTOOL_A_PODL_PSE_PW_D_STATUS`` u32 power detection status of the + PoDL PSE. + ====================================== ====== ============================= + +When set, the optional ``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` attribute identifies +the operational state of the PoDL PSE functions. The operational state of the +PSE function can be changed using the ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL`` +action. This option is corresponding to ``IEEE 802.3-2018`` 30.15.1.1.2 +aPoDLPSEAdminState. Possible values are: + +.. kernel-doc:: include/uapi/linux/ethtool.h + :identifiers: ethtool_podl_pse_admin_state + +When set, the optional ``ETHTOOL_A_PODL_PSE_PW_D_STATUS`` attribute identifies +the power detection status of the PoDL PSE. The status depend on internal PSE +state machine and automatic PD classification support. This option is +corresponding to ``IEEE 802.3-2018`` 30.15.1.1.3 aPoDLPSEPowerDetectionStatus. +Possible values are: + +.. kernel-doc:: include/uapi/linux/ethtool.h + :identifiers: ethtool_podl_pse_pw_d_status + +PSE_SET +======= + +Sets PSE parameters. + +Request contents: + + ====================================== ====== ============================= + ``ETHTOOL_A_PSE_HEADER`` nested request header + ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL`` u32 Control PoDL PSE Admin state + ====================================== ====== ============================= + +When set, the optional ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL`` attribute is used +to control PoDL PSE Admin functions. This option is implementing +``IEEE 802.3-2018`` 30.15.1.2.1 acPoDLPSEAdminControl. See +``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` for supported values. + Request translation =================== diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst index 43cdc4d34745c8d6f963c88db8e0062d8f88767e..f69da50748609e323abdc71e351ec7216571d534 100644 --- a/Documentation/networking/filter.rst +++ b/Documentation/networking/filter.rst @@ -305,7 +305,7 @@ Possible BPF extensions are shown in the following table: vlan_tci skb_vlan_tag_get(skb) vlan_avail skb_vlan_tag_present(skb) vlan_tpid skb->vlan_proto - rand prandom_u32() + rand get_random_u32() =================================== ================================================= These extensions can also be prefixed with '#'. diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst index 03b215bddde8da7264ec8e274adebf73ff932a70..16a153bcc5fe9dfdc0f6d56fddd90547b6a651fa 100644 --- a/Documentation/networking/index.rst +++ b/Documentation/networking/index.rst @@ -47,7 +47,6 @@ Contents: cdc_mbim dccp dctcp - decnet dns_resolver driver eql @@ -93,6 +92,7 @@ Contents: radiotap-headers rds regulatory + representors rxrpc sctp secid diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 56cd4ea059b2a33cde01d8db598b160762db1478..e7b3fa7bb3f73825e865b884332063355dc73ff9 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -1035,7 +1035,39 @@ tcp_limit_output_bytes - INTEGER tcp_challenge_ack_limit - INTEGER Limits number of Challenge ACK sent per second, as recommended in RFC 5961 (Improving TCP's Robustness to Blind In-Window Attacks) - Default: 1000 + Note that this per netns rate limit can allow some side channel + attacks and probably should not be enabled. + TCP stack implements per TCP socket limits anyway. + Default: INT_MAX (unlimited) + +tcp_ehash_entries - INTEGER + Show the number of hash buckets for TCP sockets in the current + networking namespace. + + A negative value means the networking namespace does not own its + hash buckets and shares the initial networking namespace's one. + +tcp_child_ehash_entries - INTEGER + Control the number of hash buckets for TCP sockets in the child + networking namespace, which must be set before clone() or unshare(). + + If the value is not 0, the kernel uses a value rounded up to 2^n + as the actual hash bucket size. 0 is a special value, meaning + the child networking namespace will share the initial networking + namespace's hash buckets. + + Note that the child will use the global one in case the kernel + fails to allocate enough memory. In addition, the global hash + buckets are spread over available NUMA nodes, but the allocation + of the child hash table depends on the current process's NUMA + policy, which could result in performance differences. + + Note also that the default value of tcp_max_tw_buckets and + tcp_max_syn_backlog depend on the hash bucket size. + + Possible values: 0, 2^n (n: 0 - 24 (16Mi)) + + Default: 0 UDP variables ============= diff --git a/Documentation/networking/ipvlan.rst b/Documentation/networking/ipvlan.rst index 694adcba36b06ccdc2b9d4af921c69595330171e..0000c1d383bc0b21e93021d5984918f5da74119f 100644 --- a/Documentation/networking/ipvlan.rst +++ b/Documentation/networking/ipvlan.rst @@ -11,7 +11,7 @@ Initial Release: ================ This is conceptually very similar to the macvlan driver with one major exception of using L3 for mux-ing /demux-ing among slaves. This property makes -the master device share the L2 with it's slave devices. I have developed this +the master device share the L2 with its slave devices. I have developed this driver in conjunction with network namespaces and not sure if there is use case outside of it. diff --git a/Documentation/networking/l2tp.rst b/Documentation/networking/l2tp.rst index 498b382d25a0b34826c42dd4ccfd4d1f12543baa..7f383e99dbada9bafe9283eb9e7c4377ed1f7442 100644 --- a/Documentation/networking/l2tp.rst +++ b/Documentation/networking/l2tp.rst @@ -530,7 +530,7 @@ its tunnel close actions. For L2TPIP sockets, the socket's close handler initiates the same tunnel close actions. All sessions are first closed. Each session drops its tunnel ref. When the tunnel ref reaches zero, the tunnel puts its socket ref. When the socket is -eventually destroyed, it's sk_destruct finally frees the L2TP tunnel +eventually destroyed, its sk_destruct finally frees the L2TP tunnel context. Sessions diff --git a/Documentation/networking/mptcp-sysctl.rst b/Documentation/networking/mptcp-sysctl.rst index e263dfcc4b402f45fc606f24c9b56b8b8c11f84f..213510698014eab523daba264ab68714ba69a09e 100644 --- a/Documentation/networking/mptcp-sysctl.rst +++ b/Documentation/networking/mptcp-sysctl.rst @@ -47,7 +47,6 @@ allow_join_initial_addr_port - BOOLEAN Default: 1 pm_type - INTEGER - Set the default path manager type to use for each new MPTCP socket. In-kernel path management will control subflow connections and address advertisements according to diff --git a/Documentation/networking/nf_conntrack-sysctl.rst b/Documentation/networking/nf_conntrack-sysctl.rst index 834945ebc4cdf3ce3fffeb477d2d9b1047b7a7a4..1120d71f28d7bca75e6a540c03c539175c3d6cc2 100644 --- a/Documentation/networking/nf_conntrack-sysctl.rst +++ b/Documentation/networking/nf_conntrack-sysctl.rst @@ -70,15 +70,6 @@ nf_conntrack_generic_timeout - INTEGER (seconds) Default for generic timeout. This refers to layer 4 unknown/unsupported protocols. -nf_conntrack_helper - BOOLEAN - - 0 - disabled (default) - - not 0 - enabled - - Enable automatic conntrack helper assignment. - If disabled it is required to set up iptables rules to assign - helpers to connections. See the CT target description in the - iptables-extensions(8) man page for further information. - nf_conntrack_icmp_timeout - INTEGER (seconds) default 30 diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index 704f31da51672e3ae4c057e8aeab6f225206f3a2..d11329a08984e024c13cd8250ec0bb73e4c72d81 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -120,7 +120,7 @@ required delays, as defined per the RGMII standard, several options may be available: * Some SoCs may offer a pin pad/mux/controller capable of configuring a given - set of pins'strength, delays, and voltage; and it may be a suitable + set of pins' strength, delays, and voltage; and it may be a suitable option to insert the expected 2ns RGMII delay. * Modifying the PCB design to include a fixed delay (e.g: using a specifically @@ -308,6 +308,21 @@ Some of the interface modes are described below: rate of 125Mpbs using a 4B/5B encoding scheme, resulting in an underlying data rate of 100Mpbs. +``PHY_INTERFACE_MODE_QUSGMII`` + This defines the Cisco the Quad USGMII mode, which is the Quad variant of + the USGMII (Universal SGMII) link. It's very similar to QSGMII, but uses + a Packet Control Header (PCH) instead of the 7 bytes preamble to carry not + only the port id, but also so-called "extensions". The only documented + extension so-far in the specification is the inclusion of timestamps, for + PTP-enabled PHYs. This mode isn't compatible with QSGMII, but offers the + same capabilities in terms of link speed and negociation. + +``PHY_INTERFACE_MODE_1000BASEKX`` + This is 1000BASE-X as defined by IEEE 802.3 Clause 36 with Clause 73 + autonegotiation. Generally, it will be used with a Clause 70 PMD. To + contrast with the 1000BASE-X phy mode used for Clause 38 and 39 PMDs, this + interface mode has different autonegotiation and only supports full duplex. + Pause frames / flow control =========================== diff --git a/Documentation/networking/representors.rst b/Documentation/networking/representors.rst new file mode 100644 index 0000000000000000000000000000000000000000..ee1f5cd54496e93f933133a10bd01c766aee9ca6 --- /dev/null +++ b/Documentation/networking/representors.rst @@ -0,0 +1,259 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================= +Network Function Representors +============================= + +This document describes the semantics and usage of representor netdevices, as +used to control internal switching on SmartNICs. For the closely-related port +representors on physical (multi-port) switches, see +:ref:`Documentation/networking/switchdev.rst `. + +Motivation +---------- + +Since the mid-2010s, network cards have started offering more complex +virtualisation capabilities than the legacy SR-IOV approach (with its simple +MAC/VLAN-based switching model) can support. This led to a desire to offload +software-defined networks (such as OpenVSwitch) to these NICs to specify the +network connectivity of each function. The resulting designs are variously +called SmartNICs or DPUs. + +Network function representors bring the standard Linux networking stack to +virtual switches and IOV devices. Just as each physical port of a Linux- +controlled switch has a separate netdev, so does each virtual port of a virtual +switch. +When the system boots, and before any offload is configured, all packets from +the virtual functions appear in the networking stack of the PF via the +representors. The PF can thus always communicate freely with the virtual +functions. +The PF can configure standard Linux forwarding between representors, the uplink +or any other netdev (routing, bridging, TC classifiers). + +Thus, a representor is both a control plane object (representing the function in +administrative commands) and a data plane object (one end of a virtual pipe). +As a virtual link endpoint, the representor can be configured like any other +netdevice; in some cases (e.g. link state) the representee will follow the +representor's configuration, while in others there are separate APIs to +configure the representee. + +Definitions +----------- + +This document uses the term "switchdev function" to refer to the PCIe function +which has administrative control over the virtual switch on the device. +Typically, this will be a PF, but conceivably a NIC could be configured to grant +these administrative privileges instead to a VF or SF (subfunction). +Depending on NIC design, a multi-port NIC might have a single switchdev function +for the whole device or might have a separate virtual switch, and hence +switchdev function, for each physical network port. +If the NIC supports nested switching, there might be separate switchdev +functions for each nested switch, in which case each switchdev function should +only create representors for the ports on the (sub-)switch it directly +administers. + +A "representee" is the object that a representor represents. So for example in +the case of a VF representor, the representee is the corresponding VF. + +What does a representor do? +--------------------------- + +A representor has three main roles. + +1. It is used to configure the network connection the representee sees, e.g. + link up/down, MTU, etc. For instance, bringing the representor + administratively UP should cause the representee to see a link up / carrier + on event. +2. It provides the slow path for traffic which does not hit any offloaded + fast-path rules in the virtual switch. Packets transmitted on the + representor netdevice should be delivered to the representee; packets + transmitted by the representee which fail to match any switching rule should + be received on the representor netdevice. (That is, there is a virtual pipe + connecting the representor to the representee, similar in concept to a veth + pair.) + This allows software switch implementations (such as OpenVSwitch or a Linux + bridge) to forward packets between representees and the rest of the network. +3. It acts as a handle by which switching rules (such as TC filters) can refer + to the representee, allowing these rules to be offloaded. + +The combination of 2) and 3) means that the behaviour (apart from performance) +should be the same whether a TC filter is offloaded or not. E.g. a TC rule +on a VF representor applies in software to packets received on that representor +netdevice, while in hardware offload it would apply to packets transmitted by +the representee VF. Conversely, a mirred egress redirect to a VF representor +corresponds in hardware to delivery directly to the representee VF. + +What functions should have a representor? +----------------------------------------- + +Essentially, for each virtual port on the device's internal switch, there +should be a representor. +Some vendors have chosen to omit representors for the uplink and the physical +network port, which can simplify usage (the uplink netdev becomes in effect the +physical port's representor) but does not generalise to devices with multiple +ports or uplinks. + +Thus, the following should all have representors: + + - VFs belonging to the switchdev function. + - Other PFs on the local PCIe controller, and any VFs belonging to them. + - PFs and VFs on external PCIe controllers on the device (e.g. for any embedded + System-on-Chip within the SmartNIC). + - PFs and VFs with other personalities, including network block devices (such + as a vDPA virtio-blk PF backed by remote/distributed storage), if (and only + if) their network access is implemented through a virtual switch port. [#]_ + Note that such functions can require a representor despite the representee + not having a netdev. + - Subfunctions (SFs) belonging to any of the above PFs or VFs, if they have + their own port on the switch (as opposed to using their parent PF's port). + - Any accelerators or plugins on the device whose interface to the network is + through a virtual switch port, even if they do not have a corresponding PCIe + PF or VF. + +This allows the entire switching behaviour of the NIC to be controlled through +representor TC rules. + +It is a common misunderstanding to conflate virtual ports with PCIe virtual +functions or their netdevs. While in simple cases there will be a 1:1 +correspondence between VF netdevices and VF representors, more advanced device +configurations may not follow this. +A PCIe function which does not have network access through the internal switch +(not even indirectly through the hardware implementation of whatever services +the function provides) should *not* have a representor (even if it has a +netdev). +Such a function has no switch virtual port for the representor to configure or +to be the other end of the virtual pipe. +The representor represents the virtual port, not the PCIe function nor the 'end +user' netdevice. + +.. [#] The concept here is that a hardware IP stack in the device performs the + translation between block DMA requests and network packets, so that only + network packets pass through the virtual port onto the switch. The network + access that the IP stack "sees" would then be configurable through tc rules; + e.g. its traffic might all be wrapped in a specific VLAN or VxLAN. However, + any needed configuration of the block device *qua* block device, not being a + networking entity, would not be appropriate for the representor and would + thus use some other channel such as devlink. + Contrast this with the case of a virtio-blk implementation which forwards the + DMA requests unchanged to another PF whose driver then initiates and + terminates IP traffic in software; in that case the DMA traffic would *not* + run over the virtual switch and the virtio-blk PF should thus *not* have a + representor. + +How are representors created? +----------------------------- + +The driver instance attached to the switchdev function should, for each virtual +port on the switch, create a pure-software netdevice which has some form of +in-kernel reference to the switchdev function's own netdevice or driver private +data (``netdev_priv()``). +This may be by enumerating ports at probe time, reacting dynamically to the +creation and destruction of ports at run time, or a combination of the two. + +The operations of the representor netdevice will generally involve acting +through the switchdev function. For example, ``ndo_start_xmit()`` might send +the packet through a hardware TX queue attached to the switchdev function, with +either packet metadata or queue configuration marking it for delivery to the +representee. + +How are representors identified? +-------------------------------- + +The representor netdevice should *not* directly refer to a PCIe device (e.g. +through ``net_dev->dev.parent`` / ``SET_NETDEV_DEV()``), either of the +representee or of the switchdev function. +Instead, it should implement the ``ndo_get_devlink_port()`` netdevice op, which +the kernel uses to provide the ``phys_switch_id`` and ``phys_port_name`` sysfs +nodes. (Some legacy drivers implement ``ndo_get_port_parent_id()`` and +``ndo_get_phys_port_name()`` directly, but this is deprecated.) See +:ref:`Documentation/networking/devlink/devlink-port.rst ` for the +details of this API. + +It is expected that userland will use this information (e.g. through udev rules) +to construct an appropriately informative name or alias for the netdevice. For +instance if the switchdev function is ``eth4`` then a representor with a +``phys_port_name`` of ``p0pf1vf2`` might be renamed ``eth4pf1vf2rep``. + +There are as yet no established conventions for naming representors which do not +correspond to PCIe functions (e.g. accelerators and plugins). + +How do representors interact with TC rules? +------------------------------------------- + +Any TC rule on a representor applies (in software TC) to packets received by +that representor netdevice. Thus, if the delivery part of the rule corresponds +to another port on the virtual switch, the driver may choose to offload it to +hardware, applying it to packets transmitted by the representee. + +Similarly, since a TC mirred egress action targeting the representor would (in +software) send the packet through the representor (and thus indirectly deliver +it to the representee), hardware offload should interpret this as delivery to +the representee. + +As a simple example, if ``PORT_DEV`` is the physical port representor and +``REP_DEV`` is a VF representor, the following rules:: + + tc filter add dev $REP_DEV parent ffff: protocol ipv4 flower \ + action mirred egress redirect dev $PORT_DEV + tc filter add dev $PORT_DEV parent ffff: protocol ipv4 flower skip_sw \ + action mirred egress mirror dev $REP_DEV + +would mean that all IPv4 packets from the VF are sent out the physical port, and +all IPv4 packets received on the physical port are delivered to the VF in +addition to ``PORT_DEV``. (Note that without ``skip_sw`` on the second rule, +the VF would get two copies, as the packet reception on ``PORT_DEV`` would +trigger the TC rule again and mirror the packet to ``REP_DEV``.) + +On devices without separate port and uplink representors, ``PORT_DEV`` would +instead be the switchdev function's own uplink netdevice. + +Of course the rules can (if supported by the NIC) include packet-modifying +actions (e.g. VLAN push/pop), which should be performed by the virtual switch. + +Tunnel encapsulation and decapsulation are rather more complicated, as they +involve a third netdevice (a tunnel netdev operating in metadata mode, such as +a VxLAN device created with ``ip link add vxlan0 type vxlan external``) and +require an IP address to be bound to the underlay device (e.g. switchdev +function uplink netdev or port representor). TC rules such as:: + + tc filter add dev $REP_DEV parent ffff: flower \ + action tunnel_key set id $VNI src_ip $LOCAL_IP dst_ip $REMOTE_IP \ + dst_port 4789 \ + action mirred egress redirect dev vxlan0 + tc filter add dev vxlan0 parent ffff: flower enc_src_ip $REMOTE_IP \ + enc_dst_ip $LOCAL_IP enc_key_id $VNI enc_dst_port 4789 \ + action tunnel_key unset action mirred egress redirect dev $REP_DEV + +where ``LOCAL_IP`` is an IP address bound to ``PORT_DEV``, and ``REMOTE_IP`` is +another IP address on the same subnet, mean that packets sent by the VF should +be VxLAN encapsulated and sent out the physical port (the driver has to deduce +this by a route lookup of ``LOCAL_IP`` leading to ``PORT_DEV``, and also +perform an ARP/neighbour table lookup to find the MAC addresses to use in the +outer Ethernet frame), while UDP packets received on the physical port with UDP +port 4789 should be parsed as VxLAN and, if their VSID matches ``$VNI``, +decapsulated and forwarded to the VF. + +If this all seems complicated, just remember the 'golden rule' of TC offload: +the hardware should ensure the same final results as if the packets were +processed through the slow path, traversed software TC (except ignoring any +``skip_hw`` rules and applying any ``skip_sw`` rules) and were transmitted or +received through the representor netdevices. + +Configuring the representee's MAC +--------------------------------- + +The representee's link state is controlled through the representor. Setting the +representor administratively UP or DOWN should cause carrier ON or OFF at the +representee. + +Setting an MTU on the representor should cause that same MTU to be reported to +the representee. +(On hardware that allows configuring separate and distinct MTU and MRU values, +the representor MTU should correspond to the representee's MRU and vice-versa.) + +Currently there is no way to use the representor to set the station permanent +MAC address of the representee; other methods available to do this include: + + - legacy SR-IOV (``ip link set DEVICE vf NUM mac LLADDR``) + - devlink port function (see **devlink-port(8)** and + :ref:`Documentation/networking/devlink/devlink-port.rst `) diff --git a/Documentation/networking/rxrpc.rst b/Documentation/networking/rxrpc.rst index 39c2249c7aa78d2fbda659d2f49676e60ede034b..39494a6ea739cf02217f088721ff23e1751521fe 100644 --- a/Documentation/networking/rxrpc.rst +++ b/Documentation/networking/rxrpc.rst @@ -1055,17 +1055,6 @@ The kernel interface functions are as follows: first function to change. Note that this must be called in TASK_RUNNING state. - (#) Get reply timestamp:: - - bool rxrpc_kernel_get_reply_time(struct socket *sock, - struct rxrpc_call *call, - ktime_t *_ts) - - This allows the timestamp on the first DATA packet of the reply of a - client call to be queried, provided that it is still in the Rx ring. If - successful, the timestamp will be stored into ``*_ts`` and true will be - returned; false will be returned otherwise. - (#) Get remote client epoch:: u32 rxrpc_kernel_get_epoch(struct socket *sock, diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst index 742e90e6d822fcf90965c239392a82cd29268d70..6d8acdbe9be1de705efef5f11504abe5ad2929d2 100644 --- a/Documentation/networking/smc-sysctl.rst +++ b/Documentation/networking/smc-sysctl.rst @@ -34,3 +34,28 @@ smcr_buf_type - INTEGER - 1 - Use virtually contiguous buffers - 2 - Mixed use of the two types. Try physically contiguous buffers first. If not available, use virtually contiguous buffers then. + +smcr_testlink_time - INTEGER + How frequently SMC-R link sends out TEST_LINK LLC messages to confirm + viability, after the last activity of connections on it. Value 0 means + disabling TEST_LINK. + + Default: 30 seconds. + +wmem - INTEGER + Initial size of send buffer used by SMC sockets. + The default value inherits from net.ipv4.tcp_wmem[1]. + + The minimum value is 16KiB and there is no hard limit for max value, but + only allowed 512KiB for SMC-R and 1MiB for SMC-D. + + Default: 16K + +rmem - INTEGER + Initial size of receive buffer (RMB) used by SMC sockets. + The default value inherits from net.ipv4.tcp_rmem[1]. + + The minimum value is 16KiB and there is no hard limit for max value, but + only allowed 512KiB for SMC-R and 1MiB for SMC-D. + + Default: 128K diff --git a/Documentation/networking/switchdev.rst b/Documentation/networking/switchdev.rst index f1f4e6a85a2973b11eb1aaab5d54c28bf5366acc..758f1dae3fce2071b130db775d9d73aeca4af214 100644 --- a/Documentation/networking/switchdev.rst +++ b/Documentation/networking/switchdev.rst @@ -1,5 +1,6 @@ .. SPDX-License-Identifier: GPL-2.0 .. include:: +.. _switchdev: =============================================== Ethernet switch device driver model (switchdev) @@ -159,7 +160,7 @@ tools such as iproute2. The switchdev driver can know a particular port's position in the topology by monitoring NETDEV_CHANGEUPPER notifications. For example, a port moved into a -bond will see it's upper master change. If that bond is moved into a bridge, +bond will see its upper master change. If that bond is moved into a bridge, the bond's upper master will change. And so on. The driver will track such movements to know what position a port is in in the overall topology by registering for netdevice events and acting on NETDEV_CHANGEUPPER. diff --git a/Documentation/powerpc/isa-versions.rst b/Documentation/powerpc/isa-versions.rst index dfcb1097dce47cd970c83a705efed6979c741eeb..a8d6b6028b3ec7cf22584290728303d03807bc1e 100644 --- a/Documentation/powerpc/isa-versions.rst +++ b/Documentation/powerpc/isa-versions.rst @@ -4,12 +4,16 @@ CPU to ISA Version Mapping Mapping of some CPU versions to relevant ISA versions. +Note Power4 and Power4+ are not supported. + ========= ==================================================================== CPU Architecture version ========= ==================================================================== Power10 Power ISA v3.1 Power9 Power ISA v3.0B Power8 Power ISA v2.07 +e6500 Power ISA v2.06 with some exceptions +e5500 Power ISA v2.06 with some exceptions, no Altivec Power7 Power ISA v2.06 Power6 Power ISA v2.05 PA6T Power ISA v2.04 @@ -24,6 +28,12 @@ PPC970 - PowerPC User Instruction Set Architecture Book I v2.01 - PowerPC Virtual Environment Architecture Book II v2.01 - PowerPC Operating Environment Architecture Book III v2.01 - Plus Altivec/VMX ~= 2.03 +Power4+ - PowerPC User Instruction Set Architecture Book I v2.01 + - PowerPC Virtual Environment Architecture Book II v2.01 + - PowerPC Operating Environment Architecture Book III v2.01 +Power4 - PowerPC User Instruction Set Architecture Book I v2.00 + - PowerPC Virtual Environment Architecture Book II v2.00 + - PowerPC Operating Environment Architecture Book III v2.00 ========= ==================================================================== @@ -36,6 +46,8 @@ CPU VMX (aka. Altivec) Power10 Yes Power9 Yes Power8 Yes +e6500 Yes +e5500 No Power7 Yes Power6 Yes PA6T Yes @@ -44,6 +56,8 @@ Power5++ No Power5+ No Power5 No PPC970 Yes +Power4+ No +Power4 No ========== ================== ========== ==== @@ -52,6 +66,8 @@ CPU VSX Power10 Yes Power9 Yes Power8 Yes +e6500 No +e5500 No Power7 Yes Power6 No PA6T No @@ -60,6 +76,8 @@ Power5++ No Power5+ No Power5 No PPC970 No +Power4+ No +Power4 No ========== ==== ========== ==================================== @@ -68,6 +86,8 @@ CPU Transactional Memory Power10 No (* see Power ISA v3.1, "Appendix A. Notes on the Removal of Transactional Memory from the Architecture") Power9 Yes (* see transactional_memory.txt) Power8 Yes +e6500 No +e5500 No Power7 No Power6 No PA6T No @@ -76,4 +96,6 @@ Power5++ No Power5+ No Power5 No PPC970 No +Power4+ No +Power4 No ========== ==================================== diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst index 906235c11c2483eaa6385481bc70868328c40382..d87f1fee4cbc5f706ba9ef3b37e0c0427fbcb807 100644 --- a/Documentation/process/5.Posting.rst +++ b/Documentation/process/5.Posting.rst @@ -256,8 +256,10 @@ The tags in common use are: - Cc: the named person received a copy of the patch and had the opportunity to comment on it. -Be careful in the addition of tags to your patches: only Cc: is appropriate -for addition without the explicit permission of the person named. +Be careful in the addition of tags to your patches, as only Cc: is appropriate +for addition without the explicit permission of the person named; using +Reported-by: is fine most of the time as well, but ask for permission if +the bug was reported in private. Sending the patch diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index 19c286c23786f17e73c5f2bf2790aa2c3da4dd4d..9844ca3a71a61689f171bc02fd10a12566ee5f0b 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -31,7 +31,9 @@ you probably needn't concern yourself with pcmciautils. ====================== =============== ======================================== GNU C 5.1 gcc --version Clang/LLVM (optional) 11.0.0 clang --version -GNU make 3.81 make --version +Rust (optional) 1.62.0 rustc --version +bindgen (optional) 0.56.0 bindgen --version +GNU make 3.82 make --version bash 4.2 bash --version binutils 2.23 ld -v flex 2.5.35 flex --version @@ -80,10 +82,33 @@ kernels. Older releases aren't guaranteed to work, and we may drop workarounds from the kernel that were used to support older versions. Please see additional docs on :ref:`Building Linux with Clang/LLVM `. +Rust (optional) +--------------- + +A particular version of the Rust toolchain is required. Newer versions may or +may not work because the kernel depends on some unstable Rust features, for +the moment. + +Each Rust toolchain comes with several "components", some of which are required +(like ``rustc``) and some that are optional. The ``rust-src`` component (which +is optional) needs to be installed to build the kernel. Other components are +useful for developing. + +Please see Documentation/rust/quick-start.rst for instructions on how to +satisfy the build requirements of Rust support. In particular, the ``Makefile`` +target ``rustavailable`` is useful to check why the Rust toolchain may not +be detected. + +bindgen (optional) +------------------ + +``bindgen`` is used to generate the Rust bindings to the C side of the kernel. +It depends on ``libclang``. + Make ---- -You will need GNU make 3.81 or later to build the kernel. +You will need GNU make 3.82 or later to build the kernel. Bash ---- @@ -348,6 +373,12 @@ Sphinx Please see :ref:`sphinx_install` in :ref:`Documentation/doc-guide/sphinx.rst ` for details about Sphinx requirements. +rustdoc +------- + +``rustdoc`` is used to generate the documentation for Rust code. Please see +Documentation/rust/general-information.rst for more information. + Getting updated software ======================== @@ -364,6 +395,16 @@ Clang/LLVM - :ref:`Getting LLVM `. +Rust +---- + +- Documentation/rust/quick-start.rst. + +bindgen +------- + +- Documentation/rust/quick-start.rst. + Make ---- diff --git a/Documentation/process/code-of-conduct-interpretation.rst b/Documentation/process/code-of-conduct-interpretation.rst index e899f14a4ba24058e07d6411f5692d6db574c326..922e0b547bc39b81d7e7edcd6e7db89aa6e128ea 100644 --- a/Documentation/process/code-of-conduct-interpretation.rst +++ b/Documentation/process/code-of-conduct-interpretation.rst @@ -51,7 +51,7 @@ the Technical Advisory Board (TAB) or other maintainers if you're uncertain how to handle situations that come up. It will not be considered a violation report unless you want it to be. If you are uncertain about approaching the TAB or any other maintainers, please -reach out to our conflict mediator, Mishi Choudhary . +reach out to our conflict mediator, Joanna Lee . In the end, "be kind to each other" is really what the end goal is for everybody. We know everyone is human and we all fail at times, but the @@ -127,10 +127,12 @@ are listed at https://kernel.org/code-of-conduct.html. Members can not access reports made before they joined or after they have left the committee. -The initial Code of Conduct Committee consists of volunteer members of -the TAB, as well as a professional mediator acting as a neutral third -party. The first task of the committee is to establish documented -processes, which will be made public. +The Code of Conduct Committee consists of volunteer community members +appointed by the TAB, as well as a professional mediator acting as a +neutral third party. The processes the Code of Conduct committee will +use to address reports is varied and will depend on the individual +circumstance, however, this file serves as documentation for the +general process used. Any member of the committee, including the mediator, can be contacted directly if a reporter does not wish to include the full committee in a @@ -141,16 +143,16 @@ processes (see above) and consults with the TAB as needed and appropriate, for instance to request and receive information about the kernel community. -Any decisions by the committee will be brought to the TAB, for -implementation of enforcement with the relevant maintainers if needed. -A decision by the Code of Conduct Committee can be overturned by the TAB -by a two-thirds vote. +Any decisions regarding enforcement recommendations will be brought to +the TAB for implementation of enforcement with the relevant maintainers +if needed. A decision by the Code of Conduct Committee can be overturned +by the TAB by a two-thirds vote. At quarterly intervals, the Code of Conduct Committee and TAB will provide a report summarizing the anonymised reports that the Code of Conduct committee has received and their status, as well details of any overridden decisions including complete and identifiable voting details. -We expect to establish a different process for Code of Conduct Committee -staffing beyond the bootstrap period. This document will be updated -with that information when this occurs. +Because how we interpret and enforce the Code of Conduct will evolve over +time, this document will be updated when necessary to reflect any +changes. diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst index 03eb53fd029a8cdeb6a6ae1b697938ccebb2955d..007e49ef6cec773e4231f543e47341a573e5ed6f 100644 --- a/Documentation/process/coding-style.rst +++ b/Documentation/process/coding-style.rst @@ -1186,6 +1186,68 @@ expression used. For instance: #endif /* CONFIG_SOMETHING */ +22) Do not crash the kernel +--------------------------- + +In general, the decision to crash the kernel belongs to the user, rather +than to the kernel developer. + +Avoid panic() +************* + +panic() should be used with care and primarily only during system boot. +panic() is, for example, acceptable when running out of memory during boot and +not being able to continue. + +Use WARN() rather than BUG() +**************************** + +Do not add new code that uses any of the BUG() variants, such as BUG(), +BUG_ON(), or VM_BUG_ON(). Instead, use a WARN*() variant, preferably +WARN_ON_ONCE(), and possibly with recovery code. Recovery code is not +required if there is no reasonable way to at least partially recover. + +"I'm too lazy to do error handling" is not an excuse for using BUG(). Major +internal corruptions with no way of continuing may still use BUG(), but need +good justification. + +Use WARN_ON_ONCE() rather than WARN() or WARN_ON() +************************************************** + +WARN_ON_ONCE() is generally preferred over WARN() or WARN_ON(), because it +is common for a given warning condition, if it occurs at all, to occur +multiple times. This can fill up and wrap the kernel log, and can even slow +the system enough that the excessive logging turns into its own, additional +problem. + +Do not WARN lightly +******************* + +WARN*() is intended for unexpected, this-should-never-happen situations. +WARN*() macros are not to be used for anything that is expected to happen +during normal operation. These are not pre- or post-condition asserts, for +example. Again: WARN*() must not be used for a condition that is expected +to trigger easily, for example, by user space actions. pr_warn_once() is a +possible alternative, if you need to notify the user of a problem. + +Do not worry about panic_on_warn users +************************************** + +A few more words about panic_on_warn: Remember that ``panic_on_warn`` is an +available kernel option, and that many users set this option. This is why +there is a "Do not WARN lightly" writeup, above. However, the existence of +panic_on_warn users is not a valid reason to avoid the judicious use +WARN*(). That is because, whoever enables panic_on_warn has explicitly +asked the kernel to crash if a WARN*() fires, and such users must be +prepared to deal with the consequences of a system that is somewhat more +likely to crash. + +Use BUILD_BUG_ON() for compile-time assertions +********************************************** + +The use of BUILD_BUG_ON() is acceptable and encouraged, because it is a +compile-time assertion that has no effect at runtime. + Appendix I) References ---------------------- diff --git a/Documentation/process/deprecated.rst b/Documentation/process/deprecated.rst index a6e36d9c3d14bb5647db46986a2fe444106ac00c..c8fd53a11a207285577696e5b3d53a6710f5806e 100644 --- a/Documentation/process/deprecated.rst +++ b/Documentation/process/deprecated.rst @@ -138,17 +138,20 @@ be NUL terminated. This can lead to various linear read overflows and other misbehavior due to the missing termination. It also NUL-pads the destination buffer if the source contents are shorter than the destination buffer size, which may be a needless performance penalty -for callers using only NUL-terminated strings. The safe replacement is +for callers using only NUL-terminated strings. + +When the destination is required to be NUL-terminated, the replacement is strscpy(), though care must be given to any cases where the return value of strncpy() was used, since strscpy() does not return a pointer to the destination, but rather a count of non-NUL bytes copied (or negative errno when it truncates). Any cases still needing NUL-padding should instead use strscpy_pad(). -If a caller is using non-NUL-terminated strings, strncpy() can -still be used, but destinations should be marked with the `__nonstring +If a caller is using non-NUL-terminated strings, strtomem() should be +used, and the destinations should be marked with the `__nonstring `_ -attribute to avoid future compiler warnings. +attribute to avoid future compiler warnings. For cases still needing +NUL-padding, strtomem_pad() can be used. strlcpy() --------- diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst index cd6997a9d20322422557cafe5980a56de07da500..bd15c393ba3cdbdaa693f437092e6917c673005a 100644 --- a/Documentation/process/howto.rst +++ b/Documentation/process/howto.rst @@ -379,7 +379,7 @@ to subscribe and unsubscribe from the list can be found at: There are archives of the mailing list on the web in many different places. Use a search engine to find these archives. For example: - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ It is highly recommended that you search the archives about the topic you want to bring up, before you post it to the list. A lot of things diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst index 2ba2a1582bbeb939a08c083e7b4d1a09da5de4b9..d4b6217472b0a047a81c7e306fcaf2f54f5a2d46 100644 --- a/Documentation/process/index.rst +++ b/Documentation/process/index.rst @@ -5,6 +5,7 @@ .. _process_index: +============================================= Working with the kernel development community ============================================= diff --git a/Documentation/process/magic-number.rst b/Documentation/process/magic-number.rst index f5ba36e964617dd68cf120f3c831f9658f6add84..64b5948fc1d49d942edcf686f482f998a99c214c 100644 --- a/Documentation/process/magic-number.rst +++ b/Documentation/process/magic-number.rst @@ -69,86 +69,17 @@ Changelog:: Magic Name Number Structure File ===================== ================ ======================== ========================================== PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/pg.h`` -CMAGIC 0x0111 user ``include/linux/a.out.h`` -MKISS_DRIVER_MAGIC 0x04bf mkiss_channel ``drivers/net/mkiss.h`` -HDLC_MAGIC 0x239e n_hdlc ``drivers/char/n_hdlc.c`` APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` -DB_MAGIC 0x4442 fc_info ``drivers/net/iph5526_novram.c`` -DL_MAGIC 0x444d fc_info ``drivers/net/iph5526_novram.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` -FF_MAGIC 0x4646 fc_info ``drivers/net/iph5526_novram.c`` -PTY_MAGIC 0x5001 ``drivers/char/pty.c`` -PPP_MAGIC 0x5002 ppp ``include/linux/if_pppvar.h`` -SSTATE_MAGIC 0x5302 serial_state ``include/linux/serial.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -STRIP_MAGIC 0x5303 strip ``drivers/net/strip.c`` -SIXPACK_MAGIC 0x5304 sixpack ``drivers/net/hamradio/6pack.h`` -AX25_MAGIC 0x5316 ax_disp ``drivers/net/mkiss.h`` -TTY_MAGIC 0x5401 tty_struct ``include/linux/tty.h`` -MGSL_MAGIC 0x5401 mgsl_info ``drivers/char/synclink.c`` -TTY_DRIVER_MAGIC 0x5402 tty_driver ``include/linux/tty_driver.h`` MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` -USB_SERIAL_MAGIC 0x6702 usb_serial ``drivers/usb/serial/usb-serial.h`` -FULL_DUPLEX_MAGIC 0x6969 ``drivers/net/ethernet/dec/tulip/de2104x.c`` -USB_BLUETOOTH_MAGIC 0x6d02 usb_bluetooth ``drivers/usb/class/bluetty.c`` -RFCOMM_TTY_MAGIC 0x6d02 ``net/bluetooth/rfcomm/tty.c`` -USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/serial/usb-serial.h`` -CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` -LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` -NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` -RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` -ISDN_X25IFACE_MAGIC 0x1e75a2b9 isdn_x25iface_proto_data ``drivers/isdn/isdn_x25iface.h`` -ECP_MAGIC 0x21504345 cdkecpsig ``include/linux/cdk.h`` -LSOMAGIC 0x27091997 lso ``drivers/fc4/fc.c`` -LSMAGIC 0x2a3b4d2a ls ``drivers/fc4/fc.c`` -WANPIPE_MAGIC 0x414C4453 sdla_{dump,exec} ``include/linux/wanpipe.h`` -CS_CARD_MAGIC 0x43525553 cs_card ``sound/oss/cs46xx.c`` -LABELCL_MAGIC 0x4857434c labelcl_info_s ``include/asm/ia64/sn/labelcl.h`` -ISDN_ASYNC_MAGIC 0x49344C01 modem_info ``include/linux/isdn.h`` -CTC_ASYNC_MAGIC 0x49344C01 ctc_tty_info ``drivers/s390/net/ctctty.c`` -ISDN_NET_MAGIC 0x49344C02 isdn_net_local_s ``drivers/isdn/i4l/isdn_net_lib.h`` -SAVEKMSG_MAGIC2 0x4B4D5347 savekmsg ``arch/*/amiga/config.c`` -CS_STATE_MAGIC 0x4c4f4749 cs_state ``sound/oss/cs46xx.c`` -SLAB_C_MAGIC 0x4f17a36d kmem_cache ``mm/slab.c`` -COW_MAGIC 0x4f4f4f4d cow_header_v1 ``arch/um/drivers/ubd_user.c`` -I810_CARD_MAGIC 0x5072696E i810_card ``sound/oss/i810_audio.c`` -TRIDENT_CARD_MAGIC 0x5072696E trident_card ``sound/oss/trident.c`` -ROUTER_MAGIC 0x524d4157 wan_device [in ``wanrouter.h`` pre 3.9] -SAVEKMSG_MAGIC1 0x53415645 savekmsg ``arch/*/amiga/config.c`` -GDA_MAGIC 0x58464552 gda ``arch/mips/include/asm/sn/gda.h`` -RED_MAGIC1 0x5a2cf071 (any) ``mm/slab.c`` -EEPROM_MAGIC_VALUE 0x5ab478d2 lanai_dev ``drivers/atm/lanai.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` -PCXX_MAGIC 0x5c6df104 channel ``drivers/char/pcxx.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` -I810_STATE_MAGIC 0x63657373 i810_state ``sound/oss/i810_audio.c`` -TRIDENT_STATE_MAGIC 0x63657373 trient_state ``sound/oss/trident.c`` -M3_CARD_MAGIC 0x646e6f50 m3_card ``sound/oss/maestro3.c`` -FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fore200e.h`` -SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` -SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` -LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` -VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` -KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` -PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/media/pwc.h`` -NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` -ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` QUEUE_MAGIC_USED 0xf7e1cc33 queue_entry ``drivers/scsi/arm/queue.c`` -HTB_CMAGIC 0xFEFAFEF1 htb_class ``net/sched/sch_htb.c`` NMI_MAGIC 0x48414d4d455201 nmi_s ``arch/mips/include/asm/sn/nmi.h`` ===================== ================ ======================== ========================================== - -Note that there are also defined special per-driver magic numbers in sound -memory management. See ``include/sound/sndmagic.h`` for complete list of them. Many -OSS sound drivers have their magic numbers constructed from the soundcard PCI -ID - these are not listed here as well. - -HFS is another larger user of magic numbers - you can find them in -``fs/hfs/hfs.h``. diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst index 29e7d7b1cd44c334a064870fa026e3e6f4295a18..40bfbd3b7648f210ed7710bd53726607b94e26d9 100644 --- a/Documentation/process/maintainer-pgp-guide.rst +++ b/Documentation/process/maintainer-pgp-guide.rst @@ -121,57 +121,56 @@ edit your ``~/.gnupg/gpg-agent.conf`` file to set your own values:: to remove anything you had in place for older versions of GnuPG, as it may not be doing the right thing any more. -Set up a refresh cronjob -~~~~~~~~~~~~~~~~~~~~~~~~ - -You will need to regularly refresh your keyring in order to get the -latest changes on other people's public keys, which is best done with a -daily cronjob:: - - @daily /usr/bin/gpg2 --refresh >/dev/null 2>&1 - -Check the full path to your ``gpg`` or ``gpg2`` command and use the -``gpg2`` command if regular ``gpg`` for you is the legacy GnuPG v.1. - -.. _master_key: +.. _protect_your_key: -Protect your master PGP key -=========================== +Protect your PGP key +==================== This guide assumes that you already have a PGP key that you use for Linux kernel development purposes. If you do not yet have one, please see the "`Protecting Code Integrity`_" document mentioned earlier for guidance on how to create a new one. -You should also make a new key if your current one is weaker than 2048 bits -(RSA). - -Master key vs. Subkeys ----------------------- - -Subkeys are fully independent PGP keypairs that are tied to the "master" -key using certifying key signatures (certificates). It is important to -understand the following: - -1. There are no technical differences between the "master key" and "subkeys." -2. At creation time, we assign functional limitations to each key by - giving it specific capabilities. -3. A PGP key can have 4 capabilities: - - - **[S]** key can be used for signing - - **[E]** key can be used for encryption - - **[A]** key can be used for authentication - - **[C]** key can be used for certifying other keys - -4. A single key may have multiple capabilities. -5. A subkey is fully independent from the master key. A message - encrypted to a subkey cannot be decrypted with the master key. If you - lose your private subkey, it cannot be recreated from the master key - in any way. - -The key carrying the **[C]** (certify) capability is considered the -"master" key because it is the only key that can be used to indicate -relationship with other keys. Only the **[C]** key can be used to: +You should also make a new key if your current one is weaker than 2048 +bits (RSA). + +Understanding PGP Subkeys +------------------------- + +A PGP key rarely consists of a single keypair -- usually it is a +collection of independent subkeys that can be used for different +purposes based on their capabilities, assigned at their creation time. +PGP defines four capabilities that a key can have: + +- **[S]** keys can be used for signing +- **[E]** keys can be used for encryption +- **[A]** keys can be used for authentication +- **[C]** keys can be used for certifying other keys + +The key with the **[C]** capability is often called the "master" key, +but this terminology is misleading because it implies that the Certify +key can be used in place of any of other subkey on the same chain (like +a physical "master key" can be used to open the locks made for other +keys). Since this is not the case, this guide will refer to it as "the +Certify key" to avoid any ambiguity. + +It is critical to fully understand the following: + +1. All subkeys are fully independent from each other. If you lose a + private subkey, it cannot be restored or recreated from any other + private key on your chain. +2. With the exception of the Certify key, there can be multiple subkeys + with identical capabilities (e.g. you can have 2 valid encryption + subkeys, 3 valid signing subkeys, but only one valid certification + subkey). All subkeys are fully independent -- a message encrypted to + one **[E]** subkey cannot be decrypted with any other **[E]** subkey + you may also have. +3. A single subkey may have multiple capabilities (e.g. your **[C]** key + can also be your **[S]** key). + +The key carrying the **[C]** (certify) capability is the only key that +can be used to indicate relationship with other keys. Only the **[C]** +key can be used to: - add or revoke other keys (subkeys) with S/E/A capabilities - add, change or revoke identities (uids) associated with the key @@ -180,7 +179,7 @@ relationship with other keys. Only the **[C]** key can be used to: By default, GnuPG creates the following when generating new keys: -- A master key carrying both Certify and Sign capabilities (**[SC]**) +- One subkey carrying both Certify and Sign capabilities (**[SC]**) - A separate subkey with the Encryption capability (**[E]**) If you used the default parameters when generating your key, then that @@ -192,9 +191,6 @@ for example:: uid [ultimate] Alice Dev ssb rsa2048 2018-01-23 [E] [expires: 2020-01-23] -Any key carrying the **[C]** capability is your master key, regardless -of any other capabilities it may have assigned to it. - The long line under the ``sec`` entry is your key fingerprint -- whenever you see ``[fpr]`` in the examples below, that 40-character string is what it refers to. @@ -215,37 +211,30 @@ strong passphrase. To set it or change it, use:: Create a separate Signing subkey -------------------------------- -Our goal is to protect your master key by moving it to offline media, so -if you only have a combined **[SC]** key, then you should create a separate -signing subkey:: +Our goal is to protect your Certify key by moving it to offline media, +so if you only have a combined **[SC]** key, then you should create a +separate signing subkey:: $ gpg --quick-addkey [fpr] ed25519 sign -Remember to tell the keyservers about this change, so others can pull down -your new subkey:: - - $ gpg --send-key [fpr] - .. note:: ECC support in GnuPG GnuPG 2.1 and later has full support for Elliptic Curve Cryptography, with ability to combine ECC subkeys with traditional - RSA master keys. The main upside of ECC cryptography is that it is - much faster computationally and creates much smaller signatures when + RSA keys. The main upside of ECC cryptography is that it is much + faster computationally and creates much smaller signatures when compared byte for byte with 2048+ bit RSA keys. Unless you plan on using a smartcard device that does not support ECC operations, we recommend that you create an ECC signing subkey for your kernel work. - If for some reason you prefer to stay with RSA subkeys, just replace - "ed25519" with "rsa2048" in the above command. Additionally, if you - plan to use a hardware device that does not support ED25519 ECC - keys, like Nitrokey Pro or a Yubikey, then you should use - "nistp256" instead or "ed25519." + Note, that if you plan to use a hardware device that does not + support ED25519 ECC keys, you should choose "nistp256" instead or + "ed25519." -Back up your master key for disaster recovery ---------------------------------------------- +Back up your Certify key for disaster recovery +---------------------------------------------- The more signatures you have on your PGP key from other developers, the more reasons you have to create a backup version that lives on something @@ -277,9 +266,7 @@ home, such as your bank vault. Your printer is probably no longer a simple dumb device connected to your parallel port, but since the output is still encrypted with your passphrase, printing out even to "cloud-integrated" modern - printers should remain a relatively safe operation. One option is to - change the passphrase on your master key immediately after you are - done with paperkey. + printers should remain a relatively safe operation. Back up your whole GnuPG directory ---------------------------------- @@ -300,7 +287,7 @@ will use for backup purposes. You will need to encrypt them using LUKS -- refer to your distro's documentation on how to accomplish this. For the encryption passphrase, you can use the same one as on your -master key. +PGP key. Once the encryption process is over, re-insert the USB drive and make sure it gets properly mounted. Copy your entire ``.gnupg`` directory @@ -319,7 +306,7 @@ far away, because you'll need to use it every now and again for things like editing identities, adding or revoking subkeys, or signing other people's keys. -Remove the master key from your homedir +Remove the Certify key from your homedir ---------------------------------------- The files in our home directory are not as well protected as we like to @@ -334,7 +321,7 @@ think. They can be leaked or stolen via many different means: Protecting your key with a good passphrase greatly helps reduce the risk of any of the above, but passphrases can be discovered via keyloggers, shoulder-surfing, or any number of other means. For this reason, the -recommended setup is to remove your master key from your home directory +recommended setup is to remove your Certify key from your home directory and store it on offline storage. .. warning:: @@ -343,7 +330,7 @@ and store it on offline storage. your GnuPG directory in its entirety. What we are about to do will render your key useless if you do not have a usable backup! -First, identify the keygrip of your master key:: +First, identify the keygrip of your Certify key:: $ gpg --with-keygrip --list-key [fpr] @@ -359,7 +346,7 @@ The output will be something like this:: Keygrip = 3333000000000000000000000000000000000000 Find the keygrip entry that is beneath the ``pub`` line (right under the -master key fingerprint). This will correspond directly to a file in your +Certify key fingerprint). This will correspond directly to a file in your ``~/.gnupg`` directory:: $ cd ~/.gnupg/private-keys-v1.d @@ -369,13 +356,13 @@ master key fingerprint). This will correspond directly to a file in your 3333000000000000000000000000000000000000.key All you have to do is simply remove the .key file that corresponds to -the master keygrip:: +the Certify key keygrip:: $ cd ~/.gnupg/private-keys-v1.d $ rm 1111000000000000000000000000000000000000.key Now, if you issue the ``--list-secret-keys`` command, it will show that -the master key is missing (the ``#`` indicates it is not available):: +the Certify key is missing (the ``#`` indicates it is not available):: $ gpg --list-secret-keys sec# rsa2048 2018-01-24 [SC] [expires: 2020-01-24] @@ -404,7 +391,7 @@ file, which still contains your private keys. Move the subkeys to a dedicated crypto device ============================================= -Even though the master key is now safe from being leaked or stolen, the +Even though the Certify key is now safe from being leaked or stolen, the subkeys are still in your home directory. Anyone who manages to get their hands on those will be able to decrypt your communication or fake your signatures (if they know the passphrase). Furthermore, each time a @@ -447,7 +434,8 @@ functionality. There are several options available: - `Yubikey 5`_: proprietary hardware and software, but cheaper than Nitrokey Pro and comes available in the USB-C form that is more useful with newer laptops. Offers additional security features such as FIDO - U2F, among others, and now finally supports ECC keys (NISTP). + U2F, among others, and now finally supports NISTP and ED25519 ECC + keys. `LWN has a good review`_ of some of the above models, as well as several others. Your choice will depend on cost, shipping availability in your @@ -460,7 +448,7 @@ geographical region, and open/proprietary hardware considerations. Foundation. .. _`Nitrokey Start`: https://shop.nitrokey.com/shop/product/nitrokey-start-6 -.. _`Nitrokey Pro 2`: https://shop.nitrokey.com/shop/product/nitrokey-pro-2-3 +.. _`Nitrokey Pro 2`: https://shop.nitrokey.com/shop/product/nkpr2-nitrokey-pro-2-3 .. _`Yubikey 5`: https://www.yubico.com/products/yubikey-5-overview/ .. _Gnuk: https://www.fsij.org/doc-gnuk/ .. _`LWN has a good review`: https://lwn.net/Articles/736231/ @@ -627,10 +615,10 @@ Other common GnuPG operations Here is a quick reference for some common operations you'll need to do with your PGP key. -Mounting your master key offline storage -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Mounting your safe offline storage +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You will need your master key for any of the operations below, so you +You will need your Certify key for any of the operations below, so you will first need to mount your backup offline storage and tell GnuPG to use it:: @@ -644,7 +632,7 @@ your regular home directory location). Extending key expiration date ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The master key has the default expiration date of 2 years from the date +The Certify key has the default expiration date of 2 years from the date of creation. This is done both for security reasons and to make obsolete keys eventually disappear from keyservers. @@ -685,6 +673,7 @@ remote end. .. _`Agent Forwarding over SSH`: https://wiki.gnupg.org/AgentForwarding +.. _pgp_with_git: Using PGP with Git ================== @@ -828,6 +817,63 @@ You can tell git to always sign commits:: .. _verify_identities: + +How to work with signed patches +------------------------------- + +It is possible to use your PGP key to sign patches sent to kernel +developer mailing lists. Since existing email signature mechanisms +(PGP-Mime or PGP-inline) tend to cause problems with regular code +review tasks, you should use the tool kernel.org created for this +purpose that puts cryptographic attestation signatures into message +headers (a-la DKIM): + +- `Patatt Patch Attestation`_ + +.. _`Patatt Patch Attestation`: https://pypi.org/project/patatt/ + +Installing and configuring patatt +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patatt is packaged for many distributions already, so please check there +first. You can also install it from pypi using "``pip install patatt``". + +If you already have your PGP key configured with git (via the +``user.signingKey`` configuration parameter), then patatt requires no +further configuration. You can start signing your patches by installing +the git-send-email hook in the repository you want:: + + patatt install-hook + +Now any patches you send with ``git send-email`` will be automatically +signed with your cryptographic signature. + +Checking patatt signatures +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you are using ``b4`` to retrieve and apply patches, then it will +automatically attempt to verify all DKIM and patatt signatures it +encounters, for example:: + + $ b4 am 20220720205013.890942-1-broonie@kernel.org + [...] + Checking attestation on all messages, may take a moment... + --- + ✓ [PATCH v1 1/3] kselftest/arm64: Correct buffer allocation for SVE Z registers + ✓ [PATCH v1 2/3] arm64/sve: Document our actual ABI for clearing registers on syscall + ✓ [PATCH v1 3/3] kselftest/arm64: Enforce actual ABI for SVE syscalls + --- + ✓ Signed: openpgp/broonie@kernel.org + ✓ Signed: DKIM/kernel.org + +.. note:: + + Patatt and b4 are still in active development and you should check + the latest documentation for these projects for any new or updated + features. + +.. _kernel_identities: + How to verify kernel developer identities ========================================= @@ -899,65 +945,17 @@ the new default in GnuPG v2). To set it, add (or modify) the trust-model tofu+pgp -How to use keyservers (more) safely ------------------------------------ - -If you get a "No public key" error when trying to validate someone's -tag, then you should attempt to lookup that key using a keyserver. It is -important to keep in mind that there is absolutely no guarantee that the -key you retrieve from PGP keyservers belongs to the actual person -- -that much is by design. You are supposed to use the Web of Trust to -establish key validity. - -How to properly maintain the Web of Trust is beyond the scope of this -document, simply because doing it properly requires both effort and -dedication that tends to be beyond the caring threshold of most human -beings. Here are some shortcuts that will help you reduce the risk of -importing a malicious key. - -First, let's say you've tried to run ``git verify-tag`` but it returned -an error saying the key is not found:: - - $ git verify-tag sunxi-fixes-for-4.15-2 - gpg: Signature made Sun 07 Jan 2018 10:51:55 PM EST - gpg: using RSA key DA73759BF8619E484E5A3B47389A54219C0F2430 - gpg: issuer "wens@...org" - gpg: Can't check signature: No public key - -Let's query the keyserver for more info about that key fingerprint (the -fingerprint probably belongs to a subkey, so we can't use it directly -without finding out the ID of the master key it is associated with):: - - $ gpg --search DA73759BF8619E484E5A3B47389A54219C0F2430 - gpg: data source: hkp://keys.gnupg.net - (1) Chen-Yu Tsai - 4096 bit RSA key C94035C21B4F2AEB, created: 2017-03-14, expires: 2019-03-15 - Keys 1-1 of 1 for "DA73759BF8619E484E5A3B47389A54219C0F2430". Enter number(s), N)ext, or Q)uit > q - -Locate the ID of the master key in the output, in our example -``C94035C21B4F2AEB``. Now display the key of Linus Torvalds that you -have on your keyring:: - - $ gpg --list-key torvalds@kernel.org - pub rsa2048 2011-09-20 [SC] - ABAF11C65A2970B130ABE3C479BE3E4300411886 - uid [ unknown] Linus Torvalds - sub rsa2048 2011-09-20 [E] - -Next, find a trust path from Linus Torvalds to the key-id you found via ``gpg ---search`` of the unknown key. For this, you can use several tools including -https://github.com/mricon/wotmate, -https://git.kernel.org/pub/scm/docs/kernel/pgpkeys.git/tree/graphs, and -https://the.earth.li/~noodles/pathfind.html. - -If you get a few decent trust paths, then it's a pretty good indication -that it is a valid key. You can add it to your keyring from the -keyserver now:: - - $ gpg --recv-key C94035C21B4F2AEB - -This process is not perfect, and you are obviously trusting the -administrators of the PGP Pathfinder service to not be malicious (in -fact, this goes against :ref:`devs_not_infra`). However, if you -do not carefully maintain your own web of trust, then it is a marked -improvement over blindly trusting keyservers. +Using the kernel.org web of trust repository +-------------------------------------------- + +Kernel.org maintains a git repository with developers' public keys as a +replacement for replicating keyserver networks that have gone mostly +dark in the past few years. The full documentation for how to set up +that repository as your source of public keys can be found here: + +- `Kernel developer PGP Keyring`_ + +If you are a kernel developer, please consider submitting your key for +inclusion into that keyring. + +.. _`Kernel developer PGP Keyring`: https://korg.docs.kernel.org/pgpkeys.html diff --git a/Documentation/process/stable-kernel-rules.rst b/Documentation/process/stable-kernel-rules.rst index c61865e91f52744cb46e475c1fe015a43170cea6..2fd8aa593a285173785241b0c21954cb8c75aa0c 100644 --- a/Documentation/process/stable-kernel-rules.rst +++ b/Documentation/process/stable-kernel-rules.rst @@ -97,6 +97,12 @@ text, like this: commit upstream. +or alternatively: + +.. code-block:: none + + [ Upstream commit ] + Additionally, some patches submitted via :ref:`option_1` may have additional patch prerequisites which can be cherry-picked. This can be specified in the following format in the sign-off area: diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index be49d8f2601b420edce604ad4fe9eca6644aa70a..7dc94555417dc7e6977e2e9845f5a3231ecad371 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -715,8 +715,8 @@ references. .. _backtraces: -Backtraces in commit mesages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Backtraces in commit messages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Backtraces help document the call chain leading to a problem. However, not all backtraces are helpful. For example, early boot call chains are diff --git a/Documentation/riscv/index.rst b/Documentation/riscv/index.rst index e23b876ad6ebb6b2c571cacfac873f542bfb5862..2e5b18fbb1451fc857a98f3d14b5d5fb3b4442cf 100644 --- a/Documentation/riscv/index.rst +++ b/Documentation/riscv/index.rst @@ -8,6 +8,7 @@ RISC-V architecture boot-image-header vm-layout patch-acceptance + uabi features diff --git a/Documentation/riscv/uabi.rst b/Documentation/riscv/uabi.rst new file mode 100644 index 0000000000000000000000000000000000000000..21a82cfb6c4dd43ae3134b67e87c778d19e01f2e --- /dev/null +++ b/Documentation/riscv/uabi.rst @@ -0,0 +1,6 @@ +.. SPDX-License-Identifier: GPL-2.0 + +RISC-V Linux User ABI +===================== + +Misaligned accesses are supported in userspace, but they may perform poorly. diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst new file mode 100644 index 0000000000000000000000000000000000000000..6982b63775da583735b8e5d52e4a784de719293c --- /dev/null +++ b/Documentation/rust/arch-support.rst @@ -0,0 +1,19 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Arch Support +============ + +Currently, the Rust compiler (``rustc``) uses LLVM for code generation, +which limits the supported architectures that can be targeted. In addition, +support for building the kernel with LLVM/Clang varies (please see +Documentation/kbuild/llvm.rst). This support is needed for ``bindgen`` +which uses ``libclang``. + +Below is a general summary of architectures that currently work. Level of +support corresponds to ``S`` values in the ``MAINTAINERS`` file. + +============ ================ ============================================== +Architecture Level of support Constraints +============ ================ ============================================== +``x86`` Maintained ``x86_64`` only. +============ ================ ============================================== diff --git a/Documentation/rust/coding-guidelines.rst b/Documentation/rust/coding-guidelines.rst new file mode 100644 index 0000000000000000000000000000000000000000..aa8ed082613e129e1058520374f4557bf4151a4e --- /dev/null +++ b/Documentation/rust/coding-guidelines.rst @@ -0,0 +1,216 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Coding Guidelines +================= + +This document describes how to write Rust code in the kernel. + + +Style & formatting +------------------ + +The code should be formatted using ``rustfmt``. In this way, a person +contributing from time to time to the kernel does not need to learn and +remember one more style guide. More importantly, reviewers and maintainers +do not need to spend time pointing out style issues anymore, and thus +less patch roundtrips may be needed to land a change. + +.. note:: Conventions on comments and documentation are not checked by + ``rustfmt``. Thus those are still needed to be taken care of. + +The default settings of ``rustfmt`` are used. This means the idiomatic Rust +style is followed. For instance, 4 spaces are used for indentation rather +than tabs. + +It is convenient to instruct editors/IDEs to format while typing, +when saving or at commit time. However, if for some reason reformatting +the entire kernel Rust sources is needed at some point, the following can be +run:: + + make LLVM=1 rustfmt + +It is also possible to check if everything is formatted (printing a diff +otherwise), for instance for a CI, with:: + + make LLVM=1 rustfmtcheck + +Like ``clang-format`` for the rest of the kernel, ``rustfmt`` works on +individual files, and does not require a kernel configuration. Sometimes it may +even work with broken code. + + +Comments +-------- + +"Normal" comments (i.e. ``//``, rather than code documentation which starts +with ``///`` or ``//!``) are written in Markdown the same way as documentation +comments are, even though they will not be rendered. This improves consistency, +simplifies the rules and allows to move content between the two kinds of +comments more easily. For instance: + +.. code-block:: rust + + // `object` is ready to be handled now. + f(object); + +Furthermore, just like documentation, comments are capitalized at the beginning +of a sentence and ended with a period (even if it is a single sentence). This +includes ``// SAFETY:``, ``// TODO:`` and other "tagged" comments, e.g.: + +.. code-block:: rust + + // FIXME: The error should be handled properly. + +Comments should not be used for documentation purposes: comments are intended +for implementation details, not users. This distinction is useful even if the +reader of the source file is both an implementor and a user of an API. In fact, +sometimes it is useful to use both comments and documentation at the same time. +For instance, for a ``TODO`` list or to comment on the documentation itself. +For the latter case, comments can be inserted in the middle; that is, closer to +the line of documentation to be commented. For any other case, comments are +written after the documentation, e.g.: + +.. code-block:: rust + + /// Returns a new [`Foo`]. + /// + /// # Examples + /// + // TODO: Find a better example. + /// ``` + /// let foo = f(42); + /// ``` + // FIXME: Use fallible approach. + pub fn f(x: i32) -> Foo { + // ... + } + +One special kind of comments are the ``// SAFETY:`` comments. These must appear +before every ``unsafe`` block, and they explain why the code inside the block is +correct/sound, i.e. why it cannot trigger undefined behavior in any case, e.g.: + +.. code-block:: rust + + // SAFETY: `p` is valid by the safety requirements. + unsafe { *p = 0; } + +``// SAFETY:`` comments are not to be confused with the ``# Safety`` sections +in code documentation. ``# Safety`` sections specify the contract that callers +(for functions) or implementors (for traits) need to abide by. ``// SAFETY:`` +comments show why a call (for functions) or implementation (for traits) actually +respects the preconditions stated in a ``# Safety`` section or the language +reference. + + +Code documentation +------------------ + +Rust kernel code is not documented like C kernel code (i.e. via kernel-doc). +Instead, the usual system for documenting Rust code is used: the ``rustdoc`` +tool, which uses Markdown (a lightweight markup language). + +To learn Markdown, there are many guides available out there. For instance, +the one at: + + https://commonmark.org/help/ + +This is how a well-documented Rust function may look like: + +.. code-block:: rust + + /// Returns the contained [`Some`] value, consuming the `self` value, + /// without checking that the value is not [`None`]. + /// + /// # Safety + /// + /// Calling this method on [`None`] is *[undefined behavior]*. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// let x = Some("air"); + /// assert_eq!(unsafe { x.unwrap_unchecked() }, "air"); + /// ``` + pub unsafe fn unwrap_unchecked(self) -> T { + match self { + Some(val) => val, + + // SAFETY: The safety contract must be upheld by the caller. + None => unsafe { hint::unreachable_unchecked() }, + } + } + +This example showcases a few ``rustdoc`` features and some conventions followed +in the kernel: + + - The first paragraph must be a single sentence briefly describing what + the documented item does. Further explanations must go in extra paragraphs. + + - Unsafe functions must document their safety preconditions under + a ``# Safety`` section. + + - While not shown here, if a function may panic, the conditions under which + that happens must be described under a ``# Panics`` section. + + Please note that panicking should be very rare and used only with a good + reason. In almost all cases, a fallible approach should be used, typically + returning a ``Result``. + + - If providing examples of usage would help readers, they must be written in + a section called ``# Examples``. + + - Rust items (functions, types, constants...) must be linked appropriately + (``rustdoc`` will create a link automatically). + + - Any ``unsafe`` block must be preceded by a ``// SAFETY:`` comment + describing why the code inside is sound. + + While sometimes the reason might look trivial and therefore unneeded, + writing these comments is not just a good way of documenting what has been + taken into account, but most importantly, it provides a way to know that + there are no *extra* implicit constraints. + +To learn more about how to write documentation for Rust and extra features, +please take a look at the ``rustdoc`` book at: + + https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html + + +Naming +------ + +Rust kernel code follows the usual Rust naming conventions: + + https://rust-lang.github.io/api-guidelines/naming.html + +When existing C concepts (e.g. macros, functions, objects...) are wrapped into +a Rust abstraction, a name as close as reasonably possible to the C side should +be used in order to avoid confusion and to improve readability when switching +back and forth between the C and Rust sides. For instance, macros such as +``pr_info`` from C are named the same in the Rust side. + +Having said that, casing should be adjusted to follow the Rust naming +conventions, and namespacing introduced by modules and types should not be +repeated in the item names. For instance, when wrapping constants like: + +.. code-block:: c + + #define GPIO_LINE_DIRECTION_IN 0 + #define GPIO_LINE_DIRECTION_OUT 1 + +The equivalent in Rust may look like (ignoring documentation): + +.. code-block:: rust + + pub mod gpio { + pub enum LineDirection { + In = bindings::GPIO_LINE_DIRECTION_IN as _, + Out = bindings::GPIO_LINE_DIRECTION_OUT as _, + } + } + +That is, the equivalent of ``GPIO_LINE_DIRECTION_IN`` would be referred to as +``gpio::LineDirection::In``. In particular, it should not be named +``gpio::gpio_line_direction::GPIO_LINE_DIRECTION_IN``. diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst new file mode 100644 index 0000000000000000000000000000000000000000..49029ee82e559e543cf583021b4c3053d8852fcb --- /dev/null +++ b/Documentation/rust/general-information.rst @@ -0,0 +1,79 @@ +.. SPDX-License-Identifier: GPL-2.0 + +General Information +=================== + +This document contains useful information to know when working with +the Rust support in the kernel. + + +Code documentation +------------------ + +Rust kernel code is documented using ``rustdoc``, its built-in documentation +generator. + +The generated HTML docs include integrated search, linked items (e.g. types, +functions, constants), source code, etc. They may be read at (TODO: link when +in mainline and generated alongside the rest of the documentation): + + http://kernel.org/ + +The docs can also be easily generated and read locally. This is quite fast +(same order as compiling the code itself) and no special tools or environment +are needed. This has the added advantage that they will be tailored to +the particular kernel configuration used. To generate them, use the ``rustdoc`` +target with the same invocation used for compilation, e.g.:: + + make LLVM=1 rustdoc + +To read the docs locally in your web browser, run e.g.:: + + xdg-open rust/doc/kernel/index.html + +To learn about how to write the documentation, please see coding-guidelines.rst. + + +Extra lints +----------- + +While ``rustc`` is a very helpful compiler, some extra lints and analyses are +available via ``clippy``, a Rust linter. To enable it, pass ``CLIPPY=1`` to +the same invocation used for compilation, e.g.:: + + make LLVM=1 CLIPPY=1 + +Please note that Clippy may change code generation, thus it should not be +enabled while building a production kernel. + + +Abstractions vs. bindings +------------------------- + +Abstractions are Rust code wrapping kernel functionality from the C side. + +In order to use functions and types from the C side, bindings are created. +Bindings are the declarations for Rust of those functions and types from +the C side. + +For instance, one may write a ``Mutex`` abstraction in Rust which wraps +a ``struct mutex`` from the C side and calls its functions through the bindings. + +Abstractions are not available for all the kernel internal APIs and concepts, +but it is intended that coverage is expanded as time goes on. "Leaf" modules +(e.g. drivers) should not use the C bindings directly. Instead, subsystems +should provide as-safe-as-possible abstractions as needed. + + +Conditional compilation +----------------------- + +Rust code has access to conditional compilation based on the kernel +configuration: + +.. code-block:: rust + + #[cfg(CONFIG_X)] // Enabled (`y` or `m`) + #[cfg(CONFIG_X="y")] // Enabled as a built-in (`y`) + #[cfg(CONFIG_X="m")] // Enabled as a module (`m`) + #[cfg(not(CONFIG_X))] // Disabled diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..4ae8c66b94faf92db55e9cb95569c21b4c10b382 --- /dev/null +++ b/Documentation/rust/index.rst @@ -0,0 +1,22 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Rust +==== + +Documentation related to Rust within the kernel. To start using Rust +in the kernel, please read the quick-start.rst guide. + +.. toctree:: + :maxdepth: 1 + + quick-start + general-information + coding-guidelines + arch-support + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst new file mode 100644 index 0000000000000000000000000000000000000000..13b7744b1e2753692c0b9c99ffecd3b663bda575 --- /dev/null +++ b/Documentation/rust/quick-start.rst @@ -0,0 +1,232 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Quick Start +=========== + +This document describes how to get started with kernel development in Rust. + + +Requirements: Building +---------------------- + +This section explains how to fetch the tools needed for building. + +Some of these requirements might be available from Linux distributions +under names like ``rustc``, ``rust-src``, ``rust-bindgen``, etc. However, +at the time of writing, they are likely not to be recent enough unless +the distribution tracks the latest releases. + +To easily check whether the requirements are met, the following target +can be used:: + + make LLVM=1 rustavailable + +This triggers the same logic used by Kconfig to determine whether +``RUST_IS_AVAILABLE`` should be enabled; but it also explains why not +if that is the case. + + +rustc +***** + +A particular version of the Rust compiler is required. Newer versions may or +may not work because, for the moment, the kernel depends on some unstable +Rust features. + +If ``rustup`` is being used, enter the checked out source code directory +and run:: + + rustup override set $(scripts/min-tool-version.sh rustc) + +Otherwise, fetch a standalone installer or install ``rustup`` from: + + https://www.rust-lang.org + + +Rust standard library source +**************************** + +The Rust standard library source is required because the build system will +cross-compile ``core`` and ``alloc``. + +If ``rustup`` is being used, run:: + + rustup component add rust-src + +The components are installed per toolchain, thus upgrading the Rust compiler +version later on requires re-adding the component. + +Otherwise, if a standalone installer is used, the Rust repository may be cloned +into the installation folder of the toolchain:: + + git clone --recurse-submodules \ + --branch $(scripts/min-tool-version.sh rustc) \ + https://github.com/rust-lang/rust \ + $(rustc --print sysroot)/lib/rustlib/src/rust + +In this case, upgrading the Rust compiler version later on requires manually +updating this clone. + + +libclang +******** + +``libclang`` (part of LLVM) is used by ``bindgen`` to understand the C code +in the kernel, which means LLVM needs to be installed; like when the kernel +is compiled with ``CC=clang`` or ``LLVM=1``. + +Linux distributions are likely to have a suitable one available, so it is +best to check that first. + +There are also some binaries for several systems and architectures uploaded at: + + https://releases.llvm.org/download.html + +Otherwise, building LLVM takes quite a while, but it is not a complex process: + + https://llvm.org/docs/GettingStarted.html#getting-the-source-code-and-building-llvm + +Please see Documentation/kbuild/llvm.rst for more information and further ways +to fetch pre-built releases and distribution packages. + + +bindgen +******* + +The bindings to the C side of the kernel are generated at build time using +the ``bindgen`` tool. A particular version is required. + +Install it via (note that this will download and build the tool from source):: + + cargo install --locked --version $(scripts/min-tool-version.sh bindgen) bindgen + + +Requirements: Developing +------------------------ + +This section explains how to fetch the tools needed for developing. That is, +they are not needed when just building the kernel. + + +rustfmt +******* + +The ``rustfmt`` tool is used to automatically format all the Rust kernel code, +including the generated C bindings (for details, please see +coding-guidelines.rst). + +If ``rustup`` is being used, its ``default`` profile already installs the tool, +thus nothing needs to be done. If another profile is being used, the component +can be installed manually:: + + rustup component add rustfmt + +The standalone installers also come with ``rustfmt``. + + +clippy +****** + +``clippy`` is a Rust linter. Running it provides extra warnings for Rust code. +It can be run by passing ``CLIPPY=1`` to ``make`` (for details, please see +general-information.rst). + +If ``rustup`` is being used, its ``default`` profile already installs the tool, +thus nothing needs to be done. If another profile is being used, the component +can be installed manually:: + + rustup component add clippy + +The standalone installers also come with ``clippy``. + + +cargo +***** + +``cargo`` is the Rust native build system. It is currently required to run +the tests since it is used to build a custom standard library that contains +the facilities provided by the custom ``alloc`` in the kernel. The tests can +be run using the ``rusttest`` Make target. + +If ``rustup`` is being used, all the profiles already install the tool, +thus nothing needs to be done. + +The standalone installers also come with ``cargo``. + + +rustdoc +******* + +``rustdoc`` is the documentation tool for Rust. It generates pretty HTML +documentation for Rust code (for details, please see +general-information.rst). + +``rustdoc`` is also used to test the examples provided in documented Rust code +(called doctests or documentation tests). The ``rusttest`` Make target uses +this feature. + +If ``rustup`` is being used, all the profiles already install the tool, +thus nothing needs to be done. + +The standalone installers also come with ``rustdoc``. + + +rust-analyzer +************* + +The `rust-analyzer `_ language server can +be used with many editors to enable syntax highlighting, completion, go to +definition, and other features. + +``rust-analyzer`` needs a configuration file, ``rust-project.json``, which +can be generated by the ``rust-analyzer`` Make target. + + +Configuration +------------- + +``Rust support`` (``CONFIG_RUST``) needs to be enabled in the ``General setup`` +menu. The option is only shown if a suitable Rust toolchain is found (see +above), as long as the other requirements are met. In turn, this will make +visible the rest of options that depend on Rust. + +Afterwards, go to:: + + Kernel hacking + -> Sample kernel code + -> Rust samples + +And enable some sample modules either as built-in or as loadable. + + +Building +-------- + +Building a kernel with a complete LLVM toolchain is the best supported setup +at the moment. That is:: + + make LLVM=1 + +For architectures that do not support a full LLVM toolchain, use:: + + make CC=clang + +Using GCC also works for some configurations, but it is very experimental at +the moment. + + +Hacking +------- + +To dive deeper, take a look at the source code of the samples +at ``samples/rust/``, the Rust support code under ``rust/`` and +the ``Rust hacking`` menu under ``Kernel hacking``. + +If GDB/Binutils is used and Rust symbols are not getting demangled, the reason +is the toolchain does not support Rust's new v0 mangling scheme yet. +There are a few ways out: + + - Install a newer release (GDB >= 10.2, Binutils >= 2.36). + + - Some versions of GDB (e.g. vanilla GDB 10.1) are able to use + the pre-demangled names embedded in the debug info (``CONFIG_DEBUG_INFO``). diff --git a/Documentation/s390/vfio-ap.rst b/Documentation/s390/vfio-ap.rst index 61a0a3c6c7b4b8a14ba3b06120ba21ff811e2f6a..00f4a04f6d4c6ac0dbb224839888b91f46c7dc42 100644 --- a/Documentation/s390/vfio-ap.rst +++ b/Documentation/s390/vfio-ap.rst @@ -297,7 +297,7 @@ of the VFIO AP mediated device driver:: | MDEV CORE | | MODULE | | mdev.ko | - | +---------+ | mdev_register_device() +--------------+ + | +---------+ | mdev_register_parent() +--------------+ | |Physical | +<-----------------------+ | | | device | | | vfio_ap.ko |<-> matrix | |interface| +----------------------->+ | device diff --git a/Documentation/s390/vfio-ccw.rst b/Documentation/s390/vfio-ccw.rst index 8aad08a8b8a50384dba9c500efc82e4b1679fdf6..ea928a3806f43f8f6133e8e8e7d657da699250e1 100644 --- a/Documentation/s390/vfio-ccw.rst +++ b/Documentation/s390/vfio-ccw.rst @@ -156,7 +156,7 @@ Below is a high Level block diagram:: | MDEV CORE | | MODULE | | mdev.ko | - | +---------+ | mdev_register_device() +--------------+ + | +---------+ | mdev_register_parent() +--------------+ | |Physical | +<-----------------------+ | | | device | | | vfio_ccw.ko |<-> subchannel | |interface| +----------------------->+ | device diff --git a/Documentation/scheduler/sched-design-CFS.rst b/Documentation/scheduler/sched-design-CFS.rst index 59b2d1fb4dc4779382a7487bad7a829f9a6c6bef..03db55504515186ad0875cd190a56129dc997771 100644 --- a/Documentation/scheduler/sched-design-CFS.rst +++ b/Documentation/scheduler/sched-design-CFS.rst @@ -94,7 +94,7 @@ other HZ detail. Thus the CFS scheduler has no notion of "timeslices" in the way the previous scheduler had, and has no heuristics whatsoever. There is only one central tunable (you have to switch on CONFIG_SCHED_DEBUG): - /proc/sys/kernel/sched_min_granularity_ns + /sys/kernel/debug/sched/min_granularity_ns which can be used to tune the scheduler from "desktop" (i.e., low latencies) to "server" (i.e., good batching) workloads. It defaults to a setting suitable diff --git a/Documentation/scsi/ChangeLog.lpfc b/Documentation/scsi/ChangeLog.lpfc index 2f6d595f95e1c5e55307a8f228968d6afa39f499..caedc8571b45fb2b4896ec0590c9efd7c62c514f 100644 --- a/Documentation/scsi/ChangeLog.lpfc +++ b/Documentation/scsi/ChangeLog.lpfc @@ -401,7 +401,7 @@ Changes from 20041213 to 20041220 structure. * Integrated patch from Christoph Hellwig Kill compile warnings on 64 bit platforms: %variables for %llx format - specifiers must be caste to long long because %(u)int64_t can + specifiers must be cast to long long because %(u)int64_t can just be long on 64bit platforms. * Integrated patch from Christoph Hellwig Removes dead code. diff --git a/Documentation/security/landlock.rst b/Documentation/security/landlock.rst index 5c77730b44791e0c9fbca8ca939973c07c3c3022..c0029d5d02eb1edf0be8e018b5bf0fb74fb70cc5 100644 --- a/Documentation/security/landlock.rst +++ b/Documentation/security/landlock.rst @@ -7,7 +7,7 @@ Landlock LSM: kernel documentation ================================== :Author: Mickaël Salaün -:Date: May 2022 +:Date: September 2022 Landlock's goal is to create scoped access-control (i.e. sandboxing). To harden a whole system, this feature should be available to any process, @@ -49,13 +49,13 @@ Filesystem access rights ------------------------ All access rights are tied to an inode and what can be accessed through it. -Reading the content of a directory doesn't imply to be allowed to read the +Reading the content of a directory does not imply to be allowed to read the content of a listed inode. Indeed, a file name is local to its parent directory, and an inode can be referenced by multiple file names thanks to (hard) links. Being able to unlink a file only has a direct impact on the directory, not the unlinked inode. This is the reason why -`LANDLOCK_ACCESS_FS_REMOVE_FILE` or `LANDLOCK_ACCESS_FS_REFER` are not allowed -to be tied to files but only to directories. +``LANDLOCK_ACCESS_FS_REMOVE_FILE`` or ``LANDLOCK_ACCESS_FS_REFER`` are not +allowed to be tied to files but only to directories. Tests ===== diff --git a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst index 176b73583b7a7b3063f6b1f9f989e0bac1d7d8cc..07a620c5ca7451eb939515b4663bb5c1d6ce4bb0 100644 --- a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst +++ b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst @@ -3565,13 +3565,17 @@ given size. The second argument (type) and the third argument (device pointer) are dependent on the bus. For normal devices, pass the device pointer (typically identical as ``card->dev``) to the third argument with -``SNDRV_DMA_TYPE_DEV`` type. For the continuous buffer unrelated to the +``SNDRV_DMA_TYPE_DEV`` type. + +For the continuous buffer unrelated to the bus can be pre-allocated with ``SNDRV_DMA_TYPE_CONTINUOUS`` type. You can pass NULL to the device pointer in that case, which is the default mode implying to allocate with ``GFP_KERNEL`` flag. -If you need a different GFP flag, you can pass it by encoding the flag -into the device pointer via a special macro -:c:func:`snd_dma_continuous_data()`. +If you need a restricted (lower) address, set up the coherent DMA mask +bits for the device, and pass the device pointer, like the normal +device memory allocations. For this type, it's still allowed to pass +NULL to the device pointer, too, if no address restriction is needed. + For the scatter-gather buffers, use ``SNDRV_DMA_TYPE_DEV_SG`` with the device pointer (see the `Non-Contiguous Buffers`_ section). @@ -3811,15 +3815,6 @@ arguments here. Since each vmalloc call should succeed at any time, we don't need to pre-allocate the buffers like other continuous pages. -If you need the 32bit DMA allocation, pass the device pointer encoded -by :c:func:`snd_dma_continuous_data()` with ``GFP_KERNEL|__GFP_DMA32`` -argument. - -:: - - snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, - snd_dma_continuous_data(GFP_KERNEL | __GFP_DMA32), 0, 0); - Proc Interface ============== diff --git a/Documentation/sphinx/kerneldoc-preamble.sty b/Documentation/sphinx/kerneldoc-preamble.sty index 2a29cbe51396d66e2fb43c5791905cc43cb0eb6e..9707e033c8c452e0d4960c598c7dfce0aa29b5b2 100644 --- a/Documentation/sphinx/kerneldoc-preamble.sty +++ b/Documentation/sphinx/kerneldoc-preamble.sty @@ -70,8 +70,16 @@ % Translations have Asian (CJK) characters which are only displayed if % xeCJK is used +\usepackage{ifthen} +\newboolean{enablecjk} +\setboolean{enablecjk}{false} \IfFontExistsTF{Noto Sans CJK SC}{ - % Load xeCJK when CJK font is available + \IfFileExists{xeCJK.sty}{ + \setboolean{enablecjk}{true} + }{} +}{} +\ifthenelse{\boolean{enablecjk}}{ + % Load xeCJK when both the Noto Sans CJK font and xeCJK.sty are available. \usepackage{xeCJK} % Noto CJK fonts don't provide slant shape. [AutoFakeSlant] permits % its emulation. @@ -196,7 +204,7 @@ % Inactivate CJK after tableofcontents \apptocmd{\sphinxtableofcontents}{\kerneldocCJKoff}{}{} \xeCJKsetup{CJKspace = true}% For inter-phrase space of Korean TOC -}{ % No CJK font found +}{ % Don't enable CJK % Custom macros to on/off CJK and switch CJK fonts (Dummy) \newcommand{\kerneldocCJKon}{} \newcommand{\kerneldocCJKoff}{} @@ -204,14 +212,16 @@ %% and ignore the argument (#1) in their definitions, whole contents of %% CJK chapters can be ignored. \newcommand{\kerneldocBeginSC}[1]{% - %% Put a note on missing CJK fonts in place of zh_CN translation. - \begin{sphinxadmonition}{note}{Note on missing fonts:} + %% Put a note on missing CJK fonts or the xecjk package in place of + %% zh_CN translation. + \begin{sphinxadmonition}{note}{Note on missing fonts and a package:} Translations of Simplified Chinese (zh\_CN), Traditional Chinese (zh\_TW), Korean (ko\_KR), and Japanese (ja\_JP) were skipped - due to the lack of suitable font families. + due to the lack of suitable font families and/or the texlive-xecjk + package. If you want them, please install ``Noto Sans CJK'' font families - by following instructions from + along with the texlive-xecjk package by following instructions from \sphinxcode{./scripts/sphinx-pre-install}. Having optional ``Noto Serif CJK'' font families will improve the looks of those translations. diff --git a/Documentation/staging/index.rst b/Documentation/staging/index.rst index abd0d18254d2bf22ead26746028502f4feb6b431..ded8254bc0d7508a321d8568f9f4844d7da6de86 100644 --- a/Documentation/staging/index.rst +++ b/Documentation/staging/index.rst @@ -14,45 +14,3 @@ Unsorted Documentation static-keys tee xz - -Atomic Types -============ - -.. raw:: latex - - \footnotesize - -.. include:: ../atomic_t.txt - :literal: - -.. raw:: latex - - \normalsize - -Atomic bitops -============= - -.. raw:: latex - - \footnotesize - -.. include:: ../atomic_bitops.txt - :literal: - -.. raw:: latex - - \normalsize - -Memory Barriers -=============== - -.. raw:: latex - - \footnotesize - -.. include:: ../memory-barriers.txt - :literal: - -.. raw:: latex - - \normalsize diff --git a/Documentation/subsystem-apis.rst b/Documentation/subsystem-apis.rst new file mode 100644 index 0000000000000000000000000000000000000000..af65004a80aa7d5532bd6e8b57349c962b51899a --- /dev/null +++ b/Documentation/subsystem-apis.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================== +Kernel subsystem documentation +============================== + +These books get into the details of how specific kernel subsystems work +from the point of view of a kernel developer. Much of the information here +is taken directly from the kernel source, with supplemental material added +as needed (or at least as we managed to add it — probably *not* all that is +needed). + +**Fixme**: much more organizational work is needed here. + +.. toctree:: + :maxdepth: 1 + + driver-api/index + core-api/index + locking/index + accounting/index + block/index + cdrom/index + cpu-freq/index + fb/index + fpga/index + hid/index + i2c/index + iio/index + isdn/index + infiniband/index + leds/index + netlabel/index + networking/index + pcmcia/index + power/index + target/index + timers/index + spi/index + w1/index + watchdog/index + virt/index + input/index + hwmon/index + gpu/index + security/index + sound/index + crypto/index + filesystems/index + mm/index + bpf/index + usb/index + PCI/index + scsi/index + misc-devices/index + scheduler/index + mhi/index + peci/index diff --git a/Documentation/tools/rtla/rtla-timerlat-top.rst b/Documentation/tools/rtla/rtla-timerlat-top.rst index 1c321de1c171efdc01a52f86b40025e5e8a88239..7c4e4b10949330f17c6e4faf9828fe2773efe921 100644 --- a/Documentation/tools/rtla/rtla-timerlat-top.rst +++ b/Documentation/tools/rtla/rtla-timerlat-top.rst @@ -39,7 +39,7 @@ higher than *30 us*. It is also set to stop the session if a *Thread* timer latency higher than *30 us* is hit. Finally, it is set to save the trace buffer if the stop condition is hit:: - [root@alien ~]# rtla timerlat top -s 30 -t 30 -T + [root@alien ~]# rtla timerlat top -s 30 -T 30 -t Timer Latency 0 00:00:59 | IRQ Timer Latency (us) | Thread Timer Latency (us) CPU COUNT | cur min avg max | cur min avg max diff --git a/Documentation/trace/coresight/coresight-cpu-debug.rst b/Documentation/trace/coresight/coresight-cpu-debug.rst index 993dd294b81bad4efec0115326eee66629c541b3..836b35532667cd1cd04375dde06b0f221e74bd58 100644 --- a/Documentation/trace/coresight/coresight-cpu-debug.rst +++ b/Documentation/trace/coresight/coresight-cpu-debug.rst @@ -117,7 +117,8 @@ divide into below cases: Device Tree Bindings -------------------- -See Documentation/devicetree/bindings/arm/coresight-cpu-debug.txt for details. +See Documentation/devicetree/bindings/arm/arm,coresight-cpu-debug.yaml for +details. How to use the module diff --git a/Documentation/trace/coresight/coresight-etm4x-reference.rst b/Documentation/trace/coresight/coresight-etm4x-reference.rst index fb7578fd937238ee5e0d7d78330801b618fe1786..70e34b8c81c1c82486ce7dce0e88699d6eaa90bc 100644 --- a/Documentation/trace/coresight/coresight-etm4x-reference.rst +++ b/Documentation/trace/coresight/coresight-etm4x-reference.rst @@ -71,6 +71,20 @@ the ‘TRC’ prefix. ---- +:File: ``ts_source`` (ro) +:Trace Registers: None. +:Notes: + When FEAT_TRF is implemented, value of TRFCR_ELx.TS used for trace session. Otherwise -1 + indicates an unknown time source. Check trcidr0.tssize to see if a global timestamp is + available. + +:Example: + ``$> cat ts_source`` + + ``$> 1`` + +---- + :File: ``addr_idx`` (rw) :Trace Registers: None. :Notes: diff --git a/Documentation/trace/coresight/coresight-perf.rst b/Documentation/trace/coresight/coresight-perf.rst new file mode 100644 index 0000000000000000000000000000000000000000..d087aae7d49286fc45d8232c25da4dccced4f2db --- /dev/null +++ b/Documentation/trace/coresight/coresight-perf.rst @@ -0,0 +1,158 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================ +CoreSight - Perf +================ + + :Author: Carsten Haitzler + :Date: June 29th, 2022 + +Perf is able to locally access CoreSight trace data and store it to the +output perf data files. This data can then be later decoded to give the +instructions that were traced for debugging or profiling purposes. You +can log such data with a perf record command like:: + + perf record -e cs_etm//u testbinary + +This would run some test binary (testbinary) until it exits and record +a perf.data trace file. That file would have AUX sections if CoreSight +is working correctly. You can dump the content of this file as +readable text with a command like:: + + perf report --stdio --dump -i perf.data + +You should find some sections of this file have AUX data blocks like:: + + 0x1e78 [0x30]: PERF_RECORD_AUXTRACE size: 0x11dd0 offset: 0 ref: 0x1b614fc1061b0ad1 idx: 0 tid: 531230 cpu: -1 + + . ... CoreSight ETM Trace data: size 73168 bytes + Idx:0; ID:10; I_ASYNC : Alignment Synchronisation. + Idx:12; ID:10; I_TRACE_INFO : Trace Info.; INFO=0x0 { CC.0 } + Idx:17; ID:10; I_ADDR_L_64IS0 : Address, Long, 64 bit, IS0.; Addr=0x0000000000000000; + Idx:26; ID:10; I_TRACE_ON : Trace On. + Idx:27; ID:10; I_ADDR_CTXT_L_64IS0 : Address & Context, Long, 64 bit, IS0.; Addr=0x0000FFFFB6069140; Ctxt: AArch64,EL0, NS; + Idx:38; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE + Idx:39; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE + Idx:40; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEEEEEEEEEEEEEE + Idx:41; ID:10; I_ATOM_F6 : Atom format 6.; EEEEEEEEEEEN + ... + +If you see these above, then your system is tracing CoreSight data +correctly. + +To compile perf with CoreSight support in the tools/perf directory do:: + + make CORESIGHT=1 + +This requires OpenCSD to build. You may install distribution packages +for the support such as libopencsd and libopencsd-dev or download it +and build yourself. Upstream OpenCSD is located at: + + https://github.com/Linaro/OpenCSD + +For complete information on building perf with CoreSight support and +more extensive usage look at: + + https://github.com/Linaro/OpenCSD/blob/master/HOWTO.md + + +Kernel CoreSight Support +------------------------ + +You will also want CoreSight support enabled in your kernel config. +Ensure it is enabled with:: + + CONFIG_CORESIGHT=y + +There are various other CoreSight options you probably also want +enabled like:: + + CONFIG_CORESIGHT_LINKS_AND_SINKS=y + CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y + CONFIG_CORESIGHT_CATU=y + CONFIG_CORESIGHT_SINK_TPIU=y + CONFIG_CORESIGHT_SINK_ETBV10=y + CONFIG_CORESIGHT_SOURCE_ETM4X=y + CONFIG_CORESIGHT_CTI=y + CONFIG_CORESIGHT_CTI_INTEGRATION_REGS=y + +Please refer to the kernel configuration help for more information. + +Perf test - Verify kernel and userspace perf CoreSight work +----------------------------------------------------------- + +When you run perf test, it will do a lot of self tests. Some of those +tests will cover CoreSight (only if enabled and on ARM64). You +generally would run perf test from the tools/perf directory in the +kernel tree. Some tests will check some internal perf support like: + + Check Arm CoreSight trace data recording and synthesized samples + Check Arm SPE trace data recording and synthesized samples + +Some others will actually use perf record and some test binaries that +are in tests/shell/coresight and will collect traces to ensure a +minimum level of functionality is met. The scripts that launch these +tests are in the same directory. These will all look like: + + CoreSight / ASM Pure Loop + CoreSight / Memcpy 16k 10 Threads + CoreSight / Thread Loop 10 Threads - Check TID + etc. + +These perf record tests will not run if the tool binaries do not exist +in tests/shell/coresight/\*/ and will be skipped. If you do not have +CoreSight support in hardware then either do not build perf with +CoreSight support or remove these binaries in order to not have these +tests fail and have them skip instead. + +These tests will log historical results in the current working +directory (e.g. tools/perf) and will be named stats-\*.csv like: + + stats-asm_pure_loop-out.csv + stats-memcpy_thread-16k_10.csv + ... + +These statistic files log some aspects of the AUX data sections in +the perf data output counting some numbers of certain encodings (a +good way to know that it's working in a very simple way). One problem +with CoreSight is that given a large enough amount of data needing to +be logged, some of it can be lost due to the processor not waking up +in time to read out all the data from buffers etc.. You will notice +that the amount of data collected can vary a lot per run of perf test. +If you wish to see how this changes over time, simply run perf test +multiple times and all these csv files will have more and more data +appended to it that you can later examine, graph and otherwise use to +figure out if things have become worse or better. + +This means sometimes these tests fail as they don't capture all the +data needed. This is about tracking quality and amount of data +produced over time and to see when changes to the Linux kernel improve +quality of traces. + +Be aware that some of these tests take quite a while to run, specifically +in processing the perf data file and dumping contents to then examine what +is inside. + +You can change where these csv logs are stored by setting the +PERF_TEST_CORESIGHT_STATDIR environment variable before running perf +test like:: + + export PERF_TEST_CORESIGHT_STATDIR=/var/tmp + perf test + +They will also store resulting perf output data in the current +directory for later inspection like:: + + perf-asm_pure_loop-out.data + perf-memcpy_thread-16k_10.data + ... + +You can alter where the perf data files are stored by setting the +PERF_TEST_CORESIGHT_DATADIR environment variable such as:: + + PERF_TEST_CORESIGHT_DATADIR=/var/tmp + perf test + +You may wish to set these above environment variables if you wish to +keep the output of tests outside of the current working directory for +longer term storage and examination. diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index b37dc19e4d40950fb46d9016cb99f4b905271580..60bceb018d6a9372cd57aaee4e16e54dc6595e10 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -564,7 +564,7 @@ of ftrace. Here is a list of some of the key files: start:: - trace_fd = open("trace_marker", WR_ONLY); + trace_fd = open("trace_marker", O_WRONLY); Note: Writing into the trace_marker file can also initiate triggers that are written into /sys/kernel/tracing/events/ftrace/print/trigger diff --git a/Documentation/trace/hisi-ptt.rst b/Documentation/trace/hisi-ptt.rst new file mode 100644 index 0000000000000000000000000000000000000000..4f87d8e21065e2da06ec121053abd6f564c6daf3 --- /dev/null +++ b/Documentation/trace/hisi-ptt.rst @@ -0,0 +1,298 @@ +.. SPDX-License-Identifier: GPL-2.0 + +====================================== +HiSilicon PCIe Tune and Trace device +====================================== + +Introduction +============ + +HiSilicon PCIe tune and trace device (PTT) is a PCIe Root Complex +integrated Endpoint (RCiEP) device, providing the capability +to dynamically monitor and tune the PCIe link's events (tune), +and trace the TLP headers (trace). The two functions are independent, +but is recommended to use them together to analyze and enhance the +PCIe link's performance. + +On Kunpeng 930 SoC, the PCIe Root Complex is composed of several +PCIe cores. Each PCIe core includes several Root Ports and a PTT +RCiEP, like below. The PTT device is capable of tuning and +tracing the links of the PCIe core. +:: + + +--------------Core 0-------+ + | | [ PTT ] | + | | [Root Port]---[Endpoint] + | | [Root Port]---[Endpoint] + | | [Root Port]---[Endpoint] + Root Complex |------Core 1-------+ + | | [ PTT ] | + | | [Root Port]---[ Switch ]---[Endpoint] + | | [Root Port]---[Endpoint] `-[Endpoint] + | | [Root Port]---[Endpoint] + +---------------------------+ + +The PTT device driver registers one PMU device for each PTT device. +The name of each PTT device is composed of 'hisi_ptt' prefix with +the id of the SICL and the Core where it locates. The Kunpeng 930 +SoC encapsulates multiple CPU dies (SCCL, Super CPU Cluster) and +IO dies (SICL, Super I/O Cluster), where there's one PCIe Root +Complex for each SICL. +:: + + /sys/devices/hisi_ptt_ + +Tune +==== + +PTT tune is designed for monitoring and adjusting PCIe link parameters (events). +Currently we support events in 2 classes. The scope of the events +covers the PCIe core to which the PTT device belongs. + +Each event is presented as a file under $(PTT PMU dir)/tune, and +a simple open/read/write/close cycle will be used to tune the event. +:: + + $ cd /sys/devices/hisi_ptt_/tune + $ ls + qos_tx_cpl qos_tx_np qos_tx_p + tx_path_rx_req_alloc_buf_level + tx_path_tx_req_alloc_buf_level + $ cat qos_tx_dp + 1 + $ echo 2 > qos_tx_dp + $ cat qos_tx_dp + 2 + +Current value (numerical value) of the event can be simply read +from the file, and the desired value written to the file to tune. + +1. Tx Path QoS Control +------------------------ + +The following files are provided to tune the QoS of the tx path of +the PCIe core. + +- qos_tx_cpl: weight of Tx completion TLPs +- qos_tx_np: weight of Tx non-posted TLPs +- qos_tx_p: weight of Tx posted TLPs + +The weight influences the proportion of certain packets on the PCIe link. +For example, for the storage scenario, increase the proportion +of the completion packets on the link to enhance the performance as +more completions are consumed. + +The available tune data of these events is [0, 1, 2]. +Writing a negative value will return an error, and out of range +values will be converted to 2. Note that the event value just +indicates a probable level, but is not precise. + +2. Tx Path Buffer Control +------------------------- + +Following files are provided to tune the buffer of tx path of the PCIe core. + +- rx_alloc_buf_level: watermark of Rx requested +- tx_alloc_buf_level: watermark of Tx requested + +These events influence the watermark of the buffer allocated for each +type. Rx means the inbound while Tx means outbound. The packets will +be stored in the buffer first and then transmitted either when the +watermark reached or when timed out. For a busy direction, you should +increase the related buffer watermark to avoid frequently posting and +thus enhance the performance. In most cases just keep the default value. + +The available tune data of above events is [0, 1, 2]. +Writing a negative value will return an error, and out of range +values will be converted to 2. Note that the event value just +indicates a probable level, but is not precise. + +Trace +===== + +PTT trace is designed for dumping the TLP headers to the memory, which +can be used to analyze the transactions and usage condition of the PCIe +Link. You can choose to filter the traced headers by either Requester ID, +or those downstream of a set of Root Ports on the same core of the PTT +device. It's also supported to trace the headers of certain type and of +certain direction. + +You can use the perf command `perf record` to set the parameters, start +trace and get the data. It's also supported to decode the trace +data with `perf report`. The control parameters for trace is inputted +as event code for each events, which will be further illustrated later. +An example usage is like +:: + + $ perf record -e hisi_ptt0_2/filter=0x80001,type=1,direction=1, + format=1/ -- sleep 5 + +This will trace the TLP headers downstream root port 0000:00:10.1 (event +code for event 'filter' is 0x80001) with type of posted TLP requests, +direction of inbound and traced data format of 8DW. + +1. Filter +--------- + +The TLP headers to trace can be filtered by the Root Ports or the Requester ID +of the Endpoint, which are located on the same core of the PTT device. You can +set the filter by specifying the `filter` parameter which is required to start +the trace. The parameter value is 20 bit. Bit 19 indicates the filter type. +1 for Root Port filter and 0 for Requester filter. Bit[15:0] indicates the +filter value. The value for a Root Port is a mask of the core port id which is +calculated from its PCI Slot ID as (slotid & 7) * 2. The value for a Requester +is the Requester ID (Device ID of the PCIe function). Bit[18:16] is currently +reserved for extension. + +For example, if the desired filter is Endpoint function 0000:01:00.1 the filter +value will be 0x00101. If the desired filter is Root Port 0000:00:10.0 then +then filter value is calculated as 0x80001. + +Note that multiple Root Ports can be specified at one time, but only one +Endpoint function can be specified in one trace. Specifying both Root Port +and function at the same time is not supported. Driver maintains a list of +available filters and will check the invalid inputs. + +Currently the available filters are detected in driver's probe. If the supported +devices are removed/added after probe, you may need to reload the driver to update +the filters. + +2. Type +------- + +You can trace the TLP headers of certain types by specifying the `type` +parameter, which is required to start the trace. The parameter value is +8 bit. Current supported types and related values are shown below: + +- 8'b00000001: posted requests (P) +- 8'b00000010: non-posted requests (NP) +- 8'b00000100: completions (CPL) + +You can specify multiple types when tracing inbound TLP headers, but can only +specify one when tracing outbound TLP headers. + +3. Direction +------------ + +You can trace the TLP headers from certain direction, which is relative +to the Root Port or the PCIe core, by specifying the `direction` parameter. +This is optional and the default parameter is inbound. The parameter value +is 4 bit. When the desired format is 4DW, directions and related values +supported are shown below: + +- 4'b0000: inbound TLPs (P, NP, CPL) +- 4'b0001: outbound TLPs (P, NP, CPL) +- 4'b0010: outbound TLPs (P, NP, CPL) and inbound TLPs (P, NP, CPL B) +- 4'b0011: outbound TLPs (P, NP, CPL) and inbound TLPs (CPL A) + +When the desired format is 8DW, directions and related values supported are +shown below: + +- 4'b0000: reserved +- 4'b0001: outbound TLPs (P, NP, CPL) +- 4'b0010: inbound TLPs (P, NP, CPL B) +- 4'b0011: inbound TLPs (CPL A) + +Inbound completions are classified into two types: + +- completion A (CPL A): completion of CHI/DMA/Native non-posted requests, except for CPL B +- completion B (CPL B): completion of DMA remote2local and P2P non-posted requests + +4. Format +-------------- + +You can change the format of the traced TLP headers by specifying the +`format` parameter. The default format is 4DW. The parameter value is 4 bit. +Current supported formats and related values are shown below: + +- 4'b0000: 4DW length per TLP header +- 4'b0001: 8DW length per TLP header + +The traced TLP header format is different from the PCIe standard. + +When using the 8DW data format, the entire TLP header is logged +(Header DW0-3 shown below). For example, the TLP header for Memory +Reads with 64-bit addresses is shown in PCIe r5.0, Figure 2-17; +the header for Configuration Requests is shown in Figure 2.20, etc. + +In addition, 8DW trace buffer entries contain a timestamp and +possibly a prefix for a PASID TLP prefix (see Figure 6-20, PCIe r5.0). +Otherwise this field will be all 0. + +The bit[31:11] of DW0 is always 0x1fffff, which can be +used to distinguish the data format. 8DW format is like +:: + + bits [ 31:11 ][ 10:0 ] + |---------------------------------------|-------------------| + DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + DW1 [ Prefix ] + DW2 [ Header DW0 ] + DW3 [ Header DW1 ] + DW4 [ Header DW2 ] + DW5 [ Header DW3 ] + DW6 [ Reserved (0x0) ] + DW7 [ Time ] + +When using the 4DW data format, DW0 of the trace buffer entry +contains selected fields of DW0 of the TLP, together with a +timestamp. DW1-DW3 of the trace buffer entry contain DW1-DW3 +directly from the TLP header. + +4DW format is like +:: + + bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + |-----|---------|---|---|---|---|-------------|-------------| + DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + DW1 [ Header DW1 ] + DW2 [ Header DW2 ] + DW3 [ Header DW3 ] + +5. Memory Management +-------------------- + +The traced TLP headers will be written to the memory allocated +by the driver. The hardware accepts 4 DMA address with same size, +and writes the buffer sequentially like below. If DMA addr 3 is +finished and the trace is still on, it will return to addr 0. +:: + + +->[DMA addr 0]->[DMA addr 1]->[DMA addr 2]->[DMA addr 3]-+ + +---------------------------------------------------------+ + +Driver will allocate each DMA buffer of 4MiB. The finished buffer +will be copied to the perf AUX buffer allocated by the perf core. +Once the AUX buffer is full while the trace is still on, driver +will commit the AUX buffer first and then apply for a new one with +the same size. The size of AUX buffer is default to 16MiB. User can +adjust the size by specifying the `-m` parameter of the perf command. + +6. Decoding +----------- + +You can decode the traced data with `perf report -D` command (currently +only support to dump the raw trace data). The traced data will be decoded +according to the format described previously (take 8DW as an example): +:: + + [...perf headers and other information] + . ... HISI PTT data: size 4194304 bytes + . 00000000: 00 00 00 00 Prefix + . 00000004: 01 00 00 60 Header DW0 + . 00000008: 0f 1e 00 01 Header DW1 + . 0000000c: 04 00 00 00 Header DW2 + . 00000010: 40 00 81 02 Header DW3 + . 00000014: 33 c0 04 00 Time + . 00000020: 00 00 00 00 Prefix + . 00000024: 01 00 00 60 Header DW0 + . 00000028: 0f 1e 00 01 Header DW1 + . 0000002c: 04 00 00 00 Header DW2 + . 00000030: 40 00 81 02 Header DW3 + . 00000034: 02 00 00 00 Time + . 00000040: 00 00 00 00 Prefix + . 00000044: 01 00 00 60 Header DW0 + . 00000048: 0f 1e 00 01 Header DW1 + . 0000004c: 04 00 00 00 Header DW2 + . 00000050: 40 00 81 02 Header DW3 + [...] diff --git a/Documentation/trace/histogram.rst b/Documentation/trace/histogram.rst index 859fd1b76c638fff44ac70b6346243d8ced2d674..c1b685a38f6b4c75c27dbef236610d7a26c4ab2f 100644 --- a/Documentation/trace/histogram.rst +++ b/Documentation/trace/histogram.rst @@ -412,7 +412,7 @@ Extended error information Because the default sort key above is 'hitcount', the above shows a the list of call_sites by increasing hitcount, so that at the bottom we see the functions that made the most kmalloc calls during the - run. If instead we we wanted to see the top kmalloc callers in + run. If instead we wanted to see the top kmalloc callers in terms of the number of bytes requested rather than the number of calls, and we wanted the top caller to appear at the top, we can use the 'sort' parameter, along with the 'descending' modifier:: diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index 2d73e8697523e2651807772a2e7af4c1732ae58e..ea25a9220f9262a85416ed97f436ac966ebb8678 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -33,3 +33,4 @@ Linux Tracing Technologies coresight/index user_events rv/index + hisi-ptt diff --git a/Documentation/trace/kprobes.rst b/Documentation/trace/kprobes.rst index f318bceda1e67c244d2c7bd0e29cbcc140b5b1bb..48cf778a246802fc6ecd6365110931846fbf374c 100644 --- a/Documentation/trace/kprobes.rst +++ b/Documentation/trace/kprobes.rst @@ -328,8 +328,8 @@ Configuring Kprobes =================== When configuring the kernel using make menuconfig/xconfig/oldconfig, -ensure that CONFIG_KPROBES is set to "y". Under "General setup", look -for "Kprobes". +ensure that CONFIG_KPROBES is set to "y", look for "Kprobes" under +"General architecture-dependent options". So that you can load and unload Kprobes-based instrumentation modules, make sure "Loadable module support" (CONFIG_MODULES) and "Module diff --git a/Documentation/trace/timerlat-tracer.rst b/Documentation/trace/timerlat-tracer.rst index d643c95c01eb1684fbdb7b75a26ab506cf8e9348..db17df312bc87a1a1c041d6620e97863e5ca1758 100644 --- a/Documentation/trace/timerlat-tracer.rst +++ b/Documentation/trace/timerlat-tracer.rst @@ -20,7 +20,7 @@ For example:: [root@f32 ~]# cd /sys/kernel/tracing/ [root@f32 tracing]# echo timerlat > current_tracer -It is possible to follow the trace by reading the trace trace file:: +It is possible to follow the trace by reading the trace file:: [root@f32 tracing]# cat trace # tracer: timerlat diff --git a/Documentation/trace/user_events.rst b/Documentation/trace/user_events.rst index c180936f49fc8b2c964a1264831192e720085b15..9f181f342a703172a7148e3dd07c8f27b2c4166b 100644 --- a/Documentation/trace/user_events.rst +++ b/Documentation/trace/user_events.rst @@ -20,14 +20,14 @@ dynamic_events is the same as the ioctl with the u: prefix applied. Typically programs will register a set of events that they wish to expose to tools that can read trace_events (such as ftrace and perf). The registration -process gives back two ints to the program for each event. The first int is the -status index. This index describes which byte in the +process gives back two ints to the program for each event. The first int is +the status bit. This describes which bit in little-endian format in the /sys/kernel/debug/tracing/user_events_status file represents this event. The -second int is the write index. This index describes the data when a write() or +second int is the write index which describes the data when a write() or writev() is called on the /sys/kernel/debug/tracing/user_events_data file. -The structures referenced in this document are contained with the -/include/uap/linux/user_events.h file in the source tree. +The structures referenced in this document are contained within the +/include/uapi/linux/user_events.h file in the source tree. **NOTE:** *Both user_events_status and user_events_data are under the tracefs filesystem and may be mounted at different paths than above.* @@ -38,18 +38,18 @@ Registering within a user process is done via ioctl() out to the /sys/kernel/debug/tracing/user_events_data file. The command to issue is DIAG_IOCSREG. -This command takes a struct user_reg as an argument:: +This command takes a packed struct user_reg as an argument:: struct user_reg { u32 size; u64 name_args; - u32 status_index; + u32 status_bit; u32 write_index; }; The struct user_reg requires two inputs, the first is the size of the structure to ensure forward and backward compatibility. The second is the command string -to issue for registering. Upon success two outputs are set, the status index +to issue for registering. Upon success two outputs are set, the status bit and the write index. User based events show up under tracefs like any other event under the @@ -111,15 +111,56 @@ in realtime. This allows user programs to only incur the cost of the write() or writev() calls when something is actively attached to the event. User programs call mmap() on /sys/kernel/debug/tracing/user_events_status to -check the status for each event that is registered. The byte to check in the -file is given back after the register ioctl() via user_reg.status_index. +check the status for each event that is registered. The bit to check in the +file is given back after the register ioctl() via user_reg.status_bit. The bit +is always in little-endian format. Programs can check if the bit is set either +using a byte-wise index with a mask or a long-wise index with a little-endian +mask. + Currently the size of user_events_status is a single page, however, custom kernel configurations can change this size to allow more user based events. In all cases the size of the file is a multiple of a page size. -For example, if the register ioctl() gives back a status_index of 3 you would -check byte 3 of the returned mmap data to see if anything is attached to that -event. +For example, if the register ioctl() gives back a status_bit of 3 you would +check byte 0 (3 / 8) of the returned mmap data and then AND the result with 8 +(1 << (3 % 8)) to see if anything is attached to that event. + +A byte-wise index check is performed as follows:: + + int index, mask; + char *status_page; + + index = status_bit / 8; + mask = 1 << (status_bit % 8); + + ... + + if (status_page[index] & mask) { + /* Enabled */ + } + +A long-wise index check is performed as follows:: + + #include + #include + + #if __BITS_PER_LONG == 64 + #define endian_swap(x) htole64(x) + #else + #define endian_swap(x) htole32(x) + #endif + + long index, mask, *status_page; + + index = status_bit / __BITS_PER_LONG; + mask = 1L << (status_bit % __BITS_PER_LONG); + mask = endian_swap(mask); + + ... + + if (status_page[index] & mask) { + /* Enabled */ + } Administrators can easily check the status of all registered events by reading the user_events_status file directly via a terminal. The output is as follows:: @@ -137,7 +178,7 @@ For example, on a system that has a single event the output looks like this:: Active: 1 Busy: 0 - Max: 4096 + Max: 32768 If a user enables the user event via ftrace, the output would change to this:: @@ -145,21 +186,10 @@ If a user enables the user event via ftrace, the output would change to this:: Active: 1 Busy: 1 - Max: 4096 - -**NOTE:** *A status index of 0 will never be returned. This allows user -programs to have an index that can be used on error cases.* - -Status Bits -^^^^^^^^^^^ -The byte being checked will be non-zero if anything is attached. Programs can -check specific bits in the byte to see what mechanism has been attached. - -The following values are defined to aid in checking what has been attached: - -**EVENT_STATUS_FTRACE** - Bit set if ftrace has been attached (Bit 0). + Max: 32768 -**EVENT_STATUS_PERF** - Bit set if perf has been attached (Bit 1). +**NOTE:** *A status bit of 0 will never be returned. This allows user programs +to have a bit that can be used on error cases.* Writing Data ------------ diff --git a/Documentation/translations/it_IT/process/howto.rst b/Documentation/translations/it_IT/process/howto.rst index 16ad5622d5495dd1899367d4f16c41aab04466cc..15c08aea1dfea27ef92170ac485b2c11b6bc6b2e 100644 --- a/Documentation/translations/it_IT/process/howto.rst +++ b/Documentation/translations/it_IT/process/howto.rst @@ -394,7 +394,7 @@ trovati al sito: Ci sono diversi archivi della lista di discussione. Usate un qualsiasi motore di ricerca per trovarli. Per esempio: - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ É caldamente consigliata una ricerca in questi archivi sul tema che volete sollevare, prima di pubblicarlo sulla lista. Molte cose sono già state diff --git a/Documentation/translations/it_IT/process/magic-number.rst b/Documentation/translations/it_IT/process/magic-number.rst index f452fafb1e84cce85fe65664dab168c9f9090872..02eb7eb2448e77af709eb02ffd370ba6da6ba284 100644 --- a/Documentation/translations/it_IT/process/magic-number.rst +++ b/Documentation/translations/it_IT/process/magic-number.rst @@ -75,87 +75,17 @@ Registro dei cambiamenti:: Nome magico Numero Struttura File ===================== ================ ======================== ========================================== PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/pg.h`` -CMAGIC 0x0111 user ``include/linux/a.out.h`` -MKISS_DRIVER_MAGIC 0x04bf mkiss_channel ``drivers/net/mkiss.h`` -HDLC_MAGIC 0x239e n_hdlc ``drivers/char/n_hdlc.c`` APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` -DB_MAGIC 0x4442 fc_info ``drivers/net/iph5526_novram.c`` -DL_MAGIC 0x444d fc_info ``drivers/net/iph5526_novram.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` -FF_MAGIC 0x4646 fc_info ``drivers/net/iph5526_novram.c`` -PTY_MAGIC 0x5001 ``drivers/char/pty.c`` -PPP_MAGIC 0x5002 ppp ``include/linux/if_pppvar.h`` -SSTATE_MAGIC 0x5302 serial_state ``include/linux/serial.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -STRIP_MAGIC 0x5303 strip ``drivers/net/strip.c`` -SIXPACK_MAGIC 0x5304 sixpack ``drivers/net/hamradio/6pack.h`` -AX25_MAGIC 0x5316 ax_disp ``drivers/net/mkiss.h`` -TTY_MAGIC 0x5401 tty_struct ``include/linux/tty.h`` -MGSL_MAGIC 0x5401 mgsl_info ``drivers/char/synclink.c`` -TTY_DRIVER_MAGIC 0x5402 tty_driver ``include/linux/tty_driver.h`` MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` -USB_SERIAL_MAGIC 0x6702 usb_serial ``drivers/usb/serial/usb-serial.h`` -FULL_DUPLEX_MAGIC 0x6969 ``drivers/net/ethernet/dec/tulip/de2104x.c`` -USB_BLUETOOTH_MAGIC 0x6d02 usb_bluetooth ``drivers/usb/class/bluetty.c`` -RFCOMM_TTY_MAGIC 0x6d02 ``net/bluetooth/rfcomm/tty.c`` -USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/serial/usb-serial.h`` -CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` -LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` -NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` -RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` -ISDN_X25IFACE_MAGIC 0x1e75a2b9 isdn_x25iface_proto_data ``drivers/isdn/isdn_x25iface.h`` -ECP_MAGIC 0x21504345 cdkecpsig ``include/linux/cdk.h`` -LSOMAGIC 0x27091997 lso ``drivers/fc4/fc.c`` -LSMAGIC 0x2a3b4d2a ls ``drivers/fc4/fc.c`` -WANPIPE_MAGIC 0x414C4453 sdla_{dump,exec} ``include/linux/wanpipe.h`` -CS_CARD_MAGIC 0x43525553 cs_card ``sound/oss/cs46xx.c`` -LABELCL_MAGIC 0x4857434c labelcl_info_s ``include/asm/ia64/sn/labelcl.h`` -ISDN_ASYNC_MAGIC 0x49344C01 modem_info ``include/linux/isdn.h`` -CTC_ASYNC_MAGIC 0x49344C01 ctc_tty_info ``drivers/s390/net/ctctty.c`` -ISDN_NET_MAGIC 0x49344C02 isdn_net_local_s ``drivers/isdn/i4l/isdn_net_lib.h`` -SAVEKMSG_MAGIC2 0x4B4D5347 savekmsg ``arch/*/amiga/config.c`` -CS_STATE_MAGIC 0x4c4f4749 cs_state ``sound/oss/cs46xx.c`` -SLAB_C_MAGIC 0x4f17a36d kmem_cache ``mm/slab.c`` -COW_MAGIC 0x4f4f4f4d cow_header_v1 ``arch/um/drivers/ubd_user.c`` -I810_CARD_MAGIC 0x5072696E i810_card ``sound/oss/i810_audio.c`` -TRIDENT_CARD_MAGIC 0x5072696E trident_card ``sound/oss/trident.c`` -ROUTER_MAGIC 0x524d4157 wan_device [in ``wanrouter.h`` pre 3.9] -SAVEKMSG_MAGIC1 0x53415645 savekmsg ``arch/*/amiga/config.c`` -GDA_MAGIC 0x58464552 gda ``arch/mips/include/asm/sn/gda.h`` -RED_MAGIC1 0x5a2cf071 (any) ``mm/slab.c`` -EEPROM_MAGIC_VALUE 0x5ab478d2 lanai_dev ``drivers/atm/lanai.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` -PCXX_MAGIC 0x5c6df104 channel ``drivers/char/pcxx.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` -I810_STATE_MAGIC 0x63657373 i810_state ``sound/oss/i810_audio.c`` -TRIDENT_STATE_MAGIC 0x63657373 trient_state ``sound/oss/trident.c`` -M3_CARD_MAGIC 0x646e6f50 m3_card ``sound/oss/maestro3.c`` -FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fore200e.h`` -SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` -SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` -LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` -VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` -KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` -PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/media/pwc.h`` -NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` -ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` QUEUE_MAGIC_USED 0xf7e1cc33 queue_entry ``drivers/scsi/arm/queue.c`` -HTB_CMAGIC 0xFEFAFEF1 htb_class ``net/sched/sch_htb.c`` NMI_MAGIC 0x48414d4d455201 nmi_s ``arch/mips/include/asm/sn/nmi.h`` ===================== ================ ======================== ========================================== - -Da notare che ci sono anche dei numeri magici specifici per driver nel -*sound memory management*. Consultate ``include/sound/sndmagic.h`` per una -lista completa. Molti driver audio OSS hanno i loro numeri magici costruiti a -partire dall'identificativo PCI della scheda audio - nemmeno questi sono -elencati in questo file. - -Il file-system HFS è un altro grande utilizzatore di numeri magici - potete -trovarli qui ``fs/hfs/hfs.h``. diff --git a/Documentation/translations/ja_JP/SubmittingPatches b/Documentation/translations/ja_JP/SubmittingPatches index 66ce0d8b05264e249eed0bb9acdbffdce6e974ba..04deb77b20c6fc16998f8707e1cd3edc21f691be 100644 --- a/Documentation/translations/ja_JP/SubmittingPatches +++ b/Documentation/translations/ja_JP/SubmittingPatches @@ -35,8 +35,7 @@ Linux カーネルに変更を加えたいと思っている個人又は会社 てもらえやすくする提案を集めたものです。 コードを投稿する前に、Documentation/process/submit-checklist.rst の項目リストに目 -を通してチェックしてください。もしあなたがドライバーを投稿しようとし -ているなら、Documentation/process/submitting-drivers.rst にも目を通してください。 +を通してチェックしてください。 -------------------------------------------- セクション1 パッチの作り方と送り方 diff --git a/Documentation/translations/ja_JP/howto.rst b/Documentation/translations/ja_JP/howto.rst index 649e2ff2a407e1a9761155ae286de2ed6e10e23b..b47a682d8dedcc81a2466d622995cfd917ad1437 100644 --- a/Documentation/translations/ja_JP/howto.rst +++ b/Documentation/translations/ja_JP/howto.rst @@ -410,7 +410,7 @@ https://bugzilla.kernel.org に行ってください。もし今後のバグレ このメーリングリストのアーカイブは web 上の多数の場所に存在します。こ れらのアーカイブを探すにはサーチエンジンを使いましょう。例えば- - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ リストに投稿する前にすでにその話題がアーカイブに存在するかどうかを検索 することを是非やってください。多数の事がすでに詳細に渡って議論されてお diff --git a/Documentation/translations/ko_KR/howto.rst b/Documentation/translations/ko_KR/howto.rst index e43970584ca4d93a0bd154c0fd6e5ca1d7d8e537..df53fafd1b10ade31b54615e0708328e7089b747 100644 --- a/Documentation/translations/ko_KR/howto.rst +++ b/Documentation/translations/ko_KR/howto.rst @@ -386,7 +386,7 @@ https://bugzilla.kernel.org 를 체크하고자 할 수도 있다; 소수의 커 웹상의 많은 다른 곳에도 메일링 리스트의 아카이브들이 있다. 이러한 아카이브들을 찾으려면 검색 엔진을 사용하라. 예를 들어: - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ 여러분이 새로운 문제에 관해 리스트에 올리기 전에 말하고 싶은 주제에 관한 것을 아카이브에서 먼저 찾아보기를 강력히 권장한다. 이미 상세하게 토론된 많은 diff --git a/Documentation/translations/zh_CN/IRQ.txt b/Documentation/translations/zh_CN/IRQ.txt deleted file mode 100644 index 9aec8dca4fcf18b46c5c0fedd9cbb2bf5878606e..0000000000000000000000000000000000000000 --- a/Documentation/translations/zh_CN/IRQ.txt +++ /dev/null @@ -1,39 +0,0 @@ -Chinese translated version of Documentation/core-api/irq/index.rst - -If you have any comment or update to the content, please contact the -original document maintainer directly. However, if you have a problem -communicating in English you can also ask the Chinese maintainer for -help. Contact the Chinese maintainer if this translation is outdated -or if there is a problem with the translation. - -Maintainer: Eric W. Biederman -Chinese maintainer: Fu Wei ---------------------------------------------------------------------- -Documentation/core-api/irq/index.rst 的中文翻译 - -如果想评论或更新本文的内容，请直接联系原文档的维护者。如果你使用英文 -交流有困难的话，也可以向中文版维护者求助。如果本翻译更新不及时或者翻 -译存在问题，请联系中文版维护者。 -英文版维护者： Eric W. Biederman -中文版维护者： 傅炜 Fu Wei -中文版翻译者： 傅炜 Fu Wei -中文版校译者： 傅炜 Fu Wei - - -以下为正文 ---------------------------------------------------------------------- -何为 IRQ? - -一个 IRQ 是来自某个设备的一个中断请求。目前，它们可以来自一个硬件引脚， -或来自一个数据包。多个设备可能连接到同个硬件引脚，从而共享一个 IRQ。 - -一个 IRQ 编号是用于告知硬件中断源的内核标识。通常情况下，这是一个 -全局 irq_desc 数组的索引，但是除了在 linux/interrupt.h 中的实现， -具体的细节是体系结构特定的。 - -一个 IRQ 编号是设备上某个可能的中断源的枚举。通常情况下，枚举的编号是 -该引脚在系统内中断控制器的所有输入引脚中的编号。对于 ISA 总线中的情况， -枚举的是在两个 i8259 中断控制器中 16 个输入引脚。 - -架构可以对 IRQ 编号指定额外的含义，在硬件涉及任何手工配置的情况下， -是被提倡的。ISA 的 IRQ 是一个分配这类额外含义的典型例子。 diff --git a/Documentation/translations/zh_CN/PCI/acpi-info.rst b/Documentation/translations/zh_CN/PCI/acpi-info.rst new file mode 100644 index 0000000000000000000000000000000000000000..a35f39dcd8584ee581169e6c7051356ffc158b92 --- /dev/null +++ b/Documentation/translations/zh_CN/PCI/acpi-info.rst @@ -0,0 +1,139 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/PCI/acpi-info.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +===================== +PCI主桥的ACPI注意事项 +===================== + +一般的规则是，ACPI命名空间应该描述操作系统可能使用的所有东西，除非有其他方法让操作系 +统找到它[1, 2]。 + +例如，没有标准的硬件机制来枚举PCI主桥，所以ACPI命名空间必须描述每个主桥、访问它 +下面的PCI配置空间的方法、主桥转发到PCI的地址空间窗口（使用_CRS）以及传统的INTx +中断的路由（使用_PRT）。 + +在主桥下面的PCI设备，通常不需要通过ACPI描述。操作系统可以通过标准的PCI枚举机制来 +发现它们，使用配置访问来发现和识别设备，并读取和测量它们的BAR。然而，如果ACPI为它们 +提供电源管理或热插拔功能，或者如果设备有由平台中断控制器连接的INTx中断，需要一个_PRT +来描述这些连接，这种情况下ACPI可以描述PCI设备。 + +ACPI资源描述是通过ACPI命名空间中设备的_CRS对象完成的[2]。_CRS就像一个通用的PCI BAR： +操作系统可以读取_CRS并找出正在消耗的资源，即使它没有该设备的驱动程序[3]。这一点很重要， +因为它意味着一个旧的操作系统可以正确地工作，即使是在操作系统不知道的新设备的系统上。新设 +备可能什么都不做，但操作系统至少可以确保没有资源与它们冲突。 + +像MCFG、HPET、ECDT等静态表，不是保留地址空间的机制。静态表是在操作系统在启动初期且在它 +能够解析ACPI命名空间之前需要知道的东西。如果定义了一个新的表，即使旧的操作系统忽略了这 +个表，它也需要正常运行。_CRS允许这样做，因为它是通用的，可以被旧的操作系统解析；而静态表 +则不允许。 + +如果操作系统要管理一个通过ACPI描述的不可发现的设备，该设备将有一个特定的_HID/_CID，以 +告诉操作系统与之绑定的驱动程序，并且_CRS告诉操作系统和驱动程序该设备的寄存器在哪里。 + +PCI主桥是PNP0A03或PNP0A08设备。它们的_CRS应该描述它们所消耗的所有地址空间。这包括它 +们转发到PCI总线上的所有窗口，以及不转发到PCI的主桥本身的寄存器。主桥的寄存器包括次要/下 +级总线寄存器，决定了桥下面的总线范围，窗口寄存器描述了桥洞，等等。这些都是设备相关的，非 +架构相关的东西，所以PNP0A03/PNP0A08驱动可以管理它们的唯一方法是通过_PRS/_CRS/_SRS， +它包含了特定于设备的细节。主桥寄存器也包括ECAM空间，因为它是由主桥消耗的。 + +ACPI定义了一个Consumer/Producer位来区分桥寄存器（“Consumer”下文译作消费者）和 +桥洞（“Producer”下文译作生产者）[4, 5]，但是早期的BIOS没有正确使用这个位。其结果 +是，目前的ACPI规范只为扩展地址空间描述符定义了消费者/生产者；在旧的QWord/Word/Word地 +址空间描述符中，该位应该被忽略。因此，操作系统必须假定所有的QWord/Word/Word描述符都是 +窗口。 + +在增加扩展地址空间描述符之前，消费者/生产者的失败意味着没有办法描述PNP0A03/PNP0A08设 +备本身的桥寄存器。解决办法是在PNP0C02捕捉器中描述桥寄存器（包括ECAM空间）[6]。 +除了ECAM之外，桥寄存器空间反正是特定于设备的，所以通用的PNP0A03/PNP0A08驱动程 +序(pci_root.c)没有必要了解它。 + +新的架构应该能够在PNP0A03设备中使用“消费者”扩展地址空间描述符，用于桥寄存器，包括 +ECAM，尽管对[6]的严格解释可能禁止这样做。旧的x86和ia64内核假定所有的地址空间描述 +符，包括“消费者”扩展地址空间的描述符，都是窗口，所以在这些架构上以这种方式描述桥寄 +存器是不安全的。 + +PNP0C02“主板”设备基本上是万能的。除了“不要将这些资源用于其他用途”之外，没有其他的编 +程模型。因此，PNP0C02 _CRS应该声明ACPI命名空间中(1)没有被_CRS声明的任何其他设备对 +象的地址空间，(2)不应该被OS分配给其他东西。 + +除非有一个标准的固件接口用于配置访问，例如ia64 SAL接口[7]，否则PCIe规范要求使用增强 +型配置访问方法（ECAM）。主桥消耗ECAM内存地址空间并将内存访问转换为PCI配置访问。该规范 +定义了ECAM地址空间的布局和功能；只有地址空间的基础是特定于设备的。ACPI操作系统从静态 +MCFG表或PNP0A03设备中的_CBA方法中了解基础地址。 + +MCFG表必须描述非热插拔主桥的ECAM空间[8]。由于MCFG是一个静态表，不能通过热插拔更新， +PNP0A03设备中的_CBA方法描述了可热插拔主桥的ECAM空间[9]。请注意，对于MCFG和_CBA， +基址总是对应于总线0，即使桥器下面的总线范围（通过_CRS报告）不从0开始。 + + +[1] ACPI 6.2, sec 6.1: + 对于任何在非枚举类型的总线上的设备（例如，ISA总线），OSPM会枚举设备的标识符，ACPI + 系统固件必须为每个设备提供一个_HID对象...以使OSPM能够做到这一点。 + +[2] ACPI 6.2, sec 3.7: + 操作系统枚举主板设备时，只需通过读取ACPI命名空间来寻找具有硬件ID的设备。 + + ACPI枚举的每个设备都包括ACPI命名空间中ACPI定义的对象，该对象报告设备可能占用的硬 + 件资源[_PRS]，报告设备当前使用的资源[_CRS]的对象，以及配置这些资源的对象[_SRS]。 + 这些信息被即插即用操作系统（OSPM）用来配置设备。 + +[3] ACPI 6.2, sec 6.2: + OSPM使用设备配置对象来配置通过ACPI列举的设备的硬件资源。设备配置对象提供了关于当前 + 和可能的资源需求的信息，共享资源之间的关系，以及配置硬件资源的方法。 + + 当OSPM枚举一个设备时，它调用_PRS来确定该设备的资源需求。它也可以调用_CRS来找到该设 + 备的当前资源设置。利用这些信息，即插即用系统决定设备应该消耗什么资源，并通过调用设备 + 的_SRS控制方法来设置这些资源。 + + 在ACPI中，设备可以消耗资源（例如，传统的键盘），提供资源（例如，一个专有的PCI桥）， + 或者两者都做。除非另有规定，设备的资源被假定为来自设备层次结构中设备上方最近的匹配资 + 源。 + +[4] ACPI 6.2, sec 6.4.3.5.1, 2, 3, 4: + QWord/DWord/Word 地址空间描述符 (.1, .2, .3) + 常规标志: Bit [0] 被忽略。 + + 扩展地址空间描述符 (.4) + 常规标志: Bit [0] 消费者/生产者: + + * 1 – 这个设备消费这个资源 + * 0 – 该设备生产和消费该资源 + +[5] ACPI 6.2, sec 19.6.43: + ResourceUsage指定内存范围是由这个设备（ResourceConsumer）消费还是传递给子设备 + （ResourceProducer）。如果没有指定，那么就假定是ResourceConsumer。 + +[6] PCI Firmware 3.2, sec 4.1.2: + 如果操作系统不能原生的懂得保留MMCFG区域，MMCFG区域必须由固件保留。在MCFG表中或通 + 过_CBA方法（见第4.1.3节）报告的地址范围必须通过声明主板资源来保留。对于大多数系统， + 主板资源将出现在ACPI命名空间的根部（在_SB下），在一个节点的_HID为EISAID（PNP0C0 + 2），在这种情况下的资源不应该要求在根PCI总线的_CRS。这些资源可以选择在Int15 E820 + 或EFIGetMemoryMap中作为保留内存返回，但必须始终通过ACPI作为主板资源报告。 + +[7] PCI Express 4.0, sec 7.2.2: + 对于PC兼容的系统，或者没有实现允许访问配置空间的处理器架构特定固件接口标准的系统，需 + 要使用本节中定义的ECAM。 + +[8] PCI Firmware 3.2, sec 4.1.2: + MCFG表是一个ACPI表，用于沟通的基础地址对应的非热的可移动的PCI段组范围内的PCI段组在 + 启动时提供给操作系统。这对PC兼容系统来说是必需的。 + + MCFG表仅用于沟通在启动时系统可用的PCI段组对应的基址。 + +[9] PCI Firmware 3.2, sec 4.1.3: + _CBA (Memory mapped Configuration Base Address) 控制方法是一个可选的ACPI对 + 象，用于返回热插拔主桥的64位内存映射的配置基址。_CBA 返回的基址是与处理器相关的地址。 + _CBA 控制方法被评估为一个整数。 + + 这个控制方法出现在主桥对象下。当_CBA方法出现在一个活动的主桥对象下时，操作系统会评 + 估这个结构，以确定内存映射的配置基址，对应于_CRS方法中指定的总线编号范围的PCI段组。 + 一个包含_CBA方法的ACPI命名空间对象也必须包含一个相应的_SEG方法。 diff --git a/Documentation/translations/zh_CN/PCI/index.rst b/Documentation/translations/zh_CN/PCI/index.rst index 16acb2bd9b58d945ba93f33ae2f3f0922d8c6fc6..cbeb33c34a987c7c97091b1598f2b28cc5917893 100644 --- a/Documentation/translations/zh_CN/PCI/index.rst +++ b/Documentation/translations/zh_CN/PCI/index.rst @@ -10,9 +10,6 @@ :校译: - -.. _cn_PCI_index.rst: - =================== Linux PCI总线子系统 =================== @@ -26,12 +23,12 @@ Linux PCI总线子系统 pci-iov-howto msi-howto sysfs-pci + acpi-info Todolist: - acpi-info - pci-error-recovery - pcieaer-howto - endpoint/index - boot-interrupts +* pci-error-recovery +* pcieaer-howto +* endpoint/index +* boot-interrupts diff --git a/Documentation/translations/zh_CN/admin-guide/README.rst b/Documentation/translations/zh_CN/admin-guide/README.rst index d20949e8bf6fdb3511c48ca15d9d6fc58bc56109..e679cbc3c89d069e35ad766a6297094d124ca5ff 100644 --- a/Documentation/translations/zh_CN/admin-guide/README.rst +++ b/Documentation/translations/zh_CN/admin-guide/README.rst @@ -6,10 +6,10 @@ 吴想成 Wu XiangCheng -Linux内核5.x版本 +Linux内核6.x版本 ========================================= -以下是Linux版本5的发行注记。仔细阅读它们， +以下是Linux版本6的发行注记。仔细阅读它们， 它们会告诉你这些都是什么，解释如何安装内核，以及遇到问题时该如何做。 什么是Linux？ @@ -61,27 +61,27 @@ Linux内核5.x版本 - 如果您要安装完整的源代码，请把内核tar档案包放在您有权限的目录中（例如您 的主目录）并将其解包:: - xz -cd linux-5.x.tar.xz | tar xvf - + xz -cd linux-6.x.tar.xz | tar xvf - 将“X”替换成最新内核的版本号。 【不要】使用 /usr/src/linux 目录！这里有一组库头文件使用的内核头文件 （通常是不完整的）。它们应该与库匹配，而不是被内核的变化搞得一团糟。 - - 您还可以通过打补丁在5.x版本之间升级。补丁以xz格式分发。要通过打补丁进行 - 安装，请获取所有较新的补丁文件，进入内核源代码（linux-5.x）的目录并 + - 您还可以通过打补丁在6.x版本之间升级。补丁以xz格式分发。要通过打补丁进行 + 安装，请获取所有较新的补丁文件，进入内核源代码（linux-6.x）的目录并 执行:: - xz -cd ../patch-5.x.xz | patch -p1 + xz -cd ../patch-6.x.xz | patch -p1 请【按顺序】替换所有大于当前源代码树版本的“x”，这样就可以了。您可能想要 删除备份文件（文件名类似xxx~ 或 xxx.orig)，并确保没有失败的补丁（文件名 类似xxx# 或 xxx.rej）。如果有，不是你就是我犯了错误。 - 与5.x内核的补丁不同，5.x.y内核（也称为稳定版内核）的补丁不是增量的，而是 - 直接应用于基本的5.x内核。例如，如果您的基本内核是5.0，并且希望应用5.0.3 - 补丁，则不应先应用5.0.1和5.0.2的补丁。类似地，如果您运行的是5.0.2内核， - 并且希望跳转到5.0.3，那么在应用5.0.3补丁之前，必须首先撤销5.0.2补丁 + 与6.x内核的补丁不同，6.x.y内核（也称为稳定版内核）的补丁不是增量的，而是 + 直接应用于基本的6.x内核。例如，如果您的基本内核是6.0，并且希望应用6.0.3 + 补丁，则不应先应用6.0.1和6.0.2的补丁。类似地，如果您运行的是6.0.2内核， + 并且希望跳转到6.0.3，那么在应用6.0.3补丁之前，必须首先撤销6.0.2补丁 （即patch -R）。更多关于这方面的内容，请阅读 :ref:`Documentation/process/applying-patches.rst ` 。 @@ -103,7 +103,7 @@ Linux内核5.x版本 软件要求 --------- - 编译和运行5.x内核需要各种软件包的最新版本。请参考 + 编译和运行6.x内核需要各种软件包的最新版本。请参考 :ref:`Documentation/process/changes.rst ` 来了解最低版本要求以及如何升级软件包。请注意，使用过旧版本的这些包可能会 导致很难追踪的间接错误，因此不要以为在生成或操作过程中出现明显问题时可以 @@ -116,12 +116,12 @@ Linux内核5.x版本 ``make O=output/dir`` 选项可以为输出文件（包括 .config）指定备用位置。 例如:: - kernel source code: /usr/src/linux-5.x + kernel source code: /usr/src/linux-6.x build directory: /home/name/build/kernel 要配置和构建内核，请使用:: - cd /usr/src/linux-5.x + cd /usr/src/linux-6.x make O=/home/name/build/kernel menuconfig make O=/home/name/build/kernel sudo make O=/home/name/build/kernel modules_install install @@ -227,8 +227,6 @@ Linux内核5.x版本 - 确保您至少有gcc 5.1可用。 有关更多信息，请参阅 :ref:`Documentation/process/changes.rst ` 。 - 请注意，您仍然可以使用此内核运行a.out用户程序。 - - 执行 ``make`` 来创建压缩内核映像。如果您安装了lilo以适配内核makefile， 那么也可以进行 ``make install`` ，但是您可能需要先检查特定的lilo设置。 @@ -282,67 +280,12 @@ Linux内核5.x版本 若遇到问题 ----------- - - 如果您发现了一些可能由于内核缺陷所导致的问题，请检查MAINTAINERS（维护者） - 文件看看是否有人与令您遇到麻烦的内核部分相关。如果无人在此列出，那么第二 - 个最好的方案就是把它们发给我（torvalds@linux-foundation.org），也可能发送 - 到任何其他相关的邮件列表或新闻组。 - - - 在所有的缺陷报告中，【请】告诉我们您在说什么内核，如何复现问题，以及您的 - 设置是什么的（使用您的常识）。如果问题是新的，请告诉我；如果问题是旧的， - 请尝试告诉我您什么时候首次注意到它。 - - - 如果缺陷导致如下消息:: - - unable to handle kernel paging request at address C0000010 - Oops: 0002 - EIP: 0010:XXXXXXXX - eax: xxxxxxxx ebx: xxxxxxxx ecx: xxxxxxxx edx: xxxxxxxx - esi: xxxxxxxx edi: xxxxxxxx ebp: xxxxxxxx - ds: xxxx es: xxxx fs: xxxx gs: xxxx - Pid: xx, process nr: xx - xx xx xx xx xx xx xx xx xx xx - - 或者类似的内核调试信息显示在屏幕上或在系统日志里，请【如实】复制它。 - 可能对你来说转储（dump）看起来不可理解，但它确实包含可能有助于调试问题的 - 信息。转储上方的文本也很重要：它说明了内核转储代码的原因（在上面的示例中， - 是由于内核指针错误）。更多关于如何理解转储的信息，请参见 - Documentation/admin-guide/bug-hunting.rst。 - - - 如果使用 CONFIG_KALLSYMS 编译内核，则可以按原样发送转储，否则必须使用 - ``ksymoops`` 程序来理解转储（但通常首选使用CONFIG_KALLSYMS编译）。 - 此实用程序可从 - https://www.kernel.org/pub/linux/utils/kernel/ksymoops/ 下载。 - 或者，您可以手动执行转储查找： - - - 在调试像上面这样的转储时，如果您可以查找EIP值的含义，这将非常有帮助。 - 十六进制值本身对我或其他任何人都没有太大帮助：它会取决于特定的内核设置。 - 您应该做的是从EIP行获取十六进制值（忽略 ``0010:`` ），然后在内核名字列表 - 中查找它，以查看哪个内核函数包含有问题的地址。 - - 要找到内核函数名，您需要找到与显示症状的内核相关联的系统二进制文件。就是 - 文件“linux/vmlinux”。要提取名字列表并将其与内核崩溃中的EIP进行匹配， - 请执行:: - - nm vmlinux | sort | less - - 这将为您提供一个按升序排序的内核地址列表，从中很容易找到包含有问题的地址 - 的函数。请注意，内核调试消息提供的地址不一定与函数地址完全匹配（事实上， - 这是不可能的），因此您不能只“grep”列表：不过列表将为您提供每个内核函数 - 的起点，因此通过查找起始地址低于你正在搜索的地址，但后一个函数的高于的 - 函数，你会找到您想要的。实际上，在您的问题报告中加入一些“上下文”可能是 - 一个好主意，给出相关的上下几行。 - - 如果您由于某些原因无法完成上述操作（如您使用预编译的内核映像或类似的映像）， - 请尽可能多地告诉我您的相关设置信息，这会有所帮助。有关详细信息请阅读 - ‘Documentation/admin-guide/reporting-issues.rst’。 - - - 或者，您可以在正在运行的内核上使用gdb（只读的；即不能更改值或设置断点）。 - 为此，请首先使用-g编译内核；适当地编辑arch/x86/Makefile，然后执行 ``make - clean`` 。您还需要启用CONFIG_PROC_FS（通过 ``make config`` ）。 - - 使用新内核重新启动后，执行 ``gdb vmlinux /proc/kcore`` 。现在可以使用所有 - 普通的gdb命令。查找系统崩溃点的命令是 ``l *0xXXXXXXXX`` （将xxx替换为EIP - 值）。 - - 用gdb无法调试一个当前未运行的内核是由于gdb（错误地）忽略了编译内核的起始 - 偏移量。 +如果您发现了一些可能由于内核缺陷所导致的问题，请参阅： +Documentation/translations/zh_CN/admin-guide/reporting-issues.rst 。 + +想要理解内核错误报告，请参阅： +Documentation/translations/zh_CN/admin-guide/bug-hunting.rst 。 + +更多用GDB调试内核的信息，请参阅： +Documentation/translations/zh_CN/dev-tools/gdb-kernel-debugging.rst +和 Documentation/dev-tools/kgdb.rst 。 diff --git a/Documentation/translations/zh_CN/admin-guide/bootconfig.rst b/Documentation/translations/zh_CN/admin-guide/bootconfig.rst new file mode 100644 index 0000000000000000000000000000000000000000..072d17f5f199b17acc01cfd8e4238c349f18746f --- /dev/null +++ b/Documentation/translations/zh_CN/admin-guide/bootconfig.rst @@ -0,0 +1,293 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/admin-guide/bootconfig.rst + +:译者: 吴想成 Wu XiangCheng + +======== +引导配置 +======== + +:作者: Masami Hiramatsu + +概述 +==== + +引导配置扩展了现有的内核命令行，以一种更有效率的方式在引导内核时进一步支持 +键值数据。这允许管理员传递一份结构化关键字的配置文件。 + +配置文件语法 +============ + +引导配置文件的语法采用非常简单的键值结构。每个关键字由点连接的单词组成，键 +和值由 ``=`` 连接。值以分号（ ``;`` ）或换行符（ ``\n`` ）结尾。数组值中每 +个元素由逗号（ ``,`` ）分隔。:: + + KEY[.WORD[...]] = VALUE[, VALUE2[...]][;] + +与内核命令行语法不同，逗号和 ``=`` 周围允许有空格。 + +关键字只允许包含字母、数字、连字符（ ``-`` ）和下划线（ ``_`` ）。值可包含 +可打印字符和空格，但分号（ ``;`` ）、换行符（ ``\n`` ）、逗号（ ``,`` ）、 +井号（ ``#`` ）和右大括号（ ``}`` ）等分隔符除外。 + +如果你需要在值中使用这些分隔符，可以用双引号（ ``"VALUE"`` ）或单引号 +（ ``'VALUE'`` ）括起来。注意，引号无法转义。 + +键的值可以为空或不存在。这些键用于检查该键是否存在（类似布尔值）。 + +键值语法 +-------- + +引导配置文件语法允许用户通过大括号合并键名部分相同的关键字。例如:: + + foo.bar.baz = value1 + foo.bar.qux.quux = value2 + +也可以写成:: + + foo.bar { + baz = value1 + qux.quux = value2 + } + +或者更紧凑一些，写成:: + + foo.bar { baz = value1; qux.quux = value2 } + +在这两种样式中，引导解析时相同的关键字都会自动合并。因此可以追加类似的树或 +键值。 + +相同关键字的值 +-------------- + +禁止两个或多个值或数组共享同一个关键字。例如:: + + foo = bar, baz + foo = qux # !错误! 我们不可以重定义相同的关键字 + +如果你想要更新值，必须显式使用覆盖操作符 ``:=`` 。例如:: + + foo = bar, baz + foo := qux + +这样 ``foo`` 关键字的值就变成了 ``qux`` 。这对于通过添加（部分）自定义引导 +配置来覆盖默认值非常有用，免于解析默认引导配置。 + +如果你想对现有关键字追加值作为数组成员，可以使用 ``+=`` 操作符。例如:: + + foo = bar, baz + foo += qux + +这样， ``foo`` 关键字就同时拥有了 ``bar`` ， ``baz`` 和 ``qux`` 。 + +此外，父关键字下可同时存在值和子关键字。 +例如，下列配置是可行的。:: + + foo = value1 + foo.bar = value2 + foo := value3 # 这会更新foo的值。 + +注意，裸值不能直接放进结构化关键字中，必须在大括号外定义它。例如:: + + foo { + bar = value1 + bar { + baz = value2 + qux = value3 + } + } + +同时，关键字下值节点的顺序是固定的。如果值和子关键字同时存在，值永远是该关 +键字的第一个子节点。因此如果用户先指定子关键字，如:: + + foo.bar = value1 + foo = value2 + +则在程序（和/proc/bootconfig）中，它会按如下显示:: + + foo = value2 + foo.bar = value1 + +注释 +---- + +配置语法接受shell脚本风格的注释。注释以井号（ ``#`` ）开始，到换行符 +（ ``\n`` ）结束。 + +:: + + # comment line + foo = value # value is set to foo. + bar = 1, # 1st element + 2, # 2nd element + 3 # 3rd element + +会被解析为:: + + foo = value + bar = 1, 2, 3 + +注意你不能把注释放在值和分隔符（ ``,`` 或 ``;`` ）之间。如下配置语法是错误的:: + + key = 1 # comment + ,2 + + +/proc/bootconfig +================ + +/proc/bootconfig是引导配置的用户空间接口。与/proc/cmdline不同，此文件内容以 +键值列表样式显示。 +每个键值对一行，样式如下:: + + KEY[.WORDS...] = "[VALUE]"[,"VALUE2"...] + + +用引导配置引导内核 +================== + +用引导配置引导内核有两种方法：将引导配置附加到initrd镜像或直接嵌入内核中。 + +*initrd: initial RAM disk，初始内存磁盘* + +将引导配置附加到initrd +---------------------- + +由于默认情况下引导配置文件是用initrd加载的，因此它将被添加到initrd（initramfs） +镜像文件的末尾，其中包含填充、大小、校验值和12字节幻数，如下所示:: + + [initrd][bootconfig][padding][size(le32)][checksum(le32)][#BOOTCONFIG\n] + +大小和校验值为小端序存放的32位无符号值。 + +当引导配置被加到initrd镜像时，整个文件大小会对齐到4字节。空字符（ ``\0`` ） +会填补对齐空隙。因此 ``size`` 就是引导配置文件的长度+填充的字节。 + +Linux内核在内存中解码initrd镜像的最后部分以获取引导配置数据。由于这种“背负式” +的方法，只要引导加载器传递了正确的initrd文件大小，就无需更改或更新引导加载器 +和内核镜像本身。如果引导加载器意外传递了更长的大小，内核将无法找到引导配置数 +据。 + +Linux内核在tools/bootconfig下提供了 ``bootconfig`` 命令来完成此操作，管理员 +可以用它从initrd镜像中删除或追加配置文件。你可以用以下命令来构建它:: + + # make -C tools/bootconfig + +要向initrd镜像添加你的引导配置文件，请按如下命令操作（旧数据会自动移除）:: + + # tools/bootconfig/bootconfig -a your-config /boot/initrd.img-X.Y.Z + +要从镜像中移除配置，可以使用-d选项:: + + # tools/bootconfig/bootconfig -d /boot/initrd.img-X.Y.Z + +然后在内核命令行上添加 ``bootconfig`` 告诉内核去initrd文件末尾寻找内核配置。 + +将引导配置嵌入内核 +------------------ + +如果你不能使用initrd，也可以通过Kconfig选项将引导配置文件嵌入内核中。在此情 +况下，你需要用以下选项重新编译内核:: + + CONFIG_BOOT_CONFIG_EMBED=y + CONFIG_BOOT_CONFIG_EMBED_FILE="/引导配置/文件/的/路径" + +``CONFIG_BOOT_CONFIG_EMBED_FILE`` 需要从源码树或对象树开始的引导配置文件的 +绝对/相对路径。内核会将其嵌入作为默认引导配置。 + +与将引导配置附加到initrd一样，你也需要在内核命令行上添加 ``bootconfig`` 告诉 +内核去启用内嵌的引导配置。 + +注意，即使你已经设置了此选项，仍可用附加到initrd的其他引导配置覆盖内嵌的引导 +配置。 + +通过引导配置传递内核参数 +======================== + +除了内核命令行，引导配置也可以用于传递内核参数。所有 ``kernel`` 关键字下的键 +值对都将直接传递给内核命令行。此外， ``init`` 下的键值对将通过命令行传递给 +init进程。参数按以下顺序与用户给定的内核命令行字符串相连，因此命令行参数可以 +覆盖引导配置参数（这取决于子系统如何处理参数，但通常前面的参数将被后面的参数 +覆盖）:: + + [bootconfig params][cmdline params] -- [bootconfig init params][cmdline init params] + +如果引导配置文件给出的kernel/init参数是:: + + kernel { + root = 01234567-89ab-cdef-0123-456789abcd + } + init { + splash + } + +这将被复制到内核命令行字符串中，如下所示:: + + root="01234567-89ab-cdef-0123-456789abcd" -- splash + +如果用户给出的其他命令行是:: + + ro bootconfig -- quiet + +则最后的内核命令行如下:: + + root="01234567-89ab-cdef-0123-456789abcd" ro bootconfig -- splash quiet + + +配置文件的限制 +============== + +当前最大的配置大小是32KB，关键字总数（不是键值条目）必须少于1024个节点。 +注意：这不是条目数而是节点数，条目必须消耗超过2个节点（一个关键字和一个值）。 +所以从理论上讲最多512个键值对。如果关键字平均包含3个单词，则可有256个键值对。 +在大多数情况下，配置项的数量将少于100个条目，小于8KB，因此这应该足够了。如果 +节点数超过1024，解析器将返回错误，即使文件大小小于32KB。（请注意，此最大尺寸 +不包括填充的空字符。） +无论如何，因为 ``bootconfig`` 命令在附加启动配置到initrd映像时会验证它，用户 +可以在引导之前注意到它。 + + +引导配置API +=========== + +用户可以查询或遍历键值对，也可以查找（前缀）根关键字节点，并在查找该节点下的 +键值。 + +如果您有一个关键字字符串，则可以直接使用 xbc_find_value() 查询该键的值。如果 +你想知道引导配置里有哪些关键字，可以使用 xbc_for_each_key_value() 迭代键值对。 +请注意，您需要使用 xbc_array_for_each_value() 访问数组的值，例如:: + + vnode = NULL; + xbc_find_value("key.word", &vnode); + if (vnode && xbc_node_is_array(vnode)) + xbc_array_for_each_value(vnode, value) { + printk("%s ", value); + } + +如果您想查找具有前缀字符串的键，可以使用 xbc_find_node() 通过前缀字符串查找 +节点，然后用 xbc_node_for_each_key_value() 迭代前缀节点下的键。 + +但最典型的用法是获取前缀下的命名值或前缀下的命名数组，例如:: + + root = xbc_find_node("key.prefix"); + value = xbc_node_find_value(root, "option", &vnode); + ... + xbc_node_for_each_array_value(root, "array-option", value, anode) { + ... + } + +这将访问值“key.prefix.option”的值和“key.prefix.array-option”的数组。 + +锁是不需要的，因为在初始化之后配置只读。如果需要修改，必须复制所有数据和关键字。 + + +函数与结构体 +============ + +相关定义的kernel-doc参见： + + - include/linux/bootconfig.h + - lib/bootconfig.c diff --git a/Documentation/translations/zh_CN/admin-guide/index.rst b/Documentation/translations/zh_CN/admin-guide/index.rst index 2f6970d0a032579e97e17951ebe0b12a4c84743e..ac2960da33e68ffb709da49e604b3a4f40ea215e 100644 --- a/Documentation/translations/zh_CN/admin-guide/index.rst +++ b/Documentation/translations/zh_CN/admin-guide/index.rst @@ -63,6 +63,7 @@ Todolist: .. toctree:: :maxdepth: 1 + bootconfig clearing-warn-once cpu-load cputopology @@ -80,7 +81,6 @@ Todolist: * binderfs * binfmt-misc * blockdev/index -* bootconfig * braille-console * btmrvl * cgroup-v1/index diff --git a/Documentation/translations/zh_CN/arch.rst b/Documentation/translations/zh_CN/arch.rst new file mode 100644 index 0000000000000000000000000000000000000000..690e173d8b2a83cc1e0da421275aa56eebb05bc0 --- /dev/null +++ b/Documentation/translations/zh_CN/arch.rst @@ -0,0 +1,29 @@ +.. SPDX-License-Identifier: GPL-2.0 + +处理器体系结构 +============== + +以下文档提供了具体架构实现的编程细节。 + +.. toctree:: + :maxdepth: 2 + + mips/index + arm64/index + riscv/index + openrisc/index + parisc/index + loongarch/index + +TODOList: + +* arm/index +* ia64/index +* m68k/index +* nios2/index +* powerpc/index +* s390/index +* sh/index +* sparc/index +* x86/index +* xtensa/index diff --git a/Documentation/translations/zh_CN/core-api/circular-buffers.rst b/Documentation/translations/zh_CN/core-api/circular-buffers.rst new file mode 100644 index 0000000000000000000000000000000000000000..694ad8e61070ac99a46f1fdd572d1c0b3197c147 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/circular-buffers.rst @@ -0,0 +1,210 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/circular-buffers.rst + +:翻译: + + 周彬彬 Binbin Zhou + +:校译: + + 司延腾 Yanteng Si + 吴想成 Wu Xiangcheng + 时奎亮 Alex Shi + +========== +环形缓冲区 +========== + +:作者: David Howells +:作者: Paul E. McKenney + + +Linux 提供了许多可用于实现循环缓冲的特性。有两组这样的特性： + + (1) 用于确定2次方大小的缓冲区信息的便利函数。 + + (2) 可以代替缓冲区中对象的生产者和消费者共享锁的内存屏障。 + +如下所述，要使用这些设施，只需要一个生产者和一个消费者。可以通过序列化来处理多个 +生产者，并通过序列化来处理多个消费者。 + +.. Contents: + + (*) 什么是环形缓冲区？ + + (*) 测量2次幂缓冲区 + + (*) 内存屏障与环形缓冲区的结合使用 + - 生产者 + - 消费者 + + (*) 延伸阅读 + + + +什么是环形缓冲区？ +================== + +首先，什么是环形缓冲区？环形缓冲区是具有固定的有限大小的缓冲区，它有两个索引： + + (1) 'head'索引 - 生产者将元素插入缓冲区的位置。 + + (2) 'tail'索引 - 消费者在缓冲区中找到下一个元素的位置。 + +通常，当tail指针等于head指针时，表明缓冲区是空的；而当head指针比tail指针少一个时， +表明缓冲区是满的。 + +添加元素时，递增head索引；删除元素时，递增tail索引。tail索引不应该跳过head索引， +两个索引在到达缓冲区末端时都应该被赋值为0，从而允许海量的数据流过缓冲区。 + +通常情况下，元素都有相同的单元大小，但这并不是使用以下技术的严格要求。如果要在缓 +冲区中包含多个元素或可变大小的元素，则索引可以增加超过1，前提是两个索引都没有超过 +另一个。然而，实现者必须小心，因为超过一个单位大小的区域可能会覆盖缓冲区的末端并 +且缓冲区会被分成两段。 + +测量2次幂缓冲区 +=============== + +计算任意大小的环形缓冲区的占用或剩余容量通常是一个费时的操作，需要使用模（除法） +指令。但是如果缓冲区的大小为2次幂，则可以使用更快的按位与指令代替。 + +Linux提供了一组用于处理2次幂环形缓冲区的宏。可以通过以下方式使用:: + + #include + +这些宏包括： + + (#) 测量缓冲区的剩余容量:: + + CIRC_SPACE(head_index, tail_index, buffer_size); + + 返回缓冲区[1]中可插入元素的剩余空间大小。 + + + (#) 测量缓冲区中的最大连续立即可用空间:: + + CIRC_SPACE_TO_END(head_index, tail_index, buffer_size); + + 返回缓冲区[1]中剩余的连续空间的大小，元素可以立即插入其中，而不必绕回到缓冲 + 区的开头。 + + + (#) 测量缓冲区的使用数:: + + CIRC_CNT(head_index, tail_index, buffer_size); + + 返回当前占用缓冲区[2]的元素数量。 + + + (#) 测量缓冲区的连续使用数:: + + CIRC_CNT_TO_END(head_index, tail_index, buffer_size); + + 返回可以从缓冲区中提取的连续元素[2]的数量，而不必绕回到缓冲区的开头。 + +这里的每一个宏名义上都会返回一个介于0和buffer_size-1之间的值，但是： + + (1) CIRC_SPACE*()是为了在生产者中使用。对生产者来说，它们将返回一个下限，因为生 + 产者控制着head索引，但消费者可能仍然在另一个CPU上耗尽缓冲区并移动tail索引。 + + 对消费者来说，它将显示一个上限，因为生产者可能正忙于耗尽空间。 + + (2) CIRC_CNT*()是为了在消费者中使用。对消费者来说，它们将返回一个下限，因为消费 + 者控制着tail索引，但生产者可能仍然在另一个CPU上填充缓冲区并移动head索引。 + + 对于生产者，它将显示一个上限，因为消费者可能正忙于清空缓冲区。 + + (3) 对于第三方来说，生产者和消费者对索引的写入顺序是无法保证的，因为它们是独立的， + 而且可能是在不同的CPU上进行的，所以在这种情况下的结果只是一种猜测，甚至可能 + 是错误的。 + +内存屏障与环形缓冲区的结合使用 +============================== + +通过将内存屏障与环形缓冲区结合使用，可以避免以下需求： + + (1) 使用单个锁来控制对缓冲区两端的访问，从而允许同时填充和清空缓冲区；以及 + + (2) 使用原子计数器操作。 + +这有两个方面：填充缓冲区的生产者和清空缓冲区的消费者。在任何时候，只应有一个生产 +者在填充缓冲区，同样的也只应有一个消费者在清空缓冲区，但双方可以同时操作。 + + +生产者 +------ + +生产者看起来像这样:: + + spin_lock(&producer_lock); + + unsigned long head = buffer->head; + /* spin_unlock()和下一个spin_lock()提供必要的排序。 */ + unsigned long tail = READ_ONCE(buffer->tail); + + if (CIRC_SPACE(head, tail, buffer->size) >= 1) { + /* 添加一个元素到缓冲区 */ + struct item *item = buffer[head]; + + produce_item(item); + + smp_store_release(buffer->head, + (head + 1) & (buffer->size - 1)); + + /* wake_up()将确保在唤醒任何人之前提交head */ + wake_up(consumer); + } + + spin_unlock(&producer_lock); + +这将表明CPU必须在head索引使其对消费者可用之前写入新项目的内容，同时CPU必须在唤醒 +消费者之前写入修改后的head索引。 + +请注意，wake_up()并不保证任何形式的屏障，除非确实唤醒了某些东西。因此我们不能依靠 +它来进行排序。但是数组中始终有一个元素留空，因此生产者必须产生两个元素，然后才可 +能破坏消费者当前正在读取的元素。同时，消费者连续调用之间成对的解锁-加锁提供了索引 +读取（指示消费者已清空给定元素）和生产者对该相同元素的写入之间的必要顺序。 + + +消费者 +------ + +消费者看起来像这样:: + + spin_lock(&consumer_lock); + + /* 读取该索引处的内容之前，先读取索引 */ + unsigned long head = smp_load_acquire(buffer->head); + unsigned long tail = buffer->tail; + + if (CIRC_CNT(head, tail, buffer->size) >= 1) { + + /* 从缓冲区中提取一个元素 */ + struct item *item = buffer[tail]; + + consume_item(item); + + /* 在递增tail之前完成对描述符的读取。 */ + smp_store_release(buffer->tail, + (tail + 1) & (buffer->size - 1)); + } + + spin_unlock(&consumer_lock); + +这表明CPU在读取新元素之前确保索引是最新的，然后在写入新的尾指针之前应确保CPU已完 +成读取该元素，这将擦除该元素。 + +请注意，使用READ_ONCE()和smp_load_acquire()来读取反向（head）索引。这可以防止编译 +器丢弃并重新加载其缓存值。如果您能确定反向（head）索引将仅使用一次，则这不是必须 +的。smp_load_acquire()还可以强制CPU对后续的内存引用进行排序。类似地，两种算法都使 +用smp_store_release()来写入线程的索引。这记录了我们正在写入可以并发读取的内容的事 +实，以防止编译器破坏存储，并强制对以前的访问进行排序。 + + +延伸阅读 +======== + +关于Linux的内存屏障设施的描述，请查看Documentation/memory-barriers.txt。 diff --git a/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst b/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst new file mode 100644 index 0000000000000000000000000000000000000000..eacd1d2ebddc79a0b19f2029f1629e58120157b9 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst @@ -0,0 +1,23 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/generic-radix-tree.rst + +:翻译: + + 周彬彬 Binbin Zhou + +=================== +通用基数树/稀疏数组 +=================== + +通用基数树/稀疏数组的相关内容请见include/linux/generic-radix-tree.h文件中的 +“DOC: Generic radix trees/sparse arrays”。 + +通用基数树函数 +-------------- + +该API在以下内核代码中: + +include/linux/generic-radix-tree.h diff --git a/Documentation/translations/zh_CN/core-api/idr.rst b/Documentation/translations/zh_CN/core-api/idr.rst new file mode 100644 index 0000000000000000000000000000000000000000..97a16e76b81b085b3fc80939ed5f37ae7e664a17 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/idr.rst @@ -0,0 +1,80 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/idr.rst + +:翻译: + + 周彬彬 Binbin Zhou + +:校译: + + 司延腾 Yanteng Si + 吴想成 Wu Xiangcheng + 时奎亮 Alex Shi + +====== +ID分配 +====== + +:作者: Matthew Wilcox + +概述 +==== + +要解决的一个常见问题是分配标识符（IDs）；它通常是标识事物的数字。比如包括文件描述 +符、进程ID、网络协议中的数据包标识符、SCSI标记和设备实例编号。IDR和IDA为这个问题 +提供了一个合理的解决方案，以避免每个人都自创。IDR提供将ID映射到指针的能力，而IDA +仅提供ID分配，因此内存效率更高。 + +IDR接口已经被废弃，请使用 ``XArray`` 。 + +IDR的用法 +========= + +首先初始化一个IDR，对于静态分配的IDR使用DEFINE_IDR()，或者对于动态分配的IDR使用 +idr_init()。 + +您可以调用idr_alloc()来分配一个未使用的ID。通过调用idr_find()查询与该ID相关的指针， +并通过调用idr_remove()释放该ID。 + +如果需要更改与一个ID相关联的指针，可以调用idr_replace()。这样做的一个常见原因是通 +过将 ``NULL`` 指针传递给分配函数来保留ID；用保留的ID初始化对象，最后将初始化的对 +象插入IDR。 + +一些用户需要分配大于 ``INT_MAX`` 的ID。到目前为止，所有这些用户都满足 ``UINT_MAX`` +的限制，他们使用idr_alloc_u32()。如果您需要超出u32的ID，我们将与您合作以满足您的 +需求。 + +如果需要按顺序分配ID，可以使用idr_alloc_cyclic()。处理较大数量的ID时，IDR的效率会 +降低，所以使用这个函数会有一点代价。 + +要对IDR使用的所有指针进行操作，您可以使用基于回调的idr_for_each()或迭代器样式的 +idr_for_each_entry()。您可能需要使用idr_for_each_entry_continue()来继续迭代。如果 +迭代器不符合您的需求，您也可以使用idr_get_next()。 + +当使用完IDR后，您可以调用idr_destroy()来释放IDR占用的内存。这并不会释放IDR指向的 +对象；如果您想这样做，请使用其中一个迭代器来执行此操作。 + +您可以使用idr_is_empty()来查看当前是否分配了任何ID。 + +如果在从IDR分配一个新ID时需要带锁，您可能需要传递一组限制性的GFP标志，但这可能导 +致IDR无法分配内存。为了解决该问题，您可以在获取锁之前调用idr_preload()，然后在分 +配之后调用idr_preload_end()。 + +IDR同步的相关内容请见include/linux/idr.h文件中的“DOC: idr sync”。 + +IDA的用法 +========= + +IDA的用法的相关内容请见lib/idr.c文件中的“DOC: IDA description”。 + +函数和数据结构 +============== + +该API在以下内核代码中: + +include/linux/idr.h + +lib/idr.c diff --git a/Documentation/translations/zh_CN/core-api/index.rst b/Documentation/translations/zh_CN/core-api/index.rst index 8a94ad87465dc90ceda43abea9915817b711f446..37756d240b5eebc0de976ef87a30b957283ca326 100644 --- a/Documentation/translations/zh_CN/core-api/index.rst +++ b/Documentation/translations/zh_CN/core-api/index.rst @@ -44,15 +44,15 @@ assoc_array xarray rbtree + idr + circular-buffers + generic-radix-tree + packing Todolist: - idr - circular-buffers - generic-radix-tree - packing this_cpu_ops timekeeping errseq diff --git a/Documentation/translations/zh_CN/core-api/packing.rst b/Documentation/translations/zh_CN/core-api/packing.rst new file mode 100644 index 0000000000000000000000000000000000000000..c0aab3a349d0548377498d2604ca910d0560c723 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/packing.rst @@ -0,0 +1,160 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/packing.rst + +:翻译: + + 周彬彬 Binbin Zhou + +:校译: + + 司延腾 Yanteng Si + 吴想成 Wu Xiangcheng + 时奎亮 Alex Shi + +======================== +通用的位域打包和解包函数 +======================== + +问题陈述 +-------- + +使用硬件时，必须在几种与其交互的方法之间进行选择。 + +可以将指针映射到在硬件设备的内存区上精心设计的结构体，并将其字段作为结构成员（可 +能声明为位域）访问。但是由于CPU和硬件设备之间潜在的字节顺序不匹配，以这种方式编写 +代码会降低其可移植性。 + +此外，必须密切注意将硬件文档中的寄存器定义转换为结构的位域索引。此外，一些硬件 +（通常是网络设备）倾向于以违反任何合理字边界（有时甚至是64位）的方式对其寄存器字 +段进行分组。这就造成了不得不在结构中定义寄存器字段的“高”和“低”部分的不便。 + +结构域定义的更可靠的替代方法是通过移动适当数量的位来提取所需的字段。但这仍然不能 +防止字节顺序不匹配，除非所有内存访问都是逐字节执行的。此外，代码很容易变得杂乱无 +章，同时可能会在所需的许多位移操作中丢失一些高层次的想法。 + +许多驱动程序采用了位移的方法，然后试图用定制的宏来减少杂乱无章的东西，但更多的时 +候，这些宏所采用的捷径依旧妨碍了代码真正的可移植性。 + +解决方案 +-------- + +该API涉及2个基本操作： + + - 将一个CPU可使用的数字打包到内存缓冲区中（具有硬件约束/特殊性）。 + - 将内存缓冲区（具有硬件约束/特殊性）解压缩为一个CPU可使用的数字。 + +该API提供了对所述硬件约束和特殊性以及CPU字节序的抽象，因此这两者之间可能不匹配。 + +这些API函数的基本单元是u64。从CPU的角度来看，位63总是意味着字节7的位偏移量7，尽管 +只是逻辑上的。问题是：我们将这个比特放在内存的什么位置？ + +以下示例介绍了打包u64字段的内存布局。打包缓冲区中的字节偏移量始终默认为0，1...7。 +示例显示的是逻辑字节和位所在的位置。 + +1. 通常情况下（无特殊性），我们会这样做： + +:: + + 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 + 7 6 5 4 + 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + 3 2 1 0 + +也就是说，CPU可使用的u64的MSByte(7)位于内存偏移量0处，而u64的LSByte(0)位于内存偏移量7处。 + +这对应于大多数人认为的“大端”，其中位i对应于数字2^i。这在代码注释中也称为“逻辑”符号。 + + +2. 如果设置了QUIRK_MSB_ON_THE_RIGHT，我们按如下方式操作： + +:: + + 56 57 58 59 60 61 62 63 48 49 50 51 52 53 54 55 40 41 42 43 44 45 46 47 32 33 34 35 36 37 38 39 + 7 6 5 4 + 24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23 8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7 + 3 2 1 0 + +也就是说，QUIRK_MSB_ON_THE_RIGHT不会影响字节定位，但会反转字节内的位偏移量。 + + +3. 如果设置了QUIRK_LITTLE_ENDIAN，我们按如下方式操作： + +:: + + 39 38 37 36 35 34 33 32 47 46 45 44 43 42 41 40 55 54 53 52 51 50 49 48 63 62 61 60 59 58 57 56 + 4 5 6 7 + 7 6 5 4 3 2 1 0 15 14 13 12 11 10 9 8 23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24 + 0 1 2 3 + +因此，QUIRK_LITTLE_ENDIAN意味着在内存区域内，每个4字节的字的每个字节都被放置在与 +该字的边界相比的镜像位置。 + + +4. 如果设置了QUIRK_MSB_ON_THE_RIGHT和QUIRK_LITTLE_ENDIAN，我们这样做： + +:: + + 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 + 4 5 6 7 + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + 0 1 2 3 + + +5. 如果只设置了QUIRK_LSW32_IS_FIRST，我们这样做： + +:: + + 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + 3 2 1 0 + 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 + 7 6 5 4 + +在这种情况下，8字节内存区域解释如下：前4字节对应最不重要的4字节的字，后4字节对应 +更重要的4字节的字。 + +6. 如果设置了QUIRK_LSW32_IS_FIRST和QUIRK_MSB_ON_THE_RIGHT，我们这样做： + +:: + + 24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23 8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7 + 3 2 1 0 + 56 57 58 59 60 61 62 63 48 49 50 51 52 53 54 55 40 41 42 43 44 45 46 47 32 33 34 35 36 37 38 39 + 7 6 5 4 + + +7. 如果设置了QUIRK_LSW32_IS_FIRST和QUIRK_LITTLE_ENDIAN，则如下所示： + +:: + + 7 6 5 4 3 2 1 0 15 14 13 12 11 10 9 8 23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24 + 0 1 2 3 + 39 38 37 36 35 34 33 32 47 46 45 44 43 42 41 40 55 54 53 52 51 50 49 48 63 62 61 60 59 58 57 56 + 4 5 6 7 + + +8. 如果设置了QUIRK_LSW32_IS_FIRST，QUIRK_LITTLE_ENDIAN和QUIRK_MSB_ON_THE_RIGHT， + 则如下所示： + +:: + + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + 0 1 2 3 + 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 + 4 5 6 7 + + +我们总是认为我们的偏移量好像没有特殊性，然后在访问内存区域之前翻译它们。 + +预期用途 +-------- + +选择使用该API的驱动程序首先需要确定上述3种quirk组合（共8种）中的哪一种与硬件文档 +中描述的相匹配。然后，他们应该封装packing()函数，创建一个新的xxx_packing()，使用 +适当的QUIRK_* one-hot 位集合来调用它。 + +packing()函数返回一个int类型的错误码，以防止程序员使用不正确的API。这些错误预计不 +会在运行时发生，因此xxx_packing()返回void并简单地接受这些错误是合理的。它可以选择 +转储栈或打印错误描述。 diff --git a/Documentation/translations/zh_CN/devicetree/changesets.rst b/Documentation/translations/zh_CN/devicetree/changesets.rst new file mode 100644 index 0000000000000000000000000000000000000000..3df1b03c5695cf70aee2cc112d72ccb62b957ec6 --- /dev/null +++ b/Documentation/translations/zh_CN/devicetree/changesets.rst @@ -0,0 +1,37 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/devicetree/changesets.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +============ +设备树变更集 +============ + +设备树变更集是一种方法，它允许人们以这样一种方式在实时树中使用变化，即要么使用全部的 +变化，要么不使用。如果在使用变更集的过程中发生错误，那么树将被回滚到之前的状态。一个 +变更集也可以在使用后被删除。 + +当一个变更集被使用时，所有的改变在发出OF_RECONFIG通知器之前被一次性使用到树上。这是 +为了让接收者在收到通知时看到一个完整的、一致的树的状态。 + +一个变化集的顺序如下。 + +1. of_changeset_init() - 初始化一个变更集。 + +2. 一些DT树变化的调用，of_changeset_attach_node(), of_changeset_detach_node(), + of_changeset_add_property(), of_changeset_remove_property, + of_changeset_update_property()来准备一组变更。此时不会对活动树做任何变更。所有 + 的变更操作都记录在of_changeset的 `entries` 列表中。 + +3. of_changeset_apply() - 将变更使用到树上。要么整个变更集被使用，要么如果有错误， + 树会被恢复到之前的状态。核心通过锁确保正确的顺序。如果需要的话，可以使用一个解锁的 + __of_changeset_apply版本。 + +如果一个成功使用的变更集需要被删除，可以用of_changeset_revert()来完成。 diff --git a/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst b/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst new file mode 100644 index 0000000000000000000000000000000000000000..6dfd946d70932cdc17dc4adc199169bbf8384a82 --- /dev/null +++ b/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst @@ -0,0 +1,31 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/devicetree/dynamic-resolution-notes.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +======================== +Devicetree动态解析器说明 +======================== + +本文描述了内核内DeviceTree解析器的实现，它位于drivers/of/resolver.c中。 + +解析器如何工作？ +---------------- + +解析器被赋予一个任意的树作为输入，该树用适当的dtc选项编译，并有一个/plugin/标签。这就产 +生了适当的__fixups__和__local_fixups__节点。 + +解析器依次通过以下步骤工作: + +1. 从实时树中获取最大的设备树phandle值 + 1. +2. 调整树的所有本地 phandles，以解决这个量。 +3. 使用 __local__fixups__ 节点信息以相同的量调整所有本地引用。 +4. 对于__fixups__节点中的每个属性，找到它在实时树中引用的节点。这是用来标记该节点的标签。 +5. 检索fixup的目标的phandle。 +6. 对于属性中的每个fixup，找到节点:属性:偏移的位置，并用phandle值替换它。 diff --git a/Documentation/translations/zh_CN/devicetree/index.rst b/Documentation/translations/zh_CN/devicetree/index.rst index 3fc355fe0037228609edab6ba010e1eb699cb161..7451dbfdd3e58ed56a1e6e3962d93d63498643fe 100644 --- a/Documentation/translations/zh_CN/devicetree/index.rst +++ b/Documentation/translations/zh_CN/devicetree/index.rst @@ -24,21 +24,16 @@ Open Firmware 和 Devicetree usage-model of_unittest - -Todolist: - -* kernel-api + kernel-api Devicetree Overlays =================== .. toctree:: :maxdepth: 1 -Todolist: - -* changesets -* dynamic-resolution-notes -* overlay-notes + changesets + dynamic-resolution-notes + overlay-notes Devicetree Bindings =================== diff --git a/Documentation/translations/zh_CN/devicetree/kernel-api.rst b/Documentation/translations/zh_CN/devicetree/kernel-api.rst new file mode 100644 index 0000000000000000000000000000000000000000..2fb729368b406c0a9efe6adddd840e6b4dbba608 --- /dev/null +++ b/Documentation/translations/zh_CN/devicetree/kernel-api.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/devicetree/kernel-api.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + +================= +内核中的设备树API +================= + +核心函数 +-------- + +该API在以下内核代码中: + +drivers/of/base.c + +include/linux/of.h + +drivers/of/property.c + +include/linux/of_graph.h + +drivers/of/address.c + +drivers/of/irq.c + +drivers/of/fdt.c + +驱动模型函数 +------------ + +该API在以下内核代码中: + +include/linux/of_device.h + +drivers/of/device.c + +include/linux/of_platform.h + +drivers/of/platform.c + +覆盖和动态DT函数 +---------------- + +该API在以下内核代码中: + +drivers/of/resolver.c + +drivers/of/dynamic.c + +drivers/of/overlay.c diff --git a/Documentation/translations/zh_CN/devicetree/overlay-notes.rst b/Documentation/translations/zh_CN/devicetree/overlay-notes.rst new file mode 100644 index 0000000000000000000000000000000000000000..43e3c0bc5a9f82358f1f17ff0e75051b084b1da8 --- /dev/null +++ b/Documentation/translations/zh_CN/devicetree/overlay-notes.rst @@ -0,0 +1,140 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/devicetree/overlay-notes.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +============== +设备树覆盖说明 +============== + +本文档描述了drivers/of/overlay.c中的内核内设备树覆盖功能的实现，是 +Documentation/devicetree/dynamic-resolution-notes.rst[1]的配套文档。 + +覆盖如何工作 +------------ + +设备树覆盖的目的是修改内核的实时树，并使修改以反映变化的方式影响内核的状态。 +由于内核主要处理的是设备，任何新的设备节点如果导致一个活动的设备，就应该创建它， +而如果设备节点被禁用或被全部删除，受影响的设备应该被取消注册。 + +让我们举个例子，我们有一个foo板，它的基本树形图如下:: + + ---- foo.dts --------------------------------------------------------------- + /* FOO平台 */ + /dts-v1/; + / { + compatible = "corp,foo"; + + /* 共享的资源 */ + res: res { + }; + + /* 芯片上的外围设备 */ + ocp: ocp { + /* 总是被实例化的外围设备 */ + peripheral1 { ... }; + }; + }; + ---- foo.dts --------------------------------------------------------------- + +覆盖bar.dts, +:: + + ---- bar.dts - 按标签覆盖目标位置 ---------------------------- + /dts-v1/; + /插件/; + &ocp { + /* bar外围 */ + bar { + compatible = "corp,bar"; + ... /* 各种属性和子节点 */ + }; + }; + ---- bar.dts --------------------------------------------------------------- + +当加载（并按照[1]中描述的方式解决）时，应该产生foo+bar.dts:: + + ---- foo+bar.dts ----------------------------------------------------------- + /* FOO平台 + bar外围 */ + / { + compatible = "corp,foo"; + + /* 共享资源 */ + res: res { + }; + + /* 芯片上的外围设备 */ + ocp: ocp { + /* 总是被实例化的外围设备 */ + peripheral1 { ... }; + + /* bar外围 */ + bar { + compatible = "corp,bar"; + ... /* 各种属性和子节点 */ + }; + }; + }; + ---- foo+bar.dts ----------------------------------------------------------- + +作为覆盖的结果，已经创建了一个新的设备节点（bar），因此将注册一个bar平台设备， +如果加载了匹配的设备驱动程序，将按预期创建设备。 + +如果基础DT不是用-@选项编译的，那么“&ocp”标签将不能用于将覆盖节点解析到基础 +DT中的适当位置。在这种情况下，可以提供目标路径。通过标签的目标位置的语法是比 +较好的，因为不管标签在DT中出现在哪里，覆盖都可以被应用到任何包含标签的基础DT上。 + +上面的bar.dts例子被修改为使用目标路径语法，即为:: + + ---- bar.dts - 通过明确的路径覆盖目标位置 -------------------- + /dts-v1/; + /插件/; + &{/ocp} { + /* bar外围 */ + bar { + compatible = "corp,bar"; + ... /* 各种外围设备和子节点 */ + } + }; + ---- bar.dts --------------------------------------------------------------- + + +内核中关于覆盖的API +------------------- + +该API相当容易使用。 + +1) 调用of_overlay_fdt_apply()来创建和应用一个覆盖的变更集。返回值是一个 + 错误或一个识别这个覆盖的cookie。 + +2) 调用of_overlay_remove()来删除和清理先前通过调用of_overlay_fdt_apply() + 而创建的覆盖变更集。不允许删除一个被另一个覆盖的覆盖变化集。 + +最后，如果你需要一次性删除所有的覆盖，只需调用of_overlay_remove_all()， +它将以正确的顺序删除每一个覆盖。 + +你可以选择注册在覆盖操作中被调用的通知器。详见 +of_overlay_notifier_register/unregister和enum of_overlay_notify_action。 + +OF_OVERLAY_PRE_APPLY、OF_OVERLAY_POST_APPLY或OF_OVERLAY_PRE_REMOVE +的通知器回调可以存储指向覆盖层中的设备树节点或其内容的指针，但这些指针不能持 +续到OF_OVERLAY_POST_REMOVE的通知器回调。在OF_OVERLAY_POST_REMOVE通 +知器被调用后，包含覆盖层的内存将被kfree()ed。请注意，即使OF_OVERLAY_POST_REMOVE +的通知器返回错误，内存也会被kfree()ed。 + +drivers/of/dynamic.c中的变更集通知器是第二种类型的通知器，可以通过应用或移除 +覆盖层来触发。这些通知器不允许在覆盖层或其内容中存储指向设备树节点的指针。当包含 +覆盖层的内存因移除覆盖层而被释放时，覆盖层代码并不能防止这类指针仍然有效。 + +任何其他保留指向覆盖层节点或数据的指针的代码都被认为是一个错误，因为在移除覆盖层 +后，该指针将指向已释放的内存。 + +覆盖层的用户必须特别注意系统上发生的整体操作，以确保其他内核代码不保留任何指向覆 +盖层节点或数据的指针。任何无意中使用这种指针的例子是，如果一个驱动或子系统模块在 +应用了覆盖后被加载，并且该驱动或子系统扫描了整个设备树或其大部分，包括覆盖节点。 diff --git a/Documentation/translations/zh_CN/driver-api/gpio/index.rst b/Documentation/translations/zh_CN/driver-api/gpio/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..9ab64e94aced4cb4affebe3f4983ce8c825e22d0 --- /dev/null +++ b/Documentation/translations/zh_CN/driver-api/gpio/index.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../../disclaimer-zh_CN.rst + +:Original: Documentation/driver-api/gpio/index.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +======================= +通用型输入/输出（GPIO） +======================= + +目录: + +.. toctree:: + :maxdepth: 2 + + legacy + +Todolist: + +* intro +* using-gpio +* driver +* consumer +* board +* drivers-on-gpio +* bt8xxgpio + +核心 +==== + +该API在以下内核代码中: + +include/linux/gpio/driver.h + +drivers/gpio/gpiolib.c + +ACPI支持 +======== + +该API在以下内核代码中: + +drivers/gpio/gpiolib-acpi.c + +设备树支持 +========== + +该API在以下内核代码中: + +drivers/gpio/gpiolib-of.c + +设备管理支持 +============ + +该API在以下内核代码中: + +drivers/gpio/gpiolib-devres.c + +sysfs帮助（函数） +================= + +该API在以下内核代码中: + +drivers/gpio/gpiolib-sysfs.c diff --git a/Documentation/translations/zh_CN/gpio.txt b/Documentation/translations/zh_CN/driver-api/gpio/legacy.rst similarity index 89% rename from Documentation/translations/zh_CN/gpio.txt rename to Documentation/translations/zh_CN/driver-api/gpio/legacy.rst index a23ee14fc927230e7aed33f449bf9395db0cef2f..6399521d05489ec0e511514d2f1070b4c19e463f 100644 --- a/Documentation/translations/zh_CN/gpio.txt +++ b/Documentation/translations/zh_CN/driver-api/gpio/legacy.rst @@ -1,39 +1,28 @@ -Chinese translated version of Documentation/admin-guide/gpio +.. SPDX-License-Identifier: GPL-2.0 -If you have any comment or update to the content, please contact the -original document maintainer directly. However, if you have a problem -communicating in English you can also ask the Chinese maintainer for -help. Contact the Chinese maintainer if this translation is outdated -or if there is a problem with the translation. +.. include:: ../../disclaimer-zh_CN.rst -Maintainer: Grant Likely - Linus Walleij -Chinese maintainer: Fu Wei ---------------------------------------------------------------------- -Documentation/admin-guide/gpio 的中文翻译 +:Original: Documentation/driver-api/gpio/legacy.rst -如果想评论或更新本文的内容，请直接联系原文档的维护者。如果你使用英文 -交流有困难的话，也可以向中文版维护者求助。如果本翻译更新不及时或者翻 -译存在问题，请联系中文版维护者。 -英文版维护者： Grant Likely - Linus Walleij -中文版维护者： 傅炜 Fu Wei -中文版翻译者： 傅炜 Fu Wei -中文版校译者： 傅炜 Fu Wei +:翻译: + 傅炜 Fu Wei + 司延腾 Yanteng Si -以下为正文 ---------------------------------------------------------------------- -GPIO 接口 +:校译: -本文档提供了一个在Linux下访问GPIO的公约概述。 + +传统GPIO接口 +============ + +本文档概述了Linux下的GPIO访问公约。 这些函数以 gpio_* 作为前缀。其他的函数不允许使用这样的前缀或相关的 __gpio_* 前缀。 -什么是GPIO? -========== +什么是GPIO？ +============ "通用输入/输出口"(GPIO)是一个灵活的由软件控制的数字信号。他们可 由多种芯片提供,且对于从事嵌入式和定制硬件的 Linux 开发者来说是 比较熟悉。每个GPIO 都代表一个连接到特定引脚或球栅阵列(BGA)封装中 @@ -99,6 +88,7 @@ GPIO 公约 标识 GPIO --------- + GPIO 是通过无符号整型来标识的,范围是 0 到 MAX_INT。保留“负”数 用于其他目的,例如标识信号“在这个板子上不可用”或指示错误。未接触底层 硬件的代码会忽略这些整数。 @@ -115,7 +105,7 @@ FPGA 的特定板子上使用 80-95。编号不一定要连续,那些平台中 如果你要初始化一个带有无效 GPIO 编号的结构体,可以使用一些负编码 (如"-EINVAL")，那将使其永远不会是有效。来测试这样一个结构体中的编号 -是否关联一个 GPIO，你可使用以下断言: +是否关联一个 GPIO，你可使用以下断言:: int gpio_is_valid(int number); @@ -128,11 +118,12 @@ FPGA 的特定板子上使用 80-95。编号不一定要连续,那些平台中 使用 GPIO --------- + 对于一个 GPIO，系统应该做的第一件事情就是通过 gpio_request() 函数分配它，见下文。 接下来是设置I/O方向，这通常是在板级启动代码中为所使用的 GPIO 设置 -platform_device 时完成。 +platform_device 时完成:: /* 设置为输入或输出, 返回 0 或负的错误代码 */ int gpio_direction_input(unsigned gpio); @@ -157,12 +148,13 @@ get/set(获取/设置)函数调用没法返回错误,且有可能是配置错误 访问自旋锁安全的 GPIO -------------------- +--------------------- + 大多数 GPIO 控制器可以通过内存读/写指令来访问。这些指令不会休眠,可以 安全地在硬(非线程)中断例程和类似的上下文中完成。 对于那些用 gpio_cansleep()测试总是返回失败的 GPIO(见下文)，使用 -以下的函数访问: +以下的函数访问:: /* GPIO 输入:返回零或非零 */ int gpio_get_value(unsigned gpio); @@ -188,17 +180,18 @@ GPIO值是布尔值，零表示低电平，非零表示高电平。当读取一 访问可能休眠的 GPIO ------------------ +------------------- + 某些 GPIO 控制器必须通过基于总线(如 I2C 或 SPI)的消息访问。读或写这些 GPIO 值的命令需要等待其信息排到队首才发送命令，再获得其反馈。期间需要 休眠，这不能在 IRQ 例程(中断上下文)中执行。 支持此类 GPIO 的平台通过以下函数返回非零值来区分出这种 GPIO。(此函数需要 -一个之前通过 gpio_request 分配到的有效 GPIO 编号): +一个之前通过 gpio_request 分配到的有效 GPIO 编号):: int gpio_cansleep(unsigned gpio); -为了访问这种 GPIO,内核定义了一套不同的函数: +为了访问这种 GPIO,内核定义了一套不同的函数:: /* GPIO 输入:返回零或非零 ,可能会休眠 */ int gpio_get_value_cansleep(unsigned gpio); @@ -214,25 +207,26 @@ GPIO 值的命令需要等待其信息排到队首才发送命令，再获得其 事实，这些处理例程实际上和自旋锁安全的函数是一样的。 ** 除此之外 ** 调用设置和配置此类 GPIO 的函数也必须在允许休眠的上下文中， -因为它们可能也需要访问 GPIO 控制器芯片: (这些设置函数通常在板级启动代码或者 -驱动探测/断开代码中，所以这是一个容易满足的约束条件。) +因为它们可能也需要访问 GPIO 控制器芯片 (这些设置函数通常在板级启动代码或者 +驱动探测/断开代码中，所以这是一个容易满足的约束条件。) :: - gpio_direction_input() - gpio_direction_output() - gpio_request() + gpio_direction_input() + gpio_direction_output() + gpio_request() -## gpio_request_one() -## gpio_request_array() -## gpio_free_array() + ## gpio_request_one() + ## gpio_request_array() + ## gpio_free_array() - gpio_free() - gpio_set_debounce() + gpio_free() + gpio_set_debounce() 声明和释放 GPIO ----------------------------- -为了有助于捕获系统配置错误,定义了两个函数。 +---------------- + +为了有助于捕获系统配置错误,定义了两个函数:: /* 申请 GPIO, 返回 0 或负的错误代码. * 非空标签可能有助于诊断. @@ -256,9 +250,9 @@ GPIO 值的命令需要等待其信息排到队首才发送命令，再获得其 某些平台可能也使用 GPIO 作为电源管理激活信号(例如通过关闭未使用芯片区和 简单地关闭未使用时钟)。 -对于 GPIO 使用 pinctrl 子系统已知的引脚，子系统应该被告知其使用情况； +对于 GPIO 使用引脚控制子系统已知的引脚，子系统应该被告知其使用情况； 一个 gpiolib 驱动的 .request()操作应调用 pinctrl_gpio_request()， -而 gpiolib 驱动的 .free()操作应调用 pinctrl_gpio_free()。pinctrl +而 gpiolib 驱动的 .free()操作应调用 pinctrl_gpio_free()。引脚控制 子系统允许 pinctrl_gpio_request()在某个引脚或引脚组以复用形式“属于” 一个设备时都成功返回。 @@ -270,7 +264,7 @@ GPIO 值的命令需要等待其信息排到队首才发送命令，再获得其 某些平台允许部分或所有 GPIO 信号使用不同的引脚。类似的，GPIO 或引脚的 其他方面也需要配置，如上拉/下拉。平台软件应该在对这些 GPIO 调用 -gpio_request()前将这类细节配置好，例如使用 pinctrl 子系统的映射表， +gpio_request()前将这类细节配置好，例如使用引脚控制子系统的映射表， 使得 GPIO 的用户无须关注这些细节。 还有一个值得注意的是在释放 GPIO 前，你必须停止使用它。 @@ -278,7 +272,7 @@ gpio_request()前将这类细节配置好，例如使用 pinctrl 子系统的映 注意:申请一个 GPIO 并没有以任何方式配置它，只不过标识那个 GPIO 处于使用 状态。必须有另外的代码来处理引脚配置(如控制 GPIO 使用的引脚、上拉/下拉)。 -考虑到大多数情况下声明 GPIO 之后就会立即配置它们,所以定义了以下三个辅助函数: +考虑到大多数情况下声明 GPIO 之后就会立即配置它们,所以定义了以下三个辅助函数:: /* 申请一个 GPIO 信号, 同时通过特定的'flags'初始化配置, * 其他和 gpio_request()的参数和返回值相同 @@ -326,7 +320,7 @@ gpio_request()前将这类细节配置好，例如使用 pinctrl 子系统的映 将来这些标志可能扩展到支持更多的属性。 更进一步,为了更简单地声明/释放多个 GPIO,'struct gpio'被引进来封装所有 -这三个领域: +这三个领域:: struct gpio { unsigned gpio; @@ -334,7 +328,7 @@ gpio_request()前将这类细节配置好，例如使用 pinctrl 子系统的映 const char *label; }; -一个典型的用例: +一个典型的用例:: static struct gpio leds_gpios[] = { { 32, GPIOF_OUT_INIT_HIGH, "Power LED" }, /* 默认开启 */ @@ -356,9 +350,10 @@ gpio_request()前将这类细节配置好，例如使用 pinctrl 子系统的映 GPIO 映射到 IRQ --------------------- +---------------- + GPIO 编号是无符号整数;IRQ 编号也是。这些构成了两个逻辑上不同的命名空间 -(GPIO 0 不一定使用 IRQ 0)。你可以通过以下函数在它们之间实现映射: +(GPIO 0 不一定使用 IRQ 0)。你可以通过以下函数在它们之间实现映射:: /* 映射 GPIO 编号到 IRQ 编号 */ int gpio_to_irq(unsigned gpio); @@ -384,7 +379,8 @@ irq_to_gpio()返回的非错误值大多数通常可以被 gpio_get_value()所 模拟开漏信号 ----------------------------- +------------ + 有时在只有低电平信号作为实际驱动结果(译者注:多个输出连接于一点，逻辑电平 结果为所有输出的逻辑与)的时候,共享的信号线需要使用“开漏”信号。(该术语 适用于 CMOS 管；而 TTL 用“集电极开路”。)一个上拉电阻使信号为高电平。这 @@ -408,9 +404,44 @@ irq_to_gpio()返回的非错误值大多数通常可以被 gpio_get_value()所 这不一定是错误的。一个常见的例子就是 I2C 时钟的延长：一个需要较慢时钟的 从设备延迟 SCK 的上升沿，而 I2C 主设备相应地调整其信号传输速率。 +GPIO控制器和引脚控制子系统 +-------------------------- + +SOC上的GPIO控制器可能与引脚控制子系统紧密结合，即引脚可以与可选的gpio功 +能一起被其他功能使用。我们已经涵盖了这样的情况，例如一个GPIO控制器需要保 +留一个引脚或通过调用以下任何一个引脚来设置其方向:: + + pinctrl_gpio_request() + pinctrl_gpio_free() + pinctrl_gpio_direction_input() + pinctrl_gpio_direction_output() + +但是，引脚控制子系统是如何将GPIO号码（这是一个全局事项）与某个引脚控制器 +上的某个引脚交叉关联的？ + +这是通过注册引脚的“范围”来实现的，这基本上是交叉参考表。这些描述是在 +Documentation/driver-api/pin-control.rst + +虽然引脚分配完全由引脚控制子系统管理，但gpio（在gpiolib下）仍由gpio驱动 +维护。可能发生的情况是，SoC中的不同引脚范围由不同的gpio驱动器管理。 + +这使得在调用 "pinctrl_gpio_request" 之前，让gpio驱动向pin ctrl子系 +统宣布它们的引脚范围是合理的，以便在使用任何gpio之前要求引脚控制子系统准 +备相应的引脚。 + +为此，gpio控制器可以用引脚控制子系统注册其引脚范围。目前有两种方法：有或 +无DT。 + +关于对DT的支持，请参考 Documentation/devicetree/bindings/gpio/gpio.txt. + +对于非DT支持，用户可以用适当的参数调用gpiochip_add_pin_range()，将一 +系列的gpio引脚注册到引脚控制驱动上。为此，必须将引脚控制设备的名称字符串 +作为参数之一传给这个程序。 + + +这些公约忽略了什么？ +==================== -这些公约忽略了什么? -================ 这些公约忽略的最大一件事就是引脚复用，因为这属于高度芯片特定的属性且 没有可移植性。某个平台可能不需要明确的复用信息；有的对于任意给定的引脚 可能只有两个功能选项；有的可能每个引脚有八个功能选项；有的可能可以将 @@ -433,8 +464,9 @@ Linux 的系统。) 当前，动态定义 GPIO 并不是标准的，例如作为配置一个带有某些 GPIO 扩展器的 附加电路板的副作用。 -GPIO 实现者的框架 (可选) -===================== +GPIO 实现者的框架（可选） +========================= + 前面提到了，有一个可选的实现框架，让平台使用相同的编程接口，更加简单地支持 不同种类的 GPIO 控制器。这个框架称为"gpiolib"。 @@ -444,15 +476,16 @@ GPIO 实现者的框架 (可选) 控制器驱动: gpio_chip -------------------- +--------------------- + 在框架中每个 GPIO 控制器都包装为一个 "struct gpio_chip"，他包含了 该类型的每个控制器的常用信息: - - 设置 GPIO 方向的方法 - - 用于访问 GPIO 值的方法 - - 告知调用其方法是否可能休眠的标志 - - 可选的 debugfs 信息导出方法 (显示类似上拉配置一样的额外状态) - - 诊断标签 + - 设置 GPIO 方向的方法 + - 用于访问 GPIO 值的方法 + - 告知调用其方法是否可能休眠的标志 + - 可选的 debugfs 信息导出方法 (显示类似上拉配置一样的额外状态) + - 诊断标签 也包含了来自 device.platform_data 的每个实例的数据：它第一个 GPIO 的 编号和它可用的 GPIO 的数量。 @@ -471,7 +504,8 @@ GPIO 实现者的框架 (可选) 平台支持 -------- +-------- + 为了支持这个框架，一个平台的 Kconfig 文件将会 "select"(选择) ARCH_REQUIRE_GPIOLIB 或 ARCH_WANT_OPTIONAL_GPIOLIB，并让它的 包含 ，同时定义三个方法: @@ -489,7 +523,7 @@ ARCH_WANT_OPTIONAL_GPIOLIB 意味着 gpiolib 核心默认关闭,且用户可以 如果这些选项都没被选择,该平台就不通过 GPIO-lib 支持 GPIO,且代码不可以 被用户使能。 -以下这些方法的实现可以直接使用框架代码,并总是通过 gpio_chip 调度: +以下这些方法的实现可以直接使用框架代码,并总是通过 gpio_chip 调度:: #define gpio_get_value __gpio_get_value #define gpio_set_value __gpio_set_value @@ -508,7 +542,8 @@ arch_initcall()或者更早的地方集成进平台初始化代码，使这些 G 且他们通常可以作为 IRQ 使用。 板级支持 -------- +-------- + 对于外部 GPIO 控制器(例如 I2C 或 SPI 扩展器、专用芯片、多功能器件、FPGA 或 CPLD)，大多数常用板级特定代码都可以注册控制器设备，并保证他们的驱动知道 gpiochip_add()所使用的 GPIO 编号。他们的起始编号通常跟在平台特定的 GPIO @@ -526,8 +561,9 @@ GPIO 可以工作之后才可被注册。解决这类依赖的的一种方法是 设备变成无效时移除它们。 -用户空间的 Sysfs 接口(可选) -======================== +用户空间的 Sysfs 接口（可选） +============================= + 使用“gpiolib”实现框架的平台可以选择配置一个 GPIO 的 sysfs 用户接口。 这不同于 debugfs 接口，因为它提供的是对 GPIO方向和值的控制，而不只显示 一个GPIO 的状态摘要。此外,它可以出现在没有调试支持的产品级系统中。 @@ -548,6 +584,7 @@ GPIO 可以工作之后才可被注册。解决这类依赖的的一种方法是 Sysfs 中的路径 -------------- + 在/sys/class/gpio 中有 3 类入口: - 用于在用户空间控制 GPIO 的控制接口; @@ -625,8 +662,9 @@ GPIO 控制器的路径类似 /sys/class/gpio/gpiochip42/ (对于从#42 GPIO 从内核代码中导出 -------------- -内核代码可以明确地管理那些已通过 gpio_request()申请的 GPIO 的导出: +---------------- + +内核代码可以明确地管理那些已通过 gpio_request()申请的 GPIO 的导出:: /* 导出 GPIO 到用户空间 */ int gpio_export(unsigned gpio, bool direction_may_change); @@ -648,3 +686,9 @@ GPIO 控制器的路径类似 /sys/class/gpio/gpiochip42/ (对于从#42 GPIO 在 GPIO 被导出之后，gpio_export_link()允许在 sysfs 文件系统的任何地方 创建一个到这个 GPIO sysfs 节点的符号链接。这样驱动就可以通过一个描述性的 名字，在 sysfs 中他们所拥有的设备下提供一个(到这个 GPIO sysfs 节点的)接口。 + + +API参考 +======= + +本节中列出的函数已被废弃。在新的代码中应该使用基于GPIO描述符的API。 diff --git a/Documentation/translations/zh_CN/driver-api/index.rst b/Documentation/translations/zh_CN/driver-api/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..ba354e1f4e6d4ce4320f8a75f678458d4a4dd900 --- /dev/null +++ b/Documentation/translations/zh_CN/driver-api/index.rst @@ -0,0 +1,132 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/driver-api/index.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + +======================== +Linux驱动实现者的API指南 +======================== + +内核提供了各种各样的接口来支持设备驱动的开发。这份文档只是对其中一些接口进行了 +一定程度的整理——希望随着时间的推移，它能变得更好！可用的小节可以在下面看到。 + +.. class:: toc-title + + 目录列表： + +.. toctree:: + :maxdepth: 2 + + gpio/index + io_ordering + +Todolist: + +* driver-model/index +* basics +* infrastructure +* ioctl +* early-userspace/index +* pm/index +* clk +* device-io +* dma-buf +* device_link +* component +* message-based +* infiniband +* aperture +* frame-buffer +* regulator +* reset +* iio/index +* input +* usb/index +* firewire +* pci/index +* cxl/index +* spi +* i2c +* ipmb +* ipmi +* i3c/index +* interconnect +* devfreq +* hsi +* edac +* scsi +* libata +* target +* mailbox +* mtdnand +* miscellaneous +* mei/index +* mtd/index +* mmc/index +* nvdimm/index +* w1 +* rapidio/index +* s390-drivers +* vme +* 80211/index +* uio-howto +* firmware/index +* pin-control +* md/index +* media/index +* misc_devices +* nfc/index +* dmaengine/index +* slimbus +* soundwire/index +* thermal/index +* fpga/index +* acpi/index +* auxiliary_bus +* backlight/lp855x-driver.rst +* connector +* console +* dcdbas +* eisa +* isa +* isapnp +* io-mapping +* generic-counter +* memory-devices/index +* men-chameleon-bus +* ntb +* nvmem +* parport-lowlevel +* pps +* ptp +* phy/index +* pwm +* pldmfw/index +* rfkill +* serial/index +* sm501 +* surface_aggregator/index +* switchtec +* sync_file +* tty/index +* vfio-mediated-device +* vfio +* vfio-pci-device-specific-driver-acceptance +* xilinx/index +* xillybus +* zorro +* hte/index + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/translations/zh_CN/driver-api/io_ordering.rst b/Documentation/translations/zh_CN/driver-api/io_ordering.rst new file mode 100644 index 0000000000000000000000000000000000000000..4dbfa4ce92a07659d9eed8c58e0f904412ac569b --- /dev/null +++ b/Documentation/translations/zh_CN/driver-api/io_ordering.rst @@ -0,0 +1,60 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/driver-api/io_ordering.rst + +:翻译: + + 林永听 Lin Yongting + 司延腾 Yanteng Si + +:校译: + +=========================== +对内存映射地址的I/O写入排序 +=========================== + +在某些平台上，所谓的内存映射I/O是弱顺序。在这些平台上，驱动开发者有责任 +保证I/O内存映射地址的写操作按程序图意的顺序达到设备。通常读取一个“安全” +设备寄存器或桥寄存器，触发IO芯片清刷未处理的写操作到达设备后才处理读操作， +而达到保证目的。驱动程序通常在spinlock保护的临界区退出之前使用这种技术。 +这也可以保证后面的写操作只在前面的写操作之后到达设备（这非常类似于内存 +屏障操作，mb()，不过仅适用于I/O）。 + +假设一个设备驱动程的具体例子:: + + ... + CPU A: spin_lock_irqsave(&dev_lock, flags) + CPU A: val = readl(my_status); + CPU A: ... + CPU A: writel(newval, ring_ptr); + CPU A: spin_unlock_irqrestore(&dev_lock, flags) + ... + CPU B: spin_lock_irqsave(&dev_lock, flags) + CPU B: val = readl(my_status); + CPU B: ... + CPU B: writel(newval2, ring_ptr); + CPU B: spin_unlock_irqrestore(&dev_lock, flags) + ... + +上述例子中，设备可能会先接收到newval2的值，然后接收到newval的值，问题就 +发生了。不过很容易通过下面方法来修复:: + + ... + CPU A: spin_lock_irqsave(&dev_lock, flags) + CPU A: val = readl(my_status); + CPU A: ... + CPU A: writel(newval, ring_ptr); + CPU A: (void)readl(safe_register); /* 配置寄存器？*/ + CPU A: spin_unlock_irqrestore(&dev_lock, flags) + ... + CPU B: spin_lock_irqsave(&dev_lock, flags) + CPU B: val = readl(my_status); + CPU B: ... + CPU B: writel(newval2, ring_ptr); + CPU B: (void)readl(safe_register); /* 配置寄存器？*/ + CPU B: spin_unlock_irqrestore(&dev_lock, flags) + +在解决方案中，读取safe_register寄存器，触发IO芯片清刷未处理的写操作， +再处理后面的读操作，防止引发数据不一致问题。 diff --git a/Documentation/translations/zh_CN/index.rst b/Documentation/translations/zh_CN/index.rst index bf85baca8b3eb5e6bf89e63d0dca11490627a11c..ec99ef5fe99030e2623f0dff77b6045a38685ec5 100644 --- a/Documentation/translations/zh_CN/index.rst +++ b/Documentation/translations/zh_CN/index.rst @@ -26,165 +26,100 @@ 顺便说下，中文文档也需要遵守内核编码风格，风格中中文和英文的主要不同就是中文 的字符标点占用两个英文字符宽度， 所以，当英文要求不要超过每行100个字符时， 中文就不要超过50个字符。另外，也要注意'-'，'=' 等符号与相关标题的对齐。在将 -补丁提交到社区之前，一定要进行必要的checkpatch.pl检查和编译测试。 +补丁提交到社区之前，一定要进行必要的 ``checkpatch.pl`` 检查和编译测试。 -许可证文档 ----------- - -下面的文档介绍了Linux内核源代码的许可证（GPLv2）、如何在源代码树中正确标记 -单个文件的许可证、以及指向完整许可证文本的链接。 - -* Documentation/translations/zh_CN/process/license-rules.rst +与Linux 内核社区一起工作 +------------------------ -用户文档 --------- - -下面的手册是为内核用户编写的——即那些试图让它在给定系统上以最佳方式工作的 -用户。 +与内核开发社区进行协作并将工作推向上游的基本指南。 .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - admin-guide/index - -TODOList: - -* kbuild/index + process/development-process + process/submitting-patches + 行为准则 + maintainer/index + 完整开发流程文档 -固件相关文档 ------------- +内部API文档 +----------- -下列文档描述了内核需要的平台固件相关信息。 +开发人员使用的内核内部交互接口手册。 .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - devicetree/index + core-api/index + driver-api/index + 内核中的锁 TODOList: -* firmware-guide/index - -应用程序开发人员文档 --------------------- - -用户空间API手册涵盖了描述应用程序开发人员可见内核接口方面的文档。 +* subsystem-apis -TODOlist: +开发工具和流程 +-------------- -* userspace-api/index - -内核开发简介 ------------- - -这些手册包含有关如何开发内核的整体信息。内核社区非常庞大，一年下来有数千名 -开发人员做出贡献。与任何大型社区一样，知道如何完成任务将使得更改合并的过程 -变得更加容易。 +为所有内核开发人员提供有用信息的各种其他手册。 .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - process/index - dev-tools/index + process/license-rules doc-guide/index + dev-tools/index + dev-tools/testing-overview kernel-hacking/index - maintainer/index TODOList: * trace/index * fault-injection/index * livepatch/index -* rust/index -内核API文档 ------------ +面向用户的文档 +-------------- -以下手册从内核开发人员的角度详细介绍了特定的内核子系统是如何工作的。这里的 -大部分信息都是直接从内核源代码获取的，并根据需要添加补充材料（或者至少是在 -我们设法添加的时候——可能不是所有的都是有需要的）。 +下列手册针对 +希望内核在给定系统上以最佳方式工作的*用户*， +和查找内核用户空间API信息的程序开发人员。 .. toctree:: - :maxdepth: 2 + :maxdepth: 1 - core-api/index - locking/index - accounting/index - cpu-freq/index - iio/index - infiniband/index - power/index - virt/index - sound/index - filesystems/index - scheduler/index - mm/index - peci/index + admin-guide/index + admin-guide/reporting-issues.rst TODOList: -* driver-api/index -* block/index -* cdrom/index -* ide/index -* fb/index -* fpga/index -* hid/index -* i2c/index -* isdn/index -* leds/index -* netlabel/index -* networking/index -* pcmcia/index -* target/index -* timers/index -* spi/index -* w1/index -* watchdog/index -* input/index -* hwmon/index -* gpu/index -* security/index -* crypto/index -* bpf/index -* usb/index -* PCI/index -* scsi/index -* misc-devices/index -* mhi/index - -体系结构无关文档 ----------------- +* 内核构建系统 +* 用户空间工具 +* userspace-api/index -TODOList: +也可参考独立于内核文档的 `Linux 手册页 `_ 。 -* asm-annotations +固件相关文档 +------------ -特定体系结构文档 ----------------- +下列文档描述了内核需要的平台固件相关信息。 .. toctree:: :maxdepth: 2 - mips/index - arm64/index - riscv/index - openrisc/index - parisc/index - loongarch/index + devicetree/index TODOList: -* arm/index -* ia64/index -* m68k/index -* nios2/index -* powerpc/index -* s390/index -* sh/index -* sparc/index -* x86/index -* xtensa/index +* firmware-guide/index + +体系结构文档 +------------ + +.. toctree:: + :maxdepth: 2 + + arch 其他文档 -------- @@ -195,9 +130,9 @@ TODOList: TODOList: * staging/index -* watch_queue -目录和表格 + +索引和表格 ---------- * :ref:`genindex` diff --git a/Documentation/translations/zh_CN/io_ordering.txt b/Documentation/translations/zh_CN/io_ordering.txt deleted file mode 100644 index 7bb3086227aec297a78bd1eb60cc921349e38a3a..0000000000000000000000000000000000000000 --- a/Documentation/translations/zh_CN/io_ordering.txt +++ /dev/null @@ -1,67 +0,0 @@ -Chinese translated version of Documentation/driver-api/io_ordering.rst - -If you have any comment or update to the content, please contact the -original document maintainer directly. However, if you have a problem -communicating in English you can also ask the Chinese maintainer for -help. Contact the Chinese maintainer if this translation is outdated -or if there is a problem with the translation. - -Chinese maintainer: Lin Yongting ---------------------------------------------------------------------- -Documentation/driver-api/io_ordering.rst 的中文翻译 - -如果想评论或更新本文的内容，请直接联系原文档的维护者。如果你使用英文 -交流有困难的话，也可以向中文版维护者求助。如果本翻译更新不及时或者翻 -译存在问题，请联系中文版维护者。 - -中文版维护者： 林永听 Lin Yongting -中文版翻译者： 林永听 Lin Yongting -中文版校译者： 林永听 Lin Yongting - - -以下为正文 ---------------------------------------------------------------------- - -在某些平台上，所谓的内存映射I/O是弱顺序。在这些平台上，驱动开发者有责任 -保证I/O内存映射地址的写操作按程序图意的顺序达到设备。通常读取一个“安全” -设备寄存器或桥寄存器，触发IO芯片清刷未处理的写操作到达设备后才处理读操作， -而达到保证目的。驱动程序通常在spinlock保护的临界区退出之前使用这种技术。 -这也可以保证后面的写操作只在前面的写操作之后到达设备（这非常类似于内存 -屏障操作，mb()，不过仅适用于I/O）。 - -假设一个设备驱动程的具体例子： - - ... -CPU A: spin_lock_irqsave(&dev_lock, flags) -CPU A: val = readl(my_status); -CPU A: ... -CPU A: writel(newval, ring_ptr); -CPU A: spin_unlock_irqrestore(&dev_lock, flags) - ... -CPU B: spin_lock_irqsave(&dev_lock, flags) -CPU B: val = readl(my_status); -CPU B: ... -CPU B: writel(newval2, ring_ptr); -CPU B: spin_unlock_irqrestore(&dev_lock, flags) - ... - -上述例子中，设备可能会先接收到newval2的值，然后接收到newval的值，问题就 -发生了。不过很容易通过下面方法来修复： - - ... -CPU A: spin_lock_irqsave(&dev_lock, flags) -CPU A: val = readl(my_status); -CPU A: ... -CPU A: writel(newval, ring_ptr); -CPU A: (void)readl(safe_register); /* 配置寄存器？*/ -CPU A: spin_unlock_irqrestore(&dev_lock, flags) - ... -CPU B: spin_lock_irqsave(&dev_lock, flags) -CPU B: val = readl(my_status); -CPU B: ... -CPU B: writel(newval2, ring_ptr); -CPU B: (void)readl(safe_register); /* 配置寄存器？*/ -CPU B: spin_unlock_irqrestore(&dev_lock, flags) - -在解决方案中，读取safe_register寄存器，触发IO芯片清刷未处理的写操作， -再处理后面的读操作，防止引发数据不一致问题。 diff --git a/Documentation/translations/zh_CN/mm/ksm.rst b/Documentation/translations/zh_CN/mm/ksm.rst index d1f82e857ad72b7f2701ce6c7629817fefd42250..f0f458753d0cdbad89cb4a417946217b98811939 100644 --- a/Documentation/translations/zh_CN/mm/ksm.rst +++ b/Documentation/translations/zh_CN/mm/ksm.rst @@ -30,7 +30,7 @@ KSM的用户空间的接口在Documentation/translations/zh_CN/admin-guide/mm/ks KSM维护着稳定树中的KSM页的逆映射信息。 当KSM页面的共享数小于 ``max_page_sharing`` 的虚拟内存区域(VMAs)时，则代表了 -KSM页的稳定树其中的节点指向了一个rmap_item结构体类型的列表。同时，这个KSM页 +KSM页的稳定树其中的节点指向了一个ksm_rmap_item结构体类型的列表。同时，这个KSM页 的 ``page->mapping`` 指向了该稳定树节点。 如果共享数超过了阈值，KSM将给稳定树添加第二个维度。稳定树就变成链接一个或多 diff --git a/Documentation/translations/zh_CN/mm/page_owner.rst b/Documentation/translations/zh_CN/mm/page_owner.rst index b7f81d7a6589c7d541e35fed8b6e152668754baa..21a6a0837d42a8342c4db7d125ae5a7927ff2389 100644 --- a/Documentation/translations/zh_CN/mm/page_owner.rst +++ b/Documentation/translations/zh_CN/mm/page_owner.rst @@ -74,15 +74,19 @@ page owner在默认情况下是禁用的。所以，如果你想使用它，你 cat /sys/kernel/debug/page_owner > page_owner_full.txt ./page_owner_sort page_owner_full.txt sorted_page_owner.txt - ``page_owner_full.txt`` 的一般输出情况如下(输出信息无翻译价值):: + ``page_owner_full.txt`` 的一般输出情况如下:: Page allocated via order XXX, ... PFN XXX ... - // Detailed stack + // 栈详情 Page allocated via order XXX, ... PFN XXX ... - // Detailed stack + // 栈详情 + 默认情况下，它将以一个给定的pfn开始，做完整的pfn转储，且page_owner支持fseek。 + + FILE *fp = fopen("/sys/kernel/debug/page_owner", "r"); + fseek(fp, pfn_start, SEEK_SET); ``page_owner_sort`` 工具忽略了 ``PFN`` 行，将剩余的行放在buf中，使用regexp提 取页序值，计算buf的次数和页数，最后根据参数进行排序。 diff --git a/Documentation/translations/zh_CN/oops-tracing.txt b/Documentation/translations/zh_CN/oops-tracing.txt deleted file mode 100644 index c5f3bda7abcb6c89c2d0840b4c7240278b5540f3..0000000000000000000000000000000000000000 --- a/Documentation/translations/zh_CN/oops-tracing.txt +++ /dev/null @@ -1,212 +0,0 @@ -Chinese translated version of Documentation/admin-guide/bug-hunting.rst - -If you have any comment or update to the content, please contact the -original document maintainer directly. However, if you have a problem -communicating in English you can also ask the Chinese maintainer for -help. Contact the Chinese maintainer if this translation is outdated -or if there is a problem with the translation. - -Chinese maintainer: Dave Young ---------------------------------------------------------------------- -Documentation/admin-guide/bug-hunting.rst 的中文翻译 - -如果想评论或更新本文的内容，请直接联系原文档的维护者。如果你使用英文 -交流有困难的话，也可以向中文版维护者求助。如果本翻译更新不及时或者翻 -译存在问题，请联系中文版维护者。 - -中文版维护者： 杨瑞 Dave Young -中文版翻译者： 杨瑞 Dave Young -中文版校译者： 李阳 Li Yang - 王聪 Wang Cong - -以下为正文 ---------------------------------------------------------------------- - -注意： ksymoops 在2.6中是没有用的。 请以原有格式使用Oops(来自dmesg，等等)。 -忽略任何这样那样关于“解码Oops”或者“通过ksymoops运行”的文档。 如果你贴出运行过 -ksymoops的来自2.6的Oops，人们只会让你重贴一次。 - -快速总结 -------------- - -发现Oops并发送给看似相关的内核领域的维护者。别太担心对不上号。如果你不确定就发给 -和你所做的事情相关的代码的负责人。 如果可重现试着描述怎样重构。 那甚至比oops更有 -价值。 - -如果你对于发送给谁一无所知， 发给linux-kernel@vger.kernel.org。感谢你帮助Linux -尽可能地稳定。 - -Oops在哪里? ----------------------- - -通常Oops文本由klogd从内核缓冲区里读取并传给syslogd，由syslogd写到syslog文件中， -典型地是/var/log/messages(依赖于/etc/syslog.conf)。有时klogd崩溃了,这种情况下你 -能够运行dmesg > file来从内核缓冲区中读取数据并保存下来。 否则你可以 -cat /proc/kmsg > file， 然而你必须介入中止传输， kmsg是一个“永不结束的文件”。如 -果机器崩溃坏到你不能输入命令或者磁盘不可用那么你有三种选择:- - -（1） 手抄屏幕上的文本待机器重启后再输入计算机。 麻烦但如果没有针对崩溃的准备， -这是仅有的选择。 另外，你可以用数码相机把屏幕拍下来-不太好，但比没有强。 如果信 -息滚动到了终端的上面，你会发现以高分辩率启动（比如，vga=791）会让你读到更多的文 -本。（注意：这需要vesafb，所以对‘早期’的oops没有帮助） - -（2）用串口终端启动（请参看Documentation/admin-guide/serial-console.rst），运行一个null -modem到另一台机器并用你喜欢的通讯工具获取输出。Minicom工作地很好。 - -（3）使用Kdump（请参看Documentation/admin-guide/kdump/kdump.rst）， -使用在Documentation/admin-guide/kdump/gdbmacros.txt中定义的dmesg gdb宏，从旧的内存中提取内核 -环形缓冲区。 - -完整信息 ----------------- - -注意：以下来自于Linus的邮件适用于2.4内核。 我因为历史原因保留了它，并且因为其中 -一些信息仍然适用。 特别注意的是，请忽略任何ksymoops的引用。 - -From: Linus Torvalds - -怎样跟踪Oops.. [原发到linux-kernel的一封邮件] - -主要的窍门是有五年和这些烦人的oops消息打交道的经验;-) - -实际上，你有办法使它更简单。我有两个不同的方法： - - gdb /usr/src/linux/vmlinux - gdb> disassemble - -那是发现问题的简单办法，至少如果bug报告做的好的情况下（象这个一样-运行ksymoops -得到oops发生的函数及函数内的偏移）。 - -哦，如果报告发生的内核以相同的编译器和相似的配置编译它会有帮助的。 - -另一件要做的事是反汇编bug报告的“Code”部分：ksymoops也会用正确的工具来做这件事， -但如果没有那些工具你可以写一个傻程序： - - char str[] = "\xXX\xXX\xXX..."; - main(){} - -并用gcc -g编译它然后执行“disassemble str”（XX部分是由Oops报告的值-你可以仅剪切 -粘贴并用“\x”替换空格-我就是这么做的，因为我懒得写程序自动做这一切）。 - -另外，你可以用scripts/decodecode这个shell脚本。它的使用方法是： -decodecode < oops.txt - -“Code”之后的十六进制字节可能（在某些架构上）有一些当前指令之前的指令字节以及 -当前和之后的指令字节 - -Code: f9 0f 8d f9 00 00 00 8d 42 0c e8 dd 26 11 c7 a1 60 ea 2b f9 8b 50 08 a1 -64 ea 2b f9 8d 34 82 8b 1e 85 db 74 6d 8b 15 60 ea 2b f9 <8b> 43 04 39 42 54 -7e 04 40 89 42 54 8b 43 04 3b 05 00 f6 52 c0 - -最后，如果你想知道代码来自哪里，你可以： - - cd /usr/src/linux - make fs/buffer.s # 或任何产生BUG的文件 - -然后你会比gdb反汇编更清楚的知道发生了什么。 - -现在，问题是把你所拥有的所有数据结合起来：C源码（关于它应该怎样的一般知识）， -汇编代码及其反汇编得到的代码（另外还有从“oops”消息得到的寄存器状态-对了解毁坏的 -指针有用，而且当你有了汇编代码你也能拿其它的寄存器和任何它们对应的C表达式做匹配 -）。 - -实际上，你仅需看看哪里不匹配（这个例子是“Code”反汇编和编译器生成的代码不匹配）。 -然后你须要找出为什么不匹配。通常很简单-你看到代码使用了空指针然后你看代码想知道 -空指针是怎么出现的，还有检查它是否合法.. - -现在，如果明白这是一项耗时的工作而且需要一丁点儿的专心，没错。这就是我为什么大多 -只是忽略那些没有符号表信息的崩溃报告的原因：简单的说太难查找了（我有一些 -程序用于在内核代码段中搜索特定的模式，而且有时我也已经能找出那些崩溃的地方，但是 -仅仅是找出正确的序列也确实需要相当扎实的内核知识） - -_有时_会发生这种情况，我仅看到崩溃中的反汇编代码序列， 然后我马上就明白问题出在 -哪里。这时我才意识到自己干这个工作已经太长时间了;-) - - Linus - - ---------------------------------------------------------------------------- -关于Oops跟踪的注解： - -为了帮助Linus和其它内核开发者，klogd纳入了大量的支持来处理保护错误。为了拥有对 -地址解析的完整支持至少应该使用1.3-pl3的sysklogd包。 - -当保护错误发生时，klogd守护进程自动把内核日志信息中的重要地址翻译成它们相应的符 -号。 - -klogd执行两种类型的地址解析。首先是静态翻译其次是动态翻译。静态翻译和ksymoops -一样使用System.map文件。为了做静态翻译klogd守护进程必须在初始化时能找到system -map文件。关于klogd怎样搜索map文件请参看klogd手册页。 - -动态地址翻译在使用内核可装载模块时很重要。 因为内核模块的内存是从内核动态内存池 -里分配的，所以不管是模块开始位置还是模块中函数和符号的位置都不是固定的。 - -内核支持允许程序决定装载哪些模块和它们在内存中位置的系统调用。使用这些系统调用 -klogd守护进程生成一张符号表用于调试发生在可装载模块中的保护错误。 - -至少klogd会提供产生保护错误的模块名。还可有额外的符号信息供可装载模块开发者选择 -以从模块中输出符号信息。 - -因为内核模块环境可能是动态的，所以必须有一种机制当模块环境发生改变时来通知klogd -守护进程。 有一些可用的命令行选项允许klogd向当前执行中的守护进程发送信号，告知符 -号信息应该被刷新了。 更多信息请参看klogd手册页。 - -sysklogd发布时包含一个补丁修改了modules-2.0.0包，无论何时一个模块装载或者卸载都 -会自动向klogd发送信号。打上这个补丁提供了必要的对调试发生于内核可装载模块的保护 -错误的无缝支持。 - -以下是被klogd处理过的发生在可装载模块中的一个保护错误例子： ---------------------------------------------------------------------------- -Aug 29 09:51:01 blizard kernel: Unable to handle kernel paging request at virtual address f15e97cc -Aug 29 09:51:01 blizard kernel: current->tss.cr3 = 0062d000, %cr3 = 0062d000 -Aug 29 09:51:01 blizard kernel: *pde = 00000000 -Aug 29 09:51:01 blizard kernel: Oops: 0002 -Aug 29 09:51:01 blizard kernel: CPU: 0 -Aug 29 09:51:01 blizard kernel: EIP: 0010:[oops:_oops+16/3868] -Aug 29 09:51:01 blizard kernel: EFLAGS: 00010212 -Aug 29 09:51:01 blizard kernel: eax: 315e97cc ebx: 003a6f80 ecx: 001be77b edx: 00237c0c -Aug 29 09:51:01 blizard kernel: esi: 00000000 edi: bffffdb3 ebp: 00589f90 esp: 00589f8c -Aug 29 09:51:01 blizard kernel: ds: 0018 es: 0018 fs: 002b gs: 002b ss: 0018 -Aug 29 09:51:01 blizard kernel: Process oops_test (pid: 3374, process nr: 21, stackpage=00589000) -Aug 29 09:51:01 blizard kernel: Stack: 315e97cc 00589f98 0100b0b4 bffffed4 0012e38e 00240c64 003a6f80 00000001 -Aug 29 09:51:01 blizard kernel: 00000000 00237810 bfffff00 0010a7fa 00000003 00000001 00000000 bfffff00 -Aug 29 09:51:01 blizard kernel: bffffdb3 bffffed4 ffffffda 0000002b 0007002b 0000002b 0000002b 00000036 -Aug 29 09:51:01 blizard kernel: Call Trace: [oops:_oops_ioctl+48/80] [_sys_ioctl+254/272] [_system_call+82/128] -Aug 29 09:51:01 blizard kernel: Code: c7 00 05 00 00 00 eb 08 90 90 90 90 90 90 90 90 89 ec 5d c3 ---------------------------------------------------------------------------- - -Dr. G.W. Wettstein Oncology Research Div. Computing Facility -Roger Maris Cancer Center INTERNET: greg@wind.rmcc.com -820 4th St. N. -Fargo, ND 58122 -Phone: 701-234-7556 - - ---------------------------------------------------------------------------- -受污染的内核 - -一些oops报告在程序记数器之后包含字符串'Tainted: '。这表明内核已经被一些东西给污 -染了。 该字符串之后紧跟着一系列的位置敏感的字符，每个代表一个特定的污染值。 - - 1：'G'如果所有装载的模块都有GPL或相容的许可证，'P'如果装载了任何的专有模块。 -没有模块MODULE_LICENSE或者带有insmod认为是与GPL不相容的的MODULE_LICENSE的模块被 -认定是专有的。 - - 2：'F'如果有任何通过“insmod -f”被强制装载的模块，' '如果所有模块都被正常装载。 - - 3：'S'如果oops发生在SMP内核中，运行于没有证明安全运行多处理器的硬件。 当前这种 -情况仅限于几种不支持SMP的速龙处理器。 - - 4：'R'如果模块通过“insmod -f”被强制装载，' '如果所有模块都被正常装载。 - - 5：'M'如果任何处理器报告了机器检查异常，' '如果没有发生机器检查异常。 - - 6：'B'如果页释放函数发现了一个错误的页引用或者一些非预期的页标志。 - - 7：'U'如果用户或者用户应用程序特别请求设置污染标志，否则' '。 - - 8：'D'如果内核刚刚死掉，比如有OOPS或者BUG。 - -使用'Tainted: '字符串的主要原因是要告诉内核调试者，这是否是一个干净的内核亦或发 -生了任何的不正常的事。污染是永久的：即使出错的模块已经被卸载了，污染值仍然存在， -以表明内核不再值得信任。 diff --git a/Documentation/translations/zh_CN/process/coding-style.rst b/Documentation/translations/zh_CN/process/coding-style.rst index 638d714bec83cbffcc26404bbd8e072d64522861..fa28ef0a7fee81785ec89141df6eea79c27fc50d 100644 --- a/Documentation/translations/zh_CN/process/coding-style.rst +++ b/Documentation/translations/zh_CN/process/coding-style.rst @@ -1,21 +1,23 @@ .. include:: ../disclaimer-zh_CN.rst -:Original: :ref:`Documentation/process/coding-style.rst ` +:Original: Documentation/process/coding-style.rst .. _cn_codingstyle: -译者:: +:译者: + - 张乐 Zhang Le + - Andy Deng + - 吴想成 - 中文版维护者： 张乐 Zhang Le - 中文版翻译者： 张乐 Zhang Le - 中文版校译者： 王聪 Wang Cong - wheelz - 管旭东 Xudong Guan - Li Zefan - Wang Chen +:校译: + - 王聪 Wang Cong + - wheelz + - 管旭东 Xudong Guan + - Li Zefan + - Wang Chen Linux 内核代码风格 -========================= +================== 这是一个简短的文档，描述了 linux 内核的首选代码风格。代码风格是因人而异的， 而且我不愿意把自己的观点强加给任何人，但这就像我去做任何事情都必须遵循的原则 @@ -29,7 +31,7 @@ Linux 内核代码风格 1) 缩进 --------------- +------- 制表符是 8 个字符，所以缩进也是 8 个字符。有些异端运动试图将缩进变为 4 (甚至 2！) 字符深，这几乎相当于尝试将圆周率的值定义为 3。 @@ -73,6 +75,22 @@ Linux 内核代码风格 if (condition) do_this; do_something_everytime; +不要使用逗号来避免使用大括号： + +.. code-block:: c + + if (condition) + do_this(), do_that(); + +使用大括号包裹多语句： + +.. code-block:: c + + if (condition) { + do_this(); + do_that(); + } + 也不要在一行里放多个赋值语句。内核代码风格超级简单。就是避免可能导致别人误读 的表达式。 @@ -83,20 +101,25 @@ Linux 内核代码风格 2) 把长的行和字符串打散 ------------------------------- +----------------------- 代码风格的意义就在于使用平常使用的工具来维持代码的可读性和可维护性。 每一行的长度的限制是 80 列，我们强烈建议您遵守这个惯例。 长于 80 列的语句要打散成有意义的片段。除非超过 80 列能显著增加可读性，并且不 -会隐藏信息。子片段要明显短于母片段，并明显靠右。这同样适用于有着很长参数列表 -的函数头。然而，绝对不要打散对用户可见的字符串，例如 printk 信息，因为这样就 +会隐藏信息。 + +子片段要明显短于母片段，并明显靠右。一种非常常用的样式是将子体与函数左括号对齐。 + +这同样适用于有着很长参数列表的函数头。 + +然而，绝对不要打散对用户可见的字符串，例如 printk 信息，因为这样就 很难对它们 grep。 3) 大括号和空格的放置 ------------------------------- +--------------------- C 语言风格中另外一个常见问题是大括号的放置。和缩进大小不同，选择或弃用某种放 置策略并没有多少技术上的原因，不过首选的方式，就像 Kernighan 和 Ritchie 展示 @@ -132,12 +155,12 @@ C 语言风格中另外一个常见问题是大括号的放置。和缩进大小 body of function } -全世界的异端可能会抱怨这个不一致性是... 呃... 不一致的，不过所有思维健全的人 +全世界的异端可能会抱怨这个不一致性是……呃……不一致，不过所有思维健全的人 都知道 (a) K&R 是 **正确的** 并且 (b) K&R 是正确的。此外，不管怎样函数都是特 殊的 (C 函数是不能嵌套的)。 注意结束大括号独自占据一行，除非它后面跟着同一个语句的剩余部分，也就是 do 语 -句中的 "while" 或者 if 语句中的 "else"，像这样： +句中的 ``while`` 或者 if 语句中的 ``else`` ，像这样： .. code-block:: c @@ -191,7 +214,7 @@ C 语言风格中另外一个常见问题是大括号的放置。和缩进大小 } 3.1) 空格 -******************** +********* Linux 内核的空格使用方式 (主要) 取决于它是用于函数还是关键字。(大多数) 关键字 后要加一个空格。值得注意的例外是 sizeof, typeof, alignof 和 __attribute__，这 @@ -254,7 +277,7 @@ Linux 内核的空格使用方式 (主要) 取决于它是用于函数还是关 4) 命名 ------------------------------- +------- C 是一个简朴的语言，你的命名也应该这样。和 Modula-2 和 Pascal 程序员不同， C 程序员不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字。C 程序员会 @@ -275,11 +298,31 @@ C 程序员不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字 可能的话。类似的， ``tmp`` 可以用来称呼任意类型的临时变量。 如果你怕混淆了你的本地变量名，你就遇到另一个问题了，叫做函数增长荷尔蒙失衡综 -合症。请看第六章 (函数)。 +合征。请看第六章 (函数)。 +对于符号名称和文档，避免引入新的“master/slave”（或独立于“master”的“slave”） +和“blacklist/whitelist”。 + +“master/slave”推荐替换为： + '{primary,main} / {secondary,replica,subordinate}' + '{initiator,requester} / {target,responder}' + '{controller,host} / {device,worker,proxy}' + 'leader/follower' + 'director/performer' + +“blacklist/whitelist”推荐替换为： + 'denylist/allowlist' + 'blocklist/passlist' + +引入新用法的例外情况是：维护用户空间ABI/API，或更新现有（截至2020年）硬件或 +协议规范的代码时要求这些术语。对于新规范，尽可能将术语的规范用法转换为内核 +编码标准。 + +.. warning:: + 以上主从、黑白名单规则不适用于中文文档，请勿更改中文术语！ 5) Typedef ------------ +---------- 不要使用类似 ``vps_t`` 之类的东西。 @@ -308,7 +351,7 @@ C 程序员不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字 .. note:: - 不透明性和 "访问函数" 本身是不好的。我们使用 pte_t 等类型的原因在于真 + 不透明性和“访问函数”本身是不好的。我们使用 pte_t 等类型的原因在于真 的是完全没有任何共用的可访问信息。 (b) 清楚的整数类型，如此，这层抽象就可以 **帮助** 消除到底是 ``int`` 还是 @@ -353,7 +396,7 @@ C 程序员不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字 6) 函数 ------------------------------- +------- 函数应该简短而漂亮，并且只完成一件事情。函数应该可以一屏或者两屏显示完 (我们 都知道 ISO/ANSI 屏幕大小是 80x24)，只做一件事情，而且把它做好。 @@ -383,12 +426,46 @@ C 程序员不使用类似 ThisVariableIsATemporaryCounter 这样华丽的名字 } EXPORT_SYMBOL(system_is_up); -在函数原型中，包含函数名和它们的数据类型。虽然 C 语言里没有这样的要求，在 +6.1) 函数原型 +************* + +在函数原型中包含参数名和它们的数据类型。虽然 C 语言里没有这样的要求，但在 Linux 里这是提倡的做法，因为这样可以很简单的给读者提供更多的有价值的信息。 +不要在函数声明里使用 ``extern`` 关键字，因为这会导致代码行变长，并且不是严格 +必需的。 + +写函数原型时，请保持 `元素顺序规则 `_ 。 +例如下列函数声明:: + + __init void * __must_check action(enum magic value, size_t size, u8 count, + char *fmt, ...) __printf(4, 5) __malloc; + +推荐的函数原型元素顺序是： + +- 储存类型（下方的 ``static __always_inline`` ，注意 ``__always_inline`` + 技术上来讲是个属性但被当做 ``inline`` ） +- 储存类型属性（上方的 ``__init`` ——即节声明，但也像 ``__cold`` ） +- 返回类型（上方的 ``void *`` ） +- 返回类型属性（上方的 ``__must_check`` ） +- 函数名（上方的 ``action`` ） +- 函数参数（上方的 ``(enum magic value, size_t size, u8 count, char *fmt, ...)`` ， + 注意必须写上参数名） +- 函数参数属性（上方的 ``__printf(4, 5)`` ） +- 函数行为属性（上方的 ``__malloc`` ） + +请注意，对于函数 **定义** （即实际函数体），编译器不允许在函数参数之后添加函 +数参数属性。在这种情况下，它们应该跟随存储类型属性（例如，与上面的 **声明** +示例相比，请注意下面的 ``__printf(4, 5)`` 的位置发生了变化）:: + + static __always_inline __init __printf(4, 5) void * __must_check action(enum magic value, + size_t size, u8 count, char *fmt, ...) __malloc + { + ... + } 7) 集中的函数退出途径 ------------------------------- +--------------------- 虽然被某些人声称已经过时，但是 goto 语句的等价物还是经常被编译器所使用，具体 形式是无条件跳转指令。 @@ -432,7 +509,7 @@ Linux 里这是提倡的做法，因为这样可以很简单的给读者提供 return result; } -一个需要注意的常见错误是 ``一个 err 错误`` ，就像这样： +一个需要注意的常见错误是 ``单 err 错误`` ，就像这样： .. code-block:: c @@ -456,19 +533,19 @@ Linux 里这是提倡的做法，因为这样可以很简单的给读者提供 8) 注释 ------------------------------- +------- 注释是好的，不过有过度注释的危险。永远不要在注释里解释你的代码是如何运作的： 更好的做法是让别人一看你的代码就可以明白，解释写的很差的代码是浪费时间。 -一般的，你想要你的注释告诉别人你的代码做了什么，而不是怎么做的。也请你不要把 +一般来说你用注释告诉别人你的代码做了什么，而不是怎么做的。也请你不要把 注释放在一个函数体内部：如果函数复杂到你需要独立的注释其中的一部分，你很可能 需要回到第六章看一看。你可以做一些小注释来注明或警告某些很聪明 (或者槽糕) 的 做法，但不要加太多。你应该做的，是把注释放在函数的头部，告诉人们它做了什么， 也可以加上它做这些事情的原因。 -当注释内核 API 函数时，请使用 kernel-doc 格式。请看 -Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信息。 +当注释内核 API 函数时，请使用 kernel-doc 格式。详见 +Documentation/translations/zh_CN/doc-guide/index.rst 和 scripts/kernel-doc 。 长 (多行) 注释的首选风格是： @@ -500,17 +577,18 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信息。 9) 你已经把事情弄糟了 ------------------------------- +--------------------- -这没什么，我们都是这样。可能你的使用了很长时间 Unix 的朋友已经告诉你 +这没什么，我们都是这样。可能你长期使用 Unix 的朋友已经告诉你 ``GNU emacs`` 能自动帮你格式化 C 源代码，而且你也注意到了，确实是这样，不过它 所使用的默认值和我们想要的相去甚远 (实际上，甚至比随机打的还要差——无数个猴子 -在 GNU emacs 里打字永远不会创造出一个好程序) (译注：Infinite Monkey Theorem) +在 GNU emacs 里打字永远不会创造出一个好程序) +*(译注：Infinite Monkey Theorem)* 所以你要么放弃 GNU emacs，要么改变它让它使用更合理的设定。要采用后一个方案， 你可以把下面这段粘贴到你的 .emacs 文件里。 -.. code-block:: none +.. code-block:: elisp (defun c-lineup-arglist-tabs-only (ignored) "Line up argument lists by tabs, not spaces" @@ -529,7 +607,7 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信息。 (c-offsets-alist . ( (arglist-close . c-lineup-arglist-tabs-only) (arglist-cont-nonempty . - (c-lineup-gcc-asm-reg c-lineup-arglist-tabs-only)) + (c-lineup-gcc-asm-reg c-lineup-arglist-tabs-only)) (arglist-intro . +) (brace-list-intro . +) (c . c-lineup-C-comments) @@ -573,9 +651,14 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信息。 ``indent`` 有很多选项，特别是重新格式化注释的时候，你可能需要看一下它的手册。 不过记住： ``indent`` 不能修正坏的编程习惯。 +请注意，您还可以使用 ``clang-format`` 工具帮助您处理这些规则，快速自动重新格 +式化部分代码，并审阅整个文件以发现代码风格错误、打字错误和可能的改进。它还可 +以方便地排序 ``#include`` ，对齐变量/宏，重排文本和其他类似任务。 +详见 Documentation/process/clang-format.rst 。 + 10) Kconfig 配置文件 ------------------------------- +-------------------- 对于遍布源码树的所有 Kconfig* 配置文件来说，它们缩进方式有所不同。紧挨着 ``config`` 定义的行，用一个制表符缩进，然而 help 信息的缩进则额外增加 2 个空 @@ -598,11 +681,11 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信息。 depends on ADFS_FS ... -要查看配置文件的完整文档，请看 Documentation/kbuild/kconfig-language.rst。 +要查看配置文件的完整文档，请看 Documentation/kbuild/kconfig-language.rst 。 11) 数据结构 ------------------------------- +------------ 如果一个数据结构，在创建和销毁它的单线执行环境之外可见，那么它必须要有一个引 用计数器。内核里没有垃圾收集 (并且内核之外的垃圾收集慢且效率低下)，这意味着你 @@ -626,7 +709,7 @@ mm_count)，和文件系统 (``struct super_block``: s_count 和 s_active) 中 12) 宏，枚举和RTL ------------------------------- +----------------- 用于定义常量的宏的名字及枚举里的标签需要大写。 @@ -638,7 +721,7 @@ mm_count)，和文件系统 (``struct super_block``: s_count 和 s_active) 中 宏的名字请用大写字母，不过形如函数的宏的名字可以用小写字母。 -一般的，如果能写成内联函数就不要写成像函数的宏。 +通常如果能写成内联函数就不要写成像函数的宏。 含有多个语句的宏应该被包含在一个 do-while 代码块里： @@ -696,18 +779,18 @@ mm_count)，和文件系统 (``struct super_block``: s_count 和 s_active) 中 (ret); \ }) -ret 是本地变量的通用名字 - __foo_ret 更不容易与一个已存在的变量冲突。 +ret 是本地变量的通用名字—— __foo_ret 更不容易与一个已存在的变量冲突。 cpp 手册对宏的讲解很详细。gcc internals 手册也详细讲解了 RTL，内核里的汇编语 言经常用到它。 13) 打印内核消息 ------------------------------- +---------------- -内核开发者应该是受过良好教育的。请一定注意内核信息的拼写，以给人以好的印象。 +内核开发者应该看起来有文化。请一定注意内核信息的拼写，以给人良好的印象。 不要用不规范的单词比如 ``dont``，而要用 ``do not`` 或者 ``don't`` 。保证这些信 -息简单明了,无歧义。 +息简单明了、无歧义。 内核信息不必以英文句号结束。 @@ -724,17 +807,18 @@ dev_info() 等等。对于那些不和某个特定设备相关连的信息，
  • ` +.. _cn_email_clients: -译者:: +:Original: Documentation/process/email-clients.rst - 中文版维护者： 贾威威 Harry Wei - 中文版翻译者： 贾威威 Harry Wei - 时奎亮 Alex Shi - 中文版校译者： Yinglin Luan - Xiaochen Wang - yaxinsn +:译者: + - 贾威威 Harry Wei + - 时奎亮 Alex Shi + - 吴想成 Wu XiangCheng + +:校译: + - Yinglin Luan + - Xiaochen Wang + - yaxinsn Linux邮件客户端配置信息 ======================= @@ -27,12 +30,17 @@ Git 改日志。如果工作正常，再将补丁发送到相应的邮件列表。 -普通配置 +通用配置 -------- + Linux内核补丁是通过邮件被提交的，最好把补丁作为邮件体的内嵌文本。有些维护者 接收附件，但是附件的内容格式应该是"text/plain"。然而，附件一般是不赞成的， 因为这会使补丁的引用部分在评论过程中变的很困难。 +同时也强烈建议在补丁或其他邮件的正文中使用纯文本格式。https://useplaintext.email +有助于了解如何配置你喜欢的邮件客户端，并在您还没有首选的情况下列出一些推荐的 +客户端。 + 用来发送Linux内核补丁的邮件客户端在发送补丁时应该处于文本的原始状态。例如， 他们不能改变或者删除制表符或者空格，甚至是在每一行的开头或者结尾。 @@ -40,17 +48,17 @@ Linux内核补丁是通过邮件被提交的，最好把补丁作为邮件体的 不要让你的邮件客户端进行自动换行。这样也会破坏你的补丁。 -邮件客户端不能改变文本的字符集编码方式。要发送的补丁只能是ASCII或者UTF-8编码方式， -如果你使用UTF-8编码方式发送邮件，那么你将会避免一些可能发生的字符集问题。 +邮件客户端不能改变文本的字符集编码方式。要发送的补丁只能是ASCII或者UTF-8编码 +方式，如果你使用UTF-8编码方式发送邮件，那么你将会避免一些可能发生的字符集问题。 -邮件客户端应该形成并且保持 References: 或者 In-Reply-To: 标题，那么 -邮件话题就不会中断。 +邮件客户端应该生成并且保持“References:”或者“In-Reply-To:”邮件头，这样邮件会话 +就不会中断。 -复制粘帖(或者剪贴粘帖)通常不能用于补丁，因为制表符会转换为空格。使用xclipboard, xclip -或者xcutsel也许可以，但是最好测试一下或者避免使用复制粘帖。 +复制粘帖(或者剪贴粘帖)通常不能用于补丁，因为制表符会转换为空格。使用xclipboard, +xclip或者xcutsel也许可以，但是最好测试一下或者避免使用复制粘帖。 -不要在使用PGP/GPG署名的邮件中包含补丁。这样会使得很多脚本不能读取和适用于你的补丁。 -（这个问题应该是可以修复的） +不要在使用PGP/GPG签名的邮件中包含补丁。这样会使得很多脚本不能读取和适用于你的 +补丁。（这个问题应该是可以修复的） 在给内核邮件列表发送补丁之前，给自己发送一个补丁是个不错的主意，保存接收到的 邮件，将补丁用'patch'命令打上，如果成功了，再给内核邮件列表发送。 @@ -58,98 +66,133 @@ Linux内核补丁是通过邮件被提交的，最好把补丁作为邮件体的 一些邮件客户端提示 ------------------ + 这里给出一些详细的MUA配置提示，可以用于给Linux内核发送补丁。这些并不意味是 所有的软件包配置总结。 说明： -TUI = 以文本为基础的用户接口 -GUI = 图形界面用户接口 + +- TUI = 以文本为基础的用户接口 +- GUI = 图形界面用户接口 Alpine (TUI) -~~~~~~~~~~~~ +************ 配置选项： -在"Sending Preferences"部分： -- "Do Not Send Flowed Text"必须开启 -- "Strip Whitespace Before Sending"必须关闭 +在 :menuselection:`Sending Preferences` 菜单： + +- :menuselection:`Do Not Send Flowed Text` 必须开启 +- :menuselection:`Strip Whitespace Before Sending` 必须关闭 + +当写邮件时，光标应该放在补丁会出现的地方，然后按下 :kbd:`CTRL-R` 组合键，使指 +定的补丁文件嵌入到邮件中。 + +Claws Mail (GUI) +**************** + +可以用，有人用它成功地发过补丁。 -当写邮件时，光标应该放在补丁会出现的地方，然后按下CTRL-R组合键，使指定的 -补丁文件嵌入到邮件中。 +用 :menuselection:`Message-->Insert File` (:kbd:`CTRL-I`) 或外置编辑器插入补丁。 + +若要在Claws编辑窗口重修改插入的补丁，需关闭 +:menuselection:`Configuration-->Preferences-->Compose-->Wrapping` +的 `Auto wrapping` 。 Evolution (GUI) -~~~~~~~~~~~~~~~ +*************** -一些开发者成功的使用它发送补丁 +一些开发者成功的使用它发送补丁。 -当选择邮件选项：Preformat - 从Format->Heading->Preformatted (Ctrl-7)或者工具栏 +撰写邮件时： +从 :menuselection:`格式-->段落样式-->预格式化` (:kbd:`CTRL-7`) +或工具栏选择 :menuselection:`预格式化` ； 然后使用： - Insert->Text File... (Alt-n x)插入补丁文件。 +:menuselection:`插入-->文本文件...` (:kbd:`ALT-N x`) 插入补丁文件。 -你还可以"diff -Nru old.c new.c | xclip"，选择Preformat，然后使用中间键进行粘帖。 +你还可以 ``diff -Nru old.c new.c | xclip`` ，选择 :menuselection:`预格式化` ， +然后使用鼠标中键进行粘帖。 Kmail (GUI) -~~~~~~~~~~~ +*********** 一些开发者成功的使用它发送补丁。 -默认设置不为HTML格式是合适的；不要启用它。 +默认撰写设置禁用HTML格式是合适的；不要启用它。 -当书写一封邮件的时候，在选项下面不要选择自动换行。唯一的缺点就是你在邮件中输入的任何文本 -都不会被自动换行，因此你必须在发送补丁之前手动换行。最简单的方法就是启用自动换行来书写邮件， -然后把它保存为草稿。一旦你在草稿中再次打开它，它已经全部自动换行了，那么你的邮件虽然没有 -选择自动换行，但是还不会失去已有的自动换行。 +当书写一封邮件的时候，在选项下面不要选择自动换行。唯一的缺点就是你在邮件中输 +入的任何文本都不会被自动换行，因此你必须在发送补丁之前手动换行。最简单的方法 +就是启用自动换行来书写邮件，然后把它保存为草稿。一旦你在草稿中再次打开它，它 +已经全部自动换行了，那么你的邮件虽然没有选择自动换行，但是还不会失去已有的自 +动换行。 -在邮件的底部，插入补丁之前，放上常用的补丁定界符：三个连字号(---)。 +在邮件的底部，插入补丁之前，放上常用的补丁定界符：三个连字符(``---``)。 -然后在"Message"菜单条目，选择插入文件，接着选取你的补丁文件。还有一个额外的选项，你可以 -通过它配置你的邮件建立工具栏菜单，还可以带上"insert file"图标。 +然后在 :menuselection:`信件` 菜单，选择 :menuselection:`插入文本文件` ，接 +着选取你的补丁文件。还有一个额外的选项，你可以通过它配置你的创建新邮件工具栏， +加上 :menuselection:`插入文本文件` 图标。 -你可以安全地通过GPG标记附件，但是内嵌补丁最好不要使用GPG标记它们。作为内嵌文本的签发补丁， -当从GPG中提取7位编码时会使他们变的更加复杂。 +将编辑器窗口拉到足够宽避免折行。对于KMail 1.13.5 (KDE 4.5.4)，它会在发送邮件 +时对编辑器窗口中显示折行的地方自动换行。在选项菜单中取消自动换行仍不能解决。 +因此，如果你的补丁中有非常长的行，必须在发送之前把编辑器窗口拉得非常宽。 +参见：https://bugs.kde.org/show_bug.cgi?id=174034 -如果你非要以附件的形式发送补丁，那么就右键点击附件，然后选中属性，突出"Suggest automatic -display"，这样内嵌附件更容易让读者看到。 +你可以安全地用GPG签名附件，但是内嵌补丁最好不要使用GPG签名它们。作为内嵌文本 +插入的签名补丁将使其难以从7-bit编码中提取。 -当你要保存将要发送的内嵌文本补丁，你可以从消息列表窗格选择包含补丁的邮件，然后右击选择 -"save as"。你可以使用一个没有更改的包含补丁的邮件，如果它是以正确的形式组成。当你正真在它 -自己的窗口之下察看，那时没有选项可以保存邮件--已经有一个这样的bug被汇报到了kmail的bugzilla -并且希望这将会被处理。邮件是以只针对某个用户可读写的权限被保存的，所以如果你想把邮件复制到其他地方， -你不得不把他们的权限改为组或者整体可读。 +如果你非要以附件的形式发送补丁，那么就右键点击附件，然后选择 +:menuselection:`属性` ，打开 :menuselection:`建议自动显示` ，使附件内联更容 +易让读者看到。 + +当你要保存将要发送的内嵌文本补丁，你可以从消息列表窗格选择包含补丁的邮件，然 +后右键选择 :menuselection:`另存为` 。如果整个电子邮件的组成正确，您可直接将 +其作为补丁使用。电子邮件以当前用户可读写权限保存，因此您必须 ``chmod`` ，以 +使其在复制到别处时用户组和其他人可读。 Lotus Notes (GUI) -~~~~~~~~~~~~~~~~~ +***************** 不要使用它。 +IBM Verse (Web GUI) +******************* + +同上条。 + Mutt (TUI) -~~~~~~~~~~ +********** + +很多Linux开发人员使用mutt客户端，这证明它肯定工作得非常漂亮。 -很多Linux开发人员使用mutt客户端，所以证明它肯定工作的非常漂亮。 +Mutt不自带编辑器，所以不管你使用什么编辑器，不自动断行就行。大多数编辑器都有 +:menuselection:`插入文件` 选项，它可以在不改变文件内容的情况下插入文件。 -Mutt不自带编辑器，所以不管你使用什么编辑器都不应该带有自动断行。大多数编辑器都带有 -一个"insert file"选项，它可以通过不改变文件内容的方式插入文件。 +用 ``vim`` 作为mutt的编辑器:: -'vim'作为mutt的编辑器： set editor="vi" - 如果使用xclip，敲入以下命令 +如果使用xclip，敲入以下命令:: + :set paste - 按中键之前或者shift-insert或者使用 + +然后再按中键或者shift-insert或者使用:: + :r filename -如果想要把补丁作为内嵌文本。 -(a)ttach工作的很好，不带有"set paste"。 +把补丁插入为内嵌文本。 +在未设置 ``set paste`` 时(a)ttach工作的很好。 你可以通过 ``git format-patch`` 生成补丁，然后用 Mutt发送它们:: - $ mutt -H 0001-some-bug-fix.patch + $ mutt -H 0001-some-bug-fix.patch 配置选项： + 它应该以默认设置的形式工作。 -然而，把"send_charset"设置为"us-ascii::utf-8"也是一个不错的主意。 +然而，把 ``send_charset`` 设置一下也是一个不错的主意:: + + set send_charset="us-ascii:utf-8" Mutt 是高度可配置的。 这里是个使用mutt通过 Gmail 发送的补丁的最小配置:: @@ -178,71 +221,107 @@ Mutt 是高度可配置的。 这里是个使用mutt通过 Gmail 发送的补丁 set from = "username@gmail.com" set use_from = yes -Mutt文档含有更多信息: +Mutt文档含有更多信息： - http://dev.mutt.org/trac/wiki/UseCases/Gmail + https://gitlab.com/muttmua/mutt/-/wikis/UseCases/Gmail - http://dev.mutt.org/doc/manual.html + http://www.mutt.org/doc/manual/ Pine (TUI) -~~~~~~~~~~ +********** Pine过去有一些空格删减问题，但是这些现在应该都被修复了。 -如果可以，请使用alpine(pine的继承者) +如果可以，请使用alpine（pine的继承者）。 配置选项： -- 最近的版本需要消除流程文本 -- "no-strip-whitespace-before-send"选项也是需要的。 + +- 最近的版本需要 ``quell-flowed-text`` +- ``no-strip-whitespace-before-send`` 选项也是需要的。 Sylpheed (GUI) -~~~~~~~~~~~~~~ +************** - 内嵌文本可以很好的工作（或者使用附件）。 - 允许使用外部的编辑器。 -- 对于目录较多时非常慢。 +- 收件箱较多时非常慢。 - 如果通过non-SSL连接，无法使用TLS SMTP授权。 -- 在组成窗口中有一个很有用的ruler bar。 -- 给地址本中添加地址就不会正确的了解显示名。 +- 撰写窗口的标尺很有用。 +- 将地址添加到通讯簿时无法正确理解显示的名称。 Thunderbird (GUI) -~~~~~~~~~~~~~~~~~ +***************** + +Thunderbird是Outlook的克隆版本，它很容易损坏文本，但也有一些方法强制修正。 + +在完成修改后（包括安装扩展），您需要重新启动Thunderbird。 + +- 允许使用外部编辑器： + + 使用Thunderbird发补丁最简单的方法是使用扩展来打开您最喜欢的外部编辑器。 + + 下面是一些能够做到这一点的扩展样例。 + + - “External Editor Revived” + + https://github.com/Frederick888/external-editor-revived + + https://addons.thunderbird.net/en-GB/thunderbird/addon/external-editor-revived/ + + 它需要安装“本地消息主机（native messaging host）”。 + 参见以下文档: + https://github.com/Frederick888/external-editor-revived/wiki + + - “External Editor” + + https://github.com/exteditor/exteditor + + 下载并安装此扩展，然后打开 :menuselection:`新建消息` 窗口, 用 + :menuselection:`查看-->工具栏-->自定义...` 给它增加一个按钮，直接点击此 + 按钮即可使用外置编辑器。 + + 请注意，“External Editor”要求你的编辑器不能fork，换句话说，编辑器必须在 + 关闭前不返回。你可能需要传递额外的参数或修改编辑器设置。最值得注意的是， + 如果您使用的是gvim，那么您必须将 :menuselection:`external editor` 设置的 + 编辑器字段设置为 ``/usr/bin/gvim --nofork"`` （假设可执行文件在 + ``/usr/bin`` ），以传递 ``-f`` 参数。如果您正在使用其他编辑器，请阅读其 + 手册了解如何处理。 -默认情况下，thunderbird很容易损坏文本，但是还有一些方法可以强制它变得更好。 +若要修正内部编辑器，请执行以下操作： -- 在用户帐号设置里，组成和寻址，不要选择"Compose messages in HTML format"。 +- 修改你的Thunderbird设置，不要使用 ``format=flowed`` ！ + 回到主窗口，按照 + :menuselection:`主菜单-->首选项-->常规-->配置编辑器...` + 打开Thunderbird的配置编辑器。 -- 编辑你的Thunderbird配置设置来使它不要拆行使用：user_pref("mailnews.wraplength", 0); + - 将 ``mailnews.send_plaintext_flowed`` 设为 ``false`` -- 编辑你的Thunderbird配置设置，使它不要使用"format=flowed"格式：user_pref("mailnews. - send_plaintext_flowed", false); + - 将 ``mailnews.wraplength`` 从 ``72`` 改为 ``0`` -- 你需要使Thunderbird变为预先格式方式： - 如果默认情况下你书写的是HTML格式，那不是很难。仅仅从标题栏的下拉框中选择"Preformat"格式。 - 如果默认情况下你书写的是文本格式，你不得把它改为HTML格式（仅仅作为一次性的）来书写新的消息， - 然后强制使它回到文本格式，否则它就会拆行。要实现它，在写信的图标上使用shift键来使它变为HTML - 格式，然后标题栏的下拉框中选择"Preformat"格式。 +- 不要写HTML邮件！ + 回到主窗口，打开 + :menuselection:`主菜单-->账户设置-->你的@邮件.地址-->通讯录/编写&地址簿` ， + 关掉 ``以HTML格式编写消息`` 。 -- 允许使用外部的编辑器： - 针对Thunderbird打补丁最简单的方法就是使用一个"external editor"扩展，然后使用你最喜欢的 - $EDITOR来读取或者合并补丁到文本中。要实现它，可以下载并且安装这个扩展，然后添加一个使用它的 - 按键View->Toolbars->Customize...最后当你书写信息的时候仅仅点击它就可以了。 +- 只用纯文本格式查看邮件！ + 回到主窗口， :menuselection:`主菜单-->查看-->消息体为-->纯文本` ！ TkRat (GUI) -~~~~~~~~~~~ +*********** 可以使用它。使用"Insert file..."或者外部的编辑器。 Gmail (Web GUI) -~~~~~~~~~~~~~~~ +*************** 不要使用它发送补丁。 Gmail网页客户端自动地把制表符转换为空格。 -虽然制表符转换为空格问题可以被外部编辑器解决，同时它还会使用回车换行把每行拆分为78个字符。 +虽然制表符转换为空格问题可以被外部编辑器解决，但它同时还会使用回车换行把每行 +拆分为78个字符。 -另一个问题是Gmail还会把任何不是ASCII的字符的信息改为base64编码。它把东西变的像欧洲人的名字。 +另一个问题是Gmail还会把任何含有非ASCII的字符的消息改用base64编码，如欧洲人的 +名字。 - ### diff --git a/Documentation/translations/zh_CN/process/howto.rst b/Documentation/translations/zh_CN/process/howto.rst index 1455190dc087af9616dcbd74debcb4fdcf07d8d0..5bf953146929f00245489c6594343c8bba95657a 100644 --- a/Documentation/translations/zh_CN/process/howto.rst +++ b/Documentation/translations/zh_CN/process/howto.rst @@ -306,7 +306,7 @@ bugzilla.kernel.org是Linux内核开发者们用来跟踪内核Bug的网站。 网上很多地方都有这个邮件列表的存档(archive)。可以使用搜索引擎来找到这些 存档。比如： - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ 在发信之前，我们强烈建议你先在存档中搜索你想要讨论的问题。很多已经被详细 讨论过的问题只在邮件列表的存档中可以找到。 diff --git a/Documentation/translations/zh_CN/process/index.rst b/Documentation/translations/zh_CN/process/index.rst index a683dbea0c83a107b391614af903c49d3096afaa..a1a35f88f4ae08ba2ac1b3b7079cf106293a50a7 100644 --- a/Documentation/translations/zh_CN/process/index.rst +++ b/Documentation/translations/zh_CN/process/index.rst @@ -10,6 +10,7 @@ .. _cn_process_index: +======================== 与Linux 内核社区一起工作 ======================== diff --git a/Documentation/translations/zh_CN/process/magic-number.rst b/Documentation/translations/zh_CN/process/magic-number.rst index 42f0635ca70a54ba803cdd2a15b1e5847c000b18..0617ce125e127b0fed587c2b043fa331b8a50571 100644 --- a/Documentation/translations/zh_CN/process/magic-number.rst +++ b/Documentation/translations/zh_CN/process/magic-number.rst @@ -58,87 +58,17 @@ Linux 魔术数 魔术数名 数字 结构 文件 ===================== ================ ======================== ========================================== PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/pg.h`` -CMAGIC 0x0111 user ``include/linux/a.out.h`` -MKISS_DRIVER_MAGIC 0x04bf mkiss_channel ``drivers/net/mkiss.h`` -HDLC_MAGIC 0x239e n_hdlc ``drivers/char/n_hdlc.c`` APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` -DB_MAGIC 0x4442 fc_info ``drivers/net/iph5526_novram.c`` -DL_MAGIC 0x444d fc_info ``drivers/net/iph5526_novram.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` -FF_MAGIC 0x4646 fc_info ``drivers/net/iph5526_novram.c`` -PTY_MAGIC 0x5001 ``drivers/char/pty.c`` -PPP_MAGIC 0x5002 ppp ``include/linux/if_pppvar.h`` -SSTATE_MAGIC 0x5302 serial_state ``include/linux/serial.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -STRIP_MAGIC 0x5303 strip ``drivers/net/strip.c`` -SIXPACK_MAGIC 0x5304 sixpack ``drivers/net/hamradio/6pack.h`` -AX25_MAGIC 0x5316 ax_disp ``drivers/net/mkiss.h`` -TTY_MAGIC 0x5401 tty_struct ``include/linux/tty.h`` -MGSL_MAGIC 0x5401 mgsl_info ``drivers/char/synclink.c`` -TTY_DRIVER_MAGIC 0x5402 tty_driver ``include/linux/tty_driver.h`` MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` -USB_SERIAL_MAGIC 0x6702 usb_serial ``drivers/usb/serial/usb-serial.h`` -FULL_DUPLEX_MAGIC 0x6969 ``drivers/net/ethernet/dec/tulip/de2104x.c`` -USB_BLUETOOTH_MAGIC 0x6d02 usb_bluetooth ``drivers/usb/class/bluetty.c`` -RFCOMM_TTY_MAGIC 0x6d02 ``net/bluetooth/rfcomm/tty.c`` -USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/serial/usb-serial.h`` -CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` -LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -GDTIOCTL_MAGIC 0x06030f07 gdth_iowr_str ``drivers/scsi/gdth_ioctl.h`` -RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` -NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` -RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` -ISDN_X25IFACE_MAGIC 0x1e75a2b9 isdn_x25iface_proto_data ``drivers/isdn/isdn_x25iface.h`` -ECP_MAGIC 0x21504345 cdkecpsig ``include/linux/cdk.h`` -LSOMAGIC 0x27091997 lso ``drivers/fc4/fc.c`` -LSMAGIC 0x2a3b4d2a ls ``drivers/fc4/fc.c`` -WANPIPE_MAGIC 0x414C4453 sdla_{dump,exec} ``include/linux/wanpipe.h`` -CS_CARD_MAGIC 0x43525553 cs_card ``sound/oss/cs46xx.c`` -LABELCL_MAGIC 0x4857434c labelcl_info_s ``include/asm/ia64/sn/labelcl.h`` -ISDN_ASYNC_MAGIC 0x49344C01 modem_info ``include/linux/isdn.h`` -CTC_ASYNC_MAGIC 0x49344C01 ctc_tty_info ``drivers/s390/net/ctctty.c`` -ISDN_NET_MAGIC 0x49344C02 isdn_net_local_s ``drivers/isdn/i4l/isdn_net_lib.h`` -SAVEKMSG_MAGIC2 0x4B4D5347 savekmsg ``arch/*/amiga/config.c`` -CS_STATE_MAGIC 0x4c4f4749 cs_state ``sound/oss/cs46xx.c`` -SLAB_C_MAGIC 0x4f17a36d kmem_cache ``mm/slab.c`` -COW_MAGIC 0x4f4f4f4d cow_header_v1 ``arch/um/drivers/ubd_user.c`` -I810_CARD_MAGIC 0x5072696E i810_card ``sound/oss/i810_audio.c`` -TRIDENT_CARD_MAGIC 0x5072696E trident_card ``sound/oss/trident.c`` -ROUTER_MAGIC 0x524d4157 wan_device [in ``wanrouter.h`` pre 3.9] -SAVEKMSG_MAGIC1 0x53415645 savekmsg ``arch/*/amiga/config.c`` -GDA_MAGIC 0x58464552 gda ``arch/mips/include/asm/sn/gda.h`` -RED_MAGIC1 0x5a2cf071 (any) ``mm/slab.c`` -EEPROM_MAGIC_VALUE 0x5ab478d2 lanai_dev ``drivers/atm/lanai.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` -PCXX_MAGIC 0x5c6df104 channel ``drivers/char/pcxx.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` -I810_STATE_MAGIC 0x63657373 i810_state ``sound/oss/i810_audio.c`` -TRIDENT_STATE_MAGIC 0x63657373 trient_state ``sound/oss/trident.c`` -M3_CARD_MAGIC 0x646e6f50 m3_card ``sound/oss/maestro3.c`` -FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fore200e.h`` -SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` -SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` -LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` -VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` -KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` -PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/media/pwc.h`` -NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` -ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` -DPMEM_MAGIC 0xc0ffee11 gdt_pci_sram ``drivers/scsi/gdth.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` QUEUE_MAGIC_USED 0xf7e1cc33 queue_entry ``drivers/scsi/arm/queue.c`` -HTB_CMAGIC 0xFEFAFEF1 htb_class ``net/sched/sch_htb.c`` NMI_MAGIC 0x48414d4d455201 nmi_s ``arch/mips/include/asm/sn/nmi.h`` ===================== ================ ======================== ========================================== - - -请注意，在声音记忆管理中仍然有一些特殊的为每个驱动定义的魔术值。查看include/sound/sndmagic.h来获取他们完整的列表信息。很多OSS声音驱动拥有自己从声卡PCI ID构建的魔术值-他们也没有被列在这里。 - -IrDA子系统也使用了大量的自己的魔术值，查看include/net/irda/irda.h来获取他们完整的信息。 - -HFS是另外一个比较大的使用魔术值的文件系统-你可以在fs/hfs/hfs.h中找到他们。 diff --git a/Documentation/translations/zh_CN/process/submit-checklist.rst b/Documentation/translations/zh_CN/process/submit-checklist.rst index a64858d321fcdc6ce9496ffc441e2777d3567201..3d6ee21c74ae07f1f19fc4e1f7b7e9f0bd5484c0 100644 --- a/Documentation/translations/zh_CN/process/submit-checklist.rst +++ b/Documentation/translations/zh_CN/process/submit-checklist.rst @@ -1,105 +1,111 @@ .. include:: ../disclaimer-zh_CN.rst -:Original: :ref:`Documentation/process/submit-checklist.rst ` -:Translator: Alex Shi +:Original: Documentation/process/submit-checklist.rst +:Translator: + - Alex Shi + - Wu XiangCheng .. _cn_submitchecklist: -Linux内核补丁提交清单 -~~~~~~~~~~~~~~~~~~~~~ +Linux内核补丁提交检查单 +~~~~~~~~~~~~~~~~~~~~~~~ 如果开发人员希望看到他们的内核补丁提交更快地被接受，那么他们应该做一些基本 的事情。 -这些都是在 -:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst ` +这些都是在 Documentation/translations/zh_CN/process/submitting-patches.rst 和其他有关提交Linux内核补丁的文档中提供的。 -1) 如果使用工具，则包括定义/声明该工具的文件。不要依赖于其他头文件拉入您使用 +1) 如果使用工具，则包括定义/声明该工具的文件。不要依赖其他头文件来引入您使用 的头文件。 2) 干净的编译： - a) 使用适用或修改的 ``CONFIG`` 选项 ``=y``、``=m`` 和 ``=n`` 。没有GCC + a) 使用合适的 ``CONFIG`` 选项 ``=y``、``=m`` 和 ``=n`` 。没有 ``gcc`` 警告/错误，没有链接器警告/错误。 - b) 通过allnoconfig、allmodconfig + b) 通过 ``allnoconfig`` 、 ``allmodconfig`` c) 使用 ``O=builddir`` 时可以成功编译 -3) 通过使用本地交叉编译工具或其他一些构建场在多个CPU体系结构上构建。 + d) 任何 Doucmentation/ 下的变更都能成功构建且不引入新警告/错误。 + 用 ``make htmldocs`` 或 ``make pdfdocs`` 检验构建情况并修复问题。 + +3) 通过使用本地交叉编译工具或其他一些构建设施在多个CPU体系结构上构建。 4) PPC64是一种很好的交叉编译检查体系结构，因为它倾向于对64位的数使用无符号 长整型。 -5) 如下所述 :ref:`Documentation/translations/zh_CN/process/coding-style.rst `. - 检查您的补丁是否为常规样式。在提交（ ``scripts/check patch.pl`` ）之前， - 使用补丁样式检查器检查是否有轻微的冲突。您应该能够处理您的补丁中存在的所有 +5) 按 Documentation/translations/zh_CN/process/coding-style.rst 所述检查您的 + 补丁是否为常规样式。在提交之前使用补丁样式检查器 ``scripts/checkpatch.pl`` + 检查是否有轻微的冲突。您应该能够处理您的补丁中存在的所有 违规行为。 -6) 任何新的或修改过的 ``CONFIG`` 选项都不会弄脏配置菜单，并默认为关闭，除非 - 它们符合 ``Documentation/kbuild/kconfig-language.rst`` 中记录的异常条件, - 菜单属性：默认值. +6) 任何新的或修改过的 ``CONFIG`` 选项都不应搞乱配置菜单，并默认为关闭，除非 + 它们符合 ``Documentation/kbuild/kconfig-language.rst`` 菜单属性：默认值中 + 记录的例外条件。 7) 所有新的 ``kconfig`` 选项都有帮助文本。 8) 已仔细审查了相关的 ``Kconfig`` 组合。这很难用测试来纠正——脑力在这里是有 回报的。 -9) 用 sparse 检查干净。 +9) 通过 sparse 清查。 + （参见 Documentation/translations/zh_CN/dev-tools/sparse.rst ） 10) 使用 ``make checkstack`` 和 ``make namespacecheck`` 并修复他们发现的任何 问题。 .. note:: - ``checkstack`` 并没有明确指出问题，但是任何一个在堆栈上使用超过512 + ``checkstack`` 并不会明确指出问题，但是任何一个在堆栈上使用超过512 字节的函数都可以进行更改。 -11) 包括 :ref:`kernel-doc ` 内核文档以记录全局内核API。（静态函数 - 不需要，但也可以。）使用 ``make htmldocs`` 或 ``make pdfdocs`` 检查 - :ref:`kernel-doc ` 并修复任何问题。 +11) 包括 :ref:`kernel-doc ` 内核文档以记录全局内核API。（静态 + 函数不需要，但也可以。）使用 ``make htmldocs`` 或 ``make pdfdocs`` 检查 + :ref:`kernel-doc ` 并修复任何问题。 -12) 通过以下选项同时启用的测试 ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``, +12) 通过以下选项同时启用的测试： ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``, ``CONFIG_DEBUG_SLAB``, ``CONFIG_DEBUG_PAGEALLOC``, ``CONFIG_DEBUG_MUTEXES``, ``CONFIG_DEBUG_SPINLOCK``, ``CONFIG_DEBUG_ATOMIC_SLEEP``, - ``CONFIG_PROVE_RCU`` and ``CONFIG_DEBUG_OBJECTS_RCU_HEAD`` - -13) 已经过构建和运行时测试，包括有或没有 ``CONFIG_SMP``, ``CONFIG_PREEMPT``. + ``CONFIG_PROVE_RCU`` 和 ``CONFIG_DEBUG_OBJECTS_RCU_HEAD`` 。 -14) 如果补丁程序影响IO/磁盘等：使用或不使用 ``CONFIG_LBDAF`` 进行测试。 +13) 在 ``CONFIG_SMP``, ``CONFIG_PREEMPT`` 开启和关闭的情况下都进行构建和运行 + 时测试。 -15) 所有代码路径都已在启用所有lockdep功能的情况下运行。 +14) 所有代码路径都已在启用所有死锁检测（lockdep）功能的情况下运行。 -16) 所有新的/proc条目都记录在 ``Documentation/`` +15) 所有新的 ``/proc`` 条目都记录在 ``Documentation/`` -17) 所有新的内核引导参数都记录在 +16) 所有新的内核引导参数都记录在 Documentation/admin-guide/kernel-parameters.rst 中。 -18) 所有新的模块参数都记录在 ``MODULE_PARM_DESC()`` +17) 所有新的模块参数都记录在 ``MODULE_PARM_DESC()`` -19) 所有新的用户空间接口都记录在 ``Documentation/ABI/`` 中。有关详细信息， +18) 所有新的用户空间接口都记录在 ``Documentation/ABI/`` 中。有关详细信息， 请参阅 ``Documentation/ABI/README`` 。更改用户空间接口的补丁应该抄送 linux-api@vger.kernel.org。 -20) 已通过至少注入slab和page分配失败进行检查。请参阅 ``Documentation/fault-injection/`` +19) 已通过至少注入slab和page分配失败进行检查。请参阅 ``Documentation/fault-injection/`` 。 如果新代码是实质性的，那么添加子系统特定的故障注入可能是合适的。 -21) 新添加的代码已经用 ``gcc -W`` 编译（使用 ``make EXTRA-CFLAGS=-W`` ）。这 +20) 新添加的代码已经用 ``gcc -W`` 编译（使用 ``make EXTRA-CFLAGS=-W`` ）。这 将产生大量噪声，但对于查找诸如“警告：有符号和无符号之间的比较”之类的错误 很有用。 -22) 在它被合并到-mm补丁集中之后进行测试，以确保它仍然与所有其他排队的补丁以 +21) 在它被合并到-mm补丁集中之后进行测试，以确保它仍然与所有其他排队的补丁以 及VM、VFS和其他子系统中的各种更改一起工作。 -23) 所有内存屏障例如 ``barrier()``, ``rmb()``, ``wmb()`` 都需要源代码中的注 +22) 所有内存屏障（例如 ``barrier()``, ``rmb()``, ``wmb()`` ）都需要源代码注 释来解释它们正在执行的操作及其原因的逻辑。 -24) 如果补丁添加了任何ioctl，那么也要更新 ``Documentation/userspace-api/ioctl/ioctl-number.rst`` +23) 如果补丁添加了任何ioctl，那么也要更新 + ``Documentation/userspace-api/ioctl/ioctl-number.rst`` 。 -25) 如果修改后的源代码依赖或使用与以下 ``Kconfig`` 符号相关的任何内核API或 +24) 如果修改后的源代码依赖或使用与以下 ``Kconfig`` 符号相关的任何内核API或 功能，则在禁用相关 ``Kconfig`` 符号和/或 ``=m`` （如果该选项可用）的情况 下测试以下多个构建[并非所有这些都同时存在，只是它们的各种/随机组合]： - ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``, ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``, - ``CONFIG_NET``, ``CONFIG_INET=n`` (但是后者伴随 ``CONFIG_NET=y``). + ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``, + ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``, + ``CONFIG_NET``, ``CONFIG_INET=n`` （但是最后一个需要 ``CONFIG_NET=y`` ）。 diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst index ebb7f37575c116dc1c59b717773fa77d7d09960c..f8978f02057c164c1bf31ea3b1989c412e6083bf 100644 --- a/Documentation/translations/zh_CN/process/submitting-patches.rst +++ b/Documentation/translations/zh_CN/process/submitting-patches.rst @@ -1,142 +1,92 @@ -.. _cn_submittingpatches: +.. SPDX-License-Identifier: GPL-2.0-or-later .. include:: ../disclaimer-zh_CN.rst -:Original: :ref:`Documentation/process/submitting-patches.rst ` +.. _cn_submittingpatches: + +:Original: Documentation/process/submitting-patches.rst -译者:: +:译者: + - 钟宇 TripleX Chung + - 时奎亮 Alex Shi + - 吴想成 Wu XiangCheng - 中文版维护者： 钟宇 TripleX Chung - 中文版翻译者： 钟宇 TripleX Chung - 时奎亮 Alex Shi - 中文版校译者： 李阳 Li Yang - 王聪 Wang Cong +:校译: + - 李阳 Li Yang + - 王聪 Wang Cong -如何让你的改动进入内核 -====================== +提交补丁：如何让你的改动进入内核 +================================ 对于想要将改动提交到 Linux 内核的个人或者公司来说，如果不熟悉“规矩”， -提交的流程会让人畏惧。本文档收集了一系列建议，这些建议可以大大的提高你 +提交的流程会让人畏惧。本文档包含了一系列建议，可以大大提高你 的改动被接受的机会. -以下文档含有大量简洁的建议， 具体请见： -:ref:`Documentation/process ` -同样，:ref:`Documentation/translations/zh_CN/process/submit-checklist.rst ` -给出在提交代码前需要检查的项目的列表。 +本文档以较为简洁的行文给出了大量建议。关于内核开发流程如何进行的详细信息， +参见： Documentation/translations/zh_CN/process/development-process.rst 。 +Documentation/translations/zh_CN/process/submit-checklist.rst 给出了一系列 +提交补丁之前要检查的事项。设备树相关的补丁，请参阅 +Documentation/devicetree/bindings/submitting-patches.rst 。 -其中许多步骤描述了Git版本控制系统的默认行为；如果您使用Git来准备补丁， -您将发现它为您完成的大部分机械工作，尽管您仍然需要准备和记录一组合理的 -补丁。一般来说，使用git将使您作为内核开发人员的生活更轻松。 +本文档假设您正在使用 ``git`` 准备你的补丁。如果您不熟悉 ``git`` ，最好学习 +如何使用它，这将使您作为内核开发人员的生活变得更加轻松。 +部分子系统和维护人员的树有一些关于其工作流程和要求的额外信息，请参阅 +Documentation/process/maintainer-handbooks.rst 。 -0) 获取当前源码树 ------------------ +获取当前源码树 +-------------- -如果您没有一个可以使用当前内核源代码的存储库，请使用git获取一个。您将要 -从主线存储库开始，它可以通过以下方式获取:: +如果您手头没有当前内核源代码的存储库，请使用 ``git`` 获取一份。您需要先获取 +主线存储库，它可以通过以下命令拉取:: - git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git -但是，请注意，您可能不希望直接针对主线树进行开发。大多数子系统维护人员运 +但是，请注意，您可能不想直接针对主线树进行开发。大多数子系统维护人员运 行自己的树，并希望看到针对这些树准备的补丁。请参见MAINTAINERS文件中子系 -统的 **T:** 项以查找该树，或者简单地询问维护者该树是否未在其中列出。 - -仍然可以通过tarballs下载内核版本（如下一节所述），但这是进行内核开发的 -一种困难的方式。 - -1) "diff -up" -------------- - -使用 "diff -up" 或者 "diff -uprN" 来创建补丁。 - -所有内核的改动，都是以补丁的形式呈现的，补丁由 diff(1) 生成。创建补丁的 -时候，要确认它是以 "unified diff" 格式创建的，这种格式由 diff(1) 的 '-u' -参数生成。而且，请使用 '-p' 参数，那样会显示每个改动所在的C函数，使得 -产生的补丁容易读得多。补丁应该基于内核源代码树的根目录，而不是里边的任 -何子目录。 - -为一个单独的文件创建补丁，一般来说这样做就够了:: - - SRCTREE=linux - MYFILE=drivers/net/mydriver.c +统的 **T:** 项以查找该树，或者直接询问维护者该树是否未在其中列出。 - cd $SRCTREE - cp $MYFILE $MYFILE.orig - vi $MYFILE # make your change - cd .. - diff -up $SRCTREE/$MYFILE{.orig,} > /tmp/patch +.. _zh_describe_changes: -为多个文件创建补丁，你可以解开一个没有修改过的内核源代码树，然后和你自 -己的代码树之间做 diff 。例如:: - - MYSRC=/devel/linux - - tar xvfz linux-3.19.tar.gz - mv linux-3.19 linux-3.19-vanilla - diff -uprN -X linux-3.19-vanilla/Documentation/dontdiff \ - linux-3.19-vanilla $MYSRC > /tmp/patch - -"dontdiff" 是内核在编译的时候产生的文件的列表，列表中的文件在 diff(1) -产生的补丁里会被跳过。 - -确定你的补丁里没有包含任何不属于这次补丁提交的额外文件。记得在用diff(1) -生成补丁之后，审阅一次补丁，以确保准确。 - -如果你的改动很散乱，你应该研究一下如何将补丁分割成独立的部分，将改动分 -割成一系列合乎逻辑的步骤。这样更容易让其他内核开发者审核，如果你想你的 -补丁被接受，这是很重要的。请参阅： -:ref:`cn_split_changes` - -如果你用 ``git`` , ``git rebase -i`` 可以帮助你这一点。如果你不用 ``git``, -``quilt`` 另外一个流行的选择。 - -.. _cn_describe_changes: - -2) 描述你的改动 ---------------- +描述你的改动 +------------ 描述你的问题。无论您的补丁是一行错误修复还是5000行新功能，都必须有一个潜在 -的问题激励您完成这项工作。让审稿人相信有一个问题值得解决，让他们读完第一段 -是有意义的。 +的问题激励您完成这项工作。说服审阅者相信有一个问题值得解决，让他们读完第一段 +后就能明白这一点。 描述用户可见的影响。直接崩溃和锁定是相当有说服力的，但并不是所有的错误都那么 -明目张胆。即使在代码审查期间发现了这个问题，也要描述一下您认为它可能对用户产 +明目张胆。即使在代码审阅期间发现了这个问题，也要描述一下您认为它可能对用户产 生的影响。请记住，大多数Linux安装运行的内核来自二级稳定树或特定于供应商/产品 的树，只从上游精选特定的补丁，因此请包含任何可以帮助您将更改定位到下游的内容： 触发的场景、DMESG的摘录、崩溃描述、性能回归、延迟尖峰、锁定等。 -量化优化和权衡。如果您声称在性能、内存消耗、堆栈占用空间或二进制大小方面有所 -改进，请包括支持它们的数字。但也要描述不明显的成本。优化通常不是免费的，而是 -在CPU、内存和可读性之间进行权衡；或者，探索性的工作，在不同的工作负载之间进 +质量优化和权衡。如果您声称在性能、内存消耗、堆栈占用空间或二进制大小方面有所 +改进，请包括支持它们的数据。但也要描述不明显的成本。优化通常不是零成本的，而是 +在CPU、内存和可读性之间进行权衡；或者，做探索性的工作，在不同的工作负载之间进 行权衡。请描述优化的预期缺点，以便审阅者可以权衡成本和收益。 -一旦问题建立起来，就要详细地描述一下您实际在做什么。对于审阅者来说，用简单的 -英语描述代码的变化是很重要的，以验证代码的行为是否符合您的意愿。 +提出问题之后，就要详细地描述一下您实际在做的技术细节。对于审阅者来说，用简练的 +英语描述代码的变化是很重要的，以验证代码的行为是否符合您的意图。 -如果您将补丁描述写在一个表单中，这个表单可以很容易地作为“提交日志”放入Linux -的源代码管理系统git中，那么维护人员将非常感谢您。见 :ref:`cn_explicit_in_reply_to`. +如果您将补丁描述写成“标准格式”，可以很容易地作为“提交日志”放入Linux的源代 +码管理系统 ``git`` 中，那么维护人员将非常感谢您。 +参见 :ref:`zh_the_canonical_patch_format` 。 每个补丁只解决一个问题。如果你的描述开始变长，这就表明你可能需要拆分你的补丁。 -请见 :ref:`cn_split_changes` +请见 :ref:`zh_split_changes` 。 -提交或重新提交修补程序或修补程序系列时，请包括完整的修补程序说明和理由。不要 +提交或重新提交补丁或补丁系列时，请包括完整的补丁说明和理由。不要 只说这是补丁（系列）的第几版。不要期望子系统维护人员引用更早的补丁版本或引用 URL来查找补丁描述并将其放入补丁中。也就是说，补丁（系列）及其描述应该是独立的。 -这对维护人员和审查人员都有好处。一些评审者可能甚至没有收到补丁的早期版本。 +这对维护人员和审阅者都有好处。一些审阅者可能甚至没有收到补丁的早期版本。 -描述你在命令语气中的变化，例如“make xyzzy do frotz”而不是“[This patch]make +用祈使句描述你的变更，例如“make xyzzy do frotz”而不是“[This patch]make xyzzy do frotz”或“[I]changed xyzzy to do frotz”，就好像你在命令代码库改变 它的行为一样。 -如果修补程序修复了一个记录的bug条目，请按编号和URL引用该bug条目。如果补丁来 -自邮件列表讨论，请给出邮件列表存档的URL；使用带有 ``Message-ID`` 的 -https://lore.kernel.org/ 重定向，以确保链接不会过时。 - -但是，在没有外部资源的情况下，尽量让你的解释可理解。除了提供邮件列表存档或 -bug的URL之外，还要总结需要提交补丁的相关讨论要点。 - -如果您想要引用一个特定的提交，不要只引用提交的 SHA-1 ID。还请包括提交的一行 +如果您想要引用一个特定的提交，不要只引用提交的SHA-1 ID。还请包括提交的一行 摘要，以便于审阅者了解它是关于什么的。例如:: Commit e21d2170f36602ae2708 ("video: remove unnecessary @@ -144,82 +94,104 @@ bug的URL之外，还要总结需要提交补丁的相关讨论要点。 platform_set_drvdata(), but left the variable "dev" unused, delete it. -您还应该确保至少使用前12位 SHA-1 ID. 内核存储库包含*许多*对象，使与较短的ID +您还应该确保至少使用前12位SHA-1 ID。内核存储库包含 *许多* 对象，使较短的ID 发生冲突的可能性很大。记住，即使现在不会与您的六个字符ID发生冲突，这种情况 -可能五年后改变。 +也可能在五年后改变。 + +如果该变更的相关讨论或背景信息可以在网上查阅，请加上“Link:”标签指向它。例如 +你的补丁修复了一个缺陷，需要添加一个带有URL的标签指向邮件列表存档或缺陷跟踪器 +的相关报告；如果该补丁是由一些早先邮件列表讨论或网络上的记录引起的，请指向它。 + +当链接到邮件列表存档时，请首选lore.kernel.org邮件存档服务。用邮件中的 +``Message-ID`` 头（去掉尖括号）可以创建链接URL。例如:: + + Link: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/ + +请检查该链接以确保可用且指向正确的邮件。 -如果修补程序修复了特定提交中的错误，例如，使用 ``git bisct`` ，请使用带有前 -12个字符SHA-1 ID 的"Fixes:"标记和单行摘要。为了简化不要将标记拆分为多个， -行、标记不受分析脚本“75列换行”规则的限制。例如:: +不过，在没有外部资源的情况下，也要尽量让你的解释可理解。除了提供邮件列表存档或 +缺陷的URL之外，还要需要总结该补丁的相关讨论要点。 - Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed") +如果补丁修复了特定提交中的错误，例如使用 ``git bisct`` 发现了一个问题，请使用 +带有前12个字符SHA-1 ID的“Fixes:”标签和单行摘要。为了简化解析脚本，不要将该 +标签拆分为多行，标签不受“75列换行”规则的限制。例如:: -下列 ``git config`` 设置可以添加让 ``git log``, ``git show`` 漂亮的显示格式:: + Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed") + +下列 ``git config`` 设置可以让 ``git log``, ``git show`` 增加上述风格的显示格式:: [core] abbrev = 12 [pretty] fixes = Fixes: %h (\"%s\") -.. _cn_split_changes: +使用示例:: + + $ git log -1 --pretty=fixes 54a4f0239f2e + Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed") -3) 拆分你的改动 ---------------- +.. _zh_split_changes: -将每个逻辑更改分隔成一个单独的补丁。 +拆分你的改动 +------------ + +将每个 **逻辑更改** 拆分成一个单独的补丁。 例如，如果你的改动里同时有bug修正和性能优化，那么把这些改动拆分到两个或 -者更多的补丁文件中。如果你的改动包含对API的修改，并且修改了驱动程序来适 -应这些新的API，那么把这些修改分成两个补丁。 +者更多的补丁文件中。如果你的改动包含对API的修改，并且增加了一个使用该新API +的驱动，那么把这些修改分成两个补丁。 另一方面，如果你将一个单独的改动做成多个补丁文件，那么将它们合并成一个 单独的补丁文件。这样一个逻辑上单独的改动只被包含在一个补丁文件里。 -如果有一个补丁依赖另外一个补丁来完成它的改动，那没问题。简单的在你的补 -丁描述里指出“这个补丁依赖某补丁”就好了。 +需要记住的一点是，每个补丁的更改都应易于理解，以便审阅者验证。每个补丁都应该 +对其价值进行阐述。 + +如果有一个补丁依赖另外一个补丁来完成它的改动，那没问题。直接在你的补 +丁描述里指出 **“这个补丁依赖某补丁”** 就好了。 -在将您的更改划分为一系列补丁时，要特别注意确保内核在系列中的每个补丁之后 -都能正常构建和运行。使用 ``git bisect`` 来追踪问题的开发者可能会在任何时 -候分割你的补丁系列；如果你在中间引入错误，他们不会感谢你。 +在将您的更改划分为一系列补丁时，要特别注意确保内核在应用系列中的每个补丁之后 +都能正常构建和运行。使用 ``git bisect`` 来追踪问题的开发者可能会在任何地方分 +割你的补丁系列；如果你在中间引入错误，他们不会感谢你。 -如果你不能将补丁浓缩成更少的文件，那么每次大约发送出15个，然后等待审查 +如果你不能将补丁系列浓缩得更小，那么每次大约发送出15个补丁，然后等待审阅 和集成。 -4) 检查你的更改风格 -------------------- +检查你的更改风格 +---------------- -检查您的补丁是否存在基本样式冲突，详细信息可在 -:ref:`Documentation/translations/zh_CN/process/coding-style.rst ` -中找到。如果不这样做，只会浪费审稿人的时间，并且会导致你的补丁被拒绝，甚至 +检查您的补丁是否违反了基本样式规定，详细信息参见 +Documentation/translations/zh_CN/process/coding-style.rst +中找到。如果不这样做，只会浪费审阅者的时间，并且会导致你的补丁被拒绝，甚至 可能没有被阅读。 一个重要的例外是在将代码从一个文件移动到另一个文件时——在这种情况下，您不应 该在移动代码的同一个补丁中修改移动的代码。这清楚地描述了移动代码和您的更改 -的行为。这大大有助于审查实际差异，并允许工具更好地跟踪代码本身的历史。 +的行为。这大大有助于审阅实际差异，并允许工具更好地跟踪代码本身的历史。 在提交之前，使用补丁样式检查程序检查补丁（scripts/check patch.pl）。不过， 请注意，样式检查程序应该被视为一个指南，而不是作为人类判断的替代品。如果您 -的代码看起来更好，但有违规行为，那么最好不要使用它。 +的代码看起来更好，但有违规行为，那么最好别管它。 检查者报告三个级别： - ERROR：很可能出错的事情 - - WARNING：需要仔细审查的事项 + - WARNING：需要仔细审阅的事项 - CHECK：需要思考的事情 您应该能够判断您的补丁中存在的所有违规行为。 -5) 选择补丁收件人 ------------------ +选择补丁收件人 +-------------- -您应该总是在任何补丁上复制相应的子系统维护人员，以获得他们维护的代码；查看 +您应该总是知会任何补丁相应代码的子系统维护人员；查看 维护人员文件和源代码修订历史记录，以了解这些维护人员是谁。脚本 -scripts/get_Maintainer.pl在这个步骤中非常有用。如果您找不到正在工作的子系统 +scripts/get_maintainer.pl在这个步骤中非常有用。如果您找不到正在工作的子系统 的维护人员，那么Andrew Morton（akpm@linux-foundation.org）将充当最后的维护 人员。 -您通常还应该选择至少一个邮件列表来接收补丁集的。linux-kernel@vger.kernel.org -作为最后一个解决办法的列表，但是这个列表上的体积已经引起了许多开发人员的拒绝。 +您通常还应该选择至少一个邮件列表来接收补丁集的副本。linux-kernel@vger.kernel.org +是所有补丁的默认列表，但是这个列表的流量已经导致了许多开发人员不再看它。 在MAINTAINERS文件中查找子系统特定的列表；您的补丁可能会在那里得到更多的关注。 不过，请不要发送垃圾邮件到无关的列表。 @@ -229,189 +201,170 @@ http://vger.kernel.org/vger-lists.html 上找到它们的列表。不过，也 不要一次发送超过15个补丁到vger邮件列表！！！！ -Linus Torvalds 是决定改动能否进入 Linux 内核的最终裁决者。他的 e-mail -地址是 。他收到的 e-mail 很多，所以一般 -的说，最好别给他发 e-mail。 - -如果您有修复可利用安全漏洞的补丁，请将该补丁发送到 security@kernel.org。对于 -严重的bug，可以考虑短期暂停以允许分销商向用户发布补丁；在这种情况下，显然不应 -将补丁发送到任何公共列表。 +Linus Torvalds是决定改动能否进入 Linux 内核的最终裁决者。他的邮件地址是 +torvalds@linux-foundation.org 。他收到的邮件很多，所以一般来说最好 **别** +给他发邮件。 -修复已发布内核中严重错误的补丁程序应该指向稳定版维护人员，方法是放这样的一行:: +如果您有修复可利用安全漏洞的补丁，请将该补丁发送到 security@kernel.org 。对于 +严重的bug，可以考虑短期禁令以允许分销商（有时间）向用户发布补丁；在这种情况下， +显然不应将补丁发送到任何公共列表。 +参见 Documentation/translations/zh_CN/admin-guide/security-bugs.rst 。 - Cc: stable@vger.kernel.org +修复已发布内核中严重错误的补丁程序应该抄送给稳定版维护人员，方法是把以下列行 +放进补丁的签准区（注意，不是电子邮件收件人）:: -进入补丁的签准区（注意，不是电子邮件收件人）。除了这个文件之外，您还应该阅读 -:ref:`Documentation/process/stable-kernel-rules.rst ` + Cc: stable@vger.kernel.org -但是，请注意，一些子系统维护人员希望得出他们自己的结论，即哪些补丁应该被放到 -稳定的树上。尤其是网络维护人员，不希望看到单个开发人员在补丁中添加像上面这样 -的行。 +除了本文件之外，您还应该阅读 +Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。 -如果更改影响到用户和内核接口，请向手册页维护人员（如维护人员文件中所列）发送 +如果更改影响到用户侧内核接口，请向手册页维护人员（如维护人员文件中所列）发送 手册页补丁，或至少发送更改通知，以便一些信息进入手册页。还应将用户空间API -更改复制到 linux-api@vger.kernel.org。 +更改抄送到 linux-api@vger.kernel.org 。 -6) 没有 MIME 编码，没有链接，没有压缩，没有附件，只有纯文本 ------------------------------------------------------------ +不要MIME编码，不要链接，不要压缩，不要附件，只要纯文本 +------------------------------------------------------ Linus 和其他的内核开发者需要阅读和评论你提交的改动。对于内核开发者来说 -，可以“引用”你的改动很重要，使用一般的 e-mail 工具，他们就可以在你的 +，可以“引用”你的改动很重要，使用一般的邮件工具，他们就可以在你的 代码的任何位置添加评论。 -因为这个原因，所有的提交的补丁都是 e-mail 中“内嵌”的。 +因为这个原因，所有的提交的补丁都是邮件中“内嵌”的。最简单（和推荐）的方法就 +是使用 ``git send-email`` 。https://git-send-email.io 有 ``git send-email`` +的交互式教程。 + +如果你选择不用 ``git send-email`` ： .. warning:: - 如果你使用剪切-粘贴你的补丁，小心你的编辑器的自动换行功能破坏你的补丁 -不要将补丁作为 MIME 编码的附件，不管是否压缩。很多流行的 e-mail 软件不 -是任何时候都将 MIME 编码的附件当作纯文本发送的，这会使得别人无法在你的 -代码中加评论。另外，MIME 编码的附件会让 Linus 多花一点时间来处理，这就 -降低了你的改动被接受的可能性。 + 如果你使用剪切-粘贴你的补丁，小心你的编辑器的自动换行功能破坏你的补丁 -例外：如果你的邮递员弄坏了补丁，那么有人可能会要求你使用mime重新发送补丁 +不要将补丁作为MIME编码的附件，不管是否压缩。很多流行的邮件软件不 +是任何时候都将MIME编码的附件当作纯文本发送的，这会使得别人无法在你的 +代码中加评论。另外，MIME编码的附件会让Linus多花一点时间来处理，这就 +降低了你的改动被接受的可能性。 -请参阅 :ref:`Documentation/translations/zh_CN/process/email-clients.rst ` -以获取有关配置电子邮件客户端以使其不受影响地发送修补程序的提示。 +例外：如果你的邮路损坏了补丁，那么有人可能会要求你使用MIME重新发送补丁。 -7) e-mail 的大小 ----------------- +请参阅 Documentation/translations/zh_CN/process/email-clients.rst +以获取有关配置电子邮件客户端以使其不受影响地发送补丁的提示。 -大的改动对邮件列表不合适，对某些维护者也不合适。如果你的补丁，在不压缩 -的情况下，超过了300kB，那么你最好将补丁放在一个能通过 internet 访问的服 -务器上，然后用指向你的补丁的 URL 替代。但是请注意，如果您的补丁超过了 -300kb，那么它几乎肯定需要被破坏。 +回复审阅意见 +------------ -8）回复评审意见 ---------------- +你的补丁几乎肯定会得到审阅者对补丁改进方法的评论（以回复邮件的形式）。您必须 +对这些评论作出回应；让补丁被忽略的一个好办法就是忽略审阅者的意见。直接回复邮 +件来回应意见即可。不会导致代码更改的意见或问题几乎肯定会带来注释或变更日志的 +改变，以便下一个审阅者更好地了解正在发生的事情。 -你的补丁几乎肯定会得到评审者对补丁改进方法的评论。您必须对这些评论作出 -回应；让补丁被忽略的一个好办法就是忽略审阅者的意见。不会导致代码更改的 -意见或问题几乎肯定会带来注释或变更日志的改变，以便下一个评审者更好地了解 -正在发生的事情。 +一定要告诉审阅者你在做什么改变，并感谢他们的时间。代码审阅是一个累人且耗时的 +过程，审阅者有时会变得暴躁。即使在这种情况下，也要礼貌地回应并解决他们指出的 +问题。当发送下一版时，在封面邮件或独立补丁里加上 ``patch changelog`` 说明与 +前一版本的不同之处（参见 :ref:`zh_the_canonical_patch_format` ）。 -一定要告诉审稿人你在做什么改变，并感谢他们的时间。代码审查是一个累人且 -耗时的过程，审查人员有时会变得暴躁。即使在这种情况下，也要礼貌地回应并 -解决他们指出的问题。 +.. _zh_resend_reminders: -9）不要泄气或不耐烦 -------------------- +不要泄气或不耐烦 +---------------- -提交更改后，请耐心等待。审阅者是忙碌的人，可能无法立即访问您的修补程序。 +提交更改后，请耐心等待。审阅者是大忙人，可能无法立即审阅您的补丁。 -曾几何时，补丁曾在没有评论的情况下消失在空白中，但开发过程比现在更加顺利。 +曾几何时，补丁曾在没收到评论的情况下消失在虚空中，但现在开发过程应该更加顺利了。 您应该在一周左右的时间内收到评论；如果没有收到评论，请确保您已将补丁发送 -到正确的位置。在重新提交或联系审阅者之前至少等待一周-在诸如合并窗口之类的 +到正确的位置。在重新提交或联系审阅者之前至少等待一周——在诸如合并窗口之类的 繁忙时间可能更长。 -10）主题中包含 PATCH --------------------- +在等了几个星期后，用带RESEND的主题重发补丁也是可以的:: -由于到linus和linux内核的电子邮件流量很高，通常会在主题行前面加上[PATCH] -前缀. 这使Linus和其他内核开发人员更容易将补丁与其他电子邮件讨论区分开。 + [PATCH Vx RESEND] sub/sys: Condensed patch summary -11）签署你的作品-开发者原始认证 -------------------------------- +当你发布补丁（系列）修改版的时候，不要加上“RESEND”——“RESEND”只适用于重 +新提交之前未经修改的补丁（系列）。 -为了加强对谁做了何事的追踪，尤其是对那些透过好几层的维护者的补丁，我们 -建议在发送出去的补丁上加一个 “sign-off” 的过程。 +主题中包含 PATCH +---------------- -"sign-off" 是在补丁的注释的最后的简单的一行文字，认证你编写了它或者其他 +由于到Linus和linux-kernel的电子邮件流量很高，通常会在主题行前面加上[PATCH] +前缀。这使Linus和其他内核开发人员更容易将补丁与其他电子邮件讨论区分开。 + +``git send-email`` 会自动为你加上。 + +签署你的作品——开发者来源认证 +------------------------------ + +为了加强对谁做了何事的追踪，尤其是对那些透过好几层维护者才最终到达的补丁，我 +们在通过邮件发送的补丁上引入了“签署（sign-off）”流程。 + +“签署”是在补丁注释最后的一行简单文字，认证你编写了它或者其他 人有权力将它作为开放源代码的补丁传递。规则很简单：如果你能认证如下信息: -开发者来源证书 1.1 +开发者来源认证 1.1 ^^^^^^^^^^^^^^^^^^ 对于本项目的贡献，我认证如下信息： - （a）这些贡献是完全或者部分的由我创建，我有权利以文件中指出 + (a) 这些贡献是完全或者部分的由我创建，我有权利以文件中指出 的开放源代码许可证提交它；或者 - （b）这些贡献基于以前的工作，据我所知，这些以前的工作受恰当的开放 - 源代码许可证保护，而且，根据许可证，我有权提交修改后的贡献， + + (b) 这些贡献基于以前的工作，据我所知，这些以前的工作受恰当的开放 + 源代码许可证保护，而且，根据文件中指出的许可证，我有权提交修改后的贡献， 无论是完全还是部分由我创造，这些贡献都使用同一个开放源代码许可证 - （除非我被允许用其它的许可证），正如文件中指出的；或者 - （c）这些贡献由认证（a），（b）或者（c）的人直接提供给我，而 + （除非我被允许用其它的许可证）；或者 + + (c) 这些贡献由认证（a），（b）或者（c）的人直接提供给我，而 且我没有修改它。 - （d）我理解并同意这个项目和贡献是公开的，贡献的记录（包括我 - 一起提交的个人记录，包括 sign-off ）被永久维护并且可以和这个项目 + + (d) 我理解并同意这个项目和贡献是公开的，贡献的记录（包括我 + 一起提交的个人记录，包括sign-off）被永久维护并且可以和这个项目 或者开放源代码的许可证同步地再发行。 那么加入这样一行:: - Signed-off-by: Random J Developer - -使用你的真名（抱歉，不能使用假名或者匿名。） - -有人在最后加上标签。现在这些东西会被忽略，但是你可以这样做，来标记公司 -内部的过程，或者只是指出关于 sign-off 的一些特殊细节。 - -如果您是子系统或分支维护人员，有时需要稍微修改收到的补丁，以便合并它们， -因为树和提交者中的代码不完全相同。如果你严格遵守规则（c），你应该要求提交者 -重新发布，但这完全是在浪费时间和精力。规则（b）允许您调整代码，但是更改一个 -提交者的代码并让他认可您的错误是非常不礼貌的。要解决此问题，建议在最后一个 -由签名行和您的行之间添加一行，指示更改的性质。虽然这并不是强制性的，但似乎 -在描述前加上您的邮件和/或姓名（全部用方括号括起来），这足以让人注意到您对最 -后一分钟的更改负有责任。例如:: - - Signed-off-by: Random J Developer - [lucky@maintainer.example.org: struct foo moved from foo.c to foo.h] - Signed-off-by: Lucky K Maintainer - -如果您维护一个稳定的分支机构，同时希望对作者进行致谢、跟踪更改、合并修复并 -保护提交者不受投诉，那么这种做法尤其有用。请注意，在任何情况下都不能更改作者 -的ID（From 头），因为它是出现在更改日志中的标识。 - -对回合（back-porters）的特别说明：在提交消息的顶部（主题行之后）插入一个补丁 -的起源指示似乎是一种常见且有用的实践，以便于跟踪。例如，下面是我们在3.x稳定 -版本中看到的内容:: - - Date: Tue Oct 7 07:26:38 2014 -0400 - - libata: Un-break ATA blacklist - - commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream. - -还有， 这里是一个旧版内核中的一个回合补丁:: + Signed-off-by: Random J Developer - Date: Tue May 13 22:12:27 2008 +0200 +使用你的真名（抱歉，不能使用假名或者匿名。）如果使用 ``git commit -s`` 的话 +将会自动完成。撤销也应当包含“Signed-off-by”， ``git revert -s`` 会帮你搞定。 - wireless, airo: waitbusy() won't delay +有些人会在最后加上额外的标签。现在这些东西会被忽略，但是你可以这样做，来标记 +公司内部的过程，或者只是指出关于签署的一些特殊细节。 - [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a] +作者签署之后的任何其他签署（Signed-off-by:'s）均来自处理和传递补丁的人员，但 +未参与其开发。签署链应当反映补丁传播到维护者并最终传播到Linus所经过的 **真实** +路径，首个签署指明单个作者的主要作者身份。 -12）何时使用Acked-by:，CC:，和Co-Developed by: ----------------------------------------------- +何时使用Acked-by:，CC:，和Co-Developed by: +------------------------------------------ -Singed-off-by: 标记表示签名者参与了补丁的开发，或者他/她在补丁的传递路径中。 +Singed-off-by: 标签表示签名者参与了补丁的开发，或者他/她在补丁的传递路径中。 -如果一个人没有直接参与补丁的准备或处理，但希望表示并记录他们对补丁的批准， -那么他们可以要求在补丁的变更日志中添加一个 Acked-by: +如果一个人没有直接参与补丁的准备或处理，但希望表示并记录他们对补丁的批准/赞成， +那么他们可以要求在补丁的变更日志中添加一个Acked-by:。 -Acked-by：通常由受影响代码的维护者使用，当该维护者既没有贡献也没有转发补丁时。 +Acked-by: 通常由受影响代码的维护者使用，当该维护者既没有贡献也没有转发补丁时。 -Acked-by: 不像签字人那样正式。这是一个记录，确认人至少审查了补丁，并表示接受。 -因此，补丁合并有时会手动将Acker的“Yep，looks good to me”转换为 Acked-By：（但 +Acked-by: 不像签署那样正式。这是一个记录，确认人至少审阅了补丁，并表示接受。 +因此，补丁合并有时会手动将Acker的“Yep，looks good to me”转换为 Acked-By:（但 请注意，通常最好要求一个明确的Ack）。 Acked-by：不一定表示对整个补丁的确认。例如，如果一个补丁影响多个子系统，并且 -有一个：来自一个子系统维护者，那么这通常表示只确认影响维护者代码的部分。这里 -应该仔细判断。如有疑问，应参考邮件列表档案中的原始讨论。 +有一个来自某个子系统维护者的Acked-By:，那么这通常表示只确认影响维护者代码的部 +分。这里应该仔细判断。如有疑问，应参考邮件列表存档中的原始讨论。 -如果某人有机会对补丁进行评论，但没有提供此类评论，您可以选择在补丁中添加 ``Cc:`` -这是唯一一个标签，它可以在没有被它命名的人显式操作的情况下添加，但它应该表明 -这个人是在补丁上抄送的。讨论中包含了潜在利益相关方。 +如果某人本应有机会对补丁进行评论，但没有提供此类评论，您可以选择在补丁中添加 +``Cc:`` 这是唯一可以在没有被该人明确同意的情况下添加的标签——但它应该表明 +这个人是在补丁上抄送的。此标签记录了讨论中包含的潜在利益相关方。 Co-developed-by: 声明补丁是由多个开发人员共同创建的；当几个人在一个补丁上工 -作时，它用于将属性赋予共同作者（除了 From: 所赋予的作者之外）。因为 -Co-developed-by: 表示作者身份，所以每个共同开发人：必须紧跟在相关合作作者的 -签名之后。标准的签核程序要求：标记的签核顺序应尽可能反映补丁的时间历史，而不 -管作者是通过 From ：还是由 Co-developed-by: 共同开发的。值得注意的是，最后一 -个签字人：必须始终是提交补丁的开发人员。 +作时，它用于给出共同作者（除了From:所给出的作者之外）。因为Co-developed-by: +表示作者身份，所以每个Co-developed-by:必须紧跟在相关合作作者的签署之后。标准 +签署程序要求Singed-off-by:标签的顺序应尽可能反映补丁的时间历史，无论作者是通 +过From:还是Co-developed-by:表明。值得注意的是，最后一个Singed-off-by:必须是 +提交补丁的开发人员。 -注意，当作者也是电子邮件标题“发件人：”行中列出的人时，“From: ” 标记是可选的。 +注意，如果From:作者也是电子邮件标题的From:行中列出的人，则From:标签是可选的。 -作者提交的补丁程序示例:: +被From:作者提交的补丁示例:: @@ -421,7 +374,7 @@ Co-developed-by: 表示作者身份，所以每个共同开发人：必须紧跟 Signed-off-by: Second Co-Author Signed-off-by: From Author -合作开发者提交的补丁示例:: +被合作开发者提交的补丁示例:: From: From Author @@ -434,76 +387,85 @@ Co-developed-by: 表示作者身份，所以每个共同开发人：必须紧跟 Signed-off-by: Submitting Co-Author -13）使用报告人：、测试人：、审核人：、建议人：、修复人： --------------------------------------------------------- +使用Reported-by:、Tested-by:、Reviewed-by:、Suggested-by:和Fixes: +----------------------------------------------------------------- Reported-by: 给那些发现错误并报告错误的人致谢，它希望激励他们在将来再次帮助 -我们。请注意，如果bug是以私有方式报告的，那么在使用Reported-by标记之前，请 -先请求权限。 +我们。请注意，如果bug是以私有方式报告的，那么在使用Reported-by标签之前，请 +先请求许可。此标签是为Bug设计的；请不要将其用于感谢功能请求。 -Tested-by: 标记表示补丁已由指定的人（在某些环境中）成功测试。这个标签通知 -维护人员已经执行了一些测试，为将来的补丁提供了一种定位测试人员的方法，并确 -保测试人员的信誉。 +Tested-by: 标签表示补丁已由指定的人（在某些环境中）成功测试。这个标签通知 +维护人员已经执行了一些测试，为将来的补丁提供了一种定位测试人员的方法，并彰显测试人员的功劳。 -Reviewed-by：相反，根据审查人的声明，表明该补丁已被审查并被认为是可接受的： +Reviewed-by：根据审阅者的监督声明，表明该补丁已被审阅并被认为是可接受的： -审查人的监督声明 +审阅者的监督声明 ^^^^^^^^^^^^^^^^ -通过提供我的 Reviewed-by，我声明： +通过提供我的Reviewed-by:标签，我声明： - (a) 我已经对这个补丁进行了一次技术审查，以评估它是否适合被包含到 + (a) 我已经对这个补丁进行了一次技术审阅，以评估它是否适合被包含到 主线内核中。 (b) 与补丁相关的任何问题、顾虑或问题都已反馈给提交者。我对提交者对 我的评论的回应感到满意。 - (c) 虽然这一提交可能会改进一些东西，但我相信，此时，（1）对内核 + (c) 虽然这一提交可能仍可被改进，但我相信，此时，（1）对内核 进行了有价值的修改，（2）没有包含争论中涉及的已知问题。 - (d) 虽然我已经审查了补丁并认为它是健全的，但我不会（除非另有明确 - 说明）作出任何保证或保证它将在任何给定情况下实现其规定的目的 + (d) 虽然我已经审阅了补丁并认为它是健全的，但我不会（除非另有明确 + 说明）作出任何保证或担保它会在任何给定情况下实现其规定的目的 或正常运行。 -Reviewed-by 是一种观点声明，即补丁是对内核的适当修改，没有任何遗留的严重技术 -问题。任何感兴趣的审阅者（完成工作的人）都可以为一个补丁提供一个 Review-by -标签。此标签用于向审阅者提供致谢，并通知维护者已在修补程序上完成的审阅程度。 -Reviewed-by: 当由已知了解主题区域并执行彻底检查的审阅者提供时，通常会增加 +Reviewed-by是一种观点声明，即补丁是对内核的适当修改，没有任何遗留的严重技术 +问题。任何感兴趣的审阅者（完成工作的人）都可以为一个补丁提供一个Reviewed-by +标签。此标签用于向审阅者提供致谢，并通知维护者补丁的审阅进度。 +当Reviewed-by:标签由已知了解主题区域并执行彻底检查的审阅者提供时，通常会增加 补丁进入内核的可能性。 +一旦从测试人员或审阅者的“Tested-by”和“Reviewed-by”标签出现在邮件列表中， +作者应在发送下一个版本时将其添加到适用的补丁中。但是，如果补丁在以下版本中发 +生了实质性更改，这些标签可能不再适用，因此应该删除。通常，在补丁更改日志中 +（在 ``---`` 分隔符之后）应该提到删除某人的测试者或审阅者标签。 + Suggested-by: 表示补丁的想法是由指定的人提出的，并确保将此想法归功于指定的 人。请注意，未经许可，不得添加此标签，特别是如果该想法未在公共论坛上发布。 -这就是说，如果我们勤快地致谢我们的创意者，他们很有希望在未来得到鼓舞，再次 +也就是说，如果我们勤快地致谢创意提供者，他们将受到鼓舞，很有希望在未来再次 帮助我们。 -Fixes: 指示补丁在以前的提交中修复了一个问题。它可以很容易地确定错误的来源， -这有助于检查错误修复。这个标记还帮助稳定内核团队确定应该接收修复的稳定内核 -版本。这是指示补丁修复的错误的首选方法。请参阅 :ref:`cn_describe_changes` -描述您的更改以了解更多详细信息。 +Fixes: 指示补丁修复了之前提交的一个问题。它可以便于确定错误的来源，这有助于 +检查错误修复。这个标签还帮助稳定内核团队确定应该接收修复的稳定内核版本。这是 +指示补丁修复的错误的首选方法。请参阅 :ref:`zh_describe_changes` 了解更多信息。 -.. _cn_the_canonical_patch_format: +.. note:: -12）标准补丁格式 ----------------- + 附加Fixes:标签不会改变稳定内核规则流程，也不改变所有稳定版补丁抄送 + stable@vger.kernel.org的要求。有关更多信息，请阅读 + Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。 + +.. _zh_the_canonical_patch_format: + +标准补丁格式 +------------ 本节描述如何格式化补丁本身。请注意，如果您的补丁存储在 ``Git`` 存储库中，则 -可以使用 ``git format-patch`` 进行正确的补丁格式设置。但是，这些工具无法创建 +可以使用 ``git format-patch`` 进行正确的补丁格式化。但是，这些工具无法创建 必要的文本，因此请务必阅读下面的说明。 -标准的补丁，标题行是:: +标准的补丁标题行是:: Subject: [PATCH 001/123] 子系统:一句话概述 -标准补丁的信体存在如下部分： +标准补丁的信体包含如下部分： - - 一个 "from" 行指出补丁作者。后跟空行（仅当发送修补程序的人不是作者时才需要）。 + - 一个 ``from`` 行指出补丁作者。后跟空行（仅当发送补丁的人不是作者时才需要）。 - - 解释的正文，行以75列包装，这将被复制到永久变更日志来描述这个补丁。 + - 说明文字，每行最长75列，这将被复制到永久变更日志来描述这个补丁。 - 一个空行 - - 上面描述的“Signed-off-by” 行，也将出现在更改日志中。 + - 上述的 ``Signed-off-by:`` 行，也将出现在更改日志中。 - 只包含 ``---`` 的标记线。 @@ -511,29 +473,29 @@ Fixes: 指示补丁在以前的提交中修复了一个问题。它可以很容 - 实际补丁（ ``diff`` 输出）。 -标题行的格式，使得对标题行按字母序排序非常的容易 - 很多 e-mail 客户端都 -可以支持 - 因为序列号是用零填充的，所以按数字排序和按字母排序是一样的。 +标题行的格式，使得对标题行按字母序排序非常的容易——很多邮件客户端都 +可以支持——因为序列号是用零填充的，所以按数字排序和按字母排序是一样的。 -e-mail 标题中的“子系统”标识哪个内核子系统将被打补丁。 +邮件标题中的“子系统”标识哪个内核子系统将被打补丁。 -e-mail 标题中的“一句话概述”扼要的描述 e-mail 中的补丁。“一句话概述” +邮件标题中的“一句话概述”扼要的描述邮件中的补丁。“一句话概述” 不应该是一个文件名。对于一个补丁系列（“补丁系列”指一系列的多个相关补 丁），不要对每个补丁都使用同样的“一句话概述”。 -记住 e-mail 的“一句话概述”会成为该补丁的全局唯一标识。它会蔓延到 git +记住邮件的“一句话概述”会成为该补丁的全局唯一标识。它会进入 ``git`` 的改动记录里。然后“一句话概述”会被用在开发者的讨论里，用来指代这个补 -丁。用户将希望通过 google 来搜索"一句话概述"来找到那些讨论这个补丁的文 +丁。用户将希望通过搜索引擎搜索“一句话概述”来找到那些讨论这个补丁的文 章。当人们在两三个月后使用诸如 ``gitk`` 或 ``git log --oneline`` 之类 的工具查看数千个补丁时，也会很快看到它。 出于这些原因，概述必须不超过70-75个字符，并且必须描述补丁的更改以及为 -什么需要补丁。既要简洁又要描述性很有挑战性，但写得好的概述应该这样做。 +什么需要补丁。既要简洁又要描述性很有挑战性，但写得好的概述应该这样。 概述的前缀可以用方括号括起来：“Subject: [PATCH ...] <概述>”。标记 不被视为概述的一部分，而是描述应该如何处理补丁。如果补丁的多个版本已发 -送出来以响应评审（即“v1，v2，v3”）或“rfc”，以指示评审请求，那么通用标记 -可能包括版本描述符。如果一个补丁系列中有四个补丁，那么各个补丁可以这样 -编号：1/4、2/4、3/4、4/4。这可以确保开发人员了解补丁应用的顺序，并且他们 +送出来以响应评审（即“v1，v2，v3”）则必须包含版本号，或包含“RFC”以指示 +评审请求。如果一个补丁系列中有四个补丁，那么各个补丁可以这样编号：1/4、2/4、 +3/4、4/4。这可以确保开发人员了解补丁应用的顺序，且 已经查看或应用了补丁系列中的所有补丁。 一些标题的例子:: @@ -541,95 +503,134 @@ e-mail 标题中的“一句话概述”扼要的描述 e-mail 中的补丁。 Subject: [patch 2/5] ext2: improve scalability of bitmap searching Subject: [PATCHv2 001/207] x86: fix eflags tracking -"From" 行是信体里的最上面一行，具有如下格式： +``From`` 行是信体里的最上面一行，具有如下格式:: + From: Patch Author -"From" 行指明在永久改动日志里，谁会被确认为作者。如果没有 "From" 行，那 -么邮件头里的 "From: " 行会被用来决定改动日志中的作者。 +``From`` 行指明在永久改动日志里，谁会被确认为作者。如果没有 ``From`` 行，那 +么邮件头里的 ``From:`` 行会被用来决定改动日志中的作者。 -说明的主题将会被提交到永久的源代码改动日志里，因此对那些早已经不记得和 -这个补丁相关的讨论细节的有能力的读者来说，是有意义的。包括补丁程序定位 -错误的（内核日志消息、OOPS消息等）症状，对于搜索提交日志以寻找适用补丁的人 -尤其有用。如果一个补丁修复了一个编译失败，那么可能不需要包含所有编译失败； +说明文字将会被提交到永久的源代码改动日志里，因此应针对那些早已经不记得和这 +个补丁相关的讨论细节的读者。包括补丁处理的故障症状（内核日志消息、oops消息 +等），这对于可能正在搜索提交日志以查找适用补丁的人特别有用。文本应该写得如 +此详细，以便在数周、数月甚至数年后阅读时，能够为读者提供所需的细节信息，以 +掌握创建补丁的 **原因** 。 + +如果一个补丁修复了一个编译失败，那么可能不需要包含 *所有* 编译失败； 只要足够让搜索补丁的人能够找到它就行了。与概述一样，既要简洁又要描述性。 -"---" 标记行对于补丁处理工具要找到哪里是改动日志信息的结束，是不可缺少 +``---`` 标记行对于补丁处理工具要找到哪里是改动日志信息的结束，是不可缺少 的。 -对于 "---" 标记之后的额外注解，一个好的用途就是用来写 diffstat，用来显 -示修改了什么文件和每个文件都增加和删除了多少行。diffstat 对于比较大的补 -丁特别有用。其余那些只是和时刻或者开发者相关的注解，不合适放到永久的改 -动日志里的，也应该放这里。 -使用 diffstat的选项 "-p 1 -w 70" 这样文件名就会从内核源代码树的目录开始 +对于 ``---`` 标记之后的额外注解，一个好的用途就是用来写 ``diffstat`` ，用来显 +示修改了什么文件和每个文件都增加和删除了多少行。 ``diffstat`` 对于比较大的补 +丁特别有用。 +使用 ``diffstat`` 的选项 ``-p 1 -w 70`` 这样文件名就会从内核源代码树的目录开始 ，不会占用太宽的空间（很容易适合80列的宽度，也许会有一些缩进。） +（ ``git`` 默认会生成合适的diffstat。） + +其余那些只适用于当时或者与维护者相关的注解，不合适放到永久的改动日志里的，也 +应该放这里。较好的例子就是 ``补丁更改记录`` ，记录了v1和v2版本补丁之间的差异。 + +请将此信息放在将变更日志与补丁的其余部分分隔开的 ``---`` 行 **之后** 。版本 +信息不是提交到git树的变更日志的一部分。只是供审阅人员使用的附加信息。如果将 +其放置在提交标记上方，则需要手动交互才能将其删除。如果它位于分隔线以下，则在 +应用补丁时会自动剥离:: + + + ... + Signed-off-by: Author + --- + V2 -> V3: Removed redundant helper function + V1 -> V2: Cleaned up coding style and addressed review comments -在后面的参考资料中能看到适当的补丁格式的更多细节。 + path/to/file | 5+++-- + ... -.. _cn_explicit_in_reply_to: +在后面的参考资料中能看到正确补丁格式的更多细节。 -15) 明确回复邮件头(In-Reply-To) -------------------------------- +.. _zh_backtraces: -手动添加回复补丁的的标题头(In-Reply_To:) 是有帮助的（例如，使用 ``git send-email`` ） -将补丁与以前的相关讨论关联起来，例如，将bug修复程序链接到电子邮件和bug报告。 +提交消息中的回溯（Backtraces） +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +回溯有助于记录导致问题的调用链。然而，并非所有回溯都有帮助。例如，早期引导调 +用链是独特而明显的。而逐字复制完整的dmesg输出则会增加时间戳、模块列表、寄存 +器和堆栈转储等分散注意力的信息。 + +因此，最有用的回溯应该从转储中提取相关信息，以更容易集中在真实问题上。下面是 +一个剪裁良好的回溯示例:: + + unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064) + at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20) + Call Trace: + mba_wrmsr + update_domains + rdtgroup_mkdir + +.. _zh_explicit_in_reply_to: + +明确回复邮件头（In-Reply-To） +----------------------------- + +手动添加回复补丁的的邮件头（In-Reply_To:）是有用的（例如，使用 ``git send-email`` ）， +可以将补丁与以前的相关讨论关联起来，例如，将bug补丁链接到电子邮件和bug报告。 但是，对于多补丁系列，最好避免在回复时使用链接到该系列的旧版本。这样， -补丁的多个版本就不会成为电子邮件客户端中无法管理的引用序列。如果链接有用， +补丁的多个版本就不会成为电子邮件客户端中无法管理的引用树。如果链接有用， 可以使用 https://lore.kernel.org/ 重定向器（例如，在封面电子邮件文本中） 链接到补丁系列的早期版本。 -16) 发送git pull请求 --------------------- - -如果您有一系列补丁，那么让维护人员通过git pull操作将它们直接拉入子系统存储 -库可能是最方便的。但是，请注意，从开发人员那里获取补丁比从邮件列表中获取补 -丁需要更高的信任度。因此，许多子系统维护人员不愿意接受请求，特别是来自新的 -未知开发人员的请求。如果有疑问，您可以在封面邮件中使用pull 请求作为补丁系列 -正常发布的一个选项，让维护人员可以选择使用其中之一。 - -pull 请求的主题行中应该有[Git Pull]。请求本身应该在一行中包含存储库名称和 -感兴趣的分支；它应该看起来像:: +给出基础树信息 +-------------- - Please pull from +当其他开发人员收到您的补丁并开始审阅时，知道应该将您的工作放到代码树历史记录 +中的什么位置通常很有用。这对于自动化持续集成流水（CI）特别有用，这些流水线试 +图运行一系列测试，以便在维护人员开始审阅之前确定提交的质量。 - git://jdelvare.pck.nerim.net/jdelvare-2.6 i2c-for-linus +如果您使用 ``git format-patch`` 生成补丁，则可以通过 ``--base`` 标志在提交中 +自动包含基础树信息。使用此选项最简单、最方便的方法是配合主题分支:: - to get these changes: + $ git checkout -t -b my-topical-branch master + Branch 'my-topical-branch' set up to track local branch 'master'. + Switched to a new branch 'my-topical-branch' + [perform your edits and commits] -pull 请求还应该包含一条整体消息，说明请求中将包含什么，一个补丁本身的 ``Git shortlog`` -以及一个显示补丁系列整体效果的 ``diffstat`` 。当然，将所有这些信息收集在一起 -的最简单方法是让 ``git`` 使用 ``git request-pull`` 命令为您完成这些工作。 + $ git format-patch --base=auto --cover-letter -o outgoing/ master + outgoing/0000-cover-letter.patch + outgoing/0001-First-Commit.patch + outgoing/... -一些维护人员（包括Linus）希望看到来自已签名提交的请求；这增加了他们对你的 -请求信心。特别是，在没有签名标签的情况下，Linus 不会从像 Github 这样的公共 -托管站点拉请求。 +当你编辑 ``outgoing/0000-cover-letter.patch`` 时，您会注意到在它的最底部有一 +行 ``base-commit:`` 尾注，它为审阅者和CI工具提供了足够的信息以正确执行 +``git am`` 而不必担心冲突:: -创建此类签名的第一步是生成一个 GNRPG 密钥，并由一个或多个核心内核开发人员对 -其进行签名。这一步对新开发人员来说可能很困难，但没有办法绕过它。参加会议是 -找到可以签署您的密钥的开发人员的好方法。 + $ git checkout -b patch-review [base-commit-id] + Switched to a new branch 'patch-review' + $ git am patches.mbox + Applying: First Commit + Applying: ... -一旦您在Git 中准备了一个您希望有人拉的补丁系列，就用 ``git tag -s`` 创建一 -个签名标记。这将创建一个新标记，标识该系列中的最后一次提交，并包含用您的私 -钥创建的签名。您还可以将changelog样式的消息添加到标记中；这是一个描述拉请求 -整体效果的理想位置。 +有关此选项的更多信息，请参阅 ``man git-format-patch`` 。 -如果维护人员将要从中提取的树不是您正在使用的存储库，请不要忘记将已签名的标记 -显式推送到公共树。 +.. note:: -生成拉请求时，请使用已签名的标记作为目标。这样的命令可以实现:: + ``--base`` 功能是在2.9.0版git中引入的。 - git request-pull master git://my.public.tree/linux.git my-signed-tag +如果您不使用git格式化补丁，仍然可以包含相同的 ``base-commit`` 尾注，以指示您 +的工作所基于的树的提交哈希。你应该在封面邮件或系列的第一个补丁中添加它，它应 +该放在 ``---`` 行的下面或所有其他内容之后，即只在你的电子邮件签名之前。 参考文献 -------- -Andrew Morton, "The perfect patch" (tpp). +Andrew Morton，“完美的补丁”（tpp） -Jeff Garzik, "Linux kernel patch submission format". +Jeff Garzik，“Linux内核补丁提交格式” -Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". +Greg Kroah-Hartman，“如何惹恼内核子系统维护人员” @@ -642,16 +643,15 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". -NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! +不！！！别再发巨型补丁炸弹给linux-kernel@vger.kernel.org的人们了！ -Kernel Documentation/process/coding-style.rst: - :ref:`Documentation/translations/zh_CN/process/coding-style.rst ` +内核 Documentation/translations/zh_CN/process/coding-style.rst -Linus Torvalds's mail on the canonical patch format: +Linus Torvalds关于标准补丁格式的邮件 -Andi Kleen, "On submitting kernel patches" - Some strategies to get difficult or controversial changes in. +Andi Kleen，“提交补丁之路” + 一些帮助合入困难或有争议的变更的策略。 http://halobates.de/on-submitting-patches.pdf diff --git a/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst b/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst index 26b0f36f793d767d234fcd27fe15d47e7c87e89a..3076402406c48e3110a8e2f7bcb681a4d1864408 100644 --- a/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst +++ b/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst @@ -80,7 +80,7 @@ p->se.vruntime。一旦p->se.vruntime变得足够大，其它的任务将成为 CFS使用纳秒粒度的计时，不依赖于任何jiffies或HZ的细节。因此CFS并不像之前的调度器那样 有“时间片”的概念，也没有任何启发式的设计。唯一可调的参数（你需要打开CONFIG_SCHED_DEBUG）是： - /proc/sys/kernel/sched_min_granularity_ns + /sys/kernel/debug/sched/min_granularity_ns 它可以用来将调度器从“桌面”模式（也就是低时延）调节为“服务器”（也就是高批处理）模式。 它的默认设置是适合桌面的工作负载。SCHED_BATCH也被CFS调度器模块处理。 diff --git a/Documentation/translations/zh_TW/oops-tracing.txt b/Documentation/translations/zh_TW/oops-tracing.txt deleted file mode 100644 index be8e59f2abafa88e45472034facc4b35a4b4f9a6..0000000000000000000000000000000000000000 --- a/Documentation/translations/zh_TW/oops-tracing.txt +++ /dev/null @@ -1,212 +0,0 @@ -Chinese translated version of Documentation/admin-guide/bug-hunting.rst - -If you have any comment or update to the content, please contact the -original document maintainer directly. However, if you have a problem -communicating in English you can also ask the Chinese maintainer for -help. Contact the Chinese maintainer if this translation is outdated -or if there is a problem with the translation. - -Traditional Chinese maintainer: Hu Haowen ---------------------------------------------------------------------- -Documentation/admin-guide/bug-hunting.rst 的繁體中文版翻譯 - -如果想評論或更新本文的內容，請直接聯繫原文檔的維護者。如果你使用英文 -交流有困難的話，也可以向繁體中文版維護者求助。如果本翻譯更新不及時或 -者翻譯存在問題，請聯繫繁體中文版維護者。 - -繁體中文版維護者： 胡皓文 Hu Haowen -繁體中文版翻譯者： 胡皓文 Hu Haowen -繁體中文版校譯者： 胡皓文 Hu Haowen - -以下爲正文 ---------------------------------------------------------------------- - -注意： ksymoops 在2.6中是沒有用的。 請以原有格式使用Oops(來自dmesg，等等)。 -忽略任何這樣那樣關於「解碼Oops」或者「通過ksymoops運行」的文檔。 如果你貼出運行過 -ksymoops的來自2.6的Oops，人們只會讓你重貼一次。 - -快速總結 -------------- - -發現Oops並發送給看似相關的內核領域的維護者。別太擔心對不上號。如果你不確定就發給 -和你所做的事情相關的代碼的負責人。 如果可重現試著描述怎樣重構。 那甚至比oops更有 -價值。 - -如果你對於發送給誰一無所知， 發給linux-kernel@vger.kernel.org。感謝你幫助Linux -儘可能地穩定。 - -Oops在哪裡? ----------------------- - -通常Oops文本由klogd從內核緩衝區里讀取並傳給syslogd，由syslogd寫到syslog文件中， -典型地是/var/log/messages(依賴於/etc/syslog.conf)。有時klogd崩潰了,這種情況下你 -能夠運行dmesg > file來從內核緩衝區中讀取數據並保存下來。 否則你可以 -cat /proc/kmsg > file， 然而你必須介入中止傳輸， kmsg是一個「永不結束的文件」。如 -果機器崩潰壞到你不能輸入命令或者磁碟不可用那麼你有三種選擇:- - -（1） 手抄屏幕上的文本待機器重啓後再輸入計算機。 麻煩但如果沒有針對崩潰的準備， -這是僅有的選擇。 另外，你可以用數位相機把屏幕拍下來-不太好，但比沒有強。 如果信 -息滾動到了終端的上面，你會發現以高分辯率啓動（比如，vga=791）會讓你讀到更多的文 -本。（注意：這需要vesafb，所以對『早期』的oops沒有幫助） - -（2）用串口終端啓動（請參看Documentation/admin-guide/serial-console.rst），運行一個null -modem到另一台機器並用你喜歡的通訊工具獲取輸出。Minicom工作地很好。 - -（3）使用Kdump（請參看Documentation/admin-guide/kdump/kdump.rst）， -使用在Documentation/admin-guide/kdump/gdbmacros.txt中定義的dmesg gdb宏，從舊的內存中提取內核 -環形緩衝區。 - -完整信息 ----------------- - -注意：以下來自於Linus的郵件適用於2.4內核。 我因爲歷史原因保留了它，並且因爲其中 -一些信息仍然適用。 特別注意的是，請忽略任何ksymoops的引用。 - -From: Linus Torvalds - -怎樣跟蹤Oops.. [原發到linux-kernel的一封郵件] - -主要的竅門是有五年和這些煩人的oops消息打交道的經驗;-) - -實際上，你有辦法使它更簡單。我有兩個不同的方法： - - gdb /usr/src/linux/vmlinux - gdb> disassemble - -那是發現問題的簡單辦法，至少如果bug報告做的好的情況下（象這個一樣-運行ksymoops -得到oops發生的函數及函數內的偏移）。 - -哦，如果報告發生的內核以相同的編譯器和相似的配置編譯它會有幫助的。 - -另一件要做的事是反彙編bug報告的「Code」部分：ksymoops也會用正確的工具來做這件事， -但如果沒有那些工具你可以寫一個傻程序： - - char str[] = "\xXX\xXX\xXX..."; - main(){} - -並用gcc -g編譯它然後執行「disassemble str」（XX部分是由Oops報告的值-你可以僅剪切 -粘貼並用「\x」替換空格-我就是這麼做的，因爲我懶得寫程序自動做這一切）。 - -另外，你可以用scripts/decodecode這個shell腳本。它的使用方法是： -decodecode < oops.txt - -「Code」之後的十六進位字節可能（在某些架構上）有一些當前指令之前的指令字節以及 -當前和之後的指令字節 - -Code: f9 0f 8d f9 00 00 00 8d 42 0c e8 dd 26 11 c7 a1 60 ea 2b f9 8b 50 08 a1 -64 ea 2b f9 8d 34 82 8b 1e 85 db 74 6d 8b 15 60 ea 2b f9 <8b> 43 04 39 42 54 -7e 04 40 89 42 54 8b 43 04 3b 05 00 f6 52 c0 - -最後，如果你想知道代碼來自哪裡，你可以： - - cd /usr/src/linux - make fs/buffer.s # 或任何產生BUG的文件 - -然後你會比gdb反彙編更清楚的知道發生了什麼。 - -現在，問題是把你所擁有的所有數據結合起來：C源碼（關於它應該怎樣的一般知識）， -彙編代碼及其反彙編得到的代碼（另外還有從「oops」消息得到的寄存器狀態-對了解毀壞的 -指針有用，而且當你有了彙編代碼你也能拿其它的寄存器和任何它們對應的C表達式做匹配 -）。 - -實際上，你僅需看看哪裡不匹配（這個例子是「Code」反彙編和編譯器生成的代碼不匹配）。 -然後你須要找出爲什麼不匹配。通常很簡單-你看到代碼使用了空指針然後你看代碼想知道 -空指針是怎麼出現的，還有檢查它是否合法.. - -現在，如果明白這是一項耗時的工作而且需要一丁點兒的專心，沒錯。這就是我爲什麼大多 -只是忽略那些沒有符號表信息的崩潰報告的原因：簡單的說太難查找了（我有一些 -程序用於在內核代碼段中搜索特定的模式，而且有時我也已經能找出那些崩潰的地方，但是 -僅僅是找出正確的序列也確實需要相當紮實的內核知識） - -_有時_會發生這種情況，我僅看到崩潰中的反彙編代碼序列， 然後我馬上就明白問題出在 -哪裡。這時我才意識到自己幹這個工作已經太長時間了;-) - - Linus - - ---------------------------------------------------------------------------- -關於Oops跟蹤的註解： - -爲了幫助Linus和其它內核開發者，klogd納入了大量的支持來處理保護錯誤。爲了擁有對 -地址解析的完整支持至少應該使用1.3-pl3的sysklogd包。 - -當保護錯誤發生時，klogd守護進程自動把內核日誌信息中的重要地址翻譯成它們相應的符 -號。 - -klogd執行兩種類型的地址解析。首先是靜態翻譯其次是動態翻譯。靜態翻譯和ksymoops -一樣使用System.map文件。爲了做靜態翻譯klogd守護進程必須在初始化時能找到system -map文件。關於klogd怎樣搜索map文件請參看klogd手冊頁。 - -動態地址翻譯在使用內核可裝載模塊時很重要。 因爲內核模塊的內存是從內核動態內存池 -里分配的，所以不管是模塊開始位置還是模塊中函數和符號的位置都不是固定的。 - -內核支持允許程序決定裝載哪些模塊和它們在內存中位置的系統調用。使用這些系統調用 -klogd守護進程生成一張符號表用於調試發生在可裝載模塊中的保護錯誤。 - -至少klogd會提供產生保護錯誤的模塊名。還可有額外的符號信息供可裝載模塊開發者選擇 -以從模塊中輸出符號信息。 - -因爲內核模塊環境可能是動態的，所以必須有一種機制當模塊環境發生改變時來通知klogd -守護進程。 有一些可用的命令行選項允許klogd向當前執行中的守護進程發送信號，告知符 -號信息應該被刷新了。 更多信息請參看klogd手冊頁。 - -sysklogd發布時包含一個補丁修改了modules-2.0.0包，無論何時一個模塊裝載或者卸載都 -會自動向klogd發送信號。打上這個補丁提供了必要的對調試發生於內核可裝載模塊的保護 -錯誤的無縫支持。 - -以下是被klogd處理過的發生在可裝載模塊中的一個保護錯誤例子： ---------------------------------------------------------------------------- -Aug 29 09:51:01 blizard kernel: Unable to handle kernel paging request at virtual address f15e97cc -Aug 29 09:51:01 blizard kernel: current->tss.cr3 = 0062d000, %cr3 = 0062d000 -Aug 29 09:51:01 blizard kernel: *pde = 00000000 -Aug 29 09:51:01 blizard kernel: Oops: 0002 -Aug 29 09:51:01 blizard kernel: CPU: 0 -Aug 29 09:51:01 blizard kernel: EIP: 0010:[oops:_oops+16/3868] -Aug 29 09:51:01 blizard kernel: EFLAGS: 00010212 -Aug 29 09:51:01 blizard kernel: eax: 315e97cc ebx: 003a6f80 ecx: 001be77b edx: 00237c0c -Aug 29 09:51:01 blizard kernel: esi: 00000000 edi: bffffdb3 ebp: 00589f90 esp: 00589f8c -Aug 29 09:51:01 blizard kernel: ds: 0018 es: 0018 fs: 002b gs: 002b ss: 0018 -Aug 29 09:51:01 blizard kernel: Process oops_test (pid: 3374, process nr: 21, stackpage=00589000) -Aug 29 09:51:01 blizard kernel: Stack: 315e97cc 00589f98 0100b0b4 bffffed4 0012e38e 00240c64 003a6f80 00000001 -Aug 29 09:51:01 blizard kernel: 00000000 00237810 bfffff00 0010a7fa 00000003 00000001 00000000 bfffff00 -Aug 29 09:51:01 blizard kernel: bffffdb3 bffffed4 ffffffda 0000002b 0007002b 0000002b 0000002b 00000036 -Aug 29 09:51:01 blizard kernel: Call Trace: [oops:_oops_ioctl+48/80] [_sys_ioctl+254/272] [_system_call+82/128] -Aug 29 09:51:01 blizard kernel: Code: c7 00 05 00 00 00 eb 08 90 90 90 90 90 90 90 90 89 ec 5d c3 ---------------------------------------------------------------------------- - -Dr. G.W. Wettstein Oncology Research Div. Computing Facility -Roger Maris Cancer Center INTERNET: greg@wind.rmcc.com -820 4th St. N. -Fargo, ND 58122 -Phone: 701-234-7556 - - ---------------------------------------------------------------------------- -受汙染的內核 - -一些oops報告在程序記數器之後包含字符串'Tainted: '。這表明內核已經被一些東西給汙 -染了。 該字符串之後緊跟著一系列的位置敏感的字符，每個代表一個特定的汙染值。 - - 1：'G'如果所有裝載的模塊都有GPL或相容的許可證，'P'如果裝載了任何的專有模塊。 -沒有模塊MODULE_LICENSE或者帶有insmod認爲是與GPL不相容的的MODULE_LICENSE的模塊被 -認定是專有的。 - - 2：'F'如果有任何通過「insmod -f」被強制裝載的模塊，' '如果所有模塊都被正常裝載。 - - 3：'S'如果oops發生在SMP內核中，運行於沒有證明安全運行多處理器的硬體。 當前這種 -情況僅限於幾種不支持SMP的速龍處理器。 - - 4：'R'如果模塊通過「insmod -f」被強制裝載，' '如果所有模塊都被正常裝載。 - - 5：'M'如果任何處理器報告了機器檢查異常，' '如果沒有發生機器檢查異常。 - - 6：'B'如果頁釋放函數發現了一個錯誤的頁引用或者一些非預期的頁標誌。 - - 7：'U'如果用戶或者用戶應用程式特別請求設置汙染標誌，否則' '。 - - 8：'D'如果內核剛剛死掉，比如有OOPS或者BUG。 - -使用'Tainted: '字符串的主要原因是要告訴內核調試者，這是否是一個乾淨的內核亦或發 -生了任何的不正常的事。汙染是永久的：即使出錯的模塊已經被卸載了，汙染值仍然存在， -以表明內核不再值得信任。 - diff --git a/Documentation/translations/zh_TW/process/howto.rst b/Documentation/translations/zh_TW/process/howto.rst index 68ae4411285b85d0ee673553ddaac5684cbee036..86b0d4c6d6f9757abe13c40633bf3553fbf1c158 100644 --- a/Documentation/translations/zh_TW/process/howto.rst +++ b/Documentation/translations/zh_TW/process/howto.rst @@ -309,7 +309,7 @@ bugzilla.kernel.org是Linux內核開發者們用來跟蹤內核Bug的網站。 網上很多地方都有這個郵件列表的存檔(archive)。可以使用搜尋引擎來找到這些 存檔。比如： - http://dir.gmane.org/gmane.linux.kernel + https://lore.kernel.org/lkml/ 在發信之前，我們強烈建議你先在存檔中搜索你想要討論的問題。很多已經被詳細 討論過的問題只在郵件列表的存檔中可以找到。 diff --git a/Documentation/translations/zh_TW/process/magic-number.rst b/Documentation/translations/zh_TW/process/magic-number.rst index ae321a9aaece20f4a787ae51e972a22bb74d3ef8..f3f7082e17c61e1020be01ac11041029d6e68d62 100644 --- a/Documentation/translations/zh_TW/process/magic-number.rst +++ b/Documentation/translations/zh_TW/process/magic-number.rst @@ -61,88 +61,17 @@ Linux 魔術數 魔術數名 數字 結構 文件 ===================== ================ ======================== ========================================== PG_MAGIC 'P' pg_{read,write}_hdr ``include/linux/pg.h`` -CMAGIC 0x0111 user ``include/linux/a.out.h`` -MKISS_DRIVER_MAGIC 0x04bf mkiss_channel ``drivers/net/mkiss.h`` -HDLC_MAGIC 0x239e n_hdlc ``drivers/char/n_hdlc.c`` APM_BIOS_MAGIC 0x4101 apm_user ``arch/x86/kernel/apm_32.c`` -DB_MAGIC 0x4442 fc_info ``drivers/net/iph5526_novram.c`` -DL_MAGIC 0x444d fc_info ``drivers/net/iph5526_novram.c`` FASYNC_MAGIC 0x4601 fasync_struct ``include/linux/fs.h`` -FF_MAGIC 0x4646 fc_info ``drivers/net/iph5526_novram.c`` -PTY_MAGIC 0x5001 ``drivers/char/pty.c`` -PPP_MAGIC 0x5002 ppp ``include/linux/if_pppvar.h`` -SSTATE_MAGIC 0x5302 serial_state ``include/linux/serial.h`` SLIP_MAGIC 0x5302 slip ``drivers/net/slip.h`` -STRIP_MAGIC 0x5303 strip ``drivers/net/strip.c`` -SIXPACK_MAGIC 0x5304 sixpack ``drivers/net/hamradio/6pack.h`` -AX25_MAGIC 0x5316 ax_disp ``drivers/net/mkiss.h`` -TTY_MAGIC 0x5401 tty_struct ``include/linux/tty.h`` -MGSL_MAGIC 0x5401 mgsl_info ``drivers/char/synclink.c`` -TTY_DRIVER_MAGIC 0x5402 tty_driver ``include/linux/tty_driver.h`` MGSLPC_MAGIC 0x5402 mgslpc_info ``drivers/char/pcmcia/synclink_cs.c`` -USB_SERIAL_MAGIC 0x6702 usb_serial ``drivers/usb/serial/usb-serial.h`` -FULL_DUPLEX_MAGIC 0x6969 ``drivers/net/ethernet/dec/tulip/de2104x.c`` -USB_BLUETOOTH_MAGIC 0x6d02 usb_bluetooth ``drivers/usb/class/bluetty.c`` -RFCOMM_TTY_MAGIC 0x6d02 ``net/bluetooth/rfcomm/tty.c`` -USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/serial/usb-serial.h`` -CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` -LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -GDTIOCTL_MAGIC 0x06030f07 gdth_iowr_str ``drivers/scsi/gdth_ioctl.h`` -RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` -NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` -RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` BAYCOM_MAGIC 0x19730510 baycom_state ``drivers/net/baycom_epp.c`` -ISDN_X25IFACE_MAGIC 0x1e75a2b9 isdn_x25iface_proto_data ``drivers/isdn/isdn_x25iface.h`` -ECP_MAGIC 0x21504345 cdkecpsig ``include/linux/cdk.h`` -LSOMAGIC 0x27091997 lso ``drivers/fc4/fc.c`` -LSMAGIC 0x2a3b4d2a ls ``drivers/fc4/fc.c`` -WANPIPE_MAGIC 0x414C4453 sdla_{dump,exec} ``include/linux/wanpipe.h`` -CS_CARD_MAGIC 0x43525553 cs_card ``sound/oss/cs46xx.c`` -LABELCL_MAGIC 0x4857434c labelcl_info_s ``include/asm/ia64/sn/labelcl.h`` -ISDN_ASYNC_MAGIC 0x49344C01 modem_info ``include/linux/isdn.h`` -CTC_ASYNC_MAGIC 0x49344C01 ctc_tty_info ``drivers/s390/net/ctctty.c`` -ISDN_NET_MAGIC 0x49344C02 isdn_net_local_s ``drivers/isdn/i4l/isdn_net_lib.h`` -SAVEKMSG_MAGIC2 0x4B4D5347 savekmsg ``arch/*/amiga/config.c`` -CS_STATE_MAGIC 0x4c4f4749 cs_state ``sound/oss/cs46xx.c`` -SLAB_C_MAGIC 0x4f17a36d kmem_cache ``mm/slab.c`` -COW_MAGIC 0x4f4f4f4d cow_header_v1 ``arch/um/drivers/ubd_user.c`` -I810_CARD_MAGIC 0x5072696E i810_card ``sound/oss/i810_audio.c`` -TRIDENT_CARD_MAGIC 0x5072696E trident_card ``sound/oss/trident.c`` -ROUTER_MAGIC 0x524d4157 wan_device [in ``wanrouter.h`` pre 3.9] -SAVEKMSG_MAGIC1 0x53415645 savekmsg ``arch/*/amiga/config.c`` -GDA_MAGIC 0x58464552 gda ``arch/mips/include/asm/sn/gda.h`` -RED_MAGIC1 0x5a2cf071 (any) ``mm/slab.c`` -EEPROM_MAGIC_VALUE 0x5ab478d2 lanai_dev ``drivers/atm/lanai.c`` HDLCDRV_MAGIC 0x5ac6e778 hdlcdrv_state ``include/linux/hdlcdrv.h`` -PCXX_MAGIC 0x5c6df104 channel ``drivers/char/pcxx.h`` KV_MAGIC 0x5f4b565f kernel_vars_s ``arch/mips/include/asm/sn/klkernvars.h`` -I810_STATE_MAGIC 0x63657373 i810_state ``sound/oss/i810_audio.c`` -TRIDENT_STATE_MAGIC 0x63657373 trient_state ``sound/oss/trident.c`` -M3_CARD_MAGIC 0x646e6f50 m3_card ``sound/oss/maestro3.c`` -FW_HEADER_MAGIC 0x65726F66 fw_header ``drivers/atm/fore200e.h`` -SLOT_MAGIC 0x67267321 slot ``drivers/hotplug/cpqphp.h`` -SLOT_MAGIC 0x67267322 slot ``drivers/hotplug/acpiphp.h`` -LO_MAGIC 0x68797548 nbd_device ``include/linux/nbd.h`` -M3_STATE_MAGIC 0x734d724d m3_state ``sound/oss/maestro3.c`` -VMALLOC_MAGIC 0x87654320 snd_alloc_track ``sound/core/memory.c`` -KMALLOC_MAGIC 0x87654321 snd_alloc_track ``sound/core/memory.c`` -PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/media/pwc.h`` -NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` -ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` -DPMEM_MAGIC 0xc0ffee11 gdt_pci_sram ``drivers/scsi/gdth.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` QUEUE_MAGIC_USED 0xf7e1cc33 queue_entry ``drivers/scsi/arm/queue.c`` -HTB_CMAGIC 0xFEFAFEF1 htb_class ``net/sched/sch_htb.c`` NMI_MAGIC 0x48414d4d455201 nmi_s ``arch/mips/include/asm/sn/nmi.h`` ===================== ================ ======================== ========================================== - - -請注意，在聲音記憶管理中仍然有一些特殊的爲每個驅動定義的魔術值。查看include/sound/sndmagic.h來獲取他們完整的列表信息。很多OSS聲音驅動擁有自己從音效卡PCI ID構建的魔術值-他們也沒有被列在這裡。 - -IrDA子系統也使用了大量的自己的魔術值，查看include/net/irda/irda.h來獲取他們完整的信息。 - -HFS是另外一個比較大的使用魔術值的文件系統-你可以在fs/hfs/hfs.h中找到他們。 - diff --git a/Documentation/usb/usbip_protocol.rst b/Documentation/usb/usbip_protocol.rst index 0b8541fda4d8476e7686b6f569e3345ef5d64a76..adc158967cc6113188d751c96b21f90393cefe88 100644 --- a/Documentation/usb/usbip_protocol.rst +++ b/Documentation/usb/usbip_protocol.rst @@ -340,13 +340,12 @@ USBIP_CMD_SUBMIT: | 0 | 20 | usbip_header_basic, 'command' shall be 0x00000001 | +-----------+--------+---------------------------------------------------+ | 0x14 | 4 | transfer_flags: possible values depend on the | -| | | URB transfer_flags (refer to URB doc in | -| | | Documentation/driver-api/usb/URB.rst) | -| | | but with URB_NO_TRANSFER_DMA_MAP masked. Refer to | -| | | function usbip_pack_cmd_submit and function | -| | | tweak_transfer_flags in drivers/usb/usbip/ | -| | | usbip_common.c. The following fields may also ref | -| | | to function usbip_pack_cmd_submit and URB doc | +| | | USBIP_URB transfer_flags. | +| | | Refer to include/uapi/linux/usbip.h and | +| | | Documentation/driver-api/usb/URB.rst. | +| | | Refer to usbip_pack_cmd_submit() and | +| | | tweak_transfer_flags() in drivers/usb/usbip/ | +| | | usbip_common.c. | +-----------+--------+---------------------------------------------------+ | 0x18 | 4 | transfer_buffer_length: | | | | use URB transfer_buffer_length | diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index a61eac0c73f8252dbb5da0afae2457ecbb4b41b1..c78da9ce0ec44e5c9feca7ee262e31ed668b9d99 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -26,6 +26,7 @@ place where this information is gathered. ioctl/index iommu media/index + netlink/index sysfs-platform_profile vduse futex2 diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 3b985b19f39d1223a6fc735101cb414ea5d48553..5f81e2a24a5c04221e49af03fcdde934a35cc855 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -308,7 +308,6 @@ Code Seq# Include File Comments 0x89 00-06 arch/x86/include/asm/sockios.h 0x89 0B-DF linux/sockios.h 0x89 E0-EF linux/sockios.h SIOCPROTOPRIVATE range -0x89 E0-EF linux/dn.h PROTOPRIVATE range 0x89 F0-FF linux/sockios.h SIOCDEVPRIVATE range 0x8B all linux/wireless.h 0x8C 00-3F WiNRADiO driver diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst index b8ea5949396481776392399e2cc34ffc41148e4a..cec780c2f4973c88d60702575f38756f50367abd 100644 --- a/Documentation/userspace-api/landlock.rst +++ b/Documentation/userspace-api/landlock.rst @@ -8,7 +8,7 @@ Landlock: unprivileged access control ===================================== :Author: Mickaël Salaün -:Date: May 2022 +:Date: September 2022 The goal of Landlock is to enable to restrict ambient rights (e.g. global filesystem access) for a set of processes. Because Landlock is a stackable @@ -69,7 +69,7 @@ should try to protect users as much as possible whatever the kernel they are using. To avoid binary enforcement (i.e. either all security features or none), we can leverage a dedicated Landlock command to get the current version of the Landlock ABI and adapt the handled accesses. Let's check if we should -remove the `LANDLOCK_ACCESS_FS_REFER` access right which is only supported +remove the ``LANDLOCK_ACCESS_FS_REFER`` access right which is only supported starting with the second version of the ABI. .. code-block:: c @@ -128,7 +128,7 @@ descriptor. It may also be required to create rules following the same logic as explained for the ruleset creation, by filtering access rights according to the Landlock ABI version. In this example, this is not required because -`LANDLOCK_ACCESS_FS_REFER` is not allowed by any rule. +``LANDLOCK_ACCESS_FS_REFER`` is not allowed by any rule. We now have a ruleset with one rule allowing read access to ``/usr`` while denying all other handled accesses for the filesystem. The next step is to @@ -154,8 +154,8 @@ The current thread is now ready to sandbox itself with the ruleset. } close(ruleset_fd); -If the `landlock_restrict_self` system call succeeds, the current thread is now -restricted and this policy will be enforced on all its subsequently created +If the ``landlock_restrict_self`` system call succeeds, the current thread is +now restricted and this policy will be enforced on all its subsequently created children as well. Once a thread is landlocked, there is no way to remove its security policy; only adding more restrictions is allowed. These threads are now in a new Landlock domain, merge of their parent one (if any) with the new @@ -170,12 +170,13 @@ It is recommended setting access rights to file hierarchy leaves as much as possible. For instance, it is better to be able to have ``~/doc/`` as a read-only hierarchy and ``~/tmp/`` as a read-write hierarchy, compared to ``~/`` as a read-only hierarchy and ``~/tmp/`` as a read-write hierarchy. -Following this good practice leads to self-sufficient hierarchies that don't +Following this good practice leads to self-sufficient hierarchies that do not depend on their location (i.e. parent directories). This is particularly relevant when we want to allow linking or renaming. Indeed, having consistent access rights per directory enables to change the location of such directory without relying on the destination directory access rights (except those that -are required for this operation, see `LANDLOCK_ACCESS_FS_REFER` documentation). +are required for this operation, see ``LANDLOCK_ACCESS_FS_REFER`` +documentation). Having self-sufficient hierarchies also helps to tighten the required access rights to the minimal set of data. This also helps avoid sinkhole directories, i.e. directories where data can be linked to but not linked from. However, @@ -259,7 +260,7 @@ Backward and forward compatibility Landlock is designed to be compatible with past and future versions of the kernel. This is achieved thanks to the system call attributes and the -associated bitflags, particularly the ruleset's `handled_access_fs`. Making +associated bitflags, particularly the ruleset's ``handled_access_fs``. Making handled access right explicit enables the kernel and user space to have a clear contract with each other. This is required to make sure sandboxing will not get stricter with a system update, which could break applications. @@ -380,8 +381,8 @@ by the Documentation/admin-guide/cgroup-v1/memory.rst. Previous limitations ==================== -File renaming and linking (ABI 1) ---------------------------------- +File renaming and linking (ABI < 2) +----------------------------------- Because Landlock targets unprivileged access controls, it needs to properly handle composition of rules. Such property also implies rules nesting. @@ -394,7 +395,7 @@ according to the potentially lost constraints. To protect against privilege escalations through renaming or linking, and for the sake of simplicity, Landlock previously limited linking and renaming to the same directory. Starting with the Landlock ABI version 2, it is now possible to securely -control renaming and linking thanks to the new `LANDLOCK_ACCESS_FS_REFER` +control renaming and linking thanks to the new ``LANDLOCK_ACCESS_FS_REFER`` access right. .. _kernel_support: @@ -403,14 +404,14 @@ Kernel support ============== Landlock was first introduced in Linux 5.13 but it must be configured at build -time with `CONFIG_SECURITY_LANDLOCK=y`. Landlock must also be enabled at boot +time with ``CONFIG_SECURITY_LANDLOCK=y``. Landlock must also be enabled at boot time as the other security modules. The list of security modules enabled by -default is set with `CONFIG_LSM`. The kernel configuration should then -contains `CONFIG_LSM=landlock,[...]` with `[...]` as the list of other +default is set with ``CONFIG_LSM``. The kernel configuration should then +contains ``CONFIG_LSM=landlock,[...]`` with ``[...]`` as the list of other potentially useful security modules for the running system (see the -`CONFIG_LSM` help). +``CONFIG_LSM`` help). -If the running kernel doesn't have `landlock` in `CONFIG_LSM`, then we can +If the running kernel does not have ``landlock`` in ``CONFIG_LSM``, then we can still enable it by adding ``lsm=landlock,[...]`` to Documentation/admin-guide/kernel-parameters.rst thanks to the bootloader configuration. diff --git a/Documentation/userspace-api/media/drivers/dw100.rst b/Documentation/userspace-api/media/drivers/dw100.rst new file mode 100644 index 0000000000000000000000000000000000000000..fceea6ece62213b1b1a91ebfbc013189ea670725 --- /dev/null +++ b/Documentation/userspace-api/media/drivers/dw100.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0 + +DW100 dewarp driver +=================== + +The Vivante DW100 Dewarp Processor IP core found on i.MX8MP SoC applies a +programmable geometrical transformation on the input image to correct distortion +introduced by lenses. + +The transformation function is exposed by the hardware as a grid map with 16x16 +pixel macroblocks indexed using X, Y vertex coordinates. +:: + + Image width + <---------------------------------------> + + ^ .-------.-------.-------.-------.-------. + | | 16x16 | | | | | + I | | pixel | | | | | + m | | block | | | | | + a | .-------.-------.-------.-------.-------. + g | | | | | | | + e | | | | | | | + | | | | | | | + h | .-------.-------.-------.-------.-------. + e | | | | | | | + i | | | | | | | + g | | | | | | | + h | .-------.-------.-------.-------.-------. + t | | | | | | | + | | | | | | | + | | | | | | | + v '-------'-------'-------'-------'-------' + + Grid of Image Blocks for Dewarping Map + + +Each x, y coordinate register uses 16 bits to record the coordinate address in +an unsigned 12.4 fixed point format (UQ12.4). +:: + + .----------------------.--------..----------------------.--------. + | 31~20 | 19~16 || 15~4 | 3~0 | + | (integer) | (frac) || (integer) | (frac) | + '----------------------'--------''----------------------'--------' + <-------------------------------><-------------------------------> + Y coordinate X coordinate + + Remap Register Layout + +The dewarping map is set from applications using the +V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP control. The control contains +an array of u32 values storing (x, y) destination coordinates for each +vertex of the grid. The x coordinate is stored in the 16 LSBs and the y +coordinate in the 16 MSBs. + +The number of elements in the array must match the image size: + +.. code-block:: C + + elems = (DIV_ROUND_UP(width, 16) + 1) * (DIV_ROUND_UP(height, 16) + 1); + +If the control has not been set by the application, the driver uses an identity +map. + +More details on the DW100 hardware operations can be found in +*chapter 13.15 DeWarp* of IMX8MP_ reference manual. + +The Vivante DW100 m2m driver implements the following driver-specific control: + +``V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP (__u32 array)`` + Specifies to DW100 driver its dewarping map (aka LUT) blob as described in + *chapter 13.15.2.3 Dewarping Remap* of IMX8MP_ reference manual as an U32 + dynamic array. The image is divided into many small 16x16 blocks. If the + width/height of the image is not divisible by 16, the size of the + rightmost/bottommost block is the remainder. The dewarping map only saves + the vertex coordinates of the block. The dewarping grid map is comprised of + vertex coordinates for x and y. Each x, y coordinate register uses 16 bits + (UQ12.4) to record the coordinate address, with the Y coordinate in the + upper bits and X in the lower bits. The driver modifies the dimensions of + this control when the sink format is changed, to reflect the new input + resolution. + +.. _IMX8MP: https://www.nxp.com/webapp/Download?colCode=IMX8MPRM diff --git a/Documentation/userspace-api/media/drivers/index.rst b/Documentation/userspace-api/media/drivers/index.rst index 1a9038f5f9fae8e708c2525ec3a9e35a481274d0..32f82aed47d91545a931a4eb5e53049d5b224384 100644 --- a/Documentation/userspace-api/media/drivers/index.rst +++ b/Documentation/userspace-api/media/drivers/index.rst @@ -33,6 +33,7 @@ For more details see the file COPYING in the source distribution of Linux. ccs cx2341x-uapi + dw100 imx-uapi max2175 meye-uapi diff --git a/Documentation/userspace-api/media/v4l/async.rst b/Documentation/userspace-api/media/v4l/async.rst deleted file mode 100644 index d6960ff5c3824158e2339b1a7d2465683f4eba8f..0000000000000000000000000000000000000000 --- a/Documentation/userspace-api/media/v4l/async.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. SPDX-License-Identifier: GFDL-1.1-no-invariants-or-later - -.. _async: - -**************** -Asynchronous I/O -**************** - -This method is not defined yet. diff --git a/Documentation/userspace-api/media/v4l/dev-raw-vbi.rst b/Documentation/userspace-api/media/v4l/dev-raw-vbi.rst index 58f97c3a77923e0d7c8d59a66ae7df04d0301635..2bec20d879287a2c449be216f356f122b3986eda 100644 --- a/Documentation/userspace-api/media/v4l/dev-raw-vbi.rst +++ b/Documentation/userspace-api/media/v4l/dev-raw-vbi.rst @@ -41,7 +41,7 @@ Devices supporting the raw VBI capturing or output API set the in the ``capabilities`` field of struct :c:type:`v4l2_capability` returned by the :ref:`VIDIOC_QUERYCAP` ioctl. At least one of the -read/write, streaming or asynchronous I/O methods must be supported. VBI +read/write or streaming I/O methods must be supported. VBI devices may or may not have a tuner or modulator. Supplemental Functions diff --git a/Documentation/userspace-api/media/v4l/dev-sdr.rst b/Documentation/userspace-api/media/v4l/dev-sdr.rst index 928884dfe09da34c29e833c4324e5fc94682be1c..dfdeddbca41f3e602aec160b91199b54ebc103d3 100644 --- a/Documentation/userspace-api/media/v4l/dev-sdr.rst +++ b/Documentation/userspace-api/media/v4l/dev-sdr.rst @@ -34,7 +34,7 @@ Devices supporting the SDR transmitter interface set the device has an Digital to Analog Converter (DAC), which is a mandatory element for the SDR transmitter. -At least one of the read/write, streaming or asynchronous I/O methods +At least one of the read/write or streaming I/O methods must be supported. diff --git a/Documentation/userspace-api/media/v4l/dev-sliced-vbi.rst b/Documentation/userspace-api/media/v4l/dev-sliced-vbi.rst index 97ec2b115c7168568381c8332aa6d4eddaffbd99..44415822c7c5a30fbb658156b5dcadecd61a23cd 100644 --- a/Documentation/userspace-api/media/v4l/dev-sliced-vbi.rst +++ b/Documentation/userspace-api/media/v4l/dev-sliced-vbi.rst @@ -36,7 +36,7 @@ Devices supporting the sliced VBI capturing or output API set the respectively, in the ``capabilities`` field of struct :c:type:`v4l2_capability` returned by the :ref:`VIDIOC_QUERYCAP` ioctl. At least one of the -read/write, streaming or asynchronous :ref:`I/O methods ` must be +read/write or streaming :ref:`I/O methods ` must be supported. Sliced VBI devices may have a tuner or modulator. Supplemental Functions diff --git a/Documentation/userspace-api/media/v4l/hist-v4l2.rst b/Documentation/userspace-api/media/v4l/hist-v4l2.rst index 28a2750d5c8cc112953617dfe14bb4caf1ebcfdb..dbc04374dc224f78c5e1e792f829227b11dadc96 100644 --- a/Documentation/userspace-api/media/v4l/hist-v4l2.rst +++ b/Documentation/userspace-api/media/v4l/hist-v4l2.rst @@ -316,7 +316,7 @@ This unnamed version was finally merged into Linux 2.5.46. There are new fields to identify the driver, a new RDS device function ``V4L2_CAP_RDS_CAPTURE``, the ``V4L2_CAP_AUDIO`` flag indicates if the device has any audio connectors, another I/O - capability ``V4L2_CAP_ASYNCIO`` can be flagged. In response to these + capability V4L2_CAP_ASYNCIO can be flagged. In response to these changes the ``type`` field became a bit set and was merged into the ``flags`` field. ``V4L2_FLAG_TUNER`` was renamed to ``V4L2_CAP_TUNER``, ``V4L2_CAP_VIDEO_OVERLAY`` replaced diff --git a/Documentation/userspace-api/media/v4l/io.rst b/Documentation/userspace-api/media/v4l/io.rst index ce0cece6f35ff676f2ba5e3f63cb6efca4eb0086..4b1964df9d73a41d1b307a7fcac2f95af24c1a7b 100644 --- a/Documentation/userspace-api/media/v4l/io.rst +++ b/Documentation/userspace-api/media/v4l/io.rst @@ -17,8 +17,7 @@ read or write will fail at any time. Other methods must be negotiated. To select the streaming I/O method with memory mapped or user buffers applications call the -:ref:`VIDIOC_REQBUFS` ioctl. The asynchronous I/O -method is not defined yet. +:ref:`VIDIOC_REQBUFS` ioctl. Video overlay can be considered another I/O method, although the application does not directly receive the image data. It is selected by @@ -46,6 +45,5 @@ The following sections describe the various I/O methods in more detail. mmap userp dmabuf - async buffer field-order diff --git a/Documentation/userspace-api/media/v4l/vidioc-dqevent.rst b/Documentation/userspace-api/media/v4l/vidioc-dqevent.rst index 6eb40073c906debaa7992336bbda3bf9b1bdc9db..8db103760930940bf5ed4c486dd9ad556ce4937c 100644 --- a/Documentation/userspace-api/media/v4l/vidioc-dqevent.rst +++ b/Documentation/userspace-api/media/v4l/vidioc-dqevent.rst @@ -332,6 +332,11 @@ call. - 0x0004 - This control event was triggered because the minimum, maximum, step or the default value of the control changed. + * - ``V4L2_EVENT_CTRL_CH_DIMENSIONS`` + - 0x0008 + - This control event was triggered because the dimensions of the + control changed. Note that the number of dimensions remains the + same. .. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.5cm}| diff --git a/Documentation/userspace-api/media/v4l/vidioc-querycap.rst b/Documentation/userspace-api/media/v4l/vidioc-querycap.rst index 63e23f6f95ee9f60e16ef931b05bb82c794ec452..6c57b84283566f362a28b3fef7b1be2571578615 100644 --- a/Documentation/userspace-api/media/v4l/vidioc-querycap.rst +++ b/Documentation/userspace-api/media/v4l/vidioc-querycap.rst @@ -244,9 +244,6 @@ specification the ioctl returns an ``EINVAL`` error code. - 0x01000000 - The device supports the :c:func:`read()` and/or :c:func:`write()` I/O methods. - * - ``V4L2_CAP_ASYNCIO`` - - 0x02000000 - - The device supports the :ref:`asynchronous ` I/O methods. * - ``V4L2_CAP_STREAMING`` - 0x04000000 - The device supports the :ref:`streaming ` I/O method. diff --git a/Documentation/userspace-api/media/videodev2.h.rst.exceptions b/Documentation/userspace-api/media/videodev2.h.rst.exceptions index 64b2c0b1f66675ae490550758d9349c9a1e75675..2a589d34b80ea3f14f18452a57aa93d77331398e 100644 --- a/Documentation/userspace-api/media/videodev2.h.rst.exceptions +++ b/Documentation/userspace-api/media/videodev2.h.rst.exceptions @@ -514,6 +514,7 @@ replace define V4L2_EVENT_PRIVATE_START event-type replace define V4L2_EVENT_CTRL_CH_VALUE ctrl-changes-flags replace define V4L2_EVENT_CTRL_CH_FLAGS ctrl-changes-flags replace define V4L2_EVENT_CTRL_CH_RANGE ctrl-changes-flags +replace define V4L2_EVENT_CTRL_CH_DIMENSIONS ctrl-changes-flags replace define V4L2_EVENT_SRC_CH_RESOLUTION src-changes-flags diff --git a/Documentation/userspace-api/netlink/index.rst b/Documentation/userspace-api/netlink/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..b0c21538d97d9a7d4c9d11e766d55ade7301320d --- /dev/null +++ b/Documentation/userspace-api/netlink/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: BSD-3-Clause + +================ +Netlink Handbook +================ + +Netlink documentation for users. + +.. toctree:: + :maxdepth: 2 + + intro diff --git a/Documentation/userspace-api/netlink/intro.rst b/Documentation/userspace-api/netlink/intro.rst new file mode 100644 index 0000000000000000000000000000000000000000..0955e9f203d3b9d4cf2a8984bde53bc573bfa4c8 --- /dev/null +++ b/Documentation/userspace-api/netlink/intro.rst @@ -0,0 +1,681 @@ +.. SPDX-License-Identifier: BSD-3-Clause + +======================= +Introduction to Netlink +======================= + +Netlink is often described as an ioctl() replacement. +It aims to replace fixed-format C structures as supplied +to ioctl() with a format which allows an easy way to add +or extended the arguments. + +To achieve this Netlink uses a minimal fixed-format metadata header +followed by multiple attributes in the TLV (type, length, value) format. + +Unfortunately the protocol has evolved over the years, in an organic +and undocumented fashion, making it hard to coherently explain. +To make the most practical sense this document starts by describing +netlink as it is used today and dives into more "historical" uses +in later sections. + +Opening a socket +================ + +Netlink communication happens over sockets, a socket needs to be +opened first: + +.. code-block:: c + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + +The use of sockets allows for a natural way of exchanging information +in both directions (to and from the kernel). The operations are still +performed synchronously when applications send() the request but +a separate recv() system call is needed to read the reply. + +A very simplified flow of a Netlink "call" will therefore look +something like: + +.. code-block:: c + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + + /* format the request */ + send(fd, &request, sizeof(request)); + n = recv(fd, &response, RSP_BUFFER_SIZE); + /* interpret the response */ + +Netlink also provides natural support for "dumping", i.e. communicating +to user space all objects of a certain type (e.g. dumping all network +interfaces). + +.. code-block:: c + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + + /* format the dump request */ + send(fd, &request, sizeof(request)); + while (1) { + n = recv(fd, &buffer, RSP_BUFFER_SIZE); + /* one recv() call can read multiple messages, hence the loop below */ + for (nl_msg in buffer) { + if (nl_msg.nlmsg_type == NLMSG_DONE) + goto dump_finished; + /* process the object */ + } + } + dump_finished: + +The first two arguments of the socket() call require little explanation - +it is opening a Netlink socket, with all headers provided by the user +(hence NETLINK, RAW). The last argument is the protocol within Netlink. +This field used to identify the subsystem with which the socket will +communicate. + +Classic vs Generic Netlink +-------------------------- + +Initial implementation of Netlink depended on a static allocation +of IDs to subsystems and provided little supporting infrastructure. +Let us refer to those protocols collectively as **Classic Netlink**. +The list of them is defined on top of the ``include/uapi/linux/netlink.h`` +file, they include among others - general networking (NETLINK_ROUTE), +iSCSI (NETLINK_ISCSI), and audit (NETLINK_AUDIT). + +**Generic Netlink** (introduced in 2005) allows for dynamic registration of +subsystems (and subsystem ID allocation), introspection and simplifies +implementing the kernel side of the interface. + +The following section describes how to use Generic Netlink, as the +number of subsystems using Generic Netlink outnumbers the older +protocols by an order of magnitude. There are also no plans for adding +more Classic Netlink protocols to the kernel. +Basic information on how communicating with core networking parts of +the Linux kernel (or another of the 20 subsystems using Classic +Netlink) differs from Generic Netlink is provided later in this document. + +Generic Netlink +=============== + +In addition to the Netlink fixed metadata header each Netlink protocol +defines its own fixed metadata header. (Similarly to how network +headers stack - Ethernet > IP > TCP we have Netlink > Generic N. > Family.) + +A Netlink message always starts with struct nlmsghdr, which is followed +by a protocol-specific header. In case of Generic Netlink the protocol +header is struct genlmsghdr. + +The practical meaning of the fields in case of Generic Netlink is as follows: + +.. code-block:: c + + struct nlmsghdr { + __u32 nlmsg_len; /* Length of message including headers */ + __u16 nlmsg_type; /* Generic Netlink Family (subsystem) ID */ + __u16 nlmsg_flags; /* Flags - request or dump */ + __u32 nlmsg_seq; /* Sequence number */ + __u32 nlmsg_pid; /* Port ID, set to 0 */ + }; + struct genlmsghdr { + __u8 cmd; /* Command, as defined by the Family */ + __u8 version; /* Irrelevant, set to 1 */ + __u16 reserved; /* Reserved, set to 0 */ + }; + /* TLV attributes follow... */ + +In Classic Netlink :c:member:`nlmsghdr.nlmsg_type` used to identify +which operation within the subsystem the message was referring to +(e.g. get information about a netdev). Generic Netlink needs to mux +multiple subsystems in a single protocol so it uses this field to +identify the subsystem, and :c:member:`genlmsghdr.cmd` identifies +the operation instead. (See :ref:`res_fam` for +information on how to find the Family ID of the subsystem of interest.) +Note that the first 16 values (0 - 15) of this field are reserved for +control messages both in Classic Netlink and Generic Netlink. +See :ref:`nl_msg_type` for more details. + +There are 3 usual types of message exchanges on a Netlink socket: + + - performing a single action (``do``); + - dumping information (``dump``); + - getting asynchronous notifications (``multicast``). + +Classic Netlink is very flexible and presumably allows other types +of exchanges to happen, but in practice those are the three that get +used. + +Asynchronous notifications are sent by the kernel and received by +the user sockets which subscribed to them. ``do`` and ``dump`` requests +are initiated by the user. :c:member:`nlmsghdr.nlmsg_flags` should +be set as follows: + + - for ``do``: ``NLM_F_REQUEST | NLM_F_ACK`` + - for ``dump``: ``NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP`` + +:c:member:`nlmsghdr.nlmsg_seq` should be a set to a monotonically +increasing value. The value gets echoed back in responses and doesn't +matter in practice, but setting it to an increasing value for each +message sent is considered good hygiene. The purpose of the field is +matching responses to requests. Asynchronous notifications will have +:c:member:`nlmsghdr.nlmsg_seq` of ``0``. + +:c:member:`nlmsghdr.nlmsg_pid` is the Netlink equivalent of an address. +This field can be set to ``0`` when talking to the kernel. +See :ref:`nlmsg_pid` for the (uncommon) uses of the field. + +The expected use for :c:member:`genlmsghdr.version` was to allow +versioning of the APIs provided by the subsystems. No subsystem to +date made significant use of this field, so setting it to ``1`` seems +like a safe bet. + +.. _nl_msg_type: + +Netlink message types +--------------------- + +As previously mentioned :c:member:`nlmsghdr.nlmsg_type` carries +protocol specific values but the first 16 identifiers are reserved +(first subsystem specific message type should be equal to +``NLMSG_MIN_TYPE`` which is ``0x10``). + +There are only 4 Netlink control messages defined: + + - ``NLMSG_NOOP`` - ignore the message, not used in practice; + - ``NLMSG_ERROR`` - carries the return code of an operation; + - ``NLMSG_DONE`` - marks the end of a dump; + - ``NLMSG_OVERRUN`` - socket buffer has overflown, not used to date. + +``NLMSG_ERROR`` and ``NLMSG_DONE`` are of practical importance. +They carry return codes for operations. Note that unless +the ``NLM_F_ACK`` flag is set on the request Netlink will not respond +with ``NLMSG_ERROR`` if there is no error. To avoid having to special-case +this quirk it is recommended to always set ``NLM_F_ACK``. + +The format of ``NLMSG_ERROR`` is described by struct nlmsgerr:: + + ---------------------------------------------- + | struct nlmsghdr - response header | + ---------------------------------------------- + | int error | + ---------------------------------------------- + | struct nlmsghdr - original request header | + ---------------------------------------------- + | ** optionally (1) payload of the request | + ---------------------------------------------- + | ** optionally (2) extended ACK | + ---------------------------------------------- + +There are two instances of struct nlmsghdr here, first of the response +and second of the request. ``NLMSG_ERROR`` carries the information about +the request which led to the error. This could be useful when trying +to match requests to responses or re-parse the request to dump it into +logs. + +The payload of the request is not echoed in messages reporting success +(``error == 0``) or if ``NETLINK_CAP_ACK`` setsockopt() was set. +The latter is common +and perhaps recommended as having to read a copy of every request back +from the kernel is rather wasteful. The absence of request payload +is indicated by ``NLM_F_CAPPED`` in :c:member:`nlmsghdr.nlmsg_flags`. + +The second optional element of ``NLMSG_ERROR`` are the extended ACK +attributes. See :ref:`ext_ack` for more details. The presence +of extended ACK is indicated by ``NLM_F_ACK_TLVS`` in +:c:member:`nlmsghdr.nlmsg_flags`. + +``NLMSG_DONE`` is simpler, the request is never echoed but the extended +ACK attributes may be present:: + + ---------------------------------------------- + | struct nlmsghdr - response header | + ---------------------------------------------- + | int error | + ---------------------------------------------- + | ** optionally extended ACK | + ---------------------------------------------- + +.. _res_fam: + +Resolving the Family ID +----------------------- + +This section explains how to find the Family ID of a subsystem. +It also serves as an example of Generic Netlink communication. + +Generic Netlink is itself a subsystem exposed via the Generic Netlink API. +To avoid a circular dependency Generic Netlink has a statically allocated +Family ID (``GENL_ID_CTRL`` which is equal to ``NLMSG_MIN_TYPE``). +The Generic Netlink family implements a command used to find out information +about other families (``CTRL_CMD_GETFAMILY``). + +To get information about the Generic Netlink family named for example +``"test1"`` we need to send a message on the previously opened Generic Netlink +socket. The message should target the Generic Netlink Family (1), be a +``do`` (2) call to ``CTRL_CMD_GETFAMILY`` (3). A ``dump`` version of this +call would make the kernel respond with information about *all* the families +it knows about. Last but not least the name of the family in question has +to be specified (4) as an attribute with the appropriate type:: + + struct nlmsghdr: + __u32 nlmsg_len: 32 + __u16 nlmsg_type: GENL_ID_CTRL // (1) + __u16 nlmsg_flags: NLM_F_REQUEST | NLM_F_ACK // (2) + __u32 nlmsg_seq: 1 + __u32 nlmsg_pid: 0 + + struct genlmsghdr: + __u8 cmd: CTRL_CMD_GETFAMILY // (3) + __u8 version: 2 /* or 1, doesn't matter */ + __u16 reserved: 0 + + struct nlattr: // (4) + __u16 nla_len: 10 + __u16 nla_type: CTRL_ATTR_FAMILY_NAME + char data: test1\0 + + (padding:) + char data: \0\0 + +The length fields in Netlink (:c:member:`nlmsghdr.nlmsg_len` +and :c:member:`nlattr.nla_len`) always *include* the header. +Attribute headers in netlink must be aligned to 4 bytes from the start +of the message, hence the extra ``\0\0`` after ``CTRL_ATTR_FAMILY_NAME``. +The attribute lengths *exclude* the padding. + +If the family is found kernel will reply with two messages, the response +with all the information about the family:: + + /* Message #1 - reply */ + struct nlmsghdr: + __u32 nlmsg_len: 136 + __u16 nlmsg_type: GENL_ID_CTRL + __u16 nlmsg_flags: 0 + __u32 nlmsg_seq: 1 /* echoed from our request */ + __u32 nlmsg_pid: 5831 /* The PID of our user space process */ + + struct genlmsghdr: + __u8 cmd: CTRL_CMD_GETFAMILY + __u8 version: 2 + __u16 reserved: 0 + + struct nlattr: + __u16 nla_len: 10 + __u16 nla_type: CTRL_ATTR_FAMILY_NAME + char data: test1\0 + + (padding:) + data: \0\0 + + struct nlattr: + __u16 nla_len: 6 + __u16 nla_type: CTRL_ATTR_FAMILY_ID + __u16: 123 /* The Family ID we are after */ + + (padding:) + char data: \0\0 + + struct nlattr: + __u16 nla_len: 9 + __u16 nla_type: CTRL_ATTR_FAMILY_VERSION + __u16: 1 + + /* ... etc, more attributes will follow. */ + +And the error code (success) since ``NLM_F_ACK`` had been set on the request:: + + /* Message #2 - the ACK */ + struct nlmsghdr: + __u32 nlmsg_len: 36 + __u16 nlmsg_type: NLMSG_ERROR + __u16 nlmsg_flags: NLM_F_CAPPED /* There won't be a payload */ + __u32 nlmsg_seq: 1 /* echoed from our request */ + __u32 nlmsg_pid: 5831 /* The PID of our user space process */ + + int error: 0 + + struct nlmsghdr: /* Copy of the request header as we sent it */ + __u32 nlmsg_len: 32 + __u16 nlmsg_type: GENL_ID_CTRL + __u16 nlmsg_flags: NLM_F_REQUEST | NLM_F_ACK + __u32 nlmsg_seq: 1 + __u32 nlmsg_pid: 0 + +The order of attributes (struct nlattr) is not guaranteed so the user +has to walk the attributes and parse them. + +Note that Generic Netlink sockets are not associated or bound to a single +family. A socket can be used to exchange messages with many different +families, selecting the recipient family on message-by-message basis using +the :c:member:`nlmsghdr.nlmsg_type` field. + +.. _ext_ack: + +Extended ACK +------------ + +Extended ACK controls reporting of additional error/warning TLVs +in ``NLMSG_ERROR`` and ``NLMSG_DONE`` messages. To maintain backward +compatibility this feature has to be explicitly enabled by setting +the ``NETLINK_EXT_ACK`` setsockopt() to ``1``. + +Types of extended ack attributes are defined in enum nlmsgerr_attrs. +The most commonly used attributes are ``NLMSGERR_ATTR_MSG``, +``NLMSGERR_ATTR_OFFS`` and ``NLMSGERR_ATTR_MISS_*``. + +``NLMSGERR_ATTR_MSG`` carries a message in English describing +the encountered problem. These messages are far more detailed +than what can be expressed thru standard UNIX error codes. + +``NLMSGERR_ATTR_OFFS`` points to the attribute which caused the problem. + +``NLMSGERR_ATTR_MISS_TYPE`` and ``NLMSGERR_ATTR_MISS_NEST`` +inform about a missing attribute. + +Extended ACKs can be reported on errors as well as in case of success. +The latter should be treated as a warning. + +Extended ACKs greatly improve the usability of Netlink and should +always be enabled, appropriately parsed and reported to the user. + +Advanced topics +=============== + +Dump consistency +---------------- + +Some of the data structures kernel uses for storing objects make +it hard to provide an atomic snapshot of all the objects in a dump +(without impacting the fast-paths updating them). + +Kernel may set the ``NLM_F_DUMP_INTR`` flag on any message in a dump +(including the ``NLMSG_DONE`` message) if the dump was interrupted and +may be inconsistent (e.g. missing objects). User space should retry +the dump if it sees the flag set. + +Introspection +------------- + +The basic introspection abilities are enabled by access to the Family +object as reported in :ref:`res_fam`. User can query information about +the Generic Netlink family, including which operations are supported +by the kernel and what attributes the kernel understands. +Family information includes the highest ID of an attribute kernel can parse, +a separate command (``CTRL_CMD_GETPOLICY``) provides detailed information +about supported attributes, including ranges of values the kernel accepts. + +Querying family information is useful in cases when user space needs +to make sure that the kernel has support for a feature before issuing +a request. + +.. _nlmsg_pid: + +nlmsg_pid +--------- + +:c:member:`nlmsghdr.nlmsg_pid` is the Netlink equivalent of an address. +It is referred to as Port ID, sometimes Process ID because for historical +reasons if the application does not select (bind() to) an explicit Port ID +kernel will automatically assign it the ID equal to its Process ID +(as reported by the getpid() system call). + +Similarly to the bind() semantics of the TCP/IP network protocols the value +of zero means "assign automatically", hence it is common for applications +to leave the :c:member:`nlmsghdr.nlmsg_pid` field initialized to ``0``. + +The field is still used today in rare cases when kernel needs to send +a unicast notification. User space application can use bind() to associate +its socket with a specific PID, it then communicates its PID to the kernel. +This way the kernel can reach the specific user space process. + +This sort of communication is utilized in UMH (User Mode Helper)-like +scenarios when kernel needs to trigger user space processing or ask user +space for a policy decision. + +Multicast notifications +----------------------- + +One of the strengths of Netlink is the ability to send event notifications +to user space. This is a unidirectional form of communication (kernel -> +user) and does not involve any control messages like ``NLMSG_ERROR`` or +``NLMSG_DONE``. + +For example the Generic Netlink family itself defines a set of multicast +notifications about registered families. When a new family is added the +sockets subscribed to the notifications will get the following message:: + + struct nlmsghdr: + __u32 nlmsg_len: 136 + __u16 nlmsg_type: GENL_ID_CTRL + __u16 nlmsg_flags: 0 + __u32 nlmsg_seq: 0 + __u32 nlmsg_pid: 0 + + struct genlmsghdr: + __u8 cmd: CTRL_CMD_NEWFAMILY + __u8 version: 2 + __u16 reserved: 0 + + struct nlattr: + __u16 nla_len: 10 + __u16 nla_type: CTRL_ATTR_FAMILY_NAME + char data: test1\0 + + (padding:) + data: \0\0 + + struct nlattr: + __u16 nla_len: 6 + __u16 nla_type: CTRL_ATTR_FAMILY_ID + __u16: 123 /* The Family ID we are after */ + + (padding:) + char data: \0\0 + + struct nlattr: + __u16 nla_len: 9 + __u16 nla_type: CTRL_ATTR_FAMILY_VERSION + __u16: 1 + + /* ... etc, more attributes will follow. */ + +The notification contains the same information as the response +to the ``CTRL_CMD_GETFAMILY`` request. + +The Netlink headers of the notification are mostly 0 and irrelevant. +The :c:member:`nlmsghdr.nlmsg_seq` may be either zero or a monotonically +increasing notification sequence number maintained by the family. + +To receive notifications the user socket must subscribe to the relevant +notification group. Much like the Family ID, the Group ID for a given +multicast group is dynamic and can be found inside the Family information. +The ``CTRL_ATTR_MCAST_GROUPS`` attribute contains nests with names +(``CTRL_ATTR_MCAST_GRP_NAME``) and IDs (``CTRL_ATTR_MCAST_GRP_ID``) of +the groups family. + +Once the Group ID is known a setsockopt() call adds the socket to the group: + +.. code-block:: c + + unsigned int group_id; + + /* .. find the group ID... */ + + setsockopt(fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP, + &group_id, sizeof(group_id)); + +The socket will now receive notifications. + +It is recommended to use separate sockets for receiving notifications +and sending requests to the kernel. The asynchronous nature of notifications +means that they may get mixed in with the responses making the message +handling much harder. + +Buffer sizing +------------- + +Netlink sockets are datagram sockets rather than stream sockets, +meaning that each message must be received in its entirety by a single +recv()/recvmsg() system call. If the buffer provided by the user is too +short, the message will be truncated and the ``MSG_TRUNC`` flag set +in struct msghdr (struct msghdr is the second argument +of the recvmsg() system call, *not* a Netlink header). + +Upon truncation the remaining part of the message is discarded. + +Netlink expects that the user buffer will be at least 8kB or a page +size of the CPU architecture, whichever is bigger. Particular Netlink +families may, however, require a larger buffer. 32kB buffer is recommended +for most efficient handling of dumps (larger buffer fits more dumped +objects and therefore fewer recvmsg() calls are needed). + +Classic Netlink +=============== + +The main differences between Classic and Generic Netlink are the dynamic +allocation of subsystem identifiers and availability of introspection. +In theory the protocol does not differ significantly, however, in practice +Classic Netlink experimented with concepts which were abandoned in Generic +Netlink (really, they usually only found use in a small corner of a single +subsystem). This section is meant as an explainer of a few of such concepts, +with the explicit goal of giving the Generic Netlink +users the confidence to ignore them when reading the uAPI headers. + +Most of the concepts and examples here refer to the ``NETLINK_ROUTE`` family, +which covers much of the configuration of the Linux networking stack. +Real documentation of that family, deserves a chapter (or a book) of its own. + +Families +-------- + +Netlink refers to subsystems as families. This is a remnant of using +sockets and the concept of protocol families, which are part of message +demultiplexing in ``NETLINK_ROUTE``. + +Sadly every layer of encapsulation likes to refer to whatever it's carrying +as "families" making the term very confusing: + + 1. AF_NETLINK is a bona fide socket protocol family + 2. AF_NETLINK's documentation refers to what comes after its own + header (struct nlmsghdr) in a message as a "Family Header" + 3. Generic Netlink is a family for AF_NETLINK (struct genlmsghdr follows + struct nlmsghdr), yet it also calls its users "Families". + +Note that the Generic Netlink Family IDs are in a different "ID space" +and overlap with Classic Netlink protocol numbers (e.g. ``NETLINK_CRYPTO`` +has the Classic Netlink protocol ID of 21 which Generic Netlink will +happily allocate to one of its families as well). + +Strict checking +--------------- + +The ``NETLINK_GET_STRICT_CHK`` socket option enables strict input checking +in ``NETLINK_ROUTE``. It was needed because historically kernel did not +validate the fields of structures it didn't process. This made it impossible +to start using those fields later without risking regressions in applications +which initialized them incorrectly or not at all. + +``NETLINK_GET_STRICT_CHK`` declares that the application is initializing +all fields correctly. It also opts into validating that message does not +contain trailing data and requests that kernel rejects attributes with +type higher than largest attribute type known to the kernel. + +``NETLINK_GET_STRICT_CHK`` is not used outside of ``NETLINK_ROUTE``. + +Unknown attributes +------------------ + +Historically Netlink ignored all unknown attributes. The thinking was that +it would free the application from having to probe what kernel supports. +The application could make a request to change the state and check which +parts of the request "stuck". + +This is no longer the case for new Generic Netlink families and those opting +in to strict checking. See enum netlink_validation for validation types +performed. + +Fixed metadata and structures +----------------------------- + +Classic Netlink made liberal use of fixed-format structures within +the messages. Messages would commonly have a structure with +a considerable number of fields after struct nlmsghdr. It was also +common to put structures with multiple members inside attributes, +without breaking each member into an attribute of its own. + +This has caused problems with validation and extensibility and +therefore using binary structures is actively discouraged for new +attributes. + +Request types +------------- + +``NETLINK_ROUTE`` categorized requests into 4 types ``NEW``, ``DEL``, ``GET``, +and ``SET``. Each object can handle all or some of those requests +(objects being netdevs, routes, addresses, qdiscs etc.) Request type +is defined by the 2 lowest bits of the message type, so commands for +new objects would always be allocated with a stride of 4. + +Each object would also have it's own fixed metadata shared by all request +types (e.g. struct ifinfomsg for netdev requests, struct ifaddrmsg for address +requests, struct tcmsg for qdisc requests). + +Even though other protocols and Generic Netlink commands often use +the same verbs in their message names (``GET``, ``SET``) the concept +of request types did not find wider adoption. + +Notification echo +----------------- + +``NLM_F_ECHO`` requests for notifications resulting from the request +to be queued onto the requesting socket. This is useful to discover +the impact of the request. + +Note that this feature is not universally implemented. + +Other request-type-specific flags +--------------------------------- + +Classic Netlink defined various flags for its ``GET``, ``NEW`` +and ``DEL`` requests in the upper byte of nlmsg_flags in struct nlmsghdr. +Since request types have not been generalized the request type specific +flags are rarely used (and considered deprecated for new families). + +For ``GET`` - ``NLM_F_ROOT`` and ``NLM_F_MATCH`` are combined into +``NLM_F_DUMP``, and not used separately. ``NLM_F_ATOMIC`` is never used. + +For ``DEL`` - ``NLM_F_NONREC`` is only used by nftables and ``NLM_F_BULK`` +only by FDB some operations. + +The flags for ``NEW`` are used most commonly in classic Netlink. Unfortunately, +the meaning is not crystal clear. The following description is based on the +best guess of the intention of the authors, and in practice all families +stray from it in one way or another. ``NLM_F_REPLACE`` asks to replace +an existing object, if no matching object exists the operation should fail. +``NLM_F_EXCL`` has the opposite semantics and only succeeds if object already +existed. +``NLM_F_CREATE`` asks for the object to be created if it does not +exist, it can be combined with ``NLM_F_REPLACE`` and ``NLM_F_EXCL``. + +A comment in the main Netlink uAPI header states:: + + 4.4BSD ADD NLM_F_CREATE|NLM_F_EXCL + 4.4BSD CHANGE NLM_F_REPLACE + + True CHANGE NLM_F_CREATE|NLM_F_REPLACE + Append NLM_F_CREATE + Check NLM_F_EXCL + +which seems to indicate that those flags predate request types. +``NLM_F_REPLACE`` without ``NLM_F_CREATE`` was initially used instead +of ``SET`` commands. +``NLM_F_EXCL`` without ``NLM_F_CREATE`` was used to check if object exists +without creating it, presumably predating ``GET`` commands. + +``NLM_F_APPEND`` indicates that if one key can have multiple objects associated +with it (e.g. multiple next-hop objects for a route) the new object should be +added to the list rather than replacing the entire list. + +uAPI reference +============== + +.. kernel-doc:: include/uapi/linux/netlink.h diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index abd7c32126ce06d9884b2986bc986a02778b3fad..eee9f857a986f52c802882f6378eccdd94c4fcb6 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -4074,7 +4074,7 @@ Queues an SMI on the thread's vcpu. 4.97 KVM_X86_SET_MSR_FILTER ---------------------------- -:Capability: KVM_X86_SET_MSR_FILTER +:Capability: KVM_CAP_X86_MSR_FILTER :Architectures: x86 :Type: vm ioctl :Parameters: struct kvm_msr_filter @@ -4173,8 +4173,10 @@ If an MSR access is not permitted through the filtering, it generates a allows user space to deflect and potentially handle various MSR accesses into user space. -If a vCPU is in running state while this ioctl is invoked, the vCPU may -experience inconsistent filtering behavior on MSR accesses. +Note, invoking this ioctl while a vCPU is running is inherently racy. However, +KVM does guarantee that vCPUs will see either the previous filter or the new +filter, e.g. MSRs with identical settings in both the old and new filter will +have deterministic behavior. 4.98 KVM_CREATE_SPAPR_TCE_64 ---------------------------- @@ -5287,110 +5289,7 @@ KVM_PV_DUMP authentication tag all of which are needed to decrypt the dump at a later time. - -4.126 KVM_X86_SET_MSR_FILTER ----------------------------- - -:Capability: KVM_CAP_X86_MSR_FILTER -:Architectures: x86 -:Type: vm ioctl -:Parameters: struct kvm_msr_filter -:Returns: 0 on success, < 0 on error - -:: - - struct kvm_msr_filter_range { - #define KVM_MSR_FILTER_READ (1 << 0) - #define KVM_MSR_FILTER_WRITE (1 << 1) - __u32 flags; - __u32 nmsrs; /* number of msrs in bitmap */ - __u32 base; /* MSR index the bitmap starts at */ - __u8 *bitmap; /* a 1 bit allows the operations in flags, 0 denies */ - }; - - #define KVM_MSR_FILTER_MAX_RANGES 16 - struct kvm_msr_filter { - #define KVM_MSR_FILTER_DEFAULT_ALLOW (0 << 0) - #define KVM_MSR_FILTER_DEFAULT_DENY (1 << 0) - __u32 flags; - struct kvm_msr_filter_range ranges[KVM_MSR_FILTER_MAX_RANGES]; - }; - -flags values for ``struct kvm_msr_filter_range``: - -``KVM_MSR_FILTER_READ`` - - Filter read accesses to MSRs using the given bitmap. A 0 in the bitmap - indicates that a read should immediately fail, while a 1 indicates that - a read for a particular MSR should be handled regardless of the default - filter action. - -``KVM_MSR_FILTER_WRITE`` - - Filter write accesses to MSRs using the given bitmap. A 0 in the bitmap - indicates that a write should immediately fail, while a 1 indicates that - a write for a particular MSR should be handled regardless of the default - filter action. - -``KVM_MSR_FILTER_READ | KVM_MSR_FILTER_WRITE`` - - Filter both read and write accesses to MSRs using the given bitmap. A 0 - in the bitmap indicates that both reads and writes should immediately fail, - while a 1 indicates that reads and writes for a particular MSR are not - filtered by this range. - -flags values for ``struct kvm_msr_filter``: - -``KVM_MSR_FILTER_DEFAULT_ALLOW`` - - If no filter range matches an MSR index that is getting accessed, KVM will - fall back to allowing access to the MSR. - -``KVM_MSR_FILTER_DEFAULT_DENY`` - - If no filter range matches an MSR index that is getting accessed, KVM will - fall back to rejecting access to the MSR. In this mode, all MSRs that should - be processed by KVM need to explicitly be marked as allowed in the bitmaps. - -This ioctl allows user space to define up to 16 bitmaps of MSR ranges to -specify whether a certain MSR access should be explicitly filtered for or not. - -If this ioctl has never been invoked, MSR accesses are not guarded and the -default KVM in-kernel emulation behavior is fully preserved. - -Calling this ioctl with an empty set of ranges (all nmsrs == 0) disables MSR -filtering. In that mode, ``KVM_MSR_FILTER_DEFAULT_DENY`` is invalid and causes -an error. - -As soon as the filtering is in place, every MSR access is processed through -the filtering except for accesses to the x2APIC MSRs (from 0x800 to 0x8ff); -x2APIC MSRs are always allowed, independent of the ``default_allow`` setting, -and their behavior depends on the ``X2APIC_ENABLE`` bit of the APIC base -register. - -If a bit is within one of the defined ranges, read and write accesses are -guarded by the bitmap's value for the MSR index if the kind of access -is included in the ``struct kvm_msr_filter_range`` flags. If no range -cover this particular access, the behavior is determined by the flags -field in the kvm_msr_filter struct: ``KVM_MSR_FILTER_DEFAULT_ALLOW`` -and ``KVM_MSR_FILTER_DEFAULT_DENY``. - -Each bitmap range specifies a range of MSRs to potentially allow access on. -The range goes from MSR index [base .. base+nmsrs]. The flags field -indicates whether reads, writes or both reads and writes are filtered -by setting a 1 bit in the bitmap for the corresponding MSR index. - -If an MSR access is not permitted through the filtering, it generates a -#GP inside the guest. When combined with KVM_CAP_X86_USER_SPACE_MSR, that -allows user space to deflect and potentially handle various MSR accesses -into user space. - -Note, invoking this ioctl with a vCPU is running is inherently racy. However, -KVM does guarantee that vCPUs will see either the previous filter or the new -filter, e.g. MSRs with identical settings in both the old and new filter will -have deterministic behavior. - -4.127 KVM_XEN_HVM_SET_ATTR +4.126 KVM_XEN_HVM_SET_ATTR -------------------------- :Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_SHARED_INFO @@ -8019,8 +7918,8 @@ guest according to the bits in the KVM_CPUID_FEATURES CPUID leaf (0x40000001). Otherwise, a guest may use the paravirtual features regardless of what has actually been exposed through the CPUID leaf. -8.29 KVM_CAP_DIRTY_LOG_RING ---------------------------- +8.29 KVM_CAP_DIRTY_LOG_RING/KVM_CAP_DIRTY_LOG_RING_ACQ_REL +---------------------------------------------------------- :Architectures: x86 :Parameters: args[0] - size of the dirty log ring @@ -8078,6 +7977,11 @@ on to the next GFN. The userspace should continue to do this until the flags of a GFN have the DIRTY bit cleared, meaning that it has harvested all the dirty GFNs that were available. +Note that on weakly ordered architectures, userspace accesses to the +ring buffer (and more specifically the 'flags' field) must be ordered, +using load-acquire/store-release accessors when available, or any +other memory barrier that will ensure this ordering. + It's not necessary for userspace to harvest the all dirty GFNs at once. However it must collect the dirty GFNs in sequence, i.e., the userspace program cannot skip one dirty GFN to collect the one next to it. @@ -8106,6 +8010,14 @@ KVM_CAP_DIRTY_LOG_RING with an acceptable dirty ring size, the virtual machine will switch to ring-buffer dirty page tracking and further KVM_GET_DIRTY_LOG or KVM_CLEAR_DIRTY_LOG ioctls will fail. +NOTE: KVM_CAP_DIRTY_LOG_RING_ACQ_REL is the only capability that +should be exposed by weakly ordered architecture, in order to indicate +the additional memory ordering requirements imposed on userspace when +reading the state of an entry and mutating it from DIRTY to HARVESTED. +Architecture with TSO-like ordering (such as x86) are allowed to +expose both KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL +to userspace. + 8.30 KVM_CAP_XEN_HVM -------------------- diff --git a/Documentation/virt/kvm/vcpu-requests.rst b/Documentation/virt/kvm/vcpu-requests.rst index 31f62b64e07b9f75c2be45ad28c133e53bf9ad04..87f04c1fa53d1c3c1efef0dff1e2fb88d2f3a258 100644 --- a/Documentation/virt/kvm/vcpu-requests.rst +++ b/Documentation/virt/kvm/vcpu-requests.rst @@ -97,7 +97,7 @@ VCPU requests are simply bit indices of the ``vcpu->requests`` bitmap. This means general bitops, like those documented in [atomic-ops]_ could also be used, e.g. :: - clear_bit(KVM_REQ_UNHALT & KVM_REQUEST_MASK, &vcpu->requests); + clear_bit(KVM_REQ_UNBLOCK & KVM_REQUEST_MASK, &vcpu->requests); However, VCPU request users should refrain from doing so, as it would break the abstraction. The first 8 bits are reserved for architecture @@ -126,17 +126,6 @@ KVM_REQ_UNBLOCK or in order to update the interrupt routing and ensure that assigned devices will wake up the vCPU. -KVM_REQ_UNHALT - - This request may be made from the KVM common function kvm_vcpu_block(), - which is used to emulate an instruction that causes a CPU to halt until - one of an architectural specific set of events and/or interrupts is - received (determined by checking kvm_arch_vcpu_runnable()). When that - event or interrupt arrives kvm_vcpu_block() makes the request. This is - in contrast to when kvm_vcpu_block() returns due to any other reason, - such as a pending signal, which does not indicate the VCPU's halt - emulation should stop, and therefore does not make the request. - KVM_REQ_OUTSIDE_GUEST_MODE This "request" ensures the target vCPU has exited guest mode prior to the @@ -297,21 +286,6 @@ architecture dependent. kvm_vcpu_block() calls kvm_arch_vcpu_runnable() to check if it should awaken. One reason to do so is to provide architectures a function where requests may be checked if necessary. -Clearing Requests ------------------ - -Generally it only makes sense for the receiving VCPU thread to clear a -request. However, in some circumstances, such as when the requesting -thread and the receiving VCPU thread are executed serially, such as when -they are the same thread, or when they are using some form of concurrency -control to temporarily execute synchronously, then it's possible to know -that the request may be cleared immediately, rather than waiting for the -receiving VCPU thread to handle the request in VCPU RUN. The only current -examples of this are kvm_vcpu_block() calls made by VCPUs to block -themselves. A possible side-effect of that call is to make the -KVM_REQ_UNHALT request, which may then be cleared immediately when the -VCPU returns from the call. - References ========== diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index 2d307811978c45ea57067163d364ee86680c5971..935aaeb97fe660cfd64476448488360b500b41f6 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -89,9 +89,8 @@ context. In a typical workflow, this command should be the first command issued. The firmware can be initialized either by using its own non-volatile storage or the OS can manage the NV storage for the firmware using the module parameter -``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create -a new NV storage file allocate the file with 32KB bytes of 0xFF as required by -the SEV spec. +``init_ex_path``. If the file specified by ``init_ex_path`` does not exist or +is invalid, the OS will create or override the file with output from PSP. Returns: 0 on success, -negative on error diff --git a/Documentation/virt/kvm/x86/mmu.rst b/Documentation/virt/kvm/x86/mmu.rst index 8739120f43006012969d02ad5572a560bc38aae2..8364afa228ecb5df661aa795152c74d0d29527d7 100644 --- a/Documentation/virt/kvm/x86/mmu.rst +++ b/Documentation/virt/kvm/x86/mmu.rst @@ -377,7 +377,7 @@ Emulating cr0.wp ================ If tdp is not enabled, the host must keep cr0.wp=1 so page write protection -works for the guest kernel, not guest guest userspace. When the guest +works for the guest kernel, not guest userspace. When the guest cr0.wp=1, this does not present a problem. However when the guest cr0.wp=0, we cannot map the permissions for gpte.u=1, gpte.w=0 to any spte (the semantics require allowing any guest kernel access plus user read access). diff --git a/Documentation/w1/masters/ds2490.rst b/Documentation/w1/masters/ds2490.rst index 7e5b50f9c0f56a80c1cad51b38404ea7b966d7d9..842e7ae80424f97f7a7ce688c7884ff9afc663c5 100644 --- a/Documentation/w1/masters/ds2490.rst +++ b/Documentation/w1/masters/ds2490.rst @@ -52,7 +52,7 @@ Notes and limitations. clear the entire bulk in buffer. It would be possible to read the maximum buffer size to not run into this error condition, only extra bytes in the buffer is a logic error in the driver. The code should - should match reads and writes as well as data sizes. Reads and + match reads and writes as well as data sizes. Reads and writes are serialized and the status verifies that the chip is idle (and data is available) before the read is executed, so it should not happen. diff --git a/Documentation/w1/w1-generic.rst b/Documentation/w1/w1-generic.rst index da4e8b4e9b01185f90a4d411ae8817ba4a437467..99255b6d0e5395616bfd8c91b26f27e6004f8adb 100644 --- a/Documentation/w1/w1-generic.rst +++ b/Documentation/w1/w1-generic.rst @@ -113,7 +113,7 @@ generally only make sense when searching is disabled, as a search will redetect manually removed devices that are present and timeout manually added devices that aren't on the bus. -Bus searches occur at an interval, specified as a summ of timeout and +Bus searches occur at an interval, specified as a sum of timeout and timeout_us module parameters (either of which may be 0) for as long as w1_master_search remains greater than 0 or is -1. Each search attempt decrements w1_master_search by 1 (down to 0) and increments diff --git a/Documentation/x86/entry_64.rst b/Documentation/x86/entry_64.rst index e433e08f701803379507003ee9d9ef610c652280..0afdce3c06f4f97d25f02803d8ae644e435d21a6 100644 --- a/Documentation/x86/entry_64.rst +++ b/Documentation/x86/entry_64.rst @@ -33,8 +33,8 @@ Some of these entries are: - interrupt: An array of entries. Every IDT vector that doesn't explicitly point somewhere else gets set to the corresponding value in interrupts. These point to a whole array of - magically-generated functions that make their way to do_IRQ with - the interrupt number as a parameter. + magically-generated functions that make their way to common_interrupt() + with the interrupt number as a parameter. - APIC interrupts: Various special-purpose interrupts for things like TLB shootdown. diff --git a/Documentation/x86/microcode.rst b/Documentation/x86/microcode.rst index a320d37982ed6dcfb772b78df934a490d2299f4d..b627c6f36bcf5a966b9f7ff7e34059f44ffe8f38 100644 --- a/Documentation/x86/microcode.rst +++ b/Documentation/x86/microcode.rst @@ -6,6 +6,7 @@ The Linux Microcode Loader :Authors: - Fenghua Yu - Borislav Petkov + - Ashok Raj The kernel has a x86 microcode loading facility which is supposed to provide microcode loading methods in the OS. Potential use cases are @@ -92,15 +93,8 @@ vendor's site. Late loading ============ -There are two legacy user space interfaces to load microcode, either through -/dev/cpu/microcode or through /sys/devices/system/cpu/microcode/reload file -in sysfs. - -The /dev/cpu/microcode method is deprecated because it needs a special -userspace tool for that. - -The easier method is simply installing the microcode packages your distro -supplies and running:: +You simply install the microcode packages your distro supplies and +run:: # echo 1 > /sys/devices/system/cpu/microcode/reload @@ -110,6 +104,110 @@ The loading mechanism looks for microcode blobs in /lib/firmware/{intel-ucode,amd-ucode}. The default distro installation packages already put them there. +Since kernel 5.19, late loading is not enabled by default. + +The /dev/cpu/microcode method has been removed in 5.19. + +Why is late loading dangerous? +============================== + +Synchronizing all CPUs +---------------------- + +The microcode engine which receives the microcode update is shared +between the two logical threads in a SMT system. Therefore, when +the update is executed on one SMT thread of the core, the sibling +"automatically" gets the update. + +Since the microcode can "simulate" MSRs too, while the microcode update +is in progress, those simulated MSRs transiently cease to exist. This +can result in unpredictable results if the SMT sibling thread happens to +be in the middle of an access to such an MSR. The usual observation is +that such MSR accesses cause #GPs to be raised to signal that former are +not present. + +The disappearing MSRs are just one common issue which is being observed. +Any other instruction that's being patched and gets concurrently +executed by the other SMT sibling, can also result in similar, +unpredictable behavior. + +To eliminate this case, a stop_machine()-based CPU synchronization was +introduced as a way to guarantee that all logical CPUs will not execute +any code but just wait in a spin loop, polling an atomic variable. + +While this took care of device or external interrupts, IPIs including +LVT ones, such as CMCI etc, it cannot address other special interrupts +that can't be shut off. Those are Machine Check (#MC), System Management +(#SMI) and Non-Maskable interrupts (#NMI). + +Machine Checks +-------------- + +Machine Checks (#MC) are non-maskable. There are two kinds of MCEs. +Fatal un-recoverable MCEs and recoverable MCEs. While un-recoverable +errors are fatal, recoverable errors can also happen in kernel context +are also treated as fatal by the kernel. + +On certain Intel machines, MCEs are also broadcast to all threads in a +system. If one thread is in the middle of executing WRMSR, a MCE will be +taken at the end of the flow. Either way, they will wait for the thread +performing the wrmsr(0x79) to rendezvous in the MCE handler and shutdown +eventually if any of the threads in the system fail to check in to the +MCE rendezvous. + +To be paranoid and get predictable behavior, the OS can choose to set +MCG_STATUS.MCIP. Since MCEs can be at most one in a system, if an +MCE was signaled, the above condition will promote to a system reset +automatically. OS can turn off MCIP at the end of the update for that +core. + +System Management Interrupt +--------------------------- + +SMIs are also broadcast to all CPUs in the platform. Microcode update +requests exclusive access to the core before writing to MSR 0x79. So if +it does happen such that, one thread is in WRMSR flow, and the 2nd got +an SMI, that thread will be stopped in the first instruction in the SMI +handler. + +Since the secondary thread is stopped in the first instruction in SMI, +there is very little chance that it would be in the middle of executing +an instruction being patched. Plus OS has no way to stop SMIs from +happening. + +Non-Maskable Interrupts +----------------------- + +When thread0 of a core is doing the microcode update, if thread1 is +pulled into NMI, that can cause unpredictable behavior due to the +reasons above. + +OS can choose a variety of methods to avoid running into this situation. + + +Is the microcode suitable for late loading? +------------------------------------------- + +Late loading is done when the system is fully operational and running +real workloads. Late loading behavior depends on what the base patch on +the CPU is before upgrading to the new patch. + +This is true for Intel CPUs. + +Consider, for example, a CPU has patch level 1 and the update is to +patch level 3. + +Between patch1 and patch3, patch2 might have deprecated a software-visible +feature. + +This is unacceptable if software is even potentially using that feature. +For instance, say MSR_X is no longer available after an update, +accessing that MSR will cause a #GP fault. + +Basically there is no way to declare a new microcode update suitable +for late-loading. This is another one of the problems that caused late +loading to be not enabled by default. + Builtin microcode ================= diff --git a/Kbuild b/Kbuild index fa441b98c9f6eac1617acf1772ae8b371cfd42aa..464b34a08f51ef9d2ae12e6902c92690b5dfa03b 100644 --- a/Kbuild +++ b/Kbuild @@ -2,18 +2,18 @@ # # Kbuild for top-level directory of the kernel -##### +# Prepare global headers and check sanity before descending into sub-directories +# --------------------------------------------------------------------------- + # Generate bounds.h bounds-file := include/generated/bounds.h -always-y := $(bounds-file) targets := kernel/bounds.s $(bounds-file): kernel/bounds.s FORCE $(call filechk,offsets,__LINUX_BOUNDS_H__) -##### # Generate timeconst.h timeconst-file := include/generated/timeconst.h @@ -23,12 +23,10 @@ filechk_gentimeconst = echo $(CONFIG_HZ) | bc -q $< $(timeconst-file): kernel/time/timeconst.bc FORCE $(call filechk,gentimeconst) -##### # Generate asm-offsets.h offsets-file := include/generated/asm-offsets.h -always-y += $(offsets-file) targets += arch/$(SRCARCH)/kernel/asm-offsets.s arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file) @@ -36,24 +34,66 @@ arch/$(SRCARCH)/kernel/asm-offsets.s: $(timeconst-file) $(bounds-file) $(offsets-file): arch/$(SRCARCH)/kernel/asm-offsets.s FORCE $(call filechk,offsets,__ASM_OFFSETS_H__) -##### # Check for missing system calls -always-y += missing-syscalls - quiet_cmd_syscalls = CALL $< cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags) -missing-syscalls: scripts/checksyscalls.sh $(offsets-file) FORCE +PHONY += missing-syscalls +missing-syscalls: scripts/checksyscalls.sh $(offsets-file) $(call cmd,syscalls) -##### -# Check atomic headers are up-to-date - -always-y += old-atomics - -quiet_cmd_atomics = CALL $< - cmd_atomics = $(CONFIG_SHELL) $< - -old-atomics: scripts/atomic/check-atomics.sh FORCE - $(call cmd,atomics) +# Check the manual modification of atomic headers + +quiet_cmd_check_sha1 = CHKSHA1 $< + cmd_check_sha1 = \ + if ! command -v sha1sum >/dev/null; then \ + echo "warning: cannot check the header due to sha1sum missing"; \ + exit 0; \ + fi; \ + if [ "$$(sed -n '$$s:// ::p' $<)" != \ + "$$(sed '$$d' $< | sha1sum | sed 's/ .*//')" ]; then \ + echo "error: $< has been modified." >&2; \ + exit 1; \ + fi; \ + touch $@ + +atomic-checks += $(addprefix $(obj)/.checked-, \ + atomic-arch-fallback.h \ + atomic-instrumented.h \ + atomic-long.h) + +targets += $(atomic-checks) +$(atomic-checks): $(obj)/.checked-%: include/linux/atomic/% FORCE + $(call if_changed,check_sha1) + +# A phony target that depends on all the preparation targets + +PHONY += prepare +prepare: $(offsets-file) missing-syscalls $(atomic-checks) + @: + +# Ordinary directory descending +# --------------------------------------------------------------------------- + +obj-y += init/ +obj-y += usr/ +obj-y += arch/$(SRCARCH)/ +obj-y += $(ARCH_CORE) +obj-y += kernel/ +obj-y += certs/ +obj-y += mm/ +obj-y += fs/ +obj-y += ipc/ +obj-y += security/ +obj-y += crypto/ +obj-$(CONFIG_BLOCK) += block/ +obj-$(CONFIG_IO_URING) += io_uring/ +obj-$(CONFIG_RUST) += rust/ +obj-y += $(ARCH_LIB) +obj-y += drivers/ +obj-y += sound/ +obj-$(CONFIG_SAMPLES) += samples/ +obj-$(CONFIG_NET) += net/ +obj-y += virt/ +obj-y += $(ARCH_DRIVERS) diff --git a/MAINTAINERS b/MAINTAINERS index 78f67095543447fe081c0ca1aae15c1290688d83..b1f3d0893393bb1e201e889cf2f2250dd2436641 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -348,7 +348,6 @@ M: "Rafael J. Wysocki" R: Len Brown L: linux-acpi@vger.kernel.org S: Supported -W: https://01.org/linux-acpi Q: https://patchwork.kernel.org/project/linux-acpi/list/ B: https://bugzilla.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm @@ -427,7 +426,6 @@ M: Rafael J. Wysocki R: Zhang Rui L: linux-acpi@vger.kernel.org S: Supported -W: https://01.org/linux-acpi B: https://bugzilla.kernel.org F: drivers/acpi/*thermal* @@ -556,7 +554,7 @@ M: Michael Hennerich S: Supported W: http://wiki.analog.com/ADP5588 W: https://ez.analog.com/linux-software-drivers -F: drivers/gpio/gpio-adp5588.c +F: Documentation/devicetree/bindings/input/adi,adp5588.yaml F: drivers/input/keyboard/adp5588-keys.c ADP8860 BACKLIGHT DRIVER (ADP8860/ADP8861/ADP8863) @@ -620,7 +618,7 @@ ADXL367 THREE-AXIS DIGITAL ACCELEROMETER DRIVER M: Cosmin Tanislav L: linux-iio@vger.kernel.org S: Supported -W: http://ez.analog.com/community/linux-device-drivers +W: https://ez.analog.com/linux-software-drivers F: Documentation/devicetree/bindings/iio/accel/adi,adxl367.yaml F: drivers/iio/accel/adxl367* @@ -671,7 +669,8 @@ F: fs/afs/ F: include/trace/events/afs.h AGPGART DRIVER -M: David Airlie +M: David Airlie +L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm F: drivers/char/agp/ @@ -749,19 +748,17 @@ S: Supported F: drivers/infiniband/hw/erdma F: include/uapi/rdma/erdma-abi.h +ALIBABA PMU DRIVER +M: Shuai Xue +S: Supported +F: Documentation/admin-guide/perf/alibaba_pmu.rst +F: drivers/perf/alibaba_uncore_drw_pmu.c + ALIENWARE WMI DRIVER L: Dell.Client.Kernel@dell.com S: Maintained F: drivers/platform/x86/dell/alienware-wmi.c -ALL SENSORS DLH SERIES PRESSURE SENSORS DRIVER -M: Tomislav Denis -L: linux-iio@vger.kernel.org -S: Maintained -W: http://www.allsensors.com/ -F: Documentation/devicetree/bindings/iio/pressure/asc,dlhl60d.yaml -F: drivers/iio/pressure/dlhl60d.c - ALLEGRO DVT VIDEO IP CORE DRIVER M: Michael Tretter R: Pengutronix Kernel Team @@ -820,6 +817,13 @@ L: linux-media@vger.kernel.org S: Maintained F: drivers/staging/media/sunxi/cedrus/ +ALLWINNER DMIC DRIVERS +M: Ban Tao +L: alsa-devel@alsa-project.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml +F: sound/soc/sunxi/sun50i-dmic.c + ALPHA PORT M: Richard Henderson M: Ivan Kokshaysky @@ -878,6 +882,13 @@ L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/altera/ +ALTERA TSE PCS +M: Maxime Chevallier +L: netdev@vger.kernel.org +S: Supported +F: drivers/net/pcs/pcs-altera-tse.c +F: include/linux/pcs-altera-tse.h + ALTERA UART/JTAG UART SERIAL DRIVERS M: Tobias Klauser L: linux-serial@vger.kernel.org @@ -1010,7 +1021,6 @@ F: drivers/spi/spi-amd.c AMD MP2 I2C DRIVER M: Elie Morisse -M: Nehal Shah M: Shyam Sundar S K L: linux-i2c@vger.kernel.org S: Maintained @@ -1022,6 +1032,13 @@ L: platform-driver-x86@vger.kernel.org S: Maintained F: drivers/platform/x86/amd/pmc.c +AMD PMF DRIVER +M: Shyam Sundar S K +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-amd-pmf +F: drivers/platform/x86/amd/pmf/ + AMD HSMP DRIVER M: Naveen Krishna Chatradhi R: Carlos Bilbao @@ -1045,6 +1062,7 @@ L: linux-pm@vger.kernel.org S: Supported F: Documentation/admin-guide/pm/amd-pstate.rst F: drivers/cpufreq/amd-pstate* +F: include/linux/amd-pstate.h F: tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py AMD PTDMA DRIVER @@ -1152,7 +1170,7 @@ ANALOG DEVICES INC AD74413R DRIVER M: Cosmin Tanislav L: linux-iio@vger.kernel.org S: Supported -W: http://ez.analog.com/community/linux-device-drivers +W: https://ez.analog.com/linux-software-drivers F: Documentation/devicetree/bindings/iio/addac/adi,ad74413r.yaml F: drivers/iio/addac/ad74413r.c F: include/dt-bindings/iio/addac/adi,ad74413r.h @@ -1327,13 +1345,23 @@ W: https://ez.analog.com/linux-software-drivers F: Documentation/ABI/testing/sysfs-bus-iio-frequency-ad9523 F: Documentation/ABI/testing/sysfs-bus-iio-frequency-adf4350 F: Documentation/devicetree/bindings/iio/*/adi,* -F: Documentation/devicetree/bindings/iio/dac/adi,ad5758.yaml +F: Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml +F: Documentation/devicetree/bindings/iio/adc/lltc,ltc2497.yaml F: drivers/iio/*/ad* F: drivers/iio/adc/ltc249* F: drivers/iio/amplifiers/hmc425a.c F: drivers/staging/iio/*/ad* X: drivers/iio/*/adjd* +ANALOG DEVICES INC MAX31760 DRIVER +M: Ibrahim Tilki +S: Maintained +W: http://wiki.analog.com/ +W: https://ez.analog.com/linux-software-drivers +F: Documentation/devicetree/bindings/hwmon/adi,max31760.yaml +F: Documentation/hwmon/max31760.rst +F: drivers/hwmon/max31760.c + ANALOGBITS PLL LIBRARIES M: Paul Walmsley S: Supported @@ -1382,7 +1410,7 @@ APEX EMBEDDED SYSTEMS STX104 IIO DRIVER M: William Breathitt Gray L: linux-iio@vger.kernel.org S: Maintained -F: drivers/iio/adc/stx104.c +F: drivers/iio/addac/stx104.c APM DRIVER M: Jiri Kosina @@ -1803,7 +1831,7 @@ N: sun[x456789]i N: sun50i ARM/Amlogic Meson SoC CLOCK FRAMEWORK -M: Neil Armstrong +M: Neil Armstrong M: Jerome Brunet L: linux-amlogic@lists.infradead.org S: Maintained @@ -1828,7 +1856,7 @@ F: Documentation/devicetree/bindings/sound/amlogic* F: sound/soc/meson/ ARM/Amlogic Meson SoC support -M: Neil Armstrong +M: Neil Armstrong M: Kevin Hilman R: Jerome Brunet R: Martin Blumenstingl @@ -1887,6 +1915,7 @@ F: drivers/dma/apple-admac.c F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/iommu/apple-dart.c +F: drivers/iommu/io-pgtable-dart.c F: drivers/irqchip/irq-apple-aic.c F: drivers/mailbox/apple-mailbox.c F: drivers/nvme/host/apple.c @@ -1899,6 +1928,15 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/apple-mailbox.h F: include/linux/soc/apple/* +ARM/APPLE MACHINE SOUND DRIVERS +M: Martin Povišer +L: asahi@lists.linux.dev +L: alsa-devel@alsa-project.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/sound/apple,* +F: sound/soc/apple/* +F: sound/soc/codecs/cs42l83-i2c.c + ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson @@ -2029,6 +2067,7 @@ F: drivers/hwtracing/coresight/* F: include/dt-bindings/arm/coresight-cti-dt.h F: include/linux/coresight* F: samples/coresight/* +F: tools/perf/tests/shell/coresight/* F: tools/perf/arch/arm/util/auxtrace.c F: tools/perf/arch/arm/util/cs-etm.c F: tools/perf/arch/arm/util/cs-etm.h @@ -2394,6 +2433,7 @@ N: atmel ARM/Microchip Sparx5 SoC support M: Lars Povlsen M: Steen Hegelund +M: Daniel Machon M: UNGLinuxDriver@microchip.com L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Supported @@ -2531,7 +2571,7 @@ W: http://www.digriz.org.uk/ts78xx/kernel F: arch/arm/mach-orion5x/ts78xx-* ARM/OXNAS platform support -M: Neil Armstrong +M: Neil Armstrong L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-oxnas@groups.io (moderated for non-subscribers) S: Maintained @@ -2579,7 +2619,7 @@ W: http://www.armlinux.org.uk/ ARM/QUALCOMM SUPPORT M: Andy Gross -M: Bjorn Andersson +M: Bjorn Andersson R: Konrad Dybcio L: linux-arm-msm@vger.kernel.org S: Maintained @@ -2588,6 +2628,7 @@ F: Documentation/devicetree/bindings/*/qcom* F: Documentation/devicetree/bindings/soc/qcom/ F: arch/arm/boot/dts/qcom-*.dts F: arch/arm/boot/dts/qcom-*.dtsi +F: arch/arm/configs/qcom_defconfig F: arch/arm/mach-qcom/ F: arch/arm64/boot/dts/qcom/ F: drivers/*/*/qcom* @@ -2650,7 +2691,7 @@ F: arch/arm/boot/dts/rtd* F: arch/arm/mach-realtek/ F: arch/arm64/boot/dts/realtek/ -ARM/RENESAS ARM64 ARCHITECTURE +ARM/RENESAS ARCHITECTURE M: Geert Uytterhoeven M: Magnus Damm L: linux-renesas-soc@vger.kernel.org @@ -2661,6 +2702,16 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git nex F: Documentation/devicetree/bindings/arm/renesas.yaml F: Documentation/devicetree/bindings/hwinfo/renesas,prr.yaml F: Documentation/devicetree/bindings/soc/renesas/ +F: arch/arm/boot/dts/emev2* +F: arch/arm/boot/dts/gr-peach* +F: arch/arm/boot/dts/iwg20d-q7* +F: arch/arm/boot/dts/r7s* +F: arch/arm/boot/dts/r8a* +F: arch/arm/boot/dts/r9a* +F: arch/arm/boot/dts/sh* +F: arch/arm/configs/shmobile_defconfig +F: arch/arm/include/debug/renesas-scif.S +F: arch/arm/mach-shmobile/ F: arch/arm64/boot/dts/renesas/ F: drivers/soc/renesas/ F: include/linux/soc/renesas/ @@ -2670,7 +2721,6 @@ M: Russell King L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained W: http://www.armlinux.org.uk/ -F: arch/arm/include/asm/hardware/entry-macro-iomd.S F: arch/arm/include/asm/hardware/ioc.h F: arch/arm/include/asm/hardware/iomd.h F: arch/arm/include/asm/hardware/memc.h @@ -2772,29 +2822,6 @@ L: linux-media@vger.kernel.org S: Maintained F: drivers/media/platform/samsung/s5p-mfc/ -ARM/SHMOBILE ARM ARCHITECTURE -M: Geert Uytterhoeven -M: Magnus Damm -L: linux-renesas-soc@vger.kernel.org -S: Supported -Q: http://patchwork.kernel.org/project/linux-renesas-soc/list/ -C: irc://irc.libera.chat/renesas-soc -T: git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git next -F: Documentation/devicetree/bindings/arm/renesas.yaml -F: Documentation/devicetree/bindings/soc/renesas/ -F: arch/arm/boot/dts/emev2* -F: arch/arm/boot/dts/gr-peach* -F: arch/arm/boot/dts/iwg20d-q7* -F: arch/arm/boot/dts/r7s* -F: arch/arm/boot/dts/r8a* -F: arch/arm/boot/dts/r9a* -F: arch/arm/boot/dts/sh* -F: arch/arm/configs/shmobile_defconfig -F: arch/arm/include/debug/renesas-scif.S -F: arch/arm/mach-shmobile/ -F: drivers/soc/renesas/ -F: include/linux/soc/renesas/ - ARM/SOCFPGA ARCHITECTURE M: Dinh Nguyen S: Maintained @@ -3087,6 +3114,8 @@ W: http://wiki.xilinx.com T: git https://github.com/Xilinx/linux-xlnx.git F: Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml F: Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml +F: Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml +F: Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml F: Documentation/devicetree/bindings/spi/xlnx,zynq-qspi.yaml F: arch/arm/mach-zynq/ F: drivers/clocksource/timer-cadence-ttc.c @@ -3210,6 +3239,13 @@ S: Maintained F: Documentation/devicetree/bindings/usb/aspeed,ast2600-udc.yaml F: drivers/usb/gadget/udc/aspeed_udc.c +ASPEED CRYPTO DRIVER +M: Neal Liu +L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) +S: Maintained +F: Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml +F: drivers/crypto/aspeed/ + ASUS NOTEBOOKS AND EEEPC ACPI/WMI EXTRAS DRIVERS M: Corentin Chary L: acpi4asus-user@lists.sourceforge.net @@ -3233,13 +3269,6 @@ L: linux-hwmon@vger.kernel.org S: Maintained F: drivers/hwmon/asus_wmi_sensors.c -ASUS WMI EC HARDWARE MONITOR DRIVER -M: Eugene Shalygin -M: Denis Pauk -L: linux-hwmon@vger.kernel.org -S: Maintained -F: drivers/hwmon/asus_wmi_ec_sensors.c - ASUS EC HARDWARE MONITOR DRIVER M: Eugene Shalygin L: linux-hwmon@vger.kernel.org @@ -3612,6 +3641,7 @@ F: include/linux/find.h F: include/linux/nodemask.h F: lib/bitmap.c F: lib/cpumask.c +F: lib/cpumask_kunit.c F: lib/find_bit.c F: lib/find_bit_benchmark.c F: lib/test_bitmap.c @@ -3679,6 +3709,7 @@ F: Documentation/networking/bonding.rst F: drivers/net/bonding/ F: include/net/bond* F: include/uapi/linux/if_bonding.h +F: tools/testing/selftests/drivers/net/bonding/ BOSCH SENSORTEC BMA400 ACCELEROMETER IIO DRIVER M: Dan Robertson @@ -3823,6 +3854,7 @@ F: kernel/bpf/dispatcher.c F: kernel/bpf/trampoline.c F: include/linux/bpf* F: include/linux/filter.h +F: include/linux/tnum.h BPF [BTF] M: Martin KaFai Lau @@ -3948,6 +3980,7 @@ M: William Zhang M: Anand Gore M: Kursad Oney M: Florian Fainelli +M: Rafał Miłecki R: Broadcom internal kernel review list L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained @@ -4894,6 +4927,7 @@ M: Prashant Malani L: chrome-platform@lists.linux.dev S: Maintained F: drivers/platform/chrome/cros_ec_typec.c +F: drivers/platform/chrome/cros_typec_switch.c CHROMEOS EC USB PD NOTIFY DRIVER M: Prashant Malani @@ -4957,7 +4991,7 @@ F: drivers/hwmon/lochnagar-hwmon.c F: drivers/mfd/lochnagar-i2c.c F: drivers/pinctrl/cirrus/pinctrl-lochnagar.c F: drivers/regulator/lochnagar-regulator.c -F: include/dt-bindings/clk/lochnagar.h +F: include/dt-bindings/clock/lochnagar.h F: include/dt-bindings/pinctrl/lochnagar.h F: include/linux/mfd/lochnagar* F: sound/soc/codecs/lochnagar-sc.c @@ -5137,6 +5171,7 @@ M: Steve French R: Paulo Alcantara (DFS, global name space) R: Ronnie Sahlberg (directory leases, sparse files) R: Shyam Prasad N (multichannel) +R: Tom Talpey (RDMA, smbdirect) L: linux-cifs@vger.kernel.org L: samba-technical@lists.samba.org (moderated for non-subscribers) S: Supported @@ -5243,6 +5278,7 @@ F: block/blk-throttle.c F: include/linux/blk-cgroup.h CONTROL GROUP - CPUSET +M: Waiman Long M: Zefan Li L: cgroups@vger.kernel.org S: Maintained @@ -5290,7 +5326,7 @@ COUNTER SUBSYSTEM M: William Breathitt Gray L: linux-iio@vger.kernel.org S: Maintained -T: git https://git.linaro.org/people/william.gray/counter.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git F: Documentation/ABI/testing/sysfs-bus-counter F: Documentation/driver-api/generic-counter.rst F: drivers/counter/ @@ -5371,8 +5407,8 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git F: drivers/cpuidle/cpuidle-big_little.c CPUIDLE DRIVER - ARM EXYNOS -M: Bartlomiej Zolnierkiewicz M: Daniel Lezcano +R: Krzysztof Kozlowski M: Kukjin Kim L: linux-pm@vger.kernel.org L: linux-samsung-soc@vger.kernel.org @@ -5630,6 +5666,12 @@ Q: http://patchwork.linuxtv.org/project/linux-media/list/ T: git git://linuxtv.org/anttip/media_tree.git F: drivers/media/common/cypress_firmware* +CYPRESS CY8C95X0 PINCTRL DRIVER +M: Patrick Rudolph +L: linux-gpio@vger.kernel.org +S: Maintained +F: drivers/pinctrl/pinctrl-cy8c95x0.c + CYPRESS CY8CTMA140 TOUCHSCREEN DRIVER M: Linus Walleij L: linux-input@vger.kernel.org @@ -5720,13 +5762,6 @@ F: include/linux/tfrc.h F: include/uapi/linux/dccp.h F: net/dccp/ -DECnet NETWORK LAYER -L: linux-decnet-user@lists.sourceforge.net -S: Orphan -W: http://linux-decnet.sourceforge.net -F: Documentation/networking/decnet.rst -F: net/decnet/ - DECSTATION PLATFORM SUPPORT M: "Maciej W. Rozycki" L: linux-mips@vger.kernel.org @@ -5895,10 +5930,9 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git F: drivers/usb/dwc2/ DESIGNWARE USB3 DRD IP DRIVER -M: Felipe Balbi +M: Thinh Nguyen L: linux-usb@vger.kernel.org S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git F: drivers/usb/dwc3/ DEVANTECH SRF ULTRASONIC RANGER IIO DRIVER @@ -6145,6 +6179,7 @@ F: include/asm-generic/dma-mapping.h F: include/linux/dma-direct.h F: include/linux/dma-mapping.h F: include/linux/dma-map-ops.h +F: include/linux/swiotlb.h F: kernel/dma/ DMA MAPPING BENCHMARK @@ -6179,7 +6214,7 @@ F: Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.y F: drivers/memory/samsung/exynos5422-dmc.c DME1737 HARDWARE MONITOR DRIVER -M: Juerg Haefliger +M: Juerg Haefliger L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/dme1737.rst @@ -6757,7 +6792,7 @@ F: Documentation/devicetree/bindings/display/panel/samsung,lms380kf01.yaml F: drivers/gpu/drm/panel/panel-widechips-ws2401.c DRM DRIVERS -M: David Airlie +M: David Airlie M: Daniel Vetter L: dri-devel@lists.freedesktop.org S: Maintained @@ -6796,7 +6831,7 @@ F: Documentation/devicetree/bindings/display/allwinner* F: drivers/gpu/drm/sun4i/ DRM DRIVERS FOR AMLOGIC SOCS -M: Neil Armstrong +M: Neil Armstrong L: dri-devel@lists.freedesktop.org L: linux-amlogic@lists.infradead.org S: Supported @@ -6818,7 +6853,7 @@ F: drivers/gpu/drm/atmel-hlcdc/ DRM DRIVERS FOR BRIDGE CHIPS M: Andrzej Hajda -M: Neil Armstrong +M: Neil Armstrong M: Robert Foss R: Laurent Pinchart R: Jonas Karlman @@ -7211,6 +7246,8 @@ M: Jason Baron S: Maintained F: include/linux/dynamic_debug.h F: lib/dynamic_debug.c +M: Jim Cromie +F: lib/test_dynamic_debug.c DYNAMIC INTERRUPT MODERATION M: Tal Gilboa @@ -7538,7 +7575,7 @@ M: Adrian Hunter M: Ritesh Harjani M: Asutosh Das L: linux-mmc@vger.kernel.org -S: Maintained +S: Supported F: drivers/mmc/host/cqhci* EMULEX 10Gbps iSCSI - OneConnect DRIVER @@ -7691,7 +7728,6 @@ R: Kees Cook L: linux-mm@kvack.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/execve -F: arch/alpha/kernel/binfmt_loader.c F: fs/*binfmt_*.c F: fs/exec.c F: include/linux/binfmts.h @@ -8008,6 +8044,7 @@ L: linux-hardening@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/hardening F: include/linux/fortify-string.h +F: lib/fortify_kunit.c F: lib/test_fortify/* F: scripts/test_fortify.sh K: \b__NO_FORTIFY\b @@ -8414,6 +8451,19 @@ L: platform-driver-x86@vger.kernel.org S: Maintained F: drivers/platform/x86/fujitsu-tablet.c +FUNCTION HOOKS (FTRACE) +M: Steven Rostedt +M: Masami Hiramatsu +R: Mark Rutland +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git +F: Documentation/trace/ftrace* +F: kernel/trace/ftrace* +F: kernel/trace/fgraph.c +F: arch/*/*/*/*ftrace* +F: arch/*/*/*ftrace* +F: include/*/ftrace.h + FUNGIBLE ETHERNET DRIVERS M: Dimitris Michailidis L: netdev@vger.kernel.org @@ -8450,7 +8500,6 @@ F: tools/testing/selftests/futex/ GATEWORKS SYSTEM CONTROLLER (GSC) DRIVER M: Tim Harvey -M: Robert Jones S: Maintained F: Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml F: drivers/mfd/gateworks-gsc.c @@ -8656,8 +8705,8 @@ F: drivers/input/touchscreen/goodix* GOOGLE ETHERNET DRIVERS M: Jeroen de Borst -R: Catherine Sullivan -R: David Awogbemila +M: Catherine Sullivan +R: Shailend Chand L: netdev@vger.kernel.org S: Supported F: Documentation/networking/device_drivers/ethernet/google/gve.rst @@ -8890,6 +8939,7 @@ T: git https://git.kernel.org/pub/scm/linux/kernel/git/ogabbay/linux.git F: Documentation/ABI/testing/debugfs-driver-habanalabs F: Documentation/ABI/testing/sysfs-driver-habanalabs F: drivers/misc/habanalabs/ +F: include/trace/events/habanalabs.h F: include/uapi/misc/habanalabs.h HACKRF MEDIA DRIVER @@ -8911,7 +8961,7 @@ S: Maintained F: Documentation/devicetree/bindings/media/nxp,imx8mq-vpu.yaml F: Documentation/devicetree/bindings/media/rockchip,rk3568-vepu.yaml F: Documentation/devicetree/bindings/media/rockchip-vpu.yaml -F: drivers/staging/media/hantro/ +F: drivers/media/platform/verisilicon/ HARD DRIVE ACTIVE PROTECTION SYSTEM (HDAPS) DRIVER M: Frank Seidel @@ -8947,7 +8997,7 @@ F: include/linux/hw_random.h HARDWARE SPINLOCK CORE M: Ohad Ben-Cohen -M: Bjorn Andersson +M: Bjorn Andersson R: Baolin Wang L: linux-remoteproc@vger.kernel.org S: Maintained @@ -9057,6 +9107,12 @@ L: linux-input@vger.kernel.org S: Supported F: drivers/hid/hid-playstation.c +HID PHOENIX RC FLIGHT CONTROLLER +M: Marcus Folkesson +L: linux-input@vger.kernel.org +S: Maintained +F: drivers/hid/hid-pxrc.c + HID SENSOR HUB DRIVERS M: Jiri Kosina M: Jonathan Cameron @@ -9069,6 +9125,12 @@ F: drivers/hid/hid-sensor-* F: drivers/iio/*/hid-* F: include/linux/hid-sensor-* +HID VRC-2 CAR CONTROLLER DRIVER +M: Marcus Folkesson +L: linux-input@vger.kernel.org +S: Maintained +F: drivers/hid/hid-vrc2.c + HID WACOM DRIVER M: Ping Cheng M: Jason Gerecke @@ -9121,12 +9183,13 @@ F: net/dsa/tag_hellcreek.c HISILICON DMA DRIVER M: Zhou Wang +M: Jie Hai L: dmaengine@vger.kernel.org S: Maintained F: drivers/dma/hisi_dma.c HISILICON GPIO DRIVER -M: Luo Jiaxing +M: Jay Fang L: linux-gpio@vger.kernel.org S: Maintained F: drivers/gpio/gpio-hisi.c @@ -9192,6 +9255,14 @@ S: Supported F: Documentation/admin-guide/perf/hns3-pmu.rst F: drivers/perf/hisilicon/hns3_pmu.c +HISILICON PTT DRIVER +M: Yicong Yang +L: linux-kernel@vger.kernel.org +S: Maintained +F: Documentation/ABI/testing/sysfs-devices-hisi_ptt +F: Documentation/trace/hisi-ptt.rst +F: drivers/hwtracing/ptt/ + HISILICON QM DRIVER M: Weili Qian M: Zhou Wang @@ -9212,8 +9283,8 @@ F: Documentation/ABI/testing/debugfs-hisi-zip F: drivers/crypto/hisilicon/zip/ HISILICON ROCE DRIVER +M: Haoyue Xu M: Wenpeng Liang -M: Weihang Li L: linux-rdma@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt @@ -9650,6 +9721,13 @@ S: Orphan F: Documentation/ia64/ F: arch/ia64/ +IBM Operation Panel Input Driver +M: Eddie James +L: linux-input@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/input/ibm,op-panel.yaml +F: drivers/input/misc/ibm-panel.c + IBM Power 842 compression accelerator M: Haren Myneni S: Supported @@ -9787,7 +9865,7 @@ M: Christian Brauner M: Seth Forshee L: linux-fsdevel@vger.kernel.org S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/brauner/linux.git +T: git://git.kernel.org/pub/scm/linux/kernel/git/vfs/idmapping.git F: Documentation/filesystems/idmappings.rst F: tools/testing/selftests/mount_setattr/ F: include/linux/mnt_idmapping.h @@ -10036,6 +10114,7 @@ F: Documentation/devicetree/bindings/input/ F: Documentation/devicetree/bindings/serio/ F: Documentation/input/ F: drivers/input/ +F: include/dt-bindings/input/ F: include/linux/input.h F: include/linux/input/ F: include/uapi/linux/input-event-codes.h @@ -10381,7 +10460,6 @@ INTEL MENLOW THERMAL DRIVER M: Sujith Thomas L: linux-pm@vger.kernel.org S: Supported -W: https://01.org/linux-acpi F: drivers/thermal/intel/intel_menlow.c INTEL P-Unit IPC DRIVER @@ -10629,8 +10707,8 @@ L: iommu@lists.linux.dev S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git F: drivers/iommu/dma-iommu.c +F: drivers/iommu/dma-iommu.h F: drivers/iommu/iova.c -F: include/linux/dma-iommu.h F: include/linux/iova.h IOMMU SUBSYSTEM @@ -10664,6 +10742,7 @@ T: git git://git.kernel.dk/linux-block T: git git://git.kernel.dk/liburing F: io_uring/ F: include/linux/io_uring.h +F: include/linux/io_uring_types.h F: include/uapi/linux/io_uring.h F: tools/io_uring/ @@ -10830,7 +10909,7 @@ F: drivers/media/tuners/it913x* ITE IT66121 HDMI BRIDGE DRIVER M: Phong LE -M: Neil Armstrong +M: Neil Armstrong S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/display/bridge/ite,it66121.yaml @@ -10941,7 +11020,6 @@ F: arch/*/include/asm/*kasan.h F: arch/*/mm/kasan_init* F: include/linux/kasan*.h F: lib/Kconfig.kasan -F: lib/test_kasan*.c F: mm/kasan/ F: scripts/Makefile.kasan @@ -11075,8 +11153,8 @@ F: tools/testing/selftests/ KERNEL SMB3 SERVER (KSMBD) M: Namjae Jeon M: Steve French -M: Hyunchul Lee R: Sergey Senozhatsky +R: Tom Talpey L: linux-cifs@vger.kernel.org S: Maintained T: git git://git.samba.org/ksmbd.git @@ -11127,7 +11205,8 @@ R: Alexandru Elisei R: Suzuki K Poulose R: Oliver Upton L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -L: kvmarm@lists.cs.columbia.edu (moderated for non-subscribers) +L: kvmarm@lists.linux.dev +L: kvmarm@lists.cs.columbia.edu (deprecated, moderated for non-subscribers) S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git F: arch/arm64/include/asm/kvm* @@ -11349,7 +11428,7 @@ F: kernel/debug/ F: kernel/module/kdb.c KHADAS MCU MFD DRIVER -M: Neil Armstrong +M: Neil Armstrong L: linux-amlogic@lists.infradead.org S: Maintained F: Documentation/devicetree/bindings/mfd/khadas,mcu.yaml @@ -11375,13 +11454,27 @@ F: kernel/kmod.c F: lib/test_kmod.c F: tools/testing/selftests/kmod/ +KMSAN +M: Alexander Potapenko +R: Marco Elver +R: Dmitry Vyukov +L: kasan-dev@googlegroups.com +S: Maintained +F: Documentation/dev-tools/kmsan.rst +F: arch/*/include/asm/kmsan.h +F: arch/*/mm/kmsan_* +F: include/linux/kmsan*.h +F: lib/Kconfig.kmsan +F: mm/kmsan/ +F: scripts/Makefile.kmsan + KPROBES M: Naveen N. Rao M: Anil S Keshavamurthy M: "David S. Miller" M: Masami Hiramatsu S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git F: Documentation/trace/kprobes.rst F: include/asm-generic/kprobes.h F: include/linux/kprobes.h @@ -11558,6 +11651,15 @@ F: drivers/ata/ahci_platform.c F: drivers/ata/libahci_platform.c F: include/linux/ahci_platform.h +LIBATA SATA AHCI SYNOPSYS DWC CONTROLLER DRIVER +M: Serge Semin +L: linux-ide@vger.kernel.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/libata.git +F: Documentation/devicetree/bindings/ata/baikal,bt1-ahci.yaml +F: Documentation/devicetree/bindings/ata/snps,dwc-ahci.yaml +F: drivers/ata/ahci_dwc.c + LIBATA SATA PROMISE TX2/TX4 CONTROLLER DRIVER M: Mikael Pettersson L: linux-ide@vger.kernel.org @@ -11632,6 +11734,7 @@ F: Documentation/process/license-rules.rst F: LICENSES/ F: scripts/spdxcheck-test.sh F: scripts/spdxcheck.py +F: scripts/spdxexclude LINEAR RANGES HELPERS M: Mark Brown @@ -11930,7 +12033,7 @@ LTC2688 IIO DAC DRIVER M: Nuno Sá L: linux-iio@vger.kernel.org S: Supported -W: http://ez.analog.com/community/linux-device-drivers +W: https://ez.analog.com/linux-software-drivers F: Documentation/ABI/testing/sysfs-bus-iio-dac-ltc2688 F: Documentation/devicetree/bindings/iio/dac/adi,ltc2688.yaml F: drivers/iio/dac/ltc2688.c @@ -12095,6 +12198,18 @@ L: linux-man@vger.kernel.org S: Maintained W: http://www.kernel.org/doc/man-pages +MAPLE TREE +M: Liam R. Howlett +L: linux-mm@kvack.org +S: Supported +F: Documentation/core-api/maple_tree.rst +F: include/linux/maple_tree.h +F: include/trace/events/maple_tree.h +F: lib/maple_tree.c +F: lib/test_maple_tree.c +F: tools/testing/radix-tree/linux/maple_tree.h +F: tools/testing/radix-tree/maple.c + MARDUK (CREATOR CI40) DEVICE TREE SUPPORT M: Rahul Bedarkar L: linux-mips@vger.kernel.org @@ -12347,6 +12462,14 @@ S: Maintained F: Documentation/devicetree/bindings/iio/proximity/maxbotix,mb1232.yaml F: drivers/iio/proximity/mb1232.c +MAXIM MAX11205 DRIVER +M: Ramona Bolboaca +L: linux-iio@vger.kernel.org +S: Supported +W: https://ez.analog.com/linux-software-drivers +F: Documentation/devicetree/bindings/iio/adc/maxim,max11205.yaml +F: drivers/iio/adc/max11205.c + MAXIM MAX17040 FAMILY FUEL GAUGE DRIVERS R: Iskren Chernev R: Krzysztof Kozlowski @@ -12412,7 +12535,6 @@ F: drivers/power/supply/max77976_charger.c MAXIM MUIC CHARGER DRIVERS FOR EXYNOS BASED BOARDS M: Krzysztof Kozlowski -M: Bartlomiej Zolnierkiewicz L: linux-pm@vger.kernel.org S: Supported B: mailto:linux-samsung-soc@vger.kernel.org @@ -12424,7 +12546,6 @@ F: drivers/power/supply/max77693_charger.c MAXIM PMIC AND MUIC DRIVERS FOR EXYNOS BASED BOARDS M: Chanwoo Choi M: Krzysztof Kozlowski -M: Bartlomiej Zolnierkiewicz L: linux-kernel@vger.kernel.org S: Supported B: mailto:linux-samsung-soc@vger.kernel.org @@ -12820,6 +12941,12 @@ S: Supported F: Documentation/devicetree/bindings/media/mediatek-jpeg-*.yaml F: drivers/media/platform/mediatek/jpeg/ +MEDIATEK KEYPAD DRIVER +M: Mattijs Korpershoek +S: Supported +F: Documentation/devicetree/bindings/input/mediatek,mt6779-keypad.yaml +F: drivers/input/keyboard/mt6779-keypad.c + MEDIATEK MDP DRIVER M: Minghsiu Tsai M: Houlong Wei @@ -12872,7 +12999,7 @@ MEDIATEK MT7621/28/88 I2C DRIVER M: Stefan Roese L: linux-i2c@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/i2c/i2c-mt7621.txt +F: Documentation/devicetree/bindings/i2c/mediatek,mt7621-i2c.yaml F: drivers/i2c/busses/i2c-mt7621.c MEDIATEK MT7621 PCIE CONTROLLER DRIVER @@ -12985,9 +13112,9 @@ F: drivers/input/touchscreen/melfas_mip4.c MELLANOX BLUEFIELD I2C DRIVER M: Khalil Blaiech +M: Asmaa Mnebhi L: linux-i2c@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml F: drivers/i2c/busses/i2c-mlxbf.c MELLANOX ETHERNET DRIVER (mlx4_en) @@ -13191,6 +13318,13 @@ F: drivers/mtd/ F: include/linux/mtd/ F: include/uapi/mtd/ +MEMSENSING MICROSYSTEMS MSA311 DRIVER +M: Dmitry Rokosov +L: linux-iio@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/iio/accel/memsensing,msa311.yaml +F: drivers/iio/accel/msa311.c + MEN A21 WATCHDOG DRIVER M: Johannes Thumshirn L: linux-watchdog@vger.kernel.org @@ -13220,7 +13354,7 @@ S: Maintained F: drivers/watchdog/menz69_wdt.c MESON AO CEC DRIVER FOR AMLOGIC SOCS -M: Neil Armstrong +M: Neil Armstrong L: linux-media@vger.kernel.org L: linux-amlogic@lists.infradead.org S: Supported @@ -13231,7 +13365,7 @@ F: drivers/media/cec/platform/meson/ao-cec-g12a.c F: drivers/media/cec/platform/meson/ao-cec.c MESON GE2D DRIVER FOR AMLOGIC SOCS -M: Neil Armstrong +M: Neil Armstrong L: linux-media@vger.kernel.org L: linux-amlogic@lists.infradead.org S: Supported @@ -13247,7 +13381,7 @@ F: Documentation/devicetree/bindings/mtd/amlogic,meson-nand.txt F: drivers/mtd/nand/raw/meson_* MESON VIDEO DECODER DRIVER FOR AMLOGIC SOCS -M: Neil Armstrong +M: Neil Armstrong L: linux-media@vger.kernel.org L: linux-amlogic@lists.infradead.org S: Supported @@ -13294,7 +13428,7 @@ F: include/dt-bindings/dma/at91.h MICROCHIP AT91 SERIAL DRIVER M: Richard Genoud S: Maintained -F: Documentation/devicetree/bindings/mfd/atmel-usart.txt +F: Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml F: drivers/tty/serial/atmel_serial.c F: drivers/tty/serial/atmel_serial.h @@ -13302,7 +13436,7 @@ MICROCHIP AT91 USART MFD DRIVER M: Radu Pirea L: linux-kernel@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/mfd/atmel-usart.txt +F: Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml F: drivers/mfd/at91-usart.c F: include/dt-bindings/mfd/at91-usart.h @@ -13310,7 +13444,7 @@ MICROCHIP AT91 USART SPI DRIVER M: Radu Pirea L: linux-spi@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/mfd/atmel-usart.txt +F: Documentation/devicetree/bindings/serial/atmel,at91-usart.yaml F: drivers/spi/spi-at91-usart.c MICROCHIP AUDIO ASOC DRIVERS @@ -13412,7 +13546,7 @@ MICROCHIP MCP3911 ADC DRIVER M: Marcus Folkesson M: Kent Gustavsson L: linux-iio@vger.kernel.org -S: Supported +S: Maintained F: Documentation/devicetree/bindings/iio/adc/microchip,mcp3911.yaml F: drivers/iio/adc/mcp3911.c @@ -13428,6 +13562,14 @@ S: Supported F: Documentation/devicetree/bindings/mtd/atmel-nand.txt F: drivers/mtd/nand/raw/atmel/* +MICROCHIP PCI1XXXX GP DRIVER +M: Kumaravel Thiagarajan +L: linux-gpio@vger.kernel.org +S: Supported +F: drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.c +F: drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.h +F: drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c + MICROCHIP OTPC DRIVER M: Claudiu Beznea L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) @@ -13436,6 +13578,14 @@ F: Documentation/devicetree/bindings/nvmem/microchip,sama7g5-otpc.yaml F: drivers/nvmem/microchip-otpc.c F: include/dt-bindings/nvmem/microchip,sama7g5-otpc.h +MICROCHIP PCI1XXXX I2C DRIVER +M: Tharun Kumar P +M: Kumaravel Thiagarajan +M: Microchip Linux Driver Support +L: linux-i2c@vger.kernel.org +S: Maintained +F: drivers/i2c/busses/i2c-mchp-pci1xxxx.c + MICROCHIP PWM DRIVER M: Claudiu Beznea L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) @@ -13495,6 +13645,7 @@ M: UNGLinuxDriver@microchip.com L: linux-mips@vger.kernel.org S: Supported F: Documentation/devicetree/bindings/mips/mscc.txt +F: Documentation/devicetree/bindings/phy/mscc,vsc7514-serdes.yaml F: Documentation/devicetree/bindings/power/reset/ocelot-reset.txt F: arch/mips/boot/dts/mscc/ F: arch/mips/configs/generic/board-ocelot.config @@ -13782,7 +13933,7 @@ MOTION EYE VAIO PICTUREBOOK CAMERA DRIVER S: Orphan W: http://popies.net/meye/ F: Documentation/userspace-api/media/drivers/meye* -F: drivers/media/pci/meye/ +F: drivers/staging/media/deprecated/meye/ F: include/uapi/linux/meye.h MOTORCOMM PHY DRIVER @@ -14451,6 +14602,7 @@ M: Willy Tarreau S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/wtarreau/nolibc.git F: tools/include/nolibc/ +F: tools/testing/selftests/nolibc/ NSDEPS M: Matthias Maennich @@ -14549,6 +14701,15 @@ F: drivers/nvme/common/ F: include/linux/nvme* F: include/uapi/linux/nvme_ioctl.h +NVM EXPRESS FABRICS AUTHENTICATION +M: Hannes Reinecke +L: linux-nvme@lists.infradead.org +S: Supported +F: drivers/nvme/host/auth.c +F: drivers/nvme/target/auth.c +F: drivers/nvme/target/fabrics-cmd-auth.c +F: include/linux/nvme-auth.h + NVM EXPRESS FC TRANSPORT DRIVERS M: James Smart L: linux-nvme@lists.infradead.org @@ -14698,6 +14859,15 @@ S: Orphan F: Documentation/devicetree/bindings/net/nfc/nxp,nci.yaml F: drivers/nfc/nxp-nci +NXP i.MX 8MP DW100 V4L2 DRIVER +M: Xavier Roumegue +L: linux-media@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/media/nxp,dw100.yaml +F: Documentation/userspace-api/media/drivers/dw100.rst +F: drivers/media/platform/nxp/dw100/ +F: include/uapi/linux/dw100.h + NXP i.MX 8QXP/8QM JPEG V4L2 DRIVER M: Mirela Rabulea R: NXP Linux Team @@ -14749,6 +14919,13 @@ F: net/dsa/tag_ocelot.c F: net/dsa/tag_ocelot_8021q.c F: tools/testing/selftests/drivers/net/ocelot/* +OCELOT EXTERNAL SWITCH CONTROL +M: Colin Foster +S: Supported +F: Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml +F: drivers/mfd/ocelot* +F: include/linux/mfd/ocelot.h + OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER M: Frederic Barrat M: Andrew Donnellan @@ -14943,6 +15120,7 @@ F: drivers/regulator/palmas-regulator*.c F: drivers/regulator/pbias-regulator.c F: drivers/regulator/tps65217-regulator.c F: drivers/regulator/tps65218-regulator.c +F: drivers/regulator/tps65219-regulator.c F: drivers/regulator/tps65910-regulator.c F: drivers/regulator/twl-regulator.c F: drivers/regulator/twl6030-regulator.c @@ -15182,17 +15360,6 @@ L: linux-rdma@vger.kernel.org S: Supported F: drivers/infiniband/ulp/opa_vnic -OPEN FIRMWARE AND DEVICE TREE OVERLAYS -M: Pantelis Antoniou -M: Frank Rowand -L: devicetree@vger.kernel.org -S: Maintained -F: Documentation/devicetree/dynamic-resolution-notes.rst -F: Documentation/devicetree/overlay-notes.rst -F: drivers/of/overlay.c -F: drivers/of/resolver.c -K: of_overlay_notifier_ - OPEN FIRMWARE AND FLATTENED DEVICE TREE M: Rob Herring M: Frank Rowand @@ -15205,6 +15372,9 @@ F: Documentation/ABI/testing/sysfs-firmware-ofw F: drivers/of/ F: include/linux/of*.h F: scripts/dtc/ +K: of_overlay_notifier_ +K: of_overlay_fdt_apply +K: of_overlay_remove OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS M: Rob Herring @@ -15242,7 +15412,7 @@ M: Stafford Horne L: openrisc@lists.librecores.org S: Maintained W: http://openrisc.io -T: git git://github.com/openrisc/linux.git +T: git https://github.com/openrisc/linux.git F: Documentation/devicetree/bindings/openrisc/ F: Documentation/openrisc/ F: arch/openrisc/ @@ -15690,6 +15860,7 @@ PCI ENDPOINT SUBSYSTEM M: Kishon Vijay Abraham I M: Lorenzo Pieralisi R: Krzysztof Wilczyński +R: Manivannan Sadhasivam L: linux-pci@vger.kernel.org S: Supported Q: https://patchwork.kernel.org/project/linux-pci/list/ @@ -15703,8 +15874,8 @@ F: drivers/pci/endpoint/ F: tools/pci/ PCI ENHANCED ERROR HANDLING (EEH) FOR POWERPC -M: Russell Currey -M: Oliver O'Halloran +M: Mahesh J Salgaonkar +R: Oliver O'Halloran L: linuxppc-dev@lists.ozlabs.org S: Supported F: Documentation/PCI/pci-error-recovery.rst @@ -16128,7 +16299,7 @@ F: drivers/gpio/gpio-sama5d2-piobu.c F: drivers/pinctrl/pinctrl-at91* PIN CONTROLLER - QUALCOMM -M: Bjorn Andersson +M: Bjorn Andersson L: linux-arm-msm@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/pinctrl/qcom,*.txt @@ -16181,6 +16352,12 @@ F: Documentation/devicetree/bindings/pinctrl/sunplus,* F: drivers/pinctrl/sunplus/ F: include/dt-bindings/pinctrl/sppctl*.h +PINE64 PINEPHONE KEYBOARD DRIVER +M: Samuel Holland +S: Supported +F: Documentation/devicetree/bindings/input/pine64,pinephone-keyboard.yaml +F: drivers/input/keyboard/pinephone-keyboard.c + PKTCDVD DRIVER M: linux-block@vger.kernel.org S: Orphan @@ -16541,14 +16718,6 @@ T: git git://linuxtv.org/media_tree.git F: drivers/media/usb/pwc/* F: include/trace/events/pwc.h -PWM FAN DRIVER -M: Bartlomiej Zolnierkiewicz -L: linux-hwmon@vger.kernel.org -S: Supported -F: Documentation/devicetree/bindings/hwmon/pwm-fan.txt -F: Documentation/hwmon/pwm-fan.rst -F: drivers/hwmon/pwm-fan.c - PWM IR Transmitter M: Sean Young L: linux-media@vger.kernel.org @@ -16617,6 +16786,9 @@ M: Srinivas Kandagatla M: Banajit Goswami L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Supported +F: Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml +F: Documentation/devicetree/bindings/sound/qcom,* +F: drivers/soc/qcom/apr.c F: include/dt-bindings/sound/qcom,wcd9335.h F: sound/soc/codecs/lpass-rx-macro.* F: sound/soc/codecs/lpass-tx-macro.* @@ -16821,7 +16993,7 @@ F: Documentation/devicetree/bindings/media/*camss* F: drivers/media/platform/qcom/camss/ QUALCOMM CLOCK DRIVERS -M: Bjorn Andersson +M: Bjorn Andersson L: linux-arm-msm@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git @@ -16860,6 +17032,7 @@ F: drivers/net/ethernet/qualcomm/emac/ QUALCOMM ETHQOS ETHERNET DRIVER M: Vinod Koul +R: Bhupesh Sharma L: netdev@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/net/qcom,ethqos.txt @@ -16870,7 +17043,7 @@ M: Srinivas Kandagatla M: Amol Maheshwari L: linux-arm-msm@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/misc/qcom,fastrpc.txt +F: Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml F: drivers/misc/fastrpc.c F: include/uapi/misc/fastrpc.h @@ -17310,7 +17483,7 @@ S: Supported F: fs/reiserfs/ REMOTE PROCESSOR (REMOTEPROC) SUBSYSTEM -M: Bjorn Andersson +M: Bjorn Andersson M: Mathieu Poirier L: linux-remoteproc@vger.kernel.org S: Maintained @@ -17323,7 +17496,7 @@ F: include/linux/remoteproc.h F: include/linux/remoteproc/ REMOTE PROCESSOR MESSAGING (RPMSG) SUBSYSTEM -M: Bjorn Andersson +M: Bjorn Andersson M: Mathieu Poirier L: linux-remoteproc@vger.kernel.org S: Maintained @@ -17449,6 +17622,12 @@ S: Maintained F: Documentation/devicetree/bindings/mtd/renesas-nandc.yaml F: drivers/mtd/nand/raw/renesas-nand-controller.c +RENESAS VERSACLOCK 7 CLOCK DRIVER +M: Alex Helms +S: Maintained +F: Documentation/devicetree/bindings/clock/renesas,versaclock7.yaml +F: drivers/clk/clk-versaclock7.c + RESET CONTROLLER FRAMEWORK M: Philipp Zabel S: Maintained @@ -17524,6 +17703,7 @@ M: Palmer Dabbelt M: Albert Ou L: linux-riscv@lists.infradead.org S: Supported +Q: https://patchwork.kernel.org/project/linux-riscv/list/ P: Documentation/riscv/patch-acceptance.rst T: git git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git F: arch/riscv/ @@ -17535,13 +17715,26 @@ M: Conor Dooley M: Daire McNamara L: linux-riscv@lists.infradead.org S: Supported +F: Documentation/devicetree/bindings/clock/microchip,mpfs*.yaml +F: Documentation/devicetree/bindings/gpio/microchip,mpfs-gpio.yaml +F: Documentation/devicetree/bindings/i2c/microchip,corei2c.yaml +F: Documentation/devicetree/bindings/mailbox/microchip,mpfs-mailbox.yaml +F: Documentation/devicetree/bindings/net/can/microchip,mpfs-can.yaml +F: Documentation/devicetree/bindings/pwm/microchip,corepwm.yaml +F: Documentation/devicetree/bindings/riscv/microchip.yaml +F: Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-sys-controller.yaml +F: Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml +F: Documentation/devicetree/bindings/usb/microchip,mpfs-musb.yaml F: arch/riscv/boot/dts/microchip/ F: drivers/char/hw_random/mpfs-rng.c F: drivers/clk/microchip/clk-mpfs.c +F: drivers/i2c/busses/i2c-microchip-core.c F: drivers/mailbox/mailbox-mpfs.c F: drivers/pci/controller/pcie-microchip-host.c +F: drivers/reset/reset-mpfs.c F: drivers/rtc/rtc-mpfs.c F: drivers/soc/microchip/ +F: drivers/spi/spi-microchip-core-qspi.c F: drivers/spi/spi-microchip-core.c F: drivers/usb/musb/mpfs.c F: include/soc/microchip/mpfs.h @@ -17738,6 +17931,35 @@ L: linux-rdma@vger.kernel.org S: Maintained F: drivers/infiniband/ulp/rtrs/ +RUNTIME VERIFICATION (RV) +M: Daniel Bristot de Oliveira +M: Steven Rostedt +L: linux-trace-devel@vger.kernel.org +S: Maintained +F: Documentation/trace/rv/ +F: include/linux/rv.h +F: include/rv/ +F: kernel/trace/rv/ +F: tools/verification/ + +RUST +M: Miguel Ojeda +M: Alex Gaynor +M: Wedson Almeida Filho +R: Boqun Feng +R: Gary Guo +R: Björn Roy Baron +L: rust-for-linux@vger.kernel.org +S: Supported +W: https://github.com/Rust-for-Linux/linux +B: https://github.com/Rust-for-Linux/linux/issues +T: git https://github.com/Rust-for-Linux/linux.git rust-next +F: Documentation/rust/ +F: rust/ +F: samples/rust/ +F: scripts/*rust* +K: \b(?i:rust)\b + RXRPC SOCKETS (AF_RXRPC) M: David Howells M: Marc Dionne @@ -17909,8 +18131,7 @@ M: Hans Verkuil L: linux-media@vger.kernel.org S: Maintained T: git git://linuxtv.org/media_tree.git -F: drivers/media/common/saa7146/ -F: drivers/media/pci/saa7146/ +F: drivers/staging/media/deprecated/saa7146/ F: include/media/drv-intf/saa7146* SAFESETID SECURITY MODULE @@ -17965,7 +18186,6 @@ F: drivers/platform/x86/samsung-laptop.c SAMSUNG MULTIFUNCTION PMIC DEVICE DRIVERS M: Krzysztof Kozlowski -M: Bartlomiej Zolnierkiewicz L: linux-kernel@vger.kernel.org L: linux-samsung-soc@vger.kernel.org S: Supported @@ -18030,12 +18250,14 @@ Q: https://patchwork.linuxtv.org/project/linux-media/list/ F: drivers/media/platform/samsung/exynos4-is/ SAMSUNG SOC CLOCK DRIVERS +M: Krzysztof Kozlowski M: Sylwester Nawrocki M: Tomasz Figa M: Chanwoo Choi R: Alim Akhtar L: linux-samsung-soc@vger.kernel.org S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/snawrocki/clk.git F: Documentation/devicetree/bindings/clock/samsung,*.yaml F: Documentation/devicetree/bindings/clock/samsung,s3c* @@ -18260,7 +18482,7 @@ F: drivers/mmc/host/sdhci-brcmstb* SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) DRIVER M: Adrian Hunter L: linux-mmc@vger.kernel.org -S: Maintained +S: Supported F: drivers/mmc/host/sdhci* SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) MICROCHIP DRIVER @@ -18283,7 +18505,7 @@ S: Maintained F: drivers/mmc/host/sdhci-spear.c SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) TI OMAP DRIVER -M: Kishon Vijay Abraham I +M: Vignesh Raghavendra L: linux-mmc@vger.kernel.org S: Maintained F: drivers/mmc/host/sdhci-omap.c @@ -18296,8 +18518,7 @@ S: Maintained F: drivers/mmc/host/sdhci-esdhc-imx.c SECURE ENCRYPTING DEVICE (SED) OPAL DRIVER -M: Jonathan Derrick -M: Revanth Rajashekar +M: Jonathan Derrick L: linux-block@vger.kernel.org S: Supported F: block/opal_proto.h @@ -18443,6 +18664,7 @@ F: drivers/misc/sgi-xp/ SHARED MEMORY COMMUNICATIONS (SMC) SOCKETS M: Karsten Graul M: Wenjia Zhang +M: Jan Karcher L: linux-s390@vger.kernel.org S: Supported W: http://www.ibm.com/developerworks/linux/linux390/ @@ -19388,8 +19610,8 @@ M: Emil Renner Berthing L: linux-gpio@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/pinctrl/starfive,jh7100-pinctrl.yaml -F: drivers/pinctrl/pinctrl-starfive.c -F: include/dt-bindings/pinctrl/pinctrl-starfive.h +F: drivers/pinctrl/starfive/ +F: include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h STARFIVE JH7100 RESET CONTROLLER DRIVER M: Emil Renner Berthing @@ -19484,6 +19706,11 @@ L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/dlink/sundance.c +SUN HAPPY MEAL ETHERNET DRIVER +M: Sean Anderson +S: Maintained +F: drivers/net/ethernet/sun/sunhme.* + SUNPLUS ETHERNET DRIVER M: Wells Lu L: netdev@vger.kernel.org @@ -19498,6 +19725,15 @@ S: Maintained F: Documentation/devicetree/bindings/nvmem/sunplus,sp7021-ocotp.yaml F: drivers/nvmem/sunplus-ocotp.c +SUNPLUS USB2 PHY DRIVER +M: Vincent Shih +L: linux-usb@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/phy/sunplus,sp7021-usb2-phy.yaml +F: drivers/phy/sunplus/Kconfig +F: drivers/phy/sunplus/Makefile +F: drivers/phy/sunplus/phy-sunplus-usb2.c + SUNPLUS PWM DRIVER M: Hammer Hsieh S: Maintained @@ -19563,16 +19799,6 @@ S: Maintained F: Documentation/admin-guide/svga.rst F: arch/x86/boot/video* -SWIOTLB SUBSYSTEM -M: Christoph Hellwig -L: iommu@lists.linux.dev -S: Supported -W: http://git.infradead.org/users/hch/dma-mapping.git -T: git git://git.infradead.org/users/hch/dma-mapping.git -F: arch/*/kernel/pci-swiotlb.c -F: include/linux/swiotlb.h -F: kernel/dma/swiotlb.c - SWITCHDEV M: Jiri Pirko M: Ivan Vecera @@ -19941,6 +20167,7 @@ S: Supported F: drivers/net/team/ F: include/linux/if_team.h F: include/uapi/linux/if_team.h +F: tools/testing/selftests/drivers/net/team/ TECHNOLOGIC SYSTEMS TS-5500 PLATFORM SUPPORT M: "Savoir-faire Linux Inc." @@ -20254,13 +20481,6 @@ M: Robert Richter S: Odd Fixes F: drivers/gpio/gpio-thunderx.c -TI ADS131E0X ADC SERIES DRIVER -M: Tomislav Denis -L: linux-iio@vger.kernel.org -S: Maintained -F: Documentation/devicetree/bindings/iio/adc/ti,ads131e08.yaml -F: drivers/iio/adc/ti-ads131e08.c - TI AM437X VPFE DRIVER M: "Lad, Prabhakar" L: linux-media@vger.kernel.org @@ -20312,6 +20532,7 @@ R: Sekhar Nori S: Maintained F: Documentation/devicetree/bindings/clock/ti/davinci/ F: drivers/clk/davinci/ +F: include/linux/clk/davinci.h TI DAVINCI SERIES GPIO DRIVER M: Keerthy @@ -20328,8 +20549,18 @@ W: https://linuxtv.org Q: http://patchwork.linuxtv.org/project/linux-media/list/ T: git git://linuxtv.org/mhadli/v4l-dvb-davinci_devices.git F: drivers/media/platform/ti/davinci/ +F: drivers/staging/media/deprecated/vpfe_capture/ F: include/media/davinci/ +TI ENHANCED CAPTURE (eCAP) DRIVER +M: Vignesh Raghavendra +R: Julien Panis +L: linux-iio@vger.kernel.org +L: linux-omap@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/counter/ti,am62-ecap-capture.yaml +F: drivers/counter/ti-ecap-capture.c + TI ENHANCED QUADRATURE ENCODER PULSE (eQEP) DRIVER R: David Lechner L: linux-iio@vger.kernel.org @@ -20468,7 +20699,7 @@ S: Odd fixes W: https://linuxtv.org T: git git://linuxtv.org/media_tree.git F: Documentation/admin-guide/media/tm6000* -F: drivers/media/usb/tm6000/ +F: drivers/staging/media/deprecated/tm6000/ TMIO/SDHI MMC DRIVER M: Wolfram Sang @@ -20568,9 +20799,10 @@ F: include/linux/toshiba.h F: include/uapi/linux/toshiba.h TOSHIBA TC358743 DRIVER -M: Mats Randgaard +M: Hans Verkuil L: linux-media@vger.kernel.org S: Maintained +F: Documentation/devicetree/bindings/media/i2c/tc358743.txt F: drivers/media/i2c/tc358743* F: include/media/i2c/tc358743.h @@ -20591,24 +20823,29 @@ Q: https://patchwork.kernel.org/project/linux-integrity/list/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git F: drivers/char/tpm/ +TPS546D24 DRIVER +M: Duke Du +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/hwmon/tps546d24.rst +F: drivers/hwmon/pmbus/tps546d24.c + TRACING M: Steven Rostedt -M: Ingo Molnar +M: Masami Hiramatsu S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-trace.git -F: Documentation/trace/ftrace.rst -F: arch/*/*/*/*ftrace* -F: arch/*/*/*ftrace* +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git +F: Documentation/trace/* F: fs/tracefs/ -F: include/*/ftrace.h F: include/linux/trace*.h F: include/trace/ F: kernel/trace/ +F: scripts/tracing/ F: tools/testing/selftests/ftrace/ TRACING MMIO ACCESSES (MMIOTRACE) M: Steven Rostedt -M: Ingo Molnar +M: Masami Hiramatsu R: Karol Herbst R: Pekka Paalanen L: linux-kernel@vger.kernel.org @@ -20739,6 +20976,7 @@ U-BOOT ENVIRONMENT VARIABLES M: Rafał Miłecki S: Maintained F: Documentation/devicetree/bindings/nvmem/u-boot,env.yaml +F: drivers/nvmem/u-boot-env.c UACCE ACCELERATOR FRAMEWORK M: Zhangfei Gao @@ -20768,6 +21006,7 @@ UBLK USERSPACE BLOCK DRIVER M: Ming Lei L: linux-block@vger.kernel.org S: Maintained +F: Documentation/block/ublk.rst F: drivers/block/ublk_drv.c F: include/uapi/linux/ublk_cmd.h @@ -21217,7 +21456,7 @@ S: Maintained W: http://royale.zerezo.com/zr364xx/ T: git git://linuxtv.org/media_tree.git F: Documentation/admin-guide/media/zr364xx* -F: drivers/media/usb/zr364xx/ +F: drivers/staging/media/deprecated/zr364xx/ USER-MODE LINUX (UML) M: Richard Weinberger @@ -21314,6 +21553,7 @@ R: Cornelia Huck L: kvm@vger.kernel.org S: Maintained T: git git://github.com/awilliam/linux-vfio.git +F: Documentation/ABI/testing/sysfs-devices-vfio-dev F: Documentation/driver-api/vfio.rst F: drivers/vfio/ F: include/linux/vfio.h @@ -21494,6 +21734,10 @@ F: include/linux/virtio*.h F: include/uapi/linux/virtio_*.h F: tools/virtio/ +IFCVF VIRTIO DATA PATH ACCELERATOR +R: Zhu Lingshan +F: drivers/vdpa/ifcvf/ + VIRTIO BALLOON M: "Michael S. Tsirkin" M: David Hildenbrand @@ -21545,7 +21789,7 @@ F: drivers/gpio/gpio-virtio.c F: include/uapi/linux/virtio_gpio.h VIRTIO GPU DRIVER -M: David Airlie +M: David Airlie M: Gerd Hoffmann R: Gurchetan Singh R: Chia-I Wu @@ -21685,7 +21929,7 @@ VMWARE BALLOON DRIVER M: Nadav Amit R: VMware PV-Drivers Reviewers L: linux-kernel@vger.kernel.org -S: Maintained +S: Supported F: drivers/misc/vmw_balloon.c VMWARE HYPERVISOR INTERFACE @@ -21704,14 +21948,14 @@ M: Bryan Tan M: Vishnu Dasa R: VMware PV-Drivers Reviewers L: linux-rdma@vger.kernel.org -S: Maintained +S: Supported F: drivers/infiniband/hw/vmw_pvrdma/ -VMware PVSCSI driver +VMWARE PVSCSI DRIVER M: Vishal Bhakta R: VMware PV-Drivers Reviewers L: linux-scsi@vger.kernel.org -S: Maintained +S: Supported F: drivers/scsi/vmw_pvscsi.c F: drivers/scsi/vmw_pvscsi.h @@ -21724,19 +21968,19 @@ F: drivers/ptp/ptp_vmw.c VMWARE VMCI DRIVER M: Bryan Tan -M: Rajesh Jalisatgi M: Vishnu Dasa R: VMware PV-Drivers Reviewers L: linux-kernel@vger.kernel.org -S: Maintained +S: Supported F: drivers/misc/vmw_vmci/ +F: include/linux/vmw_vmci* VMWARE VMMOUSE SUBDRIVER M: Zack Rusin R: VMware Graphics Reviewers R: VMware PV-Drivers Reviewers L: linux-input@vger.kernel.org -S: Maintained +S: Supported F: drivers/input/mouse/vmmouse.c F: drivers/input/mouse/vmmouse.h @@ -21744,9 +21988,17 @@ VMWARE VMXNET3 ETHERNET DRIVER M: Ronak Doshi R: VMware PV-Drivers Reviewers L: netdev@vger.kernel.org -S: Maintained +S: Supported F: drivers/net/vmxnet3/ +VMWARE VSOCK VMCI TRANSPORT DRIVER +M: Bryan Tan +M: Vishnu Dasa +R: VMware PV-Drivers Reviewers +L: linux-kernel@vger.kernel.org +S: Supported +F: net/vmw_vsock/vmci_transport* + VOCORE VOCORE2 BOARD M: Harvey Hunt L: linux-mips@vger.kernel.org @@ -21792,7 +22044,7 @@ F: lib/test_scanf.c F: lib/vsprintf.c VT1211 HARDWARE MONITOR DRIVER -M: Juerg Haefliger +M: Juerg Haefliger L: linux-hwmon@vger.kernel.org S: Maintained F: Documentation/hwmon/vt1211.rst @@ -21851,9 +22103,11 @@ F: drivers/input/tablet/wacom_serial4.c WANGXUN ETHERNET DRIVER M: Jiawen Wu +M: Mengyuan Lou +W: https://www.net-swift.com L: netdev@vger.kernel.org S: Maintained -F: Documentation/networking/device_drivers/ethernet/wangxun/txgbe.rst +F: Documentation/networking/device_drivers/ethernet/wangxun/* F: drivers/net/ethernet/wangxun/ WATCHDOG DEVICE DRIVERS @@ -22266,8 +22520,10 @@ M: Stefano Stabellini L: xen-devel@lists.xenproject.org (moderated for non-subscribers) L: iommu@lists.linux.dev S: Supported -F: arch/x86/xen/*swiotlb* -F: drivers/xen/*swiotlb* +F: arch/*/include/asm/xen/swiotlb-xen.h +F: drivers/xen/swiotlb-xen.c +F: include/xen/arm/swiotlb-xen.h +F: include/xen/swiotlb-xen.h XFS FILESYSTEM C: irc://irc.oftc.net/xfs @@ -22309,7 +22565,7 @@ M: Shubhrajyoti Datta R: Srinivas Neeli R: Michal Simek S: Maintained -F: Documentation/devicetree/bindings/gpio/gpio-xilinx.txt +F: Documentation/devicetree/bindings/gpio/xlnx,gpio-xilinx.yaml F: Documentation/devicetree/bindings/gpio/gpio-zynq.yaml F: drivers/gpio/gpio-xilinx.c F: drivers/gpio/gpio-zynq.c diff --git a/Makefile b/Makefile index c7705f7496012ff1328c306c1cf0674084fd1579..f41ec8c8426ba2a351ca18d003872db77ea20127 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 -PATCHLEVEL = 0 +PATCHLEVEL = 1 SUBLEVEL = 0 -EXTRAVERSION = -rc2 +EXTRAVERSION = -rc1 NAME = Hurr durr I'ma ninja sloth # *DOCUMENTATION* @@ -120,6 +120,15 @@ endif export KBUILD_CHECKSRC +# Enable "clippy" (a linter) as part of the Rust compilation. +# +# Use 'make CLIPPY=1' to enable it. +ifeq ("$(origin CLIPPY)", "command line") + KBUILD_CLIPPY := $(CLIPPY) +endif + +export KBUILD_CLIPPY + # Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the # directory of external module to build. Setting M= takes precedence. ifeq ("$(origin M)", "command line") @@ -270,14 +279,14 @@ no-dot-config-targets := $(clean-targets) \ cscope gtags TAGS tags help% %docs check% coccicheck \ $(version_h) headers headers_% archheaders archscripts \ %asm-generic kernelversion %src-pkg dt_binding_check \ - outputmakefile + outputmakefile rustavailable rustfmt rustfmtcheck # Installation targets should not require compiler. Unfortunately, vdso_install # is an exception where build artifacts may be updated. This must be fixed. no-compiler-targets := $(no-dot-config-targets) install dtbs_install \ headers_install modules_install kernelrelease image_name no-sync-config-targets := $(no-dot-config-targets) %install kernelrelease \ image_name -single-targets := %.a %.i %.ko %.lds %.ll %.lst %.mod %.o %.s %.symtypes %/ +single-targets := %.a %.i %.rsi %.ko %.lds %.ll %.lst %.mod %.o %.s %.symtypes %/ config-build := mixed-build := @@ -439,6 +448,7 @@ else HOSTCC = gcc HOSTCXX = g++ endif +HOSTRUSTC = rustc HOSTPKG_CONFIG = pkg-config KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ @@ -447,8 +457,26 @@ KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \ KBUILD_USERCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(USERCFLAGS) KBUILD_USERLDFLAGS := $(USERLDFLAGS) +# These flags apply to all Rust code in the tree, including the kernel and +# host programs. +export rust_common_flags := --edition=2021 \ + -Zbinary_dep_depinfo=y \ + -Dunsafe_op_in_unsafe_fn -Drust_2018_idioms \ + -Dunreachable_pub -Dnon_ascii_idents \ + -Wmissing_docs \ + -Drustdoc::missing_crate_level_docs \ + -Dclippy::correctness -Dclippy::style \ + -Dclippy::suspicious -Dclippy::complexity \ + -Dclippy::perf \ + -Dclippy::let_unit_value -Dclippy::mut_mut \ + -Dclippy::needless_bitwise_bool \ + -Dclippy::needless_continue \ + -Wclippy::dbg_macro + KBUILD_HOSTCFLAGS := $(KBUILD_USERHOSTCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS) KBUILD_HOSTCXXFLAGS := -Wall -O2 $(HOST_LFS_CFLAGS) $(HOSTCXXFLAGS) +KBUILD_HOSTRUSTFLAGS := $(rust_common_flags) -O -Cstrip=debuginfo \ + -Zallow-features= $(HOSTRUSTFLAGS) KBUILD_HOSTLDFLAGS := $(HOST_LFS_LDFLAGS) $(HOSTLDFLAGS) KBUILD_HOSTLDLIBS := $(HOST_LFS_LIBS) $(HOSTLDLIBS) @@ -473,6 +501,12 @@ OBJDUMP = $(CROSS_COMPILE)objdump READELF = $(CROSS_COMPILE)readelf STRIP = $(CROSS_COMPILE)strip endif +RUSTC = rustc +RUSTDOC = rustdoc +RUSTFMT = rustfmt +CLIPPY_DRIVER = clippy-driver +BINDGEN = bindgen +CARGO = cargo PAHOLE = pahole RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids LEX = flex @@ -498,11 +532,13 @@ CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF) NOSTDINC_FLAGS := CFLAGS_MODULE = +RUSTFLAGS_MODULE = AFLAGS_MODULE = LDFLAGS_MODULE = CFLAGS_KERNEL = +RUSTFLAGS_KERNEL = AFLAGS_KERNEL = -LDFLAGS_vmlinux = +export LDFLAGS_vmlinux = # Use USERINCLUDE when you must reference the UAPI directories only. USERINCLUDE := \ @@ -529,15 +565,43 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -Werror=return-type -Wno-format-security \ -std=gnu11 KBUILD_CPPFLAGS := -D__KERNEL__ +KBUILD_RUSTFLAGS := $(rust_common_flags) \ + --target=$(objtree)/rust/target.json \ + -Cpanic=abort -Cembed-bitcode=n -Clto=n \ + -Cforce-unwind-tables=n -Ccodegen-units=1 \ + -Csymbol-mangling-version=v0 \ + -Crelocation-model=static \ + -Zfunction-sections=n \ + -Dclippy::float_arithmetic + KBUILD_AFLAGS_KERNEL := KBUILD_CFLAGS_KERNEL := +KBUILD_RUSTFLAGS_KERNEL := KBUILD_AFLAGS_MODULE := -DMODULE KBUILD_CFLAGS_MODULE := -DMODULE +KBUILD_RUSTFLAGS_MODULE := --cfg MODULE KBUILD_LDFLAGS_MODULE := KBUILD_LDFLAGS := CLANG_FLAGS := +ifeq ($(KBUILD_CLIPPY),1) + RUSTC_OR_CLIPPY_QUIET := CLIPPY + RUSTC_OR_CLIPPY = $(CLIPPY_DRIVER) +else + RUSTC_OR_CLIPPY_QUIET := RUSTC + RUSTC_OR_CLIPPY = $(RUSTC) +endif + +ifdef RUST_LIB_SRC + export RUST_LIB_SRC +endif + +# Allows the usage of unstable features in stable compilers. +export RUSTC_BOOTSTRAP := 1 + export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG +export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO +export HOSTRUSTC KBUILD_HOSTRUSTFLAGS export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD @@ -546,9 +610,10 @@ export KBUILD_USERCFLAGS KBUILD_USERLDFLAGS export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE +export KBUILD_RUSTFLAGS RUSTFLAGS_KERNEL RUSTFLAGS_MODULE export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE -export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE -export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL +export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_RUSTFLAGS_MODULE KBUILD_LDFLAGS_MODULE +export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL KBUILD_RUSTFLAGS_KERNEL export PAHOLE_FLAGS # Files to ignore in find ... statements @@ -583,7 +648,7 @@ quiet_cmd_makefile = GEN Makefile } > Makefile outputmakefile: - $(Q)if [ -f $(srctree)/.config -o \ + @if [ -f $(srctree)/.config -o \ -d $(srctree)/include/config -o \ -d $(srctree)/arch/$(SRCARCH)/include/generated ]; then \ echo >&2 "***"; \ @@ -645,6 +710,8 @@ else __all: modules endif +targets := + # Decide whether to build built-in, modular, or both. # Normally, just do built-in. @@ -676,11 +743,8 @@ endif ifeq ($(KBUILD_EXTMOD),) # Objects we will link into vmlinux / subdirs we need to visit -core-y := init/ usr/ arch/$(SRCARCH)/ -drivers-y := drivers/ sound/ -drivers-$(CONFIG_SAMPLES) += samples/ -drivers-$(CONFIG_NET) += net/ -drivers-y += virt/ +core-y := +drivers-y := libs-y := lib/ endif # KBUILD_EXTMOD @@ -729,7 +793,7 @@ $(KCONFIG_CONFIG): # # Do not use $(call cmd,...) here. That would suppress prompts from syncconfig, # so you cannot notice that Kconfig is waiting for the user input. -%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h: $(KCONFIG_CONFIG) +%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h %/generated/rustc_cfg: $(KCONFIG_CONFIG) $(Q)$(kecho) " SYNC $@" $(Q)$(MAKE) -f $(srctree)/Makefile syncconfig else # !may-sync-config @@ -739,7 +803,7 @@ else # !may-sync-config PHONY += include/config/auto.conf include/config/auto.conf: - $(Q)test -e include/generated/autoconf.h -a -e $@ || ( \ + @test -e include/generated/autoconf.h -a -e $@ || ( \ echo >&2; \ echo >&2 " ERROR: Kernel configuration is invalid."; \ echo >&2 " include/generated/autoconf.h or $@ are missing.";\ @@ -758,10 +822,17 @@ KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 +KBUILD_RUSTFLAGS += -Copt-level=2 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE KBUILD_CFLAGS += -Os +KBUILD_RUSTFLAGS += -Copt-level=s endif +# Always set `debug-assertions` and `overflow-checks` because their default +# depends on `opt-level` and `debug-assertions`, respectively. +KBUILD_RUSTFLAGS += -Cdebug-assertions=$(if $(CONFIG_RUST_DEBUG_ASSERTIONS),y,n) +KBUILD_RUSTFLAGS += -Coverflow-checks=$(if $(CONFIG_RUST_OVERFLOW_CHECKS),y,n) + # Tell gcc to never replace conditional load with a non-conditional one ifdef CONFIG_CC_IS_GCC # gcc-10 renamed --param=allow-store-data-races=0 to @@ -790,7 +861,9 @@ KBUILD_CFLAGS += $(stackp-flags-y) KBUILD_CFLAGS-$(CONFIG_WERROR) += -Werror KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds -KBUILD_CFLAGS += $(KBUILD_CFLAGS-y) $(CONFIG_CC_IMPLICIT_FALLTHROUGH) + +KBUILD_RUSTFLAGS-$(CONFIG_WERROR) += -Dwarnings +KBUILD_RUSTFLAGS += $(KBUILD_RUSTFLAGS-y) ifdef CONFIG_CC_IS_CLANG KBUILD_CPPFLAGS += -Qunused-arguments @@ -812,12 +885,15 @@ KBUILD_CFLAGS += $(call cc-disable-warning, dangling-pointer) ifdef CONFIG_FRAME_POINTER KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls +KBUILD_RUSTFLAGS += -Cforce-frame-pointers=y else # Some targets (ARM with Thumb2, for example), can't be built with frame # pointers. For those, we don't have FUNCTION_TRACER automatically # select FRAME_POINTER. However, FUNCTION_TRACER adds -pg, and this is # incompatible with -fomit-frame-pointer with current GCC, so we don't use # -fomit-frame-pointer with FUNCTION_TRACER. +# In the Rust target specification, "frame-pointer" is set explicitly +# to "may-omit". ifndef CONFIG_FUNCTION_TRACER KBUILD_CFLAGS += -fomit-frame-pointer endif @@ -831,8 +907,8 @@ endif # Initialize all stack variables with a zero value. ifdef CONFIG_INIT_STACK_ALL_ZERO KBUILD_CFLAGS += -ftrivial-auto-var-init=zero -ifdef CONFIG_CC_IS_CLANG -# https://bugs.llvm.org/show_bug.cgi?id=45497 +ifdef CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER +# https://github.com/llvm/llvm-project/issues/44842 KBUILD_CFLAGS += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang endif endif @@ -882,8 +958,10 @@ ifdef CONFIG_DEBUG_SECTION_MISMATCH KBUILD_CFLAGS += -fno-inline-functions-called-once endif +# `rustc`'s `-Zfunction-sections` applies to data too (as of 1.59.0). ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION KBUILD_CFLAGS_KERNEL += -ffunction-sections -fdata-sections +KBUILD_RUSTFLAGS_KERNEL += -Zfunction-sections=y LDFLAGS_vmlinux += --gc-sections endif @@ -921,18 +999,7 @@ export CC_FLAGS_LTO endif ifdef CONFIG_CFI_CLANG -CC_FLAGS_CFI := -fsanitize=cfi \ - -fsanitize-cfi-cross-dso \ - -fno-sanitize-cfi-canonical-jump-tables \ - -fno-sanitize-trap=cfi \ - -fno-sanitize-blacklist - -ifdef CONFIG_CFI_PERMISSIVE -CC_FLAGS_CFI += -fsanitize-recover=cfi -endif - -# If LTO flags are filtered out, we must also filter out CFI. -CC_FLAGS_LTO += $(CC_FLAGS_CFI) +CC_FLAGS_CFI := -fsanitize=kcfi KBUILD_CFLAGS += $(CC_FLAGS_CFI) export CC_FLAGS_CFI endif @@ -972,7 +1039,6 @@ ifdef CONFIG_CC_IS_GCC KBUILD_CFLAGS += -Wno-maybe-uninitialized endif -ifdef CONFIG_CC_IS_GCC # The allocators already balk at large sizes, so silence the compiler # warnings for bounds checks involving those possible values. While # -Wno-alloc-size-larger-than would normally be used here, earlier versions @@ -984,8 +1050,8 @@ ifdef CONFIG_CC_IS_GCC # ignored, continuing to default to PTRDIFF_MAX. So, left with no other # choice, we must perform a versioned check to disable this warning. # https://lore.kernel.org/lkml/20210824115859.187f272f@canb.auug.org.au -KBUILD_CFLAGS += $(call cc-ifversion, -ge, 0901, -Wno-alloc-size-larger-than) -endif +KBUILD_CFLAGS-$(call gcc-min-version, 90100) += -Wno-alloc-size-larger-than +KBUILD_CFLAGS += $(KBUILD_CFLAGS-y) $(CONFIG_CC_IMPLICIT_FALLTHROUGH) # disable invalid "can't wrap" optimizations for signed / pointers KBUILD_CFLAGS += -fno-strict-overflow @@ -1015,6 +1081,7 @@ include-y := scripts/Makefile.extrawarn include-$(CONFIG_DEBUG_INFO) += scripts/Makefile.debug include-$(CONFIG_KASAN) += scripts/Makefile.kasan include-$(CONFIG_KCSAN) += scripts/Makefile.kcsan +include-$(CONFIG_KMSAN) += scripts/Makefile.kmsan include-$(CONFIG_UBSAN) += scripts/Makefile.ubsan include-$(CONFIG_KCOV) += scripts/Makefile.kcov include-$(CONFIG_RANDSTRUCT) += scripts/Makefile.randstruct @@ -1026,10 +1093,11 @@ include $(addprefix $(srctree)/, $(include-y)) # Do not add $(call cc-option,...) below this line. When you build the kernel # from the clean source tree, the GCC plugins do not exist at this point. -# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments +# Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTFLAGS as the last assignments KBUILD_CPPFLAGS += $(KCPPFLAGS) KBUILD_AFLAGS += $(KAFLAGS) KBUILD_CFLAGS += $(KCFLAGS) +KBUILD_RUSTFLAGS += $(KRUSTFLAGS) KBUILD_LDFLAGS_MODULE += --build-id=sha1 LDFLAGS_vmlinux += --build-id=sha1 @@ -1040,7 +1108,7 @@ KBUILD_LDFLAGS += $(call ld-option,--no-warn-rwx-segments) endif ifeq ($(CONFIG_STRIP_ASM_SYMS),y) -LDFLAGS_vmlinux += $(call ld-option, -X,) +LDFLAGS_vmlinux += -X endif ifeq ($(CONFIG_RELR),y) @@ -1101,40 +1169,27 @@ export MODORDER := $(extmod_prefix)modules.order export MODULES_NSDEPS := $(extmod_prefix)modules.nsdeps ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ -core-$(CONFIG_BLOCK) += block/ -core-$(CONFIG_IO_URING) += io_uring/ -vmlinux-dirs := $(patsubst %/,%,$(filter %/, \ - $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ - $(libs-y) $(libs-m))) - -vmlinux-alldirs := $(sort $(vmlinux-dirs) Documentation \ +build-dir := . +clean-dirs := $(sort . Documentation \ $(patsubst %/,%,$(filter %/, $(core-) \ $(drivers-) $(libs-)))) -build-dirs := $(vmlinux-dirs) -clean-dirs := $(vmlinux-alldirs) - -subdir-modorder := $(addsuffix /modules.order, $(build-dirs)) - +export ARCH_CORE := $(core-y) +export ARCH_LIB := $(filter %/, $(libs-y)) +export ARCH_DRIVERS := $(drivers-y) $(drivers-m) # Externally visible symbols (used by link-vmlinux.sh) -KBUILD_VMLINUX_OBJS := $(head-y) $(patsubst %/,%/built-in.a, $(core-y)) -KBUILD_VMLINUX_OBJS += $(addsuffix built-in.a, $(filter %/, $(libs-y))) + +KBUILD_VMLINUX_OBJS := ./built-in.a ifdef CONFIG_MODULES KBUILD_VMLINUX_OBJS += $(patsubst %/, %/lib.a, $(filter %/, $(libs-y))) KBUILD_VMLINUX_LIBS := $(filter-out %/, $(libs-y)) else KBUILD_VMLINUX_LIBS := $(patsubst %/,%/lib.a, $(libs-y)) endif -KBUILD_VMLINUX_OBJS += $(patsubst %/,%/built-in.a, $(drivers-y)) -export KBUILD_VMLINUX_OBJS KBUILD_VMLINUX_LIBS +export KBUILD_VMLINUX_LIBS export KBUILD_LDS := arch/$(SRCARCH)/kernel/vmlinux.lds -# used by scripts/Makefile.package -export KBUILD_ALLDIRS := $(sort $(filter-out arch/%,$(vmlinux-alldirs)) LICENSES arch include scripts tools) - -vmlinux-deps := $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS) # Recurse until adjust_autoksyms.sh is satisfied PHONY += autoksyms_recursive @@ -1144,7 +1199,7 @@ ifdef CONFIG_TRIM_UNUSED_KSYMS # (this can be evaluated only once include/config/auto.conf has been included) KBUILD_MODULES := 1 -autoksyms_recursive: descend modules.order +autoksyms_recursive: $(build-dir) modules.order $(Q)$(CONFIG_SHELL) $(srctree)/scripts/adjust_autoksyms.sh \ "$(MAKE) -f $(srctree)/Makefile autoksyms_recursive" endif @@ -1158,21 +1213,31 @@ quiet_cmd_autoksyms_h = GEN $@ $(autoksyms_h): $(call cmd,autoksyms_h) -ARCH_POSTLINK := $(wildcard $(srctree)/arch/$(SRCARCH)/Makefile.postlink) +# '$(AR) mPi' needs 'T' to workaround the bug of llvm-ar <= 14 +quiet_cmd_ar_vmlinux.a = AR $@ + cmd_ar_vmlinux.a = \ + rm -f $@; \ + $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ + $(AR) mPiT $$($(AR) t $@ | head -n1) $@ $$($(AR) t $@ | grep -F --file=$(srctree)/scripts/head-object-list.txt) + +targets += vmlinux.a +vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt autoksyms_recursive FORCE + $(call if_changed,ar_vmlinux.a) -# Final link of vmlinux with optional arch pass after final link -cmd_link-vmlinux = \ - $(CONFIG_SHELL) $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)"; \ - $(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true) +PHONY += vmlinux_o +vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o -vmlinux: scripts/link-vmlinux.sh autoksyms_recursive $(vmlinux-deps) FORCE - +$(call if_changed_dep,link-vmlinux) +vmlinux.o modules.builtin.modinfo modules.builtin: vmlinux_o + @: -targets := vmlinux +PHONY += vmlinux +vmlinux: vmlinux.o $(KBUILD_LDS) modpost + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux # The actual objects are generated when descending, # make sure no implicit rule kicks in -$(sort $(vmlinux-deps) $(subdir-modorder)): descend ; +$(sort $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)): . ; filechk_kernel.release = \ echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))" @@ -1198,14 +1263,18 @@ PHONY += prepare archprepare archprepare: outputmakefile archheaders archscripts scripts include/config/kernel.release \ asm-generic $(version_h) $(autoksyms_h) include/generated/utsrelease.h \ - include/generated/autoconf.h remove-stale-files + include/generated/compile.h include/generated/autoconf.h remove-stale-files prepare0: archprepare $(Q)$(MAKE) $(build)=scripts/mod - $(Q)$(MAKE) $(build)=. + $(Q)$(MAKE) $(build)=. prepare # All the preparing.. prepare: prepare0 +ifdef CONFIG_RUST + $(Q)$(CONFIG_SHELL) $(srctree)/scripts/rust_is_available.sh -v + $(Q)$(MAKE) $(build)=rust +endif PHONY += remove-stale-files remove-stale-files: @@ -1260,6 +1329,12 @@ $(version_h): FORCE include/generated/utsrelease.h: include/config/kernel.release FORCE $(call filechk,utsrelease.h) +filechk_compile.h = $(srctree)/scripts/mkcompile_h \ + "$(UTS_MACHINE)" "$(CONFIG_CC_VERSION_TEXT)" "$(LD)" + +include/generated/compile.h: FORCE + $(call filechk,compile.h) + PHONY += headerdep headerdep: $(Q)find $(srctree)/include/ -name '*.h' | xargs --max-args 1 \ @@ -1287,8 +1362,7 @@ hdr-inst := -f $(srctree)/scripts/Makefile.headersinst obj PHONY += headers headers: $(version_h) scripts_unifdef uapi-asm-generic archheaders archscripts - $(if $(wildcard $(srctree)/arch/$(SRCARCH)/include/uapi/asm/Kbuild),, \ - $(error Headers not exportable for the $(SRCARCH) architecture)) + $(if $(filter um, $(SRCARCH)), $(error Headers not exportable for UML)) $(Q)$(MAKE) $(hdr-inst)=include/uapi $(Q)$(MAKE) $(hdr-inst)=arch/$(SRCARCH)/include/uapi @@ -1417,6 +1491,10 @@ PHONY += dt_binding_check dt_binding_check: scripts_dtc $(Q)$(MAKE) $(build)=Documentation/devicetree/bindings +PHONY += dt_compatible_check +dt_compatible_check: dt_binding_check + $(Q)$(MAKE) $(build)=Documentation/devicetree/bindings $@ + # --------------------------------------------------------------------------- # Modules @@ -1435,22 +1513,16 @@ endif # Build modules # -# A module can be listed more than once in obj-m resulting in -# duplicate lines in modules.order files. Those are removed -# using awk while concatenating to the final file. - -PHONY += modules -modules: $(if $(KBUILD_BUILTIN),vmlinux) modules_check modules_prepare - -cmd_modules_order = $(AWK) '!x[$$0]++' $(real-prereqs) > $@ -modules.order: $(subdir-modorder) FORCE - $(call if_changed,modules_order) +# *.ko are usually independent of vmlinux, but CONFIG_DEBUG_INFOBTF_MODULES +# is an exception. +ifdef CONFIG_DEBUG_INFO_BTF_MODULES +modules: vmlinux +endif -targets += modules.order +modules: modules_prepare # Target to prepare building external modules -PHONY += modules_prepare modules_prepare: prepare $(Q)$(MAKE) $(build)=scripts scripts/module.lds @@ -1500,7 +1572,8 @@ endif # CONFIG_MODULES # Directories & files removed with 'make clean' CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ - compile_commands.json .thinlto-cache + compile_commands.json .thinlto-cache rust/test rust/doc \ + .vmlinux.objs .vmlinux.export.c # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1511,7 +1584,8 @@ MRPROPER_FILES += include/config include/generated \ certs/signing_key.pem \ certs/x509.genkey \ vmlinux-gdb.py \ - *.spec + *.spec \ + rust/target.json rust/libmacros.so # clean - Delete most, but leave enough to build external modules # @@ -1536,6 +1610,9 @@ $(mrproper-dirs): mrproper: clean $(mrproper-dirs) $(call cmd,rmfiles) + @find . $(RCS_FIND_IGNORE) \ + \( -name '*.rmeta' \) \ + -type f -print | xargs rm -f # distclean # @@ -1623,6 +1700,24 @@ help: @echo ' kselftest-merge - Merge all the config dependencies of' @echo ' kselftest to existing .config.' @echo '' + @echo 'Rust targets:' + @echo ' rustavailable - Checks whether the Rust toolchain is' + @echo ' available and, if not, explains why.' + @echo ' rustfmt - Reformat all the Rust code in the kernel' + @echo ' rustfmtcheck - Checks if all the Rust code in the kernel' + @echo ' is formatted, printing a diff otherwise.' + @echo ' rustdoc - Generate Rust documentation' + @echo ' (requires kernel .config)' + @echo ' rusttest - Runs the Rust tests' + @echo ' (requires kernel .config; downloads external repos)' + @echo ' rust-analyzer - Generate rust-project.json rust-analyzer support file' + @echo ' (requires kernel .config)' + @echo ' dir/file.[os] - Build specified target only' + @echo ' dir/file.rsi - Build macro expanded source, similar to C preprocessing.' + @echo ' Run with RUSTFMT=n to skip reformatting if needed.' + @echo ' The output is not intended to be compilable.' + @echo ' dir/file.ll - Build the LLVM assembly file' + @echo '' @$(if $(dtstree), \ echo 'Devicetree:'; \ echo '* dtbs - Build device tree blobs for enabled boards'; \ @@ -1695,6 +1790,52 @@ PHONY += $(DOC_TARGETS) $(DOC_TARGETS): $(Q)$(MAKE) $(build)=Documentation $@ + +# Rust targets +# --------------------------------------------------------------------------- + +# "Is Rust available?" target +PHONY += rustavailable +rustavailable: + $(Q)$(CONFIG_SHELL) $(srctree)/scripts/rust_is_available.sh -v && echo "Rust is available!" + +# Documentation target +# +# Using the singular to avoid running afoul of `no-dot-config-targets`. +PHONY += rustdoc +rustdoc: prepare + $(Q)$(MAKE) $(build)=rust $@ + +# Testing target +PHONY += rusttest +rusttest: prepare + $(Q)$(MAKE) $(build)=rust $@ + +# Formatting targets +PHONY += rustfmt rustfmtcheck + +# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream. +# +# We match using absolute paths since `find` does not resolve them +# when matching, which is a problem when e.g. `srctree` is `..`. +# We `grep` afterwards in order to remove the directory entry itself. +rustfmt: + $(Q)find $(abs_srctree) -type f -name '*.rs' \ + -o -path $(abs_srctree)/rust/alloc -prune \ + -o -path $(abs_objtree)/rust/test -prune \ + | grep -Fv $(abs_srctree)/rust/alloc \ + | grep -Fv $(abs_objtree)/rust/test \ + | grep -Fv generated \ + | xargs $(RUSTFMT) $(rustfmt_flags) + +rustfmtcheck: rustfmt_flags = --check +rustfmtcheck: rustfmt + +# IDE support targets +PHONY += rust-analyzer +rust-analyzer: + $(Q)$(MAKE) $(build)=rust $@ + # Misc # --------------------------------------------------------------------------- @@ -1720,9 +1861,7 @@ else # KBUILD_EXTMOD KBUILD_BUILTIN := KBUILD_MODULES := 1 -build-dirs := $(KBUILD_EXTMOD) -$(MODORDER): descend - @: +build-dir := $(KBUILD_EXTMOD) compile_commands.json: $(extmod_prefix)compile_commands.json PHONY += compile_commands.json @@ -1751,20 +1890,24 @@ help: @echo ' clean - remove generated files in module directory only' @echo '' -# no-op for external module builds -PHONY += modules_prepare - endif # KBUILD_EXTMOD # --------------------------------------------------------------------------- # Modules -PHONY += modules modules_install +PHONY += modules modules_install modules_prepare ifdef CONFIG_MODULES -modules: modules_check - $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost +$(MODORDER): $(build-dir) + @: + +# KBUILD_MODPOST_NOFINAL can be set to skip the final link of modules. +# This is solely useful to speed up test compiles. +modules: modpost +ifneq ($(KBUILD_MODPOST_NOFINAL),1) + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal +endif PHONY += modules_check modules_check: $(MODORDER) @@ -1791,8 +1934,15 @@ modules modules_install: @echo >&2 '***' @exit 1 +KBUILD_MODULES := + endif # CONFIG_MODULES +PHONY += modpost +modpost: $(if $(single-build),, $(if $(KBUILD_BUILTIN), vmlinux.o)) \ + $(if $(KBUILD_MODULES), modules_check) + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost + # Single targets # --------------------------------------------------------------------------- # To build individual files in subdirectories, you can do like this: @@ -1812,47 +1962,34 @@ single-ko := $(sort $(filter %.ko, $(MAKECMDGOALS))) single-no-ko := $(filter-out $(single-ko), $(MAKECMDGOALS)) \ $(foreach x, o mod, $(patsubst %.ko, %.$x, $(single-ko))) -$(single-ko): single_modpost +$(single-ko): single_modules @: -$(single-no-ko): descend +$(single-no-ko): $(build-dir) @: -ifeq ($(KBUILD_EXTMOD),) -# For the single build of in-tree modules, use a temporary file to avoid -# the situation of modules_install installing an invalid modules.order. -MODORDER := .modules.tmp -endif - -PHONY += single_modpost -single_modpost: $(single-no-ko) modules_prepare +# Remove MODORDER when done because it is not the real one. +PHONY += single_modules +single_modules: $(single-no-ko) modules_prepare $(Q){ $(foreach m, $(single-ko), echo $(extmod_prefix)$m;) } > $(MODORDER) $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modpost +ifneq ($(KBUILD_MODPOST_NOFINAL),1) + $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal +endif + $(Q)rm -f $(MODORDER) -KBUILD_MODULES := 1 - -export KBUILD_SINGLE_TARGETS := $(addprefix $(extmod_prefix), $(single-no-ko)) - -# trim unrelated directories -build-dirs := $(foreach d, $(build-dirs), \ - $(if $(filter $(d)/%, $(KBUILD_SINGLE_TARGETS)), $(d))) +single-goals := $(addprefix $(build-dir)/, $(single-no-ko)) -endif +KBUILD_MODULES := 1 -ifndef CONFIG_MODULES -KBUILD_MODULES := endif -# Handle descending into subdirectories listed in $(build-dirs) # Preset locale variables to speed up the build process. Limit locale # tweaks to this spot to avoid wrong language settings when running # make menuconfig etc. # Error messages still appears in the original language -PHONY += descend $(build-dirs) -descend: $(build-dirs) -$(build-dirs): prepare - $(Q)$(MAKE) $(build)=$@ \ - single-build=$(if $(filter-out $@/, $(filter $@/%, $(KBUILD_SINGLE_TARGETS))),1) \ - need-builtin=1 need-modorder=1 +PHONY += $(build-dir) +$(build-dir): prepare + $(Q)$(MAKE) $(build)=$@ need-builtin=1 need-modorder=1 $(single-goals) clean-dirs := $(addprefix _clean_, $(clean-dirs)) PHONY += $(clean-dirs) clean @@ -1862,7 +1999,7 @@ $(clean-dirs): clean: $(clean-dirs) $(call cmd,rmfiles) @find $(or $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ - \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ + \( -name '*.[aios]' -o -name '*.rsi' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ -o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ -o -name '*.dwo' -o -name '*.lst' \ @@ -1900,7 +2037,7 @@ quiet_cmd_gen_compile_commands = GEN $@ cmd_gen_compile_commands = $(PYTHON3) $< -a $(AR) -o $@ $(filter-out $<, $(real-prereqs)) $(extmod_prefix)compile_commands.json: scripts/clang-tools/gen_compile_commands.py \ - $(if $(KBUILD_EXTMOD),,$(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)) \ + $(if $(KBUILD_EXTMOD),, vmlinux.a $(KBUILD_VMLINUX_LIBS)) \ $(if $(CONFIG_MODULES), $(MODORDER)) FORCE $(call if_changed,gen_compile_commands) diff --git a/arch/Kconfig b/arch/Kconfig index 5dbf11a5ba4e8ea2879b780f23d543d78ac31ebc..8f138e580d1ae1f141dd0b4d3060d86e16f0287c 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -355,6 +355,12 @@ config HAVE_RSEQ This symbol should be selected by an architecture if it supports an implementation of restartable sequences. +config HAVE_RUST + bool + help + This symbol should be selected by an architecture if it + supports Rust. + config HAVE_FUNCTION_ARG_ACCESS_API bool help @@ -738,11 +744,13 @@ config ARCH_SUPPORTS_CFI_CLANG An architecture should select this option if it can support Clang's Control-Flow Integrity (CFI) checking. +config ARCH_USES_CFI_TRAPS + bool + config CFI_CLANG bool "Use Clang's Control Flow Integrity (CFI)" - depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG - depends on CLANG_VERSION >= 140000 - select KALLSYMS + depends on ARCH_SUPPORTS_CFI_CLANG + depends on $(cc-option,-fsanitize=kcfi) help This option enables Clang’s forward-edge Control Flow Integrity (CFI) checking, where the compiler injects a runtime check to each @@ -754,16 +762,6 @@ config CFI_CLANG https://clang.llvm.org/docs/ControlFlowIntegrity.html -config CFI_CLANG_SHADOW - bool "Use CFI shadow to speed up cross-module checks" - default y - depends on CFI_CLANG && MODULES - help - If you select this option, the kernel builds a fast look-up table of - CFI check functions in loaded modules to reduce performance overhead. - - If unsure, say Y. - config CFI_PERMISSIVE bool "Use CFI in permissive mode" depends on CFI_CLANG @@ -923,6 +921,9 @@ config HAVE_SOFTIRQ_ON_OWN_STACK Architecture provides a function to run __do_softirq() on a separate stack. +config SOFTIRQ_ON_OWN_STACK + def_bool HAVE_SOFTIRQ_ON_OWN_STACK && !PREEMPT_RT + config ALTERNATE_USER_ADDRESS_SPACE bool help @@ -1415,6 +1416,14 @@ config DYNAMIC_SIGFRAME config HAVE_ARCH_NODE_DEV_GROUP bool +config ARCH_HAS_NONLEAF_PMD_YOUNG + bool + help + Architectures that select this option are capable of setting the + accessed bit in non-leaf PMD entries when using them as part of linear + address translations. Page table walkers that clear the accessed bit + may use this capability to reduce their search space. + source "kernel/gcov/Kconfig" source "scripts/gcc-plugins/Kconfig" diff --git a/arch/alpha/Makefile b/arch/alpha/Makefile index 881cb913e23abbc416ffdee4e744334d0d989955..45158024085e26faa4d0e51523563758b638ac42 100644 --- a/arch/alpha/Makefile +++ b/arch/alpha/Makefile @@ -36,8 +36,6 @@ cflags-y += $(cpuflags-y) # BWX is most important, but we don't really want any emulation ever. KBUILD_CFLAGS += $(cflags-y) -Wa,-mev6 -head-y := arch/alpha/kernel/head.o - libs-y += arch/alpha/lib/ # export what is needed by arch/alpha/boot/Makefile diff --git a/arch/alpha/configs/defconfig b/arch/alpha/configs/defconfig index 7e933693088005fc071e4fb936802d3325d22c04..6a39fe8ce9e5f870c0ee484cc362fdcddf25e3b5 100644 --- a/arch/alpha/configs/defconfig +++ b/arch/alpha/configs/defconfig @@ -65,7 +65,7 @@ CONFIG_NFSD=m CONFIG_NLS_CODEPAGE_437=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_ALPHA_LEGACY_START_ADDRESS=y CONFIG_MATHEMU=y CONFIG_CRYPTO_HMAC=y diff --git a/arch/alpha/include/asm/a.out.h b/arch/alpha/include/asm/a.out.h deleted file mode 100644 index d2346b7caff1b1bf3b3ab6d1992afd2d72ac974a..0000000000000000000000000000000000000000 --- a/arch/alpha/include/asm/a.out.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ALPHA_A_OUT_H__ -#define __ALPHA_A_OUT_H__ - -#include - - -/* Assume that start addresses below 4G belong to a TASO application. - Unfortunately, there is no proper bit in the exec header to check. - Worse, we have to notice the start address before swapping to use - /sbin/loader, which of course is _not_ a TASO application. */ -#define SET_AOUT_PERSONALITY(BFPM, EX) \ - set_personality (((BFPM->taso || EX.ah.entry < 0x100000000L \ - ? ADDR_LIMIT_32BIT : 0) | PER_OSF4)) - -#endif /* __A_OUT_GNU_H__ */ diff --git a/arch/alpha/include/asm/bitops.h b/arch/alpha/include/asm/bitops.h index 492c7713ddae6d1d9cea6075326e3c6315fb3bf7..bafb1c1f0fdc169384128156505a47151e176d3c 100644 --- a/arch/alpha/include/asm/bitops.h +++ b/arch/alpha/include/asm/bitops.h @@ -283,11 +283,8 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) return (old & mask) != 0; } -static __always_inline bool -arch_test_bit(unsigned long nr, const volatile unsigned long *addr) -{ - return (1UL & (((const int *) addr)[nr >> 5] >> (nr & 31))) != 0UL; -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire /* * ffz = Find First Zero in word. Undefined if no zero exists, diff --git a/arch/alpha/include/asm/core_apecs.h b/arch/alpha/include/asm/core_apecs.h index 2d9726fc02ef8854ff0ca22f8cdaa599197e8fef..69a2fc62c9c3fe1c9f3e419adce66809260047b6 100644 --- a/arch/alpha/include/asm/core_apecs.h +++ b/arch/alpha/include/asm/core_apecs.h @@ -384,7 +384,7 @@ struct el_apecs_procdata } \ } while (0) -__EXTERN_INLINE unsigned int apecs_ioread8(const void __iomem *xaddr) +__EXTERN_INLINE u8 apecs_ioread8(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -420,7 +420,7 @@ __EXTERN_INLINE void apecs_iowrite8(u8 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int apecs_ioread16(const void __iomem *xaddr) +__EXTERN_INLINE u16 apecs_ioread16(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -456,7 +456,7 @@ __EXTERN_INLINE void apecs_iowrite16(u16 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int apecs_ioread32(const void __iomem *xaddr) +__EXTERN_INLINE u32 apecs_ioread32(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; if (addr < APECS_DENSE_MEM) @@ -472,6 +472,22 @@ __EXTERN_INLINE void apecs_iowrite32(u32 b, void __iomem *xaddr) *(vuip)addr = b; } +__EXTERN_INLINE u64 apecs_ioread64(const void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < APECS_DENSE_MEM) + addr = ((addr - APECS_IO) << 5) + APECS_IO + 0x18; + return *(vulp)addr; +} + +__EXTERN_INLINE void apecs_iowrite64(u64 b, void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < APECS_DENSE_MEM) + addr = ((addr - APECS_IO) << 5) + APECS_IO + 0x18; + *(vulp)addr = b; +} + __EXTERN_INLINE void __iomem *apecs_ioportmap(unsigned long addr) { return (void __iomem *)(addr + APECS_IO); diff --git a/arch/alpha/include/asm/core_cia.h b/arch/alpha/include/asm/core_cia.h index cb22991f676193663d3b7aba48b007b1b0996b99..d26bdfb7ca3be390b7d425e6299be4932b03f587 100644 --- a/arch/alpha/include/asm/core_cia.h +++ b/arch/alpha/include/asm/core_cia.h @@ -342,7 +342,7 @@ struct el_CIA_sysdata_mcheck { #define vuip volatile unsigned int __force * #define vulp volatile unsigned long __force * -__EXTERN_INLINE unsigned int cia_ioread8(const void __iomem *xaddr) +__EXTERN_INLINE u8 cia_ioread8(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -374,7 +374,7 @@ __EXTERN_INLINE void cia_iowrite8(u8 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int cia_ioread16(const void __iomem *xaddr) +__EXTERN_INLINE u16 cia_ioread16(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -404,7 +404,7 @@ __EXTERN_INLINE void cia_iowrite16(u16 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int cia_ioread32(const void __iomem *xaddr) +__EXTERN_INLINE u32 cia_ioread32(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; if (addr < CIA_DENSE_MEM) @@ -420,6 +420,22 @@ __EXTERN_INLINE void cia_iowrite32(u32 b, void __iomem *xaddr) *(vuip)addr = b; } +__EXTERN_INLINE u64 cia_ioread64(const void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < CIA_DENSE_MEM) + addr = ((addr - CIA_IO) << 5) + CIA_IO + 0x18; + return *(vulp)addr; +} + +__EXTERN_INLINE void cia_iowrite64(u64 b, void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < CIA_DENSE_MEM) + addr = ((addr - CIA_IO) << 5) + CIA_IO + 0x18; + *(vulp)addr = b; +} + __EXTERN_INLINE void __iomem *cia_ioportmap(unsigned long addr) { return (void __iomem *)(addr + CIA_IO); diff --git a/arch/alpha/include/asm/core_lca.h b/arch/alpha/include/asm/core_lca.h index ec86314418cb809405c9f872416550abaa522394..d8c3e72ef8f69e99759494b2d581daf0df12b195 100644 --- a/arch/alpha/include/asm/core_lca.h +++ b/arch/alpha/include/asm/core_lca.h @@ -230,7 +230,7 @@ union el_lca { } while (0) -__EXTERN_INLINE unsigned int lca_ioread8(const void __iomem *xaddr) +__EXTERN_INLINE u8 lca_ioread8(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -266,7 +266,7 @@ __EXTERN_INLINE void lca_iowrite8(u8 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int lca_ioread16(const void __iomem *xaddr) +__EXTERN_INLINE u16 lca_ioread16(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; unsigned long result, base_and_type; @@ -302,7 +302,7 @@ __EXTERN_INLINE void lca_iowrite16(u16 b, void __iomem *xaddr) *(vuip) ((addr << 5) + base_and_type) = w; } -__EXTERN_INLINE unsigned int lca_ioread32(const void __iomem *xaddr) +__EXTERN_INLINE u32 lca_ioread32(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; if (addr < LCA_DENSE_MEM) @@ -318,6 +318,22 @@ __EXTERN_INLINE void lca_iowrite32(u32 b, void __iomem *xaddr) *(vuip)addr = b; } +__EXTERN_INLINE u64 lca_ioread64(const void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < LCA_DENSE_MEM) + addr = ((addr - LCA_IO) << 5) + LCA_IO + 0x18; + return *(vulp)addr; +} + +__EXTERN_INLINE void lca_iowrite64(u64 b, void __iomem *xaddr) +{ + unsigned long addr = (unsigned long) xaddr; + if (addr < LCA_DENSE_MEM) + addr = ((addr - LCA_IO) << 5) + LCA_IO + 0x18; + *(vulp)addr = b; +} + __EXTERN_INLINE void __iomem *lca_ioportmap(unsigned long addr) { return (void __iomem *)(addr + LCA_IO); diff --git a/arch/alpha/include/asm/core_marvel.h b/arch/alpha/include/asm/core_marvel.h index b266e02e284b779481f3637c6680b0df4609e6f5..d99f3a82e0e5eea9b1e736fe15fa31a60baeab30 100644 --- a/arch/alpha/include/asm/core_marvel.h +++ b/arch/alpha/include/asm/core_marvel.h @@ -332,10 +332,10 @@ struct io7 { #define vucp volatile unsigned char __force * #define vusp volatile unsigned short __force * -extern unsigned int marvel_ioread8(const void __iomem *); +extern u8 marvel_ioread8(const void __iomem *); extern void marvel_iowrite8(u8 b, void __iomem *); -__EXTERN_INLINE unsigned int marvel_ioread16(const void __iomem *addr) +__EXTERN_INLINE u16 marvel_ioread16(const void __iomem *addr) { return __kernel_ldwu(*(vusp)addr); } diff --git a/arch/alpha/include/asm/core_mcpcia.h b/arch/alpha/include/asm/core_mcpcia.h index cb24d1bd6141969c3d1246f3f5ce936c7cb09f28..ed2bf8ad40edaaabca6711cc684434b8dd104f9f 100644 --- a/arch/alpha/include/asm/core_mcpcia.h +++ b/arch/alpha/include/asm/core_mcpcia.h @@ -248,6 +248,7 @@ struct el_MCPCIA_uncorrected_frame_mcheck { #define vip volatile int __force * #define vuip volatile unsigned int __force * +#define vulp volatile unsigned long __force * #ifndef MCPCIA_ONE_HAE_WINDOW #define MCPCIA_FROB_MMIO \ @@ -267,7 +268,7 @@ extern inline int __mcpcia_is_mmio(unsigned long addr) return (addr & 0x80000000UL) == 0; } -__EXTERN_INLINE unsigned int mcpcia_ioread8(const void __iomem *xaddr) +__EXTERN_INLINE u8 mcpcia_ioread8(const void __iomem *xaddr) { unsigned long addr = (unsigned long)xaddr & MCPCIA_MEM_MASK; unsigned long hose = (unsigned long)xaddr & ~MCPCIA_MEM_MASK; @@ -291,7 +292,7 @@ __EXTERN_INLINE void mcpcia_iowrite8(u8 b, void __iomem *xaddr) *(vuip) ((addr << 5) + hose + 0x00) = w; } -__EXTERN_INLINE unsigned int mcpcia_ioread16(const void __iomem *xaddr) +__EXTERN_INLINE u16 mcpcia_ioread16(const void __iomem *xaddr) { unsigned long addr = (unsigned long)xaddr & MCPCIA_MEM_MASK; unsigned long hose = (unsigned long)xaddr & ~MCPCIA_MEM_MASK; @@ -315,7 +316,7 @@ __EXTERN_INLINE void mcpcia_iowrite16(u16 b, void __iomem *xaddr) *(vuip) ((addr << 5) + hose + 0x08) = w; } -__EXTERN_INLINE unsigned int mcpcia_ioread32(const void __iomem *xaddr) +__EXTERN_INLINE u32 mcpcia_ioread32(const void __iomem *xaddr) { unsigned long addr = (unsigned long)xaddr; @@ -335,6 +336,26 @@ __EXTERN_INLINE void mcpcia_iowrite32(u32 b, void __iomem *xaddr) *(vuip)addr = b; } +__EXTERN_INLINE u64 mcpcia_ioread64(const void __iomem *xaddr) +{ + unsigned long addr = (unsigned long)xaddr; + + if (!__mcpcia_is_mmio(addr)) + addr = ((addr & 0xffff) << 5) + (addr & ~0xfffful) + 0x18; + + return *(vulp)addr; +} + +__EXTERN_INLINE void mcpcia_iowrite64(u64 b, void __iomem *xaddr) +{ + unsigned long addr = (unsigned long)xaddr; + + if (!__mcpcia_is_mmio(addr)) + addr = ((addr & 0xffff) << 5) + (addr & ~0xfffful) + 0x18; + + *(vulp)addr = b; +} + __EXTERN_INLINE void __iomem *mcpcia_ioportmap(unsigned long addr) { @@ -362,6 +383,7 @@ __EXTERN_INLINE int mcpcia_is_mmio(const volatile void __iomem *xaddr) #undef vip #undef vuip +#undef vulp #undef __IO_PREFIX #define __IO_PREFIX mcpcia diff --git a/arch/alpha/include/asm/core_t2.h b/arch/alpha/include/asm/core_t2.h index 12bb7addc789f78f0c41699ca0af32cb7731c2d0..ab956b1625b53a444e6225d32d937cc62c8e473d 100644 --- a/arch/alpha/include/asm/core_t2.h +++ b/arch/alpha/include/asm/core_t2.h @@ -360,6 +360,7 @@ struct el_t2_frame_corrected { #define vip volatile int * #define vuip volatile unsigned int * +#define vulp volatile unsigned long * extern inline u8 t2_inb(unsigned long addr) { @@ -402,6 +403,17 @@ extern inline void t2_outl(u32 b, unsigned long addr) mb(); } +extern inline u64 t2_inq(unsigned long addr) +{ + return *(vulp) ((addr << 5) + T2_IO + 0x18); +} + +extern inline void t2_outq(u64 b, unsigned long addr) +{ + *(vulp) ((addr << 5) + T2_IO + 0x18) = b; + mb(); +} + /* * Memory functions. @@ -572,7 +584,7 @@ __EXTERN_INLINE int t2_is_mmio(const volatile void __iomem *addr) it doesn't make sense to merge the pio and mmio routines. */ #define IOPORT(OS, NS) \ -__EXTERN_INLINE unsigned int t2_ioread##NS(const void __iomem *xaddr) \ +__EXTERN_INLINE u##NS t2_ioread##NS(const void __iomem *xaddr) \ { \ if (t2_is_mmio(xaddr)) \ return t2_read##OS(xaddr); \ @@ -590,11 +602,13 @@ __EXTERN_INLINE void t2_iowrite##NS(u##NS b, void __iomem *xaddr) \ IOPORT(b, 8) IOPORT(w, 16) IOPORT(l, 32) +IOPORT(q, 64) #undef IOPORT #undef vip #undef vuip +#undef vulp #undef __IO_PREFIX #define __IO_PREFIX t2 diff --git a/arch/alpha/include/asm/io.h b/arch/alpha/include/asm/io.h index d277189b2677ad41ef3db88203640816ee4d66e3..1c3605d874e9c21d93811fb9516130ed38465866 100644 --- a/arch/alpha/include/asm/io.h +++ b/arch/alpha/include/asm/io.h @@ -90,6 +90,8 @@ static inline void * phys_to_virt(unsigned long address) } #endif +#define virt_to_phys virt_to_phys +#define phys_to_virt phys_to_virt #define page_to_phys(page) page_to_pa(page) /* Maximum PIO space address supported? */ @@ -153,6 +155,7 @@ static inline void generic_##NAME(TYPE b, QUAL void __iomem *addr) \ REMAP1(unsigned int, ioread8, const) REMAP1(unsigned int, ioread16, const) REMAP1(unsigned int, ioread32, const) +REMAP1(u64, ioread64, const) REMAP1(u8, readb, const volatile) REMAP1(u16, readw, const volatile) REMAP1(u32, readl, const volatile) @@ -161,6 +164,7 @@ REMAP1(u64, readq, const volatile) REMAP2(u8, iowrite8, /**/) REMAP2(u16, iowrite16, /**/) REMAP2(u32, iowrite32, /**/) +REMAP2(u64, iowrite64, /**/) REMAP2(u8, writeb, volatile) REMAP2(u16, writew, volatile) REMAP2(u32, writel, volatile) @@ -242,6 +246,12 @@ extern u32 inl(unsigned long port); extern void outb(u8 b, unsigned long port); extern void outw(u16 b, unsigned long port); extern void outl(u32 b, unsigned long port); +#define inb inb +#define inw inw +#define inl inl +#define outb outb +#define outw outw +#define outl outl extern u8 readb(const volatile void __iomem *addr); extern u16 readw(const volatile void __iomem *addr); @@ -251,6 +261,14 @@ extern void writeb(u8 b, volatile void __iomem *addr); extern void writew(u16 b, volatile void __iomem *addr); extern void writel(u32 b, volatile void __iomem *addr); extern void writeq(u64 b, volatile void __iomem *addr); +#define readb readb +#define readw readw +#define readl readl +#define readq readq +#define writeb writeb +#define writew writew +#define writel writel +#define writeq writeq extern u8 __raw_readb(const volatile void __iomem *addr); extern u16 __raw_readw(const volatile void __iomem *addr); @@ -260,6 +278,14 @@ extern void __raw_writeb(u8 b, volatile void __iomem *addr); extern void __raw_writew(u16 b, volatile void __iomem *addr); extern void __raw_writel(u32 b, volatile void __iomem *addr); extern void __raw_writeq(u64 b, volatile void __iomem *addr); +#define __raw_readb __raw_readb +#define __raw_readw __raw_readw +#define __raw_readl __raw_readl +#define __raw_readq __raw_readq +#define __raw_writeb __raw_writeb +#define __raw_writew __raw_writew +#define __raw_writel __raw_writel +#define __raw_writeq __raw_writeq /* * Mapping from port numbers to __iomem space is pretty easy. @@ -277,6 +303,9 @@ extern inline void ioport_unmap(void __iomem *addr) { } +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap + static inline void __iomem *ioremap(unsigned long port, unsigned long size) { return IO_CONCAT(__IO_PREFIX,ioremap) (port, size); @@ -358,6 +387,11 @@ extern inline void outw(u16 b, unsigned long port) } #endif +#define ioread8 ioread8 +#define ioread16 ioread16 +#define iowrite8 iowrite8 +#define iowrite16 iowrite16 + #if IO_CONCAT(__IO_PREFIX,trivial_io_lq) extern inline unsigned int ioread32(const void __iomem *addr) { @@ -368,12 +402,27 @@ extern inline unsigned int ioread32(const void __iomem *addr) return ret; } +extern inline u64 ioread64(const void __iomem *addr) +{ + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread64)(addr); + mb(); + return ret; +} + extern inline void iowrite32(u32 b, void __iomem *addr) { mb(); IO_CONCAT(__IO_PREFIX, iowrite32)(b, addr); } +extern inline void iowrite64(u64 b, void __iomem *addr) +{ + mb(); + IO_CONCAT(__IO_PREFIX, iowrite64)(b, addr); +} + extern inline u32 inl(unsigned long port) { return ioread32(ioport_map(port, 4)); @@ -385,6 +434,11 @@ extern inline void outl(u32 b, unsigned long port) } #endif +#define ioread32 ioread32 +#define ioread64 ioread64 +#define iowrite32 iowrite32 +#define iowrite64 iowrite64 + #if IO_CONCAT(__IO_PREFIX,trivial_rw_bw) == 1 extern inline u8 __raw_readb(const volatile void __iomem *addr) { @@ -505,6 +559,10 @@ extern u8 readb_relaxed(const volatile void __iomem *addr); extern u16 readw_relaxed(const volatile void __iomem *addr); extern u32 readl_relaxed(const volatile void __iomem *addr); extern u64 readq_relaxed(const volatile void __iomem *addr); +#define readb_relaxed readb_relaxed +#define readw_relaxed readw_relaxed +#define readl_relaxed readl_relaxed +#define readq_relaxed readq_relaxed #if IO_CONCAT(__IO_PREFIX,trivial_io_bw) extern inline u8 readb_relaxed(const volatile void __iomem *addr) @@ -557,6 +615,10 @@ static inline void memsetw_io(volatile void __iomem *addr, u16 c, long len) _memset_c_io(addr, 0x0001000100010001UL * c, len); } +#define memset_io memset_io +#define memcpy_fromio memcpy_fromio +#define memcpy_toio memcpy_toio + /* * String versions of in/out ops: */ @@ -567,6 +629,13 @@ extern void outsb (unsigned long port, const void *src, unsigned long count); extern void outsw (unsigned long port, const void *src, unsigned long count); extern void outsl (unsigned long port, const void *src, unsigned long count); +#define insb insb +#define insw insw +#define insl insl +#define outsb outsb +#define outsw outsw +#define outsl outsl + /* * The Alpha Jensen hardware for some rather strange reason puts * the RTC clock at 0x170 instead of 0x70. Probably due to some @@ -586,22 +655,30 @@ extern void outsl (unsigned long port, const void *src, unsigned long count); #endif #define RTC_ALWAYS_BCD 0 -/* - * Some mucking forons use if[n]def writeq to check if platform has it. - * It's a bloody bad idea and we probably want ARCH_HAS_WRITEQ for them - * to play with; for now just use cpp anti-recursion logics and make sure - * that damn thing is defined and expands to itself. - */ - -#define writeq writeq -#define readq readq - /* * Convert a physical pointer to a virtual kernel pointer for /dev/mem * access */ #define xlate_dev_mem_ptr(p) __va(p) +/* + * These get provided from since alpha does not + * select GENERIC_IOMAP. + */ +#define ioread64 ioread64 +#define iowrite64 iowrite64 +#define ioread64be ioread64be +#define iowrite64be iowrite64be +#define ioread8_rep ioread8_rep +#define ioread16_rep ioread16_rep +#define ioread32_rep ioread32_rep +#define iowrite8_rep iowrite8_rep +#define iowrite16_rep iowrite16_rep +#define iowrite32_rep iowrite32_rep +#define pci_iounmap pci_iounmap + +#include + #endif /* __KERNEL__ */ #endif /* __ALPHA_IO_H */ diff --git a/arch/alpha/include/asm/io_trivial.h b/arch/alpha/include/asm/io_trivial.h index a1a29cbe02fadc29392853570348aae29e4cb0c7..00032093bcfcd42c5372036da312634b864db603 100644 --- a/arch/alpha/include/asm/io_trivial.h +++ b/arch/alpha/include/asm/io_trivial.h @@ -6,13 +6,13 @@ /* This file may be included multiple times. */ #if IO_CONCAT(__IO_PREFIX,trivial_io_bw) -__EXTERN_INLINE unsigned int +__EXTERN_INLINE u8 IO_CONCAT(__IO_PREFIX,ioread8)(const void __iomem *a) { return __kernel_ldbu(*(const volatile u8 __force *)a); } -__EXTERN_INLINE unsigned int +__EXTERN_INLINE u16 IO_CONCAT(__IO_PREFIX,ioread16)(const void __iomem *a) { return __kernel_ldwu(*(const volatile u16 __force *)a); @@ -32,7 +32,7 @@ IO_CONCAT(__IO_PREFIX,iowrite16)(u16 b, void __iomem *a) #endif #if IO_CONCAT(__IO_PREFIX,trivial_io_lq) -__EXTERN_INLINE unsigned int +__EXTERN_INLINE u32 IO_CONCAT(__IO_PREFIX,ioread32)(const void __iomem *a) { return *(const volatile u32 __force *)a; @@ -43,6 +43,18 @@ IO_CONCAT(__IO_PREFIX,iowrite32)(u32 b, void __iomem *a) { *(volatile u32 __force *)a = b; } + +__EXTERN_INLINE u64 +IO_CONCAT(__IO_PREFIX,ioread64)(const void __iomem *a) +{ + return *(const volatile u64 __force *)a; +} + +__EXTERN_INLINE void +IO_CONCAT(__IO_PREFIX,iowrite64)(u64 b, void __iomem *a) +{ + *(volatile u64 __force *)a = b; +} #endif #if IO_CONCAT(__IO_PREFIX,trivial_rw_bw) == 1 diff --git a/arch/alpha/include/asm/jensen.h b/arch/alpha/include/asm/jensen.h index 1c4131453db2ca100b2a7522289d4f8368f45756..66eb049eb421f71154c972071a0487fb315a626c 100644 --- a/arch/alpha/include/asm/jensen.h +++ b/arch/alpha/include/asm/jensen.h @@ -98,6 +98,7 @@ __EXTERN_INLINE void jensen_set_hae(unsigned long addr) } #define vuip volatile unsigned int * +#define vulp volatile unsigned long * /* * IO functions @@ -183,6 +184,12 @@ __EXTERN_INLINE u32 jensen_inl(unsigned long addr) return *(vuip) ((addr << 7) + EISA_IO + 0x60); } +__EXTERN_INLINE u64 jensen_inq(unsigned long addr) +{ + jensen_set_hae(0); + return *(vulp) ((addr << 7) + EISA_IO + 0x60); +} + __EXTERN_INLINE void jensen_outw(u16 b, unsigned long addr) { jensen_set_hae(0); @@ -197,6 +204,13 @@ __EXTERN_INLINE void jensen_outl(u32 b, unsigned long addr) mb(); } +__EXTERN_INLINE void jensen_outq(u64 b, unsigned long addr) +{ + jensen_set_hae(0); + *(vulp) ((addr << 7) + EISA_IO + 0x60) = b; + mb(); +} + /* * Memory functions. */ @@ -305,7 +319,7 @@ __EXTERN_INLINE int jensen_is_mmio(const volatile void __iomem *addr) that it doesn't make sense to merge them. */ #define IOPORT(OS, NS) \ -__EXTERN_INLINE unsigned int jensen_ioread##NS(const void __iomem *xaddr) \ +__EXTERN_INLINE u##NS jensen_ioread##NS(const void __iomem *xaddr) \ { \ if (jensen_is_mmio(xaddr)) \ return jensen_read##OS(xaddr - 0x100000000ul); \ @@ -323,10 +337,12 @@ __EXTERN_INLINE void jensen_iowrite##NS(u##NS b, void __iomem *xaddr) \ IOPORT(b, 8) IOPORT(w, 16) IOPORT(l, 32) +IOPORT(q, 64) #undef IOPORT #undef vuip +#undef vulp #undef __IO_PREFIX #define __IO_PREFIX jensen diff --git a/arch/alpha/include/asm/machvec.h b/arch/alpha/include/asm/machvec.h index e49fabce7b3317ecf92378f0df7ef78c6a2ee05e..8623f995d34c632fe325067dbeae9a415644b18c 100644 --- a/arch/alpha/include/asm/machvec.h +++ b/arch/alpha/include/asm/machvec.h @@ -46,13 +46,15 @@ struct alpha_machine_vector void (*mv_pci_tbi)(struct pci_controller *hose, dma_addr_t start, dma_addr_t end); - unsigned int (*mv_ioread8)(const void __iomem *); - unsigned int (*mv_ioread16)(const void __iomem *); - unsigned int (*mv_ioread32)(const void __iomem *); + u8 (*mv_ioread8)(const void __iomem *); + u16 (*mv_ioread16)(const void __iomem *); + u32 (*mv_ioread32)(const void __iomem *); + u64 (*mv_ioread64)(const void __iomem *); void (*mv_iowrite8)(u8, void __iomem *); void (*mv_iowrite16)(u16, void __iomem *); void (*mv_iowrite32)(u32, void __iomem *); + void (*mv_iowrite64)(u64, void __iomem *); u8 (*mv_readb)(const volatile void __iomem *); u16 (*mv_readw)(const volatile void __iomem *); diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h index 43e234c518b1c05bf92a3fa452240a5ab6d67a09..714abe494e5fdc5996be1fd46d7b217916678902 100644 --- a/arch/alpha/include/asm/processor.h +++ b/arch/alpha/include/asm/processor.h @@ -36,8 +36,6 @@ extern void start_thread(struct pt_regs *, unsigned long, unsigned long); /* Free all resources held by a thread. */ struct task_struct; -extern void release_thread(struct task_struct *); - unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) diff --git a/arch/alpha/include/asm/termios.h b/arch/alpha/include/asm/termios.h deleted file mode 100644 index b7c77bb1bfd20368a8ff95a93d5493353e58023a..0000000000000000000000000000000000000000 --- a/arch/alpha/include/asm/termios.h +++ /dev/null @@ -1,87 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ALPHA_TERMIOS_H -#define _ALPHA_TERMIOS_H - -#include - -/* eof=^D eol=\0 eol2=\0 erase=del - werase=^W kill=^U reprint=^R sxtc=\0 - intr=^C quit=^\ susp=^Z - start=^Q stop=^S lnext=^V discard=^U - vmin=\1 vtime=\0 -*/ -#define INIT_C_CC "\004\000\000\177\027\025\022\000\003\034\032\000\021\023\026\025\001\000" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ - -#define user_termio_to_kernel_termios(a_termios, u_termio) \ -({ \ - struct ktermios *k_termios = (a_termios); \ - struct termio k_termio; \ - int canon, ret; \ - \ - ret = copy_from_user(&k_termio, u_termio, sizeof(k_termio)); \ - if (!ret) { \ - /* Overwrite only the low bits. */ \ - *(unsigned short *)&k_termios->c_iflag = k_termio.c_iflag; \ - *(unsigned short *)&k_termios->c_oflag = k_termio.c_oflag; \ - *(unsigned short *)&k_termios->c_cflag = k_termio.c_cflag; \ - *(unsigned short *)&k_termios->c_lflag = k_termio.c_lflag; \ - canon = k_termio.c_lflag & ICANON; \ - \ - k_termios->c_cc[VINTR] = k_termio.c_cc[_VINTR]; \ - k_termios->c_cc[VQUIT] = k_termio.c_cc[_VQUIT]; \ - k_termios->c_cc[VERASE] = k_termio.c_cc[_VERASE]; \ - k_termios->c_cc[VKILL] = k_termio.c_cc[_VKILL]; \ - k_termios->c_cc[VEOL2] = k_termio.c_cc[_VEOL2]; \ - k_termios->c_cc[VSWTC] = k_termio.c_cc[_VSWTC]; \ - k_termios->c_cc[canon ? VEOF : VMIN] = k_termio.c_cc[_VEOF]; \ - k_termios->c_cc[canon ? VEOL : VTIME] = k_termio.c_cc[_VEOL]; \ - } \ - ret; \ -}) - -/* - * Translate a "termios" structure into a "termio". Ugh. - * - * Note the "fun" _VMIN overloading. - */ -#define kernel_termios_to_user_termio(u_termio, a_termios) \ -({ \ - struct ktermios *k_termios = (a_termios); \ - struct termio k_termio; \ - int canon; \ - \ - k_termio.c_iflag = k_termios->c_iflag; \ - k_termio.c_oflag = k_termios->c_oflag; \ - k_termio.c_cflag = k_termios->c_cflag; \ - canon = (k_termio.c_lflag = k_termios->c_lflag) & ICANON; \ - \ - k_termio.c_line = k_termios->c_line; \ - k_termio.c_cc[_VINTR] = k_termios->c_cc[VINTR]; \ - k_termio.c_cc[_VQUIT] = k_termios->c_cc[VQUIT]; \ - k_termio.c_cc[_VERASE] = k_termios->c_cc[VERASE]; \ - k_termio.c_cc[_VKILL] = k_termios->c_cc[VKILL]; \ - k_termio.c_cc[_VEOF] = k_termios->c_cc[canon ? VEOF : VMIN]; \ - k_termio.c_cc[_VEOL] = k_termios->c_cc[canon ? VEOL : VTIME]; \ - k_termio.c_cc[_VEOL2] = k_termios->c_cc[VEOL2]; \ - k_termio.c_cc[_VSWTC] = k_termios->c_cc[VSWTC]; \ - \ - copy_to_user(u_termio, &k_termio, sizeof(k_termio)); \ -}) - -#define user_termios_to_kernel_termios(k, u) \ - copy_from_user(k, u, sizeof(struct termios2)) - -#define kernel_termios_to_user_termios(u, k) \ - copy_to_user(u, k, sizeof(struct termios2)) - -#define user_termios_to_kernel_termios_1(k, u) \ - copy_from_user(k, u, sizeof(struct termios)) - -#define kernel_termios_to_user_termios_1(u, k) \ - copy_to_user(u, k, sizeof(struct termios)) - -#endif /* _ALPHA_TERMIOS_H */ diff --git a/arch/alpha/include/uapi/asm/mman.h b/arch/alpha/include/uapi/asm/mman.h index 4aa996423b0d1666f0b9d3db6d9d457d00dacef5..763929e814e9a3faa76ee2a715494867210ca14e 100644 --- a/arch/alpha/include/uapi/asm/mman.h +++ b/arch/alpha/include/uapi/asm/mman.h @@ -76,6 +76,8 @@ #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/arch/alpha/kernel/Makefile b/arch/alpha/kernel/Makefile index 5a74581bf0ee7424fe9554ee76495c02f3d2a5a0..fb4efec7cbc74147fdd29159ce7d95d8289437d4 100644 --- a/arch/alpha/kernel/Makefile +++ b/arch/alpha/kernel/Makefile @@ -3,13 +3,13 @@ # Makefile for the linux kernel. # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds asflags-y := $(KBUILD_CFLAGS) ccflags-y := -Wno-sign-compare -obj-y := entry.o traps.o process.o osf_sys.o irq.o \ +obj-y := head.o entry.o traps.o process.o osf_sys.o irq.o \ irq_alpha.o signal.o setup.o ptrace.o time.o \ - systbls.o err_common.o io.o bugs.o + systbls.o err_common.o io.o bugs.o termios.o obj-$(CONFIG_VGA_HOSE) += console.o obj-$(CONFIG_SMP) += smp.o @@ -47,10 +47,6 @@ else # Misc support obj-$(CONFIG_ALPHA_SRM) += srmcons.o -ifdef CONFIG_BINFMT_AOUT -obj-y += binfmt_loader.o -endif - # Core logic support obj-$(CONFIG_ALPHA_APECS) += core_apecs.o obj-$(CONFIG_ALPHA_CIA) += core_cia.o diff --git a/arch/alpha/kernel/binfmt_loader.c b/arch/alpha/kernel/binfmt_loader.c deleted file mode 100644 index e4be7a543ecfd4dba7f50d2e7221a915a21897f9..0000000000000000000000000000000000000000 --- a/arch/alpha/kernel/binfmt_loader.c +++ /dev/null @@ -1,46 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include - -static int load_binary(struct linux_binprm *bprm) -{ - struct exec *eh = (struct exec *)bprm->buf; - unsigned long loader; - struct file *file; - int retval; - - if (eh->fh.f_magic != 0x183 || (eh->fh.f_flags & 0x3000) != 0x3000) - return -ENOEXEC; - - if (bprm->loader) - return -ENOEXEC; - - loader = bprm->vma->vm_end - sizeof(void *); - - file = open_exec("/sbin/loader"); - retval = PTR_ERR(file); - if (IS_ERR(file)) - return retval; - - /* Remember if the application is TASO. */ - bprm->taso = eh->ah.entry < 0x100000000UL; - - bprm->interpreter = file; - bprm->loader = loader; - return 0; -} - -static struct linux_binfmt loader_format = { - .load_binary = load_binary, -}; - -static int __init init_loader_binfmt(void) -{ - insert_binfmt(&loader_format); - return 0; -} -arch_initcall(init_loader_binfmt); diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c index 1efca79ac83caa5d6916f7fadfd50838e9976679..e9348aec464990517a5e13f5f16fdbdfaef4eb9b 100644 --- a/arch/alpha/kernel/core_marvel.c +++ b/arch/alpha/kernel/core_marvel.c @@ -803,7 +803,7 @@ void __iomem *marvel_ioportmap (unsigned long addr) return (void __iomem *)addr; } -unsigned int +u8 marvel_ioread8(const void __iomem *xaddr) { unsigned long addr = (unsigned long) xaddr; diff --git a/arch/alpha/kernel/io.c b/arch/alpha/kernel/io.c index 838586abb1e02f8fbe5d8415d8f3730fdec5ad68..eda09778268f0199e7730090752aff17eb865dd4 100644 --- a/arch/alpha/kernel/io.c +++ b/arch/alpha/kernel/io.c @@ -41,6 +41,15 @@ unsigned int ioread32(const void __iomem *addr) return ret; } +u64 ioread64(const void __iomem *addr) +{ + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread64)(addr); + mb(); + return ret; +} + void iowrite8(u8 b, void __iomem *addr) { mb(); @@ -59,12 +68,20 @@ void iowrite32(u32 b, void __iomem *addr) IO_CONCAT(__IO_PREFIX,iowrite32)(b, addr); } +void iowrite64(u64 b, void __iomem *addr) +{ + mb(); + IO_CONCAT(__IO_PREFIX,iowrite64)(b, addr); +} + EXPORT_SYMBOL(ioread8); EXPORT_SYMBOL(ioread16); EXPORT_SYMBOL(ioread32); +EXPORT_SYMBOL(ioread64); EXPORT_SYMBOL(iowrite8); EXPORT_SYMBOL(iowrite16); EXPORT_SYMBOL(iowrite32); +EXPORT_SYMBOL(iowrite64); u8 inb(unsigned long port) { diff --git a/arch/alpha/kernel/machvec_impl.h b/arch/alpha/kernel/machvec_impl.h index 393d5d6ca5d28ae85151d82285e04220543fa18d..c2ebcb39e58967be49ac6ad8bb7b8646156ce659 100644 --- a/arch/alpha/kernel/machvec_impl.h +++ b/arch/alpha/kernel/machvec_impl.h @@ -78,9 +78,11 @@ .mv_ioread8 = CAT(low,_ioread8), \ .mv_ioread16 = CAT(low,_ioread16), \ .mv_ioread32 = CAT(low,_ioread32), \ + .mv_ioread64 = CAT(low,_ioread64), \ .mv_iowrite8 = CAT(low,_iowrite8), \ .mv_iowrite16 = CAT(low,_iowrite16), \ .mv_iowrite32 = CAT(low,_iowrite32), \ + .mv_iowrite64 = CAT(low,_iowrite64), \ .mv_readb = CAT(low,_readb), \ .mv_readw = CAT(low,_readw), \ .mv_readl = CAT(low,_readl), \ diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index d257293401e28ada112400d957765ca0891c4b29..c54469b369cb67a2fe8d7e94417107c7a658668d 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -108,7 +108,7 @@ struct osf_dirent_callback { int error; }; -static int +static bool osf_filldir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { @@ -120,11 +120,11 @@ osf_filldir(struct dir_context *ctx, const char *name, int namlen, buf->error = -EINVAL; /* only used if we fail */ if (reclen > buf->count) - return -EINVAL; + return false; d_ino = ino; if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { buf->error = -EOVERFLOW; - return -EOVERFLOW; + return false; } if (buf->basep) { if (put_user(offset, buf->basep)) @@ -141,10 +141,10 @@ osf_filldir(struct dir_context *ctx, const char *name, int namlen, dirent = (void __user *)dirent + reclen; buf->dirent = dirent; buf->count -= reclen; - return 0; + return true; Efault: buf->error = -EFAULT; - return -EFAULT; + return false; } SYSCALL_DEFINE4(osf_getdirentries, unsigned int, fd, @@ -1278,48 +1278,6 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, return addr; } -#ifdef CONFIG_OSF4_COMPAT -/* Clear top 32 bits of iov_len in the user's buffer for - compatibility with old versions of OSF/1 where iov_len - was defined as int. */ -static int -osf_fix_iov_len(const struct iovec __user *iov, unsigned long count) -{ - unsigned long i; - - for (i = 0 ; i < count ; i++) { - int __user *iov_len_high = (int __user *)&iov[i].iov_len + 1; - - if (put_user(0, iov_len_high)) - return -EFAULT; - } - return 0; -} -#endif - -SYSCALL_DEFINE3(osf_readv, unsigned long, fd, - const struct iovec __user *, vector, unsigned long, count) -{ -#ifdef CONFIG_OSF4_COMPAT - if (unlikely(personality(current->personality) == PER_OSF4)) - if (osf_fix_iov_len(vector, count)) - return -EFAULT; -#endif - - return sys_readv(fd, vector, count); -} - -SYSCALL_DEFINE3(osf_writev, unsigned long, fd, - const struct iovec __user *, vector, unsigned long, count) -{ -#ifdef CONFIG_OSF4_COMPAT - if (unlikely(personality(current->personality) == PER_OSF4)) - if (osf_fix_iov_len(vector, count)) - return -EFAULT; -#endif - return sys_writev(fd, vector, count); -} - SYSCALL_DEFINE2(osf_getpriority, int, which, int, who) { int prio = sys_getpriority(which, who); diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index e2e25f8b5e76c7285ec07c704b8c4887b5f3718b..dbf1bc5e2ad25d3441acdc04ed90200abc93d57e 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -225,11 +225,6 @@ flush_thread(void) current_thread_info()->pcb.unique = 0; } -void -release_thread(struct task_struct *dead_task) -{ -} - /* * Copy architecture-specific thread state */ diff --git a/arch/alpha/kernel/setup.c b/arch/alpha/kernel/setup.c index b4fbbba30aa2bdf311c49069b359982f92c8f013..33bf3a62700270ac732f3b537c0992515041f946 100644 --- a/arch/alpha/kernel/setup.c +++ b/arch/alpha/kernel/setup.c @@ -491,9 +491,9 @@ setup_arch(char **cmdline_p) boot flags depending on the boot mode, we need some shorthand. This should do for installation. */ if (strcmp(COMMAND_LINE, "INSTALL") == 0) { - strlcpy(command_line, "root=/dev/fd0 load_ramdisk=1", sizeof command_line); + strscpy(command_line, "root=/dev/fd0 load_ramdisk=1", sizeof(command_line)); } else { - strlcpy(command_line, COMMAND_LINE, sizeof command_line); + strscpy(command_line, COMMAND_LINE, sizeof(command_line)); } strcpy(boot_command_line, command_line); *cmdline_p = command_line; diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index 3515bc4f16a4ff285dcfd4c4540e7709a1d43c6a..8ebacf37a8cf4ac3050e3e2ae821245d7074cdea 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -125,8 +125,8 @@ 116 common osf_gettimeofday sys_osf_gettimeofday 117 common osf_getrusage sys_osf_getrusage 118 common getsockopt sys_getsockopt -120 common readv sys_osf_readv -121 common writev sys_osf_writev +120 common readv sys_readv +121 common writev sys_writev 122 common osf_settimeofday sys_osf_settimeofday 123 common fchown sys_fchown 124 common fchmod sys_fchmod diff --git a/arch/alpha/kernel/termios.c b/arch/alpha/kernel/termios.c new file mode 100644 index 0000000000000000000000000000000000000000..a4c29a22edf7e638d46accbae3b98c046aa1a3b5 --- /dev/null +++ b/arch/alpha/kernel/termios.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +int user_termio_to_kernel_termios(struct ktermios *termios, + struct termio __user *termio) +{ + struct termio v; + bool canon; + + if (copy_from_user(&v, termio, sizeof(struct termio))) + return -EFAULT; + + termios->c_iflag = (0xffff0000 & termios->c_iflag) | v.c_iflag; + termios->c_oflag = (0xffff0000 & termios->c_oflag) | v.c_oflag; + termios->c_cflag = (0xffff0000 & termios->c_cflag) | v.c_cflag; + termios->c_lflag = (0xffff0000 & termios->c_lflag) | v.c_lflag; + termios->c_line = (0xffff0000 & termios->c_lflag) | v.c_line; + + canon = v.c_lflag & ICANON; + termios->c_cc[VINTR] = v.c_cc[_VINTR]; + termios->c_cc[VQUIT] = v.c_cc[_VQUIT]; + termios->c_cc[VERASE] = v.c_cc[_VERASE]; + termios->c_cc[VKILL] = v.c_cc[_VKILL]; + termios->c_cc[VEOL2] = v.c_cc[_VEOL2]; + termios->c_cc[VSWTC] = v.c_cc[_VSWTC]; + termios->c_cc[canon ? VEOF : VMIN] = v.c_cc[_VEOF]; + termios->c_cc[canon ? VEOL : VTIME] = v.c_cc[_VEOL]; + + return 0; +} + +int kernel_termios_to_user_termio(struct termio __user *termio, + struct ktermios *termios) +{ + struct termio v; + bool canon; + + memset(&v, 0, sizeof(struct termio)); + v.c_iflag = termios->c_iflag; + v.c_oflag = termios->c_oflag; + v.c_cflag = termios->c_cflag; + v.c_lflag = termios->c_lflag; + v.c_line = termios->c_line; + + canon = v.c_lflag & ICANON; + v.c_cc[_VINTR] = termios->c_cc[VINTR]; + v.c_cc[_VQUIT] = termios->c_cc[VQUIT]; + v.c_cc[_VERASE] = termios->c_cc[VERASE]; + v.c_cc[_VKILL] = termios->c_cc[VKILL]; + v.c_cc[_VEOF] = termios->c_cc[canon ? VEOF : VMIN]; + v.c_cc[_VEOL] = termios->c_cc[canon ? VEOL : VTIME]; + v.c_cc[_VEOL2] = termios->c_cc[VEOL2]; + v.c_cc[_VSWTC] = termios->c_cc[VSWTC]; + + return copy_to_user(termio, &v, sizeof(struct termio)); +} diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 9e3653253ef20d5304ac8b2ce4c08f11edc7b88c..d9a13ccf89a3aa5d6a4fb5dc8158ba2e4dba4671 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -554,7 +554,7 @@ config ARC_BUILTIN_DTB_NAME endmenu # "ARC Architecture Configuration" -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" default "12" if ARC_HUGEPAGE_16M default "11" diff --git a/arch/arc/Makefile b/arch/arc/Makefile index efc54f3e35e07407777aab00356f52f873bf335f..329400a1c35598187d62faca9e84f3f286985715 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -82,8 +82,6 @@ KBUILD_CFLAGS += $(cflags-y) KBUILD_AFLAGS += $(KBUILD_CFLAGS) KBUILD_LDFLAGS += $(ldflags-y) -head-y := arch/arc/kernel/head.o - # w/o this dtb won't embed into kernel binary core-y += arch/arc/boot/dts/ diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index d93b65008d4afdc93f5826c993bbfe0816a0c732..4a94d1684ed604c80fb6c7a6fdba09e9301c3103 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -90,7 +90,7 @@ CONFIG_TMPFS=y CONFIG_CONFIGFS_FS=y # CONFIG_MISC_FILESYSTEMS is not set # CONFIG_NETWORK_FILESYSTEMS is not set -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_HEADERS_INSTALL=y diff --git a/arch/arc/include/asm/processor.h b/arch/arc/include/asm/processor.h index 54db9d7bb562d3f9acb697a89610568076b9b4f9..fb844fce1ab671552e4dc3a7dd0384da4c6e6abf 100644 --- a/arch/arc/include/asm/processor.h +++ b/arch/arc/include/asm/processor.h @@ -43,9 +43,6 @@ struct task_struct; #define task_pt_regs(p) \ ((struct pt_regs *)(THREAD_SIZE + (void *)task_stack_page(p)) - 1) -/* Free all resources held by a thread */ -#define release_thread(thread) do { } while (0) - /* * A lot of busy-wait loops in SMP are based off of non-volatile data otherwise * get optimised away by gcc diff --git a/arch/arc/kernel/Makefile b/arch/arc/kernel/Makefile index 8c4fc4b54c1446b3bb41d4b035c405576ecfc53f..0723d888ac446a30c7355b9737b3070bb9900fa2 100644 --- a/arch/arc/kernel/Makefile +++ b/arch/arc/kernel/Makefile @@ -3,7 +3,7 @@ # Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) # -obj-y := arcksyms.o setup.o irq.o reset.o ptrace.o process.o devtree.o +obj-y := head.o arcksyms.o setup.o irq.o reset.o ptrace.o process.o devtree.o obj-y += signal.o traps.o sys.o troubleshoot.o stacktrace.o disasm.o obj-$(CONFIG_ISA_ARCOMPACT) += entry-compact.o intc-compact.o obj-$(CONFIG_ISA_ARCV2) += entry-arcv2.o intc-arcv2.o @@ -31,4 +31,4 @@ else obj-y += ctx_sw_asm.o endif -extra-y := vmlinux.lds head.o +extra-y := vmlinux.lds diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 87badeae3181ea7ffd7aaca09fd12e8b3cf456b6..a08c9d092a332d8e6e3c63cd824f31ef061ac32a 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -28,7 +28,6 @@ config ARM select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT - select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7 select ARCH_SUPPORTS_ATOMIC_RMW @@ -42,6 +41,7 @@ config ARM select ARCH_WANT_LD_ORPHAN_WARN select BINFMT_FLAT_ARGVP_ENVP_ON_STACK select BUILDTIME_TABLE_SORT if MMU + select COMMON_CLK if !(ARCH_RPC || ARCH_FOOTBRIDGE) select CLONE_BACKWARDS select CPU_PM if SUSPEND || CPU_IDLE select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS @@ -110,6 +110,7 @@ config ARM select HAVE_MOD_ARCH_SPECIFIC select HAVE_NMI select HAVE_OPTPROBES if !THUMB2_KERNEL + select HAVE_PCI if MMU select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -126,13 +127,17 @@ config ARM select OF_EARLY_FLATTREE if OF select OLD_SIGACTION select OLD_SIGSUSPEND3 + select PCI_DOMAINS_GENERIC if PCI select PCI_SYSCALL if PCI select PERF_USE_VMALLOC select RTC_LIB + select SPARSE_IRQ if !(ARCH_FOOTBRIDGE || ARCH_RPC) select SYS_SUPPORTS_APM_EMULATION select THREAD_INFO_IN_TASK + select TIMER_OF if OF select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M + select USE_OF if !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100) # Above selects are sorted alphabetically; please add new ones # according to that. Thanks. help @@ -154,12 +159,8 @@ config ARM_HAS_GROUP_RELOCS which is usually sufficient, but not for allyesconfig, so we disable this feature when doing compile testing. -config ARM_HAS_SG_CHAIN - bool - config ARM_DMA_USE_IOMMU bool - select ARM_HAS_SG_CHAIN select NEED_SG_DMA_LENGTH if ARM_DMA_USE_IOMMU @@ -245,7 +246,7 @@ config ARCH_MTD_XIP config ARM_PATCH_PHYS_VIRT bool "Patch physical to virtual translations at runtime" if EMBEDDED default y - depends on !XIP_KERNEL && MMU + depends on MMU help Patch phys-to-virt and virt-to-phys translation functions at boot and module load time according to the position of the @@ -274,7 +275,7 @@ config NEED_MACH_MEMORY_H config PHYS_OFFSET hex "Physical address of main memory" if MMU - depends on !ARM_PATCH_PHYS_VIRT + depends on !ARM_PATCH_PHYS_VIRT || !AUTO_ZRELADDR default DRAM_BASE if !MMU default 0x00000000 if ARCH_FOOTBRIDGE default 0x10000000 if ARCH_OMAP1 || ARCH_RPC @@ -307,13 +308,8 @@ config MMU config ARM_SINGLE_ARMV7M def_bool !MMU select ARM_NVIC - select AUTO_ZRELADDR - select TIMER_OF - select COMMON_CLK select CPU_V7M select NO_IOPORT_MAP - select SPARSE_IRQ - select USE_OF config ARCH_MMAP_RND_BITS_MIN default 8 @@ -323,94 +319,31 @@ config ARCH_MMAP_RND_BITS_MAX default 15 if PAGE_OFFSET=0x80000000 default 16 -# -# The "ARM system type" choice list is ordered alphabetically by option -# text. Please add new entries in the option alphabetic order. -# -choice - prompt "ARM system type" - depends on MMU - default ARCH_MULTIPLATFORM - config ARCH_MULTIPLATFORM - bool "Allow multiple platforms to be selected" - select ARCH_FLATMEM_ENABLE - select ARCH_SPARSEMEM_ENABLE - select ARCH_SELECT_MEMORY_MODEL - select ARM_HAS_SG_CHAIN - select ARM_PATCH_PHYS_VIRT - select AUTO_ZRELADDR - select TIMER_OF - select COMMON_CLK - select HAVE_PCI - select PCI_DOMAINS_GENERIC if PCI - select SPARSE_IRQ - select USE_OF - -config ARCH_FOOTBRIDGE - bool "FootBridge" - depends on CPU_LITTLE_ENDIAN - depends on ATAGS - select CPU_SA110 - select FOOTBRIDGE - select NEED_MACH_MEMORY_H + bool "Require kernel to be portable to multiple machines" if EXPERT + depends on MMU && !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100) + default y help - Support for systems based on the DC21285 companion chip - ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. + In general, all Arm machines can be supported in a single + kernel image, covering either Armv4/v5 or Armv6/v7. -config ARCH_RPC - bool "RiscPC" - depends on !CC_IS_CLANG && GCC_VERSION < 90100 && GCC_VERSION >= 60000 - depends on CPU_LITTLE_ENDIAN - depends on ATAGS - select ARCH_ACORN - select ARCH_MAY_HAVE_PC_FDC - select ARCH_SPARSEMEM_ENABLE - select ARM_HAS_SG_CHAIN - select CPU_SA110 - select FIQ - select HAVE_PATA_PLATFORM - select ISA_DMA_API - select LEGACY_TIMER_TICK - select NEED_MACH_IO_H - select NEED_MACH_MEMORY_H - select NO_IOPORT_MAP - help - On the Acorn Risc-PC, Linux can support the internal IDE disk and - CD-ROM interface, serial and parallel port, and the floppy drive. + However, some configuration options require hardcoding machine + specific physical addresses or enable errata workarounds that may + break other machines. -config ARCH_SA1100 - bool "SA1100-based" - depends on CPU_LITTLE_ENDIAN - depends on ATAGS - select ARCH_MTD_XIP - select ARCH_SPARSEMEM_ENABLE - select CLKSRC_MMIO - select CLKSRC_PXA - select TIMER_OF if OF - select COMMON_CLK - select CPU_FREQ - select CPU_SA1100 - select GPIOLIB - select IRQ_DOMAIN - select ISA - select NEED_MACH_MEMORY_H - select SPARSE_IRQ - help - Support for StrongARM 11x0 based boards. + Selecting N here allows using those options, including + DEBUG_UNCOMPRESS, XIP_KERNEL and ZBOOT_ROM. If unsure, say Y. -endchoice - -menu "Multiple platform selection" - depends on ARCH_MULTIPLATFORM +menu "Platform selection" + depends on MMU comment "CPU Core family selection" config ARCH_MULTI_V4 - bool "ARMv4 based platforms (FA526)" + bool "ARMv4 based platforms (FA526, StrongARM)" depends on !ARCH_MULTI_V6_V7 select ARCH_MULTI_V4_V5 - select CPU_FA526 + select CPU_FA526 if !(CPU_SA110 || CPU_SA1100) config ARCH_MULTI_V4T bool "ARMv4T based platforms (ARM720T, ARM920T, ...)" @@ -472,7 +405,6 @@ config ARCH_AIROHA select ARM_GIC_V3 select ARM_PSCI select HAVE_ARM_ARCH_TIMER - select COMMON_CLK help Support for Airoha EN7523 SoCs @@ -573,6 +505,8 @@ source "arch/arm/mach-rda/Kconfig" source "arch/arm/mach-realtek/Kconfig" +source "arch/arm/mach-rpc/Kconfig" + source "arch/arm/mach-rockchip/Kconfig" source "arch/arm/mach-s3c/Kconfig" @@ -638,7 +572,6 @@ config ARCH_ACORN config PLAT_ORION bool select CLKSRC_MMIO - select COMMON_CLK select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -989,11 +922,6 @@ config ISA (MCA) or VESA. ISA is an older system, now being displaced by PCI; newer boards don't support it. If you have ISA, say Y, otherwise N. -# Select ISA DMA controller support -config ISA_DMA - bool - select ISA_DMA_API - # Select ISA DMA interface config ISA_DMA_API bool @@ -1054,7 +982,7 @@ config SMP config SMP_ON_UP bool "Allow booting SMP kernel on uniprocessor systems" - depends on SMP && !XIP_KERNEL && MMU + depends on SMP && MMU default y help SMP kernels contain instructions which fail on non-SMP processors. @@ -1303,7 +1231,7 @@ config THUMB2_KERNEL config ARM_PATCH_IDIV bool "Runtime patch udiv/sdiv instructions into __aeabi_{u}idiv()" - depends on CPU_32v7 && !XIP_KERNEL + depends on CPU_32v7 default y help The ARM compiler inserts calls to __aeabi_idiv() and @@ -1358,13 +1286,13 @@ config OABI_COMPAT at all). If in doubt say N. config ARCH_SELECT_MEMORY_MODEL - bool + def_bool y config ARCH_FLATMEM_ENABLE - bool + def_bool !(ARCH_RPC || ARCH_SA1100) config ARCH_SPARSEMEM_ENABLE - bool + def_bool !ARCH_FOOTBRIDGE select SPARSEMEM_STATIC if SPARSEMEM config HIGHMEM @@ -1434,7 +1362,7 @@ config ARM_MODULE_PLTS Disabling this is usually safe for small single-platform configurations. If unsure, say y. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" default "12" if SOC_AM33XX default "9" if SA1111 @@ -1671,7 +1599,6 @@ config CMDLINE choice prompt "Kernel command line type" if CMDLINE != "" default CMDLINE_FROM_BOOTLOADER - depends on ATAGS config CMDLINE_FROM_BOOTLOADER bool "Use bootloader kernel arguments if available" @@ -1698,6 +1625,7 @@ endchoice config XIP_KERNEL bool "Kernel Execute-In-Place from ROM" depends on !ARM_LPAE && !ARCH_MULTIPLATFORM + depends on !ARM_PATCH_IDIV && !ARM_PATCH_PHYS_VIRT && !SMP_ON_UP help Execute-In-Place allows the kernel to run from non-volatile storage directly addressable by the CPU, such as NOR flash. This saves RAM @@ -1772,7 +1700,8 @@ config CRASH_DUMP For more details see Documentation/admin-guide/kdump/kdump.rst config AUTO_ZRELADDR - bool "Auto calculation of the decompressed kernel image address" + bool "Auto calculation of the decompressed kernel image address" if !ARCH_MULTIPLATFORM + default !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100) help ZRELADDR is the physical address where the decompressed kernel image will be placed. If AUTO_ZRELADDR is selected, the address @@ -1921,8 +1850,4 @@ config ARCH_HIBERNATION_POSSIBLE endmenu -if CRYPTO -source "arch/arm/crypto/Kconfig" -endif - source "arch/arm/Kconfig.assembler" diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index 792796a348c3287e2a2ac2b10803b8b1a4639686..c345775f035bc4c3c3d86a1a3591554529d5d509 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1904,7 +1904,8 @@ config DEBUG_UART_8250_PALMCHIP config DEBUG_UNCOMPRESS bool "Enable decompressor debugging via DEBUG_LL output" - depends on ARCH_MULTIPLATFORM || PLAT_SAMSUNG || ARM_SINGLE_ARMV7M + depends on !ARCH_MULTIPLATFORM + depends on !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100) depends on DEBUG_LL && !DEBUG_OMAP2PLUS_UART && \ (!DEBUG_TEGRA_UART || !ZBOOT_ROM) && \ !DEBUG_BRCMSTB_UART && !DEBUG_SEMIHOSTING @@ -1921,9 +1922,8 @@ config DEBUG_UNCOMPRESS config UNCOMPRESS_INCLUDE string - default "debug/uncompress.h" if ARCH_MULTIPLATFORM || ARCH_MSM || \ - PLAT_SAMSUNG || ARM_SINGLE_ARMV7M - default "mach/uncompress.h" + default "mach/uncompress.h" if ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100 + default "debug/uncompress.h" config EARLY_PRINTK bool "Early printk" diff --git a/arch/arm/Makefile b/arch/arm/Makefile index 56f655deebb14c38beddacfa713b27933c4c7b8c..c846119c448f0425a4ebee33c7983b89d3a1225d 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -22,6 +22,9 @@ GZFLAGS :=-9 # Never generate .eh_frame KBUILD_CFLAGS += $(call cc-option,-fno-dwarf2-cfi-asm) +# Disable FDPIC ABI +KBUILD_CFLAGS += $(call cc-option,-mno-fdpic) + # This should work on most of the modern platforms KBUILD_DEFCONFIG := multi_v7_defconfig @@ -134,9 +137,6 @@ KBUILD_AFLAGS +=$(CFLAGS_ABI) $(AFLAGS_ISA) $(arch-y) $(tune-y) -include asm/uni CHECKFLAGS += -D__arm__ -#Default value -head-y := arch/arm/kernel/head$(MMUEXT).o - # Text offset. This list is sorted numerically by address in order to # provide a means to avoid/resolve conflicts in multi-arch kernels. # Note: the 32kB below this value is reserved for use by the kernel @@ -224,40 +224,24 @@ machine-$(CONFIG_ARCH_ZYNQ) += zynq machine-$(CONFIG_PLAT_VERSATILE) += versatile machine-$(CONFIG_PLAT_SPEAR) += spear -# Platform directory name. This list is sorted alphanumerically -# by CONFIG_* macro name. -plat-$(CONFIG_PLAT_ORION) += orion +# legacy platforms provide their own mach/*.h headers globally, +# these three are mutually exclusive +machdirs-$(CONFIG_ARCH_FOOTBRIDGE) += arch/arm/mach-footbridge +machdirs-$(CONFIG_ARCH_RPC) += arch/arm/mach-rpc +machdirs-$(CONFIG_ARCH_SA1100) += arch/arm/mach-sa1100 +KBUILD_CPPFLAGS += $(patsubst %,-I$(srctree)/%/include,$(machdirs-y)) # The byte offset of the kernel image in RAM from the start of RAM. TEXT_OFFSET := $(textofs-y) -# The first directory contains additional information for the boot setup code -ifneq ($(machine-y),) -MACHINE := arch/arm/mach-$(word 1,$(machine-y))/ -else -MACHINE := -endif -ifeq ($(CONFIG_ARCH_MULTIPLATFORM),y) -MACHINE := -endif - -machdirs := $(patsubst %,arch/arm/mach-%/,$(machine-y)) -platdirs := $(patsubst %,arch/arm/plat-%/,$(sort $(plat-y))) - -ifneq ($(CONFIG_ARCH_MULTIPLATFORM),y) -ifneq ($(CONFIG_ARM_SINGLE_ARMV7M),y) -KBUILD_CPPFLAGS += $(patsubst %,-I$(srctree)/%include,$(machdirs) $(platdirs)) -endif -endif - export TEXT_OFFSET GZFLAGS MMUEXT # If we have a machine-specific directory, then include it in the build. -core-y += $(machdirs) $(platdirs) - +core-y += $(patsubst %,arch/arm/mach-%/,$(machine-y)) # For cleaning -core- += $(patsubst %,arch/arm/mach-%/, $(machine-)) -core- += $(patsubst %,arch/arm/plat-%/, $(plat-)) +core- += $(patsubst %,arch/arm/mach-%/,$(machine-)) + +core-$(CONFIG_PLAT_ORION) += arch/arm/plat-orion/ libs-y := arch/arm/lib/ $(libs-y) @@ -310,7 +294,7 @@ bootpImage uImage: zImage zImage: Image $(BOOT_TARGETS): vmlinux - $(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/$@ + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ @$(kecho) ' Kernel: $(boot)/$@ is ready' $(INSTALL_TARGETS): KBUILD_IMAGE = $(boot)/$(patsubst %install,%Image,$@) @@ -324,7 +308,7 @@ ifeq ($(CONFIG_VDSO),y) endif # My testing targets (bypasses dependencies) -bp:; $(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/bootpImage +bp:; $(Q)$(MAKE) $(build)=$(boot) $(boot)/bootpImage define archhelp diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index 54a09f9464fbd9f6b65698b0b33d558a1ca9930c..abd6a2889fd0e580d8cfcb801b0cc31dd11cb2b0 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -10,22 +10,16 @@ # # Copyright (C) 1995-2002 Russell King # - OBJCOPYFLAGS :=-O binary -R .comment -S -ifneq ($(MACHINE),) -include $(MACHINE)/Makefile.boot -endif - -# Note: the following conditions must always be true: # ZRELADDR == virt_to_phys(PAGE_OFFSET + TEXT_OFFSET) -# PARAMS_PHYS must be within 4MB of ZRELADDR -# INITRD_PHYS must be in RAM -ZRELADDR := $(zreladdr-y) -PARAMS_PHYS := $(params_phys-y) -INITRD_PHYS := $(initrd_phys-y) +ifdef CONFIG_PHYS_OFFSET +add_hex = $(shell printf 0x%x $$(( $(1) + $(2) )) ) +ZRELADDR := $(call add_hex, $(CONFIG_PHYS_OFFSET), $(TEXT_OFFSET)) +endif -export ZRELADDR INITRD_PHYS PARAMS_PHYS +PHYS_OFFSET := $(CONFIG_PHYS_OFFSET) +export ZRELADDR PARAMS_PHYS PHYS_OFFSET targets := Image zImage xipImage bootpImage uImage @@ -90,17 +84,10 @@ $(obj)/uImage: $(obj)/zImage FORCE @$(check_for_multiple_loadaddr) $(call if_changed,uimage) -$(obj)/bootp/bootp: $(obj)/zImage initrd FORCE +$(obj)/bootp/bootp: $(obj)/zImage FORCE $(Q)$(MAKE) $(build)=$(obj)/bootp $@ $(obj)/bootpImage: $(obj)/bootp/bootp FORCE $(call if_changed,objcopy) -PHONY += initrd -initrd: - @test "$(INITRD_PHYS)" != "" || \ - (echo This machine does not support INITRD; exit -1) - @test "$(INITRD)" != "" || \ - (echo You must specify INITRD; exit -1) - subdir- := bootp compressed dts diff --git a/arch/arm/boot/bootp/Makefile b/arch/arm/boot/bootp/Makefile index 981a8d03f064c24f47d7fa69294a5e14a3805fa4..a2934e6fd89aa09af038be219a8f1e697eb15a1b 100644 --- a/arch/arm/boot/bootp/Makefile +++ b/arch/arm/boot/bootp/Makefile @@ -5,9 +5,40 @@ # This file is included by the global makefile so that you can add your own # architecture-specific flags and dependencies. # - GCOV_PROFILE := n +ifdef PHYS_OFFSET +add_hex = $(shell printf 0x%x $$(( $(1) + $(2) )) ) + +# If PHYS_OFFSET is set, INITRD_PHYS and PARAMS_PHYS can be derived, +# otherwise they must be passed on the command line. +# +# Note: the following conditions must always be true: +# PARAMS_PHYS must be within 4MB of ZRELADDR +# INITRD_PHYS must be in RAM + +PARAMS_PHYS := $(call add_hex, $(PHYS_OFFSET), 0x100) + +# guess an initrd location if possible +initrd_offset-$(CONFIG_ARCH_FOOTBRIDGE) += 0x00800000 +initrd_offset-$(CONFIG_ARCH_SA1100) += 0x00800000 +initrd_offset-$(CONFIG_ARCH_RPC) += 0x08000000 +INITRD_OFFSET := $(initrd_offset-y) +ifdef INITRD_OFFSET +INITRD_PHYS := $(call add_hex, $(PHYS_OFFSET), $(INITRD_OFFSET)) +endif + +endif + +PHONY += initrd +initrd: + @test "$(PARAMS_PHYS)" != "" || \ + (echo bootpImage: You must specify PHYS_OFFSET of PARAMS_PHYS ; exit -1) + @test "$(INITRD_PHYS)" != "" || \ + (echo bootpImage: You must specify INITRD_OFFSET or INITRD_PHYS ; exit -1) + @test "$(INITRD)" != "" || \ + (echo bootpImage: You must specify INITRD; exit -1) + LDFLAGS_bootp := --no-undefined -X \ --defsym initrd_phys=$(INITRD_PHYS) \ --defsym params_phys=$(PARAMS_PHYS) -T @@ -24,6 +55,6 @@ $(obj)/bootp: $(src)/bootp.lds $(addprefix $(obj)/,init.o kernel.o initrd.o) FOR $(obj)/kernel.o: arch/arm/boot/zImage FORCE -$(obj)/initrd.o: $(INITRD) FORCE +$(obj)/initrd.o: initrd $(INITRD) FORCE PHONY += $(INITRD) diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index bf79f2f78d2323068ccc223ff7fe420b5ee070d2..9f406e9c0ea6f74bc2d8fd71a7625977de891691 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -67,11 +67,7 @@ #if defined(CONFIG_ARCH_SA1100) .macro loadsp, rb, tmp1, tmp2 mov \rb, #0x80000000 @ physical base address -#ifdef CONFIG_DEBUG_LL_SER3 - add \rb, \rb, #0x00050000 @ Ser3 -#else add \rb, \rb, #0x00010000 @ Ser1 -#endif .endm #else .macro loadsp, rb, tmp1, tmp2 diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c index cb2e069dc73fd0448454d78da92943f761365821..abfed1aa2baa81381f3e6d73cbb8aa65fdd384e8 100644 --- a/arch/arm/boot/compressed/misc.c +++ b/arch/arm/boot/compressed/misc.c @@ -23,7 +23,9 @@ unsigned int __machine_arch_type; #include #include #include "misc.h" +#ifdef CONFIG_ARCH_EP93XX #include "misc-ep93xx.h" +#endif static void putstr(const char *ptr); diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S index 1bcb68ac4b0118a0dc674603bdb2a2b541f64b61..3fcb3e62dc5692c5ea8ccdb47d97c169d7b0ac76 100644 --- a/arch/arm/boot/compressed/vmlinux.lds.S +++ b/arch/arm/boot/compressed/vmlinux.lds.S @@ -23,6 +23,7 @@ SECTIONS *(.ARM.extab*) *(.note.*) *(.rel.*) + *(.printk_index) /* * Discard any r/w data - this produces a link error if we have any, * which is required for PIC decompression. Local data generates @@ -57,6 +58,7 @@ SECTIONS *(.rodata) *(.rodata.*) *(.data.rel.ro) + *(.data.rel.ro.*) } .piggydata : { *(.piggydata) diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 05d8aef6e5d252b9cb2be1d23fc47b282cf16258..6aa7dc4db2fc8affeed2f5c201d56e7e097ecc8a 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -61,6 +61,7 @@ dtb-$(CONFIG_SOC_SAM_V7) += \ at91-sama5d2_icp.dtb \ at91-sama5d2_ptc_ek.dtb \ at91-sama5d2_xplained.dtb \ + at91-sama5d3_eds.dtb \ at91-sama5d3_ksz9477_evb.dtb \ at91-sama5d3_xplained.dtb \ at91-dvk_som60.dtb \ @@ -706,8 +707,8 @@ dtb-$(CONFIG_SOC_IMX6UL) += \ imx6ul-geam.dtb \ imx6ul-isiot-emmc.dtb \ imx6ul-isiot-nand.dtb \ - imx6ul-kontron-n6310-s.dtb \ - imx6ul-kontron-n6310-s-43.dtb \ + imx6ul-kontron-bl.dtb \ + imx6ul-kontron-bl-43.dtb \ imx6ul-liteboard.dtb \ imx6ul-tqma6ul1-mba6ulx.dtb \ imx6ul-tqma6ul2-mba6ulx.dtb \ @@ -736,6 +737,7 @@ dtb-$(CONFIG_SOC_IMX6UL) += \ imx6ull-colibri-wifi-iris.dtb \ imx6ull-colibri-wifi-iris-v2.dtb \ imx6ull-jozacp.dtb \ + imx6ull-kontron-bl.dtb \ imx6ull-myir-mys-6ulx-eval.dtb \ imx6ull-opos6uldev.dtb \ imx6ull-phytec-segin-ff-rdk-nand.dtb \ @@ -788,6 +790,7 @@ dtb-$(CONFIG_SOC_IMXRT) += \ dtb-$(CONFIG_SOC_LAN966) += \ lan966x-kontron-kswitch-d10-mmt-6g-2gs.dtb \ lan966x-kontron-kswitch-d10-mmt-8g.dtb \ + lan966x-pcb8290.dtb \ lan966x-pcb8291.dtb \ lan966x-pcb8309.dtb dtb-$(CONFIG_SOC_LS1021A) += \ @@ -1047,6 +1050,9 @@ dtb-$(CONFIG_ARCH_QCOM) += \ qcom-ipq8064-rb3011.dtb \ qcom-msm8226-samsung-s3ve3g.dtb \ qcom-msm8660-surf.dtb \ + qcom-msm8916-samsung-e5.dtb \ + qcom-msm8916-samsung-e7.dtb \ + qcom-msm8916-samsung-grandmax.dtb \ qcom-msm8916-samsung-serranove.dtb \ qcom-msm8960-cdp.dtb \ qcom-msm8974-lge-nexus5-hammerhead.dtb \ @@ -1574,8 +1580,10 @@ dtb-$(CONFIG_ARCH_ASPEED) += \ aspeed-ast2500-evb.dtb \ aspeed-ast2600-evb-a1.dtb \ aspeed-ast2600-evb.dtb \ + aspeed-bmc-amd-daytonax.dtb \ aspeed-bmc-amd-ethanolx.dtb \ aspeed-bmc-ampere-mtjade.dtb \ + aspeed-bmc-ampere-mtmitchell.dtb \ aspeed-bmc-arm-stardragon4800-rep2.dtb \ aspeed-bmc-asrock-e3c246d4i.dtb \ aspeed-bmc-asrock-romed8hm3.dtb \ diff --git a/arch/arm/boot/dts/am335x-baltos-ir2110.dts b/arch/arm/boot/dts/am335x-baltos-ir2110.dts index daf4cb398070f3caccb5755d7ae3947a33144a25..75992eec830f6988da9fba623933d22979330b80 100644 --- a/arch/arm/boot/dts/am335x-baltos-ir2110.dts +++ b/arch/arm/boot/dts/am335x-baltos-ir2110.dts @@ -81,3 +81,147 @@ pinctrl-0 = <&mmc1_pins>; cd-gpios = <&gpio1 15 GPIO_ACTIVE_LOW>; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "NC", + "NC", + "I2C1_SDA", + "I2C1_SCL", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "UART1_CTSN", + "UART1_RTSN", + "UART1_RX", + "UART1_TX", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "NC", + "RMII1_TXD1", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "RMII1_TXD0", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "NC", + "NC", + "CONSOLE_RX", + "CONSOLE_TX", + "NC", + "NC", + "NC", + "SD_CD", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SW2_0", + "SW2_1", + "RMII1_RXD1", + "RMII1_RXD0", + "UART1_DTR", + "UART1_DSR", + "UART1_DCD", + "UART1_RI", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "RMII1_CRS_DV", + "RMII1_RXER", + "RMII1_TXEN", + "NC", + "NC", + "NC", + "WLAN_IRQ", + "WLAN_EN", + "SW2_2", + "SW2_3", + "NC", + "NC", + "NC", + "ModeA0", + "ModeA1", + "ModeA2", + "ModeA3", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am335x-baltos-ir3220.dts b/arch/arm/boot/dts/am335x-baltos-ir3220.dts index 2123bd5894842354b1d5e9512d443e4bd3ba55d8..087e084506d258166232b132d9112e0fe5d67c2c 100644 --- a/arch/arm/boot/dts/am335x-baltos-ir3220.dts +++ b/arch/arm/boot/dts/am335x-baltos-ir3220.dts @@ -91,6 +91,10 @@ interrupts = <20 IRQ_TYPE_EDGE_RISING>; pinctrl-names = "default"; pinctrl-0 = <&tca6416_pins>; + gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3", + "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3", + "ModeA0", "ModeA1", "ModeA2", "ModeA3", + "ModeB0", "ModeB1", "ModeB2", "ModeB3"; }; }; @@ -123,3 +127,147 @@ pinctrl-0 = <&mmc1_pins>; cd-gpios = <&gpio2 18 GPIO_ACTIVE_LOW>; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "UART2_RX", + "UART2_TX", + "I2C1_SDA", + "I2C1_SCL", + "WLAN_BTN", + "W_DISABLE", + "NC", + "NC", + "NC", + "NC", + "UART1_CTSN", + "UART1_RTSN", + "UART1_RX", + "UART1_TX", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "TCA6416_INT", + "RMII1_TXD1", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "RMII1_TXD0", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "NC", + "NC", + "CONSOLE_RX", + "CONSOLE_TX", + "UART2_DTR", + "UART2_DSR", + "UART2_DCD", + "UART2_RI", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SD_CD", + "SD_WP", + "RMII1_RXD1", + "RMII1_RXD0", + "UART1_DTR", + "UART1_DSR", + "UART1_DCD", + "UART1_RI", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "RMII1_CRS_DV", + "RMII1_RXER", + "RMII1_TXEN", + "3G_PWR_EN", + "UART2_CTSN", + "UART2_RTSN", + "WLAN_IRQ", + "WLAN_EN", + "NC", + "NC", + "NC", + "NC", + "USB1_DRVVBUS", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am335x-baltos-ir5221.dts b/arch/arm/boot/dts/am335x-baltos-ir5221.dts index 2f3872dbf4f4d9fd71f69bc8d40067b5bd8826ed..faeb39aab60a727f22ed95e9f847c1ec0fc48d06 100644 --- a/arch/arm/boot/dts/am335x-baltos-ir5221.dts +++ b/arch/arm/boot/dts/am335x-baltos-ir5221.dts @@ -99,6 +99,10 @@ interrupts = <20 IRQ_TYPE_EDGE_RISING>; pinctrl-names = "default"; pinctrl-0 = <&tca6416_pins>; + gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3", + "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3", + "ModeA0", "ModeA1", "ModeA2", "ModeA3", + "ModeB0", "ModeB1", "ModeB2", "ModeB3"; }; }; @@ -147,3 +151,147 @@ pinctrl-0 = <&mmc1_pins>; cd-gpios = <&gpio2 18 GPIO_ACTIVE_LOW>; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "UART2_RX", + "UART2_TX", + "I2C1_SDA", + "I2C1_SCL", + "WLAN_BTN", + "W_DISABLE", + "NC", + "NC", + "NC", + "NC", + "UART1_CTSN", + "UART1_RTSN", + "UART1_RX", + "UART1_TX", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "TCA6416_INT", + "RMII1_TXD1", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "RMII1_TXD0", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "DCAN1_TX", + "DCAN1_RX", + "CONSOLE_RX", + "CONSOLE_TX", + "UART2_DTR", + "UART2_DSR", + "UART2_DCD", + "UART2_RI", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SD_CD", + "SD_WP", + "RMII1_RXD1", + "RMII1_RXD0", + "UART1_DTR", + "UART1_DSR", + "UART1_DCD", + "UART1_RI", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "RMII1_CRS_DV", + "RMII1_RXER", + "RMII1_TXEN", + "3G_PWR_EN", + "UART2_CTSN", + "UART2_RTSN", + "WLAN_IRQ", + "WLAN_EN", + "NC", + "NC", + "NC", + "NC", + "USB1_DRVVBUS", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am335x-baltos.dtsi b/arch/arm/boot/dts/am335x-baltos.dtsi index d3eafee79a2379be4eac25283d13091be0008fe6..6161c8929a781ab90b8cc2163bc53c904b4f2d5b 100644 --- a/arch/arm/boot/dts/am335x-baltos.dtsi +++ b/arch/arm/boot/dts/am335x-baltos.dtsi @@ -197,7 +197,7 @@ rb-gpios = <&gpmc 0 GPIO_ACTIVE_HIGH>; /* gpmc_wait0 */ nand-bus-width = <8>; ti,nand-ecc-opt = "bch8"; - ti,nand-xfer-type = "polled"; + ti,nand-xfer-type = "prefetch-dma"; gpmc,device-nand = "true"; gpmc,device-width = <1>; diff --git a/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts b/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts index 57e756b0f19276686d3709f68e0ae802ce276ab1..2e049489ac066445e0f9f0d7fbc95a7a983ec47b 100644 --- a/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts +++ b/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts @@ -85,3 +85,147 @@ status = "okay"; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "NC", + "NC", + "I2C1_SDA", + "I2C1_SCL", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "can_data", + "can_error", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "NC", + "NC", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "NC", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "DCAN1_TX", + "DCAN1_RX", + "CONSOLE_RX", + "CONSOLE_TX", + "NC", + "NC", + "NC", + "NC", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SW2_0", + "SW2_1", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "WLAN_IRQ", + "WLAN_EN", + "SW2_2", + "SW2_3", + "NC", + "NC", + "NC", + "ModeA0", + "ModeA1", + "ModeA2", + "ModeA3", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts b/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts index c6cc1c6218a93945ccac3058ac1057a2397dc3dd..6ed886c3306be818efcc4eeade51290225b9d873 100644 --- a/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts +++ b/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts @@ -93,3 +93,147 @@ ti,dual-emac-pvid = <2>; phy-handle = <&phy1>; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "UART2_RX", + "UART2_TX", + "I2C1_SDA", + "I2C1_SCL", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "UART1_CTSN", + "UART1_RTSN", + "UART1_RX", + "UART1_TX", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "NC", + "NC", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "NC", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "NC", + "NC", + "CONSOLE_RX", + "CONSOLE_TX", + "UART2_DTR", + "UART2_DSR", + "UART2_DCD", + "UART2_RI", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SW2_0", + "SW2_1", + "NC", + "NC", + "UART1_DTR", + "UART1_DSR", + "UART1_DCD", + "UART1_RI", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "NC", + "NC", + "NC", + "NC", + "UART2_CTSN", + "UART2_RTSN", + "WLAN_IRQ", + "WLAN_EN", + "SW2_2", + "SW2_3", + "NC", + "NC", + "NC", + "ModeA0", + "ModeA1", + "ModeA2", + "ModeA3", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts b/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts index 96dffd3ffd85e40d6e7184eb21a5c0435dc9a4b2..ad3adc7679f983673e18e2304c846d7cb9d1d116 100644 --- a/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts +++ b/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts @@ -71,6 +71,10 @@ interrupts = <20 IRQ_TYPE_EDGE_RISING>; pinctrl-names = "default"; pinctrl-0 = <&tca6416_pins>; + gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3", + "GP_IN4", "GP_IN5", "GP_IN6", "GP_IN7", + "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3", + "GP_OUT4", "GP_OUT5", "GP_OUT6", "GP_OUT7"; }; }; @@ -86,6 +90,10 @@ reg = <0x20>; gpio-controller; #gpio-cells = <2>; + gpio-line-names = "CH1_M0", "CH1_M1", "CH1_M2", "CH1_M3", + "CH2_M0", "CH2_M1", "CH2_M2", "CH2_M3", + "CH3_M0", "CH3_M1", "CH3_M2", "CH3_M3", + "CH4_M0", "CH4_M1", "CH4_M2", "CH4_M3"; }; tca6416c: gpio@21 { @@ -93,6 +101,10 @@ reg = <0x21>; gpio-controller; #gpio-cells = <2>; + gpio-line-names = "CH5_M0", "CH5_M1", "CH5_M2", "CH5_M3", + "CH6_M0", "CH6_M1", "CH6_M2", "CH6_M3", + "CH7_M0", "CH7_M1", "CH7_M2", "CH7_M3", + "CH8_M0", "CH8_M1", "CH8_M2", "CH8_M3"; }; }; @@ -113,3 +125,147 @@ ti,dual-emac-pvid = <2>; phy-handle = <&phy1>; }; + +&gpio0 { + gpio-line-names = + "MDIO", + "MDC", + "NC", + "NC", + "I2C1_SDA", + "I2C1_SCL", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "I2C2_SDA", + "I2C2_SCL", + "NC", + "NC", + "onrisc:blue:wlan", + "onrisc:green:app", + "USB0_DRVVBUS", + "ETH2_INT", + "NC", + "NC", + "MMC1_DAT0", + "MMC1_DAT1", + "NC", + "NC", + "MMC1_DAT2", + "MMC1_DAT3", + "NC", + "NC", + "GPMC_WAIT0", + "GPMC_WP_N"; +}; + +&gpio1 { + gpio-line-names = + "GPMC_AD0", + "GPMC_AD1", + "GPMC_AD2", + "GPMC_AD3", + "GPMC_AD4", + "GPMC_AD5", + "GPMC_AD6", + "GPMC_AD7", + "NC", + "NC", + "CONSOLE_RX", + "CONSOLE_TX", + "SW2_0_alt", + "SW2_1_alt", + "SW2_2_alt", + "SW2_3_alt", + "RGMII2_TCTL", + "RGMII2_RCTL", + "RGMII2_TD3", + "RGMII2_TD2", + "RGMII2_TD1", + "RGMII2_TD0", + "RGMII2_TCLK", + "RGMII2_RCLK", + "RGMII2_RD3", + "RGMII2_RD2", + "RGMII2_RD1", + "RGMII2_RD0", + "PMIC_INT1", + "GPMC_CSN0_Flash", + "MMC1_CLK", + "MMC1_CMD"; +}; + +&gpio2 { + gpio-line-names = + "GPMC_CSN3_BUS", + "GPMC_CLK", + "GPMC_ADVN_ALE", + "GPMC_OEN_RE_N", + "GPMC_WE_N", + "GPMC_BEN0_CLE", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "SW2_0", + "SW2_1", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "MMC0_DAT3", + "MMC0_DAT2", + "MMC0_DAT1", + "MMC0_DAT0", + "MMC0_CLK", + "MMC0_CMD"; +}; + +&gpio3 { + gpio-line-names = + "onrisc:red:power", + "NC", + "NC", + "NC", + "3G_PWR_EN", + "NC", + "NC", + "WLAN_IRQ", + "WLAN_EN", + "SW2_2", + "SW2_3", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC", + "NC"; +}; diff --git a/arch/arm/boot/dts/am33xx-l4.dtsi b/arch/arm/boot/dts/am33xx-l4.dtsi index 7da42a5b959cf5ceee61b0a560a4ac137fc3d4db..7e50fe633d8a1946ebbca072bf4fb41d75db2810 100644 --- a/arch/arm/boot/dts/am33xx-l4.dtsi +++ b/arch/arm/boot/dts/am33xx-l4.dtsi @@ -1502,8 +1502,7 @@ mmc1: mmc@0 { compatible = "ti,am335-sdhci"; ti,needs-special-reset; - dmas = <&edma_xbar 24 0 0 - &edma_xbar 25 0 0>; + dmas = <&edma 24 0>, <&edma 25 0>; dma-names = "tx", "rx"; interrupts = <64>; reg = <0x0 0x1000>; diff --git a/arch/arm/boot/dts/am5748.dtsi b/arch/arm/boot/dts/am5748.dtsi index c260aa1a85bdb5352398f48e2ca3ad60aad4369f..a1f029e9d1f3df319cd5d59c2935a3f694422866 100644 --- a/arch/arm/boot/dts/am5748.dtsi +++ b/arch/arm/boot/dts/am5748.dtsi @@ -25,6 +25,10 @@ status = "disabled"; }; +&usb4_tm { + status = "disabled"; +}; + &atl_tm { status = "disabled"; }; diff --git a/arch/arm/boot/dts/arm-realview-eb.dtsi b/arch/arm/boot/dts/arm-realview-eb.dtsi index 2dfb32bf9d482d364364b6e9c62f96dacc1849bc..fbb2258b451f904856d6fedd41af6e67afeea5ad 100644 --- a/arch/arm/boot/dts/arm-realview-eb.dtsi +++ b/arch/arm/boot/dts/arm-realview-eb.dtsi @@ -399,7 +399,7 @@ compatible = "arm,pl022", "arm,primecell"; reg = <0x1000d000 0x1000>; clocks = <&sspclk>, <&pclk>; - clock-names = "SSPCLK", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; wdog: watchdog@10010000 { diff --git a/arch/arm/boot/dts/arm-realview-pb1176.dts b/arch/arm/boot/dts/arm-realview-pb1176.dts index 06b8723b09eb955e1894c9326dd44ab116f43e39..efed325af88d206fc5cc6f4d9b6e6d7f4f05cb77 100644 --- a/arch/arm/boot/dts/arm-realview-pb1176.dts +++ b/arch/arm/boot/dts/arm-realview-pb1176.dts @@ -410,7 +410,7 @@ interrupt-parent = <&intc_dc1176>; interrupts = <0 17 IRQ_TYPE_LEVEL_HIGH>; clocks = <&sspclk>, <&pclk>; - clock-names = "SSPCLK", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; pb1176_serial0: serial@1010c000 { diff --git a/arch/arm/boot/dts/arm-realview-pb11mp.dts b/arch/arm/boot/dts/arm-realview-pb11mp.dts index 295aef4481239bdee6541ecc4c7b69e628ff224d..89103d54ecc15c9afbdca380ab01b9427924cc9c 100644 --- a/arch/arm/boot/dts/arm-realview-pb11mp.dts +++ b/arch/arm/boot/dts/arm-realview-pb11mp.dts @@ -555,7 +555,7 @@ interrupt-parent = <&intc_pb11mp>; interrupts = <0 11 IRQ_TYPE_LEVEL_HIGH>; clocks = <&sspclk>, <&pclk>; - clock-names = "SSPCLK", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; watchdog@1000f000 { diff --git a/arch/arm/boot/dts/arm-realview-pbx.dtsi b/arch/arm/boot/dts/arm-realview-pbx.dtsi index 6f61f968d689222e4ed9d56927137693c9ed7b58..ec1507c5147c64ea978e133b5bc6bafc5238fa69 100644 --- a/arch/arm/boot/dts/arm-realview-pbx.dtsi +++ b/arch/arm/boot/dts/arm-realview-pbx.dtsi @@ -390,7 +390,7 @@ compatible = "arm,pl022", "arm,primecell"; reg = <0x1000d000 0x1000>; clocks = <&sspclk>, <&pclk>; - clock-names = "SSPCLK", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; wdog0: watchdog@1000f000 { diff --git a/arch/arm/boot/dts/armada-370.dtsi b/arch/arm/boot/dts/armada-370.dtsi index 46e6d3ed8f35a6481883b518f18f598e98531c9a..9dc928859ad3317f519f19f51b61b8e4887a2313 100644 --- a/arch/arm/boot/dts/armada-370.dtsi +++ b/arch/arm/boot/dts/armada-370.dtsi @@ -60,16 +60,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 58>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 58>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie0_intc 0>, + <0 0 0 2 &pcie0_intc 1>, + <0 0 0 3 &pcie0_intc 2>, + <0 0 0 4 &pcie0_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie0_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie2: pcie@2,0 { @@ -78,16 +88,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 62>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 62>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 9>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-375.dtsi b/arch/arm/boot/dts/armada-375.dtsi index 7f2f24a29e6c178d4f5a08c6748e1b093fa21112..929deaf312a55c6989711d57be72f1a570ea3ce9 100644 --- a/arch/arm/boot/dts/armada-375.dtsi +++ b/arch/arm/boot/dts/armada-375.dtsi @@ -568,16 +568,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie0_intc 0>, + <0 0 0 2 &pcie0_intc 1>, + <0 0 0 3 &pcie0_intc 2>, + <0 0 0 4 &pcie0_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie0_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie1: pcie@2,0 { @@ -586,16 +596,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <1>; clocks = <&gateclk 6>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-380.dtsi b/arch/arm/boot/dts/armada-380.dtsi index cff1269f3fbfd14f3b43da67f5ef9909b17c8d5d..ce1dddb2269b02edc3b9dab4817dfdd712fa4c56 100644 --- a/arch/arm/boot/dts/armada-380.dtsi +++ b/arch/arm/boot/dts/armada-380.dtsi @@ -64,16 +64,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 8>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; /* x1 port */ @@ -83,16 +93,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; /* x1 port */ @@ -102,16 +122,26 @@ reg = <0x1800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0 0x81000000 0 0 0x81000000 0x3 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3_intc 0>, + <0 0 0 2 &pcie3_intc 1>, + <0 0 0 3 &pcie3_intc 2>, + <0 0 0 4 &pcie3_intc 3>; marvell,pcie-port = <2>; marvell,pcie-lane = <0>; clocks = <&gateclk 6>; status = "disabled"; + + pcie3_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; }; diff --git a/arch/arm/boot/dts/armada-385-turris-omnia.dts b/arch/arm/boot/dts/armada-385-turris-omnia.dts index d1e0db6e573072fb4db0e2b587a643f4333e9adc..72ac807cae259cc3ded771120ad72677bcd1de84 100644 --- a/arch/arm/boot/dts/armada-385-turris-omnia.dts +++ b/arch/arm/boot/dts/armada-385-turris-omnia.dts @@ -105,6 +105,33 @@ */ status = "disabled"; }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "SPDIF"; + simple-audio-card,format = "i2s"; + + simple-audio-card,cpu { + sound-dai = <&audio_controller 1>; + }; + + simple-audio-card,codec { + sound-dai = <&spdif_out>; + }; + }; + + spdif_out: spdif-out { + #sound-dai-cells = <0>; + compatible = "linux,spdif-dit"; + }; +}; + +&audio_controller { + /* Pin header U16, GPIO51 in SPDIFO mode */ + pinctrl-0 = <&spdif_pins>; + pinctrl-names = "default"; + spdif-mode; + status = "okay"; }; &bm { @@ -166,6 +193,7 @@ buffer-manager = <&bm>; bm,pool-long = <2>; bm,pool-short = <3>; + label = "wan"; }; &i2c0 { @@ -476,7 +504,7 @@ marvell,function = "spi0"; }; - spi0cs1_pins: spi0cs1-pins { + spi0cs2_pins: spi0cs2-pins { marvell,pins = "mpp26"; marvell,function = "spi0"; }; @@ -511,7 +539,7 @@ }; }; - /* MISO, MOSI, SCLK and CS1 are routed to pin header CN11 */ + /* MISO, MOSI, SCLK and CS2 are routed to pin header CN11 */ }; &uart0 { diff --git a/arch/arm/boot/dts/armada-388-db.dts b/arch/arm/boot/dts/armada-388-db.dts index 5130eccc32af6fba6536e1e1aa55bfe007302751..2bcec5419b6661d964abdaf5f59375ec5cb43780 100644 --- a/arch/arm/boot/dts/armada-388-db.dts +++ b/arch/arm/boot/dts/armada-388-db.dts @@ -36,6 +36,11 @@ i2c@11000 { status = "okay"; clock-frequency = <100000>; + audio_codec: audio-codec@4a { + #sound-dai-cells = <0>; + compatible = "cirrus,cs42l51"; + reg = <0x4a>; + }; }; i2c@11100 { @@ -99,6 +104,12 @@ no-1-8-v; }; + audio-controller@e8000 { + pinctrl-0 = <&i2s_pins>; + pinctrl-names = "default"; + status = "disabled"; + }; + usb3@f0000 { status = "okay"; }; @@ -128,6 +139,64 @@ }; }; }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "Armada 385 DB Audio"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Headphone", "Out Jack", + "Line", "In Jack"; + simple-audio-card,routing = + "Out Jack", "HPL", + "Out Jack", "HPR", + "AIN1L", "In Jack", + "AIN1R", "In Jack"; + status = "disabled"; + + simple-audio-card,dai-link@0 { + format = "i2s"; + cpu { + sound-dai = <&audio_controller 0>; + }; + + codec { + sound-dai = <&audio_codec>; + }; + }; + + simple-audio-card,dai-link@1 { + format = "i2s"; + cpu { + sound-dai = <&audio_controller 1>; + }; + + codec { + sound-dai = <&spdif_out>; + }; + }; + + simple-audio-card,dai-link@2 { + format = "i2s"; + cpu { + sound-dai = <&audio_controller 1>; + }; + + codec { + sound-dai = <&spdif_in>; + }; + }; + }; + + spdif_out: spdif-out { + #sound-dai-cells = <0>; + compatible = "linux,spdif-dit"; + }; + + spdif_in: spdif-in { + #sound-dai-cells = <0>; + compatible = "linux,spdif-dir"; + }; }; &spi0 { diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi index df3c8d1d8f641230f9b17b8dcde7f58336918eb1..446861b6b17b2ae628a42a4a32dca94a0b5b1f21 100644 --- a/arch/arm/boot/dts/armada-38x.dtsi +++ b/arch/arm/boot/dts/armada-38x.dtsi @@ -289,6 +289,18 @@ marvell,pins = "mpp44"; marvell,function = "sata3"; }; + + i2s_pins: i2s-pins { + marvell,pins = "mpp48", "mpp49", + "mpp50", "mpp51", + "mpp52", "mpp53"; + marvell,function = "audio"; + }; + + spdif_pins: spdif-pins { + marvell,pins = "mpp51"; + marvell,function = "audio"; + }; }; gpio0: gpio@18100 { @@ -298,6 +310,7 @@ reg-names = "gpio", "pwm"; ngpios = <32>; gpio-controller; + gpio-ranges = <&pinctrl 0 0 32>; #gpio-cells = <2>; #pwm-cells = <2>; interrupt-controller; @@ -316,6 +329,7 @@ reg-names = "gpio", "pwm"; ngpios = <28>; gpio-controller; + gpio-ranges = <&pinctrl 0 32 28>; #gpio-cells = <2>; #pwm-cells = <2>; interrupt-controller; @@ -618,6 +632,18 @@ status = "disabled"; }; + audio_controller: audio-controller@e8000 { + #sound-dai-cells = <1>; + compatible = "marvell,armada-380-audio"; + reg = <0xe8000 0x4000>, <0x18410 0xc>, + <0x18204 0x4>; + reg-names = "i2s_regs", "pll_regs", "soc_ctrl"; + interrupts = ; + clocks = <&gateclk 0>; + clock-names = "internal"; + status = "disabled"; + }; + usb3_0: usb3@f0000 { compatible = "marvell,armada-380-xhci"; reg = <0xf0000 0x4000>,<0xf4000 0x4000>; diff --git a/arch/arm/boot/dts/armada-39x.dtsi b/arch/arm/boot/dts/armada-39x.dtsi index e0b7c2099831272334d9db9d9d4cea6644f7de83..923b035a3ab38e79b6a9b7c357def34ab14ea4f9 100644 --- a/arch/arm/boot/dts/armada-39x.dtsi +++ b/arch/arm/boot/dts/armada-39x.dtsi @@ -438,16 +438,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 8>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; /* x1 port */ @@ -457,16 +467,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; /* x1 port */ @@ -476,16 +496,26 @@ reg = <0x1800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0 0x81000000 0 0 0x81000000 0x3 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3_intc 0>, + <0 0 0 2 &pcie3_intc 1>, + <0 0 0 3 &pcie3_intc 2>, + <0 0 0 4 &pcie3_intc 3>; marvell,pcie-port = <2>; marvell,pcie-lane = <0>; clocks = <&gateclk 6>; status = "disabled"; + + pcie3_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; /* @@ -498,16 +528,26 @@ reg = <0x2000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&gic GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0 0x81000000 0 0 0x81000000 0x4 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &gic GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie4_intc 0>, + <0 0 0 2 &pcie4_intc 1>, + <0 0 0 3 &pcie4_intc 2>, + <0 0 0 4 &pcie4_intc 3>; marvell,pcie-port = <3>; marvell,pcie-lane = <0>; clocks = <&gateclk 7>; status = "disabled"; + + pcie4_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi index 38a052a0312d28528cf63050da46ca699311dd84..b21ffb819b1d8197a03c605166390ef8da478adf 100644 --- a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi +++ b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi @@ -76,16 +76,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 58>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 58>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts index 87dcb502f72da5fdab843d38516f8fc85d91387b..0dad95ea26c2a9e187edc22332362b498359eddb 100644 --- a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts +++ b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts @@ -164,7 +164,7 @@ }; }; - spi3 { + spi-3 { compatible = "spi-gpio"; status = "okay"; gpio-sck = <&gpio0 25 GPIO_ACTIVE_LOW>; diff --git a/arch/arm/boot/dts/armada-xp-mv78230.dtsi b/arch/arm/boot/dts/armada-xp-mv78230.dtsi index 8558bf6bb54c603c16b8a5a987c991413e3297c8..bf9360f41e0a601d80179fc3681d080e113a25db 100644 --- a/arch/arm/boot/dts/armada-xp-mv78230.dtsi +++ b/arch/arm/boot/dts/armada-xp-mv78230.dtsi @@ -83,16 +83,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 58>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 58>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie2: pcie@2,0 { @@ -101,16 +111,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 59>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 59>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <1>; clocks = <&gateclk 6>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie3: pcie@3,0 { @@ -119,16 +139,26 @@ reg = <0x1800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 60>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0 0x81000000 0 0 0x81000000 0x3 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 60>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3_intc 0>, + <0 0 0 2 &pcie3_intc 1>, + <0 0 0 3 &pcie3_intc 2>, + <0 0 0 4 &pcie3_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <2>; clocks = <&gateclk 7>; status = "disabled"; + + pcie3_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie4: pcie@4,0 { @@ -137,16 +167,26 @@ reg = <0x2000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 61>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0 0x81000000 0 0 0x81000000 0x4 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 61>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie4_intc 0>, + <0 0 0 2 &pcie4_intc 1>, + <0 0 0 3 &pcie4_intc 2>, + <0 0 0 4 &pcie4_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <3>; clocks = <&gateclk 8>; status = "disabled"; + + pcie4_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie5: pcie@5,0 { @@ -155,16 +195,26 @@ reg = <0x2800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 62>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0 0x81000000 0 0 0x81000000 0x5 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 62>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie5_intc 0>, + <0 0 0 2 &pcie5_intc 1>, + <0 0 0 3 &pcie5_intc 2>, + <0 0 0 4 &pcie5_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 9>; status = "disabled"; + + pcie5_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-xp-mv78260.dtsi b/arch/arm/boot/dts/armada-xp-mv78260.dtsi index 2d85fe8ac327296420e1050228e88fa0df48d79a..0714af52e607536a794864d1ff5b95dedc663ee5 100644 --- a/arch/arm/boot/dts/armada-xp-mv78260.dtsi +++ b/arch/arm/boot/dts/armada-xp-mv78260.dtsi @@ -98,16 +98,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 58>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 58>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie2: pcie@2,0 { @@ -116,16 +126,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 59>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 59>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <1>; clocks = <&gateclk 6>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie3: pcie@3,0 { @@ -134,16 +154,26 @@ reg = <0x1800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 60>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0 0x81000000 0 0 0x81000000 0x3 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 60>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3_intc 0>, + <0 0 0 2 &pcie3_intc 1>, + <0 0 0 3 &pcie3_intc 2>, + <0 0 0 4 &pcie3_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <2>; clocks = <&gateclk 7>; status = "disabled"; + + pcie3_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie4: pcie@4,0 { @@ -152,16 +182,26 @@ reg = <0x2000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 61>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0 0x81000000 0 0 0x81000000 0x4 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 61>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie4_intc 0>, + <0 0 0 2 &pcie4_intc 1>, + <0 0 0 3 &pcie4_intc 2>, + <0 0 0 4 &pcie4_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <3>; clocks = <&gateclk 8>; status = "disabled"; + + pcie4_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie5: pcie@5,0 { @@ -170,16 +210,26 @@ reg = <0x2800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 62>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0 0x81000000 0 0 0x81000000 0x5 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 62>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie5_intc 0>, + <0 0 0 2 &pcie5_intc 1>, + <0 0 0 3 &pcie5_intc 2>, + <0 0 0 4 &pcie5_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 9>; status = "disabled"; + + pcie5_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie6: pcie@6,0 { @@ -188,16 +238,26 @@ reg = <0x3000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 63>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x6 0 1 0 0x81000000 0 0 0x81000000 0x6 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 63>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie6_intc 0>, + <0 0 0 2 &pcie6_intc 1>, + <0 0 0 3 &pcie6_intc 2>, + <0 0 0 4 &pcie6_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <1>; clocks = <&gateclk 10>; status = "disabled"; + + pcie6_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie7: pcie@7,0 { @@ -206,16 +266,26 @@ reg = <0x3800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 64>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x7 0 1 0 0x81000000 0 0 0x81000000 0x7 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 64>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie7_intc 0>, + <0 0 0 2 &pcie7_intc 1>, + <0 0 0 3 &pcie7_intc 2>, + <0 0 0 4 &pcie7_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <2>; clocks = <&gateclk 11>; status = "disabled"; + + pcie7_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie8: pcie@8,0 { @@ -224,16 +294,26 @@ reg = <0x4000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 65>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x8 0 1 0 0x81000000 0 0 0x81000000 0x8 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 65>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie8_intc 0>, + <0 0 0 2 &pcie8_intc 1>, + <0 0 0 3 &pcie8_intc 2>, + <0 0 0 4 &pcie8_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <3>; clocks = <&gateclk 12>; status = "disabled"; + + pcie8_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie9: pcie@9,0 { @@ -242,16 +322,26 @@ reg = <0x4800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 99>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x9 0 1 0 0x81000000 0 0 0x81000000 0x9 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 99>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie9_intc 0>, + <0 0 0 2 &pcie9_intc 1>, + <0 0 0 3 &pcie9_intc 2>, + <0 0 0 4 &pcie9_intc 3>; marvell,pcie-port = <2>; marvell,pcie-lane = <0>; clocks = <&gateclk 26>; status = "disabled"; + + pcie9_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/armada-xp-mv78460.dtsi b/arch/arm/boot/dts/armada-xp-mv78460.dtsi index 230a3fd36b3048937c760784ec2fd619f49f9bdb..16185edf9aa5f00aecabe46652ca0ae740d0e81e 100644 --- a/arch/arm/boot/dts/armada-xp-mv78460.dtsi +++ b/arch/arm/boot/dts/armada-xp-mv78460.dtsi @@ -119,16 +119,26 @@ reg = <0x0800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 58>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 58>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gateclk 5>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie2: pcie@2,0 { @@ -137,16 +147,26 @@ reg = <0x1000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 59>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 59>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie2_intc 0>, + <0 0 0 2 &pcie2_intc 1>, + <0 0 0 3 &pcie2_intc 2>, + <0 0 0 4 &pcie2_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <1>; clocks = <&gateclk 6>; status = "disabled"; + + pcie2_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie3: pcie@3,0 { @@ -155,16 +175,26 @@ reg = <0x1800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 60>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0 0x81000000 0 0 0x81000000 0x3 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 60>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3_intc 0>, + <0 0 0 2 &pcie3_intc 1>, + <0 0 0 3 &pcie3_intc 2>, + <0 0 0 4 &pcie3_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <2>; clocks = <&gateclk 7>; status = "disabled"; + + pcie3_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie4: pcie@4,0 { @@ -173,16 +203,26 @@ reg = <0x2000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 61>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0 0x81000000 0 0 0x81000000 0x4 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 61>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie4_intc 0>, + <0 0 0 2 &pcie4_intc 1>, + <0 0 0 3 &pcie4_intc 2>, + <0 0 0 4 &pcie4_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <3>; clocks = <&gateclk 8>; status = "disabled"; + + pcie4_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie5: pcie@5,0 { @@ -191,16 +231,26 @@ reg = <0x2800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 62>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0 0x81000000 0 0 0x81000000 0x5 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 62>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie5_intc 0>, + <0 0 0 2 &pcie5_intc 1>, + <0 0 0 3 &pcie5_intc 2>, + <0 0 0 4 &pcie5_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gateclk 9>; status = "disabled"; + + pcie5_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie6: pcie@6,0 { @@ -209,16 +259,26 @@ reg = <0x3000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 63>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x6 0 1 0 0x81000000 0 0 0x81000000 0x6 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 63>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie6_intc 0>, + <0 0 0 2 &pcie6_intc 1>, + <0 0 0 3 &pcie6_intc 2>, + <0 0 0 4 &pcie6_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <1>; clocks = <&gateclk 10>; status = "disabled"; + + pcie6_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie7: pcie@7,0 { @@ -227,16 +287,26 @@ reg = <0x3800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 64>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x7 0 1 0 0x81000000 0 0 0x81000000 0x7 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 64>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie7_intc 0>, + <0 0 0 2 &pcie7_intc 1>, + <0 0 0 3 &pcie7_intc 2>, + <0 0 0 4 &pcie7_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <2>; clocks = <&gateclk 11>; status = "disabled"; + + pcie7_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie8: pcie@8,0 { @@ -245,16 +315,26 @@ reg = <0x4000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 65>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x8 0 1 0 0x81000000 0 0 0x81000000 0x8 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 65>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie8_intc 0>, + <0 0 0 2 &pcie8_intc 1>, + <0 0 0 3 &pcie8_intc 2>, + <0 0 0 4 &pcie8_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <3>; clocks = <&gateclk 12>; status = "disabled"; + + pcie8_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie9: pcie@9,0 { @@ -263,16 +343,26 @@ reg = <0x4800 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 99>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0x9 0 1 0 0x81000000 0 0 0x81000000 0x9 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 99>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie9_intc 0>, + <0 0 0 2 &pcie9_intc 1>, + <0 0 0 3 &pcie9_intc 2>, + <0 0 0 4 &pcie9_intc 3>; marvell,pcie-port = <2>; marvell,pcie-lane = <0>; clocks = <&gateclk 26>; status = "disabled"; + + pcie9_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie10: pcie@a,0 { @@ -281,16 +371,26 @@ reg = <0x5000 0 0 0 0>; #address-cells = <3>; #size-cells = <2>; + interrupt-names = "intx"; + interrupts-extended = <&mpic 103>; #interrupt-cells = <1>; ranges = <0x82000000 0 0 0x82000000 0xa 0 1 0 0x81000000 0 0 0x81000000 0xa 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &mpic 103>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie10_intc 0>, + <0 0 0 2 &pcie10_intc 1>, + <0 0 0 3 &pcie10_intc 2>, + <0 0 0 4 &pcie10_intc 3>; marvell,pcie-port = <3>; marvell,pcie-lane = <0>; clocks = <&gateclk 27>; status = "disabled"; + + pcie10_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts b/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts index d0a5c2ff0fec429b9dddf44616010dac52c85262..f34a2b1ec2f0fbfedaaf8d14d88858082f578029 100644 --- a/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts +++ b/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts @@ -5,7 +5,7 @@ / { model = "AST2600 A1 EVB"; - compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600"; + compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600-evb", "aspeed,ast2600"; /delete-node/regulator-vcc-sdhci0; /delete-node/regulator-vcc-sdhci1; diff --git a/arch/arm/boot/dts/aspeed-ast2600-evb.dts b/arch/arm/boot/dts/aspeed-ast2600-evb.dts index c698e653826937f0fb5566140abf42277585bdd7..de83c0eb1d6ed9e39f95d43c795c738f29258bb5 100644 --- a/arch/arm/boot/dts/aspeed-ast2600-evb.dts +++ b/arch/arm/boot/dts/aspeed-ast2600-evb.dts @@ -8,7 +8,7 @@ / { model = "AST2600 EVB"; - compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600"; + compatible = "aspeed,ast2600-evb", "aspeed,ast2600"; aliases { serial4 = &uart5; @@ -182,6 +182,7 @@ status = "okay"; m25p,fast-read; label = "bmc"; + spi-rx-bus-width = <4>; spi-max-frequency = <50000000>; #include "openbmc-flash-layout-64.dtsi" }; @@ -196,6 +197,7 @@ status = "okay"; m25p,fast-read; label = "pnor"; + spi-rx-bus-width = <4>; spi-max-frequency = <100000000>; }; }; @@ -207,11 +209,6 @@ &i2c0 { status = "okay"; - - temp@2e { - compatible = "adi,adt7490"; - reg = <0x2e>; - }; }; &i2c1 { @@ -240,10 +237,26 @@ &i2c7 { status = "okay"; + + temp@2e { + compatible = "adi,adt7490"; + reg = <0x2e>; + }; + + eeprom@50 { + compatible = "atmel,24c08"; + reg = <0x50>; + pagesize = <16>; + }; }; &i2c8 { status = "okay"; + + lm75@4d { + compatible = "national,lm75"; + reg = <0x4d>; + }; }; &i2c9 { diff --git a/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts b/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts new file mode 100644 index 0000000000000000000000000000000000000000..64bb9bf92de23e5dd2c9031cb143d08d9fa5bdcd --- /dev/null +++ b/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0 +/dts-v1/; + +#include "aspeed-g5.dtsi" +#include +#include + +/ { + model = "AMD DaytonaX BMC"; + compatible = "amd,daytonax-bmc", "aspeed,ast2500"; + + memory@80000000 { + reg = <0x80000000 0x20000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + video_engine_memory: jpegbuffer { + size = <0x02000000>; /* 32M */ + alignment = <0x01000000>; + compatible = "shared-dma-pool"; + reusable; + }; + }; + + aliases { + serial0 = &uart1; + serial4 = &uart5; + }; + + chosen { + stdout-path = &uart5; + bootargs = "console=ttyS4,115200"; + }; + + leds { + compatible = "gpio-leds"; + + led-fault { + gpios = <&gpio ASPEED_GPIO(A, 2) GPIO_ACTIVE_LOW>; + }; + + led-identify { + gpios = <&gpio ASPEED_GPIO(A, 3) GPIO_ACTIVE_LOW>; + }; + }; + + iio-hwmon { + compatible = "iio-hwmon"; + io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>, <&adc 4>, + <&adc 5>, <&adc 6>, <&adc 7>, <&adc 8>, <&adc 9>, + <&adc 10>, <&adc 11>, <&adc 12>, <&adc 13>, <&adc 14>, + <&adc 15>; + }; +}; + +&fmc { + status = "okay"; + flash@0 { + status = "okay"; + m25p,fast-read; + label = "bmc"; + #include "openbmc-flash-layout.dtsi" + }; +}; + +&mac0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_rgmii1_default &pinctrl_mdio1_default>; +}; + +&uart1 { + //Host Console + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_txd1_default + &pinctrl_rxd1_default + &pinctrl_nrts1_default + &pinctrl_ndtr1_default + &pinctrl_ndsr1_default + &pinctrl_ncts1_default + &pinctrl_ndcd1_default + &pinctrl_nri1_default>; +}; + +&uart5 { + //BMC Console + status = "okay"; +}; + +&vuart { + status = "okay"; + aspeed,lpc-io-reg = <0x3f8>; + aspeed,lpc-interrupts = <4 IRQ_TYPE_LEVEL_HIGH>; +}; + +&adc { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_adc0_default + &pinctrl_adc1_default + &pinctrl_adc2_default + &pinctrl_adc3_default + &pinctrl_adc4_default + &pinctrl_adc5_default + &pinctrl_adc6_default + &pinctrl_adc7_default + &pinctrl_adc8_default + &pinctrl_adc9_default + &pinctrl_adc10_default + &pinctrl_adc11_default + &pinctrl_adc12_default + &pinctrl_adc13_default + &pinctrl_adc14_default + &pinctrl_adc15_default>; +}; + +&gpio { + status = "okay"; + gpio-line-names = + /*A0-A7*/ "","","led-fault","led-identify","","","","", + /*B0-B7*/ "","","","","","","","", + /*C0-C7*/ "id-button","","","","","","","", + /*D0-D7*/ "","","ASSERT_BMC_READY","","","","","", + /*E0-E7*/ "reset-button","reset-control","power-button","power-control","", + "power-good","power-ok","", + /*F0-F7*/ "","","","","","","BATTERY_DETECT","", + /*G0-G7*/ "","","","","","","","", + /*H0-H7*/ "","","","","","","","", + /*I0-I7*/ "","","","","","","","", + /*J0-J7*/ "","","","","","","","", + /*K0-K7*/ "","","","","","","","", + /*L0-L7*/ "","","","","","","","", + /*M0-M7*/ "","","","","","","","", + /*N0-N7*/ "","","","","","","","", + /*O0-O7*/ "","","","","","","","", + /*P0-P7*/ "","","","","","","","", + /*Q0-Q7*/ "","","","","","","","", + /*R0-R7*/ "","","","","","","","", + /*S0-S7*/ "","","","","","","","", + /*T0-T7*/ "","","","","","","","", + /*U0-U7*/ "","","","","","","","", + /*V0-V7*/ "","","","","","","","", + /*W0-W7*/ "","","","","","","","", + /*X0-X7*/ "","","","","","","","", + /*Y0-Y7*/ "","","","","","","","", + /*Z0-Z7*/ "","","","","","","","", + /*AA0-AA7*/ "","","","","","","","", + /*AB0-AB7*/ "FM_BMC_READ_SPD_TEMP","","","","","","","", + /*AC0-AC7*/ "","","","","","","",""; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; +}; + +&i2c2 { + status = "okay"; +}; + +&i2c3 { + status = "okay"; +}; + +&i2c4 { + status = "okay"; +}; + +&i2c5 { + status = "okay"; +}; + +&i2c6 { + status = "okay"; +}; + +&i2c7 { + status = "okay"; +}; + +&i2c8 { + status = "okay"; +}; + +&i2c10 { + status = "okay"; +}; + +&i2c11 { + status = "okay"; +}; + +&i2c12 { + status = "okay"; +}; + +&kcs3 { + status = "okay"; + aspeed,lpc-io-reg = <0xca2>; +}; + +&lpc_snoop { + status = "okay"; + snoop-ports = <0x80>, <0x81>; +}; + +&lpc_ctrl { + status = "okay"; +}; + +&pwm_tacho { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm0_default + &pinctrl_pwm1_default + &pinctrl_pwm2_default + &pinctrl_pwm3_default + &pinctrl_pwm4_default + &pinctrl_pwm5_default + &pinctrl_pwm6_default + &pinctrl_pwm7_default>; + + fan@0 { + reg = <0x00>; + aspeed,fan-tach-ch = /bits/ 8 <0x00>; + }; + + fan@1 { + reg = <0x00>; + aspeed,fan-tach-ch = /bits/ 8 <0x01>; + }; + + fan@2 { + reg = <0x01>; + aspeed,fan-tach-ch = /bits/ 8 <0x02>; + }; + + fan@3 { + reg = <0x01>; + aspeed,fan-tach-ch = /bits/ 8 <0x03>; + }; + + fan@4 { + reg = <0x02>; + aspeed,fan-tach-ch = /bits/ 8 <0x04>; + }; + + fan@5 { + reg = <0x02>; + aspeed,fan-tach-ch = /bits/ 8 <0x05>; + }; + + fan@6 { + reg = <0x03>; + aspeed,fan-tach-ch = /bits/ 8 <0x06>; + }; + + fan@7 { + reg = <0x03>; + aspeed,fan-tach-ch = /bits/ 8 <0x07>; + }; + + fan@8 { + reg = <0x04>; + aspeed,fan-tach-ch = /bits/ 8 <0x08>; + }; + + fan@9 { + reg = <0x04>; + aspeed,fan-tach-ch = /bits/ 8 <0x09>; + }; + + fan@10 { + reg = <0x05>; + aspeed,fan-tach-ch = /bits/ 8 <0x0a>; + }; + + fan@11 { + reg = <0x05>; + aspeed,fan-tach-ch = /bits/ 8 <0x0b>; + }; + + fan@12 { + reg = <0x06>; + aspeed,fan-tach-ch = /bits/ 8 <0x0c>; + }; + + fan@13 { + reg = <0x06>; + aspeed,fan-tach-ch = /bits/ 8 <0x0d>; + }; + + fan@14 { + reg = <0x07>; + aspeed,fan-tach-ch = /bits/ 8 <0x0e>; + }; + + fan@15 { + reg = <0x07>; + aspeed,fan-tach-ch = /bits/ 8 <0x0f>; + }; +}; + +&video { + status = "okay"; + memory-region = <&video_engine_memory>; +}; + +&vhub { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts b/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts index 82a6f14a45f002e88dedb1c725ab5b6bd1145e60..d127cbcc799818b05c3613b24396b44b6dd71982 100644 --- a/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts +++ b/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts @@ -97,101 +97,6 @@ }; }; - gpio-keys { - compatible = "gpio-keys"; - - event-shutdown-ack { - label = "SHUTDOWN_ACK"; - gpios = <&gpio ASPEED_GPIO(G, 2) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-reboot-ack { - label = "REBOOT_ACK"; - gpios = <&gpio ASPEED_GPIO(J, 3) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s0-overtemp { - label = "S0_OVERTEMP"; - gpios = <&gpio ASPEED_GPIO(G, 3) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s0-hightemp { - label = "S0_HIGHTEMP"; - gpios = <&gpio ASPEED_GPIO(J, 0) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s0-cpu-fault { - label = "S0_CPU_FAULT"; - gpios = <&gpio ASPEED_GPIO(J, 1) GPIO_ACTIVE_HIGH>; - linux,code = ; - }; - - event-s0-scp-auth-fail { - label = "S0_SCP_AUTH_FAIL"; - gpios = <&gpio ASPEED_GPIO(J, 2) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s1-scp-auth-fail { - label = "S1_SCP_AUTH_FAIL"; - gpios = <&gpio ASPEED_GPIO(Z, 5) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s1-overtemp { - label = "S1_OVERTEMP"; - gpios = <&gpio ASPEED_GPIO(Z, 6) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s1-hightemp { - label = "S1_HIGHTEMP"; - gpios = <&gpio ASPEED_GPIO(AB, 0) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-s1-cpu-fault { - label = "S1_CPU_FAULT"; - gpios = <&gpio ASPEED_GPIO(Z, 1) GPIO_ACTIVE_HIGH>; - linux,code = ; - }; - - event-id { - label = "ID_BUTTON"; - gpios = <&gpio ASPEED_GPIO(Q, 5) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-psu1-vin-good { - label = "PSU1_VIN_GOOD"; - gpios = <&gpio ASPEED_GPIO(H, 4) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-psu2-vin-good { - label = "PSU2_VIN_GOOD"; - gpios = <&gpio ASPEED_GPIO(H, 5) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-psu1-present { - label = "PSU1_PRESENT"; - gpios = <&gpio ASPEED_GPIO(I, 0) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - event-psu2-present { - label = "PSU2_PRESENT"; - gpios = <&gpio ASPEED_GPIO(I, 1) GPIO_ACTIVE_LOW>; - linux,code = ; - }; - - }; - gpioA0mux: mux-controller { compatible = "gpio-mux"; #mux-control-cells = <0>; diff --git a/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts b/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts new file mode 100644 index 0000000000000000000000000000000000000000..606cd4be245a0fb340ed7619ba9a0cdade9fea8a --- /dev/null +++ b/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2022, Ampere Computing LLC + +/dts-v1/; + +#include "aspeed-g6.dtsi" +#include + +/ { + model = "Ampere Mt.Mitchell BMC"; + compatible = "ampere,mtmitchell-bmc", "aspeed,ast2600"; + + chosen { + stdout-path = &uart5; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x80000000>; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + gfx_memory: framebuffer { + size = <0x01000000>; + alignment = <0x01000000>; + compatible = "shared-dma-pool"; + reusable; + }; + + video_engine_memory: video { + size = <0x04000000>; + alignment = <0x01000000>; + compatible = "shared-dma-pool"; + reusable; + }; + + vga_memory: region@bf000000 { + no-map; + compatible = "shared-dma-pool"; + reg = <0xbf000000 0x01000000>; /* 16M */ + }; + }; + + voltage_mon_reg: voltage-mon-regulator { + compatible = "regulator-fixed"; + regulator-name = "ltc2497_reg"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + gpioI5mux: mux-controller { + compatible = "gpio-mux"; + #mux-control-cells = <0>; + mux-gpios = <&gpio0 ASPEED_GPIO(I, 5) GPIO_ACTIVE_HIGH>; + }; + + adc0mux: adc0mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 0>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc1mux: adc1mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 1>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc2mux: adc2mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 2>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc3mux: adc3mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 3>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc4mux: adc4mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 4>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc5mux: adc5mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 5>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc6mux: adc6mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 6>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc7mux: adc7mux { + compatible = "io-channel-mux"; + io-channels = <&adc0 7>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc8mux: adc8mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 0>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc9mux: adc9mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 1>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc10mux: adc10mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 2>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc11mux: adc11mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 3>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc12mux: adc12mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 4>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc13mux: adc13mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 5>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc14mux: adc14mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 6>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + adc15mux: adc15mux { + compatible = "io-channel-mux"; + io-channels = <&adc1 7>; + #io-channel-cells = <1>; + io-channel-names = "parent"; + mux-controls = <&gpioI5mux>; + channels = "s0", "s1"; + }; + + iio-hwmon { + compatible = "iio-hwmon"; + io-channels = <&adc0mux 0>, <&adc0mux 1>, + <&adc1mux 0>, <&adc1mux 1>, + <&adc2mux 0>, <&adc2mux 1>, + <&adc3mux 0>, <&adc3mux 1>, + <&adc4mux 0>, <&adc4mux 1>, + <&adc5mux 0>, <&adc5mux 1>, + <&adc6mux 0>, <&adc6mux 1>, + <&adc7mux 0>, <&adc7mux 1>, + <&adc8mux 0>, <&adc8mux 1>, + <&adc9mux 0>, <&adc9mux 1>, + <&adc10mux 0>, <&adc10mux 1>, + <&adc11mux 0>, <&adc11mux 1>, + <&adc12mux 0>, <&adc12mux 1>, + <&adc13mux 0>, <&adc13mux 1>, + <&adc14mux 0>, <&adc14mux 1>, + <&adc15mux 0>, <&adc15mux 1>, + <&adc_i2c 0>, <&adc_i2c 1>, + <&adc_i2c 2>, <&adc_i2c 3>, + <&adc_i2c 4>, <&adc_i2c 5>, + <&adc_i2c 6>, <&adc_i2c 7>, + <&adc_i2c 8>, <&adc_i2c 9>, + <&adc_i2c 10>, <&adc_i2c 11>, + <&adc_i2c 12>, <&adc_i2c 13>, + <&adc_i2c 14>, <&adc_i2c 15>; + }; +}; + +&mdio0 { + status = "okay"; + + ethphy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + }; +}; + +&mac0 { + status = "okay"; + + phy-mode = "rgmii"; + phy-handle = <ðphy0>; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_rgmii1_default>; +}; + +&fmc { + status = "okay"; + flash@0 { + status = "okay"; + m25p,fast-read; + label = "bmc"; + spi-max-frequency = <50000000>; +#include "openbmc-flash-layout-64.dtsi" + }; + + flash@1 { + status = "okay"; + m25p,fast-read; + label = "alt-bmc"; + spi-max-frequency = <50000000>; +#include "openbmc-flash-layout-64-alt.dtsi" + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spi1_default>; + + flash@0 { + status = "okay"; + m25p,fast-read; + label = "pnor"; + spi-max-frequency = <20000000>; + }; +}; + +&uart1 { + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&uart3 { + status = "okay"; +}; + +&uart4 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + temperature-sensor@2e { + compatible = "adi,adt7490"; + reg = <0x2e>; + }; +}; + +&i2c1 { + status = "okay"; +}; + +&i2c2 { + status = "okay"; + + psu@58 { + compatible = "pmbus"; + reg = <0x58>; + }; + + psu@59 { + compatible = "pmbus"; + reg = <0x59>; + }; +}; + +&i2c3 { + status = "okay"; +}; + +&i2c4 { + status = "okay"; + + adc_i2c: adc@16 { + compatible = "lltc,ltc2497"; + reg = <0x16>; + vref-supply = <&voltage_mon_reg>; + #io-channel-cells = <1>; + }; + + eeprom@50 { + compatible = "atmel,24c64"; + reg = <0x50>; + pagesize = <32>; + }; + + i2c-mux@70 { + compatible = "nxp,pca9545"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x70>; + i2c-mux-idle-disconnect; + + i2c4_bus70_chn0: i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0>; + + outlet_temp1: temperature-sensor@48 { + compatible = "ti,tmp75"; + reg = <0x48>; + }; + psu1_inlet_temp2: temperature-sensor@49 { + compatible = "ti,tmp75"; + reg = <0x49>; + }; + }; + + i2c4_bus70_chn1: i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x1>; + + pcie_zone_temp1: temperature-sensor@48 { + compatible = "ti,tmp75"; + reg = <0x48>; + }; + psu0_inlet_temp2: temperature-sensor@49 { + compatible = "ti,tmp75"; + reg = <0x49>; + }; + }; + + i2c4_bus70_chn2: i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x2>; + + pcie_zone_temp2: temperature-sensor@48 { + compatible = "ti,tmp75"; + reg = <0x48>; + }; + outlet_temp2: temperature-sensor@49 { + compatible = "ti,tmp75"; + reg = <0x49>; + }; + }; + + i2c4_bus70_chn3: i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x3>; + + mb_inlet_temp1: temperature-sensor@7c { + compatible = "microchip,emc1413"; + reg = <0x7c>; + }; + mb_inlet_temp2: temperature-sensor@4c { + compatible = "microchip,emc1413"; + reg = <0x4c>; + }; + }; + }; +}; + +&i2c5 { + status = "okay"; + + i2c-mux@70 { + compatible = "nxp,pca9548"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x70>; + i2c-mux-idle-disconnect; + }; +}; + +&i2c6 { + status = "okay"; + rtc@51 { + compatible = "nxp,pcf85063a"; + reg = <0x51>; + }; +}; + +&i2c7 { + status = "okay"; +}; + +&i2c9 { + status = "okay"; +}; + +&i2c11 { + status = "okay"; +}; + +&i2c14 { + status = "okay"; + eeprom@50 { + compatible = "atmel,24c64"; + reg = <0x50>; + pagesize = <32>; + }; + + bmc_ast2600_cpu: temperature-sensor@35 { + compatible = "ti,tmp175"; + reg = <0x35>; + }; +}; + +&adc0 { + ref_voltage = <2500>; + status = "okay"; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_adc0_default &pinctrl_adc1_default + &pinctrl_adc2_default &pinctrl_adc3_default + &pinctrl_adc4_default &pinctrl_adc5_default + &pinctrl_adc6_default &pinctrl_adc7_default>; +}; + +&adc1 { + ref_voltage = <2500>; + status = "okay"; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_adc8_default &pinctrl_adc9_default + &pinctrl_adc10_default &pinctrl_adc11_default + &pinctrl_adc12_default &pinctrl_adc13_default + &pinctrl_adc14_default &pinctrl_adc15_default>; +}; + +&vhub { + status = "okay"; +}; + +&video { + status = "okay"; + memory-region = <&video_engine_memory>; +}; + +&gpio0 { + gpio-line-names = + /*A0-A7*/ "","","","","","i2c2-reset-n","i2c6-reset-n","i2c4-reset-n", + /*B0-B7*/ "","","","","host0-sysreset-n","host0-pmin-n","","", + /*C0-C7*/ "s0-vrd-fault-n","s1-vrd-fault-n","","", + "irq-n","","vrd-sel","spd-sel", + /*D0-D7*/ "presence-ps0","presence-ps1","hsc-12vmain-alt2-n","ext-high-temp-n", + "","bmc-ncsi-txen","","", + /*E0-E7*/ "","","clk50m-bmc-ncsi","","","","","", + /*F0-F7*/ "s0-pcp-oc-warn-n","s1-pcp-oc-warn-n","power-chassis-control", + "cpu-bios-recover","s0-heartbeat","hs-csout-prochot", + "s0-vr-hot-n","s1-vr-hot-n", + /*G0-G7*/ "","","hsc-12vmain-alt1-n","","","","","", + /*H0-H7*/ "","","wd-disable-n","power-chassis-good","","","","", + /*I0-I7*/ "","","","","","adc-sw","power-button","rtc-battery-voltage-read-enable", + /*J0-J7*/ "","","","","","","","", + /*K0-K7*/ "","","","","","","","", + /*L0-L7*/ "","","","","","","","", + /*M0-M7*/ "","s0-ddr-save","soc-spi-nor-access","presence-cpu0", + "s0-rtc-lock","","","", + /*N0-N7*/ "hpm-fw-recovery","hpm-stby-rst-n","jtag-sel-s0","led-sw-hb", + "jtag-dbgr-prsnt-n","s1-heartbeat","","", + /*O0-O7*/ "","","","","","","","", + /*P0-P7*/ "ps0-ac-loss-n","ps1-ac-loss-n","","", + "led-fault","cpld-user-mode","jtag-srst-n","led-bmc-hb", + /*Q0-Q7*/ "","","","","","","","", + /*R0-R7*/ "","","","","","","","", + /*S0-S7*/ "","","identify-button","led-identify", + "s1-ddr-save","spi-nor-access","sys-pgood","presence-cpu1", + /*T0-T7*/ "","","","","","","","", + /*U0-U7*/ "","","","","","","","", + /*V0-V7*/ "s0-hightemp-n","s0-fault-alert","s0-sys-auth-failure-n", + "host0-reboot-ack-n","host0-ready","host0-shd-req-n", + "host0-shd-ack-n","s0-overtemp-n", + /*W0-W7*/ "ocp-aux-pwren","ocp-main-pwren","ocp-pgood","", + "bmc-ok","bmc-ready","spi0-program-sel","spi0-backup-sel", + /*X0-X7*/ "i2c-backup-sel","s1-fault-alert","s1-fw-boot-ok", + "s1-hightemp-n","s0-spi-auth-fail-n","s1-sys-auth-failure-n", + "s1-overtemp-n","s1-spi-auth-fail-n", + /*Y0-Y7*/ "","","","","","","","host0-special-boot", + /*Z0-Z7*/ "reset-button","ps0-pgood","ps1-pgood","","","","",""; +}; + +&gpio1 { + gpio-line-names = + /*18A0-18A7*/ "","","","","","","","", + /*18B0-18B7*/ "","","","","","","s0-soc-pgood","", + /*18C0-18C7*/ "uart1-mode0","uart1-mode1","uart2-mode0","uart2-mode1", + "uart3-mode0","uart3-mode1","uart4-mode0","uart4-mode1", + /*18D0-18D7*/ "","","","","","","","", + /*18E0-18E3*/ "","","",""; +}; diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts index 41d2b1535d9a1d79d9c1dfe8efcf1c4b57507710..1fc3e7cbf0d14edf51502cf4b17b12f88ff1fab7 100644 --- a/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts +++ b/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts @@ -7,6 +7,7 @@ #include #include #include +#include / { model = "Facebook Bletchley BMC"; @@ -792,11 +793,6 @@ reg = <0x4f>; }; - hdc1080@40 { - compatible = "ti,hdc1080"; - reg = <0x40>; - }; - front_leds: pca9552@67 { compatible = "nxp,pca9552"; reg = <0x67>; @@ -857,6 +853,13 @@ multi-master; aspeed,hw-timeout-ms = <1000>; status = "okay"; + + //USB Debug Connector + ipmb13@10 { + compatible = "ipmb-dev"; + reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>; + i2c-protocol; + }; }; &gpio0 { diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts index 27b43fe099f1492410468da2c4672f6bbb4302d5..8e1a1d1b282de4cbd25b6dd136ecd72cedf00a57 100644 --- a/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts +++ b/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts @@ -183,3 +183,21 @@ &i2c11 { status = "okay"; }; + +/* + * BMC's "mac3" controller is connected to BCM53134P's IMP_RGMII port + * directly (fixed link, no PHY in between). + * Note: BMC's "mdio0" controller is connected to BCM53134P's MDIO + * interface, and the MDIO channel will be enabled in dts later, when + * BCM53134 is added to "bcm53xx" DSA driver. + */ +&mac3 { + status = "okay"; + phy-mode = "rgmii"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_rgmii4_default>; + fixed-link { + speed = <1000>; + full-duplex; + }; +}; diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts index 8864e9c312a8b925a652d34b3b4ab42742aef433..6bf2ff85a40e724f17759a78f436a81c3e57a8c7 100644 --- a/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts +++ b/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts @@ -207,11 +207,16 @@ &i2c12 { status = "okay"; - //MEZZ_FRU - eeprom@51 { - compatible = "atmel,24c64"; - reg = <0x51>; - pagesize = <32>; +}; + +&i2c13 { + status = "okay"; + // Debug Card + multi-master; + ipmb13@10 { + compatible = "ipmb-dev"; + reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>; + i2c-protocol; }; }; diff --git a/arch/arm/boot/dts/aspeed-g5.dtsi b/arch/arm/boot/dts/aspeed-g5.dtsi index c89092c3905b58792b2eda42a820d8331d23f007..04f98d1dbb97c84c318c7e6a133fbf4572237c47 100644 --- a/arch/arm/boot/dts/aspeed-g5.dtsi +++ b/arch/arm/boot/dts/aspeed-g5.dtsi @@ -262,6 +262,14 @@ quality = <100>; }; + hace: crypto@1e6e3000 { + compatible = "aspeed,ast2500-hace"; + reg = <0x1e6e3000 0x100>; + interrupts = <4>; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; + gfx: display@1e6e6000 { compatible = "aspeed,ast2500-gfx", "syscon"; reg = <0x1e6e6000 0x1000>; diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi index 6660564855ff81cd9895b6c35d8bb6d8d6d885a8..ebbcfe445d9cdac63ef97f48e4f319e96f3bda98 100644 --- a/arch/arm/boot/dts/aspeed-g6.dtsi +++ b/arch/arm/boot/dts/aspeed-g6.dtsi @@ -323,6 +323,14 @@ #size-cells = <1>; ranges; + hace: crypto@1e6d0000 { + compatible = "aspeed,ast2600-hace"; + reg = <0x1e6d0000 0x200>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_YCLK>; + resets = <&syscon ASPEED_RESET_HACE>; + }; + syscon: syscon@1e6e2000 { compatible = "aspeed,ast2600-scu", "syscon", "simple-mfd"; reg = <0x1e6e2000 0x1000>; @@ -756,6 +764,62 @@ status = "disabled"; }; + uart6: serial@1e790000 { + compatible = "ns16550a"; + reg = <0x1e790000 0x20>; + reg-shift = <2>; + reg-io-width = <4>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_UART6CLK>; + no-loopback-test; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart6_default>; + + status = "disabled"; + }; + + uart7: serial@1e790100 { + compatible = "ns16550a"; + reg = <0x1e790100 0x20>; + reg-shift = <2>; + reg-io-width = <4>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_UART7CLK>; + no-loopback-test; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart7_default>; + + status = "disabled"; + }; + + uart8: serial@1e790200 { + compatible = "ns16550a"; + reg = <0x1e790200 0x20>; + reg-shift = <2>; + reg-io-width = <4>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_UART8CLK>; + no-loopback-test; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart8_default>; + + status = "disabled"; + }; + + uart9: serial@1e790300 { + compatible = "ns16550a"; + reg = <0x1e790300 0x20>; + reg-shift = <2>; + reg-io-width = <4>; + interrupts = ; + clocks = <&syscon ASPEED_CLK_GATE_UART9CLK>; + no-loopback-test; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart9_default>; + + status = "disabled"; + }; + i2c: bus@1e78a000 { compatible = "simple-bus"; #address-cells = <1>; diff --git a/arch/arm/boot/dts/at91-sam9x60ek.dts b/arch/arm/boot/dts/at91-sam9x60ek.dts index 81c38e101f586d6e7c12536a5b003d8f92904b4b..4ba52ba11dc62379457a9a0ff5cc22007ebe5772 100644 --- a/arch/arm/boot/dts/at91-sam9x60ek.dts +++ b/arch/arm/boot/dts/at91-sam9x60ek.dts @@ -34,48 +34,6 @@ }; }; - regulators: regulators { - compatible = "simple-bus"; - #address-cells = <1>; - #size-cells = <0>; - - vdd_1v8: fixed-regulator-vdd_1v8@0 { - compatible = "regulator-fixed"; - regulator-name = "VDD_1V8"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-always-on; - status = "okay"; - }; - - vdd_1v15: fixed-regulator-vdd_1v15@1 { - compatible = "regulator-fixed"; - regulator-name = "VDD_1V15"; - regulator-min-microvolt = <1150000>; - regulator-max-microvolt = <1150000>; - regulator-always-on; - status = "okay"; - }; - - vdd1_3v3: fixed-regulator-vdd1_3v3@2 { - compatible = "regulator-fixed"; - regulator-name = "VDD1_3V3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-always-on; - status = "okay"; - }; - - vdd2_3v3: regulator-fixed-vdd2_3v3@3 { - compatible = "regulator-fixed"; - regulator-name = "VDD2_3V3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-always-on; - status = "okay"; - }; - }; - gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; @@ -111,6 +69,42 @@ linux,default-trigger = "heartbeat"; }; }; + + vdd_1v8: fixed-regulator-vdd_1v8 { + compatible = "regulator-fixed"; + regulator-name = "VDD_1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + status = "okay"; + }; + + vdd_1v15: fixed-regulator-vdd_1v15 { + compatible = "regulator-fixed"; + regulator-name = "VDD_1V15"; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + regulator-always-on; + status = "okay"; + }; + + vdd1_3v3: fixed-regulator-vdd1_3v3 { + compatible = "regulator-fixed"; + regulator-name = "VDD1_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + status = "okay"; + }; + + vdd2_3v3: regulator-fixed-vdd2_3v3 { + compatible = "regulator-fixed"; + regulator-name = "VDD2_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + status = "okay"; + }; }; &adc { @@ -264,8 +258,9 @@ status = "okay"; uart1: serial@200 { - compatible = "microchip,sam9x60-usart", "atmel,at91sam9260-usart"; + compatible = "microchip,sam9x60-dbgu", "microchip,sam9x60-usart", "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <14 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | diff --git a/arch/arm/boot/dts/at91-sama5d27_wlsom1.dtsi b/arch/arm/boot/dts/at91-sama5d27_wlsom1.dtsi index 76b2025c67b4ce18c5a9ed65d3f5b88b44cab216..83bcf9fe0152052f685f0f3d8f2d1d1922361ab7 100644 --- a/arch/arm/boot/dts/at91-sama5d27_wlsom1.dtsi +++ b/arch/arm/boot/dts/at91-sama5d27_wlsom1.dtsi @@ -76,8 +76,8 @@ regulators { vdd_3v3: VDD_IO { regulator-name = "VDD_IO"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -95,8 +95,8 @@ vddio_ddr: VDD_DDR { regulator-name = "VDD_DDR"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -118,8 +118,8 @@ vdd_core: VDD_CORE { regulator-name = "VDD_CORE"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1250000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -160,8 +160,8 @@ LDO1 { regulator-name = "LDO1"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; regulator-state-standby { @@ -175,9 +175,8 @@ LDO2 { regulator-name = "LDO2"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; - regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; regulator-state-standby { regulator-on-in-suspend; diff --git a/arch/arm/boot/dts/at91-sama5d2_icp.dts b/arch/arm/boot/dts/at91-sama5d2_icp.dts index 6865be8d7787dad56946cd95211c014562981233..dd1dec9d4e07f293a86f2b4a12e2d67c9e28dcfe 100644 --- a/arch/arm/boot/dts/at91-sama5d2_icp.dts +++ b/arch/arm/boot/dts/at91-sama5d2_icp.dts @@ -196,8 +196,8 @@ regulators { vdd_io_reg: VDD_IO { regulator-name = "VDD_IO"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -215,8 +215,8 @@ VDD_DDR { regulator-name = "VDD_DDR"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -234,8 +234,8 @@ VDD_CORE { regulator-name = "VDD_CORE"; - regulator-min-microvolt = <600000>; - regulator-max-microvolt = <1850000>; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1250000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -257,7 +257,6 @@ regulator-max-microvolt = <1850000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; - regulator-always-on; regulator-state-standby { regulator-on-in-suspend; @@ -272,8 +271,8 @@ LDO1 { regulator-name = "LDO1"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; regulator-always-on; regulator-state-standby { @@ -287,8 +286,8 @@ LDO2 { regulator-name = "LDO2"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-always-on; regulator-state-standby { diff --git a/arch/arm/boot/dts/at91-sama5d3_eds.dts b/arch/arm/boot/dts/at91-sama5d3_eds.dts new file mode 100644 index 0000000000000000000000000000000000000000..c287b03d768b602962da6d77905d41c59503e16d --- /dev/null +++ b/arch/arm/boot/dts/at91-sama5d3_eds.dts @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * at91-sama5d3_eds.dts - Device Tree file for the SAMA5D3 Ethernet + * Development System board. + * + * Copyright (C) 2022 Microchip Technology Inc. and its subsidiaries + * + * Author: Jerry Ray + */ +/dts-v1/; +#include "sama5d36.dtsi" + +/ { + model = "SAMA5D3 Ethernet Development System"; + compatible = "microchip,sama5d3-eds", "atmel,sama5d36", + "atmel,sama5d3", "atmel,sama5"; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_key_gpio>; + + button-3 { + label = "PB_USER"; + gpios = <&pioE 29 GPIO_ACTIVE_LOW>; + linux,code = <0x104>; + wakeup-source; + }; + }; + + memory@20000000 { + reg = <0x20000000 0x10000000>; + }; + + vcc_3v3_reg: regulator-1 { + compatible = "regulator-fixed"; + regulator-name = "VCC_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + vcc_2v5_reg: regulator-2 { + compatible = "regulator-fixed"; + regulator-name = "VCC_2V5"; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + regulator-always-on; + vin-supply = <&vcc_3v3_reg>; + }; + + vcc_1v8_reg: regulator-3 { + compatible = "regulator-fixed"; + regulator-name = "VCC_1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + vin-supply = <&vcc_3v3_reg>; + }; + + vcc_1v2_reg: regulator-4 { + compatible = "regulator-fixed"; + regulator-name = "VCC_1V2"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-always-on; + }; + + vcc_mmc0_reg: regulator-5 { + compatible = "regulator-fixed"; + regulator-name = "mmc0-card-supply"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_vcc_mmc0_reg_gpio>; + gpio = <&pioE 2 GPIO_ACTIVE_LOW>; + }; +}; + +&can0 { + status = "okay"; +}; + +&dbgu { + status = "okay"; +}; + +&ebi { + pinctrl-0 = <&pinctrl_ebi_nand_addr>; + pinctrl-names = "default"; + status = "okay"; + + nand_controller: nand-controller { + status = "okay"; + + nand@3 { + reg = <0x3 0x0 0x2>; + atmel,rb = <0>; + nand-bus-width = <8>; + nand-ecc-mode = "hw"; + nand-ecc-strength = <4>; + nand-ecc-step-size = <512>; + nand-on-flash-bbt; + label = "atmel_nand"; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + at91bootstrap@0 { + label = "at91bootstrap"; + reg = <0x0 0x40000>; + }; + + bootloader@40000 { + label = "bootloader"; + reg = <0x40000 0xc0000>; + }; + + bootloaderenvred@100000 { + label = "bootloader env redundant"; + reg = <0x100000 0x40000>; + }; + + bootloaderenv@140000 { + label = "bootloader env"; + reg = <0x140000 0x40000>; + }; + + dtb@180000 { + label = "device tree"; + reg = <0x180000 0x80000>; + }; + + kernel@200000 { + label = "kernel"; + reg = <0x200000 0x600000>; + }; + + rootfs@800000 { + label = "rootfs"; + reg = <0x800000 0x0f800000>; + }; + }; + }; + }; +}; + +&i2c0 { + pinctrl-0 = <&pinctrl_i2c0_pu>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&pinctrl_i2c2_pu>; + status = "okay"; +}; + +&main_xtal { + clock-frequency = <12000000>; +}; + +&mmc0 { + pinctrl-0 = <&pinctrl_mmc0_clk_cmd_dat0 &pinctrl_mmc0_dat1_3 + &pinctrl_mmc0_dat4_7 &pinctrl_mmc0_cd>; + vmmc-supply = <&vcc_mmc0_reg>; + vqmmc-supply = <&vcc_3v3_reg>; + status = "okay"; + slot@0 { + reg = <0>; + bus-width = <8>; + cd-gpios = <&pioE 0 GPIO_ACTIVE_LOW>; + }; +}; + +&pinctrl { + board { + pinctrl_i2c0_pu: i2c0-pu { + atmel,pins = + , + ; + }; + + pinctrl_i2c2_pu: i2c2-pu { + atmel,pins = + , + ; + }; + + pinctrl_key_gpio: key-gpio-0 { + atmel,pins = + ; + }; + + pinctrl_mmc0_cd: mmc0-cd { + atmel,pins = + ; + }; + + /* Reserved for reset signal to the RGMII connector. */ + pinctrl_rgmii_rstn: rgmii-rstn { + atmel,pins = + ; + }; + + /* Reserved for an interrupt line from the RMII and RGMII connectors. */ + pinctrl_spi_irqn: spi-irqn { + atmel,pins = + ; + }; + + pinctrl_spi0_cs: spi0-cs-default { + atmel,pins = + ; + }; + + pinctrl_spi1_cs: spi1-cs-default { + atmel,pins = ; + }; + + pinctrl_usba_vbus: usba-vbus { + atmel,pins = + ; + }; + + pinctrl_usb_default: usb-default { + atmel,pins = + ; + }; + + /* Reserved for VBUS fault interrupt. */ + pinctrl_vbusfault_irqn: vbusfault-irqn { + atmel,pins = + ; + }; + + pinctrl_vcc_mmc0_reg_gpio: vcc-mmc0-reg-gpio-default { + atmel,pins = ; + }; + }; +}; + +&slow_xtal { + clock-frequency = <32768>; +}; + +&spi0 { + pinctrl-names = "default", "cs"; + pinctrl-1 = <&pinctrl_spi0_cs>; + cs-gpios = <&pioD 13 0>, <0>, <0>, <&pioD 16 0>; + status = "okay"; +}; + +&spi1 { + pinctrl-names = "default", "cs"; + pinctrl-1 = <&pinctrl_spi1_cs>; + cs-gpios = <&pioC 25 0>, <0>, <0>, <&pioC 28 0>; + status = "okay"; +}; + +&tcb0 { + timer0: timer@0 { + compatible = "atmel,tcb-timer"; + reg = <0>; + }; + + timer1: timer@1 { + compatible = "atmel,tcb-timer"; + reg = <1>; + }; +}; + +&usb0 { /* USB Device port with VBUS detection. */ + atmel,vbus-gpio = <&pioE 9 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usba_vbus>; + status = "okay"; +}; + +&usb1 { /* 3-port Host. First port is unused. */ + atmel,vbus-gpio = <0 + &pioE 3 GPIO_ACTIVE_HIGH + &pioE 4 GPIO_ACTIVE_HIGH + >; + num-ports = <3>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb_default>; + status = "okay"; +}; + +&usb2 { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/at91-sama7g5ek.dts b/arch/arm/boot/dts/at91-sama7g5ek.dts index de44da2e4aae24dc3e9ea086447ffa13a5618ffa..3b25c67795ddbe56fe394b6fff6a56047367bd94 100644 --- a/arch/arm/boot/dts/at91-sama7g5ek.dts +++ b/arch/arm/boot/dts/at91-sama7g5ek.dts @@ -244,8 +244,8 @@ regulators { vdd_3v3: VDD_IO { regulator-name = "VDD_IO"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -264,8 +264,8 @@ vddioddr: VDD_DDR { regulator-name = "VDD_DDR"; - regulator-min-microvolt = <1300000>; - regulator-max-microvolt = <1450000>; + regulator-min-microvolt = <1350000>; + regulator-max-microvolt = <1350000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -285,8 +285,8 @@ vddcore: VDD_CORE { regulator-name = "VDD_CORE"; - regulator-min-microvolt = <1100000>; - regulator-max-microvolt = <1850000>; + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-always-on; @@ -306,7 +306,7 @@ vddcpu: VDD_OTHER { regulator-name = "VDD_OTHER"; regulator-min-microvolt = <1050000>; - regulator-max-microvolt = <1850000>; + regulator-max-microvolt = <1250000>; regulator-initial-mode = <2>; regulator-allowed-modes = <2>, <4>; regulator-ramp-delay = <3125>; @@ -326,8 +326,8 @@ vldo1: LDO1 { regulator-name = "LDO1"; - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <3700000>; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; regulator-always-on; regulator-state-standby { diff --git a/arch/arm/boot/dts/at91rm9200.dtsi b/arch/arm/boot/dts/at91rm9200.dtsi index d1181ead18e5a86e0ada25d6ea287dffdca921f0..7a113325abb9ed958274e59a1abf477060a268eb 100644 --- a/arch/arm/boot/dts/at91rm9200.dtsi +++ b/arch/arm/boot/dts/at91rm9200.dtsi @@ -13,6 +13,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -596,6 +597,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91rm9200-dbgu", "atmel,at91rm9200-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; @@ -607,6 +609,7 @@ usart0: serial@fffc0000 { compatible = "atmel,at91rm9200-usart"; reg = <0xfffc0000 0x200>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -620,6 +623,7 @@ usart1: serial@fffc4000 { compatible = "atmel,at91rm9200-usart"; reg = <0xfffc4000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -633,6 +637,7 @@ usart2: serial@fffc8000 { compatible = "atmel,at91rm9200-usart"; reg = <0xfffc8000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -646,6 +651,7 @@ usart3: serial@fffcc000 { compatible = "atmel,at91rm9200-usart"; reg = <0xfffcc000 0x200>; + atmel,usart-mode = ; interrupts = <23 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi index 9d9820db9482c59322c233a4f445784ba857aeab..789fe356dbf60365d427968f9f926bd308ce3170 100644 --- a/arch/arm/boot/dts/at91sam9260.dtsi +++ b/arch/arm/boot/dts/at91sam9260.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -532,6 +533,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; @@ -543,6 +545,7 @@ usart0: serial@fffb0000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb0000 0x200>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -556,6 +559,7 @@ usart1: serial@fffb4000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb4000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -569,6 +573,7 @@ usart2: serial@fffb8000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb8000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -582,6 +587,7 @@ usart3: serial@fffd0000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffd0000 0x200>; + atmel,usart-mode = ; interrupts = <23 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -595,6 +601,7 @@ uart0: serial@fffd4000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffd4000 0x200>; + atmel,usart-mode = ; interrupts = <24 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -608,6 +615,7 @@ uart1: serial@fffd8000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffd8000 0x200>; + atmel,usart-mode = ; interrupts = <25 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; diff --git a/arch/arm/boot/dts/at91sam9261.dtsi b/arch/arm/boot/dts/at91sam9261.dtsi index 259aca5653051f8db6d7c4b302b595b512bfd7bb..ee0bd1aceb3f0cc16d12a926fc3d5a8d7f9b862b 100644 --- a/arch/arm/boot/dts/at91sam9261.dtsi +++ b/arch/arm/boot/dts/at91sam9261.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -179,6 +180,7 @@ usart0: serial@fffb0000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb0000 0x200>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -192,6 +194,7 @@ usart1: serial@fffb4000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb4000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -205,6 +208,7 @@ usart2: serial@fffb8000{ compatible = "atmel,at91sam9260-usart"; reg = <0xfffb8000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -301,6 +305,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi index c080df8c2312b9a74dd7d97df958b4e7ebd49584..3ce9ea9873129a0698811a2ab930c0a34a3ac048 100644 --- a/arch/arm/boot/dts/at91sam9263.dtsi +++ b/arch/arm/boot/dts/at91sam9263.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -540,6 +541,7 @@ dbgu: serial@ffffee00 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xffffee00 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; @@ -551,6 +553,7 @@ usart0: serial@fff8c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff8c000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -564,6 +567,7 @@ usart1: serial@fff90000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff90000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -577,6 +581,7 @@ usart2: serial@fff94000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff94000 0x200>; + atmel,usart-mode = ; interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi index 09794561c7ceb2dd6edd5f364c98eb56ff4aa78d..95f5d76234dbbdd6e92a8dae4cd95d1880799683 100644 --- a/arch/arm/boot/dts/at91sam9g45.dtsi +++ b/arch/arm/boot/dts/at91sam9g45.dtsi @@ -13,6 +13,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -675,6 +676,7 @@ dbgu: serial@ffffee00 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; + atmel,usart-mode = ; reg = <0xffffee00 0x200>; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; @@ -687,6 +689,7 @@ usart0: serial@fff8c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff8c000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -700,6 +703,7 @@ usart1: serial@fff90000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff90000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -713,6 +717,7 @@ usart2: serial@fff94000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff94000 0x200>; + atmel,usart-mode = ; interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -726,6 +731,7 @@ usart3: serial@fff98000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfff98000 0x200>; + atmel,usart-mode = ; interrupts = <10 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; diff --git a/arch/arm/boot/dts/at91sam9n12.dtsi b/arch/arm/boot/dts/at91sam9n12.dtsi index 556f35ce49e34a4c7952a09e96d3a08909bf2145..83114d26f10d027179ac91d65bbc2290fb1d62fa 100644 --- a/arch/arm/boot/dts/at91sam9n12.dtsi +++ b/arch/arm/boot/dts/at91sam9n12.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -593,6 +594,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; @@ -618,6 +620,7 @@ usart0: serial@f801c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf801c000 0x4000>; + atmel,usart-mode = ; interrupts = <5 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart0>; @@ -629,6 +632,7 @@ usart1: serial@f8020000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8020000 0x4000>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart1>; @@ -640,6 +644,7 @@ usart2: serial@f8024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8024000 0x4000>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart2>; @@ -651,6 +656,7 @@ usart3: serial@f8028000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8028000 0x4000>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart3>; diff --git a/arch/arm/boot/dts/at91sam9rl.dtsi b/arch/arm/boot/dts/at91sam9rl.dtsi index 12c634811820120f27f0f948de36afdfee972b9c..364a2ff0a763d9105da86fabede181ec5078459d 100644 --- a/arch/arm/boot/dts/at91sam9rl.dtsi +++ b/arch/arm/boot/dts/at91sam9rl.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -175,6 +176,7 @@ usart0: serial@fffb0000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb0000 0x200>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -188,6 +190,7 @@ usart1: serial@fffb4000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb4000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -201,6 +204,7 @@ usart2: serial@fffb8000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffb8000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -214,6 +218,7 @@ usart3: serial@fffbc000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfffbc000 0x200>; + atmel,usart-mode = ; interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>; atmel,use-dma-rx; atmel,use-dma-tx; @@ -322,6 +327,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; diff --git a/arch/arm/boot/dts/at91sam9x5.dtsi b/arch/arm/boot/dts/at91sam9x5.dtsi index ea3b11336c794f10703cfd15080555245a7a761f..0c26c925761b27822142388eb06d979a14ec9da4 100644 --- a/arch/arm/boot/dts/at91sam9x5.dtsi +++ b/arch/arm/boot/dts/at91sam9x5.dtsi @@ -13,6 +13,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -674,6 +675,7 @@ dbgu: serial@fffff200 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; @@ -688,6 +690,7 @@ usart0: serial@f801c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf801c000 0x200>; + atmel,usart-mode = ; interrupts = <5 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart0>; @@ -702,6 +705,7 @@ usart1: serial@f8020000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8020000 0x200>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart1>; @@ -716,6 +720,7 @@ usart2: serial@f8024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8024000 0x200>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart2>; @@ -775,6 +780,7 @@ uart0: serial@f8040000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8040000 0x200>; + atmel,usart-mode = ; interrupts = <15 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; @@ -786,6 +792,7 @@ uart1: serial@f8044000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8044000 0x200>; + atmel,usart-mode = ; interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; diff --git a/arch/arm/boot/dts/at91sam9x5_usart3.dtsi b/arch/arm/boot/dts/at91sam9x5_usart3.dtsi index 098d3fef5c371dcf6fb204e3f98c1e646d62675c..a47c765e1b20a825177c613d4b93a6392c393a75 100644 --- a/arch/arm/boot/dts/at91sam9x5_usart3.dtsi +++ b/arch/arm/boot/dts/at91sam9x5_usart3.dtsi @@ -8,6 +8,7 @@ #include #include +#include / { aliases { @@ -44,6 +45,7 @@ usart3: serial@f8028000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8028000 0x200>; + atmel,usart-mode = ; interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usart3>; diff --git a/arch/arm/boot/dts/bcm63178.dtsi b/arch/arm/boot/dts/bcm63178.dtsi index 5463443f07620e9af98814ff2e85c071ccc8b40e..cbd094dde6d07e2715d03510a15795ba4b656508 100644 --- a/arch/arm/boot/dts/bcm63178.dtsi +++ b/arch/arm/boot/dts/bcm63178.dtsi @@ -32,6 +32,7 @@ next-level-cache = <&L2_0>; enable-method = "psci"; }; + CA7_2: cpu@2 { device_type = "cpu"; compatible = "arm,cortex-a7"; @@ -39,6 +40,7 @@ next-level-cache = <&L2_0>; enable-method = "psci"; }; + L2_0: l2-cache0 { compatible = "cache"; }; @@ -46,10 +48,10 @@ timer { compatible = "arm,armv7-timer"; - interrupts = , - , - , - ; + interrupts = , + , + , + ; arm,cpu-registers-not-fw-configured; }; @@ -80,23 +82,23 @@ psci { compatible = "arm,psci-0.2"; method = "smc"; - cpu_off = <1>; - cpu_on = <2>; }; axi@81000000 { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; - ranges = <0 0x81000000 0x4000>; + ranges = <0 0x81000000 0x8000>; gic: interrupt-controller@1000 { compatible = "arm,cortex-a7-gic"; #interrupt-cells = <3>; - #address-cells = <0>; interrupt-controller; + interrupts = ; reg = <0x1000 0x1000>, - <0x2000 0x2000>; + <0x2000 0x2000>, + <0x4000 0x2000>, + <0x6000 0x2000>; }; }; diff --git a/arch/arm/boot/dts/bcm6846.dtsi b/arch/arm/boot/dts/bcm6846.dtsi index e610c102498fac9f70b90f323f1a3fd6ea1984f3..8aa47a2583b2985276ae7f3685237b2afdac0f6e 100644 --- a/arch/arm/boot/dts/bcm6846.dtsi +++ b/arch/arm/boot/dts/bcm6846.dtsi @@ -40,10 +40,10 @@ timer { compatible = "arm,armv7-timer"; - interrupts = , - , - , - ; + interrupts = , + , + , + ; arm,cpu-registers-not-fw-configured; }; @@ -65,23 +65,23 @@ psci { compatible = "arm,psci-0.2"; method = "smc"; - cpu_off = <1>; - cpu_on = <2>; }; axi@81000000 { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; - ranges = <0 0x81000000 0x4000>; + ranges = <0 0x81000000 0x8000>; gic: interrupt-controller@1000 { compatible = "arm,cortex-a7-gic"; #interrupt-cells = <3>; - #address-cells = <0>; interrupt-controller; + interrupts = ; reg = <0x1000 0x1000>, - <0x2000 0x2000>; + <0x2000 0x2000>, + <0x4000 0x2000>, + <0x6000 0x2000>; }; }; diff --git a/arch/arm/boot/dts/bcm6878.dtsi b/arch/arm/boot/dts/bcm6878.dtsi index a7dff596fe1e673505be4fa6438c86417a419d70..1e8b5fa96c256f57ff5830ecaf3a91d6900c2e1e 100644 --- a/arch/arm/boot/dts/bcm6878.dtsi +++ b/arch/arm/boot/dts/bcm6878.dtsi @@ -32,6 +32,7 @@ next-level-cache = <&L2_0>; enable-method = "psci"; }; + L2_0: l2-cache0 { compatible = "cache"; }; @@ -39,10 +40,10 @@ timer { compatible = "arm,armv7-timer"; - interrupts = , - , - , - ; + interrupts = , + , + , + ; arm,cpu-registers-not-fw-configured; }; diff --git a/arch/arm/boot/dts/dove.dtsi b/arch/arm/boot/dts/dove.dtsi index 89e0bdaf3a85fec8fff43db93d304755c6f7b949..00a36fba2fd234e447e227ec01881310ae49cd73 100644 --- a/arch/arm/boot/dts/dove.dtsi +++ b/arch/arm/boot/dts/dove.dtsi @@ -122,8 +122,18 @@ bus-range = <0x00 0xff>; #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 16>; + interrupt-names = "intx", "error"; + interrupts = <16>, <15>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie0_intc 0>, + <0 0 0 2 &pcie0_intc 1>, + <0 0 0 3 &pcie0_intc 2>, + <0 0 0 4 &pcie0_intc 3>; + + pcie0_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie1: pcie@2 { @@ -141,8 +151,18 @@ bus-range = <0x00 0xff>; #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 18>; + interrupt-names = "intx", "error"; + interrupts = <18>, <17>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; diff --git a/arch/arm/boot/dts/exynos4412-midas.dtsi b/arch/arm/boot/dts/exynos4412-midas.dtsi index b967397a46c5be7a64f4e68bebbdad96902039d2..8e1c19a8ad06da8d072ee9f1d25fabc7a174b323 100644 --- a/arch/arm/boot/dts/exynos4412-midas.dtsi +++ b/arch/arm/boot/dts/exynos4412-midas.dtsi @@ -586,7 +586,7 @@ clocks = <&camera 1>; clock-names = "extclk"; samsung,camclk-out = <1>; - gpios = <&gpm1 6 GPIO_ACTIVE_HIGH>; + gpios = <&gpm1 6 GPIO_ACTIVE_LOW>; port { is_s5k6a3_ep: endpoint { diff --git a/arch/arm/boot/dts/exynos4412-origen.dts b/arch/arm/boot/dts/exynos4412-origen.dts index 6db09dba07ffdcbb3c2b9e152883c3dc4228ce87..a3905e27b9cd9e2728138a3d79aa5aee6d7eeab2 100644 --- a/arch/arm/boot/dts/exynos4412-origen.dts +++ b/arch/arm/boot/dts/exynos4412-origen.dts @@ -95,7 +95,7 @@ }; &ehci { - samsung,vbus-gpio = <&gpx3 5 1>; + samsung,vbus-gpio = <&gpx3 5 GPIO_ACTIVE_HIGH>; status = "okay"; phys = <&exynos_usbphy 2>, <&exynos_usbphy 3>; phy-names = "hsic0", "hsic1"; diff --git a/arch/arm/boot/dts/gemini-ns2502.dts b/arch/arm/boot/dts/gemini-ns2502.dts index 704abd212df50dd16055e653b305832eda29b9d1..e6eeb35e88198118d442fc581c68a84c889b67c6 100644 --- a/arch/arm/boot/dts/gemini-ns2502.dts +++ b/arch/arm/boot/dts/gemini-ns2502.dts @@ -39,10 +39,6 @@ phy0: ethernet-phy@1 { reg = <1>; device_type = "ethernet-phy"; - /* We lack the knowledge of necessary GPIO to achieve - * Gigabit - */ - max-speed = <100>; }; }; }; @@ -50,7 +46,7 @@ ðernet { status = "okay"; ethernet-port@0 { - phy-mode = "rgmii"; + phy-mode = "rgmii-id"; phy-handle = <&phy0>; }; }; @@ -65,30 +61,9 @@ pinctrl-1 = <&pflash_disabled_pins>; partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - partition@0 { - label = "RedBoot"; - reg = <0x00000000 0x00020000>; - }; - partition@20000 { - label = "kernel"; - reg = <0x00020000 0x00700000>; - }; - partition@720000 { - label = "VCTL"; - reg = <0x00720000 0x00020000>; - }; - partition@740000 { - label = "CurConf"; - reg = <0x00740000 0x000a0000>; - }; - partition@7e0000 { - label = "FIS"; - reg = <0x007e0000 0x00010000>; - }; + compatible = "redboot-fis"; + /* Eraseblock at 0x7e0000 */ + fis-index-block = <0x3f>; }; }; diff --git a/arch/arm/boot/dts/gemini-ssi1328.dts b/arch/arm/boot/dts/gemini-ssi1328.dts index 2b3e7db84fedfb9dc5d30c46a8feef6747a073b6..42e85f07cf764677c9037effe28b1e76bb80cc1b 100644 --- a/arch/arm/boot/dts/gemini-ssi1328.dts +++ b/arch/arm/boot/dts/gemini-ssi1328.dts @@ -40,10 +40,6 @@ phy0: ethernet-phy@1 { reg = <1>; device_type = "ethernet-phy"; - /* We lack the knowledge of necessary GPIO to achieve - * Gigabit - */ - max-speed = <100>; }; /* WAN ICPlus IP101A */ phy1: ethernet-phy@2 { diff --git a/arch/arm/boot/dts/imx23-xfi3.dts b/arch/arm/boot/dts/imx23-xfi3.dts index a6213c590f9469eba50cc9dc1ac0677d59d8dc5e..b1d8210f3ecc84e7654f7168bc3ddddba2693a9e 100644 --- a/arch/arm/boot/dts/imx23-xfi3.dts +++ b/arch/arm/boot/dts/imx23-xfi3.dts @@ -158,19 +158,19 @@ default-brightness-level = <6>; }; - gpio_keys { + gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&key_pins_a>; - voldown { + key-voldown { label = "volume-down"; linux,code = <114>; gpios = <&gpio2 7 0>; debounce-interval = <20>; }; - volup { + key-volup { label = "volume-up"; linux,code = <115>; gpios = <&gpio2 8 0>; diff --git a/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts b/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts index 3f38c2e60a745af37743022b4de671849e6efb11..c7207ea437c404399213b2f4939af22159e1ea68 100644 --- a/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts +++ b/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts @@ -13,12 +13,12 @@ model = "Eukrea MBIMXSD25"; compatible = "eukrea,mbimxsd25-baseboard", "eukrea,cpuimx25", "fsl,imx25"; - gpio_keys { + gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpiokeys>; - bp1 { + button { label = "BP1"; gpios = <&gpio3 18 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi index bc4de0c05511b46a40ff7199b1de795ac60a85b2..5f90d72b840b0e6d8035107918ac164da540a088 100644 --- a/arch/arm/boot/dts/imx25.dtsi +++ b/arch/arm/boot/dts/imx25.dtsi @@ -515,7 +515,7 @@ #interrupt-cells = <2>; }; - sdma: sdma@53fd4000 { + sdma: dma-controller@53fd4000 { compatible = "fsl,imx25-sdma"; reg = <0x53fd4000 0x4000>; clocks = <&clks 112>, <&clks 68>; diff --git a/arch/arm/boot/dts/imx28-cfa10049.dts b/arch/arm/boot/dts/imx28-cfa10049.dts index a92b05ef390fe31af094f73b7f457cfa86c5d50c..9ef0d567ea480599d834c8cd0fa7d11a6c12c41b 100644 --- a/arch/arm/boot/dts/imx28-cfa10049.dts +++ b/arch/arm/boot/dts/imx28-cfa10049.dts @@ -327,7 +327,7 @@ }; }; - spi2 { + spi-2 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&spi2_pins_cfa10049>; @@ -351,7 +351,7 @@ }; }; - spi3 { + spi-3 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&spi3_pins_cfa10049>; @@ -388,12 +388,12 @@ }; }; - gpio_keys { + gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&rotary_btn_pins_cfa10049>; - rotary_button { + rotary-button { label = "rotary_button"; gpios = <&gpio3 26 1>; debounce-interval = <10>; diff --git a/arch/arm/boot/dts/imx28-cfa10055.dts b/arch/arm/boot/dts/imx28-cfa10055.dts index d05c370dfc178cf89a7e0d8f4da666730ff132dd..fac5bbda7a93d13e8c8f35ad3e74cb94344110b8 100644 --- a/arch/arm/boot/dts/imx28-cfa10055.dts +++ b/arch/arm/boot/dts/imx28-cfa10055.dts @@ -129,7 +129,7 @@ }; }; - spi2 { + spi-2 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&spi2_pins_cfa10055>; diff --git a/arch/arm/boot/dts/imx28-cfa10056.dts b/arch/arm/boot/dts/imx28-cfa10056.dts index c1060bd5f17f5dba96938a1ae5337fe17bd62bec..c5f3337e8b392b9052dd9f0226268068eb3b610a 100644 --- a/arch/arm/boot/dts/imx28-cfa10056.dts +++ b/arch/arm/boot/dts/imx28-cfa10056.dts @@ -88,7 +88,7 @@ }; }; - spi2 { + spi-2 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&spi2_pins_cfa10056>; diff --git a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts index bacb846f99e3c48da88ed2dfca668782b638ca0d..73f521c46c1eed0192fd48dcdc9b6ce89b371e2b 100644 --- a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts +++ b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts @@ -204,7 +204,7 @@ pinctrl-names = "default"; pinctrl-0 = <&enocean_button>; - enocean { + key-enocean { label = "EnOcean"; linux,code = ; gpios = <&gpio3 3 GPIO_ACTIVE_HIGH>; diff --git a/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi b/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi index 3280fddaaf0dea9c8e3d8b9403c9bf548f7d789f..b285a946e2c2a5983cc72b79347b532993845005 100644 --- a/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi +++ b/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi @@ -19,12 +19,12 @@ default-brightness-level = <10>; }; - button-sw3 { + gpio-keys-0 { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&gpio_button_sw3_pins_mbmx28lc>; - sw3 { + switch-sw3 { label = "SW3"; gpios = <&gpio1 21 GPIO_ACTIVE_LOW>; linux,code = ; @@ -32,12 +32,12 @@ }; }; - button-sw4 { + gpio-keys-1 { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&gpio_button_sw4_pins_mbmx28lc>; - sw4 { + switch-sw4 { label = "SW4"; gpios = <&gpio1 20 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx28-tx28.dts b/arch/arm/boot/dts/imx28-tx28.dts index 6d7b044fec5bb2b969880673044baa9b80c2cf20..096f246032c6ee7e8cd99f2e670fe0afcc810f27 100644 --- a/arch/arm/boot/dts/imx28-tx28.dts +++ b/arch/arm/boot/dts/imx28-tx28.dts @@ -221,7 +221,7 @@ linux,no-autorepeat; }; - spi_gpio: spi-gpio { + spi_gpio: spi { compatible = "spi-gpio"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi index 5c4938b0d5a198ce04955906efd997a504bcdd16..95c05f17a6d5539b416af4fb1538dc53b15d1c43 100644 --- a/arch/arm/boot/dts/imx31.dtsi +++ b/arch/arm/boot/dts/imx31.dtsi @@ -297,7 +297,7 @@ #interrupt-cells = <2>; }; - sdma: sdma@53fd4000 { + sdma: dma-controller@53fd4000 { compatible = "fsl,imx31-sdma"; reg = <0x53fd4000 0x4000>; interrupts = <34>; diff --git a/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts b/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts index b1c11170ac25a86824e160f9a40f8389053dca0b..7f4f812b08111a35881a0278949872c28fa79cf4 100644 --- a/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts +++ b/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts @@ -13,12 +13,12 @@ model = "Eukrea CPUIMX35"; compatible = "eukrea,mbimxsd35-baseboard", "eukrea,cpuimx35", "fsl,imx35"; - gpio_keys { + gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_bp1>; - bp1 { + button { label = "BP1"; gpios = <&gpio3 25 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi index 8e41c8b7bd705ff21854856bc1823a23a3502e1d..d650f54c3fc6bd2f886848638a120807c764d5a4 100644 --- a/arch/arm/boot/dts/imx35.dtsi +++ b/arch/arm/boot/dts/imx35.dtsi @@ -284,7 +284,7 @@ #interrupt-cells = <2>; }; - sdma: sdma@53fd4000 { + sdma: dma-controller@53fd4000 { compatible = "fsl,imx35-sdma"; reg = <0x53fd4000 0x4000>; clocks = <&clks 9>, <&clks 65>; diff --git a/arch/arm/boot/dts/imx50-kobo-aura.dts b/arch/arm/boot/dts/imx50-kobo-aura.dts index 82ce8c43be867efac9f21dd3c73b6960896e099d..51bf6117fb124078e3a0494c1f1a14f73aaafdef 100644 --- a/arch/arm/boot/dts/imx50-kobo-aura.dts +++ b/arch/arm/boot/dts/imx50-kobo-aura.dts @@ -38,20 +38,20 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpiokeys>; - power { + key-power { label = "Power Button"; gpios = <&gpio4 10 GPIO_ACTIVE_LOW>; linux,code = ; }; - hallsensor { + event-hallsensor { label = "Hallsensor"; gpios = <&gpio5 15 GPIO_ACTIVE_LOW>; linux,code = ; linux,input-type = ; }; - frontlight { + event-frontlight { label = "Frontlight"; gpios = <&gpio4 1 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx50.dtsi b/arch/arm/boot/dts/imx50.dtsi index c0c7575fbecf26332f8dec031c6a67f365f352bd..3d9a9f37f672bf6352c021e3983f45d2ceb27d8a 100644 --- a/arch/arm/boot/dts/imx50.dtsi +++ b/arch/arm/boot/dts/imx50.dtsi @@ -421,7 +421,7 @@ status = "disabled"; }; - sdma: sdma@63fb0000 { + sdma: dma-controller@63fb0000 { compatible = "fsl,imx50-sdma", "fsl,imx35-sdma"; reg = <0x63fb0000 0x4000>; interrupts = <6>; diff --git a/arch/arm/boot/dts/imx51-apf51dev.dts b/arch/arm/boot/dts/imx51-apf51dev.dts index c66f274ba4e94791eaff726e3ccc7c580c383342..b61d55ca146765ec8a1cf60cd0462f6458f1a071 100644 --- a/arch/arm/boot/dts/imx51-apf51dev.dts +++ b/arch/arm/boot/dts/imx51-apf51dev.dts @@ -63,7 +63,7 @@ leds { compatible = "gpio-leds"; - user { + led-user { label = "Heartbeat"; gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; diff --git a/arch/arm/boot/dts/imx51-babbage.dts b/arch/arm/boot/dts/imx51-babbage.dts index 552196d8a60a73da762f8a06b578ecc1d67d14c2..a1f9c6a722752f0b660db27cd42c66ad705748c4 100644 --- a/arch/arm/boot/dts/imx51-babbage.dts +++ b/arch/arm/boot/dts/imx51-babbage.dts @@ -154,7 +154,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_keys>; - power { + key-power { label = "Power Button"; gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; linux,code = ; diff --git a/arch/arm/boot/dts/imx51-zii-rdu1.dts b/arch/arm/boot/dts/imx51-zii-rdu1.dts index ec8ca3ac2c1c16ed3b5b0320f99057ee6baed972..3140f038aa982f9ee4ab7f4e308c774c565ccc50 100644 --- a/arch/arm/boot/dts/imx51-zii-rdu1.dts +++ b/arch/arm/boot/dts/imx51-zii-rdu1.dts @@ -137,7 +137,7 @@ }; }; - spi_gpio: spi-gpio { + spi_gpio: spi { compatible = "spi-gpio"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi index 592d9c23a447fc90968a384498eb7ce026aada69..853707574d2e3e0e60294702d3b2369db265fbdc 100644 --- a/arch/arm/boot/dts/imx51.dtsi +++ b/arch/arm/boot/dts/imx51.dtsi @@ -504,7 +504,7 @@ status = "disabled"; }; - sdma: sdma@83fb0000 { + sdma: dma-controller@83fb0000 { compatible = "fsl,imx51-sdma", "fsl,imx35-sdma"; reg = <0x83fb0000 0x4000>; interrupts = <6>; diff --git a/arch/arm/boot/dts/imx53-ard.dts b/arch/arm/boot/dts/imx53-ard.dts index 6208fbb2e74104fb13fdd896b22b6d81d997f0aa..23a7492e29294ebadb7407467afc3d087ef23e07 100644 --- a/arch/arm/boot/dts/imx53-ard.dts +++ b/arch/arm/boot/dts/imx53-ard.dts @@ -61,34 +61,34 @@ gpio-keys { compatible = "gpio-keys"; - home { + key-home { label = "Home"; gpios = <&gpio5 10 0>; linux,code = ; wakeup-source; }; - back { + key-back { label = "Back"; gpios = <&gpio5 11 0>; linux,code = ; wakeup-source; }; - program { + key-program { label = "Program"; gpios = <&gpio5 12 0>; linux,code = ; wakeup-source; }; - volume-up { + key-volume-up { label = "Volume Up"; gpios = <&gpio5 13 0>; linux,code = ; }; - volume-down { + key-volume-down { label = "Volume Down"; gpios = <&gpio4 0 0>; linux,code = ; diff --git a/arch/arm/boot/dts/imx53-qsb-common.dtsi b/arch/arm/boot/dts/imx53-qsb-common.dtsi index fe4244044a0f3be8b10e4bb7d0d4f077a194d92c..50fef8dd3675b6aed61a94b90dc89b50b19d3be1 100644 --- a/arch/arm/boot/dts/imx53-qsb-common.dtsi +++ b/arch/arm/boot/dts/imx53-qsb-common.dtsi @@ -45,20 +45,20 @@ gpio-keys { compatible = "gpio-keys"; - power { + key-power { label = "Power Button"; gpios = <&gpio1 8 GPIO_ACTIVE_LOW>; linux,code = ; }; - volume-up { + key-volume-up { label = "Volume Up"; gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; linux,code = ; wakeup-source; }; - volume-down { + key-volume-down { label = "Volume Down"; gpios = <&gpio2 15 GPIO_ACTIVE_LOW>; linux,code = ; @@ -71,7 +71,7 @@ pinctrl-names = "default"; pinctrl-0 = <&led_pin_gpio7_7>; - user { + led-user { label = "Heartbeat"; gpios = <&gpio7 7 0>; linux,default-trigger = "heartbeat"; diff --git a/arch/arm/boot/dts/imx53-smd.dts b/arch/arm/boot/dts/imx53-smd.dts index 9be44e80718865ee72cdf1ff9f1be49db2f333c4..f8d17967a67ef6e76ea8a4e526b153b89ddbd68c 100644 --- a/arch/arm/boot/dts/imx53-smd.dts +++ b/arch/arm/boot/dts/imx53-smd.dts @@ -19,13 +19,13 @@ gpio-keys { compatible = "gpio-keys"; - volume-up { + key-volume-up { label = "Volume Up"; gpios = <&gpio2 14 0>; linux,code = ; }; - volume-down { + key-volume-down { label = "Volume Down"; gpios = <&gpio2 15 0>; linux,code = ; diff --git a/arch/arm/boot/dts/imx53-tx53.dtsi b/arch/arm/boot/dts/imx53-tx53.dtsi index 8712e985146543318d1f3fafe2e9804dd033c51d..892dd1a4bac352f4f55517cf62a758c6e543e9ca 100644 --- a/arch/arm/boot/dts/imx53-tx53.dtsi +++ b/arch/arm/boot/dts/imx53-tx53.dtsi @@ -81,7 +81,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_key>; - power { + key-power { label = "Power Button"; gpios = <&gpio5 2 GPIO_ACTIVE_HIGH>; linux,code = <116>; /* KEY_POWER */ diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi index b7a6469d34721ccc637456702d0513f3d22adb9a..56b3c13f4eb7deabcdf566f4698da96cdfbcd394 100644 --- a/arch/arm/boot/dts/imx53.dtsi +++ b/arch/arm/boot/dts/imx53.dtsi @@ -710,7 +710,7 @@ status = "disabled"; }; - sdma: sdma@63fb0000 { + sdma: dma-controller@63fb0000 { compatible = "fsl,imx53-sdma", "fsl,imx35-sdma"; reg = <0x63fb0000 0x4000>; interrupts = <6>; diff --git a/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi b/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi index d9de9b4f0c52348300ea2abdc83e5e12eff9ec5d..d477a937b47a8ba8edc619879ace604bc248d4cc 100644 --- a/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi +++ b/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi @@ -6,7 +6,7 @@ keyboard { compatible = "gpio-keys"; - btn0 { + button-0 { gpios = <&pcf8575 0 GPIO_ACTIVE_LOW>; label = "btn0"; linux,code = ; @@ -14,7 +14,7 @@ wakeup-source; }; - btn1 { + button-1 { gpios = <&pcf8575 1 GPIO_ACTIVE_LOW>; label = "btn1"; linux,code = ; @@ -22,7 +22,7 @@ wakeup-source; }; - btn2 { + button-2 { gpios = <&pcf8575 2 GPIO_ACTIVE_LOW>; label = "btn2"; linux,code = ; @@ -30,7 +30,7 @@ wakeup-source; }; - btn3 { + button-3 { gpios = <&pcf8575 3 GPIO_ACTIVE_LOW>; label = "btn3"; linux,code = ; diff --git a/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi b/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi index ec5b664531565a635ffec22a201f474245ca4e54..337db29b0010a9598dd0a45f3c270b32b5c4920d 100644 --- a/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi +++ b/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi @@ -188,7 +188,7 @@ rotary-encoder-key { compatible = "gpio-keys"; - rotary-encoder-press { + rotary-encoder-event { label = "rotary-encoder press"; gpios = <&tca6424a 0 GPIO_ACTIVE_HIGH>; linux,code = ; diff --git a/arch/arm/boot/dts/imx6dl-prtmvt.dts b/arch/arm/boot/dts/imx6dl-prtmvt.dts index a35a1c66e7706cbd3bd6e91f3ab21d50e721e558..1f8cddd83ccb50625c38e610748df5006dafc2a7 100644 --- a/arch/arm/boot/dts/imx6dl-prtmvt.dts +++ b/arch/arm/boot/dts/imx6dl-prtmvt.dts @@ -51,98 +51,98 @@ pinctrl-0 = <&pinctrl_gpiokeys>; autorepeat; - power { + key-power { label = "Power Button"; gpios = <&gpio2 23 GPIO_ACTIVE_LOW>; linux,code = ; wakeup-source; }; - f1 { + key-f1 { label = "GPIO Key F1"; linux,code = ; gpios = <&gpio_pca 0 GPIO_ACTIVE_LOW>; }; - f2 { + key-f2 { label = "GPIO Key F2"; linux,code = ; gpios = <&gpio_pca 1 GPIO_ACTIVE_LOW>; }; - f3 { + key-f3 { label = "GPIO Key F3"; linux,code = ; gpios = <&gpio_pca 2 GPIO_ACTIVE_LOW>; }; - f4 { + key-f4 { label = "GPIO Key F4"; linux,code = ; gpios = <&gpio_pca 3 GPIO_ACTIVE_LOW>; }; - f5 { + key-f5 { label = "GPIO Key F5"; linux,code = ; gpios = <&gpio_pca 4 GPIO_ACTIVE_LOW>; }; - cycle { + key-cycle { label = "GPIO Key CYCLE"; linux,code = ; gpios = <&gpio_pca 5 GPIO_ACTIVE_LOW>; }; - esc { + key-esc { label = "GPIO Key ESC"; linux,code = ; gpios = <&gpio_pca 6 GPIO_ACTIVE_LOW>; }; - up { + key-up { label = "GPIO Key UP"; linux,code = ; gpios = <&gpio_pca 7 GPIO_ACTIVE_LOW>; }; - down { + key-down { label = "GPIO Key DOWN"; linux,code = ; gpios = <&gpio_pca 8 GPIO_ACTIVE_LOW>; }; - ok { + key-ok { label = "GPIO Key OK"; linux,code = ; gpios = <&gpio_pca 9 GPIO_ACTIVE_LOW>; }; - f6 { + key-f6 { label = "GPIO Key F6"; linux,code = ; gpios = <&gpio_pca 10 GPIO_ACTIVE_LOW>; }; - f7 { + key-f7 { label = "GPIO Key F7"; linux,code = ; gpios = <&gpio_pca 11 GPIO_ACTIVE_LOW>; }; - f8 { + key-f8 { label = "GPIO Key F8"; linux,code = ; gpios = <&gpio_pca 12 GPIO_ACTIVE_LOW>; }; - f9 { + key-f9 { label = "GPIO Key F9"; linux,code = ; gpios = <&gpio_pca 13 GPIO_ACTIVE_LOW>; }; - f10 { + key-f10 { label = "GPIO Key F10"; linux,code = ; gpios = <&gpio_pca 14 GPIO_ACTIVE_LOW>; diff --git a/arch/arm/boot/dts/imx6dl-prtvt7.dts b/arch/arm/boot/dts/imx6dl-prtvt7.dts index 0a0b7acddfb2369105f4c7cf0446a2a793a400f3..a1eb53851794ac5269543fdbcf9a8c0dd23d95a3 100644 --- a/arch/arm/boot/dts/imx6dl-prtvt7.dts +++ b/arch/arm/boot/dts/imx6dl-prtvt7.dts @@ -62,91 +62,91 @@ compatible = "gpio-keys"; autorepeat; - esc { + key-esc { label = "GPIO Key ESC"; linux,code = ; gpios = <&gpio_pca 0 GPIO_ACTIVE_LOW>; }; - up { + key-up { label = "GPIO Key UP"; linux,code = ; gpios = <&gpio_pca 1 GPIO_ACTIVE_LOW>; }; - down { + key-down { label = "GPIO Key DOWN"; linux,code = ; gpios = <&gpio_pca 4 GPIO_ACTIVE_LOW>; }; - enter { + key-enter { label = "GPIO Key Enter"; linux,code = ; gpios = <&gpio_pca 3 GPIO_ACTIVE_LOW>; }; - cycle { + key-cycle { label = "GPIO Key CYCLE"; linux,code = ; gpios = <&gpio_pca 2 GPIO_ACTIVE_LOW>; }; - f1 { + key-f1 { label = "GPIO Key F1"; linux,code = ; gpios = <&gpio_pca 14 GPIO_ACTIVE_LOW>; }; - f2 { + key-f2 { label = "GPIO Key F2"; linux,code = ; gpios = <&gpio_pca 13 GPIO_ACTIVE_LOW>; }; - f3 { + key-f3 { label = "GPIO Key F3"; linux,code = ; gpios = <&gpio_pca 12 GPIO_ACTIVE_LOW>; }; - f4 { + key-f4 { label = "GPIO Key F4"; linux,code = ; gpios = <&gpio_pca 11 GPIO_ACTIVE_LOW>; }; - f5 { + key-f5 { label = "GPIO Key F5"; linux,code = ; gpios = <&gpio_pca 10 GPIO_ACTIVE_LOW>; }; - f6 { + key-f6 { label = "GPIO Key F6"; linux,code = ; gpios = <&gpio_pca 5 GPIO_ACTIVE_LOW>; }; - f7 { + key-f7 { label = "GPIO Key F7"; linux,code = ; gpios = <&gpio_pca 6 GPIO_ACTIVE_LOW>; }; - f8 { + key-f8 { label = "GPIO Key F8"; linux,code = ; gpios = <&gpio_pca 7 GPIO_ACTIVE_LOW>; }; - f9 { + key-f9 { label = "GPIO Key F9"; linux,code = ; gpios = <&gpio_pca 8 GPIO_ACTIVE_LOW>; }; - f10 { + key-f10 { label = "GPIO Key F10"; linux,code = ; gpios = <&gpio_pca 9 GPIO_ACTIVE_LOW>; diff --git a/arch/arm/boot/dts/imx6dl-riotboard.dts b/arch/arm/boot/dts/imx6dl-riotboard.dts index e7d9bfbfd0e4d5b6a7d0cce0c061020f93507e7e..e7be05f205d32b6b3f6ac1c81858de4433384639 100644 --- a/arch/arm/boot/dts/imx6dl-riotboard.dts +++ b/arch/arm/boot/dts/imx6dl-riotboard.dts @@ -90,6 +90,7 @@ pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii-id"; phy-handle = <&rgmii_phy>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6dl-victgo.dts b/arch/arm/boot/dts/imx6dl-victgo.dts index 779b52858a25337f0070e5c6e4829fabed48d834..72df1dba83bec640b511fbf3763239193428d1f6 100644 --- a/arch/arm/boot/dts/imx6dl-victgo.dts +++ b/arch/arm/boot/dts/imx6dl-victgo.dts @@ -18,14 +18,14 @@ pinctrl-0 = <&pinctrl_gpiokeys>; autorepeat; - power { + key-power { label = "Power Button"; gpios = <&gpio2 23 GPIO_ACTIVE_LOW>; linux,code = ; wakeup-source; }; - enter { + key-enter { label = "Rotary Key"; gpios = <&gpio2 05 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi b/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi index 674af39c884a0e1e62dc3bbec02758f5b4d77043..52162e8c7274b4487073cc21d3fa9b9a21030d1f 100644 --- a/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi +++ b/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi @@ -55,6 +55,7 @@ panel: panel { compatible = "dataimage,scf0700c48ggu18"; power-supply = <&sw2_reg>; + backlight = <&backlight>; status = "disabled"; port { diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi index 8e0ed209ede06413df2a316d2a035e158e092107..dc919e09a505f0f17c9ae9cdd8940ea43fe3b361 100644 --- a/arch/arm/boot/dts/imx6dl.dtsi +++ b/arch/arm/boot/dts/imx6dl.dtsi @@ -84,6 +84,9 @@ ocram: sram@900000 { compatible = "mmio-sram"; reg = <0x00900000 0x20000>; + ranges = <0 0x00900000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6QDL_CLK_OCRAM>; }; diff --git a/arch/arm/boot/dts/imx6q-arm2.dts b/arch/arm/boot/dts/imx6q-arm2.dts index 0b40f52268b3c9fc5aa3e62e2eb712315507d980..75586299d9cabf4983e37861dbc1604cd027f1a5 100644 --- a/arch/arm/boot/dts/imx6q-arm2.dts +++ b/arch/arm/boot/dts/imx6q-arm2.dts @@ -178,6 +178,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6q-evi.dts b/arch/arm/boot/dts/imx6q-evi.dts index c63f371ede8b98f7952c5f03b22bb7f99918acaf..78d941fef5dfb288c69d702ed190150caa92ba7a 100644 --- a/arch/arm/boot/dts/imx6q-evi.dts +++ b/arch/arm/boot/dts/imx6q-evi.dts @@ -146,6 +146,7 @@ pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; phy-reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6q-mccmon6.dts b/arch/arm/boot/dts/imx6q-mccmon6.dts index 55692c73943d6553edad728a965b2cfbab37c2be..f08b3701029156b0e6e253cd2dac7ee8db8409bb 100644 --- a/arch/arm/boot/dts/imx6q-mccmon6.dts +++ b/arch/arm/boot/dts/imx6q-mccmon6.dts @@ -100,8 +100,10 @@ pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; + fsl,err006687-workaround-present; status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6q-novena.dts b/arch/arm/boot/dts/imx6q-novena.dts index 225cf6b7a7a4f511e2639bc82d8097c48f342546..ee8c0bd3ecfd525739da152e564a5be3235ab46e 100644 --- a/arch/arm/boot/dts/imx6q-novena.dts +++ b/arch/arm/boot/dts/imx6q-novena.dts @@ -86,7 +86,7 @@ linux,code = ; }; - lid { + lid-event { label = "Lid"; gpios = <&gpio4 12 GPIO_ACTIVE_LOW>; linux,input-type = <5>; /* EV_SW */ @@ -99,7 +99,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_leds_novena>; - heartbeat { + led-heartbeat { label = "novena:white:panel"; gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; linux,default-trigger = "default-on"; diff --git a/arch/arm/boot/dts/imx6q-pistachio.dts b/arch/arm/boot/dts/imx6q-pistachio.dts index 7a33e54cc0f1420c93fdf24da37dd7d280a24a1a..bad8d831e64e6f796b8f1b109e8d20df701982ec 100644 --- a/arch/arm/boot/dts/imx6q-pistachio.dts +++ b/arch/arm/boot/dts/imx6q-pistachio.dts @@ -100,7 +100,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_keys>; - power { + key-power { label = "Power Button"; gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; wakeup-source; diff --git a/arch/arm/boot/dts/imx6q-sabrelite.dts b/arch/arm/boot/dts/imx6q-sabrelite.dts index dc51262e7b2f270ab3a00c0337ab105f0c5dee36..7c6a2f234ccbd15d05dc74908fbf5e22d22131cf 100644 --- a/arch/arm/boot/dts/imx6q-sabrelite.dts +++ b/arch/arm/boot/dts/imx6q-sabrelite.dts @@ -1,43 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 OR X11 /* * Copyright 2011 Freescale Semiconductor, Inc. * Copyright 2011 Linaro Ltd. * - * This file is dual-licensed: you can use it either under the terms - * of the GPL or the X11 license, at your option. Note that this dual - * licensing only applies to this file, and not this project as a - * whole. - * - * a) This file is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This file is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Or, alternatively, - * - * b) Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ /dts-v1/; diff --git a/arch/arm/boot/dts/imx6q-utilite-pro.dts b/arch/arm/boot/dts/imx6q-utilite-pro.dts index d16ff2083d626c465cc0fad79b640b92152dd703..ad59b23ef27a084b80015e53c9cb5588dfc74b9a 100644 --- a/arch/arm/boot/dts/imx6q-utilite-pro.dts +++ b/arch/arm/boot/dts/imx6q-utilite-pro.dts @@ -89,7 +89,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_keys>; - power { + key-power { label = "Power Button"; gpios = <&gpio1 29 GPIO_ACTIVE_LOW>; linux,code = ; diff --git a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts index 63550351340da51e3ccfe17e1619e09918accca0..2290c1237634d9d77e8e99ead95648e3a6db3ac2 100644 --- a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts +++ b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts @@ -28,7 +28,7 @@ compatible = "gpio-keys"; autorepeat; - back { + key-back { gpios = <&gpio4 26 GPIO_ACTIVE_LOW>; linux,code = ; label = "Key Back"; @@ -37,7 +37,7 @@ wakeup-source; }; - home { + key-home { gpios = <&gpio5 11 GPIO_ACTIVE_LOW>; linux,code = ; label = "Key Home"; @@ -46,7 +46,7 @@ wakeup-source; }; - menu { + key-menu { gpios = <&gpio4 25 GPIO_ACTIVE_LOW>; linux,code = ; label = "Key Menu"; diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi index 3b77eae40e3956f0e12ead287c1cec31e277ad9c..df86049a695b1cc70cdaf50fe56a8b478fb6b6ad 100644 --- a/arch/arm/boot/dts/imx6q.dtsi +++ b/arch/arm/boot/dts/imx6q.dtsi @@ -163,6 +163,9 @@ ocram: sram@900000 { compatible = "mmio-sram"; reg = <0x00900000 0x40000>; + ranges = <0 0x00900000 0x40000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6QDL_CLK_OCRAM>; }; diff --git a/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi b/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi index fe72650295a5bd58e0d06fbd0ba73efe146db584..6248b126b5578abc7dc3066995c116a61c76256f 100644 --- a/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi +++ b/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi @@ -332,37 +332,4 @@ MX6QDL_PAD_GPIO_0__GPIO1_IO00 0xb1 /* Int */ >; }; - - pinctrl_ipu1_lcdif: ipu1-lcdif-grp { - fsl,pins = < - MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK 0x38 - MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02 0x38 - MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03 0x38 - MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15 0x38 - MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00 0x38 - MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01 0x38 - MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02 0x38 - MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03 0x38 - MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04 0x38 - MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05 0x38 - MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06 0x38 - MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07 0x38 - MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08 0x38 - MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09 0x38 - MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10 0x38 - MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11 0x38 - MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12 0x38 - MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13 0x38 - MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14 0x38 - MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15 0x38 - MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16 0x38 - MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17 0x38 - MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18 0x38 - MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19 0x38 - MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20 0x38 - MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21 0x38 - MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22 0x38 - MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23 0x38 - >; - }; }; diff --git a/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi b/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi index 5befbe13d1a3dbde79e39a80abe8763f2ad7d7ba..eaa87b3331648e72c6098342acc75217ffcc91e2 100644 --- a/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi +++ b/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi @@ -667,6 +667,39 @@ >; }; + pinctrl_ipu1_lcdif: ipu1-lcdif-grp { + fsl,pins = < + MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK 0x38 + MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02 0x38 + MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03 0x38 + MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15 0x38 + MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00 0x38 + MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01 0x38 + MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02 0x38 + MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03 0x38 + MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04 0x38 + MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05 0x38 + MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06 0x38 + MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07 0x38 + MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08 0x38 + MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09 0x38 + MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10 0x38 + MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11 0x38 + MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12 0x38 + MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13 0x38 + MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14 0x38 + MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15 0x38 + MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16 0x38 + MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17 0x38 + MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18 0x38 + MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19 0x38 + MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20 0x38 + MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21 0x38 + MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22 0x38 + MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23 0x38 + >; + }; + pinctrl_pcie: pcie-grp { fsl,pins = < MX6QDL_PAD_CSI0_DATA_EN__GPIO5_IO20 0x1b0b1 /* Wake */ diff --git a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi index b1df2beb2832c7f51e7127b22e71e600cbea9ec3..728810b9d677da92f038dc582cfa32ca4386a9ab 100644 --- a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi @@ -137,6 +137,16 @@ regulator-always-on; }; + reg_can1_stby: regulator-can1-stby { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_can1>; + regulator-name = "can1_stby"; + gpio = <&gpio1 9 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + reg_usb_otg_vbus: regulator-usb-otg-vbus { compatible = "regulator-fixed"; regulator-name = "usb_otg_vbus"; @@ -170,6 +180,7 @@ &can1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_flexcan1>; + xceiver-supply = <®_can1_stby>; status = "okay"; }; @@ -612,7 +623,6 @@ fsl,pins = < MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX 0x1b0b1 MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX 0x1b0b1 - MX6QDL_PAD_GPIO_9__GPIO1_IO09 0x4001b0b0 /* CAN_STBY */ >; }; @@ -702,6 +712,12 @@ >; }; + pinctrl_reg_can1: regcan1grp { + fsl,pins = < + MX6QDL_PAD_GPIO_9__GPIO1_IO09 0x4001b0b0 /* CAN_STBY */ + >; + }; + pinctrl_uart1: uart1grp { fsl,pins = < MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA 0x1b0b1 diff --git a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi index a0710d562766e18d97f46e7245d80a97eb18e071..6c0c109046d80989c1a0da25978867787dbf6222 100644 --- a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi @@ -129,6 +129,16 @@ regulator-always-on; }; + reg_can1_stby: regulator-can1-stby { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_can1>; + regulator-name = "can1_stby"; + gpio = <&gpio1 2 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + reg_usb_h1_vbus: regulator-usb-h1-vbus { compatible = "regulator-fixed"; regulator-name = "usb_h1_vbus"; @@ -170,6 +180,7 @@ &can1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_flexcan1>; + xceiver-supply = <®_can1_stby>; status = "okay"; }; @@ -600,7 +611,6 @@ fsl,pins = < MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX 0x1b0b1 MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX 0x1b0b1 - MX6QDL_PAD_GPIO_2__GPIO1_IO02 0x4001b0b0 /* CAN_STBY */ >; }; @@ -691,6 +701,12 @@ >; }; + pinctrl_reg_can1: regcan1grp { + fsl,pins = < + MX6QDL_PAD_GPIO_2__GPIO1_IO02 0x4001b0b0 /* CAN_STBY */ + >; + }; + pinctrl_uart1: uart1grp { fsl,pins = < MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA 0x1b0b1 diff --git a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi index cda48bf2f168d238057682094157068d87918b17..a9b04f9f1c2bc2ee7587caf86457d482b874a70a 100644 --- a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi @@ -137,6 +137,16 @@ regulator-always-on; }; + reg_can1_stby: regulator-can1-stby { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_can1>; + regulator-name = "can1_stby"; + gpio = <&gpio1 2 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + reg_usb_h1_vbus: regulator@2 { compatible = "regulator-fixed"; reg = <2>; @@ -200,6 +210,7 @@ &can1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_flexcan1>; + xceiver-supply = <®_can1_stby>; status = "okay"; }; @@ -687,7 +698,6 @@ fsl,pins = < MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX 0x1b0b1 MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX 0x1b0b1 - MX6QDL_PAD_GPIO_2__GPIO1_IO02 0x4001b0b0 /* CAN_STBY */ >; }; @@ -786,6 +796,12 @@ >; }; + pinctrl_reg_can1: regcan1grp { + fsl,pins = < + MX6QDL_PAD_GPIO_2__GPIO1_IO02 0x4001b0b0 /* CAN_STBY */ + >; + }; + pinctrl_uart1: uart1grp { fsl,pins = < MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA 0x1b0b1 diff --git a/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi b/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi index 095c9143d99a35496ac043171e457e15277ede3e..85aeebc9485dd31b63fdf418d140551682b0c270 100644 --- a/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi +++ b/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi @@ -51,16 +51,6 @@ vin-supply = <®_3p3v_s5>; }; - reg_3p3v_s0: regulator-3p3v-s0 { - compatible = "regulator-fixed"; - regulator-name = "V_3V3_S0"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-always-on; - regulator-boot-on; - vin-supply = <®_3p3v_s5>; - }; - reg_3p3v_s5: regulator-3p3v-s5 { compatible = "regulator-fixed"; regulator-name = "V_3V3_S5"; @@ -259,7 +249,7 @@ /* default boot source: workaround #1 for errata ERR006282 */ smarc_flash: flash@0 { - compatible = "winbond,w25q16dw", "jedec,spi-nor"; + compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <20000000>; }; @@ -270,7 +260,23 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; - phy-reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>; + phy-handle = <ðphy>; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>; + reset-assert-us = <1000>; + }; + }; +}; + +&hdmi { + ddc-i2c-bus = <&i2c2>; }; &i2c_intern { @@ -397,7 +403,7 @@ /* HDMI_CTRL */ &i2c2 { - clock-frequency = <375000>; + clock-frequency = <100000>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; }; diff --git a/arch/arm/boot/dts/imx6qdl-mba6.dtsi b/arch/arm/boot/dts/imx6qdl-mba6.dtsi index f4dca20669d6b7f005863df67f29745906053f7f..78555a6188510f63da0bf1a2659423cbc56205de 100644 --- a/arch/arm/boot/dts/imx6qdl-mba6.dtsi +++ b/arch/arm/boot/dts/imx6qdl-mba6.dtsi @@ -244,7 +244,6 @@ status = "okay"; }; - &uart3 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; diff --git a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi index 0ad4cb4f1e8289387815e925cc3c3960174b3b51..a53a5d0766a5126c9de7f8512105fd04866e8b86 100644 --- a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi +++ b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi @@ -192,6 +192,7 @@ phy-mode = "rgmii"; phy-handle = <ðphy>; phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi index beaa2dcd436cec5495619fadc6d0cf6bc13ee58b..57c21a01f126d9d78714a4ab5b7fd0eaeca56d40 100644 --- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi +++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi @@ -334,6 +334,7 @@ phy-mode = "rgmii"; phy-handle = <ðphy>; phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi index ee7e2371f94bd0f047936b70d6b941caf793d877..000e9dc97b1ace42ad3ca04b30857abf44eb18c7 100644 --- a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi +++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi @@ -263,6 +263,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi index 904d5d051d63c2ec1d5efec9487f14d58765de67..731759bdd7f57a8ee6512085e63d447a05149cd5 100644 --- a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi +++ b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi @@ -267,6 +267,7 @@ phy-mode = "rgmii"; phy-handle = <ðphy>; phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi index 393475cb0f4d72bc0dae8872c775388d33546a46..0020dbb1722c32ed4b3f6f0e9a737e7f953b9144 100644 --- a/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi +++ b/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi @@ -64,7 +64,7 @@ interrupt-parent = <&gpio3>; interrupts = <2 IRQ_TYPE_NONE>; status = "disabled"; - }; + }; }; &ipu1_di0_disp0 { diff --git a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi index 1368a476203721f473adfac6fa682bc51c8c55c9..3dbb460ef102e175fdd78e3ead4658b07bdf7716 100644 --- a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi +++ b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi @@ -295,6 +295,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii-id"; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi index 901b9a761b66e4069608f457731634d6e6969109..22f8e2783cdf5ec7006b5dbc754b520c120decc2 100644 --- a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi +++ b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi @@ -1,43 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 OR X11 /* * Copyright 2011 Freescale Semiconductor, Inc. * Copyright 2011 Linaro Ltd. * - * This file is dual-licensed: you can use it either under the terms - * of the GPL or the X11 license, at your option. Note that this dual - * licensing only applies to this file, and not this project as a - * whole. - * - * a) This file is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This file is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Or, alternatively, - * - * b) Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. */ #include diff --git a/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi b/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi index 8254bce1b8a22831f9f05ffb1895532ec2bb4e34..b81799d7076aa7b1a149e7b600453f0107b8f9c1 100644 --- a/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi +++ b/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi @@ -2,35 +2,60 @@ // // Copyright (C) 2020 Pengutronix, Ulrich Oelmann +/ { + touchscreen { + compatible = "resistive-adc-touch"; + io-channels = <&adc_ts 1>, <&adc_ts 3>, <&adc_ts 4>, <&adc_ts 5>; + io-channel-names = "y", "z1", "z2", "x"; + touchscreen-min-pressure = <65000>; + touchscreen-inverted-y; + touchscreen-swapped-x-y; + touchscreen-x-plate-ohms = <300>; + touchscreen-y-plate-ohms = <800>; + }; +}; + &ecspi4 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ecspi4>; cs-gpios = <&gpio3 20 GPIO_ACTIVE_LOW>; status = "okay"; - touchscreen@0 { - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_touch>; - compatible = "ti,tsc2046"; + adc_ts: adc@0 { + compatible = "ti,tsc2046e-adc"; reg = <0>; + pinctrl-0 = <&pinctrl_touch>; + pinctrl-names ="default"; spi-max-frequency = <1000000>; interrupts-extended = <&gpio3 19 IRQ_TYPE_LEVEL_LOW>; - vcc-supply = <®_3v3>; - pendown-gpio = <&gpio3 19 GPIO_ACTIVE_LOW>; - ti,x-plate-ohms = /bits/ 16 <850>; - ti,y-plate-ohms = /bits/ 16 <295>; - ti,pressure-min = /bits/ 16 <2>; - ti,pressure-max = /bits/ 16 <1500>; - ti,vref-mv = /bits/ 16 <3300>; - ti,settle-delay-usec = /bits/ 16 <15>; - ti,vref-delay-usecs = /bits/ 16 <0>; - ti,penirq-recheck-delay-usecs = /bits/ 16 <100>; - ti,debounce-max = /bits/ 16 <100>; - ti,debounce-tol = /bits/ 16 <(~0)>; - ti,debounce-rep = /bits/ 16 <4>; - touchscreen-swapped-x-y; - touchscreen-inverted-y; - wakeup-source; + #io-channel-cells = <1>; + + #address-cells = <1>; + #size-cells = <0>; + + channel@1 { + reg = <1>; + settling-time-us = <700>; + oversampling-ratio = <5>; + }; + + channel@3 { + reg = <3>; + settling-time-us = <700>; + oversampling-ratio = <5>; + }; + + channel@4 { + reg = <4>; + settling-time-us = <700>; + oversampling-ratio = <5>; + }; + + channel@5 { + reg = <5>; + settling-time-us = <700>; + oversampling-ratio = <5>; + }; }; }; diff --git a/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi b/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi index 7dc3f0005b0f0e5c213895e3ec0f31ddd3554e2b..aff46f3040c167e0cb0ee28b03ca1201d8115900 100644 --- a/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi +++ b/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi @@ -7,6 +7,7 @@ #include &fec { + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; @@ -35,6 +36,7 @@ compatible = "st,24c64", "atmel,24c64"; reg = <0x50>; pagesize = <32>; + vcc-supply = <®_3p3v>; }; }; diff --git a/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi b/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi index dd09257664403ed07c472b017b0a79c18cb87bd9..a3f6543c3aaa6127f1a1b1dce93322d2e9ba854d 100644 --- a/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi +++ b/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi @@ -29,5 +29,6 @@ compatible = "st,24c64", "atmel,24c64"; reg = <0x50>; pagesize = <32>; + vcc-supply = <®_3p3v>; }; }; diff --git a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi index d6ba4b2a60f6f53f1e4f6a64eb783de13eae7a4f..c096d25a6f5b54f56289075d1041a3ad84aa804c 100644 --- a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi +++ b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi @@ -192,6 +192,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_enet>; phy-mode = "rgmii"; + /delete-property/ interrupts; interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>, <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>; fsl,err006687-workaround-present; diff --git a/arch/arm/boot/dts/imx6qdl-vicut1.dtsi b/arch/arm/boot/dts/imx6qdl-vicut1.dtsi index a1676b5d2980fbd6c466ac00edc629f36bb1962f..c4e6cf0527ba40c4cea579a2cebc7601a16ecaad 100644 --- a/arch/arm/boot/dts/imx6qdl-vicut1.dtsi +++ b/arch/arm/boot/dts/imx6qdl-vicut1.dtsi @@ -28,7 +28,7 @@ enable-gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>; }; - backlight_led: backlight_led { + backlight_led: backlight-led { compatible = "pwm-backlight"; pwms = <&pwm3 0 5000000 0>; brightness-levels = <0 16 64 255>; @@ -37,6 +37,16 @@ power-supply = <®_3v3>; }; + /* only for backwards compatibility with old HW */ + backlight_isb: backlight-isb { + compatible = "pwm-backlight"; + pwms = <&pwm2 0 5000000 0>; + brightness-levels = <0 8 48 255>; + num-interpolated-steps = <5>; + default-brightness-level = <0>; + power-supply = <®_3v3>; + }; + connector { compatible = "composite-video-connector"; label = "Composite0"; @@ -370,6 +380,12 @@ status = "okay"; }; +&pwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm2>; + status = "okay"; +}; + &pwm3 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_pwm3>; @@ -601,6 +617,12 @@ >; }; + pinctrl_pwm2: pwm2grp { + fsl,pins = < + MX6QDL_PAD_DISP0_DAT9__PWM2_OUT 0x1b0b0 + >; + }; + pinctrl_pwm3: pwm3grp { fsl,pins = < MX6QDL_PAD_SD4_DAT1__PWM3_OUT 0x1b0b0 diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi index 4f7fefc14d0aca8332acf83e4e10cd7c8206316f..ff1e0173b39be7d7986cc5a936a3d5513b6692b6 100644 --- a/arch/arm/boot/dts/imx6qdl.dtsi +++ b/arch/arm/boot/dts/imx6qdl.dtsi @@ -929,7 +929,7 @@ interrupts = <0 125 IRQ_TYPE_LEVEL_HIGH>; }; - sdma: sdma@20ec000 { + sdma: dma-controller@20ec000 { compatible = "fsl,imx6q-sdma", "fsl,imx35-sdma"; reg = <0x020ec000 0x4000>; interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>; diff --git a/arch/arm/boot/dts/imx6qp.dtsi b/arch/arm/boot/dts/imx6qp.dtsi index 050365513836356a0f132c9ffc78020c546a5b6d..fc164991d2ae8262118e779ae410ed16870d252e 100644 --- a/arch/arm/boot/dts/imx6qp.dtsi +++ b/arch/arm/boot/dts/imx6qp.dtsi @@ -9,12 +9,18 @@ ocram2: sram@940000 { compatible = "mmio-sram"; reg = <0x00940000 0x20000>; + ranges = <0 0x00940000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6QDL_CLK_OCRAM>; }; ocram3: sram@960000 { compatible = "mmio-sram"; reg = <0x00960000 0x20000>; + ranges = <0 0x00960000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6QDL_CLK_OCRAM>; }; diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi index 06a515121dfc57ee5127cd874c60f3a150c44c99..28111efb19a6634a638ef5ff0cbf4e5dcc2d312e 100644 --- a/arch/arm/boot/dts/imx6sl.dtsi +++ b/arch/arm/boot/dts/imx6sl.dtsi @@ -61,10 +61,10 @@ <792000 1175000>, <396000 975000>; fsl,soc-operating-points = - /* ARM kHz SOC-PU uV */ - <996000 1225000>, - <792000 1175000>, - <396000 1175000>; + /* ARM kHz SOC-PU uV */ + <996000 1225000>, + <792000 1175000>, + <396000 1175000>; clock-latency = <61036>; /* two CLK32 periods */ #cooling-cells = <2>; clocks = <&clks IMX6SL_CLK_ARM>, <&clks IMX6SL_CLK_PLL2_PFD2>, @@ -115,6 +115,9 @@ ocram: sram@900000 { compatible = "mmio-sram"; reg = <0x00900000 0x20000>; + ranges = <0 0x00900000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6SL_CLK_OCRAM>; }; @@ -222,7 +225,7 @@ uart5: serial@2018000 { compatible = "fsl,imx6sl-uart", - "fsl,imx6q-uart", "fsl,imx21-uart"; + "fsl,imx6q-uart", "fsl,imx21-uart"; reg = <0x02018000 0x4000>; interrupts = <0 30 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6SL_CLK_UART>, @@ -235,7 +238,7 @@ uart1: serial@2020000 { compatible = "fsl,imx6sl-uart", - "fsl,imx6q-uart", "fsl,imx21-uart"; + "fsl,imx6q-uart", "fsl,imx21-uart"; reg = <0x02020000 0x4000>; interrupts = <0 26 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6SL_CLK_UART>, @@ -248,7 +251,7 @@ uart2: serial@2024000 { compatible = "fsl,imx6sl-uart", - "fsl,imx6q-uart", "fsl,imx21-uart"; + "fsl,imx6q-uart", "fsl,imx21-uart"; reg = <0x02024000 0x4000>; interrupts = <0 27 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6SL_CLK_UART>, @@ -309,7 +312,7 @@ uart3: serial@2034000 { compatible = "fsl,imx6sl-uart", - "fsl,imx6q-uart", "fsl,imx21-uart"; + "fsl,imx6q-uart", "fsl,imx21-uart"; reg = <0x02034000 0x4000>; interrupts = <0 28 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6SL_CLK_UART>, @@ -322,7 +325,7 @@ uart4: serial@2038000 { compatible = "fsl,imx6sl-uart", - "fsl,imx6q-uart", "fsl,imx21-uart"; + "fsl,imx6q-uart", "fsl,imx21-uart"; reg = <0x02038000 0x4000>; interrupts = <0 29 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clks IMX6SL_CLK_UART>, @@ -711,7 +714,7 @@ #power-domain-cells = <0>; power-supply = <®_pu>; clocks = <&clks IMX6SL_CLK_GPU2D_OVG>, - <&clks IMX6SL_CLK_GPU2D_PODF>; + <&clks IMX6SL_CLK_GPU2D_PODF>; }; pd_disp: power-domain@2 { @@ -747,7 +750,7 @@ interrupts = <0 6 IRQ_TYPE_LEVEL_HIGH>; }; - sdma: sdma@20ec000 { + sdma: dma-controller@20ec000 { compatible = "fsl,imx6sl-sdma", "fsl,imx6q-sdma"; reg = <0x020ec000 0x4000>; interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>; diff --git a/arch/arm/boot/dts/imx6sll.dtsi b/arch/arm/boot/dts/imx6sll.dtsi index d4a000c3dde7075dee4511dd6cc73ff89f691217..2873369a57c02885d8ec4179ae5c2bf900e268c9 100644 --- a/arch/arm/boot/dts/imx6sll.dtsi +++ b/arch/arm/boot/dts/imx6sll.dtsi @@ -115,6 +115,9 @@ ocram: sram@900000 { compatible = "mmio-sram"; reg = <0x00900000 0x20000>; + ranges = <0 0x00900000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; }; intc: interrupt-controller@a01000 { diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi index 35861bbea94e652865a02c2025e4ebbdcb88135f..c84ea1fac5e98dd3206cbee2e02e80b3f84ded23 100644 --- a/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi +++ b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi @@ -226,7 +226,7 @@ &iomuxc { pinctrl_bt_reg: btreggrp { fsl,pins = - ; + ; }; pinctrl_enet1: enet1grp { @@ -306,7 +306,6 @@ >; }; - pinctrl_uart1: uart1grp { fsl,pins = , @@ -347,24 +346,23 @@ pinctrl_otg1_reg: otg1grp { fsl,pins = - ; + ; }; - pinctrl_otg2_reg: otg2grp { fsl,pins = - ; + ; }; pinctrl_usb_otg1: usbotg1grp { fsl,pins = - , - ; + , + ; }; pinctrl_usb_otg2: usbot2ggrp { fsl,pins = - ; + ; }; pinctrl_usdhc2: usdhc2grp { diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi index 4d075e2bf7496e016fc925948885b1d3640c1c94..abc3572d699e63d822f429f433128c42d702428f 100644 --- a/arch/arm/boot/dts/imx6sx.dtsi +++ b/arch/arm/boot/dts/imx6sx.dtsi @@ -164,12 +164,18 @@ ocram_s: sram@8f8000 { compatible = "mmio-sram"; reg = <0x008f8000 0x4000>; + ranges = <0 0x008f8000 0x4000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6SX_CLK_OCRAM_S>; }; ocram: sram@900000 { compatible = "mmio-sram"; reg = <0x00900000 0x20000>; + ranges = <0 0x00900000 0x20000>; + #address-cells = <1>; + #size-cells = <1>; clocks = <&clks IMX6SX_CLK_OCRAM>; }; @@ -842,7 +848,7 @@ reg = <0x020e4000 0x4000>; }; - sdma: sdma@20ec000 { + sdma: dma-controller@20ec000 { compatible = "fsl,imx6sx-sdma", "fsl,imx6q-sdma"; reg = <0x020ec000 0x4000>; interrupts = ; diff --git a/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi b/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi index 1a18c41ce385a782db4ed449e65198913e3acc89..c83e64a62d8a8dce9268c2d6aa7e6e9d83eb3606 100644 --- a/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi +++ b/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi @@ -82,7 +82,7 @@ "AMIC", "MICB"; }; - spi4 { + spi-4 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi4>; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-s-43.dts b/arch/arm/boot/dts/imx6ul-kontron-bl-43.dts similarity index 93% rename from arch/arm/boot/dts/imx6ul-kontron-n6310-s-43.dts rename to arch/arm/boot/dts/imx6ul-kontron-bl-43.dts index 5bfad4655b22008558e10c2625407c6cf7f6de5b..0c643706a158be9ea2aff6493a0b0eeff59cf0aa 100644 --- a/arch/arm/boot/dts/imx6ul-kontron-n6310-s-43.dts +++ b/arch/arm/boot/dts/imx6ul-kontron-bl-43.dts @@ -5,12 +5,12 @@ * Copyright (c) 2019 Krzysztof Kozlowski */ -#include "imx6ul-kontron-n6310-s.dts" +#include "imx6ul-kontron-bl.dts" / { - model = "Kontron N6310 S 43"; - compatible = "kontron,imx6ul-n6310-s-43", "kontron,imx6ul-n6310-s", - "kontron,imx6ul-n6310-som", "fsl,imx6ul"; + model = "Kontron BL i.MX6UL 43 (N631X S 43)"; + compatible = "kontron,bl-imx6ul-43", "kontron,bl-imx6ul", + "kontron,sl-imx6ul", "fsl,imx6ul"; backlight { compatible = "pwm-backlight"; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-s.dtsi b/arch/arm/boot/dts/imx6ul-kontron-bl-common.dtsi similarity index 100% rename from arch/arm/boot/dts/imx6ul-kontron-n6x1x-s.dtsi rename to arch/arm/boot/dts/imx6ul-kontron-bl-common.dtsi diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-s.dts b/arch/arm/boot/dts/imx6ul-kontron-bl.dts similarity index 52% rename from arch/arm/boot/dts/imx6ul-kontron-n6310-s.dts rename to arch/arm/boot/dts/imx6ul-kontron-bl.dts index 5a3e06d6219b4ae7428e99db5e56b8894c810ad8..dadf6d3d5f5212403250d8fc8f09211c47a59a56 100644 --- a/arch/arm/boot/dts/imx6ul-kontron-n6310-s.dts +++ b/arch/arm/boot/dts/imx6ul-kontron-bl.dts @@ -7,11 +7,10 @@ /dts-v1/; -#include "imx6ul-kontron-n6310-som.dtsi" -#include "imx6ul-kontron-n6x1x-s.dtsi" +#include "imx6ul-kontron-sl.dtsi" +#include "imx6ul-kontron-bl-common.dtsi" / { - model = "Kontron N6310 S"; - compatible = "kontron,imx6ul-n6310-s", "kontron,imx6ul-n6310-som", - "fsl,imx6ul"; + model = "Kontron BL i.MX6UL (N631X S)"; + compatible = "kontron,bl-imx6ul", "kontron,sl-imx6ul", "fsl,imx6ul"; }; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi b/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi deleted file mode 100644 index acd936540d898bf45c60defb7309b408a46e6a9e..0000000000000000000000000000000000000000 --- a/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2017 exceet electronics GmbH - * Copyright (C) 2018 Kontron Electronics GmbH - * Copyright (c) 2019 Krzysztof Kozlowski - */ - -#include "imx6ul.dtsi" -#include "imx6ul-kontron-n6x1x-som-common.dtsi" - -/ { - model = "Kontron N6310 SOM"; - compatible = "kontron,imx6ul-n6310-som", "fsl,imx6ul"; - - memory@80000000 { - reg = <0x80000000 0x10000000>; - device_type = "memory"; - }; -}; - -&qspi { - flash@0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "spi-nand"; - spi-max-frequency = <108000000>; - spi-tx-bus-width = <4>; - spi-rx-bus-width = <4>; - reg = <0>; - - partition@0 { - label = "ubi1"; - reg = <0x00000000 0x08000000>; - }; - - partition@8000000 { - label = "ubi2"; - reg = <0x08000000 0x08000000>; - }; - }; -}; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts b/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts deleted file mode 100644 index 239a1af3aeaaad22bfee8057bdd28ef6752dde7e..0000000000000000000000000000000000000000 --- a/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2017 exceet electronics GmbH - * Copyright (C) 2018 Kontron Electronics GmbH - */ - -/dts-v1/; - -#include "imx6ul-kontron-n6311-som.dtsi" -#include "imx6ul-kontron-n6x1x-s.dtsi" - -/ { - model = "Kontron N6311 S"; - compatible = "kontron,imx6ul-n6311-s", "kontron,imx6ul-n6311-som", - "fsl,imx6ul"; -}; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi b/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi deleted file mode 100644 index 29ed38dce58021c7fdabf44d6e69fc4e8ecef83b..0000000000000000000000000000000000000000 --- a/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2017 exceet electronics GmbH - * Copyright (C) 2018 Kontron Electronics GmbH - */ - -#include "imx6ul.dtsi" -#include "imx6ul-kontron-n6x1x-som-common.dtsi" - -/ { - model = "Kontron N6311 SOM"; - compatible = "kontron,imx6ul-n6311-som", "fsl,imx6ul"; - - memory@80000000 { - reg = <0x80000000 0x20000000>; - device_type = "memory"; - }; -}; - -&qspi { - flash@0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "spi-nand"; - spi-max-frequency = <104000000>; - spi-tx-bus-width = <4>; - spi-rx-bus-width = <4>; - reg = <0>; - - partition@0 { - label = "ubi1"; - reg = <0x00000000 0x08000000>; - }; - - partition@8000000 { - label = "ubi2"; - reg = <0x08000000 0x18000000>; - }; - }; -}; diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-som-common.dtsi b/arch/arm/boot/dts/imx6ul-kontron-sl-common.dtsi similarity index 90% rename from arch/arm/boot/dts/imx6ul-kontron-n6x1x-som-common.dtsi rename to arch/arm/boot/dts/imx6ul-kontron-sl-common.dtsi index 09a83dbdf651059b603f3b5d0028d621625fcbed..dcf88f6103466f1afaef092b6dbae5e062ffa063 100644 --- a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-som-common.dtsi +++ b/arch/arm/boot/dts/imx6ul-kontron-sl-common.dtsi @@ -11,6 +11,11 @@ chosen { stdout-path = &uart4; }; + + memory@80000000 { + reg = <0x80000000 0x10000000>; + device_type = "memory"; + }; }; &ecspi2 { @@ -55,6 +60,16 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_qspi>; status = "okay"; + + spi-flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "spi-nand"; + spi-max-frequency = <104000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + reg = <0>; + }; }; &wdog1 { diff --git a/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi b/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..0580d043e5ae98b5964849675317492be8d87223 --- /dev/null +++ b/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 exceet electronics GmbH + * Copyright (C) 2018 Kontron Electronics GmbH + * Copyright (c) 2019 Krzysztof Kozlowski + */ + +#include "imx6ul.dtsi" +#include "imx6ul-kontron-sl-common.dtsi" + +/ { + model = "Kontron SL i.MX6UL (N631X SOM)"; + compatible = "kontron,sl-imx6ul", "fsl,imx6ul"; +}; diff --git a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi index c485d058e07908d3d5b58fcfd622b64c2619fa5c..15ee0275feaffd89920809d051a837d3a623afab 100644 --- a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi +++ b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi @@ -212,7 +212,7 @@ enable-active-high; }; - spi_gpio: spi-gpio { + spi_gpio: spi { #address-cells = <1>; #size-cells = <0>; compatible = "spi-gpio"; diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi index c95efd1d8c2dbc355b8216d6bd4a76b94ab035f5..2b5996395701a07b12a13b1294eb7d782d9ef25a 100644 --- a/arch/arm/boot/dts/imx6ul.dtsi +++ b/arch/arm/boot/dts/imx6ul.dtsi @@ -744,7 +744,7 @@ status = "disabled"; }; - sdma: sdma@20ec000 { + sdma: dma-controller@20ec000 { compatible = "fsl,imx6ul-sdma", "fsl,imx6q-sdma", "fsl,imx35-sdma"; reg = <0x020ec000 0x4000>; diff --git a/arch/arm/boot/dts/imx6ull-kontron-bl.dts b/arch/arm/boot/dts/imx6ull-kontron-bl.dts new file mode 100644 index 0000000000000000000000000000000000000000..fa016465cdbc23fb4a882559191aeae5be547ad9 --- /dev/null +++ b/arch/arm/boot/dts/imx6ull-kontron-bl.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 exceet electronics GmbH + * Copyright (C) 2019 Kontron Electronics GmbH + */ + +/dts-v1/; + +#include "imx6ull-kontron-sl.dtsi" +#include "imx6ul-kontron-bl-common.dtsi" + +/ { + model = "Kontron BL i.MX6ULL (N641X S)"; + compatible = "kontron,bl-imx6ull", "kontron,sl-imx6ull", "fsl,imx6ull"; +}; diff --git a/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts b/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts deleted file mode 100644 index 57588a5e1e342a51f784d9f0dd6755bae406a0fe..0000000000000000000000000000000000000000 --- a/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2017 exceet electronics GmbH - * Copyright (C) 2019 Kontron Electronics GmbH - */ - -/dts-v1/; - -#include "imx6ull-kontron-n6411-som.dtsi" -#include "imx6ul-kontron-n6x1x-s.dtsi" - -/ { - model = "Kontron N6411 S"; - compatible = "kontron,imx6ull-n6411-s", "kontron,imx6ull-n6411-som", - "fsl,imx6ull"; -}; diff --git a/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi b/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi deleted file mode 100644 index d000606c07049a6ba8693650806c95edec087c6b..0000000000000000000000000000000000000000 --- a/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2017 exceet electronics GmbH - * Copyright (C) 2018 Kontron Electronics GmbH - */ - -#include "imx6ull.dtsi" -#include "imx6ul-kontron-n6x1x-som-common.dtsi" - -/ { - model = "Kontron N6411 SOM"; - compatible = "kontron,imx6ull-n6311-som", "fsl,imx6ull"; - - memory@80000000 { - reg = <0x80000000 0x20000000>; - device_type = "memory"; - }; -}; - -&qspi { - flash@0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "spi-nand"; - spi-max-frequency = <104000000>; - spi-tx-bus-width = <4>; - spi-rx-bus-width = <4>; - reg = <0>; - - partition@0 { - label = "ubi1"; - reg = <0x00000000 0x08000000>; - }; - - partition@8000000 { - label = "ubi2"; - reg = <0x08000000 0x18000000>; - }; - }; -}; diff --git a/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi b/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..93f10eb3494f5060463086d43b0be602800c3aa7 --- /dev/null +++ b/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 exceet electronics GmbH + * Copyright (C) 2018 Kontron Electronics GmbH + */ + +#include "imx6ull.dtsi" +#include "imx6ul-kontron-sl-common.dtsi" + +/ { + model = "Kontron SL i.MX6ULL (N641X SOM)"; + compatible = "kontron,sl-imx6ull", "fsl,imx6ull"; +}; diff --git a/arch/arm/boot/dts/imx7d-pico.dtsi b/arch/arm/boot/dts/imx7d-pico.dtsi index e519897fae082671ada0dfb31a7a40c859b94d2f..e0bff39e8d3e1bc5e464921ab133d4baf32a2462 100644 --- a/arch/arm/boot/dts/imx7d-pico.dtsi +++ b/arch/arm/boot/dts/imx7d-pico.dtsi @@ -41,7 +41,7 @@ regulator-max-microvolt = <3300000>; gpio = <&gpio1 6 GPIO_ACTIVE_HIGH>; enable-active-high; - }; + }; reg_wlreg_on: regulator-wlreg_on { compatible = "regulator-fixed"; @@ -432,7 +432,7 @@ MX7D_PAD_ENET1_RGMII_RD2__ENET1_RGMII_RD2 0x1 MX7D_PAD_ENET1_RGMII_RD3__ENET1_RGMII_RD3 0x1 MX7D_PAD_ENET1_RGMII_RX_CTL__ENET1_RGMII_RX_CTL 0x1 - MX7D_PAD_SD3_RESET_B__GPIO6_IO11 0x1 /* Ethernet reset */ + MX7D_PAD_SD3_RESET_B__GPIO6_IO11 0x1 /* Ethernet reset */ >; }; @@ -493,19 +493,19 @@ pinctrl_pwm1: pwm1 { fsl,pins = < - MX7D_PAD_GPIO1_IO08__PWM1_OUT 0x7f + MX7D_PAD_GPIO1_IO08__PWM1_OUT 0x7f >; }; pinctrl_pwm2: pwm2 { fsl,pins = < - MX7D_PAD_GPIO1_IO09__PWM2_OUT 0x7f + MX7D_PAD_GPIO1_IO09__PWM2_OUT 0x7f >; }; pinctrl_pwm3: pwm3 { fsl,pins = < - MX7D_PAD_GPIO1_IO10__PWM3_OUT 0x7f + MX7D_PAD_GPIO1_IO10__PWM3_OUT 0x7f >; }; diff --git a/arch/arm/boot/dts/imx7d-sdb.dts b/arch/arm/boot/dts/imx7d-sdb.dts index 78f4224a9bf4eb5795c34605ae186d5cbe52d7aa..f483bc0afe5ea75f8235f32ce8c06a60ba4e0eec 100644 --- a/arch/arm/boot/dts/imx7d-sdb.dts +++ b/arch/arm/boot/dts/imx7d-sdb.dts @@ -24,14 +24,14 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_keys>; - volume-up { + key-volume-up { label = "Volume Up"; gpios = <&gpio5 11 GPIO_ACTIVE_LOW>; linux,code = ; wakeup-source; }; - volume-down { + key-volume-down { label = "Volume Down"; gpios = <&gpio5 10 GPIO_ACTIVE_LOW>; linux,code = ; @@ -39,7 +39,7 @@ }; }; - spi4 { + spi-4 { compatible = "spi-gpio"; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi4>; @@ -206,12 +206,7 @@ interrupt-parent = <&gpio2>; interrupts = <29 0>; pendown-gpio = <&gpio2 29 GPIO_ACTIVE_HIGH>; - ti,x-min = /bits/ 16 <0>; - ti,x-max = /bits/ 16 <0>; - ti,y-min = /bits/ 16 <0>; - ti,y-max = /bits/ 16 <0>; - ti,pressure-max = /bits/ 16 <0>; - ti,x-plate-ohms = /bits/ 16 <400>; + touchscreen-max-pressure = <255>; wakeup-source; }; }; diff --git a/arch/arm/boot/dts/imx7d-zii-rmu2.dts b/arch/arm/boot/dts/imx7d-zii-rmu2.dts index 1065941807e83c8044283e26475bc5cd006c9bc2..1c9f25848bf7fa5e1ad29b760c61f8fa0bd8822b 100644 --- a/arch/arm/boot/dts/imx7d-zii-rmu2.dts +++ b/arch/arm/boot/dts/imx7d-zii-rmu2.dts @@ -24,7 +24,7 @@ pinctrl-0 = <&pinctrl_leds_debug>; pinctrl-names = "default"; - debug { + led-debug { label = "zii:green:debug1"; gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; diff --git a/arch/arm/boot/dts/imx7d-zii-rpu2.dts b/arch/arm/boot/dts/imx7d-zii-rpu2.dts index 893bd30aa2a3978a1d27c351d417c53ad955762f..9d29490ab4c9b0349ecef2ecf535c062f16cbcb3 100644 --- a/arch/arm/boot/dts/imx7d-zii-rpu2.dts +++ b/arch/arm/boot/dts/imx7d-zii-rpu2.dts @@ -36,7 +36,7 @@ pinctrl-0 = <&pinctrl_leds_debug>; pinctrl-names = "default"; - debug { + led-debug { label = "zii:green:debug1"; gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi index 29148285f9fc88b3e1cef4905cf4c61c6fd7a738..0fc9e6b8b05dcfdcbc5b5da02236655afb5aa6e3 100644 --- a/arch/arm/boot/dts/imx7s.dtsi +++ b/arch/arm/boot/dts/imx7s.dtsi @@ -1224,7 +1224,7 @@ status = "disabled"; }; - sdma: sdma@30bd0000 { + sdma: dma-controller@30bd0000 { compatible = "fsl,imx7d-sdma", "fsl,imx35-sdma"; reg = <0x30bd0000 0x10000>; interrupts = ; diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi index bcec98b96411437c831d9e91ddaf2f63d657ea9e..7f7d2d5122fb629f437e84eb6b3008575de8ba4c 100644 --- a/arch/arm/boot/dts/imx7ulp.dtsi +++ b/arch/arm/boot/dts/imx7ulp.dtsi @@ -328,8 +328,9 @@ compatible = "fsl,imx7ulp-lpi2c"; reg = <0x40a40000 0x10000>; interrupts = ; - clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>; - clock-names = "ipg"; + clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>, + <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>; + clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>; assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC>; assigned-clock-rates = <48000000>; @@ -340,8 +341,9 @@ compatible = "fsl,imx7ulp-lpi2c"; reg = <0x40a50000 0x10000>; interrupts = ; - clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>; - clock-names = "ipg"; + clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>, + <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>; + clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>; assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC>; assigned-clock-rates = <48000000>; diff --git a/arch/arm/boot/dts/integratorap-im-pd1.dts b/arch/arm/boot/dts/integratorap-im-pd1.dts index d47bfb66d0692a0b0cc7c7842b887df0355e8801..cc514cf07bff7dcab39cb2b33f87b43ad0c4d0b0 100644 --- a/arch/arm/boot/dts/integratorap-im-pd1.dts +++ b/arch/arm/boot/dts/integratorap-im-pd1.dts @@ -178,12 +178,12 @@ clock-names = "uartclk", "apb_pclk"; }; - ssp@300000 { + spi@300000 { compatible = "arm,pl022", "arm,primecell"; reg = <0x00300000 0x1000>; interrupts-extended = <&impd1_vic 3>; clocks = <&impd1_sspclk>, <&sysclk>; - clock-names = "spiclk", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; impd1_gpio0: gpio@400000 { @@ -249,6 +249,7 @@ /* 640x480 16bpp @ 25.175MHz is 36827428 bytes/s */ max-memory-bandwidth = <40000000>; memory-region = <&impd1_ram>; + dma-ranges; port@0 { #address-cells = <1>; diff --git a/arch/arm/boot/dts/integratorap.dts b/arch/arm/boot/dts/integratorap.dts index 9b652cc27b14133628177c2763f147d7f00f4876..9148287fa0a930de7c329d412cde880740256469 100644 --- a/arch/arm/boot/dts/integratorap.dts +++ b/arch/arm/boot/dts/integratorap.dts @@ -160,6 +160,7 @@ pci: pciv3@62000000 { compatible = "arm,integrator-ap-pci", "v3,v360epc-pci"; + device_type = "pci"; #interrupt-cells = <1>; #size-cells = <2>; #address-cells = <3>; @@ -261,7 +262,7 @@ lm0: bus@c0000000 { compatible = "simple-bus"; ranges = <0x00000000 0xc0000000 0x10000000>; - dma-ranges = <0x00000000 0x80000000 0x10000000>; + dma-ranges = <0x00000000 0xc0000000 0x10000000>; reg = <0xc0000000 0x10000000>; #address-cells = <1>; #size-cells = <1>; @@ -269,7 +270,7 @@ lm1: bus@d0000000 { compatible = "simple-bus"; ranges = <0x00000000 0xd0000000 0x10000000>; - dma-ranges = <0x00000000 0x80000000 0x10000000>; + dma-ranges = <0x00000000 0xd0000000 0x10000000>; reg = <0xd0000000 0x10000000>; #address-cells = <1>; #size-cells = <1>; @@ -277,7 +278,7 @@ lm2: bus@e0000000 { compatible = "simple-bus"; ranges = <0x00000000 0xe0000000 0x10000000>; - dma-ranges = <0x00000000 0x80000000 0x10000000>; + dma-ranges = <0x00000000 0xe0000000 0x10000000>; reg = <0xe0000000 0x10000000>; #address-cells = <1>; #size-cells = <1>; @@ -285,7 +286,7 @@ lm3: bus@f0000000 { compatible = "simple-bus"; ranges = <0x00000000 0xf0000000 0x10000000>; - dma-ranges = <0x00000000 0x80000000 0x10000000>; + dma-ranges = <0x00000000 0xf0000000 0x10000000>; reg = <0xf0000000 0x10000000>; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm/boot/dts/kirkwood-6192.dtsi b/arch/arm/boot/dts/kirkwood-6192.dtsi index 396bcba08adba427a4b44f322d371c9201a9dbe0..705c0d7effed0054936add33474808c8c4bb3e7f 100644 --- a/arch/arm/boot/dts/kirkwood-6192.dtsi +++ b/arch/arm/boot/dts/kirkwood-6192.dtsi @@ -26,12 +26,22 @@ ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 9>; + interrupt-names = "intx", "error"; + interrupts = <9>, <44>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gate_clk 2>; status = "disabled"; + + pcie_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; }; diff --git a/arch/arm/boot/dts/kirkwood-6281.dtsi b/arch/arm/boot/dts/kirkwood-6281.dtsi index faa05849a40d49409d68b3275127ba7789050114..8e311165fd1309ede4cd550d52cbac64178320d5 100644 --- a/arch/arm/boot/dts/kirkwood-6281.dtsi +++ b/arch/arm/boot/dts/kirkwood-6281.dtsi @@ -26,12 +26,22 @@ ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 9>; + interrupt-names = "intx", "error"; + interrupts = <9>, <44>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gate_clk 2>; status = "disabled"; + + pcie_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; }; diff --git a/arch/arm/boot/dts/kirkwood-6282.dtsi b/arch/arm/boot/dts/kirkwood-6282.dtsi index e84c54b77dead570c6e25324f838f91ddb3ae771..e33723160ce7cfdcbe6bb09c8e3a3f4a088fc023 100644 --- a/arch/arm/boot/dts/kirkwood-6282.dtsi +++ b/arch/arm/boot/dts/kirkwood-6282.dtsi @@ -30,12 +30,22 @@ ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 9>; + interrupt-names = "intx", "error"; + interrupts = <9>, <44>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie0_intc 0>, + <0 0 0 2 &pcie0_intc 1>, + <0 0 0 3 &pcie0_intc 2>, + <0 0 0 4 &pcie0_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gate_clk 2>; status = "disabled"; + + pcie0_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; pcie1: pcie@2,0 { @@ -48,12 +58,22 @@ ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0 0x81000000 0 0 0x81000000 0x2 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 10>; + interrupt-names = "intx", "error"; + interrupts = <10>, <45>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie1_intc 0>, + <0 0 0 2 &pcie1_intc 1>, + <0 0 0 3 &pcie1_intc 2>, + <0 0 0 4 &pcie1_intc 3>; marvell,pcie-port = <1>; marvell,pcie-lane = <0>; clocks = <&gate_clk 18>; status = "disabled"; + + pcie1_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; }; diff --git a/arch/arm/boot/dts/kirkwood-98dx4122.dtsi b/arch/arm/boot/dts/kirkwood-98dx4122.dtsi index 299c147298c35120f9b122ef780e61ef8f18c31b..c3469a2fc58a0b1caa7cef769df6db54cdf01edc 100644 --- a/arch/arm/boot/dts/kirkwood-98dx4122.dtsi +++ b/arch/arm/boot/dts/kirkwood-98dx4122.dtsi @@ -26,12 +26,22 @@ ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0 0x81000000 0 0 0x81000000 0x1 0 1 0>; bus-range = <0x00 0xff>; - interrupt-map-mask = <0 0 0 0>; - interrupt-map = <0 0 0 0 &intc 9>; + interrupt-names = "intx", "error"; + interrupts = <9>, <44>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; marvell,pcie-port = <0>; marvell,pcie-lane = <0>; clocks = <&gate_clk 2>; status = "disabled"; + + pcie_intc: interrupt-controller { + interrupt-controller; + #interrupt-cells = <1>; + }; }; }; }; diff --git a/arch/arm/boot/dts/kirkwood-lsxl.dtsi b/arch/arm/boot/dts/kirkwood-lsxl.dtsi index 7b151acb99846f21e68404bab0bdc76d838475fd..88b70ba1c8feec67b670335c60235aa21342687e 100644 --- a/arch/arm/boot/dts/kirkwood-lsxl.dtsi +++ b/arch/arm/boot/dts/kirkwood-lsxl.dtsi @@ -10,6 +10,11 @@ ocp@f1000000 { pinctrl: pin-controller@10000 { + /* Non-default UART pins */ + pmx_uart0: pmx-uart0 { + marvell,pins = "mpp4", "mpp5"; + }; + pmx_power_hdd: pmx-power-hdd { marvell,pins = "mpp10"; marvell,function = "gpo"; @@ -213,22 +218,11 @@ &mdio { status = "okay"; - ethphy0: ethernet-phy@0 { - reg = <0>; - }; - ethphy1: ethernet-phy@8 { reg = <8>; }; }; -ð0 { - status = "okay"; - ethernet0-port@0 { - phy-handle = <ðphy0>; - }; -}; - ð1 { status = "okay"; ethernet1-port@0 { diff --git a/arch/arm/boot/dts/lan966x-pcb8290.dts b/arch/arm/boot/dts/lan966x-pcb8290.dts new file mode 100644 index 0000000000000000000000000000000000000000..77187f59f04df1f3fcd4ac22e07dc19e6101e47b --- /dev/null +++ b/arch/arm/boot/dts/lan966x-pcb8290.dts @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * lan966x-pcb8290.dts - Device Tree file for LAN966X-PCB8290 board + * + * Copyright (C) 2022 Microchip Technology Inc. and its subsidiaries + * + * Author: Horatiu Vultur + */ +/dts-v1/; +#include "lan966x.dtsi" +#include "dt-bindings/phy/phy-lan966x-serdes.h" + +/ { + model = "Microchip EVB LAN9668"; + compatible = "microchip,lan9668-pcb8290", "microchip,lan9668", "microchip,lan966"; + + gpio-restart { + compatible = "gpio-restart"; + gpios = <&gpio 56 GPIO_ACTIVE_LOW>; + priority = <200>; + }; +}; + +&aes { + status = "disabled"; /* Reserved by secure OS */ +}; + +&gpio { + miim_a_pins: mdio-pins { + /* MDC, MDIO */ + pins = "GPIO_28", "GPIO_29"; + function = "miim_a"; + }; + + pps_out_pins: pps-out-pins { + /* 1pps output */ + pins = "GPIO_38"; + function = "ptpsync_3"; + }; + + ptp_ext_pins: ptp-ext-pins { + /* 1pps input */ + pins = "GPIO_35"; + function = "ptpsync_0"; + }; + + udc_pins: ucd-pins { + /* VBUS_DET B */ + pins = "GPIO_8"; + function = "usb_slave_b"; + }; +}; + +&mdio0 { + pinctrl-0 = <&miim_a_pins>; + pinctrl-names = "default"; + status = "okay"; + + ext_phy0: ethernet-phy@7 { + reg = <7>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy1: ethernet-phy@8 { + reg = <8>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy2: ethernet-phy@9 { + reg = <9>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy3: ethernet-phy@10 { + reg = <10>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy4: ethernet-phy@15 { + reg = <15>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy5: ethernet-phy@16 { + reg = <16>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy6: ethernet-phy@17 { + reg = <17>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; + + ext_phy7: ethernet-phy@18 { + reg = <18>; + coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>; + }; +}; + +&port0 { + reg = <2>; + phy-handle = <&ext_phy2>; + phy-mode = "qsgmii"; + phys = <&serdes 0 SERDES6G(1)>; + status = "okay"; +}; + +&port1 { + reg = <3>; + phy-handle = <&ext_phy3>; + phy-mode = "qsgmii"; + phys = <&serdes 1 SERDES6G(1)>; + status = "okay"; +}; + +&port2 { + reg = <0>; + phy-handle = <&ext_phy0>; + phy-mode = "qsgmii"; + phys = <&serdes 2 SERDES6G(1)>; + status = "okay"; +}; + +&port3 { + reg = <1>; + phy-handle = <&ext_phy1>; + phy-mode = "qsgmii"; + phys = <&serdes 3 SERDES6G(1)>; + status = "okay"; +}; + +&port4 { + reg = <6>; + phy-handle = <&ext_phy6>; + phy-mode = "qsgmii"; + phys = <&serdes 4 SERDES6G(2)>; + status = "okay"; +}; + +&port5 { + reg = <7>; + phy-handle = <&ext_phy7>; + phy-mode = "qsgmii"; + phys = <&serdes 5 SERDES6G(2)>; + status = "okay"; +}; + +&port6 { + reg = <4>; + phy-handle = <&ext_phy4>; + phy-mode = "qsgmii"; + phys = <&serdes 6 SERDES6G(2)>; + status = "okay"; +}; + +&port7 { + reg = <5>; + phy-handle = <&ext_phy5>; + phy-mode = "qsgmii"; + phys = <&serdes 7 SERDES6G(2)>; + status = "okay"; +}; + +&serdes { + status = "okay"; +}; + +&switch { + pinctrl-0 = <&pps_out_pins>, <&ptp_ext_pins>; + pinctrl-names = "default"; + status = "okay"; +}; + +&udc { + pinctrl-0 = <&udc_pins>; + pinctrl-names = "default"; + atmel,vbus-gpio = <&gpio 8 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/lan966x-pcb8291.dts b/arch/arm/boot/dts/lan966x-pcb8291.dts index 24d9055c4a08c679892e77922a42bf66307fb319..f4f054cdf2a87afa9de1a3966f2e9166b701772d 100644 --- a/arch/arm/boot/dts/lan966x-pcb8291.dts +++ b/arch/arm/boot/dts/lan966x-pcb8291.dts @@ -23,6 +23,38 @@ gpios = <&gpio 56 GPIO_ACTIVE_LOW>; priority = <200>; }; + + leds { + compatible = "gpio-leds"; + + led-s0-blue { + label = "s0:blue"; + gpios = <&sgpio_out 2 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s0-green { + label = "s0:green"; + gpios = <&sgpio_out 2 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s1-blue { + label = "s1:blue"; + gpios = <&sgpio_out 3 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s1-green { + label = "s1:green"; + gpios = <&sgpio_out 3 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + }; +}; + +&aes { + status = "disabled"; /* Reserved by secure OS */ }; &gpio { diff --git a/arch/arm/boot/dts/lan966x-pcb8309.dts b/arch/arm/boot/dts/lan966x-pcb8309.dts index 05ce27ed5648f3285aea3591dace6ff1c7249688..c436cd20d4b4cf86f877d79ca28e39a16987b426 100644 --- a/arch/arm/boot/dts/lan966x-pcb8309.dts +++ b/arch/arm/boot/dts/lan966x-pcb8309.dts @@ -42,6 +42,34 @@ }; }; + leds { + compatible = "gpio-leds"; + + led-s0-green { + label = "s0:green"; + gpios = <&sgpio_out 2 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s0-red { + label = "s0:red"; + gpios = <&sgpio_out 2 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s1-green { + label = "s1:green"; + gpios = <&sgpio_out 3 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-s1-red { + label = "s1:red"; + gpios = <&sgpio_out 3 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + }; + mux: mux-controller { compatible = "gpio-mux"; #mux-control-cells = <0>; @@ -69,6 +97,10 @@ }; }; +&aes { + status = "disabled"; /* Reserved by secure OS */ +}; + &flx3 { atmel,flexcom-mode = ; status = "okay"; diff --git a/arch/arm/boot/dts/lan966x.dtsi b/arch/arm/boot/dts/lan966x.dtsi index 894bf9da19a4bc038fd2613ebfa79b91b50ba329..0bf8187134227eb99c646a5a00314f821b0d88e2 100644 --- a/arch/arm/boot/dts/lan966x.dtsi +++ b/arch/arm/boot/dts/lan966x.dtsi @@ -541,13 +541,13 @@ phy0: ethernet-phy@1 { reg = <1>; - interrupts = ; + interrupts = ; status = "disabled"; }; phy1: ethernet-phy@2 { reg = <2>; - interrupts = ; + interrupts = ; status = "disabled"; }; }; diff --git a/arch/arm/boot/dts/ls1021a.dtsi b/arch/arm/boot/dts/ls1021a.dtsi index fa761620f0731ea282b5cf37c54aed28b7a852b6..7c5510e34494fc49afa91d614d1f491103d48032 100644 --- a/arch/arm/boot/dts/ls1021a.dtsi +++ b/arch/arm/boot/dts/ls1021a.dtsi @@ -647,7 +647,7 @@ status = "disabled"; }; - edma0: edma@2c00000 { + edma0: dma-controller@2c00000 { #dma-cells = <2>; compatible = "fsl,vf610-edma"; reg = <0x0 0x2c00000 0x0 0x10000>, diff --git a/arch/arm/boot/dts/moxart-uc7112lx.dts b/arch/arm/boot/dts/moxart-uc7112lx.dts index eb5291b0ee3aa15a6928b08c52806340a234de4a..e07b807b4cec569bbdbf3dd1eb390212b178cb0b 100644 --- a/arch/arm/boot/dts/moxart-uc7112lx.dts +++ b/arch/arm/boot/dts/moxart-uc7112lx.dts @@ -79,7 +79,7 @@ clocks = <&ref12>; }; -&sdhci { +&mmc { status = "okay"; }; diff --git a/arch/arm/boot/dts/moxart.dtsi b/arch/arm/boot/dts/moxart.dtsi index f5f070a8748231077f623ceac4ce1fa785262624..764832ddfa78ace56941070a49915cc276669d11 100644 --- a/arch/arm/boot/dts/moxart.dtsi +++ b/arch/arm/boot/dts/moxart.dtsi @@ -93,8 +93,8 @@ clock-names = "PCLK"; }; - sdhci: sdhci@98e00000 { - compatible = "moxa,moxart-sdhci"; + mmc: mmc@98e00000 { + compatible = "moxa,moxart-mmc"; reg = <0x98e00000 0x5C>; interrupts = <5 IRQ_TYPE_LEVEL_HIGH>; clocks = <&clk_apb>; diff --git a/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts b/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts index ace8cea27949149d5fa5d96463e7b5b3d3a39d52..215613c652508348a17c1460dddcca5019470c25 100644 --- a/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts +++ b/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts @@ -8,6 +8,8 @@ #include "qcom-msm8226.dtsi" #include "qcom-pm8226.dtsi" +/delete-node/ &adsp_region; + / { model = "ASUS ZenWatch 2"; compatible = "asus,sparrow", "qcom,apq8026"; @@ -57,6 +59,10 @@ }; }; +&adsp { + status = "okay"; +}; + &blsp1_uart1 { status = "okay"; diff --git a/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts b/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts index 2b7e52fda6a7860eb5695d7deeac33d3db9e6b9b..193569f0ca5f7a0215615ec8543923ab4dd2b93a 100644 --- a/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts +++ b/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts @@ -8,6 +8,8 @@ #include "qcom-msm8226.dtsi" #include "qcom-pm8226.dtsi" +/delete-node/ &adsp_region; + / { model = "LG G Watch R"; compatible = "lg,lenok", "qcom,apq8026"; @@ -23,6 +25,13 @@ stdout-path = "serial0:115200n8"; }; + reserved-memory { + adsp_region: adsp@3300000 { + reg = <0x03300000 0x1400000>; + no-map; + }; + }; + vreg_wlan: wlan-regulator { compatible = "regulator-fixed"; @@ -38,6 +47,10 @@ }; }; +&adsp { + status = "okay"; +}; + &blsp1_i2c1 { status = "okay"; diff --git a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts index 70a1dd629c7aa90b0e46847d6db8e3258aaae6c3..573e4dc66bb0274b839995bb3285e398ca393ed1 100644 --- a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts +++ b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts @@ -159,6 +159,19 @@ }; }; + dragon_gsbi3_i2c_pins: gsbi3_i2c { + mux { + pins = "gpio43", "gpio44"; + function = "gsbi3"; + }; + pinconf { + pins = "gpio43", "gpio44"; + drive-strength = <8>; + /* These have external pull-up 2.2kOhm to 1.8V */ + bias-disable; + }; + }; + dragon_gsbi8_i2c_pins: gsbi8_i2c { mux { pins = "gpio64", "gpio65"; @@ -240,6 +253,22 @@ bias-pull-up; }; }; + + dragon_tma340_gpios: tma340 { + reset { + /* RESET line, TS_ATTN, WAKE_CTP */ + pins = "gpio58"; + function = "gpio"; + drive-strength = <6>; + bias-disable; + }; + irq { + pins = "gpio61"; /* IRQ line */ + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; + }; }; qcom,ssbi@500000 { @@ -444,6 +473,45 @@ }; }; + gsbi@16200000 { + qcom,mode = ; + status = "okay"; + + gsbi3_i2c: i2c@16280000 { + pinctrl-names = "default"; + pinctrl-0 = <&dragon_gsbi3_i2c_pins>; + status = "okay"; + + touchscreen@24 { + compatible = "cypress,cy8ctma340"; + reg = <0x24>; + /* Certainly we can do at least 400 kHz */ + clock-frequency = <400000>; + /* IRQ on GPIO61 called /CTP_INT */ + interrupt-parent = <&tlmm>; + interrupts = <61 IRQ_TYPE_EDGE_FALLING>; + /* + * The I2C bus is using a PCA9306 level translator from L16A + * to L2B so these two voltages are needed and L16A is + * kind of the IO voltage, however L16Aisn't really fed to + * the TMA340, which relies entirely on L2B (PM8901 L2). + */ + vcpin-supply = <&pm8058_l16>; + vdd-supply = <&pm8901_l2>; + /* GPIO58, called WAKE_CTP */ + reset-gpios = <&tlmm 58 GPIO_ACTIVE_LOW>; + touchscreen-size-x = <480>; + touchscreen-size-y = <800>; + active-interval-ms = <0>; + touch-timeout-ms = <255>; + lowpower-interval-ms = <10>; + bootloader-key = /bits/ 8 <0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07>; + pinctrl-names = "default"; + pinctrl-0 = <&dragon_tma340_gpios>; + }; + }; + }; + gsbi@19800000 { status = "okay"; qcom,mode = ; @@ -634,7 +702,8 @@ bias-pull-down; }; l2 { - regulator-min-microvolt = <2850000>; + /* TMA340 requires strictly 3.3V */ + regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; bias-pull-down; }; diff --git a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts index e3bf57cd7423c55409f979cf964b423ba3eb4df5..529629a0a9dcf431c756d8b116c916e22abeff2a 100644 --- a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts +++ b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts @@ -215,7 +215,7 @@ vdda_refclk-supply = <&v3p3_fixed>; pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - perst-gpio = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>; }; amba { diff --git a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts index 0322cb88d448051d5a47864f7173aedc5179da2b..a7f90217661b9cb0b5f55b08e5be98f9494be9db 100644 --- a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts +++ b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts @@ -287,7 +287,7 @@ vdda_refclk-supply = <&ext_3p3v>; pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - perst-gpio = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>; }; qcom,ssbi@500000 { diff --git a/arch/arm/boot/dts/qcom-apq8064.dtsi b/arch/arm/boot/dts/qcom-apq8064.dtsi index ada4c828bf2f49dff1d3bc95188fc02ba46b8061..942aa2278355d809d6fa35819e2aeb093f08654a 100644 --- a/arch/arm/boot/dts/qcom-apq8064.dtsi +++ b/arch/arm/boot/dts/qcom-apq8064.dtsi @@ -2,6 +2,7 @@ /dts-v1/; #include +#include #include #include #include @@ -815,6 +816,10 @@ #clock-cells = <1>; #power-domain-cells = <1>; #reset-cells = <1>; + clocks = <&cxo_board>, + <&pxo_board>, + <&lcc PLL4>; + clock-names = "cxo", "pxo", "pll4"; tsens: thermal-sensor { compatible = "qcom,msm8960-tsens"; @@ -834,6 +839,20 @@ reg = <0x28000000 0x1000>; #clock-cells = <1>; #reset-cells = <1>; + clocks = <&pxo_board>, + <&gcc PLL4_VOTE>, + <0>, + <0>, <0>, + <0>, <0>, + <0>; + clock-names = "pxo", + "pll4_vote", + "mi2s_codec_clk", + "codec_i2s_mic_codec_clk", + "spare_i2s_mic_codec_clk", + "codec_i2s_spkr_codec_clk", + "spare_i2s_spkr_codec_clk", + "pcm_codec_clk"; }; mmcc: clock-controller@4000000 { @@ -842,6 +861,22 @@ #clock-cells = <1>; #power-domain-cells = <1>; #reset-cells = <1>; + clocks = <&pxo_board>, + <&gcc PLL3>, + <&gcc PLL8_VOTE>, + <&dsi0_phy 1>, + <&dsi0_phy 0>, + <0>, + <0>, + <0>; + clock-names = "pxo", + "pll3", + "pll8_vote", + "dsi1pll", + "dsi1pllbyte", + "dsi2pll", + "dsi2pllbyte", + "hdmipll"; }; l2cc: clock-controller@2011000 { @@ -1384,7 +1419,7 @@ }; pcie: pci@1b500000 { - compatible = "qcom,pcie-apq8064", "snps,dw-pcie"; + compatible = "qcom,pcie-apq8064"; reg = <0x1b500000 0x1000>, <0x1b502000 0x80>, <0x1b600000 0x100>, diff --git a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts index 3051a861ff0c3b04eb00cee12e26e2de26f714f2..91716298ec5ed9e37f07fbd6b0b3079587b96eef 100644 --- a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts +++ b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include "qcom-msm8974.dtsi" #include "qcom-pm8841.dtsi" #include "qcom-pm8941.dtsi" @@ -261,7 +262,7 @@ &sdhc_2 { status = "okay"; - cd-gpios = <&tlmm 62 0x1>; + cd-gpios = <&tlmm 62 GPIO_ACTIVE_LOW>; vmmc-supply = <&pm8941_l21>; vqmmc-supply = <&pm8941_l13>; diff --git a/arch/arm/boot/dts/qcom-apq8084.dtsi b/arch/arm/boot/dts/qcom-apq8084.dtsi index 72f9255855a1dff4bfc7a34f0e953eec03b327a7..f2fb7c975af84f562f40584e79e0b38bfb050561 100644 --- a/arch/arm/boot/dts/qcom-apq8084.dtsi +++ b/arch/arm/boot/dts/qcom-apq8084.dtsi @@ -239,6 +239,11 @@ reg = <0xf9011000 0x1000>; }; + sram@fc190000 { + compatible = "qcom,apq8084-rpm-stats"; + reg = <0xfc190000 0x10000>; + }; + qfprom: qfprom@fc4bc000 { compatible = "qcom,apq8084-qfprom", "qcom,qfprom"; reg = <0xfc4bc000 0x1000>; @@ -383,14 +388,9 @@ reg = <0xfc400000 0x4000>; }; - tcsr_mutex_regs: syscon@fd484000 { - compatible = "syscon"; - reg = <0xfd484000 0x2000>; - }; - - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x80>; + tcsr_mutex: hwlock@fd484000 { + compatible = "qcom,apq8084-tcsr-mutex", "qcom,tcsr-mutex"; + reg = <0xfd484000 0x1000>; #hwlock-cells = <1>; }; @@ -422,26 +422,26 @@ mmc@f9824900 { compatible = "qcom,apq8084-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9824900 0x11c>, <0xf9824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC1_APPS_CLK>, - <&gcc GCC_SDCC1_AHB_CLK>, + clocks = <&gcc GCC_SDCC1_AHB_CLK>, + <&gcc GCC_SDCC1_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; status = "disabled"; }; mmc@f98a4900 { compatible = "qcom,apq8084-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC2_APPS_CLK>, - <&gcc GCC_SDCC2_AHB_CLK>, + clocks = <&gcc GCC_SDCC2_AHB_CLK>, + <&gcc GCC_SDCC2_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; status = "disabled"; }; diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi index 03bb9e1768c436b20deec983e8c95795ddf8817c..0505270cf508cf4e2aa4d2f94bd45f63b65d6960 100644 --- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi +++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi @@ -14,6 +14,7 @@ * */ +#include #include "qcom-ipq4019.dtsi" / { @@ -72,7 +73,7 @@ pinctrl-0 = <&spi_0_pins>; pinctrl-names = "default"; status = "okay"; - cs-gpios = <&tlmm 54 0>; + cs-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>; mx25l25635e@0 { #address-cells = <1>; diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi index 44a9597d8bfd576307a06de18d447669ede66f73..a63b3778636d4d3b173521fbf7d01dacdaa40e71 100644 --- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi +++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi @@ -87,7 +87,7 @@ pinctrl-0 = <&spi_0_pins>; pinctrl-names = "default"; status = "okay"; - cs-gpios = <&tlmm 12 0>; + cs-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>; flash@0 { #address-cells = <1>; @@ -100,7 +100,7 @@ pci@40000000 { status = "okay"; - perst-gpio = <&tlmm 38 0x1>; + perst-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>; }; qpic-nand@79b0000 { diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts index c7a6e77da272659951afc2606b14739ae4826916..ea2987fcbff8d374c4448cb8e6923d43b21d9f4b 100644 --- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts +++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (c) 2018, The Linux Foundation. All rights reserved. +#include #include "qcom-ipq4019-ap.dk07.1.dtsi" / { @@ -10,7 +11,7 @@ soc { pci@40000000 { status = "okay"; - perst-gpio = <&tlmm 38 0x1>; + perst-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>; }; spi@78b6000 { @@ -50,7 +51,7 @@ pinctrl-0 = <&spi_0_pins>; pinctrl-names = "default"; status = "okay"; - cs-gpios = <&tlmm 12 0>; + cs-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>; flash@0 { #address-cells = <1>; diff --git a/arch/arm/boot/dts/qcom-ipq4019.dtsi b/arch/arm/boot/dts/qcom-ipq4019.dtsi index bb307b8f678c2dfc0e9b7e5ccb3012578f1ad8c8..b23591110bd2babc57df3214c3039b1a72eb59a6 100644 --- a/arch/arm/boot/dts/qcom-ipq4019.dtsi +++ b/arch/arm/boot/dts/qcom-ipq4019.dtsi @@ -224,12 +224,13 @@ sdhci: mmc@7824900 { compatible = "qcom,sdhci-msm-v4"; reg = <0x7824900 0x11c>, <0x7824000 0x800>; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; bus-width = <8>; - clocks = <&gcc GCC_SDCC1_APPS_CLK>, <&gcc GCC_SDCC1_AHB_CLK>, + clocks = <&gcc GCC_SDCC1_AHB_CLK>, <&gcc GCC_SDCC1_APPS_CLK>, <&gcc GCC_DCD_XO_CLK>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; status = "disabled"; }; @@ -412,7 +413,7 @@ }; pcie0: pci@40000000 { - compatible = "qcom,pcie-ipq4019", "snps,dw-pcie"; + compatible = "qcom,pcie-ipq4019"; reg = <0x40000000 0xf1d 0x40000f20 0xa8 0x80000 0x2000 diff --git a/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..9d06255104c71c133df07a43f208cbb05d955736 --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "qcom-ipq8062.dtsi" + +&rpm { + smb208_regulators: regulators { + compatible = "qcom,rpm-smb208-regulators"; + + smb208_s1a: s1a { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s1b: s1b { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2a: s2a { + regulator-min-microvolt = < 800000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2b: s2b { + regulator-min-microvolt = < 800000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + }; +}; diff --git a/arch/arm/boot/dts/qcom-ipq8062.dtsi b/arch/arm/boot/dts/qcom-ipq8062.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..5d3ebd3e2e5144ac3c2f7141d51cca50300c4bbe --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8062.dtsi @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "qcom-ipq8064-v2.0.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. IPQ8062"; + compatible = "qcom,ipq8062", "qcom,ipq8064"; +}; diff --git a/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..0442580b22ded2c7caa4326f49762c4fe1e258b5 --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "qcom-ipq8064-v2.0.dtsi" + +&rpm { + smb208_regulators: regulators { + compatible = "qcom,rpm-smb208-regulators"; + + smb208_s1a: s1a { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s1b: s1b { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2a: s2a { + regulator-min-microvolt = < 800000>; + regulator-max-microvolt = <1250000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2b: s2b { + regulator-min-microvolt = < 800000>; + regulator-max-microvolt = <1250000>; + + qcom,switch-mode-frequency = <1200000>; + }; + }; +}; diff --git a/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi b/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..2f117d576daf3a383e641f44b5b6df8e19476aee --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "qcom-ipq8064.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. IPQ8064-v2.0"; + + aliases { + serial0 = &gsbi4_serial; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + ranges; + + rsvd@41200000 { + reg = <0x41200000 0x300000>; + no-map; + }; + }; +}; + +&gsbi4 { + qcom,mode = ; + status = "okay"; + + serial@16340000 { + status = "okay"; + }; + /* + * The i2c device on gsbi4 should not be enabled. + * On ipq806x designs gsbi4 i2c is meant for exclusive + * RPM usage. Turning this on in kernel manifests as + * i2c failure for the RPM. + */ +}; + +&pcie0 { + compatible = "qcom,pcie-ipq8064-v2"; +}; + +&pcie1 { + compatible = "qcom,pcie-ipq8064-v2"; +}; + +&pcie2 { + compatible = "qcom,pcie-ipq8064-v2"; +}; + +&sata { + ports-implemented = <0x1>; +}; + +&ss_phy_0 { + qcom,rx-eq = <2>; + qcom,tx-deamp_3_5db = <32>; + qcom,mpll = <5>; +}; + +&ss_phy_1 { + qcom,rx-eq = <2>; + qcom,tx-deamp_3_5db = <32>; + qcom,mpll = <5>; +}; diff --git a/arch/arm/boot/dts/qcom-ipq8064.dtsi b/arch/arm/boot/dts/qcom-ipq8064.dtsi index c8337c870bdb27ef239d49edcbf1990f97258286..90c08b51680aa2cd6416b4293a60b725f6932c3c 100644 --- a/arch/arm/boot/dts/qcom-ipq8064.dtsi +++ b/arch/arm/boot/dts/qcom-ipq8064.dtsi @@ -332,24 +332,64 @@ ranges; compatible = "simple-bus"; - lpass@28100000 { - compatible = "qcom,lpass-cpu"; - status = "disabled"; - clocks = <&lcc AHBIX_CLK>, - <&lcc MI2S_OSR_CLK>, - <&lcc MI2S_BIT_CLK>; - clock-names = "ahbix-clk", - "mi2s-osr-clk", - "mi2s-bit-clk"; - interrupts = ; - interrupt-names = "lpass-irq-lpaif"; - reg = <0x28100000 0x10000>; - reg-names = "lpass-lpaif"; + stmmac_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <7>; + snps,rd_osr_lmt = <7>; + snps,blen = <16 0 0 0 0 0 0>; + }; + + vsdcc_fixed: vsdcc-regulator { + compatible = "regulator-fixed"; + regulator-name = "SDCC Power"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + rpm: rpm@108000 { + compatible = "qcom,rpm-ipq8064"; + reg = <0x00108000 0x1000>; + qcom,ipc = <&l2cc 0x8 2>; + + interrupts = , + , + ; + interrupt-names = "ack", "err", "wakeup"; + + clocks = <&gcc RPM_MSG_RAM_H_CLK>; + clock-names = "ram"; + + rpmcc: clock-controller { + compatible = "qcom,rpmcc-ipq806x", "qcom,rpmcc"; + #clock-cells = <1>; + }; + }; + + qcom,ssbi@500000 { + compatible = "qcom,ssbi"; + reg = <0x00500000 0x1000>; + qcom,controller-type = "pmic-arbiter"; + }; + + qfprom: qfprom@700000 { + compatible = "qcom,ipq8064-qfprom", "qcom,qfprom"; + reg = <0x00700000 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + speedbin_efuse: speedbin@c0 { + reg = <0xc0 0x4>; + }; + tsens_calib: calib@400 { + reg = <0x400 0xb>; + }; + tsens_calib_backup: calib_backup@410 { + reg = <0x410 0xb>; + }; }; qcom_pinmux: pinmux@800000 { compatible = "qcom,ipq8064-pinctrl"; - reg = <0x800000 0x4000>; + reg = <0x00800000 0x4000>; gpio-controller; gpio-ranges = <&qcom_pinmux 0 0 69>; @@ -471,6 +511,35 @@ }; }; + gcc: clock-controller@900000 { + compatible = "qcom,gcc-ipq8064", "syscon"; + clocks = <&pxo_board>, <&cxo_board>; + clock-names = "pxo", "cxo"; + reg = <0x00900000 0x4000>; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + + tsens: thermal-sensor@900000 { + compatible = "qcom,ipq8064-tsens"; + + nvmem-cells = <&tsens_calib>, <&tsens_calib_backup>; + nvmem-cell-names = "calib", "calib_backup"; + interrupts = ; + interrupt-names = "uplow"; + + #qcom,sensors = <11>; + #thermal-sensor-cells = <1>; + }; + }; + + sfpb_mutex: hwlock@1200600 { + compatible = "qcom,sfpb-mutex"; + reg = <0x01200600 0x100>; + + #hwlock-cells = <1>; + }; + intc: interrupt-controller@2000000 { compatible = "qcom,msm-qgic2"; interrupt-controller; @@ -500,46 +569,198 @@ cpu-offset = <0x80000>; }; + l2cc: clock-controller@2011000 { + compatible = "qcom,kpss-gcc", "syscon"; + reg = <0x02011000 0x1000>; + clocks = <&gcc PLL8_VOTE>, <&pxo_board>; + clock-names = "pll8_vote", "pxo"; + clock-output-names = "acpu_l2_aux"; + }; + acc0: clock-controller@2088000 { compatible = "qcom,kpss-acc-v1"; reg = <0x02088000 0x1000>, <0x02008000 0x1000>; }; + saw0: regulator@2089000 { + compatible = "qcom,saw2"; + reg = <0x02089000 0x1000>, <0x02009000 0x1000>; + regulator; + }; + acc1: clock-controller@2098000 { compatible = "qcom,kpss-acc-v1"; reg = <0x02098000 0x1000>, <0x02008000 0x1000>; }; - adm_dma: dma-controller@18300000 { - compatible = "qcom,adm"; - reg = <0x18300000 0x100000>; - interrupts = ; - #dma-cells = <1>; + saw1: regulator@2099000 { + compatible = "qcom,saw2"; + reg = <0x02099000 0x1000>, <0x02009000 0x1000>; + regulator; + }; - clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>; - clock-names = "core", "iface"; + nss_common: syscon@03000000 { + compatible = "syscon"; + reg = <0x03000000 0x0000FFFF>; + }; - resets = <&gcc ADM0_RESET>, - <&gcc ADM0_PBUS_RESET>, - <&gcc ADM0_C0_RESET>, - <&gcc ADM0_C1_RESET>, - <&gcc ADM0_C2_RESET>; - reset-names = "clk", "pbus", "c0", "c1", "c2"; - qcom,ee = <0>; + usb3_0: usb3@100f8800 { + compatible = "qcom,ipq8064-dwc3", "qcom,dwc3"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x100f8800 0x8000>; + clocks = <&gcc USB30_0_MASTER_CLK>; + clock-names = "core"; + + ranges; + + resets = <&gcc USB30_0_MASTER_RESET>; + reset-names = "master"; status = "disabled"; + + dwc3_0: dwc3@10000000 { + compatible = "snps,dwc3"; + reg = <0x10000000 0xcd00>; + interrupts = ; + phys = <&hs_phy_0>, <&ss_phy_0>; + phy-names = "usb2-phy", "usb3-phy"; + dr_mode = "host"; + snps,dis_u3_susphy_quirk; + }; }; - saw0: regulator@2089000 { - compatible = "qcom,saw2"; - reg = <0x02089000 0x1000>, <0x02009000 0x1000>; - regulator; + hs_phy_0: phy@100f8800 { + compatible = "qcom,ipq806x-usb-phy-hs"; + reg = <0x100f8800 0x30>; + clocks = <&gcc USB30_0_UTMI_CLK>; + clock-names = "ref"; + #phy-cells = <0>; + + status = "disabled"; }; - saw1: regulator@2099000 { - compatible = "qcom,saw2"; - reg = <0x02099000 0x1000>, <0x02009000 0x1000>; - regulator; + ss_phy_0: phy@100f8830 { + compatible = "qcom,ipq806x-usb-phy-ss"; + reg = <0x100f8830 0x30>; + clocks = <&gcc USB30_0_MASTER_CLK>; + clock-names = "ref"; + #phy-cells = <0>; + + status = "disabled"; + }; + + usb3_1: usb3@110f8800 { + compatible = "qcom,ipq8064-dwc3", "qcom,dwc3"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x110f8800 0x8000>; + clocks = <&gcc USB30_1_MASTER_CLK>; + clock-names = "core"; + + ranges; + + resets = <&gcc USB30_1_MASTER_RESET>; + reset-names = "master"; + + status = "disabled"; + + dwc3_1: dwc3@11000000 { + compatible = "snps,dwc3"; + reg = <0x11000000 0xcd00>; + interrupts = ; + phys = <&hs_phy_1>, <&ss_phy_1>; + phy-names = "usb2-phy", "usb3-phy"; + dr_mode = "host"; + snps,dis_u3_susphy_quirk; + }; + }; + + hs_phy_1: phy@110f8800 { + compatible = "qcom,ipq806x-usb-phy-hs"; + reg = <0x110f8800 0x30>; + clocks = <&gcc USB30_1_UTMI_CLK>; + clock-names = "ref"; + #phy-cells = <0>; + + status = "disabled"; + }; + + ss_phy_1: phy@110f8830 { + compatible = "qcom,ipq806x-usb-phy-ss"; + reg = <0x110f8830 0x30>; + clocks = <&gcc USB30_1_MASTER_CLK>; + clock-names = "ref"; + #phy-cells = <0>; + + status = "disabled"; + }; + + sdcc3bam: dma-controller@12182000 { + compatible = "qcom,bam-v1.3.0"; + reg = <0x12182000 0x8000>; + interrupts = ; + clocks = <&gcc SDC3_H_CLK>; + clock-names = "bam_clk"; + #dma-cells = <1>; + qcom,ee = <0>; + }; + + sdcc1bam: dma-controller@12402000 { + compatible = "qcom,bam-v1.3.0"; + reg = <0x12402000 0x8000>; + interrupts = ; + clocks = <&gcc SDC1_H_CLK>; + clock-names = "bam_clk"; + #dma-cells = <1>; + qcom,ee = <0>; + }; + + amba: amba { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + sdcc3: mmc@12180000 { + compatible = "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x00051180>; + status = "disabled"; + reg = <0x12180000 0x2000>; + interrupts = ; + interrupt-names = "cmd_irq"; + clocks = <&gcc SDC3_CLK>, <&gcc SDC3_H_CLK>; + clock-names = "mclk", "apb_pclk"; + bus-width = <8>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <192000000>; + sd-uhs-sdr104; + sd-uhs-ddr50; + vqmmc-supply = <&vsdcc_fixed>; + dmas = <&sdcc3bam 2>, <&sdcc3bam 1>; + dma-names = "tx", "rx"; + }; + + sdcc1: mmc@12400000 { + status = "disabled"; + compatible = "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x00051180>; + reg = <0x12400000 0x2000>; + interrupts = ; + interrupt-names = "cmd_irq"; + clocks = <&gcc SDC1_CLK>, <&gcc SDC1_H_CLK>; + clock-names = "mclk", "apb_pclk"; + bus-width = <8>; + max-frequency = <96000000>; + non-removable; + cap-sd-highspeed; + cap-mmc-highspeed; + mmc-ddr-1_8v; + vmmc-supply = <&vsdcc_fixed>; + dmas = <&sdcc1bam 2>, <&sdcc1bam 1>; + dma-names = "tx", "rx"; + }; }; gsbi1: gsbi@12440000 { @@ -654,56 +875,6 @@ }; }; - gsbi5: gsbi@1a200000 { - compatible = "qcom,gsbi-v1.0.0"; - cell-index = <5>; - reg = <0x1a200000 0x100>; - clocks = <&gcc GSBI5_H_CLK>; - clock-names = "iface"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - status = "disabled"; - - syscon-tcsr = <&tcsr>; - - gsbi5_serial: serial@1a240000 { - compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; - reg = <0x1a240000 0x1000>, - <0x1a200000 0x1000>; - interrupts = ; - clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>; - clock-names = "core", "iface"; - status = "disabled"; - }; - - i2c@1a280000 { - compatible = "qcom,i2c-qup-v1.1.1"; - reg = <0x1a280000 0x1000>; - interrupts = ; - - clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>; - clock-names = "core", "iface"; - status = "disabled"; - - #address-cells = <1>; - #size-cells = <0>; - }; - - spi@1a280000 { - compatible = "qcom,spi-qup-v1.1.1"; - reg = <0x1a280000 0x1000>; - interrupts = ; - - clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>; - clock-names = "core", "iface"; - status = "disabled"; - - #address-cells = <1>; - #size-cells = <0>; - }; - }; - gsbi6: gsbi@16500000 { compatible = "qcom,gsbi-v1.0.0"; reg = <0x16500000 0x100>; @@ -784,151 +955,120 @@ }; }; - rng@1a500000 { - compatible = "qcom,prng"; - reg = <0x1a500000 0x200>; - clocks = <&gcc PRNG_CLK>; - clock-names = "core"; - }; + adm_dma: dma-controller@18300000 { + compatible = "qcom,adm"; + reg = <0x18300000 0x100000>; + interrupts = ; + #dma-cells = <1>; - sata_phy: sata-phy@1b400000 { - compatible = "qcom,ipq806x-sata-phy"; - reg = <0x1b400000 0x200>; + clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>; + clock-names = "core", "iface"; - clocks = <&gcc SATA_PHY_CFG_CLK>; - clock-names = "cfg"; + resets = <&gcc ADM0_RESET>, + <&gcc ADM0_PBUS_RESET>, + <&gcc ADM0_C0_RESET>, + <&gcc ADM0_C1_RESET>, + <&gcc ADM0_C2_RESET>; + reset-names = "clk", "pbus", "c0", "c1", "c2"; + qcom,ee = <0>; - #phy-cells = <0>; status = "disabled"; }; - nand: nand-controller@1ac00000 { - compatible = "qcom,ipq806x-nand"; - reg = <0x1ac00000 0x800>; - - pinctrl-0 = <&nand_pins>; - pinctrl-names = "default"; - - clocks = <&gcc EBI2_CLK>, - <&gcc EBI2_AON_CLK>; - clock-names = "core", "aon"; - - dmas = <&adm_dma 3>; - dma-names = "rxtx"; - qcom,cmd-crci = <15>; - qcom,data-crci = <3>; - + gsbi5: gsbi@1a200000 { + compatible = "qcom,gsbi-v1.0.0"; + cell-index = <5>; + reg = <0x1a200000 0x100>; + clocks = <&gcc GSBI5_H_CLK>; + clock-names = "iface"; #address-cells = <1>; - #size-cells = <0>; + #size-cells = <1>; + ranges; status = "disabled"; - }; - sata: sata@29000000 { - compatible = "qcom,ipq806x-ahci", "generic-ahci"; - reg = <0x29000000 0x180>; + syscon-tcsr = <&tcsr>; - interrupts = ; + gsbi5_serial: serial@1a240000 { + compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; + reg = <0x1a240000 0x1000>, + <0x1a200000 0x1000>; + interrupts = ; + clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>; + clock-names = "core", "iface"; + status = "disabled"; + }; - clocks = <&gcc SFAB_SATA_S_H_CLK>, - <&gcc SATA_H_CLK>, - <&gcc SATA_A_CLK>, - <&gcc SATA_RXOOB_CLK>, - <&gcc SATA_PMALIVE_CLK>; - clock-names = "slave_face", "iface", "core", - "rxoob", "pmalive"; + i2c@1a280000 { + compatible = "qcom,i2c-qup-v1.1.1"; + reg = <0x1a280000 0x1000>; + interrupts = ; - assigned-clocks = <&gcc SATA_RXOOB_CLK>, <&gcc SATA_PMALIVE_CLK>; - assigned-clock-rates = <100000000>, <100000000>; + clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>; + clock-names = "core", "iface"; + status = "disabled"; - phys = <&sata_phy>; - phy-names = "sata-phy"; - status = "disabled"; - }; + #address-cells = <1>; + #size-cells = <0>; + }; - qcom,ssbi@500000 { - compatible = "qcom,ssbi"; - reg = <0x00500000 0x1000>; - qcom,controller-type = "pmic-arbiter"; - }; + spi@1a280000 { + compatible = "qcom,spi-qup-v1.1.1"; + reg = <0x1a280000 0x1000>; + interrupts = ; - qfprom: qfprom@700000 { - compatible = "qcom,ipq8064-qfprom", "qcom,qfprom"; - reg = <0x00700000 0x1000>; - #address-cells = <1>; - #size-cells = <1>; - speedbin_efuse: speedbin@c0 { - reg = <0xc0 0x4>; - }; - tsens_calib: calib@400 { - reg = <0x400 0xb>; - }; - tsens_calib_backup: calib_backup@410 { - reg = <0x410 0xb>; + clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>; + clock-names = "core", "iface"; + status = "disabled"; + + #address-cells = <1>; + #size-cells = <0>; }; }; - gcc: clock-controller@900000 { - compatible = "qcom,gcc-ipq8064", "syscon"; - clocks = <&pxo_board>, <&cxo_board>; - clock-names = "pxo", "cxo"; - reg = <0x00900000 0x4000>; - #clock-cells = <1>; - #reset-cells = <1>; - #power-domain-cells = <1>; + tcsr: syscon@1a400000 { + compatible = "qcom,tcsr-ipq8064", "syscon"; + reg = <0x1a400000 0x100>; + }; - tsens: thermal-sensor@900000 { - compatible = "qcom,ipq8064-tsens"; + rng@1a500000 { + compatible = "qcom,prng"; + reg = <0x1a500000 0x200>; + clocks = <&gcc PRNG_CLK>; + clock-names = "core"; + }; - nvmem-cells = <&tsens_calib>, <&tsens_calib_backup>; - nvmem-cell-names = "calib", "calib_backup"; - interrupts = ; - interrupt-names = "uplow"; + nand: nand-controller@1ac00000 { + compatible = "qcom,ipq806x-nand"; + reg = <0x1ac00000 0x800>; - #qcom,sensors = <11>; - #thermal-sensor-cells = <1>; - }; - }; + pinctrl-0 = <&nand_pins>; + pinctrl-names = "default"; - rpm: rpm@108000 { - compatible = "qcom,rpm-ipq8064"; - reg = <0x108000 0x1000>; - qcom,ipc = <&l2cc 0x8 2>; + clocks = <&gcc EBI2_CLK>, + <&gcc EBI2_AON_CLK>; + clock-names = "core", "aon"; - interrupts = , - , - ; - interrupt-names = "ack", "err", "wakeup"; + dmas = <&adm_dma 3>; + dma-names = "rxtx"; + qcom,cmd-crci = <15>; + qcom,data-crci = <3>; - clocks = <&gcc RPM_MSG_RAM_H_CLK>; - clock-names = "ram"; + #address-cells = <1>; + #size-cells = <0>; - rpmcc: clock-controller { - compatible = "qcom,rpmcc-ipq806x", "qcom,rpmcc"; - #clock-cells = <1>; - clocks = <&pxo_board>; - clock-names = "pxo"; - }; + status = "disabled"; }; - tcsr: syscon@1a400000 { - compatible = "qcom,tcsr-ipq8064", "syscon"; - reg = <0x1a400000 0x100>; - }; + sata_phy: sata-phy@1b400000 { + compatible = "qcom,ipq806x-sata-phy"; + reg = <0x1b400000 0x200>; - l2cc: clock-controller@2011000 { - compatible = "qcom,kpss-gcc", "syscon"; - reg = <0x2011000 0x1000>; - clocks = <&gcc PLL8_VOTE>, <&pxo_board>; - clock-names = "pll8_vote", "pxo"; - clock-output-names = "acpu_l2_aux"; - }; + clocks = <&gcc SATA_PHY_CFG_CLK>; + clock-names = "cfg"; - lcc: clock-controller@28000000 { - compatible = "qcom,lcc-ipq8064"; - reg = <0x28000000 0x1000>; - #clock-cells = <1>; - #reset-cells = <1>; + #phy-cells = <0>; + status = "disabled"; }; pcie0: pci@1b500000 { @@ -979,7 +1119,7 @@ pinctrl-names = "default"; status = "disabled"; - perst-gpio = <&qcom_pinmux 3 GPIO_ACTIVE_LOW>; + perst-gpios = <&qcom_pinmux 3 GPIO_ACTIVE_LOW>; }; pcie1: pci@1b700000 { @@ -1030,7 +1170,7 @@ pinctrl-names = "default"; status = "disabled"; - perst-gpio = <&qcom_pinmux 48 GPIO_ACTIVE_LOW>; + perst-gpios = <&qcom_pinmux 48 GPIO_ACTIVE_LOW>; }; pcie2: pci@1b900000 { @@ -1081,12 +1221,7 @@ pinctrl-names = "default"; status = "disabled"; - perst-gpio = <&qcom_pinmux 63 GPIO_ACTIVE_LOW>; - }; - - nss_common: syscon@03000000 { - compatible = "syscon"; - reg = <0x03000000 0x0000FFFF>; + perst-gpios = <&qcom_pinmux 63 GPIO_ACTIVE_LOW>; }; qsgmii_csr: syscon@1bb00000 { @@ -1094,10 +1229,48 @@ reg = <0x1bb00000 0x000001FF>; }; - stmmac_axi_setup: stmmac-axi-config { - snps,wr_osr_lmt = <7>; - snps,rd_osr_lmt = <7>; - snps,blen = <16 0 0 0 0 0 0>; + lcc: clock-controller@28000000 { + compatible = "qcom,lcc-ipq8064"; + reg = <0x28000000 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + lpass@28100000 { + compatible = "qcom,lpass-cpu"; + status = "disabled"; + clocks = <&lcc AHBIX_CLK>, + <&lcc MI2S_OSR_CLK>, + <&lcc MI2S_BIT_CLK>; + clock-names = "ahbix-clk", + "mi2s-osr-clk", + "mi2s-bit-clk"; + interrupts = ; + interrupt-names = "lpass-irq-lpaif"; + reg = <0x28100000 0x10000>; + reg-names = "lpass-lpaif"; + }; + + sata: sata@29000000 { + compatible = "qcom,ipq806x-ahci", "generic-ahci"; + reg = <0x29000000 0x180>; + + interrupts = ; + + clocks = <&gcc SFAB_SATA_S_H_CLK>, + <&gcc SATA_H_CLK>, + <&gcc SATA_A_CLK>, + <&gcc SATA_RXOOB_CLK>, + <&gcc SATA_PMALIVE_CLK>; + clock-names = "slave_face", "iface", "core", + "rxoob", "pmalive"; + + assigned-clocks = <&gcc SATA_RXOOB_CLK>, <&gcc SATA_PMALIVE_CLK>; + assigned-clock-rates = <100000000>, <100000000>; + + phys = <&sata_phy>; + phy-names = "sata-phy"; + status = "disabled"; }; gmac0: ethernet@37000000 { @@ -1195,179 +1368,5 @@ status = "disabled"; }; - - hs_phy_0: phy@100f8800 { - compatible = "qcom,ipq806x-usb-phy-hs"; - reg = <0x100f8800 0x30>; - clocks = <&gcc USB30_0_UTMI_CLK>; - clock-names = "ref"; - #phy-cells = <0>; - - status = "disabled"; - }; - - ss_phy_0: phy@100f8830 { - compatible = "qcom,ipq806x-usb-phy-ss"; - reg = <0x100f8830 0x30>; - clocks = <&gcc USB30_0_MASTER_CLK>; - clock-names = "ref"; - #phy-cells = <0>; - - status = "disabled"; - }; - - usb3_0: usb3@100f8800 { - compatible = "qcom,ipq8064-dwc3", "qcom,dwc3"; - #address-cells = <1>; - #size-cells = <1>; - reg = <0x100f8800 0x8000>; - clocks = <&gcc USB30_0_MASTER_CLK>; - clock-names = "core"; - - ranges; - - resets = <&gcc USB30_0_MASTER_RESET>; - reset-names = "master"; - - status = "disabled"; - - dwc3_0: dwc3@10000000 { - compatible = "snps,dwc3"; - reg = <0x10000000 0xcd00>; - interrupts = ; - phys = <&hs_phy_0>, <&ss_phy_0>; - phy-names = "usb2-phy", "usb3-phy"; - dr_mode = "host"; - snps,dis_u3_susphy_quirk; - }; - }; - - hs_phy_1: phy@110f8800 { - compatible = "qcom,ipq806x-usb-phy-hs"; - reg = <0x110f8800 0x30>; - clocks = <&gcc USB30_1_UTMI_CLK>; - clock-names = "ref"; - #phy-cells = <0>; - - status = "disabled"; - }; - - ss_phy_1: phy@110f8830 { - compatible = "qcom,ipq806x-usb-phy-ss"; - reg = <0x110f8830 0x30>; - clocks = <&gcc USB30_1_MASTER_CLK>; - clock-names = "ref"; - #phy-cells = <0>; - - status = "disabled"; - }; - - usb3_1: usb3@110f8800 { - compatible = "qcom,ipq8064-dwc3", "qcom,dwc3"; - #address-cells = <1>; - #size-cells = <1>; - reg = <0x110f8800 0x8000>; - clocks = <&gcc USB30_1_MASTER_CLK>; - clock-names = "core"; - - ranges; - - resets = <&gcc USB30_1_MASTER_RESET>; - reset-names = "master"; - - status = "disabled"; - - dwc3_1: dwc3@11000000 { - compatible = "snps,dwc3"; - reg = <0x11000000 0xcd00>; - interrupts = ; - phys = <&hs_phy_1>, <&ss_phy_1>; - phy-names = "usb2-phy", "usb3-phy"; - dr_mode = "host"; - snps,dis_u3_susphy_quirk; - }; - }; - - vsdcc_fixed: vsdcc-regulator { - compatible = "regulator-fixed"; - regulator-name = "SDCC Power"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-always-on; - }; - - sdcc1bam: dma-controller@12402000 { - compatible = "qcom,bam-v1.3.0"; - reg = <0x12402000 0x8000>; - interrupts = ; - clocks = <&gcc SDC1_H_CLK>; - clock-names = "bam_clk"; - #dma-cells = <1>; - qcom,ee = <0>; - }; - - sdcc3bam: dma-controller@12182000 { - compatible = "qcom,bam-v1.3.0"; - reg = <0x12182000 0x8000>; - interrupts = ; - clocks = <&gcc SDC3_H_CLK>; - clock-names = "bam_clk"; - #dma-cells = <1>; - qcom,ee = <0>; - }; - - amba: amba { - compatible = "simple-bus"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - sdcc1: mmc@12400000 { - status = "disabled"; - compatible = "arm,pl18x", "arm,primecell"; - arm,primecell-periphid = <0x00051180>; - reg = <0x12400000 0x2000>; - interrupts = ; - interrupt-names = "cmd_irq"; - clocks = <&gcc SDC1_CLK>, <&gcc SDC1_H_CLK>; - clock-names = "mclk", "apb_pclk"; - bus-width = <8>; - max-frequency = <96000000>; - non-removable; - cap-sd-highspeed; - cap-mmc-highspeed; - mmc-ddr-1_8v; - vmmc-supply = <&vsdcc_fixed>; - dmas = <&sdcc1bam 2>, <&sdcc1bam 1>; - dma-names = "tx", "rx"; - }; - - sdcc3: mmc@12180000 { - compatible = "arm,pl18x", "arm,primecell"; - arm,primecell-periphid = <0x00051180>; - status = "disabled"; - reg = <0x12180000 0x2000>; - interrupts = ; - interrupt-names = "cmd_irq"; - clocks = <&gcc SDC3_CLK>, <&gcc SDC3_H_CLK>; - clock-names = "mclk", "apb_pclk"; - bus-width = <8>; - cap-sd-highspeed; - cap-mmc-highspeed; - max-frequency = <192000000>; - sd-uhs-sdr104; - sd-uhs-ddr50; - vqmmc-supply = <&vsdcc_fixed>; - dmas = <&sdcc3bam 2>, <&sdcc3bam 1>; - dma-names = "tx", "rx"; - }; - }; - - sfpb_mutex: hwlock@1200600 { - compatible = "qcom,sfpb-mutex"; - reg = <0x01200600 0x100>; - - #hwlock-cells = <1>; - }; }; }; diff --git a/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..803e6ff99ef8d2c9e6f81a880a60ae75756ba5e9 --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "qcom-ipq8065.dtsi" + +&rpm { + smb208_regulators: regulators { + compatible = "qcom,rpm-smb208-regulators"; + + smb208_s1a: s1a { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s1b: s1b { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1150000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2a: s2a { + regulator-min-microvolt = <775000>; + regulator-max-microvolt = <1275000>; + + qcom,switch-mode-frequency = <1200000>; + }; + + smb208_s2b: s2b { + regulator-min-microvolt = <775000>; + regulator-max-microvolt = <1275000>; + + qcom,switch-mode-frequency = <1200000>; + }; + }; +}; diff --git a/arch/arm/boot/dts/qcom-ipq8065.dtsi b/arch/arm/boot/dts/qcom-ipq8065.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..ea49f6cc416d7b96b98cb2b9ceaa27afd6e5bb99 --- /dev/null +++ b/arch/arm/boot/dts/qcom-ipq8065.dtsi @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "qcom-ipq8064-v2.0.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. IPQ8065"; + compatible = "qcom,ipq8065", "qcom,ipq8064"; +}; diff --git a/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts b/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts index d159188c8b95d64e20d46a334e72c9e56bd0b9bc..290e1df631f02d405ef534bccd86c83bc5b02917 100644 --- a/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts +++ b/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts @@ -18,8 +18,6 @@ }; }; -&soc { - serial@f991f000 { - status = "ok"; - }; +&blsp1_uart3 { + status = "ok"; }; diff --git a/arch/arm/boot/dts/qcom-msm8226.dtsi b/arch/arm/boot/dts/qcom-msm8226.dtsi index 0b5effdb269aead5c4537241f258bd180c3bbac5..cf2d56929428c9abc8268d09b3e98dce3b7c6297 100644 --- a/arch/arm/boot/dts/qcom-msm8226.dtsi +++ b/arch/arm/boot/dts/qcom-msm8226.dtsi @@ -8,6 +8,7 @@ #include #include #include +#include #include / { @@ -44,13 +45,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_block 0 0x80>; - - #hwlock-cells = <1>; - }; - reserved-memory { #address-cells = <1>; #size-cells = <1>; @@ -60,6 +54,11 @@ reg = <0x3000000 0x100000>; no-map; }; + + adsp_region: adsp@dc00000 { + reg = <0x0dc00000 0x1900000>; + no-map; + }; }; smd { @@ -115,6 +114,31 @@ hwlocks = <&tcsr_mutex 3>; }; + smp2p-adsp { + compatible = "qcom,smp2p"; + qcom,smem = <443>, <429>; + + interrupt-parent = <&intc>; + interrupts = ; + + qcom,ipc = <&apcs 8 10>; + + qcom,local-pid = <0>; + qcom,remote-pid = <2>; + + adsp_smp2p_out: master-kernel { + qcom,entry-name = "master-kernel"; + #qcom,smem-state-cells = <1>; + }; + + adsp_smp2p_in: slave-kernel { + qcom,entry-name = "slave-kernel"; + + interrupt-controller; + #interrupt-cells = <2>; + }; + }; + soc: soc { compatible = "simple-bus"; #address-cells = <1>; @@ -137,14 +161,14 @@ sdhc_1: mmc@f9824900 { compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9824900 0x11c>, <0xf9824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC1_APPS_CLK>, - <&gcc GCC_SDCC1_AHB_CLK>, + clocks = <&gcc GCC_SDCC1_AHB_CLK>, + <&gcc GCC_SDCC1_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; pinctrl-names = "default"; pinctrl-0 = <&sdhc1_default_state>; status = "disabled"; @@ -153,14 +177,14 @@ sdhc_2: mmc@f98a4900 { compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC2_APPS_CLK>, - <&gcc GCC_SDCC2_AHB_CLK>, + clocks = <&gcc GCC_SDCC2_AHB_CLK>, + <&gcc GCC_SDCC2_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; pinctrl-names = "default"; pinctrl-0 = <&sdhc2_default_state>; status = "disabled"; @@ -169,14 +193,14 @@ sdhc_3: mmc@f9864900 { compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9864900 0x11c>, <0xf9864000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC3_APPS_CLK>, - <&gcc GCC_SDCC3_AHB_CLK>, + clocks = <&gcc GCC_SDCC3_AHB_CLK>, + <&gcc GCC_SDCC3_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; pinctrl-names = "default"; pinctrl-0 = <&sdhc3_default_state>; status = "disabled"; @@ -508,9 +532,44 @@ reg = <0xfc428000 0x4000>; }; - tcsr_mutex_block: syscon@fd484000 { - compatible = "syscon"; - reg = <0xfd484000 0x2000>; + tcsr_mutex: hwlock@fd484000 { + compatible = "qcom,msm8226-tcsr-mutex", "qcom,tcsr-mutex"; + reg = <0xfd484000 0x1000>; + #hwlock-cells = <1>; + }; + + adsp: remoteproc@fe200000 { + compatible = "qcom,msm8226-adsp-pil"; + reg = <0xfe200000 0x100>; + + interrupts-extended = <&intc GIC_SPI 162 IRQ_TYPE_EDGE_RISING>, + <&adsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>, + <&adsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>, + <&adsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>, + <&adsp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>; + interrupt-names = "wdog", "fatal", "ready", "handover", "stop-ack"; + + power-domains = <&rpmpd MSM8226_VDDCX>; + power-domain-names = "cx"; + + clocks = <&xo_board>; + clock-names = "xo"; + + memory-region = <&adsp_region>; + + qcom,smem-states = <&adsp_smp2p_out 0>; + qcom,smem-state-names = "stop"; + + status = "disabled"; + + smd-edge { + interrupts = ; + + qcom,ipc = <&apcs 8 8>; + qcom,smd-edge = <1>; + + label = "lpass"; + }; }; }; diff --git a/arch/arm/boot/dts/qcom-msm8660-surf.dts b/arch/arm/boot/dts/qcom-msm8660-surf.dts index 414280d9bdbafab0564fad975809fd5c9857d119..be18f1be29a17c2b4f90565afd12bbd756011acd 100644 --- a/arch/arm/boot/dts/qcom-msm8660-surf.dts +++ b/arch/arm/boot/dts/qcom-msm8660-surf.dts @@ -15,38 +15,23 @@ stdout-path = "serial0:115200n8"; }; - soc { - gsbi@19c00000 { - status = "okay"; - qcom,mode = ; - serial@19c40000 { - status = "okay"; - }; - }; - - /* Temporary fixed regulator */ - vsdcc_fixed: vsdcc-regulator { - compatible = "regulator-fixed"; - regulator-name = "SDCC Power"; - regulator-min-microvolt = <2700000>; - regulator-max-microvolt = <2700000>; - regulator-always-on; - }; + /* Temporary fixed regulator */ + vsdcc_fixed: vsdcc-regulator { + compatible = "regulator-fixed"; + regulator-name = "SDCC Power"; + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <2700000>; + regulator-always-on; + }; +}; - amba { - /* eMMC */ - sdcc1: mmc@12400000 { - status = "okay"; - vmmc-supply = <&vsdcc_fixed>; - }; +&gsbi12 { + qcom,mode = ; + status = "okay"; +}; - /* External micro SD card */ - sdcc3: mmc@12180000 { - status = "okay"; - vmmc-supply = <&vsdcc_fixed>; - }; - }; - }; +&gsbi12_serial { + status = "okay"; }; &pm8058 { @@ -76,3 +61,15 @@ keypad,num-columns = <5>; }; }; + +/* eMMC */ +&sdcc1 { + vmmc-supply = <&vsdcc_fixed>; + status = "okay"; +}; + +/* External micro SD card */ +&sdcc3 { + vmmc-supply = <&vsdcc_fixed>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/qcom-msm8660.dtsi b/arch/arm/boot/dts/qcom-msm8660.dtsi index 63a501c63cf8e1a1e6ebe2b12e5b7439032aec9d..ddce7d64ba9911c5ecdd2c71b8945648ef192a21 100644 --- a/arch/arm/boot/dts/qcom-msm8660.dtsi +++ b/arch/arm/boot/dts/qcom-msm8660.dtsi @@ -50,22 +50,25 @@ }; clocks { - cxo_board { + cxo_board: cxo-board-clk { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <19200000>; + clock-output-names = "cxo_board"; }; - pxo_board: pxo_board { + pxo_board: pxo-board-clk { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <27000000>; + clock-output-names = "pxo_board"; }; - sleep_clk { + sleep-clk { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <32768>; + clock-output-names = "sleep_clk"; }; }; @@ -129,6 +132,59 @@ #power-domain-cells = <1>; #reset-cells = <1>; reg = <0x900000 0x4000>; + clocks = <&pxo_board>, <&cxo_board>; + clock-names = "pxo", "cxo"; + }; + + gsbi1: gsbi@16000000 { + compatible = "qcom,gsbi-v1.0.0"; + cell-index = <12>; + reg = <0x16000000 0x100>; + clocks = <&gcc GSBI1_H_CLK>; + clock-names = "iface"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + syscon-tcsr = <&tcsr>; + + status = "disabled"; + + gsbi1_spi: spi@16080000 { + compatible = "qcom,spi-qup-v1.1.1"; + reg = <0x16080000 0x1000>; + interrupts = ; + clocks = <&gcc GSBI1_QUP_CLK>, <&gcc GSBI1_H_CLK>; + clock-names = "core", "iface"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + + gsbi3: gsbi@16200000 { + compatible = "qcom,gsbi-v1.0.0"; + cell-index = <12>; + reg = <0x16200000 0x100>; + clocks = <&gcc GSBI3_H_CLK>; + clock-names = "iface"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + syscon-tcsr = <&tcsr>; + status = "disabled"; + + gsbi3_i2c: i2c@16280000 { + compatible = "qcom,i2c-qup-v1.1.1"; + reg = <0x16280000 0x1000>; + interrupts = ; + clocks = <&gcc GSBI3_QUP_CLK>, <&gcc GSBI3_H_CLK>; + clock-names = "core", "iface"; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; }; gsbi6: gsbi@16500000 { diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts new file mode 100644 index 0000000000000000000000000000000000000000..c8d34de8a71e4b4c8c7c32a1813159dee5ec1891 --- /dev/null +++ b/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "arm64/qcom/msm8916-samsung-e5.dts" +#include "qcom-msm8916-smp.dtsi" diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts new file mode 100644 index 0000000000000000000000000000000000000000..85be286c86083aa2ab10b11af0f98968c98c57e7 --- /dev/null +++ b/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "arm64/qcom/msm8916-samsung-e7.dts" +#include "qcom-msm8916-smp.dtsi" diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts new file mode 100644 index 0000000000000000000000000000000000000000..d3abe0536238c13e7f6d60e6ad30781ccf00e1da --- /dev/null +++ b/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "arm64/qcom/msm8916-samsung-grandmax.dts" +#include "qcom-msm8916-smp.dtsi" diff --git a/arch/arm/boot/dts/qcom-msm8960-cdp.dts b/arch/arm/boot/dts/qcom-msm8960-cdp.dts index d1fd0fe12ffe40e910da0e5c30706589357ac6bc..9157e3c4f48f901865b8eb58548d2028194bf92c 100644 --- a/arch/arm/boot/dts/qcom-msm8960-cdp.dts +++ b/arch/arm/boot/dts/qcom-msm8960-cdp.dts @@ -15,318 +15,6 @@ stdout-path = "serial0:115200n8"; }; - soc { - gsbi@16400000 { - status = "okay"; - qcom,mode = ; - serial@16440000 { - status = "okay"; - }; - }; - - amba { - /* eMMC */ - sdcc1: mmc@12400000 { - status = "okay"; - }; - - /* External micro SD card */ - sdcc3: mmc@12180000 { - status = "okay"; - }; - }; - - rpm@108000 { - regulators { - compatible = "qcom,rpm-pm8921-regulators"; - vin_lvs1_3_6-supply = <&pm8921_s4>; - vin_lvs2-supply = <&pm8921_s4>; - vin_lvs4_5_7-supply = <&pm8921_s4>; - vdd_ncp-supply = <&pm8921_l6>; - vdd_l1_l2_l12_l18-supply = <&pm8921_s4>; - vdd_l21_l23_l29-supply = <&pm8921_s8>; - vdd_l24-supply = <&pm8921_s1>; - vdd_l25-supply = <&pm8921_s1>; - vdd_l27-supply = <&pm8921_s7>; - vdd_l28-supply = <&pm8921_s7>; - - /* Buck SMPS */ - pm8921_s1: s1 { - regulator-always-on; - regulator-min-microvolt = <1225000>; - regulator-max-microvolt = <1225000>; - qcom,switch-mode-frequency = <3200000>; - bias-pull-down; - }; - - pm8921_s2: s2 { - regulator-min-microvolt = <1300000>; - regulator-max-microvolt = <1300000>; - qcom,switch-mode-frequency = <1600000>; - bias-pull-down; - }; - - pm8921_s3: s3 { - regulator-min-microvolt = <500000>; - regulator-max-microvolt = <1150000>; - qcom,switch-mode-frequency = <4800000>; - bias-pull-down; - }; - - pm8921_s4: s4 { - regulator-always-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - qcom,switch-mode-frequency = <1600000>; - bias-pull-down; - qcom,force-mode = ; - }; - - pm8921_s7: s7 { - regulator-min-microvolt = <1150000>; - regulator-max-microvolt = <1150000>; - qcom,switch-mode-frequency = <3200000>; - bias-pull-down; - }; - - pm8921_s8: s8 { - regulator-always-on; - regulator-min-microvolt = <2050000>; - regulator-max-microvolt = <2050000>; - qcom,switch-mode-frequency = <1600000>; - bias-pull-down; - }; - - /* PMOS LDO */ - pm8921_l1: l1 { - regulator-always-on; - regulator-min-microvolt = <1050000>; - regulator-max-microvolt = <1050000>; - bias-pull-down; - }; - - pm8921_l2: l2 { - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - bias-pull-down; - }; - - pm8921_l3: l3 { - regulator-min-microvolt = <3075000>; - regulator-max-microvolt = <3075000>; - bias-pull-down; - }; - - pm8921_l4: l4 { - regulator-always-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - bias-pull-down; - }; - - pm8921_l5: l5 { - regulator-min-microvolt = <2950000>; - regulator-max-microvolt = <2950000>; - bias-pull-down; - }; - - pm8921_l6: l6 { - regulator-min-microvolt = <2950000>; - regulator-max-microvolt = <2950000>; - bias-pull-down; - }; - - pm8921_l7: l7 { - regulator-always-on; - regulator-min-microvolt = <1850000>; - regulator-max-microvolt = <2950000>; - bias-pull-down; - }; - - pm8921_l8: l8 { - regulator-min-microvolt = <2800000>; - regulator-max-microvolt = <3000000>; - bias-pull-down; - }; - - pm8921_l9: l9 { - regulator-min-microvolt = <3000000>; - regulator-max-microvolt = <3000000>; - bias-pull-down; - }; - - pm8921_l10: l10 { - regulator-min-microvolt = <3000000>; - regulator-max-microvolt = <3000000>; - bias-pull-down; - }; - - pm8921_l11: l11 { - regulator-min-microvolt = <2850000>; - regulator-max-microvolt = <2850000>; - bias-pull-down; - }; - - pm8921_l12: l12 { - regulator-min-microvolt = <1200000>; - regulator-max-microvolt = <1200000>; - bias-pull-down; - }; - - pm8921_l14: l14 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - bias-pull-down; - }; - - pm8921_l15: l15 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <2950000>; - bias-pull-down; - }; - - pm8921_l16: l16 { - regulator-min-microvolt = <2800000>; - regulator-max-microvolt = <2800000>; - bias-pull-down; - }; - - pm8921_l17: l17 { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <2950000>; - bias-pull-down; - }; - - pm8921_l18: l18 { - regulator-min-microvolt = <1300000>; - regulator-max-microvolt = <1300000>; - bias-pull-down; - }; - - pm8921_l21: l21 { - regulator-min-microvolt = <1900000>; - regulator-max-microvolt = <1900000>; - bias-pull-down; - }; - - pm8921_l22: l22 { - regulator-min-microvolt = <2750000>; - regulator-max-microvolt = <2750000>; - bias-pull-down; - }; - - pm8921_l23: l23 { - regulator-always-on; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - bias-pull-down; - }; - - pm8921_l24: l24 { - regulator-min-microvolt = <750000>; - regulator-max-microvolt = <1150000>; - bias-pull-down; - }; - - pm8921_l25: l25 { - regulator-always-on; - regulator-min-microvolt = <1250000>; - regulator-max-microvolt = <1250000>; - bias-pull-down; - }; - - /* Low Voltage Switch */ - pm8921_lvs1: lvs1 { - bias-pull-down; - }; - - pm8921_lvs2: lvs2 { - bias-pull-down; - }; - - pm8921_lvs3: lvs3 { - bias-pull-down; - }; - - pm8921_lvs4: lvs4 { - bias-pull-down; - }; - - pm8921_lvs5: lvs5 { - bias-pull-down; - }; - - pm8921_lvs6: lvs6 { - bias-pull-down; - }; - - pm8921_lvs7: lvs7 { - bias-pull-down; - }; - - pm8921_ncp: ncp { - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - qcom,switch-mode-frequency = <1600000>; - }; - }; - }; - - gsbi@16000000 { - status = "okay"; - qcom,mode = ; - pinctrl-names = "default"; - pinctrl-0 = <&spi1_default>; - spi@16080000 { - status = "okay"; - ethernet@0 { - compatible = "micrel,ks8851"; - reg = <0>; - interrupt-parent = <&msmgpio>; - interrupts = <90 8>; - spi-max-frequency = <5400000>; - vdd-supply = <&ext_l2>; - vdd-io-supply = <&pm8921_lvs6>; - reset-gpios = <&msmgpio 89 0>; - }; - }; - }; - - pinctrl@800000 { - spi1_default: spi1_default { - mux { - pins = "gpio6", "gpio7", "gpio9"; - function = "gsbi1"; - }; - - mosi { - pins = "gpio6"; - drive-strength = <12>; - bias-disable; - }; - - miso { - pins = "gpio7"; - drive-strength = <12>; - bias-disable; - }; - - cs { - pins = "gpio8"; - drive-strength = <12>; - bias-disable; - output-low; - }; - - clk { - pins = "gpio9"; - drive-strength = <12>; - bias-disable; - }; - }; - }; - }; - regulators { compatible = "simple-bus"; @@ -340,6 +28,71 @@ }; }; +&gsbi1 { + qcom,mode = ; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_default>; + status = "okay"; +}; + +&gsbi1_spi { + status = "okay"; + + ethernet@0 { + compatible = "micrel,ks8851"; + reg = <0>; + interrupt-parent = <&msmgpio>; + interrupts = <90 8>; + spi-max-frequency = <5400000>; + vdd-supply = <&ext_l2>; + vdd-io-supply = <&pm8921_lvs6>; + reset-gpios = <&msmgpio 89 0>; + }; +}; + +&gsbi5 { + qcom,mode = ; + status = "okay"; +}; + +&gsbi5_serial { + status = "okay"; +}; + +&msmgpio { + spi1_default: spi1_default { + mux { + pins = "gpio6", "gpio7", "gpio9"; + function = "gsbi1"; + }; + + mosi { + pins = "gpio6"; + drive-strength = <12>; + bias-disable; + }; + + miso { + pins = "gpio7"; + drive-strength = <12>; + bias-disable; + }; + + cs { + pins = "gpio8"; + drive-strength = <12>; + bias-disable; + output-low; + }; + + clk { + pins = "gpio9"; + drive-strength = <12>; + bias-disable; + }; + }; +}; + &pmicintc { keypad@148 { linux,keymap = < @@ -352,3 +105,249 @@ keypad,num-columns = <5>; }; }; + +&rpm { + regulators { + compatible = "qcom,rpm-pm8921-regulators"; + vin_lvs1_3_6-supply = <&pm8921_s4>; + vin_lvs2-supply = <&pm8921_s4>; + vin_lvs4_5_7-supply = <&pm8921_s4>; + vdd_ncp-supply = <&pm8921_l6>; + vdd_l1_l2_l12_l18-supply = <&pm8921_s4>; + vdd_l21_l23_l29-supply = <&pm8921_s8>; + vdd_l24-supply = <&pm8921_s1>; + vdd_l25-supply = <&pm8921_s1>; + vdd_l27-supply = <&pm8921_s7>; + vdd_l28-supply = <&pm8921_s7>; + + /* Buck SMPS */ + pm8921_s1: s1 { + regulator-always-on; + regulator-min-microvolt = <1225000>; + regulator-max-microvolt = <1225000>; + qcom,switch-mode-frequency = <3200000>; + bias-pull-down; + }; + + pm8921_s2: s2 { + regulator-min-microvolt = <1300000>; + regulator-max-microvolt = <1300000>; + qcom,switch-mode-frequency = <1600000>; + bias-pull-down; + }; + + pm8921_s3: s3 { + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1150000>; + qcom,switch-mode-frequency = <4800000>; + bias-pull-down; + }; + + pm8921_s4: s4 { + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + qcom,switch-mode-frequency = <1600000>; + bias-pull-down; + qcom,force-mode = ; + }; + + pm8921_s7: s7 { + regulator-min-microvolt = <1150000>; + regulator-max-microvolt = <1150000>; + qcom,switch-mode-frequency = <3200000>; + bias-pull-down; + }; + + pm8921_s8: s8 { + regulator-always-on; + regulator-min-microvolt = <2050000>; + regulator-max-microvolt = <2050000>; + qcom,switch-mode-frequency = <1600000>; + bias-pull-down; + }; + + /* PMOS LDO */ + pm8921_l1: l1 { + regulator-always-on; + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + bias-pull-down; + }; + + pm8921_l2: l2 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + bias-pull-down; + }; + + pm8921_l3: l3 { + regulator-min-microvolt = <3075000>; + regulator-max-microvolt = <3075000>; + bias-pull-down; + }; + + pm8921_l4: l4 { + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + bias-pull-down; + }; + + pm8921_l5: l5 { + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + bias-pull-down; + }; + + pm8921_l6: l6 { + regulator-min-microvolt = <2950000>; + regulator-max-microvolt = <2950000>; + bias-pull-down; + }; + + pm8921_l7: l7 { + regulator-always-on; + regulator-min-microvolt = <1850000>; + regulator-max-microvolt = <2950000>; + bias-pull-down; + }; + + pm8921_l8: l8 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <3000000>; + bias-pull-down; + }; + + pm8921_l9: l9 { + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + bias-pull-down; + }; + + pm8921_l10: l10 { + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + bias-pull-down; + }; + + pm8921_l11: l11 { + regulator-min-microvolt = <2850000>; + regulator-max-microvolt = <2850000>; + bias-pull-down; + }; + + pm8921_l12: l12 { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + bias-pull-down; + }; + + pm8921_l14: l14 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + bias-pull-down; + }; + + pm8921_l15: l15 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2950000>; + bias-pull-down; + }; + + pm8921_l16: l16 { + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + bias-pull-down; + }; + + pm8921_l17: l17 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2950000>; + bias-pull-down; + }; + + pm8921_l18: l18 { + regulator-min-microvolt = <1300000>; + regulator-max-microvolt = <1300000>; + bias-pull-down; + }; + + pm8921_l21: l21 { + regulator-min-microvolt = <1900000>; + regulator-max-microvolt = <1900000>; + bias-pull-down; + }; + + pm8921_l22: l22 { + regulator-min-microvolt = <2750000>; + regulator-max-microvolt = <2750000>; + bias-pull-down; + }; + + pm8921_l23: l23 { + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + bias-pull-down; + }; + + pm8921_l24: l24 { + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1150000>; + bias-pull-down; + }; + + pm8921_l25: l25 { + regulator-always-on; + regulator-min-microvolt = <1250000>; + regulator-max-microvolt = <1250000>; + bias-pull-down; + }; + + /* Low Voltage Switch */ + pm8921_lvs1: lvs1 { + bias-pull-down; + }; + + pm8921_lvs2: lvs2 { + bias-pull-down; + }; + + pm8921_lvs3: lvs3 { + bias-pull-down; + }; + + pm8921_lvs4: lvs4 { + bias-pull-down; + }; + + pm8921_lvs5: lvs5 { + bias-pull-down; + }; + + pm8921_lvs6: lvs6 { + bias-pull-down; + }; + + pm8921_lvs7: lvs7 { + bias-pull-down; + }; + + pm8921_ncp: ncp { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + qcom,switch-mode-frequency = <1600000>; + }; + }; +}; + +/* eMMC */ +&sdcc1 { + status = "okay"; +}; + +/* External micro SD card */ +&sdcc3 { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/qcom-msm8960.dtsi b/arch/arm/boot/dts/qcom-msm8960.dtsi index 19554f3b51966bab79ab58038c57d8af72443483..c5740da3754c76345aa11807eef026ff0c382f83 100644 --- a/arch/arm/boot/dts/qcom-msm8960.dtsi +++ b/arch/arm/boot/dts/qcom-msm8960.dtsi @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -56,14 +57,14 @@ }; clocks { - cxo_board { + cxo_board: cxo_board { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <19200000>; clock-output-names = "cxo_board"; }; - pxo_board { + pxo_board: pxo_board { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <27000000>; @@ -130,6 +131,10 @@ #power-domain-cells = <1>; #reset-cells = <1>; reg = <0x900000 0x4000>; + clocks = <&cxo_board>, + <&pxo_board>, + <&lcc PLL4>; + clock-names = "cxo", "pxo", "pll4"; }; lcc: clock-controller@28000000 { @@ -137,6 +142,20 @@ reg = <0x28000000 0x1000>; #clock-cells = <1>; #reset-cells = <1>; + clocks = <&pxo_board>, + <&gcc PLL4_VOTE>, + <0>, + <0>, <0>, + <0>, <0>, + <0>; + clock-names = "pxo", + "pll4_vote", + "mi2s_codec_clk", + "codec_i2s_mic_codec_clk", + "spare_i2s_mic_codec_clk", + "codec_i2s_spkr_codec_clk", + "spare_i2s_spkr_codec_clk", + "pcm_codec_clk"; }; clock-controller@4000000 { @@ -145,6 +164,22 @@ #clock-cells = <1>; #power-domain-cells = <1>; #reset-cells = <1>; + clocks = <&pxo_board>, + <&gcc PLL3>, + <&gcc PLL8_VOTE>, + <0>, + <0>, + <0>, + <0>, + <0>; + clock-names = "pxo", + "pll3", + "pll8_vote", + "dsi1pll", + "dsi1pllbyte", + "dsi2pll", + "dsi2pllbyte", + "hdmipll"; }; l2cc: clock-controller@2011000 { @@ -152,7 +187,7 @@ reg = <0x2011000 0x1000>; }; - rpm@108000 { + rpm: rpm@108000 { compatible = "qcom,rpm-msm8960"; reg = <0x108000 0x1000>; qcom,ipc = <&l2cc 0x8 2>; @@ -307,7 +342,7 @@ reg = <0x1a400000 0x100>; }; - gsbi@16000000 { + gsbi1: gsbi@16000000 { compatible = "qcom,gsbi-v1.0.0"; cell-index = <1>; reg = <0x16000000 0x100>; @@ -317,7 +352,7 @@ #size-cells = <1>; ranges; - spi@16080000 { + gsbi1_spi: spi@16080000 { compatible = "qcom,spi-qup-v1.1.1"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts index ec5d340562b66855590886055ea5703076842e6f..6daceaa878022697f006b41e49f2b5ceef97fbc4 100644 --- a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts +++ b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts @@ -175,7 +175,7 @@ ak8963@f { compatible = "asahi-kasei,ak8963"; reg = <0x0f>; - gpios = <&tlmm 67 0>; + gpios = <&tlmm 67 GPIO_ACTIVE_HIGH>; vid-supply = <&pm8941_lvs1>; vdd-supply = <&pm8941_l17>; }; diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi index 8baca2a777176a3e8669ab1a988df9f25f878425..7a9be0acf3f5ac8e3ef8e735a7c89bbc90b73c14 100644 --- a/arch/arm/boot/dts/qcom-msm8974.dtsi +++ b/arch/arm/boot/dts/qcom-msm8974.dtsi @@ -439,14 +439,14 @@ sdhc_1: mmc@f9824900 { compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9824900 0x11c>, <0xf9824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC1_APPS_CLK>, - <&gcc GCC_SDCC1_AHB_CLK>, + clocks = <&gcc GCC_SDCC1_AHB_CLK>, + <&gcc GCC_SDCC1_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; bus-width = <8>; non-removable; @@ -456,14 +456,14 @@ sdhc_3: mmc@f9864900 { compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9864900 0x11c>, <0xf9864000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC3_APPS_CLK>, - <&gcc GCC_SDCC3_AHB_CLK>, + clocks = <&gcc GCC_SDCC3_AHB_CLK>, + <&gcc GCC_SDCC3_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; bus-width = <4>; #address-cells = <1>; @@ -475,14 +475,14 @@ sdhc_2: mmc@f98a4900 { compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; - clocks = <&gcc GCC_SDCC2_APPS_CLK>, - <&gcc GCC_SDCC2_AHB_CLK>, + clocks = <&gcc GCC_SDCC2_AHB_CLK>, + <&gcc GCC_SDCC2_APPS_CLK>, <&xo_board>; - clock-names = "core", "iface", "xo"; + clock-names = "iface", "core", "xo"; bus-width = <4>; #address-cells = <1>; @@ -762,6 +762,11 @@ }; }; + sram@fc190000 { + compatible = "qcom,msm8974-rpm-stats"; + reg = <0xfc190000 0x10000>; + }; + etf@fc307000 { compatible = "arm,coresight-tmc", "arm,primecell"; reg = <0xfc307000 0x1000>; diff --git a/arch/arm/boot/dts/qcom-msm8974pro.dtsi b/arch/arm/boot/dts/qcom-msm8974pro.dtsi index 1e882e16a221448310dd8911f76b87c7cb875fc7..58df6e75ab6da18a1afa470aa68158a73ac49457 100644 --- a/arch/arm/boot/dts/qcom-msm8974pro.dtsi +++ b/arch/arm/boot/dts/qcom-msm8974pro.dtsi @@ -10,10 +10,10 @@ }; &sdhc_1 { - clocks = <&gcc GCC_SDCC1_APPS_CLK>, - <&gcc GCC_SDCC1_AHB_CLK>, + clocks = <&gcc GCC_SDCC1_AHB_CLK>, + <&gcc GCC_SDCC1_APPS_CLK>, <&xo_board>, <&gcc GCC_SDCC1_CDCCAL_FF_CLK>, <&gcc GCC_SDCC1_CDCCAL_SLEEP_CLK>; - clock-names = "core", "iface", "xo", "cal", "sleep"; + clock-names = "iface", "core", "xo", "cal", "sleep"; }; diff --git a/arch/arm/boot/dts/qcom-pm8941.dtsi b/arch/arm/boot/dts/qcom-pm8941.dtsi index 59d0cde63251a809733d0d49a5b3080b98692382..9cd49deb9fa76e519c7f74436c3c4d8cfc13425f 100644 --- a/arch/arm/boot/dts/qcom-pm8941.dtsi +++ b/arch/arm/boot/dts/qcom-pm8941.dtsi @@ -93,7 +93,7 @@ #thermal-sensor-cells = <0>; }; - pm8941_vadc: vadc@3100 { + pm8941_vadc: adc@3100 { compatible = "qcom,spmi-vadc"; reg = <0x3100>; interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>; @@ -144,7 +144,7 @@ #address-cells = <1>; #size-cells = <0>; - pm8941_lpg: lpg { + pm8941_lpg: pwm { compatible = "qcom,pm8941-lpg"; #address-cells = <1>; diff --git a/arch/arm/boot/dts/qcom-pma8084.dtsi b/arch/arm/boot/dts/qcom-pma8084.dtsi index 7b8a8d9695da6506abc1eb2218a0e05b3a47b6e2..e77602e9f95c263753b7897a924287a575d1379b 100644 --- a/arch/arm/boot/dts/qcom-pma8084.dtsi +++ b/arch/arm/boot/dts/qcom-pma8084.dtsi @@ -56,7 +56,7 @@ io-channel-names = "thermal"; }; - pma8084_vadc: vadc@3100 { + pma8084_vadc: adc@3100 { compatible = "qcom,spmi-vadc"; reg = <0x3100>; interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>; diff --git a/arch/arm/boot/dts/qcom-pmx55.dtsi b/arch/arm/boot/dts/qcom-pmx55.dtsi index 9de7578a4c5f15c2cb48f78815cc9c9b88fd2f75..e1b869480bbd9d57f6812403cdf1c02c17589810 100644 --- a/arch/arm/boot/dts/qcom-pmx55.dtsi +++ b/arch/arm/boot/dts/qcom-pmx55.dtsi @@ -16,7 +16,7 @@ #address-cells = <1>; #size-cells = <0>; - power-on@800 { + pon@800 { compatible = "qcom,pm8916-pon"; reg = <0x0800>; diff --git a/arch/arm/boot/dts/qcom-sdx65.dtsi b/arch/arm/boot/dts/qcom-sdx65.dtsi index 8daefd50217a30eaba58524ade54bb49b074f731..4cd405db550009923dc18efb7323ee0e359b9d37 100644 --- a/arch/arm/boot/dts/qcom-sdx65.dtsi +++ b/arch/arm/boot/dts/qcom-sdx65.dtsi @@ -334,7 +334,7 @@ sdhc_1: mmc@8804000 { compatible = "qcom,sdx65-sdhci", "qcom,sdhci-msm-v5"; reg = <0x08804000 0x1000>; - reg-names = "hc_mem"; + reg-names = "hc"; interrupts = , ; interrupt-names = "hc_irq", "pwr_irq"; diff --git a/arch/arm/boot/dts/r8a7742.dtsi b/arch/arm/boot/dts/r8a7742.dtsi index 420e0b3259d48de65186fe0088f27a5f98664f7b..758a1bf02fae9a15b0f2d438d92a77d533d38773 100644 --- a/arch/arm/boot/dts/r8a7742.dtsi +++ b/arch/arm/boot/dts/r8a7742.dtsi @@ -633,7 +633,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7742", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -645,11 +645,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7743.dtsi b/arch/arm/boot/dts/r8a7743.dtsi index c0c145a5fe8da52af07fa88b064464b6d3539052..20f1d98a048dc1aff42c12c82e1fb6aa20f5d492 100644 --- a/arch/arm/boot/dts/r8a7743.dtsi +++ b/arch/arm/boot/dts/r8a7743.dtsi @@ -584,7 +584,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7743", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -596,11 +596,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7744.dtsi b/arch/arm/boot/dts/r8a7744.dtsi index 3f4fb53dd6df16963f69a7a7f22d83fd2adbf53c..96b2d5a4e8f64bd698f39e719a15e04a7bdb0c36 100644 --- a/arch/arm/boot/dts/r8a7744.dtsi +++ b/arch/arm/boot/dts/r8a7744.dtsi @@ -584,7 +584,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7744", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -596,11 +596,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7745.dtsi b/arch/arm/boot/dts/r8a7745.dtsi index fe8e98a66d93ebc8dcc93dda8df42b13ad935c3c..afc902e532d8d45a4352287731b6b3d398fcbc6c 100644 --- a/arch/arm/boot/dts/r8a7745.dtsi +++ b/arch/arm/boot/dts/r8a7745.dtsi @@ -525,7 +525,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7745", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -537,11 +537,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a77470.dtsi b/arch/arm/boot/dts/r8a77470.dtsi index c90f2a27021467013c0dbadb0967edc1741e89b7..a5cf663a0118ee1de0114db7efdd36b231b36180 100644 --- a/arch/arm/boot/dts/r8a77470.dtsi +++ b/arch/arm/boot/dts/r8a77470.dtsi @@ -357,7 +357,7 @@ status = "disabled"; }; - usbphy0: usb-phy@e6590100 { + usbphy0: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a77470", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -369,7 +369,7 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; @@ -393,7 +393,7 @@ status = "disabled"; }; - usbphy1: usb-phy@e6598100 { + usbphy1: usb-phy-controller@e6598100 { compatible = "renesas,usb-phy-r8a77470", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6598100 0 0x100>; @@ -405,7 +405,7 @@ resets = <&cpg 706>; status = "disabled"; - usb1: usb-channel@0 { + usb1: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7790.dtsi b/arch/arm/boot/dts/r8a7790.dtsi index a640488d513b2f531e3f5a9faa07515b09a7cf2c..db171e3c62f25eb879ab7f2755832bd455e16547 100644 --- a/arch/arm/boot/dts/r8a7790.dtsi +++ b/arch/arm/boot/dts/r8a7790.dtsi @@ -654,7 +654,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7790", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -666,11 +666,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7791.dtsi b/arch/arm/boot/dts/r8a7791.dtsi index 542ed0a71872959a9acf69174ae87e05fad2ed57..d8f91d9f42aeed8a024bf5b6b4f7bf6db01fd176 100644 --- a/arch/arm/boot/dts/r8a7791.dtsi +++ b/arch/arm/boot/dts/r8a7791.dtsi @@ -608,7 +608,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7791", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -620,11 +620,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi index b601ee6f7580b2852f0cb95f0869d07f3ff02864..7aa781ff3bff695234c0c0d072e5d52bd6398c5a 100644 --- a/arch/arm/boot/dts/r8a7794.dtsi +++ b/arch/arm/boot/dts/r8a7794.dtsi @@ -506,7 +506,7 @@ status = "disabled"; }; - usbphy: usb-phy@e6590100 { + usbphy: usb-phy-controller@e6590100 { compatible = "renesas,usb-phy-r8a7794", "renesas,rcar-gen2-usb-phy"; reg = <0 0xe6590100 0 0x100>; @@ -518,11 +518,11 @@ resets = <&cpg 704>; status = "disabled"; - usb0: usb-channel@0 { + usb0: usb-phy@0 { reg = <0>; #phy-cells = <1>; }; - usb2: usb-channel@2 { + usb2: usb-phy@2 { reg = <2>; #phy-cells = <1>; }; diff --git a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts index 4bf813335e2121bacc6557b3b9097e283d5ebac4..c18bbd7141c41b3c83cf324221a20a4b606793d8 100644 --- a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts +++ b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts @@ -26,6 +26,22 @@ }; }; +&can0 { + pinctrl-0 = <&pins_can0>; + pinctrl-names = "default"; + + /* Assuming CN10/CN11 are wired for CAN1 */ + status = "okay"; +}; + +&can1 { + pinctrl-0 = <&pins_can1>; + pinctrl-names = "default"; + + /* Please only enable can0 or can1, depending on CN10/CN11 */ + /* status = "okay"; */ +}; + ð_miic { status = "okay"; renesas,miic-switch-portin = ; @@ -52,6 +68,18 @@ }; &pinctrl{ + pins_can0: pins_can0 { + pinmux = , /* CAN0_TXD */ + ; /* CAN0_RXD */ + drive-strength = <6>; + }; + + pins_can1: pins_can1 { + pinmux = , /* CAN1_TXD */ + ; /* CAN1_RXD */ + drive-strength = <6>; + }; + pins_eth3: pins_eth3 { pinmux = , , diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi index 5b97fa85474fd63492283f1f9a9cb2995c1848c1..563024c9a4ae77d623f5100c12673e307845430e 100644 --- a/arch/arm/boot/dts/r9a06g032.dtsi +++ b/arch/arm/boot/dts/r9a06g032.dtsi @@ -423,6 +423,26 @@ interrupts = ; }; + + can0: can@52104000 { + compatible = "renesas,r9a06g032-sja1000","renesas,rzn1-sja1000"; + reg = <0x52104000 0x800>; + reg-io-width = <4>; + interrupts = ; + clocks = <&sysctrl R9A06G032_HCLK_CAN0>; + power-domains = <&sysctrl>; + status = "disabled"; + }; + + can1: can@52105000 { + compatible = "renesas,r9a06g032-sja1000", "renesas,rzn1-sja1000"; + reg = <0x52105000 0x800>; + reg-io-width = <4>; + interrupts = ; + clocks = <&sysctrl R9A06G032_HCLK_CAN1>; + power-domains = <&sysctrl>; + status = "disabled"; + }; }; timer { diff --git a/arch/arm/boot/dts/rk3036-evb.dts b/arch/arm/boot/dts/rk3036-evb.dts index 2a7e6624efb93127ab3167bad028b21dab33ad2a..9fd4d9db9f8f67c665eefd4bc12fbb96b2f3f21e 100644 --- a/arch/arm/boot/dts/rk3036-evb.dts +++ b/arch/arm/boot/dts/rk3036-evb.dts @@ -15,16 +15,20 @@ }; &emac { - pinctrl-names = "default"; - pinctrl-0 = <&emac_xfer>, <&emac_mdio>; phy = <&phy0>; - phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */ phy-reset-duration = <10>; /* millisecond */ - + phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */ + pinctrl-names = "default"; + pinctrl-0 = <&emac_xfer>, <&emac_mdio>; status = "okay"; - phy0: ethernet-phy@0 { - reg = <0>; + mdio { + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@0 { + reg = <0>; + }; }; }; diff --git a/arch/arm/boot/dts/rk3036-kylin.dts b/arch/arm/boot/dts/rk3036-kylin.dts index e817eba8c622baa025c5ef1d0477953bb7201480..67e1e04139e7326bc2539d555dd2be10fec1d146 100644 --- a/arch/arm/boot/dts/rk3036-kylin.dts +++ b/arch/arm/boot/dts/rk3036-kylin.dts @@ -80,16 +80,20 @@ }; &emac { - pinctrl-names = "default"; - pinctrl-0 = <&emac_xfer>, <&emac_mdio>; phy = <&phy0>; - phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */ phy-reset-duration = <10>; /* millisecond */ - + phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */ + pinctrl-names = "default"; + pinctrl-0 = <&emac_xfer>, <&emac_mdio>; status = "okay"; - phy0: ethernet-phy@0 { - reg = <0>; + mdio { + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@0 { + reg = <0>; + }; }; }; diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi index 9b0f0497567d71bfaf07c77078872879e5a08b38..78686fc72ce69ac140049e328c5d35123486f57c 100644 --- a/arch/arm/boot/dts/rk3036.dtsi +++ b/arch/arm/boot/dts/rk3036.dtsi @@ -225,11 +225,9 @@ }; emac: ethernet@10200000 { - compatible = "rockchip,rk3036-emac", "snps,arc-emac"; + compatible = "rockchip,rk3036-emac"; reg = <0x10200000 0x4000>; interrupts = ; - #address-cells = <1>; - #size-cells = <0>; rockchip,grf = <&grf>; clocks = <&cru HCLK_MAC>, <&cru SCLK_MACREF>, <&cru SCLK_MAC>; clock-names = "hclk", "macref", "macclk"; diff --git a/arch/arm/boot/dts/rk3066a-marsboard.dts b/arch/arm/boot/dts/rk3066a-marsboard.dts index a66d915aa0f65758f1e733cb70ce44bad5de7d77..8beecd628282412621b89c237cc10af8668c07fd 100644 --- a/arch/arm/boot/dts/rk3066a-marsboard.dts +++ b/arch/arm/boot/dts/rk3066a-marsboard.dts @@ -150,18 +150,21 @@ #include "tps65910.dtsi" &emac { - status = "okay"; - phy = <&phy0>; phy-supply = <&vcc_rmii>; - pinctrl-names = "default"; pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&phy_int>; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; - phy0: ethernet-phy@0 { - reg = <0>; - interrupt-parent = <&gpio1>; - interrupts = ; + phy0: ethernet-phy@0 { + reg = <0>; + interrupt-parent = <&gpio1>; + interrupts = ; + }; }; }; diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts index dbbc5170094e01ffc6a91b52322cc2564ae094d6..3eee42137b6da8bf70ace489ce5c3dd758bfc8fc 100644 --- a/arch/arm/boot/dts/rk3066a-rayeager.dts +++ b/arch/arm/boot/dts/rk3066a-rayeager.dts @@ -142,15 +142,20 @@ }; &emac { - pinctrl-names = "default"; - pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&rmii_rst>; phy = <&phy0>; phy-supply = <&vcc_rmii>; + pinctrl-names = "default"; + pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&rmii_rst>; status = "okay"; - phy0: ethernet-phy@0 { - reg = <0>; - reset-gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_LOW>; + mdio { + #address-cells = <1>; + #size-cells = <0>; + + phy0: ethernet-phy@0 { + reg = <0>; + reset-gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_LOW>; + }; }; }; diff --git a/arch/arm/boot/dts/rk3188-radxarock.dts b/arch/arm/boot/dts/rk3188-radxarock.dts index a9ed3cd2c2da64512a557a58af7753f300286fea..e7cf18823558abb2c438176d0528cdf07555f393 100644 --- a/arch/arm/boot/dts/rk3188-radxarock.dts +++ b/arch/arm/boot/dts/rk3188-radxarock.dts @@ -126,18 +126,21 @@ }; &emac { - status = "okay"; - + phy = <&phy0>; + phy-supply = <&vcc_rmii>; pinctrl-names = "default"; pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&phy_int>; + status = "okay"; - phy = <&phy0>; - phy-supply = <&vcc_rmii>; + mdio { + #address-cells = <1>; + #size-cells = <0>; - phy0: ethernet-phy@0 { - reg = <0>; - interrupt-parent = <&gpio3>; - interrupts = ; + phy0: ethernet-phy@0 { + reg = <0>; + interrupt-parent = <&gpio3>; + interrupts = ; + }; }; }; diff --git a/arch/arm/boot/dts/rk3xxx.dtsi b/arch/arm/boot/dts/rk3xxx.dtsi index 616a828e0c6e4c6f58588798d0c019767e0754a1..bf285091a9eb11b15d527218051043a726cd8dad 100644 --- a/arch/arm/boot/dts/rk3xxx.dtsi +++ b/arch/arm/boot/dts/rk3xxx.dtsi @@ -186,8 +186,6 @@ compatible = "snps,arc-emac"; reg = <0x10204000 0x3c>; interrupts = ; - #address-cells = <1>; - #size-cells = <0>; rockchip,grf = <&grf>; diff --git a/arch/arm/boot/dts/s5pv210-aries.dtsi b/arch/arm/boot/dts/s5pv210-aries.dtsi index 5541df4df6284b3a4c15cc0d038d5a62272f7bae..964c5fe517555516ec37203db601ce7cfed34541 100644 --- a/arch/arm/boot/dts/s5pv210-aries.dtsi +++ b/arch/arm/boot/dts/s5pv210-aries.dtsi @@ -738,7 +738,7 @@ samsung,pin-pud = ; }; - magnetometer_i2c_pins: yas529-i2c-pins-pins { + magnetometer_i2c_pins: yas529-i2c-pins { samsung,pins = "gpj0-0", "gpj0-1"; samsung,pin-pud = ; samsung,pin-drv = ; @@ -788,7 +788,7 @@ samsung,pin-drv = ; }; - pmic_i2c_pins: pmic-i2c-pins-pins { + pmic_i2c_pins: pmic-i2c-pins { samsung,pins = "gpj4-0", "gpj4-3"; samsung,pin-pud = ; samsung,pin-drv = ; diff --git a/arch/arm/boot/dts/s5pv210-galaxys.dts b/arch/arm/boot/dts/s5pv210-galaxys.dts index cdd3653d487f61e787b31eb7d80504331a069db2..532d3f5bceb1f24e716bbef1dcb5de80f884e52a 100644 --- a/arch/arm/boot/dts/s5pv210-galaxys.dts +++ b/arch/arm/boot/dts/s5pv210-galaxys.dts @@ -150,7 +150,7 @@ pinctrl-names = "default"; pinctrl-0 = <&sleep_cfg>; - fm_i2c_pins: fm-i2c-pins-pins { + fm_i2c_pins: fm-i2c-pins { samsung,pins = "gpd1-2", "gpd1-3"; samsung,pin-pud = ; samsung,pin-drv = ; diff --git a/arch/arm/boot/dts/sam9x60.dtsi b/arch/arm/boot/dts/sam9x60.dtsi index d3f60f6a456d6d578b188c2a966eeda38d7d98dd..8f5477e307dd49e444d66289663f03048b59b5e2 100644 --- a/arch/arm/boot/dts/sam9x60.dtsi +++ b/arch/arm/boot/dts/sam9x60.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include #include / { @@ -583,6 +584,7 @@ dbgu: serial@fffff200 { compatible = "microchip,sam9x60-dbgu", "microchip,sam9x60-usart", "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfffff200 0x200>; + atmel,usart-mode = ; interrupts = <47 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi index 2c50a021aa76831f7870fef5612906398d3bfb71..14c35c12a115fe8900d0b6248b0053ef11ffa219 100644 --- a/arch/arm/boot/dts/sama5d2.dtsi +++ b/arch/arm/boot/dts/sama5d2.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include #include / { @@ -441,6 +442,7 @@ uart0: serial@f801c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf801c000 0x100>; + atmel,usart-mode = ; interrupts = <24 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | @@ -457,6 +459,7 @@ uart1: serial@f8020000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8020000 0x100>; + atmel,usart-mode = ; interrupts = <25 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | @@ -473,6 +476,7 @@ uart2: serial@f8024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8024000 0x100>; + atmel,usart-mode = ; interrupts = <26 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | @@ -530,6 +534,7 @@ uart5: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>; clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; clock-names = "usart"; @@ -600,6 +605,7 @@ uart6: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <20 IRQ_TYPE_LEVEL_HIGH 7>; clocks = <&pmc PMC_TYPE_PERIPHERAL 20>; clock-names = "usart"; @@ -769,6 +775,7 @@ uart3: serial@fc008000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc008000 0x100>; + atmel,usart-mode = ; interrupts = <27 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma1 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | @@ -785,6 +792,7 @@ uart4: serial@fc00c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc00c000 0x100>; + atmel,usart-mode = ; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) | AT91_XDMAC_DT_PERID(43))>, @@ -810,6 +818,7 @@ uart7: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <21 IRQ_TYPE_LEVEL_HIGH 7>; clocks = <&pmc PMC_TYPE_PERIPHERAL 21>; clock-names = "usart"; @@ -880,6 +889,7 @@ uart8: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <22 IRQ_TYPE_LEVEL_HIGH 7>; clocks = <&pmc PMC_TYPE_PERIPHERAL 22>; clock-names = "usart"; @@ -951,6 +961,7 @@ uart9: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = <23 IRQ_TYPE_LEVEL_HIGH 7>; clocks = <&pmc PMC_TYPE_PERIPHERAL 23>; clock-names = "usart"; diff --git a/arch/arm/boot/dts/sama5d3.dtsi b/arch/arm/boot/dts/sama5d3.dtsi index 2d0935ad2225d6c3500e9c7034982150f2ff1071..bde8e92d60bb10973b8c53e689e9e9bcea8975cc 100644 --- a/arch/arm/boot/dts/sama5d3.dtsi +++ b/arch/arm/boot/dts/sama5d3.dtsi @@ -12,6 +12,7 @@ #include #include #include +#include / { #address-cells = <1>; @@ -194,6 +195,7 @@ usart0: serial@f001c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf001c000 0x100>; + atmel,usart-mode = ; interrupts = <12 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(3)>, <&dma0 2 (AT91_DMA_CFG_PER_ID(4) | AT91_DMA_CFG_FIFOCFG_ASAP)>; @@ -208,6 +210,7 @@ usart1: serial@f0020000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf0020000 0x100>; + atmel,usart-mode = ; interrupts = <13 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(5)>, <&dma0 2 (AT91_DMA_CFG_PER_ID(6) | AT91_DMA_CFG_FIFOCFG_ASAP)>; @@ -222,6 +225,7 @@ uart0: serial@f0024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf0024000 0x100>; + atmel,usart-mode = ; interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; @@ -356,6 +360,7 @@ usart2: serial@f8020000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8020000 0x100>; + atmel,usart-mode = ; interrupts = <14 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(7)>, <&dma1 2 (AT91_DMA_CFG_PER_ID(8) | AT91_DMA_CFG_FIFOCFG_ASAP)>; @@ -370,6 +375,7 @@ usart3: serial@f8024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8024000 0x100>; + atmel,usart-mode = ; interrupts = <15 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(9)>, <&dma1 2 (AT91_DMA_CFG_PER_ID(10) | AT91_DMA_CFG_FIFOCFG_ASAP)>; @@ -464,6 +470,7 @@ dbgu: serial@ffffee00 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xffffee00 0x200>; + atmel,usart-mode = ; interrupts = <2 IRQ_TYPE_LEVEL_HIGH 7>; dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(13)>, <&dma1 2 (AT91_DMA_CFG_PER_ID(14) | AT91_DMA_CFG_FIFOCFG_ASAP)>; diff --git a/arch/arm/boot/dts/sama5d3_uart.dtsi b/arch/arm/boot/dts/sama5d3_uart.dtsi index a3eaba995cf449244162eb52fe5bf121d324b1e2..44d1173f2ffb7c121acb85e20cc7a5102c9aa26d 100644 --- a/arch/arm/boot/dts/sama5d3_uart.dtsi +++ b/arch/arm/boot/dts/sama5d3_uart.dtsi @@ -9,6 +9,7 @@ #include #include #include +#include / { aliases { @@ -39,6 +40,7 @@ uart0: serial@f0024000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf0024000 0x100>; + atmel,usart-mode = ; interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; @@ -50,6 +52,7 @@ uart1: serial@f8028000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8028000 0x100>; + atmel,usart-mode = ; interrupts = <17 IRQ_TYPE_LEVEL_HIGH 5>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; diff --git a/arch/arm/boot/dts/sama5d4.dtsi b/arch/arm/boot/dts/sama5d4.dtsi index 1e5c01898ccff3d5eb6e54a173de7c19be56c3cc..af62157ae214f30b2bde3d88280109926e9c813f 100644 --- a/arch/arm/boot/dts/sama5d4.dtsi +++ b/arch/arm/boot/dts/sama5d4.dtsi @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -278,6 +279,7 @@ uart0: serial@f8004000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8004000 0x100>; + atmel,usart-mode = ; interrupts = <27 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -437,6 +439,7 @@ usart0: serial@f802c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf802c000 0x100>; + atmel,usart-mode = ; interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -455,6 +458,7 @@ usart1: serial@f8030000 { compatible = "atmel,at91sam9260-usart"; reg = <0xf8030000 0x100>; + atmel,usart-mode = ; interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -490,6 +494,7 @@ uart1: serial@fc004000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc004000 0x100>; + atmel,usart-mode = ; interrupts = <28 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma0 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -508,6 +513,7 @@ usart2: serial@fc008000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc008000 0x100>; + atmel,usart-mode = ; interrupts = <29 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma1 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -526,6 +532,7 @@ usart3: serial@fc00c000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc00c000 0x100>; + atmel,usart-mode = ; interrupts = <30 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma1 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -544,6 +551,7 @@ usart4: serial@fc010000 { compatible = "atmel,at91sam9260-usart"; reg = <0xfc010000 0x100>; + atmel,usart-mode = ; interrupts = <31 IRQ_TYPE_LEVEL_HIGH 5>; dmas = <&dma1 (AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) @@ -770,6 +778,7 @@ dbgu: serial@fc069000 { compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart"; reg = <0xfc069000 0x200>; + atmel,usart-mode = ; interrupts = <45 IRQ_TYPE_LEVEL_HIGH 7>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_dbgu>; diff --git a/arch/arm/boot/dts/sama7g5.dtsi b/arch/arm/boot/dts/sama7g5.dtsi index bb6d71e6dfebcdacbaafdfd68e7f1a4fa27a3d72..7bd8ae8e8d3808809aa610509dda721c4820e1b0 100644 --- a/arch/arm/boot/dts/sama7g5.dtsi +++ b/arch/arm/boot/dts/sama7g5.dtsi @@ -14,6 +14,7 @@ #include #include #include +#include / { model = "Microchip SAMA7G5 family SoC"; @@ -603,6 +604,7 @@ uart0: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = ; clocks = <&pmc PMC_TYPE_PERIPHERAL 38>; clock-names = "usart"; @@ -651,6 +653,7 @@ uart3: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = ; clocks = <&pmc PMC_TYPE_PERIPHERAL 41>; clock-names = "usart"; @@ -694,6 +697,7 @@ uart4: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = ; clocks = <&pmc PMC_TYPE_PERIPHERAL 42>; clock-names = "usart"; @@ -719,6 +723,7 @@ uart7: serial@200 { compatible = "atmel,at91sam9260-usart"; reg = <0x200 0x200>; + atmel,usart-mode = ; interrupts = ; clocks = <&pmc PMC_TYPE_PERIPHERAL 45>; clock-names = "usart"; @@ -866,9 +871,9 @@ #address-cells = <1>; #size-cells = <0>; atmel,fifo-size = <32>; - dmas = <&dma0 AT91_XDMAC_DT_PERID(27)>, - <&dma0 AT91_XDMAC_DT_PERID(28)>; - dma-names = "rx", "tx"; + dmas = <&dma0 AT91_XDMAC_DT_PERID(28)>, + <&dma0 AT91_XDMAC_DT_PERID(27)>; + dma-names = "tx", "rx"; status = "disabled"; }; }; diff --git a/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi index d2472cd8f1d03126be8a928a692078567928193c..efdd163eba30e91dbd05ac782ff258cc422d166d 100644 --- a/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi +++ b/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi @@ -6,6 +6,40 @@ #include &pinctrl { + i2c1_pins_a: i2c1-0 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c1_sleep_pins_a: i2c1-sleep-0 { + pins { + pinmux = , /* I2C1_SCL */ + ; /* I2C1_SDA */ + }; + }; + + i2c5_pins_a: i2c5-0 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + bias-disable; + drive-open-drain; + slew-rate = <0>; + }; + }; + + i2c5_sleep_pins_a: i2c5-sleep-0 { + pins { + pinmux = , /* I2C5_SCL */ + ; /* I2C5_SDA */ + }; + }; + sdmmc1_b4_pins_a: sdmmc1-b4-0 { pins { pinmux = , /* SDMMC1_D0 */ @@ -108,6 +142,29 @@ }; }; + spi5_pins_a: spi5-0 { + pins1 { + pinmux = , /* SPI5_SCK */ + ; /* SPI5_MOSI */ + bias-disable; + drive-push-pull; + slew-rate = <1>; + }; + + pins2 { + pinmux = ; /* SPI5_MISO */ + bias-disable; + }; + }; + + spi5_sleep_pins_a: spi5-sleep-0 { + pins { + pinmux = , /* SPI5_SCK */ + , /* SPI5_MISO */ + ; /* SPI5_MOSI */ + }; + }; + uart4_pins_a: uart4-0 { pins1 { pinmux = ; /* UART4_TX */ diff --git a/arch/arm/boot/dts/stm32mp131.dtsi b/arch/arm/boot/dts/stm32mp131.dtsi index 3a921db23e9f625396ea7f9f8f8964b860ba022d..dd35a607073dd69d2d4c2fb078d3a6055ebe4546 100644 --- a/arch/arm/boot/dts/stm32mp131.dtsi +++ b/arch/arm/boot/dts/stm32mp131.dtsi @@ -97,6 +97,34 @@ }; }; + spi2: spi@4000b000 { + compatible = "st,stm32h7-spi"; + reg = <0x4000b000 0x400>; + interrupts = ; + clocks = <&rcc SPI2_K>; + resets = <&rcc SPI2_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 39 0x400 0x01>, + <&dmamux1 40 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi3: spi@4000c000 { + compatible = "st,stm32h7-spi"; + reg = <0x4000c000 0x400>; + interrupts = ; + clocks = <&rcc SPI3_K>; + resets = <&rcc SPI3_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 61 0x400 0x01>, + <&dmamux1 62 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + uart4: serial@40010000 { compatible = "st,stm32h7-uart"; reg = <0x40010000 0x400>; @@ -106,6 +134,56 @@ status = "disabled"; }; + i2c1: i2c@40012000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x40012000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C1_K>; + resets = <&rcc I2C1_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 33 0x400 0x1>, + <&dmamux1 34 0x400 0x1>; + dma-names = "rx", "tx"; + st,syscfg-fmp = <&syscfg 0x4 0x1>; + i2c-analog-filter; + status = "disabled"; + }; + + i2c2: i2c@40013000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x40013000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C2_K>; + resets = <&rcc I2C2_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 35 0x400 0x1>, + <&dmamux1 36 0x400 0x1>; + dma-names = "rx", "tx"; + st,syscfg-fmp = <&syscfg 0x4 0x2>; + i2c-analog-filter; + status = "disabled"; + }; + + spi1: spi@44004000 { + compatible = "st,stm32h7-spi"; + reg = <0x44004000 0x400>; + interrupts = ; + clocks = <&rcc SPI1_K>; + resets = <&rcc SPI1_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 37 0x400 0x01>, + <&dmamux1 38 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + dma1: dma-controller@48000000 { compatible = "st,stm32-dma"; reg = <0x48000000 0x400>; @@ -153,6 +231,88 @@ dma-channels = <16>; }; + spi4: spi@4c002000 { + compatible = "st,stm32h7-spi"; + reg = <0x4c002000 0x400>; + interrupts = ; + clocks = <&rcc SPI4_K>; + resets = <&rcc SPI4_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 83 0x400 0x01>, + <&dmamux1 84 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + spi5: spi@4c003000 { + compatible = "st,stm32h7-spi"; + reg = <0x4c003000 0x400>; + interrupts = ; + clocks = <&rcc SPI5_K>; + resets = <&rcc SPI5_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 85 0x400 0x01>, + <&dmamux1 86 0x400 0x01>; + dma-names = "rx", "tx"; + status = "disabled"; + }; + + i2c3: i2c@4c004000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c004000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C3_K>; + resets = <&rcc I2C3_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 73 0x400 0x1>, + <&dmamux1 74 0x400 0x1>; + dma-names = "rx", "tx"; + st,syscfg-fmp = <&syscfg 0x4 0x4>; + i2c-analog-filter; + status = "disabled"; + }; + + i2c4: i2c@4c005000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c005000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C4_K>; + resets = <&rcc I2C4_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 75 0x400 0x1>, + <&dmamux1 76 0x400 0x1>; + dma-names = "rx", "tx"; + st,syscfg-fmp = <&syscfg 0x4 0x8>; + i2c-analog-filter; + status = "disabled"; + }; + + i2c5: i2c@4c006000 { + compatible = "st,stm32mp13-i2c"; + reg = <0x4c006000 0x400>; + interrupt-names = "event", "error"; + interrupts = , + ; + clocks = <&rcc I2C5_K>; + resets = <&rcc I2C5_R>; + #address-cells = <1>; + #size-cells = <0>; + dmas = <&dmamux1 115 0x400 0x1>, + <&dmamux1 116 0x400 0x1>; + dma-names = "rx", "tx"; + st,syscfg-fmp = <&syscfg 0x4 0x10>; + i2c-analog-filter; + status = "disabled"; + }; + rcc: rcc@50000000 { compatible = "st,stm32mp13-rcc", "syscon"; reg = <0x50000000 0x1000>; diff --git a/arch/arm/boot/dts/stm32mp135f-dk.dts b/arch/arm/boot/dts/stm32mp135f-dk.dts index e6b8ffd332c7e2dba8fb53e98bac1994cc61e896..de341d17e87d421e18efa0ea5a6e9887792f9abb 100644 --- a/arch/arm/boot/dts/stm32mp135f-dk.dts +++ b/arch/arm/boot/dts/stm32mp135f-dk.dts @@ -68,6 +68,32 @@ }; }; +&i2c1 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c1_pins_a>; + pinctrl-1 = <&i2c1_sleep_pins_a>; + i2c-scl-rising-time-ns = <96>; + i2c-scl-falling-time-ns = <3>; + clock-frequency = <1000000>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; +}; + +&i2c5 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c5_pins_a>; + pinctrl-1 = <&i2c5_sleep_pins_a>; + i2c-scl-rising-time-ns = <170>; + i2c-scl-falling-time-ns = <5>; + clock-frequency = <400000>; + status = "okay"; + /* spare dmas for other usage */ + /delete-property/dmas; + /delete-property/dma-names; +}; + &iwdg2 { timeout-sec = <32>; status = "okay"; @@ -90,6 +116,13 @@ status = "okay"; }; +&spi5 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi5_pins_a>; + pinctrl-1 = <&spi5_sleep_pins_a>; + status = "disabled"; +}; + &uart4 { pinctrl-names = "default"; pinctrl-0 = <&uart4_pins_a>; diff --git a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi index 2cc9341d43d29511ecbb55722e9dacc10624409e..a9d2bec9901412ae98cd7f3981d5269c16cb46b8 100644 --- a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi +++ b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi @@ -1261,7 +1261,7 @@ }; qspi_bk1_pins_a: qspi-bk1-0 { - pins1 { + pins { pinmux = , /* QSPI_BK1_IO0 */ , /* QSPI_BK1_IO1 */ , /* QSPI_BK1_IO2 */ @@ -1270,12 +1270,6 @@ drive-push-pull; slew-rate = <1>; }; - pins2 { - pinmux = ; /* QSPI_BK1_NCS */ - bias-pull-up; - drive-push-pull; - slew-rate = <1>; - }; }; qspi_bk1_sleep_pins_a: qspi-bk1-sleep-0 { @@ -1283,13 +1277,12 @@ pinmux = , /* QSPI_BK1_IO0 */ , /* QSPI_BK1_IO1 */ , /* QSPI_BK1_IO2 */ - , /* QSPI_BK1_IO3 */ - ; /* QSPI_BK1_NCS */ + ; /* QSPI_BK1_IO3 */ }; }; qspi_bk2_pins_a: qspi-bk2-0 { - pins1 { + pins { pinmux = , /* QSPI_BK2_IO0 */ , /* QSPI_BK2_IO1 */ , /* QSPI_BK2_IO2 */ @@ -1298,7 +1291,34 @@ drive-push-pull; slew-rate = <1>; }; - pins2 { + }; + + qspi_bk2_sleep_pins_a: qspi-bk2-sleep-0 { + pins { + pinmux = , /* QSPI_BK2_IO0 */ + , /* QSPI_BK2_IO1 */ + , /* QSPI_BK2_IO2 */ + ; /* QSPI_BK2_IO3 */ + }; + }; + + qspi_cs1_pins_a: qspi-cs1-0 { + pins { + pinmux = ; /* QSPI_BK1_NCS */ + bias-pull-up; + drive-push-pull; + slew-rate = <1>; + }; + }; + + qspi_cs1_sleep_pins_a: qspi-cs1-sleep-0 { + pins { + pinmux = ; /* QSPI_BK1_NCS */ + }; + }; + + qspi_cs2_pins_a: qspi-cs2-0 { + pins { pinmux = ; /* QSPI_BK2_NCS */ bias-pull-up; drive-push-pull; @@ -1306,13 +1326,9 @@ }; }; - qspi_bk2_sleep_pins_a: qspi-bk2-sleep-0 { + qspi_cs2_sleep_pins_a: qspi-cs2-sleep-0 { pins { - pinmux = , /* QSPI_BK2_IO0 */ - , /* QSPI_BK2_IO1 */ - , /* QSPI_BK2_IO2 */ - , /* QSPI_BK2_IO3 */ - ; /* QSPI_BK2_NCS */ + pinmux = ; /* QSPI_BK2_NCS */ }; }; diff --git a/arch/arm/boot/dts/stm32mp151.dtsi b/arch/arm/boot/dts/stm32mp151.dtsi index 742fdeeff4b604d559fd718ebc914593049e9d56..e02b3f5d44cba4e6ff7cd7f1fd764437bc69c51f 100644 --- a/arch/arm/boot/dts/stm32mp151.dtsi +++ b/arch/arm/boot/dts/stm32mp151.dtsi @@ -127,6 +127,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40000000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM2_K>; clock-names = "int"; dmas = <&dmamux1 18 0x400 0x1>, @@ -160,6 +162,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40001000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM3_K>; clock-names = "int"; dmas = <&dmamux1 23 0x400 0x1>, @@ -194,6 +198,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40002000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM4_K>; clock-names = "int"; dmas = <&dmamux1 29 0x400 0x1>, @@ -226,6 +232,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40003000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM5_K>; clock-names = "int"; dmas = <&dmamux1 55 0x400 0x1>, @@ -260,6 +268,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40004000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM6_K>; clock-names = "int"; dmas = <&dmamux1 69 0x400 0x1>; @@ -278,6 +288,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40005000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM7_K>; clock-names = "int"; dmas = <&dmamux1 70 0x400 0x1>; @@ -296,6 +308,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40006000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM12_K>; clock-names = "int"; status = "disabled"; @@ -318,6 +332,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40007000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM13_K>; clock-names = "int"; status = "disabled"; @@ -340,6 +356,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x40008000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM14_K>; clock-names = "int"; status = "disabled"; @@ -623,6 +641,11 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x44000000 0x400>; + interrupts = , + , + , + ; + interrupt-names = "brk", "up", "trg-com", "cc"; clocks = <&rcc TIM1_K>; clock-names = "int"; dmas = <&dmamux1 11 0x400 0x1>, @@ -659,6 +682,11 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x44001000 0x400>; + interrupts = , + , + , + ; + interrupt-names = "brk", "up", "trg-com", "cc"; clocks = <&rcc TIM8_K>; clock-names = "int"; dmas = <&dmamux1 47 0x400 0x1>, @@ -746,6 +774,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x44006000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM15_K>; clock-names = "int"; dmas = <&dmamux1 105 0x400 0x1>, @@ -773,6 +803,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x44007000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM16_K>; clock-names = "int"; dmas = <&dmamux1 109 0x400 0x1>, @@ -797,6 +829,8 @@ #size-cells = <0>; compatible = "st,stm32-timers"; reg = <0x44008000 0x400>; + interrupts = ; + interrupt-names = "global"; clocks = <&rcc TIM17_K>; clock-names = "int"; dmas = <&dmamux1 111 0x400 0x1>, diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts index 2a2829283456d25a9ac79c88b0c71b3dcb3709e1..9a2a4bc7d0793635684f20ec6088069fc5e837f4 100644 --- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts +++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts index 1f75f1d451813429e89fc29e264b59b0a69f8a1b..60ce4425a7fdd82ca83846a674c9cbb83708b9e0 100644 --- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts +++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts index ba92d7d8ed00377dbd4784597e402a21246ddcd0..390ee8c057547445dfe7368f28e683a8bd44e2f0 100644 --- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts +++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi index 01166ccacf2b0b1298d6466dd90a2a6e5f4ca46e..9de893101b4095b63ebd215241e723361d4c3e58 100644 --- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi +++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ / { diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts index fae656edd8209a963918fa13cc0b8694897e1aa9..0d7560ba2950f868e7de8f934ca8e490de5788ed 100644 --- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts +++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts index b9d0d3d6ad15bb7e6ed880804e63be798735c394..d949559be02070312fbe893bb47b711288d315e7 100644 --- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts +++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi index 0b85175f151e4e3e4e389db552f3843e738ae9b5..fb4600a598698837aaac7aa2fb28679c6322d7c0 100644 --- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi +++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi @@ -2,7 +2,7 @@ /* * Copyright (c) STMicroelectronics 2019 - All Rights Reserved * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ / { diff --git a/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi b/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi index ac53ee3c496b283d8373187ba5fb31a61620cff8..30156b7546ed61f01101e7e063b78114c0f2d346 100644 --- a/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi +++ b/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi @@ -435,12 +435,6 @@ pinctrl-0 = <&spi1_pins_a>; cs-gpios = <&gpioz 3 0>; status = "disabled"; - - spidev@0 { - compatible = "spidev"; - reg = <0>; - spi-max-frequency = <100000>; - }; }; &timers1 { diff --git a/arch/arm/boot/dts/stm32mp157c-ev1.dts b/arch/arm/boot/dts/stm32mp157c-ev1.dts index d142dd30e16b32cfe50f2c4e76096de9703ea5a2..050c3c27a4203680dc23d731910fccaba8f956b1 100644 --- a/arch/arm/boot/dts/stm32mp157c-ev1.dts +++ b/arch/arm/boot/dts/stm32mp157c-ev1.dts @@ -255,8 +255,16 @@ &qspi { pinctrl-names = "default", "sleep"; - pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a &qspi_bk2_pins_a>; - pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a &qspi_bk2_sleep_pins_a>; + pinctrl-0 = <&qspi_clk_pins_a + &qspi_bk1_pins_a + &qspi_cs1_pins_a + &qspi_bk2_pins_a + &qspi_cs2_pins_a>; + pinctrl-1 = <&qspi_clk_sleep_pins_a + &qspi_bk1_sleep_pins_a + &qspi_cs1_sleep_pins_a + &qspi_bk2_sleep_pins_a + &qspi_cs2_sleep_pins_a>; reg = <0x58003000 0x1000>, <0x70000000 0x4000000>; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm/boot/dts/tegra30-apalis-v1.1.dtsi b/arch/arm/boot/dts/tegra30-apalis-v1.1.dtsi index 380f22a35821a2cc14eb3bb6b16e6610962f6bc8..a1bcd67fa50591d6292165cf4cf3098c20f897f5 100644 --- a/arch/arm/boot/dts/tegra30-apalis-v1.1.dtsi +++ b/arch/arm/boot/dts/tegra30-apalis-v1.1.dtsi @@ -993,7 +993,7 @@ touchscreen@41 { compatible = "st,stmpe811"; reg = <0x41>; - irq-gpio = <&gpio TEGRA_GPIO(V, 0) IRQ_TYPE_LEVEL_LOW>; + irq-gpio = <&gpio TEGRA_GPIO(V, 0) GPIO_ACTIVE_LOW>; interrupt-controller; id = <0>; blocks = <0x5>; diff --git a/arch/arm/boot/dts/tegra30-apalis.dtsi b/arch/arm/boot/dts/tegra30-apalis.dtsi index 9bdc4cb71449c8bfb70ffd91f324ce7a0f452820..99d7dad72d29178a9e11605dfd0294696e30c6ea 100644 --- a/arch/arm/boot/dts/tegra30-apalis.dtsi +++ b/arch/arm/boot/dts/tegra30-apalis.dtsi @@ -976,7 +976,7 @@ touchscreen@41 { compatible = "st,stmpe811"; reg = <0x41>; - irq-gpio = <&gpio TEGRA_GPIO(V, 0) IRQ_TYPE_LEVEL_LOW>; + irq-gpio = <&gpio TEGRA_GPIO(V, 0) GPIO_ACTIVE_LOW>; interrupt-controller; id = <0>; blocks = <0x5>; diff --git a/arch/arm/boot/dts/tegra30-colibri.dtsi b/arch/arm/boot/dts/tegra30-colibri.dtsi index 310dff05910d5c919862a05e4f01d0832acf93ea..2867a138e011fc3866fda415942ebf23e8cd472b 100644 --- a/arch/arm/boot/dts/tegra30-colibri.dtsi +++ b/arch/arm/boot/dts/tegra30-colibri.dtsi @@ -849,7 +849,7 @@ touchscreen@41 { compatible = "st,stmpe811"; reg = <0x41>; - irq-gpio = <&gpio TEGRA_GPIO(V, 0) IRQ_TYPE_LEVEL_LOW>; + irq-gpio = <&gpio TEGRA_GPIO(V, 0) GPIO_ACTIVE_LOW>; interrupt-controller; id = <0>; blocks = <0x5>; diff --git a/arch/arm/boot/dts/uniphier-ld4-ref.dts b/arch/arm/boot/dts/uniphier-ld4-ref.dts index c46c2e8a10a7bd9dc4947f9af20769f010ff1b1a..e007db084787527f6feee14622f68647cd5ca015 100644 --- a/arch/arm/boot/dts/uniphier-ld4-ref.dts +++ b/arch/arm/boot/dts/uniphier-ld4-ref.dts @@ -36,11 +36,11 @@ }; ðsc { - interrupts = <1 8>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <1 8>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -56,7 +56,7 @@ }; &gpio { - xirq1 { + xirq1-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm/boot/dts/uniphier-ld4.dtsi b/arch/arm/boot/dts/uniphier-ld4.dtsi index b52957ccda0d5cf302ea827364b413fe4f7c3587..9dceff12a6338c8b12bd271b208fd9ec71dea90f 100644 --- a/arch/arm/boot/dts/uniphier-ld4.dtsi +++ b/arch/arm/boot/dts/uniphier-ld4.dtsi @@ -6,6 +6,7 @@ // Author: Masahiro Yamada #include +#include / { compatible = "socionext,uniphier-ld4"; @@ -55,7 +56,8 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>; - interrupts = <0 174 4>, <0 175 4>; + interrupts = , + ; cache-unified; cache-size = <(512 * 1024)>; cache-sets = <256>; @@ -69,7 +71,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -80,7 +82,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -91,7 +93,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -102,7 +104,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -113,7 +115,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 29 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -140,7 +142,7 @@ reg = <0x58400000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -154,7 +156,7 @@ reg = <0x58480000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -168,7 +170,7 @@ reg = <0x58500000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -182,7 +184,7 @@ reg = <0x58580000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -240,8 +242,13 @@ dmac: dma-controller@5a000000 { compatible = "socionext,uniphier-mio-dmac"; reg = <0x5a000000 0x1000>; - interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>, - <0 71 4>, <0 72 4>, <0 73 4>; + interrupts = , + , + , + , + , + , + ; clocks = <&mio_clk 7>; resets = <&mio_rst 7>; #dma-cells = <1>; @@ -251,7 +258,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a400000 0x200>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; @@ -271,7 +278,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a500000 0x200>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&mio_clk 1>; @@ -289,7 +296,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a800100 0x100>; - interrupts = <0 80 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>, @@ -303,7 +310,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a810100 0x100>; - interrupts = <0 81 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>, @@ -317,7 +324,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a820100 0x100>; - interrupts = <0 82 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>, @@ -358,14 +365,16 @@ timer@60000200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x60000200 0x20>; - interrupts = <1 11 0x104>; + interrupts = ; clocks = <&arm_timer_clk>; }; timer@60000600 { compatible = "arm,cortex-a9-twd-timer"; reg = <0x60000600 0x20>; - interrupts = <1 13 0x104>; + interrupts = ; clocks = <&arm_timer_clk>; }; @@ -407,7 +416,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm/boot/dts/uniphier-ld6b-ref.dts b/arch/arm/boot/dts/uniphier-ld6b-ref.dts index 5bc7fe11b517f8db1f9aecba818cdad4c5e22cc9..223a78b4a761e692175162720a3cb2f10cb6000a 100644 --- a/arch/arm/boot/dts/uniphier-ld6b-ref.dts +++ b/arch/arm/boot/dts/uniphier-ld6b-ref.dts @@ -40,11 +40,11 @@ }; ðsc { - interrupts = <4 8>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <4 8>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -60,7 +60,7 @@ }; &gpio { - xirq4 { + xirq4-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm/boot/dts/uniphier-pinctrl.dtsi b/arch/arm/boot/dts/uniphier-pinctrl.dtsi index c0fd029b37e5eedf19288a30401b3d730141db3f..f909ec2e5333bac640756a29484a0b23c2e427e4 100644 --- a/arch/arm/boot/dts/uniphier-pinctrl.dtsi +++ b/arch/arm/boot/dts/uniphier-pinctrl.dtsi @@ -196,11 +196,21 @@ function = "usb0"; }; + pinctrl_usb0_device: usb0-device { + groups = "usb0_device"; + function = "usb0"; + }; + pinctrl_usb1: usb1 { groups = "usb1"; function = "usb1"; }; + pinctrl_usb1_device: usb1-device { + groups = "usb1_device"; + function = "usb1"; + }; + pinctrl_usb2: usb2 { groups = "usb2"; function = "usb2"; diff --git a/arch/arm/boot/dts/uniphier-pro4-ace.dts b/arch/arm/boot/dts/uniphier-pro4-ace.dts index 27ff2b7b9d0e8a0ef50406caf880aa75abbfe7e2..6baee4410d9cb079402691f80fae7c223cfc13fb 100644 --- a/arch/arm/boot/dts/uniphier-pro4-ace.dts +++ b/arch/arm/boot/dts/uniphier-pro4-ace.dts @@ -99,3 +99,11 @@ &usb1 { status = "okay"; }; + +&ahci0 { + status = "okay"; +}; + +&ahci1 { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/uniphier-pro4-ref.dts b/arch/arm/boot/dts/uniphier-pro4-ref.dts index 3b9b61314d014dcf2b59f84ade7f766c9683b9dd..d2ce5c0398655ac8e93e467fede919ffafcd6b32 100644 --- a/arch/arm/boot/dts/uniphier-pro4-ref.dts +++ b/arch/arm/boot/dts/uniphier-pro4-ref.dts @@ -39,11 +39,11 @@ }; ðsc { - interrupts = <2 8>; + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <2 8>; + interrupts = <2 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -59,7 +59,7 @@ }; &gpio { - xirq2 { + xirq2-hog { gpio-hog; gpios = ; input; @@ -108,3 +108,11 @@ reg = <0>; }; }; + +&ahci0 { + status = "okay"; +}; + +&ahci1 { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/uniphier-pro4.dtsi b/arch/arm/boot/dts/uniphier-pro4.dtsi index a53b73ee93e90f38a6387b08848e3424002b95c2..a309e64c57c86b25d1b60b146d733205efafcc2f 100644 --- a/arch/arm/boot/dts/uniphier-pro4.dtsi +++ b/arch/arm/boot/dts/uniphier-pro4.dtsi @@ -6,6 +6,7 @@ // Author: Masahiro Yamada #include +#include / { compatible = "socionext,uniphier-pro4"; @@ -63,7 +64,8 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>; - interrupts = <0 174 4>, <0 175 4>; + interrupts = , + ; cache-unified; cache-size = <(768 * 1024)>; cache-sets = <256>; @@ -77,7 +79,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -88,7 +90,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -99,7 +101,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -110,7 +112,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -121,7 +123,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -148,7 +150,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -162,7 +164,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -176,7 +178,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -190,7 +192,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -206,7 +208,7 @@ reg = <0x58785000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 25 4>; + interrupts = ; clocks = <&peri_clk 9>; resets = <&peri_rst 9>; clock-frequency = <400000>; @@ -218,7 +220,7 @@ reg = <0x58786000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 26 4>; + interrupts = ; clocks = <&peri_clk 10>; resets = <&peri_rst 10>; clock-frequency = <400000>; @@ -274,8 +276,14 @@ dmac: dma-controller@5a000000 { compatible = "socionext,uniphier-mio-dmac"; reg = <0x5a000000 0x1000>; - interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>, - <0 71 4>, <0 72 4>, <0 73 4>, <0 74 4>; + interrupts = , + , + , + , + , + , + , + ; clocks = <&mio_clk 7>; resets = <&mio_rst 7>; #dma-cells = <1>; @@ -285,7 +293,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a400000 0x200>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; @@ -305,7 +313,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a500000 0x200>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&mio_clk 1>; @@ -323,7 +331,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a600000 0x200>; - interrupts = <0 85 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_sd1>; clocks = <&mio_clk 2>; @@ -339,7 +347,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a800100 0x100>; - interrupts = <0 80 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>, @@ -355,7 +363,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a810100 0x100>; - interrupts = <0 81 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb3>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>, @@ -376,7 +384,7 @@ compatible = "socionext,uniphier-pro4-pinctrl"; }; - usb-phy { + usb-controller { compatible = "socionext,uniphier-pro4-usb2-phy"; #address-cells = <1>; #size-cells = <0>; @@ -403,6 +411,11 @@ vbus-supply = <&usb1_vbus>; }; }; + + sg_clk: clock { + compatible = "socionext,uniphier-pro4-sg-clock"; + #clock-cells = <1>; + }; }; soc-glue@5f900000 { @@ -431,7 +444,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -446,14 +459,16 @@ timer@60000200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x60000200 0x20>; - interrupts = <1 11 0x304>; + interrupts = ; clocks = <&arm_timer_clk>; }; timer@60000600 { compatible = "arm,cortex-a9-twd-timer"; reg = <0x60000600 0x20>; - interrupts = <1 13 0x304>; + interrupts = ; clocks = <&arm_timer_clk>; }; @@ -485,7 +500,7 @@ compatible = "socionext,uniphier-pro4-ave4"; status = "disabled"; reg = <0x65000000 0x8500>; - interrupts = <0 66 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ether_rgmii>; clock-names = "gio", "ether", "ether-gb", "ether-phy"; @@ -503,12 +518,105 @@ }; }; + ahci0: sata@65600000 { + compatible = "socionext,uniphier-pro4-ahci", + "generic-ahci"; + status = "disabled"; + reg = <0x65600000 0x10000>; + interrupts = ; + clocks = <&sys_clk 12>, <&sys_clk 28>; + resets = <&sys_rst 12>, <&sys_rst 28>, <&ahci0_rst 3>; + ports-implemented = <1>; + phys = <&ahci0_phy>; + assigned-clocks = <&sg_clk 0>; + assigned-clock-rates = <25000000>; + }; + + sata-controller@65700000 { + compatible = "socionext,uniphier-pxs2-ahci-glue", + "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x65700000 0x100>; + + ahci0_rst: reset-controller@0 { + compatible = "socionext,uniphier-pro4-ahci-reset"; + reg = <0x0 0x4>; + clock-names = "gio", "link"; + clocks = <&sys_clk 12>, <&sys_clk 28>; + reset-names = "gio", "link"; + resets = <&sys_rst 12>, <&sys_rst 28>; + #reset-cells = <1>; + }; + + ahci0_phy: sata-phy@10 { + compatible = "socionext,uniphier-pro4-ahci-phy"; + reg = <0x10 0x40>; + clock-names = "link", "gio"; + clocks = <&sys_clk 28>, <&sys_clk 12>; + reset-names = "link", "gio", "phy", + "pm", "tx", "rx"; + resets = <&sys_rst 28>, <&sys_rst 12>, + <&sys_rst 30>, + <&ahci0_rst 0>, <&ahci0_rst 1>, + <&ahci0_rst 2>; + #phy-cells = <0>; + }; + }; + + ahci1: sata@65800000 { + compatible = "socionext,uniphier-pro4-ahci", + "generic-ahci"; + status = "disabled"; + reg = <0x65800000 0x10000>; + interrupts = ; + clocks = <&sys_clk 12>, <&sys_clk 29>; + resets = <&sys_rst 12>, <&sys_rst 29>, <&ahci1_rst 3>; + ports-implemented = <1>; + phys = <&ahci1_phy>; + assigned-clocks = <&sg_clk 0>; + assigned-clock-rates = <25000000>; + }; + + sata-controller@65900000 { + compatible = "socionext,uniphier-pro4-ahci-glue", + "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x65900000 0x100>; + + ahci1_rst: reset-controller@0 { + compatible = "socionext,uniphier-pro4-ahci-reset"; + reg = <0x0 0x4>; + clock-names = "gio", "link"; + clocks = <&sys_clk 12>, <&sys_clk 29>; + reset-names = "gio", "link"; + resets = <&sys_rst 12>, <&sys_rst 29>; + #reset-cells = <1>; + }; + + ahci1_phy: sata-phy@10 { + compatible = "socionext,uniphier-pro4-ahci-phy"; + reg = <0x10 0x40>; + clock-names = "link", "gio"; + clocks = <&sys_clk 29>, <&sys_clk 12>; + reset-names = "link", "gio", "phy", + "pm", "tx", "rx"; + resets = <&sys_rst 29>, <&sys_rst 12>, + <&sys_rst 30>, + <&ahci1_rst 0>, <&ahci1_rst 1>, + <&ahci1_rst 2>; + #phy-cells = <0>; + }; + }; + usb0: usb@65a00000 { compatible = "socionext,uniphier-dwc3", "snps,dwc3"; status = "disabled"; reg = <0x65a00000 0xcd00>; interrupt-names = "host", "peripheral"; - interrupts = <0 134 4>, <0 135 4>; + interrupts = , + ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; clock-names = "ref", "bus_early", "suspend"; @@ -518,7 +626,7 @@ dr_mode = "host"; }; - usb-glue@65b00000 { + usb-controller@65b00000 { compatible = "socionext,uniphier-pro4-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -561,7 +669,8 @@ status = "disabled"; reg = <0x65c00000 0xcd00>; interrupt-names = "host", "peripheral"; - interrupts = <0 137 4>, <0 138 4>; + interrupts = , + ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; clock-names = "ref", "bus_early", "suspend"; @@ -571,7 +680,7 @@ dr_mode = "host"; }; - usb-glue@65d00000 { + usb-controller@65d00000 { compatible = "socionext,uniphier-pro4-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -605,7 +714,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm/boot/dts/uniphier-pro5.dtsi b/arch/arm/boot/dts/uniphier-pro5.dtsi index 3525125832dd1f823ce4e0f95e9b696224af7cc3..100edd7438d8b7bb609990470f1675b49e8daedd 100644 --- a/arch/arm/boot/dts/uniphier-pro5.dtsi +++ b/arch/arm/boot/dts/uniphier-pro5.dtsi @@ -5,6 +5,8 @@ // Copyright (C) 2015-2016 Socionext Inc. // Author: Masahiro Yamada +#include + / { compatible = "socionext,uniphier-pro5"; #address-cells = <1>; @@ -135,7 +137,8 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c0000 0x2000>, <0x503c0100 0x8>, <0x506c0000 0x400>; - interrupts = <0 190 4>, <0 191 4>; + interrupts = , + ; cache-unified; cache-size = <(2 * 1024 * 1024)>; cache-sets = <512>; @@ -148,7 +151,8 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c8000 0x2000>, <0x503c8100 0x8>, <0x506c8000 0x400>; - interrupts = <0 174 4>, <0 175 4>; + interrupts = , + ; cache-unified; cache-size = <(2 * 1024 * 1024)>; cache-sets = <512>; @@ -162,7 +166,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -175,7 +179,7 @@ reg = <0x54006100 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 216 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; clocks = <&peri_clk 11>; /* common with spi0 */ @@ -186,7 +190,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -197,7 +201,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -208,7 +212,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -219,7 +223,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -246,7 +250,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -260,7 +264,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -274,7 +278,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -288,7 +292,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -304,7 +308,7 @@ reg = <0x58785000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 25 4>; + interrupts = ; clocks = <&peri_clk 9>; resets = <&peri_rst 9>; clock-frequency = <400000>; @@ -316,7 +320,7 @@ reg = <0x58786000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 26 4>; + interrupts = ; clocks = <&peri_clk 10>; resets = <&peri_rst 10>; clock-frequency = <400000>; @@ -415,7 +419,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -430,14 +434,16 @@ timer@60000200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x60000200 0x20>; - interrupts = <1 11 0x304>; + interrupts = ; clocks = <&arm_timer_clk>; }; timer@60000600 { compatible = "arm,cortex-a9-twd-timer"; reg = <0x60000600 0x20>; - interrupts = <1 13 0x304>; + interrupts = ; clocks = <&arm_timer_clk>; }; @@ -470,7 +476,7 @@ status = "disabled"; reg = <0x65a00000 0xcd00>; interrupt-names = "host"; - interrupts = <0 134 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; clock-names = "ref", "bus_early", "suspend"; @@ -480,7 +486,7 @@ dr_mode = "host"; }; - usb-glue@65b00000 { + usb-controller@65b00000 { compatible = "socionext,uniphier-pro5-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -534,7 +540,7 @@ status = "disabled"; reg = <0x65c00000 0xcd00>; interrupt-names = "host"; - interrupts = <0 137 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb2>; clock-names = "ref", "bus_early", "suspend"; @@ -544,7 +550,7 @@ dr_mode = "host"; }; - usb-glue@65d00000 { + usb-controller@65d00000 { compatible = "socionext,uniphier-pro5-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -614,8 +620,7 @@ }; pcie_ep: pcie-ep@66000000 { - compatible = "socionext,uniphier-pro5-pcie-ep", - "snps,dw-pcie-ep"; + compatible = "socionext,uniphier-pro5-pcie-ep"; status = "disabled"; reg-names = "dbi", "dbi2", "link", "addr_space"; reg = <0x66000000 0x1000>, <0x66001000 0x1000>, @@ -650,7 +655,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; @@ -663,7 +668,7 @@ compatible = "socionext,uniphier-sd-v3.1"; status = "disabled"; reg = <0x68400000 0x800>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&sd_clk 1>; @@ -679,7 +684,7 @@ compatible = "socionext,uniphier-sd-v3.1"; status = "disabled"; reg = <0x68800000 0x800>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; diff --git a/arch/arm/boot/dts/uniphier-pxs2-gentil.dts b/arch/arm/boot/dts/uniphier-pxs2-gentil.dts index 759384b606631ee0c361faa656d50bef2e16395c..5f18b926c50a97e22aee50069f3b15842493aec1 100644 --- a/arch/arm/boot/dts/uniphier-pxs2-gentil.dts +++ b/arch/arm/boot/dts/uniphier-pxs2-gentil.dts @@ -99,3 +99,7 @@ &usb1 { status = "okay"; }; + +&ahci { + status = "okay"; +}; diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi index 03301ddb3403ad855a778fade46ab9f71dfae245..ca4dccf56a67632ba45444c24a144f3cbe5cf575 100644 --- a/arch/arm/boot/dts/uniphier-pxs2.dtsi +++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi @@ -6,6 +6,7 @@ // Author: Masahiro Yamada #include +#include #include / { @@ -161,7 +162,10 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c0000 0x2000>, <0x503c0100 0x8>, <0x506c0000 0x400>; - interrupts = <0 174 4>, <0 175 4>, <0 190 4>, <0 191 4>; + interrupts = , + , + , + ; cache-unified; cache-size = <(1280 * 1024)>; cache-sets = <512>; @@ -175,7 +179,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -188,7 +192,7 @@ reg = <0x54006100 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 216 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; clocks = <&peri_clk 12>; @@ -199,7 +203,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -210,7 +214,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -221,7 +225,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -232,7 +236,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -259,7 +263,7 @@ audio@56000000 { compatible = "socionext,uniphier-pxs2-aio"; reg = <0x56000000 0x80000>; - interrupts = <0 144 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ain1>, <&pinctrl_ain2>, @@ -317,7 +321,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -331,7 +335,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -345,7 +349,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -359,7 +363,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -373,7 +377,7 @@ reg = <0x58784000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 45 4>; + interrupts = ; clocks = <&peri_clk 8>; resets = <&peri_rst 8>; clock-frequency = <400000>; @@ -385,7 +389,7 @@ reg = <0x58785000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 25 4>; + interrupts = ; clocks = <&peri_clk 9>; resets = <&peri_rst 9>; clock-frequency = <400000>; @@ -397,7 +401,7 @@ reg = <0x58786000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 26 4>; + interrupts = ; clocks = <&peri_clk 10>; resets = <&peri_rst 10>; clock-frequency = <400000>; @@ -454,7 +458,7 @@ compatible = "socionext,uniphier-sd-v3.1.1"; status = "disabled"; reg = <0x5a000000 0x800>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&sd_clk 1>; @@ -470,7 +474,7 @@ compatible = "socionext,uniphier-sd-v3.1.1"; status = "disabled"; reg = <0x5a400000 0x800>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; @@ -515,7 +519,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -530,14 +534,16 @@ timer@60000200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x60000200 0x20>; - interrupts = <1 11 0xf04>; + interrupts = ; clocks = <&arm_timer_clk>; }; timer@60000600 { compatible = "arm,cortex-a9-twd-timer"; reg = <0x60000600 0x20>; - interrupts = <1 13 0xf04>; + interrupts = ; clocks = <&arm_timer_clk>; }; @@ -564,9 +570,9 @@ #reset-cells = <1>; }; - pvtctl: pvtctl { + pvtctl: thermal-sensor { compatible = "socionext,uniphier-pxs2-thermal"; - interrupts = <0 3 4>; + interrupts = ; #thermal-sensor-cells = <0>; socionext,tmod-calibration = <0x0f86 0x6844>; }; @@ -576,7 +582,7 @@ compatible = "socionext,uniphier-pxs2-ave4"; status = "disabled"; reg = <0x65000000 0x8500>; - interrupts = <0 66 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ether_rgmii>; clock-names = "ether"; @@ -593,12 +599,52 @@ }; }; + ahci: sata@65600000 { + compatible = "socionext,uniphier-pxs2-ahci", + "generic-ahci"; + status = "disabled"; + reg = <0x65600000 0x10000>; + interrupts = ; + clocks = <&sys_clk 28>; + resets = <&sys_rst 28>, <&ahci_rst 0>; + ports-implemented = <1>; + phys = <&ahci_phy>; + }; + + sata-controller@65700000 { + compatible = "socionext,uniphier-pxs2-ahci-glue", + "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x65700000 0x100>; + + ahci_rst: reset-controller@0 { + compatible = "socionext,uniphier-pxs2-ahci-reset"; + reg = <0x0 0x4>; + clock-names = "link"; + clocks = <&sys_clk 28>; + reset-names = "link"; + resets = <&sys_rst 28>; + #reset-cells = <1>; + }; + + ahci_phy: sata-phy@10 { + compatible = "socionext,uniphier-pxs2-ahci-phy"; + reg = <0x10 0x10>; + clock-names = "link"; + clocks = <&sys_clk 28>; + reset-names = "link", "phy"; + resets = <&sys_rst 28>, <&sys_rst 30>; + #phy-cells = <0>; + }; + }; + usb0: usb@65a00000 { compatible = "socionext,uniphier-dwc3", "snps,dwc3"; status = "disabled"; reg = <0x65a00000 0xcd00>; interrupt-names = "dwc_usb3"; - interrupts = <0 134 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb2>; clock-names = "ref", "bus_early", "suspend"; @@ -609,7 +655,7 @@ dr_mode = "host"; }; - usb-glue@65b00000 { + usb-controller@65b00000 { compatible = "socionext,uniphier-pxs2-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -694,7 +740,7 @@ status = "disabled"; reg = <0x65c00000 0xcd00>; interrupt-names = "dwc_usb3"; - interrupts = <0 137 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb3>; clock-names = "ref", "bus_early", "suspend"; @@ -704,7 +750,7 @@ dr_mode = "host"; }; - usb-glue@65d00000 { + usb-controller@65d00000 { compatible = "socionext,uniphier-pxs2-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -780,7 +826,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm/boot/dts/uniphier-sld8-ref.dts b/arch/arm/boot/dts/uniphier-sld8-ref.dts index 6db949ec741112530636247801e03cceb76df43e..2446f9e153608b88d6579b211e9141800efec4bb 100644 --- a/arch/arm/boot/dts/uniphier-sld8-ref.dts +++ b/arch/arm/boot/dts/uniphier-sld8-ref.dts @@ -36,11 +36,11 @@ }; ðsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -56,7 +56,7 @@ }; &gpio { - xirq0 { + xirq0-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm/boot/dts/uniphier-sld8.dtsi b/arch/arm/boot/dts/uniphier-sld8.dtsi index 96a766deb8d118fd629d4ab62008c17f138d8eef..67b12dfe513be943d359fc041b16b622fb75e301 100644 --- a/arch/arm/boot/dts/uniphier-sld8.dtsi +++ b/arch/arm/boot/dts/uniphier-sld8.dtsi @@ -6,6 +6,7 @@ // Author: Masahiro Yamada #include +#include / { compatible = "socionext,uniphier-sld8"; @@ -55,7 +56,8 @@ compatible = "socionext,uniphier-system-cache"; reg = <0x500c0000 0x2000>, <0x503c0100 0x4>, <0x506c0000 0x400>; - interrupts = <0 174 4>, <0 175 4>; + interrupts = , + ; cache-unified; cache-size = <(256 * 1024)>; cache-sets = <256>; @@ -69,7 +71,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -80,7 +82,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -91,7 +93,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -102,7 +104,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -113,7 +115,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 29 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -144,7 +146,7 @@ reg = <0x58400000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -158,7 +160,7 @@ reg = <0x58480000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -172,7 +174,7 @@ reg = <0x58500000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -186,7 +188,7 @@ reg = <0x58580000 0x40>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 1>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -244,8 +246,13 @@ dmac: dma-controller@5a000000 { compatible = "socionext,uniphier-mio-dmac"; reg = <0x5a000000 0x1000>; - interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>, - <0 71 4>, <0 72 4>, <0 73 4>; + interrupts = , + , + , + , + , + , + ; clocks = <&mio_clk 7>; resets = <&mio_rst 7>; #dma-cells = <1>; @@ -255,7 +262,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a400000 0x200>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; @@ -275,7 +282,7 @@ compatible = "socionext,uniphier-sd-v2.91"; status = "disabled"; reg = <0x5a500000 0x200>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&mio_clk 1>; @@ -293,7 +300,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a800100 0x100>; - interrupts = <0 80 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>, @@ -307,7 +314,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a810100 0x100>; - interrupts = <0 81 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>, @@ -321,7 +328,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a820100 0x100>; - interrupts = <0 82 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>, @@ -362,14 +369,16 @@ timer@60000200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x60000200 0x20>; - interrupts = <1 11 0x104>; + interrupts = ; clocks = <&arm_timer_clk>; }; timer@60000600 { compatible = "arm,cortex-a9-twd-timer"; reg = <0x60000600 0x20>; - interrupts = <1 13 0x104>; + interrupts = ; clocks = <&arm_timer_clk>; }; @@ -411,7 +420,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm/boot/dts/uniphier-support-card.dtsi b/arch/arm/boot/dts/uniphier-support-card.dtsi index 444802fee9fb020e88479f6b046bee15178e8c9f..97e7d5db8eb853aaf1e2fb13e3f4dd166c063cde 100644 --- a/arch/arm/boot/dts/uniphier-support-card.dtsi +++ b/arch/arm/boot/dts/uniphier-support-card.dtsi @@ -8,13 +8,13 @@ &system_bus { status = "okay"; ranges = <1 0x00000000 0x42000000 0x02000000>; - interrupt-parent = <&gpio>; ethsc: ethernet@1,1f00000 { compatible = "smsc,lan9118", "smsc,lan9115"; reg = <1 0x01f00000 0x1000>; phy-mode = "mii"; reg-io-width = <4>; + interrupt-parent = <&gpio>; }; serialsc: serial@1,1fb0000 { @@ -22,5 +22,6 @@ reg = <1 0x01fb0000 0x20>; clock-frequency = <12288000>; reg-shift = <1>; + interrupt-parent = <&gpio>; }; }; diff --git a/arch/arm/boot/dts/versatile-ab.dts b/arch/arm/boot/dts/versatile-ab.dts index 79f7cc2412824fb680cdec15fad5af8ae406a3c8..a520615f4d8dbc64bee0589b4d026cc164a1d588 100644 --- a/arch/arm/boot/dts/versatile-ab.dts +++ b/arch/arm/boot/dts/versatile-ab.dts @@ -391,7 +391,7 @@ reg = <0x101f4000 0x1000>; interrupts = <11>; clocks = <&xtal24mhz>, <&pclk>; - clock-names = "SSPCLK", "apb_pclk"; + clock-names = "sspclk", "apb_pclk"; }; fpga { diff --git a/arch/arm/boot/dts/vf610-bk4.dts b/arch/arm/boot/dts/vf610-bk4.dts index 830c85476b3d8a650424afb042e50f832ebbe619..551a4c3ff4fa704cd3b263adfc546d02b1cefe2d 100644 --- a/arch/arm/boot/dts/vf610-bk4.dts +++ b/arch/arm/boot/dts/vf610-bk4.dts @@ -61,7 +61,7 @@ regulator-max-microvolt = <3300000>; }; - spi-gpio { + spi { compatible = "spi-gpio"; pinctrl-0 = <&pinctrl_gpio_spi>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/vf610-pinfunc.h b/arch/arm/boot/dts/vf610-pinfunc.h index f1e5a7cf58a942229310ce515663e590af386a64..b7b7322a2d1bcf172e87392c10c7454d05ffb026 100644 --- a/arch/arm/boot/dts/vf610-pinfunc.h +++ b/arch/arm/boot/dts/vf610-pinfunc.h @@ -420,7 +420,7 @@ #define VF610_PAD_PTD29__FTM3_CH2 0x104 0x000 ALT4 0x0 #define VF610_PAD_PTD29__DSPI2_SIN 0x104 0x000 ALT5 0x0 #define VF610_PAD_PTD29__DEBUG_OUT11 0x104 0x000 ALT7 0x0 -#define VF610_PAD_PTD28__GPIO_66 0x108 0x000 ALT0 0x0 +#define VF610_PAD_PTD28__GPIO_66 0x108 0x000 ALT0 0x0 #define VF610_PAD_PTD28__FB_AD28 0x108 0x000 ALT1 0x0 #define VF610_PAD_PTD28__NF_IO12 0x108 0x000 ALT2 0x0 #define VF610_PAD_PTD28__I2C2_SCL 0x108 0x34C ALT3 0x1 @@ -802,5 +802,55 @@ #define VF610_PAD_PTE28__EWM_OUT 0x214 0x000 ALT7 0x0 #define VF610_PAD_PTA7__GPIO_134 0x218 0x000 ALT0 0x0 #define VF610_PAD_PTA7__VIU_PIX_CLK 0x218 0x3AC ALT1 0x1 +#define VF610_PAD_DDR_RESETB 0x21c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A15__DDR_A_15 0x220 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A14__DDR_A_14 0x224 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A13__DDR_A_13 0x228 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A12__DDR_A_12 0x22c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A11__DDR_A_11 0x230 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A10__DDR_A_10 0x234 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A9__DDR_A_9 0x238 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A8__DDR_A_8 0x23c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A7__DDR_A_7 0x240 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A6__DDR_A_6 0x244 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A5__DDR_A_5 0x248 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A4__DDR_A_4 0x24c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A3__DDR_A_3 0x250 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A2__DDR_A_2 0x254 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A1__DDR_A_1 0x258 0x000 ALT0 0x0 +#define VF610_PAD_DDR_A0__DDR_A_0 0x25c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_BA2__DDR_BA_2 0x260 0x000 ALT0 0x0 +#define VF610_PAD_DDR_BA1__DDR_BA_1 0x264 0x000 ALT0 0x0 +#define VF610_PAD_DDR_BA0__DDR_BA_0 0x268 0x000 ALT0 0x0 +#define VF610_PAD_DDR_CAS__DDR_CAS_B 0x26c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_CKE__DDR_CKE_0 0x270 0x000 ALT0 0x0 +#define VF610_PAD_DDR_CLK__DDR_CLK_0 0x274 0x000 ALT0 0x0 +#define VF610_PAD_DDR_CS__DDR_CS_B_0 0x278 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D15__DDR_D_15 0x27c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D14__DDR_D_14 0x280 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D13__DDR_D_13 0x284 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D12__DDR_D_12 0x288 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D11__DDR_D_11 0x28c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D10__DDR_D_10 0x290 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D9__DDR_D_9 0x294 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D8__DDR_D_8 0x298 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D7__DDR_D_7 0x29c 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D6__DDR_D_6 0x2a0 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D5__DDR_D_5 0x2a4 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D4__DDR_D_4 0x2a8 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D3__DDR_D_3 0x2ac 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D2__DDR_D_2 0x2b0 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D1__DDR_D_1 0x2b4 0x000 ALT0 0x0 +#define VF610_PAD_DDR_D0__DDR_D_0 0x2b8 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DQM1__DDR_DQM_1 0x2bc 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DQM0__DDR_DQM_0 0x2c0 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DQS1__DDR_DQS_1 0x2c4 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DQS0__DDR_DQS_0 0x2c8 0x000 ALT0 0x0 +#define VF610_PAD_DDR_RAS__DDR_RAS_B 0x2cc 0x000 ALT0 0x0 +#define VF610_PAD_DDR_WE__DDR_WE_B 0x2d0 0x000 ALT0 0x0 +#define VF610_PAD_DDR_ODT1__DDR_ODT_0 0x2d4 0x000 ALT0 0x0 +#define VF610_PAD_DDR_ODT0__DDR_ODT_1 0x2d8 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DDRBYTE1__DDR_DDRBYTE1 0x2dc 0x000 ALT0 0x0 +#define VF610_PAD_DDR_DDRBYTE2__DDR_DDRBYTE2 0x2e0 0x000 ALT0 0x0 #endif diff --git a/arch/arm/boot/dts/vf610-twr.dts b/arch/arm/boot/dts/vf610-twr.dts index dbb5ffcdcec4388fc5173f213c1694f86f401893..6c246d5aa0328b8f38998ac8bda5e377d83271fe 100644 --- a/arch/arm/boot/dts/vf610-twr.dts +++ b/arch/arm/boot/dts/vf610-twr.dts @@ -169,7 +169,7 @@ VDDA-supply = <®_3p3v>; VDDIO-supply = <®_3p3v>; clocks = <&clks VF610_CLK_SAI2>; - }; + }; }; &iomuxc { diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts index 1f9686c33a84547d5bf420d8806ec2401c5ed6d7..42ed4a04a12e250b3e4d3421d221658397560ead 100644 --- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts +++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts @@ -288,7 +288,7 @@ }; }; - spi0 { + spi-0 { compatible = "spi-gpio"; pinctrl-0 = <&pinctrl_gpio_spi0>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/vf610.dtsi b/arch/arm/boot/dts/vf610.dtsi index 956182d08e74e3f0f11b58f75bc38830d7fd9cab..2fba923821d0253e0ce7cec832d78e91b9f8d4ee 100644 --- a/arch/arm/boot/dts/vf610.dtsi +++ b/arch/arm/boot/dts/vf610.dtsi @@ -2,7 +2,6 @@ // // Copyright 2013 Freescale Semiconductor, Inc. - #include "vf500.dtsi" &a5_cpu { diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig index 67feb060bb197da6ee671dee56326000cfc76b0a..a5c65d28ca63c67172a74d81d92a2092e72ab337 100644 --- a/arch/arm/configs/aspeed_g4_defconfig +++ b/arch/arm/configs/aspeed_g4_defconfig @@ -7,6 +7,7 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_CGROUPS=y +CONFIG_NAMESPACES=y CONFIG_BLK_DEV_INITRD=y # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZO is not set @@ -23,12 +24,9 @@ CONFIG_VMSPLIT_2G=y CONFIG_AEABI=y CONFIG_UACCESS_WITH_MEMCPY=y # CONFIG_ATAGS is not set -CONFIG_ARM_APPENDED_DTB=y -CONFIG_ARM_ATAG_DTB_COMPAT=y CONFIG_KEXEC=y CONFIG_JUMP_LABEL=y CONFIG_STRICT_KERNEL_RWX=y -# CONFIG_BLK_DEV_BSG is not set # CONFIG_BLK_DEBUG_FS is not set # CONFIG_MQ_IOSCHED_DEADLINE is not set # CONFIG_MQ_IOSCHED_KYBER is not set @@ -47,7 +45,11 @@ CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_SYN_COOKIES=y # CONFIG_INET_DIAG is not set +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y # CONFIG_IPV6_SIT is not set +CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_NETFILTER=y # CONFIG_NETFILTER_ADVANCED is not set CONFIG_VLAN_8021Q=y @@ -66,6 +68,7 @@ CONFIG_MTD_UBI=y CONFIG_MTD_UBI_FASTMAP=y CONFIG_MTD_UBI_BLOCK=y CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=y CONFIG_EEPROM_AT24=y CONFIG_NETDEVICES=y CONFIG_NETCONSOLE=y @@ -145,6 +148,7 @@ CONFIG_SENSORS_ASPEED=y CONFIG_SENSORS_IIO_HWMON=y CONFIG_SENSORS_LM75=y CONFIG_SENSORS_NCT7904=y +CONFIG_SENSORS_OCC_P8_I2C=y CONFIG_PMBUS=y CONFIG_SENSORS_ADM1275=y CONFIG_SENSORS_IBM_CFFPS=y diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig index 247ab72b25900bd64c592da20efd5a5170c5e930..c7c11cbaa39d4d8ebd3528e32e8c0c37c031a042 100644 --- a/arch/arm/configs/aspeed_g5_defconfig +++ b/arch/arm/configs/aspeed_g5_defconfig @@ -7,6 +7,7 @@ CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_CGROUPS=y +CONFIG_NAMESPACES=y CONFIG_BLK_DEV_INITRD=y # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZO is not set @@ -33,7 +34,6 @@ CONFIG_VFP=y CONFIG_NEON=y CONFIG_KERNEL_MODE_NEON=y CONFIG_JUMP_LABEL=y -# CONFIG_BLK_DEV_BSG is not set # CONFIG_BLK_DEBUG_FS is not set # CONFIG_MQ_IOSCHED_DEADLINE is not set # CONFIG_MQ_IOSCHED_KYBER is not set @@ -65,6 +65,7 @@ CONFIG_NETFILTER=y # CONFIG_NETFILTER_ADVANCED is not set CONFIG_VLAN_8021Q=y CONFIG_NET_NCSI=y +CONFIG_MCTP=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y @@ -82,6 +83,8 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_NBD=y CONFIG_EEPROM_AT24=y CONFIG_EEPROM_AT25=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y CONFIG_NETDEVICES=y CONFIG_NETCONSOLE=y # CONFIG_NET_VENDOR_ALACRITECH is not set @@ -119,6 +122,8 @@ CONFIG_FTGMAC100=y # CONFIG_NET_VENDOR_WIZNET is not set CONFIG_BROADCOM_PHY=y CONFIG_REALTEK_PHY=y +CONFIG_MCTP_SERIAL=y +CONFIG_MCTP_TRANSPORT_I2C=y # CONFIG_USB_NET_DRIVERS is not set # CONFIG_WLAN is not set CONFIG_INPUT_EVDEV=y @@ -146,6 +151,8 @@ CONFIG_IPMI_KCS_BMC_CDEV_IPMI=y CONFIG_IPMI_KCS_BMC_SERIO=y CONFIG_ASPEED_BT_IPMI_BMC=y CONFIG_HW_RANDOM_TIMERIOMEM=y +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS_I2C=y # CONFIG_I2C_COMPAT is not set CONFIG_I2C_CHARDEV=y CONFIG_I2C_MUX_GPIO=y @@ -180,6 +187,7 @@ CONFIG_SENSORS_IR38064=y CONFIG_SENSORS_ISL68137=y CONFIG_SENSORS_LM25066=y CONFIG_SENSORS_MAX31785=y +CONFIG_SENSORS_MP5023=y CONFIG_SENSORS_UCD9000=y CONFIG_SENSORS_UCD9200=y CONFIG_SENSORS_SBTSI=y @@ -200,6 +208,9 @@ CONFIG_USB_DYNAMIC_MINORS=y CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_ROOT_HUB_TT=y CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_STORAGE=y +CONFIG_USB_SERIAL=y +CONFIG_USB_SERIAL_PL2303=y CONFIG_USB_GADGET=y CONFIG_USB_ASPEED_VHUB=y CONFIG_USB_CONFIGFS=y @@ -243,9 +254,13 @@ CONFIG_FSI_MASTER_ASPEED=y CONFIG_FSI_SCOM=y CONFIG_FSI_SBEFIFO=y CONFIG_FSI_OCC=y +CONFIG_PECI=y +CONFIG_PECI_CPU=y +CONFIG_PECI_ASPEED=y CONFIG_EXT4_FS=y CONFIG_FANOTIFY=y CONFIG_OVERLAY_FS=y +CONFIG_VFAT_FS=y CONFIG_TMPFS=y CONFIG_JFFS2_FS=y # CONFIG_JFFS2_FS_WRITEBUFFER is not set @@ -261,6 +276,9 @@ CONFIG_PSTORE_PMSG=y CONFIG_PSTORE_FTRACE=y CONFIG_PSTORE_RAM=y # CONFIG_NETWORK_FILESYSTEMS is not set +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +# CONFIG_SECURITYFS is not set CONFIG_HARDENED_USERCOPY=y CONFIG_FORTIFY_SOURCE=y CONFIG_CRYPTO_HMAC=y diff --git a/arch/arm/configs/assabet_defconfig b/arch/arm/configs/assabet_defconfig index 801383e4135d7e624579c2aeb8c0ef1732c1b311..8ba8eb7a4adf2169143ddb4f700149b22037fc5e 100644 --- a/arch/arm/configs/assabet_defconfig +++ b/arch/arm/configs/assabet_defconfig @@ -1,6 +1,8 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_ASSABET=y CONFIG_CMDLINE="mem=32M console=ttySA0,38400n8 initrd=0xc0800000,3M root=/dev/ram" diff --git a/arch/arm/configs/at91_dt_defconfig b/arch/arm/configs/at91_dt_defconfig index da90ce9cd42e1b6c5af2976e6dbd9f332fd3d126..9ea08337b174cd2443525798fb3bff11d197ebd6 100644 --- a/arch/arm/configs/at91_dt_defconfig +++ b/arch/arm/configs/at91_dt_defconfig @@ -40,7 +40,6 @@ CONFIG_IP_PNP_RARP=y # CONFIG_INET_DIAG is not set CONFIG_IPV6_SIT_6RD=y CONFIG_CAN=y -CONFIG_CAN_AT91=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_DEVTMPFS=y @@ -81,6 +80,7 @@ CONFIG_DM9000=y # CONFIG_NET_VENDOR_STMICRO is not set CONFIG_DAVICOM_PHY=y CONFIG_MICREL_PHY=y +CONFIG_CAN_AT91=y CONFIG_LIBERTAS=m CONFIG_LIBERTAS_SDIO=m CONFIG_LIBERTAS_SPI=m @@ -196,7 +196,6 @@ CONFIG_RTC_DRV_AT91SAM9=y CONFIG_DMADEVICES=y CONFIG_AT_HDMAC=y CONFIG_AT_XDMAC=y -CONFIG_MICROCHIP_PIT64B=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_IIO=y CONFIG_AT91_ADC=y diff --git a/arch/arm/configs/badge4_defconfig b/arch/arm/configs/badge4_defconfig index 506f3378da07f53f7516cea86c95251480facf4f..337e5c9718ae03162773ad899a65b5a96869dd69 100644 --- a/arch/arm/configs/badge4_defconfig +++ b/arch/arm/configs/badge4_defconfig @@ -1,12 +1,13 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_BADGE4=y CONFIG_UNUSED_BOARD_FILES=y CONFIG_CMDLINE="init=/linuxrc root=/dev/mtdblock3" CONFIG_CPU_FREQ_GOV_PERFORMANCE=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=m CONFIG_MODULES=y CONFIG_MODVERSIONS=y CONFIG_PARTITION_ADVANCED=y diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig index c4d2e2334b6e621a7da186900f8223fce428d035..a51babd178c2609f27770858cd5191c23cd8cc63 100644 --- a/arch/arm/configs/bcm2835_defconfig +++ b/arch/arm/configs/bcm2835_defconfig @@ -1,6 +1,6 @@ # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT_VOLUNTARY=y CONFIG_BSD_PROCESS_ACCT=y diff --git a/arch/arm/configs/cerfcube_defconfig b/arch/arm/configs/cerfcube_defconfig index 7e6f7dfa3023126b8ddd135df8d4a227fa96cfc3..9ada868e2648b86de20e1893fd1dd83d76ef8255 100644 --- a/arch/arm/configs/cerfcube_defconfig +++ b/arch/arm/configs/cerfcube_defconfig @@ -1,6 +1,8 @@ CONFIG_SYSVIPC=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_LOG_BUF_SHIFT=14 +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_CERF=y CONFIG_SA1100_CERF_FLASH_16MB=y diff --git a/arch/arm/configs/cm_x300_defconfig b/arch/arm/configs/cm_x300_defconfig index 31f41159bef2ed1f4bd88a4a4c5510b326c68b27..95144e380b4bfa3695f19869bf7cdc5cd545618a 100644 --- a/arch/arm/configs/cm_x300_defconfig +++ b/arch/arm/configs/cm_x300_defconfig @@ -1,7 +1,7 @@ CONFIG_LOCALVERSION="-cm-x300" # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=18 diff --git a/arch/arm/configs/collie_defconfig b/arch/arm/configs/collie_defconfig index d35cc59ce8477e031af93336ae648b50e8e2ec84..2a2d2cb3ce2ea76fd14f0a254d4006cb7c73ea7e 100644 --- a/arch/arm/configs/collie_defconfig +++ b/arch/arm/configs/collie_defconfig @@ -5,6 +5,8 @@ CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_BASE_FULL is not set # CONFIG_EPOLL is not set +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_COLLIE=y CONFIG_CMDLINE="noinitrd root=/dev/mtdblock2 rootfstype=jffs2 fbcon=rotate:1" diff --git a/arch/arm/configs/corgi_defconfig b/arch/arm/configs/corgi_defconfig index 1f137f74050f242bca062ccbf474b1b421efeb6a..df84640f4f57efdad28c0d54501eec1482e7daea 100644 --- a/arch/arm/configs/corgi_defconfig +++ b/arch/arm/configs/corgi_defconfig @@ -16,7 +16,6 @@ CONFIG_MACH_HUSKY=y CONFIG_UNUSED_BOARD_FILES=y CONFIG_CMDLINE="console=ttyS0,115200n8 console=tty1 noinitrd root=/dev/mtdblock2 rootfstype=jffs2 debug" CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=m CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig index fc71a03a9c8c6ddcfaa1710ae779e5e7c03ce1e0..821d966c95a5fe2f9a4b9a1e686d49580a394fc6 100644 --- a/arch/arm/configs/davinci_all_defconfig +++ b/arch/arm/configs/davinci_all_defconfig @@ -1,6 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y diff --git a/arch/arm/configs/dove_defconfig b/arch/arm/configs/dove_defconfig index 16ed5c110e8d105f2ed7fbbe200121395c610f31..ff37f46c82fb5cf079b7dfa3258ef0fba1dcd72f 100644 --- a/arch/arm/configs/dove_defconfig +++ b/arch/arm/configs/dove_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y @@ -116,7 +116,6 @@ CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=y CONFIG_NLS_UTF8=y CONFIG_TIMER_STATS=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_CRYPTO_NULL=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m @@ -136,6 +135,7 @@ CONFIG_CRC_CCITT=y CONFIG_LIBCRC32C=y CONFIG_PRINTK_TIME=y # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index 1ce74f46e114f1d55b15bd769f636bf87b27f197..31e8e0c0ee1b77c903aa0a60a2935e4997a83ee4 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_CGROUPS=y @@ -32,7 +32,6 @@ CONFIG_KERNEL_MODE_NEON=y CONFIG_PM_DEBUG=y CONFIG_PM_ADVANCED_DEBUG=y CONFIG_ENERGY_MODEL=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m @@ -374,8 +373,8 @@ CONFIG_FONTS=y CONFIG_FONT_7x14=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y CONFIG_SOFTLOCKUP_DETECTOR=y diff --git a/arch/arm/configs/ezx_defconfig b/arch/arm/configs/ezx_defconfig index 1a41391d7367e3268cc4dc2d873ab7b66df522c8..ef7b0a0aee3a55a93e02e38ce4cf14958f53dce8 100644 --- a/arch/arm/configs/ezx_defconfig +++ b/arch/arm/configs/ezx_defconfig @@ -1,7 +1,7 @@ CONFIG_LOCALVERSION="-ezx200910312315" # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 @@ -25,7 +25,6 @@ CONFIG_CPU_FREQ_GOV_ONDEMAND=m CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m CONFIG_CPU_IDLE=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=m CONFIG_PM=y CONFIG_APM_EMULATION=y CONFIG_MODULES=y diff --git a/arch/arm/configs/footbridge_defconfig b/arch/arm/configs/footbridge_defconfig index 504070812ad0ef03a8bbb44a9c194baad67170ee..87c489337d0eb04032bc1cfad9d5b92c151d7e52 100644 --- a/arch/arm/configs/footbridge_defconfig +++ b/arch/arm/configs/footbridge_defconfig @@ -4,12 +4,13 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_HOTPLUG is not set +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_FOOTBRIDGE=y CONFIG_ARCH_EBSA285_HOST=y CONFIG_ARCH_NETWINDER=y CONFIG_FPE_NWFPE=y CONFIG_FPE_NWFPE_XP=y -CONFIG_BINFMT_AOUT=y CONFIG_MODULES=y CONFIG_PARTITION_ADVANCED=y CONFIG_ACORN_PARTITION=y diff --git a/arch/arm/configs/h3600_defconfig b/arch/arm/configs/h3600_defconfig index 5bd1ec539610bc240b905177b83fc82b7383415d..4e272875c7978bf05846fb2ef7bbf3eccb54c253 100644 --- a/arch/arm/configs/h3600_defconfig +++ b/arch/arm/configs/h3600_defconfig @@ -4,6 +4,8 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_H3600=y # CONFIG_CPU_FREQ_STAT is not set diff --git a/arch/arm/configs/hackkit_defconfig b/arch/arm/configs/hackkit_defconfig index b9327b2eacd3c41428703dff5b015843b871854c..3c91a851fd08de385e8e9f35dea25f461376c1ae 100644 --- a/arch/arm/configs/hackkit_defconfig +++ b/arch/arm/configs/hackkit_defconfig @@ -1,13 +1,14 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_HACKKIT=y CONFIG_UNUSED_BOARD_FILES=y CONFIG_CMDLINE="console=ttySA0,115200 root=/dev/ram0 initrd=0xc0400000,8M init=/rootshell" CONFIG_CPU_FREQ_GOV_PERFORMANCE=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_MODULES=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/hisi_defconfig b/arch/arm/configs/hisi_defconfig index 1db5356b1ccd91d774277b21608807a1a3cc8acc..0376a65e8bc1a3e2b650a24ce1f823cadf7168b8 100644 --- a/arch/arm/configs/hisi_defconfig +++ b/arch/arm/configs/hisi_defconfig @@ -1,4 +1,4 @@ -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_BLK_DEV_INITRD=y diff --git a/arch/arm/configs/imx_v4_v5_defconfig b/arch/arm/configs/imx_v4_v5_defconfig index bfa2a95638af8181b98f0120a75838c85b37d6e7..711a79e9be007683bbffdcb50bd0d872e6a23132 100644 --- a/arch/arm/configs/imx_v4_v5_defconfig +++ b/arch/arm/configs/imx_v4_v5_defconfig @@ -1,6 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig index 01012537a9b900076f506c3463ab8abe288eacf2..078d61b758a9a74475dc07abce617c3a54515c4f 100644 --- a/arch/arm/configs/imx_v6_v7_defconfig +++ b/arch/arm/configs/imx_v6_v7_defconfig @@ -1,6 +1,6 @@ CONFIG_KERNEL_LZO=y CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_PREEMPT_VOLUNTARY=y @@ -31,7 +31,7 @@ CONFIG_SOC_VF610=y CONFIG_SMP=y CONFIG_ARM_PSCI=y CONFIG_HIGHMEM=y -CONFIG_FORCE_MAX_ZONEORDER=14 +CONFIG_ARCH_FORCE_MAX_ORDER=14 CONFIG_CMDLINE="noinitrd console=ttymxc0,115200" CONFIG_KEXEC=y CONFIG_CPU_FREQ=y @@ -65,7 +65,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_NETFILTER=y CONFIG_CAN=y -CONFIG_CAN_FLEXCAN=y CONFIG_BT=y CONFIG_BT_BNEP=m CONFIG_BT_HCIUART=y @@ -136,6 +135,7 @@ CONFIG_SMSC911X=y # CONFIG_NET_VENDOR_STMICRO is not set CONFIG_MICREL_PHY=y CONFIG_AT803X_PHY=y +CONFIG_CAN_FLEXCAN=y CONFIG_USB_PEGASUS=m CONFIG_USB_RTL8150=m CONFIG_USB_RTL8152=y diff --git a/arch/arm/configs/integrator_defconfig b/arch/arm/configs/integrator_defconfig index 9ca43c84b45218d62e671c7377b75214bdc3dc04..61711d4bbf74b0caa8ab66a86815d108f0d4a44b 100644 --- a/arch/arm/configs/integrator_defconfig +++ b/arch/arm/configs/integrator_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y diff --git a/arch/arm/configs/iop32x_defconfig b/arch/arm/configs/iop32x_defconfig index c16e92cdfd008497022182606a7f100d14226368..19e30e790d3503a3224ba21c01fcefc2c2157ee6 100644 --- a/arch/arm/configs/iop32x_defconfig +++ b/arch/arm/configs/iop32x_defconfig @@ -12,7 +12,6 @@ CONFIG_MACH_N2100=y CONFIG_UNUSED_BOARD_FILES=y CONFIG_CMDLINE="console=ttyS0,115200 root=/dev/nfs ip=bootp cachepolicy=writealloc" CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_PARTITION_ADVANCED=y diff --git a/arch/arm/configs/ixp4xx_defconfig b/arch/arm/configs/ixp4xx_defconfig index 6b65ac2a72e7f8bf65480bf77daf7352fe41abec..3cb995b9616a3a7d4cad7d77cff80babab7d1b53 100644 --- a/arch/arm/configs/ixp4xx_defconfig +++ b/arch/arm/configs/ixp4xx_defconfig @@ -99,7 +99,7 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_EEPROM_AT24=y -CONFIG_EEPROM_LEGACY=y +# CONFIG_EEPROM_LEGACY is not set # CONFIG_SCSI_PROC_FS is not set CONFIG_BLK_DEV_SD=y # CONFIG_BLK_DEV_BSG is not set diff --git a/arch/arm/configs/jornada720_defconfig b/arch/arm/configs/jornada720_defconfig index 3dcf89d3e1f1ee1bb919f8fa56820a417b2fd60b..ae1d68da4f2acfd1b8623c4097c076c98e78170c 100644 --- a/arch/arm/configs/jornada720_defconfig +++ b/arch/arm/configs/jornada720_defconfig @@ -1,12 +1,13 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_SYSFS_DEPRECATED_V2=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_JORNADA720=y CONFIG_SA1100_JORNADA720_SSP=y CONFIG_UNUSED_BOARD_FILES=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_PM=y CONFIG_MODULES=y CONFIG_NET=y diff --git a/arch/arm/configs/keystone_defconfig b/arch/arm/configs/keystone_defconfig index 68b89b90ca29a3e5f02aa936cfcf7f98c4563bd7..4a5b9adbf2a1674c1d7e014094665b91e26f082b 100644 --- a/arch/arm/configs/keystone_defconfig +++ b/arch/arm/configs/keystone_defconfig @@ -109,8 +109,6 @@ CONFIG_IP6_NF_IPTABLES=m CONFIG_IP_SCTP=y CONFIG_VLAN_8021Q=y CONFIG_CAN=m -CONFIG_CAN_C_CAN=m -CONFIG_CAN_C_CAN_PLATFORM=m CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_DEVTMPFS=y @@ -137,6 +135,8 @@ CONFIG_TI_KEYSTONE_NETCP_ETHSS=y CONFIG_MARVELL_PHY=y CONFIG_MICREL_PHY=y CONFIG_DP83867_PHY=y +CONFIG_CAN_C_CAN=m +CONFIG_CAN_C_CAN_PLATFORM=m CONFIG_INPUT_EVDEV=m CONFIG_INPUT_MISC=y CONFIG_INPUT_GPIO_DECODER=m @@ -152,7 +152,6 @@ CONFIG_SPI=y CONFIG_SPI_CADENCE_QUADSPI=y CONFIG_SPI_DAVINCI=y CONFIG_SPI_SPIDEV=y -CONFIG_PTP_1588_CLOCK=y CONFIG_PINCTRL_SINGLE=y CONFIG_GPIOLIB=y CONFIG_GPIO_SYSFS=y @@ -224,7 +223,6 @@ CONFIG_NFSD=y CONFIG_NFSD_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_CRYPTO_USER=y CONFIG_CRYPTO_AUTHENC=y CONFIG_CRYPTO_CBC=y @@ -236,5 +234,6 @@ CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y CONFIG_DMA_CMA=y CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_SHIRQ=y CONFIG_DEBUG_USER=y diff --git a/arch/arm/configs/lart_defconfig b/arch/arm/configs/lart_defconfig index 0c2f19d756c0e11f3c5649ee59354fffe85a9e28..916177d07a394aef0735c14b863c0a70bffe838e 100644 --- a/arch/arm/configs/lart_defconfig +++ b/arch/arm/configs/lart_defconfig @@ -1,6 +1,8 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_LART=y CONFIG_UNUSED_BOARD_FILES=y @@ -8,7 +10,6 @@ CONFIG_CMDLINE="console=ttySA0,9600 root=/dev/ram" CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_PM=y CONFIG_MODULES=y CONFIG_NET=y diff --git a/arch/arm/configs/lpc18xx_defconfig b/arch/arm/configs/lpc18xx_defconfig index 142c1700f450790535e465a5a62632d8307bdb0f..56eae6a0a311b5bbdb647decab03f93fd0358a1b 100644 --- a/arch/arm/configs/lpc18xx_defconfig +++ b/arch/arm/configs/lpc18xx_defconfig @@ -150,9 +150,9 @@ CONFIG_JFFS2_FS=y CONFIG_CRC_ITU_T=y CONFIG_CRC7=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_SLUB_DEBUG is not set diff --git a/arch/arm/configs/lpc32xx_defconfig b/arch/arm/configs/lpc32xx_defconfig index 8a41fe4e62f1239eef9ec901d959a2a3c91bc330..fabb66a53350cee0019af037cd0545f43a043a79 100644 --- a/arch/arm/configs/lpc32xx_defconfig +++ b/arch/arm/configs/lpc32xx_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y diff --git a/arch/arm/configs/magician_defconfig b/arch/arm/configs/magician_defconfig index 9cbb63c694361f1e1526a3473e1114d7d1b5a390..5a8776f6aba39d7ef297ddf0c64e19f8eb85327c 100644 --- a/arch/arm/configs/magician_defconfig +++ b/arch/arm/configs/magician_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 58810e98de3d4611f52d249d02211516b0038d22..a2e25bf843ccf4b6d80b8c803ed7485948e35eb7 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_CGROUPS=y CONFIG_BLK_DEV_INITRD=y @@ -26,7 +26,7 @@ CONFIG_THUMB2_KERNEL=y # CONFIG_THUMB2_AVOID_R_ARM_THM_JUMP11 is not set # CONFIG_ARM_PATCH_IDIV is not set CONFIG_HIGHMEM=y -CONFIG_FORCE_MAX_ZONEORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=12 CONFIG_SECCOMP=y CONFIG_KEXEC=y CONFIG_EFI=y @@ -44,7 +44,6 @@ CONFIG_ARM_CPUIDLE=y CONFIG_VFP=y CONFIG_NEON=y CONFIG_KERNEL_MODE_NEON=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_SHA2_ARM_CE=m diff --git a/arch/arm/configs/mmp2_defconfig b/arch/arm/configs/mmp2_defconfig index 4d39c615117bc63e412f61fbc3ecbdcfed8be9ff..7984640e994ee3637ca4c44427195f6075740947 100644 --- a/arch/arm/configs/mmp2_defconfig +++ b/arch/arm/configs/mmp2_defconfig @@ -73,10 +73,10 @@ CONFIG_ROOT_NFS=y CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_DEBUG_PREEMPT is not set -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_DYNAMIC_DEBUG is not set CONFIG_DEBUG_USER=y CONFIG_DEBUG_LL=y diff --git a/arch/arm/configs/moxart_defconfig b/arch/arm/configs/moxart_defconfig index 082a38a14c1244326401a3f1f407fe66ca337f09..ea31f116d5776b51f673ae007fa7cb8712a2bbce 100644 --- a/arch/arm/configs/moxart_defconfig +++ b/arch/arm/configs/moxart_defconfig @@ -1,6 +1,6 @@ # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y diff --git a/arch/arm/configs/mps2_defconfig b/arch/arm/configs/mps2_defconfig index 700568474549f5bfe6db6f5c9ca76b2182041c22..3ed73f184d839fb7ec42f78f10b3f768188adf75 100644 --- a/arch/arm/configs/mps2_defconfig +++ b/arch/arm/configs/mps2_defconfig @@ -94,9 +94,9 @@ CONFIG_NFS_V4_2=y CONFIG_ROOT_NFS=y CONFIG_NLS=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y # CONFIG_SLUB_DEBUG is not set # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/multi_v4t_defconfig b/arch/arm/configs/multi_v4t_defconfig index 6c3e45b71ab5649cbeeb322f4f7a238bb373d050..e2fd822f741a833c84e51b06a2a9ad564285e1ab 100644 --- a/arch/arm/configs/multi_v4t_defconfig +++ b/arch/arm/configs/multi_v4t_defconfig @@ -71,8 +71,6 @@ CONFIG_POWER_RESET_SYSCON_POWEROFF=y CONFIG_WATCHDOG=y CONFIG_GPIO_WATCHDOG=y CONFIG_AT91RM9200_WATCHDOG=y -CONFIG_REGULATOR=y -CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_REGULATOR_GPIO=y CONFIG_FB=y CONFIG_FB_CLPS711X=y diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig index e0be0e0023f39c2cdd7d7b93d0112d1b5928484d..60fc52b956902af3e466858dc215de82e127cb8f 100644 --- a/arch/arm/configs/multi_v5_defconfig +++ b/arch/arm/configs/multi_v5_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=19 @@ -149,7 +149,6 @@ CONFIG_SPI_SUN6I=y CONFIG_GPIO_ASPEED=m CONFIG_GPIO_ASPEED_SGPIO=y CONFIG_GPIO_MXC=y -CONFIG_POWER_RESET=y CONFIG_POWER_RESET_GPIO=y CONFIG_POWER_RESET_QNAP=y CONFIG_SENSORS_ADT7475=y @@ -290,8 +289,8 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_DEV_MARVELL_CESA=y CONFIG_CRC_CCITT=y CONFIG_LIBCRC32C=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 12b35008571f701b17de27047361499b35c0e956..b61b2e3d116bcf59f0efe4e1b8806bdfc12d9359 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_CGROUPS=y CONFIG_BLK_DEV_INITRD=y @@ -132,7 +132,6 @@ CONFIG_ARM_EXYNOS_CPUIDLE=y CONFIG_ARM_TEGRA_CPUIDLE=y CONFIG_ARM_QCOM_SPM_CPUIDLE=y CONFIG_KERNEL_MODE_NEON=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA1_ARM_CE=m CONFIG_CRYPTO_SHA2_ARM_CE=m @@ -167,12 +166,6 @@ CONFIG_NET_DSA=m CONFIG_QRTR=m CONFIG_QRTR_SMD=m CONFIG_CAN=y -CONFIG_CAN_AT91=m -CONFIG_CAN_FLEXCAN=m -CONFIG_CAN_SUN4I=y -CONFIG_CAN_XILINXCAN=y -CONFIG_CAN_RCAR=m -CONFIG_CAN_MCP251X=y CONFIG_BT=m CONFIG_BT_HCIUART=m CONFIG_BT_HCIUART_BCM=y @@ -260,7 +253,6 @@ CONFIG_SATA_HIGHBANK=y CONFIG_SATA_MV=y CONFIG_SATA_RCAR=y CONFIG_NETDEVICES=y -CONFIG_NET_VENDOR_ASIX=y CONFIG_VIRTIO_NET=y CONFIG_B53_SPI_DRIVER=m CONFIG_B53_MDIO_DRIVER=m @@ -300,6 +292,12 @@ CONFIG_MARVELL_PHY=y CONFIG_AT803X_PHY=y CONFIG_ROCKCHIP_PHY=y CONFIG_DP83867_PHY=y +CONFIG_CAN_AT91=m +CONFIG_CAN_FLEXCAN=m +CONFIG_CAN_SUN4I=y +CONFIG_CAN_XILINXCAN=y +CONFIG_CAN_RCAR=m +CONFIG_CAN_MCP251X=y CONFIG_MDIO_MSCC_MIIM=m CONFIG_USB_PEGASUS=y CONFIG_USB_RTL8152=m @@ -378,8 +376,6 @@ CONFIG_SERIAL_MSM=y CONFIG_SERIAL_MSM_CONSOLE=y CONFIG_SERIAL_VT8500=y CONFIG_SERIAL_VT8500_CONSOLE=y -CONFIG_SERIAL_OMAP=y -CONFIG_SERIAL_OMAP_CONSOLE=y CONFIG_SERIAL_BCM63XX=y CONFIG_SERIAL_BCM63XX_CONSOLE=y CONFIG_SERIAL_XILINX_PS_UART=y @@ -465,7 +461,6 @@ CONFIG_SPI_TEGRA20_SLINK=y CONFIG_SPI_XILINX=y CONFIG_SPI_SPIDEV=y CONFIG_SPMI=y -CONFIG_PTP_1588_CLOCK=y CONFIG_PINCTRL_AS3722=y CONFIG_PINCTRL_MICROCHIP_SGPIO=y CONFIG_PINCTRL_OCELOT=y @@ -716,7 +711,6 @@ CONFIG_ROCKCHIP_DW_MIPI_DSI=y CONFIG_ROCKCHIP_INNO_HDMI=y CONFIG_DRM_ATMEL_HLCDC=m CONFIG_DRM_RCAR_DU=m -CONFIG_DRM_RCAR_LVDS=y CONFIG_DRM_SUN4I=m CONFIG_DRM_MSM=m CONFIG_DRM_FSL_DCU=m @@ -783,7 +777,6 @@ CONFIG_SND_ATMEL_SOC_I2S=m CONFIG_SND_BCM2835_SOC_I2S=m CONFIG_SND_IMX_SOC=m CONFIG_SND_SOC_FSL_ASOC_CARD=m -CONFIG_SND_SOC_FSL_SAI=m CONFIG_SND_PXA_SOC_SSP=m CONFIG_SND_MMP_SOC_SSPA=m CONFIG_SND_PXA910_SOC=m @@ -820,7 +813,7 @@ CONFIG_SND_SOC_TEGRA_TRIMSLICE=m CONFIG_SND_SOC_TEGRA_ALC5632=m CONFIG_SND_SOC_TEGRA_MAX98090=m CONFIG_SND_SOC_DAVINCI_MCASP=m -CONFIG_SND_SOC_AC97=m +CONFIG_SND_SOC_AC97_CODEC=m CONFIG_SND_SOC_AK4642=m CONFIG_SND_SOC_CPCAP=m CONFIG_SND_SOC_CS42L51_I2C=m @@ -875,7 +868,6 @@ CONFIG_USB_GPIO_VBUS=y CONFIG_USB_ISP1301=y CONFIG_USB_MXS_PHY=y CONFIG_USB_GADGET=y -CONFIG_USB_FSL_USB2=y CONFIG_USB_RENESAS_USBHS_UDC=m CONFIG_USB_ASPEED_VHUB=m CONFIG_USB_CONFIGFS=m @@ -1051,7 +1043,6 @@ CONFIG_MSM_MMCC_8960=y CONFIG_MSM_MMCC_8974=y CONFIG_HWSPINLOCK=y CONFIG_HWSPINLOCK_QCOM=y -CONFIG_MICROCHIP_PIT64B=y CONFIG_BCM2835_MBOX=y CONFIG_QCOM_APCS_IPC=y CONFIG_QCOM_IPCC=y @@ -1071,9 +1062,13 @@ CONFIG_ASPEED_LPC_CTRL=m CONFIG_ASPEED_LPC_SNOOP=m CONFIG_ASPEED_P2A_CTRL=m CONFIG_RASPBERRYPI_POWER=y +CONFIG_QCOM_COMMAND_DB=m CONFIG_QCOM_CPR=y CONFIG_QCOM_GSBI=y +CONFIG_QCOM_OCMEM=m CONFIG_QCOM_RMTFS_MEM=m +CONFIG_QCOM_RPMH=y +CONFIG_QCOM_RPMHPD=y CONFIG_QCOM_RPMPD=y CONFIG_QCOM_SMEM=y CONFIG_QCOM_SMD_RPM=y @@ -1193,11 +1188,11 @@ CONFIG_TI_PIPE3=y CONFIG_TWL4030_USB=m CONFIG_RAS=y CONFIG_NVMEM_IMX_OCOTP=y -CONFIG_QCOM_QFPROM=y -CONFIG_ROCKCHIP_EFUSE=m +CONFIG_NVMEM_QCOM_QFPROM=y +CONFIG_NVMEM_ROCKCHIP_EFUSE=m CONFIG_NVMEM_SUNXI_SID=y CONFIG_NVMEM_VF610_OCOTP=y -CONFIG_MESON_MX_EFUSE=m +CONFIG_NVMEM_MESON_MX_EFUSE=m CONFIG_NVMEM_RMEM=m CONFIG_FSI=m CONFIG_FSI_MASTER_GPIO=m diff --git a/arch/arm/configs/mv78xx0_defconfig b/arch/arm/configs/mv78xx0_defconfig index a53ccd49f8ffde377c9e97602d41af3c661140ca..877c5150a98768ba3ad4ac037cc510eeb7e32d2f 100644 --- a/arch/arm/configs/mv78xx0_defconfig +++ b/arch/arm/configs/mv78xx0_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 diff --git a/arch/arm/configs/mvebu_v5_defconfig b/arch/arm/configs/mvebu_v5_defconfig index ef3a33ebc29a60e158775efea8d4c4b065466ce5..2467afd32146ba7ba2bf4b71053a6747bc58d1cc 100644 --- a/arch/arm/configs/mvebu_v5_defconfig +++ b/arch/arm/configs/mvebu_v5_defconfig @@ -1,6 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_FHANDLE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=19 @@ -184,13 +184,13 @@ CONFIG_NLS_CODEPAGE_850=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_2=y CONFIG_NLS_UTF8=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_CRYPTO_CBC=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_DEV_MARVELL_CESA=y CONFIG_CRC_CCITT=y CONFIG_LIBCRC32C=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig index 155553ee06f4cae720a357bb645ce719d9b9dd8e..feb38a94c1a70a49c3d3d997957d493585bff54f 100644 --- a/arch/arm/configs/mxs_defconfig +++ b/arch/arm/configs/mxs_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT_VOLUNTARY=y CONFIG_TASKSTATS=y @@ -38,7 +38,6 @@ CONFIG_SYN_COOKIES=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set CONFIG_CAN=m -CONFIG_CAN_FLEXCAN=m # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y @@ -62,6 +61,7 @@ CONFIG_ICPLUS_PHY=y CONFIG_MICREL_PHY=y CONFIG_REALTEK_PHY=y CONFIG_SMSC_PHY=y +CONFIG_CAN_FLEXCAN=m CONFIG_USB_USBNET=y CONFIG_USB_NET_SMSC95XX=y # CONFIG_WLAN is not set @@ -163,8 +163,8 @@ CONFIG_CRC_ITU_T=m CONFIG_CRC7=m CONFIG_FONTS=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_WARN=2048 CONFIG_MAGIC_SYSRQ=y CONFIG_SOFTLOCKUP_DETECTOR=y diff --git a/arch/arm/configs/neponset_defconfig b/arch/arm/configs/neponset_defconfig index 907403529e30229fb6d9bfa1ed949c657923e13b..c333406ce5e3ff01d3ccb6455f6b175167ff79b7 100644 --- a/arch/arm/configs/neponset_defconfig +++ b/arch/arm/configs/neponset_defconfig @@ -1,6 +1,8 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_ASSABET=y CONFIG_ASSABET_NEPONSET=y @@ -9,7 +11,6 @@ CONFIG_ZBOOT_ROM_BSS=0xc1000000 CONFIG_ZBOOT_ROM=y CONFIG_CMDLINE="console=ttySA0,38400n8 cpufreq=221200 rw root=/dev/mtdblock2 mtdparts=sa1100:512K(boot),1M(kernel),2560K(initrd),4M(root) load_ramdisk=1 prompt_ramdisk=0 mem=32M noinitrd initrd=0xc0800000,3M" CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_PM=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/arm/configs/netwinder_defconfig b/arch/arm/configs/netwinder_defconfig index cf7bbcf9d98a75f9ca1b7cd2ee069abdd0815ae7..30ff6fbce5a34896f6e0fa4be8e8e44f0cd063cd 100644 --- a/arch/arm/configs/netwinder_defconfig +++ b/arch/arm/configs/netwinder_defconfig @@ -1,11 +1,12 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_FOOTBRIDGE=y CONFIG_ARCH_NETWINDER=y CONFIG_DEPRECATED_PARAM_STRUCT=y CONFIG_CMDLINE="root=0x801" CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_PARTITION_ADVANCED=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/arm/configs/omap1_defconfig b/arch/arm/configs/omap1_defconfig index 54a9f50122af172e038ec7f8428a973941ff8f0e..70511fe4b3ecfa162294aeb05b105f811876daac 100644 --- a/arch/arm/configs/omap1_defconfig +++ b/arch/arm/configs/omap1_defconfig @@ -1,6 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_BSD_PROCESS_ACCT=y @@ -218,7 +218,6 @@ CONFIG_NLS_KOI8_R=y CONFIG_NLS_UTF8=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_SECURITY=y CONFIG_CRYPTO_ECB=y CONFIG_CRYPTO_PCBC=y @@ -232,6 +231,7 @@ CONFIG_FONT_8x16=y CONFIG_FONT_6x11=y CONFIG_FONT_MINI_4x6=y # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 99d015cf8919e158b78cea4c697db5cf4248e19f..2a66850d32882bceb75e5bef24007dee0c406161 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -2,7 +2,7 @@ CONFIG_KERNEL_LZMA=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_IKCONFIG=y @@ -53,7 +53,6 @@ CONFIG_CPU_IDLE=y CONFIG_ARM_CPUIDLE=y CONFIG_KERNEL_MODE_NEON=y CONFIG_PM_DEBUG=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM_NEON=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m @@ -249,8 +248,6 @@ CONFIG_NET_ACT_POLICE=m CONFIG_NET_ACT_GACT=m CONFIG_NET_SWITCHDEV=y CONFIG_CAN=m -CONFIG_CAN_C_CAN=m -CONFIG_CAN_C_CAN_PLATFORM=m CONFIG_BT=m CONFIG_BT_RFCOMM=m CONFIG_BT_RFCOMM_TTY=y @@ -350,6 +347,8 @@ CONFIG_AT803X_PHY=y CONFIG_SMSC_PHY=y CONFIG_DP83848_PHY=y CONFIG_DP83867_PHY=y +CONFIG_CAN_C_CAN=m +CONFIG_CAN_C_CAN_PLATFORM=m CONFIG_PPP=m CONFIG_PPP_BSDCOMP=m CONFIG_PPP_DEFLATE=m @@ -419,8 +418,6 @@ CONFIG_SERIAL_8250_DETECT_IRQ=y CONFIG_SERIAL_8250_RSA=y CONFIG_SERIAL_8250_OMAP=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_OMAP=y -CONFIG_SERIAL_OMAP_CONSOLE=y CONFIG_SERIAL_DEV_BUS=y CONFIG_I2C_CHARDEV=y CONFIG_SPI=y @@ -430,7 +427,6 @@ CONFIG_SPI_TI_QSPI=m CONFIG_HSI=m CONFIG_OMAP_SSI=m CONFIG_SSI_PROTOCOL=m -CONFIG_PTP_1588_CLOCK=y CONFIG_PINCTRL_SINGLE=y CONFIG_DEBUG_GPIO=y CONFIG_GPIO_SYSFS=y diff --git a/arch/arm/configs/orion5x_defconfig b/arch/arm/configs/orion5x_defconfig index 1311d9583fccd93b91ca74ab960e4e53129abe09..0629b088a584a571bd95c90555401716431a2487 100644 --- a/arch/arm/configs/orion5x_defconfig +++ b/arch/arm/configs/orion5x_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 @@ -138,9 +138,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_DEV_MARVELL_CESA=y CONFIG_CRC_T10DIF=y # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_SLUB_DEBUG is not set CONFIG_LATENCYTOP=y # CONFIG_FTRACE is not set diff --git a/arch/arm/configs/oxnas_v6_defconfig b/arch/arm/configs/oxnas_v6_defconfig index 600f78b363dd6f496bb74f461792e9b1f05f9e06..70a67b3fc91bdc892fc81a1928e5b65dd3e55efe 100644 --- a/arch/arm/configs/oxnas_v6_defconfig +++ b/arch/arm/configs/oxnas_v6_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_CGROUPS=y CONFIG_BLK_DEV_INITRD=y @@ -12,7 +12,7 @@ CONFIG_ARCH_OXNAS=y CONFIG_MACH_OX820=y CONFIG_SMP=y CONFIG_NR_CPUS=16 -CONFIG_FORCE_MAX_ZONEORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=12 CONFIG_SECCOMP=y CONFIG_ARM_APPENDED_DTB=y CONFIG_ARM_ATAG_DTB_COMPAT=y diff --git a/arch/arm/configs/pcm027_defconfig b/arch/arm/configs/pcm027_defconfig index 06bc9a8fef900049b695c6acdef8fbef7309bc19..a392312a13ce288b9ccf04e7a5dde5eab0652495 100644 --- a/arch/arm/configs/pcm027_defconfig +++ b/arch/arm/configs/pcm027_defconfig @@ -1,6 +1,6 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_BSD_PROCESS_ACCT=y diff --git a/arch/arm/configs/pleb_defconfig b/arch/arm/configs/pleb_defconfig index d87263336cb22d215781c5e2db38327b1e4ce6b3..fd26678732736fbeef7242b8305b4b981f1aa3d9 100644 --- a/arch/arm/configs/pleb_defconfig +++ b/arch/arm/configs/pleb_defconfig @@ -4,6 +4,8 @@ CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_HOTPLUG is not set # CONFIG_SHMEM is not set +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_PLEB=y CONFIG_UNUSED_BOARD_FILES=y diff --git a/arch/arm/configs/pxa168_defconfig b/arch/arm/configs/pxa168_defconfig index 70d327895ccfe67cbdac10aa200b354073059eea..826ebbef2e3c6483885f0d52f99175882edbf2de 100644 --- a/arch/arm/configs/pxa168_defconfig +++ b/arch/arm/configs/pxa168_defconfig @@ -4,7 +4,7 @@ CONFIG_SYSFS_DEPRECATED_V2=y CONFIG_MACH_ASPENITE=y CONFIG_MACH_ZYLONITE2=y CONFIG_MACH_AVENGERS_LITE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_AEABI=y @@ -50,9 +50,9 @@ CONFIG_ROOT_NFS=y CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y # CONFIG_DEBUG_PREEMPT is not set -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_USER=y CONFIG_DEBUG_LL=y # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/arm/configs/pxa910_defconfig b/arch/arm/configs/pxa910_defconfig index 5072bde71508cd7ba56aec6d7833bc26d57f653d..353008de567832f7a2a54baaac87540cf601c1bf 100644 --- a/arch/arm/configs/pxa910_defconfig +++ b/arch/arm/configs/pxa910_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_LOG_BUF_SHIFT=14 @@ -58,9 +58,9 @@ CONFIG_ROOT_NFS=y CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y # CONFIG_DEBUG_PREEMPT is not set -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_USER=y CONFIG_DEBUG_LL=y CONFIG_DEBUG_MMP_UART2=y diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index 104a45722799933cf4726b8f1fc6c3b12bc33296..d60cc9cc4c21404490d3844c9201be8d9c710c51 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -1,7 +1,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_FHANDLE=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_BSD_PROCESS_ACCT=y @@ -21,7 +21,7 @@ CONFIG_MACH_AKITA=y CONFIG_MACH_BORZOI=y CONFIG_PXA_SYSTEMS_CPLDS=y CONFIG_AEABI=y -CONFIG_FORCE_MAX_ZONEORDER=9 +CONFIG_ARCH_FORCE_MAX_ORDER=9 CONFIG_CMDLINE="root=/dev/ram0 ro" CONFIG_KEXEC=y CONFIG_CPU_FREQ=y @@ -34,7 +34,6 @@ CONFIG_CPUFREQ_DT=m CONFIG_ARM_PXA2xx_CPUFREQ=m CONFIG_CPU_IDLE=y CONFIG_ARM_CPUIDLE=y -CONFIG_ARM_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM=m CONFIG_CRYPTO_SHA256_ARM=m CONFIG_CRYPTO_SHA512_ARM=m @@ -646,7 +645,6 @@ CONFIG_NLS_ASCII=m CONFIG_NLS_ISO8859_1=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_TIMER_STATS=y CONFIG_SECURITY=y CONFIG_CRYPTO_MANAGER=y @@ -682,6 +680,7 @@ CONFIG_FONT_6x11=y CONFIG_FONT_MINI_4x6=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_WARN=0 CONFIG_STRIP_ASM_SYMS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig index 8a59441701a80f96256bd334819427b0c7de5b97..7d8b6884fd0069a44ffce866d97f79c576c69f87 100644 --- a/arch/arm/configs/qcom_defconfig +++ b/arch/arm/configs/qcom_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y @@ -245,6 +245,7 @@ CONFIG_QCOM_Q6V5_PAS=y CONFIG_QCOM_Q6V5_PIL=y CONFIG_QCOM_WCNSS_PIL=y CONFIG_RPMSG_CHAR=y +CONFIG_RPMSG_CTRL=y CONFIG_RPMSG_QCOM_GLINK_SMEM=y CONFIG_RPMSG_QCOM_SMD=y CONFIG_QCOM_COMMAND_DB=y @@ -253,10 +254,14 @@ CONFIG_QCOM_OCMEM=y CONFIG_QCOM_PM=y CONFIG_QCOM_RPMH=y CONFIG_QCOM_RPMHPD=y +CONFIG_QCOM_RMTFS_MEM=y +CONFIG_QCOM_RPMPD=y CONFIG_QCOM_SMEM=y CONFIG_QCOM_SMD_RPM=y CONFIG_QCOM_SMP2P=y CONFIG_QCOM_SMSM=y +CONFIG_QCOM_SOCINFO=y +CONFIG_QCOM_STATS=y CONFIG_QCOM_WCNSS_CTRL=y CONFIG_EXTCON_QCOM_SPMI_MISC=y CONFIG_IIO=y @@ -278,7 +283,7 @@ CONFIG_PHY_QCOM_QMP=y CONFIG_PHY_QCOM_USB_HS=y CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=y CONFIG_PHY_QCOM_USB_HSIC=y -CONFIG_QCOM_QFPROM=y +CONFIG_NVMEM_QCOM_QFPROM=y CONFIG_INTERCONNECT=y CONFIG_INTERCONNECT_QCOM=y CONFIG_INTERCONNECT_QCOM_MSM8974=m diff --git a/arch/arm/configs/rpc_defconfig b/arch/arm/configs/rpc_defconfig index 16d74a1f027a6d2d1b61398f0064c9255cea9802..210974364d619852ca410b866f451c219159457d 100644 --- a/arch/arm/configs/rpc_defconfig +++ b/arch/arm/configs/rpc_defconfig @@ -4,10 +4,11 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_RPC=y CONFIG_CPU_SA110=y CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=y CONFIG_PARTITION_ADVANCED=y CONFIG_BSD_DISKLABEL=y CONFIG_SLAB=y diff --git a/arch/arm/configs/s3c6400_defconfig b/arch/arm/configs/s3c6400_defconfig index 4f04f583c738988c7b8370cd8e43f3cf159f3f5e..93258d5b57ff417623365d96ba987ba04e11e2fd 100644 --- a/arch/arm/configs/s3c6400_defconfig +++ b/arch/arm/configs/s3c6400_defconfig @@ -62,8 +62,8 @@ CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_CRAMFS=y CONFIG_ROMFS_FS=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_RT_MUTEXES=y CONFIG_DEBUG_SPINLOCK=y diff --git a/arch/arm/configs/s5pv210_defconfig b/arch/arm/configs/s5pv210_defconfig index 789e900a8a08e608dfc355f3895f0172fde75c6c..4c1e480b5bbdc43ed35ad7a8add5d325a2f4cead 100644 --- a/arch/arm/configs/s5pv210_defconfig +++ b/arch/arm/configs/s5pv210_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_CGROUPS=y @@ -115,8 +115,8 @@ CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_CRC_CCITT=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y # CONFIG_DEBUG_PREEMPT is not set CONFIG_DEBUG_RT_MUTEXES=y diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig index 3a6a4851ef26cff605b7b46b0cfc5689f61ee439..877bbe7b777e0491a518b9cbf8239e526c5116cc 100644 --- a/arch/arm/configs/sama5_defconfig +++ b/arch/arm/configs/sama5_defconfig @@ -43,9 +43,6 @@ CONFIG_BRIDGE_VLAN_FILTERING=y CONFIG_NET_DSA=m CONFIG_VLAN_8021Q=m CONFIG_CAN=y -CONFIG_CAN_AT91=y -CONFIG_CAN_M_CAN=y -CONFIG_CAN_M_CAN_PLATFORM=y CONFIG_CFG80211=y CONFIG_MAC80211=y CONFIG_MAC80211_LEDS=y @@ -90,6 +87,9 @@ CONFIG_MACB=y # CONFIG_NET_VENDOR_STMICRO is not set # CONFIG_NET_VENDOR_WIZNET is not set CONFIG_MICREL_PHY=y +CONFIG_CAN_AT91=y +CONFIG_CAN_M_CAN=y +CONFIG_CAN_M_CAN_PLATFORM=y CONFIG_USB_LAN78XX=m CONFIG_LIBERTAS_THINFIRM=m CONFIG_LIBERTAS_THINFIRM_USB=m diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig index 0384030d8b25a263d003302fd4c8ca65b4c75a65..8f28c9d443f07b74a2ed09c1ecc01073596cbc1a 100644 --- a/arch/arm/configs/sama7_defconfig +++ b/arch/arm/configs/sama7_defconfig @@ -19,7 +19,7 @@ CONFIG_ATMEL_CLOCKSOURCE_TCB=y # CONFIG_CACHE_L2X0 is not set # CONFIG_ARM_PATCH_IDIV is not set # CONFIG_CPU_SW_DOMAIN_PAN is not set -CONFIG_FORCE_MAX_ZONEORDER=15 +CONFIG_ARCH_FORCE_MAX_ORDER=15 CONFIG_UACCESS_WITH_MEMCPY=y # CONFIG_ATAGS is not set CONFIG_CMDLINE="console=ttyS0,115200 earlyprintk ignore_loglevel" @@ -59,8 +59,6 @@ CONFIG_BRIDGE_VLAN_FILTERING=y CONFIG_NET_DSA=m CONFIG_VLAN_8021Q=m CONFIG_CAN=y -CONFIG_CAN_M_CAN=y -CONFIG_CAN_M_CAN_PLATFORM=y CONFIG_BT=y CONFIG_BT_RFCOMM=y CONFIG_BT_RFCOMM_TTY=y @@ -107,6 +105,8 @@ CONFIG_BLK_DEV_SD=y CONFIG_NETDEVICES=y CONFIG_MACB=y CONFIG_MICREL_PHY=y +CONFIG_CAN_M_CAN=y +CONFIG_CAN_M_CAN_PLATFORM=y CONFIG_INPUT_EVDEV=y CONFIG_KEYBOARD_GPIO=y # CONFIG_INPUT_MOUSE is not set @@ -188,7 +188,6 @@ CONFIG_RTC_DRV_AT91SAM9=y CONFIG_DMADEVICES=y CONFIG_AT_XDMAC=y CONFIG_STAGING=y -CONFIG_MICROCHIP_PIT64B=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_IIO=y CONFIG_IIO_SW_TRIGGER=y diff --git a/arch/arm/configs/shannon_defconfig b/arch/arm/configs/shannon_defconfig index 42252e85ee494ce652d85cd4192262da0cbdc42d..dfcea70b8034f8c6fde51ea93e1278929ba44e23 100644 --- a/arch/arm/configs/shannon_defconfig +++ b/arch/arm/configs/shannon_defconfig @@ -1,6 +1,8 @@ CONFIG_SYSVIPC=y CONFIG_LOG_BUF_SHIFT=14 CONFIG_BLK_DEV_INITRD=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_SHANNON=y CONFIG_UNUSED_BOARD_FILES=y diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig index a29bebb3742ed386a1374ee3160bf1c588ea4815..452aef74cc5c1256128554802ab26139c2f3b60f 100644 --- a/arch/arm/configs/shmobile_defconfig +++ b/arch/arm/configs/shmobile_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_CGROUPS=y @@ -32,8 +32,8 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y +CONFIG_NET_DSA=y CONFIG_CAN=y -CONFIG_CAN_RCAR=y CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_PCI_RCAR_GEN2=y @@ -52,11 +52,14 @@ CONFIG_BLK_DEV_SD=y CONFIG_ATA=y CONFIG_SATA_RCAR=y CONFIG_NETDEVICES=y +CONFIG_NET_DSA_RZN1_A5PSW=y CONFIG_SH_ETH=y CONFIG_RAVB=y CONFIG_SMSC911X=y +CONFIG_STMMAC_ETH=y CONFIG_MICREL_PHY=y CONFIG_SMSC_PHY=y +CONFIG_CAN_RCAR=y CONFIG_INPUT_EVDEV=y CONFIG_KEYBOARD_GPIO=y # CONFIG_INPUT_MOUSE is not set @@ -91,6 +94,7 @@ CONFIG_SPI_SH_MSIOF=y CONFIG_SPI_SH_HSPI=y CONFIG_PINCTRL_RZA1=y CONFIG_PINCTRL_RZA2=y +CONFIG_PINCTRL_RZN1=y CONFIG_GPIO_EM=y CONFIG_GPIO_RCAR=y CONFIG_GPIO_PCA953X=y @@ -180,6 +184,8 @@ CONFIG_RTC_DRV_DA9063=y CONFIG_RTC_DRV_SH=y CONFIG_RTC_DRV_RZN1=y CONFIG_DMADEVICES=y +CONFIG_DW_DMAC=y +CONFIG_RZN1_DMAMUX=y CONFIG_RCAR_DMAC=y CONFIG_RENESAS_USB_DMAC=y CONFIG_STAGING=y diff --git a/arch/arm/configs/simpad_defconfig b/arch/arm/configs/simpad_defconfig index cc451728f6d9b1ec33b11c35b27f7e056c410a6a..4e00a4c2c28788f5d43e0a5290a2c235a6f8d5c9 100644 --- a/arch/arm/configs/simpad_defconfig +++ b/arch/arm/configs/simpad_defconfig @@ -5,6 +5,8 @@ CONFIG_LOG_BUF_SHIFT=14 CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y CONFIG_KALLSYMS_EXTRA_PASS=y +CONFIG_ARCH_MULTI_V4=y +# CONFIG_ARCH_MULTI_V7 is not set CONFIG_ARCH_SA1100=y CONFIG_SA1100_SIMPAD=y CONFIG_UNUSED_BOARD_FILES=y diff --git a/arch/arm/configs/socfpga_defconfig b/arch/arm/configs/socfpga_defconfig index d91ae3f0d6981c9503f54ebdc69202275f2e5cf5..70739e09d0f45fc543bddde10ed0c9b52384b0ac 100644 --- a/arch/arm/configs/socfpga_defconfig +++ b/arch/arm/configs/socfpga_defconfig @@ -33,9 +33,6 @@ CONFIG_NETWORK_PHY_TIMESTAMPING=y CONFIG_VLAN_8021Q=y CONFIG_VLAN_8021Q_GVRP=y CONFIG_CAN=y -CONFIG_CAN_C_CAN=y -CONFIG_CAN_C_CAN_PLATFORM=y -CONFIG_CAN_DEBUG_DEVICES=y CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_PCIE_ALTERA=y @@ -70,6 +67,9 @@ CONFIG_IXGBE=m CONFIG_STMMAC_ETH=y CONFIG_MARVELL_PHY=y CONFIG_MICREL_PHY=y +CONFIG_CAN_C_CAN=y +CONFIG_CAN_C_CAN_PLATFORM=y +CONFIG_CAN_DEBUG_DEVICES=y CONFIG_INPUT_EVDEV=y CONFIG_INPUT_TOUCHSCREEN=y CONFIG_TOUCHSCREEN_STMPE=y diff --git a/arch/arm/configs/sp7021_defconfig b/arch/arm/configs/sp7021_defconfig index 703b9aaa40f057388959434d9ccec3c64da5fa7e..5bca2eb59b866c1449a6d0e30f0036899ff639e1 100644 --- a/arch/arm/configs/sp7021_defconfig +++ b/arch/arm/configs/sp7021_defconfig @@ -13,18 +13,18 @@ CONFIG_LOG_BUF_SHIFT=14 # CONFIG_RD_LZ4 is not set CONFIG_CC_OPTIMIZE_FOR_SIZE=y CONFIG_PERF_EVENTS=y -CONFIG_SLAB=y CONFIG_ARCH_SUNPLUS=y # CONFIG_VDSO is not set CONFIG_SMP=y CONFIG_THUMB2_KERNEL=y -CONFIG_FORCE_MAX_ZONEORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=12 CONFIG_VFP=y CONFIG_NEON=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODVERSIONS=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_SLAB=y CONFIG_UEVENT_HELPER=y CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y diff --git a/arch/arm/configs/spear13xx_defconfig b/arch/arm/configs/spear13xx_defconfig index 0227dd566c28b2c3e2423fd0ab942d89629279fa..bfde0c86cdc5ad6c610a5e3e4e7842949a4b0a54 100644 --- a/arch/arm/configs/spear13xx_defconfig +++ b/arch/arm/configs/spear13xx_defconfig @@ -98,7 +98,7 @@ CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=m CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/arm/configs/spear3xx_defconfig b/arch/arm/configs/spear3xx_defconfig index 254d970a4011cbf4ba38fdc8aa4ec884ec5596d0..a96ed5cf778e2c2e8ec559f7bb75517db066158d 100644 --- a/arch/arm/configs/spear3xx_defconfig +++ b/arch/arm/configs/spear3xx_defconfig @@ -78,7 +78,7 @@ CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=m CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/arm/configs/spear6xx_defconfig b/arch/arm/configs/spear6xx_defconfig index 2809c4eb77e790c7b5c744ac208e1ce50156ad2e..3d631b1f3cfaa1db97f162d0cfa212bc8cd018a9 100644 --- a/arch/arm/configs/spear6xx_defconfig +++ b/arch/arm/configs/spear6xx_defconfig @@ -67,7 +67,7 @@ CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=m CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig index 1284a1d92ca3ebc39b9bc954581167dfdc689b98..66d74653f3fbaa397c68a8f051279e47b816e4d2 100644 --- a/arch/arm/configs/spitz_defconfig +++ b/arch/arm/configs/spitz_defconfig @@ -13,7 +13,6 @@ CONFIG_MACH_AKITA=y CONFIG_MACH_BORZOI=y CONFIG_CMDLINE="console=ttyS0,115200n8 console=tty1 noinitrd root=/dev/mtdblock2 rootfstype=jffs2 debug" CONFIG_FPE_NWFPE=y -CONFIG_BINFMT_AOUT=m CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y diff --git a/arch/arm/configs/stm32_defconfig b/arch/arm/configs/stm32_defconfig index 1f5446cda8b643a5cdaedfb9c3f4fb5931340a4e..dc1a32f50b7e56f890b3d5ff81505f6979775407 100644 --- a/arch/arm/configs/stm32_defconfig +++ b/arch/arm/configs/stm32_defconfig @@ -74,9 +74,9 @@ CONFIG_NLS=y CONFIG_CRC_ITU_T=y CONFIG_CRC7=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_DEBUG_BUGVERBOSE is not set +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y # CONFIG_SLUB_DEBUG is not set # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/sunxi_defconfig b/arch/arm/configs/sunxi_defconfig index 3d14827e0a31b1a70d2d5f4eb7ca3f68fb30c625..bddc82f789421191b10d5f6e00763bd53b215b7e 100644 --- a/arch/arm/configs/sunxi_defconfig +++ b/arch/arm/configs/sunxi_defconfig @@ -1,4 +1,4 @@ -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_CGROUPS=y CONFIG_BLK_DEV_INITRD=y @@ -26,7 +26,6 @@ CONFIG_IP_PNP_BOOTP=y # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set CONFIG_CAN=y -CONFIG_CAN_SUN4I=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y @@ -52,6 +51,7 @@ CONFIG_STMMAC_ETH=y # CONFIG_NET_VENDOR_WIZNET is not set CONFIG_MICREL_PHY=y CONFIG_REALTEK_PHY=y +CONFIG_CAN_SUN4I=y # CONFIG_WLAN is not set CONFIG_INPUT_EVDEV=y CONFIG_KEYBOARD_SUN4I_LRADC=y diff --git a/arch/arm/configs/tegra_defconfig b/arch/arm/configs/tegra_defconfig index 71400af6cef42df1661fb863d3d31759423f72f1..f32047e24b633eef6dfd26e32a544142ccce42da 100644 --- a/arch/arm/configs/tegra_defconfig +++ b/arch/arm/configs/tegra_defconfig @@ -1,5 +1,5 @@ CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT=y CONFIG_IKCONFIG=y @@ -54,7 +54,6 @@ CONFIG_IPV6_MIP6=y CONFIG_IPV6_TUNNEL=y CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_CAN=y -CONFIG_CAN_MCP251X=y CONFIG_BT=y CONFIG_BT_RFCOMM=y CONFIG_BT_BNEP=y @@ -99,6 +98,7 @@ CONFIG_NETDEVICES=y CONFIG_DUMMY=y CONFIG_IGB=y CONFIG_R8169=y +CONFIG_CAN_MCP251X=y CONFIG_USB_PEGASUS=y CONFIG_USB_USBNET=y CONFIG_USB_NET_SMSC75XX=y diff --git a/arch/arm/configs/vexpress_defconfig b/arch/arm/configs/vexpress_defconfig index 4e3a0133e4d38db2e0b33062dea164dbe4f5cf6d..ac3fd7523698206f4547a308f263c62c45fe1990 100644 --- a/arch/arm/configs/vexpress_defconfig +++ b/arch/arm/configs/vexpress_defconfig @@ -133,8 +133,8 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y # CONFIG_CRYPTO_HW is not set CONFIG_DMA_CMA=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/arm/configs/vt8500_v6_v7_defconfig b/arch/arm/configs/vt8500_v6_v7_defconfig index cb8d38e9562ad4edb0ec04e045f2d706c7af6e50..41607a84abc85adfc9d8d2d520354576e1e274d2 100644 --- a/arch/arm/configs/vt8500_v6_v7_defconfig +++ b/arch/arm/configs/vt8500_v6_v7_defconfig @@ -1,4 +1,4 @@ -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BLK_DEV_INITRD=y CONFIG_ARCH_MULTI_V6=y diff --git a/arch/arm/configs/xcep_defconfig b/arch/arm/configs/xcep_defconfig index 0453948d52eff67af81be370dcb074dd8098e3da..ea59e4b6bfc5e7f48b50f587a0a32453850c37e6 100644 --- a/arch/arm/configs/xcep_defconfig +++ b/arch/arm/configs/xcep_defconfig @@ -1,7 +1,7 @@ CONFIG_LOCALVERSION=".xcep-itech" # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y -CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_IKCONFIG=y diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 149a5bd6b88c195b74c82f01d9044a9b618866b1..3858c4d4cb98854d023075626878bc928062a42d 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -1,92 +1,156 @@ # SPDX-License-Identifier: GPL-2.0 -menuconfig ARM_CRYPTO - bool "ARM Accelerated Cryptographic Algorithms" - depends on ARM +menu "Accelerated Cryptographic Algorithms for CPU (arm)" + +config CRYPTO_CURVE25519_NEON + tristate "Public key crypto: Curve25519 (NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_LIB_CURVE25519_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + help + Curve25519 algorithm + + Architecture: arm with + - NEON (Advanced SIMD) extensions + +config CRYPTO_GHASH_ARM_CE + tristate "Hash functions: GHASH (PMULL/NEON/ARMv8 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_CRYPTD + select CRYPTO_GF128MUL help - Say Y here to choose from a selection of cryptographic algorithms - implemented using ARM specific CPU features or instructions. + GCM GHASH function (NIST SP800-38D) -if ARM_CRYPTO + Architecture: arm using + - PMULL (Polynomial Multiply Long) instructions + - NEON (Advanced SIMD) extensions + - ARMv8 Crypto Extensions + + Use an implementation of GHASH (used by the GCM AEAD chaining mode) + that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) + that is part of the ARMv8 Crypto Extensions, or a slower variant that + uses the vmull.p8 instruction that is part of the basic NEON ISA. + +config CRYPTO_NHPOLY1305_NEON + tristate "Hash functions: NHPoly1305 (NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function (Adiantum) + + Architecture: arm using: + - NEON (Advanced SIMD) extensions + +config CRYPTO_POLY1305_ARM + tristate "Hash functions: Poly1305 (NEON)" + select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: arm optionally using + - NEON (Advanced SIMD) extensions + +config CRYPTO_BLAKE2S_ARM + bool "Hash functions: BLAKE2s" + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: arm + + This is faster than the generic implementations of BLAKE2s and + BLAKE2b, but slower than the NEON implementation of BLAKE2b. + There is no NEON implementation of BLAKE2s, since NEON doesn't + really help with it. + +config CRYPTO_BLAKE2B_NEON + tristate "Hash functions: BLAKE2b (NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_BLAKE2B + help + BLAKE2b cryptographic hash function (RFC 7693) + + Architecture: arm using + - NEON (Advanced SIMD) extensions + + BLAKE2b digest algorithm optimized with ARM NEON instructions. + On ARM processors that have NEON support but not the ARMv8 + Crypto Extensions, typically this BLAKE2b implementation is + much faster than the SHA-2 family and slightly faster than + SHA-1. config CRYPTO_SHA1_ARM - tristate "SHA1 digest algorithm (ARM-asm)" + tristate "Hash functions: SHA-1" select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using optimized ARM assembler. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm config CRYPTO_SHA1_ARM_NEON - tristate "SHA1 digest algorithm (ARM NEON)" + tristate "Hash functions: SHA-1 (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SHA1_ARM select CRYPTO_SHA1 select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using optimized ARM NEON assembly, when NEON instructions are - available. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_SHA1_ARM_CE - tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)" + tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SHA1_ARM select CRYPTO_HASH help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using special ARMv8 Crypto Extensions. + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm using ARMv8 Crypto Extensions config CRYPTO_SHA2_ARM_CE - tristate "SHA-224/256 digest algorithm (ARM v8 Crypto Extensions)" + tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SHA256_ARM select CRYPTO_HASH help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using special ARMv8 Crypto Extensions. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm using + - ARMv8 Crypto Extensions config CRYPTO_SHA256_ARM - tristate "SHA-224/256 digest algorithm (ARM-asm and NEON)" + tristate "Hash functions: SHA-224 and SHA-256 (NEON)" select CRYPTO_HASH depends on !CPU_V7M help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using optimized ARM assembler and NEON, when available. + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_SHA512_ARM - tristate "SHA-384/512 digest algorithm (ARM-asm and NEON)" + tristate "Hash functions: SHA-384 and SHA-512 (NEON)" select CRYPTO_HASH depends on !CPU_V7M help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using optimized ARM assembler and NEON, when available. - -config CRYPTO_BLAKE2S_ARM - bool "BLAKE2s digest algorithm (ARM)" - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - help - BLAKE2s digest algorithm optimized with ARM scalar instructions. This - is faster than the generic implementations of BLAKE2s and BLAKE2b, but - slower than the NEON implementation of BLAKE2b. (There is no NEON - implementation of BLAKE2s, since NEON doesn't really help with it.) + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) -config CRYPTO_BLAKE2B_NEON - tristate "BLAKE2b digest algorithm (ARM NEON)" - depends on KERNEL_MODE_NEON - select CRYPTO_BLAKE2B - help - BLAKE2b digest algorithm optimized with ARM NEON instructions. - On ARM processors that have NEON support but not the ARMv8 - Crypto Extensions, typically this BLAKE2b implementation is - much faster than SHA-2 and slightly faster than SHA-1. + Architecture: arm using + - NEON (Advanced SIMD) extensions config CRYPTO_AES_ARM - tristate "Scalar AES cipher for ARM" + tristate "Ciphers: AES" select CRYPTO_ALGAPI select CRYPTO_AES help - Use optimized AES assembler routines for ARM platforms. + Block ciphers: AES cipher algorithms (FIPS-197) + + Architecture: arm On ARM processors without the Crypto Extensions, this is the fastest AES implementation for single blocks. For multiple @@ -98,7 +162,7 @@ config CRYPTO_AES_ARM such attacks very difficult. config CRYPTO_AES_ARM_BS - tristate "Bit sliced AES using NEON instructions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES @@ -106,8 +170,13 @@ config CRYPTO_AES_ARM_BS select CRYPTO_CBC select CRYPTO_SIMD help - Use a faster and more secure NEON based implementation of AES in CBC, - CTR and XTS modes + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode and for XTS mode encryption, CBC and XTS mode decryption speedup is @@ -116,58 +185,59 @@ config CRYPTO_AES_ARM_BS believed to be invulnerable to cache timing attacks. config CRYPTO_AES_ARM_CE - tristate "Accelerated AES using ARMv8 Crypto Extensions" + tristate "Ciphers: AES, modes: ECB/CBC/CTS/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES select CRYPTO_SIMD help - Use an implementation of AES in CBC, CTR and XTS modes that uses - ARMv8 Crypto Extensions + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - CTS (Cipher Text Stealing) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm using: + - ARMv8 Crypto Extensions -config CRYPTO_GHASH_ARM_CE - tristate "PMULL-accelerated GHASH using NEON/ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_CRYPTD - select CRYPTO_GF128MUL +config CRYPTO_CHACHA20_NEON + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (NEON)" + select CRYPTO_SKCIPHER + select CRYPTO_ARCH_HAVE_LIB_CHACHA help - Use an implementation of GHASH (used by the GCM AEAD chaining mode) - that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) - that is part of the ARMv8 Crypto Extensions, or a slower variant that - uses the vmull.p8 instruction that is part of the basic NEON ISA. + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms -config CRYPTO_CRCT10DIF_ARM_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" - depends on KERNEL_MODE_NEON - depends on CRC_T10DIF - select CRYPTO_HASH + Architecture: arm using: + - NEON (Advanced SIMD) extensions config CRYPTO_CRC32_ARM_CE - tristate "CRC32(C) digest algorithm using CRC and/or PMULL instructions" + tristate "CRC32C and CRC32" depends on KERNEL_MODE_NEON depends on CRC32 select CRYPTO_HASH + help + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + and CRC32 CRC algorithm (IEEE 802.3) -config CRYPTO_CHACHA20_NEON - tristate "NEON and scalar accelerated ChaCha stream cipher algorithms" - select CRYPTO_SKCIPHER - select CRYPTO_ARCH_HAVE_LIB_CHACHA + Architecture: arm using: + - CRC and/or PMULL instructions -config CRYPTO_POLY1305_ARM - tristate "Accelerated scalar and SIMD Poly1305 hash implementations" - select CRYPTO_HASH - select CRYPTO_ARCH_HAVE_LIB_POLY1305 + Drivers: crc32-arm-ce and crc32c-arm-ce -config CRYPTO_NHPOLY1305_NEON - tristate "NEON accelerated NHPoly1305 hash function (for Adiantum)" +config CRYPTO_CRCT10DIF_ARM_CE + tristate "CRCT10DIF" depends on KERNEL_MODE_NEON - select CRYPTO_NHPOLY1305 + depends on CRC_T10DIF + select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) -config CRYPTO_CURVE25519_NEON - tristate "NEON accelerated Curve25519 scalar multiplication library" - depends on KERNEL_MODE_NEON - select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + Architecture: arm using: + - PMULL (Polynomial Multiply Long) instructions + +endmenu -endif diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index 8754c0f5fc9041b99f97b5f298c00be6ea7b2353..c6beb1708c64b5627cf858778bf6fba378a421f0 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -9,7 +9,6 @@ struct dev_archdata { #ifdef CONFIG_ARM_DMA_USE_IOMMU struct dma_iommu_mapping *mapping; #endif - unsigned int dma_coherent:1; unsigned int dma_ops_setup:1; }; diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h deleted file mode 100644 index 4f7bcde03abb501884d029444a6a56fb0d624e72..0000000000000000000000000000000000000000 --- a/arch/arm/include/asm/dma-direct.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/arm/include/asm/dma.h b/arch/arm/include/asm/dma.h index 05f29a72150bee3867992268ca2291432aef438b..c6aded1b069cf04ea2e51e3306613ffa3669a2a6 100644 --- a/arch/arm/include/asm/dma.h +++ b/arch/arm/include/asm/dma.h @@ -106,7 +106,7 @@ extern void set_dma_sg(unsigned int chan, struct scatterlist *sg, int nr_sg); */ extern void __set_dma_addr(unsigned int chan, void *addr); #define set_dma_addr(chan, addr) \ - __set_dma_addr(chan, (void *)__bus_to_virt(addr)) + __set_dma_addr(chan, (void *)isa_bus_to_virt(addr)) /* Set the DMA byte count for this channel * diff --git a/arch/arm/include/asm/efi.h b/arch/arm/include/asm/efi.h index 3088ef72704e598b76f04df7b7c6773a594be5ef..4bdd930167c0f18c2d7dba832b0315a91264ff79 100644 --- a/arch/arm/include/asm/efi.h +++ b/arch/arm/include/asm/efi.h @@ -17,6 +17,7 @@ #ifdef CONFIG_EFI void efi_init(void); +void arm_efi_init(void); int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); int efi_set_mapping_permissions(struct mm_struct *mm, efi_memory_desc_t *md); @@ -37,7 +38,7 @@ void efi_virtmap_load(void); void efi_virtmap_unload(void); #else -#define efi_init() +#define arm_efi_init() #endif /* CONFIG_EFI */ /* arch specific definitions used by the stub code */ diff --git a/arch/arm/include/asm/fpstate.h b/arch/arm/include/asm/fpstate.h index ca42fd9ae0b35ae5ff99dc65b6199d34410f0964..e29d9c7a523883457939c57a8238b872d015a492 100644 --- a/arch/arm/include/asm/fpstate.h +++ b/arch/arm/include/asm/fpstate.h @@ -46,9 +46,6 @@ union vfp_state { struct vfp_hard_struct hard; }; -extern void vfp_flush_thread(union vfp_state *); -extern void vfp_release_thread(union vfp_state *); - #define FP_HARD_SIZE 35 struct fp_hard_struct { diff --git a/arch/arm/include/asm/hardware/dec21285.h b/arch/arm/include/asm/hardware/dec21285.h index 3f18a56a025d7141a4e9ee15db329605da252071..894f2a635cbbd1285e8cfe122bb8cd212496dff0 100644 --- a/arch/arm/include/asm/hardware/dec21285.h +++ b/arch/arm/include/asm/hardware/dec21285.h @@ -22,6 +22,13 @@ #define DC21285_IO(x) (x) #endif +/* + * The footbridge is programmed to expose the system RAM at 0xe0000000. + * The requirement is that the RAM isn't placed at bus address 0, which + * would clash with VGA cards. + */ +#define BUS_OFFSET 0xe0000000 + #define CSR_PCICMD DC21285_IO(0x0004) #define CSR_CLASSREV DC21285_IO(0x0008) #define CSR_PCICACHELINESIZE DC21285_IO(0x000c) @@ -81,19 +88,6 @@ #define SA110_CNTL_XCSDIR(x) ((x)<<28) #define SA110_CNTL_PCICFN (1 << 31) -/* - * footbridge_cfn_mode() is used when we want - * to check whether we are the central function - */ -#define __footbridge_cfn_mode() (*CSR_SA110_CNTL & SA110_CNTL_PCICFN) -#if defined(CONFIG_FOOTBRIDGE_HOST) && defined(CONFIG_FOOTBRIDGE_ADDIN) -#define footbridge_cfn_mode() __footbridge_cfn_mode() -#elif defined(CONFIG_FOOTBRIDGE_HOST) -#define footbridge_cfn_mode() (1) -#else -#define footbridge_cfn_mode() (0) -#endif - #define CSR_PCIADDR_EXTN DC21285_IO(0x0140) #define CSR_PREFETCHMEMRANGE DC21285_IO(0x0144) #define CSR_XBUS_CYCLE DC21285_IO(0x0148) diff --git a/arch/arm/include/asm/mach/dma.h b/arch/arm/include/asm/mach/dma.h index 1506422af3832201938fa4e7265a71fad7e4d20e..5ec11d7f0d0453ee5d4b548f1a661c2097d193ae 100644 --- a/arch/arm/include/asm/mach/dma.h +++ b/arch/arm/include/asm/mach/dma.h @@ -44,8 +44,3 @@ struct dma_struct { * isa_dma_add - add an ISA-style DMA channel */ extern int isa_dma_add(unsigned int, dma_t *dma); - -/* - * Add the ISA DMA controller. Always takes channels 0-7. - */ -extern void isa_init_dma(void); diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index a55a9038abc891513b619c12527b3b5cdda8ff17..d8eef4bd8c711dc4f230df9ebb2a99cce8ef4d04 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -369,17 +369,6 @@ static inline unsigned long __virt_to_idmap(unsigned long x) #define virt_to_idmap(x) __virt_to_idmap((unsigned long)(x)) -/* - * Virtual <-> DMA view memory address translations - * Again, these are *only* valid on the kernel direct mapped RAM - * memory. Use of these is *deprecated* (and that doesn't mean - * use the __ prefixed forms instead.) See dma-mapping.h. - */ -#ifndef __virt_to_bus -#define __virt_to_bus __virt_to_phys -#define __bus_to_virt __phys_to_virt -#endif - /* * Conversion between a struct page and a physical address. * diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index bdc35c0e8dfb9482ff20759d5373c15cbc95d86b..326864f79d18fe035a1e3b499b7185f8ca53caa7 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -81,9 +81,6 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset, /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - unsigned long __get_wchan(struct task_struct *p); #define task_pt_regs(p) \ diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h index 3e78f921b8b2dc460441a001fb3a5473e2133c71..36b2ff44fcbbc924fe25b50e996c8c72e3cf4c57 100644 --- a/arch/arm/include/asm/stacktrace.h +++ b/arch/arm/include/asm/stacktrace.h @@ -21,6 +21,9 @@ struct stackframe { struct llist_node *kr_cur; struct task_struct *tsk; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + bool ex_frame; +#endif }; static __always_inline @@ -34,6 +37,9 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame) frame->kr_cur = NULL; frame->tsk = current; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + frame->ex_frame = in_entry_text(frame->pc); +#endif } extern int unwind_frame(struct stackframe *frame); @@ -41,5 +47,7 @@ extern void walk_stackframe(struct stackframe *frame, int (*fn)(struct stackframe *, void *), void *data); extern void dump_mem(const char *lvl, const char *str, unsigned long bottom, unsigned long top); +extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl); #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 553866751e1a5ce66c805c083e392c1f0646088e..48737ec800eb010356e18e586815d86b71840ff7 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -45,7 +45,6 @@ obj-$(CONFIG_ISA_DMA_API) += dma.o obj-$(CONFIG_FIQ) += fiq.o fiqasm.o obj-$(CONFIG_MODULES) += armksyms.o module.o obj-$(CONFIG_ARM_MODULE_PLTS) += module-plts.o -obj-$(CONFIG_ISA_DMA) += dma-isa.o obj-$(CONFIG_PCI) += bios32.o isa.o obj-$(CONFIG_ARM_CPU_SUSPEND) += sleep.o suspend.o obj-$(CONFIG_HIBERNATION) += hibernate.o @@ -89,7 +88,7 @@ obj-$(CONFIG_VDSO) += vdso.o obj-$(CONFIG_EFI) += efi.o obj-$(CONFIG_PARAVIRT) += paravirt.o -head-y := head$(MMUEXT).o +obj-y += head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_ARM_PATCH_PHYS_VIRT) += phys2virt.o @@ -109,4 +108,4 @@ obj-$(CONFIG_HAVE_ARM_SMCCC) += smccc-call.o obj-$(CONFIG_GENERIC_CPU_VULNERABILITIES) += spectre.o -extra-y := $(head-y) vmlinux.lds +extra-y := vmlinux.lds diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 02839d8b6202027ad34743c2537268075dc61f71..264827281113b02cc66b03e4a6a752607eb95ef6 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -194,14 +194,12 @@ const struct machine_desc * __init setup_machine_fdt(void *dt_virt) { const struct machine_desc *mdesc, *mdesc_best = NULL; -#if defined(CONFIG_ARCH_MULTIPLATFORM) || defined(CONFIG_ARM_SINGLE_ARMV7M) DT_MACHINE_START(GENERIC_DT, "Generic DT based system") .l2c_aux_val = 0x0, .l2c_aux_mask = ~0x0, MACHINE_END mdesc_best = &__mach_desc_GENERIC_DT; -#endif if (!dt_virt || !early_init_dt_verify(dt_virt)) return NULL; diff --git a/arch/arm/kernel/efi.c b/arch/arm/kernel/efi.c index e57dbcc89123e1ae515fe10557fc13d3f941338e..e50ad7eefc02a6eb08b2736644f4724fa3aad2ee 100644 --- a/arch/arm/kernel/efi.c +++ b/arch/arm/kernel/efi.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -73,3 +74,81 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) return efi_set_mapping_permissions(mm, md); return 0; } + +static unsigned long __initdata screen_info_table = EFI_INVALID_TABLE_ADDR; +static unsigned long __initdata cpu_state_table = EFI_INVALID_TABLE_ADDR; + +const efi_config_table_type_t efi_arch_tables[] __initconst = { + {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table}, + {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table}, + {} +}; + +static void __init load_screen_info_table(void) +{ + struct screen_info *si; + + if (screen_info_table != EFI_INVALID_TABLE_ADDR) { + si = early_memremap_ro(screen_info_table, sizeof(*si)); + if (!si) { + pr_err("Could not map screen_info config table\n"); + return; + } + screen_info = *si; + early_memunmap(si, sizeof(*si)); + + /* dummycon on ARM needs non-zero values for columns/lines */ + screen_info.orig_video_cols = 80; + screen_info.orig_video_lines = 25; + + if (memblock_is_map_memory(screen_info.lfb_base)) + memblock_mark_nomap(screen_info.lfb_base, + screen_info.lfb_size); + } +} + +static void __init load_cpu_state_table(void) +{ + if (cpu_state_table != EFI_INVALID_TABLE_ADDR) { + struct efi_arm_entry_state *state; + bool dump_state = true; + + state = early_memremap_ro(cpu_state_table, + sizeof(struct efi_arm_entry_state)); + if (state == NULL) { + pr_warn("Unable to map CPU entry state table.\n"); + return; + } + + if ((state->sctlr_before_ebs & 1) == 0) + pr_warn(FW_BUG "EFI stub was entered with MMU and Dcache disabled, please fix your firmware!\n"); + else if ((state->sctlr_after_ebs & 1) == 0) + pr_warn(FW_BUG "ExitBootServices() returned with MMU and Dcache disabled, please fix your firmware!\n"); + else + dump_state = false; + + if (dump_state || efi_enabled(EFI_DBG)) { + pr_info("CPSR at EFI stub entry : 0x%08x\n", + state->cpsr_before_ebs); + pr_info("SCTLR at EFI stub entry : 0x%08x\n", + state->sctlr_before_ebs); + pr_info("CPSR after ExitBootServices() : 0x%08x\n", + state->cpsr_after_ebs); + pr_info("SCTLR after ExitBootServices(): 0x%08x\n", + state->sctlr_after_ebs); + } + early_memunmap(state, sizeof(struct efi_arm_entry_state)); + } +} + +void __init arm_efi_init(void) +{ + efi_init(); + + load_screen_info_table(); + + /* ARM does not permit early mappings to persist across paging_init() */ + efi_memmap_unmap(); + + load_cpu_state_table(); +} diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index 034cb48c9eeb8c7e541124d9eb34d8b40cc21d63..fe28fc1f759d9372ef6f44e43a0b959d67579c8d 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -70,7 +70,7 @@ static void __init init_irq_stacks(void) } } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK static void ____do_softirq(void *arg) { __do_softirq(); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 3d9cace638840137622a8c3f4e37209c74b99071..a2b31d91a1b6e32091edd704310376112d9fbda2 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -201,7 +201,7 @@ void __show_regs(struct pt_regs *regs) void show_regs(struct pt_regs * regs) { __show_regs(regs); - dump_stack(); + dump_backtrace(regs, NULL, KERN_DEFAULT); } ATOMIC_NOTIFIER_HEAD(thread_notify_head); @@ -232,10 +232,6 @@ void flush_thread(void) thread_notify(THREAD_NOTIFY_FLUSH, thread); } -void release_thread(struct task_struct *dead_task) -{ -} - asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) @@ -375,7 +371,7 @@ static unsigned long sigpage_addr(const struct mm_struct *mm, slots = ((last - first) >> PAGE_SHIFT) + 1; - offset = get_random_int() % slots; + offset = prandom_u32_max(slots); addr = first + (offset << PAGE_SHIFT); diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c index 8aac1e10b117aa45393a6e2df86130c62b7426fa..38f1ea9c724d5fd4bc813c389b1d03f0bb3e9820 100644 --- a/arch/arm/kernel/return_address.c +++ b/arch/arm/kernel/return_address.c @@ -47,6 +47,7 @@ here: frame.kr_cur = NULL; frame.tsk = current; #endif + frame.ex_frame = false; walk_stackframe(&frame, save_return_addr, &data); diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 1e8a50a97edf2e49dc2075b5b119a5d561010b86..cb88c6e69377319e1534776d8aeae35b979c56e4 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -1141,7 +1141,7 @@ void __init setup_arch(char **cmdline_p) #endif setup_dma_zone(mdesc); xen_early_init(); - efi_init(); + arm_efi_init(); /* * Make sure the calculation for lowmem/highmem is set appropriately * before reserving/allocating any memory diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index ea128e32e8ca8f28ae7ea8805bb986e78c7d6074..e07f359254c3cca23352984055a98a5e21230220 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -655,7 +655,7 @@ struct page *get_signal_page(void) PAGE_SIZE / sizeof(u32)); /* Give the signal return code some randomness */ - offset = 0x200 + (get_random_int() & 0x7fc); + offset = 0x200 + (get_random_u16() & 0x7fc); signal_return_offset = offset; /* Copy signal return handlers into the page */ diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c index d0fa2037460ac7906ab5bef26e5e1ad518e15d1c..85443b5d1922151e7d9bb4ac6cb38da11a626740 100644 --- a/arch/arm/kernel/stacktrace.c +++ b/arch/arm/kernel/stacktrace.c @@ -9,6 +9,8 @@ #include #include +#include "reboot.h" + #if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) /* * Unwind the current stack frame and store the new register values in the @@ -39,29 +41,74 @@ * Note that with framepointer enabled, even the leaf functions have the same * prologue and epilogue, therefore we can ignore the LR value in this case. */ -int notrace unwind_frame(struct stackframe *frame) + +extern unsigned long call_with_stack_end; + +static int frame_pointer_check(struct stackframe *frame) { unsigned long high, low; unsigned long fp = frame->fp; + unsigned long pc = frame->pc; + + /* + * call_with_stack() is the only place we allow SP to jump from one + * stack to another, with FP and SP pointing to different stacks, + * skipping the FP boundary check at this point. + */ + if (pc >= (unsigned long)&call_with_stack && + pc < (unsigned long)&call_with_stack_end) + return 0; /* only go to a higher address on the stack */ low = frame->sp; high = ALIGN(low, THREAD_SIZE); -#ifdef CONFIG_CC_IS_CLANG /* check current frame pointer is within bounds */ +#ifdef CONFIG_CC_IS_CLANG if (fp < low + 4 || fp > high - 4) return -EINVAL; - - frame->sp = frame->fp; - frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); - frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4)); #else - /* check current frame pointer is within bounds */ if (fp < low + 12 || fp > high - 4) return -EINVAL; +#endif + + return 0; +} + +int notrace unwind_frame(struct stackframe *frame) +{ + unsigned long fp = frame->fp; + + if (frame_pointer_check(frame)) + return -EINVAL; + + /* + * When we unwind through an exception stack, include the saved PC + * value into the stack trace. + */ + if (frame->ex_frame) { + struct pt_regs *regs = (struct pt_regs *)frame->sp; + + /* + * We check that 'regs + sizeof(struct pt_regs)' (that is, + * ®s[1]) does not exceed the bottom of the stack to avoid + * accessing data outside the task's stack. This may happen + * when frame->ex_frame is a false positive. + */ + if ((unsigned long)®s[1] > ALIGN(frame->sp, THREAD_SIZE)) + return -EINVAL; + + frame->pc = regs->ARM_pc; + frame->ex_frame = false; + return 0; + } /* restore the registers from the stack frame */ +#ifdef CONFIG_CC_IS_CLANG + frame->sp = frame->fp; + frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp)); + frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4)); +#else frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 12)); frame->sp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 8)); frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 4)); @@ -72,6 +119,9 @@ int notrace unwind_frame(struct stackframe *frame) (void *)frame->fp, &frame->kr_cur); #endif + if (in_entry_text(frame->pc)) + frame->ex_frame = true; + return 0; } #endif @@ -102,7 +152,6 @@ static int save_trace(struct stackframe *frame, void *d) { struct stack_trace_data *data = d; struct stack_trace *trace = data->trace; - struct pt_regs *regs; unsigned long addr = frame->pc; if (data->no_sched_functions && in_sched_functions(addr)) @@ -113,19 +162,6 @@ static int save_trace(struct stackframe *frame, void *d) } trace->entries[trace->nr_entries++] = addr; - - if (trace->nr_entries >= trace->max_entries) - return 1; - - if (!in_entry_text(frame->pc)) - return 0; - - regs = (struct pt_regs *)frame->sp; - if ((unsigned long)®s[1] > ALIGN(frame->sp, THREAD_SIZE)) - return 0; - - trace->entries[trace->nr_entries++] = regs->ARM_pc; - return trace->nr_entries >= trace->max_entries; } @@ -167,6 +203,9 @@ here: frame.kr_cur = NULL; frame.tsk = tsk; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + frame.ex_frame = false; +#endif walk_stackframe(&frame, save_trace, &data); } @@ -188,6 +227,9 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace) frame.kr_cur = NULL; frame.tsk = current; #endif +#ifdef CONFIG_UNWINDER_FRAME_POINTER + frame.ex_frame = in_entry_text(frame.pc); +#endif walk_stackframe(&frame, save_trace, &data); } diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 9283dc65be31b7d3e7ef0a3c54f8a25938fb15c3..20b2db6dcd1ced712c8627cffd629751fdcbe7ff 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -205,14 +205,14 @@ static void dump_instr(const char *lvl, struct pt_regs *regs) } #ifdef CONFIG_ARM_UNWIND -static inline void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, - const char *loglvl) +void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl) { unwind_backtrace(regs, tsk, loglvl); } #else -static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, - const char *loglvl) +void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, + const char *loglvl) { unsigned int fp, mode; int ok = 1; @@ -487,7 +487,7 @@ asmlinkage void do_undefinstr(struct pt_regs *regs) die_sig: #ifdef CONFIG_DEBUG_USER if (user_debug & UDBG_UNDEFINED) { - pr_info("%s (%d): undefined instruction: pc=%p\n", + pr_info("%s (%d): undefined instruction: pc=%px\n", current->comm, task_pid_nr(current), pc); __show_regs(regs); dump_instr(KERN_INFO, regs); @@ -920,9 +920,9 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs) { unsigned long tsk_stk = (unsigned long)current->stack; #ifdef CONFIG_IRQSTACKS - unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); + unsigned long irq_stk = (unsigned long)raw_cpu_read(irq_stack_ptr); #endif - unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr); + unsigned long ovf_stk = (unsigned long)raw_cpu_read(overflow_stack_ptr); console_verbose(); pr_emerg("Insufficient stack space to handle exception!"); diff --git a/arch/arm/lib/call_with_stack.S b/arch/arm/lib/call_with_stack.S index 0a268a6c513c8f2f369264885181be08f0678ff0..5030d4e8d1267ee006880a61cdaeae7dc154f695 100644 --- a/arch/arm/lib/call_with_stack.S +++ b/arch/arm/lib/call_with_stack.S @@ -46,4 +46,6 @@ UNWIND( .setfp fpreg, sp ) pop {fpreg, pc} UNWIND( .fnend ) #endif + .globl call_with_stack_end +call_with_stack_end: ENDPROC(call_with_stack) diff --git a/arch/arm/mach-aspeed/Kconfig b/arch/arm/mach-aspeed/Kconfig index cd8a15be0724d96a2b590d550dde0d0b90e335b2..080019aa6fcd89ef706f572477d1f72f9f90c8e9 100644 --- a/arch/arm/mach-aspeed/Kconfig +++ b/arch/arm/mach-aspeed/Kconfig @@ -19,9 +19,9 @@ config MACH_ASPEED_G4 select PINCTRL_ASPEED_G4 select FTTMR010_TIMER help - Say yes if you intend to run on an Aspeed ast2400 or similar - fourth generation BMCs, such as those used by OpenPower Power8 - systems. + Say yes if you intend to run on an Aspeed ast2400 or similar + fourth generation BMCs, such as those used by OpenPower Power8 + systems. config MACH_ASPEED_G5 bool "Aspeed SoC 5th Generation" @@ -29,8 +29,8 @@ config MACH_ASPEED_G5 select PINCTRL_ASPEED_G5 select FTTMR010_TIMER help - Say yes if you intend to run on an Aspeed ast2500 or similar - fifth generation Aspeed BMCs. + Say yes if you intend to run on an Aspeed ast2500 or similar + fifth generation Aspeed BMCs. config MACH_ASPEED_G6 bool "Aspeed SoC 6th Generation" @@ -40,7 +40,7 @@ config MACH_ASPEED_G6 select ARM_GIC select HAVE_ARM_ARCH_TIMER help - Say yes if you intend to run on an Aspeed ast2600 or similar - sixth generation Aspeed BMCs. + Say yes if you intend to run on an Aspeed ast2600 or similar + sixth generation Aspeed BMCs. endif diff --git a/arch/arm/mach-at91/Makefile.boot b/arch/arm/mach-at91/Makefile.boot deleted file mode 100644 index 5dde7328a7a95b375de473365934fe49e72baf9d..0000000000000000000000000000000000000000 --- a/arch/arm/mach-at91/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Empty file waiting for deletion once Makefile.boot isn't needed any more. -# Patch waits for application at -# https://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 . diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index df6d673e83d563ec3423aef9e8d84fb78067a6f9..60dc56d8acfb9aa08a289077e35e959621e2a1f3 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -19,8 +19,6 @@ #include #include -#include - #include #include #include @@ -541,9 +539,41 @@ extern u32 at91_pm_suspend_in_sram_sz; static int at91_suspend_finish(unsigned long val) { + unsigned char modified_gray_code[] = { + 0x00, 0x01, 0x02, 0x03, 0x06, 0x07, 0x04, 0x05, 0x0c, 0x0d, + 0x0e, 0x0f, 0x0a, 0x0b, 0x08, 0x09, 0x18, 0x19, 0x1a, 0x1b, + 0x1e, 0x1f, 0x1c, 0x1d, 0x14, 0x15, 0x16, 0x17, 0x12, 0x13, + 0x10, 0x11, + }; + unsigned int tmp, index; int i; if (soc_pm.data.mode == AT91_PM_BACKUP && soc_pm.data.ramc_phy) { + /* + * Bootloader will perform DDR recalibration and will try to + * restore the ZQ0SR0 with the value saved here. But the + * calibration is buggy and restoring some values from ZQ0SR0 + * is forbidden and risky thus we need to provide processed + * values for these (modified gray code values). + */ + tmp = readl(soc_pm.data.ramc_phy + DDR3PHY_ZQ0SR0); + + /* Store pull-down output impedance select. */ + index = (tmp >> DDR3PHY_ZQ0SR0_PDO_OFF) & 0x1f; + soc_pm.bu->ddr_phy_calibration[0] = modified_gray_code[index]; + + /* Store pull-up output impedance select. */ + index = (tmp >> DDR3PHY_ZQ0SR0_PUO_OFF) & 0x1f; + soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + + /* Store pull-down on-die termination impedance select. */ + index = (tmp >> DDR3PHY_ZQ0SR0_PDODT_OFF) & 0x1f; + soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + + /* Store pull-up on-die termination impedance select. */ + index = (tmp >> DDR3PHY_ZQ0SRO_PUODT_OFF) & 0x1f; + soc_pm.bu->ddr_phy_calibration[0] |= modified_gray_code[index]; + /* * The 1st 8 words of memory might get corrupted in the process * of DDR PHY recalibration; it is saved here in securam and it @@ -624,16 +654,6 @@ static int at91_pm_enter(suspend_state_t state) if (ret) return ret; -#ifdef CONFIG_PINCTRL_AT91 - /* - * FIXME: this is needed to communicate between the pinctrl driver and - * the PM implementation in the machine. Possibly part of the PM - * implementation should be moved down into the pinctrl driver and get - * called as part of the generic suspend/resume path. - */ - at91_pinctrl_gpio_suspend(); -#endif - switch (state) { case PM_SUSPEND_MEM: case PM_SUSPEND_STANDBY: @@ -658,9 +678,6 @@ static int at91_pm_enter(suspend_state_t state) } error: -#ifdef CONFIG_PINCTRL_AT91 - at91_pinctrl_gpio_resume(); -#endif at91_pm_config_quirks(false); return 0; } @@ -1066,10 +1083,6 @@ static int __init at91_pm_backup_init(void) of_scan_flat_dt(at91_pm_backup_scan_memcs, &located); if (!located) goto securam_fail; - - /* DDR3PHY_ZQ0SR0 */ - soc_pm.bu->ddr_phy_calibration[0] = readl(soc_pm.data.ramc_phy + - 0x188); } return 0; diff --git a/arch/arm/mach-at91/pm_suspend.S b/arch/arm/mach-at91/pm_suspend.S index abe4ced33edafbbf5f91b4a69e4a6f76ce978ff1..ffed4d9490428b30d1f87f5a5d82d6a1a7ce56fb 100644 --- a/arch/arm/mach-at91/pm_suspend.S +++ b/arch/arm/mach-at91/pm_suspend.S @@ -172,9 +172,15 @@ sr_ena_2: /* Put DDR PHY's DLL in bypass mode for non-backup modes. */ cmp r7, #AT91_PM_BACKUP beq sr_ena_3 - ldr tmp1, [r3, #DDR3PHY_PIR] - orr tmp1, tmp1, #DDR3PHY_PIR_DLLBYP - str tmp1, [r3, #DDR3PHY_PIR] + + /* Disable DX DLLs. */ + ldr tmp1, [r3, #DDR3PHY_DX0DLLCR] + orr tmp1, tmp1, #DDR3PHY_DXDLLCR_DLLDIS + str tmp1, [r3, #DDR3PHY_DX0DLLCR] + + ldr tmp1, [r3, #DDR3PHY_DX1DLLCR] + orr tmp1, tmp1, #DDR3PHY_DXDLLCR_DLLDIS + str tmp1, [r3, #DDR3PHY_DX1DLLCR] sr_ena_3: /* Power down DDR PHY data receivers. */ @@ -221,10 +227,14 @@ sr_ena_3: bic tmp1, tmp1, #DDR3PHY_DSGCR_ODTPDD_ODT0 str tmp1, [r3, #DDR3PHY_DSGCR] - /* Take DDR PHY's DLL out of bypass mode. */ - ldr tmp1, [r3, #DDR3PHY_PIR] - bic tmp1, tmp1, #DDR3PHY_PIR_DLLBYP - str tmp1, [r3, #DDR3PHY_PIR] + /* Enable DX DLLs. */ + ldr tmp1, [r3, #DDR3PHY_DX0DLLCR] + bic tmp1, tmp1, #DDR3PHY_DXDLLCR_DLLDIS + str tmp1, [r3, #DDR3PHY_DX0DLLCR] + + ldr tmp1, [r3, #DDR3PHY_DX1DLLCR] + bic tmp1, tmp1, #DDR3PHY_DXDLLCR_DLLDIS + str tmp1, [r3, #DDR3PHY_DX1DLLCR] /* Enable quasi-dynamic programming. */ mov tmp1, #0 diff --git a/arch/arm/mach-davinci/Makefile.boot b/arch/arm/mach-davinci/Makefile.boot deleted file mode 100644 index d36b251f325ba98184f4a0cfa8f8308cf8fd078a..0000000000000000000000000000000000000000 --- a/arch/arm/mach-davinci/Makefile.boot +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -zreladdr-$(CONFIG_ARCH_DAVINCI_DA8XX) += 0xc0008000 -params_phys-$(CONFIG_ARCH_DAVINCI_DA8XX) := 0xc0000100 -initrd_phys-$(CONFIG_ARCH_DAVINCI_DA8XX) := 0xc0800000 - -zreladdr-$(CONFIG_ARCH_DAVINCI_DMx) += 0x80008000 -params_phys-$(CONFIG_ARCH_DAVINCI_DMx) := 0x80000100 -initrd_phys-$(CONFIG_ARCH_DAVINCI_DMx) := 0x80800000 diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c index 92d74bc71967dd6e9310e6b1f189dfdf3616e2e5..d752ee2b30ff94919020e4343de9a1af5d4aeffb 100644 --- a/arch/arm/mach-davinci/board-da850-evm.c +++ b/arch/arm/mach-davinci/board-da850-evm.c @@ -516,8 +516,8 @@ exp_setup_sela_fail: return ret; } -static int da850_evm_ui_expander_teardown(struct i2c_client *client, - unsigned gpio, unsigned ngpio, void *c) +static void da850_evm_ui_expander_teardown(struct i2c_client *client, + unsigned gpio, unsigned ngpio, void *c) { platform_device_unregister(&da850_evm_ui_keys_device); @@ -529,8 +529,6 @@ static int da850_evm_ui_expander_teardown(struct i2c_client *client, gpio_free(gpio + DA850_EVM_UI_EXP_SEL_C); gpio_free(gpio + DA850_EVM_UI_EXP_SEL_B); gpio_free(gpio + DA850_EVM_UI_EXP_SEL_A); - - return 0; } /* assign the baseboard expander's GPIOs after the UI board's */ @@ -697,13 +695,11 @@ io_exp_setup_sw_fail: return ret; } -static int da850_evm_bb_expander_teardown(struct i2c_client *client, - unsigned gpio, unsigned ngpio, void *c) +static void da850_evm_bb_expander_teardown(struct i2c_client *client, + unsigned gpio, unsigned ngpio, void *c) { platform_device_unregister(&da850_evm_bb_leds_device); platform_device_unregister(&da850_evm_bb_keys_device); - - return 0; } static struct pca953x_platform_data da850_evm_ui_expander_info = { diff --git a/arch/arm/mach-dove/Makefile b/arch/arm/mach-dove/Makefile index e83f6492834d4ef2b7dedaf7c53c1ec25331c019..da373a5768bae513ec18be64a607a0bd59d7ba5d 100644 --- a/arch/arm/mach-dove/Makefile +++ b/arch/arm/mach-dove/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include +ccflags-y := -I$(srctree)/arch/arm/plat-orion/include obj-y += common.o obj-$(CONFIG_DOVE_LEGACY) += irq.o mpp.o diff --git a/arch/arm/mach-dove/Makefile.boot b/arch/arm/mach-dove/Makefile.boot deleted file mode 100644 index e4dd1d26038f0a0e042c89af0e6dd67e7b333add..0000000000000000000000000000000000000000 --- a/arch/arm/mach-dove/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x00008000 -params_phys-y := 0x00000100 -initrd_phys-y := 0x00800000 diff --git a/arch/arm/mach-ep93xx/Makefile.boot b/arch/arm/mach-ep93xx/Makefile.boot deleted file mode 100644 index 4c0a039a5027241fb8f973c4c9823f0009e00cc1..0000000000000000000000000000000000000000 --- a/arch/arm/mach-ep93xx/Makefile.boot +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Empty file waiting for deletion once Makefile.boot isn't needed any more. diff --git a/arch/arm/mach-ep93xx/core.c b/arch/arm/mach-ep93xx/core.c index 2d58e273c96dd9e874f15b38b129a959e4caafb7..95e731676cea4f1e84d300e245ce3655eb8010c0 100644 --- a/arch/arm/mach-ep93xx/core.c +++ b/arch/arm/mach-ep93xx/core.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm/mach-footbridge/Kconfig b/arch/arm/mach-footbridge/Kconfig index acc10b1caa694f26650155aaec7a9d4a916a61df..b5e7cbfed11963c402ab70a7d2580d10447f145b 100644 --- a/arch/arm/mach-footbridge/Kconfig +++ b/arch/arm/mach-footbridge/Kconfig @@ -1,38 +1,37 @@ # SPDX-License-Identifier: GPL-2.0-only -if ARCH_FOOTBRIDGE +menuconfig ARCH_FOOTBRIDGE + bool "FootBridge Implementations" + depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5) + depends on !(ARCH_MOXART || ARCH_GEMINI || ARCH_SA1100) + depends on ATAGS + depends on CPU_LITTLE_ENDIAN + depends on MMU + select ARCH_NO_SG_CHAIN + select CPU_SA110 + select FOOTBRIDGE + select NEED_MACH_MEMORY_H + help + Support for systems based on the DC21285 companion chip + ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder. -menu "Footbridge Implementations" +if ARCH_FOOTBRIDGE config ARCH_CATS bool "CATS" depends on UNUSED_BOARD_FILES select CLKEVT_I8253 select CLKSRC_I8253 - select FOOTBRIDGE_HOST select ISA - select ISA_DMA select FORCE_PCI help Say Y here if you intend to run this kernel on the CATS. Saying N will reduce the size of the Footbridge kernel. -config ARCH_EBSA285_ADDIN - bool "EBSA285 (addin mode)" - select ARCH_EBSA285 - select FOOTBRIDGE_ADDIN - help - Say Y here if you intend to run this kernel on the EBSA285 card - in addin mode. - - Saying N will reduce the size of the Footbridge kernel. - config ARCH_EBSA285_HOST bool "EBSA285 (host mode)" select ARCH_EBSA285 - select FOOTBRIDGE_HOST select ISA - select ISA_DMA select ARCH_MAY_HAVE_PC_FDC select FORCE_PCI help @@ -45,9 +44,7 @@ config ARCH_NETWINDER bool "NetWinder" select CLKEVT_I8253 select CLKSRC_I8253 - select FOOTBRIDGE_HOST select ISA - select ISA_DMA select FORCE_PCI help Say Y here if you intend to run this kernel on the Rebel.COM @@ -57,23 +54,12 @@ config ARCH_NETWINDER Saying N will reduce the size of the Footbridge kernel. -endmenu - # Footbridge support config FOOTBRIDGE - select ARCH_HAS_PHYS_TO_DMA - bool - -# Footbridge in host mode -config FOOTBRIDGE_HOST - bool + def_bool y select ARCH_MIGHT_HAVE_PC_SERIO + select ISA_DMA_API -# Footbridge in addin mode -config FOOTBRIDGE_ADDIN - bool - -# EBSA285 board in either host or addin mode config ARCH_EBSA285 bool diff --git a/arch/arm/mach-footbridge/Makefile b/arch/arm/mach-footbridge/Makefile index 6262993c05558592ff9c3d6467bb6bd4f950b6e3..55d570739f19847a61c4190d04aa54d8b89a6204 100644 --- a/arch/arm/mach-footbridge/Makefile +++ b/arch/arm/mach-footbridge/Makefile @@ -5,11 +5,11 @@ # Object file lists. -obj-y := common.o dma.o isa-irq.o +obj-y := common.o isa-irq.o isa.o isa-rtc.o dma-isa.o pci-y += dc21285.o pci-$(CONFIG_ARCH_CATS) += cats-pci.o -pci-$(CONFIG_ARCH_EBSA285_HOST) += ebsa285-pci.o +pci-$(CONFIG_ARCH_EBSA285) += ebsa285-pci.o pci-$(CONFIG_ARCH_NETWINDER) += netwinder-pci.o obj-$(CONFIG_ARCH_CATS) += cats-hw.o isa-timer.o @@ -18,4 +18,3 @@ obj-$(CONFIG_ARCH_NETWINDER) += netwinder-hw.o isa-timer.o obj-$(CONFIG_PCI) +=$(pci-y) -obj-$(CONFIG_ISA) += isa.o isa-rtc.o diff --git a/arch/arm/mach-footbridge/Makefile.boot b/arch/arm/mach-footbridge/Makefile.boot deleted file mode 100644 index e4313e912cacb5130224db01967f88375dbdb930..0000000000000000000000000000000000000000 --- a/arch/arm/mach-footbridge/Makefile.boot +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x00008000 -params_phys-y := 0x00000100 -initrd_phys-y := 0x00800000 - diff --git a/arch/arm/mach-footbridge/common.c b/arch/arm/mach-footbridge/common.c index 5020eb96b025dd3562fc03b069dcfdcb0d739b11..629e4676ed776059e6931fe39078367dde207ca4 100644 --- a/arch/arm/mach-footbridge/common.c +++ b/arch/arm/mach-footbridge/common.c @@ -198,9 +198,6 @@ void __init footbridge_init_irq(void) __fb_init_irq(); - if (!footbridge_cfn_mode()) - return; - if (machine_is_ebsa285()) /* The following is dependent on which slot * you plug the Southbridge card into. We @@ -221,21 +218,13 @@ void __init footbridge_init_irq(void) * commented out since there is a "No Fix" problem with it. Not mapping * it means that we have extra bullet protection on our feet. */ -static struct map_desc fb_common_io_desc[] __initdata = { +static struct map_desc ebsa285_host_io_desc[] __initdata = { { .virtual = ARMCSR_BASE, .pfn = __phys_to_pfn(DC21285_ARMCSR_BASE), .length = ARMCSR_SIZE, .type = MT_DEVICE, - } -}; - -/* - * The mapping when the footbridge is in host mode. We don't map any of - * this when we are in add-in mode. - */ -static struct map_desc ebsa285_host_io_desc[] __initdata = { -#if defined(CONFIG_ARCH_FOOTBRIDGE) && defined(CONFIG_FOOTBRIDGE_HOST) + }, { .virtual = PCIMEM_BASE, .pfn = __phys_to_pfn(DC21285_PCI_MEM), @@ -257,26 +246,12 @@ static struct map_desc ebsa285_host_io_desc[] __initdata = { .length = PCIIACK_SIZE, .type = MT_DEVICE, }, -#endif }; void __init footbridge_map_io(void) { - /* - * Set up the common mapping first; we need this to - * determine whether we're in host mode or not. - */ - iotable_init(fb_common_io_desc, ARRAY_SIZE(fb_common_io_desc)); - - /* - * Now, work out what we've got to map in addition on this - * platform. - */ - if (footbridge_cfn_mode()) { - iotable_init(ebsa285_host_io_desc, ARRAY_SIZE(ebsa285_host_io_desc)); - pci_map_io_early(__phys_to_pfn(DC21285_PCI_IO)); - } - + iotable_init(ebsa285_host_io_desc, ARRAY_SIZE(ebsa285_host_io_desc)); + pci_map_io_early(__phys_to_pfn(DC21285_PCI_IO)); vga_base = PCIMEM_BASE; } @@ -306,49 +281,3 @@ void footbridge_restart(enum reboot_mode mode, const char *cmd) *CSR_SA110_CNTL |= (1 << 13); } } - -#ifdef CONFIG_FOOTBRIDGE_ADDIN - -static inline unsigned long fb_bus_sdram_offset(void) -{ - return *CSR_PCISDRAMBASE & 0xfffffff0; -} - -/* - * These two functions convert virtual addresses to PCI addresses and PCI - * addresses to virtual addresses. Note that it is only legal to use these - * on memory obtained via get_zeroed_page or kmalloc. - */ -unsigned long __virt_to_bus(unsigned long res) -{ - WARN_ON(res < PAGE_OFFSET || res >= (unsigned long)high_memory); - - return res + (fb_bus_sdram_offset() - PAGE_OFFSET); -} -EXPORT_SYMBOL(__virt_to_bus); - -unsigned long __bus_to_virt(unsigned long res) -{ - res = res - (fb_bus_sdram_offset() - PAGE_OFFSET); - - WARN_ON(res < PAGE_OFFSET || res >= (unsigned long)high_memory); - - return res; -} -EXPORT_SYMBOL(__bus_to_virt); -#else -static inline unsigned long fb_bus_sdram_offset(void) -{ - return BUS_OFFSET; -} -#endif /* CONFIG_FOOTBRIDGE_ADDIN */ - -dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - return paddr + (fb_bus_sdram_offset() - PHYS_OFFSET); -} - -phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) -{ - return dev_addr - (fb_bus_sdram_offset() - PHYS_OFFSET); -} diff --git a/arch/arm/mach-footbridge/dc21285.c b/arch/arm/mach-footbridge/dc21285.c index f9713dc561cf77447dc1ad65cce249b3ea3ca0dd..f8920d0010de15710086b6468a7db6408e6c99d5 100644 --- a/arch/arm/mach-footbridge/dc21285.c +++ b/arch/arm/mach-footbridge/dc21285.c @@ -5,6 +5,7 @@ * Copyright (C) 1998-2001 Russell King * Copyright (C) 1998-2000 Phil Blundell */ +#include #include #include #include @@ -241,13 +242,26 @@ static irqreturn_t dc21285_parity_irq(int irq, void *dev_id) return IRQ_HANDLED; } +static int dc21285_pci_bus_notifier(struct notifier_block *nb, + unsigned long action, + void *data) +{ + if (action != BUS_NOTIFY_ADD_DEVICE) + return NOTIFY_DONE; + + dma_direct_set_offset(data, PHYS_OFFSET, BUS_OFFSET, SZ_256M); + + return NOTIFY_OK; +} + +static struct notifier_block dc21285_pci_bus_nb = { + .notifier_call = dc21285_pci_bus_notifier, +}; + int __init dc21285_setup(int nr, struct pci_sys_data *sys) { struct resource *res; - if (nr || !footbridge_cfn_mode()) - return 0; - res = kcalloc(2, sizeof(struct resource), GFP_KERNEL); if (!res) { printk("out of memory for root bus resources"); @@ -269,6 +283,8 @@ int __init dc21285_setup(int nr, struct pci_sys_data *sys) pci_add_resource_offset(&sys->resources, &res[0], sys->mem_offset); pci_add_resource_offset(&sys->resources, &res[1], sys->mem_offset); + bus_register_notifier(&pci_bus_type, &dc21285_pci_bus_nb); + return 1; } @@ -278,7 +294,6 @@ int __init dc21285_setup(int nr, struct pci_sys_data *sys) void __init dc21285_preinit(void) { unsigned int mem_size, mem_mask; - int cfn_mode; pcibios_min_mem = 0x81000000; @@ -298,21 +313,15 @@ void __init dc21285_preinit(void) *CSR_CSRBASEOFFSET = 0; *CSR_PCIADDR_EXTN = 0; - cfn_mode = __footbridge_cfn_mode(); - printk(KERN_INFO "PCI: DC21285 footbridge, revision %02lX, in " - "%s mode\n", *CSR_CLASSREV & 0xff, cfn_mode ? - "central function" : "addin"); - - if (footbridge_cfn_mode()) { - /* - * Clear any existing errors - we aren't - * interested in historical data... - */ - *CSR_SA110_CNTL = (*CSR_SA110_CNTL & 0xffffde07) | - SA110_CNTL_RXSERR; - *CSR_PCICMD = (*CSR_PCICMD & 0xffff) | PCICMD_ERROR_BITS; - } + "central function mode\n", *CSR_CLASSREV & 0xff); + + /* + * Clear any existing errors - we aren't + * interested in historical data... + */ + *CSR_SA110_CNTL = (*CSR_SA110_CNTL & 0xffffde07) | SA110_CNTL_RXSERR; + *CSR_PCICMD = (*CSR_PCICMD & 0xffff) | PCICMD_ERROR_BITS; timer_setup(&serr_timer, dc21285_enable_error, 0); timer_setup(&perr_timer, dc21285_enable_error, 0); @@ -331,29 +340,18 @@ void __init dc21285_preinit(void) dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, 0, "PCI data parity", NULL); - if (cfn_mode) { - /* - * Map our SDRAM at a known address in PCI space, just in case - * the firmware had other ideas. Using a nonzero base is - * necessary, since some VGA cards forcefully use PCI addresses - * in the range 0x000a0000 to 0x000c0000. (eg, S3 cards). - */ - *CSR_PCICSRBASE = 0xf4000000; - *CSR_PCICSRIOBASE = 0; - *CSR_PCISDRAMBASE = __virt_to_bus(PAGE_OFFSET); - *CSR_PCIROMBASE = 0; - *CSR_PCICMD = PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | - PCI_COMMAND_INVALIDATE | PCICMD_ERROR_BITS; - } else if (footbridge_cfn_mode() != 0) { - /* - * If we are not compiled to accept "add-in" mode, then - * we are using a constant virt_to_bus translation which - * can not hope to cater for the way the host BIOS has - * set up the machine. - */ - panic("PCI: this kernel is compiled for central " - "function mode only"); - } + /* + * Map our SDRAM at a known address in PCI space, just in case + * the firmware had other ideas. Using a nonzero base is + * necessary, since some VGA cards forcefully use PCI addresses + * in the range 0x000a0000 to 0x000c0000. (eg, S3 cards). + */ + *CSR_PCICSRBASE = 0xf4000000; + *CSR_PCICSRIOBASE = 0; + *CSR_PCISDRAMBASE = BUS_OFFSET; + *CSR_PCIROMBASE = 0; + *CSR_PCICMD = PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | + PCI_COMMAND_INVALIDATE | PCICMD_ERROR_BITS; } void __init dc21285_postinit(void) diff --git a/arch/arm/kernel/dma-isa.c b/arch/arm/mach-footbridge/dma-isa.c similarity index 95% rename from arch/arm/kernel/dma-isa.c rename to arch/arm/mach-footbridge/dma-isa.c index 2d90ecce5a11cdbaab150b3d34493fb761549503..937f5376d5e7749b96c1c61813b00d02284dda1f 100644 --- a/arch/arm/kernel/dma-isa.c +++ b/arch/arm/mach-footbridge/dma-isa.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/arch/arm/kernel/dma-isa.c - * * Copyright (C) 1999-2000 Russell King * * ISA DMA primitives @@ -13,6 +11,7 @@ * arch/arm/kernel/dma-ebsa285.c * Copyright (C) 1998 Phil Blundell */ +#include #include #include #include @@ -20,6 +19,7 @@ #include #include +#include #define ISA_DMA_MASK 0 #define ISA_DMA_MODE 1 @@ -157,7 +157,7 @@ static dma_t isa_dma[8]; /* * ISA DMA always starts at channel 0 */ -void __init isa_init_dma(void) +static int __init isa_dma_init(void) { /* * Try to autodetect presence of an ISA DMA controller. @@ -222,4 +222,9 @@ void __init isa_init_dma(void) request_dma(DMA_ISA_CASCADE, "cascade"); } + + dma_direct_set_offset(&isa_dma_dev, PHYS_OFFSET, BUS_OFFSET, SZ_256M); + + return 0; } +core_initcall(isa_dma_init); diff --git a/arch/arm/mach-footbridge/dma.c b/arch/arm/mach-footbridge/dma.c deleted file mode 100644 index 86618074a7a55fa87a1678ccd6ca106e7b844fad..0000000000000000000000000000000000000000 --- a/arch/arm/mach-footbridge/dma.c +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * linux/arch/arm/kernel/dma-ebsa285.c - * - * Copyright (C) 1998 Phil Blundell - * - * DMA functions specific to EBSA-285/CATS architectures - * - * Changelog: - * 09-Nov-1998 RMK Split out ISA DMA functions to dma-isa.c - * 17-Mar-1999 RMK Allow any EBSA285-like architecture to have - * ISA DMA controllers. - */ -#include -#include -#include -#include - -#include - -#include -#include - -#if 0 -static int fb_dma_request(unsigned int chan, dma_t *dma) -{ - return -EINVAL; -} - -static void fb_dma_enable(unsigned int chan, dma_t *dma) -{ -} - -static void fb_dma_disable(unsigned int chan, dma_t *dma) -{ -} - -static struct dma_ops fb_dma_ops = { - .type = "fb", - .request = fb_dma_request, - .enable = fb_dma_enable, - .disable = fb_dma_disable, -}; -#endif - -static int __init fb_dma_init(void) -{ -#if 0 - dma[_DC21285_DMA(0)].d_ops = &fb_dma_ops; - dma[_DC21285_DMA(1)].d_ops = &fb_dma_ops; -#endif -#ifdef CONFIG_ISA_DMA - if (footbridge_cfn_mode()) - isa_init_dma(); -#endif - return 0; -} -core_initcall(fb_dma_init); diff --git a/arch/arm/mach-footbridge/include/mach/dma-direct.h b/arch/arm/mach-footbridge/include/mach/dma-direct.h deleted file mode 100644 index 01f9e8367c0092b190ba6a709d46d2ccd680dc73..0000000000000000000000000000000000000000 --- a/arch/arm/mach-footbridge/include/mach/dma-direct.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef MACH_FOOTBRIDGE_DMA_DIRECT_H -#define MACH_FOOTBRIDGE_DMA_DIRECT_H 1 - -dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr); -phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr); - -#endif /* MACH_FOOTBRIDGE_DMA_DIRECT_H */ diff --git a/arch/arm/mach-footbridge/include/mach/isa-dma.h b/arch/arm/mach-footbridge/include/mach/isa-dma.h index 8a1b991076e1668c7945dc81d6fcd0f8a489dec2..b10731a1f66a8b47c59d4d5e9a9a7d9c52cbd65d 100644 --- a/arch/arm/mach-footbridge/include/mach/isa-dma.h +++ b/arch/arm/mach-footbridge/include/mach/isa-dma.h @@ -10,17 +10,9 @@ #ifndef __ASM_ARCH_DMA_H #define __ASM_ARCH_DMA_H -/* - * The 21285 has two internal DMA channels; we call these 8 and 9. - * On CATS hardware we have an additional eight ISA dma channels - * numbered 0..7. - */ -#define _ISA_DMA(x) (0+(x)) -#define _DC21285_DMA(x) (8+(x)) - -#define MAX_DMA_CHANNELS 10 +#define MAX_DMA_CHANNELS 8 -#define DMA_FLOPPY _ISA_DMA(2) -#define DMA_ISA_CASCADE _ISA_DMA(4) +#define DMA_FLOPPY (2) +#define DMA_ISA_CASCADE (4) #endif /* _ASM_ARCH_DMA_H */ diff --git a/arch/arm/mach-footbridge/include/mach/memory.h b/arch/arm/mach-footbridge/include/mach/memory.h index 3a5d2638c18f88034e1083009c7f4ab6e8c8631e..9516877667d75ade4fc34dbcae0a9a6aa22db80b 100644 --- a/arch/arm/mach-footbridge/include/mach/memory.h +++ b/arch/arm/mach-footbridge/include/mach/memory.h @@ -16,37 +16,6 @@ #ifndef __ASM_ARCH_MEMORY_H #define __ASM_ARCH_MEMORY_H - -#if defined(CONFIG_FOOTBRIDGE_ADDIN) -/* - * If we may be using add-in footbridge mode, then we must - * use the out-of-line translation that makes use of the - * PCI BAR - */ -#ifndef __ASSEMBLY__ -extern unsigned long __virt_to_bus(unsigned long); -extern unsigned long __bus_to_virt(unsigned long); -#endif -#define __virt_to_bus __virt_to_bus -#define __bus_to_virt __bus_to_virt - -#elif defined(CONFIG_FOOTBRIDGE_HOST) - -/* - * The footbridge is programmed to expose the system RAM at 0xe0000000. - * The requirement is that the RAM isn't placed at bus address 0, which - * would clash with VGA cards. - */ -#define BUS_OFFSET 0xe0000000 -#define __virt_to_bus(x) ((x) + (BUS_OFFSET - PAGE_OFFSET)) -#define __bus_to_virt(x) ((x) - (BUS_OFFSET - PAGE_OFFSET)) - -#else - -#error "Undefined footbridge mode" - -#endif - /* * Cache flushing area. */ diff --git a/arch/arm/mach-footbridge/isa.c b/arch/arm/mach-footbridge/isa.c index ec5af521cf9536a106111ff1e1c91c4a05135e1b..84caccddce44ef04eed40456ef3e625b84e9a5fb 100644 --- a/arch/arm/mach-footbridge/isa.c +++ b/arch/arm/mach-footbridge/isa.c @@ -79,16 +79,12 @@ static int __init footbridge_isa_init(void) { int err = 0; - if (!footbridge_cfn_mode()) - return 0; - /* Personal server doesn't have RTC */ - if (!machine_is_personal_server()) { - isa_rtc_init(); - err = platform_device_register(&rtc_device); - if (err) - printk(KERN_ERR "Unable to register RTC device: %d\n", err); - } + isa_rtc_init(); + err = platform_device_register(&rtc_device); + if (err) + printk(KERN_ERR "Unable to register RTC device: %d\n", err); + err = platform_device_register(&serial_device); if (err) printk(KERN_ERR "Unable to register serial device: %d\n", err); diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig index 77e435df8dfe79e45fa12e9601cbd32df08939f4..ab767f05992958eb38d34134cd0f687b04de58b7 100644 --- a/arch/arm/mach-imx/Kconfig +++ b/arch/arm/mach-imx/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only menuconfig ARCH_MXC bool "Freescale i.MX family" - depends on (ARCH_MULTI_V4_V5 && CPU_LITTLE_ENDIAN) || \ + depends on ((ARCH_MULTI_V4T || ARCH_MULTI_V5) && CPU_LITTLE_ENDIAN) || \ ARCH_MULTI_V6_V7 || ARM_SINGLE_ARMV7M select CLKSRC_IMX_GPT select GENERIC_IRQ_CHIP diff --git a/arch/arm/mach-imx/Makefile.boot b/arch/arm/mach-imx/Makefile.boot deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/arch/arm/mach-iop32x/Makefile.boot b/arch/arm/mach-iop32x/Makefile.boot deleted file mode 100644 index 5c3af01c40003bfd7cfed3895a68cfa8c9ba1756..0000000000000000000000000000000000000000 --- a/arch/arm/mach-iop32x/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0xa0008000 -params_phys-y := 0xa0000100 -initrd_phys-y := 0xa0800000 diff --git a/arch/arm/mach-ixp4xx/ixp4xx-of.c b/arch/arm/mach-ixp4xx/ixp4xx-of.c index f9904716ec7f15866db8db4f03cd84ea96192980..f543e2adae0cd154e93de751efd4188200153752 100644 --- a/arch/arm/mach-ixp4xx/ixp4xx-of.c +++ b/arch/arm/mach-ixp4xx/ixp4xx-of.c @@ -46,7 +46,7 @@ static void __init ixp4xx_of_map_io(void) } /* - * We handle 4 differen SoC families. These compatible strings are enough + * We handle 4 different SoC families. These compatible strings are enough * to provide the core so that different boards can add their more detailed * specifics. */ diff --git a/arch/arm/mach-lpc18xx/Makefile.boot b/arch/arm/mach-lpc18xx/Makefile.boot deleted file mode 100644 index cec195d4fcba2853c770a9c47d03a1380d54836c..0000000000000000000000000000000000000000 --- a/arch/arm/mach-lpc18xx/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Empty file waiting for deletion once Makefile.boot isn't needed any more. -# Patch waits for application at -# http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 . diff --git a/arch/arm/mach-lpc32xx/Makefile.boot b/arch/arm/mach-lpc32xx/Makefile.boot deleted file mode 100644 index 37d09ddb27f8411f74b89990a87960534a16de5d..0000000000000000000000000000000000000000 --- a/arch/arm/mach-lpc32xx/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x80008000 -params_phys-y := 0x80000100 -initrd_phys-y := 0x82000000 diff --git a/arch/arm/mach-mmp/devices.c b/arch/arm/mach-mmp/devices.c index 79f4a2aa5475dd3b3a4d01ff7c4e728e1225f13b..9968239d80415dfd52079a4d5efeef6ad66cc659 100644 --- a/arch/arm/mach-mmp/devices.c +++ b/arch/arm/mach-mmp/devices.c @@ -238,7 +238,7 @@ void pxa_usb_phy_deinit(void __iomem *phy_reg) static u64 __maybe_unused usb_dma_mask = ~(u32)0; #if IS_ENABLED(CONFIG_PHY_PXA_USB) -struct resource pxa168_usb_phy_resources[] = { +static struct resource pxa168_usb_phy_resources[] = { [0] = { .start = PXA168_U2O_PHYBASE, .end = PXA168_U2O_PHYBASE + USB_PHY_RANGE, @@ -259,7 +259,7 @@ struct platform_device pxa168_device_usb_phy = { #endif /* CONFIG_PHY_PXA_USB */ #if IS_ENABLED(CONFIG_USB_MV_UDC) -struct resource pxa168_u2o_resources[] = { +static struct resource pxa168_u2o_resources[] = { /* regbase */ [0] = { .start = PXA168_U2O_REGBASE + U2x_CAPREGS_OFFSET, @@ -294,7 +294,7 @@ struct platform_device pxa168_device_u2o = { #endif /* CONFIG_USB_MV_UDC */ #if IS_ENABLED(CONFIG_USB_EHCI_MV_U2O) -struct resource pxa168_u2oehci_resources[] = { +static struct resource pxa168_u2oehci_resources[] = { [0] = { .start = PXA168_U2O_REGBASE, .end = PXA168_U2O_REGBASE + USB_REG_RANGE, @@ -321,7 +321,7 @@ struct platform_device pxa168_device_u2oehci = { #endif #if IS_ENABLED(CONFIG_USB_MV_OTG) -struct resource pxa168_u2ootg_resources[] = { +static struct resource pxa168_u2ootg_resources[] = { /* regbase */ [0] = { .start = PXA168_U2O_REGBASE + U2x_CAPREGS_OFFSET, diff --git a/arch/arm/mach-mv78xx0/Makefile b/arch/arm/mach-mv78xx0/Makefile index a839e960b8c6f2fb6961090c29eb27c4cbf475c3..50aff70065f2edb326a6e1be25d380736408551d 100644 --- a/arch/arm/mach-mv78xx0/Makefile +++ b/arch/arm/mach-mv78xx0/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include +ccflags-y := -I$(srctree)/arch/arm/plat-orion/include obj-y += common.o mpp.o irq.o pcie.o obj-$(CONFIG_MACH_DB78X00_BP) += db78x00-bp-setup.o diff --git a/arch/arm/mach-mvebu/Makefile b/arch/arm/mach-mvebu/Makefile index cb106899dd7c8c220d75245add6342cc8cec1107..c21733cbb4fa4e4b3defdba9f2897e676fa1f734 100644 --- a/arch/arm/mach-mvebu/Makefile +++ b/arch/arm/mach-mvebu/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include +ccflags-y := -I$(srctree)/arch/arm/plat-orion/include AFLAGS_coherency_ll.o := -Wa,-march=armv7-a CFLAGS_pmsu.o := -march=armv7-a diff --git a/arch/arm/mach-nspire/Kconfig b/arch/arm/mach-nspire/Kconfig index eb9916233dea51a676d6ca261a6fd467b52b00d8..b7a3871876d7575e10f6b537f65861725afc643b 100644 --- a/arch/arm/mach-nspire/Kconfig +++ b/arch/arm/mach-nspire/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config ARCH_NSPIRE bool "TI-NSPIRE based" - depends on ARCH_MULTI_V4_V5 + depends on ARCH_MULTI_V4T depends on CPU_LITTLE_ENDIAN select CPU_ARM926T select GENERIC_IRQ_CHIP diff --git a/arch/arm/mach-omap1/Makefile.boot b/arch/arm/mach-omap1/Makefile.boot deleted file mode 100644 index 2c771515a60694f9c25fac6805793cfe9f83f763..0000000000000000000000000000000000000000 --- a/arch/arm/mach-omap1/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x10008000 -params_phys-y := 0x10000100 -initrd_phys-y := 0x10800000 diff --git a/arch/arm/mach-omap1/board-htcherald.c b/arch/arm/mach-omap1/board-htcherald.c index ec049cee49c6d629c31a27f12ca6e25f12d7b243..291d294b5824de8d47846b026d6c5e3c84519818 100644 --- a/arch/arm/mach-omap1/board-htcherald.c +++ b/arch/arm/mach-omap1/board-htcherald.c @@ -141,13 +141,6 @@ #define HTCPLD_GPIO_DOWN_DPAD HTCPLD_BASE(7, 4) #define HTCPLD_GPIO_ENTER_DPAD HTCPLD_BASE(7, 3) -/* - * The htcpld chip requires a gpio write to a specific line - * to re-enable interrupts after one has occurred. - */ -#define HTCPLD_GPIO_INT_RESET_HI HTCPLD_BASE(2, 7) -#define HTCPLD_GPIO_INT_RESET_LO HTCPLD_BASE(2, 0) - /* Chip 5 */ #define HTCPLD_IRQ_RIGHT_KBD HTCPLD_IRQ(0, 7) #define HTCPLD_IRQ_UP_KBD HTCPLD_IRQ(0, 6) @@ -348,8 +341,6 @@ static struct htcpld_chip_platform_data htcpld_chips[] = { }; static struct htcpld_core_platform_data htcpld_pfdata = { - .int_reset_gpio_hi = HTCPLD_GPIO_INT_RESET_HI, - .int_reset_gpio_lo = HTCPLD_GPIO_INT_RESET_LO, .i2c_adapter_id = 1, .chip = htcpld_chips, diff --git a/arch/arm/mach-orion5x/Makefile b/arch/arm/mach-orion5x/Makefile index 1a585a62d5e67de4159d111847408d50c614fe23..572c3520f7fed55c4f77a4e6ee72bd9abea4e4ab 100644 --- a/arch/arm/mach-orion5x/Makefile +++ b/arch/arm/mach-orion5x/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include +ccflags-y := -I$(srctree)/arch/arm/plat-orion/include obj-y += common.o pci.o irq.o mpp.o obj-$(CONFIG_MACH_DB88F5281) += db88f5281-setup.o diff --git a/arch/arm/mach-rpc/Kconfig b/arch/arm/mach-rpc/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..55f6d829b677ac17b484857f818109201288bc09 --- /dev/null +++ b/arch/arm/mach-rpc/Kconfig @@ -0,0 +1,21 @@ +config ARCH_RPC + bool "RiscPC" + depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5) + depends on !(ARCH_FOOTBRIDGE || ARCH_SA1100 || ARCH_MOXART || ARCH_GEMINI) + depends on !CC_IS_CLANG && GCC_VERSION < 90100 && GCC_VERSION >= 60000 + depends on CPU_LITTLE_ENDIAN + depends on ATAGS + depends on MMU + select ARCH_ACORN + select ARCH_MAY_HAVE_PC_FDC + select CPU_SA110 + select FIQ + select HAVE_PATA_PLATFORM + select ISA_DMA_API + select LEGACY_TIMER_TICK + select NEED_MACH_IO_H + select NEED_MACH_MEMORY_H + select NO_IOPORT_MAP + help + On the Acorn Risc-PC, Linux can support the internal IDE disk and + CD-ROM interface, serial and parallel port, and the floppy drive. diff --git a/arch/arm/mach-rpc/Makefile.boot b/arch/arm/mach-rpc/Makefile.boot deleted file mode 100644 index 0ed8e8fbde9985b2d8567b93725e1fc30394839b..0000000000000000000000000000000000000000 --- a/arch/arm/mach-rpc/Makefile.boot +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x10008000 -params_phys-y := 0x10000100 -initrd_phys-y := 0x18000000 - diff --git a/arch/arm/mach-s3c/Makefile.boot b/arch/arm/mach-s3c/Makefile.boot deleted file mode 100644 index 7f19e226035e7da6f46d746a03b47c429835ae22..0000000000000000000000000000000000000000 --- a/arch/arm/mach-s3c/Makefile.boot +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -ifeq ($(CONFIG_PM_H1940),y) - zreladdr-y += 0x30108000 - params_phys-y := 0x30100100 -else - zreladdr-y += 0x30008000 - params_phys-y := 0x30000100 -endif diff --git a/arch/arm/mach-s3c/devs.h b/arch/arm/mach-s3c/devs.h index 02b0c5750572936ef17152a605cc6cdaaebbc258..991b9b2006a1818de700151d863bdd81bb95bec0 100644 --- a/arch/arm/mach-s3c/devs.h +++ b/arch/arm/mach-s3c/devs.h @@ -33,8 +33,6 @@ extern struct platform_device s3c64xx_device_onenand1; extern struct platform_device s3c64xx_device_pcm0; extern struct platform_device s3c64xx_device_pcm1; extern struct platform_device s3c64xx_device_spi0; -extern struct platform_device s3c64xx_device_spi1; -extern struct platform_device s3c64xx_device_spi2; extern struct platform_device s3c_device_adc; extern struct platform_device s3c_device_cfcon; diff --git a/arch/arm/mach-s3c/mach-gta02.c b/arch/arm/mach-s3c/mach-gta02.c index abfdce765525c8b4e5065c0e1c91484c7edb3a15..d50a81d85ae15b7842fe1387765db0c6bdd67f8e 100644 --- a/arch/arm/mach-s3c/mach-gta02.c +++ b/arch/arm/mach-s3c/mach-gta02.c @@ -421,7 +421,14 @@ static struct s3c2410_platform_nand __initdata gta02_nand_info = { /* Get PMU to set USB current limit accordingly. */ static struct s3c2410_udc_mach_info gta02_udc_cfg __initdata = { .vbus_draw = gta02_udc_vbus_draw, - .pullup_pin = GTA02_GPIO_USB_PULLUP, +}; + +static struct gpiod_lookup_table gta02_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOB", 9, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, }; /* USB */ @@ -555,6 +562,7 @@ static void __init gta02_machine_init(void) s3c_gpio_cfgall_range(S3C2410_GPE(0), 5, S3C_GPIO_SFN(2), S3C_GPIO_PULL_NONE); + gpiod_add_lookup_table(>a02_udc_gpio_table); gpiod_add_lookup_table(>a02_audio_gpio_table); gpiod_add_lookup_table(>a02_mmc_gpio_table); platform_add_devices(gta02_devices, ARRAY_SIZE(gta02_devices)); diff --git a/arch/arm/mach-s3c/mach-h1940.c b/arch/arm/mach-s3c/mach-h1940.c index 032b18837855164f78cf18ddabf625d340bb0358..83ac6cfdb1d893868ef31701cb35182193c54215 100644 --- a/arch/arm/mach-s3c/mach-h1940.c +++ b/arch/arm/mach-s3c/mach-h1940.c @@ -167,9 +167,15 @@ static struct gpio_chip h1940_latch_gpiochip = { }; static struct s3c2410_udc_mach_info h1940_udc_cfg __initdata = { - .vbus_pin = S3C2410_GPG(5), - .vbus_pin_inverted = 1, - .pullup_pin = H1940_LATCH_USB_DP, +}; + +static struct gpiod_lookup_table h1940_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOG", 5, "vbus", GPIO_ACTIVE_LOW), + GPIO_LOOKUP("H1940_LATCH", 7, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, }; static struct s3c2410_ts_mach_info h1940_ts_cfg __initdata = { @@ -725,6 +731,7 @@ static void __init h1940_init(void) u32 tmp; s3c24xx_fb_set_platdata(&h1940_fb_info); + gpiod_add_lookup_table(&h1940_udc_gpio_table); gpiod_add_lookup_table(&h1940_mmc_gpio_table); gpiod_add_lookup_table(&h1940_audio_gpio_table); gpiod_add_lookup_table(&h1940_bat_gpio_table); diff --git a/arch/arm/mach-s3c/mach-jive.c b/arch/arm/mach-s3c/mach-jive.c index e3277317594447767470437f21a64dfa1b273be0..16859bb3bb13ebe23f4710c300e025a20177f60c 100644 --- a/arch/arm/mach-s3c/mach-jive.c +++ b/arch/arm/mach-s3c/mach-jive.c @@ -493,7 +493,14 @@ static struct platform_device *jive_devices[] __initdata = { }; static struct s3c2410_udc_mach_info jive_udc_cfg __initdata = { - .vbus_pin = S3C2410_GPG(1), /* detect is on GPG1 */ +}; + +static struct gpiod_lookup_table jive_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOG", 1, "vbus", GPIO_ACTIVE_HIGH), + { }, + }, }; /* Jive power management device */ @@ -669,6 +676,7 @@ static void __init jive_machine_init(void) pm_power_off = jive_power_off; + gpiod_add_lookup_table(&jive_udc_gpio_table); gpiod_add_lookup_table(&jive_lcdspi_gpiod_table); gpiod_add_lookup_table(&jive_wm8750_gpiod_table); platform_add_devices(jive_devices, ARRAY_SIZE(jive_devices)); diff --git a/arch/arm/mach-s3c/mach-mini2440.c b/arch/arm/mach-s3c/mach-mini2440.c index a6d17ffcdba13afc047a645c2a6bcaf6690147f3..283be70ca62209e70ae43443c357d80d245f160d 100644 --- a/arch/arm/mach-s3c/mach-mini2440.c +++ b/arch/arm/mach-s3c/mach-mini2440.c @@ -93,9 +93,15 @@ static struct s3c2410_uartcfg mini2440_uartcfgs[] __initdata = { /* USB device UDC support */ static struct s3c2410_udc_mach_info mini2440_udc_cfg __initdata = { - .pullup_pin = S3C2410_GPC(5), }; +static struct gpiod_lookup_table mini2440_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOC", 5, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, +}; /* LCD timing and setup */ @@ -755,6 +761,7 @@ static void __init mini2440_init(void) s3c24xx_fb_set_platdata(&mini2440_fb_info); } + gpiod_add_lookup_table(&mini2440_udc_gpio_table); s3c24xx_udc_set_platdata(&mini2440_udc_cfg); gpiod_add_lookup_table(&mini2440_mmc_gpio_table); s3c24xx_mci_set_platdata(&mini2440_mmc_cfg); diff --git a/arch/arm/mach-s3c/mach-n30.c b/arch/arm/mach-s3c/mach-n30.c index 75f5dc6351a113d6917028dc3fd34d8766501000..90122fc6b2aa65b40f2fcdb8a6b075704c25b85b 100644 --- a/arch/arm/mach-s3c/mach-n30.c +++ b/arch/arm/mach-s3c/mach-n30.c @@ -84,9 +84,15 @@ static struct s3c2410_uartcfg n30_uartcfgs[] = { }; static struct s3c2410_udc_mach_info n30_udc_cfg __initdata = { - .vbus_pin = S3C2410_GPG(1), - .vbus_pin_inverted = 0, - .pullup_pin = S3C2410_GPB(3), +}; + +static struct gpiod_lookup_table n30_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOG", 1, "vbus", GPIO_ACTIVE_HIGH), + GPIO_LOOKUP("GPIOB", 3, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, }; static struct gpio_keys_button n30_buttons[] = { @@ -595,6 +601,7 @@ static void __init n30_init(void) WARN_ON(gpio_request(S3C2410_GPG(4), "mmc power")); s3c24xx_fb_set_platdata(&n30_fb_info); + gpiod_add_lookup_table(&n30_udc_gpio_table); s3c24xx_udc_set_platdata(&n30_udc_cfg); gpiod_add_lookup_table(&n30_mci_gpio_table); s3c24xx_mci_set_platdata(&n30_mci_cfg); diff --git a/arch/arm/mach-s3c/mach-rx1950.c b/arch/arm/mach-s3c/mach-rx1950.c index 7a3e7c0a64848926954580bdd334084354121d4c..d8c49e56266066a5eaec6ec2e77393e02c19c643 100644 --- a/arch/arm/mach-s3c/mach-rx1950.c +++ b/arch/arm/mach-s3c/mach-rx1950.c @@ -643,9 +643,15 @@ static struct s3c2410_platform_nand rx1950_nand_info = { }; static struct s3c2410_udc_mach_info rx1950_udc_cfg __initdata = { - .vbus_pin = S3C2410_GPG(5), - .vbus_pin_inverted = 1, - .pullup_pin = S3C2410_GPJ(5), +}; + +static struct gpiod_lookup_table rx1950_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOG", 5, "vbus", GPIO_ACTIVE_LOW), + GPIO_LOOKUP("GPIOJ", 5, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, }; static struct s3c2410_ts_mach_info rx1950_ts_cfg __initdata = { @@ -847,6 +853,7 @@ static void __init rx1950_init_machine(void) gpio_direction_output(S3C2410_GPJ(6), 0); pwm_add_table(rx1950_pwm_lookup, ARRAY_SIZE(rx1950_pwm_lookup)); + gpiod_add_lookup_table(&rx1950_udc_gpio_table); gpiod_add_lookup_table(&rx1950_audio_gpio_table); gpiod_add_lookup_table(&rx1950_bat_gpio_table); /* Configure the I2S pins (GPE0...GPE4) in correct mode */ diff --git a/arch/arm/mach-s3c/mach-smdk2413.c b/arch/arm/mach-s3c/mach-smdk2413.c index f1f0ec174579b388d9839169cefef36c125930da..2b4e20aaa34698d66c4a98944d2e60b25cd9ed5f 100644 --- a/arch/arm/mach-s3c/mach-smdk2413.c +++ b/arch/arm/mach-s3c/mach-smdk2413.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -74,9 +74,15 @@ static struct s3c2410_uartcfg smdk2413_uartcfgs[] __initdata = { static struct s3c2410_udc_mach_info smdk2413_udc_cfg __initdata = { - .pullup_pin = S3C2410_GPF(2), }; +static struct gpiod_lookup_table smdk2413_udc_gpio_table = { + .dev_id = "s3c2410-usbgadget", + .table = { + GPIO_LOOKUP("GPIOF", 2, "pullup", GPIO_ACTIVE_HIGH), + { }, + }, +}; static struct platform_device *smdk2413_devices[] __initdata = { &s3c_device_ohci, @@ -115,7 +121,7 @@ static void __init smdk2413_machine_init(void) S3C2410_MISCCR_USBSUSPND0 | S3C2410_MISCCR_USBSUSPND1, 0x0); - + gpiod_add_lookup_table(&smdk2413_udc_gpio_table); s3c24xx_udc_set_platdata(&smdk2413_udc_cfg); s3c_i2c0_set_platdata(NULL); /* Configure the I2S pins (GPE0...GPE4) in correct mode */ diff --git a/arch/arm/mach-sa1100/Kconfig b/arch/arm/mach-sa1100/Kconfig index 7e0161cb1c1fa0c8baa976f46b37cfb7748e58ad..fb9cd10705de745a632d1af14d40af0eddb0a5db 100644 --- a/arch/arm/mach-sa1100/Kconfig +++ b/arch/arm/mach-sa1100/Kconfig @@ -1,7 +1,25 @@ # SPDX-License-Identifier: GPL-2.0-only -if ARCH_SA1100 +menuconfig ARCH_SA1100 + bool "SA11x0 Implementations" + depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5) + depends on !(ARCH_MOXART || ARCH_GEMINI) + depends on ATAGS + depends on CPU_LITTLE_ENDIAN + depends on MMU + select ARCH_NO_SG_CHAIN + select ARCH_MTD_XIP + select CLKSRC_MMIO + select CLKSRC_PXA + select CPU_FREQ + select CPU_SA1100 + select GPIOLIB + select IRQ_DOMAIN + select ISA + select NEED_MACH_MEMORY_H + help + Support for StrongARM 11x0 based boards. -menu "SA11x0 Implementations" +if ARCH_SA1100 config SA1100_ASSABET bool "Assabet" @@ -179,7 +197,4 @@ config SA1100_SSP This isn't for audio support, but for attached sensors and other devices, eg for BadgePAD 4 sensor support. -endmenu - endif - diff --git a/arch/arm/mach-sa1100/Makefile.boot b/arch/arm/mach-sa1100/Makefile.boot deleted file mode 100644 index 9d8246f2cab4e044a5145d2f509b3db5e1d9be84..0000000000000000000000000000000000000000 --- a/arch/arm/mach-sa1100/Makefile.boot +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -ifeq ($(CONFIG_SA1111),y) - zreladdr-y += 0xc0208000 -else - zreladdr-y += 0xc0008000 -endif -params_phys-y := 0xc0000100 -initrd_phys-y := 0xc0800000 - diff --git a/arch/arm/mach-shmobile/Kconfig b/arch/arm/mach-shmobile/Kconfig index 50909c4b95b20c4cb36b32f32fdf3fa57d0dba94..37f862f13c8d3569670625a84144be40fe2b6d61 100644 --- a/arch/arm/mach-shmobile/Kconfig +++ b/arch/arm/mach-shmobile/Kconfig @@ -6,5 +6,4 @@ menuconfig ARCH_RENESAS select GPIOLIB select NO_IOPORT_MAP select PINCTRL - select SOC_BUS select ZONE_DMA if ARM_LPAE diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 43b7996ab7545f120d1cffa298f943b556626d62..9e36920d4cfd12fab56893e1b66d5f1ad58ac2d8 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -25,11 +25,8 @@ extern struct pl022_ssp_controller pl022_plat_data; extern struct pl08x_platform_data pl080_plat_data; void __init spear_setup_of_timer(void); -void __init spear3xx_clk_init(void __iomem *misc_base, - void __iomem *soc_config_base); void __init spear3xx_map_io(void); void __init spear3xx_dt_init_irq(void); -void __init spear6xx_clk_init(void __iomem *misc_base); void __init spear13xx_map_io(void); void __init spear13xx_l2x0_init(void); diff --git a/arch/arm/mach-spear/spear3xx.c b/arch/arm/mach-spear/spear3xx.c index 2ba406e92c41b8a6dbafb4b630b97296dc496d31..7ef9670d302925594f0a0b18be8e52a12dff5ef8 100644 --- a/arch/arm/mach-spear/spear3xx.c +++ b/arch/arm/mach-spear/spear3xx.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include "pl080.h" diff --git a/arch/arm/mach-spear/spear6xx.c b/arch/arm/mach-spear/spear6xx.c index 58183493e06d46e633c081d66d20b70fb8dd27bc..f0a1e704ccebcb296c6b6059560b7c8c662fe57f 100644 --- a/arch/arm/mach-spear/spear6xx.c +++ b/arch/arm/mach-spear/spear6xx.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -339,7 +340,7 @@ static struct pl08x_platform_data spear6xx_pl080_plat_data = { * 0xD0000000 0xFD000000 * 0xFC000000 0xFC000000 */ -struct map_desc spear6xx_io_desc[] __initdata = { +static struct map_desc spear6xx_io_desc[] __initdata = { { .virtual = (unsigned long)VA_SPEAR6XX_ML_CPU_BASE, .pfn = __phys_to_pfn(SPEAR_ICM3_ML1_2_BASE), @@ -359,12 +360,12 @@ struct map_desc spear6xx_io_desc[] __initdata = { }; /* This will create static memory mapping for selected devices */ -void __init spear6xx_map_io(void) +static void __init spear6xx_map_io(void) { iotable_init(spear6xx_io_desc, ARRAY_SIZE(spear6xx_io_desc)); } -void __init spear6xx_timer_init(void) +static void __init spear6xx_timer_init(void) { char pclk_name[] = "pll3_clk"; struct clk *gpt_clk, *pclk; @@ -394,7 +395,7 @@ void __init spear6xx_timer_init(void) } /* Add auxdata to pass platform data */ -struct of_dev_auxdata spear6xx_auxdata_lookup[] __initdata = { +static struct of_dev_auxdata spear6xx_auxdata_lookup[] __initdata = { OF_DEV_AUXDATA("arm,pl080", SPEAR_ICM3_DMA_BASE, NULL, &spear6xx_pl080_plat_data), {} diff --git a/arch/arm/mach-stm32/Makefile.boot b/arch/arm/mach-stm32/Makefile.boot deleted file mode 100644 index 5dde7328a7a95b375de473365934fe49e72baf9d..0000000000000000000000000000000000000000 --- a/arch/arm/mach-stm32/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Empty file waiting for deletion once Makefile.boot isn't needed any more. -# Patch waits for application at -# https://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 . diff --git a/arch/arm/mach-sunplus/Kconfig b/arch/arm/mach-sunplus/Kconfig index 926cde5e3cd91cc78ebd64a3307778f3577a31b1..d0c2416e6f241bee9e22a650db166e17d624bb5b 100644 --- a/arch/arm/mach-sunplus/Kconfig +++ b/arch/arm/mach-sunplus/Kconfig @@ -18,8 +18,8 @@ config SOC_SP7021 select ARM_PSCI select PINCTRL select PINCTRL_SPPCTL - select SERIAL_SUNPLUS - select SERIAL_SUNPLUS_CONSOLE + select SERIAL_SUNPLUS if TTY + select SERIAL_SUNPLUS_CONSOLE if TTY help Support for Sunplus SP7021 SoC. It is based on ARM 4-core Cortex-A7 with various peripherals (e.g.: I2C, SPI, SDIO, diff --git a/arch/arm/mach-versatile/Kconfig b/arch/arm/mach-versatile/Kconfig index 2ef226194c3a5b3ca31b3d6a65c4ddd7068fd5ab..b1519b4dc03a0f9b69c2b8bb64f5490bd3b6d7c8 100644 --- a/arch/arm/mach-versatile/Kconfig +++ b/arch/arm/mach-versatile/Kconfig @@ -256,7 +256,6 @@ menuconfig ARCH_VEXPRESS select GPIOLIB select HAVE_ARM_SCU if SMP select HAVE_ARM_TWD if SMP - select HAVE_PATA_PLATFORM select CLK_ICST select NO_IOPORT_MAP select PLAT_VERSATILE diff --git a/arch/arm/mach-versatile/Makefile.boot b/arch/arm/mach-versatile/Makefile.boot deleted file mode 100644 index cec195d4fcba2853c770a9c47d03a1380d54836c..0000000000000000000000000000000000000000 --- a/arch/arm/mach-versatile/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# Empty file waiting for deletion once Makefile.boot isn't needed any more. -# Patch waits for application at -# http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 . diff --git a/arch/arm/mach-versatile/integrator_ap.c b/arch/arm/mach-versatile/integrator_ap.c index e216fac917d09bf99257dcda15506051827c1940..4bd6712e9f525d66a040a6e4d783da5b798914cd 100644 --- a/arch/arm/mach-versatile/integrator_ap.c +++ b/arch/arm/mach-versatile/integrator_ap.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm/mach-vt8500/Makefile.boot b/arch/arm/mach-vt8500/Makefile.boot deleted file mode 100644 index 883985f4b6c189841a53afaf9d3e5824da219fd6..0000000000000000000000000000000000000000 --- a/arch/arm/mach-vt8500/Makefile.boot +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only - zreladdr-y += 0x00008000 -params_phys-y := 0x00000100 -initrd_phys-y := 0x01000000 diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 089c9c644cce2add8859103eb1e8c8812561daaa..d7909091cf977b8fad32c447d6185947e1cf89b2 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -307,7 +307,7 @@ void __init dma_contiguous_remap(void) static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data) { - struct page *page = virt_to_page(addr); + struct page *page = virt_to_page((void *)addr); pgprot_t prot = *(pgprot_t *)data; set_pte_ext(pte, mk_pte(page, prot), 0); @@ -1769,8 +1769,14 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *iommu, bool coherent) { - dev->archdata.dma_coherent = coherent; - dev->dma_coherent = coherent; + /* + * Due to legacy code that sets the ->dma_coherent flag from a bus + * notifier we can't just assign coherent to the ->dma_coherent flag + * here, but instead have to make sure we only set but never clear it + * for now. + */ + if (coherent) + dev->dma_coherent = true; /* * Don't override the dma_ops if they have already been set. Ideally diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c index fb688003d156ea8a9c77fba4ae9d080d3be80b42..059eb4cdc9c28b8f0d5f07a8b8c8b60700fbc8a1 100644 --- a/arch/arm/mm/dump.c +++ b/arch/arm/mm/dump.c @@ -26,7 +26,7 @@ static struct addr_marker address_markers[] = { { MODULES_VADDR, "Modules" }, { PAGE_OFFSET, "Kernel Mapping" }, { 0, "vmalloc() Area" }, - { VMALLOC_END, "vmalloc() End" }, + { FDT_FIXED_BASE, "FDT Area" }, { FIXADDR_START, "Fixmap Area" }, { VECTORS_BASE, "Vectors" }, { VECTORS_BASE + PAGE_SIZE * 2, "Vectors End" }, @@ -200,6 +200,7 @@ static const struct prot_bits section_bits[] = { }; struct pg_level { + const char *name; const struct prot_bits *bits; size_t num; u64 mask; @@ -213,9 +214,11 @@ static struct pg_level pg_level[] = { }, { /* p4d */ }, { /* pud */ }, { /* pmd */ + .name = (CONFIG_PGTABLE_LEVELS > 2) ? "PMD" : "PGD", .bits = section_bits, .num = ARRAY_SIZE(section_bits), }, { /* pte */ + .name = "PTE", .bits = pte_bits, .num = ARRAY_SIZE(pte_bits), }, @@ -282,7 +285,8 @@ static void note_page(struct pg_state *st, unsigned long addr, delta >>= 10; unit++; } - pt_dump_seq_printf(st->seq, "%9lu%c", delta, *unit); + pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit, + pg_level[st->level].name); if (st->current_domain) pt_dump_seq_printf(st->seq, " %s", st->current_domain); @@ -346,7 +350,7 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) addr = start + i * PMD_SIZE; domain = get_domain_name(pmd); if (pmd_none(*pmd) || pmd_large(*pmd) || !pmd_present(*pmd)) - note_page(st, addr, 3, pmd_val(*pmd), domain); + note_page(st, addr, 4, pmd_val(*pmd), domain); else walk_pte(st, pmd, addr, domain); diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c index 29caee9c79ce3c980618ebbee1d81e17d446c419..46d9f4a622cbc4c3822a7812cc106462f3e0fa4f 100644 --- a/arch/arm/mm/kasan_init.c +++ b/arch/arm/mm/kasan_init.c @@ -268,12 +268,17 @@ void __init kasan_init(void) /* * 1. The module global variables are in MODULES_VADDR ~ MODULES_END, - * so we need to map this area. + * so we need to map this area if CONFIG_KASAN_VMALLOC=n. With + * VMALLOC support KASAN will manage this region dynamically, + * refer to kasan_populate_vmalloc() and ARM's implementation of + * module_alloc(). * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't * use kasan_populate_zero_shadow. */ - create_mapping((void *)MODULES_VADDR, (void *)(PKMAP_BASE + PMD_SIZE)); + if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES)) + create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END)); + create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE)); /* * KAsan may reuse the contents of kasan_early_shadow_pte directly, so diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index a49f0b9c0f752527f3ca38b111abc73eb118cce8..463fc2a8448f0a28725f3f26bf098fa8c504de0b 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -300,7 +300,11 @@ static struct mem_type mem_types[] __ro_after_init = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_XN | L_PTE_RDONLY, .prot_l1 = PMD_TYPE_TABLE, +#ifdef CONFIG_ARM_LPAE + .prot_sect = PMD_TYPE_SECT | L_PMD_SECT_RDONLY | PMD_SECT_AP2, +#else .prot_sect = PMD_TYPE_SECT, +#endif .domain = DOMAIN_KERNEL, }, [MT_ROM] = { diff --git a/arch/arm/plat-orion/Makefile b/arch/arm/plat-orion/Makefile index 4e3f25de13c19d1215776c58634e989810570a26..830b0be038c6bab30f1f6f84e335c037c67cc6e5 100644 --- a/arch/arm/plat-orion/Makefile +++ b/arch/arm/plat-orion/Makefile @@ -2,7 +2,7 @@ # # Makefile for the linux kernel. # -ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/$(src)/include +ccflags-y := -I$(srctree)/$(src)/include orion-gpio-$(CONFIG_GPIOLIB) += gpio.o obj-$(CONFIG_PLAT_ORION_LEGACY) += irq.o pcie.o time.o common.o mpp.o diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 571cc234d0b3f320b81d63f38bc87725b6451211..505c8a1ccbe0cd043d672a8e1192e052ac73d84a 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -149,6 +149,7 @@ config ARM64 select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE select HAVE_ARCH_COMPILER_H + select HAVE_ARCH_HUGE_VMALLOC select HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE @@ -194,7 +195,6 @@ config ARM64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KVM select HAVE_NMI - select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -209,7 +209,6 @@ config ARM64 select HAVE_KPROBES select HAVE_KRETPROBES select HAVE_GENERIC_VDSO - select IOMMU_DMA if IOMMU_SUPPORT select IRQ_DOMAIN select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN @@ -230,6 +229,7 @@ config ARM64 select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD select TRACE_IRQFLAGS_SUPPORT select TRACE_IRQFLAGS_NMI_SUPPORT + select HAVE_SOFTIRQ_ON_OWN_STACK help ARM 64-bit (AArch64) Linux support. @@ -632,6 +632,23 @@ config ARM64_ERRATUM_1530923 config ARM64_WORKAROUND_REPEAT_TLBI bool +config ARM64_ERRATUM_2441007 + bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" + default y + select ARM64_WORKAROUND_REPEAT_TLBI + help + This option adds a workaround for ARM Cortex-A55 erratum #2441007. + + Under very rare circumstances, affected Cortex-A55 CPUs + may not handle a race between a break-before-make sequence on one + CPU, and another CPU accessing the same page. This could allow a + store to a page that has been unmapped. + + Work around this by adding the affected CPUs to the list that needs + TLB sequences to be done twice. + + If unsure, say Y. + config ARM64_ERRATUM_1286807 bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation" default y @@ -733,6 +750,19 @@ config ARM64_ERRATUM_2077057 If unsure, say Y. +config ARM64_ERRATUM_2658417 + bool "Cortex-A510: 2658417: remove BF16 support due to incorrect result" + default y + help + This option adds the workaround for ARM Cortex-A510 erratum 2658417. + Affected Cortex-A510 (r0p0 to r1p1) may produce the wrong result for + BFMMLA or VMMLA instructions in rare circumstances when a pair of + A510 CPUs are using shared neon hardware. As the sharing is not + discoverable by the kernel, hide the BF16 HWCAP to indicate that + user-space should not be using these instructions. + + If unsure, say Y. + config ARM64_ERRATUM_2119858 bool "Cortex-A710/X2: 2119858: workaround TRBE overwriting trace data in FILL mode" default y @@ -917,6 +947,23 @@ config ARM64_ERRATUM_1902691 If unsure, say Y. +config ARM64_ERRATUM_2457168 + bool "Cortex-A510: 2457168: workaround for AMEVCNTR01 incrementing incorrectly" + depends on ARM64_AMU_EXTN + default y + help + This option adds the workaround for ARM Cortex-A510 erratum 2457168. + + The AMU counter AMEVCNTR01 (constant counter) should increment at the same rate + as the system counter. On affected Cortex-A510 cores AMEVCNTR01 increments + incorrectly giving a significantly higher output value. + + Work around this problem by returning 0 when reading the affected counter in + key locations that results in disabling all users of this counter. This effect + is the same to firmware disabling affected counters. + + If unsure, say Y. + config CAVIUM_ERRATUM_22375 bool "Cavium erratum 22375, 24313" default y @@ -1401,7 +1448,7 @@ config XEN help Say Y if you want to run Linux in a Virtual Machine on Xen on ARM64. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int default "14" if ARM64_64K_PAGES default "12" if ARM64_16K_PAGES @@ -1545,6 +1592,9 @@ config THUMB2_COMPAT_VDSO Compile the compat vDSO with '-mthumb -fomit-frame-pointer' if y, otherwise with '-marm'. +config COMPAT_ALIGNMENT_FIXUPS + bool "Fix up misaligned multi-word loads and stores in user space" + menuconfig ARMV8_DEPRECATED bool "Emulate deprecated/obsolete ARMv8 instructions" depends on SYSCTL @@ -1870,6 +1920,8 @@ config ARM64_BTI_KERNEL depends on CC_HAS_BRANCH_PROT_PAC_RET_BTI # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94697 depends on !CC_IS_GCC || GCC_VERSION >= 100100 + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 + depends on !CC_IS_GCC # https://github.com/llvm/llvm-project/commit/a88c722e687e6780dcd6a58718350dc76fcc4cc9 depends on !CC_IS_CLANG || CLANG_VERSION >= 120000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) @@ -2215,6 +2267,3 @@ source "drivers/acpi/Kconfig" source "arch/arm64/kvm/Kconfig" -if CRYPTO -source "arch/arm64/crypto/Kconfig" -endif # CRYPTO diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index 74e9e9de37597dd63f25e05d1a8e5a4b6d6dccf2..76580b932e446a130f35b6ebde4b136899ea7b0a 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -33,6 +33,11 @@ config ARCH_APPLE This enables support for Apple's in-house ARM SoC family, starting with the Apple M1. +menuconfig ARCH_BCM + bool "Broadcom SoC Support" + +if ARCH_BCM + config ARCH_BCM2835 bool "Broadcom BCM2835 family" select TIMER_OF @@ -47,15 +52,6 @@ config ARCH_BCM2835 This enables support for the Broadcom BCM2837 and BCM2711 SoC. These SoCs are used in the Raspberry Pi 3 and 4 devices. -config ARCH_BCM4908 - bool "Broadcom BCM4908 family" - select ARCH_BCMBCA - select GPIOLIB - help - This enables support for the Broadcom BCM4906, BCM4908 and - BCM49408 SoCs. These SoCs use Brahma-B53 cores and can be - found in home routers. - config ARCH_BCM_IPROC bool "Broadcom iProc SoC Family" select COMMON_CLK_IPROC @@ -66,6 +62,7 @@ config ARCH_BCM_IPROC config ARCH_BCMBCA bool "Broadcom Broadband Carrier Access (BCA) origin SoC" + select GPIOLIB help Say Y if you intend to run the kernel on a Broadcom Broadband ARM-based BCA chipset. @@ -73,6 +70,16 @@ config ARCH_BCMBCA This enables support for Broadcom BCA ARM-based broadband chipsets, including the DSL, PON and Wireless family of chips. +config ARCH_BRCMSTB + bool "Broadcom Set-Top-Box SoCs" + select ARCH_HAS_RESET_CONTROLLER + select GENERIC_IRQ_CHIP + select PINCTRL + help + This enables support for Broadcom's ARMv8 Set Top Box SoCs + +endif + config ARCH_BERLIN bool "Marvell Berlin SoC Family" select DW_APB_ICTL @@ -87,14 +94,6 @@ config ARCH_BITMAIN help This enables support for the Bitmain SoC Family. -config ARCH_BRCMSTB - bool "Broadcom Set-Top-Box SoCs" - select ARCH_HAS_RESET_CONTROLLER - select GENERIC_IRQ_CHIP - select PINCTRL - help - This enables support for Broadcom's ARMv8 Set Top Box SoCs - config ARCH_EXYNOS bool "ARMv8 based Samsung Exynos SoC family" select COMMON_CLK_SAMSUNG @@ -136,12 +135,6 @@ config ARCH_K3 This enables support for Texas Instruments' K3 multicore SoC architecture. -config ARCH_LAYERSCAPE - bool "ARMv8 based Freescale Layerscape SoC family" - select EDAC_SUPPORT - help - This enables support for the Freescale Layerscape SoC family. - config ARCH_LG1K bool "LG Electronics LG1K SoC Family" help @@ -200,6 +193,17 @@ config ARCH_MVEBU - Armada 8K SoC Family - 98DX2530 SoC Family +menuconfig ARCH_NXP + bool "NXP SoC support" + +if ARCH_NXP + +config ARCH_LAYERSCAPE + bool "ARMv8 based Freescale Layerscape SoC family" + select EDAC_SUPPORT + help + This enables support for the Freescale Layerscape SoC family. + config ARCH_MXC bool "ARMv8 based NXP i.MX SoC family" select ARM64_ERRATUM_843419 @@ -214,6 +218,13 @@ config ARCH_MXC This enables support for the ARMv8 based SoCs in the NXP i.MX family. +config ARCH_S32 + bool "NXP S32 SoC Family" + help + This enables support for the NXP S32 family of processors. + +endif + config ARCH_NPCM bool "Nuvoton NPCM Architecture" select PINCTRL @@ -243,7 +254,6 @@ config ARCH_RENESAS bool "Renesas SoC Platforms" select GPIOLIB select PINCTRL - select SOC_BUS help This enables support for the ARMv8 based Renesas SoCs. @@ -257,11 +267,6 @@ config ARCH_ROCKCHIP This enables support for the ARMv8 based Rockchip chipsets, like the RK3368. -config ARCH_S32 - bool "NXP S32 SoC Family" - help - This enables support for the NXP S32 family of processors. - config ARCH_SEATTLE bool "AMD Seattle SoC Family" help diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 6d9d4a58b8986bf6f44df34427c44e44aff9a7f4..5e56d26a223983546b383cdb90f4a2fbd5027833 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -133,9 +133,6 @@ ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_REGS),y) CC_FLAGS_FTRACE := -fpatchable-function-entry=2 endif -# Default value -head-y := arch/arm64/kernel/head.o - ifeq ($(CONFIG_KASAN_SW_TAGS), y) KASAN_SHADOW_SCALE_SHIFT := 4 else ifeq ($(CONFIG_KASAN_GENERIC), y) @@ -151,12 +148,17 @@ libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a # Default target when executing plain make boot := arch/arm64/boot + +ifeq ($(CONFIG_EFI_ZBOOT),) KBUILD_IMAGE := $(boot)/Image.gz +else +KBUILD_IMAGE := $(boot)/vmlinuz.efi +endif -all: Image.gz +all: $(notdir $(KBUILD_IMAGE)) -Image: vmlinux +Image vmlinuz.efi: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ Image.%: Image diff --git a/arch/arm64/boot/.gitignore b/arch/arm64/boot/.gitignore index 9a7a9009d43afef04a6b0df9813c5cd0be4b6a35..af5dc61f8b438852afac1ae66e007f53076e046a 100644 --- a/arch/arm64/boot/.gitignore +++ b/arch/arm64/boot/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only Image Image.gz +vmlinuz* diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile index a0e3dedd28839363e55e5eb4e3503f17b5dd69fa..c65aee0884103c6f7d5904d0f3680afc19b55303 100644 --- a/arch/arm64/boot/Makefile +++ b/arch/arm64/boot/Makefile @@ -38,3 +38,9 @@ $(obj)/Image.lzo: $(obj)/Image FORCE $(obj)/Image.zst: $(obj)/Image FORCE $(call if_changed,zstd) + +EFI_ZBOOT_PAYLOAD := Image +EFI_ZBOOT_BFD_TARGET := elf64-littleaarch64 +EFI_ZBOOT_MACH_TYPE := ARM64 + +include $(srctree)/drivers/firmware/efi/libstub/Makefile.zboot diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi index 548539c93ab0c8803504437f901cd1cb959a1386..97e3e6907acdf59a298d36784ac1e9e85298704d 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi @@ -101,6 +101,18 @@ #reset-cells = <1>; }; + dma: dma-controller@3002000 { + compatible = "allwinner,sun50i-a100-dma"; + reg = <0x03002000 0x1000>; + interrupts = ; + clocks = <&ccu CLK_BUS_DMA>, <&ccu CLK_MBUS_DMA>; + clock-names = "bus", "mbus"; + resets = <&ccu RST_BUS_DMA>; + dma-channels = <8>; + dma-requests = <52>; + #dma-cells = <1>; + }; + gic: interrupt-controller@3021000 { compatible = "arm,gic-400"; reg = <0x03021000 0x1000>, <0x03022000 0x2000>, @@ -209,6 +221,8 @@ interrupts = ; clocks = <&ccu CLK_BUS_I2C0>; resets = <&ccu RST_BUS_I2C0>; + dmas = <&dma 43>, <&dma 43>; + dma-names = "rx", "tx"; status = "disabled"; #address-cells = <1>; #size-cells = <0>; @@ -222,6 +236,8 @@ interrupts = ; clocks = <&ccu CLK_BUS_I2C1>; resets = <&ccu RST_BUS_I2C1>; + dmas = <&dma 44>, <&dma 44>; + dma-names = "rx", "tx"; status = "disabled"; #address-cells = <1>; #size-cells = <0>; @@ -235,6 +251,8 @@ interrupts = ; clocks = <&ccu CLK_BUS_I2C2>; resets = <&ccu RST_BUS_I2C2>; + dmas = <&dma 45>, <&dma 45>; + dma-names = "rx", "tx"; status = "disabled"; #address-cells = <1>; #size-cells = <0>; @@ -248,6 +266,8 @@ interrupts = ; clocks = <&ccu CLK_BUS_I2C3>; resets = <&ccu RST_BUS_I2C3>; + dmas = <&dma 46>, <&dma 46>; + dma-names = "rx", "tx"; status = "disabled"; #address-cells = <1>; #size-cells = <0>; @@ -325,6 +345,8 @@ interrupts = ; clocks = <&r_ccu CLK_R_APB2_I2C0>; resets = <&r_ccu RST_R_APB2_I2C0>; + dmas = <&dma 50>, <&dma 50>; + dma-names = "rx", "tx"; pinctrl-names = "default"; pinctrl-0 = <&r_i2c0_pins>; status = "disabled"; @@ -340,6 +362,8 @@ interrupts = ; clocks = <&r_ccu CLK_R_APB2_I2C1>; resets = <&r_ccu RST_R_APB2_I2C1>; + dmas = <&dma 51>, <&dma 51>; + dma-names = "rx", "tx"; pinctrl-names = "default"; pinctrl-0 = <&r_i2c1_pins>; status = "disabled"; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts index 6249e9e0292863f00684485a1eab0ffcccf8f1b3..9ec49ac2f6fd5dbc13c8694473c66bcde10cff81 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts @@ -5,6 +5,7 @@ #include "sun50i-h6.dtsi" #include "sun50i-h6-cpu-opp.dtsi" +#include "sun50i-h6-gpu-opp.dtsi" #include diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..b48049c4fc857da8bf2f18d20e2260fd1d5c8841 --- /dev/null +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +// Copyright (C) 2022 Clément Péron + +/ { + gpu_opp_table: opp-table-gpu { + compatible = "operating-points-v2"; + + opp-216000000 { + opp-hz = /bits/ 64 <216000000>; + opp-microvolt = <810000 810000 1200000>; + }; + + opp-264000000 { + opp-hz = /bits/ 64 <264000000>; + opp-microvolt = <810000 810000 1200000>; + }; + + opp-312000000 { + opp-hz = /bits/ 64 <312000000>; + opp-microvolt = <810000 810000 1200000>; + }; + + opp-336000000 { + opp-hz = /bits/ 64 <336000000>; + opp-microvolt = <810000 810000 1200000>; + }; + + opp-360000000 { + opp-hz = /bits/ 64 <360000000>; + opp-microvolt = <820000 820000 1200000>; + }; + + opp-384000000 { + opp-hz = /bits/ 64 <384000000>; + opp-microvolt = <830000 830000 1200000>; + }; + + opp-408000000 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <840000 840000 1200000>; + }; + + opp-420000000 { + opp-hz = /bits/ 64 <420000000>; + opp-microvolt = <850000 850000 1200000>; + }; + + opp-432000000 { + opp-hz = /bits/ 64 <432000000>; + opp-microvolt = <860000 860000 1200000>; + }; + + opp-456000000 { + opp-hz = /bits/ 64 <456000000>; + opp-microvolt = <870000 870000 1200000>; + }; + + opp-504000000 { + opp-hz = /bits/ 64 <504000000>; + opp-microvolt = <890000 890000 1200000>; + }; + + opp-540000000 { + opp-hz = /bits/ 64 <540000000>; + opp-microvolt = <910000 910000 1200000>; + }; + + opp-576000000 { + opp-hz = /bits/ 64 <576000000>; + opp-microvolt = <930000 930000 1200000>; + }; + + opp-624000000 { + opp-hz = /bits/ 64 <624000000>; + opp-microvolt = <950000 950000 1200000>; + }; + + opp-756000000 { + opp-hz = /bits/ 64 <756000000>; + opp-microvolt = <1040000 1040000 1200000>; + }; + }; +}; + +&gpu { + operating-points-v2 = <&gpu_opp_table>; +}; diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi index 5a28303d3d4c62c99d144132315566ee75990dd0..53f6660656ac59ef98273edfbf1435786920bb91 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi @@ -186,6 +186,7 @@ clocks = <&ccu CLK_GPU>, <&ccu CLK_BUS_GPU>; clock-names = "core", "bus"; resets = <&ccu RST_BUS_GPU>; + #cooling-cells = <2>; status = "disabled"; }; @@ -1072,9 +1073,55 @@ }; gpu-thermal { - polling-delay-passive = <0>; - polling-delay = <0>; + polling-delay-passive = <1000>; + polling-delay = <2000>; thermal-sensors = <&ths 1>; + + trips { + gpu_alert0: gpu-alert-0 { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + gpu_alert1: gpu-alert-1 { + temperature = <100000>; + hysteresis = <2000>; + type = "passive"; + }; + + gpu_alert2: gpu-alert-2 { + temperature = <105000>; + hysteresis = <2000>; + type = "passive"; + }; + + gpu-crit { + temperature = <115000>; + hysteresis = <0>; + type = "critical"; + }; + }; + + cooling-maps { + // Forbid the GPU to go over 756MHz + map0 { + trip = <&gpu_alert0>; + cooling-device = <&gpu 1 THERMAL_NO_LIMIT>; + }; + + // Forbid the GPU to go over 624MHz + map1 { + trip = <&gpu_alert1>; + cooling-device = <&gpu 2 THERMAL_NO_LIMIT>; + }; + + // Forbid the GPU to go over 576MHz + map2 { + trip = <&gpu_alert2>; + cooling-device = <&gpu 3 THERMAL_NO_LIMIT>; + }; + }; }; }; }; diff --git a/arch/arm64/boot/dts/amlogic/Makefile b/arch/arm64/boot/dts/amlogic/Makefile index 8773211df50ec2755307dbf0c6e8277492642553..e213aeebb7743c635b409d4790f5eddd5da7a387 100644 --- a/arch/arm64/boot/dts/amlogic/Makefile +++ b/arch/arm64/boot/dts/amlogic/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_MESON) += meson-a1-ad401.dtb dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j100.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j110-rev-2.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j110-rev-3.dtb dtb-$(CONFIG_ARCH_MESON) += meson-axg-s400.dtb dtb-$(CONFIG_ARCH_MESON) += meson-g12a-radxa-zero.dtb dtb-$(CONFIG_ARCH_MESON) += meson-g12a-sei510.dtb @@ -43,6 +45,7 @@ dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-libretech-cc-v2.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-libretech-cc.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-nexbox-a95x.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-p212.dtb +dtb-$(CONFIG_ARCH_MESON) += meson-gxm-gt1-ultimate.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-khadas-vim2.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-mecool-kiii-pro.dtb dtb-$(CONFIG_ARCH_MESON) += meson-gxm-minix-neo-u9h.dtb diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts index 8b0d586aa84e9fb1633407ae01d9bfba17e164b1..b2d6ba660914f37361ee05120615acda9c6502af 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts +++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts @@ -1,270 +1,29 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) /* - * Copyright (c) 2021 Vyacheslav Bocharov - * Copyright (c) 2020 JetHome - * Author: Aleksandr Kazantsev - * Author: Alexey Shevelkin + * Copyright (c) 2022 Vyacheslav Bocharov + * Copyright (c) 2022 JetHome * Author: Vyacheslav Bocharov */ /dts-v1/; -#include "meson-axg.dtsi" -#include -#include +#include "meson-axg-jethome-jethub-j1xx.dtsi" / { compatible = "jethome,jethub-j100", "amlogic,a113d", "amlogic,meson-axg"; - model = "JetHome JetHub J100"; - aliases { - serial0 = &uart_AO; /* Console */ - serial2 = &uart_AO_B; /* External UART (Wireless Module) */ - ethernet0 = ðmac; - }; - - chosen { - stdout-path = "serial0:115200n8"; - }; + model = "JetHome JetHub D1 (J100)"; /* 1024MB RAM */ memory@0 { device_type = "memory"; reg = <0x0 0x0 0x0 0x40000000>; }; - - reserved-memory { - linux,cma { - size = <0x0 0x400000>; - }; - }; - - emmc_pwrseq: emmc-pwrseq { - compatible = "mmc-pwrseq-emmc"; - reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>; - }; - - vcc_3v3: regulator-vcc_3v3 { - compatible = "regulator-fixed"; - regulator-name = "VCC_3V3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vddao_3v3>; - regulator-always-on; - }; - - vcc_5v: regulator-vcc_5v { - compatible = "regulator-fixed"; - regulator-name = "VCC5V"; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - regulator-always-on; - }; - - vddao_3v3: regulator-vddao_3v3 { - compatible = "regulator-fixed"; - regulator-name = "VDDAO_3V3"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vcc_5v>; - regulator-always-on; - }; - - vddio_ao18: regulator-vddio_ao18 { - compatible = "regulator-fixed"; - regulator-name = "VDDIO_AO18"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vddao_3v3>; - regulator-always-on; - }; - - vddio_boot: regulator-vddio_boot { - compatible = "regulator-fixed"; - regulator-name = "VDDIO_BOOT"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - vin-supply = <&vddao_3v3>; - regulator-always-on; - }; - - vccq_1v8: regulator-vccq_1v8 { - compatible = "regulator-fixed"; - regulator-name = "VCCQ_1V8"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - vin-supply = <&vddao_3v3>; - regulator-always-on; - }; - - usb_pwr: regulator-usb_pwr { - compatible = "regulator-fixed"; - regulator-name = "USB_PWR"; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc_5v>; - regulator-always-on; - }; - - sdio_pwrseq: sdio-pwrseq { - compatible = "mmc-pwrseq-simple"; - reset-gpios = <&gpio GPIOX_7 GPIO_ACTIVE_LOW>; - clocks = <&wifi32k>; - clock-names = "ext_clock"; - }; - - wifi32k: wifi32k { - compatible = "pwm-clock"; - #clock-cells = <0>; - clock-frequency = <32768>; - pwms = <&pwm_ab 0 30518 0>; /* PWM_A at 32.768KHz */ - }; - - thermal-zones { - cpu_thermal: cpu-thermal { - polling-delay-passive = <250>; - polling-delay = <1000>; - thermal-sensors = <&scpi_sensors 0>; - trips { - cpu_passive: cpu-passive { - temperature = <70000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "passive"; - }; - - cpu_hot: cpu-hot { - temperature = <80000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "hot"; - }; - - cpu_critical: cpu-critical { - temperature = <100000>; /* millicelsius */ - hysteresis = <2000>; /* millicelsius */ - type = "critical"; - }; - }; - - cpu_cooling_maps: cooling-maps { - map0 { - trip = <&cpu_passive>; - cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - - map1 { - trip = <&cpu_hot>; - cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, - <&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; - }; - - onewire { - compatible = "w1-gpio"; - gpios = <&gpio GPIOA_14 GPIO_ACTIVE_HIGH>; - #gpio-cells = <1>; - }; -}; - -&efuse { - sn: sn@32 { - reg = <0x32 0x20>; - }; - - eth_mac: eth_mac@0 { - reg = <0x0 0x6>; - }; - - bt_mac: bt_mac@6 { - reg = <0x6 0x6>; - }; - - wifi_mac: wifi_mac@c { - reg = <0xc 0x6>; - }; - - bid: bid@12 { - reg = <0x12 0x20>; - }; -}; - -ðmac { - status = "okay"; - pinctrl-0 = <ð_rmii_x_pins>; - pinctrl-names = "default"; - phy-handle = <ð_phy0>; - phy-mode = "rmii"; - - mdio { - compatible = "snps,dwmac-mdio"; - #address-cells = <1>; - #size-cells = <0>; - - /* ICPlus IP101A/G Ethernet PHY (vendor_id=0x0243, model_id=0x0c54) */ - eth_phy0: ethernet-phy@0 { - /* compatible = "ethernet-phy-id0243.0c54";*/ - max-speed = <100>; - reg = <0>; - - reset-assert-us = <10000>; - reset-deassert-us = <10000>; - reset-gpios = <&gpio GPIOZ_5 GPIO_ACTIVE_LOW>; - }; - }; -}; - -/* Internal I2C bus (on CPU module) */ -&i2c1 { - status = "okay"; - pinctrl-0 = <&i2c1_z_pins>; - pinctrl-names = "default"; - - /* RTC */ - pcf8563: pcf8563@51 { - compatible = "nxp,pcf8563"; - reg = <0x51>; - status = "okay"; - }; }; -/* Peripheral I2C bus (on motherboard) */ -&i2c_AO { - status = "okay"; - pinctrl-0 = <&i2c_ao_sck_10_pins>, <&i2c_ao_sda_11_pins>; - pinctrl-names = "default"; -}; - -&pwm_ab { - status = "okay"; - pinctrl-0 = <&pwm_a_x20_pins>; - pinctrl-names = "default"; -}; /* wifi module */ &sd_emmc_b { - status = "okay"; - #address-cells = <1>; - #size-cells = <0>; - - pinctrl-0 = <&sdio_pins>; - pinctrl-1 = <&sdio_clk_gate_pins>; - pinctrl-names = "default", "clk-gate"; - - bus-width = <4>; - cap-sd-highspeed; - max-frequency = <50000000>; non-removable; - disable-wp; - - mmc-pwrseq = <&sdio_pwrseq>; - - vmmc-supply = <&vddao_3v3>; - vqmmc-supply = <&vddio_boot>; brcmf: wifi@1 { reg = <1>; @@ -272,99 +31,10 @@ }; }; -/* emmc storage */ -&sd_emmc_c { - status = "okay"; - pinctrl-0 = <&emmc_pins>, <&emmc_ds_pins>; - pinctrl-1 = <&emmc_clk_gate_pins>; - pinctrl-names = "default", "clk-gate"; - - bus-width = <8>; - cap-mmc-highspeed; - max-frequency = <200000000>; - non-removable; - disable-wp; - mmc-ddr-1_8v; - mmc-hs200-1_8v; - - mmc-pwrseq = <&emmc_pwrseq>; - - vmmc-supply = <&vcc_3v3>; - vqmmc-supply = <&vccq_1v8>; -}; - /* UART Bluetooth */ &uart_B { - status = "okay"; - pinctrl-0 = <&uart_b_z_pins>, <&uart_b_z_cts_rts_pins>; - pinctrl-names = "default"; - uart-has-rtscts; - bluetooth { compatible = "brcm,bcm43438-bt"; shutdown-gpios = <&gpio GPIOZ_7 GPIO_ACTIVE_HIGH>; }; }; - -/* UART Console */ -&uart_AO { - status = "okay"; - pinctrl-0 = <&uart_ao_a_pins>; - pinctrl-names = "default"; -}; - -/* UART Wireless module */ -&uart_AO_B { - status = "okay"; - pinctrl-0 = <&uart_ao_b_pins>; - pinctrl-names = "default"; -}; - -&usb { - status = "okay"; - phy-supply = <&usb_pwr>; -}; - -&spicc1 { - status = "okay"; - pinctrl-0 = <&spi1_x_pins>, <&spi1_ss0_x_pins>; - pinctrl-names = "default"; -}; - -&gpio { - gpio-line-names = - "", "", "", "", "", // 0 - 4 - "", "", "", "", "", // 5 - 9 - "UserButton", "", "", "", "", // 10 - 14 - "", "", "", "", "", // 15 - 19 - "", "", "", "", "", // 20 - 24 - "", "LedRed", "LedGreen", "Output3", "Output2", // 25 - 29 - "Output1", "", "", "", "", // 30 - 34 - "", "ZigBeeBOOT", "", "", "", // 35 - 39 - "1Wire", "ZigBeeRESET", "", "Input4", "Input3", // 40 - 44 - "Input2", "Input1", "", "", "", // 45 - 49 - "", "", "", "", "", // 50 - 54 - "", "", "", "", "", // 55 - 59 - "", "", "", "", "", // 60 - 64 - "", "", "", "", "", // 65 - 69 - "", "", "", "", "", // 70 - 74 - "", "", "", "", "", // 75 - 79 - "", "", "", "", "", // 80 - 84 - "", ""; // 85-86 -}; - -&cpu0 { - #cooling-cells = <2>; -}; - -&cpu1 { - #cooling-cells = <2>; -}; - -&cpu2 { - #cooling-cells = <2>; -}; - -&cpu3 { - #cooling-cells = <2>; -}; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts new file mode 100644 index 0000000000000000000000000000000000000000..0062667c4f65f90d5b9b6edc9ae50e166a5774d9 --- /dev/null +++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2022 Vyacheslav Bocharov + * Copyright (c) 2022 JetHome + * Author: Vyacheslav Bocharov + */ + +/dts-v1/; + +#include "meson-axg-jethome-jethub-j1xx.dtsi" + +/ { + compatible = "jethome,jethub-j110", "amlogic,a113d", "amlogic,meson-axg"; + model = "JetHome JetHub D1p (J110) HW rev.2"; + + /* 2GiB or 4GiB RAM */ + memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x80000000>; + }; +}; + + +/* wifi module */ +&sd_emmc_b { + broken-cd;/* cd-gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;*/ +}; + +/* UART Bluetooth */ +&uart_B { + bluetooth { + compatible = "realtek,rtl8822cs-bt"; + enable-gpios = <&gpio GPIOZ_7 GPIO_ACTIVE_HIGH>; + host-wake-gpios = <&gpio GPIOZ_8 GPIO_ACTIVE_HIGH>; + device-wake-gpios = <&gpio GPIOZ_6 GPIO_ACTIVE_HIGH>; + }; +}; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts new file mode 100644 index 0000000000000000000000000000000000000000..c2d22b00c1cd7757f1d918ec3f1735233107ce17 --- /dev/null +++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2022 Vyacheslav Bocharov + * Copyright (c) 2022 JetHome + * Author: Vyacheslav Bocharov + */ + +/dts-v1/; + +#include "meson-axg-jethome-jethub-j1xx.dtsi" + +/ { + compatible = "jethome,jethub-j110", "amlogic,a113d", "amlogic,meson-axg"; + model = "JetHome JetHub D1p (J110) Hw rev.3"; + + /* 2GiB or 4GiB RAM */ + memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x80000000>; + }; +}; + + +/* wifi module */ +&sd_emmc_b { + broken-cd;/* cd-gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;*/ +}; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..5836b0030931231ddcaa1767b16a1b578fa54c4c --- /dev/null +++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2022 Vyacheslav Bocharov + * Copyright (c) 2022 JetHome + * Author: Vyacheslav Bocharov + * Author: Aleksandr Kazantsev + * Author: Alexey Shevelkin + */ + +/dts-v1/; + +#include "meson-axg.dtsi" +#include +#include + +/ { + aliases { + serial0 = &uart_AO; /* Console */ + serial2 = &uart_AO_B; /* External UART (Wireless Module) */ + ethernet0 = ðmac; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + reserved-memory { + linux,cma { + size = <0x0 0x400000>; + }; + }; + + emmc_pwrseq: emmc-pwrseq { + compatible = "mmc-pwrseq-emmc"; + reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>; + }; + + vcc_3v3: regulator-vcc_3v3 { + compatible = "regulator-fixed"; + regulator-name = "VCC_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vddao_3v3>; + regulator-always-on; + }; + + vcc_5v: regulator-vcc_5v { + compatible = "regulator-fixed"; + regulator-name = "VCC5V"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + }; + + vddao_3v3: regulator-vddao_3v3 { + compatible = "regulator-fixed"; + regulator-name = "VDDAO_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_5v>; + regulator-always-on; + }; + + vddio_ao18: regulator-vddio_ao18 { + compatible = "regulator-fixed"; + regulator-name = "VDDIO_AO18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vddao_3v3>; + regulator-always-on; + }; + + vddio_boot: regulator-vddio_boot { + compatible = "regulator-fixed"; + regulator-name = "VDDIO_BOOT"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vddao_3v3>; + regulator-always-on; + }; + + vccq_1v8: regulator-vccq_1v8 { + compatible = "regulator-fixed"; + regulator-name = "VCCQ_1V8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vddao_3v3>; + regulator-always-on; + }; + + usb_pwr: regulator-usb_pwr { + compatible = "regulator-fixed"; + regulator-name = "USB_PWR"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc_5v>; + regulator-always-on; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + reset-gpios = <&gpio GPIOX_7 GPIO_ACTIVE_LOW>; + clocks = <&wifi32k>; + clock-names = "ext_clock"; + }; + + wifi32k: wifi32k { + compatible = "pwm-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + pwms = <&pwm_ab 0 30518 0>; /* PWM_A at 32.768KHz */ + }; + + thermal-zones { + cpu_thermal: cpu-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + thermal-sensors = <&scpi_sensors 0>; + trips { + cpu_passive: cpu-passive { + temperature = <70000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "passive"; + }; + + cpu_hot: cpu-hot { + temperature = <80000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "hot"; + }; + + cpu_critical: cpu-critical { + temperature = <100000>; /* millicelsius */ + hysteresis = <2000>; /* millicelsius */ + type = "critical"; + }; + }; + + cpu_cooling_maps: cooling-maps { + map0 { + trip = <&cpu_passive>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + + map1 { + trip = <&cpu_hot>; + cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + }; + + onewire { + compatible = "w1-gpio"; + gpios = <&gpio GPIOA_14 GPIO_ACTIVE_HIGH>; + #gpio-cells = <1>; + }; +}; + +&efuse { + sn: sn@32 { + reg = <0x32 0x20>; + }; + + eth_mac: eth_mac@0 { + reg = <0x0 0x6>; + }; + + bt_mac: bt_mac@6 { + reg = <0x6 0x6>; + }; + + wifi_mac: wifi_mac@c { + reg = <0xc 0x6>; + }; + + bid: bid@12 { + reg = <0x12 0x20>; + }; +}; + +ðmac { + status = "okay"; + pinctrl-0 = <ð_rmii_x_pins>; + pinctrl-names = "default"; + phy-handle = <ð_phy0>; + phy-mode = "rmii"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + /* ICPlus IP101A/G Ethernet PHY (vendor_id=0x0243, model_id=0x0c54) */ + eth_phy0: ethernet-phy@0 { + /* compatible = "ethernet-phy-id0243.0c54";*/ + max-speed = <100>; + reg = <0>; + + reset-assert-us = <10000>; + reset-deassert-us = <10000>; + reset-gpios = <&gpio GPIOZ_5 GPIO_ACTIVE_LOW>; + }; + }; +}; + +/* Internal I2C bus (on CPU module) */ +&i2c1 { + status = "okay"; + pinctrl-0 = <&i2c1_z_pins>; + pinctrl-names = "default"; + + /* RTC */ + pcf8563: pcf8563@51 { + compatible = "nxp,pcf8563"; + reg = <0x51>; + status = "okay"; + }; +}; + +/* Peripheral I2C bus (on motherboard) */ +&i2c_AO { + status = "okay"; + pinctrl-0 = <&i2c_ao_sck_10_pins>, <&i2c_ao_sda_11_pins>; + pinctrl-names = "default"; +}; + +&pwm_ab { + status = "okay"; + pinctrl-0 = <&pwm_a_x20_pins>; + pinctrl-names = "default"; +}; + +/* wifi module */ +&sd_emmc_b { + status = "okay"; + #address-cells = <1>; + #size-cells = <0>; + + pinctrl-0 = <&sdio_pins>; + pinctrl-1 = <&sdio_clk_gate_pins>; + pinctrl-names = "default", "clk-gate"; + + bus-width = <4>; + cap-sd-highspeed; + max-frequency = <50000000>; + disable-wp; + + mmc-pwrseq = <&sdio_pwrseq>; + + vmmc-supply = <&vddao_3v3>; + vqmmc-supply = <&vddio_boot>; +}; + +/* emmc storage */ +&sd_emmc_c { + status = "okay"; + pinctrl-0 = <&emmc_pins>, <&emmc_ds_pins>; + pinctrl-1 = <&emmc_clk_gate_pins>; + pinctrl-names = "default", "clk-gate"; + + bus-width = <8>; + cap-mmc-highspeed; + max-frequency = <200000000>; + non-removable; + disable-wp; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + + mmc-pwrseq = <&emmc_pwrseq>; + + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&vccq_1v8>; +}; + +/* UART Bluetooth */ +&uart_B { + status = "okay"; + pinctrl-0 = <&uart_b_z_pins>, <&uart_b_z_cts_rts_pins>; + pinctrl-names = "default"; + uart-has-rtscts; +}; + +/* UART Console */ +&uart_AO { + status = "okay"; + pinctrl-0 = <&uart_ao_a_pins>; + pinctrl-names = "default"; +}; + +/* UART Wireless module */ +&uart_AO_B { + status = "okay"; + pinctrl-0 = <&uart_ao_b_pins>; + pinctrl-names = "default"; +}; + +&usb { + status = "okay"; + phy-supply = <&usb_pwr>; +}; + +&spicc1 { + status = "okay"; + pinctrl-0 = <&spi1_x_pins>, <&spi1_ss0_x_pins>; + pinctrl-names = "default"; +}; + +&gpio { + gpio-line-names = + "", "", "", "", "", // 0 - 4 + "", "", "", "", "", // 5 - 9 + "UserButton", "", "", "", "", // 10 - 14 + "", "", "", "", "", // 15 - 19 + "", "", "", "", "", // 20 - 24 + "", "LedRed", "LedGreen", "Output3", "Output2", // 25 - 29 + "Output1", "", "", "", "", // 30 - 34 + "", "ZigBeeBOOT", "", "", "", // 35 - 39 + "1Wire", "ZigBeeRESET", "", "Input4", "Input3", // 40 - 44 + "Input2", "Input1", "", "", "", // 45 - 49 + "", "", "", "", "", // 50 - 54 + "", "", "", "", "", // 55 - 59 + "", "", "", "", "", // 60 - 64 + "", "", "", "", "", // 65 - 69 + "", "", "", "", "", // 70 - 74 + "", "", "", "", "", // 75 - 79 + "", "", "", "", "", // 80 - 84 + "", ""; // 85-86 +}; + +&cpu0 { + #cooling-cells = <2>; +}; + +&cpu1 { + #cooling-cells = <2>; +}; + +&cpu2 { + #cooling-cells = <2>; +}; + +&cpu3 { + #cooling-cells = <2>; +}; diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts index b4e86196e3468223961ed9e51677b2e184e45203..b2bb94981838f2b47df39b41d53c0997f99bee8f 100644 --- a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts +++ b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts @@ -111,7 +111,6 @@ vin-supply = <&dc_in>; gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; - enable-active-low; }; vddao_1v8: regulator-vddao_1v8 { diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts new file mode 100644 index 0000000000000000000000000000000000000000..2c267884cc16b7898ca43336337a3e5035920d7b --- /dev/null +++ b/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) Christian Hewitt + */ + +/dts-v1/; + +#include "meson-gxm.dtsi" +#include "meson-gx-p23x-q20x.dtsi" +#include +#include + +/ { + compatible = "azw,gt1-ultimate", "amlogic,s912", "amlogic,meson-gxm"; + model = "Beelink GT1 Ultimate"; + + leds { + compatible = "gpio-leds"; + + led-white { + color = ; + function = LED_FUNCTION_POWER; + gpios = <&gpio_ao GPIOAO_9 GPIO_ACTIVE_HIGH>; + default-state = "on"; + panic-indicator; + }; + }; + + adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1710000>; + + button-function { + label = "update"; + linux,code = ; + press-threshold-microvolt = <10000>; + }; + }; +}; + +ðmac { + pinctrl-0 = <ð_pins>; + pinctrl-names = "default"; + phy-handle = <&external_phy>; + amlogic,tx-delay-ns = <2>; + phy-mode = "rgmii"; +}; + +&external_mdio { + external_phy: ethernet-phy@0 { + /* Realtek RTL8211F (0x001cc916) */ + reg = <0>; + max-speed = <1000>; + + reset-assert-us = <10000>; + reset-deassert-us = <80000>; + reset-gpios = <&gpio GPIOZ_14 GPIO_ACTIVE_LOW>; + + interrupt-parent = <&gpio_intc>; + /* MAC_INTR on GPIOZ_15 */ + interrupts = <25 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&ir { + linux,rc-map-name = "rc-beelink-gs1"; +}; + +&sd_emmc_a { + brcmf: wifi@1 { + reg = <1>; + compatible = "brcm,bcm4329-fmac"; + }; +}; + +&uart_A { + status = "okay"; + pinctrl-0 = <&uart_a_pins>, <&uart_a_cts_rts_pins>; + pinctrl-names = "default"; + uart-has-rtscts; + + bluetooth { + compatible = "brcm,bcm43438-bt"; + shutdown-gpios = <&gpio GPIOX_17 GPIO_ACTIVE_HIGH>; + max-speed = <2000000>; + clocks = <&wifi32k>; + clock-names = "lpo"; + }; +}; diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts index 603337ca56081e77af5cb3dc185e7103f187dfab..9068a334ea579da62e72617e65db0b8edb3761e0 100644 --- a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts +++ b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts @@ -171,7 +171,6 @@ regulator-max-microvolt = <3300000>; vin-supply = <&vddao_3v3>; gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>; - enable-active-low; regulator-always-on; }; diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 2cd429efba5b956c0705d07b0f2cfc7325ea72ba..c1f3ba9c39f6a82a107ae0f40d5361a49bdc10d4 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -21,6 +21,10 @@ }; }; +&wifi0 { + brcm,board-type = "apple,atlantisb"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 49cdf4b560a30550480cdb8edce4732f5817133f..ecb10d237a053bc377c45ef89c98d4570c779af2 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -17,6 +17,10 @@ model = "Apple MacBook Pro (13-inch, M1, 2020)"; }; +&wifi0 { + brcm,board-type = "apple,honshu"; +}; + /* * Remove unused PCIe ports and disable the associated DARTs. */ diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index b0ebb45bdb6fb031c42551775817d3785753c388..df741737b8e6f502366d26bde1e057d807da54b1 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -17,6 +17,10 @@ model = "Apple MacBook Air (M1, 2020)"; }; +&wifi0 { + brcm,board-type = "apple,shikoku"; +}; + /* * Remove unused PCIe ports and disable the associated DARTs. */ diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 884fddf7d36373fd5a97b9577f9ceeaf4c633700..8c6bf959251075cd61fd9b30fb537c01bfc27c13 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -21,6 +21,10 @@ }; }; +&wifi0 { + brcm,board-type = "apple,capri"; +}; + &i2c0 { hpm2: usb-pd@3b { compatible = "apple,cd321x"; diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index d7c622931627c5201d4d8571e39a7a4dd5355a06..fe7c0aaf7d62a7ff81ad312bff9617268315d9ab 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -21,6 +21,10 @@ }; }; +&wifi0 { + brcm,board-type = "apple,santorini"; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index fe2ae40fa9ddf9de58245ea526eda35b2a513b6d..3d15b8e2a6c1e62bb13199ed267fa41815312ca9 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -71,8 +71,10 @@ &port00 { bus-range = <1 1>; wifi0: network@0,0 { + compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; /* To be filled by the loader */ local-mac-address = [00 00 00 00 00 00]; + apple,antenna-sku = "XX"; }; }; diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi index 8d0d45d168d1338e286229dddb8fcd29d5bcb2cc..2f27619d8abd59855c46cd6f899575ae278f75e1 100644 --- a/arch/arm64/boot/dts/arm/juno-base.dtsi +++ b/arch/arm64/boot/dts/arm/juno-base.dtsi @@ -26,7 +26,8 @@ compatible = "arm,mhu", "arm,primecell"; reg = <0x0 0x2b1f0000 0x0 0x1000>; interrupts = , - ; + , + ; #mbox-cells = <1>; clocks = <&soc_refclk100mhz>; clock-names = "apb_pclk"; diff --git a/arch/arm64/boot/dts/arm/juno-cs-r1r2.dtsi b/arch/arm64/boot/dts/arm/juno-cs-r1r2.dtsi index ba88d1596f6f1f734135f2441a2e052cc16f1914..09d2b692e9e1f2b01f76e88077e18ffddd0eb6a9 100644 --- a/arch/arm64/boot/dts/arm/juno-cs-r1r2.dtsi +++ b/arch/arm64/boot/dts/arm/juno-cs-r1r2.dtsi @@ -67,7 +67,6 @@ port@0 { reg = <0>; csys2_funnel_in_port0: endpoint { - slave-mode; remote-endpoint = <&etf0_out_port>; }; }; @@ -75,7 +74,6 @@ port@1 { reg = <1>; csys2_funnel_in_port1: endpoint { - slave-mode; remote-endpoint = <&etf1_out_port>; }; }; diff --git a/arch/arm64/boot/dts/broadcom/Makefile b/arch/arm64/boot/dts/broadcom/Makefile index e8584d3b698f90f56fb1bf6830c4d574c36a7f20..05d8c5ecf3b0f546e25b8b397bac0481442ad2c0 100644 --- a/arch/arm64/boot/dts/broadcom/Makefile +++ b/arch/arm64/boot/dts/broadcom/Makefile @@ -8,7 +8,6 @@ dtb-$(CONFIG_ARCH_BCM2835) += bcm2711-rpi-400.dtb \ bcm2837-rpi-cm3-io3.dtb \ bcm2837-rpi-zero-2-w.dtb -subdir-y += bcm4908 subdir-y += bcmbca subdir-y += northstar2 subdir-y += stingray diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile b/arch/arm64/boot/dts/broadcom/bcm4908/Makefile deleted file mode 100644 index 6e364e304d4fd5d3b28477b37aefec95417e396e..0000000000000000000000000000000000000000 --- a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-netgear-r8000p.dtb -dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-tplink-archer-c2300-v1.dtb -dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-asus-gt-ac5300.dtb -dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-netgear-raxe500.dtb diff --git a/arch/arm64/boot/dts/broadcom/bcmbca/Makefile b/arch/arm64/boot/dts/broadcom/bcmbca/Makefile index 38f14307184b57518a7b417bdf3b0594f99bca33..27741b71ba9e57742897599bef7a8e778f9884d1 100644 --- a/arch/arm64/boot/dts/broadcom/bcmbca/Makefile +++ b/arch/arm64/boot/dts/broadcom/bcmbca/Makefile @@ -1,5 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_BCMBCA) += \ + bcm4906-netgear-r8000p.dtb \ + bcm4906-tplink-archer-c2300-v1.dtb \ + bcm4908-asus-gt-ac5300.dtb \ + bcm4908-netgear-raxe500.dtb \ + bcm94908.dtb \ bcm4912-asus-gt-ax6000.dtb \ bcm94912.dtb \ bcm963158.dtb \ diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-netgear-r8000p.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-netgear-r8000p.dts similarity index 96% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-netgear-r8000p.dts rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-netgear-r8000p.dts index 2dd028438c22c8482840faad804b50e8a18cd5ae..d8b60575eb4f3aaad2530a76ca85a88afdab93a6 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-netgear-r8000p.dts +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-netgear-r8000p.dts @@ -7,7 +7,7 @@ #include "bcm4906.dtsi" / { - compatible = "netgear,r8000p", "brcm,bcm4906", "brcm,bcm4908"; + compatible = "netgear,r8000p", "brcm,bcm4906", "brcm,bcm4908", "brcm,bcmbca"; model = "Netgear R8000P"; memory@0 { diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-tplink-archer-c2300-v1.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-tplink-archer-c2300-v1.dts similarity index 99% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-tplink-archer-c2300-v1.dts rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-tplink-archer-c2300-v1.dts index 064f7f5496657544714ce1a3c3efb41c5eba4c29..296393d4aaabe7ca97efc9df9909c7b08d7c58ae 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-tplink-archer-c2300-v1.dts +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-tplink-archer-c2300-v1.dts @@ -7,7 +7,7 @@ #include "bcm4906.dtsi" / { - compatible = "tplink,archer-c2300-v1", "brcm,bcm4906", "brcm,bcm4908"; + compatible = "tplink,archer-c2300-v1", "brcm,bcm4906", "brcm,bcm4908", "brcm,bcmbca"; model = "TP-Link Archer C2300 V1"; memory@0 { diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906.dtsi similarity index 100% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4906.dtsi diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-asus-gt-ac5300.dts similarity index 69% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-asus-gt-ac5300.dts index 04f8524b53351b9ef9f39e7d95cfd5db3a7ed3d9..839ca33178b01ce7f1305866f8dbe6b0d624591d 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-asus-gt-ac5300.dts @@ -2,11 +2,12 @@ #include #include +#include #include "bcm4908.dtsi" / { - compatible = "asus,gt-ac5300", "brcm,bcm4908"; + compatible = "asus,gt-ac5300", "brcm,bcm4908", "brcm,bcmbca"; model = "Asus GT-AC5300"; memory@0 { @@ -118,6 +119,53 @@ }; }; +&leds { + led-power@11 { + reg = <0x11>; + function = LED_FUNCTION_POWER; + color = ; + default-state = "on"; + active-low; + pinctrl-names = "default"; + pinctrl-0 = <&pins_led_17_a>; + }; + + led-wan-red@12 { + reg = <0x12>; + function = LED_FUNCTION_WAN; + color = ; + active-low; + pinctrl-names = "default"; + pinctrl-0 = <&pins_led_18_a>; + }; + + led-wps@14 { + reg = <0x14>; + function = LED_FUNCTION_WPS; + color = ; + active-low; + pinctrl-names = "default"; + pinctrl-0 = <&pins_led_20_a>; + }; + + led-wan-white@15 { + reg = <0x15>; + function = LED_FUNCTION_WAN; + color = ; + active-low; + pinctrl-names = "default"; + pinctrl-0 = <&pins_led_21_a>; + }; + + led-lan@19 { + reg = <0x19>; + function = LED_FUNCTION_LAN; + color = ; + pinctrl-names = "default"; + pinctrl-0 = <&pins_led_25_a>; + }; +}; + &nandcs { nand-ecc-strength = <4>; nand-ecc-step-size = <512>; diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-netgear-raxe500.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-netgear-raxe500.dts similarity index 89% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-netgear-raxe500.dts rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-netgear-raxe500.dts index 3c2cf2d238b6fff90bcfa26902a04d01b1d206d9..23b96c663239b3f3d93825fec5d5b941ccb03c1a 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-netgear-raxe500.dts +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-netgear-raxe500.dts @@ -3,7 +3,7 @@ #include "bcm4908.dtsi" / { - compatible = "netgear,raxe500", "brcm,bcm4908"; + compatible = "netgear,raxe500", "brcm,bcm4908", "brcm,bcmbca"; model = "Netgear RAXE500"; memory@0 { diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908.dtsi similarity index 86% rename from arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi rename to arch/arm64/boot/dts/broadcom/bcmbca/bcm4908.dtsi index 967d2cd3c3cee4610de6d92416b84881d2e218b0..dac9d3b4e91dff939e43fb152d6b37cd51cf9dda 100644 --- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908.dtsi @@ -352,6 +352,61 @@ groups = "led_9_grp_a"; }; + pins_led_10_a: led_10-a-pins { + function = "led_10"; + groups = "led_10_grp_a"; + }; + + pins_led_11_a: led_11-a-pins { + function = "led_11"; + groups = "led_11_grp_a"; + }; + + pins_led_12_a: led_12-a-pins { + function = "led_12"; + groups = "led_12_grp_a"; + }; + + pins_led_13_a: led_13-a-pins { + function = "led_13"; + groups = "led_13_grp_a"; + }; + + pins_led_14_a: led_14-a-pins { + function = "led_14"; + groups = "led_14_grp_a"; + }; + + pins_led_15_a: led_15-a-pins { + function = "led_15"; + groups = "led_15_grp_a"; + }; + + pins_led_16_a: led_16-a-pins { + function = "led_16"; + groups = "led_16_grp_a"; + }; + + pins_led_17_a: led_17-a-pins { + function = "led_17"; + groups = "led_17_grp_a"; + }; + + pins_led_18_a: led_18-a-pins { + function = "led_18"; + groups = "led_18_grp_a"; + }; + + pins_led_19_a: led_19-a-pins { + function = "led_19"; + groups = "led_19_grp_a"; + }; + + pins_led_20_a: led_20-a-pins { + function = "led_20"; + groups = "led_20_grp_a"; + }; + pins_led_21_a: led_21-a-pins { function = "led_21"; groups = "led_21_grp_a"; @@ -362,6 +417,21 @@ groups = "led_22_grp_a"; }; + pins_led_23_a: led_23-a-pins { + function = "led_23"; + groups = "led_23_grp_a"; + }; + + pins_led_24_a: led_24-a-pins { + function = "led_24"; + groups = "led_24_grp_a"; + }; + + pins_led_25_a: led_25-a-pins { + function = "led_25"; + groups = "led_25_grp_a"; + }; + pins_led_26_a: led_26-a-pins { function = "led_26"; groups = "led_26_grp_a"; @@ -387,6 +457,11 @@ groups = "led_30_grp_a"; }; + pins_led_31_a: led_31-a-pins { + function = "led_31"; + groups = "led_31_grp_a"; + }; + pins_hs_uart: hs_uart-pins { function = "hs_uart"; groups = "hs_uart_grp"; @@ -442,6 +517,14 @@ status = "okay"; }; + leds: leds@800 { + compatible = "brcm,bcm4908-leds", "brcm,bcm63138-leds"; + reg = <0x800 0xdc>; + + #address-cells = <1>; + #size-cells = <0>; + }; + nand-controller@1800 { #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts new file mode 100644 index 0000000000000000000000000000000000000000..fcbd3c430ace7eec1984009130946deba63eafe3 --- /dev/null +++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright 2022 Broadcom Ltd. + */ + +/dts-v1/; + +#include "bcm4908.dtsi" + +/ { + model = "Broadcom BCM94908 Reference Board"; + compatible = "brcm,bcm94908", "brcm,bcm4908", "brcm,bcmbca"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@0 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x08000000>; + }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi index 91c9bd1b47ddfda142ca3e5ada6e7c7b8756c395..bde6a6bb8dfcfcbab97e27d68de8d1b689fd931a 100644 --- a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi +++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi @@ -795,7 +795,7 @@ reg = <0x27>; interrupt-parent = <&gpa1>; interrupts = <3 IRQ_TYPE_EDGE_RISING>; - en-gpios = <&gpf1 4 GPIO_ACTIVE_HIGH>; + en-gpios = <&gpf1 4 GPIO_ACTIVE_LOW>; wake-gpios = <&gpj0 2 GPIO_ACTIVE_HIGH>; }; }; diff --git a/arch/arm64/boot/dts/exynos/exynos850.dtsi b/arch/arm64/boot/dts/exynos/exynos850.dtsi index 9076afd4bb3e70c6617b643ca41aa2b641ffc2a4..c61441f3a89a77892f199719163aa84d3fe2f22c 100644 --- a/arch/arm64/boot/dts/exynos/exynos850.dtsi +++ b/arch/arm64/boot/dts/exynos/exynos850.dtsi @@ -286,6 +286,21 @@ clock-names = "oscclk"; }; + cmu_mfcmscl: clock-controller@12c00000 { + compatible = "samsung,exynos850-cmu-mfcmscl"; + reg = <0x12c00000 0x8000>; + #clock-cells = <1>; + + clocks = <&oscclk>, + <&cmu_top CLK_DOUT_MFCMSCL_MFC>, + <&cmu_top CLK_DOUT_MFCMSCL_M2M>, + <&cmu_top CLK_DOUT_MFCMSCL_MCSC>, + <&cmu_top CLK_DOUT_MFCMSCL_JPEG>; + clock-names = "oscclk", "dout_mfcmscl_mfc", + "dout_mfcmscl_m2m", "dout_mfcmscl_mcsc", + "dout_mfcmscl_jpeg"; + }; + cmu_dpu: clock-controller@13000000 { compatible = "samsung,exynos850-cmu-dpu"; reg = <0x13000000 0x8000>; @@ -308,6 +323,29 @@ "dout_hsi_mmc_card", "dout_hsi_usb20drd"; }; + cmu_is: clock-controller@14500000 { + compatible = "samsung,exynos850-cmu-is"; + reg = <0x14500000 0x8000>; + #clock-cells = <1>; + + clocks = <&oscclk>, + <&cmu_top CLK_DOUT_IS_BUS>, + <&cmu_top CLK_DOUT_IS_ITP>, + <&cmu_top CLK_DOUT_IS_VRA>, + <&cmu_top CLK_DOUT_IS_GDC>; + clock-names = "oscclk", "dout_is_bus", "dout_is_itp", + "dout_is_vra", "dout_is_gdc"; + }; + + cmu_aud: clock-controller@14a00000 { + compatible = "samsung,exynos850-cmu-aud"; + reg = <0x14a00000 0x8000>; + #clock-cells = <1>; + + clocks = <&oscclk>, <&cmu_top CLK_DOUT_AUD>; + clock-names = "oscclk", "dout_aud"; + }; + pinctrl_alive: pinctrl@11850000 { compatible = "samsung,exynos850-pinctrl"; reg = <0x11850000 0x1000>; @@ -465,6 +503,51 @@ status = "disabled"; }; + sysmmu_mfcmscl: sysmmu@12c50000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x12c50000 0x9000>; + interrupts = ; + clock-names = "sysmmu"; + clocks = <&cmu_mfcmscl CLK_GOUT_MFCMSCL_SYSMMU_CLK>; + #iommu-cells = <0>; + }; + + sysmmu_dpu: sysmmu@130c0000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x130c0000 0x9000>; + interrupts = ; + clock-names = "sysmmu"; + clocks = <&cmu_dpu CLK_GOUT_DPU_SMMU_CLK>; + #iommu-cells = <0>; + }; + + sysmmu_is0: sysmmu@14550000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x14550000 0x9000>; + interrupts = ; + clock-names = "sysmmu"; + clocks = <&cmu_is CLK_GOUT_IS_SYSMMU_IS0_CLK>; + #iommu-cells = <0>; + }; + + sysmmu_is1: sysmmu@14570000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x14570000 0x9000>; + interrupts = ; + clock-names = "sysmmu"; + clocks = <&cmu_is CLK_GOUT_IS_SYSMMU_IS1_CLK>; + #iommu-cells = <0>; + }; + + sysmmu_aud: sysmmu@14850000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x14850000 0x9000>; + interrupts = ; + clock-names = "sysmmu"; + clocks = <&cmu_aud CLK_GOUT_AUD_SYSMMU_CLK>; + #iommu-cells = <0>; + }; + sysreg_peri: syscon@10020000 { compatible = "samsung,exynos850-sysreg", "syscon"; reg = <0x10020000 0x10000>; diff --git a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi index 2013718532f3705bec5e00c1ad4eec5fa7f897d6..5dc361734cfe42fe95388869e5924a4981ca5df9 100644 --- a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi +++ b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi @@ -207,6 +207,34 @@ "dout_clkcmu_peric1_ip"; }; + cmu_fsys1: clock-controller@17040000 { + compatible = "samsung,exynosautov9-cmu-fsys1"; + reg = <0x17040000 0x8000>; + #clock-cells = <1>; + + clocks = <&xtcxo>, + <&cmu_top DOUT_CLKCMU_FSYS1_BUS>, + <&cmu_top GOUT_CLKCMU_FSYS1_MMC_CARD>, + <&cmu_top DOUT_CLKCMU_FSYS1_USBDRD>; + clock-names = "oscclk", + "dout_clkcmu_fsys1_bus", + "gout_clkcmu_fsys1_mmc_card", + "dout_clkcmu_fsys1_usbdrd"; + }; + + cmu_fsys0: clock-controller@17700000 { + compatible = "samsung,exynosautov9-cmu-fsys0"; + reg = <0x17700000 0x8000>; + #clock-cells = <1>; + + clocks = <&xtcxo>, + <&cmu_top DOUT_CLKCMU_FSYS0_BUS>, + <&cmu_top DOUT_CLKCMU_FSYS0_PCIE>; + clock-names = "oscclk", + "dout_clkcmu_fsys0_bus", + "dout_clkcmu_fsys0_pcie"; + }; + cmu_fsys2: clock-controller@17c00000 { compatible = "samsung,exynosautov9-cmu-fsys2"; reg = <0x17c00000 0x8000>; diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile index 8bf7f7ecebaa1b8e4e45fdc37c8b6ac44892425f..3ea9edc87909a315f25aa8775d6d953062965ece 100644 --- a/arch/arm64/boot/dts/freescale/Makefile +++ b/arch/arm64/boot/dts/freescale/Makefile @@ -23,6 +23,7 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1088a-rdb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1088a-ten64.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-qds.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-rdb.dtb +dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2081a-rdb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-simu.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-qds.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-rdb.dtb @@ -48,6 +49,7 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-85bb.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-899b.dtb dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-9999.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8dxl-evk.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-beacon-kit.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-data-modul-edm-sbc.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-ddr4-evk.dtb @@ -55,7 +57,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-emcon-avari.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-evk.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-ctouch2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-edimm2.2.dtb -dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-n801x-s.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-bl.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-bl-osm-s.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-mx8menlo.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-nitrogen-r2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-phyboard-polis-rdk.dtb @@ -67,6 +70,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw73xx-0x.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7901.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7902.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7903.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7904.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-dahlia.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-dev.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-dahlia.dtb @@ -83,6 +87,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mn-venice-gw7902.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-dhcom-pdk2.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-icore-mx8mp-edimm2.2.dtb +dtb-$(CONFIG_ARCH_MXC) += imx8mp-msc-sm2s-ep1.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-rdk.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mpxl.dtb dtb-$(CONFIG_ARCH_MXC) += imx8mp-venice-gw74xx.dtb diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts index 6b575efd84a71470a47b57bcdd5147975536e62c..73eb6061c73eeaa475ef80812c2371476da36210 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts @@ -59,6 +59,10 @@ status = "okay"; }; +&enetc_port3 { + status = "okay"; +}; + &i2c3 { eeprom@57 { compatible = "atmel,24c32"; @@ -104,7 +108,10 @@ }; &mscc_felix_port4 { - ethernet = <&enetc_port2>; + status = "okay"; +}; + +&mscc_felix_port5 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts index 330e34f933a39288bc38b7d9d279f299a40e336c..113b1df74bf87cdebf0607ef3020c3e2b35ed6b2 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts @@ -39,6 +39,10 @@ status = "okay"; }; +&enetc_port3 { + status = "okay"; +}; + &mscc_felix { status = "okay"; }; @@ -60,6 +64,9 @@ }; &mscc_felix_port4 { - ethernet = <&enetc_port2>; + status = "okay"; +}; + +&mscc_felix_port5 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds-65bb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds-65bb.dts index 40d34c8384a5e10392896320a383a449559df086..b949cac0374272714a1d1d66e6d8238ceb726d22 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds-65bb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-qds-65bb.dts @@ -25,7 +25,6 @@ &enetc_port0 { phy-handle = <&slot1_sgmii>; phy-mode = "2500base-x"; - managed = "in-band-status"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts index e0cd1516d05b92d365f5310b45a70e4b419adbe3..ecd2c1ea177f8febe5832a6624b1bf10a669b8a0 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts @@ -29,6 +29,9 @@ ethernet3 = &mscc_felix_port1; ethernet4 = &mscc_felix_port2; ethernet5 = &mscc_felix_port3; + ethernet6 = &mscc_felix_port4; + ethernet7 = &mscc_felix_port5; + ethernet8 = &enetc_port3; }; chosen { @@ -151,6 +154,10 @@ status = "okay"; }; +&enetc_port3 { + status = "okay"; +}; + &esdhc { sd-uhs-sdr104; sd-uhs-sdr50; @@ -278,7 +285,10 @@ }; &mscc_felix_port4 { - ethernet = <&enetc_port2>; + status = "okay"; +}; + +&mscc_felix_port5 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi index 5627dd7734f38f6c7b7d0f3900826d3e7d6ab137..ac1c3a7e5f7a57114979865d0aa428d181e13842 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi @@ -1156,6 +1156,7 @@ mscc_felix_port4: port@4 { reg = <4>; phy-mode = "internal"; + ethernet = <&enetc_port2>; status = "disabled"; fixed-link { @@ -1168,6 +1169,7 @@ mscc_felix_port5: port@5 { reg = <5>; phy-mode = "internal"; + ethernet = <&enetc_port3>; status = "disabled"; fixed-link { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts index fea167d222cfe664b2cb34df4e7e316824f9f536..9b726c2a4842671a5a8cdb0db49e1c5eccdf5c2d 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts @@ -3,7 +3,7 @@ * Device Tree Include file for Freescale Layerscape-1043A family SoC. * * Copyright 2014-2015 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2021 NXP * * Mingkai Hu */ @@ -24,6 +24,22 @@ serial1 = &duart1; serial2 = &duart2; serial3 = &duart3; + sgmii-riser-s1-p1 = &sgmii_phy_s1_p1; + sgmii-riser-s2-p1 = &sgmii_phy_s2_p1; + sgmii-riser-s3-p1 = &sgmii_phy_s3_p1; + sgmii-riser-s4-p1 = &sgmii_phy_s4_p1; + qsgmii-s1-p1 = &qsgmii_phy_s1_p1; + qsgmii-s1-p2 = &qsgmii_phy_s1_p2; + qsgmii-s1-p3 = &qsgmii_phy_s1_p3; + qsgmii-s1-p4 = &qsgmii_phy_s1_p4; + qsgmii-s2-p1 = &qsgmii_phy_s2_p1; + qsgmii-s2-p2 = &qsgmii_phy_s2_p2; + qsgmii-s2-p3 = &qsgmii_phy_s2_p3; + qsgmii-s2-p4 = &qsgmii_phy_s2_p4; + emi1-slot1 = &ls1043mdio_s1; + emi1-slot2 = &ls1043mdio_s2; + emi1-slot3 = &ls1043mdio_s3; + emi1-slot4 = &ls1043mdio_s4; }; chosen { @@ -62,8 +78,11 @@ }; fpga: board-control@2,0 { - compatible = "fsl,ls1043aqds-fpga", "fsl,fpga-qixis"; + compatible = "fsl,ls1043aqds-fpga", "fsl,fpga-qixis", "simple-mfd"; reg = <0x2 0x0 0x0000100>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 2 0 0x100>; }; }; @@ -153,3 +172,153 @@ }; #include "fsl-ls1043-post.dtsi" + +&fman0 { + ethernet@e0000 { + phy-handle = <&qsgmii_phy_s2_p1>; + phy-connection-type = "sgmii"; + }; + + ethernet@e2000 { + phy-handle = <&qsgmii_phy_s2_p2>; + phy-connection-type = "sgmii"; + }; + + ethernet@e4000 { + phy-handle = <&rgmii_phy1>; + phy-connection-type = "rgmii"; + }; + + ethernet@e6000 { + phy-handle = <&rgmii_phy2>; + phy-connection-type = "rgmii"; + }; + + ethernet@e8000 { + phy-handle = <&qsgmii_phy_s2_p3>; + phy-connection-type = "sgmii"; + }; + + ethernet@ea000 { + phy-handle = <&qsgmii_phy_s2_p4>; + phy-connection-type = "sgmii"; + }; + + ethernet@f0000 { /* DTSEC9/10GEC1 */ + fixed-link = <1 1 10000 0 0>; + phy-connection-type = "xgmii"; + }; +}; + +&fpga { + mdio-mux-emi1@54 { + compatible = "mdio-mux-mmioreg", "mdio-mux"; + mdio-parent-bus = <&mdio0>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x54 1>; /* BRDCFG4 */ + mux-mask = <0xe0>; /* EMI1 */ + + /* On-board RGMII1 PHY */ + ls1043mdio0: mdio@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy1: ethernet-phy@1 { /* MAC3 */ + reg = <0x1>; + }; + }; + + /* On-board RGMII2 PHY */ + ls1043mdio1: mdio@20 { + reg = <0x20>; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy2: ethernet-phy@2 { /* MAC4 */ + reg = <0x2>; + }; + }; + + /* Slot 1 */ + ls1043mdio_s1: mdio@40 { + reg = <0x40>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + qsgmii_phy_s1_p1: ethernet-phy@4 { + reg = <0x4>; + }; + + qsgmii_phy_s1_p2: ethernet-phy@5 { + reg = <0x5>; + }; + + qsgmii_phy_s1_p3: ethernet-phy@6 { + reg = <0x6>; + }; + + qsgmii_phy_s1_p4: ethernet-phy@7 { + reg = <0x7>; + }; + + sgmii_phy_s1_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + }; + + /* Slot 2 */ + ls1043mdio_s2: mdio@60 { + reg = <0x60>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + qsgmii_phy_s2_p1: ethernet-phy@8 { + reg = <0x8>; + }; + + qsgmii_phy_s2_p2: ethernet-phy@9 { + reg = <0x9>; + }; + + qsgmii_phy_s2_p3: ethernet-phy@a { + reg = <0xa>; + }; + + qsgmii_phy_s2_p4: ethernet-phy@b { + reg = <0xb>; + }; + + sgmii_phy_s2_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + }; + + /* Slot 3 */ + ls1043mdio_s3: mdio@80 { + reg = <0x80>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + sgmii_phy_s3_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + }; + + /* Slot 4 */ + ls1043mdio_s4: mdio@a0 { + reg = <0xa0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + sgmii_phy_s4_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts index b290605e92cfb53e0452bb669d26ca685275fd05..26f8540cb101b121aec5774b98fd497163494077 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts @@ -29,23 +29,33 @@ &i2c0 { status = "okay"; + ina220@40 { compatible = "ti,ina220"; reg = <0x40>; shunt-resistor = <1000>; }; + adt7461a@4c { compatible = "adi,adt7461"; reg = <0x4c>; }; + + rtc@51 { + compatible = "nxp,pcf85263"; + reg = <0x51>; + }; + eeprom@52 { compatible = "atmel,24c512"; reg = <0x52>; }; + eeprom@53 { compatible = "atmel,24c512"; reg = <0x53>; }; + rtc@68 { compatible = "pericom,pt7c4338"; reg = <0x68>; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi index ca3d5a90d6d4a8ae966e7f716f978941afb504c1..704f72caddd3bbb93d271052d9f4a4c61b60a0db 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { compatible = "fsl,ls1043a"; @@ -300,6 +301,8 @@ #address-cells = <2>; #size-cells = <2>; ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>; + dma-coherent; clockgen: clocking@1ee1000 { compatible = "fsl,ls1043a-clockgen"; @@ -393,7 +396,7 @@ dcfg: dcfg@1ee0000 { compatible = "fsl,ls1043a-dcfg", "syscon"; - reg = <0x0 0x1ee0000 0x0 0x10000>; + reg = <0x0 0x1ee0000 0x0 0x1000>; big-endian; }; @@ -536,7 +539,7 @@ }; i2c0: i2c@2180000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x2180000 0x0 0x10000>; @@ -551,7 +554,7 @@ }; i2c1: i2c@2190000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x2190000 0x0 0x10000>; @@ -559,11 +562,12 @@ clock-names = "i2c"; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(1)>; + scl-gpios = <&gpio4 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; i2c2: i2c@21a0000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x21a0000 0x0 0x10000>; @@ -571,11 +575,12 @@ clock-names = "i2c"; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(1)>; + scl-gpios = <&gpio4 10 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; i2c3: i2c@21b0000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x21b0000 0x0 0x10000>; @@ -583,6 +588,7 @@ clock-names = "i2c"; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(1)>; + scl-gpios = <&gpio4 12 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -810,48 +816,59 @@ QORIQ_CLK_PLL_DIV(1)>; }; - usb0: usb@2f00000 { - compatible = "snps,dwc3"; - reg = <0x0 0x2f00000 0x0 0x10000>; - interrupts = <0 60 0x4>; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - status = "disabled"; - }; + aux_bus: aux_bus { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x100 0x00000000>; + + usb0: usb@2f00000 { + compatible = "snps,dwc3"; + reg = <0x0 0x2f00000 0x0 0x10000>; + interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + usb3-lpm-capable; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + status = "disabled"; + }; - usb1: usb@3000000 { - compatible = "snps,dwc3"; - reg = <0x0 0x3000000 0x0 0x10000>; - interrupts = <0 61 0x4>; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - status = "disabled"; - }; + usb1: usb@3000000 { + compatible = "snps,dwc3"; + reg = <0x0 0x3000000 0x0 0x10000>; + interrupts = <0 61 IRQ_TYPE_LEVEL_HIGH>; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + usb3-lpm-capable; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + status = "disabled"; + }; - usb2: usb@3100000 { - compatible = "snps,dwc3"; - reg = <0x0 0x3100000 0x0 0x10000>; - interrupts = <0 63 0x4>; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - status = "disabled"; - }; + usb2: usb@3100000 { + compatible = "snps,dwc3"; + reg = <0x0 0x3100000 0x0 0x10000>; + interrupts = <0 63 IRQ_TYPE_LEVEL_HIGH>; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + usb3-lpm-capable; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + status = "disabled"; + }; - sata: sata@3200000 { - compatible = "fsl,ls1043a-ahci"; - reg = <0x0 0x3200000 0x0 0x10000>, - <0x0 0x20140520 0x0 0x4>; - reg-names = "ahci", "sata-ecc"; - interrupts = <0 69 0x4>; - clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL - QORIQ_CLK_PLL_DIV(1)>; - dma-coherent; + sata: sata@3200000 { + compatible = "fsl,ls1043a-ahci"; + reg = <0x0 0x3200000 0x0 0x10000>, + <0x0 0x20140520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = <0 69 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(1)>; + dma-coherent; + }; }; msi1: msi-controller1@1571000 { @@ -880,13 +897,12 @@ reg = <0x00 0x03400000 0x0 0x00100000>, /* controller registers */ <0x40 0x00000000 0x0 0x00002000>; /* configuration space */ reg-names = "regs", "config"; - interrupts = <0 118 0x4>, /* controller interrupt */ - <0 117 0x4>; /* PME interrupt */ - interrupt-names = "intr", "pme"; + interrupts = <0 117 IRQ_TYPE_LEVEL_HIGH>, + <0 118 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "pme", "aer"; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <6>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x40 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -898,6 +914,8 @@ <0000 0 0 2 &gic 0 111 0x4>, <0000 0 0 3 &gic 0 112 0x4>, <0000 0 0 4 &gic 0 113 0x4>; + fsl,pcie-scfg = <&scfg 0>; + big-endian; status = "disabled"; }; @@ -906,13 +924,12 @@ reg = <0x00 0x03500000 0x0 0x00100000>, /* controller registers */ <0x48 0x00000000 0x0 0x00002000>; /* configuration space */ reg-names = "regs", "config"; - interrupts = <0 128 0x4>, - <0 127 0x4>; - interrupt-names = "intr", "pme"; + interrupts = <0 127 IRQ_TYPE_LEVEL_HIGH>, + <0 128 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "pme", "aer"; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <6>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x48 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -924,6 +941,8 @@ <0000 0 0 2 &gic 0 121 0x4>, <0000 0 0 3 &gic 0 122 0x4>, <0000 0 0 4 &gic 0 123 0x4>; + fsl,pcie-scfg = <&scfg 1>; + big-endian; status = "disabled"; }; @@ -932,13 +951,12 @@ reg = <0x00 0x03600000 0x0 0x00100000>, /* controller registers */ <0x50 0x00000000 0x0 0x00002000>; /* configuration space */ reg-names = "regs", "config"; - interrupts = <0 162 0x4>, - <0 161 0x4>; - interrupt-names = "intr", "pme"; + interrupts = <0 161 IRQ_TYPE_LEVEL_HIGH>, + <0 162 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "pme", "aer"; #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <6>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x50 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -950,6 +968,8 @@ <0000 0 0 2 &gic 0 155 0x4>, <0000 0 0 3 &gic 0 156 0x4>, <0000 0 0 4 &gic 0 157 0x4>; + fsl,pcie-scfg = <&scfg 2>; + big-endian; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts index eec62c63dafe2430a781f40b2aa4c06748698fa9..b2fcbba60d3accdf03ef8fe51d7de98e6f0818a8 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts @@ -3,7 +3,7 @@ * Device Tree Include file for Freescale Layerscape-1046A family SoC. * * Copyright 2016 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * Shaohui Xie */ @@ -17,14 +17,26 @@ compatible = "fsl,ls1046a-qds", "fsl,ls1046a"; aliases { + emi1-slot1 = &ls1046mdio_s1; + emi1-slot2 = &ls1046mdio_s2; + emi1-slot4 = &ls1046mdio_s4; gpio0 = &gpio0; gpio1 = &gpio1; gpio2 = &gpio2; gpio3 = &gpio3; + qsgmii-s2-p1 = &qsgmii_phy_s2_p1; + qsgmii-s2-p2 = &qsgmii_phy_s2_p2; + qsgmii-s2-p3 = &qsgmii_phy_s2_p3; + qsgmii-s2-p4 = &qsgmii_phy_s2_p4; serial0 = &duart0; serial1 = &duart1; serial2 = &duart2; serial3 = &duart3; + sgmii-s1-p1 = &sgmii_phy_s1_p1; + sgmii-s1-p2 = &sgmii_phy_s1_p2; + sgmii-s1-p3 = &sgmii_phy_s1_p3; + sgmii-s1-p4 = &sgmii_phy_s1_p4; + sgmii-s4-p1 = &sgmii_phy_s4_p1; }; chosen { @@ -153,8 +165,9 @@ }; fpga: board-control@2,0 { - compatible = "fsl,ls1046aqds-fpga", "fsl,fpga-qixis"; + compatible = "fsl,ls1046aqds-fpga", "fsl,fpga-qixis", "simple-mfd"; reg = <0x2 0x0 0x0000100>; + ranges = <0 2 0 0x100>; }; }; @@ -169,7 +182,7 @@ compatible = "spansion,m25p80"; #address-cells = <1>; #size-cells = <1>; - spi-max-frequency = <20000000>; + spi-max-frequency = <50000000>; spi-rx-bus-width = <4>; spi-tx-bus-width = <4>; reg = <0>; @@ -177,3 +190,141 @@ }; #include "fsl-ls1046-post.dtsi" + +&fman0 { + ethernet@e0000 { + phy-handle = <&qsgmii_phy_s2_p1>; + phy-connection-type = "sgmii"; + }; + + ethernet@e2000 { + phy-handle = <&sgmii_phy_s4_p1>; + phy-connection-type = "sgmii"; + }; + + ethernet@e4000 { + phy-handle = <&rgmii_phy1>; + phy-connection-type = "rgmii"; + }; + + ethernet@e6000 { + phy-handle = <&rgmii_phy2>; + phy-connection-type = "rgmii"; + }; + + ethernet@e8000 { + phy-handle = <&sgmii_phy_s1_p3>; + phy-connection-type = "sgmii"; + }; + + ethernet@ea000 { + phy-handle = <&sgmii_phy_s1_p4>; + phy-connection-type = "sgmii"; + }; + + ethernet@f0000 { /* DTSEC9/10GEC1 */ + phy-handle = <&sgmii_phy_s1_p1>; + phy-connection-type = "xgmii"; + }; + + ethernet@f2000 { /* DTSEC10/10GEC2 */ + phy-handle = <&sgmii_phy_s1_p2>; + phy-connection-type = "xgmii"; + }; +}; + +&fpga { + #address-cells = <1>; + #size-cells = <1>; + + mdio-mux-emi1 { + compatible = "mdio-mux-mmioreg", "mdio-mux"; + mdio-parent-bus = <&mdio0>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x54 1>; /* BRDCFG4 */ + mux-mask = <0xe0>; /* EMI1 */ + + /* On-board RGMII1 PHY */ + ls1046mdio0: mdio@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy1: ethernet-phy@1 { /* MAC3 */ + reg = <0x1>; + }; + }; + + /* On-board RGMII2 PHY */ + ls1046mdio1: mdio@1 { + reg = <0x20>; + #address-cells = <1>; + #size-cells = <0>; + + rgmii_phy2: ethernet-phy@2 { /* MAC4 */ + reg = <0x2>; + }; + }; + + /* Slot 1 */ + ls1046mdio_s1: mdio@2 { + reg = <0x40>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + sgmii_phy_s1_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + + sgmii_phy_s1_p2: ethernet-phy@1d { + reg = <0x1d>; + }; + + sgmii_phy_s1_p3: ethernet-phy@1e { + reg = <0x1e>; + }; + + sgmii_phy_s1_p4: ethernet-phy@1f { + reg = <0x1f>; + }; + }; + + /* Slot 2 */ + ls1046mdio_s2: mdio@3 { + reg = <0x60>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + qsgmii_phy_s2_p1: ethernet-phy@8 { + reg = <0x8>; + }; + + qsgmii_phy_s2_p2: ethernet-phy@9 { + reg = <0x9>; + }; + + qsgmii_phy_s2_p3: ethernet-phy@a { + reg = <0xa>; + }; + + qsgmii_phy_s2_p4: ethernet-phy@b { + reg = <0xb>; + }; + }; + + /* Slot 4 */ + ls1046mdio_s4: mdio@5 { + reg = <0x80>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + + sgmii_phy_s4_p1: ethernet-phy@1c { + reg = <0x1c>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi index feab604322cf3e48a70289f87ef54a292128b8c3..3d9e29824bb2113ac1b0db8f95ac124b97ae89b9 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi @@ -11,6 +11,7 @@ #include #include #include +#include / { compatible = "fsl,ls1046a"; @@ -272,6 +273,8 @@ #address-cells = <2>; #size-cells = <2>; ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>; + dma-coherent; ddr: memory-controller@1080000 { compatible = "fsl,qoriq-memory-controller"; @@ -354,7 +357,6 @@ ranges = <0x0 0x00 0x1700000 0x100000>; reg = <0x00 0x1700000 0x0 0x100000>; interrupts = ; - dma-coherent; sec_jr0: jr@10000 { compatible = "fsl,sec-v5.4-job-ring", @@ -500,7 +502,7 @@ }; i2c0: i2c@2180000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x2180000 0x0 0x10000>; @@ -514,35 +516,38 @@ }; i2c1: i2c@2190000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x2190000 0x0 0x10000>; interrupts = ; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(2)>; + scl-gpios = <&gpio3 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; i2c2: i2c@21a0000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x21a0000 0x0 0x10000>; interrupts = ; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(2)>; + scl-gpios = <&gpio3 10 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; i2c3: i2c@21b0000 { - compatible = "fsl,vf610-i2c"; + compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x21b0000 0x0 0x10000>; interrupts = ; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(2)>; + scl-gpios = <&gpio3 12 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -709,44 +714,55 @@ QORIQ_CLK_PLL_DIV(2)>; }; - usb0: usb@2f00000 { - compatible = "snps,dwc3"; - reg = <0x0 0x2f00000 0x0 0x10000>; - interrupts = ; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - }; - - usb1: usb@3000000 { - compatible = "snps,dwc3"; - reg = <0x0 0x3000000 0x0 0x10000>; - interrupts = ; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - }; - - usb2: usb@3100000 { - compatible = "snps,dwc3"; - reg = <0x0 0x3100000 0x0 0x10000>; - interrupts = ; - dr_mode = "host"; - snps,quirk-frame-length-adjustment = <0x20>; - snps,dis_rxdet_inp3_quirk; - snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; - }; - - sata: sata@3200000 { - compatible = "fsl,ls1046a-ahci"; - reg = <0x0 0x3200000 0x0 0x10000>, - <0x0 0x20140520 0x0 0x4>; - reg-names = "ahci", "sata-ecc"; - interrupts = ; - clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL - QORIQ_CLK_PLL_DIV(2)>; + aux_bus: aux_bus { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x100 0x00000000>; + + usb0: usb@2f00000 { + compatible = "snps,dwc3"; + reg = <0x0 0x2f00000 0x0 0x10000>; + interrupts = ; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + usb3-lpm-capable; + }; + + usb1: usb@3000000 { + compatible = "snps,dwc3"; + reg = <0x0 0x3000000 0x0 0x10000>; + interrupts = ; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + usb3-lpm-capable; + }; + + usb2: usb@3100000 { + compatible = "snps,dwc3"; + reg = <0x0 0x3100000 0x0 0x10000>; + interrupts = ; + dr_mode = "host"; + snps,quirk-frame-length-adjustment = <0x20>; + snps,dis_rxdet_inp3_quirk; + snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>; + usb3-lpm-capable; + }; + + sata: sata@3200000 { + compatible = "fsl,ls1046a-ahci"; + reg = <0x0 0x3200000 0x0 0x10000>, + <0x0 0x20140520 0x0 0x4>; + reg-names = "ahci", "sata-ecc"; + interrupts = ; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(2)>; + }; }; msi1: msi-controller@1580000 { @@ -790,7 +806,6 @@ #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <8>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x40 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -802,6 +817,7 @@ <0000 0 0 2 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 3 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 4 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>; + big-endian; status = "disabled"; }; @@ -810,8 +826,11 @@ reg = <0x00 0x03400000 0x0 0x00100000>, <0x40 0x00000000 0x8 0x00000000>; reg-names = "regs", "addr_space"; + interrupts = ; + interrupt-names = "pme"; num-ib-windows = <6>; num-ob-windows = <8>; + big-endian; status = "disabled"; }; @@ -826,7 +845,6 @@ #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <8>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x48 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -838,6 +856,7 @@ <0000 0 0 2 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 3 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 4 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>; + big-endian; status = "disabled"; }; @@ -846,8 +865,11 @@ reg = <0x00 0x03500000 0x0 0x00100000>, <0x48 0x00000000 0x8 0x00000000>; reg-names = "regs", "addr_space"; + interrupts = ; + interrupt-names = "pme"; num-ib-windows = <6>; num-ob-windows = <8>; + big-endian; status = "disabled"; }; @@ -862,7 +884,6 @@ #address-cells = <3>; #size-cells = <2>; device_type = "pci"; - dma-coherent; num-viewport = <8>; bus-range = <0x0 0xff>; ranges = <0x81000000 0x0 0x00000000 0x50 0x00010000 0x0 0x00010000 /* downstream I/O */ @@ -874,6 +895,7 @@ <0000 0 0 2 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 3 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>, <0000 0 0 4 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>; + big-endian; status = "disabled"; }; @@ -882,8 +904,11 @@ reg = <0x00 0x03600000 0x0 0x00100000>, <0x50 0x00000000 0x8 0x00000000>; reg-names = "regs", "addr_space"; + interrupts = ; + interrupt-names = "pme"; num-ib-windows = <6>; num-ob-windows = <8>; + big-endian; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts index 44894356059ce0f0f405331220536ab6e8e55d88..8b691513699721cdb2c59926da7383e08a21cbd8 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts @@ -14,6 +14,7 @@ #include "fsl-ls2080a.dtsi" #include "fsl-ls208xa-rdb.dtsi" +#include / { model = "Freescale Layerscape 2080a RDB Board"; @@ -23,3 +24,71 @@ stdout-path = "serial1:115200n8"; }; }; + +&dpmac5 { + phy-handle = <&mdio2_phy1>; + phy-connection-type = "10gbase-r"; +}; + +&dpmac6 { + phy-handle = <&mdio2_phy2>; + phy-connection-type = "10gbase-r"; +}; + +&dpmac7 { + phy-handle = <&mdio2_phy3>; + phy-connection-type = "10gbase-r"; +}; + +&dpmac8 { + phy-handle = <&mdio2_phy4>; + phy-connection-type = "10gbase-r"; +}; + +&emdio1 { + status = "disabled"; + + /* CS4340 PHYs */ + mdio1_phy1: emdio1-phy@10 { + reg = <0x10>; + }; + + mdio1_phy2: emdio1-phy@11 { + reg = <0x11>; + }; + + mdio1_phy3: emdio1-phy@12 { + reg = <0x12>; + }; + + mdio1_phy4: emdio1-phy@13 { + reg = <0x13>; + }; +}; + +&emdio2 { + /* AQR405 PHYs */ + mdio2_phy1: emdio2-phy@0 { + compatible = "ethernet-phy-ieee802.3-c45"; + interrupts = ; + reg = <0x0>; + }; + + mdio2_phy2: emdio2-phy@1 { + compatible = "ethernet-phy-ieee802.3-c45"; + interrupts = ; + reg = <0x1>; + }; + + mdio2_phy3: emdio2-phy@2 { + compatible = "ethernet-phy-ieee802.3-c45"; + interrupts = ; + reg = <0x2>; + }; + + mdio2_phy4: emdio2-phy@3 { + compatible = "ethernet-phy-ieee802.3-c45"; + interrupts = ; + reg = <0x3>; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi index 6f6667b70028efd4002bec94f82ab9d07b99d9c5..a2cadf7571482967f1d0940f7c117be3252eae0b 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi @@ -150,3 +150,7 @@ ranges = <0x81000000 0x0 0x00000000 0x16 0x00010000 0x0 0x00010000 /* downstream I/O */ 0x82000000 0x0 0x40000000 0x16 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */ }; + +&timer { + fsl,erratum-a008585; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts new file mode 100644 index 0000000000000000000000000000000000000000..4461e16fd53ad3f8f3a4c20ba5883cab3bf68d81 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device Tree file for NXP LS2081A RDB Board. + * + * Copyright 2017 NXP + * + * Priyanka Jain + * + */ + +/dts-v1/; + +#include "fsl-ls2088a.dtsi" + +/ { + model = "NXP Layerscape 2081A RDB Board"; + compatible = "fsl,ls2081a-rdb", "fsl,ls2081a"; + + aliases { + serial0 = &serial0; + serial1 = &serial1; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; +}; + +&dspi { + status = "okay"; + + n25q512a: flash@0 { + compatible = "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <3000000>; + reg = <0>; + }; +}; + +&esdhc { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + pca9547: mux@75 { + compatible = "nxp,pca9547"; + reg = <0x75>; + #address-cells = <1>; + #size-cells = <0>; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x1>; + + rtc@51 { + compatible = "nxp,pcf2129"; + reg = <0x51>; + }; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x2>; + + ina220@40 { + compatible = "ti,ina220"; + reg = <0x40>; + shunt-resistor = <500>; + }; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x3>; + + adt7481@4c { + compatible = "adi,adt7461"; + reg = <0x4c>; + }; + }; + }; +}; + +&ifc { + status = "disabled"; +}; + +&qspi { + status = "okay"; + + s25fs512s0: flash@0 { + compatible = "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + spi-max-frequency = <20000000>; + reg = <0>; + }; + + s25fs512s1: flash@1 { + compatible = "jedec,spi-nor"; + #address-cells = <1>; + #size-cells = <1>; + spi-rx-bus-width = <4>; + spi-tx-bus-width = <4>; + spi-max-frequency = <20000000>; + reg = <1>; + }; +}; + +&sata0 { + status = "okay"; +}; + +&sata1 { + status = "okay"; +}; + +&usb0 { + status = "okay"; +}; + +&usb1 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi index 6fab73d484b6f16b3e3fd7e034b258f0d6c34820..f598669e742fc940d444683f0ecaba5235e65331 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi @@ -9,6 +9,27 @@ * */ +/* Update DPMAC connections to external PHYs, under SerDes 0x2a_0x49. */ +&dpmac9 { + phy-handle = <&mdio0_phy12>; + phy-connection-type = "sgmii"; +}; + +&dpmac10 { + phy-handle = <&mdio0_phy13>; + phy-connection-type = "sgmii"; +}; + +&dpmac11 { + phy-handle = <&mdio0_phy14>; + phy-connection-type = "sgmii"; +}; + +&dpmac12 { + phy-handle = <&mdio0_phy15>; + phy-connection-type = "sgmii"; +}; + &esdhc { mmc-hs200-1_8v; status = "okay"; @@ -36,9 +57,47 @@ reg = <0x2 0x0 0x10000>; }; - cpld@3,0 { - reg = <0x3 0x0 0x10000>; - compatible = "fsl,ls2080aqds-fpga", "fsl,fpga-qixis"; + boardctrl: board-control@3,0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "fsl,ls208xaqds-fpga", "fsl,fpga-qixis", "simple-mfd"; + reg = <3 0 0x1000>; + ranges = <0 3 0 0x1000>; + + mdio-mux-emi1@54 { + compatible = "mdio-mux-mmioreg", "mdio-mux"; + mdio-parent-bus = <&emdio1>; + reg = <0x54 1>; /* BRDCFG4 */ + mux-mask = <0xe0>; /* EMI1_MDIO */ + #address-cells=<1>; + #size-cells = <0>; + + /* Child MDIO buses, one for each riser card: + * reg = 0x0, 0x20, 0x40, 0x60, 0x80, 0xa0. + * VSC8234 PHYs on the riser cards. + */ + mdio_mux3: mdio@60 { + reg = <0x60>; + #address-cells = <1>; + #size-cells = <0>; + + mdio0_phy12: mdio-phy0@1c { + reg = <0x1c>; + }; + + mdio0_phy13: mdio-phy1@1d { + reg = <0x1d>; + }; + + mdio0_phy14: mdio-phy2@1e { + reg = <0x1e>; + }; + + mdio0_phy15: mdio-phy3@1f { + reg = <0x1f>; + }; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi index f8135c5c252d048cf86b3017dc7d664a17d0e2d4..3d9647b3da14731420f72c4678aae20b3aedf5eb 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi @@ -49,6 +49,8 @@ reg = <0x75>; #address-cells = <1>; #size-cells = <0>; + idle-state = <0>; + i2c@1 { #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi index d76f1c42f3fa504aef468e3e02d70c26686d3011..f1b9cc8714dc0b40722583f28cc3cf40e156cb0a 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi @@ -239,13 +239,12 @@ }; }; - timer { + timer: timer { compatible = "arm,armv8-timer"; interrupts = <1 13 4>, /* Physical Secure PPI, active-low */ <1 14 4>, /* Physical Non-Secure PPI, active-low */ <1 11 4>, /* Virtual PPI, active-low */ <1 10 4>; /* Hypervisor PPI, active-low */ - fsl,erratum-a008585; }; pmu { diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi index 8b5cad4e270029883ae03ead30456e2d1ddbb0af..7d5183c6c5bef9b57f2ff92c5184272e43054a03 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi @@ -10,7 +10,7 @@ ddr_subsys: bus@5c000000 { #size-cells = <1>; ranges = <0x5c000000 0x0 0x5c000000 0x1000000>; - ddr-pmu@5c020000 { + ddr_pmu0: ddr-pmu@5c020000 { compatible = "fsl,imx8-ddr-pmu"; reg = <0x5c020000 0x10000>; interrupts = ; diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi index 960a802b8b6eea02635c588197787377867ffd3c..d7b4229bb4a2335a1346a71b2f89385c6ea166ef 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi @@ -111,8 +111,9 @@ dma_subsys: bus@5a000000 { i2c0: i2c@5a800000 { reg = <0x5a800000 0x4000>; interrupts = ; - clocks = <&i2c0_lpcg IMX_LPCG_CLK_0>; - clock-names = "per"; + clocks = <&i2c0_lpcg IMX_LPCG_CLK_0>, + <&i2c0_lpcg IMX_LPCG_CLK_4>; + clock-names = "per", "ipg"; assigned-clocks = <&clk IMX_SC_R_I2C_0 IMX_SC_PM_CLK_PER>; assigned-clock-rates = <24000000>; power-domains = <&pd IMX_SC_R_I2C_0>; @@ -122,8 +123,9 @@ dma_subsys: bus@5a000000 { i2c1: i2c@5a810000 { reg = <0x5a810000 0x4000>; interrupts = ; - clocks = <&i2c1_lpcg IMX_LPCG_CLK_0>; - clock-names = "per"; + clocks = <&i2c1_lpcg IMX_LPCG_CLK_0>, + <&i2c1_lpcg IMX_LPCG_CLK_4>; + clock-names = "per", "ipg"; assigned-clocks = <&clk IMX_SC_R_I2C_1 IMX_SC_PM_CLK_PER>; assigned-clock-rates = <24000000>; power-domains = <&pd IMX_SC_R_I2C_1>; @@ -133,8 +135,9 @@ dma_subsys: bus@5a000000 { i2c2: i2c@5a820000 { reg = <0x5a820000 0x4000>; interrupts = ; - clocks = <&i2c2_lpcg IMX_LPCG_CLK_0>; - clock-names = "per"; + clocks = <&i2c2_lpcg IMX_LPCG_CLK_0>, + <&i2c2_lpcg IMX_LPCG_CLK_4>; + clock-names = "per", "ipg"; assigned-clocks = <&clk IMX_SC_R_I2C_2 IMX_SC_PM_CLK_PER>; assigned-clock-rates = <24000000>; power-domains = <&pd IMX_SC_R_I2C_2>; @@ -144,8 +147,9 @@ dma_subsys: bus@5a000000 { i2c3: i2c@5a830000 { reg = <0x5a830000 0x4000>; interrupts = ; - clocks = <&i2c3_lpcg IMX_LPCG_CLK_0>; - clock-names = "per"; + clocks = <&i2c3_lpcg IMX_LPCG_CLK_0>, + <&i2c3_lpcg IMX_LPCG_CLK_4>; + clock-names = "per", "ipg"; assigned-clocks = <&clk IMX_SC_R_I2C_3 IMX_SC_PM_CLK_PER>; assigned-clock-rates = <24000000>; power-domains = <&pd IMX_SC_R_I2C_3>; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts new file mode 100644 index 0000000000000000000000000000000000000000..ca2a43e0cbf61b811df135ab52f4b4fd9379104d --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2019~2020, 2022 NXP + */ + +/dts-v1/; + +#include "imx8dxl.dtsi" + +/ { + model = "Freescale i.MX8DXL EVK"; + compatible = "fsl,imx8dxl-evk", "fsl,imx8dxl"; + + aliases { + i2c2 = &i2c2; + mmc0 = &usdhc1; + mmc1 = &usdhc2; + serial0 = &lpuart0; + }; + + chosen { + stdout-path = &lpuart0; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0 0x40000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* + * Memory reserved for optee usage. Please do not use. + * This will be automatically added to dtb if OP-TEE is installed. + * optee@96000000 { + * reg = <0 0x96000000 0 0x2000000>; + * no-map; + * }; + */ + + /* global autoconfigured region for contiguous allocations */ + linux,cma { + compatible = "shared-dma-pool"; + reusable; + size = <0 0x14000000>; + alloc-ranges = <0 0x98000000 0 0x14000000>; + linux,cma-default; + }; + }; + + mux3_en: regulator-0 { + compatible = "regulator-fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "mux3_en"; + gpio = <&pca6416_2 8 GPIO_ACTIVE_LOW>; + regulator-always-on; + }; + + reg_fec1_sel: regulator-1 { + compatible = "regulator-fixed"; + regulator-name = "fec1_supply"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&pca6416_1 11 GPIO_ACTIVE_LOW>; + regulator-always-on; + status = "disabled"; + }; + + reg_fec1_io: regulator-2 { + compatible = "regulator-fixed"; + regulator-name = "fec1_io_supply"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + gpio = <&max7322 0 GPIO_ACTIVE_HIGH>; + enable-active-high; + regulator-always-on; + status = "disabled"; + }; + + reg_usdhc2_vmmc: regulator-3 { + compatible = "regulator-fixed"; + regulator-name = "SD1_SPWR"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + gpio = <&lsio_gpio4 30 GPIO_ACTIVE_HIGH>; + enable-active-high; + off-on-delay-us = <3480>; + }; +}; + +&eqos { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_eqos>; + phy-mode = "rgmii-id"; + phy-handle = <ðphy0>; + nvmem-cells = <&fec_mac1>; + nvmem-cell-names = "mac-address"; + snps,reset-gpios = <&pca6416_1 2 GPIO_ACTIVE_LOW>; + snps,reset-delays-us = <10 20 200000>; + status = "okay"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + ethphy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + eee-broken-1000t; + qca,disable-smarteee; + vddio-supply = <&vddio0>; + + vddio0: vddio-regulator { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + }; + }; +}; + +/* + * fec1 shares the some PINs with usdhc2. + * by default usdhc2 is enabled in this dts. + * Please disable usdhc2 to enable fec1 + */ +&fec1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_fec1>; + phy-mode = "rgmii-txid"; + phy-handle = <ðphy1>; + fsl,magic-packet; + rx-internal-delay-ps = <2000>; + nvmem-cells = <&fec_mac0>; + nvmem-cell-names = "mac-address"; + status = "disabled"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy1: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + reset-gpios = <&pca6416_1 0 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + qca,disable-smarteee; + vddio-supply = <&vddio1>; + + vddio1: vddio-regulator { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + }; + }; +}; + +&i2c2 { + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c2>; + status = "okay"; + + pca6416_1: gpio@20 { + compatible = "ti,tca6416"; + reg = <0x20>; + gpio-controller; + #gpio-cells = <2>; + }; + + pca6416_2: gpio@21 { + compatible = "ti,tca6416"; + reg = <0x21>; + gpio-controller; + #gpio-cells = <2>; + }; + + pca9548_1: i2c-mux@70 { + compatible = "nxp,pca9548"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x70>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x0>; + + max7322: gpio@68 { + compatible = "maxim,max7322"; + reg = <0x68>; + gpio-controller; + #gpio-cells = <2>; + status = "disabled"; + }; + }; + + i2c@4 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x4>; + }; + + i2c@5 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x5>; + }; + + i2c@6 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0x6>; + }; + }; +}; + +&lpuart0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_lpuart0>; + status = "okay"; +}; + +&lsio_gpio4 { + status = "okay"; +}; + +&lsio_gpio5 { + status = "okay"; +}; + +&thermal_zones { + pmic-thermal0 { + polling-delay-passive = <250>; + polling-delay = <2000>; + thermal-sensors = <&tsens IMX_SC_R_PMIC_0>; + + trips { + pmic_alert0: trip0 { + temperature = <110000>; + hysteresis = <2000>; + type = "passive"; + }; + + pmic_crit0: trip1 { + temperature = <125000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&pmic_alert0>; + cooling-device = + <&A35_0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&A35_1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; +}; + +&usdhc1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usdhc1>; + bus-width = <8>; + no-sd; + no-sdio; + non-removable; + status = "okay"; +}; + +&usdhc2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>; + bus-width = <4>; + vmmc-supply = <®_usdhc2_vmmc>; + cd-gpios = <&lsio_gpio5 1 GPIO_ACTIVE_LOW>; + wp-gpios = <&lsio_gpio5 0 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&iomuxc { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_hog>; + + pinctrl_hog: hoggrp { + fsl,pins = < + IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHB_PAD 0x000514a0 + IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHK_PAD 0x000014a0 + IMX8DXL_SPI3_CS0_ADMA_ACM_MCLK_OUT1 0x0600004c + IMX8DXL_SNVS_TAMPER_OUT1_LSIO_GPIO2_IO05_IN 0x0600004c + >; + }; + + pinctrl_usbotg1: usbotg1grp { + fsl,pins = < + IMX8DXL_USB_SS3_TC0_CONN_USB_OTG1_PWR 0x00000021 + >; + }; + + pinctrl_usbotg2: usbotg2grp { + fsl,pins = < + IMX8DXL_USB_SS3_TC1_CONN_USB_OTG2_PWR 0x00000021 + >; + }; + + pinctrl_eqos: eqosgrp { + fsl,pins = < + IMX8DXL_ENET0_MDC_CONN_EQOS_MDC 0x06000020 + IMX8DXL_ENET0_MDIO_CONN_EQOS_MDIO 0x06000020 + IMX8DXL_ENET1_RGMII_RXC_CONN_EQOS_RGMII_RXC 0x06000020 + IMX8DXL_ENET1_RGMII_RXD0_CONN_EQOS_RGMII_RXD0 0x06000020 + IMX8DXL_ENET1_RGMII_RXD1_CONN_EQOS_RGMII_RXD1 0x06000020 + IMX8DXL_ENET1_RGMII_RXD2_CONN_EQOS_RGMII_RXD2 0x06000020 + IMX8DXL_ENET1_RGMII_RXD3_CONN_EQOS_RGMII_RXD3 0x06000020 + IMX8DXL_ENET1_RGMII_RX_CTL_CONN_EQOS_RGMII_RX_CTL 0x06000020 + IMX8DXL_ENET1_RGMII_TXC_CONN_EQOS_RGMII_TXC 0x06000020 + IMX8DXL_ENET1_RGMII_TXD0_CONN_EQOS_RGMII_TXD0 0x06000020 + IMX8DXL_ENET1_RGMII_TXD1_CONN_EQOS_RGMII_TXD1 0x06000020 + IMX8DXL_ENET1_RGMII_TXD2_CONN_EQOS_RGMII_TXD2 0x06000020 + IMX8DXL_ENET1_RGMII_TXD3_CONN_EQOS_RGMII_TXD3 0x06000020 + IMX8DXL_ENET1_RGMII_TX_CTL_CONN_EQOS_RGMII_TX_CTL 0x06000020 + >; + }; + + pinctrl_fec1: fec1grp { + fsl,pins = < + IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB0_PAD 0x000014a0 + IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB1_PAD 0x000014a0 + IMX8DXL_ENET0_MDC_CONN_ENET0_MDC 0x06000020 + IMX8DXL_ENET0_MDIO_CONN_ENET0_MDIO 0x06000020 + IMX8DXL_ENET0_RGMII_RXC_CONN_ENET0_RGMII_RXC 0x00000060 + IMX8DXL_ENET0_RGMII_RXD0_CONN_ENET0_RGMII_RXD0 0x00000060 + IMX8DXL_ENET0_RGMII_RXD1_CONN_ENET0_RGMII_RXD1 0x00000060 + IMX8DXL_ENET0_RGMII_RXD2_CONN_ENET0_RGMII_RXD2 0x00000060 + IMX8DXL_ENET0_RGMII_RXD3_CONN_ENET0_RGMII_RXD3 0x00000060 + IMX8DXL_ENET0_RGMII_RX_CTL_CONN_ENET0_RGMII_RX_CTL 0x00000060 + IMX8DXL_ENET0_RGMII_TXC_CONN_ENET0_RGMII_TXC 0x00000060 + IMX8DXL_ENET0_RGMII_TXD0_CONN_ENET0_RGMII_TXD0 0x00000060 + IMX8DXL_ENET0_RGMII_TXD1_CONN_ENET0_RGMII_TXD1 0x00000060 + IMX8DXL_ENET0_RGMII_TXD2_CONN_ENET0_RGMII_TXD2 0x00000060 + IMX8DXL_ENET0_RGMII_TXD3_CONN_ENET0_RGMII_TXD3 0x00000060 + IMX8DXL_ENET0_RGMII_TX_CTL_CONN_ENET0_RGMII_TX_CTL 0x00000060 + >; + }; + + pinctrl_lpspi3: lpspi3grp { + fsl,pins = < + IMX8DXL_SPI3_SCK_ADMA_SPI3_SCK 0x6000040 + IMX8DXL_SPI3_SDO_ADMA_SPI3_SDO 0x6000040 + IMX8DXL_SPI3_SDI_ADMA_SPI3_SDI 0x6000040 + IMX8DXL_SPI3_CS1_ADMA_SPI3_CS1 0x6000040 + >; + }; + + pinctrl_i2c2: i2c2grp { + fsl,pins = < + IMX8DXL_SPI1_SCK_ADMA_I2C2_SDA 0x06000021 + IMX8DXL_SPI1_SDO_ADMA_I2C2_SCL 0x06000021 + >; + }; + + pinctrl_cm40_lpuart: cm40lpuartgrp { + fsl,pins = < + IMX8DXL_ADC_IN2_M40_UART0_RX 0x06000020 + IMX8DXL_ADC_IN3_M40_UART0_TX 0x06000020 + >; + }; + + pinctrl_i2c3: i2c3grp { + fsl,pins = < + IMX8DXL_SPI1_CS0_ADMA_I2C3_SDA 0x06000021 + IMX8DXL_SPI1_SDI_ADMA_I2C3_SCL 0x06000021 + >; + }; + + pinctrl_lpuart0: lpuart0grp { + fsl,pins = < + IMX8DXL_UART0_RX_ADMA_UART0_RX 0x06000020 + IMX8DXL_UART0_TX_ADMA_UART0_TX 0x06000020 + >; + }; + + pinctrl_usdhc1: usdhc1grp { + fsl,pins = < + IMX8DXL_EMMC0_CLK_CONN_EMMC0_CLK 0x06000041 + IMX8DXL_EMMC0_CMD_CONN_EMMC0_CMD 0x00000021 + IMX8DXL_EMMC0_DATA0_CONN_EMMC0_DATA0 0x00000021 + IMX8DXL_EMMC0_DATA1_CONN_EMMC0_DATA1 0x00000021 + IMX8DXL_EMMC0_DATA2_CONN_EMMC0_DATA2 0x00000021 + IMX8DXL_EMMC0_DATA3_CONN_EMMC0_DATA3 0x00000021 + IMX8DXL_EMMC0_DATA4_CONN_EMMC0_DATA4 0x00000021 + IMX8DXL_EMMC0_DATA5_CONN_EMMC0_DATA5 0x00000021 + IMX8DXL_EMMC0_DATA6_CONN_EMMC0_DATA6 0x00000021 + IMX8DXL_EMMC0_DATA7_CONN_EMMC0_DATA7 0x00000021 + IMX8DXL_EMMC0_STROBE_CONN_EMMC0_STROBE 0x00000041 + >; + }; + + pinctrl_usdhc2_gpio: usdhc2gpiogrp { + fsl,pins = < + IMX8DXL_ENET0_RGMII_TX_CTL_LSIO_GPIO4_IO30 0x00000040 /* RESET_B */ + IMX8DXL_ENET0_RGMII_TXD1_LSIO_GPIO5_IO00 0x00000021 /* WP */ + IMX8DXL_ENET0_RGMII_TXD2_LSIO_GPIO5_IO01 0x00000021 /* CD */ + >; + }; + + pinctrl_usdhc2: usdhc2grp { + fsl,pins = < + IMX8DXL_ENET0_RGMII_RXC_CONN_USDHC1_CLK 0x06000041 + IMX8DXL_ENET0_RGMII_RX_CTL_CONN_USDHC1_CMD 0x00000021 + IMX8DXL_ENET0_RGMII_RXD0_CONN_USDHC1_DATA0 0x00000021 + IMX8DXL_ENET0_RGMII_RXD1_CONN_USDHC1_DATA1 0x00000021 + IMX8DXL_ENET0_RGMII_RXD2_CONN_USDHC1_DATA2 0x00000021 + IMX8DXL_ENET0_RGMII_RXD3_CONN_USDHC1_DATA3 0x00000021 + IMX8DXL_ENET0_RGMII_TXD0_CONN_USDHC1_VSELECT 0x00000021 + >; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..795d1d472faed079ef19807a100409160cf3e378 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2019~2020, 2022 NXP + */ + +&audio_ipg_clk { + clock-frequency = <160000000>; +}; + +&dma_ipg_clk { + clock-frequency = <160000000>; +}; + +&i2c0 { + compatible = "fsl,imx8dxl-lpi2c", "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c"; + interrupts = ; +}; + +&i2c1 { + compatible = "fsl,imx8dxl-lpi2c", "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c"; + interrupts = ; +}; + +&i2c2 { + compatible = "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c"; + interrupts = ; +}; + +&i2c3 { + compatible = "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c"; + interrupts = ; +}; + +&lpuart0 { + compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart"; + interrupts = ; +}; + +&lpuart1 { + compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart"; + interrupts = ; +}; + +&lpuart2 { + compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart"; + interrupts = ; +}; + +&lpuart3 { + compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart"; + interrupts = ; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..69c4849f21321e3ee2d5897de25e0d1cef06da44 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2019~2020, 2022 NXP + */ + +/delete-node/ &enet1_lpcg; +/delete-node/ &fec2; + +&conn_subsys { + conn_enet0_root_clk: clock-conn-enet0-root { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <250000000>; + clock-output-names = "conn_enet0_root_clk"; + }; + + eqos: ethernet@5b050000 { + compatible = "nxp,imx8dxl-dwmac-eqos", "snps,dwmac-5.10a"; + reg = <0x5b050000 0x10000>; + interrupt-parent = <&gic>; + interrupts = , + ; + interrupt-names = "eth_wake_irq", "macirq"; + clocks = <&eqos_lpcg IMX_LPCG_CLK_4>, + <&eqos_lpcg IMX_LPCG_CLK_6>, + <&eqos_lpcg IMX_LPCG_CLK_0>, + <&eqos_lpcg IMX_LPCG_CLK_5>, + <&eqos_lpcg IMX_LPCG_CLK_2>; + clock-names = "stmmaceth", "pclk", "ptp_ref", "tx", "mem"; + assigned-clocks = <&clk IMX_SC_R_ENET_1 IMX_SC_PM_CLK_PER>; + assigned-clock-rates = <125000000>; + power-domains = <&pd IMX_SC_R_ENET_1>; + status = "disabled"; + }; + + usbotg2: usb@5b0e0000 { + compatible = "fsl,imx8dxl-usb", "fsl,imx7ulp-usb"; + reg = <0x5b0e0000 0x200>; + interrupt-parent = <&gic>; + interrupts = ; + fsl,usbphy = <&usbphy2>; + fsl,usbmisc = <&usbmisc2 0>; + /* + * usbotg1 and usbotg2 share one clcok. + * scu firmware disables the access to the clock and keeps + * it always on in case other core (M4) uses one of these. + */ + clocks = <&clk_dummy>; + ahb-burst-config = <0x0>; + tx-burst-size-dword = <0x10>; + rx-burst-size-dword = <0x10>; + #stream-id-cells = <1>; + power-domains = <&pd IMX_SC_R_USB_1>; + status = "disabled"; + + clk_dummy: clock-dummy { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dummy"; + }; + }; + + usbmisc2: usbmisc@5b0e0200 { + #index-cells = <1>; + compatible = "fsl,imx7ulp-usbmisc"; + reg = <0x5b0e0200 0x200>; + }; + + usbphy2: usbphy@0x5b110000 { + compatible = "fsl,imx8dxl-usbphy", "fsl,imx7ulp-usbphy"; + reg = <0x5b110000 0x1000>; + clocks = <&usb2_2_lpcg IMX_LPCG_CLK_7>; + power-domains = <&pd IMX_SC_R_USB_1_PHY>; + status = "disabled"; + }; + + eqos_lpcg: clock-controller@5b240000 { + compatible = "fsl,imx8qxp-lpcg"; + reg = <0x5b240000 0x10000>; + #clock-cells = <1>; + clocks = <&conn_enet0_root_clk>, + <&conn_axi_clk>, + <&conn_axi_clk>, + <&clk IMX_SC_R_ENET_1 IMX_SC_PM_CLK_PER>, + <&conn_ipg_clk>; + clock-indices = , , + , , + ; + clock-output-names = "eqos_ptp", + "eqos_mem_clk", + "eqos_aclk", + "eqos_clk", + "eqos_csr_clk"; + power-domains = <&pd IMX_SC_R_ENET_1>; + }; + + usb2_2_lpcg: clock-controller@5b280000 { + compatible = "fsl,imx8qxp-lpcg"; + reg = <0x5b280000 0x10000>; + #clock-cells = <1>; + clock-indices = ; + clocks = <&conn_ipg_clk>; + clock-output-names = "usboh3_2_phy_ipg_clk"; + power-domains = <&pd IMX_SC_R_USB_1_PHY>; + }; + +}; + +&enet0_lpcg { + clocks = <&conn_enet0_root_clk>, + <&conn_enet0_root_clk>, + <&conn_axi_clk>, + <&clk IMX_SC_R_ENET_0 IMX_SC_C_TXCLK>, + <&conn_ipg_clk>, + <&conn_ipg_clk>; +}; + +&fec1 { + compatible = "fsl,imx8qm-fec"; + interrupts = , + , + , + ; + assigned-clocks = <&clk IMX_SC_R_ENET_0 IMX_SC_C_CLKDIV>; + assigned-clock-rates = <125000000>; +}; + +&usdhc1 { + compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc"; + interrupts = ; +}; + +&usdhc2 { + compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc"; + interrupts = ; +}; + +&usdhc3 { + compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc"; + interrupts = ; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..550f513708d8543b7be641cb61ba92c10b14e57b --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2020 NXP + */ + +&ddr_pmu0 { + compatible = "fsl,imx8-ddr-pmu"; + interrupts = ; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..815bd987b09b26d41131fecff40e0bd96df410b8 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2019~2020, 2022 NXP + */ + +&lsio_gpio0 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio1 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio2 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio3 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio4 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio5 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio6 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_gpio7 { + compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio"; + interrupts = ; +}; + +&lsio_mu0 { + compatible = "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; + +&lsio_mu1 { + compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; + +&lsio_mu2 { + compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; + +&lsio_mu3 { + compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; + +&lsio_mu4 { + compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; + +&lsio_mu5 { + compatible = "fsl,imx8qxp-mu", "fsl,imx6sx-mu"; + interrupts = ; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..5ddbda0b4def08fcd9d78f7d4f608319495629d6 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8dxl.dtsi @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2019~2020, 2022 NXP + */ + +#include +#include +#include +#include +#include +#include +#include + +/ { + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + ethernet0 = &fec1; + ethernet1 = &eqos; + gpio0 = &lsio_gpio0; + gpio1 = &lsio_gpio1; + gpio2 = &lsio_gpio2; + gpio3 = &lsio_gpio3; + gpio4 = &lsio_gpio4; + gpio5 = &lsio_gpio5; + gpio6 = &lsio_gpio6; + gpio7 = &lsio_gpio7; + mu1 = &lsio_mu1; + }; + + cpus: cpus { + #address-cells = <2>; + #size-cells = <0>; + + /* We have 1 clusters with 2 Cortex-A35 cores */ + A35_0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a35"; + reg = <0x0 0x0>; + enable-method = "psci"; + next-level-cache = <&A35_L2>; + clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; + #cooling-cells = <2>; + operating-points-v2 = <&a35_opp_table>; + }; + + A35_1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a35"; + reg = <0x0 0x1>; + enable-method = "psci"; + next-level-cache = <&A35_L2>; + clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>; + #cooling-cells = <2>; + operating-points-v2 = <&a35_opp_table>; + }; + + A35_L2: l2-cache0 { + compatible = "cache"; + }; + }; + + a35_opp_table: opp-table { + compatible = "operating-points-v2"; + opp-shared; + + opp-900000000 { + opp-hz = /bits/ 64 <900000000>; + opp-microvolt = <1000000>; + clock-latency-ns = <150000>; + }; + + opp-1200000000 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1100000>; + clock-latency-ns = <150000>; + opp-suspend; + }; + }; + + gic: interrupt-controller@51a00000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x51a00000 0 0x10000>, /* GIC Dist */ + <0x0 0x51b00000 0 0xc0000>; /* GICR (RD_base + SGI_base) */ + #interrupt-cells = <3>; + interrupt-controller; + interrupts = ; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + dsp_reserved: dsp@92400000 { + reg = <0 0x92400000 0 0x2000000>; + no-map; + }; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = ; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + system-controller { + compatible = "fsl,imx-scu"; + mbox-names = "tx0", + "rx0", + "gip3"; + mboxes = <&lsio_mu1 0 0 + &lsio_mu1 1 0 + &lsio_mu1 3 3>; + + pd: power-controller { + compatible = "fsl,scu-pd"; + #power-domain-cells = <1>; + wakeup-irq = <160 163 235 236 237 228 229 230 231 238 + 239 240 166 169>; + }; + + clk: clock-controller { + compatible = "fsl,imx8dxl-clk", "fsl,scu-clk"; + #clock-cells = <2>; + clocks = <&xtal32k &xtal24m>; + clock-names = "xtal_32KHz", "xtal_24Mhz"; + }; + + iomuxc: pinctrl { + compatible = "fsl,imx8dxl-iomuxc"; + }; + + ocotp: ocotp { + compatible = "fsl,imx8qxp-scu-ocotp"; + #address-cells = <1>; + #size-cells = <1>; + + fec_mac0: mac@2c4 { + reg = <0x2c4 6>; + }; + + fec_mac1: mac@2c6 { + reg = <0x2c6 6>; + }; + }; + + rtc: rtc { + compatible = "fsl,imx8qxp-sc-rtc"; + }; + + sc_pwrkey: keys { + compatible = "fsl,imx8qxp-sc-key", "fsl,imx-sc-key"; + linux,keycode = ; + wakeup-source; + }; + + watchdog { + compatible = "fsl,imx-sc-wdt"; + timeout-sec = <60>; + }; + + tsens: thermal-sensor { + compatible = "fsl,imx-sc-thermal"; + #thermal-sensor-cells = <1>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , /* Physical Secure */ + , /* Physical Non-Secure */ + , /* Virtual */ + ; /* Hypervisor */ + }; + + thermal_zones: thermal-zones { + cpu-thermal0 { + polling-delay-passive = <250>; + polling-delay = <2000>; + thermal-sensors = <&tsens IMX_SC_R_SYSTEM>; + + trips { + cpu_alert0: trip0 { + temperature = <107000>; + hysteresis = <2000>; + type = "passive"; + }; + cpu_crit0: trip1 { + temperature = <127000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_alert0>; + cooling-device = + <&A35_0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&A35_1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + }; + + /* The two values below cannot be changed by the board */ + xtal32k: clock-xtal32k { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "xtal_32KHz"; + }; + + xtal24m: clock-xtal24m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "xtal_24MHz"; + }; + + /* sorted in register address */ + #include "imx8-ss-adma.dtsi" + #include "imx8-ss-conn.dtsi" + #include "imx8-ss-ddr.dtsi" + #include "imx8-ss-lsio.dtsi" +}; + +#include "imx8dxl-ss-adma.dtsi" +#include "imx8dxl-ss-conn.dtsi" +#include "imx8dxl-ss-lsio.dtsi" +#include "imx8dxl-ss-ddr.dtsi" diff --git a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi index 9e6170d9394e41b81a9ea8b214ba63a3c408a0ce..def7bb5d37cf70aac76f56f45ad198458b8feaca 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi @@ -2,7 +2,7 @@ /* * Copyright (c) 2018 NXP * Copyright (c) 2019 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ / { diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts new file mode 100644 index 0000000000000000000000000000000000000000..8b16bd68576c0b51d67320ab4a015068834d881b --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright (C) 2022 Kontron Electronics GmbH + */ + +/dts-v1/; + +#include "imx8mm-kontron-osm-s.dtsi" + +/ { + model = "Kontron BL i.MX8MM OSM-S (N802X S)"; + compatible = "kontron,imx8mm-bl-osm-s", "kontron,imx8mm-osm-s", "fsl,imx8mm"; + + aliases { + ethernet1 = &usbnet; + }; + + /* fixed crystal dedicated to mcp2542fd */ + osc_can: clock-osc-can { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <40000000>; + clock-output-names = "osc-can"; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_led>; + + led1 { + label = "led1"; + gpios = <&gpio1 12 GPIO_ACTIVE_LOW>; + linux,default-trigger = "heartbeat"; + }; + + led2 { + label = "led2"; + gpios = <&gpio1 13 GPIO_ACTIVE_LOW>; + }; + + led3 { + label = "led3"; + gpios = <&gpio1 14 GPIO_ACTIVE_LOW>; + }; + }; + + pwm-beeper { + compatible = "pwm-beeper"; + pwms = <&pwm2 0 5000 0>; + }; + + reg_rst_eth2: regulator-rst-eth2 { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb_eth2>; + gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>; + enable-active-high; + regulator-always-on; + regulator-name = "rst-usb-eth2"; + }; + + reg_usb1_vbus: regulator-usb1-vbus { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_reg_usb1_vbus>; + gpio = <&gpio3 25 GPIO_ACTIVE_LOW>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-name = "usb1-vbus"; + }; + + reg_vdd_5v: regulator-5v { + compatible = "regulator-fixed"; + regulator-always-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-name = "vdd-5v"; + }; +}; + +&ecspi2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ecspi2>; + cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + can@0 { + compatible = "microchip,mcp251xfd"; + reg = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_can>; + clocks = <&osc_can>; + interrupts-extended = <&gpio4 28 IRQ_TYPE_LEVEL_LOW>; + /* + * Limit the SPI clock to 15 MHz to prevent issues + * with corrupted data due to chip errata. + */ + spi-max-frequency = <15000000>; + vdd-supply = <®_vdd_3v3>; + xceiver-supply = <®_vdd_5v>; + }; +}; + +&ecspi3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ecspi3>; + cs-gpios = <&gpio5 25 GPIO_ACTIVE_LOW>; + status = "okay"; + + eeram@0 { + compatible = "microchip,48l640"; + reg = <0>; + spi-max-frequency = <20000000>; + }; +}; + +&fec1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_enet>; + phy-connection-type = "rgmii-rxid"; + phy-handle = <ðphy>; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy: ethernet-phy@0 { + reg = <0>; + reset-assert-us = <1>; + reset-deassert-us = <15000>; + reset-gpios = <&gpio1 1 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&gpio1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio1>; + gpio-line-names = "", "", "", "dio1-out", "", "", "dio1-in", "dio2-out", + "dio2-in", "dio3-out", "dio3-in", "dio4-out", "", "", "", "", + "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""; +}; + +&gpio5 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio5>; + gpio-line-names = "", "", "dio4-in", "", "", "", "", "", + "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""; +}; + +&i2c4 { + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c4>; + status = "okay"; +}; + +&pwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm2>; + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1>; + uart-has-rtscts; + status = "okay"; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart2>; + linux,rs485-enabled-at-boot-time; + uart-has-rtscts; + status = "okay"; +}; + +&usbotg1 { + dr_mode = "otg"; + disable-over-current; + vbus-supply = <®_usb1_vbus>; + status = "okay"; +}; + +&usbotg2 { + dr_mode = "host"; + disable-over-current; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + usb1@1 { + compatible = "usb424,9514"; + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + usbnet: ethernet@1 { + compatible = "usb424,ec00"; + reg = <1>; + local-mac-address = [ 00 00 00 00 00 00 ]; + }; + }; +}; + +&usdhc2 { + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc2>; + pinctrl-1 = <&pinctrl_usdhc2_100mhz>; + pinctrl-2 = <&pinctrl_usdhc2_200mhz>; + vmmc-supply = <®_vdd_3v3>; + vqmmc-supply = <®_nvcc_sd>; + cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; + status = "okay"; +}; + +&iomuxc { + pinctrl_can: cangrp { + fsl,pins = < + MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28 0x19 + >; + }; + + pinctrl_ecspi2: ecspi2grp { + fsl,pins = < + MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO 0x82 + MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI 0x82 + MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK 0x82 + MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13 0x19 + >; + }; + + pinctrl_ecspi3: ecspi3grp { + fsl,pins = < + MX8MM_IOMUXC_UART2_RXD_ECSPI3_MISO 0x82 + MX8MM_IOMUXC_UART1_TXD_ECSPI3_MOSI 0x82 + MX8MM_IOMUXC_UART1_RXD_ECSPI3_SCLK 0x82 + MX8MM_IOMUXC_UART2_TXD_GPIO5_IO25 0x19 + >; + }; + + pinctrl_enet: enetgrp { + fsl,pins = < + MX8MM_IOMUXC_ENET_MDC_ENET1_MDC 0x3 + MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO 0x3 + MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3 0x1f + MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2 0x1f + MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1 0x1f + MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0 0x1f + MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3 0x91 + MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2 0x91 + MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1 0x91 + MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0 0x91 + MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC 0x1f + MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC 0x91 + MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL 0x91 + MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL 0x1f + MX8MM_IOMUXC_GPIO1_IO01_GPIO1_IO1 0x19 /* PHY RST */ + MX8MM_IOMUXC_GPIO1_IO05_GPIO1_IO5 0x19 /* ETH IRQ */ + >; + }; + + pinctrl_gpio_led: gpioledgrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12 0x19 + MX8MM_IOMUXC_GPIO1_IO13_GPIO1_IO13 0x19 + MX8MM_IOMUXC_GPIO1_IO14_GPIO1_IO14 0x19 + >; + }; + + pinctrl_gpio1: gpio1grp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3 0x19 + MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7 0x19 + MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9 0x19 + MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11 0x19 + MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6 0x19 + MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8 0x19 + MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10 0x19 + >; + }; + + pinctrl_gpio5: gpio5grp { + fsl,pins = < + MX8MM_IOMUXC_SAI3_MCLK_GPIO5_IO2 0x19 + >; + }; + + pinctrl_i2c4: i2c4grp { + fsl,pins = < + MX8MM_IOMUXC_I2C4_SCL_I2C4_SCL 0x400001c3 + MX8MM_IOMUXC_I2C4_SDA_I2C4_SDA 0x400001c3 + >; + }; + + pinctrl_pwm2: pwm2grp { + fsl,pins = < + MX8MM_IOMUXC_SPDIF_RX_PWM2_OUT 0x19 + >; + }; + + pinctrl_reg_usb1_vbus: regusb1vbusgrp { + fsl,pins = < + MX8MM_IOMUXC_SAI5_MCLK_GPIO3_IO25 0x19 + >; + }; + + pinctrl_uart1: uart1grp { + fsl,pins = < + MX8MM_IOMUXC_SAI2_RXC_UART1_DCE_RX 0x140 + MX8MM_IOMUXC_SAI2_RXFS_UART1_DCE_TX 0x140 + MX8MM_IOMUXC_SAI2_RXD0_UART1_DCE_RTS_B 0x140 + MX8MM_IOMUXC_SAI2_TXFS_UART1_DCE_CTS_B 0x140 + >; + }; + + pinctrl_uart2: uart2grp { + fsl,pins = < + MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX 0x140 + MX8MM_IOMUXC_SAI3_TXC_UART2_DCE_TX 0x140 + MX8MM_IOMUXC_SAI3_RXD_UART2_DCE_RTS_B 0x140 + MX8MM_IOMUXC_SAI3_RXC_UART2_DCE_CTS_B 0x140 + >; + }; + + pinctrl_usb_eth2: usbeth2grp { + fsl,pins = < + MX8MM_IOMUXC_NAND_CE1_B_GPIO3_IO2 0x19 + >; + }; + + pinctrl_usdhc2: usdhc2grp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x190 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d0 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d0 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d0 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d0 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 + >; + }; + + pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x194 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d4 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d4 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d4 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 + >; + }; + + pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x196 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d6 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d6 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d6 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 + >; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts similarity index 96% rename from arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts rename to arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts index 23be1ec538ba61b2057cc3f41f2e7796ae8cbcde..a079322a37931ec7726cd6bac4c5eb0e4c7bc64c 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts @@ -5,11 +5,11 @@ /dts-v1/; -#include "imx8mm-kontron-n801x-som.dtsi" +#include "imx8mm-kontron-sl.dtsi" / { - model = "Kontron i.MX8MM N801X S"; - compatible = "kontron,imx8mm-n801x-s", "kontron,imx8mm-n801x-som", "fsl,imx8mm"; + model = "Kontron BL i.MX8MM (N801X S)"; + compatible = "kontron,imx8mm-bl", "kontron,imx8mm-sl", "fsl,imx8mm"; aliases { ethernet1 = &usbnet; @@ -321,6 +321,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d0 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 >; }; @@ -333,6 +334,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 >; }; @@ -345,6 +347,7 @@ MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x019 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 >; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..8d10f5b412978b017f42e63d5746b247f46667d5 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright (C) 2022 Kontron Electronics GmbH + */ + +#include +#include "imx8mm.dtsi" + +/ { + model = "Kontron OSM-S i.MX8MM (N802X SOM)"; + compatible = "kontron,imx8mm-osm-s", "fsl,imx8mm"; + + memory@40000000 { + device_type = "memory"; + /* + * There are multiple SoM flavors with different DDR sizes. + * The smallest is 1GB. For larger sizes the bootloader will + * update the reg property. + */ + reg = <0x0 0x40000000 0 0x80000000>; + }; + + chosen { + stdout-path = &uart3; + }; +}; + +&A53_0 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_1 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_2 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_3 { + cpu-supply = <®_vdd_arm>; +}; + +&ddrc { + operating-points-v2 = <&ddrc_opp_table>; + + ddrc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100M { + opp-hz = /bits/ 64 <100000000>; + }; + + opp-750M { + opp-hz = /bits/ 64 <750000000>; + }; + }; +}; + +&ecspi1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ecspi1>; + cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>; + status = "okay"; + + flash@0 { + compatible = "mxicy,mx25r1635f", "jedec,spi-nor"; + spi-max-frequency = <80000000>; + reg = <0>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "u-boot"; + reg = <0x0 0x1e0000>; + }; + + partition@1e0000 { + label = "env"; + reg = <0x1e0000 0x10000>; + }; + + partition@1f0000 { + label = "env_redundant"; + reg = <0x1f0000 0x10000>; + }; + }; + }; +}; + +&i2c1 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c1>; + status = "okay"; + + pca9450: pmic@25 { + compatible = "nxp,pca9450a"; + reg = <0x25>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pmic>; + interrupt-parent = <&gpio1>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; + + regulators { + reg_vdd_soc: BUCK1 { + regulator-name = "+0V8_VDD_SOC (BUCK1)"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <850000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <3125>; + nxp,dvs-run-voltage = <850000>; + nxp,dvs-standby-voltage = <800000>; + }; + + reg_vdd_arm: BUCK2 { + regulator-name = "+0V9_VDD_ARM (BUCK2)"; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <950000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <3125>; + nxp,dvs-run-voltage = <950000>; + nxp,dvs-standby-voltage = <850000>; + }; + + reg_vdd_dram: BUCK3 { + regulator-name = "+0V9_VDD_DRAM&PU (BUCK3)"; + regulator-min-microvolt = <850000>; + regulator-max-microvolt = <950000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vdd_3v3: BUCK4 { + regulator-name = "+3V3 (BUCK4)"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vdd_1v8: BUCK5 { + regulator-name = "+1V8 (BUCK5)"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_nvcc_dram: BUCK6 { + regulator-name = "+1V1_NVCC_DRAM (BUCK6)"; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_nvcc_snvs: LDO1 { + regulator-name = "+1V8_NVCC_SNVS (LDO1)"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vdd_snvs: LDO2 { + regulator-name = "+0V8_VDD_SNVS (LDO2)"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <900000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vdda: LDO3 { + regulator-name = "+1V8_VDDA (LDO3)"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_vdd_phy: LDO4 { + regulator-name = "+0V9_VDD_PHY (LDO4)"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_nvcc_sd: LDO5 { + regulator-name = "NVCC_SD (LDO5)"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + }; + }; + + rtc@52 { + compatible = "microcrystal,rv3028"; + reg = <0x52>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_rtc>; + interrupts-extended = <&gpio4 1 IRQ_TYPE_LEVEL_HIGH>; + trickle-diode-disable; + }; +}; + +&uart3 { /* console */ + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3>; + status = "okay"; +}; + +&usdhc1 { + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc1>; + pinctrl-1 = <&pinctrl_usdhc1_100mhz>; + pinctrl-2 = <&pinctrl_usdhc1_200mhz>; + vmmc-supply = <®_vdd_3v3>; + vqmmc-supply = <®_vdd_1v8>; + bus-width = <8>; + non-removable; + status = "okay"; +}; + +&wdog1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_wdog>; + fsl,ext-reset-output; + status = "okay"; +}; + +&iomuxc { + pinctrl_ecspi1: ecspi1grp { + fsl,pins = < + MX8MM_IOMUXC_ECSPI1_MISO_ECSPI1_MISO 0x82 + MX8MM_IOMUXC_ECSPI1_MOSI_ECSPI1_MOSI 0x82 + MX8MM_IOMUXC_ECSPI1_SCLK_ECSPI1_SCLK 0x82 + MX8MM_IOMUXC_ECSPI1_SS0_GPIO5_IO9 0x19 + >; + }; + + pinctrl_i2c1: i2c1grp { + fsl,pins = < + MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL 0x400001c3 + MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA 0x400001c3 + >; + }; + + pinctrl_pmic: pmicgrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0 0x141 + >; + }; + + pinctrl_rtc: rtcgrp { + fsl,pins = < + MX8MM_IOMUXC_SAI1_RXC_GPIO4_IO1 0x19 + >; + }; + + pinctrl_uart3: uart3grp { + fsl,pins = < + MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX 0x140 + MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX 0x140 + >; + }; + + pinctrl_usdhc1: usdhc1grp { + fsl,pins = < + MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK 0x190 + MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD 0x1d0 + MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0 0x1d0 + MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1 0x1d0 + MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2 0x1d0 + MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3 0x1d0 + MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4 0x1d0 + MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5 0x1d0 + MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6 0x1d0 + MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7 0x1d0 + MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B 0x019 + MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE 0x190 + >; + }; + + pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK 0x194 + MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD 0x1d4 + MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0 0x1d4 + MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1 0x1d4 + MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2 0x1d4 + MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3 0x1d4 + MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4 0x1d4 + MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5 0x1d4 + MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6 0x1d4 + MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7 0x1d4 + MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B 0x019 + MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE 0x194 + >; + }; + + pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK 0x196 + MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD 0x1d6 + MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0 0x1d6 + MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1 0x1d6 + MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2 0x1d6 + MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3 0x1d6 + MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4 0x1d6 + MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5 0x1d6 + MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6 0x1d6 + MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7 0x1d6 + MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B 0x019 + MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE 0x196 + >; + }; + + pinctrl_wdog: wdoggrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B 0xc6 + >; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-sl.dtsi similarity index 87% rename from arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi rename to arch/arm64/boot/dts/freescale/imx8mm-kontron-sl.dtsi index 8f90eb02550d8e75a1fd0c1000e67238b26f4908..0679728d2489919ab4ffd30bb1c8d807d3d63003 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-sl.dtsi @@ -6,8 +6,8 @@ #include "imx8mm.dtsi" / { - model = "Kontron i.MX8MM N801X SoM"; - compatible = "kontron,imx8mm-n801x-som", "fsl,imx8mm"; + model = "Kontron SL i.MX8MM (N801X SOM)"; + compatible = "kontron,imx8mm-sl", "fsl,imx8mm"; memory@40000000 { device_type = "memory"; @@ -46,10 +46,6 @@ ddrc_opp_table: opp-table { compatible = "operating-points-v2"; - opp-25M { - opp-hz = /bits/ 64 <25000000>; - }; - opp-100M { opp-hz = /bits/ 64 <100000000>; }; @@ -70,6 +66,27 @@ compatible = "mxicy,mx25r1635f", "jedec,spi-nor"; spi-max-frequency = <80000000>; reg = <0>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "u-boot"; + reg = <0x0 0x1e0000>; + }; + + partition@1e0000 { + label = "env"; + reg = <0x1e0000 0x10000>; + }; + + partition@1f0000 { + label = "env_redundant"; + reg = <0x1f0000 0x10000>; + }; + }; }; }; @@ -86,11 +103,10 @@ pinctrl-0 = <&pinctrl_pmic>; interrupt-parent = <&gpio1>; interrupts = <0 IRQ_TYPE_LEVEL_LOW>; - sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>; regulators { reg_vdd_soc: BUCK1 { - regulator-name = "buck1"; + regulator-name = "+0V8_VDD_SOC (BUCK1)"; regulator-min-microvolt = <800000>; regulator-max-microvolt = <850000>; regulator-boot-on; @@ -101,7 +117,7 @@ }; reg_vdd_arm: BUCK2 { - regulator-name = "buck2"; + regulator-name = "+0V9_VDD_ARM (BUCK2)"; regulator-min-microvolt = <850000>; regulator-max-microvolt = <950000>; regulator-boot-on; @@ -112,7 +128,7 @@ }; reg_vdd_dram: BUCK3 { - regulator-name = "buck3"; + regulator-name = "+0V9_VDD_DRAM&PU (BUCK3)"; regulator-min-microvolt = <850000>; regulator-max-microvolt = <950000>; regulator-boot-on; @@ -120,7 +136,7 @@ }; reg_vdd_3v3: BUCK4 { - regulator-name = "buck4"; + regulator-name = "+3V3 (BUCK4)"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; regulator-boot-on; @@ -128,7 +144,7 @@ }; reg_vdd_1v8: BUCK5 { - regulator-name = "buck5"; + regulator-name = "+1V8 (BUCK5)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-boot-on; @@ -136,7 +152,7 @@ }; reg_nvcc_dram: BUCK6 { - regulator-name = "buck6"; + regulator-name = "+1V1_NVCC_DRAM (BUCK6)"; regulator-min-microvolt = <1100000>; regulator-max-microvolt = <1100000>; regulator-boot-on; @@ -144,7 +160,7 @@ }; reg_nvcc_snvs: LDO1 { - regulator-name = "ldo1"; + regulator-name = "+1V8_NVCC_SNVS (LDO1)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-boot-on; @@ -152,7 +168,7 @@ }; reg_vdd_snvs: LDO2 { - regulator-name = "ldo2"; + regulator-name = "+0V8_VDD_SNVS (LDO2)"; regulator-min-microvolt = <800000>; regulator-max-microvolt = <900000>; regulator-boot-on; @@ -160,7 +176,7 @@ }; reg_vdda: LDO3 { - regulator-name = "ldo3"; + regulator-name = "+1V8_VDDA (LDO3)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-boot-on; @@ -168,7 +184,7 @@ }; reg_vdd_phy: LDO4 { - regulator-name = "ldo4"; + regulator-name = "+0V9_VDD_PHY (LDO4)"; regulator-min-microvolt = <900000>; regulator-max-microvolt = <900000>; regulator-boot-on; @@ -176,7 +192,7 @@ }; reg_nvcc_sd: LDO5 { - regulator-name = "ldo5"; + regulator-name = "NVCC_SD (LDO5)"; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; }; @@ -229,7 +245,6 @@ pinctrl_pmic: pmicgrp { fsl,pins = < MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0 0x141 - MX8MM_IOMUXC_GPIO1_IO04_GPIO1_IO4 0x141 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts b/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts index c97f4e06ae5f304831a2f2e40af4253521f7ee87..32f6f2f50c10ca29c4c80bac23614df665657337 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts @@ -152,11 +152,11 @@ * CPLD_reset is RESET_SOFT in schematic */ gpio-line-names = - "CPLD_D[1]", "CPLD_int", "CPLD_reset", "", - "", "CPLD_D[0]", "", "", - "", "", "", "CPLD_D[2]", - "CPLD_D[3]", "CPLD_D[4]", "CPLD_D[5]", "CPLD_D[6]", - "CPLD_D[7]", "", "", "", + "CPLD_D[6]", "CPLD_int", "CPLD_reset", "", + "", "CPLD_D[7]", "", "", + "", "", "", "CPLD_D[5]", + "CPLD_D[4]", "CPLD_D[3]", "CPLD_D[2]", "CPLD_D[1]", + "CPLD_D[0]", "", "", "", "", "", "", "", "", "", "", "KBD_intK", "", "", "", ""; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts index 286d2df01cfa72e38fdacd7777d7b9516efebd66..7e0aeb2db30549b7c9469b7b1d3d1ee908e7e2a1 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts @@ -5,7 +5,6 @@ /dts-v1/; -#include #include "imx8mm-tqma8mqml.dtsi" #include "mba8mx.dtsi" diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi index 16ee9b5179e6e356ad6ad022d6b098b8c8947179..f649dfacb4b696caef22fc9e17a2d55cc20fb8e0 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi @@ -3,6 +3,7 @@ * Copyright 2020-2021 TQ-Systems GmbH */ +#include #include "imx8mm.dtsi" / { diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts index b1e7540f0281a6178c2b243924cd99627cde620c..641be3af989ddb4df09aee87ee4fe3dbbadb2a64 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts @@ -17,4 +17,3 @@ stdout-path = &uart2; }; }; - diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts index 35fb929e7bccecef9d739844fd616054d55743a0..d3ee6fc4baabd7bdba3e5936dc6fdaecc5707bac 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7901.dts @@ -626,24 +626,28 @@ lan1: port@0 { reg = <0>; label = "lan1"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan2: port@1 { reg = <1>; label = "lan2"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan3: port@2 { reg = <2>; label = "lan3"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan4: port@3 { reg = <3>; label = "lan4"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts index 6dc5eda2d2561af30ff169bfa4ba20d289bb617e..31f4c735fe4f011b081e72a6809cbe214f6e5205 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts @@ -222,7 +222,6 @@ compatible = "microchip,mcp2515"; reg = <0>; clocks = <&can20m>; - oscillator-frequency = <20000000>; interrupt-parent = <&gpio2>; interrupts = <3 IRQ_TYPE_LEVEL_LOW>; spi-max-frequency = <10000000>; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts index a65761a53f238fecba646f9f585cb3cb9b8370aa..19f6d2943d26ccc587c7ed0f50335146b4850b6c 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts @@ -250,7 +250,7 @@ }; &gpio2 { - gpio-line-names = "dig2_in", "dig2_out#", "", "", "", "", "", "", + gpio-line-names = "dig2_in", "dig2_out#", "dig2_ctl", "", "", "", "dig1_ctl", "", "dig1_out#", "dig1_in", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""; @@ -630,6 +630,8 @@ MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12 0x40000041 /* RS232# */ MX8MM_IOMUXC_SD1_DATA7_GPIO2_IO9 0x40000041 /* DIG1_IN */ MX8MM_IOMUXC_SD1_DATA6_GPIO2_IO8 0x40000041 /* DIG1_OUT */ + MX8MM_IOMUXC_SD1_DATA4_GPIO2_IO6 0x40000041 /* DIG1_CTL */ + MX8MM_IOMUXC_SD1_DATA0_GPIO2_IO2 0x40000041 /* DIG2_CTL */ MX8MM_IOMUXC_SD1_CLK_GPIO2_IO0 0x40000041 /* DIG2_IN */ MX8MM_IOMUXC_SD1_CMD_GPIO2_IO1 0x40000041 /* DIG2_OUT */ MX8MM_IOMUXC_ECSPI1_MOSI_GPIO5_IO7 0x40000041 /* SIM1DET# */ diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts new file mode 100644 index 0000000000000000000000000000000000000000..a67771d0214640b4203689b20a1ff09b9b6f49aa --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts @@ -0,0 +1,888 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright 2022 Gateworks Corporation + */ + +/dts-v1/; + +#include +#include +#include +#include + +#include "imx8mm.dtsi" + +/ { + model = "Gateworks Venice GW7904 i.MX8MM board"; + compatible = "gateworks,imx8mm-gw7904", "fsl,imx8mm"; + + chosen { + stdout-path = &uart2; + }; + + memory@40000000 { + device_type = "memory"; + reg = <0x0 0x40000000 0 0x80000000>; + }; + + gpio-keys { + compatible = "gpio-keys"; + + key-0 { + label = "user_pb"; + gpios = <&gpio 2 GPIO_ACTIVE_LOW>; + linux,code = ; + }; + + key-1 { + label = "user_pb1x"; + linux,code = ; + interrupt-parent = <&gsc>; + interrupts = <0>; + }; + + key-2 { + label = "key_erased"; + linux,code = ; + interrupt-parent = <&gsc>; + interrupts = <1>; + }; + + key-3 { + label = "eeprom_wp"; + linux,code = ; + interrupt-parent = <&gsc>; + interrupts = <2>; + }; + + key-4 { + label = "switch_hold"; + linux,code = ; + interrupt-parent = <&gsc>; + interrupts = <7>; + }; + }; + + led-controller { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_gpio_leds>; + + led-0 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led01_grn"; + gpios = <&gpioled 0 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-1 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led01_yel"; + gpios = <&gpioled 1 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-2 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led02_grn"; + gpios = <&gpioled 2 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-3 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led02_yel"; + gpios = <&gpioled 3 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-4 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led03_grn"; + gpios = <&gpioled 4 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-5 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led03_yel"; + gpios = <&gpioled 5 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-6 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led04_grn"; + gpios = <&gpioled 6 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-7 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led04_yel"; + gpios = <&gpioled 7 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-8 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led05_grn"; + gpios = <&gpioled 8 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-9 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led05_yel"; + gpios = <&gpioled 9 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-10 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led06_grn"; + gpios = <&gpio1 8 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-11 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led06_red"; + gpios = <&gpio1 9 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-12 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led07_grn"; + gpios = <&gpio1 10 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-13 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led07_red"; + gpios = <&gpio1 11 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-14 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led08_grn"; + gpios = <&gpioled 10 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-15 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led08_yel"; + gpios = <&gpioled 11 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-16 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led09_grn"; + gpios = <&gpioled 12 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-17 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led09_yel"; + gpios = <&gpioled 13 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-18 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led10_grn"; + gpios = <&gpioled 14 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + + led-19 { + function = LED_FUNCTION_STATUS; + color = ; + label = "led10_yel"; + gpios = <&gpioled 15 GPIO_ACTIVE_LOW>; + default-state = "off"; + }; + }; + + pcie0_refclk: pcie0-refclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + }; + + reg_3p3v: regulator-3p3v { + compatible = "regulator-fixed"; + regulator-name = "3P3V"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; +}; + +&A53_0 { + cpu-supply = <&buck2>; +}; + +&A53_1 { + cpu-supply = <&buck2>; +}; + +&A53_2 { + cpu-supply = <&buck2>; +}; + +&A53_3 { + cpu-supply = <&buck2>; +}; + +&ddrc { + operating-points-v2 = <&ddrc_opp_table>; + + ddrc_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-25M { + opp-hz = /bits/ 64 <25000000>; + }; + + opp-100M { + opp-hz = /bits/ 64 <100000000>; + }; + + opp-750M { + opp-hz = /bits/ 64 <750000000>; + }; + }; +}; + +&fec1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_fec1>; + phy-mode = "rgmii-id"; + phy-handle = <ðphy0>; + local-mac-address = [00 00 00 00 00 00]; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <0>; + }; + }; +}; + +&gpio1 { + gpio-line-names = "", "", "", "", "", "", "", "", + "", "", "", "", "rs232_en#", "", "", "", + "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""; +}; + +&gpio5 { + gpio-line-names = "", "", "", "", "", "", "", "", + "", "", "", "", "pci_wdis#", "", "", "", + "", "", "", "", "", "", "", "", + "", "", "", "", "", "", "", ""; +}; + +&i2c1 { + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c1>; + status = "okay"; + + gsc: gsc@20 { + compatible = "gw,gsc"; + reg = <0x20>; + pinctrl-0 = <&pinctrl_gsc>; + interrupt-parent = <&gpio4>; + interrupts = <26 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <1>; + + adc { + compatible = "gw,gsc-adc"; + #address-cells = <1>; + #size-cells = <0>; + + channel@6 { + gw,mode = <0>; + reg = <0x06>; + label = "temp"; + }; + + channel@82 { + gw,mode = <2>; + reg = <0x82>; + label = "vin"; + gw,voltage-divider-ohms = <22100 1000>; + gw,voltage-offset-microvolt = <700000>; + }; + + channel@84 { + gw,mode = <2>; + reg = <0x84>; + label = "vdd_5p0"; + gw,voltage-divider-ohms = <10000 10000>; + }; + + channel@86 { + gw,mode = <2>; + reg = <0x86>; + label = "vdd_3p3"; + gw,voltage-divider-ohms = <10000 10000>; + }; + + channel@88 { + gw,mode = <2>; + reg = <0x88>; + label = "vdd_0p9"; + }; + + channel@8c { + gw,mode = <2>; + reg = <0x8c>; + label = "vdd_soc"; + }; + + channel@8e { + gw,mode = <2>; + reg = <0x8e>; + label = "vdd_arm"; + }; + + channel@90 { + gw,mode = <2>; + reg = <0x90>; + label = "vdd_1p8"; + }; + + channel@92 { + gw,mode = <2>; + reg = <0x92>; + label = "vdd_dram"; + }; + + channel@a2 { + gw,mode = <2>; + reg = <0xa2>; + label = "vdd_gsc"; + gw,voltage-divider-ohms = <10000 10000>; + }; + }; + }; + + gpio: gpio@23 { + compatible = "nxp,pca9555"; + reg = <0x23>; + gpio-controller; + #gpio-cells = <2>; + interrupt-parent = <&gsc>; + interrupts = <4>; + }; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + pagesize = <16>; + }; + + eeprom@51 { + compatible = "atmel,24c02"; + reg = <0x51>; + pagesize = <16>; + }; + + eeprom@52 { + compatible = "atmel,24c02"; + reg = <0x52>; + pagesize = <16>; + }; + + eeprom@53 { + compatible = "atmel,24c02"; + reg = <0x53>; + pagesize = <16>; + }; + + rtc@68 { + compatible = "dallas,ds1672"; + reg = <0x68>; + }; +}; + +&i2c2 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c2>; + status = "okay"; + + pmic@4b { + compatible = "rohm,bd71847"; + reg = <0x4b>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pmic>; + interrupt-parent = <&gpio3>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + rohm,reset-snvs-powered; + #clock-cells = <0>; + clocks = <&osc_32k 0>; + clock-output-names = "clk-32k-out"; + + regulators { + /* vdd_soc: 0.805-0.900V (typ=0.8V) */ + BUCK1 { + regulator-name = "buck1"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + }; + + /* vdd_arm: 0.805-1.0V (typ=0.9V) */ + buck2: BUCK2 { + regulator-name = "buck2"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1300000>; + regulator-boot-on; + regulator-always-on; + regulator-ramp-delay = <1250>; + rohm,dvs-run-voltage = <1000000>; + rohm,dvs-idle-voltage = <900000>; + }; + + /* vdd_0p9: 0.805-1.0V (typ=0.9V) */ + BUCK3 { + regulator-name = "buck3"; + regulator-min-microvolt = <700000>; + regulator-max-microvolt = <1350000>; + regulator-boot-on; + regulator-always-on; + }; + + /* vdd_3p3 */ + BUCK4 { + regulator-name = "buck4"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + /* vdd_1p8 */ + BUCK5 { + regulator-name = "buck5"; + regulator-min-microvolt = <1605000>; + regulator-max-microvolt = <1995000>; + regulator-boot-on; + regulator-always-on; + }; + + /* vdd_dram */ + BUCK6 { + regulator-name = "buck6"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1400000>; + regulator-boot-on; + regulator-always-on; + }; + + /* nvcc_snvs_1p8 */ + LDO1 { + regulator-name = "ldo1"; + regulator-min-microvolt = <1600000>; + regulator-max-microvolt = <1900000>; + regulator-boot-on; + regulator-always-on; + }; + + /* vdd_snvs_0p8 */ + LDO2 { + regulator-name = "ldo2"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <900000>; + regulator-boot-on; + regulator-always-on; + }; + + /* vdda_1p8 */ + LDO3 { + regulator-name = "ldo3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + LDO4 { + regulator-name = "ldo4"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + LDO6 { + regulator-name = "ldo6"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + }; + }; +}; + +&i2c3 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c3>; + status = "okay"; + + accelerometer@19 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_accel>; + compatible = "st,lis2de12"; + reg = <0x19>; + st,drdy-int-pin = <1>; + interrupt-parent = <&gpio1>; + interrupts = <15 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "INT1"; + }; +}; + +&i2c4 { + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c4>; + status = "okay"; + + gpioled: gpio@27 { + compatible = "nxp,pca9555"; + reg = <0x27>; + gpio-controller; + #gpio-cells = <2>; + }; +}; + +&pcie_phy { + fsl,refclk-pad-mode = ; + fsl,clkreq-unsupported; + clocks = <&pcie0_refclk>; + clock-names = "ref"; + status = "okay"; +}; + +&pcie0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pcie0>; + reset-gpio = <&gpio5 11 GPIO_ACTIVE_LOW>; + clocks = <&clk IMX8MM_CLK_PCIE1_ROOT>, <&clk IMX8MM_CLK_PCIE1_AUX>, + <&pcie0_refclk>; + clock-names = "pcie", "pcie_aux", "pcie_bus"; + assigned-clocks = <&clk IMX8MM_CLK_PCIE1_AUX>, + <&clk IMX8MM_CLK_PCIE1_CTRL>; + assigned-clock-rates = <10000000>, <250000000>; + assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_50M>, + <&clk IMX8MM_SYS_PLL2_250M>; + status = "okay"; +}; + +&pgc_mipi { + status = "disabled"; +}; + +/* off-board RS232 */ +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1>; + status = "okay"; +}; + +/* console */ +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart2>; + status = "okay"; +}; + +/* off-board RS232 */ +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3>; + status = "okay"; +}; + +&usbotg1 { + dr_mode = "host"; + disable-over-current; + status = "okay"; +}; + +/* microSD */ +&usdhc2 { + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>; + pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>; + pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>; + cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; + bus-width = <4>; + vmmc-supply = <®_3p3v>; + status = "okay"; +}; + +/* eMMC */ +&usdhc3 { + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc3>; + pinctrl-1 = <&pinctrl_usdhc3_100mhz>; + pinctrl-2 = <&pinctrl_usdhc3_200mhz>; + bus-width = <8>; + non-removable; + status = "okay"; +}; + +&wdog1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_wdog>; + fsl,ext-reset-output; + status = "okay"; +}; + +&iomuxc { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_hog>; + + pinctrl_hog: hoggrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12 0x40000041 /* RS232# */ + MX8MM_IOMUXC_ECSPI2_MISO_GPIO5_IO12 0x40000041 /* PCI_WDIS# */ + >; + }; + + pinctrl_accel: accelgrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO15_GPIO1_IO15 0x159 + >; + }; + + pinctrl_fec1: fec1grp { + fsl,pins = < + MX8MM_IOMUXC_ENET_MDC_ENET1_MDC 0x3 + MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO 0x3 + MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3 0x1f + MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2 0x1f + MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1 0x1f + MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0 0x1f + MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3 0x91 + MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2 0x91 + MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1 0x91 + MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0 0x91 + MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC 0x1f + MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC 0x91 + MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL 0x91 + MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL 0x1f + MX8MM_IOMUXC_SAI2_TXFS_GPIO4_IO24 0x19 /* IRQ# */ + MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25 0x19 /* RST# */ + >; + }; + + pinctrl_gpio_leds: gpioledsgrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8 0x40000019 + MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9 0x40000019 + MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10 0x40000019 + MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11 0x40000019 + >; + }; + + pinctrl_gsc: gscgrp { + fsl,pins = < + MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26 0x159 + >; + }; + + pinctrl_i2c1: i2c1grp { + fsl,pins = < + MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL 0x400001c3 + MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA 0x400001c3 + >; + }; + + pinctrl_i2c2: i2c2grp { + fsl,pins = < + MX8MM_IOMUXC_I2C2_SCL_I2C2_SCL 0x400001c3 + MX8MM_IOMUXC_I2C2_SDA_I2C2_SDA 0x400001c3 + >; + }; + + pinctrl_i2c3: i2c3grp { + fsl,pins = < + MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL 0x400001c3 + MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA 0x400001c3 + >; + }; + + pinctrl_i2c4: i2c4grp { + fsl,pins = < + MX8MM_IOMUXC_I2C4_SCL_I2C4_SCL 0x400001c3 + MX8MM_IOMUXC_I2C4_SDA_I2C4_SDA 0x400001c3 + >; + }; + + pinctrl_pcie0: pciegrp { + fsl,pins = < + MX8MM_IOMUXC_ECSPI2_MOSI_GPIO5_IO11 0x41 + >; + }; + + pinctrl_pmic: pmicgrp { + fsl,pins = < + MX8MM_IOMUXC_NAND_DATA02_GPIO3_IO8 0x41 + >; + }; + + pinctrl_uart1: uart1grp { + fsl,pins = < + MX8MM_IOMUXC_UART1_RXD_UART1_DCE_RX 0x140 + MX8MM_IOMUXC_UART1_TXD_UART1_DCE_TX 0x140 + >; + }; + + pinctrl_uart2: uart2grp { + fsl,pins = < + MX8MM_IOMUXC_UART2_RXD_UART2_DCE_RX 0x140 + MX8MM_IOMUXC_UART2_TXD_UART2_DCE_TX 0x140 + >; + }; + + pinctrl_uart3: uart3grp { + fsl,pins = < + MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX 0x140 + MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX 0x140 + >; + }; + + pinctrl_usdhc2: usdhc2grp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x190 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d0 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d0 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d0 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d0 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d0 + >; + }; + + pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x194 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d4 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d4 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d4 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d4 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d4 + >; + }; + + pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK 0x196 + MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD 0x1d6 + MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0 0x1d6 + MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1 0x1d6 + MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2 0x1d6 + MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3 0x1d6 + >; + }; + + pinctrl_usdhc2_gpio: usdhc2-gpiogrp { + fsl,pins = < + MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12 0x1c4 + MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT 0x1d0 + >; + }; + + pinctrl_usdhc3: usdhc3grp { + fsl,pins = < + MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK 0x190 + MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD 0x1d0 + MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0 0x1d0 + MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1 0x1d0 + MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2 0x1d0 + MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3 0x1d0 + MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4 0x1d0 + MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5 0x1d0 + MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6 0x1d0 + MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7 0x1d0 + MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE 0x190 + >; + }; + + pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK 0x194 + MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD 0x1d4 + MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0 0x1d4 + MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1 0x1d4 + MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2 0x1d4 + MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3 0x1d4 + MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4 0x1d4 + MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5 0x1d4 + MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6 0x1d4 + MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7 0x1d4 + MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE 0x194 + >; + }; + + pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp { + fsl,pins = < + MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK 0x196 + MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD 0x1d6 + MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0 0x1d6 + MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1 0x1d6 + MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2 0x1d6 + MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3 0x1d6 + MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4 0x1d6 + MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5 0x1d6 + MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6 0x1d6 + MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7 0x1d6 + MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE 0x196 + >; + }; + + pinctrl_wdog: wdoggrp { + fsl,pins = < + MX8MM_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B 0xc6 + >; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi index d1b4582f44c4d79230cd2f85c23cbf0527a97caa..bcab830c6e95ec32468abd9ccf827d801e1295ec 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi @@ -32,10 +32,10 @@ }; /* Fixed clock dedicated to SPI CAN controller */ - clk20m: oscillator { + clk40m: oscillator { compatible = "fixed-clock"; #clock-cells = <0>; - clock-frequency = <20000000>; + clock-frequency = <40000000>; }; gpio-keys { @@ -53,6 +53,21 @@ }; }; + hdmi_connector: hdmi-connector { + compatible = "hdmi-connector"; + ddc-i2c-bus = <&i2c2>; + label = "hdmi"; + type = "a"; + status = "disabled"; + }; + + panel_lvds: panel-lvds { + compatible = "panel-lvds"; + backlight = <&backlight>; + data-mapping = "vesa-24"; + status = "disabled"; + }; + /* Carrier Board Supplies */ reg_1p8v: regulator-1p8v { compatible = "regulator-fixed"; @@ -202,8 +217,8 @@ can1: can@0 { compatible = "microchip,mcp251xfd"; - clocks = <&clk20m>; - interrupts-extended = <&gpio1 6 IRQ_TYPE_EDGE_FALLING>; + clocks = <&clk40m>; + interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_can1_int>; reg = <0>; @@ -367,8 +382,8 @@ nxp,dvs-standby-voltage = <850000>; regulator-always-on; regulator-boot-on; - regulator-max-microvolt = <950000>; - regulator-min-microvolt = <850000>; + regulator-max-microvolt = <1050000>; + regulator-min-microvolt = <805000>; regulator-name = "On-module +VDD_ARM (BUCK2)"; regulator-ramp-delay = <3125>; }; @@ -376,8 +391,8 @@ reg_vdd_dram: BUCK3 { regulator-always-on; regulator-boot-on; - regulator-max-microvolt = <950000>; - regulator-min-microvolt = <850000>; + regulator-max-microvolt = <1000000>; + regulator-min-microvolt = <805000>; regulator-name = "On-module +VDD_GPU_VPU_DDR (BUCK3)"; }; @@ -416,7 +431,7 @@ reg_vdd_snvs: LDO2 { regulator-always-on; regulator-boot-on; - regulator-max-microvolt = <900000>; + regulator-max-microvolt = <800000>; regulator-min-microvolt = <800000>; regulator-name = "On-module +V0.8_SNVS (LDO2)"; }; @@ -561,8 +576,8 @@ status = "disabled"; }; - lvds_ti_sn65dsi83: bridge@2c { - compatible = "ti,sn65dsi83"; + lvds_ti_sn65dsi84: bridge@2c { + compatible = "ti,sn65dsi84"; /* Verdin GPIO_9_DSI (SN65DSI84 IRQ, SODIMM 17, unused) */ /* Verdin GPIO_10_DSI (SODIMM 21) */ enable-gpios = <&gpio3 3 GPIO_ACTIVE_HIGH>; @@ -603,7 +618,7 @@ pinctrl-0 = <&pinctrl_gpio_9_dsi>, <&pinctrl_i2s_2_bclk_touch_reset>; reg = <0x4a>; /* Verdin I2S_2_BCLK (TOUCH_RESET#, SODIMM 42) */ - reset-gpios = <&gpio3 23 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio3 23 GPIO_ACTIVE_LOW>; status = "disabled"; }; @@ -745,6 +760,7 @@ }; &usbphynop2 { + power-domains = <&pgc_otg2>; vcc-supply = <®_vdd_3v3>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts b/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts index 636f8602b979cd5a99a04b4f6c3abbdf0cb82bbd..dd4302ac1de466a58894246706dbe45915a9b489 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts +++ b/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts @@ -213,7 +213,6 @@ compatible = "microchip,mcp2515"; reg = <0>; clocks = <&can20m>; - oscillator-frequency = <20000000>; interrupt-parent = <&gpio2>; interrupts = <3 IRQ_TYPE_LEVEL_LOW>; spi-max-frequency = <10000000>; diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi index 0c71b740a3166fbaa187439665ea58bcc20b87ad..cb2836bfbd95c7a3d7d354a6b63bea2e885f5925 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi @@ -672,7 +672,6 @@ <&clk IMX8MN_CLK_GPU_SHADER>, <&clk IMX8MN_CLK_GPU_BUS_ROOT>, <&clk IMX8MN_CLK_GPU_AHB>; - resets = <&src IMX8MQ_RESET_GPU_RESET>; }; pgc_dispmix: power-domain@3 { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts index 2ca2ede2e94e01783414a34710ce22658cbd2d16..382fbedaf6ba42c28e3ccd9b89aceac73891bfda 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts @@ -1,18 +1,23 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) /* * Copyright (C) 2022 Marek Vasut + * + * DHCOM iMX8MP variant: + * DHCM-iMX8ML8-C160-R409-F1638-SPI16-GE-CAN2-SD-RTC-WBTA-ADC-T-RGB-CSI2-HS-I-01D2 + * DHCOM PCB number: 660-100 or newer + * PDK2 PCB number: 516-400 or newer */ /dts-v1/; #include -#include #include #include "imx8mp-dhcom-som.dtsi" / { model = "DH electronics i.MX8M Plus DHCOM Premium Developer Kit (2)"; - compatible = "dh,imx8mp-dhcom-pdk2", "fsl,imx8mp"; + compatible = "dh,imx8mp-dhcom-pdk2", "dh,imx8mp-dhcom-som", + "fsl,imx8mp"; chosen { stdout-path = &uart1; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi index a616eb3780025fe892953b416f46996c4871875f..0f13ee36277151479f98e3ed3b31531620888856 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi @@ -70,7 +70,7 @@ &ecspi1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ecspi1>; - cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>; + cs-gpios = <&gpio5 17 GPIO_ACTIVE_LOW>; status = "disabled"; }; @@ -403,8 +403,8 @@ pinctrl-names = "default", "gpio"; pinctrl-0 = <&pinctrl_i2c5>; pinctrl-1 = <&pinctrl_i2c5_gpio>; - scl-gpios = <&gpio5 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; - sda-gpios = <&gpio5 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + scl-gpios = <&gpio3 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio3 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "okay"; }; @@ -648,10 +648,10 @@ pinctrl_ecspi1: dhcom-ecspi1-grp { fsl,pins = < - MX8MP_IOMUXC_ECSPI1_SCLK__ECSPI1_SCLK 0x44 - MX8MP_IOMUXC_ECSPI1_MOSI__ECSPI1_MOSI 0x44 - MX8MP_IOMUXC_ECSPI1_MISO__ECSPI1_MISO 0x44 - MX8MP_IOMUXC_ECSPI1_SS0__GPIO5_IO09 0x40 + MX8MP_IOMUXC_I2C1_SCL__ECSPI1_SCLK 0x44 + MX8MP_IOMUXC_I2C1_SDA__ECSPI1_MOSI 0x44 + MX8MP_IOMUXC_I2C2_SCL__ECSPI1_MISO 0x44 + MX8MP_IOMUXC_I2C2_SDA__GPIO5_IO17 0x40 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts index f6b017ab5f53590d319bbcdf111e6c2405a6a726..9f1469db554d3e0f85c8dccc962726a20a722c64 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts @@ -5,6 +5,7 @@ /dts-v1/; +#include #include "imx8mp.dtsi" / { @@ -33,6 +34,12 @@ <0x1 0x00000000 0 0xc0000000>; }; + pcie0_refclk: pcie0-refclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + }; + reg_can1_stby: regulator-can1-stby { compatible = "regulator-fixed"; regulator-name = "can1-stby"; @@ -55,6 +62,17 @@ enable-active-high; }; + reg_pcie0: regulator-pcie { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pcie0_reg>; + regulator-name = "MPCIE_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpio2 6 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + reg_usdhc2_vmmc: regulator-usdhc2 { compatible = "regulator-fixed"; pinctrl-names = "default"; @@ -350,6 +368,28 @@ */ }; +&pcie_phy { + fsl,refclk-pad-mode = ; + clocks = <&pcie0_refclk>; + clock-names = "ref"; + status = "okay"; +}; + +&pcie { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pcie0>; + reset-gpio = <&gpio2 7 GPIO_ACTIVE_LOW>; + clocks = <&clk IMX8MP_CLK_HSIO_ROOT>, + <&clk IMX8MP_CLK_PCIE_ROOT>, + <&clk IMX8MP_CLK_HSIO_AXI>; + clock-names = "pcie", "pcie_aux", "pcie_bus"; + assigned-clocks = <&clk IMX8MP_CLK_PCIE_AUX>; + assigned-clock-rates = <10000000>; + assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_50M>; + vpcie-supply = <®_pcie0>; + status = "okay"; +}; + &snvs_pwrkey { status = "okay"; }; @@ -502,6 +542,19 @@ >; }; + pinctrl_pcie0: pcie0grp { + fsl,pins = < + MX8MP_IOMUXC_I2C4_SCL__PCIE_CLKREQ_B 0x61 /* open drain, pull up */ + MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07 0x41 + >; + }; + + pinctrl_pcie0_reg: pcie0reggrp { + fsl,pins = < + MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06 0x41 + >; + }; + pinctrl_pmic: pmicgrp { fsl,pins = < MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x000001c0 diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts index dd703b6a5e17c91dac8860687a408ca4765310dc..a02b31c42db4872ca3d10f560789a35aece73d1c 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts @@ -2,7 +2,7 @@ /* * Copyright (c) 2018 NXP * Copyright (c) 2019 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ /dts-v1/; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi index 5116079cce22c5470d54d7ecfb6017945df8821a..a6319824ea2eb140843402fcbcd7d2c8a38a1667 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi @@ -2,7 +2,7 @@ /* * Copyright (c) 2018 NXP * Copyright (c) 2019 Engicam srl - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions(India) */ / { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..2f5cc013e8d6048bff9e3a0ae757ae122c637f48 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Avnet Embedded GmbH + */ +/dts-v1/; + +#include "imx8mp-msc-sm2s.dtsi" + +/ { + memory@40000000 { + device_type = "memory"; + reg = <0x0 0x40000000 0 0x80000000>; /* bank0, 2GiB */ + }; +}; + +&cpu_alert0 { + temperature = <95000>; +}; + +&cpu_crit0 { + temperature = <105000>; +}; + +&soc_alert0 { + temperature = <95000>; +}; + +&soc_crit0 { + temperature = <105000>; +}; + +&tca6424 { + gbe0-int-hog { + gpio-hog; + input; + gpios = <3 GPIO_ACTIVE_LOW>; + }; + + gbe1-int-hog { + gpio-hog; + input; + gpios = <4 GPIO_ACTIVE_LOW>; + }; + + cam2-rst-hog { + gpio-hog; + output-high; + gpios = <9 GPIO_ACTIVE_LOW>; + }; + + cam2-pwr-hog { + gpio-hog; + output-high; + gpios = <10 GPIO_ACTIVE_LOW>; + }; + + tpm-int-hog { + gpio-hog; + input; + gpios = <13 GPIO_ACTIVE_LOW>; + }; + + wifi-int-hog { + gpio-hog; + input; + gpios = <14 GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts new file mode 100644 index 0000000000000000000000000000000000000000..470ff8e31e3274bfd461149e30031fe06db36ec8 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Avnet Embedded GmbH + */ + +/dts-v1/; + +#include "imx8mp-msc-sm2s-14N0600E.dtsi" +#include +#include + +/ { + model = "MSC SM2-MB-EP1 Carrier Board with SM2S-IMX8PLUS-QC6-14N0600E SoM"; + compatible = "avnet,sm2s-imx8mp-14N0600E-ep1", + "avnet,sm2s-imx8mp-14N0600E", "avnet,sm2s-imx8mp", + "fsl,imx8mp"; +}; + +&flexcan1 { + status = "okay"; +}; + +&flexcan2 { + status = "okay"; +}; + +&usdhc2 { + no-1-8-v; +}; + +&iomuxc { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_smarc_gpio>; + + pinctrl_smarc_gpio: smarcgpiosgrp { + fsl,pins = + , /* GPIO0 */ + , /* GPIO1 */ + , /* GPIO2 */ + , /* GPIO3 */ + , /* GPIO4 */ + , /* GPIO5 */ + , /* GPIO6 */ + , /* GPIO7 */ + , /* GPIO8 */ + , /* GPIO9 */ + , /* GPIO10 */ + , /* GPIO11 */ + , /* GPIO12 */ + ; /* GPIO13 */ + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..5dbec71747c3bd080c08f38acec8da319c384696 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi @@ -0,0 +1,820 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Avnet Embedded GmbH + */ + +/dts-v1/; + +#include "imx8mp.dtsi" +#include + +/ { + aliases { + rtc0 = &sys_rtc; + rtc1 = &snvs_rtc; + }; + + chosen { + stdout-path = &uart2; + }; + + reg_usb0_host_vbus: regulator-usb0-vbus { + compatible = "regulator-fixed"; + regulator-name = "usb0_host_vbus"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_vbus>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + reg_usb1_host_vbus: regulator-usb1-vbus { + compatible = "regulator-fixed"; + regulator-name = "usb1_host_vbus"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb1_vbus>; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&gpio1 14 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + reg_usdhc2_vmmc: regulator-usdhc2 { + compatible = "regulator-fixed"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usdhc2_vmmc>; + regulator-name = "VSD_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>; + enable-active-high; + startup-delay-us = <100>; + off-on-delay-us = <12000>; + }; + + reg_flexcan1_xceiver: regulator-flexcan1 { + compatible = "regulator-fixed"; + regulator-name = "flexcan1-xceiver"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + reg_flexcan2_xceiver: regulator-flexcan2 { + compatible = "regulator-fixed"; + regulator-name = "flexcan2-xceiver"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + lcd0_backlight: backlight-0 { + compatible = "pwm-backlight"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_lcd0_backlight>; + pwms = <&pwm1 0 100000 0>; + brightness-levels = <0 255>; + num-interpolated-steps = <255>; + default-brightness-level = <255>; + enable-gpios = <&gpio1 5 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + lcd1_backlight: backlight-1 { + compatible = "pwm-backlight"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_lcd1_backlight>; + pwms = <&pwm2 0 100000 0>; + brightness-levels = <0 255>; + num-interpolated-steps = <255>; + default-brightness-level = <255>; + enable-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_leds>; + status = "okay"; + + led-sw { + label = "sw-led"; + gpios = <&gpio1 8 GPIO_ACTIVE_HIGH>; + default-state = "off"; + linux,default-trigger = "heartbeat"; + }; + }; + + extcon_usb0: extcon-usb0 { + compatible = "linux,extcon-usb-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0_extcon>; + id-gpio = <&gpio1 3 GPIO_ACTIVE_HIGH>; + }; +}; + +&A53_0 { + cpu-supply = <&vcc_arm>; +}; + +&A53_1 { + cpu-supply = <&vcc_arm>; +}; + +&A53_2 { + cpu-supply = <&vcc_arm>; +}; + +&A53_3 { + cpu-supply = <&vcc_arm>; +}; + +&ecspi1 { + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ecspi1>; + cs-gpios = <0>, <&gpio2 8 GPIO_ACTIVE_LOW>; +}; + +&ecspi2 { + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ecspi2>; + cs-gpios = <0>, <&gpio2 9 GPIO_ACTIVE_LOW>; +}; + +&eqos { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_eqos>; + phy-mode = "rgmii-id"; + phy-handle = <ðphy0>; + status = "okay"; + + mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + + ethphy0: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + eee-broken-1000t; + reset-gpios = <&tca6424 16 GPIO_ACTIVE_LOW>; + reset-assert-us = <1000>; + reset-deassert-us = <1000>; + ti,rx-internal-delay = ; + ti,tx-internal-delay = ; + ti,fifo-depth = ; + ti,clk-output-sel = ; + }; + }; +}; + +&fec { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_fec>; + phy-mode = "rgmii-id"; + phy-handle = <ðphy1>; + fsl,magic-packet; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy1: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + reg = <1>; + eee-broken-1000t; + reset-gpios = <&tca6424 17 GPIO_ACTIVE_LOW>; + reset-assert-us = <1000>; + reset-deassert-us = <1000>; + ti,rx-internal-delay = ; + ti,tx-internal-delay = ; + ti,fifo-depth = ; + ti,clk-output-sel = ; + }; + }; +}; + +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c1>; + clock-frequency = <400000>; + status = "okay"; + + id_eeprom: eeprom@50 { + compatible = "atmel,24c64"; + reg = <0x50>; + pagesize = <32>; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c2>; + clock-frequency = <400000>; + status = "disabled"; +}; + +&i2c3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c3>; + clock-frequency = <400000>; + status = "disabled"; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c4>; + clock-frequency = <400000>; + status = "disabled"; +}; + +&i2c5 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c5>; + clock-frequency = <400000>; + status = "disabled"; +}; + +&i2c6 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_i2c6>; + clock-frequency = <400000>; + status = "okay"; + + tca6424: gpio@22 { + compatible = "ti,tca6424"; + reg = <0x22>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_tca6424>; + gpio-controller; + #gpio-cells = <2>; + gpio-line-names = "BOOT_SEL0#", "BOOT_SEL1#", "BOOT_SEL2#", + "gbe0_int", "gbe1_int", "pmic_int", "rtc_int", "lvds_int", + "PCIE_WAKE#", "cam2_rst", "cam2_pwr", "SLEEP#", + "wifi_pd", "tpm_int", "wifi_int", "PCIE_A_RST#", + "gbe0_rst", "gbe1_rst", "LID#", "BATLOW#", "CHARGING#", + "CHARGER_PRSNT#"; + interrupt-parent = <&gpio1>; + interrupts = <9 IRQ_TYPE_EDGE_RISING>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + dsi_lvds_bridge: bridge@2d { + compatible = "ti,sn65dsi83"; + reg = <0x2d>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_lvds_bridge>; + enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; + + pmic: pmic@30 { + compatible = "ricoh,rn5t567"; + reg = <0x30>; + interrupt-parent = <&tca6424>; + interrupts = <5 IRQ_TYPE_EDGE_FALLING>; + + regulators { + DCDC1 { + regulator-name = "VCC_SOC"; + regulator-always-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + }; + + DCDC2 { + regulator-name = "VCC_DRAM"; + regulator-always-on; + regulator-min-microvolt = <1100000>; + regulator-max-microvolt = <1100000>; + }; + + vcc_arm: DCDC3 { + regulator-name = "VCC_ARM"; + regulator-always-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + }; + + DCDC4 { + regulator-name = "VCC_1V8"; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + LDO1 { + regulator-name = "VCC_LDO1_2V5"; + regulator-always-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + }; + + LDO2 { + regulator-name = "VCC_LDO2_1V8"; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + LDO3 { + regulator-name = "VCC_ETH_2V5"; + regulator-always-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + }; + + LDO4 { + regulator-name = "VCC_DDR4_2V5"; + regulator-always-on; + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + }; + + LDO5 { + regulator-name = "VCC_LDO5_1V8"; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + LDORTC1 { + regulator-name = "VCC_SNVS_1V8"; + regulator-always-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + LDORTC2 { + regulator-name = "VCC_SNVS_3V3"; + regulator-always-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + }; + }; + + sys_rtc: rtc@32 { + compatible = "ricoh,r2221tl"; + reg = <0x32>; + interrupt-parent = <&tca6424>; + interrupts = <6 IRQ_TYPE_EDGE_FALLING>; + }; + + tmp_sensor: temperature-sensor@71 { + compatible = "ti,tmp103"; + reg = <0x71>; + }; +}; + +&flexcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flexcan1>; + xceiver-supply = <®_flexcan1_xceiver>; + status = "disabled"; +}; + +&flexcan2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flexcan2>; + xceiver-supply = <®_flexcan2_xceiver>; + status = "disabled"; +}; + +&flexspi { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_flexspi0>; + status = "okay"; + + qspi_flash: flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <80000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + }; +}; + +&pwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm1>; + status = "disabled"; +}; + +&pwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm2>; + status = "disabled"; +}; + +&pwm3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm3>; + status = "disabled"; +}; + +&pwm4 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pwm4>; + status = "disabled"; +}; + +&snvs_pwrkey { + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart1>; + status = "okay"; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart2>; + uart-has-rtscts; + status = "okay"; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3>; + uart-has-rtscts; + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart4>; + status = "disabled"; +}; + +&usb3_phy0 { + vbus-supply = <®_usb0_host_vbus>; + status = "okay"; +}; + +&usb3_phy1 { + vbus-supply = <®_usb1_host_vbus>; + status = "okay"; +}; + +&usb3_0 { + status = "okay"; +}; + +&usb3_1 { + status = "okay"; +}; + +&usb_dwc3_0 { + dr_mode = "otg"; + hnp-disable; + srp-disable; + adp-disable; + extcon = <&extcon_usb0>; + status = "okay"; +}; + +&usb_dwc3_1 { + dr_mode = "host"; + status = "okay"; +}; + +&usdhc2 { + assigned-clocks = <&clk IMX8MP_CLK_USDHC2>; + assigned-clock-rates = <400000000>; + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>; + pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>; + pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>; + cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>; + wp-gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>; + bus-width = <4>; + vmmc-supply = <®_usdhc2_vmmc>; + status = "okay"; +}; + +&usdhc3 { + assigned-clocks = <&clk IMX8MP_CLK_USDHC3>; + assigned-clock-rates = <400000000>; + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc3>; + pinctrl-1 = <&pinctrl_usdhc3_100mhz>; + pinctrl-2 = <&pinctrl_usdhc3_200mhz>; + bus-width = <8>; + non-removable; + status = "okay"; +}; + +&wdog1 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_wdog>; + fsl,ext-reset-output; + status = "okay"; +}; + +&iomuxc { + pinctrl_ecspi1: ecspi1grp { + fsl,pins = + , + , + , + , + ; + }; + + pinctrl_ecspi2: ecspi2grp { + fsl,pins = + , + , + , + , + ; + }; + + pinctrl_eqos: eqosgrp { + fsl,pins = + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + }; + + pinctrl_fec: fecgrp { + fsl,pins = + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + }; + + pinctrl_flexcan1: flexcan1grp { + fsl,pins = + , + ; + }; + + pinctrl_flexcan2: flexcan2grp { + fsl,pins = + , + ; + }; + + pinctrl_flexspi0: flexspi0grp { + fsl,pins = + , + , + , + , + , + , + ; + }; + + pinctrl_i2c1: i2c1grp { + fsl,pins = + , + ; + }; + + pinctrl_i2c2: i2c2grp { + fsl,pins = + , + ; + }; + + pinctrl_i2c3: i2c3grp { + fsl,pins = + , + ; + }; + + pinctrl_i2c4: i2c4grp { + fsl,pins = + , + ; + }; + + pinctrl_i2c5: i2c5grp { + fsl,pins = + , + ; + }; + + pinctrl_i2c6: i2c6grp { + fsl,pins = + , + ; + }; + + pinctrl_lcd0_backlight: lcd0-backlightgrp { + fsl,pins = + ; + }; + + pinctrl_lcd1_backlight: lcd1-backlightgrp { + fsl,pins = + ; + }; + + pinctrl_leds: ledsgrp { + fsl,pins = + ; + }; + + pinctrl_lvds_bridge: lvds-bridgegrp { + fsl,pins = + ; + }; + + pinctrl_pwm1: pwm1grp { + fsl,pins = + ; + }; + + pinctrl_pwm2: pwm2grp { + fsl,pins = + ; + }; + + pinctrl_pwm3: pwm3grp { + fsl,pins = + ; + }; + + pinctrl_pwm4: pwm4grp { + fsl,pins = + ; + }; + + pinctrl_tca6424: tca6424grp { + fsl,pins = + ; + }; + + pinctrl_uart1: uart1grp { + fsl,pins = + , + ; + }; + + pinctrl_uart2: uart2grp { + fsl,pins = + , + , + , + ; + }; + + pinctrl_uart3: uart3grp { + fsl,pins = + , + , + , + ; + }; + + pinctrl_uart4: uart4grp { + fsl,pins = + , + ; + }; + + pinctrl_usb0_extcon: usb0-extcongrp { + fsl,pins = + ; + }; + + pinctrl_usb0_vbus: usb0-vbusgrp { + fsl,pins = + ; + }; + + pinctrl_usb1_vbus: usb1-vbusgrp { + fsl,pins = + ; + }; + + pinctrl_usdhc2_gpio: usdhc2-gpiogrp { + fsl,pins = + , + ; + }; + + pinctrl_usdhc2: usdhc2grp { + fsl,pins = + , + , + , + , + , + , + ; + }; + + pinctrl_usdhc2_vmmc: usdhc2-vmmcgrp { + fsl,pins = + ; + }; + + pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp { + fsl,pins = + , + , + , + , + , + , + ; + }; + + pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp { + fsl,pins = + , + , + , + , + , + , + ; + }; + + pinctrl_usdhc3: usdhc3grp { + fsl,pins = + , + , + , + , + , + , + , + , + , + , + ; + }; + + pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp { + fsl,pins = + , + , + , + , + , + , + , + , + , + , + ; + }; + + pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp { + fsl,pins = + , + , + , + , + , + , + , + , + , + , + ; + }; + + pinctrl_wdog: wdoggrp { + fsl,pins = + ; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts index d8ca52976170e8ce342d4c7a0c06567841a97645..7bf6f81e87b4731166a80b3bbcc2bb0ebcf79826 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts @@ -57,13 +57,13 @@ switch-1 { label = "S12"; linux,code = ; - gpios = <&gpio5 26 GPIO_ACTIVE_LOW>; + gpios = <&gpio5 27 GPIO_ACTIVE_LOW>; }; switch-2 { label = "S13"; linux,code = ; - gpios = <&gpio5 27 GPIO_ACTIVE_LOW>; + gpios = <&gpio5 26 GPIO_ACTIVE_LOW>; }; }; @@ -139,6 +139,13 @@ regulator-max-microvolt = <3300000>; }; + reg_vcc_5v0: regulator-5v0 { + compatible = "regulator-fixed"; + regulator-name = "VCC_5V0"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -394,6 +401,8 @@ &pcf85063 { /* RTC_EVENT# is connected on MBa8MPxL */ + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pcf85063>; interrupt-parent = <&gpio4>; interrupts = <28 IRQ_TYPE_EDGE_FALLING>; }; @@ -445,6 +454,38 @@ status = "okay"; }; +&usb3_0 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usb0>; + fsl,over-current-active-low; + status = "okay"; +}; + +&usb3_phy0 { + vbus-supply = <®_vcc_5v0>; + status = "okay"; +}; + +&usb_dwc3_0 { + /* dual role is implemented, but not a full featured OTG */ + hnp-disable; + srp-disable; + adp-disable; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "peripheral"; + status = "okay"; + + connector { + compatible = "gpio-usb-b-connector", "usb-b-connector"; + type = "micro"; + label = "X29"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usbcon0>; + id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>; + }; +}; + &usdhc2 { pinctrl-names = "default", "state_100mhz", "state_200mhz"; pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>; @@ -630,6 +671,10 @@ fsl,pins = ; /* Power enable */ }; + pinctrl_pcf85063: pcf85063grp { + fsl,pins = ; + }; + /* LVDS Backlight */ pinctrl_pwm2: pwm2grp { fsl,pins = ; @@ -666,6 +711,15 @@ ; }; + pinctrl_usb0: usb0grp { + fsl,pins = , + ; + }; + + pinctrl_usbcon0: usb0congrp { + fsl,pins = ; + }; + pinctrl_usdhc2: usdhc2grp { fsl,pins = , , diff --git a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts index 521215520a0f40acb3a9310f82eef1354328ee7c..06b4c93c587650e2b001a875c2a647e9763b7774 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts @@ -8,6 +8,7 @@ #include #include #include +#include #include "imx8mp.dtsi" @@ -100,6 +101,12 @@ }; }; + pcie0_refclk: pcie0-refclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + }; + pps { compatible = "pps-gpio"; pinctrl-names = "default"; @@ -123,8 +130,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_reg_can>; regulator-name = "can2_stby"; - gpio = <&gpio3 19 GPIO_ACTIVE_HIGH>; - enable-active-high; + gpio = <&gpio3 19 GPIO_ACTIVE_LOW>; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; }; @@ -135,13 +141,29 @@ compatible = "regulator-fixed"; regulator-name = "wl"; gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>; - startup-delay-us = <100>; + startup-delay-us = <70000>; enable-active-high; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; }; }; +&A53_0 { + cpu-supply = <®_arm>; +}; + +&A53_1 { + cpu-supply = <®_arm>; +}; + +&A53_2 { + cpu-supply = <®_arm>; +}; + +&A53_3 { + cpu-supply = <®_arm>; +}; + /* off-board header */ &ecspi2 { pinctrl-names = "default"; @@ -200,8 +222,8 @@ &gpio2 { gpio-line-names = "", "", "", "", "", "", "", "", - "", "", "", "", "", "", "", "", - "pcie3_wdis#", "", "", "pcie1_wdis@", "pcie2_wdis#", "", "", "", + "", "", "", "", "", "", "pcie3_wdis#", "", + "", "", "pcie2_wdis#", "", "", "", "", "", "", "", "", "", "", "", "", ""; }; @@ -362,7 +384,7 @@ regulator-ramp-delay = <3125>; }; - BUCK2 { + reg_arm: BUCK2 { regulator-name = "BUCK2"; regulator-min-microvolt = <720000>; regulator-max-microvolt = <1025000>; @@ -484,35 +506,40 @@ lan1: port@0 { reg = <0>; label = "lan1"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan2: port@1 { reg = <1>; label = "lan2"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan3: port@2 { reg = <2>; label = "lan3"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan4: port@3 { reg = <3>; label = "lan4"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; lan5: port@4 { reg = <4>; label = "lan5"; + phy-mode = "internal"; local-mac-address = [00 00 00 00 00 00]; }; - port@6 { - reg = <6>; + port@5 { + reg = <5>; label = "cpu"; ethernet = <&fec>; phy-mode = "rgmii-id"; @@ -542,6 +569,28 @@ status = "okay"; }; +&pcie_phy { + fsl,refclk-pad-mode = ; + fsl,clkreq-unsupported; + clocks = <&pcie0_refclk>; + clock-names = "ref"; + status = "okay"; +}; + +&pcie { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_pcie0>; + reset-gpio = <&gpio2 17 GPIO_ACTIVE_LOW>; + clocks = <&clk IMX8MP_CLK_HSIO_ROOT>, + <&clk IMX8MP_CLK_PCIE_ROOT>, + <&clk IMX8MP_CLK_HSIO_AXI>; + clock-names = "pcie", "pcie_aux", "pcie_bus"; + assigned-clocks = <&clk IMX8MP_CLK_PCIE_AUX>; + assigned-clock-rates = <10000000>; + assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_50M>; + status = "okay"; +}; + /* GPS / off-board header */ &uart1 { pinctrl-names = "default"; @@ -556,6 +605,21 @@ status = "okay"; }; +/* bluetooth HCI */ +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_uart3>, <&pinctrl_uart3_gpio>; + cts-gpios = <&gpio3 21 GPIO_ACTIVE_LOW>; + rts-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>; + uart-has-rtscts; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm4330-bt"; + shutdown-gpios = <&gpio3 8 GPIO_ACTIVE_HIGH>; + }; +}; + &uart4 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart4>; @@ -563,20 +627,35 @@ }; /* USB1 - Type C front panel */ -&usb3_phy0 { +&usb3_0 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; + fsl,over-current-active-low; status = "okay"; }; -&usb3_0 { - fsl,over-current-active-low; +&usb3_phy0 { status = "okay"; }; &usb_dwc3_0 { - dr_mode = "host"; + /* dual role is implemented but not a full featured OTG */ + adp-disable; + hnp-disable; + srp-disable; + dr_mode = "otg"; + usb-role-switch; + role-switch-default-mode = "peripheral"; status = "okay"; + + connector { + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_usbcon1>; + compatible = "gpio-usb-b-connector", "usb-b-connector"; + type = "micro"; + label = "Type-C"; + id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>; + }; }; /* USB2 - USB3.0 Hub */ @@ -596,6 +675,25 @@ status = "okay"; }; +/* SDIO WiFi */ +&usdhc1 { + pinctrl-names = "default", "state_100mhz", "state_200mhz"; + pinctrl-0 = <&pinctrl_usdhc1>; + pinctrl-1 = <&pinctrl_usdhc1_100mhz>; + pinctrl-2 = <&pinctrl_usdhc1_200mhz>; + bus-width = <4>; + non-removable; + vmmc-supply = <®_wifi_en>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + wifi@0 { + compatible = "cypress,cyw4373-fmac"; + reg = <0>; + }; +}; + /* eMMC */ &usdhc3 { assigned-clocks = <&clk IMX8MP_CLK_USDHC3>; @@ -625,7 +723,6 @@ MX8MP_IOMUXC_GPIO1_IO09__GPIO1_IO09 0x40000040 /* DIO0 */ MX8MP_IOMUXC_GPIO1_IO11__GPIO1_IO11 0x40000040 /* DIO1 */ MX8MP_IOMUXC_NAND_DQS__GPIO3_IO14 0x40000040 /* M2SKT_OFF# */ - MX8MP_IOMUXC_SD2_DATA2__GPIO2_IO17 0x40000150 /* PCIE1_WDIS# */ MX8MP_IOMUXC_SD2_DATA3__GPIO2_IO18 0x40000150 /* PCIE2_WDIS# */ MX8MP_IOMUXC_SD2_CMD__GPIO2_IO14 0x40000150 /* PCIE3_WDIS# */ MX8MP_IOMUXC_NAND_DATA00__GPIO3_IO06 0x40000040 /* M2SKT_RST# */ @@ -738,6 +835,12 @@ >; }; + pinctrl_pcie0: pciegrp { + fsl,pins = < + MX8MP_IOMUXC_SD2_DATA2__GPIO2_IO17 0x110 + >; + }; + pinctrl_pmic: pmicgrp { fsl,pins = < MX8MP_IOMUXC_NAND_DATA01__GPIO3_IO07 0x140 @@ -770,10 +873,10 @@ pinctrl_sai2: sai2grp { fsl,pins = < - MX8MP_IOMUXC_SAI2_TXFS__AUDIOMIX_SAI2_TX_SYNC - MX8MP_IOMUXC_SAI2_TXD0__AUDIOMIX_SAI2_TX_DATA00 - MX8MP_IOMUXC_SAI2_TXC__AUDIOMIX_SAI2_TX_BCLK - MX8MP_IOMUXC_SAI2_MCLK__AUDIOMIX_SAI2_MCLK + MX8MP_IOMUXC_SAI2_TXFS__AUDIOMIX_SAI2_TX_SYNC 0xd6 + MX8MP_IOMUXC_SAI2_TXD0__AUDIOMIX_SAI2_TX_DATA00 0xd6 + MX8MP_IOMUXC_SAI2_TXC__AUDIOMIX_SAI2_TX_BCLK 0xd6 + MX8MP_IOMUXC_SAI2_MCLK__AUDIOMIX_SAI2_MCLK 0xd6 >; }; @@ -825,7 +928,12 @@ pinctrl_usb1: usb1grp { fsl,pins = < MX8MP_IOMUXC_GPIO1_IO13__USB1_OTG_OC 0x140 - MX8MP_IOMUXC_GPIO1_IO10__USB1_OTG_ID 0x140 + >; + }; + + pinctrl_usbcon1: usb1congrp { + fsl,pins = < + MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10 0x140 >; }; @@ -840,6 +948,28 @@ >; }; + pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp { + fsl,pins = < + MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK 0x194 + MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD 0x1d4 + MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0 0x1d4 + MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1 0x1d4 + MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2 0x1d4 + MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3 0x1d4 + >; + }; + + pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp { + fsl,pins = < + MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK 0x196 + MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD 0x1d6 + MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0 0x1d6 + MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1 0x1d6 + MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2 0x1d6 + MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3 0x1d6 + >; + }; + pinctrl_usdhc3: usdhc3grp { fsl,pins = < MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK 0x190 diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi index c5987bdbb383cac59f7f1ca1ef303af3ea1a8b36..7b712d1888eadfc6e4eb60282bc6cb3ddc32f9f9 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi @@ -146,6 +146,22 @@ }; }; +&A53_0 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_1 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_2 { + cpu-supply = <®_vdd_arm>; +}; + +&A53_3 { + cpu-supply = <®_vdd_arm>; +}; + &cpu_alert0 { temperature = <95000>; }; @@ -286,7 +302,6 @@ status = "disabled"; }; - /* Verdin CAN_2 */ &flexcan2 { pinctrl-names = "default"; @@ -454,7 +469,7 @@ regulator-ramp-delay = <3125>; }; - BUCK2 { + reg_vdd_arm: BUCK2 { nxp,dvs-run-voltage = <950000>; nxp,dvs-standby-voltage = <850000>; regulator-always-on; @@ -628,7 +643,7 @@ interrupts = <5 IRQ_TYPE_EDGE_FALLING>; reg = <0x4a>; /* Verdin GPIO_2 (SODIMM 208) */ - reset-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 1 GPIO_ACTIVE_LOW>; status = "disabled"; }; }; @@ -705,7 +720,7 @@ pinctrl-0 = <&pinctrl_gpio_9_dsi>, <&pinctrl_i2s_2_bclk_touch_reset>; reg = <0x4a>; /* Verdin I2S_2_BCLK (TOUCH_RESET#, SODIMM 42) */ - reset-gpios = <&gpio5 0 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi index fe178b7d063cbac3340f20c6363ab65d89667cc2..bb916a0948a8f737d055f2925944915fb153109d 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi @@ -5,8 +5,10 @@ #include #include +#include #include #include +#include #include #include @@ -469,6 +471,11 @@ wakeup-source; status = "disabled"; }; + + snvs_lpgpr: snvs-lpgpr { + compatible = "fsl,imx8mp-snvs-lpgpr", + "fsl,imx7d-snvs-lpgpr"; + }; }; clk: clock-controller@30380000 { @@ -597,6 +604,33 @@ reg = ; clocks = <&clk IMX8MP_CLK_MEDIA_ISP_ROOT>; }; + + pgc_vpumix: power-domain@19 { + #power-domain-cells = <0>; + reg = ; + clocks =<&clk IMX8MP_CLK_VPU_ROOT>; + }; + + pgc_vpu_g1: power-domain@20 { + #power-domain-cells = <0>; + power-domains = <&pgc_vpumix>; + reg = ; + clocks = <&clk IMX8MP_CLK_VPU_G1_ROOT>; + }; + + pgc_vpu_g2: power-domain@21 { + #power-domain-cells = <0>; + power-domains = <&pgc_vpumix>; + reg = ; + clocks = <&clk IMX8MP_CLK_VPU_G2_ROOT>; + }; + + pgc_vpu_vc8000e: power-domain@22 { + #power-domain-cells = <0>; + power-domains = <&pgc_vpumix>; + reg = ; + clocks = <&clk IMX8MP_CLK_VPU_VC8KE_ROOT>; + }; }; }; }; @@ -1064,6 +1098,18 @@ "lcdif1", "isi", "mipi-csi2", "lcdif2", "isp", "dwe", "mipi-dsi2"; + interconnects = + <&noc IMX8MP_ICM_LCDIF_RD &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_LCDIF_WR &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_ISI0 &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_ISI1 &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_ISI2 &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_ISP0 &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_ISP1 &noc IMX8MP_ICN_MEDIA>, + <&noc IMX8MP_ICM_DWE &noc IMX8MP_ICN_MEDIA>; + interconnect-names = "lcdif-rd", "lcdif-wr", "isi0", + "isi1", "isi2", "isp0", "isp1", + "dwe"; clocks = <&clk IMX8MP_CLK_MEDIA_APB_ROOT>, <&clk IMX8MP_CLK_MEDIA_AXI_ROOT>, <&clk IMX8MP_CLK_MEDIA_CAM1_PIX_ROOT>, @@ -1084,6 +1130,17 @@ #power-domain-cells = <1>; }; + pcie_phy: pcie-phy@32f00000 { + compatible = "fsl,imx8mp-pcie-phy"; + reg = <0x32f00000 0x10000>; + resets = <&src IMX8MP_RESET_PCIEPHY>, + <&src IMX8MP_RESET_PCIEPHY_PERST>; + reset-names = "pciephy", "perst"; + power-domains = <&hsio_blk_ctrl IMX8MP_HSIOBLK_PD_PCIE_PHY>; + #phy-cells = <0>; + status = "disabled"; + }; + hsio_blk_ctrl: blk-ctrl@32f10000 { compatible = "fsl,imx8mp-hsio-blk-ctrl", "syscon"; reg = <0x32f10000 0x24>; @@ -1095,10 +1152,46 @@ <&pgc_hsiomix>, <&pgc_pcie_phy>; power-domain-names = "bus", "usb", "usb-phy1", "usb-phy2", "pcie", "pcie-phy"; + interconnects = <&noc IMX8MP_ICM_NOC_PCIE &noc IMX8MP_ICN_HSIO>, + <&noc IMX8MP_ICM_USB1 &noc IMX8MP_ICN_HSIO>, + <&noc IMX8MP_ICM_USB2 &noc IMX8MP_ICN_HSIO>, + <&noc IMX8MP_ICM_PCIE &noc IMX8MP_ICN_HSIO>; + interconnect-names = "noc-pcie", "usb1", "usb2", "pcie"; #power-domain-cells = <1>; }; }; + pcie: pcie@33800000 { + compatible = "fsl,imx8mp-pcie"; + reg = <0x33800000 0x400000>, <0x1ff00000 0x80000>; + reg-names = "dbi", "config"; + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + bus-range = <0x00 0xff>; + ranges = <0x81000000 0 0x00000000 0x1ff80000 0 0x00010000>, /* downstream I/O 64KB */ + <0x82000000 0 0x18000000 0x18000000 0 0x07f00000>; /* non-prefetchable memory */ + num-lanes = <1>; + num-viewport = <4>; + interrupts = ; + interrupt-names = "msi"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &gic GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &gic GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &gic GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &gic GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>; + fsl,max-link-speed = <3>; + linux,pci-domain = <0>; + power-domains = <&hsio_blk_ctrl IMX8MP_HSIOBLK_PD_PCIE>; + resets = <&src IMX8MP_RESET_PCIE_CTRL_APPS_EN>, + <&src IMX8MP_RESET_PCIE_CTRL_APPS_TURNOFF>; + reset-names = "apps", "turnoff"; + phys = <&pcie_phy>; + phy-names = "pcie-phy"; + status = "disabled"; + }; + gpu3d: gpu@38000000 { compatible = "vivante,gc"; reg = <0x38000000 0x8000>; @@ -1130,6 +1223,23 @@ power-domains = <&pgc_gpu2d>; }; + vpumix_blk_ctrl: blk-ctrl@38330000 { + compatible = "fsl,imx8mp-vpu-blk-ctrl", "syscon"; + reg = <0x38330000 0x100>; + #power-domain-cells = <1>; + power-domains = <&pgc_vpumix>, <&pgc_vpu_g1>, + <&pgc_vpu_g2>, <&pgc_vpu_vc8000e>; + power-domain-names = "bus", "g1", "g2", "vc8000e"; + clocks = <&clk IMX8MP_CLK_VPU_G1_ROOT>, + <&clk IMX8MP_CLK_VPU_G2_ROOT>, + <&clk IMX8MP_CLK_VPU_VC8KE_ROOT>; + clock-names = "g1", "g2", "vc8000e"; + interconnects = <&noc IMX8MP_ICM_VPU_G1 &noc IMX8MP_ICN_VIDEO>, + <&noc IMX8MP_ICM_VPU_G2 &noc IMX8MP_ICN_VIDEO>, + <&noc IMX8MP_ICM_VPU_H1 &noc IMX8MP_ICN_VIDEO>; + interconnect-names = "g1", "g2", "vc8000e"; + }; + gic: interrupt-controller@38800000 { compatible = "arm,gic-v3"; reg = <0x38800000 0x10000>, @@ -1189,7 +1299,7 @@ interrupts = ; phys = <&usb3_phy0>, <&usb3_phy0>; phy-names = "usb2-phy", "usb3-phy"; - snps,dis-u2-freeclk-exists-quirk; + snps,gfladj-refclk-lpm-sel-quirk; }; }; @@ -1231,7 +1341,7 @@ interrupts = ; phys = <&usb3_phy1>, <&usb3_phy1>; phy-names = "usb2-phy", "usb3-phy"; - snps,dis-u2-freeclk-exists-quirk; + snps,gfladj-refclk-lpm-sel-quirk; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi index 9eec8a7eecfc880b11d9c7e00a13ba9785802de2..ae08556b2ef2f0140f06a2027743a49c54c89023 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi @@ -7,6 +7,7 @@ #include "dt-bindings/input/input.h" #include +#include #include "dt-bindings/pwm/pwm.h" #include "dt-bindings/usb/pd.h" #include "imx8mq.dtsi" @@ -54,6 +55,31 @@ }; }; + led-controller { + compatible = "pwm-leds"; + + led-0 { + function = LED_FUNCTION_STATUS; + color = ; + max-brightness = <248>; + pwms = <&pwm2 0 50000 0>; + }; + + led-1 { + function = LED_FUNCTION_STATUS; + color = ; + max-brightness = <248>; + pwms = <&pwm4 0 50000 0>; + }; + + led-2 { + function = LED_FUNCTION_STATUS; + color = ; + max-brightness = <248>; + pwms = <&pwm3 0 50000 0>; + }; + }; + reg_aud_1v8: regulator-audio-1v8 { compatible = "regulator-fixed"; pinctrl-names = "default"; @@ -747,6 +773,10 @@ interrupt-names = "irq"; connector { + compatible = "usb-c-connector"; + label = "USB-C"; + data-role = "dual"; + ports { #address-cells = <1>; #size-cells = <0>; @@ -1070,6 +1100,12 @@ pinctrl-0 = <&pinctrl_i2c4>; status = "okay"; + vcm@c { + compatible = "dongwoon,dw9714"; + reg = <0x0c>; + vcc-supply = <®_csi_1v8>; + }; + bat: fuel-gauge@36 { compatible = "maxim,max17055"; reg = <0x36>; @@ -1077,6 +1113,7 @@ interrupts = <20 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gauge>; + power-supplies = <&bq25895>; maxim,over-heat-temp = <700>; maxim,over-volt = <4500>; maxim,rsns-microohm = <5000>; @@ -1106,8 +1143,6 @@ }; &mipi_csi1 { - #address-cells = <1>; - #size-cells = <0>; status = "okay"; ports { @@ -1265,6 +1300,7 @@ #size-cells = <0>; dr_mode = "otg"; snps,dis_u3_susphy_quirk; + usb-role-switch; status = "okay"; port@0 { diff --git a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts index 8956a46788fac2d4c94cb0410d1bcb1249868630..055031bba8c4b4a0095549081db3e1d3db2de29a 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts +++ b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts @@ -210,7 +210,6 @@ status = "okay"; }; - ®_1p8v { vin-supply = <®_main_5v>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq.dtsi index 899e8e7dbc24f20ac2097245f71f3cb43345dc1f..802ad6e5cef61790c98147ac9f596c4e1bdfbb21 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq-tqma8mq.dtsi @@ -204,7 +204,6 @@ reg = <0x51>; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_rtc>; - interrupt-names = "irq"; interrupt-parent = <&gpio1>; interrupts = <1 IRQ_TYPE_EDGE_FALLING>; quartz-load-femtofarads = <7000>; diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi index e9f0cdd10ab620e1eb456d6c0557d74ddedc9216..19eaa523564d3b7d3e8c7879b5ca8ea974f6349b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi @@ -534,7 +534,7 @@ status = "disabled"; }; - sdma2: sdma@302c0000 { + sdma2: dma-controller@302c0000 { compatible = "fsl,imx8mq-sdma","fsl,imx7d-sdma"; reg = <0x302c0000 0x10000>; interrupts = ; @@ -1302,7 +1302,7 @@ status = "disabled"; }; - sdma1: sdma@30bd0000 { + sdma1: dma-controller@30bd0000 { compatible = "fsl,imx8mq-sdma","fsl,imx7d-sdma"; reg = <0x30bd0000 0x10000>; interrupts = ; diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts index 33e84c4e9ed89a8a1a38eb05bd84a5174c8bde86..f1c6d933a17c4b47bcb44f0ac937e469071f1a14 100644 --- a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts @@ -19,6 +19,21 @@ device_type = "memory"; reg = <0x0 0x80000000 0 0x80000000>; }; + + clock_ext_rmii: clock-ext-rmii { + compatible = "fixed-clock"; + clock-frequency = <50000000>; + clock-output-names = "ext_rmii_clk"; + #clock-cells = <0>; + }; + + clock_ext_ts: clock-ext-ts { + compatible = "fixed-clock"; + /* External ts clock is 50MHZ from PHY on EVK board. */ + clock-frequency = <50000000>; + clock-output-names = "ext_ts_clk"; + #clock-cells = <0>; + }; }; &lpuart5 { @@ -38,7 +53,49 @@ status = "okay"; }; +&fec { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&pinctrl_enet>; + pinctrl-1 = <&pinctrl_enet>; + clocks = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>, + <&pcc4 IMX8ULP_CLK_ENET>, + <&cgc1 IMX8ULP_CLK_ENET_TS_SEL>, + <&clock_ext_rmii>; + clock-names = "ipg", "ahb", "ptp", "enet_clk_ref"; + assigned-clocks = <&cgc1 IMX8ULP_CLK_ENET_TS_SEL>; + assigned-clock-parents = <&clock_ext_ts>; + phy-mode = "rmii"; + phy-handle = <ðphy>; + status = "okay"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + ethphy: ethernet-phy@1 { + reg = <1>; + micrel,led-mode = <1>; + }; + }; +}; + &iomuxc1 { + pinctrl_enet: enetgrp { + fsl,pins = < + MX8ULP_PAD_PTE15__ENET0_MDC 0x43 + MX8ULP_PAD_PTE14__ENET0_MDIO 0x43 + MX8ULP_PAD_PTE17__ENET0_RXER 0x43 + MX8ULP_PAD_PTE18__ENET0_CRS_DV 0x43 + MX8ULP_PAD_PTF1__ENET0_RXD0 0x43 + MX8ULP_PAD_PTE20__ENET0_RXD1 0x43 + MX8ULP_PAD_PTE16__ENET0_TXEN 0x43 + MX8ULP_PAD_PTE23__ENET0_TXD0 0x43 + MX8ULP_PAD_PTE22__ENET0_TXD1 0x43 + MX8ULP_PAD_PTE19__ENET0_REFCLK 0x43 + MX8ULP_PAD_PTF10__ENET0_1588_CLKIN 0x43 + >; + }; + pinctrl_lpuart5: lpuart5grp { fsl,pins = < MX8ULP_PAD_PTF14__LPUART5_TX 0x3 diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h b/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h old mode 100755 new mode 100644 diff --git a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi index 60c1b018bf03db0ed9ea1ac5201129229d336b86..06ce5f19aa8aacbbcc3d8562d0f2a73fcde49e1d 100644 --- a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi @@ -16,6 +16,7 @@ #size-cells = <2>; aliases { + ethernet0 = &fec; gpio0 = &gpiod; gpio1 = &gpioe; gpio2 = &gpiof; @@ -62,6 +63,14 @@ interrupts = ; }; + pmu { + compatible = "arm,cortex-a35-pmu"; + interrupt-parent = <&gic>; + interrupts = ; + interrupt-affinity = <&A35_0>, <&A35_1>; + }; + psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -111,7 +120,7 @@ #size-cells = <1>; ranges = <0 0x0 0x2201f000 0x1000>; - scmi_buf: scmi-buf@0 { + scmi_buf: scmi-sram-section@0 { compatible = "arm,scmi-shmem"; reg = <0x0 0x400>; }; @@ -143,6 +152,13 @@ #size-cells = <1>; ranges = <0x0 0x0 0x0 0x40000000>; + s4muap: mailbox@27020000 { + compatible = "fsl,imx8ulp-mu-s4"; + reg = <0x27020000 0x10000>; + interrupts = ; + #mbox-cells = <2>; + }; + per_bridge3: bus@29000000 { compatible = "simple-bus"; reg = <0x29000000 0x800000>; @@ -150,6 +166,23 @@ #size-cells = <1>; ranges; + mu: mailbox@29220000 { + compatible = "fsl,imx8ulp-mu"; + reg = <0x29220000 0x10000>; + interrupts = ; + #mbox-cells = <2>; + status = "disabled"; + }; + + mu3: mailbox@29230000 { + compatible = "fsl,imx8ulp-mu"; + reg = <0x29230000 0x10000>; + interrupts = ; + clocks = <&pcc3 IMX8ULP_CLK_MU3_A>; + #mbox-cells = <2>; + status = "disabled"; + }; + wdog3: watchdog@292a0000 { compatible = "fsl,imx8ulp-wdt", "fsl,imx7ulp-wdt"; reg = <0x292a0000 0x10000>; @@ -163,8 +196,6 @@ cgc1: clock-controller@292c0000 { compatible = "fsl,imx8ulp-cgc1"; reg = <0x292c0000 0x10000>; - clocks = <&rosc>, <&sosc>, <&frosc>, <&lposc>; - clock-names = "rosc", "sosc", "frosc", "lposc"; #clock-cells = <1>; }; @@ -172,6 +203,7 @@ compatible = "fsl,imx8ulp-pcc3"; reg = <0x292d0000 0x10000>; #clock-cells = <1>; + #reset-cells = <1>; }; tpm5: tpm@29340000 { @@ -192,7 +224,7 @@ <&pcc3 IMX8ULP_CLK_LPI2C4>; clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX8ULP_CLK_LPI2C4>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; assigned-clock-rates = <48000000>; status = "disabled"; }; @@ -205,7 +237,7 @@ <&pcc3 IMX8ULP_CLK_LPI2C5>; clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX8ULP_CLK_LPI2C5>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; assigned-clock-rates = <48000000>; status = "disabled"; }; @@ -238,8 +270,8 @@ <&pcc3 IMX8ULP_CLK_LPSPI4>; clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX8ULP_CLK_LPSPI4>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; - assigned-clock-rates = <16000000>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; + assigned-clock-rates = <48000000>; status = "disabled"; }; @@ -253,8 +285,8 @@ <&pcc3 IMX8ULP_CLK_LPSPI5>; clock-names = "per", "ipg"; assigned-clocks = <&pcc3 IMX8ULP_CLK_LPSPI5>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; - assigned-clock-rates = <16000000>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; + assigned-clock-rates = <48000000>; status = "disabled"; }; }; @@ -270,6 +302,7 @@ compatible = "fsl,imx8ulp-pcc4"; reg = <0x29800000 0x10000>; #clock-cells = <1>; + #reset-cells = <1>; }; lpi2c6: i2c@29840000 { @@ -280,7 +313,7 @@ <&pcc4 IMX8ULP_CLK_LPI2C6>; clock-names = "per", "ipg"; assigned-clocks = <&pcc4 IMX8ULP_CLK_LPI2C6>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; assigned-clock-rates = <48000000>; status = "disabled"; }; @@ -293,7 +326,7 @@ <&pcc4 IMX8ULP_CLK_LPI2C7>; clock-names = "per", "ipg"; assigned-clocks = <&pcc4 IMX8ULP_CLK_LPI2C7>; - assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>; + assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>; assigned-clock-rates = <48000000>; status = "disabled"; }; @@ -365,6 +398,16 @@ bus-width = <4>; status = "disabled"; }; + + fec: ethernet@29950000 { + compatible = "fsl,imx8ulp-fec", "fsl,imx6ul-fec", "fsl,imx6q-fec"; + reg = <0x29950000 0x10000>; + interrupts = ; + interrupt-names = "int0"; + fsl,num-tx-queues = <1>; + fsl,num-rx-queues = <1>; + status = "disabled"; + }; }; gpioe: gpio@2d000080 { @@ -405,8 +448,6 @@ cgc2: clock-controller@2da60000 { compatible = "fsl,imx8ulp-cgc2"; reg = <0x2da60000 0x10000>; - clocks = <&sosc>, <&frosc>; - clock-names = "sosc", "frosc"; #clock-cells = <1>; }; @@ -414,6 +455,7 @@ compatible = "fsl,imx8ulp-pcc5"; reg = <0x2da70000 0x10000>; #clock-cells = <1>; + #reset-cells = <1>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi index f83a07c7c9b1e30a20bf6d362e3e932e709160d8..3a5713bb4880e44d8d78b94374aa5266c6f0b28b 100644 --- a/arch/arm64/boot/dts/freescale/imx93.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93.dtsi @@ -7,6 +7,7 @@ #include #include #include +#include #include "imx93-pinfunc.h" @@ -16,6 +17,14 @@ #size-cells = <2>; aliases { + i2c0 = &lpi2c1; + i2c1 = &lpi2c2; + i2c2 = &lpi2c3; + i2c3 = &lpi2c4; + i2c4 = &lpi2c5; + i2c5 = &lpi2c6; + i2c6 = &lpi2c7; + i2c7 = &lpi2c8; mmc0 = &usdhc1; mmc1 = &usdhc2; mmc2 = &usdhc3; @@ -72,6 +81,11 @@ clock-output-names = "clk_ext1"; }; + pmu { + compatible = "arm,cortex-a55-pmu"; + interrupts = ; + }; + psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -112,6 +126,11 @@ #size-cells = <1>; ranges; + anomix_ns_gpr: syscon@44210000 { + compatible = "fsl,imx93-aonmix-ns-syscfg", "syscon"; + reg = <0x44210000 0x1000>; + }; + mu1: mailbox@44230000 { compatible = "fsl,imx93-mu", "fsl,imx8ulp-mu"; reg = <0x44230000 0x10000>; @@ -128,6 +147,50 @@ clock-names = "per"; }; + lpi2c1: i2c@44340000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x44340000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C1_GATE>, + <&clk IMX93_CLK_BUS_AON>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpi2c2: i2c@44350000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x44350000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C2_GATE>, + <&clk IMX93_CLK_BUS_AON>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpspi1: spi@44360000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "fsl,imx93-spi", "fsl,imx7ulp-spi"; + reg = <0x44360000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPSPI1_GATE>, + <&clk IMX93_CLK_BUS_AON>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpspi2: spi@44370000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "fsl,imx93-spi", "fsl,imx7ulp-spi"; + reg = <0x44370000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPSPI2_GATE>, + <&clk IMX93_CLK_BUS_AON>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + lpuart1: serial@44380000 { compatible = "fsl,imx93-lpuart", "fsl,imx7ulp-lpuart"; reg = <0x44380000 0x1000>; @@ -161,6 +224,30 @@ status = "okay"; }; + src: system-controller@44460000 { + compatible = "fsl,imx93-src", "syscon"; + reg = <0x44460000 0x10000>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + mediamix: power-domain@44462400 { + compatible = "fsl,imx93-src-slice"; + reg = <0x44462400 0x400>, <0x44465800 0x400>; + #power-domain-cells = <0>; + clocks = <&clk IMX93_CLK_MEDIA_AXI>, + <&clk IMX93_CLK_MEDIA_APB>; + }; + + mlmix: power-domain@44461800 { + compatible = "fsl,imx93-src-slice"; + reg = <0x44461800 0x400>, <0x44464800 0x400>; + #power-domain-cells = <0>; + clocks = <&clk IMX93_CLK_ML_APB>, + <&clk IMX93_CLK_ML>; + }; + }; + anatop: anatop@44480000 { compatible = "fsl,imx93-anatop", "syscon"; reg = <0x44480000 0x10000>; @@ -174,6 +261,11 @@ #size-cells = <1>; ranges; + wakeupmix_gpr: syscon@42420000 { + compatible = "fsl,imx93-wakeupmix-syscfg", "syscon"; + reg = <0x42420000 0x1000>; + }; + mu2: mailbox@42440000 { compatible = "fsl,imx93-mu", "fsl,imx8ulp-mu"; reg = <0x42440000 0x10000>; @@ -182,6 +274,26 @@ status = "disabled"; }; + lpi2c3: i2c@42530000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x42530000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C3_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpi2c4: i2c@42540000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x42540000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C4_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + lpuart3: serial@42570000 { compatible = "fsl,imx93-lpuart", "fsl,imx7ulp-lpuart"; reg = <0x42570000 0x1000>; @@ -235,6 +347,47 @@ clock-names = "ipg"; status = "disabled"; }; + + lpi2c5: i2c@426b0000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x426b0000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C5_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpi2c6: i2c@426c0000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x426c0000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C6_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpi2c7: i2c@426d0000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x426d0000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C7_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + + lpi2c8: i2c@426e0000 { + compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c"; + reg = <0x426e0000 0x10000>; + interrupts = ; + clocks = <&clk IMX93_CLK_LPI2C8_GATE>, + <&clk IMX93_CLK_BUS_WAKEUP>; + clock-names = "per", "ipg"; + status = "disabled"; + }; + }; aips3: bus@42800000 { @@ -248,8 +401,8 @@ compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc"; reg = <0x42850000 0x10000>; interrupts = ; - clocks = <&clk IMX93_CLK_DUMMY>, - <&clk IMX93_CLK_DUMMY>, + clocks = <&clk IMX93_CLK_BUS_WAKEUP>, + <&clk IMX93_CLK_WAKEUP_AXI>, <&clk IMX93_CLK_USDHC1_GATE>; clock-names = "ipg", "ahb", "per"; bus-width = <8>; @@ -262,8 +415,8 @@ compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc"; reg = <0x42860000 0x10000>; interrupts = ; - clocks = <&clk IMX93_CLK_DUMMY>, - <&clk IMX93_CLK_DUMMY>, + clocks = <&clk IMX93_CLK_BUS_WAKEUP>, + <&clk IMX93_CLK_WAKEUP_AXI>, <&clk IMX93_CLK_USDHC2_GATE>; clock-names = "ipg", "ahb", "per"; bus-width = <4>; @@ -276,8 +429,8 @@ compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc"; reg = <0x428b0000 0x10000>; interrupts = ; - clocks = <&clk IMX93_CLK_DUMMY>, - <&clk IMX93_CLK_DUMMY>, + clocks = <&clk IMX93_CLK_BUS_WAKEUP>, + <&clk IMX93_CLK_WAKEUP_AXI>, <&clk IMX93_CLK_USDHC3_GATE>; clock-names = "ipg", "ahb", "per"; bus-width = <4>; @@ -295,6 +448,9 @@ interrupts = ; interrupt-controller; #interrupt-cells = <2>; + clocks = <&clk IMX93_CLK_GPIO2_GATE>, + <&clk IMX93_CLK_GPIO2_GATE>; + clock-names = "gpio", "port"; gpio-ranges = <&iomuxc 0 32 32>; }; @@ -306,6 +462,9 @@ interrupts = ; interrupt-controller; #interrupt-cells = <2>; + clocks = <&clk IMX93_CLK_GPIO3_GATE>, + <&clk IMX93_CLK_GPIO3_GATE>; + clock-names = "gpio", "port"; gpio-ranges = <&iomuxc 0 64 32>; }; @@ -317,6 +476,9 @@ interrupts = ; interrupt-controller; #interrupt-cells = <2>; + clocks = <&clk IMX93_CLK_GPIO4_GATE>, + <&clk IMX93_CLK_GPIO4_GATE>; + clock-names = "gpio", "port"; gpio-ranges = <&iomuxc 0 96 32>; }; @@ -328,7 +490,39 @@ interrupts = ; interrupt-controller; #interrupt-cells = <2>; + clocks = <&clk IMX93_CLK_GPIO1_GATE>, + <&clk IMX93_CLK_GPIO1_GATE>; + clock-names = "gpio", "port"; gpio-ranges = <&iomuxc 0 0 32>; }; + + s4muap: mailbox@47520000 { + compatible = "fsl,imx93-mu-s4"; + reg = <0x47520000 0x10000>; + interrupts = , + ; + interrupt-names = "txirq", "rxirq"; + #mbox-cells = <2>; + }; + + media_blk_ctrl: system-controller@4ac10000 { + compatible = "fsl,imx93-media-blk-ctrl", "syscon"; + reg = <0x4ac10000 0x10000>; + power-domains = <&mediamix>; + clocks = <&clk IMX93_CLK_MEDIA_APB>, + <&clk IMX93_CLK_MEDIA_AXI>, + <&clk IMX93_CLK_NIC_MEDIA_GATE>, + <&clk IMX93_CLK_MEDIA_DISP_PIX>, + <&clk IMX93_CLK_CAM_PIX>, + <&clk IMX93_CLK_PXP_GATE>, + <&clk IMX93_CLK_LCDIF_GATE>, + <&clk IMX93_CLK_ISI_GATE>, + <&clk IMX93_CLK_MIPI_CSI_GATE>, + <&clk IMX93_CLK_MIPI_DSI_GATE>; + clock-names = "apb", "axi", "nic", "disp", "cam", + "pxp", "lcdif", "isi", "csi", "dsi"; + #power-domain-cells = <1>; + status = "disabled"; + }; }; }; diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile index b6d493e34dc57665e04e304efa023f89bb11fb21..058237681fe51982fe80647b7ab4d2e6102c3b0e 100644 --- a/arch/arm64/boot/dts/marvell/Makefile +++ b/arch/arm64/boot/dts/marvell/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # Mvebu SoC Family dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-db.dtb +dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-eDPU.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin-emmc.dtb dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin-ultra.dtb diff --git a/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi b/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi index 80b44c7df56aad6ca4caeb597a7b12d54f1d2e54..44ed6f963b75abbe68f2e8e7c73ebe19468242d8 100644 --- a/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi +++ b/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi @@ -95,6 +95,36 @@ status = "okay"; }; + uart1: serial@12100 { + compatible = "snps,dw-apb-uart"; + reg = <0x11000 0x100>; + reg-shift = <2>; + interrupts = ; + reg-io-width = <1>; + clocks = <&cnm_clock>; + status = "disabled"; + }; + + uart2: serial@12200 { + compatible = "snps,dw-apb-uart"; + reg = <0x12200 0x100>; + reg-shift = <2>; + interrupts = ; + reg-io-width = <1>; + clocks = <&cnm_clock>; + status = "disabled"; + }; + + uart3: serial@12300 { + compatible = "snps,dw-apb-uart"; + reg = <0x12300 0x100>; + reg-shift = <2>; + interrupts = ; + reg-io-width = <1>; + clocks = <&cnm_clock>; + status = "disabled"; + }; + mdio: mdio@22004 { #address-cells = <1>; #size-cells = <0>; @@ -117,8 +147,8 @@ pinctrl-names = "default", "gpio"; pinctrl-0 = <&i2c0_pins>; pinctrl-1 = <&i2c0_gpio>; - scl_gpio = <&gpio0 26 GPIO_ACTIVE_HIGH>; - sda_gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>; + scl-gpios = <&gpio0 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -136,8 +166,8 @@ pinctrl-names = "default", "gpio"; pinctrl-0 = <&i2c1_pins>; pinctrl-1 = <&i2c1_gpio>; - scl_gpio = <&gpio0 20 GPIO_ACTIVE_HIGH>; - sda_gpio = <&gpio0 21 GPIO_ACTIVE_HIGH>; + scl-gpios = <&gpio0 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts b/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts new file mode 100644 index 0000000000000000000000000000000000000000..57fc698e55d0c3125b36dad9a6e946c107bc69d6 --- /dev/null +++ b/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/dts-v1/; + +#include "armada-3720-uDPU.dtsi" + +/ { + model = "Methode eDPU Board"; + compatible = "methode,edpu", "marvell,armada3720", "marvell,armada3710"; +}; + +ð0 { + phy-mode = "2500base-x"; +}; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts b/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts index 070725b81be51c3fff534d2141ae704d2edfaaf2..1b2ed63ae6a247a87879ee968f169ea12d1fc701 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts @@ -12,8 +12,8 @@ / { model = "Globalscale Marvell ESPRESSOBin Ultra Board"; - compatible = "globalscale,espressobin-ultra", "marvell,armada3720", - "marvell,armada3710"; + compatible = "globalscale,espressobin-ultra", "globalscale,espressobin", + "marvell,armada3720", "marvell,armada3710"; aliases { /* ethernet1 is WAN port */ diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts index b20c8e7d923b1d5dab99a86d3686c827f26553cb..c76eceabd33e61be64fc6a4b316365fe0591106c 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts @@ -1,66 +1,12 @@ // SPDX-License-Identifier: (GPL-2.0+ OR MIT) -/* - * Device tree for the uDPU board. - * Based on Marvell Armada 3720 development board (DB-88F3720-DDR3) - * Copyright (C) 2016 Marvell - * Copyright (C) 2019 Methode Electronics - * Copyright (C) 2019 Telus - * - * Vladimir Vid - */ /dts-v1/; -#include -#include "armada-372x.dtsi" +#include "armada-3720-uDPU.dtsi" / { model = "Methode uDPU Board"; - compatible = "methode,udpu", "marvell,armada3720"; - - chosen { - stdout-path = "serial0:115200n8"; - }; - - memory@0 { - device_type = "memory"; - reg = <0x00000000 0x00000000 0x00000000 0x20000000>; - }; - - leds { - pinctrl-names = "default"; - compatible = "gpio-leds"; - - power1 { - label = "udpu:green:power"; - gpios = <&gpionb 11 GPIO_ACTIVE_LOW>; - }; - - power2 { - label = "udpu:red:power"; - gpios = <&gpionb 12 GPIO_ACTIVE_LOW>; - }; - - network1 { - label = "udpu:green:network"; - gpios = <&gpionb 13 GPIO_ACTIVE_LOW>; - }; - - network2 { - label = "udpu:red:network"; - gpios = <&gpionb 14 GPIO_ACTIVE_LOW>; - }; - - alarm1 { - label = "udpu:green:alarm"; - gpios = <&gpionb 15 GPIO_ACTIVE_LOW>; - }; - - alarm2 { - label = "udpu:red:alarm"; - gpios = <&gpionb 16 GPIO_ACTIVE_LOW>; - }; - }; + compatible = "methode,udpu", "marvell,armada3720", "marvell,armada3710"; sfp_eth0: sfp-eth0 { compatible = "sff,sfp"; @@ -71,55 +17,6 @@ tx-fault-gpios = <&gpiosb 5 GPIO_ACTIVE_HIGH>; maximum-power-milliwatt = <3000>; }; - - sfp_eth1: sfp-eth1 { - compatible = "sff,sfp"; - i2c-bus = <&i2c1>; - los-gpios = <&gpiosb 7 GPIO_ACTIVE_HIGH>; - mod-def0-gpios = <&gpiosb 8 GPIO_ACTIVE_LOW>; - tx-disable-gpios = <&gpiosb 9 GPIO_ACTIVE_HIGH>; - tx-fault-gpios = <&gpiosb 10 GPIO_ACTIVE_HIGH>; - maximum-power-milliwatt = <3000>; - }; -}; - -&sdhci0 { - status = "okay"; - bus-width = <8>; - mmc-ddr-1_8v; - mmc-hs400-1_8v; - marvell,pad-type = "fixed-1-8v"; - non-removable; - no-sd; - no-sdio; -}; - -&spi0 { - status = "okay"; - pinctrl-names = "default"; - pinctrl-0 = <&spi_quad_pins>; - - flash@0 { - compatible = "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <54000000>; - - partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - partition@0 { - label = "firmware"; - reg = <0x0 0x180000>; - }; - - partition@180000 { - label = "u-boot-env"; - reg = <0x180000 0x10000>; - }; - }; - }; }; &pinctrl_nb { @@ -127,11 +24,6 @@ groups = "i2c1"; function = "gpio"; }; - - i2c2_recovery_pins: i2c2-recovery-pins { - groups = "i2c2"; - function = "gpio"; - }; }; &i2c0 { @@ -144,50 +36,7 @@ sda-gpios = <&gpionb 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; }; -&i2c1 { - status = "okay"; - pinctrl-names = "default", "recovery"; - pinctrl-0 = <&i2c2_pins>; - pinctrl-1 = <&i2c2_recovery_pins>; - /delete-property/mrvl,i2c-fast-mode; - scl-gpios = <&gpionb 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; - sda-gpios = <&gpionb 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; - - nct375@48 { - status = "okay"; - compatible = "ti,tmp75c"; - reg = <0x48>; - }; - - nct375@49 { - status = "okay"; - compatible = "ti,tmp75c"; - reg = <0x49>; - }; -}; - ð0 { phy-mode = "sgmii"; - status = "okay"; - managed = "in-band-status"; - phys = <&comphy1 0>; sfp = <&sfp_eth0>; }; - -ð1 { - phy-mode = "sgmii"; - status = "okay"; - managed = "in-band-status"; - phys = <&comphy0 1>; - sfp = <&sfp_eth1>; -}; - -&usb3 { - status = "okay"; - phys = <&usb2_utmi_otg_phy>; - phy-names = "usb2-utmi-otg-phy"; -}; - -&uart0 { - status = "okay"; -}; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..3f79923376fb28f13a20992e88364587fcae2d95 --- /dev/null +++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Device tree for the uDPU board. + * Based on Marvell Armada 3720 development board (DB-88F3720-DDR3) + * Copyright (C) 2016 Marvell + * Copyright (C) 2019 Methode Electronics + * Copyright (C) 2019 Telus + * + * Vladimir Vid + */ + +/dts-v1/; + +#include +#include "armada-372x.dtsi" + +/ { + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory@0 { + device_type = "memory"; + reg = <0x00000000 0x00000000 0x00000000 0x20000000>; + }; + + leds { + compatible = "gpio-leds"; + + led-power1 { + label = "udpu:green:power"; + gpios = <&gpionb 11 GPIO_ACTIVE_LOW>; + }; + + led-power2 { + label = "udpu:red:power"; + gpios = <&gpionb 12 GPIO_ACTIVE_LOW>; + }; + + led-network1 { + label = "udpu:green:network"; + gpios = <&gpionb 13 GPIO_ACTIVE_LOW>; + }; + + led-network2 { + label = "udpu:red:network"; + gpios = <&gpionb 14 GPIO_ACTIVE_LOW>; + }; + + led-alarm1 { + label = "udpu:green:alarm"; + gpios = <&gpionb 15 GPIO_ACTIVE_LOW>; + }; + + led-alarm2 { + label = "udpu:red:alarm"; + gpios = <&gpionb 16 GPIO_ACTIVE_LOW>; + }; + }; + + sfp_eth1: sfp-eth1 { + compatible = "sff,sfp"; + i2c-bus = <&i2c1>; + los-gpio = <&gpiosb 7 GPIO_ACTIVE_HIGH>; + mod-def0-gpio = <&gpiosb 8 GPIO_ACTIVE_LOW>; + tx-disable-gpio = <&gpiosb 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpio = <&gpiosb 10 GPIO_ACTIVE_HIGH>; + maximum-power-milliwatt = <3000>; + }; +}; + +&sdhci0 { + status = "okay"; + bus-width = <8>; + mmc-ddr-1_8v; + mmc-hs400-1_8v; + marvell,pad-type = "fixed-1-8v"; + non-removable; + no-sd; + no-sdio; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spi_quad_pins>; + + flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <54000000>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "firmware"; + reg = <0x0 0x180000>; + }; + + partition@180000 { + label = "u-boot-env"; + reg = <0x180000 0x10000>; + }; + }; + }; +}; + +&pinctrl_nb { + i2c2_recovery_pins: i2c2-recovery-pins { + groups = "i2c2"; + function = "gpio"; + }; +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default", "recovery"; + pinctrl-0 = <&i2c2_pins>; + pinctrl-1 = <&i2c2_recovery_pins>; + /delete-property/mrvl,i2c-fast-mode; + scl-gpios = <&gpionb 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpionb 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + + temp-sensor@48 { + compatible = "ti,tmp75c"; + reg = <0x48>; + }; + + temp-sensor@49 { + compatible = "ti,tmp75c"; + reg = <0x49>; + }; +}; + +ð0 { + status = "okay"; + managed = "in-band-status"; + phys = <&comphy1 0>; +}; + +ð1 { + phy-mode = "sgmii"; + status = "okay"; + managed = "in-band-status"; + phys = <&comphy0 1>; + sfp = <&sfp_eth1>; +}; + +&usb3 { + status = "okay"; + phys = <&usb2_utmi_otg_phy>; + phy-names = "usb2-utmi-otg-phy"; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile index af362a085a027d4a3d942eb26a980df60ec484b4..0ec90cb3ef289fe08f055ec52989199a7e11568c 100644 --- a/arch/arm64/boot/dts/mediatek/Makefile +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -37,6 +37,7 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kodama-sku32.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-krane-sku0.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-krane-sku176.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-pumpkin.dtb +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8186-evb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-asurada-hayato-r1.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-asurada-spherion-r0.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-evb.dtb diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi index 4797537cb368355128f59a1a9d0362cf959fdb68..e6d7453e56e0e94374ade4b89d1a6d9460c9f295 100644 --- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi @@ -756,7 +756,7 @@ snps,mtl-tx-config = <&mtl_tx_setup>; snps,txpbl = <1>; snps,rxpbl = <1>; - clk_csr = <0>; + snps,clk-csr = <0>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt6795.dtsi b/arch/arm64/boot/dts/mediatek/mt6795.dtsi index d4842b4a4eb7797d88001b3864ee81db17a6c8d0..46f0e54be766418054627ddb143dd56b54560690 100644 --- a/arch/arm64/boot/dts/mediatek/mt6795.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6795.dtsi @@ -230,6 +230,14 @@ reg = <0 0x10200620 0 0x20>; }; + systimer: timer@10200670 { + compatible = "mediatek,mt6795-systimer"; + reg = <0 0x10200670 0 0x10>; + interrupts = ; + clocks = <&system_clk>; + clock-names = "clk13m"; + }; + gic: interrupt-controller@10221000 { compatible = "arm,gic-400"; #interrupt-cells = <3>; diff --git a/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts b/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts index 882277a52b697524e84de2a386ac865bed4655fa..afe37b702eef987fa319cf02f2828fa6be2ae77b 100644 --- a/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts +++ b/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts @@ -115,6 +115,13 @@ status = "okay"; }; +&wifi { + status = "okay"; + pinctrl-names = "default", "dbdc"; + pinctrl-0 = <&wf_2g_5g_pins>; + pinctrl-1 = <&wf_dbdc_pins>; +}; + &pio { uart1_pins: uart1-pins { mux { @@ -129,4 +136,35 @@ groups = "uart2"; }; }; + + wf_2g_5g_pins: wf-2g-5g-pins { + mux { + function = "wifi"; + groups = "wf_2g", "wf_5g"; + }; + conf { + pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4", + "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6", + "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10", + "WF0_TOP_CLK", "WF0_TOP_DATA", "WF1_HB1", + "WF1_HB2", "WF1_HB3", "WF1_HB4", "WF1_HB0", + "WF1_HB5", "WF1_HB6", "WF1_HB7", "WF1_HB8", + "WF1_TOP_CLK", "WF1_TOP_DATA"; + drive-strength = <4>; + }; + }; + + wf_dbdc_pins: wf-dbdc-pins { + mux { + function = "wifi"; + groups = "wf_dbdc"; + }; + conf { + pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4", + "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6", + "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10", + "WF0_TOP_CLK", "WF0_TOP_DATA"; + drive-strength = <4>; + }; + }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi index e3a407d03551fec7c66243ce8f1439d54b514806..72e0d9722e07af74c9ab6cc9ec71b8d0e22abfcf 100644 --- a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi @@ -7,6 +7,7 @@ #include #include #include +#include / { interrupt-parent = <&gic>; @@ -70,6 +71,11 @@ reg = <0 0x43000000 0 0x30000>; no-map; }; + + wmcpu_emi: wmcpu-reserved@4fc00000 { + no-map; + reg = <0 0x4fc00000 0 0x00100000>; + }; }; timer { @@ -222,6 +228,28 @@ #reset-cells = <1>; }; + wed_pcie: wed-pcie@10003000 { + compatible = "mediatek,mt7986-wed-pcie", + "syscon"; + reg = <0 0x10003000 0 0x10>; + }; + + wed0: wed@15010000 { + compatible = "mediatek,mt7986-wed", + "syscon"; + reg = <0 0x15010000 0 0x1000>; + interrupt-parent = <&gic>; + interrupts = ; + }; + + wed1: wed@15011000 { + compatible = "mediatek,mt7986-wed", + "syscon"; + reg = <0 0x15011000 0 0x1000>; + interrupt-parent = <&gic>; + interrupts = ; + }; + eth: ethernet@15100000 { compatible = "mediatek,mt7986-eth"; reg = <0 0x15100000 0 0x80000>; @@ -256,11 +284,30 @@ <&apmixedsys CLK_APMIXED_SGMPLL>; mediatek,ethsys = <ðsys>; mediatek,sgmiisys = <&sgmiisys0>, <&sgmiisys1>; + mediatek,wed-pcie = <&wed_pcie>; + mediatek,wed = <&wed0>, <&wed1>; #reset-cells = <1>; #address-cells = <1>; #size-cells = <0>; status = "disabled"; }; + + wifi: wifi@18000000 { + compatible = "mediatek,mt7986-wmac"; + resets = <&watchdog MT7986_TOPRGU_CONSYS_SW_RST>; + reset-names = "consys"; + clocks = <&topckgen CLK_TOP_CONN_MCUSYS_SEL>, + <&topckgen CLK_TOP_AP2CNN_HOST_SEL>; + clock-names = "mcu", "ap2conn"; + reg = <0 0x18000000 0 0x1000000>, + <0 0x10003000 0 0x1000>, + <0 0x11d10000 0 0x1000>; + interrupts = , + , + , + ; + memory-region = <&wmcpu_emi>; + }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts b/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts index 0f49d5764ff3a4e74933365b9e556e55ac8ba4bb..3443013b5971eac3f1bbeefcc83395d79ec0a172 100644 --- a/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts +++ b/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts @@ -98,3 +98,43 @@ }; }; }; + +&wifi { + status = "okay"; + pinctrl-names = "default", "dbdc"; + pinctrl-0 = <&wf_2g_5g_pins>; + pinctrl-1 = <&wf_dbdc_pins>; +}; + +&pio { + wf_2g_5g_pins: wf-2g-5g-pins { + mux { + function = "wifi"; + groups = "wf_2g", "wf_5g"; + }; + conf { + pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4", + "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6", + "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10", + "WF0_TOP_CLK", "WF0_TOP_DATA", "WF1_HB1", + "WF1_HB2", "WF1_HB3", "WF1_HB4", "WF1_HB0", + "WF1_HB5", "WF1_HB6", "WF1_HB7", "WF1_HB8", + "WF1_TOP_CLK", "WF1_TOP_DATA"; + drive-strength = <4>; + }; + }; + + wf_dbdc_pins: wf-dbdc-pins { + mux { + function = "wifi"; + groups = "wf_dbdc"; + }; + conf { + pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4", + "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6", + "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10", + "WF0_TOP_CLK", "WF0_TOP_DATA"; + drive-strength = <4>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8167.dtsi b/arch/arm64/boot/dts/mediatek/mt8167.dtsi index 54655f2feb04d722e111787e9fe701ac6f4b82f1..fbe1a1128cc6ab45094cb8b9fb7246c08272cf71 100644 --- a/arch/arm64/boot/dts/mediatek/mt8167.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8167.dtsi @@ -36,9 +36,8 @@ }; scpsys: syscon@10006000 { - compatible = "syscon", "simple-mfd"; + compatible = "mediatek,mt8167-scpsys", "syscon", "simple-mfd"; reg = <0 0x10006000 0 0x1000>; - #power-domain-cells = <1>; spm: power-controller { compatible = "mediatek,mt8167-power-controller"; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 6d9513c1f5bfb2a85437c7ee1123f7b32bfe6c98..7640b5158ff9d3b24bfed2dabb16bbb475f60100 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -444,9 +444,8 @@ }; scpsys: syscon@10006000 { - compatible = "syscon", "simple-mfd"; + compatible = "mediatek,mt8173-scpsys", "syscon", "simple-mfd"; reg = <0 0x10006000 0 0x1000>; - #power-domain-cells = <1>; /* System Power Manager */ spm: power-controller { @@ -1468,7 +1467,7 @@ clock-names = "venc_sel"; assigned-clocks = <&topckgen CLK_TOP_VENC_SEL>; assigned-clock-parents = <&topckgen CLK_TOP_VCODECPLL>; - power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC>; + power-domains = <&spm MT8173_POWER_DOMAIN_VENC>; }; jpegdec: jpegdec@18004000 { @@ -1519,7 +1518,7 @@ assigned-clocks = <&topckgen CLK_TOP_VENC_LT_SEL>; assigned-clock-parents = <&topckgen CLK_TOP_VCODECPLL_370P5>; - power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC_LT>; + power-domains = <&spm MT8173_POWER_DOMAIN_VENC_LT>; }; }; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts b/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts index 530e0c9ce0c9eb9052fcaa5ea682536f0b6c0a4f..a1d01639df30a450c77165cf4b7c2789dcb3ac26 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts +++ b/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts @@ -7,6 +7,7 @@ /dts-v1/; #include +#include #include "mt8183.dtsi" #include "mt6358.dtsi" @@ -122,6 +123,18 @@ clock-frequency = <100000>; }; +&keyboard { + pinctrl-names = "default"; + pinctrl-0 = <&keyboard_pins>; + status = "okay"; + linux,keymap = ; + keypad,num-rows = <2>; + keypad,num-columns = <1>; + debounce-delay-ms = <32>; + mediatek,keys-per-group = <2>; +}; + &mmc0 { status = "okay"; pinctrl-names = "default", "state_uhs"; @@ -226,6 +239,14 @@ }; }; + keyboard_pins: keyboard { + pins_keyboard { + pinmux = , + , + ; + }; + }; + mmc0_pins_default: mmc0-pins-default { pins_cmd_dat { pinmux = , diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi index 9d32871973a29e75b720891451a82d3684dc5bea..a70b669c49baa3ccc3314320c0c4d2c8841f5c42 100644 --- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi @@ -761,9 +761,8 @@ }; scpsys: syscon@10006000 { - compatible = "syscon", "simple-mfd"; + compatible = "mediatek,mt8183-scpsys", "syscon", "simple-mfd"; reg = <0 0x10006000 0 0x1000>; - #power-domain-cells = <1>; /* System Power Manager */ spm: power-controller { @@ -943,6 +942,15 @@ clock-names = "spi", "wrap"; }; + keyboard: keyboard@10010000 { + compatible = "mediatek,mt6779-keypad"; + reg = <0 0x10010000 0 0x1000>; + interrupts = ; + clocks = <&clk26m>; + clock-names = "kpd"; + status = "disabled"; + }; + scp: scp@10500000 { compatible = "mediatek,mt8183-scp"; reg = <0 0x10500000 0 0x80000>, @@ -1691,6 +1699,60 @@ mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>; }; + mdp3-rdma0@14001000 { + compatible = "mediatek,mt8183-mdp3-rdma"; + reg = <0 0x14001000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x1000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_RDMA0>, + <&mmsys CLK_MM_MDP_RSZ1>; + iommus = <&iommu M4U_PORT_MDP_RDMA0>; + mboxes = <&gce 20 CMDQ_THR_PRIO_LOWEST 0>, + <&gce 21 CMDQ_THR_PRIO_LOWEST 0>; + }; + + mdp3-rsz0@14003000 { + compatible = "mediatek,mt8183-mdp3-rsz"; + reg = <0 0x14003000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x3000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_RSZ0>; + }; + + mdp3-rsz1@14004000 { + compatible = "mediatek,mt8183-mdp3-rsz"; + reg = <0 0x14004000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x4000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_RSZ1>; + }; + + mdp3-wrot0@14005000 { + compatible = "mediatek,mt8183-mdp3-wrot"; + reg = <0 0x14005000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x5000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_WROT0>; + iommus = <&iommu M4U_PORT_MDP_WROT0>; + }; + + mdp3-wdma@14006000 { + compatible = "mediatek,mt8183-mdp3-wdma"; + reg = <0 0x14006000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x6000 0x1000>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_MDP_WDMA0>; + iommus = <&iommu M4U_PORT_MDP_WDMA0>; + }; + ovl0: ovl@14008000 { compatible = "mediatek,mt8183-disp-ovl"; reg = <0 0x14008000 0 0x1000>; @@ -1834,6 +1896,15 @@ power-domains = <&spm MT8183_POWER_DOMAIN_DISP>; }; + mdp3-ccorr@1401c000 { + compatible = "mediatek,mt8183-mdp3-ccorr"; + reg = <0 0x1401c000 0 0x1000>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0xc000 0x1000>; + mediatek,gce-events = , + ; + clocks = <&mmsys CLK_MM_MDP_CCORR>; + }; + imgsys: syscon@15020000 { compatible = "mediatek,mt8183-imgsys", "syscon"; reg = <0 0x15020000 0 0x1000>; diff --git a/arch/arm64/boot/dts/mediatek/mt8186-evb.dts b/arch/arm64/boot/dts/mediatek/mt8186-evb.dts new file mode 100644 index 0000000000000000000000000000000000000000..ed74a3617c13521eacb103a3bd250839cbb5539e --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt8186-evb.dts @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Copyright (C) 2022 MediaTek Inc. + */ +/dts-v1/; +#include "mt8186.dtsi" + +/ { + model = "MediaTek MT8186 evaluation board"; + compatible = "mediatek,mt8186-evb", "mediatek,mt8186"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:921600n8"; + }; + + memory@40000000 { + device_type = "memory"; + reg = <0 0x40000000 0 0x80000000>; + }; +}; + +&i2c0 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; +}; + +&i2c1 { + status = "okay"; + + clock-frequency = <400000>; + i2c-scl-internal-delay-ns = <8000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; +}; + +&i2c2 { + status = "okay"; + + clock-frequency = <400000>; + i2c-scl-internal-delay-ns = <10000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; +}; + +&i2c3 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_pins>; +}; + +&i2c4 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins>; +}; + +&i2c5 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c5_pins>; +}; + +&i2c6 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c6_pins>; +}; + +&i2c7 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c7_pins>; +}; + +&i2c8 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_pins>; +}; + +&i2c9 { + status = "okay"; + + clock-frequency = <400000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c9_pins>; +}; + +&pio { + i2c0_pins: i2c0-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c1_pins: i2c1-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c2_pins: i2c2-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c3_pins: i2c3-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c4_pins: i2c4-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c5_pins: i2c5-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c6_pins: i2c6-default-pins { + pins-bus { + pinmux = , + ; + bias-pull-up = ; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c7_pins: i2c7-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c8_pins: i2c8-default-pins { + pins-bus { + pinmux = , + ; + bias-disable; + drive-strength-microamp = <1000>; + input-enable; + }; + }; + + i2c9_pins: i2c9-default-pins { + pins-bus { + pinmux = , + ; + bias-pull-up = ; + drive-strength-microamp = <1000>; + input-enable; + }; + }; +}; + +&u3phy0 { + status = "okay"; +}; + +&u3phy1 { + status = "okay"; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8186.dtsi b/arch/arm64/boot/dts/mediatek/mt8186.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..64693c17af9ec0545d02561322d97aa32573ef7c --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt8186.dtsi @@ -0,0 +1,819 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Copyright (C) 2022 MediaTek Inc. + * Author: Allen-KH Cheng + */ +/dts-v1/; +#include +#include +#include +#include +#include +#include +#include + +/ { + compatible = "mediatek,mt8186"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + + core4 { + cpu = <&cpu4>; + }; + + core5 { + cpu = <&cpu5>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu6>; + }; + + core1 { + cpu = <&cpu7>; + }; + }; + }; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x000>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu1: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x100>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu2: cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x200>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu3: cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x300>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu4: cpu@400 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x400>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu5: cpu@500 { + device_type = "cpu"; + compatible = "arm,cortex-a55"; + reg = <0x500>; + enable-method = "psci"; + clock-frequency = <2000000000>; + capacity-dmips-mhz = <382>; + cpu-idle-states = <&cpu_off_l &cluster_off_l>; + next-level-cache = <&l2_0>; + #cooling-cells = <2>; + }; + + cpu6: cpu@600 { + device_type = "cpu"; + compatible = "arm,cortex-a76"; + reg = <0x600>; + enable-method = "psci"; + clock-frequency = <2050000000>; + capacity-dmips-mhz = <1024>; + cpu-idle-states = <&cpu_off_b &cluster_off_b>; + next-level-cache = <&l2_1>; + #cooling-cells = <2>; + }; + + cpu7: cpu@700 { + device_type = "cpu"; + compatible = "arm,cortex-a76"; + reg = <0x700>; + enable-method = "psci"; + clock-frequency = <2050000000>; + capacity-dmips-mhz = <1024>; + cpu-idle-states = <&cpu_off_b &cluster_off_b>; + next-level-cache = <&l2_1>; + #cooling-cells = <2>; + }; + + idle-states { + entry-method = "psci"; + + cpu_off_l: cpu-off-l { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x00010001>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <1600>; + }; + + cpu_off_b: cpu-off-b { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x00010001>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <1400>; + }; + + cluster_off_l: cluster-off-l { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x01010001>; + local-timer-stop; + entry-latency-us = <100>; + exit-latency-us = <250>; + min-residency-us = <2100>; + }; + + cluster_off_b: cluster-off-b { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x01010001>; + local-timer-stop; + entry-latency-us = <100>; + exit-latency-us = <250>; + min-residency-us = <1900>; + }; + }; + + l2_0: l2-cache0 { + compatible = "cache"; + next-level-cache = <&l3_0>; + }; + + l2_1: l2-cache1 { + compatible = "cache"; + next-level-cache = <&l3_0>; + }; + + l3_0: l3-cache { + compatible = "cache"; + }; + }; + + clk13m: oscillator-13m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <13000000>; + clock-output-names = "clk13m"; + }; + + clk26m: oscillator-26m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + clock-output-names = "clk26m"; + }; + + clk32k: oscillator-32k { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "clk32k"; + }; + + pmu-a55 { + compatible = "arm,cortex-a55-pmu"; + interrupt-parent = <&gic>; + interrupts = ; + }; + + pmu-a76 { + compatible = "arm,cortex-a76-pmu"; + interrupt-parent = <&gic>; + interrupts = ; + }; + + psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&gic>; + interrupts = , + , + , + ; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + + gic: interrupt-controller@c000000 { + compatible = "arm,gic-v3"; + #interrupt-cells = <4>; + #redistributor-regions = <1>; + interrupt-parent = <&gic>; + interrupt-controller; + reg = <0 0x0c000000 0 0x40000>, + <0 0x0c040000 0 0x200000>; + interrupts = ; + + ppi-partitions { + ppi_cluster0: interrupt-partition-0 { + affinity = <&cpu0 &cpu1 &cpu2 &cpu3 &cpu4 &cpu5>; + }; + + ppi_cluster1: interrupt-partition-1 { + affinity = <&cpu6 &cpu7>; + }; + }; + }; + + mcusys: syscon@c53a000 { + compatible = "mediatek,mt8186-mcusys", "syscon"; + reg = <0 0xc53a000 0 0x1000>; + #clock-cells = <1>; + }; + + topckgen: syscon@10000000 { + compatible = "mediatek,mt8186-topckgen", "syscon"; + reg = <0 0x10000000 0 0x1000>; + #clock-cells = <1>; + }; + + infracfg_ao: syscon@10001000 { + compatible = "mediatek,mt8186-infracfg_ao", "syscon"; + reg = <0 0x10001000 0 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + pericfg: syscon@10003000 { + compatible = "mediatek,mt8186-pericfg", "syscon"; + reg = <0 0x10003000 0 0x1000>; + }; + + pio: pinctrl@10005000 { + compatible = "mediatek,mt8186-pinctrl"; + reg = <0 0x10005000 0 0x1000>, + <0 0x10002000 0 0x0200>, + <0 0x10002200 0 0x0200>, + <0 0x10002400 0 0x0200>, + <0 0x10002600 0 0x0200>, + <0 0x10002a00 0 0x0200>, + <0 0x10002c00 0 0x0200>, + <0 0x1000b000 0 0x1000>; + reg-names = "iocfg0", "iocfg_lt", "iocfg_lm", "iocfg_lb", + "iocfg_bl", "iocfg_rb", "iocfg_rt", "eint"; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pio 0 0 185>; + interrupt-controller; + interrupts = ; + #interrupt-cells = <2>; + }; + + watchdog: watchdog@10007000 { + compatible = "mediatek,mt8186-wdt", + "mediatek,mt6589-wdt"; + mediatek,disable-extrst; + reg = <0 0x10007000 0 0x1000>; + #reset-cells = <1>; + }; + + apmixedsys: syscon@1000c000 { + compatible = "mediatek,mt8186-apmixedsys", "syscon"; + reg = <0 0x1000c000 0 0x1000>; + #clock-cells = <1>; + }; + + pwrap: pwrap@1000d000 { + compatible = "mediatek,mt8186-pwrap", "syscon"; + reg = <0 0x1000d000 0 0x1000>; + reg-names = "pwrap"; + interrupts = ; + clocks = <&infracfg_ao CLK_INFRA_AO_PMIC_AP>, + <&infracfg_ao CLK_INFRA_AO_PMIC_TMR>; + clock-names = "spi", "wrap"; + }; + + systimer: timer@10017000 { + compatible = "mediatek,mt8186-timer", + "mediatek,mt6765-timer"; + reg = <0 0x10017000 0 0x1000>; + interrupts = ; + clocks = <&clk13m>; + }; + + scp: scp@10500000 { + compatible = "mediatek,mt8186-scp"; + reg = <0 0x10500000 0 0x40000>, + <0 0x105c0000 0 0x19080>; + reg-names = "sram", "cfg"; + interrupts = ; + }; + + nor_flash: spi@11000000 { + compatible = "mediatek,mt8186-nor"; + reg = <0 0x11000000 0 0x1000>; + clocks = <&topckgen CLK_TOP_SPINOR>, + <&infracfg_ao CLK_INFRA_AO_SPINOR>, + <&infracfg_ao CLK_INFRA_AO_FLASHIF_133M>, + <&infracfg_ao CLK_INFRA_AO_FLASHIF_66M>; + clock-names = "spi", "sf", "axi", "axi_s"; + assigned-clocks = <&topckgen CLK_TOP_SPINOR>; + assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL_D3_D8>; + interrupts = ; + status = "disabled"; + }; + + auxadc: adc@11001000 { + compatible = "mediatek,mt8186-auxadc", "mediatek,mt8173-auxadc"; + reg = <0 0x11001000 0 0x1000>; + #io-channel-cells = <1>; + clocks = <&infracfg_ao CLK_INFRA_AO_AUXADC>; + clock-names = "main"; + }; + + uart0: serial@11002000 { + compatible = "mediatek,mt8186-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11002000 0 0x1000>; + interrupts = ; + clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART0>; + clock-names = "baud", "bus"; + status = "disabled"; + }; + + uart1: serial@11003000 { + compatible = "mediatek,mt8186-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11003000 0 0x1000>; + interrupts = ; + clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART1>; + clock-names = "baud", "bus"; + status = "disabled"; + }; + + i2c0: i2c@11007000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11007000 0 0x1000>, + <0 0x10200100 0 0x100>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C0>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@11008000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11008000 0 0x1000>, + <0 0x10200200 0 0x100>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C1>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@11009000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11009000 0 0x1000>, + <0 0x10200300 0 0x180>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C2>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c@1100f000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x1100f000 0 0x1000>, + <0 0x10200480 0 0x100>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C3>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@11011000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11011000 0 0x1000>, + <0 0x10200580 0 0x180>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C4>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@11016000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11016000 0 0x1000>, + <0 0x10200700 0 0x100>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C5>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c6: i2c@1100d000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x1100d000 0 0x1000>, + <0 0x10200800 0 0x100>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C6>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c7: i2c@11004000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11004000 0 0x1000>, + <0 0x10200900 0 0x180>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C7>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c8: i2c@11005000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11005000 0 0x1000>, + <0 0x10200A80 0 0x180>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C8>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi0: spi@1100a000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x1100a000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI0>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + pwm0: pwm@1100e000 { + compatible = "mediatek,mt8186-disp-pwm", "mediatek,mt8183-disp-pwm"; + reg = <0 0x1100e000 0 0x1000>; + interrupts = ; + #pwm-cells = <2>; + clocks = <&topckgen CLK_TOP_DISP_PWM>, + <&infracfg_ao CLK_INFRA_AO_DISP_PWM>; + clock-names = "main", "mm"; + status = "disabled"; + }; + + spi1: spi@11010000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x11010000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI1>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + spi2: spi@11012000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x11012000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI2>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + spi3: spi@11013000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x11013000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI3>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + spi4: spi@11014000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x11014000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI4>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + spi5: spi@11015000 { + compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0 0x11015000 0 0x1000>; + interrupts = ; + clocks = <&topckgen CLK_TOP_MAINPLL_D5>, + <&topckgen CLK_TOP_SPI>, + <&infracfg_ao CLK_INFRA_AO_SPI5>; + clock-names = "parent-clk", "sel-clk", "spi-clk"; + status = "disabled"; + }; + + imp_iic_wrap: clock-controller@11017000 { + compatible = "mediatek,mt8186-imp_iic_wrap"; + reg = <0 0x11017000 0 0x1000>; + #clock-cells = <1>; + }; + + uart2: serial@11018000 { + compatible = "mediatek,mt8186-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11018000 0 0x1000>; + interrupts = ; + clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART2>; + clock-names = "baud", "bus"; + status = "disabled"; + }; + + i2c9: i2c@11019000 { + compatible = "mediatek,mt8186-i2c"; + reg = <0 0x11019000 0 0x1000>, + <0 0x10200c00 0 0x180>; + interrupts = ; + clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C9>, + <&infracfg_ao CLK_INFRA_AO_AP_DMA>; + clock-names = "main", "dma"; + clock-div = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + mmc0: mmc@11230000 { + compatible = "mediatek,mt8186-mmc", + "mediatek,mt8183-mmc"; + reg = <0 0x11230000 0 0x1000>, + <0 0x11cd0000 0 0x1000>; + clocks = <&topckgen CLK_TOP_MSDC50_0>, + <&infracfg_ao CLK_INFRA_AO_MSDC0>, + <&infracfg_ao CLK_INFRA_AO_MSDC0_SRC>; + clock-names = "source", "hclk", "source_cg"; + interrupts = ; + assigned-clocks = <&topckgen CLK_TOP_MSDC50_0>; + assigned-clock-parents = <&apmixedsys CLK_APMIXED_MSDCPLL>; + status = "disabled"; + }; + + mmc1: mmc@11240000 { + compatible = "mediatek,mt8186-mmc", + "mediatek,mt8183-mmc"; + reg = <0 0x11240000 0 0x1000>, + <0 0x11c90000 0 0x1000>; + clocks = <&topckgen CLK_TOP_MSDC30_1>, + <&infracfg_ao CLK_INFRA_AO_MSDC1>, + <&infracfg_ao CLK_INFRA_AO_MSDC1_SRC>; + clock-names = "source", "hclk", "source_cg"; + interrupts = ; + assigned-clocks = <&topckgen CLK_TOP_MSDC30_1>; + assigned-clock-parents = <&topckgen CLK_TOP_MSDCPLL_D2>; + status = "disabled"; + }; + + u3phy0: t-phy@11c80000 { + compatible = "mediatek,mt8186-tphy", + "mediatek,generic-tphy-v2"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x0 0x11c80000 0x1000>; + status = "disabled"; + + u2port1: usb-phy@0 { + reg = <0x0 0x700>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + }; + + u3port1: usb-phy@700 { + reg = <0x700 0x900>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + }; + }; + + u3phy1: t-phy@11ca0000 { + compatible = "mediatek,mt8186-tphy", + "mediatek,generic-tphy-v2"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x0 0x11ca0000 0x1000>; + status = "disabled"; + + u2port0: usb-phy@0 { + reg = <0x0 0x700>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + mediatek,discth = <0x8>; + }; + }; + + efuse: efuse@11cb0000 { + compatible = "mediatek,mt8186-efuse", "mediatek,efuse"; + reg = <0 0x11cb0000 0 0x1000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + mipi_tx0: dsi-phy@11cc0000 { + compatible = "mediatek,mt8183-mipi-tx"; + reg = <0 0x11cc0000 0 0x1000>; + clocks = <&clk26m>; + #clock-cells = <0>; + #phy-cells = <0>; + clock-output-names = "mipi_tx0_pll"; + status = "disabled"; + }; + + mfgsys: clock-controller@13000000 { + compatible = "mediatek,mt8186-mfgsys"; + reg = <0 0x13000000 0 0x1000>; + #clock-cells = <1>; + }; + + mmsys: syscon@14000000 { + compatible = "mediatek,mt8186-mmsys", "syscon"; + reg = <0 0x14000000 0 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + wpesys: clock-controller@14020000 { + compatible = "mediatek,mt8186-wpesys"; + reg = <0 0x14020000 0 0x1000>; + #clock-cells = <1>; + }; + + imgsys1: clock-controller@15020000 { + compatible = "mediatek,mt8186-imgsys1"; + reg = <0 0x15020000 0 0x1000>; + #clock-cells = <1>; + }; + + imgsys2: clock-controller@15820000 { + compatible = "mediatek,mt8186-imgsys2"; + reg = <0 0x15820000 0 0x1000>; + #clock-cells = <1>; + }; + + vdecsys: clock-controller@1602f000 { + compatible = "mediatek,mt8186-vdecsys"; + reg = <0 0x1602f000 0 0x1000>; + #clock-cells = <1>; + }; + + vencsys: clock-controller@17000000 { + compatible = "mediatek,mt8186-vencsys"; + reg = <0 0x17000000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys: clock-controller@1a000000 { + compatible = "mediatek,mt8186-camsys"; + reg = <0 0x1a000000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys_rawa: clock-controller@1a04f000 { + compatible = "mediatek,mt8186-camsys_rawa"; + reg = <0 0x1a04f000 0 0x1000>; + #clock-cells = <1>; + }; + + camsys_rawb: clock-controller@1a06f000 { + compatible = "mediatek,mt8186-camsys_rawb"; + reg = <0 0x1a06f000 0 0x1000>; + #clock-cells = <1>; + }; + + mdpsys: clock-controller@1b000000 { + compatible = "mediatek,mt8186-mdpsys"; + reg = <0 0x1b000000 0 0x1000>; + #clock-cells = <1>; + }; + + ipesys: clock-controller@1c000000 { + compatible = "mediatek,mt8186-ipesys"; + reg = <0 0x1c000000 0 0x1000>; + #clock-cells = <1>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8192.dtsi b/arch/arm64/boot/dts/mediatek/mt8192.dtsi index cbae5a5ee4a0b93278aa4747a3fd9b746fd83c45..6b20376191a75a6eef4e23c1caf7a8321f3f7c3d 100644 --- a/arch/arm64/boot/dts/mediatek/mt8192.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8192.dtsi @@ -6,12 +6,14 @@ /dts-v1/; #include +#include #include #include #include #include #include #include +#include / { compatible = "mediatek,mt8192"; @@ -19,6 +21,14 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + ovl0 = &ovl0; + ovl-2l0 = &ovl_2l0; + ovl-2l2 = &ovl_2l2; + rdma0 = &rdma0; + rdma4 = &rdma4; + }; + clk26m: oscillator0 { compatible = "fixed-clock"; #clock-cells = <0>; @@ -306,9 +316,8 @@ }; scpsys: syscon@10006000 { - compatible = "syscon", "simple-mfd"; + compatible = "mediatek,mt8192-scpsys", "syscon", "simple-mfd"; reg = <0 0x10006000 0 0x1000>; - #power-domain-cells = <1>; /* System Power Manager */ spm: power-controller { @@ -553,6 +562,15 @@ assigned-clock-parents = <&topckgen CLK_TOP_OSC_D10>; }; + gce: mailbox@10228000 { + compatible = "mediatek,mt8192-gce"; + reg = <0 0x10228000 0 0x4000>; + interrupts = ; + #mbox-cells = <2>; + clocks = <&infracfg CLK_INFRA_GCE>; + clock-names = "gce"; + }; + scp_adsp: clock-controller@10720000 { compatible = "mediatek,mt8192-scp_adsp"; reg = <0 0x10720000 0 0x1000>; @@ -599,6 +617,17 @@ status = "disabled"; }; + pwm0: pwm@1100e000 { + compatible = "mediatek,mt8183-disp-pwm"; + reg = <0 0x1100e000 0 0x1000>; + interrupts = ; + #pwm-cells = <2>; + clocks = <&topckgen CLK_TOP_DISP_PWM_SEL>, + <&infracfg CLK_INFRA_DISP_PWM>; + clock-names = "main", "mm"; + status = "disabled"; + }; + spi1: spi@11010000 { compatible = "mediatek,mt8192-spi", "mediatek,mt6765-spi"; @@ -724,9 +753,12 @@ assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL_D5_D4>, <&topckgen CLK_TOP_UNIVPLL_D5_D4>; clocks = <&infracfg CLK_INFRA_SSUSB>, - <&infracfg CLK_INFRA_SSUSB_XHCI>, - <&apmixedsys CLK_APMIXED_USBPLL>; - clock-names = "sys_ck", "xhci_ck", "ref_ck"; + <&apmixedsys CLK_APMIXED_USBPLL>, + <&clk26m>, + <&clk26m>, + <&infracfg CLK_INFRA_SSUSB_XHCI>; + clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck", + "xhci_ck"; wakeup-source; mediatek,syscon-wakeup = <&pericfg 0x420 102>; status = "disabled"; @@ -1084,6 +1116,16 @@ }; }; + mipi_tx0: dsi-phy@11e50000 { + compatible = "mediatek,mt8183-mipi-tx"; + reg = <0 0x11e50000 0 0x1000>; + clocks = <&apmixedsys CLK_APMIXED_MIPID26M>; + #clock-cells = <0>; + #phy-cells = <0>; + clock-output-names = "mipi_tx0_pll"; + status = "disabled"; + }; + i2c0: i2c@11f00000 { compatible = "mediatek,mt8192-i2c"; reg = <0 0x11f00000 0 0x1000>, @@ -1166,6 +1208,20 @@ compatible = "mediatek,mt8192-mmsys", "syscon"; reg = <0 0x14000000 0 0x1000>; #clock-cells = <1>; + #reset-cells = <1>; + mboxes = <&gce 0 CMDQ_THR_PRIO_HIGHEST>, + <&gce 1 CMDQ_THR_PRIO_HIGHEST>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>; + }; + + mutex: mutex@14001000 { + compatible = "mediatek,mt8192-disp-mutex"; + reg = <0 0x14001000 0 0x1000>; + interrupts = ; + clocks = <&mmsys CLK_MM_DISP_MUTEX0>; + mediatek,gce-events = , + ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; }; smi_common: smi@14002000 { @@ -1199,6 +1255,140 @@ power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; }; + ovl0: ovl@14005000 { + compatible = "mediatek,mt8192-disp-ovl"; + reg = <0 0x14005000 0 0x1000>; + interrupts = ; + clocks = <&mmsys CLK_MM_DISP_OVL0>; + iommus = <&iommu0 M4U_PORT_L0_OVL_RDMA0>, + <&iommu0 M4U_PORT_L0_OVL_RDMA0_HDR>; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x5000 0x1000>; + }; + + ovl_2l0: ovl@14006000 { + compatible = "mediatek,mt8192-disp-ovl-2l"; + reg = <0 0x14006000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_OVL0_2L>; + iommus = <&iommu0 M4U_PORT_L1_OVL_2L_RDMA0>, + <&iommu0 M4U_PORT_L1_OVL_2L_RDMA0_HDR>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x6000 0x1000>; + }; + + rdma0: rdma@14007000 { + compatible = "mediatek,mt8192-disp-rdma", + "mediatek,mt8183-disp-rdma"; + reg = <0 0x14007000 0 0x1000>; + interrupts = ; + clocks = <&mmsys CLK_MM_DISP_RDMA0>; + iommus = <&iommu0 M4U_PORT_L0_DISP_RDMA0>; + mediatek,rdma-fifo-size = <5120>; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x7000 0x1000>; + }; + + color0: color@14009000 { + compatible = "mediatek,mt8192-disp-color", + "mediatek,mt8173-disp-color"; + reg = <0 0x14009000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_COLOR0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x9000 0x1000>; + }; + + ccorr0: ccorr@1400a000 { + compatible = "mediatek,mt8192-disp-ccorr"; + reg = <0 0x1400a000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_CCORR0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xa000 0x1000>; + }; + + aal0: aal@1400b000 { + compatible = "mediatek,mt8192-disp-aal", + "mediatek,mt8183-disp-aal"; + reg = <0 0x1400b000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_AAL0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xb000 0x1000>; + }; + + gamma0: gamma@1400c000 { + compatible = "mediatek,mt8192-disp-gamma", + "mediatek,mt8183-disp-gamma"; + reg = <0 0x1400c000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_GAMMA0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xc000 0x1000>; + }; + + postmask0: postmask@1400d000 { + compatible = "mediatek,mt8192-disp-postmask"; + reg = <0 0x1400d000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_POSTMASK0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xd000 0x1000>; + }; + + dither0: dither@1400e000 { + compatible = "mediatek,mt8192-disp-dither", + "mediatek,mt8183-disp-dither"; + reg = <0 0x1400e000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_DITHER0>; + mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xe000 0x1000>; + }; + + dsi0: dsi@14010000 { + compatible = "mediatek,mt8183-dsi"; + reg = <0 0x14010000 0 0x1000>; + interrupts = ; + clocks = <&mmsys CLK_MM_DSI0>, + <&mmsys CLK_MM_DSI_DSI0>, + <&mipi_tx0>; + clock-names = "engine", "digital", "hs"; + phys = <&mipi_tx0>; + phy-names = "dphy"; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + resets = <&mmsys MT8192_MMSYS_SW0_RST_B_DISP_DSI0>; + status = "disabled"; + + port { + dsi_out: endpoint { }; + }; + }; + + ovl_2l2: ovl@14014000 { + compatible = "mediatek,mt8192-disp-ovl-2l"; + reg = <0 0x14014000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_OVL2_2L>; + iommus = <&iommu0 M4U_PORT_L1_OVL_2L_RDMA2>, + <&iommu0 M4U_PORT_L1_OVL_2L_RDMA2_HDR>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x4000 0x1000>; + }; + + rdma4: rdma@14015000 { + compatible = "mediatek,mt8192-disp-rdma", + "mediatek,mt8183-disp-rdma"; + reg = <0 0x14015000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8192_POWER_DOMAIN_DISP>; + clocks = <&mmsys CLK_MM_DISP_RDMA4>; + iommus = <&iommu0 M4U_PORT_L1_DISP_RDMA4>; + mediatek,rdma-fifo-size = <2048>; + mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x5000 0x1000>; + }; + dpi0: dpi@14016000 { compatible = "mediatek,mt8192-dpi"; reg = <0 0x14016000 0 0x1000>; diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi index fcc600674339a4b969885791d219bf531f34394b..9b62e161db2618cfc049b378549c0bfb783f43e6 100644 --- a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi @@ -4,6 +4,7 @@ */ #include +#include #include "mt8195.dtsi" #include "mt6359.dtsi" @@ -17,6 +18,7 @@ i2c5 = &i2c5; i2c7 = &i2c7; mmc0 = &mmc0; + mmc1 = &mmc1; serial0 = &uart0; }; @@ -104,6 +106,18 @@ enable-active-high; regulator-always-on; }; + + reserved_memory: reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + scp_mem: memory@50000000 { + compatible = "shared-dma-pool"; + reg = <0 0x50000000 0 0x2900000>; + no-map; + }; + }; }; &i2c0 { @@ -121,6 +135,16 @@ i2c-scl-internal-delay-ns = <12500>; pinctrl-names = "default"; pinctrl-0 = <&i2c1_pins>; + + trackpad@15 { + compatible = "elan,ekth3000"; + reg = <0x15>; + interrupts-extended = <&pio 6 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&trackpad_pins>; + vcc-supply = <&pp3300_s3>; + wakeup-source; + }; }; &i2c2 { @@ -137,6 +161,14 @@ clock-frequency = <400000>; pinctrl-names = "default"; pinctrl-0 = <&i2c3_pins>; + + tpm@50 { + compatible = "google,cr50"; + reg = <0x50>; + interrupts-extended = <&pio 88 IRQ_TYPE_EDGE_FALLING>; + pinctrl-names = "default"; + pinctrl-0 = <&cr50_int>; + }; }; &i2c4 { @@ -207,6 +239,24 @@ vqmmc-supply = <&mt6359_vufs_ldo_reg>; }; +&mmc1 { + status = "okay"; + + bus-width = <4>; + cap-sd-highspeed; + cd-gpios = <&pio 54 GPIO_ACTIVE_LOW>; + max-frequency = <200000000>; + no-mmc; + no-sdio; + pinctrl-names = "default", "state_uhs"; + pinctrl-0 = <&mmc1_pins_default>, <&mmc1_pins_detect>; + pinctrl-1 = <&mmc1_pins_default>; + sd-uhs-sdr50; + sd-uhs-sdr104; + vmmc-supply = <&mt_pmic_vmch_ldo_reg>; + vqmmc-supply = <&mt_pmic_vmc_ldo_reg>; +}; + /* for CPU-L */ &mt6359_vcore_buck_reg { regulator-always-on; @@ -414,6 +464,21 @@ "AP_SPI_FLASH_MOSI", "AP_SPI_FLASH_MISO"; + cr50_int: cr50-irq-default-pins { + pins-gsc-ap-int-odl { + pinmux = ; + input-enable; + }; + }; + + cros_ec_int: cros-ec-irq-default-pins { + pins-ec-ap-int-odl { + pinmux = ; + bias-pull-up = ; + input-enable; + }; + }; + i2c0_pins: i2c0-default-pins { pins-bus { pinmux = , @@ -540,6 +605,32 @@ }; }; + mmc1_pins_detect: mmc1-detect-pins { + pins-insert { + pinmux = ; + bias-pull-up; + }; + }; + + mmc1_pins_default: mmc1-default-pins { + pins-cmd-dat { + pinmux = , + , + , + , + ; + input-enable; + drive-strength = <8>; + bias-pull-up = ; + }; + + pins-clk { + pinmux = ; + drive-strength = <8>; + bias-pull-down = ; + }; + }; + nor_pins_default: nor-default-pins { pins-ck-io { pinmux = , @@ -600,6 +691,14 @@ }; }; + scp_pins: scp-default-pins { + pins-vreq { + pinmux = ; + bias-disable; + input-enable; + }; + }; + spi0_pins: spi0-default-pins { pins-cs-mosi-clk { pinmux = , @@ -622,6 +721,14 @@ }; }; + trackpad_pins: trackpad-default-pins { + pins-int-n { + pinmux = ; + input-enable; + bias-pull-up; + }; + }; + touchscreen_pins: touchscreen-default-pins { pins-int-n { pinmux = ; @@ -643,12 +750,128 @@ interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>; }; +&scp { + status = "okay"; + + firmware-name = "mediatek/mt8195/scp.img"; + memory-region = <&scp_mem>; + pinctrl-names = "default"; + pinctrl-0 = <&scp_pins>; + + cros-ec-rpmsg { + compatible = "google,cros-ec-rpmsg"; + mediatek,rpmsg-name = "cros-ec-rpmsg"; + }; +}; + &spi0 { status = "okay"; pinctrl-names = "default"; pinctrl-0 = <&spi0_pins>; mediatek,pad-select = <0>; + + cros_ec: ec@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "google,cros-ec-spi"; + reg = <0>; + interrupts-extended = <&pio 4 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&cros_ec_int>; + spi-max-frequency = <3000000>; + + keyboard-backlight { + compatible = "google,cros-kbd-led-backlight"; + }; + + i2c_tunnel: i2c-tunnel { + compatible = "google,cros-ec-i2c-tunnel"; + google,remote-bus = <0>; + #address-cells = <1>; + #size-cells = <0>; + }; + + mt_pmic_vmc_ldo_reg: regulator@0 { + compatible = "google,cros-ec-regulator"; + reg = <0>; + regulator-name = "mt_pmic_vmc_ldo"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3600000>; + }; + + mt_pmic_vmch_ldo_reg: regulator@1 { + compatible = "google,cros-ec-regulator"; + reg = <1>; + regulator-name = "mt_pmic_vmch_ldo"; + regulator-min-microvolt = <2700000>; + regulator-max-microvolt = <3600000>; + }; + + typec { + compatible = "google,cros-ec-typec"; + #address-cells = <1>; + #size-cells = <0>; + + usb_c0: connector@0 { + compatible = "usb-c-connector"; + reg = <0>; + power-role = "dual"; + data-role = "host"; + try-power-role = "source"; + }; + + usb_c1: connector@1 { + compatible = "usb-c-connector"; + reg = <1>; + power-role = "dual"; + data-role = "host"; + try-power-role = "source"; + }; + }; + }; +}; + +&spmi { + #address-cells = <2>; + #size-cells = <0>; + + mt6315@6 { + compatible = "mediatek,mt6315-regulator"; + reg = <0x6 SPMI_USID>; + + regulators { + mt6315_6_vbuck1: vbuck1 { + regulator-compatible = "vbuck1"; + regulator-name = "Vbcpu"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-ramp-delay = <6250>; + regulator-allowed-modes = <0 1 2>; + regulator-always-on; + }; + }; + }; + + mt6315@7 { + compatible = "mediatek,mt6315-regulator"; + reg = <0x7 SPMI_USID>; + + regulators { + mt6315_7_vbuck1: vbuck1 { + regulator-compatible = "vbuck1"; + regulator-name = "Vgpu"; + regulator-min-microvolt = <625000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-ramp-delay = <6250>; + regulator-allowed-modes = <0 1 2>; + regulator-always-on; + }; + }; + }; }; &u3phy0 { @@ -700,3 +923,36 @@ vusb33-supply = <&mt6359_vusb_ldo_reg>; vbus-supply = <&usb_vbus>; }; + +#include +#include + +&keyboard_controller { + function-row-physmap = < + MATRIX_KEY(0x00, 0x02, 0) /* T1 */ + MATRIX_KEY(0x03, 0x02, 0) /* T2 */ + MATRIX_KEY(0x02, 0x02, 0) /* T3 */ + MATRIX_KEY(0x01, 0x02, 0) /* T4 */ + MATRIX_KEY(0x03, 0x04, 0) /* T5 */ + MATRIX_KEY(0x02, 0x04, 0) /* T6 */ + MATRIX_KEY(0x01, 0x04, 0) /* T7 */ + MATRIX_KEY(0x02, 0x09, 0) /* T8 */ + MATRIX_KEY(0x01, 0x09, 0) /* T9 */ + MATRIX_KEY(0x00, 0x04, 0) /* T10 */ + >; + + linux,keymap = < + MATRIX_KEY(0x00, 0x02, KEY_BACK) + MATRIX_KEY(0x03, 0x02, KEY_REFRESH) + MATRIX_KEY(0x02, 0x02, KEY_ZOOM) + MATRIX_KEY(0x01, 0x02, KEY_SCALE) + MATRIX_KEY(0x03, 0x04, KEY_SYSRQ) + MATRIX_KEY(0x02, 0x04, KEY_BRIGHTNESSDOWN) + MATRIX_KEY(0x01, 0x04, KEY_BRIGHTNESSUP) + MATRIX_KEY(0x02, 0x09, KEY_MUTE) + MATRIX_KEY(0x01, 0x09, KEY_VOLUMEDOWN) + MATRIX_KEY(0x00, 0x04, KEY_VOLUMEUP) + + CROS_STD_MAIN_KEYMAP + >; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8195.dtsi b/arch/arm64/boot/dts/mediatek/mt8195.dtsi index 066c14989708aadd74ea9ea600c750b33f904a27..905d1a90b406c31d37e24966c4e0cd3cd9152bce 100644 --- a/arch/arm64/boot/dts/mediatek/mt8195.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8195.dtsi @@ -6,10 +6,13 @@ /dts-v1/; #include +#include #include #include +#include #include #include +#include / { compatible = "mediatek,mt8195"; @@ -17,6 +20,11 @@ #address-cells = <2>; #size-cells = <2>; + aliases { + gce0 = &gce0; + gce1 = &gce1; + }; + cpus { #address-cells = <1>; #size-cells = <0>; @@ -26,6 +34,7 @@ compatible = "arm,cortex-a55"; reg = <0x000>; enable-method = "psci"; + performance-domains = <&performance 0>; clock-frequency = <1701000000>; capacity-dmips-mhz = <578>; cpu-idle-states = <&cpu_off_l &cluster_off_l>; @@ -38,6 +47,7 @@ compatible = "arm,cortex-a55"; reg = <0x100>; enable-method = "psci"; + performance-domains = <&performance 0>; clock-frequency = <1701000000>; capacity-dmips-mhz = <578>; cpu-idle-states = <&cpu_off_l &cluster_off_l>; @@ -50,6 +60,7 @@ compatible = "arm,cortex-a55"; reg = <0x200>; enable-method = "psci"; + performance-domains = <&performance 0>; clock-frequency = <1701000000>; capacity-dmips-mhz = <578>; cpu-idle-states = <&cpu_off_l &cluster_off_l>; @@ -62,6 +73,7 @@ compatible = "arm,cortex-a55"; reg = <0x300>; enable-method = "psci"; + performance-domains = <&performance 0>; clock-frequency = <1701000000>; capacity-dmips-mhz = <578>; cpu-idle-states = <&cpu_off_l &cluster_off_l>; @@ -74,6 +86,7 @@ compatible = "arm,cortex-a78"; reg = <0x400>; enable-method = "psci"; + performance-domains = <&performance 1>; clock-frequency = <2171000000>; capacity-dmips-mhz = <1024>; cpu-idle-states = <&cpu_off_b &cluster_off_b>; @@ -86,6 +99,7 @@ compatible = "arm,cortex-a78"; reg = <0x500>; enable-method = "psci"; + performance-domains = <&performance 1>; clock-frequency = <2171000000>; capacity-dmips-mhz = <1024>; cpu-idle-states = <&cpu_off_b &cluster_off_b>; @@ -98,6 +112,7 @@ compatible = "arm,cortex-a78"; reg = <0x600>; enable-method = "psci"; + performance-domains = <&performance 1>; clock-frequency = <2171000000>; capacity-dmips-mhz = <1024>; cpu-idle-states = <&cpu_off_b &cluster_off_b>; @@ -110,6 +125,7 @@ compatible = "arm,cortex-a78"; reg = <0x700>; enable-method = "psci"; + performance-domains = <&performance 1>; clock-frequency = <2171000000>; capacity-dmips-mhz = <1024>; cpu-idle-states = <&cpu_off_b &cluster_off_b>; @@ -217,6 +233,17 @@ <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>; }; + dmic_codec: dmic-codec { + compatible = "dmic-codec"; + num-channels = <2>; + wakeup-delay-ms = <50>; + }; + + sound: mt8195-sound { + mediatek,platform = <&afe>; + status = "disabled"; + }; + clk26m: oscillator-26m { compatible = "fixed-clock"; #clock-cells = <0>; @@ -231,6 +258,12 @@ clock-output-names = "clk32k"; }; + performance: performance-controller@11bc10 { + compatible = "mediatek,cpufreq-hw"; + reg = <0 0x0011bc10 0 0x120>, <0 0x0011bd30 0 0x120>; + #performance-domain-cells = <1>; + }; + pmu-a55 { compatible = "arm,cortex-a55-pmu"; interrupt-parent = <&gic>; @@ -324,10 +357,337 @@ #interrupt-cells = <2>; }; + scpsys: syscon@10006000 { + compatible = "mediatek,mt8195-scpsys", "syscon", "simple-mfd"; + reg = <0 0x10006000 0 0x1000>; + + /* System Power Manager */ + spm: power-controller { + compatible = "mediatek,mt8195-power-controller"; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + /* power domain of the SoC */ + mfg0: power-domain@MT8195_POWER_DOMAIN_MFG0 { + reg = ; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_MFG1 { + reg = ; + clocks = <&apmixedsys CLK_APMIXED_MFGPLL>; + clock-names = "mfg"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_MFG2 { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_MFG3 { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_MFG4 { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_MFG5 { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_MFG6 { + reg = ; + #power-domain-cells = <0>; + }; + }; + }; + + power-domain@MT8195_POWER_DOMAIN_VPPSYS0 { + reg = ; + clocks = <&topckgen CLK_TOP_VPP>, + <&topckgen CLK_TOP_CAM>, + <&topckgen CLK_TOP_CCU>, + <&topckgen CLK_TOP_IMG>, + <&topckgen CLK_TOP_VENC>, + <&topckgen CLK_TOP_VDEC>, + <&topckgen CLK_TOP_WPE_VPP>, + <&topckgen CLK_TOP_CFG_VPP0>, + <&vppsys0 CLK_VPP0_SMI_COMMON>, + <&vppsys0 CLK_VPP0_GALS_VDO0_LARB0>, + <&vppsys0 CLK_VPP0_GALS_VDO0_LARB1>, + <&vppsys0 CLK_VPP0_GALS_VENCSYS>, + <&vppsys0 CLK_VPP0_GALS_VENCSYS_CORE1>, + <&vppsys0 CLK_VPP0_GALS_INFRA>, + <&vppsys0 CLK_VPP0_GALS_CAMSYS>, + <&vppsys0 CLK_VPP0_GALS_VPP1_LARB5>, + <&vppsys0 CLK_VPP0_GALS_VPP1_LARB6>, + <&vppsys0 CLK_VPP0_SMI_REORDER>, + <&vppsys0 CLK_VPP0_SMI_IOMMU>, + <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>, + <&vppsys0 CLK_VPP0_GALS_EMI0_EMI1>, + <&vppsys0 CLK_VPP0_SMI_SUB_COMMON_REORDER>, + <&vppsys0 CLK_VPP0_SMI_RSI>, + <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>, + <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>, + <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>, + <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>; + clock-names = "vppsys", "vppsys1", "vppsys2", "vppsys3", + "vppsys4", "vppsys5", "vppsys6", "vppsys7", + "vppsys0-0", "vppsys0-1", "vppsys0-2", "vppsys0-3", + "vppsys0-4", "vppsys0-5", "vppsys0-6", "vppsys0-7", + "vppsys0-8", "vppsys0-9", "vppsys0-10", "vppsys0-11", + "vppsys0-12", "vppsys0-13", "vppsys0-14", + "vppsys0-15", "vppsys0-16", "vppsys0-17", + "vppsys0-18"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_VDEC1 { + reg = ; + clocks = <&vdecsys CLK_VDEC_LARB1>; + clock-names = "vdec1-0"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VENC_CORE1 { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VDOSYS0 { + reg = ; + clocks = <&topckgen CLK_TOP_CFG_VDO0>, + <&vdosys0 CLK_VDO0_SMI_GALS>, + <&vdosys0 CLK_VDO0_SMI_COMMON>, + <&vdosys0 CLK_VDO0_SMI_EMI>, + <&vdosys0 CLK_VDO0_SMI_IOMMU>, + <&vdosys0 CLK_VDO0_SMI_LARB>, + <&vdosys0 CLK_VDO0_SMI_RSI>; + clock-names = "vdosys0", "vdosys0-0", "vdosys0-1", + "vdosys0-2", "vdosys0-3", + "vdosys0-4", "vdosys0-5"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_VPPSYS1 { + reg = ; + clocks = <&topckgen CLK_TOP_CFG_VPP1>, + <&vppsys1 CLK_VPP1_VPPSYS1_GALS>, + <&vppsys1 CLK_VPP1_VPPSYS1_LARB>; + clock-names = "vppsys1", "vppsys1-0", + "vppsys1-1"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_WPESYS { + reg = ; + clocks = <&wpesys CLK_WPE_SMI_LARB7>, + <&wpesys CLK_WPE_SMI_LARB8>, + <&wpesys CLK_WPE_SMI_LARB7_P>, + <&wpesys CLK_WPE_SMI_LARB8_P>; + clock-names = "wepsys-0", "wepsys-1", "wepsys-2", + "wepsys-3"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VDEC0 { + reg = ; + clocks = <&vdecsys_soc CLK_VDEC_SOC_LARB1>; + clock-names = "vdec0-0"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VDEC2 { + reg = ; + clocks = <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>; + clock-names = "vdec2-0"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VENC { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_VDOSYS1 { + reg = ; + clocks = <&topckgen CLK_TOP_CFG_VDO1>, + <&vdosys1 CLK_VDO1_SMI_LARB2>, + <&vdosys1 CLK_VDO1_SMI_LARB3>, + <&vdosys1 CLK_VDO1_GALS>; + clock-names = "vdosys1", "vdosys1-0", + "vdosys1-1", "vdosys1-2"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_DP_TX { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_EPD_TX { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_HDMI_TX { + reg = ; + clocks = <&topckgen CLK_TOP_HDMI_APB>; + clock-names = "hdmi_tx"; + #power-domain-cells = <0>; + }; + }; + + power-domain@MT8195_POWER_DOMAIN_IMG { + reg = ; + clocks = <&imgsys CLK_IMG_LARB9>, + <&imgsys CLK_IMG_GALS>; + clock-names = "img-0", "img-1"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_DIP { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_IPE { + reg = ; + clocks = <&topckgen CLK_TOP_IPE>, + <&imgsys CLK_IMG_IPE>, + <&ipesys CLK_IPE_SMI_LARB12>; + clock-names = "ipe", "ipe-0", "ipe-1"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + }; + + power-domain@MT8195_POWER_DOMAIN_CAM { + reg = ; + clocks = <&camsys CLK_CAM_LARB13>, + <&camsys CLK_CAM_LARB14>, + <&camsys CLK_CAM_CAM2MM0_GALS>, + <&camsys CLK_CAM_CAM2MM1_GALS>, + <&camsys CLK_CAM_CAM2SYS_GALS>; + clock-names = "cam-0", "cam-1", "cam-2", "cam-3", + "cam-4"; + mediatek,infracfg = <&infracfg_ao>; + #address-cells = <1>; + #size-cells = <0>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_CAM_RAWA { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_CAM_RAWB { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_CAM_MRAW { + reg = ; + #power-domain-cells = <0>; + }; + }; + }; + }; + + power-domain@MT8195_POWER_DOMAIN_PCIE_MAC_P0 { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_PCIE_MAC_P1 { + reg = ; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_PCIE_PHY { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_SSUSB_PCIE_PHY { + reg = ; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_CSI_RX_TOP { + reg = ; + clocks = <&topckgen CLK_TOP_SENINF>, + <&topckgen CLK_TOP_SENINF2>; + clock-names = "csi_rx_top", "csi_rx_top1"; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_ETHER { + reg = ; + clocks = <&pericfg_ao CLK_PERI_AO_ETHERNET_MAC>; + clock-names = "ether"; + #power-domain-cells = <0>; + }; + + power-domain@MT8195_POWER_DOMAIN_ADSP { + reg = ; + clocks = <&topckgen CLK_TOP_ADSP>, + <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>; + clock-names = "adsp", "adsp1"; + #address-cells = <1>; + #size-cells = <0>; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <1>; + + power-domain@MT8195_POWER_DOMAIN_AUDIO { + reg = ; + clocks = <&topckgen CLK_TOP_A1SYS_HP>, + <&topckgen CLK_TOP_AUD_INTBUS>, + <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>, + <&infracfg_ao CLK_INFRA_AO_AUDIO_26M_B>; + clock-names = "audio", "audio1", "audio2", + "audio3"; + mediatek,infracfg = <&infracfg_ao>; + #power-domain-cells = <0>; + }; + }; + }; + }; + watchdog: watchdog@10007000 { compatible = "mediatek,mt8195-wdt", "mediatek,mt6589-wdt"; + mediatek,disable-extrst; reg = <0 0x10007000 0 0x100>; + #reset-cells = <1>; }; apmixedsys: syscon@1000c000 { @@ -356,12 +716,150 @@ assigned-clock-parents = <&topckgen CLK_TOP_ULPOSC1_D10>; }; + spmi: spmi@10027000 { + compatible = "mediatek,mt8195-spmi"; + reg = <0 0x10027000 0 0x000e00>, + <0 0x10029000 0 0x000100>; + reg-names = "pmif", "spmimst"; + clocks = <&infracfg_ao CLK_INFRA_AO_PMIC_AP>, + <&infracfg_ao CLK_INFRA_AO_PMIC_TMR>, + <&topckgen CLK_TOP_SPMI_M_MST>; + clock-names = "pmif_sys_ck", + "pmif_tmr_ck", + "spmimst_clk_mux"; + assigned-clocks = <&topckgen CLK_TOP_PWRAP_ULPOSC>; + assigned-clock-parents = <&topckgen CLK_TOP_ULPOSC1_D10>; + }; + + iommu_infra: infra-iommu@10315000 { + compatible = "mediatek,mt8195-iommu-infra"; + reg = <0 0x10315000 0 0x5000>; + interrupts = , + , + , + , + ; + #iommu-cells = <1>; + }; + + gce0: mailbox@10320000 { + compatible = "mediatek,mt8195-gce"; + reg = <0 0x10320000 0 0x4000>; + interrupts = ; + #mbox-cells = <2>; + clocks = <&infracfg_ao CLK_INFRA_AO_GCE>; + }; + + gce1: mailbox@10330000 { + compatible = "mediatek,mt8195-gce"; + reg = <0 0x10330000 0 0x4000>; + interrupts = ; + #mbox-cells = <2>; + clocks = <&infracfg_ao CLK_INFRA_AO_GCE2>; + }; + + scp: scp@10500000 { + compatible = "mediatek,mt8195-scp"; + reg = <0 0x10500000 0 0x100000>, + <0 0x10720000 0 0xe0000>, + <0 0x10700000 0 0x8000>; + reg-names = "sram", "cfg", "l1tcm"; + interrupts = ; + status = "disabled"; + }; + scp_adsp: clock-controller@10720000 { compatible = "mediatek,mt8195-scp_adsp"; reg = <0 0x10720000 0 0x1000>; #clock-cells = <1>; }; + adsp: dsp@10803000 { + compatible = "mediatek,mt8195-dsp"; + reg = <0 0x10803000 0 0x1000>, + <0 0x10840000 0 0x40000>; + reg-names = "cfg", "sram"; + clocks = <&topckgen CLK_TOP_ADSP>, + <&clk26m>, + <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>, + <&topckgen CLK_TOP_MAINPLL_D7_D2>, + <&scp_adsp CLK_SCP_ADSP_AUDIODSP>, + <&topckgen CLK_TOP_AUDIO_H>; + clock-names = "adsp_sel", + "clk26m_ck", + "audio_local_bus", + "mainpll_d7_d2", + "scp_adsp_audiodsp", + "audio_h"; + power-domains = <&spm MT8195_POWER_DOMAIN_ADSP>; + mbox-names = "rx", "tx"; + mboxes = <&adsp_mailbox0>, <&adsp_mailbox1>; + status = "disabled"; + }; + + adsp_mailbox0: mailbox@10816000 { + compatible = "mediatek,mt8195-adsp-mbox"; + #mbox-cells = <0>; + reg = <0 0x10816000 0 0x1000>; + interrupts = ; + }; + + adsp_mailbox1: mailbox@10817000 { + compatible = "mediatek,mt8195-adsp-mbox"; + #mbox-cells = <0>; + reg = <0 0x10817000 0 0x1000>; + interrupts = ; + }; + + afe: mt8195-afe-pcm@10890000 { + compatible = "mediatek,mt8195-audio"; + reg = <0 0x10890000 0 0x10000>; + mediatek,topckgen = <&topckgen>; + power-domains = <&spm MT8195_POWER_DOMAIN_AUDIO>; + interrupts = ; + resets = <&watchdog 14>; + reset-names = "audiosys"; + clocks = <&clk26m>, + <&apmixedsys CLK_APMIXED_APLL1>, + <&apmixedsys CLK_APMIXED_APLL2>, + <&topckgen CLK_TOP_APLL12_DIV0>, + <&topckgen CLK_TOP_APLL12_DIV1>, + <&topckgen CLK_TOP_APLL12_DIV2>, + <&topckgen CLK_TOP_APLL12_DIV3>, + <&topckgen CLK_TOP_APLL12_DIV9>, + <&topckgen CLK_TOP_A1SYS_HP>, + <&topckgen CLK_TOP_AUD_INTBUS>, + <&topckgen CLK_TOP_AUDIO_H>, + <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>, + <&topckgen CLK_TOP_DPTX_MCK>, + <&topckgen CLK_TOP_I2SO1_MCK>, + <&topckgen CLK_TOP_I2SO2_MCK>, + <&topckgen CLK_TOP_I2SI1_MCK>, + <&topckgen CLK_TOP_I2SI2_MCK>, + <&infracfg_ao CLK_INFRA_AO_AUDIO_26M_B>, + <&scp_adsp CLK_SCP_ADSP_AUDIODSP>; + clock-names = "clk26m", + "apll1_ck", + "apll2_ck", + "apll12_div0", + "apll12_div1", + "apll12_div2", + "apll12_div3", + "apll12_div9", + "a1sys_hp_sel", + "aud_intbus_sel", + "audio_h_sel", + "audio_local_bus_sel", + "dptx_m_sel", + "i2so1_m_sel", + "i2so2_m_sel", + "i2si1_m_sel", + "i2si2_m_sel", + "infra_ao_audio_26m_b", + "scp_adsp_audiodsp"; + status = "disabled"; + }; + uart0: serial@11001100 { compatible = "mediatek,mt8195-uart", "mediatek,mt6577-uart"; @@ -560,8 +1058,10 @@ clocks = <&infracfg_ao CLK_INFRA_AO_SSUSB>, <&topckgen CLK_TOP_SSUSB_REF>, <&apmixedsys CLK_APMIXED_USB1PLL>, + <&clk26m>, <&infracfg_ao CLK_INFRA_AO_SSUSB_XHCI>; - clock-names = "sys_ck", "ref_ck", "mcu_ck", "xhci_ck"; + clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck", + "xhci_ck"; mediatek,syscon-wakeup = <&pericfg 0x400 103>; wakeup-source; status = "disabled"; @@ -625,8 +1125,10 @@ clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_1P_BUS>, <&topckgen CLK_TOP_SSUSB_P1_REF>, <&apmixedsys CLK_APMIXED_USB1PLL>, + <&clk26m>, <&pericfg_ao CLK_PERI_AO_SSUSB_1P_XHCI>; - clock-names = "sys_ck", "ref_ck", "mcu_ck","xhci_ck"; + clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck", + "xhci_ck"; mediatek,syscon-wakeup = <&pericfg 0x400 104>; wakeup-source; status = "disabled"; @@ -646,8 +1148,11 @@ <&topckgen CLK_TOP_UNIVPLL_D5_D4>; clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_2P_BUS>, <&topckgen CLK_TOP_SSUSB_P2_REF>, + <&clk26m>, + <&clk26m>, <&pericfg_ao CLK_PERI_AO_SSUSB_2P_XHCI>; - clock-names = "sys_ck", "ref_ck", "xhci_ck"; + clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck", + "xhci_ck"; mediatek,syscon-wakeup = <&pericfg 0x400 105>; wakeup-source; status = "disabled"; @@ -667,8 +1172,11 @@ <&topckgen CLK_TOP_UNIVPLL_D5_D4>; clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_3P_BUS>, <&topckgen CLK_TOP_SSUSB_P3_REF>, + <&clk26m>, + <&clk26m>, <&pericfg_ao CLK_PERI_AO_SSUSB_3P_XHCI>; - clock-names = "sys_ck", "ref_ck", "xhci_ck"; + clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck", + "xhci_ck"; mediatek,syscon-wakeup = <&pericfg 0x400 106>; wakeup-source; status = "disabled"; @@ -828,7 +1336,7 @@ clock-names = "main", "dma"; #address-cells = <1>; #size-cells = <0>; - status = "okay"; + status = "disabled"; }; i2c1: i2c@11e01000 { @@ -968,6 +1476,70 @@ #clock-cells = <1>; }; + vppsys0: clock-controller@14000000 { + compatible = "mediatek,mt8195-vppsys0"; + reg = <0 0x14000000 0 0x1000>; + #clock-cells = <1>; + }; + + smi_sub_common_vpp0_vpp1_2x1: smi@14010000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x14010000 0 0x1000>; + clocks = <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>, + <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>, + <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vpp>; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>; + }; + + smi_sub_common_vdec_vpp0_2x1: smi@14011000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x14011000 0 0x1000>; + clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>, + <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>, + <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vpp>; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>; + }; + + smi_common_vpp: smi@14012000 { + compatible = "mediatek,mt8195-smi-common-vpp"; + reg = <0 0x14012000 0 0x1000>; + clocks = <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>, + <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>, + <&vppsys0 CLK_VPP0_SMI_RSI>, + <&vppsys0 CLK_VPP0_SMI_RSI>; + clock-names = "apb", "smi", "gals0", "gals1"; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>; + }; + + larb4: larb@14013000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x14013000 0 0x1000>; + mediatek,larb-id = <4>; + mediatek,smi = <&smi_sub_common_vpp0_vpp1_2x1>; + clocks = <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>, + <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>; + }; + + iommu_vpp: iommu@14018000 { + compatible = "mediatek,mt8195-iommu-vpp"; + reg = <0 0x14018000 0 0x1000>; + mediatek,larbs = <&larb1 &larb3 &larb4 &larb6 &larb8 + &larb12 &larb14 &larb16 &larb18 + &larb20 &larb22 &larb23 &larb26 + &larb27>; + interrupts = ; + clocks = <&vppsys0 CLK_VPP0_SMI_IOMMU>; + clock-names = "bclk"; + #iommu-cells = <1>; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>; + }; + wpesys: clock-controller@14e00000 { compatible = "mediatek,mt8195-wpesys"; reg = <0 0x14e00000 0 0x1000>; @@ -986,18 +1558,116 @@ #clock-cells = <1>; }; + larb7: larb@14e04000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x14e04000 0 0x1000>; + mediatek,larb-id = <7>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&wpesys CLK_WPE_SMI_LARB7>, + <&wpesys CLK_WPE_SMI_LARB7>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_WPESYS>; + }; + + larb8: larb@14e05000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x14e05000 0 0x1000>; + mediatek,larb-id = <8>; + mediatek,smi = <&smi_common_vpp>; + clocks = <&wpesys CLK_WPE_SMI_LARB8>, + <&wpesys CLK_WPE_SMI_LARB8>, + <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_WPESYS>; + }; + + vppsys1: clock-controller@14f00000 { + compatible = "mediatek,mt8195-vppsys1"; + reg = <0 0x14f00000 0 0x1000>; + #clock-cells = <1>; + }; + + larb5: larb@14f02000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x14f02000 0 0x1000>; + mediatek,larb-id = <5>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vppsys1 CLK_VPP1_VPPSYS1_LARB>, + <&vppsys1 CLK_VPP1_VPPSYS1_GALS>, + <&vppsys0 CLK_VPP0_GALS_VPP1_LARB5>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS1>; + }; + + larb6: larb@14f03000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x14f03000 0 0x1000>; + mediatek,larb-id = <6>; + mediatek,smi = <&smi_sub_common_vpp0_vpp1_2x1>; + clocks = <&vppsys1 CLK_VPP1_VPPSYS1_LARB>, + <&vppsys1 CLK_VPP1_VPPSYS1_GALS>, + <&vppsys0 CLK_VPP0_GALS_VPP1_LARB6>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS1>; + }; + imgsys: clock-controller@15000000 { compatible = "mediatek,mt8195-imgsys"; reg = <0 0x15000000 0 0x1000>; #clock-cells = <1>; }; + larb9: larb@15001000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x15001000 0 0x1000>; + mediatek,larb-id = <9>; + mediatek,smi = <&smi_sub_common_img1_3x1>; + clocks = <&imgsys CLK_IMG_LARB9>, + <&imgsys CLK_IMG_LARB9>, + <&imgsys CLK_IMG_GALS>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_IMG>; + }; + + smi_sub_common_img0_3x1: smi@15002000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x15002000 0 0x1000>; + clocks = <&imgsys CLK_IMG_IPE>, + <&imgsys CLK_IMG_IPE>, + <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vpp>; + power-domains = <&spm MT8195_POWER_DOMAIN_IMG>; + }; + + smi_sub_common_img1_3x1: smi@15003000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x15003000 0 0x1000>; + clocks = <&imgsys CLK_IMG_LARB9>, + <&imgsys CLK_IMG_LARB9>, + <&imgsys CLK_IMG_GALS>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vdo>; + power-domains = <&spm MT8195_POWER_DOMAIN_IMG>; + }; + imgsys1_dip_top: clock-controller@15110000 { compatible = "mediatek,mt8195-imgsys1_dip_top"; reg = <0 0x15110000 0 0x1000>; #clock-cells = <1>; }; + larb10: larb@15120000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x15120000 0 0x1000>; + mediatek,larb-id = <10>; + mediatek,smi = <&smi_sub_common_img1_3x1>; + clocks = <&imgsys CLK_IMG_DIP0>, + <&imgsys1_dip_top CLK_IMG1_DIP_TOP_LARB10>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_DIP>; + }; + imgsys1_dip_nr: clock-controller@15130000 { compatible = "mediatek,mt8195-imgsys1_dip_nr"; reg = <0 0x15130000 0 0x1000>; @@ -1010,18 +1680,129 @@ #clock-cells = <1>; }; + larb11: larb@15230000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x15230000 0 0x1000>; + mediatek,larb-id = <11>; + mediatek,smi = <&smi_sub_common_img1_3x1>; + clocks = <&imgsys CLK_IMG_WPE0>, + <&imgsys1_wpe CLK_IMG1_WPE_LARB11>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_DIP>; + }; + ipesys: clock-controller@15330000 { compatible = "mediatek,mt8195-ipesys"; reg = <0 0x15330000 0 0x1000>; #clock-cells = <1>; }; + larb12: larb@15340000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x15340000 0 0x1000>; + mediatek,larb-id = <12>; + mediatek,smi = <&smi_sub_common_img0_3x1>; + clocks = <&ipesys CLK_IPE_SMI_LARB12>, + <&ipesys CLK_IPE_SMI_LARB12>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_IPE>; + }; + camsys: clock-controller@16000000 { compatible = "mediatek,mt8195-camsys"; reg = <0 0x16000000 0 0x1000>; #clock-cells = <1>; }; + larb13: larb@16001000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16001000 0 0x1000>; + mediatek,larb-id = <13>; + mediatek,smi = <&smi_sub_common_cam_4x1>; + clocks = <&camsys CLK_CAM_LARB13>, + <&camsys CLK_CAM_LARB13>, + <&camsys CLK_CAM_CAM2MM0_GALS>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM>; + }; + + larb14: larb@16002000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16002000 0 0x1000>; + mediatek,larb-id = <14>; + mediatek,smi = <&smi_sub_common_cam_7x1>; + clocks = <&camsys CLK_CAM_LARB14>, + <&camsys CLK_CAM_LARB14>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM>; + }; + + smi_sub_common_cam_4x1: smi@16004000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x16004000 0 0x1000>; + clocks = <&camsys CLK_CAM_LARB13>, + <&camsys CLK_CAM_LARB13>, + <&camsys CLK_CAM_CAM2MM0_GALS>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vdo>; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM>; + }; + + smi_sub_common_cam_7x1: smi@16005000 { + compatible = "mediatek,mt8195-smi-sub-common"; + reg = <0 0x16005000 0 0x1000>; + clocks = <&camsys CLK_CAM_LARB14>, + <&camsys CLK_CAM_CAM2MM1_GALS>, + <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>; + clock-names = "apb", "smi", "gals0"; + mediatek,smi = <&smi_common_vpp>; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM>; + }; + + larb16: larb@16012000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16012000 0 0x1000>; + mediatek,larb-id = <16>; + mediatek,smi = <&smi_sub_common_cam_7x1>; + clocks = <&camsys_rawa CLK_CAM_RAWA_LARBX>, + <&camsys_rawa CLK_CAM_RAWA_LARBX>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWA>; + }; + + larb17: larb@16013000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16013000 0 0x1000>; + mediatek,larb-id = <17>; + mediatek,smi = <&smi_sub_common_cam_4x1>; + clocks = <&camsys_yuva CLK_CAM_YUVA_LARBX>, + <&camsys_yuva CLK_CAM_YUVA_LARBX>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWA>; + }; + + larb27: larb@16014000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16014000 0 0x1000>; + mediatek,larb-id = <27>; + mediatek,smi = <&smi_sub_common_cam_7x1>; + clocks = <&camsys_rawb CLK_CAM_RAWB_LARBX>, + <&camsys_rawb CLK_CAM_RAWB_LARBX>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWB>; + }; + + larb28: larb@16015000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16015000 0 0x1000>; + mediatek,larb-id = <28>; + mediatek,smi = <&smi_sub_common_cam_4x1>; + clocks = <&camsys_yuvb CLK_CAM_YUVB_LARBX>, + <&camsys_yuvb CLK_CAM_YUVB_LARBX>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWB>; + }; + camsys_rawa: clock-controller@1604f000 { compatible = "mediatek,mt8195-camsys_rawa"; reg = <0 0x1604f000 0 0x1000>; @@ -1052,24 +1833,103 @@ #clock-cells = <1>; }; + larb25: larb@16141000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16141000 0 0x1000>; + mediatek,larb-id = <25>; + mediatek,smi = <&smi_sub_common_cam_4x1>; + clocks = <&camsys CLK_CAM_LARB13>, + <&camsys_mraw CLK_CAM_MRAW_LARBX>, + <&camsys CLK_CAM_CAM2MM0_GALS>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_MRAW>; + }; + + larb26: larb@16142000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x16142000 0 0x1000>; + mediatek,larb-id = <26>; + mediatek,smi = <&smi_sub_common_cam_7x1>; + clocks = <&camsys_mraw CLK_CAM_MRAW_LARBX>, + <&camsys_mraw CLK_CAM_MRAW_LARBX>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM_MRAW>; + + }; + ccusys: clock-controller@17200000 { compatible = "mediatek,mt8195-ccusys"; reg = <0 0x17200000 0 0x1000>; #clock-cells = <1>; }; + larb18: larb@17201000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x17201000 0 0x1000>; + mediatek,larb-id = <18>; + mediatek,smi = <&smi_sub_common_cam_7x1>; + clocks = <&ccusys CLK_CCU_LARB18>, + <&ccusys CLK_CCU_LARB18>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_CAM>; + }; + + larb24: larb@1800d000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1800d000 0 0x1000>; + mediatek,larb-id = <24>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vdecsys_soc CLK_VDEC_SOC_LARB1>, + <&vdecsys_soc CLK_VDEC_SOC_LARB1>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDEC0>; + }; + + larb23: larb@1800e000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1800e000 0 0x1000>; + mediatek,larb-id = <23>; + mediatek,smi = <&smi_sub_common_vdec_vpp0_2x1>; + clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>, + <&vdecsys_soc CLK_VDEC_SOC_LARB1>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDEC0>; + }; + vdecsys_soc: clock-controller@1800f000 { compatible = "mediatek,mt8195-vdecsys_soc"; reg = <0 0x1800f000 0 0x1000>; #clock-cells = <1>; }; + larb21: larb@1802e000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1802e000 0 0x1000>; + mediatek,larb-id = <21>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vdecsys CLK_VDEC_LARB1>, + <&vdecsys CLK_VDEC_LARB1>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDEC1>; + }; + vdecsys: clock-controller@1802f000 { compatible = "mediatek,mt8195-vdecsys"; reg = <0 0x1802f000 0 0x1000>; #clock-cells = <1>; }; + larb22: larb@1803e000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1803e000 0 0x1000>; + mediatek,larb-id = <22>; + mediatek,smi = <&smi_sub_common_vdec_vpp0_2x1>; + clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>, + <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDEC2>; + }; + vdecsys_core1: clock-controller@1803f000 { compatible = "mediatek,mt8195-vdecsys_core1"; reg = <0 0x1803f000 0 0x1000>; @@ -1088,10 +1948,212 @@ #clock-cells = <1>; }; + larb19: larb@1a010000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1a010000 0 0x1000>; + mediatek,larb-id = <19>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vencsys CLK_VENC_VENC>, + <&vencsys CLK_VENC_GALS>; + clock-names = "apb", "smi"; + power-domains = <&spm MT8195_POWER_DOMAIN_VENC>; + }; + vencsys_core1: clock-controller@1b000000 { compatible = "mediatek,mt8195-vencsys_core1"; reg = <0 0x1b000000 0 0x1000>; #clock-cells = <1>; }; + + vdosys0: syscon@1c01a000 { + compatible = "mediatek,mt8195-mmsys", "syscon"; + reg = <0 0x1c01a000 0 0x1000>; + mboxes = <&gce0 0 CMDQ_THR_PRIO_4>; + #clock-cells = <1>; + }; + + larb20: larb@1b010000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1b010000 0 0x1000>; + mediatek,larb-id = <20>; + mediatek,smi = <&smi_common_vpp>; + clocks = <&vencsys_core1 CLK_VENC_CORE1_LARB>, + <&vencsys_core1 CLK_VENC_CORE1_GALS>, + <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VENC_CORE1>; + }; + + ovl0: ovl@1c000000 { + compatible = "mediatek,mt8195-disp-ovl", "mediatek,mt8183-disp-ovl"; + reg = <0 0x1c000000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_OVL0>; + iommus = <&iommu_vdo M4U_PORT_L0_DISP_OVL0_RDMA0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x0000 0x1000>; + }; + + rdma0: rdma@1c002000 { + compatible = "mediatek,mt8195-disp-rdma"; + reg = <0 0x1c002000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_RDMA0>; + iommus = <&iommu_vdo M4U_PORT_L0_DISP_RDMA0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x2000 0x1000>; + }; + + color0: color@1c003000 { + compatible = "mediatek,mt8195-disp-color", "mediatek,mt8173-disp-color"; + reg = <0 0x1c003000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_COLOR0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x3000 0x1000>; + }; + + ccorr0: ccorr@1c004000 { + compatible = "mediatek,mt8195-disp-ccorr", "mediatek,mt8192-disp-ccorr"; + reg = <0 0x1c004000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_CCORR0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x4000 0x1000>; + }; + + aal0: aal@1c005000 { + compatible = "mediatek,mt8195-disp-aal", "mediatek,mt8183-disp-aal"; + reg = <0 0x1c005000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_AAL0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x5000 0x1000>; + }; + + gamma0: gamma@1c006000 { + compatible = "mediatek,mt8195-disp-gamma", "mediatek,mt8183-disp-gamma"; + reg = <0 0x1c006000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_GAMMA0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x6000 0x1000>; + }; + + dither0: dither@1c007000 { + compatible = "mediatek,mt8195-disp-dither", "mediatek,mt8183-disp-dither"; + reg = <0 0x1c007000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_DITHER0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x7000 0x1000>; + }; + + dsc0: dsc@1c009000 { + compatible = "mediatek,mt8195-disp-dsc"; + reg = <0 0x1c009000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DSC_WRAP0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x9000 0x1000>; + }; + + merge0: merge@1c014000 { + compatible = "mediatek,mt8195-disp-merge"; + reg = <0 0x1c014000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_VPP_MERGE0>; + mediatek,gce-client-reg = <&gce0 SUBSYS_1c01XXXX 0x4000 0x1000>; + }; + + mutex: mutex@1c016000 { + compatible = "mediatek,mt8195-disp-mutex"; + reg = <0 0x1c016000 0 0x1000>; + interrupts = ; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + clocks = <&vdosys0 CLK_VDO0_DISP_MUTEX0>; + mediatek,gce-events = ; + }; + + larb0: larb@1c018000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1c018000 0 0x1000>; + mediatek,larb-id = <0>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vdosys0 CLK_VDO0_SMI_LARB>, + <&vdosys0 CLK_VDO0_SMI_LARB>, + <&vppsys0 CLK_VPP0_GALS_VDO0_LARB0>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + }; + + larb1: larb@1c019000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1c019000 0 0x1000>; + mediatek,larb-id = <1>; + mediatek,smi = <&smi_common_vpp>; + clocks = <&vdosys0 CLK_VDO0_SMI_LARB>, + <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>, + <&vppsys0 CLK_VPP0_GALS_VDO0_LARB1>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + }; + + vdosys1: syscon@1c100000 { + compatible = "mediatek,mt8195-mmsys", "syscon"; + reg = <0 0x1c100000 0 0x1000>; + #clock-cells = <1>; + }; + + smi_common_vdo: smi@1c01b000 { + compatible = "mediatek,mt8195-smi-common-vdo"; + reg = <0 0x1c01b000 0 0x1000>; + clocks = <&vdosys0 CLK_VDO0_SMI_COMMON>, + <&vdosys0 CLK_VDO0_SMI_EMI>, + <&vdosys0 CLK_VDO0_SMI_RSI>, + <&vdosys0 CLK_VDO0_SMI_GALS>; + clock-names = "apb", "smi", "gals0", "gals1"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + + }; + + iommu_vdo: iommu@1c01f000 { + compatible = "mediatek,mt8195-iommu-vdo"; + reg = <0 0x1c01f000 0 0x1000>; + mediatek,larbs = <&larb0 &larb2 &larb5 &larb7 &larb9 + &larb10 &larb11 &larb13 &larb17 + &larb19 &larb21 &larb24 &larb25 + &larb28>; + interrupts = ; + #iommu-cells = <1>; + clocks = <&vdosys0 CLK_VDO0_SMI_IOMMU>; + clock-names = "bclk"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>; + }; + + larb2: larb@1c102000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1c102000 0 0x1000>; + mediatek,larb-id = <2>; + mediatek,smi = <&smi_common_vdo>; + clocks = <&vdosys1 CLK_VDO1_SMI_LARB2>, + <&vdosys1 CLK_VDO1_SMI_LARB2>, + <&vdosys1 CLK_VDO1_GALS>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>; + }; + + larb3: larb@1c103000 { + compatible = "mediatek,mt8195-smi-larb"; + reg = <0 0x1c103000 0 0x1000>; + mediatek,larb-id = <3>; + mediatek,smi = <&smi_common_vpp>; + clocks = <&vdosys1 CLK_VDO1_SMI_LARB3>, + <&vdosys1 CLK_VDO1_GALS>, + <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>; + clock-names = "apb", "smi", "gals"; + power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>; + }; }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi index 59a10fb184f8258cbfffa2841cda424e301f6815..6602fe421ee8d092a6f324625d2a5ea3d7e63f4a 100644 --- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi @@ -672,6 +672,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C1>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 21>, <&gpcdma 21>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -685,6 +689,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C3>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 23>, <&gpcdma 23>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -702,6 +710,10 @@ pinctrl-names = "default", "idle"; pinctrl-0 = <&state_dpaux1_i2c>; pinctrl-1 = <&state_dpaux1_off>; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 26>, <&gpcdma 26>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -733,6 +745,10 @@ pinctrl-names = "default", "idle"; pinctrl-0 = <&state_dpaux_i2c>; pinctrl-1 = <&state_dpaux_off>; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 30>, <&gpcdma 30>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -746,6 +762,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C7>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 27>, <&gpcdma 27>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -759,6 +779,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C9>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 31>, <&gpcdma 31>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1176,6 +1200,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C2>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 22>, <&gpcdma 22>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1189,6 +1217,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA186_RESET_I2C8>; reset-names = "i2c"; + iommus = <&smmu TEGRA186_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 0>, <&gpcdma 0>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1485,15 +1517,14 @@ iommus = <&smmu TEGRA186_SID_HOST1X>; /* Context isolation domains */ - iommu-map = < - 0 &smmu TEGRA186_SID_HOST1X_CTX0 1 - 1 &smmu TEGRA186_SID_HOST1X_CTX1 1 - 2 &smmu TEGRA186_SID_HOST1X_CTX2 1 - 3 &smmu TEGRA186_SID_HOST1X_CTX3 1 - 4 &smmu TEGRA186_SID_HOST1X_CTX4 1 - 5 &smmu TEGRA186_SID_HOST1X_CTX5 1 - 6 &smmu TEGRA186_SID_HOST1X_CTX6 1 - 7 &smmu TEGRA186_SID_HOST1X_CTX7 1>; + iommu-map = <0 &smmu TEGRA186_SID_HOST1X_CTX0 1>, + <1 &smmu TEGRA186_SID_HOST1X_CTX1 1>, + <2 &smmu TEGRA186_SID_HOST1X_CTX2 1>, + <3 &smmu TEGRA186_SID_HOST1X_CTX3 1>, + <4 &smmu TEGRA186_SID_HOST1X_CTX4 1>, + <5 &smmu TEGRA186_SID_HOST1X_CTX5 1>, + <6 &smmu TEGRA186_SID_HOST1X_CTX6 1>, + <7 &smmu TEGRA186_SID_HOST1X_CTX7 1>; dpaux1: dpaux@15040000 { compatible = "nvidia,tegra186-dpaux"; diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index d0ed55e5c8607b16f478f9814e8a992452d64839..41f3a7e188d08b3d5578d7bd7fb7412116d8f2b3 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -805,6 +805,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA194_RESET_I2C1>; reset-names = "i2c"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 21>, <&gpcdma 21>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -830,6 +834,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA194_RESET_I2C3>; reset-names = "i2c"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 23>, <&gpcdma 23>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -847,6 +855,10 @@ pinctrl-0 = <&state_dpaux1_i2c>; pinctrl-1 = <&state_dpaux1_off>; pinctrl-names = "default", "idle"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 26>, <&gpcdma 26>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -864,6 +876,10 @@ pinctrl-0 = <&state_dpaux0_i2c>; pinctrl-1 = <&state_dpaux0_off>; pinctrl-names = "default", "idle"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 30>, <&gpcdma 30>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -881,6 +897,10 @@ pinctrl-0 = <&state_dpaux2_i2c>; pinctrl-1 = <&state_dpaux2_off>; pinctrl-names = "default", "idle"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 27>, <&gpcdma 27>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -898,6 +918,10 @@ pinctrl-0 = <&state_dpaux3_i2c>; pinctrl-1 = <&state_dpaux3_off>; pinctrl-names = "default", "idle"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 31>, <&gpcdma 31>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1565,6 +1589,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA194_RESET_I2C2>; reset-names = "i2c"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 22>, <&gpcdma 22>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1578,6 +1606,10 @@ clock-names = "div-clk"; resets = <&bpmp TEGRA194_RESET_I2C8>; reset-names = "i2c"; + iommus = <&smmu TEGRA194_SID_GPCDMA_0>; + dma-coherent; + dmas = <&gpcdma 0>, <&gpcdma 0>; + dma-names = "rx", "tx"; status = "disabled"; }; @@ -1869,15 +1901,14 @@ iommus = <&smmu TEGRA194_SID_HOST1X>; /* Context isolation domains */ - iommu-map = < - 0 &smmu TEGRA194_SID_HOST1X_CTX0 1 - 1 &smmu TEGRA194_SID_HOST1X_CTX1 1 - 2 &smmu TEGRA194_SID_HOST1X_CTX2 1 - 3 &smmu TEGRA194_SID_HOST1X_CTX3 1 - 4 &smmu TEGRA194_SID_HOST1X_CTX4 1 - 5 &smmu TEGRA194_SID_HOST1X_CTX5 1 - 6 &smmu TEGRA194_SID_HOST1X_CTX6 1 - 7 &smmu TEGRA194_SID_HOST1X_CTX7 1>; + iommu-map = <0 &smmu TEGRA194_SID_HOST1X_CTX0 1>, + <1 &smmu TEGRA194_SID_HOST1X_CTX1 1>, + <2 &smmu TEGRA194_SID_HOST1X_CTX2 1>, + <3 &smmu TEGRA194_SID_HOST1X_CTX3 1>, + <4 &smmu TEGRA194_SID_HOST1X_CTX4 1>, + <5 &smmu TEGRA194_SID_HOST1X_CTX5 1>, + <6 &smmu TEGRA194_SID_HOST1X_CTX6 1>, + <7 &smmu TEGRA194_SID_HOST1X_CTX7 1>; nvdec@15140000 { compatible = "nvidia,tegra194-nvdec"; diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts index 5f3a1c56b2eb83615205cea2dd24545df0075892..7c569695b70523a1c10f709d412049643f15b113 100644 --- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts +++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts @@ -17,6 +17,7 @@ aliases { serial0 = &uarta; + serial3 = &uartd; }; chosen { @@ -1309,6 +1310,22 @@ status = "okay"; }; + uartd: serial@70006300 { + compatible = "nvidia,tegra30-hsuart"; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm43540-bt"; + max-speed = <4000000>; + brcm,bt-pcm-int-params = [01 02 00 01 01]; + device-wakeup-gpios = <&gpio TEGRA_GPIO(H, 3) GPIO_ACTIVE_HIGH>; + shutdown-gpios = <&gpio TEGRA_GPIO(H, 4) GPIO_ACTIVE_HIGH>; + interrupt-parent = <&gpio>; + interrupts = ; + interrupt-names = "host-wakeup"; + }; + }; + i2c@7000c400 { status = "okay"; clock-frequency = <1000000>; @@ -1692,6 +1709,25 @@ }; }; + mmc@700b0200 { + power-gpios = <&gpio TEGRA_GPIO(H, 1) GPIO_ACTIVE_HIGH>; + bus-width = <4>; + non-removable; + vqmmc-supply = <&pp1800>; + vmmc-supply = <&pp3300>; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + wifi@1 { + compatible = "brcm,bcm4354-fmac", "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&gpio>; + interrupts = ; + interrupt-names = "host-wake"; + }; + }; + mmc@700b0600 { bus-width = <8>; non-removable; diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi index 798de9226ba5c01677f13032e857c4149b46565f..9e4d72cfa69f5cac9fc9f2d6ce17ae6911847326 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi @@ -6,6 +6,42 @@ model = "NVIDIA Jetson AGX Orin"; compatible = "nvidia,p3701-0000", "nvidia,tegra234"; + vdd_1v8_ls: regulator-vdd-1v8-ls { + compatible = "regulator-fixed"; + regulator-name = "VDD_1V8_LS"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vdd_1v8_ao: regulator-vdd-1v8-ao { + compatible = "regulator-fixed"; + regulator-name = "VDD_1V8_AO"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + }; + + vdd_3v3_pcie: regulator-vdd-3v3-pcie { + compatible = "regulator-fixed"; + regulator-name = "VDD_3V3_PCIE"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpio TEGRA234_MAIN_GPIO(Z, 2) GPIO_ACTIVE_HIGH>; + regulator-boot-on; + enable-active-high; + }; + + vdd_12v_pcie: regulator-vdd-12v-pcie { + compatible = "regulator-fixed"; + regulator-name = "VDD_12V_PCIE"; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + gpio = <&gpio TEGRA234_MAIN_GPIO(A, 1) GPIO_ACTIVE_LOW>; + regulator-boot-on; + enable-active-low; + }; + bus@0 { spi@3270000 { status = "okay"; diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts index 02a10bb38562bd681278b75cd364560579319db9..57ab753288144c52dacb38140a0141d76fc47939 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts +++ b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts @@ -2009,6 +2009,7 @@ hda@3510000 { nvidia,model = "NVIDIA Jetson AGX Orin HDA"; + status = "okay"; }; }; @@ -2017,6 +2018,27 @@ stdout-path = "serial0:115200n8"; }; + bus@0 { + ethernet@6800000 { + status = "okay"; + + phy-handle = <&mgbe0_phy>; + phy-mode = "usxgmii"; + + mdio { + #address-cells = <1>; + #size-cells = <0>; + + mgbe0_phy: phy@0 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0x0>; + + #phy-cells = <0>; + }; + }; + }; + }; + gpio-keys { compatible = "gpio-keys"; status = "okay"; @@ -2111,4 +2133,55 @@ label = "NVIDIA Jetson AGX Orin APE"; }; + + pcie@14100000 { + status = "okay"; + + vddio-pex-ctl-supply = <&vdd_1v8_ao>; + + phys = <&p2u_hsio_3>; + phy-names = "p2u-0"; + }; + + pcie@14160000 { + status = "okay"; + + vddio-pex-ctl-supply = <&vdd_1v8_ao>; + + phys = <&p2u_hsio_4>, <&p2u_hsio_5>, <&p2u_hsio_6>, + <&p2u_hsio_7>; + phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3"; + }; + + pcie@141a0000 { + status = "okay"; + + vddio-pex-ctl-supply = <&vdd_1v8_ls>; + vpcie3v3-supply = <&vdd_3v3_pcie>; + vpcie12v-supply = <&vdd_12v_pcie>; + + phys = <&p2u_nvhs_0>, <&p2u_nvhs_1>, <&p2u_nvhs_2>, + <&p2u_nvhs_3>, <&p2u_nvhs_4>, <&p2u_nvhs_5>, + <&p2u_nvhs_6>, <&p2u_nvhs_7>; + phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3", "p2u-4", + "p2u-5", "p2u-6", "p2u-7"; + }; + + pcie-ep@141a0000 { + status = "disabled"; + + vddio-pex-ctl-supply = <&vdd_1v8_ls>; + + reset-gpios = <&gpio TEGRA234_MAIN_GPIO(AF, 1) GPIO_ACTIVE_LOW>; + + nvidia,refclk-select-gpios = <&gpio_aon + TEGRA234_AON_GPIO(AA, 4) + GPIO_ACTIVE_HIGH>; + + phys = <&p2u_nvhs_0>, <&p2u_nvhs_1>, <&p2u_nvhs_2>, + <&p2u_nvhs_3>, <&p2u_nvhs_4>, <&p2u_nvhs_5>, + <&p2u_nvhs_6>, <&p2u_nvhs_7>; + phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3", "p2u-4", + "p2u-5", "p2u-6", "p2u-7"; + }; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index 81a0f599685f94e14efb923875b9e073bc5c0c8b..0170bfa8a467906e862c3d8ff3b8986b41e535a6 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -23,7 +23,6 @@ gpcdma: dma-controller@2600000 { compatible = "nvidia,tegra234-gpcdma", - "nvidia,tegra194-gpcdma", "nvidia,tegra186-gpcdma"; reg = <0x2600000 0x210000>; resets = <&bpmp TEGRA234_RESET_GPCDMA>; @@ -570,6 +569,24 @@ interconnect-names = "dma-mem"; iommus = <&smmu_niso1 TEGRA234_SID_HOST1X>; + /* Context isolation domains */ + iommu-map = <0 &smmu_niso0 TEGRA234_SID_HOST1X_CTX0 1>, + <1 &smmu_niso0 TEGRA234_SID_HOST1X_CTX1 1>, + <2 &smmu_niso0 TEGRA234_SID_HOST1X_CTX2 1>, + <3 &smmu_niso0 TEGRA234_SID_HOST1X_CTX3 1>, + <4 &smmu_niso0 TEGRA234_SID_HOST1X_CTX4 1>, + <5 &smmu_niso0 TEGRA234_SID_HOST1X_CTX5 1>, + <6 &smmu_niso0 TEGRA234_SID_HOST1X_CTX6 1>, + <7 &smmu_niso0 TEGRA234_SID_HOST1X_CTX7 1>, + <8 &smmu_niso1 TEGRA234_SID_HOST1X_CTX0 1>, + <9 &smmu_niso1 TEGRA234_SID_HOST1X_CTX1 1>, + <10 &smmu_niso1 TEGRA234_SID_HOST1X_CTX2 1>, + <11 &smmu_niso1 TEGRA234_SID_HOST1X_CTX3 1>, + <12 &smmu_niso1 TEGRA234_SID_HOST1X_CTX4 1>, + <13 &smmu_niso1 TEGRA234_SID_HOST1X_CTX5 1>, + <14 &smmu_niso1 TEGRA234_SID_HOST1X_CTX6 1>, + <15 &smmu_niso1 TEGRA234_SID_HOST1X_CTX7 1>; + vic@15340000 { compatible = "nvidia,tegra234-vic"; reg = <0x15340000 0x00040000>; @@ -737,6 +754,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C1>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 21>, <&gpcdma 21>; + dma-names = "rx", "tx"; }; cam_i2c: i2c@3180000 { @@ -752,6 +773,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C3>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 23>, <&gpcdma 23>; + dma-names = "rx", "tx"; }; dp_aux_ch1_i2c: i2c@3190000 { @@ -767,6 +792,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C4>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 26>, <&gpcdma 26>; + dma-names = "rx", "tx"; }; dp_aux_ch0_i2c: i2c@31b0000 { @@ -782,6 +811,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C6>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 30>, <&gpcdma 30>; + dma-names = "rx", "tx"; }; dp_aux_ch2_i2c: i2c@31c0000 { @@ -797,6 +830,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C7>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 27>, <&gpcdma 27>; + dma-names = "rx", "tx"; }; dp_aux_ch3_i2c: i2c@31e0000 { @@ -812,6 +849,10 @@ clock-names = "div-clk", "parent"; resets = <&bpmp TEGRA234_RESET_I2C9>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 31>, <&gpcdma 31>; + dma-names = "rx", "tx"; }; spi@3270000 { @@ -897,6 +938,7 @@ interconnects = <&mc TEGRA234_MEMORY_CLIENT_HDAR &emc>, <&mc TEGRA234_MEMORY_CLIENT_HDAW &emc>; interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_HDA>; status = "disabled"; }; @@ -925,6 +967,142 @@ #mbox-cells = <2>; }; + ethernet@6800000 { + compatible = "nvidia,tegra234-mgbe"; + reg = <0x06800000 0x10000>, + <0x06810000 0x10000>, + <0x068a0000 0x10000>; + reg-names = "hypervisor", "mac", "xpcs"; + interrupts = ; + interrupt-names = "common"; + clocks = <&bpmp TEGRA234_CLK_MGBE0_APP>, + <&bpmp TEGRA234_CLK_MGBE0_MAC>, + <&bpmp TEGRA234_CLK_MGBE0_MAC_DIVIDER>, + <&bpmp TEGRA234_CLK_MGBE0_PTP_REF>, + <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT_M>, + <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT>, + <&bpmp TEGRA234_CLK_MGBE0_TX>, + <&bpmp TEGRA234_CLK_MGBE0_EEE_PCS>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_INPUT>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>, + <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>, + <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>; + clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", + "rx-pcs", "tx-pcs"; + resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>, + <&bpmp TEGRA234_RESET_MGBE0_PCS>; + reset-names = "mac", "pcs"; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEARD &emc>, + <&mc TEGRA234_MEMORY_CLIENT_MGBEAWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_MGBE>; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEA>; + status = "disabled"; + }; + + ethernet@6900000 { + compatible = "nvidia,tegra234-mgbe"; + reg = <0x06900000 0x10000>, + <0x06910000 0x10000>, + <0x069a0000 0x10000>; + reg-names = "hypervisor", "mac", "xpcs"; + interrupts = ; + interrupt-names = "common"; + clocks = <&bpmp TEGRA234_CLK_MGBE1_APP>, + <&bpmp TEGRA234_CLK_MGBE1_MAC>, + <&bpmp TEGRA234_CLK_MGBE1_MAC_DIVIDER>, + <&bpmp TEGRA234_CLK_MGBE1_PTP_REF>, + <&bpmp TEGRA234_CLK_MGBE1_RX_INPUT_M>, + <&bpmp TEGRA234_CLK_MGBE1_RX_INPUT>, + <&bpmp TEGRA234_CLK_MGBE1_TX>, + <&bpmp TEGRA234_CLK_MGBE1_EEE_PCS>, + <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_INPUT>, + <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_M>, + <&bpmp TEGRA234_CLK_MGBE1_RX_PCS>, + <&bpmp TEGRA234_CLK_MGBE1_TX_PCS>; + clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", + "rx-pcs", "tx-pcs"; + resets = <&bpmp TEGRA234_RESET_MGBE1_MAC>, + <&bpmp TEGRA234_RESET_MGBE1_PCS>; + reset-names = "mac", "pcs"; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEBRD &emc>, + <&mc TEGRA234_MEMORY_CLIENT_MGBEBWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF1>; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEB>; + status = "disabled"; + }; + + ethernet@6a00000 { + compatible = "nvidia,tegra234-mgbe"; + reg = <0x06a00000 0x10000>, + <0x06a10000 0x10000>, + <0x06aa0000 0x10000>; + reg-names = "hypervisor", "mac", "xpcs"; + interrupts = ; + interrupt-names = "common"; + clocks = <&bpmp TEGRA234_CLK_MGBE2_APP>, + <&bpmp TEGRA234_CLK_MGBE2_MAC>, + <&bpmp TEGRA234_CLK_MGBE2_MAC_DIVIDER>, + <&bpmp TEGRA234_CLK_MGBE2_PTP_REF>, + <&bpmp TEGRA234_CLK_MGBE2_RX_INPUT_M>, + <&bpmp TEGRA234_CLK_MGBE2_RX_INPUT>, + <&bpmp TEGRA234_CLK_MGBE2_TX>, + <&bpmp TEGRA234_CLK_MGBE2_EEE_PCS>, + <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_INPUT>, + <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_M>, + <&bpmp TEGRA234_CLK_MGBE2_RX_PCS>, + <&bpmp TEGRA234_CLK_MGBE2_TX_PCS>; + clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", + "rx-pcs", "tx-pcs"; + resets = <&bpmp TEGRA234_RESET_MGBE2_MAC>, + <&bpmp TEGRA234_RESET_MGBE2_PCS>; + reset-names = "mac", "pcs"; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBECRD &emc>, + <&mc TEGRA234_MEMORY_CLIENT_MGBECWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF2>; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEC>; + status = "disabled"; + }; + + ethernet@6b00000 { + compatible = "nvidia,tegra234-mgbe"; + reg = <0x06b00000 0x10000>, + <0x06b10000 0x10000>, + <0x06ba0000 0x10000>; + reg-names = "hypervisor", "mac", "xpcs"; + interrupts = ; + interrupt-names = "common"; + clocks = <&bpmp TEGRA234_CLK_MGBE3_APP>, + <&bpmp TEGRA234_CLK_MGBE3_MAC>, + <&bpmp TEGRA234_CLK_MGBE3_MAC_DIVIDER>, + <&bpmp TEGRA234_CLK_MGBE3_PTP_REF>, + <&bpmp TEGRA234_CLK_MGBE3_RX_INPUT_M>, + <&bpmp TEGRA234_CLK_MGBE3_RX_INPUT>, + <&bpmp TEGRA234_CLK_MGBE3_TX>, + <&bpmp TEGRA234_CLK_MGBE3_EEE_PCS>, + <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_INPUT>, + <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_M>, + <&bpmp TEGRA234_CLK_MGBE3_RX_PCS>, + <&bpmp TEGRA234_CLK_MGBE3_TX_PCS>; + clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m", + "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m", + "rx-pcs", "tx-pcs"; + resets = <&bpmp TEGRA234_RESET_MGBE3_MAC>, + <&bpmp TEGRA234_RESET_MGBE3_PCS>; + reset-names = "mac", "pcs"; + interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEDRD &emc>, + <&mc TEGRA234_MEMORY_CLIENT_MGBEDWR &emc>; + interconnect-names = "dma-mem", "write"; + iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF3>; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBED>; + status = "disabled"; + }; + smmu_niso1: iommu@8000000 { compatible = "nvidia,tegra234-smmu", "nvidia,smmu-500"; reg = <0x8000000 0x1000000>, @@ -1081,6 +1259,198 @@ status = "okay"; }; + p2u_hsio_0: phy@3e00000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e00000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_1: phy@3e10000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e10000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_2: phy@3e20000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e20000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_3: phy@3e30000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e30000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_4: phy@3e40000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e40000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_5: phy@3e50000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e50000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_6: phy@3e60000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e60000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_hsio_7: phy@3e70000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e70000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_0: phy@3e90000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03e90000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_1: phy@3ea0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03ea0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_2: phy@3eb0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03eb0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_3: phy@3ec0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03ec0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_4: phy@3ed0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03ed0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_5: phy@3ee0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03ee0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_6: phy@3ef0000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03ef0000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_nvhs_7: phy@3f00000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f00000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_0: phy@3f20000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f20000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_1: phy@3f30000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f30000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_2: phy@3f40000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f40000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_3: phy@3f50000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f50000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_4: phy@3f60000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f60000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_5: phy@3f70000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f70000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_6: phy@3f80000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f80000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + + p2u_gbe_7: phy@3f90000 { + compatible = "nvidia,tegra234-p2u"; + reg = <0x03f90000 0x10000>; + reg-names = "ctl"; + + #phy-cells = <0>; + }; + hsp_aon: hsp@c150000 { compatible = "nvidia,tegra234-hsp", "nvidia,tegra194-hsp"; reg = <0x0c150000 0x90000>; @@ -1109,6 +1479,10 @@ assigned-clock-parents = <&bpmp TEGRA234_CLK_PLLP_OUT0>; resets = <&bpmp TEGRA234_RESET_I2C2>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 22>, <&gpcdma 22>; + dma-names = "rx", "tx"; }; gen8_i2c: i2c@c250000 { @@ -1125,6 +1499,10 @@ assigned-clock-parents = <&bpmp TEGRA234_CLK_PLLP_OUT0>; resets = <&bpmp TEGRA234_RESET_I2C8>; reset-names = "i2c"; + iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>; + dma-coherent; + dmas = <&gpcdma 0>, <&gpcdma 0>; + dma-names = "rx", "tx"; }; rtc@c2a0000 { @@ -1495,6 +1873,741 @@ status = "okay"; }; + pcie@140a0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CA>; + reg = <0x00 0x140a0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x2a000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x2a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x2a080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <8>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C8_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_8_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_8>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 8>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x32 0x40000000 0x32 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x35 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x2a100000 0x00 0x2a100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE8AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE8AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE8 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@140c0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CB>; + reg = <0x00 0x140c0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x2c000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x2c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x2c080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <9>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C9_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_9_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_9>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 9>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x35 0x40000000 0x35 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x38 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x2c100000 0x00 0x2c100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE9AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE9AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE9 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@140e0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CC>; + reg = <0x00 0x140e0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x2e000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x2e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x2e080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <10>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C10_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_10_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_10>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 10>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x38 0x40000000 0x38 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x3b 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x2e100000 0x00 0x2e100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE10AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE10AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE10 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@14100000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>; + reg = <0x00 0x14100000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x30000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x30040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x30080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <1>; + num-viewport = <8>; + linux,pci-domain = <1>; + + clocks = <&bpmp TEGRA234_CLK_PEX0_C1_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX0_CORE_1_APB>, + <&bpmp TEGRA234_RESET_PEX0_CORE_1>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 1>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x20 0x80000000 0x20 0x80000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */ + <0x02000000 0x0 0x40000000 0x20 0xa8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x30100000 0x00 0x30100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE1R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE1W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE1 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@14120000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>; + reg = <0x00 0x14120000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x32000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x32040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x32080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <1>; + num-viewport = <8>; + linux,pci-domain = <2>; + + clocks = <&bpmp TEGRA234_CLK_PEX0_C2_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX0_CORE_2_APB>, + <&bpmp TEGRA234_RESET_PEX0_CORE_2>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 2>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x20 0xc0000000 0x20 0xc0000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */ + <0x02000000 0x0 0x40000000 0x20 0xe8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x32100000 0x00 0x32100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE2AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE2AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE2 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@14140000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>; + reg = <0x00 0x14140000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x34000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x34040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x34080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <1>; + num-viewport = <8>; + linux,pci-domain = <3>; + + clocks = <&bpmp TEGRA234_CLK_PEX0_C3_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX0_CORE_3_APB>, + <&bpmp TEGRA234_RESET_PEX0_CORE_3>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 3>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x21 0x00000000 0x21 0x00000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */ + <0x02000000 0x0 0x40000000 0x21 0xe8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x34100000 0x00 0x34100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE3R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE3W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE3 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@14160000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4BB>; + reg = <0x00 0x14160000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x36000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x36040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x36080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <4>; + + clocks = <&bpmp TEGRA234_CLK_PEX0_C4_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX0_CORE_4_APB>, + <&bpmp TEGRA234_RESET_PEX0_CORE_4>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 4>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x21 0x40000000 0x21 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x24 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x36100000 0x00 0x36100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE4R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE4W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE4 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@14180000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4BA>; + reg = <0x00 0x14180000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x38000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x38040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x38080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <0>; + + clocks = <&bpmp TEGRA234_CLK_PEX0_C0_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX0_CORE_0_APB>, + <&bpmp TEGRA234_RESET_PEX0_CORE_0>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 0>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x24 0x40000000 0x24 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x27 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x38100000 0x00 0x38100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE0R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE0W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE0 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@141a0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8A>; + reg = <0x00 0x141a0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3a000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x3a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3a080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <8>; + num-viewport = <8>; + linux,pci-domain = <5>; + + clocks = <&bpmp TEGRA234_CLK_PEX1_C5_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX1_CORE_5_APB>, + <&bpmp TEGRA234_RESET_PEX1_CORE_5>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 5>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x27 0x40000000 0x27 0x40000000 0x3 0xe8000000>, /* prefetchable memory (16000 MB) */ + <0x02000000 0x0 0x40000000 0x2b 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x3a100000 0x00 0x3a100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE5R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE5W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE5 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@141c0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4A>; + reg = <0x00 0x141c0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3c000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x3c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3c080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <4>; + num-viewport = <8>; + linux,pci-domain = <6>; + + clocks = <&bpmp TEGRA234_CLK_PEX1_C6_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX1_CORE_6_APB>, + <&bpmp TEGRA234_RESET_PEX1_CORE_6>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 6>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x2b 0x40000000 0x2b 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */ + <0x02000000 0x0 0x40000000 0x2e 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x3c100000 0x00 0x3c100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE6AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE6AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE6 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie@141e0000 { + compatible = "nvidia,tegra234-pcie"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8B>; + reg = <0x00 0x141e0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3e000000 0x0 0x00040000>, /* configuration space (256K) */ + <0x00 0x3e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3e080000 0x0 0x00040000>; /* DBI reg space (256K) */ + reg-names = "appl", "config", "atu_dma", "dbi"; + + #address-cells = <3>; + #size-cells = <2>; + device_type = "pci"; + num-lanes = <8>; + num-viewport = <8>; + linux,pci-domain = <7>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C7_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_7_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_7>; + reset-names = "apb", "core"; + + interrupts = , /* controller interrupt */ + ; /* MSI interrupt */ + interrupt-names = "intr", "msi"; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0>; + interrupt-map = <0 0 0 0 &gic GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>; + + nvidia,bpmp = <&bpmp 7>; + + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + bus-range = <0x0 0xff>; + + ranges = <0x43000000 0x2e 0x40000000 0x2e 0x40000000 0x3 0xe8000000>, /* prefetchable memory (16000 MB) */ + <0x02000000 0x0 0x40000000 0x32 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */ + <0x01000000 0x0 0x3e100000 0x00 0x3e100000 0x0 0x00100000>; /* downstream I/O (1 MB) */ + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE7AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE7AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE7 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie-ep@141a0000 { + compatible = "nvidia,tegra234-pcie-ep"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8A>; + reg = <0x00 0x141a0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3a080000 0x0 0x00040000>, /* DBI reg space (256K) */ + <0x27 0x40000000 0x4 0x00000000>; /* Address Space (16G) */ + reg-names = "appl", "atu_dma", "dbi", "addr_space"; + + num-lanes = <8>; + + clocks = <&bpmp TEGRA234_CLK_PEX1_C5_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX1_CORE_5_APB>, + <&bpmp TEGRA234_RESET_PEX1_CORE_5>; + reset-names = "apb", "core"; + + interrupts = ; /* controller interrupt */ + interrupt-names = "intr"; + + nvidia,bpmp = <&bpmp 5>; + + nvidia,enable-ext-refclk; + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE5R &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE5W &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE5 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie-ep@141c0000{ + compatible = "nvidia,tegra234-pcie-ep"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4A>; + reg = <0x00 0x141c0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3c080000 0x0 0x00040000>, /* DBI space (256K) */ + <0x2b 0x40000000 0x3 0x00000000>; /* Address Space (12G) */ + reg-names = "appl", "atu_dma", "dbi", "addr_space"; + + num-lanes = <4>; + + clocks = <&bpmp TEGRA234_CLK_PEX1_C6_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX1_CORE_6_APB>, + <&bpmp TEGRA234_RESET_PEX1_CORE_6>; + reset-names = "apb", "core"; + + interrupts = ; /* controller interrupt */ + interrupt-names = "intr"; + + nvidia,bpmp = <&bpmp 6>; + + nvidia,enable-ext-refclk; + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE6AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE6AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE6 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie-ep@141e0000{ + compatible = "nvidia,tegra234-pcie-ep"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8B>; + reg = <0x00 0x141e0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x3e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x3e080000 0x0 0x00040000>, /* DBI space (256K) */ + <0x2e 0x40000000 0x4 0x00000000>; /* Address Space (16G) */ + reg-names = "appl", "atu_dma", "dbi", "addr_space"; + + num-lanes = <8>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C7_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_7_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_7>; + reset-names = "apb", "core"; + + interrupts = ; /* controller interrupt */ + interrupt-names = "intr"; + + nvidia,bpmp = <&bpmp 7>; + + nvidia,enable-ext-refclk; + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE7AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE7AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE7 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + + pcie-ep@140e0000{ + compatible = "nvidia,tegra234-pcie-ep"; + power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CC>; + reg = <0x00 0x140e0000 0x0 0x00020000>, /* appl registers (128K) */ + <0x00 0x2e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K) */ + <0x00 0x2e080000 0x0 0x00040000>, /* DBI space (256K) */ + <0x38 0x40000000 0x3 0x00000000>; /* Address Space (12G) */ + reg-names = "appl", "atu_dma", "dbi", "addr_space"; + + num-lanes = <4>; + + clocks = <&bpmp TEGRA234_CLK_PEX2_C10_CORE>; + clock-names = "core"; + + resets = <&bpmp TEGRA234_RESET_PEX2_CORE_10_APB>, + <&bpmp TEGRA234_RESET_PEX2_CORE_10>; + reset-names = "apb", "core"; + + interrupts = ; /* controller interrupt */ + interrupt-names = "intr"; + + nvidia,bpmp = <&bpmp 10>; + + nvidia,enable-ext-refclk; + nvidia,aspm-cmrt-us = <60>; + nvidia,aspm-pwr-on-t-us = <20>; + nvidia,aspm-l0s-entrance-latency-us = <3>; + + interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE10AR &emc>, + <&mc TEGRA234_MEMORY_CLIENT_PCIE10AW &emc>; + interconnect-names = "dma-mem", "write"; + iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE10 0x1000>; + iommu-map-mask = <0x0>; + dma-coherent; + + status = "disabled"; + }; + sram@40000000 { compatible = "nvidia,tegra234-sysram", "mmio-sram"; reg = <0x0 0x40000000 0x0 0x80000>; diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile index 1d86a33de528c6cf1a8dbbcd3e31da6cecad5626..d7669a7cee9f7a8a69d78d528ddb7b12687dd5dc 100644 --- a/arch/arm64/boot/dts/qcom/Makefile +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -15,6 +15,9 @@ dtb-$(CONFIG_ARCH_QCOM) += msm8916-longcheer-l8910.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-mtp.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-a3u-eur.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-a5u-eur.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-e5.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-e7.dtb +dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-grandmax.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-j5.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-samsung-serranove.dtb dtb-$(CONFIG_ARCH_QCOM) += msm8916-wingtech-wt88047.dtb @@ -101,8 +104,11 @@ dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-wormdingler-rev1-boe-rt5682s.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7180-trogdor-r1-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-crd.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-evoker-r0.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-herobrine-r1.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-villager-r0.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-villager-r1.dtb +dtb-$(CONFIG_ARCH_QCOM) += sc7280-herobrine-villager-r1-lte.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-idp.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-idp2.dtb dtb-$(CONFIG_ARCH_QCOM) += sc7280-crd-r3.dtb @@ -152,3 +158,4 @@ dtb-$(CONFIG_ARCH_QCOM) += sm8350-sony-xperia-sagami-pdx214.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8350-sony-xperia-sagami-pdx215.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8450-hdk.dtb dtb-$(CONFIG_ARCH_QCOM) += sm8450-qrd.dtb +dtb-$(CONFIG_ARCH_QCOM) += sm8450-sony-xperia-nagara-pdx223.dtb diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts index c1cb1ba5173cc4d62a1d382d71d1a9102bc0b9e8..5cdc7ac1a9c069c6b5c2243a0a247bb5c6f743eb 100644 --- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts +++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts @@ -14,6 +14,7 @@ #include #include #include +#include /* * GPIO name legend: proper name = the GPIO line is used as GPIO @@ -502,20 +503,20 @@ &pcie0 { status = "okay"; - perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>; vddpe-3v3-supply = <&wlan_en>; vdda-supply = <&vreg_l28a_0p925>; }; &pcie1 { status = "okay"; - perst-gpio = <&tlmm 130 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>; vdda-supply = <&vreg_l28a_0p925>; }; &pcie2 { status = "okay"; - perst-gpio = <&tlmm 114 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>; vdda-supply = <&vreg_l28a_0p925>; }; @@ -1064,7 +1065,7 @@ }; codec { - sound-dai = <&wcd9335 6>; + sound-dai = <&wcd9335 AIF4_PB>; }; }; @@ -1079,7 +1080,7 @@ }; codec { - sound-dai = <&wcd9335 1>; + sound-dai = <&wcd9335 AIF1_CAP>; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts index 567b331065560ff256a64d7cb3b3394e64991142..92f264891d84be1af3bc3d598937af7229753d7b 100644 --- a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts +++ b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts @@ -368,7 +368,7 @@ bus-width = <4>; - cd-gpios = <&tlmm 38 0x1>; + cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>; vmmc-supply = <&vreg_l21a_2p95>; vqmmc-supply = <&vreg_l13a_2p95>; diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi index aaad7d9059f648374bd3fd8f0a08889fddaadfd4..a7c7ca980a71c7b6ec91d9b47b9034a376bc5fd3 100644 --- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi @@ -129,12 +129,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x80>; - #hwlock-cells = <1>; - }; - pmuv8: pmu { compatible = "arm,cortex-a53-pmu"; interrupts = ; }; - tcsr_mutex_regs: syscon@1905000 { - compatible = "syscon"; - reg = <0x0 0x01905000 0x0 0x8000>; + tcsr_mutex: hwlock@1905000 { + compatible = "qcom,ipq6018-tcsr-mutex", "qcom,tcsr-mutex"; + reg = <0x0 0x01905000 0x0 0x1000>; + #hwlock-cells = <1>; }; tcsr: syscon@1937000 { - compatible = "syscon"; + compatible = "qcom,tcsr-ipq6018", "syscon"; reg = <0x0 0x01937000 0x0 0x21000>; }; diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts index 81dc3a0bcd7d8c80fb272dac129bb3705af72605..7143c936de61e7f0528e747159913b5052d657ba 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts +++ b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts @@ -51,12 +51,12 @@ &pcie0 { status = "okay"; - perst-gpio = <&tlmm 61 0x1>; + perst-gpios = <&tlmm 61 0x1>; }; &pcie1 { status = "okay"; - perst-gpio = <&tlmm 58 0x1>; + perst-gpios = <&tlmm 58 0x1>; }; &pcie_phy0 { diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi b/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi index 40415d988e4ab81ee161ff9a19ba80d7fab24102..db4b87944cdf2e0eea05f00e864ace10f4b18ce5 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi @@ -39,12 +39,12 @@ &pcie0 { status = "ok"; - perst-gpio = <&tlmm 58 0x1>; + perst-gpios = <&tlmm 58 0x1>; }; &pcie1 { status = "ok"; - perst-gpio = <&tlmm 61 0x1>; + perst-gpios = <&tlmm 61 0x1>; }; &pcie_phy0 { diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi index d53675fc15959ab385e6bc7f8b1c5bf1814caa70..a47acf9bdf24f1c22f086aee7855f48c03d8bb30 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi @@ -199,7 +199,7 @@ pcie_qmp0: phy@86000 { compatible = "qcom,ipq8074-qmp-pcie-phy"; - reg = <0x00086000 0x1000>; + reg = <0x00086000 0x1c4>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -227,7 +227,7 @@ pcie_qmp1: phy@8e000 { compatible = "qcom,ipq8074-qmp-pcie-phy"; - reg = <0x0008e000 0x1000>; + reg = <0x0008e000 0x1c4>; #address-cells = <1>; #size-cells = <1>; ranges; @@ -383,7 +383,7 @@ sdhc_1: mmc@7824900 { compatible = "qcom,sdhci-msm-v4"; reg = <0x7824900 0x500>, <0x7824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi index 10f6509a87095fdfbfe521b165c5f9f3667452f6..3255bd3fcb55a0afef160b2bc7be78125dca45f5 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi @@ -61,9 +61,9 @@ }; }; - reg_vdd_tsp: regulator-vdd-tsp { + reg_vdd_tsp_a: regulator-vdd-tsp-a { compatible = "regulator-fixed"; - regulator-name = "vdd_tsp"; + regulator-name = "vdd_tsp_a"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts index bc198a2eea25d1f3002dcd62b2cf948a98d49bca..6db5f78ca286fdeba597bac89f4455849b44302c 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts @@ -73,8 +73,8 @@ touchscreen-size-x = <540>; touchscreen-size-y = <960>; - vdd-supply = <®_vdd_tsp>; - vddo-supply = <&pm8916_l6>; + vcca-supply = <®_vdd_tsp_a>; + vdd-supply = <&pm8916_l6>; pinctrl-names = "default"; pinctrl-0 = <&ts_int_default>; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts index 7f2ab1891d91c39daee6150ded58a2032ca1b7c0..5fb8ecd0c9ca33feedd022748edd50eaebbf1e95 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts @@ -42,7 +42,7 @@ touchscreen-size-x = <720>; touchscreen-size-y = <1280>; - avdd-supply = <®_vdd_tsp>; + avdd-supply = <®_vdd_tsp_a>; vdd-supply = <&pm8916_l6>; pinctrl-names = "default"; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..542010fdfb8a687d7ea951cc22608dad49fec967 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "msm8916-samsung-a2015-common.dtsi" + +/ { + haptic { + compatible = "regulator-haptic"; + haptic-supply = <®_motor_vdd>; + min-microvolt = <3300000>; + max-microvolt = <3300000>; + }; + + i2c-muic { + /* SM5504 MUIC instead of SM5502 */ + /delete-node/ extcon@25; + + muic: extcon@14 { + compatible = "siliconmitus,sm5504-muic"; + reg = <0x14>; + + interrupt-parent = <&msmgpio>; + interrupts = <12 IRQ_TYPE_EDGE_FALLING>; + + pinctrl-names = "default"; + pinctrl-0 = <&muic_int_default>; + }; + }; + + reg_motor_vdd: regulator-motor-vdd { + compatible = "regulator-fixed"; + regulator-name = "motor_vdd"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&msmgpio 76 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-names = "default"; + pinctrl-0 = <&motor_en_default>; + }; + + reg_touch_key: regulator-touch-key { + compatible = "regulator-fixed"; + regulator-name = "touch_key"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&msmgpio 97 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-names = "default"; + pinctrl-0 = <&tkey_en_default>; + }; +}; + +&blsp_i2c2 { + /* lis2hh12 accelerometer instead of BMC150 */ + status = "disabled"; + + /delete-node/ accelerometer@10; + /delete-node/ magnetometer@12; +}; + +&touchkey { + vcc-supply = <®_touch_key>; + vdd-supply = <®_touch_key>; +}; + +&msmgpio { + motor_en_default: motor-en-default { + pins = "gpio76"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; + + tkey_en_default: tkey-en-default { + pins = "gpio97"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts new file mode 100644 index 0000000000000000000000000000000000000000..777eb934eb4b0bd723f0b234ba4c05ed62767bcf --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/dts-v1/; + +#include "msm8916-samsung-e2015-common.dtsi" + +/* + * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some + * variants. + * Unfortunately, the firmware is signed and cannot be replaced easily. + * There seems to be no way to boot ARM64 kernels on 32-bit devices at the + * moment, even though the hardware would support it. + * + * However, it is possible to use this device tree by compiling an ARM32 kernel + * instead. For clarity and build testing this device tree is maintained next + * to the other MSM8916 device trees. However, it is actually used through + * arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts + */ + +/ { + model = "Samsung Galaxy E5"; + compatible = "samsung,e5", "qcom,msm8916"; + chassis-type = "handset"; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts new file mode 100644 index 0000000000000000000000000000000000000000..b412b61ca25853d69049becfd8b50daa656b5876 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/dts-v1/; + +#include "msm8916-samsung-e2015-common.dtsi" + +/* + * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some + * variants. + * Unfortunately, the firmware is signed and cannot be replaced easily. + * There seems to be no way to boot ARM64 kernels on 32-bit devices at the + * moment, even though the hardware would support it. + * + * However, it is possible to use this device tree by compiling an ARM32 kernel + * instead. For clarity and build testing this device tree is maintained next + * to the other MSM8916 device trees. However, it is actually used through + * arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts + */ + +/ { + model = "Samsung Galaxy E7"; + compatible = "samsung,e7", "qcom,msm8916"; + chassis-type = "handset"; +}; + +&pm8916_l17 { + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts new file mode 100644 index 0000000000000000000000000000000000000000..bc713469897899a610e98259be7b9c778597a29a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/dts-v1/; + +#include "msm8916-samsung-e2015-common.dtsi" +#include + +/* + * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some + * variants. + * Unfortunately, the firmware is signed and cannot be replaced easily. + * There seems to be no way to boot ARM64 kernels on 32-bit devices at the + * moment, even though the hardware would support it. + * + * However, it is possible to use this device tree by compiling an ARM32 kernel + * instead. For clarity and build testing this device tree is maintained next + * to the other MSM8916 device trees. However, it is actually used through + * arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts + */ + +/ { + model = "Samsung Galaxy Grand Max"; + compatible = "samsung,grandmax", "qcom,msm8916"; + chassis-type = "handset"; + + /delete-node/ gpio-hall-sensor; + /delete-node/ i2c-nfc; + /delete-node/ i2c-tkey; + + gpio-leds { + compatible = "gpio-leds"; + keyled { + gpios = <&msmgpio 60 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&gpio_leds_default>; + }; + }; +}; + +®_motor_vdd { + gpio = <&msmgpio 72 GPIO_ACTIVE_HIGH>; +}; + +®_touch_key { + status = "disabled"; +}; + +&msmgpio { + gpio_leds_default: gpio-led-default { + pins = "gpio60"; + function = "gpio"; + + drive-strength = <2>; + bias-disable; + }; +}; + +&motor_en_default { + pins = "gpio72"; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts index 439e89cf7878852b6b52691586eaf700a46975ff..bbd6bb3f4fd7b34416a3cf7b08de7bfd5d1dc1e0 100644 --- a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts +++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts @@ -210,6 +210,15 @@ pinctrl-names = "default"; pinctrl-0 = <&imu_irq_default>; }; + + magnetometer@2e { + compatible = "yamaha,yas537"; + reg = <0x2e>; + + mount-matrix = "0", "1", "0", + "1", "0", "0", + "0", "0", "-1"; + }; }; &blsp_i2c4 { diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi index 48bc2e09128d92e3aebd254b71586b926f8241d2..a831064700ee89df47c8db12dcfa1a5ffa920cb3 100644 --- a/arch/arm64/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi @@ -936,6 +936,20 @@ #reset-cells = <1>; #power-domain-cells = <1>; reg = <0x01800000 0x80000>; + clocks = <&xo_board>, + <&sleep_clk>, + <&dsi_phy0 1>, + <&dsi_phy0 0>, + <0>, + <0>, + <0>; + clock-names = "xo", + "sleep_clk", + "dsi0pll", + "dsi0pllbyte", + "ext_mclk", + "ext_pri_i2s", + "ext_sec_i2s"; }; tcsr_mutex: hwlock@1905000 { @@ -1469,7 +1483,7 @@ sdhc_1: mmc@7824000 { compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4"; reg = <0x07824900 0x11c>, <0x07824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -1487,7 +1501,7 @@ sdhc_2: mmc@7864000 { compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4"; reg = <0x07864900 0x11c>, <0x07864000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; diff --git a/arch/arm64/boot/dts/qcom/msm8953.dtsi b/arch/arm64/boot/dts/qcom/msm8953.dtsi index 8416a45ca4fd4dbcc13663fb7a599874591c0788..6b992a6d56c16d5151785df04fa2053c67ee7fad 100644 --- a/arch/arm64/boot/dts/qcom/msm8953.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8953.dtsi @@ -722,7 +722,7 @@ }; tcsr_phy_clk_scheme_sel: syscon@193f044 { - compatible = "syscon"; + compatible = "qcom,tcsr-msm8953", "syscon"; reg = <0x193f044 0x4>; }; @@ -799,7 +799,7 @@ compatible = "qcom,msm8953-sdhci", "qcom,sdhci-msm-v4"; reg = <0x7824900 0x500>, <0x7824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -859,7 +859,7 @@ compatible = "qcom,msm8953-sdhci", "qcom,sdhci-msm-v4"; reg = <0x7864900 0x500>, <0x7864000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; diff --git a/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts b/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts index cbe11c060df93647efad4fda8c19426659d4559d..c4e87d0aec42f425c4f7d9d3524784a5b3fde630 100644 --- a/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts +++ b/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts @@ -17,7 +17,7 @@ chassis-type = "handset"; /* required for bootloader to select correct board */ - qcom,msm-id = <251 0 252 0>; + qcom,msm-id = <251 0>, <252 0>; qcom,pmic-id = <65545 65546 0 0>; qcom,board-id = <12 0>; diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi index 61ec905025b074782da28e70fae175ef1ef9f39e..f9d8bd09e074ac3d5140b4a6bba8c68ffe5b66f7 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi @@ -499,7 +499,7 @@ linux,code = ; }; - volwnkey { + resin { compatible = "qcom,pm8941-resin"; interrupts = <0 8 1 IRQ_TYPE_EDGE_BOTH>; debounce = <15625>; diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi index f430d797196f5210ca2da26d7d7ee8b452623dea..ff60b7004d260cab36d771f0d19e3fca5dd04a25 100644 --- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi @@ -471,7 +471,7 @@ &sdhc2 { status = "okay"; - cd-gpios = <&tlmm 100 0>; + cd-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>; vmmc-supply = <&pm8994_l21>; vqmmc-supply = <&pm8994_l13>; }; diff --git a/arch/arm64/boot/dts/qcom/msm8994.dtsi b/arch/arm64/boot/dts/qcom/msm8994.dtsi index 8bc6c070e3066f19af54ceb11c43b3f3d29eceba..ded5b7ceeaf97fbbca2485eb3aac6eaeafeee304 100644 --- a/arch/arm64/boot/dts/qcom/msm8994.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8994.dtsi @@ -6,6 +6,7 @@ #include #include #include +#include #include / { @@ -164,12 +165,6 @@ reg = <0 0x80000000 0 0>; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x80>; - #hwlock-cells = <1>; - }; - pmu { compatible = "arm,cortex-a53-pmu"; interrupts = ; @@ -464,7 +459,7 @@ sdhc1: mmc@f9824900 { compatible = "qcom,msm8994-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf9824900 0x1a0>, <0xf9824000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -487,7 +482,7 @@ sdhc2: mmc@f98a4900 { compatible = "qcom,msm8994-sdhci", "qcom,sdhci-msm-v4"; reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -502,7 +497,7 @@ pinctrl-0 = <&sdc2_clk_on &sdc2_cmd_on &sdc2_data_on>; pinctrl-1 = <&sdc2_clk_off &sdc2_cmd_off &sdc2_data_off>; - cd-gpios = <&tlmm 100 0>; + cd-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>; bus-width = <4>; status = "disabled"; }; @@ -762,9 +757,10 @@ #interrupt-cells = <4>; }; - tcsr_mutex_regs: syscon@fd484000 { - compatible = "syscon"; - reg = <0xfd484000 0x2000>; + tcsr_mutex: hwlock@fd484000 { + compatible = "qcom,msm8994-tcsr-mutex", "qcom,tcsr-mutex"; + reg = <0xfd484000 0x1000>; + #hwlock-cells = <1>; }; tlmm: pinctrl@fd510000 { diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi index e165b5e890a09bca5a3cb8851d7b91709e13a1c4..ca7c8d2e1d3d98229466389e9a8e9476fcccbb64 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi @@ -196,8 +196,8 @@ &pcie0 { status = "okay"; - perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>; - wake-gpio = <&tlmm 37 GPIO_ACTIVE_HIGH>; + perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>; + wake-gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>; vddpe-3v3-supply = <&wlan_en>; vdda-supply = <&pm8994_l28>; }; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi index 6276499798912b110a8a911813b7000ddd6cc37f..77819186086ac54c722abfc5ac4a857da94fb1b4 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi @@ -58,6 +58,14 @@ }; }; + irled { + compatible = "pwm-ir-tx"; + pwms = <&pm8994_lpg 1 1000000>; + + pinctrl-names = "default"; + pinctrl-0 = <&irled_default>; + }; + reserved-memory { memory@88800000 { reg = <0x0 0x88800000 0x0 0x1400000>; @@ -297,6 +305,41 @@ linux,code = ; }; +&pm8994_lpg { + status = "okay"; + + qcom,power-source = <1>; +}; + +&pmi8994_lpg { + status = "okay"; + + qcom,power-source = <1>; + + multi-led { + color = ; + function = LED_FUNCTION_STATUS; + + #address-cells = <1>; + #size-cells = <0>; + + led@1 { + reg = <1>; + color = ; + }; + + led@2 { + reg = <2>; + color = ; + }; + + led@3 { + reg = <3>; + color = ; + }; + }; +}; + &slpi_pil { status = "okay"; @@ -608,6 +651,15 @@ }; &pm8994_gpios { + irled_default: irled-default-state { + pins = "gpio5"; + function = PMIC_GPIO_FUNC_FUNC1; + output-low; + qcom,drive-strength = ; + power-source = ; + bias-disable; + }; + wlan_en_default: wlan-en-state { pins = "gpio8"; function = PMIC_GPIO_FUNC_NORMAL; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts index 25f30ec277c127c52a98656a6eed4d85a18c80af..4e5264f4116a07c3120d76aea6839f7b75a22532 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts @@ -8,6 +8,7 @@ #include "msm8996-xiaomi-common.dtsi" #include #include +#include #include / { @@ -193,7 +194,7 @@ }; codec { - sound-dai = <&wcd9335 6>; + sound-dai = <&wcd9335 AIF4_PB>; }; }; @@ -208,7 +209,7 @@ }; codec { - sound-dai = <&wcd9335 1>; + sound-dai = <&wcd9335 AIF1_CAP>; }; }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts index 30a9e4bed4af4e3720200f0092c7441f87320533..79be5fb1295b0858fc6d25dd79de594d483ab6f7 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts @@ -9,6 +9,7 @@ #include "pmi8996.dtsi" #include #include +#include / { model = "Xiaomi Mi Note 2"; @@ -116,6 +117,25 @@ "qcom/msm8996/scorpio/modem.mbn"; }; +&pm8994_lpg { + pinctrl-names = "default"; + pinctrl-0 = <&keypad_default>; + + led@3 { + reg = <3>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; + function-enumerator = <1>; + }; + + led@6 { + reg = <6>; + color = ; + function = LED_FUNCTION_KBD_BACKLIGHT; + function-enumerator = <0>; + }; +}; + &q6asmdai { dai@0 { reg = <0>; @@ -171,7 +191,7 @@ }; codec { - sound-dai = <&wcd9335 6>; + sound-dai = <&wcd9335 AIF4_PB>; }; }; @@ -186,7 +206,7 @@ }; codec { - sound-dai = <&wcd9335 1>; + sound-dai = <&wcd9335 AIF1_CAP>; }; }; }; @@ -258,6 +278,15 @@ "PMIC_SLB", /* GPIO_20 */ "UIM_BATT_ALARM", /* GPIO_21 */ "NC"; /* GPIO_22 */ + + keypad_default: keypad-default-state { + pins = "gpio7", "gpio10"; + function = PMIC_GPIO_FUNC_FUNC1; + output-low; + qcom,drive-strength = ; + power-source = ; + bias-disable; + }; }; &pm8994_mpps { diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi index 742eac4ce9b3537d089be71077fb76931a4ab3a7..c0a2baffa49d530f8663128f5cfffe1881e23001 100644 --- a/arch/arm64/boot/dts/qcom/msm8996.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -359,16 +360,10 @@ firmware { scm { compatible = "qcom,scm-msm8996", "qcom,scm"; - qcom,dload-mode = <&tcsr 0x13000>; + qcom,dload-mode = <&tcsr_2 0x13000>; }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - memory@80000000 { device_type = "memory"; /* We expect the bootloader to fill in the reg */ @@ -838,12 +833,18 @@ <&rpmcc RPM_SMD_PCNOC_A_CLK>; }; - tcsr_mutex_regs: syscon@740000 { - compatible = "syscon"; - reg = <0x00740000 0x40000>; + tcsr_mutex: hwlock@740000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x00740000 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_1: syscon@760000 { + compatible = "qcom,tcsr-msm8996", "syscon"; + reg = <0x00760000 0x20000>; }; - tcsr: syscon@7a0000 { + tcsr_2: syscon@7a0000 { compatible = "qcom,tcsr-msm8996", "syscon"; reg = <0x007a0000 0x18000>; }; @@ -1161,9 +1162,13 @@ "hdmi_phy"; clocks = <&mmcc MDSS_AHB_CLK>, - <&gcc GCC_HDMI_CLKREF_CLK>; + <&gcc GCC_HDMI_CLKREF_CLK>, + <&xo_board>; clock-names = "iface", - "ref"; + "ref", + "xo"; + + #clock-cells = <0>; status = "disabled"; }; @@ -2413,7 +2418,7 @@ qcom,smem-states = <&mpss_smp2p_out 0>; qcom,smem-state-names = "stop"; - qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>; + qcom,halt-regs = <&tcsr_1 0x3000 0x5000 0x4000>; status = "disabled"; @@ -3018,7 +3023,7 @@ sdhc1: mmc@7464900 { compatible = "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4"; reg = <0x07464900 0x11c>, <0x07464000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -3042,7 +3047,7 @@ sdhc2: mmc@74a4900 { compatible = "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4"; reg = <0x074a4900 0x314>, <0x074a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; @@ -3337,7 +3342,7 @@ interrupt-names = "intr1", "intr2"; interrupt-controller; #interrupt-cells = <1>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; slim-ifc-dev = <&tasha_ifd>; @@ -3499,7 +3504,7 @@ }; saw3: syscon@9a10000 { - compatible = "syscon"; + compatible = "qcom,tcsr-msm8996", "syscon"; reg = <0x09a10000 0x1000>; }; diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 02d21bff21980a04feafdd5575254638ab857f4e..f05f16ac5cc18b968e575e8d4b1401cdfb707131 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -308,12 +308,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - psci { compatible = "arm,psci-1.0"; method = "smc"; @@ -1047,9 +1041,15 @@ }; }; - tcsr_mutex_regs: syscon@1f40000 { - compatible = "syscon"; - reg = <0x01f40000 0x40000>; + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x01f40000 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_regs_1: syscon@1f60000 { + compatible = "qcom,msm8998-tcsr", "syscon"; + reg = <0x01f60000 0x20000>; }; tlmm: pinctrl@3400000 { @@ -1340,7 +1340,7 @@ resets = <&gcc GCC_MSS_RESTART>; reset-names = "mss_restart"; - qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>; + qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>; power-domains = <&rpmpd MSM8998_VDDCX>, <&rpmpd MSM8998_VDDMX>; @@ -2076,9 +2076,9 @@ }; sdhc2: mmc@c0a4900 { - compatible = "qcom,sdhci-msm-v4"; + compatible = "qcom,msm8998-sdhci", "qcom,sdhci-msm-v4"; reg = <0x0c0a4900 0x314>, <0x0c0a4000 0x800>; - reg-names = "hc_mem", "core_mem"; + reg-names = "hc", "core"; interrupts = , ; diff --git a/arch/arm64/boot/dts/qcom/pm6150l.dtsi b/arch/arm64/boot/dts/qcom/pm6150l.dtsi index 7aa2ef90cb6ac9d1c6ff11f80ebe835bf2ea0e92..f02c223ef4485d2fbe9d5262a02072094b1a829f 100644 --- a/arch/arm64/boot/dts/qcom/pm6150l.dtsi +++ b/arch/arm64/boot/dts/qcom/pm6150l.dtsi @@ -20,10 +20,29 @@ #size-cells = <0>; #io-channel-cells = <1>; + adc-chan@0 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "ref_gnd"; + }; + + adc-chan@1 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "vref_1p25"; + }; + adc-chan@6 { reg = ; + qcom,pre-scaling = <1 1>; label = "die_temp"; }; + + adc-chan@83 { + reg = ; + qcom,pre-scaling = <1 3>; + label = "vph_pwr"; + }; }; pm6150l_adc_tm: adc-tm@3500 { diff --git a/arch/arm64/boot/dts/qcom/pm660.dtsi b/arch/arm64/boot/dts/qcom/pm660.dtsi index d0eefbb516634162526137d742132fe0efca6ffe..e1622b16c08bd9dbe185113605672c09ae541d84 100644 --- a/arch/arm64/boot/dts/qcom/pm660.dtsi +++ b/arch/arm64/boot/dts/qcom/pm660.dtsi @@ -187,7 +187,7 @@ #address-cells = <1>; #size-cells = <0>; - pm660_spmi_regulators: pm660-regulators { + pm660_spmi_regulators: regulators { compatible = "qcom,pm660-regulators"; }; }; diff --git a/arch/arm64/boot/dts/qcom/pm660l.dtsi b/arch/arm64/boot/dts/qcom/pm660l.dtsi index c7945470ffee8c777d92bc0c3b8d3733323b25c1..8aa0a5078772be4553f8ce9a15b6c36e9ac9c784 100644 --- a/arch/arm64/boot/dts/qcom/pm660l.dtsi +++ b/arch/arm64/boot/dts/qcom/pm660l.dtsi @@ -65,7 +65,7 @@ #address-cells = <1>; #size-cells = <0>; - pm660l_lpg: lpg@b100 { + pm660l_lpg: pwm { compatible = "qcom,pm660l-lpg"; status = "disabled"; @@ -81,7 +81,7 @@ status = "disabled"; }; - pm660l_spmi_regulators: pm660l-regulators { + pm660l_spmi_regulators: regulators { compatible = "qcom,pm660l-regulators"; }; }; diff --git a/arch/arm64/boot/dts/qcom/pm7250b.dtsi b/arch/arm64/boot/dts/qcom/pm7250b.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..61f7a63451505b45a9c0150c67213763470e8665 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/pm7250b.dtsi @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (C) 2022 Luca Weiss + */ + +#include +#include + +/ { + thermal-zones { + pm7250b-thermal { + polling-delay-passive = <100>; + polling-delay = <0>; + + thermal-sensors = <&pm7250b_temp>; + + trips { + trip0 { + temperature = <95000>; + hysteresis = <0>; + type = "passive"; + }; + + trip1 { + temperature = <115000>; + hysteresis = <0>; + type = "hot"; + }; + + trip2 { + temperature = <145000>; + hysteresis = <0>; + type = "critical"; + }; + }; + }; + }; +}; + +&spmi_bus { + pmic@2 { + compatible = "qcom,pm7250b", "qcom,spmi-pmic"; + reg = <0x2 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + + pm7250b_temp: temp-alarm@2400 { + compatible = "qcom,spmi-temp-alarm"; + reg = <0x2400>; + interrupts = <0x2 0x24 0x0 IRQ_TYPE_EDGE_BOTH>; + io-channels = <&pm7250b_adc ADC5_DIE_TEMP>; + io-channel-names = "thermal"; + #thermal-sensor-cells = <0>; + }; + + pm7250b_adc: adc@3100 { + compatible = "qcom,spmi-adc5"; + reg = <0x3100>; + #address-cells = <1>; + #size-cells = <0>; + #io-channel-cells = <1>; + interrupts = <0x2 0x31 0x0 IRQ_TYPE_EDGE_RISING>; + + adc-chan@0 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "ref_gnd"; + }; + + adc-chan@1 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "vref_1p25"; + }; + + adc-chan@2 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "die_temp"; + }; + + adc-chan@7 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "usb_in_i_uv"; + }; + + adc-chan@8 { + reg = ; + qcom,pre-scaling = <1 16>; + label = "usb_in_v_div_16"; + }; + + adc-chan@9 { + reg = ; + qcom,pre-scaling = <1 1>; + label = "chg_temp"; + }; + + adc-chan@e { + reg = ; + qcom,hw-settle-time = <200>; + qcom,pre-scaling = <1 1>; + label = "smb1390_therm"; + }; + + adc-chan@1e { + reg = ; + qcom,pre-scaling = <1 6>; + label = "chg_mid"; + }; + + adc-chan@83 { + reg = ; + qcom,pre-scaling = <1 3>; + label = "vph_pwr"; + }; + + adc-chan@84 { + reg = ; + qcom,pre-scaling = <1 3>; + label = "vbat_sns"; + }; + + adc-chan@99 { + reg = ; + qcom,pre-scaling = <1 3>; + label = "chg_sbux"; + }; + }; + + pm7250b_adc_tm: adc-tm@3500 { + compatible = "qcom,spmi-adc-tm5"; + reg = <0x3500>; + interrupts = <0x2 0x35 0x0 IRQ_TYPE_EDGE_RISING>; + #thermal-sensor-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + }; + + pmic@3 { + compatible = "qcom,pm7250b", "qcom,spmi-pmic"; + reg = <0x3 SPMI_USID>; + #address-cells = <1>; + #size-cells = <0>; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/pm8150.dtsi b/arch/arm64/boot/dts/qcom/pm8150.dtsi index fd8434215924e605a1d5308b6ed6739e16af57c8..574fa95a2871df7026328192c9ccaa1278fd4149 100644 --- a/arch/arm64/boot/dts/qcom/pm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8150.dtsi @@ -47,7 +47,7 @@ #address-cells = <1>; #size-cells = <0>; - pon: power-on@800 { + pon: pon@800 { compatible = "qcom,pm8998-pon"; reg = <0x0800>; mode-bootloader = <0x2>; diff --git a/arch/arm64/boot/dts/qcom/pm8150b.dtsi b/arch/arm64/boot/dts/qcom/pm8150b.dtsi index 5d1ec3a6cc3c534e9e468ab9d17547143c8f602d..cdded791d96eee88449996e8287484aa4cb6c70a 100644 --- a/arch/arm64/boot/dts/qcom/pm8150b.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8150b.dtsi @@ -46,7 +46,7 @@ #address-cells = <1>; #size-cells = <0>; - power-on@800 { + pon@800 { compatible = "qcom,pm8916-pon"; reg = <0x0800>; @@ -128,7 +128,7 @@ #address-cells = <1>; #size-cells = <0>; - pm8150b_lpg: lpg { + pm8150b_lpg: pwm { compatible = "qcom,pm8150b-lpg"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pm8150l.dtsi b/arch/arm64/boot/dts/qcom/pm8150l.dtsi index c62d023b39a2603ae3ea0a14d096338e143c22c2..135bfb8d629b50a1d572c7b5bf53067b848f4f4d 100644 --- a/arch/arm64/boot/dts/qcom/pm8150l.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8150l.dtsi @@ -46,7 +46,7 @@ #address-cells = <1>; #size-cells = <0>; - power-on@800 { + pon@800 { compatible = "qcom,pm8916-pon"; reg = <0x0800>; @@ -116,7 +116,7 @@ #address-cells = <1>; #size-cells = <0>; - pm8150l_lpg: lpg { + pm8150l_lpg: pwm { compatible = "qcom,pm8150l-lpg"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pm8350c.dtsi b/arch/arm64/boot/dts/qcom/pm8350c.dtsi index e0bbb67717fec06dee5b57dec41828a95c437496..f28e71487d5c764745b0a103821ab5e3d181617d 100644 --- a/arch/arm64/boot/dts/qcom/pm8350c.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8350c.dtsi @@ -30,9 +30,8 @@ #interrupt-cells = <2>; }; - pm8350c_pwm: pwm@e800 { + pm8350c_pwm: pwm { compatible = "qcom,pm8350c-pwm"; - reg = <0xe800>; #pwm-cells = <2>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/qcom/pm8953.dtsi b/arch/arm64/boot/dts/qcom/pm8953.dtsi index 741c538a9ceea48e37086f765bca78ef15dbacf3..a1d36f9ebbd259e41a80e3af516d9a044419afe0 100644 --- a/arch/arm64/boot/dts/qcom/pm8953.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8953.dtsi @@ -45,7 +45,7 @@ #thermal-sensor-cells = <0>; }; - pm8953_vadc: vadc@3100 { + pm8953_vadc: adc@3100 { compatible = "qcom,spmi-vadc"; reg = <0x3100>; interrupts = <0x00 0x31 0x00 0x01>; diff --git a/arch/arm64/boot/dts/qcom/pm8994.dtsi b/arch/arm64/boot/dts/qcom/pm8994.dtsi index ab342397fcd8ff2b18ecdbbe2c8bbda42ef7f374..e92e5ac414d39080f74058feec62a802f4fbcbcc 100644 --- a/arch/arm64/boot/dts/qcom/pm8994.dtsi +++ b/arch/arm64/boot/dts/qcom/pm8994.dtsi @@ -135,7 +135,7 @@ #address-cells = <1>; #size-cells = <0>; - pm8994_lpg: lpg { + pm8994_lpg: pwm { compatible = "qcom,pm8994-lpg"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pmi8994.dtsi b/arch/arm64/boot/dts/qcom/pmi8994.dtsi index 84c44912ec938a85b4ead64eeb0b09703b5d1bb6..542c215dde107027de895363673e8d3e537743a8 100644 --- a/arch/arm64/boot/dts/qcom/pmi8994.dtsi +++ b/arch/arm64/boot/dts/qcom/pmi8994.dtsi @@ -21,7 +21,7 @@ }; pmi8994_mpps: mpps@a000 { - compatible = "qcom,pmi8994-mpp"; + compatible = "qcom,pmi8994-mpp", "qcom,spmi-mpp"; reg = <0xa000>; gpio-controller; gpio-ranges = <&pmi8994_mpps 0 0 4>; @@ -37,7 +37,7 @@ #address-cells = <1>; #size-cells = <0>; - pmi8994_lpg: lpg { + pmi8994_lpg: pwm { compatible = "qcom,pmi8994-lpg"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pmi8998.dtsi b/arch/arm64/boot/dts/qcom/pmi8998.dtsi index 6d3d212560c18ed7d49a97e567dda165ad7528c2..3852a012bb0fdd704d417ab823b3a0cc6ad0f6a8 100644 --- a/arch/arm64/boot/dts/qcom/pmi8998.dtsi +++ b/arch/arm64/boot/dts/qcom/pmi8998.dtsi @@ -42,7 +42,7 @@ }; }; - pmi8998_lpg: lpg { + pmi8998_lpg: pwm { compatible = "qcom,pmi8998-lpg"; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pmk8350.dtsi b/arch/arm64/boot/dts/qcom/pmk8350.dtsi index 0f94c46a144433073e635deefff30d47420e9b6e..a7ec9d11946df654b954ef0a6aeb73b9e2336616 100644 --- a/arch/arm64/boot/dts/qcom/pmk8350.dtsi +++ b/arch/arm64/boot/dts/qcom/pmk8350.dtsi @@ -39,16 +39,13 @@ #address-cells = <1>; #size-cells = <0>; interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "eoc-int-en-set"; #io-channel-cells = <1>; - io-channel-ranges; }; pmk8350_adc_tm: adc-tm@3400 { compatible = "qcom,adc-tm7"; reg = <0x3400>; interrupts = <0x0 0x34 0x0 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "threshold"; #address-cells = <1>; #size-cells = <0>; #thermal-sensor-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi b/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi index 68e9122363aec9bfccf7262a7e491263fd2f5c90..20c5d60c8c2ce41b9dc55e2583268740dae43aba 100644 --- a/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi +++ b/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi @@ -46,7 +46,7 @@ #address-cells = <1>; #size-cells = <0>; - pon: power-on@800 { + pon: pon@800 { compatible = "qcom,pm8916-pon"; reg = <0x0800>; pwrkey { diff --git a/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi b/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi index c307fc6625118ed1ac0fc6be315335fa357c11c1..1da4606e8ee67cb4ea1a6af24c7de8545841cf1a 100644 --- a/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi +++ b/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi @@ -45,7 +45,7 @@ #address-cells = <1>; #size-cells = <0>; - power-on@800 { + pon@800 { compatible = "qcom,pm8916-pon"; reg = <0x0800>; diff --git a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi index 1721ebe5759b9f64ca0070409110968a20f89656..1678ef0f8684f17321b631f0cbc995c446bca3b7 100644 --- a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi @@ -99,7 +99,7 @@ &pcie { status = "okay"; - perst-gpio = <&tlmm 43 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 43 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&perst_state>; diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi index 9ab9900615223f381774c7dc519c13623eba8327..80f2d05595fa658b922db45b63be6304e8cff489 100644 --- a/arch/arm64/boot/dts/qcom/qcs404.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi @@ -295,12 +295,6 @@ hwlocks = <&tcsr_mutex 3>; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - soc: soc@0 { #address-cells = <1>; #size-cells = <1>; @@ -726,13 +720,14 @@ assigned-clock-rates = <19200000>; }; - tcsr_mutex_regs: syscon@1905000 { - compatible = "syscon"; + tcsr_mutex: hwlock@1905000 { + compatible = "qcom,tcsr-mutex"; reg = <0x01905000 0x20000>; + #hwlock-cells = <1>; }; tcsr: syscon@1937000 { - compatible = "syscon"; + compatible = "qcom,qcs404-tcsr", "syscon"; reg = <0x01937000 0x25000>; }; @@ -1297,7 +1292,7 @@ }; pcie: pci@10000000 { - compatible = "qcom,pcie-qcs404", "snps,dw-pcie"; + compatible = "qcom,pcie-qcs404"; reg = <0x10000000 0xf1d>, <0x10000f20 0xa8>, <0x07780000 0x2000>, diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts index ba547ca9fc6bdbfc804588f7a2e11a14118fc921..87ab0e1ecd16d5d5453e975bb009fccb9e9cf847 100644 --- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts @@ -333,9 +333,6 @@ snps,reset-active-low; snps,reset-delays-us = <0 11000 70000>; - snps,ptp-ref-clk-rate = <250000000>; - snps,ptp-req-clk-rate = <96000000>; - snps,mtl-rx-config = <&mtl_rx_setup>; snps,mtl-tx-config = <&mtl_tx_setup>; diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts index 9398f0349944ee3f3f93cca2f0a47498b935fb21..b608b82dff03c177261cb0bc14501bb26aab4d07 100644 --- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts @@ -35,7 +35,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1208000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l5a: ldo5 { @@ -43,7 +42,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l7a: ldo7 { @@ -51,7 +49,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l13a: ldo13 { @@ -59,7 +56,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; @@ -72,7 +68,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l2c: ldo2 { @@ -80,7 +75,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l3c: ldo3 { @@ -96,7 +90,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1208000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l6c: ldo6 { @@ -112,7 +105,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l10c: ldo10 { @@ -141,7 +133,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l7g: ldo7 { @@ -149,7 +140,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l8g: ldo8 { @@ -157,7 +147,6 @@ regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; }; @@ -194,9 +183,10 @@ #size-cells = <0>; pm8450a_gpios: gpio@c000 { - compatible = "qcom,pm8150-gpio"; + compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio"; reg = <0xc000>; gpio-controller; + gpio-ranges = <&pm8450a_gpios 0 0 10>; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; @@ -210,9 +200,10 @@ #size-cells = <0>; pm8450c_gpios: gpio@c000 { - compatible = "qcom,pm8150-gpio"; + compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio"; reg = <0xc000>; gpio-controller; + gpio-ranges = <&pm8450c_gpios 0 0 10>; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; @@ -226,9 +217,10 @@ #size-cells = <0>; pm8450e_gpios: gpio@c000 { - compatible = "qcom,pm8150-gpio"; + compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio"; reg = <0xc000>; gpio-controller; + gpio-ranges = <&pm8450e_gpios 0 0 10>; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; @@ -242,9 +234,10 @@ #size-cells = <0>; pm8450g_gpios: gpio@c000 { - compatible = "qcom,pm8150-gpio"; + compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio"; reg = <0xc000>; gpio-controller; + gpio-ranges = <&pm8450g_gpios 0 0 10>; #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts index 8290d036044a1418f6cf2ca39d47ec0671ac2dbb..edfcd47e1a00f8d642d46effd1537015b678bd46 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts @@ -24,8 +24,6 @@ }; &pm6150_adc { - status = "disabled"; - /delete-node/ skin-temp-thermistor@4e; /delete-node/ charger-thermistor@4f; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts index bfbf26fd2cd45076d007d9cac4c7ebb1e4daea91..d49de65aa9600368d219482ad2ef9a4abd21d934 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts @@ -16,17 +16,6 @@ compatible = "google,lazor-rev0", "qcom,sc7180"; }; -&pp3300_hub { - /* pp3300_l7c is used to power the USB hub */ - /delete-property/regulator-always-on; - /delete-property/regulator-boot-on; -}; - -&pp3300_l7c { - regulator-always-on; - regulator-boot-on; -}; - &sn65dsi86_out { /* * Lane 0 was incorrectly mapped on the cable, but we've now decided @@ -35,3 +24,11 @@ */ lane-polarities = <1 0>; }; + +&usb_hub_2_x { + vdd-supply = <&pp3300_l7c>; +}; + +&usb_hub_3_x { + vdd-supply = <&pp3300_l7c>; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts index d45a59afd7fc41c46b89ab02ffbfc4226f9c4dd3..80c7108bc51b194f1dce54c0b10301a318a93338 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts @@ -16,13 +16,11 @@ compatible = "google,lazor-rev1", "google,lazor-rev2", "qcom,sc7180"; }; -&pp3300_hub { - /* pp3300_l7c is used to power the USB hub */ - /delete-property/regulator-always-on; - /delete-property/regulator-boot-on; + +&usb_hub_2_x { + vdd-supply = <&pp3300_l7c>; }; -&pp3300_l7c { - regulator-always-on; - regulator-boot-on; +&usb_hub_3_x { + vdd-supply = <&pp3300_l7c>; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi index 2cf7d5212c61c38be457ec056faa118f4d13349f..002663d752da36623f7657ac057d9992da2f90c9 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi @@ -55,8 +55,6 @@ ap_ts_pen_1v8: &i2c4 { }; &pm6150_adc { - status = "disabled"; - /delete-node/ charger-thermistor@4f; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts index 764c451c1a857b3e1dd5ef4023bafd21d0a87259..767cb7450c0d81b8d3bc50437ad66c018612a6b9 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts @@ -14,7 +14,7 @@ / { model = "Google Pazquel (Parade,LTE)"; - compatible = "google,pazquel-sku4", "qcom,sc7180"; + compatible = "google,pazquel-sku6", "google,pazquel-sku4", "qcom,sc7180"; }; &ap_sar_sensor_i2c { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts index 76a130bad60ae518a18f10a2b1f1df556ab95f1e..8467ff41e6d539c31f637a29757857781db43de7 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts @@ -34,13 +34,10 @@ /delete-node/ charger-thermistor@0; }; -&pp3300_hub { - /* pp3300_l7c is used to power the USB hub */ - /delete-property/regulator-always-on; - /delete-property/regulator-boot-on; +&usb_hub_2_x { + vdd-supply = <&pp3300_l7c>; }; -&pp3300_l7c { - regulator-always-on; - regulator-boot-on; +&usb_hub_3_x { + vdd-supply = <&pp3300_l7c>; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts index 59a23d0e96517ed10e53d65a2a0effdf96ad6268..bc097d1b1b234db4ebcb21c74a9f242702f0167d 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts @@ -44,17 +44,6 @@ ap_ts_pen_1v8: &i2c4 { compatible = "auo,b116xa01"; }; -&pp3300_hub { - /* pp3300_l7c is used to power the USB hub */ - /delete-property/regulator-always-on; - /delete-property/regulator-boot-on; -}; - -&pp3300_l7c { - regulator-always-on; - regulator-boot-on; -}; - &sdhc_2 { status = "okay"; }; @@ -63,6 +52,14 @@ ap_ts_pen_1v8: &i2c4 { interrupts = <58 IRQ_TYPE_EDGE_FALLING>; }; +&usb_hub_2_x { + vdd-supply = <&pp3300_l7c>; +}; + +&usb_hub_3_x { + vdd-supply = <&pp3300_l7c>; +}; + /* PINCTRL - modifications to sc7180-trogdor.dtsi */ &trackpad_int_1v8_odl { diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index b5f534db135a6fed6e867aa9a2f8c2070c8f1708..eae22e6e97c15f185fac74a64b2efaf8b2815c3c 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -299,7 +299,7 @@ pinctrl-names = "default"; pinctrl-0 = <&en_pp3300_hub>; - regulator-always-on; + /* The BIOS leaves this regulator on */ regulator-boot-on; vin-supply = <&pp3300_a>; @@ -936,6 +936,24 @@ ap_spi_fp: &spi10 { &usb_1_dwc3 { dr_mode = "host"; + #address-cells = <1>; + #size-cells = <0>; + + /* 2.x hub on port 1 */ + usb_hub_2_x: hub@1 { + compatible = "usbbda,5411"; + reg = <1>; + vdd-supply = <&pp3300_hub>; + peer-hub = <&usb_hub_3_x>; + }; + + /* 3.x hub on port 2 */ + usb_hub_3_x: hub@2 { + compatible = "usbbda,411"; + reg = <2>; + vdd-supply = <&pp3300_hub>; + peer-hub = <&usb_hub_2_x>; + }; }; &usb_1_hsphy { diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index b82c335c25affe79f279ab8eb75db71c589b302e..58976a1ba06bed6af149d1dbc55cf6b9b84e3152 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -555,12 +555,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - smem { compatible = "qcom,smem"; memory-region = <&smem_mem>; @@ -1462,13 +1456,19 @@ status = "disabled"; }; - tcsr_mutex_regs: syscon@1f40000 { - compatible = "syscon"; - reg = <0 0x01f40000 0 0x40000>; + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0 0x01f40000 0 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_regs_1: syscon@1f60000 { + compatible = "qcom,sc7180-tcsr", "syscon"; + reg = <0 0x01f60000 0 0x20000>; }; - tcsr_regs: syscon@1fc0000 { - compatible = "syscon"; + tcsr_regs_2: syscon@1fc0000 { + compatible = "qcom,sc7180-tcsr", "syscon"; reg = <0 0x01fc0000 0 0x40000>; }; @@ -1932,8 +1932,8 @@ <&pdc_reset PDC_MODEM_SYNC_RESET>; reset-names = "mss_restart", "pdc_reset"; - qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>; - qcom,spare-regs = <&tcsr_regs 0xb3e4>; + qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>; + qcom,spare-regs = <&tcsr_regs_2 0xb3e4>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi b/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi index cfe2741456a1a39285aeb72eea9f02561d3fe7b7..25f31c81b2b74fcf36086d3797d80005208a30bd 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi @@ -83,17 +83,6 @@ }; }; -/* Modem setup is different on Chrome setups than typical Qualcomm setup */ -&remoteproc_mpss { - status = "okay"; - compatible = "qcom,sc7280-mss-pil"; - iommus = <&apps_smmu 0x124 0x0>, <&apps_smmu 0x488 0x7>; - interconnects = <&mc_virt MASTER_LLCC 0 &mc_virt SLAVE_EBI1 0>; - memory-region = <&mba_mem>, <&mpss_mem>; - firmware-name = "qcom/sc7280-herobrine/modem/mba.mbn", - "qcom/sc7280-herobrine/modem/qdsp6sw.mbn"; -}; - &remoteproc_wpss { status = "okay"; firmware-name = "ath11k/WCN6750/hw1.0/wpss.mdt"; diff --git a/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts b/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts index 344338ad8a0164c6cf34e2e005b86a1fb52e9c6d..dddb505e220b59ba73afd4fb574dd4e622a45ba6 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts @@ -87,6 +87,36 @@ ap_ts_pen_1v8: &i2c13 { pins = "gpio51"; }; +&sound { + audio-routing = + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "VA DMIC0", "MIC BIAS1", + "VA DMIC1", "MIC BIAS1", + "VA DMIC2", "MIC BIAS3", + "VA DMIC3", "MIC BIAS3", + "TX SWR_ADC0", "ADC1_OUTPUT", + "TX SWR_ADC1", "ADC2_OUTPUT", + "TX SWR_ADC2", "ADC3_OUTPUT", + "TX SWR_DMIC0", "DMIC1_OUTPUT", + "TX SWR_DMIC1", "DMIC2_OUTPUT", + "TX SWR_DMIC2", "DMIC3_OUTPUT", + "TX SWR_DMIC3", "DMIC4_OUTPUT", + "TX SWR_DMIC4", "DMIC5_OUTPUT", + "TX SWR_DMIC5", "DMIC6_OUTPUT", + "TX SWR_DMIC6", "DMIC7_OUTPUT", + "TX SWR_DMIC7", "DMIC8_OUTPUT"; +}; + +&wcd9385 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&wcd_reset_n>, <&us_euro_hs_sel>; + pinctrl-1 = <&wcd_reset_n_sleep>, <&us_euro_hs_sel>; + us-euro-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>; +}; + &tlmm { tp_int_odl: tp-int-odl { pins = "gpio7"; @@ -105,4 +135,11 @@ ap_ts_pen_1v8: &i2c13 { function = "gpio"; bias-disable; }; + + us_euro_hs_sel: us-euro-hs-sel { + pins = "gpio81"; + function = "gpio"; + bias-pull-down; + drive-strength = <2>; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi index 859faaa8b7e060e64f54c14e0f9833dc6e852e99..c72e53aaf997cd8ec5fbc7d7066c7ab6caf77329 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi @@ -5,6 +5,161 @@ * Copyright (c) 2022, The Linux Foundation. All rights reserved. */ +/ { + /* BOARD-SPECIFIC TOP LEVEL NODES */ + sound: sound { + compatible = "google,sc7280-herobrine"; + model = "sc7280-wcd938x-max98360a-1mic"; + + audio-routing = + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "VA DMIC0", "MIC BIAS1", + "VA DMIC1", "MIC BIAS1", + "VA DMIC2", "MIC BIAS3", + "VA DMIC3", "MIC BIAS3", + "TX SWR_ADC0", "ADC1_OUTPUT", + "TX SWR_ADC1", "ADC2_OUTPUT", + "TX SWR_ADC2", "ADC3_OUTPUT", + "TX SWR_DMIC0", "DMIC1_OUTPUT", + "TX SWR_DMIC1", "DMIC2_OUTPUT", + "TX SWR_DMIC2", "DMIC3_OUTPUT", + "TX SWR_DMIC3", "DMIC4_OUTPUT", + "TX SWR_DMIC4", "DMIC5_OUTPUT", + "TX SWR_DMIC5", "DMIC6_OUTPUT", + "TX SWR_DMIC6", "DMIC7_OUTPUT", + "TX SWR_DMIC7", "DMIC8_OUTPUT"; + + qcom,msm-mbhc-hphl-swh = <1>; + qcom,msm-mbhc-gnd-swh = <1>; + + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + + dai-link@0 { + link-name = "MAX98360A"; + reg = <0>; + + cpu { + sound-dai = <&lpass_cpu MI2S_SECONDARY>; + }; + + codec { + sound-dai = <&max98360a>; + }; + }; + + dai-link@1 { + link-name = "DisplayPort"; + reg = <1>; + + cpu { + sound-dai = <&lpass_cpu LPASS_DP_RX>; + }; + + codec { + sound-dai = <&mdss_dp>; + }; + }; + + dai-link@2 { + link-name = "WCD9385 Playback"; + reg = <2>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_RX0>; + }; + + codec { + sound-dai = <&wcd9385 0>, <&swr0 0>, <&lpass_rx_macro 0>; + }; + }; + + dai-link@3 { + link-name = "WCD9385 Capture"; + reg = <3>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_TX3>; + }; + + codec { + sound-dai = <&wcd9385 1>, <&swr1 0>, <&lpass_tx_macro 0>; + }; + }; + + dai-link@4 { + link-name = "DMIC"; + reg = <4>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_VA_TX0>; + }; + + codec { + sound-dai = <&lpass_va_macro 0>; + }; + }; + }; +}; + +/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */ + +&lpass_cpu { + status = "okay"; + + pinctrl-names = "default"; + pinctrl-0 = <&mi2s1_data0>, <&mi2s1_sclk>, <&mi2s1_ws>; + + dai-link@1 { + reg = ; + qcom,playback-sd-lines = <0>; + }; + + dai-link@5 { + reg = ; + }; + + dai-link@6 { + reg = ; + }; + + dai-link@19 { + reg = ; + }; + + dai-link@25 { + reg = ; + }; +}; + +&lpass_rx_macro { + status = "okay"; +}; + +&lpass_tx_macro { + status = "okay"; +}; + +&lpass_va_macro { + status = "okay"; +}; + +&swr0 { + status = "okay"; +}; + +&swr1 { + status = "okay"; +}; + +&wcd9385 { + status = "okay"; +}; + /* PINCTRL */ &lpass_dmic01_clk { diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts index 7881bbc641a0ba2c1f2ad0c92bd8239b2ad0ccd7..f0f26af1e42199c16b3efad5113441c5efc45389 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts @@ -9,10 +9,11 @@ #include "sc7280-herobrine.dtsi" #include "sc7280-herobrine-audio-wcd9385.dtsi" +#include "sc7280-herobrine-lte-sku.dtsi" / { model = "Qualcomm Technologies, Inc. sc7280 CRD platform (rev5+)"; - compatible = "google,hoglin", "qcom,sc7280"; + compatible = "google,zoglin", "google,hoglin", "qcom,sc7280"; /* FIXED REGULATORS */ @@ -167,7 +168,7 @@ ap_ts_pen_1v8: &i2c13 { "PMIC_EDP_BL_PWM", ""; - edp_bl_reg_en: edp-bl-reg-en { + edp_bl_reg_en: edp-bl-reg-en-state { pins = "gpio6"; function = "normal"; bias-disable; @@ -371,7 +372,5 @@ ap_ts_pen_1v8: &i2c13 { "", /* 170 */ "MOS_BLE_UART_TX", "MOS_BLE_UART_RX", - "", - "", ""; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts new file mode 100644 index 0000000000000000000000000000000000000000..ccbe50b6249abd43460fe36a3fbb4c8f40ec9855 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Google Evoker board device tree source + * + * Copyright 2022 Google LLC. + */ + +/dts-v1/; + +#include "sc7280-herobrine.dtsi" + +/ { + model = "Google Evoker"; + compatible = "google,evoker", "qcom,sc7280"; +}; + +/* + * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES + * + * Sort order matches the order in the parent files (parents before children). + */ + +&pp3300_codec { + status = "okay"; +}; + +/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */ + +ap_tp_i2c: &i2c0 { + status = "okay"; + clock-frequency = <400000>; + + trackpad: trackpad@2c { + compatible = "hid-over-i2c"; + reg = <0x2c>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_int_odl>; + + interrupt-parent = <&tlmm>; + interrupts = <7 IRQ_TYPE_EDGE_FALLING>; + + hid-descr-addr = <0x20>; + vcc-supply = <&pp3300_z1>; + + wakeup-source; + }; +}; + +ts_i2c: &i2c13 { + status = "okay"; + clock-frequency = <400000>; + + ap_ts: touchscreen@10 { + compatible = "elan,ekth6915"; + reg = <0x10>; + pinctrl-names = "default"; + pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>; + + interrupt-parent = <&tlmm>; + interrupts = <55 IRQ_TYPE_LEVEL_LOW>; + + reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>; + + vcc33-supply = <&ts_avdd>; + }; +}; + +&ap_sar_sensor_i2c { + status = "okay"; +}; + +&ap_sar_sensor0 { + status = "okay"; +}; + +&ap_sar_sensor1 { + status = "okay"; +}; + +&mdss_edp { + status = "okay"; +}; + +&mdss_edp_phy { + status = "okay"; +}; + +/* For nvme */ +&pcie1 { + status = "okay"; +}; + +/* For nvme */ +&pcie1_phy { + status = "okay"; +}; + +&pwmleds { + status = "okay"; +}; + +/* For eMMC */ +&sdhc_1 { + status = "okay"; +}; + +/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */ + +&ts_rst_conn { + bias-disable; +}; + +/* PINCTRL - BOARD-SPECIFIC */ + +/* + * Methodology for gpio-line-names: + * - If a pin goes to herobrine board and is named it gets that name. + * - If a pin goes to herobrine board and is not named, it gets no name. + * - If a pin is totally internal to Qcard then it gets Qcard name. + * - If a pin is not hooked up on Qcard, it gets no name. + */ + +&pm8350c_gpios { + gpio-line-names = "FLASH_STROBE_1", /* 1 */ + "AP_SUSPEND", + "PM8008_1_RST_N", + "", + "", + "", + "PMIC_EDP_BL_EN", + "PMIC_EDP_BL_PWM", + ""; +}; + +&tlmm { + gpio-line-names = "AP_TP_I2C_SDA", /* 0 */ + "AP_TP_I2C_SCL", + "SSD_RST_L", + "PE_WAKE_ODL", + "AP_SAR_SDA", + "AP_SAR_SCL", + "PRB_SC_GPIO_6", + "TP_INT_ODL", + "HP_I2C_SDA", + "HP_I2C_SCL", + + "GNSS_L1_EN", /* 10 */ + "GNSS_L5_EN", + "SPI_AP_MOSI", + "SPI_AP_MISO", + "SPI_AP_CLK", + "SPI_AP_CS0_L", + /* + * AP_FLASH_WP is crossystem ABI. Schematics + * call it BIOS_FLASH_WP_OD. + */ + "AP_FLASH_WP", + "", + "AP_EC_INT_L", + "", + + "UF_CAM_RST_L", /* 20 */ + "WF_CAM_RST_L", + "UART_AP_TX_DBG_RX", + "UART_DBG_TX_AP_RX", + "", + "PM8008_IRQ_1", + "HOST2WLAN_SOL", + "WLAN2HOST_SOL", + "MOS_BT_UART_CTS", + "MOS_BT_UART_RFR", + + "MOS_BT_UART_TX", /* 30 */ + "MOS_BT_UART_RX", + "PRB_SC_GPIO_32", + "HUB_RST_L", + "", + "", + "AP_SPI_FP_MISO", + "AP_SPI_FP_MOSI", + "AP_SPI_FP_CLK", + "AP_SPI_FP_CS_L", + + "AP_EC_SPI_MISO", /* 40 */ + "AP_EC_SPI_MOSI", + "AP_EC_SPI_CLK", + "AP_EC_SPI_CS_L", + "LCM_RST_L", + "EARLY_EUD_N", + "", + "DP_HOT_PLUG_DET", + "IO_BRD_MLB_ID0", + "IO_BRD_MLB_ID1", + + "IO_BRD_MLB_ID2", /* 50 */ + "SSD_EN", + "TS_I2C_SDA_CONN", + "TS_I2C_CLK_CONN", + "TS_RST_CONN", + "TS_INT_CONN", + "AP_I2C_TPM_SDA", + "AP_I2C_TPM_SCL", + "PRB_SC_GPIO_58", + "PRB_SC_GPIO_59", + + "EDP_HOT_PLUG_DET_N", /* 60 */ + "FP_TO_AP_IRQ_L", + "", + "AMP_EN", + "CAM0_MCLK_GPIO_64", + "CAM1_MCLK_GPIO_65", + "WF_CAM_MCLK", + "PRB_SC_GPIO_67", + "FPMCU_BOOT0", + "UF_CAM_SDA", + + "UF_CAM_SCL", /* 70 */ + "", + "", + "WF_CAM_SDA", + "WF_CAM_SCL", + "", + "", + "EN_FP_RAILS", + "FP_RST_L", + "PCIE1_CLKREQ_ODL", + + "EN_PP3300_DX_EDP", /* 80 */ + "SC_GPIO_81", + "FORCED_USB_BOOT", + "WCD_RESET_N", + "MOS_WLAN_EN", + "MOS_BT_EN", + "MOS_SW_CTRL", + "MOS_PCIE0_RST", + "MOS_PCIE0_CLKREQ_N", + "MOS_PCIE0_WAKE_N", + + "MOS_LAA_AS_EN", /* 90 */ + "SD_CD_ODL", + "", + "", + "MOS_BT_WLAN_SLIMBUS_CLK", + "MOS_BT_WLAN_SLIMBUS_DAT0", + "HP_MCLK", + "HP_BCLK", + "HP_DOUT", + "HP_DIN", + + "HP_LRCLK", /* 100 */ + "HP_IRQ", + "", + "", + "GSC_AP_INT_ODL", + "EN_PP3300_CODEC", + "AMP_BCLK", + "AMP_DIN", + "AMP_LRCLK", + "UIM1_DATA_GPIO_109", + + "UIM1_CLK_GPIO_110", /* 110 */ + "UIM1_RESET_GPIO_111", + "PRB_SC_GPIO_112", + "UIM0_DATA", + "UIM0_CLK", + "UIM0_RST", + "UIM0_PRESENT_ODL", + "SDM_RFFE0_CLK", + "SDM_RFFE0_DATA", + "WF_CAM_EN", + + "FASTBOOT_SEL_0", /* 120 */ + "SC_GPIO_121", + "FASTBOOT_SEL_1", + "SC_GPIO_123", + "FASTBOOT_SEL_2", + "SM_RFFE4_CLK_GRFC_8", + "SM_RFFE4_DATA_GRFC_9", + "WLAN_COEX_UART1_RX", + "WLAN_COEX_UART1_TX", + "PRB_SC_GPIO_129", + + "LCM_ID0", /* 130 */ + "LCM_ID1", + "", + "SDR_QLINK_REQ", + "SDR_QLINK_EN", + "QLINK0_WMSS_RESET_N", + "SMR526_QLINK1_REQ", + "SMR526_QLINK1_EN", + "SMR526_QLINK1_WMSS_RESET_N", + "PRB_SC_GPIO_139", + + "SAR1_IRQ_ODL", /* 140 */ + "SAR0_IRQ_ODL", + "PRB_SC_GPIO_142", + "", + "WCD_SWR_TX_CLK", + "WCD_SWR_TX_DATA0", + "WCD_SWR_TX_DATA1", + "WCD_SWR_RX_CLK", + "WCD_SWR_RX_DATA0", + "WCD_SWR_RX_DATA1", + + "DMIC01_CLK", /* 150 */ + "DMIC01_DATA", + "DMIC23_CLK", + "DMIC23_DATA", + "", + "", + "EC_IN_RW_ODL", + "HUB_EN", + "WCD_SWR_TX_DATA2", + "", + + "", /* 160 */ + "", + "", + "", + "", + "", + "", + "", + "", + "", + + "", /* 170 */ + "MOS_BLE_UART_TX", + "MOS_BLE_UART_RX", + "", + "", + ""; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts index c1647a85a371a4ff389861598907f578a70d2ab7..c1a6719687252ee397a217d68b7b9d464dafd38d 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts @@ -8,6 +8,7 @@ /dts-v1/; #include "sc7280-herobrine.dtsi" +#include "sc7280-herobrine-lte-sku.dtsi" / { model = "Google Herobrine (rev1+)"; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..a92eeccd2b2a9eef6e51ef1297e8dc116ddf1e5e --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Google Herobrine dts fragment for LTE SKUs + * + * Copyright 2022 Google LLC. + */ +/* Modem setup is different on Chrome setups than typical Qualcomm setup */ + +&remoteproc_mpss { + compatible = "qcom,sc7280-mss-pil"; + iommus = <&apps_smmu 0x124 0x0>, <&apps_smmu 0x488 0x7>; + interconnects = <&mc_virt MASTER_LLCC 0 &mc_virt SLAVE_EBI1 0>; + memory-region = <&mba_mem>, <&mpss_mem>; + firmware-name = "qcom/sc7280-herobrine/modem/mba.mbn", + "qcom/sc7280-herobrine/modem/qdsp6sw.mbn"; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts index 2cacafd8faa8dab1f3020b69049fa2c6227ec625..73e24cc55a09db5dd57d639a7171ece1a3d874e1 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts @@ -7,327 +7,10 @@ /dts-v1/; -#include "sc7280-herobrine.dtsi" +#include "sc7280-herobrine-villager.dtsi" +#include "sc7280-herobrine-lte-sku.dtsi" / { - model = "Google Villager (rev0+)"; - compatible = "google,villager", "qcom,sc7280"; -}; - -/* - * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES - * - * Sort order matches the order in the parent files (parents before children). - */ - -&pp3300_codec { - status = "okay"; -}; - -/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */ - -ap_tp_i2c: &i2c0 { - status = "okay"; - clock-frequency = <400000>; - - trackpad: trackpad@2c { - compatible = "hid-over-i2c"; - reg = <0x2c>; - pinctrl-names = "default"; - pinctrl-0 = <&tp_int_odl>; - - interrupt-parent = <&tlmm>; - interrupts = <7 IRQ_TYPE_EDGE_FALLING>; - - hid-descr-addr = <0x20>; - vcc-supply = <&pp3300_z1>; - - wakeup-source; - }; -}; - -ts_i2c: &i2c13 { - status = "okay"; - clock-frequency = <400000>; - - ap_ts: touchscreen@10 { - compatible = "elan,ekth6915"; - reg = <0x10>; - pinctrl-names = "default"; - pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>; - - interrupt-parent = <&tlmm>; - interrupts = <55 IRQ_TYPE_LEVEL_LOW>; - - reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>; - - vcc33-supply = <&ts_avdd>; - }; -}; - -&ap_sar_sensor_i2c { - status = "okay"; -}; - -&ap_sar_sensor0 { - status = "okay"; -}; - -&ap_sar_sensor1 { - status = "okay"; -}; - -&mdss_edp { - status = "okay"; -}; - -&mdss_edp_phy { - status = "okay"; -}; - -/* For nvme */ -&pcie1 { - status = "okay"; -}; - -/* For nvme */ -&pcie1_phy { - status = "okay"; -}; - -&pwmleds { - status = "okay"; -}; - -/* For eMMC */ -&sdhc_1 { - status = "okay"; -}; - -/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */ - -&ts_rst_conn { - bias-disable; -}; - -/* PINCTRL - BOARD-SPECIFIC */ - -/* - * Methodology for gpio-line-names: - * - If a pin goes to herobrine board and is named it gets that name. - * - If a pin goes to herobrine board and is not named, it gets no name. - * - If a pin is totally internal to Qcard then it gets Qcard name. - * - If a pin is not hooked up on Qcard, it gets no name. - */ - -&pm8350c_gpios { - gpio-line-names = "FLASH_STROBE_1", /* 1 */ - "AP_SUSPEND", - "PM8008_1_RST_N", - "", - "", - "", - "PMIC_EDP_BL_EN", - "PMIC_EDP_BL_PWM", - ""; -}; - -&tlmm { - gpio-line-names = "AP_TP_I2C_SDA", /* 0 */ - "AP_TP_I2C_SCL", - "SSD_RST_L", - "PE_WAKE_ODL", - "AP_SAR_SDA", - "AP_SAR_SCL", - "PRB_SC_GPIO_6", - "TP_INT_ODL", - "HP_I2C_SDA", - "HP_I2C_SCL", - - "GNSS_L1_EN", /* 10 */ - "GNSS_L5_EN", - "SPI_AP_MOSI", - "SPI_AP_MISO", - "SPI_AP_CLK", - "SPI_AP_CS0_L", - /* - * AP_FLASH_WP is crossystem ABI. Schematics - * call it BIOS_FLASH_WP_OD. - */ - "AP_FLASH_WP", - "", - "AP_EC_INT_L", - "", - - "UF_CAM_RST_L", /* 20 */ - "WF_CAM_RST_L", - "UART_AP_TX_DBG_RX", - "UART_DBG_TX_AP_RX", - "", - "PM8008_IRQ_1", - "HOST2WLAN_SOL", - "WLAN2HOST_SOL", - "MOS_BT_UART_CTS", - "MOS_BT_UART_RFR", - - "MOS_BT_UART_TX", /* 30 */ - "MOS_BT_UART_RX", - "PRB_SC_GPIO_32", - "HUB_RST_L", - "", - "", - "AP_SPI_FP_MISO", - "AP_SPI_FP_MOSI", - "AP_SPI_FP_CLK", - "AP_SPI_FP_CS_L", - - "AP_EC_SPI_MISO", /* 40 */ - "AP_EC_SPI_MOSI", - "AP_EC_SPI_CLK", - "AP_EC_SPI_CS_L", - "LCM_RST_L", - "EARLY_EUD_N", - "", - "DP_HOT_PLUG_DET", - "IO_BRD_MLB_ID0", - "IO_BRD_MLB_ID1", - - "IO_BRD_MLB_ID2", /* 50 */ - "SSD_EN", - "TS_I2C_SDA_CONN", - "TS_I2C_CLK_CONN", - "TS_RST_CONN", - "TS_INT_CONN", - "AP_I2C_TPM_SDA", - "AP_I2C_TPM_SCL", - "PRB_SC_GPIO_58", - "PRB_SC_GPIO_59", - - "EDP_HOT_PLUG_DET_N", /* 60 */ - "FP_TO_AP_IRQ_L", - "", - "AMP_EN", - "CAM0_MCLK_GPIO_64", - "CAM1_MCLK_GPIO_65", - "WF_CAM_MCLK", - "PRB_SC_GPIO_67", - "FPMCU_BOOT0", - "UF_CAM_SDA", - - "UF_CAM_SCL", /* 70 */ - "", - "", - "WF_CAM_SDA", - "WF_CAM_SCL", - "", - "", - "EN_FP_RAILS", - "FP_RST_L", - "PCIE1_CLKREQ_ODL", - - "EN_PP3300_DX_EDP", /* 80 */ - "SC_GPIO_81", - "FORCED_USB_BOOT", - "WCD_RESET_N", - "MOS_WLAN_EN", - "MOS_BT_EN", - "MOS_SW_CTRL", - "MOS_PCIE0_RST", - "MOS_PCIE0_CLKREQ_N", - "MOS_PCIE0_WAKE_N", - - "MOS_LAA_AS_EN", /* 90 */ - "SD_CD_ODL", - "", - "", - "MOS_BT_WLAN_SLIMBUS_CLK", - "MOS_BT_WLAN_SLIMBUS_DAT0", - "HP_MCLK", - "HP_BCLK", - "HP_DOUT", - "HP_DIN", - - "HP_LRCLK", /* 100 */ - "HP_IRQ", - "", - "", - "GSC_AP_INT_ODL", - "EN_PP3300_CODEC", - "AMP_BCLK", - "AMP_DIN", - "AMP_LRCLK", - "UIM1_DATA_GPIO_109", - - "UIM1_CLK_GPIO_110", /* 110 */ - "UIM1_RESET_GPIO_111", - "PRB_SC_GPIO_112", - "UIM0_DATA", - "UIM0_CLK", - "UIM0_RST", - "UIM0_PRESENT_ODL", - "SDM_RFFE0_CLK", - "SDM_RFFE0_DATA", - "WF_CAM_EN", - - "FASTBOOT_SEL_0", /* 120 */ - "SC_GPIO_121", - "FASTBOOT_SEL_1", - "SC_GPIO_123", - "FASTBOOT_SEL_2", - "SM_RFFE4_CLK_GRFC_8", - "SM_RFFE4_DATA_GRFC_9", - "WLAN_COEX_UART1_RX", - "WLAN_COEX_UART1_TX", - "PRB_SC_GPIO_129", - - "LCM_ID0", /* 130 */ - "LCM_ID1", - "", - "SDR_QLINK_REQ", - "SDR_QLINK_EN", - "QLINK0_WMSS_RESET_N", - "SMR526_QLINK1_REQ", - "SMR526_QLINK1_EN", - "SMR526_QLINK1_WMSS_RESET_N", - "PRB_SC_GPIO_139", - - "SAR1_IRQ_ODL", /* 140 */ - "SAR0_IRQ_ODL", - "PRB_SC_GPIO_142", - "", - "WCD_SWR_TX_CLK", - "WCD_SWR_TX_DATA0", - "WCD_SWR_TX_DATA1", - "WCD_SWR_RX_CLK", - "WCD_SWR_RX_DATA0", - "WCD_SWR_RX_DATA1", - - "DMIC01_CLK", /* 150 */ - "DMIC01_DATA", - "DMIC23_CLK", - "DMIC23_DATA", - "", - "", - "EC_IN_RW_ODL", - "HUB_EN", - "WCD_SWR_TX_DATA2", - "", - - "", /* 160 */ - "", - "", - "", - "", - "", - "", - "", - "", - "", - - "", /* 170 */ - "MOS_BLE_UART_TX", - "MOS_BLE_UART_RX", - "", - "", - ""; + model = "Google Villager (rev0)"; + compatible = "google,villager-rev0", "qcom,sc7280"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts new file mode 100644 index 0000000000000000000000000000000000000000..f1017809e5dae67124349fbe367b000a1b5026a3 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Google Villager board device tree source + * + * Copyright 2022 Google LLC. + */ + +#include "sc7280-herobrine-villager-r1.dts" +#include "sc7280-herobrine-lte-sku.dtsi" + +/ { + model = "Google Villager (rev1+) with LTE"; + compatible = "google,villager-sku512", "qcom,sc7280"; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts new file mode 100644 index 0000000000000000000000000000000000000000..cfc64872693052df1d8e6edd8cb8e758c046d516 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Google Villager board device tree source + * + * Copyright 2022 Google LLC. + */ + +/dts-v1/; + +#include "sc7280-herobrine-villager.dtsi" +#include "sc7280-herobrine-audio-wcd9385.dtsi" + +/ { + model = "Google Villager (rev1+)"; + compatible = "google,villager", "qcom,sc7280"; +}; + +&lpass_va_macro { + vdd-micb-supply = <&pp1800_l2c>; +}; + +&sound { + audio-routing = + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "VA DMIC0", "vdd-micb", + "VA DMIC1", "vdd-micb", + "VA DMIC2", "vdd-micb", + "VA DMIC3", "vdd-micb", + "TX SWR_ADC0", "ADC1_OUTPUT", + "TX SWR_ADC1", "ADC2_OUTPUT", + "TX SWR_ADC2", "ADC3_OUTPUT", + "TX SWR_DMIC0", "DMIC1_OUTPUT", + "TX SWR_DMIC1", "DMIC2_OUTPUT", + "TX SWR_DMIC2", "DMIC3_OUTPUT", + "TX SWR_DMIC3", "DMIC4_OUTPUT", + "TX SWR_DMIC4", "DMIC5_OUTPUT", + "TX SWR_DMIC5", "DMIC6_OUTPUT", + "TX SWR_DMIC6", "DMIC7_OUTPUT", + "TX SWR_DMIC7", "DMIC8_OUTPUT"; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..4566722bf4ddfb6ef3353a9ee4eb9a4a57ee7e9b --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Google Villager board device tree source + * + * Copyright 2022 Google LLC. + */ + +#include "sc7280-herobrine.dtsi" + +/* + * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES + * + * Sort order matches the order in the parent files (parents before children). + */ + +&pp3300_codec { + status = "okay"; +}; + +/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */ + +ap_tp_i2c: &i2c0 { + status = "okay"; + clock-frequency = <400000>; + + trackpad: trackpad@2c { + compatible = "hid-over-i2c"; + reg = <0x2c>; + pinctrl-names = "default"; + pinctrl-0 = <&tp_int_odl>; + + interrupt-parent = <&tlmm>; + interrupts = <7 IRQ_TYPE_EDGE_FALLING>; + + hid-descr-addr = <0x20>; + vcc-supply = <&pp3300_z1>; + + wakeup-source; + }; +}; + +ts_i2c: &i2c13 { + status = "okay"; + clock-frequency = <400000>; + + ap_ts: touchscreen@10 { + compatible = "elan,ekth6915"; + reg = <0x10>; + pinctrl-names = "default"; + pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>; + + interrupt-parent = <&tlmm>; + interrupts = <55 IRQ_TYPE_LEVEL_LOW>; + + reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>; + + vcc33-supply = <&ts_avdd>; + }; +}; + +&ap_sar_sensor_i2c { + status = "okay"; +}; + +&ap_sar_sensor0 { + status = "okay"; +}; + +&ap_sar_sensor1 { + status = "okay"; +}; + +&mdss_edp { + status = "okay"; +}; + +&mdss_edp_phy { + status = "okay"; +}; + +/* For nvme */ +&pcie1 { + status = "okay"; +}; + +/* For nvme */ +&pcie1_phy { + status = "okay"; +}; + +&pwmleds { + status = "okay"; +}; + +/* For eMMC */ +&sdhc_1 { + status = "okay"; +}; + +/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */ + +&ts_rst_conn { + bias-disable; +}; + +/* PINCTRL - BOARD-SPECIFIC */ + +/* + * Methodology for gpio-line-names: + * - If a pin goes to herobrine board and is named it gets that name. + * - If a pin goes to herobrine board and is not named, it gets no name. + * - If a pin is totally internal to Qcard then it gets Qcard name. + * - If a pin is not hooked up on Qcard, it gets no name. + */ + +&pm8350c_gpios { + gpio-line-names = "FLASH_STROBE_1", /* 1 */ + "AP_SUSPEND", + "PM8008_1_RST_N", + "", + "", + "", + "PMIC_EDP_BL_EN", + "PMIC_EDP_BL_PWM", + ""; +}; + +&tlmm { + gpio-line-names = "AP_TP_I2C_SDA", /* 0 */ + "AP_TP_I2C_SCL", + "SSD_RST_L", + "PE_WAKE_ODL", + "AP_SAR_SDA", + "AP_SAR_SCL", + "PRB_SC_GPIO_6", + "TP_INT_ODL", + "HP_I2C_SDA", + "HP_I2C_SCL", + + "GNSS_L1_EN", /* 10 */ + "GNSS_L5_EN", + "SPI_AP_MOSI", + "SPI_AP_MISO", + "SPI_AP_CLK", + "SPI_AP_CS0_L", + /* + * AP_FLASH_WP is crossystem ABI. Schematics + * call it BIOS_FLASH_WP_OD. + */ + "AP_FLASH_WP", + "", + "AP_EC_INT_L", + "", + + "UF_CAM_RST_L", /* 20 */ + "WF_CAM_RST_L", + "UART_AP_TX_DBG_RX", + "UART_DBG_TX_AP_RX", + "", + "PM8008_IRQ_1", + "HOST2WLAN_SOL", + "WLAN2HOST_SOL", + "MOS_BT_UART_CTS", + "MOS_BT_UART_RFR", + + "MOS_BT_UART_TX", /* 30 */ + "MOS_BT_UART_RX", + "PRB_SC_GPIO_32", + "HUB_RST_L", + "", + "", + "AP_SPI_FP_MISO", + "AP_SPI_FP_MOSI", + "AP_SPI_FP_CLK", + "AP_SPI_FP_CS_L", + + "AP_EC_SPI_MISO", /* 40 */ + "AP_EC_SPI_MOSI", + "AP_EC_SPI_CLK", + "AP_EC_SPI_CS_L", + "LCM_RST_L", + "EARLY_EUD_N", + "", + "DP_HOT_PLUG_DET", + "IO_BRD_MLB_ID0", + "IO_BRD_MLB_ID1", + + "IO_BRD_MLB_ID2", /* 50 */ + "SSD_EN", + "TS_I2C_SDA_CONN", + "TS_I2C_CLK_CONN", + "TS_RST_CONN", + "TS_INT_CONN", + "AP_I2C_TPM_SDA", + "AP_I2C_TPM_SCL", + "PRB_SC_GPIO_58", + "PRB_SC_GPIO_59", + + "EDP_HOT_PLUG_DET_N", /* 60 */ + "FP_TO_AP_IRQ_L", + "", + "AMP_EN", + "CAM0_MCLK_GPIO_64", + "CAM1_MCLK_GPIO_65", + "WF_CAM_MCLK", + "PRB_SC_GPIO_67", + "FPMCU_BOOT0", + "UF_CAM_SDA", + + "UF_CAM_SCL", /* 70 */ + "", + "", + "WF_CAM_SDA", + "WF_CAM_SCL", + "", + "", + "EN_FP_RAILS", + "FP_RST_L", + "PCIE1_CLKREQ_ODL", + + "EN_PP3300_DX_EDP", /* 80 */ + "SC_GPIO_81", + "FORCED_USB_BOOT", + "WCD_RESET_N", + "MOS_WLAN_EN", + "MOS_BT_EN", + "MOS_SW_CTRL", + "MOS_PCIE0_RST", + "MOS_PCIE0_CLKREQ_N", + "MOS_PCIE0_WAKE_N", + + "MOS_LAA_AS_EN", /* 90 */ + "SD_CD_ODL", + "", + "", + "MOS_BT_WLAN_SLIMBUS_CLK", + "MOS_BT_WLAN_SLIMBUS_DAT0", + "HP_MCLK", + "HP_BCLK", + "HP_DOUT", + "HP_DIN", + + "HP_LRCLK", /* 100 */ + "HP_IRQ", + "", + "", + "GSC_AP_INT_ODL", + "EN_PP3300_CODEC", + "AMP_BCLK", + "AMP_DIN", + "AMP_LRCLK", + "UIM1_DATA_GPIO_109", + + "UIM1_CLK_GPIO_110", /* 110 */ + "UIM1_RESET_GPIO_111", + "PRB_SC_GPIO_112", + "UIM0_DATA", + "UIM0_CLK", + "UIM0_RST", + "UIM0_PRESENT_ODL", + "SDM_RFFE0_CLK", + "SDM_RFFE0_DATA", + "WF_CAM_EN", + + "FASTBOOT_SEL_0", /* 120 */ + "SC_GPIO_121", + "FASTBOOT_SEL_1", + "SC_GPIO_123", + "FASTBOOT_SEL_2", + "SM_RFFE4_CLK_GRFC_8", + "SM_RFFE4_DATA_GRFC_9", + "WLAN_COEX_UART1_RX", + "WLAN_COEX_UART1_TX", + "PRB_SC_GPIO_129", + + "LCM_ID0", /* 130 */ + "LCM_ID1", + "", + "SDR_QLINK_REQ", + "SDR_QLINK_EN", + "QLINK0_WMSS_RESET_N", + "SMR526_QLINK1_REQ", + "SMR526_QLINK1_EN", + "SMR526_QLINK1_WMSS_RESET_N", + "PRB_SC_GPIO_139", + + "SAR1_IRQ_ODL", /* 140 */ + "SAR0_IRQ_ODL", + "PRB_SC_GPIO_142", + "", + "WCD_SWR_TX_CLK", + "WCD_SWR_TX_DATA0", + "WCD_SWR_TX_DATA1", + "WCD_SWR_RX_CLK", + "WCD_SWR_RX_DATA0", + "WCD_SWR_RX_DATA1", + + "DMIC01_CLK", /* 150 */ + "DMIC01_DATA", + "DMIC23_CLK", + "DMIC23_DATA", + "", + "", + "EC_IN_RW_ODL", + "HUB_EN", + "WCD_SWR_TX_DATA2", + "", + + "", /* 160 */ + "", + "", + "", + "", + "", + "", + "", + "", + "", + + "", /* 170 */ + "MOS_BLE_UART_TX", + "MOS_BLE_UART_RX", + "", + "", + ""; +}; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi index 3f8996c00b053bac0162dc6aeccff066470db62f..c11e37160f342be43f46dd57dd5e108fbd783922 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi @@ -144,8 +144,8 @@ regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; + /* The BIOS leaves this regulator on */ regulator-boot-on; - regulator-always-on; gpio = <&tlmm 157 GPIO_ACTIVE_HIGH>; enable-active-high; @@ -296,6 +296,14 @@ /* BOARD-SPECIFIC TOP LEVEL NODES */ + max98360a: audio-codec-0 { + compatible = "maxim,max98360a"; + pinctrl-names = "default"; + pinctrl-0 = <&_en>; + sdmode-gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; + pwmleds: pwmleds { compatible = "pwm-leds"; status = "disabled"; @@ -446,7 +454,7 @@ ap_i2c_tpm: &i2c14 { pinctrl-names = "default"; pinctrl-0 = <&pcie1_clkreq_n>, <&ssd_rst_l>, <&pe_wake_odl>; - perst-gpio = <&tlmm 2 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>; vddpe-3v3-supply = <&pp3300_ssd>; }; @@ -596,25 +604,32 @@ ap_ec_spi: &spi10 { &usb_1_dwc3 { dr_mode = "host"; -}; -&usb_1_hsphy { - status = "okay"; -}; + #address-cells = <1>; + #size-cells = <0>; -&usb_1_qmpphy { - status = "okay"; -}; + /* 2.x hub on port 1 */ + usb_hub_2_x: hub@1 { + compatible = "usbbda,5411"; + reg = <1>; + vdd-supply = <&pp3300_hub>; + peer-hub = <&usb_hub_3_x>; + }; -&usb_2 { - status = "okay"; + /* 3.x hub on port 2 */ + usb_hub_3_x: hub@2 { + compatible = "usbbda,411"; + reg = <2>; + vdd-supply = <&pp3300_hub>; + peer-hub = <&usb_hub_2_x>; + }; }; -&usb_2_dwc3 { - dr_mode = "host"; +&usb_1_hsphy { + status = "okay"; }; -&usb_2_hsphy { +&usb_1_qmpphy { status = "okay"; }; @@ -729,27 +744,27 @@ ap_ec_spi: &spi10 { pinctrl-names = "default"; pinctrl-0 = <&bios_flash_wp_od>; - amp_en: amp-en { + amp_en: amp-en-pins { pins = "gpio63"; function = "gpio"; bias-disable; drive-strength = <2>; }; - ap_ec_int_l: ap-ec-int-l { + ap_ec_int_l: ap-ec-int-l-pins { pins = "gpio18"; function = "gpio"; bias-pull-up; }; - bios_flash_wp_od: bios-flash-wp-od { + bios_flash_wp_od: bios-flash-wp-od-pins { pins = "gpio16"; function = "gpio"; /* Has external pull */ bias-disable; }; - en_fp_rails: en-fp-rails { + en_fp_rails: en-fp-rails-pins { pins = "gpio77"; function = "gpio"; bias-disable; @@ -757,60 +772,60 @@ ap_ec_spi: &spi10 { output-high; }; - en_pp3300_codec: en-pp3300-codec { + en_pp3300_codec: en-pp3300-codec-pins { pins = "gpio105"; function = "gpio"; bias-disable; drive-strength = <2>; }; - en_pp3300_dx_edp: en-pp3300-dx-edp { + en_pp3300_dx_edp: en-pp3300-dx-edp-pins { pins = "gpio80"; function = "gpio"; bias-disable; drive-strength = <2>; }; - fp_rst_l: fp-rst-l { + fp_rst_l: fp-rst-l-pins { pins = "gpio78"; function = "gpio"; bias-disable; drive-strength = <2>; }; - fp_to_ap_irq_l: fp-to-ap-irq-l { + fp_to_ap_irq_l: fp-to-ap-irq-l-pins { pins = "gpio61"; function = "gpio"; /* Has external pullup */ bias-disable; }; - fpmcu_boot0: fpmcu-boot0 { + fpmcu_boot0: fpmcu-boot0-pins { pins = "gpio68"; function = "gpio"; bias-disable; }; - gsc_ap_int_odl: gsc-ap-int-odl { + gsc_ap_int_odl: gsc-ap-int-odl-pins { pins = "gpio104"; function = "gpio"; bias-pull-up; }; - hp_irq: hp-irq { + hp_irq: hp-irq-pins { pins = "gpio101"; function = "gpio"; bias-pull-up; }; - hub_en: hub-en { + hub_en: hub-en-pins { pins = "gpio157"; function = "gpio"; bias-disable; drive-strength = <2>; }; - pe_wake_odl: pe-wake-odl { + pe_wake_odl: pe-wake-odl-pins { pins = "gpio3"; function = "gpio"; /* Has external pull */ @@ -819,45 +834,45 @@ ap_ec_spi: &spi10 { }; /* For ap_spi_fp */ - qup_spi9_cs_gpio_init_high: qup-spi9-cs-gpio-init-high { + qup_spi9_cs_gpio_init_high: qup-spi9-cs-gpio-init-high-pins { pins = "gpio39"; function = "gpio"; output-high; }; /* For ap_ec_spi */ - qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high { + qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high-pins { pins = "gpio43"; function = "gpio"; output-high; }; - sar0_irq_odl: sar0-irq-odl { + sar0_irq_odl: sar0-irq-odl-pins { pins = "gpio141"; function = "gpio"; bias-pull-up; }; - sar1_irq_odl: sar1-irq-odl { + sar1_irq_odl: sar1-irq-odl-pins { pins = "gpio140"; function = "gpio"; bias-pull-up; }; - sd_cd_odl: sd-cd-odl { + sd_cd_odl: sd-cd-odl-pins { pins = "gpio91"; function = "gpio"; bias-pull-up; }; - ssd_en: ssd-en { + ssd_en: ssd-en-pins { pins = "gpio51"; function = "gpio"; bias-disable; drive-strength = <2>; }; - ssd_rst_l: ssd-rst-l { + ssd_rst_l: ssd-rst-l-pins { pins = "gpio2"; function = "gpio"; bias-disable; @@ -865,14 +880,14 @@ ap_ec_spi: &spi10 { output-low; }; - tp_int_odl: tp-int-odl { + tp_int_odl: tp-int-odl-pins { pins = "gpio7"; function = "gpio"; /* Has external pullup */ bias-disable; }; - wf_cam_en: wf-cam-en { + wf_cam_en: wf-cam-en-pins { pins = "gpio119"; function = "gpio"; /* Has external pulldown */ diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi index a7c346aa3b020a2b63e7b5868e5c0c24da489dc6..7f5143e9bb8073167dc572f5aa67b9771c679707 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi @@ -79,26 +79,26 @@ ap_h1_spi: &spi14 { }; &tlmm { - ap_ec_int_l: ap-ec-int-l { + ap_ec_int_l: ap-ec-int-l-pins { pins = "gpio18"; function = "gpio"; input-enable; bias-pull-up; }; - h1_ap_int_odl: h1-ap-int-odl { + h1_ap_int_odl: h1-ap-int-odl-pins { pins = "gpio104"; function = "gpio"; input-enable; bias-pull-up; }; - qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high { + qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high-pins { pins = "gpio43"; output-high; }; - qup_spi14_cs_gpio_init_high: qup-spi14-cs-gpio-init-high { + qup_spi14_cs_gpio_init_high: qup-spi14-cs-gpio-init-high-pins { pins = "gpio59"; output-high; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dts b/arch/arm64/boot/dts/qcom/sc7280-idp.dts index 6d3ff80582ae96e6e3439bb79baac04eef64b2f0..7559164cdda080a6c9905c66fc02a0382345a488 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dts +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dts @@ -10,6 +10,7 @@ #include #include "sc7280-idp.dtsi" #include "pmr735a.dtsi" +#include "sc7280-herobrine-lte-sku.dtsi" / { model = "Qualcomm Technologies, Inc. sc7280 IDP SKU1 platform"; @@ -78,7 +79,7 @@ }; &pmk8350_vadc { - pmr735a_die_temp { + pmr735a-die-temp@403 { reg = ; label = "pmr735a_die_temp"; qcom,pre-scaling = <1 1>; diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi index a74e0b730db613e217bdc380b283b410c2f8baf8..cd432a2856a7b154c2fbe8b7196132451a24a473 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi @@ -20,6 +20,42 @@ serial1 = &uart7; }; + max98360a: audio-codec-0 { + compatible = "maxim,max98360a"; + pinctrl-names = "default"; + pinctrl-0 = <&_en>; + sdmode-gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; + + wcd9385: audio-codec-1 { + compatible = "qcom,wcd9385-codec"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&wcd_reset_n>; + pinctrl-1 = <&wcd_reset_n_sleep>; + + reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>; + + qcom,rx-device = <&wcd_rx>; + qcom,tx-device = <&wcd_tx>; + + vdd-rxtx-supply = <&vreg_l18b_1p8>; + vdd-io-supply = <&vreg_l18b_1p8>; + vdd-buck-supply = <&vreg_l17b_1p8>; + vdd-mic-bias-supply = <&vreg_bob>; + + qcom,micbias1-microvolt = <1800000>; + qcom,micbias2-microvolt = <1800000>; + qcom,micbias3-microvolt = <1800000>; + qcom,micbias4-microvolt = <1800000>; + + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 + 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; + #sound-dai-cells = <1>; + }; + gpio-keys { compatible = "gpio-keys"; label = "gpio-keys"; @@ -49,6 +85,104 @@ pinctrl-names = "default"; pinctrl-0 = <&nvme_pwren>; }; + + sound: sound { + compatible = "google,sc7280-herobrine"; + model = "sc7280-wcd938x-max98360a-1mic"; + + audio-routing = + "IN1_HPHL", "HPHL_OUT", + "IN2_HPHR", "HPHR_OUT", + "AMIC1", "MIC BIAS1", + "AMIC2", "MIC BIAS2", + "VA DMIC0", "MIC BIAS3", + "VA DMIC1", "MIC BIAS3", + "VA DMIC2", "MIC BIAS1", + "VA DMIC3", "MIC BIAS1", + "TX SWR_ADC0", "ADC1_OUTPUT", + "TX SWR_ADC1", "ADC2_OUTPUT", + "TX SWR_ADC2", "ADC3_OUTPUT", + "TX SWR_DMIC0", "DMIC1_OUTPUT", + "TX SWR_DMIC1", "DMIC2_OUTPUT", + "TX SWR_DMIC2", "DMIC3_OUTPUT", + "TX SWR_DMIC3", "DMIC4_OUTPUT", + "TX SWR_DMIC4", "DMIC5_OUTPUT", + "TX SWR_DMIC5", "DMIC6_OUTPUT", + "TX SWR_DMIC6", "DMIC7_OUTPUT", + "TX SWR_DMIC7", "DMIC8_OUTPUT"; + + qcom,msm-mbhc-hphl-swh = <1>; + qcom,msm-mbhc-gnd-swh = <1>; + + #address-cells = <1>; + #size-cells = <0>; + #sound-dai-cells = <0>; + + dai-link@0 { + link-name = "MAX98360A"; + reg = <0>; + + cpu { + sound-dai = <&lpass_cpu MI2S_SECONDARY>; + }; + + codec { + sound-dai = <&max98360a>; + }; + }; + + dai-link@1 { + link-name = "DisplayPort"; + reg = <1>; + + cpu { + sound-dai = <&lpass_cpu LPASS_DP_RX>; + }; + + codec { + sound-dai = <&mdss_dp>; + }; + }; + + dai-link@2 { + link-name = "WCD9385 Playback"; + reg = <2>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_RX0>; + }; + + codec { + sound-dai = <&wcd9385 0>, <&swr0 0>, <&lpass_rx_macro 0>; + }; + }; + + dai-link@3 { + link-name = "WCD9385 Capture"; + reg = <3>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_TX3>; + }; + + codec { + sound-dai = <&wcd9385 1>, <&swr1 0>, <&lpass_tx_macro 0>; + }; + }; + + dai-link@4 { + link-name = "DMIC"; + reg = <4>; + + cpu { + sound-dai = <&lpass_cpu LPASS_CDC_DMA_VA_TX0>; + }; + + codec { + sound-dai = <&lpass_va_macro 0>; + }; + }; + }; }; &apps_rsc { @@ -246,9 +380,50 @@ modem-init; }; +&lpass_cpu { + status = "okay"; + + pinctrl-names = "default"; + pinctrl-0 = <&mi2s1_data0>, <&mi2s1_sclk>, <&mi2s1_ws>; + + dai-link@1 { + reg = ; + qcom,playback-sd-lines = <0>; + }; + + dai-link@5 { + reg = ; + }; + + dai-link@6 { + reg = ; + }; + + dai-link@19 { + reg = ; + }; + + dai-link@25 { + reg = ; + }; +}; + +&lpass_rx_macro { + status = "okay"; +}; + +&lpass_tx_macro { + status = "okay"; +}; + +&lpass_va_macro { + status = "okay"; + vdd-micb-supply = <&vreg_bob>; +}; + &pcie1 { status = "okay"; - perst-gpio = <&tlmm 2 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>; vddpe-3v3-supply = <&nvme_3v3_regulator>; @@ -264,7 +439,7 @@ }; &pmk8350_vadc { - pmk8350_die_temp { + pmk8350-die-temp@3 { reg = ; label = "pmk8350_die_temp"; qcom,pre-scaling = <1 1>; @@ -306,6 +481,28 @@ cd-gpios = <&tlmm 91 GPIO_ACTIVE_LOW>; }; +&swr0 { + status = "okay"; + + wcd_rx: codec@0,4 { + compatible = "sdw20217010d00"; + reg = <0 4>; + #sound-dai-cells = <1>; + qcom,rx-port-mapping = <1 2 3 4 5>; + }; +}; + +&swr1 { + status = "okay"; + + wcd_tx: codec@0,3 { + compatible = "sdw20217010d00"; + reg = <0 3>; + #sound-dai-cells = <1>; + qcom,tx-port-mapping = <1 2 3 4>; + }; +}; + &uart5 { compatible = "qcom,geni-debug-uart"; status = "okay"; @@ -550,18 +747,24 @@ }; &tlmm { - bt_en: bt-en { + amp_en: amp-en { + pins = "gpio63"; + bias-pull-down; + drive-strength = <2>; + }; + + bt_en: bt-en-pins { pins = "gpio85"; function = "gpio"; output-low; bias-disable; }; - nvme_pwren: nvme-pwren { + nvme_pwren: nvme-pwren-pins { function = "gpio"; }; - pcie1_reset_n: pcie1-reset-n { + pcie1_reset_n: pcie1-reset-n-pins { pins = "gpio2"; function = "gpio"; @@ -570,7 +773,7 @@ bias-disable; }; - pcie1_wake_n: pcie1-wake-n { + pcie1_wake_n: pcie1-wake-n-pins { pins = "gpio3"; function = "gpio"; @@ -578,7 +781,7 @@ bias-pull-up; }; - qup_uart7_sleep_cts: qup-uart7-sleep-cts { + qup_uart7_sleep_cts: qup-uart7-sleep-cts-pins { pins = "gpio28"; function = "gpio"; /* @@ -591,7 +794,7 @@ bias-bus-hold; }; - qup_uart7_sleep_rts: qup-uart7-sleep-rts { + qup_uart7_sleep_rts: qup-uart7-sleep-rts-pins { pins = "gpio29"; function = "gpio"; /* @@ -603,7 +806,7 @@ bias-pull-down; }; - qup_uart7_sleep_tx: qup-uart7-sleep-tx { + qup_uart7_sleep_tx: qup-uart7-sleep-tx-pins { pins = "gpio30"; function = "gpio"; /* @@ -613,7 +816,7 @@ bias-pull-up; }; - qup_uart7_sleep_rx: qup-uart7-sleep-rx { + qup_uart7_sleep_rx: qup-uart7-sleep-rx-pins { pins = "gpio31"; function = "gpio"; /* @@ -624,15 +827,28 @@ bias-pull-up; }; - sd_cd: sd-cd { + sd_cd: sd-cd-pins { pins = "gpio91"; function = "gpio"; bias-pull-up; }; - sw_ctrl: sw-ctrl { + sw_ctrl: sw-ctrl-pins { pins = "gpio86"; function = "gpio"; bias-pull-down; }; + + wcd_reset_n: wcd-reset-n { + pins = "gpio83"; + function = "gpio"; + drive-strength = <8>; + }; + + wcd_reset_n_sleep: wcd-reset-n-sleep { + pins = "gpio83"; + function = "gpio"; + drive-strength = <8>; + bias-disable; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi index 7adf31bb98272a39c630c5029d894b9575a790bb..4b8c676b0bb196e784d2324a17b24b0e757db451 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi @@ -28,6 +28,38 @@ bluetooth0 = &bluetooth; serial0 = &uart5; serial1 = &uart7; + wifi0 = &wifi; + }; + + wcd9385: audio-codec-1 { + compatible = "qcom,wcd9385-codec"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&wcd_reset_n>, <&us_euro_hs_sel>; + pinctrl-1 = <&wcd_reset_n_sleep>, <&us_euro_hs_sel>; + + reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>; + us-euro-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>; + + qcom,rx-device = <&wcd_rx>; + qcom,tx-device = <&wcd_tx>; + + vdd-rxtx-supply = <&vreg_l18b_1p8>; + vdd-io-supply = <&vreg_l18b_1p8>; + vdd-buck-supply = <&vreg_l17b_1p8>; + vdd-mic-bias-supply = <&vreg_bob>; + + qcom,micbias1-microvolt = <1800000>; + qcom,micbias2-microvolt = <1800000>; + qcom,micbias3-microvolt = <1800000>; + qcom,micbias4-microvolt = <1800000>; + + qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000 + 500000 500000 500000>; + qcom,mbhc-headset-vthreshold-microvolt = <1700000>; + qcom,mbhc-headphone-vthreshold-microvolt = <50000>; + #sound-dai-cells = <1>; + + status = "disabled"; }; pm8350c_pwm_backlight: backlight { @@ -309,6 +341,10 @@ modem-init; }; +&lpass_va_macro { + vdd-micb-supply = <&vreg_bob>; +}; + /* NOTE: Not all Qcards have eDP connector stuffed */ &mdss_edp { aux-bus { @@ -378,6 +414,24 @@ no-sdio; }; +&swr0 { + wcd_rx: codec@0,4 { + compatible = "sdw20217010d00"; + reg = <0 4>; + #sound-dai-cells = <1>; + qcom,rx-port-mapping = <1 2 3 4 5>; + }; +}; + +&swr1 { + wcd_tx: codec@0,3 { + compatible = "sdw20217010d00"; + reg = <0 3>; + #sound-dai-cells = <1>; + qcom,tx-port-mapping = <1 2 3 4>; + }; +}; + uart_dbg: &uart5 { compatible = "qcom,geni-debug-uart"; status = "okay"; @@ -541,7 +595,7 @@ mos_bt_uart: &uart7 { }; &tlmm { - mos_bt_en: mos-bt-en { + mos_bt_en: mos-bt-en-pins { pins = "gpio85"; function = "gpio"; drive-strength = <2>; @@ -549,7 +603,7 @@ mos_bt_uart: &uart7 { }; /* For mos_bt_uart */ - qup_uart7_sleep_cts: qup-uart7-sleep-cts { + qup_uart7_sleep_cts: qup-uart7-sleep-cts-pins { pins = "gpio28"; function = "gpio"; /* @@ -563,7 +617,7 @@ mos_bt_uart: &uart7 { }; /* For mos_bt_uart */ - qup_uart7_sleep_rts: qup-uart7-sleep-rts { + qup_uart7_sleep_rts: qup-uart7-sleep-rts-pins { pins = "gpio29"; function = "gpio"; /* @@ -576,7 +630,7 @@ mos_bt_uart: &uart7 { }; /* For mos_bt_uart */ - qup_uart7_sleep_rx: qup-uart7-sleep-rx { + qup_uart7_sleep_rx: qup-uart7-sleep-rx-pins { pins = "gpio31"; function = "gpio"; /* @@ -588,7 +642,7 @@ mos_bt_uart: &uart7 { }; /* For mos_bt_uart */ - qup_uart7_sleep_tx: qup-uart7-sleep-tx { + qup_uart7_sleep_tx: qup-uart7-sleep-tx-pins { pins = "gpio30"; function = "gpio"; /* @@ -598,15 +652,35 @@ mos_bt_uart: &uart7 { bias-pull-up; }; - ts_int_conn: ts-int-conn { + ts_int_conn: ts-int-conn-pins { pins = "gpio55"; function = "gpio"; bias-pull-up; }; - ts_rst_conn: ts-rst-conn { + ts_rst_conn: ts-rst-conn-pins { pins = "gpio54"; function = "gpio"; drive-strength = <2>; }; + + us_euro_hs_sel: us-euro-hs-sel { + pins = "gpio81"; + function = "gpio"; + bias-pull-down; + drive-strength = <2>; + }; + + wcd_reset_n: wcd-reset-n { + pins = "gpio83"; + function = "gpio"; + drive-strength = <8>; + }; + + wcd_reset_n_sleep: wcd-reset-n-sleep { + pins = "gpio83"; + function = "gpio"; + drive-strength = <8>; + bias-disable; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi index 13d7f267b2891a2c916e147aba99863f7b04cca2..212580316d3e6a46e750b8cc8f466d4bd39dd14e 100644 --- a/arch/arm64/boot/dts/qcom/sc7280.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi @@ -22,6 +22,7 @@ #include #include #include +#include #include / { @@ -828,6 +829,7 @@ #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; + power-domains = <&rpmhpd SC7280_CX>; }; ipcc: mailbox@408000 { @@ -2043,7 +2045,9 @@ <&gcc GCC_PCIE_1_SLV_AXI_CLK>, <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>, <&gcc GCC_AGGRE_NOC_PCIE_TBU_CLK>, - <&gcc GCC_DDRSS_PCIE_SF_CLK>; + <&gcc GCC_DDRSS_PCIE_SF_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_CENTER_SF_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_PCIE_1_AXI_CLK>; clock-names = "pipe", "pipe_mux", @@ -2055,7 +2059,9 @@ "bus_slave", "slave_q2a", "tbu", - "ddrss_sf_tbu"; + "ddrss_sf_tbu", + "aggre0", + "aggre1"; assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; assigned-clock-rates = <19200000>; @@ -2155,12 +2161,17 @@ }; tcsr_mutex: hwlock@1f40000 { - compatible = "qcom,tcsr-mutex", "syscon"; - reg = <0 0x01f40000 0 0x40000>; + compatible = "qcom,tcsr-mutex"; + reg = <0 0x01f40000 0 0x20000>; #hwlock-cells = <1>; }; - tcsr: syscon@1fc0000 { + tcsr_1: syscon@1f60000 { + compatible = "qcom,sc7280-tcsr", "syscon"; + reg = <0 0x01f60000 0 0x20000>; + }; + + tcsr_2: syscon@1fc0000 { compatible = "qcom,sc7280-tcsr", "syscon"; reg = <0 0x01fc0000 0 0x30000>; }; @@ -2168,14 +2179,121 @@ lpasscc: lpasscc@3000000 { compatible = "qcom,sc7280-lpasscc"; reg = <0 0x03000000 0 0x40>, - <0 0x03c04000 0 0x4>, - <0 0x03389000 0 0x24>; - reg-names = "qdsp6ss", "top_cc", "cc"; + <0 0x03c04000 0 0x4>; + reg-names = "qdsp6ss", "top_cc"; clocks = <&gcc GCC_CFG_NOC_LPASS_CLK>; clock-names = "iface"; #clock-cells = <1>; }; + lpass_rx_macro: codec@3200000 { + compatible = "qcom,sc7280-lpass-rx-macro"; + reg = <0 0x03200000 0 0x1000>; + + pinctrl-names = "default"; + pinctrl-0 = <&lpass_rx_swr_clk>, <&lpass_rx_swr_data>; + + clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>, + <&lpass_aon LPASS_AON_CC_TX_MCLK_2X_CLK>, + <&lpass_va_macro>; + clock-names = "mclk", "npl", "fsgen"; + + power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>, + <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>; + power-domain-names = "macro", "dcodec"; + + #clock-cells = <0>; + #sound-dai-cells = <1>; + + status = "disabled"; + }; + + swr0: soundwire@3210000 { + compatible = "qcom,soundwire-v1.6.0"; + reg = <0 0x03210000 0 0x2000>; + + interrupts = ; + clocks = <&lpass_rx_macro>; + clock-names = "iface"; + + qcom,din-ports = <0>; + qcom,dout-ports = <5>; + + resets = <&lpass_audiocc LPASS_AUDIO_SWR_RX_CGCR>; + reset-names = "swr_audio_cgcr"; + + qcom,ports-word-length = /bits/ 8 <0x01 0x07 0x04 0xff 0xff>; + qcom,ports-sinterval-low = /bits/ 8 <0x03 0x3f 0x1f 0x03 0x03>; + qcom,ports-offset1 = /bits/ 8 <0x00 0x00 0x0b 0x01 0x01>; + qcom,ports-offset2 = /bits/ 8 <0x00 0x00 0x0b 0x00 0x00>; + qcom,ports-lane-control = /bits/ 8 <0x01 0x00 0x00 0x00 0x00>; + qcom,ports-block-pack-mode = /bits/ 8 <0xff 0x00 0x01 0xff 0xff>; + qcom,ports-hstart = /bits/ 8 <0xff 0x03 0xff 0xff 0xff>; + qcom,ports-hstop = /bits/ 8 <0xff 0x06 0xff 0xff 0xff>; + qcom,ports-block-group-count = /bits/ 8 <0xff 0xff 0xff 0xff 0x00>; + + #sound-dai-cells = <1>; + #address-cells = <2>; + #size-cells = <0>; + + status = "disabled"; + }; + + lpass_tx_macro: codec@3220000 { + compatible = "qcom,sc7280-lpass-tx-macro"; + reg = <0 0x03220000 0 0x1000>; + + pinctrl-names = "default"; + pinctrl-0 = <&lpass_tx_swr_clk>, <&lpass_tx_swr_data>; + + clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>, + <&lpass_aon LPASS_AON_CC_TX_MCLK_2X_CLK>, + <&lpass_va_macro>; + clock-names = "mclk", "npl", "fsgen"; + + power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>, + <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>; + power-domain-names = "macro", "dcodec"; + + #clock-cells = <0>; + #sound-dai-cells = <1>; + + status = "disabled"; + }; + + swr1: soundwire@3230000 { + compatible = "qcom,soundwire-v1.6.0"; + reg = <0 0x03230000 0 0x2000>; + + interrupts-extended = <&intc GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>, + <&pdc 130 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&lpass_tx_macro>; + clock-names = "iface"; + + qcom,din-ports = <3>; + qcom,dout-ports = <0>; + + resets = <&lpass_audiocc LPASS_AUDIO_SWR_TX_CGCR>; + reset-names = "swr_audio_cgcr"; + + qcom,ports-sinterval-low = /bits/ 8 <0x01 0x03 0x03>; + qcom,ports-offset1 = /bits/ 8 <0x01 0x00 0x02>; + qcom,ports-offset2 = /bits/ 8 <0x00 0x00 0x00>; + qcom,ports-hstart = /bits/ 8 <0xff 0xff 0xff>; + qcom,ports-hstop = /bits/ 8 <0xff 0xff 0xff>; + qcom,ports-word-length = /bits/ 8 <0xff 0x00 0xff>; + qcom,ports-block-pack-mode = /bits/ 8 <0xff 0xff 0xff>; + qcom,ports-block-group-count = /bits/ 8 <0xff 0xff 0xff>; + qcom,ports-lane-control = /bits/ 8 <0x00 0x01 0x00>; + qcom,port-offset = <1>; + + #sound-dai-cells = <1>; + #address-cells = <2>; + #size-cells = <0>; + + status = "disabled"; + }; + lpass_audiocc: clock-controller@3300000 { compatible = "qcom,sc7280-lpassaudiocc"; reg = <0 0x03300000 0 0x30000>; @@ -2185,6 +2303,27 @@ power-domains = <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>; #clock-cells = <1>; #power-domain-cells = <1>; + #reset-cells = <1>; + }; + + lpass_va_macro: codec@3370000 { + compatible = "qcom,sc7280-lpass-va-macro"; + reg = <0 0x03370000 0 0x1000>; + + pinctrl-names = "default"; + pinctrl-0 = <&lpass_dmic01_clk>, <&lpass_dmic01_data>; + + clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>; + clock-names = "mclk"; + + power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>, + <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>; + power-domain-names = "macro", "dcodec"; + + #clock-cells = <0>; + #sound-dai-cells = <1>; + + status = "disabled"; }; lpass_aon: clock-controller@3380000 { @@ -2192,13 +2331,13 @@ reg = <0 0x03380000 0 0x30000>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>, - <&lpasscore LPASS_CORE_CC_CORE_CLK>; + <&lpass_core LPASS_CORE_CC_CORE_CLK>; clock-names = "bi_tcxo", "bi_tcxo_ao", "iface"; #clock-cells = <1>; #power-domain-cells = <1>; }; - lpasscore: clock-controller@3900000 { + lpass_core: clock-controller@3900000 { compatible = "qcom,sc7280-lpasscorecc"; reg = <0 0x03900000 0 0x50000>; clocks = <&rpmhcc RPMH_CXO_CLK>; @@ -2208,6 +2347,67 @@ #power-domain-cells = <1>; }; + lpass_cpu: audio@3987000 { + compatible = "qcom,sc7280-lpass-cpu"; + + reg = <0 0x03987000 0 0x68000>, + <0 0x03b00000 0 0x29000>, + <0 0x03260000 0 0xc000>, + <0 0x03280000 0 0x29000>, + <0 0x03340000 0 0x29000>, + <0 0x0336c000 0 0x3000>; + reg-names = "lpass-hdmiif", + "lpass-lpaif", + "lpass-rxtx-cdc-dma-lpm", + "lpass-rxtx-lpaif", + "lpass-va-lpaif", + "lpass-va-cdc-dma-lpm"; + + iommus = <&apps_smmu 0x1820 0>, + <&apps_smmu 0x1821 0>, + <&apps_smmu 0x1832 0>; + + power-domains = <&rpmhpd SC7280_LCX>; + power-domain-names = "lcx"; + required-opps = <&rpmhpd_opp_nom>; + + clocks = <&lpass_aon LPASS_AON_CC_AUDIO_HM_H_CLK>, + <&lpass_core LPASS_CORE_CC_EXT_MCLK0_CLK>, + <&lpass_core LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK>, + <&lpass_core LPASS_CORE_CC_EXT_IF0_IBIT_CLK>, + <&lpass_core LPASS_CORE_CC_EXT_IF1_IBIT_CLK>, + <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM_CLK>, + <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM0_CLK>, + <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM1_CLK>, + <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM2_CLK>, + <&lpass_aon LPASS_AON_CC_VA_MEM0_CLK>; + clock-names = "aon_cc_audio_hm_h", + "audio_cc_ext_mclk0", + "core_cc_sysnoc_mport_core", + "core_cc_ext_if0_ibit", + "core_cc_ext_if1_ibit", + "audio_cc_codec_mem", + "audio_cc_codec_mem0", + "audio_cc_codec_mem1", + "audio_cc_codec_mem2", + "aon_cc_va_mem0"; + + #sound-dai-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + + interrupts = , + , + , + ; + interrupt-names = "lpass-irq-lpaif", + "lpass-irq-hdmi", + "lpass-irq-vaif", + "lpass-irq-rxtxif"; + + status = "disabled"; + }; + lpass_hm: clock-controller@3c00000 { compatible = "qcom,sc7280-lpasshm"; reg = <0 0x3c00000 0 0x28>; @@ -2352,11 +2552,19 @@ opp-supported-hw = <0x03>; }; - opp-550000000 { + /* Only applicable for SKUs which has 550Mhz as Fmax */ + opp-550000000-0 { + opp-hz = /bits/ 64 <550000000>; + opp-level = ; + opp-peak-kBps = <8368000>; + opp-supported-hw = <0x01>; + }; + + opp-550000000-1 { opp-hz = /bits/ 64 <550000000>; opp-level = ; opp-peak-kBps = <6832000>; - opp-supported-hw = <0x03>; + opp-supported-hw = <0x02>; }; opp-608000000 { @@ -2522,9 +2730,9 @@ <&pdc_reset PDC_MODEM_SYNC_RESET>; reset-names = "mss_restart", "pdc_reset"; - qcom,halt-regs = <&tcsr_mutex 0x23000 0x25000 0x28000 0x33000>; - qcom,ext-regs = <&tcsr 0x10000 0x10004 &tcsr_mutex 0x26004 0x26008>; - qcom,qaccept-regs = <&tcsr_mutex 0x23030 0x23040 0x23020>; + qcom,halt-regs = <&tcsr_1 0x3000 0x5000 0x8000 0x13000>; + qcom,ext-regs = <&tcsr_2 0x10000 0x10004 &tcsr_1 0x6004 0x6008>; + qcom,qaccept-regs = <&tcsr_1 0x3030 0x3040 0x3020>; status = "disabled"; @@ -3181,6 +3389,7 @@ "dm_hs_phy_irq"; power-domains = <&gcc GCC_USB30_SEC_GDSC>; + required-opps = <&rpmhpd_opp_nom>; resets = <&gcc GCC_USB30_SEC_BCR>; @@ -3259,7 +3468,7 @@ <&pdc_reset PDC_WPSS_SYNC_RESET>; reset-names = "restart", "pdc_sync"; - qcom,halt-regs = <&tcsr_mutex 0x37000>; + qcom,halt-regs = <&tcsr_1 0x17000>; status = "disabled"; @@ -3275,6 +3484,82 @@ }; }; + pmu@9091000 { + compatible = "qcom,sc7280-llcc-bwmon"; + reg = <0 0x9091000 0 0x1000>; + + interrupts = ; + + interconnects = <&mc_virt MASTER_LLCC 3 &mc_virt SLAVE_EBI1 3>; + + operating-points-v2 = <&llcc_bwmon_opp_table>; + + llcc_bwmon_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-0 { + opp-peak-kBps = <800000>; + }; + opp-1 { + opp-peak-kBps = <1804000>; + }; + opp-2 { + opp-peak-kBps = <2188000>; + }; + opp-3 { + opp-peak-kBps = <3072000>; + }; + opp-4 { + opp-peak-kBps = <4068000>; + }; + opp-5 { + opp-peak-kBps = <6220000>; + }; + opp-6 { + opp-peak-kBps = <6832000>; + }; + opp-7 { + opp-peak-kBps = <8532000>; + }; + }; + }; + + pmu@90b6400 { + compatible = "qcom,sc7280-cpu-bwmon", "qcom,msm8998-bwmon"; + reg = <0 0x090b6400 0 0x600>; + + interrupts = ; + + interconnects = <&gem_noc MASTER_APPSS_PROC 3 &gem_noc SLAVE_LLCC 3>; + operating-points-v2 = <&cpu_bwmon_opp_table>; + + cpu_bwmon_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-0 { + opp-peak-kBps = <2400000>; + }; + opp-1 { + opp-peak-kBps = <4800000>; + }; + opp-2 { + opp-peak-kBps = <7456000>; + }; + opp-3 { + opp-peak-kBps = <9600000>; + }; + opp-4 { + opp-peak-kBps = <12896000>; + }; + opp-5 { + opp-peak-kBps = <14928000>; + }; + opp-6 { + opp-peak-kBps = <17056000>; + }; + }; + }; + dc_noc: interconnect@90e0000 { reg = <0 0x090e0000 0 0x5080>; compatible = "qcom,sc7280-dc-noc"; @@ -3367,6 +3652,7 @@ "ss_phy_irq"; power-domains = <&gcc GCC_USB30_PRIM_GDSC>; + required-opps = <&rpmhpd_opp_nom>; resets = <&gcc GCC_USB30_PRIM_BCR>; @@ -3374,6 +3660,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_USB3_0 0>; interconnect-names = "usb-ddr", "apps-usb"; + wakeup-source; + usb_1_dwc3: usb@a600000 { compatible = "snps,dwc3"; reg = <0 0x0a600000 0 0xe000>; @@ -3384,7 +3672,6 @@ phys = <&usb_1_hsphy>, <&usb_1_ssphy>; phy-names = "usb2-phy", "usb3-phy"; maximum-speed = "super-speed"; - wakeup-source; }; }; @@ -3971,791 +4258,791 @@ gpio-ranges = <&tlmm 0 0 175>; wakeup-parent = <&pdc>; - dp_hot_plug_det: dp-hot-plug-det { + dp_hot_plug_det: dp-hot-plug-det-pins { pins = "gpio47"; function = "dp_hot"; }; - edp_hot_plug_det: edp-hot-plug-det { + edp_hot_plug_det: edp-hot-plug-det-pins { pins = "gpio60"; function = "edp_hot"; }; - mi2s0_data0: mi2s0-data0 { + mi2s0_data0: mi2s0-data0-pins { pins = "gpio98"; function = "mi2s0_data0"; }; - mi2s0_data1: mi2s0-data1 { + mi2s0_data1: mi2s0-data1-pins { pins = "gpio99"; function = "mi2s0_data1"; }; - mi2s0_mclk: mi2s0-mclk { + mi2s0_mclk: mi2s0-mclk-pins { pins = "gpio96"; function = "pri_mi2s"; }; - mi2s0_sclk: mi2s0-sclk { + mi2s0_sclk: mi2s0-sclk-pins { pins = "gpio97"; function = "mi2s0_sck"; }; - mi2s0_ws: mi2s0-ws { + mi2s0_ws: mi2s0-ws-pins { pins = "gpio100"; function = "mi2s0_ws"; }; - mi2s1_data0: mi2s1-data0 { + mi2s1_data0: mi2s1-data0-pins { pins = "gpio107"; function = "mi2s1_data0"; }; - mi2s1_sclk: mi2s1-sclk { + mi2s1_sclk: mi2s1-sclk-pins { pins = "gpio106"; function = "mi2s1_sck"; }; - mi2s1_ws: mi2s1-ws { + mi2s1_ws: mi2s1-ws-pins { pins = "gpio108"; function = "mi2s1_ws"; }; - pcie1_clkreq_n: pcie1-clkreq-n { + pcie1_clkreq_n: pcie1-clkreq-n-pins { pins = "gpio79"; function = "pcie1_clkreqn"; }; - qspi_clk: qspi-clk { + qspi_clk: qspi-clk-pins { pins = "gpio14"; function = "qspi_clk"; }; - qspi_cs0: qspi-cs0 { + qspi_cs0: qspi-cs0-pins { pins = "gpio15"; function = "qspi_cs"; }; - qspi_cs1: qspi-cs1 { + qspi_cs1: qspi-cs1-pins { pins = "gpio19"; function = "qspi_cs"; }; - qspi_data01: qspi-data01 { + qspi_data01: qspi-data01-pins { pins = "gpio12", "gpio13"; function = "qspi_data"; }; - qspi_data12: qspi-data12 { + qspi_data12: qspi-data12-pins { pins = "gpio16", "gpio17"; function = "qspi_data"; }; - qup_i2c0_data_clk: qup-i2c0-data-clk { + qup_i2c0_data_clk: qup-i2c0-data-clk-pins { pins = "gpio0", "gpio1"; function = "qup00"; }; - qup_i2c1_data_clk: qup-i2c1-data-clk { + qup_i2c1_data_clk: qup-i2c1-data-clk-pins { pins = "gpio4", "gpio5"; function = "qup01"; }; - qup_i2c2_data_clk: qup-i2c2-data-clk { + qup_i2c2_data_clk: qup-i2c2-data-clk-pins { pins = "gpio8", "gpio9"; function = "qup02"; }; - qup_i2c3_data_clk: qup-i2c3-data-clk { + qup_i2c3_data_clk: qup-i2c3-data-clk-pins { pins = "gpio12", "gpio13"; function = "qup03"; }; - qup_i2c4_data_clk: qup-i2c4-data-clk { + qup_i2c4_data_clk: qup-i2c4-data-clk-pins { pins = "gpio16", "gpio17"; function = "qup04"; }; - qup_i2c5_data_clk: qup-i2c5-data-clk { + qup_i2c5_data_clk: qup-i2c5-data-clk-pins { pins = "gpio20", "gpio21"; function = "qup05"; }; - qup_i2c6_data_clk: qup-i2c6-data-clk { + qup_i2c6_data_clk: qup-i2c6-data-clk-pins { pins = "gpio24", "gpio25"; function = "qup06"; }; - qup_i2c7_data_clk: qup-i2c7-data-clk { + qup_i2c7_data_clk: qup-i2c7-data-clk-pins { pins = "gpio28", "gpio29"; function = "qup07"; }; - qup_i2c8_data_clk: qup-i2c8-data-clk { + qup_i2c8_data_clk: qup-i2c8-data-clk-pins { pins = "gpio32", "gpio33"; function = "qup10"; }; - qup_i2c9_data_clk: qup-i2c9-data-clk { + qup_i2c9_data_clk: qup-i2c9-data-clk-pins { pins = "gpio36", "gpio37"; function = "qup11"; }; - qup_i2c10_data_clk: qup-i2c10-data-clk { + qup_i2c10_data_clk: qup-i2c10-data-clk-pins { pins = "gpio40", "gpio41"; function = "qup12"; }; - qup_i2c11_data_clk: qup-i2c11-data-clk { + qup_i2c11_data_clk: qup-i2c11-data-clk-pins { pins = "gpio44", "gpio45"; function = "qup13"; }; - qup_i2c12_data_clk: qup-i2c12-data-clk { + qup_i2c12_data_clk: qup-i2c12-data-clk-pins { pins = "gpio48", "gpio49"; function = "qup14"; }; - qup_i2c13_data_clk: qup-i2c13-data-clk { + qup_i2c13_data_clk: qup-i2c13-data-clk-pins { pins = "gpio52", "gpio53"; function = "qup15"; }; - qup_i2c14_data_clk: qup-i2c14-data-clk { + qup_i2c14_data_clk: qup-i2c14-data-clk-pins { pins = "gpio56", "gpio57"; function = "qup16"; }; - qup_i2c15_data_clk: qup-i2c15-data-clk { + qup_i2c15_data_clk: qup-i2c15-data-clk-pins { pins = "gpio60", "gpio61"; function = "qup17"; }; - qup_spi0_data_clk: qup-spi0-data-clk { + qup_spi0_data_clk: qup-spi0-data-clk-pins { pins = "gpio0", "gpio1", "gpio2"; function = "qup00"; }; - qup_spi0_cs: qup-spi0-cs { + qup_spi0_cs: qup-spi0-cs-pins { pins = "gpio3"; function = "qup00"; }; - qup_spi0_cs_gpio: qup-spi0-cs-gpio { + qup_spi0_cs_gpio: qup-spi0-cs-gpio-pins { pins = "gpio3"; function = "gpio"; }; - qup_spi1_data_clk: qup-spi1-data-clk { + qup_spi1_data_clk: qup-spi1-data-clk-pins { pins = "gpio4", "gpio5", "gpio6"; function = "qup01"; }; - qup_spi1_cs: qup-spi1-cs { + qup_spi1_cs: qup-spi1-cs-pins { pins = "gpio7"; function = "qup01"; }; - qup_spi1_cs_gpio: qup-spi1-cs-gpio { + qup_spi1_cs_gpio: qup-spi1-cs-gpio-pins { pins = "gpio7"; function = "gpio"; }; - qup_spi2_data_clk: qup-spi2-data-clk { + qup_spi2_data_clk: qup-spi2-data-clk-pins { pins = "gpio8", "gpio9", "gpio10"; function = "qup02"; }; - qup_spi2_cs: qup-spi2-cs { + qup_spi2_cs: qup-spi2-cs-pins { pins = "gpio11"; function = "qup02"; }; - qup_spi2_cs_gpio: qup-spi2-cs-gpio { + qup_spi2_cs_gpio: qup-spi2-cs-gpio-pins { pins = "gpio11"; function = "gpio"; }; - qup_spi3_data_clk: qup-spi3-data-clk { + qup_spi3_data_clk: qup-spi3-data-clk-pins { pins = "gpio12", "gpio13", "gpio14"; function = "qup03"; }; - qup_spi3_cs: qup-spi3-cs { + qup_spi3_cs: qup-spi3-cs-pins { pins = "gpio15"; function = "qup03"; }; - qup_spi3_cs_gpio: qup-spi3-cs-gpio { + qup_spi3_cs_gpio: qup-spi3-cs-gpio-pins { pins = "gpio15"; function = "gpio"; }; - qup_spi4_data_clk: qup-spi4-data-clk { + qup_spi4_data_clk: qup-spi4-data-clk-pins { pins = "gpio16", "gpio17", "gpio18"; function = "qup04"; }; - qup_spi4_cs: qup-spi4-cs { + qup_spi4_cs: qup-spi4-cs-pins { pins = "gpio19"; function = "qup04"; }; - qup_spi4_cs_gpio: qup-spi4-cs-gpio { + qup_spi4_cs_gpio: qup-spi4-cs-gpio-pins { pins = "gpio19"; function = "gpio"; }; - qup_spi5_data_clk: qup-spi5-data-clk { + qup_spi5_data_clk: qup-spi5-data-clk-pins { pins = "gpio20", "gpio21", "gpio22"; function = "qup05"; }; - qup_spi5_cs: qup-spi5-cs { + qup_spi5_cs: qup-spi5-cs-pins { pins = "gpio23"; function = "qup05"; }; - qup_spi5_cs_gpio: qup-spi5-cs-gpio { + qup_spi5_cs_gpio: qup-spi5-cs-gpio-pins { pins = "gpio23"; function = "gpio"; }; - qup_spi6_data_clk: qup-spi6-data-clk { + qup_spi6_data_clk: qup-spi6-data-clk-pins { pins = "gpio24", "gpio25", "gpio26"; function = "qup06"; }; - qup_spi6_cs: qup-spi6-cs { + qup_spi6_cs: qup-spi6-cs-pins { pins = "gpio27"; function = "qup06"; }; - qup_spi6_cs_gpio: qup-spi6-cs-gpio { + qup_spi6_cs_gpio: qup-spi6-cs-gpio-pins { pins = "gpio27"; function = "gpio"; }; - qup_spi7_data_clk: qup-spi7-data-clk { + qup_spi7_data_clk: qup-spi7-data-clk-pins { pins = "gpio28", "gpio29", "gpio30"; function = "qup07"; }; - qup_spi7_cs: qup-spi7-cs { + qup_spi7_cs: qup-spi7-cs-pins { pins = "gpio31"; function = "qup07"; }; - qup_spi7_cs_gpio: qup-spi7-cs-gpio { + qup_spi7_cs_gpio: qup-spi7-cs-gpio-pins { pins = "gpio31"; function = "gpio"; }; - qup_spi8_data_clk: qup-spi8-data-clk { + qup_spi8_data_clk: qup-spi8-data-clk-pins { pins = "gpio32", "gpio33", "gpio34"; function = "qup10"; }; - qup_spi8_cs: qup-spi8-cs { + qup_spi8_cs: qup-spi8-cs-pins { pins = "gpio35"; function = "qup10"; }; - qup_spi8_cs_gpio: qup-spi8-cs-gpio { + qup_spi8_cs_gpio: qup-spi8-cs-gpio-pins { pins = "gpio35"; function = "gpio"; }; - qup_spi9_data_clk: qup-spi9-data-clk { + qup_spi9_data_clk: qup-spi9-data-clk-pins { pins = "gpio36", "gpio37", "gpio38"; function = "qup11"; }; - qup_spi9_cs: qup-spi9-cs { + qup_spi9_cs: qup-spi9-cs-pins { pins = "gpio39"; function = "qup11"; }; - qup_spi9_cs_gpio: qup-spi9-cs-gpio { + qup_spi9_cs_gpio: qup-spi9-cs-gpio-pins { pins = "gpio39"; function = "gpio"; }; - qup_spi10_data_clk: qup-spi10-data-clk { + qup_spi10_data_clk: qup-spi10-data-clk-pins { pins = "gpio40", "gpio41", "gpio42"; function = "qup12"; }; - qup_spi10_cs: qup-spi10-cs { + qup_spi10_cs: qup-spi10-cs-pins { pins = "gpio43"; function = "qup12"; }; - qup_spi10_cs_gpio: qup-spi10-cs-gpio { + qup_spi10_cs_gpio: qup-spi10-cs-gpio-pins { pins = "gpio43"; function = "gpio"; }; - qup_spi11_data_clk: qup-spi11-data-clk { + qup_spi11_data_clk: qup-spi11-data-clk-pins { pins = "gpio44", "gpio45", "gpio46"; function = "qup13"; }; - qup_spi11_cs: qup-spi11-cs { + qup_spi11_cs: qup-spi11-cs-pins { pins = "gpio47"; function = "qup13"; }; - qup_spi11_cs_gpio: qup-spi11-cs-gpio { + qup_spi11_cs_gpio: qup-spi11-cs-gpio-pins { pins = "gpio47"; function = "gpio"; }; - qup_spi12_data_clk: qup-spi12-data-clk { + qup_spi12_data_clk: qup-spi12-data-clk-pins { pins = "gpio48", "gpio49", "gpio50"; function = "qup14"; }; - qup_spi12_cs: qup-spi12-cs { + qup_spi12_cs: qup-spi12-cs-pins { pins = "gpio51"; function = "qup14"; }; - qup_spi12_cs_gpio: qup-spi12-cs-gpio { + qup_spi12_cs_gpio: qup-spi12-cs-gpio-pins { pins = "gpio51"; function = "gpio"; }; - qup_spi13_data_clk: qup-spi13-data-clk { + qup_spi13_data_clk: qup-spi13-data-clk-pins { pins = "gpio52", "gpio53", "gpio54"; function = "qup15"; }; - qup_spi13_cs: qup-spi13-cs { + qup_spi13_cs: qup-spi13-cs-pins { pins = "gpio55"; function = "qup15"; }; - qup_spi13_cs_gpio: qup-spi13-cs-gpio { + qup_spi13_cs_gpio: qup-spi13-cs-gpio-pins { pins = "gpio55"; function = "gpio"; }; - qup_spi14_data_clk: qup-spi14-data-clk { + qup_spi14_data_clk: qup-spi14-data-clk-pins { pins = "gpio56", "gpio57", "gpio58"; function = "qup16"; }; - qup_spi14_cs: qup-spi14-cs { + qup_spi14_cs: qup-spi14-cs-pins { pins = "gpio59"; function = "qup16"; }; - qup_spi14_cs_gpio: qup-spi14-cs-gpio { + qup_spi14_cs_gpio: qup-spi14-cs-gpio-pins { pins = "gpio59"; function = "gpio"; }; - qup_spi15_data_clk: qup-spi15-data-clk { + qup_spi15_data_clk: qup-spi15-data-clk-pins { pins = "gpio60", "gpio61", "gpio62"; function = "qup17"; }; - qup_spi15_cs: qup-spi15-cs { + qup_spi15_cs: qup-spi15-cs-pins { pins = "gpio63"; function = "qup17"; }; - qup_spi15_cs_gpio: qup-spi15-cs-gpio { + qup_spi15_cs_gpio: qup-spi15-cs-gpio-pins { pins = "gpio63"; function = "gpio"; }; - qup_uart0_cts: qup-uart0-cts { + qup_uart0_cts: qup-uart0-cts-pins { pins = "gpio0"; function = "qup00"; }; - qup_uart0_rts: qup-uart0-rts { + qup_uart0_rts: qup-uart0-rts-pins { pins = "gpio1"; function = "qup00"; }; - qup_uart0_tx: qup-uart0-tx { + qup_uart0_tx: qup-uart0-tx-pins { pins = "gpio2"; function = "qup00"; }; - qup_uart0_rx: qup-uart0-rx { + qup_uart0_rx: qup-uart0-rx-pins { pins = "gpio3"; function = "qup00"; }; - qup_uart1_cts: qup-uart1-cts { + qup_uart1_cts: qup-uart1-cts-pins { pins = "gpio4"; function = "qup01"; }; - qup_uart1_rts: qup-uart1-rts { + qup_uart1_rts: qup-uart1-rts-pins { pins = "gpio5"; function = "qup01"; }; - qup_uart1_tx: qup-uart1-tx { + qup_uart1_tx: qup-uart1-tx-pins { pins = "gpio6"; function = "qup01"; }; - qup_uart1_rx: qup-uart1-rx { + qup_uart1_rx: qup-uart1-rx-pins { pins = "gpio7"; function = "qup01"; }; - qup_uart2_cts: qup-uart2-cts { + qup_uart2_cts: qup-uart2-cts-pins { pins = "gpio8"; function = "qup02"; }; - qup_uart2_rts: qup-uart2-rts { + qup_uart2_rts: qup-uart2-rts-pins { pins = "gpio9"; function = "qup02"; }; - qup_uart2_tx: qup-uart2-tx { + qup_uart2_tx: qup-uart2-tx-pins { pins = "gpio10"; function = "qup02"; }; - qup_uart2_rx: qup-uart2-rx { + qup_uart2_rx: qup-uart2-rx-pins { pins = "gpio11"; function = "qup02"; }; - qup_uart3_cts: qup-uart3-cts { + qup_uart3_cts: qup-uart3-cts-pins { pins = "gpio12"; function = "qup03"; }; - qup_uart3_rts: qup-uart3-rts { + qup_uart3_rts: qup-uart3-rts-pins { pins = "gpio13"; function = "qup03"; }; - qup_uart3_tx: qup-uart3-tx { + qup_uart3_tx: qup-uart3-tx-pins { pins = "gpio14"; function = "qup03"; }; - qup_uart3_rx: qup-uart3-rx { + qup_uart3_rx: qup-uart3-rx-pins { pins = "gpio15"; function = "qup03"; }; - qup_uart4_cts: qup-uart4-cts { + qup_uart4_cts: qup-uart4-cts-pins { pins = "gpio16"; function = "qup04"; }; - qup_uart4_rts: qup-uart4-rts { + qup_uart4_rts: qup-uart4-rts-pins { pins = "gpio17"; function = "qup04"; }; - qup_uart4_tx: qup-uart4-tx { + qup_uart4_tx: qup-uart4-tx-pins { pins = "gpio18"; function = "qup04"; }; - qup_uart4_rx: qup-uart4-rx { + qup_uart4_rx: qup-uart4-rx-pins { pins = "gpio19"; function = "qup04"; }; - qup_uart5_cts: qup-uart5-cts { + qup_uart5_cts: qup-uart5-cts-pins { pins = "gpio20"; function = "qup05"; }; - qup_uart5_rts: qup-uart5-rts { + qup_uart5_rts: qup-uart5-rts-pins { pins = "gpio21"; function = "qup05"; }; - qup_uart5_tx: qup-uart5-tx { + qup_uart5_tx: qup-uart5-tx-pins { pins = "gpio22"; function = "qup05"; }; - qup_uart5_rx: qup-uart5-rx { + qup_uart5_rx: qup-uart5-rx-pins { pins = "gpio23"; function = "qup05"; }; - qup_uart6_cts: qup-uart6-cts { + qup_uart6_cts: qup-uart6-cts-pins { pins = "gpio24"; function = "qup06"; }; - qup_uart6_rts: qup-uart6-rts { + qup_uart6_rts: qup-uart6-rts-pins { pins = "gpio25"; function = "qup06"; }; - qup_uart6_tx: qup-uart6-tx { + qup_uart6_tx: qup-uart6-tx-pins { pins = "gpio26"; function = "qup06"; }; - qup_uart6_rx: qup-uart6-rx { + qup_uart6_rx: qup-uart6-rx-pins { pins = "gpio27"; function = "qup06"; }; - qup_uart7_cts: qup-uart7-cts { + qup_uart7_cts: qup-uart7-cts-pins { pins = "gpio28"; function = "qup07"; }; - qup_uart7_rts: qup-uart7-rts { + qup_uart7_rts: qup-uart7-rts-pins { pins = "gpio29"; function = "qup07"; }; - qup_uart7_tx: qup-uart7-tx { + qup_uart7_tx: qup-uart7-tx-pins { pins = "gpio30"; function = "qup07"; }; - qup_uart7_rx: qup-uart7-rx { + qup_uart7_rx: qup-uart7-rx-pins { pins = "gpio31"; function = "qup07"; }; - qup_uart8_cts: qup-uart8-cts { + qup_uart8_cts: qup-uart8-cts-pins { pins = "gpio32"; function = "qup10"; }; - qup_uart8_rts: qup-uart8-rts { + qup_uart8_rts: qup-uart8-rts-pins { pins = "gpio33"; function = "qup10"; }; - qup_uart8_tx: qup-uart8-tx { + qup_uart8_tx: qup-uart8-tx-pins { pins = "gpio34"; function = "qup10"; }; - qup_uart8_rx: qup-uart8-rx { + qup_uart8_rx: qup-uart8-rx-pins { pins = "gpio35"; function = "qup10"; }; - qup_uart9_cts: qup-uart9-cts { + qup_uart9_cts: qup-uart9-cts-pins { pins = "gpio36"; function = "qup11"; }; - qup_uart9_rts: qup-uart9-rts { + qup_uart9_rts: qup-uart9-rts-pins { pins = "gpio37"; function = "qup11"; }; - qup_uart9_tx: qup-uart9-tx { + qup_uart9_tx: qup-uart9-tx-pins { pins = "gpio38"; function = "qup11"; }; - qup_uart9_rx: qup-uart9-rx { + qup_uart9_rx: qup-uart9-rx-pins { pins = "gpio39"; function = "qup11"; }; - qup_uart10_cts: qup-uart10-cts { + qup_uart10_cts: qup-uart10-cts-pins { pins = "gpio40"; function = "qup12"; }; - qup_uart10_rts: qup-uart10-rts { + qup_uart10_rts: qup-uart10-rts-pins { pins = "gpio41"; function = "qup12"; }; - qup_uart10_tx: qup-uart10-tx { + qup_uart10_tx: qup-uart10-tx-pins { pins = "gpio42"; function = "qup12"; }; - qup_uart10_rx: qup-uart10-rx { + qup_uart10_rx: qup-uart10-rx-pins { pins = "gpio43"; function = "qup12"; }; - qup_uart11_cts: qup-uart11-cts { + qup_uart11_cts: qup-uart11-cts-pins { pins = "gpio44"; function = "qup13"; }; - qup_uart11_rts: qup-uart11-rts { + qup_uart11_rts: qup-uart11-rts-pins { pins = "gpio45"; function = "qup13"; }; - qup_uart11_tx: qup-uart11-tx { + qup_uart11_tx: qup-uart11-tx-pins { pins = "gpio46"; function = "qup13"; }; - qup_uart11_rx: qup-uart11-rx { + qup_uart11_rx: qup-uart11-rx-pins { pins = "gpio47"; function = "qup13"; }; - qup_uart12_cts: qup-uart12-cts { + qup_uart12_cts: qup-uart12-cts-pins { pins = "gpio48"; function = "qup14"; }; - qup_uart12_rts: qup-uart12-rts { + qup_uart12_rts: qup-uart12-rts-pins { pins = "gpio49"; function = "qup14"; }; - qup_uart12_tx: qup-uart12-tx { + qup_uart12_tx: qup-uart12-tx-pins { pins = "gpio50"; function = "qup14"; }; - qup_uart12_rx: qup-uart12-rx { + qup_uart12_rx: qup-uart12-rx-pins { pins = "gpio51"; function = "qup14"; }; - qup_uart13_cts: qup-uart13-cts { + qup_uart13_cts: qup-uart13-cts-pins { pins = "gpio52"; function = "qup15"; }; - qup_uart13_rts: qup-uart13-rts { + qup_uart13_rts: qup-uart13-rts-pins { pins = "gpio53"; function = "qup15"; }; - qup_uart13_tx: qup-uart13-tx { + qup_uart13_tx: qup-uart13-tx-pins { pins = "gpio54"; function = "qup15"; }; - qup_uart13_rx: qup-uart13-rx { + qup_uart13_rx: qup-uart13-rx-pins { pins = "gpio55"; function = "qup15"; }; - qup_uart14_cts: qup-uart14-cts { + qup_uart14_cts: qup-uart14-cts-pins { pins = "gpio56"; function = "qup16"; }; - qup_uart14_rts: qup-uart14-rts { + qup_uart14_rts: qup-uart14-rts-pins { pins = "gpio57"; function = "qup16"; }; - qup_uart14_tx: qup-uart14-tx { + qup_uart14_tx: qup-uart14-tx-pins { pins = "gpio58"; function = "qup16"; }; - qup_uart14_rx: qup-uart14-rx { + qup_uart14_rx: qup-uart14-rx-pins { pins = "gpio59"; function = "qup16"; }; - qup_uart15_cts: qup-uart15-cts { + qup_uart15_cts: qup-uart15-cts-pins { pins = "gpio60"; function = "qup17"; }; - qup_uart15_rts: qup-uart15-rts { + qup_uart15_rts: qup-uart15-rts-pins { pins = "gpio61"; function = "qup17"; }; - qup_uart15_tx: qup-uart15-tx { + qup_uart15_tx: qup-uart15-tx-pins { pins = "gpio62"; function = "qup17"; }; - qup_uart15_rx: qup-uart15-rx { + qup_uart15_rx: qup-uart15-rx-pins { pins = "gpio63"; function = "qup17"; }; - sdc1_clk: sdc1-clk { + sdc1_clk: sdc1-clk-pins { pins = "sdc1_clk"; }; - sdc1_cmd: sdc1-cmd { + sdc1_cmd: sdc1-cmd-pins { pins = "sdc1_cmd"; }; - sdc1_data: sdc1-data { + sdc1_data: sdc1-data-pins { pins = "sdc1_data"; }; - sdc1_rclk: sdc1-rclk { + sdc1_rclk: sdc1-rclk-pins { pins = "sdc1_rclk"; }; - sdc1_clk_sleep: sdc1-clk-sleep { + sdc1_clk_sleep: sdc1-clk-sleep-pins { pins = "sdc1_clk"; drive-strength = <2>; bias-bus-hold; }; - sdc1_cmd_sleep: sdc1-cmd-sleep { + sdc1_cmd_sleep: sdc1-cmd-sleep-pins { pins = "sdc1_cmd"; drive-strength = <2>; bias-bus-hold; }; - sdc1_data_sleep: sdc1-data-sleep { + sdc1_data_sleep: sdc1-data-sleep-pins { pins = "sdc1_data"; drive-strength = <2>; bias-bus-hold; }; - sdc1_rclk_sleep: sdc1-rclk-sleep { + sdc1_rclk_sleep: sdc1-rclk-sleep-pins { pins = "sdc1_rclk"; drive-strength = <2>; bias-bus-hold; }; - sdc2_clk: sdc2-clk { + sdc2_clk: sdc2-clk-pins { pins = "sdc2_clk"; }; - sdc2_cmd: sdc2-cmd { + sdc2_cmd: sdc2-cmd-pins { pins = "sdc2_cmd"; }; - sdc2_data: sdc2-data { + sdc2_data: sdc2-data-pins { pins = "sdc2_data"; }; - sdc2_clk_sleep: sdc2-clk-sleep { + sdc2_clk_sleep: sdc2-clk-sleep-pins { pins = "sdc2_clk"; drive-strength = <2>; bias-bus-hold; }; - sdc2_cmd_sleep: sdc2-cmd-sleep { + sdc2_cmd_sleep: sdc2-cmd-sleep-pins { pins = "sdc2_cmd"; drive-strength = <2>; bias-bus-hold; }; - sdc2_data_sleep: sdc2-data-sleep { + sdc2_data_sleep: sdc2-data-sleep-pins { pins = "sdc2_data"; drive-strength = <2>; bias-bus-hold; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts index 45058ad0a1c8adea816dddd1e0565fefdf2ad447..fea7d8273ccde66c775ab36920191c0a2953937f 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts @@ -87,7 +87,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; regulator-boot-on; regulator-always-on; }; @@ -97,7 +96,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l6b: ldo6 { @@ -105,7 +103,6 @@ regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; regulator-initial-mode = ; - regulator-allow-set-load; regulator-boot-on; }; }; @@ -119,7 +116,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l7c: ldo7 { @@ -135,7 +131,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; @@ -158,7 +153,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l6d: ldo6 { @@ -166,7 +160,6 @@ regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l7d: ldo7 { @@ -174,7 +167,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l9d: ldo9 { @@ -182,7 +174,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; }; @@ -203,16 +194,20 @@ clock-frequency = <400000>; pinctrl-names = "default"; - pinctrl-0 = <&qup0_i2c4_default>, <&ts0_default>; + pinctrl-0 = <&qup0_i2c4_default>; status = "okay"; touchscreen@10 { compatible = "hid-over-i2c"; reg = <0x10>; + hid-descr-addr = <0x1>; interrupts-extended = <&tlmm 175 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&ts0_default>; }; }; @@ -228,24 +223,36 @@ clock-frequency = <400000>; pinctrl-names = "default"; - pinctrl-0 = <&qup2_i2c5_default>, <&kybd_default>, <&tpad_default>; + pinctrl-0 = <&qup2_i2c5_default>; status = "okay"; touchpad@15 { compatible = "hid-over-i2c"; reg = <0x15>; + hid-descr-addr = <0x1>; interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&tpad_default>; + + wakeup-source; }; keyboard@68 { compatible = "hid-over-i2c"; reg = <0x68>; + hid-descr-addr = <0x1>; interrupts-extended = <&tlmm 104 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&kybd_default>; + + wakeup-source; }; }; @@ -414,7 +421,7 @@ int-n { pins = "gpio175"; function = "gpio"; - bias-pull-up; + bias-disable; }; reset-n { diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts index 84dc92dda0b80cdde54e969d7dcdb77a9b67626a..b2b744bb8a538f0111d4ba96ffbfc5e896d1b017 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts @@ -79,7 +79,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; regulator-boot-on; }; @@ -88,7 +87,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l6b: ldo6 { @@ -96,7 +94,6 @@ regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; regulator-initial-mode = ; - regulator-allow-set-load; regulator-boot-on; regulator-always-on; // FIXME: VDD_A_EDP_0_0P9 }; @@ -111,7 +108,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l12c: ldo12 { @@ -119,7 +115,6 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l13c: ldo13 { @@ -127,7 +122,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; @@ -142,7 +136,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l4d: ldo4 { @@ -150,7 +143,6 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l7d: ldo7 { @@ -158,7 +150,6 @@ regulator-min-microvolt = <3072000>; regulator-max-microvolt = <3072000>; regulator-initial-mode = ; - regulator-allow-set-load; }; vreg_l9d: ldo9 { @@ -166,7 +157,6 @@ regulator-min-microvolt = <912000>; regulator-max-microvolt = <912000>; regulator-initial-mode = ; - regulator-allow-set-load; }; }; }; @@ -187,7 +177,7 @@ clock-frequency = <400000>; pinctrl-names = "default"; - pinctrl-0 = <&qup0_i2c4_default>, <&ts0_default>; + pinctrl-0 = <&qup0_i2c4_default>; status = "okay"; @@ -195,9 +185,13 @@ touchscreen@10 { compatible = "hid-over-i2c"; reg = <0x10>; + hid-descr-addr = <0x1>; interrupts-extended = <&tlmm 175 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&ts0_default>; }; }; @@ -213,35 +207,63 @@ clock-frequency = <400000>; pinctrl-names = "default"; - pinctrl-0 = <&qup2_i2c5_default>, <&kybd_default>, <&tpad_default>; + pinctrl-0 = <&qup2_i2c5_default>; status = "okay"; + touchpad@15 { + compatible = "hid-over-i2c"; + reg = <0x15>; + + hid-descr-addr = <0x1>; + interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>; + vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&tpad_default>; + + wakeup-source; + + status = "disabled"; + }; + touchpad@2c { compatible = "hid-over-i2c"; reg = <0x2c>; + hid-descr-addr = <0x20>; interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&tpad_default>; + + wakeup-source; }; keyboard@68 { compatible = "hid-over-i2c"; reg = <0x68>; + hid-descr-addr = <0x1>; interrupts-extended = <&tlmm 104 IRQ_TYPE_LEVEL_LOW>; vdd-supply = <&vreg_misc_3p3>; + + pinctrl-names = "default"; + pinctrl-0 = <&kybd_default>; + + wakeup-source; }; }; &remoteproc_adsp { - firmware-name = "qcom/sc8280xp/qcadsp8280.mbn"; + firmware-name = "qcom/sc8280xp/LENOVO/21BX/qcadsp8280.mbn"; status = "okay"; }; &remoteproc_nsp0 { - firmware-name = "qcom/sc8280xp/qccdsp8280.mbn"; + firmware-name = "qcom/sc8280xp/LENOVO/21BX/qccdsp8280.mbn"; status = "okay"; }; @@ -373,7 +395,7 @@ int-n { pins = "gpio175"; function = "gpio"; - bias-pull-up; + bias-disable; }; reset-n { diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi index ae90b97aecb8e49a1b50fb2584d503343e11d06f..24836b6b9bbc971b7852c5c21fefea2b42961dd6 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi @@ -60,9 +60,8 @@ #interrupt-cells = <2>; }; - pmc8280c_lpg: lpg@e800 { + pmc8280c_lpg: pwm { compatible = "qcom,pm8350c-pwm"; - reg = <0xe800>; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index 49ea8b5612fc2d1cb07f3460aa314f74f14ed3b1..c32bcded2aef19b30b375b2332c8cb9d6a1abc14 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -1312,6 +1312,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_USB3_0 0>; interconnect-names = "usb-ddr", "apps-usb"; + wakeup-source; + status = "disabled"; usb_0_dwc3: usb@a600000 { @@ -1364,6 +1366,8 @@ <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_USB3_1 0>; interconnect-names = "usb-ddr", "apps-usb"; + wakeup-source; + status = "disabled"; usb_1_dwc3: usb@a800000 { diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi index 1bc9091cad2a8a1acbdf5a000093056358c4cfdd..b51b85f583e5d2803a28419b2d5ffeea6f184371 100644 --- a/arch/arm64/boot/dts/qcom/sdm630.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi @@ -697,9 +697,15 @@ #thermal-sensor-cells = <1>; }; - tcsr_mutex_regs: syscon@1f40000 { - compatible = "syscon"; - reg = <0x01f40000 0x40000>; + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x01f40000 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_regs_1: syscon@1f60000 { + compatible = "qcom,sdm630-tcsr", "syscon"; + reg = <0x01f60000 0x20000>; }; tlmm: pinctrl@3100000 { @@ -2351,12 +2357,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - sound: sound { }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts index c6e2c571b45270600d61190b6e842da410f74edd..132417e2d11e52a7a706dbcfc45c40ecb6748303 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts @@ -522,7 +522,7 @@ &pcie0 { status = "okay"; - perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>; enable-gpio = <&tlmm 134 GPIO_ACTIVE_HIGH>; vddpe-3v3-supply = <&pcie0_3p3v_dual>; @@ -540,7 +540,7 @@ &pcie1 { status = "okay"; - perst-gpio = <&tlmm 102 GPIO_ACTIVE_LOW>; + perst-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&pcie1_default_state>; @@ -1081,7 +1081,7 @@ pinctrl-names = "default"; clock-names = "extclk"; clocks = <&rpmhcc RPMH_LN_BB_CLK2>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-buck-sido-supply = <&vreg_s4a_1p8>; vdd-tx-supply = <&vreg_s4a_1p8>; @@ -1214,8 +1214,6 @@ reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&cam0_default>; - gpios = <&tlmm 13 0>, - <&tlmm 9 GPIO_ACTIVE_LOW>; clocks = <&clock_camcc CAM_CC_MCLK0_CLK>; clock-names = "xvclk"; @@ -1228,8 +1226,6 @@ * both have to be enabled through the power management * gpios. */ - power-domains = <&clock_camcc TITAN_TOP_GDSC>; - dovdd-supply = <&vreg_lvs1a_1p8>; avdd-supply = <&cam0_avdd_2v8>; dvdd-supply = <&cam0_dvdd_1v2>; @@ -1255,11 +1251,9 @@ reg = <0x60>; // CAM3_RST_N - enable-gpios = <&tlmm 21 0>; + enable-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>; pinctrl-names = "default"; pinctrl-0 = <&cam3_default>; - gpios = <&tlmm 16 0>, - <&tlmm 21 0>; clocks = <&clock_camcc CAM_CC_MCLK3_CLK>; clock-names = "xclk"; @@ -1273,8 +1267,6 @@ * * No 1.2V vddd-supply regulator is used. */ - power-domains = <&clock_camcc TITAN_TOP_GDSC>; - vdddo-supply = <&vreg_lvs1a_1p8>; vdda-supply = <&cam3_avdd_2v8>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts index 7713e8060c5b64c138bfe14192b963c90e2dab6c..de2d10e0315af96ae6a09e38a6e86f9fae15f2d0 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts @@ -536,42 +536,42 @@ reg = ; label = "xo_therm"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; adc-chan@4d { reg = ; label = "msm_therm"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; adc-chan@4f { reg = ; label = "pa_therm1"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; adc-chan@51 { reg = ; label = "quiet_therm"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; adc-chan@83 { reg = ; label = "vph_pwr"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; adc-chan@85 { reg = ; label = "vcoin"; qcom,ratiometric; - qcom,hw-settle-time-us = <200>; + qcom,hw-settle-time = <200>; }; }; diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts index 82c27f90d300d26715d6ac67f5ba018238ceffd7..0f470cf1ed1c1f9859f64cb0b444a1922cd444cc 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts @@ -546,7 +546,7 @@ pinctrl-names = "default"; clock-names = "extclk"; clocks = <&rpmhcc RPMH_LN_BB_CLK2>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-buck-sido-supply = <&vreg_s4a_1p8>; vdd-tx-supply = <&vreg_s4a_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts index 7747081b98875aad6d592421df3dbf95ce699e37..afc17e4d403fcea2c5a391e0860a3dd457fef290 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts @@ -126,7 +126,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; - gpio = <&tlmm 23 0>; + gpio = <&tlmm 23 GPIO_ACTIVE_HIGH>; regulator-always-on; regulator-boot-on; enable-active-high; @@ -522,7 +522,7 @@ }; &pm8998_gpio { - volume_up_gpio: pm8998_gpio6 { + volume_up_gpio: pm8998-gpio6-state { pinconf { qcom,drive-strength = ; function = "normal"; @@ -617,7 +617,7 @@ pins = "gpio6", "gpio10"; function = "gpio"; drive-strength = <8>; - bias-disable = <0>; + bias-disable; }; sde_dsi_suspend: sde-dsi-suspend { @@ -712,7 +712,7 @@ pinctrl-names = "default"; clock-names = "extclk"; clocks = <&rpmhcc RPMH_LN_BB_CLK2>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; vdd-buck-sido-supply = <&vreg_s4a_1p8>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-tx-supply = <&vreg_s4a_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi index f0e286715d1bdbe643510b4b8dbc0b0da1dad0dd..d761da47220dd33ee10235d628a17987fba8b963 100644 --- a/arch/arm64/boot/dts/qcom/sdm845.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi @@ -919,12 +919,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - smp2p-cdsp { compatible = "qcom,smp2p"; qcom,smem = <94>, <432>; @@ -2138,11 +2132,48 @@ llcc: system-cache-controller@1100000 { compatible = "qcom,sdm845-llcc"; - reg = <0 0x01100000 0 0x200000>, <0 0x01300000 0 0x50000>; + reg = <0 0x01100000 0 0x31000>, <0 0x01300000 0 0x50000>; reg-names = "llcc_base", "llcc_broadcast_base"; interrupts = ; }; + pmu@114a000 { + compatible = "qcom,sdm845-llcc-bwmon"; + reg = <0 0x0114a000 0 0x1000>; + interrupts = ; + interconnects = <&mem_noc MASTER_LLCC 3 &mem_noc SLAVE_EBI1 3>; + + operating-points-v2 = <&llcc_bwmon_opp_table>; + + llcc_bwmon_opp_table: opp-table { + compatible = "operating-points-v2"; + + /* + * The interconnect path bandwidth taken from + * cpu4_opp_table bandwidth for gladiator_noc-mem_noc + * interconnect. This also matches the + * bandwidth table of qcom,llccbw (qcom,bw-tbl, + * bus width: 4 bytes) from msm-4.9 downstream + * kernel. + */ + opp-0 { + opp-peak-kBps = <800000>; + }; + opp-1 { + opp-peak-kBps = <1804000>; + }; + opp-2 { + opp-peak-kBps = <3072000>; + }; + opp-3 { + opp-peak-kBps = <5412000>; + }; + opp-4 { + opp-peak-kBps = <7216000>; + }; + }; + }; + pmu@1436400 { compatible = "qcom,sdm845-bwmon", "qcom,msm8998-bwmon"; reg = <0 0x01436400 0 0x600>; @@ -2588,9 +2619,15 @@ status = "disabled"; }; - tcsr_mutex_regs: syscon@1f40000 { - compatible = "syscon"; - reg = <0 0x01f40000 0 0x40000>; + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0 0x01f40000 0 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_regs_1: syscon@1f60000 { + compatible = "qcom,sdm845-tcsr", "syscon"; + reg = <0 0x01f60000 0 0x20000>; }; tlmm: pinctrl@3400000 { @@ -3207,7 +3244,7 @@ <&pdc_reset PDC_MODEM_SYNC_RESET>; reset-names = "mss_restart", "pdc_reset"; - qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>; + qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>; power-domains = <&rpmhpd SDM845_CX>, <&rpmhpd SDM845_MX>, @@ -4836,7 +4873,7 @@ aoss_qmp: power-controller@c300000 { compatible = "qcom,sdm845-aoss-qmp", "qcom,aoss-qmp"; - reg = <0 0x0c300000 0 0x100000>; + reg = <0 0x0c300000 0 0x400>; interrupts = ; mboxes = <&apss_shared 0>; @@ -4851,6 +4888,11 @@ }; }; + sram@c3f0000 { + compatible = "qcom,sdm845-rpmh-stats"; + reg = <0 0x0c3f0000 0 0x400>; + }; + spmi_bus: spmi@c440000 { compatible = "qcom,spmi-pmic-arb"; reg = <0 0x0c440000 0 0x1100>, diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index a7af1bed431296e2c091d68ce49099b459a8f1ed..be59a8ba9c1fe60767d4c53ce267518c2d5a1185 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -772,7 +772,7 @@ pinctrl-names = "default"; clock-names = "extclk"; clocks = <&rpmhcc RPMH_LN_BB_CLK2>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-buck-sido-supply = <&vreg_s4a_1p8>; vdd-tx-supply = <&vreg_s4a_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts index b0315eeb132054c3fd352742c9e6b5ecc9ceb7ae..f954fe5cb61ab22641b18c88539d01ac89a68afc 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts @@ -704,7 +704,7 @@ pinctrl-names = "default"; clock-names = "extclk"; clocks = <&rpmhcc RPMH_LN_BB_CLK2>; - reset-gpios = <&tlmm 64 0>; + reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-buck-sido-supply = <&vreg_s4a_1p8>; vdd-tx-supply = <&vreg_s4a_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts index 0aad2e94e757518df7dead5a0e491b1d67698161..6a8b88cc438533766221983efdc50ee6332f6a7b 100644 --- a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts +++ b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts @@ -87,7 +87,7 @@ }; &sdc2_off_state { - sd-cd { + sd-cd-pins { pins = "gpio98"; drive-strength = <2>; bias-disable; @@ -95,7 +95,7 @@ }; &sdc2_on_state { - sd-cd { + sd-cd-pins { pins = "gpio98"; drive-strength = <2>; bias-pull-up; diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi index 8c582a9e4ada4f0bef8f066fa71d4129ff1c3b84..1fe3fa3ad877067fab49ebd543434c3df25787d9 100644 --- a/arch/arm64/boot/dts/qcom/sm6125.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi @@ -387,19 +387,19 @@ #interrupt-cells = <2>; sdc2_off_state: sdc2-off-state { - clk { + clk-pins { pins = "sdc2_clk"; drive-strength = <2>; bias-disable; }; - cmd { + cmd-pins { pins = "sdc2_cmd"; drive-strength = <2>; bias-pull-up; }; - data { + data-pins { pins = "sdc2_data"; drive-strength = <2>; bias-pull-up; @@ -413,13 +413,13 @@ bias-disable; }; - cmd { + cmd-pins-pins { pins = "sdc2_cmd"; drive-strength = <10>; bias-pull-up; }; - data { + data-pins { pins = "sdc2_data"; drive-strength = <10>; bias-pull-up; diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi index d06aefdf3d9edff5063d4f8ac0921a7866a79317..c39de7d3ace0bc61af41a69253e5fc287d64c6dd 100644 --- a/arch/arm64/boot/dts/qcom/sm6350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi @@ -1,11 +1,14 @@ // SPDX-License-Identifier: BSD-3-Clause /* * Copyright (c) 2021, Konrad Dybcio + * Copyright (c) 2022, Luca Weiss */ #include #include +#include #include +#include #include #include #include @@ -517,6 +520,26 @@ }; }; + gpi_dma0: dma-controller@800000 { + compatible = "qcom,sm6350-gpi-dma"; + reg = <0 0x00800000 0 0x60000>; + interrupts = , + , + , + , + , + , + , + , + , + ; + dma-channels = <10>; + dma-channel-mask = <0x1f>; + iommus = <&apps_smmu 0x56 0x0>; + #dma-cells = <3>; + status = "disabled"; + }; + qupv3_id_0: geniqup@8c0000 { compatible = "qcom,geni-se-qup"; reg = <0x0 0x8c0000 0x0 0x2000>; @@ -537,8 +560,15 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c0_default>; interrupts = ; + dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>, + <&gpi_dma0 1 0 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; @@ -550,12 +580,39 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c2_default>; interrupts = ; + dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>, + <&gpi_dma0 1 2 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_0 0>, + <&aggre1_noc MASTER_QUP_0 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; }; + gpi_dma1: dma-controller@900000 { + compatible = "qcom,sm6350-gpi-dma"; + reg = <0 0x00900000 0 0x60000>; + interrupts = , + , + , + , + , + , + , + , + , + ; + dma-channels = <10>; + dma-channel-mask = <0x3f>; + iommus = <&apps_smmu 0x4d6 0x0>; + #dma-cells = <3>; + status = "disabled"; + }; + qupv3_id_1: geniqup@9c0000 { compatible = "qcom,geni-se-qup"; reg = <0x0 0x9c0000 0x0 0x2000>; @@ -576,8 +633,15 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c6_default>; interrupts = ; + dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>, + <&gpi_dma1 1 0 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; @@ -589,8 +653,15 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c7_default>; interrupts = ; + dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>, + <&gpi_dma1 1 1 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; @@ -602,8 +673,15 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c8_default>; interrupts = ; + dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>, + <&gpi_dma1 1 2 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; @@ -615,6 +693,9 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_uart9_default>; interrupts = ; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>; + interconnect-names = "qup-core", "qup-config"; status = "disabled"; }; @@ -626,13 +707,67 @@ pinctrl-names = "default"; pinctrl-0 = <&qup_i2c10_default>; interrupts = ; + dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>, + <&gpi_dma1 1 4 QCOM_GPI_I2C>; + dma-names = "tx", "rx"; #address-cells = <1>; #size-cells = <0>; + interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>, + <&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>; + interconnect-names = "qup-core", "qup-config", "qup-memory"; status = "disabled"; }; }; + config_noc: interconnect@1500000 { + compatible = "qcom,sm6350-config-noc"; + reg = <0 0x01500000 0 0x28000>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + system_noc: interconnect@1620000 { + compatible = "qcom,sm6350-system-noc"; + reg = <0 0x01620000 0 0x17080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + + clk_virt: interconnect-clk-virt { + compatible = "qcom,sm6350-clk-virt"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + }; + + aggre1_noc: interconnect@16e0000 { + compatible = "qcom,sm6350-aggre1-noc"; + reg = <0 0x016e0000 0 0x15080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + aggre2_noc: interconnect@1700000 { + compatible = "qcom,sm6350-aggre2-noc"; + reg = <0 0x01700000 0 0x1f880>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + + compute_noc: interconnect-compute-noc { + compatible = "qcom,sm6350-compute-noc"; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + }; + + mmss_noc: interconnect@1740000 { + compatible = "qcom,sm6350-mmss-noc"; + reg = <0 0x01740000 0 0x1c100>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + ufs_mem_hc: ufs@1d84000 { compatible = "qcom,sm6350-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; @@ -933,6 +1068,10 @@ <&gcc GCC_SDCC2_APPS_CLK>, <&rpmhcc RPMH_CXO_CLK>; clock-names = "iface", "core", "xo"; + interconnects = <&aggre2_noc MASTER_SDCC_2 0 &clk_virt SLAVE_EBI_CH0 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_SDCC_2 0>; + interconnect-names = "sdhc-ddr", "cpu-sdhc"; + qcom,dll-config = <0x0007642c>; qcom,ddr-config = <0x80040868>; power-domains = <&rpmhpd SM6350_CX>; @@ -947,11 +1086,15 @@ opp-100000000 { opp-hz = /bits/ 64 <100000000>; required-opps = <&rpmhpd_opp_svs_l1>; + opp-peak-kBps = <790000 131000>; + opp-avg-kBps = <50000 50000>; }; opp-202000000 { opp-hz = /bits/ 64 <202000000>; required-opps = <&rpmhpd_opp_nom>; + opp-peak-kBps = <3190000 294000>; + opp-avg-kBps = <261438 300000>; }; }; }; @@ -1017,12 +1160,33 @@ }; }; + dc_noc: interconnect@9160000 { + compatible = "qcom,sm6350-dc-noc"; + reg = <0 0x09160000 0 0x3200>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + system-cache-controller@9200000 { compatible = "qcom,sm6350-llcc"; reg = <0 0x09200000 0 0x50000>, <0 0x09600000 0 0x50000>; reg-names = "llcc_base", "llcc_broadcast_base"; }; + gem_noc: interconnect@9680000 { + compatible = "qcom,sm6350-gem-noc"; + reg = <0 0x09680000 0 0x3e200>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + npu_noc: interconnect@9990000 { + compatible = "qcom,sm6350-npu-noc"; + reg = <0 0x09990000 0 0x1600>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + usb_1: usb@a6f8800 { compatible = "qcom,sm6350-dwc3", "qcom,dwc3"; reg = <0 0x0a6f8800 0 0x400>; @@ -1054,6 +1218,10 @@ resets = <&gcc GCC_USB30_PRIM_BCR>; + interconnects = <&aggre2_noc MASTER_USB3 0 &clk_virt SLAVE_EBI_CH0 0>, + <&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_USB3 0>; + interconnect-names = "usb-ddr", "apps-usb"; + usb_1_dwc3: usb@a600000 { compatible = "snps,dwc3"; reg = <0 0x0a600000 0 0xcd00>; @@ -1146,49 +1314,49 @@ #interrupt-cells = <2>; gpio-ranges = <&tlmm 0 0 157>; - qup_uart9_default: qup-uart9-default { + qup_uart9_default: qup-uart9-default-state { pins = "gpio25", "gpio26"; function = "qup13_f2"; drive-strength = <2>; bias-disable; }; - qup_i2c0_default: qup-i2c0-default { + qup_i2c0_default: qup-i2c0-default-state { pins = "gpio0", "gpio1"; function = "qup00"; drive-strength = <2>; bias-pull-up; }; - qup_i2c2_default: qup-i2c2-default { + qup_i2c2_default: qup-i2c2-default-state { pins = "gpio45", "gpio46"; function = "qup02"; drive-strength = <2>; bias-pull-up; }; - qup_i2c6_default: qup-i2c6-default { + qup_i2c6_default: qup-i2c6-default-state { pins = "gpio13", "gpio14"; function = "qup10"; drive-strength = <2>; bias-pull-up; }; - qup_i2c7_default: qup-i2c7-default { + qup_i2c7_default: qup-i2c7-default-state { pins = "gpio27", "gpio28"; function = "qup11"; drive-strength = <2>; bias-pull-up; }; - qup_i2c8_default: qup-i2c8-default { + qup_i2c8_default: qup-i2c8-default-state { pins = "gpio19", "gpio20"; function = "qup12"; drive-strength = <2>; bias-pull-up; }; - qup_i2c10_default: qup-i2c10-default { + qup_i2c10_default: qup-i2c10-default-state { pins = "gpio4", "gpio5"; function = "qup14"; drive-strength = <2>; diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts index c76abe7587b4f9f340c3be9f8bb40058a023ff20..30c94fd4fe61f81ae049f3c79c0a984150deee3d 100644 --- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts +++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts @@ -12,6 +12,7 @@ #include "sm7225.dtsi" #include "pm6150l.dtsi" #include "pm6350.dtsi" +#include "pm7250b.dtsi" / { model = "Fairphone 4"; @@ -70,6 +71,36 @@ qcom,vmid = <15>; }; }; + + thermal-zones { + chg-skin-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&pm7250b_adc_tm 0>; + + trips { + active-config0 { + temperature = <125000>; + hysteresis = <1000>; + type = "passive"; + }; + }; + }; + + conn-thermal { + polling-delay-passive = <0>; + polling-delay = <0>; + thermal-sensors = <&pm7250b_adc_tm 1>; + + trips { + active-config0 { + temperature = <125000>; + hysteresis = <1000>; + type = "passive"; + }; + }; + }; + }; }; &adsp { @@ -353,6 +384,42 @@ linux,code = ; }; +&pm7250b_adc { + adc-chan@4d { + reg = ; + qcom,ratiometric; + qcom,hw-settle-time = <200>; + qcom,pre-scaling = <1 1>; + label = "charger_skin_therm"; + }; + + adc-chan@4f { + reg = ; + qcom,ratiometric; + qcom,hw-settle-time = <200>; + qcom,pre-scaling = <1 1>; + label = "conn_therm"; + }; +}; + +&pm7250b_adc_tm { + status = "okay"; + + charger-skin-therm@0 { + reg = <0>; + io-channels = <&pm7250b_adc ADC5_AMUX_THM1_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + + conn-therm@1 { + reg = <1>; + io-channels = <&pm7250b_adc ADC5_AMUX_THM3_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; +}; + &qupv3_id_1 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index 7d509ecd44dabcdcbe032c7d6fb95bd11a69934b..cef8c4f4f0ff278c8d894061d12c4b6ea74d69d3 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -585,12 +585,6 @@ }; }; - tcsr_mutex: hwlock { - compatible = "qcom,tcsr-mutex"; - syscon = <&tcsr_mutex_regs 0 0x1000>; - #hwlock-cells = <1>; - }; - memory@80000000 { device_type = "memory"; /* We expect the bootloader to fill in the size */ @@ -2054,9 +2048,15 @@ qcom,bcm-voters = <&apps_bcm_voter>; }; - tcsr_mutex_regs: syscon@1f40000 { - compatible = "syscon"; - reg = <0x0 0x01f40000 0x0 0x40000>; + tcsr_mutex: hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x0 0x01f40000 0x0 0x20000>; + #hwlock-cells = <1>; + }; + + tcsr_regs_1: syscon@1f60000 { + compatible = "qcom,sm8150-tcsr", "syscon"; + reg = <0x0 0x01f60000 0x0 0x20000>; }; remoteproc_slpi: remoteproc@2400000 { @@ -3394,57 +3394,49 @@ compute-cb@1 { compatible = "qcom,fastrpc-compute-cb"; reg = <1>; - iommus = <&apps_smmu 0x1401 0x2040>, - <&apps_smmu 0x1421 0x0>, - <&apps_smmu 0x2001 0x420>, - <&apps_smmu 0x2041 0x0>; + iommus = <&apps_smmu 0x1001 0x0460>; }; compute-cb@2 { compatible = "qcom,fastrpc-compute-cb"; reg = <2>; - iommus = <&apps_smmu 0x2 0x3440>, - <&apps_smmu 0x22 0x3400>; + iommus = <&apps_smmu 0x1002 0x0460>; }; compute-cb@3 { compatible = "qcom,fastrpc-compute-cb"; reg = <3>; - iommus = <&apps_smmu 0x3 0x3440>, - <&apps_smmu 0x1423 0x0>, - <&apps_smmu 0x2023 0x0>; + iommus = <&apps_smmu 0x1003 0x0460>; }; compute-cb@4 { compatible = "qcom,fastrpc-compute-cb"; reg = <4>; - iommus = <&apps_smmu 0x4 0x3440>, - <&apps_smmu 0x24 0x3400>; + iommus = <&apps_smmu 0x1004 0x0460>; }; compute-cb@5 { compatible = "qcom,fastrpc-compute-cb"; reg = <5>; - iommus = <&apps_smmu 0x5 0x3440>, - <&apps_smmu 0x25 0x3400>; + iommus = <&apps_smmu 0x1005 0x0460>; }; compute-cb@6 { compatible = "qcom,fastrpc-compute-cb"; reg = <6>; - iommus = <&apps_smmu 0x6 0x3460>; + iommus = <&apps_smmu 0x1006 0x0460>; }; compute-cb@7 { compatible = "qcom,fastrpc-compute-cb"; reg = <7>; - iommus = <&apps_smmu 0x7 0x3460>; + iommus = <&apps_smmu 0x1007 0x0460>; }; compute-cb@8 { compatible = "qcom,fastrpc-compute-cb"; reg = <8>; - iommus = <&apps_smmu 0x8 0x3460>; + iommus = <&apps_smmu 0x1008 0x0460>; }; /* note: secure cb9 in downstream */ diff --git a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts index 7ab3627cc347d6a15c6fd9b8efad50a0f62468ca..a102aa5efa3266d33ef11ffec9d343f869430b41 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts +++ b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts @@ -635,7 +635,7 @@ wcd938x: codec { compatible = "qcom,wcd9380-codec"; #sound-dai-cells = <1>; - reset-gpios = <&tlmm 32 0>; + reset-gpios = <&tlmm 32 GPIO_ACTIVE_HIGH>; vdd-buck-supply = <&vreg_s4a_1p8>; vdd-rxtx-supply = <&vreg_s4a_1p8>; vdd-io-supply = <&vreg_s4a_1p8>; diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index bc773e210023cba0ea1ce5f4aa426de1de60eec6..a5b62cadb1298b52d8e8b56bcfd48ebd43b3d072 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -1792,7 +1792,7 @@ }; pcie0: pci@1c00000 { - compatible = "qcom,pcie-sm8250", "snps,dw-pcie"; + compatible = "qcom,pcie-sm8250"; reg = <0 0x01c00000 0 0x3000>, <0 0x60000000 0 0xf1d>, <0 0x60000f20 0 0xa8>, @@ -1810,8 +1810,16 @@ ranges = <0x01000000 0x0 0x60200000 0 0x60200000 0x0 0x100000>, <0x02000000 0x0 0x60300000 0 0x60300000 0x0 0x3d00000>; - interrupts = ; - interrupt-names = "msi"; + interrupts = , + , + , + , + , + , + , + ; + interrupt-names = "msi0", "msi1", "msi2", "msi3", + "msi4", "msi5", "msi6", "msi7"; #interrupt-cells = <1>; interrupt-map-mask = <0 0 0 0x7>; interrupt-map = <0 0 0 1 &intc 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */ @@ -1893,7 +1901,7 @@ }; pcie1: pci@1c08000 { - compatible = "qcom,pcie-sm8250", "snps,dw-pcie"; + compatible = "qcom,pcie-sm8250"; reg = <0 0x01c08000 0 0x3000>, <0 0x40000000 0 0xf1d>, <0 0x40000f20 0 0xa8>, @@ -2001,7 +2009,7 @@ }; pcie2: pci@1c10000 { - compatible = "qcom,pcie-sm8250", "snps,dw-pcie"; + compatible = "qcom,pcie-sm8250"; reg = <0 0x01c10000 0 0x3000>, <0 0x64000000 0 0xf1d>, <0 0x64000f20 0 0xa8>, @@ -3571,6 +3579,25 @@ }; }; }; + + dsi_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-187500000 { + opp-hz = /bits/ 64 <187500000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-358000000 { + opp-hz = /bits/ 64 <358000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; }; dsi0_phy: dsi-phy@ae94400 { @@ -3663,25 +3690,6 @@ clock-names = "iface", "ref"; status = "disabled"; - - dsi_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-187500000 { - opp-hz = /bits/ 64 <187500000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-300000000 { - opp-hz = /bits/ 64 <300000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-358000000 { - opp-hz = /bits/ 64 <358000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - }; }; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi index cb9bbd234b7bc1ee928cd8ef1ff18c35b6406cec..b3c9952ac1735048733d4867586d9f25107a41e9 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi @@ -222,8 +222,8 @@ &tlmm { gpio-reserved-ranges = <44 4>; - ts_int_default: ts-int-default { - pin = "gpio23"; + ts_int_default: ts-int-default-state { + pins = "gpio23"; function = "gpio"; drive-strength = <2>; bias-disable; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index e72a04411888cb011eb18ac6374275e65fdcfbe3..a86d9ea93b9d4af71dd0509fe554d282e9fe4820 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -1762,150 +1762,150 @@ wakeup-parent = <&pdc>; qup_uart3_default_state: qup-uart3-default-state { - rx { + rx-pins { pins = "gpio18"; function = "qup3"; }; - tx { + tx-pins { pins = "gpio19"; function = "qup3"; }; }; - qup_uart6_default: qup-uart6-default { + qup_uart6_default: qup-uart6-default-state { pins = "gpio30", "gpio31"; function = "qup6"; drive-strength = <2>; bias-disable; }; - qup_uart18_default: qup-uart18-default { + qup_uart18_default: qup-uart18-default-state { pins = "gpio58", "gpio59"; function = "qup18"; drive-strength = <2>; bias-disable; }; - qup_i2c0_default: qup-i2c0-default { + qup_i2c0_default: qup-i2c0-default-state { pins = "gpio4", "gpio5"; function = "qup0"; drive-strength = <2>; bias-pull-up; }; - qup_i2c1_default: qup-i2c1-default { + qup_i2c1_default: qup-i2c1-default-state { pins = "gpio8", "gpio9"; function = "qup1"; drive-strength = <2>; bias-pull-up; }; - qup_i2c2_default: qup-i2c2-default { + qup_i2c2_default: qup-i2c2-default-state { pins = "gpio12", "gpio13"; function = "qup2"; drive-strength = <2>; bias-pull-up; }; - qup_i2c4_default: qup-i2c4-default { + qup_i2c4_default: qup-i2c4-default-state { pins = "gpio20", "gpio21"; function = "qup4"; drive-strength = <2>; bias-pull-up; }; - qup_i2c5_default: qup-i2c5-default { + qup_i2c5_default: qup-i2c5-default-state { pins = "gpio24", "gpio25"; function = "qup5"; drive-strength = <2>; bias-pull-up; }; - qup_i2c6_default: qup-i2c6-default { + qup_i2c6_default: qup-i2c6-default-state { pins = "gpio28", "gpio29"; function = "qup6"; drive-strength = <2>; bias-pull-up; }; - qup_i2c7_default: qup-i2c7-default { + qup_i2c7_default: qup-i2c7-default-state { pins = "gpio32", "gpio33"; function = "qup7"; drive-strength = <2>; bias-disable; }; - qup_i2c8_default: qup-i2c8-default { + qup_i2c8_default: qup-i2c8-default-state { pins = "gpio36", "gpio37"; function = "qup8"; drive-strength = <2>; bias-pull-up; }; - qup_i2c9_default: qup-i2c9-default { + qup_i2c9_default: qup-i2c9-default-state { pins = "gpio40", "gpio41"; function = "qup9"; drive-strength = <2>; bias-pull-up; }; - qup_i2c10_default: qup-i2c10-default { + qup_i2c10_default: qup-i2c10-default-state { pins = "gpio44", "gpio45"; function = "qup10"; drive-strength = <2>; bias-pull-up; }; - qup_i2c11_default: qup-i2c11-default { + qup_i2c11_default: qup-i2c11-default-state { pins = "gpio48", "gpio49"; function = "qup11"; drive-strength = <2>; bias-pull-up; }; - qup_i2c12_default: qup-i2c12-default { + qup_i2c12_default: qup-i2c12-default-state { pins = "gpio52", "gpio53"; function = "qup12"; drive-strength = <2>; bias-pull-up; }; - qup_i2c13_default: qup-i2c13-default { + qup_i2c13_default: qup-i2c13-default-state { pins = "gpio0", "gpio1"; function = "qup13"; drive-strength = <2>; bias-pull-up; }; - qup_i2c14_default: qup-i2c14-default { + qup_i2c14_default: qup-i2c14-default-state { pins = "gpio56", "gpio57"; function = "qup14"; drive-strength = <2>; bias-disable; }; - qup_i2c15_default: qup-i2c15-default { + qup_i2c15_default: qup-i2c15-default-state { pins = "gpio60", "gpio61"; function = "qup15"; drive-strength = <2>; bias-disable; }; - qup_i2c16_default: qup-i2c16-default { + qup_i2c16_default: qup-i2c16-default-state { pins = "gpio64", "gpio65"; function = "qup16"; drive-strength = <2>; bias-disable; }; - qup_i2c17_default: qup-i2c17-default { + qup_i2c17_default: qup-i2c17-default-state { pins = "gpio72", "gpio73"; function = "qup17"; drive-strength = <2>; bias-disable; }; - qup_i2c19_default: qup-i2c19-default { + qup_i2c19_default: qup-i2c19-default-state { pins = "gpio76", "gpio77"; function = "qup19"; drive-strength = <2>; @@ -2128,7 +2128,7 @@ ufs_mem_phy: phy@1d87000 { compatible = "qcom,sm8350-qmp-ufs-phy"; - reg = <0 0x01d87000 0 0xe10>; + reg = <0 0x01d87000 0 0x1c4>; #address-cells = <2>; #size-cells = <2>; ranges; diff --git a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts new file mode 100644 index 0000000000000000000000000000000000000000..d68765eb6d4f97527e3bc96addb2e9036d748076 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2022, Konrad Dybcio + */ + +/dts-v1/; + +#include +#include "sm8450.dtsi" + +/delete-node/ &adsp_mem; +/delete-node/ &rmtfs_mem; +/delete-node/ &video_mem; + +/ { + model = "Sony Xperia 1 IV"; + compatible = "sony,pdx223", "qcom,sm8450"; + chassis-type = "handset"; + + aliases { + serial0 = &uart7; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + reserved-memory { + adsp_mem: memory@85700000 { + reg = <0x0 0x85700000 0x0 0x2800000>; + no-map; + }; + + video_mem: memory@9fd00000 { + reg = <0x0 0x9fd00000 0x0 0x700000>; + no-map; + }; + + rmtfs_mem: memory@f3300000 { + compatible = "qcom,rmtfs-mem"; + reg = <0x0 0xf3300000 0x0 0x280000>; + no-map; + + qcom,client-id = <1>; + qcom,vmid = <15>; + }; + + ramoops@ffc00000 { + compatible = "ramoops"; + reg = <0 0xffc00000 0 0x200000>; + console-size = <0x40000>; + record-size = <0x1000>; + ecc-size = <16>; + no-map; + }; + }; + + /* Sadly, the voltages for these GPIO regulators are unknown. */ + imx650_vana_vreg: imx650-vana-regulator { + compatible = "regulator-fixed"; + regulator-name = "imx650_vana_vreg"; + gpio = <&tlmm 23 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; + + vph_pwr: vph-pwr-regulator { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + + regulator-always-on; + regulator-boot-on; + }; +}; + +&apps_rsc { + pm8350-rpmh-regulators { + compatible = "qcom,pm8350-rpmh-regulators"; + qcom,pmic-id = "b"; + + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + vdd-s4-supply = <&vph_pwr>; + vdd-s5-supply = <&vph_pwr>; + vdd-s6-supply = <&vph_pwr>; + vdd-s7-supply = <&vph_pwr>; + vdd-s8-supply = <&vph_pwr>; + vdd-s9-supply = <&vph_pwr>; + vdd-s10-supply = <&vph_pwr>; + vdd-s11-supply = <&vph_pwr>; + vdd-s12-supply = <&vph_pwr>; + + vdd-l1-l4-supply = <&pm8350_s11>; + vdd-l2-l7-supply = <&vreg_bob>; + vdd-l3-l5-supply = <&vreg_bob>; + vdd-l6-l9-l10-supply = <&pm8350_s12>; + + /* + * ARC regulators: + * s5 - gfx.lvl + * l8 - lcx.lvl + */ + + pm8350_s10: smps10 { + regulator-name = "pm8350_s10"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + pm8350_s11: smps11 { + regulator-name = "pm8350_s11"; + regulator-min-microvolt = <848000>; + regulator-max-microvolt = <1104000>; + }; + + pm8350_s12: smps12 { + regulator-name = "pm8350_s12"; + regulator-min-microvolt = <1224000>; + regulator-max-microvolt = <1400000>; + }; + + pm8350_l1: ldo1 { + regulator-name = "pm8350_l1"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <920000>; + regulator-initial-mode = ; + }; + + pm8350_l2: ldo2 { + regulator-name = "pm8350_l2"; + regulator-min-microvolt = <3072000>; + regulator-max-microvolt = <3072000>; + regulator-initial-mode = ; + }; + + pm8350_l3: ldo3 { + regulator-name = "pm8350_l3"; + regulator-min-microvolt = <904000>; + regulator-max-microvolt = <904000>; + regulator-initial-mode = ; + }; + + pm8350_l5: ldo5 { + regulator-name = "pm8350_l5"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + }; + + pm8350_l6: ldo6 { + regulator-name = "pm8350_l6"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + pm8350_l7: ldo7 { + regulator-name = "pm8350_l7"; + regulator-min-microvolt = <2504000>; + regulator-max-microvolt = <2504000>; + regulator-initial-mode = ; + }; + + pm8350_l9: ldo9 { + regulator-name = "pm8350_l9"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + }; + + pm8350c-rpmh-regulators { + compatible = "qcom,pm8350c-rpmh-regulators"; + qcom,pmic-id = "c"; + + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + vdd-s4-supply = <&vph_pwr>; + vdd-s5-supply = <&vph_pwr>; + vdd-s6-supply = <&vph_pwr>; + vdd-s7-supply = <&vph_pwr>; + vdd-s8-supply = <&vph_pwr>; + vdd-s9-supply = <&vph_pwr>; + vdd-s10-supply = <&vph_pwr>; + + vdd-l1-l12-supply = <&vreg_bob>; + vdd-l2-l8-supply = <&vreg_bob>; + vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>; + vdd-l6-l9-l11-supply = <&vreg_bob>; + vdd-l10-supply = <&pm8350_s12>; + + vdd-bob-supply = <&vph_pwr>; + + /* + * ARC regulators: + * s2 - mxc.lvl + * s4 - mss.lvl + * s6 - cx.lvl + */ + + pm8350c_s1: smps1 { + regulator-name = "pm8350c_s1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <2024000>; + }; + + pm8350c_s10: smps10 { + regulator-name = "pm8350c_s10"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1100000>; + }; + + vreg_bob: bob { + regulator-name = "vreg_bob"; + regulator-min-microvolt = <3400000>; + regulator-max-microvolt = <3960000>; + regulator-initial-mode = ; + }; + + pm8350c_l1: ldo1 { + regulator-name = "pm8350c_l1"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + pm8350c_l2: ldo2 { + regulator-name = "pm8350c_l2"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + pm8350c_l3: ldo3 { + regulator-name = "pm8350c_l3"; + regulator-min-microvolt = <3296000>; + regulator-max-microvolt = <3304000>; + regulator-initial-mode = ; + }; + + pm8350c_l4: ldo4 { + regulator-name = "pm8350c_l4"; + regulator-min-microvolt = <1704000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + }; + + pm8350c_l5: ldo5 { + regulator-name = "pm8350c_l5"; + regulator-min-microvolt = <1704000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + }; + + pm8350c_l6: ldo6 { + regulator-name = "pm8350c_l6"; + regulator-min-microvolt = <2960000>; + /* Originally max = 3008000 but SDHCI expects 2960000 */ + regulator-max-microvolt = <2960000>; + regulator-initial-mode = ; + }; + + pm8350c_l7: ldo7 { + regulator-name = "pm8350c_l7"; + regulator-min-microvolt = <3008000>; + regulator-max-microvolt = <3008000>; + regulator-initial-mode = ; + }; + + pm8350c_l8: ldo8 { + regulator-name = "pm8350c_l8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-initial-mode = ; + }; + + pm8350c_l9: ldo9 { + regulator-name = "pm8350c_l9"; + regulator-min-microvolt = <2960000>; + /* Originally max = 3008000 but SDHCI expects 2960000 */ + regulator-max-microvolt = <2960000>; + regulator-initial-mode = ; + }; + + pm8350c_l10: ldo10 { + regulator-name = "pm8350c_l10"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-initial-mode = ; + }; + + pm8350c_l12: ldo12 { + regulator-name = "pm8350c_l12"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1968000>; + regulator-initial-mode = ; + }; + + pm8350c_l13: ldo13 { + regulator-name = "pm8350c_l13"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-initial-mode = ; + }; + }; + + pm8450-rpmh-regulators { + compatible = "qcom,pm8450-rpmh-regulators"; + qcom,pmic-id = "h"; + + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + vdd-s4-supply = <&vph_pwr>; + vdd-s5-supply = <&vph_pwr>; + vdd-s6-supply = <&vph_pwr>; + + vdd-l2-supply = <&vreg_bob>; + vdd-l3-supply = <&vreg_bob>; + vdd-l4-supply = <&vreg_bob>; + + /* + * ARC regulators: + * S2 - ebi.lvl + * S4 - mmcx.lvl + * S6 - mx.lvl + * L1 - lmx.lvl + */ + + pm8450_s3: smps3 { + regulator-name = "pm8450_s3"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <600000>; + }; + + pm8450_l2: ldo2 { + regulator-name = "pm8450_l2"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + }; + + pm8450_l3: ldo3 { + regulator-name = "pm8450_l3"; + regulator-min-microvolt = <912000>; + regulator-max-microvolt = <912000>; + regulator-initial-mode = ; + }; + }; + + pmr735a-rpmh-regulators { + compatible = "qcom,pmr735a-rpmh-regulators"; + qcom,pmic-id = "e"; + + vdd-s1-supply = <&vph_pwr>; + vdd-s2-supply = <&vph_pwr>; + vdd-s3-supply = <&vph_pwr>; + + vdd-l1-l2-supply = <&pmr735a_s2>; + vdd-l3-supply = <&pmr735a_s1>; + vdd-l4-supply = <&pm8350c_s1>; + vdd-l5-l6-supply = <&pm8350c_s1>; + vdd-l7-bob-supply = <&vreg_bob>; + + pmr735a_s1: smps1 { + regulator-name = "pmr735a_s1"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1296000>; + }; + + pmr735a_s2: smps2 { + regulator-name = "pmr735a_s2"; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1040000>; + }; + + pmr735a_s3: smps3 { + regulator-name = "pmr735a_s3"; + regulator-min-microvolt = <435000>; + regulator-max-microvolt = <2352000>; + }; + + pmr735a_l1: ldo1 { + regulator-name = "pmr735a_l1"; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <800000>; + }; + + pmr735a_l2: ldo2 { + regulator-name = "pmr735a_l2"; + regulator-min-microvolt = <480000>; + regulator-max-microvolt = <912000>; + }; + + pmr735a_l3: ldo3 { + regulator-name = "pmr735a_l3"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + }; + + pmr735a_l4: ldo4 { + regulator-name = "pmr735a_l4"; + regulator-min-microvolt = <1776000>; + regulator-max-microvolt = <1776000>; + }; + + pmr735a_l5: ldo5 { + regulator-name = "pmr735a_l5"; + regulator-min-microvolt = <880000>; + regulator-max-microvolt = <880000>; + }; + + pmr735a_l6: ldo6 { + regulator-name = "pmr735a_l6"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + }; + + pmr735a_l7: ldo7 { + regulator-name = "pmr735a_l7"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + }; + }; +}; + +&gpi_dma0 { + status = "okay"; +}; + +&gpi_dma1 { + status = "okay"; +}; + +&gpi_dma2 { + status = "okay"; +}; + +/* I2C4 is used, it hosts a Samsung touchscreen, but GPI DMA is broken.. */ + +&i2c5 { + clock-frequency = <400000>; + status = "okay"; + + /* Dialog SLG51000 CMIC @ 75 */ +}; + +&i2c9 { + clock-frequency = <400000>; + status = "okay"; + + /* NXP SN1X0 NFC @ 28 */ +}; + +&i2c13 { + clock-frequency = <400000>; + status = "okay"; + + /* Richwave RTC6226 FM Radio Receiver @ 64 */ +}; + +&i2c14 { + clock-frequency = <1000000>; + status = "okay"; + + cs35l41_l: speaker-amp@40 { + compatible = "cirrus,cs35l41"; + reg = <0x40>; + interrupt-parent = <&tlmm>; + interrupts = <182 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&tlmm 183 GPIO_ACTIVE_HIGH>; + cirrus,boost-peak-milliamp = <4000>; + cirrus,boost-ind-nanohenry = <1000>; + cirrus,boost-cap-microfarad = <15>; + cirrus,gpio2-src-select = <2>; + cirrus,gpio2-output-enable; + cirrus,asp-sdout-hiz = <3>; + #sound-dai-cells = <1>; + }; + + cs35l41_r: speaker-amp@41 { + compatible = "cirrus,cs35l41"; + reg = <0x41>; + interrupt-parent = <&tlmm>; + interrupts = <182 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&tlmm 183 GPIO_ACTIVE_HIGH>; + cirrus,boost-peak-milliamp = <4000>; + cirrus,boost-ind-nanohenry = <1000>; + cirrus,boost-cap-microfarad = <15>; + cirrus,gpio2-src-select = <2>; + cirrus,gpio2-output-enable; + cirrus,asp-sdout-hiz = <3>; + #sound-dai-cells = <1>; + }; +}; + +&i2c15 { + clock-frequency = <400000>; + status = "okay"; + + /* AMS TCS3490 RGB+IR color sensor @ 72 */ +}; + +&i2c19 { + clock-frequency = <1000000>; + status = "okay"; + + /* Cirrus Logic CS40L25A boosted haptics driver @ 40 */ +}; + +&pcie0 { + max-link-speed = <2>; + status = "okay"; +}; + +&pcie0_phy { + vdda-phy-supply = <&pm8350_l5>; + vdda-pll-supply = <&pm8350_l6>; + status = "okay"; +}; + +&remoteproc_adsp { + firmware-name = "qcom/adsp.mbn"; + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/cdsp.mbn"; + status = "okay"; +}; + +&remoteproc_slpi { + firmware-name = "qcom/slpi.mbn"; + status = "okay"; +}; + +&qupv3_id_0 { + status = "okay"; +}; + +&qupv3_id_1 { + status = "okay"; +}; + +&qupv3_id_2 { + status = "okay"; +}; + +&sdhc_2 { + cd-gpios = <&tlmm 92 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&sdc2_default_state &sdc2_card_det_n>; + pinctrl-1 = <&sdc2_sleep_state &sdc2_card_det_n>; + vmmc-supply = <&pm8350c_l9>; + vqmmc-supply = <&pm8350c_l6>; + /* Forbid SDR104/SDR50 - broken hw! */ + sdhci-caps-mask = <0x3 0x0>; + no-sdio; + no-mmc; + status = "okay"; +}; + +&spi10 { + status = "okay"; + + /* NXP SN1X0 NFC Secure Element @ 0 */ +}; + +&tlmm { + gpio-reserved-ranges = <28 4>; + + sdc2_default_state: sdc2-default-state { + clk-pins { + pins = "sdc2_clk"; + drive-strength = <16>; + bias-disable; + }; + + cmd-pins { + pins = "sdc2_cmd"; + drive-strength = <16>; + bias-pull-up; + }; + + data-pins { + pins = "sdc2_data"; + drive-strength = <16>; + bias-pull-up; + }; + }; + + ts_int_default: ts-int-default-state { + pins = "gpio23"; + function = "gpio"; + drive-strength = <2>; + bias-disable; + input-enable; + }; + + sdc2_card_det_n: sd-card-det-n-state { + pins = "gpio92"; + function = "gpio"; + drive-strength = <2>; + bias-pull-up; + }; +}; + +&uart7 { + status = "okay"; +}; + +&usb_1 { + status = "okay"; +}; + +&usb_1_dwc3 { + dr_mode = "peripheral"; +}; + +&usb_1_hsphy { + vdda-pll-supply = <&pm8350_l5>; + vdda18-supply = <&pm8350c_l1>; + vdda33-supply = <&pm8350_l2>; + status = "okay"; +}; + +&usb_1_qmpphy { + vdda-phy-supply = <&pm8350_l6>; + vdda-pll-supply = <&pm8350_l1>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 4978c5ba5dd0858025f262880d92587e5ad97cb7..d32f08df743d8162f7d8cb2ecf087dfdb1b3ddf7 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -438,11 +438,6 @@ no-map; }; - camera_mem: memory@85200000 { - reg = <0x0 0x85200000 0x0 0x500000>; - no-map; - }; - video_mem: memory@85700000 { reg = <0x0 0x85700000 0x0 0x700000>; no-map; @@ -505,6 +500,11 @@ no-map; }; + camera_mem: memory@9f500000 { + reg = <0x0 0x9f500000 0x0 0x800000>; + no-map; + }; + rmtfs_mem: memory@9fd00000 { compatible = "qcom,rmtfs-mem"; reg = <0x0 0x9fd00000 0x0 0x280000>; @@ -514,6 +514,11 @@ qcom,vmid = <15>; }; + xbl_sc_mem2: memory@a6e00000 { + reg = <0x0 0xa6e00000 0x0 0x40000>; + no-map; + }; + global_sync_mem: memory@a6f00000 { reg = <0x0 0xa6f00000 0x0 0x100000>; no-map; @@ -2379,22 +2384,42 @@ gpio-ranges = <&tlmm 0 0 211>; wakeup-parent = <&pdc>; + sdc2_sleep_state: sdc2-sleep-state { + clk-pins { + pins = "sdc2_clk"; + drive-strength = <2>; + bias-disable; + }; + + cmd-pins { + pins = "sdc2_cmd"; + drive-strength = <2>; + bias-pull-up; + }; + + data-pins { + pins = "sdc2_data"; + drive-strength = <2>; + bias-pull-up; + }; + }; + pcie0_default_state: pcie0-default-state { - perst { + perst-pins { pins = "gpio94"; function = "gpio"; drive-strength = <2>; bias-pull-down; }; - clkreq { + clkreq-pins { pins = "gpio95"; function = "pcie0_clkreqn"; drive-strength = <2>; bias-pull-up; }; - wake { + wake-pins { pins = "gpio96"; function = "gpio"; drive-strength = <2>; @@ -2403,21 +2428,21 @@ }; pcie1_default_state: pcie1-default-state { - perst { + perst-pins { pins = "gpio97"; function = "gpio"; drive-strength = <2>; bias-pull-down; }; - clkreq { + clkreq-pins { pins = "gpio98"; function = "pcie1_clkreqn"; drive-strength = <2>; bias-pull-up; }; - wake { + wake-pins { pins = "gpio99"; function = "gpio"; drive-strength = <2>; @@ -2425,350 +2450,350 @@ }; }; - qup_i2c0_data_clk: qup-i2c0-data-clk { + qup_i2c0_data_clk: qup-i2c0-data-clk-state { pins = "gpio0", "gpio1"; function = "qup0"; }; - qup_i2c1_data_clk: qup-i2c1-data-clk { + qup_i2c1_data_clk: qup-i2c1-data-clk-state { pins = "gpio4", "gpio5"; function = "qup1"; }; - qup_i2c2_data_clk: qup-i2c2-data-clk { + qup_i2c2_data_clk: qup-i2c2-data-clk-state { pins = "gpio8", "gpio9"; function = "qup2"; }; - qup_i2c3_data_clk: qup-i2c3-data-clk { + qup_i2c3_data_clk: qup-i2c3-data-clk-state { pins = "gpio12", "gpio13"; function = "qup3"; }; - qup_i2c4_data_clk: qup-i2c4-data-clk { + qup_i2c4_data_clk: qup-i2c4-data-clk-state { pins = "gpio16", "gpio17"; function = "qup4"; }; - qup_i2c5_data_clk: qup-i2c5-data-clk { + qup_i2c5_data_clk: qup-i2c5-data-clk-state { pins = "gpio206", "gpio207"; function = "qup5"; }; - qup_i2c6_data_clk: qup-i2c6-data-clk { + qup_i2c6_data_clk: qup-i2c6-data-clk-state { pins = "gpio20", "gpio21"; function = "qup6"; }; - qup_i2c8_data_clk: qup-i2c8-data-clk { + qup_i2c8_data_clk: qup-i2c8-data-clk-state { pins = "gpio28", "gpio29"; function = "qup8"; }; - qup_i2c9_data_clk: qup-i2c9-data-clk { + qup_i2c9_data_clk: qup-i2c9-data-clk-state { pins = "gpio32", "gpio33"; function = "qup9"; }; - qup_i2c10_data_clk: qup-i2c10-data-clk { + qup_i2c10_data_clk: qup-i2c10-data-clk-state { pins = "gpio36", "gpio37"; function = "qup10"; }; - qup_i2c11_data_clk: qup-i2c11-data-clk { + qup_i2c11_data_clk: qup-i2c11-data-clk-state { pins = "gpio40", "gpio41"; function = "qup11"; }; - qup_i2c12_data_clk: qup-i2c12-data-clk { + qup_i2c12_data_clk: qup-i2c12-data-clk-state { pins = "gpio44", "gpio45"; function = "qup12"; }; - qup_i2c13_data_clk: qup-i2c13-data-clk { + qup_i2c13_data_clk: qup-i2c13-data-clk-state { pins = "gpio48", "gpio49"; function = "qup13"; drive-strength = <2>; bias-pull-up; }; - qup_i2c14_data_clk: qup-i2c14-data-clk { + qup_i2c14_data_clk: qup-i2c14-data-clk-state { pins = "gpio52", "gpio53"; function = "qup14"; drive-strength = <2>; bias-pull-up; }; - qup_i2c15_data_clk: qup-i2c15-data-clk { + qup_i2c15_data_clk: qup-i2c15-data-clk-state { pins = "gpio56", "gpio57"; function = "qup15"; }; - qup_i2c16_data_clk: qup-i2c16-data-clk { + qup_i2c16_data_clk: qup-i2c16-data-clk-state { pins = "gpio60", "gpio61"; function = "qup16"; }; - qup_i2c17_data_clk: qup-i2c17-data-clk { + qup_i2c17_data_clk: qup-i2c17-data-clk-state { pins = "gpio64", "gpio65"; function = "qup17"; }; - qup_i2c18_data_clk: qup-i2c18-data-clk { + qup_i2c18_data_clk: qup-i2c18-data-clk-state { pins = "gpio68", "gpio69"; function = "qup18"; }; - qup_i2c19_data_clk: qup-i2c19-data-clk { + qup_i2c19_data_clk: qup-i2c19-data-clk-state { pins = "gpio72", "gpio73"; function = "qup19"; }; - qup_i2c20_data_clk: qup-i2c20-data-clk { + qup_i2c20_data_clk: qup-i2c20-data-clk-state { pins = "gpio76", "gpio77"; function = "qup20"; }; - qup_i2c21_data_clk: qup-i2c21-data-clk { + qup_i2c21_data_clk: qup-i2c21-data-clk-state { pins = "gpio80", "gpio81"; function = "qup21"; }; - qup_spi0_cs: qup-spi0-cs { + qup_spi0_cs: qup-spi0-cs-state { pins = "gpio3"; function = "qup0"; }; - qup_spi0_data_clk: qup-spi0-data-clk { + qup_spi0_data_clk: qup-spi0-data-clk-state { pins = "gpio0", "gpio1", "gpio2"; function = "qup0"; }; - qup_spi1_cs: qup-spi1-cs { + qup_spi1_cs: qup-spi1-cs-state { pins = "gpio7"; function = "qup1"; }; - qup_spi1_data_clk: qup-spi1-data-clk { + qup_spi1_data_clk: qup-spi1-data-clk-state { pins = "gpio4", "gpio5", "gpio6"; function = "qup1"; }; - qup_spi2_cs: qup-spi2-cs { + qup_spi2_cs: qup-spi2-cs-state { pins = "gpio11"; function = "qup2"; }; - qup_spi2_data_clk: qup-spi2-data-clk { + qup_spi2_data_clk: qup-spi2-data-clk-state { pins = "gpio8", "gpio9", "gpio10"; function = "qup2"; }; - qup_spi3_cs: qup-spi3-cs { + qup_spi3_cs: qup-spi3-cs-state { pins = "gpio15"; function = "qup3"; }; - qup_spi3_data_clk: qup-spi3-data-clk { + qup_spi3_data_clk: qup-spi3-data-clk-state { pins = "gpio12", "gpio13", "gpio14"; function = "qup3"; }; - qup_spi4_cs: qup-spi4-cs { + qup_spi4_cs: qup-spi4-cs-state { pins = "gpio19"; function = "qup4"; drive-strength = <6>; bias-disable; }; - qup_spi4_data_clk: qup-spi4-data-clk { + qup_spi4_data_clk: qup-spi4-data-clk-state { pins = "gpio16", "gpio17", "gpio18"; function = "qup4"; }; - qup_spi5_cs: qup-spi5-cs { + qup_spi5_cs: qup-spi5-cs-state { pins = "gpio85"; function = "qup5"; }; - qup_spi5_data_clk: qup-spi5-data-clk { + qup_spi5_data_clk: qup-spi5-data-clk-state { pins = "gpio206", "gpio207", "gpio84"; function = "qup5"; }; - qup_spi6_cs: qup-spi6-cs { + qup_spi6_cs: qup-spi6-cs-state { pins = "gpio23"; function = "qup6"; }; - qup_spi6_data_clk: qup-spi6-data-clk { + qup_spi6_data_clk: qup-spi6-data-clk-state { pins = "gpio20", "gpio21", "gpio22"; function = "qup6"; }; - qup_spi8_cs: qup-spi8-cs { + qup_spi8_cs: qup-spi8-cs-state { pins = "gpio31"; function = "qup8"; }; - qup_spi8_data_clk: qup-spi8-data-clk { + qup_spi8_data_clk: qup-spi8-data-clk-state { pins = "gpio28", "gpio29", "gpio30"; function = "qup8"; }; - qup_spi9_cs: qup-spi9-cs { + qup_spi9_cs: qup-spi9-cs-state { pins = "gpio35"; function = "qup9"; }; - qup_spi9_data_clk: qup-spi9-data-clk { + qup_spi9_data_clk: qup-spi9-data-clk-state { pins = "gpio32", "gpio33", "gpio34"; function = "qup9"; }; - qup_spi10_cs: qup-spi10-cs { + qup_spi10_cs: qup-spi10-cs-state { pins = "gpio39"; function = "qup10"; }; - qup_spi10_data_clk: qup-spi10-data-clk { + qup_spi10_data_clk: qup-spi10-data-clk-state { pins = "gpio36", "gpio37", "gpio38"; function = "qup10"; }; - qup_spi11_cs: qup-spi11-cs { + qup_spi11_cs: qup-spi11-cs-state { pins = "gpio43"; function = "qup11"; }; - qup_spi11_data_clk: qup-spi11-data-clk { + qup_spi11_data_clk: qup-spi11-data-clk-state { pins = "gpio40", "gpio41", "gpio42"; function = "qup11"; }; - qup_spi12_cs: qup-spi12-cs { + qup_spi12_cs: qup-spi12-cs-state { pins = "gpio47"; function = "qup12"; }; - qup_spi12_data_clk: qup-spi12-data-clk { + qup_spi12_data_clk: qup-spi12-data-clk-state { pins = "gpio44", "gpio45", "gpio46"; function = "qup12"; }; - qup_spi13_cs: qup-spi13-cs { + qup_spi13_cs: qup-spi13-cs-state { pins = "gpio51"; function = "qup13"; }; - qup_spi13_data_clk: qup-spi13-data-clk { + qup_spi13_data_clk: qup-spi13-data-clk-state { pins = "gpio48", "gpio49", "gpio50"; function = "qup13"; }; - qup_spi14_cs: qup-spi14-cs { + qup_spi14_cs: qup-spi14-cs-state { pins = "gpio55"; function = "qup14"; }; - qup_spi14_data_clk: qup-spi14-data-clk { + qup_spi14_data_clk: qup-spi14-data-clk-state { pins = "gpio52", "gpio53", "gpio54"; function = "qup14"; }; - qup_spi15_cs: qup-spi15-cs { + qup_spi15_cs: qup-spi15-cs-state { pins = "gpio59"; function = "qup15"; }; - qup_spi15_data_clk: qup-spi15-data-clk { + qup_spi15_data_clk: qup-spi15-data-clk-state { pins = "gpio56", "gpio57", "gpio58"; function = "qup15"; }; - qup_spi16_cs: qup-spi16-cs { + qup_spi16_cs: qup-spi16-cs-state { pins = "gpio63"; function = "qup16"; }; - qup_spi16_data_clk: qup-spi16-data-clk { + qup_spi16_data_clk: qup-spi16-data-clk-state { pins = "gpio60", "gpio61", "gpio62"; function = "qup16"; }; - qup_spi17_cs: qup-spi17-cs { + qup_spi17_cs: qup-spi17-cs-state { pins = "gpio67"; function = "qup17"; }; - qup_spi17_data_clk: qup-spi17-data-clk { + qup_spi17_data_clk: qup-spi17-data-clk-state { pins = "gpio64", "gpio65", "gpio66"; function = "qup17"; }; - qup_spi18_cs: qup-spi18-cs { + qup_spi18_cs: qup-spi18-cs-state { pins = "gpio71"; function = "qup18"; drive-strength = <6>; bias-disable; }; - qup_spi18_data_clk: qup-spi18-data-clk { + qup_spi18_data_clk: qup-spi18-data-clk-state { pins = "gpio68", "gpio69", "gpio70"; function = "qup18"; drive-strength = <6>; bias-disable; }; - qup_spi19_cs: qup-spi19-cs { + qup_spi19_cs: qup-spi19-cs-state { pins = "gpio75"; function = "qup19"; drive-strength = <6>; bias-disable; }; - qup_spi19_data_clk: qup-spi19-data-clk { + qup_spi19_data_clk: qup-spi19-data-clk-state { pins = "gpio72", "gpio73", "gpio74"; function = "qup19"; drive-strength = <6>; bias-disable; }; - qup_spi20_cs: qup-spi20-cs { + qup_spi20_cs: qup-spi20-cs-state { pins = "gpio79"; function = "qup20"; }; - qup_spi20_data_clk: qup-spi20-data-clk { + qup_spi20_data_clk: qup-spi20-data-clk-state { pins = "gpio76", "gpio77", "gpio78"; function = "qup20"; }; - qup_spi21_cs: qup-spi21-cs { + qup_spi21_cs: qup-spi21-cs-state { pins = "gpio83"; function = "qup21"; }; - qup_spi21_data_clk: qup-spi21-data-clk { + qup_spi21_data_clk: qup-spi21-data-clk-state { pins = "gpio80", "gpio81", "gpio82"; function = "qup21"; }; - qup_uart7_rx: qup-uart7-rx { + qup_uart7_rx: qup-uart7-rx-state { pins = "gpio26"; function = "qup7"; drive-strength = <2>; bias-disable; }; - qup_uart7_tx: qup-uart7-tx { + qup_uart7_tx: qup-uart7-tx-state { pins = "gpio27"; function = "qup7"; drive-strength = <2>; bias-disable; }; - qup_uart20_default: qup-uart20-default { + qup_uart20_default: qup-uart20-default-state { pins = "gpio76", "gpio77", "gpio78", "gpio79"; function = "qup20"; }; @@ -3069,7 +3094,9 @@ ufs_mem_hc: ufshc@1d84000 { compatible = "qcom,sm8450-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; - reg = <0 0x01d84000 0 0x3000>; + reg = <0 0x01d84000 0 0x3000>, + <0 0x01d88000 0 0x8000>; + reg-names = "std", "ice"; interrupts = ; phys = <&ufs_mem_phy_lanes>; phy-names = "ufsphy"; @@ -3093,7 +3120,8 @@ "ref_clk", "tx_lane0_sync_clk", "rx_lane0_sync_clk", - "rx_lane1_sync_clk"; + "rx_lane1_sync_clk", + "ice_core_clk"; clocks = <&gcc GCC_UFS_PHY_AXI_CLK>, <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, @@ -3102,7 +3130,8 @@ <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>, <&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>, - <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>; + <&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>, + <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; freq-table-hz = <75000000 300000000>, <0 0>, @@ -3111,13 +3140,14 @@ <75000000 300000000>, <0 0>, <0 0>, - <0 0>; + <0 0>, + <75000000 300000000>; status = "disabled"; }; ufs_mem_phy: phy@1d87000 { compatible = "qcom,sm8450-qmp-ufs-phy"; - reg = <0 0x01d87000 0 0xe10>; + reg = <0 0x01d87000 0 0x1c4>; #address-cells = <2>; #size-cells = <2>; ranges; @@ -3140,6 +3170,45 @@ }; }; + sdhc_2: sdhci@8804000 { + compatible = "qcom,sm8450-sdhci", "qcom,sdhci-msm-v5"; + reg = <0 0x08804000 0 0x1000>; + + interrupts = , + ; + interrupt-names = "hc_irq", "pwr_irq"; + + clocks = <&gcc GCC_SDCC2_AHB_CLK>, + <&gcc GCC_SDCC2_APPS_CLK>, + <&rpmhcc RPMH_CXO_CLK>; + clock-names = "iface", "core", "xo"; + resets = <&gcc GCC_SDCC2_BCR>; + interconnects = <&aggre2_noc MASTER_SDCC_2 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_SDCC_2 0>; + interconnect-names = "sdhc-ddr","cpu-sdhc"; + iommus = <&apps_smmu 0x4a0 0x0>; + power-domains = <&rpmhpd SM8450_CX>; + operating-points-v2 = <&sdhc2_opp_table>; + bus-width = <4>; + dma-coherent; + + status = "disabled"; + + sdhc2_opp_table: opp-table { + compatible = "operating-points-v2"; + + opp-100000000 { + opp-hz = /bits/ 64 <100000000>; + required-opps = <&rpmhpd_opp_low_svs>; + }; + + opp-202000000 { + opp-hz = /bits/ 64 <202000000>; + required-opps = <&rpmhpd_opp_svs_l1>; + }; + }; + }; + usb_1: usb@a6f8800 { compatible = "qcom,sm8450-dwc3", "qcom,dwc3"; reg = <0 0x0a6f8800 0 0x400>; diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile index 7a647860ef355582001b054c890e292c23fc1919..0699b51c124717ae1e6b2ee0ce501ace17c2a0c0 100644 --- a/arch/arm64/boot/dts/renesas/Makefile +++ b/arch/arm64/boot/dts/renesas/Makefile @@ -56,6 +56,7 @@ dtb-$(CONFIG_ARCH_R8A77970) += r8a77970-v3msk.dtb dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-condor.dtb dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-v3hsk.dtb +dtb-$(CONFIG_ARCH_R8A77980) += r8a77980a-condor-i.dtb dtb-$(CONFIG_ARCH_R8A77990) += r8a77990-ebisu.dtb diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi index 63e7a39e100e367c87d0102214b35678a17b8557..8166e3c1ff4e58f6c6c3e4c83f3fd6ca7bbc2ea8 100644 --- a/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi +++ b/arch/arm64/boot/dts/renesas/beacon-renesom-baseboard.dtsi @@ -5,7 +5,7 @@ #include #include -#include +#include / { backlight_lvds: backlight-lvds { diff --git a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi index f5c1d74b738b9344dc620ed10233c4798cbfe288..d3fc8ffd5b4c1a2e53cf0f07452eae09ff519a76 100644 --- a/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi +++ b/arch/arm64/boot/dts/renesas/beacon-renesom-som.dtsi @@ -4,7 +4,7 @@ */ #include -#include +#include / { memory@48000000 { diff --git a/arch/arm64/boot/dts/renesas/condor-common.dtsi b/arch/arm64/boot/dts/renesas/condor-common.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..dfbe35bf46e00c72fef4baddab9ccaeb6bcb4dac --- /dev/null +++ b/arch/arm64/boot/dts/renesas/condor-common.dtsi @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for the Condor board with R-Car V3H + * + * Copyright (C) 2018 Renesas Electronics Corp. + * Copyright (C) 2018 Cogent Embedded, Inc. + */ +#include + +/ { + aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + serial0 = &scif0; + ethernet0 = &gether; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + d1_8v: regulator-2 { + compatible = "regulator-fixed"; + regulator-name = "D1.8V"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + d3_3v: regulator-0 { + compatible = "regulator-fixed"; + regulator-name = "D3.3V"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; + + hdmi-out { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con: endpoint { + remote-endpoint = <&adv7511_out>; + }; + }; + }; + + lvds-decoder { + compatible = "thine,thc63lvd1024"; + vcc-supply = <&d3_3v>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + thc63lvd1024_in: endpoint { + remote-endpoint = <&lvds0_out>; + }; + }; + + port@2 { + reg = <2>; + thc63lvd1024_out: endpoint { + remote-endpoint = <&adv7511_in>; + }; + }; + }; + }; + + memory@48000000 { + device_type = "memory"; + /* first 128MB is reserved for secure area. */ + reg = <0 0x48000000 0 0x78000000>; + }; + + vddq_vin01: regulator-1 { + compatible = "regulator-fixed"; + regulator-name = "VDDQ_VIN01"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + x1_clk: x1-clock { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <148500000>; + }; +}; + +&canfd { + pinctrl-0 = <&canfd0_pins>; + pinctrl-names = "default"; + status = "okay"; + + channel0 { + status = "okay"; + }; +}; + +&csi40 { + status = "okay"; + + ports { + port@0 { + csi40_in: endpoint { + clock-lanes = <0>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&max9286_out0>; + }; + }; + }; +}; + +&csi41 { + status = "okay"; + + ports { + port@0 { + csi41_in: endpoint { + clock-lanes = <0>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&max9286_out1>; + }; + }; + }; +}; + +&du { + clocks = <&cpg CPG_MOD 724>, + <&x1_clk>; + clock-names = "du.0", "dclkin.0"; + status = "okay"; +}; + +&extal_clk { + clock-frequency = <16666666>; +}; + +&extalr_clk { + clock-frequency = <32768>; +}; + +&gether { + pinctrl-0 = <&gether_pins>; + pinctrl-names = "default"; + + phy-mode = "rgmii-id"; + phy-handle = <&phy0>; + renesas,no-ether-link; + status = "okay"; + + phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; + rxc-skew-ps = <1500>; + reg = <0>; + interrupt-parent = <&gpio4>; + interrupts = <23 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; + }; +}; + +&i2c0 { + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + + status = "okay"; + clock-frequency = <400000>; + + io_expander0: gpio@20 { + compatible = "onnn,pca9654"; + reg = <0x20>; + gpio-controller; + #gpio-cells = <2>; + }; + + io_expander1: gpio@21 { + compatible = "onnn,pca9654"; + reg = <0x21>; + gpio-controller; + #gpio-cells = <2>; + }; + + hdmi@39 { + compatible = "adi,adv7511w"; + reg = <0x39>; + interrupt-parent = <&gpio1>; + interrupts = <20 IRQ_TYPE_LEVEL_LOW>; + avdd-supply = <&d1_8v>; + dvdd-supply = <&d1_8v>; + pvdd-supply = <&d1_8v>; + bgvdd-supply = <&d1_8v>; + dvdd-3v-supply = <&d3_3v>; + + adi,input-depth = <8>; + adi,input-colorspace = "rgb"; + adi,input-clock = "1x"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + adv7511_in: endpoint { + remote-endpoint = <&thc63lvd1024_out>; + }; + }; + + port@1 { + reg = <1>; + adv7511_out: endpoint { + remote-endpoint = <&hdmi_con>; + }; + }; + }; + }; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + + status = "okay"; + clock-frequency = <400000>; + + gmsl0: gmsl-deserializer@48 { + compatible = "maxim,max9286"; + reg = <0x48>; + + maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>; + enable-gpios = <&io_expander0 0 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + }; + + port@2 { + reg = <2>; + }; + + port@3 { + reg = <3>; + }; + + port@4 { + reg = <4>; + max9286_out0: endpoint { + clock-lanes = <0>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&csi40_in>; + }; + }; + }; + + i2c-mux { + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + status = "disabled"; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + status = "disabled"; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + status = "disabled"; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + status = "disabled"; + }; + }; + }; + + gmsl1: gmsl-deserializer@4a { + compatible = "maxim,max9286"; + reg = <0x4a>; + + maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>; + enable-gpios = <&io_expander1 0 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + }; + + port@2 { + reg = <2>; + }; + + port@3 { + reg = <3>; + }; + + port@4 { + reg = <4>; + max9286_out1: endpoint { + clock-lanes = <0>; + data-lanes = <1 2 3 4>; + remote-endpoint = <&csi41_in>; + }; + }; + }; + + i2c-mux { + #address-cells = <1>; + #size-cells = <0>; + + i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + + status = "disabled"; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + + status = "disabled"; + }; + + i2c@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <2>; + + status = "disabled"; + }; + + i2c@3 { + #address-cells = <1>; + #size-cells = <0>; + reg = <3>; + + status = "disabled"; + }; + }; + }; +}; + +&lvds0 { + status = "okay"; + + ports { + port@1 { + lvds0_out: endpoint { + remote-endpoint = <&thc63lvd1024_in>; + }; + }; + }; +}; + +&mmc0 { + pinctrl-0 = <&mmc_pins>; + pinctrl-1 = <&mmc_pins>; + pinctrl-names = "default", "state_uhs"; + + vmmc-supply = <&d3_3v>; + vqmmc-supply = <&vddq_vin01>; + mmc-hs200-1_8v; + bus-width = <8>; + no-sd; + no-sdio; + non-removable; + status = "okay"; +}; + +&pciec { + status = "okay"; +}; + +&pcie_bus_clk { + clock-frequency = <100000000>; +}; + +&pcie_phy { + status = "okay"; +}; + +&pfc { + canfd0_pins: canfd0 { + groups = "canfd0_data_a"; + function = "canfd0"; + }; + + gether_pins: gether { + groups = "gether_mdio_a", "gether_rgmii", + "gether_txcrefclk", "gether_txcrefclk_mega"; + function = "gether"; + }; + + i2c0_pins: i2c0 { + groups = "i2c0"; + function = "i2c0"; + }; + + i2c1_pins: i2c1 { + groups = "i2c1"; + function = "i2c1"; + }; + + mmc_pins: mmc { + groups = "mmc_data8", "mmc_ctrl", "mmc_ds"; + function = "mmc"; + power-source = <1800>; + }; + + qspi0_pins: qspi0 { + groups = "qspi0_ctrl", "qspi0_data4"; + function = "qspi0"; + }; + + scif0_pins: scif0 { + groups = "scif0_data"; + function = "scif0"; + }; + + scif_clk_pins: scif_clk { + groups = "scif_clk_b"; + function = "scif_clk"; + }; +}; + +&rpc { + pinctrl-0 = <&qspi0_pins>; + pinctrl-names = "default"; + + status = "okay"; + + flash@0 { + compatible = "spansion,s25fs512s", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + spi-rx-bus-width = <4>; + + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + + bootparam@0 { + reg = <0x00000000 0x040000>; + read-only; + }; + cr7@40000 { + reg = <0x00040000 0x080000>; + read-only; + }; + cert_header_sa3@c0000 { + reg = <0x000c0000 0x080000>; + read-only; + }; + bl2@140000 { + reg = <0x00140000 0x040000>; + read-only; + }; + cert_header_sa6@180000 { + reg = <0x00180000 0x040000>; + read-only; + }; + bl31@1c0000 { + reg = <0x001c0000 0x460000>; + read-only; + }; + uboot@640000 { + reg = <0x00640000 0x0c0000>; + read-only; + }; + uboot-env@700000 { + reg = <0x00700000 0x040000>; + read-only; + }; + dtb@740000 { + reg = <0x00740000 0x080000>; + }; + kernel@7c0000 { + reg = <0x007c0000 0x1400000>; + }; + user@1bc0000 { + reg = <0x01bc0000 0x2440000>; + }; + }; + }; +}; + +&rwdt { + timeout-sec = <60>; + status = "okay"; +}; + +&scif0 { + pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>; + pinctrl-names = "default"; + + status = "okay"; +}; + +&scif_clk { + clock-frequency = <14745600>; +}; diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi index 8fc03491a11c431ef7ae6ace91ad080f9f006a4d..bbc29452d1be92f0d76ab4627f5741b28c09e478 100644 --- a/arch/arm64/boot/dts/renesas/ebisu.dtsi +++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi @@ -13,6 +13,14 @@ compatible = "renesas,ebisu"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &i2c7; serial0 = &scif2; ethernet0 = &avb; mmc0 = &sdhi3; diff --git a/arch/arm64/boot/dts/renesas/hihope-common.dtsi b/arch/arm64/boot/dts/renesas/hihope-common.dtsi index b062f41ee270124b9927a4e54392ee4395d58b47..83104af2813eb4a08323eda9eb44fdc5fc981448 100644 --- a/arch/arm64/boot/dts/renesas/hihope-common.dtsi +++ b/arch/arm64/boot/dts/renesas/hihope-common.dtsi @@ -10,6 +10,14 @@ / { aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &iic_pmic; serial0 = &scif2; serial1 = &hscif0; mmc0 = &sdhi3; diff --git a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts index 3cf2e076940f3b5a8740ff9d97b5a362c1f52b0d..9ae67263c0df3fed712aa6f79247be1be2a4125c 100644 --- a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts +++ b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts @@ -14,6 +14,14 @@ compatible = "beacon,beacon-rzg2m", "renesas,r8a774a1"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &iic_pmic; serial0 = &scif2; serial1 = &hscif0; serial2 = &hscif1; diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi index e7d17776624dee7f9c0e3fc615430174b43b9999..7e643243c3be6e89ef7f7afa4182429fc36b194b 100644 --- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi @@ -17,17 +17,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - i2c7 = &iic_pmic; - }; - /* * The external audio clocks are configured as 0 Hz fixed frequency * clocks by default. @@ -2334,7 +2323,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi index f62d95760e82dade6426ffaab332b9a64d797a1a..d541b48c7e384dab41ec065af054ddda5021bba3 100644 --- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi @@ -2191,7 +2191,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi index c563d26a7a71cc3347951fd28a1a04b1ead9c9d3..151e32ac03683bf8c098f5749ff55c2c9c70e7e9 100644 --- a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi @@ -1671,7 +1671,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A774C0_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi index 8ec59094882b543d143db4a2e42a41fa97444376..c5a0e7866b2ffc7667111bb0f194efb2c6d9d458 100644 --- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi @@ -769,7 +769,7 @@ status = "disabled"; }; - i2c_dvfs: i2c@e60b0000 { + iic_pmic: i2c@e60b0000 { #address-cells = <1>; #size-cells = <0>; compatible = "renesas,iic-r8a774e1", @@ -2423,7 +2423,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi index a297af22a195fa1f124692913907eebdf090990c..07c8763c1e77fc8a73ef081c90cc564ebe045b66 100644 --- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi @@ -21,17 +21,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - i2c7 = &i2c_dvfs; - }; - /* * The external audio clocks are configured as 0 Hz fixed frequency * clocks by default. diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi index 4159c23d38743e30563a629c02ebe93328c3fe66..1424d4ad941f44b4011311fe12a9dde3715d0c93 100644 --- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi @@ -16,17 +16,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - i2c7 = &i2c_dvfs; - }; - /* * The external audio clocks are configured as 0 Hz fixed frequency * clocks by default. diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi index 21a5e1cdd9f1c80ace35a1b1dc280f049acf2a68..997f29521f66c35433fcf28de56046bbf2097586 100644 --- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi @@ -21,17 +21,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - i2c7 = &i2c_dvfs; - }; - /* * The external audio clocks are configured as 0 Hz fixed frequency * clocks by default. diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts index 49d1a929aef7560aa93872a27f0de53ee63adc65..004a5eacd460da9517be440663cf7fd080fd2b1b 100644 --- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts +++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts @@ -15,6 +15,11 @@ compatible = "renesas,eagle", "renesas,r8a77970"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; serial0 = &scif0; ethernet0 = &avb; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts index 39f3e6cbba3d500ff57a47853ced3939c1c8ddd4..c2b65f8de547ce7ec4e5da00c9157d3775773d08 100644 --- a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts +++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts @@ -15,6 +15,11 @@ compatible = "renesas,v3msk", "renesas,r8a77970"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; serial0 = &scif0; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi index 2703ef3a38c203a302e38dffa8d5208f7125195e..ed6e2e47c60479efbb28f43679ebd222e936c9f1 100644 --- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi @@ -16,14 +16,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - }; - /* External CAN clock - to be overridden by boards that provide it */ can_clk: can { compatible = "fixed-clock"; @@ -1053,7 +1045,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A77970_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts index 43ed033eb512505e98e7b327f902bb3eee9fe374..1d326552e2facd06b93fe0e0cd21d48c8c7f60c9 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts +++ b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts @@ -8,541 +8,9 @@ /dts-v1/; #include "r8a77980.dtsi" -#include +#include "condor-common.dtsi" / { model = "Renesas Condor board based on r8a77980"; compatible = "renesas,condor", "renesas,r8a77980"; - - aliases { - serial0 = &scif0; - ethernet0 = &gether; - }; - - chosen { - stdout-path = "serial0:115200n8"; - }; - - d1_8v: regulator-2 { - compatible = "regulator-fixed"; - regulator-name = "D1.8V"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-boot-on; - regulator-always-on; - }; - - d3_3v: regulator-0 { - compatible = "regulator-fixed"; - regulator-name = "D3.3V"; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; - regulator-boot-on; - regulator-always-on; - }; - - hdmi-out { - compatible = "hdmi-connector"; - type = "a"; - - port { - hdmi_con: endpoint { - remote-endpoint = <&adv7511_out>; - }; - }; - }; - - lvds-decoder { - compatible = "thine,thc63lvd1024"; - vcc-supply = <&d3_3v>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - thc63lvd1024_in: endpoint { - remote-endpoint = <&lvds0_out>; - }; - }; - - port@2 { - reg = <2>; - thc63lvd1024_out: endpoint { - remote-endpoint = <&adv7511_in>; - }; - }; - }; - }; - - memory@48000000 { - device_type = "memory"; - /* first 128MB is reserved for secure area. */ - reg = <0 0x48000000 0 0x78000000>; - }; - - vddq_vin01: regulator-1 { - compatible = "regulator-fixed"; - regulator-name = "VDDQ_VIN01"; - regulator-min-microvolt = <1800000>; - regulator-max-microvolt = <1800000>; - regulator-boot-on; - regulator-always-on; - }; - - x1_clk: x1-clock { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <148500000>; - }; -}; - -&canfd { - pinctrl-0 = <&canfd0_pins>; - pinctrl-names = "default"; - status = "okay"; - - channel0 { - status = "okay"; - }; -}; - -&csi40 { - status = "okay"; - - ports { - port@0 { - csi40_in: endpoint { - clock-lanes = <0>; - data-lanes = <1 2 3 4>; - remote-endpoint = <&max9286_out0>; - }; - }; - }; -}; - -&csi41 { - status = "okay"; - - ports { - port@0 { - csi41_in: endpoint { - clock-lanes = <0>; - data-lanes = <1 2 3 4>; - remote-endpoint = <&max9286_out1>; - }; - }; - }; -}; - -&du { - clocks = <&cpg CPG_MOD 724>, - <&x1_clk>; - clock-names = "du.0", "dclkin.0"; - status = "okay"; -}; - -&extal_clk { - clock-frequency = <16666666>; -}; - -&extalr_clk { - clock-frequency = <32768>; -}; - -&gether { - pinctrl-0 = <&gether_pins>; - pinctrl-names = "default"; - - phy-mode = "rgmii-id"; - phy-handle = <&phy0>; - renesas,no-ether-link; - status = "okay"; - - phy0: ethernet-phy@0 { - compatible = "ethernet-phy-id0022.1622", - "ethernet-phy-ieee802.3-c22"; - rxc-skew-ps = <1500>; - reg = <0>; - interrupt-parent = <&gpio4>; - interrupts = <23 IRQ_TYPE_LEVEL_LOW>; - reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>; - }; -}; - -&i2c0 { - pinctrl-0 = <&i2c0_pins>; - pinctrl-names = "default"; - - status = "okay"; - clock-frequency = <400000>; - - io_expander0: gpio@20 { - compatible = "onnn,pca9654"; - reg = <0x20>; - gpio-controller; - #gpio-cells = <2>; - }; - - io_expander1: gpio@21 { - compatible = "onnn,pca9654"; - reg = <0x21>; - gpio-controller; - #gpio-cells = <2>; - }; - - hdmi@39 { - compatible = "adi,adv7511w"; - reg = <0x39>; - interrupt-parent = <&gpio1>; - interrupts = <20 IRQ_TYPE_LEVEL_LOW>; - avdd-supply = <&d1_8v>; - dvdd-supply = <&d1_8v>; - pvdd-supply = <&d1_8v>; - bgvdd-supply = <&d1_8v>; - dvdd-3v-supply = <&d3_3v>; - - adi,input-depth = <8>; - adi,input-colorspace = "rgb"; - adi,input-clock = "1x"; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - adv7511_in: endpoint { - remote-endpoint = <&thc63lvd1024_out>; - }; - }; - - port@1 { - reg = <1>; - adv7511_out: endpoint { - remote-endpoint = <&hdmi_con>; - }; - }; - }; - }; -}; - -&i2c1 { - pinctrl-0 = <&i2c1_pins>; - pinctrl-names = "default"; - - status = "okay"; - clock-frequency = <400000>; - - gmsl0: gmsl-deserializer@48 { - compatible = "maxim,max9286"; - reg = <0x48>; - - maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>; - enable-gpios = <&io_expander0 0 GPIO_ACTIVE_HIGH>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - }; - - port@1 { - reg = <1>; - }; - - port@2 { - reg = <2>; - }; - - port@3 { - reg = <3>; - }; - - port@4 { - reg = <4>; - max9286_out0: endpoint { - clock-lanes = <0>; - data-lanes = <1 2 3 4>; - remote-endpoint = <&csi40_in>; - }; - }; - }; - - i2c-mux { - #address-cells = <1>; - #size-cells = <0>; - - i2c@0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0>; - - status = "disabled"; - }; - - i2c@1 { - #address-cells = <1>; - #size-cells = <0>; - reg = <1>; - - status = "disabled"; - }; - - i2c@2 { - #address-cells = <1>; - #size-cells = <0>; - reg = <2>; - - status = "disabled"; - }; - - i2c@3 { - #address-cells = <1>; - #size-cells = <0>; - reg = <3>; - - status = "disabled"; - }; - }; - }; - - gmsl1: gmsl-deserializer@4a { - compatible = "maxim,max9286"; - reg = <0x4a>; - - maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>; - enable-gpios = <&io_expander1 0 GPIO_ACTIVE_HIGH>; - - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - reg = <0>; - }; - - port@1 { - reg = <1>; - }; - - port@2 { - reg = <2>; - }; - - port@3 { - reg = <3>; - }; - - port@4 { - reg = <4>; - max9286_out1: endpoint { - clock-lanes = <0>; - data-lanes = <1 2 3 4>; - remote-endpoint = <&csi41_in>; - }; - }; - }; - - i2c-mux { - #address-cells = <1>; - #size-cells = <0>; - - i2c@0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0>; - - status = "disabled"; - }; - - i2c@1 { - #address-cells = <1>; - #size-cells = <0>; - reg = <1>; - - status = "disabled"; - }; - - i2c@2 { - #address-cells = <1>; - #size-cells = <0>; - reg = <2>; - - status = "disabled"; - }; - - i2c@3 { - #address-cells = <1>; - #size-cells = <0>; - reg = <3>; - - status = "disabled"; - }; - }; - }; -}; - -&lvds0 { - status = "okay"; - - ports { - port@1 { - lvds0_out: endpoint { - remote-endpoint = <&thc63lvd1024_in>; - }; - }; - }; -}; - -&mmc0 { - pinctrl-0 = <&mmc_pins>; - pinctrl-1 = <&mmc_pins>; - pinctrl-names = "default", "state_uhs"; - - vmmc-supply = <&d3_3v>; - vqmmc-supply = <&vddq_vin01>; - mmc-hs200-1_8v; - bus-width = <8>; - no-sd; - no-sdio; - non-removable; - status = "okay"; -}; - -&pciec { - status = "okay"; -}; - -&pcie_bus_clk { - clock-frequency = <100000000>; -}; - -&pcie_phy { - status = "okay"; -}; - -&pfc { - canfd0_pins: canfd0 { - groups = "canfd0_data_a"; - function = "canfd0"; - }; - - gether_pins: gether { - groups = "gether_mdio_a", "gether_rgmii", - "gether_txcrefclk", "gether_txcrefclk_mega"; - function = "gether"; - }; - - i2c0_pins: i2c0 { - groups = "i2c0"; - function = "i2c0"; - }; - - i2c1_pins: i2c1 { - groups = "i2c1"; - function = "i2c1"; - }; - - mmc_pins: mmc { - groups = "mmc_data8", "mmc_ctrl", "mmc_ds"; - function = "mmc"; - power-source = <1800>; - }; - - qspi0_pins: qspi0 { - groups = "qspi0_ctrl", "qspi0_data4"; - function = "qspi0"; - }; - - scif0_pins: scif0 { - groups = "scif0_data"; - function = "scif0"; - }; - - scif_clk_pins: scif_clk { - groups = "scif_clk_b"; - function = "scif_clk"; - }; -}; - -&rpc { - pinctrl-0 = <&qspi0_pins>; - pinctrl-names = "default"; - - status = "okay"; - - flash@0 { - compatible = "spansion,s25fs512s", "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <50000000>; - spi-rx-bus-width = <4>; - - partitions { - compatible = "fixed-partitions"; - #address-cells = <1>; - #size-cells = <1>; - - bootparam@0 { - reg = <0x00000000 0x040000>; - read-only; - }; - cr7@40000 { - reg = <0x00040000 0x080000>; - read-only; - }; - cert_header_sa3@c0000 { - reg = <0x000c0000 0x080000>; - read-only; - }; - bl2@140000 { - reg = <0x00140000 0x040000>; - read-only; - }; - cert_header_sa6@180000 { - reg = <0x00180000 0x040000>; - read-only; - }; - bl31@1c0000 { - reg = <0x001c0000 0x460000>; - read-only; - }; - uboot@640000 { - reg = <0x00640000 0x0c0000>; - read-only; - }; - uboot-env@700000 { - reg = <0x00700000 0x040000>; - read-only; - }; - dtb@740000 { - reg = <0x00740000 0x080000>; - }; - kernel@7c0000 { - reg = <0x007c0000 0x1400000>; - }; - user@1bc0000 { - reg = <0x01bc0000 0x2440000>; - }; - }; - }; -}; - -&rwdt { - timeout-sec = <60>; - status = "okay"; -}; - -&scif0 { - pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>; - pinctrl-names = "default"; - - status = "okay"; -}; - -&scif_clk { - clock-frequency = <14745600>; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts index 1d09d8867651a06b9f6a9fa5a130a8e4b200a785..d168b0e7747d349428956cc267491f398768c2ee 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts +++ b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts @@ -15,6 +15,12 @@ compatible = "renesas,v3hsk", "renesas,r8a77980"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; serial0 = &scif0; ethernet0 = &gether; }; diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi index 8594be72f2219d83294eabaebf2cd25f27363f5b..c4ac28a0f7161d000714571af0cb6b44fa01ec21 100644 --- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi @@ -16,15 +16,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - }; - /* External CAN clock - to be overridden by boards that provide it */ can_clk: can { compatible = "fixed-clock"; @@ -1359,7 +1350,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 917>; - clock-names = "rpc"; power-domains = <&sysc R8A77980_PD_ALWAYS_ON>; resets = <&cpg 917>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts b/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts new file mode 100644 index 0000000000000000000000000000000000000000..9f488dea0f341eedbcfcb2d185a895b5faaf78c7 --- /dev/null +++ b/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for the Condor-I board on r8a77980A (ES2.0) + * + * Copyright (C) 2022 Renesas Electronics Corp. + */ + +/dts-v1/; +#include "r8a77980a.dtsi" +#include "condor-common.dtsi" + +/ { + model = "Renesas Condor-I board based on r8a77980A (ES2.0)"; + compatible = "renesas,condor-i", "renesas,r8a77980a", "renesas,r8a77980"; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a77980a.dtsi b/arch/arm64/boot/dts/renesas/r8a77980a.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..25b2d27b616710226ff786c37d2e18a2fe1b891e --- /dev/null +++ b/arch/arm64/boot/dts/renesas/r8a77980a.dtsi @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: (GPL-2.0 or MIT) +/* + * Device Tree Source for the R-Car V3H2 (R8A77980A) SoC + * + * Copyright (C) 2022 Renesas Electronics Corp. + */ +#include "r8a77980.dtsi" + +/ { + compatible = "renesas,r8a77980a", "renesas,r8a77980"; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi index 565e9d85946e6f7ff9b3da4db27c2163df8e9268..3053b4b2149788c6ace5958bb7fd2616a18f4546 100644 --- a/arch/arm64/boot/dts/renesas/r8a77990.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi @@ -14,17 +14,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - i2c7 = &i2c7; - }; - /* * The external audio clocks are configured as 0 Hz fixed frequency * clocks by default. diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi index 53c4a26198e3db26d084f1dfa5699265b1570b21..99b73e21c82c2b18261fd98bc29be7982d926c09 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi @@ -16,6 +16,13 @@ compatible = "renesas,falcon-cpu", "renesas,r8a779a0"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; serial0 = &scif0; }; diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi index 3d668709d8a8d09f5bae98d23007809e05444d86..ed9400f903c9ecefde6a268fcb0b4d739b87303b 100644 --- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi @@ -14,16 +14,6 @@ #address-cells = <2>; #size-cells = <2>; - aliases { - i2c0 = &i2c0; - i2c1 = &i2c1; - i2c2 = &i2c2; - i2c3 = &i2c3; - i2c4 = &i2c4; - i2c5 = &i2c5; - i2c6 = &i2c6; - }; - /* External CAN clock - to be overridden by boards that provide it */ can_clk: can { compatible = "fixed-clock"; @@ -257,7 +247,7 @@ cmt0: timer@e60f0000 { compatible = "renesas,r8a779a0-cmt0", - "renesas,rcar-gen3-cmt0"; + "renesas,rcar-gen4-cmt0"; reg = <0 0xe60f0000 0 0x1004>; interrupts = , ; @@ -270,7 +260,7 @@ cmt1: timer@e6130000 { compatible = "renesas,r8a779a0-cmt1", - "renesas,rcar-gen3-cmt1"; + "renesas,rcar-gen4-cmt1"; reg = <0 0xe6130000 0 0x1004>; interrupts = , , @@ -289,7 +279,7 @@ cmt2: timer@e6140000 { compatible = "renesas,r8a779a0-cmt1", - "renesas,rcar-gen3-cmt1"; + "renesas,rcar-gen4-cmt1"; reg = <0 0xe6140000 0 0x1004>; interrupts = , , @@ -308,7 +298,7 @@ cmt3: timer@e6148000 { compatible = "renesas,r8a779a0-cmt1", - "renesas,rcar-gen3-cmt1"; + "renesas,rcar-gen4-cmt1"; reg = <0 0xe6148000 0 0x1004>; interrupts = , , @@ -2075,7 +2065,7 @@ mmc0: mmc@ee140000 { compatible = "renesas,sdhi-r8a779a0", - "renesas,rcar-gen3-sdhi"; + "renesas,rcar-gen4-sdhi"; reg = <0 0xee140000 0 0x2000>; interrupts = ; clocks = <&cpg CPG_MOD 706>, <&cpg CPG_CORE R8A779A0_CLK_SD0H>; @@ -2096,7 +2086,6 @@ reg-names = "regs", "dirmap", "wbuf"; interrupts = ; clocks = <&cpg CPG_MOD 629>; - clock-names = "rpc"; power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>; resets = <&cpg 629>; #address-cells = <1>; diff --git a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi index 28fbf7bc1eb4df20f0d625ec3e197f10face202b..a45df1041705159fb1e6f4efd191be3080fe7b50 100644 --- a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi @@ -11,6 +11,16 @@ model = "Renesas Spider CPU board"; compatible = "renesas,spider-cpu", "renesas,r8a779f0"; + aliases { + serial0 = &scif3; + serial1 = &scif0; + }; + + chosen { + bootargs = "ignore_loglevel rw root=/dev/nfs ip=on"; + stdout-path = "serial0:115200n8"; + }; + memory@48000000 { device_type = "memory"; /* first 128MB is reserved for secure area. */ @@ -21,6 +31,24 @@ device_type = "memory"; reg = <0x4 0x80000000 0x0 0x80000000>; }; + + reg_1p8v: regulator-1p8v { + compatible = "regulator-fixed"; + regulator-name = "fixed-1.8V"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-boot-on; + regulator-always-on; + }; + + reg_3p3v: regulator-3p3v { + compatible = "regulator-fixed"; + regulator-name = "fixed-3.3V"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + regulator-always-on; + }; }; &extal_clk { @@ -46,6 +74,27 @@ }; }; +/* + * This board also has a microSD slot which we will not support upstream + * because we cannot directly switch voltages in software. + */ +&mmc0 { + pinctrl-0 = <&mmc_pins>; + pinctrl-1 = <&mmc_pins>; + pinctrl-names = "default", "state_uhs"; + + vmmc-supply = <®_3p3v>; + vqmmc-supply = <®_1p8v>; + mmc-hs200-1_8v; + mmc-hs400-1_8v; + bus-width = <8>; + no-sd; + no-sdio; + non-removable; + full-pwr-cycle-in-suspend; + status = "okay"; +}; + &pfc { pinctrl-0 = <&scif_clk_pins>; pinctrl-names = "default"; @@ -55,6 +104,12 @@ function = "i2c4"; }; + mmc_pins: mmc { + groups = "mmc_data8", "mmc_ctrl", "mmc_ds"; + function = "mmc"; + power-source = <1800>; + }; + scif0_pins: scif0 { groups = "scif0_data", "scif0_ctrl"; function = "scif0"; diff --git a/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts b/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts index 7a7c8ffba7118a1f61bae97d6d5290ef0ae7602b..7aac3f4d319c330cf5e8ff570f66fa2bb808e876 100644 --- a/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts +++ b/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts @@ -12,15 +12,6 @@ / { model = "Renesas Spider CPU and Breakout boards based on r8a779f0"; compatible = "renesas,spider-breakout", "renesas,spider-cpu", "renesas,r8a779f0"; - - aliases { - serial0 = &scif3; - serial1 = &scif0; - }; - - chosen { - stdout-path = "serial0:115200n8"; - }; }; &i2c4 { diff --git a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi index 384817ffa4deb73c8f88e95ca474d7462845a7bc..c2f152bcf10ec43890fdb1ab8139f93fbd5b8ec3 100644 --- a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi @@ -301,6 +301,76 @@ #interrupt-cells = <2>; }; + cmt0: timer@e60f0000 { + compatible = "renesas,r8a779f0-cmt0", + "renesas,rcar-gen4-cmt0"; + reg = <0 0xe60f0000 0 0x1004>; + interrupts = , + ; + clocks = <&cpg CPG_MOD 910>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 910>; + status = "disabled"; + }; + + cmt1: timer@e6130000 { + compatible = "renesas,r8a779f0-cmt1", + "renesas,rcar-gen4-cmt1"; + reg = <0 0xe6130000 0 0x1004>; + interrupts = , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD 911>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 911>; + status = "disabled"; + }; + + cmt2: timer@e6140000 { + compatible = "renesas,r8a779f0-cmt1", + "renesas,rcar-gen4-cmt1"; + reg = <0 0xe6140000 0 0x1004>; + interrupts = , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD 912>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 912>; + status = "disabled"; + }; + + cmt3: timer@e6148000 { + compatible = "renesas,r8a779f0-cmt1", + "renesas,rcar-gen4-cmt1"; + reg = <0 0xe6148000 0 0x1004>; + interrupts = , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD 913>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 913>; + status = "disabled"; + }; + cpg: clock-controller@e6150000 { compatible = "renesas,r8a779f0-cpg-mssr"; reg = <0 0xe6150000 0 0x4000>; @@ -334,6 +404,71 @@ #thermal-sensor-cells = <1>; }; + tmu0: timer@e61e0000 { + compatible = "renesas,tmu-r8a779f0", "renesas,tmu"; + reg = <0 0xe61e0000 0 0x30>; + interrupts = , + , + ; + clocks = <&cpg CPG_MOD 713>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 713>; + status = "disabled"; + }; + + tmu1: timer@e6fc0000 { + compatible = "renesas,tmu-r8a779f0", "renesas,tmu"; + reg = <0 0xe6fc0000 0 0x30>; + interrupts = , + , + ; + clocks = <&cpg CPG_MOD 714>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 714>; + status = "disabled"; + }; + + tmu2: timer@e6fd0000 { + compatible = "renesas,tmu-r8a779f0", "renesas,tmu"; + reg = <0 0xe6fd0000 0 0x30>; + interrupts = , + , + ; + clocks = <&cpg CPG_MOD 715>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 715>; + status = "disabled"; + }; + + tmu3: timer@e6fe0000 { + compatible = "renesas,tmu-r8a779f0", "renesas,tmu"; + reg = <0 0xe6fe0000 0 0x30>; + interrupts = , + , + ; + clocks = <&cpg CPG_MOD 716>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 716>; + status = "disabled"; + }; + + tmu4: timer@ffc00000 { + compatible = "renesas,tmu-r8a779f0", "renesas,tmu"; + reg = <0 0xffc00000 0 0x30>; + interrupts = , + , + ; + clocks = <&cpg CPG_MOD 717>; + clock-names = "fck"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 717>; + status = "disabled"; + }; + i2c0: i2c@e6500000 { compatible = "renesas,i2c-r8a779f0", "renesas,rcar-gen4-i2c"; @@ -584,6 +719,70 @@ status = "disabled"; }; + msiof0: spi@e6e90000 { + compatible = "renesas,msiof-r8a779f0", + "renesas,rcar-gen4-msiof"; + reg = <0 0xe6e90000 0 0x0064>; + interrupts = ; + clocks = <&cpg CPG_MOD 618>; + dmas = <&dmac0 0x41>, <&dmac0 0x40>, + <&dmac1 0x41>, <&dmac1 0x40>; + dma-names = "tx", "rx", "tx", "rx"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 618>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + msiof1: spi@e6ea0000 { + compatible = "renesas,msiof-r8a779f0", + "renesas,rcar-gen4-msiof"; + reg = <0 0xe6ea0000 0 0x0064>; + interrupts = ; + clocks = <&cpg CPG_MOD 619>; + dmas = <&dmac0 0x43>, <&dmac0 0x42>, + <&dmac1 0x43>, <&dmac1 0x42>; + dma-names = "tx", "rx", "tx", "rx"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 619>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + msiof2: spi@e6c00000 { + compatible = "renesas,msiof-r8a779f0", + "renesas,rcar-gen4-msiof"; + reg = <0 0xe6c00000 0 0x0064>; + interrupts = ; + clocks = <&cpg CPG_MOD 620>; + dmas = <&dmac0 0x45>, <&dmac0 0x44>, + <&dmac1 0x45>, <&dmac1 0x44>; + dma-names = "tx", "rx", "tx", "rx"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 620>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + msiof3: spi@e6c10000 { + compatible = "renesas,msiof-r8a779f0", + "renesas,rcar-gen4-msiof"; + reg = <0 0xe6c10000 0 0x0064>; + interrupts = ; + clocks = <&cpg CPG_MOD 621>; + dmas = <&dmac0 0x47>, <&dmac0 0x46>, + <&dmac1 0x47>, <&dmac1 0x46>; + dma-names = "tx", "rx", "tx", "rx"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 621>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + dmac0: dma-controller@e7350000 { compatible = "renesas,dmac-r8a779f0", "renesas,rcar-gen4-dmac"; @@ -670,6 +869,19 @@ <&ipmmu_ds0 30>, <&ipmmu_ds0 31>; }; + mmc0: mmc@ee140000 { + compatible = "renesas,sdhi-r8a779f0", + "renesas,rcar-gen4-sdhi"; + reg = <0 0xee140000 0 0x2000>; + interrupts = ; + clocks = <&cpg CPG_MOD 706>, <&cpg CPG_CORE R8A779F0_CLK_SD0H>; + clock-names = "core", "clkh"; + power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>; + resets = <&cpg 706>; + max-frequency = <200000000>; + status = "disabled"; + }; + ipmmu_rt0: iommu@ee480000 { compatible = "renesas,ipmmu-r8a779f0", "renesas,rcar-gen4-ipmmu-vmsa"; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi index ea4ae4b893abf931e8ee9f3d35b37926a13548dd..895f0bd9f754080feb6227ae6dfd394b1cedf2ea 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi @@ -7,10 +7,80 @@ #include "r8a779g0.dtsi" +#include +#include +#include + / { model = "Renesas White Hawk CPU board"; compatible = "renesas,white-hawk-cpu", "renesas,r8a779g0"; + aliases { + ethernet0 = &avb0; + serial0 = &hscif0; + }; + + chosen { + bootargs = "ignore_loglevel rw root=/dev/nfs ip=on"; + stdout-path = "serial0:921600n8"; + }; + + keys { + compatible = "gpio-keys"; + + pinctrl-0 = <&keys_pins>; + pinctrl-names = "default"; + + key-1 { + gpios = <&gpio5 0 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW47"; + wakeup-source; + debounce-interval = <20>; + }; + + key-2 { + gpios = <&gpio5 1 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW48"; + wakeup-source; + debounce-interval = <20>; + }; + + key-3 { + gpios = <&gpio5 2 GPIO_ACTIVE_LOW>; + linux,code = ; + label = "SW49"; + wakeup-source; + debounce-interval = <20>; + }; + }; + + leds { + compatible = "gpio-leds"; + + led-1 { + gpios = <&gpio7 0 GPIO_ACTIVE_HIGH>; + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <1>; + }; + + led-2 { + gpios = <&gpio7 1 GPIO_ACTIVE_HIGH>; + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <2>; + }; + + led-3 { + gpios = <&gpio7 2 GPIO_ACTIVE_HIGH>; + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <3>; + }; + }; + memory@48000000 { device_type = "memory"; /* first 128MB is reserved for secure area. */ @@ -28,6 +98,24 @@ }; }; +&avb0 { + pinctrl-0 = <&avb0_pins>; + pinctrl-names = "default"; + phy-handle = <&phy0>; + tx-internal-delay-ps = <2000>; + status = "okay"; + + phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id0022.1622", + "ethernet-phy-ieee802.3-c22"; + rxc-skew-ps = <1500>; + reg = <0>; + interrupt-parent = <&gpio7>; + interrupts = <5 IRQ_TYPE_LEVEL_LOW>; + reset-gpios = <&gpio7 10 GPIO_ACTIVE_LOW>; + }; +}; + &extal_clk { clock-frequency = <16666666>; }; @@ -40,6 +128,69 @@ status = "okay"; }; +&i2c0 { + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + + status = "okay"; + clock-frequency = <400000>; + + eeprom@50 { + compatible = "rohm,br24g01", "atmel,24c01"; + label = "cpu-board"; + reg = <0x50>; + pagesize = <8>; + }; +}; + +&pfc { + pinctrl-0 = <&scif_clk_pins>; + pinctrl-names = "default"; + + avb0_pins: avb0 { + mux { + groups = "avb0_link", "avb0_mdio", "avb0_rgmii", + "avb0_txcrefclk"; + function = "avb0"; + }; + + pins_mdio { + groups = "avb0_mdio"; + drive-strength = <21>; + }; + + pins_mii { + groups = "avb0_rgmii"; + drive-strength = <21>; + }; + + }; + hscif0_pins: hscif0 { + groups = "hscif0_data"; + function = "hscif0"; + }; + + i2c0_pins: i2c0 { + groups = "i2c0"; + function = "i2c0"; + }; + + keys_pins: keys { + pins = "GP_5_0", "GP_5_1", "GP_5_2"; + bias-pull-up; + }; + + scif_clk_pins: scif_clk { + groups = "scif_clk"; + function = "scif_clk"; + }; +}; + &scif_clk { clock-frequency = <24000000>; }; + +&rwdt { + timeout-sec = <60>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..ae7522b60e5dbad16c0a61767117ee9841f267a7 --- /dev/null +++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Device Tree Source for the R-Car V4H White Hawk CSI/DSI sub-board + * + * Copyright (C) 2022 Glider bv + */ + +&i2c0 { + eeprom@52 { + compatible = "rohm,br24g01", "atmel,24c01"; + label = "csi-dsi-sub-board-id"; + reg = <0x52>; + pagesize = <8>; + }; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..4f411f95c674bd51507ac7f6fe9ff0282850390a --- /dev/null +++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * Device Tree Source for the R-Car V4H White Hawk RAVB/Ethernet(1000Base-T1) + * sub-board + * + * Copyright (C) 2022 Glider bv + */ + +&i2c0 { + eeprom@53 { + compatible = "rohm,br24g01", "atmel,24c01"; + label = "ethernet-sub-board-id"; + reg = <0x53>; + pagesize = <8>; + }; +}; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts index bc0ac109b17cec3066b68d61e33c39c1fd019020..04a2b6b83e743f326c9b88d8c98c915175512254 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts +++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts @@ -7,16 +7,19 @@ /dts-v1/; #include "r8a779g0-white-hawk-cpu.dtsi" +#include "r8a779g0-white-hawk-csi-dsi.dtsi" +#include "r8a779g0-white-hawk-ethernet.dtsi" / { model = "Renesas White Hawk CPU and Breakout boards based on r8a779g0"; compatible = "renesas,white-hawk-breakout", "renesas,white-hawk-cpu", "renesas,r8a779g0"; +}; - aliases { - serial0 = &hscif0; - }; - - chosen { - stdout-path = "serial0:921600n8"; +&i2c0 { + eeprom@51 { + compatible = "rohm,br24g01", "atmel,24c01"; + label = "breakout-board"; + reg = <0x51>; + pagesize = <8>; }; }; diff --git a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi index 7cbb0de060ddc03b6df2cdcbd0a4087a56c00d84..d70f0600ae5a946950569ee701ab6efa1b4bb9fe 100644 --- a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi +++ b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi @@ -59,6 +59,161 @@ #size-cells = <2>; ranges; + rwdt: watchdog@e6020000 { + compatible = "renesas,r8a779g0-wdt", + "renesas,rcar-gen4-wdt"; + reg = <0 0xe6020000 0 0x0c>; + interrupts = ; + clocks = <&cpg CPG_MOD 907>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 907>; + status = "disabled"; + }; + + pfc: pinctrl@e6050000 { + compatible = "renesas,pfc-r8a779g0"; + reg = <0 0xe6050000 0 0x16c>, <0 0xe6050800 0 0x16c>, + <0 0xe6058000 0 0x16c>, <0 0xe6058800 0 0x16c>, + <0 0xe6060000 0 0x16c>, <0 0xe6060800 0 0x16c>, + <0 0xe6061000 0 0x16c>, <0 0xe6061800 0 0x16c>, + <0 0xe6068000 0 0x16c>; + }; + + gpio0: gpio@e6050180 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6050180 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 915>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 915>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 0 19>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio1: gpio@e6050980 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6050980 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 915>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 915>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 32 29>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio2: gpio@e6058180 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6058180 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 916>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 916>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 64 20>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio3: gpio@e6058980 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6058980 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 916>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 916>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 96 30>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio4: gpio@e6060180 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6060180 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 917>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 917>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 128 25>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio5: gpio@e6060980 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6060980 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 917>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 917>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 160 21>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio6: gpio@e6061180 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6061180 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 917>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 917>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 192 21>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio7: gpio@e6061980 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6061980 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 917>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 917>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 224 21>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpio8: gpio@e6068180 { + compatible = "renesas,gpio-r8a779g0", + "renesas,rcar-gen4-gpio"; + reg = <0 0xe6068180 0 0x54>; + interrupts = ; + clocks = <&cpg CPG_MOD 918>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 918>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pfc 0 256 14>; + interrupt-controller; + #interrupt-cells = <2>; + }; + cpg: clock-controller@e6150000 { compatible = "renesas,r8a779g0-cpg-mssr"; reg = <0 0xe6150000 0 0x4000>; @@ -80,12 +235,96 @@ #power-domain-cells = <1>; }; + i2c0: i2c@e6500000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe6500000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 518>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 518>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@e6508000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe6508000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 519>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 519>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@e6510000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe6510000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 520>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 520>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c@e66d0000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe66d0000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 521>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 521>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@e66d8000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe66d8000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 522>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 522>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@e66e0000 { + compatible = "renesas,i2c-r8a779g0", + "renesas,rcar-gen4-i2c"; + reg = <0 0xe66e0000 0 0x40>; + interrupts = ; + clocks = <&cpg CPG_MOD 523>; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 523>; + i2c-scl-internal-delay-ns = <110>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + hscif0: serial@e6540000 { compatible = "renesas,hscif-r8a779g0", "renesas,rcar-gen4-hscif", "renesas,hscif"; reg = <0 0xe6540000 0 96>; - interrupts = ; + interrupts = ; clocks = <&cpg CPG_MOD 514>, <&cpg CPG_CORE R8A779G0_CLK_S0D3_PER>, <&scif_clk>; @@ -95,6 +334,147 @@ status = "disabled"; }; + avb0: ethernet@e6800000 { + compatible = "renesas,etheravb-r8a779g0", + "renesas,etheravb-rcar-gen4"; + reg = <0 0xe6800000 0 0x800>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4", + "ch5", "ch6", "ch7", "ch8", "ch9", + "ch10", "ch11", "ch12", "ch13", + "ch14", "ch15", "ch16", "ch17", + "ch18", "ch19", "ch20", "ch21", + "ch22", "ch23", "ch24"; + clocks = <&cpg CPG_MOD 211>; + clock-names = "fck"; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 211>; + phy-mode = "rgmii"; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + avb1: ethernet@e6810000 { + compatible = "renesas,etheravb-r8a779g0", + "renesas,etheravb-rcar-gen4"; + reg = <0 0xe6810000 0 0x800>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4", + "ch5", "ch6", "ch7", "ch8", "ch9", + "ch10", "ch11", "ch12", "ch13", + "ch14", "ch15", "ch16", "ch17", + "ch18", "ch19", "ch20", "ch21", + "ch22", "ch23", "ch24"; + clocks = <&cpg CPG_MOD 212>; + clock-names = "fck"; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 212>; + phy-mode = "rgmii"; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + avb2: ethernet@e6820000 { + compatible = "renesas,etheravb-r8a779g0", + "renesas,etheravb-rcar-gen4"; + reg = <0 0xe6820000 0 0x1000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4", + "ch5", "ch6", "ch7", "ch8", "ch9", + "ch10", "ch11", "ch12", "ch13", + "ch14", "ch15", "ch16", "ch17", + "ch18", "ch19", "ch20", "ch21", + "ch22", "ch23", "ch24"; + clocks = <&cpg CPG_MOD 213>; + clock-names = "fck"; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 213>; + phy-mode = "rgmii"; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + gic: interrupt-controller@f1000000 { compatible = "arm,gic-v3"; #interrupt-cells = <3>; diff --git a/arch/arm64/boot/dts/renesas/r8a779mb.dtsi b/arch/arm64/boot/dts/renesas/r8a779mb.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..40d1dce2f350f4851831bd68768085319a29a586 --- /dev/null +++ b/arch/arm64/boot/dts/renesas/r8a779mb.dtsi @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: (GPL-2.0 or MIT) +/* + * Device Tree Source for the R-Car H3Ne-1.7G (R8A779MB) SoC + * + * Copyright (C) 2022 Glider bv + */ + +#include "r8a77951.dtsi" + +/ { + compatible = "renesas,r8a779mb", "renesas,r8a7795"; +}; diff --git a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi index 40201a16d653c4bb02158d1ac7c2ca55a694500f..689aa4ba416b8dbc1f39808e186a44eec6a5c24c 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi @@ -13,14 +13,14 @@ #address-cells = <2>; #size-cells = <2>; - audio_clk1: audio-clk1 { + audio_clk1: audio1-clk { compatible = "fixed-clock"; #clock-cells = <0>; /* This value must be overridden by boards that provide it */ clock-frequency = <0>; }; - audio_clk2: audio-clk2 { + audio_clk2: audio2-clk { compatible = "fixed-clock"; #clock-cells = <0>; /* This value must be overridden by boards that provide it */ @@ -196,6 +196,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G043_RSPI0_CLKB>; resets = <&cpg R9A07G043_RSPI0_RST>; + dmas = <&dmac 0x2e95>, <&dmac 0x2e96>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -212,6 +214,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G043_RSPI1_CLKB>; resets = <&cpg R9A07G043_RSPI1_RST>; + dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -228,6 +232,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G043_RSPI2_CLKB>; resets = <&cpg R9A07G043_RSPI2_RST>; + dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -334,8 +340,8 @@ compatible = "renesas,r9a07g043-sci", "renesas,sci"; reg = <0 0x1004d000 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G043_SCI0_CLKP>; @@ -349,8 +355,8 @@ compatible = "renesas,r9a07g043-sci", "renesas,sci"; reg = <0 0x1004d400 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G043_SCI1_CLKP>; @@ -613,7 +619,7 @@ interrupts = ; }; - sdhi0: mmc@11c00000 { + sdhi0: mmc@11c00000 { compatible = "renesas,sdhi-r9a07g043", "renesas,rcar-gen3-sdhi"; reg = <0x0 0x11c00000 0 0x10000>; diff --git a/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts b/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts index 121e55282d1818572a79a1672338ba83d0f6ec29..059885a01ede9cb3d6a4b5742ca187e2e55a4094 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts +++ b/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts @@ -6,7 +6,19 @@ */ /dts-v1/; + +/* + * DIP-Switch SW1 setting + * 1 : High; 0: Low + * SW1-2 : SW_SD0_DEV_SEL (0: uSD; 1: eMMC) + * SW1-3 : SW_ET0_EN_N (0: ETHER0; 1: CAN0, CAN1, SSI1, RSPI1) + * Please change below macros according to SW1 setting on the SoM + */ +#define SW_SW0_DEV_SEL 1 +#define SW_ET0_EN_N 1 + #include "r9a07g043.dtsi" +#include "rzg2ul-smarc-som.dtsi" #include "rzg2ul-smarc.dtsi" / { diff --git a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi index 3652e511160fb5a758de96a939081227db337877..2283d4fb873634a5780f88fab9584f968874e454 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi @@ -261,6 +261,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G044_RSPI0_CLKB>; resets = <&cpg R9A07G044_RSPI0_RST>; + dmas = <&dmac 0x2e95>, <&dmac 0x2e96>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -277,6 +279,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G044_RSPI1_CLKB>; resets = <&cpg R9A07G044_RSPI1_RST>; + dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -293,6 +297,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G044_RSPI2_CLKB>; resets = <&cpg R9A07G044_RSPI2_RST>; + dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -394,8 +400,8 @@ compatible = "renesas,r9a07g044-sci", "renesas,sci"; reg = <0 0x1004d000 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G044_SCI0_CLKP>; @@ -409,8 +415,8 @@ compatible = "renesas,r9a07g044-sci", "renesas,sci"; reg = <0 0x1004d400 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G044_SCI1_CLKP>; @@ -638,6 +644,10 @@ reg = <0 0x11030000 0 0x10000>; gpio-controller; #gpio-cells = <2>; + #address-cells = <2>; + #interrupt-cells = <2>; + interrupt-parent = <&irqc>; + interrupt-controller; gpio-ranges = <&pinctrl 0 0 392>; clocks = <&cpg CPG_MOD R9A07G044_GPIO_HCLK>; power-domains = <&cpg>; @@ -646,6 +656,61 @@ <&cpg R9A07G044_GPIO_SPARE_RESETN>; }; + irqc: interrupt-controller@110a0000 { + compatible = "renesas,r9a07g044-irqc", + "renesas,rzg2l-irqc"; + #interrupt-cells = <2>; + #address-cells = <0>; + interrupt-controller; + reg = <0 0x110a0000 0 0x10000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD R9A07G044_IA55_CLK>, + <&cpg CPG_MOD R9A07G044_IA55_PCLK>; + clock-names = "clk", "pclk"; + power-domains = <&cpg>; + resets = <&cpg R9A07G044_IA55_RESETN>; + }; + dmac: dma-controller@11820000 { compatible = "renesas,r9a07g044-dmac", "renesas,rz-dmac"; @@ -713,7 +778,7 @@ interrupts = ; }; - sdhi0: mmc@11c00000 { + sdhi0: mmc@11c00000 { compatible = "renesas,sdhi-r9a07g044", "renesas,rcar-gen3-sdhi"; reg = <0x0 0x11c00000 0 0x10000>; diff --git a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi index 4d6b9d7684c94600993d7a75db78e72543231c39..358d4c34465fabfb870aa697636271187dbe2ab4 100644 --- a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi @@ -261,6 +261,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G054_RSPI0_CLKB>; resets = <&cpg R9A07G054_RSPI0_RST>; + dmas = <&dmac 0x2e95>, <&dmac 0x2e96>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -277,6 +279,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G054_RSPI1_CLKB>; resets = <&cpg R9A07G054_RSPI1_RST>; + dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -293,6 +297,8 @@ interrupt-names = "error", "rx", "tx"; clocks = <&cpg CPG_MOD R9A07G054_RSPI2_CLKB>; resets = <&cpg R9A07G054_RSPI2_RST>; + dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>; + dma-names = "tx", "rx"; power-domains = <&cpg>; num-cs = <1>; #address-cells = <1>; @@ -399,8 +405,8 @@ compatible = "renesas,r9a07g054-sci", "renesas,sci"; reg = <0 0x1004d000 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G054_SCI0_CLKP>; @@ -414,8 +420,8 @@ compatible = "renesas,r9a07g054-sci", "renesas,sci"; reg = <0 0x1004d400 0 0x400>; interrupts = , - , - , + , + , ; interrupt-names = "eri", "rxi", "txi", "tei"; clocks = <&cpg CPG_MOD R9A07G054_SCI1_CLKP>; @@ -644,6 +650,10 @@ reg = <0 0x11030000 0 0x10000>; gpio-controller; #gpio-cells = <2>; + #address-cells = <2>; + #interrupt-cells = <2>; + interrupt-parent = <&irqc>; + interrupt-controller; gpio-ranges = <&pinctrl 0 0 392>; clocks = <&cpg CPG_MOD R9A07G054_GPIO_HCLK>; power-domains = <&cpg>; @@ -652,6 +662,61 @@ <&cpg R9A07G054_GPIO_SPARE_RESETN>; }; + irqc: interrupt-controller@110a0000 { + compatible = "renesas,r9a07g054-irqc", + "renesas,rzg2l-irqc"; + #interrupt-cells = <2>; + #address-cells = <0>; + interrupt-controller; + reg = <0 0x110a0000 0 0x10000>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD R9A07G054_IA55_CLK>, + <&cpg CPG_MOD R9A07G054_IA55_PCLK>; + clock-names = "clk", "pclk"; + power-domains = <&cpg>; + resets = <&cpg R9A07G054_IA55_RESETN>; + }; + dmac: dma-controller@11820000 { compatible = "renesas,r9a07g054-dmac", "renesas,rz-dmac"; @@ -719,7 +784,7 @@ interrupts = ; }; - sdhi0: mmc@11c00000 { + sdhi0: mmc@11c00000 { compatible = "renesas,sdhi-r9a07g054", "renesas,rcar-gen3-sdhi"; reg = <0x0 0x11c00000 0 0x10000>; diff --git a/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts b/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts index c3a52fa0b16ecbfc8e892845d6afb284ffe9cead..5c15d73d059ff8f60b4711753e68bc0cde9415b1 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts +++ b/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts @@ -7,6 +7,7 @@ /dts-v1/; #include "r9a09g011.dtsi" +#include / { model = "RZ/V2M Evaluation Kit 2.0"; @@ -53,6 +54,32 @@ clock-frequency = <48000000>; }; +&i2c0 { + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&i2c2 { + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + clock-frequency = <100000>; + status = "okay"; +}; + +&pinctrl { + i2c0_pins: i2c0 { + pinmux = , /* SDA */ + ; /* SCL */ + }; + + i2c2_pins: i2c2 { + pinmux = , /* SDA */ + ; /* SCL */ + }; +}; + &uart0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi index d4cc5459fbb76d62149f19e70836fcbe39bf8ba6..fb1a97202c387dbc9352eb979ac83008f8ac1a5f 100644 --- a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi +++ b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi @@ -123,6 +123,34 @@ #power-domain-cells = <0>; }; + i2c0: i2c@a4030000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "renesas,i2c-r9a09g011", "renesas,rzv2m-i2c"; + reg = <0 0xa4030000 0 0x80>; + interrupts = , + ; + interrupt-names = "tia", "tis"; + clocks = <&cpg CPG_MOD R9A09G011_IIC_PCLK0>; + resets = <&cpg R9A09G011_IIC_GPA_PRESETN>; + power-domains = <&cpg>; + status = "disabled"; + }; + + i2c2: i2c@a4030100 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "renesas,i2c-r9a09g011", "renesas,rzv2m-i2c"; + reg = <0 0xa4030100 0 0x80>; + interrupts = , + ; + interrupt-names = "tia", "tis"; + clocks = <&cpg CPG_MOD R9A09G011_IIC_PCLK1>; + resets = <&cpg R9A09G011_IIC_GPB_PRESETN>; + power-domains = <&cpg>; + status = "disabled"; + }; + uart0: serial@a4040000 { compatible = "renesas,r9a09g011-uart", "renesas,em-uart"; reg = <0 0xa4040000 0 0x80>; @@ -132,6 +160,56 @@ clock-names = "sclk", "pclk"; status = "disabled"; }; + + pinctrl: pinctrl@b6250000 { + compatible = "renesas,r9a09g011-pinctrl"; + reg = <0 0xb6250000 0 0x800>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl 0 0 352>; + interrupts = , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ; + clocks = <&cpg CPG_MOD R9A09G011_PFC_PCLK>; + power-domains = <&cpg>; + resets = <&cpg R9A09G011_PFC_PRESETN>; + }; }; timer { diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi index 9410796c8ad6b72de1d159bd2a48223399f3ab5f..c4faff0923800ab35ecb27371d61119ae8b4b529 100644 --- a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi @@ -6,6 +6,7 @@ */ #include +#include #include /* SW1[2] should be at position 2/OFF to enable 64 GB eMMC */ @@ -94,6 +95,8 @@ compatible = "ethernet-phy-id0022.1640", "ethernet-phy-ieee802.3-c22"; reg = <7>; + interrupt-parent = <&irqc>; + interrupts = ; rxc-skew-psec = <2400>; txc-skew-psec = <2400>; rxdv-skew-psec = <0>; @@ -120,6 +123,8 @@ compatible = "ethernet-phy-id0022.1640", "ethernet-phy-ieee802.3-c22"; reg = <7>; + interrupt-parent = <&irqc>; + interrupts = ; rxc-skew-psec = <2400>; txc-skew-psec = <2400>; rxdv-skew-psec = <0>; @@ -171,7 +176,8 @@ , /* ET0_RXD0 */ , /* ET0_RXD1 */ , /* ET0_RXD2 */ - ; /* ET0_RXD3 */ + , /* ET0_RXD3 */ + ; /* IRQ2 */ }; eth1_pins: eth1 { @@ -189,7 +195,8 @@ , /* ET1_RXD0 */ , /* ET1_RXD1 */ , /* ET1_RXD2 */ - ; /* ET1_RXD3 */ + , /* ET1_RXD3 */ + ; /* IRQ3 */ }; gpio-sd0-pwr-en-hog { diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi index cf3b3d118ef170ceae53ef8e83b92fad2dfa980c..2a0feb53f0dcbc6ebb2f47f102f0010358654be4 100644 --- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi @@ -263,8 +263,3 @@ status = "okay"; timeout-sec = <60>; }; - -&wdt2 { - status = "okay"; - timeout-sec = <60>; -}; diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi index f9835c12023e063ce2e455546423e3aebfda353e..2a1331ed1a5c0faca2eb76ba798da4daefa5c76c 100644 --- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi +++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi @@ -5,17 +5,6 @@ * Copyright (C) 2022 Renesas Electronics Corp. */ -/* - * DIP-Switch SW1 setting - * 1 : High; 0: Low - * SW1-2 : SW_SD0_DEV_SEL (0: uSD; 1: eMMC) - * SW1-3 : SW_ET0_EN_N (0: ETHER0; 1: CAN0, CAN1, SSI1, RSPI1) - * Please change below macros according to SW1 setting - */ -#define SW_SW0_DEV_SEL 1 -#define SW_ET0_EN_N 1 - -#include "rzg2ul-smarc-som.dtsi" #include "rzg2ul-smarc-pinfunction.dtsi" #include "rz-smarc-common.dtsi" diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi index b7c7911858b2c742d192ba60473be39abf105ab8..d9747340181929aa3b3696cadf49aa4a7cec07e3 100644 --- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi +++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi @@ -33,6 +33,14 @@ / { aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &i2c_dvfs; serial0 = &scif2; serial1 = &hscif1; ethernet0 = &avb; diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi index 0772dfe4adffeebeb3cafe274b97748fbd47b2a0..29cedf4dc1a9df19fa6278c3fd08d612c8fe15e2 100644 --- a/arch/arm64/boot/dts/renesas/ulcb.dtsi +++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi @@ -21,6 +21,14 @@ model = "Renesas R-Car Gen3 ULCB board"; aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + i2c6 = &i2c6; + i2c7 = &i2c_dvfs; serial0 = &scif2; ethernet0 = &avb; mmc0 = &sdhi2; diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile index ef79a672804a1ebceb26a6d01a02dcc9ada836c8..8c15593c0ca4abf27a8fb894eefda6c519912865 100644 --- a/arch/arm64/boot/dts/rockchip/Makefile +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -21,6 +21,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-lion-haikou.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-orion-r68-meta.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-px5-evb.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-r88.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-eaidk-610.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-evb.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-ficus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-firefly.dtb @@ -40,12 +41,15 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4b.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-neo4.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-r4s.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-r4s-enterprise.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-orangepi.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-pinebook-pro.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-pinephone-pro.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-puma-haikou.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-mezzanine.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-plus.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-4c-plus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a-plus.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4b.dtb @@ -57,6 +61,8 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg353p.dtb +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg503.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.1.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.2.dtb dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-a.dtb diff --git a/arch/arm64/boot/dts/rockchip/px30-engicam-px30-core.dtsi b/arch/arm64/boot/dts/rockchip/px30-engicam-px30-core.dtsi index 7249871530ab96c778b09a6366a7a8ed817b3a00..5eecbefa8a336e83435c661cc189129ad08ac355 100644 --- a/arch/arm64/boot/dts/rockchip/px30-engicam-px30-core.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-engicam-px30-core.dtsi @@ -2,8 +2,8 @@ /* * Copyright (c) 2020 Fuzhou Rockchip Electronics Co., Ltd * Copyright (c) 2020 Engicam srl - * Copyright (c) 2020 Amarula Solutons - * Copyright (c) 2020 Amarula Solutons(India) + * Copyright (c) 2020 Amarula Solutions + * Copyright (c) 2020 Amarula Solutions(India) */ #include diff --git a/arch/arm64/boot/dts/rockchip/px30-evb.dts b/arch/arm64/boot/dts/rockchip/px30-evb.dts index 848bc39cf86ac32e6ef82cc6d65ec7f49f83acc3..07008d84434c19a7331edc16ec3cea774bc65153 100644 --- a/arch/arm64/boot/dts/rockchip/px30-evb.dts +++ b/arch/arm64/boot/dts/rockchip/px30-evb.dts @@ -450,8 +450,8 @@ dvdd-supply = <&vcc1v5_dvp>; dovdd-supply = <&vcc1v8_dvp>; pinctrl-names = "default"; - pinctrl-0 = <&cif_clkout_m0>; - reset-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>; + pinctrl-0 = <&cif_clkout_m0 &mipi_pdn>; + reset-gpios = <&gpio2 RK_PB6 GPIO_ACTIVE_LOW>; port { ucam_out: endpoint { @@ -537,6 +537,19 @@ <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; }; }; + + cif-m0 { + cif_clkout_m0: cif-clkout-m0 { + rockchip,pins = + <2 RK_PB3 1 &pcfg_pull_none_12ma>; + }; + }; + + mipi { + mipi_pdn: mipi-pdn { + rockchip,pins = <2 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; }; &pmu_io_domains { diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi index 214f94fea3dca4db87e5a541579e723283c53481..bfa3580429d10e010e736ad929243583287f0dea 100644 --- a/arch/arm64/boot/dts/rockchip/px30.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30.dtsi @@ -365,6 +365,28 @@ status = "disabled"; }; + i2s0_8ch: i2s@ff060000 { + compatible = "rockchip,px30-i2s-tdm"; + reg = <0x0 0xff060000 0x0 0x1000>; + interrupts = ; + clocks = <&cru SCLK_I2S0_TX>, <&cru SCLK_I2S0_RX>, <&cru HCLK_I2S0>; + clock-names = "mclk_tx", "mclk_rx", "hclk"; + dmas = <&dmac 16>, <&dmac 17>; + dma-names = "tx", "rx"; + rockchip,grf = <&grf>; + resets = <&cru SRST_I2S0_TX>, <&cru SRST_I2S0_RX>; + reset-names = "tx-m", "rx-m"; + pinctrl-names = "default"; + pinctrl-0 = <&i2s0_8ch_sclktx &i2s0_8ch_sclkrx + &i2s0_8ch_lrcktx &i2s0_8ch_lrckrx + &i2s0_8ch_sdo0 &i2s0_8ch_sdi0 + &i2s0_8ch_sdo1 &i2s0_8ch_sdi1 + &i2s0_8ch_sdo2 &i2s0_8ch_sdi2 + &i2s0_8ch_sdo3 &i2s0_8ch_sdi3>; + #sound-dai-cells = <0>; + status = "disabled"; + }; + i2s1_2ch: i2s@ff070000 { compatible = "rockchip,px30-i2s", "rockchip,rk3066-i2s"; reg = <0x0 0xff070000 0x0 0x1000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts index 415aa9ff8bd482ad43e6530e733dba6575ec72f6..72899a7143104bb1fc62a65efa80c8f6c4088156 100644 --- a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts +++ b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts @@ -52,6 +52,25 @@ pwms = <&pwm1 0 25000 0>; }; + battery: battery { + compatible = "simple-battery"; + charge-full-design-microamp-hours = <3000000>; + charge-term-current-microamp = <300000>; + constant-charge-current-max-microamp = <2000000>; + constant-charge-voltage-max-microvolt = <4200000>; + factory-internal-resistance-micro-ohms = <180000>; + voltage-max-design-microvolt = <4100000>; + voltage-min-design-microvolt = <3500000>; + + ocv-capacity-celsius = <20>; + ocv-capacity-table-0 = <4046950 100>, <4001920 95>, <3967900 90>, <3919950 85>, + <3888450 80>, <3861850 75>, <3831540 70>, <3799130 65>, + <3768190 60>, <3745650 55>, <3726610 50>, <3711630 45>, + <3696720 40>, <3685660 35>, <3674950 30>, <3663050 25>, + <3649470 20>, <3635260 15>, <3616920 10>, <3592440 5>, + <3574170 0>; + }; + gpio-keys { compatible = "gpio-keys"; pinctrl-names = "default"; @@ -472,6 +491,13 @@ }; }; + rk817_charger: charger { + monitored-battery = <&battery>; + rockchip,resistor-sense-micro-ohms = <10000>; + rockchip,sleep-enter-current-microamp = <300000>; + rockchip,sleep-filter-current-microamp = <100000>; + }; + rk817_codec: codec { rockchip,mic-in-differential; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts b/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts new file mode 100644 index 0000000000000000000000000000000000000000..d1f343345f67499a737d83b3d791b47efce32deb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts @@ -0,0 +1,939 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2022 Fuzhou Rockchip Electronics Co., Ltd. + */ + +/dts-v1/; +#include +#include +#include +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" + +/ { + model = "OPEN AI LAB EAIDK-610"; + compatible = "openailab,eaidk-610", "rockchip,rk3399"; + + aliases { + mmc0 = &sdio0; + mmc1 = &sdmmc; + mmc2 = &sdhci; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm0 0 25000 0>; + brightness-levels = < + 0 1 2 3 4 5 6 7 + 8 9 10 11 12 13 14 15 + 16 17 18 19 20 21 22 23 + 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 + 48 49 50 51 52 53 54 55 + 56 57 58 59 60 61 62 63 + 64 65 66 67 68 69 70 71 + 72 73 74 75 76 77 78 79 + 80 81 82 83 84 85 86 87 + 88 89 90 91 92 93 94 95 + 96 97 98 99 100 101 102 103 + 104 105 106 107 108 109 110 111 + 112 113 114 115 116 117 118 119 + 120 121 122 123 124 125 126 127 + 128 129 130 131 132 133 134 135 + 136 137 138 139 140 141 142 143 + 144 145 146 147 148 149 150 151 + 152 153 154 155 156 157 158 159 + 160 161 162 163 164 165 166 167 + 168 169 170 171 172 173 174 175 + 176 177 178 179 180 181 182 183 + 184 185 186 187 188 189 190 191 + 192 193 194 195 196 197 198 199 + 200 201 202 203 204 205 206 207 + 208 209 210 211 212 213 214 215 + 216 217 218 219 220 221 222 223 + 224 225 226 227 228 229 230 231 + 232 233 234 235 236 237 238 239 + 240 241 242 243 244 245 246 247 + 248 249 250 251 252 253 254 255>; + default-brightness-level = <200>; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + dc_12v: dc-12v { + compatible = "regulator-fixed"; + regulator-name = "dc_12v"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; + }; + + gpio-keys { + compatible = "gpio-keys"; + autorepeat; + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn>; + + key-power { + debounce-interval = <100>; + gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + label = "GPIO Key Power"; + linux,code = ; + wakeup-source; + }; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&work_led_pin>, <&user_led_pin>, + <&heartbeat_led_pin>, <&wlan_active_led_pin>, + <&bt_active_led_pin>; + + work_led: led-0 { + label = "blue:work"; + default-state = "on"; + gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>; + }; + + user_led: led-1 { + label = "read:user"; + default-state = "off"; + gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>; + }; + + heartbeat_led: led-2 { + label = "green:heartbeat"; + linux,default-trigger = "heartbeat"; + gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>; + }; + + wlan_active_led: led-3 { + label = "yellow:wlan"; + gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "phy0tx"; + default-state = "off"; + }; + + bt_active_led: led-4 { + label = "blue:bt"; + gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "hci0-power"; + default-state = "off"; + }; + }; + + rt5651-sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "realtek,rt5651-codec"; + simple-audio-card,format = "i2s"; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "Mic Jack", "MICBIAS1", + "IN1P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + simple-audio-card,cpu { + sound-dai = <&i2s1>; + }; + simple-audio-card,codec { + sound-dai = <&rt5651>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk808 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + + /* + * On the module itself this is one of these (depending + * on the actual card populated): + * - SDIO_RESET_L_WL_REG_ON + * - PDN (power down when low) + */ + reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; + }; + + /* switched by pmic_sleep */ + vcc1v8_s3: vcca1v8_s3: vcc1v8-s3 { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8_s3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc_1v8>; + }; + + vcc3v3_sys: vcc3v3-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&dc_12v>; + }; + + vcc5v0_sys: vcc5v0-sys { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&dc_12v>; + }; + + /* For USB3.0 Port1/2 */ + vcc5v0_host1: vcc5v0-host1-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host1_en>; + regulator-name = "vcc5v0_host1"; + regulator-always-on; + vin-supply = <&vcc5v0_sys>; + }; + + /* For USB2.0 Port1/2 */ + vcc5v0_host3: vcc5v0-host3-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host3_en>; + regulator-name = "vcc5v0_host3"; + regulator-always-on; + vin-supply = <&vcc5v0_sys>; + }; + + vcc5v0_typec: vcc5v0-typec-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_typec_en>; + regulator-name = "vcc5v0_typec"; + regulator-always-on; + vin-supply = <&vcc3v3_sys>; + }; + + vdd_log: vdd-log { + compatible = "regulator-fixed"; + regulator-name = "vdd_log"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&emmc_phy { + status = "okay"; +}; + +&gmac { + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + clock_in_out = "input"; + phy-supply = <&vcc_lan>; + phy-mode = "rgmii"; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + tx_delay = <0x28>; + rx_delay = <0x11>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + ddc-i2c-bus = <&i2c3>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_cec>; + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk808: pmic@1b { + compatible = "rockchip,rk808"; + reg = <0x1b>; + interrupt-parent = <&gpio1>; + interrupts = <21 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + #clock-cells = <1>; + clock-output-names = "xin32k", "rk808-clkout2"; + + vcc1-supply = <&vcc3v3_sys>; + vcc2-supply = <&vcc3v3_sys>; + vcc3-supply = <&vcc3v3_sys>; + vcc4-supply = <&vcc3v3_sys>; + vcc6-supply = <&vcc3v3_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc3v3_sys>; + vcc10-supply = <&vcc3v3_sys>; + vcc11-supply = <&vcc3v3_sys>; + vcc12-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_3v0>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-name = "vdd_center"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-name = "vdd_cpu_l"; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-name = "vcc_1v8"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc1v8_dvp: LDO_REG1 { + regulator-name = "vcc1v8_dvp"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG2 { + regulator-name = "vcc2v8_dvp"; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_pmu: LDO_REG3 { + regulator-name = "vcc1v8_pmu"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_sdio: LDO_REG4 { + regulator-name = "vcc_sdio"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcca3v0_codec: LDO_REG5 { + regulator-name = "vcca3v0_codec"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-name = "vcc_1v5"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1500000>; + }; + }; + + vcca1v8_codec: LDO_REG7 { + regulator-name = "vcca1v8_codec"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0: LDO_REG8 { + regulator-name = "vcc_3v0"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3000000>; + }; + }; + + vcc3v3_s3: vcc_lan: SWITCH_REG1 { + regulator-name = "vcc3v3_s3"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_s0: SWITCH_REG2 { + regulator-name = "vcc3v3_s0"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + + vdd_cpu_b: regulator@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + regulator-name = "vdd_cpu_b"; + pinctrl-names = "default"; + pinctrl-0 = <&vsel1_pin>; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc3v3_sys>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: regulator@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + fcs,suspend-voltage-selector = <1>; + regulator-name = "vdd_gpu"; + pinctrl-names = "default"; + pinctrl-0 = <&vsel2_pin>; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc3v3_sys>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&i2c1 { + i2c-scl-rising-time-ns = <300>; + i2c-scl-falling-time-ns = <15>; + status = "okay"; + + rt5651: audio-codec@1a { + compatible = "rockchip,rt5651"; + reg = <0x1a>; + clocks = <&cru SCLK_I2S_8CH_OUT>; + clock-names = "mclk"; + hp-det-gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>; + spk-con-gpio = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + }; + +}; + +&i2c3 { + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + status = "okay"; +}; + +&i2c4 { + i2c-scl-rising-time-ns = <600>; + i2c-scl-falling-time-ns = <20>; + status = "okay"; + + fusb0: typec-portc@22 { + compatible = "fcs,fusb302"; + reg = <0x22>; + interrupt-parent = <&gpio1>; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&fusb0_int>; + vbus-supply = <&vcc5v0_typec>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + usbc0_role_sw: endpoint@0 { + remote-endpoint = <&dwc3_0_role_switch>; + }; + }; + }; + + connector { + compatible = "usb-c-connector"; + data-role = "dual"; + label = "USB-C"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + usbc_hs: endpoint { + remote-endpoint = <&u2phy0_typec_hs>; + }; + }; + + port@1 { + reg = <1>; + + usbc_ss: endpoint { + remote-endpoint = <&tcphy0_typec_ss>; + }; + }; + }; + }; + }; +}; + +&i2s1 { + rockchip,playback-channels = <2>; + rockchip,capture-channels = <2>; + status = "okay"; +}; + +&i2s2 { + status = "okay"; +}; + +&io_domains { + status = "okay"; + + audio-supply = <&vcca1v8_codec>; + bt656-supply = <&vcc_3v0>; + gpio1830-supply = <&vcc_3v0>; + sdmmc-supply = <&vcc_sdio>; +}; + +&pmu_io_domains { + status = "okay"; + + pmu1830-supply = <&vcc_3v0>; +}; + +&pinctrl { + buttons { + pwrbtn: pwrbtn { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + bt { + bt_enable_h: bt-enable-h { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_host_wake_l: bt-host-wake-l { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_l: bt-wake-l { + rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + fusb302x { + fusb0_int: fusb0-int { + rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + lcd-panel { + lcd_panel_reset: lcd-panel-reset { + rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + leds { + work_led_pin: work-led-pin { + rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + user_led_pin: user-led-pin { + rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + heartbeat_led_pin: heartbeat-led-pin { + rockchip,pins = <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wlan_active_led_pin: wlan-led-pin { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_active_led_pin: bt-led-pin { + rockchip,pins = <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_pin: vsel1-pin { + rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_pin: vsel2-pin { + rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + rt5651 { + rt5651_hpcon: rt5640-hpcon { + rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb-typec { + vcc5v0_typec_en: vcc5v0_typec_en { + rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + vcc5v0_host3_en: vcc5v0-host3-en { + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + vcc5v0_host1_en: vcc5v0-host1-en { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + wifi { + wifi_host_wake_l: wifi-host-wake-l { + rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pwm0 { + status = "okay"; +}; + +&saradc { + vref-supply = <&vcca1v8_s3>; + status = "okay"; +}; + +&sdio0 { + /* WiFi & BT combo module AMPAK AP6255 */ + #address-cells = <1>; + #size-cells = <0>; + bus-width = <4>; + clock-frequency = <50000000>; + cap-sdio-irq; + cap-sd-highspeed; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; + + brcmf: wifi@1 { + compatible = "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&gpio0>; + interrupts = ; + interrupt-names = "host-wake"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_l>; + }; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; + disable-wp; + max-frequency = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_bus4>; + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + non-removable; + status = "okay"; +}; + +&tcphy0 { + status = "okay"; +}; + +&tcphy0_usb3 { + orientation-switch; + port { + tcphy0_typec_ss: endpoint { + remote-endpoint = <&usbc_ss>; + }; + }; +}; + +&tcphy1 { + status = "okay"; +}; + +&tsadc { + /* tshut mode 0:CRU 1:GPIO */ + rockchip,hw-tshut-mode = <1>; + /* tshut polarity 0:LOW 1:HIGH */ + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host3>; + status = "okay"; + }; + + port { + u2phy0_typec_hs: endpoint { + remote-endpoint = <&usbc_hs>; + }; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host3>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm4345c5"; + clocks = <&rk808 1>; + clock-names = "lpo"; + device-wakeup-gpios = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>; + host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; + shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + max-speed = <1500000>; + pinctrl-names = "default"; + pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>; + vbat-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8>; + }; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + usb-role-switch; + + port { + #address-cells = <1>; + #size-cells = <0>; + dwc3_0_role_switch: endpoint@0 { + reg = <0>; + remote-endpoint = <&usbc0_role_sw>; + }; + }; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + dr_mode = "host"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts index 31ebb4e5fd3307335008ecafd1ca8be1df54a607..0f9cc042d9bf06b3445c2cb125435c823f3b26b4 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-bob.dts @@ -88,3 +88,8 @@ }; }; }; + +&wlan_host_wake_l { + /* Kevin has an external pull up, but Bob does not. */ + rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi index cd074641884bfe51cd12293954eb83000e54d95d..ee6095baba4d3a0679f1ce01106ae81c05920760 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi @@ -244,6 +244,14 @@ &edp { status = "okay"; + /* + * eDP PHY/clk don't sync reliably at anything other than 24 MHz. Only + * set this here, because rk3399-gru.dtsi ensures we can generate this + * off GPLL=600MHz, whereas some other RK3399 boards may not. + */ + assigned-clocks = <&cru PCLK_EDP>; + assigned-clock-rates = <24000000>; + ports { edp_out: port@1 { reg = <1>; @@ -578,6 +586,7 @@ ap_i2c_tp: &i2c5 { }; wlan_host_wake_l: wlan-host-wake-l { + /* Kevin has an external pull up, but Bob does not */ rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; }; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts index 2d721a97479034d5b85aabbf0d9f3de5c0026f74..5d1879033e7c0f063f681f44470b0027bc25b5b9 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts @@ -11,17 +11,29 @@ / { model = "Google Scarlet"; - compatible = "google,scarlet-rev15-sku6", "google,scarlet-rev15", + compatible = "google,scarlet-rev15-sku2", "google,scarlet-rev15-sku4", + "google,scarlet-rev15-sku6", "google,scarlet-rev15", + "google,scarlet-rev14-sku2", "google,scarlet-rev14-sku4", "google,scarlet-rev14-sku6", "google,scarlet-rev14", + "google,scarlet-rev13-sku2", "google,scarlet-rev13-sku4", "google,scarlet-rev13-sku6", "google,scarlet-rev13", + "google,scarlet-rev12-sku2", "google,scarlet-rev12-sku4", "google,scarlet-rev12-sku6", "google,scarlet-rev12", + "google,scarlet-rev11-sku2", "google,scarlet-rev11-sku4", "google,scarlet-rev11-sku6", "google,scarlet-rev11", + "google,scarlet-rev10-sku2", "google,scarlet-rev10-sku4", "google,scarlet-rev10-sku6", "google,scarlet-rev10", + "google,scarlet-rev9-sku2", "google,scarlet-rev9-sku4", "google,scarlet-rev9-sku6", "google,scarlet-rev9", + "google,scarlet-rev8-sku2", "google,scarlet-rev8-sku4", "google,scarlet-rev8-sku6", "google,scarlet-rev8", + "google,scarlet-rev7-sku2", "google,scarlet-rev7-sku4", "google,scarlet-rev7-sku6", "google,scarlet-rev7", + "google,scarlet-rev6-sku2", "google,scarlet-rev6-sku4", "google,scarlet-rev6-sku6", "google,scarlet-rev6", + "google,scarlet-rev5-sku2", "google,scarlet-rev5-sku4", "google,scarlet-rev5-sku6", "google,scarlet-rev5", + "google,scarlet-rev4-sku2", "google,scarlet-rev4-sku4", "google,scarlet-rev4-sku6", "google,scarlet-rev4", "google,scarlet", "google,gru", "rockchip,rk3399"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi index 40d4053fba80dfe94c88537ed28a7d4491f2f3b1..ed3348b558f8e3b161a75aece6f32e249b1ad6d5 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi @@ -768,6 +768,16 @@ camera: &i2c7 { <4 RK_PA0 1 &pcfg_pull_none_6ma>; }; +&i2s0_8ch_bus_bclk_off { + rockchip,pins = + <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none_6ma>, + <3 RK_PD1 1 &pcfg_pull_none_6ma>, + <3 RK_PD2 1 &pcfg_pull_none_6ma>, + <3 RK_PD3 1 &pcfg_pull_none_6ma>, + <3 RK_PD7 1 &pcfg_pull_none_6ma>, + <4 RK_PA0 1 &pcfg_pull_none_6ma>; +}; + /* there is no external pull up, so need to set this pin pull up */ &sdmmc_cd_pin { rockchip,pins = <1 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts new file mode 100644 index 0000000000000000000000000000000000000000..a23d11ca0eb6af3cb7976c83ddbaa760fe4cf3d5 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/dts-v1/; +#include "rk3399-nanopi-r4s.dts" + +/ { + model = "FriendlyElec NanoPi R4S Enterprise Edition"; + compatible = "friendlyarm,nanopi-r4s-enterprise", "rockchip,rk3399"; +}; + +&gmac { + nvmem-cells = <&mac_address>; + nvmem-cell-names = "mac-address"; +}; + +&i2c2 { + eeprom@51 { + compatible = "microchip,24c02", "atmel,24c02"; + reg = <0x51>; + pagesize = <16>; + size = <256>; + #address-cells = <1>; + #size-cells = <1>; + + mac_address: mac-address@fa { + reg = <0xfa 0x06>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts new file mode 100644 index 0000000000000000000000000000000000000000..2e058c3150256fb43bc3c83a6eba2cc5e1f122a2 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2020 Martijn Braam + * Copyright (c) 2021 Kamil Trzciński + */ + +/* + * PinePhone Pro datasheet: + * https://files.pine64.org/doc/PinePhonePro/PinephonePro-Schematic-V1.0-20211127.pdf + */ + +/dts-v1/; +#include +#include "rk3399.dtsi" +#include "rk3399-opp.dtsi" + +/ { + model = "Pine64 PinePhonePro"; + compatible = "pine64,pinephone-pro", "rockchip,rk3399"; + chassis-type = "handset"; + + aliases { + mmc0 = &sdio0; + mmc1 = &sdmmc; + mmc2 = &sdhci; + }; + + chosen { + stdout-path = "serial2:115200n8"; + }; + + gpio-keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&pwrbtn_pin>; + + key-power { + debounce-interval = <20>; + gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; + label = "Power"; + linux,code = ; + wakeup-source; + }; + }; + + vcc_sys: vcc-sys-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-always-on; + regulator-boot-on; + }; + + vcc3v3_sys: vcc3v3-sys-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_sys>; + }; + + vcca1v8_s3: vcc1v8-s3-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcca1v8_s3"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + regulator-always-on; + regulator-boot-on; + }; + + vcc1v8_codec: vcc1v8-codec-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc1v8_codec_en>; + regulator-name = "vcc1v8_codec"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&emmc_phy { + status = "okay"; +}; + +&i2c0 { + clock-frequency = <400000>; + i2c-scl-rising-time-ns = <168>; + i2c-scl-falling-time-ns = <4>; + status = "okay"; + + rk818: pmic@1c { + compatible = "rockchip,rk818"; + reg = <0x1c>; + interrupt-parent = <&gpio1>; + interrupts = ; + #clock-cells = <1>; + clock-output-names = "xin32k", "rk808-clkout2"; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc3v3_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&vcc3v3_sys>; + + regulators { + vdd_cpu_l: DCDC_REG1 { + regulator-name = "vdd_cpu_l"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <875000>; + regulator-max-microvolt = <975000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_center: DCDC_REG2 { + regulator-name = "vdd_center"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <800000>; + regulator-max-microvolt = <1000000>; + regulator-ramp-delay = <6001>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-name = "vcc_ddr"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v8: DCDC_REG4 { + regulator-name = "vcc_1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcca3v0_codec: LDO_REG1 { + regulator-name = "vcca3v0_codec"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + }; + + vcc3v0_touch: LDO_REG2 { + regulator-name = "vcc3v0_touch"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + }; + + vcca1v8_codec: LDO_REG3 { + regulator-name = "vcca1v8_codec"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + rk818_pwr_on: LDO_REG4 { + regulator-name = "rk818_pwr_on"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v0: LDO_REG5 { + regulator-name = "vcc_3v0"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_1v5: LDO_REG6 { + regulator-name = "vcc_1v5"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG7 { + regulator-name = "vcc1v8_dvp"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; + + vcc3v3_s3: LDO_REG8 { + regulator-name = "vcc3v3_s3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG9 { + regulator-name = "vccio_sd"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + }; + + vcc3v3_s0: SWITCH_REG { + regulator-name = "vcc3v3_s0"; + regulator-always-on; + regulator-boot-on; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + }; + }; + + vdd_cpu_b: regulator@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&vsel1_pin>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <875000>; + regulator-max-microvolt = <1150000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: regulator@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + fcs,suspend-voltage-selector = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&vsel2_pin>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <875000>; + regulator-max-microvolt = <975000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&cluster0_opp { + opp04 { + status = "disabled"; + }; + + opp05 { + status = "disabled"; + }; +}; + +&cluster1_opp { + opp06 { + opp-hz = /bits/ 64 <1500000000>; + opp-microvolt = <1100000 1100000 1150000>; + }; + + opp07 { + status = "disabled"; + }; +}; + +&io_domains { + bt656-supply = <&vcc1v8_dvp>; + audio-supply = <&vcca1v8_codec>; + sdmmc-supply = <&vccio_sd>; + gpio1830-supply = <&vcc_3v0>; + status = "okay"; +}; + +&pmu_io_domains { + pmu1830-supply = <&vcc_1v8>; + status = "okay"; +}; + +&pinctrl { + buttons { + pwrbtn_pin: pwrbtn-pin { + rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_pin: vsel1-pin { + rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_pin: vsel2-pin { + rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sound { + vcc1v8_codec_en: vcc1v8-codec-en { + rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; +}; + +&sdmmc { + bus-width = <4>; + cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; + disable-wp; + max-frequency = <150000000>; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; + vmmc-supply = <&vcc3v3_sys>; + vqmmc-supply = <&vccio_sd>; + status = "okay"; +}; + +&sdhci { + bus-width = <8>; + mmc-hs200-1_8v; + non-removable; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; + rockchip,hw-tshut-polarity = <1>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi index b1ac3a89f259cdcab1842f2e161774a554f814a6..aa3e21bd6c8f44ff5f9b249dd43b56316dbc2a60 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-puma.dtsi @@ -62,7 +62,6 @@ vcc5v0_host: vcc5v0-host-regulator { compatible = "regulator-fixed"; gpio = <&gpio4 RK_PA3 GPIO_ACTIVE_LOW>; - enable-active-low; pinctrl-names = "default"; pinctrl-0 = <&vcc5v0_host_en>; regulator-name = "vcc5v0_host"; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi index acb174d3a8c5fffdaa216aa55bea68d1bda815b8..2f4b1b2e3ac7cc98d2995abea3bed15a9e8eb0b8 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi @@ -271,6 +271,8 @@ }; &hdmi { + avdd-0v9-supply = <&vcca0v9_hdmi>; + avdd-1v8-supply = <&vcca1v8_hdmi>; ddc-i2c-bus = <&i2c3>; pinctrl-names = "default"; pinctrl-0 = <&hdmi_cec>; @@ -369,8 +371,8 @@ }; }; - vcc1v8_hdmi: LDO_REG2 { - regulator-name = "vcc1v8_hdmi"; + vcca1v8_hdmi: LDO_REG2 { + regulator-name = "vcca1v8_hdmi"; regulator-always-on; regulator-boot-on; regulator-min-microvolt = <1800000>; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts new file mode 100644 index 0000000000000000000000000000000000000000..f9884902f8745b4dc396f4d5f19bbb5b1c63230c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts @@ -0,0 +1,703 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd + * Copyright (c) 2019 Radxa Limited + * Copyright (c) 2022 Amarula Solutions(India) + */ + +/dts-v1/; +#include +#include "rk3399.dtsi" +#include "rk3399-t-opp.dtsi" + +/ { + model = "Radxa ROCK 4C+"; + compatible = "radxa,rock-4c-plus", "rockchip,rk3399"; + + aliases { + mmc0 = &sdmmc; + mmc1 = &sdhci; + }; + + chosen { + stdout-path = "serial2:1500000n8"; + }; + + clkin_gmac: external-gmac-clock { + compatible = "fixed-clock"; + clock-frequency = <125000000>; + clock-output-names = "clkin_gmac"; + #clock-cells = <0>; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&user_led1 &user_led2>; + + /* USER_LED1 */ + led-0 { + function = LED_FUNCTION_POWER; + color = ; + gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_LOW>; + linux,default-trigger = "default-on"; + }; + + /* USER_LED2 */ + led-1 { + function = LED_FUNCTION_STATUS; + color = ; + gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk809 1>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_enable_h>; + reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>; + }; + + vcc_3v3: vcc-3v3-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc3v3_sys>; + }; + + vcc3v3_phy1: vcc3v3-phy1-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_phy1"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3>; + }; + + vcc5v0_host1: vcc5v0-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio3 RK_PD6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_host_en>; + regulator-name = "vcc5v0_host1"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_host0_s0>; + }; + + vcc5v0_sys: vcc5v0-sys-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc5v0_sys"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + }; + + vcc5v0_typec: vcc5v0-typec-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_typec0_en>; + regulator-name = "vcc5v0_typec"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + }; + + vdd_log: vdd-log-regulator { + compatible = "regulator-fixed"; + regulator-name = "vdd_log"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <950000>; + vin-supply = <&vcc5v0_sys>; + }; +}; + +&cpu_l0 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l1 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l2 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_l3 { + cpu-supply = <&vdd_cpu_l>; +}; + +&cpu_b0 { + cpu-supply = <&vdd_cpu_b>; +}; + +&cpu_b1 { + cpu-supply = <&vdd_cpu_b>; +}; + +&emmc_phy { + status = "okay"; +}; + +&gmac { + assigned-clocks = <&cru SCLK_RMII_SRC>; + assigned-clock-parents = <&clkin_gmac>; + clock_in_out = "input"; + phy-supply = <&vcc3v3_phy1>; + phy-mode = "rgmii"; + pinctrl-names = "default"; + pinctrl-0 = <&rgmii_pins>; + snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 50000>; + tx_delay = <0x2a>; + rx_delay = <0x21>; + status = "okay"; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + avdd-0v9-supply = <&vcc_0v9_s0>; + avdd-1v8-supply = <&vcc_1v8_s0>; + ddc-i2c-bus = <&i2c3>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_cec>; + status = "okay"; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + i2c-scl-falling-time-ns = <30>; + i2c-scl-rising-time-ns = <180>; + clock-frequency = <400000>; + + rk809: pmic@20 { + compatible = "rockchip,rk809"; + reg = <0x20>; + interrupt-parent = <&gpio1>; + interrupts = ; + #clock-cells = <1>; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_int_l>; + rockchip,system-power-controller; + wakeup-source; + + vcc1-supply = <&vcc5v0_sys>; + vcc2-supply = <&vcc5v0_sys>; + vcc3-supply = <&vcc5v0_sys>; + vcc4-supply = <&vcc5v0_sys>; + vcc5-supply = <&vcc_buck5_s3>; + vcc6-supply = <&vcc_buck5_s3>; + vcc7-supply = <&vcc5v0_sys>; + vcc8-supply = <&vcc3v3_sys>; + vcc9-supply = <&vcc5v0_sys>; + + regulators { + vdd_center: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_center"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_cpu_l: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <750000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_cpu_l"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc_ddr"; + regulator-initial-mode = <0x2>; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc3v3_sys: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc3v3_sys"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_buck5_s3: DCDC_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_buck5_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_0v9_s3: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcc_0v9_s3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v8_s3: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_s3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vcc_0v9_s0: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vcc_0v9_s0"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vcc_1v8_s0: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_mipi: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_mipi"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_1v5_s0: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <1500000>; + regulator-name = "vcc_1v5_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_3v0_s0: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3000000>; + regulator-name = "vcc_3v0_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sdio_s0: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_sdio_s0"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_cam: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_cam"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc5v0_host0_s0: SWITCH_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-name = "vcc5v0_host0_s0"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + lcd_3v3: SWITCH_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-name = "lcd_3v3"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + + vdd_cpu_b: regulator@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel1_gpio>; + vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_cpu_b"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdd_gpu: regulator@41 { + compatible = "silergy,syr828"; + reg = <0x41>; + fcs,suspend-voltage-selector = <1>; + regulator-compatible = "fan53555-reg"; + pinctrl-0 = <&vsel2_gpio>; + vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>; + regulator-name = "vdd_gpu"; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <1000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc5v0_sys>; + regulator-initial-mode = <1>; /* 1:force PWM 2:auto */ + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&i2c3 { + i2c-scl-rising-time-ns = <450>; + i2c-scl-falling-time-ns = <15>; + status = "okay"; +}; + +&i2s2 { + status = "okay"; +}; + +&io_domains { + audio-supply = <&vcc_1v8_s0>; + bt656-supply = <&vcc_3v0_s0>; + gpio1830-supply = <&vcc_3v0_s0>; + sdmmc-supply = <&vcc_sdio_s0>; + status = "okay"; +}; + +&pinctrl { + bt { + bt_enable_h: bt-enable-h { + rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_host_wake_l: bt-host-wake-l { + rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_wake_l: bt-wake-l { + rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + leds { + user_led1: user-led1 { + rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + user_led2: user-led2 { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_gpio: vsel1-gpio { + rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_gpio: vsel2-gpio { + rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + + sdmmc { + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = <4 8 1 &pcfg_pull_up_8ma>, + <4 9 1 &pcfg_pull_up_8ma>, + <4 10 1 &pcfg_pull_up_8ma>, + <4 11 1 &pcfg_pull_up_8ma>; + }; + + sdmmc_clk: sdmmc-clk { + rockchip,pins = <4 12 1 &pcfg_pull_none_18ma>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = <4 13 1 &pcfg_pull_up_8ma>; + }; + }; + + usb-typec { + vcc5v0_typec0_en: vcc5v0-typec-en { + rockchip,pins = <1 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + usb2 { + vcc5v0_host_en: vcc5v0-host-en { + rockchip,pins = <3 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + wifi { + wifi_enable_h: wifi-enable-h { + rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + wifi_host_wake_l: wifi-host-wake-l { + rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + pmu1830-supply = <&vcc_3v0_s0>; + status = "okay"; +}; + +&saradc { + status = "okay"; + vref-supply = <&vcc_1v8_s3>; +}; + +&sdhci { + max-frequency = <150000000>; + bus-width = <8>; + mmc-hs400-1_8v; + non-removable; + mmc-hs400-enhanced-strobe; + status = "okay"; +}; + +&sdio0 { + #address-cells = <1>; + #size-cells = <0>; + bus-width = <4>; + clock-frequency = <50000000>; + cap-sdio-irq; + cap-sd-highspeed; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; + sd-uhs-sdr104; + status = "okay"; + + brcmf: wifi@1 { + compatible = "brcm,bcm4329-fmac"; + reg = <1>; + interrupt-parent = <&gpio0>; + interrupts = ; + interrupt-names = "host-wake"; + pinctrl-names = "default"; + pinctrl-0 = <&wifi_host_wake_l>; + }; +}; + +&sdmmc { + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + card-detect-delay = <800>; + disable-wp; + pinctrl-names = "default"; + pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_bus4>; + cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>; + vqmmc-supply = <&vcc_sdio_s0>; + status = "okay"; +}; + +&tcphy0 { + status = "okay"; +}; + +&tcphy1 { + status = "okay"; +}; + +&u2phy0 { + status = "okay"; + + u2phy0_otg: otg-port { + status = "okay"; + }; + + u2phy0_host: host-port { + phy-supply = <&vcc5v0_host1>; + status = "okay"; + }; +}; + +&u2phy1 { + status = "okay"; + + u2phy1_otg: otg-port { + status = "okay"; + }; + + u2phy1_host: host-port { + phy-supply = <&vcc5v0_host1>; + status = "okay"; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>; + status = "okay"; + + bluetooth { + compatible = "brcm,bcm4345c5"; + clocks = <&rk809 1>; + clock-names = "lpo"; + device-wakeup-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>; + host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>; + shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>; + max-speed = <1500000>; + pinctrl-names = "default"; + pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>; + vbat-supply = <&vcc3v3_sys>; + vddio-supply = <&vcc_1v8_s3>; + }; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_host0_ohci { + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usbdrd3_0 { + extcon = <&u2phy0>; + status = "okay"; +}; + +&usbdrd_dwc3_0 { + status = "okay"; + dr_mode = "host"; +}; + +&usbdrd3_1 { + status = "okay"; +}; + +&usbdrd_dwc3_1 { + status = "okay"; + dr_mode = "host"; +}; + +&vopb { + status = "okay"; +}; + +&vopb_mmu { + status = "okay"; +}; + +&vopl { + status = "okay"; +}; + +&vopl_mmu { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi index 401e1ae9d94432c19b5c271fec9329c3a695cd05..645ced6617a65380fedd729b5b8952769e1d4903 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi @@ -6,6 +6,7 @@ /dts-v1/; #include +#include #include #include "rk3399.dtsi" #include "rk3399-opp.dtsi" @@ -27,6 +28,20 @@ #clock-cells = <0>; }; + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&user_led2>; + + /* USER_LED2 */ + led-0 { + function = LED_FUNCTION_STATUS; + color = ; + gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + }; + }; + sdio_pwrseq: sdio-pwrseq { compatible = "mmc-pwrseq-simple"; clocks = <&rk808 1>; @@ -59,32 +74,33 @@ }; }; - vcc12v_dcin: dc-12v { + vbus_typec: vbus-typec-regulator { compatible = "regulator-fixed"; - regulator-name = "vcc12v_dcin"; + enable-active-high; + gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc5v0_typec_en>; + regulator-name = "vbus_typec"; regulator-always-on; - regulator-boot-on; - regulator-min-microvolt = <12000000>; - regulator-max-microvolt = <12000000>; + vin-supply = <&vcc5v0_sys>; }; - vcc5v0_sys: vcc-sys { + vcc12v_dcin: dc-12v { compatible = "regulator-fixed"; - regulator-name = "vcc5v0_sys"; + regulator-name = "vcc12v_dcin"; regulator-always-on; regulator-boot-on; - regulator-min-microvolt = <5000000>; - regulator-max-microvolt = <5000000>; - vin-supply = <&vcc12v_dcin>; + regulator-min-microvolt = <12000000>; + regulator-max-microvolt = <12000000>; }; - vcc_0v9: vcc-0v9 { + vcc3v3_lan: vcc3v3-lan-regulator { compatible = "regulator-fixed"; - regulator-name = "vcc_0v9"; + regulator-name = "vcc3v3_lan"; regulator-always-on; regulator-boot-on; - regulator-min-microvolt = <900000>; - regulator-max-microvolt = <900000>; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; vin-supply = <&vcc3v3_sys>; }; @@ -121,24 +137,24 @@ vin-supply = <&vcc5v0_sys>; }; - vcc5v0_typec: vcc5v0-typec-regulator { + vcc5v0_sys: vcc-sys { compatible = "regulator-fixed"; - enable-active-high; - gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>; - pinctrl-names = "default"; - pinctrl-0 = <&vcc5v0_typec_en>; - regulator-name = "vcc5v0_typec"; + regulator-name = "vcc5v0_sys"; regulator-always-on; - vin-supply = <&vcc5v0_sys>; + regulator-boot-on; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vcc12v_dcin>; }; - vcc_lan: vcc3v3-phy-regulator { + vcc_0v9: vcc-0v9 { compatible = "regulator-fixed"; - regulator-name = "vcc_lan"; + regulator-name = "vcc_0v9"; regulator-always-on; regulator-boot-on; - regulator-min-microvolt = <3300000>; - regulator-max-microvolt = <3300000>; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; }; vdd_log: vdd-log { @@ -185,7 +201,7 @@ assigned-clocks = <&cru SCLK_RMII_SRC>; assigned-clock-parents = <&clkin_gmac>; clock_in_out = "input"; - phy-supply = <&vcc_lan>; + phy-supply = <&vcc3v3_lan>; phy-mode = "rgmii"; pinctrl-names = "default"; pinctrl-0 = <&rgmii_pins>; @@ -203,6 +219,8 @@ }; &hdmi { + avdd-0v9-supply = <&vcca0v9_hdmi>; + avdd-1v8-supply = <&vcca1v8_hdmi>; ddc-i2c-bus = <&i2c3>; pinctrl-names = "default"; pinctrl-0 = <&hdmi_cec>; @@ -290,8 +308,8 @@ }; }; - vcc1v8_codec: LDO_REG1 { - regulator-name = "vcc1v8_codec"; + vcca1v8_codec: LDO_REG1 { + regulator-name = "vcca1v8_codec"; regulator-always-on; regulator-boot-on; regulator-min-microvolt = <1800000>; @@ -301,8 +319,8 @@ }; }; - vcc1v8_hdmi: LDO_REG2 { - regulator-name = "vcc1v8_hdmi"; + vcca1v8_hdmi: LDO_REG2 { + regulator-name = "vcca1v8_hdmi"; regulator-always-on; regulator-boot-on; regulator-min-microvolt = <1800000>; @@ -359,8 +377,8 @@ }; }; - vcc0v9_hdmi: LDO_REG7 { - regulator-name = "vcc0v9_hdmi"; + vcca0v9_hdmi: LDO_REG7 { + regulator-name = "vcca0v9_hdmi"; regulator-always-on; regulator-boot-on; regulator-min-microvolt = <900000>; @@ -499,21 +517,10 @@ }; &io_domains { - status = "okay"; - + audio-supply = <&vcca1v8_codec>; bt656-supply = <&vcc_3v0>; - audio-supply = <&vcc1v8_codec>; - sdmmc-supply = <&vcc_sdio>; gpio1830-supply = <&vcc_3v0>; -}; - -&pmu_io_domains { - status = "okay"; - - pmu1830-supply = <&vcc_3v0>; -}; - -&pcie_phy { + sdmmc-supply = <&vcc_sdio>; status = "okay"; }; @@ -528,6 +535,10 @@ status = "okay"; }; +&pcie_phy { + status = "okay"; +}; + &pinctrl { bt { bt_enable_h: bt-enable-h { @@ -553,12 +564,32 @@ }; }; + leds { + user_led2: user-led2 { + rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + pcie { pcie_pwr_en: pcie-pwr-en { rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; }; }; + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + vsel1_pin: vsel1-pin { + rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; + }; + + vsel2_pin: vsel2-pin { + rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; + }; + }; + sdio0 { sdio0_bus4: sdio0-bus4 { rockchip,pins = <2 RK_PC4 1 &pcfg_pull_up_20ma>, @@ -576,20 +607,6 @@ }; }; - pmic { - pmic_int_l: pmic-int-l { - rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>; - }; - - vsel1_pin: vsel1-pin { - rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>; - }; - - vsel2_pin: vsel2-pin { - rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>; - }; - }; - usb-typec { vcc5v0_typec_en: vcc5v0-typec-en { rockchip,pins = <1 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; @@ -613,6 +630,11 @@ }; }; +&pmu_io_domains { + pmu1830-supply = <&vcc_3v0>; + status = "okay"; +}; + &pwm2 { status = "okay"; }; @@ -623,6 +645,14 @@ vref-supply = <&vcc_1v8>; }; +&sdhci { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + status = "okay"; +}; + &sdio0 { #address-cells = <1>; #size-cells = <0>; @@ -650,14 +680,6 @@ status = "okay"; }; -&sdhci { - bus-width = <8>; - mmc-hs400-1_8v; - mmc-hs400-enhanced-strobe; - non-removable; - status = "okay"; -}; - &spdif { spdif_p0: port { @@ -739,13 +761,13 @@ status = "okay"; }; -&usbdrd_dwc3_0 { +&usbdrd3_1 { status = "okay"; - dr_mode = "host"; }; -&usbdrd3_1 { +&usbdrd_dwc3_0 { status = "okay"; + dr_mode = "host"; }; &usbdrd_dwc3_1 { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..1ababadda9df638fe65b490c1ce0740513746db7 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2016-2017 Fuzhou Rockchip Electronics Co., Ltd + * Copyright (c) 2022 Radxa Limited + */ + +/ { + cluster0_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp00 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <875000 875000 1250000>; + clock-latency-ns = <40000>; + }; + opp01 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <875000 875000 1250000>; + }; + opp02 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <900000 900000 1250000>; + }; + opp03 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <975000 975000 1250000>; + }; + }; + + cluster1_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp00 { + opp-hz = /bits/ 64 <408000000>; + opp-microvolt = <875000 875000 1250000>; + clock-latency-ns = <40000>; + }; + opp01 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <875000 875000 1250000>; + }; + opp02 { + opp-hz = /bits/ 64 <816000000>; + opp-microvolt = <875000 875000 1250000>; + }; + opp03 { + opp-hz = /bits/ 64 <1008000000>; + opp-microvolt = <925000 925000 1250000>; + }; + opp04 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1000000 1000000 1250000>; + }; + opp05 { + opp-hz = /bits/ 64 <1416000000>; + opp-microvolt = <1075000 1075000 1250000>; + }; + opp06 { + opp-hz = /bits/ 64 <1512000000>; + opp-microvolt = <1150000 1150000 1250000>; + }; + }; + + gpu_opp_table: opp-table-2 { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <875000 875000 1150000>; + }; + opp01 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <875000 875000 1150000>; + }; + opp02 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <875000 875000 1150000>; + }; + opp03 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <975000 975000 1150000>; + }; + }; +}; + +&cpu_l0 { + operating-points-v2 = <&cluster0_opp>; +}; + +&cpu_l1 { + operating-points-v2 = <&cluster0_opp>; +}; + +&cpu_l2 { + operating-points-v2 = <&cluster0_opp>; +}; + +&cpu_l3 { + operating-points-v2 = <&cluster0_opp>; +}; + +&cpu_b0 { + operating-points-v2 = <&cluster1_opp>; +}; + +&cpu_b1 { + operating-points-v2 = <&cluster1_opp>; +}; + +&gpu { + operating-points-v2 = <&gpu_opp_table>; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi index 9d5b0e8c9cca72af9630f4e26496b294458bc071..92c2207e686ceee13d8863873350c8d4b080630f 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi @@ -1664,8 +1664,9 @@ dma-names = "tx", "rx"; clock-names = "i2s_clk", "i2s_hclk"; clocks = <&cru SCLK_I2S0_8CH>, <&cru HCLK_I2S0_8CH>; - pinctrl-names = "default"; + pinctrl-names = "bclk_on", "bclk_off"; pinctrl-0 = <&i2s0_8ch_bus>; + pinctrl-1 = <&i2s0_8ch_bus_bclk_off>; power-domains = <&power RK3399_PD_SDIOAUDIO>; #sound-dai-cells = <0>; status = "disabled"; @@ -1701,7 +1702,7 @@ vopl: vop@ff8f0000 { compatible = "rockchip,rk3399-vop-lit"; - reg = <0x0 0xff8f0000 0x0 0x3efc>; + reg = <0x0 0xff8f0000 0x0 0x2000>, <0x0 0xff8f2000 0x0 0x400>; interrupts = ; assigned-clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>; assigned-clock-rates = <400000000>, <100000000>; @@ -1757,7 +1758,7 @@ vopb: vop@ff900000 { compatible = "rockchip,rk3399-vop-big"; - reg = <0x0 0xff900000 0x0 0x3efc>; + reg = <0x0 0xff900000 0x0 0x2000>, <0x0 0xff902000 0x0 0x1000>; interrupts = ; assigned-clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>; assigned-clock-rates = <400000000>, <100000000>; @@ -2409,6 +2410,19 @@ <3 RK_PD7 1 &pcfg_pull_none>, <4 RK_PA0 1 &pcfg_pull_none>; }; + + i2s0_8ch_bus_bclk_off: i2s0-8ch-bus-bclk-off { + rockchip,pins = + <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, + <3 RK_PD1 1 &pcfg_pull_none>, + <3 RK_PD2 1 &pcfg_pull_none>, + <3 RK_PD3 1 &pcfg_pull_none>, + <3 RK_PD4 1 &pcfg_pull_none>, + <3 RK_PD5 1 &pcfg_pull_none>, + <3 RK_PD6 1 &pcfg_pull_none>, + <3 RK_PD7 1 &pcfg_pull_none>, + <4 RK_PA0 1 &pcfg_pull_none>; + }; }; i2s1 { @@ -2420,6 +2434,15 @@ <4 RK_PA6 1 &pcfg_pull_none>, <4 RK_PA7 1 &pcfg_pull_none>; }; + + i2s1_2ch_bus_bclk_off: i2s1-2ch-bus-bclk-off { + rockchip,pins = + <4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>, + <4 RK_PA4 1 &pcfg_pull_none>, + <4 RK_PA5 1 &pcfg_pull_none>, + <4 RK_PA6 1 &pcfg_pull_none>, + <4 RK_PA7 1 &pcfg_pull_none>; + }; }; sdio0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts new file mode 100644 index 0000000000000000000000000000000000000000..7a20e2d6876a0941530aa3e8954a1695c47c03e8 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/dts-v1/; + +#include +#include +#include +#include "rk3566-anbernic-rgxx3.dtsi" + +/ { + model = "RG353P"; + compatible = "anbernic,rg353p", "rockchip,rk3566"; + + aliases { + mmc0 = &sdhci; + mmc1 = &sdmmc0; + mmc2 = &sdmmc1; + mmc3 = &sdmmc2; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + power-supply = <&vcc_sys>; + pwms = <&pwm4 0 25000 0>; + }; +}; + +&gpio_keys_control { + button-a { + gpios = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + label = "EAST"; + linux,code = ; + }; + + button-left { + gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + label = "DPAD-LEFT"; + linux,code = ; + }; + + button-r1 { + gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>; + label = "TR"; + linux,code = ; + }; + + button-r2 { + gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>; + label = "TR2"; + linux,code = ; + }; + + button-right { + gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; + label = "DPAD-RIGHT"; + linux,code = ; + }; + + button-y { + gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>; + label = "WEST"; + linux,code = ; + }; +}; + +&i2c0 { + /* This hardware is physically present but unused. */ + power-monitor@62 { + compatible = "cellwise,cw2015"; + reg = <0x62>; + status = "disabled"; + }; +}; + +&i2c2 { + pintctrl-names = "default"; + pinctrl-0 = <&i2c2m1_xfer>; + status = "okay"; +}; + +&pwm4 { + status = "okay"; +}; + +&sdhci { + pinctrl-0 = <&emmc_bus8>, <&emmc_clk>, <&emmc_cmd>, <&emmc_datastrobe>, <&emmc_rstnout>; + pinctrl-names = "default"; + bus-width = <8>; + mmc-hs200-1_8v; + non-removable; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&vcc_1v8>; + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts new file mode 100644 index 0000000000000000000000000000000000000000..3dc01549a5b45a6d08e48243aa15a1312ddf941a --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/dts-v1/; + +#include +#include +#include +#include "rk3566-anbernic-rgxx3.dtsi" + +/ { + model = "RG503"; + compatible = "anbernic,rg503", "rockchip,rk3566"; + + aliases { + mmc0 = &sdmmc0; + mmc1 = &sdmmc1; + mmc2 = &sdmmc2; + }; + + gpio_spi: spi { + compatible = "spi-gpio"; + pinctrl-names = "default"; + pinctrl-0 = <&spi_pins>; + #address-cells = <1>; + #size-cells = <0>; + + sck-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>; + mosi-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>; + num-chipselects = <0>; + }; +}; + +&gpio_keys_control { + button-a { + gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>; + label = "EAST"; + linux,code = ; + }; + + button-left { + gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>; + label = "DPAD-LEFT"; + linux,code = ; + }; + + button-right { + gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + label = "DPAD-RIGHT"; + linux,code = ; + }; + + button-r1 { + gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>; + label = "TR"; + linux,code = ; + }; + + button-r2 { + gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>; + label = "TR2"; + linux,code = ; + }; + + button-right { + gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>; + label = "DPAD-RIGHT"; + linux,code = ; + }; + + button-y { + gpios = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>; + label = "WEST"; + linux,code = ; + }; +}; + +&pinctrl { + gpio-spi { + spi_pins: spi-pins { + rockchip,pins = + <4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>, + <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>, + <4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..2b455143b86d184af8ce2fdaa4eadb09361168eb --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi @@ -0,0 +1,831 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +/dts-v1/; + +#include +#include +#include +#include +#include +#include "rk3566.dtsi" + +/ { + chosen: chosen { + stdout-path = "serial2:1500000n8"; + }; + + adc-joystick { + compatible = "adc-joystick"; + io-channels = <&adc_mux 0>, + <&adc_mux 1>, + <&adc_mux 2>, + <&adc_mux 3>; + pinctrl-0 = <&joy_mux_en>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + + axis@0 { + reg = <0>; + abs-flat = <32>; + abs-fuzz = <32>; + abs-range = <1023 15>; + linux,code = ; + }; + + axis@1 { + reg = <1>; + abs-flat = <32>; + abs-fuzz = <32>; + abs-range = <15 1023>; + linux,code = ; + }; + + axis@2 { + reg = <2>; + abs-flat = <32>; + abs-fuzz = <32>; + abs-range = <15 1023>; + linux,code = ; + }; + + axis@3 { + reg = <3>; + abs-flat = <32>; + abs-fuzz = <32>; + abs-range = <1023 15>; + linux,code = ; + }; + }; + + adc_keys: adc-keys { + compatible = "adc-keys"; + io-channels = <&saradc 0>; + io-channel-names = "buttons"; + keyup-threshold-microvolt = <1800000>; + poll-interval = <60>; + + /* + * Button is mapped to F key in BSP kernel, but + * according to input guidelines it should be mode. + */ + button-mode { + label = "MODE"; + linux,code = ; + press-threshold-microvolt = <1750>; + }; + }; + + adc_mux: adc-mux { + compatible = "io-channel-mux"; + channels = "left_x", "right_x", "left_y", "right_y"; + #io-channel-cells = <1>; + io-channels = <&saradc 3>; + io-channel-names = "parent"; + mux-controls = <&gpio_mux>; + settle-time-us = <100>; + }; + + gpio_keys_control: gpio-keys-control { + compatible = "gpio-keys"; + pinctrl-0 = <&btn_pins_ctrl>; + pinctrl-names = "default"; + + button-b { + gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>; + label = "SOUTH"; + linux,code = ; + }; + + button-down { + gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>; + label = "DPAD-DOWN"; + linux,code = ; + }; + + button-l1 { + gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>; + label = "TL"; + linux,code = ; + }; + + button-l2 { + gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>; + label = "TL2"; + linux,code = ; + }; + + button-select { + gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>; + label = "SELECT"; + linux,code = ; + }; + + button-start { + gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>; + label = "START"; + linux,code = ; + }; + + button-thumbl { + gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>; + label = "THUMBL"; + linux,code = ; + }; + + button-thumbr { + gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_LOW>; + label = "THUMBR"; + linux,code = ; + }; + + button-up { + gpios = <&gpio3 RK_PA3 GPIO_ACTIVE_LOW>; + label = "DPAD-UP"; + linux,code = ; + }; + + button-x { + gpios = <&gpio3 RK_PC0 GPIO_ACTIVE_LOW>; + label = "NORTH"; + linux,code = ; + }; + }; + + gpio_keys_vol: gpio-keys-vol { + compatible = "gpio-keys"; + autorepeat; + pinctrl-0 = <&btn_pins_vol>; + pinctrl-names = "default"; + + button-vol-down { + gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>; + label = "VOLUMEDOWN"; + linux,code = ; + }; + + button-vol-up { + gpios = <&gpio3 RK_PA7 GPIO_ACTIVE_LOW>; + label = "VOLUMEUP"; + linux,code = ; + }; + }; + + gpio_mux: mux-controller { + compatible = "gpio-mux"; + mux-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_LOW>, + <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>; + #mux-control-cells = <0>; + }; + + hdmi-con { + compatible = "hdmi-connector"; + ddc-i2c-bus = <&i2c5>; + type = "c"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + pinctrl-0 = <&led_pins>; + pinctrl-names = "default"; + + green_led: led-0 { + color = ; + default-state = "on"; + function = LED_FUNCTION_POWER; + gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>; + }; + + amber_led: led-1 { + color = ; + function = LED_FUNCTION_CHARGING; + gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + retain-state-suspended; + }; + + red_led: led-2 { + color = ; + default-state = "off"; + function = LED_FUNCTION_STATUS; + gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>; + }; + }; + + /* Channels reversed for both headphones and speakers. */ + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "anbernic_rk817"; + simple-audio-card,aux-devs = <&spk_amp>; + simple-audio-card,format = "i2s"; + simple-audio-card,hp-det-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>; + simple-audio-card,mclk-fs = <256>; + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphones", + "Speaker", "Internal Speakers"; + simple-audio-card,routing = + "MICL", "Mic Jack", + "Headphones", "HPOL", + "Headphones", "HPOR", + "Internal Speakers", "Speaker Amp OUTL", + "Internal Speakers", "Speaker Amp OUTR", + "Speaker Amp INL", "HPOL", + "Speaker Amp INR", "HPOR"; + simple-audio-card,pin-switches = "Internal Speakers"; + + simple-audio-card,codec { + sound-dai = <&rk817>; + }; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&rk817 1>; + clock-names = "ext_clock"; + pinctrl-0 = <&wifi_enable_h>; + pinctrl-names = "default"; + post-power-on-delay-ms = <200>; + reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_LOW>; + }; + + spk_amp: audio-amplifier { + compatible = "simple-audio-amplifier"; + enable-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&spk_amp_enable_h>; + pinctrl-names = "default"; + sound-name-prefix = "Speaker Amp"; + }; + + vcc3v3_lcd0_n: regulator-vcc3v3-lcd0 { + compatible = "regulator-fixed"; + gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>; + enable-active-high; + pinctrl-0 = <&vcc_lcd_h>; + pinctrl-names = "default"; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_lcd0_n"; + vin-supply = <&vcc_3v3>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_sys: regulator-vcc-sys { + compatible = "regulator-fixed"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3800000>; + regulator-max-microvolt = <3800000>; + regulator-name = "vcc_sys"; + }; + + vcc_wifi: regulator-vcc-wifi { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&vcc_wifi_h>; + pinctrl-names = "default"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc_wifi"; + }; + + vibrator: pwm-vibrator { + compatible = "pwm-vibrator"; + pwm-names = "enable"; + pwms = <&pwm5 0 1000000000 0>; + }; +}; + +&combphy1 { + status = "okay"; +}; + +&cpu0 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu1 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu2 { + cpu-supply = <&vdd_cpu>; +}; + +&cpu3 { + cpu-supply = <&vdd_cpu>; +}; + +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + ddc-i2c-bus = <&i2c5>; + pinctrl-0 = <&hdmitxm0_cec>; + pinctrl-names = "default"; + status = "okay"; +}; + +&hdmi_in { + hdmi_in_vp0: endpoint { + remote-endpoint = <&vp0_out_hdmi>; + }; +}; + +&hdmi_out { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + +&hdmi_sound { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + rk817: pmic@20 { + compatible = "rockchip,rk817"; + reg = <0x20>; + interrupt-parent = <&gpio0>; + interrupts = ; + clock-output-names = "rk808-clkout1", "rk808-clkout2"; + clock-names = "mclk"; + clocks = <&cru I2S1_MCLKOUT_TX>; + assigned-clocks = <&cru I2S1_MCLKOUT_TX>; + assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>; + #clock-cells = <1>; + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_mclk>, <&pmic_int_l>; + wakeup-source; + + vcc1-supply = <&vcc_sys>; + vcc2-supply = <&vcc_sys>; + vcc3-supply = <&vcc_sys>; + vcc4-supply = <&vcc_sys>; + vcc5-supply = <&vcc_sys>; + vcc6-supply = <&vcc_sys>; + vcc7-supply = <&vcc_sys>; + vcc8-supply = <&vcc_sys>; + vcc9-supply = <&dcdc_boost>; + + regulators { + vdd_logic: DCDC_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_logic"; + regulator-state-mem { + regulator-off-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vdd_gpu: DCDC_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <500000>; + regulator-max-microvolt = <1350000>; + regulator-init-microvolt = <900000>; + regulator-ramp-delay = <6001>; + regulator-initial-mode = <0x2>; + regulator-name = "vdd_gpu"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_ddr: DCDC_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_ddr"; + regulator-state-mem { + regulator-on-in-suspend; + }; + }; + + vcc_3v3: DCDC_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-initial-mode = <0x2>; + regulator-name = "vcc_3v3"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcca1v8_pmu: LDO_REG1 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcca1v8_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <1800000>; + }; + }; + + vdda_0v9: LDO_REG2 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda_0v9"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vdda0v9_pmu: LDO_REG3 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-name = "vdda0v9_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <900000>; + }; + }; + + vccio_acodec: LDO_REG4 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_acodec"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vccio_sd: LDO_REG5 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vccio_sd"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc3v3_pmu: LDO_REG6 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc3v3_pmu"; + regulator-state-mem { + regulator-on-in-suspend; + regulator-suspend-microvolt = <3300000>; + }; + }; + + vcc_1v8: LDO_REG7 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-name = "vcc_1v8"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc1v8_dvp: LDO_REG8 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-name = "vcc1v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc2v8_dvp: LDO_REG9 { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <2800000>; + regulator-max-microvolt = <2800000>; + regulator-name = "vcc2v8_dvp"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + dcdc_boost: BOOST { + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <4700000>; + regulator-max-microvolt = <5400000>; + regulator-name = "boost"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + otg_switch: OTG_SWITCH { + regulator-name = "otg_switch"; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + }; + }; + + vdd_cpu: regulator@40 { + compatible = "fcs,fan53555"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1390000>; + regulator-init-microvolt = <900000>; + regulator-name = "vdd_cpu"; + regulator-ramp-delay = <2300>; + vin-supply = <&vcc_sys>; + regulator-state-mem { + regulator-off-in-suspend; + }; + }; +}; + +&i2c1 { + /* Unknown/unused device at 0x3c */ + status = "disabled"; +}; + +&i2c5 { + pinctrl-0 = <&i2c5m1_xfer>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; + pinctrl-names = "default"; + rockchip,trcm-sync-tx-only; + status = "okay"; +}; + +&pinctrl { + audio-amplifier { + spk_amp_enable_h: spk-amp-enable-h { + rockchip,pins = + <4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + gpio-btns { + btn_pins_ctrl: btn-pins-ctrl { + rockchip,pins = + <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + + btn_pins_vol: btn-pins-vol { + rockchip,pins = + <3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>, + <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + gpio-led { + led_pins: led-pins { + rockchip,pins = + <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>, + <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>, + <0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + joy-mux { + joy_mux_en: joy-mux-en { + rockchip,pins = + <0 RK_PB5 RK_FUNC_GPIO &pcfg_output_low>; + }; + }; + + pmic { + pmic_int_l: pmic-int-l { + rockchip,pins = + <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + sdio-pwrseq { + wifi_enable_h: wifi-enable-h { + rockchip,pins = + <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc3v3-lcd { + vcc_lcd_h: vcc-lcd-h { + rockchip,pins = + <0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + vcc-wifi { + vcc_wifi_h: vcc-wifi-h { + rockchip,pins = + <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&pmu_io_domains { + status = "okay"; + pmuio1-supply = <&vcc3v3_pmu>; + pmuio2-supply = <&vcc3v3_pmu>; + vccio1-supply = <&vccio_acodec>; + vccio3-supply = <&vccio_sd>; + vccio4-supply = <&vcc_1v8>; + vccio5-supply = <&vcc_3v3>; + vccio6-supply = <&vcc1v8_dvp>; + vccio7-supply = <&vcc_3v3>; +}; + +&pwm5 { + status = "okay"; +}; + +&saradc { + vref-supply = <&vcc_1v8>; + status = "okay"; +}; + +&sdmmc0 { + bus-width = <4>; + cap-sd-highspeed; + cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>; + disable-wp; + pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; + pinctrl-names = "default"; + sd-uhs-sdr104; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&vccio_sd>; + status = "okay"; +}; + +&sdmmc1 { + bus-width = <4>; + cap-sd-highspeed; + cd-gpios = <&gpio2 RK_PB2 GPIO_ACTIVE_LOW>; + disable-wp; + pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk &sdmmc1_det>; + pinctrl-names = "default"; + sd-uhs-sdr104; + vmmc-supply = <&vcc_3v3>; + vqmmc-supply = <&vcc1v8_dvp>; + status = "okay"; +}; + +&sdmmc2 { + bus-width = <4>; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>; + pinctrl-names = "default"; + vmmc-supply = <&vcc_wifi>; + vqmmc-supply = <&vcca1v8_pmu>; + status = "okay"; +}; + +&tsadc { + rockchip,hw-tshut-mode = <1>; + rockchip,hw-tshut-polarity = <0>; + status = "okay"; +}; + +&uart1 { + pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn &uart1m1_rtsn>; + pinctrl-names = "default"; + uart-has-rtscts; + status = "okay"; + + bluetooth { + compatible = "realtek,rtl8821cs-bt"; + device-wake-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>; + enable-gpios = <&gpio4 3 GPIO_ACTIVE_HIGH>; + host-wake-gpios = <&gpio4 5 GPIO_ACTIVE_HIGH>; + }; +}; + +&uart2 { + status = "okay"; +}; + +/* + * Lack the schematics to verify, but port works as a peripheral + * (and not a host or OTG port). + */ +&usb_host0_xhci { + dr_mode = "peripheral"; + phys = <&usb2phy0_otg>; + phy-names = "usb2-phy"; + status = "okay"; +}; + +&usb_host1_ehci { + status = "okay"; +}; + +&usb_host1_ohci { + status = "okay"; +}; + +&usb_host1_xhci { + phy-names = "usb2-phy", "usb3-phy"; + phys = <&usb2phy1_host>, <&combphy1 PHY_TYPE_USB3>; + status = "okay"; +}; + +&usb2phy0 { + status = "okay"; +}; + +&usb2phy0_otg { + status = "okay"; +}; + +&usb2phy1 { + status = "okay"; +}; + +&usb2phy1_host { + status = "okay"; +}; + +&vop { + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; + +&vp0 { + vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 { + reg = ; + remote-endpoint = <&hdmi_in_vp0>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts index d943559b157cec08d62dbc2cc4748f033131e5a6..a05460b924153ec3df73fb9e311096b9ed3e4f87 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-a.dts @@ -189,7 +189,6 @@ vcc3v3_sd: vcc3v3_sd { compatible = "regulator-fixed"; - enable-active-low; gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>; pinctrl-names = "default"; pinctrl-0 = <&vcc_sd_h>; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts index 02d5f5a8ca036171f715114212eb9df5b7042d0c..77b179cd20e729d05e16c8af7fb2e596db572018 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts @@ -4,6 +4,7 @@ #include #include +#include #include "rk3566.dtsi" / { @@ -28,6 +29,17 @@ #clock-cells = <0>; }; + hdmi-con { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + leds { compatible = "gpio-leds"; @@ -42,6 +54,21 @@ }; }; + sound { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "Analog RK809"; + simple-audio-card,mclk-fs = <256>; + + simple-audio-card,cpu { + sound-dai = <&i2s1_8ch>; + }; + + simple-audio-card,codec { + sound-dai = <&rk809>; + }; + }; + sdio_pwrseq: sdio-pwrseq { status = "okay"; compatible = "mmc-pwrseq-simple"; @@ -54,6 +81,18 @@ power-off-delay-us = <5000000>; }; + vcc3v3_pcie_p: vcc3v3-pcie-p-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_enable_h>; + regulator-name = "vcc3v3_pcie_p"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc_3v3>; + }; + vcc5v0_in: vcc5v0-in-regulator { compatible = "regulator-fixed"; regulator-name = "vcc5v0_in"; @@ -113,6 +152,10 @@ status = "okay"; }; +&combphy2 { + status = "okay"; +}; + &cpu0 { cpu-supply = <&vdd_cpu>; }; @@ -152,6 +195,33 @@ status = "okay"; }; +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + avdd-0v9-supply = <&vdda0v9_image>; + avdd-1v8-supply = <&vcca1v8_image>; + status = "okay"; +}; + +&hdmi_in { + hdmi_in_vp0: endpoint { + remote-endpoint = <&vp0_out_hdmi>; + }; +}; + +&hdmi_out { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + +&hdmi_sound { + status = "okay"; +}; + &i2c0 { status = "okay"; @@ -177,11 +247,16 @@ reg = <0x20>; interrupt-parent = <&gpio0>; interrupts = ; + assigned-clocks = <&cru I2S1_MCLKOUT_TX>; + assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>; + clock-names = "mclk"; + clocks = <&cru I2S1_MCLKOUT_TX>; clock-output-names = "rk808-clkout1", "rk808-clkout2"; pinctrl-names = "default"; - pinctrl-0 = <&pmic_int>; + pinctrl-0 = <&pmic_int>, <&i2s1m0_mclk>; rockchip,system-power-controller; + #sound-dai-cells = <0>; wakeup-source; #clock-cells = <1>; @@ -420,6 +495,20 @@ status = "disabled"; }; +&i2s0_8ch { + status = "okay"; +}; + +&i2s1_8ch { + pinctrl-names = "default"; + pinctrl-0 = <&i2s1m0_sclktx + &i2s1m0_lrcktx + &i2s1m0_sdi0 + &i2s1m0_sdo0>; + rockchip,trcm-sync-tx-only; + status = "okay"; +}; + &mdio1 { rgmii_phy1: ethernet-phy@1 { compatible = "ethernet-phy-ieee802.3-c22"; @@ -427,6 +516,14 @@ }; }; +&pcie2x1 { + pinctrl-names = "default"; + pinctrl-0 = <&pcie_reset_h>; + reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc3v3_pcie_p>; + status = "okay"; +}; + &pinctrl { bt { bt_enable_h: bt-enable-h { @@ -448,6 +545,16 @@ }; }; + pcie { + pcie_enable_h: pcie-enable-h { + rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + pcie_reset_h: pcie-reset-h { + rockchip,pins = <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + pmic { pmic_int: pmic_int { rockchip,pins = @@ -506,7 +613,7 @@ disable-wp; pinctrl-names = "default"; pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>; - sd-uhs-sdr104; + sd-uhs-sdr50; vmmc-supply = <&vcc3v3_sd>; vqmmc-supply = <&vccio_sd>; status = "okay"; @@ -613,3 +720,20 @@ &usb_host0_ohci { status = "okay"; }; + +&vop { + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; + +&vp0 { + vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 { + reg = ; + remote-endpoint = <&hdmi_in_vp0>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts index 57759b66d44d0a2a958891be31cc6b5daeee228e..dba648c2f57e87294e8c7dc32b831dc3b13c02f7 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts @@ -4,6 +4,7 @@ #include #include +#include #include "rk3566.dtsi" / { @@ -27,6 +28,17 @@ #clock-cells = <0>; }; + hdmi-con { + compatible = "hdmi-connector"; + type = "a"; + + port { + hdmi_con_in: endpoint { + remote-endpoint = <&hdmi_out_con>; + }; + }; + }; + leds { compatible = "gpio-leds"; @@ -149,6 +161,29 @@ status = "okay"; }; +&gpu { + mali-supply = <&vdd_gpu>; + status = "okay"; +}; + +&hdmi { + avdd-0v9-supply = <&vdda0v9_image>; + avdd-1v8-supply = <&vcca1v8_image>; + status = "okay"; +}; + +&hdmi_in { + hdmi_in_vp0: endpoint { + remote-endpoint = <&vp0_out_hdmi>; + }; +}; + +&hdmi_out { + hdmi_out_con: endpoint { + remote-endpoint = <&hdmi_con_in>; + }; +}; + &i2c0 { status = "okay"; @@ -577,3 +612,20 @@ &usb_host0_ohci { status = "okay"; }; + +&vop { + assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; + assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; + status = "okay"; +}; + +&vop_mmu { + status = "okay"; +}; + +&vp0 { + vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 { + reg = ; + remote-endpoint = <&hdmi_in_vp0>; + }; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts index 5e34bd0b214d624c0ec13c0c9683a25403d68965..c282f6e7996078da5c6dbcb1ad45a4e31f8b9b15 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts @@ -46,7 +46,7 @@ }; }; - dc_12v: dc-12v { + dc_12v: dc-12v-regulator { compatible = "regulator-fixed"; regulator-name = "dc_12v"; regulator-always-on; @@ -66,7 +66,7 @@ }; }; - vcc3v3_sys: vcc3v3-sys { + vcc3v3_sys: vcc3v3-sys-regulator { compatible = "regulator-fixed"; regulator-name = "vcc3v3_sys"; regulator-always-on; @@ -76,7 +76,7 @@ vin-supply = <&dc_12v>; }; - vcc5v0_sys: vcc5v0-sys { + vcc5v0_sys: vcc5v0-sys-regulator { compatible = "regulator-fixed"; regulator-name = "vcc5v0_sys"; regulator-always-on; @@ -86,7 +86,67 @@ vin-supply = <&dc_12v>; }; - vcc5v0_usb: vcc5v0_usb { + pcie30_avdd0v9: pcie30-avdd0v9-regulator { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd0v9"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + vin-supply = <&vcc3v3_sys>; + }; + + pcie30_avdd1v8: pcie30-avdd1v8-regulator { + compatible = "regulator-fixed"; + regulator-name = "pcie30_avdd1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + vin-supply = <&vcc3v3_sys>; + }; + + /* pi6c pcie clock generator feeds both ports */ + vcc3v3_pi6c_05: vcc3v3-pi6c-05-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_pcie"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-high; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + startup-delay-us = <200000>; + vin-supply = <&vcc5v0_sys>; + }; + + /* actually fed by vcc3v3_sys, dependent on pi6c clock generator */ + vcc3v3_minipcie: vcc3v3-minipcie-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_minipcie"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-high; + gpio = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&minipcie_enable_h>; + startup-delay-us = <50000>; + vin-supply = <&vcc3v3_pi6c_05>; + }; + + /* actually fed by vcc3v3_sys, dependent on pi6c clock generator */ + vcc3v3_ngff: vcc3v3-ngff-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc3v3_ngff"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + enable-active-high; + gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&ngffpcie_enable_h>; + startup-delay-us = <50000>; + vin-supply = <&vcc3v3_pi6c_05>; + }; + + vcc5v0_usb: vcc5v0-usb-regulator { compatible = "regulator-fixed"; regulator-name = "vcc5v0_usb"; regulator-always-on; @@ -96,7 +156,7 @@ vin-supply = <&dc_12v>; }; - vcc5v0_usb_host: vcc5v0-usb-host { + vcc5v0_usb_host: vcc5v0-usb-host-regulator { compatible = "regulator-fixed"; enable-active-high; gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>; @@ -108,7 +168,7 @@ vin-supply = <&vcc5v0_usb>; }; - vcc5v0_usb_otg: vcc5v0-usb-otg { + vcc5v0_usb_otg: vcc5v0-usb-otg-regulator { compatible = "regulator-fixed"; enable-active-high; gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>; @@ -513,6 +573,32 @@ }; }; +&pcie30phy { + data-lanes = <1 2>; + phy-supply = <&vcc3v3_pi6c_05>; + status = "okay"; +}; + +&pcie3x1 { + /* M.2 slot */ + num-lanes = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&ngffpcie_reset_h>; + reset-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc3v3_ngff>; + status = "okay"; +}; + +&pcie3x2 { + /* mPCIe slot */ + num-lanes = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&minipcie_reset_h>; + reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc3v3_minipcie>; + status = "okay"; +}; + &pinctrl { leds { blue_led_pin: blue-led-pin { @@ -529,6 +615,24 @@ }; }; + pcie { + minipcie_enable_h: minipcie-enable-h { + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>; + }; + + ngffpcie_enable_h: ngffpcie-enable-h { + rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>; + }; + + minipcie_reset_h: minipcie-reset-h { + rockchip,pins = <2 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>; + }; + + ngffpcie_reset_h: ngffpcie-reset-h { + rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>; + }; + }; + pmic { pmic_int: pmic_int { rockchip,pins = @@ -678,7 +782,7 @@ }; &usb_host0_xhci { - extcon = <&usb2phy0>; + dr_mode = "host"; status = "okay"; }; @@ -708,6 +812,19 @@ status = "okay"; }; +&usb2phy1 { + /* USB for PCIe/M2 */ + status = "okay"; +}; + +&usb2phy1_host { + status = "okay"; +}; + +&usb2phy1_otg { + status = "okay"; +}; + &vop { assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>; assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts index 6ff89ff95ad1c9ff00ca3855af6e7be3fc8e8bd9..674792567fa6e288c94169406cd69b0e2262d5b2 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-evb1-v10.dts @@ -656,7 +656,7 @@ }; &usb2phy0_otg { - vbus-supply = <&vcc5v0_usb_otg>; + phy-supply = <&vcc5v0_usb_otg>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts b/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts index 6b5093a1a6cf5ca242309eb635b7adac49a4185e..fb87a168fe9676e2cafd79dec8787a0301bba7a3 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts +++ b/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts @@ -67,6 +67,18 @@ regulator-boot-on; }; + vcc3v3_pcie: vcc3v3-pcie-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&pcie_enable_h>; + regulator-name = "vcc3v3_pcie"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc5v0_sys>; + }; + vcc3v3_sys: vcc3v3-sys { compatible = "regulator-fixed"; regulator-name = "vcc3v3_sys"; @@ -131,6 +143,38 @@ regulator-max-microvolt = <5000000>; vin-supply = <&vcc5v0_usb>; }; + + vcc_cam: vcc-cam { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio1 RK_PB1 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_cam_en>; + regulator-name = "vcc_cam"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc3v3_sys>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; + + vcc_mipi: vcc-mipi { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio3 RK_PC0 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&vcc_mipi_en>; + regulator-name = "vcc_mipi"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc3v3_sys>; + + regulator-state-mem { + regulator-off-in-suspend; + }; + }; }; &combphy0 { @@ -141,6 +185,10 @@ status = "okay"; }; +&combphy2 { + status = "okay"; +}; + &cpu0 { cpu-supply = <&vdd_cpu>; }; @@ -440,6 +488,35 @@ }; }; +&i2c3 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c3m1_xfer>; + status = "disabled"; +}; + +&i2c4 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c4m1_xfer>; + status = "disabled"; +}; + +&i2c5 { + status = "okay"; + + hym8563: rtc@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + interrupt-parent = <&gpio0>; + interrupts = ; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "rtcic_32kout"; + pinctrl-names = "default"; + pinctrl-0 = <&hym8563_int>; + wakeup-source; + }; +}; + &i2s0_8ch { status = "okay"; }; @@ -461,19 +538,55 @@ }; }; +&pcie2x1 { + pinctrl-names = "default"; + pinctrl-0 = <&pcie_reset_h>; + reset-gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_HIGH>; + vpcie3v3-supply = <&vcc3v3_pcie>; + status = "okay"; +}; + &pinctrl { + cam { + vcc_cam_en: vcc_cam_en { + rockchip,pins = <1 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + display { + vcc_mipi_en: vcc_mipi_en { + rockchip,pins = <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + ethernet { eth_phy_rst: eth_phy_rst { rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>; }; }; + hym8563 { + hym8563_int: hym8563-int { + rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + leds { led_user_en: led_user_en { rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>; }; }; + pcie { + pcie_enable_h: pcie-enable-h { + rockchip,pins = <0 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + pcie_reset_h: pcie-reset-h { + rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + pmic { pmic_int: pmic_int { rockchip,pins = @@ -581,7 +694,7 @@ }; &usb2phy0_otg { - vbus-supply = <&vcc5v0_usb_otg>; + phy-supply = <&vcc5v0_usb_otg>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi index 2bdf8c7e9765b54e8dd294696cc39d91bfd677f2..ba67b58f05b790c55d63427aaa87215441b6f082 100644 --- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi @@ -42,6 +42,128 @@ reg = <0x0 0xfe190200 0x0 0x20>; }; + pcie30_phy_grf: syscon@fdcb8000 { + compatible = "rockchip,rk3568-pcie3-phy-grf", "syscon"; + reg = <0x0 0xfdcb8000 0x0 0x10000>; + }; + + pcie30phy: phy@fe8c0000 { + compatible = "rockchip,rk3568-pcie3-phy"; + reg = <0x0 0xfe8c0000 0x0 0x20000>; + #phy-cells = <0>; + clocks = <&pmucru CLK_PCIE30PHY_REF_M>, <&pmucru CLK_PCIE30PHY_REF_N>, + <&cru PCLK_PCIE30PHY>; + clock-names = "refclk_m", "refclk_n", "pclk"; + resets = <&cru SRST_PCIE30PHY>; + reset-names = "phy"; + rockchip,phy-grf = <&pcie30_phy_grf>; + status = "disabled"; + }; + + pcie3x1: pcie@fe270000 { + compatible = "rockchip,rk3568-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x0 0xf>; + clocks = <&cru ACLK_PCIE30X1_MST>, <&cru ACLK_PCIE30X1_SLV>, + <&cru ACLK_PCIE30X1_DBI>, <&cru PCLK_PCIE30X1>, + <&cru CLK_PCIE30X1_AUX_NDFT>; + clock-names = "aclk_mst", "aclk_slv", + "aclk_dbi", "pclk", "aux"; + device_type = "pci"; + interrupts = , + , + , + , + ; + interrupt-names = "sys", "pmc", "msg", "legacy", "err"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3x1_intc 0>, + <0 0 0 2 &pcie3x1_intc 1>, + <0 0 0 3 &pcie3x1_intc 2>, + <0 0 0 4 &pcie3x1_intc 3>; + linux,pci-domain = <1>; + num-ib-windows = <6>; + num-ob-windows = <2>; + max-link-speed = <3>; + msi-map = <0x0 &gic 0x1000 0x1000>; + num-lanes = <1>; + phys = <&pcie30phy>; + phy-names = "pcie-phy"; + power-domains = <&power RK3568_PD_PIPE>; + reg = <0x3 0xc0400000 0x0 0x00400000>, + <0x0 0xfe270000 0x0 0x00010000>, + <0x3 0x7f000000 0x0 0x01000000>; + ranges = <0x01000000 0x0 0x3ef00000 0x3 0x7ef00000 0x0 0x00100000>, + <0x02000000 0x0 0x00000000 0x3 0x40000000 0x0 0x3ef00000>; + reg-names = "dbi", "apb", "config"; + resets = <&cru SRST_PCIE30X1_POWERUP>; + reset-names = "pipe"; + /* bifurcation; lane1 when using 1+1 */ + status = "disabled"; + + pcie3x1_intc: legacy-interrupt-controller { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + }; + + pcie3x2: pcie@fe280000 { + compatible = "rockchip,rk3568-pcie"; + #address-cells = <3>; + #size-cells = <2>; + bus-range = <0x0 0xf>; + clocks = <&cru ACLK_PCIE30X2_MST>, <&cru ACLK_PCIE30X2_SLV>, + <&cru ACLK_PCIE30X2_DBI>, <&cru PCLK_PCIE30X2>, + <&cru CLK_PCIE30X2_AUX_NDFT>; + clock-names = "aclk_mst", "aclk_slv", + "aclk_dbi", "pclk", "aux"; + device_type = "pci"; + interrupts = , + , + , + , + ; + interrupt-names = "sys", "pmc", "msg", "legacy", "err"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &pcie3x2_intc 0>, + <0 0 0 2 &pcie3x2_intc 1>, + <0 0 0 3 &pcie3x2_intc 2>, + <0 0 0 4 &pcie3x2_intc 3>; + linux,pci-domain = <2>; + num-ib-windows = <6>; + num-ob-windows = <2>; + max-link-speed = <3>; + msi-map = <0x0 &gic 0x2000 0x1000>; + num-lanes = <2>; + phys = <&pcie30phy>; + phy-names = "pcie-phy"; + power-domains = <&power RK3568_PD_PIPE>; + reg = <0x3 0xc0800000 0x0 0x00400000>, + <0x0 0xfe280000 0x0 0x00010000>, + <0x3 0xbf000000 0x0 0x01000000>; + ranges = <0x01000000 0x0 0x3ef00000 0x3 0xbef00000 0x0 0x00100000>, + <0x02000000 0x0 0x00000000 0x3 0x80000000 0x0 0x3ef00000>; + reg-names = "dbi", "apb", "config"; + resets = <&cru SRST_PCIE30X2_POWERUP>; + reset-names = "pipe"; + /* bifurcation; lane0 when using 1+1 */ + status = "disabled"; + + pcie3x2_intc: legacy-interrupt-controller { + interrupt-controller; + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + }; + gmac0: ethernet@fe2a0000 { compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a"; reg = <0x0 0xfe2a0000 0x0 0x10000>; diff --git a/arch/arm64/boot/dts/rockchip/rk356x.dtsi b/arch/arm64/boot/dts/rockchip/rk356x.dtsi index 319981c3e9f72a12c73a4784c122a24afe1b5263..164708f1eb674772219356a27435e6ddd211c415 100644 --- a/arch/arm64/boot/dts/rockchip/rk356x.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk356x.dtsi @@ -592,6 +592,46 @@ status = "disabled"; }; + vpu: video-codec@fdea0400 { + compatible = "rockchip,rk3568-vpu"; + reg = <0x0 0xfdea0000 0x0 0x800>; + interrupts = ; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + clock-names = "aclk", "hclk"; + iommus = <&vdpu_mmu>; + power-domains = <&power RK3568_PD_VPU>; + }; + + vdpu_mmu: iommu@fdea0800 { + compatible = "rockchip,rk3568-iommu"; + reg = <0x0 0xfdea0800 0x0 0x40>; + interrupts = ; + clock-names = "aclk", "iface"; + clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>; + power-domains = <&power RK3568_PD_VPU>; + #iommu-cells = <0>; + }; + + vepu: video-codec@fdee0000 { + compatible = "rockchip,rk3568-vepu"; + reg = <0x0 0xfdee0000 0x0 0x800>; + interrupts = ; + clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; + clock-names = "aclk", "hclk"; + iommus = <&vepu_mmu>; + power-domains = <&power RK3568_PD_RGA>; + }; + + vepu_mmu: iommu@fdee0800 { + compatible = "rockchip,rk3568-iommu"; + reg = <0x0 0xfdee0800 0x0 0x40>; + interrupts = ; + clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>; + clock-names = "aclk", "iface"; + power-domains = <&power RK3568_PD_RGA>; + #iommu-cells = <0>; + }; + sdmmc2: mmc@fe000000 { compatible = "rockchip,rk3568-dw-mshc", "rockchip,rk3288-dw-mshc"; reg = <0x0 0xfe000000 0x0 0x4000>; @@ -699,6 +739,62 @@ status = "disabled"; }; + dsi0: dsi@fe060000 { + compatible = "rockchip,rk3568-mipi-dsi", "snps,dw-mipi-dsi"; + reg = <0x00 0xfe060000 0x00 0x10000>; + interrupts = ; + clock-names = "pclk", "hclk"; + clocks = <&cru PCLK_DSITX_0>, <&cru HCLK_VO>; + phy-names = "dphy"; + phys = <&dsi_dphy0>; + power-domains = <&power RK3568_PD_VO>; + reset-names = "apb"; + resets = <&cru SRST_P_DSITX_0>; + rockchip,grf = <&grf>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dsi0_in: port@0 { + reg = <0>; + }; + + dsi0_out: port@1 { + reg = <1>; + }; + }; + }; + + dsi1: dsi@fe070000 { + compatible = "rockchip,rk3568-mipi-dsi", "snps,dw-mipi-dsi"; + reg = <0x0 0xfe070000 0x0 0x10000>; + interrupts = ; + clock-names = "pclk", "hclk"; + clocks = <&cru PCLK_DSITX_1>, <&cru HCLK_VO>; + phy-names = "dphy"; + phys = <&dsi_dphy1>; + power-domains = <&power RK3568_PD_VO>; + reset-names = "apb"; + resets = <&cru SRST_P_DSITX_1>; + rockchip,grf = <&grf>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + dsi1_in: port@0 { + reg = <0>; + }; + + dsi1_out: port@1 { + reg = <1>; + }; + }; + }; + hdmi: hdmi@fe0a0000 { compatible = "rockchip,rk3568-dw-hdmi"; reg = <0x0 0xfe0a0000 0x0 0x20000>; @@ -1594,6 +1690,42 @@ status = "disabled"; }; + csi_dphy: phy@fe870000 { + compatible = "rockchip,rk3568-csi-dphy"; + reg = <0x0 0xfe870000 0x0 0x10000>; + clocks = <&cru PCLK_MIPICSIPHY>; + clock-names = "pclk"; + #phy-cells = <0>; + resets = <&cru SRST_P_MIPICSIPHY>; + reset-names = "apb"; + rockchip,grf = <&grf>; + status = "disabled"; + }; + + dsi_dphy0: mipi-dphy@fe850000 { + compatible = "rockchip,rk3568-dsi-dphy"; + reg = <0x0 0xfe850000 0x0 0x10000>; + clock-names = "ref", "pclk"; + clocks = <&pmucru CLK_MIPIDSIPHY0_REF>, <&cru PCLK_MIPIDSIPHY0>; + #phy-cells = <0>; + power-domains = <&power RK3568_PD_VO>; + reset-names = "apb"; + resets = <&cru SRST_P_MIPIDSIPHY0>; + status = "disabled"; + }; + + dsi_dphy1: mipi-dphy@fe860000 { + compatible = "rockchip,rk3568-dsi-dphy"; + reg = <0x0 0xfe860000 0x0 0x10000>; + clock-names = "ref", "pclk"; + clocks = <&pmucru CLK_MIPIDSIPHY1_REF>, <&cru PCLK_MIPIDSIPHY1>; + #phy-cells = <0>; + power-domains = <&power RK3568_PD_VO>; + reset-names = "apb"; + resets = <&cru SRST_P_MIPIDSIPHY1>; + status = "disabled"; + }; + usb2phy0: usb2phy@fe8a0000 { compatible = "rockchip,rk3568-usb2phy"; reg = <0x0 0xfe8a0000 0x0 0x10000>; diff --git a/arch/arm64/boot/dts/socionext/Makefile b/arch/arm64/boot/dts/socionext/Makefile index dda3da33614b8f416e1d7b89c29d2851eecda957..33989a9643ac881c67fbc02194dad679d3c6e71f 100644 --- a/arch/arm64/boot/dts/socionext/Makefile +++ b/arch/arm64/boot/dts/socionext/Makefile @@ -5,4 +5,6 @@ dtb-$(CONFIG_ARCH_UNIPHIER) += \ uniphier-ld20-akebi96.dtb \ uniphier-ld20-global.dtb \ uniphier-ld20-ref.dtb \ - uniphier-pxs3-ref.dtb + uniphier-pxs3-ref.dtb \ + uniphier-pxs3-ref-gadget0.dtb \ + uniphier-pxs3-ref-gadget1.dtb diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts index 617d2b1e9b1ef9c7178a4de66a03c53e4f2ce9b8..414aeb99e68f9b70f0ab5971626be0667b1f84db 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts +++ b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts @@ -39,11 +39,11 @@ }; ðsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -51,7 +51,7 @@ }; &gpio { - xirq0 { + xirq0-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi index 15dcfc2598542b284f987a3006484673ff0b90ba..1c76b4375b2ea473a9708124d32ba8be61bd504e 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi @@ -7,6 +7,7 @@ #include #include +#include / { compatible = "socionext,uniphier-ld11"; @@ -35,6 +36,7 @@ reg = <0 0x000>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; }; @@ -44,8 +46,13 @@ reg = <0 0x001>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; }; + + l2: l2-cache { + compatible = "cache"; + }; }; cluster0_opp: opp-table { @@ -102,10 +109,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 4>, - <1 14 4>, - <1 11 4>, - <1 10 4>; + interrupts = , + , + , + ; }; reserved-memory { @@ -131,7 +138,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -144,7 +151,7 @@ reg = <0x54006100 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 216 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; clocks = <&peri_clk 12>; @@ -155,7 +162,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -166,7 +173,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -177,7 +184,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -188,7 +195,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -223,7 +230,7 @@ audio@56000000 { compatible = "socionext,uniphier-ld11-aio"; reg = <0x56000000 0x80000>; - interrupts = <0 144 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_aout1>, <&pinctrl_aoutiec1>; @@ -323,7 +330,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -337,7 +344,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -350,7 +357,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; clocks = <&peri_clk 6>; resets = <&peri_rst 6>; clock-frequency = <400000>; @@ -362,7 +369,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -376,7 +383,7 @@ reg = <0x58784000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 45 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c4>; clocks = <&peri_clk 8>; @@ -389,7 +396,7 @@ reg = <0x58785000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 25 4>; + interrupts = ; clocks = <&peri_clk 9>; resets = <&peri_rst 9>; clock-frequency = <400000>; @@ -440,7 +447,7 @@ emmc: mmc@5a000000 { compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc"; reg = <0x5a000000 0x400>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&sys_clk 4>; @@ -460,7 +467,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a800100 0x100>; - interrupts = <0 243 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>, @@ -476,7 +483,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a810100 0x100>; - interrupts = <0 244 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>, @@ -492,7 +499,7 @@ compatible = "socionext,uniphier-ehci", "generic-ehci"; status = "disabled"; reg = <0x5a820100 0x100>; - interrupts = <0 245 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb2>; clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>, @@ -530,7 +537,7 @@ compatible = "socionext,uniphier-ld11-pinctrl"; }; - usb-phy { + usb-controller { compatible = "socionext,uniphier-ld11-usb2-phy"; #address-cells = <1>; #size-cells = <0>; @@ -573,7 +580,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -591,7 +598,7 @@ <0x5fe40000 0x80000>; /* GICR */ interrupt-controller; #interrupt-cells = <3>; - interrupts = <1 9 4>; + interrupts = ; }; sysctrl@61840000 { @@ -618,7 +625,7 @@ compatible = "socionext,uniphier-ld11-ave4"; status = "disabled"; reg = <0x65000000 0x8500>; - interrupts = <0 66 4>; + interrupts = ; clock-names = "ether"; clocks = <&sys_clk 6>; reset-names = "ether"; @@ -640,7 +647,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts b/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts index aa159a11292c3e31c06a00b1e3baee37063b5bb7..fba454adae7d451ad98843f807426b63ed6ae7f0 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts +++ b/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts @@ -110,7 +110,7 @@ spi-max-frequency = <12500000>; interrupt-parent = <&gpio>; interrupt-names = "udc"; - interrupts = <0 2>; + interrupts = <0 IRQ_TYPE_EDGE_FALLING>; }; }; @@ -168,12 +168,12 @@ &gpio { /* IRQs for Max3421 */ - xirq0 { + xirq0-hog { gpio-hog; gpios = ; input; }; - xirq10 { + xirq10-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts index 39ee279a1eb97a2aceae0781d9a791b7a643e58f..a5f2083f8b75ff9efdb951693c2fef97da418791 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts +++ b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts @@ -39,11 +39,11 @@ }; ðsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <0 8>; + interrupts = <0 IRQ_TYPE_LEVEL_LOW>; }; &serial0 { @@ -51,7 +51,7 @@ }; &gpio { - xirq0 { + xirq0-hog { gpio-hog; gpios = ; input; diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi index 8f2c1c1e2c64e0b96de408caa129849bab4cc445..9308458f96111c4ccb0b27a306150a122926b6c2 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi @@ -7,6 +7,7 @@ #include #include +#include #include / { @@ -45,6 +46,7 @@ reg = <0 0x000>; clocks = <&sys_clk 32>; enable-method = "psci"; + next-level-cache = <&a72_l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; @@ -55,6 +57,7 @@ reg = <0 0x001>; clocks = <&sys_clk 32>; enable-method = "psci"; + next-level-cache = <&a72_l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; @@ -65,6 +68,7 @@ reg = <0 0x100>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&a53_l2>; operating-points-v2 = <&cluster1_opp>; #cooling-cells = <2>; }; @@ -75,12 +79,21 @@ reg = <0 0x101>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&a53_l2>; operating-points-v2 = <&cluster1_opp>; #cooling-cells = <2>; }; + + a72_l2: l2-cache0 { + compatible = "cache"; + }; + + a53_l2: l2-cache1 { + compatible = "cache"; + }; }; - cluster0_opp: opp-table0 { + cluster0_opp: opp-table-0 { compatible = "operating-points-v2"; opp-shared; @@ -118,7 +131,7 @@ }; }; - cluster1_opp: opp-table1 { + cluster1_opp: opp-table-1 { compatible = "operating-points-v2"; opp-shared; @@ -176,10 +189,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 4>, - <1 14 4>, - <1 11 4>, - <1 10 4>; + interrupts = , + , + , + ; }; thermal-zones { @@ -236,7 +249,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -249,7 +262,7 @@ reg = <0x54006100 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 216 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; clocks = <&peri_clk 12>; @@ -262,7 +275,7 @@ reg = <0x54006200 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 229 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi2>; clocks = <&peri_clk 13>; @@ -275,7 +288,7 @@ reg = <0x54006300 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 230 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi3>; clocks = <&peri_clk 14>; @@ -286,7 +299,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -297,7 +310,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -308,7 +321,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -319,7 +332,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -348,7 +361,7 @@ audio@56000000 { compatible = "socionext,uniphier-ld20-aio"; reg = <0x56000000 0x80000>; - interrupts = <0 144 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_aout1>, <&pinctrl_aoutiec1>; @@ -448,7 +461,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -462,7 +475,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -475,7 +488,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; clocks = <&peri_clk 6>; resets = <&peri_rst 6>; clock-frequency = <400000>; @@ -487,7 +500,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -501,7 +514,7 @@ reg = <0x58784000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 45 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c4>; clocks = <&peri_clk 8>; @@ -514,7 +527,7 @@ reg = <0x58785000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 25 4>; + interrupts = ; clocks = <&peri_clk 9>; resets = <&peri_rst 9>; clock-frequency = <400000>; @@ -570,7 +583,7 @@ emmc: mmc@5a000000 { compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc"; reg = <0x5a000000 0x400>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&sys_clk 4>; @@ -590,7 +603,7 @@ compatible = "socionext,uniphier-sd-v3.1.1"; status = "disabled"; reg = <0x5a400000 0x800>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_sd>; clocks = <&sd_clk 0>; @@ -675,7 +688,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -693,7 +706,7 @@ <0x5fe80000 0x80000>; /* GICR */ interrupt-controller; #interrupt-cells = <3>; - interrupts = <1 9 4>; + interrupts = ; }; sysctrl@61840000 { @@ -715,9 +728,9 @@ compatible = "socionext,uniphier-wdt"; }; - pvtctl: pvtctl { + pvtctl: thermal-sensor { compatible = "socionext,uniphier-ld20-thermal"; - interrupts = <0 3 4>; + interrupts = ; #thermal-sensor-cells = <0>; socionext,tmod-calibration = <0x0f22 0x68ee>; }; @@ -727,7 +740,7 @@ compatible = "socionext,uniphier-ld20-ave4"; status = "disabled"; reg = <0x65000000 0x8500>; - interrupts = <0 66 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ether_rgmii>; clock-names = "ether"; @@ -749,7 +762,7 @@ status = "disabled"; reg = <0x65a00000 0xcd00>; interrupt-names = "host"; - interrupts = <0 134 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb1>, <&pinctrl_usb2>, <&pinctrl_usb3>; @@ -762,7 +775,7 @@ dr_mode = "host"; }; - usb-glue@65b00000 { + usb-controller@65b00000 { compatible = "socionext,uniphier-ld20-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -895,7 +908,7 @@ }; pcie: pcie@66000000 { - compatible = "socionext,uniphier-pcie", "snps,dw-pcie"; + compatible = "socionext,uniphier-pcie"; status = "disabled"; reg-names = "dbi", "link", "config"; reg = <0x66000000 0x1000>, <0x66010000 0x10000>, @@ -915,7 +928,8 @@ <0x82000000 0 0x20000000 0x20000000 0 0x0ffe0000>; #interrupt-cells = <1>; interrupt-names = "dma", "msi"; - interrupts = <0 224 4>, <0 225 4>; + interrupts = , + ; interrupt-map-mask = <0 0 0 7>; interrupt-map = <0 0 0 1 &pcie_intc 0>, /* INTA */ <0 0 0 2 &pcie_intc 1>, /* INTB */ @@ -928,7 +942,7 @@ interrupt-controller; #interrupt-cells = <1>; interrupt-parent = <&gic>; - interrupts = <0 226 4>; + interrupts = ; }; }; @@ -950,7 +964,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts new file mode 100644 index 0000000000000000000000000000000000000000..7069f51bc120e9d0bcd5eb28a61ac93981a4952f --- /dev/null +++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT +// +// Device Tree Source for UniPhier PXs3 Reference Board (for USB-Device #0) +// +// Copyright (C) 2021 Socionext Inc. +// Author: Kunihiko Hayashi + +/dts-v1/; +#include "uniphier-pxs3-ref.dts" + +/ { + model = "UniPhier PXs3 Reference Board (USB-Device #0)"; +}; + +/* I2C3 pinctrl is shared with USB*VBUSIN */ +&i2c3 { + status = "disabled"; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + pinctrl-0 = <&pinctrl_usb0_device>; + snps,dis_enblslpm_quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,usb2_gadget_lpm_disable; + phy-names = "usb2-phy", "usb3-phy"; + phys = <&usb0_hsphy0>, <&usb0_ssphy0>; +}; + +&usb0_hsphy0 { + /delete-property/ vbus-supply; +}; + +&usb0_ssphy0 { + /delete-property/ vbus-supply; +}; + +/delete-node/ &usb0_hsphy1; +/delete-node/ &usb0_ssphy1; diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts new file mode 100644 index 0000000000000000000000000000000000000000..a3cfa8113ffb2029d65dfd22c5a6030456e4c16a --- /dev/null +++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR MIT +// +// Device Tree Source for UniPhier PXs3 Reference Board (for USB-Device #1) +// +// Copyright (C) 2021 Socionext Inc. +// Author: Kunihiko Hayashi + +/dts-v1/; +#include "uniphier-pxs3-ref.dts" + +/ { + model = "UniPhier PXs3 Reference Board (USB-Device #1)"; +}; + +/* I2C3 pinctrl is shared with USB*VBUSIN */ +&i2c3 { + status = "disabled"; +}; + +&usb1 { + status = "okay"; + dr_mode = "peripheral"; + pinctrl-0 = <&pinctrl_usb1_device>; + snps,dis_enblslpm_quirk; + snps,dis_u2_susphy_quirk; + snps,dis_u3_susphy_quirk; + snps,usb2_gadget_lpm_disable; + phy-names = "usb2-phy", "usb3-phy"; + phys = <&usb1_hsphy0>, <&usb1_ssphy0>; +}; + +&usb1_hsphy0 { + /delete-property/ vbus-supply; +}; + +&usb1_ssphy0 { + /delete-property/ vbus-supply; +}; + +/delete-node/ &usb1_hsphy1; diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts index 086040306fb36546290b552b1ca7b8c29e192036..1ced6190ab2b255715d2108693314819865c50b4 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts +++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts @@ -40,11 +40,11 @@ }; ðsc { - interrupts = <4 8>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; }; &serialsc { - interrupts = <4 8>; + interrupts = <4 IRQ_TYPE_LEVEL_LOW>; }; &spi0 { @@ -68,7 +68,7 @@ }; &gpio { - xirq4 { + xirq4-hog { gpio-hog; gpios = ; input; @@ -137,6 +137,14 @@ }; }; +&ahci0 { + status = "okay"; +}; + +&ahci1 { + status = "okay"; +}; + &pinctrl_ether_rgmii { tx { pins = "RGMII0_TXCLK", "RGMII0_TXD0", "RGMII0_TXD1", diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi index ba75adedbf79b1b37c026edda7ea555e9f2c94eb..b0c29510a7dadcc01d1e89e68f208482b8dad6d7 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi @@ -7,6 +7,7 @@ #include #include +#include #include / { @@ -42,6 +43,7 @@ reg = <0 0x000>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; @@ -52,6 +54,7 @@ reg = <0 0x001>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; @@ -62,6 +65,7 @@ reg = <0 0x002>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; @@ -72,9 +76,14 @@ reg = <0 0x003>; clocks = <&sys_clk 33>; enable-method = "psci"; + next-level-cache = <&l2>; operating-points-v2 = <&cluster0_opp>; #cooling-cells = <2>; }; + + l2: l2-cache { + compatible = "cache"; + }; }; cluster0_opp: opp-table { @@ -135,10 +144,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 4>, - <1 14 4>, - <1 11 4>, - <1 10 4>; + interrupts = , + , + , + ; }; thermal-zones { @@ -195,7 +204,7 @@ reg = <0x54006000 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 39 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi0>; clocks = <&peri_clk 11>; @@ -208,7 +217,7 @@ reg = <0x54006100 0x100>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 216 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_spi1>; clocks = <&peri_clk 12>; @@ -219,7 +228,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006800 0x40>; - interrupts = <0 33 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart0>; clocks = <&peri_clk 0>; @@ -230,7 +239,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006900 0x40>; - interrupts = <0 35 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart1>; clocks = <&peri_clk 1>; @@ -241,7 +250,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006a00 0x40>; - interrupts = <0 37 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart2>; clocks = <&peri_clk 2>; @@ -252,7 +261,7 @@ compatible = "socionext,uniphier-uart"; status = "disabled"; reg = <0x54006b00 0x40>; - interrupts = <0 177 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_uart3>; clocks = <&peri_clk 3>; @@ -284,7 +293,7 @@ reg = <0x58780000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 41 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c0>; clocks = <&peri_clk 4>; @@ -298,7 +307,7 @@ reg = <0x58781000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 42 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c1>; clocks = <&peri_clk 5>; @@ -312,7 +321,7 @@ reg = <0x58782000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 43 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c2>; clocks = <&peri_clk 6>; @@ -326,7 +335,7 @@ reg = <0x58783000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 44 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_i2c3>; clocks = <&peri_clk 7>; @@ -340,7 +349,7 @@ reg = <0x58786000 0x80>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 26 4>; + interrupts = ; clocks = <&peri_clk 10>; resets = <&peri_rst 10>; clock-frequency = <400000>; @@ -396,7 +405,7 @@ emmc: mmc@5a000000 { compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc"; reg = <0x5a000000 0x400>; - interrupts = <0 78 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_emmc>; clocks = <&sys_clk 4>; @@ -416,7 +425,7 @@ compatible = "socionext,uniphier-sd-v3.1.1"; status = "disabled"; reg = <0x5a400000 0x800>; - interrupts = <0 76 4>; + interrupts = ; pinctrl-names = "default", "uhs"; pinctrl-0 = <&pinctrl_sd>; pinctrl-1 = <&pinctrl_sd_uhs>; @@ -505,7 +514,7 @@ xdmac: dma-controller@5fc10000 { compatible = "socionext,uniphier-xdmac"; reg = <0x5fc10000 0x5300>; - interrupts = <0 188 4>; + interrupts = ; dma-channels = <16>; #dma-cells = <2>; }; @@ -523,7 +532,7 @@ <0x5fe80000 0x80000>; /* GICR */ interrupt-controller; #interrupt-cells = <3>; - interrupts = <1 9 4>; + interrupts = ; }; sysctrl@61840000 { @@ -545,9 +554,9 @@ compatible = "socionext,uniphier-wdt"; }; - pvtctl: pvtctl { + pvtctl: thermal-sensor { compatible = "socionext,uniphier-pxs3-thermal"; - interrupts = <0 3 4>; + interrupts = ; #thermal-sensor-cells = <0>; socionext,tmod-calibration = <0x0f22 0x68ee>; }; @@ -557,7 +566,7 @@ compatible = "socionext,uniphier-pxs3-ave4"; status = "disabled"; reg = <0x65000000 0x8500>; - interrupts = <0 66 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ether_rgmii>; clock-names = "ether"; @@ -578,7 +587,7 @@ compatible = "socionext,uniphier-pxs3-ave4"; status = "disabled"; reg = <0x65200000 0x8500>; - interrupts = <0 67 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_ether1_rgmii>; clock-names = "ether"; @@ -595,12 +604,92 @@ }; }; + ahci0: sata@65600000 { + compatible = "socionext,uniphier-pxs3-ahci", + "generic-ahci"; + status = "disabled"; + reg = <0x65600000 0x10000>; + interrupts = ; + clocks = <&sys_clk 28>; + resets = <&sys_rst 28>, <&ahci0_rst 0>; + ports-implemented = <1>; + phys = <&ahci0_phy>; + }; + + sata-controller@65700000 { + compatible = "socionext,uniphier-pxs3-ahci-glue", + "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x65700000 0x100>; + + ahci0_rst: reset-controller@0 { + compatible = "socionext,uniphier-pxs3-ahci-reset"; + reg = <0x0 0x4>; + clock-names = "link"; + clocks = <&sys_clk 28>; + reset-names = "link"; + resets = <&sys_rst 28>; + #reset-cells = <1>; + }; + + ahci0_phy: sata-phy@10 { + compatible = "socionext,uniphier-pxs3-ahci-phy"; + reg = <0x10 0x10>; + clock-names = "link", "phy"; + clocks = <&sys_clk 28>, <&sys_clk 30>; + reset-names = "link", "phy"; + resets = <&sys_rst 28>, <&sys_rst 30>; + #phy-cells = <0>; + }; + }; + + ahci1: sata@65800000 { + compatible = "socionext,uniphier-pxs3-ahci", + "generic-ahci"; + status = "disabled"; + reg = <0x65800000 0x10000>; + interrupts = ; + clocks = <&sys_clk 29>; + resets = <&sys_rst 29>, <&ahci1_rst 0>; + ports-implemented = <1>; + phys = <&ahci1_phy>; + }; + + sata-controller@65900000 { + compatible = "socionext,uniphier-pxs3-ahci-glue", + "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0x65900000 0x100>; + + ahci1_rst: reset-controller@0 { + compatible = "socionext,uniphier-pxs3-ahci-reset"; + reg = <0x0 0x4>; + clock-names = "link"; + clocks = <&sys_clk 29>; + reset-names = "link"; + resets = <&sys_rst 29>; + #reset-cells = <1>; + }; + + ahci1_phy: sata-phy@10 { + compatible = "socionext,uniphier-pxs3-ahci-phy"; + reg = <0x10 0x10>; + clock-names = "link", "phy"; + clocks = <&sys_clk 29>, <&sys_clk 30>; + reset-names = "link", "phy"; + resets = <&sys_rst 29>, <&sys_rst 30>; + #phy-cells = <0>; + }; + }; + usb0: usb@65a00000 { compatible = "socionext,uniphier-dwc3", "snps,dwc3"; status = "disabled"; reg = <0x65a00000 0xcd00>; interrupt-names = "dwc_usb3"; - interrupts = <0 134 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb2>; clock-names = "ref", "bus_early", "suspend"; @@ -611,7 +700,7 @@ dr_mode = "host"; }; - usb-glue@65b00000 { + usb-controller@65b00000 { compatible = "socionext,uniphier-pxs3-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -702,7 +791,7 @@ status = "disabled"; reg = <0x65c00000 0xcd00>; interrupt-names = "dwc_usb3"; - interrupts = <0 137 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb3>; clock-names = "ref", "bus_early", "suspend"; @@ -713,7 +802,7 @@ dr_mode = "host"; }; - usb-glue@65d00000 { + usb-controller@65d00000 { compatible = "socionext,uniphier-pxs3-dwc3-glue", "simple-mfd"; #address-cells = <1>; @@ -792,7 +881,7 @@ }; pcie: pcie@66000000 { - compatible = "socionext,uniphier-pcie", "snps,dw-pcie"; + compatible = "socionext,uniphier-pcie"; status = "disabled"; reg-names = "dbi", "link", "config"; reg = <0x66000000 0x1000>, <0x66010000 0x10000>, @@ -812,7 +901,8 @@ <0x82000000 0 0x20000000 0x20000000 0 0x0ffe0000>; #interrupt-cells = <1>; interrupt-names = "dma", "msi"; - interrupts = <0 224 4>, <0 225 4>; + interrupts = , + ; interrupt-map-mask = <0 0 0 7>; interrupt-map = <0 0 0 1 &pcie_intc 0>, /* INTA */ <0 0 0 2 &pcie_intc 1>, /* INTB */ @@ -825,7 +915,7 @@ interrupt-controller; #interrupt-cells = <1>; interrupt-parent = <&gic>; - interrupts = <0 226 4>; + interrupts = ; }; }; @@ -847,7 +937,7 @@ reg = <0x68000000 0x20>, <0x68100000 0x1000>; #address-cells = <1>; #size-cells = <0>; - interrupts = <0 65 4>; + interrupts = ; pinctrl-names = "default"; pinctrl-0 = <&pinctrl_nand>; clock-names = "nand", "nand_x", "ecc"; diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile index 02e5d80344d00cf8bdb4ad9bc29c0fad2925b324..4555a5be2257788a1c2f1d61078ea3b54b72e0f2 100644 --- a/arch/arm64/boot/dts/ti/Makefile +++ b/arch/arm64/boot/dts/ti/Makefile @@ -23,3 +23,5 @@ dtb-$(CONFIG_ARCH_K3) += k3-am642-evm.dtb dtb-$(CONFIG_ARCH_K3) += k3-am642-sk.dtb dtb-$(CONFIG_ARCH_K3) += k3-am625-sk.dtb + +dtb-$(CONFIG_ARCH_K3) += k3-am62a7-sk.dtb diff --git a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi index 12ab7548dc7721a560d64b097e126adf012304f5..03660476364f37623927dbcf33a3bee25425f82d 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi @@ -54,6 +54,12 @@ reg = <0x4044 0x8>; #phy-cells = <1>; }; + + epwm_tbclk: clock@4130 { + compatible = "ti,am62-epwm-tbclk", "syscon"; + reg = <0x4130 0x4>; + #clock-cells = <1>; + }; }; dmss: bus@48000000 { @@ -584,4 +590,31 @@ interrupt-names = "int0", "int1"; bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; }; + + epwm0: pwm@23000000 { + compatible = "ti,am64-epwm", "ti,am3352-ehrpwm"; + #pwm-cells = <3>; + reg = <0x00 0x23000000 0x00 0x100>; + power-domains = <&k3_pds 86 TI_SCI_PD_EXCLUSIVE>; + clocks = <&epwm_tbclk 0>, <&k3_clks 86 0>; + clock-names = "tbclk", "fck"; + }; + + epwm1: pwm@23010000 { + compatible = "ti,am64-epwm", "ti,am3352-ehrpwm"; + #pwm-cells = <3>; + reg = <0x00 0x23010000 0x00 0x100>; + power-domains = <&k3_pds 87 TI_SCI_PD_EXCLUSIVE>; + clocks = <&epwm_tbclk 1>, <&k3_clks 87 0>; + clock-names = "tbclk", "fck"; + }; + + epwm2: pwm@23020000 { + compatible = "ti,am64-epwm", "ti,am3352-ehrpwm"; + #pwm-cells = <3>; + reg = <0x00 0x23020000 0x00 0x100>; + power-domains = <&k3_pds 88 TI_SCI_PD_EXCLUSIVE>; + clocks = <&epwm_tbclk 2>, <&k3_clks 88 0>; + clock-names = "tbclk", "fck"; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am625-sk.dts b/arch/arm64/boot/dts/ti/k3-am625-sk.dts index 9b4dbae9d4aadfdc7384c541b2ca81561dd8eef9..93a5f0817efc4539825c00152229460c893cee85 100644 --- a/arch/arm64/boot/dts/ti/k3-am625-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am625-sk.dts @@ -502,3 +502,15 @@ &main_mcan0 { status = "disabled"; }; + +&epwm0 { + status = "disabled"; +}; + +&epwm1 { + status = "disabled"; +}; + +&epwm2 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..bc4b50bcd1773dc1d0de9cc6e68fec13506d0032 --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for AM62A SoC Family Main Domain peripherals + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +&cbass_main { + oc_sram: sram@70000000 { + compatible = "mmio-sram"; + reg = <0x00 0x70000000 0x00 0x10000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x00 0x70000000 0x10000>; + }; + + gic500: interrupt-controller@1800000 { + compatible = "arm,gic-v3"; + reg = <0x00 0x01800000 0x00 0x10000>, /* GICD */ + <0x00 0x01880000 0x00 0xc0000>, /* GICR */ + <0x00 0x01880000 0x00 0xc0000>, /* GICR */ + <0x01 0x00000000 0x00 0x2000>, /* GICC */ + <0x01 0x00010000 0x00 0x1000>, /* GICH */ + <0x01 0x00020000 0x00 0x2000>; /* GICV */ + #address-cells = <2>; + #size-cells = <2>; + ranges; + #interrupt-cells = <3>; + interrupt-controller; + /* + * vcpumntirq: + * virtual CPU interface maintenance interrupt + */ + interrupts = ; + + gic_its: msi-controller@1820000 { + compatible = "arm,gic-v3-its"; + reg = <0x00 0x01820000 0x00 0x10000>; + socionext,synquacer-pre-its = <0x1000000 0x400000>; + msi-controller; + #msi-cells = <1>; + }; + }; + + main_conf: syscon@100000 { + compatible = "ti,j721e-system-controller", "syscon", "simple-mfd"; + reg = <0x00 0x00100000 0x00 0x20000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x00 0x00 0x00100000 0x20000>; + }; + + dmss: bus@48000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + dma-ranges; + ranges = <0x00 0x48000000 0x00 0x48000000 0x00 0x06000000>; + + ti,sci-dev-id = <25>; + + secure_proxy_main: mailbox@4d000000 { + compatible = "ti,am654-secure-proxy"; + reg = <0x00 0x4d000000 0x00 0x80000>, + <0x00 0x4a600000 0x00 0x80000>, + <0x00 0x4a400000 0x00 0x80000>; + reg-names = "target_data", "rt", "scfg"; + #mbox-cells = <1>; + interrupt-names = "rx_012"; + interrupts = ; + }; + }; + + dmsc: system-controller@44043000 { + compatible = "ti,k2g-sci"; + reg = <0x00 0x44043000 0x00 0xfe0>; + reg-names = "debug_messages"; + ti,host-id = <12>; + mbox-names = "rx", "tx"; + mboxes= <&secure_proxy_main 12>, + <&secure_proxy_main 13>; + + k3_pds: power-controller { + compatible = "ti,sci-pm-domain"; + #power-domain-cells = <2>; + }; + + k3_clks: clock-controller { + compatible = "ti,k2g-sci-clk"; + #clock-cells = <2>; + }; + + k3_reset: reset-controller { + compatible = "ti,sci-reset"; + #reset-cells = <2>; + }; + }; + + main_pmx0: pinctrl@f4000 { + compatible = "pinctrl-single"; + reg = <0x00 0xf4000 0x00 0x2ac>; + #pinctrl-cells = <1>; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0xffffffff>; + }; + + main_uart0: serial@2800000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02800000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 146 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 146 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart1: serial@2810000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02810000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 152 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 152 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart2: serial@2820000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02820000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 153 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 153 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart3: serial@2830000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02830000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 154 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 154 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart4: serial@2840000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02840000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 155 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 155 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart5: serial@2850000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02850000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 156 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 156 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_uart6: serial@2860000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x02860000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 158 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 158 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + main_i2c0: i2c@20000000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x20000000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 102 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 102 2>; + clock-names = "fck"; + status = "disabled"; + }; + + main_i2c1: i2c@20010000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x20010000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 103 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 103 2>; + clock-names = "fck"; + status = "disabled"; + }; + + main_i2c2: i2c@20020000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x20020000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 104 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 104 2>; + clock-names = "fck"; + status = "disabled"; + }; + + main_i2c3: i2c@20030000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x20030000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 105 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 105 2>; + clock-names = "fck"; + status = "disabled"; + }; + + main_gpio_intr: interrupt-controller@a00000 { + compatible = "ti,sci-intr"; + reg = <0x00 0x00a00000 0x00 0x800>; + ti,intr-trigger-type = <1>; + interrupt-controller; + interrupt-parent = <&gic500>; + #interrupt-cells = <1>; + ti,sci = <&dmsc>; + ti,sci-dev-id = <3>; + ti,interrupt-ranges = <0 32 16>; + status = "disabled"; + }; + + main_gpio0: gpio@600000 { + compatible = "ti,am64-gpio", "ti,keystone-gpio"; + reg = <0x00 0x00600000 0x0 0x100>; + gpio-controller; + #gpio-cells = <2>; + interrupt-parent = <&main_gpio_intr>; + interrupts = <190>, <191>, <192>, + <193>, <194>, <195>; + interrupt-controller; + #interrupt-cells = <2>; + ti,ngpio = <87>; + ti,davinci-gpio-unbanked = <0>; + power-domains = <&k3_pds 77 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 77 0>; + clock-names = "gpio"; + status = "disabled"; + }; + + main_gpio1: gpio@601000 { + compatible = "ti,am64-gpio", "ti,keystone-gpio"; + reg = <0x00 0x00601000 0x0 0x100>; + gpio-controller; + #gpio-cells = <2>; + interrupt-parent = <&main_gpio_intr>; + interrupts = <180>, <181>, <182>, + <183>, <184>, <185>; + interrupt-controller; + #interrupt-cells = <2>; + ti,ngpio = <88>; + ti,davinci-gpio-unbanked = <0>; + power-domains = <&k3_pds 78 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 78 0>; + clock-names = "gpio"; + status = "disabled"; + }; + + sdhci1: mmc@fa00000 { + compatible = "ti,am62-sdhci"; + reg = <0x00 0xfa00000 0x00 0x260>, <0x00 0xfa08000 0x00 0x134>; + interrupts = ; + power-domains = <&k3_pds 58 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 58 5>, <&k3_clks 58 6>; + clock-names = "clk_ahb", "clk_xin"; + ti,trm-icp = <0x2>; + ti,otap-del-sel-legacy = <0x0>; + ti,otap-del-sel-sd-hs = <0x0>; + ti,otap-del-sel-sdr12 = <0xf>; + ti,otap-del-sel-sdr25 = <0xf>; + ti,otap-del-sel-sdr50 = <0xc>; + ti,otap-del-sel-sdr104 = <0x6>; + ti,otap-del-sel-ddr50 = <0x9>; + ti,itap-del-sel-legacy = <0x0>; + ti,itap-del-sel-sd-hs = <0x0>; + ti,itap-del-sel-sdr12 = <0x0>; + ti,itap-del-sel-sdr25 = <0x0>; + ti,clkbuf-sel = <0x7>; + bus-width = <4>; + no-1-8-v; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..6d1e501b94abf83521758ee42b10306cd0c9b933 --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for AM625 SoC Family MCU Domain peripherals + * + * Copyright (C) 2020-2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +&cbass_mcu { + mcu_pmx0: pinctrl@4084000 { + compatible = "pinctrl-single"; + reg = <0x00 0x04084000 0x00 0x88>; + #pinctrl-cells = <1>; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0xffffffff>; + status = "disabled"; + }; + + mcu_uart0: serial@4a00000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x04a00000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 149 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 149 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + mcu_i2c0: i2c@4900000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x04900000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 106 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 106 2>; + clock-names = "fck"; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..99afac40e8d4b33b0e2e94519f07e6ad746317eb --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for AM62A SoC Family Wakeup Domain peripherals + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +&cbass_wakeup { + wkup_conf: syscon@43000000 { + compatible = "ti,j721e-system-controller", "syscon", "simple-mfd"; + reg = <0x00 0x43000000 0x00 0x20000>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x00 0x00 0x43000000 0x20000>; + + chipid: chipid@14 { + compatible = "ti,am654-chipid"; + reg = <0x14 0x4>; + }; + }; + + wkup_uart0: serial@2b300000 { + compatible = "ti,am64-uart", "ti,am654-uart"; + reg = <0x00 0x2b300000 0x00 0x100>; + interrupts = ; + power-domains = <&k3_pds 114 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 114 0>; + clock-names = "fclk"; + status = "disabled"; + }; + + wkup_i2c0: i2c@2b200000 { + compatible = "ti,am64-i2c", "ti,omap4-i2c"; + reg = <0x00 0x02b200000 0x00 0x100>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&k3_pds 107 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 107 4>; + clock-names = "fck"; + status = "disabled"; + }; + + wkup_rtc0: rtc@2b1f0000 { + compatible = "ti,am62-rtc"; + reg = <0x00 0x2b1f0000 0x00 0x100>; + interrupts = ; + clocks = <&k3_clks 117 6> , <&k3_clks 117 0>; + clock-names = "vbus", "osc32k"; + power-domains = <&k3_pds 117 TI_SCI_PD_EXCLUSIVE>; + wakeup-source; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am62a.dtsi b/arch/arm64/boot/dts/ti/k3-am62a.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..6eb87c3f9f3cee40452bec6890685196d0cd105a --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a.dtsi @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for AM62A SoC Family + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +#include +#include +#include +#include +#include + +/ { + model = "Texas Instruments K3 AM62A SoC"; + compatible = "ti,am62a7"; + interrupt-parent = <&gic500>; + #address-cells = <2>; + #size-cells = <2>; + + chosen { }; + + firmware { + optee { + compatible = "linaro,optee-tz"; + method = "smc"; + }; + + psci: psci { + compatible = "arm,psci-1.0"; + method = "smc"; + }; + }; + + a53_timer0: timer-cl0-cpu0 { + compatible = "arm,armv8-timer"; + interrupts = , /* cntpsirq */ + , /* cntpnsirq */ + , /* cntvirq */ + ; /* cnthpirq */ + }; + + pmu: pmu { + compatible = "arm,cortex-a53-pmu"; + interrupts = ; + }; + + cbass_main: bus@f0000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges = <0x00 0x000f0000 0x00 0x000f0000 0x00 0x00030000>, /* Main MMRs */ + <0x00 0x00420000 0x00 0x00420000 0x00 0x00001000>, /* ESM0 */ + <0x00 0x00600000 0x00 0x00600000 0x00 0x00001100>, /* GPIO */ + <0x00 0x00703000 0x00 0x00703000 0x00 0x00000200>, /* USB0 debug trace */ + <0x00 0x0070c000 0x00 0x0070c000 0x00 0x00000200>, /* USB1 debug trace */ + <0x00 0x00a40000 0x00 0x00a40000 0x00 0x00000800>, /* Timesync router */ + <0x00 0x01000000 0x00 0x01000000 0x00 0x01b28400>, /* First peripheral window */ + <0x00 0x08000000 0x00 0x08000000 0x00 0x00200000>, /* Main CPSW */ + <0x00 0x0e000000 0x00 0x0e000000 0x00 0x01d20000>, /* Second peripheral window */ + <0x00 0x0fd00000 0x00 0x0fd00000 0x00 0x00020000>, /* GPU */ + <0x00 0x20000000 0x00 0x20000000 0x00 0x0a008000>, /* Third peripheral window */ + <0x00 0x30040000 0x00 0x30040000 0x00 0x00080000>, /* PRUSS-M */ + <0x00 0x30101000 0x00 0x30101000 0x00 0x00010100>, /* CSI window */ + <0x00 0x30200000 0x00 0x30200000 0x00 0x00010000>, /* DSS */ + <0x00 0x30210000 0x00 0x30210000 0x00 0x00010000>, /* VPU */ + <0x00 0x31000000 0x00 0x31000000 0x00 0x00050000>, /* USB0 DWC3 Core window */ + <0x00 0x31100000 0x00 0x31100000 0x00 0x00050000>, /* USB1 DWC3 Core window */ + <0x00 0x40900000 0x00 0x40900000 0x00 0x00030000>, /* SA3UL */ + <0x00 0x43600000 0x00 0x43600000 0x00 0x00010000>, /* SA3 sproxy data */ + <0x00 0x44043000 0x00 0x44043000 0x00 0x00000fe0>, /* TI SCI DEBUG */ + <0x00 0x44860000 0x00 0x44860000 0x00 0x00040000>, /* SA3 sproxy config */ + <0x00 0x48000000 0x00 0x48000000 0x00 0x06400000>, /* DMSS */ + <0x00 0x60000000 0x00 0x60000000 0x00 0x08000000>, /* FSS0 DAT1 */ + <0x00 0x70000000 0x00 0x70000000 0x00 0x00010000>, /* OCSRAM */ + <0x00 0x7e000000 0x00 0x7e000000 0x00 0x00100000>, /* C7x_0 */ + <0x01 0x00000000 0x01 0x00000000 0x00 0x00310000>, /* A53 PERIPHBASE */ + <0x05 0x00000000 0x05 0x00000000 0x01 0x00000000>, /* FSS0 DAT3 */ + + /* MCU Domain Range */ + <0x00 0x04000000 0x00 0x04000000 0x00 0x01ff1400>, + <0x00 0x79000000 0x00 0x79000000 0x00 0x00008000>, /* MCU R5 ATCM */ + <0x00 0x79020000 0x00 0x79020000 0x00 0x00008000>, /* MCU R5 BTCM */ + <0x00 0x79100000 0x00 0x79100000 0x00 0x00040000>, /* MCU R5 IRAM0 */ + <0x00 0x79140000 0x00 0x79140000 0x00 0x00040000>, /* MCU R5 IRAM1 */ + + /* Wakeup Domain Range */ + <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00002400>, + <0x00 0x2b000000 0x00 0x2b000000 0x00 0x00300400>, + <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>, + <0x00 0x78000000 0x00 0x78000000 0x00 0x00008000>, /* DM R5 ATCM */ + <0x00 0x78100000 0x00 0x78100000 0x00 0x00008000>; /* DM R5 BTCM */ + + cbass_mcu: bus@4000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x00 0x04000000 0x00 0x04000000 0x00 0x01ff1400>, /* Peripheral window */ + <0x00 0x79000000 0x00 0x79000000 0x00 0x00008000>, /* MCU R5 ATCM */ + <0x00 0x79020000 0x00 0x79020000 0x00 0x00008000>, /* MCU R5 BTCM */ + <0x00 0x79100000 0x00 0x79100000 0x00 0x00040000>, /* MCU IRAM0 */ + <0x00 0x79140000 0x00 0x79140000 0x00 0x00040000>; /* MCU IRAM1 */ + }; + + cbass_wakeup: bus@b00000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00002400>, /* VTM */ + <0x00 0x2b000000 0x00 0x2b000000 0x00 0x00300400>, /* Peripheral Window */ + <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>, /* WKUP CTRL MMR */ + <0x00 0x78000000 0x00 0x78000000 0x00 0x00008000>, /* DM R5 ATCM*/ + <0x00 0x78100000 0x00 0x78100000 0x00 0x00008000>; /* DM R5 BTCM*/ + }; + }; +}; + +/* Now include the peripherals for each bus segments */ +#include "k3-am62a-main.dtsi" +#include "k3-am62a-mcu.dtsi" +#include "k3-am62a-wakeup.dtsi" diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts new file mode 100644 index 0000000000000000000000000000000000000000..576dbce80ad8363ddd783fcaacbc44ea9e4326c8 --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AM62A SK: https://www.ti.com/lit/zip/sprr459 + * + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +/dts-v1/; + +#include +#include +#include "k3-am62a7.dtsi" + +/ { + compatible = "ti,am62a7-sk", "ti,am62a7"; + model = "Texas Instruments AM62A7 SK"; + + aliases { + serial2 = &main_uart0; + mmc1 = &sdhci1; + }; + + chosen { + stdout-path = "serial2:115200n8"; + }; + + memory@80000000 { + device_type = "memory"; + /* 2G RAM */ + reg = <0x00000000 0x80000000 0x00000000 0x80000000>; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + secure_tfa_ddr: tfa@9e780000 { + reg = <0x00 0x9e780000 0x00 0x80000>; + alignment = <0x1000>; + no-map; + }; + + secure_ddr: optee@9e800000 { + reg = <0x00 0x9e800000 0x00 0x01800000>; /* for OP-TEE */ + alignment = <0x1000>; + no-map; + }; + + wkup_r5fss0_core0_memory_region: r5f-dma-memory@9c900000 { + compatible = "shared-dma-pool"; + reg = <0x00 0x9c900000 0x00 0x01e00000>; + no-map; + }; + }; + + vmain_pd: regulator-0 { + /* TPS25750 PD CONTROLLER OUTPUT */ + compatible = "regulator-fixed"; + regulator-name = "vmain_pd"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + regulator-boot-on; + }; + + vcc_5v0: regulator-1 { + /* Output of TPS63070 */ + compatible = "regulator-fixed"; + regulator-name = "vcc_5v0"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + vin-supply = <&vmain_pd>; + regulator-always-on; + regulator-boot-on; + }; + + vcc_3v3_sys: regulator-2 { + /* output of LM5141-Q1 */ + compatible = "regulator-fixed"; + regulator-name = "vcc_3v3_sys"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vmain_pd>; + regulator-always-on; + regulator-boot-on; + }; + + vdd_mmc1: regulator-3 { + /* TPS22918DBVR */ + compatible = "regulator-fixed"; + regulator-name = "vdd_mmc1"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-boot-on; + enable-active-high; + gpio = <&exp1 3 GPIO_ACTIVE_HIGH>; + }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&usr_led_pins_default>; + + led-0 { + label = "am62a-sk:green:heartbeat"; + gpios = <&main_gpio1 49 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + function = LED_FUNCTION_HEARTBEAT; + default-state = "off"; + }; + }; +}; + +&main_pmx0 { + main_uart0_pins_default: main-uart0-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x1c8, PIN_INPUT, 0) /* (D14) UART0_RXD */ + AM62AX_IOPAD(0x1cc, PIN_OUTPUT, 0) /* (E14) UART0_TXD */ + >; + }; + + main_i2c0_pins_default: main-i2c0-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x1e0, PIN_INPUT_PULLUP, 0) /* (B16) I2C0_SCL */ + AM62AX_IOPAD(0x1e4, PIN_INPUT_PULLUP, 0) /* (A16) I2C0_SDA */ + >; + }; + + main_i2c1_pins_default: main-i2c1-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x1e8, PIN_INPUT_PULLUP, 0) /* (B17) I2C1_SCL */ + AM62AX_IOPAD(0x1ec, PIN_INPUT_PULLUP, 0) /* (A17) I2C1_SDA */ + >; + }; + + main_i2c2_pins_default: main-i2c2-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x0b0, PIN_INPUT_PULLUP, 1) /* (K22) GPMC0_CSn2.I2C2_SCL */ + AM62AX_IOPAD(0x0b4, PIN_INPUT_PULLUP, 1) /* (K24) GPMC0_CSn3.I2C2_SDA */ + >; + }; + + main_mmc1_pins_default: main-mmc1-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x23c, PIN_INPUT, 0) /* (A21) MMC1_CMD */ + AM62AX_IOPAD(0x234, PIN_INPUT, 0) /* (B22) MMC1_CLK */ + AM62AX_IOPAD(0x230, PIN_INPUT, 0) /* (A22) MMC1_DAT0 */ + AM62AX_IOPAD(0x22c, PIN_INPUT, 0) /* (B21) MMC1_DAT1 */ + AM62AX_IOPAD(0x228, PIN_INPUT, 0) /* (C21) MMC1_DAT2 */ + AM62AX_IOPAD(0x224, PIN_INPUT, 0) /* (D22) MMC1_DAT3 */ + AM62AX_IOPAD(0x240, PIN_INPUT, 0) /* (D17) MMC1_SDCD */ + >; + }; + + usr_led_pins_default: usr-led-pins-default { + pinctrl-single,pins = < + AM62AX_IOPAD(0x244, PIN_OUTPUT, 7) /* (D18) MMC1_SDWP.GPIO1_49 */ + >; + }; +}; + +&main_i2c0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&main_i2c0_pins_default>; + clock-frequency = <400000>; +}; + +&main_i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&main_i2c1_pins_default>; + clock-frequency = <400000>; + + exp1: gpio@22 { + compatible = "ti,tca6424"; + reg = <0x22>; + gpio-controller; + #gpio-cells = <2>; + + gpio-line-names = "GPIO_CPSW2_RST", "GPIO_CPSW1_RST", + "BT_EN_SOC", "MMC1_SD_EN", + "VPP_EN", "EXP_PS_3V3_En", + "EXP_PS_5V0_En", "EXP_HAT_DETECT", + "GPIO_AUD_RSTn", "GPIO_eMMC_RSTn", + "UART1_FET_BUF_EN", "BT_UART_WAKE_SOC", + "GPIO_HDMI_RSTn", "CSI_GPIO0", + "CSI_GPIO1", "WLAN_ALERTn", + "HDMI_INTn", "TEST_GPIO2", + "MCASP1_FET_EN", "MCASP1_BUF_BT_EN", + "MCASP1_FET_SEL", "UART1_FET_SEL", + "PD_I2C_IRQ", "IO_EXP_TEST_LED"; + }; +}; + +&sdhci1 { + /* SD/MMC */ + status = "okay"; + vmmc-supply = <&vdd_mmc1>; + pinctrl-names = "default"; + pinctrl-0 = <&main_mmc1_pins_default>; + ti,driver-strength-ohm = <50>; + disable-wp; +}; + +&main_gpio0 { + status = "okay"; +}; + +&main_gpio1 { + status = "okay"; +}; + +&main_gpio_intr { + status = "okay"; +}; + +&main_uart0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&main_uart0_pins_default>; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am62a7.dtsi b/arch/arm64/boot/dts/ti/k3-am62a7.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..331d89fda29d039e48a11e7e5b0fd62a026d3ac6 --- /dev/null +++ b/arch/arm64/boot/dts/ti/k3-am62a7.dtsi @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device Tree Source for AM62A7 SoC family in Quad core configuration + * + * TRM: https://www.ti.com/lit/zip/spruj16 + * + * Copyright (C) 2020-2022 Texas Instruments Incorporated - https://www.ti.com/ + */ + +/dts-v1/; + +#include "k3-am62a.dtsi" + +/ { + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu-map { + cluster0: cluster0 { + core0 { + cpu = <&cpu0>; + }; + + core1 { + cpu = <&cpu1>; + }; + + core2 { + cpu = <&cpu2>; + }; + + core3 { + cpu = <&cpu3>; + }; + }; + }; + + cpu0: cpu@0 { + compatible = "arm,cortex-a53"; + reg = <0x000>; + device_type = "cpu"; + enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + next-level-cache = <&L2_0>; + }; + + cpu1: cpu@1 { + compatible = "arm,cortex-a53"; + reg = <0x001>; + device_type = "cpu"; + enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + next-level-cache = <&L2_0>; + }; + + cpu2: cpu@2 { + compatible = "arm,cortex-a53"; + reg = <0x002>; + device_type = "cpu"; + enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + next-level-cache = <&L2_0>; + }; + + cpu3: cpu@3 { + compatible = "arm,cortex-a53"; + reg = <0x003>; + device_type = "cpu"; + enable-method = "psci"; + i-cache-size = <0x8000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <128>; + next-level-cache = <&L2_0>; + }; + }; + + L2_0: l2-cache0 { + compatible = "cache"; + cache-level = <2>; + cache-size = <0x40000>; + cache-line-size = <64>; + cache-sets = <512>; + }; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi index ada00575f0f2e743236f3b7c94857b63adc12aa7..d6aa23681bbe51d9b624c5473e9b2cc766c079ed 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi @@ -550,7 +550,7 @@ }; }; - cpts@39000000 { + main_cpts0: cpts@39000000 { compatible = "ti,j721e-cpts"; reg = <0x0 0x39000000 0x0 0x400>; reg-names = "cpts"; @@ -1308,4 +1308,52 @@ interrupt-names = "int0", "int1"; bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>; }; + + crypto: crypto@40900000 { + compatible = "ti,am64-sa2ul"; + reg = <0x00 0x40900000 0x00 0x1200>; + power-domains = <&k3_pds 133 TI_SCI_PD_SHARED>; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x00 0x40900000 0x00 0x40900000 0x00 0x30000>; + dmas = <&main_pktdma 0xc001 0>, <&main_pktdma 0x4002 0>, + <&main_pktdma 0x4003 0>; + dma-names = "tx", "rx1", "rx2"; + + rng: rng@40910000 { + compatible = "inside-secure,safexcel-eip76"; + reg = <0x00 0x40910000 0x00 0x7d>; + interrupts = ; + clocks = <&k3_clks 133 1>; + status = "disabled"; /* Used by OP-TEE */ + }; + }; + + gpmc0: memory-controller@3b000000 { + compatible = "ti,am64-gpmc"; + power-domains = <&k3_pds 80 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 80 0>; + clock-names = "fck"; + reg = <0x00 0x03b000000 0x00 0x400>, + <0x00 0x050000000 0x00 0x8000000>; + reg-names = "cfg", "data"; + interrupts = ; + gpmc,num-cs = <3>; + gpmc,num-waitpins = <2>; + #address-cells = <2>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-controller; + #gpio-cells = <2>; + }; + + elm0: ecc@25010000 { + compatible = "ti,am64-elm"; + reg = <0x00 0x25010000 0x00 0x2000>; + interrupts = ; + power-domains = <&k3_pds 54 TI_SCI_PD_EXCLUSIVE>; + clocks = <&k3_clks 54 0>; + clock-names = "fck"; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-am64.dtsi b/arch/arm64/boot/dts/ti/k3-am64.dtsi index 016dd8511ca6f37f651a5274686fc4f5d168a81d..c858725133af49a27cd7f1535527d530dbb42f97 100644 --- a/arch/arm64/boot/dts/ti/k3-am64.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64.dtsi @@ -82,6 +82,7 @@ <0x00 0x3b000000 0x00 0x3b000000 0x00 0x00000400>, /* GPMC0_CFG */ <0x00 0x3cd00000 0x00 0x3cd00000 0x00 0x00000200>, /* TIMERMGR0_CONFIG */ <0x00 0x3f004000 0x00 0x3f004000 0x00 0x00000400>, /* GICSS0_REGS */ + <0x00 0x40900000 0x00 0x40900000 0x00 0x00030000>, /* SA2_UL0 */ <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>, /* CTRL_MMR0 */ <0x00 0x44043000 0x00 0x44043000 0x00 0x00000fe0>, /* TI SCI DEBUG */ <0x00 0x48000000 0x00 0x48000000 0x00 0x06400000>, /* DMASS */ diff --git a/arch/arm64/boot/dts/ti/k3-am642-evm.dts b/arch/arm64/boot/dts/ti/k3-am642-evm.dts index ad150c704623581b2b7ef06cbd755e4ecf06f3c1..5cf913860f800e1415b79b73c9dc1a1aedf5472e 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-evm.dts +++ b/arch/arm64/boot/dts/ti/k3-am642-evm.dts @@ -676,3 +676,11 @@ pinctrl-0 = <&main_mcan1_pins_default>; phys = <&transceiver2>; }; + +&gpmc0 { + status = "disabled"; +}; + +&elm0 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am642-sk.dts b/arch/arm64/boot/dts/ti/k3-am642-sk.dts index 2620469a75179f22a9ddef9f72411ee89b49ff77..738d0cf6c40ac9b44d597d97d22e0d2067be503f 100644 --- a/arch/arm64/boot/dts/ti/k3-am642-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am642-sk.dts @@ -9,6 +9,7 @@ #include #include #include +#include #include "k3-am642.dtsi" / { @@ -150,6 +151,74 @@ vin-supply = <&com8_ls_en>; gpio = <&main_gpio0 48 GPIO_ACTIVE_HIGH>; }; + + led-controller { + compatible = "gpio-leds"; + + led-0 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <1>; + gpios = <&exp2 0 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-1 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <2>; + gpios = <&exp2 1 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-2 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <3>; + gpios = <&exp2 2 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-3 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <4>; + gpios = <&exp2 3 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-4 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <5>; + gpios = <&exp2 4 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-5 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <6>; + gpios = <&exp2 5 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-6 { + color = ; + function = LED_FUNCTION_INDICATOR; + function-enumerator = <7>; + gpios = <&exp2 6 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + led-7 { + color = ; + function = LED_FUNCTION_HEARTBEAT; + function-enumerator = <8>; + linux,default-trigger = "heartbeat"; + gpios = <&exp2 7 GPIO_ACTIVE_HIGH>; + }; + }; }; &main_pmx0 { @@ -330,6 +399,14 @@ "VPP_LDO_EN", "RPI_PS_3V3_En", "RPI_PS_5V0_En", "RPI_HAT_DETECT"; }; + + exp2: gpio@60 { + compatible = "ti,tpic2810"; + reg = <0x60>; + gpio-controller; + #gpio-cells = <2>; + gpio-line-names = "LED1","LED2","LED3","LED4","LED5","LED6","LED7","LED8"; + }; }; &main_i2c3 { @@ -607,3 +684,11 @@ &main_mcan1 { status = "disabled"; }; + +&gpmc0 { + status = "disabled"; +}; + +&elm0 { + status = "disabled"; +}; diff --git a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi index 8919fede3cd76fc97f93a149eff7de777201ccb9..4005a73cfea990bce88fd50e4972e76a1e03e306 100644 --- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi @@ -112,13 +112,13 @@ crypto: crypto@4e00000 { compatible = "ti,am654-sa2ul"; reg = <0x0 0x4e00000 0x0 0x1200>; - power-domains = <&k3_pds 136 TI_SCI_PD_EXCLUSIVE>; + power-domains = <&k3_pds 136 TI_SCI_PD_SHARED>; #address-cells = <2>; #size-cells = <2>; ranges = <0x0 0x04e00000 0x00 0x04e00000 0x0 0x30000>; - dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>, - <&main_udmap 0x4001>; + dmas = <&main_udmap 0xc001>, <&main_udmap 0x4002>, + <&main_udmap 0x4003>; dma-names = "tx", "rx1", "rx2"; dma-coherent; @@ -127,6 +127,7 @@ reg = <0x0 0x4e10000 0x0 0x7d>; interrupts = ; clocks = <&k3_clks 136 1>; + status = "disabled"; /* Used by OP-TEE */ }; }; diff --git a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts index 121975dc82397ec03be9ab559a9bf0aa82f8a0b5..7e8552fd2b6ae67fdd91af683093434f9edb311e 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts +++ b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts @@ -134,15 +134,17 @@ >; }; - main_usbss0_pins_default: main-usbss0-pins-default { + vdd_sd_dv_pins_default: vdd-sd-dv-pins-default { pinctrl-single,pins = < - J721E_IOPAD(0x120, PIN_OUTPUT, 0) /* (T4) USB0_DRVVBUS */ + J721E_IOPAD(0xd0, PIN_OUTPUT, 7) /* (T5) SPI0_D1.GPIO0_55 */ >; }; +}; - vdd_sd_dv_pins_default: vdd-sd-dv-pins-default { +&main_pmx1 { + main_usbss0_pins_default: main-usbss0-pins-default { pinctrl-single,pins = < - J721E_IOPAD(0xd0, PIN_OUTPUT, 7) /* (T5) SPI0_D1.GPIO0_55 */ + J721E_IOPAD(0x04, PIN_OUTPUT, 0) /* (T4) USB0_DRVVBUS */ >; }; }; diff --git a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi index 16684a2f054d987ad22f107ff2be021fa776fac3..80a57916bcb3e391a999e23aedd884b8754bcd98 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi @@ -295,7 +295,16 @@ main_pmx0: pinctrl@11c000 { compatible = "pinctrl-single"; /* Proxy 0 addressing */ - reg = <0x00 0x11c000 0x00 0x2b4>; + reg = <0x00 0x11c000 0x00 0x10c>; + #pinctrl-cells = <1>; + pinctrl-single,register-width = <32>; + pinctrl-single,function-mask = <0xffffffff>; + }; + + main_pmx1: pinctrl@11c11c { + compatible = "pinctrl-single"; + /* Proxy 0 addressing */ + reg = <0x00 0x11c11c 0x00 0xc>; #pinctrl-cells = <1>; pinctrl-single,register-width = <32>; pinctrl-single,function-mask = <0xffffffff>; @@ -739,6 +748,24 @@ clock-names = "gpio"; }; + watchdog0: watchdog@2200000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x0 0x2200000 0x0 0x100>; + clocks = <&k3_clks 252 1>; + power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 252 1>; + assigned-clock-parents = <&k3_clks 252 5>; + }; + + watchdog1: watchdog@2210000 { + compatible = "ti,j7-rti-wdt"; + reg = <0x0 0x2210000 0x0 0x100>; + clocks = <&k3_clks 253 1>; + power-domains = <&k3_pds 253 TI_SCI_PD_EXCLUSIVE>; + assigned-clocks = <&k3_clks 253 1>; + assigned-clock-parents = <&k3_clks 253 5>; + }; + main_r5fss0: r5fss@5c00000 { compatible = "ti,j7200-r5fss"; ti,cluster-mode = <1>; diff --git a/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi index ff13bbeed30c91e657c9d935cdae8077ed18ef72..e5be78a58682d542a3ce90a0956409b8bcef9a91 100644 --- a/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi @@ -375,4 +375,24 @@ ti,loczrama = <1>; }; }; + + mcu_crypto: crypto@40900000 { + compatible = "ti,j721e-sa2ul"; + reg = <0x00 0x40900000 0x00 0x1200>; + power-domains = <&k3_pds 265 TI_SCI_PD_SHARED>; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x00 0x40900000 0x00 0x40900000 0x00 0x30000>; + dmas = <&mcu_udmap 0xf501>, <&mcu_udmap 0x7502>, + <&mcu_udmap 0x7503>; + dma-names = "tx", "rx1", "rx2"; + dma-coherent; + + rng: rng@40910000 { + compatible = "inside-secure,safexcel-eip76"; + reg = <0x00 0x40910000 0x00 0x7d>; + interrupts = ; + status = "disabled"; /* Used by OP-TEE */ + }; + }; }; diff --git a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi index 43b6cf5791eee943985c98058d26de83b998c325..917c9dc99efaa867d82dde4a33394dc025799363 100644 --- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi @@ -343,7 +343,7 @@ compatible = "inside-secure,safexcel-eip76"; reg = <0x0 0x4e10000 0x0 0x7d>; interrupts = ; - clocks = <&k3_clks 264 1>; + clocks = <&k3_clks 264 2>; }; }; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index d5b2d2dd49043d26aaa71a7e996c083fc75963df..0b6af3348e791a8c995dc7f263bf3709baf04208 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -18,6 +18,7 @@ CONFIG_NUMA_BALANCING=y CONFIG_MEMCG=y CONFIG_BLK_CGROUP=y CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_HUGETLB=y CONFIG_CPUSETS=y CONFIG_CGROUP_DEVICE=y @@ -33,8 +34,8 @@ CONFIG_ARCH_ACTIONS=y CONFIG_ARCH_SUNXI=y CONFIG_ARCH_ALPINE=y CONFIG_ARCH_APPLE=y +CONFIG_ARCH_BCM=y CONFIG_ARCH_BCM2835=y -CONFIG_ARCH_BCM4908=y CONFIG_ARCH_BCMBCA=y CONFIG_ARCH_BCM_IPROC=y CONFIG_ARCH_BERLIN=y @@ -48,6 +49,7 @@ CONFIG_ARCH_KEEMBAY=y CONFIG_ARCH_MEDIATEK=y CONFIG_ARCH_MESON=y CONFIG_ARCH_MVEBU=y +CONFIG_ARCH_NXP=y CONFIG_ARCH_MXC=y CONFIG_ARCH_NPCM=y CONFIG_ARCH_QCOM=y @@ -80,7 +82,6 @@ CONFIG_RANDOMIZE_BASE=y CONFIG_HIBERNATION=y CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y CONFIG_ENERGY_MODEL=y -CONFIG_ARM_CPUIDLE=y CONFIG_ARM_PSCI_CPUIDLE=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_STAT=y @@ -102,6 +103,8 @@ CONFIG_ARM_SCMI_CPUFREQ=y CONFIG_ARM_TEGRA186_CPUFREQ=y CONFIG_QORIQ_CPUFREQ=y CONFIG_ACPI=y +CONFIG_ACPI_HOTPLUG_MEMORY=y +CONFIG_ACPI_HMAT=y CONFIG_ACPI_APEI=y CONFIG_ACPI_APEI_GHES=y CONFIG_ACPI_APEI_PCIEAER=y @@ -109,7 +112,6 @@ CONFIG_ACPI_APEI_MEMORY_FAILURE=y CONFIG_ACPI_APEI_EINJ=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=y -CONFIG_ARM64_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_SHA512_ARM64_CE=m @@ -126,6 +128,8 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_COMPAT_BRK is not set +CONFIG_MEMORY_HOTPLUG=y +CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y CONFIG_MEMORY_FAILURE=y CONFIG_TRANSPARENT_HUGEPAGE=y @@ -139,12 +143,16 @@ CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IPV6=m CONFIG_NETFILTER=y +CONFIG_BRIDGE_NETFILTER=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NETFILTER_XT_MARK=m CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m CONFIG_NETFILTER_XT_TARGET_LOG=m CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_IP_VS=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_FILTER=m CONFIG_IP_NF_TARGET_REJECT=m @@ -354,6 +362,7 @@ CONFIG_SMSC911X=y CONFIG_SNI_AVE=y CONFIG_SNI_NETSEC=y CONFIG_STMMAC_ETH=m +CONFIG_DWMAC_TEGRA=m CONFIG_TI_K3_AM65_CPSW_NUSS=y CONFIG_QCOM_IPA=m CONFIG_MESON_GXL_PHY=m @@ -474,6 +483,7 @@ CONFIG_I2C_QCOM_GENI=m CONFIG_I2C_QUP=y CONFIG_I2C_RIIC=y CONFIG_I2C_RK3X=y +CONFIG_I2C_RZV2M=m CONFIG_I2C_S3C2410=y CONFIG_I2C_SH_MOBILE=y CONFIG_I2C_TEGRA=y @@ -506,6 +516,8 @@ CONFIG_SPI_QCOM_GENI=m CONFIG_SPI_S3C64XX=y CONFIG_SPI_SH_MSIOF=m CONFIG_SPI_SUN6I=y +CONFIG_SPI_TEGRA210_QUAD=m +CONFIG_SPI_TEGRA114=m CONFIG_SPI_SPIDEV=m CONFIG_SPMI=y CONFIG_PINCTRL_MAX77620=y @@ -534,6 +546,7 @@ CONFIG_PINCTRL_QDF2XXX=y CONFIG_PINCTRL_QCOM_SPMI_PMIC=y CONFIG_PINCTRL_SC7180=y CONFIG_PINCTRL_SC7280=y +CONFIG_PINCTRL_SC8180X=y CONFIG_PINCTRL_SC8280XP=y CONFIG_PINCTRL_SDM845=y CONFIG_PINCTRL_SM8150=y @@ -584,6 +597,7 @@ CONFIG_SENSORS_INA2XX=m CONFIG_SENSORS_INA3221=m CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y CONFIG_CPU_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y CONFIG_THERMAL_EMULATION=y CONFIG_IMX_SC_THERMAL=m CONFIG_IMX8MM_THERMAL=m @@ -794,6 +808,8 @@ CONFIG_SND_SOC_APQ8016_SBC=m CONFIG_SND_SOC_MSM8996=m CONFIG_SND_SOC_SDM845=m CONFIG_SND_SOC_SM8250=m +CONFIG_SND_SOC_SC7180=m +CONFIG_SND_SOC_SC7280=m CONFIG_SND_SOC_ROCKCHIP=m CONFIG_SND_SOC_ROCKCHIP_SPDIF=m CONFIG_SND_SOC_ROCKCHIP_RT5645=m @@ -914,6 +930,7 @@ CONFIG_MMC_SDHCI=y CONFIG_MMC_SDHCI_ACPI=y CONFIG_MMC_SDHCI_PLTFM=y CONFIG_MMC_SDHCI_OF_ARASAN=y +CONFIG_MMC_SDHCI_OF_DWCMSHC=y CONFIG_MMC_SDHCI_OF_ESDHC=y CONFIG_MMC_SDHCI_CADENCE=y CONFIG_MMC_SDHCI_ESDHC_IMX=y @@ -939,6 +956,7 @@ CONFIG_SCSI_UFSHCD=y CONFIG_SCSI_UFSHCD_PLATFORM=y CONFIG_SCSI_UFS_QCOM=m CONFIG_SCSI_UFS_HISI=y +CONFIG_SCSI_UFS_RENESAS=m CONFIG_SCSI_UFS_EXYNOS=y CONFIG_NEW_LEDS=y CONFIG_LEDS_CLASS=y @@ -995,7 +1013,7 @@ CONFIG_MV_XOR=y CONFIG_MV_XOR_V2=y CONFIG_OWL_DMA=y CONFIG_PL330_DMA=y -CONFIG_TEGRA186_GPC_DMA=m +CONFIG_TEGRA186_GPC_DMA=y CONFIG_TEGRA20_APB_DMA=y CONFIG_TEGRA210_ADMA=m CONFIG_QCOM_BAM_DMA=y @@ -1058,6 +1076,7 @@ CONFIG_MSM_GCC_8998=y CONFIG_QCS_GCC_404=y CONFIG_SC_GCC_7180=y CONFIG_SC_GCC_7280=y +CONFIG_SC_GCC_8180X=y CONFIG_SC_GCC_8280XP=y CONFIG_SDM_CAMCC_845=m CONFIG_SDM_GPUCC_845=y @@ -1088,11 +1107,13 @@ CONFIG_ARM_SMMU_V3=y CONFIG_MTK_IOMMU=y CONFIG_QCOM_IOMMU=y CONFIG_REMOTEPROC=y +CONFIG_QCOM_Q6V5_ADSP=m CONFIG_QCOM_Q6V5_MSS=m CONFIG_QCOM_Q6V5_PAS=m CONFIG_QCOM_SYSMON=m CONFIG_QCOM_WCNSS_PIL=m CONFIG_RPMSG_CHAR=m +CONFIG_RPMSG_CTRL=m CONFIG_RPMSG_QCOM_GLINK_RPM=y CONFIG_RPMSG_QCOM_GLINK_SMEM=m CONFIG_RPMSG_QCOM_SMD=y @@ -1109,6 +1130,8 @@ CONFIG_QCOM_AOSS_QMP=y CONFIG_QCOM_COMMAND_DB=y CONFIG_QCOM_CPR=y CONFIG_QCOM_GENI_SE=y +CONFIG_QCOM_LLCC=m +CONFIG_QCOM_OCMEM=m CONFIG_QCOM_RMTFS_MEM=m CONFIG_QCOM_RPMH=y CONFIG_QCOM_RPMHPD=y @@ -1118,6 +1141,7 @@ CONFIG_QCOM_SMD_RPM=y CONFIG_QCOM_SMP2P=y CONFIG_QCOM_SMSM=y CONFIG_QCOM_SOCINFO=m +CONFIG_QCOM_SPM=m CONFIG_QCOM_STATS=m CONFIG_QCOM_WCNSS_CTRL=m CONFIG_QCOM_APR=m @@ -1207,6 +1231,7 @@ CONFIG_PHY_HISTB_COMBPHY=y CONFIG_PHY_HISI_INNO_USB2=y CONFIG_PHY_MVEBU_CP110_COMPHY=y CONFIG_PHY_MTK_TPHY=y +CONFIG_PHY_QCOM_EDP=m CONFIG_PHY_QCOM_PCIE2=m CONFIG_PHY_QCOM_QMP=m CONFIG_PHY_QCOM_QUSB2=m @@ -1229,19 +1254,25 @@ CONFIG_PHY_UNIPHIER_USB3=y CONFIG_PHY_TEGRA_XUSB=y CONFIG_PHY_AM654_SERDES=m CONFIG_PHY_J721E_WIZ=m +CONFIG_ARM_CCI_PMU=m +CONFIG_ARM_CCN=m +CONFIG_ARM_CMN=m CONFIG_ARM_SMMU_V3_PMU=m +CONFIG_ARM_DSU_PMU=m CONFIG_FSL_IMX8_DDR_PMU=m +CONFIG_ARM_SPE_PMU=m +CONFIG_ARM_DMC620_PMU=m CONFIG_QCOM_L2_PMU=y CONFIG_QCOM_L3_PMU=y CONFIG_HISI_PMU=y CONFIG_NVMEM_IMX_OCOTP=y CONFIG_NVMEM_IMX_OCOTP_SCU=y -CONFIG_MTK_EFUSE=y -CONFIG_QCOM_QFPROM=y -CONFIG_ROCKCHIP_EFUSE=y +CONFIG_NVMEM_MTK_EFUSE=y +CONFIG_NVMEM_QCOM_QFPROM=y +CONFIG_NVMEM_ROCKCHIP_EFUSE=y CONFIG_NVMEM_SUNXI_SID=y -CONFIG_UNIPHIER_EFUSE=y -CONFIG_MESON_EFUSE=m +CONFIG_NVMEM_UNIPHIER_EFUSE=y +CONFIG_NVMEM_MESON_EFUSE=m CONFIG_NVMEM_RMEM=m CONFIG_NVMEM_LAYERSCAPE_SFP=m CONFIG_FPGA=y @@ -1269,6 +1300,7 @@ CONFIG_INTERCONNECT_QCOM_OSM_L3=m CONFIG_INTERCONNECT_QCOM_QCS404=m CONFIG_INTERCONNECT_QCOM_SC7180=m CONFIG_INTERCONNECT_QCOM_SC7280=y +CONFIG_INTERCONNECT_QCOM_SC8180X=y CONFIG_INTERCONNECT_QCOM_SC8280XP=y CONFIG_INTERCONNECT_QCOM_SDM845=y CONFIG_INTERCONNECT_QCOM_SM8150=m @@ -1325,4 +1357,12 @@ CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_PREEMPT is not set # CONFIG_FTRACE is not set +CONFIG_CORESIGHT=m +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m +CONFIG_CORESIGHT_CATU=m +CONFIG_CORESIGHT_SINK_TPIU=m +CONFIG_CORESIGHT_SINK_ETBV10=m +CONFIG_CORESIGHT_STM=m +CONFIG_CORESIGHT_CPU_DEBUG=m +CONFIG_CORESIGHT_CTI=m CONFIG_MEMTEST=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 60db5bb2ddda5038bda4f60b12c42c4c58082058..8bd80508a710d1120afc320701111b8397f34f86 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -1,141 +1,282 @@ # SPDX-License-Identifier: GPL-2.0 -menuconfig ARM64_CRYPTO - bool "ARM64 Accelerated Cryptographic Algorithms" - depends on ARM64 +menu "Accelerated Cryptographic Algorithms for CPU (arm64)" + +config CRYPTO_GHASH_ARM64_CE + tristate "Hash functions: GHASH (ARMv8 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_GF128MUL + select CRYPTO_LIB_AES + select CRYPTO_AEAD help - Say Y here to choose from a selection of cryptographic algorithms - implemented using ARM64 specific CPU features or instructions. + GCM GHASH function (NIST SP800-38D) -if ARM64_CRYPTO + Architecture: arm64 using: + - ARMv8 Crypto Extensions -config CRYPTO_SHA256_ARM64 - tristate "SHA-224/SHA-256 digest algorithm for arm64" - select CRYPTO_HASH +config CRYPTO_NHPOLY1305_NEON + tristate "Hash functions: NHPoly1305 (NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function (Adiantum) -config CRYPTO_SHA512_ARM64 - tristate "SHA-384/SHA-512 digest algorithm for arm64" + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions + +config CRYPTO_POLY1305_NEON + tristate "Hash functions: Poly1305 (NEON)" + depends on KERNEL_MODE_NEON select CRYPTO_HASH + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_SHA1_ARM64_CE - tristate "SHA-1 digest algorithm (ARMv8 Crypto Extensions)" + tristate "Hash functions: SHA-1 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA1 + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + +config CRYPTO_SHA256_ARM64 + tristate "Hash functions: SHA-224 and SHA-256" + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm64 config CRYPTO_SHA2_ARM64_CE - tristate "SHA-224/SHA-256 digest algorithm (ARMv8 Crypto Extensions)" + tristate "Hash functions: SHA-224 and SHA-256 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA256_ARM64 + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + +config CRYPTO_SHA512_ARM64 + tristate "Hash functions: SHA-384 and SHA-512" + select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: arm64 config CRYPTO_SHA512_ARM64_CE - tristate "SHA-384/SHA-512 digest algorithm (ARMv8 Crypto Extensions)" + tristate "Hash functions: SHA-384 and SHA-512 (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA512_ARM64 + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_SHA3_ARM64 - tristate "SHA3 digest algorithm (ARMv8.2 Crypto Extensions)" + tristate "Hash functions: SHA-3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SHA3 + help + SHA-3 secure hash algorithms (FIPS 202) + + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions config CRYPTO_SM3_ARM64_CE - tristate "SM3 digest algorithm (ARMv8.2 Crypto Extensions)" + tristate "Hash functions: SM3 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_SM3 + help + SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012) -config CRYPTO_SM4_ARM64_CE - tristate "SM4 symmetric cipher (ARMv8.2 Crypto Extensions)" - depends on KERNEL_MODE_NEON - select CRYPTO_ALGAPI - select CRYPTO_SM4 - -config CRYPTO_SM4_ARM64_CE_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER - select CRYPTO_SM4 - -config CRYPTO_SM4_ARM64_NEON_BLK - tristate "SM4 in ECB/CBC/CFB/CTR modes using NEON instructions" - depends on KERNEL_MODE_NEON - select CRYPTO_SKCIPHER - select CRYPTO_SM4 - -config CRYPTO_GHASH_ARM64_CE - tristate "GHASH/AES-GCM using ARMv8 Crypto Extensions" - depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_GF128MUL - select CRYPTO_LIB_AES - select CRYPTO_AEAD + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions config CRYPTO_POLYVAL_ARM64_CE - tristate "POLYVAL using ARMv8 Crypto Extensions (for HCTR2)" + tristate "Hash functions: POLYVAL (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_POLYVAL + help + POLYVAL hash function for HCTR2 -config CRYPTO_CRCT10DIF_ARM64_CE - tristate "CRCT10DIF digest algorithm using PMULL instructions" - depends on KERNEL_MODE_NEON && CRC_T10DIF - select CRYPTO_HASH + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64 - tristate "AES core cipher using scalar instructions" + tristate "Ciphers: AES, modes: ECB, CBC, CTR, CTS, XCTR, XTS" select CRYPTO_AES + help + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preserving ciphers: AES with ECB, CBC, CTR, CTS, + XCTR, and XTS modes + AEAD cipher: AES with CBC, ESSIV, and SHA-256 + for fscrypt and dm-crypt + + Architecture: arm64 config CRYPTO_AES_ARM64_CE - tristate "AES core cipher using ARMv8 Crypto Extensions" + tristate "Ciphers: AES (ARMv8 Crypto Extensions)" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_ALGAPI select CRYPTO_LIB_AES + help + Block ciphers: AES cipher algorithms (FIPS-197) -config CRYPTO_AES_ARM64_CE_CCM - tristate "AES in CCM mode using ARMv8 Crypto Extensions" - depends on ARM64 && KERNEL_MODE_NEON - select CRYPTO_ALGAPI - select CRYPTO_AES_ARM64_CE - select CRYPTO_AEAD - select CRYPTO_LIB_AES + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64_CE_BLK - tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using ARMv8 Crypto Extensions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_AES_ARM64_CE + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions config CRYPTO_AES_ARM64_NEON_BLK - tristate "AES in ECB/CBC/CTR/XTS/XCTR modes using NEON instructions" + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_AES + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions config CRYPTO_CHACHA20_NEON - tristate "ChaCha20, XChaCha20, and XChaCha12 stream ciphers using NEON instructions" + tristate "Ciphers: ChaCha (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms -config CRYPTO_POLY1305_NEON - tristate "Poly1305 hash function using scalar or NEON instructions" + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions + +config CRYPTO_AES_ARM64_BS + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XCTR/XTS modes (bit-sliced NEON)" depends on KERNEL_MODE_NEON - select CRYPTO_HASH - select CRYPTO_ARCH_HAVE_LIB_POLY1305 + select CRYPTO_SKCIPHER + select CRYPTO_AES_ARM64_NEON_BLK + select CRYPTO_LIB_AES + help + Length-preserving ciphers: AES cipher algorithms (FIPS-197) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + - XCTR mode for HCTR2 + - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) -config CRYPTO_NHPOLY1305_NEON - tristate "NHPoly1305 hash function using NEON instructions (for Adiantum)" + Architecture: arm64 using: + - bit-sliced algorithm + - NEON (Advanced SIMD) extensions + +config CRYPTO_SM4_ARM64_CE + tristate "Ciphers: SM4 (ARMv8.2 Crypto Extensions)" depends on KERNEL_MODE_NEON - select CRYPTO_NHPOLY1305 + select CRYPTO_ALGAPI + select CRYPTO_SM4 + help + Block ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) -config CRYPTO_AES_ARM64_BS - tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm" + Architecture: arm64 using: + - ARMv8.2 Crypto Extensions + - NEON (Advanced SIMD) extensions + +config CRYPTO_SM4_ARM64_CE_BLK + tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER - select CRYPTO_AES_ARM64_NEON_BLK + select CRYPTO_SM4 + help + Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CFB (Cipher Feedback) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + - NEON (Advanced SIMD) extensions + +config CRYPTO_SM4_ARM64_NEON_BLK + tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR (NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_SKCIPHER + select CRYPTO_SM4 + help + Length-preserving ciphers: SM4 cipher algorithms (OSCCA GB/T 32907-2016) + with block cipher modes: + - ECB (Electronic Codebook) mode (NIST SP800-38A) + - CBC (Cipher Block Chaining) mode (NIST SP800-38A) + - CFB (Cipher Feedback) mode (NIST SP800-38A) + - CTR (Counter) mode (NIST SP800-38A) + + Architecture: arm64 using: + - NEON (Advanced SIMD) extensions + +config CRYPTO_AES_ARM64_CE_CCM + tristate "AEAD cipher: AES in CCM mode (ARMv8 Crypto Extensions)" + depends on ARM64 && KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM64_CE + select CRYPTO_AEAD select CRYPTO_LIB_AES + help + AEAD cipher: AES cipher algorithms (FIPS-197) with + CCM (Counter with Cipher Block Chaining-Message Authentication Code) + authenticated encryption mode (NIST SP800-38C) + + Architecture: arm64 using: + - ARMv8 Crypto Extensions + - NEON (Advanced SIMD) extensions + +config CRYPTO_CRCT10DIF_ARM64_CE + tristate "CRCT10DIF (PMULL)" + depends on KERNEL_MODE_NEON && CRC_T10DIF + select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: arm64 using + - PMULL (Polynomial Multiply Long) instructions + +endmenu -endif diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 7868330dd54ec2f22faf815b1568abfd339787ce..ebe5558929b7bba68fa2802906fbf88a759de077 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -6,6 +6,7 @@ */ #include +#include #include SHASH .req v0 @@ -350,11 +351,11 @@ CPU_LE( rev64 T1.16b, T1.16b ) * void pmull_ghash_update(int blocks, u64 dg[], const char *src, * struct ghash_key const *k, const char *head) */ -SYM_FUNC_START(pmull_ghash_update_p64) +SYM_TYPED_FUNC_START(pmull_ghash_update_p64) __pmull_ghash p64 SYM_FUNC_END(pmull_ghash_update_p64) -SYM_FUNC_START(pmull_ghash_update_p8) +SYM_TYPED_FUNC_START(pmull_ghash_update_p8) __pmull_ghash p8 SYM_FUNC_END(pmull_ghash_update_p8) diff --git a/arch/arm64/crypto/sm3-ce-core.S b/arch/arm64/crypto/sm3-ce-core.S index ef97d3187cb7fce869b4165b0e8f3dc64b8e7325..ca70cfacd0d0aa094a9a2d4e7088c559cdde9276 100644 --- a/arch/arm64/crypto/sm3-ce-core.S +++ b/arch/arm64/crypto/sm3-ce-core.S @@ -6,6 +6,7 @@ */ #include +#include #include .irp b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 @@ -73,7 +74,7 @@ * int blocks) */ .text -SYM_FUNC_START(sm3_ce_transform) +SYM_TYPED_FUNC_START(sm3_ce_transform) /* load state */ ld1 {v8.4s-v9.4s}, [x0] rev64 v8.4s, v8.4s diff --git a/arch/arm64/hyperv/mshyperv.c b/arch/arm64/hyperv/mshyperv.c index bbbe351e9045c0dec8cca926a5e598262ac6e990..a406454578f07e2a59c5a34639503ea0692f8fa6 100644 --- a/arch/arm64/hyperv/mshyperv.c +++ b/arch/arm64/hyperv/mshyperv.c @@ -38,7 +38,7 @@ static int __init hyperv_init(void) return 0; /* Setup the guest ID */ - guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); + guest_id = hv_generate_guest_id(LINUX_VERSION_CODE); hv_set_vpreg(HV_REGISTER_GUEST_OSID, guest_id); /* Get the features and hints from Hyper-V */ diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h index 7e157ab6cd505376a9c809f6941658ae71a3c9f3..3622e9f4fb442b9e8301568a2447b99f78beeea2 100644 --- a/arch/arm64/include/asm/alternative-macros.h +++ b/arch/arm64/include/asm/alternative-macros.h @@ -2,10 +2,22 @@ #ifndef __ASM_ALTERNATIVE_MACROS_H #define __ASM_ALTERNATIVE_MACROS_H +#include +#include + #include #include -#define ARM64_CB_PATCH ARM64_NCAPS +/* + * Binutils 2.27.0 can't handle a 'UL' suffix on constants, so for the assembly + * macros below we must use we must use `(1 << ARM64_CB_SHIFT)`. + */ +#define ARM64_CB_SHIFT 15 +#define ARM64_CB_BIT BIT(ARM64_CB_SHIFT) + +#if ARM64_NCAPS >= ARM64_CB_BIT +#error "cpucaps have overflown ARM64_CB_BIT" +#endif #ifndef __ASSEMBLY__ @@ -73,8 +85,8 @@ #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) -#define ALTERNATIVE_CB(oldinstr, cb) \ - __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb) +#define ALTERNATIVE_CB(oldinstr, feature, cb) \ + __ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (feature), 1, cb) #else #include @@ -82,7 +94,7 @@ .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len .word \orig_offset - . .word \alt_offset - . - .hword \feature + .hword (\feature) .byte \orig_len .byte \alt_len .endm @@ -141,10 +153,10 @@ 661: .endm -.macro alternative_cb cb +.macro alternative_cb cap, cb .set .Lasm_alt_mode, 0 .pushsection .altinstructions, "a" - altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0 + altinstruction_entry 661f, \cb, (1 << ARM64_CB_SHIFT) | \cap, 662f-661f, 0 .popsection 661: .endm @@ -207,4 +219,46 @@ alternative_endif #define ALTERNATIVE(oldinstr, newinstr, ...) \ _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) +#ifndef __ASSEMBLY__ + +#include + +static __always_inline bool +alternative_has_feature_likely(unsigned long feature) +{ + compiletime_assert(feature < ARM64_NCAPS, + "feature must be < ARM64_NCAPS"); + + asm_volatile_goto( + ALTERNATIVE_CB("b %l[l_no]", %[feature], alt_cb_patch_nops) + : + : [feature] "i" (feature) + : + : l_no); + + return true; +l_no: + return false; +} + +static __always_inline bool +alternative_has_feature_unlikely(unsigned long feature) +{ + compiletime_assert(feature < ARM64_NCAPS, + "feature must be < ARM64_NCAPS"); + + asm_volatile_goto( + ALTERNATIVE("nop", "b %l[l_yes]", %[feature]) + : + : [feature] "i" (feature) + : + : l_yes); + + return false; +l_yes: + return true; +} + +#endif /* __ASSEMBLY__ */ + #endif /* __ASM_ALTERNATIVE_MACROS_H */ diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 5846145be523c64c4e5c5be4e57cd9e9c2104ad0..e5957a53be3983acdf50e8b5750457ea97112e10 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -293,7 +293,7 @@ alternative_endif alternative_if_not ARM64_KVM_PROTECTED_MODE ASM_BUG() alternative_else_nop_endif -alternative_cb kvm_compute_final_ctr_el0 +alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 movz \reg, #0 movk \reg, #0, lsl #16 movk \reg, #0, lsl #32 @@ -384,8 +384,8 @@ alternative_cb_end .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 mrs \tmp0, ID_AA64MMFR0_EL1 // Narrow PARange to fit the PS field in TCR_ELx - ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3 - mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX + ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 + mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX cmp \tmp0, \tmp1 csel \tmp0, \tmp1, \tmp0, hi bfi \tcr, \tmp0, \pos, #3 @@ -512,7 +512,7 @@ alternative_endif */ .macro reset_pmuserenr_el0, tmpreg mrs \tmpreg, id_aa64dfr0_el1 - sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4 + sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 cmp \tmpreg, #1 // Skip if no PMU present b.lt 9000f msr pmuserenr_el0, xzr // Disable PMU access from EL0 @@ -524,7 +524,7 @@ alternative_endif */ .macro reset_amuserenr_el0, tmpreg mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 - ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4 + ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 .Lskip_\@: @@ -612,7 +612,7 @@ alternative_endif .macro offset_ttbr1, ttbr, tmp #ifdef CONFIG_ARM64_VA_BITS_52 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1 - and \tmp, \tmp, #(0xf << ID_AA64MMFR2_LVA_SHIFT) + and \tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT) cbnz \tmp, .Lskipoffs_\@ orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET .Lskipoffs_\@ : @@ -877,7 +877,7 @@ alternative_endif .macro __mitigate_spectre_bhb_loop tmp #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY -alternative_cb spectre_bhb_patch_loop_iter +alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter mov \tmp, #32 // Patched to correct the immediate alternative_cb_end .Lspectre_bhb_loop\@: @@ -890,7 +890,7 @@ alternative_cb_end .macro mitigate_spectre_bhb_loop tmp #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY -alternative_cb spectre_bhb_patch_loop_mitigation_enable +alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable b .L_spectre_bhb_loop_done\@ // Patched to NOP alternative_cb_end __mitigate_spectre_bhb_loop \tmp @@ -904,7 +904,7 @@ alternative_cb_end stp x0, x1, [sp, #-16]! stp x2, x3, [sp, #-16]! mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 -alternative_cb smccc_patch_fw_mitigation_conduit +alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit nop // Patched to SMC/HVC #0 alternative_cb_end ldp x2, x3, [sp], #16 @@ -914,7 +914,7 @@ alternative_cb_end .macro mitigate_spectre_bhb_clear_insn #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY -alternative_cb spectre_bhb_patch_clearbhb +alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb /* Patched to NOP when not supported */ clearbhb isb diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h index fe0db8d416fb2b83173cead513ec25c536b7196e..0890e4f568fb7f9804566fbf82b2b79bea0732df 100644 --- a/arch/arm64/include/asm/atomic_ll_sc.h +++ b/arch/arm64/include/asm/atomic_ll_sc.h @@ -12,19 +12,6 @@ #include -#ifdef CONFIG_ARM64_LSE_ATOMICS -#define __LL_SC_FALLBACK(asm_ops) \ -" b 3f\n" \ -" .subsection 1\n" \ -"3:\n" \ -asm_ops "\n" \ -" b 4f\n" \ -" .previous\n" \ -"4:\n" -#else -#define __LL_SC_FALLBACK(asm_ops) asm_ops -#endif - #ifndef CONFIG_CC_HAS_K_CONSTRAINT #define K #endif @@ -36,38 +23,36 @@ asm_ops "\n" \ */ #define ATOMIC_OP(op, asm_op, constraint) \ -static inline void \ +static __always_inline void \ __ll_sc_atomic_##op(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ \ asm volatile("// atomic_" #op "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %2\n" \ "1: ldxr %w0, %2\n" \ " " #asm_op " %w0, %w0, %w3\n" \ " stxr %w1, %w0, %2\n" \ - " cbnz %w1, 1b\n") \ + " cbnz %w1, 1b\n" \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i)); \ } #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ -static inline int \ +static __always_inline int \ __ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ \ asm volatile("// atomic_" #op "_return" #name "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %2\n" \ "1: ld" #acq "xr %w0, %2\n" \ " " #asm_op " %w0, %w0, %w3\n" \ " st" #rel "xr %w1, %w0, %2\n" \ " cbnz %w1, 1b\n" \ - " " #mb ) \ + " " #mb \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -76,20 +61,19 @@ __ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ } #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \ -static inline int \ +static __always_inline int \ __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \ { \ unsigned long tmp; \ int val, result; \ \ asm volatile("// atomic_fetch_" #op #name "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %3\n" \ "1: ld" #acq "xr %w0, %3\n" \ " " #asm_op " %w1, %w0, %w4\n" \ " st" #rel "xr %w2, %w1, %3\n" \ " cbnz %w2, 1b\n" \ - " " #mb ) \ + " " #mb \ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -135,38 +119,36 @@ ATOMIC_OPS(andnot, bic, ) #undef ATOMIC_OP #define ATOMIC64_OP(op, asm_op, constraint) \ -static inline void \ +static __always_inline void \ __ll_sc_atomic64_##op(s64 i, atomic64_t *v) \ { \ s64 result; \ unsigned long tmp; \ \ asm volatile("// atomic64_" #op "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %2\n" \ "1: ldxr %0, %2\n" \ " " #asm_op " %0, %0, %3\n" \ " stxr %w1, %0, %2\n" \ - " cbnz %w1, 1b") \ + " cbnz %w1, 1b" \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i)); \ } #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ -static inline long \ +static __always_inline long \ __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ { \ s64 result; \ unsigned long tmp; \ \ asm volatile("// atomic64_" #op "_return" #name "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %2\n" \ "1: ld" #acq "xr %0, %2\n" \ " " #asm_op " %0, %0, %3\n" \ " st" #rel "xr %w1, %0, %2\n" \ " cbnz %w1, 1b\n" \ - " " #mb ) \ + " " #mb \ : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -175,20 +157,19 @@ __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ } #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\ -static inline long \ +static __always_inline long \ __ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ { \ s64 result, val; \ unsigned long tmp; \ \ asm volatile("// atomic64_fetch_" #op #name "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %3\n" \ "1: ld" #acq "xr %0, %3\n" \ " " #asm_op " %1, %0, %4\n" \ " st" #rel "xr %w2, %1, %3\n" \ " cbnz %w2, 1b\n" \ - " " #mb ) \ + " " #mb \ : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ : __stringify(constraint) "r" (i) \ : cl); \ @@ -233,14 +214,13 @@ ATOMIC64_OPS(andnot, bic, ) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -static inline s64 +static __always_inline s64 __ll_sc_atomic64_dec_if_positive(atomic64_t *v) { s64 result; unsigned long tmp; asm volatile("// atomic64_dec_if_positive\n" - __LL_SC_FALLBACK( " prfm pstl1strm, %2\n" "1: ldxr %0, %2\n" " subs %0, %0, #1\n" @@ -248,7 +228,7 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v) " stlxr %w1, %0, %2\n" " cbnz %w1, 1b\n" " dmb ish\n" - "2:") + "2:" : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) : : "cc", "memory"); @@ -257,7 +237,7 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v) } #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \ -static inline u##sz \ +static __always_inline u##sz \ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ unsigned long old, \ u##sz new) \ @@ -274,7 +254,6 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ old = (u##sz)old; \ \ asm volatile( \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %[v]\n" \ "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \ " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ @@ -282,7 +261,7 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \ " cbnz %w[tmp], 1b\n" \ " " #mb "\n" \ - "2:") \ + "2:" \ : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ [v] "+Q" (*(u##sz *)ptr) \ : [old] __stringify(constraint) "r" (old), [new] "r" (new) \ @@ -316,7 +295,7 @@ __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) #undef __CMPXCHG_CASE #define __CMPXCHG_DBL(name, mb, rel, cl) \ -static inline long \ +static __always_inline long \ __ll_sc__cmpxchg_double##name(unsigned long old1, \ unsigned long old2, \ unsigned long new1, \ @@ -326,7 +305,6 @@ __ll_sc__cmpxchg_double##name(unsigned long old1, \ unsigned long tmp, ret; \ \ asm volatile("// __cmpxchg_double" #name "\n" \ - __LL_SC_FALLBACK( \ " prfm pstl1strm, %2\n" \ "1: ldxp %0, %1, %2\n" \ " eor %0, %0, %3\n" \ @@ -336,7 +314,7 @@ __ll_sc__cmpxchg_double##name(unsigned long old1, \ " st" #rel "xp %w0, %5, %6, %2\n" \ " cbnz %w0, 1b\n" \ " " #mb "\n" \ - "2:") \ + "2:" \ : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr) \ : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ : cl); \ diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h index 5d460f6b7675e9fa8f68822c7adf1798daba98a0..52075e93de6c01a0b7034e0a14887e43a9167137 100644 --- a/arch/arm64/include/asm/atomic_lse.h +++ b/arch/arm64/include/asm/atomic_lse.h @@ -11,7 +11,8 @@ #define __ASM_ATOMIC_LSE_H #define ATOMIC_OP(op, asm_op) \ -static inline void __lse_atomic_##op(int i, atomic_t *v) \ +static __always_inline void \ +__lse_atomic_##op(int i, atomic_t *v) \ { \ asm volatile( \ __LSE_PREAMBLE \ @@ -25,7 +26,7 @@ ATOMIC_OP(or, stset) ATOMIC_OP(xor, steor) ATOMIC_OP(add, stadd) -static inline void __lse_atomic_sub(int i, atomic_t *v) +static __always_inline void __lse_atomic_sub(int i, atomic_t *v) { __lse_atomic_add(-i, v); } @@ -33,7 +34,8 @@ static inline void __lse_atomic_sub(int i, atomic_t *v) #undef ATOMIC_OP #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \ -static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v) \ +static __always_inline int \ +__lse_atomic_fetch_##op##name(int i, atomic_t *v) \ { \ int old; \ \ @@ -63,7 +65,8 @@ ATOMIC_FETCH_OPS(add, ldadd) #undef ATOMIC_FETCH_OPS #define ATOMIC_FETCH_OP_SUB(name) \ -static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v) \ +static __always_inline int \ +__lse_atomic_fetch_sub##name(int i, atomic_t *v) \ { \ return __lse_atomic_fetch_add##name(-i, v); \ } @@ -76,12 +79,14 @@ ATOMIC_FETCH_OP_SUB( ) #undef ATOMIC_FETCH_OP_SUB #define ATOMIC_OP_ADD_SUB_RETURN(name) \ -static inline int __lse_atomic_add_return##name(int i, atomic_t *v) \ +static __always_inline int \ +__lse_atomic_add_return##name(int i, atomic_t *v) \ { \ return __lse_atomic_fetch_add##name(i, v) + i; \ } \ \ -static inline int __lse_atomic_sub_return##name(int i, atomic_t *v) \ +static __always_inline int \ +__lse_atomic_sub_return##name(int i, atomic_t *v) \ { \ return __lse_atomic_fetch_sub(i, v) - i; \ } @@ -93,13 +98,14 @@ ATOMIC_OP_ADD_SUB_RETURN( ) #undef ATOMIC_OP_ADD_SUB_RETURN -static inline void __lse_atomic_and(int i, atomic_t *v) +static __always_inline void __lse_atomic_and(int i, atomic_t *v) { return __lse_atomic_andnot(~i, v); } #define ATOMIC_FETCH_OP_AND(name, mb, cl...) \ -static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v) \ +static __always_inline int \ +__lse_atomic_fetch_and##name(int i, atomic_t *v) \ { \ return __lse_atomic_fetch_andnot##name(~i, v); \ } @@ -112,7 +118,8 @@ ATOMIC_FETCH_OP_AND( , al, "memory") #undef ATOMIC_FETCH_OP_AND #define ATOMIC64_OP(op, asm_op) \ -static inline void __lse_atomic64_##op(s64 i, atomic64_t *v) \ +static __always_inline void \ +__lse_atomic64_##op(s64 i, atomic64_t *v) \ { \ asm volatile( \ __LSE_PREAMBLE \ @@ -126,7 +133,7 @@ ATOMIC64_OP(or, stset) ATOMIC64_OP(xor, steor) ATOMIC64_OP(add, stadd) -static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) +static __always_inline void __lse_atomic64_sub(s64 i, atomic64_t *v) { __lse_atomic64_add(-i, v); } @@ -134,7 +141,8 @@ static inline void __lse_atomic64_sub(s64 i, atomic64_t *v) #undef ATOMIC64_OP #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \ -static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\ +static __always_inline long \ +__lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ { \ s64 old; \ \ @@ -164,7 +172,8 @@ ATOMIC64_FETCH_OPS(add, ldadd) #undef ATOMIC64_FETCH_OPS #define ATOMIC64_FETCH_OP_SUB(name) \ -static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ +static __always_inline long \ +__lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v) \ { \ return __lse_atomic64_fetch_add##name(-i, v); \ } @@ -177,12 +186,14 @@ ATOMIC64_FETCH_OP_SUB( ) #undef ATOMIC64_FETCH_OP_SUB #define ATOMIC64_OP_ADD_SUB_RETURN(name) \ -static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\ +static __always_inline long \ +__lse_atomic64_add_return##name(s64 i, atomic64_t *v) \ { \ return __lse_atomic64_fetch_add##name(i, v) + i; \ } \ \ -static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)\ +static __always_inline long \ +__lse_atomic64_sub_return##name(s64 i, atomic64_t *v) \ { \ return __lse_atomic64_fetch_sub##name(i, v) - i; \ } @@ -194,13 +205,14 @@ ATOMIC64_OP_ADD_SUB_RETURN( ) #undef ATOMIC64_OP_ADD_SUB_RETURN -static inline void __lse_atomic64_and(s64 i, atomic64_t *v) +static __always_inline void __lse_atomic64_and(s64 i, atomic64_t *v) { return __lse_atomic64_andnot(~i, v); } #define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \ -static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \ +static __always_inline long \ +__lse_atomic64_fetch_and##name(s64 i, atomic64_t *v) \ { \ return __lse_atomic64_fetch_andnot##name(~i, v); \ } @@ -212,7 +224,7 @@ ATOMIC64_FETCH_OP_AND( , al, "memory") #undef ATOMIC64_FETCH_OP_AND -static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v) +static __always_inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v) { unsigned long tmp; diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index ec7720dbe2c801bc44344d7893cfc510fed7d0d7..6e000113e508a452cbecb6cea2762eddda70fa00 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h @@ -17,6 +17,7 @@ * 0x401: for compile time BRK instruction * 0x800: kernel-mode BUG() and WARN() traps * 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff) + * 0x8xxx: Control-Flow Integrity traps */ #define KPROBES_BRK_IMM 0x004 #define UPROBES_BRK_IMM 0x005 @@ -28,4 +29,9 @@ #define KASAN_BRK_IMM 0x900 #define KASAN_BRK_MASK 0x0ff +#define CFI_BRK_IMM_TARGET GENMASK(4, 0) +#define CFI_BRK_IMM_TYPE GENMASK(9, 5) +#define CFI_BRK_IMM_BASE 0x8000 +#define CFI_BRK_IMM_MASK (CFI_BRK_IMM_TARGET | CFI_BRK_IMM_TYPE) + #endif diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index ca9b487112ccbfa6c1c2d55dea2a781004356bdc..c0b178d1bb4f06b6fc99460084c26718798df35b 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -45,10 +45,6 @@ static inline unsigned int arch_slab_minalign(void) #define arch_slab_minalign() arch_slab_minalign() #endif -#define CTR_CACHE_MINLINE_MASK \ - (0xf << CTR_EL0_DMINLINE_SHIFT | \ - CTR_EL0_IMINLINE_MASK << CTR_EL0_IMINLINE_SHIFT) - #define CTR_L1IP(ctr) SYS_FIELD_GET(CTR_EL0, L1Ip, ctr) #define ICACHEF_ALIASING 0 @@ -71,7 +67,7 @@ static __always_inline int icache_is_vpipt(void) static inline u32 cache_type_cwg(void) { - return (read_cpuid_cachetype() >> CTR_EL0_CWG_SHIFT) & CTR_EL0_CWG_MASK; + return SYS_FIELD_GET(CTR_EL0, CWG, read_cpuid_cachetype()); } #define __read_mostly __section(".data..read_mostly") diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index fd7d75a275f6c1c8c66110312d457b26501d9d5e..f73f11b5504254be3eca687824b6869dcc946196 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -6,6 +6,7 @@ #ifndef __ASM_CPUFEATURE_H #define __ASM_CPUFEATURE_H +#include #include #include #include @@ -419,12 +420,8 @@ static __always_inline bool is_hyp_code(void) } extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); -extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; -extern struct static_key_false arm64_const_caps_ready; -/* ARM64 CAPS + alternative_cb */ -#define ARM64_NPATCHABLE (ARM64_NCAPS + 1) -extern DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); +extern DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS); #define for_each_available_cap(cap) \ for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS) @@ -440,7 +437,7 @@ unsigned long cpu_get_elf_hwcap2(void); static __always_inline bool system_capabilities_finalized(void) { - return static_branch_likely(&arm64_const_caps_ready); + return alternative_has_feature_likely(ARM64_ALWAYS_SYSTEM); } /* @@ -448,11 +445,11 @@ static __always_inline bool system_capabilities_finalized(void) * * Before the capability is detected, this returns false. */ -static inline bool cpus_have_cap(unsigned int num) +static __always_inline bool cpus_have_cap(unsigned int num) { if (num >= ARM64_NCAPS) return false; - return test_bit(num, cpu_hwcaps); + return arch_test_bit(num, cpu_hwcaps); } /* @@ -467,7 +464,7 @@ static __always_inline bool __cpus_have_const_cap(int num) { if (num >= ARM64_NCAPS) return false; - return static_branch_unlikely(&cpu_hwcap_keys[num]); + return alternative_has_feature_unlikely(num); } /* @@ -553,7 +550,7 @@ cpuid_feature_cap_perfmon_field(u64 features, int field, u64 cap) u64 mask = GENMASK_ULL(field + 3, field); /* Treat IMPLEMENTATION DEFINED functionality as unimplemented */ - if (val == ID_AA64DFR0_PMUVER_IMP_DEF) + if (val == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) val = 0; if (val > cap) { @@ -597,43 +594,43 @@ static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val) static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) { - return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 || - cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1; + return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_BIGEND_SHIFT) == 0x1 || + cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT) == 0x1; } static inline bool id_aa64pfr0_32bit_el1(u64 pfr0) { - u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_SHIFT); + u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_EL1_SHIFT); - return val == ID_AA64PFR0_ELx_32BIT_64BIT; + return val == ID_AA64PFR0_EL1_ELx_32BIT_64BIT; } static inline bool id_aa64pfr0_32bit_el0(u64 pfr0) { - u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT); + u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_EL0_SHIFT); - return val == ID_AA64PFR0_ELx_32BIT_64BIT; + return val == ID_AA64PFR0_EL1_ELx_32BIT_64BIT; } static inline bool id_aa64pfr0_sve(u64 pfr0) { - u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT); + u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_SVE_SHIFT); return val > 0; } static inline bool id_aa64pfr1_sme(u64 pfr1) { - u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_SME_SHIFT); + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_SME_SHIFT); return val > 0; } static inline bool id_aa64pfr1_mte(u64 pfr1) { - u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT); + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT); - return val >= ID_AA64PFR1_MTE; + return val >= ID_AA64PFR1_EL1_MTE_MTE2; } void __init setup_cpu_features(void); @@ -659,7 +656,7 @@ static inline bool supports_csv2p3(int scope) pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); csv2_val = cpuid_feature_extract_unsigned_field(pfr0, - ID_AA64PFR0_CSV2_SHIFT); + ID_AA64PFR0_EL1_CSV2_SHIFT); return csv2_val == 3; } @@ -694,10 +691,10 @@ static inline bool system_supports_4kb_granule(void) mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); val = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_TGRAN4_SHIFT); + ID_AA64MMFR0_EL1_TGRAN4_SHIFT); - return (val >= ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN) && - (val <= ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX); + return (val >= ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN) && + (val <= ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX); } static inline bool system_supports_64kb_granule(void) @@ -707,10 +704,10 @@ static inline bool system_supports_64kb_granule(void) mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); val = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_TGRAN64_SHIFT); + ID_AA64MMFR0_EL1_TGRAN64_SHIFT); - return (val >= ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN) && - (val <= ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX); + return (val >= ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN) && + (val <= ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX); } static inline bool system_supports_16kb_granule(void) @@ -720,10 +717,10 @@ static inline bool system_supports_16kb_granule(void) mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); val = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_TGRAN16_SHIFT); + ID_AA64MMFR0_EL1_TGRAN16_SHIFT); - return (val >= ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN) && - (val <= ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX); + return (val >= ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN) && + (val <= ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX); } static inline bool system_supports_mixed_endian_el0(void) @@ -738,7 +735,7 @@ static inline bool system_supports_mixed_endian(void) mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); val = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_BIGENDEL_SHIFT); + ID_AA64MMFR0_EL1_BIGEND_SHIFT); return val == 0x1; } @@ -840,13 +837,13 @@ extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange) { switch (parange) { - case ID_AA64MMFR0_PARANGE_32: return 32; - case ID_AA64MMFR0_PARANGE_36: return 36; - case ID_AA64MMFR0_PARANGE_40: return 40; - case ID_AA64MMFR0_PARANGE_42: return 42; - case ID_AA64MMFR0_PARANGE_44: return 44; - case ID_AA64MMFR0_PARANGE_48: return 48; - case ID_AA64MMFR0_PARANGE_52: return 52; + case ID_AA64MMFR0_EL1_PARANGE_32: return 32; + case ID_AA64MMFR0_EL1_PARANGE_36: return 36; + case ID_AA64MMFR0_EL1_PARANGE_40: return 40; + case ID_AA64MMFR0_EL1_PARANGE_42: return 42; + case ID_AA64MMFR0_EL1_PARANGE_44: return 44; + case ID_AA64MMFR0_EL1_PARANGE_48: return 48; + case ID_AA64MMFR0_EL1_PARANGE_52: return 52; /* * A future PE could use a value unknown to the kernel. * However, by the "D10.1.4 Principles of the ID scheme @@ -868,14 +865,14 @@ static inline bool cpu_has_hw_af(void) mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); return cpuid_feature_extract_unsigned_field(mmfr1, - ID_AA64MMFR1_HADBS_SHIFT); + ID_AA64MMFR1_EL1_HAFDBS_SHIFT); } static inline bool cpu_has_pan(void) { u64 mmfr1 = read_cpuid(ID_AA64MMFR1_EL1); return cpuid_feature_extract_unsigned_field(mmfr1, - ID_AA64MMFR1_PAN_SHIFT); + ID_AA64MMFR1_EL1_PAN_SHIFT); } #ifdef CONFIG_ARM64_AMU_EXTN @@ -896,8 +893,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) int vmid_bits; vmid_bits = cpuid_feature_extract_unsigned_field(mmfr1, - ID_AA64MMFR1_VMIDBITS_SHIFT); - if (vmid_bits == ID_AA64MMFR1_VMIDBITS_16) + ID_AA64MMFR1_EL1_VMIDBits_SHIFT); + if (vmid_bits == ID_AA64MMFR1_EL1_VMIDBits_16) return 16; /* @@ -907,6 +904,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); + extern struct arm64_ftr_override id_aa64mmfr1_override; extern struct arm64_ftr_override id_aa64pfr0_override; extern struct arm64_ftr_override id_aa64pfr1_override; diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 8aa0d276a63628f51d1d533c0f7a9e41fc9bfdb0..abc418650fec04fda7f79c2c46f3ec255124781d 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -60,6 +60,7 @@ #define ARM_CPU_IMP_FUJITSU 0x46 #define ARM_CPU_IMP_HISI 0x48 #define ARM_CPU_IMP_APPLE 0x61 +#define ARM_CPU_IMP_AMPERE 0xC0 #define ARM_CPU_PART_AEM_V8 0xD0F #define ARM_CPU_PART_FOUNDATION 0xD00 @@ -123,6 +124,8 @@ #define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 #define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 +#define AMPERE_CPU_PART_AMPERE1 0xAC3 + #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) #define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) @@ -172,6 +175,7 @@ #define MIDR_APPLE_M1_FIRESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) #define MIDR_APPLE_M1_ICESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) #define MIDR_APPLE_M1_FIRESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) +#define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ #define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 2630faa5bc08fc5879cb6595fb39f8fd2d44ab42..668569adf4d33138849ba9f93d3035a4afee0f7f 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -40,7 +40,7 @@ .macro __init_el2_debug mrs x1, id_aa64dfr0_el1 - sbfx x0, x1, #ID_AA64DFR0_PMUVER_SHIFT, #4 + sbfx x0, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 cmp x0, #1 b.lt .Lskip_pmu_\@ // Skip if no PMU present mrs x0, pmcr_el0 // Disable debug access traps @@ -49,7 +49,7 @@ csel x2, xzr, x0, lt // all PMU counters from EL1 /* Statistical profiling */ - ubfx x0, x1, #ID_AA64DFR0_PMSVER_SHIFT, #4 + ubfx x0, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4 cbz x0, .Lskip_spe_\@ // Skip if SPE not present mrs_s x0, SYS_PMBIDR_EL1 // If SPE available at EL2, @@ -65,7 +65,7 @@ .Lskip_spe_\@: /* Trace buffer */ - ubfx x0, x1, #ID_AA64DFR0_TRBE_SHIFT, #4 + ubfx x0, x1, #ID_AA64DFR0_EL1_TraceBuffer_SHIFT, #4 cbz x0, .Lskip_trace_\@ // Skip if TraceBuffer is not present mrs_s x0, SYS_TRBIDR_EL1 @@ -83,7 +83,7 @@ /* LORegions */ .macro __init_el2_lor mrs x1, id_aa64mmfr1_el1 - ubfx x0, x1, #ID_AA64MMFR1_LOR_SHIFT, 4 + ubfx x0, x1, #ID_AA64MMFR1_EL1_LO_SHIFT, 4 cbz x0, .Lskip_lor_\@ msr_s SYS_LORC_EL1, xzr .Lskip_lor_\@: @@ -97,7 +97,7 @@ /* GICv3 system register access */ .macro __init_el2_gicv3 mrs x0, id_aa64pfr0_el1 - ubfx x0, x0, #ID_AA64PFR0_GIC_SHIFT, #4 + ubfx x0, x0, #ID_AA64PFR0_EL1_GIC_SHIFT, #4 cbz x0, .Lskip_gicv3_\@ mrs_s x0, SYS_ICC_SRE_EL2 @@ -132,12 +132,12 @@ /* Disable any fine grained traps */ .macro __init_el2_fgt mrs x1, id_aa64mmfr0_el1 - ubfx x1, x1, #ID_AA64MMFR0_FGT_SHIFT, #4 + ubfx x1, x1, #ID_AA64MMFR0_EL1_FGT_SHIFT, #4 cbz x1, .Lskip_fgt_\@ mov x0, xzr mrs x1, id_aa64dfr0_el1 - ubfx x1, x1, #ID_AA64DFR0_PMSVER_SHIFT, #4 + ubfx x1, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4 cmp x1, #3 b.lt .Lset_debug_fgt_\@ /* Disable PMSNEVFR_EL1 read and write traps */ @@ -149,7 +149,7 @@ mov x0, xzr mrs x1, id_aa64pfr1_el1 - ubfx x1, x1, #ID_AA64PFR1_SME_SHIFT, #4 + ubfx x1, x1, #ID_AA64PFR1_EL1_SME_SHIFT, #4 cbz x1, .Lset_fgt_\@ /* Disable nVHE traps of TPIDR2 and SMPRI */ @@ -162,7 +162,7 @@ msr_s SYS_HFGITR_EL2, xzr mrs x1, id_aa64pfr0_el1 // AMU traps UNDEF without AMU - ubfx x1, x1, #ID_AA64PFR0_AMU_SHIFT, #4 + ubfx x1, x1, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 cbz x1, .Lskip_fgt_\@ msr_s SYS_HAFGRTR_EL2, xzr diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index d94aecff969029c90154c04f4e3333a952f7a569..19713d0f013b7635341e0ddc34769c076626bfe9 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -58,8 +58,9 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs, asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs); void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs); -void do_undefinstr(struct pt_regs *regs); -void do_bti(struct pt_regs *regs); +void do_undefinstr(struct pt_regs *regs, unsigned long esr); +void do_el0_bti(struct pt_regs *regs); +void do_el1_bti(struct pt_regs *regs, unsigned long esr); void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr, struct pt_regs *regs); void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs); @@ -70,9 +71,11 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs); void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs); void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr); void do_cp15instr(unsigned long esr, struct pt_regs *regs); +int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs); void do_el0_svc(struct pt_regs *regs); void do_el0_svc_compat(struct pt_regs *regs); -void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr); +void do_el0_fpac(struct pt_regs *regs, unsigned long esr); +void do_el1_fpac(struct pt_regs *regs, unsigned long esr); void do_serror(struct pt_regs *regs, unsigned long esr); void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags); diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 9bb1873f529515a9ec9e1ee487a4e9e8a02e1f3b..6f86b7ab6c28f00d38573336c9e0425df75d80c1 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -153,7 +153,7 @@ struct vl_info { #ifdef CONFIG_ARM64_SVE -extern void sve_alloc(struct task_struct *task); +extern void sve_alloc(struct task_struct *task, bool flush); extern void fpsimd_release_task(struct task_struct *task); extern void fpsimd_sync_to_sve(struct task_struct *task); extern void fpsimd_force_sync_to_sve(struct task_struct *task); @@ -256,7 +256,7 @@ size_t sve_state_size(struct task_struct const *task); #else /* ! CONFIG_ARM64_SVE */ -static inline void sve_alloc(struct task_struct *task) { } +static inline void sve_alloc(struct task_struct *task, bool flush) { } static inline void fpsimd_release_task(struct task_struct *task) { } static inline void sve_sync_to_fpsimd(struct task_struct *task) { } static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { } diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index dbc45a4157fa7b955df2cdfb6c22446d8ade36ae..329dbbd4d50b6d6efd76b73a2b93eb02828967ed 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -26,7 +26,7 @@ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS #define ARCH_SUPPORTS_FTRACE_OPS 1 #else -#define MCOUNT_ADDR ((unsigned long)function_nocfi(_mcount)) +#define MCOUNT_ADDR ((unsigned long)_mcount) #endif /* The BL at the callsite's adjusted rec->ip */ diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index bc7aaed4b34e958386ae844a6795ba6d576bc182..fa4c6ff3aa9bde5e117a37f123c8268a605b186f 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -142,7 +142,7 @@ static inline int get_num_brps(void) u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); return 1 + cpuid_feature_extract_unsigned_field(dfr0, - ID_AA64DFR0_BRPS_SHIFT); + ID_AA64DFR0_EL1_BRPs_SHIFT); } /* Determine number of WRP registers available. */ @@ -151,7 +151,7 @@ static inline int get_num_wrps(void) u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); return 1 + cpuid_feature_extract_unsigned_field(dfr0, - ID_AA64DFR0_WRPS_SHIFT); + ID_AA64DFR0_EL1_WRPs_SHIFT); } #endif /* __ASM_BREAKPOINT_H */ diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index cef4ae7a3d8b1f7191dd310e6ffd17b10555baec..298b386d3ebe326c60d52c458c88c45d38df3cc6 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -119,6 +119,7 @@ #define KERNEL_HWCAP_SME_FA64 __khwcap2_feature(SME_FA64) #define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT) #define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16) +#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 02e59fa8f2930c8fa81d54311c4fefe55277d001..32d14f481f0c3f375e6ff011e6be840f51ed641d 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -64,28 +64,28 @@ #define EARLY_KASLR (0) #endif -#define EARLY_ENTRIES(vstart, vend, shift) \ - ((((vend) - 1) >> (shift)) - ((vstart) >> (shift)) + 1 + EARLY_KASLR) +#define EARLY_ENTRIES(vstart, vend, shift, add) \ + ((((vend) - 1) >> (shift)) - ((vstart) >> (shift)) + 1 + add) -#define EARLY_PGDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT)) +#define EARLY_PGDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT, add)) #if SWAPPER_PGTABLE_LEVELS > 3 -#define EARLY_PUDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT)) +#define EARLY_PUDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT, add)) #else -#define EARLY_PUDS(vstart, vend) (0) +#define EARLY_PUDS(vstart, vend, add) (0) #endif #if SWAPPER_PGTABLE_LEVELS > 2 -#define EARLY_PMDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT)) +#define EARLY_PMDS(vstart, vend, add) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT, add)) #else -#define EARLY_PMDS(vstart, vend) (0) +#define EARLY_PMDS(vstart, vend, add) (0) #endif -#define EARLY_PAGES(vstart, vend) ( 1 /* PGDIR page */ \ - + EARLY_PGDS((vstart), (vend)) /* each PGDIR needs a next level page table */ \ - + EARLY_PUDS((vstart), (vend)) /* each PUD needs a next level page table */ \ - + EARLY_PMDS((vstart), (vend))) /* each PMD needs a next level page table */ -#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end)) +#define EARLY_PAGES(vstart, vend, add) ( 1 /* PGDIR page */ \ + + EARLY_PGDS((vstart), (vend), add) /* each PGDIR needs a next level page table */ \ + + EARLY_PUDS((vstart), (vend), add) /* each PUD needs a next level page table */ \ + + EARLY_PMDS((vstart), (vend), add)) /* each PMD needs a next level page table */ +#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end, EARLY_KASLR)) /* the initial ID map may need two extra pages if it needs to be extended */ #if VA_BITS < 48 @@ -93,7 +93,7 @@ #else #define INIT_IDMAP_DIR_SIZE (INIT_IDMAP_DIR_PAGES * PAGE_SIZE) #endif -#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE) +#define INIT_IDMAP_DIR_PAGES EARLY_PAGES(KIMAGE_VADDR, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE, 1) /* Initial memory map size */ #if ARM64_KERNEL_USES_PMD_MAPS diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index e9c9388ccc024e5fd9b1dfcb932af2140ece0950..45e2136322ba264eb53ba1f3538df5d2c11bdb83 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -393,6 +393,7 @@ struct kvm_vcpu_arch { */ struct { u32 mdscr_el1; + bool pstate_ss; } guest_debug_preserved; /* vcpu power state */ @@ -535,6 +536,9 @@ struct kvm_vcpu_arch { #define IN_WFIT __vcpu_single_flag(sflags, BIT(3)) /* vcpu system registers loaded on physical CPU */ #define SYSREGS_ON_CPU __vcpu_single_flag(sflags, BIT(4)) +/* Software step state is Active-pending */ +#define DBG_SS_ACTIVE_PENDING __vcpu_single_flag(sflags, BIT(5)) + /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */ #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \ diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index b208da3bebec82e7b59467f155561ce3a1cc3047..7784081088e78f84c7aad3aaf9553a276a6d1b8a 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -63,7 +63,7 @@ * specific registers encoded in the instructions). */ .macro kern_hyp_va reg -alternative_cb kvm_update_va_mask +alternative_cb ARM64_ALWAYS_SYSTEM, kvm_update_va_mask and \reg, \reg, #1 /* mask with va_mask */ ror \reg, \reg, #1 /* rotate to the first tag bit */ add \reg, \reg, #0 /* insert the low 12 bits of the tag */ @@ -97,7 +97,7 @@ alternative_cb_end hyp_pa \reg, \tmp /* Load kimage_voffset. */ -alternative_cb kvm_get_kimage_voffset +alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset movz \tmp, #0 movk \tmp, #0, lsl #16 movk \tmp, #0, lsl #32 @@ -131,6 +131,7 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) "add %0, %0, #0\n" "add %0, %0, #0, lsl 12\n" "ror %0, %0, #63\n", + ARM64_ALWAYS_SYSTEM, kvm_update_va_mask) : "+r" (v)); return v; diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 9f339dffbc1a7e12c29df923badbb66dde48f60f..1b098bd4cd37826539b899fb77e31aff87827be6 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -16,9 +16,9 @@ static inline u64 kvm_get_parange(u64 mmfr0) { u64 parange = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_PARANGE_SHIFT); - if (parange > ID_AA64MMFR0_PARANGE_MAX) - parange = ID_AA64MMFR0_PARANGE_MAX; + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + if (parange > ID_AA64MMFR0_EL1_PARANGE_MAX) + parange = ID_AA64MMFR0_EL1_PARANGE_MAX; return parange; } diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index 43f8c25b3fda655577859cf7a8ce59c1a049ed6b..1436fa1cde24d7edb0bc1b389ea8d631a4108e0f 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -39,4 +39,8 @@ SYM_START(name, SYM_L_WEAK, SYM_A_NONE) \ bti c ; +#define SYM_TYPED_FUNC_START(name) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + bti c ; + #endif diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h index 29c85810ae69052e3198e79ff4770b5f05d97177..c503db8e73b01710d935d556143272b846d5f786 100644 --- a/arch/arm64/include/asm/lse.h +++ b/arch/arm64/include/asm/lse.h @@ -13,14 +13,13 @@ #include #include #include +#include #include #include -extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS]; - static __always_inline bool system_uses_lse_atomics(void) { - return static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]); + return alternative_has_feature_likely(ARM64_HAS_LSE_ATOMICS); } #define __lse_ll_sc_body(op, ...) \ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index c7ccd82db1d24bb3b8a3dfe7b95ff25bde59dc70..d3f8b5df0c1fe31532eb270f2b6cec2b33872625 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -147,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz) * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD, * avoiding the possibility of conflicting TLB entries being allocated. */ -static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) +static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) { typedef void (ttbr_replace_func)(phys_addr_t); extern ttbr_replace_func idmap_cpu_replace_ttbr1; @@ -168,7 +168,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap) ttbr1 |= TTBR_CNP_BIT; } - replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1)); + replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1); __cpu_install_idmap(idmap); replace_phys(ttbr1); diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h index 4e7fa2623896b008afb775c2cd4d41f7a72491c5..18734fed3bdd7609bf02b6ebbffc338e80591206 100644 --- a/arch/arm64/include/asm/module.h +++ b/arch/arm64/include/asm/module.h @@ -58,11 +58,20 @@ static inline bool is_forbidden_offset_for_adrp(void *place) } struct plt_entry get_plt_entry(u64 dst, void *pc); -bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b); -static inline bool plt_entry_is_initialized(const struct plt_entry *e) +static inline const Elf_Shdr *find_section(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, + const char *name) { - return e->adrp || e->add || e->br; + const Elf_Shdr *s, *se; + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { + if (strcmp(name, secstrs + s->sh_name) == 0) + return s; + } + + return NULL; } #endif /* __ASM_MODULE_H */ diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index aa523591a44e5635ee1d2ed2fd4f22e418513fa6..760c62f8e22f84f95c48de4f1470d9cea67ca536 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -42,7 +42,9 @@ void mte_sync_tags(pte_t old_pte, pte_t pte); void mte_copy_page_tags(void *kto, const void *kfrom); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); +void mte_cpu_setup(void); void mte_suspend_enter(void); +void mte_suspend_exit(void); long set_mte_ctrl(struct task_struct *task, unsigned long arg); long get_mte_ctrl(struct task_struct *task); int mte_ptrace_copy_tags(struct task_struct *child, long request, @@ -72,6 +74,9 @@ static inline void mte_thread_switch(struct task_struct *next) static inline void mte_suspend_enter(void) { } +static inline void mte_suspend_exit(void) +{ +} static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg) { return 0; diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index b5df82aa99e64ba30cafa436c4ae5181cd6894eb..71a1af42f0e89775163c9d6428c22901b9a6f1e9 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -1082,24 +1082,13 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, * page after fork() + CoW for pfn mappings. We don't always have a * hardware-managed access flag on arm64. */ -static inline bool arch_faults_on_old_pte(void) -{ - /* The register read below requires a stable CPU to make any sense */ - cant_migrate(); - - return !cpu_has_hw_af(); -} -#define arch_faults_on_old_pte arch_faults_on_old_pte +#define arch_has_hw_pte_young cpu_has_hw_af /* * Experimentally, it's cheap to set the access flag in hardware and we * benefit from prefaulting mappings as 'old' to start with. */ -static inline bool arch_wants_old_prefaulted_pte(void) -{ - return !arch_faults_on_old_pte(); -} -#define arch_wants_old_prefaulted_pte arch_wants_old_prefaulted_pte +#define arch_wants_old_prefaulted_pte cpu_has_hw_af static inline bool pud_sect_supported(void) { diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 86eb0bfe3b380a562299216e8862a58edf2ca9b7..445aa3af3b76239e724f0b8ea09284512961a753 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -323,9 +323,6 @@ static inline bool is_ttbr1_addr(unsigned long addr) /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - unsigned long __get_wchan(struct task_struct *p); void update_sctlr_el1(u64 sctlr); @@ -410,7 +407,7 @@ long get_tagged_addr_ctrl(struct task_struct *task); * The top of the current task's task stack */ #define current_top_of_stack() ((unsigned long)current->stack + THREAD_SIZE) -#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1, NULL)) +#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1)) #endif /* __ASSEMBLY__ */ #endif /* __ASM_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h index 7bea1d705dd6408cf53c41b9491d8b68b1851730..4292d9bafb9d2f85f8106fb6cba63306b614c4fd 100644 --- a/arch/arm64/include/asm/sdei.h +++ b/arch/arm64/include/asm/sdei.h @@ -43,22 +43,5 @@ unsigned long do_sdei_event(struct pt_regs *regs, unsigned long sdei_arch_get_entry_point(int conduit); #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) -struct stack_info; - -bool _on_sdei_stack(unsigned long sp, unsigned long size, - struct stack_info *info); -static inline bool on_sdei_stack(unsigned long sp, unsigned long size, - struct stack_info *info) -{ - if (!IS_ENABLED(CONFIG_VMAP_STACK)) - return false; - if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE)) - return false; - if (in_nmi()) - return _on_sdei_stack(sp, size, info); - - return false; -} - #endif /* __ASSEMBLY__ */ #endif /* __ASM_SDEI_H */ diff --git a/arch/arm64/include/asm/setup.h b/arch/arm64/include/asm/setup.h index 6437df661700920473aa9ccb1f217985436f856c..f4af547ef54caa70a521387d442ffeb6d04a230b 100644 --- a/arch/arm64/include/asm/setup.h +++ b/arch/arm64/include/asm/setup.h @@ -3,6 +3,8 @@ #ifndef __ARM64_ASM_SETUP_H #define __ARM64_ASM_SETUP_H +#include + #include void *get_early_fdt_ptr(void); @@ -14,4 +16,19 @@ void early_fdt_map(u64 dt_phys); extern phys_addr_t __fdt_pointer __initdata; extern u64 __cacheline_aligned boot_args[4]; +static inline bool arch_parse_debug_rodata(char *arg) +{ + extern bool rodata_enabled; + extern bool rodata_full; + + if (arg && !strcmp(arg, "full")) { + rodata_enabled = true; + rodata_full = true; + return true; + } + + return false; +} +#define arch_parse_debug_rodata arch_parse_debug_rodata + #endif diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h index 6ebdcdff77f56c5e8502e45cb04daa2856954b7f..5a0edb064ea478bbf18c0abfb8ac2623a8ad5f2f 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -22,39 +22,86 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, DECLARE_PER_CPU(unsigned long *, irq_stack_ptr); -static inline bool on_irq_stack(unsigned long sp, unsigned long size, - struct stack_info *info) +static inline struct stack_info stackinfo_get_irq(void) { unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr); unsigned long high = low + IRQ_STACK_SIZE; - return on_stack(sp, size, low, high, STACK_TYPE_IRQ, info); + return (struct stack_info) { + .low = low, + .high = high, + }; } -static inline bool on_task_stack(const struct task_struct *tsk, - unsigned long sp, unsigned long size, - struct stack_info *info) +static inline bool on_irq_stack(unsigned long sp, unsigned long size) +{ + struct stack_info info = stackinfo_get_irq(); + return stackinfo_on_stack(&info, sp, size); +} + +static inline struct stack_info stackinfo_get_task(const struct task_struct *tsk) { unsigned long low = (unsigned long)task_stack_page(tsk); unsigned long high = low + THREAD_SIZE; - return on_stack(sp, size, low, high, STACK_TYPE_TASK, info); + return (struct stack_info) { + .low = low, + .high = high, + }; +} + +static inline bool on_task_stack(const struct task_struct *tsk, + unsigned long sp, unsigned long size) +{ + struct stack_info info = stackinfo_get_task(tsk); + return stackinfo_on_stack(&info, sp, size); } #ifdef CONFIG_VMAP_STACK DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); -static inline bool on_overflow_stack(unsigned long sp, unsigned long size, - struct stack_info *info) +static inline struct stack_info stackinfo_get_overflow(void) { unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack); unsigned long high = low + OVERFLOW_STACK_SIZE; - return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info); + return (struct stack_info) { + .low = low, + .high = high, + }; +} +#else +#define stackinfo_get_overflow() stackinfo_get_unknown() +#endif + +#if defined(CONFIG_ARM_SDE_INTERFACE) && defined(CONFIG_VMAP_STACK) +DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr); +DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr); + +static inline struct stack_info stackinfo_get_sdei_normal(void) +{ + unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr); + unsigned long high = low + SDEI_STACK_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; +} + +static inline struct stack_info stackinfo_get_sdei_critical(void) +{ + unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr); + unsigned long high = low + SDEI_STACK_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; } #else -static inline bool on_overflow_stack(unsigned long sp, unsigned long size, - struct stack_info *info) { return false; } +#define stackinfo_get_sdei_normal() stackinfo_get_unknown() +#define stackinfo_get_sdei_critical() stackinfo_get_unknown() #endif #endif /* __ASM_STACKTRACE_H */ diff --git a/arch/arm64/include/asm/stacktrace/common.h b/arch/arm64/include/asm/stacktrace/common.h index f58eb944c46fb11c78d541892561fa1d904e5fdf..508f734de46ee207e9f7e28a47ad5f55624655ed 100644 --- a/arch/arm64/include/asm/stacktrace/common.h +++ b/arch/arm64/include/asm/stacktrace/common.h @@ -2,13 +2,6 @@ /* * Common arm64 stack unwinder code. * - * To implement a new arm64 stack unwinder: - * 1) Include this header - * - * 2) Call into unwind_next_common() from your top level unwind - * function, passing it the validation and translation callbacks - * (though the later can be NULL if no translation is required). - * * See: arch/arm64/kernel/stacktrace.c for the reference implementation. * * Copyright (C) 2012 ARM Ltd. @@ -16,78 +9,60 @@ #ifndef __ASM_STACKTRACE_COMMON_H #define __ASM_STACKTRACE_COMMON_H -#include -#include #include #include -enum stack_type { - STACK_TYPE_UNKNOWN, - STACK_TYPE_TASK, - STACK_TYPE_IRQ, - STACK_TYPE_OVERFLOW, - STACK_TYPE_SDEI_NORMAL, - STACK_TYPE_SDEI_CRITICAL, - STACK_TYPE_HYP, - __NR_STACK_TYPES -}; - struct stack_info { unsigned long low; unsigned long high; - enum stack_type type; }; -/* - * A snapshot of a frame record or fp/lr register values, along with some - * accounting information necessary for robust unwinding. +/** + * struct unwind_state - state used for robust unwinding. * * @fp: The fp value in the frame record (or the real fp) * @pc: The lr value in the frame record (or the real lr) * - * @stacks_done: Stacks which have been entirely unwound, for which it is no - * longer valid to unwind to. - * - * @prev_fp: The fp that pointed to this frame record, or a synthetic value - * of 0. This is used to ensure that within a stack, each - * subsequent frame record is at an increasing address. - * @prev_type: The type of stack this frame record was on, or a synthetic - * value of STACK_TYPE_UNKNOWN. This is used to detect a - * transition from one stack to another. - * * @kr_cur: When KRETPROBES is selected, holds the kretprobe instance * associated with the most recently encountered replacement lr * value. * * @task: The task being unwound. + * + * @stack: The stack currently being unwound. + * @stacks: An array of stacks which can be unwound. + * @nr_stacks: The number of stacks in @stacks. */ struct unwind_state { unsigned long fp; unsigned long pc; - DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES); - unsigned long prev_fp; - enum stack_type prev_type; #ifdef CONFIG_KRETPROBES struct llist_node *kr_cur; #endif struct task_struct *task; + + struct stack_info stack; + struct stack_info *stacks; + int nr_stacks; }; -static inline bool on_stack(unsigned long sp, unsigned long size, - unsigned long low, unsigned long high, - enum stack_type type, struct stack_info *info) +static inline struct stack_info stackinfo_get_unknown(void) +{ + return (struct stack_info) { + .low = 0, + .high = 0, + }; +} + +static inline bool stackinfo_on_stack(const struct stack_info *info, + unsigned long sp, unsigned long size) { - if (!low) + if (!info->low) return false; - if (sp < low || sp + size < sp || sp + size > high) + if (sp < info->low || sp + size < sp || sp + size > info->high) return false; - if (info) { - info->low = low; - info->high = high; - info->type = type; - } return true; } @@ -99,99 +74,101 @@ static inline void unwind_init_common(struct unwind_state *state, state->kr_cur = NULL; #endif - /* - * Prime the first unwind. - * - * In unwind_next() we'll check that the FP points to a valid stack, - * which can't be STACK_TYPE_UNKNOWN, and the first unwind will be - * treated as a transition to whichever stack that happens to be. The - * prev_fp value won't be used, but we set it to 0 such that it is - * definitely not an accessible stack address. - */ - bitmap_zero(state->stacks_done, __NR_STACK_TYPES); - state->prev_fp = 0; - state->prev_type = STACK_TYPE_UNKNOWN; + state->stack = stackinfo_get_unknown(); } -/* - * stack_trace_translate_fp_fn() - Translates a non-kernel frame pointer to - * a kernel address. - * - * @fp: the frame pointer to be updated to its kernel address. - * @type: the stack type associated with frame pointer @fp - * - * Returns true and success and @fp is updated to the corresponding - * kernel virtual address; otherwise returns false. - */ -typedef bool (*stack_trace_translate_fp_fn)(unsigned long *fp, - enum stack_type type); +static struct stack_info *unwind_find_next_stack(const struct unwind_state *state, + unsigned long sp, + unsigned long size) +{ + for (int i = 0; i < state->nr_stacks; i++) { + struct stack_info *info = &state->stacks[i]; -/* - * on_accessible_stack_fn() - Check whether a stack range is on any - * of the possible stacks. + if (stackinfo_on_stack(info, sp, size)) + return info; + } + + return NULL; +} + +/** + * unwind_consume_stack() - Check if an object is on an accessible stack, + * updating stack boundaries so that future unwind steps cannot consume this + * object again. * - * @tsk: task whose stack is being unwound - * @sp: stack address being checked - * @size: size of the stack range being checked - * @info: stack unwinding context + * @state: the current unwind state. + * @sp: the base address of the object. + * @size: the size of the object. + * + * Return: 0 upon success, an error code otherwise. */ -typedef bool (*on_accessible_stack_fn)(const struct task_struct *tsk, - unsigned long sp, unsigned long size, - struct stack_info *info); - -static inline int unwind_next_common(struct unwind_state *state, - struct stack_info *info, - on_accessible_stack_fn accessible, - stack_trace_translate_fp_fn translate_fp) +static inline int unwind_consume_stack(struct unwind_state *state, + unsigned long sp, + unsigned long size) { - unsigned long fp = state->fp, kern_fp = fp; - struct task_struct *tsk = state->task; + struct stack_info *next; - if (fp & 0x7) - return -EINVAL; + if (stackinfo_on_stack(&state->stack, sp, size)) + goto found; - if (!accessible(tsk, fp, 16, info)) - return -EINVAL; - - if (test_bit(info->type, state->stacks_done)) + next = unwind_find_next_stack(state, sp, size); + if (!next) return -EINVAL; /* - * If fp is not from the current address space perform the necessary - * translation before dereferencing it to get the next fp. - */ - if (translate_fp && !translate_fp(&kern_fp, info->type)) - return -EINVAL; - - /* - * As stacks grow downward, any valid record on the same stack must be - * at a strictly higher address than the prior record. + * Stack transitions are strictly one-way, and once we've + * transitioned from one stack to another, it's never valid to + * unwind back to the old stack. + * + * Remove the current stack from the list of stacks so that it cannot + * be found on a subsequent transition. * - * Stacks can nest in several valid orders, e.g. + * Note that stacks can nest in several valid orders, e.g. * - * TASK -> IRQ -> OVERFLOW -> SDEI_NORMAL - * TASK -> SDEI_NORMAL -> SDEI_CRITICAL -> OVERFLOW - * HYP -> OVERFLOW + * TASK -> IRQ -> OVERFLOW -> SDEI_NORMAL + * TASK -> SDEI_NORMAL -> SDEI_CRITICAL -> OVERFLOW + * HYP -> OVERFLOW * - * ... but the nesting itself is strict. Once we transition from one - * stack to another, it's never valid to unwind back to that first - * stack. + * ... so we do not check the specific order of stack + * transitions. */ - if (info->type == state->prev_type) { - if (fp <= state->prev_fp) - return -EINVAL; - } else { - __set_bit(state->prev_type, state->stacks_done); - } + state->stack = *next; + *next = stackinfo_get_unknown(); + +found: + /* + * Future unwind steps can only consume stack above this frame record. + * Update the current stack to start immediately above it. + */ + state->stack.low = sp + size; + return 0; +} + +/** + * unwind_next_frame_record() - Unwind to the next frame record. + * + * @state: the current unwind state. + * + * Return: 0 upon success, an error code otherwise. + */ +static inline int +unwind_next_frame_record(struct unwind_state *state) +{ + unsigned long fp = state->fp; + int err; + + if (fp & 0x7) + return -EINVAL; + + err = unwind_consume_stack(state, fp, 16); + if (err) + return err; /* - * Record this frame record's values and location. The prev_fp and - * prev_type are only meaningful to the next unwind_next() invocation. + * Record this frame record's values. */ - state->fp = READ_ONCE(*(unsigned long *)(kern_fp)); - state->pc = READ_ONCE(*(unsigned long *)(kern_fp + 8)); - state->prev_fp = fp; - state->prev_type = info->type; + state->fp = READ_ONCE(*(unsigned long *)(fp)); + state->pc = READ_ONCE(*(unsigned long *)(fp + 8)); return 0; } diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h index d5527b60039086518f4ad8f1b2fceb076548cf9d..25ab83a315a76d7b8efac158858b7b4e29acc573 100644 --- a/arch/arm64/include/asm/stacktrace/nvhe.h +++ b/arch/arm64/include/asm/stacktrace/nvhe.h @@ -20,8 +20,8 @@ #include -/* - * kvm_nvhe_unwind_init - Start an unwind from the given nVHE HYP fp and pc +/** + * kvm_nvhe_unwind_init() - Start an unwind from the given nVHE HYP fp and pc * * @state : unwind_state to initialize * @fp : frame pointer at which to start the unwinding. diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7c71358d44c4ab811fc05af7ff6d405aa737d4cf..7d301700d1a9369216889c39c6577cdc11f2cec8 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -190,19 +190,6 @@ #define SYS_MVFR1_EL1 sys_reg(3, 0, 0, 3, 1) #define SYS_MVFR2_EL1 sys_reg(3, 0, 0, 3, 2) -#define SYS_ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 4, 0) -#define SYS_ID_AA64PFR1_EL1 sys_reg(3, 0, 0, 4, 1) - -#define SYS_ID_AA64DFR0_EL1 sys_reg(3, 0, 0, 5, 0) -#define SYS_ID_AA64DFR1_EL1 sys_reg(3, 0, 0, 5, 1) - -#define SYS_ID_AA64AFR0_EL1 sys_reg(3, 0, 0, 5, 4) -#define SYS_ID_AA64AFR1_EL1 sys_reg(3, 0, 0, 5, 5) - -#define SYS_ID_AA64MMFR0_EL1 sys_reg(3, 0, 0, 7, 0) -#define SYS_ID_AA64MMFR1_EL1 sys_reg(3, 0, 0, 7, 1) -#define SYS_ID_AA64MMFR2_EL1 sys_reg(3, 0, 0, 7, 2) - #define SYS_ACTLR_EL1 sys_reg(3, 0, 1, 0, 1) #define SYS_RGSR_EL1 sys_reg(3, 0, 1, 0, 5) #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) @@ -436,19 +423,11 @@ #define SYS_ICC_IGRPEN0_EL1 sys_reg(3, 0, 12, 12, 6) #define SYS_ICC_IGRPEN1_EL1 sys_reg(3, 0, 12, 12, 7) -#define SYS_TPIDR_EL1 sys_reg(3, 0, 13, 0, 4) - -#define SYS_SCXTNUM_EL1 sys_reg(3, 0, 13, 0, 7) - #define SYS_CNTKCTL_EL1 sys_reg(3, 0, 14, 1, 0) #define SYS_CCSIDR_EL1 sys_reg(3, 1, 0, 0, 0) #define SYS_AIDR_EL1 sys_reg(3, 1, 0, 0, 7) -#define SMIDR_EL1_IMPLEMENTER_SHIFT 24 -#define SMIDR_EL1_SMPS_SHIFT 15 -#define SMIDR_EL1_AFFINITY_SHIFT 0 - #define SYS_RNDR_EL0 sys_reg(3, 3, 2, 4, 0) #define SYS_RNDRRS_EL0 sys_reg(3, 3, 2, 4, 1) @@ -537,7 +516,6 @@ #define SYS_HFGWTR_EL2 sys_reg(3, 4, 1, 1, 5) #define SYS_HFGITR_EL2 sys_reg(3, 4, 1, 1, 6) #define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) -#define SYS_HCRX_EL2 sys_reg(3, 4, 1, 2, 2) #define SYS_HDFGRTR_EL2 sys_reg(3, 4, 3, 1, 4) #define SYS_HDFGWTR_EL2 sys_reg(3, 4, 3, 1, 5) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) @@ -690,164 +668,30 @@ #define MAIR_ATTRIDX(attr, idx) ((attr) << ((idx) * 8)) /* id_aa64pfr0 */ -#define ID_AA64PFR0_CSV3_SHIFT 60 -#define ID_AA64PFR0_CSV2_SHIFT 56 -#define ID_AA64PFR0_DIT_SHIFT 48 -#define ID_AA64PFR0_AMU_SHIFT 44 -#define ID_AA64PFR0_MPAM_SHIFT 40 -#define ID_AA64PFR0_SEL2_SHIFT 36 -#define ID_AA64PFR0_SVE_SHIFT 32 -#define ID_AA64PFR0_RAS_SHIFT 28 -#define ID_AA64PFR0_GIC_SHIFT 24 -#define ID_AA64PFR0_ASIMD_SHIFT 20 -#define ID_AA64PFR0_FP_SHIFT 16 -#define ID_AA64PFR0_EL3_SHIFT 12 -#define ID_AA64PFR0_EL2_SHIFT 8 -#define ID_AA64PFR0_EL1_SHIFT 4 -#define ID_AA64PFR0_EL0_SHIFT 0 - -#define ID_AA64PFR0_AMU 0x1 -#define ID_AA64PFR0_SVE 0x1 -#define ID_AA64PFR0_RAS_V1 0x1 -#define ID_AA64PFR0_RAS_V1P1 0x2 -#define ID_AA64PFR0_FP_NI 0xf -#define ID_AA64PFR0_FP_SUPPORTED 0x0 -#define ID_AA64PFR0_ASIMD_NI 0xf -#define ID_AA64PFR0_ASIMD_SUPPORTED 0x0 -#define ID_AA64PFR0_ELx_64BIT_ONLY 0x1 -#define ID_AA64PFR0_ELx_32BIT_64BIT 0x2 - -/* id_aa64pfr1 */ -#define ID_AA64PFR1_SME_SHIFT 24 -#define ID_AA64PFR1_MPAMFRAC_SHIFT 16 -#define ID_AA64PFR1_RASFRAC_SHIFT 12 -#define ID_AA64PFR1_MTE_SHIFT 8 -#define ID_AA64PFR1_SSBS_SHIFT 4 -#define ID_AA64PFR1_BT_SHIFT 0 - -#define ID_AA64PFR1_SSBS_PSTATE_NI 0 -#define ID_AA64PFR1_SSBS_PSTATE_ONLY 1 -#define ID_AA64PFR1_SSBS_PSTATE_INSNS 2 -#define ID_AA64PFR1_BT_BTI 0x1 -#define ID_AA64PFR1_SME 1 - -#define ID_AA64PFR1_MTE_NI 0x0 -#define ID_AA64PFR1_MTE_EL0 0x1 -#define ID_AA64PFR1_MTE 0x2 -#define ID_AA64PFR1_MTE_ASYMM 0x3 +#define ID_AA64PFR0_EL1_ELx_64BIT_ONLY 0x1 +#define ID_AA64PFR0_EL1_ELx_32BIT_64BIT 0x2 /* id_aa64mmfr0 */ -#define ID_AA64MMFR0_ECV_SHIFT 60 -#define ID_AA64MMFR0_FGT_SHIFT 56 -#define ID_AA64MMFR0_EXS_SHIFT 44 -#define ID_AA64MMFR0_TGRAN4_2_SHIFT 40 -#define ID_AA64MMFR0_TGRAN64_2_SHIFT 36 -#define ID_AA64MMFR0_TGRAN16_2_SHIFT 32 -#define ID_AA64MMFR0_TGRAN4_SHIFT 28 -#define ID_AA64MMFR0_TGRAN64_SHIFT 24 -#define ID_AA64MMFR0_TGRAN16_SHIFT 20 -#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16 -#define ID_AA64MMFR0_SNSMEM_SHIFT 12 -#define ID_AA64MMFR0_BIGENDEL_SHIFT 8 -#define ID_AA64MMFR0_ASID_SHIFT 4 -#define ID_AA64MMFR0_PARANGE_SHIFT 0 - -#define ID_AA64MMFR0_ASID_8 0x0 -#define ID_AA64MMFR0_ASID_16 0x2 - -#define ID_AA64MMFR0_TGRAN4_NI 0xf -#define ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN 0x0 -#define ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX 0x7 -#define ID_AA64MMFR0_TGRAN64_NI 0xf -#define ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN 0x0 -#define ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX 0x7 -#define ID_AA64MMFR0_TGRAN16_NI 0x0 -#define ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN 0x1 -#define ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX 0xf - -#define ID_AA64MMFR0_PARANGE_32 0x0 -#define ID_AA64MMFR0_PARANGE_36 0x1 -#define ID_AA64MMFR0_PARANGE_40 0x2 -#define ID_AA64MMFR0_PARANGE_42 0x3 -#define ID_AA64MMFR0_PARANGE_44 0x4 -#define ID_AA64MMFR0_PARANGE_48 0x5 -#define ID_AA64MMFR0_PARANGE_52 0x6 +#define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN 0x0 +#define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX 0x7 +#define ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN 0x0 +#define ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX 0x7 +#define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN 0x1 +#define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX 0xf #define ARM64_MIN_PARANGE_BITS 32 -#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_DEFAULT 0x0 -#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_NONE 0x1 -#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_MIN 0x2 -#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_MAX 0x7 +#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT 0x0 +#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE 0x1 +#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN 0x2 +#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX 0x7 #ifdef CONFIG_ARM64_PA_BITS_52 -#define ID_AA64MMFR0_PARANGE_MAX ID_AA64MMFR0_PARANGE_52 +#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_52 #else -#define ID_AA64MMFR0_PARANGE_MAX ID_AA64MMFR0_PARANGE_48 +#define ID_AA64MMFR0_EL1_PARANGE_MAX ID_AA64MMFR0_EL1_PARANGE_48 #endif -/* id_aa64mmfr1 */ -#define ID_AA64MMFR1_ECBHB_SHIFT 60 -#define ID_AA64MMFR1_TIDCP1_SHIFT 52 -#define ID_AA64MMFR1_HCX_SHIFT 40 -#define ID_AA64MMFR1_AFP_SHIFT 44 -#define ID_AA64MMFR1_ETS_SHIFT 36 -#define ID_AA64MMFR1_TWED_SHIFT 32 -#define ID_AA64MMFR1_XNX_SHIFT 28 -#define ID_AA64MMFR1_SPECSEI_SHIFT 24 -#define ID_AA64MMFR1_PAN_SHIFT 20 -#define ID_AA64MMFR1_LOR_SHIFT 16 -#define ID_AA64MMFR1_HPD_SHIFT 12 -#define ID_AA64MMFR1_VHE_SHIFT 8 -#define ID_AA64MMFR1_VMIDBITS_SHIFT 4 -#define ID_AA64MMFR1_HADBS_SHIFT 0 - -#define ID_AA64MMFR1_VMIDBITS_8 0 -#define ID_AA64MMFR1_VMIDBITS_16 2 - -#define ID_AA64MMFR1_TIDCP1_NI 0 -#define ID_AA64MMFR1_TIDCP1_IMP 1 - -/* id_aa64mmfr2 */ -#define ID_AA64MMFR2_E0PD_SHIFT 60 -#define ID_AA64MMFR2_EVT_SHIFT 56 -#define ID_AA64MMFR2_BBM_SHIFT 52 -#define ID_AA64MMFR2_TTL_SHIFT 48 -#define ID_AA64MMFR2_FWB_SHIFT 40 -#define ID_AA64MMFR2_IDS_SHIFT 36 -#define ID_AA64MMFR2_AT_SHIFT 32 -#define ID_AA64MMFR2_ST_SHIFT 28 -#define ID_AA64MMFR2_NV_SHIFT 24 -#define ID_AA64MMFR2_CCIDX_SHIFT 20 -#define ID_AA64MMFR2_LVA_SHIFT 16 -#define ID_AA64MMFR2_IESB_SHIFT 12 -#define ID_AA64MMFR2_LSM_SHIFT 8 -#define ID_AA64MMFR2_UAO_SHIFT 4 -#define ID_AA64MMFR2_CNP_SHIFT 0 - -/* id_aa64dfr0 */ -#define ID_AA64DFR0_MTPMU_SHIFT 48 -#define ID_AA64DFR0_TRBE_SHIFT 44 -#define ID_AA64DFR0_TRACE_FILT_SHIFT 40 -#define ID_AA64DFR0_DOUBLELOCK_SHIFT 36 -#define ID_AA64DFR0_PMSVER_SHIFT 32 -#define ID_AA64DFR0_CTX_CMPS_SHIFT 28 -#define ID_AA64DFR0_WRPS_SHIFT 20 -#define ID_AA64DFR0_BRPS_SHIFT 12 -#define ID_AA64DFR0_PMUVER_SHIFT 8 -#define ID_AA64DFR0_TRACEVER_SHIFT 4 -#define ID_AA64DFR0_DEBUGVER_SHIFT 0 - -#define ID_AA64DFR0_PMUVER_8_0 0x1 -#define ID_AA64DFR0_PMUVER_8_1 0x4 -#define ID_AA64DFR0_PMUVER_8_4 0x5 -#define ID_AA64DFR0_PMUVER_8_5 0x6 -#define ID_AA64DFR0_PMUVER_8_7 0x7 -#define ID_AA64DFR0_PMUVER_IMP_DEF 0xf - -#define ID_AA64DFR0_PMSVER_8_2 0x1 -#define ID_AA64DFR0_PMSVER_8_3 0x2 - #define ID_DFR0_PERFMON_SHIFT 24 #define ID_DFR0_PERFMON_8_0 0x3 @@ -955,20 +799,20 @@ #define ID_PFR1_PROGMOD_SHIFT 0 #if defined(CONFIG_ARM64_4K_PAGES) -#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN4_SHIFT -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX -#define ID_AA64MMFR0_TGRAN_2_SHIFT ID_AA64MMFR0_TGRAN4_2_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN4_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX +#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT #elif defined(CONFIG_ARM64_16K_PAGES) -#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN16_SHIFT -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX -#define ID_AA64MMFR0_TGRAN_2_SHIFT ID_AA64MMFR0_TGRAN16_2_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN16_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX +#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT #elif defined(CONFIG_ARM64_64K_PAGES) -#define ID_AA64MMFR0_TGRAN_SHIFT ID_AA64MMFR0_TGRAN64_SHIFT -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN -#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX -#define ID_AA64MMFR0_TGRAN_2_SHIFT ID_AA64MMFR0_TGRAN64_2_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SHIFT ID_AA64MMFR0_EL1_TGRAN64_SHIFT +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN +#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX +#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT #endif #define MVFR2_FPMISC_SHIFT 4 @@ -1021,6 +865,7 @@ #define SYS_MPIDR_SAFE_VAL (BIT(31)) #define TRFCR_ELx_TS_SHIFT 5 +#define TRFCR_ELx_TS_MASK ((0x3UL) << TRFCR_ELx_TS_SHIFT) #define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) #define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) #define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) @@ -1028,9 +873,6 @@ #define TRFCR_ELx_ExTRE BIT(1) #define TRFCR_ELx_E0TRE BIT(0) -/* HCRX_EL2 definitions */ -#define HCRX_EL2_SMPME_MASK (1 << 5) - /* GIC Hypervisor interface registers */ /* ICH_MISR_EL2 bit definitions */ #define ICH_MISR_EOI (1 << 0) @@ -1116,6 +958,7 @@ #else +#include #include #include #include @@ -1209,8 +1052,6 @@ par; \ }) -#endif - #define SYS_FIELD_GET(reg, field, val) \ FIELD_GET(reg##_##field##_MASK, val) @@ -1220,4 +1061,6 @@ #define SYS_FIELD_PREP_ENUM(reg, field, val) \ FIELD_PREP(reg##_##field##_MASK, reg##_##field##_##val) +#endif + #endif /* __ASM_SYSREG_H */ diff --git a/arch/arm64/include/asm/system_misc.h b/arch/arm64/include/asm/system_misc.h index 0eb7709422e292ce97860b7c7fc84a4b5bdc4216..c343442567625d58b3158e6ee95c12ebc1027326 100644 --- a/arch/arm64/include/asm/system_misc.h +++ b/arch/arm64/include/asm/system_misc.h @@ -18,7 +18,7 @@ struct pt_regs; -void die(const char *msg, struct pt_regs *regs, int err); +void die(const char *msg, struct pt_regs *regs, long err); struct siginfo; void arm64_notify_die(const char *str, struct pt_regs *regs, diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 2fc9f0861769a76e9ee835ab1007be4547a930c8..5c7b2f9d5913759472a1cb79335ca7263c6a5b4d 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -203,9 +203,11 @@ static inline void uaccess_enable_privileged(void) } /* - * Sanitise a uaccess pointer such that it becomes NULL if above the maximum - * user address. In case the pointer is tagged (has the top byte set), untag - * the pointer before checking. + * Sanitize a uaccess pointer such that it cannot reach any kernel address. + * + * Clearing bit 55 ensures the pointer cannot address any portion of the TTBR1 + * address range (i.e. any kernel address), and either the pointer falls within + * the TTBR0 address range or must cause a fault. */ #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr) static inline void __user *__uaccess_mask_ptr(const void __user *ptr) @@ -213,14 +215,12 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr) void __user *safe_ptr; asm volatile( - " bics xzr, %3, %2\n" - " csel %0, %1, xzr, eq\n" - : "=&r" (safe_ptr) - : "r" (ptr), "r" (TASK_SIZE_MAX - 1), - "r" (untagged_addr(ptr)) - : "cc"); - - csdb(); + " bic %0, %1, %2\n" + : "=r" (safe_ptr) + : "r" (ptr), + "i" (BIT(55)) + ); + return safe_ptr; } diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h index f99dcb94b438d21482bf7938542abf24f9dfdaa3..b4ae3210993273e8fd709b8f4d17a081bf39ff3d 100644 --- a/arch/arm64/include/asm/vdso.h +++ b/arch/arm64/include/asm/vdso.h @@ -26,6 +26,9 @@ (void *)(vdso_offset_##name - VDSO_LBASE + (unsigned long)(base)); \ }) +extern char vdso_start[], vdso_end[]; +extern char vdso32_start[], vdso32_end[]; + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_VDSO_H */ diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h index 4f7a629df81f7416e9797794406029dd3024a355..764d13e2916c559bc8fd7efa032b6f468ac0d4a2 100644 --- a/arch/arm64/include/asm/vdso/gettimeofday.h +++ b/arch/arm64/include/asm/vdso/gettimeofday.h @@ -7,8 +7,10 @@ #ifndef __ASSEMBLY__ +#include #include #include +#include #define VDSO_HAS_CLOCK_GETRES 1 @@ -78,11 +80,20 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, return 0; /* - * This isb() is required to prevent that the counter value + * If FEAT_ECV is available, use the self-synchronizing counter. + * Otherwise the isb is required to prevent that the counter value * is speculated. - */ - isb(); - asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory"); + */ + asm volatile( + ALTERNATIVE("isb\n" + "mrs %0, cntvct_el0", + "nop\n" + __mrs_s("%0", SYS_CNTVCTSS_EL0), + ARM64_HAS_ECV) + : "=r" (res) + : + : "memory"); + arch_counter_enforce_ordering(res); return res; diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 1ad2568a2569f481d1cb52042c4c5e7f70f4935f..9b245da6f507ae96b2e84adb88689b202878e8ac 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -92,5 +92,6 @@ #define HWCAP2_SME_FA64 (1 << 30) #define HWCAP2_WFXT (1UL << 31) #define HWCAP2_EBF16 (1UL << 32) +#define HWCAP2_SVE_EBF16 (1UL << 33) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/include/uapi/asm/perf_regs.h b/arch/arm64/include/uapi/asm/perf_regs.h index d54daafa89e3ff21782fff617d52d89b9dd388b6..86e556429e0eb61bac0c873c68c0ebf28e862c61 100644 --- a/arch/arm64/include/uapi/asm/perf_regs.h +++ b/arch/arm64/include/uapi/asm/perf_regs.h @@ -37,5 +37,12 @@ enum perf_event_arm_regs { PERF_REG_ARM64_SP, PERF_REG_ARM64_PC, PERF_REG_ARM64_MAX, + + /* Extended/pseudo registers */ + PERF_REG_ARM64_VG = 46, /* SVE Vector Granule */ + PERF_REG_ARM64_EXTENDED_MAX }; + +#define PERF_REG_EXTENDED_MASK (1ULL << PERF_REG_ARM64_VG) + #endif /* _ASM_ARM64_PERF_REGS_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 1add7b01efa72ed016fbfacd88f80bd5289ce39d..2f361a883d8c93a138fabc5150e067d781fa870d 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -45,6 +45,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ sys_compat.o obj-$(CONFIG_COMPAT) += sigreturn32.o +obj-$(CONFIG_COMPAT_ALIGNMENT_FIXUPS) += compat_alignment.o obj-$(CONFIG_KUSER_HELPERS) += kuser32.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o obj-$(CONFIG_MODULES) += module.o @@ -85,8 +86,8 @@ $(obj)/vdso-wrap.o: $(obj)/vdso/vdso.so $(obj)/vdso32-wrap.o: $(obj)/vdso32/vdso.so obj-y += probes/ -head-y := head.o -extra-y += $(head-y) vmlinux.lds +obj-y += head.o +extra-y += vmlinux.lds ifeq ($(CONFIG_DEBUG_EFI),y) AFLAGS_head.o += -DVMLINUX_PATH="\"$(realpath $(objtree)/vmlinux)\"" diff --git a/arch/arm64/kernel/acpi_parking_protocol.c b/arch/arm64/kernel/acpi_parking_protocol.c index bfeeb5319abf9851c5b6fca759e3de03e39c0b41..b1990e38aed0a5bd97c8a0f13d3677a48344c79f 100644 --- a/arch/arm64/kernel/acpi_parking_protocol.c +++ b/arch/arm64/kernel/acpi_parking_protocol.c @@ -99,7 +99,7 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu) * that read this address need to convert this address to the * Boot-Loader's endianness before jumping. */ - writeq_relaxed(__pa_symbol(function_nocfi(secondary_entry)), + writeq_relaxed(__pa_symbol(secondary_entry), &mailbox->entry_point); writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id); diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index 9bcaa5eacf16cf1ccf76d64f1886f52c54caef77..91263d09ea650ab1f4b1ada14d258f4651082497 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -10,17 +10,23 @@ #include #include +#include #include #include #include #include +#include #include +#include #include #define __ALT_PTR(a, f) ((void *)&(a)->f + (a)->f) #define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) #define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) +#define ALT_CAP(a) ((a)->cpufeature & ~ARM64_CB_BIT) +#define ALT_HAS_CB(a) ((a)->cpufeature & ARM64_CB_BIT) + /* Volatile, as we may be patching the guts of READ_ONCE() */ static volatile int all_alternatives_applied; @@ -133,7 +139,8 @@ static void clean_dcache_range_nopatch(u64 start, u64 end) } while (cur += d_size, cur < end); } -static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module, +static void __apply_alternatives(const struct alt_region *region, + bool is_module, unsigned long *feature_mask) { struct alt_instr *alt; @@ -142,30 +149,27 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu for (alt = region->begin; alt < region->end; alt++) { int nr_inst; + int cap = ALT_CAP(alt); - if (!test_bit(alt->cpufeature, feature_mask)) + if (!test_bit(cap, feature_mask)) continue; - /* Use ARM64_CB_PATCH as an unconditional patch */ - if (alt->cpufeature < ARM64_CB_PATCH && - !cpus_have_cap(alt->cpufeature)) + if (!cpus_have_cap(cap)) continue; - if (alt->cpufeature == ARM64_CB_PATCH) + if (ALT_HAS_CB(alt)) BUG_ON(alt->alt_len != 0); else BUG_ON(alt->alt_len != alt->orig_len); - pr_info_once("patching kernel code\n"); - origptr = ALT_ORIG_PTR(alt); updptr = is_module ? origptr : lm_alias(origptr); nr_inst = alt->orig_len / AARCH64_INSN_SIZE; - if (alt->cpufeature < ARM64_CB_PATCH) - alt_cb = patch_alternative; - else + if (ALT_HAS_CB(alt)) alt_cb = ALT_REPL_PTR(alt); + else + alt_cb = patch_alternative; alt_cb(alt, origptr, updptr, nr_inst); @@ -192,30 +196,55 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu } } +void apply_alternatives_vdso(void) +{ + struct alt_region region; + const struct elf64_hdr *hdr; + const struct elf64_shdr *shdr; + const struct elf64_shdr *alt; + DECLARE_BITMAP(all_capabilities, ARM64_NCAPS); + + bitmap_fill(all_capabilities, ARM64_NCAPS); + + hdr = (struct elf64_hdr *)vdso_start; + shdr = (void *)hdr + hdr->e_shoff; + alt = find_section(hdr, shdr, ".altinstructions"); + if (!alt) + return; + + region = (struct alt_region){ + .begin = (void *)hdr + alt->sh_offset, + .end = (void *)hdr + alt->sh_offset + alt->sh_size, + }; + + __apply_alternatives(®ion, false, &all_capabilities[0]); +} + +static const struct alt_region kernel_alternatives = { + .begin = (struct alt_instr *)__alt_instructions, + .end = (struct alt_instr *)__alt_instructions_end, +}; + /* * We might be patching the stop_machine state machine, so implement a * really simple polling protocol here. */ static int __apply_alternatives_multi_stop(void *unused) { - struct alt_region region = { - .begin = (struct alt_instr *)__alt_instructions, - .end = (struct alt_instr *)__alt_instructions_end, - }; - /* We always have a CPU 0 at this point (__init) */ if (smp_processor_id()) { while (!all_alternatives_applied) cpu_relax(); isb(); } else { - DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE); + DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS); bitmap_complement(remaining_capabilities, boot_capabilities, - ARM64_NPATCHABLE); + ARM64_NCAPS); BUG_ON(all_alternatives_applied); - __apply_alternatives(®ion, false, remaining_capabilities); + __apply_alternatives(&kernel_alternatives, false, + remaining_capabilities); /* Barriers provided by the cache flushing */ all_alternatives_applied = 1; } @@ -225,6 +254,9 @@ static int __apply_alternatives_multi_stop(void *unused) void __init apply_alternatives_all(void) { + pr_info("applying system-wide alternatives\n"); + + apply_alternatives_vdso(); /* better not try code patching on a live SMP system */ stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask); } @@ -236,15 +268,13 @@ void __init apply_alternatives_all(void) */ void __init apply_boot_alternatives(void) { - struct alt_region region = { - .begin = (struct alt_instr *)__alt_instructions, - .end = (struct alt_instr *)__alt_instructions_end, - }; - /* If called on non-boot cpu things could go wrong */ WARN_ON(smp_processor_id() != 0); - __apply_alternatives(®ion, false, &boot_capabilities[0]); + pr_info("applying boot alternatives\n"); + + __apply_alternatives(&kernel_alternatives, false, + &boot_capabilities[0]); } #ifdef CONFIG_MODULES @@ -254,10 +284,18 @@ void apply_alternatives_module(void *start, size_t length) .begin = start, .end = start + length, }; - DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE); + DECLARE_BITMAP(all_capabilities, ARM64_NCAPS); - bitmap_fill(all_capabilities, ARM64_NPATCHABLE); + bitmap_fill(all_capabilities, ARM64_NCAPS); __apply_alternatives(®ion, true, &all_capabilities[0]); } #endif + +noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr, + __le32 *updptr, int nr_inst) +{ + for (int i = 0; i < nr_inst; i++) + updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); +} +EXPORT_SYMBOL(alt_cb_patch_nops); diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c index 587543c6c51cb0fdbc7a55595be1310d1f6d6221..97c42be71338a9ee29b79ce823592d238c5ba0fe 100644 --- a/arch/arm64/kernel/cacheinfo.c +++ b/arch/arm64/kernel/cacheinfo.c @@ -45,7 +45,8 @@ static void ci_leaf_init(struct cacheinfo *this_leaf, int init_cache_level(unsigned int cpu) { - unsigned int ctype, level, leaves, fw_level; + unsigned int ctype, level, leaves; + int fw_level; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) { @@ -63,6 +64,9 @@ int init_cache_level(unsigned int cpu) else fw_level = acpi_find_last_cache_level(cpu); + if (fw_level < 0) + return fw_level; + if (level < fw_level) { /* * some external caches not specified in CLIDR_EL1 diff --git a/arch/arm64/kernel/compat_alignment.c b/arch/arm64/kernel/compat_alignment.c new file mode 100644 index 0000000000000000000000000000000000000000..5edec2f49ec98c9b040d7698db52a62be0315b48 --- /dev/null +++ b/arch/arm64/kernel/compat_alignment.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +// based on arch/arm/mm/alignment.c + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* + * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998 + * + * Speed optimisations and better fault handling by Russell King. + */ +#define CODING_BITS(i) (i & 0x0e000000) + +#define LDST_P_BIT(i) (i & (1 << 24)) /* Preindex */ +#define LDST_U_BIT(i) (i & (1 << 23)) /* Add offset */ +#define LDST_W_BIT(i) (i & (1 << 21)) /* Writeback */ +#define LDST_L_BIT(i) (i & (1 << 20)) /* Load */ + +#define LDST_P_EQ_U(i) ((((i) ^ ((i) >> 1)) & (1 << 23)) == 0) + +#define LDSTHD_I_BIT(i) (i & (1 << 22)) /* double/half-word immed */ + +#define RN_BITS(i) ((i >> 16) & 15) /* Rn */ +#define RD_BITS(i) ((i >> 12) & 15) /* Rd */ +#define RM_BITS(i) (i & 15) /* Rm */ + +#define REGMASK_BITS(i) (i & 0xffff) + +#define BAD_INSTR 0xdeadc0de + +/* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */ +#define IS_T32(hi16) \ + (((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800)) + +union offset_union { + unsigned long un; + signed long sn; +}; + +#define TYPE_ERROR 0 +#define TYPE_FAULT 1 +#define TYPE_LDST 2 +#define TYPE_DONE 3 + +static void +do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs, + union offset_union offset) +{ + if (!LDST_U_BIT(instr)) + offset.un = -offset.un; + + if (!LDST_P_BIT(instr)) + addr += offset.un; + + if (!LDST_P_BIT(instr) || LDST_W_BIT(instr)) + regs->regs[RN_BITS(instr)] = addr; +} + +static int +do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) +{ + unsigned int rd = RD_BITS(instr); + unsigned int rd2; + int load; + + if ((instr & 0xfe000000) == 0xe8000000) { + /* ARMv7 Thumb-2 32-bit LDRD/STRD */ + rd2 = (instr >> 8) & 0xf; + load = !!(LDST_L_BIT(instr)); + } else if (((rd & 1) == 1) || (rd == 14)) { + return TYPE_ERROR; + } else { + load = ((instr & 0xf0) == 0xd0); + rd2 = rd + 1; + } + + if (load) { + unsigned int val, val2; + + if (get_user(val, (u32 __user *)addr) || + get_user(val2, (u32 __user *)(addr + 4))) + return TYPE_FAULT; + regs->regs[rd] = val; + regs->regs[rd2] = val2; + } else { + if (put_user(regs->regs[rd], (u32 __user *)addr) || + put_user(regs->regs[rd2], (u32 __user *)(addr + 4))) + return TYPE_FAULT; + } + return TYPE_LDST; +} + +/* + * LDM/STM alignment handler. + * + * There are 4 variants of this instruction: + * + * B = rn pointer before instruction, A = rn pointer after instruction + * ------ increasing address -----> + * | | r0 | r1 | ... | rx | | + * PU = 01 B A + * PU = 11 B A + * PU = 00 A B + * PU = 10 A B + */ +static int +do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) +{ + unsigned int rd, rn, nr_regs, regbits; + unsigned long eaddr, newaddr; + unsigned int val; + + /* count the number of registers in the mask to be transferred */ + nr_regs = hweight16(REGMASK_BITS(instr)) * 4; + + rn = RN_BITS(instr); + newaddr = eaddr = regs->regs[rn]; + + if (!LDST_U_BIT(instr)) + nr_regs = -nr_regs; + newaddr += nr_regs; + if (!LDST_U_BIT(instr)) + eaddr = newaddr; + + if (LDST_P_EQ_U(instr)) /* U = P */ + eaddr += 4; + + for (regbits = REGMASK_BITS(instr), rd = 0; regbits; + regbits >>= 1, rd += 1) + if (regbits & 1) { + if (LDST_L_BIT(instr)) { + if (get_user(val, (u32 __user *)eaddr)) + return TYPE_FAULT; + if (rd < 15) + regs->regs[rd] = val; + else + regs->pc = val; + } else { + /* + * The PC register has a bias of +8 in ARM mode + * and +4 in Thumb mode. This means that a read + * of the value of PC should account for this. + * Since Thumb does not permit STM instructions + * to refer to PC, just add 8 here. + */ + val = (rd < 15) ? regs->regs[rd] : regs->pc + 8; + if (put_user(val, (u32 __user *)eaddr)) + return TYPE_FAULT; + } + eaddr += 4; + } + + if (LDST_W_BIT(instr)) + regs->regs[rn] = newaddr; + + return TYPE_DONE; +} + +/* + * Convert Thumb multi-word load/store instruction forms to equivalent ARM + * instructions so we can reuse ARM userland alignment fault fixups for Thumb. + * + * This implementation was initially based on the algorithm found in + * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same + * to convert only Thumb ld/st instruction forms to equivalent ARM forms. + * + * NOTES: + * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections. + * 2. If for some reason we're passed an non-ld/st Thumb instruction to + * decode, we return 0xdeadc0de. This should never happen under normal + * circumstances but if it does, we've got other problems to deal with + * elsewhere and we obviously can't fix those problems here. + */ + +static unsigned long thumb2arm(u16 tinstr) +{ + u32 L = (tinstr & (1<<11)) >> 11; + + switch ((tinstr & 0xf800) >> 11) { + /* 6.6.1 Format 1: */ + case 0xc000 >> 11: /* 7.1.51 STMIA */ + case 0xc800 >> 11: /* 7.1.25 LDMIA */ + { + u32 Rn = (tinstr & (7<<8)) >> 8; + u32 W = ((L<> 11: /* 7.1.48 PUSH */ + case 0xb800 >> 11: /* 7.1.47 POP */ + if ((tinstr & (3 << 9)) == 0x0400) { + static const u32 subset[4] = { + 0xe92d0000, /* STMDB sp!,{registers} */ + 0xe92d4000, /* STMDB sp!,{registers,lr} */ + 0xe8bd0000, /* LDMIA sp!,{registers} */ + 0xe8bd8000 /* LDMIA sp!,{registers,pc} */ + }; + return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] | + (tinstr & 255); /* register_list */ + } + fallthrough; /* for illegal instruction case */ + + default: + return BAD_INSTR; + } +} + +/* + * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction + * handlable by ARM alignment handler, also find the corresponding handler, + * so that we can reuse ARM userland alignment fault fixups for Thumb. + * + * @pinstr: original Thumb-2 instruction; returns new handlable instruction + * @regs: register context. + * @poffset: return offset from faulted addr for later writeback + * + * NOTES: + * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections. + * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt) + */ +static void * +do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs, + union offset_union *poffset) +{ + u32 instr = *pinstr; + u16 tinst1 = (instr >> 16) & 0xffff; + u16 tinst2 = instr & 0xffff; + + switch (tinst1 & 0xffe0) { + /* A6.3.5 Load/Store multiple */ + case 0xe880: /* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */ + case 0xe8a0: /* ...above writeback version */ + case 0xe900: /* STMDB/STMFD, LDMDB/LDMEA */ + case 0xe920: /* ...above writeback version */ + /* no need offset decision since handler calculates it */ + return do_alignment_ldmstm; + + case 0xf840: /* POP/PUSH T3 (single register) */ + if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) { + u32 L = !!(LDST_L_BIT(instr)); + const u32 subset[2] = { + 0xe92d0000, /* STMDB sp!,{registers} */ + 0xe8bd0000, /* LDMIA sp!,{registers} */ + }; + *pinstr = subset[L] | (1<un = (tinst2 & 0xff) << 2; + fallthrough; + + case 0xe940: + case 0xe9c0: + return do_alignment_ldrdstrd; + + /* + * No need to handle load/store instructions up to word size + * since ARMv6 and later CPUs can perform unaligned accesses. + */ + default: + break; + } + return NULL; +} + +static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst) +{ + __le32 instr = 0; + int fault; + + fault = get_user(instr, ip); + if (fault) + return fault; + + *inst = __le32_to_cpu(instr); + return 0; +} + +static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst) +{ + __le16 instr = 0; + int fault; + + fault = get_user(instr, ip); + if (fault) + return fault; + + *inst = __le16_to_cpu(instr); + return 0; +} + +int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs) +{ + union offset_union offset; + unsigned long instrptr; + int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs); + unsigned int type; + u32 instr = 0; + u16 tinstr = 0; + int isize = 4; + int thumb2_32b = 0; + int fault; + + instrptr = instruction_pointer(regs); + + if (compat_thumb_mode(regs)) { + __le16 __user *ptr = (__le16 __user *)(instrptr & ~1); + + fault = alignment_get_thumb(regs, ptr, &tinstr); + if (!fault) { + if (IS_T32(tinstr)) { + /* Thumb-2 32-bit */ + u16 tinst2; + fault = alignment_get_thumb(regs, ptr + 1, &tinst2); + instr = ((u32)tinstr << 16) | tinst2; + thumb2_32b = 1; + } else { + isize = 2; + instr = thumb2arm(tinstr); + } + } + } else { + fault = alignment_get_arm(regs, (__le32 __user *)instrptr, &instr); + } + + if (fault) + return 1; + + switch (CODING_BITS(instr)) { + case 0x00000000: /* 3.13.4 load/store instruction extensions */ + if (LDSTHD_I_BIT(instr)) + offset.un = (instr & 0xf00) >> 4 | (instr & 15); + else + offset.un = regs->regs[RM_BITS(instr)]; + + if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */ + (instr & 0x001000f0) == 0x000000f0) /* STRD */ + handler = do_alignment_ldrdstrd; + else + return 1; + break; + + case 0x08000000: /* ldm or stm, or thumb-2 32bit instruction */ + if (thumb2_32b) { + offset.un = 0; + handler = do_alignment_t32_to_handler(&instr, regs, &offset); + } else { + offset.un = 0; + handler = do_alignment_ldmstm; + } + break; + + default: + return 1; + } + + type = handler(addr, instr, regs); + + if (type == TYPE_ERROR || type == TYPE_FAULT) + return 1; + + if (type == TYPE_LDST) + do_alignment_finish_ldst(addr, instr, regs, offset); + + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->pc); + arm64_skip_faulting_instruction(regs, isize); + + return 0; +} diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S index 48a8af97faa9ab32eb9d29374bb7ca88288fe75f..6b752fe89745196b9d844c27810ccbdffeb3b7cf 100644 --- a/arch/arm64/kernel/cpu-reset.S +++ b/arch/arm64/kernel/cpu-reset.S @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -28,7 +29,7 @@ * branch to what would be the reset vector. It must be executed with the * flat identity mapping. */ -SYM_CODE_START(cpu_soft_restart) +SYM_TYPED_FUNC_START(cpu_soft_restart) mov_q x12, INIT_SCTLR_EL1_MMU_OFF pre_disable_mmu_workaround /* @@ -47,6 +48,6 @@ SYM_CODE_START(cpu_soft_restart) mov x1, x3 // arg1 mov x2, x4 // arg2 br x8 -SYM_CODE_END(cpu_soft_restart) +SYM_FUNC_END(cpu_soft_restart) .popsection diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 7e6289e709fc898f983967e0cb7b067deb257b47..89ac00084f38a4584924bd2da8e2e0e9c5341a07 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -121,6 +121,22 @@ cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused) sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0); } +static DEFINE_RAW_SPINLOCK(reg_user_mask_modification); +static void __maybe_unused +cpu_clear_bf16_from_user_emulation(const struct arm64_cpu_capabilities *__unused) +{ + struct arm64_ftr_reg *regp; + + regp = get_arm64_ftr_reg(SYS_ID_AA64ISAR1_EL1); + if (!regp) + return; + + raw_spin_lock(®_user_mask_modification); + if (regp->user_mask & ID_AA64ISAR1_EL1_BF16_MASK) + regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK; + raw_spin_unlock(®_user_mask_modification); +} + #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max) \ .matches = is_affected_midr_range, \ .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max) @@ -208,10 +224,17 @@ static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { #ifdef CONFIG_ARM64_ERRATUM_1286807 { ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0), + }, + { /* Kryo4xx Gold (rcpe to rfpe) => (r0p0 to r3p0) */ ERRATA_MIDR_RANGE(MIDR_QCOM_KRYO_4XX_GOLD, 0xc, 0xe, 0xf, 0xe), }, #endif +#ifdef CONFIG_ARM64_ERRATUM_2441007 + { + ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A55), + }, +#endif #ifdef CONFIG_ARM64_ERRATUM_2441009 { /* Cortex-A510 r0p0 -> r1p1. Fixed in r1p2 */ @@ -654,6 +677,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2) }, #endif +#ifdef CONFIG_ARM64_ERRATUM_2457168 + { + .desc = "ARM erratum 2457168", + .capability = ARM64_WORKAROUND_2457168, + .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE, + + /* Cortex-A510 r0p0-r1p1 */ + CAP_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1) + }, +#endif #ifdef CONFIG_ARM64_ERRATUM_2038923 { .desc = "ARM erratum 2038923", @@ -679,6 +712,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = { CAP_MIDR_RANGE_LIST(broken_aarch32_aes), .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_2658417 + { + .desc = "ARM erratum 2658417", + .capability = ARM64_WORKAROUND_2658417, + /* Cortex-A510 r0p0 - r1p1 */ + ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1), + MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)), + .cpu_enable = cpu_clear_bf16_from_user_emulation, + }, #endif { } diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 907401e4fffb19cf91b3620a8db8cf90306cfc14..6062454a9067431769a7ad64c1c7d59336406e7d 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -108,8 +108,7 @@ DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); EXPORT_SYMBOL(cpu_hwcaps); static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS]; -/* Need also bit for ARM64_CB_PATCH */ -DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); +DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS); bool arm64_use_ng_mappings = false; EXPORT_SYMBOL(arm64_use_ng_mappings); @@ -134,31 +133,12 @@ DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); */ static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; -/* - * Flag to indicate if we have computed the system wide - * capabilities based on the boot time active CPUs. This - * will be used to determine if a new booting CPU should - * go through the verification process to make sure that it - * supports the system capabilities, without using a hotplug - * notifier. This is also used to decide if we could use - * the fast path for checking constant CPU caps. - */ -DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready); -EXPORT_SYMBOL(arm64_const_caps_ready); -static inline void finalize_system_capabilities(void) -{ - static_branch_enable(&arm64_const_caps_ready); -} - void dump_cpu_features(void) { /* file-wide pr_fmt adds "CPU features: " prefix */ pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps); } -DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS); -EXPORT_SYMBOL(cpu_hwcap_keys); - #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \ { \ .sign = SIGNED, \ @@ -243,35 +223,35 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { }; static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_AMU_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_MPAM_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SEL2_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_DIT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AMU_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_MPAM_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SEL2_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE), - FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0), - S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI), - S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_ELx_64BIT_ONLY), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_ELx_64BIT_ONLY), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SVE_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_RAS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_GIC_SHIFT, 4, 0), + S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, ID_AA64PFR0_EL1_AdvSIMD_NI), + S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_FP_SHIFT, 4, ID_AA64PFR0_EL1_FP_NI), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL3_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL2_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL1_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL0_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY), ARM64_FTR_END, }; static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME), - FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SME_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MPAMFRAC_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_RASFRAC_SHIFT, 4, 0), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SME_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_RAS_frac_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE), - FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI), - ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_SHIFT, 4, ID_AA64PFR1_EL1_MTE_NI), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, ID_AA64PFR1_EL1_SSBS_NI), ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI), - FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_BT_SHIFT, 4, 0), + FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_BT_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -316,9 +296,9 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { - ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_EXS_SHIFT, 4, 0), /* * Page size not being supported at Stage-2 is not fatal. You * just give up KVM if PAGE_SIZE isn't supported there. Go fix @@ -334,9 +314,9 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { * fields are inconsistent across vCPUs, then it isn't worth * trying to bring KVM up. */ - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_2_SHIFT, 4, 1), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_2_SHIFT, 4, 1), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_2_SHIFT, 4, 1), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT, 4, 1), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT, 4, 1), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT, 4, 1), /* * We already refuse to boot CPUs that don't support our configured * page size, so we can only detect mismatches for a page size other @@ -344,55 +324,55 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = { * exist in the wild so, even though we don't like it, we'll have to go * along with it and treat them as non-strict. */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI), - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI), + S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN4_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN4_NI), + S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN64_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN64_NI), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN16_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN16_NI), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT, 4, 0), /* Linux shouldn't care about secure memory */ - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_SNSMEM_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGEND_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT, 4, 0), /* * Differing PARange is fine as long as all peripherals and memory are mapped * within the minimum PARange of all CPUs */ - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_PARANGE_SHIFT, 4, 0), ARM64_FTR_END, }; static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = { - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TIDCP1_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_AFP_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_ETS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TWED_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_XNX_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_SPECSEI_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1_SpecSEI_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_PAN_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_LO_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HPDS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VH_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VMIDBits_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, 4, 0), ARM64_FTR_END, }; static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = { - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_E0PD_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EVT_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_BBM_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_TTL_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IDS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_ST_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_NV_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CCIDX_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_E0PD_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_EVT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_BBM_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_TTL_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_FWB_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IDS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_AT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_ST_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_NV_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CCIDX_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_VARange_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IESB_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_LSM_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_UAO_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CnP_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -434,17 +414,17 @@ static const struct arm64_ftr_bits ftr_id_mmfr0[] = { }; static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_DOUBLELOCK_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0), + S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_DoubleLock_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_PMSVer_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_CTX_CMPs_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_WRPs_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_BRPs_SHIFT, 4, 0), /* * We can instantiate multiple PMU instances with different levels * of support. */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6), + S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6), ARM64_FTR_END, }; @@ -750,7 +730,7 @@ static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id) * returns - Upon success, matching ftr_reg entry for id. * - NULL on failure but with an WARN_ON(). */ -static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id) +struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id) { struct arm64_ftr_reg *reg; @@ -1391,6 +1371,12 @@ u64 __read_sysreg_by_encoding(u32 sys_id) #include +static bool +has_always(const struct arm64_cpu_capabilities *entry, int scope) +{ + return true; +} + static bool feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { @@ -1401,20 +1387,43 @@ feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) return val >= entry->min_field_value; } -static bool -has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) +static u64 +read_scoped_sysreg(const struct arm64_cpu_capabilities *entry, int scope) { - u64 val; - WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); if (scope == SCOPE_SYSTEM) - val = read_sanitised_ftr_reg(entry->sys_reg); + return read_sanitised_ftr_reg(entry->sys_reg); else - val = __read_sysreg_by_encoding(entry->sys_reg); + return __read_sysreg_by_encoding(entry->sys_reg); +} + +static bool +has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + int mask; + struct arm64_ftr_reg *regp; + u64 val = read_scoped_sysreg(entry, scope); + + regp = get_arm64_ftr_reg(entry->sys_reg); + if (!regp) + return false; + + mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask, + entry->field_pos, + entry->field_width); + if (!mask) + return false; return feature_matches(val, entry); } +static bool +has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val = read_scoped_sysreg(entry, scope); + return feature_matches(val, entry); +} + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -1492,7 +1501,7 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); return cpuid_feature_extract_signed_field(pfr0, - ID_AA64PFR0_FP_SHIFT) < 0; + ID_AA64PFR0_EL1_FP_SHIFT) < 0; } static bool has_cache_idc(const struct arm64_cpu_capabilities *entry, @@ -1571,7 +1580,7 @@ bool kaslr_requires_kpti(void) if (IS_ENABLED(CONFIG_ARM64_E0PD)) { u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1); if (cpuid_feature_extract_unsigned_field(mmfr2, - ID_AA64MMFR2_E0PD_SHIFT)) + ID_AA64MMFR2_EL1_E0PD_SHIFT)) return false; } @@ -1685,7 +1694,7 @@ static phys_addr_t kpti_ng_pgd_alloc(int shift) return kpti_ng_temp_alloc; } -static void __nocfi +static void kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) { typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long); @@ -1713,7 +1722,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused) if (arm64_use_ng_mappings) return; - remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings)); + remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings); if (!cpu) { alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); @@ -1870,7 +1879,10 @@ static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap) pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n", smp_processor_id()); cpumask_set_cpu(smp_processor_id(), &amu_cpus); - update_freq_counters_refs(); + + /* 0 reference values signal broken/disabled counters */ + if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168)) + update_freq_counters_refs(); } } @@ -2031,7 +2043,8 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused) static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) { sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0); - isb(); + + mte_cpu_setup(); /* * Clear the tags in the zero page. This needs to be done via the @@ -2084,13 +2097,23 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap) } static const struct arm64_cpu_capabilities arm64_features[] = { + { + .capability = ARM64_ALWAYS_BOOT, + .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, + .matches = has_always, + }, + { + .capability = ARM64_ALWAYS_SYSTEM, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_always, + }, { .desc = "GIC system register CPU interface", .capability = ARM64_HAS_SYSREG_GIC_CPUIF, .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, .matches = has_useable_gicv3_cpuif, .sys_reg = SYS_ID_AA64PFR0_EL1, - .field_pos = ID_AA64PFR0_GIC_SHIFT, + .field_pos = ID_AA64PFR0_EL1_GIC_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, @@ -2101,7 +2124,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR0_EL1, - .field_pos = ID_AA64MMFR0_ECV_SHIFT, + .field_pos = ID_AA64MMFR0_EL1_ECV_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, @@ -2113,7 +2136,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR1_EL1, - .field_pos = ID_AA64MMFR1_PAN_SHIFT, + .field_pos = ID_AA64MMFR1_EL1_PAN_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, @@ -2127,7 +2150,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64MMFR1_EL1, - .field_pos = ID_AA64MMFR1_PAN_SHIFT, + .field_pos = ID_AA64MMFR1_EL1_PAN_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 3, @@ -2165,9 +2188,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_32bit_el0, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR0_EL0_SHIFT, + .field_pos = ID_AA64PFR0_EL1_EL0_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT, + .min_field_value = ID_AA64PFR0_EL1_ELx_32BIT_64BIT, }, #ifdef CONFIG_KVM { @@ -2177,9 +2200,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR0_EL1_SHIFT, + .field_pos = ID_AA64PFR0_EL1_EL1_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT, + .min_field_value = ID_AA64PFR0_EL1_ELx_32BIT_64BIT, }, { .desc = "Protected KVM", @@ -2198,7 +2221,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { * more details. */ .sys_reg = SYS_ID_AA64PFR0_EL1, - .field_pos = ID_AA64PFR0_CSV3_SHIFT, + .field_pos = ID_AA64PFR0_EL1_CSV3_SHIFT, .field_width = 4, .min_field_value = 1, .matches = unmap_kernel_at_el0, @@ -2241,9 +2264,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_SVE, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR0_SVE_SHIFT, + .field_pos = ID_AA64PFR0_EL1_SVE_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR0_SVE, + .min_field_value = ID_AA64PFR0_EL1_SVE_IMP, .matches = has_cpuid_feature, .cpu_enable = sve_kernel_enable, }, @@ -2256,9 +2279,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR0_RAS_SHIFT, + .field_pos = ID_AA64PFR0_EL1_RAS_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR0_RAS_V1, + .min_field_value = ID_AA64PFR0_EL1_RAS_IMP, .cpu_enable = cpu_clear_disr, }, #endif /* CONFIG_ARM64_RAS_EXTN */ @@ -2275,9 +2298,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_amu, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR0_AMU_SHIFT, + .field_pos = ID_AA64PFR0_EL1_AMU_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR0_AMU, + .min_field_value = ID_AA64PFR0_EL1_AMU_IMP, .cpu_enable = cpu_amu_enable, }, #endif /* CONFIG_ARM64_AMU_EXTN */ @@ -2300,7 +2323,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_STAGE2_FWB, .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64MMFR2_FWB_SHIFT, + .field_pos = ID_AA64MMFR2_EL1_FWB_SHIFT, .field_width = 4, .min_field_value = 1, .matches = has_cpuid_feature, @@ -2311,7 +2334,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HAS_ARMv8_4_TTL, .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64MMFR2_TTL_SHIFT, + .field_pos = ID_AA64MMFR2_EL1_TTL_SHIFT, .field_width = 4, .min_field_value = 1, .matches = has_cpuid_feature, @@ -2341,7 +2364,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_HW_DBM, .sys_reg = SYS_ID_AA64MMFR1_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64MMFR1_HADBS_SHIFT, + .field_pos = ID_AA64MMFR1_EL1_HAFDBS_SHIFT, .field_width = 4, .min_field_value = 2, .matches = has_hw_dbm, @@ -2364,10 +2387,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, - .field_pos = ID_AA64PFR1_SSBS_SHIFT, + .field_pos = ID_AA64PFR1_EL1_SSBS_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, - .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY, + .min_field_value = ID_AA64PFR1_EL1_SSBS_IMP, }, #ifdef CONFIG_ARM64_CNP { @@ -2377,7 +2400,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_useable_cnp, .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64MMFR2_CNP_SHIFT, + .field_pos = ID_AA64MMFR2_EL1_CnP_SHIFT, .field_width = 4, .min_field_value = 1, .cpu_enable = cpu_enable_cnp, @@ -2482,7 +2505,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, .matches = can_use_gic_priorities, .sys_reg = SYS_ID_AA64PFR0_EL1, - .field_pos = ID_AA64PFR0_GIC_SHIFT, + .field_pos = ID_AA64PFR0_EL1_GIC_SHIFT, .field_width = 4, .sign = FTR_UNSIGNED, .min_field_value = 1, @@ -2496,7 +2519,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .sys_reg = SYS_ID_AA64MMFR2_EL1, .sign = FTR_UNSIGNED, .field_width = 4, - .field_pos = ID_AA64MMFR2_E0PD_SHIFT, + .field_pos = ID_AA64MMFR2_EL1_E0PD_SHIFT, .matches = has_cpuid_feature, .min_field_value = 1, .cpu_enable = cpu_enable_e0pd, @@ -2525,9 +2548,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .cpu_enable = bti_enable, .sys_reg = SYS_ID_AA64PFR1_EL1, - .field_pos = ID_AA64PFR1_BT_SHIFT, + .field_pos = ID_AA64PFR1_EL1_BT_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR1_BT_BTI, + .min_field_value = ID_AA64PFR1_EL1_BT_IMP, .sign = FTR_UNSIGNED, }, #endif @@ -2538,9 +2561,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, - .field_pos = ID_AA64PFR1_MTE_SHIFT, + .field_pos = ID_AA64PFR1_EL1_MTE_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR1_MTE, + .min_field_value = ID_AA64PFR1_EL1_MTE_MTE2, .sign = FTR_UNSIGNED, .cpu_enable = cpu_enable_mte, }, @@ -2550,9 +2573,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, .matches = has_cpuid_feature, .sys_reg = SYS_ID_AA64PFR1_EL1, - .field_pos = ID_AA64PFR1_MTE_SHIFT, + .field_pos = ID_AA64PFR1_EL1_MTE_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR1_MTE_ASYMM, + .min_field_value = ID_AA64PFR1_EL1_MTE_MTE3, .sign = FTR_UNSIGNED, }, #endif /* CONFIG_ARM64_MTE */ @@ -2574,9 +2597,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .capability = ARM64_SME, .sys_reg = SYS_ID_AA64PFR1_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64PFR1_SME_SHIFT, + .field_pos = ID_AA64PFR1_EL1_SME_SHIFT, .field_width = 4, - .min_field_value = ID_AA64PFR1_SME, + .min_field_value = ID_AA64PFR1_EL1_SME_IMP, .matches = has_cpuid_feature, .cpu_enable = sme_kernel_enable, }, @@ -2611,9 +2634,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .type = ARM64_CPUCAP_SYSTEM_FEATURE, .sys_reg = SYS_ID_AA64MMFR1_EL1, .sign = FTR_UNSIGNED, - .field_pos = ID_AA64MMFR1_TIDCP1_SHIFT, + .field_pos = ID_AA64MMFR1_EL1_TIDCP1_SHIFT, .field_width = 4, - .min_field_value = ID_AA64MMFR1_TIDCP1_IMP, + .min_field_value = ID_AA64MMFR1_EL1_TIDCP1_IMP, .matches = has_cpuid_feature, .cpu_enable = cpu_trap_el0_impdef, }, @@ -2621,7 +2644,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = { }; #define HWCAP_CPUID_MATCH(reg, field, width, s, min_value) \ - .matches = has_cpuid_feature, \ + .matches = has_user_cpuid_feature, \ .sys_reg = reg, \ .field_pos = field, \ .field_width = width, \ @@ -2705,11 +2728,11 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_TS_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FLAGM), HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_TS_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2), HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_RNDR_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RNG), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_FP_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_FP_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_DIT_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DCPOP), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_DCPODP), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_JSCVT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_JSCVT), @@ -2722,38 +2745,39 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_EBF16), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DGH), HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_I8MM), - HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT), + HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT), #ifdef CONFIG_ARM64_SVE - HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, KERNEL_HWCAP_SVE), + HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_EL1_SVE_IMP, CAP_HWCAP, KERNEL_HWCAP_SVE), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BitPerm_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BF16_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16), + HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BF16_EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SHA3_IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SM4_IMP, CAP_HWCAP, KERNEL_HWCAP_SVESM4), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_I8MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_F32MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM), HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_F64MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM), #endif - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_SSBS_SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS), #ifdef CONFIG_ARM64_BTI - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_BT_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_BT_IMP, CAP_HWCAP, KERNEL_HWCAP_BTI), #endif #ifdef CONFIG_ARM64_PTR_AUTH HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA), HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG), #endif #ifdef CONFIG_ARM64_MTE - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE), - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE_ASYMM, CAP_HWCAP, KERNEL_HWCAP_MTE3), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_MTE_MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_MTE_MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3), #endif /* CONFIG_ARM64_MTE */ - HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), - HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), + HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), + HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT), #ifdef CONFIG_ARM64_SME - HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SME_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_SME, CAP_HWCAP, KERNEL_HWCAP_SME), + HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_SME_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_SME_IMP, CAP_HWCAP, KERNEL_HWCAP_SME), HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, FTR_UNSIGNED, ID_AA64SMFR0_EL1_FA64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64), HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_I16I64_SHIFT, 4, FTR_UNSIGNED, ID_AA64SMFR0_EL1_I16I64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64), HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_F64F64_SHIFT, 1, FTR_UNSIGNED, ID_AA64SMFR0_EL1_F64F64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64), @@ -2926,9 +2950,6 @@ static void __init enable_cpu_capabilities(u16 scope_mask) if (!cpus_have_cap(num)) continue; - /* Ensure cpus_have_const_cap(num) works */ - static_branch_enable(&cpu_hwcap_keys[num]); - if (boot_scope && caps->cpu_enable) /* * Capabilities with SCOPE_BOOT_CPU scope are finalised @@ -3099,7 +3120,7 @@ static void verify_hyp_capabilities(void) /* Verify IPA range */ parange = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_PARANGE_SHIFT); + ID_AA64MMFR0_EL1_PARANGE_SHIFT); ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange); if (ipa_max < get_kvm_ipa_limit()) { pr_crit("CPU%d: IPA range mismatch\n", smp_processor_id()); @@ -3250,9 +3271,6 @@ void __init setup_cpu_features(void) sme_setup(); minsigstksz_setup(); - /* Advertise that we have computed the system capabilities */ - finalize_system_capabilities(); - /* * Check for sane CTR_EL0.CWG value. */ diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index d7702f39b4d338eafe96f0c68e58f8bad0a693ca..28d4f442b0bc1b476e96e19064c71d58ecf207b5 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -115,6 +115,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_SME_FA64] = "smefa64", [KERNEL_HWCAP_WFXT] = "wfxt", [KERNEL_HWCAP_EBF16] = "ebf16", + [KERNEL_HWCAP_SVE_EBF16] = "sveebf16", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index bf9fe71589bcaca54e8d18f08d5794e9830c9570..3da09778267ecdfc5edcc08d1b165c41ff93c654 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -28,7 +28,7 @@ u8 debug_monitors_arch(void) { return cpuid_feature_extract_unsigned_field(read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1), - ID_AA64DFR0_DEBUGVER_SHIFT); + ID_AA64DFR0_EL1_DebugVer_SHIFT); } /* diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c index 98d67444a5b615246a9bd0f01c59a06660015b5f..27ef7ad3ffd2e27a5c6fdc1e1a63c179df2e56e2 100644 --- a/arch/arm64/kernel/elfcore.c +++ b/arch/arm64/kernel/elfcore.c @@ -8,9 +8,9 @@ #include #include -#define for_each_mte_vma(tsk, vma) \ +#define for_each_mte_vma(vmi, vma) \ if (system_supports_mte()) \ - for (vma = tsk->mm->mmap; vma; vma = vma->vm_next) \ + for_each_vma(vmi, vma) \ if (vma->vm_flags & VM_MTE) static unsigned long mte_vma_tag_dump_size(struct vm_area_struct *vma) @@ -81,8 +81,9 @@ Elf_Half elf_core_extra_phdrs(void) { struct vm_area_struct *vma; int vma_count = 0; + VMA_ITERATOR(vmi, current->mm, 0); - for_each_mte_vma(current, vma) + for_each_mte_vma(vmi, vma) vma_count++; return vma_count; @@ -91,8 +92,9 @@ Elf_Half elf_core_extra_phdrs(void) int elf_core_write_extra_phdrs(struct coredump_params *cprm, loff_t offset) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, current->mm, 0); - for_each_mte_vma(current, vma) { + for_each_mte_vma(vmi, vma) { struct elf_phdr phdr; phdr.p_type = PT_AARCH64_MEMTAG_MTE; @@ -116,8 +118,9 @@ size_t elf_core_extra_data_size(void) { struct vm_area_struct *vma; size_t data_size = 0; + VMA_ITERATOR(vmi, current->mm, 0); - for_each_mte_vma(current, vma) + for_each_mte_vma(vmi, vma) data_size += mte_vma_tag_dump_size(vma); return data_size; @@ -126,8 +129,9 @@ size_t elf_core_extra_data_size(void) int elf_core_write_extra_data(struct coredump_params *cprm) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, current->mm, 0); - for_each_mte_vma(current, vma) { + for_each_mte_vma(vmi, vma) { if (vma->vm_flags & VM_DONTDUMP) continue; diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index c75ca36b4a49113091b804ebdcfd695350b9b487..9173fad279af9333ff4bee8cb0b8153132f2af4a 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -379,11 +379,20 @@ static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) exit_to_kernel_mode(regs); } -static void noinstr el1_undef(struct pt_regs *regs) +static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) { enter_from_kernel_mode(regs); local_daif_inherit(regs); - do_undefinstr(regs); + do_undefinstr(regs, esr); + local_daif_mask(); + exit_to_kernel_mode(regs); +} + +static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) +{ + enter_from_kernel_mode(regs); + local_daif_inherit(regs); + do_el1_bti(regs, esr); local_daif_mask(); exit_to_kernel_mode(regs); } @@ -402,7 +411,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) { enter_from_kernel_mode(regs); local_daif_inherit(regs); - do_ptrauth_fault(regs, esr); + do_el1_fpac(regs, esr); local_daif_mask(); exit_to_kernel_mode(regs); } @@ -425,7 +434,10 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) break; case ESR_ELx_EC_SYS64: case ESR_ELx_EC_UNKNOWN: - el1_undef(regs); + el1_undef(regs, esr); + break; + case ESR_ELx_EC_BTI: + el1_bti(regs, esr); break; case ESR_ELx_EC_BREAKPT_CUR: case ESR_ELx_EC_SOFTSTP_CUR: @@ -582,11 +594,11 @@ static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) exit_to_user_mode(regs); } -static void noinstr el0_undef(struct pt_regs *regs) +static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) { enter_from_user_mode(regs); local_daif_restore(DAIF_PROCCTX); - do_undefinstr(regs); + do_undefinstr(regs, esr); exit_to_user_mode(regs); } @@ -594,7 +606,7 @@ static void noinstr el0_bti(struct pt_regs *regs) { enter_from_user_mode(regs); local_daif_restore(DAIF_PROCCTX); - do_bti(regs); + do_el0_bti(regs); exit_to_user_mode(regs); } @@ -629,7 +641,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) { enter_from_user_mode(regs); local_daif_restore(DAIF_PROCCTX); - do_ptrauth_fault(regs, esr); + do_el0_fpac(regs, esr); exit_to_user_mode(regs); } @@ -670,7 +682,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) el0_pc(regs, esr); break; case ESR_ELx_EC_UNKNOWN: - el0_undef(regs); + el0_undef(regs, esr); break; case ESR_ELx_EC_BTI: el0_bti(regs); @@ -788,7 +800,7 @@ asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) case ESR_ELx_EC_CP14_MR: case ESR_ELx_EC_CP14_LS: case ESR_ELx_EC_CP14_64: - el0_undef(regs); + el0_undef(regs, esr); break; case ESR_ELx_EC_CP15_32: case ESR_ELx_EC_CP15_64: diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 254fe31c03a07d048c41e2156a95aff997d68d06..e28137d64b7688e2a7f127eac9ce4f49f1118d6e 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -114,7 +114,7 @@ * them if required. */ .macro apply_ssbd, state, tmp1, tmp2 -alternative_cb spectre_v4_patch_fw_mitigation_enable +alternative_cb ARM64_ALWAYS_SYSTEM, spectre_v4_patch_fw_mitigation_enable b .L__asm_ssbd_skip\@ // Patched to NOP alternative_cb_end ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1 @@ -123,7 +123,7 @@ alternative_cb_end tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2 mov w1, #\state -alternative_cb smccc_patch_fw_mitigation_conduit +alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit nop // Patched to SMC/HVC #0 alternative_cb_end .L__asm_ssbd_skip\@: @@ -175,7 +175,7 @@ alternative_else_nop_endif .macro mte_set_kernel_gcr, tmp, tmp2 #ifdef CONFIG_KASAN_HW_TAGS -alternative_cb kasan_hw_tags_enable +alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable b 1f alternative_cb_end mov \tmp, KERNEL_GCR_EL1 @@ -186,7 +186,7 @@ alternative_cb_end .macro mte_set_user_gcr, tsk, tmp, tmp2 #ifdef CONFIG_KASAN_HW_TAGS -alternative_cb kasan_hw_tags_enable +alternative_cb ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable b 1f alternative_cb_end ldr \tmp, [\tsk, #THREAD_MTE_CTRL] @@ -502,7 +502,7 @@ tsk .req x28 // current thread_info SYM_CODE_START(vectors) kernel_ventry 1, t, 64, sync // Synchronous EL1t kernel_ventry 1, t, 64, irq // IRQ EL1t - kernel_ventry 1, t, 64, fiq // FIQ EL1h + kernel_ventry 1, t, 64, fiq // FIQ EL1t kernel_ventry 1, t, 64, error // Error EL1t kernel_ventry 1, h, 64, sync // Synchronous EL1h diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index dd63ffc3a2fa2782823fec804c8327405c69a4cd..23834d96d1e781d16e34e760f5a7e72af7422525 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -715,10 +715,12 @@ size_t sve_state_size(struct task_struct const *task) * do_sve_acc() case, there is no ABI requirement to hide stale data * written previously be task. */ -void sve_alloc(struct task_struct *task) +void sve_alloc(struct task_struct *task, bool flush) { if (task->thread.sve_state) { - memset(task->thread.sve_state, 0, sve_state_size(task)); + if (flush) + memset(task->thread.sve_state, 0, + sve_state_size(task)); return; } @@ -1388,7 +1390,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs) return; } - sve_alloc(current); + sve_alloc(current, true); if (!current->thread.sve_state) { force_sig(SIGKILL); return; @@ -1439,7 +1441,7 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) return; } - sve_alloc(current); + sve_alloc(current, false); sme_alloc(current); if (!current->thread.sve_state || !current->thread.za_state) { force_sig(SIGKILL); @@ -1460,17 +1462,6 @@ void do_sme_acc(unsigned long esr, struct pt_regs *regs) fpsimd_bind_task_to_cpu(); } - /* - * If SVE was not already active initialise the SVE registers, - * any non-shared state between the streaming and regular SVE - * registers is architecturally guaranteed to be zeroed when - * we enter streaming mode. We do not need to initialize ZA - * since ZA must be disabled at this point and enabling ZA is - * architecturally defined to zero ZA. - */ - if (system_supports_sve() && !test_thread_flag(TIF_SVE)) - sve_init_regs(); - put_cpu_fpsimd_context(); } diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index ea5dc7c90f465f8bada797603df16249b6979a09..8745175f4a754c8808804c9569da96777909c2a3 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -56,7 +56,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func) unsigned long pc; u32 new; - pc = (unsigned long)function_nocfi(ftrace_call); + pc = (unsigned long)ftrace_call; new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, AARCH64_INSN_BRANCH_LINK); @@ -217,11 +217,26 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long pc = rec->ip; u32 old = 0, new; + new = aarch64_insn_gen_nop(); + + /* + * When using mcount, callsites in modules may have been initalized to + * call an arbitrary module PLT (which redirects to the _mcount stub) + * rather than the ftrace PLT we'll use at runtime (which redirects to + * the ftrace trampoline). We can ignore the old PLT when initializing + * the callsite. + * + * Note: 'mod' is only set at module load time. + */ + if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) && + IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) { + return aarch64_insn_patch_text_nosync((void *)pc, new); + } + if (!ftrace_find_callable_addr(rec, mod, &addr)) return -EINVAL; old = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK); - new = aarch64_insn_gen_nop(); return ftrace_modify_code(pc, old, new, true); } diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index cefe6a73ee546c26e8971bce4abf4b01f6e5f96d..2196aad7b55bcef05b2554b5fae086e3cdc84159 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -99,7 +99,7 @@ SYM_CODE_START(primary_entry) */ #if VA_BITS > 48 mrs_s x0, SYS_ID_AA64MMFR2_EL1 - tst x0, #0xf << ID_AA64MMFR2_LVA_SHIFT + tst x0, #0xf << ID_AA64MMFR2_EL1_VARange_SHIFT mov x0, #VA_BITS mov x25, #VA_BITS_MIN csel x25, x25, x0, eq @@ -371,7 +371,9 @@ SYM_FUNC_END(create_idmap) SYM_FUNC_START_LOCAL(create_kernel_mapping) adrp x0, init_pg_dir mov_q x5, KIMAGE_VADDR // compile time __va(_text) +#ifdef CONFIG_RELOCATABLE add x5, x5, x23 // add KASLR displacement +#endif adrp x6, _end // runtime __pa(_end) adrp x3, _text // runtime __pa(_text) sub x6, x6, x3 // _end - _text @@ -656,10 +658,10 @@ SYM_FUNC_END(__secondary_too_slow) */ SYM_FUNC_START(__enable_mmu) mrs x3, ID_AA64MMFR0_EL1 - ubfx x3, x3, #ID_AA64MMFR0_TGRAN_SHIFT, 4 - cmp x3, #ID_AA64MMFR0_TGRAN_SUPPORTED_MIN + ubfx x3, x3, #ID_AA64MMFR0_EL1_TGRAN_SHIFT, 4 + cmp x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN b.lt __no_granule_support - cmp x3, #ID_AA64MMFR0_TGRAN_SUPPORTED_MAX + cmp x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX b.gt __no_granule_support phys_to_ttbr x2, x2 msr ttbr0_el1, x2 // load TTBR0 @@ -677,7 +679,7 @@ SYM_FUNC_START(__cpu_secondary_check52bitva) b.ne 2f mrs_s x0, SYS_ID_AA64MMFR2_EL1 - and x0, x0, #(0xf << ID_AA64MMFR2_LVA_SHIFT) + and x0, x0, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT) cbnz x0, 2f update_early_cpu_boot_status \ diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 12c7fad02ae517a1c17f999ce52a01669afedd8d..2ee18c860f2ab61de44444c7faa5ec77de7b3ca4 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -98,7 +98,7 @@ SYM_CODE_START_LOCAL(elx_sync) SYM_CODE_END(elx_sync) SYM_CODE_START_LOCAL(__finalise_el2) - check_override id_aa64pfr0 ID_AA64PFR0_SVE_SHIFT .Linit_sve .Lskip_sve + check_override id_aa64pfr0 ID_AA64PFR0_EL1_SVE_SHIFT .Linit_sve .Lskip_sve .Linit_sve: /* SVE register access */ mrs x0, cptr_el2 // Disable SVE traps @@ -109,7 +109,7 @@ SYM_CODE_START_LOCAL(__finalise_el2) msr_s SYS_ZCR_EL2, x1 // length for EL1. .Lskip_sve: - check_override id_aa64pfr1 ID_AA64PFR1_SME_SHIFT .Linit_sme .Lskip_sme + check_override id_aa64pfr1 ID_AA64PFR1_EL1_SME_SHIFT .Linit_sme .Lskip_sme .Linit_sme: /* SME register access and priority mapping */ mrs x0, cptr_el2 // Disable SME traps @@ -142,7 +142,7 @@ SYM_CODE_START_LOCAL(__finalise_el2) msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present? - ubfx x1, x1, #ID_AA64MMFR1_HCX_SHIFT, #4 + ubfx x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4 cbz x1, .Lskip_sme mrs_s x1, SYS_HCRX_EL2 @@ -157,7 +157,7 @@ SYM_CODE_START_LOCAL(__finalise_el2) tbnz x1, #0, 1f // Needs to be VHE capable, obviously - check_override id_aa64mmfr1 ID_AA64MMFR1_VHE_SHIFT 2f 1f + check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 2f 1f 1: mov_q x0, HVC_STUB_ERR eret diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 1b0542c69738eca272afbd899965d247a8393526..95133765ed29a0e4f9c68e1fecfb9d9332034454 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -50,7 +50,7 @@ static const struct ftr_set_desc mmfr1 __initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { - FIELD("vh", ID_AA64MMFR1_VHE_SHIFT, mmfr1_vh_filter), + FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), {} }, }; @@ -74,7 +74,7 @@ static const struct ftr_set_desc pfr0 __initconst = { .name = "id_aa64pfr0", .override = &id_aa64pfr0_override, .fields = { - FIELD("sve", ID_AA64PFR0_SVE_SHIFT, pfr0_sve_filter), + FIELD("sve", ID_AA64PFR0_EL1_SVE_SHIFT, pfr0_sve_filter), {} }, }; @@ -98,9 +98,9 @@ static const struct ftr_set_desc pfr1 __initconst = { .name = "id_aa64pfr1", .override = &id_aa64pfr1_override, .fields = { - FIELD("bt", ID_AA64PFR1_BT_SHIFT, NULL ), - FIELD("mte", ID_AA64PFR1_MTE_SHIFT, NULL), - FIELD("sme", ID_AA64PFR1_SME_SHIFT, pfr1_sme_filter), + FIELD("bt", ID_AA64PFR1_EL1_BT_SHIFT, NULL ), + FIELD("mte", ID_AA64PFR1_EL1_MTE_SHIFT, NULL), + FIELD("sme", ID_AA64PFR1_EL1_SME_SHIFT, pfr1_sme_filter), {} }, }; diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index afa69e04e75eddfb5b303efb61dedb22679f6620..8151412653de209cc9006881d2e496b132cbd72e 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -24,9 +24,6 @@ PROVIDE(__efistub_primary_entry_offset = primary_entry - _text); */ PROVIDE(__efistub_memcmp = __pi_memcmp); PROVIDE(__efistub_memchr = __pi_memchr); -PROVIDE(__efistub_memcpy = __pi_memcpy); -PROVIDE(__efistub_memmove = __pi_memmove); -PROVIDE(__efistub_memset = __pi_memset); PROVIDE(__efistub_strlen = __pi_strlen); PROVIDE(__efistub_strnlen = __pi_strnlen); PROVIDE(__efistub_strcmp = __pi_strcmp); @@ -40,16 +37,6 @@ PROVIDE(__efistub__edata = _edata); PROVIDE(__efistub_screen_info = screen_info); PROVIDE(__efistub__ctype = _ctype); -/* - * The __ prefixed memcpy/memset/memmove symbols are provided by KASAN, which - * instruments the conventional ones. Therefore, any references from the EFI - * stub or other position independent, low level C code should be redirected to - * the non-instrumented versions as well. - */ -PROVIDE(__efistub___memcpy = __pi_memcpy); -PROVIDE(__efistub___memmove = __pi_memmove); -PROVIDE(__efistub___memset = __pi_memset); - PROVIDE(__pi___memcpy = __pi_memcpy); PROVIDE(__pi___memmove = __pi_memmove); PROVIDE(__pi___memset = __pi_memset); @@ -73,6 +60,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter); KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable); KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); +KVM_NVHE_ALIAS(alt_cb_patch_nops); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); @@ -89,10 +77,6 @@ KVM_NVHE_ALIAS(__icache_flags); /* VMID bits set by the KVM VMID allocator */ KVM_NVHE_ALIAS(kvm_arm_vmid_bits); -/* Kernel symbols needed for cpus_have_final/const_caps checks. */ -KVM_NVHE_ALIAS(arm64_const_caps_ready); -KVM_NVHE_ALIAS(cpu_hwcap_keys); - /* Static keys which are set if a vGIC trap should be handled in hyp. */ KVM_NVHE_ALIAS(vgic_v2_cpuif_trap); KVM_NVHE_ALIAS(vgic_v3_cpuif_trap); diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index bda49430c9ea3c9beb0669689e760bc6bd26c3ce..38dbd3828f139106ff6f1755a79e1d511e5108b2 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -21,7 +21,9 @@ #include #include #include +#include #include +#include /* Only access this in an NMI enter/exit */ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); @@ -71,6 +73,18 @@ static void init_irq_stacks(void) } #endif +#ifndef CONFIG_PREEMPT_RT +static void ____do_softirq(struct pt_regs *regs) +{ + __do_softirq(); +} + +void do_softirq_own_stack(void) +{ + call_on_irq_stack(NULL, ____do_softirq); +} +#endif + static void default_handle_irq(struct pt_regs *regs) { panic("IRQ taken without a root IRQ handler\n"); diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 19c2d487cb08feb66052642f7bfca776e97445d0..ce3d40120f72f03283926246e4db4b266ac6d179 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -204,7 +204,7 @@ void machine_kexec(struct kimage *kimage) typeof(cpu_soft_restart) *restart; cpu_install_idmap(); - restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart)); + restart = (void *)__pa_symbol(cpu_soft_restart); restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem, 0, 0); } else { diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 889951291cc0f9cea9e9fccb9bf0619ad040f9c8..a11a6e14ba89f648aac6e378beb2eb9d0d066c33 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -47,7 +47,7 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) u64 i; phys_addr_t start, end; - nr_ranges = 1; /* for exclusion of crashkernel region */ + nr_ranges = 2; /* for exclusion of crashkernel region */ for_each_mem_range(i, &start, &end) nr_ranges++; diff --git a/arch/arm64/kernel/module-plts.c b/arch/arm64/kernel/module-plts.c index a3d0494f25a91de87c8e15a04fe404d516e6ef2e..5a0a8f552a6106e66c931e45eaf1956a1f82636c 100644 --- a/arch/arm64/kernel/module-plts.c +++ b/arch/arm64/kernel/module-plts.c @@ -37,7 +37,8 @@ struct plt_entry get_plt_entry(u64 dst, void *pc) return plt; } -bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b) +static bool plt_entries_equal(const struct plt_entry *a, + const struct plt_entry *b) { u64 p, q; diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index f2d4bb14bfabe28e7fa79333c6c290bc9a3043c4..76b41e4ca9fa3a06d72849717c1354e9a92ced8e 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -476,21 +476,6 @@ overflow: return -ENOEXEC; } -static const Elf_Shdr *find_section(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - const char *name) -{ - const Elf_Shdr *s, *se; - const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; - - for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { - if (strcmp(name, secstrs + s->sh_name) == 0) - return s; - } - - return NULL; -} - static inline void __init_plt(struct plt_entry *plt, unsigned long addr) { *plt = get_plt_entry(addr, plt); diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index b2b730233274b72f4e1c619d39a0e68d22835f13..7467217c1eaf372d19a0eed8398e2e89e04f44c7 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -48,7 +48,12 @@ static void mte_sync_page_tags(struct page *page, pte_t old_pte, if (!pte_is_tagged) return; - mte_clear_page_tags(page_address(page)); + /* + * Test PG_mte_tagged again in case it was racing with another + * set_pte_at(). + */ + if (!test_and_set_bit(PG_mte_tagged, &page->flags)) + mte_clear_page_tags(page_address(page)); } void mte_sync_tags(pte_t old_pte, pte_t pte) @@ -64,7 +69,7 @@ void mte_sync_tags(pte_t old_pte, pte_t pte) /* if PG_mte_tagged is set, tags have already been initialised */ for (i = 0; i < nr_pages; i++, page++) { - if (!test_and_set_bit(PG_mte_tagged, &page->flags)) + if (!test_bit(PG_mte_tagged, &page->flags)) mte_sync_page_tags(page, old_pte, check_swap, pte_is_tagged); } @@ -285,6 +290,49 @@ void mte_thread_switch(struct task_struct *next) mte_check_tfsr_el1(); } +void mte_cpu_setup(void) +{ + u64 rgsr; + + /* + * CnP must be enabled only after the MAIR_EL1 register has been set + * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may + * lead to the wrong memory type being used for a brief window during + * CPU power-up. + * + * CnP is not a boot feature so MTE gets enabled before CnP, but let's + * make sure that is the case. + */ + BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT); + BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT); + + /* Normal Tagged memory type at the corresponding MAIR index */ + sysreg_clear_set(mair_el1, + MAIR_ATTRIDX(MAIR_ATTR_MASK, MT_NORMAL_TAGGED), + MAIR_ATTRIDX(MAIR_ATTR_NORMAL_TAGGED, + MT_NORMAL_TAGGED)); + + write_sysreg_s(KERNEL_GCR_EL1, SYS_GCR_EL1); + + /* + * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then + * RGSR_EL1.SEED must be non-zero for IRG to produce + * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we + * must initialize it. + */ + rgsr = (read_sysreg(CNTVCT_EL0) & SYS_RGSR_EL1_SEED_MASK) << + SYS_RGSR_EL1_SEED_SHIFT; + if (rgsr == 0) + rgsr = 1 << SYS_RGSR_EL1_SEED_SHIFT; + write_sysreg_s(rgsr, SYS_RGSR_EL1); + + /* clear any pending tag check faults in TFSR*_EL1 */ + write_sysreg_s(0, SYS_TFSR_EL1); + write_sysreg_s(0, SYS_TFSRE0_EL1); + + local_flush_tlb_all(); +} + void mte_suspend_enter(void) { if (!system_supports_mte()) @@ -301,6 +349,14 @@ void mte_suspend_enter(void) mte_check_tfsr_el1(); } +void mte_suspend_exit(void) +{ + if (!system_supports_mte()) + return; + + mte_cpu_setup(); +} + long set_mte_ctrl(struct task_struct *task, unsigned long arg) { u64 mte_ctrl = (~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) & diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index cb69ff1e61380b7f5371b6dde6436dd09f672c74..7b0643fe2f1347db07bc09277e5b179881d3b003 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -390,7 +390,7 @@ static const struct attribute_group armv8_pmuv3_caps_attr_group = { */ static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu) { - return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5); + return (cpu_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P5); } static inline bool armv8pmu_event_has_user_read(struct perf_event *event) @@ -1145,8 +1145,8 @@ static void __armv8pmu_probe_pmu(void *info) dfr0 = read_sysreg(id_aa64dfr0_el1); pmuver = cpuid_feature_extract_unsigned_field(dfr0, - ID_AA64DFR0_PMUVER_SHIFT); - if (pmuver == ID_AA64DFR0_PMUVER_IMP_DEF || pmuver == 0) + ID_AA64DFR0_EL1_PMUVer_SHIFT); + if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF || pmuver == 0) return; cpu_pmu->pmuver = pmuver; @@ -1172,7 +1172,7 @@ static void __armv8pmu_probe_pmu(void *info) pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS); /* store PMMIR_EL1 register for sysfs */ - if (pmuver >= ID_AA64DFR0_PMUVER_8_4 && (pmceid_raw[1] & BIT(31))) + if (pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4 && (pmceid_raw[1] & BIT(31))) cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1); else cpu_pmu->reg_pmmir = 0; diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index f6f58e6265df89d8c1ce7a49ff052728cbdb9c8c..b4eece3eb17d0336d4a735579f8a958c25198f96 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -9,9 +9,27 @@ #include #include +static u64 perf_ext_regs_value(int idx) +{ + switch (idx) { + case PERF_REG_ARM64_VG: + if (WARN_ON_ONCE(!system_supports_sve())) + return 0; + + /* + * Vector granule is current length in bits of SVE registers + * divided by 64. + */ + return (task_get_sve_vl(current) * 8) / 64; + default: + WARN_ON_ONCE(true); + return 0; + } +} + u64 perf_reg_value(struct pt_regs *regs, int idx) { - if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_MAX)) + if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_EXTENDED_MAX)) return 0; /* @@ -51,6 +69,9 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) if ((u32)idx == PERF_REG_ARM64_PC) return regs->pc; + if ((u32)idx >= PERF_REG_ARM64_MAX) + return perf_ext_regs_value(idx); + return regs->regs[idx]; } @@ -58,7 +79,12 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) int perf_reg_validate(u64 mask) { - if (!mask || mask & REG_RESERVED) + u64 reserved_mask = REG_RESERVED; + + if (system_supports_sve()) + reserved_mask &= ~(1ULL << PERF_REG_ARM64_VG); + + if (!mask || mask & reserved_mask) return -EINVAL; return 0; diff --git a/arch/arm64/kernel/pi/kaslr_early.c b/arch/arm64/kernel/pi/kaslr_early.c index 6c3855e693956499b3f20f497574efc8111a1cfb..17bff6e399e46b0bb33efa2bf9f1ebe58d1a770d 100644 --- a/arch/arm64/kernel/pi/kaslr_early.c +++ b/arch/arm64/kernel/pi/kaslr_early.c @@ -94,11 +94,9 @@ asmlinkage u64 kaslr_early_init(void *fdt) seed = get_kaslr_seed(fdt); if (!seed) { -#ifdef CONFIG_ARCH_RANDOM - if (!__early_cpu_has_rndr() || - !__arm64_rndr((unsigned long *)&seed)) -#endif - return 0; + if (!__early_cpu_has_rndr() || + !__arm64_rndr((unsigned long *)&seed)) + return 0; } /* diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index d1d1823202453d2259536636099b75b7d0f09d99..c9e4d072028561d8b0730a8da68f193ad14197ee 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -44,13 +44,28 @@ post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *); static void __kprobes arch_prepare_ss_slot(struct kprobe *p) { kprobe_opcode_t *addr = p->ainsn.api.insn; - void *addrs[] = {addr, addr + 1}; - u32 insns[] = {p->opcode, BRK64_OPCODE_KPROBES_SS}; - /* prepare insn slot */ - aarch64_insn_patch_text(addrs, insns, 2); - - flush_icache_range((uintptr_t)addr, (uintptr_t)(addr + MAX_INSN_SIZE)); + /* + * Prepare insn slot, Mark Rutland points out it depends on a coupe of + * subtleties: + * + * - That the I-cache maintenance for these instructions is complete + * *before* the kprobe BRK is written (and aarch64_insn_patch_text_nosync() + * ensures this, but just omits causing a Context-Synchronization-Event + * on all CPUS). + * + * - That the kprobe BRK results in an exception (and consequently a + * Context-Synchronoization-Event), which ensures that the CPU will + * fetch thesingle-step slot instructions *after* this, ensuring that + * the new instructions are used + * + * It supposes to place ISB after patching to guarantee I-cache maintenance + * is observed on all CPUS, however, single-step slot is installed in + * the BRK exception handler, so it is unnecessary to generate + * Contex-Synchronization-Event via ISB again. + */ + aarch64_insn_patch_text_nosync(addr, p->opcode); + aarch64_insn_patch_text_nosync(addr + 1, BRK64_OPCODE_KPROBES_SS); /* * Needs restoring of return address after stepping xol. diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 92bcc1768f0b997b761771ba2dad659a0fe16953..044a7d7f1f6adb49326129cfb8724afe55f1b5d0 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -279,10 +279,6 @@ void flush_thread(void) flush_tagged_addr_state(); } -void release_thread(struct task_struct *dead_task) -{ -} - void arch_release_task_struct(struct task_struct *tsk) { fpsimd_release_task(tsk); @@ -595,7 +591,7 @@ unsigned long __get_wchan(struct task_struct *p) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= prandom_u32_max(PAGE_SIZE); return sp & ~0xf; } diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 40be3a7c2c53154a17be367d56badf57b9e1c48a..bfce41c2a53b3a396f7231ef359946aca0027de9 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -168,7 +168,7 @@ static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void) /* If the CPU has CSV2 set, we're safe */ pfr0 = read_cpuid(ID_AA64PFR0_EL1); - if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT)) + if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT)) return SPECTRE_UNAFFECTED; /* Alternatively, we have a list of unaffected CPUs */ @@ -586,7 +586,7 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt, if (spectre_v4_mitigations_off()) return; - if (cpus_have_final_cap(ARM64_SSBS)) + if (cpus_have_cap(ARM64_SSBS)) return; if (spectre_v4_mitigations_dynamic()) @@ -868,6 +868,10 @@ u8 spectre_bhb_loop_affected(int scope) MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), {}, }; + static const struct midr_range spectre_bhb_k11_list[] = { + MIDR_ALL_VERSIONS(MIDR_AMPERE1), + {}, + }; static const struct midr_range spectre_bhb_k8_list[] = { MIDR_ALL_VERSIONS(MIDR_CORTEX_A72), MIDR_ALL_VERSIONS(MIDR_CORTEX_A57), @@ -878,6 +882,8 @@ u8 spectre_bhb_loop_affected(int scope) k = 32; else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k24_list)) k = 24; + else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k11_list)) + k = 11; else if (is_midr_in_range_list(read_cpuid_id(), spectre_bhb_k8_list)) k = 8; @@ -945,7 +951,7 @@ static bool supports_ecbhb(int scope) mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); return cpuid_feature_extract_unsigned_field(mmfr1, - ID_AA64MMFR1_ECBHB_SHIFT); + ID_AA64MMFR1_EL1_ECBHB_SHIFT); } bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry, @@ -988,6 +994,14 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot) isb(); } +static bool __read_mostly __nospectre_bhb; +static int __init parse_spectre_bhb_param(char *str) +{ + __nospectre_bhb = true; + return 0; +} +early_param("nospectre_bhb", parse_spectre_bhb_param); + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1001,7 +1015,7 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { pr_info_once("spectre-bhb mitigation disabled by compile time option\n"); - } else if (cpu_mitigations_off()) { + } else if (cpu_mitigations_off() || __nospectre_bhb) { pr_info_once("spectre-bhb mitigation disabled by command line option\n"); } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index ab7f4c4761046a38adfaec8dcfe38db5db405210..29a8e444db836a70a17fecf2794f358a5f8f3003 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -38,7 +38,7 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu) static int cpu_psci_cpu_boot(unsigned int cpu) { - phys_addr_t pa_secondary_entry = __pa_symbol(function_nocfi(secondary_entry)); + phys_addr_t pa_secondary_entry = __pa_symbol(secondary_entry); int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry); if (err) pr_err("failed to boot CPU%d (%d)\n", cpu, err); diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 21da83187a602c499e4240d538d28da3bed5624d..c2fb5755bbecbde36117f27a6a6e0c52cca412f5 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -121,7 +121,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) { return ((addr & ~(THREAD_SIZE - 1)) == (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || - on_irq_stack(addr, sizeof(unsigned long), NULL); + on_irq_stack(addr, sizeof(unsigned long)); } /** @@ -666,10 +666,18 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset, static int tls_get(struct task_struct *target, const struct user_regset *regset, struct membuf to) { + int ret; + if (target == current) tls_preserve_current_state(); - return membuf_store(&to, target->thread.uw.tp_value); + ret = membuf_store(&to, target->thread.uw.tp_value); + if (system_supports_tpidr2()) + ret = membuf_store(&to, target->thread.tpidr2_el0); + else + ret = membuf_zero(&to, sizeof(u64)); + + return ret; } static int tls_set(struct task_struct *target, const struct user_regset *regset, @@ -677,13 +685,20 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset, const void *kbuf, const void __user *ubuf) { int ret; - unsigned long tls = target->thread.uw.tp_value; + unsigned long tls[2]; - ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); + tls[0] = target->thread.uw.tp_value; + if (system_supports_sme()) + tls[1] = target->thread.tpidr2_el0; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count); if (ret) return ret; - target->thread.uw.tp_value = tls; + target->thread.uw.tp_value = tls[0]; + if (system_supports_sme()) + target->thread.tpidr2_el0 = tls[1]; + return ret; } @@ -882,7 +897,7 @@ static int sve_set_common(struct task_struct *target, * state and ensure there's storage. */ if (target->thread.svcr != old_svcr) - sve_alloc(target); + sve_alloc(target, true); } /* Registers: FPSIMD-only case */ @@ -912,7 +927,7 @@ static int sve_set_common(struct task_struct *target, goto out; } - sve_alloc(target); + sve_alloc(target, true); if (!target->thread.sve_state) { ret = -ENOMEM; clear_tsk_thread_flag(target, TIF_SVE); @@ -1082,9 +1097,8 @@ static int za_set(struct task_struct *target, /* Ensure there is some SVE storage for streaming mode */ if (!target->thread.sve_state) { - sve_alloc(target); + sve_alloc(target, false); if (!target->thread.sve_state) { - clear_thread_flag(TIF_SME); ret = -ENOMEM; goto out; } @@ -1094,7 +1108,6 @@ static int za_set(struct task_struct *target, sme_alloc(target); if (!target->thread.za_state) { ret = -ENOMEM; - clear_tsk_thread_flag(target, TIF_SME); goto out; } @@ -1392,7 +1405,7 @@ static const struct user_regset aarch64_regsets[] = { }, [REGSET_TLS] = { .core_note_type = NT_ARM_TLS, - .n = 1, + .n = 2, .size = sizeof(void *), .align = sizeof(void *), .regset_get = tls_get, diff --git a/arch/arm64/kernel/reloc_test_core.c b/arch/arm64/kernel/reloc_test_core.c index e87a2b7f20f613830daec851755813e7195bd1c0..99f2ffe9fc0576400fa03e593b9ae7658892031e 100644 --- a/arch/arm64/kernel/reloc_test_core.c +++ b/arch/arm64/kernel/reloc_test_core.c @@ -48,7 +48,7 @@ static struct { { "R_AARCH64_PREL16", relative_data16, (u64)&sym64_rel }, }; -static int reloc_test_init(void) +static int __init reloc_test_init(void) { int i; @@ -67,7 +67,7 @@ static int reloc_test_init(void) return 0; } -static void reloc_test_exit(void) +static void __exit reloc_test_exit(void) { } diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index d20620a1c51a470ed3e43f0ac0370e8c68d9b8e0..d56e170e1ca7c637c61bfdc08c94fa85f09edeb2 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -162,38 +162,6 @@ static int init_sdei_scs(void) return err; } -static bool on_sdei_normal_stack(unsigned long sp, unsigned long size, - struct stack_info *info) -{ - unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr); - unsigned long high = low + SDEI_STACK_SIZE; - - return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info); -} - -static bool on_sdei_critical_stack(unsigned long sp, unsigned long size, - struct stack_info *info) -{ - unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr); - unsigned long high = low + SDEI_STACK_SIZE; - - return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info); -} - -bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info) -{ - if (!IS_ENABLED(CONFIG_VMAP_STACK)) - return false; - - if (on_sdei_critical_stack(sp, size, info)) - return true; - - if (on_sdei_normal_stack(sp, size, info)) - return true; - - return false; -} - unsigned long sdei_arch_get_entry_point(int conduit) { /* diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 3e6d0352d7d36469d85853af6059a9523a5fa8e4..9ad911f1647c8a204424b992255f11de92279332 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -91,7 +91,7 @@ static size_t sigframe_size(struct rt_sigframe_user_layout const *user) * not taken into account. This limit is not a guarantee and is * NOT ABI. */ -#define SIGFRAME_MAXSZ SZ_64K +#define SIGFRAME_MAXSZ SZ_256K static int __sigframe_alloc(struct rt_sigframe_user_layout *user, unsigned long *offset, size_t size, bool extend) @@ -310,7 +310,7 @@ static int restore_sve_fpsimd_context(struct user_ctxs *user) fpsimd_flush_task_state(current); /* From now, fpsimd_thread_switch() won't touch thread.sve_state */ - sve_alloc(current); + sve_alloc(current, true); if (!current->thread.sve_state) { clear_thread_flag(TIF_SVE); return -ENOMEM; @@ -926,6 +926,16 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* Signal handlers are invoked with ZA and streaming mode disabled */ if (system_supports_sme()) { + /* + * If we were in streaming mode the saved register + * state was SVE but we will exit SM and use the + * FPSIMD register state - flush the saved FPSIMD + * register state in case it gets loaded. + */ + if (current->thread.svcr & SVCR_SM_MASK) + memset(¤t->thread.uw.fpsimd_state, 0, + sizeof(current->thread.uw.fpsimd_state)); + current->thread.svcr &= ~(SVCR_ZA_MASK | SVCR_SM_MASK); sme_smstop(); diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index 617f78ad43a185c277f9d0193252039f216dba7d..97c9de57725dfddb59ee4a19ccbd883dd3b24cc9 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -101,6 +101,9 @@ SYM_FUNC_END(__cpu_suspend_enter) SYM_CODE_START(cpu_resume) bl init_kernel_el bl finalise_el2 +#if VA_BITS > 48 + ldr_l x0, vabits_actual +#endif bl __cpu_setup /* enable the MMU early - so we can access sleep_save_stash by va */ adrp x1, swapper_pg_dir diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 7e1624ecab3c80ea6b34437d91672630e434a52a..49029eace3ad9c8b789c5bfcdb6639867d2dd6cc 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -66,7 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu) static int smp_spin_table_cpu_prepare(unsigned int cpu) { __le64 __iomem *release_addr; - phys_addr_t pa_holding_pen = __pa_symbol(function_nocfi(secondary_holding_pen)); + phys_addr_t pa_holding_pen = __pa_symbol(secondary_holding_pen); if (!cpu_release_addr[cpu]) return -ENODEV; diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index ce190ee18a20121893470ab005ff0bccb6aa5f14..634279b3b03d1b078e7554fe82191ec302cd2bbe 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -67,31 +67,6 @@ static inline void unwind_init_from_task(struct unwind_state *state, state->pc = thread_saved_pc(task); } -/* - * We can only safely access per-cpu stacks from current in a non-preemptible - * context. - */ -static bool on_accessible_stack(const struct task_struct *tsk, - unsigned long sp, unsigned long size, - struct stack_info *info) -{ - if (info) - info->type = STACK_TYPE_UNKNOWN; - - if (on_task_stack(tsk, sp, size, info)) - return true; - if (tsk != current || preemptible()) - return false; - if (on_irq_stack(sp, size, info)) - return true; - if (on_overflow_stack(sp, size, info)) - return true; - if (on_sdei_stack(sp, size, info)) - return true; - - return false; -} - /* * Unwind from one frame record (A) to the next frame record (B). * @@ -103,14 +78,13 @@ static int notrace unwind_next(struct unwind_state *state) { struct task_struct *tsk = state->task; unsigned long fp = state->fp; - struct stack_info info; int err; /* Final frame; nothing to unwind */ if (fp == (unsigned long)task_pt_regs(tsk)->stackframe) return -ENOENT; - err = unwind_next_common(state, &info, on_accessible_stack, NULL); + err = unwind_next_frame_record(state); if (err) return err; @@ -190,11 +164,47 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl) barrier(); } +/* + * Per-cpu stacks are only accessible when unwinding the current task in a + * non-preemptible context. + */ +#define STACKINFO_CPU(name) \ + ({ \ + ((task == current) && !preemptible()) \ + ? stackinfo_get_##name() \ + : stackinfo_get_unknown(); \ + }) + +/* + * SDEI stacks are only accessible when unwinding the current task in an NMI + * context. + */ +#define STACKINFO_SDEI(name) \ + ({ \ + ((task == current) && in_nmi()) \ + ? stackinfo_get_sdei_##name() \ + : stackinfo_get_unknown(); \ + }) + noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, struct task_struct *task, struct pt_regs *regs) { - struct unwind_state state; + struct stack_info stacks[] = { + stackinfo_get_task(task), + STACKINFO_CPU(irq), +#if defined(CONFIG_VMAP_STACK) + STACKINFO_CPU(overflow), +#endif +#if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE) + STACKINFO_SDEI(normal), + STACKINFO_SDEI(critical), +#endif + }; + struct unwind_state state = { + .stacks = stacks, + .nr_stacks = ARRAY_SIZE(stacks), + }; if (regs) { if (task != current) diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index 9135fe0f3df536f4fe77832f2758ed5815c877b1..8b02d310838f92404c7d61a1083ffa8398194d20 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -43,6 +43,8 @@ void notrace __cpu_suspend_exit(void) { unsigned int cpu = smp_processor_id(); + mte_suspend_exit(); + /* * We are resuming from reset with the idmap active in TTBR0_EL1. * We must uninstall the idmap and restore the expected MMU diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 733451fe7e41f031e2b5cf34396521d8ec0a0639..d72e8f23422daf6dc0b812855d5d28a36bbd4c54 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -67,7 +67,7 @@ static void invoke_syscall(struct pt_regs *regs, unsigned int scno, * * The resulting 5 bits of entropy is seen in SP[8:4]. */ - choose_random_kstack_offset(get_random_int() & 0x1FF); + choose_random_kstack_offset(get_random_u16() & 0x1FF); } static inline bool has_syscall_work(unsigned long flags) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index 869ffc4d4484777310d2bf905c3d7346cef04c4c..817d788cd86669439cf4b55af45214f016cc5efb 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -22,46 +22,6 @@ #include #include -void store_cpu_topology(unsigned int cpuid) -{ - struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; - u64 mpidr; - - if (cpuid_topo->package_id != -1) - goto topology_populated; - - mpidr = read_cpuid_mpidr(); - - /* Uniprocessor systems can rely on default topology values */ - if (mpidr & MPIDR_UP_BITMASK) - return; - - /* - * This would be the place to create cpu topology based on MPIDR. - * - * However, it cannot be trusted to depict the actual topology; some - * pieces of the architecture enforce an artificial cap on Aff0 values - * (e.g. GICv3's ICC_SGI1R_EL1 limits it to 15), leading to an - * artificial cycling of Aff1, Aff2 and Aff3 values. IOW, these end up - * having absolutely no relationship to the actual underlying system - * topology, and cannot be reasonably used as core / package ID. - * - * If the MT bit is set, Aff0 *could* be used to define a thread ID, but - * we still wouldn't be able to obtain a sane core ID. This means we - * need to entirely ignore MPIDR for any topology deduction. - */ - cpuid_topo->thread_id = -1; - cpuid_topo->core_id = cpuid; - cpuid_topo->package_id = cpu_to_node(cpuid); - - pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n", - cpuid, cpuid_topo->package_id, cpuid_topo->core_id, - cpuid_topo->thread_id, mpidr); - -topology_populated: - update_siblings_masks(cpuid); -} - #ifdef CONFIG_ACPI static bool __init acpi_cpu_is_threaded(int cpu) { @@ -237,7 +197,7 @@ static void amu_fie_setup(const struct cpumask *cpus) for_each_cpu(cpu, cpus) { if (!freq_counters_valid(cpu) || freq_inv_set_max_ratio(cpu, - cpufreq_get_hw_max_freq(cpu) * 1000, + cpufreq_get_hw_max_freq(cpu) * 1000ULL, arch_timer_get_rate())) return; } @@ -296,12 +256,25 @@ core_initcall(init_amu_fie); static void cpu_read_corecnt(void *val) { + /* + * A value of 0 can be returned if the current CPU does not support AMUs + * or if the counter is disabled for this CPU. A return value of 0 at + * counter read is properly handled as an error case by the users of the + * counter. + */ *(u64 *)val = read_corecnt(); } static void cpu_read_constcnt(void *val) { - *(u64 *)val = read_constcnt(); + /* + * Return 0 if the current CPU is affected by erratum 2457168. A value + * of 0 is also returned if the current CPU does not support AMUs or if + * the counter is disabled. A return value of 0 at counter read is + * properly handled as an error case by the users of the counter. + */ + *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ? + 0UL : read_constcnt(); } static inline @@ -328,7 +301,22 @@ int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) */ bool cpc_ffh_supported(void) { - return freq_counters_valid(get_cpu_with_amu_feat()); + int cpu = get_cpu_with_amu_feat(); + + /* + * FFH is considered supported if there is at least one present CPU that + * supports AMUs. Using FFH to read core and reference counters for CPUs + * that do not support AMUs, have counters disabled or that are affected + * by errata, will result in a return value of 0. + * + * This is done to allow any enabled and valid counters to be read + * through FFH, knowing that potentially returning 0 as counter value is + * properly handled by the users of these counters. + */ + if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) + return false; + + return true; } int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index b7fed33981f7b7683900f7e7f961d67fed6c51bc..23d281ed7621e4eae8f6772b323b2a5e578db53b 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -180,12 +181,12 @@ static void dump_kernel_instr(const char *lvl, struct pt_regs *regs) #define S_SMP " SMP" -static int __die(const char *str, int err, struct pt_regs *regs) +static int __die(const char *str, long err, struct pt_regs *regs) { static int die_counter; int ret; - pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", + pr_emerg("Internal error: %s: %016lx [#%d]" S_PREEMPT S_SMP "\n", str, err, ++die_counter); /* trap and error numbers are mostly meaningless on ARM */ @@ -206,7 +207,7 @@ static DEFINE_RAW_SPINLOCK(die_lock); /* * This function is protected against re-entrancy. */ -void die(const char *str, struct pt_regs *regs, int err) +void die(const char *str, struct pt_regs *regs, long err) { int ret; unsigned long flags; @@ -485,7 +486,7 @@ void arm64_notify_segfault(unsigned long addr) force_signal_inject(SIGSEGV, code, addr, 0); } -void do_undefinstr(struct pt_regs *regs) +void do_undefinstr(struct pt_regs *regs, unsigned long esr) { /* check for AArch32 breakpoint instructions */ if (!aarch32_break_handler(regs)) @@ -494,28 +495,38 @@ void do_undefinstr(struct pt_regs *regs) if (call_undef_hook(regs) == 0) return; - BUG_ON(!user_mode(regs)); + if (!user_mode(regs)) + die("Oops - Undefined instruction", regs, esr); + force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); } NOKPROBE_SYMBOL(do_undefinstr); -void do_bti(struct pt_regs *regs) +void do_el0_bti(struct pt_regs *regs) { - BUG_ON(!user_mode(regs)); force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); } -NOKPROBE_SYMBOL(do_bti); -void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr) +void do_el1_bti(struct pt_regs *regs, unsigned long esr) +{ + die("Oops - BTI", regs, esr); +} +NOKPROBE_SYMBOL(do_el1_bti); + +void do_el0_fpac(struct pt_regs *regs, unsigned long esr) +{ + force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr); +} + +void do_el1_fpac(struct pt_regs *regs, unsigned long esr) { /* - * Unexpected FPAC exception or pointer authentication failure in - * the kernel: kill the task before it does any more harm. + * Unexpected FPAC exception in the kernel: kill the task before it + * does any more harm. */ - BUG_ON(!user_mode(regs)); - force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr); + die("Oops - FPAC", regs, esr); } -NOKPROBE_SYMBOL(do_ptrauth_fault); +NOKPROBE_SYMBOL(do_el1_fpac) #define __user_cache_maint(insn, address, res) \ if (address >= TASK_SIZE_MAX) { \ @@ -758,7 +769,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs) hook_base = cp15_64_hooks; break; default: - do_undefinstr(regs); + do_undefinstr(regs, esr); return; } @@ -773,7 +784,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs) * EL0. Fall back to our usual undefined instruction handler * so that we handle these consistently. */ - do_undefinstr(regs); + do_undefinstr(regs, esr); } NOKPROBE_SYMBOL(do_cp15instr); #endif @@ -793,7 +804,7 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs) * back to our usual undefined instruction handler so that we handle * these consistently. */ - do_undefinstr(regs); + do_undefinstr(regs, esr); } NOKPROBE_SYMBOL(do_sysinstr); @@ -970,7 +981,7 @@ static int bug_handler(struct pt_regs *regs, unsigned long esr) { switch (report_bug(regs->pc, regs)) { case BUG_TRAP_TYPE_BUG: - die("Oops - BUG", regs, 0); + die("Oops - BUG", regs, esr); break; case BUG_TRAP_TYPE_WARN: @@ -991,6 +1002,38 @@ static struct break_hook bug_break_hook = { .imm = BUG_BRK_IMM, }; +#ifdef CONFIG_CFI_CLANG +static int cfi_handler(struct pt_regs *regs, unsigned long esr) +{ + unsigned long target; + u32 type; + + target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr)); + type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr)); + + switch (report_cfi_failure(regs, regs->pc, &target, type)) { + case BUG_TRAP_TYPE_BUG: + die("Oops - CFI", regs, 0); + break; + + case BUG_TRAP_TYPE_WARN: + break; + + default: + return DBG_HOOK_ERROR; + } + + arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); + return DBG_HOOK_HANDLED; +} + +static struct break_hook cfi_break_hook = { + .fn = cfi_handler, + .imm = CFI_BRK_IMM_BASE, + .mask = CFI_BRK_IMM_MASK, +}; +#endif /* CONFIG_CFI_CLANG */ + static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr) { pr_err("%s generated an invalid instruction at %pS!\n", @@ -1038,7 +1081,7 @@ static int kasan_handler(struct pt_regs *regs, unsigned long esr) * This is something that might be fixed at some point in the future. */ if (!recover) - die("Oops - KASAN", regs, 0); + die("Oops - KASAN", regs, esr); /* If thread survives, skip over the brk instruction and continue: */ arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE); @@ -1052,6 +1095,9 @@ static struct break_hook kasan_break_hook = { }; #endif + +#define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK) + /* * Initial handler for AArch64 BRK exceptions * This handler only used until debug_traps_init(). @@ -1059,10 +1105,12 @@ static struct break_hook kasan_break_hook = { int __init early_brk64(unsigned long addr, unsigned long esr, struct pt_regs *regs) { +#ifdef CONFIG_CFI_CLANG + if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE) + return cfi_handler(regs, esr) != DBG_HOOK_HANDLED; +#endif #ifdef CONFIG_KASAN_SW_TAGS - unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK; - - if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) + if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM) return kasan_handler(regs, esr) != DBG_HOOK_HANDLED; #endif return bug_handler(regs, esr) != DBG_HOOK_HANDLED; @@ -1071,6 +1119,9 @@ int __init early_brk64(unsigned long addr, unsigned long esr, void __init trap_init(void) { register_kernel_break_hook(&bug_break_hook); +#ifdef CONFIG_CFI_CLANG + register_kernel_break_hook(&cfi_break_hook); +#endif register_kernel_break_hook(&fault_break_hook); #ifdef CONFIG_KASAN_SW_TAGS register_kernel_break_hook(&kasan_break_hook); diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index a61fc4f989b37bfe57f05931926edb03146ab89e..99ae81ab91a749e5a9ee37f546425ea96e9df70e 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -29,9 +29,6 @@ #include #include -extern char vdso_start[], vdso_end[]; -extern char vdso32_start[], vdso32_end[]; - enum vdso_abi { VDSO_ABI_AA64, VDSO_ABI_AA32, @@ -136,10 +133,11 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, vdso_info[VDSO_ABI_AA64].dm)) diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index bafbf78fab772d62286b895ba1823d366593b914..619e2dc7ee14cce1b906c1962c5429194260ad80 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -40,7 +40,8 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO # kernel with CONFIG_WERROR enabled. CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \ $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \ - $(CC_FLAGS_LTO) -Wmissing-prototypes -Wmissing-declarations + $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \ + -Wmissing-prototypes -Wmissing-declarations KASAN_SANITIZE := n KCSAN_SANITIZE := n UBSAN_SANITIZE := n diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S index e69fb4aaaf3ece102ba619cd2c6c403dc6f1b3cb..6028f1fe2d1cbf593af2e3b0973aff64c940a83d 100644 --- a/arch/arm64/kernel/vdso/vdso.lds.S +++ b/arch/arm64/kernel/vdso/vdso.lds.S @@ -48,6 +48,13 @@ SECTIONS PROVIDE (_etext = .); PROVIDE (etext = .); + . = ALIGN(4); + .altinstructions : { + __alt_instructions = .; + *(.altinstructions) + __alt_instructions_end = .; + } + .dynamic : { *(.dynamic) } :text :dynamic .rela.dyn : ALIGN(8) { *(.rela .rela*) } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 2ff0ef62abadc8cbf01a23b4b1df4f46039b55f7..94d33e296e10c57c34008af96d7a9adcb72d99ef 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -666,7 +666,6 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) kvm_vcpu_halt(vcpu); vcpu_clear_flag(vcpu, IN_WFIT); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); preempt_disable(); vgic_v4_load(vcpu); @@ -2114,7 +2113,7 @@ static int finalize_hyp_mode(void) * at, which would end badly once inaccessible. */ kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start); - kmemleak_free_part(__va(hyp_mem_base), hyp_mem_size); + kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); return pkvm_drop_host_privileges(); } @@ -2270,6 +2269,16 @@ static int __init early_kvm_mode_cfg(char *arg) if (!arg) return -EINVAL; + if (strcmp(arg, "none") == 0) { + kvm_mode = KVM_MODE_NONE; + return 0; + } + + if (!is_hyp_mode_available()) { + pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n"); + return 0; + } + if (strcmp(arg, "protected") == 0) { if (!is_kernel_in_hyp_mode()) kvm_mode = KVM_MODE_PROTECTED; @@ -2284,11 +2293,6 @@ static int __init early_kvm_mode_cfg(char *arg) return 0; } - if (strcmp(arg, "none") == 0) { - kvm_mode = KVM_MODE_NONE; - return 0; - } - return -EINVAL; } early_param("kvm-arm.mode", early_kvm_mode_cfg); diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c index 0b28d7db7c766a4b2e282589abf89da12148192b..fccf9ec01813f510a7d130ddcb7c4d7bbfc5358f 100644 --- a/arch/arm64/kvm/debug.c +++ b/arch/arm64/kvm/debug.c @@ -32,6 +32,10 @@ static DEFINE_PER_CPU(u64, mdcr_el2); * * Guest access to MDSCR_EL1 is trapped by the hypervisor and handled * after we have restored the preserved value to the main context. + * + * When single-step is enabled by userspace, we tweak PSTATE.SS on every + * guest entry. Preserve PSTATE.SS so we can restore the original value + * for the vcpu after the single-step is disabled. */ static void save_guest_debug_regs(struct kvm_vcpu *vcpu) { @@ -41,6 +45,9 @@ static void save_guest_debug_regs(struct kvm_vcpu *vcpu) trace_kvm_arm_set_dreg32("Saved MDSCR_EL1", vcpu->arch.guest_debug_preserved.mdscr_el1); + + vcpu->arch.guest_debug_preserved.pstate_ss = + (*vcpu_cpsr(vcpu) & DBG_SPSR_SS); } static void restore_guest_debug_regs(struct kvm_vcpu *vcpu) @@ -51,6 +58,11 @@ static void restore_guest_debug_regs(struct kvm_vcpu *vcpu) trace_kvm_arm_set_dreg32("Restored MDSCR_EL1", vcpu_read_sys_reg(vcpu, MDSCR_EL1)); + + if (vcpu->arch.guest_debug_preserved.pstate_ss) + *vcpu_cpsr(vcpu) |= DBG_SPSR_SS; + else + *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; } /** @@ -188,7 +200,18 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) * debugging the system. */ if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { - *vcpu_cpsr(vcpu) |= DBG_SPSR_SS; + /* + * If the software step state at the last guest exit + * was Active-pending, we don't set DBG_SPSR_SS so + * that the state is maintained (to not run another + * single-step until the pending Software Step + * exception is taken). + */ + if (!vcpu_get_flag(vcpu, DBG_SS_ACTIVE_PENDING)) + *vcpu_cpsr(vcpu) |= DBG_SPSR_SS; + else + *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS; + mdscr = vcpu_read_sys_reg(vcpu, MDSCR_EL1); mdscr |= DBG_MDSCR_SS; vcpu_write_sys_reg(vcpu, mdscr, MDSCR_EL1); @@ -262,6 +285,15 @@ void kvm_arm_clear_debug(struct kvm_vcpu *vcpu) * Restore the guest's debug registers if we were using them. */ if (vcpu->guest_debug || kvm_vcpu_os_lock_enabled(vcpu)) { + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { + if (!(*vcpu_cpsr(vcpu) & DBG_SPSR_SS)) + /* + * Mark the vcpu as ACTIVE_PENDING + * until Software Step exception is taken. + */ + vcpu_set_flag(vcpu, DBG_SS_ACTIVE_PENDING); + } + restore_guest_debug_regs(vcpu); /* @@ -295,12 +327,12 @@ void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu) * If SPE is present on this CPU and is available at current EL, * we may need to check if the host state needs to be saved. */ - if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_PMSVER_SHIFT) && + if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) && !(read_sysreg_s(SYS_PMBIDR_EL1) & BIT(SYS_PMBIDR_EL1_P_SHIFT))) vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_SPE); /* Check if we have TRBE implemented and available at the host */ - if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRBE_SHIFT) && + if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceBuffer_SHIFT) && !(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_PROG)) vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_TRBE); } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index f802a3b3f8dbc86664d280601b1180ff2da0378f..2ff13a3f847966c2268cb21c91045e1609869265 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -937,6 +937,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, } else { /* If not enabled clear all flags */ vcpu->guest_debug = 0; + vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING); } out: diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index bbe5b393d689f0a1933c554887a674ca4d948890..e778eefcf214d8876f14eefd8bf8b881ddf96e09 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -152,8 +152,14 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu) run->debug.arch.hsr_high = upper_32_bits(esr); run->flags = KVM_DEBUG_ARCH_HSR_HIGH_VALID; - if (ESR_ELx_EC(esr) == ESR_ELx_EC_WATCHPT_LOW) + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_WATCHPT_LOW: run->debug.arch.far = vcpu->arch.fault.far_el2; + break; + case ESR_ELx_EC_SOFTSTP_LOW: + vcpu_clear_flag(vcpu, DBG_SS_ACTIVE_PENDING); + break; + } return 0; } diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 7839d075729b1601f28fad70443f405e98ea9498..8f3f93fa119ed85145ad14a9529df9530c381149 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -196,7 +196,7 @@ SYM_CODE_END(__kvm_hyp_vector) sub sp, sp, #(8 * 4) stp x2, x3, [sp, #(8 * 0)] stp x0, x1, [sp, #(8 * 2)] - alternative_cb spectre_bhb_patch_wa3 + alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3 /* Patched to mov WA3 when supported */ mov w0, #ARM_SMCCC_ARCH_WORKAROUND_1 alternative_cb_end @@ -216,7 +216,7 @@ SYM_CODE_END(__kvm_hyp_vector) mitigate_spectre_bhb_clear_insn .endif .if \indirect != 0 - alternative_cb kvm_patch_vector_branch + alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch /* * For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with: * diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h index fa6e466ed57f60640d5a502aac53517bbad2d592..07edfc7524c942eb2e199578b306bf92b869acbe 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h +++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h @@ -35,9 +35,9 @@ * - Data Independent Timing */ #define PVM_ID_AA64PFR0_ALLOW (\ - ARM64_FEATURE_MASK(ID_AA64PFR0_FP) | \ - ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD) | \ - ARM64_FEATURE_MASK(ID_AA64PFR0_DIT) \ + ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP) | \ + ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD) | \ + ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_DIT) \ ) /* @@ -49,11 +49,11 @@ * Supported by KVM */ #define PVM_ID_AA64PFR0_RESTRICT_UNSIGNED (\ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0), ID_AA64PFR0_ELx_64BIT_ONLY) | \ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), ID_AA64PFR0_ELx_64BIT_ONLY) | \ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL2), ID_AA64PFR0_ELx_64BIT_ONLY) | \ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL3), ID_AA64PFR0_ELx_64BIT_ONLY) | \ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_RAS), ID_AA64PFR0_RAS_V1) \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL2), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL3), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), ID_AA64PFR0_EL1_RAS_IMP) \ ) /* @@ -62,8 +62,8 @@ * - Speculative Store Bypassing */ #define PVM_ID_AA64PFR1_ALLOW (\ - ARM64_FEATURE_MASK(ID_AA64PFR1_BT) | \ - ARM64_FEATURE_MASK(ID_AA64PFR1_SSBS) \ + ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_BT) | \ + ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SSBS) \ ) /* @@ -74,10 +74,10 @@ * - Non-context synchronizing exception entry and exit */ #define PVM_ID_AA64MMFR0_ALLOW (\ - ARM64_FEATURE_MASK(ID_AA64MMFR0_BIGENDEL) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR0_SNSMEM) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR0_BIGENDEL0) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR0_EXS) \ + ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGEND) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_SNSMEM) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGENDEL0) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_EXS) \ ) /* @@ -86,8 +86,8 @@ * - 16-bit ASID */ #define PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED (\ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_PARANGE), ID_AA64MMFR0_PARANGE_40) | \ - FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_ASID), ID_AA64MMFR0_ASID_16) \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_PARANGE), ID_AA64MMFR0_EL1_PARANGE_40) | \ + FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_ASIDBITS), ID_AA64MMFR0_EL1_ASIDBITS_16) \ ) /* @@ -100,12 +100,12 @@ * - Enhanced Translation Synchronization */ #define PVM_ID_AA64MMFR1_ALLOW (\ - ARM64_FEATURE_MASK(ID_AA64MMFR1_HADBS) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_VMIDBITS) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_HPD) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_PAN) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_SPECSEI) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR1_ETS) \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_VMIDBits) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HPDS) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_PAN) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_SpecSEI) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_ETS) \ ) /* @@ -120,14 +120,14 @@ * - E0PDx mechanism */ #define PVM_ID_AA64MMFR2_ALLOW (\ - ARM64_FEATURE_MASK(ID_AA64MMFR2_CNP) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_UAO) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_IESB) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_AT) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_IDS) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_TTL) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_BBM) | \ - ARM64_FEATURE_MASK(ID_AA64MMFR2_E0PD) \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_CnP) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_UAO) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IESB) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_AT) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IDS) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_TTL) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_BBM) | \ + ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_E0PD) \ ) /* diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 99c8d8b73e70479b74f97e813aca4b21561ef929..85d3b7ae720fb0ae78e79709e9c555ab01531e0d 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -20,35 +20,35 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu) u64 cptr_set = 0; /* Protected KVM does not support AArch32 guests. */ - BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0), - PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_ELx_64BIT_ONLY); - BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), - PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_ELx_64BIT_ONLY); + BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), + PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY); + BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), + PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY); /* * Linux guests assume support for floating-point and Advanced SIMD. Do * not change the trapping behavior for these from the KVM default. */ - BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_FP), + BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP), PVM_ID_AA64PFR0_ALLOW)); - BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD), + BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD), PVM_ID_AA64PFR0_ALLOW)); /* Trap RAS unless all current versions are supported */ - if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_RAS), feature_ids) < - ID_AA64PFR0_RAS_V1P1) { + if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) < + ID_AA64PFR0_EL1_RAS_V1P1) { hcr_set |= HCR_TERR | HCR_TEA; hcr_clear |= HCR_FIEN; } /* Trap AMU */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_AMU), feature_ids)) { + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) { hcr_clear |= HCR_AMVOFFEN; cptr_set |= CPTR_EL2_TAM; } /* Trap SVE */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_SVE), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids)) cptr_set |= CPTR_EL2_TZ; vcpu->arch.hcr_el2 |= hcr_set; @@ -66,7 +66,7 @@ static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu) u64 hcr_clear = 0; /* Memory Tagging: Trap and Treat as Untagged if not supported. */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_MTE), feature_ids)) { + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) { hcr_set |= HCR_TID5; hcr_clear |= HCR_DCT | HCR_ATA; } @@ -86,32 +86,32 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu) u64 cptr_set = 0; /* Trap/constrain PMU */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_PMUVER), feature_ids)) { + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) { mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME | MDCR_EL2_HPMN_MASK; } /* Trap Debug */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids)) mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA | MDCR_EL2_TDE; /* Trap OS Double Lock */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DOUBLELOCK), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids)) mdcr_set |= MDCR_EL2_TDOSA; /* Trap SPE */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_PMSVER), feature_ids)) { + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) { mdcr_set |= MDCR_EL2_TPMS; mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; } /* Trap Trace Filter */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_TRACE_FILT), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids)) mdcr_set |= MDCR_EL2_TTRF; /* Trap Trace */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_TRACEVER), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids)) cptr_set |= CPTR_EL2_TTA; vcpu->arch.mdcr_el2 |= mdcr_set; @@ -128,7 +128,7 @@ static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu) u64 mdcr_set = 0; /* Trap Debug Communications Channel registers */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_FGT), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids)) mdcr_set |= MDCR_EL2_TDCC; vcpu->arch.mdcr_el2 |= mdcr_set; @@ -143,7 +143,7 @@ static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu) u64 hcr_set = 0; /* Trap LOR */ - if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_LOR), feature_ids)) + if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids)) hcr_set |= HCR_TLOR; vcpu->arch.hcr_el2 |= hcr_set; diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c index 58f645ad66bcbec800a36ecf900ff229c1ac7f92..ed6b58b19cfa53c4a7c4b799c593f2b91f517062 100644 --- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c +++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c @@ -39,41 +39,32 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc) DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace); -static bool on_overflow_stack(unsigned long sp, unsigned long size, - struct stack_info *info) +static struct stack_info stackinfo_get_overflow(void) { unsigned long low = (unsigned long)this_cpu_ptr(overflow_stack); unsigned long high = low + OVERFLOW_STACK_SIZE; - return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info); + return (struct stack_info) { + .low = low, + .high = high, + }; } -static bool on_hyp_stack(unsigned long sp, unsigned long size, - struct stack_info *info) +static struct stack_info stackinfo_get_hyp(void) { struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params); unsigned long high = params->stack_hyp_va; unsigned long low = high - PAGE_SIZE; - return on_stack(sp, size, low, high, STACK_TYPE_HYP, info); -} - -static bool on_accessible_stack(const struct task_struct *tsk, - unsigned long sp, unsigned long size, - struct stack_info *info) -{ - if (info) - info->type = STACK_TYPE_UNKNOWN; - - return (on_overflow_stack(sp, size, info) || - on_hyp_stack(sp, size, info)); + return (struct stack_info) { + .low = low, + .high = high, + }; } static int unwind_next(struct unwind_state *state) { - struct stack_info info; - - return unwind_next_common(state, &info, on_accessible_stack, NULL); + return unwind_next_frame_record(state); } static void notrace unwind(struct unwind_state *state, @@ -129,7 +120,14 @@ static bool pkvm_save_backtrace_entry(void *arg, unsigned long where) */ static void pkvm_save_backtrace(unsigned long fp, unsigned long pc) { - struct unwind_state state; + struct stack_info stacks[] = { + stackinfo_get_overflow(), + stackinfo_get_hyp(), + }; + struct unwind_state state = { + .stacks = stacks, + .nr_stacks = ARRAY_SIZE(stacks), + }; int idx = 0; kvm_nvhe_unwind_init(&state, fp, pc); diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 9f638570206188bc6cc54da60ae4ca64849edb26..8e9d49a964be61690d02998ec84e06e985265f66 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -143,7 +143,7 @@ static void __hyp_vgic_save_state(struct kvm_vcpu *vcpu) } } -/* Restore VGICv3 state on non_VEH systems */ +/* Restore VGICv3 state on non-VHE systems */ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index e20fa4475dac69673a95c4aa4034106a42b36e05..0f9ac25afdf40218b0059f36d541648e7e6facb3 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -92,9 +92,9 @@ static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu) PVM_ID_AA64PFR0_RESTRICT_UNSIGNED); /* Spectre and Meltdown mitigation in KVM */ - set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), + set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), (u64)kvm->arch.pfr0_csv2); - set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), + set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3), (u64)kvm->arch.pfr0_csv3); return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask; @@ -106,7 +106,7 @@ static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu) u64 allow_mask = PVM_ID_AA64PFR1_ALLOW; if (!kvm_has_mte(kvm)) - allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); + allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE); return id_aa64pfr1_el1_sys_val & allow_mask; } @@ -281,8 +281,8 @@ static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu, * No support for AArch32 guests, therefore, pKVM has no sanitized copy * of AArch32 feature id registers. */ - BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), - PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY); + BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), + PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_EL1_ELx_64BIT_ONLY); return pvm_access_raz_wi(vcpu, p, r); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index 2cb3867eb7c2bad97ed2e9eb1695e4dcd9e233bb..cdf8e76b0be141380c95a39e76398a222be75c93 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -61,7 +61,7 @@ struct kvm_pgtable_walk_data { static bool kvm_phys_is_valid(u64 phys) { - return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_PARANGE_MAX)); + return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX)); } static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index c9a13e487187cc790eb546d96a9b3d55db425a51..34c5feed9dc17480f516a4743ef5e3be78065220 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -92,9 +92,13 @@ static bool kvm_is_device_pfn(unsigned long pfn) static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; + void *virt; /* Allocated with __GFP_ZERO, so no need to zero */ - return kvm_mmu_memory_cache_alloc(mc); + virt = kvm_mmu_memory_cache_alloc(mc); + if (virt) + kvm_account_pgtable_pages(virt, 1); + return virt; } static void *kvm_host_zalloc_pages_exact(size_t size) @@ -102,6 +106,21 @@ static void *kvm_host_zalloc_pages_exact(size_t size) return alloc_pages_exact(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO); } +static void *kvm_s2_zalloc_pages_exact(size_t size) +{ + void *virt = kvm_host_zalloc_pages_exact(size); + + if (virt) + kvm_account_pgtable_pages(virt, (size >> PAGE_SHIFT)); + return virt; +} + +static void kvm_s2_free_pages_exact(void *virt, size_t size) +{ + kvm_account_pgtable_pages(virt, -(size >> PAGE_SHIFT)); + free_pages_exact(virt, size); +} + static void kvm_host_get_page(void *addr) { get_page(virt_to_page(addr)); @@ -112,6 +131,15 @@ static void kvm_host_put_page(void *addr) put_page(virt_to_page(addr)); } +static void kvm_s2_put_page(void *addr) +{ + struct page *p = virt_to_page(addr); + /* Dropping last refcount, the page will be freed */ + if (page_count(p) == 1) + kvm_account_pgtable_pages(addr, -1); + put_page(p); +} + static int kvm_host_page_count(void *addr) { return page_count(virt_to_page(addr)); @@ -625,10 +653,10 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { .zalloc_page = stage2_memcache_zalloc_page, - .zalloc_pages_exact = kvm_host_zalloc_pages_exact, - .free_pages_exact = free_pages_exact, + .zalloc_pages_exact = kvm_s2_zalloc_pages_exact, + .free_pages_exact = kvm_s2_free_pages_exact, .get_page = kvm_host_get_page, - .put_page = kvm_host_put_page, + .put_page = kvm_s2_put_page, .page_count = kvm_host_page_count, .phys_to_virt = kvm_host_va, .virt_to_phys = kvm_host_pa, diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 11c43bed5f975bcef3bbc6dc8335f8ae8aa30956..0003c7d37533a87ea5be34414b812996cf094fc8 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -33,12 +33,12 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm) pmuver = kvm->arch.arm_pmu->pmuver; switch (pmuver) { - case ID_AA64DFR0_PMUVER_8_0: + case ID_AA64DFR0_EL1_PMUVer_IMP: return GENMASK(9, 0); - case ID_AA64DFR0_PMUVER_8_1: - case ID_AA64DFR0_PMUVER_8_4: - case ID_AA64DFR0_PMUVER_8_5: - case ID_AA64DFR0_PMUVER_8_7: + case ID_AA64DFR0_EL1_PMUVer_V3P1: + case ID_AA64DFR0_EL1_PMUVer_V3P4: + case ID_AA64DFR0_EL1_PMUVer_V3P5: + case ID_AA64DFR0_EL1_PMUVer_V3P7: return GENMASK(15, 0); default: /* Shouldn't be here, just for sanity */ WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); @@ -774,7 +774,7 @@ void kvm_host_pmu_init(struct arm_pmu *pmu) { struct arm_pmu_entry *entry; - if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF) + if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) return; mutex_lock(&arm_pmus_lock); @@ -828,7 +828,7 @@ static struct arm_pmu *kvm_pmu_probe_armpmu(void) if (event->pmu) { pmu = to_arm_pmu(event->pmu); if (pmu->pmuver == 0 || - pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF) + pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF) pmu = NULL; } @@ -856,7 +856,7 @@ u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled * as RAZ */ - if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_4) + if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4) val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32); base = 32; } diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 0e08fbe68715e8e26623f086c1aedde9e8069b36..5ae18472205a9f34b73eb087b1ba383edee46ad0 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -359,7 +359,7 @@ int kvm_set_ipa_limit(void) mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); parange = cpuid_feature_extract_unsigned_field(mmfr0, - ID_AA64MMFR0_PARANGE_SHIFT); + ID_AA64MMFR0_EL1_PARANGE_SHIFT); /* * IPA size beyond 48 bits could not be supported * on either 4K or 16K page size. Hence let's cap @@ -367,20 +367,20 @@ int kvm_set_ipa_limit(void) * on the system. */ if (PAGE_SIZE != SZ_64K) - parange = min(parange, (unsigned int)ID_AA64MMFR0_PARANGE_48); + parange = min(parange, (unsigned int)ID_AA64MMFR0_EL1_PARANGE_48); /* * Check with ARMv8.5-GTG that our PAGE_SIZE is supported at * Stage-2. If not, things will stop very quickly. */ - switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_TGRAN_2_SHIFT)) { - case ID_AA64MMFR0_TGRAN_2_SUPPORTED_NONE: + switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_TGRAN_2_SHIFT)) { + case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE: kvm_err("PAGE_SIZE not supported at Stage-2, giving up\n"); return -EINVAL; - case ID_AA64MMFR0_TGRAN_2_SUPPORTED_DEFAULT: + case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT: kvm_debug("PAGE_SIZE supported at Stage-2 (default)\n"); break; - case ID_AA64MMFR0_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_TGRAN_2_SUPPORTED_MAX: + case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX: kvm_debug("PAGE_SIZE supported at Stage-2 (advertised)\n"); break; default: diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c index 949d19d603fbad9e59e433ca8ffc9bf62b1a7a06..3ace5b75813bd5aaf30352e31cc39ef4eb2de3fc 100644 --- a/arch/arm64/kvm/stacktrace.c +++ b/arch/arm64/kvm/stacktrace.c @@ -21,6 +21,54 @@ #include +static struct stack_info stackinfo_get_overflow(void) +{ + struct kvm_nvhe_stacktrace_info *stacktrace_info + = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); + unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base; + unsigned long high = low + OVERFLOW_STACK_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; +} + +static struct stack_info stackinfo_get_overflow_kern_va(void) +{ + unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack); + unsigned long high = low + OVERFLOW_STACK_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; +} + +static struct stack_info stackinfo_get_hyp(void) +{ + struct kvm_nvhe_stacktrace_info *stacktrace_info + = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); + unsigned long low = (unsigned long)stacktrace_info->stack_base; + unsigned long high = low + PAGE_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; +} + +static struct stack_info stackinfo_get_hyp_kern_va(void) +{ + unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page); + unsigned long high = low + PAGE_SIZE; + + return (struct stack_info) { + .low = low, + .high = high, + }; +} + /* * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs * @@ -34,73 +82,45 @@ * Returns true on success and updates @addr to its corresponding kernel VA; * otherwise returns false. */ -static bool kvm_nvhe_stack_kern_va(unsigned long *addr, - enum stack_type type) +static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size) { - struct kvm_nvhe_stacktrace_info *stacktrace_info; - unsigned long hyp_base, kern_base, hyp_offset; + struct stack_info stack_hyp, stack_kern; - stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); + stack_hyp = stackinfo_get_hyp(); + stack_kern = stackinfo_get_hyp_kern_va(); + if (stackinfo_on_stack(&stack_hyp, *addr, size)) + goto found; - switch (type) { - case STACK_TYPE_HYP: - kern_base = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page); - hyp_base = (unsigned long)stacktrace_info->stack_base; - break; - case STACK_TYPE_OVERFLOW: - kern_base = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack); - hyp_base = (unsigned long)stacktrace_info->overflow_stack_base; - break; - default: - return false; - } + stack_hyp = stackinfo_get_overflow(); + stack_kern = stackinfo_get_overflow_kern_va(); + if (stackinfo_on_stack(&stack_hyp, *addr, size)) + goto found; - hyp_offset = *addr - hyp_base; - - *addr = kern_base + hyp_offset; + return false; +found: + *addr = *addr - stack_hyp.low + stack_kern.low; return true; } -static bool on_overflow_stack(unsigned long sp, unsigned long size, - struct stack_info *info) -{ - struct kvm_nvhe_stacktrace_info *stacktrace_info - = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); - unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base; - unsigned long high = low + OVERFLOW_STACK_SIZE; - - return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info); -} - -static bool on_hyp_stack(unsigned long sp, unsigned long size, - struct stack_info *info) -{ - struct kvm_nvhe_stacktrace_info *stacktrace_info - = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); - unsigned long low = (unsigned long)stacktrace_info->stack_base; - unsigned long high = low + PAGE_SIZE; - - return on_stack(sp, size, low, high, STACK_TYPE_HYP, info); -} - -static bool on_accessible_stack(const struct task_struct *tsk, - unsigned long sp, unsigned long size, - struct stack_info *info) +/* + * Convert a KVN nVHE HYP frame record address to a kernel VA + */ +static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr) { - if (info) - info->type = STACK_TYPE_UNKNOWN; - - return (on_overflow_stack(sp, size, info) || - on_hyp_stack(sp, size, info)); + return kvm_nvhe_stack_kern_va(addr, 16); } static int unwind_next(struct unwind_state *state) { - struct stack_info info; - - return unwind_next_common(state, &info, on_accessible_stack, - kvm_nvhe_stack_kern_va); + /* + * The FP is in the hypervisor VA space. Convert it to the kernel VA + * space so it can be unwound by the regular unwind functions. + */ + if (!kvm_nvhe_stack_kern_record_va(&state->fp)) + return -EINVAL; + + return unwind_next_frame_record(state); } static void unwind(struct unwind_state *state, @@ -158,7 +178,14 @@ static void kvm_nvhe_dump_backtrace_end(void) static void hyp_dump_backtrace(unsigned long hyp_offset) { struct kvm_nvhe_stacktrace_info *stacktrace_info; - struct unwind_state state; + struct stack_info stacks[] = { + stackinfo_get_overflow_kern_va(), + stackinfo_get_hyp_kern_va(), + }; + struct unwind_state state = { + .stacks = stacks, + .nr_stacks = ARRAY_SIZE(stacks), + }; stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info); diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 3234f50b8c4b25160aea442a1c212fa88f6ecd2b..f4a7c5abcbca44070292e9a60db6f302acac783d 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -273,7 +273,7 @@ static bool trap_loregion(struct kvm_vcpu *vcpu, u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1); u32 sr = reg_to_encoding(r); - if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) { + if (!(val & (0xfUL << ID_AA64MMFR1_EL1_LO_SHIFT))) { kvm_inject_undefined(vcpu); return false; } @@ -1063,13 +1063,12 @@ static bool access_arch_timer(struct kvm_vcpu *vcpu, } /* Read a sanitised cpufeature ID register by sys_reg_desc */ -static u64 read_id_reg(const struct kvm_vcpu *vcpu, - struct sys_reg_desc const *r, bool raz) +static u64 read_id_reg(const struct kvm_vcpu *vcpu, struct sys_reg_desc const *r) { u32 id = reg_to_encoding(r); u64 val; - if (raz) + if (sysreg_visible_as_raz(vcpu, r)) return 0; val = read_sanitised_ftr_reg(id); @@ -1077,22 +1076,22 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, switch (id) { case SYS_ID_AA64PFR0_EL1: if (!vcpu_has_sve(vcpu)) - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_AMU); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2); + val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3); + val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3); if (kvm_vgic_global_state.type == VGIC_V3) { - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_GIC); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_GIC), 1); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC); + val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC), 1); } break; case SYS_ID_AA64PFR1_EL1: if (!kvm_has_mte(vcpu->kvm)) - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE); - val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_SME); + val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME); break; case SYS_ID_AA64ISAR1_EL1: if (!vcpu_has_ptrauth(vcpu)) @@ -1110,14 +1109,14 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, break; case SYS_ID_AA64DFR0_EL1: /* Limit debug to ARMv8.0 */ - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER); - val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), 6); + val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer); + val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6); /* Limit guests to PMUv3 for ARMv8.4 */ val = cpuid_feature_cap_perfmon_field(val, - ID_AA64DFR0_PMUVER_SHIFT, - kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0); + ID_AA64DFR0_EL1_PMUVer_SHIFT, + kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_EL1_PMUVer_V3P4 : 0); /* Hide SPE from guests */ - val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_PMSVER); + val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer); break; case SYS_ID_DFR0_EL1: /* Limit guests to PMUv3 for ARMv8.4 */ @@ -1145,34 +1144,37 @@ static unsigned int id_visibility(const struct kvm_vcpu *vcpu, return 0; } -/* cpufeature ID register access trap handlers */ - -static bool __access_id_reg(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r, - bool raz) +static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) { - if (p->is_write) - return write_to_read_only(vcpu, p, r); + /* + * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any + * EL. Promote to RAZ/WI in order to guarantee consistency between + * systems. + */ + if (!kvm_supports_32bit_el0()) + return REG_RAZ | REG_USER_WI; - p->regval = read_id_reg(vcpu, r, raz); - return true; + return id_visibility(vcpu, r); } +static unsigned int raz_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) +{ + return REG_RAZ; +} + +/* cpufeature ID register access trap handlers */ + static bool access_id_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { - bool raz = sysreg_visible_as_raz(vcpu, r); - - return __access_id_reg(vcpu, p, r, raz); -} + if (p->is_write) + return write_to_read_only(vcpu, p, r); -static bool access_raz_id_reg(struct kvm_vcpu *vcpu, - struct sys_reg_params *p, - const struct sys_reg_desc *r) -{ - return __access_id_reg(vcpu, p, r, true); + p->regval = read_id_reg(vcpu, r); + return true; } /* Visibility overrides for SVE-specific control registers */ @@ -1196,21 +1198,21 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, * it doesn't promise more than what is actually provided (the * guest could otherwise be covered in ectoplasmic residue). */ - csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT); + csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV2_SHIFT); if (csv2 > 1 || (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED)) return -EINVAL; /* Same thing for CSV3 */ - csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT); + csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV3_SHIFT); if (csv3 > 1 || (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED)) return -EINVAL; /* We can only differ with CSV[23], and anything else is an error */ - val ^= read_id_reg(vcpu, rd, false); - val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) | - (0xFUL << ID_AA64PFR0_CSV3_SHIFT)); + val ^= read_id_reg(vcpu, rd); + val &= ~(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2) | + ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3)); if (val) return -EINVAL; @@ -1227,45 +1229,21 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu, * are stored, and for set_id_reg() we don't allow the effective value * to be changed. */ -static int __get_id_reg(const struct kvm_vcpu *vcpu, - const struct sys_reg_desc *rd, u64 *val, - bool raz) -{ - *val = read_id_reg(vcpu, rd, raz); - return 0; -} - -static int __set_id_reg(const struct kvm_vcpu *vcpu, - const struct sys_reg_desc *rd, u64 val, - bool raz) -{ - /* This is what we mean by invariant: you can't change it. */ - if (val != read_id_reg(vcpu, rd, raz)) - return -EINVAL; - - return 0; -} - static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 *val) { - bool raz = sysreg_visible_as_raz(vcpu, rd); - - return __get_id_reg(vcpu, rd, val, raz); + *val = read_id_reg(vcpu, rd); + return 0; } static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, u64 val) { - bool raz = sysreg_visible_as_raz(vcpu, rd); - - return __set_id_reg(vcpu, rd, val, raz); -} + /* This is what we mean by invariant: you can't change it. */ + if (val != read_id_reg(vcpu, rd)) + return -EINVAL; -static int set_raz_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, - u64 val) -{ - return __set_id_reg(vcpu, rd, val, true); + return 0; } static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, @@ -1367,6 +1345,15 @@ static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, .visibility = id_visibility, \ } +/* sys_reg_desc initialiser for known cpufeature ID registers */ +#define AA32_ID_SANITISED(name) { \ + SYS_DESC(SYS_##name), \ + .access = access_id_reg, \ + .get_user = get_id_reg, \ + .set_user = set_id_reg, \ + .visibility = aa32_id_visibility, \ +} + /* * sys_reg_desc initialiser for architecturally unallocated cpufeature ID * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2 @@ -1374,9 +1361,10 @@ static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, */ #define ID_UNALLOCATED(crm, op2) { \ Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \ - .access = access_raz_id_reg, \ - .get_user = get_raz_reg, \ - .set_user = set_raz_id_reg, \ + .access = access_id_reg, \ + .get_user = get_id_reg, \ + .set_user = set_id_reg, \ + .visibility = raz_visibility \ } /* @@ -1386,9 +1374,10 @@ static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, */ #define ID_HIDDEN(name) { \ SYS_DESC(SYS_##name), \ - .access = access_raz_id_reg, \ - .get_user = get_raz_reg, \ - .set_user = set_raz_id_reg, \ + .access = access_id_reg, \ + .get_user = get_id_reg, \ + .set_user = set_id_reg, \ + .visibility = raz_visibility, \ } /* @@ -1452,33 +1441,33 @@ static const struct sys_reg_desc sys_reg_descs[] = { /* AArch64 mappings of the AArch32 ID registers */ /* CRm=1 */ - ID_SANITISED(ID_PFR0_EL1), - ID_SANITISED(ID_PFR1_EL1), - ID_SANITISED(ID_DFR0_EL1), + AA32_ID_SANITISED(ID_PFR0_EL1), + AA32_ID_SANITISED(ID_PFR1_EL1), + AA32_ID_SANITISED(ID_DFR0_EL1), ID_HIDDEN(ID_AFR0_EL1), - ID_SANITISED(ID_MMFR0_EL1), - ID_SANITISED(ID_MMFR1_EL1), - ID_SANITISED(ID_MMFR2_EL1), - ID_SANITISED(ID_MMFR3_EL1), + AA32_ID_SANITISED(ID_MMFR0_EL1), + AA32_ID_SANITISED(ID_MMFR1_EL1), + AA32_ID_SANITISED(ID_MMFR2_EL1), + AA32_ID_SANITISED(ID_MMFR3_EL1), /* CRm=2 */ - ID_SANITISED(ID_ISAR0_EL1), - ID_SANITISED(ID_ISAR1_EL1), - ID_SANITISED(ID_ISAR2_EL1), - ID_SANITISED(ID_ISAR3_EL1), - ID_SANITISED(ID_ISAR4_EL1), - ID_SANITISED(ID_ISAR5_EL1), - ID_SANITISED(ID_MMFR4_EL1), - ID_SANITISED(ID_ISAR6_EL1), + AA32_ID_SANITISED(ID_ISAR0_EL1), + AA32_ID_SANITISED(ID_ISAR1_EL1), + AA32_ID_SANITISED(ID_ISAR2_EL1), + AA32_ID_SANITISED(ID_ISAR3_EL1), + AA32_ID_SANITISED(ID_ISAR4_EL1), + AA32_ID_SANITISED(ID_ISAR5_EL1), + AA32_ID_SANITISED(ID_MMFR4_EL1), + AA32_ID_SANITISED(ID_ISAR6_EL1), /* CRm=3 */ - ID_SANITISED(MVFR0_EL1), - ID_SANITISED(MVFR1_EL1), - ID_SANITISED(MVFR2_EL1), + AA32_ID_SANITISED(MVFR0_EL1), + AA32_ID_SANITISED(MVFR1_EL1), + AA32_ID_SANITISED(MVFR2_EL1), ID_UNALLOCATED(3,3), - ID_SANITISED(ID_PFR2_EL1), + AA32_ID_SANITISED(ID_PFR2_EL1), ID_HIDDEN(ID_DFR1_EL1), - ID_SANITISED(ID_MMFR5_EL1), + AA32_ID_SANITISED(ID_MMFR5_EL1), ID_UNALLOCATED(3,7), /* AArch64 ID registers */ @@ -1825,11 +1814,11 @@ static bool trap_dbgdidr(struct kvm_vcpu *vcpu, } else { u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1); u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); - u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT); + u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL1_EL3_SHIFT); - p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) | - (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) | - (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20) + p->regval = ((((dfr >> ID_AA64DFR0_EL1_WRPs_SHIFT) & 0xf) << 28) | + (((dfr >> ID_AA64DFR0_EL1_BRPs_SHIFT) & 0xf) << 24) | + (((dfr >> ID_AA64DFR0_EL1_CTX_CMPs_SHIFT) & 0xf) << 20) | (6 << 16) | (1 << 15) | (el3 << 14) | (el3 << 12)); return true; } @@ -2809,6 +2798,9 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, if (!r) return -ENOENT; + if (sysreg_user_write_ignore(vcpu, r)) + return 0; + if (r->set_user) { ret = (r->set_user)(vcpu, r, val); } else { diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index a8c4cc32eb9af0f16f1847adf8c8ed57cd506d6e..e4ebb3a379fdb95e1d4fcc45e8e7d3221f0cf64d 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -86,6 +86,7 @@ struct sys_reg_desc { #define REG_HIDDEN (1 << 0) /* hidden from userspace and guest */ #define REG_RAZ (1 << 1) /* RAZ from userspace and guest */ +#define REG_USER_WI (1 << 2) /* WI from userspace only */ static __printf(2, 3) inline void print_sys_reg_msg(const struct sys_reg_params *p, @@ -136,22 +137,31 @@ static inline void reset_val(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r __vcpu_sys_reg(vcpu, r->reg) = r->val; } -static inline bool sysreg_hidden(const struct kvm_vcpu *vcpu, - const struct sys_reg_desc *r) +static inline unsigned int sysreg_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) { if (likely(!r->visibility)) - return false; + return 0; - return r->visibility(vcpu, r) & REG_HIDDEN; + return r->visibility(vcpu, r); +} + +static inline bool sysreg_hidden(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) +{ + return sysreg_visibility(vcpu, r) & REG_HIDDEN; } static inline bool sysreg_visible_as_raz(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - if (likely(!r->visibility)) - return false; + return sysreg_visibility(vcpu, r) & REG_RAZ; +} - return r->visibility(vcpu, r) & REG_RAZ; +static inline bool sysreg_user_write_ignore(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r) +{ + return sysreg_visibility(vcpu, r) & REG_USER_WI; } static inline int cmp_sys_reg(const struct sys_reg_desc *i1, diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c index acdb7b3cc97d65aaa9a540ea3773e46af1ea04cf..91b22a014610b22ffcc2883c9b499151d8e77f26 100644 --- a/arch/arm64/kvm/va_layout.c +++ b/arch/arm64/kvm/va_layout.c @@ -169,7 +169,7 @@ void __init kvm_update_va_mask(struct alt_instr *alt, * dictates it and we don't have any spare bits in the * address), NOP everything after masking the kernel VA. */ - if (has_vhe() || (!tag_val && i > 0)) { + if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) { updptr[i] = cpu_to_le32(aarch64_insn_gen_nop()); continue; } @@ -193,7 +193,8 @@ void kvm_patch_vector_branch(struct alt_instr *alt, BUG_ON(nr_inst != 4); - if (!cpus_have_const_cap(ARM64_SPECTRE_V3A) || WARN_ON_ONCE(has_vhe())) + if (!cpus_have_cap(ARM64_SPECTRE_V3A) || + WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))) return; /* diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9d3299a70242399e18b09585e1c7a0936d04f6a2..24d7778d1ce63fd64315e545c1bb32e0a92a007c 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -406,7 +406,7 @@ static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its, struct its_ite *ite; for_each_lpi_its(device, ite, its) { - if (!ite->collection || coll != ite->collection) + if (ite->collection != coll) continue; update_affinity_ite(kvm, ite); diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index b8b4cf0bcf39b795c6703a90d73c2b2712cf2efd..e1e0dca01839232dd186e1a9bd7b2264f15f8be6 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -43,17 +43,17 @@ static u32 get_cpu_asid_bits(void) { u32 asid; int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1), - ID_AA64MMFR0_ASID_SHIFT); + ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); switch (fld) { default: pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n", smp_processor_id(), fld); fallthrough; - case ID_AA64MMFR0_ASID_8: + case ID_AA64MMFR0_EL1_ASIDBITS_8: asid = 8; break; - case ID_AA64MMFR0_ASID_16: + case ID_AA64MMFR0_EL1_ASIDBITS_16: asid = 16; } diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 599cf81f568514123a3ee5874c2802f082a837a1..3cb101e8cb29baca75d3fd25287c9dfe932f7677 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -36,7 +36,7 @@ void arch_dma_prep_coherent(struct page *page, size_t size) { unsigned long start = (unsigned long)page_address(page); - dcache_clean_inval_poc(start, start + size); + dcache_clean_poc(start, start + size); } #ifdef CONFIG_IOMMU_DMA diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index c33f1fad27450307eb0fc4440914355e4de11e02..5b391490e045be91b9cf1e85a2b964474f4d8c4d 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -691,6 +691,9 @@ static int __kprobes do_translation_fault(unsigned long far, static int do_alignment_fault(unsigned long far, unsigned long esr, struct pt_regs *regs) { + if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) && + compat_user_mode(regs)) + return do_compat_alignment_fixup(far, regs); do_bad_area(far, esr, regs); return 0; } diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 0795028f017c212aaf893b29cfc203a58494d5a4..35e9a468d13e6ac68093c7516350815df5b009b5 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -245,7 +245,7 @@ static inline struct folio *hugetlb_swap_entry_to_folio(swp_entry_t entry) { VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry)); - return page_folio(pfn_to_page(swp_offset(entry))); + return page_folio(pfn_to_page(swp_offset_pfn(entry))); } void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index b9af30be813eefa89d8385fc90d098175e102cd8..4b4651ee47f271a76281055a8b542e2573711611 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -360,7 +360,7 @@ void __init arm64_memblock_init(void) extern u16 memstart_offset_seed; u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); int parange = cpuid_feature_extract_unsigned_field( - mmfr0, ID_AA64MMFR0_PARANGE_SHIFT); + mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT); s64 range = linear_region_size - BIT(id_aa64mmfr0_parange_to_phys_shift(parange)); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index db7c4e6ae57bbe02774748c9e050d246486b4f9f..9a7c38965154081eebea1936146d349989a222cb 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -331,12 +331,6 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, } BUG_ON(p4d_bad(p4d)); - /* - * No need for locking during early boot. And it doesn't work as - * expected with KASLR enabled. - */ - if (system_state != SYSTEM_BOOTING) - mutex_lock(&fixmap_lock); pudp = pud_set_fixmap_offset(p4dp, addr); do { pud_t old_pud = READ_ONCE(*pudp); @@ -368,15 +362,13 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end, } while (pudp++, addr = next, addr != end); pud_clear_fixmap(); - if (system_state != SYSTEM_BOOTING) - mutex_unlock(&fixmap_lock); } -static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - pgprot_t prot, - phys_addr_t (*pgtable_alloc)(int), - int flags) +static void __create_pgd_mapping_locked(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) { unsigned long addr, end, next; pgd_t *pgdp = pgd_offset_pgd(pgdir, virt); @@ -400,8 +392,20 @@ static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, } while (pgdp++, addr = next, addr != end); } +static void __create_pgd_mapping(pgd_t *pgdir, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot, + phys_addr_t (*pgtable_alloc)(int), + int flags) +{ + mutex_lock(&fixmap_lock); + __create_pgd_mapping_locked(pgdir, phys, virt, size, prot, + pgtable_alloc, flags); + mutex_unlock(&fixmap_lock); +} + #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 -extern __alias(__create_pgd_mapping) +extern __alias(__create_pgd_mapping_locked) void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot, phys_addr_t (*pgtable_alloc)(int), int flags); @@ -535,7 +539,7 @@ static void __init map_mem(pgd_t *pgdp) */ BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end)); - if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE)) + if (can_set_direct_map()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; /* @@ -642,24 +646,6 @@ static void __init map_kernel_segment(pgd_t *pgdp, void *va_start, void *va_end, vm_area_add_early(vma); } -static int __init parse_rodata(char *arg) -{ - int ret = strtobool(arg, &rodata_enabled); - if (!ret) { - rodata_full = false; - return 0; - } - - /* permit 'full' in addition to boolean options */ - if (strcmp(arg, "full")) - return -EINVAL; - - rodata_enabled = true; - rodata_full = true; - return 0; -} -early_param("rodata", parse_rodata); - #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 static int __init map_entry_trampoline(void) { @@ -704,7 +690,7 @@ static bool arm64_early_this_cpu_has_bti(void) pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1); return cpuid_feature_extract_unsigned_field(pfr1, - ID_AA64PFR1_BT_SHIFT); + ID_AA64PFR1_EL1_BT_SHIFT); } /* @@ -1198,14 +1184,6 @@ static void free_empty_tables(unsigned long addr, unsigned long end, } #endif -#if !ARM64_KERNEL_USES_PMD_MAPS -int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, - struct vmem_altmap *altmap) -{ - WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); - return vmemmap_populate_basepages(start, end, node, altmap); -} -#else /* !ARM64_KERNEL_USES_PMD_MAPS */ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { @@ -1217,6 +1195,10 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, pmd_t *pmdp; WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); + + if (!ARM64_KERNEL_USES_PMD_MAPS) + return vmemmap_populate_basepages(start, end, node, altmap); + do { next = pmd_addr_end(addr, end); @@ -1250,7 +1232,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return 0; } -#endif /* !ARM64_KERNEL_USES_PMD_MAPS */ #ifdef CONFIG_MEMORY_HOTPLUG void vmemmap_free(unsigned long start, unsigned long end, @@ -1565,11 +1546,7 @@ int arch_add_memory(int nid, u64 start, u64 size, VM_BUG_ON(!mhp_range_allowed(start, size, true)); - /* - * KFENCE requires linear map to be mapped at page granularity, so that - * it is possible to protect/unprotect single pages in the KFENCE pool. - */ - if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE)) + if (can_set_direct_map()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c index 4334dec93bd441eb285d27b97d2b86cfdf1d00c0..bed803d8e15856b56468fcbc6b271f82b4466261 100644 --- a/arch/arm64/mm/mteswap.c +++ b/arch/arm64/mm/mteswap.c @@ -53,7 +53,12 @@ bool mte_restore_tags(swp_entry_t entry, struct page *page) if (!tags) return false; - mte_restore_page_tags(page_address(page), tags); + /* + * Test PG_mte_tagged again in case it was racing with another + * set_pte_at(). + */ + if (!test_and_set_bit(PG_mte_tagged, &page->flags)) + mte_restore_page_tags(page_address(page), tags); return true; } diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 64e985eaa52d8df35244427a0479c0b173ee8668..d107c3d434e224552ae34011a4df3319cc977516 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -21,7 +21,13 @@ bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED bool can_set_direct_map(void) { - return rodata_full || debug_pagealloc_enabled(); + /* + * rodata_full, DEBUG_PAGEALLOC and KFENCE require linear map to be + * mapped at page granularity, so that it is possible to + * protect/unprotect single pages. + */ + return rodata_full || debug_pagealloc_enabled() || + IS_ENABLED(CONFIG_KFENCE); } static int change_page_range(pte_t *ptep, unsigned long addr, void *data) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 7837a69524c530deead50aab490ccadd686d19f9..b9ecbbae1e1abca1bdadfdc33f30c56bbb0074cd 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -48,17 +49,19 @@ #ifdef CONFIG_KASAN_HW_TAGS #define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1 -#else +#elif defined(CONFIG_ARM64_MTE) /* * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on * TBI being enabled at EL1. */ #define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 +#else +#define TCR_MTE_FLAGS 0 #endif /* * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and - * changed during __cpu_setup to Normal Tagged if the system supports MTE. + * changed during mte_cpu_setup to Normal Tagged if the system supports MTE. */ #define MAIR_EL1_SET \ (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ @@ -185,7 +188,7 @@ SYM_FUNC_END(cpu_do_resume) * This is the low-level counterpart to cpu_replace_ttbr1, and should not be * called by anything else. It can only be executed from a TTBR0 mapping. */ -SYM_FUNC_START(idmap_cpu_replace_ttbr1) +SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1) save_and_disable_daif flags=x2 __idmap_cpu_set_reserved_ttbr1 x1, x3 @@ -253,7 +256,7 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1) SYM_DATA(__idmap_kpti_flag, .long 1) .popsection -SYM_FUNC_START(idmap_kpti_install_ng_mappings) +SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cpu .req w0 temp_pte .req x0 num_cpus .req w1 @@ -426,46 +429,8 @@ SYM_FUNC_START(__cpu_setup) mov_q mair, MAIR_EL1_SET mov_q tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ - TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS - -#ifdef CONFIG_ARM64_MTE - /* - * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported - * (ID_AA64PFR1_EL1[11:8] > 1). - */ - mrs x10, ID_AA64PFR1_EL1 - ubfx x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4 - cmp x10, #ID_AA64PFR1_MTE - b.lt 1f - - /* Normal Tagged memory type at the corresponding MAIR index */ - mov x10, #MAIR_ATTR_NORMAL_TAGGED - bfi mair, x10, #(8 * MT_NORMAL_TAGGED), #8 + TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS - mov x10, #KERNEL_GCR_EL1 - msr_s SYS_GCR_EL1, x10 - - /* - * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then - * RGSR_EL1.SEED must be non-zero for IRG to produce - * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we - * must initialize it. - */ - mrs x10, CNTVCT_EL0 - ands x10, x10, #SYS_RGSR_EL1_SEED_MASK - csinc x10, x10, xzr, ne - lsl x10, x10, #SYS_RGSR_EL1_SEED_SHIFT - msr_s SYS_RGSR_EL1, x10 - - /* clear any pending tag check faults in TFSR*_EL1 */ - msr_s SYS_TFSR_EL1, xzr - msr_s SYS_TFSRE0_EL1, xzr - - /* set the TCR_EL1 bits */ - mov_q x10, TCR_MTE_FLAGS - orr tcr, tcr, x10 -1: -#endif tcr_clear_errata_bits tcr, x9, x5 #ifdef CONFIG_ARM64_VA_BITS_52 diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 389623ae5a91c375e1d78ae84552781180a54d33..30f76178608b360d5bb4f7b18310949d4aebcee4 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1970,7 +1970,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, u32 flags, struct bpf_tramp_links *tlinks, void *orig_call) { - int ret; + int i, ret; int nargs = m->nr_args; int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE; struct jit_ctx ctx = { @@ -1982,6 +1982,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, if (nargs > 8) return -ENOTSUPP; + /* don't support struct argument */ + for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { + if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) + return -ENOTSUPP; + } + ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags); if (ret < 0) return ret; diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 779653771507a03c5d83960c8d687d132161da62..f1c0347ec31a85c7c85dcb2d969def4aee154f06 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -2,6 +2,8 @@ # # Internal CPU capabilities constants, keep this list sorted +ALWAYS_BOOT +ALWAYS_SYSTEM BTI # Unreliable: use system_supports_32bit_el0() instead. HAS_32BIT_EL0_DO_NOT_USE @@ -67,6 +69,8 @@ WORKAROUND_1902691 WORKAROUND_2038923 WORKAROUND_2064142 WORKAROUND_2077057 +WORKAROUND_2457168 +WORKAROUND_2658417 WORKAROUND_TRBE_OVERWRITE_FILL_MODE WORKAROUND_TSB_FLUSH_FAILURE WORKAROUND_TRBE_WRITE_OUT_OF_RANGE diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 9ae483ec1e56ecdafbf42cbccb1b14da517d3b86..384757a7eda9e3749c00610bde4fe022a7c944ea 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -46,6 +46,127 @@ # feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration # item ACCDATA) though it may be more taseful to do something else. +Sysreg ID_AA64PFR0_EL1 3 0 0 4 0 +Enum 63:60 CSV3 + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 59:56 CSV2 + 0b0000 NI + 0b0001 IMP + 0b0010 CSV2_2 + 0b0011 CSV2_3 +EndEnum +Enum 55:52 RME + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 51:48 DIT + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 47:44 AMU + 0b0000 NI + 0b0001 IMP + 0b0010 V1P1 +EndEnum +Enum 43:40 MPAM + 0b0000 0 + 0b0001 1 +EndEnum +Enum 39:36 SEL2 + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 35:32 SVE + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 31:28 RAS + 0b0000 NI + 0b0001 IMP + 0b0010 V1P1 +EndEnum +Enum 27:24 GIC + 0b0000 NI + 0b0001 IMP + 0b0010 V4P1 +EndEnum +Enum 23:20 AdvSIMD + 0b0000 IMP + 0b0001 FP16 + 0b1111 NI +EndEnum +Enum 19:16 FP + 0b0000 IMP + 0b0001 FP16 + 0b1111 NI +EndEnum +Enum 15:12 EL3 + 0b0000 NI + 0b0001 IMP + 0b0010 AARCH32 +EndEnum +Enum 11:8 EL2 + 0b0000 NI + 0b0001 IMP + 0b0010 AARCH32 +EndEnum +Enum 7:4 EL1 + 0b0001 IMP + 0b0010 AARCH32 +EndEnum +Enum 3:0 EL0 + 0b0001 IMP + 0b0010 AARCH32 +EndEnum +EndSysreg + +Sysreg ID_AA64PFR1_EL1 3 0 0 4 1 +Res0 63:40 +Enum 39:36 NMI + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 35:32 CSV2_frac + 0b0000 NI + 0b0001 CSV2_1p1 + 0b0010 CSV2_1p2 +EndEnum +Enum 31:28 RNDR_trap + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 27:24 SME + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 23:20 +Enum 19:16 MPAM_frac + 0b0000 MINOR_0 + 0b0001 MINOR_1 +EndEnum +Enum 15:12 RAS_frac + 0b0000 NI + 0b0001 RASv1p1 +EndEnum +Enum 11:8 MTE + 0b0000 NI + 0b0001 IMP + 0b0010 MTE2 + 0b0011 MTE3 +EndEnum +Enum 7:4 SSBS + 0b0000 NI + 0b0001 IMP + 0b0010 SSBS2 +EndEnum +Enum 3:0 BT + 0b0000 NI + 0b0001 IMP +EndEnum +EndSysreg + Sysreg ID_AA64ZFR0_EL1 3 0 0 4 4 Res0 63:60 Enum 59:56 F64MM @@ -98,7 +219,9 @@ Enum 63 FA64 0b1 IMP EndEnum Res0 62:60 -Field 59:56 SMEver +Enum 59:56 SMEver + 0b0000 IMP +EndEnum Enum 55:52 I16I64 0b0000 NI 0b1111 IMP @@ -129,6 +252,89 @@ EndEnum Res0 31:0 EndSysreg +Sysreg ID_AA64DFR0_EL1 3 0 0 5 0 +Enum 63:60 HPMN0 + 0b0000 UNPREDICTABLE + 0b0001 DEF +EndEnum +Res0 59:56 +Enum 55:52 BRBE + 0b0000 NI + 0b0001 IMP + 0b0010 BRBE_V1P1 +EndEnum +Enum 51:48 MTPMU + 0b0000 NI_IMPDEF + 0b0001 IMP + 0b1111 NI +EndEnum +Enum 47:44 TraceBuffer + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 43:40 TraceFilt + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 39:36 DoubleLock + 0b0000 IMP + 0b1111 NI +EndEnum +Enum 35:32 PMSVer + 0b0000 NI + 0b0001 IMP + 0b0010 V1P1 + 0b0011 V1P2 + 0b0100 V1P3 +EndEnum +Field 31:28 CTX_CMPs +Res0 27:24 +Field 23:20 WRPs +Res0 19:16 +Field 15:12 BRPs +Enum 11:8 PMUVer + 0b0000 NI + 0b0001 IMP + 0b0100 V3P1 + 0b0101 V3P4 + 0b0110 V3P5 + 0b0111 V3P7 + 0b1000 V3P8 + 0b1111 IMP_DEF +EndEnum +Enum 7:4 TraceVer + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 3:0 DebugVer + 0b0110 IMP + 0b0111 VHE + 0b1000 V8P2 + 0b1001 V8P4 + 0b1010 V8P8 +EndEnum +EndSysreg + +Sysreg ID_AA64DFR1_EL1 3 0 0 5 1 +Res0 63:0 +EndSysreg + +Sysreg ID_AA64AFR0_EL1 3 0 0 5 4 +Res0 63:32 +Field 31:28 IMPDEF7 +Field 27:24 IMPDEF6 +Field 23:20 IMPDEF5 +Field 19:16 IMPDEF4 +Field 15:12 IMPDEF3 +Field 11:8 IMPDEF2 +Field 7:4 IMPDEF1 +Field 3:0 IMPDEF0 +EndSysreg + +Sysreg ID_AA64AFR1_EL1 3 0 0 5 5 +Res0 63:0 +EndSysreg + Sysreg ID_AA64ISAR0_EL1 3 0 0 6 0 Enum 63:60 RNDR 0b0000 NI @@ -313,9 +519,220 @@ Enum 3:0 WFxT EndEnum EndSysreg +Sysreg ID_AA64MMFR0_EL1 3 0 0 7 0 +Enum 63:60 ECV + 0b0000 NI + 0b0001 IMP + 0b0010 CNTPOFF +EndEnum +Enum 59:56 FGT + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 55:48 +Enum 47:44 EXS + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 43:40 TGRAN4_2 + 0b0000 TGRAN4 + 0b0001 NI + 0b0010 IMP + 0b0011 52_BIT +EndEnum +Enum 39:36 TGRAN64_2 + 0b0000 TGRAN64 + 0b0001 NI + 0b0010 IMP +EndEnum +Enum 35:32 TGRAN16_2 + 0b0000 TGRAN16 + 0b0001 NI + 0b0010 IMP + 0b0011 52_BIT +EndEnum +Enum 31:28 TGRAN4 + 0b0000 IMP + 0b0001 52_BIT + 0b1111 NI +EndEnum +Enum 27:24 TGRAN64 + 0b0000 IMP + 0b1111 NI +EndEnum +Enum 23:20 TGRAN16 + 0b0000 NI + 0b0001 IMP + 0b0010 52_BIT +EndEnum +Enum 19:16 BIGENDEL0 + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 15:12 SNSMEM + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 11:8 BIGEND + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 7:4 ASIDBITS + 0b0000 8 + 0b0010 16 +EndEnum +Enum 3:0 PARANGE + 0b0000 32 + 0b0001 36 + 0b0010 40 + 0b0011 42 + 0b0100 44 + 0b0101 48 + 0b0110 52 +EndEnum +EndSysreg + +Sysreg ID_AA64MMFR1_EL1 3 0 0 7 1 +Enum 63:60 ECBHB + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 59:56 CMOW + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 55:52 TIDCP1 + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 51:48 nTLBPA + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 47:44 AFP + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 43:40 HCX + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 39:36 ETS + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 35:32 TWED + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 31:28 XNX + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 27:24 SpecSEI + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 23:20 PAN + 0b0000 NI + 0b0001 IMP + 0b0010 PAN2 + 0b0011 PAN3 +EndEnum +Enum 19:16 LO + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 15:12 HPDS + 0b0000 NI + 0b0001 IMP + 0b0010 HPDS2 +EndEnum +Enum 11:8 VH + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 7:4 VMIDBits + 0b0000 8 + 0b0010 16 +EndEnum +Enum 3:0 HAFDBS + 0b0000 NI + 0b0001 AF + 0b0010 DBM +EndEnum +EndSysreg + +Sysreg ID_AA64MMFR2_EL1 3 0 0 7 2 +Enum 63:60 E0PD + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 59:56 EVT + 0b0000 NI + 0b0001 IMP + 0b0010 TTLBxS +EndEnum +Enum 55:52 BBM + 0b0000 0 + 0b0001 1 + 0b0010 2 +EndEnum +Enum 51:48 TTL + 0b0000 NI + 0b0001 IMP +EndEnum +Res0 47:44 +Enum 43:40 FWB + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 39:36 IDS + 0b0000 0x0 + 0b0001 0x18 +EndEnum +Enum 35:32 AT + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 31:28 ST + 0b0000 39 + 0b0001 48_47 +EndEnum +Enum 27:24 NV + 0b0000 NI + 0b0001 IMP + 0b0010 NV2 +EndEnum +Enum 23:20 CCIDX + 0b0000 32 + 0b0001 64 +EndEnum +Enum 19:16 VARange + 0b0000 48 + 0b0001 52 +EndEnum +Enum 15:12 IESB + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 11:8 LSM + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 7:4 UAO + 0b0000 NI + 0b0001 IMP +EndEnum +Enum 3:0 CnP + 0b0000 NI + 0b0001 IMP +EndEnum +EndSysreg + Sysreg SCTLR_EL1 3 0 1 0 0 Field 63 TIDCP -Field 62 SPINMASK +Field 62 SPINTMASK Field 61 NMI Field 60 EnTP2 Res0 59:58 @@ -427,6 +844,12 @@ Sysreg SMCR_EL1 3 0 1 2 6 Fields SMCR_ELx EndSysreg +Sysreg ALLINT 3 0 4 3 0 +Res0 63:14 +Field 13 ALLINT +Res0 12:0 +EndSysreg + Sysreg FAR_EL1 3 0 6 0 0 Field 63:0 ADDR EndSysreg @@ -440,6 +863,14 @@ Sysreg CONTEXTIDR_EL1 3 0 13 0 1 Fields CONTEXTIDR_ELx EndSysreg +Sysreg TPIDR_EL1 3 0 13 0 4 +Field 63:0 ThreadID +EndSysreg + +Sysreg SCXTNUM_EL1 3 0 13 0 7 +Field 63:0 SoftwareContextNumber +EndSysreg + Sysreg CLIDR_EL1 3 1 0 0 1 Res0 63:47 Field 46:33 Ttypen @@ -514,6 +945,22 @@ Sysreg ZCR_EL2 3 4 1 2 0 Fields ZCR_ELx EndSysreg +Sysreg HCRX_EL2 3 4 1 2 2 +Res0 63:12 +Field 11 MSCEn +Field 10 MCE2 +Field 9 CMOW +Field 8 VFNMI +Field 7 VINMI +Field 6 TALLINT +Field 5 SMPME +Field 4 FGTnXS +Field 3 FnXS +Field 2 EnASR +Field 1 EnALS +Field 0 EnAS0 +EndSysreg + Sysreg SMPRIMAP_EL2 3 4 1 2 5 Field 63:60 P15 Field 59:56 P14 diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 3cbc2dc62bafc2fa7805e79fc2316dee50be05ae..adee6ab36862e6a1a2a92ff68bfd93f57d0fd14c 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -332,7 +332,7 @@ config HIGHMEM select KMAP_LOCAL default y -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" default "11" diff --git a/arch/csky/Makefile b/arch/csky/Makefile index 4e1d619fd5c63d4e7f0342e3a9550d1bfb14da73..0e4237e55758b38aa2d9518ef26669c782571fc3 100644 --- a/arch/csky/Makefile +++ b/arch/csky/Makefile @@ -59,8 +59,6 @@ LDFLAGS += -EL KBUILD_AFLAGS += $(KBUILD_CFLAGS) -head-y := arch/csky/kernel/head.o - core-y += arch/csky/$(CSKYABI)/ libs-y += arch/csky/lib/ \ diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 9638206bc44f74cf62a40d56998493c709c6451f..63ad71fab30d7bbe5af9a42b7581e3c486554a41 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -69,11 +69,6 @@ do { \ /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - /* Prepare to copy thread state - unlazy all lazy status */ #define prepare_to_copy(tsk) do { } while (0) diff --git a/arch/csky/kernel/Makefile b/arch/csky/kernel/Makefile index 6f14c924b20d261f92280f9cc40c9da9359db034..8a868316b912bfbb387b7ed6b0a94c1e15367410 100644 --- a/arch/csky/kernel/Makefile +++ b/arch/csky/kernel/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o vdso/ +obj-y += head.o entry.o atomic.o signal.o traps.o irq.o time.o vdso.o vdso/ obj-y += power.o syscall.o syscall_table.o setup.o io.o obj-y += process.o cpu-probe.o ptrace.o stacktrace.o obj-y += probes/ diff --git a/arch/hexagon/Makefile b/arch/hexagon/Makefile index 44312bc147d83854ec85c649fe6b57734df0b6be..92d005958dfb232d48a4ca843b46262a84a08eb4 100644 --- a/arch/hexagon/Makefile +++ b/arch/hexagon/Makefile @@ -32,5 +32,3 @@ KBUILD_LDFLAGS += $(ldflags-y) TIR_NAME := r19 KBUILD_CFLAGS += -ffixed-$(TIR_NAME) -DTHREADINFO_REG=$(TIR_NAME) -D__linux__ KBUILD_AFLAGS += -DTHREADINFO_REG=$(TIR_NAME) - -head-y := arch/hexagon/kernel/head.o diff --git a/arch/hexagon/include/asm/bitops.h b/arch/hexagon/include/asm/bitops.h index da500471ac73ccb0a6569992c4f189d1bce5f2f4..160d8f37fa1a34c237282940ae6cf2947893f382 100644 --- a/arch/hexagon/include/asm/bitops.h +++ b/arch/hexagon/include/asm/bitops.h @@ -179,6 +179,21 @@ arch_test_bit(unsigned long nr, const volatile unsigned long *addr) return retval; } +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + int retval; + + asm volatile( + "{P0 = tstbit(%1,%2); if (P0.new) %0 = #1; if (!P0.new) %0 = #0;}\n" + : "=&r" (retval) + : "r" (addr[BIT_WORD(nr)]), "r" (nr % BITS_PER_LONG) + : "p0", "memory" + ); + + return retval; +} + /* * ffz - find first zero in word. * @word: The word to search diff --git a/arch/hexagon/include/asm/io.h b/arch/hexagon/include/asm/io.h index c33241425a5c2ecfc6a0ed95abbbbd008fd375bc..46a099de85b7f342410ce5ab140a970a20e51a82 100644 --- a/arch/hexagon/include/asm/io.h +++ b/arch/hexagon/include/asm/io.h @@ -308,6 +308,31 @@ static inline void outsl(unsigned long port, const void *buffer, int count) } } +/* + * These defines are necessary to use the generic io.h for filling in + * the missing parts of the API contract. This is because the platform + * uses (inline) functions rather than defines and the generic helper + * fills in the undefined. + */ +#define virt_to_phys virt_to_phys +#define phys_to_virt phys_to_virt +#define memset_io memset_io +#define memcpy_fromio memcpy_fromio +#define memcpy_toio memcpy_toio +#define readb readb +#define readw readw +#define readl readl +#define writeb writeb +#define writew writew +#define writel writel +#define insb insb +#define insw insw +#define insl insl +#define outsb outsb +#define outsw outsw +#define outsl outsl +#include + #endif /* __KERNEL__ */ #endif diff --git a/arch/hexagon/include/asm/processor.h b/arch/hexagon/include/asm/processor.h index 615f7e49968e619626f423b10af853f2a80ddbf1..0cd39c2cdf8f7a4073a3cca54a4847935414724b 100644 --- a/arch/hexagon/include/asm/processor.h +++ b/arch/hexagon/include/asm/processor.h @@ -60,10 +60,6 @@ struct thread_struct { #define KSTK_EIP(tsk) (pt_elr(task_pt_regs(tsk))) #define KSTK_ESP(tsk) (pt_psp(task_pt_regs(tsk))) -/* Free all resources held by a thread; defined in process.c */ -extern void release_thread(struct task_struct *dead_task); - -/* Get wait channel for task P. */ extern unsigned long __get_wchan(struct task_struct *p); /* The following stuff is pretty HEXAGON specific. */ diff --git a/arch/hexagon/kernel/Makefile b/arch/hexagon/kernel/Makefile index fae3dce32fde81fc7767fe25a78577726fbe1156..e73cb321630ec218b2b25c669c97b2144d3ae3ad 100644 --- a/arch/hexagon/kernel/Makefile +++ b/arch/hexagon/kernel/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds +obj-y += head.o obj-$(CONFIG_SMP) += smp.o obj-y += setup.o irq_cpu.o traps.o syscalltab.o signal.o time.o diff --git a/arch/hexagon/kernel/process.c b/arch/hexagon/kernel/process.c index f0552f98a7bae116a1d5fb133adffc78695c8b62..e15eeaebd78530e90eace2211582a1bb2bf1f32b 100644 --- a/arch/hexagon/kernel/process.c +++ b/arch/hexagon/kernel/process.c @@ -112,13 +112,6 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) return 0; } -/* - * Release any architecture-specific resources locked by thread - */ -void release_thread(struct task_struct *dead_task) -{ -} - /* * Some archs flush debug and FPU info here */ diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 26ac8ea15a9e3d3bf91832a86aa44964725b42fe..c6e06cdc738f0bb0696c8451a64ec8c64f7835d2 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -200,7 +200,7 @@ config IA64_CYCLONE Say Y here to enable support for IBM EXA Cyclone time source. If you're unsure, answer N. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "MAX_ORDER (11 - 17)" if !HUGETLB_PAGE range 11 17 if !HUGETLB_PAGE default "17" if HUGETLB_PAGE diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile index e55c2f138656abb82d8808ec2047d73568179d76..56c4bb276b6eda0e19c91fca6b2fd216be0e7467 100644 --- a/arch/ia64/Makefile +++ b/arch/ia64/Makefile @@ -44,7 +44,6 @@ quiet_cmd_objcopy = OBJCOPY $@ cmd_objcopy = $(OBJCOPY) $(OBJCOPYFLAGS) $(OBJCOPYFLAGS_$(@F)) $< $@ KBUILD_CFLAGS += $(cflags-y) -head-y := arch/ia64/kernel/head.o libs-y += arch/ia64/lib/ diff --git a/arch/ia64/configs/bigsur_defconfig b/arch/ia64/configs/bigsur_defconfig index a3724882295cd9b1e263ec83cdd9935728280c5b..3e1337aceb37154d361e03746a2baec1d25a9255 100644 --- a/arch/ia64/configs/bigsur_defconfig +++ b/arch/ia64/configs/bigsur_defconfig @@ -20,7 +20,6 @@ CONFIG_UNIX=y CONFIG_INET=y # CONFIG_IPV6 is not set CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=m CONFIG_ATA=m @@ -91,7 +90,6 @@ CONFIG_NFS_V4=m CONFIG_NFSD=m CONFIG_NFSD_V4=y CONFIG_CIFS=m -CONFIG_CIFS_STATS=y CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y CONFIG_NLS_CODEPAGE_437=y diff --git a/arch/ia64/configs/generic_defconfig b/arch/ia64/configs/generic_defconfig index a3dff482a3d70bbb53cfc9ecea76b270412398fc..f8033bacea89e4912f18904bded9e0efb131159c 100644 --- a/arch/ia64/configs/generic_defconfig +++ b/arch/ia64/configs/generic_defconfig @@ -39,7 +39,6 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_CONNECTOR=y # CONFIG_PNP_DEBUG_MESSAGES is not set CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_SGI_XP=m @@ -91,7 +90,6 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y # CONFIG_HW_RANDOM is not set CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y -CONFIG_RAW_DRIVER=m CONFIG_HPET=y CONFIG_AGP=m CONFIG_AGP_I460=m diff --git a/arch/ia64/configs/gensparse_defconfig b/arch/ia64/configs/gensparse_defconfig index 4cd46105b0201f4878f41803d828e7c48b2f17c8..ffebe6c503f5119c3777828229a4d11e2f7ac489 100644 --- a/arch/ia64/configs/gensparse_defconfig +++ b/arch/ia64/configs/gensparse_defconfig @@ -31,11 +31,9 @@ CONFIG_IP_MULTICAST=y CONFIG_SYN_COOKIES=y # CONFIG_IPV6 is not set CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_ATA=y -CONFIG_BLK_DEV_IDECD=y CONFIG_ATA_GENERIC=y CONFIG_PATA_CMD64X=y CONFIG_ATA_PIIX=y @@ -81,7 +79,6 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y # CONFIG_HW_RANDOM is not set CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y -CONFIG_RAW_DRIVER=m CONFIG_HPET=y CONFIG_AGP=m CONFIG_AGP_I460=m diff --git a/arch/ia64/configs/tiger_defconfig b/arch/ia64/configs/tiger_defconfig index a2045d73adfad5d132fb259d7d9e610d6cbb0d22..45f5d6e2da0afc6a4656daaff8fce668beb34540 100644 --- a/arch/ia64/configs/tiger_defconfig +++ b/arch/ia64/configs/tiger_defconfig @@ -36,7 +36,6 @@ CONFIG_IP_MULTICAST=y CONFIG_SYN_COOKIES=y # CONFIG_IPV6 is not set CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_ATA=y @@ -85,7 +84,6 @@ CONFIG_SERIAL_8250_SHARE_IRQ=y # CONFIG_HW_RANDOM is not set CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y -CONFIG_RAW_DRIVER=m CONFIG_HPET=y CONFIG_AGP=m CONFIG_AGP_I460=m diff --git a/arch/ia64/configs/zx1_defconfig b/arch/ia64/configs/zx1_defconfig index 99f8b2a0332bc1541bb5dafc60fb75639e068d23..ed104550d0d519a26157d900ae8b22b18229d4ea 100644 --- a/arch/ia64/configs/zx1_defconfig +++ b/arch/ia64/configs/zx1_defconfig @@ -30,7 +30,6 @@ CONFIG_PATA_CMD64X=y CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_ST=y -CONFIG_CHR_DEV_OSST=y CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=y CONFIG_SCSI_CONSTANTS=y diff --git a/arch/ia64/include/asm/bitops.h b/arch/ia64/include/asm/bitops.h index 9f62af7fd7c42c9d0f621a06058274863786a0a3..1accb7842f58806446720a8992ce0d1d2f71c99c 100644 --- a/arch/ia64/include/asm/bitops.h +++ b/arch/ia64/include/asm/bitops.h @@ -331,11 +331,8 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) return (old & bit) != 0; } -static __always_inline bool -arch_test_bit(unsigned long nr, const volatile unsigned long *addr) -{ - return 1 & (((const volatile __u32 *) addr)[nr >> 5] >> (nr & 31)); -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire /** * ffz - find the first zero bit in a long word diff --git a/arch/ia64/include/asm/processor.h b/arch/ia64/include/asm/processor.h index 757c2f6d8d4b8128eec011f7f6ba20dcc91f2ac7..d1978e0040548fbacd2f63bb7aca1bbf08068d66 100644 --- a/arch/ia64/include/asm/processor.h +++ b/arch/ia64/include/asm/processor.h @@ -318,13 +318,6 @@ struct thread_struct { struct mm_struct; struct task_struct; -/* - * Free all resources held by a thread. This is called after the - * parent of DEAD_TASK has collected the exit status of the task via - * wait(). - */ -#define release_thread(dead_task) - /* Get wait channel for task P. */ extern unsigned long __get_wchan (struct task_struct *p); diff --git a/arch/ia64/include/asm/sparsemem.h b/arch/ia64/include/asm/sparsemem.h index 42ed5248fae9876875a71f5b66c40a0a884c8749..84e8ce387b698629c1172bf5882a3f7acd4db6c8 100644 --- a/arch/ia64/include/asm/sparsemem.h +++ b/arch/ia64/include/asm/sparsemem.h @@ -11,10 +11,10 @@ #define SECTION_SIZE_BITS (30) #define MAX_PHYSMEM_BITS (50) -#ifdef CONFIG_FORCE_MAX_ZONEORDER -#if ((CONFIG_FORCE_MAX_ZONEORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS) +#ifdef CONFIG_ARCH_FORCE_MAX_ORDER +#if ((CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS) #undef SECTION_SIZE_BITS -#define SECTION_SIZE_BITS (CONFIG_FORCE_MAX_ZONEORDER - 1 + PAGE_SHIFT) +#define SECTION_SIZE_BITS (CONFIG_ARCH_FORCE_MAX_ORDER - 1 + PAGE_SHIFT) #endif #endif diff --git a/arch/ia64/include/asm/termios.h b/arch/ia64/include/asm/termios.h deleted file mode 100644 index 589c026444cc5fb48ea58fb07ce3074a2ac873d8..0000000000000000000000000000000000000000 --- a/arch/ia64/include/asm/termios.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Modified 1999 - * David Mosberger-Tang , Hewlett-Packard Co - * - * 99/01/28 Added N_IRDA and N_SMSBLOCK - */ -#ifndef _ASM_IA64_TERMIOS_H -#define _ASM_IA64_TERMIOS_H - -#include - - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D vtime=\0 vmin=\1 sxtc=\0 - start=^Q stop=^S susp=^Z eol=\0 - reprint=^R discard=^U werase=^W lnext=^V - eol2=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -#define SET_LOW_TERMIOS_BITS(termios, termio, x) { \ - unsigned short __tmp; \ - get_user(__tmp,&(termio)->x); \ - *(unsigned short *) &(termios)->x = __tmp; \ -} - -#define user_termio_to_kernel_termios(termios, termio) \ -({ \ - SET_LOW_TERMIOS_BITS(termios, termio, c_iflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_oflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_cflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_lflag); \ - copy_from_user((termios)->c_cc, (termio)->c_cc, NCC); \ -}) - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -#define kernel_termios_to_user_termio(termio, termios) \ -({ \ - put_user((termios)->c_iflag, &(termio)->c_iflag); \ - put_user((termios)->c_oflag, &(termio)->c_oflag); \ - put_user((termios)->c_cflag, &(termio)->c_cflag); \ - put_user((termios)->c_lflag, &(termio)->c_lflag); \ - put_user((termios)->c_line, &(termio)->c_line); \ - copy_to_user((termio)->c_cc, (termios)->c_cc, NCC); \ -}) - -#define user_termios_to_kernel_termios(k, u) copy_from_user(k, u, sizeof(struct termios2)) -#define kernel_termios_to_user_termios(u, k) copy_to_user(u, k, sizeof(struct termios2)) -#define user_termios_to_kernel_termios_1(k, u) copy_from_user(k, u, sizeof(struct termios)) -#define kernel_termios_to_user_termios_1(u, k) copy_to_user(u, k, sizeof(struct termios)) - -#endif /* _ASM_IA64_TERMIOS_H */ diff --git a/arch/ia64/include/uapi/asm/termios.h b/arch/ia64/include/uapi/asm/termios.h deleted file mode 100644 index 199742d08f2c527ee1c5df1020872254954055a5..0000000000000000000000000000000000000000 --- a/arch/ia64/include/uapi/asm/termios.h +++ /dev/null @@ -1,51 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * Modified 1999 - * David Mosberger-Tang , Hewlett-Packard Co - * - * 99/01/28 Added N_IRDA and N_SMSBLOCK - */ -#ifndef _UAPI_ASM_IA64_TERMIOS_H -#define _UAPI_ASM_IA64_TERMIOS_H - - -#include -#include - -struct winsize { - unsigned short ws_row; - unsigned short ws_col; - unsigned short ws_xpixel; - unsigned short ws_ypixel; -}; - -#define NCC 8 -struct termio { - unsigned short c_iflag; /* input mode flags */ - unsigned short c_oflag; /* output mode flags */ - unsigned short c_cflag; /* control mode flags */ - unsigned short c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - -/* modem lines */ -#define TIOCM_LE 0x001 -#define TIOCM_DTR 0x002 -#define TIOCM_RTS 0x004 -#define TIOCM_ST 0x008 -#define TIOCM_SR 0x010 -#define TIOCM_CTS 0x020 -#define TIOCM_CAR 0x040 -#define TIOCM_RNG 0x080 -#define TIOCM_DSR 0x100 -#define TIOCM_CD TIOCM_CAR -#define TIOCM_RI TIOCM_RNG -#define TIOCM_OUT1 0x2000 -#define TIOCM_OUT2 0x4000 -#define TIOCM_LOOP 0x8000 - -/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ - - -#endif /* _UAPI_ASM_IA64_TERMIOS_H */ diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index 08d4a2ba06520fd5ba6a7a23e751d4985004bdaa..ae9ff07de4abe419f3fdf8f740d148de1436332b 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -7,9 +7,9 @@ ifdef CONFIG_DYNAMIC_FTRACE CFLAGS_REMOVE_ftrace.o = -pg endif -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y := entry.o efi.o efi_stub.o gate-data.o fsys.o irq.o irq_ia64.o \ +obj-y := head.o entry.o efi.o efi_stub.o gate-data.o fsys.o irq.o irq_ia64.o \ irq_lsapic.o ivt.o pal.o patch.o process.o ptrace.o sal.o \ salinfo.o setup.o signal.o sys_ia64.o time.o traps.o unaligned.o \ unwind.o mca.o mca_asm.o topology.o dma-mapping.o iosapic.o acpi.o \ @@ -34,10 +34,7 @@ mca_recovery-y += mca_drv.o mca_drv_asm.o obj-$(CONFIG_IA64_MC_ERR_INJECT)+= err_inject.o obj-$(CONFIG_STACKTRACE) += stacktrace.o -obj-$(CONFIG_IA64_ESI) += esi.o -ifneq ($(CONFIG_IA64_ESI),) -obj-y += esi_stub.o # must be in kernel proper -endif +obj-$(CONFIG_IA64_ESI) += esi.o esi_stub.o # must be in kernel proper obj-$(CONFIG_INTEL_IOMMU) += pci-dma.o obj-$(CONFIG_ELF_CORE) += elfcore.o diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index c62a66710ad6d42476e3b7eaab83988c4f1c486c..92ede80d17fea68d7aa69a636a37f8db159631dd 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -1793,7 +1793,7 @@ format_mca_init_stack(void *mca_data, unsigned long offset, p->parent = p->real_parent = p->group_leader = p; INIT_LIST_HEAD(&p->children); INIT_LIST_HEAD(&p->sibling); - strncpy(p->comm, type, sizeof(p->comm)-1); + strscpy(p->comm, type, sizeof(p->comm)-1); } /* Caller prevents this from being called after init */ diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index fd6301eafa9d58a2d01ce240dda907a2cb601268..c0572804427275e7837c8b6524411f16e453fafc 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -552,7 +552,7 @@ setup_arch (char **cmdline_p) ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist); *cmdline_p = __va(ia64_boot_param->command_line); - strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); + strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); efi_init(); io_port_init(); diff --git a/arch/ia64/kernel/sys_ia64.c b/arch/ia64/kernel/sys_ia64.c index e14db25146c222cdc8aef027f691fef6a73ac9cf..215bf3f8cb2043fefb26db7f0bc91c69b0440bb1 100644 --- a/arch/ia64/kernel/sys_ia64.c +++ b/arch/ia64/kernel/sys_ia64.c @@ -166,3 +166,29 @@ ia64_mremap (unsigned long addr, unsigned long old_len, unsigned long new_len, u force_successful_syscall_return(); return addr; } + +asmlinkage long +ia64_clock_getres(const clockid_t which_clock, struct __kernel_timespec __user *tp) +{ + /* + * ia64's clock_gettime() syscall is implemented as a vdso call + * fsys_clock_gettime(). Currently it handles only + * CLOCK_REALTIME and CLOCK_MONOTONIC. Both are based on + * 'ar.itc' counter which gets incremented at a constant + * frequency. It's usually 400MHz, ~2.5x times slower than CPU + * clock frequency. Which is almost a 1ns hrtimer, but not quite. + * + * Let's special-case these timers to report correct precision + * based on ITC frequency and not HZ frequency for supported + * clocks. + */ + switch (which_clock) { + case CLOCK_REALTIME: + case CLOCK_MONOTONIC: + s64 tick_ns = DIV_ROUND_UP(NSEC_PER_SEC, local_cpu_data->itc_freq); + struct timespec64 rtn_tp = ns_to_timespec64(tick_ns); + return put_timespec64(&rtn_tp, tp); + } + + return sys_clock_getres(which_clock, tp); +} diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index 78b1d03e86e1dc1e7b71dbaaf2649d5e7e6c48cd..72c929d9902b99601ed98f25f89ed88cc2d9935e 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -240,7 +240,7 @@ 228 common timer_delete sys_timer_delete 229 common clock_settime sys_clock_settime 230 common clock_gettime sys_clock_gettime -231 common clock_getres sys_clock_getres +231 common clock_getres ia64_clock_getres 232 common clock_nanosleep sys_clock_nanosleep 233 common fstatfs64 sys_fstatfs64 234 common statfs64 sys_statfs64 diff --git a/arch/ia64/mm/numa.c b/arch/ia64/mm/numa.c index d6579ec3ea324f9370c5b19c75c0429cff796f9d..4c7b1f50e3b7d50c29ee814890c8a4542f287556 100644 --- a/arch/ia64/mm/numa.c +++ b/arch/ia64/mm/numa.c @@ -75,5 +75,6 @@ int memory_add_physaddr_to_nid(u64 addr) return 0; return nid; } +EXPORT_SYMBOL(memory_add_physaddr_to_nid); #endif #endif diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild index ab5373d0a24ffa024386c451e553c105bf11da55..b01f5cdb27e03d778dfa400e370037c39cd7abed 100644 --- a/arch/loongarch/Kbuild +++ b/arch/loongarch/Kbuild @@ -1,5 +1,6 @@ obj-y += kernel/ obj-y += mm/ +obj-y += net/ obj-y += vdso/ # for cleaning diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 4abc9a28aba4eee5959f7a1e17f4814a945165c0..903096bd87f8829273a0f7bba4134d3f4c468825 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -10,7 +10,6 @@ config LOONGARCH select ARCH_ENABLE_MEMORY_HOTPLUG select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI - select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_INLINE_READ_LOCK if !PREEMPTION @@ -39,6 +38,7 @@ config LOONGARCH select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION + select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_SPARSEMEM_ENABLE @@ -50,7 +50,9 @@ config LOONGARCH select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT + select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANTS_NO_INSTR select BUILDTIME_TABLE_SORT select COMMON_CLK @@ -60,6 +62,7 @@ config LOONGARCH select GENERIC_CPU_AUTOPROBE select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY + select GENERIC_IOREMAP if !ARCH_IOREMAP select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW @@ -68,6 +71,7 @@ config LOONGARCH select GENERIC_LIB_CMPDI2 select GENERIC_LIB_LSHRDI3 select GENERIC_LIB_UCMPDI2 + select GENERIC_LIB_DEVMEM_IS_ALLOWED select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD @@ -82,6 +86,7 @@ config LOONGARCH select HAVE_CONTEXT_TRACKING_USER select HAVE_DEBUG_STACKOVERFLOW select HAVE_DMA_CONTIGUOUS + select HAVE_EBPF_JIT select HAVE_EXIT_THREAD select HAVE_FAST_GUP select HAVE_GENERIC_VDSO @@ -92,6 +97,8 @@ config LOONGARCH select HAVE_NMI select HAVE_PCI select HAVE_PERF_EVENTS + select HAVE_PERF_REGS + select HAVE_PERF_USER_STACK_DUMP select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RSEQ select HAVE_SETUP_PER_CPU_AREA if NUMA @@ -104,13 +111,12 @@ config LOONGARCH select MODULES_USE_ELF_RELA if MODULES select NEED_PER_CPU_EMBED_FIRST_CHUNK select NEED_PER_CPU_PAGE_FIRST_CHUNK - select OF - select OF_EARLY_FLATTREE select PCI select PCI_DOMAINS_GENERIC select PCI_ECAM if ACPI select PCI_LOONGSON select PCI_MSI_ARCH_FALLBACKS + select PCI_QUIRKS select PERF_USE_VMALLOC select RTC_LIB select SMP @@ -136,6 +142,14 @@ config CPU_HAS_PREFETCH bool default y +config GENERIC_BUG + def_bool y + depends on BUG + +config GENERIC_BUG_RELATIVE_POINTERS + def_bool y + depends on GENERIC_BUG + config GENERIC_CALIBRATE_DELAY def_bool y @@ -157,7 +171,7 @@ config STACKTRACE_SUPPORT bool default y -# MACH_LOONGSON32 and MACH_LOONGSON64 are delibrately carried over from the +# MACH_LOONGSON32 and MACH_LOONGSON64 are deliberately carried over from the # MIPS Loongson code, to preserve Loongson-specific code paths in drivers that # are shared between architectures, and specifically expecting the symbols. config MACH_LOONGSON32 @@ -166,6 +180,9 @@ config MACH_LOONGSON32 config MACH_LOONGSON64 def_bool 64BIT +config FIX_EARLYCON_MEM + def_bool y + config PAGE_SIZE_4KB bool @@ -194,6 +211,9 @@ config SCHED_OMIT_FRAME_POINTER bool default y +config AS_HAS_EXPLICIT_RELOCS + def_bool $(as-instr,x:pcalau12i \$t0$(comma)%pc_hi20(x)) + menu "Kernel type and options" source "kernel/Kconfig.hz" @@ -311,12 +331,20 @@ config DMI config EFI bool "EFI runtime service support" select UCS2_STRING - select EFI_PARAMS_FROM_FDT select EFI_RUNTIME_WRAPPERS help This enables the kernel to use EFI runtime services that are available (such as the EFI variable services). +config EFI_STUB + bool "EFI boot stub support" + default y + depends on EFI + select EFI_GENERIC_STUB + help + This kernel feature allows the kernel to be loaded directly by + EFI firmware without the use of a bootloader. + config SMP bool "Multi-Processing support" help @@ -369,7 +397,7 @@ config NODES_SHIFT default "6" depends on NUMA -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" range 14 64 if PAGE_SIZE_64KB default "14" if PAGE_SIZE_64KB @@ -391,6 +419,46 @@ config FORCE_MAX_ZONEORDER The page size is not necessarily 4KB. Keep this in mind when choosing a value for this option. +config ARCH_IOREMAP + bool "Enable LoongArch DMW-based ioremap()" + help + We use generic TLB-based ioremap() by default since it has page + protection support. However, you can enable LoongArch DMW-based + ioremap() for better performance. + +config KEXEC + bool "Kexec system call" + select KEXEC_CORE + help + kexec is a system call that implements the ability to shutdown your + current kernel, and to start another kernel. It is like a reboot + but it is independent of the system firmware. And like a reboot + you can start any kernel with it, not just Linux. + + The name comes from the similarity to the exec system call. + +config CRASH_DUMP + bool "Build kdump crash kernel" + help + Generate crash dump after being started by kexec. This should + be normally only set in special crash dump kernels which are + loaded in the main kernel with kexec-tools into a specially + reserved region and then later executed after a crash by + kdump/kexec. + + For more details see Documentation/admin-guide/kdump/kdump.rst + +config PHYSICAL_START + hex "Physical address where the kernel is loaded" + default "0x90000000a0000000" + depends on CRASH_DUMP + help + This gives the XKPRANGE address where the kernel is loaded. + If you plan to use kernel for capturing the crash dump change + this value to start of the reserved region (the "X" value as + specified in the "crashkernel=YM@XM" command line boot parameter + passed to the panic-ed kernel). + config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on PROC_FS diff --git a/arch/loongarch/Makefile b/arch/loongarch/Makefile index ec3de619127655a074ab17a95873e21f640030f5..f4cb54d5afd669b7f177f1f60c8937dc4b05e7eb 100644 --- a/arch/loongarch/Makefile +++ b/arch/loongarch/Makefile @@ -7,7 +7,14 @@ boot := arch/loongarch/boot KBUILD_DEFCONFIG := loongson3_defconfig -KBUILD_IMAGE = $(boot)/vmlinux +image-name-y := vmlinux +image-name-$(CONFIG_EFI_ZBOOT) := vmlinuz + +ifndef CONFIG_EFI_STUB +KBUILD_IMAGE := $(boot)/vmlinux.elf +else +KBUILD_IMAGE := $(boot)/$(image-name-y).efi +endif # # Select the object file format to substitute into the linker script. @@ -36,15 +43,37 @@ endif cflags-y += -G0 -pipe -msoft-float LDFLAGS_vmlinux += -G0 -static -n -nostdlib + +# When the assembler supports explicit relocation hint, we must use it. +# GCC may have -mexplicit-relocs off by default if it was built with an old +# assembler, so we force it via an option. +# +# When the assembler does not supports explicit relocation hint, we can't use +# it. Disable it if the compiler supports it. +# +# If you've seen "unknown reloc hint" message building the kernel and you are +# now wondering why "-mexplicit-relocs" is not wrapped with cc-option: the +# combination of a "new" assembler and "old" compiler is not supported. Either +# upgrade the compiler or downgrade the assembler. +ifdef CONFIG_AS_HAS_EXPLICIT_RELOCS +cflags-y += -mexplicit-relocs +KBUILD_CFLAGS_KERNEL += -mdirect-extern-access +else +cflags-y += $(call cc-option,-mno-explicit-relocs) KBUILD_AFLAGS_KERNEL += -Wa,-mla-global-with-pcrel KBUILD_CFLAGS_KERNEL += -Wa,-mla-global-with-pcrel KBUILD_AFLAGS_MODULE += -Wa,-mla-global-with-abs KBUILD_CFLAGS_MODULE += -fplt -Wa,-mla-global-with-abs,-mla-local-with-abs +endif cflags-y += -ffreestanding cflags-y += $(call cc-option, -mno-check-zero-division) +ifndef CONFIG_PHYSICAL_START load-y = 0x9000000000200000 +else +load-y = $(CONFIG_PHYSICAL_START) +endif bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) drivers-$(CONFIG_PCI) += arch/loongarch/pci/ @@ -72,9 +101,8 @@ CHECKFLAGS += $(shell $(CC) $(KBUILD_CFLAGS) -dM -E -x c /dev/null | \ sed -e "s/^\#define /-D'/" -e "s/ /'='/" -e "s/$$/'/" -e 's/\$$/&&/g') endif -head-y := arch/loongarch/kernel/head.o - libs-y += arch/loongarch/lib/ +libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a ifeq ($(KBUILD_EXTMOD),) prepare: vdso_prepare @@ -86,13 +114,13 @@ PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/loongarch/vdso $@ -all: $(KBUILD_IMAGE) +all: $(notdir $(KBUILD_IMAGE)) -$(KBUILD_IMAGE): vmlinux - $(Q)$(MAKE) $(build)=$(boot) $(bootvars-y) $@ +vmlinux.elf vmlinux.efi vmlinuz.efi: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(bootvars-y) $(boot)/$@ install: - $(Q)install -D -m 755 $(KBUILD_IMAGE) $(INSTALL_PATH)/vmlinux-$(KERNELRELEASE) + $(Q)install -D -m 755 $(KBUILD_IMAGE) $(INSTALL_PATH)/$(image-name-y)-$(KERNELRELEASE) $(Q)install -D -m 644 .config $(INSTALL_PATH)/config-$(KERNELRELEASE) $(Q)install -D -m 644 System.map $(INSTALL_PATH)/System.map-$(KERNELRELEASE) diff --git a/arch/loongarch/boot/.gitignore b/arch/loongarch/boot/.gitignore index 49423ee96ef30390d5ce50642a5495cc491817a0..e5dc594dc4b6a024962068c59164f3bbaff465ce 100644 --- a/arch/loongarch/boot/.gitignore +++ b/arch/loongarch/boot/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only vmlinux* +vmlinuz* diff --git a/arch/loongarch/boot/Makefile b/arch/loongarch/boot/Makefile index 0125b17edc98c837d97854aca474b90d021c92cc..4e1c374c57823ee995201c2b5d21ed70c504c5f1 100644 --- a/arch/loongarch/boot/Makefile +++ b/arch/loongarch/boot/Makefile @@ -8,9 +8,19 @@ drop-sections := .comment .note .options .note.gnu.build-id strip-flags := $(addprefix --remove-section=,$(drop-sections)) -S OBJCOPYFLAGS_vmlinux.efi := -O binary $(strip-flags) -targets := vmlinux quiet_cmd_strip = STRIP $@ cmd_strip = $(STRIP) -s -o $@ $< -$(obj)/vmlinux: vmlinux FORCE +targets := vmlinux.elf +$(obj)/vmlinux.elf: vmlinux FORCE $(call if_changed,strip) + +targets += vmlinux.efi +$(obj)/vmlinux.efi: vmlinux FORCE + $(call if_changed,objcopy) + +EFI_ZBOOT_PAYLOAD := vmlinux.efi +EFI_ZBOOT_BFD_TARGET := elf64-loongarch +EFI_ZBOOT_MACH_TYPE := LOONGARCH64 + +include $(srctree)/drivers/firmware/efi/libstub/Makefile.zboot diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 3712552e18d39dfe3164fc2c90771db26f1cd46d..3540e9c0a631066081726c44c3cbebc22618771e 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -4,6 +4,7 @@ CONFIG_POSIX_MQUEUE=y CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y CONFIG_PREEMPT=y CONFIG_BSD_PROCESS_ACCT=y CONFIG_BSD_PROCESS_ACCT_V3=y @@ -45,6 +46,7 @@ CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y CONFIG_NR_CPUS=64 CONFIG_NUMA=y +CONFIG_KEXEC=y CONFIG_PAGE_SIZE_16KB=y CONFIG_HZ_250=y CONFIG_ACPI=y @@ -55,6 +57,7 @@ CONFIG_ACPI_DOCK=y CONFIG_ACPI_IPMI=m CONFIG_ACPI_PCI_SLOT=y CONFIG_ACPI_HOTPLUG_MEMORY=y +CONFIG_EFI_ZBOOT=y CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER=y CONFIG_EFI_CAPSULE_LOADER=m CONFIG_EFI_TEST=m @@ -65,6 +68,8 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_PARTITION_ADVANCED=y +CONFIG_BSD_DISKLABEL=y +CONFIG_UNIXWARE_DISKLABEL=y CONFIG_IOSCHED_BFQ=y CONFIG_BFQ_GROUP_IOSCHED=y CONFIG_BINFMT_MISC=m @@ -82,8 +87,11 @@ CONFIG_ZSMALLOC=m CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y +CONFIG_TLS=m +CONFIG_TLS_DEVICE=y CONFIG_XFRM_USER=y CONFIG_NET_KEY=y +CONFIG_XDP_SOCKETS=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_IP_ADVANCED_ROUTER=y @@ -95,6 +103,7 @@ CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m CONFIG_IP_MROUTE=y CONFIG_INET_ESP=m CONFIG_INET_UDP_DIAG=y @@ -102,6 +111,7 @@ CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_CONG_BBR=m CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y +CONFIG_INET6_ESP=m CONFIG_IPV6_MROUTE=y CONFIG_NETWORK_PHY_TIMESTAMPING=y CONFIG_NETFILTER=y @@ -112,10 +122,11 @@ CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m CONFIG_NF_CONNTRACK_TFTP=m CONFIG_NF_CT_NETLINK=m CONFIG_NF_TABLES=m -CONFIG_NFT_COUNTER=m CONFIG_NFT_CONNLIMIT=m CONFIG_NFT_LOG=m CONFIG_NFT_LIMIT=m @@ -200,7 +211,6 @@ CONFIG_NF_TABLES_IPV4=y CONFIG_NFT_DUP_IPV4=m CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y -CONFIG_NF_LOG_ARP=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -254,10 +264,14 @@ CONFIG_BPFILTER=y CONFIG_IP_SCTP=m CONFIG_RDS=y CONFIG_L2TP=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m CONFIG_BRIDGE=m CONFIG_VLAN_8021Q=m CONFIG_VLAN_8021Q_GVRP=y CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m CONFIG_NET_SCHED=y CONFIG_NET_SCH_HTB=m CONFIG_NET_SCH_PRIO=m @@ -282,9 +296,33 @@ CONFIG_VSOCKETS=m CONFIG_VIRTIO_VSOCKETS=m CONFIG_NETLINK_DIAG=y CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_STREAM_PARSER=y CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_HS=y CONFIG_BT_HCIBTUSB=m -# CONFIG_BT_HCIBTUSB_BCM is not set +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIBTUSB_MTK=y +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_BCSP=y +CONFIG_BT_HCIUART_ATH3K=y +CONFIG_BT_HCIUART_INTEL=y +CONFIG_BT_HCIUART_AG6XX=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIDTL1=m +CONFIG_BT_HCIBT3C=m +CONFIG_BT_HCIBLUECARD=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_ATH3K=m +CONFIG_BT_VIRTIO=m CONFIG_CFG80211=m CONFIG_CFG80211_WEXT=y CONFIG_MAC80211=m @@ -329,7 +367,6 @@ CONFIG_PARPORT_PC_FIFO=y CONFIG_ZRAM=m CONFIG_ZRAM_DEF_COMP_ZSTD=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=y CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 @@ -486,6 +523,7 @@ CONFIG_PPP_FILTER=y CONFIG_PPP_MPPE=m CONFIG_PPP_MULTILINK=y CONFIG_PPPOE=m +CONFIG_PPTP=m CONFIG_PPPOL2TP=m CONFIG_PPP_ASYNC=m CONFIG_PPP_SYNC_TTY=m @@ -505,7 +543,6 @@ CONFIG_ATH9K_HTC=m CONFIG_IWLWIFI=m CONFIG_IWLDVM=m CONFIG_IWLMVM=m -CONFIG_IWLWIFI_BCAST_FILTERING=y CONFIG_HOSTAP=m CONFIG_MT7601U=m CONFIG_RT2X00=m @@ -521,6 +558,14 @@ CONFIG_RTL8821AE=m CONFIG_RTL8192CU=m # CONFIG_RTLWIFI_DEBUG is not set CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8822BE=m +CONFIG_RTW88_8822CE=m +CONFIG_RTW88_8723DE=m +CONFIG_RTW88_8821CE=m +CONFIG_RTW89=m +CONFIG_RTW89_8852AE=m +CONFIG_RTW89_8852CE=m CONFIG_ZD1211RW=m CONFIG_USB_NET_RNDIS_WLAN=m CONFIG_INPUT_MOUSEDEV=y @@ -651,6 +696,11 @@ CONFIG_USB_SERIAL_FTDI_SIO=m CONFIG_USB_SERIAL_PL2303=m CONFIG_USB_SERIAL_OPTION=m CONFIG_USB_GADGET=y +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_TCPCI=m +CONFIG_TYPEC_UCSI=m +CONFIG_UCSI_ACPI=m CONFIG_INFINIBAND=m CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y @@ -688,7 +738,6 @@ CONFIG_COMEDI_NI_PCIDIO=m CONFIG_COMEDI_NI_PCIMIO=m CONFIG_STAGING=y CONFIG_R8188EU=m -# CONFIG_88EU_AP_MODE is not set CONFIG_PM_DEVFREQ=y CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y CONFIG_DEVFREQ_GOV_PERFORMANCE=y @@ -772,14 +821,12 @@ CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/loongarch/include/asm/Kbuild b/arch/loongarch/include/asm/Kbuild index 83bc0681e72b4428ed55e324d32024f35a7119c6..77ad8e6f0906c4be1e7e1b52c568de5ac977f416 100644 --- a/arch/loongarch/include/asm/Kbuild +++ b/arch/loongarch/include/asm/Kbuild @@ -1,12 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 generic-y += dma-contiguous.h generic-y += export.h +generic-y += mcs_spinlock.h generic-y += parport.h generic-y += early_ioremap.h generic-y += qrwlock.h -generic-y += qrwlock_types.h -generic-y += spinlock.h -generic-y += spinlock_types.h +generic-y += qspinlock.h generic-y += rwsem.h generic-y += segment.h generic-y += user.h @@ -21,7 +20,6 @@ generic-y += shmbuf.h generic-y += statfs.h generic-y += socket.h generic-y += sockios.h -generic-y += termios.h generic-y += termbits.h generic-y += poll.h generic-y += param.h diff --git a/arch/loongarch/include/asm/acpi.h b/arch/loongarch/include/asm/acpi.h index c5108213876cf1d479dc3f0b35f3e112cdaf8d85..17162f494b9b13039fbd266087701eef106797f7 100644 --- a/arch/loongarch/include/asm/acpi.h +++ b/arch/loongarch/include/asm/acpi.h @@ -15,7 +15,7 @@ extern int acpi_pci_disabled; extern int acpi_noirq; #define acpi_os_ioremap acpi_os_ioremap -void __init __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size); +void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size); static inline void disable_acpi(void) { diff --git a/arch/loongarch/include/asm/addrspace.h b/arch/loongarch/include/asm/addrspace.h index b91e0733b2e571aa79885d8764f11148384586d0..d342935e5a72d1de92c496087ba3f2e96e46d352 100644 --- a/arch/loongarch/include/asm/addrspace.h +++ b/arch/loongarch/include/asm/addrspace.h @@ -109,4 +109,20 @@ extern unsigned long vm_map_base; */ #define PHYSADDR(a) ((_ACAST64_(a)) & TO_PHYS_MASK) +/* + * On LoongArch, I/O ports mappring is following: + * + * | .... | + * |-----------------------| + * | pci io ports(16K~32M) | + * |-----------------------| + * | isa io ports(0 ~16K) | + * PCI_IOBASE ->|-----------------------| + * | .... | + */ +#define PCI_IOBASE ((void __iomem *)(vm_map_base + (2 * PAGE_SIZE))) +#define PCI_IOSIZE SZ_32M +#define ISA_IOSIZE SZ_16K +#define IO_SPACE_LIMIT (PCI_IOSIZE - 1) + #endif /* _ASM_ADDRSPACE_H */ diff --git a/arch/loongarch/include/asm/bootinfo.h b/arch/loongarch/include/asm/bootinfo.h index e02ac4af7f6e86515e9cfffd320be41cace56028..ed0910e8b856b8804c2d1027f44e74be8e23f620 100644 --- a/arch/loongarch/include/asm/bootinfo.h +++ b/arch/loongarch/include/asm/bootinfo.h @@ -36,8 +36,13 @@ struct loongson_system_configuration { }; extern u64 efi_system_table; -extern unsigned long fw_arg0, fw_arg1; +extern unsigned long fw_arg0, fw_arg1, fw_arg2; extern struct loongson_board_info b_info; extern struct loongson_system_configuration loongson_sysconf; +static inline bool io_master(int cpu) +{ + return test_bit(cpu, &loongson_sysconf.cores_io_master); +} + #endif /* _ASM_BOOTINFO_H */ diff --git a/arch/loongarch/include/asm/bug.h b/arch/loongarch/include/asm/bug.h index bda49108a76d0bba56a93b3139b3d7706298b238..d4ca3ba2541885c0a7d28229b1adacadad9ec0a5 100644 --- a/arch/loongarch/include/asm/bug.h +++ b/arch/loongarch/include/asm/bug.h @@ -2,21 +2,59 @@ #ifndef __ASM_BUG_H #define __ASM_BUG_H -#include +#include +#include + +#ifndef CONFIG_DEBUG_BUGVERBOSE +#define _BUGVERBOSE_LOCATION(file, line) +#else +#define __BUGVERBOSE_LOCATION(file, line) \ + .pushsection .rodata.str, "aMS", @progbits, 1; \ + 10002: .string file; \ + .popsection; \ + \ + .long 10002b - .; \ + .short line; +#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) +#endif -#ifdef CONFIG_BUG +#ifndef CONFIG_GENERIC_BUG +#define __BUG_ENTRY(flags) +#else +#define __BUG_ENTRY(flags) \ + .pushsection __bug_table, "aw"; \ + .align 2; \ + 10000: .long 10001f - .; \ + _BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ + .short flags; \ + .popsection; \ + 10001: +#endif -#include +#define ASM_BUG_FLAGS(flags) \ + __BUG_ENTRY(flags) \ + break BRK_BUG -static inline void __noreturn BUG(void) -{ - __asm__ __volatile__("break %0" : : "i" (BRK_BUG)); - unreachable(); -} +#define ASM_BUG() ASM_BUG_FLAGS(0) -#define HAVE_ARCH_BUG +#define __BUG_FLAGS(flags) \ + asm_inline volatile (__stringify(ASM_BUG_FLAGS(flags))); -#endif +#define __WARN_FLAGS(flags) \ +do { \ + instrumentation_begin(); \ + __BUG_FLAGS(BUGFLAG_WARNING|(flags)); \ + instrumentation_end(); \ +} while (0) + +#define BUG() \ +do { \ + instrumentation_begin(); \ + __BUG_FLAGS(0); \ + unreachable(); \ +} while (0) + +#define HAVE_ARCH_BUG #include diff --git a/arch/loongarch/include/asm/cacheflush.h b/arch/loongarch/include/asm/cacheflush.h index 670900141b7c80df0ae2041ef086c7e921cdc1a6..0681788eb474aa1a74b591afde869be5bd14dd6d 100644 --- a/arch/loongarch/include/asm/cacheflush.h +++ b/arch/loongarch/include/asm/cacheflush.h @@ -6,10 +6,33 @@ #define _ASM_CACHEFLUSH_H #include -#include +#include #include -extern void local_flush_icache_range(unsigned long start, unsigned long end); +static inline bool cache_present(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_PRESENT; +} + +static inline bool cache_private(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_PRIVATE; +} + +static inline bool cache_inclusive(struct cache_desc *cdesc) +{ + return cdesc->flags & CACHE_INCLUSIVE; +} + +static inline unsigned int cpu_last_level_cache_line_size(void) +{ + int cache_present = boot_cpu_data.cache_leaves_present; + + return boot_cpu_data.cache_leaves[cache_present - 1].linesz; +} + +asmlinkage void __flush_cache_all(void); +void local_flush_icache_range(unsigned long start, unsigned long end); #define flush_icache_range local_flush_icache_range #define flush_icache_user_range local_flush_icache_range @@ -35,44 +58,30 @@ extern void local_flush_icache_range(unsigned long start, unsigned long end); : \ : "i" (op), "ZC" (*(unsigned char *)(addr))) -static inline void flush_icache_line_indexed(unsigned long addr) -{ - cache_op(Index_Invalidate_I, addr); -} - -static inline void flush_dcache_line_indexed(unsigned long addr) -{ - cache_op(Index_Writeback_Inv_D, addr); -} - -static inline void flush_vcache_line_indexed(unsigned long addr) -{ - cache_op(Index_Writeback_Inv_V, addr); -} - -static inline void flush_scache_line_indexed(unsigned long addr) -{ - cache_op(Index_Writeback_Inv_S, addr); -} - -static inline void flush_icache_line(unsigned long addr) -{ - cache_op(Hit_Invalidate_I, addr); -} - -static inline void flush_dcache_line(unsigned long addr) -{ - cache_op(Hit_Writeback_Inv_D, addr); -} - -static inline void flush_vcache_line(unsigned long addr) -{ - cache_op(Hit_Writeback_Inv_V, addr); -} - -static inline void flush_scache_line(unsigned long addr) +static inline void flush_cache_line(int leaf, unsigned long addr) { - cache_op(Hit_Writeback_Inv_S, addr); + switch (leaf) { + case Cache_LEAF0: + cache_op(Index_Writeback_Inv_LEAF0, addr); + break; + case Cache_LEAF1: + cache_op(Index_Writeback_Inv_LEAF1, addr); + break; + case Cache_LEAF2: + cache_op(Index_Writeback_Inv_LEAF2, addr); + break; + case Cache_LEAF3: + cache_op(Index_Writeback_Inv_LEAF3, addr); + break; + case Cache_LEAF4: + cache_op(Index_Writeback_Inv_LEAF4, addr); + break; + case Cache_LEAF5: + cache_op(Index_Writeback_Inv_LEAF5, addr); + break; + default: + break; + } } #include diff --git a/arch/loongarch/include/asm/cacheops.h b/arch/loongarch/include/asm/cacheops.h index dc280efecebd8d5c091ef9957904128824f636a5..0f4a86f8e2bea78cd0796fb0848acd50111d79a3 100644 --- a/arch/loongarch/include/asm/cacheops.h +++ b/arch/loongarch/include/asm/cacheops.h @@ -8,16 +8,18 @@ #define __ASM_CACHEOPS_H /* - * Most cache ops are split into a 2 bit field identifying the cache, and a 3 + * Most cache ops are split into a 3 bit field identifying the cache, and a 2 * bit field identifying the cache operation. */ -#define CacheOp_Cache 0x03 -#define CacheOp_Op 0x1c +#define CacheOp_Cache 0x07 +#define CacheOp_Op 0x18 -#define Cache_I 0x00 -#define Cache_D 0x01 -#define Cache_V 0x02 -#define Cache_S 0x03 +#define Cache_LEAF0 0x00 +#define Cache_LEAF1 0x01 +#define Cache_LEAF2 0x02 +#define Cache_LEAF3 0x03 +#define Cache_LEAF4 0x04 +#define Cache_LEAF5 0x05 #define Index_Invalidate 0x08 #define Index_Writeback_Inv 0x08 @@ -25,13 +27,17 @@ #define Hit_Writeback_Inv 0x10 #define CacheOp_User_Defined 0x18 -#define Index_Invalidate_I (Cache_I | Index_Invalidate) -#define Index_Writeback_Inv_D (Cache_D | Index_Writeback_Inv) -#define Index_Writeback_Inv_V (Cache_V | Index_Writeback_Inv) -#define Index_Writeback_Inv_S (Cache_S | Index_Writeback_Inv) -#define Hit_Invalidate_I (Cache_I | Hit_Invalidate) -#define Hit_Writeback_Inv_D (Cache_D | Hit_Writeback_Inv) -#define Hit_Writeback_Inv_V (Cache_V | Hit_Writeback_Inv) -#define Hit_Writeback_Inv_S (Cache_S | Hit_Writeback_Inv) +#define Index_Writeback_Inv_LEAF0 (Cache_LEAF0 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF1 (Cache_LEAF1 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF2 (Cache_LEAF2 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF3 (Cache_LEAF3 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF4 (Cache_LEAF4 | Index_Writeback_Inv) +#define Index_Writeback_Inv_LEAF5 (Cache_LEAF5 | Index_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF0 (Cache_LEAF0 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF1 (Cache_LEAF1 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF2 (Cache_LEAF2 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF3 (Cache_LEAF3 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF4 (Cache_LEAF4 | Hit_Writeback_Inv) +#define Hit_Writeback_Inv_LEAF5 (Cache_LEAF5 | Hit_Writeback_Inv) #endif /* __ASM_CACHEOPS_H */ diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h index 0a9b0fac1eeeb6115bfcd444d35b1975f45a1277..ecfa6cf79806e68ec5aee96914c2a826e8f0c9fa 100644 --- a/arch/loongarch/include/asm/cmpxchg.h +++ b/arch/loongarch/include/asm/cmpxchg.h @@ -5,8 +5,9 @@ #ifndef __ASM_CMPXCHG_H #define __ASM_CMPXCHG_H -#include +#include #include +#include #define __xchg_asm(amswap_db, m, val) \ ({ \ @@ -21,10 +22,53 @@ __ret; \ }) -static inline unsigned long __xchg(volatile void *ptr, unsigned long x, - int size) +static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val, + unsigned int size) +{ + unsigned int shift; + u32 old32, mask, temp; + volatile u32 *ptr32; + + /* Mask value to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + val &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * exchange within the naturally aligned 4 byte integerthat includes + * it. + */ + shift = (unsigned long)ptr & 0x3; + shift *= BITS_PER_BYTE; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + + asm volatile ( + "1: ll.w %0, %3 \n" + " andn %1, %0, %z4 \n" + " or %1, %1, %z5 \n" + " sc.w %1, %2 \n" + " beqz %1, 1b \n" + : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) + : "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift) + : "memory"); + + return (old32 & mask) >> shift; +} + +static __always_inline unsigned long +__xchg(volatile void *ptr, unsigned long x, int size) { switch (size) { + case 1: + case 2: + return __xchg_small(ptr, x, size); + case 4: return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); @@ -67,10 +111,62 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x, __ret; \ }) -static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, - unsigned long new, unsigned int size) +static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old, + unsigned int new, unsigned int size) +{ + unsigned int shift; + u32 old32, mask, temp; + volatile u32 *ptr32; + + /* Mask inputs to the correct size. */ + mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); + old &= mask; + new &= mask; + + /* + * Calculate a shift & mask that correspond to the value we wish to + * compare & exchange within the naturally aligned 4 byte integer + * that includes it. + */ + shift = (unsigned long)ptr & 0x3; + shift *= BITS_PER_BYTE; + old <<= shift; + new <<= shift; + mask <<= shift; + + /* + * Calculate a pointer to the naturally aligned 4 byte integer that + * includes our byte of interest, and load its value. + */ + ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); + + asm volatile ( + "1: ll.w %0, %3 \n" + " and %1, %0, %z4 \n" + " bne %1, %z5, 2f \n" + " andn %1, %0, %z4 \n" + " or %1, %1, %z6 \n" + " sc.w %1, %2 \n" + " beqz %1, 1b \n" + " b 3f \n" + "2: \n" + __WEAK_LLSC_MB + "3: \n" + : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) + : "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new) + : "memory"); + + return (old32 & mask) >> shift; +} + +static __always_inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { + case 1: + case 2: + return __cmpxchg_small(ptr, old, new, size); + case 4: return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)old, new); diff --git a/arch/loongarch/include/asm/cpu-features.h b/arch/loongarch/include/asm/cpu-features.h index a8d87c40a0eb0606bc06cb90e1cf5263f5f8392d..b07974218393d1dd6f47fa45f429363134f4d605 100644 --- a/arch/loongarch/include/asm/cpu-features.h +++ b/arch/loongarch/include/asm/cpu-features.h @@ -19,11 +19,6 @@ #define cpu_has_loongarch32 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_32BIT) #define cpu_has_loongarch64 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) -#define cpu_icache_line_size() cpu_data[0].icache.linesz -#define cpu_dcache_line_size() cpu_data[0].dcache.linesz -#define cpu_vcache_line_size() cpu_data[0].vcache.linesz -#define cpu_scache_line_size() cpu_data[0].scache.linesz - #ifdef CONFIG_32BIT # define cpu_has_64bits (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) # define cpu_vabits 31 diff --git a/arch/loongarch/include/asm/cpu-info.h b/arch/loongarch/include/asm/cpu-info.h index b6c4f96079dfedf2bc63e96a20f9feded8cff238..cd73a6f57fe37c13600aa3e1b7a6f8ea1b087302 100644 --- a/arch/loongarch/include/asm/cpu-info.h +++ b/arch/loongarch/include/asm/cpu-info.h @@ -10,18 +10,28 @@ #include +/* cache_desc->flags */ +enum { + CACHE_PRESENT = (1 << 0), + CACHE_PRIVATE = (1 << 1), /* core private cache */ + CACHE_INCLUSIVE = (1 << 2), /* include the inner level caches */ +}; + /* * Descriptor for a cache */ struct cache_desc { - unsigned int waysize; /* Bytes per way */ + unsigned char type; + unsigned char level; unsigned short sets; /* Number of lines per set */ unsigned char ways; /* Number of ways */ unsigned char linesz; /* Size of line in bytes */ - unsigned char waybit; /* Bits to select in a cache set */ unsigned char flags; /* Flags describing cache properties */ }; +#define CACHE_LEVEL_MAX 3 +#define CACHE_LEAVES_MAX 6 + struct cpuinfo_loongarch { u64 asid_cache; unsigned long asid_mask; @@ -40,11 +50,8 @@ struct cpuinfo_loongarch { int tlbsizemtlb; int tlbsizestlbsets; int tlbsizestlbways; - struct cache_desc icache; /* Primary I-cache */ - struct cache_desc dcache; /* Primary D or combined I/D cache */ - struct cache_desc vcache; /* Victim cache, between pcache and scache */ - struct cache_desc scache; /* Secondary cache */ - struct cache_desc tcache; /* Tertiary/split secondary cache */ + int cache_leaves_present; /* number of cache_leaves[] elements */ + struct cache_desc cache_leaves[CACHE_LEAVES_MAX]; int core; /* physical core number in package */ int package;/* physical package number */ int vabits; /* Virtual Address size in bits */ diff --git a/arch/loongarch/include/asm/efi.h b/arch/loongarch/include/asm/efi.h index 9d44c6948be1bdc9d45dda22ce172e16b0c4f81f..174567b00ddb907d32241ae45a7d71ff2e902050 100644 --- a/arch/loongarch/include/asm/efi.h +++ b/arch/loongarch/include/asm/efi.h @@ -17,9 +17,16 @@ void efifb_setup_from_dmi(struct screen_info *si, const char *opt); #define arch_efi_call_virt_teardown() #define EFI_ALLOC_ALIGN SZ_64K +#define EFI_RT_VIRTUAL_OFFSET CSR_DMW0_BASE -struct screen_info *alloc_screen_info(void); -void free_screen_info(struct screen_info *si); +static inline struct screen_info *alloc_screen_info(void) +{ + return &screen_info; +} + +static inline void free_screen_info(struct screen_info *si) +{ +} static inline unsigned long efi_get_max_initrd_addr(unsigned long image_addr) { diff --git a/arch/loongarch/include/asm/elf.h b/arch/loongarch/include/asm/elf.h index 5f3ff4781fda83fd9e861a862f1a4ea8611d8619..7af0cebf28d73c5d3551eb10ecc36beb6b08d2d7 100644 --- a/arch/loongarch/include/asm/elf.h +++ b/arch/loongarch/include/asm/elf.h @@ -74,6 +74,43 @@ #define R_LARCH_SUB64 56 #define R_LARCH_GNU_VTINHERIT 57 #define R_LARCH_GNU_VTENTRY 58 +#define R_LARCH_B16 64 +#define R_LARCH_B21 65 +#define R_LARCH_B26 66 +#define R_LARCH_ABS_HI20 67 +#define R_LARCH_ABS_LO12 68 +#define R_LARCH_ABS64_LO20 69 +#define R_LARCH_ABS64_HI12 70 +#define R_LARCH_PCALA_HI20 71 +#define R_LARCH_PCALA_LO12 72 +#define R_LARCH_PCALA64_LO20 73 +#define R_LARCH_PCALA64_HI12 74 +#define R_LARCH_GOT_PC_HI20 75 +#define R_LARCH_GOT_PC_LO12 76 +#define R_LARCH_GOT64_PC_LO20 77 +#define R_LARCH_GOT64_PC_HI12 78 +#define R_LARCH_GOT_HI20 79 +#define R_LARCH_GOT_LO12 80 +#define R_LARCH_GOT64_LO20 81 +#define R_LARCH_GOT64_HI12 82 +#define R_LARCH_TLS_LE_HI20 83 +#define R_LARCH_TLS_LE_LO12 84 +#define R_LARCH_TLS_LE64_LO20 85 +#define R_LARCH_TLS_LE64_HI12 86 +#define R_LARCH_TLS_IE_PC_HI20 87 +#define R_LARCH_TLS_IE_PC_LO12 88 +#define R_LARCH_TLS_IE64_PC_LO20 89 +#define R_LARCH_TLS_IE64_PC_HI12 90 +#define R_LARCH_TLS_IE_HI20 91 +#define R_LARCH_TLS_IE_LO12 92 +#define R_LARCH_TLS_IE64_LO20 93 +#define R_LARCH_TLS_IE64_HI12 94 +#define R_LARCH_TLS_LD_PC_HI20 95 +#define R_LARCH_TLS_LD_HI20 96 +#define R_LARCH_TLS_GD_PC_HI20 97 +#define R_LARCH_TLS_GD_HI20 98 +#define R_LARCH_32_PCREL 99 +#define R_LARCH_RELAX 100 #ifndef ELF_ARCH diff --git a/arch/loongarch/include/asm/fixmap.h b/arch/loongarch/include/asm/fixmap.h index b3541dfa201382073c611886400eb24280012d14..d2e55ae55bb9c499b9a685509bc26d663054b58e 100644 --- a/arch/loongarch/include/asm/fixmap.h +++ b/arch/loongarch/include/asm/fixmap.h @@ -10,4 +10,19 @@ #define NR_FIX_BTMAPS 64 +enum fixed_addresses { + FIX_HOLE, + FIX_EARLYCON_MEM_BASE, + __end_of_fixed_addresses +}; + +#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) +#define FIXMAP_PAGE_IO PAGE_KERNEL_SUC + +extern void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags); + +#include + #endif diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 7b07cbb3188c0a4c1c8df07007ed1d5e788b67cf..fce1843ceebb36a30e67fd01554bca37932dd383 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -8,6 +8,8 @@ #include #include +#define INSN_BREAK 0x002a0000 + #define ADDR_IMMMASK_LU52ID 0xFFF0000000000000 #define ADDR_IMMMASK_LU32ID 0x000FFFFF00000000 #define ADDR_IMMMASK_ADDU16ID 0x00000000FFFF0000 @@ -18,9 +20,16 @@ #define ADDR_IMM(addr, INSN) ((addr & ADDR_IMMMASK_##INSN) >> ADDR_IMMSHIFT_##INSN) +enum reg0i26_op { + b_op = 0x14, + bl_op = 0x15, +}; + enum reg1i20_op { lu12iw_op = 0x0a, lu32id_op = 0x0b, + pcaddu12i_op = 0x0e, + pcaddu18i_op = 0x0f, }; enum reg1i21_op { @@ -28,10 +37,34 @@ enum reg1i21_op { bnez_op = 0x11, }; +enum reg2_op { + revb2h_op = 0x0c, + revb4h_op = 0x0d, + revb2w_op = 0x0e, + revbd_op = 0x0f, + revh2w_op = 0x10, + revhd_op = 0x11, +}; + +enum reg2i5_op { + slliw_op = 0x81, + srliw_op = 0x89, + sraiw_op = 0x91, +}; + +enum reg2i6_op { + sllid_op = 0x41, + srlid_op = 0x45, + sraid_op = 0x49, +}; + enum reg2i12_op { addiw_op = 0x0a, addid_op = 0x0b, lu52id_op = 0x0c, + andi_op = 0x0d, + ori_op = 0x0e, + xori_op = 0x0f, ldb_op = 0xa0, ldh_op = 0xa1, ldw_op = 0xa2, @@ -40,6 +73,20 @@ enum reg2i12_op { sth_op = 0xa5, stw_op = 0xa6, std_op = 0xa7, + ldbu_op = 0xa8, + ldhu_op = 0xa9, + ldwu_op = 0xaa, +}; + +enum reg2i14_op { + llw_op = 0x20, + scw_op = 0x21, + lld_op = 0x22, + scd_op = 0x23, + ldptrw_op = 0x24, + stptrw_op = 0x25, + ldptrd_op = 0x26, + stptrd_op = 0x27, }; enum reg2i16_op { @@ -52,6 +99,71 @@ enum reg2i16_op { bgeu_op = 0x1b, }; +enum reg2bstrd_op { + bstrinsd_op = 0x2, + bstrpickd_op = 0x3, +}; + +enum reg3_op { + addw_op = 0x20, + addd_op = 0x21, + subw_op = 0x22, + subd_op = 0x23, + nor_op = 0x28, + and_op = 0x29, + or_op = 0x2a, + xor_op = 0x2b, + orn_op = 0x2c, + andn_op = 0x2d, + sllw_op = 0x2e, + srlw_op = 0x2f, + sraw_op = 0x30, + slld_op = 0x31, + srld_op = 0x32, + srad_op = 0x33, + mulw_op = 0x38, + mulhw_op = 0x39, + mulhwu_op = 0x3a, + muld_op = 0x3b, + mulhd_op = 0x3c, + mulhdu_op = 0x3d, + divw_op = 0x40, + modw_op = 0x41, + divwu_op = 0x42, + modwu_op = 0x43, + divd_op = 0x44, + modd_op = 0x45, + divdu_op = 0x46, + moddu_op = 0x47, + ldxb_op = 0x7000, + ldxh_op = 0x7008, + ldxw_op = 0x7010, + ldxd_op = 0x7018, + stxb_op = 0x7020, + stxh_op = 0x7028, + stxw_op = 0x7030, + stxd_op = 0x7038, + ldxbu_op = 0x7040, + ldxhu_op = 0x7048, + ldxwu_op = 0x7050, + amswapw_op = 0x70c0, + amswapd_op = 0x70c1, + amaddw_op = 0x70c2, + amaddd_op = 0x70c3, + amandw_op = 0x70c4, + amandd_op = 0x70c5, + amorw_op = 0x70c6, + amord_op = 0x70c7, + amxorw_op = 0x70c8, + amxord_op = 0x70c9, +}; + +enum reg3sa2_op { + alslw_op = 0x02, + alslwu_op = 0x03, + alsld_op = 0x16, +}; + struct reg0i26_format { unsigned int immediate_h : 10; unsigned int immediate_l : 16; @@ -71,6 +183,26 @@ struct reg1i21_format { unsigned int opcode : 6; }; +struct reg2_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int opcode : 22; +}; + +struct reg2i5_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int immediate : 5; + unsigned int opcode : 17; +}; + +struct reg2i6_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int immediate : 6; + unsigned int opcode : 16; +}; + struct reg2i12_format { unsigned int rd : 5; unsigned int rj : 5; @@ -78,6 +210,13 @@ struct reg2i12_format { unsigned int opcode : 10; }; +struct reg2i14_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int immediate : 14; + unsigned int opcode : 8; +}; + struct reg2i16_format { unsigned int rd : 5; unsigned int rj : 5; @@ -85,13 +224,43 @@ struct reg2i16_format { unsigned int opcode : 6; }; +struct reg2bstrd_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int lsbd : 6; + unsigned int msbd : 6; + unsigned int opcode : 10; +}; + +struct reg3_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int rk : 5; + unsigned int opcode : 17; +}; + +struct reg3sa2_format { + unsigned int rd : 5; + unsigned int rj : 5; + unsigned int rk : 5; + unsigned int immediate : 2; + unsigned int opcode : 15; +}; + union loongarch_instruction { unsigned int word; - struct reg0i26_format reg0i26_format; - struct reg1i20_format reg1i20_format; - struct reg1i21_format reg1i21_format; - struct reg2i12_format reg2i12_format; - struct reg2i16_format reg2i16_format; + struct reg0i26_format reg0i26_format; + struct reg1i20_format reg1i20_format; + struct reg1i21_format reg1i21_format; + struct reg2_format reg2_format; + struct reg2i5_format reg2i5_format; + struct reg2i6_format reg2i6_format; + struct reg2i12_format reg2i12_format; + struct reg2i14_format reg2i14_format; + struct reg2i16_format reg2i16_format; + struct reg2bstrd_format reg2bstrd_format; + struct reg3_format reg3_format; + struct reg3sa2_format reg3sa2_format; }; #define LOONGARCH_INSN_SIZE sizeof(union loongarch_instruction) @@ -166,4 +335,235 @@ u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, unsigned long pc, unsigned long dest); +static inline bool signed_imm_check(long val, unsigned int bit) +{ + return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1)); +} + +static inline bool unsigned_imm_check(unsigned long val, unsigned int bit) +{ + return val < (1UL << bit); +} + +#define DEF_EMIT_REG0I26_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + int offset) \ +{ \ + unsigned int immediate_l, immediate_h; \ + \ + immediate_l = offset & 0xffff; \ + offset >>= 16; \ + immediate_h = offset & 0x3ff; \ + \ + insn->reg0i26_format.opcode = OP; \ + insn->reg0i26_format.immediate_l = immediate_l; \ + insn->reg0i26_format.immediate_h = immediate_h; \ +} + +DEF_EMIT_REG0I26_FORMAT(b, b_op) + +#define DEF_EMIT_REG1I20_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, int imm) \ +{ \ + insn->reg1i20_format.opcode = OP; \ + insn->reg1i20_format.immediate = imm; \ + insn->reg1i20_format.rd = rd; \ +} + +DEF_EMIT_REG1I20_FORMAT(lu12iw, lu12iw_op) +DEF_EMIT_REG1I20_FORMAT(lu32id, lu32id_op) +DEF_EMIT_REG1I20_FORMAT(pcaddu18i, pcaddu18i_op) + +#define DEF_EMIT_REG2_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj) \ +{ \ + insn->reg2_format.opcode = OP; \ + insn->reg2_format.rd = rd; \ + insn->reg2_format.rj = rj; \ +} + +DEF_EMIT_REG2_FORMAT(revb2h, revb2h_op) +DEF_EMIT_REG2_FORMAT(revb2w, revb2w_op) +DEF_EMIT_REG2_FORMAT(revbd, revbd_op) + +#define DEF_EMIT_REG2I5_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + int imm) \ +{ \ + insn->reg2i5_format.opcode = OP; \ + insn->reg2i5_format.immediate = imm; \ + insn->reg2i5_format.rd = rd; \ + insn->reg2i5_format.rj = rj; \ +} + +DEF_EMIT_REG2I5_FORMAT(slliw, slliw_op) +DEF_EMIT_REG2I5_FORMAT(srliw, srliw_op) +DEF_EMIT_REG2I5_FORMAT(sraiw, sraiw_op) + +#define DEF_EMIT_REG2I6_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + int imm) \ +{ \ + insn->reg2i6_format.opcode = OP; \ + insn->reg2i6_format.immediate = imm; \ + insn->reg2i6_format.rd = rd; \ + insn->reg2i6_format.rj = rj; \ +} + +DEF_EMIT_REG2I6_FORMAT(sllid, sllid_op) +DEF_EMIT_REG2I6_FORMAT(srlid, srlid_op) +DEF_EMIT_REG2I6_FORMAT(sraid, sraid_op) + +#define DEF_EMIT_REG2I12_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + int imm) \ +{ \ + insn->reg2i12_format.opcode = OP; \ + insn->reg2i12_format.immediate = imm; \ + insn->reg2i12_format.rd = rd; \ + insn->reg2i12_format.rj = rj; \ +} + +DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op) +DEF_EMIT_REG2I12_FORMAT(addid, addid_op) +DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op) +DEF_EMIT_REG2I12_FORMAT(andi, andi_op) +DEF_EMIT_REG2I12_FORMAT(ori, ori_op) +DEF_EMIT_REG2I12_FORMAT(xori, xori_op) +DEF_EMIT_REG2I12_FORMAT(ldbu, ldbu_op) +DEF_EMIT_REG2I12_FORMAT(ldhu, ldhu_op) +DEF_EMIT_REG2I12_FORMAT(ldwu, ldwu_op) +DEF_EMIT_REG2I12_FORMAT(ldd, ldd_op) +DEF_EMIT_REG2I12_FORMAT(stb, stb_op) +DEF_EMIT_REG2I12_FORMAT(sth, sth_op) +DEF_EMIT_REG2I12_FORMAT(stw, stw_op) +DEF_EMIT_REG2I12_FORMAT(std, std_op) + +#define DEF_EMIT_REG2I14_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + int imm) \ +{ \ + insn->reg2i14_format.opcode = OP; \ + insn->reg2i14_format.immediate = imm; \ + insn->reg2i14_format.rd = rd; \ + insn->reg2i14_format.rj = rj; \ +} + +DEF_EMIT_REG2I14_FORMAT(llw, llw_op) +DEF_EMIT_REG2I14_FORMAT(scw, scw_op) +DEF_EMIT_REG2I14_FORMAT(lld, lld_op) +DEF_EMIT_REG2I14_FORMAT(scd, scd_op) +DEF_EMIT_REG2I14_FORMAT(ldptrw, ldptrw_op) +DEF_EMIT_REG2I14_FORMAT(stptrw, stptrw_op) +DEF_EMIT_REG2I14_FORMAT(ldptrd, ldptrd_op) +DEF_EMIT_REG2I14_FORMAT(stptrd, stptrd_op) + +#define DEF_EMIT_REG2I16_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rj, \ + enum loongarch_gpr rd, \ + int offset) \ +{ \ + insn->reg2i16_format.opcode = OP; \ + insn->reg2i16_format.immediate = offset; \ + insn->reg2i16_format.rj = rj; \ + insn->reg2i16_format.rd = rd; \ +} + +DEF_EMIT_REG2I16_FORMAT(beq, beq_op) +DEF_EMIT_REG2I16_FORMAT(bne, bne_op) +DEF_EMIT_REG2I16_FORMAT(blt, blt_op) +DEF_EMIT_REG2I16_FORMAT(bge, bge_op) +DEF_EMIT_REG2I16_FORMAT(bltu, bltu_op) +DEF_EMIT_REG2I16_FORMAT(bgeu, bgeu_op) +DEF_EMIT_REG2I16_FORMAT(jirl, jirl_op) + +#define DEF_EMIT_REG2BSTRD_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + int msbd, \ + int lsbd) \ +{ \ + insn->reg2bstrd_format.opcode = OP; \ + insn->reg2bstrd_format.msbd = msbd; \ + insn->reg2bstrd_format.lsbd = lsbd; \ + insn->reg2bstrd_format.rj = rj; \ + insn->reg2bstrd_format.rd = rd; \ +} + +DEF_EMIT_REG2BSTRD_FORMAT(bstrpickd, bstrpickd_op) + +#define DEF_EMIT_REG3_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + enum loongarch_gpr rk) \ +{ \ + insn->reg3_format.opcode = OP; \ + insn->reg3_format.rd = rd; \ + insn->reg3_format.rj = rj; \ + insn->reg3_format.rk = rk; \ +} + +DEF_EMIT_REG3_FORMAT(addd, addd_op) +DEF_EMIT_REG3_FORMAT(subd, subd_op) +DEF_EMIT_REG3_FORMAT(muld, muld_op) +DEF_EMIT_REG3_FORMAT(divdu, divdu_op) +DEF_EMIT_REG3_FORMAT(moddu, moddu_op) +DEF_EMIT_REG3_FORMAT(and, and_op) +DEF_EMIT_REG3_FORMAT(or, or_op) +DEF_EMIT_REG3_FORMAT(xor, xor_op) +DEF_EMIT_REG3_FORMAT(sllw, sllw_op) +DEF_EMIT_REG3_FORMAT(slld, slld_op) +DEF_EMIT_REG3_FORMAT(srlw, srlw_op) +DEF_EMIT_REG3_FORMAT(srld, srld_op) +DEF_EMIT_REG3_FORMAT(sraw, sraw_op) +DEF_EMIT_REG3_FORMAT(srad, srad_op) +DEF_EMIT_REG3_FORMAT(ldxbu, ldxbu_op) +DEF_EMIT_REG3_FORMAT(ldxhu, ldxhu_op) +DEF_EMIT_REG3_FORMAT(ldxwu, ldxwu_op) +DEF_EMIT_REG3_FORMAT(ldxd, ldxd_op) +DEF_EMIT_REG3_FORMAT(stxb, stxb_op) +DEF_EMIT_REG3_FORMAT(stxh, stxh_op) +DEF_EMIT_REG3_FORMAT(stxw, stxw_op) +DEF_EMIT_REG3_FORMAT(stxd, stxd_op) +DEF_EMIT_REG3_FORMAT(amaddw, amaddw_op) +DEF_EMIT_REG3_FORMAT(amaddd, amaddd_op) +DEF_EMIT_REG3_FORMAT(amandw, amandw_op) +DEF_EMIT_REG3_FORMAT(amandd, amandd_op) +DEF_EMIT_REG3_FORMAT(amorw, amorw_op) +DEF_EMIT_REG3_FORMAT(amord, amord_op) +DEF_EMIT_REG3_FORMAT(amxorw, amxorw_op) +DEF_EMIT_REG3_FORMAT(amxord, amxord_op) +DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op) +DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op) + +#define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \ +static inline void emit_##NAME(union loongarch_instruction *insn, \ + enum loongarch_gpr rd, \ + enum loongarch_gpr rj, \ + enum loongarch_gpr rk, \ + int imm) \ +{ \ + insn->reg3sa2_format.opcode = OP; \ + insn->reg3sa2_format.immediate = imm; \ + insn->reg3sa2_format.rd = rd; \ + insn->reg3sa2_format.rj = rj; \ + insn->reg3sa2_format.rk = rk; \ +} + +DEF_EMIT_REG3SA2_FORMAT(alsld, alsld_op) + #endif /* _ASM_INST_H */ diff --git a/arch/loongarch/include/asm/io.h b/arch/loongarch/include/asm/io.h index 884599739b3676d3e693dc1d56918f3cbb964f75..402a7d9e3a53eafed41c4170de5627f673d142f4 100644 --- a/arch/loongarch/include/asm/io.h +++ b/arch/loongarch/include/asm/io.h @@ -7,34 +7,15 @@ #define ARCH_HAS_IOREMAP_WC -#include #include #include #include -#include -#include #include #include #include #include -/* - * On LoongArch, I/O ports mappring is following: - * - * | .... | - * |-----------------------| - * | pci io ports(64K~32M) | - * |-----------------------| - * | isa io ports(0 ~16K) | - * PCI_IOBASE ->|-----------------------| - * | .... | - */ -#define PCI_IOBASE ((void __iomem *)(vm_map_base + (2 * PAGE_SIZE))) -#define PCI_IOSIZE SZ_32M -#define ISA_IOSIZE SZ_16K -#define IO_SPACE_LIMIT (PCI_IOSIZE - 1) - /* * Change "struct page" to physical address. */ @@ -46,71 +27,38 @@ extern void __init early_iounmap(void __iomem *addr, unsigned long size); #define early_memremap early_ioremap #define early_memunmap early_iounmap +#ifdef CONFIG_ARCH_IOREMAP + static inline void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, unsigned long prot_val) { - if (prot_val == _CACHE_CC) + if (prot_val & _CACHE_CC) return (void __iomem *)(unsigned long)(CACHE_BASE + offset); else return (void __iomem *)(unsigned long)(UNCACHE_BASE + offset); } -/* - * ioremap - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. - */ -#define ioremap(offset, size) \ - ioremap_prot((offset), (size), _CACHE_SUC) +#define ioremap(offset, size) \ + ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL_SUC)) -/* - * ioremap_wc - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap_wc performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. - * - * This version of ioremap ensures that the memory is marked uncachable - * but accelerated by means of write-combining feature. It is specifically - * useful for PCIe prefetchable windows, which may vastly improve a - * communications performance. If it was determined on boot stage, what - * CPU CCA doesn't support WUC, the method shall fall-back to the - * _CACHE_SUC option (see cpu_probe() method). - */ -#define ioremap_wc(offset, size) \ - ioremap_prot((offset), (size), _CACHE_WUC) +#define iounmap(addr) ((void)(addr)) + +#endif /* - * ioremap_cache - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap_cache performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. + * On LoongArch, ioremap() has two variants, ioremap_wc() and ioremap_cache(). + * They map bus memory into CPU space, the mapped memory is marked uncachable + * (_CACHE_SUC), uncachable but accelerated by write-combine (_CACHE_WUC) and + * cachable (_CACHE_CC) respectively for CPU access. * - * This version of ioremap ensures that the memory is marked cachable by - * the CPU. Also enables full write-combining. Useful for some - * memory-like regions on I/O busses. + * @offset: bus address of the memory + * @size: size of the resource to map */ -#define ioremap_cache(offset, size) \ - ioremap_prot((offset), (size), _CACHE_CC) +#define ioremap_wc(offset, size) \ + ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL_WUC)) -static inline void iounmap(const volatile void __iomem *addr) -{ -} +#define ioremap_cache(offset, size) \ + ioremap_prot((offset), (size), pgprot_val(PAGE_KERNEL)) #define mmiowb() asm volatile ("dbar 0" ::: "memory") @@ -126,4 +74,8 @@ extern void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t #include +#define ARCH_HAS_VALID_PHYS_ADDR_RANGE +extern int valid_phys_addr_range(phys_addr_t addr, size_t size); +extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); + #endif /* _ASM_IO_H */ diff --git a/arch/loongarch/include/asm/kexec.h b/arch/loongarch/include/asm/kexec.h new file mode 100644 index 0000000000000000000000000000000000000000..cf95cd3eb2deaefb065719e970062a98a8356346 --- /dev/null +++ b/arch/loongarch/include/asm/kexec.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * kexec.h for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#ifndef _ASM_KEXEC_H +#define _ASM_KEXEC_H + +#include +#include + +/* Maximum physical address we can use pages from */ +#define KEXEC_SOURCE_MEMORY_LIMIT (-1UL) +/* Maximum address we can reach in physical address mode */ +#define KEXEC_DESTINATION_MEMORY_LIMIT (-1UL) + /* Maximum address we can use for the control code buffer */ +#define KEXEC_CONTROL_MEMORY_LIMIT (-1UL) + +/* Reserve a page for the control code buffer */ +#define KEXEC_CONTROL_PAGE_SIZE PAGE_SIZE + +/* The native architecture */ +#define KEXEC_ARCH KEXEC_ARCH_LOONGARCH + +static inline void crash_setup_regs(struct pt_regs *newregs, + struct pt_regs *oldregs) +{ + if (oldregs) + memcpy(newregs, oldregs, sizeof(*newregs)); + else + prepare_frametrace(newregs); +} + +#define ARCH_HAS_KIMAGE_ARCH + +struct kimage_arch { + unsigned long efi_boot; + unsigned long cmdline_ptr; + unsigned long systable_ptr; +}; + +typedef void (*do_kexec_t)(unsigned long efi_boot, + unsigned long cmdline_ptr, + unsigned long systable_ptr, + unsigned long start_addr, + unsigned long first_ind_entry); + +struct kimage; +extern const unsigned char relocate_new_kernel[]; +extern const size_t relocate_new_kernel_size; +extern void kexec_reboot(void); + +#ifdef CONFIG_SMP +extern atomic_t kexec_ready_to_reboot; +extern const unsigned char kexec_smp_wait[]; +#endif + +#endif /* !_ASM_KEXEC_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 3ba4f7e87cd254404b223395b9567f09fd718821..7f8d57a61c8bdd1c7cd9d2d77fdbe23bc10e89e3 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -187,36 +187,15 @@ static inline u32 read_cpucfg(u32 reg) #define CPUCFG16_L3_DINCL BIT(16) #define LOONGARCH_CPUCFG17 0x11 -#define CPUCFG17_L1I_WAYS_M GENMASK(15, 0) -#define CPUCFG17_L1I_SETS_M GENMASK(23, 16) -#define CPUCFG17_L1I_SIZE_M GENMASK(30, 24) -#define CPUCFG17_L1I_WAYS 0 -#define CPUCFG17_L1I_SETS 16 -#define CPUCFG17_L1I_SIZE 24 - #define LOONGARCH_CPUCFG18 0x12 -#define CPUCFG18_L1D_WAYS_M GENMASK(15, 0) -#define CPUCFG18_L1D_SETS_M GENMASK(23, 16) -#define CPUCFG18_L1D_SIZE_M GENMASK(30, 24) -#define CPUCFG18_L1D_WAYS 0 -#define CPUCFG18_L1D_SETS 16 -#define CPUCFG18_L1D_SIZE 24 - #define LOONGARCH_CPUCFG19 0x13 -#define CPUCFG19_L2_WAYS_M GENMASK(15, 0) -#define CPUCFG19_L2_SETS_M GENMASK(23, 16) -#define CPUCFG19_L2_SIZE_M GENMASK(30, 24) -#define CPUCFG19_L2_WAYS 0 -#define CPUCFG19_L2_SETS 16 -#define CPUCFG19_L2_SIZE 24 - #define LOONGARCH_CPUCFG20 0x14 -#define CPUCFG20_L3_WAYS_M GENMASK(15, 0) -#define CPUCFG20_L3_SETS_M GENMASK(23, 16) -#define CPUCFG20_L3_SIZE_M GENMASK(30, 24) -#define CPUCFG20_L3_WAYS 0 -#define CPUCFG20_L3_SETS 16 -#define CPUCFG20_L3_SIZE 24 +#define CPUCFG_CACHE_WAYS_M GENMASK(15, 0) +#define CPUCFG_CACHE_SETS_M GENMASK(23, 16) +#define CPUCFG_CACHE_LSIZE_M GENMASK(30, 24) +#define CPUCFG_CACHE_WAYS 0 +#define CPUCFG_CACHE_SETS 16 +#define CPUCFG_CACHE_LSIZE 24 #define LOONGARCH_CPUCFG48 0x30 #define CPUCFG48_MCSR_LCK BIT(0) diff --git a/arch/loongarch/include/asm/loongson.h b/arch/loongarch/include/asm/loongson.h index 6e8f6972ceb614c9cd96b3814063006507d71064..00db93edae1ba3f6c37ad0db0ac01b3b06ef06b8 100644 --- a/arch/loongarch/include/asm/loongson.h +++ b/arch/loongarch/include/asm/loongson.h @@ -14,8 +14,6 @@ #include #include -extern const struct plat_smp_ops loongson3_smp_ops; - #define LOONGSON_REG(x) \ (*(volatile u32 *)((char *)TO_UNCACHE(LOONGSON_REG_BASE) + (x))) diff --git a/arch/loongarch/include/asm/module.h b/arch/loongarch/include/asm/module.h index 9f6718df18547e3a29e81c5a44b01b2de7a22c1d..b29b19a46f4270553c03ce85d1a5d59be78b9d26 100644 --- a/arch/loongarch/include/asm/module.h +++ b/arch/loongarch/include/asm/module.h @@ -17,10 +17,15 @@ struct mod_section { }; struct mod_arch_specific { + struct mod_section got; struct mod_section plt; struct mod_section plt_idx; }; +struct got_entry { + Elf_Addr symbol_addr; +}; + struct plt_entry { u32 inst_lu12iw; u32 inst_lu32id; @@ -29,10 +34,16 @@ struct plt_entry { }; struct plt_idx_entry { - unsigned long symbol_addr; + Elf_Addr symbol_addr; }; -Elf_Addr module_emit_plt_entry(struct module *mod, unsigned long val); +Elf_Addr module_emit_got_entry(struct module *mod, Elf_Addr val); +Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Addr val); + +static inline struct got_entry emit_got_entry(Elf_Addr val) +{ + return (struct got_entry) { val }; +} static inline struct plt_entry emit_plt_entry(unsigned long val) { @@ -77,4 +88,16 @@ static inline struct plt_entry *get_plt_entry(unsigned long val, return plt + plt_idx; } +static inline struct got_entry *get_got_entry(Elf_Addr val, + const struct mod_section *sec) +{ + struct got_entry *got = (struct got_entry *)sec->shdr->sh_addr; + int i; + + for (i = 0; i < sec->num_entries; i++) + if (got[i].symbol_addr == val) + return &got[i]; + return NULL; +} + #endif /* _ASM_MODULE_H */ diff --git a/arch/loongarch/include/asm/module.lds.h b/arch/loongarch/include/asm/module.lds.h index 31c1c0db11a3a3690a36252271cfdbdfae8dcdcb..a3d1bc0fcc72e99dcab35aa623f8956fdbec41f3 100644 --- a/arch/loongarch/include/asm/module.lds.h +++ b/arch/loongarch/include/asm/module.lds.h @@ -2,6 +2,7 @@ /* Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ SECTIONS { . = ALIGN(4); + .got : { BYTE(0) } .plt : { BYTE(0) } .plt.idx : { BYTE(0) } } diff --git a/arch/loongarch/include/asm/page.h b/arch/loongarch/include/asm/page.h index a37324ac460b6e0a002ca1abd00fc255f42ff08f..53f284a961823c9e3f9cad4613a45d42eece9617 100644 --- a/arch/loongarch/include/asm/page.h +++ b/arch/loongarch/include/asm/page.h @@ -95,7 +95,7 @@ static inline int pfn_valid(unsigned long pfn) #endif -#define virt_to_pfn(kaddr) PFN_DOWN(virt_to_phys((void *)(kaddr))) +#define virt_to_pfn(kaddr) PFN_DOWN(PHYSADDR(kaddr)) #define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr)) extern int __virt_addr_valid(volatile void *kaddr); diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h index e6569f18c6ddfcd5e8d75bde426c4e4b5a7071fe..ad8d88494554a73951ac1c7b12c5b322c9750ab5 100644 --- a/arch/loongarch/include/asm/percpu.h +++ b/arch/loongarch/include/asm/percpu.h @@ -8,6 +8,15 @@ #include #include +/* + * The "address" (in fact, offset from $r21) of a per-CPU variable is close to + * the loading address of main kernel image, but far from where the modules are + * loaded. Tell the compiler this fact when using explicit relocs. + */ +#if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS) +#define PER_CPU_ATTRIBUTES __attribute__((model("extreme"))) +#endif + /* Use r21 for fast access */ register unsigned long __my_cpu_offset __asm__("$r21"); @@ -123,6 +132,10 @@ static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size) { switch (size) { + case 1: + case 2: + return __xchg_small((volatile void *)ptr, val, size); + case 4: return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val); @@ -204,9 +217,13 @@ do { \ #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) diff --git a/arch/loongarch/include/asm/perf_event.h b/arch/loongarch/include/asm/perf_event.h index dcb3b17053a83fdadedf459abb3b42325f0a6a82..2a35a0bc2aaabf128cb5336d25dcbec1738d646b 100644 --- a/arch/loongarch/include/asm/perf_event.h +++ b/arch/loongarch/include/asm/perf_event.h @@ -6,5 +6,7 @@ #ifndef __LOONGARCH_PERF_EVENT_H__ #define __LOONGARCH_PERF_EVENT_H__ -/* Nothing to show here; the file is required by linux/perf_event.h. */ + +#define perf_arch_bpf_user_pt_regs(regs) (struct user_pt_regs *)regs + #endif /* __LOONGARCH_PERF_EVENT_H__ */ diff --git a/arch/loongarch/include/asm/pgtable-bits.h b/arch/loongarch/include/asm/pgtable-bits.h index 9ca147a29bab81040e7f9322c2331ca4c25c43b4..3d1e0a69975a59f56fac3855c7b5d344be6da463 100644 --- a/arch/loongarch/include/asm/pgtable-bits.h +++ b/arch/loongarch/include/asm/pgtable-bits.h @@ -83,8 +83,11 @@ _PAGE_GLOBAL | _PAGE_KERN | _CACHE_SUC) #define PAGE_KERNEL_WUC __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ _PAGE_GLOBAL | _PAGE_KERN | _CACHE_WUC) + #ifndef __ASSEMBLY__ +#define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL_SUC) + #define pgprot_noncached pgprot_noncached static inline pgprot_t pgprot_noncached(pgprot_t _prot) diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index e03443abaf7daab97400a1e5f954d9f124282e9f..946704bee599ee843c9abc154355683550b1c6e0 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -59,7 +59,6 @@ #include #include #include -#include struct mm_struct; struct vm_area_struct; @@ -145,7 +144,7 @@ static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) *p4d = p4dval; } -#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) +#define p4d_phys(p4d) PHYSADDR(p4d_val(p4d)) #define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) #endif @@ -188,7 +187,7 @@ static inline pmd_t *pud_pgtable(pud_t pud) #define set_pud(pudptr, pudval) do { *(pudptr) = (pudval); } while (0) -#define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) +#define pud_phys(pud) PHYSADDR(pud_val(pud)) #define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) #endif @@ -221,7 +220,7 @@ static inline void pmd_clear(pmd_t *pmdp) #define set_pmd(pmdptr, pmdval) do { *(pmdptr) = (pmdval); } while (0) -#define pmd_phys(pmd) virt_to_phys((void *)pmd_val(pmd)) +#define pmd_phys(pmd) PHYSADDR(pmd_val(pmd)) #ifndef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) @@ -413,6 +412,9 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, __update_tlb(vma, address, ptep); } +#define __HAVE_ARCH_UPDATE_MMU_TLB +#define update_mmu_tlb update_mmu_cache + static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h index 1c4b4308378d43424aa31d4df2b7c9cd239747f5..6954dc5d24e9df599b184f63487cd8a1b05a250f 100644 --- a/arch/loongarch/include/asm/processor.h +++ b/arch/loongarch/include/asm/processor.h @@ -176,9 +176,6 @@ struct thread_struct { struct task_struct; -/* Free all resources held by a thread. */ -#define release_thread(thread) do { } while (0) - enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL}; extern unsigned long boot_option_idle_override; diff --git a/arch/loongarch/include/asm/reboot.h b/arch/loongarch/include/asm/reboot.h deleted file mode 100644 index 51151749d8f05794f6e1a4d72321d362661c1563..0000000000000000000000000000000000000000 --- a/arch/loongarch/include/asm/reboot.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2020-2022 Loongson Technology Corporation Limited - */ -#ifndef _ASM_REBOOT_H -#define _ASM_REBOOT_H - -extern void (*pm_restart)(void); - -#endif /* _ASM_REBOOT_H */ diff --git a/arch/loongarch/include/asm/setup.h b/arch/loongarch/include/asm/setup.h index 6d7d2a3e23dd696fdec4fa305e5b33c6f6a889b6..ca373f8e3c4db29a42fccdcca72a816135ea286c 100644 --- a/arch/loongarch/include/asm/setup.h +++ b/arch/loongarch/include/asm/setup.h @@ -13,7 +13,9 @@ extern unsigned long eentry; extern unsigned long tlbrentry; +extern void tlb_init(int cpu); extern void cpu_cache_init(void); +extern void cache_error_setup(void); extern void per_cpu_trap_init(int cpu); extern void set_handler(unsigned long offset, void *addr, unsigned long len); extern void set_merr_handler(unsigned long offset, void *addr, unsigned long len); diff --git a/arch/loongarch/include/asm/spinlock.h b/arch/loongarch/include/asm/spinlock.h new file mode 100644 index 0000000000000000000000000000000000000000..7cb3476999becc549ed99d3c56137c8dd617b008 --- /dev/null +++ b/arch/loongarch/include/asm/spinlock.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SPINLOCK_H +#define _ASM_SPINLOCK_H + +#include +#include +#include + +#endif /* _ASM_SPINLOCK_H */ diff --git a/arch/loongarch/include/asm/spinlock_types.h b/arch/loongarch/include/asm/spinlock_types.h new file mode 100644 index 0000000000000000000000000000000000000000..7458d036c161dd19db2fd38b4acbc226431956ef --- /dev/null +++ b/arch/loongarch/include/asm/spinlock_types.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#ifndef _ASM_SPINLOCK_TYPES_H +#define _ASM_SPINLOCK_TYPES_H + +#include +#include + +#endif diff --git a/arch/loongarch/include/uapi/asm/bpf_perf_event.h b/arch/loongarch/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 0000000000000000000000000000000000000000..eb6e2fd2a1f01eaddfd0dfb31bdc868e1b12fb81 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ +#define _UAPI__ASM_BPF_PERF_EVENT_H__ + +#include + +typedef struct user_pt_regs bpf_user_pt_regs_t; + +#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ diff --git a/arch/loongarch/include/uapi/asm/perf_regs.h b/arch/loongarch/include/uapi/asm/perf_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..29d69c00fc7a665f743d931223916c43cd0b4849 --- /dev/null +++ b/arch/loongarch/include/uapi/asm/perf_regs.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASM_LOONGARCH_PERF_REGS_H +#define _ASM_LOONGARCH_PERF_REGS_H + +enum perf_event_loongarch_regs { + PERF_REG_LOONGARCH_PC, + PERF_REG_LOONGARCH_R1, + PERF_REG_LOONGARCH_R2, + PERF_REG_LOONGARCH_R3, + PERF_REG_LOONGARCH_R4, + PERF_REG_LOONGARCH_R5, + PERF_REG_LOONGARCH_R6, + PERF_REG_LOONGARCH_R7, + PERF_REG_LOONGARCH_R8, + PERF_REG_LOONGARCH_R9, + PERF_REG_LOONGARCH_R10, + PERF_REG_LOONGARCH_R11, + PERF_REG_LOONGARCH_R12, + PERF_REG_LOONGARCH_R13, + PERF_REG_LOONGARCH_R14, + PERF_REG_LOONGARCH_R15, + PERF_REG_LOONGARCH_R16, + PERF_REG_LOONGARCH_R17, + PERF_REG_LOONGARCH_R18, + PERF_REG_LOONGARCH_R19, + PERF_REG_LOONGARCH_R20, + PERF_REG_LOONGARCH_R21, + PERF_REG_LOONGARCH_R22, + PERF_REG_LOONGARCH_R23, + PERF_REG_LOONGARCH_R24, + PERF_REG_LOONGARCH_R25, + PERF_REG_LOONGARCH_R26, + PERF_REG_LOONGARCH_R27, + PERF_REG_LOONGARCH_R28, + PERF_REG_LOONGARCH_R29, + PERF_REG_LOONGARCH_R30, + PERF_REG_LOONGARCH_R31, + PERF_REG_LOONGARCH_MAX, +}; +#endif /* _ASM_LOONGARCH_PERF_REGS_H */ diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile index e5be17009fe8a4ba456bf2644888811d49ef4b93..42be564278fa1948167ca660672ccd839353ecb9 100644 --- a/arch/loongarch/kernel/Makefile +++ b/arch/loongarch/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for the Linux/LoongArch kernel. # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y += cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \ +obj-y += head.o cpu-probe.o cacheinfo.o env.o setup.o entry.o genex.o \ traps.o irq.o idle.o process.o dma.o mem.o io.o reset.o switch.o \ elf.o syscall.o signal.o time.o topology.o inst.o ptrace.o vdso.o @@ -23,7 +23,14 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_NUMA) += numa.o +obj-$(CONFIG_MAGIC_SYSRQ) += sysrq.o + +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o + obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_UNWINDER_PROLOGUE) += unwind_prologue.o +obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_regs.o + CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index f1c928648a4a6178b4dc5449126429292374af27..3353984820388349b4e961e79a4767717be92a6f 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -48,7 +48,7 @@ void __init __acpi_unmap_table(void __iomem *map, unsigned long size) early_memunmap(map, size); } -void __init __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) +void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) { if (!memblock_is_memory(phys)) return ioremap(phys, size); diff --git a/arch/loongarch/kernel/cacheinfo.c b/arch/loongarch/kernel/cacheinfo.c index 4662b06269f42eea27830bd2439fc693b4e487b5..c7988f757281cbc22116286aad2921acd5b64c83 100644 --- a/arch/loongarch/kernel/cacheinfo.c +++ b/arch/loongarch/kernel/cacheinfo.c @@ -5,73 +5,34 @@ * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ #include +#include #include #include -/* Populates leaf and increments to next leaf */ -#define populate_cache(cache, leaf, c_level, c_type) \ -do { \ - leaf->type = c_type; \ - leaf->level = c_level; \ - leaf->coherency_line_size = c->cache.linesz; \ - leaf->number_of_sets = c->cache.sets; \ - leaf->ways_of_associativity = c->cache.ways; \ - leaf->size = c->cache.linesz * c->cache.sets * \ - c->cache.ways; \ - if (leaf->level > 2) \ - leaf->size *= nodes_per_package; \ - leaf++; \ -} while (0) - int init_cache_level(unsigned int cpu) { - struct cpuinfo_loongarch *c = ¤t_cpu_data; + int cache_present = current_cpu_data.cache_leaves_present; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - int levels = 0, leaves = 0; - - /* - * If Dcache is not set, we assume the cache structures - * are not properly initialized. - */ - if (c->dcache.waysize) - levels += 1; - else - return -ENOENT; - - - leaves += (c->icache.waysize) ? 2 : 1; - - if (c->vcache.waysize) { - levels++; - leaves++; - } - if (c->scache.waysize) { - levels++; - leaves++; - } + this_cpu_ci->num_levels = + current_cpu_data.cache_leaves[cache_present - 1].level; + this_cpu_ci->num_leaves = cache_present; - if (c->tcache.waysize) { - levels++; - leaves++; - } - - this_cpu_ci->num_levels = levels; - this_cpu_ci->num_leaves = leaves; return 0; } static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, struct cacheinfo *sib_leaf) { - return !((this_leaf->level == 1) || (this_leaf->level == 2)); + return (!(*(unsigned char *)(this_leaf->priv) & CACHE_PRIVATE) + && !(*(unsigned char *)(sib_leaf->priv) & CACHE_PRIVATE)); } static void cache_cpumap_setup(unsigned int cpu) { - struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); - struct cacheinfo *this_leaf, *sib_leaf; unsigned int index; + struct cacheinfo *this_leaf, *sib_leaf; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); for (index = 0; index < this_cpu_ci->num_leaves; index++) { unsigned int i; @@ -85,8 +46,10 @@ static void cache_cpumap_setup(unsigned int cpu) for_each_online_cpu(i) { struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i); - if (i == cpu || !sib_cpu_ci->info_list) - continue;/* skip if itself or no cacheinfo */ + if (i == cpu || !sib_cpu_ci->info_list || + (cpu_to_node(i) != cpu_to_node(cpu))) + continue; + sib_leaf = sib_cpu_ci->info_list + index; if (cache_leaves_are_shared(this_leaf, sib_leaf)) { cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); @@ -98,31 +61,24 @@ static void cache_cpumap_setup(unsigned int cpu) int populate_cache_leaves(unsigned int cpu) { - int level = 1, nodes_per_package = 1; - struct cpuinfo_loongarch *c = ¤t_cpu_data; + int i, cache_present = current_cpu_data.cache_leaves_present; struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); struct cacheinfo *this_leaf = this_cpu_ci->info_list; - - if (loongson_sysconf.nr_nodes > 1) - nodes_per_package = loongson_sysconf.cores_per_package - / loongson_sysconf.cores_per_node; - - if (c->icache.waysize) { - populate_cache(dcache, this_leaf, level, CACHE_TYPE_DATA); - populate_cache(icache, this_leaf, level++, CACHE_TYPE_INST); - } else { - populate_cache(dcache, this_leaf, level++, CACHE_TYPE_UNIFIED); + struct cache_desc *cd, *cdesc = current_cpu_data.cache_leaves; + + for (i = 0; i < cache_present; i++) { + cd = cdesc + i; + + this_leaf->type = cd->type; + this_leaf->level = cd->level; + this_leaf->coherency_line_size = cd->linesz; + this_leaf->number_of_sets = cd->sets; + this_leaf->ways_of_associativity = cd->ways; + this_leaf->size = cd->linesz * cd->sets * cd->ways; + this_leaf->priv = &cd->flags; + this_leaf++; } - if (c->vcache.waysize) - populate_cache(vcache, this_leaf, level++, CACHE_TYPE_UNIFIED); - - if (c->scache.waysize) - populate_cache(scache, this_leaf, level++, CACHE_TYPE_UNIFIED); - - if (c->tcache.waysize) - populate_cache(tcache, this_leaf, level++, CACHE_TYPE_UNIFIED); - cache_cpumap_setup(cpu); this_cpu_ci->cpu_map_populated = true; diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 529ab8f44ec6d9fcecbba29099c224fa087d9c08..255a09876ef28d1b10c67fd1b9f8ca4814f8aefc 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -187,7 +187,9 @@ static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]); uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]); - __cpu_full_name[cpu] = cpu_full_name; + if (!__cpu_full_name[cpu]) + __cpu_full_name[cpu] = cpu_full_name; + *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR); *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME); diff --git a/arch/loongarch/kernel/crash_dump.c b/arch/loongarch/kernel/crash_dump.c new file mode 100644 index 0000000000000000000000000000000000000000..e559307c10929a6974442aa0762ad5e43f5fc701 --- /dev/null +++ b/arch/loongarch/kernel/crash_dump.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +ssize_t copy_oldmem_page(struct iov_iter *iter, unsigned long pfn, + size_t csize, unsigned long offset) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = memremap(__pfn_to_phys(pfn), PAGE_SIZE, MEMREMAP_WB); + if (!vaddr) + return -ENOMEM; + + csize = copy_to_iter(vaddr + offset, csize, iter); + + memunmap(vaddr); + + return csize; +} diff --git a/arch/loongarch/kernel/dma.c b/arch/loongarch/kernel/dma.c index 8c9b5314a13edf0b1034cd8ec7c565984d58a1b3..7a9c6a9dd2d01fb429b67aea02deada5e5b1f3be 100644 --- a/arch/loongarch/kernel/dma.c +++ b/arch/loongarch/kernel/dma.c @@ -2,39 +2,29 @@ /* * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ -#include +#include #include -#include -#include -#include -#include -#include -#include - -/* - * We extract 4bit node id (bit 44~47) from Loongson-3's - * 48bit physical address space and embed it into 40bit. - */ - -static int node_id_offset; - -dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) -{ - long nid = (paddr >> 44) & 0xf; - - return ((nid << 44) ^ paddr) | (nid << node_id_offset); -} - -phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) +void acpi_arch_dma_setup(struct device *dev) { - long nid = (daddr >> node_id_offset) & 0xf; + int ret; + u64 mask, end = 0; + const struct bus_dma_region *map = NULL; + + ret = acpi_dma_get_range(dev, &map); + if (!ret && map) { + const struct bus_dma_region *r = map; + + for (end = 0; r->size; r++) { + if (r->dma_start + r->size - 1 > end) + end = r->dma_start + r->size - 1; + } + + mask = DMA_BIT_MASK(ilog2(end) + 1); + dev->bus_dma_limit = end; + dev->dma_range_map = map; + dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); + *dev->dma_mask = min(*dev->dma_mask, mask); + } - return ((nid << node_id_offset) ^ daddr) | (nid << 44); -} - -void __init plat_swiotlb_setup(void) -{ - swiotlb_init(true, SWIOTLB_VERBOSE); - node_id_offset = ((readl(LS7A_DMA_CFG) & LS7A_DMA_NODE_MASK) >> LS7A_DMA_NODE_SHF) + 36; } diff --git a/arch/loongarch/kernel/efi-header.S b/arch/loongarch/kernel/efi-header.S new file mode 100644 index 0000000000000000000000000000000000000000..8c1d229a2afa10b69f2a3f3688c0900ff3399f78 --- /dev/null +++ b/arch/loongarch/kernel/efi-header.S @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include + + .macro __EFI_PE_HEADER + .long PE_MAGIC +.Lcoff_header: + .short IMAGE_FILE_MACHINE_LOONGARCH64 /* Machine */ + .short .Lsection_count /* NumberOfSections */ + .long 0 /* TimeDateStamp */ + .long 0 /* PointerToSymbolTable */ + .long 0 /* NumberOfSymbols */ + .short .Lsection_table - .Loptional_header /* SizeOfOptionalHeader */ + .short IMAGE_FILE_DEBUG_STRIPPED | \ + IMAGE_FILE_EXECUTABLE_IMAGE | \ + IMAGE_FILE_LINE_NUMS_STRIPPED /* Characteristics */ + +.Loptional_header: + .short PE_OPT_MAGIC_PE32PLUS /* PE32+ format */ + .byte 0x02 /* MajorLinkerVersion */ + .byte 0x14 /* MinorLinkerVersion */ + .long __inittext_end - .Lefi_header_end /* SizeOfCode */ + .long _end - __initdata_begin /* SizeOfInitializedData */ + .long 0 /* SizeOfUninitializedData */ + .long __efistub_efi_pe_entry - _head /* AddressOfEntryPoint */ + .long .Lefi_header_end - _head /* BaseOfCode */ + +.Lextra_header_fields: + .quad 0 /* ImageBase */ + .long PECOFF_SEGMENT_ALIGN /* SectionAlignment */ + .long PECOFF_FILE_ALIGN /* FileAlignment */ + .short 0 /* MajorOperatingSystemVersion */ + .short 0 /* MinorOperatingSystemVersion */ + .short LINUX_EFISTUB_MAJOR_VERSION /* MajorImageVersion */ + .short LINUX_EFISTUB_MINOR_VERSION /* MinorImageVersion */ + .short 0 /* MajorSubsystemVersion */ + .short 0 /* MinorSubsystemVersion */ + .long 0 /* Win32VersionValue */ + + .long _end - _head /* SizeOfImage */ + + /* Everything before the kernel image is considered part of the header */ + .long .Lefi_header_end - _head /* SizeOfHeaders */ + .long 0 /* CheckSum */ + .short IMAGE_SUBSYSTEM_EFI_APPLICATION /* Subsystem */ + .short 0 /* DllCharacteristics */ + .quad 0 /* SizeOfStackReserve */ + .quad 0 /* SizeOfStackCommit */ + .quad 0 /* SizeOfHeapReserve */ + .quad 0 /* SizeOfHeapCommit */ + .long 0 /* LoaderFlags */ + .long (.Lsection_table - .) / 8 /* NumberOfRvaAndSizes */ + + .quad 0 /* ExportTable */ + .quad 0 /* ImportTable */ + .quad 0 /* ResourceTable */ + .quad 0 /* ExceptionTable */ + .quad 0 /* CertificationTable */ + .quad 0 /* BaseRelocationTable */ + + /* Section table */ +.Lsection_table: + .ascii ".text\0\0\0" + .long __inittext_end - .Lefi_header_end /* VirtualSize */ + .long .Lefi_header_end - _head /* VirtualAddress */ + .long __inittext_end - .Lefi_header_end /* SizeOfRawData */ + .long .Lefi_header_end - _head /* PointerToRawData */ + + .long 0 /* PointerToRelocations */ + .long 0 /* PointerToLineNumbers */ + .short 0 /* NumberOfRelocations */ + .short 0 /* NumberOfLineNumbers */ + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE /* Characteristics */ + + .ascii ".data\0\0\0" + .long _end - __initdata_begin /* VirtualSize */ + .long __initdata_begin - _head /* VirtualAddress */ + .long _edata - __initdata_begin /* SizeOfRawData */ + .long __initdata_begin - _head /* PointerToRawData */ + + .long 0 /* PointerToRelocations */ + .long 0 /* PointerToLineNumbers */ + .short 0 /* NumberOfRelocations */ + .short 0 /* NumberOfLineNumbers */ + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_WRITE /* Characteristics */ + + .set .Lsection_count, (. - .Lsection_table) / 40 + + .balign 0x10000 /* PECOFF_SEGMENT_ALIGN */ +.Lefi_header_end: + .endm diff --git a/arch/loongarch/kernel/efi.c b/arch/loongarch/kernel/efi.c index a50b60c587fa0fb89a3838fe0e7b7be8767ba51d..a31329971133946d383fc16fc2eaf99d78f0bcbb 100644 --- a/arch/loongarch/kernel/efi.c +++ b/arch/loongarch/kernel/efi.c @@ -27,8 +27,13 @@ static unsigned long efi_nr_tables; static unsigned long efi_config_table; +static unsigned long __initdata boot_memmap = EFI_INVALID_TABLE_ADDR; + static efi_system_table_t *efi_systab; -static efi_config_table_type_t arch_tables[] __initdata = {{},}; +static efi_config_table_type_t arch_tables[] __initdata = { + {LINUX_EFI_BOOT_MEMMAP_GUID, &boot_memmap, "MEMMAP" }, + {}, +}; void __init efi_runtime_init(void) { @@ -51,6 +56,7 @@ void __init efi_init(void) { int size; void *config_tables; + struct efi_boot_memmap *tbl; if (!efi_system_table) return; @@ -61,6 +67,8 @@ void __init efi_init(void) return; } + efi_systab_report_header(&efi_systab->hdr, efi_systab->fw_vendor); + set_bit(EFI_64BIT, &efi.flags); efi_nr_tables = efi_systab->nr_tables; efi_config_table = (unsigned long)efi_systab->tables; @@ -69,4 +77,27 @@ void __init efi_init(void) config_tables = early_memremap(efi_config_table, efi_nr_tables * size); efi_config_parse_tables(config_tables, efi_systab->nr_tables, arch_tables); early_memunmap(config_tables, efi_nr_tables * size); + + set_bit(EFI_CONFIG_TABLES, &efi.flags); + + if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI) + memblock_reserve(screen_info.lfb_base, screen_info.lfb_size); + + if (boot_memmap == EFI_INVALID_TABLE_ADDR) + return; + + tbl = early_memremap_ro(boot_memmap, sizeof(*tbl)); + if (tbl) { + struct efi_memory_map_data data; + + data.phys_map = boot_memmap + sizeof(*tbl); + data.size = tbl->map_size; + data.desc_size = tbl->desc_size; + data.desc_version = tbl->desc_ver; + + if (efi_memmap_init_early(&data) < 0) + panic("Unable to map EFI memory map.\n"); + + early_memunmap(tbl, sizeof(*tbl)); + } } diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 82b478a5c665746e90ed72dc17b00d6c6089807e..6d56a463b091c0f6069380565ea6e0883e7571a6 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -20,21 +19,17 @@ EXPORT_SYMBOL(loongson_sysconf); void __init init_environ(void) { int efi_boot = fw_arg0; - struct efi_memory_map_data data; - void *fdt_ptr = early_memremap_ro(fw_arg1, SZ_64K); + char *cmdline = early_memremap_ro(fw_arg1, COMMAND_LINE_SIZE); if (efi_boot) set_bit(EFI_BOOT, &efi.flags); else clear_bit(EFI_BOOT, &efi.flags); - early_init_dt_scan(fdt_ptr); - early_init_fdt_reserve_self(); - efi_system_table = efi_get_fdt_params(&data); + strscpy(boot_command_line, cmdline, COMMAND_LINE_SIZE); + early_memunmap(cmdline, COMMAND_LINE_SIZE); - efi_memmap_init_early(&data); - memblock_reserve(data.phys_map & PAGE_MASK, - PAGE_ALIGN(data.size + (data.phys_map & ~PAGE_MASK))); + efi_system_table = fw_arg2; } static int __init init_cpu_fullname(void) diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index c60eb66793e351cb0a91114cab9478903b44ebdb..97425779ce9f3499ec27a8ef68b739cc002cc8a6 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -8,12 +8,39 @@ #include #include #include +#include #include #include #include +#ifdef CONFIG_EFI_STUB + +#include "efi-header.S" + + __HEAD + +_head: + .word MZ_MAGIC /* "MZ", MS-DOS header */ + .org 0x8 + .dword kernel_entry /* Kernel entry point */ + .dword _end - _text /* Kernel image effective size */ + .quad 0 /* Kernel image load offset from start of RAM */ + .org 0x3c /* 0x20 ~ 0x3b reserved */ + .long pe_header - _head /* Offset to the PE header */ + +pe_header: + __EFI_PE_HEADER + +SYM_DATA(kernel_asize, .long _end - _text); +SYM_DATA(kernel_fsize, .long _edata - _text); +SYM_DATA(kernel_offset, .long kernel_offset - _text); + +#endif + __REF + .align 12 + SYM_CODE_START(kernel_entry) # kernel entry point /* Config direct window and set PG */ @@ -35,25 +62,27 @@ SYM_CODE_START(kernel_entry) # kernel entry point li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0 csrwr t0, LOONGARCH_CSR_EUEN - la t0, __bss_start # clear .bss + la.pcrel t0, __bss_start # clear .bss st.d zero, t0, 0 - la t1, __bss_stop - LONGSIZE + la.pcrel t1, __bss_stop - LONGSIZE 1: addi.d t0, t0, LONGSIZE st.d zero, t0, 0 bne t0, t1, 1b - la t0, fw_arg0 + la.pcrel t0, fw_arg0 st.d a0, t0, 0 # firmware arguments - la t0, fw_arg1 + la.pcrel t0, fw_arg1 st.d a1, t0, 0 + la.pcrel t0, fw_arg2 + st.d a2, t0, 0 /* KSave3 used for percpu base, initialized as 0 */ csrwr zero, PERCPU_BASE_KS /* GPR21 used for percpu base (runtime), initialized as 0 */ move u0, zero - la tp, init_thread_union + la.pcrel tp, init_thread_union /* Set the SP after an empty pt_regs. */ PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE) PTR_ADD sp, sp, tp @@ -61,6 +90,7 @@ SYM_CODE_START(kernel_entry) # kernel entry point PTR_ADDI sp, sp, -4 * SZREG # init stack pointer bl start_kernel + ASM_BUG() SYM_CODE_END(kernel_entry) @@ -92,6 +122,8 @@ SYM_CODE_START(smpboot_entry) ld.d tp, t0, CPU_BOOT_TINFO bl start_secondary + ASM_BUG() + SYM_CODE_END(smpboot_entry) #endif /* CONFIG_SMP */ diff --git a/arch/loongarch/kernel/image-vars.h b/arch/loongarch/kernel/image-vars.h new file mode 100644 index 0000000000000000000000000000000000000000..88f5d81702dfcf688dd44defac9a1ebd194a6c66 --- /dev/null +++ b/arch/loongarch/kernel/image-vars.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#ifndef __LOONGARCH_KERNEL_IMAGE_VARS_H +#define __LOONGARCH_KERNEL_IMAGE_VARS_H + +#ifdef CONFIG_EFI_STUB + +__efistub_memcmp = memcmp; +__efistub_memchr = memchr; +__efistub_strcat = strcat; +__efistub_strcmp = strcmp; +__efistub_strlen = strlen; +__efistub_strncat = strncat; +__efistub_strnstr = strnstr; +__efistub_strnlen = strnlen; +__efistub_strrchr = strrchr; +__efistub_kernel_entry = kernel_entry; +__efistub_kernel_asize = kernel_asize; +__efistub_kernel_fsize = kernel_fsize; +__efistub_kernel_offset = kernel_offset; +__efistub_screen_info = screen_info; + +#endif + +#endif /* __LOONGARCH_KERNEL_IMAGE_VARS_H */ diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c new file mode 100644 index 0000000000000000000000000000000000000000..2dcb9e003657c848adff71078870fe682451e416 --- /dev/null +++ b/arch/loongarch/kernel/machine_kexec.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * machine_kexec.c for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* 0x100000 ~ 0x200000 is safe */ +#define KEXEC_CONTROL_CODE TO_CACHE(0x100000UL) +#define KEXEC_CMDLINE_ADDR TO_CACHE(0x108000UL) + +static unsigned long reboot_code_buffer; +static cpumask_t cpus_in_crash = CPU_MASK_NONE; + +#ifdef CONFIG_SMP +static void (*relocated_kexec_smp_wait)(void *); +atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); +#endif + +static unsigned long efi_boot; +static unsigned long cmdline_ptr; +static unsigned long systable_ptr; +static unsigned long start_addr; +static unsigned long first_ind_entry; + +static void kexec_image_info(const struct kimage *kimage) +{ + unsigned long i; + + pr_debug("kexec kimage info:\n"); + pr_debug("\ttype: %d\n", kimage->type); + pr_debug("\tstart: %lx\n", kimage->start); + pr_debug("\thead: %lx\n", kimage->head); + pr_debug("\tnr_segments: %lu\n", kimage->nr_segments); + + for (i = 0; i < kimage->nr_segments; i++) { + pr_debug("\t segment[%lu]: %016lx - %016lx", i, + kimage->segment[i].mem, + kimage->segment[i].mem + kimage->segment[i].memsz); + pr_debug("\t\t0x%lx bytes, %lu pages\n", + (unsigned long)kimage->segment[i].memsz, + (unsigned long)kimage->segment[i].memsz / PAGE_SIZE); + } +} + +int machine_kexec_prepare(struct kimage *kimage) +{ + int i; + char *bootloader = "kexec"; + void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; + + kexec_image_info(kimage); + + kimage->arch.efi_boot = fw_arg0; + kimage->arch.systable_ptr = fw_arg2; + + /* Find the command line */ + for (i = 0; i < kimage->nr_segments; i++) { + if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { + if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) + kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; + break; + } + } + + if (!kimage->arch.cmdline_ptr) { + pr_err("Command line not included in the provided image\n"); + return -EINVAL; + } + + /* kexec/kdump need a safe page to save reboot_code_buffer */ + kimage->control_code_page = virt_to_page((void *)KEXEC_CONTROL_CODE); + + reboot_code_buffer = (unsigned long)page_address(kimage->control_code_page); + memcpy((void *)reboot_code_buffer, relocate_new_kernel, relocate_new_kernel_size); + +#ifdef CONFIG_SMP + /* All secondary cpus now may jump to kexec_smp_wait cycle */ + relocated_kexec_smp_wait = reboot_code_buffer + (void *)(kexec_smp_wait - relocate_new_kernel); +#endif + + return 0; +} + +void machine_kexec_cleanup(struct kimage *kimage) +{ +} + +void kexec_reboot(void) +{ + do_kexec_t do_kexec = NULL; + + /* + * We know we were online, and there will be no incoming IPIs at + * this point. Mark online again before rebooting so that the crash + * analysis tool will see us correctly. + */ + set_cpu_online(smp_processor_id(), true); + + /* Ensure remote CPUs observe that we're online before rebooting. */ + smp_mb__after_atomic(); + + /* + * Make sure we get correct instructions written by the + * machine_kexec_prepare() CPU. + */ + __asm__ __volatile__ ("\tibar 0\n"::); + +#ifdef CONFIG_SMP + /* All secondary cpus go to kexec_smp_wait */ + if (smp_processor_id() > 0) { + relocated_kexec_smp_wait(NULL); + unreachable(); + } +#endif + + do_kexec = (void *)reboot_code_buffer; + do_kexec(efi_boot, cmdline_ptr, systable_ptr, start_addr, first_ind_entry); + + unreachable(); +} + + +#ifdef CONFIG_SMP +static void kexec_shutdown_secondary(void *regs) +{ + int cpu = smp_processor_id(); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} + +static void crash_shutdown_secondary(void *passed_regs) +{ + int cpu = smp_processor_id(); + struct pt_regs *regs = passed_regs; + + /* + * If we are passed registers, use those. Otherwise get the + * regs from the last interrupt, which should be correct, as + * we are in an interrupt. But if the regs are not there, + * pull them from the top of the stack. They are probably + * wrong, but we need something to keep from crashing again. + */ + if (!regs) + regs = get_irq_regs(); + if (!regs) + regs = task_pt_regs(current); + + if (!cpu_online(cpu)) + return; + + /* We won't be sent IPIs any more. */ + set_cpu_online(cpu, false); + + local_irq_disable(); + if (!cpumask_test_cpu(cpu, &cpus_in_crash)) + crash_save_cpu(regs, cpu); + cpumask_set_cpu(cpu, &cpus_in_crash); + + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + + kexec_reboot(); +} + +void crash_smp_send_stop(void) +{ + unsigned int ncpus; + unsigned long timeout; + static int cpus_stopped; + + /* + * This function can be called twice in panic path, but obviously + * we should execute this only once. + */ + if (cpus_stopped) + return; + + cpus_stopped = 1; + + /* Excluding the panic cpu */ + ncpus = num_online_cpus() - 1; + + smp_call_function(crash_shutdown_secondary, NULL, 0); + smp_wmb(); + + /* + * The crash CPU sends an IPI and wait for other CPUs to + * respond. Delay of at least 10 seconds. + */ + timeout = MSEC_PER_SEC * 10; + pr_emerg("Sending IPI to other cpus...\n"); + while ((cpumask_weight(&cpus_in_crash) < ncpus) && timeout--) { + mdelay(1); + cpu_relax(); + } +} +#endif /* defined(CONFIG_SMP) */ + +void machine_shutdown(void) +{ + int cpu; + + /* All CPUs go to reboot_code_buffer */ + for_each_possible_cpu(cpu) + if (!cpu_online(cpu)) + cpu_device_up(get_cpu_device(cpu)); + +#ifdef CONFIG_SMP + smp_call_function(kexec_shutdown_secondary, NULL, 0); +#endif +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + int crashing_cpu; + + local_irq_disable(); + + crashing_cpu = smp_processor_id(); + crash_save_cpu(regs, crashing_cpu); + +#ifdef CONFIG_SMP + crash_smp_send_stop(); +#endif + cpumask_set_cpu(crashing_cpu, &cpus_in_crash); + + pr_info("Starting crashdump kernel...\n"); +} + +void machine_kexec(struct kimage *image) +{ + unsigned long entry, *ptr; + struct kimage_arch *internal = &image->arch; + + efi_boot = internal->efi_boot; + cmdline_ptr = internal->cmdline_ptr; + systable_ptr = internal->systable_ptr; + + start_addr = (unsigned long)phys_to_virt(image->start); + + first_ind_entry = (image->type == KEXEC_TYPE_DEFAULT) ? + (unsigned long)phys_to_virt(image->head & PAGE_MASK) : 0; + + /* + * The generic kexec code builds a page list with physical + * addresses. they are directly accessible through XKPRANGE + * hence the phys_to_virt() call. + */ + for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); + ptr = (entry & IND_INDIRECTION) ? + phys_to_virt(entry & PAGE_MASK) : ptr + 1) { + if (*ptr & IND_SOURCE || *ptr & IND_INDIRECTION || + *ptr & IND_DESTINATION) + *ptr = (unsigned long) phys_to_virt(*ptr); + } + + /* Mark offline before disabling local irq. */ + set_cpu_online(smp_processor_id(), false); + + /* We do not want to be bothered. */ + local_irq_disable(); + + pr_notice("EFI boot flag 0x%lx\n", efi_boot); + pr_notice("Command line at 0x%lx\n", cmdline_ptr); + pr_notice("System table at 0x%lx\n", systable_ptr); + pr_notice("We will call new kernel at 0x%lx\n", start_addr); + pr_notice("Bye ...\n"); + + /* Make reboot code buffer available to the boot CPU. */ + flush_cache_all(); + +#ifdef CONFIG_SMP + atomic_set(&kexec_ready_to_reboot, 1); +#endif + + kexec_reboot(); +} diff --git a/arch/loongarch/kernel/mem.c b/arch/loongarch/kernel/mem.c index 7423361b0ebc9b69864b9cec624b48081d20ee37..4a4107a6a9651535fc6cfb72f21cc1fb64f7eeed 100644 --- a/arch/loongarch/kernel/mem.c +++ b/arch/loongarch/kernel/mem.c @@ -58,7 +58,4 @@ void __init memblock_init(void) /* Reserve the kernel text/data/bss */ memblock_reserve(__pa_symbol(&_text), __pa_symbol(&_end) - __pa_symbol(&_text)); - - /* Reserve the initrd */ - reserve_initrd_mem(); } diff --git a/arch/loongarch/kernel/module-sections.c b/arch/loongarch/kernel/module-sections.c index 6d498288977d2a44df89dc979fad84e3571b0066..d296a70b758fd60a5963d1298bbd1189e645c93c 100644 --- a/arch/loongarch/kernel/module-sections.c +++ b/arch/loongarch/kernel/module-sections.c @@ -7,7 +7,33 @@ #include #include -Elf_Addr module_emit_plt_entry(struct module *mod, unsigned long val) +Elf_Addr module_emit_got_entry(struct module *mod, Elf_Addr val) +{ + struct mod_section *got_sec = &mod->arch.got; + int i = got_sec->num_entries; + struct got_entry *got = get_got_entry(val, got_sec); + + if (got) + return (Elf_Addr)got; + + /* There is no GOT entry for val yet, create a new one. */ + got = (struct got_entry *)got_sec->shdr->sh_addr; + got[i] = emit_got_entry(val); + + got_sec->num_entries++; + if (got_sec->num_entries > got_sec->max_entries) { + /* + * This may happen when the module contains a GOT_HI20 without + * a paired GOT_LO12. Such a module is broken, reject it. + */ + pr_err("%s: module contains bad GOT relocation\n", mod->name); + return 0; + } + + return (Elf_Addr)&got[i]; +} + +Elf_Addr module_emit_plt_entry(struct module *mod, Elf_Addr val) { int nr; struct mod_section *plt_sec = &mod->arch.plt; @@ -50,15 +76,25 @@ static bool duplicate_rela(const Elf_Rela *rela, int idx) return false; } -static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts) +static void count_max_entries(Elf_Rela *relas, int num, + unsigned int *plts, unsigned int *gots) { unsigned int i, type; for (i = 0; i < num; i++) { type = ELF_R_TYPE(relas[i].r_info); - if (type == R_LARCH_SOP_PUSH_PLT_PCREL) { + switch (type) { + case R_LARCH_SOP_PUSH_PLT_PCREL: + case R_LARCH_B26: if (!duplicate_rela(relas, i)) (*plts)++; + break; + case R_LARCH_GOT_PC_HI20: + if (!duplicate_rela(relas, i)) + (*gots)++; + break; + default: + break; /* Do nothing. */ } } } @@ -66,18 +102,24 @@ static void count_max_entries(Elf_Rela *relas, int num, unsigned int *plts) int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, char *secstrings, struct module *mod) { - unsigned int i, num_plts = 0; + unsigned int i, num_plts = 0, num_gots = 0; /* * Find the empty .plt sections. */ for (i = 0; i < ehdr->e_shnum; i++) { - if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt")) + if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) + mod->arch.got.shdr = sechdrs + i; + else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt")) mod->arch.plt.shdr = sechdrs + i; else if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt.idx")) mod->arch.plt_idx.shdr = sechdrs + i; } + if (!mod->arch.got.shdr) { + pr_err("%s: module GOT section(s) missing\n", mod->name); + return -ENOEXEC; + } if (!mod->arch.plt.shdr) { pr_err("%s: module PLT section(s) missing\n", mod->name); return -ENOEXEC; @@ -100,9 +142,16 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs, if (!(dst_sec->sh_flags & SHF_EXECINSTR)) continue; - count_max_entries(relas, num_rela, &num_plts); + count_max_entries(relas, num_rela, &num_plts, &num_gots); } + mod->arch.got.shdr->sh_type = SHT_NOBITS; + mod->arch.got.shdr->sh_flags = SHF_ALLOC; + mod->arch.got.shdr->sh_addralign = L1_CACHE_BYTES; + mod->arch.got.shdr->sh_size = (num_gots + 1) * sizeof(struct got_entry); + mod->arch.got.num_entries = 0; + mod->arch.got.max_entries = num_gots; + mod->arch.plt.shdr->sh_type = SHT_NOBITS; mod->arch.plt.shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC; mod->arch.plt.shdr->sh_addralign = L1_CACHE_BYTES; diff --git a/arch/loongarch/kernel/module.c b/arch/loongarch/kernel/module.c index 638427ff0d5150cbffacef7d9ac75c0f10b661b8..097595b2fc14bce0fc0ee059a855e71c7233eca0 100644 --- a/arch/loongarch/kernel/module.c +++ b/arch/loongarch/kernel/module.c @@ -18,16 +18,6 @@ #include #include -static inline bool signed_imm_check(long val, unsigned int bit) -{ - return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1)); -} - -static inline bool unsigned_imm_check(unsigned long val, unsigned int bit) -{ - return val < (1UL << bit); -} - static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) { if (*rela_stack_top >= RELA_STACK_DEPTH) @@ -281,6 +271,96 @@ static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, } } +static int apply_r_larch_b26(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + ptrdiff_t offset = (void *)v - (void *)location; + union loongarch_instruction *insn = (union loongarch_instruction *)location; + + if (offset >= SZ_128M) + v = module_emit_plt_entry(mod, v); + + if (offset < -SZ_128M) + v = module_emit_plt_entry(mod, v); + + offset = (void *)v - (void *)location; + + if (offset & 3) { + pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", + mod->name, (long long)offset, type); + return -ENOEXEC; + } + + if (!signed_imm_check(offset, 28)) { + pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", + mod->name, (long long)offset, type); + return -ENOEXEC; + } + + offset >>= 2; + insn->reg0i26_format.immediate_l = offset & 0xffff; + insn->reg0i26_format.immediate_h = (offset >> 16) & 0x3ff; + + return 0; +} + +static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + union loongarch_instruction *insn = (union loongarch_instruction *)location; + /* Use s32 for a sign-extension deliberately. */ + s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - + (void *)((Elf_Addr)location & ~0xfff); + Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; + ptrdiff_t offset_rem = (void *)v - (void *)anchor; + + switch (type) { + case R_LARCH_PCALA_LO12: + insn->reg2i12_format.immediate = v & 0xfff; + break; + case R_LARCH_PCALA_HI20: + v = offset_hi20 >> 12; + insn->reg1i20_format.immediate = v & 0xfffff; + break; + case R_LARCH_PCALA64_LO20: + v = offset_rem >> 32; + insn->reg1i20_format.immediate = v & 0xfffff; + break; + case R_LARCH_PCALA64_HI12: + v = offset_rem >> 52; + insn->reg2i12_format.immediate = v & 0xfff; + break; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + + return 0; +} + +static int apply_r_larch_got_pc(struct module *mod, u32 *location, Elf_Addr v, + s64 *rela_stack, size_t *rela_stack_top, unsigned int type) +{ + Elf_Addr got = module_emit_got_entry(mod, v); + + if (!got) + return -EINVAL; + + switch (type) { + case R_LARCH_GOT_PC_LO12: + type = R_LARCH_PCALA_LO12; + break; + case R_LARCH_GOT_PC_HI20: + type = R_LARCH_PCALA_HI20; + break; + default: + pr_err("%s: Unsupport relocation type %u\n", mod->name, type); + return -EINVAL; + } + + return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); +} + /* * reloc_handlers_rela() - Apply a particular relocation to a module * @mod: the module to apply the reloc to @@ -296,7 +376,7 @@ typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, /* The handlers for known reloc types */ static reloc_rela_handler reloc_rela_handlers[] = { - [R_LARCH_NONE ... R_LARCH_SUB64] = apply_r_larch_error, + [R_LARCH_NONE ... R_LARCH_RELAX] = apply_r_larch_error, [R_LARCH_NONE] = apply_r_larch_none, [R_LARCH_32] = apply_r_larch_32, @@ -310,6 +390,9 @@ static reloc_rela_handler reloc_rela_handlers[] = { [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, + [R_LARCH_B26] = apply_r_larch_b26, + [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, + [R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12] = apply_r_larch_got_pc, }; int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, diff --git a/arch/loongarch/kernel/perf_event.c b/arch/loongarch/kernel/perf_event.c new file mode 100644 index 0000000000000000000000000000000000000000..707bd32e5c4ff84b415a22fdb8701df7266e7f33 --- /dev/null +++ b/arch/loongarch/kernel/perf_event.c @@ -0,0 +1,887 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Linux performance counter support for LoongArch. + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 2010 MIPS Technologies, Inc. + * Copyright (C) 2011 Cavium Networks, Inc. + * Author: Deng-Cheng Zhu + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static unsigned long +user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) +{ + unsigned long err; + unsigned long __user *user_frame_tail; + struct stack_frame buftail; + + user_frame_tail = (unsigned long __user *)(fp - sizeof(struct stack_frame)); + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(user_frame_tail, sizeof(buftail))) + return 0; + + pagefault_disable(); + err = __copy_from_user_inatomic(&buftail, user_frame_tail, sizeof(buftail)); + pagefault_enable(); + + if (err || (unsigned long)user_frame_tail >= buftail.fp) + return 0; + + perf_callchain_store(entry, buftail.ra); + + return buftail.fp; +} + +void perf_callchain_user(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long fp; + + if (perf_guest_state()) { + /* We don't support guest os callchain now */ + return; + } + + perf_callchain_store(entry, regs->csr_era); + + fp = regs->regs[22]; + + while (entry->nr < entry->max_stack && fp && !((unsigned long)fp & 0xf)) + fp = user_backtrace(entry, fp); +} + +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + struct unwind_state state; + unsigned long addr; + + for (unwind_start(&state, current, regs); + !unwind_done(&state); unwind_next_frame(&state)) { + addr = unwind_get_return_address(&state); + if (!addr || perf_callchain_store(entry, addr)) + return; + } +} + +#define LOONGARCH_MAX_HWEVENTS 32 + +struct cpu_hw_events { + /* Array of events on this cpu. */ + struct perf_event *events[LOONGARCH_MAX_HWEVENTS]; + + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(LOONGARCH_MAX_HWEVENTS)]; + + /* + * Software copy of the control register for each performance counter. + */ + unsigned int saved_ctrl[LOONGARCH_MAX_HWEVENTS]; +}; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { + .saved_ctrl = {0}, +}; + +/* The description of LoongArch performance events. */ +struct loongarch_perf_event { + unsigned int event_id; +}; + +static struct loongarch_perf_event raw_event; +static DEFINE_MUTEX(raw_event_mutex); + +#define C(x) PERF_COUNT_HW_CACHE_##x +#define HW_OP_UNSUPPORTED 0xffffffff +#define CACHE_OP_UNSUPPORTED 0xffffffff + +#define PERF_MAP_ALL_UNSUPPORTED \ + [0 ... PERF_COUNT_HW_MAX - 1] = {HW_OP_UNSUPPORTED} + +#define PERF_CACHE_MAP_ALL_UNSUPPORTED \ +[0 ... C(MAX) - 1] = { \ + [0 ... C(OP_MAX) - 1] = { \ + [0 ... C(RESULT_MAX) - 1] = {CACHE_OP_UNSUPPORTED}, \ + }, \ +} + +struct loongarch_pmu { + u64 max_period; + u64 valid_count; + u64 overflow; + const char *name; + unsigned int num_counters; + u64 (*read_counter)(unsigned int idx); + void (*write_counter)(unsigned int idx, u64 val); + const struct loongarch_perf_event *(*map_raw_event)(u64 config); + const struct loongarch_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; + const struct loongarch_perf_event (*cache_event_map) + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; +}; + +static struct loongarch_pmu loongarch_pmu; + +#define M_PERFCTL_EVENT(event) (event & CSR_PERFCTRL_EVENT) + +#define M_PERFCTL_COUNT_EVENT_WHENEVER (CSR_PERFCTRL_PLV0 | \ + CSR_PERFCTRL_PLV1 | \ + CSR_PERFCTRL_PLV2 | \ + CSR_PERFCTRL_PLV3 | \ + CSR_PERFCTRL_IE) + +#define M_PERFCTL_CONFIG_MASK 0x1f0000 + +static void pause_local_counters(void); +static void resume_local_counters(void); + +static u64 loongarch_pmu_read_counter(unsigned int idx) +{ + u64 val = -1; + + switch (idx) { + case 0: + val = read_csr_perfcntr0(); + break; + case 1: + val = read_csr_perfcntr1(); + break; + case 2: + val = read_csr_perfcntr2(); + break; + case 3: + val = read_csr_perfcntr3(); + break; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } + + return val; +} + +static void loongarch_pmu_write_counter(unsigned int idx, u64 val) +{ + switch (idx) { + case 0: + write_csr_perfcntr0(val); + return; + case 1: + write_csr_perfcntr1(val); + return; + case 2: + write_csr_perfcntr2(val); + return; + case 3: + write_csr_perfcntr3(val); + return; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return; + } +} + +static unsigned int loongarch_pmu_read_control(unsigned int idx) +{ + unsigned int val = -1; + + switch (idx) { + case 0: + val = read_csr_perfctrl0(); + break; + case 1: + val = read_csr_perfctrl1(); + break; + case 2: + val = read_csr_perfctrl2(); + break; + case 3: + val = read_csr_perfctrl3(); + break; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; + } + + return val; +} + +static void loongarch_pmu_write_control(unsigned int idx, unsigned int val) +{ + switch (idx) { + case 0: + write_csr_perfctrl0(val); + return; + case 1: + write_csr_perfctrl1(val); + return; + case 2: + write_csr_perfctrl2(val); + return; + case 3: + write_csr_perfctrl3(val); + return; + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return; + } +} + +static int loongarch_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc) +{ + int i; + + for (i = 0; i < loongarch_pmu.num_counters; i++) { + if (!test_and_set_bit(i, cpuc->used_mask)) + return i; + } + + return -EAGAIN; +} + +static void loongarch_pmu_enable_event(struct hw_perf_event *evt, int idx) +{ + unsigned int cpu; + struct perf_event *event = container_of(evt, struct perf_event, hw); + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + /* Make sure interrupt enabled. */ + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | CSR_PERFCTRL_IE; + + cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id(); + + /* + * We do not actually let the counter run. Leave it until start(). + */ + pr_debug("Enabling perf counter for CPU%d\n", cpu); +} + +static void loongarch_pmu_disable_event(int idx) +{ + unsigned long flags; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + local_irq_save(flags); + cpuc->saved_ctrl[idx] = loongarch_pmu_read_control(idx) & + ~M_PERFCTL_COUNT_EVENT_WHENEVER; + loongarch_pmu_write_control(idx, cpuc->saved_ctrl[idx]); + local_irq_restore(flags); +} + +static int loongarch_pmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + int ret = 0; + u64 left = local64_read(&hwc->period_left); + u64 period = hwc->sample_period; + + if (unlikely((left + period) & (1ULL << 63))) { + /* left underflowed by more than period. */ + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } else if (unlikely((left + period) <= period)) { + /* left underflowed by less than period. */ + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (left > loongarch_pmu.max_period) { + left = loongarch_pmu.max_period; + local64_set(&hwc->period_left, left); + } + + local64_set(&hwc->prev_count, loongarch_pmu.overflow - left); + + loongarch_pmu.write_counter(idx, loongarch_pmu.overflow - left); + + perf_event_update_userpage(event); + + return ret; +} + +static void loongarch_pmu_event_update(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + u64 delta; + u64 prev_raw_count, new_raw_count; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = loongarch_pmu.read_counter(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = new_raw_count - prev_raw_count; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); +} + +static void loongarch_pmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + + hwc->state = 0; + + /* Set the period for the event. */ + loongarch_pmu_event_set_period(event, hwc, hwc->idx); + + /* Enable the event. */ + loongarch_pmu_enable_event(hwc, hwc->idx); +} + +static void loongarch_pmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!(hwc->state & PERF_HES_STOPPED)) { + /* We are working on a local event. */ + loongarch_pmu_disable_event(hwc->idx); + barrier(); + loongarch_pmu_event_update(event, hwc, hwc->idx); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; + } +} + +static int loongarch_pmu_add(struct perf_event *event, int flags) +{ + int idx, err = 0; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + + perf_pmu_disable(event->pmu); + + /* To look for a free counter for this event. */ + idx = loongarch_pmu_alloc_counter(cpuc, hwc); + if (idx < 0) { + err = idx; + goto out; + } + + /* + * If there is an event in the counter we are going to use then + * make sure it is disabled. + */ + event->hw.idx = idx; + loongarch_pmu_disable_event(idx); + cpuc->events[idx] = event; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + loongarch_pmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + perf_pmu_enable(event->pmu); + return err; +} + +static void loongarch_pmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); + + loongarch_pmu_stop(event, PERF_EF_UPDATE); + cpuc->events[idx] = NULL; + clear_bit(idx, cpuc->used_mask); + + perf_event_update_userpage(event); +} + +static void loongarch_pmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* Don't read disabled counters! */ + if (hwc->idx < 0) + return; + + loongarch_pmu_event_update(event, hwc, hwc->idx); +} + +static void loongarch_pmu_enable(struct pmu *pmu) +{ + resume_local_counters(); +} + +static void loongarch_pmu_disable(struct pmu *pmu) +{ + pause_local_counters(); +} + +static DEFINE_MUTEX(pmu_reserve_mutex); +static atomic_t active_events = ATOMIC_INIT(0); + +static int get_pmc_irq(void) +{ + struct irq_domain *d = irq_find_matching_fwnode(cpuintc_handle, DOMAIN_BUS_ANY); + + if (d) + return irq_create_mapping(d, EXCCODE_PMC - EXCCODE_INT_START); + + return -EINVAL; +} + +static void reset_counters(void *arg); +static int __hw_perf_event_init(struct perf_event *event); + +static void hw_perf_event_destroy(struct perf_event *event) +{ + if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) { + on_each_cpu(reset_counters, NULL, 1); + free_irq(get_pmc_irq(), &loongarch_pmu); + mutex_unlock(&pmu_reserve_mutex); + } +} + +static void handle_associated_event(struct cpu_hw_events *cpuc, int idx, + struct perf_sample_data *data, struct pt_regs *regs) +{ + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc = &event->hw; + + loongarch_pmu_event_update(event, hwc, idx); + data->period = event->hw.last_period; + if (!loongarch_pmu_event_set_period(event, hwc, idx)) + return; + + if (perf_event_overflow(event, data, regs)) + loongarch_pmu_disable_event(idx); +} + +static irqreturn_t pmu_handle_irq(int irq, void *dev) +{ + int n; + int handled = IRQ_NONE; + uint64_t counter; + struct pt_regs *regs; + struct perf_sample_data data; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + /* + * First we pause the local counters, so that when we are locked + * here, the counters are all paused. When it gets locked due to + * perf_disable(), the timer interrupt handler will be delayed. + * + * See also loongarch_pmu_start(). + */ + pause_local_counters(); + + regs = get_irq_regs(); + + perf_sample_data_init(&data, 0, 0); + + for (n = 0; n < loongarch_pmu.num_counters; n++) { + if (test_bit(n, cpuc->used_mask)) { + counter = loongarch_pmu.read_counter(n); + if (counter & loongarch_pmu.overflow) { + handle_associated_event(cpuc, n, &data, regs); + handled = IRQ_HANDLED; + } + } + } + + resume_local_counters(); + + /* + * Do all the work for the pending perf events. We can do this + * in here because the performance counter interrupt is a regular + * interrupt, not NMI. + */ + if (handled == IRQ_HANDLED) + irq_work_run(); + + return handled; +} + +static int loongarch_pmu_event_init(struct perf_event *event) +{ + int r, irq; + unsigned long flags; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + + default: + /* Init it to avoid false validate_group */ + event->hw.event_base = 0xffffffff; + return -ENOENT; + } + + if (event->cpu >= 0 && !cpu_online(event->cpu)) + return -ENODEV; + + irq = get_pmc_irq(); + flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_SUSPEND | IRQF_SHARED; + if (!atomic_inc_not_zero(&active_events)) { + mutex_lock(&pmu_reserve_mutex); + if (atomic_read(&active_events) == 0) { + r = request_irq(irq, pmu_handle_irq, flags, "Perf_PMU", &loongarch_pmu); + if (r < 0) { + mutex_unlock(&pmu_reserve_mutex); + pr_warn("PMU IRQ request failed\n"); + return -ENODEV; + } + } + atomic_inc(&active_events); + mutex_unlock(&pmu_reserve_mutex); + } + + return __hw_perf_event_init(event); +} + +static struct pmu pmu = { + .pmu_enable = loongarch_pmu_enable, + .pmu_disable = loongarch_pmu_disable, + .event_init = loongarch_pmu_event_init, + .add = loongarch_pmu_add, + .del = loongarch_pmu_del, + .start = loongarch_pmu_start, + .stop = loongarch_pmu_stop, + .read = loongarch_pmu_read, +}; + +static unsigned int loongarch_pmu_perf_event_encode(const struct loongarch_perf_event *pev) +{ + return (pev->event_id & 0xff); +} + +static const struct loongarch_perf_event *loongarch_pmu_map_general_event(int idx) +{ + const struct loongarch_perf_event *pev; + + pev = &(*loongarch_pmu.general_event_map)[idx]; + + if (pev->event_id == HW_OP_UNSUPPORTED) + return ERR_PTR(-ENOENT); + + return pev; +} + +static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct loongarch_perf_event *pev; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + pev = &((*loongarch_pmu.cache_event_map) + [cache_type] + [cache_op] + [cache_result]); + + if (pev->event_id == CACHE_OP_UNSUPPORTED) + return ERR_PTR(-ENOENT); + + return pev; +} + +static int validate_group(struct perf_event *event) +{ + struct cpu_hw_events fake_cpuc; + struct perf_event *sibling, *leader = event->group_leader; + + memset(&fake_cpuc, 0, sizeof(fake_cpuc)); + + if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) + return -EINVAL; + + for_each_sibling_event(sibling, leader) { + if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) + return -EINVAL; + } + + if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) + return -EINVAL; + + return 0; +} + +static void reset_counters(void *arg) +{ + int n; + int counters = loongarch_pmu.num_counters; + + for (n = 0; n < counters; n++) { + loongarch_pmu_write_control(n, 0); + loongarch_pmu.write_counter(n, 0); + } +} + +static const struct loongarch_perf_event loongson_event_map[PERF_COUNT_HW_MAX] = { + PERF_MAP_ALL_UNSUPPORTED, + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00 }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01 }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x08 }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x09 }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02 }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x03 }, +}; + +static const struct loongarch_perf_event loongson_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +PERF_CACHE_MAP_ALL_UNSUPPORTED, +[C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x8 }, + [C(RESULT_MISS)] = { 0x9 }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x8 }, + [C(RESULT_MISS)] = { 0x9 }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0xaa }, + [C(RESULT_MISS)] = { 0xa9 }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x6 }, + [C(RESULT_MISS)] = { 0x7 }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0xc }, + [C(RESULT_MISS)] = { 0xd }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0xc }, + [C(RESULT_MISS)] = { 0xd }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x3b }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x4 }, + [C(RESULT_MISS)] = { 0x3c }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x4 }, + [C(RESULT_MISS)] = { 0x3c }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x02 }, + [C(RESULT_MISS)] = { 0x03 }, + }, +}, +}; + +static int __hw_perf_event_init(struct perf_event *event) +{ + int err; + struct hw_perf_event *hwc = &event->hw; + struct perf_event_attr *attr = &event->attr; + const struct loongarch_perf_event *pev; + + /* Returning LoongArch event descriptor for generic perf event. */ + if (PERF_TYPE_HARDWARE == event->attr.type) { + if (event->attr.config >= PERF_COUNT_HW_MAX) + return -EINVAL; + pev = loongarch_pmu_map_general_event(event->attr.config); + } else if (PERF_TYPE_HW_CACHE == event->attr.type) { + pev = loongarch_pmu_map_cache_event(event->attr.config); + } else if (PERF_TYPE_RAW == event->attr.type) { + /* We are working on the global raw event. */ + mutex_lock(&raw_event_mutex); + pev = loongarch_pmu.map_raw_event(event->attr.config); + } else { + /* The event type is not (yet) supported. */ + return -EOPNOTSUPP; + } + + if (IS_ERR(pev)) { + if (PERF_TYPE_RAW == event->attr.type) + mutex_unlock(&raw_event_mutex); + return PTR_ERR(pev); + } + + /* + * We allow max flexibility on how each individual counter shared + * by the single CPU operates (the mode exclusion and the range). + */ + hwc->config_base = CSR_PERFCTRL_IE; + + hwc->event_base = loongarch_pmu_perf_event_encode(pev); + if (PERF_TYPE_RAW == event->attr.type) + mutex_unlock(&raw_event_mutex); + + if (!attr->exclude_user) { + hwc->config_base |= CSR_PERFCTRL_PLV3; + hwc->config_base |= CSR_PERFCTRL_PLV2; + } + if (!attr->exclude_kernel) { + hwc->config_base |= CSR_PERFCTRL_PLV0; + } + if (!attr->exclude_hv) { + hwc->config_base |= CSR_PERFCTRL_PLV1; + } + + hwc->config_base &= M_PERFCTL_CONFIG_MASK; + /* + * The event can belong to another cpu. We do not assign a local + * counter for it for now. + */ + hwc->idx = -1; + hwc->config = 0; + + if (!hwc->sample_period) { + hwc->sample_period = loongarch_pmu.max_period; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + err = 0; + if (event->group_leader != event) + err = validate_group(event); + + event->destroy = hw_perf_event_destroy; + + if (err) + event->destroy(event); + + return err; +} + +static void pause_local_counters(void) +{ + unsigned long flags; + int ctr = loongarch_pmu.num_counters; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + local_irq_save(flags); + do { + ctr--; + cpuc->saved_ctrl[ctr] = loongarch_pmu_read_control(ctr); + loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] & + ~M_PERFCTL_COUNT_EVENT_WHENEVER); + } while (ctr > 0); + local_irq_restore(flags); +} + +static void resume_local_counters(void) +{ + int ctr = loongarch_pmu.num_counters; + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + do { + ctr--; + loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]); + } while (ctr > 0); +} + +static const struct loongarch_perf_event *loongarch_pmu_map_raw_event(u64 config) +{ + raw_event.event_id = config & 0xff; + + return &raw_event; +} + +static int __init init_hw_perf_events(void) +{ + int counters; + + if (!cpu_has_pmp) + return -ENODEV; + + pr_info("Performance counters: "); + counters = ((read_cpucfg(LOONGARCH_CPUCFG6) & CPUCFG6_PMNUM) >> 4) + 1; + + loongarch_pmu.num_counters = counters; + loongarch_pmu.max_period = (1ULL << 63) - 1; + loongarch_pmu.valid_count = (1ULL << 63) - 1; + loongarch_pmu.overflow = 1ULL << 63; + loongarch_pmu.name = "loongarch/loongson64"; + loongarch_pmu.read_counter = loongarch_pmu_read_counter; + loongarch_pmu.write_counter = loongarch_pmu_write_counter; + loongarch_pmu.map_raw_event = loongarch_pmu_map_raw_event; + loongarch_pmu.general_event_map = &loongson_event_map; + loongarch_pmu.cache_event_map = &loongson_cache_map; + + on_each_cpu(reset_counters, NULL, 1); + + pr_cont("%s PMU enabled, %d %d-bit counters available to each CPU.\n", + loongarch_pmu.name, counters, 64); + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); diff --git a/arch/loongarch/kernel/perf_regs.c b/arch/loongarch/kernel/perf_regs.c new file mode 100644 index 0000000000000000000000000000000000000000..263ac4ab5af684ac88aff4d59dfa55c899326ef0 --- /dev/null +++ b/arch/loongarch/kernel/perf_regs.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Loongson Technology Corporation Limited + * + * Derived from MIPS: + * Copyright (C) 2013 Cavium, Inc. + */ + +#include + +#include + +#ifdef CONFIG_32BIT +u64 perf_reg_abi(struct task_struct *tsk) +{ + return PERF_SAMPLE_REGS_ABI_32; +} +#else /* Must be CONFIG_64BIT */ +u64 perf_reg_abi(struct task_struct *tsk) +{ + if (test_tsk_thread_flag(tsk, TIF_32BIT_REGS)) + return PERF_SAMPLE_REGS_ABI_32; + else + return PERF_SAMPLE_REGS_ABI_64; +} +#endif /* CONFIG_32BIT */ + +int perf_reg_validate(u64 mask) +{ + if (!mask) + return -EINVAL; + if (mask & ~((1ull << PERF_REG_LOONGARCH_MAX) - 1)) + return -EINVAL; + return 0; +} + +u64 perf_reg_value(struct pt_regs *regs, int idx) +{ + if (WARN_ON_ONCE((u32)idx >= PERF_REG_LOONGARCH_MAX)) + return 0; + + if ((u32)idx == PERF_REG_LOONGARCH_PC) + return regs->csr_era; + + return regs->regs[idx]; +} + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index 660492f064e7e4584fa2f04088506fc8ed7076f2..1256e3582475fbc06f4f7b712252ddaf2eeb84e0 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -293,7 +293,7 @@ unsigned long stack_top(void) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= prandom_u32_max(PAGE_SIZE); return sp & STACK_ALIGN; } diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S new file mode 100644 index 0000000000000000000000000000000000000000..d13252553a7c7d6f7916fda58f7fcdc1d9bae126 --- /dev/null +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * relocate_kernel.S for kexec + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#include + +#include +#include +#include +#include +#include +#include + +SYM_CODE_START(relocate_new_kernel) + /* + * a0: EFI boot flag for the new kernel + * a1: Command line pointer for the new kernel + * a2: System table pointer for the new kernel + * a3: Start address to jump to after relocation + * a4: Pointer to the current indirection page entry + */ + move s0, a4 + + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beqz s0, done + +process_entry: + PTR_L s1, s0, 0 + PTR_ADDI s0, s0, SZREG + + /* destination page */ + andi s2, s1, IND_DESTINATION + beqz s2, 1f + li.w t0, ~0x1 + and s3, s1, t0 /* store destination addr in s3 */ + b process_entry + +1: + /* indirection page, update s0 */ + andi s2, s1, IND_INDIRECTION + beqz s2, 1f + li.w t0, ~0x2 + and s0, s1, t0 + b process_entry + +1: + /* done page */ + andi s2, s1, IND_DONE + beqz s2, 1f + b done + +1: + /* source page */ + andi s2, s1, IND_SOURCE + beqz s2, process_entry + li.w t0, ~0x8 + and s1, s1, t0 + li.w s5, (1 << _PAGE_SHIFT) / SZREG + +copy_word: + /* copy page word by word */ + REG_L s4, s1, 0 + REG_S s4, s3, 0 + PTR_ADDI s3, s3, SZREG + PTR_ADDI s1, s1, SZREG + LONG_ADDI s5, s5, -1 + beqz s5, process_entry + b copy_word + b process_entry + +done: + ibar 0 + dbar 0 + + /* + * Jump to the new kernel, + * make sure the values of a0, a1, a2 and a3 are not changed. + */ + jr a3 +SYM_CODE_END(relocate_new_kernel) + +#ifdef CONFIG_SMP +/* + * Other CPUs should wait until code is relocated and + * then start at the entry point from LOONGARCH_IOCSR_MBUF0. + */ +SYM_CODE_START(kexec_smp_wait) +1: li.w t0, 0x100 /* wait for init loop */ +2: addi.w t0, t0, -1 /* limit mailbox access */ + bnez t0, 2b + li.w t1, LOONGARCH_IOCSR_MBUF0 + iocsrrd.w s0, t1 /* check PC as an indicator */ + beqz s0, 1b + iocsrrd.d s0, t1 /* get PC via mailbox */ + + li.d t0, CACHE_BASE + or s0, s0, t0 /* s0 = TO_CACHE(s0) */ + jr s0 /* jump to initial PC */ +SYM_CODE_END(kexec_smp_wait) +#endif + +relocate_new_kernel_end: + +SYM_DATA_START(relocate_new_kernel_size) + PTR relocate_new_kernel_end - relocate_new_kernel +SYM_DATA_END(relocate_new_kernel_size) diff --git a/arch/loongarch/kernel/reset.c b/arch/loongarch/kernel/reset.c index 800c965a17eaa81dc5b5d94064e7df5a802c4b1f..8c82021eb2f447d867560a77b3f5e6c276804bf4 100644 --- a/arch/loongarch/kernel/reset.c +++ b/arch/loongarch/kernel/reset.c @@ -15,10 +15,16 @@ #include #include #include -#include -static void default_halt(void) +void (*pm_power_off)(void); +EXPORT_SYMBOL(pm_power_off); + +void machine_halt(void) { +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif local_irq_disable(); clear_csr_ecfg(ECFG0_IM); @@ -30,18 +36,29 @@ static void default_halt(void) } } -static void default_poweroff(void) +void machine_power_off(void) { +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + do_kernel_power_off(); #ifdef CONFIG_EFI efi.reset_system(EFI_RESET_SHUTDOWN, EFI_SUCCESS, 0, NULL); #endif + while (true) { __arch_cpu_idle(); } } -static void default_restart(void) +void machine_restart(char *command) { +#ifdef CONFIG_SMP + preempt_disable(); + smp_send_stop(); +#endif + do_kernel_restart(command); #ifdef CONFIG_EFI if (efi_capsule_pending(NULL)) efi_reboot(REBOOT_WARM, NULL); @@ -55,47 +72,3 @@ static void default_restart(void) __arch_cpu_idle(); } } - -void (*pm_restart)(void); -EXPORT_SYMBOL(pm_restart); - -void (*pm_power_off)(void); -EXPORT_SYMBOL(pm_power_off); - -void machine_halt(void) -{ -#ifdef CONFIG_SMP - preempt_disable(); - smp_send_stop(); -#endif - default_halt(); -} - -void machine_power_off(void) -{ -#ifdef CONFIG_SMP - preempt_disable(); - smp_send_stop(); -#endif - pm_power_off(); -} - -void machine_restart(char *command) -{ -#ifdef CONFIG_SMP - preempt_disable(); - smp_send_stop(); -#endif - do_kernel_restart(command); - pm_restart(); -} - -static int __init loongarch_reboot_setup(void) -{ - pm_restart = default_restart; - pm_power_off = default_poweroff; - - return 0; -} - -arch_initcall(loongarch_reboot_setup); diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 8f5c2f9a1a835de4435cec1a5e92439387017697..1eb63fa9bc81aa7af2640b6bdaed95b57e7a4498 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -49,11 +51,9 @@ #define SMBIOS_CORE_PACKAGE_OFFSET 0x23 #define LOONGSON_EFI_ENABLE (1 << 3) -#ifdef CONFIG_VT -struct screen_info screen_info; -#endif +struct screen_info screen_info __section(".data"); -unsigned long fw_arg0, fw_arg1; +unsigned long fw_arg0, fw_arg1, fw_arg2; DEFINE_PER_CPU(unsigned long, kernelsp); struct cpuinfo_loongarch cpu_data[NR_CPUS] __read_mostly; @@ -122,16 +122,9 @@ static void __init parse_cpu_table(const struct dmi_header *dm) static void __init parse_bios_table(const struct dmi_header *dm) { - int bios_extern; char *dmi_data = (char *)dm; - bios_extern = *(dmi_data + SMBIOS_BIOSEXTERN_OFFSET); b_info.bios_size = (*(dmi_data + SMBIOS_BIOSSIZE_OFFSET) + 1) << 6; - - if (bios_extern & LOONGSON_EFI_ENABLE) - set_bit(EFI_BOOT, &efi.flags); - else - clear_bit(EFI_BOOT, &efi.flags); } static void __init find_tokens(const struct dmi_header *dm, void *dummy) @@ -194,9 +187,70 @@ static int __init early_parse_mem(char *p) } early_param("mem", early_parse_mem); +static void __init arch_reserve_vmcore(void) +{ +#ifdef CONFIG_PROC_VMCORE + u64 i; + phys_addr_t start, end; + + if (!is_kdump_kernel()) + return; + + if (!elfcorehdr_size) { + for_each_mem_range(i, &start, &end) { + if (elfcorehdr_addr >= start && elfcorehdr_addr < end) { + /* + * Reserve from the elf core header to the end of + * the memory segment, that should all be kdump + * reserved memory. + */ + elfcorehdr_size = end - elfcorehdr_addr; + break; + } + } + } + + if (memblock_is_region_reserved(elfcorehdr_addr, elfcorehdr_size)) { + pr_warn("elfcorehdr is overlapped\n"); + return; + } + + memblock_reserve(elfcorehdr_addr, elfcorehdr_size); + + pr_info("Reserving %llu KiB of memory at 0x%llx for elfcorehdr\n", + elfcorehdr_size >> 10, elfcorehdr_addr); +#endif +} + +static void __init arch_parse_crashkernel(void) +{ +#ifdef CONFIG_KEXEC + int ret; + unsigned long long start; + unsigned long long total_mem; + unsigned long long crash_base, crash_size; + + total_mem = memblock_phys_mem_size(); + ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base); + if (ret < 0 || crash_size <= 0) + return; + + start = memblock_phys_alloc_range(crash_size, 1, crash_base, crash_base + crash_size); + if (start != crash_base) { + pr_warn("Invalid memory region reserved for crash kernel\n"); + return; + } + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; +#endif +} + void __init platform_init(void) { - efi_init(); + arch_reserve_vmcore(); + arch_parse_crashkernel(); + #ifdef CONFIG_ACPI_TABLE_UPGRADE acpi_table_upgrade(); #endif @@ -247,7 +301,7 @@ static void __init arch_mem_init(char **cmdline_p) sparse_init(); memblock_set_bottom_up(true); - plat_swiotlb_setup(); + swiotlb_init(true, SWIOTLB_VERBOSE); dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); @@ -299,6 +353,15 @@ static void __init resource_init(void) request_resource(res, &data_resource); request_resource(res, &bss_resource); } + +#ifdef CONFIG_KEXEC + if (crashk_res.start < crashk_res.end) { + insert_resource(&iomem_resource, &crashk_res); + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n", + (unsigned long)((crashk_res.end - crashk_res.start + 1) >> 20), + (unsigned long)(crashk_res.start >> 20)); + } +#endif } static int __init reserve_memblock_reserved_regions(void) @@ -346,7 +409,7 @@ static void __init prefill_possible_map(void) for (; i < NR_CPUS; i++) set_cpu_possible(i, false); - nr_cpu_ids = possible; + set_nr_cpu_ids(possible); } #endif @@ -356,11 +419,13 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; init_environ(); + efi_init(); memblock_init(); + pagetable_init(); parse_early_param(); + reserve_initrd_mem(); platform_init(); - pagetable_init(); arch_mem_init(cmdline_p); resource_init(); diff --git a/arch/loongarch/kernel/signal.c b/arch/loongarch/kernel/signal.c index 7f4889df4a17e4063bde7e303750738ef080fdc7..8f5b7986374b52afc237853cc743a61695d079ac 100644 --- a/arch/loongarch/kernel/signal.c +++ b/arch/loongarch/kernel/signal.c @@ -529,11 +529,11 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) signal_setup_done(ret, ksig, 0); } -void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) +void arch_do_signal_or_restart(struct pt_regs *regs) { struct ksignal ksig; - if (has_signal && get_signal(&ksig)) { + if (get_signal(&ksig)) { /* Whee! Actually deliver the signal. */ handle_signal(&ksig, regs); return; diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index b5fab308dcf25a3693ded821c584cee6952ead1d..781a4d4bdddc994f6f6c6c8e473382ccad1fe355 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -240,11 +240,6 @@ void loongson3_smp_finish(void) #ifdef CONFIG_HOTPLUG_CPU -static bool io_master(int cpu) -{ - return test_bit(cpu, &loongson_sysconf.cores_io_master); -} - int loongson3_cpu_disable(void) { unsigned long flags; diff --git a/arch/loongarch/kernel/sysrq.c b/arch/loongarch/kernel/sysrq.c new file mode 100644 index 0000000000000000000000000000000000000000..366baef72d297654b4d5722ad8683ca929207b10 --- /dev/null +++ b/arch/loongarch/kernel/sysrq.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * LoongArch specific sysrq operations. + * + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include + +#include +#include + +/* + * Dump TLB entries on all CPUs. + */ + +static DEFINE_SPINLOCK(show_lock); + +static void sysrq_tlbdump_single(void *dummy) +{ + unsigned long flags; + + spin_lock_irqsave(&show_lock, flags); + + pr_info("CPU%d:\n", smp_processor_id()); + dump_tlb_regs(); + pr_info("\n"); + dump_tlb_all(); + pr_info("\n"); + + spin_unlock_irqrestore(&show_lock, flags); +} + +#ifdef CONFIG_SMP +static void sysrq_tlbdump_othercpus(struct work_struct *dummy) +{ + smp_call_function(sysrq_tlbdump_single, NULL, 0); +} + +static DECLARE_WORK(sysrq_tlbdump, sysrq_tlbdump_othercpus); +#endif + +static void sysrq_handle_tlbdump(int key) +{ + sysrq_tlbdump_single(NULL); +#ifdef CONFIG_SMP + schedule_work(&sysrq_tlbdump); +#endif +} + +static struct sysrq_key_op sysrq_tlbdump_op = { + .handler = sysrq_handle_tlbdump, + .help_msg = "show-tlbs(x)", + .action_msg = "Show TLB entries", + .enable_mask = SYSRQ_ENABLE_DUMP, +}; + +static int __init loongarch_sysrq_init(void) +{ + return register_sysrq_key('x', &sysrq_tlbdump_op); +} +arch_initcall(loongarch_sysrq_init); diff --git a/arch/loongarch/kernel/topology.c b/arch/loongarch/kernel/topology.c index ab1a75c0b5a64572c60c0062a2926812067cfe1c..caa7cd8590788ca39cdc27205951bc1a368fcff7 100644 --- a/arch/loongarch/kernel/topology.c +++ b/arch/loongarch/kernel/topology.c @@ -5,6 +5,7 @@ #include #include #include +#include static DEFINE_PER_CPU(struct cpu, cpu_devices); @@ -40,7 +41,7 @@ static int __init topology_init(void) for_each_present_cpu(i) { struct cpu *c = &per_cpu(cpu_devices, i); - c->hotpluggable = !!i; + c->hotpluggable = !io_master(i); ret = register_cpu(c, i); if (ret < 0) pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret); diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c index aa1c95aaf595ba0402e80d10f82cffb74db3c735..1a4dce84ebc60f5208e19893d8d4b943b47c5504 100644 --- a/arch/loongarch/kernel/traps.c +++ b/arch/loongarch/kernel/traps.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -246,6 +247,9 @@ void __noreturn die(const char *str, struct pt_regs *regs) oops_exit(); + if (regs && kexec_should_crash(current)) + crash_kexec(regs); + if (in_interrupt()) panic("Fatal exception in interrupt"); @@ -374,6 +378,29 @@ asmlinkage void noinstr do_ale(struct pt_regs *regs) irqentry_exit(regs, state); } +#ifdef CONFIG_GENERIC_BUG +int is_valid_bugaddr(unsigned long addr) +{ + return 1; +} +#endif /* CONFIG_GENERIC_BUG */ + +static void bug_handler(struct pt_regs *regs) +{ + switch (report_bug(regs->csr_era, regs)) { + case BUG_TRAP_TYPE_BUG: + case BUG_TRAP_TYPE_NONE: + die_if_kernel("Oops - BUG", regs); + force_sig(SIGTRAP); + break; + + case BUG_TRAP_TYPE_WARN: + /* Skip the BUG instruction and continue */ + regs->csr_era += LOONGARCH_INSN_SIZE; + break; + } +} + asmlinkage void noinstr do_bp(struct pt_regs *regs) { bool user = user_mode(regs); @@ -427,8 +454,7 @@ asmlinkage void noinstr do_bp(struct pt_regs *regs) switch (bcode) { case BRK_BUG: - die_if_kernel("Kernel bug detected", regs); - force_sig(SIGTRAP); + bug_handler(regs); break; case BRK_DIVZERO: die_if_kernel("Break instruction in kernel code", regs); @@ -461,11 +487,9 @@ asmlinkage void noinstr do_watch(struct pt_regs *regs) asmlinkage void noinstr do_ri(struct pt_regs *regs) { - int status = -1; + int status = SIGILL; unsigned int opcode = 0; unsigned int __user *era = (unsigned int __user *)exception_era(regs); - unsigned long old_era = regs->csr_era; - unsigned long old_ra = regs->regs[1]; irqentry_state_t state = irqentry_enter(regs); local_irq_enable(); @@ -477,21 +501,12 @@ asmlinkage void noinstr do_ri(struct pt_regs *regs) die_if_kernel("Reserved instruction in kernel code", regs); - compute_return_era(regs); - if (unlikely(get_user(opcode, era) < 0)) { status = SIGSEGV; current->thread.error_code = 1; } - if (status < 0) - status = SIGILL; - - if (unlikely(status > 0)) { - regs->csr_era = old_era; /* Undo skip-over. */ - regs->regs[1] = old_ra; - force_sig(status); - } + force_sig(status); out: local_irq_disable(); @@ -631,9 +646,6 @@ asmlinkage void noinstr do_vint(struct pt_regs *regs, unsigned long sp) irqentry_exit(regs, state); } -extern void tlb_init(int cpu); -extern void cache_error_setup(void); - unsigned long eentry; unsigned long tlbrentry; diff --git a/arch/loongarch/kernel/vdso.c b/arch/loongarch/kernel/vdso.c index f32c38abd791589527bbba2eda8bd91a7065244a..8c9826062652e3b39a6ee2c03d02370b519eb07d 100644 --- a/arch/loongarch/kernel/vdso.c +++ b/arch/loongarch/kernel/vdso.c @@ -78,7 +78,7 @@ static unsigned long vdso_base(void) unsigned long base = STACK_TOP; if (current->flags & PF_RANDOMIZE) { - base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1); + base += prandom_u32_max(VDSO_RANDOMIZE_SIZE); base = PAGE_ALIGN(base); } diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S index 69c76f26c1c57e03ba4ff6389111abd8cbedf03a..b3309a5e695b23ceed7f156331805015d00255ff 100644 --- a/arch/loongarch/kernel/vmlinux.lds.S +++ b/arch/loongarch/kernel/vmlinux.lds.S @@ -12,6 +12,7 @@ #define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) #include +#include "image-vars.h" /* * Max avaliable Page Size is 64K, so we set SectionAlignment @@ -54,6 +55,10 @@ SECTIONS EXCEPTION_TABLE(16) + .got : ALIGN(16) { *(.got) } + .plt : ALIGN(16) { *(.plt) } + .got.plt : ALIGN(16) { *(.got.plt) } + . = ALIGN(PECOFF_SEGMENT_ALIGN); __init_begin = .; __inittext_begin = .; @@ -77,6 +82,8 @@ SECTIONS PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT) #endif + .rela.dyn : ALIGN(8) { *(.rela.dyn) *(.rela*) } + .init.bss : { *(.init.bss) } diff --git a/arch/loongarch/lib/dump_tlb.c b/arch/loongarch/lib/dump_tlb.c index cda2c6bc7f09405ce4e2f8c0e8e751fa52675f4e..c2cc7ce343c9bb3c914bb531bd44bb80d79be425 100644 --- a/arch/loongarch/lib/dump_tlb.c +++ b/arch/loongarch/lib/dump_tlb.c @@ -18,11 +18,11 @@ void dump_tlb_regs(void) { const int field = 2 * sizeof(unsigned long); - pr_info("Index : %0x\n", read_csr_tlbidx()); - pr_info("PageSize : %0x\n", read_csr_pagesize()); - pr_info("EntryHi : %0*llx\n", field, read_csr_entryhi()); - pr_info("EntryLo0 : %0*llx\n", field, read_csr_entrylo0()); - pr_info("EntryLo1 : %0*llx\n", field, read_csr_entrylo1()); + pr_info("Index : 0x%0x\n", read_csr_tlbidx()); + pr_info("PageSize : 0x%0x\n", read_csr_pagesize()); + pr_info("EntryHi : 0x%0*llx\n", field, read_csr_entryhi()); + pr_info("EntryLo0 : 0x%0*llx\n", field, read_csr_entrylo0()); + pr_info("EntryLo1 : 0x%0*llx\n", field, read_csr_entrylo1()); } static void dump_tlb(int first, int last) @@ -33,8 +33,8 @@ static void dump_tlb(int first, int last) unsigned int s_index, s_asid; unsigned int pagesize, c0, c1, i; unsigned long asidmask = cpu_asid_mask(¤t_cpu_data); - int pwidth = 11; - int vwidth = 11; + int pwidth = 16; + int vwidth = 16; int asidwidth = DIV_ROUND_UP(ilog2(asidmask) + 1, 4); s_entryhi = read_csr_entryhi(); @@ -64,22 +64,22 @@ static void dump_tlb(int first, int last) /* * Only print entries in use */ - pr_info("Index: %2d pgsize=%x ", i, (1 << pagesize)); + pr_info("Index: %4d pgsize=0x%x ", i, (1 << pagesize)); c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; - pr_cont("va=%0*lx asid=%0*lx", + pr_cont("va=0x%0*lx asid=0x%0*lx", vwidth, (entryhi & ~0x1fffUL), asidwidth, asid & asidmask); /* NR/NX are in awkward places, so mask them off separately */ pa = entrylo0 & ~(ENTRYLO_NR | ENTRYLO_NX); pa = pa & PAGE_MASK; pr_cont("\n\t["); - pr_cont("ri=%d xi=%d ", + pr_cont("nr=%d nx=%d ", (entrylo0 & ENTRYLO_NR) ? 1 : 0, (entrylo0 & ENTRYLO_NX) ? 1 : 0); - pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d plv=%lld] [", + pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld] [", pwidth, pa, c0, (entrylo0 & ENTRYLO_D) ? 1 : 0, (entrylo0 & ENTRYLO_V) ? 1 : 0, @@ -88,10 +88,10 @@ static void dump_tlb(int first, int last) /* NR/NX are in awkward places, so mask them off separately */ pa = entrylo1 & ~(ENTRYLO_NR | ENTRYLO_NX); pa = pa & PAGE_MASK; - pr_cont("ri=%d xi=%d ", + pr_cont("nr=%d nx=%d ", (entrylo1 & ENTRYLO_NR) ? 1 : 0, (entrylo1 & ENTRYLO_NX) ? 1 : 0); - pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d plv=%lld]\n", + pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld]\n", pwidth, pa, c1, (entrylo1 & ENTRYLO_D) ? 1 : 0, (entrylo1 & ENTRYLO_V) ? 1 : 0, diff --git a/arch/loongarch/mm/cache.c b/arch/loongarch/mm/cache.c index e8c68dcf6ab20bb2d5000912534229343e8ea187..72685a48eaf084f61ce967bcd16fd0bb79bf9ce4 100644 --- a/arch/loongarch/mm/cache.c +++ b/arch/loongarch/mm/cache.c @@ -6,8 +6,8 @@ * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) * Copyright (C) 2007 MIPS Technologies, Inc. */ +#include #include -#include #include #include #include @@ -16,14 +16,21 @@ #include #include +#include #include #include #include -#include #include +#include #include #include +void cache_error_setup(void) +{ + extern char __weak except_vec_cex; + set_merr_handler(0x0, &except_vec_cex, 0x80); +} + /* * LoongArch maintains ICache/DCache coherency by hardware, * we just need "ibar" to avoid instruction hazard here. @@ -34,109 +41,121 @@ void local_flush_icache_range(unsigned long start, unsigned long end) } EXPORT_SYMBOL(local_flush_icache_range); -void cache_error_setup(void) -{ - extern char __weak except_vec_cex; - set_merr_handler(0x0, &except_vec_cex, 0x80); -} - -static unsigned long icache_size __read_mostly; -static unsigned long dcache_size __read_mostly; -static unsigned long vcache_size __read_mostly; -static unsigned long scache_size __read_mostly; - -static char *way_string[] = { NULL, "direct mapped", "2-way", - "3-way", "4-way", "5-way", "6-way", "7-way", "8-way", - "9-way", "10-way", "11-way", "12-way", - "13-way", "14-way", "15-way", "16-way", -}; - -static void probe_pcache(void) +static void flush_cache_leaf(unsigned int leaf) { - struct cpuinfo_loongarch *c = ¤t_cpu_data; - unsigned int lsize, sets, ways; - unsigned int config; - - config = read_cpucfg(LOONGARCH_CPUCFG17); - lsize = 1 << ((config & CPUCFG17_L1I_SIZE_M) >> CPUCFG17_L1I_SIZE); - sets = 1 << ((config & CPUCFG17_L1I_SETS_M) >> CPUCFG17_L1I_SETS); - ways = ((config & CPUCFG17_L1I_WAYS_M) >> CPUCFG17_L1I_WAYS) + 1; - - c->icache.linesz = lsize; - c->icache.sets = sets; - c->icache.ways = ways; - icache_size = sets * ways * lsize; - c->icache.waysize = icache_size / c->icache.ways; - - config = read_cpucfg(LOONGARCH_CPUCFG18); - lsize = 1 << ((config & CPUCFG18_L1D_SIZE_M) >> CPUCFG18_L1D_SIZE); - sets = 1 << ((config & CPUCFG18_L1D_SETS_M) >> CPUCFG18_L1D_SETS); - ways = ((config & CPUCFG18_L1D_WAYS_M) >> CPUCFG18_L1D_WAYS) + 1; - - c->dcache.linesz = lsize; - c->dcache.sets = sets; - c->dcache.ways = ways; - dcache_size = sets * ways * lsize; - c->dcache.waysize = dcache_size / c->dcache.ways; - - c->options |= LOONGARCH_CPU_PREFETCH; - - pr_info("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", - icache_size >> 10, way_string[c->icache.ways], "VIPT", c->icache.linesz); - - pr_info("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", - dcache_size >> 10, way_string[c->dcache.ways], "VIPT", "no aliases", c->dcache.linesz); + int i, j, nr_nodes; + uint64_t addr = CSR_DMW0_BASE; + struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; + + nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; + + do { + for (i = 0; i < cdesc->sets; i++) { + for (j = 0; j < cdesc->ways; j++) { + flush_cache_line(leaf, addr); + addr++; + } + + addr -= cdesc->ways; + addr += cdesc->linesz; + } + addr += (1ULL << NODE_ADDRSPACE_SHIFT); + } while (--nr_nodes > 0); } -static void probe_vcache(void) +asmlinkage __visible void __flush_cache_all(void) { - struct cpuinfo_loongarch *c = ¤t_cpu_data; - unsigned int lsize, sets, ways; - unsigned int config; - - config = read_cpucfg(LOONGARCH_CPUCFG19); - lsize = 1 << ((config & CPUCFG19_L2_SIZE_M) >> CPUCFG19_L2_SIZE); - sets = 1 << ((config & CPUCFG19_L2_SETS_M) >> CPUCFG19_L2_SETS); - ways = ((config & CPUCFG19_L2_WAYS_M) >> CPUCFG19_L2_WAYS) + 1; - - c->vcache.linesz = lsize; - c->vcache.sets = sets; - c->vcache.ways = ways; - vcache_size = lsize * sets * ways; - c->vcache.waysize = vcache_size / c->vcache.ways; - - pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n", - vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz); + int leaf; + struct cache_desc *cdesc = current_cpu_data.cache_leaves; + unsigned int cache_present = current_cpu_data.cache_leaves_present; + + leaf = cache_present - 1; + if (cache_inclusive(cdesc + leaf)) { + flush_cache_leaf(leaf); + return; + } + + for (leaf = 0; leaf < cache_present; leaf++) + flush_cache_leaf(leaf); } -static void probe_scache(void) -{ - struct cpuinfo_loongarch *c = ¤t_cpu_data; - unsigned int lsize, sets, ways; - unsigned int config; - - config = read_cpucfg(LOONGARCH_CPUCFG20); - lsize = 1 << ((config & CPUCFG20_L3_SIZE_M) >> CPUCFG20_L3_SIZE); - sets = 1 << ((config & CPUCFG20_L3_SETS_M) >> CPUCFG20_L3_SETS); - ways = ((config & CPUCFG20_L3_WAYS_M) >> CPUCFG20_L3_WAYS) + 1; - - c->scache.linesz = lsize; - c->scache.sets = sets; - c->scache.ways = ways; - /* 4 cores. scaches are shared */ - scache_size = lsize * sets * ways; - c->scache.waysize = scache_size / c->scache.ways; - - pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", - scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); -} +#define L1IUPRE (1 << 0) +#define L1IUUNIFY (1 << 1) +#define L1DPRE (1 << 2) + +#define LXIUPRE (1 << 0) +#define LXIUUNIFY (1 << 1) +#define LXIUPRIV (1 << 2) +#define LXIUINCL (1 << 3) +#define LXDPRE (1 << 4) +#define LXDPRIV (1 << 5) +#define LXDINCL (1 << 6) + +#define populate_cache_properties(cfg0, cdesc, level, leaf) \ +do { \ + unsigned int cfg1; \ + \ + cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ + if (level == 1) { \ + cdesc->flags |= CACHE_PRIVATE; \ + } else { \ + if (cfg0 & LXIUPRIV) \ + cdesc->flags |= CACHE_PRIVATE; \ + if (cfg0 & LXIUINCL) \ + cdesc->flags |= CACHE_INCLUSIVE; \ + } \ + cdesc->level = level; \ + cdesc->flags |= CACHE_PRESENT; \ + cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ + cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ + cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ + cdesc++; leaf++; \ +} while (0) void cpu_cache_init(void) { - probe_pcache(); - probe_vcache(); - probe_scache(); - + unsigned int leaf = 0, level = 1; + unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); + struct cache_desc *cdesc = current_cpu_data.cache_leaves; + + if (config & L1IUPRE) { + if (config & L1IUUNIFY) + cdesc->type = CACHE_TYPE_UNIFIED; + else + cdesc->type = CACHE_TYPE_INST; + populate_cache_properties(config, cdesc, level, leaf); + } + + if (config & L1DPRE) { + cdesc->type = CACHE_TYPE_DATA; + populate_cache_properties(config, cdesc, level, leaf); + } + + config = config >> 3; + for (level = 2; level <= CACHE_LEVEL_MAX; level++) { + if (!config) + break; + + if (config & LXIUPRE) { + if (config & LXIUUNIFY) + cdesc->type = CACHE_TYPE_UNIFIED; + else + cdesc->type = CACHE_TYPE_INST; + populate_cache_properties(config, cdesc, level, leaf); + } + + if (config & LXDPRE) { + cdesc->type = CACHE_TYPE_DATA; + populate_cache_properties(config, cdesc, level, leaf); + } + + config = config >> 7; + } + + BUG_ON(leaf > CACHE_LEAVES_MAX); + + current_cpu_data.cache_leaves_present = leaf; + current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; shm_align_mask = PAGE_SIZE - 1; } diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c index 605579b19a002ee0252e7506f126d55314bbd5b2..1ccd53655cab097f02ed09a5c1bd566de38d01ec 100644 --- a/arch/loongarch/mm/fault.c +++ b/arch/loongarch/mm/fault.c @@ -216,6 +216,10 @@ good_area: return; } + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + if (unlikely(fault & VM_FAULT_RETRY)) { flags |= FAULT_FLAG_TRIED; diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c index 7094a68c9b8329db56836de78738daa29526d7ff..080061793c859d58d6875207a20961bd08191b68 100644 --- a/arch/loongarch/mm/init.c +++ b/arch/loongarch/mm/init.c @@ -131,18 +131,6 @@ int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params) return ret; } -#ifdef CONFIG_NUMA -int memory_add_physaddr_to_nid(u64 start) -{ - int nid; - - nid = pa_to_nid(start); - return nid; -} -EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); -#endif - -#ifdef CONFIG_MEMORY_HOTREMOVE void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) { unsigned long start_pfn = start >> PAGE_SHIFT; @@ -154,8 +142,79 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap) page += vmem_altmap_offset(altmap); __remove_pages(start_pfn, nr_pages, altmap); } + +#ifdef CONFIG_NUMA +int memory_add_physaddr_to_nid(u64 start) +{ + return pa_to_nid(start); +} +EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); +#endif +#endif + +static pte_t *fixmap_pte(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset_k(addr); + p4d = p4d_offset(pgd, addr); + + if (pgd_none(*pgd)) { + pud_t *new __maybe_unused; + + new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + pgd_populate(&init_mm, pgd, new); +#ifndef __PAGETABLE_PUD_FOLDED + pud_init((unsigned long)new, (unsigned long)invalid_pmd_table); #endif + } + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + pmd_t *new __maybe_unused; + + new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + pud_populate(&init_mm, pud, new); +#ifndef __PAGETABLE_PMD_FOLDED + pmd_init((unsigned long)new, (unsigned long)invalid_pte_table); #endif + } + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + pte_t *new __maybe_unused; + + new = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); + pmd_populate_kernel(&init_mm, pmd, new); + } + + return pte_offset_kernel(pmd, addr); +} + +void __init __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *ptep; + + BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + ptep = fixmap_pte(addr); + if (!pte_none(*ptep)) { + pte_ERROR(*ptep); + return; + } + + if (pgprot_val(flags)) + set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, flags)); + else { + pte_clear(&init_mm, addr, ptep); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); + } +} /* * Align swapper_pg_dir in to 64K, allows its address to be loaded diff --git a/arch/loongarch/mm/mmap.c b/arch/loongarch/mm/mmap.c index 52e40f0ba732d715de4d9c12878694ebedcd6f1a..fbe1a4856fc42d4f255d9e9c3a9e349c3fa09e9a 100644 --- a/arch/loongarch/mm/mmap.c +++ b/arch/loongarch/mm/mmap.c @@ -2,16 +2,11 @@ /* * Copyright (C) 2020-2022 Loongson Technology Corporation Limited */ -#include -#include -#include +#include +#include +#include #include #include -#include -#include -#include -#include -#include unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */ EXPORT_SYMBOL(shm_align_mask); @@ -120,6 +115,33 @@ int __virt_addr_valid(volatile void *kaddr) if ((vaddr < PAGE_OFFSET) || (vaddr >= vm_map_base)) return 0; - return pfn_valid(PFN_DOWN(virt_to_phys(kaddr))); + return pfn_valid(PFN_DOWN(PHYSADDR(kaddr))); } EXPORT_SYMBOL_GPL(__virt_addr_valid); + +/* + * You really shouldn't be using read() or write() on /dev/mem. This might go + * away in the future. + */ +int valid_phys_addr_range(phys_addr_t addr, size_t size) +{ + /* + * Check whether addr is covered by a memory region without the + * MEMBLOCK_NOMAP attribute, and whether that region covers the + * entire range. In theory, this could lead to false negatives + * if the range is covered by distinct but adjacent memory regions + * that only differ in other attributes. However, few of such + * attributes have been defined, and it is debatable whether it + * follows that /dev/mem read() calls should be able traverse + * such boundaries. + */ + return memblock_is_region_memory(addr, size) && memblock_is_map_memory(addr); +} + +/* + * Do not allow /dev/mem mappings beyond the supported physical range. + */ +int valid_mmap_phys_addr_range(unsigned long pfn, size_t size) +{ + return !(((pfn << PAGE_SHIFT) + size) & ~(GENMASK_ULL(cpu_pabits, 0))); +} diff --git a/arch/loongarch/mm/tlb.c b/arch/loongarch/mm/tlb.c index 9818ce11546bcb502415dbe7e000d1798ae4aac4..da3681f131c8d8a722b7f479775f7a48be47fc0a 100644 --- a/arch/loongarch/mm/tlb.c +++ b/arch/loongarch/mm/tlb.c @@ -258,7 +258,7 @@ extern long exception_handlers[VECSIZE * 128 / sizeof(long)]; void setup_tlb_handler(int cpu) { setup_ptwalker(); - output_pgtable_bits_defines(); + local_flush_tlb_all(); /* The tlb handlers are generated only once */ if (cpu == 0) { @@ -301,6 +301,7 @@ void tlb_init(int cpu) write_csr_pagesize(PS_DEFAULT_SIZE); write_csr_stlbpgsize(PS_DEFAULT_SIZE); write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE); + setup_tlb_handler(cpu); - local_flush_tlb_all(); + output_pgtable_bits_defines(); } diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S index 39743337999e98f5458fe08a42d72e120ffc7d1b..d8ee8fbc8c67321b92adbeabe9eea923aa2780ff 100644 --- a/arch/loongarch/mm/tlbex.S +++ b/arch/loongarch/mm/tlbex.S @@ -10,15 +10,20 @@ #include #include +#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) +#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) + .macro tlb_do_page_fault, write SYM_FUNC_START(tlb_do_page_fault_\write) SAVE_ALL - csrrd a2, LOONGARCH_CSR_BADV - move a0, sp - REG_S a2, sp, PT_BVADDR - li.w a1, \write - la.abs t0, do_page_fault - jirl ra, t0, 0 + csrrd a2, LOONGARCH_CSR_BADV + move a0, sp + REG_S a2, sp, PT_BVADDR + li.w a1, \write + la.abs t0, do_page_fault + jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(tlb_do_page_fault_\write) .endm @@ -29,133 +34,115 @@ SYM_FUNC_START(handle_tlb_protect) BACKUP_T0T1 SAVE_ALL - move a0, sp - move a1, zero - csrrd a2, LOONGARCH_CSR_BADV - REG_S a2, sp, PT_BVADDR - la.abs t0, do_page_fault - jirl ra, t0, 0 + move a0, sp + move a1, zero + csrrd a2, LOONGARCH_CSR_BADV + REG_S a2, sp, PT_BVADDR + la.abs t0, do_page_fault + jirl ra, t0, 0 RESTORE_ALL_AND_RET SYM_FUNC_END(handle_tlb_protect) SYM_FUNC_START(handle_tlb_load) - csrwr t0, EXCEPTION_KS0 - csrwr t1, EXCEPTION_KS1 - csrwr ra, EXCEPTION_KS2 + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ - csrrd t0, LOONGARCH_CSR_BADV - bltz t0, vmalloc_load - csrrd t1, LOONGARCH_CSR_PGDL + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_load + csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_load: /* Get PGD offset in bytes */ - srli.d t0, t0, PGDIR_SHIFT - andi t0, t0, (PTRS_PER_PGD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PUD_SHIFT - andi t0, t0, (PTRS_PER_PUD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PMD_SHIFT - andi t0, t0, (PTRS_PER_PMD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 #endif - ld.d ra, t1, 0 + ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ - andi t0, ra, _PAGE_HUGE - bnez t0, tlb_huge_update_load + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_load - csrrd t0, LOONGARCH_CSR_BADV - srli.d t0, t0, PAGE_SHIFT - andi t0, t0, (PTRS_PER_PTE - 1) - slli.d t0, t0, _PTE_T_LOG2 - add.d t1, ra, t0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_load: -#endif -#ifdef CONFIG_SMP - ll.d t0, t1, 0 + ll.d t0, t1, 0 #else - ld.d t0, t1, 0 + ld.d t0, t1, 0 #endif - tlbsrch - - srli.d ra, t0, _PAGE_PRESENT_SHIFT - andi ra, ra, 1 - beqz ra, nopage_tlb_load + andi ra, t0, _PAGE_PRESENT + beqz ra, nopage_tlb_load - ori t0, t0, _PAGE_VALID + ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, smp_pgtable_change_load + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_load #else - st.d t0, t1, 0 + st.d t0, t1, 0 #endif - ori t1, t1, 8 - xori t1, t1, 8 - ld.d t0, t1, 0 - ld.d t1, t1, 8 - csrwr t0, LOONGARCH_CSR_TLBELO0 - csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr -leave_load: - csrrd t0, EXCEPTION_KS0 - csrrd t1, EXCEPTION_KS1 - csrrd ra, EXCEPTION_KS2 + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 ertn + #ifdef CONFIG_64BIT vmalloc_load: - la.abs t1, swapper_pg_dir - b vmalloc_done_load + la.abs t1, swapper_pg_dir + b vmalloc_done_load #endif - /* - * This is the entry point when build_tlbchange_handler_head - * spots a huge page. - */ + /* This is the entry point of a huge page. */ tlb_huge_update_load: #ifdef CONFIG_SMP - ll.d t0, t1, 0 -#else - ld.d t0, t1, 0 + ll.d ra, t1, 0 #endif - srli.d ra, t0, _PAGE_PRESENT_SHIFT - andi ra, ra, 1 - beqz ra, nopage_tlb_load - tlbsrch + andi t0, ra, _PAGE_PRESENT + beqz t0, nopage_tlb_load - ori t0, t0, _PAGE_VALID #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, tlb_huge_update_load - ld.d t0, t1, 0 + ori t0, ra, _PAGE_VALID + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_load + ori t0, ra, _PAGE_VALID #else - st.d t0, t1, 0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, _PAGE_VALID + st.d t0, t1, 0 #endif + tlbsrch addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16) addi.d ra, t1, 0 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX tlbwr - csrxchg zero, t1, LOONGARCH_CSR_TLBIDX + csrxchg zero, t1, LOONGARCH_CSR_TLBIDX /* * A huge PTE describes an area the size of the @@ -167,21 +154,20 @@ tlb_huge_update_load: * address space. */ /* Huge page: Move Global bit */ - xori t0, t0, _PAGE_HUGE - lu12i.w t1, _PAGE_HGLOBAL >> 12 - and t1, t0, t1 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) - or t0, t0, t1 + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 - addi.d ra, t0, 0 - csrwr t0, LOONGARCH_CSR_TLBELO0 - addi.d t0, ra, 0 + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ - addi.d t1, zero, 1 - slli.d t1, t1, (HPAGE_SHIFT - 1) - add.d t0, t0, t1 - csrwr t0, LOONGARCH_CSR_TLBELO1 + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) @@ -194,136 +180,120 @@ tlb_huge_update_load: addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + nopage_tlb_load: - dbar 0 - csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_0 - jr t0 + dbar 0 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_0 + jr t0 SYM_FUNC_END(handle_tlb_load) SYM_FUNC_START(handle_tlb_store) - csrwr t0, EXCEPTION_KS0 - csrwr t1, EXCEPTION_KS1 - csrwr ra, EXCEPTION_KS2 + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ - csrrd t0, LOONGARCH_CSR_BADV - bltz t0, vmalloc_store - csrrd t1, LOONGARCH_CSR_PGDL + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_store + csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_store: /* Get PGD offset in bytes */ - srli.d t0, t0, PGDIR_SHIFT - andi t0, t0, (PTRS_PER_PGD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 - + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PUD_SHIFT - andi t0, t0, (PTRS_PER_PUD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PMD_SHIFT - andi t0, t0, (PTRS_PER_PMD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 #endif - ld.d ra, t1, 0 + ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ - andi t0, ra, _PAGE_HUGE - bnez t0, tlb_huge_update_store + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_store - csrrd t0, LOONGARCH_CSR_BADV - srli.d t0, t0, PAGE_SHIFT - andi t0, t0, (PTRS_PER_PTE - 1) - slli.d t0, t0, _PTE_T_LOG2 - add.d t1, ra, t0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_store: -#endif -#ifdef CONFIG_SMP - ll.d t0, t1, 0 + ll.d t0, t1, 0 #else - ld.d t0, t1, 0 + ld.d t0, t1, 0 #endif - tlbsrch - - srli.d ra, t0, _PAGE_PRESENT_SHIFT - andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) - xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) - bnez ra, nopage_tlb_store + andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE + xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE + bnez ra, nopage_tlb_store - ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, smp_pgtable_change_store + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_store #else - st.d t0, t1, 0 + st.d t0, t1, 0 #endif - - ori t1, t1, 8 - xori t1, t1, 8 - ld.d t0, t1, 0 - ld.d t1, t1, 8 - csrwr t0, LOONGARCH_CSR_TLBELO0 - csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr -leave_store: - csrrd t0, EXCEPTION_KS0 - csrrd t1, EXCEPTION_KS1 - csrrd ra, EXCEPTION_KS2 + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 ertn + #ifdef CONFIG_64BIT vmalloc_store: - la.abs t1, swapper_pg_dir - b vmalloc_done_store + la.abs t1, swapper_pg_dir + b vmalloc_done_store #endif - /* - * This is the entry point when build_tlbchange_handler_head - * spots a huge page. - */ + /* This is the entry point of a huge page. */ tlb_huge_update_store: #ifdef CONFIG_SMP - ll.d t0, t1, 0 -#else - ld.d t0, t1, 0 + ll.d ra, t1, 0 #endif - srli.d ra, t0, _PAGE_PRESENT_SHIFT - andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) - xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT) - bnez ra, nopage_tlb_store - - tlbsrch - ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE + xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE + bnez t0, nopage_tlb_store #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, tlb_huge_update_store - ld.d t0, t1, 0 + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_store + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else - st.d t0, t1, 0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + st.d t0, t1, 0 #endif + tlbsrch addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16) addi.d ra, t1, 0 csrxchg ra, t1, LOONGARCH_CSR_TLBIDX tlbwr - csrxchg zero, t1, LOONGARCH_CSR_TLBIDX + csrxchg zero, t1, LOONGARCH_CSR_TLBIDX /* * A huge PTE describes an area the size of the * configured huge page size. This is twice the @@ -334,21 +304,20 @@ tlb_huge_update_store: * address space. */ /* Huge page: Move Global bit */ - xori t0, t0, _PAGE_HUGE - lu12i.w t1, _PAGE_HGLOBAL >> 12 - and t1, t0, t1 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) - or t0, t0, t1 + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 - addi.d ra, t0, 0 - csrwr t0, LOONGARCH_CSR_TLBELO0 - addi.d t0, ra, 0 + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ - addi.d t1, zero, 1 - slli.d t1, t1, (HPAGE_SHIFT - 1) - add.d t0, t0, t1 - csrwr t0, LOONGARCH_CSR_TLBELO1 + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) @@ -362,126 +331,110 @@ tlb_huge_update_store: addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + nopage_tlb_store: - dbar 0 - csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 - jr t0 + dbar 0 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_1 + jr t0 SYM_FUNC_END(handle_tlb_store) SYM_FUNC_START(handle_tlb_modify) - csrwr t0, EXCEPTION_KS0 - csrwr t1, EXCEPTION_KS1 - csrwr ra, EXCEPTION_KS2 + csrwr t0, EXCEPTION_KS0 + csrwr t1, EXCEPTION_KS1 + csrwr ra, EXCEPTION_KS2 /* * The vmalloc handling is not in the hotpath. */ - csrrd t0, LOONGARCH_CSR_BADV - bltz t0, vmalloc_modify - csrrd t1, LOONGARCH_CSR_PGDL + csrrd t0, LOONGARCH_CSR_BADV + bltz t0, vmalloc_modify + csrrd t1, LOONGARCH_CSR_PGDL vmalloc_done_modify: /* Get PGD offset in bytes */ - srli.d t0, t0, PGDIR_SHIFT - andi t0, t0, (PTRS_PER_PGD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT + alsl.d t1, ra, t1, 3 #if CONFIG_PGTABLE_LEVELS > 3 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PUD_SHIFT - andi t0, t0, (PTRS_PER_PUD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT + alsl.d t1, ra, t1, 3 #endif #if CONFIG_PGTABLE_LEVELS > 2 - csrrd t0, LOONGARCH_CSR_BADV - ld.d t1, t1, 0 - srli.d t0, t0, PMD_SHIFT - andi t0, t0, (PTRS_PER_PMD - 1) - slli.d t0, t0, 3 - add.d t1, t1, t0 + ld.d t1, t1, 0 + bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT + alsl.d t1, ra, t1, 3 #endif - ld.d ra, t1, 0 + ld.d ra, t1, 0 /* * For huge tlb entries, pmde doesn't contain an address but * instead contains the tlb pte. Check the PAGE_HUGE bit and * see if we need to jump to huge tlb processing. */ - andi t0, ra, _PAGE_HUGE - bnez t0, tlb_huge_update_modify + rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 + bltz ra, tlb_huge_update_modify - csrrd t0, LOONGARCH_CSR_BADV - srli.d t0, t0, PAGE_SHIFT - andi t0, t0, (PTRS_PER_PTE - 1) - slli.d t0, t0, _PTE_T_LOG2 - add.d t1, ra, t0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT + alsl.d t1, t0, ra, _PTE_T_LOG2 #ifdef CONFIG_SMP smp_pgtable_change_modify: -#endif -#ifdef CONFIG_SMP - ll.d t0, t1, 0 + ll.d t0, t1, 0 #else - ld.d t0, t1, 0 + ld.d t0, t1, 0 #endif - tlbsrch - - srli.d ra, t0, _PAGE_WRITE_SHIFT - andi ra, ra, 1 - beqz ra, nopage_tlb_modify + andi ra, t0, _PAGE_WRITE + beqz ra, nopage_tlb_modify - ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, smp_pgtable_change_modify + sc.d t0, t1, 0 + beqz t0, smp_pgtable_change_modify #else - st.d t0, t1, 0 + st.d t0, t1, 0 #endif - ori t1, t1, 8 - xori t1, t1, 8 - ld.d t0, t1, 0 - ld.d t1, t1, 8 - csrwr t0, LOONGARCH_CSR_TLBELO0 - csrwr t1, LOONGARCH_CSR_TLBELO1 + tlbsrch + bstrins.d t1, zero, 3, 3 + ld.d t0, t1, 0 + ld.d t1, t1, 8 + csrwr t0, LOONGARCH_CSR_TLBELO0 + csrwr t1, LOONGARCH_CSR_TLBELO1 tlbwr -leave_modify: - csrrd t0, EXCEPTION_KS0 - csrrd t1, EXCEPTION_KS1 - csrrd ra, EXCEPTION_KS2 + + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 ertn + #ifdef CONFIG_64BIT vmalloc_modify: - la.abs t1, swapper_pg_dir - b vmalloc_done_modify + la.abs t1, swapper_pg_dir + b vmalloc_done_modify #endif - /* - * This is the entry point when - * build_tlbchange_handler_head spots a huge page. - */ + /* This is the entry point of a huge page. */ tlb_huge_update_modify: #ifdef CONFIG_SMP - ll.d t0, t1, 0 -#else - ld.d t0, t1, 0 + ll.d ra, t1, 0 #endif - - srli.d ra, t0, _PAGE_WRITE_SHIFT - andi ra, ra, 1 - beqz ra, nopage_tlb_modify - - tlbsrch - ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + andi t0, ra, _PAGE_WRITE + beqz t0, nopage_tlb_modify #ifdef CONFIG_SMP - sc.d t0, t1, 0 - beqz t0, tlb_huge_update_modify - ld.d t0, t1, 0 + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + sc.d t0, t1, 0 + beqz t0, tlb_huge_update_modify + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) #else - st.d t0, t1, 0 + rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) + ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) + st.d t0, t1, 0 #endif /* * A huge PTE describes an area the size of the @@ -493,21 +446,20 @@ tlb_huge_update_modify: * address space. */ /* Huge page: Move Global bit */ - xori t0, t0, _PAGE_HUGE - lu12i.w t1, _PAGE_HGLOBAL >> 12 - and t1, t0, t1 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) - or t0, t0, t1 + xori t0, t0, _PAGE_HUGE + lu12i.w t1, _PAGE_HGLOBAL >> 12 + and t1, t0, t1 + srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) + or t0, t0, t1 - addi.d ra, t0, 0 - csrwr t0, LOONGARCH_CSR_TLBELO0 - addi.d t0, ra, 0 + move ra, t0 + csrwr ra, LOONGARCH_CSR_TLBELO0 /* Convert to entrylo1 */ - addi.d t1, zero, 1 - slli.d t1, t1, (HPAGE_SHIFT - 1) - add.d t0, t0, t1 - csrwr t0, LOONGARCH_CSR_TLBELO1 + addi.d t1, zero, 1 + slli.d t1, t1, (HPAGE_SHIFT - 1) + add.d t0, t0, t1 + csrwr t0, LOONGARCH_CSR_TLBELO1 /* Set huge page tlb entry size */ addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) @@ -521,26 +473,31 @@ tlb_huge_update_modify: addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) csrxchg t1, t0, LOONGARCH_CSR_TLBIDX + csrrd t0, EXCEPTION_KS0 + csrrd t1, EXCEPTION_KS1 + csrrd ra, EXCEPTION_KS2 + ertn + nopage_tlb_modify: - dbar 0 - csrrd ra, EXCEPTION_KS2 - la.abs t0, tlb_do_page_fault_1 - jr t0 + dbar 0 + csrrd ra, EXCEPTION_KS2 + la.abs t0, tlb_do_page_fault_1 + jr t0 SYM_FUNC_END(handle_tlb_modify) SYM_FUNC_START(handle_tlb_refill) - csrwr t0, LOONGARCH_CSR_TLBRSAVE - csrrd t0, LOONGARCH_CSR_PGD - lddir t0, t0, 3 + csrwr t0, LOONGARCH_CSR_TLBRSAVE + csrrd t0, LOONGARCH_CSR_PGD + lddir t0, t0, 3 #if CONFIG_PGTABLE_LEVELS > 3 - lddir t0, t0, 2 + lddir t0, t0, 2 #endif #if CONFIG_PGTABLE_LEVELS > 2 - lddir t0, t0, 1 + lddir t0, t0, 1 #endif - ldpte t0, 0 - ldpte t0, 1 + ldpte t0, 0 + ldpte t0, 1 tlbfill - csrrd t0, LOONGARCH_CSR_TLBRSAVE + csrrd t0, LOONGARCH_CSR_TLBRSAVE ertn SYM_FUNC_END(handle_tlb_refill) diff --git a/arch/loongarch/net/Makefile b/arch/loongarch/net/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..1ec12a0c324a50079d1efdcb34e9f2fe0f8393bf --- /dev/null +++ b/arch/loongarch/net/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for arch/loongarch/net +# +# Copyright (C) 2022 Loongson Technology Corporation Limited +# +obj-$(CONFIG_BPF_JIT) += bpf_jit.o diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c new file mode 100644 index 0000000000000000000000000000000000000000..43f0a98efe3808d046123808386424bd8466da29 --- /dev/null +++ b/arch/loongarch/net/bpf_jit.c @@ -0,0 +1,1179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * BPF JIT compiler for LoongArch + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include "bpf_jit.h" + +#define REG_TCC LOONGARCH_GPR_A6 +#define TCC_SAVED LOONGARCH_GPR_S5 + +#define SAVE_RA BIT(0) +#define SAVE_TCC BIT(1) + +static const int regmap[] = { + /* return value from in-kernel function, and exit value for eBPF program */ + [BPF_REG_0] = LOONGARCH_GPR_A5, + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = LOONGARCH_GPR_A0, + [BPF_REG_2] = LOONGARCH_GPR_A1, + [BPF_REG_3] = LOONGARCH_GPR_A2, + [BPF_REG_4] = LOONGARCH_GPR_A3, + [BPF_REG_5] = LOONGARCH_GPR_A4, + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = LOONGARCH_GPR_S0, + [BPF_REG_7] = LOONGARCH_GPR_S1, + [BPF_REG_8] = LOONGARCH_GPR_S2, + [BPF_REG_9] = LOONGARCH_GPR_S3, + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = LOONGARCH_GPR_S4, + /* temporary register for blinding constants */ + [BPF_REG_AX] = LOONGARCH_GPR_T0, +}; + +static void mark_call(struct jit_ctx *ctx) +{ + ctx->flags |= SAVE_RA; +} + +static void mark_tail_call(struct jit_ctx *ctx) +{ + ctx->flags |= SAVE_TCC; +} + +static bool seen_call(struct jit_ctx *ctx) +{ + return (ctx->flags & SAVE_RA); +} + +static bool seen_tail_call(struct jit_ctx *ctx) +{ + return (ctx->flags & SAVE_TCC); +} + +static u8 tail_call_reg(struct jit_ctx *ctx) +{ + if (seen_call(ctx)) + return TCC_SAVED; + + return REG_TCC; +} + +/* + * eBPF prog stack layout: + * + * high + * original $sp ------------> +-------------------------+ <--LOONGARCH_GPR_FP + * | $ra | + * +-------------------------+ + * | $fp | + * +-------------------------+ + * | $s0 | + * +-------------------------+ + * | $s1 | + * +-------------------------+ + * | $s2 | + * +-------------------------+ + * | $s3 | + * +-------------------------+ + * | $s4 | + * +-------------------------+ + * | $s5 | + * +-------------------------+ <--BPF_REG_FP + * | prog->aux->stack_depth | + * | (optional) | + * current $sp -------------> +-------------------------+ + * low + */ +static void build_prologue(struct jit_ctx *ctx) +{ + int stack_adjust = 0, store_offset, bpf_stack_adjust; + + bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + + /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */ + stack_adjust += sizeof(long) * 8; + + stack_adjust = round_up(stack_adjust, 16); + stack_adjust += bpf_stack_adjust; + + /* + * First instruction initializes the tail call count (TCC). + * On tail call we skip this instruction, and the TCC is + * passed in REG_TCC from the caller. + */ + emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); + + store_offset = stack_adjust - sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset); + + store_offset -= sizeof(long); + emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); + + if (bpf_stack_adjust) + emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); + + /* + * Program contains calls and tail calls, so REG_TCC need + * to be saved across calls. + */ + if (seen_tail_call(ctx) && seen_call(ctx)) + move_reg(ctx, TCC_SAVED, REG_TCC); + + ctx->stack_size = stack_adjust; +} + +static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) +{ + int stack_adjust = ctx->stack_size; + int load_offset; + + load_offset = stack_adjust - sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset); + + load_offset -= sizeof(long); + emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); + + if (!is_tail_call) { + /* Set return value */ + move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]); + /* Return to the caller */ + emit_insn(ctx, jirl, LOONGARCH_GPR_RA, LOONGARCH_GPR_ZERO, 0); + } else { + /* + * Call the next bpf prog and skip the first instruction + * of TCC initialization. + */ + emit_insn(ctx, jirl, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, 1); + } +} + +static void build_epilogue(struct jit_ctx *ctx) +{ + __build_epilogue(ctx, false); +} + +bool bpf_jit_supports_kfunc_call(void) +{ + return true; +} + +/* initialized on the first pass of build_body() */ +static int out_offset = -1; +static int emit_bpf_tail_call(struct jit_ctx *ctx) +{ + int off; + u8 tcc = tail_call_reg(ctx); + u8 a1 = LOONGARCH_GPR_A1; + u8 a2 = LOONGARCH_GPR_A2; + u8 t1 = LOONGARCH_GPR_T1; + u8 t2 = LOONGARCH_GPR_T2; + u8 t3 = LOONGARCH_GPR_T3; + const int idx0 = ctx->idx; + +#define cur_offset (ctx->idx - idx0) +#define jmp_offset (out_offset - (cur_offset)) + + /* + * a0: &ctx + * a1: &array + * a2: index + * + * if (index >= array->map.max_entries) + * goto out; + */ + off = offsetof(struct bpf_array, map.max_entries); + emit_insn(ctx, ldwu, t1, a1, off); + /* bgeu $a2, $t1, jmp_offset */ + if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) + goto toofar; + + /* + * if (--TCC < 0) + * goto out; + */ + emit_insn(ctx, addid, REG_TCC, tcc, -1); + if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0) + goto toofar; + + /* + * prog = array->ptrs[index]; + * if (!prog) + * goto out; + */ + emit_insn(ctx, alsld, t2, a2, a1, 2); + off = offsetof(struct bpf_array, ptrs); + emit_insn(ctx, ldd, t2, t2, off); + /* beq $t2, $zero, jmp_offset */ + if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) + goto toofar; + + /* goto *(prog->bpf_func + 4); */ + off = offsetof(struct bpf_prog, bpf_func); + emit_insn(ctx, ldd, t3, t2, off); + __build_epilogue(ctx, true); + + /* out: */ + if (out_offset == -1) + out_offset = cur_offset; + if (cur_offset != out_offset) { + pr_err_once("tail_call out_offset = %d, expected %d!\n", + cur_offset, out_offset); + return -1; + } + + return 0; + +toofar: + pr_info_once("tail_call: jump too far\n"); + return -1; +#undef cur_offset +#undef jmp_offset +} + +static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 t1 = LOONGARCH_GPR_T1; + const u8 t2 = LOONGARCH_GPR_T2; + const u8 t3 = LOONGARCH_GPR_T3; + const u8 src = regmap[insn->src_reg]; + const u8 dst = regmap[insn->dst_reg]; + const s16 off = insn->off; + const s32 imm = insn->imm; + const bool isdw = BPF_SIZE(insn->code) == BPF_DW; + + move_imm(ctx, t1, off, false); + emit_insn(ctx, addd, t1, dst, t1); + move_reg(ctx, t3, src); + + switch (imm) { + /* lock *(size *)(dst + off) = src */ + case BPF_ADD: + if (isdw) + emit_insn(ctx, amaddd, t2, t1, src); + else + emit_insn(ctx, amaddw, t2, t1, src); + break; + case BPF_AND: + if (isdw) + emit_insn(ctx, amandd, t2, t1, src); + else + emit_insn(ctx, amandw, t2, t1, src); + break; + case BPF_OR: + if (isdw) + emit_insn(ctx, amord, t2, t1, src); + else + emit_insn(ctx, amorw, t2, t1, src); + break; + case BPF_XOR: + if (isdw) + emit_insn(ctx, amxord, t2, t1, src); + else + emit_insn(ctx, amxorw, t2, t1, src); + break; + /* src = atomic_fetch_(dst + off, src) */ + case BPF_ADD | BPF_FETCH: + if (isdw) { + emit_insn(ctx, amaddd, src, t1, t3); + } else { + emit_insn(ctx, amaddw, src, t1, t3); + emit_zext_32(ctx, src, true); + } + break; + case BPF_AND | BPF_FETCH: + if (isdw) { + emit_insn(ctx, amandd, src, t1, t3); + } else { + emit_insn(ctx, amandw, src, t1, t3); + emit_zext_32(ctx, src, true); + } + break; + case BPF_OR | BPF_FETCH: + if (isdw) { + emit_insn(ctx, amord, src, t1, t3); + } else { + emit_insn(ctx, amorw, src, t1, t3); + emit_zext_32(ctx, src, true); + } + break; + case BPF_XOR | BPF_FETCH: + if (isdw) { + emit_insn(ctx, amxord, src, t1, t3); + } else { + emit_insn(ctx, amxorw, src, t1, t3); + emit_zext_32(ctx, src, true); + } + break; + /* src = atomic_xchg(dst + off, src); */ + case BPF_XCHG: + if (isdw) { + emit_insn(ctx, amswapd, src, t1, t3); + } else { + emit_insn(ctx, amswapw, src, t1, t3); + emit_zext_32(ctx, src, true); + } + break; + /* r0 = atomic_cmpxchg(dst + off, r0, src); */ + case BPF_CMPXCHG: + u8 r0 = regmap[BPF_REG_0]; + + move_reg(ctx, t2, r0); + if (isdw) { + emit_insn(ctx, lld, r0, t1, 0); + emit_insn(ctx, bne, t2, r0, 4); + move_reg(ctx, t3, src); + emit_insn(ctx, scd, t3, t1, 0); + emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4); + } else { + emit_insn(ctx, llw, r0, t1, 0); + emit_zext_32(ctx, t2, true); + emit_zext_32(ctx, r0, true); + emit_insn(ctx, bne, t2, r0, 4); + move_reg(ctx, t3, src); + emit_insn(ctx, scw, t3, t1, 0); + emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6); + emit_zext_32(ctx, r0, true); + } + break; + } +} + +static bool is_signed_bpf_cond(u8 cond) +{ + return cond == BPF_JSGT || cond == BPF_JSLT || + cond == BPF_JSGE || cond == BPF_JSLE; +} + +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) +{ + const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || + BPF_CLASS(insn->code) == BPF_JMP32; + const u8 code = insn->code; + const u8 cond = BPF_OP(code); + const u8 t1 = LOONGARCH_GPR_T1; + const u8 t2 = LOONGARCH_GPR_T2; + const u8 src = regmap[insn->src_reg]; + const u8 dst = regmap[insn->dst_reg]; + const s16 off = insn->off; + const s32 imm = insn->imm; + int jmp_offset; + int i = insn - ctx->prog->insnsi; + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + case BPF_ALU64 | BPF_MOV | BPF_X: + move_reg(ctx, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + case BPF_ALU64 | BPF_MOV | BPF_K: + move_imm(ctx, dst, imm, is32); + break; + + /* dst = dst + src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit_insn(ctx, addd, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst + imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + if (is_signed_imm12(imm)) { + emit_insn(ctx, addid, dst, dst, imm); + } else { + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, addd, dst, dst, t1); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst - src */ + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit_insn(ctx, subd, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst - imm */ + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + if (is_signed_imm12(-imm)) { + emit_insn(ctx, addid, dst, dst, -imm); + } else { + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, subd, dst, dst, t1); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst * src */ + case BPF_ALU | BPF_MUL | BPF_X: + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_insn(ctx, muld, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst * imm */ + case BPF_ALU | BPF_MUL | BPF_K: + case BPF_ALU64 | BPF_MUL | BPF_K: + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, muld, dst, dst, t1); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst / src */ + case BPF_ALU | BPF_DIV | BPF_X: + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_zext_32(ctx, dst, is32); + move_reg(ctx, t1, src); + emit_zext_32(ctx, t1, is32); + emit_insn(ctx, divdu, dst, dst, t1); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst / imm */ + case BPF_ALU | BPF_DIV | BPF_K: + case BPF_ALU64 | BPF_DIV | BPF_K: + move_imm(ctx, t1, imm, is32); + emit_zext_32(ctx, dst, is32); + emit_insn(ctx, divdu, dst, dst, t1); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst % src */ + case BPF_ALU | BPF_MOD | BPF_X: + case BPF_ALU64 | BPF_MOD | BPF_X: + emit_zext_32(ctx, dst, is32); + move_reg(ctx, t1, src); + emit_zext_32(ctx, t1, is32); + emit_insn(ctx, moddu, dst, dst, t1); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst % imm */ + case BPF_ALU | BPF_MOD | BPF_K: + case BPF_ALU64 | BPF_MOD | BPF_K: + move_imm(ctx, t1, imm, is32); + emit_zext_32(ctx, dst, is32); + emit_insn(ctx, moddu, dst, dst, t1); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst & src */ + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit_insn(ctx, and, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst & imm */ + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, andi, dst, dst, imm); + } else { + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, and, dst, dst, t1); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst | src */ + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit_insn(ctx, or, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst | imm */ + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, ori, dst, dst, imm); + } else { + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, or, dst, dst, t1); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst ^ src */ + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit_insn(ctx, xor, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst ^ imm */ + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, xori, dst, dst, imm); + } else { + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, xor, dst, dst, t1); + } + emit_zext_32(ctx, dst, is32); + break; + + /* dst = dst << src (logical) */ + case BPF_ALU | BPF_LSH | BPF_X: + emit_insn(ctx, sllw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_insn(ctx, slld, dst, dst, src); + break; + + /* dst = dst << imm (logical) */ + case BPF_ALU | BPF_LSH | BPF_K: + emit_insn(ctx, slliw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_LSH | BPF_K: + emit_insn(ctx, sllid, dst, dst, imm); + break; + + /* dst = dst >> src (logical) */ + case BPF_ALU | BPF_RSH | BPF_X: + emit_insn(ctx, srlw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_RSH | BPF_X: + emit_insn(ctx, srld, dst, dst, src); + break; + + /* dst = dst >> imm (logical) */ + case BPF_ALU | BPF_RSH | BPF_K: + emit_insn(ctx, srliw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_RSH | BPF_K: + emit_insn(ctx, srlid, dst, dst, imm); + break; + + /* dst = dst >> src (arithmetic) */ + case BPF_ALU | BPF_ARSH | BPF_X: + emit_insn(ctx, sraw, dst, dst, src); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_insn(ctx, srad, dst, dst, src); + break; + + /* dst = dst >> imm (arithmetic) */ + case BPF_ALU | BPF_ARSH | BPF_K: + emit_insn(ctx, sraiw, dst, dst, imm); + emit_zext_32(ctx, dst, is32); + break; + + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_insn(ctx, sraid, dst, dst, imm); + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_LE: + switch (imm) { + case 16: + /* zero-extend 16 bits into 64 bits */ + emit_insn(ctx, bstrpickd, dst, dst, 15, 0); + break; + case 32: + /* zero-extend 32 bits into 64 bits */ + emit_zext_32(ctx, dst, is32); + break; + case 64: + /* do nothing */ + break; + } + break; + + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + emit_insn(ctx, revb2h, dst, dst); + /* zero-extend 16 bits into 64 bits */ + emit_insn(ctx, bstrpickd, dst, dst, 15, 0); + break; + case 32: + emit_insn(ctx, revb2w, dst, dst); + /* zero-extend 32 bits into 64 bits */ + emit_zext_32(ctx, dst, is32); + break; + case 64: + emit_insn(ctx, revbd, dst, dst); + break; + } + break; + + /* PC += off if dst cond src */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP32 | BPF_JEQ | BPF_X: + case BPF_JMP32 | BPF_JNE | BPF_X: + case BPF_JMP32 | BPF_JGT | BPF_X: + case BPF_JMP32 | BPF_JGE | BPF_X: + case BPF_JMP32 | BPF_JLT | BPF_X: + case BPF_JMP32 | BPF_JLE | BPF_X: + case BPF_JMP32 | BPF_JSGT | BPF_X: + case BPF_JMP32 | BPF_JSGE | BPF_X: + case BPF_JMP32 | BPF_JSLT | BPF_X: + case BPF_JMP32 | BPF_JSLE | BPF_X: + jmp_offset = bpf2la_offset(i, off, ctx); + move_reg(ctx, t1, dst); + move_reg(ctx, t2, src); + if (is_signed_bpf_cond(BPF_OP(code))) { + emit_sext_32(ctx, t1, is32); + emit_sext_32(ctx, t2, is32); + } else { + emit_zext_32(ctx, t1, is32); + emit_zext_32(ctx, t2, is32); + } + if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0) + goto toofar; + break; + + /* PC += off if dst cond imm */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP32 | BPF_JEQ | BPF_K: + case BPF_JMP32 | BPF_JNE | BPF_K: + case BPF_JMP32 | BPF_JGT | BPF_K: + case BPF_JMP32 | BPF_JGE | BPF_K: + case BPF_JMP32 | BPF_JLT | BPF_K: + case BPF_JMP32 | BPF_JLE | BPF_K: + case BPF_JMP32 | BPF_JSGT | BPF_K: + case BPF_JMP32 | BPF_JSGE | BPF_K: + case BPF_JMP32 | BPF_JSLT | BPF_K: + case BPF_JMP32 | BPF_JSLE | BPF_K: + u8 t7 = -1; + jmp_offset = bpf2la_offset(i, off, ctx); + if (imm) { + move_imm(ctx, t1, imm, false); + t7 = t1; + } else { + /* If imm is 0, simply use zero register. */ + t7 = LOONGARCH_GPR_ZERO; + } + move_reg(ctx, t2, dst); + if (is_signed_bpf_cond(BPF_OP(code))) { + emit_sext_32(ctx, t7, is32); + emit_sext_32(ctx, t2, is32); + } else { + emit_zext_32(ctx, t7, is32); + emit_zext_32(ctx, t2, is32); + } + if (emit_cond_jmp(ctx, cond, t2, t7, jmp_offset) < 0) + goto toofar; + break; + + /* PC += off if dst & src */ + case BPF_JMP | BPF_JSET | BPF_X: + case BPF_JMP32 | BPF_JSET | BPF_X: + jmp_offset = bpf2la_offset(i, off, ctx); + emit_insn(ctx, and, t1, dst, src); + emit_zext_32(ctx, t1, is32); + if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) + goto toofar; + break; + + /* PC += off if dst & imm */ + case BPF_JMP | BPF_JSET | BPF_K: + case BPF_JMP32 | BPF_JSET | BPF_K: + jmp_offset = bpf2la_offset(i, off, ctx); + move_imm(ctx, t1, imm, is32); + emit_insn(ctx, and, t1, dst, t1); + emit_zext_32(ctx, t1, is32); + if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0) + goto toofar; + break; + + /* PC += off */ + case BPF_JMP | BPF_JA: + jmp_offset = bpf2la_offset(i, off, ctx); + if (emit_uncond_jmp(ctx, jmp_offset) < 0) + goto toofar; + break; + + /* function call */ + case BPF_JMP | BPF_CALL: + int ret; + u64 func_addr; + bool func_addr_fixed; + + mark_call(ctx); + ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, + &func_addr, &func_addr_fixed); + if (ret < 0) + return ret; + + move_imm(ctx, t1, func_addr, is32); + emit_insn(ctx, jirl, t1, LOONGARCH_GPR_RA, 0); + move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0); + break; + + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + mark_tail_call(ctx); + if (emit_bpf_tail_call(ctx) < 0) + return -EINVAL; + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + emit_sext_32(ctx, regmap[BPF_REG_0], true); + + if (i == ctx->prog->len - 1) + break; + + jmp_offset = epilogue_offset(ctx); + if (emit_uncond_jmp(ctx, jmp_offset) < 0) + goto toofar; + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; + + move_imm(ctx, dst, imm64, is32); + return 1; + + /* dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: + switch (BPF_SIZE(code)) { + case BPF_B: + if (is_signed_imm12(off)) { + emit_insn(ctx, ldbu, dst, src, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, ldxbu, dst, src, t1); + } + break; + case BPF_H: + if (is_signed_imm12(off)) { + emit_insn(ctx, ldhu, dst, src, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, ldxhu, dst, src, t1); + } + break; + case BPF_W: + if (is_signed_imm12(off)) { + emit_insn(ctx, ldwu, dst, src, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, ldptrw, dst, src, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, ldxwu, dst, src, t1); + } + break; + case BPF_DW: + if (is_signed_imm12(off)) { + emit_insn(ctx, ldd, dst, src, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, ldptrd, dst, src, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, ldxd, dst, src, t1); + } + break; + } + break; + + /* *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_DW: + switch (BPF_SIZE(code)) { + case BPF_B: + move_imm(ctx, t1, imm, is32); + if (is_signed_imm12(off)) { + emit_insn(ctx, stb, t1, dst, off); + } else { + move_imm(ctx, t2, off, is32); + emit_insn(ctx, stxb, t1, dst, t2); + } + break; + case BPF_H: + move_imm(ctx, t1, imm, is32); + if (is_signed_imm12(off)) { + emit_insn(ctx, sth, t1, dst, off); + } else { + move_imm(ctx, t2, off, is32); + emit_insn(ctx, stxh, t1, dst, t2); + } + break; + case BPF_W: + move_imm(ctx, t1, imm, is32); + if (is_signed_imm12(off)) { + emit_insn(ctx, stw, t1, dst, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, stptrw, t1, dst, off); + } else { + move_imm(ctx, t2, off, is32); + emit_insn(ctx, stxw, t1, dst, t2); + } + break; + case BPF_DW: + move_imm(ctx, t1, imm, is32); + if (is_signed_imm12(off)) { + emit_insn(ctx, std, t1, dst, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, stptrd, t1, dst, off); + } else { + move_imm(ctx, t2, off, is32); + emit_insn(ctx, stxd, t1, dst, t2); + } + break; + } + break; + + /* *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_DW: + switch (BPF_SIZE(code)) { + case BPF_B: + if (is_signed_imm12(off)) { + emit_insn(ctx, stb, src, dst, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, stxb, src, dst, t1); + } + break; + case BPF_H: + if (is_signed_imm12(off)) { + emit_insn(ctx, sth, src, dst, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, stxh, src, dst, t1); + } + break; + case BPF_W: + if (is_signed_imm12(off)) { + emit_insn(ctx, stw, src, dst, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, stptrw, src, dst, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, stxw, src, dst, t1); + } + break; + case BPF_DW: + if (is_signed_imm12(off)) { + emit_insn(ctx, std, src, dst, off); + } else if (is_signed_imm14(off)) { + emit_insn(ctx, stptrd, src, dst, off); + } else { + move_imm(ctx, t1, off, is32); + emit_insn(ctx, stxd, src, dst, t1); + } + break; + } + break; + + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + emit_atomic(insn, ctx); + break; + + default: + pr_err("bpf_jit: unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; + +toofar: + pr_info_once("bpf_jit: opcode %02x, jump too far\n", code); + return -E2BIG; +} + +static int build_body(struct jit_ctx *ctx, bool extra_pass) +{ + int i; + const struct bpf_prog *prog = ctx->prog; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + if (ctx->image == NULL) + ctx->offset[i] = ctx->idx; + + ret = build_insn(insn, ctx, extra_pass); + if (ret > 0) { + i++; + if (ctx->image == NULL) + ctx->offset[i] = ctx->idx; + continue; + } + if (ret) + return ret; + } + + if (ctx->image == NULL) + ctx->offset[i] = ctx->idx; + + return 0; +} + +/* Fill space with break instructions */ +static void jit_fill_hole(void *area, unsigned int size) +{ + u32 *ptr; + + /* We are guaranteed to have aligned memory */ + for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) + *ptr++ = INSN_BREAK; +} + +static int validate_code(struct jit_ctx *ctx) +{ + int i; + union loongarch_instruction insn; + + for (i = 0; i < ctx->idx; i++) { + insn = ctx->image[i]; + /* Check INSN_BREAK */ + if (insn.word == INSN_BREAK) + return -1; + } + + return 0; +} + +struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) +{ + bool tmp_blinded = false, extra_pass = false; + u8 *image_ptr; + int image_size; + struct jit_ctx ctx; + struct jit_data *jit_data; + struct bpf_binary_header *header; + struct bpf_prog *tmp, *orig_prog = prog; + + /* + * If BPF JIT was not enabled then we must fall back to + * the interpreter. + */ + if (!prog->jit_requested) + return orig_prog; + + tmp = bpf_jit_blind_constants(prog); + /* + * If blinding was requested and we failed during blinding, + * we must fall back to the interpreter. Otherwise, we save + * the new JITed code. + */ + if (IS_ERR(tmp)) + return orig_prog; + + if (tmp != prog) { + tmp_blinded = true; + prog = tmp; + } + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); + if (!jit_data) { + prog = orig_prog; + goto out; + } + prog->aux->jit_data = jit_data; + } + if (jit_data->ctx.offset) { + ctx = jit_data->ctx; + image_ptr = jit_data->image; + header = jit_data->header; + extra_pass = true; + image_size = sizeof(u32) * ctx.idx; + goto skip_init_ctx; + } + + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); + if (ctx.offset == NULL) { + prog = orig_prog; + goto out_offset; + } + + /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ + build_prologue(&ctx); + if (build_body(&ctx, extra_pass)) { + prog = orig_prog; + goto out_offset; + } + ctx.epilogue_offset = ctx.idx; + build_epilogue(&ctx); + + /* Now we know the actual image size. + * As each LoongArch instruction is of length 32bit, + * we are translating number of JITed intructions into + * the size required to store these JITed code. + */ + image_size = sizeof(u32) * ctx.idx; + /* Now we know the size of the structure to make */ + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (header == NULL) { + prog = orig_prog; + goto out_offset; + } + + /* 2. Now, the actual pass to generate final JIT code */ + ctx.image = (union loongarch_instruction *)image_ptr; + +skip_init_ctx: + ctx.idx = 0; + + build_prologue(&ctx); + if (build_body(&ctx, extra_pass)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_offset; + } + build_epilogue(&ctx); + + /* 3. Extra pass to validate JITed code */ + if (validate_code(&ctx)) { + bpf_jit_binary_free(header); + prog = orig_prog; + goto out_offset; + } + + /* And we're done */ + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, image_size, 2, ctx.image); + + /* Update the icache */ + flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx)); + + if (!prog->is_func || extra_pass) { + if (extra_pass && ctx.idx != jit_data->ctx.idx) { + pr_err_once("multi-func JIT bug %d != %d\n", + ctx.idx, jit_data->ctx.idx); + bpf_jit_binary_free(header); + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + goto out_offset; + } + bpf_jit_binary_lock_ro(header); + } else { + jit_data->ctx = ctx; + jit_data->image = image_ptr; + jit_data->header = header; + } + prog->jited = 1; + prog->jited_len = image_size; + prog->bpf_func = (void *)ctx.image; + + if (!prog->is_func || extra_pass) { + int i; + + /* offset[prog->len] is the size of program */ + for (i = 0; i <= prog->len; i++) + ctx.offset[i] *= LOONGARCH_INSN_SIZE; + bpf_prog_fill_jited_linfo(prog, ctx.offset + 1); + +out_offset: + kvfree(ctx.offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } + +out: + if (tmp_blinded) + bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); + + out_offset = -1; + + return prog; +} diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h new file mode 100644 index 0000000000000000000000000000000000000000..e665ddb0aeb85580d583098c88fd6995e1353da1 --- /dev/null +++ b/arch/loongarch/net/bpf_jit.h @@ -0,0 +1,282 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * BPF JIT compiler for LoongArch + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include + +struct jit_ctx { + const struct bpf_prog *prog; + unsigned int idx; + unsigned int flags; + unsigned int epilogue_offset; + u32 *offset; + union loongarch_instruction *image; + u32 stack_size; +}; + +struct jit_data { + struct bpf_binary_header *header; + u8 *image; + struct jit_ctx ctx; +}; + +#define emit_insn(ctx, func, ...) \ +do { \ + if (ctx->image != NULL) { \ + union loongarch_instruction *insn = &ctx->image[ctx->idx]; \ + emit_##func(insn, ##__VA_ARGS__); \ + } \ + ctx->idx++; \ +} while (0) + +#define is_signed_imm12(val) signed_imm_check(val, 12) +#define is_signed_imm14(val) signed_imm_check(val, 14) +#define is_signed_imm16(val) signed_imm_check(val, 16) +#define is_signed_imm26(val) signed_imm_check(val, 26) +#define is_signed_imm32(val) signed_imm_check(val, 32) +#define is_signed_imm52(val) signed_imm_check(val, 52) +#define is_unsigned_imm12(val) unsigned_imm_check(val, 12) + +static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx) +{ + /* BPF JMP offset is relative to the next instruction */ + bpf_insn++; + /* + * Whereas LoongArch branch instructions encode the offset + * from the branch itself, so we must subtract 1 from the + * instruction offset. + */ + return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1)); +} + +static inline int epilogue_offset(const struct jit_ctx *ctx) +{ + int from = ctx->idx; + int to = ctx->epilogue_offset; + + return (to - from); +} + +/* Zero-extend 32 bits into 64 bits */ +static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) +{ + if (!is32) + return; + + emit_insn(ctx, lu32id, reg, 0); +} + +/* Signed-extend 32 bits into 64 bits */ +static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) +{ + if (!is32) + return; + + emit_insn(ctx, addiw, reg, reg, 0); +} + +static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm, bool is32) +{ + long imm_11_0, imm_31_12, imm_51_32, imm_63_52, imm_51_0, imm_51_31; + + /* or rd, $zero, $zero */ + if (imm == 0) { + emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO); + return; + } + + /* addiw rd, $zero, imm_11_0 */ + if (is_signed_imm12(imm)) { + emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm); + goto zext; + } + + /* ori rd, $zero, imm_11_0 */ + if (is_unsigned_imm12(imm)) { + emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm); + goto zext; + } + + /* lu52id rd, $zero, imm_63_52 */ + imm_63_52 = (imm >> 52) & 0xfff; + imm_51_0 = imm & 0xfffffffffffff; + if (imm_63_52 != 0 && imm_51_0 == 0) { + emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, imm_63_52); + return; + } + + /* lu12iw rd, imm_31_12 */ + imm_31_12 = (imm >> 12) & 0xfffff; + emit_insn(ctx, lu12iw, rd, imm_31_12); + + /* ori rd, rd, imm_11_0 */ + imm_11_0 = imm & 0xfff; + if (imm_11_0 != 0) + emit_insn(ctx, ori, rd, rd, imm_11_0); + + if (!is_signed_imm32(imm)) { + if (imm_51_0 != 0) { + /* + * If bit[51:31] is all 0 or all 1, + * it means bit[51:32] is sign extended by lu12iw, + * no need to call lu32id to do a new filled operation. + */ + imm_51_31 = (imm >> 31) & 0x1fffff; + if (imm_51_31 != 0 || imm_51_31 != 0x1fffff) { + /* lu32id rd, imm_51_32 */ + imm_51_32 = (imm >> 32) & 0xfffff; + emit_insn(ctx, lu32id, rd, imm_51_32); + } + } + + /* lu52id rd, rd, imm_63_52 */ + if (!is_signed_imm52(imm)) + emit_insn(ctx, lu52id, rd, rd, imm_63_52); + } + +zext: + emit_zext_32(ctx, rd, is32); +} + +static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd, + enum loongarch_gpr rj) +{ + emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO); +} + +static inline int invert_jmp_cond(u8 cond) +{ + switch (cond) { + case BPF_JEQ: + return BPF_JNE; + case BPF_JNE: + case BPF_JSET: + return BPF_JEQ; + case BPF_JGT: + return BPF_JLE; + case BPF_JGE: + return BPF_JLT; + case BPF_JLT: + return BPF_JGE; + case BPF_JLE: + return BPF_JGT; + case BPF_JSGT: + return BPF_JSLE; + case BPF_JSGE: + return BPF_JSLT; + case BPF_JSLT: + return BPF_JSGE; + case BPF_JSLE: + return BPF_JSGT; + } + return -1; +} + +static inline void cond_jmp_offset(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + switch (cond) { + case BPF_JEQ: + /* PC += jmp_offset if rj == rd */ + emit_insn(ctx, beq, rj, rd, jmp_offset); + return; + case BPF_JNE: + case BPF_JSET: + /* PC += jmp_offset if rj != rd */ + emit_insn(ctx, bne, rj, rd, jmp_offset); + return; + case BPF_JGT: + /* PC += jmp_offset if rj > rd (unsigned) */ + emit_insn(ctx, bltu, rd, rj, jmp_offset); + return; + case BPF_JLT: + /* PC += jmp_offset if rj < rd (unsigned) */ + emit_insn(ctx, bltu, rj, rd, jmp_offset); + return; + case BPF_JGE: + /* PC += jmp_offset if rj >= rd (unsigned) */ + emit_insn(ctx, bgeu, rj, rd, jmp_offset); + return; + case BPF_JLE: + /* PC += jmp_offset if rj <= rd (unsigned) */ + emit_insn(ctx, bgeu, rd, rj, jmp_offset); + return; + case BPF_JSGT: + /* PC += jmp_offset if rj > rd (signed) */ + emit_insn(ctx, blt, rd, rj, jmp_offset); + return; + case BPF_JSLT: + /* PC += jmp_offset if rj < rd (signed) */ + emit_insn(ctx, blt, rj, rd, jmp_offset); + return; + case BPF_JSGE: + /* PC += jmp_offset if rj >= rd (signed) */ + emit_insn(ctx, bge, rj, rd, jmp_offset); + return; + case BPF_JSLE: + /* PC += jmp_offset if rj <= rd (signed) */ + emit_insn(ctx, bge, rd, rj, jmp_offset); + return; + } +} + +static inline void cond_jmp_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + cond = invert_jmp_cond(cond); + cond_jmp_offset(ctx, cond, rj, rd, 2); + emit_insn(ctx, b, jmp_offset); +} + +static inline void uncond_jmp_offs26(struct jit_ctx *ctx, int jmp_offset) +{ + emit_insn(ctx, b, jmp_offset); +} + +static inline int emit_cond_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + /* + * A large PC-relative jump offset may overflow the immediate field of + * the native conditional branch instruction, triggering a conversion + * to use an absolute jump instead, this jump sequence is particularly + * nasty. For now, use cond_jmp_offs26() directly to keep it simple. + * In the future, maybe we can add support for far branching, the branch + * relaxation requires more than two passes to converge, the code seems + * too complex to understand, not quite sure whether it is necessary and + * worth the extra pain. Anyway, just leave it as it is to enhance code + * readability now. + */ + if (is_signed_imm26(jmp_offset)) { + cond_jmp_offs26(ctx, cond, rj, rd, jmp_offset); + return 0; + } + + return -EINVAL; +} + +static inline int emit_uncond_jmp(struct jit_ctx *ctx, int jmp_offset) +{ + if (is_signed_imm26(jmp_offset)) { + uncond_jmp_offs26(ctx, jmp_offset); + return 0; + } + + return -EINVAL; +} + +static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, + enum loongarch_gpr rd, int jmp_offset) +{ + if (is_signed_imm16(jmp_offset)) { + cond_jmp_offset(ctx, cond, rj, rd, jmp_offset); + return 0; + } + + return -EINVAL; +} diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c index bf921487333c65dc035bc246094961390125eb02..8235ec92b41fe2b35b66848447a94698d3aaeb04 100644 --- a/arch/loongarch/pci/acpi.c +++ b/arch/loongarch/pci/acpi.c @@ -82,6 +82,69 @@ static int acpi_prepare_root_resources(struct acpi_pci_root_info *ci) return 0; } +/* + * Create a PCI config space window + * - reserve mem region + * - alloc struct pci_config_window with space for all mappings + * - ioremap the config space + */ +static struct pci_config_window *arch_pci_ecam_create(struct device *dev, + struct resource *cfgres, struct resource *busr, const struct pci_ecam_ops *ops) +{ + int bsz, bus_range, err; + struct resource *conflict; + struct pci_config_window *cfg; + + if (busr->start > busr->end) + return ERR_PTR(-EINVAL); + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return ERR_PTR(-ENOMEM); + + cfg->parent = dev; + cfg->ops = ops; + cfg->busr.start = busr->start; + cfg->busr.end = busr->end; + cfg->busr.flags = IORESOURCE_BUS; + bus_range = resource_size(cfgres) >> ops->bus_shift; + + bsz = 1 << ops->bus_shift; + + cfg->res.start = cfgres->start; + cfg->res.end = cfgres->end; + cfg->res.flags = IORESOURCE_MEM | IORESOURCE_BUSY; + cfg->res.name = "PCI ECAM"; + + conflict = request_resource_conflict(&iomem_resource, &cfg->res); + if (conflict) { + err = -EBUSY; + dev_err(dev, "can't claim ECAM area %pR: address conflict with %s %pR\n", + &cfg->res, conflict->name, conflict); + goto err_exit; + } + + cfg->win = pci_remap_cfgspace(cfgres->start, bus_range * bsz); + if (!cfg->win) + goto err_exit_iomap; + + if (ops->init) { + err = ops->init(cfg); + if (err) + goto err_exit; + } + dev_info(dev, "ECAM at %pR for %pR\n", &cfg->res, &cfg->busr); + + return cfg; + +err_exit_iomap: + err = -ENOMEM; + dev_err(dev, "ECAM ioremap failed\n"); +err_exit: + pci_ecam_free(cfg); + return ERR_PTR(err); +} + /* * Lookup the bus range for the domain in MCFG, and set up config space * mapping. @@ -106,11 +169,16 @@ pci_acpi_setup_ecam_mapping(struct acpi_pci_root *root) bus_shift = ecam_ops->bus_shift ? : 20; - cfgres.start = root->mcfg_addr + (bus_res->start << bus_shift); - cfgres.end = cfgres.start + (resource_size(bus_res) << bus_shift) - 1; - cfgres.flags = IORESOURCE_MEM; + if (bus_shift == 20) + cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops); + else { + cfgres.start = root->mcfg_addr + (bus_res->start << bus_shift); + cfgres.end = cfgres.start + (resource_size(bus_res) << bus_shift) - 1; + cfgres.end |= BIT(28) + (((PCI_CFG_SPACE_EXP_SIZE - 1) & 0xf00) << 16); + cfgres.flags = IORESOURCE_MEM; + cfg = arch_pci_ecam_create(dev, &cfgres, bus_res, ecam_ops); + } - cfg = pci_ecam_create(dev, &cfgres, bus_res, ecam_ops); if (IS_ERR(cfg)) { dev_err(dev, "%04x:%pR error %ld mapping ECAM\n", seg, bus_res, PTR_ERR(cfg)); return NULL; diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index e9b7c34d9b6d899daa0cefcce597fd34817813ff..2726639150bc7ab66402381a09fe3d1d7b4af8c3 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #define PCI_DEVICE_ID_LOONGSON_HOST 0x7a00 @@ -45,12 +46,10 @@ static int __init pcibios_init(void) unsigned int lsize; /* - * Set PCI cacheline size to that of the highest level in the + * Set PCI cacheline size to that of the last level in the * cache hierarchy. */ - lsize = cpu_dcache_line_size(); - lsize = cpu_vcache_line_size() ? : lsize; - lsize = cpu_scache_line_size() ? : lsize; + lsize = cpu_last_level_cache_line_size(); BUG_ON(!lsize); diff --git a/arch/loongarch/vdso/vgetcpu.c b/arch/loongarch/vdso/vgetcpu.c index 43a0078e441854cf2787a8fcf3fb97f2ed442c9f..e02e775f53608119ccd32969398f8279634ea814 100644 --- a/arch/loongarch/vdso/vgetcpu.c +++ b/arch/loongarch/vdso/vgetcpu.c @@ -24,6 +24,8 @@ static __always_inline const struct vdso_pcpu_data *get_pcpu_data(void) return (struct vdso_pcpu_data *)(get_vdso_base() - VDSO_DATA_SIZE); } +extern +int __vdso_getcpu(unsigned int *cpu, unsigned int *node, struct getcpu_cache *unused); int __vdso_getcpu(unsigned int *cpu, unsigned int *node, struct getcpu_cache *unused) { int cpu_id; diff --git a/arch/loongarch/vdso/vgettimeofday.c b/arch/loongarch/vdso/vgettimeofday.c index b1f4548dae924c455df460a8650397e6babc8481..8f22863bd7ea813bee095b2d377328fecd4af709 100644 --- a/arch/loongarch/vdso/vgettimeofday.c +++ b/arch/loongarch/vdso/vgettimeofday.c @@ -6,20 +6,23 @@ */ #include -int __vdso_clock_gettime(clockid_t clock, - struct __kernel_timespec *ts) +extern +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) { return __cvdso_clock_gettime(clock, ts); } -int __vdso_gettimeofday(struct __kernel_old_timeval *tv, - struct timezone *tz) +extern +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) { return __cvdso_gettimeofday(tv, tz); } -int __vdso_clock_getres(clockid_t clock_id, - struct __kernel_timespec *res) +extern +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res); +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) { return __cvdso_clock_getres(clock_id, res); } diff --git a/arch/m68k/68000/Makefile b/arch/m68k/68000/Makefile index 674541fdf5b892f934253ac55cae3f4cb1a5484f..279560add5776261545b52a5e07ef49f56345413 100644 --- a/arch/m68k/68000/Makefile +++ b/arch/m68k/68000/Makefile @@ -17,4 +17,4 @@ obj-$(CONFIG_DRAGEN2) += dragen2.o obj-$(CONFIG_UCSIMM) += ucsimm.o obj-$(CONFIG_UCDIMM) += ucsimm.o -extra-y := head.o +obj-y += head.o diff --git a/arch/m68k/68000/ints.c b/arch/m68k/68000/ints.c index cda49b12d7be9ce40520bcd55544351cf9a81d0e..f9a5ec781408f332d6a86005b02330925513e320 100644 --- a/arch/m68k/68000/ints.c +++ b/arch/m68k/68000/ints.c @@ -18,12 +18,12 @@ #include #include -#if defined(CONFIG_M68328) -#include -#elif defined(CONFIG_M68EZ328) +#if defined(CONFIG_M68EZ328) #include #elif defined(CONFIG_M68VZ328) #include +#else +#include #endif /* assembler routines */ diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index b06faf6c0b27c2656ed446f5c26140720ee1cc49..7bff881185070767189289019951edcf2d858a75 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -87,7 +87,7 @@ config MMU_SUN3 config KEXEC bool "kexec system call" - depends on M68KCLASSIC + depends on M68KCLASSIC && MMU select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index e0e9e31339c1250dce4ea39d95540689186533a2..9380f6e3bb6609f08bd9f091bd6985f96e31d2ab 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -46,6 +46,7 @@ config M68000 select GENERIC_CSUM select CPU_NO_EFFICIENT_FFS select HAVE_ARCH_HASH + select LEGACY_TIMER_TICK help The Freescale (was Motorola) 68000 CPU is the first generation of the well known M68K family of processors. The CPU core as well as @@ -97,7 +98,6 @@ config M68060 config M68328 bool depends on !MMU - select LEGACY_TIMER_TICK select M68000 help Motorola 68328 processor support. @@ -105,7 +105,6 @@ config M68328 config M68EZ328 bool depends on !MMU - select LEGACY_TIMER_TICK select M68000 help Motorola 68EX328 processor support. @@ -113,7 +112,6 @@ config M68EZ328 config M68VZ328 bool depends on !MMU - select LEGACY_TIMER_TICK select M68000 help Motorola 68VZ328 processor support. @@ -399,7 +397,7 @@ config SINGLE_MEMORY_CHUNK order" to save memory that could be wasted for unused memory map. Say N if not sure. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" if ADVANCED depends on !SINGLE_MEMORY_CHUNK default "11" diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile index e358605b70baafc5cebfbc09c96c9c60ffa5d6cb..43e39040d3ac6cd38a4bd4fc3dc04e03d5c71bf5 100644 --- a/arch/m68k/Makefile +++ b/arch/m68k/Makefile @@ -86,15 +86,6 @@ ifdef CONFIG_KGDB KBUILD_CFLAGS := $(subst -fomit-frame-pointer,,$(KBUILD_CFLAGS)) -g endif -# -# Select the assembler head startup code. Order is important. The default -# head code is first, processor specific selections can override it after. -# -head-y := arch/m68k/kernel/head.o -head-$(CONFIG_SUN3) := arch/m68k/kernel/sun3-head.o -head-$(CONFIG_M68000) := arch/m68k/68000/head.o -head-$(CONFIG_COLDFIRE) := arch/m68k/coldfire/head.o - libs-y += arch/m68k/lib/ diff --git a/arch/m68k/coldfire/Makefile b/arch/m68k/coldfire/Makefile index 9419a6c1f036dd6f304dafecdf375928786d7ad5..c56bc0dc7f2e338c8275cfd15656f80e78825254 100644 --- a/arch/m68k/coldfire/Makefile +++ b/arch/m68k/coldfire/Makefile @@ -45,4 +45,4 @@ obj-$(CONFIG_STMARK2) += stmark2.o obj-$(CONFIG_PCI) += pci.o obj-y += gpio.o -extra-y := head.o +obj-y += head.o diff --git a/arch/m68k/configs/amcore_defconfig b/arch/m68k/configs/amcore_defconfig index 6d9ed2198170a984ea0679f173c10c0951568025..041adcf6ecfc34be4f4e7ed685d488f1de695560 100644 --- a/arch/m68k/configs/amcore_defconfig +++ b/arch/m68k/configs/amcore_defconfig @@ -27,9 +27,6 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set # CONFIG_UEVENT_HELPER is not set @@ -85,7 +82,6 @@ CONFIG_ROMFS_FS=y CONFIG_ROMFS_BACKED_BY_BOTH=y # CONFIG_NETWORK_FILESYSTEMS is not set CONFIG_PRINTK_TIME=y -# CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set CONFIG_PANIC_ON_OOPS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index a6a886a89be28382533966173a447f1c75f04200..e2038d9499e479cdc4e524173fcf8cde02f1c601 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -84,7 +84,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -573,9 +572,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -594,6 +593,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index bffd24c2755e711716066174ea13881edf2671c2..ddd201259e43a95a528e7572de256112f242e62e 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -80,7 +80,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -530,9 +529,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -551,6 +550,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 0013425b1e08ed4b4261a03012cfc50b7d99873f..d9f7837073876721c5a54d9e8d132f01f8885a49 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -87,7 +87,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -550,9 +549,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -571,6 +570,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 42d969697f7f0352332fb241df30293e75bef33d..68957c6bcff1a1be5d71d44e4b7d4633dd080e07 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -77,7 +77,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -522,9 +521,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -543,6 +542,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 97d6d9acb39520ab07ae8cf44532b933a43b92ec..825c6a02fa9d102f535404355821a001edfb25b7 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -79,7 +79,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -532,9 +531,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -553,6 +552,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/m5208evb_defconfig b/arch/m68k/configs/m5208evb_defconfig index 0ee3079f6ca983f19091c7747b37629002bd34d1..31035a0b924735afe60c5a3debbbc165098217a7 100644 --- a/arch/m68k/configs/m5208evb_defconfig +++ b/arch/m68k/configs/m5208evb_defconfig @@ -21,9 +21,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/m5249evb_defconfig b/arch/m68k/configs/m5249evb_defconfig index f84f68c04065b13109de2f11119c1b80e7adce62..5706d7a1daba33de84690aafeab9799749e07559 100644 --- a/arch/m68k/configs/m5249evb_defconfig +++ b/arch/m68k/configs/m5249evb_defconfig @@ -22,9 +22,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/m5272c3_defconfig b/arch/m68k/configs/m5272c3_defconfig index eca65020aae356f983e3ed5e9b4f940d98e574bf..f02fe144f4adca3e0c0d02a62f1371b452cb48e0 100644 --- a/arch/m68k/configs/m5272c3_defconfig +++ b/arch/m68k/configs/m5272c3_defconfig @@ -22,9 +22,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/m5275evb_defconfig b/arch/m68k/configs/m5275evb_defconfig index 9402c7a3e9c77f8a342280a7a796f86f9a270529..781f307ff330d42e941c081967dcb2aac423fb1e 100644 --- a/arch/m68k/configs/m5275evb_defconfig +++ b/arch/m68k/configs/m5275evb_defconfig @@ -22,9 +22,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/m5307c3_defconfig b/arch/m68k/configs/m5307c3_defconfig index bb8b0eb4bdfcfb77610e93736477bcaab52e4751..6eac482356ca768db5177e24371a671d7e36c06e 100644 --- a/arch/m68k/configs/m5307c3_defconfig +++ b/arch/m68k/configs/m5307c3_defconfig @@ -22,9 +22,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/m5407c3_defconfig b/arch/m68k/configs/m5407c3_defconfig index ce9ccf13c7c0861b275e36188f2a0b226c3c977e..496dcccb1c182c9b29025c665cf8caf790a339b6 100644 --- a/arch/m68k/configs/m5407c3_defconfig +++ b/arch/m68k/configs/m5407c3_defconfig @@ -23,9 +23,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_FW_LOADER is not set diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 8cbfc1c659a311b60a6da3b870e5af0fcec05ebd..17f64c562bf119e8085d28655e153cc105ef4dee 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -78,7 +78,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -552,9 +551,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -573,6 +572,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 9f45fe60757fab1cdf07c845955ec467b6add4d1..f5f4c572b6949698d3f3b716b5dcf55942ae417f 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -98,7 +98,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -638,9 +637,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -659,6 +658,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 4736cfacf6a2801cdcebdf745900fb2a011d94d7..b4a0bbef7e39b490be03ff7d78e746467dc232b4 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -76,7 +76,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -521,9 +520,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -542,6 +541,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 638cd38aa7d2730328915e6674b8365a09cd45df..c6a6d592679340215e929b0bae24e5bcfb3737e5 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -77,7 +77,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -522,9 +521,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -543,6 +542,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index ec8b6bb70ebdcaf35f0e6b9e1062ca4d4a572cfd..49c9c89f0caf76ee672bb1054e17dc59aa42576f 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -78,7 +78,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -539,9 +538,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -560,6 +559,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 7d8dc578d59c6ffb306312b7ff7196c9142e5dbe..9b44eeb9c07f7aa91fdbe346c656edb41e02dccb 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -74,7 +74,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -521,9 +520,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -542,6 +541,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 96290aee530211e30af2abd55ca8007e5ca64f87..d2ffb0a65b44ac471e71a03241aae3c5b934d92b 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -74,7 +74,6 @@ CONFIG_NETFILTER=y CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_ZONES=y -# CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set CONFIG_NF_CONNTRACK_AMANDA=m CONFIG_NF_CONNTRACK_FTP=m @@ -520,9 +519,9 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_BLAKE2S=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m @@ -541,6 +540,7 @@ CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m diff --git a/arch/m68k/include/asm/bitops.h b/arch/m68k/include/asm/bitops.h index 470aed978590394b63c3337471e2cfc1ef3c81e9..e984af71df6bee14a90af4ad9d21b57c95e6f8ed 100644 --- a/arch/m68k/include/asm/bitops.h +++ b/arch/m68k/include/asm/bitops.h @@ -157,11 +157,8 @@ arch___change_bit(unsigned long nr, volatile unsigned long *addr) change_bit(nr, addr); } -static __always_inline bool -arch_test_bit(unsigned long nr, const volatile unsigned long *addr) -{ - return (addr[nr >> 5] & (1UL << (nr & 31))) != 0; -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire static inline int bset_reg_test_and_set_bit(int nr, volatile unsigned long *vaddr) diff --git a/arch/m68k/include/asm/processor.h b/arch/m68k/include/asm/processor.h index d86b4009880b4e790c4abeb4cda8fbc38be7e238..7a2da780830b8f628a8cbaa25b5333cbc36a8e50 100644 --- a/arch/m68k/include/asm/processor.h +++ b/arch/m68k/include/asm/processor.h @@ -145,11 +145,6 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, /* Forward declaration, a strange C thing */ struct task_struct; -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - unsigned long __get_wchan(struct task_struct *p); void show_registers(struct pt_regs *regs); diff --git a/arch/m68k/include/uapi/asm/bootinfo-virt.h b/arch/m68k/include/uapi/asm/bootinfo-virt.h index b091ee9b06e05ba4dec172fa47fed5a29dc1d687..7dbcd7bec103446de14401831395d915b257f577 100644 --- a/arch/m68k/include/uapi/asm/bootinfo-virt.h +++ b/arch/m68k/include/uapi/asm/bootinfo-virt.h @@ -13,13 +13,8 @@ #define BI_VIRT_VIRTIO_BASE 0x8004 #define BI_VIRT_CTRL_BASE 0x8005 -/* - * A random seed used to initialize the RNG. Record format: - * - * - length [ 2 bytes, 16-bit big endian ] - * - seed data [ `length` bytes, padded to preserve 2-byte alignment ] - */ -#define BI_VIRT_RNG_SEED 0x8006 +/* No longer used -- replaced with BI_RNG_SEED -- but don't reuse this index: + * #define BI_VIRT_RNG_SEED 0x8006 */ #define VIRT_BOOTI_VERSION MK_BI_VERSION(2, 0) diff --git a/arch/m68k/include/uapi/asm/bootinfo.h b/arch/m68k/include/uapi/asm/bootinfo.h index 95ecf3ae4c49ffaf163ab64560009d1ff15e3052..024e87d7095f8feffe13084f47592219c323e690 100644 --- a/arch/m68k/include/uapi/asm/bootinfo.h +++ b/arch/m68k/include/uapi/asm/bootinfo.h @@ -64,6 +64,13 @@ struct mem_info { /* (struct mem_info) */ #define BI_COMMAND_LINE 0x0007 /* kernel command line parameters */ /* (string) */ +/* + * A random seed used to initialize the RNG. Record format: + * + * - length [ 2 bytes, 16-bit big endian ] + * - seed data [ `length` bytes, padded to preserve 4-byte struct alignment ] + */ +#define BI_RNG_SEED 0x0008 /* diff --git a/arch/m68k/kernel/Makefile b/arch/m68k/kernel/Makefile index c0833da6a2ca6b8f608267ad6aa41c363faccca6..af015447dfb4c1f98e12830b8ff9d8e5d774a484 100644 --- a/arch/m68k/kernel/Makefile +++ b/arch/m68k/kernel/Makefile @@ -3,19 +3,20 @@ # Makefile for the linux kernel. # -extra-$(CONFIG_AMIGA) := head.o -extra-$(CONFIG_ATARI) := head.o -extra-$(CONFIG_MAC) := head.o -extra-$(CONFIG_APOLLO) := head.o -extra-$(CONFIG_VME) := head.o -extra-$(CONFIG_HP300) := head.o -extra-$(CONFIG_Q40) := head.o -extra-$(CONFIG_SUN3X) := head.o -extra-$(CONFIG_VIRT) := head.o -extra-$(CONFIG_SUN3) := sun3-head.o extra-y += vmlinux.lds -obj-y := entry.o irq.o module.o process.o ptrace.o +obj-$(CONFIG_AMIGA) := head.o +obj-$(CONFIG_ATARI) := head.o +obj-$(CONFIG_MAC) := head.o +obj-$(CONFIG_APOLLO) := head.o +obj-$(CONFIG_VME) := head.o +obj-$(CONFIG_HP300) := head.o +obj-$(CONFIG_Q40) := head.o +obj-$(CONFIG_SUN3X) := head.o +obj-$(CONFIG_VIRT) := head.o +obj-$(CONFIG_SUN3) := sun3-head.o + +obj-y += entry.o irq.o module.o process.o ptrace.o obj-y += setup.o signal.o sys_m68k.o syscalltable.o time.o traps.o obj-$(CONFIG_MMU_MOTOROLA) += ints.o vectors.o diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index e62fa8f2149b3574df6bed602e28571aa7d73d0c..3a2bb2e8fdad47b44f3ff442325d3d7582eaea4b 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -109,10 +110,9 @@ extern void paging_init(void); static void __init m68k_parse_bootinfo(const struct bi_record *record) { + const struct bi_record *first_record = record; uint16_t tag; - save_bootinfo(record); - while ((tag = be16_to_cpu(record->tag)) != BI_LAST) { int unknown = 0; const void *data = record->data; @@ -148,10 +148,21 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record) break; case BI_COMMAND_LINE: - strlcpy(m68k_command_line, data, + strscpy(m68k_command_line, data, sizeof(m68k_command_line)); break; + case BI_RNG_SEED: { + u16 len = be16_to_cpup(data); + add_bootloader_randomness(data + 2, len); + /* + * Zero the data to preserve forward secrecy, and zero the + * length to prevent kexec from using it. + */ + memzero_explicit((void *)data, len + 2); + break; + } + default: if (MACH_IS_AMIGA) unknown = amiga_parse_bootinfo(record); @@ -182,6 +193,8 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record) record = (struct bi_record *)((unsigned long)record + size); } + save_bootinfo(first_record); + m68k_realnum_memory = m68k_num_memory; #ifdef CONFIG_SINGLE_MEMORY_CHUNK if (m68k_num_memory > 1) { diff --git a/arch/m68k/virt/config.c b/arch/m68k/virt/config.c index 4ab22946ff68fec81b8f2fc45b029b99fa51b179..632ba200ad425245eb22f0f73b15307e523b0cd5 100644 --- a/arch/m68k/virt/config.c +++ b/arch/m68k/virt/config.c @@ -2,7 +2,6 @@ #include #include -#include #include #include @@ -93,16 +92,6 @@ int __init virt_parse_bootinfo(const struct bi_record *record) data += 4; virt_bi_data.virtio.irq = be32_to_cpup(data); break; - case BI_VIRT_RNG_SEED: { - u16 len = be16_to_cpup(data); - add_bootloader_randomness(data + 2, len); - /* - * Zero the data to preserve forward secrecy, and zero the - * length to prevent kexec from using it. - */ - memzero_explicit((void *)data, len + 2); - break; - } default: unknown = 1; break; diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 996132a5ef3505c31d536f63cea9fa1fbf5b1bae..4ebb56d6d9592c6a04dc70c4b7a56d45102e4130 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -204,6 +204,16 @@ config TASK_SIZE hex "Size of user task space" if TASK_SIZE_BOOL default "0x80000000" +config MB_MANAGER + bool "Support for Microblaze Manager" + depends on ADVANCED_OPTIONS + help + This option enables API for configuring the MicroBlaze manager + control register, which is consumed by the break handler to + block the break. + + Say N here unless you know what you are doing. + endmenu menu "Bus Options" diff --git a/arch/microblaze/Makefile b/arch/microblaze/Makefile index 1826d9ce4459e9491cc2b414c3a6da9ac36e06f2..3f8a86c4336a86329728678fcbc0a8d3d1380686 100644 --- a/arch/microblaze/Makefile +++ b/arch/microblaze/Makefile @@ -48,7 +48,6 @@ CPUFLAGS-1 += $(call cc-option,-mcpu=v$(CPU_VER)) # r31 holds current when in kernel mode KBUILD_CFLAGS += -ffixed-r31 $(CPUFLAGS-y) $(CPUFLAGS-1) $(CPUFLAGS-2) -head-y := arch/microblaze/kernel/head.o libs-y += arch/microblaze/lib/ boot := arch/microblaze/boot diff --git a/arch/microblaze/configs/mmu_defconfig b/arch/microblaze/configs/mmu_defconfig index 51337fffb9473104bb57920877056f4807becb0f..8150daf04a76c37255eaee624d57504e40e6770c 100644 --- a/arch/microblaze/configs/mmu_defconfig +++ b/arch/microblaze/configs/mmu_defconfig @@ -83,7 +83,7 @@ CONFIG_CIFS=y CONFIG_CIFS_STATS2=y CONFIG_ENCRYPTED_KEYS=y CONFIG_DMA_CMA=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_KGDB=y CONFIG_KGDB_TESTS=y CONFIG_KGDB_KDB=y diff --git a/arch/microblaze/include/asm/processor.h b/arch/microblaze/include/asm/processor.h index 7e9e92670df3340f2acd27f8563f3f6493a5de39..4e193c7550dfa21cc7fb5cb8b55c22a0b0f00030 100644 --- a/arch/microblaze/include/asm/processor.h +++ b/arch/microblaze/include/asm/processor.h @@ -63,11 +63,6 @@ struct thread_struct { .pgdir = swapper_pg_dir, \ } -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - unsigned long __get_wchan(struct task_struct *p); /* The size allocated for kernel stacks. This _must_ be a power of two! */ diff --git a/arch/microblaze/include/asm/xilinx_mb_manager.h b/arch/microblaze/include/asm/xilinx_mb_manager.h new file mode 100644 index 0000000000000000000000000000000000000000..7b6995722b0c0a62e52966171d64faaf0ceff4a3 --- /dev/null +++ b/arch/microblaze/include/asm/xilinx_mb_manager.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Xilinx, Inc. + */ +#ifndef _XILINX_MB_MANAGER_H +#define _XILINX_MB_MANAGER_H + +# ifndef __ASSEMBLY__ + +#include + +/* + * When the break vector gets asserted because of error injection, the break + * signal must be blocked before exiting from the break handler, Below api + * updates the manager address and control register and error counter callback + * arguments, which will be used by the break handler to block the break and + * call the callback function. + */ +void xmb_manager_register(uintptr_t phys_baseaddr, u32 cr_val, + void (*callback)(void *data), + void *priv, void (*reset_callback)(void *data)); +asmlinkage void xmb_inject_err(void); + +# endif /* __ASSEMBLY__ */ + +/* Error injection offset */ +#define XMB_INJECT_ERR_OFFSET 0x200 + +#endif /* _XILINX_MB_MANAGER_H */ diff --git a/arch/microblaze/kernel/Makefile b/arch/microblaze/kernel/Makefile index 15a20eb814cef263976023c4bcee4823407a5cfe..4393bee64eaf80b5a3d2dee8aa59d31fdccf295b 100644 --- a/arch/microblaze/kernel/Makefile +++ b/arch/microblaze/kernel/Makefile @@ -12,9 +12,9 @@ CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_process.o = -pg endif -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y += dma.o exceptions.o \ +obj-y += head.o dma.o exceptions.o \ hw_exception_handler.o irq.o \ process.o prom.o ptrace.o \ reset.o setup.o signal.o sys_microblaze.o timer.o traps.o unwind.o diff --git a/arch/microblaze/kernel/asm-offsets.c b/arch/microblaze/kernel/asm-offsets.c index 47ee409508b1c227e13ac7ace2a62e96a75328cc..104c3ac5f30c88785b21055b43f38e72cca24a11 100644 --- a/arch/microblaze/kernel/asm-offsets.c +++ b/arch/microblaze/kernel/asm-offsets.c @@ -120,5 +120,12 @@ int main(int argc, char *argv[]) DEFINE(CC_FSR, offsetof(struct cpu_context, fsr)); BLANK(); + /* struct cpuinfo */ + DEFINE(CI_DCS, offsetof(struct cpuinfo, dcache_size)); + DEFINE(CI_DCL, offsetof(struct cpuinfo, dcache_line_length)); + DEFINE(CI_ICS, offsetof(struct cpuinfo, icache_size)); + DEFINE(CI_ICL, offsetof(struct cpuinfo, icache_line_length)); + BLANK(); + return 0; } diff --git a/arch/microblaze/kernel/entry.S b/arch/microblaze/kernel/entry.S index d875a0c01032bc258b30b22325e78bc64dbbe0a1..582d7256d8154726fbd0eb5e06be177da36c5cd6 100644 --- a/arch/microblaze/kernel/entry.S +++ b/arch/microblaze/kernel/entry.S @@ -27,9 +27,11 @@ #include #include +#include #include #include +#include #undef DEBUG @@ -287,6 +289,44 @@ syscall_debug_table: .text +.extern cpuinfo + +C_ENTRY(mb_flush_dcache): + addik r1, r1, -PT_SIZE + SAVE_REGS + + addik r3, r0, cpuinfo + lwi r7, r3, CI_DCS + lwi r8, r3, CI_DCL + sub r9, r7, r8 +1: + wdc.flush r9, r0 + bgtid r9, 1b + addk r9, r9, r8 + + RESTORE_REGS + addik r1, r1, PT_SIZE + rtsd r15, 8 + nop + +C_ENTRY(mb_invalidate_icache): + addik r1, r1, -PT_SIZE + SAVE_REGS + + addik r3, r0, cpuinfo + lwi r7, r3, CI_ICS + lwi r8, r3, CI_ICL + sub r9, r7, r8 +1: + wic r9, r0 + bgtid r9, 1b + addk r9, r9, r8 + + RESTORE_REGS + addik r1, r1, PT_SIZE + rtsd r15, 8 + nop + /* * User trap. * @@ -753,6 +793,160 @@ IRQ_return: /* MS: Make global symbol for debugging */ rtid r14, 0 nop +#ifdef CONFIG_MB_MANAGER + +#define PT_PID PT_SIZE +#define PT_TLBI PT_SIZE + 4 +#define PT_ZPR PT_SIZE + 8 +#define PT_TLBL0 PT_SIZE + 12 +#define PT_TLBH0 PT_SIZE + 16 + +C_ENTRY(_xtmr_manager_reset): + lwi r1, r0, xmb_manager_stackpointer + + /* Restore MSR */ + lwi r2, r1, PT_MSR + mts rmsr, r2 + bri 4 + + /* restore Special purpose registers */ + lwi r2, r1, PT_PID + mts rpid, r2 + + lwi r2, r1, PT_TLBI + mts rtlbx, r2 + + lwi r2, r1, PT_ZPR + mts rzpr, r2 + +#if CONFIG_XILINX_MICROBLAZE0_USE_FPU + lwi r2, r1, PT_FSR + mts rfsr, r2 +#endif + + /* restore all the tlb's */ + addik r3, r0, TOPHYS(tlb_skip) + addik r6, r0, PT_TLBL0 + addik r7, r0, PT_TLBH0 +restore_tlb: + add r6, r6, r1 + add r7, r7, r1 + lwi r2, r6, 0 + mts rtlblo, r2 + lwi r2, r7, 0 + mts rtlbhi, r2 + addik r6, r6, 4 + addik r7, r7, 4 + bgtid r3, restore_tlb + addik r3, r3, -1 + + lwi r5, r0, TOPHYS(xmb_manager_dev) + lwi r8, r0, TOPHYS(xmb_manager_reset_callback) + set_vms + /* return from reset need -8 to adjust for rtsd r15, 8 */ + addik r15, r0, ret_from_reset - 8 + rtbd r8, 0 + nop + +ret_from_reset: + set_bip /* Ints masked for state restore */ + VM_OFF + /* MS: Restore all regs */ + RESTORE_REGS + lwi r14, r1, PT_R14 + lwi r16, r1, PT_PC + addik r1, r1, PT_SIZE + 36 + rtbd r16, 0 + nop + +/* + * Break handler for MB Manager. Enter to _xmb_manager_break by + * injecting fault in one of the TMR Microblaze core. + * FIXME: This break handler supports getting + * called from kernel space only. + */ +C_ENTRY(_xmb_manager_break): + /* + * Reserve memory in the stack for context store/restore + * (which includes memory for storing tlbs (max two tlbs)) + */ + addik r1, r1, -PT_SIZE - 36 + swi r1, r0, xmb_manager_stackpointer + SAVE_REGS + swi r14, r1, PT_R14 /* rewrite saved R14 value */ + swi r16, r1, PT_PC; /* PC and r16 are the same */ + + lwi r6, r0, TOPHYS(xmb_manager_baseaddr) + lwi r7, r0, TOPHYS(xmb_manager_crval) + /* + * When the break vector gets asserted because of error injection, + * the break signal must be blocked before exiting from the + * break handler, below code configures the tmr manager + * control register to block break signal. + */ + swi r7, r6, 0 + + /* Save the special purpose registers */ + mfs r2, rpid + swi r2, r1, PT_PID + + mfs r2, rtlbx + swi r2, r1, PT_TLBI + + mfs r2, rzpr + swi r2, r1, PT_ZPR + +#if CONFIG_XILINX_MICROBLAZE0_USE_FPU + mfs r2, rfsr + swi r2, r1, PT_FSR +#endif + mfs r2, rmsr + swi r2, r1, PT_MSR + + /* Save all the tlb's */ + addik r3, r0, TOPHYS(tlb_skip) + addik r6, r0, PT_TLBL0 + addik r7, r0, PT_TLBH0 +save_tlb: + add r6, r6, r1 + add r7, r7, r1 + mfs r2, rtlblo + swi r2, r6, 0 + mfs r2, rtlbhi + swi r2, r7, 0 + addik r6, r6, 4 + addik r7, r7, 4 + bgtid r3, save_tlb + addik r3, r3, -1 + + lwi r5, r0, TOPHYS(xmb_manager_dev) + lwi r8, r0, TOPHYS(xmb_manager_callback) + /* return from break need -8 to adjust for rtsd r15, 8 */ + addik r15, r0, ret_from_break - 8 + rtbd r8, 0 + nop + +ret_from_break: + /* flush the d-cache */ + bralid r15, mb_flush_dcache + nop + + /* + * To make sure microblaze i-cache is in a proper state + * invalidate the i-cache. + */ + bralid r15, mb_invalidate_icache + nop + + set_bip; /* Ints masked for state restore */ + VM_OFF; + mbar 1 + mbar 2 + bri 4 + suspend + nop +#endif + /* * Debug trap for KGDB. Enter to _debug_exception by brki r16, 0x18 * and call handling function with saved pt_regs @@ -957,6 +1151,88 @@ ENTRY(_switch_to) rtsd r15, 8 nop +#ifdef CONFIG_MB_MANAGER +.global xmb_inject_err +.section .text +.align 2 +.ent xmb_inject_err +.type xmb_inject_err, @function +xmb_inject_err: + addik r1, r1, -PT_SIZE + SAVE_REGS + + /* Switch to real mode */ + VM_OFF; + set_bip; + mbar 1 + mbar 2 + bralid r15, XMB_INJECT_ERR_OFFSET + nop; + + /* enable virtual mode */ + set_vms; + /* barrier for instructions and data accesses */ + mbar 1 + mbar 2 + /* + * Enable Interrupts, Virtual Protected Mode, equalize + * initial state for all possible entries. + */ + rtbd r0, 1f + nop; +1: + RESTORE_REGS + addik r1, r1, PT_SIZE + rtsd r15, 8; + nop; +.end xmb_inject_err + +.section .data +.global xmb_manager_dev +.global xmb_manager_baseaddr +.global xmb_manager_crval +.global xmb_manager_callback +.global xmb_manager_reset_callback +.global xmb_manager_stackpointer +.align 4 +xmb_manager_dev: + .long 0 +xmb_manager_baseaddr: + .long 0 +xmb_manager_crval: + .long 0 +xmb_manager_callback: + .long 0 +xmb_manager_reset_callback: + .long 0 +xmb_manager_stackpointer: + .long 0 + +/* + * When the break vector gets asserted because of error injection, + * the break signal must be blocked before exiting from the + * break handler, Below api updates the manager address and + * control register and error count callback arguments, + * which will be used by the break handler to block the + * break and call the callback function. + */ +.global xmb_manager_register +.section .text +.align 2 +.ent xmb_manager_register +.type xmb_manager_register, @function +xmb_manager_register: + swi r5, r0, xmb_manager_baseaddr + swi r6, r0, xmb_manager_crval + swi r7, r0, xmb_manager_callback + swi r8, r0, xmb_manager_dev + swi r9, r0, xmb_manager_reset_callback + + rtsd r15, 8; + nop; +.end xmb_manager_register +#endif + ENTRY(_reset) VM_OFF brai 0; /* Jump to reset vector */ @@ -964,19 +1240,43 @@ ENTRY(_reset) /* These are compiled and loaded into high memory, then * copied into place in mach_early_setup */ .section .init.ivt, "ax" -#if CONFIG_MANUAL_RESET_VECTOR +#if CONFIG_MANUAL_RESET_VECTOR && !defined(CONFIG_MB_MANAGER) .org 0x0 brai CONFIG_MANUAL_RESET_VECTOR +#elif defined(CONFIG_MB_MANAGER) + .org 0x0 + brai TOPHYS(_xtmr_manager_reset); #endif .org 0x8 brai TOPHYS(_user_exception); /* syscall handler */ .org 0x10 brai TOPHYS(_interrupt); /* Interrupt handler */ +#ifdef CONFIG_MB_MANAGER + .org 0x18 + brai TOPHYS(_xmb_manager_break); /* microblaze manager break handler */ +#else .org 0x18 brai TOPHYS(_debug_exception); /* debug trap handler */ +#endif .org 0x20 brai TOPHYS(_hw_exception_handler); /* HW exception handler */ +#ifdef CONFIG_MB_MANAGER + /* + * For TMR Inject API which injects the error should + * be executed from LMB. + * TMR Inject is programmed with address of 0x200 so that + * when program counter matches with this address error will + * be injected. 0x200 is expected to be next available bram + * offset, hence used for this api. + */ + .org XMB_INJECT_ERR_OFFSET +xmb_inject_error: + nop + rtsd r15, 8 + nop +#endif + .section .rodata,"a" #include "syscall_table.S" diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index ec21f899924985fe8e57811a35d3790f439aeafa..b26b77673c2cc307882a1b029ac1f0d16ded1df1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -2140,7 +2140,7 @@ config PAGE_SIZE_64KB endchoice -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" range 14 64 if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB default "14" if MIPS_HUGE_TLB_SUPPORT && PAGE_SIZE_64KB @@ -2669,7 +2669,6 @@ config ARCH_FLATMEM_ENABLE config ARCH_SPARSEMEM_ENABLE bool - select SPARSEMEM_STATIC if !SGI_IP27 config NUMA bool "NUMA Support" diff --git a/arch/mips/Makefile b/arch/mips/Makefile index 4d2a3e73fc45b61b3001e6feef1e0c3f114182a4..b296e33f8e333ad518ac74d7d2a77d444149d778 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -324,8 +324,6 @@ endif OBJCOPYFLAGS += --remove-section=.reginfo -head-y := arch/mips/kernel/head.o - libs-y += arch/mips/lib/ libs-$(CONFIG_MIPS_FP_SUPPORT) += arch/mips/math-emu/ diff --git a/arch/mips/bcm47xx/prom.c b/arch/mips/bcm47xx/prom.c index ab203e66ba0dd419a5162ee9ecfee86d645f1312..a9bea411d92827d1240908fbf24af5deb95f5446 100644 --- a/arch/mips/bcm47xx/prom.c +++ b/arch/mips/bcm47xx/prom.c @@ -86,7 +86,7 @@ static __init void prom_init_mem(void) pr_debug("Assume 128MB RAM\n"); break; } - if (!memcmp(prom_init, prom_init + mem, 32)) + if (!memcmp((void *)prom_init, (void *)prom_init + mem, 32)) break; } lowmem = mem; @@ -159,7 +159,7 @@ void __init bcm47xx_prom_highmem_init(void) off = EXTVBASE + __pa(off); for (extmem = 128 << 20; extmem < 512 << 20; extmem <<= 1) { - if (!memcmp(prom_init, (void *)(off + extmem), 16)) + if (!memcmp((void *)prom_init, (void *)(off + extmem), 16)) break; } extmem -= lowmem; diff --git a/arch/mips/boot/dts/brcm/bcm63268.dtsi b/arch/mips/boot/dts/brcm/bcm63268.dtsi index c3ce49ec675f3cd7e36ce56d936a07012606504f..8926417a8fbccc97bbc1d16c63b5c363d3b292a6 100644 --- a/arch/mips/boot/dts/brcm/bcm63268.dtsi +++ b/arch/mips/boot/dts/brcm/bcm63268.dtsi @@ -105,14 +105,20 @@ interrupts = <2>, <3>; }; - wdt: watchdog@1000009c { - compatible = "brcm,bcm7038-wdt"; - reg = <0x1000009c 0xc>; + timer-mfd@10000080 { + compatible = "brcm,bcm7038-twd", "simple-mfd", "syscon"; + reg = <0x10000080 0x30>; + ranges = <0x0 0x10000080 0x30>; - clocks = <&periph_osc>; - clock-names = "refclk"; + wdt: watchdog@1c { + compatible = "brcm,bcm7038-wdt"; + reg = <0x1c 0xc>; - timeout-sec = <30>; + clocks = <&periph_osc>; + clock-names = "refclk"; + + timeout-sec = <30>; + }; }; uart0: serial@10000180 { diff --git a/arch/mips/boot/dts/lantiq/Makefile b/arch/mips/boot/dts/lantiq/Makefile index f5dfc06242b9b7bf826ec77f00f47374d79b8106..ae6e3e21ebeb0753a6336ac3f97e2df78a7b2b51 100644 --- a/arch/mips/boot/dts/lantiq/Makefile +++ b/arch/mips/boot/dts/lantiq/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_DT_EASY50712) += easy50712.dtb +dtb-$(CONFIG_DT_EASY50712) += danube_easy50712.dtb obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/mips/boot/dts/lantiq/easy50712.dts b/arch/mips/boot/dts/lantiq/danube_easy50712.dts similarity index 100% rename from arch/mips/boot/dts/lantiq/easy50712.dts rename to arch/mips/boot/dts/lantiq/danube_easy50712.dts diff --git a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts index 24eebc5a85b1dc6c3305a98b73e4a3dd19959cb1..0128bd8fa7edae3531a99dad0b7f5b6a96648e66 100644 --- a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts +++ b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts @@ -20,12 +20,6 @@ bootargs = "console=ttyS0,57600"; }; - palmbus: palmbus@1e000000 { - i2c@900 { - status = "okay"; - }; - }; - gpio-keys { compatible = "gpio-keys"; @@ -53,7 +47,7 @@ }; }; -&sdhci { +&mmc { status = "okay"; }; diff --git a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts index 34006e6677806a4bd5dc1e4595a3c6bd52abe0e9..7515555388ae987a0338069dabf7fdefe379a655 100644 --- a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts +++ b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts @@ -20,12 +20,6 @@ bootargs = "console=ttyS0,57600"; }; - palmbus: palmbus@1e000000 { - i2c@900 { - status = "okay"; - }; - }; - gpio-keys { compatible = "gpio-keys"; @@ -35,9 +29,45 @@ linux,code = ; }; }; + + gpio-leds { + compatible = "gpio-leds"; + + ethblack-green { + label = "green:ethblack"; + gpios = <&gpio 3 GPIO_ACTIVE_LOW>; + }; + + ethblue-green { + label = "green:ethblue"; + gpios = <&gpio 4 GPIO_ACTIVE_LOW>; + }; + + ethyellow-green { + label = "green:ethyellow"; + gpios = <&gpio 15 GPIO_ACTIVE_LOW>; + }; + + ethyellow-orange { + label = "orange:ethyellow"; + gpios = <&gpio 13 GPIO_ACTIVE_LOW>; + }; + + power { + label = "green:power"; + gpios = <&gpio 6 GPIO_ACTIVE_LOW>; + linux,default-trigger = "default-on"; + }; + + system { + label = "green:system"; + gpios = <&gpio 8 GPIO_ACTIVE_LOW>; + linux,default-trigger = "disk-activity"; + }; + }; }; -&sdhci { +&mmc { status = "okay"; }; @@ -83,12 +113,12 @@ &gmac1 { status = "okay"; - phy-handle = <ðphy7>; + phy-handle = <ðphy5>; }; &mdio { - ethphy7: ethernet-phy@7 { - reg = <7>; + ethphy5: ethernet-phy@5 { + reg = <5>; phy-mode = "rgmii-rxid"; }; }; diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi index ee46ace0bcc11a41630e96cefd00e0628c896b88..f3f4c1f26e0199bebbb883c4a22578ff5030a26b 100644 --- a/arch/mips/boot/dts/ralink/mt7621.dtsi +++ b/arch/mips/boot/dts/ralink/mt7621.dtsi @@ -33,11 +33,6 @@ compatible = "mti,cpu-interrupt-controller"; }; - aliases { - serial0 = &uartlite; - }; - - mmc_fixed_3v3: regulator-3v3 { compatible = "regulator-fixed"; regulator-name = "mmc_power"; @@ -110,17 +105,16 @@ pinctrl-0 = <&i2c_pins>; }; - memc: syscon@5000 { + memc: memory-controller@5000 { compatible = "mediatek,mt7621-memc", "syscon"; reg = <0x5000 0x1000>; }; - uartlite: uartlite@c00 { + serial0: serial@c00 { compatible = "ns16550a"; reg = <0xc00 0x100>; clocks = <&sysc MT7621_CLK_UART1>; - clock-names = "uart1"; interrupt-parent = <&gic>; interrupts = ; @@ -236,7 +230,7 @@ }; }; - sdhci: sdhci@1e130000 { + mmc: mmc@1e130000 { status = "disabled"; compatible = "mediatek,mt7620-mmc"; @@ -262,8 +256,8 @@ interrupts = ; }; - xhci: xhci@1e1c0000 { - compatible = "mediatek,mt8173-xhci"; + usb: usb@1e1c0000 { + compatible = "mediatek,mt8173-xhci", "mediatek,mtk-xhci"; reg = <0x1e1c0000 0x1000 0x1e1d0700 0x0100>; reg-names = "mac", "ippc"; @@ -338,23 +332,22 @@ gmac1: mac@1 { compatible = "mediatek,eth-mac"; reg = <1>; - status = "off"; - phy-mode = "rgmii-rxid"; + status = "disabled"; + phy-mode = "rgmii"; }; mdio: mdio-bus { #address-cells = <1>; #size-cells = <0>; - switch0: switch0@0 { + switch0: switch@1f { compatible = "mediatek,mt7621"; - reg = <0>; + reg = <0x1f>; mediatek,mcm; resets = <&sysc MT7621_RST_MCM>; reset-names = "mcm"; interrupt-controller; #interrupt-cells = <1>; - interrupt-parent = <&gic>; interrupts = ; ports { @@ -362,31 +355,31 @@ #size-cells = <0>; port@0 { - status = "off"; + status = "disabled"; reg = <0>; label = "lan0"; }; port@1 { - status = "off"; + status = "disabled"; reg = <1>; label = "lan1"; }; port@2 { - status = "off"; + status = "disabled"; reg = <2>; label = "lan2"; }; port@3 { - status = "off"; + status = "disabled"; reg = <3>; label = "lan3"; }; port@4 { - status = "off"; + status = "disabled"; reg = <4>; label = "lan4"; }; diff --git a/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c b/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c index bf13e35871b213939a6bc6b1f5c0b6df2b6dee04..aa7bbf8d0df558c74f4bb438a18985ff7c30c37c 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c +++ b/arch/mips/cavium-octeon/executive/cvmx-cmd-queue.c @@ -57,14 +57,11 @@ EXPORT_SYMBOL_GPL(__cvmx_cmd_queue_state_ptr); static cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(void) { char *alloc_name = "cvmx_cmd_queues"; -#if defined(CONFIG_CAVIUM_RESERVE32) && CONFIG_CAVIUM_RESERVE32 extern uint64_t octeon_reserve32_memory; -#endif if (likely(__cvmx_cmd_queue_state_ptr)) return CVMX_CMD_QUEUE_SUCCESS; -#if defined(CONFIG_CAVIUM_RESERVE32) && CONFIG_CAVIUM_RESERVE32 if (octeon_reserve32_memory) __cvmx_cmd_queue_state_ptr = cvmx_bootmem_alloc_named_range(sizeof(*__cvmx_cmd_queue_state_ptr), @@ -73,7 +70,6 @@ static cvmx_cmd_queue_result_t __cvmx_cmd_queue_init_state_ptr(void) (CONFIG_CAVIUM_RESERVE32 << 20) - 1, 128, alloc_name); else -#endif __cvmx_cmd_queue_state_ptr = cvmx_bootmem_alloc_named(sizeof(*__cvmx_cmd_queue_state_ptr), 128, diff --git a/arch/mips/cavium-octeon/oct_ilm.c b/arch/mips/cavium-octeon/oct_ilm.c index 6a4694538bb6eacb38a46e62f0de1ed5468c8c5d..dc05262e85ff98ce14912b52a5c68f38fc64ba40 100644 --- a/arch/mips/cavium-octeon/oct_ilm.c +++ b/arch/mips/cavium-octeon/oct_ilm.c @@ -28,7 +28,7 @@ struct latency_info { static struct latency_info li; static struct dentry *dir; -static int show_latency(struct seq_file *m, void *v) +static int oct_ilm_show(struct seq_file *m, void *v) { u64 cpuclk, avg, max, min; struct latency_info curr_li = li; @@ -43,18 +43,7 @@ static int show_latency(struct seq_file *m, void *v) curr_li.interrupt_cnt, avg, max, min); return 0; } - -static int oct_ilm_open(struct inode *inode, struct file *file) -{ - return single_open(file, show_latency, NULL); -} - -static const struct file_operations oct_ilm_ops = { - .open = oct_ilm_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(oct_ilm); static int reset_statistics(void *data, u64 value) { @@ -67,7 +56,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(reset_statistics_ops, NULL, reset_statistics, "%llu\n") static void init_debugfs(void) { dir = debugfs_create_dir("oct_ilm", 0); - debugfs_create_file("statistics", 0222, dir, NULL, &oct_ilm_ops); + debugfs_create_file("statistics", 0222, dir, NULL, &oct_ilm_fops); debugfs_create_file("reset", 0222, dir, NULL, &reset_statistics_ops); } diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index 9cb9ed44bcafa149c18e08fb35bbbcd663ae0d96..fd8043f6ff8ae65a5769b5c24056560ac5c1fbd9 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -127,6 +127,16 @@ static void octeon_irq_free_cd(struct irq_domain *d, unsigned int irq) static int octeon_irq_force_ciu_mapping(struct irq_domain *domain, int irq, int line, int bit) { + struct device_node *of_node; + int ret; + + of_node = irq_domain_get_of_node(domain); + if (!of_node) + return -EINVAL; + ret = irq_alloc_desc_at(irq, of_node_to_nid(of_node)); + if (ret < 0) + return ret; + return irq_domain_associate(domain, irq, line << 6 | bit); } diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index cbd83205518d7d1d50e1aa249964f546a843586b..a71727f7a608c0133b066ab4d6e0a7973a95a344 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -284,10 +284,8 @@ void octeon_crash_smp_send_stop(void) #endif /* CONFIG_KEXEC */ -#ifdef CONFIG_CAVIUM_RESERVE32 uint64_t octeon_reserve32_memory; EXPORT_SYMBOL(octeon_reserve32_memory); -#endif #ifdef CONFIG_KEXEC /* crashkernel cmdline parameter is parsed _after_ memory setup @@ -532,7 +530,7 @@ void octeon_user_io_init(void) /* Get the current settings for CP0_CVMMEMCTL_REG */ cvmmemctl.u64 = read_c0_cvmmemctl(); /* R/W If set, marked write-buffer entries time out the same - * as as other entries; if clear, marked write-buffer entries + * as other entries; if clear, marked write-buffer entries * use the maximum timeout. */ cvmmemctl.s.dismarkwblongto = 1; /* R/W If set, a merged store does not clear the write-buffer @@ -666,9 +664,6 @@ void __init prom_init(void) int i; u64 t; int argc; -#ifdef CONFIG_CAVIUM_RESERVE32 - int64_t addr = -1; -#endif /* * The bootloader passes a pointer to the boot descriptor in * $a3, this is available as fw_arg3. @@ -783,7 +778,7 @@ void __init prom_init(void) cvmx_write_csr(CVMX_LED_UDD_DATX(1), 0); cvmx_write_csr(CVMX_LED_EN, 1); } -#ifdef CONFIG_CAVIUM_RESERVE32 + /* * We need to temporarily allocate all memory in the reserve32 * region. This makes sure the kernel doesn't allocate this @@ -794,14 +789,16 @@ void __init prom_init(void) * Allocate memory for RESERVED32 aligned on 2MB boundary. This * is in case we later use hugetlb entries with it. */ - addr = cvmx_bootmem_phy_named_block_alloc(CONFIG_CAVIUM_RESERVE32 << 20, - 0, 0, 2 << 20, - "CAVIUM_RESERVE32", 0); - if (addr < 0) - pr_err("Failed to allocate CAVIUM_RESERVE32 memory area\n"); - else - octeon_reserve32_memory = addr; -#endif + if (CONFIG_CAVIUM_RESERVE32) { + int64_t addr = + cvmx_bootmem_phy_named_block_alloc(CONFIG_CAVIUM_RESERVE32 << 20, + 0, 0, 2 << 20, + "CAVIUM_RESERVE32", 0); + if (addr < 0) + pr_err("Failed to allocate CAVIUM_RESERVE32 memory area\n"); + else + octeon_reserve32_memory = addr; + } #ifdef CONFIG_CAVIUM_OCTEON_LOCK_L2 if (cvmx_read_csr(CVMX_L2D_FUS3) & (3ull << 34)) { @@ -1079,7 +1076,6 @@ void __init plat_mem_setup(void) cvmx_bootmem_unlock(); #endif /* CONFIG_CRASH_DUMP */ -#ifdef CONFIG_CAVIUM_RESERVE32 /* * Now that we've allocated the kernel memory it is safe to * free the reserved region. We free it here so that builtin @@ -1087,7 +1083,6 @@ void __init plat_mem_setup(void) */ if (octeon_reserve32_memory) cvmx_bootmem_free_named("CAVIUM_RESERVE32"); -#endif /* CONFIG_CAVIUM_RESERVE32 */ if (total == 0) panic("Unable to allocate memory from " diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig index cf9c6329b807f9012a540e29951493fcd5963a48..ed4a6388791ee493c80a466ed9ed73a449ec0cb7 100644 --- a/arch/mips/configs/ar7_defconfig +++ b/arch/mips/configs/ar7_defconfig @@ -32,9 +32,6 @@ CONFIG_IP_ROUTE_MULTIPATH=y CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IP_MROUTE=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set CONFIG_TCP_CONG_ADVANCED=y # CONFIG_TCP_CONG_BIC is not set @@ -117,7 +114,6 @@ CONFIG_JFFS2_SUMMARY=y CONFIG_JFFS2_COMPRESSION_OPTIONS=y CONFIG_SQUASHFS=y # CONFIG_CRYPTO_HW is not set -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_CMDLINE_BOOL=y diff --git a/arch/mips/configs/ath25_defconfig b/arch/mips/configs/ath25_defconfig index 7143441f5476ae2737ed1cda61b00d4c5721b91c..afd1c16242e9dffef56ca6a0eee561a468d8f18e 100644 --- a/arch/mips/configs/ath25_defconfig +++ b/arch/mips/configs/ath25_defconfig @@ -29,9 +29,6 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_IP_ADVANCED_ROUTER=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_CFG80211=m CONFIG_MAC80211=m @@ -108,7 +105,6 @@ CONFIG_SQUASHFS_XZ=y # CONFIG_XZ_DEC_ARMTHUMB is not set # CONFIG_XZ_DEC_SPARC is not set CONFIG_PRINTK_TIME=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig index 96622a2ad3330c3355d803338c5e1e7f048a96b3..0b741716c8523c63e1cef2d0eeb0dbabeb861f27 100644 --- a/arch/mips/configs/ath79_defconfig +++ b/arch/mips/configs/ath79_defconfig @@ -10,12 +10,6 @@ CONFIG_EMBEDDED=y # CONFIG_SLUB_DEBUG is not set # CONFIG_COMPAT_BRK is not set CONFIG_ATH79=y -CONFIG_ATH79_MACH_AP121=y -CONFIG_ATH79_MACH_AP136=y -CONFIG_ATH79_MACH_AP81=y -CONFIG_ATH79_MACH_DB120=y -CONFIG_ATH79_MACH_PB44=y -CONFIG_ATH79_MACH_UBNT_XM=y CONFIG_HZ_100=y # CONFIG_SECCOMP is not set CONFIG_PCI=y @@ -29,9 +23,6 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_IP_ADVANCED_ROUTER=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_CFG80211=m CONFIG_MAC80211=m @@ -92,7 +83,6 @@ CONFIG_LEDS_GPIO=y # CONFIG_DNOTIFY is not set # CONFIG_PROC_PAGE_MONITOR is not set CONFIG_CRC_ITU_T=m -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/bcm47xx_defconfig b/arch/mips/configs/bcm47xx_defconfig index 91ce75edbfb46da1e52348e2dbb675ada346aa5a..22ffde722bb9e031c100180ed7fb90d0a88b3b12 100644 --- a/arch/mips/configs/bcm47xx_defconfig +++ b/arch/mips/configs/bcm47xx_defconfig @@ -72,7 +72,7 @@ CONFIG_LEDS_TRIGGER_TIMER=y CONFIG_LEDS_TRIGGER_DEFAULT_ON=y CONFIG_CRC32_SARWATE=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_INFO_REDUCED=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/configs/bcm63xx_defconfig b/arch/mips/configs/bcm63xx_defconfig index 861f680184b9523fb163e19d22705a7defcf601a..34d0ca638ef00720483d70b688016949ac9d6bdc 100644 --- a/arch/mips/configs/bcm63xx_defconfig +++ b/arch/mips/configs/bcm63xx_defconfig @@ -24,9 +24,6 @@ CONFIG_PCMCIA_BCM63XX=y CONFIG_NET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set CONFIG_CFG80211=y diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig index d83e7d600b0a56760a4c7d21885e329b794173f1..d15961f00ece2d26dcfa2168571bac1e0a148d1d 100644 --- a/arch/mips/configs/bigsur_defconfig +++ b/arch/mips/configs/bigsur_defconfig @@ -49,8 +49,6 @@ CONFIG_IP_PIMSM_V2=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -59,7 +57,6 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_SIT_6RD=y CONFIG_IPV6_TUNNEL=m CONFIG_IPV6_MULTIPLE_TABLES=y @@ -101,7 +98,6 @@ CONFIG_BAYCOM_SER_HDX=m CONFIG_YAM=m CONFIG_FW_LOADER=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_EEPROM_LEGACY=y CONFIG_EEPROM_MAX6875=y @@ -230,12 +226,8 @@ CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -243,7 +235,6 @@ CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/bmips_be_defconfig b/arch/mips/configs/bmips_be_defconfig index 032bb51defe8a0ba386c41322f8b3a437dfe76c3..daef132d000b393cf04e97adeddd255aa2454c83 100644 --- a/arch/mips/configs/bmips_be_defconfig +++ b/arch/mips/configs/bmips_be_defconfig @@ -17,9 +17,6 @@ CONFIG_PACKET=y CONFIG_PACKET_DIAG=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y diff --git a/arch/mips/configs/bmips_stb_defconfig b/arch/mips/configs/bmips_stb_defconfig index 5956fb95c19fa6b2af57a2ad0a0d5c9ce3579d12..cd0dc37c3d84496dfe012806c414bf16e3cf81e5 100644 --- a/arch/mips/configs/bmips_stb_defconfig +++ b/arch/mips/configs/bmips_stb_defconfig @@ -12,7 +12,6 @@ CONFIG_HIGHMEM=y CONFIG_HIGH_RES_TIMERS=y CONFIG_SMP=y CONFIG_NR_CPUS=4 -CONFIG_CC_STACKPROTECTOR_STRONG=y # CONFIG_SECCOMP is not set CONFIG_MIPS_O32_FP64_SUPPORT=y # CONFIG_RD_GZIP is not set @@ -21,8 +20,6 @@ CONFIG_MIPS_O32_FP64_SUPPORT=y CONFIG_RD_XZ=y # CONFIG_RD_LZO is not set # CONFIG_RD_LZ4 is not set -# CONFIG_IOSCHED_DEADLINE is not set -# CONFIG_IOSCHED_CFQ is not set CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_PCIEASPM_POWERSAVE=y @@ -30,7 +27,6 @@ CONFIG_PCIEPORTBUS=y CONFIG_PCIE_BRCMSTB=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_STAT=y -CONFIG_CPU_FREQ_STAT_DETAILS=y CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y CONFIG_CPU_FREQ_GOV_PERFORMANCE=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y @@ -44,15 +40,11 @@ CONFIG_PACKET=y CONFIG_PACKET_DIAG=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set CONFIG_CFG80211=y CONFIG_NL80211_TESTMODE=y CONFIG_WIRELESS=y CONFIG_MAC80211=y -CONFIG_NL80211=y CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_STANDALONE is not set @@ -70,10 +62,6 @@ CONFIG_IP_PNP_RARP=y CONFIG_IP_MROUTE=y CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_LRO is not set CONFIG_INET_UDP_DIAG=y CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_CONG_BIC=y @@ -93,7 +81,6 @@ CONFIG_NET_SWITCHDEV=y CONFIG_DMA_CMA=y CONFIG_CMA_ALIGNMENT=12 CONFIG_SPI=y -CONFIG_SPI_BRCMSTB=y CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y @@ -105,14 +92,11 @@ CONFIG_MTD_CFI_STAA=y CONFIG_MTD_ROM=y CONFIG_MTD_ABSENT=y CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_M25P80=y -CONFIG_MTD_NAND=y CONFIG_MTD_NAND_BRCMNAND=y CONFIG_MTD_SPI_NOR=y # CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is not set CONFIG_MTD_UBI=y CONFIG_MTD_UBI_GLUEBI=y -CONFIG_PROC_DEVICETREE=y CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 @@ -120,7 +104,6 @@ CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y -CONFIG_SCSI_MULTI_LUN=y # CONFIG_SCSI_LOWLEVEL is not set CONFIG_NETDEVICES=y CONFIG_VLAN_8021Q=y @@ -135,7 +118,6 @@ CONFIG_INPUT_UINPUT=y # CONFIG_SERIO is not set CONFIG_VT=y CONFIG_VT_HW_CONSOLE_BINDING=y -# CONFIG_DEVKMEM is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y @@ -203,17 +185,14 @@ CONFIG_CMDLINE="earlycon" CONFIG_MIPS_CMDLINE_DTB_EXTEND=y # CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER is not set # CONFIG_CRYPTO_HW is not set -CONFIG_DT_BCM974XX=y CONFIG_FW_CFE=y CONFIG_ATA=y CONFIG_SATA_AHCI_PLATFORM=y -CONFIG_AHCI_BRCMSTB=y CONFIG_GENERIC_PHY=y CONFIG_GPIOLIB=y CONFIG_GPIO_SYSFS=y CONFIG_PHY_BRCM_USB=y CONFIG_PHY_BRCM_SATA=y -CONFIG_PM_RUNTIME=y CONFIG_PM_DEBUG=y CONFIG_SYSVIPC=y CONFIG_FUNCTION_GRAPH_TRACER=y @@ -227,3 +206,5 @@ CONFIG_FTRACE_SYSCALLS=y CONFIG_TRACER_SNAPSHOT=y CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y CONFIG_STACK_TRACER=y +CONFIG_AHCI_BRCM=y +CONFIG_MTD_RAW_NAND=y diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig index 97ceaf080c0c780f9497b0c8cbd8e6f29b150ac3..0bc2e3cc573b6164a239d6fa487fff88406e48a9 100644 --- a/arch/mips/configs/cavium_octeon_defconfig +++ b/arch/mips/configs/cavium_octeon_defconfig @@ -71,7 +71,6 @@ CONFIG_NETDEVICES=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -162,7 +161,7 @@ CONFIG_CRYPTO_SHA1_OCTEON=m CONFIG_CRYPTO_SHA256_OCTEON=m CONFIG_CRYPTO_SHA512_OCTEON=m CONFIG_CRYPTO_DES=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index cc69b215854ea336d942eeff2d485f374dcab307..11f08b6a3013f1d71fc58fa771cbc701c3a7a2be 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -143,7 +143,7 @@ CONFIG_MEMORY=y CONFIG_JZ4780_NEMC=y CONFIG_PWM=y CONFIG_PWM_JZ4740=m -CONFIG_JZ4780_EFUSE=y +CONFIG_NVMEM_JZ4780_EFUSE=y CONFIG_JZ4770_PHY=y CONFIG_EXT4_FS=y # CONFIG_DNOTIFY is not set @@ -199,7 +199,7 @@ CONFIG_NLS_UTF8=y CONFIG_DMA_CMA=y CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/configs/cu1000-neo_defconfig b/arch/mips/configs/cu1000-neo_defconfig index 5bd55eb32fe55ec256263018312b44b312a419fd..1cbc9302e1d140ba6b1a06efd29b2d9c3066ad86 100644 --- a/arch/mips/configs/cu1000-neo_defconfig +++ b/arch/mips/configs/cu1000-neo_defconfig @@ -113,7 +113,7 @@ CONFIG_PRINTK_TIME=y CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15 CONFIG_CONSOLE_LOGLEVEL_QUIET=15 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7 -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/configs/cu1830-neo_defconfig b/arch/mips/configs/cu1830-neo_defconfig index cc69688962e8c348131a0ef4f730ff673958db6c..a0f73f3cd6ce7b787bf6c584ea9ef8ec8a005c13 100644 --- a/arch/mips/configs/cu1830-neo_defconfig +++ b/arch/mips/configs/cu1830-neo_defconfig @@ -116,7 +116,7 @@ CONFIG_PRINTK_TIME=y CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15 CONFIG_CONSOLE_LOGLEVEL_QUIET=15 CONFIG_MESSAGE_LOGLEVEL_DEFAULT=7 -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y diff --git a/arch/mips/configs/db1xxx_defconfig b/arch/mips/configs/db1xxx_defconfig index b8bd663009969abf1f1bbbea992b80dffcb80963..af070be1b583ae087f8f25dfacac2214c6eabd08 100644 --- a/arch/mips/configs/db1xxx_defconfig +++ b/arch/mips/configs/db1xxx_defconfig @@ -9,7 +9,6 @@ CONFIG_HIGH_RES_TIMERS=y CONFIG_LOG_BUF_SHIFT=16 CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y CONFIG_BLK_CGROUP=y CONFIG_CGROUP_SCHED=y CONFIG_CFS_BANDWIDTH=y @@ -61,7 +60,6 @@ CONFIG_INET6_AH=y CONFIG_INET6_ESP=y CONFIG_INET6_IPCOMP=y CONFIG_IPV6_MIP6=y -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=y CONFIG_IPV6_VTI=y CONFIG_IPV6_SIT_6RD=y CONFIG_IPV6_GRE=y diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 0021427a1bbe7b8c97dfa09ad67d20a4ecf1f088..49ec1575234ee31a68ed265015fd133ebfe7a187 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -37,9 +37,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -47,14 +44,11 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y CONFIG_NETWORK_SECMARK=y CONFIG_IP_SCTP=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m -CONFIG_DECNET_ROUTER=y # CONFIG_WIRELESS is not set # CONFIG_UEVENT_HELPER is not set # CONFIG_FW_LOADER is not set @@ -79,7 +73,6 @@ CONFIG_NETDEVICES=y CONFIG_DECLANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_VENDOR_AURORA is not set # CONFIG_NET_VENDOR_BROADCOM is not set # CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set @@ -193,12 +186,8 @@ CONFIG_CRYPTO_CRC32=m CONFIG_CRYPTO_CRCT10DIF=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARC4=m @@ -208,7 +197,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 7a97a0818ce4209b099f4f5114cb8922ef0e77b5..5cec4c096e2c220c7759d9eda8fc24f66364e974 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -33,9 +33,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -43,14 +40,11 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y CONFIG_NETWORK_SECMARK=y CONFIG_IP_SCTP=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m -CONFIG_DECNET_ROUTER=y # CONFIG_WIRELESS is not set # CONFIG_UEVENT_HELPER is not set # CONFIG_FW_LOADER is not set @@ -75,7 +69,6 @@ CONFIG_NETDEVICES=y CONFIG_DECLANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_VENDOR_AURORA is not set # CONFIG_NET_VENDOR_BROADCOM is not set # CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set @@ -188,12 +181,8 @@ CONFIG_CRYPTO_CRC32=m CONFIG_CRYPTO_CRCT10DIF=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARC4=m @@ -203,7 +192,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index a0643363526d72f525ac6868f8c307e87bf37b24..af37e26d9b5b65607b46758145f736c337eb1d67 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -32,9 +32,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -42,14 +39,11 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y CONFIG_NETWORK_SECMARK=y CONFIG_IP_SCTP=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m -CONFIG_DECNET_ROUTER=y # CONFIG_WIRELESS is not set # CONFIG_UEVENT_HELPER is not set # CONFIG_FW_LOADER is not set @@ -74,7 +68,6 @@ CONFIG_NETDEVICES=y CONFIG_DECLANCE=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_VENDOR_AURORA is not set # CONFIG_NET_VENDOR_BROADCOM is not set # CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set @@ -188,12 +181,8 @@ CONFIG_CRYPTO_CRC32=m CONFIG_CRYPTO_CRCT10DIF=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARC4=m @@ -203,7 +192,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/fuloong2e_defconfig b/arch/mips/configs/fuloong2e_defconfig index ba47c5e929b7f368816215267bae2df692b20504..843d6a5a4f61581b77288d5da833ef7f0b2e3fc6 100644 --- a/arch/mips/configs/fuloong2e_defconfig +++ b/arch/mips/configs/fuloong2e_defconfig @@ -35,8 +35,6 @@ CONFIG_IP_MULTICAST=y CONFIG_IP_PNP=y CONFIG_IP_PNP_BOOTP=y CONFIG_NET_IPIP=m -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set CONFIG_NETFILTER=y @@ -92,7 +90,6 @@ CONFIG_MTD_CFI_AMDSTD=m CONFIG_MTD_CFI_STAA=m CONFIG_MTD_PHYSMAP=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_RAM=m CONFIG_CDROM_PKTCDVD=m CONFIG_ATA_OVER_ETH=m @@ -159,7 +156,6 @@ CONFIG_USB_MOUSE=y CONFIG_USB=y CONFIG_USB_ANNOUNCE_NEW_DEVICES=y CONFIG_USB_OTG_PRODUCTLIST=y -CONFIG_USB_WUSB_CBAF=m CONFIG_USB_C67X00_HCD=m CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_ROOT_HUB_TT=y @@ -219,15 +215,10 @@ CONFIG_CRYPTO_CTS=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC_CCITT=y CONFIG_CRC7=m -# CONFIG_ENABLE_MUST_CHECK is not set diff --git a/arch/mips/configs/generic/board-ocelot.config b/arch/mips/configs/generic/board-ocelot.config index 5107095654046bcb16a245533a3baf2c2cf00568..8cfbafa532e0cf2fc88979f65604806365ff81b8 100644 --- a/arch/mips/configs/generic/board-ocelot.config +++ b/arch/mips/configs/generic/board-ocelot.config @@ -25,7 +25,6 @@ CONFIG_NETDEVICES=y CONFIG_NET_SWITCHDEV=y CONFIG_NET_DSA=y CONFIG_MSCC_OCELOT_SWITCH=y -CONFIG_MSCC_OCELOT_SWITCH_OCELOT=y CONFIG_MDIO_MSCC_MIIM=y CONFIG_MICROSEMI_PHY=y diff --git a/arch/mips/configs/generic_defconfig b/arch/mips/configs/generic_defconfig index 714169e411cf07df260a04a6f3f89d93358063d1..c2cd2b181ef39b60bfa0bc17a7b868c436eaa9a1 100644 --- a/arch/mips/configs/generic_defconfig +++ b/arch/mips/configs/generic_defconfig @@ -3,7 +3,6 @@ CONFIG_NO_HZ_IDLE=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y CONFIG_BLK_CGROUP=y CONFIG_CFS_BANDWIDTH=y CONFIG_RT_GROUP_SCHED=y @@ -83,7 +82,7 @@ CONFIG_ROOT_NFS=y # CONFIG_XZ_DEC_ARMTHUMB is not set # CONFIG_XZ_DEC_SPARC is not set CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_INFO_REDUCED=y CONFIG_DEBUG_FS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index d82f4ebf687fc425325a3d21a19fcdaf2174f298..eb755650f8212daaef7fc2fc6bdef1472302eae5 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -29,9 +29,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IP_PNP=y CONFIG_IP_PNP_BOOTP=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_NETWORK_SECMARK=y CONFIG_NETFILTER=y @@ -69,7 +66,6 @@ CONFIG_IP_NF_RAW=m CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_DECNET_NF_GRABULATOR=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m CONFIG_BRIDGE_EBT_T_FILTER=m @@ -99,7 +95,6 @@ CONFIG_ATM_MPOA=m CONFIG_ATM_BR2684=m CONFIG_BRIDGE=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m CONFIG_LLC2=m CONFIG_ATALK=m CONFIG_DEV_APPLETALK=m @@ -220,9 +215,6 @@ CONFIG_HDLC_X25=m CONFIG_PCI200SYN=m CONFIG_WANXL=m CONFIG_FARSYNC=m -CONFIG_DSCC4=m -CONFIG_DSCC4_PCISYNC=y -CONFIG_DSCC4_PCI_RST=y CONFIG_LAPBETHER=m # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set @@ -288,7 +280,6 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -299,7 +290,6 @@ CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_DEFLATE=m -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="console=ttyS0,115200 root=/dev/nfs rw ip=auto" diff --git a/arch/mips/configs/ip22_defconfig b/arch/mips/configs/ip22_defconfig index 70a4ba90f49176c4259c33a20826cdc932a1af74..13df299012378780234c521da984e29dae7c1eb4 100644 --- a/arch/mips/configs/ip22_defconfig +++ b/arch/mips/configs/ip22_defconfig @@ -36,9 +36,6 @@ CONFIG_IP_PNP_BOOTP=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -47,7 +44,6 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_TUNNEL=m CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y @@ -226,7 +222,6 @@ CONFIG_SERIO_RAW=m CONFIG_VT_HW_CONSOLE_BINDING=y CONFIG_SERIAL_IP22_ZILOG=m # CONFIG_HW_RANDOM is not set -CONFIG_RAW_DRIVER=m # CONFIG_HWMON is not set CONFIG_THERMAL=y CONFIG_WATCHDOG=y @@ -320,11 +315,7 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -333,7 +324,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig index 821630ac1be7ff1f850c14463eb981cffc0a8ede..3e86f8106ba054d0d118969247737d92b9b57e00 100644 --- a/arch/mips/configs/ip27_defconfig +++ b/arch/mips/configs/ip27_defconfig @@ -33,9 +33,6 @@ CONFIG_NET_KEY_MIGRATE=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_IP_PNP=y -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -44,10 +41,6 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_TRANSPORT=m -CONFIG_INET6_XFRM_MODE_TUNNEL=m -CONFIG_INET6_XFRM_MODE_BEET=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_SIT=m CONFIG_IPV6_SIT_6RD=y CONFIG_IPV6_TUNNEL=m @@ -92,7 +85,6 @@ CONFIG_CFG80211=m CONFIG_MAC80211=m CONFIG_RFKILL=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_CDROM_PKTCDVD=m CONFIG_ATA_OVER_ETH=m CONFIG_SCSI=y @@ -115,7 +107,6 @@ CONFIG_SCSI_AIC94XX=m # CONFIG_AIC94XX_DEBUG is not set CONFIG_SCSI_MVSAS=m # CONFIG_SCSI_MVSAS_DEBUG is not set -CONFIG_SCSI_DPT_I2O=m CONFIG_SCSI_MPT2SAS=m CONFIG_LIBFC=m CONFIG_SCSI_QLOGIC_1280=y @@ -126,8 +117,6 @@ CONFIG_SCSI_DH_RDAC=m CONFIG_SCSI_DH_HP_SW=m CONFIG_SCSI_DH_EMC=m CONFIG_SCSI_DH_ALUA=m -CONFIG_SCSI_OSD_INITIATOR=m -CONFIG_SCSI_OSD_ULD=m CONFIG_MD=y CONFIG_BLK_DEV_MD=y CONFIG_MD_LINEAR=m @@ -166,7 +155,6 @@ CONFIG_JME=m CONFIG_MLX4_EN=m # CONFIG_MLX4_DEBUG is not set CONFIG_KS8851_MLL=m -CONFIG_VXGE=m CONFIG_AX88796=m CONFIG_AX88796_93CX6=y CONFIG_ETHOC=m @@ -264,7 +252,6 @@ CONFIG_I2C_VIAPRO=m CONFIG_I2C_OCORES=m CONFIG_I2C_PCA_PLATFORM=m CONFIG_I2C_SIMTEC=m -CONFIG_I2C_PARPORT_LIGHT=m CONFIG_I2C_TAOS_EVM=m CONFIG_I2C_STUB=m # CONFIG_HWMON is not set @@ -309,7 +296,6 @@ CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y CONFIG_SQUASHFS=m CONFIG_OMFS_FS=m -CONFIG_EXOFS_FS=m CONFIG_NFS_FS=y CONFIG_SECURITYFS=y CONFIG_CRYPTO_CRYPTD=m @@ -321,12 +307,8 @@ CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -335,7 +317,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/ip28_defconfig b/arch/mips/configs/ip28_defconfig index 0921ef38e9fb4377297c21c1c5bf847a08de028f..ba13eea0509fd7f05dd50684b8a22fdfb450f6d0 100644 --- a/arch/mips/configs/ip28_defconfig +++ b/arch/mips/configs/ip28_defconfig @@ -29,9 +29,6 @@ CONFIG_IP_MULTICAST=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set CONFIG_TCP_MD5SIG=y # CONFIG_IPV6 is not set CONFIG_SCSI=y diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig index 74020aa3440b18a04223f06bb4a89b0b9ff4be76..8ced2224c328b6485086b503e50c6d44f8d33078 100644 --- a/arch/mips/configs/ip32_defconfig +++ b/arch/mips/configs/ip32_defconfig @@ -43,7 +43,6 @@ CONFIG_IPV6_TUNNEL=m CONFIG_NETWORK_SECMARK=y CONFIG_CONNECTOR=y CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_RAID_ATTRS=y CONFIG_SCSI=y @@ -165,7 +164,6 @@ CONFIG_CRYPTO_MICHAEL_MIC=y CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_SHA512=y -CONFIG_CRYPTO_TGR192=y CONFIG_CRYPTO_WP512=y CONFIG_CRYPTO_ANUBIS=y CONFIG_CRYPTO_ARC4=y diff --git a/arch/mips/configs/jazz_defconfig b/arch/mips/configs/jazz_defconfig index 843f360da5f2bf57a41e85671de5d1f07f23ba64..106b21cb677fca61eb9cfd09d8f97e32691ad0b0 100644 --- a/arch/mips/configs/jazz_defconfig +++ b/arch/mips/configs/jazz_defconfig @@ -32,7 +32,6 @@ CONFIG_PARPORT_1284=y CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_FD=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=m CONFIG_CDROM_PKTCDVD=m diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig index 791894c4d8fbc2c43b9e76957113e425f72de800..7e598d3389798a9f1276ccc5c2114b50d9f1154d 100644 --- a/arch/mips/configs/lemote2f_defconfig +++ b/arch/mips/configs/lemote2f_defconfig @@ -43,9 +43,6 @@ CONFIG_IP_MROUTE=y CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y CONFIG_SYN_COOKIES=y -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_CONG_BIC=y CONFIG_DEFAULT_BIC=y @@ -77,7 +74,6 @@ CONFIG_MAC80211_LEDS=y CONFIG_RFKILL=m CONFIG_RFKILL_INPUT=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_BLK_DEV_SD=y @@ -312,12 +308,8 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -326,7 +318,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/mips/configs/loongson1b_defconfig b/arch/mips/configs/loongson1b_defconfig index 25e70423e17dfa431bc5e4b5eb7324b1a2ed6e22..68207b31dc2028b42bd2df20273a42908ca1898a 100644 --- a/arch/mips/configs/loongson1b_defconfig +++ b/arch/mips/configs/loongson1b_defconfig @@ -28,9 +28,6 @@ CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set @@ -115,7 +112,6 @@ CONFIG_NLS_ISO8859_1=m # CONFIG_CRYPTO_ECHAINIV is not set # CONFIG_CRYPTO_HW is not set CONFIG_DYNAMIC_DEBUG=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/loongson1c_defconfig b/arch/mips/configs/loongson1c_defconfig index 3a158d4d2faba3db85e2883e803f3514efba7e1f..c3910a9dee9e87a7e1e933a1aacfb0f15f023e59 100644 --- a/arch/mips/configs/loongson1c_defconfig +++ b/arch/mips/configs/loongson1c_defconfig @@ -29,9 +29,6 @@ CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set @@ -116,7 +113,6 @@ CONFIG_NLS_ISO8859_1=m # CONFIG_CRYPTO_ECHAINIV is not set # CONFIG_CRYPTO_HW is not set CONFIG_DYNAMIC_DEBUG=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/mips/configs/loongson2k_defconfig b/arch/mips/configs/loongson2k_defconfig index e948ca487e2d82c0be008fd8fbaae5bc18875c90..728bef666f7a3ca1c5d3cd64ff54cf257b67097f 100644 --- a/arch/mips/configs/loongson2k_defconfig +++ b/arch/mips/configs/loongson2k_defconfig @@ -95,7 +95,6 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_MTD=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_RAID_ATTRS=m @@ -229,7 +228,6 @@ CONFIG_SERIAL_8250_RSA=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_NONSTANDARD=y CONFIG_HW_RANDOM=y -CONFIG_RAW_DRIVER=m CONFIG_I2C_CHARDEV=y CONFIG_I2C_PIIX4=y CONFIG_GPIO_LOONGSON=y @@ -336,7 +334,6 @@ CONFIG_DEFAULT_SECURITY_DAC=y CONFIG_CRYPTO_SEQIV=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 25ecd15bc952ff31924a03f85cad2300c0e21022..aca66a5f330dde9a01b18238ff832479d528ad39 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -143,7 +143,6 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y CONFIG_MTD=m CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=8192 CONFIG_VIRTIO_BLK=y @@ -268,7 +267,6 @@ CONFIG_SERIAL_OF_PLATFORM=y CONFIG_SERIAL_NONSTANDARD=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y -CONFIG_RAW_DRIVER=m CONFIG_I2C_CHARDEV=y CONFIG_I2C_PIIX4=y CONFIG_GPIO_LOONGSON=y diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig index 7a5bdd236a2af5da38816b8d9c1c32c7e0b2b737..265d38dffbf6e17eb01565fe341ca30a425d431c 100644 --- a/arch/mips/configs/malta_defconfig +++ b/arch/mips/configs/malta_defconfig @@ -42,8 +42,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -229,7 +227,6 @@ CONFIG_MTD_UBI=m CONFIG_MTD_UBI_GLUEBI=m CONFIG_BLK_DEV_FD=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_CDROM_PKTCDVD=m @@ -237,7 +234,6 @@ CONFIG_ATA_OVER_ETH=m CONFIG_RAID_ATTRS=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_ST=m -CONFIG_CHR_DEV_OSST=m CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=m CONFIG_SCSI_CONSTANTS=y @@ -408,7 +404,6 @@ CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig index b5ba08d7ab57523b5a61a90e319a510dafde51c5..1d2b248c7cd3f7450cdd8031010a39df73f9e42a 100644 --- a/arch/mips/configs/malta_kvm_defconfig +++ b/arch/mips/configs/malta_kvm_defconfig @@ -46,8 +46,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -233,7 +231,6 @@ CONFIG_MTD_UBI=m CONFIG_MTD_UBI_GLUEBI=m CONFIG_BLK_DEV_FD=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_CDROM_PKTCDVD=m @@ -241,7 +238,6 @@ CONFIG_ATA_OVER_ETH=m CONFIG_RAID_ATTRS=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_ST=m -CONFIG_CHR_DEV_OSST=m CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=m CONFIG_SCSI_CONSTANTS=y @@ -415,7 +411,6 @@ CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig index 6fb9bc29f4a03eca70001e2c0c17ddf393c18c78..fd63a2b152f689945f269a31d672b561f607b88b 100644 --- a/arch/mips/configs/malta_qemu_32r6_defconfig +++ b/arch/mips/configs/malta_qemu_32r6_defconfig @@ -76,7 +76,6 @@ CONFIG_NET_ACT_POLICE=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y # CONFIG_SCSI_LOWLEVEL is not set @@ -98,7 +97,6 @@ CONFIG_PCNET32=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -172,7 +170,6 @@ CONFIG_NLS_ISO8859_1=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig index eb72df528243a668b11409529c844b13d8fe07fc..1f07e354c9544d8495962ea2af96f4cf4ac250ad 100644 --- a/arch/mips/configs/maltaaprp_defconfig +++ b/arch/mips/configs/maltaaprp_defconfig @@ -78,7 +78,6 @@ CONFIG_NET_ACT_POLICE=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y # CONFIG_SCSI_LOWLEVEL is not set @@ -100,7 +99,6 @@ CONFIG_PCNET32=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -173,7 +171,6 @@ CONFIG_NLS_ISO8859_1=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig index 1fb40d310f49c080345c34dd109380944981d9d0..5cd3eca236de2c57761f559e46374cedd803331c 100644 --- a/arch/mips/configs/maltasmvp_defconfig +++ b/arch/mips/configs/maltasmvp_defconfig @@ -79,7 +79,6 @@ CONFIG_NET_ACT_POLICE=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y # CONFIG_SCSI_LOWLEVEL is not set @@ -99,7 +98,6 @@ CONFIG_PCNET32=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -174,7 +172,6 @@ CONFIG_NLS_ISO8859_1=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig index 75cb778c6149612c497a7993565183e26978cd94..45688e742a15a35cbf0a4c85bdd8b595eb9361fb 100644 --- a/arch/mips/configs/maltasmvp_eva_defconfig +++ b/arch/mips/configs/maltasmvp_eva_defconfig @@ -80,7 +80,6 @@ CONFIG_NET_ACT_POLICE=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y # CONFIG_SCSI_LOWLEVEL is not set @@ -102,7 +101,6 @@ CONFIG_PCNET32=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -176,7 +174,6 @@ CONFIG_NLS_ISO8859_1=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig index 7b4f247dc60cc1c1a7dae8005d93aab73bea3ab0..136f965784db47def816b8ce5f032782b5bacf9c 100644 --- a/arch/mips/configs/maltaup_defconfig +++ b/arch/mips/configs/maltaup_defconfig @@ -77,7 +77,6 @@ CONFIG_NET_ACT_POLICE=y # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_SG=y # CONFIG_SCSI_LOWLEVEL is not set @@ -99,7 +98,6 @@ CONFIG_PCNET32=y # CONFIG_NET_VENDOR_DEC is not set # CONFIG_NET_VENDOR_DLINK is not set # CONFIG_NET_VENDOR_EMULEX is not set -# CONFIG_NET_VENDOR_HP is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set # CONFIG_NET_VENDOR_MELLANOX is not set @@ -172,7 +170,6 @@ CONFIG_NLS_ISO8859_1=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig index 8d58653f1b4ed5e6bd220d41c209ac550762dce3..75b8da8d992714f9c4e8fbec135dc69933c15b5a 100644 --- a/arch/mips/configs/maltaup_xpa_defconfig +++ b/arch/mips/configs/maltaup_xpa_defconfig @@ -45,8 +45,6 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -231,7 +229,6 @@ CONFIG_MTD_UBI=m CONFIG_MTD_UBI_GLUEBI=m CONFIG_BLK_DEV_FD=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_CDROM_PKTCDVD=m @@ -239,7 +236,6 @@ CONFIG_ATA_OVER_ETH=m CONFIG_RAID_ATTRS=m CONFIG_BLK_DEV_SD=y CONFIG_CHR_DEV_ST=m -CONFIG_CHR_DEV_OSST=m CONFIG_BLK_DEV_SR=y CONFIG_CHR_DEV_SG=m CONFIG_SCSI_CONSTANTS=y @@ -414,7 +410,6 @@ CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig index 4194e79b435c83f5e38584ac4f30aa54b536423a..edf9634aa8ee1d1b904252c66895ef5dbce7ec2c 100644 --- a/arch/mips/configs/mtx1_defconfig +++ b/arch/mips/configs/mtx1_defconfig @@ -58,13 +58,9 @@ CONFIG_SYN_COOKIES=y CONFIG_INET_AH=m CONFIG_INET_ESP=m CONFIG_INET_IPCOMP=m -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_TUNNEL=m CONFIG_NETWORK_SECMARK=y CONFIG_NETFILTER=y @@ -116,7 +112,6 @@ CONFIG_IP6_NF_FILTER=m CONFIG_IP6_NF_TARGET_REJECT=m CONFIG_IP6_NF_MANGLE=m CONFIG_IP6_NF_RAW=m -CONFIG_DECNET_NF_GRABULATOR=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m CONFIG_BRIDGE_EBT_T_FILTER=m @@ -146,7 +141,6 @@ CONFIG_ATM_MPOA=m CONFIG_ATM_BR2684=m CONFIG_BRIDGE=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m CONFIG_LLC2=m CONFIG_ATALK=m CONFIG_DEV_APPLETALK=m @@ -284,7 +278,6 @@ CONFIG_PCMCIA_XIRCOM=m CONFIG_DL2K=m CONFIG_SUNDANCE=m CONFIG_PCMCIA_FMVJ18X=m -CONFIG_HP100=m CONFIG_E100=m CONFIG_E1000=m CONFIG_IXGB=m @@ -368,9 +361,6 @@ CONFIG_HDLC_X25=m CONFIG_PCI200SYN=m CONFIG_WANXL=m CONFIG_FARSYNC=m -CONFIG_DSCC4=m -CONFIG_DSCC4_PCISYNC=y -CONFIG_DSCC4_PCI_RST=y CONFIG_LAPBETHER=m # CONFIG_KEYBOARD_ATKBD is not set CONFIG_KEYBOARD_GPIO=y @@ -683,7 +673,6 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -692,5 +681,4 @@ CONFIG_CRYPTO_KHAZAD=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/configs/omega2p_defconfig b/arch/mips/configs/omega2p_defconfig index fc39ddf610a9bc41af23af21c26c5783b3fe2e99..91fe2822f8971463bbbbdda79e853489ec4ffde8 100644 --- a/arch/mips/configs/omega2p_defconfig +++ b/arch/mips/configs/omega2p_defconfig @@ -35,9 +35,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set @@ -116,7 +113,7 @@ CONFIG_CRYPTO_LZO=y CONFIG_CRC16=y CONFIG_XZ_DEC=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/configs/pic32mzda_defconfig b/arch/mips/configs/pic32mzda_defconfig index fd567247adc741f0fe4a2e4b2e4c3a54f1e8a0b7..48dd02d01ac192fa570a75559e3f3d0143442619 100644 --- a/arch/mips/configs/pic32mzda_defconfig +++ b/arch/mips/configs/pic32mzda_defconfig @@ -45,7 +45,6 @@ CONFIG_KEYBOARD_GPIO_POLLED=m CONFIG_SERIAL_PIC32=y CONFIG_SERIAL_PIC32_CONSOLE=y CONFIG_HW_RANDOM=y -CONFIG_RAW_DRIVER=m CONFIG_GPIO_SYSFS=y # CONFIG_HWMON is not set CONFIG_HIDRAW=y diff --git a/arch/mips/configs/qi_lb60_defconfig b/arch/mips/configs/qi_lb60_defconfig index b4448d0876d57ae98bd4817ae58ba7c5bd68236c..7e5d9741bd5ddb53f5631d1747b50046baa033fc 100644 --- a/arch/mips/configs/qi_lb60_defconfig +++ b/arch/mips/configs/qi_lb60_defconfig @@ -166,7 +166,7 @@ CONFIG_NLS_UTF8=y CONFIG_FONTS=y CONFIG_FONT_SUN8x16=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_READABLE_ASM=y CONFIG_KGDB=y diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 252d472387aae5f060288e53ce89e7f15e4e84cd..93306f5e045b41e689e79dd3ec1f654440a14fba 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -33,9 +33,6 @@ CONFIG_IP_MULTIPLE_TABLES=y CONFIG_IP_ROUTE_MULTIPATH=y CONFIG_IP_ROUTE_VERBOSE=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set CONFIG_INET_DIAG=m CONFIG_TCP_CONG_ADVANCED=y CONFIG_TCP_CONG_CUBIC=m @@ -162,5 +159,4 @@ CONFIG_SQUASHFS=y CONFIG_CRYPTO_TEST=m # CONFIG_CRYPTO_HW is not set CONFIG_CRC16=m -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y diff --git a/arch/mips/configs/rbtx49xx_defconfig b/arch/mips/configs/rbtx49xx_defconfig index f8212a813be7b1bde1fed0c8d9bb5554b9f81458..30c195f2827816b345fb2fe7dc9c2f3eb94ff0b5 100644 --- a/arch/mips/configs/rbtx49xx_defconfig +++ b/arch/mips/configs/rbtx49xx_defconfig @@ -21,9 +21,6 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_MULTICAST=y CONFIG_IP_PNP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set CONFIG_MTD=y @@ -51,7 +48,6 @@ CONFIG_TC35815=y CONFIG_SERIAL_TXX9_CONSOLE=y CONFIG_SERIAL_TXX9_STDSERIAL=y CONFIG_SPI=y -CONFIG_SPI_TXX9=y # CONFIG_HWMON is not set CONFIG_WATCHDOG=y CONFIG_TXX9_WDT=m @@ -65,8 +61,6 @@ CONFIG_SND=m # CONFIG_SND_SPI is not set # CONFIG_SND_MIPS is not set CONFIG_SND_SOC=m -CONFIG_SND_SOC_TXX9ACLC=m -CONFIG_SND_SOC_TXX9ACLC_GENERIC=m # CONFIG_USB_SUPPORT is not set CONFIG_NEW_LEDS=y CONFIG_LEDS_CLASS=y @@ -78,7 +72,6 @@ CONFIG_RTC_CLASS=y CONFIG_RTC_INTF_DEV_UIE_EMUL=y CONFIG_RTC_DRV_RS5C348=y CONFIG_RTC_DRV_DS1742=y -CONFIG_RTC_DRV_TX4939=y CONFIG_DMADEVICES=y CONFIG_TXX9_DMAC=m # CONFIG_DNOTIFY is not set diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig index 7d6f235e8ccbcc09a0d95a93fce65f37dcd94239..9932a593e3c39604e932333290ac6b0d45c20310 100644 --- a/arch/mips/configs/rm200_defconfig +++ b/arch/mips/configs/rm200_defconfig @@ -29,9 +29,6 @@ CONFIG_NET_IPIP=m CONFIG_IP_MROUTE=y CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m CONFIG_TCP_MD5SIG=y CONFIG_IPV6_ROUTER_PREF=y CONFIG_IPV6_ROUTE_INFO=y @@ -39,7 +36,6 @@ CONFIG_INET6_AH=m CONFIG_INET6_ESP=m CONFIG_INET6_IPCOMP=m CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m CONFIG_IPV6_TUNNEL=m CONFIG_IPV6_MULTIPLE_TABLES=y CONFIG_IPV6_SUBTREES=y @@ -116,7 +112,6 @@ CONFIG_IP6_NF_FILTER=m CONFIG_IP6_NF_TARGET_REJECT=m CONFIG_IP6_NF_MANGLE=m CONFIG_IP6_NF_RAW=m -CONFIG_DECNET_NF_GRABULATOR=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m CONFIG_BRIDGE_EBT_T_FILTER=m @@ -137,7 +132,6 @@ CONFIG_BRIDGE_EBT_REDIRECT=m CONFIG_BRIDGE_EBT_SNAT=m CONFIG_BRIDGE_EBT_LOG=m CONFIG_BRIDGE=m -CONFIG_DECNET=m CONFIG_NET_SCHED=y CONFIG_NET_SCH_CBQ=m CONFIG_NET_SCH_HTB=m @@ -192,9 +186,7 @@ CONFIG_PARIDE_KTTI=m CONFIG_PARIDE_ON20=m CONFIG_PARIDE_ON26=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m -CONFIG_BLK_DEV_SX8=m CONFIG_BLK_DEV_RAM=m CONFIG_CDROM_PKTCDVD=m CONFIG_ATA_OVER_ETH=m @@ -400,7 +392,6 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig index eb359db15dba5681d03562b9268c097b1eb43e37..bf017d4930027a76faa866155dced945ed68ebb3 100644 --- a/arch/mips/configs/rt305x_defconfig +++ b/arch/mips/configs/rt305x_defconfig @@ -35,9 +35,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IP_MROUTE=y CONFIG_IP_MROUTE_MULTIPLE_TABLES=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set CONFIG_TCP_CONG_ADVANCED=y # CONFIG_TCP_CONG_BIC is not set @@ -140,7 +137,6 @@ CONFIG_CRC32_SARWATE=y # CONFIG_XZ_DEC_ARMTHUMB is not set # CONFIG_XZ_DEC_SPARC is not set CONFIG_PRINTK_TIME=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/configs/sb1250_swarm_defconfig b/arch/mips/configs/sb1250_swarm_defconfig index de94bf756a939ffb7d1571f9eb2639ff09d5e4e1..030186f895016ad227ebe72ccaa13991e5251650 100644 --- a/arch/mips/configs/sb1250_swarm_defconfig +++ b/arch/mips/configs/sb1250_swarm_defconfig @@ -88,7 +88,6 @@ CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAMELLIA=m @@ -96,7 +95,6 @@ CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m CONFIG_CRYPTO_FCRYPT=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_DEFLATE=m diff --git a/arch/mips/configs/vocore2_defconfig b/arch/mips/configs/vocore2_defconfig index a14f8ea5c38675f1ba2a04cc3285e24b43c728e1..e47d4cc3353be20623e23dffc3e70e9d5c0961de 100644 --- a/arch/mips/configs/vocore2_defconfig +++ b/arch/mips/configs/vocore2_defconfig @@ -35,9 +35,6 @@ CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set @@ -116,7 +113,7 @@ CONFIG_CRYPTO_LZO=y CONFIG_CRC16=y CONFIG_XZ_DEC=y CONFIG_PRINTK_TIME=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/configs/xway_defconfig b/arch/mips/configs/xway_defconfig index eeb689f715cb9e9fa852970776a9b59613777e2c..eb5acf1f24aeebc268a68351f1e96c885d8e8211 100644 --- a/arch/mips/configs/xway_defconfig +++ b/arch/mips/configs/xway_defconfig @@ -37,9 +37,6 @@ CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IP_MROUTE=y CONFIG_IP_MROUTE_MULTIPLE_TABLES=y CONFIG_SYN_COOKIES=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set CONFIG_TCP_CONG_ADVANCED=y # CONFIG_TCP_CONG_BIC is not set @@ -146,7 +143,6 @@ CONFIG_CRYPTO_ARC4=m CONFIG_CRC_ITU_T=m CONFIG_CRC32_SARWATE=y CONFIG_PRINTK_TIME=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..9003a5c1e879fa78e8d264db1a236824cde53aa8 --- /dev/null +++ b/arch/mips/crypto/Kconfig @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (mips)" + +config CRYPTO_CRC32_MIPS + tristate "CRC32c and CRC32" + depends on MIPS_CRC_SUPPORT + select CRYPTO_HASH + help + CRC32c and CRC32 CRC algorithms + + Architecture: mips + +config CRYPTO_POLY1305_MIPS + tristate "Hash functions: Poly1305" + depends on MIPS + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: mips + +config CRYPTO_MD5_OCTEON + tristate "Digests: MD5 (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) + + Architecture: mips OCTEON using crypto instructions, when available + +config CRYPTO_SHA1_OCTEON + tristate "Hash functions: SHA-1 (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: mips OCTEON + +config CRYPTO_SHA256_OCTEON + tristate "Hash functions: SHA-224 and SHA-256 (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: mips OCTEON using crypto instructions, when available + +config CRYPTO_SHA512_OCTEON + tristate "Hash functions: SHA-384 and SHA-512 (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: mips OCTEON using crypto instructions, when available + +config CRYPTO_CHACHA_MIPS + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (MIPS32r2)" + depends on CPU_MIPS32_R2 + select CRYPTO_SKCIPHER + select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: MIPS32r2 + +endmenu diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index 57561e0e6e8db7ad003e84be9cf7bf151638591e..44f9824c1d8c3b855fcf4b5c6a814476d3e9452a 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -63,10 +63,6 @@ extern void do_domain_IRQ(struct irq_domain *domain, unsigned int irq); extern void arch_init_irq(void); extern void spurious_interrupt(void); -extern int allocate_irqno(void); -extern void alloc_legacy_irqno(void); -extern void free_irqno(unsigned int irq); - /* * Before R2 the timer and performance counter interrupts were both fixed to * IE7. Since R2 their number has to be read from the c0_intctl register. diff --git a/arch/mips/include/asm/mach-ar7/ar7.h b/arch/mips/include/asm/mach-ar7/ar7.h index cbe75ade3277f51be62b646978f140938290d905..1e8621a6afa310e55cae10dcfa03710339790c73 100644 --- a/arch/mips/include/asm/mach-ar7/ar7.h +++ b/arch/mips/include/asm/mach-ar7/ar7.h @@ -104,8 +104,6 @@ struct plat_dsl_data { int reset_bit_sar; }; -extern int ar7_cpu_clock, ar7_bus_clock, ar7_dsp_clock; - static inline int ar7_is_titan(void) { return (readl((void *)KSEG1ADDR(AR7_REGS_GPIO + 0x24)) & 0xffff) == diff --git a/arch/mips/include/asm/octeon/cvmx-fpa.h b/arch/mips/include/asm/octeon/cvmx-fpa.h index 29ae63606ab44663ed98b31fc55ce241e7e59277..f6dfcca97f19c83d4594c46de6996f1d8fee099e 100644 --- a/arch/mips/include/asm/octeon/cvmx-fpa.h +++ b/arch/mips/include/asm/octeon/cvmx-fpa.h @@ -263,26 +263,6 @@ static inline void cvmx_fpa_free(void *ptr, uint64_t pool, cvmx_write_io(newptr.u64, num_cache_lines); } -/** - * Setup a FPA pool to control a new block of memory. - * This can only be called once per pool. Make sure proper - * locking enforces this. - * - * @pool: Pool to initialize - * 0 <= pool < 8 - * @name: Constant character string to name this pool. - * String is not copied. - * @buffer: Pointer to the block of memory to use. This must be - * accessible by all processors and external hardware. - * @block_size: Size for each block controlled by the FPA - * @num_blocks: Number of blocks - * - * Returns 0 on Success, - * -1 on failure - */ -extern int cvmx_fpa_setup_pool(uint64_t pool, const char *name, void *buffer, - uint64_t block_size, uint64_t num_blocks); - /** * Shutdown a Memory pool and validate that it had all of * the buffers originally placed in it. This should only be diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h index 7e714aefc76d80941f6feb27bcf1db78d6220ea2..5c1d726c702fe40fd06db9a4d34c5b1f8ba7d75f 100644 --- a/arch/mips/include/asm/octeon/octeon.h +++ b/arch/mips/include/asm/octeon/octeon.h @@ -43,7 +43,6 @@ extern int octeon_get_southbridge_interrupt(void); extern int octeon_get_boot_coremask(void); extern int octeon_get_boot_num_arguments(void); extern const char *octeon_get_boot_argument(int arg); -extern void octeon_hal_setup_reserved32(void); extern void octeon_user_io_init(void); extern void octeon_init_cvmcount(void); diff --git a/arch/mips/include/asm/octeon/pci-octeon.h b/arch/mips/include/asm/octeon/pci-octeon.h index b12d9a3fbfb6c040aab5b5b1a55fe67f517f44ec..2f46f6c6e3d0bafb91b74c51e0895e9280c71963 100644 --- a/arch/mips/include/asm/octeon/pci-octeon.h +++ b/arch/mips/include/asm/octeon/pci-octeon.h @@ -64,6 +64,4 @@ enum octeon_dma_bar_type { extern enum octeon_dma_bar_type octeon_dma_bar_type; void octeon_pci_dma_init(void); -extern char *octeon_swiotlb; - #endif diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index 4bb24579d12e43ff119c48e91b0023cafc140bf9..3fde1ff72bd167e8ce322a2a73dd5612c9b5423b 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -344,9 +344,6 @@ struct thread_struct { struct task_struct; -/* Free all resources held by a thread. */ -#define release_thread(thread) do { } while(0) - /* * Do necessary setup to start up a newly executed thread. */ diff --git a/arch/mips/include/asm/sibyte/sb1250.h b/arch/mips/include/asm/sibyte/sb1250.h index dbde5f93f0ddbe2e40cf8ee0e3282be7d8951500..495b31925ed77640508e96be7c1ae150b0319bed 100644 --- a/arch/mips/include/asm/sibyte/sb1250.h +++ b/arch/mips/include/asm/sibyte/sb1250.h @@ -32,7 +32,6 @@ extern unsigned int soc_type; extern unsigned int periph_rev; extern unsigned int zbbus_mhz; -extern void sb1250_time_init(void); extern void sb1250_mask_irq(int cpu, int irq); extern void sb1250_unmask_irq(int cpu, int irq); diff --git a/arch/mips/include/asm/sn/gda.h b/arch/mips/include/asm/sn/gda.h index d52f8162066130d4ddfe0670e15fae636d39e32e..5b8c96d5b5870e4acc11019b5f592ad3cd73bceb 100644 --- a/arch/mips/include/asm/sn/gda.h +++ b/arch/mips/include/asm/sn/gda.h @@ -16,8 +16,6 @@ #include -#define GDA_MAGIC 0x58464552 - /* * GDA Version History * diff --git a/arch/mips/include/asm/sni.h b/arch/mips/include/asm/sni.h index 7dfa297ce597a94586b11a215e2cb6d5f6ef9f32..7fb6656a6bfd62febe864827f042e959f13d8efc 100644 --- a/arch/mips/include/asm/sni.h +++ b/arch/mips/include/asm/sni.h @@ -226,9 +226,6 @@ extern void sni_pcit_cplus_irq_init(void); extern void sni_rm200_irq_init(void); extern void sni_pcimt_irq_init(void); -/* timer inits */ -extern void sni_cpu_time_init(void); - /* eisa init for RM200/400 */ #ifdef CONFIG_EISA extern int sni_eisa_root_init(void); diff --git a/arch/mips/include/asm/termios.h b/arch/mips/include/asm/termios.h deleted file mode 100644 index bc29eeacc55adb17ac7b95759d62222636619c9e..0000000000000000000000000000000000000000 --- a/arch/mips/include/asm/termios.h +++ /dev/null @@ -1,105 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1995, 1996, 2000, 2001 by Ralf Baechle - * Copyright (C) 2000, 2001 Silicon Graphics, Inc. - */ -#ifndef _ASM_TERMIOS_H -#define _ASM_TERMIOS_H - -#include -#include - -/* - * intr=^C quit=^\ erase=del kill=^U - * vmin=\1 vtime=\0 eol2=\0 swtc=\0 - * start=^Q stop=^S susp=^Z vdsusp= - * reprint=^R discard=^U werase=^W lnext=^V - * eof=^D eol=\0 - */ -#define INIT_C_CC "\003\034\177\025\1\0\0\0\021\023\032\0\022\017\027\026\004\0" - -#include - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -static inline int user_termio_to_kernel_termios(struct ktermios *termios, - struct termio __user *termio) -{ - unsigned short iflag, oflag, cflag, lflag; - unsigned int err; - - if (!access_ok(termio, sizeof(struct termio))) - return -EFAULT; - - err = __get_user(iflag, &termio->c_iflag); - termios->c_iflag = (termios->c_iflag & 0xffff0000) | iflag; - err |=__get_user(oflag, &termio->c_oflag); - termios->c_oflag = (termios->c_oflag & 0xffff0000) | oflag; - err |=__get_user(cflag, &termio->c_cflag); - termios->c_cflag = (termios->c_cflag & 0xffff0000) | cflag; - err |=__get_user(lflag, &termio->c_lflag); - termios->c_lflag = (termios->c_lflag & 0xffff0000) | lflag; - err |=__get_user(termios->c_line, &termio->c_line); - if (err) - return -EFAULT; - - if (__copy_from_user(termios->c_cc, termio->c_cc, NCC)) - return -EFAULT; - - return 0; -} - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -static inline int kernel_termios_to_user_termio(struct termio __user *termio, - struct ktermios *termios) -{ - int err; - - if (!access_ok(termio, sizeof(struct termio))) - return -EFAULT; - - err = __put_user(termios->c_iflag, &termio->c_iflag); - err |= __put_user(termios->c_oflag, &termio->c_oflag); - err |= __put_user(termios->c_cflag, &termio->c_cflag); - err |= __put_user(termios->c_lflag, &termio->c_lflag); - err |= __put_user(termios->c_line, &termio->c_line); - if (err) - return -EFAULT; - - if (__copy_to_user(termio->c_cc, termios->c_cc, NCC)) - return -EFAULT; - - return 0; -} - -static inline int user_termios_to_kernel_termios(struct ktermios __user *k, - struct termios2 *u) -{ - return copy_from_user(k, u, sizeof(struct termios2)) ? -EFAULT : 0; -} - -static inline int kernel_termios_to_user_termios(struct termios2 __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios2)) ? -EFAULT : 0; -} - -static inline int user_termios_to_kernel_termios_1(struct ktermios *k, - struct termios __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios)) ? -EFAULT : 0; -} - -static inline int kernel_termios_to_user_termios_1(struct termios __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios)) ? -EFAULT : 0; -} - -#endif /* _ASM_TERMIOS_H */ diff --git a/arch/mips/include/uapi/asm/mman.h b/arch/mips/include/uapi/asm/mman.h index 1be428663c102701e26a1516bb6ca8eb0a083a75..c6e1fc77c9968874feefc79b7c94a165d0ad89d2 100644 --- a/arch/mips/include/uapi/asm/mman.h +++ b/arch/mips/include/uapi/asm/mman.h @@ -103,6 +103,8 @@ #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index 7c96282bff2e3d8171f6ecec016eb424881cea5f..5d1addac5e284cfdc3e7c177773f39ff27158c2c 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for the Linux/MIPS kernel. # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y += branch.o cmpxchg.o elf.o entry.o genex.o idle.o irq.o \ +obj-y += head.o branch.o cmpxchg.o elf.o entry.o genex.o idle.o irq.o \ process.o prom.o ptrace.o reset.o setup.o signal.o \ syscall.o time.o topology.o traps.o unaligned.o watch.o \ vdso.o cacheinfo.o diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 35b912bce42975340d6674ef2a607fa84f165696..bbe9ce471791e7895fd57c4756474b36ffa1a4bf 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -711,7 +711,7 @@ unsigned long mips_stack_top(void) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= prandom_u32_max(PAGE_SIZE); return sp & ALMASK; } diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c index 7db6ff9aed7dff9cef8c56544be894b71c307b31..f88ce78e13e3a2de9f51d33531398c96d807d7f2 100644 --- a/arch/mips/kernel/prom.c +++ b/arch/mips/kernel/prom.c @@ -26,7 +26,7 @@ __init void mips_set_machine_name(const char *name) if (name == NULL) return; - strlcpy(mips_machine_name, name, sizeof(mips_machine_name)); + strscpy(mips_machine_name, name, sizeof(mips_machine_name)); pr_info("MIPS: machine is %s\n", mips_get_machine_name()); } @@ -52,9 +52,9 @@ int __init __dt_register_buses(const char *bus0, const char *bus1) if (!of_have_populated_dt()) panic("device tree not present"); - strlcpy(of_ids[0].compatible, bus0, sizeof(of_ids[0].compatible)); + strscpy(of_ids[0].compatible, bus0, sizeof(of_ids[0].compatible)); if (bus1) { - strlcpy(of_ids[1].compatible, bus1, + strscpy(of_ids[1].compatible, bus1, sizeof(of_ids[1].compatible)); } diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index 56b51de2dc51b3e00e1e34f2fe4df6d90c04e146..58fc8d089402bd86649596ef422aaee7c7814fcb 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -340,7 +340,7 @@ void *__init relocate_kernel(void) early_init_dt_scan(fdt); if (boot_command_line[0]) { /* Boot command line was passed in device tree */ - strlcpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); + strscpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); } #endif /* CONFIG_USE_OF */ diff --git a/arch/mips/kernel/segment.c b/arch/mips/kernel/segment.c index 0a9bd7b0983b93333d7fcfadc1cece892cc0aa55..24560501c70d6818e7ed112c09d0cfce5240b4c8 100644 --- a/arch/mips/kernel/segment.c +++ b/arch/mips/kernel/segment.c @@ -46,7 +46,7 @@ static void build_segment_config(char *str, unsigned int cfg) ((cfg & MIPS_SEGCFG_EU) >> MIPS_SEGCFG_EU_SHIFT)); } -static int show_segments(struct seq_file *m, void *v) +static int segments_show(struct seq_file *m, void *v) { unsigned int segcfg; char str[42]; @@ -80,18 +80,7 @@ static int show_segments(struct seq_file *m, void *v) return 0; } - -static int segments_open(struct inode *inode, struct file *file) -{ - return single_open(file, show_segments, NULL); -} - -static const struct file_operations segments_fops = { - .open = segments_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(segments); static int __init segments_info(void) { diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 2ca156a5b23198cff29d60b27148d8bdac40286f..f1c88f8a1dc512e7b7a837f8f1ac62048c69e573 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -42,6 +42,7 @@ #include #include #include +#include #ifdef CONFIG_MIPS_ELF_APPENDED_DTB char __section(".appended_dtb") __appended_dtb[0x100000]; @@ -750,12 +751,30 @@ static void __init prefill_possible_map(void) for (; i < NR_CPUS; i++) set_cpu_possible(i, false); - nr_cpu_ids = possible; + set_nr_cpu_ids(possible); } #else static inline void prefill_possible_map(void) {} #endif +static void __init setup_rng_seed(void) +{ + char *rng_seed_hex = fw_getenv("rngseed"); + u8 rng_seed[512]; + size_t len; + + if (!rng_seed_hex) + return; + + len = min(sizeof(rng_seed), strlen(rng_seed_hex) / 2); + if (hex2bin(rng_seed, rng_seed_hex, len)) + return; + + add_bootloader_randomness(rng_seed, len); + memzero_explicit(rng_seed, len); + memzero_explicit(rng_seed_hex, len * 2); +} + void __init setup_arch(char **cmdline_p) { cpu_probe(); @@ -786,6 +805,8 @@ void __init setup_arch(char **cmdline_p) paging_init(); memblock_dump_all(); + + setup_rng_seed(); } unsigned long kernelsp[NR_CPUS]; diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index b2cc2c2dd4bfc44493dd11526b1eebe3b23ef2d2..5fd9bf1d596c6451b6d819952427c11389671f2a 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -79,7 +79,7 @@ static unsigned long vdso_base(void) } if (current->flags & PF_RANDOMIZE) { - base += get_random_int() & (VDSO_RANDOMIZE_SIZE - 1); + base += prandom_u32_max(VDSO_RANDOMIZE_SIZE); base = PAGE_ALIGN(base); } diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index b494d8d39290b0c3b189a204d03161e92ca0d939..edaec93a1a1feb6737e0ba8467ff76d2d66c6893 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -955,13 +955,11 @@ enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu) kvm_vcpu_halt(vcpu); /* - * We we are runnable, then definitely go off to user space to + * We are runnable, then definitely go off to user space to * check if any I/O interrupts are pending. */ - if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) { - kvm_clear_request(KVM_REQ_UNHALT, vcpu); + if (kvm_arch_vcpu_runnable(vcpu)) vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; - } } return EMULATE_DONE; diff --git a/arch/mips/lantiq/clk.c b/arch/mips/lantiq/clk.c index 7a623684d9b5ed415e167af74e0b9af22ea5d74c..2d5a0bcb0cec156dc5f0daedbdd7c56ff8d62ca4 100644 --- a/arch/mips/lantiq/clk.c +++ b/arch/mips/lantiq/clk.c @@ -50,6 +50,7 @@ struct clk *clk_get_io(void) { return &cpu_clk_generic[2]; } +EXPORT_SYMBOL_GPL(clk_get_io); struct clk *clk_get_ppe(void) { diff --git a/arch/mips/lantiq/prom.c b/arch/mips/lantiq/prom.c index c731082a0c42da11e6aef7c088b17f6a43cccfa5..be4829cc7a3a07941980cfc716b4a59e8509c6a8 100644 --- a/arch/mips/lantiq/prom.c +++ b/arch/mips/lantiq/prom.c @@ -34,6 +34,14 @@ unsigned long physical_memsize = 0L; */ static struct ltq_soc_info soc_info; +/* + * These structs are used to override vsmp_init_secondary() + */ +#if defined(CONFIG_MIPS_MT_SMP) +extern const struct plat_smp_ops vsmp_smp_ops; +static struct plat_smp_ops lantiq_smp_ops; +#endif + const char *get_system_type(void) { return soc_info.sys_type; @@ -84,6 +92,17 @@ void __init plat_mem_setup(void) __dt_setup_arch(dtb); } +#if defined(CONFIG_MIPS_MT_SMP) +static void lantiq_init_secondary(void) +{ + /* + * MIPS CPU startup function vsmp_init_secondary() will only + * enable some of the interrupts for the second CPU/VPE. + */ + set_c0_status(ST0_IM); +} +#endif + void __init prom_init(void) { /* call the soc specific detetcion code and get it to fill soc_info */ @@ -95,7 +114,10 @@ void __init prom_init(void) prom_init_cmdline(); #if defined(CONFIG_MIPS_MT_SMP) - if (register_vsmp_smp_ops()) - panic("failed to register_vsmp_smp_ops()"); + if (cpu_has_mipsmt) { + lantiq_smp_ops = vsmp_smp_ops; + lantiq_smp_ops.init_secondary = lantiq_init_secondary; + register_smp_ops(&lantiq_smp_ops); + } #endif } diff --git a/arch/mips/lantiq/xway/vmmc.c b/arch/mips/lantiq/xway/vmmc.c index 7a14da8d9d15ed44c43c691394b2925d276e50f8..2796e87dfcae31fbcebb60e218ae4bd6b5107d1f 100644 --- a/arch/mips/lantiq/xway/vmmc.c +++ b/arch/mips/lantiq/xway/vmmc.c @@ -4,9 +4,10 @@ * Copyright (C) 2012 John Crispin */ +#include #include +#include #include -#include #include #include @@ -25,23 +26,28 @@ EXPORT_SYMBOL(ltq_get_cp1_base); static int vmmc_probe(struct platform_device *pdev) { #define CP1_SIZE (1 << 20) + struct gpio_desc *gpio; int gpio_count; dma_addr_t dma; + int error; cp1_base = (void *) CPHYSADDR(dma_alloc_coherent(&pdev->dev, CP1_SIZE, &dma, GFP_KERNEL)); - gpio_count = of_gpio_count(pdev->dev.of_node); + gpio_count = gpiod_count(&pdev->dev, NULL); while (gpio_count > 0) { - enum of_gpio_flags flags; - int gpio = of_get_gpio_flags(pdev->dev.of_node, - --gpio_count, &flags); - if (gpio_request(gpio, "vmmc-relay")) + gpio = devm_gpiod_get_index(&pdev->dev, + NULL, --gpio_count, GPIOD_OUT_HIGH); + error = PTR_ERR_OR_ZERO(gpio); + if (error) { + dev_err(&pdev->dev, + "failed to request GPIO idx %d: %d\n", + gpio_count, error); continue; - dev_info(&pdev->dev, "requested GPIO %d\n", gpio); - gpio_direction_output(gpio, - (flags & OF_GPIO_ACTIVE_LOW) ? (0) : (1)); + } + + gpiod_set_consumer_name(gpio, "vmmc-relay"); } dev_info(&pdev->dev, "reserved %dMB at 0x%p", CP1_SIZE >> 20, cp1_base); diff --git a/arch/mips/lib/bswapdi.c b/arch/mips/lib/bswapdi.c index fcef74084492c97a4e874d471ca954e68dd1f6ec..88242dc7de17676e09d6750352f28c0b401a6293 100644 --- a/arch/mips/lib/bswapdi.c +++ b/arch/mips/lib/bswapdi.c @@ -1,17 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include + +/* To silence -Wmissing-prototypes. */ +unsigned long long __bswapdi2(unsigned long long u); unsigned long long notrace __bswapdi2(unsigned long long u) { - return (((u) & 0xff00000000000000ull) >> 56) | - (((u) & 0x00ff000000000000ull) >> 40) | - (((u) & 0x0000ff0000000000ull) >> 24) | - (((u) & 0x000000ff00000000ull) >> 8) | - (((u) & 0x00000000ff000000ull) << 8) | - (((u) & 0x0000000000ff0000ull) << 24) | - (((u) & 0x000000000000ff00ull) << 40) | - (((u) & 0x00000000000000ffull) << 56); + return ___constant_swab64(u); } - EXPORT_SYMBOL(__bswapdi2); diff --git a/arch/mips/lib/bswapsi.c b/arch/mips/lib/bswapsi.c index 22d8e4f6d66ec5c178a48c189a4e75bbb98abd47..2ed655497de5c5b2d5145b4de1fc1effd18294c7 100644 --- a/arch/mips/lib/bswapsi.c +++ b/arch/mips/lib/bswapsi.c @@ -1,13 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include + +/* To silence -Wmissing-prototypes. */ +unsigned int __bswapsi2(unsigned int u); unsigned int notrace __bswapsi2(unsigned int u) { - return (((u) & 0xff000000) >> 24) | - (((u) & 0x00ff0000) >> 8) | - (((u) & 0x0000ff00) << 8) | - (((u) & 0x000000ff) << 24); + return ___constant_swab32(u); } - EXPORT_SYMBOL(__bswapsi2); diff --git a/arch/mips/loongson2ef/common/pci.c b/arch/mips/loongson2ef/common/pci.c index 200916925e95e944d5a8960063d8c95932d0df1f..7d9ea51e8c01ec200bedfcc5a87036399b043022 100644 --- a/arch/mips/loongson2ef/common/pci.c +++ b/arch/mips/loongson2ef/common/pci.c @@ -73,8 +73,6 @@ static void __init setup_pcimap(void) #endif } -extern int sbx00_acpi_init(void); - static int __init pcibios_init(void) { setup_pcimap(); diff --git a/arch/mips/loongson32/common/platform.c b/arch/mips/loongson32/common/platform.c index 794c96c2a4cdd96cbaedb651b36a0421b09cbf37..311dc1580bbde98e59f33f508afba7695714923d 100644 --- a/arch/mips/loongson32/common/platform.c +++ b/arch/mips/loongson32/common/platform.c @@ -98,7 +98,7 @@ int ls1x_eth_mux_init(struct platform_device *pdev, void *priv) if (plat_dat->bus_id) { __raw_writel(__raw_readl(LS1X_MUX_CTRL0) | GMAC1_USE_UART1 | GMAC1_USE_UART0, LS1X_MUX_CTRL0); - switch (plat_dat->interface) { + switch (plat_dat->phy_interface) { case PHY_INTERFACE_MODE_RGMII: val &= ~(GMAC1_USE_TXCLK | GMAC1_USE_PWM23); break; @@ -107,12 +107,12 @@ int ls1x_eth_mux_init(struct platform_device *pdev, void *priv) break; default: pr_err("unsupported mii mode %d\n", - plat_dat->interface); + plat_dat->phy_interface); return -ENOTSUPP; } val &= ~GMAC1_SHUT; } else { - switch (plat_dat->interface) { + switch (plat_dat->phy_interface) { case PHY_INTERFACE_MODE_RGMII: val &= ~(GMAC0_USE_TXCLK | GMAC0_USE_PWM01); break; @@ -121,7 +121,7 @@ int ls1x_eth_mux_init(struct platform_device *pdev, void *priv) break; default: pr_err("unsupported mii mode %d\n", - plat_dat->interface); + plat_dat->phy_interface); return -ENOTSUPP; } val &= ~GMAC0_SHUT; @@ -131,7 +131,7 @@ int ls1x_eth_mux_init(struct platform_device *pdev, void *priv) plat_dat = dev_get_platdata(&pdev->dev); val &= ~PHY_INTF_SELI; - if (plat_dat->interface == PHY_INTERFACE_MODE_RMII) + if (plat_dat->phy_interface == PHY_INTERFACE_MODE_RMII) val |= 0x4 << PHY_INTF_SELI_SHIFT; __raw_writel(val, LS1X_MUX_CTRL1); @@ -146,9 +146,9 @@ static struct plat_stmmacenet_data ls1x_eth0_pdata = { .bus_id = 0, .phy_addr = -1, #if defined(CONFIG_LOONGSON1_LS1B) - .interface = PHY_INTERFACE_MODE_MII, + .phy_interface = PHY_INTERFACE_MODE_MII, #elif defined(CONFIG_LOONGSON1_LS1C) - .interface = PHY_INTERFACE_MODE_RMII, + .phy_interface = PHY_INTERFACE_MODE_RMII, #endif .mdio_bus_data = &ls1x_mdio_bus_data, .dma_cfg = &ls1x_eth_dma_cfg, @@ -186,7 +186,7 @@ struct platform_device ls1x_eth0_pdev = { static struct plat_stmmacenet_data ls1x_eth1_pdata = { .bus_id = 1, .phy_addr = -1, - .interface = PHY_INTERFACE_MODE_MII, + .phy_interface = PHY_INTERFACE_MODE_MII, .mdio_bus_data = &ls1x_mdio_bus_data, .dma_cfg = &ls1x_eth_dma_cfg, .has_gmac = 1, diff --git a/arch/mips/loongson32/ls1c/board.c b/arch/mips/loongson32/ls1c/board.c index e9de6da0ce51ff652ddcdbb5fdb7c79104f49454..9dcfe9de55b0ae983a2c4f8ebcf9ae23344a9b6f 100644 --- a/arch/mips/loongson32/ls1c/board.c +++ b/arch/mips/loongson32/ls1c/board.c @@ -15,7 +15,6 @@ static struct platform_device *ls1c_platform_devices[] __initdata = { static int __init ls1c_platform_init(void) { ls1x_serial_set_uartclk(&ls1x_uart_pdev); - ls1x_rtc_set_extclk(&ls1x_rtc_pdev); return platform_add_devices(ls1c_platform_devices, ARRAY_SIZE(ls1c_platform_devices)); diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c index 587cf1d115e8faac610a45823645c472f9c7e565..265bc57819dfb561b7698183fc5cf21e58132b05 100644 --- a/arch/mips/math-emu/cp1emu.c +++ b/arch/mips/math-emu/cp1emu.c @@ -1032,7 +1032,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, */ if (dec_insn.micro_mips_mode) { /* - * If next instruction is a 16-bit instruction, then it + * If next instruction is a 16-bit instruction, then * it cannot be a FPU instruction. This could happen * since we can be called for non-FPU instructions. */ diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c index 83c975d5cca27d6135c3fe0b44ab9ddda45c6449..ace5db3fbd171fc63421cc716c2f00d4967a8501 100644 --- a/arch/mips/net/bpf_jit_comp32.c +++ b/arch/mips/net/bpf_jit_comp32.c @@ -1376,12 +1376,20 @@ void build_prologue(struct jit_context *ctx) const u8 *fp = bpf2mips32[BPF_REG_FP]; int stack, saved, locals, reserved; + /* + * In the unlikely event that the TCC limit is raised to more + * than 16 bits, it is clamped to the maximum value allowed for + * the generated code (0xffff). It is better fail to compile + * instead of degrading gracefully. + */ + BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff); + /* * The first two instructions initialize TCC in the reserved (for us) * 16-byte area in the parent's stack frame. On a tail call, the * calling function jumps into the prologue after these instructions. */ - emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff)); + emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); /* diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c index 6475828ffb36d5544e3457ed670a35b8e83734b9..0e7c1bdcf9148e07fced608a435bc8d3987169a0 100644 --- a/arch/mips/net/bpf_jit_comp64.c +++ b/arch/mips/net/bpf_jit_comp64.c @@ -547,12 +547,20 @@ void build_prologue(struct jit_context *ctx) u8 zx = bpf2mips64[JIT_REG_ZX]; int stack, saved, locals, reserved; + /* + * In the unlikely event that the TCC limit is raised to more + * than 16 bits, it is clamped to the maximum value allowed for + * the generated code (0xffff). It is better fail to compile + * instead of degrading gracefully. + */ + BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff); + /* * The first instruction initializes the tail call count register. * On a tail call, the calling function jumps into the prologue * after this instruction. */ - emit(ctx, ori, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff)); + emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); /* === Entry-point for tail calls === */ diff --git a/arch/mips/pci/pci-ar2315.c b/arch/mips/pci/pci-ar2315.c index 30e0922f4ceaecf3ec4f64653c58c15068ae7f7c..e17d862cfa4c6ab586017506f23ca3459d96c6f1 100644 --- a/arch/mips/pci/pci-ar2315.c +++ b/arch/mips/pci/pci-ar2315.c @@ -2,7 +2,7 @@ /* */ -/** +/* * Both AR2315 and AR2316 chips have PCI interface unit, which supports DMA * and interrupt. PCI interface supports MMIO access method, but does not * seem to support I/O ports. diff --git a/arch/mips/pci/pci-lantiq.c b/arch/mips/pci/pci-lantiq.c index 1ca42f482130204c103c532ac781e8853724b0f9..8d16cd021f6043872138b8632a7e4e049b05c852 100644 --- a/arch/mips/pci/pci-lantiq.c +++ b/arch/mips/pci/pci-lantiq.c @@ -9,11 +9,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include @@ -62,7 +62,7 @@ __iomem void *ltq_pci_mapped_cfg; static __iomem void *ltq_pci_membase; -static int reset_gpio; +static struct gpio_desc *reset_gpio; static struct clk *clk_pci, *clk_external; static struct resource pci_io_resource; static struct resource pci_mem_resource; @@ -95,6 +95,7 @@ static int ltq_pci_startup(struct platform_device *pdev) struct device_node *node = pdev->dev.of_node; const __be32 *req_mask, *bus_clk; u32 temp_buffer; + int error; /* get our clocks */ clk_pci = clk_get(&pdev->dev, NULL); @@ -123,17 +124,14 @@ static int ltq_pci_startup(struct platform_device *pdev) clk_disable(clk_external); /* setup reset gpio used by pci */ - reset_gpio = of_get_named_gpio(node, "gpio-reset", 0); - if (gpio_is_valid(reset_gpio)) { - int ret = devm_gpio_request(&pdev->dev, - reset_gpio, "pci-reset"); - if (ret) { - dev_err(&pdev->dev, - "failed to request gpio %d\n", reset_gpio); - return ret; - } - gpio_direction_output(reset_gpio, 1); + reset_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", + GPIOD_OUT_LOW); + error = PTR_ERR_OR_ZERO(reset_gpio); + if (error) { + dev_err(&pdev->dev, "failed to request gpio: %d\n", error); + return error; } + gpiod_set_consumer_name(reset_gpio, "pci_reset"); /* enable auto-switching between PCI and EBU */ ltq_pci_w32(0xa, PCI_CR_CLK_CTRL); @@ -195,11 +193,11 @@ static int ltq_pci_startup(struct platform_device *pdev) ltq_ebu_w32(ltq_ebu_r32(LTQ_EBU_PCC_IEN) | 0x10, LTQ_EBU_PCC_IEN); /* toggle reset pin */ - if (gpio_is_valid(reset_gpio)) { - __gpio_set_value(reset_gpio, 0); + if (reset_gpio) { + gpiod_set_value_cansleep(reset_gpio, 1); wmb(); mdelay(1); - __gpio_set_value(reset_gpio, 1); + gpiod_set_value_cansleep(reset_gpio, 0); } return 0; } diff --git a/arch/mips/pic32/pic32mzda/init.c b/arch/mips/pic32/pic32mzda/init.c index d9c8c4e46aff99e1b86ba734a9a57f3522bec43d..08c46cf122d73576c9998c861ebe1fcd3f4076f7 100644 --- a/arch/mips/pic32/pic32mzda/init.c +++ b/arch/mips/pic32/pic32mzda/init.c @@ -44,7 +44,7 @@ void __init plat_mem_setup(void) pr_info(" builtin_cmdline : %s\n", CONFIG_CMDLINE); #endif if (dtb != __dtb_start) - strlcpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); + strscpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); #ifdef CONFIG_EARLY_PRINTK fw_init_early_console(-1); diff --git a/arch/mips/ralink/bootrom.c b/arch/mips/ralink/bootrom.c index 94ca8379b83c9031170817c10b489fb65cb90439..8c8cc0a81ed8628c9fd1aa50ea4812cadec3a9d2 100644 --- a/arch/mips/ralink/bootrom.c +++ b/arch/mips/ralink/bootrom.c @@ -18,22 +18,11 @@ static int bootrom_show(struct seq_file *s, void *unused) return 0; } - -static int bootrom_open(struct inode *inode, struct file *file) -{ - return single_open(file, bootrom_show, NULL); -} - -static const struct file_operations bootrom_file_ops = { - .open = bootrom_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(bootrom); static int __init bootrom_setup(void) { - debugfs_create_file("bootrom", 0444, NULL, NULL, &bootrom_file_ops); + debugfs_create_file("bootrom", 0444, NULL, NULL, &bootrom_fops); return 0; } diff --git a/arch/mips/sgi-ip27/ip27-xtalk.c b/arch/mips/sgi-ip27/ip27-xtalk.c index e762886d1dda9f900866b47139fd9c5f711d532c..5143d1cf8984c5b2ec9139831bc1e07a1cbad94d 100644 --- a/arch/mips/sgi-ip27/ip27-xtalk.c +++ b/arch/mips/sgi-ip27/ip27-xtalk.c @@ -27,15 +27,18 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid) { struct xtalk_bridge_platform_data *bd; struct sgi_w1_platform_data *wd; - struct platform_device *pdev; + struct platform_device *pdev_wd; + struct platform_device *pdev_bd; struct resource w1_res; unsigned long offset; offset = NODE_OFFSET(nasid); wd = kzalloc(sizeof(*wd), GFP_KERNEL); - if (!wd) - goto no_mem; + if (!wd) { + pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget); + return; + } snprintf(wd->dev_id, sizeof(wd->dev_id), "bridge-%012lx", offset + (widget << SWIN_SIZE_BITS)); @@ -46,24 +49,35 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid) w1_res.end = w1_res.start + 3; w1_res.flags = IORESOURCE_MEM; - pdev = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO); - if (!pdev) { - kfree(wd); - goto no_mem; + pdev_wd = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO); + if (!pdev_wd) { + pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget); + goto err_kfree_wd; + } + if (platform_device_add_resources(pdev_wd, &w1_res, 1)) { + pr_warn("xtalk:n%d/%x bridge failed to add platform resources.\n", nasid, widget); + goto err_put_pdev_wd; + } + if (platform_device_add_data(pdev_wd, wd, sizeof(*wd))) { + pr_warn("xtalk:n%d/%x bridge failed to add platform data.\n", nasid, widget); + goto err_put_pdev_wd; + } + if (platform_device_add(pdev_wd)) { + pr_warn("xtalk:n%d/%x bridge failed to add platform device.\n", nasid, widget); + goto err_put_pdev_wd; } - platform_device_add_resources(pdev, &w1_res, 1); - platform_device_add_data(pdev, wd, sizeof(*wd)); /* platform_device_add_data() duplicates the data */ kfree(wd); - platform_device_add(pdev); bd = kzalloc(sizeof(*bd), GFP_KERNEL); - if (!bd) - goto no_mem; - pdev = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO); - if (!pdev) { - kfree(bd); - goto no_mem; + if (!bd) { + pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget); + goto err_unregister_pdev_wd; + } + pdev_bd = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO); + if (!pdev_bd) { + pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget); + goto err_kfree_bd; } @@ -84,15 +98,31 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid) bd->io.flags = IORESOURCE_IO; bd->io_offset = offset; - platform_device_add_data(pdev, bd, sizeof(*bd)); + if (platform_device_add_data(pdev_bd, bd, sizeof(*bd))) { + pr_warn("xtalk:n%d/%x bridge failed to add platform data.\n", nasid, widget); + goto err_put_pdev_bd; + } + if (platform_device_add(pdev_bd)) { + pr_warn("xtalk:n%d/%x bridge failed to add platform device.\n", nasid, widget); + goto err_put_pdev_bd; + } /* platform_device_add_data() duplicates the data */ kfree(bd); - platform_device_add(pdev); pr_info("xtalk:n%d/%x bridge widget\n", nasid, widget); return; -no_mem: - pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget); +err_put_pdev_bd: + platform_device_put(pdev_bd); +err_kfree_bd: + kfree(bd); +err_unregister_pdev_wd: + platform_device_unregister(pdev_wd); + return; +err_put_pdev_wd: + platform_device_put(pdev_wd); +err_kfree_wd: + kfree(wd); + return; } static int probe_one_port(nasid_t nasid, int widget, int masterwid) diff --git a/arch/mips/sgi-ip30/ip30-xtalk.c b/arch/mips/sgi-ip30/ip30-xtalk.c index 8129524421cb0af89d486fd4d089b4ede5ef33af..7ceb2b23ea1cf5690658144c538ef281d42fce1c 100644 --- a/arch/mips/sgi-ip30/ip30-xtalk.c +++ b/arch/mips/sgi-ip30/ip30-xtalk.c @@ -40,12 +40,15 @@ static void bridge_platform_create(int widget, int masterwid) { struct xtalk_bridge_platform_data *bd; struct sgi_w1_platform_data *wd; - struct platform_device *pdev; + struct platform_device *pdev_wd; + struct platform_device *pdev_bd; struct resource w1_res; wd = kzalloc(sizeof(*wd), GFP_KERNEL); - if (!wd) - goto no_mem; + if (!wd) { + pr_warn("xtalk:%x bridge create out of memory\n", widget); + return; + } snprintf(wd->dev_id, sizeof(wd->dev_id), "bridge-%012lx", IP30_SWIN_BASE(widget)); @@ -56,24 +59,35 @@ static void bridge_platform_create(int widget, int masterwid) w1_res.end = w1_res.start + 3; w1_res.flags = IORESOURCE_MEM; - pdev = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO); - if (!pdev) { - kfree(wd); - goto no_mem; + pdev_wd = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO); + if (!pdev_wd) { + pr_warn("xtalk:%x bridge create out of memory\n", widget); + goto err_kfree_wd; + } + if (platform_device_add_resources(pdev_wd, &w1_res, 1)) { + pr_warn("xtalk:%x bridge failed to add platform resources.\n", widget); + goto err_put_pdev_wd; + } + if (platform_device_add_data(pdev_wd, wd, sizeof(*wd))) { + pr_warn("xtalk:%x bridge failed to add platform data.\n", widget); + goto err_put_pdev_wd; + } + if (platform_device_add(pdev_wd)) { + pr_warn("xtalk:%x bridge failed to add platform device.\n", widget); + goto err_put_pdev_wd; } - platform_device_add_resources(pdev, &w1_res, 1); - platform_device_add_data(pdev, wd, sizeof(*wd)); /* platform_device_add_data() duplicates the data */ kfree(wd); - platform_device_add(pdev); bd = kzalloc(sizeof(*bd), GFP_KERNEL); - if (!bd) - goto no_mem; - pdev = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO); - if (!pdev) { - kfree(bd); - goto no_mem; + if (!bd) { + pr_warn("xtalk:%x bridge create out of memory\n", widget); + goto err_unregister_pdev_wd; + } + pdev_bd = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO); + if (!pdev_bd) { + pr_warn("xtalk:%x bridge create out of memory\n", widget); + goto err_kfree_bd; } bd->bridge_addr = IP30_RAW_SWIN_BASE(widget); @@ -93,15 +107,31 @@ static void bridge_platform_create(int widget, int masterwid) bd->io.flags = IORESOURCE_IO; bd->io_offset = IP30_SWIN_BASE(widget); - platform_device_add_data(pdev, bd, sizeof(*bd)); + if (platform_device_add_data(pdev_bd, bd, sizeof(*bd))) { + pr_warn("xtalk:%x bridge failed to add platform data.\n", widget); + goto err_put_pdev_bd; + } + if (platform_device_add(pdev_bd)) { + pr_warn("xtalk:%x bridge failed to add platform device.\n", widget); + goto err_put_pdev_bd; + } /* platform_device_add_data() duplicates the data */ kfree(bd); - platform_device_add(pdev); pr_info("xtalk:%x bridge widget\n", widget); return; -no_mem: - pr_warn("xtalk:%x bridge create out of memory\n", widget); +err_put_pdev_bd: + platform_device_put(pdev_bd); +err_kfree_bd: + kfree(bd); +err_unregister_pdev_wd: + platform_device_unregister(pdev_wd); + return; +err_put_pdev_wd: + platform_device_put(pdev_wd); +err_kfree_wd: + kfree(wd); + return; } static unsigned int __init xbow_widget_active(s8 wid) diff --git a/arch/mips/sibyte/sb1250/irq.c b/arch/mips/sibyte/sb1250/irq.c index 86f49c48fc34d2d9d696757830b4d7f419a1a538..2f08ad267a11e6b24e0d4818248c9272edf5012e 100644 --- a/arch/mips/sibyte/sb1250/irq.c +++ b/arch/mips/sibyte/sb1250/irq.c @@ -262,12 +262,6 @@ void __init arch_init_irq(void) __raw_writeq(tmp, IOADDR(A_IMR_REGISTER(0, R_IMR_INTERRUPT_MASK))); __raw_writeq(tmp, IOADDR(A_IMR_REGISTER(1, R_IMR_INTERRUPT_MASK))); - /* - * Note that the timer interrupts are also mapped, but this is - * done in sb1250_time_init(). Also, the profiling driver - * does its own management of IP7. - */ - /* Enable necessary IPs, disable the rest */ change_c0_status(ST0_IM, imask); } diff --git a/arch/nios2/Kbuild b/arch/nios2/Kbuild index 4e39f7abdeb6dc90d4018a927f11f139515a4ce8..fc2952edd2dec40f3b6c136a02f89d4068d56a63 100644 --- a/arch/nios2/Kbuild +++ b/arch/nios2/Kbuild @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-y += kernel/ mm/ platform/ boot/dts/ + # for cleaning subdir- += boot diff --git a/arch/nios2/Kconfig b/arch/nios2/Kconfig index 4167f1eb4cd83321333c4c5eea81dfce5646a995..a582f72104f39229ca524cc2aa5f645f3bc1d5f3 100644 --- a/arch/nios2/Kconfig +++ b/arch/nios2/Kconfig @@ -44,7 +44,7 @@ menu "Kernel features" source "kernel/Kconfig.hz" -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" range 9 20 default "11" diff --git a/arch/nios2/Makefile b/arch/nios2/Makefile index d6a7499b814c9364eddeb7354b017872fa5a712c..f1ff4ce0f1a2cf4e5b9674dffaf533bd2787d882 100644 --- a/arch/nios2/Makefile +++ b/arch/nios2/Makefile @@ -37,10 +37,7 @@ KBUILD_CFLAGS += -DUTS_SYSNAME=\"$(UTS_SYSNAME)\" KBUILD_CFLAGS += -fno-builtin KBUILD_CFLAGS += -G 0 -head-y := arch/nios2/kernel/head.o libs-y += arch/nios2/lib/ $(LIBGCC) -core-y += arch/nios2/kernel/ arch/nios2/mm/ -core-y += arch/nios2/platform/ INSTALL_PATH ?= /tftpboot nios2-boot := arch/$(ARCH)/boot @@ -48,8 +45,6 @@ BOOT_TARGETS = vmImage zImage PHONY += $(BOOT_TARGETS) install KBUILD_IMAGE := $(nios2-boot)/vmImage -core-y += $(nios2-boot)/dts/ - all: vmImage $(BOOT_TARGETS): vmlinux diff --git a/arch/nios2/configs/10m50_defconfig b/arch/nios2/configs/10m50_defconfig index a7967b4cfb6ed6979ff864473b427f2b650dd204..91c3fce4dc7fec49cf051819ae5bb3bc02841eed 100644 --- a/arch/nios2/configs/10m50_defconfig +++ b/arch/nios2/configs/10m50_defconfig @@ -74,4 +74,4 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_ROOT_NFS=y CONFIG_SUNRPC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/nios2/configs/3c120_defconfig b/arch/nios2/configs/3c120_defconfig index 423a0c40a1627d882676348b084be7417658f1fe..c42ad7e162a3640d2476b30b8e468f55e696a3fe 100644 --- a/arch/nios2/configs/3c120_defconfig +++ b/arch/nios2/configs/3c120_defconfig @@ -71,4 +71,4 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_ROOT_NFS=y CONFIG_SUNRPC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y diff --git a/arch/nios2/include/asm/processor.h b/arch/nios2/include/asm/processor.h index b8125dfbcad2d6cbe42239aeb3b2afa77bf6a79b..8916d93d5c2d0ba37dae84bf47979c3a21131838 100644 --- a/arch/nios2/include/asm/processor.h +++ b/arch/nios2/include/asm/processor.h @@ -64,11 +64,6 @@ extern void start_thread(struct pt_regs *regs, unsigned long pc, struct task_struct; -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - extern unsigned long __get_wchan(struct task_struct *p); #define task_pt_regs(p) \ diff --git a/arch/nios2/kernel/Makefile b/arch/nios2/kernel/Makefile index 0b645e1e3158e8e5fa4004621859643ea1090f3d..78a913181fa19dece7f9b746f6a22435315c3b61 100644 --- a/arch/nios2/kernel/Makefile +++ b/arch/nios2/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for the nios2 linux kernel. # -extra-y += head.o extra-y += vmlinux.lds +obj-y += head.o obj-y += cpuinfo.o obj-y += entry.o obj-y += insnemu.o diff --git a/arch/openrisc/Makefile b/arch/openrisc/Makefile index b446510173cde94ba0ab9bf79a20a98c5d638c42..68249521db5ab6afd87663df9dae22f1db71e72a 100644 --- a/arch/openrisc/Makefile +++ b/arch/openrisc/Makefile @@ -55,8 +55,6 @@ ifeq ($(CONFIG_OPENRISC_HAVE_INST_SEXT),y) KBUILD_CFLAGS += $(call cc-option,-msext) endif -head-y := arch/openrisc/kernel/head.o - libs-y += $(LIBGCC) PHONY += vmlinux.bin diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h index aa1699c18add85ee7ec81918ba0f4b0129b2ce1a..ed9efb430afa17c9b655ef84789ebd7ee7b47aa2 100644 --- a/arch/openrisc/include/asm/processor.h +++ b/arch/openrisc/include/asm/processor.h @@ -72,7 +72,6 @@ struct thread_struct { void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp); -void release_thread(struct task_struct *); unsigned long __get_wchan(struct task_struct *p); #define cpu_relax() barrier() diff --git a/arch/openrisc/kernel/Makefile b/arch/openrisc/kernel/Makefile index 2d172e79f58d014f344446d8849bb7645dfabd7a..79129161f3e031be9aae62bc78242b8fa542e893 100644 --- a/arch/openrisc/kernel/Makefile +++ b/arch/openrisc/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for the linux kernel. # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y := setup.o or32_ksyms.o process.o dma.o \ +obj-y := head.o setup.o or32_ksyms.o process.o dma.o \ traps.o time.o irq.o entry.o ptrace.o signal.o \ sys_call_table.o unwinder.o diff --git a/arch/openrisc/kernel/dma.c b/arch/openrisc/kernel/dma.c index a82b2caaa560d6e3dc8e2b5bbcfc2240cfb54291..b3edbb33b621d0431665374c9d87313163b0312c 100644 --- a/arch/openrisc/kernel/dma.c +++ b/arch/openrisc/kernel/dma.c @@ -74,10 +74,10 @@ void *arch_dma_set_uncached(void *cpu_addr, size_t size) * We need to iterate through the pages, clearing the dcache for * them and setting the cache-inhibit bit. */ - mmap_read_lock(&init_mm); - error = walk_page_range(&init_mm, va, va + size, &set_nocache_walk_ops, - NULL); - mmap_read_unlock(&init_mm); + mmap_write_lock(&init_mm); + error = walk_page_range_novma(&init_mm, va, va + size, + &set_nocache_walk_ops, NULL, NULL); + mmap_write_unlock(&init_mm); if (error) return ERR_PTR(error); @@ -88,11 +88,11 @@ void arch_dma_clear_uncached(void *cpu_addr, size_t size) { unsigned long va = (unsigned long)cpu_addr; - mmap_read_lock(&init_mm); + mmap_write_lock(&init_mm); /* walk_page_range shouldn't be able to fail here */ - WARN_ON(walk_page_range(&init_mm, va, va + size, - &clear_nocache_walk_ops, NULL)); - mmap_read_unlock(&init_mm); + WARN_ON(walk_page_range_novma(&init_mm, va, va + size, + &clear_nocache_walk_ops, NULL, NULL)); + mmap_write_unlock(&init_mm); } void arch_sync_dma_for_device(phys_addr_t addr, size_t size, diff --git a/arch/openrisc/kernel/process.c b/arch/openrisc/kernel/process.c index 52dc983ddeba30c6e978aa683e256be746da7139..f94b5ec06786e9ed10a5b3b80c61d0696558536f 100644 --- a/arch/openrisc/kernel/process.c +++ b/arch/openrisc/kernel/process.c @@ -125,10 +125,6 @@ void show_regs(struct pt_regs *regs) show_registers(regs); } -void release_thread(struct task_struct *dead_task) -{ -} - /* * Copy the thread-specific (arch specific) info from the current * process to the new one p diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 7f059cd1196a15e99dbe31a577697735113f0c97..a98940e6424327fe5a5232dc0613404c5c559ae4 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -146,10 +146,10 @@ menu "Processor type and features" choice prompt "Processor type" - default PA7000 + default PA7000 if "$(ARCH)" = "parisc" config PA7000 - bool "PA7000/PA7100" + bool "PA7000/PA7100" if "$(ARCH)" = "parisc" help This is the processor type of your CPU. This information is used for optimizing purposes. In order to compile a kernel @@ -160,21 +160,21 @@ config PA7000 which is required on some machines. config PA7100LC - bool "PA7100LC" + bool "PA7100LC" if "$(ARCH)" = "parisc" help Select this option for the PCX-L processor, as used in the 712, 715/64, 715/80, 715/100, 715/100XC, 725/100, 743, 748, D200, D210, D300, D310 and E-class config PA7200 - bool "PA7200" + bool "PA7200" if "$(ARCH)" = "parisc" help Select this option for the PCX-T' processor, as used in the C100, C110, J100, J110, J210XC, D250, D260, D350, D360, K100, K200, K210, K220, K400, K410 and K420 config PA7300LC - bool "PA7300LC" + bool "PA7300LC" if "$(ARCH)" = "parisc" help Select this option for the PCX-L2 processor, as used in the 744, A180, B132L, B160L, B180L, C132L, C160L, C180L, @@ -224,7 +224,8 @@ config MLONGCALLS Enabling this option will probably slow down your kernel. config 64BIT - bool "64-bit kernel" + def_bool y if "$(ARCH)" = "parisc64" + bool "64-bit kernel" if "$(ARCH)" = "parisc" depends on PA8X00 help Enable this if you want to support 64bit kernel on PA-RISC platform. diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index e38d993d87f2166abc60d883eddc1e1426889aa8..a2d8600521f9320e908828f2ed87315556e02106 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -113,8 +113,6 @@ cflags-$(CONFIG_PA7100LC) += -march=1.1 -mschedule=7100LC cflags-$(CONFIG_PA7300LC) += -march=1.1 -mschedule=7300 cflags-$(CONFIG_PA8X00) += -march=2.0 -mschedule=8000 -head-y := arch/parisc/kernel/head.o - KBUILD_CFLAGS += $(cflags-y) LIBGCC := $(shell $(CC) -print-libgcc-file-name) export LIBGCC diff --git a/arch/parisc/include/asm/alternative.h b/arch/parisc/include/asm/alternative.h index 0ec54f43d6d250e69400f69be7ee3ea1461793bc..1ed45fd085d3b80ece36baf4c815fa91e709df63 100644 --- a/arch/parisc/include/asm/alternative.h +++ b/arch/parisc/include/asm/alternative.h @@ -22,10 +22,10 @@ struct alt_instr { s32 orig_offset; /* offset to original instructions */ - s32 len; /* end of original instructions */ - u32 cond; /* see ALT_COND_XXX */ + s16 len; /* end of original instructions */ + u16 cond; /* see ALT_COND_XXX */ u32 replacement; /* replacement instruction or code */ -}; +} __packed; void set_kernel_text_rw(int enable_read_write); void apply_alternatives_all(void); @@ -35,8 +35,9 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end, /* Alternative SMP implementation. */ #define ALTERNATIVE(cond, replacement) "!0:" \ ".section .altinstructions, \"aw\" !" \ - ".word (0b-4-.), 1, " __stringify(cond) "," \ - __stringify(replacement) " !" \ + ".word (0b-4-.) !" \ + ".hword 1, " __stringify(cond) " !" \ + ".word " __stringify(replacement) " !" \ ".previous" #else @@ -44,15 +45,17 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end, /* to replace one single instructions by a new instruction */ #define ALTERNATIVE(from, to, cond, replacement)\ .section .altinstructions, "aw" ! \ - .word (from - .), (to - from)/4 ! \ - .word cond, replacement ! \ + .word (from - .) ! \ + .hword (to - from)/4, cond ! \ + .word replacement ! \ .previous /* to replace multiple instructions by new code */ #define ALTERNATIVE_CODE(from, num_instructions, cond, new_instr_ptr)\ .section .altinstructions, "aw" ! \ - .word (from - .), -num_instructions ! \ - .word cond, (new_instr_ptr - .) ! \ + .word (from - .) ! \ + .hword -num_instructions, cond ! \ + .word (new_instr_ptr - .) ! \ .previous #endif /* __ASSEMBLY__ */ diff --git a/arch/parisc/include/asm/bitops.h b/arch/parisc/include/asm/bitops.h index 56ffd260c669b7d9ce50838edb07acb1de004600..0ec9cfc5131fc3d23b9694f1c8814d0e2d415630 100644 --- a/arch/parisc/include/asm/bitops.h +++ b/arch/parisc/include/asm/bitops.h @@ -12,14 +12,6 @@ #include #include -/* compiler build environment sanity checks: */ -#if !defined(CONFIG_64BIT) && defined(__LP64__) -#error "Please use 'ARCH=parisc' to build the 32-bit kernel." -#endif -#if defined(CONFIG_64BIT) && !defined(__LP64__) -#error "Please use 'ARCH=parisc64' to build the 64-bit kernel." -#endif - /* See http://marc.theaimsgroup.com/?t=108826637900003 for discussion * on use of volatile and __*_bit() (set/clear/change): * *_bit() want use of volatile. diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h index 42ffb60a6ea9ca58c54516e2d6439c1697a251c8..c05e781be2f5bd5e12e5c356162694d5daa9d0f7 100644 --- a/arch/parisc/include/asm/io.h +++ b/arch/parisc/include/asm/io.h @@ -128,98 +128,16 @@ static inline void gsc_writeq(unsigned long long val, unsigned long addr) void __iomem *ioremap(unsigned long offset, unsigned long size); #define ioremap_wc ioremap #define ioremap_uc ioremap +#define pci_iounmap pci_iounmap extern void iounmap(const volatile void __iomem *addr); -static inline unsigned char __raw_readb(const volatile void __iomem *addr) -{ - return (*(volatile unsigned char __force *) (addr)); -} -static inline unsigned short __raw_readw(const volatile void __iomem *addr) -{ - return *(volatile unsigned short __force *) addr; -} -static inline unsigned int __raw_readl(const volatile void __iomem *addr) -{ - return *(volatile unsigned int __force *) addr; -} -static inline unsigned long long __raw_readq(const volatile void __iomem *addr) -{ - return *(volatile unsigned long long __force *) addr; -} - -static inline void __raw_writeb(unsigned char b, volatile void __iomem *addr) -{ - *(volatile unsigned char __force *) addr = b; -} -static inline void __raw_writew(unsigned short b, volatile void __iomem *addr) -{ - *(volatile unsigned short __force *) addr = b; -} -static inline void __raw_writel(unsigned int b, volatile void __iomem *addr) -{ - *(volatile unsigned int __force *) addr = b; -} -static inline void __raw_writeq(unsigned long long b, volatile void __iomem *addr) -{ - *(volatile unsigned long long __force *) addr = b; -} - -static inline unsigned char readb(const volatile void __iomem *addr) -{ - return __raw_readb(addr); -} -static inline unsigned short readw(const volatile void __iomem *addr) -{ - return le16_to_cpu((__le16 __force) __raw_readw(addr)); -} -static inline unsigned int readl(const volatile void __iomem *addr) -{ - return le32_to_cpu((__le32 __force) __raw_readl(addr)); -} -static inline unsigned long long readq(const volatile void __iomem *addr) -{ - return le64_to_cpu((__le64 __force) __raw_readq(addr)); -} - -static inline void writeb(unsigned char b, volatile void __iomem *addr) -{ - __raw_writeb(b, addr); -} -static inline void writew(unsigned short w, volatile void __iomem *addr) -{ - __raw_writew((__u16 __force) cpu_to_le16(w), addr); -} -static inline void writel(unsigned int l, volatile void __iomem *addr) -{ - __raw_writel((__u32 __force) cpu_to_le32(l), addr); -} -static inline void writeq(unsigned long long q, volatile void __iomem *addr) -{ - __raw_writeq((__u64 __force) cpu_to_le64(q), addr); -} - -#define readb readb -#define readw readw -#define readl readl -#define readq readq -#define writeb writeb -#define writew writew -#define writel writel -#define writeq writeq - -#define readb_relaxed(addr) readb(addr) -#define readw_relaxed(addr) readw(addr) -#define readl_relaxed(addr) readl(addr) -#define readq_relaxed(addr) readq(addr) -#define writeb_relaxed(b, addr) writeb(b, addr) -#define writew_relaxed(w, addr) writew(w, addr) -#define writel_relaxed(l, addr) writel(l, addr) -#define writeq_relaxed(q, addr) writeq(q, addr) - void memset_io(volatile void __iomem *addr, unsigned char val, int count); void memcpy_fromio(void *dst, const volatile void __iomem *src, int count); void memcpy_toio(volatile void __iomem *dst, const void *src, int count); +#define memset_io memset_io +#define memcpy_fromio memcpy_fromio +#define memcpy_toio memcpy_toio /* Port-space IO */ @@ -241,10 +159,15 @@ extern void eisa_out32(unsigned int data, unsigned short port); extern unsigned char inb(int addr); extern unsigned short inw(int addr); extern unsigned int inl(int addr); - extern void outb(unsigned char b, int addr); extern void outw(unsigned short b, int addr); extern void outl(unsigned int b, int addr); +#define inb inb +#define inw inw +#define inl inl +#define outb outb +#define outw outw +#define outl outl #elif defined(CONFIG_EISA) #define inb eisa_in8 #define inw eisa_in16 @@ -270,7 +193,9 @@ static inline int inl(unsigned long addr) BUG(); return -1; } - +#define inb inb +#define inw inw +#define inl inl #define outb(x, y) ({(void)(x); (void)(y); BUG(); 0;}) #define outw(x, y) ({(void)(x); (void)(y); BUG(); 0;}) #define outl(x, y) ({(void)(x); (void)(y); BUG(); 0;}) @@ -285,7 +210,12 @@ extern void insl (unsigned long port, void *dst, unsigned long count); extern void outsb (unsigned long port, const void *src, unsigned long count); extern void outsw (unsigned long port, const void *src, unsigned long count); extern void outsl (unsigned long port, const void *src, unsigned long count); - +#define insb insb +#define insw insw +#define insl insl +#define outsb outsb +#define outsw outsw +#define outsl outsl /* IO Port space is : BBiiii where BB is HBA number. */ #define IO_SPACE_LIMIT 0x00ffffff @@ -297,6 +227,7 @@ extern void outsl (unsigned long port, const void *src, unsigned long count); * value for either 32 or 64 bit mode */ #define F_EXTEND(x) ((unsigned long)((x) | (0xffffffff00000000ULL))) +#ifdef CONFIG_64BIT #define ioread64 ioread64 #define ioread64be ioread64be #define iowrite64 iowrite64 @@ -305,8 +236,31 @@ extern u64 ioread64(const void __iomem *addr); extern u64 ioread64be(const void __iomem *addr); extern void iowrite64(u64 val, void __iomem *addr); extern void iowrite64be(u64 val, void __iomem *addr); +#endif #include +/* + * These get provided from since parisc does not + * select GENERIC_IOMAP. + */ +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap +#define ioread8 ioread8 +#define ioread16 ioread16 +#define ioread32 ioread32 +#define ioread16be ioread16be +#define ioread32be ioread32be +#define iowrite8 iowrite8 +#define iowrite16 iowrite16 +#define iowrite32 iowrite32 +#define iowrite16be iowrite16be +#define iowrite32be iowrite32be +#define ioread8_rep ioread8_rep +#define ioread16_rep ioread16_rep +#define ioread32_rep ioread32_rep +#define iowrite8_rep iowrite8_rep +#define iowrite16_rep iowrite16_rep +#define iowrite32_rep iowrite32_rep /* * Convert a physical pointer to a virtual kernel pointer for /dev/mem @@ -316,4 +270,6 @@ extern void iowrite64be(u64 val, void __iomem *addr); extern int devmem_is_allowed(unsigned long pfn); +#include + #endif diff --git a/arch/parisc/include/asm/pdc.h b/arch/parisc/include/asm/pdc.h index b643092d4b985370702d13cbef4e6d9d143bfc85..fcbcf9a96c1112d39de10868cab3ac544a5b161e 100644 --- a/arch/parisc/include/asm/pdc.h +++ b/arch/parisc/include/asm/pdc.h @@ -19,9 +19,6 @@ extern unsigned long parisc_pat_pdc_cap; /* PDC capabilities (PAT) */ #define PDC_TYPE_SYSTEM_MAP 1 /* 32-bit, but supports PDC_SYSTEM_MAP */ #define PDC_TYPE_SNAKE 2 /* Doesn't support SYSTEM_MAP */ -void pdc_console_init(void); /* in pdc_console.c */ -void pdc_console_restart(void); - void setup_pdc(void); /* in inventory.c */ /* wrapper-functions from pdc.c */ diff --git a/arch/parisc/include/asm/pgtable.h b/arch/parisc/include/asm/pgtable.h index df7b931865d224fe42c3f12dfee9d947f4d0ee8b..ecd028854469856e0493e191e10b8738cf4fefed 100644 --- a/arch/parisc/include/asm/pgtable.h +++ b/arch/parisc/include/asm/pgtable.h @@ -192,6 +192,11 @@ extern void __update_cache(pte_t pte); #define _PAGE_PRESENT_BIT 22 /* (0x200) Software: translation valid */ #define _PAGE_HPAGE_BIT 21 /* (0x400) Software: Huge Page */ #define _PAGE_USER_BIT 20 /* (0x800) Software: User accessible page */ +#ifdef CONFIG_HUGETLB_PAGE +#define _PAGE_SPECIAL_BIT _PAGE_DMB_BIT /* DMB feature is currently unused */ +#else +#define _PAGE_SPECIAL_BIT _PAGE_HPAGE_BIT /* use unused HUGE PAGE bit */ +#endif /* N.B. The bits are defined in terms of a 32 bit word above, so the */ /* following macro is ok for both 32 and 64 bit. */ @@ -219,7 +224,7 @@ extern void __update_cache(pte_t pte); #define _PAGE_PRESENT (1 << xlate_pabit(_PAGE_PRESENT_BIT)) #define _PAGE_HUGE (1 << xlate_pabit(_PAGE_HPAGE_BIT)) #define _PAGE_USER (1 << xlate_pabit(_PAGE_USER_BIT)) -#define _PAGE_SPECIAL (_PAGE_DMB) +#define _PAGE_SPECIAL (1 << xlate_pabit(_PAGE_SPECIAL_BIT)) #define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED) #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_SPECIAL) diff --git a/arch/parisc/include/asm/processor.h b/arch/parisc/include/asm/processor.h index 4621ceb51314702a066b241bc6c8a36ec6e5b261..a608970b249af7fbbf3cef99a22fe9549f6c0c22 100644 --- a/arch/parisc/include/asm/processor.h +++ b/arch/parisc/include/asm/processor.h @@ -266,9 +266,6 @@ on downward growing arches, it looks like this: struct mm_struct; -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - extern unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) ((tsk)->thread.regs.iaoq[0]) diff --git a/arch/parisc/include/asm/termios.h b/arch/parisc/include/asm/termios.h deleted file mode 100644 index cded9dc90c1b89a45ead004c93fb90eb950b84b9..0000000000000000000000000000000000000000 --- a/arch/parisc/include/asm/termios.h +++ /dev/null @@ -1,52 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _PARISC_TERMIOS_H -#define _PARISC_TERMIOS_H - -#include - - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D vtime=\0 vmin=\1 sxtc=\0 - start=^Q stop=^S susp=^Z eol=\0 - reprint=^R discard=^U werase=^W lnext=^V - eol2=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -#define SET_LOW_TERMIOS_BITS(termios, termio, x) { \ - unsigned short __tmp; \ - get_user(__tmp,&(termio)->x); \ - *(unsigned short *) &(termios)->x = __tmp; \ -} - -#define user_termio_to_kernel_termios(termios, termio) \ -({ \ - SET_LOW_TERMIOS_BITS(termios, termio, c_iflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_oflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_cflag); \ - SET_LOW_TERMIOS_BITS(termios, termio, c_lflag); \ - copy_from_user((termios)->c_cc, (termio)->c_cc, NCC); \ -}) - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -#define kernel_termios_to_user_termio(termio, termios) \ -({ \ - put_user((termios)->c_iflag, &(termio)->c_iflag); \ - put_user((termios)->c_oflag, &(termio)->c_oflag); \ - put_user((termios)->c_cflag, &(termio)->c_cflag); \ - put_user((termios)->c_lflag, &(termio)->c_lflag); \ - put_user((termios)->c_line, &(termio)->c_line); \ - copy_to_user((termio)->c_cc, (termios)->c_cc, NCC); \ -}) - -#define user_termios_to_kernel_termios(k, u) copy_from_user(k, u, sizeof(struct termios2)) -#define kernel_termios_to_user_termios(u, k) copy_to_user(u, k, sizeof(struct termios2)) -#define user_termios_to_kernel_termios_1(k, u) copy_from_user(k, u, sizeof(struct termios)) -#define kernel_termios_to_user_termios_1(u, k) copy_to_user(u, k, sizeof(struct termios)) - -#endif /* _PARISC_TERMIOS_H */ diff --git a/arch/parisc/include/uapi/asm/mman.h b/arch/parisc/include/uapi/asm/mman.h index a7ea3204a5faa0ed79e3a473ea915ec92aead4c1..22133a6a506efba842e7286deed19c4fe9c1e6ce 100644 --- a/arch/parisc/include/uapi/asm/mman.h +++ b/arch/parisc/include/uapi/asm/mman.h @@ -70,6 +70,8 @@ #define MADV_WIPEONFORK 71 /* Zero memory on fork, child only */ #define MADV_KEEPONFORK 72 /* Undo MADV_WIPEONFORK */ +#define MADV_COLLAPSE 73 /* Synchronous hugepage collapse */ + #define MADV_HWPOISON 100 /* poison a page for testing */ #define MADV_SOFT_OFFLINE 101 /* soft offline page for testing */ diff --git a/arch/parisc/include/uapi/asm/termios.h b/arch/parisc/include/uapi/asm/termios.h deleted file mode 100644 index aba174f23ef003901f39c407e03b5cf6c71600a1..0000000000000000000000000000000000000000 --- a/arch/parisc/include/uapi/asm/termios.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _UAPI_PARISC_TERMIOS_H -#define _UAPI_PARISC_TERMIOS_H - -#include -#include - -struct winsize { - unsigned short ws_row; - unsigned short ws_col; - unsigned short ws_xpixel; - unsigned short ws_ypixel; -}; - -#define NCC 8 -struct termio { - unsigned short c_iflag; /* input mode flags */ - unsigned short c_oflag; /* output mode flags */ - unsigned short c_cflag; /* control mode flags */ - unsigned short c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - -/* modem lines */ -#define TIOCM_LE 0x001 -#define TIOCM_DTR 0x002 -#define TIOCM_RTS 0x004 -#define TIOCM_ST 0x008 -#define TIOCM_SR 0x010 -#define TIOCM_CTS 0x020 -#define TIOCM_CAR 0x040 -#define TIOCM_RNG 0x080 -#define TIOCM_DSR 0x100 -#define TIOCM_CD TIOCM_CAR -#define TIOCM_RI TIOCM_RNG -#define TIOCM_OUT1 0x2000 -#define TIOCM_OUT2 0x4000 -#define TIOCM_LOOP 0x8000 - -/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ - - -#endif /* _UAPI_PARISC_TERMIOS_H */ diff --git a/arch/parisc/kernel/Makefile b/arch/parisc/kernel/Makefile index d0bfac89a84272e1e8ae621ee4dc70b41ed6dbf3..3d138c9cf9ce9ebe9a68591831e676d605164dc3 100644 --- a/arch/parisc/kernel/Makefile +++ b/arch/parisc/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for arch/parisc/kernel # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y := cache.o pacache.o setup.o pdt.o traps.o time.o irq.o \ +obj-y := head.o cache.o pacache.o setup.o pdt.o traps.o time.o irq.o \ pa7300lc.o syscall.o entry.o sys_parisc.o firmware.o \ ptrace.o hardware.o inventory.o drivers.o alternative.o \ signal.o hpmc.o real2.o parisc_ksyms.o unaligned.o \ diff --git a/arch/parisc/kernel/alternative.c b/arch/parisc/kernel/alternative.c index daa1e9047275b0c1f683272a84f14642d1856117..66f5672c70bd469e6e1a261c4a7c080be1d808f5 100644 --- a/arch/parisc/kernel/alternative.c +++ b/arch/parisc/kernel/alternative.c @@ -26,7 +26,7 @@ void __init_or_module apply_alternatives(struct alt_instr *start, struct alt_instr *entry; int index = 0, applied = 0; int num_cpus = num_online_cpus(); - u32 cond_check; + u16 cond_check; cond_check = ALT_COND_ALWAYS | ((num_cpus == 1) ? ALT_COND_NO_SMP : 0) | @@ -45,8 +45,9 @@ void __init_or_module apply_alternatives(struct alt_instr *start, for (entry = start; entry < end; entry++, index++) { - u32 *from, cond, replacement; - s32 len; + u32 *from, replacement; + u16 cond; + s16 len; from = (u32 *)((ulong)&entry->orig_offset + entry->orig_offset); len = entry->len; diff --git a/arch/parisc/kernel/cache.c b/arch/parisc/kernel/cache.c index 3feb7694e0ca4fc3737898e9abea0ead3bbf6462..1d3b8bc8a6233641c4eeea6d8b775887b0de9a4e 100644 --- a/arch/parisc/kernel/cache.c +++ b/arch/parisc/kernel/cache.c @@ -657,15 +657,20 @@ static inline unsigned long mm_total_size(struct mm_struct *mm) { struct vm_area_struct *vma; unsigned long usize = 0; + VMA_ITERATOR(vmi, mm, 0); - for (vma = mm->mmap; vma && usize < parisc_cache_flush_threshold; vma = vma->vm_next) + for_each_vma(vmi, vma) { + if (usize >= parisc_cache_flush_threshold) + break; usize += vma->vm_end - vma->vm_start; + } return usize; } void flush_cache_mm(struct mm_struct *mm) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); /* * Flushing the whole cache on each cpu takes forever on @@ -685,7 +690,7 @@ void flush_cache_mm(struct mm_struct *mm) } /* Flush mm */ - for (vma = mm->mmap; vma; vma = vma->vm_next) + for_each_vma(vmi, vma) flush_cache_pages(vma, vma->vm_start, vma->vm_end); } diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index df8102fb435fced706266fe9786c7d46f24e9ccd..0e5ebfe8d9d293efc549fa2df3069fa16c560a3b 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -499,6 +499,10 @@ * Finally, _PAGE_READ goes in the top bit of PL1 (so we * trigger an access rights trap in user space if the user * tries to read an unreadable page */ +#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT + /* need to drop DMB bit, as it's used as SPECIAL flag */ + depi 0,_PAGE_SPECIAL_BIT,1,\pte +#endif depd \pte,8,7,\prot /* PAGE_USER indicates the page can be read with user privileges, @@ -529,6 +533,10 @@ * makes the tlb entry for the differently formatted pa11 * insertion instructions */ .macro make_insert_tlb_11 spc,pte,prot +#if _PAGE_SPECIAL_BIT == _PAGE_DMB_BIT + /* need to drop DMB bit, as it's used as SPECIAL flag */ + depi 0,_PAGE_SPECIAL_BIT,1,\pte +#endif zdep \spc,30,15,\prot dep \pte,8,7,\prot extru,= \pte,_PAGE_NO_CACHE_BIT,1,%r0 diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index e0a9e96576221a614522acc63e45f13e5248462e..fd15fd4bbb61b8f4cd7567b20a7b6d29f36b7864 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -22,7 +22,7 @@ #include #include - .level PA_ASM_LEVEL + .level 1.1 __INITDATA ENTRY(boot_args) @@ -70,6 +70,47 @@ $bss_loop: stw,ma %arg2,4(%r1) stw,ma %arg3,4(%r1) +#if !defined(CONFIG_64BIT) && defined(CONFIG_PA20) + /* This 32-bit kernel was compiled for PA2.0 CPUs. Check current CPU + * and halt kernel if we detect a PA1.x CPU. */ + ldi 32,%r10 + mtctl %r10,%cr11 + .level 2.0 + mfctl,w %cr11,%r10 + .level 1.1 + comib,<>,n 0,%r10,$cpu_ok + + load32 PA(msg1),%arg0 + ldi msg1_end-msg1,%arg1 +$iodc_panic: + copy %arg0, %r10 + copy %arg1, %r11 + load32 PA(init_stack),%sp +#define MEM_CONS 0x3A0 + ldw MEM_CONS+32(%r0),%arg0 // HPA + ldi ENTRY_IO_COUT,%arg1 + ldw MEM_CONS+36(%r0),%arg2 // SPA + ldw MEM_CONS+8(%r0),%arg3 // layers + load32 PA(__bss_start),%r1 + stw %r1,-52(%sp) // arg4 + stw %r0,-56(%sp) // arg5 + stw %r10,-60(%sp) // arg6 = ptr to text + stw %r11,-64(%sp) // arg7 = len + stw %r0,-68(%sp) // arg8 + load32 PA(.iodc_panic_ret), %rp + ldw MEM_CONS+40(%r0),%r1 // ENTRY_IODC + bv,n (%r1) +.iodc_panic_ret: + b . /* wait endless with ... */ + or %r10,%r10,%r10 /* qemu idle sleep */ +msg1: .ascii "Can't boot kernel which was built for PA8x00 CPUs on this machine.\r\n" +msg1_end: + +$cpu_ok: +#endif + + .level PA_ASM_LEVEL + /* Initialize startup VM. Just map first 16/32 MB of memory */ load32 PA(swapper_pg_dir),%r4 mtctl %r4,%cr24 /* Initialize kernel root pointer */ diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index fbb882cb8dbb5f9316af05f7c9f45faf90701cec..b05055f3ba4b8aa406c3a0881bb65b5984ce51f7 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -480,7 +480,7 @@ static void execute_on_irq_stack(void *func, unsigned long param1) *irq_stack_in_use = 1; } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void) { execute_on_irq_stack(__do_softirq, 0); diff --git a/arch/parisc/kernel/pdc_cons.c b/arch/parisc/kernel/pdc_cons.c index 2661cdd256ae7deabd69d06d492bc833f50e2a0d..7d0989f523d031bf74888e4ab96ed22b291ea457 100644 --- a/arch/parisc/kernel/pdc_cons.c +++ b/arch/parisc/kernel/pdc_cons.c @@ -1,46 +1,18 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * PDC Console support - ie use firmware to dump text via boot console + * PDC early console support - use PDC firmware to dump text via boot console * - * Copyright (C) 1999-2003 Matthew Wilcox - * Copyright (C) 2000 Martin K Petersen - * Copyright (C) 2000 John Marvin - * Copyright (C) 2000-2003 Paul Bame - * Copyright (C) 2000 Philipp Rumpf - * Copyright (C) 2000 Michael Ang - * Copyright (C) 2000 Grant Grundler - * Copyright (C) 2001-2002 Ryan Bradetich - * Copyright (C) 2001 Helge Deller - * Copyright (C) 2001 Thomas Bogendoerfer - * Copyright (C) 2002 Randolph Chung - * Copyright (C) 2010 Guy Martin + * Copyright (C) 2001-2022 Helge Deller */ -/* - * The PDC console is a simple console, which can be used for debugging - * boot related problems on HP PA-RISC machines. It is also useful when no - * other console works. - * - * This code uses the ROM (=PDC) based functions to read and write characters - * from and to PDC's boot path. - */ - -/* Define EARLY_BOOTUP_DEBUG to debug kernel related boot problems. - * On production kernels EARLY_BOOTUP_DEBUG should be undefined. */ -#define EARLY_BOOTUP_DEBUG - - -#include #include -#include #include -#include -#include +#include +#include #include /* for PAGE0 */ #include /* for iodc_call() proto and friends */ static DEFINE_SPINLOCK(pdc_console_lock); -static struct console pdc_cons; static void pdc_console_write(struct console *co, const char *s, unsigned count) { @@ -54,7 +26,8 @@ static void pdc_console_write(struct console *co, const char *s, unsigned count) spin_unlock_irqrestore(&pdc_console_lock, flags); } -int pdc_console_poll_key(struct console *co) +#ifdef CONFIG_KGDB +static int kgdb_pdc_read_char(void) { int c; unsigned long flags; @@ -63,201 +36,40 @@ int pdc_console_poll_key(struct console *co) c = pdc_iodc_getc(); spin_unlock_irqrestore(&pdc_console_lock, flags); - return c; -} - -static int pdc_console_setup(struct console *co, char *options) -{ - return 0; -} - -#if defined(CONFIG_PDC_CONSOLE) -#include -#include - -#define PDC_CONS_POLL_DELAY (30 * HZ / 1000) - -static void pdc_console_poll(struct timer_list *unused); -static DEFINE_TIMER(pdc_console_timer, pdc_console_poll); -static struct tty_port tty_port; - -static int pdc_console_tty_open(struct tty_struct *tty, struct file *filp) -{ - tty_port_tty_set(&tty_port, tty); - mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY); - - return 0; + return (c <= 0) ? NO_POLL_CHAR : c; } -static void pdc_console_tty_close(struct tty_struct *tty, struct file *filp) +static void kgdb_pdc_write_char(u8 chr) { - if (tty->count == 1) { - del_timer_sync(&pdc_console_timer); - tty_port_tty_set(&tty_port, NULL); - } + if (PAGE0->mem_cons.cl_class != CL_DUPLEX) + pdc_console_write(NULL, &chr, 1); } -static int pdc_console_tty_write(struct tty_struct *tty, const unsigned char *buf, int count) -{ - pdc_console_write(NULL, buf, count); - return count; -} - -static unsigned int pdc_console_tty_write_room(struct tty_struct *tty) -{ - return 32768; /* no limit, no buffer used */ -} - -static const struct tty_operations pdc_console_tty_ops = { - .open = pdc_console_tty_open, - .close = pdc_console_tty_close, - .write = pdc_console_tty_write, - .write_room = pdc_console_tty_write_room, +static struct kgdb_io kgdb_pdc_io_ops = { + .name = "kgdb_pdc", + .read_char = kgdb_pdc_read_char, + .write_char = kgdb_pdc_write_char, }; - -static void pdc_console_poll(struct timer_list *unused) -{ - int data, count = 0; - - while (1) { - data = pdc_console_poll_key(NULL); - if (data == -1) - break; - tty_insert_flip_char(&tty_port, data & 0xFF, TTY_NORMAL); - count ++; - } - - if (count) - tty_flip_buffer_push(&tty_port); - - if (pdc_cons.flags & CON_ENABLED) - mod_timer(&pdc_console_timer, jiffies + PDC_CONS_POLL_DELAY); -} - -static struct tty_driver *pdc_console_tty_driver; - -static int __init pdc_console_tty_driver_init(void) -{ - struct tty_driver *driver; - int err; - - /* Check if the console driver is still registered. - * It is unregistered if the pdc console was not selected as the - * primary console. */ - - struct console *tmp; - - console_lock(); - for_each_console(tmp) - if (tmp == &pdc_cons) - break; - console_unlock(); - - if (!tmp) { - printk(KERN_INFO "PDC console driver not registered anymore, not creating %s\n", pdc_cons.name); - return -ENODEV; - } - - printk(KERN_INFO "The PDC console driver is still registered, removing CON_BOOT flag\n"); - pdc_cons.flags &= ~CON_BOOT; - - driver = tty_alloc_driver(1, TTY_DRIVER_REAL_RAW | - TTY_DRIVER_RESET_TERMIOS); - if (IS_ERR(driver)) - return PTR_ERR(driver); - - tty_port_init(&tty_port); - - driver->driver_name = "pdc_cons"; - driver->name = "ttyB"; - driver->major = MUX_MAJOR; - driver->minor_start = 0; - driver->type = TTY_DRIVER_TYPE_SYSTEM; - driver->init_termios = tty_std_termios; - tty_set_operations(driver, &pdc_console_tty_ops); - tty_port_link_device(&tty_port, driver, 0); - - err = tty_register_driver(driver); - if (err) { - printk(KERN_ERR "Unable to register the PDC console TTY driver\n"); - tty_port_destroy(&tty_port); - tty_driver_kref_put(driver); - return err; - } - - pdc_console_tty_driver = driver; - - return 0; -} -device_initcall(pdc_console_tty_driver_init); - -static struct tty_driver * pdc_console_device (struct console *c, int *index) -{ - *index = c->index; - return pdc_console_tty_driver; -} -#else -#define pdc_console_device NULL #endif -static struct console pdc_cons = { - .name = "ttyB", - .write = pdc_console_write, - .device = pdc_console_device, - .setup = pdc_console_setup, - .flags = CON_BOOT | CON_PRINTBUFFER, - .index = -1, -}; - -static int pdc_console_initialized; - -static void pdc_console_init_force(void) +static int __init pdc_earlycon_setup(struct earlycon_device *device, + const char *opt) { - if (pdc_console_initialized) - return; - ++pdc_console_initialized; - + struct console *earlycon_console; + /* If the console is duplex then copy the COUT parameters to CIN. */ if (PAGE0->mem_cons.cl_class == CL_DUPLEX) memcpy(&PAGE0->mem_kbd, &PAGE0->mem_cons, sizeof(PAGE0->mem_cons)); - /* register the pdc console */ - register_console(&pdc_cons); -} + earlycon_console = device->con; + earlycon_console->write = pdc_console_write; + device->port.iotype = UPIO_MEM32BE; -void __init pdc_console_init(void) -{ -#if defined(EARLY_BOOTUP_DEBUG) || defined(CONFIG_PDC_CONSOLE) - pdc_console_init_force(); +#ifdef CONFIG_KGDB + kgdb_register_io_module(&kgdb_pdc_io_ops); #endif -#ifdef EARLY_BOOTUP_DEBUG - printk(KERN_INFO "Initialized PDC Console for debugging.\n"); -#endif -} - - -/* - * Used for emergencies. Currently only used if an HPMC occurs. If an - * HPMC occurs, it is possible that the current console may not be - * properly initialised after the PDC IO reset. This routine unregisters - * all of the current consoles, reinitializes the pdc console and - * registers it. - */ - -void pdc_console_restart(void) -{ - struct console *console; - - if (pdc_console_initialized) - return; - /* If we've already seen the output, don't bother to print it again */ - if (console_drivers != NULL) - pdc_cons.flags &= ~CON_PRINTBUFFER; - - while ((console = console_drivers) != NULL) - unregister_console(console_drivers); - - /* force registering the pdc console */ - pdc_console_init_force(); + return 0; } + +EARLYCON_DECLARE(pdc, pdc_earlycon_setup); diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index 7c37e09c92da6fe0e6e5cbd3ced4604d55a1024d..c4f8374c7018d644f3e1e06f11f50922f490466c 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -146,10 +146,6 @@ void flush_thread(void) */ } -void release_thread(struct task_struct *dead_task) -{ -} - /* * Idle thread support * @@ -288,7 +284,7 @@ __get_wchan(struct task_struct *p) static inline unsigned long brk_rnd(void) { - return (get_random_int() & BRK_RND_MASK) << PAGE_SHIFT; + return (get_random_u32() & BRK_RND_MASK) << PAGE_SHIFT; } unsigned long arch_randomize_brk(struct mm_struct *mm) diff --git a/arch/parisc/kernel/setup.c b/arch/parisc/kernel/setup.c index f005ddedb50e469653bcec1d5eb10c062a414311..375f38d6e1a4de1b3d73d330facb9c484ba84427 100644 --- a/arch/parisc/kernel/setup.c +++ b/arch/parisc/kernel/setup.c @@ -70,6 +70,10 @@ void __init setup_cmdline(char **cmdline_p) strlcat(p, "tty0", COMMAND_LINE_SIZE); } + /* default to use early console */ + if (!strstr(p, "earlycon")) + strlcat(p, " earlycon=pdc", COMMAND_LINE_SIZE); + #ifdef CONFIG_BLK_DEV_INITRD if (boot_args[2] != 0) /* did palo pass us a ramdisk? */ { @@ -139,8 +143,6 @@ void __init setup_arch(char **cmdline_p) if (__pa((unsigned long) &_end) >= KERNEL_INITIAL_SIZE) panic("KERNEL_INITIAL_ORDER too small!"); - pdc_console_init(); - #ifdef CONFIG_64BIT if(parisc_narrow_firmware) { printk(KERN_INFO "Kernel is using PDC in 32-bit mode.\n"); diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index 2b34294517a15a72b17dc290b985366eb8aa1c35..848b0702005d6178bceeed876a5ee3ce24b8d295 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -239,14 +239,14 @@ static unsigned long mmap_rnd(void) unsigned long rnd = 0; if (current->flags & PF_RANDOMIZE) - rnd = get_random_int() & MMAP_RND_MASK; + rnd = get_random_u32() & MMAP_RND_MASK; return rnd << PAGE_SHIFT; } unsigned long arch_mmap_rnd(void) { - return (get_random_int() & MMAP_RND_MASK) << PAGE_SHIFT; + return (get_random_u32() & MMAP_RND_MASK) << PAGE_SHIFT; } static unsigned long mmap_legacy_base(void) diff --git a/arch/parisc/kernel/traps.c b/arch/parisc/kernel/traps.c index b78f1b9d45c18b1e382e6c68961db149fa03c5f0..f9696fbf646c473cffbe26f316b1bcd289054431 100644 --- a/arch/parisc/kernel/traps.c +++ b/arch/parisc/kernel/traps.c @@ -239,13 +239,6 @@ void die_if_kernel(char *str, struct pt_regs *regs, long err) /* unlock the pdc lock if necessary */ pdc_emergency_unlock(); - /* maybe the kernel hasn't booted very far yet and hasn't been able - * to initialize the serial or STI console. In that case we should - * re-enable the pdc console, so that the user will be able to - * identify the problem. */ - if (!console_drivers) - pdc_console_restart(); - if (err) printk(KERN_CRIT "%s (pid %d): %s (code %ld)\n", current->comm, task_pid_nr(current), str, err); @@ -429,10 +422,6 @@ void parisc_terminate(char *msg, struct pt_regs *regs, int code, unsigned long o /* unlock the pdc lock if necessary */ pdc_emergency_unlock(); - /* restart pdc console if necessary */ - if (!console_drivers) - pdc_console_restart(); - /* Not all paths will gutter the processor... */ switch(code){ @@ -482,9 +471,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs) unsigned long fault_space = 0; int si_code; - if (code == 1) - pdc_console_restart(); /* switch back to pdc if HPMC */ - else if (!irqs_disabled_flags(regs->gr[0])) + if (!irqs_disabled_flags(regs->gr[0])) local_irq_enable(); /* Security check: diff --git a/arch/parisc/kernel/unaligned.c b/arch/parisc/kernel/unaligned.c index bac581b5ecfc5f85e3cb9a3c63efaa020690756b..e8a4d77cff53a77ee20d0e0319fa360d208cdd6d 100644 --- a/arch/parisc/kernel/unaligned.c +++ b/arch/parisc/kernel/unaligned.c @@ -93,7 +93,7 @@ #define R1(i) (((i)>>21)&0x1f) #define R2(i) (((i)>>16)&0x1f) #define R3(i) ((i)&0x1f) -#define FR3(i) ((((i)<<1)&0x1f)|(((i)>>6)&1)) +#define FR3(i) ((((i)&0x1f)<<1)|(((i)>>6)&1)) #define IM(i,n) (((i)>>1&((1<<(n-1))-1))|((i)&1?((0-1L)<<(n-1)):0)) #define IM5_2(i) IM((i)>>16,5) #define IM5_3(i) IM((i),5) diff --git a/arch/parisc/kernel/vdso.c b/arch/parisc/kernel/vdso.c index 63dc44c4c246bc755c3325faaa8f07b3188bb810..47e5960a2f961f44c70336a35be3c970ae7d2c3b 100644 --- a/arch/parisc/kernel/vdso.c +++ b/arch/parisc/kernel/vdso.c @@ -75,7 +75,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, map_base = mm->mmap_base; if (current->flags & PF_RANDOMIZE) - map_base -= (get_random_int() & 0x1f) * PAGE_SIZE; + map_base -= prandom_u32_max(0x20) * PAGE_SIZE; vdso_text_start = get_unmapped_area(NULL, map_base, vdso_text_len, 0, 0); diff --git a/arch/parisc/lib/iomap.c b/arch/parisc/lib/iomap.c index 86038505808573ce7bd45d306912b262177c2a9f..915c0c4da66385c2e5297fc6950bbf7fc56cff4b 100644 --- a/arch/parisc/lib/iomap.c +++ b/arch/parisc/lib/iomap.c @@ -48,15 +48,19 @@ struct iomap_ops { unsigned int (*read16be)(const void __iomem *); unsigned int (*read32)(const void __iomem *); unsigned int (*read32be)(const void __iomem *); +#ifdef CONFIG_64BIT u64 (*read64)(const void __iomem *); u64 (*read64be)(const void __iomem *); +#endif void (*write8)(u8, void __iomem *); void (*write16)(u16, void __iomem *); void (*write16be)(u16, void __iomem *); void (*write32)(u32, void __iomem *); void (*write32be)(u32, void __iomem *); +#ifdef CONFIG_64BIT void (*write64)(u64, void __iomem *); void (*write64be)(u64, void __iomem *); +#endif void (*read8r)(const void __iomem *, void *, unsigned long); void (*read16r)(const void __iomem *, void *, unsigned long); void (*read32r)(const void __iomem *, void *, unsigned long); @@ -175,6 +179,7 @@ static unsigned int iomem_read32be(const void __iomem *addr) return __raw_readl(addr); } +#ifdef CONFIG_64BIT static u64 iomem_read64(const void __iomem *addr) { return readq(addr); @@ -184,6 +189,7 @@ static u64 iomem_read64be(const void __iomem *addr) { return __raw_readq(addr); } +#endif static void iomem_write8(u8 datum, void __iomem *addr) { @@ -210,15 +216,17 @@ static void iomem_write32be(u32 datum, void __iomem *addr) __raw_writel(datum, addr); } +#ifdef CONFIG_64BIT static void iomem_write64(u64 datum, void __iomem *addr) { - writel(datum, addr); + writeq(datum, addr); } static void iomem_write64be(u64 datum, void __iomem *addr) { - __raw_writel(datum, addr); + __raw_writeq(datum, addr); } +#endif static void iomem_read8r(const void __iomem *addr, void *dst, unsigned long count) { @@ -274,15 +282,19 @@ static const struct iomap_ops iomem_ops = { .read16be = iomem_read16be, .read32 = iomem_read32, .read32be = iomem_read32be, +#ifdef CONFIG_64BIT .read64 = iomem_read64, .read64be = iomem_read64be, +#endif .write8 = iomem_write8, .write16 = iomem_write16, .write16be = iomem_write16be, .write32 = iomem_write32, .write32be = iomem_write32be, +#ifdef CONFIG_64BIT .write64 = iomem_write64, .write64be = iomem_write64be, +#endif .read8r = iomem_read8r, .read16r = iomem_read16r, .read32r = iomem_read32r, @@ -332,6 +344,7 @@ unsigned int ioread32be(const void __iomem *addr) return *((u32 *)addr); } +#ifdef CONFIG_64BIT u64 ioread64(const void __iomem *addr) { if (unlikely(INDIRECT_ADDR(addr))) @@ -345,26 +358,7 @@ u64 ioread64be(const void __iomem *addr) return iomap_ops[ADDR_TO_REGION(addr)]->read64be(addr); return *((u64 *)addr); } - -u64 ioread64_lo_hi(const void __iomem *addr) -{ - u32 low, high; - - low = ioread32(addr); - high = ioread32(addr + sizeof(u32)); - - return low + ((u64)high << 32); -} - -u64 ioread64_hi_lo(const void __iomem *addr) -{ - u32 low, high; - - high = ioread32(addr + sizeof(u32)); - low = ioread32(addr); - - return low + ((u64)high << 32); -} +#endif void iowrite8(u8 datum, void __iomem *addr) { @@ -411,6 +405,7 @@ void iowrite32be(u32 datum, void __iomem *addr) } } +#ifdef CONFIG_64BIT void iowrite64(u64 datum, void __iomem *addr) { if (unlikely(INDIRECT_ADDR(addr))) { @@ -428,18 +423,7 @@ void iowrite64be(u64 datum, void __iomem *addr) *((u64 *)addr) = datum; } } - -void iowrite64_lo_hi(u64 val, void __iomem *addr) -{ - iowrite32(val, addr); - iowrite32(val >> 32, addr + sizeof(u32)); -} - -void iowrite64_hi_lo(u64 val, void __iomem *addr) -{ - iowrite32(val >> 32, addr + sizeof(u32)); - iowrite32(val, addr); -} +#endif /* Repeating interfaces */ @@ -544,19 +528,19 @@ EXPORT_SYMBOL(ioread16); EXPORT_SYMBOL(ioread16be); EXPORT_SYMBOL(ioread32); EXPORT_SYMBOL(ioread32be); +#ifdef CONFIG_64BIT EXPORT_SYMBOL(ioread64); EXPORT_SYMBOL(ioread64be); -EXPORT_SYMBOL(ioread64_lo_hi); -EXPORT_SYMBOL(ioread64_hi_lo); +#endif EXPORT_SYMBOL(iowrite8); EXPORT_SYMBOL(iowrite16); EXPORT_SYMBOL(iowrite16be); EXPORT_SYMBOL(iowrite32); EXPORT_SYMBOL(iowrite32be); +#ifdef CONFIG_64BIT EXPORT_SYMBOL(iowrite64); EXPORT_SYMBOL(iowrite64be); -EXPORT_SYMBOL(iowrite64_lo_hi); -EXPORT_SYMBOL(iowrite64_hi_lo); +#endif EXPORT_SYMBOL(ioread8_rep); EXPORT_SYMBOL(ioread16_rep); EXPORT_SYMBOL(ioread32_rep); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 4c466acdc70d4cbe2d94271f9051c4e73868c863..699df27b0e2fc2f413c481e3e44ac22e2f134ce2 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -135,8 +135,9 @@ config PPC select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64 select ARCH_HAS_SET_MEMORY select ARCH_HAS_STRICT_KERNEL_RWX if (PPC_BOOK3S || PPC_8xx || 40x) && !HIBERNATION - select ARCH_HAS_STRICT_KERNEL_RWX if FSL_BOOKE && !HIBERNATION && !RANDOMIZE_BASE + select ARCH_HAS_STRICT_KERNEL_RWX if PPC_85xx && !HIBERNATION && !RANDOMIZE_BASE select ARCH_HAS_STRICT_MODULE_RWX if ARCH_HAS_STRICT_KERNEL_RWX + select ARCH_HAS_SYSCALL_WRAPPER if !SPU_BASE && !COMPAT select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE select ARCH_HAS_UBSAN_SANITIZE_ALL @@ -194,7 +195,7 @@ config PPC select HAVE_ARCH_KASAN if PPC_RADIX_MMU select HAVE_ARCH_KASAN if PPC_BOOK3E_64 select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN - select HAVE_ARCH_KFENCE if PPC_BOOK3S_32 || PPC_8xx || 40x + select HAVE_ARCH_KFENCE if ARCH_SUPPORTS_DEBUG_PAGEALLOC select HAVE_ARCH_RANDOMIZE_KSTACK_OFFSET select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS @@ -211,7 +212,7 @@ config PPC select HAVE_DYNAMIC_FTRACE_WITH_ARGS if MPROFILE_KERNEL || PPC32 select HAVE_DYNAMIC_FTRACE_WITH_REGS if MPROFILE_KERNEL || PPC32 select HAVE_EBPF_JIT - select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU) + select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_DESCRIPTORS if PPC64_ELF_ABI_V1 @@ -290,7 +291,7 @@ config PPC_LONG_DOUBLE_128 config PPC_BARRIER_NOSPEC bool default y - depends on PPC_BOOK3S_64 || PPC_FSL_BOOK3E + depends on PPC_BOOK3S_64 || PPC_E500 config EARLY_PRINTK bool @@ -548,7 +549,7 @@ config PPC64_SUPPORTS_MEMORY_FAILURE config KEXEC bool "kexec system call" - depends on (PPC_BOOK3S || FSL_BOOKE || (44x && !SMP)) || PPC_BOOK3E + depends on PPC_BOOK3S || PPC_E500 || (44x && !SMP) select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your @@ -583,7 +584,7 @@ config ARCH_HAS_KEXEC_PURGATORY config RELOCATABLE bool "Build a relocatable kernel" - depends on PPC64 || (FLATMEM && (44x || FSL_BOOKE)) + depends on PPC64 || (FLATMEM && (44x || PPC_85xx)) select NONSTATIC_KERNEL help This builds a kernel image that is capable of running at the @@ -606,7 +607,7 @@ config RELOCATABLE config RANDOMIZE_BASE bool "Randomize the address of the kernel image" - depends on (FSL_BOOKE && FLATMEM && PPC32) + depends on PPC_85xx && FLATMEM depends on RELOCATABLE help Randomizes the virtual address at which the kernel image is @@ -625,8 +626,8 @@ config RELOCATABLE_TEST config CRASH_DUMP bool "Build a dump capture kernel" - depends on PPC64 || PPC_BOOK3S_32 || FSL_BOOKE || (44x && !SMP) - select RELOCATABLE if PPC64 || 44x || FSL_BOOKE + depends on PPC64 || PPC_BOOK3S_32 || PPC_85xx || (44x && !SMP) + select RELOCATABLE if PPC64 || 44x || PPC_85xx help Build a kernel suitable for use as a dump capture kernel. The same kernel binary can be used as production kernel and dump @@ -815,7 +816,7 @@ config DATA_SHIFT_BOOL depends on ADVANCED_OPTIONS depends on STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE depends on PPC_BOOK3S_32 || (PPC_8xx && !PIN_TLB_DATA && !STRICT_KERNEL_RWX) || \ - FSL_BOOKE + PPC_85xx help This option allows you to set the kernel data alignment. When RAM is mapped by blocks, the alignment needs to fit the size and @@ -828,13 +829,13 @@ config DATA_SHIFT default 24 if STRICT_KERNEL_RWX && PPC64 range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx - range 20 24 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_FSL_BOOKE + range 20 24 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_85xx default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32 default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 default 23 if STRICT_KERNEL_RWX && PPC_8xx default 23 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx && PIN_TLB_DATA default 19 if (DEBUG_PAGEALLOC || KFENCE) && PPC_8xx - default 24 if STRICT_KERNEL_RWX && FSL_BOOKE + default 24 if STRICT_KERNEL_RWX && PPC_85xx default PPC_PAGE_SHIFT help On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO. @@ -845,7 +846,7 @@ config DATA_SHIFT in that case. If PIN_TLB is selected, it must be aligned to 8M as 8M pages will be pinned. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" range 8 9 if PPC64 && PPC_64K_PAGES default "9" if PPC64 && PPC_64K_PAGES @@ -1150,7 +1151,7 @@ config LOWMEM_SIZE config LOWMEM_CAM_NUM_BOOL bool "Set number of CAMs to use to map low memory" - depends on ADVANCED_OPTIONS && FSL_BOOKE + depends on ADVANCED_OPTIONS && PPC_85xx help This option allows you to set the maximum number of CAM slots that will be used to map low memory. There are a limited number of slots @@ -1161,7 +1162,7 @@ config LOWMEM_CAM_NUM_BOOL Say N here unless you know what you are doing. config LOWMEM_CAM_NUM - depends on FSL_BOOKE + depends on PPC_85xx int "Number of CAMs to use to map low memory" if LOWMEM_CAM_NUM_BOOL default 3 if !STRICT_KERNEL_RWX default 9 if DATA_SHIFT >= 24 @@ -1170,7 +1171,7 @@ config LOWMEM_CAM_NUM config DYNAMIC_MEMSTART bool "Enable page aligned dynamic load address for kernel" - depends on ADVANCED_OPTIONS && FLATMEM && (FSL_BOOKE || 44x) + depends on ADVANCED_OPTIONS && FLATMEM && (PPC_85xx || 44x) select NONSTATIC_KERNEL help This option enables the kernel to be loaded at any page aligned @@ -1219,7 +1220,7 @@ config KERNEL_START config PHYSICAL_START_BOOL bool "Set physical address where the kernel is loaded" - depends on ADVANCED_OPTIONS && FLATMEM && FSL_BOOKE + depends on ADVANCED_OPTIONS && FLATMEM && PPC_85xx help This gives the physical address where the kernel is loaded. @@ -1232,7 +1233,7 @@ config PHYSICAL_START config PHYSICAL_ALIGN hex - default "0x04000000" if FSL_BOOKE + default "0x04000000" if PPC_85xx help This value puts the alignment restrictions on physical address where kernel is loaded and run from. Kernel is compiled for an diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index ae727d4218b90a5f9e8f246b3a8973eda87ad40e..6aaf8dc60610d1ce9be9be4680672c8a86096781 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -283,6 +283,12 @@ config PPC_EARLY_DEBUG_MEMCONS This console provides input and output buffers stored within the kernel BSS and should be safe to select on any system. A debugger can then be used to read kernel output or send input to the console. + +config PPC_EARLY_DEBUG_16550 + bool "Serial 16550" + depends on PPC_UDBG_16550 + help + Select this to enable early debugging via Serial 16550 console endchoice config PPC_MEMCONS_OUTPUT_SIZE @@ -354,6 +360,15 @@ config PPC_EARLY_DEBUG_CPM_ADDR platform probing is done, all platforms selected must share the same address. +config PPC_EARLY_DEBUG_16550_PHYSADDR + hex "Early debug Serial 16550 physical address" + depends on PPC_EARLY_DEBUG_16550 + +config PPC_EARLY_DEBUG_16550_STRIDE + int "Early debug Serial 16550 stride" + depends on PPC_EARLY_DEBUG_16550 + default 1 + config FAIL_IOMMU bool "Fault-injection capability for IOMMU" depends on FAULT_INJECTION diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index 02742facf89511d6e6a4f0a1312157b85c8ddff8..dc4cbf0a5ca953bbf0498c08e54166aa10e41124 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -149,11 +149,12 @@ CFLAGS-$(CONFIG_PPC32) += $(call cc-option,-mno-readonly-in-sdata) ifdef CONFIG_PPC_BOOK3S_64 ifdef CONFIG_CPU_LITTLE_ENDIAN CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=power8 -CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power9,-mtune=power8) else -CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power7,$(call cc-option,-mtune=power5)) -CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mcpu=power5,-mcpu=power4) +CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=power4 endif +CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power10, \ + $(call cc-option,-mtune=power9, \ + $(call cc-option,-mtune=power8))) else ifdef CONFIG_PPC_BOOK3E_64 CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=powerpc64 endif @@ -191,9 +192,14 @@ ifdef CONFIG_476FPE_ERR46 -T $(srctree)/arch/powerpc/platforms/44x/ppc476_modules.lds endif -# No AltiVec or VSX instructions when building kernel +# No prefix or pcrel +KBUILD_CFLAGS += $(call cc-option,-mno-prefixed) +KBUILD_CFLAGS += $(call cc-option,-mno-pcrel) + +# No AltiVec or VSX or MMA instructions when building kernel KBUILD_CFLAGS += $(call cc-option,-mno-altivec) KBUILD_CFLAGS += $(call cc-option,-mno-vsx) +KBUILD_CFLAGS += $(call cc-option,-mno-mma) # No SPE instruction when building kernel # (We use all available options to help semi-broken compilers) @@ -210,7 +216,7 @@ KBUILD_CFLAGS += $(call cc-option,-mno-string) cpu-as-$(CONFIG_40x) += -Wa,-m405 cpu-as-$(CONFIG_44x) += -Wa,-m440 cpu-as-$(CONFIG_ALTIVEC) += $(call as-option,-Wa$(comma)-maltivec) -cpu-as-$(CONFIG_E500) += -Wa,-me500 +cpu-as-$(CONFIG_PPC_E500) += -Wa,-me500 # When using '-many -mpower4' gas will first try and find a matching power4 # mnemonic and failing that it will allow any valid mnemonic that GAS knows @@ -226,18 +232,6 @@ KBUILD_CFLAGS += $(cpu-as-y) KBUILD_AFLAGS += $(aflags-y) KBUILD_CFLAGS += $(cflags-y) -head-$(CONFIG_PPC64) := arch/powerpc/kernel/head_64.o -head-$(CONFIG_PPC_BOOK3S_32) := arch/powerpc/kernel/head_book3s_32.o -head-$(CONFIG_PPC_8xx) := arch/powerpc/kernel/head_8xx.o -head-$(CONFIG_40x) := arch/powerpc/kernel/head_40x.o -head-$(CONFIG_44x) := arch/powerpc/kernel/head_44x.o -head-$(CONFIG_FSL_BOOKE) := arch/powerpc/kernel/head_fsl_booke.o - -head-$(CONFIG_PPC64) += arch/powerpc/kernel/entry_64.o -head-$(CONFIG_PPC_FPU) += arch/powerpc/kernel/fpu.o -head-$(CONFIG_ALTIVEC) += arch/powerpc/kernel/vector.o -head-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += arch/powerpc/kernel/prom_init.o - # Default to zImage, override when needed all: zImage diff --git a/arch/powerpc/boot/44x.h b/arch/powerpc/boot/44x.h index 02563443788afd09e187b6c8f9bdb8badc40bd98..9b15e59522d6f0519a643a1b45e07b0345a96f78 100644 --- a/arch/powerpc/boot/44x.h +++ b/arch/powerpc/boot/44x.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * PowerPC 44x related functions * * Copyright 2007 David Gibson, IBM Corporation. - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #ifndef _PPC_BOOT_44X_H_ #define _PPC_BOOT_44X_H_ diff --git a/arch/powerpc/boot/4xx.h b/arch/powerpc/boot/4xx.h index 7dc5d45361bccf76251c66af3226ac374cfe966c..77f15d124c81cff73fbd688ba5f8c7093a6ed5cd 100644 --- a/arch/powerpc/boot/4xx.h +++ b/arch/powerpc/boot/4xx.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * PowerPC 4xx related functions * * Copyright 2007 IBM Corporation. * Josh Boyer - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #ifndef _POWERPC_BOOT_4XX_H_ #define _POWERPC_BOOT_4XX_H_ diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index a9cd2ea4a8617dd86d74a1d32b2715ab89386a5e..d32d95aea5d6f4c3e14d79f0e36923c332776078 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -34,6 +34,7 @@ endif BOOTCFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -O2 -msoft-float -mno-altivec -mno-vsx \ + $(call cc-option,-mno-spe) $(call cc-option,-mspe=no) \ -pipe -fomit-frame-pointer -fno-builtin -fPIC -nostdinc \ $(LINUXINCLUDE) diff --git a/arch/powerpc/boot/dts/fsl/e500v1_power_isa.dtsi b/arch/powerpc/boot/dts/fsl/e500v1_power_isa.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..7e2a90cde72e52c2ffdfa7375300c14219c2a006 --- /dev/null +++ b/arch/powerpc/boot/dts/fsl/e500v1_power_isa.dtsi @@ -0,0 +1,51 @@ +/* + * e500v1 Power ISA Device Tree Source (include) + * + * Copyright 2012 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/ { + cpus { + power-isa-version = "2.03"; + power-isa-b; // Base + power-isa-e; // Embedded + power-isa-atb; // Alternate Time Base + power-isa-cs; // Cache Specification + power-isa-e.le; // Embedded.Little-Endian + power-isa-e.pm; // Embedded.Performance Monitor + power-isa-ecl; // Embedded Cache Locking + power-isa-mmc; // Memory Coherence + power-isa-sp; // Signal Processing Engine + power-isa-sp.fs; // SPE.Embedded Float Scalar Single + power-isa-sp.fv; // SPE.Embedded Float Vector + mmu-type = "power-embedded"; + }; +}; diff --git a/arch/powerpc/boot/dts/fsl/mpc8540ads.dts b/arch/powerpc/boot/dts/fsl/mpc8540ads.dts index 18a885130538a483d4ede5ac51549cebc0d63e12..e03ae130162baa407e03a0cda01311842301903d 100644 --- a/arch/powerpc/boot/dts/fsl/mpc8540ads.dts +++ b/arch/powerpc/boot/dts/fsl/mpc8540ads.dts @@ -7,7 +7,7 @@ /dts-v1/; -/include/ "e500v2_power_isa.dtsi" +/include/ "e500v1_power_isa.dtsi" / { model = "MPC8540ADS"; diff --git a/arch/powerpc/boot/dts/fsl/mpc8541cds.dts b/arch/powerpc/boot/dts/fsl/mpc8541cds.dts index ac381e7b1c60ece2b7f1a54ca7d34f5d26651488..a2a6c5cf852e971f7ce6928f572dd93b4c5699cb 100644 --- a/arch/powerpc/boot/dts/fsl/mpc8541cds.dts +++ b/arch/powerpc/boot/dts/fsl/mpc8541cds.dts @@ -7,7 +7,7 @@ /dts-v1/; -/include/ "e500v2_power_isa.dtsi" +/include/ "e500v1_power_isa.dtsi" / { model = "MPC8541CDS"; diff --git a/arch/powerpc/boot/dts/fsl/mpc8555cds.dts b/arch/powerpc/boot/dts/fsl/mpc8555cds.dts index 9f58db2a7e66165f5abad4ff5aac3ccbaf16f57f..901b6ff06dfbb4b7e4a8b3c81bc6ddb338879283 100644 --- a/arch/powerpc/boot/dts/fsl/mpc8555cds.dts +++ b/arch/powerpc/boot/dts/fsl/mpc8555cds.dts @@ -7,7 +7,7 @@ /dts-v1/; -/include/ "e500v2_power_isa.dtsi" +/include/ "e500v1_power_isa.dtsi" / { model = "MPC8555CDS"; diff --git a/arch/powerpc/boot/dts/fsl/mpc8560ads.dts b/arch/powerpc/boot/dts/fsl/mpc8560ads.dts index a24722ccaebf1cef5a6eb62bc2fe15a43ac9f6a3..c2f9aea78b29ff3b8e513ab60498292a2d1927bc 100644 --- a/arch/powerpc/boot/dts/fsl/mpc8560ads.dts +++ b/arch/powerpc/boot/dts/fsl/mpc8560ads.dts @@ -7,7 +7,7 @@ /dts-v1/; -/include/ "e500v2_power_isa.dtsi" +/include/ "e500v1_power_isa.dtsi" / { model = "MPC8560ADS"; diff --git a/arch/powerpc/boot/dts/ksi8560.dts b/arch/powerpc/boot/dts/ksi8560.dts index fe6c17c8812a62545fd279cba29906c67090733a..37a7eb576d02ee0670e36ebbcf133df6956abb10 100644 --- a/arch/powerpc/boot/dts/ksi8560.dts +++ b/arch/powerpc/boot/dts/ksi8560.dts @@ -14,6 +14,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "KSI8560"; compatible = "emerson,KSI8560"; diff --git a/arch/powerpc/boot/dts/mgcoge.dts b/arch/powerpc/boot/dts/mgcoge.dts index 7de068991bde6155dc2cd1670ef31858d485cd92..9cefed20723454a81a9b9219b96905f5b1c6815f 100644 --- a/arch/powerpc/boot/dts/mgcoge.dts +++ b/arch/powerpc/boot/dts/mgcoge.dts @@ -225,13 +225,6 @@ interrupts = <2 8>; interrupt-parent = <&PIC>; cs-gpios = < &cpm2_pio_d 19 0>; - #address-cells = <1>; - #size-cells = <0>; - ds3106@1 { - compatible = "gen,spidev"; - reg = <0>; - spi-max-frequency = <8000000>; - }; }; }; diff --git a/arch/powerpc/boot/dts/stx_gp3_8560.dts b/arch/powerpc/boot/dts/stx_gp3_8560.dts index d1ab698eef36d887636b0268c6b9723594ba78fb..e73f7e75b0b4341b434aed716f27e8e7fb8511f5 100644 --- a/arch/powerpc/boot/dts/stx_gp3_8560.dts +++ b/arch/powerpc/boot/dts/stx_gp3_8560.dts @@ -7,6 +7,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "stx,gp3"; compatible = "stx,gp3-8560", "stx,gp3"; diff --git a/arch/powerpc/boot/dts/stxssa8555.dts b/arch/powerpc/boot/dts/stxssa8555.dts index 5dca2a91c41f6092d734c1dfdfbed4f4afe93ac5..96add25c904ba6ae22ff8ccbe2a5ed2f37057d4d 100644 --- a/arch/powerpc/boot/dts/stxssa8555.dts +++ b/arch/powerpc/boot/dts/stxssa8555.dts @@ -9,6 +9,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "stx,gp3"; compatible = "stx,gp3-8560", "stx,gp3"; diff --git a/arch/powerpc/boot/dts/tqm8540.dts b/arch/powerpc/boot/dts/tqm8540.dts index 9c1eb9779108c1edee8122d47e76b8fe2d5dc634..eb4d8fd3f7aa3c0d09f6368d75562fee34153114 100644 --- a/arch/powerpc/boot/dts/tqm8540.dts +++ b/arch/powerpc/boot/dts/tqm8540.dts @@ -7,6 +7,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "tqc,tqm8540"; compatible = "tqc,tqm8540"; diff --git a/arch/powerpc/boot/dts/tqm8541.dts b/arch/powerpc/boot/dts/tqm8541.dts index 44595cf675d01fb556aabb49d5243ac203bf9484..fe5d3d873ec9e9adff66e0c0ad60e57779fbded9 100644 --- a/arch/powerpc/boot/dts/tqm8541.dts +++ b/arch/powerpc/boot/dts/tqm8541.dts @@ -7,6 +7,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "tqc,tqm8541"; compatible = "tqc,tqm8541"; diff --git a/arch/powerpc/boot/dts/tqm8555.dts b/arch/powerpc/boot/dts/tqm8555.dts index 54f3e82907d699b34bfdd321e3210e93a11a318c..4be05b7d225d3969664a10f1814014573d75a5d9 100644 --- a/arch/powerpc/boot/dts/tqm8555.dts +++ b/arch/powerpc/boot/dts/tqm8555.dts @@ -7,6 +7,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "tqc,tqm8555"; compatible = "tqc,tqm8555"; diff --git a/arch/powerpc/boot/dts/tqm8560.dts b/arch/powerpc/boot/dts/tqm8560.dts index 7415cb69f60d5579d14c8125b0e5b085031e0be0..8ea48502420bfb77a4c448a76f7e2b80f7808425 100644 --- a/arch/powerpc/boot/dts/tqm8560.dts +++ b/arch/powerpc/boot/dts/tqm8560.dts @@ -8,6 +8,8 @@ /dts-v1/; +/include/ "fsl/e500v1_power_isa.dtsi" + / { model = "tqc,tqm8560"; compatible = "tqc,tqm8560"; diff --git a/arch/powerpc/boot/dts/turris1x.dts b/arch/powerpc/boot/dts/turris1x.dts index 12e08271e61f08ea5f8f11b81d945b80787faf64..045af668e92847844c2431ef6532da430289adcf 100644 --- a/arch/powerpc/boot/dts/turris1x.dts +++ b/arch/powerpc/boot/dts/turris1x.dts @@ -147,7 +147,7 @@ port@0 { reg = <0>; - label = "cpu1"; + label = "cpu"; ethernet = <&enet1>; phy-mode = "rgmii-id"; @@ -184,7 +184,7 @@ port@6 { reg = <6>; - label = "cpu0"; + label = "cpu"; ethernet = <&enet0>; phy-mode = "rgmii-id"; @@ -263,21 +263,21 @@ }; partition@20000 { - /* 1.7 MB for Rescue Linux Kernel Image */ + /* 1.7 MB for Linux Kernel Image */ reg = <0x00020000 0x001a0000>; - label = "rescue-kernel"; + label = "kernel"; }; partition@1c0000 { /* 1.5 MB for Rescue JFFS2 Root File System */ reg = <0x001c0000 0x00180000>; - label = "rescue-rootfs"; + label = "rescue"; }; partition@340000 { - /* 11 MB for TAR.XZ Backup with content of NAND Root File System */ + /* 11 MB for TAR.XZ Archive with Factory content of NAND Root File System */ reg = <0x00340000 0x00b00000>; - label = "backup-rootfs"; + label = "factory"; }; partition@e40000 { diff --git a/arch/powerpc/boot/dummy.c b/arch/powerpc/boot/dummy.c deleted file mode 100644 index 31dbf45bf99c5996ca0ee62cba186d912b08e666..0000000000000000000000000000000000000000 --- a/arch/powerpc/boot/dummy.c +++ /dev/null @@ -1,4 +0,0 @@ -int main(void) -{ - return 0; -} diff --git a/arch/powerpc/boot/opal-calls.S b/arch/powerpc/boot/opal-calls.S index ad0e15d930c4e759fcb87e1db2887071a143d6c4..1f2f330a459e86453187e2d41541dcbb50b67a73 100644 --- a/arch/powerpc/boot/opal-calls.S +++ b/arch/powerpc/boot/opal-calls.S @@ -16,7 +16,7 @@ opal_kentry: li r5, 0 li r6, 0 li r7, 0 - ld r11,opal@got(r2) + LOAD_REG_ADDR(r11, opal) ld r8,0(r11) ld r9,8(r11) bctr @@ -35,7 +35,7 @@ opal_call: mr r13,r2 /* Set opal return address */ - ld r11,opal_return@got(r2) + LOAD_REG_ADDR(r11, opal_return) mtlr r11 mfmsr r12 @@ -45,7 +45,7 @@ opal_call: mtspr SPRN_HSRR1,r12 /* load the opal call entry point and base */ - ld r11,opal@got(r2) + LOAD_REG_ADDR(r11, opal) ld r12,8(r11) ld r2,0(r11) mtspr SPRN_HSRR0,r12 diff --git a/arch/powerpc/boot/ops.h b/arch/powerpc/boot/ops.h index 8334bc3cbe499a166edcfaa74e59d91f69bf5051..a40c2162a4e9aa66afe95d8a31717b73dcfb44b6 100644 --- a/arch/powerpc/boot/ops.h +++ b/arch/powerpc/boot/ops.h @@ -1,12 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Global definition of all the bootwrapper operations. * * Author: Mark A. Greer * - * 2006 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2006 (c) MontaVista Software, Inc. */ #ifndef _PPC_BOOT_OPS_H_ #define _PPC_BOOT_OPS_H_ diff --git a/arch/powerpc/boot/ppc_asm.h b/arch/powerpc/boot/ppc_asm.h index 192b97523b0503d59a53583659fe92fba3e67b1f..a66cfd76fa4d245946247aac7e89792badfa8466 100644 --- a/arch/powerpc/boot/ppc_asm.h +++ b/arch/powerpc/boot/ppc_asm.h @@ -84,4 +84,14 @@ #define MFTBU(dest) mfspr dest, SPRN_TBRU #endif +#ifdef CONFIG_PPC64_BOOT_WRAPPER +#define LOAD_REG_ADDR(reg,name) \ + addis reg,r2,name@toc@ha; \ + addi reg,reg,name@toc@l +#else +#define LOAD_REG_ADDR(reg,name) \ + lis reg,name@ha; \ + addi reg,reg,name@l +#endif + #endif /* _PPC64_PPC_ASM_H */ diff --git a/arch/powerpc/boot/serial.c b/arch/powerpc/boot/serial.c index 54d2522be48542098b64b7933e14e900ad74edce..c6d32a8c361219f47e389696c2b7677af4c9e292 100644 --- a/arch/powerpc/boot/serial.c +++ b/arch/powerpc/boot/serial.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Generic serial console support * @@ -6,10 +7,7 @@ * Code in serial_edit_cmdline() copied from * and was written by Matt Porter . * - * 2001,2006 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2001,2006 (c) MontaVista Software, Inc. */ #include #include diff --git a/arch/powerpc/boot/simple_alloc.c b/arch/powerpc/boot/simple_alloc.c index 65ec135d015798490560c3a5bf419cc08e42aa9e..267d6524caac47d5897fec42fa1873d0740bc21a 100644 --- a/arch/powerpc/boot/simple_alloc.c +++ b/arch/powerpc/boot/simple_alloc.c @@ -1,12 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Implement primitive realloc(3) functionality. * * Author: Mark A. Greer * - * 2006 (c) MontaVista, Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. + * 2006 (c) MontaVista, Software, Inc. */ #include diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index 55978f32fa7751373b24868e93603d57ad916d00..5bdd4dd20bbb18e5d360b7ad12b14e0c12494e3b 100755 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -433,7 +433,7 @@ fi # Extract kernel version information, some platforms want to include # it in the image header version=`${CROSS}strings "$kernel" | grep '^Linux version [-0-9.]' | \ - cut -d' ' -f3` + head -n1 | cut -d' ' -f3` if [ -n "$version" ]; then uboot_version="-n Linux-$version" fi diff --git a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig index cbcae2a927e99e9ba9f0302b9f551791b4375824..4e3373381ab6e4cd708b5aed41efa46bcd17d6d4 100644 --- a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig +++ b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig @@ -77,6 +77,5 @@ CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_CRC_T10DIF=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/85xx/ge_imp3a_defconfig b/arch/powerpc/configs/85xx/ge_imp3a_defconfig index f29c166998af5948a7c5fb6843b2cbd6b40c0fa2..ea719898b581cd796bf3c5d5621a717222ac396f 100644 --- a/arch/powerpc/configs/85xx/ge_imp3a_defconfig +++ b/arch/powerpc/configs/85xx/ge_imp3a_defconfig @@ -30,7 +30,7 @@ CONFIG_PREEMPT=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m CONFIG_MATH_EMULATION=y -CONFIG_FORCE_MAX_ZONEORDER=17 +CONFIG_ARCH_FORCE_MAX_ORDER=17 CONFIG_PCI=y CONFIG_PCIEPORTBUS=y CONFIG_PCI_MSI=y @@ -74,7 +74,6 @@ CONFIG_MTD_PHYSMAP_OF=y CONFIG_MTD_RAW_NAND=y CONFIG_MTD_NAND_FSL_ELBC=y CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=131072 diff --git a/arch/powerpc/configs/85xx/ppa8548_defconfig b/arch/powerpc/configs/85xx/ppa8548_defconfig index 190978a5b7d5a5d75afba8be8daf926033556aa3..4bd5f993d26ae56c1f387019dee5a4a81594e7f9 100644 --- a/arch/powerpc/configs/85xx/ppa8548_defconfig +++ b/arch/powerpc/configs/85xx/ppa8548_defconfig @@ -7,9 +7,7 @@ CONFIG_RAPIDIO=y CONFIG_FSL_RIO=y CONFIG_RAPIDIO_DMA_ENGINE=y CONFIG_RAPIDIO_ENUM_BASIC=y -CONFIG_RAPIDIO_TSI57X=y CONFIG_RAPIDIO_CPS_XX=y -CONFIG_RAPIDIO_TSI568=y CONFIG_RAPIDIO_CPS_GEN2=y CONFIG_ADVANCED_OPTIONS=y CONFIG_LOWMEM_SIZE_BOOL=y diff --git a/arch/powerpc/configs/cell_defconfig b/arch/powerpc/configs/cell_defconfig index 7fd9e596ea331f1324cfd2947d2e65f1358a5e7a..06391cc2af3afc2b1b7840b824da2365dfb553fb 100644 --- a/arch/powerpc/configs/cell_defconfig +++ b/arch/powerpc/configs/cell_defconfig @@ -195,7 +195,6 @@ CONFIG_NLS_ISO8859_9=m CONFIG_NLS_ISO8859_13=m CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config index f14c6dbd7346cd2da36c067b3f0b1760b913bf95..ab8a8c4530d9259e30b19935c5caf0cbcde4ed9a 100644 --- a/arch/powerpc/configs/fsl-emb-nonhw.config +++ b/arch/powerpc/configs/fsl-emb-nonhw.config @@ -41,7 +41,7 @@ CONFIG_FIXED_PHY=y CONFIG_FONT_8x16=y CONFIG_FONT_8x8=y CONFIG_FONTS=y -CONFIG_FORCE_MAX_ZONEORDER=13 +CONFIG_ARCH_FORCE_MAX_ORDER=13 CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_FRAME_WARN=1024 CONFIG_FTL=y diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 9d6212a8b1951a95a0290788369457e2d32d5c5f..71d9d112c0b635f976476c2169c3f585467ea23f 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -119,7 +119,6 @@ CONFIG_INPUT_EVDEV=y # CONFIG_SERIO_I8042 is not set # CONFIG_SERIO_SERPORT is not set # CONFIG_HW_RANDOM is not set -CONFIG_RAW_DRIVER=y CONFIG_I2C_CHARDEV=y CONFIG_AGP=m CONFIG_AGP_UNINORTH=m diff --git a/arch/powerpc/configs/microwatt_defconfig b/arch/powerpc/configs/microwatt_defconfig index eff933ebbb9ecd62dccde2084f1afa4c3346517d..ea2dbd778aadf6be0ee264426af165a87e73b12e 100644 --- a/arch/powerpc/configs/microwatt_defconfig +++ b/arch/powerpc/configs/microwatt_defconfig @@ -75,7 +75,12 @@ CONFIG_SPI_BITBANG=y CONFIG_SPI_SPIDEV=y # CONFIG_HWMON is not set # CONFIG_USB_SUPPORT is not set +CONFIG_MMC=y +# CONFIG_PWRSEQ_EMMC is not set +# CONFIG_PWRSEQ_SIMPLE is not set +CONFIG_MMC_LITEX=y # CONFIG_VIRTIO_MENU is not set +CONFIG_COMMON_CLK=y # CONFIG_IOMMU_SUPPORT is not set # CONFIG_NVMEM is not set CONFIG_EXT4_FS=y diff --git a/arch/powerpc/configs/mpc512x_defconfig b/arch/powerpc/configs/mpc512x_defconfig index e75d3f3060c9e4b9cf11e818efa2564bfd41f312..10fe061c5e6d13b1f6cce7c5515c2ac878c1e654 100644 --- a/arch/powerpc/configs/mpc512x_defconfig +++ b/arch/powerpc/configs/mpc512x_defconfig @@ -114,5 +114,4 @@ CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_CRYPTO_HW is not set diff --git a/arch/powerpc/configs/mpc885_ads_defconfig b/arch/powerpc/configs/mpc885_ads_defconfig index 700115d85d6fb0d7d4c5aa6d7da09530d1d78e16..56b876e418e9154acc19fcb74b3656e299654209 100644 --- a/arch/powerpc/configs/mpc885_ads_defconfig +++ b/arch/powerpc/configs/mpc885_ads_defconfig @@ -78,4 +78,4 @@ CONFIG_DEBUG_VM_PGTABLE=y CONFIG_DETECT_HUNG_TASK=y CONFIG_BDI_SWITCH=y CONFIG_PPC_EARLY_DEBUG=y -CONFIG_PPC_PTDUMP=y +CONFIG_GENERIC_PTDUMP=y diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig index e00a703581c3854865f23b98ac745a631820b8d0..96aa5355911f1fbf7ecd62339f5efa9d51d079b5 100644 --- a/arch/powerpc/configs/pasemi_defconfig +++ b/arch/powerpc/configs/pasemi_defconfig @@ -92,7 +92,6 @@ CONFIG_LEGACY_PTY_COUNT=4 CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_HW_RANDOM=y -CONFIG_RAW_DRIVER=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_PASEMI=y CONFIG_SENSORS_LM85=y diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig index 13885ec563d159cdf744842f3eefca451f50acb3..019163c2571e67bd05f834a443533826635aeb8f 100644 --- a/arch/powerpc/configs/pmac32_defconfig +++ b/arch/powerpc/configs/pmac32_defconfig @@ -284,7 +284,6 @@ CONFIG_BOOTX_TEXT=y CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 49f49c2639350b7cd8f9514a50fc551b463d2bbf..c92652575064d18d4aa3b415089dd256e3987550 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -17,7 +17,6 @@ CONFIG_LOG_CPU_MAX_BUF_SHIFT=13 CONFIG_NUMA_BALANCING=y CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y CONFIG_CGROUP_SCHED=y CONFIG_CGROUP_FREEZER=y CONFIG_CPUSETS=y @@ -50,6 +49,7 @@ CONFIG_CPU_IDLE=y CONFIG_HZ_100=y CONFIG_BINFMT_MISC=m CONFIG_PPC_TRANSACTIONAL_MEM=y +CONFIG_PPC_UV=y CONFIG_HOTPLUG_CPU=y CONFIG_KEXEC=y CONFIG_KEXEC_FILE=y @@ -65,6 +65,8 @@ CONFIG_DEFERRED_STRUCT_PAGE_INIT=y CONFIG_SCHED_SMT=y CONFIG_PM=y CONFIG_HOTPLUG_PCI=y +CONFIG_ZONE_DEVICE=y +CONFIG_DEVICE_PRIVATE=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -252,7 +254,6 @@ CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_GENERIC=y # CONFIG_VIRTIO_MENU is not set CONFIG_LIBNVDIMM=y -# CONFIG_ND_BLK is not set CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y @@ -325,13 +326,11 @@ CONFIG_CRYPTO_MD5_PPC=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index c8b0e80d613bf2c97bf5b6d8ccadcea1d4cf377b..d6949a6c5b2b5e6b171e41f85a1c93208f145675 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -1,7 +1,9 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y +# CONFIG_CONTEXT_TRACKING_USER_FORCE is not set CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y +CONFIG_VIRT_CPU_ACCOUNTING_GEN=y CONFIG_TASKSTATS=y CONFIG_TASK_DELAY_ACCT=y CONFIG_IKCONFIG=y @@ -213,7 +215,6 @@ CONFIG_HVC_RTAS=y CONFIG_HVCS=m CONFIG_VIRTIO_CONSOLE=m CONFIG_IBM_BSR=m -CONFIG_RAW_DRIVER=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_AMD8111=y CONFIG_I2C_PASEMI=y @@ -342,13 +343,11 @@ CONFIG_CRYPTO_MD5_PPC=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig index 5cf49a515f8e6cb941f9389983769da3226bae03..f97a2d31bbf70d394384bf6be83a2fbe49355676 100644 --- a/arch/powerpc/configs/ppc64e_defconfig +++ b/arch/powerpc/configs/ppc64e_defconfig @@ -118,7 +118,6 @@ CONFIG_INPUT_MISC=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y # CONFIG_HW_RANDOM is not set -CONFIG_RAW_DRIVER=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_AMD8111=y CONFIG_FB=y @@ -234,13 +233,11 @@ CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index 91967824272ef125b66445f8f9201de22a372790..d23deb94b36e75a47f649a8ff9d3802872e5cb67 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -243,8 +243,6 @@ CONFIG_ATM_LANE=m CONFIG_ATM_BR2684=m CONFIG_BRIDGE=m CONFIG_VLAN_8021Q=m -CONFIG_DECNET=m -CONFIG_DECNET_ROUTER=y CONFIG_ATALK=m CONFIG_DEV_APPLETALK=m CONFIG_IPDDP=m @@ -323,7 +321,6 @@ CONFIG_PNP=y CONFIG_ISAPNP=y CONFIG_MAC_FLOPPY=m CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m CONFIG_BLK_DEV_NBD=m CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=16384 @@ -592,7 +589,6 @@ CONFIG_GAMEPORT_EMU10K1=m CONFIG_GAMEPORT_FM801=m # CONFIG_LEGACY_PTYS is not set CONFIG_SERIAL_NONSTANDARD=y -CONFIG_ROCKETPORT=m CONFIG_SYNCLINK_GT=m CONFIG_NOZOMI=m CONFIG_N_HDLC=m @@ -1109,13 +1105,9 @@ CONFIG_CRYPTO_XTS=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD128=m CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_RMD256=m -CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA1=y CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m @@ -1123,7 +1115,6 @@ CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SEED=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig index 2d9ac233da685603afba12ad139ae6168bf14c5d..0a1b42c4f26a40335495d76d8f370f0d8728f7c2 100644 --- a/arch/powerpc/configs/ps3_defconfig +++ b/arch/powerpc/configs/ps3_defconfig @@ -165,6 +165,5 @@ CONFIG_RCU_CPU_STALL_TIMEOUT=60 # CONFIG_FTRACE is not set CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_LZO=m CONFIG_PRINTK_TIME=y diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig index b571d084c148b84d4fd5d05df1ca6467fd3bed37..7497e17ea657f414ddf93e2001266ab28f365e07 100644 --- a/arch/powerpc/configs/pseries_defconfig +++ b/arch/powerpc/configs/pseries_defconfig @@ -3,8 +3,10 @@ CONFIG_NR_CPUS=2048 CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_AUDIT=y +# CONFIG_CONTEXT_TRACKING_USER_FORCE is not set CONFIG_NO_HZ=y CONFIG_HIGH_RES_TIMERS=y +CONFIG_VIRT_CPU_ACCOUNTING_GEN=y CONFIG_TASKSTATS=y CONFIG_TASK_DELAY_ACCT=y CONFIG_TASK_XACCT=y @@ -16,7 +18,6 @@ CONFIG_LOG_CPU_MAX_BUF_SHIFT=13 CONFIG_NUMA_BALANCING=y CONFIG_CGROUPS=y CONFIG_MEMCG=y -CONFIG_MEMCG_SWAP=y CONFIG_CGROUP_SCHED=y CONFIG_CGROUP_FREEZER=y CONFIG_CPUSETS=y @@ -40,6 +41,7 @@ CONFIG_PPC_SPLPAR=y CONFIG_DTL=y CONFIG_PPC_SMLPAR=y CONFIG_IBMEBUS=y +CONFIG_LIBNVDIMM=m CONFIG_PAPR_SCM=m CONFIG_PPC_SVM=y # CONFIG_PPC_PMAC is not set @@ -187,7 +189,6 @@ CONFIG_HVC_RTAS=y CONFIG_HVCS=m CONFIG_VIRTIO_CONSOLE=m CONFIG_IBM_BSR=m -CONFIG_RAW_DRIVER=y CONFIG_I2C_CHARDEV=y CONFIG_FB=y CONFIG_FIRMWARE_EDID=y @@ -302,13 +303,11 @@ CONFIG_CRYPTO_MD5_PPC=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SALSA20=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig index f491875700e8a317b7469c395a47534719d98aa3..e0964210f2599b4b9b1a2d8a6cb94dda506621de 100644 --- a/arch/powerpc/configs/skiroot_defconfig +++ b/arch/powerpc/configs/skiroot_defconfig @@ -133,7 +133,6 @@ CONFIG_ACENIC_OMIT_TIGON_I=y # CONFIG_NET_VENDOR_AQUANTIA is not set # CONFIG_NET_VENDOR_ARC is not set # CONFIG_NET_VENDOR_ATHEROS is not set -# CONFIG_NET_VENDOR_AURORA is not set CONFIG_TIGON3=m CONFIG_BNX2X=m # CONFIG_NET_VENDOR_BROCADE is not set @@ -274,7 +273,6 @@ CONFIG_NLS_UTF8=y CONFIG_ENCRYPTED_KEYS=y CONFIG_SECURITY=y CONFIG_HARDENED_USERCOPY=y -CONFIG_HARDENED_USERCOPY_PAGESPAN=y CONFIG_FORTIFY_SOURCE=y CONFIG_SECURITY_LOCKDOWN_LSM=y CONFIG_SECURITY_LOCKDOWN_LSM_EARLY=y diff --git a/arch/powerpc/configs/storcenter_defconfig b/arch/powerpc/configs/storcenter_defconfig index 47dcfaddc1acafbf46a333e6e77e06efe850e346..7a978d39699119cfe8b0527b10f69342267e30a4 100644 --- a/arch/powerpc/configs/storcenter_defconfig +++ b/arch/powerpc/configs/storcenter_defconfig @@ -76,4 +76,3 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_CRC_T10DIF=y -# CONFIG_ENABLE_MUST_CHECK is not set diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..c1b964447401634e9c55ba7f34695a0f63750789 --- /dev/null +++ b/arch/powerpc/crypto/Kconfig @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" + +config CRYPTO_CRC32C_VPMSUM + tristate "CRC32c" + depends on PPC64 && ALTIVEC + select CRYPTO_HASH + select CRC32 + help + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: powerpc64 using + - AltiVec extensions + + Enable on POWER8 and newer processors for improved performance. + +config CRYPTO_CRCT10DIF_VPMSUM + tristate "CRC32T10DIF" + depends on PPC64 && ALTIVEC && CRC_T10DIF + select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: powerpc64 using + - AltiVec extensions + + Enable on POWER8 and newer processors for improved performance. + +config CRYPTO_VPMSUM_TESTER + tristate "CRC32c and CRC32T10DIF hardware acceleration tester" + depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM + help + Stress test for CRC32c and CRCT10DIF algorithms implemented with + powerpc64 AltiVec extensions (POWER8 vpmsum instructions). + Unless you are testing these algorithms, you don't need this. + +config CRYPTO_MD5_PPC + tristate "Digests: MD5" + depends on PPC + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) + + Architecture: powerpc + +config CRYPTO_SHA1_PPC + tristate "Hash functions: SHA-1" + depends on PPC + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: powerpc + +config CRYPTO_SHA1_PPC_SPE + tristate "Hash functions: SHA-1 (SPE)" + depends on PPC && SPE + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: powerpc using + - SPE (Signal Processing Engine) extensions + +config CRYPTO_SHA256_PPC_SPE + tristate "Hash functions: SHA-224 and SHA-256 (SPE)" + depends on PPC && SPE + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: powerpc using + - SPE (Signal Processing Engine) extensions + +config CRYPTO_AES_PPC_SPE + tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" + depends on PPC && SPE + select CRYPTO_SKCIPHER + help + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes + + Architecture: powerpc using: + - SPE (Signal Processing Engine) extensions + + SPE is available for: + - Processor Type: Freescale 8500 + - CPU selection: e500 (8540) + + This module should only be used for low power (router) devices + without hardware AES acceleration (e.g. caam crypto). It reduces the + size of the AES tables from 16KB to 8KB + 256 bytes and mitigates + timining attacks. Nevertheless it might be not as secure as other + architecture specific assembler implementations that work on 1KB + tables or 256 bytes S-boxes. + +endmenu diff --git a/arch/powerpc/crypto/crc-vpmsum_test.c b/arch/powerpc/crypto/crc-vpmsum_test.c index c1c1ef9457fb494279873799c373b68b9485e09d..273c527868db2e6dd5f81a18d71373d6df616250 100644 --- a/arch/powerpc/crypto/crc-vpmsum_test.c +++ b/arch/powerpc/crypto/crc-vpmsum_test.c @@ -82,7 +82,7 @@ static int __init crc_test_init(void) if (len <= offset) continue; - prandom_bytes(data, len); + get_random_bytes(data, len); len -= offset; crypto_shash_update(crct10dif_shash, data+offset, len); diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index 81631e64dbeb7135463fbaf86fefee442dc3ed74..274bce76f5da238aae281739a675236af097d104 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -36,6 +36,7 @@ int64_t __opal_call(int64_t a0, int64_t a1, int64_t a2, int64_t a3, int64_t opcode, uint64_t msr); /* misc runtime */ +void enable_machine_check(void); extern u64 __bswapdi2(u64); extern s64 __lshrdi3(s64, int); extern s64 __ashldi3(s64, int); @@ -55,19 +56,6 @@ struct kvm_vcpu; void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); -/* Patch sites */ -extern s32 patch__call_flush_branch_caches1; -extern s32 patch__call_flush_branch_caches2; -extern s32 patch__call_flush_branch_caches3; -extern s32 patch__flush_count_cache_return; -extern s32 patch__flush_link_stack_return; -extern s32 patch__call_kvm_flush_link_stack; -extern s32 patch__call_kvm_flush_link_stack_p9; -extern s32 patch__memset_nocache, patch__memcpy_nocache; - -extern long flush_branch_caches; -extern long kvm_flush_link_stack; - #ifdef CONFIG_PPC_TRANSACTIONAL_MEM void kvmppc_save_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv); void kvmppc_restore_tm_hv(struct kvm_vcpu *vcpu, u64 msr, bool preserve_nv); diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index ef2d8b15eaabe3d876174588886d76851c6d6142..e80b2c0e9315a1fc2515d207c7cadcba5f425ca8 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -86,7 +86,7 @@ do { \ #ifdef CONFIG_PPC_BOOK3S_64 #define NOSPEC_BARRIER_SLOT nop -#elif defined(CONFIG_PPC_FSL_BOOK3E) +#elif defined(CONFIG_PPC_E500) #define NOSPEC_BARRIER_SLOT nop; nop #endif diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 40041ac713d9822bf248eae03567e5ad3847ce19..75823f39e04241d5fb41190aafd2bdd5ca30b17c 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -112,31 +112,11 @@ static inline bool pte_user(pte_t pte) /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) #define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE) -#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_NO_CACHE | _PAGE_GUARDED) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE | _PAGE_GUARDED) #define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) #define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) #define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) -/* - * Protection used for kernel text. We want the debuggers to be able to - * set breakpoints anywhere, so don't write protect the kernel text - * on platforms where such control is possible. - */ -#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\ - defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) -#define PAGE_KERNEL_TEXT PAGE_KERNEL_X -#else -#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX -#endif - -/* Make modules code happy. We don't set RO yet */ -#define PAGE_KERNEL_EXEC PAGE_KERNEL_X - -/* Advertise special mapping type for AGP */ -#define PAGE_AGP (PAGE_KERNEL_NC) -#define HAVE_PAGE_AGP - #define PTE_INDEX_SIZE PTE_SHIFT #define PMD_INDEX_SIZE 0 #define PUD_INDEX_SIZE 0 diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h index e1af0b394cebec3ae5e04a3ad2ff3586c2ab9695..dd2cff53a111e7715a16d38b869e1012018fd6fc 100644 --- a/arch/powerpc/include/asm/book3s/64/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h @@ -113,9 +113,11 @@ static inline void __pud_free(pud_t *pud) /* * Early pud pages allocated via memblock allocator - * can't be directly freed to slab + * can't be directly freed to slab. KFENCE pages have + * both reserved and slab flags set so need to be freed + * kmem_cache_free. */ - if (PageReserved(page)) + if (PageReserved(page) && !PageSlab(page)) free_reserved_page(page); else kmem_cache_free(PGT_CACHE(PUD_CACHE_INDEX), pud); diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h index 4e697bc2f4cde914db1ef09732764f8500b158a7..48f21820afe209ea3de1e5df7b00c1a363064da6 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-4k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-4k.h @@ -26,16 +26,6 @@ static inline int pud_huge(pud_t pud) return 0; } -static inline int pgd_huge(pgd_t pgd) -{ - /* - * leaf pte for huge page - */ - if (radix_enabled()) - return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); - return 0; -} -#define pgd_huge pgd_huge /* * With radix , we have hugepage ptes in the pud and pmd entries. We don't * need to setup hugepage directory for them. Our pte and page directory format diff --git a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h index 34d1018896b3e805dab2df5983eed34d6fb80145..2fce3498b000e29541c33138fc625dfe7b95d7ab 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable-64k.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable-64k.h @@ -30,15 +30,6 @@ static inline int pud_huge(pud_t pud) return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); } -static inline int pgd_huge(pgd_t pgd) -{ - /* - * leaf pte for huge page - */ - return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE)); -} -#define pgd_huge pgd_huge - /* * With 64k page size, we have hugepage ptes in the pgd and pmd entries. We don't * need to setup hugepage directory for them. Our pte and page directory format diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 392ff48f77df34b4e809138612e56eb4114b0666..c436d84226540f60d8e6947454d5eb290d0f4ac9 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -117,8 +117,7 @@ #define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_RW | _PAGE_DIRTY) #define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_READ) #define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED | _PAGE_READ | _PAGE_EXEC) -#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | \ - _PAGE_RW | _PAGE_EXEC) +#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | _PAGE_EXEC) /* * _PAGE_CHG_MASK masks of bits that are to be preserved across * pgprot changes @@ -151,33 +150,17 @@ #define PAGE_COPY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) #define PAGE_READONLY __pgprot(_PAGE_BASE | _PAGE_READ) #define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_EXEC) +/* Radix only, Hash uses PAGE_READONLY_X + execute-only pkey instead */ +#define PAGE_EXECONLY __pgprot(_PAGE_BASE | _PAGE_EXEC) /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) -#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_TOLERANT) -#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_NON_IDEMPOTENT) +#define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_TOLERANT) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NON_IDEMPOTENT) #define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) #define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) #define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) -/* - * Protection used for kernel text. We want the debuggers to be able to - * set breakpoints anywhere, so don't write protect the kernel text - * on platforms where such control is possible. - */ -#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \ - defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) -#define PAGE_KERNEL_TEXT PAGE_KERNEL_X -#else -#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX -#endif - -/* Make modules code happy. We don't set RO yet */ -#define PAGE_KERNEL_EXEC PAGE_KERNEL_X -#define PAGE_AGP (PAGE_KERNEL_NC) - #ifndef __ASSEMBLY__ /* * page table defines @@ -333,9 +316,6 @@ extern unsigned long pci_io_base; #define IOREMAP_END (KERN_IO_END - FIXADDR_SIZE) #define FIXADDR_SIZE SZ_32M -/* Advertise special mapping type for AGP */ -#define HAVE_PAGE_AGP - #ifndef __ASSEMBLY__ /* @@ -411,6 +391,9 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, * event of it not getting flushed for a long time the delay * shouldn't really matter because there's no real memory * pressure for swapout to react to. ] + * + * Note: this optimisation also exists in pte_needs_flush() and + * huge_pmd_needs_flush(). */ #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH #define ptep_clear_flush_young ptep_test_and_clear_young @@ -1123,7 +1106,7 @@ static inline void vmemmap_remove_mapping(unsigned long start, } #endif -#ifdef CONFIG_DEBUG_PAGEALLOC +#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) static inline void __kernel_map_pages(struct page *page, int numpages, int enable) { if (radix_enabled()) @@ -1457,12 +1440,5 @@ static inline bool pud_is_leaf(pud_t pud) return !!(pud_raw(pud) & cpu_to_be64(_PAGE_PTE)); } -#define p4d_is_leaf p4d_is_leaf -#define p4d_leaf p4d_is_leaf -static inline bool p4d_is_leaf(p4d_t p4d) -{ - return !!(p4d_raw(p4d) & cpu_to_be64(_PAGE_PTE)); -} - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h index 8b762f28219018f821cd1c01335d736c31c09885..fab8332fe1add8d6854d0a39025a50a568e57b16 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -112,13 +112,11 @@ static inline void hash__flush_tlb_kernel_range(unsigned long start, struct mmu_gather; extern void hash__tlb_flush(struct mmu_gather *tlb); -void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr); #ifdef CONFIG_PPC_64S_HASH_MMU /* Private function for use by PCI IO mapping code */ extern void __flush_hash_table_range(unsigned long start, unsigned long end); -extern void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr); +void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr); #else static inline void __flush_hash_table_range(unsigned long start, unsigned long end) { } #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 206f920fe5b9655f979d001ef4b893648babe2d0..67655cd6054563bbdcbd84396921a5f05ff419d4 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -163,6 +163,62 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, */ } +static inline bool __pte_flags_need_flush(unsigned long oldval, + unsigned long newval) +{ + unsigned long delta = oldval ^ newval; + + /* + * The return value of this function doesn't matter for hash, + * ptep_modify_prot_start() does a pte_update() which does or schedules + * any necessary hash table update and flush. + */ + if (!radix_enabled()) + return true; + + /* + * We do not expect kernel mappings or non-PTEs or not-present PTEs. + */ + VM_WARN_ON_ONCE(oldval & _PAGE_PRIVILEGED); + VM_WARN_ON_ONCE(newval & _PAGE_PRIVILEGED); + VM_WARN_ON_ONCE(!(oldval & _PAGE_PTE)); + VM_WARN_ON_ONCE(!(newval & _PAGE_PTE)); + VM_WARN_ON_ONCE(!(oldval & _PAGE_PRESENT)); + VM_WARN_ON_ONCE(!(newval & _PAGE_PRESENT)); + + /* + * Must flush on any change except READ, WRITE, EXEC, DIRTY, ACCESSED. + * + * In theory, some changed software bits could be tolerated, in + * practice those should rarely if ever matter. + */ + + if (delta & ~(_PAGE_RWX | _PAGE_DIRTY | _PAGE_ACCESSED)) + return true; + + /* + * If any of the above was present in old but cleared in new, flush. + * With the exception of _PAGE_ACCESSED, don't worry about flushing + * if that was cleared (see the comment in ptep_clear_flush_young()). + */ + if ((delta & ~_PAGE_ACCESSED) & oldval) + return true; + + return false; +} + +static inline bool pte_needs_flush(pte_t oldpte, pte_t newpte) +{ + return __pte_flags_need_flush(pte_val(oldpte), pte_val(newpte)); +} +#define pte_needs_flush pte_needs_flush + +static inline bool huge_pmd_needs_flush(pmd_t oldpmd, pmd_t newpmd) +{ + return __pte_flags_need_flush(pmd_val(oldpmd), pmd_val(newpmd)); +} +#define huge_pmd_needs_flush huge_pmd_needs_flush + extern bool tlbie_capable; extern bool tlbie_enabled; diff --git a/arch/powerpc/include/asm/book3s/pgtable.h b/arch/powerpc/include/asm/book3s/pgtable.h index e8269434ecbec54c2967021f1a52102b611850a3..d18b748ea3ae0fed84aa9b4fb4e82ebad0eb2764 100644 --- a/arch/powerpc/include/asm/book3s/pgtable.h +++ b/arch/powerpc/include/asm/book3s/pgtable.h @@ -25,7 +25,8 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot); #define __HAVE_PHYS_MEM_ACCESS_PROT -#if defined(CONFIG_PPC32) || defined(CONFIG_PPC_64S_HASH_MMU) +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); + /* * This gets called at the end of handling a page fault, when * the kernel has put a new PTE into the page table for the process. @@ -35,10 +36,14 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, * corresponding HPTE into the hash table ahead of time, instead of * waiting for the inevitable extra hash-table miss exception. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); -#else -static inline void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) {} -#endif +static inline void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) +{ + if (IS_ENABLED(CONFIG_PPC32) && !mmu_has_feature(MMU_FTR_HPTE_TABLE)) + return; + if (radix_enabled()) + return; + __update_mmu_cache(vma, address, ptep); +} #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/powerpc/include/asm/cpu_setup.h b/arch/powerpc/include/asm/cpu_setup.h new file mode 100644 index 0000000000000000000000000000000000000000..30e2fe38950241321093a869079e42a138bea445 --- /dev/null +++ b/arch/powerpc/include/asm/cpu_setup.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 IBM Corporation + */ + +#ifndef _ASM_POWERPC_CPU_SETUP_H +#define _ASM_POWERPC_CPU_SETUP_H +void __setup_cpu_power7(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power8(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power9(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_power10(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_power7(void); +void __restore_cpu_power8(void); +void __restore_cpu_power9(void); +void __restore_cpu_power10(void); + +void __setup_cpu_e500v1(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e500v2(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e500mc(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440ep(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440epx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440gx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440grx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440spe(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_440x5(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460ex(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460gt(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_460sx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_apm821xx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_603(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_604(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750cx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_750fx(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_7400(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_7410(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_745x(unsigned long offset, struct cpu_spec *spec); + +void __setup_cpu_ppc970(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_ppc970MP(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_pa6t(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_pa6t(void); +void __restore_cpu_ppc970(void); + +void __setup_cpu_e5500(unsigned long offset, struct cpu_spec *spec); +void __setup_cpu_e6500(unsigned long offset, struct cpu_spec *spec); +void __restore_cpu_e5500(void); +void __restore_cpu_e6500(void); +#endif /* _ASM_POWERPC_CPU_SETUP_H */ diff --git a/arch/powerpc/include/asm/cpu_setup_power.h b/arch/powerpc/include/asm/cpu_setup_power.h deleted file mode 100644 index 24be9131f8032d80b002f2bf0b055e54981717bf..0000000000000000000000000000000000000000 --- a/arch/powerpc/include/asm/cpu_setup_power.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2020 IBM Corporation - */ -void __setup_cpu_power7(unsigned long offset, struct cpu_spec *spec); -void __restore_cpu_power7(void); -void __setup_cpu_power8(unsigned long offset, struct cpu_spec *spec); -void __restore_cpu_power8(void); -void __setup_cpu_power9(unsigned long offset, struct cpu_spec *spec); -void __restore_cpu_power9(void); -void __setup_cpu_power10(unsigned long offset, struct cpu_spec *spec); -void __restore_cpu_power10(void); diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index ae8c3e13cfce4fa4b98141f7d75a1a845679d1c8..757dbded11dcfa3c9de381cb10ee49821091f5ba 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -463,7 +463,7 @@ static inline void cpu_feature_keys_init(void) { } #define CPU_FTRS_COMPATIBLE (CPU_FTR_PPCAS_ARCH_V2) #ifdef CONFIG_PPC64 -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 #define CPU_FTRS_POSSIBLE (CPU_FTRS_E6500 | CPU_FTRS_E5500) #else #ifdef CONFIG_CPU_LITTLE_ENDIAN @@ -510,7 +510,7 @@ enum { #elif defined(CONFIG_44x) CPU_FTRS_44X | CPU_FTRS_440x6 | #endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 CPU_FTRS_E500 | CPU_FTRS_E500_2 | #endif #ifdef CONFIG_PPC_E500MC @@ -521,7 +521,7 @@ enum { #endif /* __powerpc64__ */ #ifdef CONFIG_PPC64 -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 #define CPU_FTRS_ALWAYS (CPU_FTRS_E6500 & CPU_FTRS_E5500) #else @@ -584,7 +584,7 @@ enum { #elif defined(CONFIG_44x) CPU_FTRS_44X & CPU_FTRS_440x6 & #endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 CPU_FTRS_E500 & CPU_FTRS_E500_2 & #endif #ifdef CONFIG_PPC_E500MC diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index 6d2b279974929237dab43ac03eef633c8a1a8926..431ae234302227905f42d1c99745ef47ef356023 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -95,7 +95,7 @@ static notrace inline void account_stolen_time(void) struct lppaca *lp = local_paca->lppaca_ptr; if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) - accumulate_stolen_time(); + pseries_accumulate_stolen_time(); } #endif } diff --git a/arch/powerpc/include/asm/dtl.h b/arch/powerpc/include/asm/dtl.h index 1625888f27ef6b48f51560fa0b776f9e5927bd8c..4bcb9f9ac76491c249b271be333543727c849764 100644 --- a/arch/powerpc/include/asm/dtl.h +++ b/arch/powerpc/include/asm/dtl.h @@ -37,14 +37,6 @@ struct dtl_entry { extern struct kmem_cache *dtl_cache; extern rwlock_t dtl_access_lock; -/* - * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls - * reading from the dispatch trace log. If other code wants to consume - * DTL entries, it can set this pointer to a function that will get - * called once for each DTL entry that gets processed. - */ -extern void (*dtl_consumer)(struct dtl_entry *entry, u64 index); - extern void register_dtl_buffer(int cpu); extern void alloc_dtl_buffers(unsigned long *time_limit); extern long hcall_vphn(unsigned long cpu, u64 flags, __be32 *associativity); diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 398e0b5e485f66a8da5908ffae4841ff567e8b58..ed6db13a1d7c45405a565461b95205996392c28b 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -83,6 +83,8 @@ enum { FW_FEATURE_POWERNV_ALWAYS = 0, FW_FEATURE_PS3_POSSIBLE = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, FW_FEATURE_PS3_ALWAYS = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, + FW_FEATURE_NATIVE_POSSIBLE = 0, + FW_FEATURE_NATIVE_ALWAYS = 0, FW_FEATURE_POSSIBLE = #ifdef CONFIG_PPC_PSERIES FW_FEATURE_PSERIES_POSSIBLE | @@ -92,6 +94,9 @@ enum { #endif #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_POSSIBLE | +#endif +#ifdef CONFIG_PPC_HASH_MMU_NATIVE + FW_FEATURE_NATIVE_ALWAYS | #endif 0, FW_FEATURE_ALWAYS = @@ -103,6 +108,9 @@ enum { #endif #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_ALWAYS & +#endif +#ifdef CONFIG_PPC_HASH_MMU_NATIVE + FW_FEATURE_NATIVE_ALWAYS & #endif FW_FEATURE_POSSIBLE, diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 32ce0fb7548f83712fe4a6b7fcc7be5e06c51f67..ea71f7245a63e5b4ae522b7767a3b01845a86e36 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -7,8 +7,8 @@ #ifdef CONFIG_PPC_BOOK3S_64 #include -#elif defined(CONFIG_PPC_FSL_BOOK3E) -#include +#elif defined(CONFIG_PPC_E500) +#include #elif defined(CONFIG_PPC_8xx) #include #endif /* CONFIG_PPC_BOOK3S_64 */ diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 26ede09c521df2829656b1db183b29ccffc6ccf4..77fa88c2aed0dc0febfe52073e7cba464821a4e7 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -113,7 +113,14 @@ static inline void __hard_RI_enable(void) static inline notrace unsigned long irq_soft_mask_return(void) { - return READ_ONCE(local_paca->irq_soft_mask); + unsigned long flags; + + asm volatile( + "lbz %0,%1(13)" + : "=r" (flags) + : "i" (offsetof(struct paca_struct, irq_soft_mask))); + + return flags; } /* @@ -140,8 +147,12 @@ static inline notrace void irq_soft_mask_set(unsigned long mask) if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) WARN_ON(mask && !(mask & IRQS_DISABLED)); - WRITE_ONCE(local_paca->irq_soft_mask, mask); - barrier(); + asm volatile( + "stb %0,%1(13)" + : + : "r" (mask), + "i" (offsetof(struct paca_struct, irq_soft_mask)) + : "memory"); } static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask) @@ -282,7 +293,8 @@ static inline bool pmi_irq_pending(void) flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED); \ local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ if (!arch_irqs_disabled_flags(flags)) { \ - WRITE_ONCE(local_paca->saved_r1, current_stack_pointer);\ + asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \ + : "r" (current_stack_pointer)); \ trace_hardirqs_off(); \ } \ } while(0) @@ -459,6 +471,30 @@ static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned l } #endif /* CONFIG_PPC64 */ +static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr) +{ +#ifdef CONFIG_PPC64 + if (arch_irqs_disabled()) { + /* + * With soft-masking, MSR[EE] can change from 1 to 0 + * asynchronously when irqs are disabled, and we don't want to + * set MSR[EE] back to 1 here if that has happened. A race-free + * way to do this is ensure EE is already 0. Another way it + * could be done is with a RESTART_TABLE handler, but that's + * probably overkill here. + */ + msr &= ~MSR_EE; + mtmsr_isync(msr); + irq_soft_mask_set(IRQS_ALL_DISABLED); + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + } else +#endif + mtmsr_isync(msr); + + return msr; +} + + #define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h index 8069dbc4b8d185ea81c89eda2449b4dda0b55f6e..4745bb9998bd7fa902b8c9826132d17317cc44fc 100644 --- a/arch/powerpc/include/asm/interrupt.h +++ b/arch/powerpc/include/asm/interrupt.h @@ -74,6 +74,19 @@ #include #include +#ifdef CONFIG_PPC64 +/* + * WARN/BUG is handled with a program interrupt so minimise checks here to + * avoid recursion and maximise the chance of getting the first oops handled. + */ +#define INT_SOFT_MASK_BUG_ON(regs, cond) \ +do { \ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && \ + (user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM))) \ + BUG_ON(cond); \ +} while (0) +#endif + #ifdef CONFIG_PPC_BOOK3S_64 extern char __end_soft_masked[]; bool search_kernel_soft_mask_table(unsigned long addr); @@ -170,8 +183,7 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs) * context. */ if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) { - if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) - BUG_ON(!(regs->msr & MSR_EE)); + INT_SOFT_MASK_BUG_ON(regs, !(regs->msr & MSR_EE)); __hard_irq_enable(); } else { __hard_RI_enable(); @@ -194,19 +206,15 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs) * CT_WARN_ON comes here via program_check_exception, * so avoid recursion. */ - if (TRAP(regs) != INTERRUPT_PROGRAM) { - CT_WARN_ON(ct_state() != CONTEXT_KERNEL); - if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) - BUG_ON(is_implicit_soft_masked(regs)); - } - - /* Move this under a debugging check */ - if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && - arch_irq_disabled_regs(regs)) - BUG_ON(search_kernel_restart_table(regs->nip)); + if (TRAP(regs) != INTERRUPT_PROGRAM) + CT_WARN_ON(ct_state() != CONTEXT_KERNEL && + ct_state() != CONTEXT_IDLE); + INT_SOFT_MASK_BUG_ON(regs, is_implicit_soft_masked(regs)); + INT_SOFT_MASK_BUG_ON(regs, arch_irq_disabled_regs(regs) && + search_kernel_restart_table(regs->nip)); } - if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) - BUG_ON(!arch_irq_disabled_regs(regs) && !(regs->msr & MSR_EE)); + INT_SOFT_MASK_BUG_ON(regs, !arch_irq_disabled_regs(regs) && + !(regs->msr & MSR_EE)); #endif booke_restore_dbcr0(); @@ -281,7 +289,7 @@ static inline bool nmi_disables_ftrace(struct pt_regs *regs) if (TRAP(regs) == INTERRUPT_PERFMON) return false; } - if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { + if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) { if (TRAP(regs) == INTERRUPT_PERFMON) return false; } @@ -665,8 +673,7 @@ static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) local_irq_enable(); } -long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, - unsigned long r0, struct pt_regs *regs); +long system_call_exception(struct pt_regs *regs, unsigned long r0); notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv); notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs); notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs); diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index f8d122d16af4d59d31a0a8195931c4f40bbab4e3..a1ddba01e7d13b9fd654805d0831f5240428c98c 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -3,7 +3,7 @@ #define _ASM_POWERPC_KEXEC_H #ifdef __KERNEL__ -#if defined(CONFIG_FSL_BOOKE) || defined(CONFIG_44x) +#if defined(CONFIG_PPC_85xx) || defined(CONFIG_44x) /* * On FSL-BookE we setup a 1:1 mapping which covers the first 2GiB of memory diff --git a/arch/powerpc/include/asm/kfence.h b/arch/powerpc/include/asm/kfence.h index a9846b68c6b9eab427578c342cb8f87590492435..6fd2b4d486c5257a1343002703829a6ac5d31f2b 100644 --- a/arch/powerpc/include/asm/kfence.h +++ b/arch/powerpc/include/asm/kfence.h @@ -11,11 +11,25 @@ #include #include +#ifdef CONFIG_PPC64_ELF_ABI_V1 +#define ARCH_FUNC_PREFIX "." +#endif + static inline bool arch_kfence_init_pool(void) { return true; } +#ifdef CONFIG_PPC64 +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + struct page *page = virt_to_page(addr); + + __kernel_map_pages(page, 1, !protect); + + return true; +} +#else static inline bool kfence_protect_page(unsigned long addr, bool protect) { pte_t *kpte = virt_to_kpte(addr); @@ -29,5 +43,6 @@ static inline bool kfence_protect_page(unsigned long addr, bool protect) return true; } +#endif #endif /* __ASM_POWERPC_KFENCE_H */ diff --git a/arch/powerpc/include/asm/kgdb.h b/arch/powerpc/include/asm/kgdb.h index a9e098a3b881f29f0d18d6c96a6f4f4a31376173..715c18b7533463fa545766ab142ebbc216b026ea 100644 --- a/arch/powerpc/include/asm/kgdb.h +++ b/arch/powerpc/include/asm/kgdb.h @@ -52,7 +52,7 @@ static inline void arch_kgdb_breakpoint(void) /* On non-E500 family PPC32 we determine the size by picking the last * register we need, but on E500 we skip sections so we list what we * need to store, and add it up. */ -#ifndef CONFIG_E500 +#ifndef CONFIG_PPC_E500 #define MAXREG (PT_FPSCR+1) #else /* 32 GPRs (8 bytes), nip, msr, ccr, link, ctr, xer, acc (8 bytes), spefscr*/ diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index c2b003550dc9845e1645bc0e93de80622da9284a..caea15dcb91dd0d192100615315fd863f22b3b03 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -443,7 +443,7 @@ struct kvmppc_passthru_irqmap { }; #endif -# ifdef CONFIG_PPC_FSL_BOOK3E +# ifdef CONFIG_PPC_E500 #define KVMPPC_BOOKE_IAC_NUM 2 #define KVMPPC_BOOKE_DAC_NUM 2 # else diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 9f625af3b65b53227c5e1702d6ed9bcbebec6175..bfacf12784dde98be592ac7a4091a22acea92fe3 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -104,7 +104,6 @@ extern void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_map(struct kvm_vcpu *vcpu, u64 gvaddr, gpa_t gpaddr, unsigned int gtlb_idx); -extern void kvmppc_mmu_priv_switch(struct kvm_vcpu *vcpu, int usermode); extern void kvmppc_mmu_switch_pid(struct kvm_vcpu *vcpu, u32 pid); extern int kvmppc_mmu_dtlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); extern int kvmppc_mmu_itlb_index(struct kvm_vcpu *vcpu, gva_t eaddr); @@ -153,7 +152,6 @@ extern int kvmppc_core_check_requests(struct kvm_vcpu *vcpu); extern int kvmppc_booke_init(void); extern void kvmppc_booke_exit(void); -extern void kvmppc_core_destroy_mmu(struct kvm_vcpu *vcpu); extern int kvmppc_kvm_pv(struct kvm_vcpu *vcpu); extern void kvmppc_map_magic(struct kvm_vcpu *vcpu); @@ -162,8 +160,6 @@ extern void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info); extern long kvmppc_alloc_reset_hpt(struct kvm *kvm, int order); extern void kvmppc_free_hpt(struct kvm_hpt_info *info); extern void kvmppc_rmap_reset(struct kvm *kvm); -extern long kvmppc_prepare_vrma(struct kvm *kvm, - struct kvm_userspace_memory_region *mem); extern void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, unsigned long porder); extern int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu); diff --git a/arch/powerpc/include/asm/lppaca.h b/arch/powerpc/include/asm/lppaca.h index c390ec377baed0bb79504c3c4fa4faf9877aa98a..34d44cb17c874969c57ab8c3b03ec90ae27e9a6a 100644 --- a/arch/powerpc/include/asm/lppaca.h +++ b/arch/powerpc/include/asm/lppaca.h @@ -104,14 +104,18 @@ struct lppaca { volatile __be32 dispersion_count; /* dispatch changed physical cpu */ volatile __be64 cmo_faults; /* CMO page fault count */ volatile __be64 cmo_fault_time; /* CMO page fault time */ - u8 reserved10[104]; + u8 reserved10[64]; /* [S]PURR expropriated/donated */ + volatile __be64 enqueue_dispatch_tb; /* Total TB enqueue->dispatch */ + volatile __be64 ready_enqueue_tb; /* Total TB ready->enqueue */ + volatile __be64 wait_ready_tb; /* Total TB wait->ready */ + u8 reserved11[16]; /* cacheline 4-5 */ __be32 page_ins; /* CMO Hint - # page ins by OS */ - u8 reserved11[148]; + u8 reserved12[148]; volatile __be64 dtl_idx; /* Dispatch Trace Log head index */ - u8 reserved12[96]; + u8 reserved13[96]; } ____cacheline_aligned; #define lppaca_of(cpu) (*paca_ptrs[cpu]->lppaca_ptr) diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 8cb83600c43461c23d58ded17beeda37ac904284..378b8d5836a7b8fe249a19fe43ffa8edeb673e92 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -204,7 +204,6 @@ struct machdep_calls { extern void e500_idle(void); extern void power4_idle(void); extern void ppc6xx_idle(void); -extern void book3e_idle(void); /* * ppc_md contains a copy of the machine description structure for the diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 860d0290ca4def85a6527d9214f07c5859ffb4cc..94b981152667c8f9b5ed51620eeb835effc25bc9 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -120,6 +120,9 @@ */ #define MMU_FTR_1T_SEGMENT ASM_CONST(0x40000000) +// NX paste RMA reject in DSI +#define MMU_FTR_NX_DSI ASM_CONST(0x80000000) + /* MMU feature bit sets for various CPUs */ #define MMU_FTRS_DEFAULT_HPTE_ARCH_V2 (MMU_FTR_HPTE_TABLE | MMU_FTR_TLBIEL | MMU_FTR_16M_PAGE) #define MMU_FTRS_POWER MMU_FTRS_DEFAULT_HPTE_ARCH_V2 @@ -141,7 +144,7 @@ typedef pte_t *pgtable_t; -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 #include DECLARE_PER_CPU(int, next_tlbcam_idx); #endif @@ -162,7 +165,7 @@ enum { #elif defined(CONFIG_44x) MMU_FTR_TYPE_44x | #endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX | #endif #ifdef CONFIG_PPC_BOOK3S_32 @@ -181,7 +184,7 @@ enum { #endif #ifdef CONFIG_PPC_RADIX_MMU MMU_FTR_TYPE_RADIX | - MMU_FTR_GTSE | + MMU_FTR_GTSE | MMU_FTR_NX_DSI | #endif /* CONFIG_PPC_RADIX_MMU */ #endif #ifdef CONFIG_PPC_KUAP @@ -211,7 +214,7 @@ enum { #elif defined(CONFIG_44x) #define MMU_FTRS_ALWAYS MMU_FTR_TYPE_44x #endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 #define MMU_FTRS_ALWAYS MMU_FTR_TYPE_FSL_E #endif diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 3f25bd3e14eb8419847f750e5b44a6ac397f0671..c1ea270bb8486d881c7dcbfe4c7fce0d58ac6751 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -31,7 +31,6 @@ extern long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua, extern long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem); extern void mm_iommu_init(struct mm_struct *mm); -extern void mm_iommu_cleanup(struct mm_struct *mm); extern struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, unsigned long ua, unsigned long size); extern struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm, @@ -117,7 +116,6 @@ static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea) } #endif -extern void switch_cop(struct mm_struct *next); extern int use_cop(unsigned long acop, struct mm_struct *mm); extern void drop_cop(unsigned long acop, struct mm_struct *mm); diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 9091e4904a6b6a3174b61fcbfe13de3f60a4813d..0d40b33184ebe9a7c1f3372c3dbaaa78aa1b4e9a 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -130,10 +130,10 @@ void unmap_kernel_page(unsigned long va); #include #elif defined(CONFIG_44x) #include -#elif defined(CONFIG_FSL_BOOKE) && defined(CONFIG_PTE_64BIT) -#include -#elif defined(CONFIG_FSL_BOOKE) -#include +#elif defined(CONFIG_PPC_85xx) && defined(CONFIG_PTE_64BIT) +#include +#elif defined(CONFIG_PPC_85xx) +#include #elif defined(CONFIG_PPC_8xx) #include #endif diff --git a/arch/powerpc/include/asm/nohash/32/pte-fsl-booke.h b/arch/powerpc/include/asm/nohash/32/pte-85xx.h similarity index 94% rename from arch/powerpc/include/asm/nohash/32/pte-fsl-booke.h rename to arch/powerpc/include/asm/nohash/32/pte-85xx.h index 0fc1bd42bb3ee3c8c879565105477c479484240c..93fb8e11a3f12ae43ec50cec05c90ce29200ff69 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-fsl-booke.h +++ b/arch/powerpc/include/asm/nohash/32/pte-85xx.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_POWERPC_NOHASH_32_PTE_FSL_BOOKE_H -#define _ASM_POWERPC_NOHASH_32_PTE_FSL_BOOKE_H +#ifndef _ASM_POWERPC_NOHASH_32_PTE_85xx_H +#define _ASM_POWERPC_NOHASH_32_PTE_85xx_H #ifdef __KERNEL__ /* PTE bit definitions for Freescale BookE SW loaded TLB MMU based @@ -71,4 +71,4 @@ #define PAGE_READONLY_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_EXEC) #endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_NOHASH_32_PTE_FSL_BOOKE_H */ +#endif /* _ASM_POWERPC_NOHASH_32_PTE_FSL_85xx_H */ diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index 599921cc257e0ec4a6ea932356c081769aa9162d..879e9a6e5a870a49c1dbd5f8c37ac5086f359b22 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -70,7 +70,7 @@ /* * Include the PTE bits definitions */ -#include +#include #define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) diff --git a/arch/powerpc/include/asm/nohash/hugetlb-book3e.h b/arch/powerpc/include/asm/nohash/hugetlb-e500.h similarity index 84% rename from arch/powerpc/include/asm/nohash/hugetlb-book3e.h rename to arch/powerpc/include/asm/nohash/hugetlb-e500.h index ecd8694cb229bc53c949f28dff7df2269d656477..8f04ad20e0403eaa8cc7a6ae843fa7052a19f454 100644 --- a/arch/powerpc/include/asm/nohash/hugetlb-book3e.h +++ b/arch/powerpc/include/asm/nohash/hugetlb-e500.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_POWERPC_NOHASH_HUGETLB_BOOK3E_H -#define _ASM_POWERPC_NOHASH_HUGETLB_BOOK3E_H +#ifndef _ASM_POWERPC_NOHASH_HUGETLB_E500_H +#define _ASM_POWERPC_NOHASH_HUGETLB_E500_H static inline pte_t *hugepd_page(hugepd_t hpd) { @@ -30,7 +30,7 @@ void flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr); static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshift) { - /* We use the old format for PPC_FSL_BOOK3E */ + /* We use the old format for PPC_E500 */ *hpdp = __hugepd(((unsigned long)new & ~PD_HUGE) | pshift); } @@ -42,4 +42,4 @@ static inline int check_and_get_huge_psize(int shift) return shift_to_mmu_psize(shift); } -#endif /* _ASM_POWERPC_NOHASH_HUGETLB_BOOK3E_H */ +#endif /* _ASM_POWERPC_NOHASH_HUGETLB_E500_H */ diff --git a/arch/powerpc/include/asm/nohash/mmu-book3e.h b/arch/powerpc/include/asm/nohash/mmu-e500.h similarity index 100% rename from arch/powerpc/include/asm/nohash/mmu-book3e.h rename to arch/powerpc/include/asm/nohash/mmu-e500.h diff --git a/arch/powerpc/include/asm/nohash/mmu.h b/arch/powerpc/include/asm/nohash/mmu.h index edc793e5f08f900394b63f3056c619be84795bc0..e264be219fdb893753cc2f8f31d4df3686b2bb50 100644 --- a/arch/powerpc/include/asm/nohash/mmu.h +++ b/arch/powerpc/include/asm/nohash/mmu.h @@ -8,9 +8,9 @@ #elif defined(CONFIG_44x) /* 44x-style software loaded TLB */ #include -#elif defined(CONFIG_PPC_BOOK3E_MMU) +#elif defined(CONFIG_PPC_E500) /* Freescale Book-E software loaded TLB or Book-3e (ISA 2.06+) MMU */ -#include +#include #elif defined (CONFIG_PPC_8xx) /* Motorola/Freescale 8xx software loaded TLB */ #include diff --git a/arch/powerpc/include/asm/nohash/pgalloc.h b/arch/powerpc/include/asm/nohash/pgalloc.h index 29c43665a75367df7fec4b9397691473fc50e99c..4b62376318e11c2e1cedf7e1e676c8381a3606bf 100644 --- a/arch/powerpc/include/asm/nohash/pgalloc.h +++ b/arch/powerpc/include/asm/nohash/pgalloc.h @@ -15,7 +15,7 @@ static inline void tlb_flush_pgtable(struct mmu_gather *tlb, { } -#endif /* !CONFIG_PPC_BOOK3E */ +#endif /* !CONFIG_PPC_BOOK3E_64 */ static inline pgd_t *pgd_alloc(struct mm_struct *mm) { diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index b499da6c1a994b485e5149e50624032015e7c0f6..d9067dfc531ccdd3f739a7fb29407a38a355f5d9 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -11,31 +11,11 @@ /* Permission masks used for kernel mappings */ #define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_KERNEL_RW) #define PAGE_KERNEL_NC __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE) -#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | \ - _PAGE_NO_CACHE | _PAGE_GUARDED) +#define PAGE_KERNEL_NCG __pgprot(_PAGE_BASE_NC | _PAGE_KERNEL_RW | _PAGE_NO_CACHE | _PAGE_GUARDED) #define PAGE_KERNEL_X __pgprot(_PAGE_BASE | _PAGE_KERNEL_RWX) #define PAGE_KERNEL_RO __pgprot(_PAGE_BASE | _PAGE_KERNEL_RO) #define PAGE_KERNEL_ROX __pgprot(_PAGE_BASE | _PAGE_KERNEL_ROX) -/* - * Protection used for kernel text. We want the debuggers to be able to - * set breakpoints anywhere, so don't write protect the kernel text - * on platforms where such control is possible. - */ -#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\ - defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) -#define PAGE_KERNEL_TEXT PAGE_KERNEL_X -#else -#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX -#endif - -/* Make modules code happy. We don't set RO yet */ -#define PAGE_KERNEL_EXEC PAGE_KERNEL_X - -/* Advertise special mapping type for AGP */ -#define PAGE_AGP (PAGE_KERNEL_NC) -#define HAVE_PAGE_AGP - #ifndef __ASSEMBLY__ /* Generic accessors to PTE bits */ @@ -277,12 +257,6 @@ static inline int pud_huge(pud_t pud) return 0; } -static inline int pgd_huge(pgd_t pgd) -{ - return 0; -} -#define pgd_huge pgd_huge - #define is_hugepd(hpd) (hugepd_ok(hpd)) #endif @@ -292,7 +266,7 @@ static inline int pgd_huge(pgd_t pgd) * We use it to ensure coherency between the i-cache and d-cache * for the page which has just been mapped in. */ -#if defined(CONFIG_PPC_FSL_BOOK3E) && defined(CONFIG_HUGETLB_PAGE) +#if defined(CONFIG_PPC_E500) && defined(CONFIG_HUGETLB_PAGE) void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); #else static inline diff --git a/arch/powerpc/include/asm/nohash/pte-book3e.h b/arch/powerpc/include/asm/nohash/pte-e500.h similarity index 96% rename from arch/powerpc/include/asm/nohash/pte-book3e.h rename to arch/powerpc/include/asm/nohash/pte-e500.h index f798640422c2d6831de682e3b5a80a0b9c1ca389..0934e8965e4ed8ccb1c2cb6df13e24cde614b233 100644 --- a/arch/powerpc/include/asm/nohash/pte-book3e.h +++ b/arch/powerpc/include/asm/nohash/pte-e500.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_POWERPC_NOHASH_PTE_BOOK3E_H -#define _ASM_POWERPC_NOHASH_PTE_BOOK3E_H +#ifndef _ASM_POWERPC_NOHASH_PTE_E500_H +#define _ASM_POWERPC_NOHASH_PTE_E500_H #ifdef __KERNEL__ /* PTE bit definitions for processors compliant to the Book3E @@ -126,4 +126,4 @@ static inline pte_t pte_mkexec(pte_t pte) #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_NOHASH_PTE_BOOK3E_H */ +#endif /* _ASM_POWERPC_NOHASH_PTE_E500_H */ diff --git a/arch/powerpc/include/asm/nohash/tlbflush.h b/arch/powerpc/include/asm/nohash/tlbflush.h index 698935d4f72dddd59f1681f142a74e766820c22e..bdaf34ad41ea6a36d4d56d837901f867cbeded75 100644 --- a/arch/powerpc/include/asm/nohash/tlbflush.h +++ b/arch/powerpc/include/asm/nohash/tlbflush.h @@ -18,7 +18,7 @@ /* * TLB flushing for software loaded TLB chips * - * TODO: (CONFIG_FSL_BOOKE) determine if flush_tlb_range & + * TODO: (CONFIG_PPC_85xx) determine if flush_tlb_range & * flush_tlb_kernel_range are best implemented as tlbia vs * specific tlbie's */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index bfd3142cd0ba7a7435e5e4b98854acce7c5325c4..726125a534de80817fdd0e99742477ae704b0e69 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -324,16 +324,10 @@ extern int opal_flush_console(uint32_t vtermno); extern void hvc_opal_init_early(void); -extern int opal_notifier_register(struct notifier_block *nb); -extern int opal_notifier_unregister(struct notifier_block *nb); - extern int opal_message_notifier_register(enum opal_msg_type msg_type, struct notifier_block *nb); extern int opal_message_notifier_unregister(enum opal_msg_type msg_type, struct notifier_block *nb); -extern void opal_notifier_enable(void); -extern void opal_notifier_disable(void); -extern void opal_notifier_update_evt(uint64_t evt_mask, uint64_t evt_val); extern int opal_async_get_token_interruptible(void); extern int opal_async_release_token(int token); diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 4d7aaab8270232446fd6715d3fde0a3d1596294c..09f1790d0ae165c5475ffd2ec235776d98ddab9f 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -18,7 +18,7 @@ #include #include #include -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 #include #else #include @@ -127,7 +127,7 @@ struct paca_struct { #endif #endif /* CONFIG_PPC_BOOK3S_64 */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 u64 exgen[8] __aligned(0x40); /* Keep pgd in the same cacheline as the start of extlb */ pgd_t *pgd __aligned(0x40); /* Current PGD */ @@ -151,7 +151,7 @@ struct paca_struct { void *dbg_kstack; struct tlb_core_data tcd; -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ #ifdef CONFIG_PPC_64S_HASH_MMU unsigned char mm_ctx_low_slices_psize[BITS_PER_LONG / BITS_PER_BYTE]; @@ -168,7 +168,7 @@ struct paca_struct { #ifdef CONFIG_PPC64 u64 exit_save_r1; /* Syscall/interrupt R1 save */ #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 u16 trap_save; /* Used when bad stack is encountered */ #endif #ifdef CONFIG_PPC_BOOK3S_64 @@ -263,7 +263,6 @@ struct paca_struct { u64 l1d_flush_size; #endif #ifdef CONFIG_PPC_PSERIES - struct rtas_args *rtas_args_reentrant; u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */ #endif /* CONFIG_PPC_PSERIES */ diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h index e5f75c70eda8366c79f3f3fc821c3ed406f23408..edf1dd1b0ca99e8de7fb58db22a6743ec27995ff 100644 --- a/arch/powerpc/include/asm/page.h +++ b/arch/powerpc/include/asm/page.h @@ -31,7 +31,7 @@ extern unsigned int hpage_shift; #define HPAGE_SHIFT hpage_shift #elif defined(CONFIG_PPC_8xx) #define HPAGE_SHIFT 19 /* 512k pages */ -#elif defined(CONFIG_PPC_FSL_BOOK3E) +#elif defined(CONFIG_PPC_E500) #define HPAGE_SHIFT 22 /* 4M pages */ #endif #define HPAGE_SIZE ((1UL) << HPAGE_SHIFT) @@ -308,12 +308,6 @@ static inline bool pfn_valid(unsigned long pfn) #include #endif - -#ifndef CONFIG_HUGETLB_PAGE -#define is_hugepd(pdep) (0) -#define pgd_huge(pgd) (0) -#endif /* CONFIG_HUGETLB_PAGE */ - struct page; extern void clear_user_page(void *page, unsigned long vaddr, struct page *pg); extern void copy_user_page(void *to, void *from, unsigned long vaddr, diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index eb7df559ae7497c8bee9bb57ac9c9bc59302f3a0..f5ba1a3c41f8e508fd9d5e94224b24bbb4827bc0 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -21,6 +21,18 @@ static inline bool is_shared_processor(void) return static_branch_unlikely(&shared_processor); } +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING +extern struct static_key paravirt_steal_enabled; +extern struct static_key paravirt_steal_rq_enabled; + +u64 pseries_paravirt_steal_clock(int cpu); + +static inline u64 paravirt_steal_clock(int cpu) +{ + return pseries_paravirt_steal_clock(cpu); +} +#endif + /* If bit 0 is set, the cpu has been ceded, conferred, or preempted */ static inline u32 yield_count_of(int cpu) { diff --git a/arch/powerpc/include/asm/paravirt_api_clock.h b/arch/powerpc/include/asm/paravirt_api_clock.h new file mode 100644 index 0000000000000000000000000000000000000000..d25ca7ac57c714b8f2a3ad309a198e83f9327af9 --- /dev/null +++ b/arch/powerpc/include/asm/paravirt_api_clock.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include diff --git a/arch/powerpc/include/asm/pgtable-be-types.h b/arch/powerpc/include/asm/pgtable-be-types.h index b169bbf95fcbe2232438442ccd9e827de872ed61..82633200b500dababfc5616081e130b3493d4bba 100644 --- a/arch/powerpc/include/asm/pgtable-be-types.h +++ b/arch/powerpc/include/asm/pgtable-be-types.h @@ -101,6 +101,7 @@ static inline bool pmd_xchg(pmd_t *pmdp, pmd_t old, pmd_t new) return pmd_raw(old) == prev; } +#ifdef CONFIG_ARCH_HAS_HUGEPD typedef struct { __be64 pdbe; } hugepd_t; #define __hugepd(x) ((hugepd_t) { cpu_to_be64(x) }) @@ -108,5 +109,6 @@ static inline unsigned long hpd_val(hugepd_t x) { return be64_to_cpu(x.pdbe); } +#endif #endif /* _ASM_POWERPC_PGTABLE_BE_TYPES_H */ diff --git a/arch/powerpc/include/asm/pgtable-types.h b/arch/powerpc/include/asm/pgtable-types.h index efed0db7b1db66c88da3fd7036b2bd6026a6397e..082c85cc09b1442dd3e743524f425399f5bb1473 100644 --- a/arch/powerpc/include/asm/pgtable-types.h +++ b/arch/powerpc/include/asm/pgtable-types.h @@ -83,11 +83,13 @@ static inline bool pte_xchg(pte_t *ptep, pte_t old, pte_t new) } #endif +#ifdef CONFIG_ARCH_HAS_HUGEPD typedef struct { unsigned long pd; } hugepd_t; #define __hugepd(x) ((hugepd_t) { (x) }) static inline unsigned long hpd_val(hugepd_t x) { return x.pd; } +#endif #endif /* _ASM_POWERPC_PGTABLE_TYPES_H */ diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 33f4bf8d22b01a7e662367542fdc5302a7cd566b..283f40d05a4d7ee725a3d4bf4c77ee9482ca8feb 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -20,6 +20,25 @@ struct mm_struct; #include #endif /* !CONFIG_PPC_BOOK3S */ +/* + * Protection used for kernel text. We want the debuggers to be able to + * set breakpoints anywhere, so don't write protect the kernel text + * on platforms where such control is possible. + */ +#if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) || \ + defined(CONFIG_KPROBES) || defined(CONFIG_DYNAMIC_FTRACE) +#define PAGE_KERNEL_TEXT PAGE_KERNEL_X +#else +#define PAGE_KERNEL_TEXT PAGE_KERNEL_ROX +#endif + +/* Make modules code happy. We don't set RO yet */ +#define PAGE_KERNEL_EXEC PAGE_KERNEL_X + +/* Advertise special mapping type for AGP */ +#define PAGE_AGP (PAGE_KERNEL_NC) +#define HAVE_PAGE_AGP + #ifndef __ASSEMBLY__ #ifndef MAX_PTRS_PER_PGD diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index c6d724104ed1a08b24b0fa60a48305329afd5deb..21e33e46f4b8cf8c90fb2a2ca7d10956ee314cf2 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -330,6 +330,7 @@ #define __PPC_XSP(s) ((((s) & 0x1e) | (((s) >> 5) & 0x1)) << 21) #define __PPC_XTP(s) __PPC_XSP(s) #define __PPC_T_TLB(t) (((t) & 0x3) << 21) +#define __PPC_PL(p) (((p) & 0x3) << 16) #define __PPC_WC(w) (((w) & 0x3) << 21) #define __PPC_WS(w) (((w) & 0x1f) << 11) #define __PPC_SH(s) __PPC_WS(s) @@ -388,7 +389,8 @@ #define PPC_RAW_RFDI (0x4c00004e) #define PPC_RAW_RFMCI (0x4c00004c) #define PPC_RAW_TLBILX(t, a, b) (0x7c000024 | __PPC_T_TLB(t) | __PPC_RA0(a) | __PPC_RB(b)) -#define PPC_RAW_WAIT(w) (0x7c00007c | __PPC_WC(w)) +#define PPC_RAW_WAIT_v203 (0x7c00007c) +#define PPC_RAW_WAIT(w, p) (0x7c00003c | __PPC_WC(w) | __PPC_PL(p)) #define PPC_RAW_TLBIE(lp, a) (0x7c000264 | ___PPC_RB(a) | ___PPC_RS(lp)) #define PPC_RAW_TLBIE_5(rb, rs, ric, prs, r) \ (0x7c000264 | ___PPC_RB(rb) | ___PPC_RS(rs) | ___PPC_RIC(ric) | ___PPC_PRS(prs) | ___PPC_R(r)) @@ -606,7 +608,8 @@ #define PPC_TLBILX_ALL(a, b) PPC_TLBILX(0, a, b) #define PPC_TLBILX_PID(a, b) PPC_TLBILX(1, a, b) #define PPC_TLBILX_VA(a, b) PPC_TLBILX(3, a, b) -#define PPC_WAIT(w) stringify_in_c(.long PPC_RAW_WAIT(w)) +#define PPC_WAIT_v203 stringify_in_c(.long PPC_RAW_WAIT_v203) +#define PPC_WAIT(w, p) stringify_in_c(.long PPC_RAW_WAIT(w, p)) #define PPC_TLBIE(lp, a) stringify_in_c(.long PPC_RAW_TLBIE(lp, a)) #define PPC_TLBIE_5(rb, rs, ric, prs, r) \ stringify_in_c(.long PPC_RAW_TLBIE_5(rb, rs, ric, prs, r)) diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 83c02f5a7f2ab11d056f4e5c114172563ac2462d..753a2757bcd4f3c78927a0b5ded949d6c5d3b19a 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -33,6 +33,20 @@ .endr .endm +/* + * This expands to a sequence of register clears for regs start to end + * inclusive, of the form: + * + * li rN, 0 + */ +.macro ZEROIZE_REGS start, end + .Lreg=\start + .rept (\end - \start + 1) + li .Lreg, 0 + .Lreg=.Lreg+1 + .endr +.endm + /* * Macros for storing registers into and loading registers from * exception frames. @@ -49,6 +63,14 @@ #define REST_NVGPRS(base) REST_GPRS(13, 31, base) #endif +#define ZEROIZE_GPRS(start, end) ZEROIZE_REGS start, end +#ifdef __powerpc64__ +#define ZEROIZE_NVGPRS() ZEROIZE_GPRS(14, 31) +#else +#define ZEROIZE_NVGPRS() ZEROIZE_GPRS(13, 31) +#endif +#define ZEROIZE_GPR(n) ZEROIZE_GPRS(n, n) + #define SAVE_GPR(n, base) SAVE_GPRS(n, n, base) #define REST_GPR(n, base) REST_GPRS(n, n, base) @@ -305,6 +327,12 @@ n: #ifdef __powerpc64__ +#define __LOAD_PACA_TOC(reg) \ + ld reg,PACATOC(r13) + +#define LOAD_PACA_TOC() \ + __LOAD_PACA_TOC(r2) + #define LOAD_REG_IMMEDIATE(reg, expr) __LOAD_REG_IMMEDIATE reg, expr #define LOAD_REG_IMMEDIATE_SYM(reg, tmp, expr) \ @@ -315,7 +343,19 @@ n: rldimi reg, tmp, 32, 0 #define LOAD_REG_ADDR(reg,name) \ - ld reg,name@got(r2) + addis reg,r2,name@toc@ha; \ + addi reg,reg,name@toc@l + +#ifdef CONFIG_PPC_BOOK3E_64 +/* + * This is used in register-constrained interrupt handlers. Not to be used + * by BOOK3S. ld complains with "got/toc optimization is not supported" if r2 + * is not used for the TOC offset, so use @got(tocreg). If the interrupt + * handlers saved r2 instead, LOAD_REG_ADDR could be used. + */ +#define LOAD_REG_ADDR_ALTTOC(reg,tocreg,name) \ + ld reg,name@got(tocreg) +#endif #define LOAD_REG_ADDRBASE(reg,name) LOAD_REG_ADDR(reg,name) #define ADDROFF(name) 0 @@ -342,7 +382,7 @@ n: #endif /* various errata or part fixups */ -#if defined(CONFIG_PPC_CELL) || defined(CONFIG_PPC_FSL_BOOK3E) +#if defined(CONFIG_PPC_CELL) || defined(CONFIG_PPC_E500) #define MFTB(dest) \ 90: mfspr dest, SPRN_TBRL; \ BEGIN_FTR_SECTION_NESTED(96); \ @@ -709,7 +749,7 @@ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96) * kernel is built for. */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 #define FIXUP_ENDIAN #else /* @@ -749,7 +789,7 @@ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96) .long 0x2402004c; /* hrfid */ \ 191: -#endif /* !CONFIG_PPC_BOOK3E */ +#endif /* !CONFIG_PPC_BOOK3E_64 */ #endif /* __ASSEMBLY__ */ @@ -768,7 +808,7 @@ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96) stringify_in_c(.llong (_target);) \ stringify_in_c(.previous) -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 #define BTB_FLUSH(reg) \ lis reg,BUCSR_INIT@h; \ ori reg,reg,BUCSR_INIT@l; \ @@ -776,6 +816,6 @@ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96) isync; #else #define BTB_FLUSH(reg) -#endif /* CONFIG_PPC_FSL_BOOK3E */ +#endif /* CONFIG_PPC_E500 */ #endif /* _ASM_POWERPC_PPC_ASM_H */ diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index fdfaae194ddd5c17a978425db9501916ac02f3b8..631802999d5988f1aa7bf2f993a794563454ca06 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -75,7 +75,6 @@ extern int _chrp_type; struct task_struct; void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp); -void release_thread(struct task_struct *); #define TS_FPR(i) fp_state.fpr[i][TS_FPROFFSET] #define TS_CKFPR(i) ckfp_state.fpr[i][TS_FPROFFSET] @@ -355,11 +354,23 @@ static inline unsigned long __pack_fe01(unsigned int fpmode) #ifdef CONFIG_PPC64 -#define spin_begin() HMT_low() +#define spin_begin() \ + asm volatile(ASM_FTR_IFCLR( \ + "or 1,1,1", /* HMT_LOW */ \ + "nop", /* v3.1 uses pause_short in cpu_relax instead */ \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") -#define spin_cpu_relax() barrier() +#define spin_cpu_relax() \ + asm volatile(ASM_FTR_IFCLR( \ + "nop", /* Before v3.1 use priority nops in spin_begin/end */ \ + PPC_WAIT(2, 0), /* aka pause_short */ \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") -#define spin_end() HMT_medium() +#define spin_end() \ + asm volatile(ASM_FTR_IFCLR( \ + "or 2,2,2", /* HMT_MEDIUM */ \ + "nop", \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") #endif @@ -426,6 +437,8 @@ extern int fix_alignment(struct pt_regs *); #endif int do_mathemu(struct pt_regs *regs); +int do_spe_mathemu(struct pt_regs *regs); +int speround_handler(struct pt_regs *regs); /* VMX copying */ int enter_vmx_usercopy(void); diff --git a/arch/powerpc/include/asm/ps3av.h b/arch/powerpc/include/asm/ps3av.h index 82db78fc169dd2d65a1a6088e7a6555886e5da54..c8b0f2ffcd3567f94e549f8b940a48c192dac540 100644 --- a/arch/powerpc/include/asm/ps3av.h +++ b/arch/powerpc/include/asm/ps3av.h @@ -726,6 +726,4 @@ extern int ps3av_video_mode2res(u32, u32 *, u32 *); extern int ps3av_video_mute(int); extern int ps3av_audio_mute(int); extern int ps3av_audio_mute_analog(int); -extern int ps3av_dev_open(void); -extern int ps3av_dev_close(void); #endif /* _ASM_POWERPC_PS3AV_H_ */ diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index a03403695cd4ee6f266a0ad307b43f7d53e8bd65..2efec6d87049ea76f8bacd00f2797208a45fec66 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -99,6 +99,13 @@ struct pt_regs #define STACK_FRAME_WITH_PT_REGS (STACK_FRAME_OVERHEAD + sizeof(struct pt_regs)) +// Always displays as "REGS" in memory dumps +#ifdef CONFIG_CPU_BIG_ENDIAN +#define STACK_FRAME_REGS_MARKER ASM_CONST(0x52454753) +#else +#define STACK_FRAME_REGS_MARKER ASM_CONST(0x53474552) +#endif + #ifdef __powerpc64__ /* @@ -115,7 +122,6 @@ struct pt_regs #define STACK_FRAME_OVERHEAD 112 /* size of minimum stack frame */ #define STACK_FRAME_LR_SAVE 2 /* Location of LR in stack frame */ -#define STACK_FRAME_REGS_MARKER ASM_CONST(0x7265677368657265) #define STACK_INT_FRAME_SIZE (sizeof(struct pt_regs) + \ STACK_FRAME_OVERHEAD + KERNEL_REDZONE_SIZE) #define STACK_FRAME_MARKER 12 @@ -136,7 +142,6 @@ struct pt_regs #define KERNEL_REDZONE_SIZE 0 #define STACK_FRAME_OVERHEAD 16 /* size of minimum stack frame */ #define STACK_FRAME_LR_SAVE 1 /* Location of LR in stack frame */ -#define STACK_FRAME_REGS_MARKER ASM_CONST(0x72656773) #define STACK_INT_FRAME_SIZE (sizeof(struct pt_regs) + STACK_FRAME_OVERHEAD) #define STACK_FRAME_MARKER 2 #define STACK_FRAME_MIN_SIZE STACK_FRAME_OVERHEAD diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index 17b8dcd9a40dedf137c1842bbbc6fe504e3e83d0..af56980b6cdbea124382ab335be85cf488ac04d4 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -246,7 +246,7 @@ #define PPC47x_MCSR_FPR 0x00800000 /* FPR parity error */ #define PPC47x_MCSR_IPR 0x00400000 /* Imprecise Machine Check Exception */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 /* All e500 */ #define MCSR_MCP 0x80000000UL /* Machine Check Input Pin */ #define MCSR_ICPERR 0x40000000UL /* I-Cache Parity Error */ @@ -282,7 +282,7 @@ #endif /* Bit definitions for the HID1 */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 /* e500v1/v2 */ #define HID1_PLL_CFG_MASK 0xfc000000 /* PLL_CFG input pins */ #define HID1_RFXE 0x00020000 /* Read fault exception enable */ @@ -545,7 +545,7 @@ #define TCR_FIE 0x00800000 /* FIT Interrupt Enable */ #define TCR_ARE 0x00400000 /* Auto Reload Enable */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 #define TCR_GET_WP(tcr) ((((tcr) & 0xC0000000) >> 30) | \ (((tcr) & 0x1E0000) >> 15)) #else diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 00531af17ce05380fd54e91a0586c05f13c6e0f3..56319aea646e6a6b0f3fa7a0540f4f9963cd372c 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -240,7 +240,6 @@ extern struct rtas_t rtas; extern int rtas_token(const char *service); extern int rtas_service_present(const char *service); extern int rtas_call(int token, int, int, int *, ...); -int rtas_call_reentrant(int token, int nargs, int nret, int *outputs, ...); void rtas_call_unlocked(struct rtas_args *args, int token, int nargs, int nret, ...); extern void __noreturn rtas_restart(char *cmd); diff --git a/arch/powerpc/include/asm/runlatch.h b/arch/powerpc/include/asm/runlatch.h index cfb390edf7d0112f6046e85c82fecaaa82c06b6f..ceb66d761fe1f7444317ec5797e2e0983da9e699 100644 --- a/arch/powerpc/include/asm/runlatch.h +++ b/arch/powerpc/include/asm/runlatch.h @@ -19,10 +19,9 @@ extern void __ppc64_runlatch_off(void); do { \ if (cpu_has_feature(CPU_FTR_CTRL) && \ test_thread_local_flags(_TLF_RUNLATCH)) { \ - unsigned long msr = mfmsr(); \ __hard_irq_disable(); \ __ppc64_runlatch_off(); \ - if (msr & MSR_EE) \ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) \ __hard_irq_enable(); \ } \ } while (0) @@ -31,10 +30,9 @@ extern void __ppc64_runlatch_off(void); do { \ if (cpu_has_feature(CPU_FTR_CTRL) && \ !test_thread_local_flags(_TLF_RUNLATCH)) { \ - unsigned long msr = mfmsr(); \ __hard_irq_disable(); \ __ppc64_runlatch_on(); \ - if (msr & MSR_EE) \ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) \ __hard_irq_enable(); \ } \ } while (0) diff --git a/arch/powerpc/include/asm/sections.h b/arch/powerpc/include/asm/sections.h index 8be2c491c73319526549baf1a2cb32611f9448bc..9c00c9c0ca8f1724cf575c5f0301cfd852fefb2d 100644 --- a/arch/powerpc/include/asm/sections.h +++ b/arch/powerpc/include/asm/sections.h @@ -13,15 +13,26 @@ typedef struct func_desc func_desc_t; #include extern char __head_end[]; +extern char __srwx_boundary[]; + +/* Patch sites */ +extern s32 patch__call_flush_branch_caches1; +extern s32 patch__call_flush_branch_caches2; +extern s32 patch__call_flush_branch_caches3; +extern s32 patch__flush_count_cache_return; +extern s32 patch__flush_link_stack_return; +extern s32 patch__call_kvm_flush_link_stack; +extern s32 patch__call_kvm_flush_link_stack_p9; +extern s32 patch__memset_nocache, patch__memcpy_nocache; + +extern long flush_branch_caches; +extern long kvm_flush_link_stack; #ifdef __powerpc64__ extern char __start_interrupts[]; extern char __end_interrupts[]; -extern char __prom_init_toc_start[]; -extern char __prom_init_toc_end[]; - #ifdef CONFIG_PPC_POWERNV extern char start_real_trampolines[]; extern char end_real_trampolines[]; diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index d8c28902cf597b0d52ebf7297733625708ed492c..e29e83f8a89c8a64322ed291682b73332be62d62 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -7,7 +7,6 @@ #ifndef __ASSEMBLY__ extern void ppc_printk_progress(char *s, unsigned short hex); -extern unsigned int rtas_data; extern unsigned long long memory_limit; extern void *zalloc_maybe_bootmem(size_t size, gfp_t mask); @@ -70,7 +69,7 @@ void do_barrier_nospec_fixups_range(bool enable, void *start, void *end); static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { } #endif -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 void __init setup_spectre_v2(void); #else static inline void setup_spectre_v2(void) {} @@ -89,6 +88,8 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4, unsigned long pp, unsigned long r6, unsigned long r7, unsigned long kbase); +extern struct seq_buf ppc_hw_desc; + #endif /* !__ASSEMBLY__ */ #endif /* _ASM_POWERPC_SETUP_H */ diff --git a/arch/powerpc/include/asm/synch.h b/arch/powerpc/include/asm/synch.h index 7130176d8cb8826dfac2770dc303bc352fcdb4bf..b0b4c64870d77c75123ee2396359699553cc816b 100644 --- a/arch/powerpc/include/asm/synch.h +++ b/arch/powerpc/include/asm/synch.h @@ -44,7 +44,7 @@ static inline void ppc_after_tlbiel_barrier(void) #if defined(__powerpc64__) # define LWSYNC lwsync -#elif defined(CONFIG_E500) +#elif defined(CONFIG_PPC_E500) # define LWSYNC \ START_LWSYNC_SECTION(96); \ sync; \ diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index 25fc8ad9a27afaba9b17b8cc74b82fb7caa3c5f8..3dd36c5e334a9d628b01a738bd32a8f80c45976a 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -14,9 +14,16 @@ #include #include +#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER +typedef long (*syscall_fn)(const struct pt_regs *); +#else +typedef long (*syscall_fn)(unsigned long, unsigned long, unsigned long, + unsigned long, unsigned long, unsigned long); +#endif + /* ftrace syscalls requires exporting the sys_call_table */ -extern const unsigned long sys_call_table[]; -extern const unsigned long compat_sys_call_table[]; +extern const syscall_fn sys_call_table[]; +extern const syscall_fn compat_sys_call_table[]; static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { diff --git a/arch/powerpc/include/asm/syscall_wrapper.h b/arch/powerpc/include/asm/syscall_wrapper.h new file mode 100644 index 0000000000000000000000000000000000000000..67486c67e8a249912eb1c871e16e75a7e9d0612d --- /dev/null +++ b/arch/powerpc/include/asm/syscall_wrapper.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * syscall_wrapper.h - powerpc specific wrappers to syscall definitions + * + * Based on arch/{x86,arm64}/include/asm/syscall_wrapper.h + */ + +#ifndef __ASM_POWERPC_SYSCALL_WRAPPER_H +#define __ASM_POWERPC_SYSCALL_WRAPPER_H + +struct pt_regs; + +#define SC_POWERPC_REGS_TO_ARGS(x, ...) \ + __MAP(x,__SC_ARGS \ + ,,regs->gpr[3],,regs->gpr[4],,regs->gpr[5] \ + ,,regs->gpr[6],,regs->gpr[7],,regs->gpr[8]) + +#define __SYSCALL_DEFINEx(x, name, ...) \ + long sys##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(sys##name, ERRNO); \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ + static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ + long sys##name(const struct pt_regs *regs) \ + { \ + return __se_sys##name(SC_POWERPC_REGS_TO_ARGS(x,__VA_ARGS__)); \ + } \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ + { \ + long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \ + __MAP(x,__SC_TEST,__VA_ARGS__); \ + __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \ + return ret; \ + } \ + static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) + +#define SYSCALL_DEFINE0(sname) \ + SYSCALL_METADATA(_##sname, 0); \ + long sys_##sname(const struct pt_regs *__unused); \ + ALLOW_ERROR_INJECTION(sys_##sname, ERRNO); \ + long sys_##sname(const struct pt_regs *__unused) + +#define COND_SYSCALL(name) \ + long sys_##name(const struct pt_regs *regs); \ + long __weak sys_##name(const struct pt_regs *regs) \ + { \ + return sys_ni_syscall(); \ + } + +#endif // __ASM_POWERPC_SYSCALL_WRAPPER_H diff --git a/arch/powerpc/include/asm/syscalls.h b/arch/powerpc/include/asm/syscalls.h index a2b13e55254fb8c3df05340c9638b08553a9d66f..a1142496cd588569e99dc18c2f8fb4f5670d0bab 100644 --- a/arch/powerpc/include/asm/syscalls.h +++ b/arch/powerpc/include/asm/syscalls.h @@ -8,49 +8,147 @@ #include #include +#include +#ifdef CONFIG_PPC64 +#include +#endif +#include +#include + +#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER +long sys_ni_syscall(void); +#else +long sys_ni_syscall(const struct pt_regs *regs); +#endif + struct rtas_args; -asmlinkage long sys_mmap(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, off_t offset); -asmlinkage long sys_mmap2(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff); -asmlinkage long ppc64_personality(unsigned long personality); -asmlinkage long sys_rtas(struct rtas_args __user *uargs); -int ppc_select(int n, fd_set __user *inp, fd_set __user *outp, - fd_set __user *exp, struct __kernel_old_timeval __user *tvp); -long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low, - u32 len_high, u32 len_low); +/* + * long long munging: + * The 32 bit ABI passes long longs in an odd even register pair. + * High and low parts are swapped depending on endian mode, + * so define a macro (similar to mips linux32) to handle that. + */ +#ifdef __LITTLE_ENDIAN__ +#define merge_64(low, high) (((u64)high << 32) | low) +#else +#define merge_64(high, low) (((u64)high << 32) | low) +#endif + +/* + * PowerPC architecture-specific syscalls + */ + +#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER + +long sys_rtas(struct rtas_args __user *uargs); +#ifdef CONFIG_PPC64 +long sys_ppc64_personality(unsigned long personality); #ifdef CONFIG_COMPAT -unsigned long compat_sys_mmap2(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff); +long compat_sys_ppc64_personality(unsigned long personality); +#endif /* CONFIG_COMPAT */ +#endif /* CONFIG_PPC64 */ -compat_ssize_t compat_sys_pread64(unsigned int fd, char __user *ubuf, compat_size_t count, - u32 reg6, u32 pos1, u32 pos2); +long sys_swapcontext(struct ucontext __user *old_ctx, + struct ucontext __user *new_ctx, long ctx_size); +long sys_mmap(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, off_t offset); +long sys_mmap2(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long sys_switch_endian(void); -compat_ssize_t compat_sys_pwrite64(unsigned int fd, const char __user *ubuf, compat_size_t count, - u32 reg6, u32 pos1, u32 pos2); +#ifdef CONFIG_PPC32 +long sys_sigreturn(void); +long sys_debug_setcontext(struct ucontext __user *ctx, int ndbg, + struct sig_dbg_op __user *dbg); +#endif -compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offset1, u32 offset2, u32 count); +long sys_rt_sigreturn(void); -int compat_sys_truncate64(const char __user *path, u32 reg4, - unsigned long len1, unsigned long len2); +long sys_subpage_prot(unsigned long addr, + unsigned long len, u32 __user *map); -long compat_sys_fallocate(int fd, int mode, u32 offset1, u32 offset2, u32 len1, u32 len2); +#ifdef CONFIG_COMPAT +long compat_sys_swapcontext(struct ucontext32 __user *old_ctx, + struct ucontext32 __user *new_ctx, + int ctx_size); +long compat_sys_old_getrlimit(unsigned int resource, + struct compat_rlimit __user *rlim); +long compat_sys_sigreturn(void); +long compat_sys_rt_sigreturn(void); +#endif /* CONFIG_COMPAT */ -int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long len1, - unsigned long len2); +/* + * Architecture specific signatures required by long long munging: + * The 32 bit ABI passes long longs in an odd even register pair. + * The following signatures provide a machine long parameter for + * each register that will be supplied. The implementation is + * responsible for combining parameter pairs. + */ -long ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, - size_t len, int advice); +#ifdef CONFIG_PPC32 +long sys_ppc_pread64(unsigned int fd, + char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long sys_ppc_pwrite64(unsigned int fd, + const char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long sys_ppc_readahead(int fd, u32 r4, + u32 offset1, u32 offset2, u32 count); +long sys_ppc_truncate64(const char __user *path, u32 reg4, + unsigned long len1, unsigned long len2); +long sys_ppc_ftruncate64(unsigned int fd, u32 reg4, + unsigned long len1, unsigned long len2); +long sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, + size_t len, int advice); +#endif +#ifdef CONFIG_COMPAT +long compat_sys_mmap2(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long compat_sys_ppc_pread64(unsigned int fd, + char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long compat_sys_ppc_pwrite64(unsigned int fd, + const char __user *ubuf, compat_size_t count, + u32 reg6, u32 pos1, u32 pos2); +long compat_sys_ppc_readahead(int fd, u32 r4, + u32 offset1, u32 offset2, u32 count); +long compat_sys_ppc_truncate64(const char __user *path, u32 reg4, + unsigned long len1, unsigned long len2); +long compat_sys_ppc_ftruncate64(unsigned int fd, u32 reg4, + unsigned long len1, unsigned long len2); +long compat_sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, + size_t len, int advice); +long compat_sys_ppc_sync_file_range2(int fd, unsigned int flags, + unsigned int offset1, + unsigned int offset2, + unsigned int nbytes1, + unsigned int nbytes2); +#endif /* CONFIG_COMPAT */ -long compat_sys_sync_file_range2(int fd, unsigned int flags, - unsigned int offset1, unsigned int offset2, - unsigned int nbytes1, unsigned int nbytes2); +#if defined(CONFIG_PPC32) || defined(CONFIG_COMPAT) +long sys_ppc_fadvise64_64(int fd, int advice, + u32 offset_high, u32 offset_low, + u32 len_high, u32 len_low); #endif +#else + +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#define __SYSCALL(nr, entry) \ + long entry(const struct pt_regs *regs); + +#ifdef CONFIG_PPC64 +#include +#else +#include +#endif /* CONFIG_PPC64 */ + +#endif /* CONFIG_ARCH_HAS_SYSCALL_WRAPPER */ + #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_SYSCALLS_H */ diff --git a/arch/powerpc/kernel/ppc32.h b/arch/powerpc/include/asm/syscalls_32.h similarity index 93% rename from arch/powerpc/kernel/ppc32.h rename to arch/powerpc/include/asm/syscalls_32.h index 2346f8c7ff2ed1a4e4795c1a9ad1fb2398770019..749255568be90a78cca88dc7cc1fe8b409a6d186 100644 --- a/arch/powerpc/kernel/ppc32.h +++ b/arch/powerpc/include/asm/syscalls_32.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _PPC64_PPC32_H -#define _PPC64_PPC32_H +#ifndef _ASM_POWERPC_SYSCALLS_32_H +#define _ASM_POWERPC_SYSCALLS_32_H #include #include @@ -57,4 +57,4 @@ struct ucontext32 { struct mcontext32 uc_mcontext; }; -#endif /* _PPC64_PPC32_H */ +#endif // _ASM_POWERPC_SYSCALLS_32_H diff --git a/arch/powerpc/include/asm/termios.h b/arch/powerpc/include/asm/termios.h deleted file mode 100644 index 205de8f8a9d3ab5904bb91d85c39bb81759d97c7..0000000000000000000000000000000000000000 --- a/arch/powerpc/include/asm/termios.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Liberally adapted from alpha/termios.h. In particular, the c_cc[] - * fields have been reordered so that termio & termios share the - * common subset in the same order (for brain dead programs that don't - * know or care about the differences). - */ -#ifndef _ASM_POWERPC_TERMIOS_H -#define _ASM_POWERPC_TERMIOS_H - -#include - -/* ^C ^\ del ^U ^D 1 0 0 0 0 ^W ^R ^Z ^Q ^S ^V ^U */ -#define INIT_C_CC "\003\034\177\025\004\001\000\000\000\000\027\022\032\021\023\026\025" - -#include - -#endif /* _ASM_POWERPC_TERMIOS_H */ diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h index 1e5643a9b1f2eb4cefc432810f5946b206d0440b..9f50766c4623e5722e1721a13ea349b524acb190 100644 --- a/arch/powerpc/include/asm/time.h +++ b/arch/powerpc/include/asm/time.h @@ -116,8 +116,9 @@ unsigned long long tb_to_ns(unsigned long long tb_ticks); void timer_broadcast_interrupt(void); -/* SPLPAR */ -void accumulate_stolen_time(void); +/* SPLPAR and VIRT_CPU_ACCOUNTING_NATIVE */ +void pseries_accumulate_stolen_time(void); +u64 pseries_calculate_stolen_time(u64 stop_tb); #endif /* __KERNEL__ */ #endif /* __POWERPC_TIME_H */ diff --git a/arch/powerpc/include/asm/udbg.h b/arch/powerpc/include/asm/udbg.h index b4aa0d88ce2c51f42840f10e1e750690c9e7bc11..b1f094728b35911a449986cf62f8a332b89176ed 100644 --- a/arch/powerpc/include/asm/udbg.h +++ b/arch/powerpc/include/asm/udbg.h @@ -15,13 +15,13 @@ extern void (*udbg_flush)(void); extern int (*udbg_getc)(void); extern int (*udbg_getc_poll)(void); -extern void udbg_puts(const char *s); -extern int udbg_write(const char *s, int n); +void udbg_puts(const char *s); +int udbg_write(const char *s, int n); -extern void register_early_udbg_console(void); -extern void udbg_printf(const char *fmt, ...) +void register_early_udbg_console(void); +void udbg_printf(const char *fmt, ...) __attribute__ ((format (printf, 1, 2))); -extern void udbg_progress(char *s, unsigned short hex); +void udbg_progress(char *s, unsigned short hex); void __init udbg_uart_init_mmio(void __iomem *addr, unsigned int stride); void __init udbg_uart_init_pio(unsigned long port, unsigned int stride); @@ -31,28 +31,28 @@ unsigned int __init udbg_probe_uart_speed(unsigned int clock); struct device_node; void __init udbg_scc_init(int force_scc); -extern int udbg_adb_init(int force_btext); -extern void udbg_adb_init_early(void); - -extern void __init udbg_early_init(void); -extern void __init udbg_init_debug_lpar(void); -extern void __init udbg_init_debug_lpar_hvsi(void); -extern void __init udbg_init_pmac_realmode(void); -extern void __init udbg_init_maple_realmode(void); -extern void __init udbg_init_pas_realmode(void); -extern void __init udbg_init_rtas_panel(void); -extern void __init udbg_init_rtas_console(void); -extern void __init udbg_init_debug_beat(void); -extern void __init udbg_init_btext(void); -extern void __init udbg_init_44x_as1(void); -extern void __init udbg_init_40x_realmode(void); -extern void __init udbg_init_cpm(void); -extern void __init udbg_init_usbgecko(void); -extern void __init udbg_init_memcons(void); -extern void __init udbg_init_ehv_bc(void); -extern void __init udbg_init_ps3gelic(void); -extern void __init udbg_init_debug_opal_raw(void); -extern void __init udbg_init_debug_opal_hvsi(void); +int udbg_adb_init(int force_btext); +void udbg_adb_init_early(void); + +void __init udbg_early_init(void); +void __init udbg_init_debug_lpar(void); +void __init udbg_init_debug_lpar_hvsi(void); +void __init udbg_init_pmac_realmode(void); +void __init udbg_init_maple_realmode(void); +void __init udbg_init_pas_realmode(void); +void __init udbg_init_rtas_panel(void); +void __init udbg_init_rtas_console(void); +void __init udbg_init_btext(void); +void __init udbg_init_44x_as1(void); +void __init udbg_init_40x_realmode(void); +void __init udbg_init_cpm(void); +void __init udbg_init_usbgecko(void); +void __init udbg_init_memcons(void); +void __init udbg_init_ehv_bc(void); +void __init udbg_init_ps3gelic(void); +void __init udbg_init_debug_opal_raw(void); +void __init udbg_init_debug_opal_hvsi(void); +void __init udbg_init_debug_16550(void); #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_UDBG_H */ diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index b1129b4ef57d90c2ca06c0f4fb87c008b715e193..659a996c75aa8cbae741457c531cc2d0df58d02a 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h @@ -45,6 +45,7 @@ #define __ARCH_WANT_SYS_UTIME #define __ARCH_WANT_SYS_NEWFSTATAT #define __ARCH_WANT_COMPAT_STAT +#define __ARCH_WANT_COMPAT_FALLOCATE #define __ARCH_WANT_COMPAT_SYS_SENDFILE #endif #define __ARCH_WANT_SYS_FORK diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 8542e9bbeead30c388aa318d373d786e1c580b30..7650b6ce14c85ad1e6e7ee248721604d059b8446 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -2,9 +2,6 @@ #ifndef _ASM_POWERPC_VDSO_H #define _ASM_POWERPC_VDSO_H -/* Default map addresses for 32bit vDSO */ -#define VDSO32_MBASE 0x100000 - #define VDSO_VERSION_STRING LINUX_2.6.15 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/vdso/processor.h b/arch/powerpc/include/asm/vdso/processor.h index 8d79f994b4aa20555b455438537f427ff9dd821d..80d13207c5688d73954822aede2bbe2d0e05c054 100644 --- a/arch/powerpc/include/asm/vdso/processor.h +++ b/arch/powerpc/include/asm/vdso/processor.h @@ -22,7 +22,13 @@ #endif #ifdef CONFIG_PPC64 -#define cpu_relax() do { HMT_low(); HMT_medium(); barrier(); } while (0) +#define cpu_relax() \ + asm volatile(ASM_FTR_IFCLR( \ + /* Pre-POWER10 uses low ; medium priority nops */ \ + "or 1,1,1 ; or 2,2,2", \ + /* POWER10 onward uses pause_short (wait 2,0) */ \ + PPC_WAIT(2, 0), \ + %0) :: "i" (CPU_FTR_ARCH_31) : "memory") #else #define cpu_relax() barrier() #endif diff --git a/arch/powerpc/include/asm/vdso/timebase.h b/arch/powerpc/include/asm/vdso/timebase.h index 891c9d5eaabe10e494013271491ff3a72a813b6a..e9245f86a46cfba4d0173af0aac9a12528d94887 100644 --- a/arch/powerpc/include/asm/vdso/timebase.h +++ b/arch/powerpc/include/asm/vdso/timebase.h @@ -12,7 +12,7 @@ * We use __powerpc64__ here because we want the compat VDSO to use the 32-bit * version below in the else case of the ifdef. */ -#if defined(__powerpc64__) && (defined(CONFIG_PPC_CELL) || defined(CONFIG_E500)) +#if defined(__powerpc64__) && (defined(CONFIG_PPC_CELL) || defined(CONFIG_PPC_E500)) #define mftb() ({unsigned long rval; \ asm volatile( \ "90: mfspr %0, %2;\n" \ diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index e2e704eca5f65b62daf02742303eeff0fe733cf3..89090485bec18f4e29f1f668ce8fdaae40728a44 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -159,7 +159,6 @@ extern void xics_setup_cpu(void); extern void xics_update_irq_servers(void); extern void xics_set_cpu_giq(unsigned int gserver, unsigned int join); extern void xics_mask_unknown_vec(unsigned int vec); -extern irqreturn_t xics_ipi_dispatch(int cpu); extern void xics_smp_probe(void); extern void xics_register_ics(struct ics *ics); extern void xics_teardown_cpu(void); diff --git a/arch/powerpc/kernel/fsl_booke_entry_mapping.S b/arch/powerpc/kernel/85xx_entry_mapping.S similarity index 100% rename from arch/powerpc/kernel/fsl_booke_entry_mapping.S rename to arch/powerpc/kernel/85xx_entry_mapping.S diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 06d2d1f78f71b4f7c32e4b63d2d1eb922f8956a2..9b6146056e48b592b75d527c1732e18b2e68e813 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -73,6 +73,7 @@ obj-y := cputable.o syscalls.o \ obj-y += ptrace/ obj-$(CONFIG_PPC64) += setup_64.o irq_64.o\ paca.o nvram_64.o note.o +obj-$(CONFIG_PPC32) += sys_ppc32.o obj-$(CONFIG_COMPAT) += sys_ppc32.o signal_32.o obj-$(CONFIG_VDSO32) += vdso32_wrapper.o obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o @@ -81,7 +82,7 @@ obj-$(CONFIG_PPC_DAWR) += dawr.o obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_ppc970.o cpu_setup_pa6t.o obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o obj-$(CONFIG_PPC_BOOK3S_64) += mce.o mce_power.o -obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_book3e.o +obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_64e.o obj-$(CONFIG_PPC_BARRIER_NOSPEC) += security.o obj-$(CONFIG_PPC64) += vdso64_wrapper.o obj-$(CONFIG_ALTIVEC) += vecemu.o @@ -100,30 +101,28 @@ obj-$(CONFIG_GENERIC_TBSYNC) += smp-tbsync.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_FA_DUMP) += fadump.o obj-$(CONFIG_PRESERVE_FA_DUMP) += fadump.o -ifdef CONFIG_PPC32 -obj-$(CONFIG_E500) += idle_e500.o -endif +obj-$(CONFIG_PPC_85xx) += idle_85xx.o obj-$(CONFIG_PPC_BOOK3S_32) += idle_6xx.o l2cr_6xx.o cpu_setup_6xx.o obj-$(CONFIG_TAU) += tau_6xx.o obj-$(CONFIG_HIBERNATION) += swsusp.o suspend.o -ifdef CONFIG_FSL_BOOKE -obj-$(CONFIG_HIBERNATION) += swsusp_booke.o +ifdef CONFIG_PPC_85xx +obj-$(CONFIG_HIBERNATION) += swsusp_85xx.o else obj-$(CONFIG_HIBERNATION) += swsusp_$(BITS).o endif obj64-$(CONFIG_HIBERNATION) += swsusp_asm64.o obj-$(CONFIG_MODULES) += module.o module_$(BITS).o obj-$(CONFIG_44x) += cpu_setup_44x.o -obj-$(CONFIG_PPC_FSL_BOOK3E) += cpu_setup_fsl_booke.o +obj-$(CONFIG_PPC_E500) += cpu_setup_e500.o obj-$(CONFIG_PPC_DOORBELL) += dbell.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o -extra-$(CONFIG_PPC64) := head_64.o -extra-$(CONFIG_PPC_BOOK3S_32) := head_book3s_32.o -extra-$(CONFIG_40x) := head_40x.o -extra-$(CONFIG_44x) := head_44x.o -extra-$(CONFIG_FSL_BOOKE) := head_fsl_booke.o -extra-$(CONFIG_PPC_8xx) := head_8xx.o +obj-$(CONFIG_PPC64) += head_64.o +obj-$(CONFIG_PPC_BOOK3S_32) += head_book3s_32.o +obj-$(CONFIG_40x) += head_40x.o +obj-$(CONFIG_44x) += head_44x.o +obj-$(CONFIG_PPC_8xx) += head_8xx.o +obj-$(CONFIG_PPC_85xx) += head_85xx.o extra-y += vmlinux.lds obj-$(CONFIG_RELOCATABLE) += reloc_$(BITS).o @@ -198,10 +197,10 @@ KCOV_INSTRUMENT_paca.o := n CFLAGS_setup_64.o += -fno-stack-protector CFLAGS_paca.o += -fno-stack-protector -extra-$(CONFIG_PPC_FPU) += fpu.o -extra-$(CONFIG_ALTIVEC) += vector.o -extra-$(CONFIG_PPC64) += entry_64.o -extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o +obj-$(CONFIG_PPC_FPU) += fpu.o +obj-$(CONFIG_ALTIVEC) += vector.o +obj-$(CONFIG_PPC64) += entry_64.o +obj-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init.o extra-$(CONFIG_PPC_OF_BOOT_TRAMPOLINE) += prom_init_check diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 8c10f536e47891abf3051234e1fae42ebccdc51f..4ce2a4aa3985436e3ff788c3b06223a931f86269 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -59,7 +59,7 @@ #endif #endif -#if defined(CONFIG_PPC_FSL_BOOK3E) +#if defined(CONFIG_PPC_E500) #include "../mm/mmu_decl.h" #endif @@ -197,7 +197,7 @@ int main(void) OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened); OFFSET(PACA_FTRACE_ENABLED, paca_struct, ftrace_enabled); -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 OFFSET(PACAPGD, paca_struct, pgd); OFFSET(PACA_KERNELPGD, paca_struct, kernel_pgd); OFFSET(PACA_EXGEN, paca_struct, exgen); @@ -213,7 +213,7 @@ int main(void) OFFSET(TCD_ESEL_NEXT, tlb_core_data, esel_next); OFFSET(TCD_ESEL_MAX, tlb_core_data, esel_max); OFFSET(TCD_ESEL_FIRST, tlb_core_data, esel_first); -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ #ifdef CONFIG_PPC_BOOK3S_64 OFFSET(PACA_EXGEN, paca_struct, exgen); @@ -248,7 +248,7 @@ int main(void) #ifdef CONFIG_PPC64 OFFSET(PACA_EXIT_SAVE_R1, paca_struct, exit_save_r1); #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save); #endif OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso); @@ -651,7 +651,7 @@ int main(void) DEFINE(PGD_T_LOG2, PGD_T_LOG2); DEFINE(PTE_T_LOG2, PTE_T_LOG2); #endif -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 DEFINE(TLBCAM_SIZE, sizeof(struct tlbcam)); OFFSET(TLBCAM_MAS0, tlbcam, MAS0); OFFSET(TLBCAM_MAS1, tlbcam, MAS1); diff --git a/arch/powerpc/kernel/cpu_setup_fsl_booke.S b/arch/powerpc/kernel/cpu_setup_e500.S similarity index 98% rename from arch/powerpc/kernel/cpu_setup_fsl_booke.S rename to arch/powerpc/kernel/cpu_setup_e500.S index 4bf33f1b4193cf8c56fd2e85e654227f428269a2..2ab25161b0adc94deac23e15c587722bc04c4d66 100644 --- a/arch/powerpc/kernel/cpu_setup_fsl_booke.S +++ b/arch/powerpc/kernel/cpu_setup_e500.S @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -108,7 +108,7 @@ _GLOBAL(__setup_cpu_e6500) #endif /* CONFIG_PPC_E500MC */ #ifdef CONFIG_PPC32 -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 #ifndef CONFIG_PPC_E500MC _GLOBAL(__setup_cpu_e500v1) _GLOBAL(__setup_cpu_e500v2) @@ -156,7 +156,7 @@ _GLOBAL(__setup_cpu_e5500) mtlr r5 blr #endif /* CONFIG_PPC_E500MC */ -#endif /* CONFIG_E500 */ +#endif /* CONFIG_PPC_E500 */ #endif /* CONFIG_PPC32 */ #ifdef CONFIG_PPC_BOOK3E_64 diff --git a/arch/powerpc/kernel/cpu_setup_power.c b/arch/powerpc/kernel/cpu_setup_power.c index 3dc61e203f37dcc63c9dd39b1874c4a7ae44c650..097c033668f0faee68f3a9d3410137142a778e53 100644 --- a/arch/powerpc/kernel/cpu_setup_power.c +++ b/arch/powerpc/kernel/cpu_setup_power.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include /* Disable CPU_FTR_HVMODE and return false if MSR:HV is not set */ static bool init_hvmode_206(struct cpu_spec *t) diff --git a/arch/powerpc/kernel/cpu_specs.h b/arch/powerpc/kernel/cpu_specs.h new file mode 100644 index 0000000000000000000000000000000000000000..85ded3f77204cfdb0301ef2c94bebc9e973913be --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifdef CONFIG_40x +#include "cpu_specs_40x.h" +#endif + +#ifdef CONFIG_PPC_47x +#include "cpu_specs_47x.h" +#elif defined(CONFIG_44x) +#include "cpu_specs_44x.h" +#endif + +#ifdef CONFIG_PPC_8xx +#include "cpu_specs_8xx.h" +#endif + +#ifdef CONFIG_PPC_E500MC +#include "cpu_specs_e500mc.h" +#elif defined(CONFIG_PPC_85xx) +#include "cpu_specs_85xx.h" +#endif + +#ifdef CONFIG_PPC_BOOK3S_32 +#include "cpu_specs_book3s_32.h" +#endif + +#ifdef CONFIG_PPC_BOOK3S_64 +#include "cpu_specs_book3s_64.h" +#endif diff --git a/arch/powerpc/kernel/cpu_specs_40x.h b/arch/powerpc/kernel/cpu_specs_40x.h new file mode 100644 index 0000000000000000000000000000000000000000..a1362a75b8c8766fc268587734b64572814619aa --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_40x.h @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +static struct cpu_spec cpu_specs[] __initdata = { + { /* STB 04xxx */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x41810000, + .cpu_name = "STB04xxx", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* NP405L */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x41610000, + .cpu_name = "NP405L", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* NP4GS3 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x40B10000, + .cpu_name = "NP4GS3", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* NP405H */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x41410000, + .cpu_name = "NP405H", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405GPr */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x50910000, + .cpu_name = "405GPr", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* STBx25xx */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x51510000, + .cpu_name = "STBx25xx", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405LP */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x41F10000, + .cpu_name = "405LP", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EP */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x51210000, + .cpu_name = "405EP", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EX Rev. A/B with Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910007, + .cpu_name = "405EX Rev. A/B", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EX Rev. C without Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x1291000d, + .cpu_name = "405EX Rev. C", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EX Rev. C with Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x1291000f, + .cpu_name = "405EX Rev. C", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EX Rev. D without Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910003, + .cpu_name = "405EX Rev. D", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EX Rev. D with Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910005, + .cpu_name = "405EX Rev. D", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EXr Rev. A/B without Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910001, + .cpu_name = "405EXr Rev. A/B", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EXr Rev. C without Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910009, + .cpu_name = "405EXr Rev. C", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EXr Rev. C with Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x1291000b, + .cpu_name = "405EXr Rev. C", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EXr Rev. D without Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910000, + .cpu_name = "405EXr Rev. D", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* 405EXr Rev. D with Security */ + .pvr_mask = 0xffff000f, + .pvr_value = 0x12910002, + .cpu_name = "405EXr Rev. D", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { + /* 405EZ */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x41510000, + .cpu_name = "405EZ", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* APM8018X */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x7ff11432, + .cpu_name = "APM8018X", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + }, + { /* default match */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "(generic 40x PPC)", + .cpu_features = CPU_FTRS_40X, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | + PPC_FEATURE_HAS_4xxMAC, + .mmu_features = MMU_FTR_TYPE_40x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc405", + } +}; diff --git a/arch/powerpc/kernel/cpu_specs_44x.h b/arch/powerpc/kernel/cpu_specs_44x.h new file mode 100644 index 0000000000000000000000000000000000000000..69c4cdc0cdeed36298becdabd83e80a1ab1b8af3 --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_44x.h @@ -0,0 +1,304 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ + PPC_FEATURE_BOOKE) + +static struct cpu_spec cpu_specs[] __initdata = { + { + .pvr_mask = 0xf0000fff, + .pvr_value = 0x40000850, + .cpu_name = "440GR Rev. A", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { /* Use logical PVR for 440EP (logical pvr = pvr | 0x8) */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x40000858, + .cpu_name = "440EP Rev. A", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440ep, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { + .pvr_mask = 0xf0000fff, + .pvr_value = 0x400008d3, + .cpu_name = "440GR Rev. B", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { /* Matches both physical and logical PVR for 440EP (logical pvr = pvr | 0x8) */ + .pvr_mask = 0xf0000ff7, + .pvr_value = 0x400008d4, + .cpu_name = "440EP Rev. C", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440ep, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { /* Use logical PVR for 440EP (logical pvr = pvr | 0x8) */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x400008db, + .cpu_name = "440EP Rev. B", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440ep, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { /* 440GRX */ + .pvr_mask = 0xf0000ffb, + .pvr_value = 0x200008D0, + .cpu_name = "440GRX", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440grx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* Use logical PVR for 440EPx (logical pvr = pvr | 0x8) */ + .pvr_mask = 0xf0000ffb, + .pvr_value = 0x200008D8, + .cpu_name = "440EPX", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440epx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440GP Rev. B */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x40000440, + .cpu_name = "440GP Rev. B", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440gp", + }, + { /* 440GP Rev. C */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x40000481, + .cpu_name = "440GP Rev. C", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440gp", + }, + { /* 440GX Rev. A */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x50000850, + .cpu_name = "440GX Rev. A", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440gx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440GX Rev. B */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x50000851, + .cpu_name = "440GX Rev. B", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440gx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440GX Rev. C */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x50000892, + .cpu_name = "440GX Rev. C", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440gx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440GX Rev. F */ + .pvr_mask = 0xf0000fff, + .pvr_value = 0x50000894, + .cpu_name = "440GX Rev. F", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440gx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440SP Rev. A */ + .pvr_mask = 0xfff00fff, + .pvr_value = 0x53200891, + .cpu_name = "440SP Rev. A", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440", + }, + { /* 440SPe Rev. A */ + .pvr_mask = 0xfff00fff, + .pvr_value = 0x53400890, + .cpu_name = "440SPe Rev. A", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440spe, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 440SPe Rev. B */ + .pvr_mask = 0xfff00fff, + .pvr_value = 0x53400891, + .cpu_name = "440SPe Rev. B", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_440spe, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 460EX */ + .pvr_mask = 0xffff0006, + .pvr_value = 0x13020002, + .cpu_name = "460EX", + .cpu_features = CPU_FTRS_440x6, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_460ex, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 460EX Rev B */ + .pvr_mask = 0xffff0007, + .pvr_value = 0x13020004, + .cpu_name = "460EX Rev. B", + .cpu_features = CPU_FTRS_440x6, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_460ex, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 460GT */ + .pvr_mask = 0xffff0006, + .pvr_value = 0x13020000, + .cpu_name = "460GT", + .cpu_features = CPU_FTRS_440x6, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_460gt, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 460GT Rev B */ + .pvr_mask = 0xffff0007, + .pvr_value = 0x13020005, + .cpu_name = "460GT Rev. B", + .cpu_features = CPU_FTRS_440x6, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_460gt, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 460SX */ + .pvr_mask = 0xffffff00, + .pvr_value = 0x13541800, + .cpu_name = "460SX", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_460sx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* 464 in APM821xx */ + .pvr_mask = 0xfffffff0, + .pvr_value = 0x12C41C80, + .cpu_name = "APM821XX", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE | + PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_apm821xx, + .machine_check = machine_check_440A, + .platform = "ppc440", + }, + { /* default match */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "(generic 44x PPC)", + .cpu_features = CPU_FTRS_44X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_44x, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_4xx, + .platform = "ppc440", + } +}; diff --git a/arch/powerpc/kernel/cpu_specs_47x.h b/arch/powerpc/kernel/cpu_specs_47x.h new file mode 100644 index 0000000000000000000000000000000000000000..3143cd504a51995b630457813bdf2a19c836b7cd --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_47x.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ + PPC_FEATURE_BOOKE) + +static struct cpu_spec cpu_specs[] __initdata = { + { /* 476 DD2 core */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x11a52080, + .cpu_name = "476", + .cpu_features = CPU_FTRS_47X | CPU_FTR_476_DD2, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | + MMU_FTR_LOCK_BCAST_INVAL, + .icache_bsize = 32, + .dcache_bsize = 128, + .machine_check = machine_check_47x, + .platform = "ppc470", + }, + { /* 476fpe */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x7ff50000, + .cpu_name = "476fpe", + .cpu_features = CPU_FTRS_47X | CPU_FTR_476_DD2, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | + MMU_FTR_LOCK_BCAST_INVAL, + .icache_bsize = 32, + .dcache_bsize = 128, + .machine_check = machine_check_47x, + .platform = "ppc470", + }, + { /* 476 iss */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00050000, + .cpu_name = "476", + .cpu_features = CPU_FTRS_47X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | + MMU_FTR_LOCK_BCAST_INVAL, + .icache_bsize = 32, + .dcache_bsize = 128, + .machine_check = machine_check_47x, + .platform = "ppc470", + }, + { /* 476 others */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x11a50000, + .cpu_name = "476", + .cpu_features = CPU_FTRS_47X, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .mmu_features = MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | + MMU_FTR_LOCK_BCAST_INVAL, + .icache_bsize = 32, + .dcache_bsize = 128, + .machine_check = machine_check_47x, + .platform = "ppc470", + }, + { /* default match */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "(generic 47x PPC)", + .cpu_features = CPU_FTRS_47X, + .cpu_user_features = COMMON_USER_BOOKE, + .mmu_features = MMU_FTR_TYPE_47x, + .icache_bsize = 32, + .dcache_bsize = 128, + .machine_check = machine_check_47x, + .platform = "ppc470", + } +}; diff --git a/arch/powerpc/kernel/cpu_specs_85xx.h b/arch/powerpc/kernel/cpu_specs_85xx.h new file mode 100644 index 0000000000000000000000000000000000000000..aaae202c1a8947344791e345ef94526434d1fa99 --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_85xx.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ + PPC_FEATURE_BOOKE) + +static struct cpu_spec cpu_specs[] __initdata = { + { /* e500 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80200000, + .cpu_name = "e500", + .cpu_features = CPU_FTRS_E500, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_SPE_COMP | + PPC_FEATURE_HAS_EFP_SINGLE_COMP, + .cpu_user_features2 = PPC_FEATURE2_ISEL, + .mmu_features = MMU_FTR_TYPE_FSL_E, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_e500v1, + .machine_check = machine_check_e500, + .platform = "ppc8540", + }, + { /* e500v2 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80210000, + .cpu_name = "e500v2", + .cpu_features = CPU_FTRS_E500_2, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_SPE_COMP | + PPC_FEATURE_HAS_EFP_SINGLE_COMP | + PPC_FEATURE_HAS_EFP_DOUBLE_COMP, + .cpu_user_features2 = PPC_FEATURE2_ISEL, + .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_e500v2, + .machine_check = machine_check_e500, + .platform = "ppc8548", + .cpu_down_flush = cpu_down_flush_e500v2, + }, + { /* default match */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "(generic E500 PPC)", + .cpu_features = CPU_FTRS_E500, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_SPE_COMP | + PPC_FEATURE_HAS_EFP_SINGLE_COMP, + .mmu_features = MMU_FTR_TYPE_FSL_E, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_e500, + .platform = "powerpc", + } +}; diff --git a/arch/powerpc/kernel/cpu_specs_8xx.h b/arch/powerpc/kernel/cpu_specs_8xx.h new file mode 100644 index 0000000000000000000000000000000000000000..93ddbc202ba34a9fe8302d3151ad3b7b4871f290 --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_8xx.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +static struct cpu_spec cpu_specs[] __initdata = { + { /* 8xx */ + .pvr_mask = 0xffff0000, + .pvr_value = PVR_8xx, + .cpu_name = "8xx", + /* + * CPU_FTR_MAYBE_CAN_DOZE is possible, + * if the 8xx code is there.... + */ + .cpu_features = CPU_FTRS_8XX, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, + .mmu_features = MMU_FTR_TYPE_8xx, + .icache_bsize = 16, + .dcache_bsize = 16, + .machine_check = machine_check_8xx, + .platform = "ppc823", + }, +}; diff --git a/arch/powerpc/kernel/cpu_specs_book3s_32.h b/arch/powerpc/kernel/cpu_specs_book3s_32.h new file mode 100644 index 0000000000000000000000000000000000000000..3714634d194a1d0be0e922b17ecef17c4a8f25a8 --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_book3s_32.h @@ -0,0 +1,605 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + */ + +#define COMMON_USER (PPC_FEATURE_32 | PPC_FEATURE_HAS_FPU | \ + PPC_FEATURE_HAS_MMU) + +static struct cpu_spec cpu_specs[] __initdata = { +#ifdef CONFIG_PPC_BOOK3S_603 + { /* 603 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00030000, + .cpu_name = "603", + .cpu_features = CPU_FTRS_603, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = 0, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, + { /* 603e */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00060000, + .cpu_name = "603e", + .cpu_features = CPU_FTRS_603, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = 0, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, + { /* 603ev */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00070000, + .cpu_name = "603ev", + .cpu_features = CPU_FTRS_603, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = 0, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, + { /* 82xx (8240, 8245, 8260 are all 603e cores) */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00810000, + .cpu_name = "82xx", + .cpu_features = CPU_FTRS_82XX, + .cpu_user_features = COMMON_USER, + .mmu_features = 0, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, + { /* All G2_LE (603e core, plus some) have the same pvr */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00820000, + .cpu_name = "G2_LE", + .cpu_features = CPU_FTRS_G2_LE, + .cpu_user_features = COMMON_USER, + .mmu_features = MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, +#ifdef CONFIG_PPC_83xx + { /* e300c1 (a 603e core, plus some) on 83xx */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00830000, + .cpu_name = "e300c1", + .cpu_features = CPU_FTRS_E300, + .cpu_user_features = COMMON_USER, + .mmu_features = MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_83xx, + .platform = "ppc603", + }, + { /* e300c2 (an e300c1 core, plus some, minus FPU) on 83xx */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00840000, + .cpu_name = "e300c2", + .cpu_features = CPU_FTRS_E300C2, + .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, + .mmu_features = MMU_FTR_USE_HIGH_BATS | MMU_FTR_NEED_DTLB_SW_LRU, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_83xx, + .platform = "ppc603", + }, + { /* e300c3 (e300c1, plus one IU, half cache size) on 83xx */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00850000, + .cpu_name = "e300c3", + .cpu_features = CPU_FTRS_E300, + .cpu_user_features = COMMON_USER, + .mmu_features = MMU_FTR_USE_HIGH_BATS | MMU_FTR_NEED_DTLB_SW_LRU, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_83xx, + .num_pmcs = 4, + .platform = "ppc603", + }, + { /* e300c4 (e300c1, plus one IU) */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00860000, + .cpu_name = "e300c4", + .cpu_features = CPU_FTRS_E300, + .cpu_user_features = COMMON_USER, + .mmu_features = MMU_FTR_USE_HIGH_BATS | MMU_FTR_NEED_DTLB_SW_LRU, + .icache_bsize = 32, + .dcache_bsize = 32, + .cpu_setup = __setup_cpu_603, + .machine_check = machine_check_83xx, + .num_pmcs = 4, + .platform = "ppc603", + }, +#endif +#endif /* CONFIG_PPC_BOOK3S_603 */ +#ifdef CONFIG_PPC_BOOK3S_604 + { /* 604 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00040000, + .cpu_name = "604", + .cpu_features = CPU_FTRS_604, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 2, + .cpu_setup = __setup_cpu_604, + .machine_check = machine_check_generic, + .platform = "ppc604", + }, + { /* 604e */ + .pvr_mask = 0xfffff000, + .pvr_value = 0x00090000, + .cpu_name = "604e", + .cpu_features = CPU_FTRS_604, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_604, + .machine_check = machine_check_generic, + .platform = "ppc604", + }, + { /* 604r */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00090000, + .cpu_name = "604r", + .cpu_features = CPU_FTRS_604, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_604, + .machine_check = machine_check_generic, + .platform = "ppc604", + }, + { /* 604ev */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x000a0000, + .cpu_name = "604ev", + .cpu_features = CPU_FTRS_604, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_604, + .machine_check = machine_check_generic, + .platform = "ppc604", + }, + { /* 740/750 (0x4202, don't support TAU ?) */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x00084202, + .cpu_name = "740/750", + .cpu_features = CPU_FTRS_740_NOTAU, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750CX (80100 and 8010x?) */ + .pvr_mask = 0xfffffff0, + .pvr_value = 0x00080100, + .cpu_name = "750CX", + .cpu_features = CPU_FTRS_750, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_750cx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750CX (82201 and 82202) */ + .pvr_mask = 0xfffffff0, + .pvr_value = 0x00082200, + .cpu_name = "750CX", + .cpu_features = CPU_FTRS_750, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750cx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750CXe (82214) */ + .pvr_mask = 0xfffffff0, + .pvr_value = 0x00082210, + .cpu_name = "750CXe", + .cpu_features = CPU_FTRS_750, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750cx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750CXe "Gekko" (83214) */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x00083214, + .cpu_name = "750CXe", + .cpu_features = CPU_FTRS_750, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750cx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750CL (and "Broadway") */ + .pvr_mask = 0xfffff0e0, + .pvr_value = 0x00087000, + .cpu_name = "750CL", + .cpu_features = CPU_FTRS_750CL, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 745/755 */ + .pvr_mask = 0xfffff000, + .pvr_value = 0x00083000, + .cpu_name = "745/755", + .cpu_features = CPU_FTRS_750, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750FX rev 1.x */ + .pvr_mask = 0xffffff00, + .pvr_value = 0x70000100, + .cpu_name = "750FX", + .cpu_features = CPU_FTRS_750FX1, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750FX rev 2.0 must disable HID0[DPM] */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x70000200, + .cpu_name = "750FX", + .cpu_features = CPU_FTRS_750FX2, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750FX (All revs except 2.0) */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x70000000, + .cpu_name = "750FX", + .cpu_features = CPU_FTRS_750FX, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750fx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 750GX */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x70020000, + .cpu_name = "750GX", + .cpu_features = CPU_FTRS_750GX, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750fx, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 740/750 (L2CR bit need fixup for 740) */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00080000, + .cpu_name = "740/750", + .cpu_features = CPU_FTRS_740, + .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_750, + .machine_check = machine_check_generic, + .platform = "ppc750", + }, + { /* 7400 rev 1.1 ? (no TAU) */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x000c1101, + .cpu_name = "7400 (1.1)", + .cpu_features = CPU_FTRS_7400_NOTAU, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_7400, + .machine_check = machine_check_generic, + .platform = "ppc7400", + }, + { /* 7400 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x000c0000, + .cpu_name = "7400", + .cpu_features = CPU_FTRS_7400, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_7400, + .machine_check = machine_check_generic, + .platform = "ppc7400", + }, + { /* 7410 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x800c0000, + .cpu_name = "7410", + .cpu_features = CPU_FTRS_7400, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 4, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_7410, + .machine_check = machine_check_generic, + .platform = "ppc7400", + }, + { /* 7450 2.0 - no doze/nap */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x80000200, + .cpu_name = "7450", + .cpu_features = CPU_FTRS_7450_20, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7450 2.1 */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x80000201, + .cpu_name = "7450", + .cpu_features = CPU_FTRS_7450_21, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7450 2.3 and newer */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80000000, + .cpu_name = "7450", + .cpu_features = CPU_FTRS_7450_23, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7455 rev 1.x */ + .pvr_mask = 0xffffff00, + .pvr_value = 0x80010100, + .cpu_name = "7455", + .cpu_features = CPU_FTRS_7455_1, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7455 rev 2.0 */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x80010200, + .cpu_name = "7455", + .cpu_features = CPU_FTRS_7455_20, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7455 others */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80010000, + .cpu_name = "7455", + .cpu_features = CPU_FTRS_7455, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7447/7457 Rev 1.0 */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x80020100, + .cpu_name = "7447/7457", + .cpu_features = CPU_FTRS_7447_10, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7447/7457 Rev 1.1 */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x80020101, + .cpu_name = "7447/7457", + .cpu_features = CPU_FTRS_7447_10, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7447/7457 Rev 1.2 and later */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80020000, + .cpu_name = "7447/7457", + .cpu_features = CPU_FTRS_7447, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7447A */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80030000, + .cpu_name = "7447A", + .cpu_features = CPU_FTRS_7447A, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* 7448 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80040000, + .cpu_name = "7448", + .cpu_features = CPU_FTRS_7448, + .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | + PPC_FEATURE_PPC_LE, + .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, + .icache_bsize = 32, + .dcache_bsize = 32, + .num_pmcs = 6, + .pmc_type = PPC_PMC_G4, + .cpu_setup = __setup_cpu_745x, + .machine_check = machine_check_generic, + .platform = "ppc7450", + }, + { /* default match, we assume split I/D cache & TB (non-601)... */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "(generic PPC)", + .cpu_features = CPU_FTRS_CLASSIC32, + .cpu_user_features = COMMON_USER, + .mmu_features = MMU_FTR_HPTE_TABLE, + .icache_bsize = 32, + .dcache_bsize = 32, + .machine_check = machine_check_generic, + .platform = "ppc603", + }, +#endif /* CONFIG_PPC_BOOK3S_604 */ +}; diff --git a/arch/powerpc/kernel/cpu_specs_book3s_64.h b/arch/powerpc/kernel/cpu_specs_book3s_64.h new file mode 100644 index 0000000000000000000000000000000000000000..c370c1b804a9036774530ad8ee2d1a0f1d7591bd --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_book3s_64.h @@ -0,0 +1,481 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + * + * Modifications for ppc64: + * Copyright (C) 2003 Dave Engebretsen + */ + +/* NOTE: + * Unlike ppc32, ppc64 will only call cpu_setup() for the boot CPU, it's + * the responsibility of the appropriate CPU save/restore functions to + * eventually copy these settings over. Those save/restore aren't yet + * part of the cputable though. That has to be fixed for both ppc32 + * and ppc64 + */ +#define COMMON_USER_PPC64 (PPC_FEATURE_32 | PPC_FEATURE_HAS_FPU | \ + PPC_FEATURE_HAS_MMU | PPC_FEATURE_64) +#define COMMON_USER_POWER4 (COMMON_USER_PPC64 | PPC_FEATURE_POWER4) +#define COMMON_USER_POWER5 (COMMON_USER_PPC64 | PPC_FEATURE_POWER5 |\ + PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP) +#define COMMON_USER_POWER5_PLUS (COMMON_USER_PPC64 | PPC_FEATURE_POWER5_PLUS|\ + PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP) +#define COMMON_USER_POWER6 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_05 |\ + PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ + PPC_FEATURE_TRUE_LE | \ + PPC_FEATURE_PSERIES_PERFMON_COMPAT) +#define COMMON_USER_POWER7 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_06 |\ + PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ + PPC_FEATURE_TRUE_LE | \ + PPC_FEATURE_PSERIES_PERFMON_COMPAT) +#define COMMON_USER2_POWER7 (PPC_FEATURE2_DSCR) +#define COMMON_USER_POWER8 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_06 |\ + PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ + PPC_FEATURE_TRUE_LE | \ + PPC_FEATURE_PSERIES_PERFMON_COMPAT) +#define COMMON_USER2_POWER8 (PPC_FEATURE2_ARCH_2_07 | \ + PPC_FEATURE2_HTM_COMP | \ + PPC_FEATURE2_HTM_NOSC_COMP | \ + PPC_FEATURE2_DSCR | \ + PPC_FEATURE2_ISEL | PPC_FEATURE2_TAR | \ + PPC_FEATURE2_VEC_CRYPTO) +#define COMMON_USER_PA6T (COMMON_USER_PPC64 | PPC_FEATURE_PA6T |\ + PPC_FEATURE_TRUE_LE | \ + PPC_FEATURE_HAS_ALTIVEC_COMP) +#define COMMON_USER_POWER9 COMMON_USER_POWER8 +#define COMMON_USER2_POWER9 (COMMON_USER2_POWER8 | \ + PPC_FEATURE2_ARCH_3_00 | \ + PPC_FEATURE2_HAS_IEEE128 | \ + PPC_FEATURE2_DARN | \ + PPC_FEATURE2_SCV) +#define COMMON_USER_POWER10 COMMON_USER_POWER9 +#define COMMON_USER2_POWER10 (PPC_FEATURE2_ARCH_3_1 | \ + PPC_FEATURE2_MMA | \ + PPC_FEATURE2_ARCH_3_00 | \ + PPC_FEATURE2_HAS_IEEE128 | \ + PPC_FEATURE2_DARN | \ + PPC_FEATURE2_SCV | \ + PPC_FEATURE2_ARCH_2_07 | \ + PPC_FEATURE2_DSCR | \ + PPC_FEATURE2_ISEL | PPC_FEATURE2_TAR | \ + PPC_FEATURE2_VEC_CRYPTO) + +static struct cpu_spec cpu_specs[] __initdata = { + { /* PPC970 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00390000, + .cpu_name = "PPC970", + .cpu_features = CPU_FTRS_PPC970, + .cpu_user_features = COMMON_USER_POWER4 | PPC_FEATURE_HAS_ALTIVEC_COMP, + .mmu_features = MMU_FTRS_PPC970, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 8, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_ppc970, + .cpu_restore = __restore_cpu_ppc970, + .platform = "ppc970", + }, + { /* PPC970FX */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x003c0000, + .cpu_name = "PPC970FX", + .cpu_features = CPU_FTRS_PPC970, + .cpu_user_features = COMMON_USER_POWER4 | PPC_FEATURE_HAS_ALTIVEC_COMP, + .mmu_features = MMU_FTRS_PPC970, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 8, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_ppc970, + .cpu_restore = __restore_cpu_ppc970, + .platform = "ppc970", + }, + { /* PPC970MP DD1.0 - no DEEPNAP, use regular 970 init */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x00440100, + .cpu_name = "PPC970MP", + .cpu_features = CPU_FTRS_PPC970, + .cpu_user_features = COMMON_USER_POWER4 | PPC_FEATURE_HAS_ALTIVEC_COMP, + .mmu_features = MMU_FTRS_PPC970, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 8, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_ppc970, + .cpu_restore = __restore_cpu_ppc970, + .platform = "ppc970", + }, + { /* PPC970MP */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00440000, + .cpu_name = "PPC970MP", + .cpu_features = CPU_FTRS_PPC970, + .cpu_user_features = COMMON_USER_POWER4 | PPC_FEATURE_HAS_ALTIVEC_COMP, + .mmu_features = MMU_FTRS_PPC970, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 8, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_ppc970MP, + .cpu_restore = __restore_cpu_ppc970, + .platform = "ppc970", + }, + { /* PPC970GX */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00450000, + .cpu_name = "PPC970GX", + .cpu_features = CPU_FTRS_PPC970, + .cpu_user_features = COMMON_USER_POWER4 | PPC_FEATURE_HAS_ALTIVEC_COMP, + .mmu_features = MMU_FTRS_PPC970, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 8, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_ppc970, + .platform = "ppc970", + }, + { /* Power5 GR */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x003a0000, + .cpu_name = "POWER5 (gr)", + .cpu_features = CPU_FTRS_POWER5, + .cpu_user_features = COMMON_USER_POWER5, + .mmu_features = MMU_FTRS_POWER5, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .platform = "power5", + }, + { /* Power5++ */ + .pvr_mask = 0xffffff00, + .pvr_value = 0x003b0300, + .cpu_name = "POWER5+ (gs)", + .cpu_features = CPU_FTRS_POWER5, + .cpu_user_features = COMMON_USER_POWER5_PLUS, + .mmu_features = MMU_FTRS_POWER5, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .platform = "power5+", + }, + { /* Power5 GS */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x003b0000, + .cpu_name = "POWER5+ (gs)", + .cpu_features = CPU_FTRS_POWER5, + .cpu_user_features = COMMON_USER_POWER5_PLUS, + .mmu_features = MMU_FTRS_POWER5, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .platform = "power5+", + }, + { /* POWER6 in P5+ mode; 2.04-compliant processor */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000001, + .cpu_name = "POWER5+", + .cpu_features = CPU_FTRS_POWER5, + .cpu_user_features = COMMON_USER_POWER5_PLUS, + .mmu_features = MMU_FTRS_POWER5, + .icache_bsize = 128, + .dcache_bsize = 128, + .platform = "power5+", + }, + { /* Power6 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x003e0000, + .cpu_name = "POWER6 (raw)", + .cpu_features = CPU_FTRS_POWER6, + .cpu_user_features = COMMON_USER_POWER6 | PPC_FEATURE_POWER6_EXT, + .mmu_features = MMU_FTRS_POWER6, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .platform = "power6x", + }, + { /* 2.05-compliant processor, i.e. Power6 "architected" mode */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000002, + .cpu_name = "POWER6 (architected)", + .cpu_features = CPU_FTRS_POWER6, + .cpu_user_features = COMMON_USER_POWER6, + .mmu_features = MMU_FTRS_POWER6, + .icache_bsize = 128, + .dcache_bsize = 128, + .platform = "power6", + }, + { /* 2.06-compliant processor, i.e. Power7 "architected" mode */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000003, + .cpu_name = "POWER7 (architected)", + .cpu_features = CPU_FTRS_POWER7, + .cpu_user_features = COMMON_USER_POWER7, + .cpu_user_features2 = COMMON_USER2_POWER7, + .mmu_features = MMU_FTRS_POWER7, + .icache_bsize = 128, + .dcache_bsize = 128, + .cpu_setup = __setup_cpu_power7, + .cpu_restore = __restore_cpu_power7, + .machine_check_early = __machine_check_early_realmode_p7, + .platform = "power7", + }, + { /* 2.07-compliant processor, i.e. Power8 "architected" mode */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000004, + .cpu_name = "POWER8 (architected)", + .cpu_features = CPU_FTRS_POWER8, + .cpu_user_features = COMMON_USER_POWER8, + .cpu_user_features2 = COMMON_USER2_POWER8, + .mmu_features = MMU_FTRS_POWER8, + .icache_bsize = 128, + .dcache_bsize = 128, + .cpu_setup = __setup_cpu_power8, + .cpu_restore = __restore_cpu_power8, + .machine_check_early = __machine_check_early_realmode_p8, + .platform = "power8", + }, + { /* 3.00-compliant processor, i.e. Power9 "architected" mode */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000005, + .cpu_name = "POWER9 (architected)", + .cpu_features = CPU_FTRS_POWER9, + .cpu_user_features = COMMON_USER_POWER9, + .cpu_user_features2 = COMMON_USER2_POWER9, + .mmu_features = MMU_FTRS_POWER9, + .icache_bsize = 128, + .dcache_bsize = 128, + .cpu_setup = __setup_cpu_power9, + .cpu_restore = __restore_cpu_power9, + .platform = "power9", + }, + { /* 3.1-compliant processor, i.e. Power10 "architected" mode */ + .pvr_mask = 0xffffffff, + .pvr_value = 0x0f000006, + .cpu_name = "POWER10 (architected)", + .cpu_features = CPU_FTRS_POWER10, + .cpu_user_features = COMMON_USER_POWER10, + .cpu_user_features2 = COMMON_USER2_POWER10, + .mmu_features = MMU_FTRS_POWER10, + .icache_bsize = 128, + .dcache_bsize = 128, + .cpu_setup = __setup_cpu_power10, + .cpu_restore = __restore_cpu_power10, + .platform = "power10", + }, + { /* Power7 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x003f0000, + .cpu_name = "POWER7 (raw)", + .cpu_features = CPU_FTRS_POWER7, + .cpu_user_features = COMMON_USER_POWER7, + .cpu_user_features2 = COMMON_USER2_POWER7, + .mmu_features = MMU_FTRS_POWER7, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power7, + .cpu_restore = __restore_cpu_power7, + .machine_check_early = __machine_check_early_realmode_p7, + .platform = "power7", + }, + { /* Power7+ */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004A0000, + .cpu_name = "POWER7+ (raw)", + .cpu_features = CPU_FTRS_POWER7, + .cpu_user_features = COMMON_USER_POWER7, + .cpu_user_features2 = COMMON_USER2_POWER7, + .mmu_features = MMU_FTRS_POWER7, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power7, + .cpu_restore = __restore_cpu_power7, + .machine_check_early = __machine_check_early_realmode_p7, + .platform = "power7+", + }, + { /* Power8E */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004b0000, + .cpu_name = "POWER8E (raw)", + .cpu_features = CPU_FTRS_POWER8E, + .cpu_user_features = COMMON_USER_POWER8, + .cpu_user_features2 = COMMON_USER2_POWER8, + .mmu_features = MMU_FTRS_POWER8, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power8, + .cpu_restore = __restore_cpu_power8, + .machine_check_early = __machine_check_early_realmode_p8, + .platform = "power8", + }, + { /* Power8NVL */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004c0000, + .cpu_name = "POWER8NVL (raw)", + .cpu_features = CPU_FTRS_POWER8, + .cpu_user_features = COMMON_USER_POWER8, + .cpu_user_features2 = COMMON_USER2_POWER8, + .mmu_features = MMU_FTRS_POWER8, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power8, + .cpu_restore = __restore_cpu_power8, + .machine_check_early = __machine_check_early_realmode_p8, + .platform = "power8", + }, + { /* Power8 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004d0000, + .cpu_name = "POWER8 (raw)", + .cpu_features = CPU_FTRS_POWER8, + .cpu_user_features = COMMON_USER_POWER8, + .cpu_user_features2 = COMMON_USER2_POWER8, + .mmu_features = MMU_FTRS_POWER8, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power8, + .cpu_restore = __restore_cpu_power8, + .machine_check_early = __machine_check_early_realmode_p8, + .platform = "power8", + }, + { /* Power9 DD2.0 */ + .pvr_mask = 0xffffefff, + .pvr_value = 0x004e0200, + .cpu_name = "POWER9 (raw)", + .cpu_features = CPU_FTRS_POWER9_DD2_0, + .cpu_user_features = COMMON_USER_POWER9, + .cpu_user_features2 = COMMON_USER2_POWER9, + .mmu_features = MMU_FTRS_POWER9, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power9, + .cpu_restore = __restore_cpu_power9, + .machine_check_early = __machine_check_early_realmode_p9, + .platform = "power9", + }, + { /* Power9 DD 2.1 */ + .pvr_mask = 0xffffefff, + .pvr_value = 0x004e0201, + .cpu_name = "POWER9 (raw)", + .cpu_features = CPU_FTRS_POWER9_DD2_1, + .cpu_user_features = COMMON_USER_POWER9, + .cpu_user_features2 = COMMON_USER2_POWER9, + .mmu_features = MMU_FTRS_POWER9, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power9, + .cpu_restore = __restore_cpu_power9, + .machine_check_early = __machine_check_early_realmode_p9, + .platform = "power9", + }, + { /* Power9 DD2.2 */ + .pvr_mask = 0xffffefff, + .pvr_value = 0x004e0202, + .cpu_name = "POWER9 (raw)", + .cpu_features = CPU_FTRS_POWER9_DD2_2, + .cpu_user_features = COMMON_USER_POWER9, + .cpu_user_features2 = COMMON_USER2_POWER9, + .mmu_features = MMU_FTRS_POWER9, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power9, + .cpu_restore = __restore_cpu_power9, + .machine_check_early = __machine_check_early_realmode_p9, + .platform = "power9", + }, + { /* Power9 DD2.3 or later */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x004e0000, + .cpu_name = "POWER9 (raw)", + .cpu_features = CPU_FTRS_POWER9_DD2_3, + .cpu_user_features = COMMON_USER_POWER9, + .cpu_user_features2 = COMMON_USER2_POWER9, + .mmu_features = MMU_FTRS_POWER9, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power9, + .cpu_restore = __restore_cpu_power9, + .machine_check_early = __machine_check_early_realmode_p9, + .platform = "power9", + }, + { /* Power10 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00800000, + .cpu_name = "POWER10 (raw)", + .cpu_features = CPU_FTRS_POWER10, + .cpu_user_features = COMMON_USER_POWER10, + .cpu_user_features2 = COMMON_USER2_POWER10, + .mmu_features = MMU_FTRS_POWER10, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .cpu_setup = __setup_cpu_power10, + .cpu_restore = __restore_cpu_power10, + .machine_check_early = __machine_check_early_realmode_p10, + .platform = "power10", + }, + { /* Cell Broadband Engine */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00700000, + .cpu_name = "Cell Broadband Engine", + .cpu_features = CPU_FTRS_CELL, + .cpu_user_features = COMMON_USER_PPC64 | PPC_FEATURE_CELL | + PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_SMT, + .mmu_features = MMU_FTRS_CELL, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 4, + .pmc_type = PPC_PMC_IBM, + .platform = "ppc-cell-be", + }, + { /* PA Semi PA6T */ + .pvr_mask = 0x7fff0000, + .pvr_value = 0x00900000, + .cpu_name = "PA6T", + .cpu_features = CPU_FTRS_PA6T, + .cpu_user_features = COMMON_USER_PA6T, + .mmu_features = MMU_FTRS_PA6T, + .icache_bsize = 64, + .dcache_bsize = 64, + .num_pmcs = 6, + .pmc_type = PPC_PMC_PA6T, + .cpu_setup = __setup_cpu_pa6t, + .cpu_restore = __restore_cpu_pa6t, + .platform = "pa6t", + }, + { /* default match */ + .pvr_mask = 0x00000000, + .pvr_value = 0x00000000, + .cpu_name = "POWER5 (compatible)", + .cpu_features = CPU_FTRS_COMPATIBLE, + .cpu_user_features = COMMON_USER_PPC64, + .mmu_features = MMU_FTRS_POWER, + .icache_bsize = 128, + .dcache_bsize = 128, + .num_pmcs = 6, + .pmc_type = PPC_PMC_IBM, + .platform = "power5", + } +}; diff --git a/arch/powerpc/kernel/cpu_specs_e500mc.h b/arch/powerpc/kernel/cpu_specs_e500mc.h new file mode 100644 index 0000000000000000000000000000000000000000..ceb06b109f831355a833a0e929ef68d86ccbc321 --- /dev/null +++ b/arch/powerpc/kernel/cpu_specs_e500mc.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) + * + * Modifications for ppc64: + * Copyright (C) 2003 Dave Engebretsen + */ + +#ifdef CONFIG_PPC64 +#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ + PPC_FEATURE_HAS_FPU | PPC_FEATURE_64) +#else +#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ + PPC_FEATURE_BOOKE) +#endif + +static struct cpu_spec cpu_specs[] __initdata = { +#ifdef CONFIG_PPC32 + { /* e500mc */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80230000, + .cpu_name = "e500mc", + .cpu_features = CPU_FTRS_E500MC, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .cpu_user_features2 = PPC_FEATURE2_ISEL, + .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX, + .icache_bsize = 64, + .dcache_bsize = 64, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_e500mc, + .machine_check = machine_check_e500mc, + .platform = "ppce500mc", + .cpu_down_flush = cpu_down_flush_e500mc, + }, +#endif /* CONFIG_PPC32 */ + { /* e5500 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80240000, + .cpu_name = "e5500", + .cpu_features = CPU_FTRS_E5500, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, + .cpu_user_features2 = PPC_FEATURE2_ISEL, + .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX, + .icache_bsize = 64, + .dcache_bsize = 64, + .num_pmcs = 4, + .cpu_setup = __setup_cpu_e5500, +#ifndef CONFIG_PPC32 + .cpu_restore = __restore_cpu_e5500, +#endif + .machine_check = machine_check_e500mc, + .platform = "ppce5500", + .cpu_down_flush = cpu_down_flush_e5500, + }, + { /* e6500 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x80400000, + .cpu_name = "e6500", + .cpu_features = CPU_FTRS_E6500, + .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU | + PPC_FEATURE_HAS_ALTIVEC_COMP, + .cpu_user_features2 = PPC_FEATURE2_ISEL, + .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX, + .icache_bsize = 64, + .dcache_bsize = 64, + .num_pmcs = 6, + .cpu_setup = __setup_cpu_e6500, +#ifndef CONFIG_PPC32 + .cpu_restore = __restore_cpu_e6500, +#endif + .machine_check = machine_check_e500mc, + .platform = "ppce6500", + .cpu_down_flush = cpu_down_flush_e6500, + }, +}; diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index d8e42ef750f1a9b5a941cf85831e4ea80c1f3830..8a32bffefa5b716f90c989606acd5b225c07ad52 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -18,6 +18,7 @@ #include #include #include +#include static struct cpu_spec the_cpu_spec __read_mostly; @@ -27,1922 +28,7 @@ EXPORT_SYMBOL(cur_cpu_spec); /* The platform string corresponding to the real PVR */ const char *powerpc_base_platform; -/* NOTE: - * Unlike ppc32, ppc64 will only call this once for the boot CPU, it's - * the responsibility of the appropriate CPU save/restore functions to - * eventually copy these settings over. Those save/restore aren't yet - * part of the cputable though. That has to be fixed for both ppc32 - * and ppc64 - */ -#ifdef CONFIG_PPC32 -extern void __setup_cpu_e500v1(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_e500v2(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_e500mc(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440ep(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440epx(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440gx(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440grx(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440spe(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_440x5(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_460ex(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_460gt(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_460sx(unsigned long offset, struct cpu_spec *spec); -extern void __setup_cpu_apm821xx(unsigned long offset, struct cpu_spec *spec); -extern void __setup_cpu_603(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_604(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_750(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_750cx(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_750fx(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_7400(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_7410(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_745x(unsigned long offset, struct cpu_spec* spec); -#endif /* CONFIG_PPC32 */ -#ifdef CONFIG_PPC64 -#include -extern void __setup_cpu_ppc970(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_ppc970MP(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_pa6t(unsigned long offset, struct cpu_spec* spec); -extern void __restore_cpu_pa6t(void); -extern void __restore_cpu_ppc970(void); -extern long __machine_check_early_realmode_p7(struct pt_regs *regs); -extern long __machine_check_early_realmode_p8(struct pt_regs *regs); -extern long __machine_check_early_realmode_p9(struct pt_regs *regs); -#endif /* CONFIG_PPC64 */ -#if defined(CONFIG_E500) -extern void __setup_cpu_e5500(unsigned long offset, struct cpu_spec* spec); -extern void __setup_cpu_e6500(unsigned long offset, struct cpu_spec* spec); -extern void __restore_cpu_e5500(void); -extern void __restore_cpu_e6500(void); -#endif /* CONFIG_E500 */ - -/* This table only contains "desktop" CPUs, it need to be filled with embedded - * ones as well... - */ -#define COMMON_USER (PPC_FEATURE_32 | PPC_FEATURE_HAS_FPU | \ - PPC_FEATURE_HAS_MMU) -#define COMMON_USER_PPC64 (COMMON_USER | PPC_FEATURE_64) -#define COMMON_USER_POWER4 (COMMON_USER_PPC64 | PPC_FEATURE_POWER4) -#define COMMON_USER_POWER5 (COMMON_USER_PPC64 | PPC_FEATURE_POWER5 |\ - PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP) -#define COMMON_USER_POWER5_PLUS (COMMON_USER_PPC64 | PPC_FEATURE_POWER5_PLUS|\ - PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP) -#define COMMON_USER_POWER6 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_05 |\ - PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ - PPC_FEATURE_TRUE_LE | \ - PPC_FEATURE_PSERIES_PERFMON_COMPAT) -#define COMMON_USER_POWER7 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_06 |\ - PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ - PPC_FEATURE_TRUE_LE | \ - PPC_FEATURE_PSERIES_PERFMON_COMPAT) -#define COMMON_USER2_POWER7 (PPC_FEATURE2_DSCR) -#define COMMON_USER_POWER8 (COMMON_USER_PPC64 | PPC_FEATURE_ARCH_2_06 |\ - PPC_FEATURE_SMT | PPC_FEATURE_ICACHE_SNOOP | \ - PPC_FEATURE_TRUE_LE | \ - PPC_FEATURE_PSERIES_PERFMON_COMPAT) -#define COMMON_USER2_POWER8 (PPC_FEATURE2_ARCH_2_07 | \ - PPC_FEATURE2_HTM_COMP | \ - PPC_FEATURE2_HTM_NOSC_COMP | \ - PPC_FEATURE2_DSCR | \ - PPC_FEATURE2_ISEL | PPC_FEATURE2_TAR | \ - PPC_FEATURE2_VEC_CRYPTO) -#define COMMON_USER_PA6T (COMMON_USER_PPC64 | PPC_FEATURE_PA6T |\ - PPC_FEATURE_TRUE_LE | \ - PPC_FEATURE_HAS_ALTIVEC_COMP) -#define COMMON_USER_POWER9 COMMON_USER_POWER8 -#define COMMON_USER2_POWER9 (COMMON_USER2_POWER8 | \ - PPC_FEATURE2_ARCH_3_00 | \ - PPC_FEATURE2_HAS_IEEE128 | \ - PPC_FEATURE2_DARN | \ - PPC_FEATURE2_SCV) -#define COMMON_USER_POWER10 COMMON_USER_POWER9 -#define COMMON_USER2_POWER10 (PPC_FEATURE2_ARCH_3_1 | \ - PPC_FEATURE2_MMA | \ - PPC_FEATURE2_ARCH_3_00 | \ - PPC_FEATURE2_HAS_IEEE128 | \ - PPC_FEATURE2_DARN | \ - PPC_FEATURE2_SCV | \ - PPC_FEATURE2_ARCH_2_07 | \ - PPC_FEATURE2_DSCR | \ - PPC_FEATURE2_ISEL | PPC_FEATURE2_TAR | \ - PPC_FEATURE2_VEC_CRYPTO) - -#ifdef CONFIG_PPC_BOOK3E_64 -#define COMMON_USER_BOOKE (COMMON_USER_PPC64 | PPC_FEATURE_BOOKE) -#else -#define COMMON_USER_BOOKE (PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU | \ - PPC_FEATURE_BOOKE) -#endif - -static struct cpu_spec __initdata cpu_specs[] = { -#ifdef CONFIG_PPC_BOOK3S_64 - { /* PPC970 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00390000, - .cpu_name = "PPC970", - .cpu_features = CPU_FTRS_PPC970, - .cpu_user_features = COMMON_USER_POWER4 | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .mmu_features = MMU_FTRS_PPC970, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 8, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_ppc970, - .cpu_restore = __restore_cpu_ppc970, - .platform = "ppc970", - }, - { /* PPC970FX */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x003c0000, - .cpu_name = "PPC970FX", - .cpu_features = CPU_FTRS_PPC970, - .cpu_user_features = COMMON_USER_POWER4 | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .mmu_features = MMU_FTRS_PPC970, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 8, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_ppc970, - .cpu_restore = __restore_cpu_ppc970, - .platform = "ppc970", - }, - { /* PPC970MP DD1.0 - no DEEPNAP, use regular 970 init */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x00440100, - .cpu_name = "PPC970MP", - .cpu_features = CPU_FTRS_PPC970, - .cpu_user_features = COMMON_USER_POWER4 | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .mmu_features = MMU_FTRS_PPC970, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 8, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_ppc970, - .cpu_restore = __restore_cpu_ppc970, - .platform = "ppc970", - }, - { /* PPC970MP */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00440000, - .cpu_name = "PPC970MP", - .cpu_features = CPU_FTRS_PPC970, - .cpu_user_features = COMMON_USER_POWER4 | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .mmu_features = MMU_FTRS_PPC970, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 8, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_ppc970MP, - .cpu_restore = __restore_cpu_ppc970, - .platform = "ppc970", - }, - { /* PPC970GX */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00450000, - .cpu_name = "PPC970GX", - .cpu_features = CPU_FTRS_PPC970, - .cpu_user_features = COMMON_USER_POWER4 | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .mmu_features = MMU_FTRS_PPC970, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 8, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_ppc970, - .platform = "ppc970", - }, - { /* Power5 GR */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x003a0000, - .cpu_name = "POWER5 (gr)", - .cpu_features = CPU_FTRS_POWER5, - .cpu_user_features = COMMON_USER_POWER5, - .mmu_features = MMU_FTRS_POWER5, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .platform = "power5", - }, - { /* Power5++ */ - .pvr_mask = 0xffffff00, - .pvr_value = 0x003b0300, - .cpu_name = "POWER5+ (gs)", - .cpu_features = CPU_FTRS_POWER5, - .cpu_user_features = COMMON_USER_POWER5_PLUS, - .mmu_features = MMU_FTRS_POWER5, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .platform = "power5+", - }, - { /* Power5 GS */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x003b0000, - .cpu_name = "POWER5+ (gs)", - .cpu_features = CPU_FTRS_POWER5, - .cpu_user_features = COMMON_USER_POWER5_PLUS, - .mmu_features = MMU_FTRS_POWER5, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .platform = "power5+", - }, - { /* POWER6 in P5+ mode; 2.04-compliant processor */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000001, - .cpu_name = "POWER5+", - .cpu_features = CPU_FTRS_POWER5, - .cpu_user_features = COMMON_USER_POWER5_PLUS, - .mmu_features = MMU_FTRS_POWER5, - .icache_bsize = 128, - .dcache_bsize = 128, - .platform = "power5+", - }, - { /* Power6 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x003e0000, - .cpu_name = "POWER6 (raw)", - .cpu_features = CPU_FTRS_POWER6, - .cpu_user_features = COMMON_USER_POWER6 | - PPC_FEATURE_POWER6_EXT, - .mmu_features = MMU_FTRS_POWER6, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .platform = "power6x", - }, - { /* 2.05-compliant processor, i.e. Power6 "architected" mode */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000002, - .cpu_name = "POWER6 (architected)", - .cpu_features = CPU_FTRS_POWER6, - .cpu_user_features = COMMON_USER_POWER6, - .mmu_features = MMU_FTRS_POWER6, - .icache_bsize = 128, - .dcache_bsize = 128, - .platform = "power6", - }, - { /* 2.06-compliant processor, i.e. Power7 "architected" mode */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000003, - .cpu_name = "POWER7 (architected)", - .cpu_features = CPU_FTRS_POWER7, - .cpu_user_features = COMMON_USER_POWER7, - .cpu_user_features2 = COMMON_USER2_POWER7, - .mmu_features = MMU_FTRS_POWER7, - .icache_bsize = 128, - .dcache_bsize = 128, - .cpu_setup = __setup_cpu_power7, - .cpu_restore = __restore_cpu_power7, - .machine_check_early = __machine_check_early_realmode_p7, - .platform = "power7", - }, - { /* 2.07-compliant processor, i.e. Power8 "architected" mode */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000004, - .cpu_name = "POWER8 (architected)", - .cpu_features = CPU_FTRS_POWER8, - .cpu_user_features = COMMON_USER_POWER8, - .cpu_user_features2 = COMMON_USER2_POWER8, - .mmu_features = MMU_FTRS_POWER8, - .icache_bsize = 128, - .dcache_bsize = 128, - .cpu_setup = __setup_cpu_power8, - .cpu_restore = __restore_cpu_power8, - .machine_check_early = __machine_check_early_realmode_p8, - .platform = "power8", - }, - { /* 3.00-compliant processor, i.e. Power9 "architected" mode */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000005, - .cpu_name = "POWER9 (architected)", - .cpu_features = CPU_FTRS_POWER9, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .platform = "power9", - }, - { /* 3.1-compliant processor, i.e. Power10 "architected" mode */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x0f000006, - .cpu_name = "POWER10 (architected)", - .cpu_features = CPU_FTRS_POWER10, - .cpu_user_features = COMMON_USER_POWER10, - .cpu_user_features2 = COMMON_USER2_POWER10, - .mmu_features = MMU_FTRS_POWER10, - .icache_bsize = 128, - .dcache_bsize = 128, - .cpu_setup = __setup_cpu_power10, - .cpu_restore = __restore_cpu_power10, - .platform = "power10", - }, - { /* Power7 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x003f0000, - .cpu_name = "POWER7 (raw)", - .cpu_features = CPU_FTRS_POWER7, - .cpu_user_features = COMMON_USER_POWER7, - .cpu_user_features2 = COMMON_USER2_POWER7, - .mmu_features = MMU_FTRS_POWER7, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power7, - .cpu_restore = __restore_cpu_power7, - .machine_check_early = __machine_check_early_realmode_p7, - .platform = "power7", - }, - { /* Power7+ */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x004A0000, - .cpu_name = "POWER7+ (raw)", - .cpu_features = CPU_FTRS_POWER7, - .cpu_user_features = COMMON_USER_POWER7, - .cpu_user_features2 = COMMON_USER2_POWER7, - .mmu_features = MMU_FTRS_POWER7, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power7, - .cpu_restore = __restore_cpu_power7, - .machine_check_early = __machine_check_early_realmode_p7, - .platform = "power7+", - }, - { /* Power8E */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x004b0000, - .cpu_name = "POWER8E (raw)", - .cpu_features = CPU_FTRS_POWER8E, - .cpu_user_features = COMMON_USER_POWER8, - .cpu_user_features2 = COMMON_USER2_POWER8, - .mmu_features = MMU_FTRS_POWER8, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power8, - .cpu_restore = __restore_cpu_power8, - .machine_check_early = __machine_check_early_realmode_p8, - .platform = "power8", - }, - { /* Power8NVL */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x004c0000, - .cpu_name = "POWER8NVL (raw)", - .cpu_features = CPU_FTRS_POWER8, - .cpu_user_features = COMMON_USER_POWER8, - .cpu_user_features2 = COMMON_USER2_POWER8, - .mmu_features = MMU_FTRS_POWER8, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power8, - .cpu_restore = __restore_cpu_power8, - .machine_check_early = __machine_check_early_realmode_p8, - .platform = "power8", - }, - { /* Power8 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x004d0000, - .cpu_name = "POWER8 (raw)", - .cpu_features = CPU_FTRS_POWER8, - .cpu_user_features = COMMON_USER_POWER8, - .cpu_user_features2 = COMMON_USER2_POWER8, - .mmu_features = MMU_FTRS_POWER8, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power8, - .cpu_restore = __restore_cpu_power8, - .machine_check_early = __machine_check_early_realmode_p8, - .platform = "power8", - }, - { /* Power9 DD2.0 */ - .pvr_mask = 0xffffefff, - .pvr_value = 0x004e0200, - .cpu_name = "POWER9 (raw)", - .cpu_features = CPU_FTRS_POWER9_DD2_0, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .machine_check_early = __machine_check_early_realmode_p9, - .platform = "power9", - }, - { /* Power9 DD 2.1 */ - .pvr_mask = 0xffffefff, - .pvr_value = 0x004e0201, - .cpu_name = "POWER9 (raw)", - .cpu_features = CPU_FTRS_POWER9_DD2_1, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .machine_check_early = __machine_check_early_realmode_p9, - .platform = "power9", - }, - { /* Power9 DD2.2 */ - .pvr_mask = 0xffffefff, - .pvr_value = 0x004e0202, - .cpu_name = "POWER9 (raw)", - .cpu_features = CPU_FTRS_POWER9_DD2_2, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .machine_check_early = __machine_check_early_realmode_p9, - .platform = "power9", - }, - { /* Power9 DD2.3 or later */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x004e0000, - .cpu_name = "POWER9 (raw)", - .cpu_features = CPU_FTRS_POWER9_DD2_3, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .machine_check_early = __machine_check_early_realmode_p9, - .platform = "power9", - }, - { /* Power10 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00800000, - .cpu_name = "POWER10 (raw)", - .cpu_features = CPU_FTRS_POWER10, - .cpu_user_features = COMMON_USER_POWER10, - .cpu_user_features2 = COMMON_USER2_POWER10, - .mmu_features = MMU_FTRS_POWER10, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_power10, - .cpu_restore = __restore_cpu_power10, - .machine_check_early = __machine_check_early_realmode_p10, - .platform = "power10", - }, - { /* Cell Broadband Engine */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00700000, - .cpu_name = "Cell Broadband Engine", - .cpu_features = CPU_FTRS_CELL, - .cpu_user_features = COMMON_USER_PPC64 | - PPC_FEATURE_CELL | PPC_FEATURE_HAS_ALTIVEC_COMP | - PPC_FEATURE_SMT, - .mmu_features = MMU_FTRS_CELL, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .platform = "ppc-cell-be", - }, - { /* PA Semi PA6T */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00900000, - .cpu_name = "PA6T", - .cpu_features = CPU_FTRS_PA6T, - .cpu_user_features = COMMON_USER_PA6T, - .mmu_features = MMU_FTRS_PA6T, - .icache_bsize = 64, - .dcache_bsize = 64, - .num_pmcs = 6, - .pmc_type = PPC_PMC_PA6T, - .cpu_setup = __setup_cpu_pa6t, - .cpu_restore = __restore_cpu_pa6t, - .platform = "pa6t", - }, - { /* default match */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "POWER5 (compatible)", - .cpu_features = CPU_FTRS_COMPATIBLE, - .cpu_user_features = COMMON_USER_PPC64, - .mmu_features = MMU_FTRS_POWER, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .platform = "power5", - } -#endif /* CONFIG_PPC_BOOK3S_64 */ - -#ifdef CONFIG_PPC32 -#ifdef CONFIG_PPC_BOOK3S_32 -#ifdef CONFIG_PPC_BOOK3S_604 - { /* 604 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00040000, - .cpu_name = "604", - .cpu_features = CPU_FTRS_604, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 2, - .cpu_setup = __setup_cpu_604, - .machine_check = machine_check_generic, - .platform = "ppc604", - }, - { /* 604e */ - .pvr_mask = 0xfffff000, - .pvr_value = 0x00090000, - .cpu_name = "604e", - .cpu_features = CPU_FTRS_604, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_604, - .machine_check = machine_check_generic, - .platform = "ppc604", - }, - { /* 604r */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00090000, - .cpu_name = "604r", - .cpu_features = CPU_FTRS_604, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_604, - .machine_check = machine_check_generic, - .platform = "ppc604", - }, - { /* 604ev */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x000a0000, - .cpu_name = "604ev", - .cpu_features = CPU_FTRS_604, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_604, - .machine_check = machine_check_generic, - .platform = "ppc604", - }, - { /* 740/750 (0x4202, don't support TAU ?) */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x00084202, - .cpu_name = "740/750", - .cpu_features = CPU_FTRS_740_NOTAU, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750CX (80100 and 8010x?) */ - .pvr_mask = 0xfffffff0, - .pvr_value = 0x00080100, - .cpu_name = "750CX", - .cpu_features = CPU_FTRS_750, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_750cx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750CX (82201 and 82202) */ - .pvr_mask = 0xfffffff0, - .pvr_value = 0x00082200, - .cpu_name = "750CX", - .cpu_features = CPU_FTRS_750, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750cx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750CXe (82214) */ - .pvr_mask = 0xfffffff0, - .pvr_value = 0x00082210, - .cpu_name = "750CXe", - .cpu_features = CPU_FTRS_750, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750cx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750CXe "Gekko" (83214) */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x00083214, - .cpu_name = "750CXe", - .cpu_features = CPU_FTRS_750, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750cx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750CL (and "Broadway") */ - .pvr_mask = 0xfffff0e0, - .pvr_value = 0x00087000, - .cpu_name = "750CL", - .cpu_features = CPU_FTRS_750CL, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 745/755 */ - .pvr_mask = 0xfffff000, - .pvr_value = 0x00083000, - .cpu_name = "745/755", - .cpu_features = CPU_FTRS_750, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750FX rev 1.x */ - .pvr_mask = 0xffffff00, - .pvr_value = 0x70000100, - .cpu_name = "750FX", - .cpu_features = CPU_FTRS_750FX1, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750FX rev 2.0 must disable HID0[DPM] */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x70000200, - .cpu_name = "750FX", - .cpu_features = CPU_FTRS_750FX2, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750FX (All revs except 2.0) */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x70000000, - .cpu_name = "750FX", - .cpu_features = CPU_FTRS_750FX, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750fx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 750GX */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x70020000, - .cpu_name = "750GX", - .cpu_features = CPU_FTRS_750GX, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750fx, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 740/750 (L2CR bit need fixup for 740) */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00080000, - .cpu_name = "740/750", - .cpu_features = CPU_FTRS_740, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_IBM, - .cpu_setup = __setup_cpu_750, - .machine_check = machine_check_generic, - .platform = "ppc750", - }, - { /* 7400 rev 1.1 ? (no TAU) */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x000c1101, - .cpu_name = "7400 (1.1)", - .cpu_features = CPU_FTRS_7400_NOTAU, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_7400, - .machine_check = machine_check_generic, - .platform = "ppc7400", - }, - { /* 7400 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x000c0000, - .cpu_name = "7400", - .cpu_features = CPU_FTRS_7400, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_7400, - .machine_check = machine_check_generic, - .platform = "ppc7400", - }, - { /* 7410 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x800c0000, - .cpu_name = "7410", - .cpu_features = CPU_FTRS_7400, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_7410, - .machine_check = machine_check_generic, - .platform = "ppc7400", - }, - { /* 7450 2.0 - no doze/nap */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x80000200, - .cpu_name = "7450", - .cpu_features = CPU_FTRS_7450_20, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7450 2.1 */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x80000201, - .cpu_name = "7450", - .cpu_features = CPU_FTRS_7450_21, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7450 2.3 and newer */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80000000, - .cpu_name = "7450", - .cpu_features = CPU_FTRS_7450_23, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7455 rev 1.x */ - .pvr_mask = 0xffffff00, - .pvr_value = 0x80010100, - .cpu_name = "7455", - .cpu_features = CPU_FTRS_7455_1, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7455 rev 2.0 */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x80010200, - .cpu_name = "7455", - .cpu_features = CPU_FTRS_7455_20, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7455 others */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80010000, - .cpu_name = "7455", - .cpu_features = CPU_FTRS_7455, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7447/7457 Rev 1.0 */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x80020100, - .cpu_name = "7447/7457", - .cpu_features = CPU_FTRS_7447_10, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7447/7457 Rev 1.1 */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x80020101, - .cpu_name = "7447/7457", - .cpu_features = CPU_FTRS_7447_10, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7447/7457 Rev 1.2 and later */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80020000, - .cpu_name = "7447/7457", - .cpu_features = CPU_FTRS_7447, - .cpu_user_features = COMMON_USER | PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7447A */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80030000, - .cpu_name = "7447A", - .cpu_features = CPU_FTRS_7447A, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, - { /* 7448 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80040000, - .cpu_name = "7448", - .cpu_features = CPU_FTRS_7448, - .cpu_user_features = COMMON_USER | - PPC_FEATURE_HAS_ALTIVEC_COMP | PPC_FEATURE_PPC_LE, - .mmu_features = MMU_FTR_HPTE_TABLE | MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 6, - .pmc_type = PPC_PMC_G4, - .cpu_setup = __setup_cpu_745x, - .machine_check = machine_check_generic, - .platform = "ppc7450", - }, -#endif /* CONFIG_PPC_BOOK3S_604 */ -#ifdef CONFIG_PPC_BOOK3S_603 - { /* 603 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00030000, - .cpu_name = "603", - .cpu_features = CPU_FTRS_603, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = 0, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, - { /* 603e */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00060000, - .cpu_name = "603e", - .cpu_features = CPU_FTRS_603, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = 0, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, - { /* 603ev */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00070000, - .cpu_name = "603ev", - .cpu_features = CPU_FTRS_603, - .cpu_user_features = COMMON_USER | PPC_FEATURE_PPC_LE, - .mmu_features = 0, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, - { /* 82xx (8240, 8245, 8260 are all 603e cores) */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00810000, - .cpu_name = "82xx", - .cpu_features = CPU_FTRS_82XX, - .cpu_user_features = COMMON_USER, - .mmu_features = 0, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, - { /* All G2_LE (603e core, plus some) have the same pvr */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00820000, - .cpu_name = "G2_LE", - .cpu_features = CPU_FTRS_G2_LE, - .cpu_user_features = COMMON_USER, - .mmu_features = MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, -#ifdef CONFIG_PPC_83xx - { /* e300c1 (a 603e core, plus some) on 83xx */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00830000, - .cpu_name = "e300c1", - .cpu_features = CPU_FTRS_E300, - .cpu_user_features = COMMON_USER, - .mmu_features = MMU_FTR_USE_HIGH_BATS, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_83xx, - .platform = "ppc603", - }, - { /* e300c2 (an e300c1 core, plus some, minus FPU) on 83xx */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00840000, - .cpu_name = "e300c2", - .cpu_features = CPU_FTRS_E300C2, - .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, - .mmu_features = MMU_FTR_USE_HIGH_BATS | - MMU_FTR_NEED_DTLB_SW_LRU, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_83xx, - .platform = "ppc603", - }, - { /* e300c3 (e300c1, plus one IU, half cache size) on 83xx */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00850000, - .cpu_name = "e300c3", - .cpu_features = CPU_FTRS_E300, - .cpu_user_features = COMMON_USER, - .mmu_features = MMU_FTR_USE_HIGH_BATS | - MMU_FTR_NEED_DTLB_SW_LRU, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_83xx, - .num_pmcs = 4, - .platform = "ppc603", - }, - { /* e300c4 (e300c1, plus one IU) */ - .pvr_mask = 0x7fff0000, - .pvr_value = 0x00860000, - .cpu_name = "e300c4", - .cpu_features = CPU_FTRS_E300, - .cpu_user_features = COMMON_USER, - .mmu_features = MMU_FTR_USE_HIGH_BATS | - MMU_FTR_NEED_DTLB_SW_LRU, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_603, - .machine_check = machine_check_83xx, - .num_pmcs = 4, - .platform = "ppc603", - }, -#endif -#endif /* CONFIG_PPC_BOOK3S_603 */ -#ifdef CONFIG_PPC_BOOK3S_604 - { /* default match, we assume split I/D cache & TB (non-601)... */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "(generic PPC)", - .cpu_features = CPU_FTRS_CLASSIC32, - .cpu_user_features = COMMON_USER, - .mmu_features = MMU_FTR_HPTE_TABLE, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_generic, - .platform = "ppc603", - }, -#endif /* CONFIG_PPC_BOOK3S_604 */ -#endif /* CONFIG_PPC_BOOK3S_32 */ -#ifdef CONFIG_PPC_8xx - { /* 8xx */ - .pvr_mask = 0xffff0000, - .pvr_value = PVR_8xx, - .cpu_name = "8xx", - /* CPU_FTR_MAYBE_CAN_DOZE is possible, - * if the 8xx code is there.... */ - .cpu_features = CPU_FTRS_8XX, - .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, - .mmu_features = MMU_FTR_TYPE_8xx, - .icache_bsize = 16, - .dcache_bsize = 16, - .machine_check = machine_check_8xx, - .platform = "ppc823", - }, -#endif /* CONFIG_PPC_8xx */ -#ifdef CONFIG_40x - { /* STB 04xxx */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x41810000, - .cpu_name = "STB04xxx", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* NP405L */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x41610000, - .cpu_name = "NP405L", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* NP4GS3 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x40B10000, - .cpu_name = "NP4GS3", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* NP405H */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x41410000, - .cpu_name = "NP405H", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405GPr */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x50910000, - .cpu_name = "405GPr", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* STBx25xx */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x51510000, - .cpu_name = "STBx25xx", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405LP */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x41F10000, - .cpu_name = "405LP", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | PPC_FEATURE_HAS_MMU, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EP */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x51210000, - .cpu_name = "405EP", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EX Rev. A/B with Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910007, - .cpu_name = "405EX Rev. A/B", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EX Rev. C without Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x1291000d, - .cpu_name = "405EX Rev. C", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EX Rev. C with Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x1291000f, - .cpu_name = "405EX Rev. C", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EX Rev. D without Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910003, - .cpu_name = "405EX Rev. D", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EX Rev. D with Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910005, - .cpu_name = "405EX Rev. D", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EXr Rev. A/B without Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910001, - .cpu_name = "405EXr Rev. A/B", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EXr Rev. C without Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910009, - .cpu_name = "405EXr Rev. C", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EXr Rev. C with Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x1291000b, - .cpu_name = "405EXr Rev. C", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EXr Rev. D without Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910000, - .cpu_name = "405EXr Rev. D", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* 405EXr Rev. D with Security */ - .pvr_mask = 0xffff000f, - .pvr_value = 0x12910002, - .cpu_name = "405EXr Rev. D", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { - /* 405EZ */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x41510000, - .cpu_name = "405EZ", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* APM8018X */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x7ff11432, - .cpu_name = "APM8018X", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - }, - { /* default match */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "(generic 40x PPC)", - .cpu_features = CPU_FTRS_40X, - .cpu_user_features = PPC_FEATURE_32 | - PPC_FEATURE_HAS_MMU | PPC_FEATURE_HAS_4xxMAC, - .mmu_features = MMU_FTR_TYPE_40x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc405", - } - -#endif /* CONFIG_40x */ -#ifdef CONFIG_44x -#ifndef CONFIG_PPC_47x - { - .pvr_mask = 0xf0000fff, - .pvr_value = 0x40000850, - .cpu_name = "440GR Rev. A", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { /* Use logical PVR for 440EP (logical pvr = pvr | 0x8) */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x40000858, - .cpu_name = "440EP Rev. A", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440ep, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { - .pvr_mask = 0xf0000fff, - .pvr_value = 0x400008d3, - .cpu_name = "440GR Rev. B", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { /* Matches both physical and logical PVR for 440EP (logical pvr = pvr | 0x8) */ - .pvr_mask = 0xf0000ff7, - .pvr_value = 0x400008d4, - .cpu_name = "440EP Rev. C", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440ep, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { /* Use logical PVR for 440EP (logical pvr = pvr | 0x8) */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x400008db, - .cpu_name = "440EP Rev. B", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440ep, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { /* 440GRX */ - .pvr_mask = 0xf0000ffb, - .pvr_value = 0x200008D0, - .cpu_name = "440GRX", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440grx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* Use logical PVR for 440EPx (logical pvr = pvr | 0x8) */ - .pvr_mask = 0xf0000ffb, - .pvr_value = 0x200008D8, - .cpu_name = "440EPX", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440epx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440GP Rev. B */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x40000440, - .cpu_name = "440GP Rev. B", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440gp", - }, - { /* 440GP Rev. C */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x40000481, - .cpu_name = "440GP Rev. C", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440gp", - }, - { /* 440GX Rev. A */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x50000850, - .cpu_name = "440GX Rev. A", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440gx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440GX Rev. B */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x50000851, - .cpu_name = "440GX Rev. B", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440gx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440GX Rev. C */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x50000892, - .cpu_name = "440GX Rev. C", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440gx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440GX Rev. F */ - .pvr_mask = 0xf0000fff, - .pvr_value = 0x50000894, - .cpu_name = "440GX Rev. F", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440gx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440SP Rev. A */ - .pvr_mask = 0xfff00fff, - .pvr_value = 0x53200891, - .cpu_name = "440SP Rev. A", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440", - }, - { /* 440SPe Rev. A */ - .pvr_mask = 0xfff00fff, - .pvr_value = 0x53400890, - .cpu_name = "440SPe Rev. A", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440spe, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 440SPe Rev. B */ - .pvr_mask = 0xfff00fff, - .pvr_value = 0x53400891, - .cpu_name = "440SPe Rev. B", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_440spe, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 460EX */ - .pvr_mask = 0xffff0006, - .pvr_value = 0x13020002, - .cpu_name = "460EX", - .cpu_features = CPU_FTRS_440x6, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_460ex, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 460EX Rev B */ - .pvr_mask = 0xffff0007, - .pvr_value = 0x13020004, - .cpu_name = "460EX Rev. B", - .cpu_features = CPU_FTRS_440x6, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_460ex, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 460GT */ - .pvr_mask = 0xffff0006, - .pvr_value = 0x13020000, - .cpu_name = "460GT", - .cpu_features = CPU_FTRS_440x6, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_460gt, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 460GT Rev B */ - .pvr_mask = 0xffff0007, - .pvr_value = 0x13020005, - .cpu_name = "460GT Rev. B", - .cpu_features = CPU_FTRS_440x6, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_460gt, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 460SX */ - .pvr_mask = 0xffffff00, - .pvr_value = 0x13541800, - .cpu_name = "460SX", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_460sx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* 464 in APM821xx */ - .pvr_mask = 0xfffffff0, - .pvr_value = 0x12C41C80, - .cpu_name = "APM821XX", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .cpu_setup = __setup_cpu_apm821xx, - .machine_check = machine_check_440A, - .platform = "ppc440", - }, - { /* default match */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "(generic 44x PPC)", - .cpu_features = CPU_FTRS_44X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_44x, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_4xx, - .platform = "ppc440", - } -#else /* CONFIG_PPC_47x */ - { /* 476 DD2 core */ - .pvr_mask = 0xffffffff, - .pvr_value = 0x11a52080, - .cpu_name = "476", - .cpu_features = CPU_FTRS_47X | CPU_FTR_476_DD2, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_47x | - MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL, - .icache_bsize = 32, - .dcache_bsize = 128, - .machine_check = machine_check_47x, - .platform = "ppc470", - }, - { /* 476fpe */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x7ff50000, - .cpu_name = "476fpe", - .cpu_features = CPU_FTRS_47X | CPU_FTR_476_DD2, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_47x | - MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL, - .icache_bsize = 32, - .dcache_bsize = 128, - .machine_check = machine_check_47x, - .platform = "ppc470", - }, - { /* 476 iss */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x00050000, - .cpu_name = "476", - .cpu_features = CPU_FTRS_47X, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_47x | - MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL, - .icache_bsize = 32, - .dcache_bsize = 128, - .machine_check = machine_check_47x, - .platform = "ppc470", - }, - { /* 476 others */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x11a50000, - .cpu_name = "476", - .cpu_features = CPU_FTRS_47X, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_FPU, - .mmu_features = MMU_FTR_TYPE_47x | - MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL, - .icache_bsize = 32, - .dcache_bsize = 128, - .machine_check = machine_check_47x, - .platform = "ppc470", - }, - { /* default match */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "(generic 47x PPC)", - .cpu_features = CPU_FTRS_47X, - .cpu_user_features = COMMON_USER_BOOKE, - .mmu_features = MMU_FTR_TYPE_47x, - .icache_bsize = 32, - .dcache_bsize = 128, - .machine_check = machine_check_47x, - .platform = "ppc470", - } -#endif /* CONFIG_PPC_47x */ -#endif /* CONFIG_44x */ -#endif /* CONFIG_PPC32 */ -#ifdef CONFIG_E500 -#ifdef CONFIG_PPC32 -#ifndef CONFIG_PPC_E500MC - { /* e500 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80200000, - .cpu_name = "e500", - .cpu_features = CPU_FTRS_E500, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_SPE_COMP | - PPC_FEATURE_HAS_EFP_SINGLE_COMP, - .cpu_user_features2 = PPC_FEATURE2_ISEL, - .mmu_features = MMU_FTR_TYPE_FSL_E, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_e500v1, - .machine_check = machine_check_e500, - .platform = "ppc8540", - }, - { /* e500v2 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80210000, - .cpu_name = "e500v2", - .cpu_features = CPU_FTRS_E500_2, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_SPE_COMP | - PPC_FEATURE_HAS_EFP_SINGLE_COMP | - PPC_FEATURE_HAS_EFP_DOUBLE_COMP, - .cpu_user_features2 = PPC_FEATURE2_ISEL, - .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS, - .icache_bsize = 32, - .dcache_bsize = 32, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_e500v2, - .machine_check = machine_check_e500, - .platform = "ppc8548", - .cpu_down_flush = cpu_down_flush_e500v2, - }, -#else - { /* e500mc */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80230000, - .cpu_name = "e500mc", - .cpu_features = CPU_FTRS_E500MC, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .cpu_user_features2 = PPC_FEATURE2_ISEL, - .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | - MMU_FTR_USE_TLBILX, - .icache_bsize = 64, - .dcache_bsize = 64, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_e500mc, - .machine_check = machine_check_e500mc, - .platform = "ppce500mc", - .cpu_down_flush = cpu_down_flush_e500mc, - }, -#endif /* CONFIG_PPC_E500MC */ -#endif /* CONFIG_PPC32 */ -#ifdef CONFIG_PPC_E500MC - { /* e5500 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80240000, - .cpu_name = "e5500", - .cpu_features = CPU_FTRS_E5500, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU, - .cpu_user_features2 = PPC_FEATURE2_ISEL, - .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | - MMU_FTR_USE_TLBILX, - .icache_bsize = 64, - .dcache_bsize = 64, - .num_pmcs = 4, - .cpu_setup = __setup_cpu_e5500, -#ifndef CONFIG_PPC32 - .cpu_restore = __restore_cpu_e5500, -#endif - .machine_check = machine_check_e500mc, - .platform = "ppce5500", - .cpu_down_flush = cpu_down_flush_e5500, - }, - { /* e6500 */ - .pvr_mask = 0xffff0000, - .pvr_value = 0x80400000, - .cpu_name = "e6500", - .cpu_features = CPU_FTRS_E6500, - .cpu_user_features = COMMON_USER_BOOKE | PPC_FEATURE_HAS_FPU | - PPC_FEATURE_HAS_ALTIVEC_COMP, - .cpu_user_features2 = PPC_FEATURE2_ISEL, - .mmu_features = MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | - MMU_FTR_USE_TLBILX, - .icache_bsize = 64, - .dcache_bsize = 64, - .num_pmcs = 6, - .cpu_setup = __setup_cpu_e6500, -#ifndef CONFIG_PPC32 - .cpu_restore = __restore_cpu_e6500, -#endif - .machine_check = machine_check_e500mc, - .platform = "ppce6500", - .cpu_down_flush = cpu_down_flush_e6500, - }, -#endif /* CONFIG_PPC_E500MC */ -#ifdef CONFIG_PPC32 - { /* default match */ - .pvr_mask = 0x00000000, - .pvr_value = 0x00000000, - .cpu_name = "(generic E500 PPC)", - .cpu_features = CPU_FTRS_E500, - .cpu_user_features = COMMON_USER_BOOKE | - PPC_FEATURE_HAS_SPE_COMP | - PPC_FEATURE_HAS_EFP_SINGLE_COMP, - .mmu_features = MMU_FTR_TYPE_FSL_E, - .icache_bsize = 32, - .dcache_bsize = 32, - .machine_check = machine_check_e500, - .platform = "powerpc", - } -#endif /* CONFIG_PPC32 */ -#endif /* CONFIG_E500 */ -}; +#include "cpu_specs.h" void __init set_cur_cpu_spec(struct cpu_spec *s) { @@ -2018,6 +104,8 @@ struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr) struct cpu_spec *s = cpu_specs; int i; + BUILD_BUG_ON(!ARRAY_SIZE(cpu_specs)); + s = PTRRELOC(s); for (i = 0; i < ARRAY_SIZE(cpu_specs); i++,s++) { diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index fc800a9fb2c4fb370b863a26b02e454c5b9663e3..c3fb9fdf5bd782e121571efd679e812a4540b2a5 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -1099,7 +1099,7 @@ static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char prop = of_get_flat_dt_prop(node, "display-name", NULL); if (prop && strlen((char *)prop) != 0) { - strlcpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); + strscpy(dt_cpu_name, (char *)prop, sizeof(dt_cpu_name)); cur_cpu_spec->cpu_name = dt_cpu_name; } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 1d599df6f16906054531c6b54d6a9d50a432796c..3fc7c9886bb709be106d1d5ebcb6c63b59301edb 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -49,7 +49,7 @@ */ .align 12 -#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_E500) +#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_E500) .globl prepare_transfer_to_handler prepare_transfer_to_handler: /* if from kernel, check interrupted DOZE/NAP mode */ @@ -68,10 +68,10 @@ prepare_transfer_to_handler: lwz r9,_MSR(r11) /* if sleeping, clear MSR.EE */ rlwinm r9,r9,0,~MSR_EE lwz r12,_LINK(r11) /* and return to address in LR */ - lwz r2, GPR2(r11) + REST_GPR(2, r11) b fast_exception_return _ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler) -#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_E500 */ +#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_PPC_E500 */ #if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32) .globl __kuep_lock @@ -101,6 +101,7 @@ __kuep_unlock: .globl transfer_to_syscall transfer_to_syscall: + stw r3, ORIG_GPR3(r1) stw r11, GPR1(r1) stw r11, 0(r1) mflr r12 @@ -121,9 +122,9 @@ transfer_to_syscall: SAVE_NVGPRS(r1) kuep_lock - /* Calling convention has r9 = orig r0, r10 = regs */ - addi r10,r1,STACK_FRAME_OVERHEAD - mr r9,r0 + /* Calling convention has r3 = regs, r4 = orig r0 */ + addi r3,r1,STACK_FRAME_OVERHEAD + mr r4,r0 bl system_call_exception ret_from_syscall: @@ -143,7 +144,7 @@ ret_from_syscall: lwz r7,_NIP(r1) lwz r8,_MSR(r1) cmpwi r3,0 - lwz r3,GPR3(r1) + REST_GPR(3, r1) syscall_exit_finish: mtspr SPRN_SRR0,r7 mtspr SPRN_SRR1,r8 @@ -151,8 +152,8 @@ syscall_exit_finish: bne 3f mtcr r5 -1: lwz r2,GPR2(r1) - lwz r1,GPR1(r1) +1: REST_GPR(2, r1) + REST_GPR(1, r1) rfi #ifdef CONFIG_40x b . /* Prevent prefetch past rfi */ @@ -164,10 +165,8 @@ syscall_exit_finish: REST_NVGPRS(r1) mtctr r4 mtxer r5 - lwz r0,GPR0(r1) - lwz r3,GPR3(r1) - REST_GPRS(4, 11, r1) - lwz r12,GPR12(r1) + REST_GPR(0, r1) + REST_GPRS(3, 12, r1) b 1b #ifdef CONFIG_44x @@ -259,13 +258,12 @@ fast_exception_return: beq 3f /* if not, we've got problems */ #endif -2: REST_GPRS(3, 6, r11) - lwz r10,_CCR(r11) - REST_GPRS(1, 2, r11) +2: lwz r10,_CCR(r11) + REST_GPRS(1, 6, r11) mtcr r10 lwz r10,_LINK(r11) mtlr r10 - /* Clear the exception_marker on the stack to avoid confusing stacktrace */ + /* Clear the exception marker on the stack to avoid confusing stacktrace */ li r10, 0 stw r10, 8(r11) REST_GPR(10, r11) @@ -276,7 +274,7 @@ fast_exception_return: mtspr SPRN_SRR0,r12 REST_GPR(9, r11) REST_GPR(12, r11) - lwz r11,GPR11(r11) + REST_GPR(11, r11) rfi #ifdef CONFIG_40x b . /* Prevent prefetch past rfi */ @@ -322,7 +320,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) li r0,0 /* - * Leaving a stale exception_marker on the stack can confuse + * Leaving a stale exception marker on the stack can confuse * the reliable stack unwinder later on. Clear it. */ stw r0,8(r1) @@ -374,7 +372,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) mtspr SPRN_XER,r5 /* - * Leaving a stale exception_marker on the stack can confuse + * Leaving a stale exception marker on the stack can confuse * the reliable stack unwinder later on. Clear it. */ stw r0,8(r1) @@ -453,9 +451,8 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) lwz r3,_MSR(r1); \ andi. r3,r3,MSR_PR; \ bne interrupt_return; \ - lwz r0,GPR0(r1); \ - lwz r2,GPR2(r1); \ - REST_GPRS(3, 8, r1); \ + REST_GPR(0, r1); \ + REST_GPRS(2, 8, r1); \ lwz r10,_XER(r1); \ lwz r11,_CTR(r1); \ mtspr SPRN_XER,r10; \ @@ -474,11 +471,8 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) lwz r12,_MSR(r1); \ mtspr exc_lvl_srr0,r11; \ mtspr exc_lvl_srr1,r12; \ - lwz r9,GPR9(r1); \ - lwz r12,GPR12(r1); \ - lwz r10,GPR10(r1); \ - lwz r11,GPR11(r1); \ - lwz r1,GPR1(r1); \ + REST_GPRS(9, 12, r1); \ + REST_GPR(1, r1); \ exc_lvl_rfi; \ b .; /* prevent prefetch past exc_lvl_rfi */ @@ -488,7 +482,7 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return) mtspr SPRN_##exc_lvl_srr0,r9; \ mtspr SPRN_##exc_lvl_srr1,r10; -#if defined(CONFIG_PPC_BOOK3E_MMU) +#if defined(CONFIG_PPC_E500) #ifdef CONFIG_PHYS_64BIT #define RESTORE_MAS7 \ lwz r11,MAS7(r1); \ diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 01ace4c56104dcb8c613fcff810c167d12fcf05f..3e2e37e6ecabe9a8b81f74641e9130b28019af02 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -292,16 +292,16 @@ _GLOBAL(enter_prom) /* Prepare a 32-bit mode big endian MSR */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 rlwinm r11,r11,0,1,31 mtsrr1 r11 rfi -#else /* CONFIG_PPC_BOOK3E */ +#else /* CONFIG_PPC_BOOK3E_64 */ LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_LE) andc r11,r11,r12 mtsrr1 r11 RFI_TO_KERNEL -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ 1: /* Return from OF */ FIXUP_ENDIAN diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 67dc4e3179a02a45b89871bcf035ed3680d8e644..930e360990152ef8677df2279a048b7e616455f0 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -216,17 +216,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) mtlr r10 mtcr r11 - ld r10,GPR10(r1) - ld r11,GPR11(r1) - ld r12,GPR12(r1) + REST_GPRS(10, 12, r1) mtspr \scratch,r0 std r10,\paca_ex+EX_R10(r13); std r11,\paca_ex+EX_R11(r13); ld r10,_NIP(r1) ld r11,_MSR(r1) - ld r0,GPR0(r1) - ld r1,GPR1(r1) + REST_GPR(0, r1) + REST_GPR(1, r1) mtspr \srr0,r10 mtspr \srr1,r11 ld r10,\paca_ex+EX_R10(r13) @@ -291,7 +289,6 @@ ret_from_mc_except: #define SPRN_MC_SRR0 SPRN_MCSRR0 #define SPRN_MC_SRR1 SPRN_MCSRR1 -#ifdef CONFIG_PPC_FSL_BOOK3E #define GEN_BTB_FLUSH \ START_BTB_FLUSH_SECTION \ beq 1f; \ @@ -307,13 +304,6 @@ ret_from_mc_except: #define DBG_BTB_FLUSH CRIT_BTB_FLUSH #define MC_BTB_FLUSH CRIT_BTB_FLUSH #define GDBELL_BTB_FLUSH GEN_BTB_FLUSH -#else -#define GEN_BTB_FLUSH -#define CRIT_BTB_FLUSH -#define DBG_BTB_FLUSH -#define MC_BTB_FLUSH -#define GDBELL_BTB_FLUSH -#endif #define NORMAL_EXCEPTION_PROLOG(n, intnum, addition) \ EXCEPTION_PROLOG(n, intnum, GEN, addition##_GEN(n)) @@ -372,25 +362,24 @@ ret_from_mc_except: /* Core exception code for all exceptions except TLB misses. */ #define EXCEPTION_COMMON_LVL(n, scratch, excf) \ exc_##n##_common: \ - std r0,GPR0(r1); /* save r0 in stackframe */ \ - std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_GPRS(3, 9, r1); /* save r3 - r9 in stackframe */ \ + SAVE_GPR(0, r1); /* save r0 in stackframe */ \ + SAVE_GPRS(2, 9, r1); /* save r2 - r9 in stackframe */ \ std r10,_NIP(r1); /* save SRR0 to stackframe */ \ std r11,_MSR(r1); /* save SRR1 to stackframe */ \ beq 2f; /* if from kernel mode */ \ 2: ld r3,excf+EX_R10(r13); /* get back r10 */ \ ld r4,excf+EX_R11(r13); /* get back r11 */ \ mfspr r5,scratch; /* get back r13 */ \ - std r12,GPR12(r1); /* save r12 in stackframe */ \ - ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ + SAVE_GPR(12, r1); /* save r12 in stackframe */ \ + LOAD_PACA_TOC(); /* get kernel TOC into r2 */ \ mflr r6; /* save LR in stackframe */ \ mfctr r7; /* save CTR in stackframe */ \ mfspr r8,SPRN_XER; /* save XER in stackframe */ \ ld r9,excf+EX_R1(r13); /* load orig r1 back from PACA */ \ lwz r10,excf+EX_CR(r13); /* load orig CR back from PACA */ \ lbz r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */ \ - ld r12,exception_marker@toc(r2); \ - li r0,0; \ + LOAD_REG_IMMEDIATE(r12, STACK_FRAME_REGS_MARKER); \ + ZEROIZE_GPR(0); \ std r3,GPR10(r1); /* save r10 to stackframe */ \ std r4,GPR11(r1); /* save r11 to stackframe */ \ std r5,GPR13(r1); /* save it to stackframe */ \ @@ -470,12 +459,6 @@ exc_##n##_bad_stack: \ bl hdlr; \ b interrupt_return -/* This value is used to mark exception frames on the stack. */ - .section ".toc","aw" -exception_marker: - .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER - - /* * And here we have the exception vectors ! */ @@ -704,9 +687,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) beq+ 1f #ifdef CONFIG_RELOCATABLE - ld r15,PACATOC(r13) - ld r14,interrupt_base_book3e@got(r15) - ld r15,__end_interrupts@got(r15) + __LOAD_PACA_TOC(r15) + LOAD_REG_ADDR_ALTTOC(r14, r15, interrupt_base_book3e) + LOAD_REG_ADDR_ALTTOC(r15, r15, __end_interrupts) cmpld cr0,r10,r14 cmpld cr1,r10,r15 #else @@ -775,9 +758,9 @@ kernel_dbg_exc: beq+ 1f #ifdef CONFIG_RELOCATABLE - ld r15,PACATOC(r13) - ld r14,interrupt_base_book3e@got(r15) - ld r15,__end_interrupts@got(r15) + __LOAD_PACA_TOC(r15) + LOAD_REG_ADDR_ALTTOC(r14, r15, interrupt_base_book3e) + LOAD_REG_ADDR_ALTTOC(r15, r15, __end_interrupts) cmpld cr0,r10,r14 cmpld cr1,r10,r15 #else @@ -900,9 +883,9 @@ kernel_dbg_exc: .macro SEARCH_RESTART_TABLE #ifdef CONFIG_RELOCATABLE - ld r11,PACATOC(r13) - ld r14,__start___restart_table@got(r11) - ld r15,__stop___restart_table@got(r11) + __LOAD_PACA_TOC(r11) + LOAD_REG_ADDR_ALTTOC(r14, r11, __start___restart_table) + LOAD_REG_ADDR_ALTTOC(r15, r11, __stop___restart_table) #else LOAD_REG_IMMEDIATE_SYM(r14, r11, __start___restart_table) LOAD_REG_IMMEDIATE_SYM(r15, r11, __stop___restart_table) @@ -1056,15 +1039,14 @@ bad_stack_book3e: mfspr r11,SPRN_ESR std r10,_DEAR(r1) std r11,_ESR(r1) - std r0,GPR0(r1); /* save r0 in stackframe */ \ - std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_GPRS(3, 9, r1); /* save r3 - r9 in stackframe */ \ + SAVE_GPR(0, r1); /* save r0 in stackframe */ \ + SAVE_GPRS(2, 9, r1); /* save r2 - r9 in stackframe */ \ ld r3,PACA_EXGEN+EX_R10(r13);/* get back r10 */ \ ld r4,PACA_EXGEN+EX_R11(r13);/* get back r11 */ \ mfspr r5,SPRN_SPRG_GEN_SCRATCH;/* get back r13 XXX can be wrong */ \ std r3,GPR10(r1); /* save r10 to stackframe */ \ std r4,GPR11(r1); /* save r11 to stackframe */ \ - std r12,GPR12(r1); /* save r12 in stackframe */ \ + SAVE_GPR(12, r1); /* save r12 in stackframe */ \ std r5,GPR13(r1); /* save it to stackframe */ \ mflr r10 mfctr r11 @@ -1072,14 +1054,14 @@ bad_stack_book3e: std r10,_LINK(r1) std r11,_CTR(r1) std r12,_XER(r1) - SAVE_GPRS(14, 31, r1) + SAVE_NVGPRS(r1) lhz r12,PACA_TRAP_SAVE(r13) std r12,_TRAP(r1) addi r11,r1,INT_FRAME_SIZE std r11,0(r1) - li r12,0 + ZEROIZE_GPR(12) std r12,0(r11) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() 1: addi r3,r1,STACK_FRAME_OVERHEAD bl kernel_bad_stack b 1b @@ -1320,8 +1302,8 @@ a2_tlbinit_after_linear_map: /* Now we branch the new virtual address mapped by this entry */ #ifdef CONFIG_RELOCATABLE - ld r5,PACATOC(r13) - ld r3,1f@got(r5) + __LOAD_PACA_TOC(r5) + LOAD_REG_ADDR_ALTTOC(r3, r5, 1f) #else LOAD_REG_IMMEDIATE_SYM(r3, r5, 1f) #endif diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 3d0dc133a9ae10ca8496cc456fe8e0a1543e6df0..5381a43e50fef990c36b60c5a383713145d259a7 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -281,7 +281,7 @@ BEGIN_FTR_SECTION mfspr r9,SPRN_PPR END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) HMT_MEDIUM - std r10,IAREA+EX_R10(r13) /* save r10 - r12 */ + std r10,IAREA+EX_R10(r13) /* save r10 */ .if ICFAR BEGIN_FTR_SECTION mfspr r10,SPRN_CFAR @@ -321,7 +321,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) mfctr r10 std r10,IAREA+EX_CTR(r13) mfcr r9 - std r11,IAREA+EX_R11(r13) + std r11,IAREA+EX_R11(r13) /* save r11 - r12 */ std r12,IAREA+EX_R12(r13) /* @@ -580,7 +580,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) std r2,GPR2(r1) /* save r2 in stackframe */ SAVE_GPRS(3, 8, r1) /* save r3 - r8 in stackframe */ mflr r9 /* Get LR, later save to stack */ - ld r2,PACATOC(r13) /* get kernel TOC into r2 */ + LOAD_PACA_TOC() /* get kernel TOC into r2 */ std r9,_LINK(r1) lbz r10,PACAIRQSOFTMASK(r13) mfspr r11,SPRN_XER /* save XER in stackframe */ @@ -589,7 +589,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) li r9,IVEC std r9,_TRAP(r1) /* set trap number */ li r10,0 - ld r11,exception_marker@toc(r2) + LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) std r10,RESULT(r1) /* clear regs->result */ std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ .endm @@ -610,7 +610,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .macro SEARCH_RESTART_TABLE #ifdef CONFIG_RELOCATABLE mr r12,r2 - ld r2,PACATOC(r13) + LOAD_PACA_TOC() LOAD_REG_ADDR(r9, __start___restart_table) LOAD_REG_ADDR(r10, __stop___restart_table) mr r2,r12 @@ -640,7 +640,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) .macro SEARCH_SOFT_MASK_TABLE #ifdef CONFIG_RELOCATABLE mr r12,r2 - ld r2,PACATOC(r13) + LOAD_PACA_TOC() LOAD_REG_ADDR(r9, __start___soft_mask_table) LOAD_REG_ADDR(r10, __stop___soft_mask_table) mr r2,r12 @@ -702,6 +702,71 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r1,GPR1(r1) .endm +/* + * EARLY_BOOT_FIXUP - Fix real-mode interrupt with wrong endian in early boot. + * + * There's a short window during boot where although the kernel is running + * little endian, any exceptions will cause the CPU to switch back to big + * endian. For example a WARN() boils down to a trap instruction, which will + * cause a program check, and we end up here but with the CPU in big endian + * mode. The first instruction of the program check handler (in GEN_INT_ENTRY + * below) is an mtsprg, which when executed in the wrong endian is an lhzu with + * a ~3GB displacement from r3. The content of r3 is random, so that is a load + * from some random location, and depending on the system can easily lead to a + * checkstop, or an infinitely recursive page fault. + * + * So to handle that case we have a trampoline here that can detect we are in + * the wrong endian and flip us back to the correct endian. We can't flip + * MSR[LE] using mtmsr, so we have to use rfid. That requires backing up SRR0/1 + * as well as a GPR. To do that we use SPRG0/2/3, as SPRG1 is already used for + * the paca. SPRG3 is user readable, but this trampoline is only active very + * early in boot, and SPRG3 will be reinitialised in vdso_getcpu_init() before + * userspace starts. + */ +.macro EARLY_BOOT_FIXUP +BEGIN_FTR_SECTION +#ifdef CONFIG_CPU_LITTLE_ENDIAN + tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 + b 2f // Skip trampoline if endian is correct + .long 0xa643707d // mtsprg 0, r11 Backup r11 + .long 0xa6027a7d // mfsrr0 r11 + .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 + .long 0xa6027b7d // mfsrr1 r11 + .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 + .long 0xa600607d // mfmsr r11 + .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] + .long 0xa6037b7d // mtsrr1 r11 + /* + * This is 'li r11,1f' where 1f is the absolute address of that + * label, byteswapped into the SI field of the instruction. + */ + .long 0x00006039 | \ + ((ABS_ADDR(1f, real_vectors) & 0x00ff) << 24) | \ + ((ABS_ADDR(1f, real_vectors) & 0xff00) << 8) + .long 0xa6037a7d // mtsrr0 r11 + .long 0x2400004c // rfid +1: + mfsprg r11, 3 + mtsrr1 r11 // Restore SRR1 + mfsprg r11, 2 + mtsrr0 r11 // Restore SRR0 + mfsprg r11, 0 // Restore r11 +2: +#endif + /* + * program check could hit at any time, and pseries can not block + * MSR[ME] in early boot. So check if there is anything useful in r13 + * yet, and spin forever if not. + */ + mtsprg 0, r11 + mfcr r11 + cmpdi r13, 0 + beq . + mtcr r11 + mfsprg r11, 0 +END_FTR_SECTION(0, 1) // nop out after boot +.endm + /* * There are a few constraints to be concerned with. * - Real mode exceptions code/data must be located at their physical location. @@ -1079,6 +1144,7 @@ INT_DEFINE_BEGIN(machine_check) INT_DEFINE_END(machine_check) EXC_REAL_BEGIN(machine_check, 0x200, 0x100) + EARLY_BOOT_FIXUP GEN_INT_ENTRY machine_check_early, virt=0 EXC_REAL_END(machine_check, 0x200, 0x100) EXC_VIRT_NONE(0x4200, 0x100) @@ -1143,6 +1209,9 @@ BEGIN_FTR_SECTION bl enable_machine_check END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) addi r3,r1,STACK_FRAME_OVERHEAD +BEGIN_FTR_SECTION + bl machine_check_early_boot +END_FTR_SECTION(0, 1) // nop out after boot bl machine_check_early std r3,RESULT(r1) /* Save result */ ld r12,_MSR(r1) @@ -1619,51 +1688,7 @@ INT_DEFINE_BEGIN(program_check) INT_DEFINE_END(program_check) EXC_REAL_BEGIN(program_check, 0x700, 0x100) - -#ifdef CONFIG_CPU_LITTLE_ENDIAN - /* - * There's a short window during boot where although the kernel is - * running little endian, any exceptions will cause the CPU to switch - * back to big endian. For example a WARN() boils down to a trap - * instruction, which will cause a program check, and we end up here but - * with the CPU in big endian mode. The first instruction of the program - * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when - * executed in the wrong endian is an lhzu with a ~3GB displacement from - * r3. The content of r3 is random, so that is a load from some random - * location, and depending on the system can easily lead to a checkstop, - * or an infinitely recursive page fault. - * - * So to handle that case we have a trampoline here that can detect we - * are in the wrong endian and flip us back to the correct endian. We - * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires - * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as - * SPRG1 is already used for the paca. SPRG3 is user readable, but this - * trampoline is only active very early in boot, and SPRG3 will be - * reinitialised in vdso_getcpu_init() before userspace starts. - */ -BEGIN_FTR_SECTION - tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 - b 1f // Skip trampoline if endian is correct - .long 0xa643707d // mtsprg 0, r11 Backup r11 - .long 0xa6027a7d // mfsrr0 r11 - .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 - .long 0xa6027b7d // mfsrr1 r11 - .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 - .long 0xa600607d // mfmsr r11 - .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] - .long 0xa6037b7d // mtsrr1 r11 - .long 0x34076039 // li r11, 0x734 - .long 0xa6037a7d // mtsrr0 r11 - .long 0x2400004c // rfid - mfsprg r11, 3 - mtsrr1 r11 // Restore SRR1 - mfsprg r11, 2 - mtsrr0 r11 // Restore SRR0 - mfsprg r11, 0 // Restore r11 -1: -END_FTR_SECTION(0, 1) // nop out after boot -#endif /* CONFIG_CPU_LITTLE_ENDIAN */ - + EARLY_BOOT_FIXUP GEN_INT_ENTRY program_check, virt=0 EXC_REAL_END(program_check, 0x700, 0x100) EXC_VIRT_BEGIN(program_check, 0x4700, 0x100) @@ -2794,6 +2819,20 @@ masked_Hinterrupt: masked_interrupt: .endif stw r9,PACA_EXGEN+EX_CCR(r13) +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG + /* + * Ensure there was no previous MUST_HARD_MASK interrupt or + * HARD_DIS setting. If this does fire, the interrupt is still + * masked and MSR[EE] will be cleared on return, so no need to + * panic, but somebody probably enabled MSR[EE] under + * PACA_IRQ_HARD_DIS, mtmsr(mfmsr() | MSR_x) being a common + * cause. + */ + lbz r9,PACAIRQHAPPENED(r13) + andi. r9,r9,(PACA_IRQ_MUST_HARD_MASK|PACA_IRQ_HARD_DIS) +0: tdnei r9,0 + EMIT_WARN_ENTRY 0b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE) +#endif lbz r9,PACAIRQHAPPENED(r13) or r9,r9,r10 stb r9,PACAIRQHAPPENED(r13) @@ -3034,22 +3073,6 @@ EXPORT_SYMBOL(do_uaccess_flush) MASKED_INTERRUPT MASKED_INTERRUPT hsrr=1 - /* - * Relocation-on interrupts: A subset of the interrupts can be delivered - * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering - * it. Addresses are the same as the original interrupt addresses, but - * offset by 0xc000000000004000. - * It's impossible to receive interrupts below 0x300 via this mechanism. - * KVM: None of these traps are from the guest ; anything that escalated - * to HV=1 from HV=0 is delivered via real mode handlers. - */ - - /* - * This uses the standard macro, since the original 0x300 vector - * only has extra guff for STAB-based processors -- which never - * come here. - */ - USE_FIXED_SECTION(virt_trampolines) /* * All code below __end_soft_masked is treated as soft-masked. If @@ -3075,7 +3098,7 @@ CLOSE_FIXED_SECTION(virt_trampolines); USE_TEXT_SECTION() /* MSR[RI] should be clear because this uses SRR[01] */ -enable_machine_check: +_GLOBAL(enable_machine_check) mflr r0 bcl 20,31,$+4 0: mflr r3 diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index cf2c08902c0569da7c075c6ef386088b98d14125..dedcc6fe2263a1d75159a12f5cd2f17ea384be15 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -143,7 +143,7 @@ DEFINE_FIXED_SYMBOL(__run_at_load, first_256B) .globl __secondary_hold __secondary_hold: FIXUP_ENDIAN -#ifndef CONFIG_PPC_BOOK3E +#ifndef CONFIG_PPC_BOOK3E_64 mfmsr r24 ori r24,r24,MSR_RI mtmsrd r24 /* RI on */ @@ -160,7 +160,7 @@ __secondary_hold: sync li r26,0 -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 tovirt(r26,r26) #endif /* All secondary cpus wait here until told to start. */ @@ -169,7 +169,7 @@ __secondary_hold: beq 100b #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE) -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 tovirt(r12,r12) #endif mtctr r12 @@ -178,7 +178,7 @@ __secondary_hold: * it may be the case that other platforms have r4 right to * begin with, this gives us some safety in case it is not */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 mr r4,r25 #else li r4,0 @@ -192,13 +192,6 @@ __secondary_hold: #endif CLOSE_FIXED_SECTION(first_256B) -/* This value is used to mark exception frames on the stack. */ - .section ".toc","aw" -/* This value is used to mark exception frames on the stack. */ -exception_marker: - .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER - .previous - /* * On server, we include the exception vectors code here as it * relies on absolute addressing which is only possible within @@ -214,7 +207,7 @@ USE_TEXT_SECTION() #include "interrupt_64.S" -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* * The booting_thread_hwid holds the thread id we want to boot in cpu * hotplug case. It is set by cpu hotplug code, and is invalid by default. @@ -322,7 +315,7 @@ _GLOBAL(fsl_secondary_thread_init) bl book3e_secondary_thread_init b generic_secondary_common_init -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ /* * On pSeries and most other platforms, secondary processors spin @@ -345,7 +338,7 @@ _GLOBAL(generic_secondary_smp_init) bl relative_toc tovirt(r2,r2) -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* Book3E initialization */ mr r3,r24 mr r4,r25 @@ -400,8 +393,12 @@ generic_secondary_common_init: #else LOAD_REG_ADDR(r8, paca_ptrs) /* Load paca_ptrs pointe */ ld r8,0(r8) /* Get base vaddr of array */ +#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) + LOAD_REG_IMMEDIATE(r7, NR_CPUS) +#else LOAD_REG_ADDR(r7, nr_cpu_ids) /* Load nr_cpu_ids address */ lwz r7,0(r7) /* also the max paca allocated */ +#endif li r5,0 /* logical cpu id */ 1: sldi r9,r5,3 /* get paca_ptrs[] index from cpu id */ @@ -417,7 +414,7 @@ generic_secondary_common_init: b kexec_wait /* next kernel might do better */ 2: SET_PACA(r13) -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 addi r12,r13,PACA_EXTLB /* and TLB exc frame in another */ mtspr SPRN_SPRG_TLB_EXFRAME,r12 #endif @@ -494,6 +491,9 @@ __start_initialization_multiplatform: /* Make sure we are running in 64 bits mode */ bl enable_64b_mode + /* Zero r13 (paca) so early program check / mce don't use it */ + li r13,0 + /* Get TOC pointer (current runtime address) */ bl relative_toc @@ -519,7 +519,7 @@ __start_initialization_multiplatform: mr r29,r9 #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 bl start_initialization_book3e b __after_prom_start #else @@ -540,7 +540,7 @@ __start_initialization_multiplatform: /* Switch off MMU if not already off */ bl __mmu_off b __after_prom_start -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ __REF __boot_from_prom: @@ -587,11 +587,11 @@ __after_prom_start: /* process relocations for the final address of the kernel */ lis r25,PAGE_OFFSET@highest /* compute virtual base of kernel */ sldi r25,r25,32 -#if defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3E_64) tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */ #endif lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26) -#if defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3E_64) tophys(r26,r26) #endif cmplwi cr0,r7,1 /* flagged to stay where we are ? */ @@ -599,7 +599,7 @@ __after_prom_start: add r25,r25,r26 1: mr r3,r25 bl relocate -#if defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3E_64) /* IVPR needs to be set after relocation. */ bl init_core_book3e #endif @@ -613,11 +613,11 @@ __after_prom_start: * Note: This process overwrites the OF exception vectors. */ li r3,0 /* target addr */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 tovirt(r3,r3) /* on booke, we already run at PAGE_OFFSET */ #endif mr. r4,r26 /* In some cases the loader may */ -#if defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3E_64) tovirt(r4,r4) #endif beq 9f /* have already put us at zero */ @@ -630,14 +630,14 @@ __after_prom_start: * variable __run_at_load, if it is set the kernel is treated as relocatable * kernel, otherwise it will be moved to PHYSICAL_START */ -#if defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3E_64) tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */ #endif lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26) cmplwi cr0,r7,1 bne 3f -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 LOAD_REG_ADDR(r5, __end_interrupts) LOAD_REG_ADDR(r11, _stext) sub r5,r5,r11 @@ -848,7 +848,7 @@ __secondary_start: * before going into C code. */ start_secondary_prolog: - ld r2,PACATOC(r13) + LOAD_PACA_TOC() li r3,0 std r3,0(r1) /* Zero the stack frame pointer */ bl start_secondary @@ -871,10 +871,10 @@ _GLOBAL(start_secondary_resume) */ enable_64b_mode: mfmsr r11 /* grab the current MSR */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 oris r11,r11,0x8000 /* CM bit set, we'll set ICM later */ mtmsr r11 -#else /* CONFIG_PPC_BOOK3E */ +#else /* CONFIG_PPC_BOOK3E_64 */ LOAD_REG_IMMEDIATE(r12, MSR_64BIT) or r11,r11,r12 mtmsrd r11 @@ -940,7 +940,7 @@ start_here_multiplatform: std r29,8(r11); #endif -#ifndef CONFIG_PPC_BOOK3E +#ifndef CONFIG_PPC_BOOK3E_64 mfmsr r6 ori r6,r6,MSR_RI mtmsrd r6 /* RI on */ @@ -988,7 +988,7 @@ start_here_common: std r1,PACAKSAVE(r13) /* Load the TOC (virtual address) */ - ld r2,PACATOC(r13) + LOAD_PACA_TOC() /* Mark interrupts soft and hard disabled (they might be enabled * in the PACA when doing hotplug) diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_85xx.S similarity index 99% rename from arch/powerpc/kernel/head_fsl_booke.S rename to arch/powerpc/kernel/head_85xx.S index f0db4f52bc001d87d1a4304781c4160e0271b5b2..52c0ab416326a8ee543b91292f9acf52d4801c7d 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_85xx.S @@ -129,7 +129,7 @@ _GLOBAL(_start); /* * For the second relocation, we already set the right tlb entries - * for the kernel space, so skip the code in fsl_booke_entry_mapping.S + * for the kernel space, so skip the code in 85xx_entry_mapping.S */ cmpwi r19,1 beq set_ivor @@ -159,7 +159,7 @@ _GLOBAL(__early_start) lwz r20,0(r20) #define ENTRY_MAPPING_BOOT_SETUP -#include "fsl_booke_entry_mapping.S" +#include "85xx_entry_mapping.S" #undef ENTRY_MAPPING_BOOT_SETUP set_ivor: @@ -912,7 +912,7 @@ get_phys_addr: * Global functions */ -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 #ifndef CONFIG_PPC_E500MC /* Adjust or setup IVORs for e500v1/v2 */ _GLOBAL(__setup_e500_ivors) @@ -955,7 +955,7 @@ _GLOBAL(__setup_ehv_ivors) sync blr #endif /* CONFIG_PPC_E500MC */ -#endif /* CONFIG_E500 */ +#endif /* CONFIG_PPC_E500 */ #ifdef CONFIG_SPE /* diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h index bb6d5d0fc4ac8eb9c45ca795738344150f878868..1cb9d0f7cbf257962799cb4201512e2134adda0a 100644 --- a/arch/powerpc/kernel/head_booke.h +++ b/arch/powerpc/kernel/head_booke.h @@ -34,7 +34,7 @@ */ #define THREAD_NORMSAVE(offset) (THREAD_NORMSAVES + (offset * 4)) -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 #define BOOKE_CLEAR_BTB(reg) \ START_BTB_FLUSH_SECTION \ BTB_FLUSH(reg) \ @@ -103,7 +103,7 @@ END_BTB_FLUSH_SECTION .endm .macro prepare_transfer_to_handler -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 andi. r12,r9,MSR_PR bne 777f bl prepare_transfer_to_handler @@ -242,7 +242,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) .macro SAVE_MMU_REGS -#ifdef CONFIG_PPC_BOOK3E_MMU +#ifdef CONFIG_PPC_E500 mfspr r0,SPRN_MAS0 stw r0,MAS0(r1) mfspr r0,SPRN_MAS1 @@ -257,7 +257,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) mfspr r0,SPRN_MAS7 stw r0,MAS7(r1) #endif /* CONFIG_PHYS_64BIT */ -#endif /* CONFIG_PPC_BOOK3E_MMU */ +#endif /* CONFIG_PPC_E500 */ #ifdef CONFIG_44x mfspr r0,SPRN_MMUCR stw r0,MMUCR(r1) diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index 2669f80b3a4973fafb3b9104dfc47c42366f0f82..8db1a15d7acbe17815f2a843242b5ea5339fbcf1 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -129,7 +130,14 @@ struct breakpoint { bool ptrace_bp; }; +/* + * While kernel/events/hw_breakpoint.c does its own synchronization, we cannot + * rely on it safely synchronizing internals here; however, we can rely on it + * not requesting more breakpoints than available. + */ +static DEFINE_SPINLOCK(cpu_bps_lock); static DEFINE_PER_CPU(struct breakpoint *, cpu_bps[HBP_NUM_MAX]); +static DEFINE_SPINLOCK(task_bps_lock); static LIST_HEAD(task_bps); static struct breakpoint *alloc_breakpoint(struct perf_event *bp) @@ -174,7 +182,9 @@ static int task_bps_add(struct perf_event *bp) if (IS_ERR(tmp)) return PTR_ERR(tmp); + spin_lock(&task_bps_lock); list_add(&tmp->list, &task_bps); + spin_unlock(&task_bps_lock); return 0; } @@ -182,6 +192,7 @@ static void task_bps_remove(struct perf_event *bp) { struct list_head *pos, *q; + spin_lock(&task_bps_lock); list_for_each_safe(pos, q, &task_bps) { struct breakpoint *tmp = list_entry(pos, struct breakpoint, list); @@ -191,6 +202,7 @@ static void task_bps_remove(struct perf_event *bp) break; } } + spin_unlock(&task_bps_lock); } /* @@ -200,12 +212,17 @@ static void task_bps_remove(struct perf_event *bp) static bool all_task_bps_check(struct perf_event *bp) { struct breakpoint *tmp; + bool ret = false; + spin_lock(&task_bps_lock); list_for_each_entry(tmp, &task_bps, list) { - if (!can_co_exist(tmp, bp)) - return true; + if (!can_co_exist(tmp, bp)) { + ret = true; + break; + } } - return false; + spin_unlock(&task_bps_lock); + return ret; } /* @@ -215,13 +232,18 @@ static bool all_task_bps_check(struct perf_event *bp) static bool same_task_bps_check(struct perf_event *bp) { struct breakpoint *tmp; + bool ret = false; + spin_lock(&task_bps_lock); list_for_each_entry(tmp, &task_bps, list) { if (tmp->bp->hw.target == bp->hw.target && - !can_co_exist(tmp, bp)) - return true; + !can_co_exist(tmp, bp)) { + ret = true; + break; + } } - return false; + spin_unlock(&task_bps_lock); + return ret; } static int cpu_bps_add(struct perf_event *bp) @@ -234,6 +256,7 @@ static int cpu_bps_add(struct perf_event *bp) if (IS_ERR(tmp)) return PTR_ERR(tmp); + spin_lock(&cpu_bps_lock); cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu); for (i = 0; i < nr_wp_slots(); i++) { if (!cpu_bp[i]) { @@ -241,6 +264,7 @@ static int cpu_bps_add(struct perf_event *bp) break; } } + spin_unlock(&cpu_bps_lock); return 0; } @@ -249,6 +273,7 @@ static void cpu_bps_remove(struct perf_event *bp) struct breakpoint **cpu_bp; int i = 0; + spin_lock(&cpu_bps_lock); cpu_bp = per_cpu_ptr(cpu_bps, bp->cpu); for (i = 0; i < nr_wp_slots(); i++) { if (!cpu_bp[i]) @@ -260,19 +285,25 @@ static void cpu_bps_remove(struct perf_event *bp) break; } } + spin_unlock(&cpu_bps_lock); } static bool cpu_bps_check(int cpu, struct perf_event *bp) { struct breakpoint **cpu_bp; + bool ret = false; int i; + spin_lock(&cpu_bps_lock); cpu_bp = per_cpu_ptr(cpu_bps, cpu); for (i = 0; i < nr_wp_slots(); i++) { - if (cpu_bp[i] && !can_co_exist(cpu_bp[i], bp)) - return true; + if (cpu_bp[i] && !can_co_exist(cpu_bp[i], bp)) { + ret = true; + break; + } } - return false; + spin_unlock(&cpu_bps_lock); + return ret; } static bool all_cpu_bps_check(struct perf_event *bp) @@ -286,10 +317,6 @@ static bool all_cpu_bps_check(struct perf_event *bp) return false; } -/* - * We don't use any locks to serialize accesses to cpu_bps or task_bps - * because are already inside nr_bp_mutex. - */ int arch_reserve_bp_slot(struct perf_event *bp) { int ret; diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_64e.S similarity index 93% rename from arch/powerpc/kernel/idle_book3e.S rename to arch/powerpc/kernel/idle_64e.S index cc008de58b050925f81a2c05407f87e1bd2f3185..0fc680e03deea9f44217a2fc7c3cf064af24030e 100644 --- a/arch/powerpc/kernel/idle_book3e.S +++ b/arch/powerpc/kernel/idle_64e.S @@ -2,7 +2,7 @@ /* * Copyright 2010 IBM Corp, Benjamin Herrenschmidt * - * Generic idle routine for Book3E processors + * Generic idle routine for 64 bits e500 processors */ #include @@ -16,8 +16,6 @@ #include /* 64-bit version only for now */ -#ifdef CONFIG_PPC64 - .macro BOOK3E_IDLE name loop _GLOBAL(\name) /* Save LR for later */ @@ -77,7 +75,7 @@ _GLOBAL(\name) .macro BOOK3E_IDLE_LOOP 1: - PPC_WAIT(0) + PPC_WAIT_v203 b 1b .endm @@ -98,6 +96,4 @@ epapr_ev_idle_start: BOOK3E_IDLE epapr_ev_idle EPAPR_EV_IDLE_LOOP -BOOK3E_IDLE book3e_idle BOOK3E_IDLE_LOOP - -#endif /* CONFIG_PPC64 */ +BOOK3E_IDLE e500_idle BOOK3E_IDLE_LOOP diff --git a/arch/powerpc/kernel/idle_e500.S b/arch/powerpc/kernel/idle_85xx.S similarity index 100% rename from arch/powerpc/kernel/idle_e500.S rename to arch/powerpc/kernel/idle_85xx.S diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 0e75cb03244adefa0918b9c590e43ac8700ddf2c..f9db0a172401a5e8fd6752fb858179f0acf6a166 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -431,16 +431,6 @@ again: if (unlikely(stack_store)) __hard_EE_RI_disable(); - /* - * Returning to a kernel context with local irqs disabled. - * Here, if EE was enabled in the interrupted context, enable - * it on return as well. A problem exists here where a soft - * masked interrupt may have cleared MSR[EE] and set HARD_DIS - * here, and it will still exist on return to the caller. This - * will be resolved by the masked interrupt firing again. - */ - if (regs->msr & MSR_EE) - local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; #endif /* CONFIG_PPC64 */ } diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S index ce25b28cf418ef6154fd46fad165c21c62c90b7b..978a173eb33964c60c63ff766477b540eb783900 100644 --- a/arch/powerpc/kernel/interrupt_64.S +++ b/arch/powerpc/kernel/interrupt_64.S @@ -13,16 +13,6 @@ #include #include - .section ".toc","aw" -SYS_CALL_TABLE: - .tc sys_call_table[TC],sys_call_table - -#ifdef CONFIG_COMPAT -COMPAT_SYS_CALL_TABLE: - .tc compat_sys_call_table[TC],compat_sys_call_table -#endif - .previous - .align 7 .macro DEBUG_SRR_VALID srr @@ -67,16 +57,11 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_\name) std r0,GPR0(r1) std r10,GPR1(r1) std r2,GPR2(r1) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() mfcr r12 li r11,0 - /* Can we avoid saving r3-r8 in common case? */ - std r3,GPR3(r1) - std r4,GPR4(r1) - std r5,GPR5(r1) - std r6,GPR6(r1) - std r7,GPR7(r1) - std r8,GPR8(r1) + /* Save syscall parameters in r3-r8 */ + SAVE_GPRS(3, 8, r1) /* Zero r9-r12, this should only be required when restoring all GPRs */ std r11,GPR9(r1) std r11,GPR10(r1) @@ -91,9 +76,12 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_\name) li r11,\trapnr std r11,_TRAP(r1) std r12,_CCR(r1) - addi r10,r1,STACK_FRAME_OVERHEAD - ld r11,exception_marker@toc(r2) - std r11,-16(r10) /* "regshere" marker */ + std r3,ORIG_GPR3(r1) + /* Calling convention has r3 = regs, r4 = orig r0 */ + addi r3,r1,STACK_FRAME_OVERHEAD + mr r4,r0 + LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) + std r11,-16(r3) /* "regshere" marker */ BEGIN_FTR_SECTION HMT_MEDIUM @@ -108,8 +96,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) * but this is the best we can do. */ - /* Calling convention has r9 = orig r0, r10 = regs */ - mr r9,r0 bl system_call_exception .Lsyscall_vectored_\name\()_exit: @@ -148,17 +134,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) /* Could zero these as per ABI, but we may consider a stricter ABI * which preserves these if libc implementations can benefit, so * restore them for now until further measurement is done. */ - ld r0,GPR0(r1) - ld r4,GPR4(r1) - ld r5,GPR5(r1) - ld r6,GPR6(r1) - ld r7,GPR7(r1) - ld r8,GPR8(r1) + REST_GPR(0, r1) + REST_GPRS(4, 8, r1) /* Zero volatile regs that may contain sensitive kernel data */ - li r9,0 - li r10,0 - li r11,0 - li r12,0 + ZEROIZE_GPRS(9, 12) mtspr SPRN_XER,r0 /* @@ -181,7 +160,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) ld r5,_XER(r1) REST_NVGPRS(r1) - ld r0,GPR0(r1) + REST_GPR(0, r1) mtcr r2 mtctr r3 mtlr r4 @@ -195,7 +174,7 @@ syscall_vectored_\name\()_restart: _ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart) GET_PACA(r13) ld r1,PACA_EXIT_SAVE_R1(r13) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() ld r3,RESULT(r1) addi r4,r1,STACK_FRAME_OVERHEAD li r11,IRQS_ALL_DISABLED @@ -240,21 +219,16 @@ _ASM_NOKPROBE_SYMBOL(system_call_common) std r0,GPR0(r1) std r10,GPR1(r1) std r2,GPR2(r1) -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 START_BTB_FLUSH_SECTION BTB_FLUSH(r10) END_BTB_FLUSH_SECTION #endif - ld r2,PACATOC(r13) + LOAD_PACA_TOC() mfcr r12 li r11,0 - /* Can we avoid saving r3-r8 in common case? */ - std r3,GPR3(r1) - std r4,GPR4(r1) - std r5,GPR5(r1) - std r6,GPR6(r1) - std r7,GPR7(r1) - std r8,GPR8(r1) + /* Save syscall parameters in r3-r8 */ + SAVE_GPRS(3, 8, r1) /* Zero r9-r12, this should only be required when restoring all GPRs */ std r11,GPR9(r1) std r11,GPR10(r1) @@ -275,9 +249,12 @@ END_BTB_FLUSH_SECTION std r10,_LINK(r1) std r11,_TRAP(r1) std r12,_CCR(r1) - addi r10,r1,STACK_FRAME_OVERHEAD - ld r11,exception_marker@toc(r2) - std r11,-16(r10) /* "regshere" marker */ + std r3,ORIG_GPR3(r1) + /* Calling convention has r3 = regs, r4 = orig r0 */ + addi r3,r1,STACK_FRAME_OVERHEAD + mr r4,r0 + LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER) + std r11,-16(r3) /* "regshere" marker */ #ifdef CONFIG_PPC_BOOK3S li r11,1 @@ -298,8 +275,6 @@ END_BTB_FLUSH_SECTION wrteei 1 #endif - /* Calling convention has r9 = orig r0, r10 = regs */ - mr r9,r0 bl system_call_exception .Lsyscall_exit: @@ -343,16 +318,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) cmpdi r3,0 bne .Lsyscall_restore_regs /* Zero volatile regs that may contain sensitive kernel data */ - li r0,0 - li r4,0 - li r5,0 - li r6,0 - li r7,0 - li r8,0 - li r9,0 - li r10,0 - li r11,0 - li r12,0 + ZEROIZE_GPR(0) + ZEROIZE_GPRS(4, 12) mtctr r0 mtspr SPRN_XER,r0 .Lsyscall_restore_regs_cont: @@ -378,7 +345,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) REST_NVGPRS(r1) mtctr r3 mtspr SPRN_XER,r4 - ld r0,GPR0(r1) + REST_GPR(0, r1) REST_GPRS(4, 12, r1) b .Lsyscall_restore_regs_cont .Lsyscall_rst_end: @@ -388,7 +355,7 @@ syscall_restart: _ASM_NOKPROBE_SYMBOL(syscall_restart) GET_PACA(r13) ld r1,PACA_EXIT_SAVE_R1(r13) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() ld r3,RESULT(r1) addi r4,r1,STACK_FRAME_OVERHEAD li r11,IRQS_ALL_DISABLED @@ -535,7 +502,7 @@ interrupt_return_\srr\()_user_restart: _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart) GET_PACA(r13) ld r1,PACA_EXIT_SAVE_R1(r13) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() addi r3,r1,STACK_FRAME_OVERHEAD li r11,IRQS_ALL_DISABLED stb r11,PACAIRQSOFTMASK(r13) @@ -559,15 +526,54 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel) ld r11,SOFTE(r1) cmpwi r11,IRQS_ENABLED stb r11,PACAIRQSOFTMASK(r13) - bne 1f + beq .Linterrupt_return_\srr\()_soft_enabled + + /* + * Returning to soft-disabled context. + * Check if a MUST_HARD_MASK interrupt has become pending, in which + * case we need to disable MSR[EE] in the return context. + */ + ld r12,_MSR(r1) + andi. r10,r12,MSR_EE + beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled + lbz r11,PACAIRQHAPPENED(r13) + andi. r10,r11,PACA_IRQ_MUST_HARD_MASK + beq .Lfast_kernel_interrupt_return_\srr\() // No HARD_MASK pending + + /* Must clear MSR_EE from _MSR */ +#ifdef CONFIG_PPC_BOOK3S + li r10,0 + /* Clear valid before changing _MSR */ + .ifc \srr,srr + stb r10,PACASRR_VALID(r13) + .else + stb r10,PACAHSRR_VALID(r13) + .endif +#endif + xori r12,r12,MSR_EE + std r12,_MSR(r1) + b .Lfast_kernel_interrupt_return_\srr\() + +.Linterrupt_return_\srr\()_soft_enabled: + /* + * In the soft-enabled case, need to double-check that we have no + * pending interrupts that might have come in before we reached the + * restart section of code, and restart the exit so those can be + * handled. + * + * If there are none, it is be possible that the interrupt still + * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the + * interrupted context. This clear will not clobber a new pending + * interrupt coming in, because we're in the restart section, so + * such would return to the restart location. + */ #ifdef CONFIG_PPC_BOOK3S lbz r11,PACAIRQHAPPENED(r13) andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l bne- interrupt_return_\srr\()_kernel_restart #endif li r11,0 - stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS -1: + stb r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS .Lfast_kernel_interrupt_return_\srr\(): cmpdi cr1,r3,0 @@ -619,7 +625,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS) mtspr SPRN_XER,r5 /* - * Leaving a stale exception_marker on the stack can confuse + * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse * the reliable stack unwinder later on. Clear it. */ std r0,STACK_FRAME_OVERHEAD-16(r1) @@ -668,7 +674,7 @@ interrupt_return_\srr\()_kernel_restart: _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart) GET_PACA(r13) ld r1,PACA_EXIT_SAVE_R1(r13) - ld r2,PACATOC(r13) + LOAD_PACA_TOC() addi r3,r1,STACK_FRAME_OVERHEAD li r11,IRQS_ALL_DISABLED stb r11,PACAIRQSOFTMASK(r13) diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 0f17268c1f0bbdcb6126eb2108cc0a7c30fbbbaa..9ede61a5a469efb7f4252412c2a5c98c5f77c48b 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -199,7 +199,7 @@ static inline void check_stack_overflow(unsigned long sp) } } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK static __always_inline void call_do_softirq(const void *sp) { /* Temporarily switch r1 to sp, call __do_softirq() then restore r1. */ @@ -335,7 +335,7 @@ void *mcheckirq_ctx[NR_CPUS] __read_mostly; void *softirq_ctx[NR_CPUS] __read_mostly; void *hardirq_ctx[NR_CPUS] __read_mostly; -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void) { call_do_softirq(softirq_ctx[smp_processor_id()]); diff --git a/arch/powerpc/kernel/irq_64.c b/arch/powerpc/kernel/irq_64.c index 01645e03e9f044ccbf8e617ce5c071e6da3e9efe..eb2b380e52a0d0416139a0cafeca14b2d2bb0c79 100644 --- a/arch/powerpc/kernel/irq_64.c +++ b/arch/powerpc/kernel/irq_64.c @@ -68,6 +68,35 @@ int distribute_irqs = 1; +static inline void next_interrupt(struct pt_regs *regs) +{ + /* + * Softirq processing can enable/disable irqs, which will leave + * MSR[EE] enabled and the soft mask set to IRQS_DISABLED. Fix + * this up. + */ + if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) + hard_irq_disable(); + else + irq_soft_mask_set(IRQS_ALL_DISABLED); + + /* + * We are responding to the next interrupt, so interrupt-off + * latencies should be reset here. + */ + trace_hardirqs_on(); + trace_hardirqs_off(); +} + +static inline bool irq_happened_test_and_clear(u8 irq) +{ + if (local_paca->irq_happened & irq) { + local_paca->irq_happened &= ~irq; + return true; + } + return false; +} + void replay_soft_interrupts(void) { struct pt_regs regs; @@ -79,18 +108,25 @@ void replay_soft_interrupts(void) * recurse into this function. Don't keep any state across * interrupt handler calls which may change underneath us. * + * Softirqs can not be disabled over replay to stop this recursion + * because interrupts taken in idle code may require RCU softirq + * to run in the irq RCU tracking context. This is a hard problem + * to fix without changes to the softirq or idle layer. + * * We use local_paca rather than get_paca() to avoid all the * debug_smp_processor_id() business in this low level function. */ + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) { + WARN_ON_ONCE(mfmsr() & MSR_EE); + WARN_ON(!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)); + } + ppc_save_regs(®s); regs.softe = IRQS_ENABLED; regs.msr |= MSR_EE; again: - if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) - WARN_ON_ONCE(mfmsr() & MSR_EE); - /* * Force the delivery of pending soft-disabled interrupts on PS3. * Any HV call will have this side effect. @@ -105,56 +141,47 @@ again: * This is a higher priority interrupt than the others, so * replay it first. */ - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) { - local_paca->irq_happened &= ~PACA_IRQ_HMI; + if (IS_ENABLED(CONFIG_PPC_BOOK3S) && + irq_happened_test_and_clear(PACA_IRQ_HMI)) { regs.trap = INTERRUPT_HMI; handle_hmi_exception(®s); - if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) - hard_irq_disable(); + next_interrupt(®s); } - if (local_paca->irq_happened & PACA_IRQ_DEC) { - local_paca->irq_happened &= ~PACA_IRQ_DEC; + if (irq_happened_test_and_clear(PACA_IRQ_DEC)) { regs.trap = INTERRUPT_DECREMENTER; timer_interrupt(®s); - if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) - hard_irq_disable(); + next_interrupt(®s); } - if (local_paca->irq_happened & PACA_IRQ_EE) { - local_paca->irq_happened &= ~PACA_IRQ_EE; + if (irq_happened_test_and_clear(PACA_IRQ_EE)) { regs.trap = INTERRUPT_EXTERNAL; do_IRQ(®s); - if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) - hard_irq_disable(); + next_interrupt(®s); } - if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) { - local_paca->irq_happened &= ~PACA_IRQ_DBELL; + if (IS_ENABLED(CONFIG_PPC_DOORBELL) && + irq_happened_test_and_clear(PACA_IRQ_DBELL)) { regs.trap = INTERRUPT_DOORBELL; doorbell_exception(®s); - if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) - hard_irq_disable(); + next_interrupt(®s); } /* Book3E does not support soft-masking PMI interrupts */ - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) { - local_paca->irq_happened &= ~PACA_IRQ_PMI; + if (IS_ENABLED(CONFIG_PPC_BOOK3S) && + irq_happened_test_and_clear(PACA_IRQ_PMI)) { regs.trap = INTERRUPT_PERFMON; performance_monitor_exception(®s); - if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) - hard_irq_disable(); + next_interrupt(®s); } - if (local_paca->irq_happened & ~PACA_IRQ_HARD_DIS) { - /* - * We are responding to the next interrupt, so interrupt-off - * latencies should be reset here. - */ - trace_hardirqs_on(); - trace_hardirqs_off(); + /* + * Softirq processing can enable and disable interrupts, which can + * result in new irqs becoming pending. Must keep looping until we + * have cleared out all pending interrupts. + */ + if (local_paca->irq_happened & ~PACA_IRQ_HARD_DIS) goto again; - } } #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP) @@ -270,10 +297,12 @@ happened: trace_hardirqs_off(); replay_soft_interrupts_irqrestore(); - local_paca->irq_happened = 0; trace_hardirqs_on(); irq_soft_mask_set(IRQS_ENABLED); + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) + WARN_ON(local_paca->irq_happened != PACA_IRQ_HARD_DIS); + local_paca->irq_happened = 0; __hard_irq_enable(); preempt_enable(); } diff --git a/arch/powerpc/kernel/kgdb.c b/arch/powerpc/kernel/kgdb.c index a20deebf233fd0f5a9182998ac85aaa211bb870d..1a1e9995dae35b5745a9e0ae7ed2afef07417ada 100644 --- a/arch/powerpc/kernel/kgdb.c +++ b/arch/powerpc/kernel/kgdb.c @@ -47,7 +47,7 @@ static struct hard_trap_info { 0x0c00, 0x14 /* SIGCHLD */ }, /* system call */ #ifdef CONFIG_BOOKE_OR_40x { 0x2002, 0x05 /* SIGTRAP */ }, /* debug */ -#if defined(CONFIG_FSL_BOOKE) +#if defined(CONFIG_PPC_85xx) { 0x2010, 0x08 /* SIGFPE */ }, /* spe unavailable */ { 0x2020, 0x08 /* SIGFPE */ }, /* spe unavailable */ { 0x2030, 0x08 /* SIGFPE */ }, /* spe fp data */ @@ -57,7 +57,7 @@ static struct hard_trap_info { 0x2900, 0x08 /* SIGFPE */ }, /* apu unavailable */ { 0x3100, 0x0e /* SIGALRM */ }, /* fixed interval timer */ { 0x3200, 0x02 /* SIGINT */ }, /* watchdog */ -#else /* ! CONFIG_FSL_BOOKE */ +#else /* ! CONFIG_PPC_85xx */ { 0x1000, 0x0e /* SIGALRM */ }, /* prog interval timer */ { 0x1010, 0x0e /* SIGALRM */ }, /* fixed interval timer */ { 0x1020, 0x02 /* SIGINT */ }, /* watchdog */ @@ -208,7 +208,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) for (reg = 14; reg < 32; reg++) PACK64(ptr, regs->gpr[reg]); -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx #ifdef CONFIG_SPE for (reg = 0; reg < 32; reg++) PACK64(ptr, p->thread.evr[reg]); @@ -234,7 +234,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) #define GDB_SIZEOF_REG sizeof(unsigned long) #define GDB_SIZEOF_REG_U32 sizeof(u32) -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx #define GDB_SIZEOF_FLOAT_REG sizeof(unsigned long) #else #define GDB_SIZEOF_FLOAT_REG sizeof(u64) @@ -329,7 +329,7 @@ char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) if (regno >= 32 && regno < 64) { /* FP registers 32 -> 63 */ -#if defined(CONFIG_FSL_BOOKE) && defined(CONFIG_SPE) +#if defined(CONFIG_PPC_85xx) && defined(CONFIG_SPE) if (current) memcpy(mem, ¤t->thread.evr[regno-32], dbg_reg_def[regno].size); @@ -355,7 +355,7 @@ int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) if (regno >= 32 && regno < 64) { /* FP registers 32 -> 63 */ -#if defined(CONFIG_FSL_BOOKE) && defined(CONFIG_SPE) +#if defined(CONFIG_PPC_85xx) && defined(CONFIG_SPE) memcpy(¤t->thread.evr[regno-32], mem, dbg_reg_def[regno].size); #else diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index 912d4f8a13beb2b15257d9cc60ccf76fe6b90f84..bd7b1a035459488d12c3d5c615e2569524a3a7b3 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -161,7 +161,13 @@ int arch_prepare_kprobe(struct kprobe *p) preempt_disable(); prev = get_kprobe(p->addr - 1); preempt_enable_no_resched(); - if (prev && ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { + + /* + * When prev is a ftrace-based kprobe, we don't have an insn, and it + * doesn't probe for prefixed instruction. + */ + if (prev && !kprobe_ftrace(prev) && + ppc_inst_prefixed(ppc_inst_read(prev->ainsn.insn))) { printk("Cannot register a kprobe on the second word of prefixed instruction\n"); ret = -EINVAL; } diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c index 6568823cf3063086f02f050d4e7a4f66a4a672c6..5b3c093611baf142b679315fe61c2e2c1459a4ee 100644 --- a/arch/powerpc/kernel/kvm.c +++ b/arch/powerpc/kernel/kvm.c @@ -455,7 +455,7 @@ static void __init kvm_check_ins(u32 *inst, u32 features) kvm_patch_ins_lwz(inst, magic_var(dsisr), inst_rt); break; -#ifdef CONFIG_PPC_BOOK3E_MMU +#ifdef CONFIG_PPC_E500 case KVM_INST_MFSPR(SPRN_MAS0): if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) kvm_patch_ins_lwz(inst, magic_var(mas0), inst_rt); @@ -484,7 +484,7 @@ static void __init kvm_check_ins(u32 *inst, u32 features) if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) kvm_patch_ins_lwz(inst, magic_var(mas7_3), inst_rt); break; -#endif /* CONFIG_PPC_BOOK3E_MMU */ +#endif /* CONFIG_PPC_E500 */ case KVM_INST_MFSPR(SPRN_SPRG4): #ifdef CONFIG_BOOKE @@ -557,7 +557,7 @@ static void __init kvm_check_ins(u32 *inst, u32 features) case KVM_INST_MTSPR(SPRN_DSISR): kvm_patch_ins_stw(inst, magic_var(dsisr), inst_rt); break; -#ifdef CONFIG_PPC_BOOK3E_MMU +#ifdef CONFIG_PPC_E500 case KVM_INST_MTSPR(SPRN_MAS0): if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) kvm_patch_ins_stw(inst, magic_var(mas0), inst_rt); @@ -586,7 +586,7 @@ static void __init kvm_check_ins(u32 *inst, u32 features) if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) kvm_patch_ins_stw(inst, magic_var(mas7_3), inst_rt); break; -#endif /* CONFIG_PPC_BOOK3E_MMU */ +#endif /* CONFIG_PPC_E500 */ case KVM_INST_MTSPR(SPRN_SPRG4): if (features & KVM_MAGIC_FEAT_MAS0_TO_SPRG7) diff --git a/arch/powerpc/kernel/legacy_serial.c b/arch/powerpc/kernel/legacy_serial.c index 5c58460b269ac3f6057cbd026e97dcb35053d707..f048c424c525b0a735fe3f8503873d72a9049c0b 100644 --- a/arch/powerpc/kernel/legacy_serial.c +++ b/arch/powerpc/kernel/legacy_serial.c @@ -471,6 +471,8 @@ void __init find_legacy_serial_ports(void) } #endif + of_node_put(stdout); + DBG("legacy_serial_console = %d\n", legacy_serial_console); if (legacy_serial_console >= 0) setup_legacy_serial_console(legacy_serial_console); diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index fd6d8d3a548e0cc17cbbb5b70beea22275da6ffe..36184cada00b1107a67885da80dc6a96dfa5a4e8 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -286,7 +286,7 @@ kexec_flag: #ifdef CONFIG_KEXEC_CORE -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* * BOOK3E has no real MMU mode, so we have to setup the initial TLB * for a core to identity map v:0 to p:0. This current implementation @@ -354,7 +354,7 @@ _GLOBAL(kexec_smp_wait) * don't overwrite r3 here, it is live for kexec_wait above. */ real_mode: /* assume normal blr return */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* Create an identity mapping. */ b kexec_create_tlb #else @@ -413,7 +413,7 @@ _GLOBAL(kexec_sequence) lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ /* disable interrupts, we are overwriting kernel data next */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 wrteei 0 #else mfmsr r3 diff --git a/arch/powerpc/kernel/optprobes_head.S b/arch/powerpc/kernel/optprobes_head.S index 5c7f0b4b784b2687a960eef6839bc42b31556f1a..cd4e7bc32609d32e6892a670ade162992a737ea1 100644 --- a/arch/powerpc/kernel/optprobes_head.S +++ b/arch/powerpc/kernel/optprobes_head.S @@ -73,7 +73,7 @@ optprobe_template_entry: * further below. */ #ifdef CONFIG_PPC64 - ld r2,PACATOC(r13) + LOAD_PACA_TOC() #endif .global optprobe_template_op_address diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c index ba593fd6012450b658ac66b210212705404cbe51..be8db402e96390e964d72022be9cf83dfd7a1e4f 100644 --- a/arch/powerpc/kernel/paca.c +++ b/arch/powerpc/kernel/paca.c @@ -16,7 +16,6 @@ #include #include #include -#include #include "setup.h" @@ -170,30 +169,6 @@ static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) } #endif /* CONFIG_PPC_64S_HASH_MMU */ -#ifdef CONFIG_PPC_PSERIES -/** - * new_rtas_args() - Allocates rtas args - * @cpu: CPU number - * @limit: Memory limit for this allocation - * - * Allocates a struct rtas_args and return it's pointer, - * if not in Hypervisor mode - * - * Return: Pointer to allocated rtas_args - * NULL if CPU in Hypervisor Mode - */ -static struct rtas_args * __init new_rtas_args(int cpu, unsigned long limit) -{ - limit = min_t(unsigned long, limit, RTAS_INSTANTIATE_MAX); - - if (early_cpu_has_feature(CPU_FTR_HVMODE)) - return NULL; - - return alloc_paca_data(sizeof(struct rtas_args), L1_CACHE_BYTES, - limit, cpu); -} -#endif /* CONFIG_PPC_PSERIES */ - /* The Paca is an array with one entry per processor. Each contains an * lppaca, which contains the information shared between the * hypervisor and Linux. @@ -211,7 +186,7 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu) #ifdef CONFIG_PPC_PSERIES new_paca->lppaca_ptr = NULL; #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 new_paca->kernel_pgd = swapper_pg_dir; #endif new_paca->lock_token = 0x8000; @@ -228,14 +203,10 @@ void __init initialise_paca(struct paca_struct *new_paca, int cpu) new_paca->slb_shadow_ptr = NULL; #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* For now -- if we have threads this will be adjusted later */ new_paca->tcd_ptr = &new_paca->tcd; #endif - -#ifdef CONFIG_PPC_PSERIES - new_paca->rtas_args_reentrant = NULL; -#endif } /* Put the paca pointer into r13 and SPRG_PACA */ @@ -244,7 +215,7 @@ void setup_paca(struct paca_struct *new_paca) /* Setup r13 */ local_paca = new_paca; -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* On Book3E, initialize the TLB miss exception frames */ mtspr(SPRN_SPRG_TLB_EXFRAME, local_paca->extlb); #else @@ -307,9 +278,6 @@ void __init allocate_paca(int cpu) #endif #ifdef CONFIG_PPC_64S_HASH_MMU paca->slb_shadow_ptr = new_slb_shadow(cpu, limit); -#endif -#ifdef CONFIG_PPC_PSERIES - paca->rtas_args_reentrant = new_rtas_args(cpu, limit); #endif paca_struct_size += sizeof(struct paca_struct); } diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 31de91c8359c11ff6b44952e568c5f78eef82be4..d67cf79bf5d03456fbfc8aae5544b40c21de388c 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -135,7 +135,7 @@ struct pci_controller *pcibios_alloc_controller(struct device_node *dev) list_add_tail(&phb->list_node, &hose_list); spin_unlock(&hose_spinlock); - phb->dn = dev; + phb->dn = of_node_get(dev); phb->is_dynamic = slab_is_available(); #ifdef CONFIG_PPC64 if (dev) { @@ -158,7 +158,7 @@ void pcibios_free_controller(struct pci_controller *phb) /* Clear bit of phb_bitmap to allow reuse of this PHB number. */ if (phb->global_number < MAX_PHBS) clear_bit(phb->global_number, phb_bitmap); - + of_node_put(phb->dn); list_del(&phb->list_node); spin_unlock(&hose_spinlock); diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c index 433965bf37b4be29a25484a2e775d767b753d7cf..855b59892c5c93ea07dad7a4e4f6cfca6a170cd3 100644 --- a/arch/powerpc/kernel/pci_32.c +++ b/arch/powerpc/kernel/pci_32.c @@ -245,6 +245,15 @@ static int __init pcibios_init(void) printk(KERN_INFO "PCI: Probing PCI hardware\n"); +#ifdef CONFIG_PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT + /* + * Enable PCI domains in /proc when PCI bus numbers are not unique + * across all PCI domains to prevent conflicts. And keep PCI domain 0 + * backward compatible in /proc for video cards. + */ + pci_add_flags(PCI_ENABLE_PROC_DOMAINS | PCI_COMPAT_DOMAIN_0); +#endif + if (pci_has_flag(PCI_REASSIGN_ALL_BUS)) pci_assign_all_buses = 1; diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c index 7a35fc25a304653d637d92c8030b31789a8bb5e7..38561d6a2079216b7472e78f6f05d6d9151bab03 100644 --- a/arch/powerpc/kernel/pci_dn.c +++ b/arch/powerpc/kernel/pci_dn.c @@ -330,6 +330,7 @@ struct pci_dn *pci_add_device_node_info(struct pci_controller *hose, INIT_LIST_HEAD(&pdn->list); parent = of_get_parent(dn); pdn->parent = parent ? PCI_DN(parent) : NULL; + of_node_put(parent); if (pdn->parent) list_add_tail(&pdn->list, &pdn->parent->child_list); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 0fbda89cd1bb53b43024953469d9c39490dfeb5b..67da147fe34dc5cf2558f6304f87b8c20d0f3d45 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -127,7 +127,7 @@ unsigned long notrace msr_check_and_set(unsigned long bits) newmsr |= MSR_VSX; if (oldmsr != newmsr) - mtmsr_isync(newmsr); + newmsr = mtmsr_isync_irqsafe(newmsr); return newmsr; } @@ -145,7 +145,7 @@ void notrace __msr_check_and_clear(unsigned long bits) newmsr &= ~MSR_VSX; if (oldmsr != newmsr) - mtmsr_isync(newmsr); + mtmsr_isync_irqsafe(newmsr); } EXPORT_SYMBOL(__msr_check_and_clear); @@ -1655,11 +1655,6 @@ EXPORT_SYMBOL_GPL(set_thread_tidr); #endif /* CONFIG_PPC64 */ -void -release_thread(struct task_struct *t) -{ -} - /* * this gets called so that we can store coprocessor state into memory and * copy the current task into the new thread. @@ -2308,6 +2303,6 @@ void notrace __ppc64_runlatch_off(void) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= prandom_u32_max(PAGE_SIZE); return sp & ~0xf; } diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index a730b951b64bfe847b3de3190a3d5135e48f3797..1eed87d954ba8fb4129f258eb47f23bb5f6bf547 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -137,7 +138,7 @@ static void __init move_device_tree(void) } /* - * ibm,pa-features is a per-cpu property that contains a string of + * ibm,pa/pi-features is a per-cpu property that contains a string of * attribute descriptors, each of which has a 2 byte header plus up * to 254 bytes worth of processor attribute bits. First header * byte specifies the number of bytes following the header. @@ -149,15 +150,17 @@ static void __init move_device_tree(void) * is supported/not supported. Note that the bit numbers are * big-endian to match the definition in PAPR. */ -static struct ibm_pa_feature { +struct ibm_feature { unsigned long cpu_features; /* CPU_FTR_xxx bit */ unsigned long mmu_features; /* MMU_FTR_xxx bit */ unsigned int cpu_user_ftrs; /* PPC_FEATURE_xxx bit */ unsigned int cpu_user_ftrs2; /* PPC_FEATURE2_xxx bit */ - unsigned char pabyte; /* byte number in ibm,pa-features */ + unsigned char pabyte; /* byte number in ibm,pa/pi-features */ unsigned char pabit; /* bit number (big-endian) */ unsigned char invert; /* if 1, pa bit set => clear feature */ -} ibm_pa_features[] __initdata = { +}; + +static struct ibm_feature ibm_pa_features[] __initdata = { { .pabyte = 0, .pabit = 0, .cpu_user_ftrs = PPC_FEATURE_HAS_MMU }, { .pabyte = 0, .pabit = 1, .cpu_user_ftrs = PPC_FEATURE_HAS_FPU }, { .pabyte = 0, .pabit = 3, .cpu_features = CPU_FTR_CTRL }, @@ -179,9 +182,19 @@ static struct ibm_pa_feature { { .pabyte = 64, .pabit = 0, .cpu_features = CPU_FTR_DAWR1 }, }; +/* + * ibm,pi-features property provides the support of processor specific + * options not described in ibm,pa-features. Right now use byte 0, bit 3 + * which indicates the occurrence of DSI interrupt when the paste operation + * on the suspended NX window. + */ +static struct ibm_feature ibm_pi_features[] __initdata = { + { .pabyte = 0, .pabit = 3, .mmu_features = MMU_FTR_NX_DSI }, +}; + static void __init scan_features(unsigned long node, const unsigned char *ftrs, unsigned long tablelen, - struct ibm_pa_feature *fp, + struct ibm_feature *fp, unsigned long ft_size) { unsigned long i, len, bit; @@ -218,17 +231,18 @@ static void __init scan_features(unsigned long node, const unsigned char *ftrs, } } -static void __init check_cpu_pa_features(unsigned long node) +static void __init check_cpu_features(unsigned long node, char *name, + struct ibm_feature *fp, + unsigned long size) { const unsigned char *pa_ftrs; int tablelen; - pa_ftrs = of_get_flat_dt_prop(node, "ibm,pa-features", &tablelen); + pa_ftrs = of_get_flat_dt_prop(node, name, &tablelen); if (pa_ftrs == NULL) return; - scan_features(node, pa_ftrs, tablelen, - ibm_pa_features, ARRAY_SIZE(ibm_pa_features)); + scan_features(node, pa_ftrs, tablelen, fp, size); } #ifdef CONFIG_PPC_64S_HASH_MMU @@ -376,11 +390,16 @@ static int __init early_init_dt_scan_cpus(unsigned long node, */ if (!dt_cpu_ftrs_in_use()) { prop = of_get_flat_dt_prop(node, "cpu-version", NULL); - if (prop && (be32_to_cpup(prop) & 0xff000000) == 0x0f000000) + if (prop && (be32_to_cpup(prop) & 0xff000000) == 0x0f000000) { identify_cpu(0, be32_to_cpup(prop)); + seq_buf_printf(&ppc_hw_desc, "0x%04x ", be32_to_cpup(prop)); + } check_cpu_feature_properties(node); - check_cpu_pa_features(node); + check_cpu_features(node, "ibm,pa-features", ibm_pa_features, + ARRAY_SIZE(ibm_pa_features)); + check_cpu_features(node, "ibm,pi-features", ibm_pi_features, + ARRAY_SIZE(ibm_pi_features)); } identical_pvr_fixup(node); @@ -696,6 +715,23 @@ static void __init tm_init(void) static void tm_init(void) { } #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ +static int __init +early_init_dt_scan_model(unsigned long node, const char *uname, + int depth, void *data) +{ + const char *prop; + + if (depth != 0) + return 0; + + prop = of_get_flat_dt_prop(node, "model", NULL); + if (prop) + seq_buf_printf(&ppc_hw_desc, "%s ", prop); + + /* break now */ + return 1; +} + #ifdef CONFIG_PPC64 static void __init save_fscr_to_task(void) { @@ -724,6 +760,8 @@ void __init early_init_devtree(void *params) if (!early_init_dt_verify(params)) panic("BUG: Failed verifying flat device tree, bad version?"); + of_scan_flat_dt(early_init_dt_scan_model, NULL); + #ifdef CONFIG_PPC_RTAS /* Some machines might need RTAS info for debugging, grab it now. */ of_scan_flat_dt(early_init_dt_scan_rtas, NULL); @@ -803,6 +841,9 @@ void __init early_init_devtree(void *params) dt_cpu_ftrs_scan(); + // We can now add the CPU name & PVR to the hardware description + seq_buf_printf(&ppc_hw_desc, "%s 0x%04lx ", cur_cpu_spec->cpu_name, mfspr(SPRN_PVR)); + /* Retrieve CPU related informations from the flat tree * (altivec support, boot CPU ID, ...) */ diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index a6669c40c1dbec3789b194198ea8a0b9dd56a114..d464ba412084d10e5d4d7cd9dd27485a1c556aca 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -96,12 +96,6 @@ static int of_workarounds __prombss; #define OF_WA_CLAIM 1 /* do phys/virt claim separately, then map */ #define OF_WA_LONGTRAIL 2 /* work around longtrail bugs */ -#define PROM_BUG() do { \ - prom_printf("kernel BUG at %s line 0x%x!\n", \ - __FILE__, __LINE__); \ - __builtin_trap(); \ -} while (0) - #ifdef DEBUG_PROM #define prom_debug(x...) prom_printf(x) #else diff --git a/arch/powerpc/kernel/prom_init_check.sh b/arch/powerpc/kernel/prom_init_check.sh index dfa5f729f774d6607f6e417bb5ca379430b96257..311890d71c4c9870ac7f5bf10f8dc2572f9300af 100644 --- a/arch/powerpc/kernel/prom_init_check.sh +++ b/arch/powerpc/kernel/prom_init_check.sh @@ -26,8 +26,7 @@ _end enter_prom $MEM_FUNCS reloc_offset __secondary_hold __secondary_hold_acknowledge __secondary_hold_spinloop __start logo_linux_clut224 btext_prepare_BAT reloc_got2 kernstart_addr memstart_addr linux_banner _stext -__prom_init_toc_start __prom_init_toc_end btext_setup_display TOC. -relocate" +btext_setup_display TOC. relocate" NM="$1" OBJ="$2" diff --git a/arch/powerpc/kernel/reloc_64.S b/arch/powerpc/kernel/reloc_64.S index 232e4549defe1f79b78f148bf94989565b45d973..efd52f2e70339919535983e86a143f62f6a17f47 100644 --- a/arch/powerpc/kernel/reloc_64.S +++ b/arch/powerpc/kernel/reloc_64.S @@ -27,8 +27,8 @@ _GLOBAL(relocate) add r9,r9,r12 /* r9 has runtime addr of .rela.dyn section */ ld r10,(p_st - 0b)(r12) add r10,r10,r12 /* r10 has runtime addr of _stext */ - ld r13,(p_sym - 0b)(r12) - add r13,r13,r12 /* r13 has runtime addr of .dynsym */ + ld r4,(p_sym - 0b)(r12) + add r4,r4,r12 /* r4 has runtime addr of .dynsym */ /* * Scan the dynamic section for the RELA, RELASZ and RELAENT entries. @@ -84,16 +84,16 @@ _GLOBAL(relocate) ld r0,16(r9) /* reloc->r_addend */ b .Lstore .Luaddr64: - srdi r14,r0,32 /* ELF64_R_SYM(reloc->r_info) */ + srdi r5,r0,32 /* ELF64_R_SYM(reloc->r_info) */ clrldi r0,r0,32 cmpdi r0,R_PPC64_UADDR64 bne .Lnext ld r6,0(r9) ld r0,16(r9) - mulli r14,r14,24 /* 24 == sizeof(elf64_sym) */ - add r14,r14,r13 /* elf64_sym[ELF64_R_SYM] */ - ld r14,8(r14) - add r0,r0,r14 + mulli r5,r5,24 /* 24 == sizeof(elf64_sym) */ + add r5,r5,r4 /* elf64_sym[ELF64_R_SYM] */ + ld r5,8(r5) + add r0,r0,r5 .Lstore: add r0,r0,r3 stdx r0,r7,r6 diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 6931339722948bb154792930dd14d72c2ec78295..e847f9b1c5b9fdd8190817207156f05e86ad70f7 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -43,7 +44,6 @@ #include #include #include -#include /* This is here deliberately so it's only used in this file */ void enter_rtas(unsigned long); @@ -464,6 +464,9 @@ void rtas_call_unlocked(struct rtas_args *args, int token, int nargs, int nret, va_end(list); } +static int ibm_open_errinjct_token; +static int ibm_errinjct_token; + int rtas_call(int token, int nargs, int nret, int *outputs, ...) { va_list list; @@ -476,6 +479,16 @@ int rtas_call(int token, int nargs, int nret, int *outputs, ...) if (!rtas.entry || token == RTAS_UNKNOWN_SERVICE) return -1; + if (token == ibm_open_errinjct_token || token == ibm_errinjct_token) { + /* + * It would be nicer to not discard the error value + * from security_locked_down(), but callers expect an + * RTAS status, not an errno. + */ + if (security_locked_down(LOCKDOWN_RTAS_ERROR_INJECTION)) + return -1; + } + if ((mfmsr() & (MSR_IR|MSR_DR)) != (MSR_IR|MSR_DR)) { WARN_ON_ONCE(1); return -1; @@ -932,59 +945,6 @@ void rtas_activate_firmware(void) pr_err("ibm,activate-firmware failed (%i)\n", fwrc); } -#ifdef CONFIG_PPC_PSERIES -/** - * rtas_call_reentrant() - Used for reentrant rtas calls - * @token: Token for desired reentrant RTAS call - * @nargs: Number of Input Parameters - * @nret: Number of Output Parameters - * @outputs: Array of outputs - * @...: Inputs for desired RTAS call - * - * According to LoPAR documentation, only "ibm,int-on", "ibm,int-off", - * "ibm,get-xive" and "ibm,set-xive" are currently reentrant. - * Reentrant calls need their own rtas_args buffer, so not using rtas.args, but - * PACA one instead. - * - * Return: -1 on error, - * First output value of RTAS call if (nret > 0), - * 0 otherwise, - */ -int rtas_call_reentrant(int token, int nargs, int nret, int *outputs, ...) -{ - va_list list; - struct rtas_args *args; - unsigned long flags; - int i, ret = 0; - - if (!rtas.entry || token == RTAS_UNKNOWN_SERVICE) - return -1; - - local_irq_save(flags); - preempt_disable(); - - /* We use the per-cpu (PACA) rtas args buffer */ - args = local_paca->rtas_args_reentrant; - - va_start(list, outputs); - va_rtas_call_unlocked(args, token, nargs, nret, list); - va_end(list); - - if (nret > 1 && outputs) - for (i = 0; i < nret - 1; ++i) - outputs[i] = be32_to_cpu(args->rets[i + 1]); - - if (nret > 0) - ret = be32_to_cpu(args->rets[0]); - - local_irq_restore(flags); - preempt_enable(); - - return ret; -} - -#endif /* CONFIG_PPC_PSERIES */ - /** * get_pseries_errorlog() - Find a specific pseries error log in an RTAS * extended event log. @@ -1227,6 +1187,14 @@ SYSCALL_DEFINE1(rtas, struct rtas_args __user *, uargs) if (block_rtas_call(token, nargs, &args)) return -EINVAL; + if (token == ibm_open_errinjct_token || token == ibm_errinjct_token) { + int err; + + err = security_locked_down(LOCKDOWN_RTAS_ERROR_INJECTION); + if (err) + return err; + } + /* Need to handle ibm,suspend_me call specially */ if (token == rtas_token("ibm,suspend-me")) { @@ -1325,7 +1293,8 @@ void __init rtas_initialize(void) #ifdef CONFIG_RTAS_ERROR_LOGGING rtas_last_error_token = rtas_token("rtas-last-error"); #endif - + ibm_open_errinjct_token = rtas_token("ibm,open-errinjct"); + ibm_errinjct_token = rtas_token("ibm,errinjct"); rtas_syscall_filter_init(); } diff --git a/arch/powerpc/kernel/rtas_entry.S b/arch/powerpc/kernel/rtas_entry.S index 9a434d42e660ac8ee8d13415b324a4da1bbf9142..6ce95ddadbcdbe38ddc7044737f543f7a3674e8d 100644 --- a/arch/powerpc/kernel/rtas_entry.S +++ b/arch/powerpc/kernel/rtas_entry.S @@ -109,8 +109,12 @@ __enter_rtas: * its critical regions (as specified in PAPR+ section 7.2.1). MSR[S] * is not impacted by RFI_TO_KERNEL (only urfid can unset it). So if * MSR[S] is set, it will remain when entering RTAS. + * If we're in HV mode, RTAS must also run in HV mode, so extract MSR_HV + * from the saved MSR value and insert into the value RTAS will use. */ + extrdi r0, r6, 1, 63 - MSR_HV_LG LOAD_REG_IMMEDIATE(r6, MSR_ME | MSR_RI) + insrdi r6, r0, 1, 63 - MSR_HV_LG li r0,0 mtmsrd r0,1 /* disable RI before using SRR0/1 */ diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c index d96fd14bd7c9c614036f758b466502bb4d84b0aa..206475e3e0b480116719b69b0ddd0873f18dd529 100644 --- a/arch/powerpc/kernel/security.c +++ b/arch/powerpc/kernel/security.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -34,7 +35,7 @@ static enum branch_cache_flush_type link_stack_flush_type = BRANCH_CACHE_FLUSH_N bool barrier_nospec_enabled; static bool no_nospec; static bool btb_flush_enabled; -#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_BOOK3S_64) +#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64) static bool no_spectrev2; #endif @@ -121,7 +122,7 @@ static __init int security_feature_debugfs_init(void) device_initcall(security_feature_debugfs_init); #endif /* CONFIG_DEBUG_FS */ -#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_BOOK3S_64) +#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64) static int __init handle_nospectre_v2(char *p) { no_spectrev2 = true; @@ -129,9 +130,9 @@ static int __init handle_nospectre_v2(char *p) return 0; } early_param("nospectre_v2", handle_nospectre_v2); -#endif /* CONFIG_PPC_FSL_BOOK3E || CONFIG_PPC_BOOK3S_64 */ +#endif /* CONFIG_PPC_E500 || CONFIG_PPC_BOOK3S_64 */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 void __init setup_spectre_v2(void) { if (no_spectrev2 || cpu_mitigations_off()) @@ -139,7 +140,7 @@ void __init setup_spectre_v2(void) else btb_flush_enabled = true; } -#endif /* CONFIG_PPC_FSL_BOOK3E */ +#endif /* CONFIG_PPC_E500 */ #ifdef CONFIG_PPC_BOOK3S_64 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index dd98f43bd685c9df4000ab44454b9f7a22b39e3d..6d041993a45dccf868bccb2d7fca38d2583b15bc 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -588,6 +590,15 @@ static __init int add_pcspkr(void) device_initcall(add_pcspkr); #endif /* CONFIG_PCSPKR_PLATFORM */ +static char ppc_hw_desc_buf[128] __initdata; + +struct seq_buf ppc_hw_desc __initdata = { + .buffer = ppc_hw_desc_buf, + .size = sizeof(ppc_hw_desc_buf), + .len = 0, + .readpos = 0, +}; + static __init void probe_machine(void) { extern struct machdep_calls __machine_desc_start; @@ -628,7 +639,13 @@ static __init void probe_machine(void) for (;;); } - printk(KERN_INFO "Using %s machine description\n", ppc_md.name); + // Append the machine name to other info we've gathered + seq_buf_puts(&ppc_hw_desc, ppc_md.name); + + // Set the generic hardware description shown in oopses + dump_stack_set_arch_desc(ppc_hw_desc.buffer); + + pr_info("Hardware name: %s\n", ppc_hw_desc.buffer); } /* Match a class of boards, not a specific device configuration. */ diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h index 93f22da12abe78c4982c637da3f3afb70669248f..7912bb50a7cb82d6dddab83c09bb4adc3079e632 100644 --- a/arch/powerpc/kernel/setup.h +++ b/arch/powerpc/kernel/setup.h @@ -23,7 +23,7 @@ void check_smt_enabled(void); static inline void check_smt_enabled(void) { } #endif -#if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP) +#if defined(CONFIG_PPC_BOOK3E_64) && defined(CONFIG_SMP) void setup_tlb_core_data(void); #else static inline void setup_tlb_core_data(void) { } diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 8132617893035cfa04ae18b81e6399c691bf7740..b761cc1a403c3df4af9858758a5f635b546b0e58 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -207,7 +207,7 @@ void __init setup_power_save(void) ppc_md.power_save = ppc6xx_idle; #endif -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 if (cpu_has_feature(CPU_FTR_CAN_DOZE) || cpu_has_feature(CPU_FTR_CAN_NAP)) ppc_md.power_save = e500_idle; diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 2b2d0b0fbb30d8252b5db3e427be1cbc1177c657..a0dee7354fe6b5032c0d334487fe2ec30f2fae87 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -86,7 +87,7 @@ struct ppc64_caches ppc64_caches = { }; EXPORT_SYMBOL_GPL(ppc64_caches); -#if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP) +#if defined(CONFIG_PPC_BOOK3E_64) && defined(CONFIG_SMP) void __init setup_tlb_core_data(void) { int cpu; @@ -176,14 +177,26 @@ early_param("smt-enabled", early_smt_enabled); #endif /* CONFIG_SMP */ /** Fix up paca fields required for the boot cpu */ -static void __init fixup_boot_paca(void) +static void __init fixup_boot_paca(struct paca_struct *boot_paca) { /* The boot cpu is started */ - get_paca()->cpu_start = 1; + boot_paca->cpu_start = 1; +#ifdef CONFIG_PPC_BOOK3S_64 + /* + * Give the early boot machine check stack somewhere to use, use + * half of the init stack. This is a bit hacky but there should not be + * deep stack usage in early init so shouldn't overflow it or overwrite + * things. + */ + boot_paca->mc_emergency_sp = (void *)&init_thread_union + + (THREAD_SIZE/2); +#endif /* Allow percpu accesses to work until we setup percpu data */ - get_paca()->data_offset = 0; - /* Mark interrupts disabled in PACA */ - irq_soft_mask_set(IRQS_DISABLED); + boot_paca->data_offset = 0; + /* Mark interrupts soft and hard disabled in PACA */ + boot_paca->irq_soft_mask = IRQS_DISABLED; + boot_paca->irq_happened = PACA_IRQ_HARD_DIS; + WARN_ON(mfmsr() & MSR_EE); } static void __init configure_exceptions(void) @@ -350,11 +363,15 @@ void __init early_setup(unsigned long dt_ptr) * what CPU we are on. */ initialise_paca(&boot_paca, 0); - setup_paca(&boot_paca); - fixup_boot_paca(); + fixup_boot_paca(&boot_paca); + WARN_ON(local_paca != 0); + setup_paca(&boot_paca); /* install the paca into registers */ /* -------- printk is now safe to use ------- */ + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && (mfmsr() & MSR_HV)) + enable_machine_check(); + /* Try new device tree based feature discovery ... */ if (!dt_cpu_ftrs_init(__va(dt_ptr))) /* Otherwise use the old style CPU table */ @@ -377,8 +394,8 @@ void __init early_setup(unsigned long dt_ptr) /* Poison paca_ptrs[0] again if it's not the boot cpu */ memset(&paca_ptrs[0], 0x88, sizeof(paca_ptrs[0])); } - setup_paca(paca_ptrs[boot_cpuid]); - fixup_boot_paca(); + fixup_boot_paca(paca_ptrs[boot_cpuid]); + setup_paca(paca_ptrs[boot_cpuid]); /* install the paca into registers */ /* * Configure exception handlers. This include setting up trampolines @@ -673,7 +690,7 @@ void __init initialize_cache_info(void) */ __init u64 ppc64_bolted_size(void) { -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 /* Freescale BookE bolts the entire linear mapping */ /* XXX: BookE ppc64_rma_limit setup seems to disagree? */ if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E)) @@ -723,7 +740,7 @@ void __init irqstack_early_init(void) } } -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 void __init exc_lvl_early_init(void) { unsigned int i; @@ -825,7 +842,7 @@ void __init setup_per_cpu_areas(void) /* * BookE and BookS radix are historical values and should be revisited. */ - if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { + if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) { atom_size = SZ_1M; } else if (radix_enabled()) { atom_size = PAGE_SIZE; diff --git a/arch/powerpc/kernel/signal.h b/arch/powerpc/kernel/signal.h index 618aeccdf69184464ec8dde6db4208c2fc3ab8eb..a429c57ed4331805a2ce60140d9254a8f9316e7f 100644 --- a/arch/powerpc/kernel/signal.h +++ b/arch/powerpc/kernel/signal.h @@ -196,9 +196,6 @@ extern int handle_rt_signal64(struct ksignal *ksig, sigset_t *set, #else /* CONFIG_PPC64 */ -extern long sys_rt_sigreturn(void); -extern long sys_sigreturn(void); - static inline int handle_rt_signal64(struct ksignal *ksig, sigset_t *set, struct task_struct *tsk) { diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index 157a7403e3eb3e3f39002c2f23e0927323fef6f9..c114c7f25645c0b41f810e0f95705eb906c61fbc 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -43,7 +43,7 @@ #include #include #ifdef CONFIG_PPC64 -#include "ppc32.h" +#include #include #else #include diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 169703fead57668a8667ab483079f161546547d7..0da6e59161cd494678ebabce5feb8dd72e5fe017 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -708,7 +708,7 @@ static struct task_struct *current_set[NR_CPUS]; static void smp_store_cpu_info(int id) { per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR); -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 per_cpu(next_tlbcam_idx, id) = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1; #endif @@ -1257,7 +1257,12 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle) int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - int rc, c; + const unsigned long boot_spin_ms = 5 * MSEC_PER_SEC; + const bool booting = system_state < SYSTEM_RUNNING; + const unsigned long hp_spin_ms = 1; + unsigned long deadline; + int rc; + const unsigned long spin_wait_ms = booting ? boot_spin_ms : hp_spin_ms; /* * Don't allow secondary threads to come online if inhibited @@ -1302,22 +1307,23 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) } /* - * wait to see if the cpu made a callin (is actually up). - * use this value that I found through experimentation. - * -- Cort + * At boot time, simply spin on the callin word until the + * deadline passes. + * + * At run time, spin for an optimistic amount of time to avoid + * sleeping in the common case. */ - if (system_state < SYSTEM_RUNNING) - for (c = 50000; c && !cpu_callin_map[cpu]; c--) - udelay(100); -#ifdef CONFIG_HOTPLUG_CPU - else - /* - * CPUs can take much longer to come up in the - * hotplug case. Wait five seconds. - */ - for (c = 5000; c && !cpu_callin_map[cpu]; c--) - msleep(1); -#endif + deadline = jiffies + msecs_to_jiffies(spin_wait_ms); + spin_until_cond(cpu_callin_map[cpu] || time_is_before_jiffies(deadline)); + + if (!cpu_callin_map[cpu] && system_state >= SYSTEM_RUNNING) { + const unsigned long sleep_interval_us = 10 * USEC_PER_MSEC; + const unsigned long sleep_wait_ms = 100 * MSEC_PER_SEC; + + deadline = jiffies + msecs_to_jiffies(sleep_wait_ms); + while (!cpu_callin_map[cpu] && time_is_after_jiffies(deadline)) + fsleep(sleep_interval_us); + } if (!cpu_callin_map[cpu]) { printk(KERN_ERR "Processor %u is stuck.\n", cpu); diff --git a/arch/powerpc/kernel/swsusp_booke.S b/arch/powerpc/kernel/swsusp_85xx.S similarity index 100% rename from arch/powerpc/kernel/swsusp_booke.S rename to arch/powerpc/kernel/swsusp_85xx.S diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S index 9f1903c7f54092f66a36055d72bccdf9a5b667f9..f645652c265465772e674b609e22eb2c90ac108b 100644 --- a/arch/powerpc/kernel/swsusp_asm64.S +++ b/arch/powerpc/kernel/swsusp_asm64.S @@ -76,16 +76,10 @@ swsusp_save_area: .space SL_SIZE - .section ".toc","aw" -swsusp_save_area_ptr: - .tc swsusp_save_area[TC],swsusp_save_area -restore_pblist_ptr: - .tc restore_pblist[TC],restore_pblist - .section .text .align 5 _GLOBAL(swsusp_arch_suspend) - ld r11,swsusp_save_area_ptr@toc(r2) + LOAD_REG_ADDR(r11, swsusp_save_area) SAVE_SPECIAL(LR) SAVE_REGISTER(r1) SAVE_SPECIAL(CR) @@ -131,7 +125,7 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR) bl swsusp_save /* restore LR */ - ld r11,swsusp_save_area_ptr@toc(r2) + LOAD_REG_ADDR(r11, swsusp_save_area) RESTORE_SPECIAL(LR) addi r1,r1,128 @@ -145,7 +139,7 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) sync - ld r12,restore_pblist_ptr@toc(r2) + LOAD_REG_ADDR(r11, restore_pblist) ld r12,0(r12) cmpdi r12,0 @@ -187,7 +181,7 @@ nothing_to_copy: tlbia #endif - ld r11,swsusp_save_area_ptr@toc(r2) + LOAD_REG_ADDR(r11, swsusp_save_area) RESTORE_SPECIAL(CR) @@ -265,7 +259,7 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR) bl do_after_copyback addi r1,r1,128 - ld r11,swsusp_save_area_ptr@toc(r2) + LOAD_REG_ADDR(r11, swsusp_save_area) RESTORE_SPECIAL(LR) li r3, 0 diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index 16ff0399a257499fee59294c534fbfa97032172d..1ab4a4d95abafa50649f0848d41bcf09a1b847f0 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c @@ -1,13 +1,23 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * sys_ppc32.c: Conversion between 32bit and 64bit native syscalls. + * sys_ppc32.c: 32-bit system calls with complex calling conventions. * * Copyright (C) 2001 IBM * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) * - * These routines maintain argument size conversion between 32bit and 64bit - * environment. + * 32-bit system calls with 64-bit arguments pass those in register pairs. + * This must be specially dealt with on 64-bit kernels. The compat_arg_u64_dual + * in generic compat syscalls is not always usable because the register + * pairing is constrained depending on preceding arguments. + * + * An analogous problem exists on 32-bit kernels with ARCH_HAS_SYSCALL_WRAPPER, + * the defined system call functions take the pt_regs as an argument, and there + * is a mapping macro which maps registers to arguments + * (SC_POWERPC_REGS_TO_ARGS) which also does not deal with these 64-bit + * arguments. + * + * This file contains these system calls. */ #include @@ -25,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -48,72 +57,65 @@ #include #include -unsigned long compat_sys_mmap2(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) -{ - /* This should remain 12 even if PAGE_SIZE changes */ - return sys_mmap(addr, len, prot, flags, fd, pgoff << 12); -} - -/* - * long long munging: - * The 32 bit ABI passes long longs in an odd even register pair. - * High and low parts are swapped depending on endian mode, - * so define a macro (similar to mips linux32) to handle that. - */ -#ifdef __LITTLE_ENDIAN__ -#define merge_64(low, high) ((u64)high << 32) | low +#ifdef CONFIG_PPC32 +#define PPC32_SYSCALL_DEFINE4 SYSCALL_DEFINE4 +#define PPC32_SYSCALL_DEFINE5 SYSCALL_DEFINE5 +#define PPC32_SYSCALL_DEFINE6 SYSCALL_DEFINE6 #else -#define merge_64(high, low) ((u64)high << 32) | low +#define PPC32_SYSCALL_DEFINE4 COMPAT_SYSCALL_DEFINE4 +#define PPC32_SYSCALL_DEFINE5 COMPAT_SYSCALL_DEFINE5 +#define PPC32_SYSCALL_DEFINE6 COMPAT_SYSCALL_DEFINE6 #endif -compat_ssize_t compat_sys_pread64(unsigned int fd, char __user *ubuf, compat_size_t count, - u32 reg6, u32 pos1, u32 pos2) +PPC32_SYSCALL_DEFINE6(ppc_pread64, + unsigned int, fd, + char __user *, ubuf, compat_size_t, count, + u32, reg6, u32, pos1, u32, pos2) { return ksys_pread64(fd, ubuf, count, merge_64(pos1, pos2)); } -compat_ssize_t compat_sys_pwrite64(unsigned int fd, const char __user *ubuf, compat_size_t count, - u32 reg6, u32 pos1, u32 pos2) +PPC32_SYSCALL_DEFINE6(ppc_pwrite64, + unsigned int, fd, + const char __user *, ubuf, compat_size_t, count, + u32, reg6, u32, pos1, u32, pos2) { return ksys_pwrite64(fd, ubuf, count, merge_64(pos1, pos2)); } -compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offset1, u32 offset2, u32 count) +PPC32_SYSCALL_DEFINE5(ppc_readahead, + int, fd, u32, r4, + u32, offset1, u32, offset2, u32, count) { return ksys_readahead(fd, merge_64(offset1, offset2), count); } -asmlinkage int compat_sys_truncate64(const char __user * path, u32 reg4, - unsigned long len1, unsigned long len2) +PPC32_SYSCALL_DEFINE4(ppc_truncate64, + const char __user *, path, u32, reg4, + unsigned long, len1, unsigned long, len2) { return ksys_truncate(path, merge_64(len1, len2)); } -asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offset1, u32 offset2, - u32 len1, u32 len2) -{ - return ksys_fallocate(fd, mode, ((loff_t)offset1 << 32) | offset2, - merge_64(len1, len2)); -} - -asmlinkage int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long len1, - unsigned long len2) +PPC32_SYSCALL_DEFINE4(ppc_ftruncate64, + unsigned int, fd, u32, reg4, + unsigned long, len1, unsigned long, len2) { return ksys_ftruncate(fd, merge_64(len1, len2)); } -long ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, - size_t len, int advice) +PPC32_SYSCALL_DEFINE6(ppc32_fadvise64, + int, fd, u32, unused, u32, offset1, u32, offset2, + size_t, len, int, advice) { return ksys_fadvise64_64(fd, merge_64(offset1, offset2), len, advice); } -asmlinkage long compat_sys_sync_file_range2(int fd, unsigned int flags, - unsigned offset1, unsigned offset2, - unsigned nbytes1, unsigned nbytes2) +COMPAT_SYSCALL_DEFINE6(ppc_sync_file_range2, + int, fd, unsigned int, flags, + unsigned int, offset1, unsigned int, offset2, + unsigned int, nbytes1, unsigned int, nbytes2) { loff_t offset = merge_64(offset1, offset2); loff_t nbytes = merge_64(nbytes1, nbytes2); diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c index 81ace9e8b72b685b93d5fa788aa5b77ab150c1ad..18b9d325395f5e593930577ef178b1dd55abae99 100644 --- a/arch/powerpc/kernel/syscall.c +++ b/arch/powerpc/kernel/syscall.c @@ -12,12 +12,8 @@ #include -typedef long (*syscall_fn)(long, long, long, long, long, long); - /* Has to run notrace because it is entered not completely "reconciled" */ -notrace long system_call_exception(long r3, long r4, long r5, - long r6, long r7, long r8, - unsigned long r0, struct pt_regs *regs) +notrace long system_call_exception(struct pt_regs *regs, unsigned long r0) { long ret; syscall_fn f; @@ -25,7 +21,6 @@ notrace long system_call_exception(long r3, long r4, long r5, kuap_lock(); add_random_kstack_offset(); - regs->orig_gpr3 = r3; if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); @@ -139,12 +134,6 @@ notrace long system_call_exception(long r3, long r4, long r5, r0 = do_syscall_trace_enter(regs); if (unlikely(r0 >= NR_syscalls)) return regs->gpr[3]; - r3 = regs->gpr[3]; - r4 = regs->gpr[4]; - r5 = regs->gpr[5]; - r6 = regs->gpr[6]; - r7 = regs->gpr[7]; - r8 = regs->gpr[8]; } else if (unlikely(r0 >= NR_syscalls)) { if (unlikely(trap_is_unsupported_scv(regs))) { @@ -158,21 +147,31 @@ notrace long system_call_exception(long r3, long r4, long r5, /* May be faster to do array_index_nospec? */ barrier_nospec(); +#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER + // No COMPAT if we have SYSCALL_WRAPPER, see Kconfig + f = (void *)sys_call_table[r0]; + ret = f(regs); +#else if (unlikely(is_compat_task())) { + unsigned long r3, r4, r5, r6, r7, r8; + f = (void *)compat_sys_call_table[r0]; - r3 &= 0x00000000ffffffffULL; - r4 &= 0x00000000ffffffffULL; - r5 &= 0x00000000ffffffffULL; - r6 &= 0x00000000ffffffffULL; - r7 &= 0x00000000ffffffffULL; - r8 &= 0x00000000ffffffffULL; + r3 = regs->gpr[3] & 0x00000000ffffffffULL; + r4 = regs->gpr[4] & 0x00000000ffffffffULL; + r5 = regs->gpr[5] & 0x00000000ffffffffULL; + r6 = regs->gpr[6] & 0x00000000ffffffffULL; + r7 = regs->gpr[7] & 0x00000000ffffffffULL; + r8 = regs->gpr[8] & 0x00000000ffffffffULL; + ret = f(r3, r4, r5, r6, r7, r8); } else { f = (void *)sys_call_table[r0]; - } - ret = f(r3, r4, r5, r6, r7, r8); + ret = f(regs->gpr[3], regs->gpr[4], regs->gpr[5], + regs->gpr[6], regs->gpr[7], regs->gpr[8]); + } +#endif /* * Ultimately, this value will get limited by KSTACK_OFFSET_MAX(), diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c index fc999140bc27e94229856e80f0cac221f87c59b8..68ebb23a5af4bb74c0006903b0a2aa5e5124fd4e 100644 --- a/arch/powerpc/kernel/syscalls.c +++ b/arch/powerpc/kernel/syscalls.c @@ -36,9 +36,9 @@ #include #include -static inline long do_mmap2(unsigned long addr, size_t len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long off, int shift) +static long do_mmap2(unsigned long addr, size_t len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long off, int shift) { if (!arch_validate_prot(prot, addr)) return -EINVAL; @@ -56,6 +56,16 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, size_t, len, return do_mmap2(addr, len, prot, flags, fd, pgoff, PAGE_SHIFT-12); } +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE6(mmap2, + unsigned long, addr, size_t, len, + unsigned long, prot, unsigned long, flags, + unsigned long, fd, unsigned long, off_4k) +{ + return do_mmap2(addr, len, prot, flags, fd, off_4k, PAGE_SHIFT-12); +} +#endif + SYSCALL_DEFINE6(mmap, unsigned long, addr, size_t, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, off_t, offset) @@ -63,43 +73,39 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, size_t, len, return do_mmap2(addr, len, prot, flags, fd, offset, PAGE_SHIFT); } -#ifdef CONFIG_PPC32 -/* - * Due to some executables calling the wrong select we sometimes - * get wrong args. This determines how the args are being passed - * (a single ptr to them all args passed) then calls - * sys_select() with the appropriate args. -- Cort - */ -int -ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp, struct __kernel_old_timeval __user *tvp) -{ - if ((unsigned long)n >= 4096) - return sys_old_select((void __user *)n); - - return sys_select(n, inp, outp, exp, tvp); -} -#endif - #ifdef CONFIG_PPC64 -long ppc64_personality(unsigned long personality) +static long do_ppc64_personality(unsigned long personality) { long ret; if (personality(current->personality) == PER_LINUX32 && personality(personality) == PER_LINUX) personality = (personality & ~PER_MASK) | PER_LINUX32; - ret = sys_personality(personality); + ret = ksys_personality(personality); if (personality(ret) == PER_LINUX32) ret = (ret & ~PER_MASK) | PER_LINUX; return ret; } -#endif -long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low, - u32 len_high, u32 len_low) +SYSCALL_DEFINE1(ppc64_personality, unsigned long, personality) +{ + return do_ppc64_personality(personality); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE1(ppc64_personality, unsigned long, personality) +{ + return do_ppc64_personality(personality); +} +#endif /* CONFIG_COMPAT */ +#endif /* CONFIG_PPC64 */ + +SYSCALL_DEFINE6(ppc_fadvise64_64, + int, fd, int, advice, u32, offset_high, u32, offset_low, + u32, len_high, u32, len_low) { - return ksys_fadvise64_64(fd, (u64)offset_high << 32 | offset_low, - (u64)len_high << 32 | len_low, advice); + return ksys_fadvise64_64(fd, merge_64(offset_high, offset_low), + merge_64(len_high, len_low), advice); } SYSCALL_DEFINE0(switch_endian) diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index 2600b4237292c78c44363159a02eaca9a2dd88c8..e9e0df4f9a61a494a2344bd6589a02d29170179a 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -110,7 +110,7 @@ 79 common settimeofday sys_settimeofday compat_sys_settimeofday 80 common getgroups sys_getgroups 81 common setgroups sys_setgroups -82 32 select ppc_select sys_ni_syscall +82 32 select sys_old_select compat_sys_old_select 82 64 select sys_ni_syscall 82 spu select sys_ni_syscall 83 common symlink sys_symlink @@ -178,9 +178,9 @@ 133 common fchdir sys_fchdir 134 common bdflush sys_ni_syscall 135 common sysfs sys_sysfs -136 32 personality sys_personality ppc64_personality -136 64 personality ppc64_personality -136 spu personality ppc64_personality +136 32 personality sys_personality compat_sys_ppc64_personality +136 64 personality sys_ppc64_personality +136 spu personality sys_ppc64_personality 137 common afs_syscall sys_ni_syscall 138 common setfsuid sys_setfsuid 139 common setfsgid sys_setfsgid @@ -228,8 +228,10 @@ 176 64 rt_sigtimedwait sys_rt_sigtimedwait 177 nospu rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo 178 nospu rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend -179 common pread64 sys_pread64 compat_sys_pread64 -180 common pwrite64 sys_pwrite64 compat_sys_pwrite64 +179 32 pread64 sys_ppc_pread64 compat_sys_ppc_pread64 +179 64 pread64 sys_pread64 +180 32 pwrite64 sys_ppc_pwrite64 compat_sys_ppc_pwrite64 +180 64 pwrite64 sys_pwrite64 181 common chown sys_chown 182 common getcwd sys_getcwd 183 common capget sys_capget @@ -242,10 +244,11 @@ 188 common putpmsg sys_ni_syscall 189 nospu vfork sys_vfork 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit -191 common readahead sys_readahead compat_sys_readahead +191 32 readahead sys_ppc_readahead compat_sys_ppc_readahead +191 64 readahead sys_readahead 192 32 mmap2 sys_mmap2 compat_sys_mmap2 -193 32 truncate64 sys_truncate64 compat_sys_truncate64 -194 32 ftruncate64 sys_ftruncate64 compat_sys_ftruncate64 +193 32 truncate64 sys_ppc_truncate64 compat_sys_ppc_truncate64 +194 32 ftruncate64 sys_ppc_ftruncate64 compat_sys_ppc_ftruncate64 195 32 stat64 sys_stat64 196 32 lstat64 sys_lstat64 197 32 fstat64 sys_fstat64 @@ -288,7 +291,8 @@ 230 common io_submit sys_io_submit compat_sys_io_submit 231 common io_cancel sys_io_cancel 232 nospu set_tid_address sys_set_tid_address -233 common fadvise64 sys_fadvise64 ppc32_fadvise64 +233 32 fadvise64 sys_ppc32_fadvise64 compat_sys_ppc32_fadvise64 +233 64 fadvise64 sys_fadvise64 234 nospu exit_group sys_exit_group 235 nospu lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie 236 common epoll_create sys_epoll_create @@ -323,7 +327,7 @@ 251 spu utimes sys_utimes 252 common statfs64 sys_statfs64 compat_sys_statfs64 253 common fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 -254 32 fadvise64_64 ppc_fadvise64_64 +254 32 fadvise64_64 sys_ppc_fadvise64_64 254 spu fadvise64_64 sys_ni_syscall 255 common rtas sys_rtas 256 32 sys_debug_setcontext sys_debug_setcontext sys_ni_syscall @@ -390,7 +394,7 @@ 305 common signalfd sys_signalfd compat_sys_signalfd 306 common timerfd_create sys_timerfd_create 307 common eventfd sys_eventfd -308 common sync_file_range2 sys_sync_file_range2 compat_sys_sync_file_range2 +308 common sync_file_range2 sys_sync_file_range2 compat_sys_ppc_sync_file_range2 309 nospu fallocate sys_fallocate compat_sys_fallocate 310 nospu subpage_prot sys_subpage_prot 311 32 timerfd_settime sys_timerfd_settime32 diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index 3a10cda9c05ec775787e7199ed5adc4b7ccc6ce7..ef9a61718940369f887c187bddf0a6dbff3def06 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -228,7 +228,7 @@ static void __init sysfs_create_dscr_default(void) } #endif /* CONFIG_PPC64 */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 #define MAX_BIT 63 static u64 pw20_wt; @@ -907,7 +907,7 @@ static int register_cpu_online(unsigned int cpu) device_create_file(s, &dev_attr_tscr); #endif /* CONFIG_PPC64 */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (PVR_VER(cur_cpu_spec->pvr_value) == PVR_VER_E6500) { device_create_file(s, &dev_attr_pw20_state); device_create_file(s, &dev_attr_pw20_wait_time); @@ -1003,7 +1003,7 @@ static int unregister_cpu_online(unsigned int cpu) device_remove_file(s, &dev_attr_tscr); #endif /* CONFIG_PPC64 */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (PVR_VER(cur_cpu_spec->pvr_value) == PVR_VER_E6500) { device_remove_file(s, &dev_attr_pw20_state); device_remove_file(s, &dev_attr_pw20_wait_time); diff --git a/arch/powerpc/kernel/systbl.S b/arch/powerpc/kernel/systbl.c similarity index 52% rename from arch/powerpc/kernel/systbl.S rename to arch/powerpc/kernel/systbl.c index cb3358886203e9028182c82e577954057c8cb521..4305f2a2162fe7cc2ce6d36980b70b354ca71000 100644 --- a/arch/powerpc/kernel/systbl.S +++ b/arch/powerpc/kernel/systbl.c @@ -10,31 +10,37 @@ * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) */ -#include +#include +#include +#include +#include -.section .rodata,"a" +#undef __SYSCALL_WITH_COMPAT +#define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry) -#ifdef CONFIG_PPC64 - .p2align 3 -#define __SYSCALL(nr, entry) .8byte entry +#undef __SYSCALL +#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER +#define __SYSCALL(nr, entry) [nr] = entry, #else -#define __SYSCALL(nr, entry) .long entry +/* + * Coerce syscall handlers with arbitrary parameters to common type + * requires cast to void* to avoid -Wcast-function-type. + */ +#define __SYSCALL(nr, entry) [nr] = (void *) entry, #endif -#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) -.globl sys_call_table -sys_call_table: +const syscall_fn sys_call_table[] = { #ifdef CONFIG_PPC64 #include #else #include #endif +}; #ifdef CONFIG_COMPAT #undef __SYSCALL_WITH_COMPAT #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) -.globl compat_sys_call_table -compat_sys_call_table: -#define compat_sys_sigsuspend sys_sigsuspend +const syscall_fn compat_sys_call_table[] = { #include -#endif +}; +#endif /* CONFIG_COMPAT */ diff --git a/arch/powerpc/kernel/systbl_chk.sh b/arch/powerpc/kernel/systbl_chk.sh deleted file mode 100644 index c7ac3ed657c482423eb862670ea00467b35cc197..0000000000000000000000000000000000000000 --- a/arch/powerpc/kernel/systbl_chk.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0-or-later -# -# Just process the CPP output from systbl_chk.c and complain -# if anything is out of order. -# -# Copyright © 2008 IBM Corporation -# - -awk 'BEGIN { num = -1; } # Ignore the beginning of the file - /^#/ { next; } - /^[ \t]*$/ { next; } - /^START_TABLE/ { num = 0; next; } - /^END_TABLE/ { - if (num != $2) { - printf "Error: NR_syscalls (%s) is not one more than the last syscall (%s)\n", - $2, num - 1; - exit(1); - } - num = -1; # Ignore the rest of the file - } - { - if (num == -1) next; - if (($1 != -1) && ($1 != num)) { - printf "Error: Syscall %s out of order (expected %s)\n", - $1, num; - exit(1); - }; - num++; - }' "$1" diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 587adcc12860bd96f1867cea9e4121aff2b0c07c..a2ab397065c6667bdd26a2dcbd6561984b2158ed 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -178,92 +178,6 @@ static inline unsigned long read_spurr(unsigned long tb) return tb; } -#ifdef CONFIG_PPC_SPLPAR - -#include - -void (*dtl_consumer)(struct dtl_entry *, u64); - -/* - * Scan the dispatch trace log and count up the stolen time. - * Should be called with interrupts disabled. - */ -static u64 scan_dispatch_log(u64 stop_tb) -{ - u64 i = local_paca->dtl_ridx; - struct dtl_entry *dtl = local_paca->dtl_curr; - struct dtl_entry *dtl_end = local_paca->dispatch_log_end; - struct lppaca *vpa = local_paca->lppaca_ptr; - u64 tb_delta; - u64 stolen = 0; - u64 dtb; - - if (!dtl) - return 0; - - if (i == be64_to_cpu(vpa->dtl_idx)) - return 0; - while (i < be64_to_cpu(vpa->dtl_idx)) { - dtb = be64_to_cpu(dtl->timebase); - tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) + - be32_to_cpu(dtl->ready_to_enqueue_time); - barrier(); - if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) { - /* buffer has overflowed */ - i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG; - dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG); - continue; - } - if (dtb > stop_tb) - break; - if (dtl_consumer) - dtl_consumer(dtl, i); - stolen += tb_delta; - ++i; - ++dtl; - if (dtl == dtl_end) - dtl = local_paca->dispatch_log; - } - local_paca->dtl_ridx = i; - local_paca->dtl_curr = dtl; - return stolen; -} - -/* - * Accumulate stolen time by scanning the dispatch trace log. - * Called on entry from user mode. - */ -void notrace accumulate_stolen_time(void) -{ - u64 sst, ust; - struct cpu_accounting_data *acct = &local_paca->accounting; - - sst = scan_dispatch_log(acct->starttime_user); - ust = scan_dispatch_log(acct->starttime); - acct->stime -= sst; - acct->utime -= ust; - acct->steal_time += ust + sst; -} - -static inline u64 calculate_stolen_time(u64 stop_tb) -{ - if (!firmware_has_feature(FW_FEATURE_SPLPAR)) - return 0; - - if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) - return scan_dispatch_log(stop_tb); - - return 0; -} - -#else /* CONFIG_PPC_SPLPAR */ -static inline u64 calculate_stolen_time(u64 stop_tb) -{ - return 0; -} - -#endif /* CONFIG_PPC_SPLPAR */ - /* * Account time for a transition between system, hard irq * or soft irq state. @@ -322,7 +236,11 @@ static unsigned long vtime_delta(struct cpu_accounting_data *acct, *stime_scaled = vtime_delta_scaled(acct, now, stime); - *steal_time = calculate_stolen_time(now); + if (IS_ENABLED(CONFIG_PPC_SPLPAR) && + firmware_has_feature(FW_FEATURE_SPLPAR)) + *steal_time = pseries_calculate_stolen_time(now); + else + *steal_time = 0; return stime; } @@ -614,22 +532,23 @@ DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) return; } - /* - * Ensure a positive value is written to the decrementer, or - * else some CPUs will continue to take decrementer exceptions. - * When the PPC_WATCHDOG (decrementer based) is configured, - * keep this at most 31 bits, which is about 4 seconds on most - * systems, which gives the watchdog a chance of catching timer - * interrupt hard lockups. - */ - if (IS_ENABLED(CONFIG_PPC_WATCHDOG)) - set_dec(0x7fffffff); - else - set_dec(decrementer_max); - /* Conditionally hard-enable interrupts. */ - if (should_hard_irq_enable()) + if (should_hard_irq_enable()) { + /* + * Ensure a positive value is written to the decrementer, or + * else some CPUs will continue to take decrementer exceptions. + * When the PPC_WATCHDOG (decrementer based) is configured, + * keep this at most 31 bits, which is about 4 seconds on most + * systems, which gives the watchdog a chance of catching timer + * interrupt hard lockups. + */ + if (IS_ENABLED(CONFIG_PPC_WATCHDOG)) + set_dec(0x7fffffff); + else + set_dec(decrementer_max); + do_hard_irq_enable(); + } #if defined(CONFIG_PPC32) && defined(CONFIG_PPC_PMAC) if (atomic_read(&ppc_n_lost_interrupts) != 0) diff --git a/arch/powerpc/kernel/trace/ftrace_low.S b/arch/powerpc/kernel/trace/ftrace_low.S index 0bddf1fa663619aae35e56b1d3bdfad35ee2393e..294d1e05958aaeff6f4497273eca0a96c85135f7 100644 --- a/arch/powerpc/kernel/trace/ftrace_low.S +++ b/arch/powerpc/kernel/trace/ftrace_low.S @@ -48,7 +48,7 @@ _GLOBAL(return_to_handler) * We might be called from a module. * Switch to our TOC to run inside the core kernel. */ - ld r2, PACATOC(r13) + LOAD_PACA_TOC() #else stwu r1, -16(r1) stw r3, 8(r1) diff --git a/arch/powerpc/kernel/trace/ftrace_mprofile.S b/arch/powerpc/kernel/trace/ftrace_mprofile.S index 4fa23e260cabe00dd63382653e52368d150dc9d7..d031093bc43671309e07c52ea300842e35246b04 100644 --- a/arch/powerpc/kernel/trace/ftrace_mprofile.S +++ b/arch/powerpc/kernel/trace/ftrace_mprofile.S @@ -83,10 +83,8 @@ #ifdef CONFIG_PPC64 /* Save callee's TOC in the ABI compliant location */ std r2, STK_GOT(r1) - ld r2,PACATOC(r13) /* get kernel TOC in r2 */ - - addis r3,r2,function_trace_op@toc@ha - addi r3,r3,function_trace_op@toc@l + LOAD_PACA_TOC() /* get kernel TOC in r2 */ + LOAD_REG_ADDR(r3, function_trace_op) ld r5,0(r3) #else lis r3,function_trace_op@ha diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index dadfcef5d6db4bb0b1e89f38122deeabab0f11c1..9bdd79aa51cfc12422476e00b4994534b7192a54 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -68,6 +68,7 @@ #include #include #include +#include #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) int (*__debugger)(struct pt_regs *regs) __read_mostly; @@ -600,7 +601,7 @@ static inline int check_io_access(struct pt_regs *regs) #define inst_length(reason) (((reason) & REASON_PREFIXED) ? 8 : 4) -#if defined(CONFIG_E500) +#if defined(CONFIG_PPC_E500) int machine_check_e500mc(struct pt_regs *regs) { unsigned long mcsr = mfspr(SPRN_MCSR); @@ -850,6 +851,19 @@ bail: } #ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER_RAW(machine_check_early_boot) +{ + udbg_printf("Machine check (early boot)\n"); + udbg_printf("SRR0=0x%016lx SRR1=0x%016lx\n", regs->nip, regs->msr); + udbg_printf(" DAR=0x%016lx DSISR=0x%08lx\n", regs->dar, regs->dsisr); + udbg_printf(" LR=0x%016lx R1=0x%08lx\n", regs->link, regs->gpr[1]); + udbg_printf("------\n"); + die("Machine check (early boot)", regs, SIGBUS); + for (;;) + ; + return 0; +} + DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async) { __machine_check_exception(regs); @@ -2085,7 +2099,7 @@ DEFINE_INTERRUPT_HANDLER(altivec_assist_exception) } #endif /* CONFIG_ALTIVEC */ -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx DEFINE_INTERRUPT_HANDLER(CacheLockingException) { unsigned long error_code = regs->dsisr; @@ -2098,12 +2112,11 @@ DEFINE_INTERRUPT_HANDLER(CacheLockingException) _exception(SIGILL, regs, ILL_PRVOPC, regs->nip); return; } -#endif /* CONFIG_FSL_BOOKE */ +#endif /* CONFIG_PPC_85xx */ #ifdef CONFIG_SPE DEFINE_INTERRUPT_HANDLER(SPEFloatingPointException) { - extern int do_spe_mathemu(struct pt_regs *regs); unsigned long spefscr; int fpexc_mode; int code = FPE_FLTUNK; @@ -2153,7 +2166,6 @@ DEFINE_INTERRUPT_HANDLER(SPEFloatingPointException) DEFINE_INTERRUPT_HANDLER(SPEFloatingPointRoundException) { - extern int speround_handler(struct pt_regs *regs); int err; interrupt_cond_local_irq_enable(regs); diff --git a/arch/powerpc/kernel/udbg.c b/arch/powerpc/kernel/udbg.c index b1544b2f6321b2746ef553ad5e30dd14eda49d40..92b3fc258d1108fd66c3d411211c4878a84b341e 100644 --- a/arch/powerpc/kernel/udbg.c +++ b/arch/powerpc/kernel/udbg.c @@ -67,6 +67,8 @@ void __init udbg_early_init(void) udbg_init_debug_opal_raw(); #elif defined(CONFIG_PPC_EARLY_DEBUG_OPAL_HVSI) udbg_init_debug_opal_hvsi(); +#elif defined(CONFIG_PPC_EARLY_DEBUG_16550) + udbg_init_debug_16550(); #endif #ifdef CONFIG_PPC_EARLY_DEBUG diff --git a/arch/powerpc/kernel/udbg_16550.c b/arch/powerpc/kernel/udbg_16550.c index d3942de254c68d01f0a2e5ba2c6157609f3e6470..74ddf836f7a264ed37fdaa14f64cb06eede4d2dc 100644 --- a/arch/powerpc/kernel/udbg_16550.c +++ b/arch/powerpc/kernel/udbg_16550.c @@ -8,6 +8,7 @@ #include #include #include +#include extern u8 real_readb(volatile u8 __iomem *addr); extern void real_writeb(u8 data, volatile u8 __iomem *addr); @@ -297,41 +298,34 @@ void __init udbg_init_40x_realmode(void) #endif /* CONFIG_PPC_EARLY_DEBUG_40x */ -#ifdef CONFIG_PPC_EARLY_DEBUG_MICROWATT +#ifdef CONFIG_PPC_EARLY_DEBUG_16550 -#define UDBG_UART_MW_ADDR ((void __iomem *)0xc0002000) +static void __iomem *udbg_uart_early_addr; -static u8 udbg_uart_in_isa300_rm(unsigned int reg) +void __init udbg_init_debug_16550(void) { - uint64_t msr = mfmsr(); - uint8_t c; - - mtmsr(msr & ~(MSR_EE|MSR_DR)); - isync(); - eieio(); - c = __raw_rm_readb(UDBG_UART_MW_ADDR + (reg << 2)); - mtmsr(msr); - isync(); - return c; + udbg_uart_early_addr = early_ioremap(CONFIG_PPC_EARLY_DEBUG_16550_PHYSADDR, 0x1000); + udbg_uart_init_mmio(udbg_uart_early_addr, CONFIG_PPC_EARLY_DEBUG_16550_STRIDE); } -static void udbg_uart_out_isa300_rm(unsigned int reg, u8 val) +static int __init udbg_init_debug_16550_ioremap(void) { - uint64_t msr = mfmsr(); - - mtmsr(msr & ~(MSR_EE|MSR_DR)); - isync(); - eieio(); - __raw_rm_writeb(val, UDBG_UART_MW_ADDR + (reg << 2)); - mtmsr(msr); - isync(); -} + void __iomem *addr; -void __init udbg_init_debug_microwatt(void) -{ - udbg_uart_in = udbg_uart_in_isa300_rm; - udbg_uart_out = udbg_uart_out_isa300_rm; - udbg_use_uart(); + if (!udbg_uart_early_addr) + return 0; + + addr = ioremap(CONFIG_PPC_EARLY_DEBUG_16550_PHYSADDR, 0x1000); + if (WARN_ON(!addr)) + return -ENOMEM; + + udbg_uart_init_mmio(addr, CONFIG_PPC_EARLY_DEBUG_16550_STRIDE); + early_iounmap(udbg_uart_early_addr, 0x1000); + udbg_uart_early_addr = NULL; + + return 0; } -#endif /* CONFIG_PPC_EARLY_DEBUG_MICROWATT */ +early_initcall(udbg_init_debug_16550_ioremap); + +#endif /* CONFIG_PPC_EARLY_DEBUG_16550 */ diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 0da287544054f55000dc8df90a402be7d5615466..4abc0194970208713f428ec0eb4545d8016ea481 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -39,6 +39,8 @@ extern char vdso32_start, vdso32_end; extern char vdso64_start, vdso64_end; +long sys_ni_syscall(void); + /* * The vdso data page (aka. systemcfg for old ppc64 fans) is here. * Once the early boot kernel code no longer needs to muck around @@ -113,18 +115,18 @@ struct vdso_data *arch_get_vdso_data(void *vvar_page) int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; mmap_read_lock(mm); - - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, &vvar_spec)) zap_page_range(vma, vma->vm_start, size); } - mmap_read_unlock(mm); + return 0; } @@ -200,28 +202,19 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int if (is_32bit_task()) { vdso_spec = &vdso32_spec; vdso_size = &vdso32_end - &vdso32_start; - vdso_base = VDSO32_MBASE; } else { vdso_spec = &vdso64_spec; vdso_size = &vdso64_end - &vdso64_start; - /* - * On 64bit we don't have a preferred map address. This - * allows get_unmapped_area to find an area near other mmaps - * and most likely share a SLB entry. - */ - vdso_base = 0; } mappings_size = vdso_size + vvar_size; mappings_size += (VDSO_ALIGNMENT - 1) & PAGE_MASK; /* - * pick a base address for the vDSO in process space. We try to put it - * at vdso_base which is the "natural" base for it, but we might fail - * and end up putting it elsewhere. + * Pick a base address for the vDSO in process space. * Add enough to the size so that the result can be aligned. */ - vdso_base = get_unmapped_area(NULL, vdso_base, mappings_size, 0, 0); + vdso_base = get_unmapped_area(NULL, 0, mappings_size, 0, 0); if (IS_ERR_VALUE(vdso_base)) return vdso_base; @@ -313,10 +306,10 @@ static void __init vdso_setup_syscall_map(void) unsigned int i; for (i = 0; i < NR_syscalls; i++) { - if (sys_call_table[i] != (unsigned long)&sys_ni_syscall) + if (sys_call_table[i] != (void *)&sys_ni_syscall) vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); if (IS_ENABLED(CONFIG_COMPAT) && - compat_sys_call_table[i] != (unsigned long)&sys_ni_syscall) + compat_sys_call_table[i] != (void *)&sys_ni_syscall) vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); } } diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 096b0bf1335ffc7d73fc15bb4fe23adfc4acd68a..a2e7b0ce5b1913536e1b24fb7072c1ea49c4687f 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -92,13 +92,13 @@ include/generated/vdso64-offsets.h: $(obj)/vdso64.so.dbg FORCE # actual build commands quiet_cmd_vdso32ld_and_check = VDSO32L $@ - cmd_vdso32ld_and_check = $(VDSOCC) $(c_flags) $(CC32FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^) ; $(cmd_vdso_check) + cmd_vdso32ld_and_check = $(VDSOCC) $(c_flags) $(CC32FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^) -z noexecstack ; $(cmd_vdso_check) quiet_cmd_vdso32as = VDSO32A $@ cmd_vdso32as = $(VDSOCC) $(a_flags) $(CC32FLAGS) $(AS32FLAGS) -c -o $@ $< quiet_cmd_vdso32cc = VDSO32C $@ cmd_vdso32cc = $(VDSOCC) $(c_flags) $(CC32FLAGS) -c -o $@ $< quiet_cmd_vdso64ld_and_check = VDSO64L $@ - cmd_vdso64ld_and_check = $(VDSOCC) $(c_flags) $(CC64FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^) ; $(cmd_vdso_check) + cmd_vdso64ld_and_check = $(VDSOCC) $(c_flags) $(CC64FLAGS) -o $@ -Wl,-T$(filter %.lds,$^) $(filter %.o,$^) -z noexecstack ; $(cmd_vdso_check) quiet_cmd_vdso64as = VDSO64A $@ cmd_vdso64as = $(VDSOCC) $(a_flags) $(CC64FLAGS) $(AS64FLAGS) -c -o $@ $< diff --git a/arch/powerpc/kernel/vdso/vdso32.lds.S b/arch/powerpc/kernel/vdso/vdso32.lds.S index e0d19d74455fd880c9f710b7572438d1fef0a4dc..bc0be274a9ac24d8ae87a4f5dddb63aefa4fa443 100644 --- a/arch/powerpc/kernel/vdso/vdso32.lds.S +++ b/arch/powerpc/kernel/vdso/vdso32.lds.S @@ -78,7 +78,6 @@ SECTIONS __end = .; PROVIDE(end = .); - STABS_DEBUG DWARF_DEBUG ELF_DETAILS diff --git a/arch/powerpc/kernel/vdso/vdso64.lds.S b/arch/powerpc/kernel/vdso/vdso64.lds.S index 1a4a7bc4c8157fa468ebc7a1468ab47424ad0e30..744ae5363e6c8581c1ae839729154e68f8989745 100644 --- a/arch/powerpc/kernel/vdso/vdso64.lds.S +++ b/arch/powerpc/kernel/vdso/vdso64.lds.S @@ -76,7 +76,6 @@ SECTIONS _end = .; PROVIDE(end = .); - STABS_DEBUG DWARF_DEBUG ELF_DETAILS diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S index 5cc24d8cce94e2ff2ad514297b07ee9674336c26..5cf64740edb82db60c9d14d0625dbbaf2d98da17 100644 --- a/arch/powerpc/kernel/vector.S +++ b/arch/powerpc/kernel/vector.S @@ -155,8 +155,8 @@ _GLOBAL(load_up_vsx) * usage of floating-point registers. These routines must be called * with preempt disabled. */ -#ifdef CONFIG_PPC32 .data +#ifdef CONFIG_PPC32 fpzero: .long 0 fpone: @@ -169,18 +169,17 @@ fphalf: lfs fr,name@l(r11) #else - .section ".toc","aw" fpzero: - .tc FD_0_0[TC],0 + .quad 0 fpone: - .tc FD_3ff00000_0[TC],0x3ff0000000000000 /* 1.0 */ + .quad 0x3ff0000000000000 /* 1.0 */ fphalf: - .tc FD_3fe00000_0[TC],0x3fe0000000000000 /* 0.5 */ + .quad 0x3fe0000000000000 /* 0.5 */ -#define LDCONST(fr, name) \ - lfd fr,name@toc(r2) +#define LDCONST(fr, name) \ + addis r11,r2,name@toc@ha; \ + lfd fr,name@toc@l(r11) #endif - .text /* * Internal routine to enable floating point and set FPSCR to 0. diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index fe22d940412fda1a0aec1dd01ba36b9b8875de1b..7786e3ac7611cf8dcbce38ab1fe3b2f4c83c5df4 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -32,6 +32,10 @@ #define STRICT_ALIGN_SIZE (1 << CONFIG_DATA_SHIFT) +#if STRICT_ALIGN_SIZE < PAGE_SIZE +#error "CONFIG_DATA_SHIFT must be >= PAGE_SHIFT" +#endif + ENTRY(_stext) PHDRS { @@ -67,7 +71,7 @@ SECTIONS .head.text : AT(ADDR(.head.text) - LOAD_OFFSET) { #ifdef CONFIG_PPC64 KEEP(*(.head.text.first_256B)); -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 #else KEEP(*(.head.text.real_vectors)); *(.head.text.real_trampolines); @@ -122,14 +126,6 @@ SECTIONS *(.sfpr); MEM_KEEP(init.text) MEM_KEEP(exit.text) - -#ifdef CONFIG_PPC32 - *(.got1) - __got2_start = .; - *(.got2) - __got2_end = .; -#endif /* CONFIG_PPC32 */ - } :text . = ALIGN(PAGE_SIZE); @@ -139,15 +135,57 @@ SECTIONS /* Read-only data */ RO_DATA(PAGE_SIZE) -#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC32 + .sdata2 : AT(ADDR(.sdata2) - LOAD_OFFSET) { + *(.sdata2) + } +#endif + + .data.rel.ro : AT(ADDR(.data.rel.ro) - LOAD_OFFSET) { + *(.data.rel.ro*) + } + + .branch_lt : AT(ADDR(.branch_lt) - LOAD_OFFSET) { + *(.branch_lt) + } + +#ifdef CONFIG_PPC32 + .got1 : AT(ADDR(.got1) - LOAD_OFFSET) { + *(.got1) + } + .got2 : AT(ADDR(.got2) - LOAD_OFFSET) { + __got2_start = .; + *(.got2) + __got2_end = .; + } + .got : AT(ADDR(.got) - LOAD_OFFSET) { + *(.got) + *(.got.plt) + } + .plt : AT(ADDR(.plt) - LOAD_OFFSET) { + /* XXX: is .plt (and .got.plt) required? */ + *(.plt) + } + +#else /* CONFIG_PPC32 */ + .toc1 : AT(ADDR(.toc1) - LOAD_OFFSET) { + *(.toc1) + } + + .got : AT(ADDR(.got) - LOAD_OFFSET) ALIGN(256) { + *(.got .toc) + } + SOFT_MASK_TABLE(8) RESTART_TABLE(8) +#ifdef CONFIG_PPC64_ELF_ABI_V1 .opd : AT(ADDR(.opd) - LOAD_OFFSET) { __start_opd = .; KEEP(*(.opd)) __end_opd = .; } +#endif . = ALIGN(8); __stf_entry_barrier_fixup : AT(ADDR(__stf_entry_barrier_fixup) - LOAD_OFFSET) { @@ -190,7 +228,7 @@ SECTIONS *(__rfi_flush_fixup) __stop___rfi_flush_fixup = .; } -#endif /* CONFIG_PPC64 */ +#endif /* CONFIG_PPC32 */ #ifdef CONFIG_PPC_BARRIER_NOSPEC . = ALIGN(8); @@ -201,7 +239,7 @@ SECTIONS } #endif /* CONFIG_PPC_BARRIER_NOSPEC */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 . = ALIGN(8); __spec_btb_flush_fixup : AT(ADDR(__spec_btb_flush_fixup) - LOAD_OFFSET) { __start__btb_flush_fixup = .; @@ -210,12 +248,17 @@ SECTIONS } #endif + /* + * Various code relies on __init_begin being at the strict RWX boundary. + */ + . = ALIGN(STRICT_ALIGN_SIZE); + __srwx_boundary = .; + __end_rodata = .; + __init_begin = .; + /* * Init sections discarded at runtime */ - . = ALIGN(STRICT_ALIGN_SIZE); - __init_begin = .; - . = ALIGN(PAGE_SIZE); .init.text : AT(ADDR(.init.text) - LOAD_OFFSET) { _sinittext = .; INIT_TEXT @@ -317,34 +360,13 @@ SECTIONS . = ALIGN(PAGE_SIZE); _sdata = .; -#ifdef CONFIG_PPC32 .data : AT(ADDR(.data) - LOAD_OFFSET) { DATA_DATA *(.data.rel*) +#ifdef CONFIG_PPC32 *(SDATA_MAIN) - *(.sdata2) - *(.got.plt) *(.got) - *(.plt) - *(.branch_lt) - } -#else - .data : AT(ADDR(.data) - LOAD_OFFSET) { - DATA_DATA - *(.data.rel*) - *(.toc1) - *(.branch_lt) - } - - .got : AT(ADDR(.got) - LOAD_OFFSET) ALIGN(256) { - *(.got) -#ifndef CONFIG_RELOCATABLE - __prom_init_toc_start = .; - arch/powerpc/kernel/prom_init.o*(.toc) - __prom_init_toc_end = .; #endif - *(.toc) } -#endif /* The initial task and kernel stack */ INIT_TASK_DATA_SECTION(THREAD_ALIGN) @@ -382,7 +404,6 @@ SECTIONS _end = . ; PROVIDE32 (end = .); - STABS_DEBUG DWARF_DEBUG ELF_DETAILS diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index cf84bfe9e27e57a631540278d8c5bce0f30e400a..de64c7962991214b285626a4cb79b96643c10f56 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -136,7 +136,7 @@ void __init reserve_crashkernel(void) #ifdef CONFIG_PPC64 /* * On the LPAR platform place the crash kernel to mid of - * RMA size (512MB or more) to ensure the crash kernel + * RMA size (max. of 512MB) to ensure the crash kernel * gets enough space to place itself and some stack to be * in the first segment. At the same time normal kernel * also get enough space to allocate memory for essential @@ -144,9 +144,9 @@ void __init reserve_crashkernel(void) * kernel starts at 128MB offset on other platforms. */ if (firmware_has_feature(FW_FEATURE_LPAR)) - crashk_res.start = ppc64_rma_size / 2; + crashk_res.start = min_t(u64, ppc64_rma_size / 2, SZ_512M); else - crashk_res.start = min(0x8000000ULL, (ppc64_rma_size / 2)); + crashk_res.start = min_t(u64, ppc64_rma_size / 2, SZ_128M); #else crashk_res.start = KDUMP_KERNELBASE; #endif diff --git a/arch/powerpc/kexec/core_32.c b/arch/powerpc/kexec/core_32.c index b50aed48d09db628443e653a76db9758daee0d47..c95f96850c9e13350d80ebc4f85cc0b00ba3215e 100644 --- a/arch/powerpc/kexec/core_32.c +++ b/arch/powerpc/kexec/core_32.c @@ -55,7 +55,7 @@ void default_machine_kexec(struct kimage *image) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE); printk(KERN_INFO "Bye!\n"); - if (!IS_ENABLED(CONFIG_FSL_BOOKE) && !IS_ENABLED(CONFIG_44x)) + if (!IS_ENABLED(CONFIG_PPC_85xx) && !IS_ENABLED(CONFIG_44x)) relocate_new_kernel(page_list, reboot_code_buffer_phys, image->start); /* now call it */ diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c index c2bea9db1c1e145fbdaf2624137f209bb5cd2fe8..a79e28c91e2be312efb2f1bf177b14e4c07adb66 100644 --- a/arch/powerpc/kexec/core_64.c +++ b/arch/powerpc/kexec/core_64.c @@ -360,7 +360,7 @@ void default_machine_kexec(struct kimage *image) * the RMA. On BookE there is no real MMU off mode, so we have to * keep it enabled as well (but then we have bolted TLB entries). */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 copy_with_mmu_off = false; #else copy_with_mmu_off = radix_enabled() || diff --git a/arch/powerpc/kexec/relocate_32.S b/arch/powerpc/kexec/relocate_32.S index cf6e52bdf8d8331565cdf82e8483ef750ae3fb0a..d9f0dd9b34ffbfcb9cc6274bed12dfa3886ebef1 100644 --- a/arch/powerpc/kexec/relocate_32.S +++ b/arch/powerpc/kexec/relocate_32.S @@ -25,14 +25,14 @@ relocate_new_kernel: /* r4 = reboot_code_buffer */ /* r5 = start_address */ -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx mr r29, r3 mr r30, r4 mr r31, r5 #define ENTRY_MAPPING_KEXEC_SETUP -#include +#include #undef ENTRY_MAPPING_KEXEC_SETUP mr r3, r29 diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index dcb398d5e0093f69c53db0c9f07b76653aac5b67..61cdd782d3c5ec6cec8b70e9799644a6df7e2414 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -189,7 +189,7 @@ config KVM_EXIT_TIMING config KVM_E500V2 bool "KVM support for PowerPC E500v2 processors" - depends on E500 && !PPC_E500MC + depends on PPC_E500 && !PPC_E500MC select KVM select KVM_MMIO select MMU_NOTIFIER @@ -220,7 +220,7 @@ config KVM_E500MC config KVM_MPIC bool "KVM in-kernel MPIC emulation" - depends on KVM && E500 + depends on KVM && PPC_E500 select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING diff --git a/arch/powerpc/kvm/book3s_64_entry.S b/arch/powerpc/kvm/book3s_64_entry.S index e43704547a1eee455c44ed291aa6044413c796af..6c2b1d17cb636a9c09ef5317c3a2aea1983af4bf 100644 --- a/arch/powerpc/kvm/book3s_64_entry.S +++ b/arch/powerpc/kvm/book3s_64_entry.S @@ -315,7 +315,7 @@ kvmppc_p9_exit_interrupt: reg = reg + 1 .endr - ld r2,PACATOC(r13) + LOAD_PACA_TOC() mflr r4 std r4,VCPU_LR(r3) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 57d0835e56fdb99ba2be1bc16900d05a1c622448..6ba68dd6190bd30a4ab31c417df2e909077d62a5 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -249,6 +249,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) /* * We use the vcpu_load/put functions to measure stolen time. + * * Stolen time is counted as time when either the vcpu is able to * run as part of a virtual core, but the task running the vcore * is preempted or sleeping, or when the vcpu needs something done @@ -278,6 +279,12 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * lock. The stolen times are measured in units of timebase ticks. * (Note that the != TB_NIL checks below are purely defensive; * they should never fail.) + * + * The POWER9 path is simpler, one vcpu per virtual core so the + * former case does not exist. If a vcpu is preempted when it is + * BUSY_IN_HOST and not ceded or otherwise blocked, then accumulate + * the stolen cycles in busy_stolen. RUNNING is not a preemptible + * state in the P9 path. */ static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc, u64 tb) @@ -311,8 +318,14 @@ static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) unsigned long flags; u64 now; - if (cpu_has_feature(CPU_FTR_ARCH_300)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + if (vcpu->arch.busy_preempt != TB_NIL) { + WARN_ON_ONCE(vcpu->arch.state != KVMPPC_VCPU_BUSY_IN_HOST); + vc->stolen_tb += mftb() - vcpu->arch.busy_preempt; + vcpu->arch.busy_preempt = TB_NIL; + } return; + } now = mftb(); @@ -340,8 +353,21 @@ static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) unsigned long flags; u64 now; - if (cpu_has_feature(CPU_FTR_ARCH_300)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + /* + * In the P9 path, RUNNABLE is not preemptible + * (nor takes host interrupts) + */ + WARN_ON_ONCE(vcpu->arch.state == KVMPPC_VCPU_RUNNABLE); + /* + * Account stolen time when preempted while the vcpu task is + * running in the kernel (but not in qemu, which is INACTIVE). + */ + if (task_is_running(current) && + vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) + vcpu->arch.busy_preempt = mftb(); return; + } now = mftb(); @@ -707,16 +733,15 @@ static u64 vcore_stolen_time(struct kvmppc_vcore *vc, u64 now) } static void __kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, + struct lppaca *vpa, unsigned int pcpu, u64 now, unsigned long stolen) { struct dtl_entry *dt; - struct lppaca *vpa; dt = vcpu->arch.dtl_ptr; - vpa = vcpu->arch.vpa.pinned_addr; - if (!dt || !vpa) + if (!dt) return; dt->dispatch_reason = 7; @@ -737,17 +762,23 @@ static void __kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, /* order writing *dt vs. writing vpa->dtl_idx */ smp_wmb(); vpa->dtl_idx = cpu_to_be64(++vcpu->arch.dtl_index); - vcpu->arch.dtl.dirty = true; + + /* vcpu->arch.dtl.dirty is set by the caller */ } -static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, - struct kvmppc_vcore *vc) +static void kvmppc_update_vpa_dispatch(struct kvm_vcpu *vcpu, + struct kvmppc_vcore *vc) { + struct lppaca *vpa; unsigned long stolen; unsigned long core_stolen; u64 now; unsigned long flags; + vpa = vcpu->arch.vpa.pinned_addr; + if (!vpa) + return; + now = mftb(); core_stolen = vcore_stolen_time(vc, now); @@ -758,7 +789,34 @@ static void kvmppc_create_dtl_entry(struct kvm_vcpu *vcpu, vcpu->arch.busy_stolen = 0; spin_unlock_irqrestore(&vcpu->arch.tbacct_lock, flags); - __kvmppc_create_dtl_entry(vcpu, vc->pcpu, now + vc->tb_offset, stolen); + vpa->enqueue_dispatch_tb = cpu_to_be64(be64_to_cpu(vpa->enqueue_dispatch_tb) + stolen); + + __kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now + vc->tb_offset, stolen); + + vcpu->arch.vpa.dirty = true; +} + +static void kvmppc_update_vpa_dispatch_p9(struct kvm_vcpu *vcpu, + struct kvmppc_vcore *vc, + u64 now) +{ + struct lppaca *vpa; + unsigned long stolen; + unsigned long stolen_delta; + + vpa = vcpu->arch.vpa.pinned_addr; + if (!vpa) + return; + + stolen = vc->stolen_tb; + stolen_delta = stolen - vcpu->arch.stolen_logged; + vcpu->arch.stolen_logged = stolen; + + vpa->enqueue_dispatch_tb = cpu_to_be64(stolen); + + __kvmppc_create_dtl_entry(vcpu, vpa, vc->pcpu, now, stolen_delta); + + vcpu->arch.vpa.dirty = true; } /* See if there is a doorbell interrupt pending for a vcpu */ @@ -2517,10 +2575,24 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); break; case KVM_REG_PPC_TB_OFFSET: + { /* round up to multiple of 2^24 */ - vcpu->arch.vcore->tb_offset = - ALIGN(set_reg_val(id, *val), 1UL << 24); + u64 tb_offset = ALIGN(set_reg_val(id, *val), 1UL << 24); + + /* + * Now that we know the timebase offset, update the + * decrementer expiry with a guest timebase value. If + * the userspace does not set DEC_EXPIRY, this ensures + * a migrated vcpu at least starts with an expired + * decrementer, which is better than a large one that + * causes a hang. + */ + if (!vcpu->arch.dec_expires && tb_offset) + vcpu->arch.dec_expires = get_tb() + tb_offset; + + vcpu->arch.vcore->tb_offset = tb_offset; break; + } case KVM_REG_PPC_LPCR: kvmppc_set_lpcr(vcpu, set_reg_val(id, *val), true); break; @@ -3800,7 +3872,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) * kvmppc_core_prepare_to_enter. */ kvmppc_start_thread(vcpu, pvc); - kvmppc_create_dtl_entry(vcpu, pvc); + kvmppc_update_vpa_dispatch(vcpu, pvc); trace_kvm_guest_enter(vcpu); if (!vcpu->arch.ptid) thr0_done = true; @@ -3840,23 +3912,17 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) for (sub = 0; sub < core_info.n_subcores; ++sub) spin_unlock(&core_info.vc[sub]->lock); - guest_enter_irqoff(); + guest_timing_enter_irqoff(); srcu_idx = srcu_read_lock(&vc->kvm->srcu); + guest_state_enter_irqoff(); this_cpu_disable_ftrace(); - /* - * Interrupts will be enabled once we get into the guest, - * so tell lockdep that we're about to enable interrupts. - */ - trace_hardirqs_on(); - trap = __kvmppc_vcore_entry(); - trace_hardirqs_off(); - this_cpu_enable_ftrace(); + guest_state_exit_irqoff(); srcu_read_unlock(&vc->kvm->srcu, srcu_idx); @@ -3891,11 +3957,10 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) kvmppc_set_host_core(pcpu); - context_tracking_guest_exit(); if (!vtime_accounting_enabled_this_cpu()) { local_irq_enable(); /* - * Service IRQs here before vtime_account_guest_exit() so any + * Service IRQs here before guest_timing_exit_irqoff() so any * ticks that occurred while running the guest are accounted to * the guest. If vtime accounting is enabled, accounting uses * TB rather than ticks, so it can be done without enabling @@ -3904,7 +3969,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) */ local_irq_disable(); } - vtime_account_guest_exit(); + guest_timing_exit_irqoff(); local_irq_enable(); @@ -4404,7 +4469,7 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu) if ((vc->vcore_state == VCORE_PIGGYBACK || vc->vcore_state == VCORE_RUNNING) && !VCORE_IS_EXITING(vc)) { - kvmppc_create_dtl_entry(vcpu, vc); + kvmppc_update_vpa_dispatch(vcpu, vc); kvmppc_start_thread(vcpu, vc); trace_kvm_guest_enter(vcpu); } else if (vc->vcore_state == VCORE_SLEEPING) { @@ -4520,7 +4585,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vc = vcpu->arch.vcore; vcpu->arch.ceded = 0; vcpu->arch.run_task = current; - vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; /* See if the MMU is ready to go */ @@ -4547,6 +4611,8 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, /* flags save not required, but irq_pmu has no disable/enable API */ powerpc_local_irq_pmu_save(flags); + vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; + if (signal_pending(current)) goto sigpend; if (need_resched() || !kvm->arch.mmu_ready) @@ -4591,47 +4657,44 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, tb = mftb(); - __kvmppc_create_dtl_entry(vcpu, pcpu, tb + vc->tb_offset, 0); + kvmppc_update_vpa_dispatch_p9(vcpu, vc, tb + vc->tb_offset); trace_kvm_guest_enter(vcpu); - guest_enter_irqoff(); + guest_timing_enter_irqoff(); srcu_idx = srcu_read_lock(&kvm->srcu); + guest_state_enter_irqoff(); this_cpu_disable_ftrace(); - /* Tell lockdep that we're about to enable interrupts */ - trace_hardirqs_on(); - trap = kvmhv_p9_guest_entry(vcpu, time_limit, lpcr, &tb); vcpu->arch.trap = trap; - trace_hardirqs_off(); - this_cpu_enable_ftrace(); + guest_state_exit_irqoff(); srcu_read_unlock(&kvm->srcu, srcu_idx); set_irq_happened(trap); - context_tracking_guest_exit(); + vcpu->cpu = -1; + vcpu->arch.thread_cpu = -1; + vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; + if (!vtime_accounting_enabled_this_cpu()) { - local_irq_enable(); + powerpc_local_irq_pmu_restore(flags); /* - * Service IRQs here before vtime_account_guest_exit() so any + * Service IRQs here before guest_timing_exit_irqoff() so any * ticks that occurred while running the guest are accounted to * the guest. If vtime accounting is enabled, accounting uses * TB rather than ticks, so it can be done without enabling * interrupts here, which has the problem that it accounts * interrupt processing overhead to the host. */ - local_irq_disable(); + powerpc_local_irq_pmu_save(flags); } - vtime_account_guest_exit(); - - vcpu->cpu = -1; - vcpu->arch.thread_cpu = -1; + guest_timing_exit_irqoff(); powerpc_local_irq_pmu_restore(flags); @@ -4694,6 +4757,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, out: vcpu->cpu = -1; vcpu->arch.thread_cpu = -1; + vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; powerpc_local_irq_pmu_restore(flags); preempt_enable(); goto done; diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 7ded202bf9959978af0eb0284f5e96c02c8c5c08..37f50861dd98fe538af9424d68424675dee3e612 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -1024,7 +1024,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) /* Restore R1/R2 so we can handle faults */ ld r1, HSTATE_HOST_R1(r13) - ld r2, PACATOC(r13) + LOAD_PACA_TOC() mfspr r10, SPRN_SRR0 mfspr r11, SPRN_SRR1 @@ -2727,8 +2727,8 @@ kvmppc_bad_host_intr: std r4, _CTR(r1) std r5, _XER(r1) std r6, SOFTE(r1) - ld r2, PACATOC(r13) - LOAD_REG_IMMEDIATE(3, 0x7265677368657265) + LOAD_PACA_TOC() + LOAD_REG_IMMEDIATE(3, STACK_FRAME_REGS_MARKER) std r3, STACK_FRAME_OVERHEAD-16(r1) /* diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index 5980063016207f94594cfdbb22f3b966e7f56974..e2f11f9c3f2aa691bd4b1a130d06d38d989ecc5c 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -508,10 +508,10 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) static int __kvmppc_svm_page_out(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long page_shift, - struct kvm *kvm, unsigned long gpa) + struct kvm *kvm, unsigned long gpa, struct page *fault_page) { unsigned long src_pfn, dst_pfn = 0; - struct migrate_vma mig; + struct migrate_vma mig = { 0 }; struct page *dpage, *spage; struct kvmppc_uvmem_page_pvt *pvt; unsigned long pfn; @@ -525,6 +525,7 @@ static int __kvmppc_svm_page_out(struct vm_area_struct *vma, mig.dst = &dst_pfn; mig.pgmap_owner = &kvmppc_uvmem_pgmap; mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE; + mig.fault_page = fault_page; /* The requested page is already paged-out, nothing to do */ if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL)) @@ -580,12 +581,14 @@ out_finalize: static inline int kvmppc_svm_page_out(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long page_shift, - struct kvm *kvm, unsigned long gpa) + struct kvm *kvm, unsigned long gpa, + struct page *fault_page) { int ret; mutex_lock(&kvm->arch.uvmem_lock); - ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa); + ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, + fault_page); mutex_unlock(&kvm->arch.uvmem_lock); return ret; @@ -634,7 +637,7 @@ void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot, pvt->remove_gfn = true; if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE, - PAGE_SHIFT, kvm, pvt->gpa)) + PAGE_SHIFT, kvm, pvt->gpa, NULL)) pr_err("Can't page out gpa:0x%lx addr:0x%lx\n", pvt->gpa, addr); } else { @@ -715,7 +718,7 @@ static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) dpage = pfn_to_page(uvmem_pfn); dpage->zone_device_data = pvt; - lock_page(dpage); + zone_device_page_init(dpage); return dpage; out_clear: spin_lock(&kvmppc_uvmem_bitmap_lock); @@ -736,7 +739,7 @@ static int kvmppc_svm_page_in(struct vm_area_struct *vma, bool pagein) { unsigned long src_pfn, dst_pfn = 0; - struct migrate_vma mig; + struct migrate_vma mig = { 0 }; struct page *spage; unsigned long pfn; struct page *dpage; @@ -994,7 +997,7 @@ static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf) if (kvmppc_svm_page_out(vmf->vma, vmf->address, vmf->address + PAGE_SIZE, PAGE_SHIFT, - pvt->kvm, pvt->gpa)) + pvt->kvm, pvt->gpa, vmf->page)) return VM_FAULT_SIGBUS; else return 0; @@ -1065,7 +1068,7 @@ kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa, if (!vma || vma->vm_start > start || vma->vm_end < end) goto out; - if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa)) + if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL)) ret = H_SUCCESS; out: mmap_read_unlock(kvm->mm); diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index d6abed6e51e6995a66df51b59d544fce56a6dfee..9fc4dd8f66ebc365b1e211c880f298d90266167e 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -499,7 +499,6 @@ static void kvmppc_set_msr_pr(struct kvm_vcpu *vcpu, u64 msr) if (msr & MSR_POW) { if (!vcpu->arch.pending_exceptions) { kvm_vcpu_halt(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); vcpu->stat.generic.halt_wakeup++; /* Unset POW bit after we woke up */ diff --git a/arch/powerpc/kvm/book3s_pr_papr.c b/arch/powerpc/kvm/book3s_pr_papr.c index a1f2978b2a8654ff2daddc6834a66e5166d41229..b2c89e850d7ae84f87b658057318dca9c8bfc3dc 100644 --- a/arch/powerpc/kvm/book3s_pr_papr.c +++ b/arch/powerpc/kvm/book3s_pr_papr.c @@ -393,7 +393,6 @@ int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd) case H_CEDE: kvmppc_set_msr_fast(vcpu, kvmppc_get_msr(vcpu) | MSR_EE); kvm_vcpu_halt(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); vcpu->stat.generic.halt_wakeup++; return EMULATE_DONE; case H_LOGICAL_CI_LOAD: diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 06c5830a93f9d6cd8f849dbd7aad0f619e5aceb3..7b4920e9fd26304bcf47ee43dd746d162f8d63a8 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -719,7 +719,6 @@ int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu) if (vcpu->arch.shared->msr & MSR_WE) { local_irq_enable(); kvm_vcpu_halt(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); hard_irq_disable(); kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS); diff --git a/arch/powerpc/kvm/booke_interrupts.S b/arch/powerpc/kvm/booke_interrupts.S index 6fa82efe833bc4f8e7fdc71862cecfe8120f3010..205545d820a115b3af6e414be4de8d56802d2237 100644 --- a/arch/powerpc/kvm/booke_interrupts.S +++ b/arch/powerpc/kvm/booke_interrupts.S @@ -223,7 +223,7 @@ _GLOBAL(kvmppc_resume_host) lwz r3, VCPU_HOST_PID(r4) mtspr SPRN_PID, r3 -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx /* we cheat and know that Linux doesn't use PID1 which is always 0 */ lis r3, 0 mtspr SPRN_PID1, r3 @@ -406,7 +406,7 @@ lightweight_exit: lwz r3, VCPU_SHADOW_PID(r4) mtspr SPRN_PID, r3 -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx lwz r3, VCPU_SHADOW_PID1(r4) mtspr SPRN_PID1, r3 #endif diff --git a/arch/powerpc/kvm/e500.h b/arch/powerpc/kvm/e500.h index c3ef751465fb300f07210e4773de166900791e9a..6d0d329cbb35c2321c7ad3a9e4ec65ad65b899e7 100644 --- a/arch/powerpc/kvm/e500.h +++ b/arch/powerpc/kvm/e500.h @@ -17,7 +17,7 @@ #define KVM_E500_H #include -#include +#include #include #include diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index fb1490761c87e973c4f5f322ab16cccaf5eff19d..b850b0efa201a35bc49b4c24bffc56581c8a34d4 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -239,7 +239,6 @@ int kvmppc_kvm_pv(struct kvm_vcpu *vcpu) case EV_HCALL_TOKEN(EV_IDLE): r = EV_SUCCESS; kvm_vcpu_halt(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); break; default: r = EV_UNIMPLEMENTED; @@ -786,7 +785,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; - vcpu->arch.dec_expires = get_tb(); #ifdef CONFIG_KVM_EXIT_TIMING mutex_init(&vcpu->arch.exit_timing_lock); diff --git a/arch/powerpc/kvm/tm.S b/arch/powerpc/kvm/tm.S index 3bf17c854be4470bbcd69f7c537a4f5d762d562b..2158f61e317fc0418be1cdc51fce7bfe5dd8136d 100644 --- a/arch/powerpc/kvm/tm.S +++ b/arch/powerpc/kvm/tm.S @@ -110,7 +110,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST) mtmsrd r2, 1 /* Reload TOC pointer. */ - ld r2, PACATOC(r13) + LOAD_PACA_TOC() /* Save all but r0-r2, r9 & r13 */ reg = 3 diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index 6edf0697a526b22c91b08b05a34c169726fae603..ad0cf3108dd09c78c5fb50d1395253ef5d43d6f6 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -94,17 +94,20 @@ void __init poking_init(void) static_branch_enable(&poking_init_done); } +static unsigned long get_patch_pfn(void *addr) +{ + if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr)) + return vmalloc_to_pfn(addr); + else + return __pa_symbol(addr) >> PAGE_SHIFT; +} + /* * This can be called for kernel text or a module. */ static int map_patch_area(void *addr, unsigned long text_poke_addr) { - unsigned long pfn; - - if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr)) - pfn = vmalloc_to_pfn(addr); - else - pfn = __pa_symbol(addr) >> PAGE_SHIFT; + unsigned long pfn = get_patch_pfn(addr); return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL); } @@ -149,17 +152,22 @@ static int __do_patch_instruction(u32 *addr, ppc_inst_t instr) int err; u32 *patch_addr; unsigned long text_poke_addr; + pte_t *pte; + unsigned long pfn = get_patch_pfn(addr); - text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr; + text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr & PAGE_MASK; patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr)); - err = map_patch_area(addr, text_poke_addr); - if (err) - return err; + pte = virt_to_kpte(text_poke_addr); + __set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0); + /* See ptesync comment in radix__set_pte_at() */ + if (radix_enabled()) + asm volatile("ptesync": : :"memory"); err = __patch_instruction(addr, instr, patch_addr); - unmap_patch_area(text_poke_addr); + pte_clear(&init_mm, text_poke_addr, pte); + flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE); return err; } diff --git a/arch/powerpc/lib/copypage_64.S b/arch/powerpc/lib/copypage_64.S index d1091b5ee5da9ae508416d48dd5c31b0b3992d85..6812cb19d04aff99e6413927e2f3358baef173c2 100644 --- a/arch/powerpc/lib/copypage_64.S +++ b/arch/powerpc/lib/copypage_64.S @@ -9,11 +9,6 @@ #include #include - .section ".toc","aw" -PPC64_CACHES: - .tc ppc64_caches[TC],ppc64_caches - .section ".text" - _GLOBAL_TOC(copy_page) BEGIN_FTR_SECTION lis r5,PAGE_SIZE@h @@ -24,7 +19,7 @@ FTR_SECTION_ELSE ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) ori r5,r5,PAGE_SIZE@l BEGIN_FTR_SECTION - ld r10,PPC64_CACHES@toc(r2) + LOAD_REG_ADDR(r10, ppc64_caches) lwz r11,DCACHEL1LOGBLOCKSIZE(r10) /* log2 of cache block size */ lwz r12,DCACHEL1BLOCKSIZE(r10) /* get cache block size */ li r9,0 diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 993d3f31832af676303ace05ff3443e4cbe930f6..31f40f544de547b826a9287d924376efa93a2f52 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -550,7 +550,7 @@ void do_barrier_nospec_fixups(bool enable) } #endif /* CONFIG_PPC_BARRIER_NOSPEC */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) { unsigned int instr[2], *dest; @@ -602,7 +602,7 @@ void __init do_btb_flush_fixups(void) for (; start < end; start += 2) patch_btb_flush_section(start); } -#endif /* CONFIG_PPC_FSL_BOOK3E */ +#endif /* CONFIG_PPC_E500 */ void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) { diff --git a/arch/powerpc/lib/string_64.S b/arch/powerpc/lib/string_64.S index 169872bc08928aa5fc566355ac0e5165985a1d00..df41ce06f86bd30f2e4ef19340cef77e48ea6193 100644 --- a/arch/powerpc/lib/string_64.S +++ b/arch/powerpc/lib/string_64.S @@ -11,11 +11,6 @@ #include #include - .section ".toc","aw" -PPC64_CACHES: - .tc ppc64_caches[TC],ppc64_caches - .section ".text" - /** * __arch_clear_user: - Zero a block of memory in user space, with less checking. * @to: Destination address, in user space. @@ -133,7 +128,7 @@ err1; stb r0,0(r3) blr .Llong_clear: - ld r5,PPC64_CACHES@toc(r2) + LOAD_REG_ADDR(r5, ppc64_caches) bf cr7*4+0,11f err2; std r0,0(r3) diff --git a/arch/powerpc/math-emu/Makefile b/arch/powerpc/math-emu/Makefile index a8794032f15ff35419b2ca25bd94009d18667662..603e59c3db109201406ed4f5ee47ee9b5ce3d897 100644 --- a/arch/powerpc/math-emu/Makefile +++ b/arch/powerpc/math-emu/Makefile @@ -17,4 +17,9 @@ obj-$(CONFIG_SPE) += math_efp.o CFLAGS_fabs.o = -fno-builtin-fabs CFLAGS_math.o = -fno-builtin-fabs -ccflags-y = -w +ccflags-remove-y = -Wmissing-prototypes -Wmissing-declarations -Wunused-but-set-variable + +ifdef KBUILD_EXTRA_WARN +CFLAGS_math.o += -Wmissing-prototypes -Wmissing-declarations -Wunused-but-set-variable +CFLAGS_math_efp.o += -Wmissing-prototypes -Wmissing-declarations -Wunused-but-set-variable +endif diff --git a/arch/powerpc/math-emu/math.c b/arch/powerpc/math-emu/math.c index 36761bd00f385d2d1256ec24e40b4f5ebe766f9b..936a9a149037c59d7f926302e3a2f0dc8dae51fc 100644 --- a/arch/powerpc/math-emu/math.c +++ b/arch/powerpc/math-emu/math.c @@ -24,9 +24,9 @@ FLOATFUNC(mtfsf); FLOATFUNC(mtfsfi); #ifdef CONFIG_MATH_EMULATION_HW_UNIMPLEMENTED -#undef FLOATFUNC(x) +#undef FLOATFUNC #define FLOATFUNC(x) static inline int x(void *op1, void *op2, void *op3, \ - void *op4) { } + void *op4) { return 0; } #endif FLOATFUNC(fadd); @@ -396,28 +396,28 @@ do_mathemu(struct pt_regs *regs) case XCR: op0 = (void *)®s->ccr; - op1 = (void *)((insn >> 23) & 0x7); + op1 = (void *)(long)((insn >> 23) & 0x7); op2 = (void *)¤t->thread.TS_FPR((insn >> 16) & 0x1f); op3 = (void *)¤t->thread.TS_FPR((insn >> 11) & 0x1f); break; case XCRL: op0 = (void *)®s->ccr; - op1 = (void *)((insn >> 23) & 0x7); - op2 = (void *)((insn >> 18) & 0x7); + op1 = (void *)(long)((insn >> 23) & 0x7); + op2 = (void *)(long)((insn >> 18) & 0x7); break; case XCRB: - op0 = (void *)((insn >> 21) & 0x1f); + op0 = (void *)(long)((insn >> 21) & 0x1f); break; case XCRI: - op0 = (void *)((insn >> 23) & 0x7); - op1 = (void *)((insn >> 12) & 0xf); + op0 = (void *)(long)((insn >> 23) & 0x7); + op1 = (void *)(long)((insn >> 12) & 0xf); break; case XFLB: - op0 = (void *)((insn >> 17) & 0xff); + op0 = (void *)(long)((insn >> 17) & 0xff); op1 = (void *)¤t->thread.TS_FPR((insn >> 11) & 0x1f); break; diff --git a/arch/powerpc/math-emu/math_efp.c b/arch/powerpc/math-emu/math_efp.c index 39b84e7452e1b4560f3d9b5347ec85326e446285..34f62aafe706e192e93943596e8d891d51123ec7 100644 --- a/arch/powerpc/math-emu/math_efp.c +++ b/arch/powerpc/math-emu/math_efp.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -218,6 +219,7 @@ int do_spe_mathemu(struct pt_regs *regs) case AB: case XCR: FP_UNPACK_SP(SA, va.wp + 1); + fallthrough; case XB: FP_UNPACK_SP(SB, vb.wp + 1); break; @@ -226,8 +228,8 @@ int do_spe_mathemu(struct pt_regs *regs) break; } - pr_debug("SA: %ld %08lx %ld (%ld)\n", SA_s, SA_f, SA_e, SA_c); - pr_debug("SB: %ld %08lx %ld (%ld)\n", SB_s, SB_f, SB_e, SB_c); + pr_debug("SA: %d %08x %d (%d)\n", SA_s, SA_f, SA_e, SA_c); + pr_debug("SB: %d %08x %d (%d)\n", SB_s, SB_f, SB_e, SB_c); switch (func) { case EFSABS: @@ -278,7 +280,7 @@ int do_spe_mathemu(struct pt_regs *regs) } else { SB_e += (func == EFSCTSF ? 31 : 32); FP_TO_INT_ROUND_S(vc.wp[1], SB, 32, - (func == EFSCTSF)); + (func == EFSCTSF) ? 1 : 0); } goto update_regs; @@ -287,7 +289,7 @@ int do_spe_mathemu(struct pt_regs *regs) FP_CLEAR_EXCEPTIONS; FP_UNPACK_DP(DB, vb.dp); - pr_debug("DB: %ld %08lx %08lx %ld (%ld)\n", + pr_debug("DB: %d %08x %08x %d (%d)\n", DB_s, DB_f1, DB_f0, DB_e, DB_c); FP_CONV(S, D, 1, 2, SR, DB); @@ -301,7 +303,7 @@ int do_spe_mathemu(struct pt_regs *regs) FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_ROUND_S(vc.wp[1], SB, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -312,7 +314,7 @@ int do_spe_mathemu(struct pt_regs *regs) FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_S(vc.wp[1], SB, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -322,7 +324,7 @@ int do_spe_mathemu(struct pt_regs *regs) break; pack_s: - pr_debug("SR: %ld %08lx %ld (%ld)\n", SR_s, SR_f, SR_e, SR_c); + pr_debug("SR: %d %08x %d (%d)\n", SR_s, SR_f, SR_e, SR_c); FP_PACK_SP(vc.wp + 1, SR); goto update_regs; @@ -346,6 +348,7 @@ cmp_s: case AB: case XCR: FP_UNPACK_DP(DA, va.dp); + fallthrough; case XB: FP_UNPACK_DP(DB, vb.dp); break; @@ -354,9 +357,9 @@ cmp_s: break; } - pr_debug("DA: %ld %08lx %08lx %ld (%ld)\n", + pr_debug("DA: %d %08x %08x %d (%d)\n", DA_s, DA_f1, DA_f0, DA_e, DA_c); - pr_debug("DB: %ld %08lx %08lx %ld (%ld)\n", + pr_debug("DB: %d %08x %08x %d (%d)\n", DB_s, DB_f1, DB_f0, DB_e, DB_c); switch (func) { @@ -408,7 +411,7 @@ cmp_s: } else { DB_e += (func == EFDCTSF ? 31 : 32); FP_TO_INT_ROUND_D(vc.wp[1], DB, 32, - (func == EFDCTSF)); + (func == EFDCTSF) ? 1 : 0); } goto update_regs; @@ -417,7 +420,7 @@ cmp_s: FP_CLEAR_EXCEPTIONS; FP_UNPACK_SP(SB, vb.wp + 1); - pr_debug("SB: %ld %08lx %ld (%ld)\n", + pr_debug("SB: %d %08x %d (%d)\n", SB_s, SB_f, SB_e, SB_c); FP_CONV(D, S, 2, 1, DR, SB); @@ -431,7 +434,7 @@ cmp_s: FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_D(vc.dp[0], DB, 64, - ((func & 0x1) == 0)); + ((func & 0x1) == 0) ? 1 : 0); } goto update_regs; @@ -442,7 +445,7 @@ cmp_s: FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_ROUND_D(vc.wp[1], DB, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -453,7 +456,7 @@ cmp_s: FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_D(vc.wp[1], DB, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -463,7 +466,7 @@ cmp_s: break; pack_d: - pr_debug("DR: %ld %08lx %08lx %ld (%ld)\n", + pr_debug("DR: %d %08x %08x %d (%d)\n", DR_s, DR_f1, DR_f0, DR_e, DR_c); FP_PACK_DP(vc.dp, DR); @@ -492,6 +495,7 @@ cmp_d: case XCR: FP_UNPACK_SP(SA0, va.wp); FP_UNPACK_SP(SA1, va.wp + 1); + fallthrough; case XB: FP_UNPACK_SP(SB0, vb.wp); FP_UNPACK_SP(SB1, vb.wp + 1); @@ -502,13 +506,13 @@ cmp_d: break; } - pr_debug("SA0: %ld %08lx %ld (%ld)\n", + pr_debug("SA0: %d %08x %d (%d)\n", SA0_s, SA0_f, SA0_e, SA0_c); - pr_debug("SA1: %ld %08lx %ld (%ld)\n", + pr_debug("SA1: %d %08x %d (%d)\n", SA1_s, SA1_f, SA1_e, SA1_c); - pr_debug("SB0: %ld %08lx %ld (%ld)\n", + pr_debug("SB0: %d %08x %d (%d)\n", SB0_s, SB0_f, SB0_e, SB0_c); - pr_debug("SB1: %ld %08lx %ld (%ld)\n", + pr_debug("SB1: %d %08x %d (%d)\n", SB1_s, SB1_f, SB1_e, SB1_c); switch (func) { @@ -567,7 +571,7 @@ cmp_d: } else { SB0_e += (func == EVFSCTSF ? 31 : 32); FP_TO_INT_ROUND_S(vc.wp[0], SB0, 32, - (func == EVFSCTSF)); + (func == EVFSCTSF) ? 1 : 0); } if (SB1_c == FP_CLS_NAN) { vc.wp[1] = 0; @@ -575,7 +579,7 @@ cmp_d: } else { SB1_e += (func == EVFSCTSF ? 31 : 32); FP_TO_INT_ROUND_S(vc.wp[1], SB1, 32, - (func == EVFSCTSF)); + (func == EVFSCTSF) ? 1 : 0); } goto update_regs; @@ -586,14 +590,14 @@ cmp_d: FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_ROUND_S(vc.wp[0], SB0, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } if (SB1_c == FP_CLS_NAN) { vc.wp[1] = 0; FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_ROUND_S(vc.wp[1], SB1, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -604,14 +608,14 @@ cmp_d: FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_S(vc.wp[0], SB0, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } if (SB1_c == FP_CLS_NAN) { vc.wp[1] = 0; FP_SET_EXCEPTION(FP_EX_INVALID); } else { FP_TO_INT_S(vc.wp[1], SB1, 32, - ((func & 0x3) != 0)); + ((func & 0x3) != 0) ? 1 : 0); } goto update_regs; @@ -621,9 +625,9 @@ cmp_d: break; pack_vs: - pr_debug("SR0: %ld %08lx %ld (%ld)\n", + pr_debug("SR0: %d %08x %d (%d)\n", SR0_s, SR0_f, SR0_e, SR0_c); - pr_debug("SR1: %ld %08lx %ld (%ld)\n", + pr_debug("SR1: %d %08x %d (%d)\n", SR1_s, SR1_f, SR1_e, SR1_c); FP_PACK_SP(vc.wp, SR0); @@ -886,7 +890,7 @@ int speround_handler(struct pt_regs *regs) return 0; } -int __init spe_mathemu_init(void) +static int __init spe_mathemu_init(void) { u32 pvr, maj, min; diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index a96b73006dfb1481d99f8128ed9f9375482dbcd2..850783cfa9c730bf04d7d8308852e388f120e4f4 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -158,7 +158,7 @@ static unsigned long __init __mmu_mapin_ram(unsigned long base, unsigned long to unsigned long __init mmu_mapin_ram(unsigned long base, unsigned long top) { unsigned long done; - unsigned long border = (unsigned long)__init_begin - PAGE_OFFSET; + unsigned long border = (unsigned long)__srwx_boundary - PAGE_OFFSET; unsigned long size; size = roundup_pow_of_two((unsigned long)_einittext - PAGE_OFFSET); @@ -240,7 +240,7 @@ void mmu_mark_rodata_ro(void) for (i = 0; i < nb; i++) { struct ppc_bat *bat = BATS[i]; - if (bat_addrs[i].start < (unsigned long)__init_begin) + if (bat_addrs[i].start < (unsigned long)__end_rodata) bat[1].batl = (bat[1].batl & ~BPP_RW) | BPP_RX; } @@ -314,11 +314,9 @@ static void hash_preload(struct mm_struct *mm, unsigned long ea) * * This must always be called with the pte lock held. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { - if (!mmu_has_feature(MMU_FTR_HPTE_TABLE)) - return; /* * We don't need to worry about _PAGE_PRESENT here because we are * called with either mm->page_table_lock held or ptl lock held diff --git a/arch/powerpc/mm/book3s32/tlb.c b/arch/powerpc/mm/book3s32/tlb.c index 19f0ef950d77322d5d0be1ccf18c3820e6b4bcb1..9ad6b56bfec96e989b96f027d075ad5812500854 100644 --- a/arch/powerpc/mm/book3s32/tlb.c +++ b/arch/powerpc/mm/book3s32/tlb.c @@ -81,14 +81,15 @@ EXPORT_SYMBOL(hash__flush_range); void hash__flush_tlb_mm(struct mm_struct *mm) { struct vm_area_struct *mp; + VMA_ITERATOR(vmi, mm, 0); /* - * It is safe to go down the mm's list of vmas when called - * from dup_mmap, holding mmap_lock. It would also be safe from - * unmap_region or exit_mmap, but not from vmtruncate on SMP - - * but it seems dup_mmap is the only SMP case which gets here. + * It is safe to iterate the vmas when called from dup_mmap, + * holding mmap_lock. It would also be safe from unmap_region + * or exit_mmap, but not from vmtruncate on SMP - but it seems + * dup_mmap is the only SMP case which gets here. */ - for (mp = mm->mmap; mp != NULL; mp = mp->vm_next) + for_each_vma(vmi, mp) hash__flush_range(mp->vm_mm, mp->vm_start, mp->vm_end); } EXPORT_SYMBOL(hash__flush_tlb_mm); diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index ae008b9df0e670ffd7ef3a61dcf27725129c5051..747492edb75abbb023c7fc7df383480a2d540347 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -256,7 +256,7 @@ pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addres * the __collapse_huge_page_copy can result in copying * the old content. */ - flush_tlb_pmd_range(vma->vm_mm, &pmd, address); + flush_hash_table_pmd_range(vma->vm_mm, &pmd, address); return pmd; } @@ -541,7 +541,7 @@ void hash__mark_rodata_ro(void) unsigned long start, end, pp; start = (unsigned long)_stext; - end = (unsigned long)__init_begin; + end = (unsigned long)__end_rodata; pp = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL_ROX), HPTE_USE_KERNEL_KEY); diff --git a/arch/powerpc/mm/book3s64/hash_tlb.c b/arch/powerpc/mm/book3s64/hash_tlb.c index eb0bccaf221eaa9326e1561e355d78da6dc63490..a64ea0a7ef96d2acf9f6e738a52a30523ab95cb4 100644 --- a/arch/powerpc/mm/book3s64/hash_tlb.c +++ b/arch/powerpc/mm/book3s64/hash_tlb.c @@ -221,7 +221,7 @@ void __flush_hash_table_range(unsigned long start, unsigned long end) local_irq_restore(flags); } -void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) +void flush_hash_table_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) { pte_t *pte; pte_t *start_pte; diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index 363a9447d63d866751ef53021454f6430698bb82..df008edf7be09e233ae993ab4fb4cfd9a8d4b5e4 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -123,11 +123,8 @@ EXPORT_SYMBOL_GPL(mmu_slb_size); #ifdef CONFIG_PPC_64K_PAGES int mmu_ci_restrictions; #endif -#ifdef CONFIG_DEBUG_PAGEALLOC static u8 *linear_map_hash_slots; static unsigned long linear_map_hash_count; -static DEFINE_SPINLOCK(linear_map_hash_lock); -#endif /* CONFIG_DEBUG_PAGEALLOC */ struct mmu_hash_ops mmu_hash_ops; EXPORT_SYMBOL(mmu_hash_ops); @@ -427,11 +424,9 @@ repeat: break; cond_resched(); -#ifdef CONFIG_DEBUG_PAGEALLOC - if (debug_pagealloc_enabled() && + if (debug_pagealloc_enabled_or_kfence() && (paddr >> PAGE_SHIFT) < linear_map_hash_count) linear_map_hash_slots[paddr >> PAGE_SHIFT] = ret | 0x80; -#endif /* CONFIG_DEBUG_PAGEALLOC */ } return ret < 0 ? ret : 0; } @@ -778,7 +773,7 @@ static void __init htab_init_page_sizes(void) bool aligned = true; init_hpte_page_sizes(); - if (!debug_pagealloc_enabled()) { + if (!debug_pagealloc_enabled_or_kfence()) { /* * Pick a size for the linear mapping. Currently, we only * support 16M, 1M and 4K which is the default @@ -1066,8 +1061,7 @@ static void __init htab_initialize(void) prot = pgprot_val(PAGE_KERNEL); -#ifdef CONFIG_DEBUG_PAGEALLOC - if (debug_pagealloc_enabled()) { + if (debug_pagealloc_enabled_or_kfence()) { linear_map_hash_count = memblock_end_of_DRAM() >> PAGE_SHIFT; linear_map_hash_slots = memblock_alloc_try_nid( linear_map_hash_count, 1, MEMBLOCK_LOW_LIMIT, @@ -1076,7 +1070,6 @@ static void __init htab_initialize(void) panic("%s: Failed to allocate %lu bytes max_addr=%pa\n", __func__, linear_map_hash_count, &ppc64_rma_size); } -#endif /* CONFIG_DEBUG_PAGEALLOC */ /* create bolted the linear mapping in the hash table */ for_each_mem_range(i, &base, &end) { @@ -1781,7 +1774,7 @@ static void hash_preload(struct mm_struct *mm, pte_t *ptep, unsigned long ea, * * This must always be called with the pte lock held. */ -void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, +void __update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { /* @@ -1791,9 +1784,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, unsigned long trap; bool is_exec; - if (radix_enabled()) - return; - /* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */ if (!pte_young(*ptep) || address >= TASK_SIZE) return; @@ -1990,7 +1980,9 @@ repeat: return slot; } -#ifdef CONFIG_DEBUG_PAGEALLOC +#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) +static DEFINE_SPINLOCK(linear_map_hash_lock); + static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) { unsigned long hash; @@ -2005,6 +1997,9 @@ static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) if (!vsid) return; + if (linear_map_hash_slots[lmi] & 0x80) + return; + ret = hpte_insert_repeating(hash, vpn, __pa(vaddr), mode, HPTE_V_BOLTED, mmu_linear_psize, mmu_kernel_ssize); @@ -2024,7 +2019,10 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize); spin_lock(&linear_map_hash_lock); - BUG_ON(!(linear_map_hash_slots[lmi] & 0x80)); + if (!(linear_map_hash_slots[lmi] & 0x80)) { + spin_unlock(&linear_map_hash_lock); + return; + } hidx = linear_map_hash_slots[lmi] & 0x7f; linear_map_hash_slots[lmi] = 0; spin_unlock(&linear_map_hash_lock); @@ -2055,7 +2053,7 @@ void hash__kernel_map_pages(struct page *page, int numpages, int enable) } local_irq_restore(flags); } -#endif /* CONFIG_DEBUG_PAGEALLOC */ +#endif /* CONFIG_DEBUG_PAGEALLOC || CONFIG_KFENCE */ void hash__setup_initial_memory_limit(phys_addr_t first_memblock_base, phys_addr_t first_memblock_size) diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 7b9966402b25bc4e26b5fdc300e54932b367bd8e..f6151a5892982ac288934e3ce0f3533bb4743911 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -553,8 +553,15 @@ EXPORT_SYMBOL_GPL(memremap_compat_align); pgprot_t vm_get_page_prot(unsigned long vm_flags) { - unsigned long prot = pgprot_val(protection_map[vm_flags & - (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); + unsigned long prot; + + /* Radix supports execute-only, but protection_map maps X -> RX */ + if (radix_enabled() && ((vm_flags & VM_ACCESS_FLAGS) == VM_EXEC)) { + prot = pgprot_val(PAGE_EXECONLY); + } else { + prot = pgprot_val(protection_map[vm_flags & + (VM_ACCESS_FLAGS | VM_SHARED)]); + } if (vm_flags & VM_SAO) prot |= _PAGE_SAO; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 698274109c91f39a1c28468c310d0bbbd2eb42c0..cac727b0179990706fd1dd235a6e3928fc55a024 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -30,9 +30,12 @@ #include #include #include +#include #include +#include + unsigned int mmu_base_pid; unsigned long radix_mem_block_size __ro_after_init; @@ -228,7 +231,7 @@ void radix__mark_rodata_ro(void) unsigned long start, end; start = (unsigned long)_stext; - end = (unsigned long)__init_begin; + end = (unsigned long)__end_rodata; radix__change_memory_range(start, end, _PAGE_WRITE); } @@ -259,21 +262,24 @@ print_mapping(unsigned long start, unsigned long end, unsigned long size, bool e static unsigned long next_boundary(unsigned long addr, unsigned long end) { #ifdef CONFIG_STRICT_KERNEL_RWX - if (addr < __pa_symbol(__init_begin)) - return __pa_symbol(__init_begin); + if (addr < __pa_symbol(__srwx_boundary)) + return __pa_symbol(__srwx_boundary); #endif return end; } static int __meminit create_physical_mapping(unsigned long start, unsigned long end, - unsigned long max_mapping_size, int nid, pgprot_t _prot) { unsigned long vaddr, addr, mapping_size = 0; bool prev_exec, exec = false; pgprot_t prot; int psize; + unsigned long max_mapping_size = radix_mem_block_size; + + if (debug_pagealloc_enabled_or_kfence()) + max_mapping_size = PAGE_SIZE; start = ALIGN(start, PAGE_SIZE); end = ALIGN_DOWN(end, PAGE_SIZE); @@ -352,7 +358,6 @@ static void __init radix_init_pgtable(void) } WARN_ON(create_physical_mapping(start, end, - radix_mem_block_size, -1, PAGE_KERNEL)); } @@ -850,7 +855,7 @@ int __meminit radix__create_section_mapping(unsigned long start, } return create_physical_mapping(__pa(start), __pa(end), - radix_mem_block_size, nid, prot); + nid, prot); } int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end) @@ -896,10 +901,17 @@ void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long #endif #endif -#ifdef CONFIG_DEBUG_PAGEALLOC +#if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) void radix__kernel_map_pages(struct page *page, int numpages, int enable) { - pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n"); + unsigned long addr; + + addr = (unsigned long)page_address(page); + + if (enable) + set_memory_p(addr, numpages); + else + set_memory_np(addr, numpages); } #endif @@ -937,15 +949,6 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre pmd = *pmdp; pmd_clear(pmdp); - /* - * pmdp collapse_flush need to ensure that there are no parallel gup - * walk after this call. This is needed so that we can have stable - * page ref count when collapsing a page. We don't allow a collapse page - * if we have gup taken on the page. We can ensure that by sending IPI - * because gup walk happens with IRQ disabled. - */ - serialize_against_pte_lookup(vma->vm_mm); - radix__flush_tlb_collapsed_pmd(vma->vm_mm, address); return pmd; diff --git a/arch/powerpc/mm/book3s64/subpage_prot.c b/arch/powerpc/mm/book3s64/subpage_prot.c index 60c6ea16a972afc6e26bd615bd3eba2968c1a8a8..d73b3b4176e81d2112a4e5e1cbd6bdff8d678aa4 100644 --- a/arch/powerpc/mm/book3s64/subpage_prot.c +++ b/arch/powerpc/mm/book3s64/subpage_prot.c @@ -149,24 +149,15 @@ static void subpage_mark_vma_nohuge(struct mm_struct *mm, unsigned long addr, unsigned long len) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, addr); /* * We don't try too hard, we just mark all the vma in that range * VM_NOHUGEPAGE and split them. */ - vma = find_vma(mm, addr); - /* - * If the range is in unmapped range, just return - */ - if (vma && ((addr + len) <= vma->vm_start)) - return; - - while (vma) { - if (vma->vm_start >= (addr + len)) - break; + for_each_vma_range(vmi, vma, addr + len) { vma->vm_flags |= VM_NOHUGEPAGE; walk_page_vma(vma, &subpage_walk_ops, NULL); - vma = vma->vm_next; } } #else diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 01400542868731439de4b6a3ff20c058215cc2d2..2bef19cc1b98c659b38109ba47ea2811d0976b66 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -270,7 +270,11 @@ static bool access_error(bool is_write, bool is_exec, struct vm_area_struct *vma return false; } - if (unlikely(!vma_is_accessible(vma))) + /* + * Check for a read fault. This could be caused by a read on an + * inaccessible page (i.e. PROT_NONE), or a Radix MMU execute-only page. + */ + if (unlikely(!(vma->vm_flags & VM_READ))) return true; /* * We should ideally do the vma pkey access check here. But in the @@ -367,7 +371,22 @@ static void sanity_check_fault(bool is_write, bool is_user, #elif defined(CONFIG_PPC_8xx) #define page_fault_is_bad(__err) ((__err) & DSISR_NOEXEC_OR_G) #elif defined(CONFIG_PPC64) -#define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_64S) +static int page_fault_is_bad(unsigned long err) +{ + unsigned long flag = DSISR_BAD_FAULT_64S; + + /* + * PAPR+ v2.11 § 14.15.3.4.1 (unreleased) + * If byte 0, bit 3 of pi-attribute-specifier-type in + * ibm,pi-features property is defined, ignore the DSI error + * which is caused by the paste instruction on the + * suspended NX window. + */ + if (mmu_has_feature(MMU_FTR_NX_DSI)) + flag &= ~DSISR_BAD_COPYPASTE; + + return err & flag; +} #else #define page_fault_is_bad(__err) ((__err) & DSISR_BAD_FAULT_32S) #endif diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index bc84a594ca62cb90cd6fe6056972fa5b2583c3c6..5852a86d990d66f6b0ddf24db4dcee6fd0ab7fb9 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -392,7 +392,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, * single hugepage, but all of them point to * the same kmem cache that holds the hugepte. */ - more = addr + (1 << hugepd_shift(*(hugepd_t *)pmd)); + more = addr + (1UL << hugepd_shift(*(hugepd_t *)pmd)); if (more > next) next = more; @@ -434,7 +434,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, * single hugepage, but all of them point to * the same kmem cache that holds the hugepte. */ - more = addr + (1 << hugepd_shift(*(hugepd_t *)pud)); + more = addr + (1UL << hugepd_shift(*(hugepd_t *)pud)); if (more > next) next = more; @@ -496,7 +496,7 @@ void hugetlb_free_pgd_range(struct mmu_gather *tlb, * for a single hugepage, but all of them point to the * same kmem cache that holds the hugepte. */ - more = addr + (1 << hugepd_shift(*(hugepd_t *)pgd)); + more = addr + (1UL << hugepd_shift(*(hugepd_t *)pgd)); if (more > next) next = more; @@ -623,7 +623,7 @@ static int __init hugetlbpage_init(void) if (pdshift > shift) { if (!IS_ENABLED(CONFIG_PPC_8xx)) pgtable_cache_add(pdshift - shift); - } else if (IS_ENABLED(CONFIG_PPC_FSL_BOOK3E) || + } else if (IS_ENABLED(CONFIG_PPC_E500) || IS_ENABLED(CONFIG_PPC_8xx)) { pgtable_cache_add(PTE_T_ORDER); } diff --git a/arch/powerpc/mm/init_32.c b/arch/powerpc/mm/init_32.c index 62d9af6606cd291c70c72b0b5114d7ceeb8f1b64..d4cc3749e6214864f1a51b3e68186a53a0e4048f 100644 --- a/arch/powerpc/mm/init_32.c +++ b/arch/powerpc/mm/init_32.c @@ -82,28 +82,15 @@ void __init MMU_init(void) if (ppc_md.progress) ppc_md.progress("MMU:enter", 0x111); - /* - * Reserve gigantic pages for hugetlb. This MUST occur before - * lowmem_end_addr is initialized below. - */ - if (memblock.memory.cnt > 1) { -#ifndef CONFIG_WII - memblock_enforce_memory_limit(memblock.memory.regions[0].size); - pr_warn("Only using first contiguous memory region\n"); -#else - wii_memory_fixups(); -#endif - } - total_lowmem = total_memory = memblock_end_of_DRAM() - memstart_addr; lowmem_end_addr = memstart_addr + total_lowmem; -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx /* Freescale Book-E parts expect lowmem to be mapped by fixed TLB * entries, so we need to adjust lowmem to match the amount we can map * in the fixed entries */ adjust_total_lowmem(); -#endif /* CONFIG_FSL_BOOKE */ +#endif /* CONFIG_PPC_85xx */ if (total_lowmem > __max_low_memory) { total_lowmem = __max_low_memory; diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 01772e79fd93e507c050f149ffe9b4813f83f226..84d171953ba44eb4fbcfb42772da104404554ff3 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -302,13 +302,13 @@ void __init mem_init(void) for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) { phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT; struct page *page = pfn_to_page(pfn); - if (!memblock_is_reserved(paddr)) + if (memblock_is_memory(paddr) && !memblock_is_reserved(paddr)) free_highmem_page(page); } } #endif /* CONFIG_HIGHMEM */ -#if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP) +#if defined(CONFIG_PPC_E500) && !defined(CONFIG_SMP) /* * If smp is enabled, next_tlbcam_idx is initialized in the cpu up * functions.... do it here for the non-smp case. diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h index 229c72e491985a90825e22a8b7768ae1f7adcded..bd9784f77f2ee23b295ba792df843cf82f00fd33 100644 --- a/arch/powerpc/mm/mmu_decl.h +++ b/arch/powerpc/mm/mmu_decl.h @@ -38,7 +38,7 @@ static inline void _tlbil_pid(unsigned int pid) #else /* CONFIG_40x || CONFIG_PPC_8xx */ extern void _tlbil_all(void); extern void _tlbil_pid(unsigned int pid); -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 extern void _tlbil_pid_noind(unsigned int pid); #else #define _tlbil_pid_noind(pid) _tlbil_pid(pid) @@ -55,7 +55,7 @@ static inline void _tlbil_va(unsigned long address, unsigned int pid, asm volatile ("tlbie %0; sync" : : "r" (address) : "memory"); trace_tlbie(0, 0, address, pid, 0, 0, 0); } -#elif defined(CONFIG_PPC_BOOK3E) +#elif defined(CONFIG_PPC_BOOK3E_64) extern void _tlbil_va(unsigned long address, unsigned int pid, unsigned int tsize, unsigned int ind); #else @@ -67,7 +67,7 @@ static inline void _tlbil_va(unsigned long address, unsigned int pid, } #endif /* CONFIG_PPC_8xx */ -#if defined(CONFIG_PPC_BOOK3E) || defined(CONFIG_PPC_47x) +#if defined(CONFIG_PPC_BOOK3E_64) || defined(CONFIG_PPC_47x) extern void _tlbivax_bcast(unsigned long address, unsigned int pid, unsigned int tsize, unsigned int ind); #else @@ -92,28 +92,16 @@ extern void mapin_ram(void); extern void setbat(int index, unsigned long virt, phys_addr_t phys, unsigned int size, pgprot_t prot); -extern unsigned int rtas_data, rtas_size; - -struct hash_pte; extern u8 early_hash[]; #endif /* CONFIG_PPC32 */ extern unsigned long __max_low_memory; -extern phys_addr_t __initial_memory_limit_addr; extern phys_addr_t total_memory; extern phys_addr_t total_lowmem; extern phys_addr_t memstart_addr; extern phys_addr_t lowmem_end_addr; -#ifdef CONFIG_WII -extern unsigned long wii_hole_start; -extern unsigned long wii_hole_size; - -extern unsigned long wii_mmu_mapin_mem2(unsigned long top); -extern void wii_memory_fixups(void); -#endif - /* ...and now those things that may be slightly different between processor * architectures. -- Dan */ @@ -123,11 +111,9 @@ void MMU_init_hw_patch(void); unsigned long mmu_mapin_ram(unsigned long base, unsigned long top); #endif -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 extern unsigned long map_mem_in_cams(unsigned long ram, int max_cam_idx, bool dryrun, bool init); -extern unsigned long calc_cam_sz(unsigned long ram, unsigned long virt, - phys_addr_t phys); #ifdef CONFIG_PPC32 extern void adjust_total_lowmem(void); extern int switch_to_as1(void); @@ -160,9 +146,9 @@ struct tlbcam { extern struct tlbcam TLBCAM[NUM_TLBCAMS]; #endif -#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_FSL_BOOKE) || defined(CONFIG_PPC_8xx) +#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_85xx) || defined(CONFIG_PPC_8xx) /* 6xx have BATS */ -/* FSL_BOOKE have TLBCAM */ +/* PPC_85xx have TLBCAM */ /* 8xx have LTLB */ phys_addr_t v_block_mapped(unsigned long va); unsigned long p_block_mapped(phys_addr_t pa); @@ -171,7 +157,7 @@ static inline phys_addr_t v_block_mapped(unsigned long va) { return 0; } static inline unsigned long p_block_mapped(phys_addr_t pa) { return 0; } #endif -#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC_FSL_BOOK3E) +#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC_E500) void mmu_mark_initmem_nx(void); void mmu_mark_rodata_ro(void); #else diff --git a/arch/powerpc/mm/nohash/Makefile b/arch/powerpc/mm/nohash/Makefile index b467a25ee155d16e61295a4da2a58816f9a42ec6..f3894e79d5f700f57d9f7a35e92c030ad1f49f5f 100644 --- a/arch/powerpc/mm/nohash/Makefile +++ b/arch/powerpc/mm/nohash/Makefile @@ -7,13 +7,13 @@ obj-$(CONFIG_PPC_BOOK3E_64) += tlb_low_64e.o book3e_pgtable.o obj-$(CONFIG_40x) += 40x.o obj-$(CONFIG_44x) += 44x.o obj-$(CONFIG_PPC_8xx) += 8xx.o -obj-$(CONFIG_PPC_FSL_BOOK3E) += fsl_book3e.o +obj-$(CONFIG_PPC_E500) += e500.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr_booke.o ifdef CONFIG_HUGETLB_PAGE -obj-$(CONFIG_PPC_FSL_BOOK3E) += book3e_hugetlbpage.o +obj-$(CONFIG_PPC_E500) += e500_hugetlbpage.o endif # Disable kcov instrumentation on sensitive code # This is necessary for booting with kcov enabled on book3e machines KCOV_INSTRUMENT_tlb.o := n -KCOV_INSTRUMENT_fsl_book3e.o := n +KCOV_INSTRUMENT_e500.o := n diff --git a/arch/powerpc/mm/nohash/fsl_book3e.c b/arch/powerpc/mm/nohash/e500.c similarity index 98% rename from arch/powerpc/mm/nohash/fsl_book3e.c rename to arch/powerpc/mm/nohash/e500.c index b8ae6c08c06fdce37e67b09c2fba5e813401c2be..40a4e69ae1a91ba39c80878cc68471f4f802c604 100644 --- a/arch/powerpc/mm/nohash/fsl_book3e.c +++ b/arch/powerpc/mm/nohash/e500.c @@ -59,7 +59,7 @@ static struct { phys_addr_t phys; } tlbcam_addrs[NUM_TLBCAMS]; -#ifdef CONFIG_FSL_BOOKE +#ifdef CONFIG_PPC_85xx /* * Return PA for this VA if it is mapped by a CAM, or 0 */ @@ -135,8 +135,8 @@ static void settlbcam(int index, unsigned long virt, phys_addr_t phys, tlbcam_addrs[index].phys = phys; } -unsigned long calc_cam_sz(unsigned long ram, unsigned long virt, - phys_addr_t phys) +static unsigned long calc_cam_sz(unsigned long ram, unsigned long virt, + phys_addr_t phys) { unsigned int camsize = __ilog2(ram); unsigned int align = __ffs(virt | phys); diff --git a/arch/powerpc/mm/nohash/book3e_hugetlbpage.c b/arch/powerpc/mm/nohash/e500_hugetlbpage.c similarity index 100% rename from arch/powerpc/mm/nohash/book3e_hugetlbpage.c rename to arch/powerpc/mm/nohash/e500_hugetlbpage.c diff --git a/arch/powerpc/mm/nohash/tlb.c b/arch/powerpc/mm/nohash/tlb.c index 5e7ccb48b79c0579619a7dba1fde067d52c358a1..2c15c86c701571d8c4c1d7533da9dd5774cd4304 100644 --- a/arch/powerpc/mm/nohash/tlb.c +++ b/arch/powerpc/mm/nohash/tlb.c @@ -49,8 +49,7 @@ * other sizes not listed here. The .ind field is only used on MMUs that have * indirect page table entries. */ -#if defined(CONFIG_PPC_BOOK3E_MMU) || defined(CONFIG_PPC_8xx) -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { [MMU_PAGE_4K] = { .shift = 12, @@ -81,7 +80,20 @@ struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { .enc = BOOK3E_PAGESZ_1GB, }, }; -#elif defined(CONFIG_PPC_8xx) + +static inline int mmu_get_tsize(int psize) +{ + return mmu_psize_defs[psize].enc; +} +#else +static inline int mmu_get_tsize(int psize) +{ + /* This isn't used on !Book3E for now */ + return 0; +} +#endif + +#ifdef CONFIG_PPC_8xx struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { [MMU_PAGE_4K] = { .shift = 12, @@ -96,53 +108,7 @@ struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { .shift = 23, }, }; -#else -struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { - [MMU_PAGE_4K] = { - .shift = 12, - .ind = 20, - .enc = BOOK3E_PAGESZ_4K, - }, - [MMU_PAGE_16K] = { - .shift = 14, - .enc = BOOK3E_PAGESZ_16K, - }, - [MMU_PAGE_64K] = { - .shift = 16, - .ind = 28, - .enc = BOOK3E_PAGESZ_64K, - }, - [MMU_PAGE_1M] = { - .shift = 20, - .enc = BOOK3E_PAGESZ_1M, - }, - [MMU_PAGE_16M] = { - .shift = 24, - .ind = 36, - .enc = BOOK3E_PAGESZ_16M, - }, - [MMU_PAGE_256M] = { - .shift = 28, - .enc = BOOK3E_PAGESZ_256M, - }, - [MMU_PAGE_1G] = { - .shift = 30, - .enc = BOOK3E_PAGESZ_1GB, - }, -}; -#endif /* CONFIG_FSL_BOOKE */ - -static inline int mmu_get_tsize(int psize) -{ - return mmu_psize_defs[psize].enc; -} -#else -static inline int mmu_get_tsize(int psize) -{ - /* This isn't used on !Book3E for now */ - return 0; -} -#endif /* CONFIG_PPC_BOOK3E_MMU */ +#endif /* The variables below are currently only used on 64-bit Book3E * though this will probably be made common with other nohash @@ -166,7 +132,7 @@ int extlb_level_exc; #endif /* CONFIG_PPC64 */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */ DEFINE_PER_CPU(int, next_tlbcam_idx); EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx); @@ -441,7 +407,7 @@ static void __init setup_page_sizes(void) unsigned int eptcfg; int i, psize; -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 unsigned int mmucfg = mfspr(SPRN_MMUCFG); int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E); @@ -584,7 +550,7 @@ static void __init setup_mmu_htw(void) patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e); patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e); break; -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 case PPC_HTW_E6500: extlb_level_exc = EX_TLB_SIZE; patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e); @@ -627,7 +593,7 @@ static void early_init_this_mmu(void) } mtspr(SPRN_MAS4, mas4); -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { unsigned int num_cams; bool map = true; @@ -680,7 +646,7 @@ static void __init early_init_mmu_global(void) /* Look for HW tablewalk support */ setup_mmu_htw(); -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { if (book3e_htw_mode == PPC_HTW_NONE) { extlb_level_exc = EX_TLB_SIZE; @@ -701,7 +667,7 @@ static void __init early_init_mmu_global(void) static void __init early_mmu_set_memory_limit(void) { -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { /* * Limit memory so we dont have linear faults. @@ -750,7 +716,7 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base, * We crop it to the size of the first MEMBLOCK to * avoid going over total available memory just in case... */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { unsigned long linear_sz; unsigned int num_cams; diff --git a/arch/powerpc/mm/nohash/tlb_low.S b/arch/powerpc/mm/nohash/tlb_low.S index d62b613a0d5d17495d249be61da38bdfbb745204..e1199608ff4dcdfc878b68d4af01d127cd568685 100644 --- a/arch/powerpc/mm/nohash/tlb_low.S +++ b/arch/powerpc/mm/nohash/tlb_low.S @@ -221,7 +221,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_476_DD2) blr #endif /* CONFIG_PPC_47x */ -#elif defined(CONFIG_FSL_BOOKE) +#elif defined(CONFIG_PPC_85xx) /* * FSL BookE implementations. * @@ -294,7 +294,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX) isync 1: wrtee r10 blr -#elif defined(CONFIG_PPC_BOOK3E) +#elif defined(CONFIG_PPC_BOOK3E_64) /* * New Book3E (>= 2.06) implementation * @@ -364,7 +364,7 @@ _GLOBAL(_tlbivax_bcast) #error Unsupported processor type ! #endif -#if defined(CONFIG_PPC_FSL_BOOK3E) +#if defined(CONFIG_PPC_E500) /* * extern void loadcam_entry(unsigned int index) * diff --git a/arch/powerpc/mm/nohash/tlb_low_64e.S b/arch/powerpc/mm/nohash/tlb_low_64e.S index 68ffbfdba8944ba74429725746159514c433850a..76cf456d797624b37147d63a36af1b90dd8cc27a 100644 --- a/arch/powerpc/mm/nohash/tlb_low_64e.S +++ b/arch/powerpc/mm/nohash/tlb_low_64e.S @@ -61,7 +61,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) ld r14,PACAPGD(r13) std r15,EX_TLB_R15(r12) std r10,EX_TLB_CR(r12) -#ifdef CONFIG_PPC_FSL_BOOK3E START_BTB_FLUSH_SECTION mfspr r11, SPRN_SRR1 andi. r10,r11,MSR_PR @@ -70,14 +69,11 @@ START_BTB_FLUSH_SECTION 1: END_BTB_FLUSH_SECTION std r7,EX_TLB_R7(r12) -#endif .endm .macro tlb_epilog_bolted ld r14,EX_TLB_CR(r12) -#ifdef CONFIG_PPC_FSL_BOOK3E ld r7,EX_TLB_R7(r12) -#endif ld r10,EX_TLB_R10(r12) ld r11,EX_TLB_R11(r12) ld r13,EX_TLB_R13(r12) @@ -248,7 +244,6 @@ itlb_miss_fault_bolted: beq tlb_miss_user_bolted b itlb_miss_kernel_bolted -#ifdef CONFIG_PPC_FSL_BOOK3E /* * TLB miss handling for e6500 and derivatives, using hardware tablewalk. * @@ -515,7 +510,6 @@ dtlb_miss_fault_e6500: itlb_miss_fault_e6500: tlb_epilog_bolted b exc_instruction_storage_book3e -#endif /* CONFIG_PPC_FSL_BOOK3E */ /********************************************************************** * * @@ -1124,8 +1118,8 @@ tlb_load_linear: * we only use 1G pages for now. That might have to be changed in a * final implementation, especially when dealing with hypervisors */ - ld r11,PACATOC(r13) - ld r11,linear_map_top@got(r11) + __LOAD_PACA_TOC(r11) + LOAD_REG_ADDR_ALTTOC(r11, r11, linear_map_top) ld r10,0(r11) tovirt(10,10) cmpld cr0,r16,r10 diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 0801b2ce9b7d1053d051521c372eb3d86af2c8ee..b44ce71917d75a9668d42e0fe6fe328e653a2790 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -1160,6 +1160,9 @@ void __init mem_topology_setup(void) { int cpu; + max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; + min_low_pfn = MEMORY_START >> PAGE_SHIFT; + /* * Linux/mm assumes node 0 to be online at boot. However this is not * true on PowerPC, where node 0 is similar to any other node, it @@ -1204,9 +1207,6 @@ void __init initmem_init(void) { int nid; - max_low_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; - max_pfn = max_low_pfn; - memblock_dump_all(); for_each_online_node(nid) { diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 3ac73f9fb5d59870f91e540f8c186df7033bde6b..5c02fd08d61eff04253b5069a852f4da907348fe 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -158,10 +158,11 @@ void mark_rodata_ro(void) } /* - * mark .text and .rodata as read only. Use __init_begin rather than - * __end_rodata to cover NOTES and EXCEPTION_TABLE. + * mark text and rodata as read only. __end_rodata is set by + * powerpc's linker script and includes tables and data + * requiring relocation which are not put in RO_DATA. */ - numpages = PFN_UP((unsigned long)__init_begin) - + numpages = PFN_UP((unsigned long)__end_rodata) - PFN_DOWN((unsigned long)_stext); set_memory_ro((unsigned long)_stext, numpages); diff --git a/arch/powerpc/mm/ptdump/Makefile b/arch/powerpc/mm/ptdump/Makefile index b533caaf0910ca994d90004e8a9edc8537a7d862..dc896d2874f33fd3fcda1cafbd38030362640ccd 100644 --- a/arch/powerpc/mm/ptdump/Makefile +++ b/arch/powerpc/mm/ptdump/Makefile @@ -4,7 +4,7 @@ obj-y += ptdump.o obj-$(CONFIG_4xx) += shared.o obj-$(CONFIG_PPC_8xx) += 8xx.o -obj-$(CONFIG_PPC_BOOK3E_MMU) += shared.o +obj-$(CONFIG_PPC_E500) += shared.o obj-$(CONFIG_PPC_BOOK3S_32) += shared.o obj-$(CONFIG_PPC_BOOK3S_64) += book3s64.o diff --git a/arch/powerpc/perf/bhrb.S b/arch/powerpc/perf/bhrb.S index 1aa3259716b8c92cc96dd32bdb5659267d24a421..47ba05d5ae7693d4fec7b79e1198bd8a4fa78036 100644 --- a/arch/powerpc/perf/bhrb.S +++ b/arch/powerpc/perf/bhrb.S @@ -21,7 +21,7 @@ _GLOBAL(read_bhrb) cmpldi r3,31 bgt 1f - ld r4,bhrb_table@got(r2) + LOAD_REG_ADDR(r4, bhrb_table) sldi r3,r3,3 add r3,r4,r3 mtctr r3 diff --git a/arch/powerpc/perf/callchain_32.c b/arch/powerpc/perf/callchain_32.c index b83c47b7947f077a0b13812492c7b433436335b1..ea8cfe3806dce39aaf9e807d4b4699b73b7bda1f 100644 --- a/arch/powerpc/perf/callchain_32.c +++ b/arch/powerpc/perf/callchain_32.c @@ -19,7 +19,7 @@ #include "callchain.h" #ifdef CONFIG_PPC64 -#include "../kernel/ppc32.h" +#include #else /* CONFIG_PPC64 */ #define __SIGNAL_FRAMESIZE32 __SIGNAL_FRAMESIZE diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 13919eb96931b352958c27feadf7d3c769bc3d3b..942aa830e110eaf0f944c9194596f45ad5b14f47 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -2131,6 +2131,23 @@ static int power_pmu_event_init(struct perf_event *event) if (has_branch_stack(event)) { u64 bhrb_filter = -1; + /* + * Currently no PMU supports having multiple branch filters + * at the same time. Branch filters are set via MMCRA IFM[32:33] + * bits for Power8 and above. Return EOPNOTSUPP when multiple + * branch filters are requested in the event attr. + * + * When opening event via perf_event_open(), branch_sample_type + * gets adjusted in perf_copy_attr(). Kernel will automatically + * adjust the branch_sample_type based on the event modifier + * settings to include PERF_SAMPLE_BRANCH_PLM_ALL. Hence drop + * the check for PERF_SAMPLE_BRANCH_PLM_ALL. + */ + if (hweight64(event->attr.branch_sample_type & ~PERF_SAMPLE_BRANCH_PLM_ALL) > 1) { + local_irq_restore(irq_flags); + return -EOPNOTSUPP; + } + if (ppmu->bhrb_filter_map) bhrb_filter = ppmu->bhrb_filter_map( event->attr.branch_sample_type); @@ -2297,16 +2314,20 @@ static void record_and_restart(struct perf_event *event, unsigned long val, cpuhw = this_cpu_ptr(&cpu_hw_events); power_pmu_bhrb_read(event, cpuhw); data.br_stack = &cpuhw->bhrb_stack; + data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; } if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && - ppmu->get_mem_data_src) + ppmu->get_mem_data_src) { ppmu->get_mem_data_src(&data.data_src, ppmu->flags, regs); + data.sample_flags |= PERF_SAMPLE_DATA_SRC; + } if (event->attr.sample_type & PERF_SAMPLE_WEIGHT_TYPE && - ppmu->get_mem_weight) + ppmu->get_mem_weight) { ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type); - + data.sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } if (perf_event_overflow(event, &data, regs)) power_pmu_stop(event, 0); } else if (period) { diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index d7976ab40d386d73f1ba36a49037759ececf1166..d517aba94d1bc4cb8e176700e2b29894a71a19fa 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -240,8 +240,10 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) ct = of_get_child_count(pmu_events); /* Get the event prefix */ - if (of_property_read_string(node, "events-prefix", &prefix)) + if (of_property_read_string(node, "events-prefix", &prefix)) { + of_node_put(pmu_events); return 0; + } /* Get a global unit and scale data if available */ if (of_property_read_string(node, "scale", &g_scale)) @@ -255,8 +257,10 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) /* Allocate memory for the events */ pmu->events = kcalloc(ct, sizeof(struct imc_events), GFP_KERNEL); - if (!pmu->events) + if (!pmu->events) { + of_node_put(pmu_events); return -ENOMEM; + } ct = 0; /* Parse the events and update the struct */ @@ -266,6 +270,8 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) ct++; } + of_node_put(pmu_events); + /* Allocate memory for attribute group */ attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL); if (!attr_group) { diff --git a/arch/powerpc/platforms/44x/ppc476.c b/arch/powerpc/platforms/44x/ppc476.c index 20cc8f80b086fad6262d5c7c3ad2a0d8b20f57e1..7c91ac5a5241b594ea049107f11c91525cf88e9a 100644 --- a/arch/powerpc/platforms/44x/ppc476.c +++ b/arch/powerpc/platforms/44x/ppc476.c @@ -140,6 +140,8 @@ static void __init ppc47x_init_irq(void) ppc_md.get_irq = mpic_get_irq; } else panic("Unrecognized top level interrupt controller"); + + of_node_put(np); } #ifdef CONFIG_SMP diff --git a/arch/powerpc/platforms/512x/clock-commonclk.c b/arch/powerpc/platforms/512x/clock-commonclk.c index 0652c7e6922560ccb75955f211735944388d3b6d..42abeba4f69832bc8c4eee0b972aedae9c00ee97 100644 --- a/arch/powerpc/platforms/512x/clock-commonclk.c +++ b/arch/powerpc/platforms/512x/clock-commonclk.c @@ -950,7 +950,7 @@ static void __init mpc5121_clk_register_of_provider(struct device_node *np) */ static void __init mpc5121_clk_provide_migration_support(void) { - + struct device_node *np; /* * pre-enable those clock items which are not yet appropriately * acquired by their peripheral driver @@ -970,7 +970,9 @@ static void __init mpc5121_clk_provide_migration_support(void) * unused and so it gets disabled */ clk_prepare_enable(clks[MPC512x_CLK_PSC3_MCLK]);/* serial console */ - if (of_find_compatible_node(NULL, "pci", "fsl,mpc5121-pci")) + np = of_find_compatible_node(NULL, "pci", "fsl,mpc5121-pci"); + of_node_put(np); + if (np) clk_prepare_enable(clks[MPC512x_CLK_PCI]); } @@ -1208,6 +1210,8 @@ int __init mpc5121_clk_init(void) /* register as an OF clock provider */ mpc5121_clk_register_of_provider(clk_np); + of_node_put(clk_np); + /* * unbreak not yet adjusted peripheral drivers during migration * towards fully operational common clock support, and allow diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index ee367ff3ec8a49e5ce80ba04638162f0b82fb0c9..33a35fff11b5cb043567fb1e70362fee85093cbb 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -174,6 +174,8 @@ static void __init media5200_init_irq(void) goto out; pr_debug("%s: allocated irqhost\n", __func__); + of_node_put(fpga_np); + irq_set_handler_data(cascade_virq, &media5200_irq); irq_set_chained_handler(cascade_virq, media5200_irq_cascade); @@ -181,6 +183,7 @@ static void __init media5200_init_irq(void) out: pr_err("Could not find Media5200 FPGA; PCI interrupts will not work\n"); + of_node_put(fpga_np); } /* diff --git a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c index abb62fa630effbc7a5444de55f07411a96c5b87c..77ed61306a73a9ba80de101f7a6f302da110b001 100644 --- a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c +++ b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c @@ -178,7 +178,7 @@ err: return ret; } -static int mcu_remove(struct i2c_client *client) +static void mcu_remove(struct i2c_client *client) { struct mcu *mcu = i2c_get_clientdata(client); @@ -193,7 +193,6 @@ static int mcu_remove(struct i2c_client *client) mcu_gpiochip_remove(mcu); kfree(mcu); - return 0; } static const struct i2c_device_id mcu_ids[] = { diff --git a/arch/powerpc/platforms/83xx/mpc832x_rdb.c b/arch/powerpc/platforms/83xx/mpc832x_rdb.c index bb8caa5071f8cf356fde2934a22154d2f43cb20f..e12cb44e717f1f084226235b3386c7e79e96bc09 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc832x_rdb.c @@ -162,6 +162,8 @@ static struct spi_board_info mpc832x_spi_boardinfo = { static int __init mpc832x_spi_init(void) { + struct device_node *np; + par_io_config_pin(3, 0, 3, 0, 1, 0); /* SPI1 MOSI, I/O */ par_io_config_pin(3, 1, 3, 0, 1, 0); /* SPI1 MISO, I/O */ par_io_config_pin(3, 2, 3, 0, 1, 0); /* SPI1 CLK, I/O */ @@ -175,7 +177,9 @@ static int __init mpc832x_spi_init(void) * Don't bother with legacy stuff when device tree contains * mmc-spi-slot node. */ - if (of_find_compatible_node(NULL, NULL, "mmc-spi-slot")) + np = of_find_compatible_node(NULL, NULL, "mmc-spi-slot"); + of_node_put(np); + if (np) return 0; return fsl_spi_init(&mpc832x_spi_boardinfo, 1, mpc83xx_spi_cs_control); } diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig index be16eba0f704dc384cf37f1e1cc408f99cc06c60..b92cb2b4d54d178c2cae4da5dda95f10665e1678 100644 --- a/arch/powerpc/platforms/85xx/Kconfig +++ b/arch/powerpc/platforms/85xx/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 menuconfig FSL_SOC_BOOKE bool "Freescale Book-E Machine Type" - depends on PPC_85xx || PPC_BOOK3E + depends on PPC_E500 select FSL_SOC select PPC_UDBG_16550 select MPIC @@ -241,8 +241,6 @@ endif # PPC32 config PPC_QEMU_E500 bool "QEMU generic e500 platform" select DEFAULT_UIMAGE - select E500 - select PPC_E500MC if PPC64 help This option enables support for running as a QEMU guest using QEMU's generic e500 machine. This is not required if you're @@ -258,7 +256,6 @@ config PPC_QEMU_E500 config CORENET_GENERIC bool "Freescale CoreNet Generic" select DEFAULT_UIMAGE - select E500 select PPC_E500MC select PHYS_64BIT select SWIOTLB diff --git a/arch/powerpc/platforms/85xx/corenet_generic.c b/arch/powerpc/platforms/85xx/corenet_generic.c index 28d6b36f1ccdb96eae6fe2ba912d3dd0e88e6ad0..2c539de2d629a01ce6ae9e998b5419fe30f86eb8 100644 --- a/arch/powerpc/platforms/85xx/corenet_generic.c +++ b/arch/powerpc/platforms/85xx/corenet_generic.c @@ -200,9 +200,5 @@ define_machine(corenet_generic) { #endif .calibrate_decr = generic_calibrate_decr, .progress = udbg_progress, -#ifdef CONFIG_PPC64 - .power_save = book3e_idle, -#else .power_save = e500_idle, -#endif }; diff --git a/arch/powerpc/platforms/85xx/ge_imp3a.c b/arch/powerpc/platforms/85xx/ge_imp3a.c index 8e827376d97b8969046ed60b77b507d9c23394e1..e3e8f18825a1b5ff66fb48f811b9de4d8a5d2346 100644 --- a/arch/powerpc/platforms/85xx/ge_imp3a.c +++ b/arch/powerpc/platforms/85xx/ge_imp3a.c @@ -89,8 +89,10 @@ static void __init ge_imp3a_pci_assign_primary(void) of_device_is_compatible(np, "fsl,mpc8548-pcie") || of_device_is_compatible(np, "fsl,p2020-pcie")) { of_address_to_resource(np, 0, &rsrc); - if ((rsrc.start & 0xfffff) == 0x9000) - fsl_pci_primary = np; + if ((rsrc.start & 0xfffff) == 0x9000) { + of_node_put(fsl_pci_primary); + fsl_pci_primary = of_node_get(np); + } } } #endif diff --git a/arch/powerpc/platforms/85xx/ksi8560.c b/arch/powerpc/platforms/85xx/ksi8560.c index bdf9d42f8521d58c5f13e36a418a6dd06fa4ebae..a22f02b0fc77c43cbfe12c333732d286350450c0 100644 --- a/arch/powerpc/platforms/85xx/ksi8560.c +++ b/arch/powerpc/platforms/85xx/ksi8560.c @@ -133,6 +133,8 @@ static void __init ksi8560_setup_arch(void) else printk(KERN_ERR "Can't find CPLD in device tree\n"); + of_node_put(cpld); + if (ppc_md.progress) ppc_md.progress("ksi8560_setup_arch()", 0); diff --git a/arch/powerpc/platforms/85xx/mpc85xx_cds.c b/arch/powerpc/platforms/85xx/mpc85xx_cds.c index 48f3acfece0b2aaca7cc2fab328419fedaaa1af1..0b8f2101c5fba9f6af6884e0639781cb7a79ac71 100644 --- a/arch/powerpc/platforms/85xx/mpc85xx_cds.c +++ b/arch/powerpc/platforms/85xx/mpc85xx_cds.c @@ -159,6 +159,7 @@ static void __init mpc85xx_cds_pci_irq_fixup(struct pci_dev *dev) else dev->irq = 10; pci_write_config_byte(dev, PCI_INTERRUPT_LINE, dev->irq); + break; default: break; } diff --git a/arch/powerpc/platforms/85xx/qemu_e500.c b/arch/powerpc/platforms/85xx/qemu_e500.c index 64109ad6736c14ca97833adc5fff251e338b93e5..1639e222cc33f015a239aab0ab31cda4085ca60c 100644 --- a/arch/powerpc/platforms/85xx/qemu_e500.c +++ b/arch/powerpc/platforms/85xx/qemu_e500.c @@ -68,9 +68,5 @@ define_machine(qemu_e500) { .get_irq = mpic_get_coreint_irq, .calibrate_decr = generic_calibrate_decr, .progress = udbg_progress, -#ifdef CONFIG_PPC64 - .power_save = book3e_idle, -#else .power_save = e500_idle, -#endif }; diff --git a/arch/powerpc/platforms/85xx/sgy_cts1000.c b/arch/powerpc/platforms/85xx/sgy_cts1000.c index 98ae640751934d0723ec0335214717332cb93783..e14d1b74d4e4c5b2864447e2f32df255b5cc47b7 100644 --- a/arch/powerpc/platforms/85xx/sgy_cts1000.c +++ b/arch/powerpc/platforms/85xx/sgy_cts1000.c @@ -71,6 +71,7 @@ static int gpio_halt_probe(struct platform_device *pdev) { enum of_gpio_flags flags; struct device_node *node = pdev->dev.of_node; + struct device_node *child_node; int gpio, err, irq; int trigger; @@ -78,26 +79,29 @@ static int gpio_halt_probe(struct platform_device *pdev) return -ENODEV; /* If there's no matching child, this isn't really an error */ - halt_node = of_find_matching_node(node, child_match); - if (!halt_node) + child_node = of_find_matching_node(node, child_match); + if (!child_node) return 0; /* Technically we could just read the first one, but punish * DT writers for invalid form. */ - if (of_gpio_count(halt_node) != 1) - return -EINVAL; + if (of_gpio_count(child_node) != 1) { + err = -EINVAL; + goto err_put; + } /* Get the gpio number relative to the dynamic base. */ - gpio = of_get_gpio_flags(halt_node, 0, &flags); - if (!gpio_is_valid(gpio)) - return -EINVAL; + gpio = of_get_gpio_flags(child_node, 0, &flags); + if (!gpio_is_valid(gpio)) { + err = -EINVAL; + goto err_put; + } err = gpio_request(gpio, "gpio-halt"); if (err) { printk(KERN_ERR "gpio-halt: error requesting GPIO %d.\n", gpio); - halt_node = NULL; - return err; + goto err_put; } trigger = (flags == OF_GPIO_ACTIVE_LOW); @@ -105,15 +109,14 @@ static int gpio_halt_probe(struct platform_device *pdev) gpio_direction_output(gpio, !trigger); /* Now get the IRQ which tells us when the power button is hit */ - irq = irq_of_parse_and_map(halt_node, 0); + irq = irq_of_parse_and_map(child_node, 0); err = request_irq(irq, gpio_halt_irq, IRQF_TRIGGER_RISING | - IRQF_TRIGGER_FALLING, "gpio-halt", halt_node); + IRQF_TRIGGER_FALLING, "gpio-halt", child_node); if (err) { printk(KERN_ERR "gpio-halt: error requesting IRQ %d for " "GPIO %d.\n", irq, gpio); gpio_free(gpio); - halt_node = NULL; - return err; + goto err_put; } /* Register our halt function */ @@ -123,7 +126,12 @@ static int gpio_halt_probe(struct platform_device *pdev) printk(KERN_INFO "gpio-halt: registered GPIO %d (%d trigger, %d" " irq).\n", gpio, trigger, irq); + halt_node = child_node; return 0; + +err_put: + of_node_put(child_node); + return err; } static int gpio_halt_remove(struct platform_device *pdev) @@ -139,6 +147,7 @@ static int gpio_halt_remove(struct platform_device *pdev) gpio_free(gpio); + of_node_put(halt_node); halt_node = NULL; } diff --git a/arch/powerpc/platforms/8xx/tqm8xx_setup.c b/arch/powerpc/platforms/8xx/tqm8xx_setup.c index 3725d51248df155c9c625d485d1aa905ead24ce5..ffcfd17a5fa30734db9ed5750c34230022f2b76e 100644 --- a/arch/powerpc/platforms/8xx/tqm8xx_setup.c +++ b/arch/powerpc/platforms/8xx/tqm8xx_setup.c @@ -105,6 +105,9 @@ static void __init init_ioports(void) if (dnode == NULL) return; prop = of_find_property(dnode, "ethernet1", &len); + + of_node_put(dnode); + if (prop == NULL) return; diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 5185d942b4555d328217ac95a9b9770cdde9297d..0c4eed9aea80623e673e93f370b38f8572fd82cd 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -33,7 +33,7 @@ config PPC_BOOK3S_32 config PPC_85xx bool "Freescale 85xx" - select E500 + select PPC_E500 config PPC_8xx bool "Freescale 8xx" @@ -107,7 +107,8 @@ config PPC_BOOK3S_64 config PPC_BOOK3E_64 bool "Embedded processors" - select PPC_FSL_BOOK3E + select PPC_E500 + select PPC_E500MC select PPC_FPU # Make it a choice ? select PPC_SMP_MUXED_IPI select PPC_DOORBELL @@ -125,7 +126,7 @@ choice If unsure, select Generic. config GENERIC_CPU - bool "Generic (POWER4 and above)" + bool "Generic (POWER5 and PowerPC 970 and above)" depends on PPC_BOOK3S_64 && !CPU_LITTLE_ENDIAN select PPC_64S_HASH_MMU @@ -144,8 +145,8 @@ config CELL_CPU depends on PPC_BOOK3S_64 && !CPU_LITTLE_ENDIAN select PPC_64S_HASH_MMU -config POWER5_CPU - bool "POWER5" +config PPC_970_CPU + bool "PowerPC 970 (including PowerPC G5)" depends on PPC_BOOK3S_64 && !CPU_LITTLE_ENDIAN select PPC_64S_HASH_MMU @@ -171,13 +172,18 @@ config POWER9_CPU depends on PPC_BOOK3S_64 select ARCH_HAS_FAST_MULTIPLIER +config POWER10_CPU + bool "POWER10" + depends on PPC_BOOK3S_64 + select ARCH_HAS_FAST_MULTIPLIER + config E5500_CPU bool "Freescale e5500" - depends on PPC64 && E500 + depends on PPC64 && PPC_E500 config E6500_CPU bool "Freescale e6500" - depends on PPC64 && E500 + depends on PPC64 && PPC_E500 config 405_CPU bool "40x family" @@ -234,11 +240,12 @@ config TARGET_CPU string depends on TARGET_CPU_BOOL default "cell" if CELL_CPU - default "power5" if POWER5_CPU + default "970" if PPC_970_CPU default "power6" if POWER6_CPU default "power7" if POWER7_CPU default "power8" if POWER8_CPU default "power9" if POWER9_CPU + default "power10" if POWER10_CPU default "405" if 405_CPU default "440" if 440_CPU default "464" if 464_CPU @@ -255,20 +262,19 @@ config PPC_BOOK3S def_bool y depends on PPC_BOOK3S_32 || PPC_BOOK3S_64 -config PPC_BOOK3E - def_bool y - depends on PPC_BOOK3E_64 - -config E500 +config PPC_E500 select FSL_EMB_PERFMON - select PPC_FSL_BOOK3E bool + select ARCH_SUPPORTS_HUGETLBFS if PHYS_64BIT || PPC64 + select PPC_SMP_MUXED_IPI + select PPC_DOORBELL + select PPC_KUEP config PPC_E500MC bool "e500mc Support" select PPC_FPU select COMMON_CLK - depends on E500 + depends on PPC_E500 help This must be enabled for running on e500mc (and derivatives such as e5500/e6500), and must be disabled for running on @@ -291,7 +297,7 @@ config PPC_FPU config FSL_EMB_PERFMON bool "Freescale Embedded Perfmon" - depends on E500 || PPC_83xx + depends on PPC_E500 || PPC_83xx help This is the Performance Monitor support found on the e500 core and some e300 cores (c3 and c4). Select this only if your @@ -304,7 +310,7 @@ config FSL_EMB_PERF_EVENT config FSL_EMB_PERF_EVENT_E500 bool - depends on FSL_EMB_PERF_EVENT && E500 + depends on FSL_EMB_PERF_EVENT && PPC_E500 default y config 4xx @@ -314,7 +320,7 @@ config 4xx config BOOKE bool - depends on E500 || 44x || PPC_BOOK3E + depends on PPC_E500 || 44x default y config BOOKE_OR_40x @@ -322,29 +328,14 @@ config BOOKE_OR_40x depends on BOOKE || 40x default y -config FSL_BOOKE - bool - depends on E500 && PPC32 - default y - -# this is for common code between PPC32 & PPC64 FSL BOOKE -config PPC_FSL_BOOK3E - bool - select ARCH_SUPPORTS_HUGETLBFS if PHYS_64BIT || PPC64 - imply FSL_EMB_PERFMON - select PPC_SMP_MUXED_IPI - select PPC_DOORBELL - select PPC_KUEP - default y if FSL_BOOKE - config PTE_64BIT bool - depends on 44x || E500 || PPC_86xx + depends on 44x || PPC_E500 || PPC_86xx default y if PHYS_64BIT config PHYS_64BIT - bool 'Large physical address support' if E500 || PPC_86xx - depends on (44x || E500 || PPC_86xx) && !PPC_83xx && !PPC_82xx + bool 'Large physical address support' if PPC_E500 || PPC_86xx + depends on (44x || PPC_E500 || PPC_86xx) && !PPC_83xx && !PPC_82xx select PHYS_ADDR_T_64BIT help This option enables kernel support for larger than 32-bit physical @@ -391,7 +382,7 @@ config VSX config SPE_POSSIBLE def_bool y - depends on E500 && !PPC_E500MC + depends on PPC_E500 && !PPC_E500MC config SPE bool "SPE Support" @@ -481,10 +472,6 @@ config PPC_MMU_NOHASH def_bool y depends on !PPC_BOOK3S -config PPC_BOOK3E_MMU - def_bool y - depends on FSL_BOOKE || PPC_BOOK3E - config PPC_HAVE_PMU_SUPPORT bool @@ -506,7 +493,7 @@ config FORCE_SMP select SMP config SMP - depends on PPC_BOOK3S || PPC_BOOK3E || FSL_BOOKE || PPC_47x + depends on PPC_BOOK3S || PPC_E500 || PPC_47x select GENERIC_IRQ_MIGRATION bool "Symmetric multi-processing support" if !FORCE_SMP help diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c index c0799fb26b6d6080a3c0f638f70eafd716af53ed..40f5ae5e1238d633ba71a020ba6a5b46ff296468 100644 --- a/arch/powerpc/platforms/book3s/vas-api.c +++ b/arch/powerpc/platforms/book3s/vas-api.c @@ -431,7 +431,7 @@ static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) * The window may be inactive due to lost credit (Ex: core * removal with DLPAR). If the window is active again when * the credit is available, map the new paste address at the - * the window virtual address. + * window virtual address. */ if (txwin->status == VAS_WIN_ACTIVE) { paste_addr = cp_inst->coproc->vops->paste_addr(txwin); diff --git a/arch/powerpc/platforms/cell/cbe_regs.c b/arch/powerpc/platforms/cell/cbe_regs.c index 316e533afc0054808fa5d3f26ae1dfb0db52d2c5..fb4023f9ea6be6d312e0704c02b12c0c35b36442 100644 --- a/arch/powerpc/platforms/cell/cbe_regs.c +++ b/arch/powerpc/platforms/cell/cbe_regs.c @@ -182,9 +182,16 @@ static struct device_node *__init cbe_get_be_node(int cpu_id) if (WARN_ON_ONCE(!cpu_handle)) return np; - for (i=0; ibe_node) { - struct device_node *be, *np; + struct device_node *be, *np, *parent_np; be = map->be_node; - for_each_node_by_type(np, "pervasive") - if (of_get_parent(np) == be) + for_each_node_by_type(np, "pervasive") { + parent_np = of_get_parent(np); + if (parent_np == be) map->pmd_regs = of_iomap(np, 0); + of_node_put(parent_np); + } - for_each_node_by_type(np, "CBEA-Internal-Interrupt-Controller") - if (of_get_parent(np) == be) + for_each_node_by_type(np, "CBEA-Internal-Interrupt-Controller") { + parent_np = of_get_parent(np); + if (parent_np == be) map->iic_regs = of_iomap(np, 2); + of_node_put(parent_np); + } - for_each_node_by_type(np, "mic-tm") - if (of_get_parent(np) == be) + for_each_node_by_type(np, "mic-tm") { + parent_np = of_get_parent(np); + if (parent_np == be) map->mic_tm_regs = of_iomap(np, 0); + of_node_put(parent_np); + } } else { struct device_node *cpu; /* That hack must die die die ! */ @@ -261,7 +277,8 @@ void __init cbe_regs_init(void) of_node_put(cpu); return; } - map->cpu_node = cpu; + of_node_put(map->cpu_node); + map->cpu_node = of_node_get(cpu); for_each_possible_cpu(i) { struct cbe_thread_map *thread = &cbe_thread_map[i]; diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c index 0ca3efeef293bdd58c3b42086f733ea04a4a7102..8c7133039566911365fb3548dbba1938bb8e82d2 100644 --- a/arch/powerpc/platforms/cell/iommu.c +++ b/arch/powerpc/platforms/cell/iommu.c @@ -720,8 +720,10 @@ static int __init cell_iommu_init_disabled(void) cell_disable_iommus(); /* If we have no Axon, we set up the spider DMA magic offset */ - if (of_find_node_by_name(NULL, "axon") == NULL) + np = of_find_node_by_name(NULL, "axon"); + if (!np) cell_dma_nommu_offset = SPIDER_DMA_OFFSET; + of_node_put(np); /* Now we need to check to see where the memory is mapped * in PCI space. We assume that all busses use the same dma diff --git a/arch/powerpc/platforms/cell/setup.c b/arch/powerpc/platforms/cell/setup.c index 52de014983c97f73d1488a4a357878c3a769e293..47eaf75349f2fcc1773569bdfaa25f68a54d0a2f 100644 --- a/arch/powerpc/platforms/cell/setup.c +++ b/arch/powerpc/platforms/cell/setup.c @@ -167,6 +167,8 @@ static int __init cell_publish_devices(void) of_platform_device_create(np, NULL, NULL); } + of_node_put(root); + /* There is no device for the MIC memory controller, thus we create * a platform device for it to attach the EDAC driver to. */ diff --git a/arch/powerpc/platforms/cell/spu_callbacks.c b/arch/powerpc/platforms/cell/spu_callbacks.c index fe0d8797a00a3bd40082a5133b0cc82839392ebf..e780c14c5733cb21b2b5f3f19141d886706f5002 100644 --- a/arch/powerpc/platforms/cell/spu_callbacks.c +++ b/arch/powerpc/platforms/cell/spu_callbacks.c @@ -34,15 +34,15 @@ * mbind, mq_open, ipc, ... */ -static void *spu_syscall_table[] = { +static const syscall_fn spu_syscall_table[] = { #define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry) -#define __SYSCALL(nr, entry) [nr] = entry, +#define __SYSCALL(nr, entry) [nr] = (void *) entry, #include }; long spu_sys_callback(struct spu_syscall_block *s) { - long (*syscall)(u64 a1, u64 a2, u64 a3, u64 a4, u64 a5, u64 a6); + syscall_fn syscall; if (s->nr_ret >= ARRAY_SIZE(spu_syscall_table)) { pr_debug("%s: invalid syscall #%lld", __func__, s->nr_ret); diff --git a/arch/powerpc/platforms/cell/spu_manage.c b/arch/powerpc/platforms/cell/spu_manage.c index ae09c5a91b4078d63ad4490b95e092a8b80a2976..f1ac4c7420690a5d5f5792966c55ba29aa3f29c4 100644 --- a/arch/powerpc/platforms/cell/spu_manage.c +++ b/arch/powerpc/platforms/cell/spu_manage.c @@ -488,6 +488,8 @@ static void __init init_affinity_node(int cbe) avoid_ph = vic_dn->phandle; } + of_node_put(vic_dn); + list_add_tail(&spu->aff_list, &last_spu->aff_list); last_spu = spu; break; diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 320008528edd96d5f9cb25c03fe390ff131e78c7..dbcfe361831a663bbe199d4e9c11cc073516adff 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -275,7 +275,7 @@ spufs_mkdir(struct inode *dir, struct dentry *dentry, unsigned int flags, return ret; } -static int spufs_context_open(struct path *path) +static int spufs_context_open(const struct path *path) { int ret; struct file *filp; @@ -491,7 +491,7 @@ out: return ret; } -static int spufs_gang_open(struct path *path) +static int spufs_gang_open(const struct path *path) { int ret; struct file *filp; @@ -536,7 +536,7 @@ static int spufs_create_gang(struct inode *inode, static struct file_system_type spufs_type; -long spufs_create(struct path *path, struct dentry *dentry, +long spufs_create(const struct path *path, struct dentry *dentry, unsigned int flags, umode_t mode, struct file *filp) { struct inode *dir = d_inode(path->dentry); diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index 23c6799cfa5a0eb0da96629384f395957af68cef..84958487f696a44bf35f95ad747018ac68e45330 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -232,7 +232,7 @@ extern const struct spufs_tree_descr spufs_dir_debug_contents[]; extern struct spufs_calls spufs_calls; struct coredump_params; long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *status); -long spufs_create(struct path *nd, struct dentry *dentry, unsigned int flags, +long spufs_create(const struct path *nd, struct dentry *dentry, unsigned int flags, umode_t mode, struct file *filp); /* ELF coredump callbacks for writing SPU ELF notes */ extern int spufs_coredump_extra_notes_size(void); @@ -333,7 +333,6 @@ void spufs_stop_callback(struct spu *spu, int irq); void spufs_mfc_callback(struct spu *spu); void spufs_dma_callback(struct spu *spu, int type); -extern struct spu_coredump_calls spufs_coredump_calls; struct spufs_coredump_reader { char *name; ssize_t (*dump)(struct spu_context *ctx, struct coredump_params *cprm); @@ -341,7 +340,6 @@ struct spufs_coredump_reader { size_t size; }; extern const struct spufs_coredump_reader spufs_coredump_read[]; -extern int spufs_coredump_num_notes; extern int spu_init_csa(struct spu_state *csa); extern void spu_fini_csa(struct spu_state *csa); diff --git a/arch/powerpc/platforms/chrp/chrp.h b/arch/powerpc/platforms/chrp/chrp.h index a5a7c338caf948a5350b350d5d0d9d3634822520..6ff4631d9db4b15a06e9c01cab18b406872779a5 100644 --- a/arch/powerpc/platforms/chrp/chrp.h +++ b/arch/powerpc/platforms/chrp/chrp.h @@ -9,4 +9,3 @@ extern int chrp_set_rtc_time(struct rtc_time *); extern long chrp_time_init(void); extern void chrp_find_bridges(void); -extern void chrp_event_scan(unsigned long); diff --git a/arch/powerpc/platforms/embedded6xx/holly.c b/arch/powerpc/platforms/embedded6xx/holly.c index 78f2378d9223f3765df3f9c7fb1bd0467cb0887b..bebc5a972694e33d26809ce09557cc7a2593a4d3 100644 --- a/arch/powerpc/platforms/embedded6xx/holly.c +++ b/arch/powerpc/platforms/embedded6xx/holly.c @@ -123,6 +123,8 @@ static void __init holly_init_pci(void) if (np) tsi108_setup_pci(np, HOLLY_PCI_CFG_PHYS, 1); + of_node_put(np); + ppc_md.pci_exclude_device = holly_exclude_device; if (ppc_md.progress) ppc_md.progress("tsi108: resources set", 0x100); @@ -184,6 +186,9 @@ static void __init holly_init_IRQ(void) tsi108_pci_int_init(cascade_node); irq_set_handler_data(cascade_pci_irq, mpic); irq_set_chained_handler(cascade_pci_irq, tsi108_irq_cascade); + + of_node_put(tsi_pci); + of_node_put(cascade_node); #endif /* Configure MPIC outputs to CPU0 */ tsi108_write_reg(TSI108_MPIC_OFFSET + 0x30c, 0); @@ -210,6 +215,7 @@ static void __noreturn holly_restart(char *cmd) if (bridge) { prop = of_get_property(bridge, "reg", &size); addr = of_translate_address(bridge, prop); + of_node_put(bridge); } addr += (TSI108_PB_OFFSET + 0x414); diff --git a/arch/powerpc/platforms/embedded6xx/ls_uart.c b/arch/powerpc/platforms/embedded6xx/ls_uart.c index 0133e175a0fcd26c81afee1ff3b316a91eef9129..4ecbc55b37c0b96fba8fde89f39093015b45981f 100644 --- a/arch/powerpc/platforms/embedded6xx/ls_uart.c +++ b/arch/powerpc/platforms/embedded6xx/ls_uart.c @@ -124,6 +124,8 @@ static int __init ls_uarts_init(void) avr_clock = *(u32*)of_get_property(avr, "clock-frequency", &len); phys_addr = ((u32*)of_get_property(avr, "reg", &len))[0]; + of_node_put(avr); + if (!avr_clock || !phys_addr) return -EINVAL; diff --git a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c index 8b2b422103563ff6f8196a89651424692710d140..ddf0c652af8000f058594e43f1b841d928d0c683 100644 --- a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c +++ b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c @@ -135,6 +135,9 @@ static void __init mpc7448_hpc2_init_IRQ(void) tsi108_pci_int_init(cascade_node); irq_set_handler_data(cascade_pci_irq, mpic); irq_set_chained_handler(cascade_pci_irq, tsi108_irq_cascade); + + of_node_put(tsi_pci); + of_node_put(cascade_node); #endif /* Configure MPIC outputs to CPU0 */ tsi108_write_reg(TSI108_MPIC_OFFSET + 0x30c, 0); diff --git a/arch/powerpc/platforms/embedded6xx/wii.c b/arch/powerpc/platforms/embedded6xx/wii.c index 9e03ff8f631c4f14d929c37149acbe306d99fb4a..f4e654a9d4ff6d79f4216ccb09fc3e0a6dce909d 100644 --- a/arch/powerpc/platforms/embedded6xx/wii.c +++ b/arch/powerpc/platforms/embedded6xx/wii.c @@ -15,8 +15,6 @@ #include #include #include -#include -#include #include #include @@ -49,19 +47,6 @@ static void __iomem *hw_ctrl; static void __iomem *hw_gpio; -static int __init page_aligned(unsigned long x) -{ - return !(x & (PAGE_SIZE-1)); -} - -void __init wii_memory_fixups(void) -{ - struct memblock_region *p = memblock.memory.regions; - - BUG_ON(memblock.memory.cnt != 2); - BUG_ON(!page_aligned(p[0].base) || !page_aligned(p[1].base)); -} - static void __noreturn wii_spin(void) { local_irq_disable(); diff --git a/arch/powerpc/platforms/maple/time.c b/arch/powerpc/platforms/maple/time.c index 823e219ef8ee7c86560445d19e4314a599174403..91606411d2e08b2ca0c5938829fb2ca5b088cbda 100644 --- a/arch/powerpc/platforms/maple/time.c +++ b/arch/powerpc/platforms/maple/time.c @@ -153,6 +153,7 @@ time64_t __init maple_get_boot_time(void) maple_rtc_addr); } bail: + of_node_put(rtcs); if (maple_rtc_addr == 0) { maple_rtc_addr = RTC_PORT(0); /* legacy address */ printk(KERN_INFO "Maple: No device node for RTC, assuming " diff --git a/arch/powerpc/platforms/pasemi/misc.c b/arch/powerpc/platforms/pasemi/misc.c index f859ada29074a31903a333e2e90babea31603441..9e9a7e46288a884554a1d21cb07c28daef186680 100644 --- a/arch/powerpc/platforms/pasemi/misc.c +++ b/arch/powerpc/platforms/pasemi/misc.c @@ -36,8 +36,7 @@ static int __init find_i2c_driver(struct device_node *node, for (i = 0; i < ARRAY_SIZE(i2c_devices); i++) { if (!of_device_is_compatible(node, i2c_devices[i].of_device)) continue; - if (strlcpy(info->type, i2c_devices[i].i2c_type, - I2C_NAME_SIZE) >= I2C_NAME_SIZE) + if (strscpy(info->type, i2c_devices[i].i2c_type, I2C_NAME_SIZE) < 0) return -ENOMEM; return 0; } diff --git a/arch/powerpc/platforms/pasemi/pci.c b/arch/powerpc/platforms/pasemi/pci.c index 55f0160910bf487fa94d36231eb728610e6112bc..f27d314147373294b6f476980014275edbdaa32c 100644 --- a/arch/powerpc/platforms/pasemi/pci.c +++ b/arch/powerpc/platforms/pasemi/pci.c @@ -270,18 +270,12 @@ static int __init pas_add_bridge(struct device_node *dev) void __init pas_pci_init(void) { - struct device_node *np, *root; + struct device_node *np; int res; - root = of_find_node_by_path("/"); - if (!root) { - pr_crit("pas_pci_init: can't find root of device tree\n"); - return; - } - pci_set_flags(PCI_SCAN_ALL_PCIE_DEVS); - np = of_find_compatible_node(root, NULL, "pasemi,rootbus"); + np = of_find_compatible_node(of_root, NULL, "pasemi,rootbus"); if (np) { res = pas_add_bridge(np); of_node_put(np); diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c index 5cc958adba13f1280b089261a4db3e8d3d161bd7..0382d20b56193446cdf2037b0dc30451a30cd8e1 100644 --- a/arch/powerpc/platforms/powermac/feature.c +++ b/arch/powerpc/platforms/powermac/feature.c @@ -2632,31 +2632,31 @@ static void __init probe_one_macio(const char *name, const char *compat, int typ if (!macio_chips[i].of_node) break; if (macio_chips[i].of_node == node) - return; + goto out_put; } if (i >= MAX_MACIO_CHIPS) { printk(KERN_ERR "pmac_feature: Please increase MAX_MACIO_CHIPS !\n"); printk(KERN_ERR "pmac_feature: %pOF skipped\n", node); - return; + goto out_put; } addrp = of_get_pci_address(node, 0, &size, NULL); if (addrp == NULL) { printk(KERN_ERR "pmac_feature: %pOF: can't find base !\n", node); - return; + goto out_put; } addr = of_translate_address(node, addrp); if (addr == 0) { printk(KERN_ERR "pmac_feature: %pOF, can't translate base !\n", node); - return; + goto out_put; } base = ioremap(addr, (unsigned long)size); if (!base) { printk(KERN_ERR "pmac_feature: %pOF, can't map mac-io chip !\n", node); - return; + goto out_put; } if (type == macio_keylargo || type == macio_keylargo2) { const u32 *did = of_get_property(node, "device-id", NULL); @@ -2677,6 +2677,11 @@ static void __init probe_one_macio(const char *name, const char *compat, int typ macio_chips[i].rev = *revp; printk(KERN_INFO "Found a %s mac-io controller, rev: %d, mapped at 0x%p\n", macio_names[type], macio_chips[i].rev, macio_chips[i].base); + + return; + +out_put: + of_node_put(node); } static int __init diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index c1c430c66dc9e9aa7ea36574e3f0e1648c27a769..40f3aa432fbacd8c07c7dd716810a97dc45bb16e 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -627,6 +627,7 @@ static void __init kw_i2c_probe(void) if (parent == NULL) continue; chans = parent->name[0] == 'u' ? 2 : 1; + of_node_put(parent); for (i = 0; i < chans; i++) kw_i2c_add(host, np, np, i); } else { diff --git a/arch/powerpc/platforms/powermac/pfunc_base.c b/arch/powerpc/platforms/powermac/pfunc_base.c index 9c2947a3edd5f4fc938a9dedcf551297d9116595..085e0ad20eba588501641259a4d6831af6a1b362 100644 --- a/arch/powerpc/platforms/powermac/pfunc_base.c +++ b/arch/powerpc/platforms/powermac/pfunc_base.c @@ -136,6 +136,8 @@ static void __init macio_gpio_init_one(struct macio_chip *macio) for_each_child_of_node(gparent, gp) pmf_do_functions(gp, NULL, 0, PMF_FLAGS_ON_INIT, NULL); + of_node_put(gparent); + /* Note: We do not at this point implement the "at sleep" or "at wake" * functions. I yet to find any for GPIOs anyway */ diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index d9df45741ecefbaaffcf85be7a5a706541532ce0..5b26a9012d2e2be6e3d33635c3508c38b2cb0645 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -711,6 +711,7 @@ static void __init smp_core99_setup(int ncpus) printk(KERN_INFO "Processor timebase sync using" " platform function\n"); } + of_node_put(cpus); } #else /* CONFIG_PPC64 */ diff --git a/arch/powerpc/platforms/powermac/udbg_scc.c b/arch/powerpc/platforms/powermac/udbg_scc.c index 734df5a32f994fc1ecbed4156e49a027eb9f3949..1b7c39e841ee7d7c73796bf6ec1b930f4d1c8f82 100644 --- a/arch/powerpc/platforms/powermac/udbg_scc.c +++ b/arch/powerpc/platforms/powermac/udbg_scc.c @@ -81,10 +81,14 @@ void __init udbg_scc_init(int force_scc) if (path != NULL) stdout = of_find_node_by_path(path); for_each_child_of_node(escc, ch) { - if (ch == stdout) + if (ch == stdout) { + of_node_put(ch_def); ch_def = of_node_get(ch); - if (of_node_name_eq(ch, "ch-a")) + } + if (of_node_name_eq(ch, "ch-a")) { + of_node_put(ch_a); ch_a = of_node_get(ch); + } } if (ch_def == NULL && !force_scc) goto bail; diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index 6f94b808dd39ac3d7369ee2fb6e3514d88d03bf7..841cb7f31f4f4d0f17641a2c38e3d2a391d385c3 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -1411,7 +1411,7 @@ static int __init pnv_parse_cpuidle_dt(void) goto out; } for (i = 0; i < nr_idle_states; i++) - strlcpy(pnv_idle_states[i].name, temp_string[i], + strscpy(pnv_idle_states[i].name, temp_string[i], PNV_IDLE_NAME_LEN); nr_pnv_idle_states = nr_idle_states; rc = 0; @@ -1419,6 +1419,7 @@ out: kfree(temp_u32); kfree(temp_u64); kfree(temp_string); + of_node_put(np); return rc; } diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c index 27c936075031a652c869210c83b4ca3922713889..629067781cec04773ae5187db1dccd0dad0793bd 100644 --- a/arch/powerpc/platforms/powernv/ocxl.c +++ b/arch/powerpc/platforms/powernv/ocxl.c @@ -478,10 +478,8 @@ EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release); int pnv_ocxl_spa_remove_pe_from_cache(void *platform_data, int pe_handle) { struct spa_data *data = (struct spa_data *) platform_data; - int rc; - rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); - return rc; + return opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); } EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe_from_cache); diff --git a/arch/powerpc/platforms/powernv/opal-core.c b/arch/powerpc/platforms/powernv/opal-core.c index adcb1a1a2bfe805519e1774101ecdc5c5eb913c9..bb7657115f1d276372d53b6edea9be4b89c7ce4b 100644 --- a/arch/powerpc/platforms/powernv/opal-core.c +++ b/arch/powerpc/platforms/powernv/opal-core.c @@ -348,6 +348,8 @@ static int __init create_opalcore(void) if (!dn || ret) pr_warn("WARNING: Failed to read OPAL base & entry values\n"); + of_node_put(dn); + /* Use count to keep track of the program headers */ count = 0; diff --git a/arch/powerpc/platforms/powernv/opal-powercap.c b/arch/powerpc/platforms/powernv/opal-powercap.c index 64506b46e77bd16c099dad55948381a798577c3a..7bfe4cbeb35a9934978b384c070d29cb14709c8f 100644 --- a/arch/powerpc/platforms/powernv/opal-powercap.c +++ b/arch/powerpc/platforms/powernv/opal-powercap.c @@ -153,7 +153,7 @@ void __init opal_powercap_init(void) pcaps = kcalloc(of_get_child_count(powercap), sizeof(*pcaps), GFP_KERNEL); if (!pcaps) - return; + goto out_put_powercap; powercap_kobj = kobject_create_and_add("powercap", opal_kobj); if (!powercap_kobj) { @@ -226,6 +226,7 @@ void __init opal_powercap_init(void) } i++; } + of_node_put(powercap); return; @@ -236,6 +237,9 @@ out_pcaps_pattrs: kfree(pcaps[i].pg.name); } kobject_put(powercap_kobj); + of_node_put(node); out_pcaps: kfree(pcaps); +out_put_powercap: + of_node_put(powercap); } diff --git a/arch/powerpc/platforms/powernv/opal-psr.c b/arch/powerpc/platforms/powernv/opal-psr.c index 69d7e75950d135952d5408f3887ba93eb7746c97..6441e17b699659f2011f733ef8e0708135995cf7 100644 --- a/arch/powerpc/platforms/powernv/opal-psr.c +++ b/arch/powerpc/platforms/powernv/opal-psr.c @@ -135,7 +135,7 @@ void __init opal_psr_init(void) psr_attrs = kcalloc(of_get_child_count(psr), sizeof(*psr_attrs), GFP_KERNEL); if (!psr_attrs) - return; + goto out_put_psr; psr_kobj = kobject_create_and_add("psr", opal_kobj); if (!psr_kobj) { @@ -162,10 +162,14 @@ void __init opal_psr_init(void) } i++; } + of_node_put(psr); return; out_kobj: + of_node_put(node); kobject_put(psr_kobj); out: kfree(psr_attrs); +out_put_psr: + of_node_put(psr); } diff --git a/arch/powerpc/platforms/powernv/opal-sensor-groups.c b/arch/powerpc/platforms/powernv/opal-sensor-groups.c index 8fba7d25ae565768d7a7ae386576f353d5af765e..9944376b115c5e367936672ad0aa852d1b324871 100644 --- a/arch/powerpc/platforms/powernv/opal-sensor-groups.c +++ b/arch/powerpc/platforms/powernv/opal-sensor-groups.c @@ -170,7 +170,7 @@ void __init opal_sensor_groups_init(void) sgs = kcalloc(of_get_child_count(sg), sizeof(*sgs), GFP_KERNEL); if (!sgs) - return; + goto out_sg_put; sg_kobj = kobject_create_and_add("sensor_groups", opal_kobj); if (!sg_kobj) { @@ -222,6 +222,7 @@ void __init opal_sensor_groups_init(void) } i++; } + of_node_put(sg); return; @@ -231,6 +232,9 @@ out_sgs_sgattrs: kfree(sgs[i].sg.attrs); } kobject_put(sg_kobj); + of_node_put(node); out_sgs: kfree(sgs); +out_sg_put: + of_node_put(sg); } diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index e5acc33b3b200bda05d37c872930dc91cc430b30..0ed95f753416bfa3b3afd38211a51841866b8df2 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -57,7 +57,7 @@ opal_return: .long 0xa64b7b7d /* mthsrr1 r11 */ .long 0x2402004c /* hrfid */ #endif - ld r2,PACATOC(r13) + LOAD_PACA_TOC() ld r0,PPC_LR_STKOFF(r1) mtlr r0 blr diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 55a8fbfdb5b2874c673eb9dce310fbed5e01be2b..cdf3838f08d3762406e3880b82b3f7da0255989f 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -892,6 +892,7 @@ static void opal_export_attrs(void) kobj = kobject_create_and_add("exports", opal_kobj); if (!kobj) { pr_warn("kobject_create_and_add() of exports failed\n"); + of_node_put(np); return; } @@ -952,6 +953,8 @@ static void __init opal_imc_init_dev(void) np = of_find_compatible_node(NULL, NULL, IMC_DTB_COMPAT); if (np) of_platform_device_create(np, NULL, NULL); + + of_node_put(np); } static int kopald(void *unused) diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 9de9b2fb163d35cdce4f2e4fe797dc4f7813be0d..5c144c05cbfd81d470b80953294043a6a0704a71 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -67,7 +67,7 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, vaf.va = &args; if (pe->flags & PNV_IODA_PE_DEV) - strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix)); + strscpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix)); else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) sprintf(pfix, "%04x:%02x ", pci_domain_nr(pe->pbus), pe->pbus->number); diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index dac545aa030824f12c2c87e98c9b000e1991ab14..61ab2d38ff4be98993ed85afd7edb8e796c9891b 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -207,8 +208,29 @@ static void __init pnv_setup_arch(void) pnv_rng_init(); } +static void __init pnv_add_hw_description(void) +{ + struct device_node *dn; + const char *s; + + dn = of_find_node_by_path("/ibm,opal/firmware"); + if (!dn) + return; + + if (of_property_read_string(dn, "version", &s) == 0 || + of_property_read_string(dn, "git-id", &s) == 0) + seq_buf_printf(&ppc_hw_desc, "opal:%s ", s); + + if (of_property_read_string(dn, "mi-version", &s) == 0) + seq_buf_printf(&ppc_hw_desc, "mi:%s ", s); + + of_node_put(dn); +} + static void __init pnv_init(void) { + pnv_add_hw_description(); + /* * Initialize the LPC bus now so that legacy serial * ports can be found on it diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index fb6499977f99a56d4fa5055c035701f61f4a510d..a3b4d99567cbec70bdfbc49e49a920269648561b 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -23,13 +23,21 @@ config PPC_PSERIES select SWIOTLB default y +config PARAVIRT + bool + config PARAVIRT_SPINLOCKS bool +config PARAVIRT_TIME_ACCOUNTING + select PARAVIRT + bool + config PPC_SPLPAR bool "Support for shared-processor logical partitions" depends on PPC_PSERIES select PARAVIRT_SPINLOCKS if PPC_QUEUED_SPINLOCKS + select PARAVIRT_TIME_ACCOUNTING if VIRT_CPU_ACCOUNTING_GEN default y help Enabling this option will make the kernel run more efficiently diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile index 14e143b946a36822c260674cc9d26ab9fcc92d61..92310202bdd7604268c3058dc6b17f376f274aa0 100644 --- a/arch/powerpc/platforms/pseries/Makefile +++ b/arch/powerpc/platforms/pseries/Makefile @@ -7,7 +7,7 @@ obj-y := lpar.o hvCall.o nvram.o reconfig.o \ setup.o iommu.o event_sources.o ras.o \ firmware.o power.o dlpar.o mobility.o rng.o \ pci.o pci_dlpar.o eeh_pseries.o msi.o \ - papr_platform_attributes.o + papr_platform_attributes.o dtl.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_KEXEC_CORE) += kexec.o obj-$(CONFIG_PSERIES_ENERGY) += pseries_energy.o @@ -19,7 +19,6 @@ obj-$(CONFIG_HVC_CONSOLE) += hvconsole.o obj-$(CONFIG_HVCS) += hvcserver.o obj-$(CONFIG_HCALL_STATS) += hvCall_inst.o obj-$(CONFIG_CMM) += cmm.o -obj-$(CONFIG_DTL) += dtl.o obj-$(CONFIG_IO_EVENT_IRQ) += io_event_irq.o obj-$(CONFIG_LPARCFG) += lparcfg.o obj-$(CONFIG_IBMVIO) += vio.o diff --git a/arch/powerpc/platforms/pseries/dtl.c b/arch/powerpc/platforms/pseries/dtl.c index 352af5b14a0f7bc62ec2f9d1ecdcc3a9dea49613..3f1cdccebc9c156e48ec3f19d08f412bc9653a70 100644 --- a/arch/powerpc/platforms/pseries/dtl.c +++ b/arch/powerpc/platforms/pseries/dtl.c @@ -18,6 +18,7 @@ #include #include +#ifdef CONFIG_DTL struct dtl { struct dtl_entry *buf; int cpu; @@ -37,6 +38,15 @@ static u8 dtl_event_mask = DTL_LOG_ALL; static int dtl_buf_entries = N_DISPATCH_LOG; #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE + +/* + * When CONFIG_VIRT_CPU_ACCOUNTING_NATIVE = y, the cpu accounting code controls + * reading from the dispatch trace log. If other code wants to consume + * DTL entries, it can set this pointer to a function that will get + * called once for each DTL entry that gets processed. + */ +static void (*dtl_consumer)(struct dtl_entry *entry, u64 index); + struct dtl_ring { u64 write_index; struct dtl_entry *write_ptr; @@ -355,3 +365,81 @@ static int dtl_init(void) return 0; } machine_arch_initcall(pseries, dtl_init); +#endif /* CONFIG_DTL */ + +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE +/* + * Scan the dispatch trace log and count up the stolen time. + * Should be called with interrupts disabled. + */ +static notrace u64 scan_dispatch_log(u64 stop_tb) +{ + u64 i = local_paca->dtl_ridx; + struct dtl_entry *dtl = local_paca->dtl_curr; + struct dtl_entry *dtl_end = local_paca->dispatch_log_end; + struct lppaca *vpa = local_paca->lppaca_ptr; + u64 tb_delta; + u64 stolen = 0; + u64 dtb; + + if (!dtl) + return 0; + + if (i == be64_to_cpu(vpa->dtl_idx)) + return 0; + while (i < be64_to_cpu(vpa->dtl_idx)) { + dtb = be64_to_cpu(dtl->timebase); + tb_delta = be32_to_cpu(dtl->enqueue_to_dispatch_time) + + be32_to_cpu(dtl->ready_to_enqueue_time); + barrier(); + if (i + N_DISPATCH_LOG < be64_to_cpu(vpa->dtl_idx)) { + /* buffer has overflowed */ + i = be64_to_cpu(vpa->dtl_idx) - N_DISPATCH_LOG; + dtl = local_paca->dispatch_log + (i % N_DISPATCH_LOG); + continue; + } + if (dtb > stop_tb) + break; +#ifdef CONFIG_DTL + if (dtl_consumer) + dtl_consumer(dtl, i); +#endif + stolen += tb_delta; + ++i; + ++dtl; + if (dtl == dtl_end) + dtl = local_paca->dispatch_log; + } + local_paca->dtl_ridx = i; + local_paca->dtl_curr = dtl; + return stolen; +} + +/* + * Accumulate stolen time by scanning the dispatch trace log. + * Called on entry from user mode. + */ +void notrace pseries_accumulate_stolen_time(void) +{ + u64 sst, ust; + struct cpu_accounting_data *acct = &local_paca->accounting; + + sst = scan_dispatch_log(acct->starttime_user); + ust = scan_dispatch_log(acct->starttime); + acct->stime -= sst; + acct->utime -= ust; + acct->steal_time += ust + sst; +} + +u64 pseries_calculate_stolen_time(u64 stop_tb) +{ + if (!firmware_has_feature(FW_FEATURE_SPLPAR)) + return 0; + + if (get_paca()->dtl_ridx != be64_to_cpu(get_lppaca()->dtl_idx)) + return scan_dispatch_log(stop_tb); + + return 0; +} + +#endif diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c index 0f8cd8b06432d21a02bd0ddae4a0b8677de962ae..e0a7ac5db15d9fa22684b61ce6737c0d31efd80a 100644 --- a/arch/powerpc/platforms/pseries/hotplug-cpu.c +++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c @@ -619,17 +619,21 @@ static ssize_t dlpar_cpu_add(u32 drc_index) static unsigned int pseries_cpuhp_cache_use_count(const struct device_node *cachedn) { unsigned int use_count = 0; - struct device_node *dn; + struct device_node *dn, *tn; WARN_ON(!of_node_is_type(cachedn, "cache")); for_each_of_cpu_node(dn) { - if (of_find_next_cache_node(dn) == cachedn) + tn = of_find_next_cache_node(dn); + of_node_put(tn); + if (tn == cachedn) use_count++; } for_each_node_by_type(dn, "cache") { - if (of_find_next_cache_node(dn) == cachedn) + tn = of_find_next_cache_node(dn); + of_node_put(tn); + if (tn == cachedn) use_count++; } @@ -649,10 +653,13 @@ static int pseries_cpuhp_detach_nodes(struct device_node *cpudn) dn = cpudn; while ((dn = of_find_next_cache_node(dn))) { - if (pseries_cpuhp_cache_use_count(dn) > 1) + if (pseries_cpuhp_cache_use_count(dn) > 1) { + of_node_put(dn); break; + } ret = of_changeset_detach_node(&cs, dn); + of_node_put(dn); if (ret) goto out; } diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S index ab9fc65068617b570d881bd85ed5880e76aae13a..762eb15d3bd42645d3a13287986eb7fef3c558a4 100644 --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -16,7 +16,7 @@ #ifdef CONFIG_TRACEPOINTS #ifndef CONFIG_JUMP_LABEL - .section ".toc","aw" + .data .globl hcall_tracepoint_refcount hcall_tracepoint_refcount: @@ -88,7 +88,7 @@ hcall_tracepoint_refcount: BEGIN_FTR_SECTION; \ b 1f; \ END_FTR_SECTION(0, 1); \ - ld r12,hcall_tracepoint_refcount@toc(r2); \ + LOAD_REG_ADDR(r12, hcall_tracepoint_refcount) ; \ std r12,32(r1); \ cmpdi r12,0; \ bne- LABEL; \ diff --git a/arch/powerpc/platforms/pseries/hvcserver.c b/arch/powerpc/platforms/pseries/hvcserver.c index 96e18d3b2fcf5e53bd82afe00b9b684199f69111..d48c9c7ce10f1c036f6f08fb5672633dc0e20c88 100644 --- a/arch/powerpc/platforms/pseries/hvcserver.c +++ b/arch/powerpc/platforms/pseries/hvcserver.c @@ -176,7 +176,7 @@ int hvcs_get_partner_info(uint32_t unit_address, struct list_head *head, = (unsigned int)last_p_partition_ID; /* copy the Null-term char too */ - strlcpy(&next_partner_info->location_code[0], + strscpy(&next_partner_info->location_code[0], (char *)&pi_buff[2], sizeof(next_partner_info->location_code)); diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c index 7ee3ed7d6cc211018773207d029735106f7c190f..a870cada7acd2313ec8c0d134a39a27a529507fe 100644 --- a/arch/powerpc/platforms/pseries/ibmebus.c +++ b/arch/powerpc/platforms/pseries/ibmebus.c @@ -152,7 +152,11 @@ static const struct dma_map_ops ibmebus_dma_ops = { static int ibmebus_match_path(struct device *dev, const void *data) { struct device_node *dn = to_platform_device(dev)->dev.of_node; - return (of_find_node_by_path(data) == dn); + struct device_node *tn = of_find_node_by_path(data); + + of_node_put(tn); + + return (tn == dn); } static int ibmebus_match_node(struct device *dev, const void *data) diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index e6c117fb64913fe45cda16c169deb5b7080a0a75..97ef6499e50191708afe9f4d72bdcad75717795b 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -660,6 +660,17 @@ static int __init vcpudispatch_stats_procfs_init(void) } machine_device_initcall(pseries, vcpudispatch_stats_procfs_init); + +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING +u64 pseries_paravirt_steal_clock(int cpu) +{ + struct lppaca *lppaca = &lppaca_of(cpu); + + return be64_to_cpu(READ_ONCE(lppaca->enqueue_dispatch_tb)) + + be64_to_cpu(READ_ONCE(lppaca->ready_enqueue_tb)); +} +#endif + #endif /* CONFIG_PPC_SPLPAR */ void vpa_init(int cpu) diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c index 3d36a8955eaf87456657e7b520285436559ab952..634fac5db3f98da9400e235a107e4cf5318570d7 100644 --- a/arch/powerpc/platforms/pseries/mobility.c +++ b/arch/powerpc/platforms/pseries/mobility.c @@ -216,7 +216,7 @@ static int update_dt_node(struct device_node *dn, s32 scope) nprops = be32_to_cpu(upwa->nprops); /* On the first call to ibm,update-properties for a node the - * the first property value descriptor contains an empty + * first property value descriptor contains an empty * property name, the property value length encoded as u32, * and the property value is the node path being updated. */ @@ -740,11 +740,19 @@ static int pseries_migrate_partition(u64 handle) #ifdef CONFIG_PPC_WATCHDOG factor = nmi_wd_lpm_factor; #endif + /* + * When the migration is initiated, the hypervisor changes VAS + * mappings to prepare before OS gets the notification and + * closes all VAS windows. NX generates continuous faults during + * this time and the user space can not differentiate these + * faults from the migration event. So reduce this time window + * by closing VAS windows at the beginning of this function. + */ + vas_migration_handler(VAS_SUSPEND); + ret = wait_for_vasi_session_suspending(handle); if (ret) - return ret; - - vas_migration_handler(VAS_SUSPEND); + goto out; if (factor) watchdog_nmi_set_timeout_pct(factor); @@ -765,6 +773,7 @@ static int pseries_migrate_partition(u64 handle) if (factor) watchdog_nmi_set_timeout_pct(0); +out: vas_migration_handler(VAS_RESUME); return ret; diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index 20f6ed813bffb917365fec841ef0d153eb709517..2f8385523a1320047925af07a7219618a48c60d5 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -124,9 +124,6 @@ struct papr_scm_priv { /* The bits which needs to be overridden */ u64 health_bitmap_inject_mask; - - /* array to have event_code and stat_id mappings */ - u8 *nvdimm_events_map; }; static int papr_scm_pmem_flush(struct nd_region *nd_region, @@ -350,6 +347,25 @@ static ssize_t drc_pmem_query_stats(struct papr_scm_priv *p, #ifdef CONFIG_PERF_EVENTS #define to_nvdimm_pmu(_pmu) container_of(_pmu, struct nvdimm_pmu, pmu) +static const char * const nvdimm_events_map[] = { + [1] = "CtlResCt", + [2] = "CtlResTm", + [3] = "PonSecs ", + [4] = "MemLife ", + [5] = "CritRscU", + [6] = "HostLCnt", + [7] = "HostSCnt", + [8] = "HostSDur", + [9] = "HostLDur", + [10] = "MedRCnt ", + [11] = "MedWCnt ", + [12] = "MedRDur ", + [13] = "MedWDur ", + [14] = "CchRHCnt", + [15] = "CchWHCnt", + [16] = "FastWCnt", +}; + static int papr_scm_pmu_get_value(struct perf_event *event, struct device *dev, u64 *count) { struct papr_scm_perf_stat *stat; @@ -357,11 +373,15 @@ static int papr_scm_pmu_get_value(struct perf_event *event, struct device *dev, struct papr_scm_priv *p = dev_get_drvdata(dev); int rc, size; + /* Invalid eventcode */ + if (event->attr.config == 0 || event->attr.config >= ARRAY_SIZE(nvdimm_events_map)) + return -EINVAL; + /* Allocate request buffer enough to hold single performance stat */ size = sizeof(struct papr_scm_perf_stats) + sizeof(struct papr_scm_perf_stat); - if (!p || !p->nvdimm_events_map) + if (!p) return -EINVAL; stats = kzalloc(size, GFP_KERNEL); @@ -370,7 +390,7 @@ static int papr_scm_pmu_get_value(struct perf_event *event, struct device *dev, stat = &stats->scm_statistic[0]; memcpy(&stat->stat_id, - &p->nvdimm_events_map[event->attr.config * sizeof(stat->stat_id)], + nvdimm_events_map[event->attr.config], sizeof(stat->stat_id)); stat->stat_val = 0; @@ -458,56 +478,6 @@ static void papr_scm_pmu_del(struct perf_event *event, int flags) papr_scm_pmu_read(event); } -static int papr_scm_pmu_check_events(struct papr_scm_priv *p, struct nvdimm_pmu *nd_pmu) -{ - struct papr_scm_perf_stat *stat; - struct papr_scm_perf_stats *stats; - u32 available_events; - int index, rc = 0; - - if (!p->stat_buffer_len) - return -ENOENT; - - available_events = (p->stat_buffer_len - sizeof(struct papr_scm_perf_stats)) - / sizeof(struct papr_scm_perf_stat); - if (available_events == 0) - return -EOPNOTSUPP; - - /* Allocate the buffer for phyp where stats are written */ - stats = kzalloc(p->stat_buffer_len, GFP_KERNEL); - if (!stats) { - rc = -ENOMEM; - return rc; - } - - /* Called to get list of events supported */ - rc = drc_pmem_query_stats(p, stats, 0); - if (rc) - goto out; - - /* - * Allocate memory and populate nvdimm_event_map. - * Allocate an extra element for NULL entry - */ - p->nvdimm_events_map = kcalloc(available_events + 1, - sizeof(stat->stat_id), - GFP_KERNEL); - if (!p->nvdimm_events_map) { - rc = -ENOMEM; - goto out; - } - - /* Copy all stat_ids to event map */ - for (index = 0, stat = stats->scm_statistic; - index < available_events; index++, ++stat) { - memcpy(&p->nvdimm_events_map[index * sizeof(stat->stat_id)], - &stat->stat_id, sizeof(stat->stat_id)); - } -out: - kfree(stats); - return rc; -} - static void papr_scm_pmu_register(struct papr_scm_priv *p) { struct nvdimm_pmu *nd_pmu; @@ -519,9 +489,10 @@ static void papr_scm_pmu_register(struct papr_scm_priv *p) goto pmu_err_print; } - rc = papr_scm_pmu_check_events(p, nd_pmu); - if (rc) + if (!p->stat_buffer_len) { + rc = -ENOENT; goto pmu_check_events_err; + } nd_pmu->pmu.task_ctx_nr = perf_invalid_context; nd_pmu->pmu.name = nvdimm_name(p->nvdimm); @@ -539,7 +510,7 @@ static void papr_scm_pmu_register(struct papr_scm_priv *p) rc = register_nvdimm_pmu(nd_pmu, p->pdev); if (rc) - goto pmu_register_err; + goto pmu_check_events_err; /* * Set archdata.priv value to nvdimm_pmu structure, to handle the @@ -548,8 +519,6 @@ static void papr_scm_pmu_register(struct papr_scm_priv *p) p->pdev->archdata.priv = nd_pmu; return; -pmu_register_err: - kfree(p->nvdimm_events_map); pmu_check_events_err: kfree(nd_pmu); pmu_err_print: @@ -1560,7 +1529,6 @@ static int papr_scm_remove(struct platform_device *pdev) unregister_nvdimm_pmu(pdev->archdata.priv); pdev->archdata.priv = NULL; - kfree(p->nvdimm_events_map); kfree(p->bus_desc.provider_name); kfree(p); diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index 52aaa2894606835d0849a01e24d0fd57648b9223..f4b5b5a64db3d3a0e9ae65282f39ec7c60e160e6 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "plpks.h" @@ -457,4 +458,4 @@ static __init int pseries_plpks_init(void) return rc; } -arch_initcall(pseries_plpks_init); +machine_arch_initcall(pseries, pseries_plpks_init); diff --git a/arch/powerpc/platforms/pseries/reconfig.c b/arch/powerpc/platforms/pseries/reconfig.c index cad7a0c9311705912a64b1e9fd9e321cc532f39f..599bd2c7851444af9d60abde6b37504b2bd3c014 100644 --- a/arch/powerpc/platforms/pseries/reconfig.c +++ b/arch/powerpc/platforms/pseries/reconfig.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -361,6 +362,10 @@ static ssize_t ofdt_write(struct file *file, const char __user *buf, size_t coun char *kbuf; char *tmp; + rv = security_locked_down(LOCKDOWN_DEVICE_TREE); + if (rv) + return rv; + kbuf = memdup_user_nul(buf, count); if (IS_ERR(kbuf)) return PTR_ERR(kbuf); diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 489f4c4df468a1e49634c49ffe6cc3e508e24a62..8ef3270515a9b649cd8005886a14a99f2075347b 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -80,6 +81,20 @@ DEFINE_STATIC_KEY_FALSE(shared_processor); EXPORT_SYMBOL(shared_processor); +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING +struct static_key paravirt_steal_enabled; +struct static_key paravirt_steal_rq_enabled; + +static bool steal_acc = true; +static int __init parse_no_stealacc(char *arg) +{ + steal_acc = false; + return 0; +} + +early_param("no-steal-acc", parse_no_stealacc); +#endif + int CMO_PrPSP = -1; int CMO_SecPSP = -1; unsigned long CMO_PageSize = (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K); @@ -834,6 +849,11 @@ static void __init pSeries_setup_arch(void) if (lppaca_shared_proc(get_lppaca())) { static_branch_enable(&shared_processor); pv_spinlocks_init(); +#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING + static_key_slow_inc(¶virt_steal_enabled); + if (steal_acc) + static_key_slow_inc(¶virt_steal_rq_enabled); +#endif } ppc_md.power_save = pseries_lpar_idle; @@ -992,6 +1012,33 @@ static void __init pSeries_cmo_feature_init(void) pr_debug(" <- fw_cmo_feature_init()\n"); } +static void __init pseries_add_hw_description(void) +{ + struct device_node *dn; + const char *s; + + dn = of_find_node_by_path("/openprom"); + if (dn) { + if (of_property_read_string(dn, "model", &s) == 0) + seq_buf_printf(&ppc_hw_desc, "of:%s ", s); + + of_node_put(dn); + } + + dn = of_find_node_by_path("/hypervisor"); + if (dn) { + if (of_property_read_string(dn, "compatible", &s) == 0) + seq_buf_printf(&ppc_hw_desc, "hv:%s ", s); + + of_node_put(dn); + return; + } + + if (of_property_read_bool(of_root, "ibm,powervm-partition") || + of_property_read_bool(of_root, "ibm,fw-net-version")) + seq_buf_printf(&ppc_hw_desc, "hv:phyp "); +} + /* * Early initialization. Relocation is on but do not reference unbolted pages */ @@ -999,6 +1046,8 @@ static void __init pseries_init(void) { pr_debug(" -> pseries_init()\n"); + pseries_add_hw_description(); + #ifdef CONFIG_HVC_CONSOLE if (firmware_has_feature(FW_FEATURE_LPAR)) hvc_vio_init_early(); diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c index 7e6e6dd2e33e9b11ce40f49d31347d32e39041f6..0e0524cbe20c946e20698b47c1bb7a9d066d45a1 100644 --- a/arch/powerpc/platforms/pseries/vas.c +++ b/arch/powerpc/platforms/pseries/vas.c @@ -333,7 +333,7 @@ static struct vas_window *vas_allocate_window(int vas_id, u64 flags, * So no unpacking needs to be done. */ rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, domain, - VPHN_FLAG_VCPU, smp_processor_id()); + VPHN_FLAG_VCPU, hard_smp_processor_id()); if (rc != H_SUCCESS) { pr_err("H_HOME_NODE_ASSOCIATIVITY error: %d\n", rc); goto out; @@ -501,14 +501,10 @@ static const struct vas_user_win_ops vops_pseries = { int vas_register_api_pseries(struct module *mod, enum vas_cop_type cop_type, const char *name) { - int rc; - if (!copypaste_feat) return -ENOTSUPP; - rc = vas_register_coproc_api(mod, cop_type, name, &vops_pseries); - - return rc; + return vas_register_coproc_api(mod, cop_type, name, &vops_pseries); } EXPORT_SYMBOL_GPL(vas_register_api_pseries); diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index ef9a5999fa93ddfef9e8851701a8a2d517c60761..73c2d70706c0add48ea9553fec807c6953cb5a7d 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -209,8 +209,10 @@ static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) dev_err(&pdev->dev, "node %pOF has an invalid fsl,msi phandle %u\n", hose->dn, np->phandle); + of_node_put(np); return -EINVAL; } + of_node_put(np); } msi_for_each_desc(entry, &pdev->dev, MSI_DESC_NOTASSOCIATED) { diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index af6c8ca824d399ac6400b412f1b064ce1c4277b6..974d3db6faab237dc7d56ce5a89eb0420e8e8c31 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -181,6 +181,7 @@ static int setup_one_atmu(struct ccsr_pci __iomem *pci, static bool is_kdump(void) { struct device_node *node; + bool ret; node = of_find_node_by_type(NULL, "memory"); if (!node) { @@ -188,7 +189,10 @@ static bool is_kdump(void) return false; } - return of_property_read_bool(node, "linux,usable-memory"); + ret = of_property_read_bool(node, "linux,usable-memory"); + of_node_put(node); + + return ret; } /* atmu setup for fsl pci/pcie controller */ @@ -939,7 +943,7 @@ u64 fsl_pci_immrbar_base(struct pci_controller *hose) return 0; } -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 static int mcheck_handle_load(struct pt_regs *regs, u32 inst) { unsigned int rd, ra, rb, d; @@ -1142,7 +1146,6 @@ void __init fsl_pci_assign_primary(void) for_each_matching_node(np, pci_ids) { if (of_device_is_available(np)) { fsl_pci_primary = np; - of_node_put(np); return; } } diff --git a/arch/powerpc/sysdev/fsl_rio.c b/arch/powerpc/sysdev/fsl_rio.c index 4647c6074f3b8f949ff1406bc5462938c65f69b0..c8f044d62fe2e8d7418ef200ae0f859c50e7bc51 100644 --- a/arch/powerpc/sysdev/fsl_rio.c +++ b/arch/powerpc/sysdev/fsl_rio.c @@ -98,7 +98,7 @@ resource_size_t rio_law_start; struct fsl_rio_dbell *dbell; struct fsl_rio_pw *pw; -#ifdef CONFIG_E500 +#ifdef CONFIG_PPC_E500 int fsl_rio_mcheck_exception(struct pt_regs *regs) { const struct exception_table_entry *entry; diff --git a/arch/powerpc/sysdev/mpic_msgr.c b/arch/powerpc/sysdev/mpic_msgr.c index 698fefaaa6dd8a5fe080122e90f8a4a8059039be..a439e33eae0618e2d227e3f7338362781f33c396 100644 --- a/arch/powerpc/sysdev/mpic_msgr.c +++ b/arch/powerpc/sysdev/mpic_msgr.c @@ -121,6 +121,7 @@ static unsigned int mpic_msgr_number_of_blocks(void) count += 1; } + of_node_put(aliases); } return count; @@ -144,12 +145,18 @@ static int mpic_msgr_block_number(struct device_node *node) for (index = 0; index < number_of_blocks; ++index) { struct property *prop; + struct device_node *tn; snprintf(buf, sizeof(buf), "mpic-msgr-block%d", index); prop = of_find_property(aliases, buf, NULL); - if (node == of_find_node_by_path(prop->value)) + tn = of_find_node_by_path(prop->value); + if (node == tn) { + of_node_put(tn); break; + } + of_node_put(tn); } + of_node_put(aliases); return index == number_of_blocks ? -1 : index; } diff --git a/arch/powerpc/sysdev/xics/ics-rtas.c b/arch/powerpc/sysdev/xics/ics-rtas.c index 9e7007f9aca5cbe369ca5941a4ee6c3f964cd83e..f8320f8e5bc79a7a2731a97b6dee2661e41ac9bc 100644 --- a/arch/powerpc/sysdev/xics/ics-rtas.c +++ b/arch/powerpc/sysdev/xics/ics-rtas.c @@ -36,8 +36,8 @@ static void ics_rtas_unmask_irq(struct irq_data *d) server = xics_get_irq_server(d->irq, irq_data_get_affinity_mask(d), 0); - call_status = rtas_call_reentrant(ibm_set_xive, 3, 1, NULL, hw_irq, - server, DEFAULT_PRIORITY); + call_status = rtas_call(ibm_set_xive, 3, 1, NULL, hw_irq, server, + DEFAULT_PRIORITY); if (call_status != 0) { printk(KERN_ERR "%s: ibm_set_xive irq %u server %x returned %d\n", @@ -46,7 +46,7 @@ static void ics_rtas_unmask_irq(struct irq_data *d) } /* Now unmask the interrupt (often a no-op) */ - call_status = rtas_call_reentrant(ibm_int_on, 1, 1, NULL, hw_irq); + call_status = rtas_call(ibm_int_on, 1, 1, NULL, hw_irq); if (call_status != 0) { printk(KERN_ERR "%s: ibm_int_on irq=%u returned %d\n", __func__, hw_irq, call_status); @@ -68,7 +68,7 @@ static void ics_rtas_mask_real_irq(unsigned int hw_irq) if (hw_irq == XICS_IPI) return; - call_status = rtas_call_reentrant(ibm_int_off, 1, 1, NULL, hw_irq); + call_status = rtas_call(ibm_int_off, 1, 1, NULL, hw_irq); if (call_status != 0) { printk(KERN_ERR "%s: ibm_int_off irq=%u returned %d\n", __func__, hw_irq, call_status); @@ -76,8 +76,8 @@ static void ics_rtas_mask_real_irq(unsigned int hw_irq) } /* Have to set XIVE to 0xff to be able to remove a slot */ - call_status = rtas_call_reentrant(ibm_set_xive, 3, 1, NULL, hw_irq, - xics_default_server, 0xff); + call_status = rtas_call(ibm_set_xive, 3, 1, NULL, hw_irq, + xics_default_server, 0xff); if (call_status != 0) { printk(KERN_ERR "%s: ibm_set_xive(0xff) irq=%u returned %d\n", __func__, hw_irq, call_status); @@ -108,7 +108,7 @@ static int ics_rtas_set_affinity(struct irq_data *d, if (hw_irq == XICS_IPI || hw_irq == XICS_IRQ_SPURIOUS) return -1; - status = rtas_call_reentrant(ibm_get_xive, 1, 3, xics_status, hw_irq); + status = rtas_call(ibm_get_xive, 1, 3, xics_status, hw_irq); if (status) { printk(KERN_ERR "%s: ibm,get-xive irq=%u returns %d\n", @@ -126,8 +126,8 @@ static int ics_rtas_set_affinity(struct irq_data *d, pr_debug("%s: irq %d [hw 0x%x] server: 0x%x\n", __func__, d->irq, hw_irq, irq_server); - status = rtas_call_reentrant(ibm_set_xive, 3, 1, NULL, - hw_irq, irq_server, xics_status[1]); + status = rtas_call(ibm_set_xive, 3, 1, NULL, + hw_irq, irq_server, xics_status[1]); if (status) { printk(KERN_ERR "%s: ibm,set-xive irq=%u returns %d\n", @@ -158,7 +158,7 @@ static int ics_rtas_check(struct ics *ics, unsigned int hw_irq) return -EINVAL; /* Check if RTAS knows about this interrupt */ - rc = rtas_call_reentrant(ibm_get_xive, 1, 3, status, hw_irq); + rc = rtas_call(ibm_get_xive, 1, 3, status, hw_irq); if (rc) return -ENXIO; @@ -174,7 +174,7 @@ static long ics_rtas_get_server(struct ics *ics, unsigned long vec) { int rc, status[2]; - rc = rtas_call_reentrant(ibm_get_xive, 1, 3, status, vec); + rc = rtas_call(ibm_get_xive, 1, 3, status, vec); if (rc) return -1; return status[0]; diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index 61b9f98dfd4a4bdb26ed3d759d2464368ff70272..a289cb97c1d7b50526440248576adab3fe387577 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -783,7 +783,7 @@ static int xive_irq_set_type(struct irq_data *d, unsigned int flow_type) * the corresponding descriptor bits mind you but those will in turn * affect the resend function when re-enabling an edge interrupt. * - * Set set the default to edge as explained in map(). + * Set the default to edge as explained in map(). */ if (flow_type == IRQ_TYPE_DEFAULT || flow_type == IRQ_TYPE_NONE) flow_type = IRQ_TYPE_EDGE_RISING; diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c index d25d8c69290942aa4b55461217307ce7967e1b30..3925825954bcceba55ba2a8dffcbbdc5df5dcf68 100644 --- a/arch/powerpc/sysdev/xive/native.c +++ b/arch/powerpc/sysdev/xive/native.c @@ -579,12 +579,12 @@ bool __init xive_native_init(void) /* Resource 1 is HV window */ if (of_address_to_resource(np, 1, &r)) { pr_err("Failed to get thread mgmnt area resource\n"); - return false; + goto err_put; } tima = ioremap(r.start, resource_size(&r)); if (!tima) { pr_err("Failed to map thread mgmnt area\n"); - return false; + goto err_put; } /* Read number of priorities */ @@ -612,7 +612,7 @@ bool __init xive_native_init(void) /* Resource 2 is OS window */ if (of_address_to_resource(np, 2, &r)) { pr_err("Failed to get thread mgmnt area resource\n"); - return false; + goto err_put; } xive_tima_os = r.start; @@ -624,7 +624,7 @@ bool __init xive_native_init(void) rc = opal_xive_reset(OPAL_XIVE_MODE_EXPL); if (rc) { pr_err("Switch to exploitation mode failed with error %lld\n", rc); - return false; + goto err_put; } /* Setup some dummy HV pool VPs */ @@ -634,10 +634,15 @@ bool __init xive_native_init(void) if (!xive_core_init(np, &xive_native_ops, tima, TM_QW3_HV_PHYS, max_prio)) { opal_xive_reset(OPAL_XIVE_MODE_EMU); - return false; + goto err_put; } + of_node_put(np); pr_info("Using %dkB queues\n", 1 << (xive_queue_shift - 10)); return true; + +err_put: + of_node_put(np); + return false; } static bool xive_native_provision_pages(void) diff --git a/arch/powerpc/xmon/ppc.h b/arch/powerpc/xmon/ppc.h index d00f33dcf192b9906684ff95db16427dc8e06de0..1d98b8dd134efeea96799fee847a7fd144706578 100644 --- a/arch/powerpc/xmon/ppc.h +++ b/arch/powerpc/xmon/ppc.h @@ -435,8 +435,6 @@ struct powerpc_macro extern const struct powerpc_macro powerpc_macros[]; extern const int powerpc_num_macros; -extern ppc_cpu_t ppc_parse_cpu (ppc_cpu_t, ppc_cpu_t *, const char *); - static inline long ppc_optional_operand_value (const struct powerpc_operand *operand) { diff --git a/arch/powerpc/xmon/spr_access.S b/arch/powerpc/xmon/spr_access.S index 720a52afdd5813849c473abcdc91cb5667b7fac6..c308ddf268fbc15d274fbebf85e68dd7881f4c8e 100644 --- a/arch/powerpc/xmon/spr_access.S +++ b/arch/powerpc/xmon/spr_access.S @@ -4,12 +4,12 @@ /* unsigned long xmon_mfspr(sprn, default_value) */ _GLOBAL(xmon_mfspr) - PPC_LL r5, .Lmfspr_table@got(r2) + LOAD_REG_ADDR(r5, .Lmfspr_table) b xmon_mxspr /* void xmon_mtspr(sprn, new_value) */ _GLOBAL(xmon_mtspr) - PPC_LL r5, .Lmtspr_table@got(r2) + LOAD_REG_ADDR(r5, .Lmtspr_table) b xmon_mxspr /* diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 26ef3388c24c493cc2dcd67e54c7ae283070f930..f51c882bf9023af3e8007c79fd4bbfee16b49a3e 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -195,7 +195,7 @@ static int do_spu_cmd(void); #ifdef CONFIG_44x static void dump_tlb_44x(void); #endif -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 static void dump_tlb_book3e(void); #endif @@ -288,11 +288,11 @@ Commands:\n\ t print backtrace\n\ x exit monitor and recover\n\ X exit monitor and don't recover\n" -#if defined(CONFIG_PPC64) && !defined(CONFIG_PPC_BOOK3E) +#if defined(CONFIG_PPC_BOOK3S_64) " u dump segment table or SLB\n" #elif defined(CONFIG_PPC_BOOK3S_32) " u dump segment registers\n" -#elif defined(CONFIG_44x) || defined(CONFIG_PPC_BOOK3E) +#elif defined(CONFIG_44x) || defined(CONFIG_PPC_BOOK3E_64) " u dump TLB\n" #endif " U show uptime information\n" @@ -1166,7 +1166,7 @@ cmds(struct pt_regs *excp) case 'u': dump_tlb_44x(); break; -#elif defined(CONFIG_PPC_BOOK3E) +#elif defined(CONFIG_PPC_BOOK3E_64) case 'u': dump_tlb_book3e(); break; @@ -2686,7 +2686,7 @@ static void dump_one_paca(int cpu) DUMP(p, rfi_flush_fallback_area, "%-*px"); #endif DUMP(p, dscr_default, "%#-*llx"); -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 DUMP(p, pgd, "%-*px"); DUMP(p, kernel_pgd, "%-*px"); DUMP(p, tcd_ptr, "%-*px"); @@ -2701,7 +2701,7 @@ static void dump_one_paca(int cpu) DUMP(p, canary, "%#-*lx"); #endif DUMP(p, saved_r1, "%#-*llx"); -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 DUMP(p, trap_save, "%#-*x"); #endif DUMP(p, irq_soft_mask, "%#-*x"); @@ -3823,7 +3823,7 @@ static void dump_tlb_44x(void) } #endif /* CONFIG_44x */ -#ifdef CONFIG_PPC_BOOK3E +#ifdef CONFIG_PPC_BOOK3E_64 static void dump_tlb_book3e(void) { u32 mmucfg, pidmask, lpidmask; @@ -3965,7 +3965,7 @@ static void dump_tlb_book3e(void) } } } -#endif /* CONFIG_PPC_BOOK3E */ +#endif /* CONFIG_PPC_BOOK3E_64 */ static void xmon_init(int enable) { diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ed66c31e465590f94c447a1fee2b590e237073e2..6b48a3ae984394016f8fbd3ec60bc76763baba5d 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -45,6 +45,7 @@ config RISCV select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE if 64BIT + select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU select BUILDTIME_TABLE_SORT if MMU select CLONE_BACKWARDS @@ -52,7 +53,7 @@ config RISCV select COMMON_CLK select CPU_PM if CPU_IDLE select EDAC_SUPPORT - select GENERIC_ARCH_TOPOLOGY if SMP + select GENERIC_ARCH_TOPOLOGY select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_EARLY_IOREMAP @@ -69,6 +70,7 @@ config RISCV select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL if MMU && 64BIT select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO + select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_JUMP_LABEL if !XIP_KERNEL select HAVE_ARCH_JUMP_LABEL_RELATIVE if !XIP_KERNEL @@ -103,6 +105,7 @@ config RISCV select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_POSIX_CPU_TIMERS_TASK_WORK select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_STACKPROTECTOR @@ -227,6 +230,9 @@ config RISCV_DMA_NONCOHERENT select ARCH_HAS_SETUP_DMA_OPS select DMA_DIRECT_REMAP +config AS_HAS_INSN + def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero) + source "arch/riscv/Kconfig.socs" source "arch/riscv/Kconfig.erratas" @@ -309,10 +315,13 @@ config SMP If you don't know what to do here, say N. config NR_CPUS - int "Maximum number of CPUs (2-32)" - range 2 32 + int "Maximum number of CPUs (2-512)" depends on SMP - default "8" + range 2 512 if !SBI_V01 + range 2 32 if SBI_V01 && 32BIT + range 2 64 if SBI_V01 && 64BIT + default "32" if 32BIT + default "64" if 64BIT config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" @@ -386,6 +395,7 @@ config RISCV_ISA_C config RISCV_ISA_SVPBMT bool "SVPBMT extension support" depends on 64BIT && MMU + depends on !XIP_KERNEL select RISCV_ALTERNATIVE default y help diff --git a/arch/riscv/Kconfig.erratas b/arch/riscv/Kconfig.erratas index 6850e9389930240eab28852e1c4f8f7bd3f07fc3..f3623df23b5fcd902339552e303c257a4dd6efa4 100644 --- a/arch/riscv/Kconfig.erratas +++ b/arch/riscv/Kconfig.erratas @@ -46,7 +46,7 @@ config ERRATA_THEAD config ERRATA_THEAD_PBMT bool "Apply T-Head memory type errata" - depends on ERRATA_THEAD && 64BIT + depends on ERRATA_THEAD && 64BIT && MMU select RISCV_ALTERNATIVE_EARLY default y help @@ -57,7 +57,7 @@ config ERRATA_THEAD_PBMT config ERRATA_THEAD_CMO bool "Apply T-Head cache management errata" - depends on ERRATA_THEAD + depends on ERRATA_THEAD && MMU select RISCV_DMA_NONCOHERENT default y help diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 3fa8ef336822438570a95b613d7d89d2ed8872ce..1c8ec656e916248e8da7f1d087d42e7d0bbbfc36 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -37,6 +37,7 @@ else endif ifeq ($(CONFIG_LD_IS_LLD),y) +ifeq ($(shell test $(CONFIG_LLD_VERSION) -lt 150000; echo $$?),0) KBUILD_CFLAGS += -mno-relax KBUILD_AFLAGS += -mno-relax ifndef CONFIG_AS_IS_LLVM @@ -44,6 +45,7 @@ ifndef CONFIG_AS_IS_LLVM KBUILD_AFLAGS += -Wa,-mno-relax endif endif +endif # ISA string setting riscv-march-$(CONFIG_ARCH_RV32I) := rv32ima @@ -110,8 +112,6 @@ else KBUILD_IMAGE := $(boot)/Image.gz endif -head-y := arch/riscv/kernel/head.o - libs-y += arch/riscv/lib/ libs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a @@ -136,10 +136,14 @@ ifneq ($(CONFIG_XIP_KERNEL),y) ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_CANAAN),yy) KBUILD_IMAGE := $(boot)/loader.bin else +ifeq ($(CONFIG_EFI_ZBOOT),) KBUILD_IMAGE := $(boot)/Image.gz +else +KBUILD_IMAGE := $(boot)/vmlinuz.efi +endif endif endif -BOOT_TARGETS := Image Image.gz loader loader.bin xipImage +BOOT_TARGETS := Image Image.gz loader loader.bin xipImage vmlinuz.efi all: $(notdir $(KBUILD_IMAGE)) diff --git a/arch/riscv/boot/.gitignore b/arch/riscv/boot/.gitignore index 0cea9f7fa9d5c86a70eadb312592619045400122..e1bc507e8cb25ea60b4e46b8928d3b7211a7923a 100644 --- a/arch/riscv/boot/.gitignore +++ b/arch/riscv/boot/.gitignore @@ -4,4 +4,5 @@ Image.* loader loader.lds loader.bin +vmlinuz* xipImage diff --git a/arch/riscv/boot/Makefile b/arch/riscv/boot/Makefile index becd0621071cc38a54aca18804cac7f478d5df3d..d1a49adcb1d76283233785ec3bed58e2d1278e47 100644 --- a/arch/riscv/boot/Makefile +++ b/arch/riscv/boot/Makefile @@ -58,3 +58,9 @@ $(obj)/Image.lzo: $(obj)/Image FORCE $(obj)/loader.bin: $(obj)/loader FORCE $(call if_changed,objcopy) + +EFI_ZBOOT_PAYLOAD := Image +EFI_ZBOOT_BFD_TARGET := elf$(BITS)-littleriscv +EFI_ZBOOT_MACH_TYPE := RISCV$(BITS) + +include $(srctree)/drivers/firmware/efi/libstub/Makefile.zboot diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile index 39aae7b04f1cba08cef29b4c0f4475815d330847..7427a20934f375933db2a616c630864c6c0e0366 100644 --- a/arch/riscv/boot/dts/microchip/Makefile +++ b/arch/riscv/boot/dts/microchip/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-icicle-kit.dtb +dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb +dtb-$(CONFIG_SOC_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi index 0d28858b83f2875691a2ea22d2ebe8d1bd17eaf9..24b1cfb9a73e432f2881595ecf2a4319aa6d6fd9 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi @@ -2,20 +2,21 @@ /* Copyright (c) 2020-2021 Microchip Technology Inc */ / { - compatible = "microchip,mpfs-icicle-reference-rtlv2203", "microchip,mpfs"; + compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit", + "microchip,mpfs"; - core_pwm0: pwm@41000000 { + core_pwm0: pwm@40000000 { compatible = "microchip,corepwm-rtl-v4"; - reg = <0x0 0x41000000 0x0 0xF0>; + reg = <0x0 0x40000000 0x0 0xF0>; microchip,sync-update-mask = /bits/ 32 <0>; #pwm-cells = <2>; clocks = <&fabric_clk3>; status = "disabled"; }; - i2c2: i2c@44000000 { + i2c2: i2c@40000200 { compatible = "microchip,corei2c-rtl-v7"; - reg = <0x0 0x44000000 0x0 0x1000>; + reg = <0x0 0x40000200 0x0 0x100>; #address-cells = <1>; #size-cells = <0>; clocks = <&fabric_clk3>; @@ -28,7 +29,7 @@ fabric_clk3: fabric-clk3 { compatible = "fixed-clock"; #clock-cells = <0>; - clock-frequency = <62500000>; + clock-frequency = <50000000>; }; fabric_clk1: fabric-clk1 { @@ -36,4 +37,34 @@ #clock-cells = <0>; clock-frequency = <125000000>; }; + + pcie: pcie@3000000000 { + compatible = "microchip,pcie-host-1.0"; + #address-cells = <0x3>; + #interrupt-cells = <0x1>; + #size-cells = <0x2>; + device_type = "pci"; + reg = <0x30 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; + reg-names = "cfg", "apb"; + bus-range = <0x0 0x7f>; + interrupt-parent = <&plic>; + interrupts = <119>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; + interrupt-map-mask = <0 0 0 7>; + clocks = <&fabric_clk1>, <&fabric_clk3>; + clock-names = "fic1", "fic3"; + ranges = <0x3000000 0x0 0x8000000 0x30 0x8000000 0x0 0x80000000>; + dma-ranges = <0x02000000 0x0 0x00000000 0x0 0x00000000 0x1 0x00000000>; + msi-parent = <&pcie>; + msi-controller; + status = "disabled"; + pcie_intc: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts index 044982a11df5045b2eb7abd13d74dec8ebb69523..ec7b7c2a3ce2827e819f31b855f7ced83d55e482 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts @@ -11,7 +11,8 @@ / { model = "Microchip PolarFire-SoC Icicle Kit"; - compatible = "microchip,mpfs-icicle-kit", "microchip,mpfs"; + compatible = "microchip,mpfs-icicle-reference-rtlv2210", "microchip,mpfs-icicle-kit", + "microchip,mpfs"; aliases { ethernet0 = &mac1; @@ -32,15 +33,26 @@ ddrc_cache_lo: memory@80000000 { device_type = "memory"; - reg = <0x0 0x80000000 0x0 0x2e000000>; + reg = <0x0 0x80000000 0x0 0x40000000>; status = "okay"; }; ddrc_cache_hi: memory@1000000000 { device_type = "memory"; - reg = <0x10 0x0 0x0 0x40000000>; + reg = <0x10 0x40000000 0x0 0x40000000>; status = "okay"; }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + hss_payload: region@BFC00000 { + reg = <0x0 0xBFC00000 0x0 0x400000>; + no-map; + }; + }; }; &core_pwm0 { @@ -84,12 +96,10 @@ phy1: ethernet-phy@9 { reg = <9>; - ti,fifo-depth = <0x1>; }; phy0: ethernet-phy@8 { reg = <8>; - ti,fifo-depth = <0x1>; }; }; @@ -102,7 +112,6 @@ disable-wp; cap-sd-highspeed; cap-mmc-highspeed; - card-detect-delay = <200>; mmc-ddr-1_8v; mmc-hs200-1_8v; sd-uhs-sdr12; diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..7b9ee13b6a3af017b2558e7fbbd0994368618cc3 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfs-fabric.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2022 Microchip Technology Inc */ + +/ { + fabric_clk3: fabric-clk3 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <62500000>; + }; + + fabric_clk1: fabric-clk1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + pcie: pcie@2000000000 { + compatible = "microchip,pcie-host-1.0"; + #address-cells = <0x3>; + #interrupt-cells = <0x1>; + #size-cells = <0x2>; + device_type = "pci"; + reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; + reg-names = "cfg", "apb"; + bus-range = <0x0 0x7f>; + interrupt-parent = <&plic>; + interrupts = <119>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; + interrupt-map-mask = <0 0 0 7>; + clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>; + clock-names = "fic0", "fic1", "fic3"; + ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>; + msi-parent = <&pcie>; + msi-controller; + status = "disabled"; + pcie_intc: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts new file mode 100644 index 0000000000000000000000000000000000000000..184cb36a175e40742763510b8b1e7eca6d9c784a --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Original all-in-one devicetree: + * Copyright (C) 2021-2022 - Wolfgang Grandegger + * Rewritten to use includes: + * Copyright (C) 2022 - Conor Dooley + */ +/dts-v1/; + +#include "mpfs.dtsi" +#include "mpfs-m100pfs-fabric.dtsi" + +/* Clock frequency (in Hz) of the rtcclk */ +#define MTIMER_FREQ 1000000 + +/ { + model = "Aries Embedded M100PFEVPS"; + compatible = "aries,m100pfsevp", "microchip,mpfs"; + + aliases { + ethernet0 = &mac0; + ethernet1 = &mac1; + serial0 = &mmuart0; + serial1 = &mmuart1; + serial2 = &mmuart2; + serial3 = &mmuart3; + serial4 = &mmuart4; + gpio0 = &gpio0; + gpio1 = &gpio2; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; + + cpus { + timebase-frequency = ; + }; + + ddrc_cache_lo: memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x0 0x40000000>; + }; + ddrc_cache_hi: memory@1040000000 { + device_type = "memory"; + reg = <0x10 0x40000000 0x0 0x40000000>; + }; +}; + +&can0 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; +}; + +&i2c1 { + status = "okay"; +}; + +&gpio0 { + interrupts = <13>, <14>, <15>, <16>, + <17>, <18>, <19>, <20>, + <21>, <22>, <23>, <24>, + <25>, <26>; + ngpios = <14>; + status = "okay"; + + pmic-irq-hog { + gpio-hog; + gpios = <13 0>; + input; + }; + + /* Set to low for eMMC, high for SD-card */ + mmc-sel-hog { + gpio-hog; + gpios = <12 0>; + output-high; + }; +}; + +&gpio2 { + interrupts = <13>, <14>, <15>, <16>, + <17>, <18>, <19>, <20>, + <21>, <22>, <23>, <24>, + <25>, <26>, <27>, <28>, + <29>, <30>, <31>, <32>, + <33>, <34>, <35>, <36>, + <37>, <38>, <39>, <40>, + <41>, <42>, <43>, <44>; + status = "okay"; +}; + +&mac0 { + status = "okay"; + phy-mode = "gmii"; + phy-handle = <&phy0>; + phy0: ethernet-phy@0 { + reg = <0>; + }; +}; + +&mac1 { + status = "okay"; + phy-mode = "gmii"; + phy-handle = <&phy1>; + phy1: ethernet-phy@0 { + reg = <0>; + }; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + max-frequency = <50000000>; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + no-1-8-v; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; + disable-wp; + status = "okay"; +}; + +&mmuart1 { + status = "okay"; +}; + +&mmuart2 { + status = "okay"; +}; + +&mmuart3 { + status = "okay"; +}; + +&mmuart4 { + status = "okay"; +}; + +&pcie { + status = "okay"; +}; + +&qspi { + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&rtc { + status = "okay"; +}; + +&spi0 { + status = "okay"; +}; + +&spi1 { + status = "okay"; +}; + +&syscontroller { + status = "okay"; +}; + +&usb { + status = "okay"; + dr_mode = "host"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi index 49380c428ec917cc75c06d18a69693af78c698e4..67303bc0e451bc1b180f6a4ab2914b6d4b71b62f 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry-fabric.dtsi @@ -13,4 +13,33 @@ #clock-cells = <0>; clock-frequency = <125000000>; }; + + pcie: pcie@2000000000 { + compatible = "microchip,pcie-host-1.0"; + #address-cells = <0x3>; + #interrupt-cells = <0x1>; + #size-cells = <0x2>; + device_type = "pci"; + reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; + reg-names = "cfg", "apb"; + bus-range = <0x0 0x7f>; + interrupt-parent = <&plic>; + interrupts = <119>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; + interrupt-map-mask = <0 0 0 7>; + clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>; + clock-names = "fic0", "fic1", "fic3"; + ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>; + msi-parent = <&pcie>; + msi-controller; + status = "disabled"; + pcie_intc: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; }; diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts index 82c93c8f5c17e6c39cdb532296ba1bcb1d2546ed..c87cc2d8fe29fa2174bbedca08c272c7331283b8 100644 --- a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts +++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts @@ -54,12 +54,10 @@ phy1: ethernet-phy@5 { reg = <5>; - ti,fifo-depth = <0x01>; }; phy0: ethernet-phy@4 { reg = <4>; - ti,fifo-depth = <0x01>; }; }; @@ -72,7 +70,6 @@ disable-wp; cap-sd-highspeed; cap-mmc-highspeed; - card-detect-delay = <200>; mmc-ddr-1_8v; mmc-hs200-1_8v; sd-uhs-sdr12; diff --git a/arch/riscv/boot/dts/microchip/mpfs-sev-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-sev-kit-fabric.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..8545baf4d1290b49251fbf60c8739520e631bbb8 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-sev-kit-fabric.dtsi @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2022 Microchip Technology Inc */ + +/ { + fabric_clk3: fabric-clk3 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + }; + + fabric_clk1: fabric-clk1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + }; + + pcie: pcie@2000000000 { + compatible = "microchip,pcie-host-1.0"; + #address-cells = <0x3>; + #interrupt-cells = <0x1>; + #size-cells = <0x2>; + device_type = "pci"; + reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; + reg-names = "cfg", "apb"; + bus-range = <0x0 0x7f>; + interrupt-parent = <&plic>; + interrupts = <119>; + interrupt-map = <0 0 0 1 &pcie_intc 0>, + <0 0 0 2 &pcie_intc 1>, + <0 0 0 3 &pcie_intc 2>, + <0 0 0 4 &pcie_intc 3>; + interrupt-map-mask = <0 0 0 7>; + clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>; + clock-names = "fic0", "fic1", "fic3"; + ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>; + msi-parent = <&pcie>; + msi-controller; + status = "disabled"; + pcie_intc: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts new file mode 100644 index 0000000000000000000000000000000000000000..013cb666c72da8e539a4bfbc8c5ddeb560272160 --- /dev/null +++ b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2022 Microchip Technology Inc */ + +/dts-v1/; + +#include "mpfs.dtsi" +#include "mpfs-sev-kit-fabric.dtsi" + +/* Clock frequency (in Hz) of the rtcclk */ +#define MTIMER_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "Microchip PolarFire-SoC SEV Kit"; + compatible = "microchip,mpfs-sev-kit", "microchip,mpfs"; + + aliases { + ethernet0 = &mac1; + serial0 = &mmuart0; + serial1 = &mmuart1; + serial2 = &mmuart2; + serial3 = &mmuart3; + serial4 = &mmuart4; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; + + cpus { + timebase-frequency = ; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + fabricbuf0ddrc: buffer@80000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0x80000000 0x0 0x2000000>; + }; + + fabricbuf1ddrnc: buffer@c4000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0xc4000000 0x0 0x4000000>; + }; + + fabricbuf2ddrncwcb: buffer@d4000000 { + compatible = "shared-dma-pool"; + reg = <0x0 0xd4000000 0x0 0x4000000>; + }; + }; + + ddrc_cache: memory@1000000000 { + device_type = "memory"; + reg = <0x10 0x0 0x0 0x76000000>; + }; +}; + +&i2c0 { + status = "okay"; +}; + +&gpio2 { + interrupts = <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>, + <53>, <53>, <53>, <53>; + status = "okay"; +}; + +&mac0 { + status = "okay"; + phy-mode = "sgmii"; + phy-handle = <&phy0>; + phy1: ethernet-phy@9 { + reg = <9>; + }; + phy0: ethernet-phy@8 { + reg = <8>; + }; +}; + +&mac1 { + status = "okay"; + phy-mode = "sgmii"; + phy-handle = <&phy1>; +}; + +&mbox { + status = "okay"; +}; + +&mmc { + status = "okay"; + bus-width = <4>; + disable-wp; + cap-sd-highspeed; + cap-mmc-highspeed; + mmc-ddr-1_8v; + mmc-hs200-1_8v; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-sdr104; +}; + +&mmuart1 { + status = "okay"; +}; + +&mmuart2 { + status = "okay"; +}; + +&mmuart3 { + status = "okay"; +}; + +&mmuart4 { + status = "okay"; +}; + +&refclk { + clock-frequency = <125000000>; +}; + +&rtc { + status = "okay"; +}; + +&syscontroller { + status = "okay"; +}; + +&usb { + status = "okay"; + dr_mode = "otg"; +}; diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 499c2e63ad35e160f0adaf3b6f319496622ab4b8..8f463399a5687ef81250dd4d320f0456f18f85b2 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -185,7 +185,7 @@ ranges; cctrllr: cache-controller@2010000 { - compatible = "sifive,fu540-c000-ccache", "cache"; + compatible = "microchip,mpfs-ccache", "sifive,fu540-c000-ccache", "cache"; reg = <0x0 0x2010000 0x0 0x1000>; cache-block-size = <64>; cache-level = <2>; @@ -193,7 +193,7 @@ cache-size = <2097152>; cache-unified; interrupt-parent = <&plic>; - interrupts = <1>, <2>, <3>; + interrupts = <1>, <3>, <4>, <2>; }; clint: clint@2000000 { @@ -330,7 +330,7 @@ }; qspi: spi@21000000 { - compatible = "microchip,mpfs-qspi"; + compatible = "microchip,mpfs-qspi", "microchip,coreqspi-rtl-v2"; #address-cells = <1>; #size-cells = <0>; reg = <0x0 0x21000000 0x0 0x1000>; @@ -464,36 +464,6 @@ status = "disabled"; }; - pcie: pcie@2000000000 { - compatible = "microchip,pcie-host-1.0"; - #address-cells = <0x3>; - #interrupt-cells = <0x1>; - #size-cells = <0x2>; - device_type = "pci"; - reg = <0x20 0x0 0x0 0x8000000>, <0x0 0x43000000 0x0 0x10000>; - reg-names = "cfg", "apb"; - bus-range = <0x0 0x7f>; - interrupt-parent = <&plic>; - interrupts = <119>; - interrupt-map = <0 0 0 1 &pcie_intc 0>, - <0 0 0 2 &pcie_intc 1>, - <0 0 0 3 &pcie_intc 2>, - <0 0 0 4 &pcie_intc 3>; - interrupt-map-mask = <0 0 0 7>; - clocks = <&fabric_clk1>, <&fabric_clk1>, <&fabric_clk3>; - clock-names = "fic0", "fic1", "fic3"; - ranges = <0x3000000 0x0 0x8000000 0x20 0x8000000 0x0 0x80000000>; - msi-parent = <&pcie>; - msi-controller; - microchip,axi-m-atr0 = <0x10 0x0>; - status = "disabled"; - pcie_intc: legacy-interrupt-controller { - #address-cells = <0>; - #interrupt-cells = <1>; - interrupt-controller; - }; - }; - mbox: mailbox@37020000 { compatible = "microchip,mpfs-mailbox"; reg = <0x0 0x37020000 0x0 0x1000>, <0x0 0x2000318C 0x0 0x40>; diff --git a/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts b/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts index c9af67f7a0d201753df77c182dff2aae77fb8539..f7a23011051282097ef221cad8eb78537682acc2 100644 --- a/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts +++ b/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts @@ -8,7 +8,7 @@ #include "jh7100.dtsi" #include #include -#include +#include / { model = "BeagleV Starlight Beta"; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index aed332a9d4ea21099396e3c5c7b6cf4df59b486c..05fd5fcf24f9bf0e5d35e0a61083e87080e5e416 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -166,6 +166,9 @@ CONFIG_BTRFS_FS=m CONFIG_BTRFS_FS_POSIX_ACL=y CONFIG_AUTOFS4_FS=y CONFIG_OVERLAY_FS=m +CONFIG_ISO9660_FS=y +CONFIG_JOLIET=y +CONFIG_ZISOFS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y CONFIG_TMPFS=y diff --git a/arch/riscv/errata/thead/errata.c b/arch/riscv/errata/thead/errata.c index 202c83f677b2ede1bc0992fffb91442930ffa74b..21546937db39bf5fb968af60a7ee5b096242925b 100644 --- a/arch/riscv/errata/thead/errata.c +++ b/arch/riscv/errata/thead/errata.c @@ -17,6 +17,9 @@ static bool errata_probe_pbmt(unsigned int stage, unsigned long arch_id, unsigned long impid) { + if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PBMT)) + return false; + if (arch_id != 0 || impid != 0) return false; @@ -30,18 +33,18 @@ static bool errata_probe_pbmt(unsigned int stage, static bool errata_probe_cmo(unsigned int stage, unsigned long arch_id, unsigned long impid) { -#ifdef CONFIG_ERRATA_THEAD_CMO + if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO)) + return false; + if (arch_id != 0 || impid != 0) return false; if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) return false; + riscv_cbom_block_size = L1_CACHE_BYTES; riscv_noncoherent_supported(); return true; -#else - return false; -#endif } static u32 thead_errata_probe(unsigned int stage, @@ -50,10 +53,10 @@ static u32 thead_errata_probe(unsigned int stage, u32 cpu_req_errata = 0; if (errata_probe_pbmt(stage, archid, impid)) - cpu_req_errata |= (1U << ERRATA_THEAD_PBMT); + cpu_req_errata |= BIT(ERRATA_THEAD_PBMT); if (errata_probe_cmo(stage, archid, impid)) - cpu_req_errata |= (1U << ERRATA_THEAD_CMO); + cpu_req_errata |= BIT(ERRATA_THEAD_CMO); return cpu_req_errata; } diff --git a/arch/riscv/include/asm/cacheflush.h b/arch/riscv/include/asm/cacheflush.h index a60acaecfedab6ad09e578b1f1a98b04a08541c7..8a5c246b0a216c6d9da8e835f84fc640d2dbd2fb 100644 --- a/arch/riscv/include/asm/cacheflush.h +++ b/arch/riscv/include/asm/cacheflush.h @@ -42,6 +42,11 @@ void flush_icache_mm(struct mm_struct *mm, bool local); #endif /* CONFIG_SMP */ +/* + * The T-Head CMO errata internally probe the CBOM block size, but otherwise + * don't depend on Zicbom. + */ +extern unsigned int riscv_cbom_block_size; #ifdef CONFIG_RISCV_ISA_ZICBOM void riscv_init_cbom_blocksize(void); #else @@ -50,6 +55,8 @@ static inline void riscv_init_cbom_blocksize(void) { } #ifdef CONFIG_RISCV_DMA_NONCOHERENT void riscv_noncoherent_supported(void); +#else +static inline void riscv_noncoherent_supported(void) {} #endif /* diff --git a/arch/riscv/include/asm/elf.h b/arch/riscv/include/asm/elf.h index 14fc7342490bfc6151e500e34cc774cbfeb49005..e7acffdf21d2663b659f9dc212d14780277fa7f5 100644 --- a/arch/riscv/include/asm/elf.h +++ b/arch/riscv/include/asm/elf.h @@ -99,6 +99,10 @@ do { \ get_cache_size(2, CACHE_TYPE_UNIFIED)); \ NEW_AUX_ENT(AT_L2_CACHEGEOMETRY, \ get_cache_geometry(2, CACHE_TYPE_UNIFIED)); \ + NEW_AUX_ENT(AT_L3_CACHESIZE, \ + get_cache_size(3, CACHE_TYPE_UNIFIED)); \ + NEW_AUX_ENT(AT_L3_CACHEGEOMETRY, \ + get_cache_geometry(3, CACHE_TYPE_UNIFIED)); \ } while (0) #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; diff --git a/arch/riscv/include/asm/gpr-num.h b/arch/riscv/include/asm/gpr-num.h index dfee2829fc7cb0ad7dc2ca27f4414262e0075c84..efeb5edf8a3af15052c68fc8a95a3091a2d3af46 100644 --- a/arch/riscv/include/asm/gpr-num.h +++ b/arch/riscv/include/asm/gpr-num.h @@ -3,6 +3,11 @@ #define __ASM_GPR_NUM_H #ifdef __ASSEMBLY__ + + .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .equ .L__gpr_num_x\num, \num + .endr + .equ .L__gpr_num_zero, 0 .equ .L__gpr_num_ra, 1 .equ .L__gpr_num_sp, 2 @@ -39,6 +44,9 @@ #else /* __ASSEMBLY__ */ #define __DEFINE_ASM_GPR_NUMS \ +" .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31\n" \ +" .equ .L__gpr_num_x\\num, \\num\n" \ +" .endr\n" \ " .equ .L__gpr_num_zero, 0\n" \ " .equ .L__gpr_num_ra, 1\n" \ " .equ .L__gpr_num_sp, 2\n" \ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index 6f59ec64175efd2781635c01e34bb26f80d689fb..b2252529007301767f4dac92f4bc93d128b020b5 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -58,6 +58,7 @@ enum riscv_isa_ext_id { RISCV_ISA_EXT_ZICBOM, RISCV_ISA_EXT_ZIHINTPAUSE, RISCV_ISA_EXT_SSTC, + RISCV_ISA_EXT_SVINVAL, RISCV_ISA_EXT_ID_MAX = RISCV_ISA_EXT_MAX, }; @@ -69,6 +70,7 @@ enum riscv_isa_ext_id { enum riscv_isa_ext_key { RISCV_ISA_EXT_KEY_FPU, /* For 'F' and 'D' */ RISCV_ISA_EXT_KEY_ZIHINTPAUSE, + RISCV_ISA_EXT_KEY_SVINVAL, RISCV_ISA_EXT_KEY_MAX, }; @@ -90,6 +92,8 @@ static __always_inline int riscv_isa_ext2key(int num) return RISCV_ISA_EXT_KEY_FPU; case RISCV_ISA_EXT_ZIHINTPAUSE: return RISCV_ISA_EXT_KEY_ZIHINTPAUSE; + case RISCV_ISA_EXT_SVINVAL: + return RISCV_ISA_EXT_KEY_SVINVAL; default: return -EINVAL; } diff --git a/arch/riscv/include/asm/insn-def.h b/arch/riscv/include/asm/insn-def.h new file mode 100644 index 0000000000000000000000000000000000000000..16044affa57cc7e6d30e8d068ebffa6bf87e7729 --- /dev/null +++ b/arch/riscv/include/asm/insn-def.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_INSN_DEF_H +#define __ASM_INSN_DEF_H + +#include + +#define INSN_R_FUNC7_SHIFT 25 +#define INSN_R_RS2_SHIFT 20 +#define INSN_R_RS1_SHIFT 15 +#define INSN_R_FUNC3_SHIFT 12 +#define INSN_R_RD_SHIFT 7 +#define INSN_R_OPCODE_SHIFT 0 + +#ifdef __ASSEMBLY__ + +#ifdef CONFIG_AS_HAS_INSN + + .macro insn_r, opcode, func3, func7, rd, rs1, rs2 + .insn r \opcode, \func3, \func7, \rd, \rs1, \rs2 + .endm + +#else + +#include + + .macro insn_r, opcode, func3, func7, rd, rs1, rs2 + .4byte ((\opcode << INSN_R_OPCODE_SHIFT) | \ + (\func3 << INSN_R_FUNC3_SHIFT) | \ + (\func7 << INSN_R_FUNC7_SHIFT) | \ + (.L__gpr_num_\rd << INSN_R_RD_SHIFT) | \ + (.L__gpr_num_\rs1 << INSN_R_RS1_SHIFT) | \ + (.L__gpr_num_\rs2 << INSN_R_RS2_SHIFT)) + .endm + +#endif + +#define __INSN_R(...) insn_r __VA_ARGS__ + +#else /* ! __ASSEMBLY__ */ + +#ifdef CONFIG_AS_HAS_INSN + +#define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \ + ".insn r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" + +#else + +#include +#include + +#define DEFINE_INSN_R \ + __DEFINE_ASM_GPR_NUMS \ +" .macro insn_r, opcode, func3, func7, rd, rs1, rs2\n" \ +" .4byte ((\\opcode << " __stringify(INSN_R_OPCODE_SHIFT) ") |" \ +" (\\func3 << " __stringify(INSN_R_FUNC3_SHIFT) ") |" \ +" (\\func7 << " __stringify(INSN_R_FUNC7_SHIFT) ") |" \ +" (.L__gpr_num_\\rd << " __stringify(INSN_R_RD_SHIFT) ") |" \ +" (.L__gpr_num_\\rs1 << " __stringify(INSN_R_RS1_SHIFT) ") |" \ +" (.L__gpr_num_\\rs2 << " __stringify(INSN_R_RS2_SHIFT) "))\n" \ +" .endm\n" + +#define UNDEFINE_INSN_R \ +" .purgem insn_r\n" + +#define __INSN_R(opcode, func3, func7, rd, rs1, rs2) \ + DEFINE_INSN_R \ + "insn_r " opcode ", " func3 ", " func7 ", " rd ", " rs1 ", " rs2 "\n" \ + UNDEFINE_INSN_R + +#endif + +#endif /* ! __ASSEMBLY__ */ + +#define INSN_R(opcode, func3, func7, rd, rs1, rs2) \ + __INSN_R(RV_##opcode, RV_##func3, RV_##func7, \ + RV_##rd, RV_##rs1, RV_##rs2) + +#define RV_OPCODE(v) __ASM_STR(v) +#define RV_FUNC3(v) __ASM_STR(v) +#define RV_FUNC7(v) __ASM_STR(v) +#define RV_RD(v) __ASM_STR(v) +#define RV_RS1(v) __ASM_STR(v) +#define RV_RS2(v) __ASM_STR(v) +#define __RV_REG(v) __ASM_STR(x ## v) +#define RV___RD(v) __RV_REG(v) +#define RV___RS1(v) __RV_REG(v) +#define RV___RS2(v) __RV_REG(v) + +#define RV_OPCODE_SYSTEM RV_OPCODE(115) + +#define HFENCE_VVMA(vaddr, asid) \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(17), \ + __RD(0), RS1(vaddr), RS2(asid)) + +#define HFENCE_GVMA(gaddr, vmid) \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(49), \ + __RD(0), RS1(gaddr), RS2(vmid)) + +#define HLVX_HU(dest, addr) \ + INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(50), \ + RD(dest), RS1(addr), __RS2(3)) + +#define HLV_W(dest, addr) \ + INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(52), \ + RD(dest), RS1(addr), __RS2(0)) + +#ifdef CONFIG_64BIT +#define HLV_D(dest, addr) \ + INSN_R(OPCODE_SYSTEM, FUNC3(4), FUNC7(54), \ + RD(dest), RS1(addr), __RS2(0)) +#else +#define HLV_D(dest, addr) \ + __ASM_STR(.error "hlv.d requires 64-bit support") +#endif + +#define SINVAL_VMA(vaddr, asid) \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(11), \ + __RD(0), RS1(vaddr), RS2(asid)) + +#define SFENCE_W_INVAL() \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(12), \ + __RD(0), __RS1(0), __RS2(0)) + +#define SFENCE_INVAL_IR() \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(12), \ + __RD(0), __RS1(0), __RS2(1)) + +#define HINVAL_VVMA(vaddr, asid) \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(19), \ + __RD(0), RS1(vaddr), RS2(asid)) + +#define HINVAL_GVMA(gaddr, vmid) \ + INSN_R(OPCODE_SYSTEM, FUNC3(0), FUNC7(51), \ + __RD(0), RS1(gaddr), RS2(vmid)) + +#endif /* __ASM_INSN_DEF_H */ diff --git a/arch/riscv/include/asm/io.h b/arch/riscv/include/asm/io.h index 69605a47427065a688cd18ed3a6b13210cec800d..92080a227937283b2d094c2d204b979928d9e16d 100644 --- a/arch/riscv/include/asm/io.h +++ b/arch/riscv/include/asm/io.h @@ -101,9 +101,9 @@ __io_reads_ins(reads, u32, l, __io_br(), __io_ar(addr)) __io_reads_ins(ins, u8, b, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u16, w, __io_pbr(), __io_par(addr)) __io_reads_ins(ins, u32, l, __io_pbr(), __io_par(addr)) -#define insb(addr, buffer, count) __insb((void __iomem *)(long)addr, buffer, count) -#define insw(addr, buffer, count) __insw((void __iomem *)(long)addr, buffer, count) -#define insl(addr, buffer, count) __insl((void __iomem *)(long)addr, buffer, count) +#define insb(addr, buffer, count) __insb(PCI_IOBASE + (addr), buffer, count) +#define insw(addr, buffer, count) __insw(PCI_IOBASE + (addr), buffer, count) +#define insl(addr, buffer, count) __insl(PCI_IOBASE + (addr), buffer, count) __io_writes_outs(writes, u8, b, __io_bw(), __io_aw()) __io_writes_outs(writes, u16, w, __io_bw(), __io_aw()) @@ -115,22 +115,22 @@ __io_writes_outs(writes, u32, l, __io_bw(), __io_aw()) __io_writes_outs(outs, u8, b, __io_pbw(), __io_paw()) __io_writes_outs(outs, u16, w, __io_pbw(), __io_paw()) __io_writes_outs(outs, u32, l, __io_pbw(), __io_paw()) -#define outsb(addr, buffer, count) __outsb((void __iomem *)(long)addr, buffer, count) -#define outsw(addr, buffer, count) __outsw((void __iomem *)(long)addr, buffer, count) -#define outsl(addr, buffer, count) __outsl((void __iomem *)(long)addr, buffer, count) +#define outsb(addr, buffer, count) __outsb(PCI_IOBASE + (addr), buffer, count) +#define outsw(addr, buffer, count) __outsw(PCI_IOBASE + (addr), buffer, count) +#define outsl(addr, buffer, count) __outsl(PCI_IOBASE + (addr), buffer, count) #ifdef CONFIG_64BIT __io_reads_ins(reads, u64, q, __io_br(), __io_ar(addr)) #define readsq(addr, buffer, count) __readsq(addr, buffer, count) __io_reads_ins(ins, u64, q, __io_pbr(), __io_par(addr)) -#define insq(addr, buffer, count) __insq((void __iomem *)addr, buffer, count) +#define insq(addr, buffer, count) __insq(PCI_IOBASE + (addr), buffer, count) __io_writes_outs(writes, u64, q, __io_bw(), __io_aw()) #define writesq(addr, buffer, count) __writesq(addr, buffer, count) __io_writes_outs(outs, u64, q, __io_pbr(), __io_paw()) -#define outsq(addr, buffer, count) __outsq((void __iomem *)addr, buffer, count) +#define outsq(addr, buffer, count) __outsq(PCI_IOBASE + (addr), buffer, count) #endif #include diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 60c517e4d57643708b64469ac78ffdb64978e613..dbbf43d526234822d396e26d6e0c18a2ecca9699 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -67,6 +67,7 @@ struct kvm_vcpu_stat { u64 mmio_exit_kernel; u64 csr_exit_user; u64 csr_exit_kernel; + u64 signal_exits; u64 exits; }; diff --git a/arch/riscv/include/asm/kvm_vcpu_sbi.h b/arch/riscv/include/asm/kvm_vcpu_sbi.h index 83d6d4d2b1dffe81aae58add2f7403eeade32f6d..d4e3e600beefb2d502d4a0fb4ae6b9ba357913ea 100644 --- a/arch/riscv/include/asm/kvm_vcpu_sbi.h +++ b/arch/riscv/include/asm/kvm_vcpu_sbi.h @@ -11,8 +11,8 @@ #define KVM_SBI_IMPID 3 -#define KVM_SBI_VERSION_MAJOR 0 -#define KVM_SBI_VERSION_MINOR 3 +#define KVM_SBI_VERSION_MAJOR 1 +#define KVM_SBI_VERSION_MINOR 0 struct kvm_vcpu_sbi_extension { unsigned long extid_start; @@ -33,4 +33,16 @@ void kvm_riscv_vcpu_sbi_system_reset(struct kvm_vcpu *vcpu, u32 type, u64 flags); const struct kvm_vcpu_sbi_extension *kvm_vcpu_sbi_find_ext(unsigned long extid); +#ifdef CONFIG_RISCV_SBI_V01 +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; +#endif +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_base; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_time; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_ipi; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_rfence; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_srst; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; +extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; + #endif /* __RISCV_KVM_VCPU_SBI_H__ */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index cedcf8ea3c7664cc144601dc67ba9c584624577f..0099dc1161683ddd1e3c45460309e331b7e6b0a7 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -16,7 +16,6 @@ typedef struct { atomic_long_t id; #endif void *vdso; - void *vdso_info; #ifdef CONFIG_SMP /* A local icache flush is needed before user execution can resume. */ cpumask_t icache_stale_mask; diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 19eedd4af4cdee092b54ef60e5c4d7270bba44a0..94a0590c69710b54e2e1f7258bd6692858ce5f90 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -65,11 +65,6 @@ static inline void arch_thread_struct_whitelist(unsigned long *offset, extern void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp); -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *dead_task) -{ -} - extern unsigned long __get_wchan(struct task_struct *p); diff --git a/arch/riscv/include/asm/signal.h b/arch/riscv/include/asm/signal.h new file mode 100644 index 0000000000000000000000000000000000000000..532c29ef037698dc2e04c4faf4ad3574d2d02429 --- /dev/null +++ b/arch/riscv/include/asm/signal.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_SIGNAL_H +#define __ASM_SIGNAL_H + +#include +#include + +asmlinkage __visible +void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags); + +#endif diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 78933ac04995b2d22f545bd8d7853743440c5e49..67322f878e0d7828178a51fb0cff047270d59a2a 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -42,6 +42,8 @@ #ifndef __ASSEMBLY__ +extern long shadow_stack[SHADOW_OVERFLOW_STACK_SIZE / sizeof(long)]; + #include #include diff --git a/arch/riscv/include/uapi/asm/auxvec.h b/arch/riscv/include/uapi/asm/auxvec.h index 32c73ba1d531391bb6621d0b1dc750f430609f2a..fb187a33ce5897e7b671d7253324d5eadb00c5f3 100644 --- a/arch/riscv/include/uapi/asm/auxvec.h +++ b/arch/riscv/include/uapi/asm/auxvec.h @@ -30,8 +30,10 @@ #define AT_L1D_CACHEGEOMETRY 43 #define AT_L2_CACHESIZE 44 #define AT_L2_CACHEGEOMETRY 45 +#define AT_L3_CACHESIZE 46 +#define AT_L3_CACHEGEOMETRY 47 /* entries in ARCH_DLINFO */ -#define AT_VECTOR_SIZE_ARCH 7 +#define AT_VECTOR_SIZE_ARCH 9 #endif /* _UAPI_ASM_RISCV_AUXVEC_H */ diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index 7351417afd62e32c69eaa6ec9bc57675815ad322..8985ff234c01cd03ce660dfaad7cf5abedfd2a78 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -48,6 +48,7 @@ struct kvm_sregs { /* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ struct kvm_riscv_config { unsigned long isa; + unsigned long zicbom_block_size; }; /* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */ @@ -98,6 +99,9 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_M, KVM_RISCV_ISA_EXT_SVPBMT, KVM_RISCV_ISA_EXT_SSTC, + KVM_RISCV_ISA_EXT_SVINVAL, + KVM_RISCV_ISA_EXT_ZIHINTPAUSE, + KVM_RISCV_ISA_EXT_ZICBOM, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 33bb60a354cd20633fd40cd8b5faa01c407d3814..db6e4b1294ba3761bed0a3e1a249dd2ac7e2d25e 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -28,9 +28,9 @@ KASAN_SANITIZE_cpufeature.o := n endif endif -extra-y += head.o extra-y += vmlinux.lds +obj-y += head.o obj-y += soc.o obj-$(CONFIG_RISCV_ALTERNATIVE) += alternative.o obj-y += cpu.o diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 0be8a2403212d4a2b9a8a30af78437bc5b597b5f..fa427bdcf773d20cba656fdb0477ff6819ef20ab 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -3,10 +3,13 @@ * Copyright (C) 2012 Regents of the University of California */ +#include #include #include #include +#include #include +#include #include #include @@ -68,6 +71,50 @@ int riscv_of_parent_hartid(struct device_node *node, unsigned long *hartid) } #ifdef CONFIG_PROC_FS + +struct riscv_cpuinfo { + unsigned long mvendorid; + unsigned long marchid; + unsigned long mimpid; +}; +static DEFINE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); + +static int riscv_cpuinfo_starting(unsigned int cpu) +{ + struct riscv_cpuinfo *ci = this_cpu_ptr(&riscv_cpuinfo); + +#if IS_ENABLED(CONFIG_RISCV_SBI) + ci->mvendorid = sbi_spec_is_0_1() ? 0 : sbi_get_mvendorid(); + ci->marchid = sbi_spec_is_0_1() ? 0 : sbi_get_marchid(); + ci->mimpid = sbi_spec_is_0_1() ? 0 : sbi_get_mimpid(); +#elif IS_ENABLED(CONFIG_RISCV_M_MODE) + ci->mvendorid = csr_read(CSR_MVENDORID); + ci->marchid = csr_read(CSR_MARCHID); + ci->mimpid = csr_read(CSR_MIMPID); +#else + ci->mvendorid = 0; + ci->marchid = 0; + ci->mimpid = 0; +#endif + + return 0; +} + +static int __init riscv_cpuinfo_init(void) +{ + int ret; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "riscv/cpuinfo:starting", + riscv_cpuinfo_starting, NULL); + if (ret < 0) { + pr_err("cpuinfo: failed to register hotplug callbacks.\n"); + return ret; + } + + return 0; +} +device_initcall(riscv_cpuinfo_init); + #define __RISCV_ISA_EXT_DATA(UPROP, EXTID) \ { \ .uprop = #UPROP, \ @@ -92,10 +139,11 @@ int riscv_of_parent_hartid(struct device_node *node, unsigned long *hartid) */ static struct riscv_isa_ext_data isa_ext_arr[] = { __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF), + __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), + __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL), __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE), - __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC), __RISCV_ISA_EXT_DATA("", RISCV_ISA_EXT_MAX), }; @@ -185,6 +233,7 @@ static int c_show(struct seq_file *m, void *v) { unsigned long cpu_id = (unsigned long)v - 1; struct device_node *node = of_get_cpu_node(cpu_id, NULL); + struct riscv_cpuinfo *ci = per_cpu_ptr(&riscv_cpuinfo, cpu_id); const char *compat, *isa; seq_printf(m, "processor\t: %lu\n", cpu_id); @@ -195,6 +244,9 @@ static int c_show(struct seq_file *m, void *v) if (!of_property_read_string(node, "compatible", &compat) && strcmp(compat, "riscv")) seq_printf(m, "uarch\t\t: %s\n", compat); + seq_printf(m, "mvendorid\t: 0x%lx\n", ci->mvendorid); + seq_printf(m, "marchid\t\t: 0x%lx\n", ci->marchid); + seq_printf(m, "mimpid\t\t: 0x%lx\n", ci->mimpid); seq_puts(m, "\n"); of_node_put(node); diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 3b5583db9d80ee4a175a5290ccb77ceea33c97c4..694267d1fe8149a6a29aab095e441aafaa07b0ec 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -204,6 +204,7 @@ void __init riscv_fill_hwcap(void) SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM); SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE); SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC); + SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL); } #undef SET_ISA_EXT_MAP } @@ -253,35 +254,28 @@ void __init riscv_fill_hwcap(void) #ifdef CONFIG_RISCV_ALTERNATIVE static bool __init_or_module cpufeature_probe_svpbmt(unsigned int stage) { -#ifdef CONFIG_RISCV_ISA_SVPBMT - switch (stage) { - case RISCV_ALTERNATIVES_EARLY_BOOT: + if (!IS_ENABLED(CONFIG_RISCV_ISA_SVPBMT)) + return false; + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) return false; - default: - return riscv_isa_extension_available(NULL, SVPBMT); - } -#endif - return false; + return riscv_isa_extension_available(NULL, SVPBMT); } static bool __init_or_module cpufeature_probe_zicbom(unsigned int stage) { -#ifdef CONFIG_RISCV_ISA_ZICBOM - switch (stage) { - case RISCV_ALTERNATIVES_EARLY_BOOT: + if (!IS_ENABLED(CONFIG_RISCV_ISA_ZICBOM)) + return false; + + if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) + return false; + + if (!riscv_isa_extension_available(NULL, ZICBOM)) return false; - default: - if (riscv_isa_extension_available(NULL, ZICBOM)) { - riscv_noncoherent_supported(); - return true; - } else { - return false; - } - } -#endif - return false; + riscv_noncoherent_supported(); + return true; } /* @@ -296,10 +290,10 @@ static u32 __init_or_module cpufeature_probe(unsigned int stage) u32 cpu_req_feature = 0; if (cpufeature_probe_svpbmt(stage)) - cpu_req_feature |= (1U << CPUFEATURE_SVPBMT); + cpu_req_feature |= BIT(CPUFEATURE_SVPBMT); if (cpufeature_probe_zicbom(stage)) - cpu_req_feature |= (1U << CPUFEATURE_ZICBOM); + cpu_req_feature |= BIT(CPUFEATURE_ZICBOM); return cpu_req_feature; } diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 71a76a62325778c4e030b83f24e80d18c49f62bb..d6e5f739905e740c783249424e0f51a37cb7c36a 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -25,21 +25,12 @@ */ __efistub_memcmp = memcmp; __efistub_memchr = memchr; -__efistub_memcpy = memcpy; -__efistub_memmove = memmove; -__efistub_memset = memset; __efistub_strlen = strlen; __efistub_strnlen = strnlen; __efistub_strcmp = strcmp; __efistub_strncmp = strncmp; __efistub_strrchr = strrchr; -#ifdef CONFIG_KASAN -__efistub___memcpy = memcpy; -__efistub___memmove = memmove; -__efistub___memset = memset; -#endif - __efistub__start = _start; __efistub__start_kernel = _start_kernel; __efistub__end = _end; diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index ceb9ebab6558cba7c64dae26ea911abec09df6b1..b0c63e8e867ef6979231d749ad98d2381d827a67 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -105,7 +105,7 @@ static int __init compat_mode_detect(void) csr_write(CSR_STATUS, tmp); pr_info("riscv: ELF compat mode %s", - compat_mode_supported ? "supported" : "failed"); + compat_mode_supported ? "supported" : "unsupported"); return 0; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 95ef6e2bf45c02faffaa4228ae50393d9146c6fd..ad76bb59b0590d8e67fa2825d69de741de7f8f31 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -252,10 +252,10 @@ static void __init parse_dtb(void) pr_info("Machine model: %s\n", name); dump_stack_set_arch_desc("%s (DT)", name); } - return; + } else { + pr_err("No DTB passed to the kernel\n"); } - pr_err("No DTB passed to the kernel\n"); #ifdef CONFIG_CMDLINE_FORCE strscpy(boot_command_line, CONFIG_CMDLINE, COMMAND_LINE_SIZE); pr_info("Forcing kernel command line to: %s\n", boot_command_line); @@ -296,8 +296,8 @@ void __init setup_arch(char **cmdline_p) setup_smp(); #endif - riscv_fill_hwcap(); riscv_init_cbom_blocksize(); + riscv_fill_hwcap(); apply_boot_alternatives(); } diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 38b05ca6fe66983426e7bfba231c0775ae8c81d4..5c591123c440910594f0051be1bb6387d72d05d6 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -123,6 +124,8 @@ SYSCALL_DEFINE0(rt_sigreturn) if (restore_altstack(&frame->uc.uc_stack)) goto badframe; + regs->cause = -1UL; + return regs->a0; badframe: diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index a752c7b416838a35873363b50c5dd24dfd1ade42..3373df413c8820682ea4770f622b6fbbc0003865 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -49,6 +49,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) unsigned int curr_cpuid; curr_cpuid = smp_processor_id(); + store_cpu_topology(curr_cpuid); numa_store_cpu_info(curr_cpuid); numa_add_cpu(curr_cpuid); @@ -162,9 +163,9 @@ asmlinkage __visible void smp_callin(void) mmgrab(mm); current->active_mm = mm; + store_cpu_topology(curr_cpuid); notify_cpu_starting(curr_cpuid); numa_add_cpu(curr_cpuid); - update_siblings_masks(curr_cpuid); set_cpu_online(curr_cpuid, 1); /* diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c index 571556bb9261a9686f57f4336dbbe8447031e04b..5d3f2fbeb33c7b72eeca2b88f627a404039aefc8 100644 --- a/arch/riscv/kernel/sys_riscv.c +++ b/arch/riscv/kernel/sys_riscv.c @@ -18,9 +18,6 @@ static long riscv_sys_mmap(unsigned long addr, unsigned long len, if (unlikely(offset & (~PAGE_MASK >> page_shift_offset))) return -EINVAL; - if (unlikely((prot & PROT_WRITE) && !(prot & PROT_READ))) - return -EINVAL; - return ksys_mmap_pgoff(addr, len, prot, flags, fd, offset >> (PAGE_SHIFT - page_shift_offset)); } diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 39d0f8bba4b40e3e22ac8125c628f50d919bd346..f3e96d60a2ff384a4ff7b342ba8daa45e22aaff1 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -20,9 +20,10 @@ #include #include +#include #include #include -#include +#include int show_unhandled_signals = 1; @@ -32,6 +33,7 @@ void die(struct pt_regs *regs, const char *str) { static int die_counter; int ret; + long cause; oops_enter(); @@ -41,11 +43,13 @@ void die(struct pt_regs *regs, const char *str) pr_emerg("%s [#%d]\n", str, ++die_counter); print_modules(); - show_regs(regs); + if (regs) + show_regs(regs); - ret = notify_die(DIE_OOPS, str, regs, 0, regs->cause, SIGSEGV); + cause = regs ? regs->cause : -1; + ret = notify_die(DIE_OOPS, str, regs, 0, cause, SIGSEGV); - if (regs && kexec_should_crash(current)) + if (kexec_should_crash(current)) crash_kexec(regs); bust_spinlocks(0); diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 69b05b6c181b6d252c6d95d11605b76154619c0c..123d05255fcfa63b4ff418e51383d54391e5d7f8 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -60,6 +60,11 @@ struct __vdso_info { struct vm_special_mapping *cm; }; +static struct __vdso_info vdso_info; +#ifdef CONFIG_COMPAT +static struct __vdso_info compat_vdso_info; +#endif + static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma) { @@ -114,15 +119,19 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; struct vm_area_struct *vma; - struct __vdso_info *vdso_info = mm->context.vdso_info; + VMA_ITERATOR(vmi, mm, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long size = vma->vm_end - vma->vm_start; - if (vma_is_special_mapping(vma, vdso_info->dm)) + if (vma_is_special_mapping(vma, vdso_info.dm)) zap_page_range(vma, vma->vm_start, size); +#ifdef CONFIG_COMPAT + if (vma_is_special_mapping(vma, compat_vdso_info.dm)) + zap_page_range(vma, vma->vm_start, size); +#endif } mmap_read_unlock(mm); @@ -264,7 +273,6 @@ static int __setup_additional_pages(struct mm_struct *mm, vdso_base += VVAR_SIZE; mm->context.vdso = (void *)vdso_base; - mm->context.vdso_info = (void *)vdso_info; ret = _install_special_mapping(mm, vdso_base, vdso_text_len, diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index f5a342fa1b1d26fc5f4148ef2cb21c798085b9b2..f36a737d5f96d253d42075fdcb9082d500e3d08b 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -24,6 +24,7 @@ config KVM select PREEMPT_NOTIFIERS select KVM_MMIO select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select KVM_XFER_TO_GUEST_WORK select HAVE_KVM_VCPU_ASYNC_IOCTL select HAVE_KVM_EVENTFD select SRCU diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 1549205fe5feb22cb1bb3807cec74f499c542d9a..df2d8716851f20cd226419dca6a35c2bf9595aef 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -122,7 +122,7 @@ void kvm_arch_exit(void) { } -static int riscv_kvm_init(void) +static int __init riscv_kvm_init(void) { return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); } diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 1a76d0b1907d54b43dca41f55878aba4da199ba7..309d79b3e5cd58bc4235a6604bba6c77bb9aa469 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -12,22 +12,11 @@ #include #include #include +#include +#include -/* - * Instruction encoding of hfence.gvma is: - * HFENCE.GVMA rs1, rs2 - * HFENCE.GVMA zero, rs2 - * HFENCE.GVMA rs1 - * HFENCE.GVMA - * - * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2 - * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2 - * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1 - * rs1==zero and rs2==zero ==> HFENCE.GVMA - * - * Instruction encoding of HFENCE.GVMA is: - * 0110001 rs2(5) rs1(5) 000 00000 1110011 - */ +#define has_svinval() \ + static_branch_unlikely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_SVINVAL]) void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, @@ -40,32 +29,22 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, return; } - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { - /* - * rs1 = a0 (GPA >> 2) - * rs2 = a1 (VMID) - * HFENCE.GVMA a0, a1 - * 0110001 01011 01010 000 00000 1110011 - */ - asm volatile ("srli a0, %0, 2\n" - "add a1, %1, zero\n" - ".word 0x62b50073\n" - :: "r" (pos), "r" (vmid) - : "a0", "a1", "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile (HINVAL_GVMA(%0, %1) + : : "r" (pos >> 2), "r" (vmid) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile (HFENCE_GVMA(%0, %1) + : : "r" (pos >> 2), "r" (vmid) : "memory"); } } void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) { - /* - * rs1 = zero - * rs2 = a0 (VMID) - * HFENCE.GVMA zero, a0 - * 0110001 01010 00000 000 00000 1110011 - */ - asm volatile ("add a0, %0, zero\n" - ".word 0x62a00073\n" - :: "r" (vmid) : "a0", "memory"); + asm volatile(HFENCE_GVMA(zero, %0) : : "r" (vmid) : "memory"); } void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, @@ -78,46 +57,24 @@ void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, return; } - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { - /* - * rs1 = a0 (GPA >> 2) - * rs2 = zero - * HFENCE.GVMA a0 - * 0110001 00000 01010 000 00000 1110011 - */ - asm volatile ("srli a0, %0, 2\n" - ".word 0x62050073\n" - :: "r" (pos) : "a0", "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile(HINVAL_GVMA(%0, zero) + : : "r" (pos >> 2) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile(HFENCE_GVMA(%0, zero) + : : "r" (pos >> 2) : "memory"); } } void kvm_riscv_local_hfence_gvma_all(void) { - /* - * rs1 = zero - * rs2 = zero - * HFENCE.GVMA - * 0110001 00000 00000 000 00000 1110011 - */ - asm volatile (".word 0x62000073" ::: "memory"); + asm volatile(HFENCE_GVMA(zero, zero) : : : "memory"); } -/* - * Instruction encoding of hfence.gvma is: - * HFENCE.VVMA rs1, rs2 - * HFENCE.VVMA zero, rs2 - * HFENCE.VVMA rs1 - * HFENCE.VVMA - * - * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2 - * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2 - * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1 - * rs1==zero and rs2==zero ==> HFENCE.VVMA - * - * Instruction encoding of HFENCE.VVMA is: - * 0010001 rs2(5) rs1(5) 000 00000 1110011 - */ - void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, unsigned long asid, unsigned long gva, @@ -133,18 +90,16 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { - /* - * rs1 = a0 (GVA) - * rs2 = a1 (ASID) - * HFENCE.VVMA a0, a1 - * 0010001 01011 01010 000 00000 1110011 - */ - asm volatile ("add a0, %0, zero\n" - "add a1, %1, zero\n" - ".word 0x22b50073\n" - :: "r" (pos), "r" (asid) - : "a0", "a1", "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HINVAL_VVMA(%0, %1) + : : "r" (pos), "r" (asid) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HFENCE_VVMA(%0, %1) + : : "r" (pos), "r" (asid) : "memory"); } csr_write(CSR_HGATP, hgatp); @@ -157,15 +112,7 @@ void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - /* - * rs1 = zero - * rs2 = a0 (ASID) - * HFENCE.VVMA zero, a0 - * 0010001 01010 00000 000 00000 1110011 - */ - asm volatile ("add a0, %0, zero\n" - ".word 0x22a00073\n" - :: "r" (asid) : "a0", "memory"); + asm volatile(HFENCE_VVMA(zero, %0) : : "r" (asid) : "memory"); csr_write(CSR_HGATP, hgatp); } @@ -183,16 +130,16 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { - /* - * rs1 = a0 (GVA) - * rs2 = zero - * HFENCE.VVMA a0 - * 0010001 00000 01010 000 00000 1110011 - */ - asm volatile ("add a0, %0, zero\n" - ".word 0x22050073\n" - :: "r" (pos) : "a0", "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HINVAL_VVMA(%0, zero) + : : "r" (pos) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HFENCE_VVMA(%0, zero) + : : "r" (pos) : "memory"); } csr_write(CSR_HGATP, hgatp); @@ -204,13 +151,7 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - /* - * rs1 = zero - * rs2 = zero - * HFENCE.VVMA - * 0010001 00000 00000 000 00000 1110011 - */ - asm volatile (".word 0x22000073" ::: "memory"); + asm volatile(HFENCE_VVMA(zero, zero) : : : "memory"); csr_write(CSR_HGATP, hgatp); } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index d0f08d5b4282952c82a462cb8f4ba9bc27a5f9eb..a032c4f0d60061a3ffc29c9a12f5abc0fb450876 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { @@ -28,6 +30,7 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), STATS_DESC_COUNTER(VCPU, csr_exit_user), STATS_DESC_COUNTER(VCPU, csr_exit_kernel), + STATS_DESC_COUNTER(VCPU, signal_exits), STATS_DESC_COUNTER(VCPU, exits) }; @@ -42,17 +45,23 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) +#define KVM_ISA_EXT_ARR(ext) [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext + /* Mapping between KVM ISA Extension ID & Host ISA extension ID */ static const unsigned long kvm_isa_ext_arr[] = { - RISCV_ISA_EXT_a, - RISCV_ISA_EXT_c, - RISCV_ISA_EXT_d, - RISCV_ISA_EXT_f, - RISCV_ISA_EXT_h, - RISCV_ISA_EXT_i, - RISCV_ISA_EXT_m, - RISCV_ISA_EXT_SVPBMT, - RISCV_ISA_EXT_SSTC, + [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, + [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, + [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, + [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, + [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, + [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, + [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, + + KVM_ISA_EXT_ARR(SSTC), + KVM_ISA_EXT_ARR(SVINVAL), + KVM_ISA_EXT_ARR(SVPBMT), + KVM_ISA_EXT_ARR(ZIHINTPAUSE), + KVM_ISA_EXT_ARR(ZICBOM), }; static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) @@ -87,6 +96,8 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_I: case KVM_RISCV_ISA_EXT_M: case KVM_RISCV_ISA_EXT_SSTC: + case KVM_RISCV_ISA_EXT_SVINVAL: + case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: return false; default: break; @@ -254,6 +265,11 @@ static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, case KVM_REG_RISCV_CONFIG_REG(isa): reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; break; + case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): + if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) + return -EINVAL; + reg_val = riscv_cbom_block_size; + break; default: return -EINVAL; } @@ -311,6 +327,8 @@ static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, return -EOPNOTSUPP; } break; + case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): + return -EOPNOTSUPP; default: return -EINVAL; } @@ -784,11 +802,15 @@ static void kvm_riscv_vcpu_update_config(const unsigned long *isa) { u64 henvcfg = 0; - if (__riscv_isa_extension_available(isa, RISCV_ISA_EXT_SVPBMT)) + if (riscv_isa_extension_available(isa, SVPBMT)) henvcfg |= ENVCFG_PBMTE; - if (__riscv_isa_extension_available(isa, RISCV_ISA_EXT_SSTC)) + if (riscv_isa_extension_available(isa, SSTC)) henvcfg |= ENVCFG_STCE; + + if (riscv_isa_extension_available(isa, ZICBOM)) + henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); + csr_write(CSR_HENVCFG, henvcfg); #ifdef CONFIG_32BIT csr_write(CSR_HENVCFGH, henvcfg >> 32); @@ -958,7 +980,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_UNKNOWN; while (ret > 0) { /* Check conditions before entering the guest */ - cond_resched(); + ret = xfer_to_guest_mode_handle_work(vcpu); + if (!ret) + ret = 1; kvm_riscv_gstage_vmid_update(vcpu); @@ -966,15 +990,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) local_irq_disable(); - /* - * Exit if we have a signal pending so that we can deliver - * the signal to user space. - */ - if (signal_pending(current)) { - ret = -EINTR; - run->exit_reason = KVM_EXIT_INTR; - } - /* * Ensure we set mode to IN_GUEST_MODE after we disable * interrupts and before the final VCPU requests check. @@ -997,7 +1012,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) if (ret <= 0 || kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || - kvm_request_pending(vcpu)) { + kvm_request_pending(vcpu) || + xfer_to_guest_mode_work_pending()) { vcpu->mode = OUTSIDE_GUEST_MODE; local_irq_enable(); kvm_vcpu_srcu_read_lock(vcpu); diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index d5c36386878a37f8947ad0d7f05cf0539d6f4b22..c9f741ab26f5b8a9d70af94cdad121a36fb5e11d 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -8,6 +8,7 @@ #include #include +#include static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap) @@ -62,11 +63,7 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, { register unsigned long taddr asm("a0") = (unsigned long)trap; register unsigned long ttmp asm("a1"); - register unsigned long val asm("t0"); - register unsigned long tmp asm("t1"); - register unsigned long addr asm("t2") = guest_addr; - unsigned long flags; - unsigned long old_stvec, old_hstatus; + unsigned long flags, val, tmp, old_stvec, old_hstatus; local_irq_save(flags); @@ -82,29 +79,19 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, ".option push\n" ".option norvc\n" "add %[ttmp], %[taddr], 0\n" - /* - * HLVX.HU %[val], (%[addr]) - * HLVX.HU t0, (t2) - * 0110010 00011 00111 100 00101 1110011 - */ - ".word 0x6433c2f3\n" + HLVX_HU(%[val], %[addr]) "andi %[tmp], %[val], 3\n" "addi %[tmp], %[tmp], -3\n" "bne %[tmp], zero, 2f\n" "addi %[addr], %[addr], 2\n" - /* - * HLVX.HU %[tmp], (%[addr]) - * HLVX.HU t1, (t2) - * 0110010 00011 00111 100 00110 1110011 - */ - ".word 0x6433c373\n" + HLVX_HU(%[tmp], %[addr]) "sll %[tmp], %[tmp], 16\n" "add %[val], %[val], %[tmp]\n" "2:\n" ".option pop" : [val] "=&r" (val), [tmp] "=&r" (tmp), [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp), - [addr] "+&r" (addr) : : "memory"); + [addr] "+&r" (guest_addr) : : "memory"); if (trap->scause == EXC_LOAD_PAGE_FAULT) trap->scause = EXC_INST_PAGE_FAULT; @@ -121,24 +108,14 @@ unsigned long kvm_riscv_vcpu_unpriv_read(struct kvm_vcpu *vcpu, ".option norvc\n" "add %[ttmp], %[taddr], 0\n" #ifdef CONFIG_64BIT - /* - * HLV.D %[val], (%[addr]) - * HLV.D t0, (t2) - * 0110110 00000 00111 100 00101 1110011 - */ - ".word 0x6c03c2f3\n" + HLV_D(%[val], %[addr]) #else - /* - * HLV.W %[val], (%[addr]) - * HLV.W t0, (t2) - * 0110100 00000 00111 100 00101 1110011 - */ - ".word 0x6803c2f3\n" + HLV_W(%[val], %[addr]) #endif ".option pop" : [val] "=&r" (val), [taddr] "+&r" (taddr), [ttmp] "+&r" (ttmp) - : [addr] "r" (addr) : "memory"); + : [addr] "r" (guest_addr) : "memory"); } csr_write(CSR_STVEC, old_stvec); diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 7eb90a47b57168c4e271ceea14784f07ac3d0dd6..0bb52761a3f73ee92fa23b189b317954525d596a 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -191,7 +191,6 @@ void kvm_riscv_vcpu_wfi(struct kvm_vcpu *vcpu) kvm_vcpu_srcu_read_unlock(vcpu); kvm_vcpu_halt(vcpu); kvm_vcpu_srcu_read_lock(vcpu); - kvm_clear_request(KVM_REQ_UNHALT, vcpu); } } diff --git a/arch/riscv/kvm/vcpu_sbi.c b/arch/riscv/kvm/vcpu_sbi.c index d45e7da3f0d324e61c7215ddec8f8a61b7446a9c..f96991d230bfc33fe2eef7de81e89a6791c96e76 100644 --- a/arch/riscv/kvm/vcpu_sbi.c +++ b/arch/riscv/kvm/vcpu_sbi.c @@ -32,23 +32,13 @@ static int kvm_linux_err_map_sbi(int err) }; } -#ifdef CONFIG_RISCV_SBI_V01 -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01; -#else +#ifndef CONFIG_RISCV_SBI_V01 static const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_v01 = { .extid_start = -1UL, .extid_end = -1UL, .handler = NULL, }; #endif -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_base; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_time; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_ipi; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_rfence; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_srst; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_hsm; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_experimental; -extern const struct kvm_vcpu_sbi_extension vcpu_sbi_ext_vendor; static const struct kvm_vcpu_sbi_extension *sbi_ext[] = { &vcpu_sbi_ext_v01, diff --git a/arch/riscv/kvm/vcpu_timer.c b/arch/riscv/kvm/vcpu_timer.c index 16f50c46ba39430ee95e0a2a0434ea599671765d..185f2386a747ed12f60b668de5c611ac14cda413 100644 --- a/arch/riscv/kvm/vcpu_timer.c +++ b/arch/riscv/kvm/vcpu_timer.c @@ -299,7 +299,6 @@ static void kvm_riscv_vcpu_update_timedelta(struct kvm_vcpu *vcpu) void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu) { - struct kvm_vcpu_csr *csr; struct kvm_vcpu_timer *t = &vcpu->arch.timer; kvm_riscv_vcpu_update_timedelta(vcpu); @@ -307,7 +306,6 @@ void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu) if (!t->sstc_enabled) return; - csr = &vcpu->arch.guest_csr; #if defined(CONFIG_32BIT) csr_write(CSR_VSTIMECMP, (u32)t->next_cycles); csr_write(CSR_VSTIMECMPH, (u32)(t->next_cycles >> 32)); @@ -324,13 +322,11 @@ void kvm_riscv_vcpu_timer_restore(struct kvm_vcpu *vcpu) void kvm_riscv_vcpu_timer_save(struct kvm_vcpu *vcpu) { - struct kvm_vcpu_csr *csr; struct kvm_vcpu_timer *t = &vcpu->arch.timer; if (!t->sstc_enabled) return; - csr = &vcpu->arch.guest_csr; t = &vcpu->arch.timer; #if defined(CONFIG_32BIT) t->next_cycles = csr_read(CSR_VSTIMECMP); diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c index cd2225304c82e2c4efd349a04e93f41a2adfa005..b0add983530abb5c1b28b093a02e73b173118617 100644 --- a/arch/riscv/mm/dma-noncoherent.c +++ b/arch/riscv/mm/dma-noncoherent.c @@ -12,7 +12,9 @@ #include #include -static unsigned int riscv_cbom_block_size = L1_CACHE_BYTES; +unsigned int riscv_cbom_block_size; +EXPORT_SYMBOL_GPL(riscv_cbom_block_size); + static bool noncoherent_supported; void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, @@ -79,38 +81,41 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, void riscv_init_cbom_blocksize(void) { struct device_node *node; + unsigned long cbom_hartid; + u32 val, probed_block_size; int ret; - u32 val; + probed_block_size = 0; for_each_of_cpu_node(node) { unsigned long hartid; - int cbom_hartid; ret = riscv_of_processor_hartid(node, &hartid); if (ret) continue; - if (hartid < 0) - continue; - /* set block-size for cbom extension if available */ ret = of_property_read_u32(node, "riscv,cbom-block-size", &val); if (ret) continue; - if (!riscv_cbom_block_size) { - riscv_cbom_block_size = val; + if (!probed_block_size) { + probed_block_size = val; cbom_hartid = hartid; } else { - if (riscv_cbom_block_size != val) - pr_warn("cbom-block-size mismatched between harts %d and %lu\n", + if (probed_block_size != val) + pr_warn("cbom-block-size mismatched between harts %lu and %lu\n", cbom_hartid, hartid); } } + + if (probed_block_size) + riscv_cbom_block_size = probed_block_size; } #endif void riscv_noncoherent_supported(void) { + WARN(!riscv_cbom_block_size, + "Non-coherent DMA support enabled without a block size\n"); noncoherent_supported = true; } diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index f2fbd1400b7c9ade4fffd340a64dc96597e4c729..d86f7cebd4a7ef3799089cbbbbe61f1a1b099605 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -184,7 +184,8 @@ static inline bool access_error(unsigned long cause, struct vm_area_struct *vma) } break; case EXC_LOAD_PAGE_FAULT: - if (!(vma->vm_flags & VM_READ)) { + /* Write implies read */ + if (!(vma->vm_flags & (VM_READ | VM_WRITE))) { return true; } break; diff --git a/arch/riscv/mm/pageattr.c b/arch/riscv/mm/pageattr.c index 5e49e4b4a4cccc08df7282af5aabb193fb377daf..86c56616e5dea9c0d8d68748c17725e9fc031023 100644 --- a/arch/riscv/mm/pageattr.c +++ b/arch/riscv/mm/pageattr.c @@ -118,10 +118,10 @@ static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask, if (!numpages) return 0; - mmap_read_lock(&init_mm); + mmap_write_lock(&init_mm); ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL, &masks); - mmap_read_unlock(&init_mm); + mmap_write_unlock(&init_mm); flush_tlb_kernel_range(start, end); diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 4cb5d17e7ead620ee1a84db0347cf68f18a009c9..de6d8b2ea4d8f86e0551bb762893df4228553634 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -119,8 +119,6 @@ export KBUILD_CFLAGS_DECOMPRESSOR OBJCOPYFLAGS := -O binary -head-y := arch/s390/kernel/head64.o - libs-y += arch/s390/lib/ drivers-y += drivers/s390/ diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index bc48fe82d949a4dc60ebb2dda2f73d2b308d36e1..6e7f01ca53e6da2014b1e881cc2d720ae2b2c1fd 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -10,11 +10,14 @@ #include #include #include +#include #include "decompressor.h" #include "boot.h" #include "uv.h" unsigned long __bootdata_preserved(__kaslr_offset); +unsigned long __bootdata_preserved(__abs_lowcore); +unsigned long __bootdata_preserved(__memcpy_real_area); unsigned long __bootdata(__amode31_base); unsigned long __bootdata_preserved(VMALLOC_START); unsigned long __bootdata_preserved(VMALLOC_END); @@ -180,7 +183,10 @@ static void setup_kernel_memory_layout(void) /* force vmalloc and modules below kasan shadow */ vmax = min(vmax, KASAN_SHADOW_START); #endif - MODULES_END = vmax; + __memcpy_real_area = round_down(vmax - PAGE_SIZE, PAGE_SIZE); + __abs_lowcore = round_down(__memcpy_real_area - ABS_LOWCORE_MAP_SIZE, + sizeof(struct lowcore)); + MODULES_END = round_down(__abs_lowcore, _SEGMENT_SIZE); MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; diff --git a/arch/s390/boot/version.c b/arch/s390/boot/version.c index d32e58bdda6ad2c7c15c8f104a4234f99eefb77b..fd32f038777f43e9a0cd0bd0b7c38c1ea76adc45 100644 --- a/arch/s390/boot/version.c +++ b/arch/s390/boot/version.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include "boot.h" diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index f6dfde577ce83049db124250e3ebbb3af5c56924..2a827002934bc65466bb42e36a19736c1b2ed814 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -40,8 +40,6 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_SCHED_AUTOGROUP=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set -CONFIG_USERFAULTFD=y -# CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_LIVEPATCH=y CONFIG_MARCH_ZEC12=y @@ -74,6 +72,7 @@ CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODULE_UNLOAD_TAINT_TRACKING=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y @@ -93,6 +92,10 @@ CONFIG_UNIXWARE_DISKLABEL=y CONFIG_IOSCHED_BFQ=y CONFIG_BFQ_GROUP_IOSCHED=y CONFIG_BINFMT_MISC=m +CONFIG_ZSWAP=y +CONFIG_ZSMALLOC_STAT=y +CONFIG_SLUB_STATS=y +# CONFIG_COMPAT_BRK is not set CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y @@ -102,14 +105,12 @@ CONFIG_CMA_DEBUGFS=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=7 CONFIG_MEM_SOFT_DIRTY=y -CONFIG_ZSWAP=y -CONFIG_ZSMALLOC=y -CONFIG_ZSMALLOC_STAT=y CONFIG_DEFERRED_STRUCT_PAGE_INIT=y CONFIG_IDLE_PAGE_TRACKING=y CONFIG_PERCPU_STATS=y CONFIG_GUP_TEST=y CONFIG_ANON_VMA_NAME=y +CONFIG_USERFAULTFD=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=m @@ -167,6 +168,7 @@ CONFIG_BRIDGE_NETFILTER=m CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_PROCFS=y CONFIG_NF_CONNTRACK_EVENTS=y CONFIG_NF_CONNTRACK_TIMEOUT=y CONFIG_NF_CONNTRACK_TIMESTAMP=y @@ -493,7 +495,6 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_ASIX is not set # CONFIG_NET_VENDOR_ATHEROS is not set # CONFIG_NET_VENDOR_BROADCOM is not set -# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set # CONFIG_NET_VENDOR_CHELSIO is not set @@ -509,7 +510,7 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_GOOGLE is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set -# CONFIG_NET_VENDOR_MICROSOFT is not set +# CONFIG_NET_VENDOR_WANGXUN is not set # CONFIG_NET_VENDOR_LITEX is not set # CONFIG_NET_VENDOR_MARVELL is not set CONFIG_MLX4_EN=m @@ -518,16 +519,18 @@ CONFIG_MLX5_CORE_EN=y # CONFIG_NET_VENDOR_MICREL is not set # CONFIG_NET_VENDOR_MICROCHIP is not set # CONFIG_NET_VENDOR_MICROSEMI is not set +# CONFIG_NET_VENDOR_MICROSOFT is not set # CONFIG_NET_VENDOR_MYRI is not set +# CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETERION is not set # CONFIG_NET_VENDOR_NETRONOME is not set -# CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set # CONFIG_NET_VENDOR_PACKET_ENGINES is not set # CONFIG_NET_VENDOR_PENSANDO is not set # CONFIG_NET_VENDOR_QLOGIC is not set +# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_QUALCOMM is not set # CONFIG_NET_VENDOR_RDC is not set # CONFIG_NET_VENDOR_REALTEK is not set @@ -535,9 +538,9 @@ CONFIG_MLX5_CORE_EN=y # CONFIG_NET_VENDOR_ROCKER is not set # CONFIG_NET_VENDOR_SAMSUNG is not set # CONFIG_NET_VENDOR_SEEQ is not set -# CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SILAN is not set # CONFIG_NET_VENDOR_SIS is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SMSC is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set @@ -570,6 +573,8 @@ CONFIG_VIRTIO_CONSOLE=m CONFIG_HW_RANDOM_VIRTIO=m CONFIG_HANGCHECK_TIMER=m CONFIG_TN3270_FS=y +# CONFIG_RANDOM_TRUST_CPU is not set +# CONFIG_RANDOM_TRUST_BOOTLOADER is not set CONFIG_PPS=m # CONFIG_PTP_1588_CLOCK is not set # CONFIG_HWMON is not set @@ -727,18 +732,26 @@ CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_CRC32_S390=y CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA512_S390=m +CONFIG_CRYPTO_SHA1_S390=m +CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m +CONFIG_CRYPTO_SHA3_256_S390=m +CONFIG_CRYPTO_SHA3_512_S390=m +CONFIG_CRYPTO_SM3_GENERIC=m CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_TI=m +CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARC4=m CONFIG_CRYPTO_BLOWFISH=m @@ -746,11 +759,14 @@ CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m +CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_SM4=m +CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_842=m @@ -766,16 +782,6 @@ CONFIG_CRYPTO_STATS=y CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m -CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA3_256_S390=m -CONFIG_CRYPTO_SHA3_512_S390=m -CONFIG_CRYPTO_DES_S390=m -CONFIG_CRYPTO_AES_S390=m -CONFIG_CRYPTO_CHACHA_S390=m -CONFIG_CRYPTO_GHASH_S390=m -CONFIG_CRYPTO_CRC32_S390=y CONFIG_CRYPTO_DEV_VIRTIO=m CONFIG_CORDIC=m CONFIG_CRYPTO_LIB_CURVE25519=m @@ -797,6 +803,7 @@ CONFIG_HEADERS_INSTALL=y CONFIG_DEBUG_SECTION_MISMATCH=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_PAGEALLOC=y +CONFIG_SLUB_DEBUG_ON=y CONFIG_PAGE_OWNER=y CONFIG_DEBUG_RODATA_TEST=y CONFIG_DEBUG_WX=y @@ -808,8 +815,6 @@ CONFIG_DEBUG_OBJECTS_TIMERS=y CONFIG_DEBUG_OBJECTS_WORK=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=y CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER=y -CONFIG_SLUB_DEBUG_ON=y -CONFIG_SLUB_STATS=y CONFIG_DEBUG_STACK_USAGE=y CONFIG_DEBUG_VM=y CONFIG_DEBUG_VM_PGFLAGS=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 706df3a4a867f0619aeae6e8be6efefffd8ea26e..fb780e80e4c8f72dd9203489c0921a1edbd23430 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -38,8 +38,6 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_SCHED_AUTOGROUP=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set -CONFIG_USERFAULTFD=y -# CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_LIVEPATCH=y CONFIG_MARCH_ZEC12=y @@ -69,6 +67,7 @@ CONFIG_MODULES=y CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODULE_UNLOAD_TAINT_TRACKING=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y @@ -88,6 +87,9 @@ CONFIG_UNIXWARE_DISKLABEL=y CONFIG_IOSCHED_BFQ=y CONFIG_BFQ_GROUP_IOSCHED=y CONFIG_BINFMT_MISC=m +CONFIG_ZSWAP=y +CONFIG_ZSMALLOC_STAT=y +# CONFIG_COMPAT_BRK is not set CONFIG_MEMORY_HOTPLUG=y CONFIG_MEMORY_HOTREMOVE=y CONFIG_KSM=y @@ -95,13 +97,11 @@ CONFIG_TRANSPARENT_HUGEPAGE=y CONFIG_CMA_SYSFS=y CONFIG_CMA_AREAS=7 CONFIG_MEM_SOFT_DIRTY=y -CONFIG_ZSWAP=y -CONFIG_ZSMALLOC=y -CONFIG_ZSMALLOC_STAT=y CONFIG_DEFERRED_STRUCT_PAGE_INIT=y CONFIG_IDLE_PAGE_TRACKING=y CONFIG_PERCPU_STATS=y CONFIG_ANON_VMA_NAME=y +CONFIG_USERFAULTFD=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_PACKET_DIAG=m @@ -159,6 +159,7 @@ CONFIG_BRIDGE_NETFILTER=m CONFIG_NETFILTER_NETLINK_HOOK=m CONFIG_NF_CONNTRACK=m CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_PROCFS=y CONFIG_NF_CONNTRACK_EVENTS=y CONFIG_NF_CONNTRACK_TIMEOUT=y CONFIG_NF_CONNTRACK_TIMESTAMP=y @@ -484,7 +485,6 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_ASIX is not set # CONFIG_NET_VENDOR_ATHEROS is not set # CONFIG_NET_VENDOR_BROADCOM is not set -# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_CADENCE is not set # CONFIG_NET_VENDOR_CAVIUM is not set # CONFIG_NET_VENDOR_CHELSIO is not set @@ -500,7 +500,7 @@ CONFIG_NLMON=m # CONFIG_NET_VENDOR_GOOGLE is not set # CONFIG_NET_VENDOR_HUAWEI is not set # CONFIG_NET_VENDOR_INTEL is not set -# CONFIG_NET_VENDOR_MICROSOFT is not set +# CONFIG_NET_VENDOR_WANGXUN is not set # CONFIG_NET_VENDOR_LITEX is not set # CONFIG_NET_VENDOR_MARVELL is not set CONFIG_MLX4_EN=m @@ -509,16 +509,18 @@ CONFIG_MLX5_CORE_EN=y # CONFIG_NET_VENDOR_MICREL is not set # CONFIG_NET_VENDOR_MICROCHIP is not set # CONFIG_NET_VENDOR_MICROSEMI is not set +# CONFIG_NET_VENDOR_MICROSOFT is not set # CONFIG_NET_VENDOR_MYRI is not set +# CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NATSEMI is not set # CONFIG_NET_VENDOR_NETERION is not set # CONFIG_NET_VENDOR_NETRONOME is not set -# CONFIG_NET_VENDOR_NI is not set # CONFIG_NET_VENDOR_NVIDIA is not set # CONFIG_NET_VENDOR_OKI is not set # CONFIG_NET_VENDOR_PACKET_ENGINES is not set # CONFIG_NET_VENDOR_PENSANDO is not set # CONFIG_NET_VENDOR_QLOGIC is not set +# CONFIG_NET_VENDOR_BROCADE is not set # CONFIG_NET_VENDOR_QUALCOMM is not set # CONFIG_NET_VENDOR_RDC is not set # CONFIG_NET_VENDOR_REALTEK is not set @@ -526,9 +528,9 @@ CONFIG_MLX5_CORE_EN=y # CONFIG_NET_VENDOR_ROCKER is not set # CONFIG_NET_VENDOR_SAMSUNG is not set # CONFIG_NET_VENDOR_SEEQ is not set -# CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SILAN is not set # CONFIG_NET_VENDOR_SIS is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set # CONFIG_NET_VENDOR_SMSC is not set # CONFIG_NET_VENDOR_SOCIONEXT is not set # CONFIG_NET_VENDOR_STMICRO is not set @@ -561,6 +563,8 @@ CONFIG_VIRTIO_CONSOLE=m CONFIG_HW_RANDOM_VIRTIO=m CONFIG_HANGCHECK_TIMER=m CONFIG_TN3270_FS=y +# CONFIG_RANDOM_TRUST_CPU is not set +# CONFIG_RANDOM_TRUST_BOOTLOADER is not set # CONFIG_PTP_1588_CLOCK is not set # CONFIG_HWMON is not set CONFIG_WATCHDOG=y @@ -713,18 +717,26 @@ CONFIG_CRYPTO_OFB=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_KEYWRAP=m CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_HCTR2=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_VMAC=m CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_BLAKE2S=m +CONFIG_CRYPTO_CRC32_S390=y CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA512_S390=m +CONFIG_CRYPTO_SHA1_S390=m +CONFIG_CRYPTO_SHA256_S390=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m +CONFIG_CRYPTO_SHA3_256_S390=m +CONFIG_CRYPTO_SHA3_512_S390=m +CONFIG_CRYPTO_SM3_GENERIC=m CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_TI=m +CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_ANUBIS=m CONFIG_CRYPTO_ARC4=m CONFIG_CRYPTO_BLOWFISH=m @@ -732,11 +744,14 @@ CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m +CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_SM4=m +CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_842=m @@ -752,16 +767,6 @@ CONFIG_CRYPTO_STATS=y CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m -CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA3_256_S390=m -CONFIG_CRYPTO_SHA3_512_S390=m -CONFIG_CRYPTO_DES_S390=m -CONFIG_CRYPTO_AES_S390=m -CONFIG_CRYPTO_CHACHA_S390=m -CONFIG_CRYPTO_GHASH_S390=m -CONFIG_CRYPTO_CRC32_S390=y CONFIG_CRYPTO_DEV_VIRTIO=m CONFIG_CORDIC=m CONFIG_PRIME_NUMBERS=m diff --git a/arch/s390/configs/zfcpdump_defconfig b/arch/s390/configs/zfcpdump_defconfig index f4976f611b9447dad5510c4856cbdd202f8443a2..a5576b8d4081e959bc44b3fe23a435cf1d76dfb5 100644 --- a/arch/s390/configs/zfcpdump_defconfig +++ b/arch/s390/configs/zfcpdump_defconfig @@ -1,4 +1,3 @@ -# CONFIG_SWAP is not set CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y @@ -9,7 +8,6 @@ CONFIG_BPF_SYSCALL=y # CONFIG_NET_NS is not set CONFIG_BLK_DEV_INITRD=y CONFIG_CC_OPTIMIZE_FOR_SIZE=y -# CONFIG_COMPAT_BRK is not set CONFIG_MARCH_ZEC12=y CONFIG_TUNE_ZEC12=y # CONFIG_COMPAT is not set @@ -28,6 +26,8 @@ CONFIG_CRASH_DUMP=y # CONFIG_BLOCK_LEGACY_AUTOLOAD is not set CONFIG_PARTITION_ADVANCED=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +# CONFIG_SWAP is not set +# CONFIG_COMPAT_BRK is not set # CONFIG_COMPACTION is not set # CONFIG_MIGRATION is not set CONFIG_NET=y @@ -53,10 +53,12 @@ CONFIG_ZFCP=y # CONFIG_HVC_IUCV is not set # CONFIG_HW_RANDOM_S390 is not set # CONFIG_HMC_DRV is not set +# CONFIG_S390_UV_UAPI is not set # CONFIG_S390_TAPE is not set # CONFIG_VMCP is not set # CONFIG_MONWRITER is not set # CONFIG_S390_VMUR is not set +# CONFIG_RANDOM_TRUST_BOOTLOADER is not set # CONFIG_HID is not set # CONFIG_VIRTIO_MENU is not set # CONFIG_VHOST_MENU is not set diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..06ee706b0d7886e403e50ca9fcea79c39e7de50e --- /dev/null +++ b/arch/s390/crypto/Kconfig @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (s390)" + +config CRYPTO_CRC32_S390 + tristate "CRC32c and CRC32" + depends on S390 + select CRYPTO_HASH + select CRC32 + help + CRC32c and CRC32 CRC algorithms + + Architecture: s390 + + It is available with IBM z13 or later. + +config CRYPTO_SHA512_S390 + tristate "Hash functions: SHA-384 and SHA-512" + depends on S390 + select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: s390 + + It is available as of z10. + +config CRYPTO_SHA1_S390 + tristate "Hash functions: SHA-1" + depends on S390 + select CRYPTO_HASH + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: s390 + + It is available as of z990. + +config CRYPTO_SHA256_S390 + tristate "Hash functions: SHA-224 and SHA-256" + depends on S390 + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: s390 + + It is available as of z9. + +config CRYPTO_SHA3_256_S390 + tristate "Hash functions: SHA3-224 and SHA3-256" + depends on S390 + select CRYPTO_HASH + help + SHA3-224 and SHA3-256 secure hash algorithms (FIPS 202) + + Architecture: s390 + + It is available as of z14. + +config CRYPTO_SHA3_512_S390 + tristate "Hash functions: SHA3-384 and SHA3-512" + depends on S390 + select CRYPTO_HASH + help + SHA3-384 and SHA3-512 secure hash algorithms (FIPS 202) + + Architecture: s390 + + It is available as of z14. + +config CRYPTO_GHASH_S390 + tristate "Hash functions: GHASH" + depends on S390 + select CRYPTO_HASH + help + GCM GHASH hash function (NIST SP800-38D) + + Architecture: s390 + + It is available as of z196. + +config CRYPTO_AES_S390 + tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM" + depends on S390 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Block cipher: AES cipher algorithms (FIPS 197) + AEAD cipher: AES with GCM + Length-preserving ciphers: AES with ECB, CBC, XTS, and CTR modes + + Architecture: s390 + + As of z9 the ECB and CBC modes are hardware accelerated + for 128 bit keys. + + As of z10 the ECB and CBC modes are hardware accelerated + for all AES key sizes. + + As of z196 the CTR mode is hardware accelerated for all AES + key sizes and XTS mode is hardware accelerated for 256 and + 512 bit keys. + +config CRYPTO_DES_S390 + tristate "Ciphers: DES and Triple DES EDE, modes: ECB, CBC, CTR" + depends on S390 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + help + Block ciphers: DES (FIPS 46-2) cipher algorithm + Block ciphers: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: DES with ECB, CBC, and CTR modes + Length-preserving ciphers: Triple DES EDED with ECB, CBC, and CTR modes + + Architecture: s390 + + As of z990 the ECB and CBC mode are hardware accelerated. + As of z196 the CTR mode is hardware accelerated. + +config CRYPTO_CHACHA_S390 + tristate "Ciphers: ChaCha20" + depends on S390 + select CRYPTO_SKCIPHER + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving cipher: ChaCha20 stream cipher (RFC 7539) + + Architecture: s390 + + It is available as of z13. + +endmenu diff --git a/arch/s390/include/asm/abs_lowcore.h b/arch/s390/include/asm/abs_lowcore.h new file mode 100644 index 0000000000000000000000000000000000000000..4c61b14ee928610bd91cec2a03edad355d99c733 --- /dev/null +++ b/arch/s390/include/asm/abs_lowcore.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_ABS_LOWCORE_H +#define _ASM_S390_ABS_LOWCORE_H + +#include + +#define ABS_LOWCORE_MAP_SIZE (NR_CPUS * sizeof(struct lowcore)) + +extern unsigned long __abs_lowcore; +extern bool abs_lowcore_mapped; + +struct lowcore *get_abs_lowcore(unsigned long *flags); +void put_abs_lowcore(struct lowcore *lc, unsigned long flags); +int abs_lowcore_map(int cpu, struct lowcore *lc, bool alloc); +void abs_lowcore_unmap(int cpu); + +#endif /* _ASM_S390_ABS_LOWCORE_H */ diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 9a7d15da966e34db8b485cec09f02907542f2a46..2de74fcd0578f42c25d178f4f3ef1dbbaa30158c 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -176,14 +176,8 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) return old & mask; } -static __always_inline bool -arch_test_bit(unsigned long nr, const volatile unsigned long *addr) -{ - const volatile unsigned long *p = __bitops_word(nr, addr); - unsigned long mask = __bitops_mask(nr); - - return *p & mask; -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire static inline bool arch_test_and_set_bit_lock(unsigned long nr, volatile unsigned long *ptr) diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index d4e90f2ba77e6f7cd0db6521aae8580607fb9116..bd1596810cc1ed97623b3351b4f2106249978be1 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -214,7 +214,6 @@ extern struct ccw_device *ccw_device_create_console(struct ccw_driver *); extern void ccw_device_destroy_console(struct ccw_device *); extern int ccw_device_enable_console(struct ccw_device *); extern void ccw_device_wait_idle(struct ccw_device *); -extern int ccw_device_force_console(struct ccw_device *); extern void *ccw_device_dma_zalloc(struct ccw_device *cdev, size_t size); extern void ccw_device_dma_free(struct ccw_device *cdev, diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index 267a8f88e1435da57e41da2be04185aa675f176c..adf7d8cdac7e64de4b9ba849f862452d0142666a 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -95,7 +95,8 @@ union ctlreg0 { Interruption-Filtering Override */ unsigned long : 3; unsigned long ccc : 1; /* Cryptography counter control */ - unsigned long : 18; + unsigned long pec : 1; /* PAI extension control */ + unsigned long : 17; unsigned long : 3; unsigned long lap : 1; /* Low-address-protection control */ unsigned long : 4; diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h index f22beda9e6d5cd40b3cebe77077e15e3150622ad..ccdbccfde148c3afd2c102328161e8f433930adf 100644 --- a/arch/s390/include/asm/hugetlb.h +++ b/arch/s390/include/asm/hugetlb.h @@ -28,9 +28,11 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, static inline int prepare_hugepage_range(struct file *file, unsigned long addr, unsigned long len) { - if (len & ~HPAGE_MASK) + struct hstate *h = hstate_file(file); + + if (len & ~huge_page_mask(h)) return -EINVAL; - if (addr & ~HPAGE_MASK) + if (addr & ~huge_page_mask(h)) return -EINVAL; return 0; } diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index f39092e0ceaad3984b8365c43f3f9e97fb2a1b76..b1e98a9ed152b91365f014c7a54b3b5cfbeff459 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -1038,16 +1038,11 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} #define __KVM_HAVE_ARCH_VM_FREE void kvm_arch_free_vm(struct kvm *kvm); -#ifdef CONFIG_VFIO_PCI_ZDEV_KVM -int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm); -void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev); -#else -static inline int kvm_s390_pci_register_kvm(struct zpci_dev *dev, - struct kvm *kvm) -{ - return -EPERM; -} -static inline void kvm_s390_pci_unregister_kvm(struct zpci_dev *dev) {} -#endif +struct zpci_kvm_hook { + int (*kvm_register)(void *opaque, struct kvm *kvm); + void (*kvm_unregister)(void *opaque); +}; + +extern struct zpci_kvm_hook zpci_kvm_hook; #endif diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 26fe5e535728d3923fd0f87f1269fb63a5c30def..8aa1f6530a3e42c71e8d5fc03fb1d8e83e75fa2b 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -203,7 +203,9 @@ struct lowcore { __u8 pad_0x1400[0x1500-0x1400]; /* 0x1400 */ /* Cryptography-counter designation */ __u64 ccd; /* 0x1500 */ - __u8 pad_0x1508[0x1800-0x1508]; /* 0x1508 */ + /* AI-extension counter designation */ + __u64 aicd; /* 0x1508 */ + __u8 pad_0x1510[0x1800-0x1510]; /* 0x1510 */ /* Transaction abort diagnostic block */ struct pgm_tdb pgm_tdb; /* 0x1800 */ diff --git a/arch/s390/include/asm/maccess.h b/arch/s390/include/asm/maccess.h new file mode 100644 index 0000000000000000000000000000000000000000..c7fa838cf6b90d135f3047fbf7296a38dbb060a4 --- /dev/null +++ b/arch/s390/include/asm/maccess.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_S390_MACCESS_H +#define __ASM_S390_MACCESS_H + +#include + +struct iov_iter; + +extern unsigned long __memcpy_real_area; +void memcpy_real_init(void); +size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count); +int memcpy_real(void *dest, unsigned long src, size_t count); +#ifdef CONFIG_CRASH_DUMP +int copy_oldmem_kernel(void *dst, unsigned long src, size_t count); +#endif + +#endif /* __ASM_S390_MACCESS_H */ diff --git a/arch/s390/include/asm/os_info.h b/arch/s390/include/asm/os_info.h index 85248d8fee0cd11c3ea2ba2f99c75757a602c196..0d1c74a7a65044f86b7dcff62a7aab17a5064b48 100644 --- a/arch/s390/include/asm/os_info.h +++ b/arch/s390/include/asm/os_info.h @@ -41,20 +41,6 @@ u32 os_info_csum(struct os_info *os_info); #ifdef CONFIG_CRASH_DUMP void *os_info_old_entry(int nr, unsigned long *size); -size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count); - -static inline int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) -{ - struct iov_iter iter; - struct kvec kvec; - - kvec.iov_base = dst; - kvec.iov_len = count; - iov_iter_kvec(&iter, WRITE, &kvec, 1, count); - if (copy_oldmem_iter(&iter, src, count) < count) - return -EFAULT; - return 0; -} #else static inline void *os_info_old_entry(int nr, unsigned long *size) { diff --git a/arch/s390/include/asm/pai.h b/arch/s390/include/asm/pai.h index 5b7e33ac6f0bb03af659e5805a6d80e38461f068..1a8a6b15d12139112dd7e946887f8019f9bad8fb 100644 --- a/arch/s390/include/asm/pai.h +++ b/arch/s390/include/asm/pai.h @@ -17,7 +17,9 @@ struct qpaci_info_block { struct { u64 : 8; u64 num_cc : 8; /* # of supported crypto counters */ - u64 : 48; + u64 : 9; + u64 num_nnpa : 7; /* # of supported NNPA counters */ + u64 : 32; }; }; @@ -42,6 +44,8 @@ static inline int qpaci(struct qpaci_info_block *info) #define PAI_CRYPTO_BASE 0x1000 /* First event number */ #define PAI_CRYPTO_MAXCTR 256 /* Max # of event counters */ #define PAI_CRYPTO_KERNEL_OFFSET 2048 +#define PAI_NNPA_BASE 0x1800 /* First event number */ +#define PAI_NNPA_MAXCTR 128 /* Max # of event counters */ DECLARE_STATIC_KEY_FALSE(pai_key); diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index 7b4cdadbc023c7a95346da3dab99ecd5b368a591..108e732d7b140def7918271a0c3fb75f038dde3a 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -117,7 +117,6 @@ struct zpci_bus { struct zpci_dev { struct zpci_bus *zbus; struct list_head entry; /* list of all zpci_devices, needed for hotplug, etc. */ - struct list_head bus_next; struct kref kref; struct hotplug_slot hotplug_slot; diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index f019df19884df0fa86155699f105a6dc801aa0a8..f1cb9391190db93f051602be1ac3834492fe5278 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1777,6 +1777,10 @@ static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) extern int vmem_add_mapping(unsigned long start, unsigned long size); extern void vmem_remove_mapping(unsigned long start, unsigned long size); +extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc); +extern int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot); +extern void vmem_unmap_4k_page(unsigned long addr); +extern pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc); extern int s390_enable_sie(void); extern int s390_enable_skey(void); extern void s390_reset_cmma(struct mm_struct *mm); diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index bd66f8e349492f2e0b70f51a43a40732151fd703..87be3e855bf72703f28d97f0d118bf6bd75f23bd 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -186,9 +186,6 @@ struct pt_regs; void show_registers(struct pt_regs *regs); void show_cacheinfo(struct seq_file *m); -/* Free all resources held by a thread. */ -static inline void release_thread(struct task_struct *tsk) { } - /* Free guarded storage control block */ void guarded_storage_release(struct task_struct *tsk); void gs_load_bc_cb(struct pt_regs *regs); @@ -306,23 +303,6 @@ static __always_inline void __noreturn disabled_wait(void) #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL -extern int memcpy_real(void *, unsigned long, size_t); -extern void memcpy_absolute(void *, void *, size_t); - -#define put_abs_lowcore(member, x) do { \ - unsigned long __abs_address = offsetof(struct lowcore, member); \ - __typeof__(((struct lowcore *)0)->member) __tmp = (x); \ - \ - memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp)); \ -} while (0) - -#define get_abs_lowcore(x, member) do { \ - unsigned long __abs_address = offsetof(struct lowcore, member); \ - __typeof__(((struct lowcore *)0)->member) *__ptr = &(x); \ - \ - memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr)); \ -} while (0) - extern int s390_isolate_bp(void); extern int s390_isolate_bp_guest(void); diff --git a/arch/s390/include/asm/scsw.h b/arch/s390/include/asm/scsw.h index 7ce584aff5bb5def0ea00f323c37d1fcdb55b30a..322bdcd4b616ada7601cc550f883ad4b71b119a0 100644 --- a/arch/s390/include/asm/scsw.h +++ b/arch/s390/include/asm/scsw.h @@ -215,6 +215,11 @@ union scsw { #define SNS2_ENV_DATA_PRESENT 0x10 #define SNS2_INPRECISE_END 0x04 +/* + * architectured values for PPRC errors + */ +#define SNS7_INVALID_ON_SEC 0x0e + /** * scsw_is_tm - check for transport mode scsw * @scsw: pointer to scsw diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 7f5d4763357b433c51f902c0c38c66d41fd33562..73ed2781073b260f5621bcb9142d5d2919e518dd 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -30,7 +30,8 @@ extern void smp_emergency_stop(void); extern int smp_find_processor_id(u16 address); extern int smp_store_status(int cpu); -extern void smp_save_dump_cpus(void); +extern void smp_save_dump_ipl_cpu(void); +extern void smp_save_dump_secondary_cpus(void); extern void smp_yield_cpu(int cpu); extern void smp_cpu_set_polarization(int cpu, int val); extern int smp_cpu_get_polarization(int cpu); @@ -58,6 +59,7 @@ static inline void smp_cpus_done(unsigned int max_cpus) { } +extern int smp_reinit_ipl_cpu(void); extern int smp_rescan_cpus(void); extern void __noreturn cpu_die(void); extern void __cpu_die(unsigned int cpu); diff --git a/arch/s390/include/asm/softirq_stack.h b/arch/s390/include/asm/softirq_stack.h index af68d6c1d5840ecf0d06e65cd7dcd449d1a27219..1ac5115d3115ee5738ed27cb7411c6702b5d4aae 100644 --- a/arch/s390/include/asm/softirq_stack.h +++ b/arch/s390/include/asm/softirq_stack.h @@ -5,7 +5,7 @@ #include #include -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK static inline void do_softirq_own_stack(void) { call_on_stack(0, S390_lowcore.async_stack, void, __do_softirq); diff --git a/arch/s390/include/asm/termios.h b/arch/s390/include/asm/termios.h deleted file mode 100644 index 46fa3020b41e75e626d23e268c55b7dca2ef3c86..0000000000000000000000000000000000000000 --- a/arch/s390/include/asm/termios.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * S390 version - * - * Derived from "include/asm-i386/termios.h" - */ -#ifndef _S390_TERMIOS_H -#define _S390_TERMIOS_H - -#include - - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D vtime=\0 vmin=\1 sxtc=\0 - start=^Q stop=^S susp=^Z eol=\0 - reprint=^R discard=^U werase=^W lnext=^V - eol2=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0" - -#define user_termios_to_kernel_termios(k, u) copy_from_user(k, u, sizeof(struct termios2)) -#define kernel_termios_to_user_termios(u, k) copy_to_user(u, k, sizeof(struct termios2)) - -#include - -#endif /* _S390_TERMIOS_H */ diff --git a/arch/s390/include/uapi/asm/dasd.h b/arch/s390/include/uapi/asm/dasd.h index 9ec86fae998050881dfcf56b760be156b5ca6384..93d1ccd3304c706f591d98932090eb994f200d4c 100644 --- a/arch/s390/include/uapi/asm/dasd.h +++ b/arch/s390/include/uapi/asm/dasd.h @@ -182,6 +182,18 @@ typedef struct format_data_t { unsigned int intensity; } format_data_t; +/* + * struct dasd_copypair_swap_data_t + * represents all data necessary to issue a swap of the copy pair relation + */ +struct dasd_copypair_swap_data_t { + char primary[20]; /* BUSID of primary */ + char secondary[20]; /* BUSID of secondary */ + + /* Reserved for future updates. */ + __u8 reserved[64]; +}; + /* * values to be used for format_data_t.intensity * 0/8: normal format @@ -326,6 +338,8 @@ struct dasd_snid_ioctl_data { #define BIODASDSATTR _IOW(DASD_IOCTL_LETTER,2,attrib_data_t) /* Release Allocated Space */ #define BIODASDRAS _IOW(DASD_IOCTL_LETTER, 3, format_data_t) +/* Swap copy pair relation */ +#define BIODASDCOPYPAIRSWAP _IOW(DASD_IOCTL_LETTER, 4, struct dasd_copypair_swap_data_t) /* Get Sense Path Group ID (SNID) data */ #define BIODASDSNID _IOWR(DASD_IOCTL_LETTER, 1, struct dasd_snid_ioctl_data) diff --git a/arch/s390/include/uapi/asm/termios.h b/arch/s390/include/uapi/asm/termios.h deleted file mode 100644 index 54223169c806e6c7cdf859b111b272eda6a0d9aa..0000000000000000000000000000000000000000 --- a/arch/s390/include/uapi/asm/termios.h +++ /dev/null @@ -1,50 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -/* - * S390 version - * - * Derived from "include/asm-i386/termios.h" - */ - -#ifndef _UAPI_S390_TERMIOS_H -#define _UAPI_S390_TERMIOS_H - -#include -#include - -struct winsize { - unsigned short ws_row; - unsigned short ws_col; - unsigned short ws_xpixel; - unsigned short ws_ypixel; -}; - -#define NCC 8 -struct termio { - unsigned short c_iflag; /* input mode flags */ - unsigned short c_oflag; /* output mode flags */ - unsigned short c_cflag; /* control mode flags */ - unsigned short c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - -/* modem lines */ -#define TIOCM_LE 0x001 -#define TIOCM_DTR 0x002 -#define TIOCM_RTS 0x004 -#define TIOCM_ST 0x008 -#define TIOCM_SR 0x010 -#define TIOCM_CTS 0x020 -#define TIOCM_CAR 0x040 -#define TIOCM_RNG 0x080 -#define TIOCM_DSR 0x100 -#define TIOCM_CD TIOCM_CAR -#define TIOCM_RI TIOCM_RNG -#define TIOCM_OUT1 0x2000 -#define TIOCM_OUT2 0x4000 -#define TIOCM_LOOP 0x8000 - -/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ - - -#endif /* _UAPI_S390_TERMIOS_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 3cbfa9fddd9a924ffcfe875e3ae14ec0a31484eb..5e6a23299790ffdf91f02fcf79b015410edf665e 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -33,16 +33,16 @@ CFLAGS_stacktrace.o += -fno-optimize-sibling-calls CFLAGS_dumpstack.o += -fno-optimize-sibling-calls CFLAGS_unwind_bc.o += -fno-optimize-sibling-calls -obj-y := traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o +obj-y := head64.o traps.o time.o process.o earlypgm.o early.o setup.o idle.o vtime.o obj-y += processor.o syscall.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o vdso.o cpufeature.o obj-y += sysinfo.o lgr.o os_info.o machine_kexec.o obj-y += runtime_instr.o cache.o fpu.o dumpstack.o guarded_storage.o sthyi.o obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o obj-y += nospec-branch.o ipl_vmparm.o machine_kexec_reloc.o unwind_bc.o -obj-y += smp.o text_amode31.o stacktrace.o +obj-y += smp.o text_amode31.o stacktrace.o abs_lowcore.o -extra-y += head64.o vmlinux.lds +extra-y += vmlinux.lds obj-$(CONFIG_SYSFS) += nospec-sysfs.o CFLAGS_REMOVE_nospec-branch.o += $(CC_FLAGS_EXPOLINE) @@ -72,7 +72,7 @@ obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf_common.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o perf_regs.o -obj-$(CONFIG_PERF_EVENTS) += perf_pai_crypto.o +obj-$(CONFIG_PERF_EVENTS) += perf_pai_crypto.o perf_pai_ext.o obj-$(CONFIG_TRACEPOINTS) += trace.o obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o diff --git a/arch/s390/kernel/abs_lowcore.c b/arch/s390/kernel/abs_lowcore.c new file mode 100644 index 0000000000000000000000000000000000000000..fb92e8ed05258d4c1c5bd246155cc3e3adff3acb --- /dev/null +++ b/arch/s390/kernel/abs_lowcore.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#define ABS_LOWCORE_UNMAPPED 1 +#define ABS_LOWCORE_LAP_ON 2 +#define ABS_LOWCORE_IRQS_ON 4 + +unsigned long __bootdata_preserved(__abs_lowcore); +bool __ro_after_init abs_lowcore_mapped; + +int abs_lowcore_map(int cpu, struct lowcore *lc, bool alloc) +{ + unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); + unsigned long phys = __pa(lc); + int rc, i; + + for (i = 0; i < LC_PAGES; i++) { + rc = __vmem_map_4k_page(addr, phys, PAGE_KERNEL, alloc); + if (rc) { + /* + * Do not unmap allocated page tables in case the + * allocation was not requested. In such a case the + * request is expected coming from an atomic context, + * while the unmap attempt might sleep. + */ + if (alloc) { + for (--i; i >= 0; i--) { + addr -= PAGE_SIZE; + vmem_unmap_4k_page(addr); + } + } + return rc; + } + addr += PAGE_SIZE; + phys += PAGE_SIZE; + } + return 0; +} + +void abs_lowcore_unmap(int cpu) +{ + unsigned long addr = __abs_lowcore + (cpu * sizeof(struct lowcore)); + int i; + + for (i = 0; i < LC_PAGES; i++) { + vmem_unmap_4k_page(addr); + addr += PAGE_SIZE; + } +} + +struct lowcore *get_abs_lowcore(unsigned long *flags) +{ + unsigned long irq_flags; + union ctlreg0 cr0; + int cpu; + + *flags = 0; + cpu = get_cpu(); + if (abs_lowcore_mapped) { + return ((struct lowcore *)__abs_lowcore) + cpu; + } else { + if (cpu != 0) + panic("Invalid unmapped absolute lowcore access\n"); + local_irq_save(irq_flags); + if (!irqs_disabled_flags(irq_flags)) + *flags |= ABS_LOWCORE_IRQS_ON; + __ctl_store(cr0.val, 0, 0); + if (cr0.lap) { + *flags |= ABS_LOWCORE_LAP_ON; + __ctl_clear_bit(0, 28); + } + *flags |= ABS_LOWCORE_UNMAPPED; + return lowcore_ptr[0]; + } +} + +void put_abs_lowcore(struct lowcore *lc, unsigned long flags) +{ + if (abs_lowcore_mapped) { + if (flags) + panic("Invalid mapped absolute lowcore release\n"); + } else { + if (smp_processor_id() != 0) + panic("Invalid mapped absolute lowcore access\n"); + if (!(flags & ABS_LOWCORE_UNMAPPED)) + panic("Invalid unmapped absolute lowcore release\n"); + if (flags & ABS_LOWCORE_LAP_ON) + __ctl_set_bit(0, 28); + if (flags & ABS_LOWCORE_IRQS_ON) + local_irq_enable(); + } + put_cpu(); +} diff --git a/arch/s390/kernel/crash_dump.c b/arch/s390/kernel/crash_dump.c index bad8f47fc5d696a7cba7de8651bfa8d2f3290136..dd74fe664ed12e0e90619c783b2b4c840390c474 100644 --- a/arch/s390/kernel/crash_dump.c +++ b/arch/s390/kernel/crash_dump.c @@ -21,6 +21,7 @@ #include #include #include +#include #define PTR_ADD(x, y) (((char *) (x)) + ((unsigned long) (y))) #define PTR_SUB(x, y) (((char *) (x)) - ((unsigned long) (y))) @@ -53,8 +54,6 @@ struct save_area { }; static LIST_HEAD(dump_save_areas); -static DEFINE_MUTEX(memcpy_real_mutex); -static char memcpy_real_buf[PAGE_SIZE]; /* * Allocate a save area @@ -116,27 +115,7 @@ void __init save_area_add_vxrs(struct save_area *sa, __vector128 *vxrs) memcpy(sa->vxrs_high, vxrs + 16, 16 * sizeof(__vector128)); } -static size_t copy_to_iter_real(struct iov_iter *iter, unsigned long src, size_t count) -{ - size_t len, copied, res = 0; - - mutex_lock(&memcpy_real_mutex); - while (count) { - len = min(PAGE_SIZE, count); - if (memcpy_real(memcpy_real_buf, src, len)) - break; - copied = copy_to_iter(memcpy_real_buf, len, iter); - count -= copied; - src += copied; - res += copied; - if (copied < len) - break; - } - mutex_unlock(&memcpy_real_mutex); - return res; -} - -size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count) +static size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count) { size_t len, copied, res = 0; @@ -156,7 +135,7 @@ size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count) } else { len = count; } - copied = copy_to_iter_real(iter, src, len); + copied = memcpy_real_iter(iter, src, len); } count -= copied; src += copied; @@ -167,6 +146,19 @@ size_t copy_oldmem_iter(struct iov_iter *iter, unsigned long src, size_t count) return res; } +int copy_oldmem_kernel(void *dst, unsigned long src, size_t count) +{ + struct iov_iter iter; + struct kvec kvec; + + kvec.iov_base = dst; + kvec.iov_len = count; + iov_iter_kvec(&iter, WRITE, &kvec, 1, count); + if (copy_oldmem_iter(&iter, src, count) < count) + return -EFAULT; + return 0; +} + /* * Copy one page from "oldmem" */ diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 4331c7e6e1c0320083735780b2b2b1b5a2075a37..d7a82066a638a942549e433c96338c2aa91c436c 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -250,7 +250,7 @@ static debug_info_t *debug_info_alloc(const char *name, int pages_per_area, rc->level = level; rc->buf_size = buf_size; rc->entry_size = sizeof(debug_entry_t) + buf_size; - strlcpy(rc->name, name, sizeof(rc->name)); + strscpy(rc->name, name, sizeof(rc->name)); memset(rc->views, 0, DEBUG_MAX_VIEWS * sizeof(struct debug_view *)); memset(rc->debugfs_entries, 0, DEBUG_MAX_VIEWS * sizeof(struct dentry *)); refcount_set(&(rc->ref_count), 0); diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 432c8c987256bbed390f5e82ffcf8da4131bd544..6030fdd6997bc224d58d509d7828512d1dcae650 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -267,7 +267,7 @@ char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; static void __init setup_boot_command_line(void) { /* copy arch command line */ - strlcpy(boot_command_line, early_command_line, COMMAND_LINE_SIZE); + strscpy(boot_command_line, early_command_line, COMMAND_LINE_SIZE); } static void __init check_image_bootable(void) diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 1cc85b8ff42e5d30a055f5fb1d286941f4b78f08..325cbf69ebbde014bdb5744dbac50ba1ac8192b9 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1642,12 +1643,16 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { unsigned long ipib = (unsigned long) reipl_block_actual; + struct lowcore *abs_lc; + unsigned long flags; unsigned int csum; csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); - put_abs_lowcore(ipib, ipib); - put_abs_lowcore(ipib_checksum, csum); + abs_lc = get_abs_lowcore(&flags); + abs_lc->ipib = ipib; + abs_lc->ipib_checksum = csum; + put_abs_lowcore(abs_lc, flags); dump_run(trigger); } diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index ab761c008f9813e8a4526b05e1947c244c1e3c6b..4579b42286d5fbd713f12dabbbfd654c7d94655e 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -222,13 +223,18 @@ void machine_kexec_cleanup(struct kimage *image) void arch_crash_save_vmcoreinfo(void) { + struct lowcore *abs_lc; + unsigned long flags; + VMCOREINFO_SYMBOL(lowcore_ptr); VMCOREINFO_SYMBOL(high_memory); VMCOREINFO_LENGTH(lowcore_ptr, NR_CPUS); vmcoreinfo_append_str("SAMODE31=%lx\n", __samode31); vmcoreinfo_append_str("EAMODE31=%lx\n", __eamode31); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - put_abs_lowcore(vmcore_info, paddr_vmcoreinfo_note()); + abs_lc = get_abs_lowcore(&flags); + abs_lc->vmcore_info = paddr_vmcoreinfo_note(); + put_abs_lowcore(abs_lc, flags); } void machine_shutdown(void) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 60ac66aab1635b7578be79b41ac6bd8ae2eded0e..31cb9b00a36bb41bcec1e9090066cb029fa08e9d 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -64,7 +64,7 @@ static inline unsigned long nmi_get_mcesa_size(void) * structure. The structure is required for machine check happening * early in the boot process. */ -static struct mcesa boot_mcesa __initdata __aligned(MCESA_MAX_SIZE); +static struct mcesa boot_mcesa __aligned(MCESA_MAX_SIZE); void __init nmi_alloc_mcesa_early(u64 *mcesad) { diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 1acc2e05d70f07bffebe98ea7605d0580596bf76..ec0bd9457e90ae5f852f3af6915aba46730e9442 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -13,8 +13,9 @@ #include #include #include -#include +#include #include +#include #include /* @@ -57,13 +58,16 @@ void os_info_entry_add(int nr, void *ptr, u64 size) */ void __init os_info_init(void) { - void *ptr = &os_info; + struct lowcore *abs_lc; + unsigned long flags; os_info.version_major = OS_INFO_VERSION_MAJOR; os_info.version_minor = OS_INFO_VERSION_MINOR; os_info.magic = OS_INFO_MAGIC; os_info.csum = os_info_csum(&os_info); - put_abs_lowcore(os_info, __pa(ptr)); + abs_lc = get_abs_lowcore(&flags); + abs_lc->os_info = __pa(&os_info); + put_abs_lowcore(abs_lc, flags); } #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index f7dd3c849e68ca6e2dcc396113d788b0d4714f85..f043a7ff220b72e5e5008e8db4cf4c8d80147e9c 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -664,6 +664,7 @@ static int cfdiag_push_sample(struct perf_event *event, raw.frag.data = cpuhw->stop; raw.size = raw.frag.size; data.raw = &raw; + data.sample_flags |= PERF_SAMPLE_RAW; } overflow = perf_event_overflow(event, &data, ®s); diff --git a/arch/s390/kernel/perf_pai_crypto.c b/arch/s390/kernel/perf_pai_crypto.c index b38b4ae01589b20d58fa91d78b05844977f2f5e1..6826e2a69a21678aae0bd23bebaf96de0306ef04 100644 --- a/arch/s390/kernel/perf_pai_crypto.c +++ b/arch/s390/kernel/perf_pai_crypto.c @@ -366,6 +366,7 @@ static int paicrypt_push_sample(void) raw.frag.data = cpump->save; raw.size = raw.frag.size; data.raw = &raw; + data.sample_flags |= PERF_SAMPLE_RAW; } overflow = perf_event_overflow(event, &data, ®s); diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c new file mode 100644 index 0000000000000000000000000000000000000000..d5c7c1e30c1790d247ab50d51d8474b4dd6b9dd2 --- /dev/null +++ b/arch/s390/kernel/perf_pai_ext.c @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Performance event support - Processor Activity Instrumentation Extension + * Facility + * + * Copyright IBM Corp. 2022 + * Author(s): Thomas Richter + */ +#define KMSG_COMPONENT "pai_ext" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define PAIE1_CB_SZ 0x200 /* Size of PAIE1 control block */ +#define PAIE1_CTRBLOCK_SZ 0x400 /* Size of PAIE1 counter blocks */ + +static debug_info_t *paiext_dbg; +static unsigned int paiext_cnt; /* Extracted with QPACI instruction */ + +enum paiext_mode { + PAI_MODE_NONE, + PAI_MODE_SAMPLING, + PAI_MODE_COUNTER, +}; + +struct pai_userdata { + u16 num; + u64 value; +} __packed; + +/* Create the PAI extension 1 control block area. + * The PAI extension control block 1 is pointed to by lowcore + * address 0x1508 for each CPU. This control block is 512 bytes in size + * and requires a 512 byte boundary alignment. + */ +struct paiext_cb { /* PAI extension 1 control block */ + u64 header; /* Not used */ + u64 reserved1; + u64 acc; /* Addr to analytics counter control block */ + u8 reserved2[488]; +} __packed; + +struct paiext_map { + unsigned long *area; /* Area for CPU to store counters */ + struct pai_userdata *save; /* Area to store non-zero counters */ + enum paiext_mode mode; /* Type of event */ + unsigned int active_events; /* # of PAI Extension users */ + unsigned int refcnt; + struct perf_event *event; /* Perf event for sampling */ + struct paiext_cb *paiext_cb; /* PAI extension control block area */ +}; + +struct paiext_mapptr { + struct paiext_map *mapptr; +}; + +static struct paiext_root { /* Anchor to per CPU data */ + int refcnt; /* Overall active events */ + struct paiext_mapptr __percpu *mapptr; +} paiext_root; + +/* Free per CPU data when the last event is removed. */ +static void paiext_root_free(void) +{ + if (!--paiext_root.refcnt) { + free_percpu(paiext_root.mapptr); + paiext_root.mapptr = NULL; + } +} + +/* On initialization of first event also allocate per CPU data dynamically. + * Start with an array of pointers, the array size is the maximum number of + * CPUs possible, which might be larger than the number of CPUs currently + * online. + */ +static int paiext_root_alloc(void) +{ + if (++paiext_root.refcnt == 1) { + /* The memory is already zeroed. */ + paiext_root.mapptr = alloc_percpu(struct paiext_mapptr); + if (!paiext_root.mapptr) { + /* Returing without refcnt adjustment is ok. The + * error code is handled by paiext_alloc() which + * decrements refcnt when an event can not be + * created. + */ + return -ENOMEM; + } + } + return 0; +} + +/* Protects against concurrent increment of sampler and counter member + * increments at the same time and prohibits concurrent execution of + * counting and sampling events. + * Ensures that analytics counter block is deallocated only when the + * sampling and counting on that cpu is zero. + * For details see paiext_alloc(). + */ +static DEFINE_MUTEX(paiext_reserve_mutex); + +/* Free all memory allocated for event counting/sampling setup */ +static void paiext_free(struct paiext_mapptr *mp) +{ + kfree(mp->mapptr->area); + kfree(mp->mapptr->paiext_cb); + kvfree(mp->mapptr->save); + kfree(mp->mapptr); + mp->mapptr = NULL; +} + +/* Release the PMU if event is the last perf event */ +static void paiext_event_destroy(struct perf_event *event) +{ + struct paiext_mapptr *mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); + struct paiext_map *cpump = mp->mapptr; + + mutex_lock(&paiext_reserve_mutex); + cpump->event = NULL; + if (!--cpump->refcnt) /* Last reference gone */ + paiext_free(mp); + paiext_root_free(); + mutex_unlock(&paiext_reserve_mutex); + debug_sprintf_event(paiext_dbg, 4, "%s cpu %d mapptr %p\n", __func__, + event->cpu, mp->mapptr); + +} + +/* Used to avoid races in checking concurrent access of counting and + * sampling for pai_extension events. + * + * Only one instance of event pai_ext/NNPA_ALL/ for sampling is + * allowed and when this event is running, no counting event is allowed. + * Several counting events are allowed in parallel, but no sampling event + * is allowed while one (or more) counting events are running. + * + * This function is called in process context and it is safe to block. + * When the event initialization functions fails, no other call back will + * be invoked. + * + * Allocate the memory for the event. + */ +static int paiext_alloc(struct perf_event_attr *a, struct perf_event *event) +{ + struct paiext_mapptr *mp; + struct paiext_map *cpump; + int rc; + + mutex_lock(&paiext_reserve_mutex); + + rc = paiext_root_alloc(); + if (rc) + goto unlock; + + mp = per_cpu_ptr(paiext_root.mapptr, event->cpu); + cpump = mp->mapptr; + if (!cpump) { /* Paiext_map allocated? */ + rc = -ENOMEM; + cpump = kzalloc(sizeof(*cpump), GFP_KERNEL); + if (!cpump) + goto unlock; + + /* Allocate memory for counter area and counter extraction. + * These are + * - a 512 byte block and requires 512 byte boundary alignment. + * - a 1KB byte block and requires 1KB boundary alignment. + * Only the first counting event has to allocate the area. + * + * Note: This works with commit 59bb47985c1d by default. + * Backporting this to kernels without this commit might + * need adjustment. + */ + mp->mapptr = cpump; + cpump->area = kzalloc(PAIE1_CTRBLOCK_SZ, GFP_KERNEL); + cpump->paiext_cb = kzalloc(PAIE1_CB_SZ, GFP_KERNEL); + cpump->save = kvmalloc_array(paiext_cnt + 1, + sizeof(struct pai_userdata), + GFP_KERNEL); + if (!cpump->save || !cpump->area || !cpump->paiext_cb) { + paiext_free(mp); + goto unlock; + } + cpump->mode = a->sample_period ? PAI_MODE_SAMPLING + : PAI_MODE_COUNTER; + } else { + /* Multiple invocation, check whats active. + * Supported are multiple counter events or only one sampling + * event concurrently at any one time. + */ + if (cpump->mode == PAI_MODE_SAMPLING || + (cpump->mode == PAI_MODE_COUNTER && a->sample_period)) { + rc = -EBUSY; + goto unlock; + } + } + + rc = 0; + cpump->event = event; + ++cpump->refcnt; + +unlock: + if (rc) { + /* Error in allocation of event, decrement anchor. Since + * the event in not created, its destroy() function is never + * invoked. Adjust the reference counter for the anchor. + */ + paiext_root_free(); + } + mutex_unlock(&paiext_reserve_mutex); + /* If rc is non-zero, no increment of counter/sampler was done. */ + return rc; +} + +/* The PAI extension 1 control block supports up to 128 entries. Return + * the index within PAIE1_CB given the event number. Also validate event + * number. + */ +static int paiext_event_valid(struct perf_event *event) +{ + u64 cfg = event->attr.config; + + if (cfg >= PAI_NNPA_BASE && cfg <= PAI_NNPA_BASE + paiext_cnt) { + /* Offset NNPA in paiext_cb */ + event->hw.config_base = offsetof(struct paiext_cb, acc); + return 0; + } + return -EINVAL; +} + +/* Might be called on different CPU than the one the event is intended for. */ +static int paiext_event_init(struct perf_event *event) +{ + struct perf_event_attr *a = &event->attr; + int rc; + + /* PMU pai_ext registered as PERF_TYPE_RAW, check event type */ + if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type) + return -ENOENT; + /* PAI extension event must be valid and in supported range */ + rc = paiext_event_valid(event); + if (rc) + return rc; + /* Allow only CPU wide operation, no process context for now. */ + if (event->hw.target || event->cpu == -1) + return -ENOENT; + /* Allow only event NNPA_ALL for sampling. */ + if (a->sample_period && a->config != PAI_NNPA_BASE) + return -EINVAL; + /* Prohibit exclude_user event selection */ + if (a->exclude_user) + return -EINVAL; + + rc = paiext_alloc(a, event); + if (rc) + return rc; + event->hw.last_tag = 0; + event->destroy = paiext_event_destroy; + + if (a->sample_period) { + a->sample_period = 1; + a->freq = 0; + /* Register for paicrypt_sched_task() to be called */ + event->attach_state |= PERF_ATTACH_SCHED_CB; + /* Add raw data which are the memory mapped counters */ + a->sample_type |= PERF_SAMPLE_RAW; + /* Turn off inheritance */ + a->inherit = 0; + } + + return 0; +} + +static u64 paiext_getctr(struct paiext_map *cpump, int nr) +{ + return cpump->area[nr]; +} + +/* Read the counter values. Return value from location in buffer. For event + * NNPA_ALL sum up all events. + */ +static u64 paiext_getdata(struct perf_event *event) +{ + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; + u64 sum = 0; + int i; + + if (event->attr.config != PAI_NNPA_BASE) + return paiext_getctr(cpump, event->attr.config - PAI_NNPA_BASE); + + for (i = 1; i <= paiext_cnt; i++) + sum += paiext_getctr(cpump, i); + + return sum; +} + +static u64 paiext_getall(struct perf_event *event) +{ + return paiext_getdata(event); +} + +static void paiext_read(struct perf_event *event) +{ + u64 prev, new, delta; + + prev = local64_read(&event->hw.prev_count); + new = paiext_getall(event); + local64_set(&event->hw.prev_count, new); + delta = new - prev; + local64_add(delta, &event->count); +} + +static void paiext_start(struct perf_event *event, int flags) +{ + u64 sum; + + if (event->hw.last_tag) + return; + event->hw.last_tag = 1; + sum = paiext_getall(event); /* Get current value */ + local64_set(&event->hw.prev_count, sum); + local64_set(&event->count, 0); +} + +static int paiext_add(struct perf_event *event, int flags) +{ + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; + struct paiext_cb *pcb = cpump->paiext_cb; + + if (++cpump->active_events == 1) { + S390_lowcore.aicd = virt_to_phys(cpump->paiext_cb); + pcb->acc = virt_to_phys(cpump->area) | 0x1; + /* Enable CPU instruction lookup for PAIE1 control block */ + __ctl_set_bit(0, 49); + debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", + __func__, S390_lowcore.aicd, pcb->acc); + } + if (flags & PERF_EF_START && !event->attr.sample_period) { + /* Only counting needs initial counter value */ + paiext_start(event, PERF_EF_RELOAD); + } + event->hw.state = 0; + if (event->attr.sample_period) { + cpump->event = event; + perf_sched_cb_inc(event->pmu); + } + return 0; +} + +static void paiext_stop(struct perf_event *event, int flags) +{ + paiext_read(event); + event->hw.state = PERF_HES_STOPPED; +} + +static void paiext_del(struct perf_event *event, int flags) +{ + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; + struct paiext_cb *pcb = cpump->paiext_cb; + + if (event->attr.sample_period) + perf_sched_cb_dec(event->pmu); + if (!event->attr.sample_period) { + /* Only counting needs to read counter */ + paiext_stop(event, PERF_EF_UPDATE); + } + if (--cpump->active_events == 0) { + /* Disable CPU instruction lookup for PAIE1 control block */ + __ctl_clear_bit(0, 49); + pcb->acc = 0; + S390_lowcore.aicd = 0; + debug_sprintf_event(paiext_dbg, 4, "%s 1508 %llx acc %llx\n", + __func__, S390_lowcore.aicd, pcb->acc); + } +} + +/* Create raw data and save it in buffer. Returns number of bytes copied. + * Saves only positive counter entries of the form + * 2 bytes: Number of counter + * 8 bytes: Value of counter + */ +static size_t paiext_copy(struct paiext_map *cpump) +{ + struct pai_userdata *userdata = cpump->save; + int i, outidx = 0; + + for (i = 1; i <= paiext_cnt; i++) { + u64 val = paiext_getctr(cpump, i); + + if (val) { + userdata[outidx].num = i; + userdata[outidx].value = val; + outidx++; + } + } + return outidx * sizeof(*userdata); +} + +/* Write sample when one or more counters values are nonzero. + * + * Note: The function paiext_sched_task() and paiext_push_sample() are not + * invoked after function paiext_del() has been called because of function + * perf_sched_cb_dec(). + * The function paiext_sched_task() and paiext_push_sample() are only + * called when sampling is active. Function perf_sched_cb_inc() + * has been invoked to install function paiext_sched_task() as call back + * to run at context switch time (see paiext_add()). + * + * This causes function perf_event_context_sched_out() and + * perf_event_context_sched_in() to check whether the PMU has installed an + * sched_task() callback. That callback is not active after paiext_del() + * returns and has deleted the event on that CPU. + */ +static int paiext_push_sample(void) +{ + struct paiext_mapptr *mp = this_cpu_ptr(paiext_root.mapptr); + struct paiext_map *cpump = mp->mapptr; + struct perf_event *event = cpump->event; + struct perf_sample_data data; + struct perf_raw_record raw; + struct pt_regs regs; + size_t rawsize; + int overflow; + + rawsize = paiext_copy(cpump); + if (!rawsize) /* No incremented counters */ + return 0; + + /* Setup perf sample */ + memset(®s, 0, sizeof(regs)); + memset(&raw, 0, sizeof(raw)); + memset(&data, 0, sizeof(data)); + perf_sample_data_init(&data, 0, event->hw.last_period); + if (event->attr.sample_type & PERF_SAMPLE_TID) { + data.tid_entry.pid = task_tgid_nr(current); + data.tid_entry.tid = task_pid_nr(current); + } + if (event->attr.sample_type & PERF_SAMPLE_TIME) + data.time = event->clock(); + if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) + data.id = event->id; + if (event->attr.sample_type & PERF_SAMPLE_CPU) + data.cpu_entry.cpu = smp_processor_id(); + if (event->attr.sample_type & PERF_SAMPLE_RAW) { + raw.frag.size = rawsize; + raw.frag.data = cpump->save; + raw.size = raw.frag.size; + data.raw = &raw; + } + + overflow = perf_event_overflow(event, &data, ®s); + perf_event_update_userpage(event); + /* Clear lowcore area after read */ + memset(cpump->area, 0, PAIE1_CTRBLOCK_SZ); + return overflow; +} + +/* Called on schedule-in and schedule-out. No access to event structure, + * but for sampling only event NNPA_ALL is allowed. + */ +static void paiext_sched_task(struct perf_event_context *ctx, bool sched_in) +{ + /* We started with a clean page on event installation. So read out + * results on schedule_out and if page was dirty, clear values. + */ + if (!sched_in) + paiext_push_sample(); +} + +/* Attribute definitions for pai extension1 interface. As with other CPU + * Measurement Facilities, there is one attribute per mapped counter. + * The number of mapped counters may vary per machine generation. Use + * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction + * to determine the number of mapped counters. The instructions returns + * a positive number, which is the highest number of supported counters. + * All counters less than this number are also supported, there are no + * holes. A returned number of zero means no support for mapped counters. + * + * The identification of the counter is a unique number. The chosen range + * is 0x1800 + offset in mapped kernel page. + * All CPU Measurement Facility counters identifiers must be unique and + * the numbers from 0 to 496 are already used for the CPU Measurement + * Counter facility. Number 0x1000 to 0x103e are used for PAI cryptography + * counters. + * Numbers 0xb0000, 0xbc000 and 0xbd000 are already + * used for the CPU Measurement Sampling facility. + */ +PMU_FORMAT_ATTR(event, "config:0-63"); + +static struct attribute *paiext_format_attr[] = { + &format_attr_event.attr, + NULL, +}; + +static struct attribute_group paiext_events_group = { + .name = "events", + .attrs = NULL, /* Filled in attr_event_init() */ +}; + +static struct attribute_group paiext_format_group = { + .name = "format", + .attrs = paiext_format_attr, +}; + +static const struct attribute_group *paiext_attr_groups[] = { + &paiext_events_group, + &paiext_format_group, + NULL, +}; + +/* Performance monitoring unit for mapped counters */ +static struct pmu paiext = { + .task_ctx_nr = perf_invalid_context, + .event_init = paiext_event_init, + .add = paiext_add, + .del = paiext_del, + .start = paiext_start, + .stop = paiext_stop, + .read = paiext_read, + .sched_task = paiext_sched_task, + .attr_groups = paiext_attr_groups, +}; + +/* List of symbolic PAI extension 1 NNPA counter names. */ +static const char * const paiext_ctrnames[] = { + [0] = "NNPA_ALL", + [1] = "NNPA_ADD", + [2] = "NNPA_SUB", + [3] = "NNPA_MUL", + [4] = "NNPA_DIV", + [5] = "NNPA_MIN", + [6] = "NNPA_MAX", + [7] = "NNPA_LOG", + [8] = "NNPA_EXP", + [9] = "NNPA_IBM_RESERVED_9", + [10] = "NNPA_RELU", + [11] = "NNPA_TANH", + [12] = "NNPA_SIGMOID", + [13] = "NNPA_SOFTMAX", + [14] = "NNPA_BATCHNORM", + [15] = "NNPA_MAXPOOL2D", + [16] = "NNPA_AVGPOOL2D", + [17] = "NNPA_LSTMACT", + [18] = "NNPA_GRUACT", + [19] = "NNPA_CONVOLUTION", + [20] = "NNPA_MATMUL_OP", + [21] = "NNPA_MATMUL_OP_BCAST23", + [22] = "NNPA_SMALLBATCH", + [23] = "NNPA_LARGEDIM", + [24] = "NNPA_SMALLTENSOR", + [25] = "NNPA_1MFRAME", + [26] = "NNPA_2GFRAME", + [27] = "NNPA_ACCESSEXCEPT", +}; + +static void __init attr_event_free(struct attribute **attrs, int num) +{ + struct perf_pmu_events_attr *pa; + struct device_attribute *dap; + int i; + + for (i = 0; i < num; i++) { + dap = container_of(attrs[i], struct device_attribute, attr); + pa = container_of(dap, struct perf_pmu_events_attr, attr); + kfree(pa); + } + kfree(attrs); +} + +static int __init attr_event_init_one(struct attribute **attrs, int num) +{ + struct perf_pmu_events_attr *pa; + + pa = kzalloc(sizeof(*pa), GFP_KERNEL); + if (!pa) + return -ENOMEM; + + sysfs_attr_init(&pa->attr.attr); + pa->id = PAI_NNPA_BASE + num; + pa->attr.attr.name = paiext_ctrnames[num]; + pa->attr.attr.mode = 0444; + pa->attr.show = cpumf_events_sysfs_show; + pa->attr.store = NULL; + attrs[num] = &pa->attr.attr; + return 0; +} + +/* Create PMU sysfs event attributes on the fly. */ +static int __init attr_event_init(void) +{ + struct attribute **attrs; + int ret, i; + + attrs = kmalloc_array(ARRAY_SIZE(paiext_ctrnames) + 1, sizeof(*attrs), + GFP_KERNEL); + if (!attrs) + return -ENOMEM; + for (i = 0; i < ARRAY_SIZE(paiext_ctrnames); i++) { + ret = attr_event_init_one(attrs, i); + if (ret) { + attr_event_free(attrs, i - 1); + return ret; + } + } + attrs[i] = NULL; + paiext_events_group.attrs = attrs; + return 0; +} + +static int __init paiext_init(void) +{ + struct qpaci_info_block ib; + int rc = -ENOMEM; + + if (!test_facility(197)) + return 0; + + qpaci(&ib); + paiext_cnt = ib.num_nnpa; + if (paiext_cnt >= PAI_NNPA_MAXCTR) + paiext_cnt = PAI_NNPA_MAXCTR; + if (!paiext_cnt) + return 0; + + rc = attr_event_init(); + if (rc) { + pr_err("Creation of PMU " KMSG_COMPONENT " /sysfs failed\n"); + return rc; + } + + /* Setup s390dbf facility */ + paiext_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128); + if (!paiext_dbg) { + pr_err("Registration of s390dbf " KMSG_COMPONENT " failed\n"); + rc = -ENOMEM; + goto out_init; + } + debug_register_view(paiext_dbg, &debug_sprintf_view); + + rc = perf_pmu_register(&paiext, KMSG_COMPONENT, -1); + if (rc) { + pr_err("Registration of " KMSG_COMPONENT " PMU failed with " + "rc=%i\n", rc); + goto out_pmu; + } + + return 0; + +out_pmu: + debug_unregister_view(paiext_dbg, &debug_sprintf_view); + debug_unregister(paiext_dbg); +out_init: + attr_event_free(paiext_events_group.attrs, + ARRAY_SIZE(paiext_ctrnames) + 1); + return rc; +} + +device_initcall(paiext_init); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 89949b9f3cf88eb12af09377eda67030f9a5f126..42af4b3aa02b85d0d117f7931cd8eb28f0784004 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -91,6 +91,18 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) memcpy(dst, src, arch_task_struct_size); dst->thread.fpu.regs = dst->thread.fpu.fprs; + + /* + * Don't transfer over the runtime instrumentation or the guarded + * storage control block pointers. These fields are cleared here instead + * of in copy_thread() to avoid premature freeing of associated memory + * on fork() failure. Wait to clear the RI flag because ->stack still + * refers to the source thread. + */ + dst->thread.ri_cb = NULL; + dst->thread.gs_cb = NULL; + dst->thread.gs_bc_cb = NULL; + return 0; } @@ -150,13 +162,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) frame->childregs.flags = 0; if (new_stackp) frame->childregs.gprs[15] = new_stackp; - - /* Don't copy runtime instrumentation info */ - p->thread.ri_cb = NULL; + /* + * Clear the runtime instrumentation flag after the above childregs + * copy. The CB pointer was already cleared in arch_dup_task_struct(). + */ frame->childregs.psw.mask &= ~PSW_MASK_RI; - /* Don't copy guarded storage control block */ - p->thread.gs_cb = NULL; - p->thread.gs_bc_cb = NULL; /* Set a new TLS ? */ if (clone_flags & CLONE_SETTLS) { @@ -214,13 +224,13 @@ unsigned long __get_wchan(struct task_struct *p) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() & ~PAGE_MASK; + sp -= prandom_u32_max(PAGE_SIZE); return sp & ~0xf; } static inline unsigned long brk_rnd(void) { - return (get_random_int() & BRK_RND_MASK) << PAGE_SHIFT; + return (get_random_u16() & BRK_RND_MASK) << PAGE_SHIFT; } unsigned long arch_randomize_brk(struct mm_struct *mm) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index ed4fbbbdd1b0787c405d4fe2ebee9042e0f51bac..ab19ddb09d65eda37b7214760fd9d549ef9d063e 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -58,7 +58,7 @@ #include #include #include -#include +#include #include #include #include @@ -74,6 +74,7 @@ #include #include #include +#include #include #include #include "entry.h" @@ -395,6 +396,7 @@ void __init arch_call_rest_init(void) { unsigned long stack; + smp_reinit_ipl_cpu(); stack = stack_alloc(); if (!stack) panic("Couldn't allocate kernel stack"); @@ -411,8 +413,9 @@ void __init arch_call_rest_init(void) static void __init setup_lowcore_dat_off(void) { unsigned long int_psw_mask = PSW_KERNEL_BITS; + struct lowcore *abs_lc, *lc; unsigned long mcck_stack; - struct lowcore *lc; + unsigned long flags; if (IS_ENABLED(CONFIG_KASAN)) int_psw_mask |= PSW_MASK_DAT; @@ -474,11 +477,14 @@ static void __init setup_lowcore_dat_off(void) lc->restart_data = 0; lc->restart_source = -1U; - put_abs_lowcore(restart_stack, lc->restart_stack); - put_abs_lowcore(restart_fn, lc->restart_fn); - put_abs_lowcore(restart_data, lc->restart_data); - put_abs_lowcore(restart_source, lc->restart_source); - put_abs_lowcore(restart_psw, lc->restart_psw); + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_stack = lc->restart_stack; + abs_lc->restart_fn = lc->restart_fn; + abs_lc->restart_data = lc->restart_data; + abs_lc->restart_source = lc->restart_source; + abs_lc->restart_psw = lc->restart_psw; + abs_lc->mcesad = lc->mcesad; + put_abs_lowcore(abs_lc, flags); mcck_stack = (unsigned long)memblock_alloc(THREAD_SIZE, THREAD_SIZE); if (!mcck_stack) @@ -499,20 +505,25 @@ static void __init setup_lowcore_dat_off(void) static void __init setup_lowcore_dat_on(void) { - struct lowcore *lc = lowcore_ptr[0]; - int cr; + struct lowcore *abs_lc; + unsigned long flags; __ctl_clear_bit(0, 28); S390_lowcore.external_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.svc_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.program_new_psw.mask |= PSW_MASK_DAT; S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; - __ctl_store(S390_lowcore.cregs_save_area, 0, 15); __ctl_set_bit(0, 28); - put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); - put_abs_lowcore(program_new_psw, lc->program_new_psw); - for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) - put_abs_lowcore(cregs_save_area[cr], lc->cregs_save_area[cr]); + __ctl_store(S390_lowcore.cregs_save_area, 0, 15); + if (abs_lowcore_map(0, lowcore_ptr[0], true)) + panic("Couldn't setup absolute lowcore"); + abs_lowcore_mapped = true; + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_flags = RESTART_FLAG_CTLREGS; + abs_lc->program_new_psw = S390_lowcore.program_new_psw; + memcpy(abs_lc->cregs_save_area, S390_lowcore.cregs_save_area, + sizeof(abs_lc->cregs_save_area)); + put_abs_lowcore(abs_lc, flags); } static struct resource code_resource = { @@ -1018,10 +1029,10 @@ void __init setup_arch(char **cmdline_p) reserve_crashkernel(); #ifdef CONFIG_CRASH_DUMP /* - * Be aware that smp_save_dump_cpus() triggers a system reset. + * Be aware that smp_save_dump_secondary_cpus() triggers a system reset. * Therefore CPU and device initialization should be done afterwards. */ - smp_save_dump_cpus(); + smp_save_dump_secondary_cpus(); #endif setup_resources(); @@ -1040,12 +1051,15 @@ void __init setup_arch(char **cmdline_p) * Create kernel page tables and switch to virtual addressing. */ paging_init(); - + memcpy_real_init(); /* * After paging_init created the kernel page table, the new PSWs * in lowcore can now run with DAT enabled. */ setup_lowcore_dat_on(); +#ifdef CONFIG_CRASH_DUMP + smp_save_dump_ipl_cpu(); +#endif /* Setup default console */ conmode_default(); diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 30c91d56593374fcb8b43a1be5a5fe6eefa00a3c..0031325ce4bc9f1656c16151d0eb5cdef41593de 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -45,7 +45,7 @@ #include #include #include -#include +#include #include #include #include @@ -55,6 +55,7 @@ #include #include #include +#include #include "entry.h" enum { @@ -212,10 +213,14 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->preempt_count = PREEMPT_DISABLED; if (nmi_alloc_mcesa(&lc->mcesad)) goto out; + if (abs_lowcore_map(cpu, lc, true)) + goto out_mcesa; lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc)); return 0; +out_mcesa: + nmi_free_mcesa(&lc->mcesad); out: stack_free(mcck_stack); stack_free(async_stack); @@ -237,6 +242,7 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) mcck_stack = lc->mcck_stack - STACK_INIT_OFFSET; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[cpu] = NULL; + abs_lowcore_unmap(cpu); nmi_free_mcesa(&lc->mcesad); stack_free(async_stack); stack_free(mcck_stack); @@ -315,9 +321,12 @@ static void pcpu_delegate(struct pcpu *pcpu, pcpu_delegate_fn *func, void *data, unsigned long stack) { - struct lowcore *lc = lowcore_ptr[pcpu - pcpu_devices]; - unsigned int source_cpu = stap(); + struct lowcore *lc, *abs_lc; + unsigned int source_cpu; + unsigned long flags; + lc = lowcore_ptr[pcpu - pcpu_devices]; + source_cpu = stap(); __load_psw_mask(PSW_KERNEL_BITS | PSW_MASK_DAT); if (pcpu->address == source_cpu) { call_on_stack(2, stack, void, __pcpu_delegate, @@ -332,10 +341,12 @@ static void pcpu_delegate(struct pcpu *pcpu, lc->restart_data = (unsigned long)data; lc->restart_source = source_cpu; } else { - put_abs_lowcore(restart_stack, stack); - put_abs_lowcore(restart_fn, (unsigned long)func); - put_abs_lowcore(restart_data, (unsigned long)data); - put_abs_lowcore(restart_source, source_cpu); + abs_lc = get_abs_lowcore(&flags); + abs_lc->restart_stack = stack; + abs_lc->restart_fn = (unsigned long)func; + abs_lc->restart_data = (unsigned long)data; + abs_lc->restart_source = source_cpu; + put_abs_lowcore(abs_lc, flags); } __bpon(); asm volatile( @@ -581,6 +592,8 @@ static DEFINE_SPINLOCK(ctl_lock); void smp_ctl_set_clear_bit(int cr, int bit, bool set) { struct ec_creg_mask_parms parms = { .cr = cr, }; + struct lowcore *abs_lc; + unsigned long flags; u64 ctlreg; if (set) { @@ -591,9 +604,11 @@ void smp_ctl_set_clear_bit(int cr, int bit, bool set) parms.andval = ~(1UL << bit); } spin_lock(&ctl_lock); - get_abs_lowcore(ctlreg, cregs_save_area[cr]); + abs_lc = get_abs_lowcore(&flags); + ctlreg = abs_lc->cregs_save_area[cr]; ctlreg = (ctlreg & parms.andval) | parms.orval; - put_abs_lowcore(cregs_save_area[cr], ctlreg); + abs_lc->cregs_save_area[cr] = ctlreg; + put_abs_lowcore(abs_lc, flags); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } @@ -650,35 +665,36 @@ int smp_store_status(int cpu) * This case does not exist for s390 anymore, setup_arch explicitly * deactivates the elfcorehdr= kernel parameter */ -static __init void smp_save_cpu_vxrs(struct save_area *sa, u16 addr, - bool is_boot_cpu, __vector128 *vxrs) +static bool dump_available(void) { - if (is_boot_cpu) - vxrs = boot_cpu_vector_save_area; - else - __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(vxrs)); - save_area_add_vxrs(sa, vxrs); + return oldmem_data.start || is_ipl_type_dump(); } -static __init void smp_save_cpu_regs(struct save_area *sa, u16 addr, - bool is_boot_cpu, void *regs) +void __init smp_save_dump_ipl_cpu(void) { - if (is_boot_cpu) - copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512); - else - __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(regs)); + struct save_area *sa; + void *regs; + + if (!dump_available()) + return; + sa = save_area_alloc(true); + regs = memblock_alloc(512, 8); + if (!sa || !regs) + panic("could not allocate memory for boot CPU save area\n"); + copy_oldmem_kernel(regs, __LC_FPREGS_SAVE_AREA, 512); save_area_add_regs(sa, regs); + memblock_free(regs, 512); + if (MACHINE_HAS_VX) + save_area_add_vxrs(sa, boot_cpu_vector_save_area); } -void __init smp_save_dump_cpus(void) +void __init smp_save_dump_secondary_cpus(void) { int addr, boot_cpu_addr, max_cpu_addr; struct save_area *sa; - bool is_boot_cpu; void *page; - if (!(oldmem_data.start || is_ipl_type_dump())) - /* No previous system present, normal boot. */ + if (!dump_available()) return; /* Allocate a page as dumping area for the store status sigps */ page = memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); @@ -691,26 +707,20 @@ void __init smp_save_dump_cpus(void) boot_cpu_addr = stap(); max_cpu_addr = SCLP_MAX_CORES << sclp.mtid_prev; for (addr = 0; addr <= max_cpu_addr; addr++) { + if (addr == boot_cpu_addr) + continue; if (__pcpu_sigp_relax(addr, SIGP_SENSE, 0) == SIGP_CC_NOT_OPERATIONAL) continue; - is_boot_cpu = (addr == boot_cpu_addr); - /* Allocate save area */ - sa = save_area_alloc(is_boot_cpu); + sa = save_area_alloc(false); if (!sa) panic("could not allocate memory for save area\n"); - if (MACHINE_HAS_VX) - /* Get the vector registers */ - smp_save_cpu_vxrs(sa, addr, is_boot_cpu, page); - /* - * For a zfcp/nvme dump OLDMEM_BASE == NULL and the registers - * of the boot CPU are stored in the HSA. To retrieve - * these registers an SCLP request is required which is - * done by drivers/s390/char/zcore.c:init_cpu_info() - */ - if (!is_boot_cpu || oldmem_data.start) - /* Get the CPU registers */ - smp_save_cpu_regs(sa, addr, is_boot_cpu, page); + __pcpu_sigp_relax(addr, SIGP_STORE_STATUS_AT_ADDRESS, __pa(page)); + save_area_add_regs(sa, page); + if (MACHINE_HAS_VX) { + __pcpu_sigp_relax(addr, SIGP_STORE_ADDITIONAL_STATUS, __pa(page)); + save_area_add_vxrs(sa, page); + } } memblock_free(page, PAGE_SIZE); diag_amode31_ops.diag308_reset(); @@ -1256,7 +1266,7 @@ static __always_inline void set_new_lowcore(struct lowcore *lc) : "memory", "cc"); } -static int __init smp_reinit_ipl_cpu(void) +int __init smp_reinit_ipl_cpu(void) { unsigned long async_stack, nodat_stack, mcck_stack; struct lowcore *lc, *lc_ipl; @@ -1281,6 +1291,8 @@ static int __init smp_reinit_ipl_cpu(void) __ctl_clear_bit(0, 28); /* disable lowcore protection */ S390_lowcore.mcesad = mcesad; __ctl_load(cr0, 0, 0); + if (abs_lowcore_map(0, lc, false)) + panic("Couldn't remap absolute lowcore"); lowcore_ptr[0] = lc; local_mcck_enable(); local_irq_restore(flags); @@ -1291,4 +1303,3 @@ static int __init smp_reinit_ipl_cpu(void) return 0; } -early_initcall(smp_reinit_ipl_cpu); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 5075cde77b29202977f7aad304267f49b6aea33e..3105ca5bd4701d14831078559431aaed0f70d1d1 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -69,10 +69,11 @@ static struct page *find_timens_vvar_page(struct vm_area_struct *vma) int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long size = vma->vm_end - vma->vm_start; if (!vma_is_special_mapping(vma, &vvar_mapping)) @@ -226,7 +227,7 @@ static unsigned long vdso_addr(unsigned long start, unsigned long len) end -= len; if (end > start) { - offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1); + offset = prandom_u32_max(((end - start) >> PAGE_SHIFT) + 1); addr = start + (offset << PAGE_SHIFT); } else { addr = start; diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 2e526f11b91e2e734d90d2754f58eac6bf7e2416..5ea3830af0ccff11c6a18a06a2392cf690742b34 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -131,6 +131,7 @@ SECTIONS /* * Table with the patch locations to undo expolines */ + . = ALIGN(4); .nospec_call_table : { __nospec_call_start = . ; *(.s390_indirect*) diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index 082ec5f2c3a542dbeb48d1f7358bfb1b4545ad23..0243b6e38d364cf005d99803742d55d4c73631b0 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -489,6 +489,8 @@ enum prot_type { PROT_TYPE_ALC = 2, PROT_TYPE_DAT = 3, PROT_TYPE_IEP = 4, + /* Dummy value for passing an initialized value when code != PGM_PROTECTION */ + PROT_NONE, }; static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, u8 ar, @@ -504,6 +506,10 @@ static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, switch (code) { case PGM_PROTECTION: switch (prot) { + case PROT_NONE: + /* We should never get here, acts like termination */ + WARN_ON_ONCE(1); + break; case PROT_TYPE_IEP: tec->b61 = 1; fallthrough; @@ -968,8 +974,10 @@ static int guest_range_to_gpas(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, return rc; } else { gpa = kvm_s390_real_to_abs(vcpu, ga); - if (kvm_is_error_gpa(vcpu->kvm, gpa)) + if (kvm_is_error_gpa(vcpu->kvm, gpa)) { rc = PGM_ADDRESSING; + prot = PROT_NONE; + } } if (rc) return trans_exc(vcpu, rc, ga, ar, mode, prot); @@ -1112,8 +1120,6 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, if (rc == PGM_PROTECTION && try_storage_prot_override) rc = access_guest_page_with_key(vcpu->kvm, mode, gpas[idx], data, fragment_len, PAGE_SPO_ACC); - if (rc == PGM_PROTECTION) - prot = PROT_TYPE_KEYC; if (rc) break; len -= fragment_len; @@ -1123,6 +1129,10 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, if (rc > 0) { bool terminate = (mode == GACC_STORE) && (idx > 0); + if (rc == PGM_PROTECTION) + prot = PROT_TYPE_KEYC; + else + prot = PROT_NONE; rc = trans_exc_ending(vcpu, rc, ga, ar, mode, prot, terminate); } out_unlock: diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index b9c944b262c7955c3a7ce0d51fbf352867d065e0..ab569faf0df241a8bbc03a05e9aa28870c22b61c 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3324,7 +3324,7 @@ static void aen_host_forward(unsigned long si) if (gaite->count == 0) return; if (gaite->aisb != 0) - set_bit_inv(gaite->aisbo, (unsigned long *)gaite->aisb); + set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb)); kvm = kvm_s390_pci_si_to_kvm(aift, si); if (!kvm) diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index edfd4bbd0cbae345c8375b35e9ffa2b9e368bdf7..45d4b8182b0734c27e70e8582d043b8ffba7b554 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -505,7 +505,7 @@ int kvm_arch_init(void *opaque) goto out; } - if (kvm_s390_pci_interp_allowed()) { + if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) { rc = kvm_s390_pci_init(); if (rc) { pr_err("Unable to allocate AIFT for PCI\n"); @@ -527,7 +527,7 @@ out: void kvm_arch_exit(void) { kvm_s390_gib_destroy(); - if (kvm_s390_pci_interp_allowed()) + if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) kvm_s390_pci_exit(); debug_unregister(kvm_s390_dbf); debug_unregister(kvm_s390_dbf_uv); @@ -4343,8 +4343,6 @@ retry: goto retry; } - /* nothing to do, just clear the request */ - kvm_clear_request(KVM_REQ_UNHALT, vcpu); /* we left the vsie handler, nothing to do, just clear the request */ kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu); diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 4946fb7757d6bbe44ca6a013a50ba0854990eda6..c50c1645c0aeca81dab6dc2aa9b61b91604adda2 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -58,7 +58,7 @@ static int zpci_setup_aipb(u8 nisc) if (!zpci_aipb) return -ENOMEM; - aift->sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC, 0); + aift->sbv = airq_iv_create(ZPCI_NR_DEVICES, AIRQ_IV_ALLOC, NULL); if (!aift->sbv) { rc = -ENOMEM; goto free_aipb; @@ -71,7 +71,7 @@ static int zpci_setup_aipb(u8 nisc) rc = -ENOMEM; goto free_sbv; } - aift->gait = (struct zpci_gaite *)page_to_phys(page); + aift->gait = (struct zpci_gaite *)page_to_virt(page); zpci_aipb->aipb.faisb = virt_to_phys(aift->sbv->vector); zpci_aipb->aipb.gait = virt_to_phys(aift->gait); @@ -373,7 +373,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) gaite->gisc = 0; gaite->aisbo = 0; gaite->gisa = 0; - aift->kzdev[zdev->aisb] = 0; + aift->kzdev[zdev->aisb] = NULL; /* Clear zdev info */ airq_iv_free_bit(aift->sbv, zdev->aisb); airq_iv_release(zdev->aibv); @@ -431,8 +431,9 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev) * available, enable them and let userspace indicate whether or not they will * be used (specify SHM bit to disable). */ -int kvm_s390_pci_register_kvm(struct zpci_dev *zdev, struct kvm *kvm) +static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm) { + struct zpci_dev *zdev = opaque; int rc; if (!zdev) @@ -510,10 +511,10 @@ err: kvm_put_kvm(kvm); return rc; } -EXPORT_SYMBOL_GPL(kvm_s390_pci_register_kvm); -void kvm_s390_pci_unregister_kvm(struct zpci_dev *zdev) +static void kvm_s390_pci_unregister_kvm(void *opaque) { + struct zpci_dev *zdev = opaque; struct kvm *kvm; if (!zdev) @@ -566,7 +567,6 @@ out: kvm_put_kvm(kvm); } -EXPORT_SYMBOL_GPL(kvm_s390_pci_unregister_kvm); void kvm_s390_pci_init_list(struct kvm *kvm) { @@ -672,6 +672,12 @@ out: int kvm_s390_pci_init(void) { + zpci_kvm_hook.kvm_register = kvm_s390_pci_register_kvm; + zpci_kvm_hook.kvm_unregister = kvm_s390_pci_unregister_kvm; + + if (!kvm_s390_pci_interp_allowed()) + return 0; + aift = kzalloc(sizeof(struct zpci_aift), GFP_KERNEL); if (!aift) return -ENOMEM; @@ -684,6 +690,12 @@ int kvm_s390_pci_init(void) void kvm_s390_pci_exit(void) { + zpci_kvm_hook.kvm_register = NULL; + zpci_kvm_hook.kvm_unregister = NULL; + + if (!kvm_s390_pci_interp_allowed()) + return; + mutex_destroy(&aift->aift_lock); kfree(aift); diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index 3a3606c3a0fe3d5d558032138d7fd76553387e21..486d06ef563f4f65572dda0668b67b976c7f7932 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -46,9 +46,9 @@ extern struct zpci_aift *aift; static inline struct kvm *kvm_s390_pci_si_to_kvm(struct zpci_aift *aift, unsigned long si) { - if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM) || aift->kzdev == 0 || - aift->kzdev[si] == 0) - return 0; + if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM) || !aift->kzdev || + !aift->kzdev[si]) + return NULL; return aift->kzdev[si]->kvm; }; diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index f7f5adea8940f8116fb7fd3db3722bca88e3941a..be14c58cb9894bbc8bce0df4389586b759614e92 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -13,13 +13,10 @@ void __delay(unsigned long loops) { - /* - * To end the bloody studid and useless discussion about the - * BogoMips number I took the liberty to define the __delay - * function in a way that that resulting BogoMips number will - * yield the megahertz number of the cpu. The important function - * is udelay and that is done using the tod clock. -- martin. - */ + /* + * Loop 'loops' times. Callers must not assume a specific + * amount of time passes before this function returns. + */ asm volatile("0: brct %0,0b" : : "d" ((loops/2) + 1)); } EXPORT_SYMBOL(__delay); diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index d7b3b193d1088ca498d735f63e7a3b7bff744393..58033dfcb6d45f4b6d28e2bd8a865729d9baf572 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -81,8 +81,9 @@ unsigned long _copy_from_user_key(void *to, const void __user *from, might_fault(); if (!should_fail_usercopy()) { - instrument_copy_from_user(to, from, n); + instrument_copy_from_user_before(to, from, n); res = raw_copy_from_user_key(to, from, n, key); + instrument_copy_from_user_after(to, from, n, res); } if (unlikely(res)) memset(to + (n - res), 0, res); diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 9f9af5298dd6e33bdd7f4f698ed8e81ac074d29f..9953819d795968c1d9ab09d362e415d1cf04e474 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -8,8 +8,10 @@ #include #include #include +#include #include #include +#include static unsigned long max_addr; @@ -21,6 +23,8 @@ struct addr_marker { enum address_markers_idx { IDENTITY_BEFORE_NR = 0, IDENTITY_BEFORE_END_NR, + AMODE31_START_NR, + AMODE31_END_NR, KERNEL_START_NR, KERNEL_END_NR, #ifdef CONFIG_KFENCE @@ -39,11 +43,17 @@ enum address_markers_idx { VMALLOC_END_NR, MODULES_NR, MODULES_END_NR, + ABS_LOWCORE_NR, + ABS_LOWCORE_END_NR, + MEMCPY_REAL_NR, + MEMCPY_REAL_END_NR, }; static struct addr_marker address_markers[] = { [IDENTITY_BEFORE_NR] = {0, "Identity Mapping Start"}, [IDENTITY_BEFORE_END_NR] = {(unsigned long)_stext, "Identity Mapping End"}, + [AMODE31_START_NR] = {0, "Amode31 Area Start"}, + [AMODE31_END_NR] = {0, "Amode31 Area End"}, [KERNEL_START_NR] = {(unsigned long)_stext, "Kernel Image Start"}, [KERNEL_END_NR] = {(unsigned long)_end, "Kernel Image End"}, #ifdef CONFIG_KFENCE @@ -62,6 +72,10 @@ static struct addr_marker address_markers[] = { [VMALLOC_END_NR] = {0, "vmalloc Area End"}, [MODULES_NR] = {0, "Modules Area Start"}, [MODULES_END_NR] = {0, "Modules Area End"}, + [ABS_LOWCORE_NR] = {0, "Lowcore Area Start"}, + [ABS_LOWCORE_END_NR] = {0, "Lowcore Area End"}, + [MEMCPY_REAL_NR] = {0, "Real Memory Copy Area Start"}, + [MEMCPY_REAL_END_NR] = {0, "Real Memory Copy Area End"}, { -1, NULL } }; @@ -276,8 +290,14 @@ static int pt_dump_init(void) max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; + address_markers[AMODE31_START_NR].start_address = __samode31; + address_markers[AMODE31_END_NR].start_address = __eamode31; address_markers[MODULES_NR].start_address = MODULES_VADDR; address_markers[MODULES_END_NR].start_address = MODULES_END; + address_markers[ABS_LOWCORE_NR].start_address = __abs_lowcore; + address_markers[ABS_LOWCORE_END_NR].start_address = __abs_lowcore + ABS_LOWCORE_MAP_SIZE; + address_markers[MEMCPY_REAL_NR].start_address = __memcpy_real_area; + address_markers[MEMCPY_REAL_END_NR].start_address = __memcpy_real_area + PAGE_SIZE; address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; address_markers[VMEMMAP_END_NR].start_address = (unsigned long)vmemmap + vmemmap_size; address_markers[VMALLOC_NR].start_address = VMALLOC_START; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 13449941516c2502bfd0974482abefc590238324..9649d9382e0ae1276959a7b2464d54840657eb57 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -268,8 +268,7 @@ static noinline void do_sigbus(struct pt_regs *regs) (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK)); } -static noinline void do_fault_error(struct pt_regs *regs, int access, - vm_fault_t fault) +static noinline void do_fault_error(struct pt_regs *regs, vm_fault_t fault) { int si_code; @@ -379,7 +378,9 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) flags = FAULT_FLAG_DEFAULT; if (user_mode(regs)) flags |= FAULT_FLAG_USER; - if (access == VM_WRITE || is_write) + if (is_write) + access = VM_WRITE; + if (access == VM_WRITE) flags |= FAULT_FLAG_WRITE; mmap_read_lock(mm); @@ -419,8 +420,6 @@ retry: if (unlikely(!(vma->vm_flags & access))) goto out_up; - if (is_vm_hugetlb_page(vma)) - address &= HPAGE_MASK; /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo @@ -516,7 +515,7 @@ void do_protection_exception(struct pt_regs *regs) fault = do_exception(regs, access); } if (unlikely(fault)) - do_fault_error(regs, access, fault); + do_fault_error(regs, fault); } NOKPROBE_SYMBOL(do_protection_exception); @@ -528,7 +527,7 @@ void do_dat_exception(struct pt_regs *regs) access = VM_ACCESS_FLAGS; fault = do_exception(regs, access); if (unlikely(fault)) - do_fault_error(regs, access, fault); + do_fault_error(regs, fault); } NOKPROBE_SYMBOL(do_dat_exception); @@ -803,7 +802,7 @@ void do_secure_storage_access(struct pt_regs *regs) addr = __gmap_translate(gmap, addr); mmap_read_unlock(mm); if (IS_ERR_VALUE(addr)) { - do_fault_error(regs, VM_ACCESS_FLAGS, VM_FAULT_BADMAP); + do_fault_error(regs, VM_FAULT_BADMAP); break; } fallthrough; @@ -813,7 +812,7 @@ void do_secure_storage_access(struct pt_regs *regs) vma = find_vma(mm, addr); if (!vma) { mmap_read_unlock(mm); - do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + do_fault_error(regs, VM_FAULT_BADMAP); break; } page = follow_page(vma, addr, FOLL_WRITE | FOLL_GET); @@ -836,7 +835,7 @@ void do_secure_storage_access(struct pt_regs *regs) BUG(); break; default: - do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + do_fault_error(regs, VM_FAULT_BADMAP); WARN_ON_ONCE(1); } } @@ -848,7 +847,7 @@ void do_non_secure_storage_access(struct pt_regs *regs) struct gmap *gmap = (struct gmap *)S390_lowcore.gmap; if (get_fault_type(regs) != GMAP_FAULT) { - do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); + do_fault_error(regs, VM_FAULT_BADMAP); WARN_ON_ONCE(1); return; } diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 62758cb5872f1da06abbca4dd21cbe2e107ec13a..02d15c8dc92e994692f2322cf3232f85a032f76a 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -2515,8 +2515,9 @@ static const struct mm_walk_ops thp_split_walk_ops = { static inline void thp_split_mm(struct mm_struct *mm) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); - for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) { + for_each_vma(vmi, vma) { vma->vm_flags &= ~VM_HUGEPAGE; vma->vm_flags |= VM_NOHUGEPAGE; walk_page_vma(vma, &thp_split_walk_ops, NULL); @@ -2584,8 +2585,9 @@ int gmap_mark_unmergeable(void) struct mm_struct *mm = current->mm; struct vm_area_struct *vma; int ret; + VMA_ITERATOR(vmi, mm, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, MADV_UNMERGEABLE, &vma->vm_flags); if (ret) diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index 10e51ef9c79a793b658890c71db84e68b07b15c9..c299a18273ffeca9c30f89d75f14b6fdfc5b49ed 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -237,16 +237,6 @@ int pud_huge(pud_t pud) return pud_large(pud); } -struct page * -follow_huge_pud(struct mm_struct *mm, unsigned long address, - pud_t *pud, int flags) -{ - if (flags & FOLL_GET) - return NULL; - - return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT); -} - bool __init arch_hugetlb_valid_size(unsigned long size) { if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 4a154a08496600a49afd9e9be109309fa5a7f7da..97d66a3e60fb2a445a7e1cd9c926c7a57f283ce1 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index d6d84e02f35ab2cb6ac20c35ccea6c6be4faabb1..1571cdcb0c50c110900a7660f0e1c68c49243239 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -12,10 +12,17 @@ #include #include #include +#include #include #include #include +#include #include +#include + +unsigned long __bootdata_preserved(__memcpy_real_area); +static __ro_after_init pte_t *memcpy_real_ptep; +static DEFINE_MUTEX(memcpy_real_mutex); static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) { @@ -76,118 +83,72 @@ notrace void *s390_kernel_write(void *dst, const void *src, size_t size) return dst; } -static int __no_sanitize_address __memcpy_real(void *dest, void *src, size_t count) +void __init memcpy_real_init(void) { - union register_pair _dst, _src; - int rc = -EFAULT; - - _dst.even = (unsigned long) dest; - _dst.odd = (unsigned long) count; - _src.even = (unsigned long) src; - _src.odd = (unsigned long) count; - asm volatile ( - "0: mvcle %[dst],%[src],0\n" - "1: jo 0b\n" - " lhi %[rc],0\n" - "2:\n" - EX_TABLE(1b,2b) - : [rc] "+&d" (rc), [dst] "+&d" (_dst.pair), [src] "+&d" (_src.pair) - : : "cc", "memory"); - return rc; -} - -static unsigned long __no_sanitize_address _memcpy_real(unsigned long dest, - unsigned long src, - unsigned long count) -{ - int irqs_disabled, rc; - unsigned long flags; - - if (!count) - return 0; - flags = arch_local_irq_save(); - irqs_disabled = arch_irqs_disabled_flags(flags); - if (!irqs_disabled) - trace_hardirqs_off(); - __arch_local_irq_stnsm(0xf8); // disable DAT - rc = __memcpy_real((void *) dest, (void *) src, (size_t) count); - if (flags & PSW_MASK_DAT) - __arch_local_irq_stosm(0x04); // enable DAT - if (!irqs_disabled) - trace_hardirqs_on(); - __arch_local_irq_ssm(flags); - return rc; + memcpy_real_ptep = vmem_get_alloc_pte(__memcpy_real_area, true); + if (!memcpy_real_ptep) + panic("Couldn't setup memcpy real area"); } -/* - * Copy memory in real mode (kernel to kernel) - */ -int memcpy_real(void *dest, unsigned long src, size_t count) +size_t memcpy_real_iter(struct iov_iter *iter, unsigned long src, size_t count) { - unsigned long _dest = (unsigned long)dest; - unsigned long _src = (unsigned long)src; - unsigned long _count = (unsigned long)count; - int rc; - - if (S390_lowcore.nodat_stack != 0) { - preempt_disable(); - rc = call_on_stack(3, S390_lowcore.nodat_stack, - unsigned long, _memcpy_real, - unsigned long, _dest, - unsigned long, _src, - unsigned long, _count); - preempt_enable(); - return rc; + size_t len, copied, res = 0; + unsigned long phys, offset; + void *chunk; + pte_t pte; + + while (count) { + phys = src & PAGE_MASK; + offset = src & ~PAGE_MASK; + chunk = (void *)(__memcpy_real_area + offset); + len = min(count, PAGE_SIZE - offset); + pte = mk_pte_phys(phys, PAGE_KERNEL_RO); + + mutex_lock(&memcpy_real_mutex); + if (pte_val(pte) != pte_val(*memcpy_real_ptep)) { + __ptep_ipte(__memcpy_real_area, memcpy_real_ptep, 0, 0, IPTE_GLOBAL); + set_pte(memcpy_real_ptep, pte); + } + copied = copy_to_iter(chunk, len, iter); + mutex_unlock(&memcpy_real_mutex); + + count -= copied; + src += copied; + res += copied; + if (copied < len) + break; } - /* - * This is a really early memcpy_real call, the stacks are - * not set up yet. Just call _memcpy_real on the early boot - * stack - */ - return _memcpy_real(_dest, _src, _count); + return res; } -/* - * Copy memory in absolute mode (kernel to kernel) - */ -void memcpy_absolute(void *dest, void *src, size_t count) +int memcpy_real(void *dest, unsigned long src, size_t count) { - unsigned long cr0, flags, prefix; - - flags = arch_local_irq_save(); - __ctl_store(cr0, 0, 0); - __ctl_clear_bit(0, 28); /* disable lowcore protection */ - prefix = store_prefix(); - if (prefix) { - local_mcck_disable(); - set_prefix(0); - memcpy(dest, src, count); - set_prefix(prefix); - local_mcck_enable(); - } else { - memcpy(dest, src, count); - } - __ctl_load(cr0, 0, 0); - arch_local_irq_restore(flags); + struct iov_iter iter; + struct kvec kvec; + + kvec.iov_base = dest; + kvec.iov_len = count; + iov_iter_kvec(&iter, WRITE, &kvec, 1, count); + if (memcpy_real_iter(&iter, src, count) < count) + return -EFAULT; + return 0; } /* - * Check if physical address is within prefix or zero page + * Find CPU that owns swapped prefix page */ -static int is_swapped(phys_addr_t addr) +static int get_swapped_owner(phys_addr_t addr) { phys_addr_t lc; int cpu; - if (addr < sizeof(struct lowcore)) - return 1; for_each_online_cpu(cpu) { lc = virt_to_phys(lowcore_ptr[cpu]); if (addr > lc + sizeof(struct lowcore) - 1 || addr < lc) continue; - return 1; + return cpu; } - return 0; + return -1; } /* @@ -200,17 +161,35 @@ void *xlate_dev_mem_ptr(phys_addr_t addr) { void *ptr = phys_to_virt(addr); void *bounce = ptr; + struct lowcore *abs_lc; + unsigned long flags; unsigned long size; + int this_cpu, cpu; cpus_read_lock(); - preempt_disable(); - if (is_swapped(addr)) { - size = PAGE_SIZE - (addr & ~PAGE_MASK); - bounce = (void *) __get_free_page(GFP_ATOMIC); - if (bounce) - memcpy_absolute(bounce, ptr, size); + this_cpu = get_cpu(); + if (addr >= sizeof(struct lowcore)) { + cpu = get_swapped_owner(addr); + if (cpu < 0) + goto out; + } + bounce = (void *)__get_free_page(GFP_ATOMIC); + if (!bounce) + goto out; + size = PAGE_SIZE - (addr & ~PAGE_MASK); + if (addr < sizeof(struct lowcore)) { + abs_lc = get_abs_lowcore(&flags); + ptr = (void *)abs_lc + addr; + memcpy(bounce, ptr, size); + put_abs_lowcore(abs_lc, flags); + } else if (cpu == this_cpu) { + ptr = (void *)(addr - virt_to_phys(lowcore_ptr[cpu])); + memcpy(bounce, ptr, size); + } else { + memcpy(bounce, ptr, size); } - preempt_enable(); +out: + put_cpu(); cpus_read_unlock(); return bounce; } diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 5980ce3488325c5869499da117d919698638fb00..3327c47bc18149826339d49fa5cf48b7ecee5d1a 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -37,7 +37,7 @@ static inline int mmap_is_legacy(struct rlimit *rlim_stack) unsigned long arch_mmap_rnd(void) { - return (get_random_int() & MMAP_RND_MASK) << PAGE_SHIFT; + return (get_random_u32() & MMAP_RND_MASK) << PAGE_SHIFT; } static unsigned long mmap_base_legacy(unsigned long rnd) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index c2583f921ca8d100251079ab701a4ce11ea1ac95..ee1a97078527b7bd01e35a281746232fa3d42585 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -240,7 +240,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, } else if (pmd_none(*pmd)) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE) && - MACHINE_HAS_EDAT1 && addr && direct && + MACHINE_HAS_EDAT1 && direct && !debug_pagealloc_enabled()) { set_pmd(pmd, __pmd(__pa(addr) | prot)); pages++; @@ -336,7 +336,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, } else if (pud_none(*pud)) { if (IS_ALIGNED(addr, PUD_SIZE) && IS_ALIGNED(next, PUD_SIZE) && - MACHINE_HAS_EDAT2 && addr && direct && + MACHINE_HAS_EDAT2 && direct && !debug_pagealloc_enabled()) { set_pud(pud, __pud(__pa(addr) | prot)); pages++; @@ -560,6 +560,103 @@ int vmem_add_mapping(unsigned long start, unsigned long size) return ret; } +/* + * Allocate new or return existing page-table entry, but do not map it + * to any physical address. If missing, allocate segment- and region- + * table entries along. Meeting a large segment- or region-table entry + * while traversing is an error, since the function is expected to be + * called against virtual regions reserverd for 4KB mappings only. + */ +pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc) +{ + pte_t *ptep = NULL; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) { + if (!alloc) + goto out; + p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY); + if (!p4d) + goto out; + pgd_populate(&init_mm, pgd, p4d); + } + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) { + if (!alloc) + goto out; + pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY); + if (!pud) + goto out; + p4d_populate(&init_mm, p4d, pud); + } + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) { + if (!alloc) + goto out; + pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY); + if (!pmd) + goto out; + pud_populate(&init_mm, pud, pmd); + } else if (WARN_ON_ONCE(pud_large(*pud))) { + goto out; + } + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) { + if (!alloc) + goto out; + pte = vmem_pte_alloc(); + if (!pte) + goto out; + pmd_populate(&init_mm, pmd, pte); + } else if (WARN_ON_ONCE(pmd_large(*pmd))) { + goto out; + } + ptep = pte_offset_kernel(pmd, addr); +out: + return ptep; +} + +int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc) +{ + pte_t *ptep, pte; + + if (!IS_ALIGNED(addr, PAGE_SIZE)) + return -EINVAL; + ptep = vmem_get_alloc_pte(addr, alloc); + if (!ptep) + return -ENOMEM; + __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); + pte = mk_pte_phys(phys, prot); + set_pte(ptep, pte); + return 0; +} + +int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot) +{ + int rc; + + mutex_lock(&vmem_mutex); + rc = __vmem_map_4k_page(addr, phys, prot, true); + mutex_unlock(&vmem_mutex); + return rc; +} + +void vmem_unmap_4k_page(unsigned long addr) +{ + pte_t *ptep; + + mutex_lock(&vmem_mutex); + ptep = virt_to_kpte(addr); + __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL); + pte_clear(&init_mm, addr, ptep); + mutex_unlock(&vmem_mutex); +} + /* * map whole physical memory to virtual memory (identity mapping) * we reserve enough space in the vmalloc area for vmemmap to hotplug @@ -584,6 +681,9 @@ void __init vmem_map_init(void) __set_memory(__stext_amode31, (__etext_amode31 - __stext_amode31) >> PAGE_SHIFT, SET_MEMORY_RO | SET_MEMORY_X); + /* lowcore requires 4k mapping for real addresses / prefixing */ + set_memory_4k(0, LC_PAGES); + /* lowcore must be executable for LPSWE */ if (!static_key_enabled(&cpu_has_bear)) set_memory_x(0, 1); diff --git a/arch/s390/pci/Makefile b/arch/s390/pci/Makefile index bf557a1b789c75a3b3bf5a812770269368532be1..5ae31ca9dd441d6180b13e624c0adaca4e49fc23 100644 --- a/arch/s390/pci/Makefile +++ b/arch/s390/pci/Makefile @@ -5,5 +5,5 @@ obj-$(CONFIG_PCI) += pci.o pci_irq.o pci_dma.o pci_clp.o pci_sysfs.o \ pci_event.o pci_debug.o pci_insn.o pci_mmio.o \ - pci_bus.o + pci_bus.o pci_kvm_hook.o obj-$(CONFIG_PCI_IOV) += pci_iov.o diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index f46833a25526358c510c0332677ce111f784d939..227cf0a62800b4cd353aead646240fe6bc93f90d 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -666,7 +666,7 @@ static int __init dma_alloc_cpu_table_caches(void) int __init zpci_dma_init(void) { - s390_iommu_aperture = (u64)high_memory; + s390_iommu_aperture = (u64)virt_to_phys(high_memory); if (!s390_iommu_aperture_factor) s390_iommu_aperture = ULONG_MAX; else diff --git a/arch/s390/pci/pci_kvm_hook.c b/arch/s390/pci/pci_kvm_hook.c new file mode 100644 index 0000000000000000000000000000000000000000..ff34baf50a3e680dee1683967f7de566ad001c0b --- /dev/null +++ b/arch/s390/pci/pci_kvm_hook.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * VFIO ZPCI devices support + * + * Copyright (C) IBM Corp. 2022. All rights reserved. + * Author(s): Pierre Morel + */ +#include + +struct zpci_kvm_hook zpci_kvm_hook; +EXPORT_SYMBOL_GPL(zpci_kvm_hook); diff --git a/arch/sh/Makefile b/arch/sh/Makefile index b39412bf91fb0afafee74140321d5d328be74290..5c8776482530c35a653f888fd05da38da5ab197a 100644 --- a/arch/sh/Makefile +++ b/arch/sh/Makefile @@ -114,8 +114,6 @@ endif export ld-bfd -head-y := arch/sh/kernel/head_32.o - # Mach groups machdir-$(CONFIG_SOLUTION_ENGINE) += mach-se machdir-$(CONFIG_SH_HP6XX) += mach-hp6xx diff --git a/arch/sh/configs/apsh4a3a_defconfig b/arch/sh/configs/apsh4a3a_defconfig index 530498f189904ca9fee38833dce40ececadc96db..99931a13a74da9d7e7bb7066a163e4a28c6ea637 100644 --- a/arch/sh/configs/apsh4a3a_defconfig +++ b/arch/sh/configs/apsh4a3a_defconfig @@ -85,7 +85,7 @@ CONFIG_DEBUG_FS=y CONFIG_DEBUG_KERNEL=y # CONFIG_DEBUG_PREEMPT is not set # CONFIG_DEBUG_BUGVERBOSE is not set -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_FTRACE is not set # CONFIG_CRYPTO_ANSI_CPRNG is not set # CONFIG_CRYPTO_HW is not set diff --git a/arch/sh/configs/apsh4ad0a_defconfig b/arch/sh/configs/apsh4ad0a_defconfig index 6abd9bd70106107d65243ff625f594ec959b52bc..d9fb124bf015a21089ee28976c035609fc0eac9d 100644 --- a/arch/sh/configs/apsh4ad0a_defconfig +++ b/arch/sh/configs/apsh4ad0a_defconfig @@ -116,7 +116,7 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_VM=y CONFIG_DWARF_UNWINDER=y # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/configs/ecovec24_defconfig b/arch/sh/configs/ecovec24_defconfig index e699e2e041284313a1e49dd5587fb8bfd42c04f0..b52e14ccb450d4c63868810bcd388a3c7372ce77 100644 --- a/arch/sh/configs/ecovec24_defconfig +++ b/arch/sh/configs/ecovec24_defconfig @@ -8,7 +8,7 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set CONFIG_CPU_SUBTYPE_SH7724=y -CONFIG_FORCE_MAX_ZONEORDER=12 +CONFIG_ARCH_FORCE_MAX_ORDER=12 CONFIG_MEMORY_SIZE=0x10000000 CONFIG_FLATMEM_MANUAL=y CONFIG_SH_ECOVEC=y diff --git a/arch/sh/configs/edosk7760_defconfig b/arch/sh/configs/edosk7760_defconfig index d77f54e906fd04fece65ad96cbd6a3fa402db36f..f427a95bcd21e5a4de0a9f14ae402af66ccbb12e 100644 --- a/arch/sh/configs/edosk7760_defconfig +++ b/arch/sh/configs/edosk7760_defconfig @@ -107,7 +107,7 @@ CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set CONFIG_TIMER_STATS=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_CRYPTO=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_DES=y diff --git a/arch/sh/configs/magicpanelr2_defconfig b/arch/sh/configs/magicpanelr2_defconfig index 0989ed9295408f9cd14a2b32850300ada0e22249..ef1d98e35c91f7dc93f0b25a9d3bf62517cb3d47 100644 --- a/arch/sh/configs/magicpanelr2_defconfig +++ b/arch/sh/configs/magicpanelr2_defconfig @@ -84,7 +84,7 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y # CONFIG_SCHED_DEBUG is not set CONFIG_DEBUG_KOBJECT=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_POINTER=y CONFIG_CRC_CCITT=m CONFIG_CRC16=m diff --git a/arch/sh/configs/polaris_defconfig b/arch/sh/configs/polaris_defconfig index 246408ec7462b3b43b006c08311c97f5155276ba..f42e4867ddc1a7e5f9b92cfcf2ea4121ff5d1f46 100644 --- a/arch/sh/configs/polaris_defconfig +++ b/arch/sh/configs/polaris_defconfig @@ -79,5 +79,5 @@ CONFIG_DETECT_HUNG_TASK=y CONFIG_DEBUG_RT_MUTEXES=y CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_DEBUG_SPINLOCK_SLEEP=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_SG=y diff --git a/arch/sh/configs/r7780mp_defconfig b/arch/sh/configs/r7780mp_defconfig index f823cc6b18f91d91b1ac46d34e93621f3e6ffc64..e527cd60a1910531dcbde65ca62e255dd53a778f 100644 --- a/arch/sh/configs/r7780mp_defconfig +++ b/arch/sh/configs/r7780mp_defconfig @@ -101,7 +101,7 @@ CONFIG_DEBUG_FS=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_DEBUG_PREEMPT is not set -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y diff --git a/arch/sh/configs/r7785rp_defconfig b/arch/sh/configs/r7785rp_defconfig index f96bc20d4b1ac53c0ae0f73accdb3a7f87466ef3..a3f952a83d970f810d242678291e9bb647f9f115 100644 --- a/arch/sh/configs/r7785rp_defconfig +++ b/arch/sh/configs/r7785rp_defconfig @@ -96,7 +96,7 @@ CONFIG_DEBUG_KERNEL=y # CONFIG_DEBUG_PREEMPT is not set CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_DEBUG_LOCKING_API_SELFTESTS=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_SH_STANDARD_BIOS=y CONFIG_DEBUG_STACK_USAGE=y CONFIG_4KSTACKS=y diff --git a/arch/sh/configs/rsk7203_defconfig b/arch/sh/configs/rsk7203_defconfig index 5a54e2b883f0a33577b1a1b5058d6feb784fd141..d00fafc021e1ac8ef34d55946cc6a9f497dbd17e 100644 --- a/arch/sh/configs/rsk7203_defconfig +++ b/arch/sh/configs/rsk7203_defconfig @@ -112,7 +112,7 @@ CONFIG_DETECT_HUNG_TASK=y CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_MUTEXES=y CONFIG_DEBUG_SPINLOCK_SLEEP=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_VM=y CONFIG_DEBUG_LIST=y CONFIG_DEBUG_SG=y diff --git a/arch/sh/configs/sdk7780_defconfig b/arch/sh/configs/sdk7780_defconfig index 7d6d323598480c9f250c9c6c48f875dd74e7252b..41cb588ca99cb7e6f33eb3635084d608c1542fc9 100644 --- a/arch/sh/configs/sdk7780_defconfig +++ b/arch/sh/configs/sdk7780_defconfig @@ -131,7 +131,7 @@ CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set CONFIG_TIMER_STATS=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_SH_STANDARD_BIOS=y CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_DES=y diff --git a/arch/sh/configs/sdk7786_defconfig b/arch/sh/configs/sdk7786_defconfig index a8662b6927ec721cc7c2147b8b15ba6312f9847b..97b7356639ed82fbca6a8677a46ab3b8589d8ad4 100644 --- a/arch/sh/configs/sdk7786_defconfig +++ b/arch/sh/configs/sdk7786_defconfig @@ -16,7 +16,6 @@ CONFIG_CPUSETS=y # CONFIG_PROC_PID_CPUSET is not set CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_MEMCG=y -CONFIG_CGROUP_MEMCG_SWAP=y CONFIG_CGROUP_SCHED=y CONFIG_RT_GROUP_SCHED=y CONFIG_BLK_CGROUP=y diff --git a/arch/sh/configs/se7712_defconfig b/arch/sh/configs/se7712_defconfig index ee6d28ae08deb82e1032caed666ecf266a08384c..36356223d51c837d24bf3a857b7a2e705c672100 100644 --- a/arch/sh/configs/se7712_defconfig +++ b/arch/sh/configs/se7712_defconfig @@ -93,7 +93,7 @@ CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_POINTER=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/sh/configs/se7721_defconfig b/arch/sh/configs/se7721_defconfig index bad921bc10f8f36eb76f3658cf71ecf9f82ef99c..46c5a263a239255b1fc11e5d92aaa3794293c1ab 100644 --- a/arch/sh/configs/se7721_defconfig +++ b/arch/sh/configs/se7721_defconfig @@ -121,7 +121,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_1=y CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_POINTER=y # CONFIG_CRYPTO_ANSI_CPRNG is not set CONFIG_CRC_CCITT=y diff --git a/arch/sh/configs/sh2007_defconfig b/arch/sh/configs/sh2007_defconfig index 79f02f1c0dc83031006c7a1fd630167415e0b9dd..259c69e3fa22796c9651d982255057bff3ffea6a 100644 --- a/arch/sh/configs/sh2007_defconfig +++ b/arch/sh/configs/sh2007_defconfig @@ -159,7 +159,7 @@ CONFIG_DEBUG_FS=y CONFIG_DEBUG_KERNEL=y # CONFIG_DETECT_SOFTLOCKUP is not set # CONFIG_SCHED_DEBUG is not set -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_POINTER=y CONFIG_SH_STANDARD_BIOS=y CONFIG_CRYPTO_NULL=y diff --git a/arch/sh/configs/sh7757lcr_defconfig b/arch/sh/configs/sh7757lcr_defconfig index a2700ab165afb19ac8b61c5c4a9c11e6663a5eca..2579dc4bc0c8f6f963dfedd625bc4bf3224a0c94 100644 --- a/arch/sh/configs/sh7757lcr_defconfig +++ b/arch/sh/configs/sh7757lcr_defconfig @@ -80,6 +80,6 @@ CONFIG_NLS_ISO8859_1=y CONFIG_DEBUG_KERNEL=y # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_BUGVERBOSE is not set -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_FTRACE is not set # CONFIG_CRYPTO_ANSI_CPRNG is not set diff --git a/arch/sh/configs/sh7785lcr_32bit_defconfig b/arch/sh/configs/sh7785lcr_32bit_defconfig index 7eb3c10f28ad2fc4b6ff802f4346229e34d7f093..781ff13227fc99dfb47410f7addabce9b85e43bc 100644 --- a/arch/sh/configs/sh7785lcr_32bit_defconfig +++ b/arch/sh/configs/sh7785lcr_32bit_defconfig @@ -141,7 +141,7 @@ CONFIG_DEBUG_KMEMLEAK=y CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y CONFIG_DEBUG_SPINLOCK_SLEEP=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_LATENCYTOP=y # CONFIG_FTRACE is not set CONFIG_CRYPTO_HMAC=y diff --git a/arch/sh/configs/urquell_defconfig b/arch/sh/configs/urquell_defconfig index cb2f56468fe0268bb834ce9183ddfef34d7fc442..8fc687c98fd195e8dd6c3845789772dd88020b5f 100644 --- a/arch/sh/configs/urquell_defconfig +++ b/arch/sh/configs/urquell_defconfig @@ -14,7 +14,6 @@ CONFIG_CPUSETS=y # CONFIG_PROC_PID_CPUSET is not set CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_MEMCG=y -CONFIG_CGROUP_MEMCG_SWAP=y CONFIG_CGROUP_SCHED=y CONFIG_RT_GROUP_SCHED=y CONFIG_BLK_DEV_INITRD=y @@ -139,7 +138,7 @@ CONFIG_PRINTK_TIME=y CONFIG_DEBUG_FS=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_POINTER=y # CONFIG_FTRACE is not set # CONFIG_DUMP_CODE is not set diff --git a/arch/sh/include/asm/bitops-op32.h b/arch/sh/include/asm/bitops-op32.h index 565a85d8b7fb014275ce981de1df7fe0bbf95da8..5ace89b4650791d2bfb0d3c3567219a52c6b07fc 100644 --- a/arch/sh/include/asm/bitops-op32.h +++ b/arch/sh/include/asm/bitops-op32.h @@ -135,16 +135,8 @@ arch___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) return (old & mask) != 0; } -/** - * arch_test_bit - Determine whether a bit is set - * @nr: bit number to test - * @addr: Address to start counting from - */ -static __always_inline bool -arch_test_bit(unsigned long nr, const volatile unsigned long *addr) -{ - return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); -} +#define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire #include diff --git a/arch/sh/include/asm/hw_breakpoint.h b/arch/sh/include/asm/hw_breakpoint.h index 199d17b765f2b9424d9c3eea0225260ed39b175a..361a0f57bdebda6ef10e97c6bfe708bb9a1dfbba 100644 --- a/arch/sh/include/asm/hw_breakpoint.h +++ b/arch/sh/include/asm/hw_breakpoint.h @@ -48,10 +48,7 @@ struct pmu; /* Maximum number of UBC channels */ #define HBP_NUM 2 -static inline int hw_breakpoint_slots(int type) -{ - return HBP_NUM; -} +#define hw_breakpoint_slots(type) (HBP_NUM) /* arch/sh/kernel/hw_breakpoint.c */ extern int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw); diff --git a/arch/sh/include/asm/processor_32.h b/arch/sh/include/asm/processor_32.h index 45240ec6b85a43de127afb13b786c1142fcdb178..27aebf1e75a2003003cffaf82c43d46b3e206357 100644 --- a/arch/sh/include/asm/processor_32.h +++ b/arch/sh/include/asm/processor_32.h @@ -127,9 +127,6 @@ struct task_struct; extern void start_thread(struct pt_regs *regs, unsigned long new_pc, unsigned long new_sp); -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - /* * FPU lazy state save handling. */ diff --git a/arch/sh/include/asm/sections.h b/arch/sh/include/asm/sections.h index 8edb824049b9eeb4f7e8e6af14e8eeb4b45718a9..0cb0ca149ac34a62e4d5e8191a2597d6029dd5ec 100644 --- a/arch/sh/include/asm/sections.h +++ b/arch/sh/include/asm/sections.h @@ -4,7 +4,7 @@ #include -extern long __machvec_start, __machvec_end; +extern char __machvec_start[], __machvec_end[]; extern char __uncached_start, __uncached_end; extern char __start_eh_frame[], __stop_eh_frame[]; diff --git a/arch/sh/kernel/Makefile b/arch/sh/kernel/Makefile index aa0fbc9202b14e89464a82e22691619b1812c215..69cd9ac4b2ab8a55c290a42d52bb4cfc0f4926ba 100644 --- a/arch/sh/kernel/Makefile +++ b/arch/sh/kernel/Makefile @@ -3,7 +3,7 @@ # Makefile for the Linux/SuperH kernel. # -extra-y := head_32.o vmlinux.lds +extra-y := vmlinux.lds ifdef CONFIG_FUNCTION_TRACER # Do not profile debug and lowlevel utilities @@ -12,7 +12,7 @@ endif CFLAGS_REMOVE_return_address.o = -pg -obj-y := debugtraps.o dumpstack.o \ +obj-y := head_32.o debugtraps.o dumpstack.o \ idle.o io.o irq.o irq_32.o kdebugfs.o \ machvec.o nmi_debug.o process.o \ process_32.o ptrace.o ptrace_32.o \ diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index 90927673807802cb9ee4157994045c100e9dcc1e..4e6835de54cf8e773bd2b5b53df29e6ce7d2aa88 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -149,7 +149,7 @@ void irq_ctx_exit(int cpu) hardirq_ctx[cpu] = NULL; } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void) { struct thread_info *curctx; diff --git a/arch/sh/kernel/machvec.c b/arch/sh/kernel/machvec.c index d606679a211e1bdb4095faf46523d0c562155cef..57efaf5b82ae07e4f744aaeded508143dd248ccf 100644 --- a/arch/sh/kernel/machvec.c +++ b/arch/sh/kernel/machvec.c @@ -20,8 +20,8 @@ #define MV_NAME_SIZE 32 #define for_each_mv(mv) \ - for ((mv) = (struct sh_machine_vector *)&__machvec_start; \ - (mv) && (unsigned long)(mv) < (unsigned long)&__machvec_end; \ + for ((mv) = (struct sh_machine_vector *)__machvec_start; \ + (mv) && (unsigned long)(mv) < (unsigned long)__machvec_end; \ (mv)++) static struct sh_machine_vector * __init get_mv_byname(const char *name) @@ -87,8 +87,8 @@ void __init sh_mv_setup(void) if (!machvec_selected) { unsigned long machvec_size; - machvec_size = ((unsigned long)&__machvec_end - - (unsigned long)&__machvec_start); + machvec_size = ((unsigned long)__machvec_end - + (unsigned long)__machvec_start); /* * Sanity check for machvec section alignment. Ensure @@ -102,7 +102,7 @@ void __init sh_mv_setup(void) * vector (usually the only one) from .machvec.init. */ if (machvec_size >= sizeof(struct sh_machine_vector)) - sh_mv = *(struct sh_machine_vector *)&__machvec_start; + sh_mv = *(struct sh_machine_vector *)__machvec_start; } pr_notice("Booting machvec: %s\n", get_system_type()); diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index a808843375e713eb5e7c037079e3cb1d2a503b69..92b6649d492952ce6943ed2c535337cfd27508ca 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -84,11 +84,6 @@ void flush_thread(void) #endif } -void release_thread(struct task_struct *dead_task) -{ - /* do nothing */ -} - asmlinkage void ret_from_fork(void); asmlinkage void ret_from_kernel_thread(void); diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig index ba569cfb43684eee0f7f71c3cb263f316250715f..411fdc0901f7ea75a5f1f4e8b95029f1aa3c7129 100644 --- a/arch/sh/mm/Kconfig +++ b/arch/sh/mm/Kconfig @@ -18,7 +18,7 @@ config PAGE_OFFSET default "0x80000000" if MMU default "0x00000000" -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" range 9 64 if PAGE_SIZE_16KB default "9" if PAGE_SIZE_16KB diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 1c852bb530ecc949de22242c36cd8802c63f94f9..4d3d1af90d5217e1f16f1a92085afb9c4eef936e 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -269,7 +269,7 @@ config ARCH_SPARSEMEM_ENABLE config ARCH_SPARSEMEM_DEFAULT def_bool y if SPARC64 -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" default "13" help diff --git a/arch/sparc/Makefile b/arch/sparc/Makefile index fe58a410b4ce68379acf754e44dcb423b5455c81..a4ea5b05f2885712834ef68902937d720640b1a5 100644 --- a/arch/sparc/Makefile +++ b/arch/sparc/Makefile @@ -56,8 +56,6 @@ endif endif -head-y := arch/sparc/kernel/head_$(BITS).o - libs-y += arch/sparc/prom/ libs-y += arch/sparc/lib/ diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..cfe5102b1c683593d0191acbf8b90d1ef11e1ce0 --- /dev/null +++ b/arch/sparc/crypto/Kconfig @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (sparc64)" + +config CRYPTO_DES_SPARC64 + tristate "Ciphers: DES and Triple DES EDE, modes: ECB/CBC" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES + select CRYPTO_SKCIPHER + help + Block cipher: DES (FIPS 46-2) cipher algorithm + Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: DES with ECB and CBC modes + Length-preserving ciphers: Tripe DES EDE with ECB and CBC modes + + Architecture: sparc64 + +config CRYPTO_CRC32C_SPARC64 + tristate "CRC32c" + depends on SPARC64 + select CRYPTO_HASH + select CRC32 + help + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: sparc64 + +config CRYPTO_MD5_SPARC64 + tristate "Digests: MD5" + depends on SPARC64 + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) + + Architecture: sparc64 using crypto instructions, when available + +config CRYPTO_SHA1_SPARC64 + tristate "Hash functions: SHA-1" + depends on SPARC64 + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: sparc64 + +config CRYPTO_SHA256_SPARC64 + tristate "Hash functions: SHA-224 and SHA-256" + depends on SPARC64 + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: sparc64 using crypto instructions, when available + +config CRYPTO_SHA512_SPARC64 + tristate "Hash functions: SHA-384 and SHA-512" + depends on SPARC64 + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: sparc64 using crypto instructions, when available + +config CRYPTO_AES_SPARC64 + tristate "Ciphers: AES, modes: ECB, CBC, CTR" + depends on SPARC64 + select CRYPTO_SKCIPHER + help + Block ciphers: AES cipher algorithms (FIPS-197) + Length-preseving ciphers: AES with ECB, CBC, and CTR modes + + Architecture: sparc64 using crypto instructions + +config CRYPTO_CAMELLIA_SPARC64 + tristate "Ciphers: Camellia, modes: ECB, CBC" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Block ciphers: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: sparc64 + +endmenu diff --git a/arch/sparc/include/asm/io.h b/arch/sparc/include/asm/io.h index 2eefa526b38f034df9ef71437231f93c6d458d69..2dad9be9ec752fa27ef3acb149418691035ad4d5 100644 --- a/arch/sparc/include/asm/io.h +++ b/arch/sparc/include/asm/io.h @@ -19,4 +19,6 @@ #define writel_be(__w, __addr) __raw_writel(__w, __addr) #define writew_be(__l, __addr) __raw_writew(__l, __addr) +#include + #endif diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h index 5ffa820dcd4d5c4437d8912f801e3bf8385759e1..9303270b22f3cfd4df85d3abfddebe7824725c75 100644 --- a/arch/sparc/include/asm/io_64.h +++ b/arch/sparc/include/asm/io_64.h @@ -9,6 +9,7 @@ #include /* IO address mapping routines need this */ #include #include +#define pci_iomap pci_iomap /* BIO layer definitions. */ extern unsigned long kern_base, kern_size; @@ -239,38 +240,51 @@ static inline void outl(u32 l, unsigned long addr) void outsb(unsigned long, const void *, unsigned long); void outsw(unsigned long, const void *, unsigned long); void outsl(unsigned long, const void *, unsigned long); +#define outsb outsb +#define outsw outsw +#define outsl outsl void insb(unsigned long, void *, unsigned long); void insw(unsigned long, void *, unsigned long); void insl(unsigned long, void *, unsigned long); +#define insb insb +#define insw insw +#define insl insl static inline void readsb(void __iomem *port, void *buf, unsigned long count) { insb((unsigned long __force)port, buf, count); } +#define readsb readsb + static inline void readsw(void __iomem *port, void *buf, unsigned long count) { insw((unsigned long __force)port, buf, count); } +#define readsw readsw static inline void readsl(void __iomem *port, void *buf, unsigned long count) { insl((unsigned long __force)port, buf, count); } +#define readsl readsl static inline void writesb(void __iomem *port, const void *buf, unsigned long count) { outsb((unsigned long __force)port, buf, count); } +#define writesb writesb static inline void writesw(void __iomem *port, const void *buf, unsigned long count) { outsw((unsigned long __force)port, buf, count); } +#define writesw writesw static inline void writesl(void __iomem *port, const void *buf, unsigned long count) { outsl((unsigned long __force)port, buf, count); } +#define writesl writesl #define ioread8_rep(p,d,l) readsb(p,d,l) #define ioread16_rep(p,d,l) readsw(p,d,l) @@ -344,6 +358,7 @@ static inline void memset_io(volatile void __iomem *dst, int c, __kernel_size_t d++; } } +#define memset_io memset_io static inline void sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, __kernel_size_t n) @@ -369,6 +384,7 @@ static inline void memcpy_fromio(void *dst, const volatile void __iomem *src, src++; } } +#define memcpy_fromio memcpy_fromio static inline void sbus_memcpy_toio(volatile void __iomem *dst, const void *src, __kernel_size_t n) @@ -395,6 +411,7 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, d++; } } +#define memcpy_toio memcpy_toio #ifdef __KERNEL__ @@ -412,7 +429,9 @@ static inline void __iomem *ioremap(unsigned long offset, unsigned long size) static inline void __iomem *ioremap_np(unsigned long offset, unsigned long size) { return NULL; + } +#define ioremap_np ioremap_np static inline void iounmap(volatile void __iomem *addr) { @@ -432,10 +451,13 @@ static inline void iounmap(volatile void __iomem *addr) /* Create a virtual mapping cookie for an IO port range */ void __iomem *ioport_map(unsigned long port, unsigned int nr); void ioport_unmap(void __iomem *); +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ struct pci_dev; void pci_iounmap(struct pci_dev *dev, void __iomem *); +#define pci_iounmap pci_iounmap static inline int sbus_can_dma_64bit(void) { diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index b26c35336b51d180a9485297dd785c49d773a0cd..ba8b70ffec085feb17de9050f37de98e0039f7c3 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -80,9 +80,6 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, : "memory"); } -/* Free all resources held by a thread. */ -#define release_thread(tsk) do { } while(0) - unsigned long __get_wchan(struct task_struct *); #define task_pt_regs(tsk) ((tsk)->thread.kregs) diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 89850dff6b033da352da7c39b58a066e4bca3df6..2667f35d5ea5678c64cc99ded20af2b797754967 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -176,9 +176,6 @@ do { \ regs->tstate &= ~TSTATE_PEF; \ } while (0) -/* Free all resources held by a thread. */ -#define release_thread(tsk) do { } while (0) - unsigned long __get_wchan(struct task_struct *task); #define task_pt_regs(tsk) (task_thread_info(tsk)->kregs) diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h index 856081761b0fc3c11fe640cdd30d4009f727f309..2cf7971d7f6c9e3e9377bb90aa7c883b5d327b4d 100644 --- a/arch/sparc/include/asm/smp_32.h +++ b/arch/sparc/include/asm/smp_32.h @@ -33,9 +33,6 @@ extern volatile unsigned long cpu_callin_map[NR_CPUS]; extern cpumask_t smp_commenced_mask; extern struct linux_prom_registers smp_penguin_ctable; -typedef void (*smpfunc_t)(unsigned long, unsigned long, unsigned long, - unsigned long, unsigned long); - void cpu_panic(void); /* @@ -57,7 +54,7 @@ void smp_bogo(struct seq_file *); void smp_info(struct seq_file *); struct sparc32_ipi_ops { - void (*cross_call)(smpfunc_t func, cpumask_t mask, unsigned long arg1, + void (*cross_call)(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4); void (*resched)(int cpu); @@ -66,28 +63,28 @@ struct sparc32_ipi_ops { }; extern const struct sparc32_ipi_ops *sparc32_ipi_ops; -static inline void xc0(smpfunc_t func) +static inline void xc0(void *func) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, 0, 0, 0, 0); } -static inline void xc1(smpfunc_t func, unsigned long arg1) +static inline void xc1(void *func, unsigned long arg1) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, 0, 0, 0); } -static inline void xc2(smpfunc_t func, unsigned long arg1, unsigned long arg2) +static inline void xc2(void *func, unsigned long arg1, unsigned long arg2) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, 0, 0); } -static inline void xc3(smpfunc_t func, unsigned long arg1, unsigned long arg2, +static inline void xc3(void *func, unsigned long arg1, unsigned long arg2, unsigned long arg3) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, arg3, 0); } -static inline void xc4(smpfunc_t func, unsigned long arg1, unsigned long arg2, +static inline void xc4(void *func, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { sparc32_ipi_ops->cross_call(func, *cpu_online_mask, diff --git a/arch/sparc/include/asm/termios.h b/arch/sparc/include/asm/termios.h deleted file mode 100644 index 4a558efdfa9375921b724c90ceb2bdecd3a90d88..0000000000000000000000000000000000000000 --- a/arch/sparc/include/asm/termios.h +++ /dev/null @@ -1,147 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _SPARC_TERMIOS_H -#define _SPARC_TERMIOS_H - -#include - - -/* - * c_cc characters in the termio structure. Oh, how I love being - * backwardly compatible. Notice that character 4 and 5 are - * interpreted differently depending on whether ICANON is set in - * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise - * as _VMIN and V_TIME. This is for compatibility with OSF/1 (which - * is compatible with sysV)... - */ -#define _VMIN 4 -#define _VTIME 5 - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D eol=\0 eol2=\0 sxtc=\0 - start=^Q stop=^S susp=^Z dsusp=^Y - reprint=^R discard=^U werase=^W lnext=^V - vmin=\1 vtime=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\000\000\000\021\023\032\031\022\025\027\026\001" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -#define user_termio_to_kernel_termios(termios, termio) \ -({ \ - unsigned short tmp; \ - int err; \ - err = get_user(tmp, &(termio)->c_iflag); \ - (termios)->c_iflag = (0xffff0000 & ((termios)->c_iflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_oflag); \ - (termios)->c_oflag = (0xffff0000 & ((termios)->c_oflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_cflag); \ - (termios)->c_cflag = (0xffff0000 & ((termios)->c_cflag)) | tmp; \ - err |= get_user(tmp, &(termio)->c_lflag); \ - (termios)->c_lflag = (0xffff0000 & ((termios)->c_lflag)) | tmp; \ - err |= copy_from_user((termios)->c_cc, (termio)->c_cc, NCC); \ - err; \ -}) - -/* - * Translate a "termios" structure into a "termio". Ugh. - * - * Note the "fun" _VMIN overloading. - */ -#define kernel_termios_to_user_termio(termio, termios) \ -({ \ - int err; \ - err = put_user((termios)->c_iflag, &(termio)->c_iflag); \ - err |= put_user((termios)->c_oflag, &(termio)->c_oflag); \ - err |= put_user((termios)->c_cflag, &(termio)->c_cflag); \ - err |= put_user((termios)->c_lflag, &(termio)->c_lflag); \ - err |= put_user((termios)->c_line, &(termio)->c_line); \ - err |= copy_to_user((termio)->c_cc, (termios)->c_cc, NCC); \ - if (!((termios)->c_lflag & ICANON)) { \ - err |= put_user((termios)->c_cc[VMIN], &(termio)->c_cc[_VMIN]); \ - err |= put_user((termios)->c_cc[VTIME], &(termio)->c_cc[_VTIME]); \ - } \ - err; \ -}) - -#define user_termios_to_kernel_termios(k, u) \ -({ \ - int err; \ - err = get_user((k)->c_iflag, &(u)->c_iflag); \ - err |= get_user((k)->c_oflag, &(u)->c_oflag); \ - err |= get_user((k)->c_cflag, &(u)->c_cflag); \ - err |= get_user((k)->c_lflag, &(u)->c_lflag); \ - err |= get_user((k)->c_line, &(u)->c_line); \ - err |= copy_from_user((k)->c_cc, (u)->c_cc, NCCS); \ - if ((k)->c_lflag & ICANON) { \ - err |= get_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= get_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } else { \ - err |= get_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= get_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } \ - err |= get_user((k)->c_ispeed, &(u)->c_ispeed); \ - err |= get_user((k)->c_ospeed, &(u)->c_ospeed); \ - err; \ -}) - -#define kernel_termios_to_user_termios(u, k) \ -({ \ - int err; \ - err = put_user((k)->c_iflag, &(u)->c_iflag); \ - err |= put_user((k)->c_oflag, &(u)->c_oflag); \ - err |= put_user((k)->c_cflag, &(u)->c_cflag); \ - err |= put_user((k)->c_lflag, &(u)->c_lflag); \ - err |= put_user((k)->c_line, &(u)->c_line); \ - err |= copy_to_user((u)->c_cc, (k)->c_cc, NCCS); \ - if (!((k)->c_lflag & ICANON)) { \ - err |= put_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= put_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } else { \ - err |= put_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= put_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } \ - err |= put_user((k)->c_ispeed, &(u)->c_ispeed); \ - err |= put_user((k)->c_ospeed, &(u)->c_ospeed); \ - err; \ -}) - -#define user_termios_to_kernel_termios_1(k, u) \ -({ \ - int err; \ - err = get_user((k)->c_iflag, &(u)->c_iflag); \ - err |= get_user((k)->c_oflag, &(u)->c_oflag); \ - err |= get_user((k)->c_cflag, &(u)->c_cflag); \ - err |= get_user((k)->c_lflag, &(u)->c_lflag); \ - err |= get_user((k)->c_line, &(u)->c_line); \ - err |= copy_from_user((k)->c_cc, (u)->c_cc, NCCS); \ - if ((k)->c_lflag & ICANON) { \ - err |= get_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= get_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } else { \ - err |= get_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= get_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } \ - err; \ -}) - -#define kernel_termios_to_user_termios_1(u, k) \ -({ \ - int err; \ - err = put_user((k)->c_iflag, &(u)->c_iflag); \ - err |= put_user((k)->c_oflag, &(u)->c_oflag); \ - err |= put_user((k)->c_cflag, &(u)->c_cflag); \ - err |= put_user((k)->c_lflag, &(u)->c_lflag); \ - err |= put_user((k)->c_line, &(u)->c_line); \ - err |= copy_to_user((u)->c_cc, (k)->c_cc, NCCS); \ - if (!((k)->c_lflag & ICANON)) { \ - err |= put_user((k)->c_cc[VMIN], &(u)->c_cc[_VMIN]); \ - err |= put_user((k)->c_cc[VTIME], &(u)->c_cc[_VTIME]); \ - } else { \ - err |= put_user((k)->c_cc[VEOF], &(u)->c_cc[VEOF]); \ - err |= put_user((k)->c_cc[VEOL], &(u)->c_cc[VEOL]); \ - } \ - err; \ -}) - -#endif /* _SPARC_TERMIOS_H */ diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index d3a0e072ebe8cf6191913a01d66c8c269b9ca1f2..0984bb6f0f17b177b6b368911a5efd5069b5e0ac 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -7,8 +7,6 @@ asflags-y := -ansi ccflags-y := -Werror -extra-y := head_$(BITS).o - # Undefine sparc when processing vmlinux.lds - it is used # And teach CPP we are doing $(BITS) builds (for this case) CPPFLAGS_vmlinux.lds := -Usparc -m$(BITS) @@ -22,6 +20,7 @@ CFLAGS_REMOVE_perf_event.o := -pg CFLAGS_REMOVE_pcr.o := -pg endif +obj-y := head_$(BITS).o obj-$(CONFIG_SPARC64) += urtt_fill.o obj-$(CONFIG_SPARC32) += entry.o wof.o wuf.o obj-$(CONFIG_SPARC32) += etrap_32.o @@ -87,12 +86,13 @@ obj-$(CONFIG_SPARC64_SMP) += hvtramp.o obj-y += auxio_$(BITS).o obj-$(CONFIG_SUN_PM) += apc.o pmc.o +obj-y += termios.o + obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_MODULES) += sparc_ksyms.o obj-$(CONFIG_SPARC_LED) += led.o obj-$(CONFIG_KGDB) += kgdb_$(BITS).o - obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o @@ -104,6 +104,7 @@ obj-$(CONFIG_SPARC64_PCI) += pci_psycho.o pci_sabre.o pci_schizo.o obj-$(CONFIG_SPARC64_PCI) += pci_sun4v.o pci_sun4v_asm.o pci_fire.o obj-$(CONFIG_SPARC64_PCI_MSI) += pci_msi.o + obj-$(CONFIG_COMPAT) += sys32.o sys_sparc32.o signal32.o obj-$(CONFIG_US3_MC) += chmc.o diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 41fa1be980a33463adbde4c50b0dbade80203e9f..72da2e10e2559ab8ea1dc19c63af917b6ca2a492 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -855,7 +855,7 @@ void __irq_entry handler_irq(int pil, struct pt_regs *regs) set_irq_regs(old_regs); } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void) { void *orig_sp, *sp = softirq_stack[smp_processor_id()]; diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index 1eed26d423fb2da3580cff885048696cfbce03e9..991e9ad3d3e8f953894ca1015306b7d8e7e6688e 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -359,7 +359,7 @@ void leonsmp_ipi_interrupt(void) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -372,7 +372,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void leon_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -384,7 +384,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, { /* If you make changes here, make sure gcc generates proper code... */ - register smpfunc_t f asm("i0") = func; + register void *f asm("i0") = func; register unsigned long a1 asm("i1") = arg1; register unsigned long a2 asm("i2") = arg2; register unsigned long a3 asm("i3") = arg3; @@ -444,11 +444,13 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void leon_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index ff30f03beb7c785f951421f4b6feb944171c45a3..9a62a5cf33370e73273055f0b8598c6a2f30a769 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -268,7 +268,7 @@ static void sun4d_ipi_resched(int cpu) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -281,7 +281,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4d_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -296,7 +296,7 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, * If you make changes here, make sure * gcc generates proper code... */ - register smpfunc_t f asm("i0") = func; + register void *f asm("i0") = func; register unsigned long a1 asm("i1") = arg1; register unsigned long a2 asm("i2") = arg2; register unsigned long a3 asm("i3") = arg3; @@ -353,11 +353,13 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void smp4d_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = hard_smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 228a6527082dc2184a754b17b472d3b8ee4ba8ee..056df034e79eebb9fa0779044b1d082805cc24f6 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -157,7 +157,7 @@ static void sun4m_ipi_mask_one(int cpu) } static struct smp_funcall { - smpfunc_t func; + void *func; unsigned long arg1; unsigned long arg2; unsigned long arg3; @@ -170,7 +170,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4m_cross_call(void *func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -230,11 +230,13 @@ static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void smp4m_cross_call_irq(void) { + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; int i = smp_processor_id(); ccall_info.processors_in[i] = 1; - ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, - ccall_info.arg4, ccall_info.arg5); + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); ccall_info.processors_out[i] = 1; } diff --git a/arch/sparc/kernel/termios.c b/arch/sparc/kernel/termios.c new file mode 100644 index 0000000000000000000000000000000000000000..ee64965c27cd48a7ed55c50dd1488f6bc9f2e7a1 --- /dev/null +++ b/arch/sparc/kernel/termios.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +/* + * c_cc characters in the termio structure. Oh, how I love being + * backwardly compatible. Notice that character 4 and 5 are + * interpreted differently depending on whether ICANON is set in + * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise + * as _VMIN and V_TIME. This is for compatibility with OSF/1 (which + * is compatible with sysV)... + */ +#define _VMIN 4 +#define _VTIME 5 + +int kernel_termios_to_user_termio(struct termio __user *termio, + struct ktermios *termios) +{ + struct termio v; + memset(&v, 0, sizeof(struct termio)); + v.c_iflag = termios->c_iflag; + v.c_oflag = termios->c_oflag; + v.c_cflag = termios->c_cflag; + v.c_lflag = termios->c_lflag; + v.c_line = termios->c_line; + memcpy(v.c_cc, termios->c_cc, NCC); + if (!(v.c_lflag & ICANON)) { + v.c_cc[_VMIN] = termios->c_cc[VMIN]; + v.c_cc[_VTIME] = termios->c_cc[VTIME]; + } + return copy_to_user(termio, &v, sizeof(struct termio)); +} + +int user_termios_to_kernel_termios(struct ktermios *k, + struct termios2 __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + err |= get_user(k->c_ispeed, &u->c_ispeed); + err |= get_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int kernel_termios_to_user_termios(struct termios2 __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + err |= put_user(k->c_ispeed, &u->c_ispeed); + err |= put_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int user_termios_to_kernel_termios_1(struct ktermios *k, + struct termios __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + return err; +} + +int kernel_termios_to_user_termios_1(struct termios __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + return err; +} diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index a9aa6a92c7fee280b5ae408e6b4f65c9e96d36dc..13f027afc875c89d27f857e00e29b604d487d7e6 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -1636,19 +1636,19 @@ static void __init get_srmmu_type(void) /* Local cross-calls. */ static void smp_flush_page_for_dma(unsigned long page) { - xc1((smpfunc_t) local_ops->page_for_dma, page); + xc1(local_ops->page_for_dma, page); local_ops->page_for_dma(page); } static void smp_flush_cache_all(void) { - xc0((smpfunc_t) local_ops->cache_all); + xc0(local_ops->cache_all); local_ops->cache_all(); } static void smp_flush_tlb_all(void) { - xc0((smpfunc_t) local_ops->tlb_all); + xc0(local_ops->tlb_all); local_ops->tlb_all(); } @@ -1659,7 +1659,7 @@ static void smp_flush_cache_mm(struct mm_struct *mm) cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm); + xc1(local_ops->cache_mm, (unsigned long)mm); local_ops->cache_mm(mm); } } @@ -1671,7 +1671,7 @@ static void smp_flush_tlb_mm(struct mm_struct *mm) cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) { - xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm); + xc1(local_ops->tlb_mm, (unsigned long)mm); if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id())); @@ -1691,8 +1691,8 @@ static void smp_flush_cache_range(struct vm_area_struct *vma, cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc3((smpfunc_t) local_ops->cache_range, - (unsigned long) vma, start, end); + xc3(local_ops->cache_range, (unsigned long)vma, start, + end); local_ops->cache_range(vma, start, end); } } @@ -1708,8 +1708,8 @@ static void smp_flush_tlb_range(struct vm_area_struct *vma, cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc3((smpfunc_t) local_ops->tlb_range, - (unsigned long) vma, start, end); + xc3(local_ops->tlb_range, (unsigned long)vma, start, + end); local_ops->tlb_range(vma, start, end); } } @@ -1723,8 +1723,7 @@ static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->cache_page, - (unsigned long) vma, page); + xc2(local_ops->cache_page, (unsigned long)vma, page); local_ops->cache_page(vma, page); } } @@ -1738,8 +1737,7 @@ static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->tlb_page, - (unsigned long) vma, page); + xc2(local_ops->tlb_page, (unsigned long)vma, page); local_ops->tlb_page(vma, page); } } @@ -1753,7 +1751,7 @@ static void smp_flush_page_to_ram(unsigned long page) * XXX This experiment failed, research further... -DaveM */ #if 1 - xc1((smpfunc_t) local_ops->page_to_ram, page); + xc1(local_ops->page_to_ram, page); #endif local_ops->page_to_ram(page); } @@ -1764,8 +1762,7 @@ static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) cpumask_copy(&cpu_mask, mm_cpumask(mm)); cpumask_clear_cpu(smp_processor_id(), &cpu_mask); if (!cpumask_empty(&cpu_mask)) - xc2((smpfunc_t) local_ops->sig_insns, - (unsigned long) mm, insn_addr); + xc2(local_ops->sig_insns, (unsigned long)mm, insn_addr); local_ops->sig_insns(mm, insn_addr); } diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c index cc19e09b0fa1e545b629050eded90701e50267e0..ae9a86cb6f3d978245005c06777f12990b9fcad3 100644 --- a/arch/sparc/vdso/vma.c +++ b/arch/sparc/vdso/vma.c @@ -354,7 +354,7 @@ static unsigned long vdso_addr(unsigned long start, unsigned int len) unsigned int offset; /* This loses some more bits than a modulo, but is cheaper */ - offset = get_random_int() & (PTRS_PER_PTE - 1); + offset = prandom_u32_max(PTRS_PER_PTE); return start + (offset << PAGE_SHIFT); } diff --git a/arch/um/Kconfig b/arch/um/Kconfig index 78de31ac1da7c66422153cb6bca05ce6dab801f2..ad4ff3b0e91e5775b0642160a0d2a258ea2f8dff 100644 --- a/arch/um/Kconfig +++ b/arch/um/Kconfig @@ -6,6 +6,7 @@ config UML bool default y select ARCH_EPHEMERAL_INODES + select ARCH_HAS_FORTIFY_SOURCE select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_KCOV select ARCH_HAS_STRNCPY_FROM_USER diff --git a/arch/um/Makefile b/arch/um/Makefile index f2fe63bfd819f5f07971d4e4d50cbecec983173d..f1d4d67157be0bf568ca476aed3508b9c9fd2f8f 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -132,10 +132,18 @@ export LDS_ELF_FORMAT := $(ELF_FORMAT) # The wrappers will select whether using "malloc" or the kernel allocator. LINK_WRAPS = -Wl,--wrap,malloc -Wl,--wrap,free -Wl,--wrap,calloc +# Avoid binutils 2.39+ warnings by marking the stack non-executable and +# ignorning warnings for the kallsyms sections. +LDFLAGS_EXECSTACK = -z noexecstack +ifeq ($(CONFIG_LD_IS_BFD),y) +LDFLAGS_EXECSTACK += $(call ld-option,--no-warn-rwx-segments) +endif + LD_FLAGS_CMDLINE = $(foreach opt,$(KBUILD_LDFLAGS),-Wl,$(opt)) # Used by link-vmlinux.sh which has special support for um link export CFLAGS_vmlinux := $(LINK-y) $(LINK_WRAPS) $(LD_FLAGS_CMDLINE) +export LDFLAGS_vmlinux := $(LDFLAGS_EXECSTACK) # When cleaning we don't include .config, so we don't include # TT or skas makefiles and don't clean skas_ptregs.h. diff --git a/arch/um/configs/i386_defconfig b/arch/um/configs/i386_defconfig index fb51bd206dbed329e45b885a3de7318bbb4afe7c..c0162286d68b7e6f70317af06c501bf8e3a2b454 100644 --- a/arch/um/configs/i386_defconfig +++ b/arch/um/configs/i386_defconfig @@ -69,5 +69,5 @@ CONFIG_JOLIET=y CONFIG_PROC_KCORE=y CONFIG_TMPFS=y CONFIG_NLS=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/um/configs/x86_64_defconfig b/arch/um/configs/x86_64_defconfig index 477b8731742430a67c4b6eb6084ba79cde72ebc6..bec6e5d956873089736bcfc1ef0523103d46c678 100644 --- a/arch/um/configs/x86_64_defconfig +++ b/arch/um/configs/x86_64_defconfig @@ -67,6 +67,6 @@ CONFIG_JOLIET=y CONFIG_PROC_KCORE=y CONFIG_TMPFS=y CONFIG_NLS=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_FRAME_WARN=1024 CONFIG_DEBUG_KERNEL=y diff --git a/arch/um/drivers/chan.h b/arch/um/drivers/chan.h index c37cc4f26f91c7e592978337352a1c8e60a4b7cf..3fec3b8406e98a2bad48c19630ce0cd801f738b4 100644 --- a/arch/um/drivers/chan.h +++ b/arch/um/drivers/chan.h @@ -36,7 +36,6 @@ extern int console_write_chan(struct chan *chan, const char *buf, int len); extern int console_open_chan(struct line *line, struct console *co); extern void deactivate_chan(struct chan *chan, int irq); -extern void reactivate_chan(struct chan *chan, int irq); extern void chan_enable_winch(struct chan *chan, struct tty_port *port); extern int enable_chan(struct line *line); extern void close_chan(struct line *line); diff --git a/arch/um/drivers/mconsole_kern.c b/arch/um/drivers/mconsole_kern.c index 8ca67a69268306c8c3ca17857d5efcd9c380e0e6..5026e7b9adfe54960544600244e5b6b59cc15c7e 100644 --- a/arch/um/drivers/mconsole_kern.c +++ b/arch/um/drivers/mconsole_kern.c @@ -283,7 +283,7 @@ struct unplugged_pages { }; static DEFINE_MUTEX(plug_mem_mutex); -static unsigned long long unplugged_pages_count = 0; +static unsigned long long unplugged_pages_count; static LIST_HEAD(unplugged_pages); static int unplug_index = UNPLUGGED_PER_PAGE; @@ -846,13 +846,12 @@ static int notify_panic(struct notifier_block *self, unsigned long unused1, mconsole_notify(notify_socket, MCONSOLE_PANIC, message, strlen(message) + 1); - return 0; + return NOTIFY_DONE; } static struct notifier_block panic_exit_notifier = { - .notifier_call = notify_panic, - .next = NULL, - .priority = 1 + .notifier_call = notify_panic, + .priority = INT_MAX, /* run as soon as possible */ }; static int add_notifier(void) diff --git a/arch/um/drivers/mmapper_kern.c b/arch/um/drivers/mmapper_kern.c index 0bf78ff8901102f55c19369fce6f030932b0ef2d..807cd33587405f47648dd8a37de184df3931b67b 100644 --- a/arch/um/drivers/mmapper_kern.c +++ b/arch/um/drivers/mmapper_kern.c @@ -122,7 +122,7 @@ static int __init mmapper_init(void) return 0; } -static void mmapper_exit(void) +static void __exit mmapper_exit(void) { misc_deregister(&mmapper_dev); } diff --git a/arch/um/drivers/net_kern.c b/arch/um/drivers/net_kern.c index 59331384c2d38914ac811356e832ee9ed2f1fa26..3d7836c46507010b03f057873665a090e3afa40d 100644 --- a/arch/um/drivers/net_kern.c +++ b/arch/um/drivers/net_kern.c @@ -265,7 +265,7 @@ static void uml_net_poll_controller(struct net_device *dev) static void uml_net_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); } static const struct ethtool_ops uml_net_ethtool_ops = { diff --git a/arch/um/drivers/ssl.c b/arch/um/drivers/ssl.c index 8514966778d5376ff4a209ee4ecff11f8a1234e7..277cea3d30eb5946d55628ad4d91d8b49d73dfcc 100644 --- a/arch/um/drivers/ssl.c +++ b/arch/um/drivers/ssl.c @@ -106,7 +106,7 @@ static const struct tty_operations ssl_ops = { /* Changed by ssl_init and referenced by ssl_exit, which are both serialized * by being an initcall and exitcall, respectively. */ -static int ssl_init_done = 0; +static int ssl_init_done; static void ssl_console_write(struct console *c, const char *string, unsigned len) diff --git a/arch/um/drivers/stdio_console.c b/arch/um/drivers/stdio_console.c index 489d5a746ed3363ce90c6a8e698ba718b0981ae5..1c239737d88ec75644151f19341f9362103952ce 100644 --- a/arch/um/drivers/stdio_console.c +++ b/arch/um/drivers/stdio_console.c @@ -88,7 +88,7 @@ static int con_remove(int n, char **error_out) } /* Set in an initcall, checked in an exitcall */ -static int con_init_done = 0; +static int con_init_done; static int con_install(struct tty_driver *driver, struct tty_struct *tty) { diff --git a/arch/um/drivers/ubd_kern.c b/arch/um/drivers/ubd_kern.c index eb2d2f0f0bccace7c6095b629f5acc667610146a..f4c1e6e97ad520ef0e5cdabd622ec473862c7b72 100644 --- a/arch/um/drivers/ubd_kern.c +++ b/arch/um/drivers/ubd_kern.c @@ -1555,7 +1555,7 @@ static void do_io(struct io_thread_req *req, struct io_desc *desc) int kernel_fd = -1; /* Only changed by the io thread. XXX: currently unused. */ -static int io_count = 0; +static int io_count; int io_thread(void *arg) { diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 54826531274366fc0d98e9cd8a99855da7610c53..ded7c47d2fbe5a87da2ff550224694baa95d6053 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1372,7 +1372,7 @@ static void vector_net_poll_controller(struct net_device *dev) static void vector_net_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); } static int vector_net_load_bpf_flash(struct net_device *dev, diff --git a/arch/um/drivers/virt-pci.c b/arch/um/drivers/virt-pci.c index 02784702318419227156d7a0b199171bc4abcb79..acb55b302b14c34d7dfe14fb6ace9067d5441308 100644 --- a/arch/um/drivers/virt-pci.c +++ b/arch/um/drivers/virt-pci.c @@ -857,7 +857,7 @@ void *pci_root_bus_fwnode(struct pci_bus *bus) return um_pci_fwnode; } -static int um_pci_init(void) +static int __init um_pci_init(void) { int err, i; @@ -940,7 +940,7 @@ free: } module_init(um_pci_init); -static void um_pci_exit(void) +static void __exit um_pci_exit(void) { unregister_virtio_driver(&um_pci_virtio_driver); irq_domain_remove(um_pci_msi_domain); diff --git a/arch/um/drivers/virtio_uml.c b/arch/um/drivers/virtio_uml.c index e719af8bdf56d3e9b6f98e0cf685f1ef0a78715d..588930a0ced17d8ae1026e62474e15cc41b7f881 100644 --- a/arch/um/drivers/virtio_uml.c +++ b/arch/um/drivers/virtio_uml.c @@ -374,45 +374,48 @@ static irqreturn_t vu_req_read_message(struct virtio_uml_device *vu_dev, u8 extra_payload[512]; } msg; int rc; + irqreturn_t irq_rc = IRQ_NONE; - rc = vhost_user_recv_req(vu_dev, &msg.msg, - sizeof(msg.msg.payload) + - sizeof(msg.extra_payload)); - - vu_dev->recv_rc = rc; - if (rc) - return IRQ_NONE; - - switch (msg.msg.header.request) { - case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG: - vu_dev->config_changed_irq = true; - response = 0; - break; - case VHOST_USER_SLAVE_VRING_CALL: - virtio_device_for_each_vq((&vu_dev->vdev), vq) { - if (vq->index == msg.msg.payload.vring_state.index) { - response = 0; - vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index); - break; + while (1) { + rc = vhost_user_recv_req(vu_dev, &msg.msg, + sizeof(msg.msg.payload) + + sizeof(msg.extra_payload)); + if (rc) + break; + + switch (msg.msg.header.request) { + case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG: + vu_dev->config_changed_irq = true; + response = 0; + break; + case VHOST_USER_SLAVE_VRING_CALL: + virtio_device_for_each_vq((&vu_dev->vdev), vq) { + if (vq->index == msg.msg.payload.vring_state.index) { + response = 0; + vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index); + break; + } } + break; + case VHOST_USER_SLAVE_IOTLB_MSG: + /* not supported - VIRTIO_F_ACCESS_PLATFORM */ + case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: + /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */ + default: + vu_err(vu_dev, "unexpected slave request %d\n", + msg.msg.header.request); } - break; - case VHOST_USER_SLAVE_IOTLB_MSG: - /* not supported - VIRTIO_F_ACCESS_PLATFORM */ - case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG: - /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */ - default: - vu_err(vu_dev, "unexpected slave request %d\n", - msg.msg.header.request); - } - - if (ev && !vu_dev->suspended) - time_travel_add_irq_event(ev); - if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY) - vhost_user_reply(vu_dev, &msg.msg, response); + if (ev && !vu_dev->suspended) + time_travel_add_irq_event(ev); - return IRQ_HANDLED; + if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY) + vhost_user_reply(vu_dev, &msg.msg, response); + irq_rc = IRQ_HANDLED; + }; + /* mask EAGAIN as we try non-blocking read until socket is empty */ + vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc; + return irq_rc; } static irqreturn_t vu_req_interrupt(int irq, void *data) diff --git a/arch/um/include/asm/processor-generic.h b/arch/um/include/asm/processor-generic.h index d0fc1862da957dea0c5c0655e441c6a2885e05d8..bb5f06480da95237ff37a5b58394522f40d031cf 100644 --- a/arch/um/include/asm/processor-generic.h +++ b/arch/um/include/asm/processor-generic.h @@ -55,10 +55,6 @@ struct thread_struct { .request = { 0 } \ } -static inline void release_thread(struct task_struct *task) -{ -} - /* * User space process size: 3GB (default). */ diff --git a/arch/um/kernel/physmem.c b/arch/um/kernel/physmem.c index e7c7b53a1435bacc8a118cfc2f7d1499534ed501..91485119ae67a7f6282e1792b8b93974e63ce332 100644 --- a/arch/um/kernel/physmem.c +++ b/arch/um/kernel/physmem.c @@ -169,7 +169,7 @@ __uml_setup("iomem=", parse_iomem, ); /* - * This list is constructed in parse_iomem and addresses filled in in + * This list is constructed in parse_iomem and addresses filled in * setup_iomem, both of which run during early boot. Afterwards, it's * unchanged. */ diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 80b90b1276a1937030f570df079d9f72cb11b72f..010bc422a09dd0ad29093b661f9afe5f7f32e54f 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -356,7 +356,7 @@ int singlestepping(void * t) unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() % 8192; + sp -= prandom_u32_max(8192); return sp & ~0xf; } #endif diff --git a/arch/um/kernel/sysrq.c b/arch/um/kernel/sysrq.c index 7452f70d50d06924dbe0039db140ca8a0d0db2db..746715379f12a84abcfe991b3a23218eec7e0fc3 100644 --- a/arch/um/kernel/sysrq.c +++ b/arch/um/kernel/sysrq.c @@ -48,7 +48,8 @@ void show_stack(struct task_struct *task, unsigned long *stack, break; if (i && ((i % STACKSLOTS_PER_LINE) == 0)) pr_cont("\n"); - pr_cont(" %08lx", *stack++); + pr_cont(" %08lx", READ_ONCE_NOCHECK(*stack)); + stack++; } printk("%sCall Trace:\n", loglvl); diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index bc38f79ca3a38242a5165685910528e18c1f98a4..ad449173a1a1cdf20f3973fdfea96539392bc343 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -584,21 +584,19 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, void flush_tlb_mm(struct mm_struct *mm) { - struct vm_area_struct *vma = mm->mmap; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); - while (vma != NULL) { + for_each_vma(vmi, vma) fix_range(mm, vma->vm_start, vma->vm_end, 0); - vma = vma->vm_next; - } } void force_flush_all(void) { struct mm_struct *mm = current->mm; - struct vm_area_struct *vma = mm->mmap; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); - while (vma != NULL) { + for_each_vma(vmi, vma) fix_range(mm, vma->vm_start, vma->vm_end, 1); - vma = vma->vm_next; - } } diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index e0de60e503b983c2ab6a2f215c688bf14161e59e..8adf8e89b25588763799f12a676958a3f99e75ae 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -33,7 +33,7 @@ #include "um_arch.h" #define DEFAULT_COMMAND_LINE_ROOT "root=98:0" -#define DEFAULT_COMMAND_LINE_CONSOLE "console=tty" +#define DEFAULT_COMMAND_LINE_CONSOLE "console=tty0" /* Changed in add_arg and setup_arch, which run before SMP is started */ static char __initdata command_line[COMMAND_LINE_SIZE] = { 0 }; @@ -96,7 +96,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) static void *c_start(struct seq_file *m, loff_t *pos) { - return *pos < NR_CPUS ? cpu_data + *pos : NULL; + return *pos < nr_cpu_ids ? cpu_data + *pos : NULL; } static void *c_next(struct seq_file *m, void *v, loff_t *pos) @@ -132,7 +132,7 @@ static int have_root __initdata; static int have_console __initdata; /* Set in uml_mem_setup and modified in linux_main */ -long long physmem_size = 32 * 1024 * 1024; +long long physmem_size = 64 * 1024 * 1024; EXPORT_SYMBOL(physmem_size); static const char *usage_string = @@ -247,13 +247,13 @@ static int panic_exit(struct notifier_block *self, unsigned long unused1, bust_spinlocks(0); uml_exitcode = 1; os_dump_core(); - return 0; + + return NOTIFY_DONE; } static struct notifier_block panic_exit_notifier = { - .notifier_call = panic_exit, - .next = NULL, - .priority = 0 + .notifier_call = panic_exit, + .priority = INT_MAX - 1, /* run as 2nd notifier, won't return */ }; void uml_finishsetup(void) @@ -416,7 +416,7 @@ void __init setup_arch(char **cmdline_p) read_initrd(); paging_init(); - strlcpy(boot_command_line, command_line, COMMAND_LINE_SIZE); + strscpy(boot_command_line, command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; setup_hostinfo(host_info, sizeof host_info); diff --git a/arch/um/kernel/umid.c b/arch/um/kernel/umid.c index 8031a038eb5889b35da5582ceceba7688c13f1c9..72bc60ade347bc5cd029601aeb5b6e54ddfe1d20 100644 --- a/arch/um/kernel/umid.c +++ b/arch/um/kernel/umid.c @@ -9,7 +9,7 @@ #include /* Changed by set_umid_arg */ -static int umid_inited = 0; +static int umid_inited; static int __init set_umid_arg(char *name, int *add) { diff --git a/arch/um/os-Linux/user_syms.c b/arch/um/os-Linux/user_syms.c index cb667c9225abd7ae344fe602e9d0ac619913ad95..fd575ecbcaece3628b5ede5353adbea11a42d1a3 100644 --- a/arch/um/os-Linux/user_syms.c +++ b/arch/um/os-Linux/user_syms.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#define __NO_FORTIFY #include #include diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f9920f1341c8d46d0680c81eda74f568f35092ac..6d1879ef933a2c129ad05d2f7c3254b0347c2e38 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -85,6 +85,7 @@ config X86 select ARCH_HAS_PMEM_API if X86_64 select ARCH_HAS_PTE_DEVMAP if X86_64 select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_NONLEAF_PMD_YOUNG if PGTABLE_LEVELS > 2 select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64 select ARCH_HAS_COPY_MC if X86_64 select ARCH_HAS_SET_MEMORY @@ -107,6 +108,8 @@ config X86 select ARCH_SUPPORTS_PAGE_TABLE_CHECK if X86_64 select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 + select ARCH_SUPPORTS_CFI_CLANG if X86_64 + select ARCH_USES_CFI_TRAPS if X86_64 && CFI_CLANG select ARCH_SUPPORTS_LTO_CLANG select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_USE_BUILTIN_BSWAP @@ -128,7 +131,9 @@ config X86 select CLKEVT_I8253 select CLOCKSOURCE_VALIDATE_LAST_CYCLE select CLOCKSOURCE_WATCHDOG - select DCACHE_WORD_ACCESS + # Word-size accesses may read uninitialized data past the trailing \0 + # in strings and cause false KMSAN reports. + select DCACHE_WORD_ACCESS if !KMSAN select DYNAMIC_SIGFRAME select EDAC_ATOMIC_SCRUB select EDAC_SUPPORT @@ -166,6 +171,7 @@ config X86 select HAVE_ARCH_KASAN if X86_64 select HAVE_ARCH_KASAN_VMALLOC if X86_64 select HAVE_ARCH_KFENCE + select HAVE_ARCH_KMSAN if X86_64 select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT @@ -257,6 +263,7 @@ config X86 select HAVE_STATIC_CALL_INLINE if HAVE_OBJTOOL select HAVE_PREEMPT_DYNAMIC_CALL select HAVE_RSEQ + select HAVE_RUST if X86_64 select HAVE_SYSCALL_TRACEPOINTS select HAVE_UACCESS_VALIDATION if HAVE_OBJTOOL select HAVE_UNSTABLE_SCHED_CLOCK @@ -284,6 +291,7 @@ config X86 select PROC_PID_ARCH_STATUS if PROC_FS select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI + select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE config INSTRUCTION_DECODER def_bool y @@ -324,6 +332,10 @@ config GENERIC_ISA_DMA def_bool y depends on ISA_DMA_API +config GENERIC_CSUM + bool + default y if KMSAN || KASAN + config GENERIC_BUG def_bool y depends on BUG @@ -448,6 +460,11 @@ config X86_X2APIC This allows 32-bit apic IDs (so it can support very large systems), and accesses the local apic via MSRs not via mmio. + Some Intel systems circa 2022 and later are locked into x2APIC mode + and can not fall back to the legacy APIC modes if SGX or TDX are + enabled in the BIOS. They will be unable to boot without enabling + this option. + If you don't know what to do here, say N. config X86_MPPARSE @@ -1919,7 +1936,7 @@ endchoice config X86_SGX bool "Software Guard eXtensions (SGX)" - depends on X86_64 && CPU_SUP_INTEL + depends on X86_64 && CPU_SUP_INTEL && X86_X2APIC depends on CRYPTO=y depends on CRYPTO_SHA256=y select SRCU @@ -2569,7 +2586,7 @@ menuconfig APM 1) make sure that you have enough swap space and that it is enabled. - 2) pass the "no-hlt" option to the kernel + 2) pass the "idle=poll" option to the kernel 3) switch on floating point emulation in the kernel and pass the "no387" option to the kernel 4) pass the "floppy=nodma" option to the kernel diff --git a/arch/x86/Makefile b/arch/x86/Makefile index bafbd905e6e7c4857064156d0d70a8275b9dd19d..415a5d138de47c37c90890a71056775d023f5d7d 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -68,6 +68,7 @@ export BITS # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383 # KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx +KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2 ifeq ($(CONFIG_X86_KERNEL_IBT),y) # @@ -155,8 +156,17 @@ else cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic KBUILD_CFLAGS += $(cflags-y) + rustflags-$(CONFIG_MK8) += -Ctarget-cpu=k8 + rustflags-$(CONFIG_MPSC) += -Ctarget-cpu=nocona + rustflags-$(CONFIG_MCORE2) += -Ctarget-cpu=core2 + rustflags-$(CONFIG_MATOM) += -Ctarget-cpu=atom + rustflags-$(CONFIG_GENERIC_CPU) += -Ztune-cpu=generic + KBUILD_RUSTFLAGS += $(rustflags-y) + KBUILD_CFLAGS += -mno-red-zone KBUILD_CFLAGS += -mcmodel=kernel + KBUILD_RUSTFLAGS += -Cno-redzone=y + KBUILD_RUSTFLAGS += -Ccode-model=kernel endif # @@ -234,11 +244,6 @@ archheaders: ### # Kernel objects -head-y := arch/x86/kernel/head_$(BITS).o -head-y += arch/x86/kernel/head$(BITS).o -head-y += arch/x86/kernel/ebda.o -head-y += arch/x86/kernel/platform-quirks.o - libs-y += arch/x86/lib/ # drivers-y are linked after core-y diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index ffec8bb01ba8c20372b8cf4a37bd6810f054d0cc..9860ca5979f8aa508ced8ff09db0632c2e3f7dcc 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -12,6 +12,7 @@ # Sanitizer runtimes are unavailable and cannot be linked for early boot code. KASAN_SANITIZE := n KCSAN_SANITIZE := n +KMSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Kernel does not boot with kcov instrumentation here. diff --git a/arch/x86/boot/bitops.h b/arch/x86/boot/bitops.h index 02e1dea11d94bd397a8cc152b77783f7ece06017..8518ae214c9b335917e00c3d976369d6d71edaff 100644 --- a/arch/x86/boot/bitops.h +++ b/arch/x86/boot/bitops.h @@ -19,13 +19,13 @@ static inline bool constant_test_bit(int nr, const void *addr) { - const u32 *p = (const u32 *)addr; + const u32 *p = addr; return ((1UL << (nr & 31)) & (p[nr >> 5])) != 0; } static inline bool variable_test_bit(int nr, const void *addr) { bool v; - const u32 *p = (const u32 *)addr; + const u32 *p = addr; asm("btl %2,%1" CC_SET(c) : CC_OUT(c) (v) : "m" (*p), "Ir" (nr)); return v; diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 35ce1a64068b76d2871a31c65f28324ecf18f17f..3a261abb6d158d62c9592f115ec39542d23ca2ab 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -20,6 +20,7 @@ # Sanitizer runtimes are unavailable and cannot be linked for early boot code. KASAN_SANITIZE := n KCSAN_SANITIZE := n +KMSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c index 4a3f223973f40f85bc633359f1a41c92ce6ca084..e476bcbd9b422f6e2a980b59ac59d4e0cb30ce0d 100644 --- a/arch/x86/boot/compressed/kaslr.c +++ b/arch/x86/boot/compressed/kaslr.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #define _SETUP diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 4910bf230d7b4afbb44f83d958e0b1902fe26458..62208ec04ca4b800e7cf3eb8949f6cf6a228fee2 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -132,7 +132,17 @@ void snp_set_page_private(unsigned long paddr); void snp_set_page_shared(unsigned long paddr); void sev_prep_identity_maps(unsigned long top_level_pgt); #else -static inline void sev_enable(struct boot_params *bp) { } +static inline void sev_enable(struct boot_params *bp) +{ + /* + * bp->cc_blob_address should only be set by boot/compressed kernel. + * Initialize it to 0 unconditionally (thus here in this stub too) to + * ensure that uninitialized values from buggy bootloaders aren't + * propagated. + */ + if (bp) + bp->cc_blob_address = 0; +} static inline void sev_es_shutdown_ghcb(void) { } static inline bool sev_es_check_ghcb_fault(unsigned long address) { diff --git a/arch/x86/boot/compressed/sev.c b/arch/x86/boot/compressed/sev.c index 52f989f6acc281f95815bc76e0976348b5b8f635..c93930d5ccbd0f1b4894012e09c885e20ee1ba0a 100644 --- a/arch/x86/boot/compressed/sev.c +++ b/arch/x86/boot/compressed/sev.c @@ -276,6 +276,14 @@ void sev_enable(struct boot_params *bp) struct msr m; bool snp; + /* + * bp->cc_blob_address should only be set by boot/compressed kernel. + * Initialize it to 0 to ensure that uninitialized values from + * buggy bootloaders aren't propagated. + */ + if (bp) + bp->cc_blob_address = 0; + /* * Setup/preliminary detection of SNP. This will be sanity-checked * against CPUID/MSR values later. diff --git a/arch/x86/boot/version.c b/arch/x86/boot/version.c index a1aaaf6c06a6434d0f8a6b5a8defdc34f225616d..945383f0f606abb850df5442d8cca356d6380042 100644 --- a/arch/x86/boot/version.c +++ b/arch/x86/boot/version.c @@ -11,6 +11,7 @@ */ #include "boot.h" +#include #include #include diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 7207219509f64c8b348fec83333d4f371960b9a0..3cf34912abfe2c961fbfb851c5e27d5c3759623e 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -27,7 +27,6 @@ CONFIG_CGROUP_MISC=y CONFIG_CGROUP_DEBUG=y CONFIG_BLK_DEV_INITRD=y CONFIG_KALLSYMS_ALL=y -# CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_SMP=y CONFIG_HYPERVISOR_GUEST=y @@ -44,6 +43,7 @@ CONFIG_EFI_STUB=y CONFIG_HZ_1000=y CONFIG_KEXEC=y CONFIG_CRASH_DUMP=y +# CONFIG_RETHUNK is not set CONFIG_HIBERNATION=y CONFIG_PM_DEBUG=y CONFIG_PM_TRACE_RTC=y @@ -62,6 +62,7 @@ CONFIG_BLK_CGROUP_IOLATENCY=y CONFIG_BLK_CGROUP_IOCOST=y CONFIG_BLK_CGROUP_IOPRIO=y CONFIG_BINFMT_MISC=y +# CONFIG_COMPAT_BRK is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -269,9 +270,10 @@ CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y CONFIG_PRINTK_TIME=y +CONFIG_DEBUG_KERNEL=y CONFIG_FRAME_WARN=1024 CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_WX=y CONFIG_DEBUG_STACK_USAGE=y # CONFIG_SCHED_DEBUG is not set CONFIG_SCHEDSTATS=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 5ce67b73e2182742037f03d778ba15d68aaa9bc0..27759236fd60ec1cd200d4cf6e430e353fe6aa76 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -26,7 +26,6 @@ CONFIG_CGROUP_MISC=y CONFIG_CGROUP_DEBUG=y CONFIG_BLK_DEV_INITRD=y CONFIG_KALLSYMS_ALL=y -# CONFIG_COMPAT_BRK is not set CONFIG_PROFILING=y CONFIG_SMP=y CONFIG_HYPERVISOR_GUEST=y @@ -62,6 +61,7 @@ CONFIG_BLK_CGROUP_IOLATENCY=y CONFIG_BLK_CGROUP_IOCOST=y CONFIG_BLK_CGROUP_IOPRIO=y CONFIG_BINFMT_MISC=y +# CONFIG_COMPAT_BRK is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -267,8 +267,9 @@ CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y CONFIG_PRINTK_TIME=y -CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_WX=y CONFIG_DEBUG_STACK_USAGE=y # CONFIG_SCHED_DEBUG is not set CONFIG_SCHEDSTATS=y diff --git a/arch/x86/configs/xen.config b/arch/x86/configs/xen.config index d9fc7139fd46edc708565fa27135ee6f11ca56a3..581296255b39e403246d4ad076786d10cb86dc60 100644 --- a/arch/x86/configs/xen.config +++ b/arch/x86/configs/xen.config @@ -14,7 +14,6 @@ CONFIG_CPU_FREQ=y # x86 xen specific config options CONFIG_XEN_PVH=y -CONFIG_XEN_MAX_DOMAIN_MEMORY=500 CONFIG_XEN_SAVE_RESTORE=y # CONFIG_XEN_DEBUG_FS is not set CONFIG_XEN_MCE_LOG=y diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..71c4c473d34b898ddddac299e2d6f294b41b8477 --- /dev/null +++ b/arch/x86/crypto/Kconfig @@ -0,0 +1,484 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (x86)" + +config CRYPTO_CURVE25519_X86 + tristate "Public key crypto: Curve25519 (ADX)" + depends on X86 && 64BIT + select CRYPTO_LIB_CURVE25519_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + help + Curve25519 algorithm + + Architecture: x86_64 using: + - ADX (large integer arithmetic) + +config CRYPTO_AES_NI_INTEL + tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTR, XTS, GCM (AES-NI)" + depends on X86 + select CRYPTO_AEAD + select CRYPTO_LIB_AES + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + help + Block cipher: AES cipher algorithms + AEAD cipher: AES with GCM + Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTR, XTS + + Architecture: x86 (32-bit and 64-bit) using: + - AES-NI (AES new instructions) + +config CRYPTO_BLOWFISH_X86_64 + tristate "Ciphers: Blowfish, modes: ECB, CBC" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_BLOWFISH_COMMON + imply CRYPTO_CTR + help + Block cipher: Blowfish cipher algorithm + Length-preserving ciphers: Blowfish with ECB and CBC modes + + Architecture: x86_64 + +config CRYPTO_CAMELLIA_X86_64 + tristate "Ciphers: Camellia with modes: ECB, CBC" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + imply CRYPTO_CTR + help + Block cipher: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: x86_64 + +config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 + tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAMELLIA_X86_64 + select CRYPTO_SIMD + imply CRYPTO_XTS + help + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) + +config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 + tristate "Ciphers: Camellia with modes: ECB, CBC (AES-NI/AVX2)" + depends on X86 && 64BIT + select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 + help + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX2 (Advanced Vector Extensions 2) + +config CRYPTO_CAST5_AVX_X86_64 + tristate "Ciphers: CAST5 with modes: ECB, CBC (AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAST5 + select CRYPTO_CAST_COMMON + select CRYPTO_SIMD + imply CRYPTO_CTR + help + Length-preserving ciphers: CAST5 (CAST-128) cipher algorithm + (RFC2144) with ECB and CBC modes + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + Processes 16 blocks in parallel. + +config CRYPTO_CAST6_AVX_X86_64 + tristate "Ciphers: CAST6 with modes: ECB, CBC (AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_CAST6 + select CRYPTO_CAST_COMMON + select CRYPTO_SIMD + imply CRYPTO_XTS + imply CRYPTO_CTR + help + Length-preserving ciphers: CAST6 (CAST-256) cipher algorithm + (RFC2612) with ECB and CBC modes + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + Processes eight blocks in parallel. + +config CRYPTO_DES3_EDE_X86_64 + tristate "Ciphers: Triple DES EDE with modes: ECB, CBC" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_LIB_DES + imply CRYPTO_CTR + help + Block cipher: Triple DES EDE (FIPS 46-3) cipher algorithm + Length-preserving ciphers: Triple DES EDE with ECB and CBC modes + + Architecture: x86_64 + + Processes one or three blocks in parallel. + +config CRYPTO_SERPENT_SSE2_X86_64 + tristate "Ciphers: Serpent with modes: ECB, CBC (SSE2)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_CTR + help + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes + + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) + + Processes eight blocks in parallel. + +config CRYPTO_SERPENT_SSE2_586 + tristate "Ciphers: Serpent with modes: ECB, CBC (32-bit with SSE2)" + depends on X86 && !64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_CTR + help + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes + + Architecture: x86 (32-bit) using: + - SSE2 (Streaming SIMD Extensions 2) + + Processes four blocks in parallel. + +config CRYPTO_SERPENT_AVX_X86_64 + tristate "Ciphers: Serpent with modes: ECB, CBC (AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SERPENT + select CRYPTO_SIMD + imply CRYPTO_XTS + imply CRYPTO_CTR + help + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + Processes eight blocks in parallel. + +config CRYPTO_SERPENT_AVX2_X86_64 + tristate "Ciphers: Serpent with modes: ECB, CBC (AVX2)" + depends on X86 && 64BIT + select CRYPTO_SERPENT_AVX_X86_64 + help + Length-preserving ciphers: Serpent cipher algorithm + with ECB and CBC modes + + Architecture: x86_64 using: + - AVX2 (Advanced Vector Extensions 2) + + Processes 16 blocks in parallel. + +config CRYPTO_SM4_AESNI_AVX_X86_64 + tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_SM4 + help + Length-preserving ciphers: SM4 cipher algorithms + (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) + + Through two affine transforms, + we can use the AES S-Box to simulate the SM4 S-Box to achieve the + effect of instruction acceleration. + + If unsure, say N. + +config CRYPTO_SM4_AESNI_AVX2_X86_64 + tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_SM4 + select CRYPTO_SM4_AESNI_AVX_X86_64 + help + Length-preserving ciphers: SM4 cipher algorithms + (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX2 (Advanced Vector Extensions 2) + + Through two affine transforms, + we can use the AES S-Box to simulate the SM4 S-Box to achieve the + effect of instruction acceleration. + + If unsure, say N. + +config CRYPTO_TWOFISH_586 + tristate "Ciphers: Twofish (32-bit)" + depends on (X86 || UML_X86) && !64BIT + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR + help + Block cipher: Twofish cipher algorithm + + Architecture: x86 (32-bit) + +config CRYPTO_TWOFISH_X86_64 + tristate "Ciphers: Twofish" + depends on (X86 || UML_X86) && 64BIT + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR + help + Block cipher: Twofish cipher algorithm + + Architecture: x86_64 + +config CRYPTO_TWOFISH_X86_64_3WAY + tristate "Ciphers: Twofish with modes: ECB, CBC (3-way parallel)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_TWOFISH_COMMON + select CRYPTO_TWOFISH_X86_64 + help + Length-preserving cipher: Twofish cipher algorithm + with ECB and CBC modes + + Architecture: x86_64 + + Processes three blocks in parallel, better utilizing resources of + out-of-order CPUs. + +config CRYPTO_TWOFISH_AVX_X86_64 + tristate "Ciphers: Twofish with modes: ECB, CBC (AVX)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_TWOFISH_COMMON + select CRYPTO_TWOFISH_X86_64 + select CRYPTO_TWOFISH_X86_64_3WAY + imply CRYPTO_XTS + help + Length-preserving cipher: Twofish cipher algorithm + with ECB and CBC modes + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + Processes eight blocks in parallel. + +config CRYPTO_ARIA_AESNI_AVX_X86_64 + tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX/GFNI)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_SIMD + select CRYPTO_ALGAPI + select CRYPTO_ARIA + help + Length-preserving cipher: ARIA cipher algorithms + (RFC 5794) with ECB and CTR modes + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - AVX (Advanced Vector Extensions) + - GFNI (Galois Field New Instructions) + + Processes 16 blocks in parallel. + +config CRYPTO_CHACHA20_X86_64 + tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)" + depends on X86 && 64BIT + select CRYPTO_SKCIPHER + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_ARCH_HAVE_LIB_CHACHA + help + Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 + stream cipher algorithms + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX2 (Advanced Vector Extensions 2) + - AVX-512VL (Advanced Vector Extensions-512VL) + +config CRYPTO_AEGIS128_AESNI_SSE2 + tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE2)" + depends on X86 && 64BIT + select CRYPTO_AEAD + select CRYPTO_SIMD + help + AEGIS-128 AEAD algorithm + + Architecture: x86_64 using: + - AES-NI (AES New Instructions) + - SSE2 (Streaming SIMD Extensions 2) + +config CRYPTO_NHPOLY1305_SSE2 + tristate "Hash functions: NHPoly1305 (SSE2)" + depends on X86 && 64BIT + select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function for Adiantum + + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) + +config CRYPTO_NHPOLY1305_AVX2 + tristate "Hash functions: NHPoly1305 (AVX2)" + depends on X86 && 64BIT + select CRYPTO_NHPOLY1305 + help + NHPoly1305 hash function for Adiantum + + Architecture: x86_64 using: + - AVX2 (Advanced Vector Extensions 2) + +config CRYPTO_BLAKE2S_X86 + bool "Hash functions: BLAKE2s (SSSE3/AVX-512)" + depends on X86 && 64BIT + select CRYPTO_LIB_BLAKE2S_GENERIC + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s cryptographic hash function (RFC 7693) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX-512 (Advanced Vector Extensions-512) + +config CRYPTO_POLYVAL_CLMUL_NI + tristate "Hash functions: POLYVAL (CLMUL-NI)" + depends on X86 && 64BIT + select CRYPTO_POLYVAL + help + POLYVAL hash function for HCTR2 + + Architecture: x86_64 using: + - CLMUL-NI (carry-less multiplication new instructions) + +config CRYPTO_POLY1305_X86_64 + tristate "Hash functions: Poly1305 (SSE2/AVX2)" + depends on X86 && 64BIT + select CRYPTO_LIB_POLY1305_GENERIC + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + help + Poly1305 authenticator algorithm (RFC7539) + + Architecture: x86_64 using: + - SSE2 (Streaming SIMD Extensions 2) + - AVX2 (Advanced Vector Extensions 2) + +config CRYPTO_SHA1_SSSE3 + tristate "Hash functions: SHA-1 (SSSE3/AVX/AVX2/SHA-NI)" + depends on X86 && 64BIT + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash algorithm (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) + - SHA-NI (SHA Extensions New Instructions) + +config CRYPTO_SHA256_SSSE3 + tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)" + depends on X86 && 64BIT + select CRYPTO_SHA256 + select CRYPTO_HASH + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) + - SHA-NI (SHA Extensions New Instructions) + +config CRYPTO_SHA512_SSSE3 + tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" + depends on X86 && 64BIT + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-384 and SHA-512 secure hash algorithms (FIPS 180) + + Architecture: x86_64 using: + - SSSE3 (Supplemental SSE3) + - AVX (Advanced Vector Extensions) + - AVX2 (Advanced Vector Extensions 2) + +config CRYPTO_SM3_AVX_X86_64 + tristate "Hash functions: SM3 (AVX)" + depends on X86 && 64BIT + select CRYPTO_HASH + select CRYPTO_SM3 + help + SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3 + + Architecture: x86_64 using: + - AVX (Advanced Vector Extensions) + + If unsure, say N. + +config CRYPTO_GHASH_CLMUL_NI_INTEL + tristate "Hash functions: GHASH (CLMUL-NI)" + depends on X86 && 64BIT + select CRYPTO_CRYPTD + help + GCM GHASH hash function (NIST SP800-38D) + + Architecture: x86_64 using: + - CLMUL-NI (carry-less multiplication new instructions) + +config CRYPTO_CRC32C_INTEL + tristate "CRC32c (SSE4.2/PCLMULQDQ)" + depends on X86 + select CRYPTO_HASH + help + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) + + Architecture: x86 (32-bit and 64-bit) using: + - SSE4.2 (Streaming SIMD Extensions 4.2) CRC32 instruction + - PCLMULQDQ (carry-less multiplication) + +config CRYPTO_CRC32_PCLMUL + tristate "CRC32 (PCLMULQDQ)" + depends on X86 + select CRYPTO_HASH + select CRC32 + help + CRC32 CRC algorithm (IEEE 802.3) + + Architecture: x86 (32-bit and 64-bit) using: + - PCLMULQDQ (carry-less multiplication) + +config CRYPTO_CRCT10DIF_PCLMUL + tristate "CRCT10DIF (PCLMULQDQ)" + depends on X86 && 64BIT && CRC_T10DIF + select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) + + Architecture: x86_64 using: + - PCLMULQDQ (carry-less multiplication) + +endmenu diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 04d07ab744b2e75dde534d6cc3cabd35f36edd11..3b1d701a4f6c509841998e8b3699096e11ea7dce 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -100,6 +100,9 @@ sm4-aesni-avx-x86_64-y := sm4-aesni-avx-asm_64.o sm4_aesni_avx_glue.o obj-$(CONFIG_CRYPTO_SM4_AESNI_AVX2_X86_64) += sm4-aesni-avx2-x86_64.o sm4-aesni-avx2-x86_64-y := sm4-aesni-avx2-asm_64.o sm4_aesni_avx2_glue.o +obj-$(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64) += aria-aesni-avx-x86_64.o +aria-aesni-avx-x86_64-y := aria-aesni-avx-asm_64.o aria_aesni_avx_glue.o + quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ $(obj)/%.S: $(src)/%.pl FORCE diff --git a/arch/x86/crypto/aria-aesni-avx-asm_64.S b/arch/x86/crypto/aria-aesni-avx-asm_64.S new file mode 100644 index 0000000000000000000000000000000000000000..c75fd7d015ed8c958819f30d363b59b8b972321c --- /dev/null +++ b/arch/x86/crypto/aria-aesni-avx-asm_64.S @@ -0,0 +1,1303 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ARIA Cipher 16-way parallel algorithm (AVX) + * + * Copyright (c) 2022 Taehee Yoo + * + */ + +#include +#include + +/* struct aria_ctx: */ +#define enc_key 0 +#define dec_key 272 +#define rounds 544 + +/* register macros */ +#define CTX %rdi + + +#define BV8(a0, a1, a2, a3, a4, a5, a6, a7) \ + ( (((a0) & 1) << 0) | \ + (((a1) & 1) << 1) | \ + (((a2) & 1) << 2) | \ + (((a3) & 1) << 3) | \ + (((a4) & 1) << 4) | \ + (((a5) & 1) << 5) | \ + (((a6) & 1) << 6) | \ + (((a7) & 1) << 7) ) + +#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7) \ + ( ((l7) << (0 * 8)) | \ + ((l6) << (1 * 8)) | \ + ((l5) << (2 * 8)) | \ + ((l4) << (3 * 8)) | \ + ((l3) << (4 * 8)) | \ + ((l2) << (5 * 8)) | \ + ((l1) << (6 * 8)) | \ + ((l0) << (7 * 8)) ) + +#define inc_le128(x, minus_one, tmp) \ + vpcmpeqq minus_one, x, tmp; \ + vpsubq minus_one, x, x; \ + vpslldq $8, tmp, tmp; \ + vpsubq tmp, x, x; + +#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ + vpand x, mask4bit, tmp0; \ + vpandn x, mask4bit, x; \ + vpsrld $4, x, x; \ + \ + vpshufb tmp0, lo_t, tmp0; \ + vpshufb x, hi_t, x; \ + vpxor tmp0, x, x; + +#define transpose_4x4(x0, x1, x2, x3, t1, t2) \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x1, x0, x0; \ + \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x2; \ + \ + vpunpckhqdq t1, x0, x1; \ + vpunpcklqdq t1, x0, x0; \ + \ + vpunpckhqdq x2, t2, x3; \ + vpunpcklqdq x2, t2, x2; + +#define byteslice_16x16b(a0, b0, c0, d0, \ + a1, b1, c1, d1, \ + a2, b2, c2, d2, \ + a3, b3, c3, d3, \ + st0, st1) \ + vmovdqu d2, st0; \ + vmovdqu d3, st1; \ + transpose_4x4(a0, a1, a2, a3, d2, d3); \ + transpose_4x4(b0, b1, b2, b3, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu a0, st0; \ + vmovdqu a1, st1; \ + transpose_4x4(c0, c1, c2, c3, a0, a1); \ + transpose_4x4(d0, d1, d2, d3, a0, a1); \ + \ + vmovdqu .Lshufb_16x16b, a0; \ + vmovdqu st1, a1; \ + vpshufb a0, a2, a2; \ + vpshufb a0, a3, a3; \ + vpshufb a0, b0, b0; \ + vpshufb a0, b1, b1; \ + vpshufb a0, b2, b2; \ + vpshufb a0, b3, b3; \ + vpshufb a0, a1, a1; \ + vpshufb a0, c0, c0; \ + vpshufb a0, c1, c1; \ + vpshufb a0, c2, c2; \ + vpshufb a0, c3, c3; \ + vpshufb a0, d0, d0; \ + vpshufb a0, d1, d1; \ + vpshufb a0, d2, d2; \ + vpshufb a0, d3, d3; \ + vmovdqu d3, st1; \ + vmovdqu st0, d3; \ + vpshufb a0, d3, a0; \ + vmovdqu d2, st0; \ + \ + transpose_4x4(a0, b0, c0, d0, d2, d3); \ + transpose_4x4(a1, b1, c1, d1, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu b0, st0; \ + vmovdqu b1, st1; \ + transpose_4x4(a2, b2, c2, d2, b0, b1); \ + transpose_4x4(a3, b3, c3, d3, b0, b1); \ + vmovdqu st0, b0; \ + vmovdqu st1, b1; \ + /* does not adjust output bytes inside vectors */ + +#define debyteslice_16x16b(a0, b0, c0, d0, \ + a1, b1, c1, d1, \ + a2, b2, c2, d2, \ + a3, b3, c3, d3, \ + st0, st1) \ + vmovdqu d2, st0; \ + vmovdqu d3, st1; \ + transpose_4x4(a0, a1, a2, a3, d2, d3); \ + transpose_4x4(b0, b1, b2, b3, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu a0, st0; \ + vmovdqu a1, st1; \ + transpose_4x4(c0, c1, c2, c3, a0, a1); \ + transpose_4x4(d0, d1, d2, d3, a0, a1); \ + \ + vmovdqu .Lshufb_16x16b, a0; \ + vmovdqu st1, a1; \ + vpshufb a0, a2, a2; \ + vpshufb a0, a3, a3; \ + vpshufb a0, b0, b0; \ + vpshufb a0, b1, b1; \ + vpshufb a0, b2, b2; \ + vpshufb a0, b3, b3; \ + vpshufb a0, a1, a1; \ + vpshufb a0, c0, c0; \ + vpshufb a0, c1, c1; \ + vpshufb a0, c2, c2; \ + vpshufb a0, c3, c3; \ + vpshufb a0, d0, d0; \ + vpshufb a0, d1, d1; \ + vpshufb a0, d2, d2; \ + vpshufb a0, d3, d3; \ + vmovdqu d3, st1; \ + vmovdqu st0, d3; \ + vpshufb a0, d3, a0; \ + vmovdqu d2, st0; \ + \ + transpose_4x4(c0, d0, a0, b0, d2, d3); \ + transpose_4x4(c1, d1, a1, b1, d2, d3); \ + vmovdqu st0, d2; \ + vmovdqu st1, d3; \ + \ + vmovdqu b0, st0; \ + vmovdqu b1, st1; \ + transpose_4x4(c2, d2, a2, b2, b0, b1); \ + transpose_4x4(c3, d3, a3, b3, b0, b1); \ + vmovdqu st0, b0; \ + vmovdqu st1, b1; \ + /* does not adjust output bytes inside vectors */ + +/* load blocks to registers and apply pre-whitening */ +#define inpack16_pre(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + rio) \ + vmovdqu (0 * 16)(rio), x0; \ + vmovdqu (1 * 16)(rio), x1; \ + vmovdqu (2 * 16)(rio), x2; \ + vmovdqu (3 * 16)(rio), x3; \ + vmovdqu (4 * 16)(rio), x4; \ + vmovdqu (5 * 16)(rio), x5; \ + vmovdqu (6 * 16)(rio), x6; \ + vmovdqu (7 * 16)(rio), x7; \ + vmovdqu (8 * 16)(rio), y0; \ + vmovdqu (9 * 16)(rio), y1; \ + vmovdqu (10 * 16)(rio), y2; \ + vmovdqu (11 * 16)(rio), y3; \ + vmovdqu (12 * 16)(rio), y4; \ + vmovdqu (13 * 16)(rio), y5; \ + vmovdqu (14 * 16)(rio), y6; \ + vmovdqu (15 * 16)(rio), y7; + +/* byteslice pre-whitened blocks and store to temporary memory */ +#define inpack16_post(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_ab, mem_cd) \ + byteslice_16x16b(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + (mem_ab), (mem_cd)); \ + \ + vmovdqu x0, 0 * 16(mem_ab); \ + vmovdqu x1, 1 * 16(mem_ab); \ + vmovdqu x2, 2 * 16(mem_ab); \ + vmovdqu x3, 3 * 16(mem_ab); \ + vmovdqu x4, 4 * 16(mem_ab); \ + vmovdqu x5, 5 * 16(mem_ab); \ + vmovdqu x6, 6 * 16(mem_ab); \ + vmovdqu x7, 7 * 16(mem_ab); \ + vmovdqu y0, 0 * 16(mem_cd); \ + vmovdqu y1, 1 * 16(mem_cd); \ + vmovdqu y2, 2 * 16(mem_cd); \ + vmovdqu y3, 3 * 16(mem_cd); \ + vmovdqu y4, 4 * 16(mem_cd); \ + vmovdqu y5, 5 * 16(mem_cd); \ + vmovdqu y6, 6 * 16(mem_cd); \ + vmovdqu y7, 7 * 16(mem_cd); + +#define write_output(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem) \ + vmovdqu x0, 0 * 16(mem); \ + vmovdqu x1, 1 * 16(mem); \ + vmovdqu x2, 2 * 16(mem); \ + vmovdqu x3, 3 * 16(mem); \ + vmovdqu x4, 4 * 16(mem); \ + vmovdqu x5, 5 * 16(mem); \ + vmovdqu x6, 6 * 16(mem); \ + vmovdqu x7, 7 * 16(mem); \ + vmovdqu y0, 8 * 16(mem); \ + vmovdqu y1, 9 * 16(mem); \ + vmovdqu y2, 10 * 16(mem); \ + vmovdqu y3, 11 * 16(mem); \ + vmovdqu y4, 12 * 16(mem); \ + vmovdqu y5, 13 * 16(mem); \ + vmovdqu y6, 14 * 16(mem); \ + vmovdqu y7, 15 * 16(mem); \ + +#define aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, idx) \ + vmovdqu x0, ((idx + 0) * 16)(mem_tmp); \ + vmovdqu x1, ((idx + 1) * 16)(mem_tmp); \ + vmovdqu x2, ((idx + 2) * 16)(mem_tmp); \ + vmovdqu x3, ((idx + 3) * 16)(mem_tmp); \ + vmovdqu x4, ((idx + 4) * 16)(mem_tmp); \ + vmovdqu x5, ((idx + 5) * 16)(mem_tmp); \ + vmovdqu x6, ((idx + 6) * 16)(mem_tmp); \ + vmovdqu x7, ((idx + 7) * 16)(mem_tmp); + +#define aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, idx) \ + vmovdqu ((idx + 0) * 16)(mem_tmp), x0; \ + vmovdqu ((idx + 1) * 16)(mem_tmp), x1; \ + vmovdqu ((idx + 2) * 16)(mem_tmp), x2; \ + vmovdqu ((idx + 3) * 16)(mem_tmp), x3; \ + vmovdqu ((idx + 4) * 16)(mem_tmp), x4; \ + vmovdqu ((idx + 5) * 16)(mem_tmp), x5; \ + vmovdqu ((idx + 6) * 16)(mem_tmp), x6; \ + vmovdqu ((idx + 7) * 16)(mem_tmp), x7; + +#define aria_ark_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, rk, idx, round) \ + /* AddRoundKey */ \ + vpbroadcastb ((round * 16) + idx + 3)(rk), t0; \ + vpxor t0, x0, x0; \ + vpbroadcastb ((round * 16) + idx + 2)(rk), t0; \ + vpxor t0, x1, x1; \ + vpbroadcastb ((round * 16) + idx + 1)(rk), t0; \ + vpxor t0, x2, x2; \ + vpbroadcastb ((round * 16) + idx + 0)(rk), t0; \ + vpxor t0, x3, x3; \ + vpbroadcastb ((round * 16) + idx + 7)(rk), t0; \ + vpxor t0, x4, x4; \ + vpbroadcastb ((round * 16) + idx + 6)(rk), t0; \ + vpxor t0, x5, x5; \ + vpbroadcastb ((round * 16) + idx + 5)(rk), t0; \ + vpxor t0, x6, x6; \ + vpbroadcastb ((round * 16) + idx + 4)(rk), t0; \ + vpxor t0, x7, x7; + +#define aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, t1, t2, t3, \ + t4, t5, t6, t7) \ + vpbroadcastq .Ltf_s2_bitmatrix, t0; \ + vpbroadcastq .Ltf_inv_bitmatrix, t1; \ + vpbroadcastq .Ltf_id_bitmatrix, t2; \ + vpbroadcastq .Ltf_aff_bitmatrix, t3; \ + vpbroadcastq .Ltf_x2_bitmatrix, t4; \ + vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \ + vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \ + vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \ + vgf2p8affineqb $(tf_inv_const), t1, x6, x6; \ + vgf2p8affineinvqb $0, t2, x2, x2; \ + vgf2p8affineinvqb $0, t2, x6, x6; \ + vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0; \ + vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4; \ + vgf2p8affineqb $(tf_x2_const), t4, x3, x3; \ + vgf2p8affineqb $(tf_x2_const), t4, x7, x7; \ + vgf2p8affineinvqb $0, t2, x3, x3; \ + vgf2p8affineinvqb $0, t2, x7, x7 + +#define aria_sbox_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + t0, t1, t2, t3, \ + t4, t5, t6, t7) \ + vpxor t7, t7, t7; \ + vmovdqa .Linv_shift_row, t0; \ + vmovdqa .Lshift_row, t1; \ + vpbroadcastd .L0f0f0f0f, t6; \ + vmovdqa .Ltf_lo__inv_aff__and__s2, t2; \ + vmovdqa .Ltf_hi__inv_aff__and__s2, t3; \ + vmovdqa .Ltf_lo__x2__and__fwd_aff, t4; \ + vmovdqa .Ltf_hi__x2__and__fwd_aff, t5; \ + \ + vaesenclast t7, x0, x0; \ + vaesenclast t7, x4, x4; \ + vaesenclast t7, x1, x1; \ + vaesenclast t7, x5, x5; \ + vaesdeclast t7, x2, x2; \ + vaesdeclast t7, x6, x6; \ + \ + /* AES inverse shift rows */ \ + vpshufb t0, x0, x0; \ + vpshufb t0, x4, x4; \ + vpshufb t0, x1, x1; \ + vpshufb t0, x5, x5; \ + vpshufb t1, x3, x3; \ + vpshufb t1, x7, x7; \ + vpshufb t1, x2, x2; \ + vpshufb t1, x6, x6; \ + \ + /* affine transformation for S2 */ \ + filter_8bit(x1, t2, t3, t6, t0); \ + /* affine transformation for S2 */ \ + filter_8bit(x5, t2, t3, t6, t0); \ + \ + /* affine transformation for X2 */ \ + filter_8bit(x3, t4, t5, t6, t0); \ + /* affine transformation for X2 */ \ + filter_8bit(x7, t4, t5, t6, t0); \ + vaesdeclast t7, x3, x3; \ + vaesdeclast t7, x7, x7; + +#define aria_diff_m(x0, x1, x2, x3, \ + t0, t1, t2, t3) \ + /* T = rotr32(X, 8); */ \ + /* X ^= T */ \ + vpxor x0, x3, t0; \ + vpxor x1, x0, t1; \ + vpxor x2, x1, t2; \ + vpxor x3, x2, t3; \ + /* X = T ^ rotr(X, 16); */ \ + vpxor t2, x0, x0; \ + vpxor x1, t3, t3; \ + vpxor t0, x2, x2; \ + vpxor t1, x3, x1; \ + vmovdqu t3, x3; + +#define aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7) \ + /* t1 ^= t2; */ \ + vpxor y0, x4, x4; \ + vpxor y1, x5, x5; \ + vpxor y2, x6, x6; \ + vpxor y3, x7, x7; \ + \ + /* t2 ^= t3; */ \ + vpxor y4, y0, y0; \ + vpxor y5, y1, y1; \ + vpxor y6, y2, y2; \ + vpxor y7, y3, y3; \ + \ + /* t0 ^= t1; */ \ + vpxor x4, x0, x0; \ + vpxor x5, x1, x1; \ + vpxor x6, x2, x2; \ + vpxor x7, x3, x3; \ + \ + /* t3 ^= t1; */ \ + vpxor x4, y4, y4; \ + vpxor x5, y5, y5; \ + vpxor x6, y6, y6; \ + vpxor x7, y7, y7; \ + \ + /* t2 ^= t0; */ \ + vpxor x0, y0, y0; \ + vpxor x1, y1, y1; \ + vpxor x2, y2, y2; \ + vpxor x3, y3, y3; \ + \ + /* t1 ^= t2; */ \ + vpxor y0, x4, x4; \ + vpxor y1, x5, x5; \ + vpxor y2, x6, x6; \ + vpxor y3, x7, x7; + +#define aria_fe(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T3 = ABCD -> BADC \ + * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ + * T0 = ABCD -> CDAB \ + * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ + * T1 = ABCD -> DCBA \ + * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ + */ \ + aria_diff_word(x2, x3, x0, x1, \ + x7, x6, x5, x4, \ + y0, y1, y2, y3, \ + y5, y4, y7, y6); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_fo(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T1 = ABCD -> BADC \ + * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ + * T2 = ABCD -> CDAB \ + * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ + * T3 = ABCD -> DCBA \ + * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ + */ \ + aria_diff_word(x0, x1, x2, x3, \ + x5, x4, x7, x6, \ + y2, y3, y0, y1, \ + y7, y6, y5, y4); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_ff(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round, last_round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, last_round); \ + \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ + y0, y1, y2, y3, y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, last_round); \ + \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); + +#define aria_fe_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T3 = ABCD -> BADC \ + * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ + * T0 = ABCD -> CDAB \ + * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ + * T1 = ABCD -> DCBA \ + * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ + */ \ + aria_diff_word(x2, x3, x0, x1, \ + x7, x6, x5, x4, \ + y0, y1, y2, y3, \ + y5, y4, y7, y6); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_fo_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ + aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); \ + aria_diff_word(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + /* aria_diff_byte() \ + * T1 = ABCD -> BADC \ + * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ + * T2 = ABCD -> CDAB \ + * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ + * T3 = ABCD -> DCBA \ + * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ + */ \ + aria_diff_word(x0, x1, x2, x3, \ + x5, x4, x7, x6, \ + y2, y3, y0, y1, \ + y7, y6, y5, y4); \ + aria_store_state_8way(x3, x2, x1, x0, \ + x6, x7, x4, x5, \ + mem_tmp, 0); + +#define aria_ff_gfni(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, rk, round, last_round) \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 8, last_round); \ + \ + aria_store_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 8); \ + \ + aria_load_state_8way(x0, x1, x2, x3, \ + x4, x5, x6, x7, \ + mem_tmp, 0); \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, round); \ + \ + aria_sbox_8way_gfni(x2, x3, x0, x1, \ + x6, x7, x4, x5, \ + y0, y1, y2, y3, \ + y4, y5, y6, y7); \ + \ + aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ + y0, rk, 0, last_round); \ + \ + aria_load_state_8way(y0, y1, y2, y3, \ + y4, y5, y6, y7, \ + mem_tmp, 8); + +/* NB: section is mergeable, all elements must be aligned 16-byte blocks */ +.section .rodata.cst16, "aM", @progbits, 16 +.align 16 + +#define SHUFB_BYTES(idx) \ + 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) + +.Lshufb_16x16b: + .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3); +/* For isolating SubBytes from AESENCLAST, inverse shift row */ +.Linv_shift_row: + .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b + .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 +.Lshift_row: + .byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03 + .byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b +/* For CTR-mode IV byteswap */ +.Lbswap128_mask: + .byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 + .byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 + +/* AES inverse affine and S2 combined: + * 1 1 0 0 0 0 0 1 x0 0 + * 0 1 0 0 1 0 0 0 x1 0 + * 1 1 0 0 1 1 1 1 x2 0 + * 0 1 1 0 1 0 0 1 x3 1 + * 0 1 0 0 1 1 0 0 * x4 + 0 + * 0 1 0 1 1 0 0 0 x5 0 + * 0 0 0 0 0 1 0 1 x6 0 + * 1 1 1 0 0 1 1 1 x7 1 + */ +.Ltf_lo__inv_aff__and__s2: + .octa 0x92172DA81A9FA520B2370D883ABF8500 +.Ltf_hi__inv_aff__and__s2: + .octa 0x2B15FFC1AF917B45E6D8320C625CB688 + +/* X2 and AES forward affine combined: + * 1 0 1 1 0 0 0 1 x0 0 + * 0 1 1 1 1 0 1 1 x1 0 + * 0 0 0 1 1 0 1 0 x2 1 + * 0 1 0 0 0 1 0 0 x3 0 + * 0 0 1 1 1 0 1 1 * x4 + 0 + * 0 1 0 0 1 0 0 0 x5 0 + * 1 1 0 1 0 0 1 1 x6 0 + * 0 1 0 0 1 0 1 0 x7 0 + */ +.Ltf_lo__x2__and__fwd_aff: + .octa 0xEFAE0544FCBD1657B8F95213ABEA4100 +.Ltf_hi__x2__and__fwd_aff: + .octa 0x3F893781E95FE1576CDA64D2BA0CB204 + +.section .rodata.cst8, "aM", @progbits, 8 +.align 8 +/* AES affine: */ +#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0) +.Ltf_aff_bitmatrix: + .quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1), + BV8(1, 1, 0, 0, 0, 1, 1, 1), + BV8(1, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 1, 0, 0, 0, 1), + BV8(1, 1, 1, 1, 1, 0, 0, 0), + BV8(0, 1, 1, 1, 1, 1, 0, 0), + BV8(0, 0, 1, 1, 1, 1, 1, 0), + BV8(0, 0, 0, 1, 1, 1, 1, 1)) + +/* AES inverse affine: */ +#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0) +.Ltf_inv_bitmatrix: + .quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1), + BV8(1, 0, 0, 1, 0, 0, 1, 0), + BV8(0, 1, 0, 0, 1, 0, 0, 1), + BV8(1, 0, 1, 0, 0, 1, 0, 0), + BV8(0, 1, 0, 1, 0, 0, 1, 0), + BV8(0, 0, 1, 0, 1, 0, 0, 1), + BV8(1, 0, 0, 1, 0, 1, 0, 0), + BV8(0, 1, 0, 0, 1, 0, 1, 0)) + +/* S2: */ +#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1) +.Ltf_s2_bitmatrix: + .quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1), + BV8(0, 0, 1, 1, 1, 1, 1, 1), + BV8(1, 1, 1, 0, 1, 1, 0, 1), + BV8(1, 1, 0, 0, 0, 0, 1, 1), + BV8(0, 1, 0, 0, 0, 0, 1, 1), + BV8(1, 1, 0, 0, 1, 1, 1, 0), + BV8(0, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 1, 0, 1, 1, 0)) + +/* X2: */ +#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0) +.Ltf_x2_bitmatrix: + .quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0), + BV8(0, 0, 1, 0, 0, 1, 1, 0), + BV8(0, 0, 0, 0, 1, 0, 1, 0), + BV8(1, 1, 1, 0, 0, 0, 1, 1), + BV8(1, 1, 1, 0, 1, 1, 0, 0), + BV8(0, 1, 1, 0, 1, 0, 1, 1), + BV8(1, 0, 1, 1, 1, 1, 0, 1), + BV8(1, 0, 0, 1, 0, 0, 1, 1)) + +/* Identity matrix: */ +.Ltf_id_bitmatrix: + .quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0), + BV8(0, 1, 0, 0, 0, 0, 0, 0), + BV8(0, 0, 1, 0, 0, 0, 0, 0), + BV8(0, 0, 0, 1, 0, 0, 0, 0), + BV8(0, 0, 0, 0, 1, 0, 0, 0), + BV8(0, 0, 0, 0, 0, 1, 0, 0), + BV8(0, 0, 0, 0, 0, 0, 1, 0), + BV8(0, 0, 0, 0, 0, 0, 0, 1)) + +/* 4-bit mask */ +.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4 +.align 4 +.L0f0f0f0f: + .long 0x0f0f0f0f + +.text + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way) + /* input: + * %r9: rk + * %rsi: dst + * %rdx: src + * %xmm0..%xmm15: 16 byte-sliced blocks + */ + + FRAME_BEGIN + + movq %rsi, %rax; + leaq 8 * 16(%rax), %r8; + + inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r8); + aria_fo(%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 0); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 1); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 2); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 3); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 4); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 5); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 6); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 7); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 8); + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 9); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 10); + cmpl $12, rounds(CTX); + jne .Laria_192; + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11, 12); + jmp .Laria_end; +.Laria_192: + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 12); + cmpl $14, rounds(CTX); + jne .Laria_256; + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13, 14); + jmp .Laria_end; +.Laria_256: + aria_fe(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13); + aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 14); + aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 15, 16); +.Laria_end: + debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4, + %xmm9, %xmm13, %xmm0, %xmm5, + %xmm10, %xmm14, %xmm3, %xmm6, + %xmm11, %xmm15, %xmm2, %xmm7, + (%rax), (%r8)); + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_crypt_16way) + +SYM_FUNC_START(aria_aesni_avx_encrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq enc_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_encrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_decrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq dec_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_decrypt_16way) + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_ctr_gen_keystream_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + + FRAME_BEGIN + /* load IV and byteswap */ + vmovdqu (%r8), %xmm8; + + vmovdqa .Lbswap128_mask (%rip), %xmm1; + vpshufb %xmm1, %xmm8, %xmm3; /* be => le */ + + vpcmpeqd %xmm0, %xmm0, %xmm0; + vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */ + + /* construct IVs */ + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm9; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm10; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm11; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm12; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm13; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm14; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm15; + vmovdqu %xmm8, (0 * 16)(%rcx); + vmovdqu %xmm9, (1 * 16)(%rcx); + vmovdqu %xmm10, (2 * 16)(%rcx); + vmovdqu %xmm11, (3 * 16)(%rcx); + vmovdqu %xmm12, (4 * 16)(%rcx); + vmovdqu %xmm13, (5 * 16)(%rcx); + vmovdqu %xmm14, (6 * 16)(%rcx); + vmovdqu %xmm15, (7 * 16)(%rcx); + + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm8; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm9; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm10; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm11; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm12; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm13; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm14; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm15; + inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpshufb %xmm1, %xmm3, %xmm4; + vmovdqu %xmm4, (%r8); + + vmovdqu (0 * 16)(%rcx), %xmm0; + vmovdqu (1 * 16)(%rcx), %xmm1; + vmovdqu (2 * 16)(%rcx), %xmm2; + vmovdqu (3 * 16)(%rcx), %xmm3; + vmovdqu (4 * 16)(%rcx), %xmm4; + vmovdqu (5 * 16)(%rcx), %xmm5; + vmovdqu (6 * 16)(%rcx), %xmm6; + vmovdqu (7 * 16)(%rcx), %xmm7; + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_ctr_gen_keystream_16way) + +SYM_FUNC_START(aria_aesni_avx_ctr_crypt_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + FRAME_BEGIN + + call __aria_aesni_avx_ctr_gen_keystream_16way; + + leaq (%rsi), %r10; + leaq (%rdx), %r11; + leaq (%rcx), %rsi; + leaq (%rcx), %rdx; + leaq enc_key(CTX), %r9; + + call __aria_aesni_avx_crypt_16way; + + vpxor (0 * 16)(%r11), %xmm1, %xmm1; + vpxor (1 * 16)(%r11), %xmm0, %xmm0; + vpxor (2 * 16)(%r11), %xmm3, %xmm3; + vpxor (3 * 16)(%r11), %xmm2, %xmm2; + vpxor (4 * 16)(%r11), %xmm4, %xmm4; + vpxor (5 * 16)(%r11), %xmm5, %xmm5; + vpxor (6 * 16)(%r11), %xmm6, %xmm6; + vpxor (7 * 16)(%r11), %xmm7, %xmm7; + vpxor (8 * 16)(%r11), %xmm8, %xmm8; + vpxor (9 * 16)(%r11), %xmm9, %xmm9; + vpxor (10 * 16)(%r11), %xmm10, %xmm10; + vpxor (11 * 16)(%r11), %xmm11, %xmm11; + vpxor (12 * 16)(%r11), %xmm12, %xmm12; + vpxor (13 * 16)(%r11), %xmm13, %xmm13; + vpxor (14 * 16)(%r11), %xmm14, %xmm14; + vpxor (15 * 16)(%r11), %xmm15, %xmm15; + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %r10); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way) + +SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way) + /* input: + * %r9: rk + * %rsi: dst + * %rdx: src + * %xmm0..%xmm15: 16 byte-sliced blocks + */ + + FRAME_BEGIN + + movq %rsi, %rax; + leaq 8 * 16(%rax), %r8; + + inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r8); + aria_fo_gfni(%xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 0); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 1); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 2); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 3); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 4); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 5); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 6); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 7); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 8); + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 9); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 10); + cmpl $12, rounds(CTX); + jne .Laria_gfni_192; + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11, 12); + jmp .Laria_gfni_end; +.Laria_gfni_192: + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 11); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 12); + cmpl $14, rounds(CTX); + jne .Laria_gfni_256; + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13, 14); + jmp .Laria_gfni_end; +.Laria_gfni_256: + aria_fe_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 13); + aria_fo_gfni(%xmm9, %xmm8, %xmm11, %xmm10, + %xmm12, %xmm13, %xmm14, %xmm15, + %xmm0, %xmm1, %xmm2, %xmm3, + %xmm4, %xmm5, %xmm6, %xmm7, + %rax, %r9, 14); + aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, + %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, + %xmm12, %xmm13, %xmm14, + %xmm15, %rax, %r9, 15, 16); +.Laria_gfni_end: + debyteslice_16x16b(%xmm8, %xmm12, %xmm1, %xmm4, + %xmm9, %xmm13, %xmm0, %xmm5, + %xmm10, %xmm14, %xmm3, %xmm6, + %xmm11, %xmm15, %xmm2, %xmm7, + (%rax), (%r8)); + + FRAME_END + RET; +SYM_FUNC_END(__aria_aesni_avx_gfni_crypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_encrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq enc_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_gfni_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_encrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_decrypt_16way) + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + FRAME_BEGIN + + leaq dec_key(CTX), %r9; + + inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rdx); + + call __aria_aesni_avx_gfni_crypt_16way; + + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %rax); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_decrypt_16way) + +SYM_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way) + /* input: + * %rdi: ctx + * %rsi: dst + * %rdx: src + * %rcx: keystream + * %r8: iv (big endian, 128bit) + */ + FRAME_BEGIN + + call __aria_aesni_avx_ctr_gen_keystream_16way + + leaq (%rsi), %r10; + leaq (%rdx), %r11; + leaq (%rcx), %rsi; + leaq (%rcx), %rdx; + leaq enc_key(CTX), %r9; + + call __aria_aesni_avx_gfni_crypt_16way; + + vpxor (0 * 16)(%r11), %xmm1, %xmm1; + vpxor (1 * 16)(%r11), %xmm0, %xmm0; + vpxor (2 * 16)(%r11), %xmm3, %xmm3; + vpxor (3 * 16)(%r11), %xmm2, %xmm2; + vpxor (4 * 16)(%r11), %xmm4, %xmm4; + vpxor (5 * 16)(%r11), %xmm5, %xmm5; + vpxor (6 * 16)(%r11), %xmm6, %xmm6; + vpxor (7 * 16)(%r11), %xmm7, %xmm7; + vpxor (8 * 16)(%r11), %xmm8, %xmm8; + vpxor (9 * 16)(%r11), %xmm9, %xmm9; + vpxor (10 * 16)(%r11), %xmm10, %xmm10; + vpxor (11 * 16)(%r11), %xmm11, %xmm11; + vpxor (12 * 16)(%r11), %xmm12, %xmm12; + vpxor (13 * 16)(%r11), %xmm13, %xmm13; + vpxor (14 * 16)(%r11), %xmm14, %xmm14; + vpxor (15 * 16)(%r11), %xmm15, %xmm15; + write_output(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7, + %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, + %xmm15, %r10); + + FRAME_END + RET; +SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way) diff --git a/arch/x86/crypto/aria-avx.h b/arch/x86/crypto/aria-avx.h new file mode 100644 index 0000000000000000000000000000000000000000..01e9a01dc1574d40eb9d5dc9b91b214858adedb9 --- /dev/null +++ b/arch/x86/crypto/aria-avx.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef ASM_X86_ARIA_AVX_H +#define ASM_X86_ARIA_AVX_H + +#include + +#define ARIA_AESNI_PARALLEL_BLOCKS 16 +#define ARIA_AESNI_PARALLEL_BLOCK_SIZE (ARIA_BLOCK_SIZE * 16) + +struct aria_avx_ops { + void (*aria_encrypt_16way)(const void *ctx, u8 *dst, const u8 *src); + void (*aria_decrypt_16way)(const void *ctx, u8 *dst, const u8 *src); + void (*aria_ctr_crypt_16way)(const void *ctx, u8 *dst, const u8 *src, + u8 *keystream, u8 *iv); +}; +#endif diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c new file mode 100644 index 0000000000000000000000000000000000000000..c561ea4fefa5e6f7387168b2dcf3af8008026339 --- /dev/null +++ b/arch/x86/crypto/aria_aesni_avx_glue.c @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Glue Code for the AVX/AES-NI/GFNI assembler implementation of the ARIA Cipher + * + * Copyright (c) 2022 Taehee Yoo + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ecb_cbc_helpers.h" +#include "aria-avx.h" + +asmlinkage void aria_aesni_avx_encrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_decrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_ctr_crypt_16way(const void *ctx, u8 *dst, + const u8 *src, + u8 *keystream, u8 *iv); +asmlinkage void aria_aesni_avx_gfni_encrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_gfni_decrypt_16way(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void aria_aesni_avx_gfni_ctr_crypt_16way(const void *ctx, u8 *dst, + const u8 *src, + u8 *keystream, u8 *iv); + +static struct aria_avx_ops aria_ops; + +static int ecb_do_encrypt(struct skcipher_request *req, const u32 *rkey) +{ + ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_encrypt_16way); + ECB_BLOCK(1, aria_encrypt); + ECB_WALK_END(); +} + +static int ecb_do_decrypt(struct skcipher_request *req, const u32 *rkey) +{ + ECB_WALK_START(req, ARIA_BLOCK_SIZE, ARIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(ARIA_AESNI_PARALLEL_BLOCKS, aria_ops.aria_decrypt_16way); + ECB_BLOCK(1, aria_decrypt); + ECB_WALK_END(); +} + +static int aria_avx_ecb_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + + return ecb_do_encrypt(req, ctx->enc_key[0]); +} + +static int aria_avx_ecb_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + + return ecb_do_decrypt(req, ctx->dec_key[0]); +} + +static int aria_avx_set_key(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + return aria_set_key(&tfm->base, key, keylen); +} + +static int aria_avx_ctr_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aria_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, false); + + while ((nbytes = walk.nbytes) > 0) { + const u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + + while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) { + u8 keystream[ARIA_AESNI_PARALLEL_BLOCK_SIZE]; + + kernel_fpu_begin(); + aria_ops.aria_ctr_crypt_16way(ctx, dst, src, keystream, + walk.iv); + kernel_fpu_end(); + dst += ARIA_AESNI_PARALLEL_BLOCK_SIZE; + src += ARIA_AESNI_PARALLEL_BLOCK_SIZE; + nbytes -= ARIA_AESNI_PARALLEL_BLOCK_SIZE; + } + + while (nbytes >= ARIA_BLOCK_SIZE) { + u8 keystream[ARIA_BLOCK_SIZE]; + + memcpy(keystream, walk.iv, ARIA_BLOCK_SIZE); + crypto_inc(walk.iv, ARIA_BLOCK_SIZE); + + aria_encrypt(ctx, keystream, keystream); + + crypto_xor_cpy(dst, src, keystream, ARIA_BLOCK_SIZE); + dst += ARIA_BLOCK_SIZE; + src += ARIA_BLOCK_SIZE; + nbytes -= ARIA_BLOCK_SIZE; + } + + if (walk.nbytes == walk.total && nbytes > 0) { + u8 keystream[ARIA_BLOCK_SIZE]; + + memcpy(keystream, walk.iv, ARIA_BLOCK_SIZE); + crypto_inc(walk.iv, ARIA_BLOCK_SIZE); + + aria_encrypt(ctx, keystream, keystream); + + crypto_xor_cpy(dst, src, keystream, nbytes); + dst += nbytes; + src += nbytes; + nbytes = 0; + } + err = skcipher_walk_done(&walk, nbytes); + } + + return err; +} + +static struct skcipher_alg aria_algs[] = { + { + .base.cra_name = "__ecb(aria)", + .base.cra_driver_name = "__ecb-aria-avx", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = ARIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aria_ctx), + .base.cra_module = THIS_MODULE, + .min_keysize = ARIA_MIN_KEY_SIZE, + .max_keysize = ARIA_MAX_KEY_SIZE, + .setkey = aria_avx_set_key, + .encrypt = aria_avx_ecb_encrypt, + .decrypt = aria_avx_ecb_decrypt, + }, { + .base.cra_name = "__ctr(aria)", + .base.cra_driver_name = "__ctr-aria-avx", + .base.cra_priority = 400, + .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aria_ctx), + .base.cra_module = THIS_MODULE, + .min_keysize = ARIA_MIN_KEY_SIZE, + .max_keysize = ARIA_MAX_KEY_SIZE, + .ivsize = ARIA_BLOCK_SIZE, + .chunksize = ARIA_BLOCK_SIZE, + .walksize = 16 * ARIA_BLOCK_SIZE, + .setkey = aria_avx_set_key, + .encrypt = aria_avx_ctr_encrypt, + .decrypt = aria_avx_ctr_encrypt, + } +}; + +static struct simd_skcipher_alg *aria_simd_algs[ARRAY_SIZE(aria_algs)]; + +static int __init aria_avx_init(void) +{ + const char *feature_name; + + if (!boot_cpu_has(X86_FEATURE_AVX) || + !boot_cpu_has(X86_FEATURE_AES) || + !boot_cpu_has(X86_FEATURE_OSXSAVE)) { + pr_info("AVX or AES-NI instructions are not detected.\n"); + return -ENODEV; + } + + if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, + &feature_name)) { + pr_info("CPU feature '%s' is not supported.\n", feature_name); + return -ENODEV; + } + + if (boot_cpu_has(X86_FEATURE_GFNI)) { + aria_ops.aria_encrypt_16way = aria_aesni_avx_gfni_encrypt_16way; + aria_ops.aria_decrypt_16way = aria_aesni_avx_gfni_decrypt_16way; + aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_gfni_ctr_crypt_16way; + } else { + aria_ops.aria_encrypt_16way = aria_aesni_avx_encrypt_16way; + aria_ops.aria_decrypt_16way = aria_aesni_avx_decrypt_16way; + aria_ops.aria_ctr_crypt_16way = aria_aesni_avx_ctr_crypt_16way; + } + + return simd_register_skciphers_compat(aria_algs, + ARRAY_SIZE(aria_algs), + aria_simd_algs); +} + +static void __exit aria_avx_exit(void) +{ + simd_unregister_skciphers(aria_algs, ARRAY_SIZE(aria_algs), + aria_simd_algs); +} + +module_init(aria_avx_init); +module_exit(aria_avx_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Taehee Yoo "); +MODULE_DESCRIPTION("ARIA Cipher Algorithm, AVX/AES-NI/GFNI optimized"); +MODULE_ALIAS_CRYPTO("aria"); +MODULE_ALIAS_CRYPTO("aria-aesni-avx"); diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S index 802d715826891a5aab7afcd3e2cbfb1cc68edeac..4a43e072d2d1d9f3fe194eac8a42d5fe18b72e6f 100644 --- a/arch/x86/crypto/blowfish-x86_64-asm_64.S +++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S @@ -6,6 +6,7 @@ */ #include +#include .file "blowfish-x86_64-asm.S" .text @@ -141,7 +142,7 @@ SYM_FUNC_START(__blowfish_enc_blk) RET; SYM_FUNC_END(__blowfish_enc_blk) -SYM_FUNC_START(blowfish_dec_blk) +SYM_TYPED_FUNC_START(blowfish_dec_blk) /* input: * %rdi: ctx * %rsi: dst @@ -332,7 +333,7 @@ SYM_FUNC_START(__blowfish_enc_blk_4way) RET; SYM_FUNC_END(__blowfish_enc_blk_4way) -SYM_FUNC_START(blowfish_dec_blk_4way) +SYM_TYPED_FUNC_START(blowfish_dec_blk_4way) /* input: * %rdi: ctx * %rsi: dst diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 30e70f4fe2f7a4ba9d3869079179d6ddc1601f4d..6d3b85e53d0eeb352479407ba7de326d150bf924 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -36,6 +36,7 @@ #include #include #include +#include #include asmlinkage void sha512_transform_ssse3(struct sha512_state *state, @@ -284,6 +285,13 @@ static int register_sha512_avx2(void) ARRAY_SIZE(sha512_avx2_algs)); return 0; } +static const struct x86_cpu_id module_cpu_ids[] = { + X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), + X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), + X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); static void unregister_sha512_avx2(void) { @@ -294,6 +302,8 @@ static void unregister_sha512_avx2(void) static int __init sha512_ssse3_mod_init(void) { + if (!x86_match_cpu(module_cpu_ids)) + return -ENODEV; if (register_sha512_ssse3()) goto fail; diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S index 682338e7e2a38dbedcdef57ac6cbb6078e8abe75..4dd19819053a5848d1ee0ff3ab3c44c0a09c283e 100644 --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -311,7 +311,7 @@ SYM_CODE_START(entry_INT80_compat) * Interrupts are off on entry. */ ASM_CLAC /* Do this early to minimize exposure */ - SWAPGS + ALTERNATIVE "swapgs", "", X86_FEATURE_XENPV /* * User tracing code (ptrace or signal handlers) might assume that diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 12f6c4d714cd63c7579066b7201aa2d33440cc69..3e88b9df8c8f137104566f4fb7c4b3d27ef03f09 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -11,6 +11,9 @@ include $(srctree)/lib/vdso/Makefile # Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n +KMSAN_SANITIZE_vclock_gettime.o := n +KMSAN_SANITIZE_vgetcpu.o := n + UBSAN_SANITIZE := n KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y @@ -91,7 +94,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) endif endif -$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) +$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO # @@ -153,6 +156,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 1000d457c3321e2caf3c9428e7961b0e1c572458..311eae30e08944c767995572bce85965038cb145 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -127,17 +127,17 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) { struct mm_struct *mm = task->mm; struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); mmap_read_lock(mm); - - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long size = vma->vm_end - vma->vm_start; if (vma_is_special_mapping(vma, &vvar_mapping)) zap_page_range(vma, vma->vm_start, size); } - mmap_read_unlock(mm); + return 0; } #else @@ -327,7 +327,7 @@ static unsigned long vdso_addr(unsigned long start, unsigned len) end -= len; if (end > start) { - offset = get_random_int() % (((end - start) >> PAGE_SHIFT) + 1); + offset = prandom_u32_max(((end - start) >> PAGE_SHIFT) + 1); addr = start + (offset << PAGE_SHIFT); } else { addr = start; @@ -354,6 +354,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); mmap_write_lock(mm); /* @@ -363,7 +364,7 @@ int map_vdso_once(const struct vdso_image *image, unsigned long addr) * We could search vma near context.vdso, but it's a slowpath, * so let's explicitly check all VMAs to be completely sure. */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (vma_is_special_mapping(vma, &vdso_mapping) || vma_is_special_mapping(vma, &vvar_mapping)) { mmap_write_unlock(mm); diff --git a/arch/x86/events/Makefile b/arch/x86/events/Makefile index 9933c0e8e97a94982696417da36a05346a5ff9d1..86a76efa8bb6d1fb0d84f7f31b4d5b9f4ffd5a34 100644 --- a/arch/x86/events/Makefile +++ b/arch/x86/events/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += core.o probe.o +obj-y += core.o probe.o utils.o obj-$(CONFIG_PERF_EVENTS_INTEL_RAPL) += rapl.o obj-y += amd/ obj-$(CONFIG_X86_LOCAL_APIC) += msr.o diff --git a/arch/x86/events/amd/Makefile b/arch/x86/events/amd/Makefile index b9f5d4610256d16f2816126efb6a5f83da849628..527d947eb76b92edfbe7a94e12a4f816e9a9b47f 100644 --- a/arch/x86/events/amd/Makefile +++ b/arch/x86/events/amd/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_CPU_SUP_AMD) += core.o +obj-$(CONFIG_CPU_SUP_AMD) += core.o lbr.o obj-$(CONFIG_PERF_EVENTS_AMD_BRS) += brs.o obj-$(CONFIG_PERF_EVENTS_AMD_POWER) += power.o obj-$(CONFIG_X86_LOCAL_APIC) += ibs.o diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index bee8765a1e9b631febe5c05e2e53f3ed9e1ddc7b..f1bff153d94578a549e59416a9754b559ad6f0b4 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -81,7 +81,7 @@ static bool __init amd_brs_detect(void) * a br_sel_map. Software filtering is not supported because it would not correlate well * with a sampling period. */ -int amd_brs_setup_filter(struct perf_event *event) +static int amd_brs_setup_filter(struct perf_event *event) { u64 type = event->attr.branch_sample_type; @@ -96,6 +96,73 @@ int amd_brs_setup_filter(struct perf_event *event) return 0; } +static inline int amd_is_brs_event(struct perf_event *e) +{ + return (e->hw.config & AMD64_RAW_EVENT_MASK) == AMD_FAM19H_BRS_EVENT; +} + +int amd_brs_hw_config(struct perf_event *event) +{ + int ret = 0; + + /* + * Due to interrupt holding, BRS is not recommended in + * counting mode. + */ + if (!is_sampling_event(event)) + return -EINVAL; + + /* + * Due to the way BRS operates by holding the interrupt until + * lbr_nr entries have been captured, it does not make sense + * to allow sampling on BRS with an event that does not match + * what BRS is capturing, i.e., retired taken branches. + * Otherwise the correlation with the event's period is even + * more loose: + * + * With retired taken branch: + * Effective P = P + 16 + X + * With any other event: + * Effective P = P + Y + X + * + * Where X is the number of taken branches due to interrupt + * skid. Skid is large. + * + * Where Y is the occurences of the event while BRS is + * capturing the lbr_nr entries. + * + * By using retired taken branches, we limit the impact on the + * Y variable. We know it cannot be more than the depth of + * BRS. + */ + if (!amd_is_brs_event(event)) + return -EINVAL; + + /* + * BRS implementation does not work with frequency mode + * reprogramming of the period. + */ + if (event->attr.freq) + return -EINVAL; + /* + * The kernel subtracts BRS depth from period, so it must + * be big enough. + */ + if (event->attr.sample_period <= x86_pmu.lbr_nr) + return -EINVAL; + + /* + * Check if we can allow PERF_SAMPLE_BRANCH_STACK + */ + ret = amd_brs_setup_filter(event); + + /* only set in case of success */ + if (!ret) + event->hw.flags |= PERF_X86_EVENT_AMD_BRS; + + return ret; +} + /* tos = top of stack, i.e., last valid entry written */ static inline int amd_brs_get_tos(union amd_debug_extn_cfg *cfg) { diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 9ac3718410ce4bf4173ee566f6bce208938ea08e..8b70237c33f76a720d67437fe7d697ab8e105b2b 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -330,16 +330,10 @@ static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc) } } -#define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */ -static inline int amd_is_brs_event(struct perf_event *e) -{ - return (e->hw.config & AMD64_RAW_EVENT_MASK) == AMD_FAM19H_BRS_EVENT; -} +DEFINE_STATIC_CALL_RET0(amd_pmu_branch_hw_config, *x86_pmu.hw_config); static int amd_core_hw_config(struct perf_event *event) { - int ret = 0; - if (event->attr.exclude_host && event->attr.exclude_guest) /* * When HO == GO == 1 the hardware treats that as GO == HO == 0 @@ -356,66 +350,10 @@ static int amd_core_hw_config(struct perf_event *event) if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(&event->hw)) event->hw.flags |= PERF_X86_EVENT_PAIR; - /* - * if branch stack is requested - */ - if (has_branch_stack(event)) { - /* - * Due to interrupt holding, BRS is not recommended in - * counting mode. - */ - if (!is_sampling_event(event)) - return -EINVAL; + if (has_branch_stack(event)) + return static_call(amd_pmu_branch_hw_config)(event); - /* - * Due to the way BRS operates by holding the interrupt until - * lbr_nr entries have been captured, it does not make sense - * to allow sampling on BRS with an event that does not match - * what BRS is capturing, i.e., retired taken branches. - * Otherwise the correlation with the event's period is even - * more loose: - * - * With retired taken branch: - * Effective P = P + 16 + X - * With any other event: - * Effective P = P + Y + X - * - * Where X is the number of taken branches due to interrupt - * skid. Skid is large. - * - * Where Y is the occurences of the event while BRS is - * capturing the lbr_nr entries. - * - * By using retired taken branches, we limit the impact on the - * Y variable. We know it cannot be more than the depth of - * BRS. - */ - if (!amd_is_brs_event(event)) - return -EINVAL; - - /* - * BRS implementation does not work with frequency mode - * reprogramming of the period. - */ - if (event->attr.freq) - return -EINVAL; - /* - * The kernel subtracts BRS depth from period, so it must - * be big enough. - */ - if (event->attr.sample_period <= x86_pmu.lbr_nr) - return -EINVAL; - - /* - * Check if we can allow PERF_SAMPLE_BRANCH_STACK - */ - ret = amd_brs_setup_filter(event); - - /* only set in case of success */ - if (!ret) - event->hw.flags |= PERF_X86_EVENT_AMD_BRS; - } - return ret; + return 0; } static inline int amd_is_nb_event(struct hw_perf_event *hwc) @@ -582,8 +520,14 @@ static struct amd_nb *amd_alloc_nb(int cpu) return nb; } +typedef void (amd_pmu_branch_reset_t)(void); +DEFINE_STATIC_CALL_NULL(amd_pmu_branch_reset, amd_pmu_branch_reset_t); + static void amd_pmu_cpu_reset(int cpu) { + if (x86_pmu.lbr_nr) + static_call(amd_pmu_branch_reset)(); + if (x86_pmu.version < 2) return; @@ -598,16 +542,24 @@ static int amd_pmu_cpu_prepare(int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); + cpuc->lbr_sel = kzalloc_node(sizeof(struct er_account), GFP_KERNEL, + cpu_to_node(cpu)); + if (!cpuc->lbr_sel) + return -ENOMEM; + WARN_ON_ONCE(cpuc->amd_nb); if (!x86_pmu.amd_nb_constraints) return 0; cpuc->amd_nb = amd_alloc_nb(cpu); - if (!cpuc->amd_nb) - return -ENOMEM; + if (cpuc->amd_nb) + return 0; - return 0; + kfree(cpuc->lbr_sel); + cpuc->lbr_sel = NULL; + + return -ENOMEM; } static void amd_pmu_cpu_starting(int cpu) @@ -640,19 +592,19 @@ static void amd_pmu_cpu_starting(int cpu) cpuc->amd_nb->nb_id = nb_id; cpuc->amd_nb->refcnt++; - amd_brs_reset(); amd_pmu_cpu_reset(cpu); } static void amd_pmu_cpu_dead(int cpu) { - struct cpu_hw_events *cpuhw; + struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu); + + kfree(cpuhw->lbr_sel); + cpuhw->lbr_sel = NULL; if (!x86_pmu.amd_nb_constraints) return; - cpuhw = &per_cpu(cpu_hw_events, cpu); - if (cpuhw->amd_nb) { struct amd_nb *nb = cpuhw->amd_nb; @@ -677,7 +629,7 @@ static inline u64 amd_pmu_get_global_status(void) /* PerfCntrGlobalStatus is read-only */ rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status); - return status & amd_pmu_global_cntr_mask; + return status; } static inline void amd_pmu_ack_global_status(u64 status) @@ -688,8 +640,6 @@ static inline void amd_pmu_ack_global_status(u64 status) * clears the same bit in PerfCntrGlobalStatus */ - /* Only allow modifications to PerfCntrGlobalStatus.PerfCntrOvfl */ - status &= amd_pmu_global_cntr_mask; wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status); } @@ -799,11 +749,17 @@ static void amd_pmu_v2_enable_event(struct perf_event *event) __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); } -static void amd_pmu_v2_enable_all(int added) +static __always_inline void amd_pmu_core_enable_all(void) { amd_pmu_set_global_ctl(amd_pmu_global_cntr_mask); } +static void amd_pmu_v2_enable_all(int added) +{ + amd_pmu_lbr_enable_all(); + amd_pmu_core_enable_all(); +} + static void amd_pmu_disable_event(struct perf_event *event) { x86_pmu_disable_event(event); @@ -828,23 +784,32 @@ static void amd_pmu_disable_all(void) amd_pmu_check_overflow(); } -static void amd_pmu_v2_disable_all(void) +static __always_inline void amd_pmu_core_disable_all(void) { - /* Disable all PMCs */ amd_pmu_set_global_ctl(0); +} + +static void amd_pmu_v2_disable_all(void) +{ + amd_pmu_core_disable_all(); + amd_pmu_lbr_disable_all(); amd_pmu_check_overflow(); } +DEFINE_STATIC_CALL_NULL(amd_pmu_branch_add, *x86_pmu.add); + static void amd_pmu_add_event(struct perf_event *event) { if (needs_branch_stack(event)) - amd_pmu_brs_add(event); + static_call(amd_pmu_branch_add)(event); } +DEFINE_STATIC_CALL_NULL(amd_pmu_branch_del, *x86_pmu.del); + static void amd_pmu_del_event(struct perf_event *event) { if (needs_branch_stack(event)) - amd_pmu_brs_del(event); + static_call(amd_pmu_branch_del)(event); } /* @@ -930,8 +895,8 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) pmu_enabled = cpuc->enabled; cpuc->enabled = 0; - /* Stop counting */ - amd_pmu_v2_disable_all(); + /* Stop counting but do not disable LBR */ + amd_pmu_core_disable_all(); status = amd_pmu_get_global_status(); @@ -939,6 +904,12 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) if (!status) goto done; + /* Read branch records before unfreezing */ + if (status & GLOBAL_STATUS_LBRS_FROZEN) { + amd_pmu_lbr_read(); + status &= ~GLOBAL_STATUS_LBRS_FROZEN; + } + for (idx = 0; idx < x86_pmu.num_counters; idx++) { if (!test_bit(idx, cpuc->active_mask)) continue; @@ -958,6 +929,11 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) if (!x86_perf_event_set_period(event)) continue; + if (has_branch_stack(event)) { + data.br_stack = &cpuc->lbr_stack; + data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; + } + if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); @@ -971,7 +947,7 @@ static int amd_pmu_v2_handle_irq(struct pt_regs *regs) */ WARN_ON(status > 0); - /* Clear overflow bits */ + /* Clear overflow and freeze bits */ amd_pmu_ack_global_status(~status); /* @@ -985,7 +961,7 @@ done: /* Resume counting only if PMU is active */ if (pmu_enabled) - amd_pmu_v2_enable_all(0); + amd_pmu_core_enable_all(); return amd_pmu_adjust_nmi_window(handled); } @@ -1248,23 +1224,14 @@ static ssize_t amd_event_sysfs_show(char *page, u64 config) return x86_event_sysfs_show(page, config, event); } -static void amd_pmu_sched_task(struct perf_event_context *ctx, - bool sched_in) -{ - if (sched_in && x86_pmu.lbr_nr) - amd_pmu_brs_sched_task(ctx, sched_in); -} - -static u64 amd_pmu_limit_period(struct perf_event *event, u64 left) +static void amd_pmu_limit_period(struct perf_event *event, s64 *left) { /* * Decrease period by the depth of the BRS feature to get the last N * taken branches and approximate the desired period */ - if (has_branch_stack(event) && left > x86_pmu.lbr_nr) - left -= x86_pmu.lbr_nr; - - return left; + if (has_branch_stack(event) && *left > x86_pmu.lbr_nr) + *left -= x86_pmu.lbr_nr; } static __initconst const struct x86_pmu amd_pmu = { @@ -1311,23 +1278,25 @@ static ssize_t branches_show(struct device *cdev, static DEVICE_ATTR_RO(branches); -static struct attribute *amd_pmu_brs_attrs[] = { +static struct attribute *amd_pmu_branches_attrs[] = { &dev_attr_branches.attr, NULL, }; static umode_t -amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i) +amd_branches_is_visible(struct kobject *kobj, struct attribute *attr, int i) { return x86_pmu.lbr_nr ? attr->mode : 0; } -static struct attribute_group group_caps_amd_brs = { +static struct attribute_group group_caps_amd_branches = { .name = "caps", - .attrs = amd_pmu_brs_attrs, - .is_visible = amd_brs_is_visible, + .attrs = amd_pmu_branches_attrs, + .is_visible = amd_branches_is_visible, }; +#ifdef CONFIG_PERF_EVENTS_AMD_BRS + EVENT_ATTR_STR(branch-brs, amd_branch_brs, "event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n"); @@ -1336,15 +1305,26 @@ static struct attribute *amd_brs_events_attrs[] = { NULL, }; +static umode_t +amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i) +{ + return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ? + attr->mode : 0; +} + static struct attribute_group group_events_amd_brs = { .name = "events", .attrs = amd_brs_events_attrs, .is_visible = amd_brs_is_visible, }; +#endif /* CONFIG_PERF_EVENTS_AMD_BRS */ + static const struct attribute_group *amd_attr_update[] = { - &group_caps_amd_brs, + &group_caps_amd_branches, +#ifdef CONFIG_PERF_EVENTS_AMD_BRS &group_events_amd_brs, +#endif NULL, }; @@ -1421,13 +1401,27 @@ static int __init amd_core_pmu_init(void) x86_pmu.flags |= PMU_FL_PAIR; } - /* - * BRS requires special event constraints and flushing on ctxsw. - */ - if (boot_cpu_data.x86 >= 0x19 && !amd_brs_init()) { + /* LBR and BRS are mutually exclusive features */ + if (!amd_pmu_lbr_init()) { + /* LBR requires flushing on context switch */ + x86_pmu.sched_task = amd_pmu_lbr_sched_task; + static_call_update(amd_pmu_branch_hw_config, amd_pmu_lbr_hw_config); + static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset); + static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add); + static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del); + } else if (!amd_brs_init()) { + /* + * BRS requires special event constraints and flushing on ctxsw. + */ x86_pmu.get_event_constraints = amd_get_event_constraints_f19h; - x86_pmu.sched_task = amd_pmu_sched_task; + x86_pmu.sched_task = amd_pmu_brs_sched_task; x86_pmu.limit_period = amd_pmu_limit_period; + + static_call_update(amd_pmu_branch_hw_config, amd_brs_hw_config); + static_call_update(amd_pmu_branch_reset, amd_brs_reset); + static_call_update(amd_pmu_branch_add, amd_pmu_brs_add); + static_call_update(amd_pmu_branch_del, amd_pmu_brs_del); + /* * put_event_constraints callback same as Fam17h, set above */ diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index c251bc44c088d8382b9bfac688e1901f78642bf1..3271735f00702de86cfcc95fa95adfdb98083667 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -300,16 +300,6 @@ static int perf_ibs_init(struct perf_event *event) hwc->config_base = perf_ibs->msr; hwc->config = config; - /* - * rip recorded by IbsOpRip will not be consistent with rsp and rbp - * recorded as part of interrupt regs. Thus we need to use rip from - * interrupt regs while unwinding call stack. Setting _EARLY flag - * makes sure we unwind call-stack before perf sample rip is set to - * IbsOpRip. - */ - if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; - return 0; } @@ -688,6 +678,339 @@ static struct perf_ibs perf_ibs_op = { .get_count = get_ibs_op_count, }; +static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3, + struct perf_sample_data *data) +{ + union perf_mem_data_src *data_src = &data->data_src; + + data_src->mem_op = PERF_MEM_OP_NA; + + if (op_data3->ld_op) + data_src->mem_op = PERF_MEM_OP_LOAD; + else if (op_data3->st_op) + data_src->mem_op = PERF_MEM_OP_STORE; +} + +/* + * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has + * more fine granular DataSrc encodings. Others have coarse. + */ +static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2) +{ + if (ibs_caps & IBS_CAPS_ZEN4) + return (op_data2->data_src_hi << 3) | op_data2->data_src_lo; + + return op_data2->data_src_lo; +} + +static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, + union ibs_op_data3 *op_data3, + struct perf_sample_data *data) +{ + union perf_mem_data_src *data_src = &data->data_src; + u8 ibs_data_src = perf_ibs_data_src(op_data2); + + data_src->mem_lvl = 0; + + /* + * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached + * memory accesses. So, check DcUcMemAcc bit early. + */ + if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) { + data_src->mem_lvl = PERF_MEM_LVL_UNC | PERF_MEM_LVL_HIT; + return; + } + + /* L1 Hit */ + if (op_data3->dc_miss == 0) { + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + return; + } + + /* L2 Hit */ + if (op_data3->l2_miss == 0) { + /* Erratum #1293 */ + if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF || + !(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + return; + } + } + + /* + * OP_DATA2 is valid only for load ops. Skip all checks which + * uses OP_DATA2[DataSrc]. + */ + if (data_src->mem_op != PERF_MEM_OP_LOAD) + goto check_mab; + + /* L3 Hit */ + if (ibs_caps & IBS_CAPS_ZEN4) { + if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + return; + } + } else { + if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_REM_CCE1 | + PERF_MEM_LVL_HIT; + return; + } + } + + /* A peer cache in a near CCX */ + if (ibs_caps & IBS_CAPS_ZEN4 && + ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE) { + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; + return; + } + + /* A peer cache in a far CCX */ + if (ibs_caps & IBS_CAPS_ZEN4) { + if (ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) { + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT; + return; + } + } else { + if (ibs_data_src == IBS_DATA_SRC_REM_CACHE) { + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT; + return; + } + } + + /* DRAM */ + if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) { + if (op_data2->rmt_node == 0) + data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; + else + data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT; + return; + } + + /* PMEM */ + if (ibs_caps & IBS_CAPS_ZEN4 && ibs_data_src == IBS_DATA_SRC_EXT_PMEM) { + data_src->mem_lvl_num = PERF_MEM_LVLNUM_PMEM; + if (op_data2->rmt_node) { + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + /* IBS doesn't provide Remote socket detail */ + data_src->mem_hops = PERF_MEM_HOPS_1; + } + return; + } + + /* Extension Memory */ + if (ibs_caps & IBS_CAPS_ZEN4 && + ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) { + data_src->mem_lvl_num = PERF_MEM_LVLNUM_EXTN_MEM; + if (op_data2->rmt_node) { + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + /* IBS doesn't provide Remote socket detail */ + data_src->mem_hops = PERF_MEM_HOPS_1; + } + return; + } + + /* IO */ + if (ibs_data_src == IBS_DATA_SRC_EXT_IO) { + data_src->mem_lvl = PERF_MEM_LVL_IO; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO; + if (op_data2->rmt_node) { + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + /* IBS doesn't provide Remote socket detail */ + data_src->mem_hops = PERF_MEM_HOPS_1; + } + return; + } + +check_mab: + /* + * MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding + * DC misses. However, such data may come from any level in mem + * hierarchy. IBS provides detail about both MAB as well as actual + * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set + * MAB only when IBS fails to provide DataSrc. + */ + if (op_data3->dc_miss_no_mab_alloc) { + data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT; + return; + } + + data_src->mem_lvl = PERF_MEM_LVL_NA; +} + +static bool perf_ibs_cache_hit_st_valid(void) +{ + /* 0: Uninitialized, 1: Valid, -1: Invalid */ + static int cache_hit_st_valid; + + if (unlikely(!cache_hit_st_valid)) { + if (boot_cpu_data.x86 == 0x19 && + (boot_cpu_data.x86_model <= 0xF || + (boot_cpu_data.x86_model >= 0x20 && + boot_cpu_data.x86_model <= 0x5F))) { + cache_hit_st_valid = -1; + } else { + cache_hit_st_valid = 1; + } + } + + return cache_hit_st_valid == 1; +} + +static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2, + struct perf_sample_data *data) +{ + union perf_mem_data_src *data_src = &data->data_src; + u8 ibs_data_src; + + data_src->mem_snoop = PERF_MEM_SNOOP_NA; + + if (!perf_ibs_cache_hit_st_valid() || + data_src->mem_op != PERF_MEM_OP_LOAD || + data_src->mem_lvl & PERF_MEM_LVL_L1 || + data_src->mem_lvl & PERF_MEM_LVL_L2 || + op_data2->cache_hit_st) + return; + + ibs_data_src = perf_ibs_data_src(op_data2); + + if (ibs_caps & IBS_CAPS_ZEN4) { + if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE || + ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE || + ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + } else if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) { + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + } +} + +static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3, + struct perf_sample_data *data) +{ + union perf_mem_data_src *data_src = &data->data_src; + + data_src->mem_dtlb = PERF_MEM_TLB_NA; + + if (!op_data3->dc_lin_addr_valid) + return; + + if (!op_data3->dc_l1tlb_miss) { + data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT; + return; + } + + if (!op_data3->dc_l2tlb_miss) { + data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT; + return; + } + + data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS; +} + +static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3, + struct perf_sample_data *data) +{ + union perf_mem_data_src *data_src = &data->data_src; + + data_src->mem_lock = PERF_MEM_LOCK_NA; + + if (op_data3->dc_locked_op) + data_src->mem_lock = PERF_MEM_LOCK_LOCKED; +} + +#define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL) + +static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data, + struct perf_sample_data *data, + union ibs_op_data2 *op_data2, + union ibs_op_data3 *op_data3) +{ + perf_ibs_get_mem_lvl(op_data2, op_data3, data); + perf_ibs_get_mem_snoop(op_data2, data); + perf_ibs_get_tlb_lvl(op_data3, data); + perf_ibs_get_mem_lock(op_data3, data); +} + +static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data, + union ibs_op_data3 *op_data3) +{ + __u64 val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)]; + + /* Erratum #1293 */ + if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF && + (op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) { + /* + * OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode. + * DataSrc=0 is 'No valid status' and RmtNode is invalid when + * DataSrc=0. + */ + val = 0; + } + return val; +} + +static void perf_ibs_parse_ld_st_data(__u64 sample_type, + struct perf_ibs_data *ibs_data, + struct perf_sample_data *data) +{ + union ibs_op_data3 op_data3; + union ibs_op_data2 op_data2; + union ibs_op_data op_data; + + data->data_src.val = PERF_MEM_NA; + op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)]; + + perf_ibs_get_mem_op(&op_data3, data); + if (data->data_src.mem_op != PERF_MEM_OP_LOAD && + data->data_src.mem_op != PERF_MEM_OP_STORE) + return; + + op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3); + + if (sample_type & PERF_SAMPLE_DATA_SRC) { + perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3); + data->sample_flags |= PERF_SAMPLE_DATA_SRC; + } + + if (sample_type & PERF_SAMPLE_WEIGHT_TYPE && op_data3.dc_miss && + data->data_src.mem_op == PERF_MEM_OP_LOAD) { + op_data.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA)]; + + if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) { + data->weight.var1_dw = op_data3.dc_miss_lat; + data->weight.var2_w = op_data.tag_to_ret_ctr; + } else if (sample_type & PERF_SAMPLE_WEIGHT) { + data->weight.full = op_data3.dc_miss_lat; + } + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } + + if (sample_type & PERF_SAMPLE_ADDR && op_data3.dc_lin_addr_valid) { + data->addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)]; + data->sample_flags |= PERF_SAMPLE_ADDR; + } + + if (sample_type & PERF_SAMPLE_PHYS_ADDR && op_data3.dc_phy_addr_valid) { + data->phys_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)]; + data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; + } +} + +static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type, + int check_rip) +{ + if (sample_type & PERF_SAMPLE_RAW || + (perf_ibs == &perf_ibs_op && + (sample_type & PERF_SAMPLE_DATA_SRC || + sample_type & PERF_SAMPLE_WEIGHT_TYPE || + sample_type & PERF_SAMPLE_ADDR || + sample_type & PERF_SAMPLE_PHYS_ADDR))) + return perf_ibs->offset_max; + else if (check_rip) + return 3; + return 1; +} + static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) { struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); @@ -735,12 +1058,9 @@ fail: size = 1; offset = 1; check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); - if (event->attr.sample_type & PERF_SAMPLE_RAW) - offset_max = perf_ibs->offset_max; - else if (check_rip) - offset_max = 3; - else - offset_max = 1; + + offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip); + do { rdmsrl(msr + offset, *buf++); size++; @@ -791,15 +1111,21 @@ fail: }, }; data.raw = &raw; + data.sample_flags |= PERF_SAMPLE_RAW; } + if (perf_ibs == &perf_ibs_op) + perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data); + /* * rip recorded by IbsOpRip will not be consistent with rsp and rbp * recorded as part of interrupt regs. Thus we need to use rip from * interrupt regs while unwinding call stack. */ - if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) + if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { data.callchain = perf_callchain(event, iregs); + data.sample_flags |= PERF_SAMPLE_CALLCHAIN; + } throttle = perf_event_overflow(event, &data, ®s); out: diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c new file mode 100644 index 0000000000000000000000000000000000000000..38a75216c12cffc89a49e19f747e7a0adf272d9b --- /dev/null +++ b/arch/x86/events/amd/lbr.c @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "../perf_event.h" + +/* LBR Branch Select valid bits */ +#define LBR_SELECT_MASK 0x1ff + +/* + * LBR Branch Select filter bits which when set, ensures that the + * corresponding type of branches are not recorded + */ +#define LBR_SELECT_KERNEL 0 /* Branches ending in CPL = 0 */ +#define LBR_SELECT_USER 1 /* Branches ending in CPL > 0 */ +#define LBR_SELECT_JCC 2 /* Conditional branches */ +#define LBR_SELECT_CALL_NEAR_REL 3 /* Near relative calls */ +#define LBR_SELECT_CALL_NEAR_IND 4 /* Indirect relative calls */ +#define LBR_SELECT_RET_NEAR 5 /* Near returns */ +#define LBR_SELECT_JMP_NEAR_IND 6 /* Near indirect jumps (excl. calls and returns) */ +#define LBR_SELECT_JMP_NEAR_REL 7 /* Near relative jumps (excl. calls) */ +#define LBR_SELECT_FAR_BRANCH 8 /* Far branches */ + +#define LBR_KERNEL BIT(LBR_SELECT_KERNEL) +#define LBR_USER BIT(LBR_SELECT_USER) +#define LBR_JCC BIT(LBR_SELECT_JCC) +#define LBR_REL_CALL BIT(LBR_SELECT_CALL_NEAR_REL) +#define LBR_IND_CALL BIT(LBR_SELECT_CALL_NEAR_IND) +#define LBR_RETURN BIT(LBR_SELECT_RET_NEAR) +#define LBR_REL_JMP BIT(LBR_SELECT_JMP_NEAR_REL) +#define LBR_IND_JMP BIT(LBR_SELECT_JMP_NEAR_IND) +#define LBR_FAR BIT(LBR_SELECT_FAR_BRANCH) +#define LBR_NOT_SUPP -1 /* unsupported filter */ +#define LBR_IGNORE 0 + +#define LBR_ANY \ + (LBR_JCC | LBR_REL_CALL | LBR_IND_CALL | LBR_RETURN | \ + LBR_REL_JMP | LBR_IND_JMP | LBR_FAR) + +struct branch_entry { + union { + struct { + u64 ip:58; + u64 ip_sign_ext:5; + u64 mispredict:1; + } split; + u64 full; + } from; + + union { + struct { + u64 ip:58; + u64 ip_sign_ext:3; + u64 reserved:1; + u64 spec:1; + u64 valid:1; + } split; + u64 full; + } to; +}; + +static __always_inline void amd_pmu_lbr_set_from(unsigned int idx, u64 val) +{ + wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); +} + +static __always_inline void amd_pmu_lbr_set_to(unsigned int idx, u64 val) +{ + wrmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); +} + +static __always_inline u64 amd_pmu_lbr_get_from(unsigned int idx) +{ + u64 val; + + rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2, val); + + return val; +} + +static __always_inline u64 amd_pmu_lbr_get_to(unsigned int idx) +{ + u64 val; + + rdmsrl(MSR_AMD_SAMP_BR_FROM + idx * 2 + 1, val); + + return val; +} + +static __always_inline u64 sign_ext_branch_ip(u64 ip) +{ + u32 shift = 64 - boot_cpu_data.x86_virt_bits; + + return (u64)(((s64)ip << shift) >> shift); +} + +static void amd_pmu_lbr_filter(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int br_sel = cpuc->br_sel, offset, type, i, j; + bool compress = false; + bool fused_only = false; + u64 from, to; + + /* If sampling all branches, there is nothing to filter */ + if (((br_sel & X86_BR_ALL) == X86_BR_ALL) && + ((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE)) + fused_only = true; + + for (i = 0; i < cpuc->lbr_stack.nr; i++) { + from = cpuc->lbr_entries[i].from; + to = cpuc->lbr_entries[i].to; + type = branch_type_fused(from, to, 0, &offset); + + /* + * Adjust the branch from address in case of instruction + * fusion where it points to an instruction preceding the + * actual branch + */ + if (offset) { + cpuc->lbr_entries[i].from += offset; + if (fused_only) + continue; + } + + /* If type does not correspond, then discard */ + if (type == X86_BR_NONE || (br_sel & type) != type) { + cpuc->lbr_entries[i].from = 0; /* mark invalid */ + compress = true; + } + + if ((br_sel & X86_BR_TYPE_SAVE) == X86_BR_TYPE_SAVE) + cpuc->lbr_entries[i].type = common_branch_type(type); + } + + if (!compress) + return; + + /* Remove all invalid entries */ + for (i = 0; i < cpuc->lbr_stack.nr; ) { + if (!cpuc->lbr_entries[i].from) { + j = i; + while (++j < cpuc->lbr_stack.nr) + cpuc->lbr_entries[j - 1] = cpuc->lbr_entries[j]; + cpuc->lbr_stack.nr--; + if (!cpuc->lbr_entries[i].from) + continue; + } + i++; + } +} + +static const int lbr_spec_map[PERF_BR_SPEC_MAX] = { + PERF_BR_SPEC_NA, + PERF_BR_SPEC_WRONG_PATH, + PERF_BR_NON_SPEC_CORRECT_PATH, + PERF_BR_SPEC_CORRECT_PATH, +}; + +void amd_pmu_lbr_read(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct perf_branch_entry *br = cpuc->lbr_entries; + struct branch_entry entry; + int out = 0, idx, i; + + if (!cpuc->lbr_users) + return; + + for (i = 0; i < x86_pmu.lbr_nr; i++) { + entry.from.full = amd_pmu_lbr_get_from(i); + entry.to.full = amd_pmu_lbr_get_to(i); + + /* + * Check if a branch has been logged; if valid = 0, spec = 0 + * then no branch was recorded + */ + if (!entry.to.split.valid && !entry.to.split.spec) + continue; + + perf_clear_branch_entry_bitfields(br + out); + + br[out].from = sign_ext_branch_ip(entry.from.split.ip); + br[out].to = sign_ext_branch_ip(entry.to.split.ip); + br[out].mispred = entry.from.split.mispredict; + br[out].predicted = !br[out].mispred; + + /* + * Set branch speculation information using the status of + * the valid and spec bits. + * + * When valid = 0, spec = 0, no branch was recorded and the + * entry is discarded as seen above. + * + * When valid = 0, spec = 1, the recorded branch was + * speculative but took the wrong path. + * + * When valid = 1, spec = 0, the recorded branch was + * non-speculative but took the correct path. + * + * When valid = 1, spec = 1, the recorded branch was + * speculative and took the correct path + */ + idx = (entry.to.split.valid << 1) | entry.to.split.spec; + br[out].spec = lbr_spec_map[idx]; + out++; + } + + cpuc->lbr_stack.nr = out; + + /* + * Internal register renaming always ensures that LBR From[0] and + * LBR To[0] always represent the TOS + */ + cpuc->lbr_stack.hw_idx = 0; + + /* Perform further software filtering */ + amd_pmu_lbr_filter(); +} + +static const int lbr_select_map[PERF_SAMPLE_BRANCH_MAX_SHIFT] = { + [PERF_SAMPLE_BRANCH_USER_SHIFT] = LBR_USER, + [PERF_SAMPLE_BRANCH_KERNEL_SHIFT] = LBR_KERNEL, + [PERF_SAMPLE_BRANCH_HV_SHIFT] = LBR_IGNORE, + + [PERF_SAMPLE_BRANCH_ANY_SHIFT] = LBR_ANY, + [PERF_SAMPLE_BRANCH_ANY_CALL_SHIFT] = LBR_REL_CALL | LBR_IND_CALL | LBR_FAR, + [PERF_SAMPLE_BRANCH_ANY_RETURN_SHIFT] = LBR_RETURN | LBR_FAR, + [PERF_SAMPLE_BRANCH_IND_CALL_SHIFT] = LBR_IND_CALL, + [PERF_SAMPLE_BRANCH_ABORT_TX_SHIFT] = LBR_NOT_SUPP, + [PERF_SAMPLE_BRANCH_IN_TX_SHIFT] = LBR_NOT_SUPP, + [PERF_SAMPLE_BRANCH_NO_TX_SHIFT] = LBR_NOT_SUPP, + [PERF_SAMPLE_BRANCH_COND_SHIFT] = LBR_JCC, + + [PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] = LBR_NOT_SUPP, + [PERF_SAMPLE_BRANCH_IND_JUMP_SHIFT] = LBR_IND_JMP, + [PERF_SAMPLE_BRANCH_CALL_SHIFT] = LBR_REL_CALL, + + [PERF_SAMPLE_BRANCH_NO_FLAGS_SHIFT] = LBR_NOT_SUPP, + [PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT] = LBR_NOT_SUPP, +}; + +static int amd_pmu_lbr_setup_filter(struct perf_event *event) +{ + struct hw_perf_event_extra *reg = &event->hw.branch_reg; + u64 br_type = event->attr.branch_sample_type; + u64 mask = 0, v; + int i; + + /* No LBR support */ + if (!x86_pmu.lbr_nr) + return -EOPNOTSUPP; + + if (br_type & PERF_SAMPLE_BRANCH_USER) + mask |= X86_BR_USER; + + if (br_type & PERF_SAMPLE_BRANCH_KERNEL) + mask |= X86_BR_KERNEL; + + /* Ignore BRANCH_HV here */ + + if (br_type & PERF_SAMPLE_BRANCH_ANY) + mask |= X86_BR_ANY; + + if (br_type & PERF_SAMPLE_BRANCH_ANY_CALL) + mask |= X86_BR_ANY_CALL; + + if (br_type & PERF_SAMPLE_BRANCH_ANY_RETURN) + mask |= X86_BR_RET | X86_BR_IRET | X86_BR_SYSRET; + + if (br_type & PERF_SAMPLE_BRANCH_IND_CALL) + mask |= X86_BR_IND_CALL; + + if (br_type & PERF_SAMPLE_BRANCH_COND) + mask |= X86_BR_JCC; + + if (br_type & PERF_SAMPLE_BRANCH_IND_JUMP) + mask |= X86_BR_IND_JMP; + + if (br_type & PERF_SAMPLE_BRANCH_CALL) + mask |= X86_BR_CALL | X86_BR_ZERO_CALL; + + if (br_type & PERF_SAMPLE_BRANCH_TYPE_SAVE) + mask |= X86_BR_TYPE_SAVE; + + reg->reg = mask; + mask = 0; + + for (i = 0; i < PERF_SAMPLE_BRANCH_MAX_SHIFT; i++) { + if (!(br_type & BIT_ULL(i))) + continue; + + v = lbr_select_map[i]; + if (v == LBR_NOT_SUPP) + return -EOPNOTSUPP; + + if (v != LBR_IGNORE) + mask |= v; + } + + /* Filter bits operate in suppress mode */ + reg->config = mask ^ LBR_SELECT_MASK; + + return 0; +} + +int amd_pmu_lbr_hw_config(struct perf_event *event) +{ + int ret = 0; + + /* LBR is not recommended in counting mode */ + if (!is_sampling_event(event)) + return -EINVAL; + + ret = amd_pmu_lbr_setup_filter(event); + if (!ret) + event->attach_state |= PERF_ATTACH_SCHED_CB; + + return ret; +} + +void amd_pmu_lbr_reset(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int i; + + if (!x86_pmu.lbr_nr) + return; + + /* Reset all branch records individually */ + for (i = 0; i < x86_pmu.lbr_nr; i++) { + amd_pmu_lbr_set_from(i, 0); + amd_pmu_lbr_set_to(i, 0); + } + + cpuc->last_task_ctx = NULL; + cpuc->last_log_id = 0; + wrmsrl(MSR_AMD64_LBR_SELECT, 0); +} + +void amd_pmu_lbr_add(struct perf_event *event) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + struct hw_perf_event_extra *reg = &event->hw.branch_reg; + + if (!x86_pmu.lbr_nr) + return; + + if (has_branch_stack(event)) { + cpuc->lbr_select = 1; + cpuc->lbr_sel->config = reg->config; + cpuc->br_sel = reg->reg; + } + + perf_sched_cb_inc(event->ctx->pmu); + + if (!cpuc->lbr_users++ && !event->total_time_running) + amd_pmu_lbr_reset(); +} + +void amd_pmu_lbr_del(struct perf_event *event) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + if (!x86_pmu.lbr_nr) + return; + + if (has_branch_stack(event)) + cpuc->lbr_select = 0; + + cpuc->lbr_users--; + WARN_ON_ONCE(cpuc->lbr_users < 0); + perf_sched_cb_dec(event->ctx->pmu); +} + +void amd_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + /* + * A context switch can flip the address space and LBR entries are + * not tagged with an identifier. Hence, branches cannot be resolved + * from the old address space and the LBR records should be wiped. + */ + if (cpuc->lbr_users && sched_in) + amd_pmu_lbr_reset(); +} + +void amd_pmu_lbr_enable_all(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + u64 lbr_select, dbg_ctl, dbg_extn_cfg; + + if (!cpuc->lbr_users || !x86_pmu.lbr_nr) + return; + + /* Set hardware branch filter */ + if (cpuc->lbr_select) { + lbr_select = cpuc->lbr_sel->config & LBR_SELECT_MASK; + wrmsrl(MSR_AMD64_LBR_SELECT, lbr_select); + } + + rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); + rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); + + wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); + wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg | DBG_EXTN_CFG_LBRV2EN); +} + +void amd_pmu_lbr_disable_all(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + u64 dbg_ctl, dbg_extn_cfg; + + if (!cpuc->lbr_users || !x86_pmu.lbr_nr) + return; + + rdmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg); + rdmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl); + + wrmsrl(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN); + wrmsrl(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI); +} + +__init int amd_pmu_lbr_init(void) +{ + union cpuid_0x80000022_ebx ebx; + + if (x86_pmu.version < 2 || !boot_cpu_has(X86_FEATURE_AMD_LBR_V2)) + return -EOPNOTSUPP; + + /* Set number of entries */ + ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); + x86_pmu.lbr_nr = ebx.split.lbr_v2_stack_sz; + + pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr); + + return 0; +} diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index f969410d0c9044d4d428cfbccebee9eebc66c14c..b30b8bbcd1e2277a292145f2436bf0fa2cfb1a28 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -72,6 +72,10 @@ DEFINE_STATIC_CALL_NULL(x86_pmu_add, *x86_pmu.add); DEFINE_STATIC_CALL_NULL(x86_pmu_del, *x86_pmu.del); DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read); +DEFINE_STATIC_CALL_NULL(x86_pmu_set_period, *x86_pmu.set_period); +DEFINE_STATIC_CALL_NULL(x86_pmu_update, *x86_pmu.update); +DEFINE_STATIC_CALL_NULL(x86_pmu_limit_period, *x86_pmu.limit_period); + DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events, *x86_pmu.schedule_events); DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints); DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints); @@ -116,9 +120,6 @@ u64 x86_perf_event_update(struct perf_event *event) if (unlikely(!hwc->event_base)) return 0; - if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event) - return x86_pmu.update_topdown_event(event); - /* * Careful: an NMI might modify the previous event value. * @@ -621,8 +622,9 @@ int x86_pmu_hw_config(struct perf_event *event) event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; if (event->attr.sample_period && x86_pmu.limit_period) { - if (x86_pmu.limit_period(event, event->attr.sample_period) > - event->attr.sample_period) + s64 left = event->attr.sample_period; + x86_pmu.limit_period(event, &left); + if (left > event->attr.sample_period) return -EINVAL; } @@ -1354,7 +1356,7 @@ static void x86_pmu_enable(struct pmu *pmu) static_call(x86_pmu_enable_all)(added); } -static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); +DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); /* * Set the next IRQ period, based on the hwc->period_left value. @@ -1370,10 +1372,6 @@ int x86_perf_event_set_period(struct perf_event *event) if (unlikely(!hwc->event_base)) return 0; - if (unlikely(is_topdown_count(event)) && - x86_pmu.set_topdown_event_period) - return x86_pmu.set_topdown_event_period(event); - /* * If we are way outside a reasonable range then just skip forward: */ @@ -1399,10 +1397,9 @@ int x86_perf_event_set_period(struct perf_event *event) if (left > x86_pmu.max_period) left = x86_pmu.max_period; - if (x86_pmu.limit_period) - left = x86_pmu.limit_period(event, left); + static_call_cond(x86_pmu_limit_period)(event, &left); - per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; + this_cpu_write(pmc_prev_left[idx], left); /* * The hw event starts counting from this event offset, @@ -1419,16 +1416,6 @@ int x86_perf_event_set_period(struct perf_event *event) if (is_counter_pair(hwc)) wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff); - /* - * Due to erratum on certan cpu we need - * a second write to be sure the register - * is updated properly - */ - if (x86_pmu.perfctr_second_write) { - wrmsrl(hwc->event_base, - (u64)(-left) & x86_pmu.cntval_mask); - } - perf_event_update_userpage(event); return ret; @@ -1518,7 +1505,7 @@ static void x86_pmu_start(struct perf_event *event, int flags) if (flags & PERF_EF_RELOAD) { WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); - x86_perf_event_set_period(event); + static_call(x86_pmu_set_period)(event); } event->hw.state = 0; @@ -1610,7 +1597,7 @@ void x86_pmu_stop(struct perf_event *event, int flags) * Drain the remaining delta count out of a event * that we are disabling: */ - x86_perf_event_update(event); + static_call(x86_pmu_update)(event); hwc->state |= PERF_HES_UPTODATE; } } @@ -1700,7 +1687,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) event = cpuc->events[idx]; - val = x86_perf_event_update(event); + val = static_call(x86_pmu_update)(event); if (val & (1ULL << (x86_pmu.cntval_bits - 1))) continue; @@ -1709,13 +1696,15 @@ int x86_pmu_handle_irq(struct pt_regs *regs) */ handled++; - if (!x86_perf_event_set_period(event)) + if (!static_call(x86_pmu_set_period)(event)) continue; perf_sample_data_init(&data, 0, event->hw.last_period); - if (has_branch_stack(event)) + if (has_branch_stack(event)) { data.br_stack = &cpuc->lbr_stack; + data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; + } if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); @@ -2023,6 +2012,10 @@ static void x86_pmu_static_call_update(void) static_call_update(x86_pmu_del, x86_pmu.del); static_call_update(x86_pmu_read, x86_pmu.read); + static_call_update(x86_pmu_set_period, x86_pmu.set_period); + static_call_update(x86_pmu_update, x86_pmu.update); + static_call_update(x86_pmu_limit_period, x86_pmu.limit_period); + static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events); static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints); static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints); @@ -2042,7 +2035,7 @@ static void x86_pmu_static_call_update(void) static void _x86_pmu_read(struct perf_event *event) { - x86_perf_event_update(event); + static_call(x86_pmu_update)(event); } void x86_pmu_show_pmu_cap(int num_counters, int num_counters_fixed, @@ -2149,6 +2142,12 @@ static int __init init_hw_perf_events(void) if (!x86_pmu.guest_get_msrs) x86_pmu.guest_get_msrs = (void *)&__static_call_return0; + if (!x86_pmu.set_period) + x86_pmu.set_period = x86_perf_event_set_period; + + if (!x86_pmu.update) + x86_pmu.update = x86_perf_event_update; + x86_pmu_static_call_update(); /* @@ -2670,7 +2669,9 @@ static int x86_pmu_check_period(struct perf_event *event, u64 value) return -EINVAL; if (value && x86_pmu.limit_period) { - if (x86_pmu.limit_period(event, value) > value) + s64 left = value; + x86_pmu.limit_period(event, &left); + if (left > value) return -EINVAL; } diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2db93498ff7119518f8731b1a6efc88d95d57b20..a646a5f9a235c1055e13990ab615369e1fdc1a40 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2102,6 +2102,15 @@ static struct extra_reg intel_tnt_extra_regs[] __read_mostly = { EVENT_EXTRA_END }; +EVENT_ATTR_STR(mem-loads, mem_ld_grt, "event=0xd0,umask=0x5,ldlat=3"); +EVENT_ATTR_STR(mem-stores, mem_st_grt, "event=0xd0,umask=0x6"); + +static struct attribute *grt_mem_attrs[] = { + EVENT_PTR(mem_ld_grt), + EVENT_PTR(mem_st_grt), + NULL +}; + static struct extra_reg intel_grt_extra_regs[] __read_mostly = { /* must define OFFCORE_RSP_X first, see intel_fixup_er() */ INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0), @@ -2190,6 +2199,12 @@ static void __intel_pmu_enable_all(int added, bool pmi) u64 intel_ctrl = hybrid(cpuc->pmu, intel_ctrl); intel_pmu_lbr_enable_all(pmi); + + if (cpuc->fixed_ctrl_val != cpuc->active_fixed_ctrl_val) { + wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, cpuc->fixed_ctrl_val); + cpuc->active_fixed_ctrl_val = cpuc->fixed_ctrl_val; + } + wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, intel_ctrl & ~cpuc->intel_ctrl_guest_mask); @@ -2302,7 +2317,7 @@ static void intel_pmu_nhm_workaround(void) for (i = 0; i < 4; i++) { event = cpuc->events[i]; if (event) - x86_perf_event_update(event); + static_call(x86_pmu_update)(event); } for (i = 0; i < 4; i++) { @@ -2317,7 +2332,7 @@ static void intel_pmu_nhm_workaround(void) event = cpuc->events[i]; if (event) { - x86_perf_event_set_period(event); + static_call(x86_pmu_set_period)(event); __x86_pmu_enable_event(&event->hw, ARCH_PERFMON_EVENTSEL_ENABLE); } else @@ -2407,9 +2422,10 @@ static inline void intel_clear_masks(struct perf_event *event, int idx) static void intel_pmu_disable_fixed(struct perf_event *event) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; - u64 ctrl_val, mask; int idx = hwc->idx; + u64 mask; if (is_topdown_idx(idx)) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); @@ -2426,9 +2442,7 @@ static void intel_pmu_disable_fixed(struct perf_event *event) intel_clear_masks(event, idx); mask = 0xfULL << ((idx - INTEL_PMC_IDX_FIXED) * 4); - rdmsrl(hwc->config_base, ctrl_val); - ctrl_val &= ~mask; - wrmsrl(hwc->config_base, ctrl_val); + cpuc->fixed_ctrl_val &= ~mask; } static void intel_pmu_disable_event(struct perf_event *event) @@ -2521,6 +2535,8 @@ static int adl_set_topdown_event_period(struct perf_event *event) return icl_set_topdown_event_period(event); } +DEFINE_STATIC_CALL(intel_pmu_set_topdown_event_period, x86_perf_event_set_period); + static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx) { u32 val; @@ -2671,6 +2687,7 @@ static u64 adl_update_topdown_event(struct perf_event *event) return icl_update_topdown_event(event); } +DEFINE_STATIC_CALL(intel_pmu_update_topdown_event, x86_perf_event_update); static void intel_pmu_read_topdown_event(struct perf_event *event) { @@ -2682,7 +2699,7 @@ static void intel_pmu_read_topdown_event(struct perf_event *event) return; perf_pmu_disable(event->pmu); - x86_pmu.update_topdown_event(event); + static_call(intel_pmu_update_topdown_event)(event); perf_pmu_enable(event->pmu); } @@ -2690,7 +2707,7 @@ static void intel_pmu_read_event(struct perf_event *event) { if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD) intel_pmu_auto_reload_read(event); - else if (is_topdown_count(event) && x86_pmu.update_topdown_event) + else if (is_topdown_count(event)) intel_pmu_read_topdown_event(event); else x86_perf_event_update(event); @@ -2698,8 +2715,9 @@ static void intel_pmu_read_event(struct perf_event *event) static void intel_pmu_enable_fixed(struct perf_event *event) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct hw_perf_event *hwc = &event->hw; - u64 ctrl_val, mask, bits = 0; + u64 mask, bits = 0; int idx = hwc->idx; if (is_topdown_idx(idx)) { @@ -2743,10 +2761,8 @@ static void intel_pmu_enable_fixed(struct perf_event *event) mask |= ICL_FIXED_0_ADAPTIVE << (idx * 4); } - rdmsrl(hwc->config_base, ctrl_val); - ctrl_val &= ~mask; - ctrl_val |= bits; - wrmsrl(hwc->config_base, ctrl_val); + cpuc->fixed_ctrl_val &= ~mask; + cpuc->fixed_ctrl_val |= bits; } static void intel_pmu_enable_event(struct perf_event *event) @@ -2794,7 +2810,7 @@ static void intel_pmu_add_event(struct perf_event *event) */ int intel_pmu_save_and_restart(struct perf_event *event) { - x86_perf_event_update(event); + static_call(x86_pmu_update)(event); /* * For a checkpointed counter always reset back to 0. This * avoids a situation where the counter overflows, aborts the @@ -2806,9 +2822,25 @@ int intel_pmu_save_and_restart(struct perf_event *event) wrmsrl(event->hw.event_base, 0); local64_set(&event->hw.prev_count, 0); } + return static_call(x86_pmu_set_period)(event); +} + +static int intel_pmu_set_period(struct perf_event *event) +{ + if (unlikely(is_topdown_count(event))) + return static_call(intel_pmu_set_topdown_event_period)(event); + return x86_perf_event_set_period(event); } +static u64 intel_pmu_update(struct perf_event *event) +{ + if (unlikely(is_topdown_count(event))) + return static_call(intel_pmu_update_topdown_event)(event); + + return x86_perf_event_update(event); +} + static void intel_pmu_reset(void) { struct debug_store *ds = __this_cpu_read(cpu_hw_events.ds); @@ -2971,8 +3003,7 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) */ if (__test_and_clear_bit(GLOBAL_STATUS_PERF_METRICS_OVF_BIT, (unsigned long *)&status)) { handled++; - if (x86_pmu.update_topdown_event) - x86_pmu.update_topdown_event(NULL); + static_call(intel_pmu_update_topdown_event)(NULL); } /* @@ -2995,8 +3026,10 @@ static int handle_pmi_common(struct pt_regs *regs, u64 status) perf_sample_data_init(&data, 0, event->hw.last_period); - if (has_branch_stack(event)) + if (has_branch_stack(event)) { data.br_stack = &cpuc->lbr_stack; + data.sample_flags |= PERF_SAMPLE_BRANCH_STACK; + } if (perf_event_overflow(event, &data, regs)) x86_pmu_stop(event, 0); @@ -3844,9 +3877,6 @@ static int intel_pmu_hw_config(struct perf_event *event) } if (x86_pmu.pebs_aliases) x86_pmu.pebs_aliases(event); - - if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) - event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY; } if (needs_branch_stack(event)) { @@ -4052,8 +4082,9 @@ static struct perf_guest_switch_msr *intel_guest_get_msrs(int *nr, void *data) /* Disable guest PEBS if host PEBS is enabled. */ arr[pebs_enable].guest = 0; } else { - /* Disable guest PEBS for cross-mapped PEBS counters. */ + /* Disable guest PEBS thoroughly for cross-mapped PEBS counters. */ arr[pebs_enable].guest &= ~kvm_pmu->host_cross_mapped_mask; + arr[global_ctrl].guest &= ~kvm_pmu->host_cross_mapped_mask; /* Set hw GLOBAL_CTRL bits for PEBS counter when it runs for guest */ arr[global_ctrl].guest |= arr[pebs_enable].guest; } @@ -4324,28 +4355,25 @@ static u8 adl_get_hybrid_cpu_type(void) * Therefore the effective (average) period matches the requested period, * despite coarser hardware granularity. */ -static u64 bdw_limit_period(struct perf_event *event, u64 left) +static void bdw_limit_period(struct perf_event *event, s64 *left) { if ((event->hw.config & INTEL_ARCH_EVENT_MASK) == X86_CONFIG(.event=0xc0, .umask=0x01)) { - if (left < 128) - left = 128; - left &= ~0x3fULL; + if (*left < 128) + *left = 128; + *left &= ~0x3fULL; } - return left; } -static u64 nhm_limit_period(struct perf_event *event, u64 left) +static void nhm_limit_period(struct perf_event *event, s64 *left) { - return max(left, 32ULL); + *left = max(*left, 32LL); } -static u64 spr_limit_period(struct perf_event *event, u64 left) +static void spr_limit_period(struct perf_event *event, s64 *left) { if (event->attr.precise_ip == 3) - return max(left, 128ULL); - - return left; + *left = max(*left, 128LL); } PMU_FORMAT_ATTR(event, "config:0-7" ); @@ -4784,6 +4812,8 @@ static __initconst const struct x86_pmu intel_pmu = { .add = intel_pmu_add_event, .del = intel_pmu_del_event, .read = intel_pmu_read_event, + .set_period = intel_pmu_set_period, + .update = intel_pmu_update, .hw_config = intel_pmu_hw_config, .schedule_events = x86_schedule_events, .eventsel = MSR_ARCH_PERFMON_EVENTSEL0, @@ -5974,6 +6004,36 @@ __init int intel_pmu_init(void) name = "Tremont"; break; + case INTEL_FAM6_ALDERLAKE_N: + x86_pmu.mid_ack = true; + memcpy(hw_cache_event_ids, glp_hw_cache_event_ids, + sizeof(hw_cache_event_ids)); + memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs, + sizeof(hw_cache_extra_regs)); + hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1; + + x86_pmu.event_constraints = intel_slm_event_constraints; + x86_pmu.pebs_constraints = intel_grt_pebs_event_constraints; + x86_pmu.extra_regs = intel_grt_extra_regs; + + x86_pmu.pebs_aliases = NULL; + x86_pmu.pebs_prec_dist = true; + x86_pmu.pebs_block = true; + x86_pmu.lbr_pt_coexist = true; + x86_pmu.flags |= PMU_FL_HAS_RSP_1; + x86_pmu.flags |= PMU_FL_INSTR_LATENCY; + + intel_pmu_pebs_data_source_grt(); + x86_pmu.pebs_latency_data = adl_latency_data_small; + x86_pmu.get_event_constraints = tnt_get_event_constraints; + x86_pmu.limit_period = spr_limit_period; + td_attr = tnt_events_attrs; + mem_attr = grt_mem_attrs; + extra_attr = nhm_format_attr; + pr_cont("Gracemont events, "); + name = "gracemont"; + break; + case INTEL_FAM6_WESTMERE: case INTEL_FAM6_WESTMERE_EP: case INTEL_FAM6_WESTMERE_EX: @@ -6272,8 +6332,10 @@ __init int intel_pmu_init(void) x86_pmu.lbr_pt_coexist = true; intel_pmu_pebs_data_source_skl(pmem); x86_pmu.num_topdown_events = 4; - x86_pmu.update_topdown_event = icl_update_topdown_event; - x86_pmu.set_topdown_event_period = icl_set_topdown_event_period; + static_call_update(intel_pmu_update_topdown_event, + &icl_update_topdown_event); + static_call_update(intel_pmu_set_topdown_event_period, + &icl_set_topdown_event_period); pr_cont("Icelake events, "); name = "icelake"; break; @@ -6291,10 +6353,8 @@ __init int intel_pmu_init(void) x86_pmu.pebs_aliases = NULL; x86_pmu.pebs_prec_dist = true; x86_pmu.pebs_block = true; - x86_pmu.pebs_capable = ~0ULL; x86_pmu.flags |= PMU_FL_HAS_RSP_1; x86_pmu.flags |= PMU_FL_NO_HT_SHARING; - x86_pmu.flags |= PMU_FL_PEBS_ALL; x86_pmu.flags |= PMU_FL_INSTR_LATENCY; x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX; @@ -6310,17 +6370,19 @@ __init int intel_pmu_init(void) x86_pmu.lbr_pt_coexist = true; intel_pmu_pebs_data_source_skl(pmem); x86_pmu.num_topdown_events = 8; - x86_pmu.update_topdown_event = icl_update_topdown_event; - x86_pmu.set_topdown_event_period = icl_set_topdown_event_period; + static_call_update(intel_pmu_update_topdown_event, + &icl_update_topdown_event); + static_call_update(intel_pmu_set_topdown_event_period, + &icl_set_topdown_event_period); pr_cont("Sapphire Rapids events, "); name = "sapphire_rapids"; break; case INTEL_FAM6_ALDERLAKE: case INTEL_FAM6_ALDERLAKE_L: - case INTEL_FAM6_ALDERLAKE_N: case INTEL_FAM6_RAPTORLAKE: case INTEL_FAM6_RAPTORLAKE_P: + case INTEL_FAM6_RAPTORLAKE_S: /* * Alder Lake has 2 types of CPU, core and atom. * @@ -6337,18 +6399,18 @@ __init int intel_pmu_init(void) x86_pmu.pebs_aliases = NULL; x86_pmu.pebs_prec_dist = true; x86_pmu.pebs_block = true; - x86_pmu.pebs_capable = ~0ULL; x86_pmu.flags |= PMU_FL_HAS_RSP_1; x86_pmu.flags |= PMU_FL_NO_HT_SHARING; - x86_pmu.flags |= PMU_FL_PEBS_ALL; x86_pmu.flags |= PMU_FL_INSTR_LATENCY; x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX; x86_pmu.lbr_pt_coexist = true; intel_pmu_pebs_data_source_adl(); x86_pmu.pebs_latency_data = adl_latency_data_small; x86_pmu.num_topdown_events = 8; - x86_pmu.update_topdown_event = adl_update_topdown_event; - x86_pmu.set_topdown_event_period = adl_set_topdown_event_period; + static_call_update(intel_pmu_update_topdown_event, + &adl_update_topdown_event); + static_call_update(intel_pmu_set_topdown_event_period, + &adl_set_topdown_event_period); x86_pmu.filter_match = intel_pmu_filter_match; x86_pmu.get_event_constraints = adl_get_event_constraints; diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 8ec23f47fee9a7a4dd10632c76528d4c61549a7c..a2834bc93149a735295706e2599e5f4b5b667678 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -685,6 +685,7 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &adl_cstates), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_cstates), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_cstates), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index ba60427caa6d36553672509c22a292ea7e2b7d1e..7839507b38448d10a6a393e9ee9cfe40aaaac47b 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -110,13 +110,18 @@ void __init intel_pmu_pebs_data_source_skl(bool pmem) __intel_pmu_pebs_data_source_skl(pmem, pebs_data_source); } -static void __init intel_pmu_pebs_data_source_grt(u64 *data_source) +static void __init __intel_pmu_pebs_data_source_grt(u64 *data_source) { data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT); data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM); data_source[0x08] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD); } +void __init intel_pmu_pebs_data_source_grt(void) +{ + __intel_pmu_pebs_data_source_grt(pebs_data_source); +} + void __init intel_pmu_pebs_data_source_adl(void) { u64 *data_source; @@ -127,7 +132,7 @@ void __init intel_pmu_pebs_data_source_adl(void) data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source; memcpy(data_source, pebs_data_source, sizeof(pebs_data_source)); - intel_pmu_pebs_data_source_grt(data_source); + __intel_pmu_pebs_data_source_grt(data_source); } static u64 precise_store_data(u64 status) @@ -291,6 +296,7 @@ static u64 load_latency_data(struct perf_event *event, u64 status) static u64 store_latency_data(struct perf_event *event, u64 status) { union intel_x86_pebs_dse dse; + union perf_mem_data_src src; u64 val; dse.val = status; @@ -304,7 +310,14 @@ static u64 store_latency_data(struct perf_event *event, u64 status) val |= P(BLK, NA); - return val; + /* + * the pebs_data_source table is only for loads + * so override the mem_op to say STORE instead + */ + src.val = val; + src.mem_op = P(OP,STORE); + + return src.val; } struct pebs_record_core { @@ -822,7 +835,7 @@ struct event_constraint intel_glm_pebs_event_constraints[] = { struct event_constraint intel_grt_pebs_event_constraints[] = { /* Allow all events as PEBS with no flags */ - INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xf), + INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3), INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf), EVENT_CONSTRAINT_END }; @@ -1527,14 +1540,18 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, /* * Use latency for weight (only avail with PEBS-LL) */ - if (fll && (sample_type & PERF_SAMPLE_WEIGHT_TYPE)) + if (fll && (sample_type & PERF_SAMPLE_WEIGHT_TYPE)) { data->weight.full = pebs->lat; + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } /* * data.data_src encodes the data source */ - if (sample_type & PERF_SAMPLE_DATA_SRC) + if (sample_type & PERF_SAMPLE_DATA_SRC) { data->data_src.val = get_data_src(event, pebs->dse); + data->sample_flags |= PERF_SAMPLE_DATA_SRC; + } /* * We must however always use iregs for the unwinder to stay sane; the @@ -1542,8 +1559,10 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, * previous PMI context or an (I)RET happened between the record and * PMI. */ - if (sample_type & PERF_SAMPLE_CALLCHAIN) + if (sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, iregs); + data->sample_flags |= PERF_SAMPLE_CALLCHAIN; + } /* * We use the interrupt regs as a base because the PEBS record does not @@ -1615,17 +1634,22 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, if ((sample_type & PERF_SAMPLE_ADDR_TYPE) && - x86_pmu.intel_cap.pebs_format >= 1) + x86_pmu.intel_cap.pebs_format >= 1) { data->addr = pebs->dla; + data->sample_flags |= PERF_SAMPLE_ADDR; + } if (x86_pmu.intel_cap.pebs_format >= 2) { /* Only set the TSX weight when no memory weight. */ - if ((sample_type & PERF_SAMPLE_WEIGHT_TYPE) && !fll) + if ((sample_type & PERF_SAMPLE_WEIGHT_TYPE) && !fll) { data->weight.full = intel_get_tsx_weight(pebs->tsx_tuning); - - if (sample_type & PERF_SAMPLE_TRANSACTION) + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } + if (sample_type & PERF_SAMPLE_TRANSACTION) { data->txn = intel_get_tsx_transaction(pebs->tsx_tuning, pebs->ax); + data->sample_flags |= PERF_SAMPLE_TRANSACTION; + } } /* @@ -1635,11 +1659,15 @@ static void setup_pebs_fixed_sample_data(struct perf_event *event, * We can only do this for the default trace clock. */ if (x86_pmu.intel_cap.pebs_format >= 3 && - event->attr.use_clockid == 0) + event->attr.use_clockid == 0) { data->time = native_sched_clock_from_tsc(pebs->tsc); + data->sample_flags |= PERF_SAMPLE_TIME; + } - if (has_branch_stack(event)) + if (has_branch_stack(event)) { data->br_stack = &cpuc->lbr_stack; + data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; + } } static void adaptive_pebs_save_regs(struct pt_regs *regs, @@ -1697,8 +1725,10 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, perf_sample_data_init(data, 0, event->hw.last_period); data->period = event->hw.last_period; - if (event->attr.use_clockid == 0) + if (event->attr.use_clockid == 0) { data->time = native_sched_clock_from_tsc(basic->tsc); + data->sample_flags |= PERF_SAMPLE_TIME; + } /* * We must however always use iregs for the unwinder to stay sane; the @@ -1706,8 +1736,10 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, * previous PMI context or an (I)RET happened between the record and * PMI. */ - if (sample_type & PERF_SAMPLE_CALLCHAIN) + if (sample_type & PERF_SAMPLE_CALLCHAIN) { data->callchain = perf_callchain(event, iregs); + data->sample_flags |= PERF_SAMPLE_CALLCHAIN; + } *regs = *iregs; /* The ip in basic is EventingIP */ @@ -1758,17 +1790,24 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, data->weight.var1_dw = (u32)(weight & PEBS_LATENCY_MASK) ?: intel_get_tsx_weight(meminfo->tsx_tuning); } + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; } - if (sample_type & PERF_SAMPLE_DATA_SRC) + if (sample_type & PERF_SAMPLE_DATA_SRC) { data->data_src.val = get_data_src(event, meminfo->aux); + data->sample_flags |= PERF_SAMPLE_DATA_SRC; + } - if (sample_type & PERF_SAMPLE_ADDR_TYPE) + if (sample_type & PERF_SAMPLE_ADDR_TYPE) { data->addr = meminfo->address; + data->sample_flags |= PERF_SAMPLE_ADDR; + } - if (sample_type & PERF_SAMPLE_TRANSACTION) + if (sample_type & PERF_SAMPLE_TRANSACTION) { data->txn = intel_get_tsx_transaction(meminfo->tsx_tuning, gprs ? gprs->ax : 0); + data->sample_flags |= PERF_SAMPLE_TRANSACTION; + } } if (format_size & PEBS_DATACFG_XMMS) { @@ -1787,6 +1826,7 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, if (has_branch_stack(event)) { intel_pmu_store_pebs_lbrs(lbr); data->br_stack = &cpuc->lbr_stack; + data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; } } @@ -2262,6 +2302,7 @@ void __init intel_ds_init(void) PERF_SAMPLE_BRANCH_STACK | PERF_SAMPLE_TIME; x86_pmu.flags |= PMU_FL_PEBS_ALL; + x86_pmu.pebs_capable = ~0ULL; pebs_qual = "-baseline"; x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_EXTENDED_REGS; } else { diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 4f70fb6c2c1eb78e35f717ffb3085ee9f459e145..4fce1a4226e3de27721997811289f9635410e1ee 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -4,7 +4,6 @@ #include #include -#include #include "../perf_event.h" @@ -65,65 +64,6 @@ #define LBR_FROM_SIGNEXT_2MSB (BIT_ULL(60) | BIT_ULL(59)) -/* - * x86control flow change classification - * x86control flow changes include branches, interrupts, traps, faults - */ -enum { - X86_BR_NONE = 0, /* unknown */ - - X86_BR_USER = 1 << 0, /* branch target is user */ - X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ - - X86_BR_CALL = 1 << 2, /* call */ - X86_BR_RET = 1 << 3, /* return */ - X86_BR_SYSCALL = 1 << 4, /* syscall */ - X86_BR_SYSRET = 1 << 5, /* syscall return */ - X86_BR_INT = 1 << 6, /* sw interrupt */ - X86_BR_IRET = 1 << 7, /* return from interrupt */ - X86_BR_JCC = 1 << 8, /* conditional */ - X86_BR_JMP = 1 << 9, /* jump */ - X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ - X86_BR_IND_CALL = 1 << 11,/* indirect calls */ - X86_BR_ABORT = 1 << 12,/* transaction abort */ - X86_BR_IN_TX = 1 << 13,/* in transaction */ - X86_BR_NO_TX = 1 << 14,/* not in transaction */ - X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ - X86_BR_CALL_STACK = 1 << 16,/* call stack */ - X86_BR_IND_JMP = 1 << 17,/* indirect jump */ - - X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */ - -}; - -#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) -#define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) - -#define X86_BR_ANY \ - (X86_BR_CALL |\ - X86_BR_RET |\ - X86_BR_SYSCALL |\ - X86_BR_SYSRET |\ - X86_BR_INT |\ - X86_BR_IRET |\ - X86_BR_JCC |\ - X86_BR_JMP |\ - X86_BR_IRQ |\ - X86_BR_ABORT |\ - X86_BR_IND_CALL |\ - X86_BR_IND_JMP |\ - X86_BR_ZERO_CALL) - -#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) - -#define X86_BR_ANY_CALL \ - (X86_BR_CALL |\ - X86_BR_IND_CALL |\ - X86_BR_ZERO_CALL |\ - X86_BR_SYSCALL |\ - X86_BR_IRQ |\ - X86_BR_INT) - /* * Intel LBR_CTL bits * @@ -1097,6 +1037,14 @@ static int intel_pmu_setup_hw_lbr_filter(struct perf_event *event) if (static_cpu_has(X86_FEATURE_ARCH_LBR)) { reg->config = mask; + + /* + * The Arch LBR HW can retrieve the common branch types + * from the LBR_INFO. It doesn't require the high overhead + * SW disassemble. + * Enable the branch type by default for the Arch LBR. + */ + reg->reg |= X86_BR_TYPE_SAVE; return 0; } @@ -1143,219 +1091,6 @@ int intel_pmu_setup_lbr_filter(struct perf_event *event) return ret; } -/* - * return the type of control flow change at address "from" - * instruction is not necessarily a branch (in case of interrupt). - * - * The branch type returned also includes the priv level of the - * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). - * - * If a branch type is unknown OR the instruction cannot be - * decoded (e.g., text page not present), then X86_BR_NONE is - * returned. - */ -static int branch_type(unsigned long from, unsigned long to, int abort) -{ - struct insn insn; - void *addr; - int bytes_read, bytes_left; - int ret = X86_BR_NONE; - int ext, to_plm, from_plm; - u8 buf[MAX_INSN_SIZE]; - int is64 = 0; - - to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; - from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; - - /* - * maybe zero if lbr did not fill up after a reset by the time - * we get a PMU interrupt - */ - if (from == 0 || to == 0) - return X86_BR_NONE; - - if (abort) - return X86_BR_ABORT | to_plm; - - if (from_plm == X86_BR_USER) { - /* - * can happen if measuring at the user level only - * and we interrupt in a kernel thread, e.g., idle. - */ - if (!current->mm) - return X86_BR_NONE; - - /* may fail if text not present */ - bytes_left = copy_from_user_nmi(buf, (void __user *)from, - MAX_INSN_SIZE); - bytes_read = MAX_INSN_SIZE - bytes_left; - if (!bytes_read) - return X86_BR_NONE; - - addr = buf; - } else { - /* - * The LBR logs any address in the IP, even if the IP just - * faulted. This means userspace can control the from address. - * Ensure we don't blindly read any address by validating it is - * a known text address. - */ - if (kernel_text_address(from)) { - addr = (void *)from; - /* - * Assume we can get the maximum possible size - * when grabbing kernel data. This is not - * _strictly_ true since we could possibly be - * executing up next to a memory hole, but - * it is very unlikely to be a problem. - */ - bytes_read = MAX_INSN_SIZE; - } else { - return X86_BR_NONE; - } - } - - /* - * decoder needs to know the ABI especially - * on 64-bit systems running 32-bit apps - */ -#ifdef CONFIG_X86_64 - is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs()); -#endif - insn_init(&insn, addr, bytes_read, is64); - if (insn_get_opcode(&insn)) - return X86_BR_ABORT; - - switch (insn.opcode.bytes[0]) { - case 0xf: - switch (insn.opcode.bytes[1]) { - case 0x05: /* syscall */ - case 0x34: /* sysenter */ - ret = X86_BR_SYSCALL; - break; - case 0x07: /* sysret */ - case 0x35: /* sysexit */ - ret = X86_BR_SYSRET; - break; - case 0x80 ... 0x8f: /* conditional */ - ret = X86_BR_JCC; - break; - default: - ret = X86_BR_NONE; - } - break; - case 0x70 ... 0x7f: /* conditional */ - ret = X86_BR_JCC; - break; - case 0xc2: /* near ret */ - case 0xc3: /* near ret */ - case 0xca: /* far ret */ - case 0xcb: /* far ret */ - ret = X86_BR_RET; - break; - case 0xcf: /* iret */ - ret = X86_BR_IRET; - break; - case 0xcc ... 0xce: /* int */ - ret = X86_BR_INT; - break; - case 0xe8: /* call near rel */ - if (insn_get_immediate(&insn) || insn.immediate1.value == 0) { - /* zero length call */ - ret = X86_BR_ZERO_CALL; - break; - } - fallthrough; - case 0x9a: /* call far absolute */ - ret = X86_BR_CALL; - break; - case 0xe0 ... 0xe3: /* loop jmp */ - ret = X86_BR_JCC; - break; - case 0xe9 ... 0xeb: /* jmp */ - ret = X86_BR_JMP; - break; - case 0xff: /* call near absolute, call far absolute ind */ - if (insn_get_modrm(&insn)) - return X86_BR_ABORT; - - ext = (insn.modrm.bytes[0] >> 3) & 0x7; - switch (ext) { - case 2: /* near ind call */ - case 3: /* far ind call */ - ret = X86_BR_IND_CALL; - break; - case 4: - case 5: - ret = X86_BR_IND_JMP; - break; - } - break; - default: - ret = X86_BR_NONE; - } - /* - * interrupts, traps, faults (and thus ring transition) may - * occur on any instructions. Thus, to classify them correctly, - * we need to first look at the from and to priv levels. If they - * are different and to is in the kernel, then it indicates - * a ring transition. If the from instruction is not a ring - * transition instr (syscall, systenter, int), then it means - * it was a irq, trap or fault. - * - * we have no way of detecting kernel to kernel faults. - */ - if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL - && ret != X86_BR_SYSCALL && ret != X86_BR_INT) - ret = X86_BR_IRQ; - - /* - * branch priv level determined by target as - * is done by HW when LBR_SELECT is implemented - */ - if (ret != X86_BR_NONE) - ret |= to_plm; - - return ret; -} - -#define X86_BR_TYPE_MAP_MAX 16 - -static int branch_map[X86_BR_TYPE_MAP_MAX] = { - PERF_BR_CALL, /* X86_BR_CALL */ - PERF_BR_RET, /* X86_BR_RET */ - PERF_BR_SYSCALL, /* X86_BR_SYSCALL */ - PERF_BR_SYSRET, /* X86_BR_SYSRET */ - PERF_BR_UNKNOWN, /* X86_BR_INT */ - PERF_BR_ERET, /* X86_BR_IRET */ - PERF_BR_COND, /* X86_BR_JCC */ - PERF_BR_UNCOND, /* X86_BR_JMP */ - PERF_BR_IRQ, /* X86_BR_IRQ */ - PERF_BR_IND_CALL, /* X86_BR_IND_CALL */ - PERF_BR_UNKNOWN, /* X86_BR_ABORT */ - PERF_BR_UNKNOWN, /* X86_BR_IN_TX */ - PERF_BR_UNKNOWN, /* X86_BR_NO_TX */ - PERF_BR_CALL, /* X86_BR_ZERO_CALL */ - PERF_BR_UNKNOWN, /* X86_BR_CALL_STACK */ - PERF_BR_IND, /* X86_BR_IND_JMP */ -}; - -static int -common_branch_type(int type) -{ - int i; - - type >>= 2; /* skip X86_BR_USER and X86_BR_KERNEL */ - - if (type) { - i = __ffs(type); - if (i < X86_BR_TYPE_MAP_MAX) - return branch_map[i]; - } - - return PERF_BR_UNKNOWN; -} - enum { ARCH_LBR_BR_TYPE_JCC = 0, ARCH_LBR_BR_TYPE_NEAR_IND_JMP = 1, diff --git a/arch/x86/events/intel/p4.c b/arch/x86/events/intel/p4.c index 7951a5dc73b630d5bc55b7cb95fefc321ddd0fee..03bbcc2fa2ff8804a9f1e236a1c6546f29a14299 100644 --- a/arch/x86/events/intel/p4.c +++ b/arch/x86/events/intel/p4.c @@ -1006,6 +1006,29 @@ static void p4_pmu_enable_all(int added) } } +static int p4_pmu_set_period(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + s64 left = this_cpu_read(pmc_prev_left[hwc->idx]); + int ret; + + ret = x86_perf_event_set_period(event); + + if (hwc->event_base) { + /* + * This handles erratum N15 in intel doc 249199-029, + * the counter may not be updated correctly on write + * so we need a second write operation to do the trick + * (the official workaround didn't work) + * + * the former idea is taken from OProfile code + */ + wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); + } + + return ret; +} + static int p4_pmu_handle_irq(struct pt_regs *regs) { struct perf_sample_data data; @@ -1044,7 +1067,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) /* event overflow for sure */ perf_sample_data_init(&data, 0, hwc->last_period); - if (!x86_perf_event_set_period(event)) + if (!static_call(x86_pmu_set_period)(event)) continue; @@ -1316,6 +1339,9 @@ static __initconst const struct x86_pmu p4_pmu = { .enable_all = p4_pmu_enable_all, .enable = p4_pmu_enable_event, .disable = p4_pmu_disable_event, + + .set_period = p4_pmu_set_period, + .eventsel = MSR_P4_BPU_CCCR0, .perfctr = MSR_P4_BPU_PERFCTR0, .event_map = p4_pmu_event_map, @@ -1334,15 +1360,6 @@ static __initconst const struct x86_pmu p4_pmu = { .max_period = (1ULL << (ARCH_P4_CNTRVAL_BITS - 1)) - 1, .hw_config = p4_hw_config, .schedule_events = p4_pmu_schedule_events, - /* - * This handles erratum N15 in intel doc 249199-029, - * the counter may not be updated correctly on write - * so we need a second write operation to do the trick - * (the official workaround didn't work) - * - * the former idea is taken from OProfile code - */ - .perfctr_second_write = 1, .format_attrs = intel_p4_formats_attr, }; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index db6c31bca80927cb14bd244da9daab6e6cf9a550..6f1ccc57a6921fd0ffd2d7f924f96bf0e2907a6c 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -1831,6 +1831,7 @@ static const struct x86_cpu_id intel_uncore_match[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &adl_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &adl_uncore_init), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &spr_uncore_init), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &snr_uncore_init), {}, diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index ce440011cc4e411892ae6cd4009b4acba86df1f5..1ef4f7861e2ecb323bad131846cf9df92fc291f0 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -841,6 +841,22 @@ int snb_pci2phy_map_init(int devid) return 0; } +static u64 snb_uncore_imc_read_counter(struct intel_uncore_box *box, struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* + * SNB IMC counters are 32-bit and are laid out back to back + * in MMIO space. Therefore we must use a 32-bit accessor function + * using readq() from uncore_mmio_read_counter() causes problems + * because it is reading 64-bit at a time. This is okay for the + * uncore_perf_event_update() function because it drops the upper + * 32-bits but not okay for plain uncore_read_counter() as invoked + * in uncore_pmu_event_start(). + */ + return (u64)readl(box->io_addr + hwc->event_base); +} + static struct pmu snb_uncore_imc_pmu = { .task_ctx_nr = perf_invalid_context, .event_init = snb_uncore_imc_event_init, @@ -860,7 +876,7 @@ static struct intel_uncore_ops snb_uncore_imc_ops = { .disable_event = snb_uncore_imc_disable_event, .enable_event = snb_uncore_imc_enable_event, .hw_config = snb_uncore_imc_hw_config, - .read_counter = uncore_mmio_read_counter, + .read_counter = snb_uncore_imc_read_counter, }; static struct intel_uncore_type snb_uncore_imc = { diff --git a/arch/x86/events/msr.c b/arch/x86/events/msr.c index ac542f98c0705e1220f0fa0689097deda823165c..ecced3a52668a129e3fbc38d38facaeff9c2ca61 100644 --- a/arch/x86/events/msr.c +++ b/arch/x86/events/msr.c @@ -106,6 +106,7 @@ static bool test_intel(int idx, void *data) case INTEL_FAM6_ALDERLAKE_N: case INTEL_FAM6_RAPTORLAKE: case INTEL_FAM6_RAPTORLAKE_P: + case INTEL_FAM6_RAPTORLAKE_S: if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF) return true; break; diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index ba3d24a6a4ec282369db8842cfe85e2e6166f815..332d2e6d8ae4d87f52cc001986cddeaec8190fc5 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -64,27 +64,25 @@ static inline bool constraint_match(struct event_constraint *c, u64 ecode) return ((ecode & c->cmask) - c->code) <= (u64)c->size; } +#define PERF_ARCH(name, val) \ + PERF_X86_EVENT_##name = val, + /* * struct hw_perf_event.flags flags */ -#define PERF_X86_EVENT_PEBS_LDLAT 0x00001 /* ld+ldlat data address sampling */ -#define PERF_X86_EVENT_PEBS_ST 0x00002 /* st data address sampling */ -#define PERF_X86_EVENT_PEBS_ST_HSW 0x00004 /* haswell style datala, store */ -#define PERF_X86_EVENT_PEBS_LD_HSW 0x00008 /* haswell style datala, load */ -#define PERF_X86_EVENT_PEBS_NA_HSW 0x00010 /* haswell style datala, unknown */ -#define PERF_X86_EVENT_EXCL 0x00020 /* HT exclusivity on counter */ -#define PERF_X86_EVENT_DYNAMIC 0x00040 /* dynamic alloc'd constraint */ - -#define PERF_X86_EVENT_EXCL_ACCT 0x00100 /* accounted EXCL event */ -#define PERF_X86_EVENT_AUTO_RELOAD 0x00200 /* use PEBS auto-reload */ -#define PERF_X86_EVENT_LARGE_PEBS 0x00400 /* use large PEBS */ -#define PERF_X86_EVENT_PEBS_VIA_PT 0x00800 /* use PT buffer for PEBS */ -#define PERF_X86_EVENT_PAIR 0x01000 /* Large Increment per Cycle */ -#define PERF_X86_EVENT_LBR_SELECT 0x02000 /* Save/Restore MSR_LBR_SELECT */ -#define PERF_X86_EVENT_TOPDOWN 0x04000 /* Count Topdown slots/metrics events */ -#define PERF_X86_EVENT_PEBS_STLAT 0x08000 /* st+stlat data address sampling */ -#define PERF_X86_EVENT_AMD_BRS 0x10000 /* AMD Branch Sampling */ -#define PERF_X86_EVENT_PEBS_LAT_HYBRID 0x20000 /* ld and st lat for hybrid */ +enum { +#include "perf_event_flags.h" +}; + +#undef PERF_ARCH + +#define PERF_ARCH(name, val) \ + static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) == \ + PERF_X86_EVENT_##name); + +#include "perf_event_flags.h" + +#undef PERF_ARCH static inline bool is_topdown_count(struct perf_event *event) { @@ -272,6 +270,10 @@ struct cpu_hw_events { u64 active_pebs_data_cfg; int pebs_record_size; + /* Intel Fixed counter configuration */ + u64 fixed_ctrl_val; + u64 active_fixed_ctrl_val; + /* * Intel LBR bits */ @@ -745,6 +747,8 @@ struct x86_pmu { void (*add)(struct perf_event *); void (*del)(struct perf_event *); void (*read)(struct perf_event *event); + int (*set_period)(struct perf_event *event); + u64 (*update)(struct perf_event *event); int (*hw_config)(struct perf_event *event); int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); unsigned eventsel; @@ -780,8 +784,7 @@ struct x86_pmu { struct event_constraint *event_constraints; struct x86_pmu_quirk *quirks; - int perfctr_second_write; - u64 (*limit_period)(struct perf_event *event, u64 l); + void (*limit_period)(struct perf_event *event, s64 *l); /* PMI handler bits */ unsigned int late_ack :1, @@ -889,8 +892,6 @@ struct x86_pmu { * Intel perf metrics */ int num_topdown_events; - u64 (*update_topdown_event)(struct perf_event *event); - int (*set_topdown_event_period)(struct perf_event *event); /* * perf task context (i.e. struct perf_event_context::task_ctx_data) @@ -1044,6 +1045,9 @@ static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\ struct pmu *x86_get_pmu(unsigned int cpu); extern struct x86_pmu x86_pmu __read_mostly; +DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period); +DECLARE_STATIC_CALL(x86_pmu_update, *x86_pmu.update); + static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx) { if (static_cpu_has(X86_FEATURE_ARCH_LBR)) @@ -1059,6 +1063,7 @@ static inline bool x86_pmu_has_lbr_callstack(void) } DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events); +DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); int x86_perf_event_set_period(struct perf_event *event); @@ -1210,6 +1215,70 @@ static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip) regs->ip = ip; } +/* + * x86control flow change classification + * x86control flow changes include branches, interrupts, traps, faults + */ +enum { + X86_BR_NONE = 0, /* unknown */ + + X86_BR_USER = 1 << 0, /* branch target is user */ + X86_BR_KERNEL = 1 << 1, /* branch target is kernel */ + + X86_BR_CALL = 1 << 2, /* call */ + X86_BR_RET = 1 << 3, /* return */ + X86_BR_SYSCALL = 1 << 4, /* syscall */ + X86_BR_SYSRET = 1 << 5, /* syscall return */ + X86_BR_INT = 1 << 6, /* sw interrupt */ + X86_BR_IRET = 1 << 7, /* return from interrupt */ + X86_BR_JCC = 1 << 8, /* conditional */ + X86_BR_JMP = 1 << 9, /* jump */ + X86_BR_IRQ = 1 << 10,/* hw interrupt or trap or fault */ + X86_BR_IND_CALL = 1 << 11,/* indirect calls */ + X86_BR_ABORT = 1 << 12,/* transaction abort */ + X86_BR_IN_TX = 1 << 13,/* in transaction */ + X86_BR_NO_TX = 1 << 14,/* not in transaction */ + X86_BR_ZERO_CALL = 1 << 15,/* zero length call */ + X86_BR_CALL_STACK = 1 << 16,/* call stack */ + X86_BR_IND_JMP = 1 << 17,/* indirect jump */ + + X86_BR_TYPE_SAVE = 1 << 18,/* indicate to save branch type */ + +}; + +#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL) +#define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX) + +#define X86_BR_ANY \ + (X86_BR_CALL |\ + X86_BR_RET |\ + X86_BR_SYSCALL |\ + X86_BR_SYSRET |\ + X86_BR_INT |\ + X86_BR_IRET |\ + X86_BR_JCC |\ + X86_BR_JMP |\ + X86_BR_IRQ |\ + X86_BR_ABORT |\ + X86_BR_IND_CALL |\ + X86_BR_IND_JMP |\ + X86_BR_ZERO_CALL) + +#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY) + +#define X86_BR_ANY_CALL \ + (X86_BR_CALL |\ + X86_BR_IND_CALL |\ + X86_BR_ZERO_CALL |\ + X86_BR_SYSCALL |\ + X86_BR_IRQ |\ + X86_BR_INT) + +int common_branch_type(int type); +int branch_type(unsigned long from, unsigned long to, int abort); +int branch_type_fused(unsigned long from, unsigned long to, int abort, + int *offset); + ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event); ssize_t intel_event_sysfs_show(char *page, u64 config); @@ -1232,7 +1301,20 @@ static inline bool fixed_counter_disabled(int i, struct pmu *pmu) int amd_pmu_init(void); +int amd_pmu_lbr_init(void); +void amd_pmu_lbr_reset(void); +void amd_pmu_lbr_read(void); +void amd_pmu_lbr_add(struct perf_event *event); +void amd_pmu_lbr_del(struct perf_event *event); +void amd_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in); +void amd_pmu_lbr_enable_all(void); +void amd_pmu_lbr_disable_all(void); +int amd_pmu_lbr_hw_config(struct perf_event *event); + #ifdef CONFIG_PERF_EVENTS_AMD_BRS + +#define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */ + int amd_brs_init(void); void amd_brs_disable(void); void amd_brs_enable(void); @@ -1241,7 +1323,7 @@ void amd_brs_disable_all(void); void amd_brs_drain(void); void amd_brs_lopwr_init(void); void amd_brs_disable_all(void); -int amd_brs_setup_filter(struct perf_event *event); +int amd_brs_hw_config(struct perf_event *event); void amd_brs_reset(void); static inline void amd_pmu_brs_add(struct perf_event *event) @@ -1277,7 +1359,7 @@ static inline void amd_brs_enable(void) {} static inline void amd_brs_drain(void) {} static inline void amd_brs_lopwr_init(void) {} static inline void amd_brs_disable_all(void) {} -static inline int amd_brs_setup_filter(struct perf_event *event) +static inline int amd_brs_hw_config(struct perf_event *event) { return 0; } @@ -1516,6 +1598,8 @@ void intel_pmu_pebs_data_source_skl(bool pmem); void intel_pmu_pebs_data_source_adl(void); +void intel_pmu_pebs_data_source_grt(void); + int intel_pmu_setup_lbr_filter(struct perf_event *event); void intel_pt_interrupt(void); diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h new file mode 100644 index 0000000000000000000000000000000000000000..1dc19b9b4426abde8e1b005a7561e8f97b5a3995 --- /dev/null +++ b/arch/x86/events/perf_event_flags.h @@ -0,0 +1,22 @@ + +/* + * struct hw_perf_event.flags flags + */ +PERF_ARCH(PEBS_LDLAT, 0x00001) /* ld+ldlat data address sampling */ +PERF_ARCH(PEBS_ST, 0x00002) /* st data address sampling */ +PERF_ARCH(PEBS_ST_HSW, 0x00004) /* haswell style datala, store */ +PERF_ARCH(PEBS_LD_HSW, 0x00008) /* haswell style datala, load */ +PERF_ARCH(PEBS_NA_HSW, 0x00010) /* haswell style datala, unknown */ +PERF_ARCH(EXCL, 0x00020) /* HT exclusivity on counter */ +PERF_ARCH(DYNAMIC, 0x00040) /* dynamic alloc'd constraint */ + /* 0x00080 */ +PERF_ARCH(EXCL_ACCT, 0x00100) /* accounted EXCL event */ +PERF_ARCH(AUTO_RELOAD, 0x00200) /* use PEBS auto-reload */ +PERF_ARCH(LARGE_PEBS, 0x00400) /* use large PEBS */ +PERF_ARCH(PEBS_VIA_PT, 0x00800) /* use PT buffer for PEBS */ +PERF_ARCH(PAIR, 0x01000) /* Large Increment per Cycle */ +PERF_ARCH(LBR_SELECT, 0x02000) /* Save/Restore MSR_LBR_SELECT */ +PERF_ARCH(TOPDOWN, 0x04000) /* Count Topdown slots/metrics events */ +PERF_ARCH(PEBS_STLAT, 0x08000) /* st+stlat data address sampling */ +PERF_ARCH(AMD_BRS, 0x10000) /* AMD Branch Sampling */ +PERF_ARCH(PEBS_LAT_HYBRID, 0x20000) /* ld and st lat for hybrid */ diff --git a/arch/x86/events/utils.c b/arch/x86/events/utils.c new file mode 100644 index 0000000000000000000000000000000000000000..76b1f8bb0fd5f1898a837278813f66258399274d --- /dev/null +++ b/arch/x86/events/utils.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include "perf_event.h" + +static int decode_branch_type(struct insn *insn) +{ + int ext; + + if (insn_get_opcode(insn)) + return X86_BR_ABORT; + + switch (insn->opcode.bytes[0]) { + case 0xf: + switch (insn->opcode.bytes[1]) { + case 0x05: /* syscall */ + case 0x34: /* sysenter */ + return X86_BR_SYSCALL; + case 0x07: /* sysret */ + case 0x35: /* sysexit */ + return X86_BR_SYSRET; + case 0x80 ... 0x8f: /* conditional */ + return X86_BR_JCC; + } + return X86_BR_NONE; + case 0x70 ... 0x7f: /* conditional */ + return X86_BR_JCC; + case 0xc2: /* near ret */ + case 0xc3: /* near ret */ + case 0xca: /* far ret */ + case 0xcb: /* far ret */ + return X86_BR_RET; + case 0xcf: /* iret */ + return X86_BR_IRET; + case 0xcc ... 0xce: /* int */ + return X86_BR_INT; + case 0xe8: /* call near rel */ + if (insn_get_immediate(insn) || insn->immediate1.value == 0) { + /* zero length call */ + return X86_BR_ZERO_CALL; + } + fallthrough; + case 0x9a: /* call far absolute */ + return X86_BR_CALL; + case 0xe0 ... 0xe3: /* loop jmp */ + return X86_BR_JCC; + case 0xe9 ... 0xeb: /* jmp */ + return X86_BR_JMP; + case 0xff: /* call near absolute, call far absolute ind */ + if (insn_get_modrm(insn)) + return X86_BR_ABORT; + + ext = (insn->modrm.bytes[0] >> 3) & 0x7; + switch (ext) { + case 2: /* near ind call */ + case 3: /* far ind call */ + return X86_BR_IND_CALL; + case 4: + case 5: + return X86_BR_IND_JMP; + } + return X86_BR_NONE; + } + + return X86_BR_NONE; +} + +/* + * return the type of control flow change at address "from" + * instruction is not necessarily a branch (in case of interrupt). + * + * The branch type returned also includes the priv level of the + * target of the control flow change (X86_BR_USER, X86_BR_KERNEL). + * + * If a branch type is unknown OR the instruction cannot be + * decoded (e.g., text page not present), then X86_BR_NONE is + * returned. + * + * While recording branches, some processors can report the "from" + * address to be that of an instruction preceding the actual branch + * when instruction fusion occurs. If fusion is expected, attempt to + * find the type of the first branch instruction within the next + * MAX_INSN_SIZE bytes and if found, provide the offset between the + * reported "from" address and the actual branch instruction address. + */ +static int get_branch_type(unsigned long from, unsigned long to, int abort, + bool fused, int *offset) +{ + struct insn insn; + void *addr; + int bytes_read, bytes_left, insn_offset; + int ret = X86_BR_NONE; + int to_plm, from_plm; + u8 buf[MAX_INSN_SIZE]; + int is64 = 0; + + /* make sure we initialize offset */ + if (offset) + *offset = 0; + + to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + + /* + * maybe zero if lbr did not fill up after a reset by the time + * we get a PMU interrupt + */ + if (from == 0 || to == 0) + return X86_BR_NONE; + + if (abort) + return X86_BR_ABORT | to_plm; + + if (from_plm == X86_BR_USER) { + /* + * can happen if measuring at the user level only + * and we interrupt in a kernel thread, e.g., idle. + */ + if (!current->mm) + return X86_BR_NONE; + + /* may fail if text not present */ + bytes_left = copy_from_user_nmi(buf, (void __user *)from, + MAX_INSN_SIZE); + bytes_read = MAX_INSN_SIZE - bytes_left; + if (!bytes_read) + return X86_BR_NONE; + + addr = buf; + } else { + /* + * The LBR logs any address in the IP, even if the IP just + * faulted. This means userspace can control the from address. + * Ensure we don't blindly read any address by validating it is + * a known text address. + */ + if (kernel_text_address(from)) { + addr = (void *)from; + /* + * Assume we can get the maximum possible size + * when grabbing kernel data. This is not + * _strictly_ true since we could possibly be + * executing up next to a memory hole, but + * it is very unlikely to be a problem. + */ + bytes_read = MAX_INSN_SIZE; + } else { + return X86_BR_NONE; + } + } + + /* + * decoder needs to know the ABI especially + * on 64-bit systems running 32-bit apps + */ +#ifdef CONFIG_X86_64 + is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs()); +#endif + insn_init(&insn, addr, bytes_read, is64); + ret = decode_branch_type(&insn); + insn_offset = 0; + + /* Check for the possibility of branch fusion */ + while (fused && ret == X86_BR_NONE) { + /* Check for decoding errors */ + if (insn_get_length(&insn) || !insn.length) + break; + + insn_offset += insn.length; + bytes_read -= insn.length; + if (bytes_read < 0) + break; + + insn_init(&insn, addr + insn_offset, bytes_read, is64); + ret = decode_branch_type(&insn); + } + + if (offset) + *offset = insn_offset; + + /* + * interrupts, traps, faults (and thus ring transition) may + * occur on any instructions. Thus, to classify them correctly, + * we need to first look at the from and to priv levels. If they + * are different and to is in the kernel, then it indicates + * a ring transition. If the from instruction is not a ring + * transition instr (syscall, systenter, int), then it means + * it was a irq, trap or fault. + * + * we have no way of detecting kernel to kernel faults. + */ + if (from_plm == X86_BR_USER && to_plm == X86_BR_KERNEL + && ret != X86_BR_SYSCALL && ret != X86_BR_INT) + ret = X86_BR_IRQ; + + /* + * branch priv level determined by target as + * is done by HW when LBR_SELECT is implemented + */ + if (ret != X86_BR_NONE) + ret |= to_plm; + + return ret; +} + +int branch_type(unsigned long from, unsigned long to, int abort) +{ + return get_branch_type(from, to, abort, false, NULL); +} + +int branch_type_fused(unsigned long from, unsigned long to, int abort, + int *offset) +{ + return get_branch_type(from, to, abort, true, offset); +} + +#define X86_BR_TYPE_MAP_MAX 16 + +static int branch_map[X86_BR_TYPE_MAP_MAX] = { + PERF_BR_CALL, /* X86_BR_CALL */ + PERF_BR_RET, /* X86_BR_RET */ + PERF_BR_SYSCALL, /* X86_BR_SYSCALL */ + PERF_BR_SYSRET, /* X86_BR_SYSRET */ + PERF_BR_UNKNOWN, /* X86_BR_INT */ + PERF_BR_ERET, /* X86_BR_IRET */ + PERF_BR_COND, /* X86_BR_JCC */ + PERF_BR_UNCOND, /* X86_BR_JMP */ + PERF_BR_IRQ, /* X86_BR_IRQ */ + PERF_BR_IND_CALL, /* X86_BR_IND_CALL */ + PERF_BR_UNKNOWN, /* X86_BR_ABORT */ + PERF_BR_UNKNOWN, /* X86_BR_IN_TX */ + PERF_BR_NO_TX, /* X86_BR_NO_TX */ + PERF_BR_CALL, /* X86_BR_ZERO_CALL */ + PERF_BR_UNKNOWN, /* X86_BR_CALL_STACK */ + PERF_BR_IND, /* X86_BR_IND_JMP */ +}; + +int common_branch_type(int type) +{ + int i; + + type >>= 2; /* skip X86_BR_USER and X86_BR_KERNEL */ + + if (type) { + i = __ffs(type); + if (i < X86_BR_TYPE_MAP_MAX) + return branch_map[i]; + } + + return PERF_BR_UNKNOWN; +} diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 3de6d8b533672bea57e9b15468de3ce835e4e76b..29774126e93144f48fe209dead216f078e18d0ce 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -426,7 +426,7 @@ void __init hyperv_init(void) * 1. Register the guest ID * 2. Enable the hypercall and register the hypercall page */ - guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0); + guest_id = hv_generate_guest_id(LINUX_VERSION_CODE); wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id); /* Hyper-V requires to write guest os id via ghcb in SNP IVM. */ @@ -459,13 +459,13 @@ void __init hyperv_init(void) wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); pg = vmalloc_to_page(hv_hypercall_pg); - dst = kmap(pg); + dst = kmap_local_page(pg); src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE, MEMREMAP_WB); BUG_ON(!(src && dst)); memcpy(dst, src, HV_HYP_PAGE_SIZE); memunmap(src); - kunmap(pg); + kunmap_local(dst); } else { hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h index e003a01b7c6721f6dfcddd148e4be6847b3d67ce..1dd14381bcb6301115026799e2851ba287a16a46 100644 --- a/arch/x86/include/asm/acrn.h +++ b/arch/x86/include/asm/acrn.h @@ -10,6 +10,15 @@ /* Bit 0 indicates whether guest VM is privileged */ #define ACRN_FEATURE_PRIVILEGED_VM BIT(0) +/* + * Timing Information. + * This leaf returns the current TSC frequency in kHz. + * + * EAX: (Virtual) TSC frequency in kHz. + * EBX, ECX, EDX: RESERVED (reserved fields are set to zero). + */ +#define ACRN_CPUID_TIMING_INFO 0x40000010 + void acrn_setup_intr_handler(void (*handler)(void)); void acrn_remove_intr_handler(void); @@ -21,6 +30,11 @@ static inline u32 acrn_cpuid_base(void) return 0; } +static inline unsigned long acrn_get_tsc_khz(void) +{ + return cpuid_eax(ACRN_CPUID_TIMING_INFO); +} + /* * Hypercalls for ACRN * diff --git a/arch/x86/include/asm/amd-ibs.h b/arch/x86/include/asm/amd-ibs.h index f3eb098d63d4bcf8e3d84fadd561d21f7d0f16f3..cb2a5e113daa75d50ae8dd1ed338bd1c77db9273 100644 --- a/arch/x86/include/asm/amd-ibs.h +++ b/arch/x86/include/asm/amd-ibs.h @@ -6,6 +6,22 @@ #include +/* IBS_OP_DATA2 DataSrc */ +#define IBS_DATA_SRC_LOC_CACHE 2 +#define IBS_DATA_SRC_DRAM 3 +#define IBS_DATA_SRC_REM_CACHE 4 +#define IBS_DATA_SRC_IO 7 + +/* IBS_OP_DATA2 DataSrc Extension */ +#define IBS_DATA_SRC_EXT_LOC_CACHE 1 +#define IBS_DATA_SRC_EXT_NEAR_CCX_CACHE 2 +#define IBS_DATA_SRC_EXT_DRAM 3 +#define IBS_DATA_SRC_EXT_FAR_CCX_CACHE 5 +#define IBS_DATA_SRC_EXT_PMEM 6 +#define IBS_DATA_SRC_EXT_IO 7 +#define IBS_DATA_SRC_EXT_EXT_MEM 8 +#define IBS_DATA_SRC_EXT_PEER_AGENT_MEM 12 + /* * IBS Hardware MSRs */ diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 973c6bd17f98e31c65b38693c48f54f701ac32a9..2edf68475fec4138895968c90d7845e08da54d62 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -207,6 +207,20 @@ static __always_inline bool constant_test_bit(long nr, const volatile unsigned l (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } +static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr) +{ + bool oldbit; + + asm volatile("testb %2,%1" + CC_SET(nz) + : CC_OUT(nz) (oldbit) + : "m" (((unsigned char *)addr)[nr >> 3]), + "i" (1 << (nr & 7)) + :"memory"); + + return oldbit; +} + static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr) { bool oldbit; @@ -226,17 +240,37 @@ arch_test_bit(unsigned long nr, const volatile unsigned long *addr) variable_test_bit(nr, addr); } +static __always_inline bool +arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) : + variable_test_bit(nr, addr); +} + +static __always_inline unsigned long variable__ffs(unsigned long word) +{ + asm("rep; bsf %1,%0" + : "=r" (word) + : "rm" (word)); + return word; +} + /** * __ffs - find first set bit in word * @word: The word to search * * Undefined if no bit exists, so code should check against 0 first. */ -static __always_inline unsigned long __ffs(unsigned long word) +#define __ffs(word) \ + (__builtin_constant_p(word) ? \ + (unsigned long)__builtin_ctzl(word) : \ + variable__ffs(word)) + +static __always_inline unsigned long variable_ffz(unsigned long word) { asm("rep; bsf %1,%0" : "=r" (word) - : "rm" (word)); + : "r" (~word)); return word; } @@ -246,13 +280,10 @@ static __always_inline unsigned long __ffs(unsigned long word) * * Undefined if no zero exists, so code should check against ~0UL first. */ -static __always_inline unsigned long ffz(unsigned long word) -{ - asm("rep; bsf %1,%0" - : "=r" (word) - : "r" (~word)); - return word; -} +#define ffz(word) \ + (__builtin_constant_p(word) ? \ + (unsigned long)__builtin_ctzl(~word) : \ + variable_ffz(word)) /* * __fls: find last set bit in word @@ -271,18 +302,7 @@ static __always_inline unsigned long __fls(unsigned long word) #undef ADDR #ifdef __KERNEL__ -/** - * ffs - find first set bit in word - * @x: the word to search - * - * This is defined the same way as the libc and compiler builtin ffs - * routines, therefore differs in spirit from the other bitops. - * - * ffs(value) returns 0 if value is 0 or the position of the first - * set bit if value is nonzero. The first (least significant) bit - * is at position 1. - */ -static __always_inline int ffs(int x) +static __always_inline int variable_ffs(int x) { int r; @@ -312,6 +332,19 @@ static __always_inline int ffs(int x) return r + 1; } +/** + * ffs - find first set bit in word + * @x: the word to search + * + * This is defined the same way as the libc and compiler builtin ffs + * routines, therefore differs in spirit from the other bitops. + * + * ffs(value) returns 0 if value is 0 or the position of the first + * set bit if value is nonzero. The first (least significant) bit + * is at position 1. + */ +#define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x)) + /** * fls - find last set bit in word * @x: the word to search diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h new file mode 100644 index 0000000000000000000000000000000000000000..58dacd90daefc4308d98407b7ac0bf02b717126d --- /dev/null +++ b/arch/x86/include/asm/cfi.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_CFI_H +#define _ASM_X86_CFI_H + +/* + * Clang Control Flow Integrity (CFI) support. + * + * Copyright (C) 2022 Google LLC + */ + +#include + +#ifdef CONFIG_CFI_CLANG +enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#else +static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) +{ + return BUG_TRAP_TYPE_NONE; +} +#endif /* CONFIG_CFI_CLANG */ + +#endif /* _ASM_X86_CFI_H */ diff --git a/arch/x86/include/asm/checksum.h b/arch/x86/include/asm/checksum.h index bca625a60186c9ad77eebefc6ce10c650a02d5f4..6df6ece8a28ecbec0c91090182e3454eb689cb18 100644 --- a/arch/x86/include/asm/checksum.h +++ b/arch/x86/include/asm/checksum.h @@ -1,9 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1 -#define HAVE_CSUM_COPY_USER -#define _HAVE_ARCH_CSUM_AND_COPY -#ifdef CONFIG_X86_32 -# include +#ifdef CONFIG_GENERIC_CSUM +# include #else -# include +# define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1 +# define HAVE_CSUM_COPY_USER +# define _HAVE_ARCH_CSUM_AND_COPY +# ifdef CONFIG_X86_32 +# include +# else +# include +# endif #endif diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h index 8cbf623f0ecfb6aec905ac8e376c7b0fe75933dc..b472ef76826ad93033128dd8198f32f83a9d584c 100644 --- a/arch/x86/include/asm/cpu.h +++ b/arch/x86/include/asm/cpu.h @@ -94,4 +94,6 @@ static inline bool intel_cpu_signatures_match(unsigned int s1, unsigned int p1, return p1 & p2; } +extern u64 x86_read_arch_cap_msr(void); + #endif /* _ASM_X86_CPU_H */ diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 235dc85c91c3e372980b8b428e9713869115ead4..b71f4f2ecdd571a44a808e443118278ea30cbe04 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -96,7 +96,7 @@ #define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in IA32 userspace */ #define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in IA32 userspace */ #define X86_FEATURE_REP_GOOD ( 3*32+16) /* REP microcode works well */ -/* FREE! ( 3*32+17) */ +#define X86_FEATURE_AMD_LBR_V2 ( 3*32+17) /* AMD Last Branch Record Extension Version 2 */ #define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" LFENCE synchronizes RDTSC */ #define X86_FEATURE_ACC_POWER ( 3*32+19) /* AMD Accumulated Power Mechanism */ #define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ @@ -457,7 +457,8 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ -#define X86_BUG_RETBLEED X86_BUG(26) /* CPU is affected by RETBleed */ -#define X86_BUG_EIBRS_PBRSB X86_BUG(27) /* EIBRS is vulnerable to Post Barrier RSB Predictions */ +#define X86_BUG_MMIO_UNKNOWN X86_BUG(26) /* CPU is too old and its MMIO Stale Data status is unknown */ +#define X86_BUG_RETBLEED X86_BUG(27) /* CPU is affected by RETBleed */ +#define X86_BUG_EIBRS_PBRSB X86_BUG(28) /* EIBRS is vulnerable to Post Barrier RSB Predictions */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b5ef474be858d4278f05e778bf8985ed5636b998..908d99b127d31b1c446009471794160fd96d032a 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -23,7 +23,6 @@ #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR #ifndef __ASSEMBLY__ -extern atomic_t modifying_ftrace_code; extern void __fentry__(void); static inline unsigned long ftrace_call_adjust(unsigned long addr) diff --git a/arch/x86/include/asm/hw_breakpoint.h b/arch/x86/include/asm/hw_breakpoint.h index a1f0e90d0818054c2bfb886fff380cbf336c5929..0bc931cd0698bf0214689821743088bae22e9c0a 100644 --- a/arch/x86/include/asm/hw_breakpoint.h +++ b/arch/x86/include/asm/hw_breakpoint.h @@ -44,10 +44,7 @@ struct arch_hw_breakpoint { /* Total number of available HW breakpoint registers */ #define HBP_NUM 4 -static inline int hw_breakpoint_slots(int type) -{ - return HBP_NUM; -} +#define hw_breakpoint_slots(type) (HBP_NUM) struct perf_event_attr; struct perf_event; diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 0a9407dc085980fd147a58e80e4eb168d80d5441..3089ec352743b6e13a3fde63c01e09029cb797ad 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -138,6 +138,9 @@ #define HV_X64_NESTED_GUEST_MAPPING_FLUSH BIT(18) #define HV_X64_NESTED_MSR_BITMAP BIT(19) +/* Nested features #2. These are HYPERV_CPUID_NESTED_FEATURES.EBX bits. */ +#define HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL BIT(0) + /* * This is specific to AMD and specifies that enlightened TLB flush is * supported. If guest opts in to this feature, ASID invalidations only @@ -546,7 +549,7 @@ struct hv_enlightened_vmcs { u64 guest_rip; u32 hv_clean_fields; - u32 hv_padding_32; + u32 padding32_1; u32 hv_synthetic_controls; struct { u32 nested_flush_hypercall:1; @@ -554,14 +557,25 @@ struct hv_enlightened_vmcs { u32 reserved:30; } __packed hv_enlightenments_control; u32 hv_vp_id; - + u32 padding32_2; u64 hv_vm_id; u64 partition_assist_page; u64 padding64_4[4]; u64 guest_bndcfgs; - u64 padding64_5[7]; + u64 guest_ia32_perf_global_ctrl; + u64 guest_ia32_s_cet; + u64 guest_ssp; + u64 guest_ia32_int_ssp_table_addr; + u64 guest_ia32_lbr_ctl; + u64 padding64_5[2]; u64 xss_exit_bitmap; - u64 padding64_6[7]; + u64 encls_exiting_bitmap; + u64 host_ia32_perf_global_ctrl; + u64 tsc_multiplier; + u64 host_ia32_s_cet; + u64 host_ssp; + u64 host_ia32_int_ssp_table_addr; + u64 padding64_6; } __packed; #define HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE 0 diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index def6ca121111ce2873749ea418c43169caadb412..5d75fe22934217bf6d01084b5da943803099798c 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -27,6 +27,7 @@ * _X - regular server parts * _D - micro server parts * _N,_P - other mobile parts + * _S - other client parts * * Historical OPTDIFFs: * @@ -112,6 +113,10 @@ #define INTEL_FAM6_RAPTORLAKE 0xB7 #define INTEL_FAM6_RAPTORLAKE_P 0xBA +#define INTEL_FAM6_RAPTORLAKE_S 0xBF + +#define INTEL_FAM6_METEORLAKE 0xAC +#define INTEL_FAM6_METEORLAKE_L 0xAA /* "Small Core" Processors (Atom) */ diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 63f818aedf770c65579d4bd48a38f9147a104542..147cb8fdda92e9e0a9a2e74bb876968af8ba29dc 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -203,7 +203,7 @@ IRQ_CONSTRAINTS, regs, vector); \ } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK /* * Macro to invoke __do_softirq on the irq stack. This is only called from * task context when bottom halves are about to be reenabled and soft diff --git a/arch/x86/include/asm/kmsan.h b/arch/x86/include/asm/kmsan.h new file mode 100644 index 0000000000000000000000000000000000000000..8fa6ac0e2d7665f936756748c0e1b4ab08a2c5a7 --- /dev/null +++ b/arch/x86/include/asm/kmsan.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * x86 KMSAN support. + * + * Copyright (C) 2022, Google LLC + * Author: Alexander Potapenko + */ + +#ifndef _ASM_X86_KMSAN_H +#define _ASM_X86_KMSAN_H + +#ifndef MODULE + +#include +#include +#include + +DECLARE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow); +DECLARE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin); + +/* + * Functions below are declared in the header to make sure they are inlined. + * They all are called from kmsan_get_metadata() for every memory access in + * the kernel, so speed is important here. + */ + +/* + * Compute metadata addresses for the CPU entry area on x86. + */ +static inline void *arch_kmsan_get_meta_or_null(void *addr, bool is_origin) +{ + unsigned long addr64 = (unsigned long)addr; + char *metadata_array; + unsigned long off; + int cpu; + + if ((addr64 < CPU_ENTRY_AREA_BASE) || + (addr64 >= (CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE))) + return NULL; + cpu = (addr64 - CPU_ENTRY_AREA_BASE) / CPU_ENTRY_AREA_SIZE; + off = addr64 - (unsigned long)get_cpu_entry_area(cpu); + if ((off < 0) || (off >= CPU_ENTRY_AREA_SIZE)) + return NULL; + metadata_array = is_origin ? cpu_entry_area_origin : + cpu_entry_area_shadow; + return &per_cpu(metadata_array[off], cpu); +} + +/* + * Taken from arch/x86/mm/physaddr.h to avoid using an instrumented version. + */ +static inline bool kmsan_phys_addr_valid(unsigned long addr) +{ + if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT)) + return !(addr >> boot_cpu_data.x86_phys_bits); + else + return true; +} + +/* + * Taken from arch/x86/mm/physaddr.c to avoid using an instrumented version. + */ +static inline bool kmsan_virt_addr_valid(void *addr) +{ + unsigned long x = (unsigned long)addr; + unsigned long y = x - __START_KERNEL_map; + + /* use the carry flag to determine if x was < __START_KERNEL_map */ + if (unlikely(x > y)) { + x = y + phys_base; + + if (y >= KERNEL_IMAGE_SIZE) + return false; + } else { + x = y + (__START_KERNEL_map - PAGE_OFFSET); + + /* carry flag will be set if starting x was >= PAGE_OFFSET */ + if ((x > y) || !kmsan_phys_addr_valid(x)) + return false; + } + + return pfn_valid(x >> PAGE_SHIFT); +} + +#endif /* !MODULE */ + +#endif /* _ASM_X86_KMSAN_H */ diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h index 71ea2eab43d5102639d7f46d56d02a473ce1d597..a2e9317aad4955c1110e9f19d5b94ffd837b3865 100644 --- a/arch/x86/include/asm/kprobes.h +++ b/arch/x86/include/asm/kprobes.h @@ -50,8 +50,6 @@ extern const int kretprobe_blacklist_size; void arch_remove_kprobe(struct kprobe *p); -extern void arch_kprobe_override_function(struct pt_regs *regs); - /* Architecture specific copy of original instruction*/ struct arch_specific_insn { /* copy of the original instruction */ diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 51f7770715848968da6ece4c5cb95ec18f089c10..82ba4a564e5875abd05f5e1a34e76473eb20a109 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -67,7 +67,7 @@ KVM_X86_OP(get_interrupt_shadow) KVM_X86_OP(patch_hypercall) KVM_X86_OP(inject_irq) KVM_X86_OP(inject_nmi) -KVM_X86_OP(queue_exception) +KVM_X86_OP(inject_exception) KVM_X86_OP(cancel_injection) KVM_X86_OP(interrupt_allowed) KVM_X86_OP(nmi_allowed) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 2c96c43c313a260659f82a822afe2a7f9c8a78ce..7551b6f9c31c52246c76e32d09214e071efdb5b5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -615,6 +615,8 @@ struct kvm_vcpu_hv { u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ + u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */ + u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */ } cpuid_cache; }; @@ -639,6 +641,16 @@ struct kvm_vcpu_xen { struct timer_list poll_timer; }; +struct kvm_queued_exception { + bool pending; + bool injected; + bool has_error_code; + u8 vector; + u32 error_code; + unsigned long payload; + bool has_payload; +}; + struct kvm_vcpu_arch { /* * rip and regs accesses must go through @@ -729,6 +741,7 @@ struct kvm_vcpu_arch { struct fpu_guest guest_fpu; u64 xcr0; + u64 guest_supported_xcr0; struct kvm_pio_request pio; void *pio_data; @@ -737,16 +750,12 @@ struct kvm_vcpu_arch { u8 event_exit_inst_len; - struct kvm_queued_exception { - bool pending; - bool injected; - bool has_error_code; - u8 nr; - u32 error_code; - unsigned long payload; - bool has_payload; - u8 nested_apf; - } exception; + bool exception_from_userspace; + + /* Exceptions to be injected to the guest. */ + struct kvm_queued_exception exception; + /* Exception VM-Exits to be synthesized to L1. */ + struct kvm_queued_exception exception_vmexit; struct kvm_queued_interrupt { bool injected; @@ -857,7 +866,6 @@ struct kvm_vcpu_arch { u32 id; bool send_user_only; u32 host_apf_flags; - unsigned long nested_apf_token; bool delivery_as_pf_vmexit; bool pageready_pending; } apf; @@ -1272,8 +1280,8 @@ struct kvm_arch { bool tdp_mmu_enabled; /* - * List of struct kvm_mmu_pages being used as roots. - * All struct kvm_mmu_pages in the list should have + * List of kvm_mmu_page structs being used as roots. + * All kvm_mmu_page structs in the list should have * tdp_mmu_page set. * * For reads, this list is protected by: @@ -1292,8 +1300,8 @@ struct kvm_arch { struct list_head tdp_mmu_roots; /* - * List of struct kvmp_mmu_pages not being used as roots. - * All struct kvm_mmu_pages in the list should have + * List of kvm_mmu_page structs not being used as roots. + * All kvm_mmu_page structs in the list should have * tdp_mmu_page set and a tdp_mmu_root_count of 0. */ struct list_head tdp_mmu_pages; @@ -1303,9 +1311,9 @@ struct kvm_arch { * is held in read mode: * - tdp_mmu_roots (above) * - tdp_mmu_pages (above) - * - the link field of struct kvm_mmu_pages used by the TDP MMU + * - the link field of kvm_mmu_page structs used by the TDP MMU * - lpage_disallowed_mmu_pages - * - the lpage_disallowed_link field of struct kvm_mmu_pages used + * - the lpage_disallowed_link field of kvm_mmu_page structs used * by the TDP MMU * It is acceptable, but not necessary, to acquire this lock when * the thread holds the MMU lock in write mode. @@ -1523,7 +1531,7 @@ struct kvm_x86_ops { unsigned char *hypercall_addr); void (*inject_irq)(struct kvm_vcpu *vcpu, bool reinjected); void (*inject_nmi)(struct kvm_vcpu *vcpu); - void (*queue_exception)(struct kvm_vcpu *vcpu); + void (*inject_exception)(struct kvm_vcpu *vcpu); void (*cancel_injection)(struct kvm_vcpu *vcpu); int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); @@ -1633,10 +1641,10 @@ struct kvm_x86_ops { struct kvm_x86_nested_ops { void (*leave_nested)(struct kvm_vcpu *vcpu); + bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, + u32 error_code); int (*check_events)(struct kvm_vcpu *vcpu); - bool (*handle_page_fault_workaround)(struct kvm_vcpu *vcpu, - struct x86_exception *fault); - bool (*hv_timer_pending)(struct kvm_vcpu *vcpu); + bool (*has_events)(struct kvm_vcpu *vcpu); void (*triple_fault)(struct kvm_vcpu *vcpu); int (*get_state)(struct kvm_vcpu *vcpu, struct kvm_nested_state __user *user_kvm_nested_state, @@ -1862,7 +1870,7 @@ void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long pay void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); -bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, +void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl); bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr); diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index 73ca2004983562ca3774d5a6796cf80e231c312f..f484d656d34ee0a8ae607d85c3f586ca608c5d90 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -43,6 +43,18 @@ #endif /* __ASSEMBLY__ */ +#define __CFI_TYPE(name) \ + SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE) \ + .fill 11, 1, 0x90 ASM_NL \ + .byte 0xb8 ASM_NL \ + .long __kcfi_typeid_##name ASM_NL \ + SYM_FUNC_END(__cfi_##name) + +/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */ +#define SYM_TYPED_FUNC_START(name) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ + ENDBR + /* SYM_FUNC_START -- use for global functions */ #define SYM_FUNC_START(name) \ SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) \ diff --git a/arch/x86/include/asm/mc146818rtc.h b/arch/x86/include/asm/mc146818rtc.h index 97198001e56760b793b73a7175fc962783316658..6115bb3d5795f19a80d75e60a769514a35792adb 100644 --- a/arch/x86/include/asm/mc146818rtc.h +++ b/arch/x86/include/asm/mc146818rtc.h @@ -95,7 +95,7 @@ static inline unsigned char current_lock_cmos_reg(void) unsigned char rtc_cmos_read(unsigned char addr); void rtc_cmos_write(unsigned char val, unsigned char addr); -extern int mach_set_rtc_mmss(const struct timespec64 *now); +extern int mach_set_cmos_time(const struct timespec64 *now); extern void mach_get_cmos_time(struct timespec64 *now); #define RTC_IRQ 8 diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index cc73061e7255dd6796041c3f199b26e231c098e4..6e986088817d510be6e957247533495f2b5594d8 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -42,6 +42,7 @@ #define MCI_STATUS_CEC_SHIFT 38 /* Corrected Error Count */ #define MCI_STATUS_CEC_MASK GENMASK_ULL(52,38) #define MCI_STATUS_CEC(c) (((c) & MCI_STATUS_CEC_MASK) >> MCI_STATUS_CEC_SHIFT) +#define MCI_STATUS_MSCOD(m) (((m) >> 16) & 0xffff) /* AMD-specific bits */ #define MCI_STATUS_TCC BIT_ULL(55) /* Task context corrupt */ diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h index 0c3d3440fe27876c0438d16d9dead603498ea3e8..74ecc2bd6cd0f1ace97a9b5340d78d5b77893a94 100644 --- a/arch/x86/include/asm/microcode.h +++ b/arch/x86/include/asm/microcode.h @@ -9,6 +9,7 @@ struct ucode_patch { struct list_head plist; void *data; /* Intel uses only this one */ + unsigned int size; u32 patch_id; u16 equiv_cpu; }; @@ -32,9 +33,6 @@ enum ucode_state { }; struct microcode_ops { - enum ucode_state (*request_microcode_user) (int cpu, - const void __user *buf, size_t size); - enum ucode_state (*request_microcode_fw) (int cpu, struct device *, bool refresh_fw); diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 6674bdb096f346d940e353e3c4df7491fd5e0779..10ac52705892a1680415d144877574a2457f0344 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -155,6 +155,11 @@ * Return Stack Buffer Predictions. */ +#define ARCH_CAP_XAPIC_DISABLE BIT(21) /* + * IA32_XAPIC_DISABLE_STATUS MSR + * supported + */ + #define MSR_IA32_FLUSH_CMD 0x0000010b #define L1D_FLUSH BIT(0) /* * Writeback and invalidate the @@ -585,6 +590,9 @@ #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 +/* AMD Last Branch Record MSRs */ +#define MSR_AMD64_LBR_SELECT 0xc000010e + /* Fam 17h MSRs */ #define MSR_F17H_IRPERF 0xc00000e9 @@ -756,6 +764,8 @@ #define MSR_AMD_DBG_EXTN_CFG 0xc000010f #define MSR_AMD_SAMP_BR_FROM 0xc0010300 +#define DBG_EXTN_CFG_LBRV2EN BIT_ULL(6) + #define MSR_IA32_MPERF 0x000000e7 #define MSR_IA32_APERF 0x000000e8 @@ -1054,4 +1064,12 @@ #define MSR_IA32_HW_FEEDBACK_PTR 0x17d0 #define MSR_IA32_HW_FEEDBACK_CONFIG 0x17d1 +/* x2APIC locked status */ +#define MSR_IA32_XAPIC_DISABLE_STATUS 0xBD +#define LEGACY_XAPIC_DISABLED BIT(0) /* + * x2APIC mode is locked and + * disabling x2APIC will cause + * a #GP + */ + #endif /* _ASM_X86_MSR_INDEX_H */ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index e64fd20778b61a23b59c8d04eba6cd5efc8cde0f..c936ce9f0c47c896efc83fc1f9370db429be142b 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -35,33 +35,56 @@ #define RSB_CLEAR_LOOPS 32 /* To forcibly overwrite all entries */ /* + * Common helper for __FILL_RETURN_BUFFER and __FILL_ONE_RETURN. + */ +#define __FILL_RETURN_SLOT \ + ANNOTATE_INTRA_FUNCTION_CALL; \ + call 772f; \ + int3; \ +772: + +/* + * Stuff the entire RSB. + * * Google experimented with loop-unrolling and this turned out to be * the optimal version - two calls, each with their own speculation * trap should their return address end up getting used, in a loop. */ -#define __FILL_RETURN_BUFFER(reg, nr, sp) \ - mov $(nr/2), reg; \ -771: \ - ANNOTATE_INTRA_FUNCTION_CALL; \ - call 772f; \ -773: /* speculation trap */ \ - UNWIND_HINT_EMPTY; \ - pause; \ - lfence; \ - jmp 773b; \ -772: \ - ANNOTATE_INTRA_FUNCTION_CALL; \ - call 774f; \ -775: /* speculation trap */ \ - UNWIND_HINT_EMPTY; \ - pause; \ - lfence; \ - jmp 775b; \ -774: \ - add $(BITS_PER_LONG/8) * 2, sp; \ - dec reg; \ - jnz 771b; \ - /* barrier for jnz misprediction */ \ +#ifdef CONFIG_X86_64 +#define __FILL_RETURN_BUFFER(reg, nr) \ + mov $(nr/2), reg; \ +771: \ + __FILL_RETURN_SLOT \ + __FILL_RETURN_SLOT \ + add $(BITS_PER_LONG/8) * 2, %_ASM_SP; \ + dec reg; \ + jnz 771b; \ + /* barrier for jnz misprediction */ \ + lfence; +#else +/* + * i386 doesn't unconditionally have LFENCE, as such it can't + * do a loop. + */ +#define __FILL_RETURN_BUFFER(reg, nr) \ + .rept nr; \ + __FILL_RETURN_SLOT; \ + .endr; \ + add $(BITS_PER_LONG/8) * nr, %_ASM_SP; +#endif + +/* + * Stuff a single RSB slot. + * + * To mitigate Post-Barrier RSB speculation, one CALL instruction must be + * forced to retire before letting a RET instruction execute. + * + * On PBRSB-vulnerable CPUs, it is not safe for a RET to be executed + * before this point. + */ +#define __FILL_ONE_RETURN \ + __FILL_RETURN_SLOT \ + add $(BITS_PER_LONG/8), %_ASM_SP; \ lfence; #ifdef __ASSEMBLY__ @@ -132,28 +155,15 @@ #endif .endm -.macro ISSUE_UNBALANCED_RET_GUARD - ANNOTATE_INTRA_FUNCTION_CALL - call .Lunbalanced_ret_guard_\@ - int3 -.Lunbalanced_ret_guard_\@: - add $(BITS_PER_LONG/8), %_ASM_SP - lfence -.endm - /* * A simpler FILL_RETURN_BUFFER macro. Don't make people use the CPP * monstrosity above, manually. */ -.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2 -.ifb \ftr2 - ALTERNATIVE "jmp .Lskip_rsb_\@", "", \ftr -.else - ALTERNATIVE_2 "jmp .Lskip_rsb_\@", "", \ftr, "jmp .Lunbalanced_\@", \ftr2 -.endif - __FILL_RETURN_BUFFER(\reg,\nr,%_ASM_SP) -.Lunbalanced_\@: - ISSUE_UNBALANCED_RET_GUARD +.macro FILL_RETURN_BUFFER reg:req nr:req ftr:req ftr2=ALT_NOT(X86_FEATURE_ALWAYS) + ALTERNATIVE_2 "jmp .Lskip_rsb_\@", \ + __stringify(__FILL_RETURN_BUFFER(\reg,\nr)), \ftr, \ + __stringify(__FILL_ONE_RETURN), \ftr2 + .Lskip_rsb_\@: .endm diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index baa70451b8df5d20ef0731b82f7409f9f9ff05d5..198e03e59ca19ab0ccd9670d4c5eafc2937a9ff8 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -8,6 +8,8 @@ #include #include +#include + /* duplicated to the one in bootmem.h */ extern unsigned long max_pfn; extern unsigned long phys_base; @@ -47,6 +49,11 @@ void clear_page_erms(void *page); static inline void clear_page(void *page) { + /* + * Clean up KMSAN metadata for the page being cleared. The assembly call + * below clobbers @page, so we perform unpoisoning before it. + */ + kmsan_unpoison_memory(page, PAGE_SIZE); alternative_call_2(clear_page_orig, clear_page_rep, X86_FEATURE_REP_GOOD, clear_page_erms, X86_FEATURE_ERMS, diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 964442b99245318df7d18ae80be6560d8682cd3e..2a0b8dd4ec33553c30584f520bed2f818f85ce62 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -743,6 +743,7 @@ extern void default_banner(void); word 771b; \ .byte ptype; \ .byte 772b-771b; \ + _ASM_ALIGN; \ .popsection diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 89df6c6617f50b439f4a24ef22e1bc81ee4633f1..f3d601574730ce117295d4f69bc43202f635b599 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -294,6 +294,7 @@ extern struct paravirt_patch_template pv_ops; " .byte " type "\n" \ " .byte 772b-771b\n" \ " .short " clobber "\n" \ + _ASM_ALIGN "\n" \ ".popsection\n" /* Generate patchable code, with the default asm parameters. */ @@ -328,7 +329,7 @@ int paravirt_disable_iospace(void); * Unfortunately, this is a relatively slow operation for modern CPUs, * because it cannot necessarily determine what the destination * address is. In this case, the address is a runtime constant, so at - * the very least we can patch the call to e a simple direct call, or + * the very least we can patch the call to a simple direct call, or, * ideally, patch an inline implementation into the callsite. (Direct * calls are essentially free, because the call and return addresses * are completely predictable.) @@ -339,10 +340,10 @@ int paravirt_disable_iospace(void); * on the stack. All caller-save registers (eax,edx,ecx) are expected * to be modified (either clobbered or used for return values). * X86_64, on the other hand, already specifies a register-based calling - * conventions, returning at %rax, with parameters going on %rdi, %rsi, + * conventions, returning at %rax, with parameters going in %rdi, %rsi, * %rdx, and %rcx. Note that for this reason, x86_64 does not need any * special handling for dealing with 4 arguments, unlike i386. - * However, x86_64 also have to clobber all caller saved registers, which + * However, x86_64 also has to clobber all caller saved registers, which * unfortunately, are quite a bit (r8 - r11) * * The call instruction itself is marked by placing its start address @@ -360,22 +361,22 @@ int paravirt_disable_iospace(void); * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments. * It could be extended to more arguments, but there would be little * to be gained from that. For each number of arguments, there are - * the two VCALL and CALL variants for void and non-void functions. + * two VCALL and CALL variants for void and non-void functions. * * When there is a return value, the invoker of the macro must specify * the return type. The macro then uses sizeof() on that type to - * determine whether its a 32 or 64 bit value, and places the return + * determine whether it's a 32 or 64 bit value and places the return * in the right register(s) (just %eax for 32-bit, and %edx:%eax for - * 64-bit). For x86_64 machines, it just returns at %rax regardless of + * 64-bit). For x86_64 machines, it just returns in %rax regardless of * the return value size. * - * 64-bit arguments are passed as a pair of adjacent 32-bit arguments + * 64-bit arguments are passed as a pair of adjacent 32-bit arguments; * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments * in low,high order * * Small structures are passed and returned in registers. The macro * calling convention can't directly deal with this, so the wrapper - * functions must do this. + * functions must do it. * * These PVOP_* macros are only defined within this header. This * means that all uses must be wrapped in inline functions. This also @@ -414,8 +415,17 @@ int paravirt_disable_iospace(void); "=c" (__ecx) #define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax) -/* void functions are still allowed [re]ax for scratch */ +/* + * void functions are still allowed [re]ax for scratch. + * + * The ZERO_CALL_USED REGS feature may end up zeroing out callee-saved + * registers. Make sure we model this with the appropriate clobbers. + */ +#ifdef CONFIG_ZERO_CALL_USED_REGS +#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), PVOP_VCALL_CLOBBERS +#else #define PVOP_VCALLEE_CLOBBERS "=a" (__eax) +#endif #define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS #define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11" diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h index f6fc8dd51ef477f894d41a59f30e49819fecedb4..9ac46dbe57d48fc3d6eb53471ef725899917f1d7 100644 --- a/arch/x86/include/asm/perf_event.h +++ b/arch/x86/include/asm/perf_event.h @@ -207,7 +207,8 @@ union cpuid_0x80000022_ebx { struct { /* Number of Core Performance Counters */ unsigned int num_core_pmc:4; - unsigned int reserved:6; + /* Number of available LBR Stack Entries */ + unsigned int lbr_v2_stack_sz:6; /* Number of Data Fabric Counters */ unsigned int num_df_pmc:6; } split; diff --git a/arch/x86/include/asm/pgtable-3level.h b/arch/x86/include/asm/pgtable-3level.h index e896ebef8c24cbeb2273b490f28f4e2f7f086afb..28421a88720939e20ef4354684012e3a5f021bd8 100644 --- a/arch/x86/include/asm/pgtable-3level.h +++ b/arch/x86/include/asm/pgtable-3level.h @@ -256,10 +256,10 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp) /* We always extract/encode the offset by shifting it all the way up, and then down again */ #define SWP_OFFSET_SHIFT (SWP_OFFSET_FIRST_BIT + SWP_TYPE_BITS) -#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > 5) -#define __swp_type(x) (((x).val) & 0x1f) -#define __swp_offset(x) ((x).val >> 5) -#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << 5}) +#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) +#define __swp_type(x) (((x).val) & ((1UL << SWP_TYPE_BITS) - 1)) +#define __swp_offset(x) ((x).val >> SWP_TYPE_BITS) +#define __swp_entry(type, offset) ((swp_entry_t){(type) | (offset) << SWP_TYPE_BITS}) /* * Normally, __swp_entry() converts from arch-independent swp_entry_t to diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index 44e2d6f1dbaa875c8e1e1e08217102b75d358ff4..5059799bebe36d4b3a2c34d5aeda8b35a92ca96d 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -815,7 +815,8 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) static inline int pmd_bad(pmd_t pmd) { - return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; + return (pmd_flags(pmd) & ~(_PAGE_USER | _PAGE_ACCESSED)) != + (_KERNPG_TABLE & ~_PAGE_ACCESSED); } static inline unsigned long pages_to_mb(unsigned long npg) @@ -1431,10 +1432,10 @@ static inline bool arch_has_pfn_modify_check(void) return boot_cpu_has_bug(X86_BUG_L1TF); } -#define arch_faults_on_old_pte arch_faults_on_old_pte -static inline bool arch_faults_on_old_pte(void) +#define arch_has_hw_pte_young arch_has_hw_pte_young +static inline bool arch_has_hw_pte_young(void) { - return false; + return true; } #ifdef CONFIG_PAGE_TABLE_CHECK diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h index 70e360a2e5fb7974de5bb139b8a9a9fcb6718fa6..04f36063ad5468a9917247329046d34464b254d1 100644 --- a/arch/x86/include/asm/pgtable_64_types.h +++ b/arch/x86/include/asm/pgtable_64_types.h @@ -139,7 +139,52 @@ extern unsigned int ptrs_per_p4d; # define VMEMMAP_START __VMEMMAP_BASE_L4 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */ -#define VMALLOC_END (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1) +/* + * End of the region for which vmalloc page tables are pre-allocated. + * For non-KMSAN builds, this is the same as VMALLOC_END. + * For KMSAN builds, VMALLOC_START..VMEMORY_END is 4 times bigger than + * VMALLOC_START..VMALLOC_END (see below). + */ +#define VMEMORY_END (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1) + +#ifndef CONFIG_KMSAN +#define VMALLOC_END VMEMORY_END +#else +/* + * In KMSAN builds vmalloc area is four times smaller, and the remaining 3/4 + * are used to keep the metadata for virtual pages. The memory formerly + * belonging to vmalloc area is now laid out as follows: + * + * 1st quarter: VMALLOC_START to VMALLOC_END - new vmalloc area + * 2nd quarter: KMSAN_VMALLOC_SHADOW_START to + * VMALLOC_END+KMSAN_VMALLOC_SHADOW_OFFSET - vmalloc area shadow + * 3rd quarter: KMSAN_VMALLOC_ORIGIN_START to + * VMALLOC_END+KMSAN_VMALLOC_ORIGIN_OFFSET - vmalloc area origins + * 4th quarter: KMSAN_MODULES_SHADOW_START to KMSAN_MODULES_ORIGIN_START + * - shadow for modules, + * KMSAN_MODULES_ORIGIN_START to + * KMSAN_MODULES_ORIGIN_START + MODULES_LEN - origins for modules. + */ +#define VMALLOC_QUARTER_SIZE ((VMALLOC_SIZE_TB << 40) >> 2) +#define VMALLOC_END (VMALLOC_START + VMALLOC_QUARTER_SIZE - 1) + +/* + * vmalloc metadata addresses are calculated by adding shadow/origin offsets + * to vmalloc address. + */ +#define KMSAN_VMALLOC_SHADOW_OFFSET VMALLOC_QUARTER_SIZE +#define KMSAN_VMALLOC_ORIGIN_OFFSET (VMALLOC_QUARTER_SIZE << 1) + +#define KMSAN_VMALLOC_SHADOW_START (VMALLOC_START + KMSAN_VMALLOC_SHADOW_OFFSET) +#define KMSAN_VMALLOC_ORIGIN_START (VMALLOC_START + KMSAN_VMALLOC_ORIGIN_OFFSET) + +/* + * The shadow/origin for modules are placed one by one in the last 1/4 of + * vmalloc space. + */ +#define KMSAN_MODULES_SHADOW_START (VMALLOC_END + KMSAN_VMALLOC_ORIGIN_OFFSET + 1) +#define KMSAN_MODULES_ORIGIN_START (KMSAN_MODULES_SHADOW_START + MODULES_LEN) +#endif /* CONFIG_KMSAN */ #define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE) /* The module sections ends with the start of the fixmap */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 356308c73951400d44e5b285de5ce0181d988ed0..67c9d73b31faa1590c0aa4abd72da7d5b20ff0ed 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -587,9 +587,6 @@ static inline void load_sp0(unsigned long sp0) #endif /* CONFIG_PARAVIRT_XXL */ -/* Free all resources held by a thread. */ -extern void release_thread(struct task_struct *); - unsigned long __get_wchan(struct task_struct *p); /* diff --git a/arch/x86/include/asm/qspinlock_paravirt.h b/arch/x86/include/asm/qspinlock_paravirt.h index 892fd8c3a6f72eb0ceba7b3d5caaabbd74c8d015..60ece592b22077328a89b5979f7f340bcca46e8c 100644 --- a/arch/x86/include/asm/qspinlock_paravirt.h +++ b/arch/x86/include/asm/qspinlock_paravirt.h @@ -12,7 +12,7 @@ */ #ifdef CONFIG_64BIT -PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); +__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text"); #define __pv_queued_spin_unlock __pv_queued_spin_unlock #define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock" #define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath" @@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); /* * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock * which combines the registers saving trunk and the body of the following - * C code: + * C code. Note that it puts the code in the .spinlock.text section which + * is equivalent to adding __lockfunc in the C code: * - * void __pv_queued_spin_unlock(struct qspinlock *lock) + * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock) * { * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); * @@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); * rsi = lockval (second argument) * rdx = internal variable (set to 0) */ -asm (".pushsection .text;" +asm (".pushsection .spinlock.text;" ".globl " PV_UNLOCK ";" ".type " PV_UNLOCK ", @function;" ".align 4,0x90;" @@ -65,8 +66,8 @@ asm (".pushsection .text;" #else /* CONFIG_64BIT */ -extern void __pv_queued_spin_unlock(struct qspinlock *lock); -PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock); +extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock); +__PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text"); #endif /* CONFIG_64BIT */ #endif diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h index d60ed0668a59335833ee6fded25f1b60fae3d232..d24b04ebf950ed1a78952317e8af262e9681c67d 100644 --- a/arch/x86/include/asm/resctrl.h +++ b/arch/x86/include/asm/resctrl.h @@ -81,6 +81,15 @@ static void __resctrl_sched_in(void) } } +static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) +{ + unsigned int scale = boot_cpu_data.x86_cache_occ_scale; + + /* h/w works in units of "boot_cpu_data.x86_cache_occ_scale" */ + val /= scale; + return val * scale; +} + static inline void resctrl_sched_in(void) { if (static_branch_likely(&rdt_enable_key)) diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h index 4a23e52fe0ee1632eb64fb1ed6ab183c5b7bb81f..ebc271bb6d8ed1d74e07194604b25244646ffc0a 100644 --- a/arch/x86/include/asm/sev.h +++ b/arch/x86/include/asm/sev.h @@ -195,7 +195,7 @@ void snp_set_memory_shared(unsigned long vaddr, unsigned int npages); void snp_set_memory_private(unsigned long vaddr, unsigned int npages); void snp_set_wakeup_secondary_cpu(void); bool snp_init(struct boot_params *bp); -void snp_abort(void); +void __init __noreturn snp_abort(void); int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, unsigned long *fw_err); #else static inline void sev_es_ist_enter(struct pt_regs *regs) { } diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 81a0211a372d3d268671ecfa1d554816481006bc..a73bced40e2419d0cec28c1fe8327db24342f1d1 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -21,16 +21,6 @@ DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id); DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id); DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); -static inline struct cpumask *cpu_llc_shared_mask(int cpu) -{ - return per_cpu(cpu_llc_shared_map, cpu); -} - -static inline struct cpumask *cpu_l2c_shared_mask(int cpu) -{ - return per_cpu(cpu_l2c_shared_map, cpu); -} - DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid); DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); @@ -172,6 +162,16 @@ extern int safe_smp_processor_id(void); # define safe_smp_processor_id() smp_processor_id() #endif +static inline struct cpumask *cpu_llc_shared_mask(int cpu) +{ + return per_cpu(cpu_llc_shared_map, cpu); +} + +static inline struct cpumask *cpu_l2c_shared_mask(int cpu) +{ + return per_cpu(cpu_l2c_shared_map, cpu); +} + #else /* !CONFIG_SMP */ #define wbinvd_on_cpu(cpu) wbinvd() static inline int wbinvd_on_all_cpus(void) @@ -179,6 +179,11 @@ static inline int wbinvd_on_all_cpus(void) wbinvd(); return 0; } + +static inline struct cpumask *cpu_llc_shared_mask(int cpu) +{ + return (struct cpumask *)cpumask_of(0); +} #endif /* CONFIG_SMP */ extern unsigned disabled_cpus; diff --git a/arch/x86/include/asm/sparsemem.h b/arch/x86/include/asm/sparsemem.h index 6a9ccc1b2be5d1b8ade62585ef50753ea8b8d679..64df897c0ee30f9332bd332ba9e484877bebe551 100644 --- a/arch/x86/include/asm/sparsemem.h +++ b/arch/x86/include/asm/sparsemem.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_SPARSEMEM_H #define _ASM_X86_SPARSEMEM_H +#include + #ifdef CONFIG_SPARSEMEM /* * generic non-linear memory support: diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 6e450827f677a9ec81401fb1949981f50142415d..3b87d889b6e16a111662b40168fb9101981ce8ea 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -11,11 +11,23 @@ function. */ #define __HAVE_ARCH_MEMCPY 1 +#if defined(__SANITIZE_MEMORY__) +#undef memcpy +void *__msan_memcpy(void *dst, const void *src, size_t size); +#define memcpy __msan_memcpy +#else extern void *memcpy(void *to, const void *from, size_t len); +#endif extern void *__memcpy(void *to, const void *from, size_t len); #define __HAVE_ARCH_MEMSET +#if defined(__SANITIZE_MEMORY__) +extern void *__msan_memset(void *s, int c, size_t n); +#undef memset +#define memset __msan_memset +#else void *memset(void *s, int c, size_t n); +#endif void *__memset(void *s, int c, size_t n); #define __HAVE_ARCH_MEMSET16 @@ -55,7 +67,13 @@ static inline void *memset64(uint64_t *s, uint64_t v, size_t n) } #define __HAVE_ARCH_MEMMOVE +#if defined(__SANITIZE_MEMORY__) +#undef memmove +void *__msan_memmove(void *dest, const void *src, size_t len); +#define memmove __msan_memmove +#else void *memmove(void *dest, const void *src, size_t count); +#endif void *__memmove(void *dest, const void *src, size_t count); int memcmp(const void *cs, const void *ct, size_t count); @@ -64,8 +82,7 @@ char *strcpy(char *dest, const char *src); char *strcat(char *dest, const char *src); int strcmp(const char *cs, const char *ct); -#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) - +#if (defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)) /* * For files that not instrumented (e.g. mm/slub.c) we * should use not instrumented version of mem* functions. @@ -73,7 +90,9 @@ int strcmp(const char *cs, const char *ct); #undef memcpy #define memcpy(dst, src, len) __memcpy(dst, src, len) +#undef memmove #define memmove(dst, src, len) __memmove(dst, src, len) +#undef memset #define memset(s, c, n) __memset(s, c, n) #ifndef __NO_FORTIFY diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 913e593a3b45fb5de1e16170a23320bb464a0ee6..8bc614cfe21b99cd089827b14a4672397ebecbe9 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -5,6 +5,7 @@ * User space memory access functions */ #include +#include #include #include #include @@ -103,6 +104,7 @@ extern int __get_user_bad(void); : "=a" (__ret_gu), "=r" (__val_gu), \ ASM_CALL_CONSTRAINT \ : "0" (ptr), "i" (sizeof(*(ptr)))); \ + instrument_get_user(__val_gu); \ (x) = (__force __typeof__(*(ptr))) __val_gu; \ __builtin_expect(__ret_gu, 0); \ }) @@ -192,9 +194,11 @@ extern void __put_user_nocheck_8(void); int __ret_pu; \ void __user *__ptr_pu; \ register __typeof__(*(ptr)) __val_pu asm("%"_ASM_AX); \ - __chk_user_ptr(ptr); \ - __ptr_pu = (ptr); \ - __val_pu = (x); \ + __typeof__(*(ptr)) __x = (x); /* eval x once */ \ + __typeof__(ptr) __ptr = (ptr); /* eval ptr once */ \ + __chk_user_ptr(__ptr); \ + __ptr_pu = __ptr; \ + __val_pu = __x; \ asm volatile("call __" #fn "_%P[size]" \ : "=c" (__ret_pu), \ ASM_CALL_CONSTRAINT \ @@ -202,6 +206,7 @@ extern void __put_user_nocheck_8(void); "r" (__val_pu), \ [size] "i" (sizeof(*(ptr))) \ :"ebx"); \ + instrument_put_user(__x, __ptr, sizeof(*(ptr))); \ __builtin_expect(__ret_pu, 0); \ }) @@ -248,23 +253,25 @@ extern void __put_user_nocheck_8(void); #define __put_user_size(x, ptr, size, label) \ do { \ + __typeof__(*(ptr)) __x = (x); /* eval x once */ \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: \ - __put_user_goto(x, ptr, "b", "iq", label); \ + __put_user_goto(__x, ptr, "b", "iq", label); \ break; \ case 2: \ - __put_user_goto(x, ptr, "w", "ir", label); \ + __put_user_goto(__x, ptr, "w", "ir", label); \ break; \ case 4: \ - __put_user_goto(x, ptr, "l", "ir", label); \ + __put_user_goto(__x, ptr, "l", "ir", label); \ break; \ case 8: \ - __put_user_goto_u64(x, ptr, label); \ + __put_user_goto_u64(__x, ptr, label); \ break; \ default: \ __put_user_bad(); \ } \ + instrument_put_user(__x, ptr, size); \ } while (0) #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT @@ -305,6 +312,7 @@ do { \ default: \ (x) = __get_user_bad(); \ } \ + instrument_get_user(x); \ } while (0) #define __get_user_asm(x, addr, itype, ltype, label) \ @@ -448,7 +456,7 @@ do { \ #ifdef CONFIG_X86_32 /* - * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error. + * Unlike the normal CMPXCHG, use output GPR for both success/fail and error. * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses * both ESI and EDI for the memory operand, compilation will fail if the error @@ -461,11 +469,12 @@ do { \ __typeof__(*(_ptr)) __new = (_new); \ asm volatile("\n" \ "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ - "mov $0, %%ecx\n\t" \ - "setz %%cl\n" \ + "mov $0, %[result]\n\t" \ + "setz %b[result]\n" \ "2:\n" \ - _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \ - : [result]"=c" (__result), \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \ + %[result]) \ + : [result] "=q" (__result), \ "+A" (__old), \ [ptr] "+m" (*_ptr) \ : "b" ((u32)__new), \ @@ -502,9 +511,6 @@ strncpy_from_user(char *dst, const char __user *src, long count); extern __must_check long strnlen_user(const char __user *str, long n); -unsigned long __must_check clear_user(void __user *mem, unsigned long len); -unsigned long __must_check __clear_user(void __user *mem, unsigned long len); - #ifdef CONFIG_ARCH_HAS_COPY_MC unsigned long __must_check copy_mc_to_kernel(void *to, const void *from, unsigned len); @@ -526,6 +532,8 @@ extern struct movsl_mask { #define ARCH_HAS_NOCACHE_UACCESS 1 #ifdef CONFIG_X86_32 +unsigned long __must_check clear_user(void __user *mem, unsigned long len); +unsigned long __must_check __clear_user(void __user *mem, unsigned long len); # include #else # include diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 45697e04d77106971f20f3f825bce819e1f2f771..d13d71af5cf61045147d84ea66703ec44ee3da82 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -79,4 +79,49 @@ __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) kasan_check_write(dst, size); return __copy_user_flushcache(dst, src, size); } + +/* + * Zero Userspace. + */ + +__must_check unsigned long +clear_user_original(void __user *addr, unsigned long len); +__must_check unsigned long +clear_user_rep_good(void __user *addr, unsigned long len); +__must_check unsigned long +clear_user_erms(void __user *addr, unsigned long len); + +static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size) +{ + might_fault(); + stac(); + + /* + * No memory constraint because it doesn't change any memory gcc + * knows about. + */ + asm volatile( + "1:\n\t" + ALTERNATIVE_3("rep stosb", + "call clear_user_erms", ALT_NOT(X86_FEATURE_FSRM), + "call clear_user_rep_good", ALT_NOT(X86_FEATURE_ERMS), + "call clear_user_original", ALT_NOT(X86_FEATURE_REP_GOOD)) + "2:\n" + _ASM_EXTABLE_UA(1b, 2b) + : "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT + : "a" (0) + /* rep_good clobbers %rdx */ + : "rdx"); + + clac(); + + return size; +} + +static __always_inline unsigned long clear_user(void __user *to, unsigned long n) +{ + if (access_ok(to, n)) + return __clear_user(to, n); + return n; +} #endif /* _ASM_X86_UACCESS_64_H */ diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index c371ef695fcc09fe82688da0549fd79a61b49e4b..498dc600bd5c8ef55a4de29ecb87396d4d192859 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -309,7 +309,7 @@ enum vmcs_field { GUEST_LDTR_AR_BYTES = 0x00004820, GUEST_TR_AR_BYTES = 0x00004822, GUEST_INTERRUPTIBILITY_INFO = 0x00004824, - GUEST_ACTIVITY_STATE = 0X00004826, + GUEST_ACTIVITY_STATE = 0x00004826, GUEST_SYSENTER_CS = 0x0000482A, VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, HOST_IA32_SYSENTER_CS = 0x00004c00, diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index a20a5ebfacd73f772249b986c55f3e5f5e15a044..f901658d9f7c0833912e1be0436c11a0d9b35761 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -3,10 +3,6 @@ # Makefile for the linux kernel. # -extra-y := head_$(BITS).o -extra-y += head$(BITS).o -extra-y += ebda.o -extra-y += platform-quirks.o extra-y += vmlinux.lds CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE) @@ -33,6 +29,8 @@ KASAN_SANITIZE_sev.o := n # With some compiler versions the generated code results in boot hangs, caused # by several compilation units. To be safe, disable all instrumentation. KCSAN_SANITIZE := n +KMSAN_SANITIZE_head$(BITS).o := n +KMSAN_SANITIZE_nmi.o := n # If instrumentation of this dir is enabled, boot hangs during first second. # Probably could be more selective here, but note that files related to irqs, @@ -42,7 +40,11 @@ KCOV_INSTRUMENT := n CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace -obj-y := process_$(BITS).o signal.o +obj-y += head_$(BITS).o +obj-y += head$(BITS).o +obj-y += ebda.o +obj-y += platform-quirks.o +obj-y += process_$(BITS).o signal.o obj-$(CONFIG_COMPAT) += signal_compat.o obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o obj-y += time.o ioport.o dumpstack.o nmi.o @@ -139,6 +141,8 @@ obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o obj-$(CONFIG_AMD_MEM_ENCRYPT) += sev.o +obj-$(CONFIG_CFI_CLANG) += cfi.o + ### # 64 bit specific files ifeq ($(CONFIG_X86_64),y) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 62f6b8b7c4a5285933b80013ff6fe2aa7bf6e183..5cadcea035e044b47972675f0bd9792b8fc46fc0 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes) return ret; i += ret; + /* + * The compiler is supposed to EMIT an INT3 after every unconditional + * JMP instruction due to AMD BTC. However, if the compiler is too old + * or SLS isn't enabled, we still need an INT3 after indirect JMPs + * even on Intel. + */ + if (op == JMP32_INSN_OPCODE && i < insn->length) + bytes[i++] = INT3_INSN_OPCODE; + for (; i < insn->length;) bytes[i++] = BYTES_NOP1; @@ -1319,22 +1328,23 @@ struct bp_patching_desc { atomic_t refs; }; -static struct bp_patching_desc *bp_desc; +static struct bp_patching_desc bp_desc; static __always_inline -struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) +struct bp_patching_desc *try_get_desc(void) { - /* rcu_dereference */ - struct bp_patching_desc *desc = __READ_ONCE(*descp); + struct bp_patching_desc *desc = &bp_desc; - if (!desc || !arch_atomic_inc_not_zero(&desc->refs)) + if (!arch_atomic_inc_not_zero(&desc->refs)) return NULL; return desc; } -static __always_inline void put_desc(struct bp_patching_desc *desc) +static __always_inline void put_desc(void) { + struct bp_patching_desc *desc = &bp_desc; + smp_mb__before_atomic(); arch_atomic_dec(&desc->refs); } @@ -1367,15 +1377,15 @@ noinstr int poke_int3_handler(struct pt_regs *regs) /* * Having observed our INT3 instruction, we now must observe - * bp_desc: + * bp_desc with non-zero refcount: * - * bp_desc = desc INT3 + * bp_desc.refs = 1 INT3 * WMB RMB - * write INT3 if (desc) + * write INT3 if (bp_desc.refs != 0) */ smp_rmb(); - desc = try_get_desc(&bp_desc); + desc = try_get_desc(); if (!desc) return 0; @@ -1429,7 +1439,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs) ret = 1; out_put: - put_desc(desc); + put_desc(); return ret; } @@ -1460,18 +1470,20 @@ static int tp_vec_nr; */ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries) { - struct bp_patching_desc desc = { - .vec = tp, - .nr_entries = nr_entries, - .refs = ATOMIC_INIT(1), - }; unsigned char int3 = INT3_INSN_OPCODE; unsigned int i; int do_sync; lockdep_assert_held(&text_mutex); - smp_store_release(&bp_desc, &desc); /* rcu_assign_pointer */ + bp_desc.vec = tp; + bp_desc.nr_entries = nr_entries; + + /* + * Corresponds to the implicit memory barrier in try_get_desc() to + * ensure reading a non-zero refcount provides up to date bp_desc data. + */ + atomic_set_release(&bp_desc.refs, 1); /* * Corresponding read barrier in int3 notifier for making sure the @@ -1559,12 +1571,10 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries text_poke_sync(); /* - * Remove and synchronize_rcu(), except we have a very primitive - * refcount based completion. + * Remove and wait for refs to be zero. */ - WRITE_ONCE(bp_desc, NULL); /* RCU_INIT_POINTER */ - if (!atomic_dec_and_test(&desc.refs)) - atomic_cond_read_acquire(&desc.refs, !VAL); + if (!atomic_dec_and_test(&bp_desc.refs)) + atomic_cond_read_acquire(&bp_desc.refs, !VAL); } static void text_poke_loc_init(struct text_poke_loc *tp, void *addr, diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index 194d54eed5376f188da5691f825caa0304df3892..19a0207e529fe33f6d340ae36a6f5fcbf60ea475 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -53,7 +53,7 @@ static u32 *iommu_gatt_base; /* Remapping table */ * of only flushing when an mapping is reused. With it true the GART is * flushed for every mapping. Problem is that doing the lazy flush seems * to trigger bugs with some popular PCI cards, in particular 3ware (but - * has been also also seen with Qlogic at least). + * has been also seen with Qlogic at least). */ static int iommu_fullflush = 1; diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c index 7a5630d904b23dc511163f0ff3a3f5becb12f885..4feaa670d5783b8fe9bb226d0635c09bb15939da 100644 --- a/arch/x86/kernel/aperture_64.c +++ b/arch/x86/kernel/aperture_64.c @@ -36,7 +36,7 @@ /* * Using 512M as goal, in case kexec will load kernel_big * that will do the on-position decompress, and could overlap with - * with the gart aperture that is used. + * the gart aperture that is used. * Sequence: * kernel_small * ==> kexec (with kdump trigger path or gart still enabled) diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 6d303d1d276c3751448f4d517f1e4b9316d189d8..c6876d3ea4b17893462828f9dfb6e94c1747a83a 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -61,6 +61,7 @@ #include #include #include +#include unsigned int num_processors; @@ -1751,11 +1752,26 @@ EXPORT_SYMBOL_GPL(x2apic_mode); enum { X2APIC_OFF, - X2APIC_ON, X2APIC_DISABLED, + /* All states below here have X2APIC enabled */ + X2APIC_ON, + X2APIC_ON_LOCKED }; static int x2apic_state; +static bool x2apic_hw_locked(void) +{ + u64 ia32_cap; + u64 msr; + + ia32_cap = x86_read_arch_cap_msr(); + if (ia32_cap & ARCH_CAP_XAPIC_DISABLE) { + rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr); + return (msr & LEGACY_XAPIC_DISABLED); + } + return false; +} + static void __x2apic_disable(void) { u64 msr; @@ -1793,6 +1809,10 @@ static int __init setup_nox2apic(char *str) apicid); return 0; } + if (x2apic_hw_locked()) { + pr_warn("APIC locked in x2apic mode, can't disable\n"); + return 0; + } pr_warn("x2apic already enabled.\n"); __x2apic_disable(); } @@ -1807,10 +1827,18 @@ early_param("nox2apic", setup_nox2apic); void x2apic_setup(void) { /* - * If x2apic is not in ON state, disable it if already enabled + * Try to make the AP's APIC state match that of the BSP, but if the + * BSP is unlocked and the AP is locked then there is a state mismatch. + * Warn about the mismatch in case a GP fault occurs due to a locked AP + * trying to be turned off. + */ + if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked()) + pr_warn("x2apic lock mismatch between BSP and AP.\n"); + /* + * If x2apic is not in ON or LOCKED state, disable it if already enabled * from BIOS. */ - if (x2apic_state != X2APIC_ON) { + if (x2apic_state < X2APIC_ON) { __x2apic_disable(); return; } @@ -1831,6 +1859,11 @@ static __init void x2apic_disable(void) if (x2apic_id >= 255) panic("Cannot disable x2apic, id: %08x\n", x2apic_id); + if (x2apic_hw_locked()) { + pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id); + return; + } + __x2apic_disable(); register_lapic_address(mp_lapic_addr); } @@ -1889,7 +1922,10 @@ void __init check_x2apic(void) if (x2apic_enabled()) { pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n"); x2apic_mode = 1; - x2apic_state = X2APIC_ON; + if (x2apic_hw_locked()) + x2apic_state = X2APIC_ON_LOCKED; + else + x2apic_state = X2APIC_ON; } else if (!boot_cpu_has(X86_FEATURE_X2APIC)) { x2apic_state = X2APIC_DISABLED; } diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c new file mode 100644 index 0000000000000000000000000000000000000000..8674a5c0c031d3dd3c0a2708ff39369efb52ff7e --- /dev/null +++ b/arch/x86/kernel/cfi.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Clang Control Flow Integrity (CFI) support. + * + * Copyright (C) 2022 Google LLC + */ +#include +#include +#include +#include + +/* + * Returns the target address and the expected type when regs->ip points + * to a compiler-generated CFI trap. + */ +static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target, + u32 *type) +{ + char buffer[MAX_INSN_SIZE]; + struct insn insn; + int offset = 0; + + *target = *type = 0; + + /* + * The compiler generates the following instruction sequence + * for indirect call checks: + * + *   movl -, %r10d ; 6 bytes + * addl -4(%reg), %r10d ; 4 bytes + * je .Ltmp1 ; 2 bytes + * ud2 ; <- regs->ip + * .Ltmp1: + * + * We can decode the expected type and the target address from the + * movl/addl instructions. + */ + if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 12, MAX_INSN_SIZE)) + return false; + if (insn_decode_kernel(&insn, &buffer[offset])) + return false; + if (insn.opcode.value != 0xBA) + return false; + + *type = -(u32)insn.immediate.value; + + if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 6, MAX_INSN_SIZE)) + return false; + if (insn_decode_kernel(&insn, &buffer[offset])) + return false; + if (insn.opcode.value != 0x3) + return false; + + /* Read the target address from the register. */ + offset = insn_get_modrm_rm_off(&insn, regs); + if (offset < 0) + return false; + + *target = *(unsigned long *)((void *)regs + offset); + + return true; +} + +/* + * Checks if a ud2 trap is because of a CFI failure, and handles the trap + * if needed. Returns a bug_trap_type value similarly to report_bug. + */ +enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) +{ + unsigned long target; + u32 type; + + if (!is_cfi_trap(regs->ip)) + return BUG_TRAP_TYPE_NONE; + + if (!decode_cfi_insn(regs, &target, &type)) + return report_cfi_failure_noaddr(regs, regs->ip); + + return report_cfi_failure(regs, regs->ip, &target, type); +} + +/* + * Ensure that __kcfi_typeid_ symbols are emitted for functions that may + * not be indirectly called with all configurations. + */ +__ADDRESSABLE(__memcpy) diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 9661e3e802be56e937f137e467b7d4a1669d4410..f10a921ee75658980b92d0d81238d9c16fb642ce 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -12,6 +12,7 @@ endif # If these files are instrumented, boot hangs during the first second. KCOV_INSTRUMENT_common.o := n KCOV_INSTRUMENT_perf_event.o := n +KMSAN_SANITIZE_common.o := n # As above, instrumenting secondary CPU boot code causes boot hangs. KCSAN_SANITIZE_common.o := n diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 23f5f27b5a024865054b19b43c766c3f6afa8bfa..485441b7f03036fba446e0e0fba99f48d1addd2b 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -28,6 +28,9 @@ static void __init acrn_init_platform(void) { /* Setup the IDT for ACRN hypervisor callback */ alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_acrn_hv_callback); + + x86_platform.calibrate_tsc = acrn_get_tsc_khz; + x86_platform.calibrate_cpu = acrn_get_tsc_khz; } static bool acrn_x2apic_available(void) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 48276c0e479d824bb014f92fbe413086b372c13a..860b60273df3ff38eb406449a768166c8253153b 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -503,7 +503,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) va_align.flags = ALIGN_VA_32 | ALIGN_VA_64; /* A random value per boot for bit slice [12:upper_bit) */ - va_align.bits = get_random_int() & va_align.mask; + va_align.bits = get_random_u32() & va_align.mask; } if (cpu_has(c, X86_FEATURE_MWAITX)) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 510d85261132b06dc4825ec8e562ac89273043bb..da7c361f47e0d9ab39ec4b0f63eb326baa44a842 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -433,7 +433,8 @@ static void __init mmio_select_mitigation(void) u64 ia32_cap; if (!boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA) || - cpu_mitigations_off()) { + boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN) || + cpu_mitigations_off()) { mmio_mitigation = MMIO_MITIGATION_OFF; return; } @@ -538,6 +539,8 @@ out: pr_info("TAA: %s\n", taa_strings[taa_mitigation]); if (boot_cpu_has_bug(X86_BUG_MMIO_STALE_DATA)) pr_info("MMIO Stale Data: %s\n", mmio_strings[mmio_mitigation]); + else if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) + pr_info("MMIO Stale Data: Unknown: No mitigations\n"); } static void __init md_clear_select_mitigation(void) @@ -2275,6 +2278,9 @@ static ssize_t tsx_async_abort_show_state(char *buf) static ssize_t mmio_stale_data_show_state(char *buf) { + if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) + return sysfs_emit(buf, "Unknown: No mitigations\n"); + if (mmio_mitigation == MMIO_MITIGATION_OFF) return sysfs_emit(buf, "%s\n", mmio_strings[mmio_mitigation]); @@ -2421,6 +2427,7 @@ static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr return srbds_show_state(buf); case X86_BUG_MMIO_STALE_DATA: + case X86_BUG_MMIO_UNKNOWN: return mmio_stale_data_show_state(buf); case X86_BUG_RETBLEED: @@ -2480,7 +2487,10 @@ ssize_t cpu_show_srbds(struct device *dev, struct device_attribute *attr, char * ssize_t cpu_show_mmio_stale_data(struct device *dev, struct device_attribute *attr, char *buf) { - return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); + if (boot_cpu_has_bug(X86_BUG_MMIO_UNKNOWN)) + return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_UNKNOWN); + else + return cpu_show_common(dev, attr, buf, X86_BUG_MMIO_STALE_DATA); } ssize_t cpu_show_retbleed(struct device *dev, struct device_attribute *attr, char *buf) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 64a73f415f036432e9e6cc66e05259e3fcf68d38..3e508f23909830fd78fbd6b168ad04aaeb57ef18 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1135,7 +1135,8 @@ static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c) #define NO_SWAPGS BIT(6) #define NO_ITLB_MULTIHIT BIT(7) #define NO_SPECTRE_V2 BIT(8) -#define NO_EIBRS_PBRSB BIT(9) +#define NO_MMIO BIT(9) +#define NO_EIBRS_PBRSB BIT(10) #define VULNWL(vendor, family, model, whitelist) \ X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist) @@ -1158,6 +1159,11 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { VULNWL(VORTEX, 6, X86_MODEL_ANY, NO_SPECULATION), /* Intel Family 6 */ + VULNWL_INTEL(TIGERLAKE, NO_MMIO), + VULNWL_INTEL(TIGERLAKE_L, NO_MMIO), + VULNWL_INTEL(ALDERLAKE, NO_MMIO), + VULNWL_INTEL(ALDERLAKE_L, NO_MMIO), + VULNWL_INTEL(ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT), VULNWL_INTEL(ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT), VULNWL_INTEL(ATOM_SALTWELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT), @@ -1176,9 +1182,9 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { VULNWL_INTEL(ATOM_AIRMONT_MID, NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT), VULNWL_INTEL(ATOM_AIRMONT_NP, NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_INTEL(ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_INTEL(ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_INTEL(ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB), + VULNWL_INTEL(ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_INTEL(ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_INTEL(ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO | NO_EIBRS_PBRSB), /* * Technically, swapgs isn't serializing on AMD (despite it previously @@ -1193,18 +1199,18 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { VULNWL_INTEL(ATOM_TREMONT_D, NO_ITLB_MULTIHIT | NO_EIBRS_PBRSB), /* AMD Family 0xf - 0x12 */ - VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), + VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), /* FAMILY_ANY must be last, otherwise 0x0f - 0x12 matches won't work */ - VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), - VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT), + VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), + VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT | NO_MMIO), /* Zhaoxin Family 7 */ - VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS), - VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS), + VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO), + VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS | NO_MMIO), {} }; @@ -1358,10 +1364,16 @@ static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c) * Affected CPU list is generally enough to enumerate the vulnerability, * but for virtualization case check for ARCH_CAP MSR bits also, VMM may * not want the guest to enumerate the bug. + * + * Set X86_BUG_MMIO_UNKNOWN for CPUs that are neither in the blacklist, + * nor in the whitelist and also don't enumerate MSR ARCH_CAP MMIO bits. */ - if (cpu_matches(cpu_vuln_blacklist, MMIO) && - !arch_cap_mmio_immune(ia32_cap)) - setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA); + if (!arch_cap_mmio_immune(ia32_cap)) { + if (cpu_matches(cpu_vuln_blacklist, MMIO)) + setup_force_cpu_bug(X86_BUG_MMIO_STALE_DATA); + else if (!cpu_matches(cpu_vuln_whitelist, NO_MMIO)) + setup_force_cpu_bug(X86_BUG_MMIO_UNKNOWN); + } if (!cpu_has(c, X86_FEATURE_BTC_NO)) { if (cpu_matches(cpu_vuln_blacklist, RETBLEED) || (ia32_cap & ARCH_CAP_RSBA)) diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 993697e71854c0de3c8a02c054980df3eb57b390..03851240c3e36d4ed5e9ad250eee76410830d6e9 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -1,11 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include #include -#include "cpu.h" #undef pr_fmt #define pr_fmt(fmt) "x86/cpu: " fmt diff --git a/arch/x86/kernel/cpu/mce/apei.c b/arch/x86/kernel/cpu/mce/apei.c index 717192915f28a010cbe94504dcde628c279112ac..8ed341714686a842be633b9356cbf88e87b933bc 100644 --- a/arch/x86/kernel/cpu/mce/apei.c +++ b/arch/x86/kernel/cpu/mce/apei.c @@ -29,15 +29,26 @@ void apei_mce_report_mem_error(int severity, struct cper_sec_mem_err *mem_err) { struct mce m; + int lsb; if (!(mem_err->validation_bits & CPER_MEM_VALID_PA)) return; + /* + * Even if the ->validation_bits are set for address mask, + * to be extra safe, check and reject an error radius '0', + * and fall back to the default page size. + */ + if (mem_err->validation_bits & CPER_MEM_VALID_PA_MASK) + lsb = find_first_bit((void *)&mem_err->physical_addr_mask, PAGE_SHIFT); + else + lsb = PAGE_SHIFT; + mce_setup(&m); m.bank = -1; /* Fake a memory read error with unknown channel */ m.status = MCI_STATUS_VAL | MCI_STATUS_EN | MCI_STATUS_ADDRV | MCI_STATUS_MISCV | 0x9f; - m.misc = (MCI_MISC_ADDR_PHYS << 6) | PAGE_SHIFT; + m.misc = (MCI_MISC_ADDR_PHYS << 6) | lsb; if (severity >= GHES_SEV_RECOVERABLE) m.status |= MCI_STATUS_UC; diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 8b2fcdfa6d316816d0db2b1f46cd1f9cc8120ff4..e7410e98fc1f9a55a57ab3f485beb584789df9b7 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -788,6 +788,7 @@ static int verify_and_add_patch(u8 family, u8 *fw, unsigned int leftover, kfree(patch); return -EINVAL; } + patch->size = *patch_size; mc_hdr = (struct microcode_header_amd *)(fw + SECTION_HDR_SIZE); proc_id = mc_hdr->processor_rev_id; @@ -869,7 +870,7 @@ load_microcode_amd(bool save, u8 family, const u8 *data, size_t size) return ret; memset(amd_ucode_patch, 0, PATCH_MAX_SIZE); - memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data), PATCH_MAX_SIZE)); + memcpy(amd_ucode_patch, p->data, min_t(u32, p->size, PATCH_MAX_SIZE)); return ret; } @@ -924,12 +925,6 @@ static enum ucode_state request_microcode_amd(int cpu, struct device *device, return ret; } -static enum ucode_state -request_microcode_user(int cpu, const void __user *buf, size_t size) -{ - return UCODE_ERROR; -} - static void microcode_fini_cpu_amd(int cpu) { struct ucode_cpu_info *uci = ucode_cpu_info + cpu; @@ -938,7 +933,6 @@ static void microcode_fini_cpu_amd(int cpu) } static struct microcode_ops microcode_amd_ops = { - .request_microcode_user = request_microcode_user, .request_microcode_fw = request_microcode_amd, .collect_cpu_info = collect_cpu_info_amd, .apply_microcode = apply_microcode_amd, diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index ad57e0e4d6742399bbf9ce89fd905821728201dd..6a41cee242f6d710fb8db5c0c7e26b7061dcc8c0 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -491,7 +491,7 @@ wait_for_siblings: */ static int microcode_reload_late(void) { - int ret; + int old = boot_cpu_data.microcode, ret; pr_err("Attempting late microcode loading - it is dangerous and taints the kernel.\n"); pr_err("You should switch to early loading, if possible.\n"); @@ -503,7 +503,8 @@ static int microcode_reload_late(void) if (ret == 0) microcode_check(); - pr_info("Reload completed, microcode revision: 0x%x\n", boot_cpu_data.microcode); + pr_info("Reload completed, microcode revision: 0x%x -> 0x%x\n", + old, boot_cpu_data.microcode); return ret; } diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c index 025c8f0cd948c5483445e256a7944af033784717..1fcbd671f1dffc515672ccd2c57d5fdbf5d42436 100644 --- a/arch/x86/kernel/cpu/microcode/intel.c +++ b/arch/x86/kernel/cpu/microcode/intel.c @@ -916,24 +916,7 @@ static enum ucode_state request_microcode_fw(int cpu, struct device *device, return ret; } -static enum ucode_state -request_microcode_user(int cpu, const void __user *buf, size_t size) -{ - struct iov_iter iter; - struct iovec iov; - - if (is_blacklisted(cpu)) - return UCODE_NFOUND; - - iov.iov_base = (void __user *)buf; - iov.iov_len = size; - iov_iter_init(&iter, WRITE, &iov, 1, size); - - return generic_load_microcode(cpu, &iter); -} - static struct microcode_ops microcode_intel_ops = { - .request_microcode_user = request_microcode_user, .request_microcode_fw = request_microcode_fw, .collect_cpu_info = collect_cpu_info, .apply_microcode = apply_microcode_intel, diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index bb1c3f5f60c81d3969e0c84e269f3f4883dd0467..de62b0b87cedfb4225af0fc0f63dc1e0799bcd02 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -147,7 +147,6 @@ static inline void cache_alloc_hsw_probe(void) r->cache.shareable_bits = 0xc0000; r->cache.min_cbm_bits = 2; r->alloc_capable = true; - r->alloc_enabled = true; rdt_alloc_capable = true; } @@ -211,7 +210,6 @@ static bool __get_mem_config_intel(struct rdt_resource *r) thread_throttle_mode_init(); r->alloc_capable = true; - r->alloc_enabled = true; return true; } @@ -242,7 +240,6 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r) r->data_width = 4; r->alloc_capable = true; - r->alloc_enabled = true; return true; } @@ -261,7 +258,6 @@ static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r) r->cache.shareable_bits = ebx & r->default_ctrl; r->data_width = (r->cache.cbm_len + 3) / 4; r->alloc_capable = true; - r->alloc_enabled = true; } static void rdt_get_cdp_config(int level) @@ -300,7 +296,7 @@ mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r) * that can be written to QOS_MSRs. * There are currently no SKUs which support non linear delay values. */ -u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) +static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r) { if (r->membw.delay_linear) return MAX_MBA_BW - bw; @@ -401,7 +397,7 @@ struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, return NULL; } -void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm) +static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc) { struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); int i; @@ -410,12 +406,17 @@ void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm) * Initialize the Control MSRs to having no control. * For Cache Allocation: Set all bits in cbm * For Memory Allocation: Set b/w requested to 100% - * and the bandwidth in MBps to U32_MAX */ - for (i = 0; i < hw_res->num_closid; i++, dc++, dm++) { + for (i = 0; i < hw_res->num_closid; i++, dc++) *dc = r->default_ctrl; - *dm = MBA_MAX_MBPS; - } +} + +static void domain_free(struct rdt_hw_domain *hw_dom) +{ + kfree(hw_dom->arch_mbm_total); + kfree(hw_dom->arch_mbm_local); + kfree(hw_dom->ctrl_val); + kfree(hw_dom); } static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) @@ -423,23 +424,15 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); struct msr_param m; - u32 *dc, *dm; + u32 *dc; dc = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->ctrl_val), GFP_KERNEL); if (!dc) return -ENOMEM; - dm = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->mbps_val), - GFP_KERNEL); - if (!dm) { - kfree(dc); - return -ENOMEM; - } - hw_dom->ctrl_val = dc; - hw_dom->mbps_val = dm; - setup_default_ctrlval(r, dc, dm); + setup_default_ctrlval(r, dc); m.low = 0; m.high = hw_res->num_closid; @@ -447,39 +440,31 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d) return 0; } -static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d) +/** + * arch_domain_mbm_alloc() - Allocate arch private storage for the MBM counters + * @num_rmid: The size of the MBM counter array + * @hw_dom: The domain that owns the allocated arrays + */ +static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_domain *hw_dom) { size_t tsize; - if (is_llc_occupancy_enabled()) { - d->rmid_busy_llc = bitmap_zalloc(r->num_rmid, GFP_KERNEL); - if (!d->rmid_busy_llc) - return -ENOMEM; - INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo); - } if (is_mbm_total_enabled()) { - tsize = sizeof(*d->mbm_total); - d->mbm_total = kcalloc(r->num_rmid, tsize, GFP_KERNEL); - if (!d->mbm_total) { - bitmap_free(d->rmid_busy_llc); + tsize = sizeof(*hw_dom->arch_mbm_total); + hw_dom->arch_mbm_total = kcalloc(num_rmid, tsize, GFP_KERNEL); + if (!hw_dom->arch_mbm_total) return -ENOMEM; - } } if (is_mbm_local_enabled()) { - tsize = sizeof(*d->mbm_local); - d->mbm_local = kcalloc(r->num_rmid, tsize, GFP_KERNEL); - if (!d->mbm_local) { - bitmap_free(d->rmid_busy_llc); - kfree(d->mbm_total); + tsize = sizeof(*hw_dom->arch_mbm_local); + hw_dom->arch_mbm_local = kcalloc(num_rmid, tsize, GFP_KERNEL); + if (!hw_dom->arch_mbm_local) { + kfree(hw_dom->arch_mbm_total); + hw_dom->arch_mbm_total = NULL; return -ENOMEM; } } - if (is_mbm_enabled()) { - INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow); - mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL); - } - return 0; } @@ -502,6 +487,7 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r) struct list_head *add_pos = NULL; struct rdt_hw_domain *hw_dom; struct rdt_domain *d; + int err; d = rdt_find_domain(r, id, &add_pos); if (IS_ERR(d)) { @@ -527,25 +513,22 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r) rdt_domain_reconfigure_cdp(r); if (r->alloc_capable && domain_setup_ctrlval(r, d)) { - kfree(hw_dom); + domain_free(hw_dom); return; } - if (r->mon_capable && domain_setup_mon_state(r, d)) { - kfree(hw_dom->ctrl_val); - kfree(hw_dom->mbps_val); - kfree(hw_dom); + if (r->mon_capable && arch_domain_mbm_alloc(r->num_rmid, hw_dom)) { + domain_free(hw_dom); return; } list_add_tail(&d->list, add_pos); - /* - * If resctrl is mounted, add - * per domain monitor data directories. - */ - if (static_branch_unlikely(&rdt_mon_enable_key)) - mkdir_mondata_subdir_allrdtgrp(r, d); + err = resctrl_online_domain(r, d); + if (err) { + list_del(&d->list); + domain_free(hw_dom); + } } static void domain_remove_cpu(int cpu, struct rdt_resource *r) @@ -563,27 +546,8 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) cpumask_clear_cpu(cpu, &d->cpu_mask); if (cpumask_empty(&d->cpu_mask)) { - /* - * If resctrl is mounted, remove all the - * per domain monitor data directories. - */ - if (static_branch_unlikely(&rdt_mon_enable_key)) - rmdir_mondata_subdir_allrdtgrp(r, d->id); + resctrl_offline_domain(r, d); list_del(&d->list); - if (r->mon_capable && is_mbm_enabled()) - cancel_delayed_work(&d->mbm_over); - if (is_llc_occupancy_enabled() && has_busy_rmid(r, d)) { - /* - * When a package is going down, forcefully - * decrement rmid->ebusy. There is no way to know - * that the L3 was flushed and hence may lead to - * incorrect counts in rare scenarios, but leaving - * the RMID as busy creates RMID leaks if the - * package never comes back. - */ - __check_limbo(d, true); - cancel_delayed_work(&d->cqm_limbo); - } /* * rdt_domain "d" is going to be freed below, so clear @@ -591,13 +555,8 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r) */ if (d->plr) d->plr->d = NULL; + domain_free(hw_dom); - kfree(hw_dom->ctrl_val); - kfree(hw_dom->mbps_val); - bitmap_free(d->rmid_busy_llc); - kfree(d->mbm_total); - kfree(d->mbm_local); - kfree(hw_dom); return; } diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c index 87666275eed920654868ccf3fd9bb2a0b6b81004..1dafbdc5ac31627e9391832fdf032685871eb637 100644 --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c @@ -61,6 +61,7 @@ int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, struct rdt_domain *d) { struct resctrl_staged_config *cfg; + u32 closid = data->rdtgrp->closid; struct rdt_resource *r = s->res; unsigned long bw_val; @@ -72,6 +73,12 @@ int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, if (!bw_validate(data->buf, &bw_val, r)) return -EINVAL; + + if (is_mba_sc(r)) { + d->mbps_val[closid] = bw_val; + return 0; + } + cfg->new_ctrl = bw_val; cfg->have_new_ctrl = true; @@ -261,14 +268,13 @@ static u32 get_config_index(u32 closid, enum resctrl_conf_type type) static bool apply_config(struct rdt_hw_domain *hw_dom, struct resctrl_staged_config *cfg, u32 idx, - cpumask_var_t cpu_mask, bool mba_sc) + cpumask_var_t cpu_mask) { struct rdt_domain *dom = &hw_dom->d_resctrl; - u32 *dc = !mba_sc ? hw_dom->ctrl_val : hw_dom->mbps_val; - if (cfg->new_ctrl != dc[idx]) { + if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) { cpumask_set_cpu(cpumask_any(&dom->cpu_mask), cpu_mask); - dc[idx] = cfg->new_ctrl; + hw_dom->ctrl_val[idx] = cfg->new_ctrl; return true; } @@ -276,6 +282,27 @@ static bool apply_config(struct rdt_hw_domain *hw_dom, return false; } +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, + u32 closid, enum resctrl_conf_type t, u32 cfg_val) +{ + struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); + struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + u32 idx = get_config_index(closid, t); + struct msr_param msr_param; + + if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) + return -EINVAL; + + hw_dom->ctrl_val[idx] = cfg_val; + + msr_param.res = r; + msr_param.low = idx; + msr_param.high = idx + 1; + hw_res->msr_update(d, &msr_param, r); + + return 0; +} + int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) { struct resctrl_staged_config *cfg; @@ -284,14 +311,12 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) enum resctrl_conf_type t; cpumask_var_t cpu_mask; struct rdt_domain *d; - bool mba_sc; int cpu; u32 idx; if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL)) return -ENOMEM; - mba_sc = is_mba_sc(r); msr_param.res = NULL; list_for_each_entry(d, &r->domains, list) { hw_dom = resctrl_to_arch_dom(d); @@ -301,7 +326,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) continue; idx = get_config_index(closid, t); - if (!apply_config(hw_dom, cfg, idx, cpu_mask, mba_sc)) + if (!apply_config(hw_dom, cfg, idx, cpu_mask)) continue; if (!msr_param.res) { @@ -315,11 +340,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid) } } - /* - * Avoid writing the control msr with control values when - * MBA software controller is enabled - */ - if (cpumask_empty(cpu_mask) || mba_sc) + if (cpumask_empty(cpu_mask)) goto done; cpu = get_cpu(); /* Update resource control msr on this CPU if it's in cpu_mask. */ @@ -406,6 +427,14 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; + + /* + * Writes to mba_sc resources update the software controller, + * not the control MSR. + */ + if (is_mba_sc(r)) + continue; + ret = resctrl_arch_update_domains(r, rdtgrp->closid); if (ret) goto out; @@ -433,9 +462,7 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); u32 idx = get_config_index(closid, type); - if (!is_mba_sc(r)) - return hw_dom->ctrl_val[idx]; - return hw_dom->mbps_val[idx]; + return hw_dom->ctrl_val[idx]; } static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int closid) @@ -450,8 +477,12 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int clo if (sep) seq_puts(s, ";"); - ctrl_val = resctrl_arch_get_config(r, dom, closid, - schema->conf_type); + if (is_mba_sc(r)) + ctrl_val = dom->mbps_val[closid]; + else + ctrl_val = resctrl_arch_get_config(r, dom, closid, + schema->conf_type); + seq_printf(s, r->format_str, dom->id, max_data_width, ctrl_val); sep = true; @@ -518,7 +549,6 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, int rdtgroup_mondata_show(struct seq_file *m, void *arg) { struct kernfs_open_file *of = m->private; - struct rdt_hw_resource *hw_res; u32 resid, evtid, domid; struct rdtgroup *rdtgrp; struct rdt_resource *r; @@ -538,8 +568,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) domid = md.u.domid; evtid = md.u.evtid; - hw_res = &rdt_resources_all[resid]; - r = &hw_res->r_resctrl; + r = &rdt_resources_all[resid].r_resctrl; d = rdt_find_domain(r, domid, NULL); if (IS_ERR_OR_NULL(d)) { ret = -ENOENT; @@ -548,12 +577,12 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) mon_event_read(&rr, r, d, rdtgrp, evtid, false); - if (rr.val & RMID_VAL_ERROR) + if (rr.err == -EIO) seq_puts(m, "Error\n"); - else if (rr.val & RMID_VAL_UNAVAIL) + else if (rr.err == -EINVAL) seq_puts(m, "Unavailable\n"); else - seq_printf(m, "%llu\n", rr.val * hw_res->mon_scale); + seq_printf(m, "%llu\n", rr.val); out: rdtgroup_kn_unlock(of->kn); diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 1d647188a43bfc9b673dcfd55301f897bc140ffd..5f7128686cfd2f851b92707afedd6e7d313dfeab 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -22,21 +22,12 @@ #define L2_QOS_CDP_ENABLE 0x01ULL -/* - * Event IDs are used to program IA32_QM_EVTSEL before reading event - * counter from IA32_QM_CTR - */ -#define QOS_L3_OCCUP_EVENT_ID 0x01 -#define QOS_L3_MBM_TOTAL_EVENT_ID 0x02 -#define QOS_L3_MBM_LOCAL_EVENT_ID 0x03 - #define CQM_LIMBOCHECK_INTERVAL 1000 #define MBM_CNTR_WIDTH_BASE 24 #define MBM_OVERFLOW_INTERVAL 1000 #define MAX_MBA_BW 100u #define MBA_IS_LINEAR 0x4 -#define MBA_MAX_MBPS U32_MAX #define MAX_MBA_BW_AMD 0x800 #define MBM_CNTR_WIDTH_OFFSET_AMD 20 @@ -74,7 +65,7 @@ DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key); * @list: entry in &rdt_resource->evt_list */ struct mon_evt { - u32 evtid; + enum resctrl_event_id evtid; char *name; struct list_head list; }; @@ -91,9 +82,9 @@ struct mon_evt { union mon_data_bits { void *priv; struct { - unsigned int rid : 10; - unsigned int evtid : 8; - unsigned int domid : 14; + unsigned int rid : 10; + enum resctrl_event_id evtid : 8; + unsigned int domid : 14; } u; }; @@ -101,12 +92,12 @@ struct rmid_read { struct rdtgroup *rgrp; struct rdt_resource *r; struct rdt_domain *d; - int evtid; + enum resctrl_event_id evtid; bool first; + int err; u64 val; }; -extern unsigned int resctrl_cqm_threshold; extern bool rdt_alloc_capable; extern bool rdt_mon_capable; extern unsigned int rdt_mon_features; @@ -288,35 +279,45 @@ struct rftype { /** * struct mbm_state - status for each MBM counter in each domain - * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes) - * @prev_msr: Value of IA32_QM_CTR for this RMID last time we read it - * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting + * @prev_bw_bytes: Previous bytes value read for bandwidth calculation * @prev_bw: The most recent bandwidth in MBps * @delta_bw: Difference between the current and previous bandwidth * @delta_comp: Indicates whether to compute the delta_bw */ struct mbm_state { - u64 chunks; - u64 prev_msr; - u64 prev_bw_msr; + u64 prev_bw_bytes; u32 prev_bw; u32 delta_bw; bool delta_comp; }; +/** + * struct arch_mbm_state - values used to compute resctrl_arch_rmid_read()s + * return value. + * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes) + * @prev_msr: Value of IA32_QM_CTR last time it was read for the RMID used to + * find this struct. + */ +struct arch_mbm_state { + u64 chunks; + u64 prev_msr; +}; + /** * struct rdt_hw_domain - Arch private attributes of a set of CPUs that share * a resource * @d_resctrl: Properties exposed to the resctrl file system * @ctrl_val: array of cache or mem ctrl values (indexed by CLOSID) - * @mbps_val: When mba_sc is enabled, this holds the bandwidth in MBps + * @arch_mbm_total: arch private state for MBM total bandwidth + * @arch_mbm_local: arch private state for MBM local bandwidth * * Members of this structure are accessed via helpers that provide abstraction. */ struct rdt_hw_domain { struct rdt_domain d_resctrl; u32 *ctrl_val; - u32 *mbps_val; + struct arch_mbm_state *arch_mbm_total; + struct arch_mbm_state *arch_mbm_local; }; static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r) @@ -459,14 +460,6 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable); for_each_rdt_resource(r) \ if (r->mon_capable) -#define for_each_alloc_enabled_rdt_resource(r) \ - for_each_rdt_resource(r) \ - if (r->alloc_enabled) - -#define for_each_mon_enabled_rdt_resource(r) \ - for_each_rdt_resource(r) \ - if (r->mon_enabled) - /* CPUID.(EAX=10H, ECX=ResID=1).EAX */ union cpuid_0x10_1_eax { struct { @@ -530,10 +523,6 @@ void free_rmid(u32 rmid); int rdt_get_mon_l3_config(struct rdt_resource *r); void mon_event_count(void *info); int rdtgroup_mondata_show(struct seq_file *m, void *arg); -void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - unsigned int dom_id); -void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - struct rdt_domain *d); void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, struct rdt_domain *d, struct rdtgroup *rdtgrp, int evtid, int first); @@ -542,8 +531,6 @@ void mbm_setup_overflow_handler(struct rdt_domain *dom, void mbm_handle_overflow(struct work_struct *work); void __init intel_rdt_mbm_apply_quirk(void); bool is_mba_sc(struct rdt_resource *r); -void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm); -u32 delay_bw_map(unsigned long bw, struct rdt_resource *r); void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms); void cqm_handle_limbo(struct work_struct *work); bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d); diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index eaf25a234ff5de954bfad39969c88110c59a769d..efe0c30d3a12d5f126a129d81bcac921dda97252 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -16,8 +16,12 @@ */ #include +#include #include + #include +#include + #include "internal.h" struct rmid_entry { @@ -37,8 +41,8 @@ static LIST_HEAD(rmid_free_lru); * @rmid_limbo_count count of currently unused but (potentially) * dirty RMIDs. * This counts RMIDs that no one is currently using but that - * may have a occupancy value > intel_cqm_threshold. User can change - * the threshold occupancy value. + * may have a occupancy value > resctrl_rmid_realloc_threshold. User can + * change the threshold occupancy value. */ static unsigned int rmid_limbo_count; @@ -59,10 +63,15 @@ bool rdt_mon_capable; unsigned int rdt_mon_features; /* - * This is the threshold cache occupancy at which we will consider an + * This is the threshold cache occupancy in bytes at which we will consider an * RMID available for re-allocation. */ -unsigned int resctrl_cqm_threshold; +unsigned int resctrl_rmid_realloc_threshold; + +/* + * This is the maximum value for the reallocation threshold, in bytes. + */ +unsigned int resctrl_rmid_realloc_limit; #define CF(cf) ((unsigned long)(1048576 * (cf) + 0.5)) @@ -137,9 +146,54 @@ static inline struct rmid_entry *__rmid_entry(u32 rmid) return entry; } -static u64 __rmid_read(u32 rmid, u32 eventid) +static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_domain *hw_dom, + u32 rmid, + enum resctrl_event_id eventid) { - u64 val; + switch (eventid) { + case QOS_L3_OCCUP_EVENT_ID: + return NULL; + case QOS_L3_MBM_TOTAL_EVENT_ID: + return &hw_dom->arch_mbm_total[rmid]; + case QOS_L3_MBM_LOCAL_EVENT_ID: + return &hw_dom->arch_mbm_local[rmid]; + } + + /* Never expect to get here */ + WARN_ON_ONCE(1); + + return NULL; +} + +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, + u32 rmid, enum resctrl_event_id eventid) +{ + struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + struct arch_mbm_state *am; + + am = get_arch_mbm_state(hw_dom, rmid, eventid); + if (am) + memset(am, 0, sizeof(*am)); +} + +static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) +{ + u64 shift = 64 - width, chunks; + + chunks = (cur_msr << shift) - (prev_msr << shift); + return chunks >> shift; +} + +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, + u32 rmid, enum resctrl_event_id eventid, u64 *val) +{ + struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); + struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d); + struct arch_mbm_state *am; + u64 msr_val, chunks; + + if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask)) + return -EINVAL; /* * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured @@ -150,16 +204,26 @@ static u64 __rmid_read(u32 rmid, u32 eventid) * are error bits. */ wrmsr(MSR_IA32_QM_EVTSEL, eventid, rmid); - rdmsrl(MSR_IA32_QM_CTR, val); - - return val; -} + rdmsrl(MSR_IA32_QM_CTR, msr_val); + + if (msr_val & RMID_VAL_ERROR) + return -EIO; + if (msr_val & RMID_VAL_UNAVAIL) + return -EINVAL; + + am = get_arch_mbm_state(hw_dom, rmid, eventid); + if (am) { + am->chunks += mbm_overflow_count(am->prev_msr, msr_val, + hw_res->mbm_width); + chunks = get_corrected_mbm_count(rmid, am->chunks); + am->prev_msr = msr_val; + } else { + chunks = msr_val; + } -static bool rmid_dirty(struct rmid_entry *entry) -{ - u64 val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID); + *val = chunks * hw_res->mon_scale; - return val >= resctrl_cqm_threshold; + return 0; } /* @@ -170,11 +234,11 @@ static bool rmid_dirty(struct rmid_entry *entry) */ void __check_limbo(struct rdt_domain *d, bool force_free) { + struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; struct rmid_entry *entry; - struct rdt_resource *r; u32 crmid = 1, nrmid; - - r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; + bool rmid_dirty; + u64 val = 0; /* * Skip RMID 0 and start from RMID 1 and check all the RMIDs that @@ -188,7 +252,15 @@ void __check_limbo(struct rdt_domain *d, bool force_free) break; entry = __rmid_entry(nrmid); - if (force_free || !rmid_dirty(entry)) { + + if (resctrl_arch_rmid_read(r, d, entry->rmid, + QOS_L3_OCCUP_EVENT_ID, &val)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + } + + if (force_free || !rmid_dirty) { clear_bit(entry->rmid, d->rmid_busy_llc); if (!--entry->busy) { rmid_limbo_count--; @@ -227,19 +299,19 @@ int alloc_rmid(void) static void add_rmid_to_limbo(struct rmid_entry *entry) { - struct rdt_resource *r; + struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; struct rdt_domain *d; - int cpu; - u64 val; - - r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl; + int cpu, err; + u64 val = 0; entry->busy = 0; cpu = get_cpu(); list_for_each_entry(d, &r->domains, list) { if (cpumask_test_cpu(cpu, &d->cpu_mask)) { - val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID); - if (val <= resctrl_cqm_threshold) + err = resctrl_arch_rmid_read(r, d, entry->rmid, + QOS_L3_OCCUP_EVENT_ID, + &val); + if (err || val <= resctrl_rmid_realloc_threshold) continue; } @@ -277,24 +349,18 @@ void free_rmid(u32 rmid) list_add_tail(&entry->list, &rmid_free_lru); } -static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) +static int __mon_event_count(u32 rmid, struct rmid_read *rr) { - u64 shift = 64 - width, chunks; + struct mbm_state *m; + u64 tval = 0; - chunks = (cur_msr << shift) - (prev_msr << shift); - return chunks >> shift; -} + if (rr->first) + resctrl_arch_reset_rmid(rr->r, rr->d, rmid, rr->evtid); -static u64 __mon_event_count(u32 rmid, struct rmid_read *rr) -{ - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(rr->r); - struct mbm_state *m; - u64 chunks, tval; + rr->err = resctrl_arch_rmid_read(rr->r, rr->d, rmid, rr->evtid, &tval); + if (rr->err) + return rr->err; - tval = __rmid_read(rmid, rr->evtid); - if (tval & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL)) { - return tval; - } switch (rr->evtid) { case QOS_L3_OCCUP_EVENT_ID: rr->val += tval; @@ -308,48 +374,47 @@ static u64 __mon_event_count(u32 rmid, struct rmid_read *rr) default: /* * Code would never reach here because an invalid - * event id would fail the __rmid_read. + * event id would fail in resctrl_arch_rmid_read(). */ - return RMID_VAL_ERROR; + return -EINVAL; } if (rr->first) { memset(m, 0, sizeof(struct mbm_state)); - m->prev_bw_msr = m->prev_msr = tval; return 0; } - chunks = mbm_overflow_count(m->prev_msr, tval, hw_res->mbm_width); - m->chunks += chunks; - m->prev_msr = tval; - - rr->val += get_corrected_mbm_count(rmid, m->chunks); + rr->val += tval; return 0; } /* + * mbm_bw_count() - Update bw count from values previously read by + * __mon_event_count(). + * @rmid: The rmid used to identify the cached mbm_state. + * @rr: The struct rmid_read populated by __mon_event_count(). + * * Supporting function to calculate the memory bandwidth - * and delta bandwidth in MBps. + * and delta bandwidth in MBps. The chunks value previously read by + * __mon_event_count() is compared with the chunks value from the previous + * invocation. This must be called once per second to maintain values in MBps. */ static void mbm_bw_count(u32 rmid, struct rmid_read *rr) { - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(rr->r); struct mbm_state *m = &rr->d->mbm_local[rmid]; - u64 tval, cur_bw, chunks; + u64 cur_bw, bytes, cur_bytes; - tval = __rmid_read(rmid, rr->evtid); - if (tval & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL)) - return; + cur_bytes = rr->val; + bytes = cur_bytes - m->prev_bw_bytes; + m->prev_bw_bytes = cur_bytes; - chunks = mbm_overflow_count(m->prev_bw_msr, tval, hw_res->mbm_width); - cur_bw = (get_corrected_mbm_count(rmid, chunks) * hw_res->mon_scale) >> 20; + cur_bw = bytes / SZ_1M; if (m->delta_comp) m->delta_bw = abs(cur_bw - m->prev_bw); m->delta_comp = false; m->prev_bw = cur_bw; - m->prev_bw_msr = tval; } /* @@ -361,11 +426,11 @@ void mon_event_count(void *info) struct rdtgroup *rdtgrp, *entry; struct rmid_read *rr = info; struct list_head *head; - u64 ret_val; + int ret; rdtgrp = rr->rgrp; - ret_val = __mon_event_count(rdtgrp->mon.rmid, rr); + ret = __mon_event_count(rdtgrp->mon.rmid, rr); /* * For Ctrl groups read data from child monitor groups and @@ -377,13 +442,17 @@ void mon_event_count(void *info) if (rdtgrp->type == RDTCTRL_GROUP) { list_for_each_entry(entry, head, mon.crdtgrp_list) { if (__mon_event_count(entry->mon.rmid, rr) == 0) - ret_val = 0; + ret = 0; } } - /* Report error if none of rmid_reads are successful */ - if (ret_val) - rr->val = ret_val; + /* + * __mon_event_count() calls for newly created monitor groups may + * report -EINVAL/Unavailable if the monitor hasn't seen any traffic. + * Discard error if any of the monitor event reads succeeded. + */ + if (ret == 0) + rr->err = 0; } /* @@ -420,10 +489,8 @@ void mon_event_count(void *info) */ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) { - u32 closid, rmid, cur_msr, cur_msr_val, new_msr_val; + u32 closid, rmid, cur_msr_val, new_msr_val; struct mbm_state *pmbm_data, *cmbm_data; - struct rdt_hw_resource *hw_r_mba; - struct rdt_hw_domain *hw_dom_mba; u32 cur_bw, delta_bw, user_bw; struct rdt_resource *r_mba; struct rdt_domain *dom_mba; @@ -433,8 +500,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) if (!is_mbm_local_enabled()) return; - hw_r_mba = &rdt_resources_all[RDT_RESOURCE_MBA]; - r_mba = &hw_r_mba->r_resctrl; + r_mba = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl; + closid = rgrp->closid; rmid = rgrp->mon.rmid; pmbm_data = &dom_mbm->mbm_local[rmid]; @@ -444,16 +511,13 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) pr_warn_once("Failure to get domain for MBA update\n"); return; } - hw_dom_mba = resctrl_to_arch_dom(dom_mba); cur_bw = pmbm_data->prev_bw; - user_bw = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE); + user_bw = dom_mba->mbps_val[closid]; delta_bw = pmbm_data->delta_bw; - /* - * resctrl_arch_get_config() chooses the mbps/ctrl value to return - * based on is_mba_sc(). For now, reach into the hw_dom. - */ - cur_msr_val = hw_dom_mba->ctrl_val[closid]; + + /* MBA resource doesn't support CDP */ + cur_msr_val = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE); /* * For Ctrl groups read data from child monitor groups. @@ -488,9 +552,7 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm) return; } - cur_msr = hw_r_mba->msr_base + closid; - wrmsrl(cur_msr, delay_bw_map(new_msr_val, r_mba)); - hw_dom_mba->ctrl_val[closid] = new_msr_val; + resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val); /* * Delta values are updated dynamically package wise for each @@ -523,10 +585,12 @@ static void mbm_update(struct rdt_resource *r, struct rdt_domain *d, int rmid) */ if (is_mbm_total_enabled()) { rr.evtid = QOS_L3_MBM_TOTAL_EVENT_ID; + rr.val = 0; __mon_event_count(rmid, &rr); } if (is_mbm_local_enabled()) { rr.evtid = QOS_L3_MBM_LOCAL_EVENT_ID; + rr.val = 0; __mon_event_count(rmid, &rr); /* @@ -686,9 +750,10 @@ int rdt_get_mon_l3_config(struct rdt_resource *r) { unsigned int mbm_offset = boot_cpu_data.x86_cache_mbm_width_offset; struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - unsigned int cl_size = boot_cpu_data.x86_cache_size; + unsigned int threshold; int ret; + resctrl_rmid_realloc_limit = boot_cpu_data.x86_cache_size * 1024; hw_res->mon_scale = boot_cpu_data.x86_cache_occ_scale; r->num_rmid = boot_cpu_data.x86_cache_max_rmid + 1; hw_res->mbm_width = MBM_CNTR_WIDTH_BASE; @@ -705,10 +770,14 @@ int rdt_get_mon_l3_config(struct rdt_resource *r) * * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC. */ - resctrl_cqm_threshold = cl_size * 1024 / r->num_rmid; + threshold = resctrl_rmid_realloc_limit / r->num_rmid; - /* h/w works in units of "boot_cpu_data.x86_cache_occ_scale" */ - resctrl_cqm_threshold /= hw_res->mon_scale; + /* + * Because num_rmid may not be a power of two, round the value + * to the nearest multiple of hw_res->mon_scale so it matches a + * value the hardware will measure. mon_scale may not be a power of 2. + */ + resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(threshold); ret = dom_data_init(r); if (ret) @@ -717,7 +786,6 @@ int rdt_get_mon_l3_config(struct rdt_resource *r) l3_mon_evt_init(r); r->mon_capable = true; - r->mon_enabled = true; return 0; } diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index db813f819ad6cb071290eac33dc3c6ff4bcc2059..d961ae3ed96e720e1a346bf1775abee9bfbe83a5 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -420,6 +420,7 @@ static int pseudo_lock_fn(void *_rdtgrp) struct pseudo_lock_region *plr = rdtgrp->plr; u32 rmid_p, closid_p; unsigned long i; + u64 saved_msr; #ifdef CONFIG_KASAN /* * The registers used for local register variables are also used @@ -463,6 +464,7 @@ static int pseudo_lock_fn(void *_rdtgrp) * the buffer and evict pseudo-locked memory read earlier from the * cache. */ + saved_msr = __rdmsr(MSR_MISC_FEATURE_CONTROL); __wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); closid_p = this_cpu_read(pqr_state.cur_closid); rmid_p = this_cpu_read(pqr_state.cur_rmid); @@ -514,7 +516,7 @@ static int pseudo_lock_fn(void *_rdtgrp) __wrmsr(IA32_PQR_ASSOC, rmid_p, closid_p); /* Re-enable the hardware prefetcher(s) */ - wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + wrmsrl(MSR_MISC_FEATURE_CONTROL, saved_msr); local_irq_enable(); plr->thread_done = 1; @@ -835,7 +837,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) * First determine which cpus have pseudo-locked regions * associated with them. */ - for_each_alloc_enabled_rdt_resource(r) { + for_each_alloc_capable_rdt_resource(r) { list_for_each_entry(d_i, &r->domains, list) { if (d_i->plr) cpumask_or(cpu_with_psl, cpu_with_psl, @@ -871,6 +873,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d) static int measure_cycles_lat_fn(void *_plr) { struct pseudo_lock_region *plr = _plr; + u32 saved_low, saved_high; unsigned long i; u64 start, end; void *mem_r; @@ -879,6 +882,7 @@ static int measure_cycles_lat_fn(void *_plr) /* * Disable hardware prefetchers. */ + rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); mem_r = READ_ONCE(plr->kmem); /* @@ -895,7 +899,7 @@ static int measure_cycles_lat_fn(void *_plr) end = rdtsc_ordered(); trace_pseudo_lock_mem_latency((u32)(end - start)); } - wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); local_irq_enable(); plr->thread_done = 1; wake_up_interruptible(&plr->lock_thread_wq); @@ -940,6 +944,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0; struct perf_event *miss_event, *hit_event; int hit_pmcnum, miss_pmcnum; + u32 saved_low, saved_high; unsigned int line_size; unsigned int size; unsigned long i; @@ -973,6 +978,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, /* * Disable hardware prefetchers. */ + rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0); /* Initialize rest of local variables */ @@ -1031,7 +1037,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr, */ rmb(); /* Re-enable hardware prefetchers */ - wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0); + wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high); local_irq_enable(); out_hit: perf_event_release_kernel(hit_event); diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index f276aff521e8b851b0cf2641aaec1adef59a4c15..e5a48f05e7876adcfb942e48f930603c2375fd19 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -1030,10 +1030,7 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of, static int max_threshold_occ_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) { - struct rdt_resource *r = of->kn->parent->priv; - struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); - - seq_printf(seq, "%u\n", resctrl_cqm_threshold * hw_res->mon_scale); + seq_printf(seq, "%u\n", resctrl_rmid_realloc_threshold); return 0; } @@ -1055,7 +1052,6 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, static ssize_t max_threshold_occ_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - struct rdt_hw_resource *hw_res; unsigned int bytes; int ret; @@ -1063,11 +1059,10 @@ static ssize_t max_threshold_occ_write(struct kernfs_open_file *of, if (ret) return ret; - if (bytes > (boot_cpu_data.x86_cache_size * 1024)) + if (bytes > resctrl_rmid_realloc_limit) return -EINVAL; - hw_res = resctrl_to_arch_res(of->kn->parent->priv); - resctrl_cqm_threshold = bytes / hw_res->mon_scale; + resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(bytes); return nbytes; } @@ -1356,11 +1351,13 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, struct seq_file *s, void *v) { struct resctrl_schema *schema; + enum resctrl_conf_type type; struct rdtgroup *rdtgrp; struct rdt_resource *r; struct rdt_domain *d; unsigned int size; int ret = 0; + u32 closid; bool sep; u32 ctrl; @@ -1386,8 +1383,11 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, goto out; } + closid = rdtgrp->closid; + list_for_each_entry(schema, &resctrl_schema_all, list) { r = schema->res; + type = schema->conf_type; sep = false; seq_printf(s, "%*s:", max_name_width, schema->name); list_for_each_entry(d, &r->domains, list) { @@ -1396,9 +1396,12 @@ static int rdtgroup_size_show(struct kernfs_open_file *of, if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { size = 0; } else { - ctrl = resctrl_arch_get_config(r, d, - rdtgrp->closid, - schema->conf_type); + if (is_mba_sc(r)) + ctrl = d->mbps_val[closid]; + else + ctrl = resctrl_arch_get_config(r, d, + closid, + type); if (r->rid == RDT_RESOURCE_MBA) size = ctrl; else @@ -1756,7 +1759,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) if (ret) goto out_destroy; - /* loop over enabled controls, these are all alloc_enabled */ + /* loop over enabled controls, these are all alloc_capable */ list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; fflags = r->fflags | RF_CTRL_INFO; @@ -1765,7 +1768,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) goto out_destroy; } - for_each_mon_enabled_rdt_resource(r) { + for_each_mon_capable_rdt_resource(r) { fflags = r->fflags | RF_MON_INFO; sprintf(name, "%s_MON", r->name); ret = rdtgroup_mkdir_info_resdir(r, name, fflags); @@ -1889,26 +1892,61 @@ void rdt_domain_reconfigure_cdp(struct rdt_resource *r) l3_qos_cfg_update(&hw_res->cdp_enabled); } +static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_domain *d) +{ + u32 num_closid = resctrl_arch_get_num_closid(r); + int cpu = cpumask_any(&d->cpu_mask); + int i; + + d->mbps_val = kcalloc_node(num_closid, sizeof(*d->mbps_val), + GFP_KERNEL, cpu_to_node(cpu)); + if (!d->mbps_val) + return -ENOMEM; + + for (i = 0; i < num_closid; i++) + d->mbps_val[i] = MBA_MAX_MBPS; + + return 0; +} + +static void mba_sc_domain_destroy(struct rdt_resource *r, + struct rdt_domain *d) +{ + kfree(d->mbps_val); + d->mbps_val = NULL; +} + /* - * Enable or disable the MBA software controller - * which helps user specify bandwidth in MBps. * MBA software controller is supported only if * MBM is supported and MBA is in linear scale. */ +static bool supports_mba_mbps(void) +{ + struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl; + + return (is_mbm_local_enabled() && + r->alloc_capable && is_mba_linear()); +} + +/* + * Enable or disable the MBA software controller + * which helps user specify bandwidth in MBps. + */ static int set_mba_sc(bool mba_sc) { struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl; - struct rdt_hw_domain *hw_dom; + u32 num_closid = resctrl_arch_get_num_closid(r); struct rdt_domain *d; + int i; - if (!is_mbm_enabled() || !is_mba_linear() || - mba_sc == is_mba_sc(r)) + if (!supports_mba_mbps() || mba_sc == is_mba_sc(r)) return -EINVAL; r->membw.mba_sc = mba_sc; + list_for_each_entry(d, &r->domains, list) { - hw_dom = resctrl_to_arch_dom(d); - setup_default_ctrlval(r, hw_dom->ctrl_val, hw_dom->mbps_val); + for (i = 0; i < num_closid; i++) + d->mbps_val[i] = MBA_MAX_MBPS; } return 0; @@ -2106,7 +2144,7 @@ static int schemata_list_create(void) struct rdt_resource *r; int ret = 0; - for_each_alloc_enabled_rdt_resource(r) { + for_each_alloc_capable_rdt_resource(r) { if (resctrl_arch_get_cdp_enabled(r->rid)) { ret = schemata_list_add(r, CDP_CODE); if (ret) @@ -2261,7 +2299,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx->enable_cdpl2 = true; return 0; case Opt_mba_mbps: - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + if (!supports_mba_mbps()) return -EINVAL; ctx->enable_mba_mbps = true; return 0; @@ -2452,7 +2490,7 @@ static void rdt_kill_sb(struct super_block *sb) set_mba_sc(false); /*Put everything back to default values. */ - for_each_alloc_enabled_rdt_resource(r) + for_each_alloc_capable_rdt_resource(r) reset_all_ctrls(r); cdp_disable_all(); rmdir_all_sub(); @@ -2499,14 +2537,12 @@ static int mon_addfile(struct kernfs_node *parent_kn, const char *name, * Remove all subdirectories of mon_data of ctrl_mon groups * and monitor groups with given domain id. */ -void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, unsigned int dom_id) +static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, + unsigned int dom_id) { struct rdtgroup *prgrp, *crgrp; char name[32]; - if (!r->mon_enabled) - return; - list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { sprintf(name, "mon_%s_%02d", r->name, dom_id); kernfs_remove_by_name(prgrp->mon.mon_data_kn, name); @@ -2565,16 +2601,13 @@ out_destroy: * Add all subdirectories of mon_data for "ctrl_mon" groups * and "monitor" groups with given domain id. */ -void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - struct rdt_domain *d) +static void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, + struct rdt_domain *d) { struct kernfs_node *parent_kn; struct rdtgroup *prgrp, *crgrp; struct list_head *head; - if (!r->mon_enabled) - return; - list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { parent_kn = prgrp->mon.mon_data_kn; mkdir_mondata_subdir(parent_kn, d, r, prgrp); @@ -2642,7 +2675,7 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn, * Create the subdirectories for each domain. Note that all events * in a domain like L3 are grouped into a resource whose domain is L3 */ - for_each_mon_enabled_rdt_resource(r) { + for_each_mon_capable_rdt_resource(r) { ret = mkdir_mondata_subdir_alldom(kn, r, prgrp); if (ret) goto out_destroy; @@ -2786,14 +2819,19 @@ static int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) } /* Initialize MBA resource with default values. */ -static void rdtgroup_init_mba(struct rdt_resource *r) +static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) { struct resctrl_staged_config *cfg; struct rdt_domain *d; list_for_each_entry(d, &r->domains, list) { + if (is_mba_sc(r)) { + d->mbps_val[closid] = MBA_MAX_MBPS; + continue; + } + cfg = &d->staged_config[CDP_NONE]; - cfg->new_ctrl = is_mba_sc(r) ? MBA_MAX_MBPS : r->default_ctrl; + cfg->new_ctrl = r->default_ctrl; cfg->have_new_ctrl = true; } } @@ -2808,7 +2846,9 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp) list_for_each_entry(s, &resctrl_schema_all, list) { r = s->res; if (r->rid == RDT_RESOURCE_MBA) { - rdtgroup_init_mba(r); + rdtgroup_init_mba(r, rdtgrp->closid); + if (is_mba_sc(r)) + continue; } else { ret = rdtgroup_init_cat(s, rdtgrp->closid); if (ret < 0) @@ -3236,6 +3276,110 @@ out: return ret; } +static void domain_destroy_mon_state(struct rdt_domain *d) +{ + bitmap_free(d->rmid_busy_llc); + kfree(d->mbm_total); + kfree(d->mbm_local); +} + +void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d) +{ + lockdep_assert_held(&rdtgroup_mutex); + + if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) + mba_sc_domain_destroy(r, d); + + if (!r->mon_capable) + return; + + /* + * If resctrl is mounted, remove all the + * per domain monitor data directories. + */ + if (static_branch_unlikely(&rdt_mon_enable_key)) + rmdir_mondata_subdir_allrdtgrp(r, d->id); + + if (is_mbm_enabled()) + cancel_delayed_work(&d->mbm_over); + if (is_llc_occupancy_enabled() && has_busy_rmid(r, d)) { + /* + * When a package is going down, forcefully + * decrement rmid->ebusy. There is no way to know + * that the L3 was flushed and hence may lead to + * incorrect counts in rare scenarios, but leaving + * the RMID as busy creates RMID leaks if the + * package never comes back. + */ + __check_limbo(d, true); + cancel_delayed_work(&d->cqm_limbo); + } + + domain_destroy_mon_state(d); +} + +static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d) +{ + size_t tsize; + + if (is_llc_occupancy_enabled()) { + d->rmid_busy_llc = bitmap_zalloc(r->num_rmid, GFP_KERNEL); + if (!d->rmid_busy_llc) + return -ENOMEM; + } + if (is_mbm_total_enabled()) { + tsize = sizeof(*d->mbm_total); + d->mbm_total = kcalloc(r->num_rmid, tsize, GFP_KERNEL); + if (!d->mbm_total) { + bitmap_free(d->rmid_busy_llc); + return -ENOMEM; + } + } + if (is_mbm_local_enabled()) { + tsize = sizeof(*d->mbm_local); + d->mbm_local = kcalloc(r->num_rmid, tsize, GFP_KERNEL); + if (!d->mbm_local) { + bitmap_free(d->rmid_busy_llc); + kfree(d->mbm_total); + return -ENOMEM; + } + } + + return 0; +} + +int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d) +{ + int err; + + lockdep_assert_held(&rdtgroup_mutex); + + if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) + /* RDT_RESOURCE_MBA is never mon_capable */ + return mba_sc_domain_allocate(r, d); + + if (!r->mon_capable) + return 0; + + err = domain_setup_mon_state(r, d); + if (err) + return err; + + if (is_mbm_enabled()) { + INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow); + mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL); + } + + if (is_llc_occupancy_enabled()) + INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo); + + /* If resctrl is mounted, add per domain monitor data directories. */ + if (static_branch_unlikely(&rdt_mon_enable_key)) + mkdir_mondata_subdir_allrdtgrp(r, d); + + return 0; +} + /* * rdtgroup_init - rdtgroup initialization * diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index fd44b54c90d50483646909e581e8cc7eaad67188..fc01f81f6e2a382b4daccdac85a9fe0e79d95fc2 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -45,6 +45,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_PROC_FEEDBACK, CPUID_EDX, 11, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, { X86_FEATURE_PERFMON_V2, CPUID_EAX, 0, 0x80000022, 0 }, + { X86_FEATURE_AMD_LBR_V2, CPUID_EAX, 1, 0x80000022, 0 }, { 0, 0, 0, 0, 0 } }; diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index 24c1bb8eb196252708444245e8b5b684c68d184a..1ec20807de1e84642cd6c5cfee1397accb7d4c40 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -12,6 +12,9 @@ #include "encls.h" #include "sgx.h" +static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index, + struct sgx_backing *backing); + #define PCMDS_PER_PAGE (PAGE_SIZE / sizeof(struct sgx_pcmd)) /* * 32 PCMD entries share a PCMD page. PCMD_FIRST_MASK is used to @@ -344,8 +347,11 @@ static vm_fault_t sgx_encl_eaug_page(struct vm_area_struct *vma, } va_page = sgx_encl_grow(encl, false); - if (IS_ERR(va_page)) + if (IS_ERR(va_page)) { + if (PTR_ERR(va_page) == -EBUSY) + vmret = VM_FAULT_NOPAGE; goto err_out_epc; + } if (va_page) list_add(&va_page->list, &encl->va_pages); @@ -906,15 +912,14 @@ const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl) static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, pgoff_t index) { - struct inode *inode = encl->backing->f_path.dentry->d_inode; - struct address_space *mapping = inode->i_mapping; + struct address_space *mapping = encl->backing->f_mapping; gfp_t gfpmask = mapping_gfp_mask(mapping); return shmem_read_mapping_page_gfp(mapping, index, gfpmask); } /** - * sgx_encl_get_backing() - Pin the backing storage + * __sgx_encl_get_backing() - Pin the backing storage * @encl: an enclave pointer * @page_index: enclave page index * @backing: data for accessing backing storage for the page @@ -926,7 +931,7 @@ static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, * 0 on success, * -errno otherwise. */ -static int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, +static int __sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, struct sgx_backing *backing) { pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index); @@ -1001,7 +1006,7 @@ static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl) } /** - * sgx_encl_alloc_backing() - allocate a new backing storage page + * sgx_encl_alloc_backing() - create a new backing storage page * @encl: an enclave pointer * @page_index: enclave page index * @backing: data for accessing backing storage for the page @@ -1009,7 +1014,9 @@ static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl) * When called from ksgxd, sets the active memcg from one of the * mms in the enclave's mm_list prior to any backing page allocation, * in order to ensure that shmem page allocations are charged to the - * enclave. + * enclave. Create a backing page for loading data back into an EPC page with + * ELDU. This function takes a reference on a new backing page which + * must be dropped with a corresponding call to sgx_encl_put_backing(). * * Return: * 0 on success, @@ -1022,7 +1029,7 @@ int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index, struct mem_cgroup *memcg = set_active_memcg(encl_memcg); int ret; - ret = sgx_encl_get_backing(encl, page_index, backing); + ret = __sgx_encl_get_backing(encl, page_index, backing); set_active_memcg(memcg); mem_cgroup_put(encl_memcg); @@ -1040,15 +1047,17 @@ int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index, * It is the caller's responsibility to ensure that it is appropriate to use * sgx_encl_lookup_backing() rather than sgx_encl_alloc_backing(). If lookup is * not used correctly, this will cause an allocation which is not accounted for. + * This function takes a reference on an existing backing page which must be + * dropped with a corresponding call to sgx_encl_put_backing(). * * Return: * 0 on success, * -errno otherwise. */ -int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index, +static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index, struct sgx_backing *backing) { - return sgx_encl_get_backing(encl, page_index, backing); + return __sgx_encl_get_backing(encl, page_index, backing); } /** diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h index a65a952116fd1c2c8fac3db27b9f6453e14f5a59..f94ff14c948698140a1fa84e5efd0d2905ac7951 100644 --- a/arch/x86/kernel/cpu/sgx/encl.h +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -107,8 +107,6 @@ bool current_is_ksgxd(void); void sgx_encl_release(struct kref *ref); int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm); const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl); -int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index, - struct sgx_backing *backing); int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index, struct sgx_backing *backing); void sgx_encl_put_backing(struct sgx_backing *backing); diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 515e2a5f25bbc8e4519a177106fcd5777dcade55..0aad028f04d40366c5846553cbcfcc4d85f1ec5a 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -49,9 +49,13 @@ static LIST_HEAD(sgx_dirty_page_list); * Reset post-kexec EPC pages to the uninitialized state. The pages are removed * from the input list, and made available for the page allocator. SECS pages * prepending their children in the input list are left intact. + * + * Return 0 when sanitization was successful or kthread was stopped, and the + * number of unsanitized pages otherwise. */ -static void __sgx_sanitize_pages(struct list_head *dirty_page_list) +static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list) { + unsigned long left_dirty = 0; struct sgx_epc_page *page; LIST_HEAD(dirty); int ret; @@ -59,7 +63,7 @@ static void __sgx_sanitize_pages(struct list_head *dirty_page_list) /* dirty_page_list is thread-local, no need for a lock: */ while (!list_empty(dirty_page_list)) { if (kthread_should_stop()) - return; + return 0; page = list_first_entry(dirty_page_list, struct sgx_epc_page, list); @@ -92,12 +96,14 @@ static void __sgx_sanitize_pages(struct list_head *dirty_page_list) } else { /* The page is not yet clean - move to the dirty list. */ list_move_tail(&page->list, &dirty); + left_dirty++; } cond_resched(); } list_splice(&dirty, dirty_page_list); + return left_dirty; } static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) @@ -395,10 +401,7 @@ static int ksgxd(void *p) * required for SECS pages, whose child pages blocked EREMOVE. */ __sgx_sanitize_pages(&sgx_dirty_page_list); - __sgx_sanitize_pages(&sgx_dirty_page_list); - - /* sanity check: */ - WARN_ON(!list_empty(&sgx_dirty_page_list)); + WARN_ON(__sgx_sanitize_pages(&sgx_dirty_page_list)); while (!kthread_should_stop()) { if (try_to_freeze()) diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index afae4dd77495173ca742ffbc2afaa23e5777ba34..0bf6779187dda158a8deba21e14215d2b0cbddb7 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -128,7 +128,7 @@ void show_opcodes(struct pt_regs *regs, const char *loglvl) /* No access to the user space stack of other tasks. Ignore. */ break; default: - printk("%sCode: Unable to access opcode bytes at RIP 0x%lx.\n", + printk("%sCode: Unable to access opcode bytes at 0x%lx.\n", loglvl, prologue); break; } @@ -177,6 +177,12 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs, } } +/* + * This function reads pointers from the stack and dereferences them. The + * pointers may not have their KMSAN shadow set up properly, which may result + * in false positive reports. Disable instrumentation to avoid those. + */ +__no_kmsan_checks static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, unsigned long *stack, const char *log_lvl) { diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c index 68b38925a74f085bcd87b24698b9074b95b5060e..44f937015e1e25bf41532eb7e1031a6be32a6523 100644 --- a/arch/x86/kernel/early_printk.c +++ b/arch/x86/kernel/early_printk.c @@ -264,11 +264,11 @@ static __init void early_pci_serial_init(char *s) bar0 = read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_0); /* - * Verify it is a UART type device + * Verify it is a 16550-UART type device */ if (((classcode >> 16 != PCI_CLASS_COMMUNICATION_MODEM) && (classcode >> 16 != PCI_CLASS_COMMUNICATION_SERIAL)) || - (((classcode >> 8) & 0xff) != 0x02)) /* 16550 I/F at BAR0 */ { + (((classcode >> 8) & 0xff) != PCI_SERIAL_16550_COMPATIBLE)) { if (!force) return; } @@ -276,22 +276,22 @@ static __init void early_pci_serial_init(char *s) /* * Determine if it is IO or memory mapped */ - if (bar0 & 0x01) { + if ((bar0 & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { /* it is IO mapped */ serial_in = io_serial_in; serial_out = io_serial_out; - early_serial_base = bar0&0xfffffffc; + early_serial_base = bar0 & PCI_BASE_ADDRESS_IO_MASK; write_pci_config(bus, slot, func, PCI_COMMAND, - cmdreg|PCI_COMMAND_IO); + cmdreg|PCI_COMMAND_IO); } else { /* It is memory mapped - assume 32-bit alignment */ serial_in = mem32_serial_in; serial_out = mem32_serial_out; /* WARNING! assuming the address is always in the first 4G */ early_serial_base = - (unsigned long)early_ioremap(bar0 & 0xfffffff0, 0x10); + (unsigned long)early_ioremap(bar0 & PCI_BASE_ADDRESS_MEM_MASK, 0x10); write_pci_config(bus, slot, func, PCI_COMMAND, - cmdreg|PCI_COMMAND_MEMORY); + cmdreg|PCI_COMMAND_MEMORY); } /* diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index e5dd6da78713bc1f37def623d2bf07577ea981ae..01833ebf5e8e3582992a8860460bbe555923eb30 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -132,7 +132,7 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; } -#ifndef CONFIG_PREEMPT_RT +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void) { struct irq_stack *irqstk; diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 4c3c27b6aea3b56b2ffe9b8c2375c0353f9ce206..eb8bc82846b997031da4529d7009bbc737bee0a9 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -59,8 +59,6 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); -#define stack_addr(regs) ((unsigned long *)regs->sp) - #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\ (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \ (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \ diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c index b1abf663417cdef7eff8f58fbbdfa3732a8ff6c9..c032edcd3d95e06941a4810e09ba18da1707b196 100644 --- a/arch/x86/kernel/module.c +++ b/arch/x86/kernel/module.c @@ -53,7 +53,7 @@ static unsigned long int get_module_load_offset(void) */ if (module_load_offset == 0) module_load_offset = - (get_random_int() % 1024 + 1) * PAGE_SIZE; + (prandom_u32_max(1024) + 1) * PAGE_SIZE; mutex_unlock(&module_kaslr_mutex); } return module_load_offset; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 58a6ea472db92b4efdb7ddaf5eea68034525bbb7..c21b7347a26dd5f26df8cf1802375059dbc659fd 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -965,7 +965,7 @@ early_param("idle", idle_setup); unsigned long arch_align_stack(unsigned long sp) { if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space) - sp -= get_random_int() % 8192; + sp -= prandom_u32_max(8192); return sp & ~0xf; } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 1962008fe7437f89be48e2511199e0356348573f..6b3418bff32611f556cef28d4a0dfc57ff2bd151 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -553,6 +553,7 @@ void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp, bool x32) * Kprobes not supported here. Set the probe on schedule instead. * Function graph tracer not supported too. */ +__no_kmsan_checks __visible __notrace_funcgraph struct task_struct * __switch_to(struct task_struct *prev_p, struct task_struct *next_p) { diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 586f718b8e9511681fd843c4f9e9630249c88844..34904643451332f88de1f62a2a34a01833d4a304 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -4,11 +4,8 @@ */ #include #include -#include -#include #include #include -#include #include #include @@ -20,26 +17,23 @@ /* * This is a special lock that is owned by the CPU and holds the index * register we are working with. It is required for NMI access to the - * CMOS/RTC registers. See include/asm-i386/mc146818rtc.h for details. + * CMOS/RTC registers. See arch/x86/include/asm/mc146818rtc.h for details. */ volatile unsigned long cmos_lock; EXPORT_SYMBOL(cmos_lock); #endif /* CONFIG_X86_32 */ -/* For two digit years assume time is always after that */ -#define CMOS_YEARS_OFFS 2000 - DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL(rtc_lock); /* - * In order to set the CMOS clock precisely, set_rtc_mmss has to be + * In order to set the CMOS clock precisely, mach_set_cmos_time has to be * called 500 ms after the second nowtime has started, because when * nowtime is written into the registers of the CMOS clock, it will * jump to the next second precisely 500 ms later. Check the Motorola * MC146818A or Dallas DS12887 data sheet for details. */ -int mach_set_rtc_mmss(const struct timespec64 *now) +int mach_set_cmos_time(const struct timespec64 *now) { unsigned long long nowtime = now->tv_sec; struct rtc_time tm; @@ -62,8 +56,7 @@ int mach_set_rtc_mmss(const struct timespec64 *now) void mach_get_cmos_time(struct timespec64 *now) { - unsigned int status, year, mon, day, hour, min, sec, century = 0; - unsigned long flags; + struct rtc_time tm; /* * If pm_trace abused the RTC as storage, set the timespec to 0, @@ -74,51 +67,13 @@ void mach_get_cmos_time(struct timespec64 *now) return; } - spin_lock_irqsave(&rtc_lock, flags); - - /* - * If UIP is clear, then we have >= 244 microseconds before - * RTC registers will be updated. Spec sheet says that this - * is the reliable way to read RTC - registers. If UIP is set - * then the register access might be invalid. - */ - while ((CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP)) - cpu_relax(); - - sec = CMOS_READ(RTC_SECONDS); - min = CMOS_READ(RTC_MINUTES); - hour = CMOS_READ(RTC_HOURS); - day = CMOS_READ(RTC_DAY_OF_MONTH); - mon = CMOS_READ(RTC_MONTH); - year = CMOS_READ(RTC_YEAR); - -#ifdef CONFIG_ACPI - if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && - acpi_gbl_FADT.century) - century = CMOS_READ(acpi_gbl_FADT.century); -#endif - - status = CMOS_READ(RTC_CONTROL); - WARN_ON_ONCE(RTC_ALWAYS_BCD && (status & RTC_DM_BINARY)); - - spin_unlock_irqrestore(&rtc_lock, flags); - - if (RTC_ALWAYS_BCD || !(status & RTC_DM_BINARY)) { - sec = bcd2bin(sec); - min = bcd2bin(min); - hour = bcd2bin(hour); - day = bcd2bin(day); - mon = bcd2bin(mon); - year = bcd2bin(year); + if (mc146818_get_time(&tm)) { + pr_err("Unable to read current time from RTC\n"); + now->tv_sec = now->tv_nsec = 0; + return; } - if (century) { - century = bcd2bin(century); - year += century * 100; - } else - year += CMOS_YEARS_OFFS; - - now->tv_sec = mktime64(year, mon, day, hour, min, sec); + now->tv_sec = rtc_tm_to_time64(&tm); now->tv_nsec = 0; } diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c index 63dc626627a03ed1a4d0df7b5fb19fa9021b10b0..a428c62330d371a6742ada0a2e9e578dd3f1def6 100644 --- a/arch/x86/kernel/sev.c +++ b/arch/x86/kernel/sev.c @@ -701,7 +701,13 @@ e_term: void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, unsigned int npages) { - if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) + /* + * This can be invoked in early boot while running identity mapped, so + * use an open coded check for SNP instead of using cc_platform_has(). + * This eliminates worries about jump tables or checking boot_cpu_data + * in the cc_platform_has() function. + */ + if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED)) return; /* @@ -717,7 +723,13 @@ void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long padd void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, unsigned int npages) { - if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) + /* + * This can be invoked in early boot while running identity mapped, so + * use an open coded check for SNP instead of using cc_platform_has(). + * This eliminates worries about jump tables or checking boot_cpu_data + * in the cc_platform_has() function. + */ + if (!(sev_status & MSR_AMD64_SEV_SNP_ENABLED)) return; /* Invalidate the memory pages before they are marked shared in the RMP table. */ @@ -2100,7 +2112,7 @@ bool __init snp_init(struct boot_params *bp) return true; } -void __init snp_abort(void) +void __init __noreturn snp_abort(void) { sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); } diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index f24227bc3220a05918539b8f9cca2ce8f3d0c79a..3f3ea0287f694f6a5a674ef7d63c1b5f8839fbcb 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -1316,7 +1316,7 @@ static void __init smp_sanity_check(void) nr++; } - nr_cpu_ids = 8; + set_nr_cpu_ids(8); } #endif @@ -1569,7 +1569,7 @@ __init void prefill_possible_map(void) possible = i; } - nr_cpu_ids = possible; + set_nr_cpu_ids(possible); pr_info("Allowing %d CPUs, %d hotplug CPUs\n", possible, max_t(int, possible - num_processors, 0)); diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index 3bacd935f840b933b28081dd25b27eb02d9bdf26..4c1bcb6053fc07d54ea8c668164fe8494745f490 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -95,7 +95,7 @@ void __init tboot_probe(void) static pgd_t *tboot_pg_dir; static struct mm_struct tboot_mm = { - .mm_rb = RB_ROOT, + .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, tboot_mm.mmap_lock), .pgd = swapper_pg_dir, .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d62b2cb85ceae55407d02a7ff5678e0df67f3ec3..178015a820f080304f1889da2296abca9de2c644 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -63,6 +63,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 #include @@ -313,7 +314,8 @@ static noinstr bool handle_bug(struct pt_regs *regs) */ if (regs->flags & X86_EFLAGS_IF) raw_local_irq_enable(); - if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) { + if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN || + handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) { regs->ip += LEN_UD2; handled = true; } diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c index 8e1c50c86e5db161c38cd1c0a630f0372c828ecf..d8ba93778ae325886f92f8bcba9d7db5f19fc97a 100644 --- a/arch/x86/kernel/unwind_frame.c +++ b/arch/x86/kernel/unwind_frame.c @@ -183,6 +183,16 @@ static struct pt_regs *decode_frame_pointer(unsigned long *bp) } #endif +/* + * While walking the stack, KMSAN may stomp on stale locals from other + * functions that were marked as uninitialized upon function exit, and + * now hold the call frame information for the current function (e.g. the frame + * pointer). Because KMSAN does not specifically mark call frames as + * initialized, false positive reports are possible. To prevent such reports, + * we mark the functions scanning the stack (here and below) with + * __no_kmsan_checks. + */ +__no_kmsan_checks static bool update_stack_state(struct unwind_state *state, unsigned long *next_bp) { @@ -250,6 +260,7 @@ static bool update_stack_state(struct unwind_state *state, return true; } +__no_kmsan_checks bool unwind_next_frame(struct unwind_state *state) { struct pt_regs *regs; diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 38185aedf7d1622103e83f7e771eb724a02196f2..0ea57da929407378221c29756fb6635596d444d9 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -93,22 +93,27 @@ static struct orc_entry *orc_find(unsigned long ip); static struct orc_entry *orc_ftrace_find(unsigned long ip) { struct ftrace_ops *ops; - unsigned long caller; + unsigned long tramp_addr, offset; ops = ftrace_ops_trampoline(ip); if (!ops) return NULL; + /* Set tramp_addr to the start of the code copied by the trampoline */ if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) - caller = (unsigned long)ftrace_regs_call; + tramp_addr = (unsigned long)ftrace_regs_caller; else - caller = (unsigned long)ftrace_call; + tramp_addr = (unsigned long)ftrace_caller; + + /* Now place tramp_addr to the location within the trampoline ip is at */ + offset = ip - ops->trampoline; + tramp_addr += offset; /* Prevent unlikely recursion */ - if (ip == caller) + if (ip == tramp_addr) return NULL; - return orc_find(caller); + return orc_find(tramp_addr); } #else static struct orc_entry *orc_ftrace_find(unsigned long ip) diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index e84ee5cdbd8c6eb43060e16cb97036b9a5214470..57353519bc119e7458f9a35d172d5f199ba9e75b 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -138,7 +138,7 @@ struct x86_platform_ops x86_platform __ro_after_init = { .calibrate_cpu = native_calibrate_cpu_early, .calibrate_tsc = native_calibrate_tsc, .get_wallclock = mach_get_cmos_time, - .set_wallclock = mach_set_rtc_mmss, + .set_wallclock = mach_set_cmos_time, .iommu_shutdown = iommu_shutdown_noop, .is_untracked_pat_range = is_ISA_range, .nmi_init = default_nmi_init, diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index e3cbd7706136438fc7775febc1970270280ce1c9..67be7f217e37bdad646c288ee3a3b18ceba5939b 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -28,7 +28,8 @@ config KVM select HAVE_KVM_IRQCHIP select HAVE_KVM_PFNCACHE select HAVE_KVM_IRQFD - select HAVE_KVM_DIRTY_RING + select HAVE_KVM_DIRTY_RING_TSO + select HAVE_KVM_DIRTY_RING_ACQ_REL select IRQ_BYPASS_MANAGER select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 75dcf7a72605f335d0f104b26986d17cc5cd81ec..7065462378e2933d7c76711a54cc64c70140443a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -311,11 +311,19 @@ void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_update_cpuid_runtime); +static bool kvm_cpuid_has_hyperv(struct kvm_cpuid_entry2 *entries, int nent) +{ + struct kvm_cpuid_entry2 *entry; + + entry = cpuid_entry2_find(entries, nent, HYPERV_CPUID_INTERFACE, + KVM_CPUID_INDEX_NOT_SIGNIFICANT); + return entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX; +} + static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; struct kvm_cpuid_entry2 *best; - u64 guest_supported_xcr0; best = kvm_find_cpuid_entry(vcpu, 1); if (best && apic) { @@ -327,10 +335,16 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) kvm_apic_set_version(vcpu); } - guest_supported_xcr0 = + vcpu->arch.guest_supported_xcr0 = cpuid_get_supported_xcr0(vcpu->arch.cpuid_entries, vcpu->arch.cpuid_nent); - vcpu->arch.guest_fpu.fpstate->user_xfeatures = guest_supported_xcr0; + /* + * FP+SSE can always be saved/restored via KVM_{G,S}ET_XSAVE, even if + * XSAVE/XCRO are not exposed to the guest, and even if XSAVE isn't + * supported by the host. + */ + vcpu->arch.guest_fpu.fpstate->user_xfeatures = vcpu->arch.guest_supported_xcr0 | + XFEATURE_MASK_FPSSE; kvm_update_pv_runtime(vcpu); @@ -341,7 +355,8 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.cr4_guest_rsvd_bits = __cr4_reserved_bits(guest_cpuid_has, vcpu); - kvm_hv_set_cpuid(vcpu); + kvm_hv_set_cpuid(vcpu, kvm_cpuid_has_hyperv(vcpu->arch.cpuid_entries, + vcpu->arch.cpuid_nent)); /* Invoke the vendor callback only after the above state is updated. */ static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu); @@ -404,6 +419,12 @@ static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2, return 0; } + if (kvm_cpuid_has_hyperv(e2, nent)) { + r = kvm_hv_vcpu_init(vcpu); + if (r) + return r; + } + r = kvm_check_cpuid(vcpu, e2, nent); if (r) return r; @@ -897,8 +918,6 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->edx = 0; } break; - case 9: - break; case 0xa: { /* Architectural Performance Monitoring */ union cpuid10_eax eax; union cpuid10_edx edx; diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index d5ec3a2ed5a44f32e01a50684ae2dde6b658516d..3b27622d46425b58c7285f11f60b346a550c8f84 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -1137,9 +1137,11 @@ static int em_fnstsw(struct x86_emulate_ctxt *ctxt) static void decode_register_operand(struct x86_emulate_ctxt *ctxt, struct operand *op) { - unsigned reg = ctxt->modrm_reg; + unsigned int reg; - if (!(ctxt->d & ModRM)) + if (ctxt->d & ModRM) + reg = ctxt->modrm_reg; + else reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3); if (ctxt->d & Sse) { @@ -1953,7 +1955,7 @@ static int em_pop_sreg(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; - if (ctxt->modrm_reg == VCPU_SREG_SS) + if (seg == VCPU_SREG_SS) ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS; if (ctxt->op_bytes > 2) rsp_increment(ctxt, ctxt->op_bytes - 2); @@ -3645,13 +3647,10 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt) | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32); r = ctxt->ops->set_msr_with_filter(ctxt, msr_index, msr_data); - if (r == X86EMUL_IO_NEEDED) - return r; - - if (r > 0) + if (r == X86EMUL_PROPAGATE_FAULT) return emulate_gp(ctxt, 0); - return r < 0 ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; + return r; } static int em_rdmsr(struct x86_emulate_ctxt *ctxt) @@ -3662,15 +3661,14 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt) r = ctxt->ops->get_msr_with_filter(ctxt, msr_index, &msr_data); - if (r == X86EMUL_IO_NEEDED) - return r; - - if (r) + if (r == X86EMUL_PROPAGATE_FAULT) return emulate_gp(ctxt, 0); - *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data; - *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32; - return X86EMUL_CONTINUE; + if (r == X86EMUL_CONTINUE) { + *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data; + *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32; + } + return r; } static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment) @@ -4132,6 +4130,9 @@ static int em_xsetbv(struct x86_emulate_ctxt *ctxt) { u32 eax, ecx, edx; + if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSXSAVE)) + return emulate_ud(ctxt); + eax = reg_read(ctxt, VCPU_REGS_RAX); edx = reg_read(ctxt, VCPU_REGS_RDX); ecx = reg_read(ctxt, VCPU_REGS_RCX); @@ -4168,8 +4169,7 @@ static int check_dr7_gd(struct x86_emulate_ctxt *ctxt) ctxt->ops->get_dr(ctxt, 7, &dr7); - /* Check if DR7.Global_Enable is set */ - return dr7 & (1 << 13); + return dr7 & DR7_GD; } static int check_dr_read(struct x86_emulate_ctxt *ctxt) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index ed804447589c9353765413ae41e0dc4252fb6f78..0adf4a437e85f6f7e67b7b281ebc2484b680ee2f 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -38,9 +38,6 @@ #include "irq.h" #include "fpu.h" -/* "Hv#1" signature */ -#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 - #define KVM_HV_MAX_SPARSE_VCPU_SET_BITS DIV_ROUND_UP(KVM_MAX_VCPUS, 64) static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, @@ -934,11 +931,14 @@ static void stimer_init(struct kvm_vcpu_hv_stimer *stimer, int timer_index) stimer_prepare_msg(stimer); } -static int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) +int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) { - struct kvm_vcpu_hv *hv_vcpu; + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); int i; + if (hv_vcpu) + return 0; + hv_vcpu = kzalloc(sizeof(struct kvm_vcpu_hv), GFP_KERNEL_ACCOUNT); if (!hv_vcpu) return -ENOMEM; @@ -962,11 +962,9 @@ int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) struct kvm_vcpu_hv_synic *synic; int r; - if (!to_hv_vcpu(vcpu)) { - r = kvm_hv_vcpu_init(vcpu); - if (r) - return r; - } + r = kvm_hv_vcpu_init(vcpu); + if (r) + return r; synic = to_hv_synic(vcpu); @@ -1660,10 +1658,8 @@ int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) if (!host && !vcpu->arch.hyperv_enabled) return 1; - if (!to_hv_vcpu(vcpu)) { - if (kvm_hv_vcpu_init(vcpu)) - return 1; - } + if (kvm_hv_vcpu_init(vcpu)) + return 1; if (kvm_hv_msr_partition_wide(msr)) { int r; @@ -1683,10 +1679,8 @@ int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) if (!host && !vcpu->arch.hyperv_enabled) return 1; - if (!to_hv_vcpu(vcpu)) { - if (kvm_hv_vcpu_init(vcpu)) - return 1; - } + if (kvm_hv_vcpu_init(vcpu)) + return 1; if (kvm_hv_msr_partition_wide(msr)) { int r; @@ -1987,49 +1981,49 @@ ret_success: return HV_STATUS_SUCCESS; } -void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu) +void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled) { + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct kvm_cpuid_entry2 *entry; - struct kvm_vcpu_hv *hv_vcpu; - entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE); - if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX) { - vcpu->arch.hyperv_enabled = true; - } else { - vcpu->arch.hyperv_enabled = false; + vcpu->arch.hyperv_enabled = hyperv_enabled; + + if (!hv_vcpu) { + /* + * KVM should have already allocated kvm_vcpu_hv if Hyper-V is + * enabled in CPUID. + */ + WARN_ON_ONCE(vcpu->arch.hyperv_enabled); return; } - if (!to_hv_vcpu(vcpu) && kvm_hv_vcpu_init(vcpu)) - return; + memset(&hv_vcpu->cpuid_cache, 0, sizeof(hv_vcpu->cpuid_cache)); - hv_vcpu = to_hv_vcpu(vcpu); + if (!vcpu->arch.hyperv_enabled) + return; entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES); if (entry) { hv_vcpu->cpuid_cache.features_eax = entry->eax; hv_vcpu->cpuid_cache.features_ebx = entry->ebx; hv_vcpu->cpuid_cache.features_edx = entry->edx; - } else { - hv_vcpu->cpuid_cache.features_eax = 0; - hv_vcpu->cpuid_cache.features_ebx = 0; - hv_vcpu->cpuid_cache.features_edx = 0; } entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO); if (entry) { hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; - } else { - hv_vcpu->cpuid_cache.enlightenments_eax = 0; - hv_vcpu->cpuid_cache.enlightenments_ebx = 0; } entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES); if (entry) hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; - else - hv_vcpu->cpuid_cache.syndbg_cap_eax = 0; + + entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_NESTED_FEATURES); + if (entry) { + hv_vcpu->cpuid_cache.nested_eax = entry->eax; + hv_vcpu->cpuid_cache.nested_ebx = entry->ebx; + } } int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) @@ -2552,7 +2546,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, case HYPERV_CPUID_NESTED_FEATURES: ent->eax = evmcs_ver; ent->eax |= HV_X64_NESTED_MSR_BITMAP; - + ent->ebx |= HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; break; case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index da2737f2a956c2ab054af924b83012e4960997f6..1030b1b50552373666a59d371e99202389abfab0 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -23,6 +23,9 @@ #include +/* "Hv#1" signature */ +#define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 + /* * The #defines related to the synthetic debugger are required by KDNet, but * they are not documented in the Hyper-V TLFS because the synthetic debugger @@ -141,7 +144,8 @@ void kvm_hv_request_tsc_page_update(struct kvm *kvm); void kvm_hv_init_vm(struct kvm *kvm); void kvm_hv_destroy_vm(struct kvm *kvm); -void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu); +int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu); +void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu, bool hyperv_enabled); int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce); int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args); int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 9dda989a1cf0198efceb27086da9416aeac9c128..d7639d126e6c7ae255f6bce48634ebe1102a5b84 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -3025,17 +3025,8 @@ int kvm_apic_accept_events(struct kvm_vcpu *vcpu) struct kvm_lapic *apic = vcpu->arch.apic; u8 sipi_vector; int r; - unsigned long pe; - if (!lapic_in_kernel(vcpu)) - return 0; - - /* - * Read pending events before calling the check_events - * callback. - */ - pe = smp_load_acquire(&apic->pending_events); - if (!pe) + if (!kvm_apic_has_pending_init_or_sipi(vcpu)) return 0; if (is_guest_mode(vcpu)) { @@ -3043,38 +3034,31 @@ int kvm_apic_accept_events(struct kvm_vcpu *vcpu) if (r < 0) return r == -EBUSY ? 0 : r; /* - * If an event has happened and caused a vmexit, - * we know INITs are latched and therefore - * we will not incorrectly deliver an APIC - * event instead of a vmexit. + * Continue processing INIT/SIPI even if a nested VM-Exit + * occurred, e.g. pending SIPIs should be dropped if INIT+SIPI + * are blocked as a result of transitioning to VMX root mode. */ } /* - * INITs are latched while CPU is in specific states - * (SMM, VMX root mode, SVM with GIF=0). - * Because a CPU cannot be in these states immediately - * after it has processed an INIT signal (and thus in - * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs - * and leave the INIT pending. + * INITs are blocked while CPU is in specific states (SMM, VMX root + * mode, SVM with GIF=0), while SIPIs are dropped if the CPU isn't in + * wait-for-SIPI (WFS). */ - if (kvm_vcpu_latch_init(vcpu)) { + if (!kvm_apic_init_sipi_allowed(vcpu)) { WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); - if (test_bit(KVM_APIC_SIPI, &pe)) - clear_bit(KVM_APIC_SIPI, &apic->pending_events); + clear_bit(KVM_APIC_SIPI, &apic->pending_events); return 0; } - if (test_bit(KVM_APIC_INIT, &pe)) { - clear_bit(KVM_APIC_INIT, &apic->pending_events); + if (test_and_clear_bit(KVM_APIC_INIT, &apic->pending_events)) { kvm_vcpu_reset(vcpu, true); if (kvm_vcpu_is_bsp(apic->vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; else vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED; } - if (test_bit(KVM_APIC_SIPI, &pe)) { - clear_bit(KVM_APIC_SIPI, &apic->pending_events); + if (test_and_clear_bit(KVM_APIC_SIPI, &apic->pending_events)) { if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { /* evaluate pending_events before reading the vector */ smp_rmb(); diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 117a46df5cc1e4d366eb294b1d56e39a52986d7e..a5ac4a5a517920ed8dff0ba1f14cde110606832d 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -7,6 +7,7 @@ #include #include "hyperv.h" +#include "kvm_cache_regs.h" #define KVM_APIC_INIT 0 #define KVM_APIC_SIPI 1 @@ -223,11 +224,17 @@ static inline bool kvm_vcpu_apicv_active(struct kvm_vcpu *vcpu) return lapic_in_kernel(vcpu) && vcpu->arch.apic->apicv_active; } -static inline bool kvm_apic_has_events(struct kvm_vcpu *vcpu) +static inline bool kvm_apic_has_pending_init_or_sipi(struct kvm_vcpu *vcpu) { return lapic_in_kernel(vcpu) && vcpu->arch.apic->pending_events; } +static inline bool kvm_apic_init_sipi_allowed(struct kvm_vcpu *vcpu) +{ + return !is_smm(vcpu) && + !static_call(kvm_x86_apic_init_signal_blocked)(vcpu); +} + static inline bool kvm_lowest_prio_delivery(struct kvm_lapic_irq *irq) { return (irq->delivery_mode == APIC_DM_LOWEST || diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 126fa9aec64cda2b4f80b9558426ffafa34cfb48..6f81539061d6485905e5a2e50a49293096f16035 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1596,6 +1596,8 @@ static void __rmap_add(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, sp->role.level, slot); rmap_count = pte_list_add(cache, spte, rmap_head); + if (rmap_count > kvm->stat.max_mmu_rmap_size) + kvm->stat.max_mmu_rmap_size = rmap_count; if (rmap_count > RMAP_RECYCLE_THRESHOLD) { kvm_zap_all_rmap_sptes(kvm, rmap_head); kvm_flush_remote_tlbs_with_address( @@ -1665,6 +1667,18 @@ static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr) percpu_counter_add(&kvm_total_used_mmu_pages, nr); } +static void kvm_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + kvm_mod_used_mmu_pages(kvm, +1); + kvm_account_pgtable_pages((void *)sp->spt, +1); +} + +static void kvm_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + kvm_mod_used_mmu_pages(kvm, -1); + kvm_account_pgtable_pages((void *)sp->spt, -1); +} + static void kvm_mmu_free_shadow_page(struct kvm_mmu_page *sp) { MMU_WARN_ON(!is_empty_shadow_page(sp->spt)); @@ -2122,7 +2136,7 @@ static struct kvm_mmu_page *kvm_mmu_alloc_shadow_page(struct kvm *kvm, */ sp->mmu_valid_gen = kvm->arch.mmu_valid_gen; list_add(&sp->link, &kvm->arch.active_mmu_pages); - kvm_mod_used_mmu_pages(kvm, +1); + kvm_account_mmu_page(kvm, sp); sp->gfn = gfn; sp->role = role; @@ -2456,7 +2470,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, list_add(&sp->link, invalid_list); else list_move(&sp->link, invalid_list); - kvm_mod_used_mmu_pages(kvm, -1); + kvm_unaccount_mmu_page(kvm, sp); } else { /* * Remove the active root from the active page list, the root @@ -4290,7 +4304,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, vcpu->arch.l1tf_flush_l1d = true; if (!flags) { - trace_kvm_page_fault(fault_address, error_code); + trace_kvm_page_fault(vcpu, fault_address, error_code); if (kvm_event_needs_reinjection(vcpu)) kvm_mmu_unprotect_page_virt(vcpu, fault_address); @@ -5361,19 +5375,6 @@ void kvm_mmu_free_obsolete_roots(struct kvm_vcpu *vcpu) __kvm_mmu_free_obsolete_roots(vcpu->kvm, &vcpu->arch.guest_mmu); } -static bool need_remote_flush(u64 old, u64 new) -{ - if (!is_shadow_present_pte(old)) - return false; - if (!is_shadow_present_pte(new)) - return true; - if ((old ^ new) & SPTE_BASE_ADDR_MASK) - return true; - old ^= shadow_nx_mask; - new ^= shadow_nx_mask; - return (old & ~new & SPTE_PERM_MASK) != 0; -} - static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa, int *bytes) { @@ -5519,7 +5520,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL); if (gentry && sp->role.level != PG_LEVEL_4K) ++vcpu->kvm->stat.mmu_pde_zapped; - if (need_remote_flush(entry, *spte)) + if (is_shadow_present_pte(entry)) flush = true; ++spte; } @@ -6085,47 +6086,18 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, const struct kvm_memory_slot *memslot, int start_level) { - bool flush = false; - if (kvm_memslots_have_rmaps(kvm)) { write_lock(&kvm->mmu_lock); - flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, - start_level, KVM_MAX_HUGEPAGE_LEVEL, - false); + slot_handle_level(kvm, memslot, slot_rmap_write_protect, + start_level, KVM_MAX_HUGEPAGE_LEVEL, false); write_unlock(&kvm->mmu_lock); } if (is_tdp_mmu_enabled(kvm)) { read_lock(&kvm->mmu_lock); - flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level); + kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level); read_unlock(&kvm->mmu_lock); } - - /* - * Flush TLBs if any SPTEs had to be write-protected to ensure that - * guest writes are reflected in the dirty bitmap before the memslot - * update completes, i.e. before enabling dirty logging is visible to - * userspace. - * - * Perform the TLB flush outside the mmu_lock to reduce the amount of - * time the lock is held. However, this does mean that another CPU can - * now grab mmu_lock and encounter a write-protected SPTE while CPUs - * still have a writable mapping for the associated GFN in their TLB. - * - * This is safe but requires KVM to be careful when making decisions - * based on the write-protection status of an SPTE. Specifically, KVM - * also write-protects SPTEs to monitor changes to guest page tables - * during shadow paging, and must guarantee no CPUs can write to those - * page before the lock is dropped. As mentioned in the previous - * paragraph, a write-protected SPTE is no guarantee that CPU cannot - * perform writes. So to determine if a TLB flush is truly required, KVM - * will clear a separate software-only bit (MMU-writable) and skip the - * flush if-and-only-if this bit was already clear. - * - * See is_writable_pte() for more details. - */ - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } static inline bool need_topup(struct kvm_mmu_memory_cache *cache, int min) @@ -6493,32 +6465,30 @@ void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, const struct kvm_memory_slot *memslot) { - bool flush = false; - if (kvm_memslots_have_rmaps(kvm)) { write_lock(&kvm->mmu_lock); /* * Clear dirty bits only on 4k SPTEs since the legacy MMU only * support dirty logging at a 4k granularity. */ - flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false); + slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false); write_unlock(&kvm->mmu_lock); } if (is_tdp_mmu_enabled(kvm)) { read_lock(&kvm->mmu_lock); - flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot); + kvm_tdp_mmu_clear_dirty_slot(kvm, memslot); read_unlock(&kvm->mmu_lock); } /* + * The caller will flush the TLBs after this function returns. + * * It's also safe to flush TLBs out of mmu lock here as currently this * function is only used for dirty logging, in which case flushing TLB * out of mmu lock also guarantees no dirty pages will be lost in * dirty_bitmap. */ - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } void kvm_mmu_zap_all(struct kvm *kvm) @@ -6746,10 +6716,12 @@ int kvm_mmu_vendor_module_init(void) ret = register_shrinker(&mmu_shrinker, "x86-mmu"); if (ret) - goto out; + goto out_shrinker; return 0; +out_shrinker: + percpu_counter_destroy(&kvm_total_used_mmu_pages); out: mmu_destroy_caches(); return ret; diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 39e0205e7300ac160da1fc547de73d2a738680e6..5ab5f94dcb6fdb3c95140dbc31bee11595665832 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -472,7 +472,7 @@ error: #if PTTYPE == PTTYPE_EPT /* - * Use PFERR_RSVD_MASK in error_code to to tell if EPT + * Use PFERR_RSVD_MASK in error_code to tell if EPT * misconfiguration requires to be injected. The detection is * done by is_rsvd_bits_set() above. * diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index f3744eea45f5dbb000240a688a81974f3e9a03db..7670c13ce251b6e08698eef477317b932e5bfdb7 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -343,7 +343,7 @@ static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, } /* - * An shadow-present leaf SPTE may be non-writable for 3 possible reasons: + * A shadow-present leaf SPTE may be non-writable for 4 possible reasons: * * 1. To intercept writes for dirty logging. KVM write-protects huge pages * so that they can be split be split down into the dirty logging @@ -361,8 +361,13 @@ static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, * read-only memslot or guest memory backed by a read-only VMA. Writes to * such pages are disallowed entirely. * - * To keep track of why a given SPTE is write-protected, KVM uses 2 - * software-only bits in the SPTE: + * 4. To emulate the Accessed bit for SPTEs without A/D bits. Note, in this + * case, the SPTE is access-protected, not just write-protected! + * + * For cases #1 and #4, KVM can safely make such SPTEs writable without taking + * mmu_lock as capturing the Accessed/Dirty state doesn't require taking it. + * To differentiate #1 and #4 from #2 and #3, KVM uses two software-only bits + * in the SPTE: * * shadow_mmu_writable_mask, aka MMU-writable - * Cleared on SPTEs that KVM is currently write-protecting for shadow paging @@ -391,7 +396,8 @@ static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, * shadow page tables between vCPUs. Write-protecting an SPTE for dirty logging * (which does not clear the MMU-writable bit), does not flush TLBs before * dropping the lock, as it only needs to synchronize guest writes with the - * dirty bitmap. + * dirty bitmap. Similarly, making the SPTE inaccessible (and non-writable) for + * access-tracking via the clear_young() MMU notifier also does not flush TLBs. * * So, there is the problem: clearing the MMU-writable bit can encounter a * write-protected SPTE while CPUs still have writable mappings for that SPTE diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index bf2ccf9debcaadc9e3d3e8efb7c038eadd43dc59..672f0432d7777fca9414da2bb6fc6cc13d28220b 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -372,6 +372,16 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, } } +static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + kvm_account_pgtable_pages((void *)sp->spt, +1); +} + +static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + kvm_account_pgtable_pages((void *)sp->spt, -1); +} + /** * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages * @@ -384,6 +394,7 @@ static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn, static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp, bool shared) { + tdp_unaccount_mmu_page(kvm, sp); if (shared) spin_lock(&kvm->arch.tdp_mmu_pages_lock); else @@ -1132,6 +1143,7 @@ static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, if (account_nx) account_huge_nx_page(kvm, sp); spin_unlock(&kvm->arch.tdp_mmu_pages_lock); + tdp_account_mmu_page(kvm, sp); return 0; } diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 02f9e4f245bd0f63f91944453c0639c72c94d250..d9b9a0f0db17cb3f329e749ca1a804fef498590b 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -106,9 +106,19 @@ static inline void __kvm_perf_overflow(struct kvm_pmc *pmc, bool in_pmi) return; if (pmc->perf_event && pmc->perf_event->attr.precise_ip) { - /* Indicate PEBS overflow PMI to guest. */ - skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, - (unsigned long *)&pmu->global_status); + if (!in_pmi) { + /* + * TODO: KVM is currently _choosing_ to not generate records + * for emulated instructions, avoiding BUFFER_OVF PMI when + * there are no records. Strictly speaking, it should be done + * as well in the right context to improve sampling accuracy. + */ + skip_pmi = true; + } else { + /* Indicate PEBS overflow PMI to guest. */ + skip_pmi = __test_and_set_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, + (unsigned long *)&pmu->global_status); + } } else { __set_bit(pmc->idx, (unsigned long *)&pmu->global_status); } @@ -227,8 +237,8 @@ static bool pmc_resume_counter(struct kvm_pmc *pmc) get_sample_period(pmc, pmc->counter))) return false; - if (!test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) && - pmc->perf_event->attr.precise_ip) + if (test_bit(pmc->idx, (unsigned long *)&pmc_to_pmu(pmc)->pebs_enable) != + (!!pmc->perf_event->attr.precise_ip)) return false; /* reuse perf_event to serve as pmc_reprogram_counter() does*/ diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 76dcc8a3e8496e3c477b91970661f71f7ed302db..4c620999d230a507dfcfd3340e381c3436da9921 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -55,28 +55,6 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, nested_svm_vmexit(svm); } -static bool nested_svm_handle_page_fault_workaround(struct kvm_vcpu *vcpu, - struct x86_exception *fault) -{ - struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb *vmcb = svm->vmcb; - - WARN_ON(!is_guest_mode(vcpu)); - - if (vmcb12_is_intercept(&svm->nested.ctl, - INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && - !WARN_ON_ONCE(svm->nested.nested_run_pending)) { - vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; - vmcb->control.exit_code_hi = 0; - vmcb->control.exit_info_1 = fault->error_code; - vmcb->control.exit_info_2 = fault->address; - nested_svm_vmexit(svm); - return true; - } - - return false; -} - static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) { struct vcpu_svm *svm = to_svm(vcpu); @@ -468,7 +446,7 @@ static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, unsigned int nr; if (vcpu->arch.exception.injected) { - nr = vcpu->arch.exception.nr; + nr = vcpu->arch.exception.vector; exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; if (vcpu->arch.exception.has_error_code) { @@ -781,11 +759,15 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, struct vcpu_svm *svm = to_svm(vcpu); int ret; - trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, - vmcb12->save.rip, - vmcb12->control.int_ctl, - vmcb12->control.event_inj, - vmcb12->control.nested_ctl); + trace_kvm_nested_vmenter(svm->vmcb->save.rip, + vmcb12_gpa, + vmcb12->save.rip, + vmcb12->control.int_ctl, + vmcb12->control.event_inj, + vmcb12->control.nested_ctl, + vmcb12->control.nested_cr3, + vmcb12->save.cr3, + KVM_ISA_SVM); trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, vmcb12->control.intercepts[INTERCEPT_CR] >> 16, @@ -1304,44 +1286,46 @@ int nested_svm_check_permissions(struct kvm_vcpu *vcpu) return 0; } -static bool nested_exit_on_exception(struct vcpu_svm *svm) +static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, + u32 error_code) { - unsigned int nr = svm->vcpu.arch.exception.nr; + struct vcpu_svm *svm = to_svm(vcpu); - return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(nr)); + return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector)); } -static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) +static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu) { - unsigned int nr = svm->vcpu.arch.exception.nr; + struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; + vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector; vmcb->control.exit_code_hi = 0; - if (svm->vcpu.arch.exception.has_error_code) - vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; + if (ex->has_error_code) + vmcb->control.exit_info_1 = ex->error_code; /* * EXITINFO2 is undefined for all exception intercepts other * than #PF. */ - if (nr == PF_VECTOR) { - if (svm->vcpu.arch.exception.nested_apf) - vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; - else if (svm->vcpu.arch.exception.has_payload) - vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; + if (ex->vector == PF_VECTOR) { + if (ex->has_payload) + vmcb->control.exit_info_2 = ex->payload; else - vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; - } else if (nr == DB_VECTOR) { - /* See inject_pending_event. */ - kvm_deliver_exception_payload(&svm->vcpu); - if (svm->vcpu.arch.dr7 & DR7_GD) { - svm->vcpu.arch.dr7 &= ~DR7_GD; - kvm_update_dr7(&svm->vcpu); + vmcb->control.exit_info_2 = vcpu->arch.cr2; + } else if (ex->vector == DB_VECTOR) { + /* See kvm_check_and_inject_events(). */ + kvm_deliver_exception_payload(vcpu, ex); + + if (vcpu->arch.dr7 & DR7_GD) { + vcpu->arch.dr7 &= ~DR7_GD; + kvm_update_dr7(vcpu); } - } else - WARN_ON(svm->vcpu.arch.exception.has_payload); + } else { + WARN_ON(ex->has_payload); + } nested_svm_vmexit(svm); } @@ -1353,10 +1337,22 @@ static inline bool nested_exit_on_init(struct vcpu_svm *svm) static int svm_check_nested_events(struct kvm_vcpu *vcpu) { - struct vcpu_svm *svm = to_svm(vcpu); - bool block_nested_events = - kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; struct kvm_lapic *apic = vcpu->arch.apic; + struct vcpu_svm *svm = to_svm(vcpu); + /* + * Only a pending nested run blocks a pending exception. If there is a + * previously injected event, the pending exception occurred while said + * event was being delivered and thus needs to be handled. + */ + bool block_nested_exceptions = svm->nested.nested_run_pending; + /* + * New events (not exceptions) are only recognized at instruction + * boundaries. If an event needs reinjection, then KVM is handling a + * VM-Exit that occurred _during_ instruction execution; new events are + * blocked until the instruction completes. + */ + bool block_nested_events = block_nested_exceptions || + kvm_event_needs_reinjection(vcpu); if (lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &apic->pending_events)) { @@ -1368,18 +1364,16 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (vcpu->arch.exception.pending) { - /* - * Only a pending nested run can block a pending exception. - * Otherwise an injected NMI/interrupt should either be - * lost or delivered to the nested hypervisor in the EXITINTINFO - * vmcb field, while delivering the pending exception. - */ - if (svm->nested.nested_run_pending) + if (vcpu->arch.exception_vmexit.pending) { + if (block_nested_exceptions) return -EBUSY; - if (!nested_exit_on_exception(svm)) - return 0; - nested_svm_inject_exception_vmexit(svm); + nested_svm_inject_exception_vmexit(vcpu); + return 0; + } + + if (vcpu->arch.exception.pending) { + if (block_nested_exceptions) + return -EBUSY; return 0; } @@ -1720,8 +1714,8 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) struct kvm_x86_nested_ops svm_nested_ops = { .leave_nested = svm_leave_nested, + .is_exception_vmexit = nested_svm_is_exception_vmexit, .check_events = svm_check_nested_events, - .handle_page_fault_workaround = nested_svm_handle_page_fault_workaround, .triple_fault = nested_svm_triple_fault, .get_nested_state_pages = svm_get_nested_state_pages, .get_state = svm_get_nested_state, diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index f24613a108c5302a81a7a8a41dee7f05fda39697..b68956299fa8ec1240a280a7e3718a41a154efcd 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -23,107 +23,52 @@ enum pmu_type { PMU_TYPE_EVNTSEL, }; -enum index { - INDEX_ZERO = 0, - INDEX_ONE, - INDEX_TWO, - INDEX_THREE, - INDEX_FOUR, - INDEX_FIVE, - INDEX_ERROR, -}; - -static unsigned int get_msr_base(struct kvm_pmu *pmu, enum pmu_type type) +static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); + unsigned int num_counters = pmu->nr_arch_gp_counters; - if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) { - if (type == PMU_TYPE_COUNTER) - return MSR_F15H_PERF_CTR; - else - return MSR_F15H_PERF_CTL; - } else { - if (type == PMU_TYPE_COUNTER) - return MSR_K7_PERFCTR0; - else - return MSR_K7_EVNTSEL0; - } -} + if (pmc_idx >= num_counters) + return NULL; -static enum index msr_to_index(u32 msr) -{ - switch (msr) { - case MSR_F15H_PERF_CTL0: - case MSR_F15H_PERF_CTR0: - case MSR_K7_EVNTSEL0: - case MSR_K7_PERFCTR0: - return INDEX_ZERO; - case MSR_F15H_PERF_CTL1: - case MSR_F15H_PERF_CTR1: - case MSR_K7_EVNTSEL1: - case MSR_K7_PERFCTR1: - return INDEX_ONE; - case MSR_F15H_PERF_CTL2: - case MSR_F15H_PERF_CTR2: - case MSR_K7_EVNTSEL2: - case MSR_K7_PERFCTR2: - return INDEX_TWO; - case MSR_F15H_PERF_CTL3: - case MSR_F15H_PERF_CTR3: - case MSR_K7_EVNTSEL3: - case MSR_K7_PERFCTR3: - return INDEX_THREE; - case MSR_F15H_PERF_CTL4: - case MSR_F15H_PERF_CTR4: - return INDEX_FOUR; - case MSR_F15H_PERF_CTL5: - case MSR_F15H_PERF_CTR5: - return INDEX_FIVE; - default: - return INDEX_ERROR; - } + return &pmu->gp_counters[array_index_nospec(pmc_idx, num_counters)]; } static inline struct kvm_pmc *get_gp_pmc_amd(struct kvm_pmu *pmu, u32 msr, enum pmu_type type) { struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); + unsigned int idx; if (!vcpu->kvm->arch.enable_pmu) return NULL; switch (msr) { - case MSR_F15H_PERF_CTL0: - case MSR_F15H_PERF_CTL1: - case MSR_F15H_PERF_CTL2: - case MSR_F15H_PERF_CTL3: - case MSR_F15H_PERF_CTL4: - case MSR_F15H_PERF_CTL5: + case MSR_F15H_PERF_CTL0 ... MSR_F15H_PERF_CTR5: if (!guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) return NULL; - fallthrough; + /* + * Each PMU counter has a pair of CTL and CTR MSRs. CTLn + * MSRs (accessed via EVNTSEL) are even, CTRn MSRs are odd. + */ + idx = (unsigned int)((msr - MSR_F15H_PERF_CTL0) / 2); + if (!(msr & 0x1) != (type == PMU_TYPE_EVNTSEL)) + return NULL; + break; case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: if (type != PMU_TYPE_EVNTSEL) return NULL; + idx = msr - MSR_K7_EVNTSEL0; break; - case MSR_F15H_PERF_CTR0: - case MSR_F15H_PERF_CTR1: - case MSR_F15H_PERF_CTR2: - case MSR_F15H_PERF_CTR3: - case MSR_F15H_PERF_CTR4: - case MSR_F15H_PERF_CTR5: - if (!guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) - return NULL; - fallthrough; case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: if (type != PMU_TYPE_COUNTER) return NULL; + idx = msr - MSR_K7_PERFCTR0; break; default: return NULL; } - return &pmu->gp_counters[msr_to_index(msr)]; + return amd_pmc_idx_to_pmc(pmu, idx); } static bool amd_hw_event_available(struct kvm_pmc *pmc) @@ -139,22 +84,6 @@ static bool amd_pmc_is_enabled(struct kvm_pmc *pmc) return true; } -static struct kvm_pmc *amd_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) -{ - unsigned int base = get_msr_base(pmu, PMU_TYPE_COUNTER); - struct kvm_vcpu *vcpu = pmu_to_vcpu(pmu); - - if (guest_cpuid_has(vcpu, X86_FEATURE_PERFCTR_CORE)) { - /* - * The idx is contiguous. The MSRs are not. The counter MSRs - * are interleaved with the event select MSRs. - */ - pmc_idx *= 2; - } - - return get_gp_pmc_amd(pmu, base + pmc_idx, PMU_TYPE_COUNTER); -} - static bool amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -168,15 +97,7 @@ static bool amd_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) static struct kvm_pmc *amd_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, unsigned int idx, u64 *mask) { - struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - struct kvm_pmc *counters; - - idx &= ~(3u << 30); - if (idx >= pmu->nr_arch_gp_counters) - return NULL; - counters = pmu->gp_counters; - - return &counters[idx]; + return amd_pmc_idx_to_pmc(vcpu_to_pmu(vcpu), idx & ~(3u << 30)); } static bool amd_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index f3813dbacb9f1c5983b61b9db8bfa9e4fe9709d3..58f0077d935799eaa5922bf9da4dde5e4a02d2b2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -461,24 +461,22 @@ static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu) return 0; } -static void svm_queue_exception(struct kvm_vcpu *vcpu) +static void svm_inject_exception(struct kvm_vcpu *vcpu) { + struct kvm_queued_exception *ex = &vcpu->arch.exception; struct vcpu_svm *svm = to_svm(vcpu); - unsigned nr = vcpu->arch.exception.nr; - bool has_error_code = vcpu->arch.exception.has_error_code; - u32 error_code = vcpu->arch.exception.error_code; - kvm_deliver_exception_payload(vcpu); + kvm_deliver_exception_payload(vcpu, ex); - if (kvm_exception_is_soft(nr) && + if (kvm_exception_is_soft(ex->vector) && svm_update_soft_interrupt_rip(vcpu)) return; - svm->vmcb->control.event_inj = nr + svm->vmcb->control.event_inj = ex->vector | SVM_EVTINJ_VALID - | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) + | (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0) | SVM_EVTINJ_TYPE_EXEPT; - svm->vmcb->control.event_inj_err = error_code; + svm->vmcb->control.event_inj_err = ex->error_code; } static void svm_init_erratum_383(void) @@ -1975,7 +1973,7 @@ static int npf_interception(struct kvm_vcpu *vcpu) u64 fault_address = svm->vmcb->control.exit_info_2; u64 error_code = svm->vmcb->control.exit_info_1; - trace_kvm_page_fault(fault_address, error_code); + trace_kvm_page_fault(vcpu, fault_address, error_code); return kvm_mmu_page_fault(vcpu, fault_address, error_code, static_cpu_has(X86_FEATURE_DECODEASSISTS) ? svm->vmcb->control.insn_bytes : NULL, @@ -2341,7 +2339,8 @@ void svm_set_gif(struct vcpu_svm *svm, bool value) enable_gif(svm); if (svm->vcpu.arch.smi_pending || svm->vcpu.arch.nmi_pending || - kvm_cpu_has_injectable_intr(&svm->vcpu)) + kvm_cpu_has_injectable_intr(&svm->vcpu) || + kvm_apic_has_pending_init_or_sipi(&svm->vcpu)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); } else { disable_gif(svm); @@ -3522,7 +3521,7 @@ void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, /* Note, this is called iff the local APIC is in-kernel. */ if (!READ_ONCE(vcpu->arch.apic->apicv_active)) { - /* Process the interrupt via inject_pending_event */ + /* Process the interrupt via kvm_check_and_inject_events(). */ kvm_make_request(KVM_REQ_EVENT, vcpu); kvm_vcpu_kick(vcpu); return; @@ -4697,15 +4696,7 @@ static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - /* - * TODO: Last condition latch INIT signals on vCPU when - * vCPU is in guest-mode and vmcb12 defines intercept on INIT. - * To properly emulate the INIT intercept, - * svm_check_nested_events() should call nested_svm_vmexit() - * if an INIT signal is pending. - */ - return !gif_set(svm) || - (vmcb_is_intercept(&svm->vmcb->control, INTERCEPT_INIT)); + return !gif_set(svm); } static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) @@ -4798,7 +4789,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .patch_hypercall = svm_patch_hypercall, .inject_irq = svm_inject_irq, .inject_nmi = svm_inject_nmi, - .queue_exception = svm_queue_exception, + .inject_exception = svm_inject_exception, .cancel_injection = svm_cancel_injection, .interrupt_allowed = svm_interrupt_allowed, .nmi_allowed = svm_nmi_allowed, diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 2120d7c060a9a3ac081cf739ec228ebdece79676..bc25589ad588676974770919234b67f9ce198c43 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -394,20 +394,25 @@ TRACE_EVENT(kvm_inj_exception, * Tracepoint for page fault. */ TRACE_EVENT(kvm_page_fault, - TP_PROTO(unsigned long fault_address, unsigned int error_code), - TP_ARGS(fault_address, error_code), + TP_PROTO(struct kvm_vcpu *vcpu, u64 fault_address, u64 error_code), + TP_ARGS(vcpu, fault_address, error_code), TP_STRUCT__entry( - __field( unsigned long, fault_address ) - __field( unsigned int, error_code ) + __field( unsigned int, vcpu_id ) + __field( unsigned long, guest_rip ) + __field( u64, fault_address ) + __field( u64, error_code ) ), TP_fast_assign( + __entry->vcpu_id = vcpu->vcpu_id; + __entry->guest_rip = kvm_rip_read(vcpu); __entry->fault_address = fault_address; __entry->error_code = error_code; ), - TP_printk("address %lx error_code %x", + TP_printk("vcpu %u rip 0x%lx address 0x%016llx error_code 0x%llx", + __entry->vcpu_id, __entry->guest_rip, __entry->fault_address, __entry->error_code) ); @@ -589,10 +594,12 @@ TRACE_EVENT(kvm_pv_eoi, /* * Tracepoint for nested VMRUN */ -TRACE_EVENT(kvm_nested_vmrun, +TRACE_EVENT(kvm_nested_vmenter, TP_PROTO(__u64 rip, __u64 vmcb, __u64 nested_rip, __u32 int_ctl, - __u32 event_inj, bool npt), - TP_ARGS(rip, vmcb, nested_rip, int_ctl, event_inj, npt), + __u32 event_inj, bool tdp_enabled, __u64 guest_tdp_pgd, + __u64 guest_cr3, __u32 isa), + TP_ARGS(rip, vmcb, nested_rip, int_ctl, event_inj, tdp_enabled, + guest_tdp_pgd, guest_cr3, isa), TP_STRUCT__entry( __field( __u64, rip ) @@ -600,7 +607,9 @@ TRACE_EVENT(kvm_nested_vmrun, __field( __u64, nested_rip ) __field( __u32, int_ctl ) __field( __u32, event_inj ) - __field( bool, npt ) + __field( bool, tdp_enabled ) + __field( __u64, guest_pgd ) + __field( __u32, isa ) ), TP_fast_assign( @@ -609,14 +618,24 @@ TRACE_EVENT(kvm_nested_vmrun, __entry->nested_rip = nested_rip; __entry->int_ctl = int_ctl; __entry->event_inj = event_inj; - __entry->npt = npt; - ), - - TP_printk("rip: 0x%016llx vmcb: 0x%016llx nrip: 0x%016llx int_ctl: 0x%08x " - "event_inj: 0x%08x npt: %s", - __entry->rip, __entry->vmcb, __entry->nested_rip, - __entry->int_ctl, __entry->event_inj, - __entry->npt ? "on" : "off") + __entry->tdp_enabled = tdp_enabled; + __entry->guest_pgd = tdp_enabled ? guest_tdp_pgd : guest_cr3; + __entry->isa = isa; + ), + + TP_printk("rip: 0x%016llx %s: 0x%016llx nested_rip: 0x%016llx " + "int_ctl: 0x%08x event_inj: 0x%08x nested_%s=%s %s: 0x%016llx", + __entry->rip, + __entry->isa == KVM_ISA_VMX ? "vmcs" : "vmcb", + __entry->vmcb, + __entry->nested_rip, + __entry->int_ctl, + __entry->event_inj, + __entry->isa == KVM_ISA_VMX ? "ept" : "npt", + __entry->tdp_enabled ? "y" : "n", + !__entry->tdp_enabled ? "guest_cr3" : + __entry->isa == KVM_ISA_VMX ? "nested_eptp" : "nested_cr3", + __entry->guest_pgd) ); TRACE_EVENT(kvm_nested_intercepts, diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index c5e5dfef69c7f848bbb7bd7244322f2e8346e5ec..87c4e46daf3725953f1ecac77dd8a64b9c65704c 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -65,6 +65,7 @@ struct vmcs_config { u64 cpu_based_3rd_exec_ctrl; u32 vmexit_ctrl; u32 vmentry_ctrl; + u64 misc; struct nested_vmx_msrs nested; }; extern struct vmcs_config vmcs_config; @@ -82,7 +83,8 @@ static inline bool cpu_has_vmx_basic_inout(void) static inline bool cpu_has_virtual_nmis(void) { - return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS; + return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS && + vmcs_config.cpu_based_exec_ctrl & CPU_BASED_NMI_WINDOW_EXITING; } static inline bool cpu_has_vmx_preemption_timer(void) @@ -224,11 +226,8 @@ static inline bool cpu_has_vmx_vmfunc(void) static inline bool cpu_has_vmx_shadow_vmcs(void) { - u64 vmx_msr; - /* check if the cpu supports writing r/o exit information fields */ - rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); - if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS)) + if (!(vmcs_config.misc & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS)) return false; return vmcs_config.cpu_based_2nd_exec_ctrl & @@ -370,10 +369,7 @@ static inline bool cpu_has_vmx_invvpid_global(void) static inline bool cpu_has_vmx_intel_pt(void) { - u64 vmx_msr; - - rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); - return (vmx_msr & MSR_IA32_VMX_MISC_INTEL_PT) && + return (vmcs_config.misc & MSR_IA32_VMX_MISC_INTEL_PT) && (vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_PT_USE_GPA) && (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_RTIT_CTL); } diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c index 6a61b1ae794209d01c8e18a0b71e8830d8292b23..d8b23c96d627234e67f50d3bad7eceef444291a5 100644 --- a/arch/x86/kvm/vmx/evmcs.c +++ b/arch/x86/kvm/vmx/evmcs.c @@ -10,6 +10,8 @@ #include "vmx.h" #include "trace.h" +#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK + DEFINE_STATIC_KEY_FALSE(enable_evmcs); #define EVMCS1_OFFSET(x) offsetof(struct hv_enlightened_vmcs, x) @@ -28,6 +30,8 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), EVMCS1_FIELD(HOST_IA32_EFER, host_ia32_efer, HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + EVMCS1_FIELD(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), EVMCS1_FIELD(HOST_CR0, host_cr0, HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), EVMCS1_FIELD(HOST_CR3, host_cr3, @@ -78,6 +82,8 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), EVMCS1_FIELD(GUEST_IA32_EFER, guest_ia32_efer, HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + EVMCS1_FIELD(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), EVMCS1_FIELD(GUEST_PDPTR0, guest_pdptr0, HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), EVMCS1_FIELD(GUEST_PDPTR1, guest_pdptr1, @@ -126,6 +132,28 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), EVMCS1_FIELD(XSS_EXIT_BITMAP, xss_exit_bitmap, HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(ENCLS_EXITING_BITMAP, encls_exiting_bitmap, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + EVMCS1_FIELD(TSC_MULTIPLIER, tsc_multiplier, + HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2), + /* + * Not used by KVM: + * + * EVMCS1_FIELD(0x00006828, guest_ia32_s_cet, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x0000682A, guest_ssp, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC), + * EVMCS1_FIELD(0x0000682C, guest_ia32_int_ssp_table_addr, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x00002816, guest_ia32_lbr_ctl, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1), + * EVMCS1_FIELD(0x00006C18, host_ia32_s_cet, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + * EVMCS1_FIELD(0x00006C1A, host_ssp, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + * EVMCS1_FIELD(0x00006C1C, host_ia32_int_ssp_table_addr, + * HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1), + */ /* 64 bit read only */ EVMCS1_FIELD(GUEST_PHYSICAL_ADDRESS, guest_physical_address, @@ -294,19 +322,6 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { }; const unsigned int nr_evmcs_1_fields = ARRAY_SIZE(vmcs_field_to_evmcs_1); -#if IS_ENABLED(CONFIG_HYPERV) -__init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) -{ - vmcs_conf->cpu_based_exec_ctrl &= ~EVMCS1_UNSUPPORTED_EXEC_CTRL; - vmcs_conf->pin_based_exec_ctrl &= ~EVMCS1_UNSUPPORTED_PINCTRL; - vmcs_conf->cpu_based_2nd_exec_ctrl &= ~EVMCS1_UNSUPPORTED_2NDEXEC; - vmcs_conf->cpu_based_3rd_exec_ctrl = 0; - - vmcs_conf->vmexit_ctrl &= ~EVMCS1_UNSUPPORTED_VMEXIT_CTRL; - vmcs_conf->vmentry_ctrl &= ~EVMCS1_UNSUPPORTED_VMENTRY_CTRL; -} -#endif - bool nested_enlightened_vmentry(struct kvm_vcpu *vcpu, u64 *evmcs_gpa) { struct hv_vp_assist_page assist_page; @@ -334,6 +349,9 @@ uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu) * versions: lower 8 bits is the minimal version, higher 8 bits is the * maximum supported version. KVM supports versions from 1 to * KVM_EVMCS_VERSION. + * + * Note, do not check the Hyper-V is fully enabled in guest CPUID, this + * helper is used to _get_ the vCPU's supported CPUID. */ if (kvm_cpu_cap_get(X86_FEATURE_VMX) && (!vcpu || to_vmx(vcpu)->nested.enlightened_vmcs_enabled)) @@ -342,10 +360,67 @@ uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu) return 0; } -void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata) +enum evmcs_revision { + EVMCSv1_LEGACY, + NR_EVMCS_REVISIONS, +}; + +enum evmcs_ctrl_type { + EVMCS_EXIT_CTRLS, + EVMCS_ENTRY_CTRLS, + EVMCS_2NDEXEC, + EVMCS_PINCTRL, + EVMCS_VMFUNC, + NR_EVMCS_CTRLS, +}; + +static const u32 evmcs_unsupported_ctrls[NR_EVMCS_CTRLS][NR_EVMCS_REVISIONS] = { + [EVMCS_EXIT_CTRLS] = { + [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMEXIT_CTRL, + }, + [EVMCS_ENTRY_CTRLS] = { + [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMENTRY_CTRL, + }, + [EVMCS_2NDEXEC] = { + [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_2NDEXEC, + }, + [EVMCS_PINCTRL] = { + [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_PINCTRL, + }, + [EVMCS_VMFUNC] = { + [EVMCSv1_LEGACY] = EVMCS1_UNSUPPORTED_VMFUNC, + }, +}; + +static u32 evmcs_get_unsupported_ctls(enum evmcs_ctrl_type ctrl_type) +{ + enum evmcs_revision evmcs_rev = EVMCSv1_LEGACY; + + return evmcs_unsupported_ctrls[ctrl_type][evmcs_rev]; +} + +static bool evmcs_has_perf_global_ctrl(struct kvm_vcpu *vcpu) +{ + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + + /* + * PERF_GLOBAL_CTRL has a quirk where some Windows guests may fail to + * boot if a PV CPUID feature flag is not also set. Treat the fields + * as unsupported if the flag is not set in guest CPUID. This should + * be called only for guest accesses, and all guest accesses should be + * gated on Hyper-V being enabled and initialized. + */ + if (WARN_ON_ONCE(!hv_vcpu)) + return false; + + return hv_vcpu->cpuid_cache.nested_ebx & HV_X64_NESTED_EVMCS1_PERF_GLOBAL_CTRL; +} + +void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) { u32 ctl_low = (u32)*pdata; u32 ctl_high = (u32)(*pdata >> 32); + u32 unsupported_ctrls; /* * Hyper-V 2016 and 2019 try using these features even when eVMCS @@ -354,77 +429,70 @@ void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata) switch (msr_index) { case MSR_IA32_VMX_EXIT_CTLS: case MSR_IA32_VMX_TRUE_EXIT_CTLS: - ctl_high &= ~EVMCS1_UNSUPPORTED_VMEXIT_CTRL; + unsupported_ctrls = evmcs_get_unsupported_ctls(EVMCS_EXIT_CTRLS); + if (!evmcs_has_perf_global_ctrl(vcpu)) + unsupported_ctrls |= VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; + ctl_high &= ~unsupported_ctrls; break; case MSR_IA32_VMX_ENTRY_CTLS: case MSR_IA32_VMX_TRUE_ENTRY_CTLS: - ctl_high &= ~EVMCS1_UNSUPPORTED_VMENTRY_CTRL; + unsupported_ctrls = evmcs_get_unsupported_ctls(EVMCS_ENTRY_CTRLS); + if (!evmcs_has_perf_global_ctrl(vcpu)) + unsupported_ctrls |= VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; + ctl_high &= ~unsupported_ctrls; break; case MSR_IA32_VMX_PROCBASED_CTLS2: - ctl_high &= ~EVMCS1_UNSUPPORTED_2NDEXEC; + ctl_high &= ~evmcs_get_unsupported_ctls(EVMCS_2NDEXEC); break; case MSR_IA32_VMX_TRUE_PINBASED_CTLS: case MSR_IA32_VMX_PINBASED_CTLS: - ctl_high &= ~EVMCS1_UNSUPPORTED_PINCTRL; + ctl_high &= ~evmcs_get_unsupported_ctls(EVMCS_PINCTRL); break; case MSR_IA32_VMX_VMFUNC: - ctl_low &= ~EVMCS1_UNSUPPORTED_VMFUNC; + ctl_low &= ~evmcs_get_unsupported_ctls(EVMCS_VMFUNC); break; } *pdata = ctl_low | ((u64)ctl_high << 32); } +static bool nested_evmcs_is_valid_controls(enum evmcs_ctrl_type ctrl_type, + u32 val) +{ + return !(val & evmcs_get_unsupported_ctls(ctrl_type)); +} + int nested_evmcs_check_controls(struct vmcs12 *vmcs12) { - int ret = 0; - u32 unsupp_ctl; - - unsupp_ctl = vmcs12->pin_based_vm_exec_control & - EVMCS1_UNSUPPORTED_PINCTRL; - if (unsupp_ctl) { - trace_kvm_nested_vmenter_failed( - "eVMCS: unsupported pin-based VM-execution controls", - unsupp_ctl); - ret = -EINVAL; - } + if (CC(!nested_evmcs_is_valid_controls(EVMCS_PINCTRL, + vmcs12->pin_based_vm_exec_control))) + return -EINVAL; - unsupp_ctl = vmcs12->secondary_vm_exec_control & - EVMCS1_UNSUPPORTED_2NDEXEC; - if (unsupp_ctl) { - trace_kvm_nested_vmenter_failed( - "eVMCS: unsupported secondary VM-execution controls", - unsupp_ctl); - ret = -EINVAL; - } + if (CC(!nested_evmcs_is_valid_controls(EVMCS_2NDEXEC, + vmcs12->secondary_vm_exec_control))) + return -EINVAL; - unsupp_ctl = vmcs12->vm_exit_controls & - EVMCS1_UNSUPPORTED_VMEXIT_CTRL; - if (unsupp_ctl) { - trace_kvm_nested_vmenter_failed( - "eVMCS: unsupported VM-exit controls", - unsupp_ctl); - ret = -EINVAL; - } + if (CC(!nested_evmcs_is_valid_controls(EVMCS_EXIT_CTRLS, + vmcs12->vm_exit_controls))) + return -EINVAL; - unsupp_ctl = vmcs12->vm_entry_controls & - EVMCS1_UNSUPPORTED_VMENTRY_CTRL; - if (unsupp_ctl) { - trace_kvm_nested_vmenter_failed( - "eVMCS: unsupported VM-entry controls", - unsupp_ctl); - ret = -EINVAL; - } + if (CC(!nested_evmcs_is_valid_controls(EVMCS_ENTRY_CTRLS, + vmcs12->vm_entry_controls))) + return -EINVAL; - unsupp_ctl = vmcs12->vm_function_control & EVMCS1_UNSUPPORTED_VMFUNC; - if (unsupp_ctl) { - trace_kvm_nested_vmenter_failed( - "eVMCS: unsupported VM-function controls", - unsupp_ctl); - ret = -EINVAL; - } + /* + * VM-Func controls are 64-bit, but KVM currently doesn't support any + * controls in bits 63:32, i.e. dropping those bits on the consistency + * check is intentional. + */ + if (WARN_ON_ONCE(vmcs12->vm_function_control >> 32)) + return -EINVAL; - return ret; + if (CC(!nested_evmcs_is_valid_controls(EVMCS_VMFUNC, + vmcs12->vm_function_control))) + return -EINVAL; + + return 0; } int nested_enable_evmcs(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/vmx/evmcs.h b/arch/x86/kvm/vmx/evmcs.h index f886a8ff03426f3bbf28edd3714cf84e126b623d..6f746ef3c0386ce277ef520d9a1746d0728a4ad8 100644 --- a/arch/x86/kvm/vmx/evmcs.h +++ b/arch/x86/kvm/vmx/evmcs.h @@ -42,8 +42,6 @@ DECLARE_STATIC_KEY_FALSE(enable_evmcs); * PLE_GAP = 0x00004020, * PLE_WINDOW = 0x00004022, * VMX_PREEMPTION_TIMER_VALUE = 0x0000482E, - * GUEST_IA32_PERF_GLOBAL_CTRL = 0x00002808, - * HOST_IA32_PERF_GLOBAL_CTRL = 0x00002c04, * * Currently unsupported in KVM: * GUEST_IA32_RTIT_CTL = 0x00002814, @@ -61,9 +59,8 @@ DECLARE_STATIC_KEY_FALSE(enable_evmcs); SECONDARY_EXEC_TSC_SCALING | \ SECONDARY_EXEC_PAUSE_LOOP_EXITING) #define EVMCS1_UNSUPPORTED_VMEXIT_CTRL \ - (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ - VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) -#define EVMCS1_UNSUPPORTED_VMENTRY_CTRL (VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) + (VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) +#define EVMCS1_UNSUPPORTED_VMENTRY_CTRL (0) #define EVMCS1_UNSUPPORTED_VMFUNC (VMX_VMFUNC_EPTP_SWITCHING) struct evmcs_field { @@ -212,7 +209,6 @@ static inline void evmcs_load(u64 phys_addr) vp_ap->enlighten_vmentry = 1; } -__init void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf); #else /* !IS_ENABLED(CONFIG_HYPERV) */ static __always_inline void evmcs_write64(unsigned long field, u64 value) {} static inline void evmcs_write32(unsigned long field, u32 value) {} @@ -243,7 +239,7 @@ bool nested_enlightened_vmentry(struct kvm_vcpu *vcpu, u64 *evmcs_gpa); uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu); int nested_enable_evmcs(struct kvm_vcpu *vcpu, uint16_t *vmcs_version); -void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata); +void nested_evmcs_filter_control_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); int nested_evmcs_check_controls(struct vmcs12 *vmcs12); #endif /* __KVM_X86_VMX_EVMCS_H */ diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ddd4367d482657290644c1771688d946c24cbce4..0c62352dda6abc9bf72dfaaaa760cc5bb78bbcbf 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -439,61 +439,22 @@ static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, return inequality ^ bit; } - -/* - * KVM wants to inject page-faults which it got to the guest. This function - * checks whether in a nested guest, we need to inject them to L1 or L2. - */ -static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual) +static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, + u32 error_code) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - unsigned int nr = vcpu->arch.exception.nr; - bool has_payload = vcpu->arch.exception.has_payload; - unsigned long payload = vcpu->arch.exception.payload; - - if (nr == PF_VECTOR) { - if (vcpu->arch.exception.nested_apf) { - *exit_qual = vcpu->arch.apf.nested_apf_token; - return 1; - } - if (nested_vmx_is_page_fault_vmexit(vmcs12, - vcpu->arch.exception.error_code)) { - *exit_qual = has_payload ? payload : vcpu->arch.cr2; - return 1; - } - } else if (vmcs12->exception_bitmap & (1u << nr)) { - if (nr == DB_VECTOR) { - if (!has_payload) { - payload = vcpu->arch.dr6; - payload &= ~DR6_BT; - payload ^= DR6_ACTIVE_LOW; - } - *exit_qual = payload; - } else - *exit_qual = 0; - return 1; - } - return 0; -} - -static bool nested_vmx_handle_page_fault_workaround(struct kvm_vcpu *vcpu, - struct x86_exception *fault) -{ - struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - - WARN_ON(!is_guest_mode(vcpu)); + /* + * Drop bits 31:16 of the error code when performing the #PF mask+match + * check. All VMCS fields involved are 32 bits, but Intel CPUs never + * set bits 31:16 and VMX disallows setting bits 31:16 in the injected + * error code. Including the to-be-dropped bits in the check might + * result in an "impossible" or missed exit from L1's perspective. + */ + if (vector == PF_VECTOR) + return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); - if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && - !WARN_ON_ONCE(to_vmx(vcpu)->nested.nested_run_pending)) { - vmcs12->vm_exit_intr_error_code = fault->error_code; - nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, - PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | - INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, - fault->address); - return true; - } - return false; + return (vmcs12->exception_bitmap & (1u << vector)); } static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, @@ -1607,6 +1568,10 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->guest_rflags = evmcs->guest_rflags; vmcs12->guest_interruptibility_info = evmcs->guest_interruptibility_info; + /* + * Not present in struct vmcs12: + * vmcs12->guest_ssp = evmcs->guest_ssp; + */ } if (unlikely(!(hv_clean_fields & @@ -1653,6 +1618,13 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->host_fs_selector = evmcs->host_fs_selector; vmcs12->host_gs_selector = evmcs->host_gs_selector; vmcs12->host_tr_selector = evmcs->host_tr_selector; + vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; + /* + * Not present in struct vmcs12: + * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; + * vmcs12->host_ssp = evmcs->host_ssp; + * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; + */ } if (unlikely(!(hv_clean_fields & @@ -1720,6 +1692,8 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->tsc_offset = evmcs->tsc_offset; vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; + vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; + vmcs12->tsc_multiplier = evmcs->tsc_multiplier; } if (unlikely(!(hv_clean_fields & @@ -1767,6 +1741,13 @@ static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; vmcs12->guest_activity_state = evmcs->guest_activity_state; vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; + vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; + /* + * Not present in struct vmcs12: + * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; + * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; + * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; + */ } /* @@ -1869,12 +1850,23 @@ static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; + * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; + * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; + * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; + * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; * * Not present in struct vmcs12: * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; + * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; + * evmcs->host_ssp = vmcs12->host_ssp; + * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; + * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; + * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; + * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; + * evmcs->guest_ssp = vmcs12->guest_ssp; */ evmcs->guest_es_selector = vmcs12->guest_es_selector; @@ -1982,7 +1974,7 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( bool evmcs_gpa_changed = false; u64 evmcs_gpa; - if (likely(!vmx->nested.enlightened_vmcs_enabled)) + if (likely(!guest_cpuid_has_evmcs(vcpu))) return EVMPTRLD_DISABLED; if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) { @@ -2328,9 +2320,14 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate * on the related bits (if supported by the CPU) in the hope that * we can avoid VMWrites during vmx_set_efer(). + * + * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is + * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to + * do the same for L2. */ exec_control = __vm_entry_controls_get(vmcs01); - exec_control |= vmcs12->vm_entry_controls; + exec_control |= (vmcs12->vm_entry_controls & + ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); if (cpu_has_load_ia32_efer()) { if (guest_efer & EFER_LMA) @@ -2570,7 +2567,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * bits which we consider mandatory enabled. * The CR0_READ_SHADOW is what L2 should have expected to read given * the specifications by L1; It's not enough to take - * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we + * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we * have more bits than L1 expected. */ vmx_set_cr0(vcpu, vmcs12->guest_cr0); @@ -2863,7 +2860,7 @@ static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, nested_check_vm_entry_controls(vcpu, vmcs12)) return -EINVAL; - if (to_vmx(vcpu)->nested.enlightened_vmcs_enabled) + if (guest_cpuid_has_evmcs(vcpu)) return nested_evmcs_check_controls(vmcs12); return 0; @@ -3145,7 +3142,7 @@ static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) * L2 was running), map it here to make sure vmcs12 changes are * properly reflected. */ - if (vmx->nested.enlightened_vmcs_enabled && + if (guest_cpuid_has_evmcs(vcpu) && vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { enum nested_evmptrld_status evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, false); @@ -3364,12 +3361,24 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, }; u32 failed_index; + trace_kvm_nested_vmenter(kvm_rip_read(vcpu), + vmx->nested.current_vmptr, + vmcs12->guest_rip, + vmcs12->guest_intr_status, + vmcs12->vm_entry_intr_info_field, + vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT, + vmcs12->ept_pointer, + vmcs12->guest_cr3, + KVM_ISA_VMX); + kvm_service_local_tlb_flush_requests(vcpu); evaluate_pending_interrupts = exec_controls_get(vmx) & (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu); + if (!evaluate_pending_interrupts) + evaluate_pending_interrupts |= kvm_apic_has_pending_init_or_sipi(vcpu); if (!vmx->nested.nested_run_pending || !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) @@ -3450,18 +3459,10 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, } /* - * If L1 had a pending IRQ/NMI until it executed - * VMLAUNCH/VMRESUME which wasn't delivered because it was - * disallowed (e.g. interrupts disabled), L0 needs to - * evaluate if this pending event should cause an exit from L2 - * to L1 or delivered directly to L2 (e.g. In case L1 don't - * intercept EXTERNAL_INTERRUPT). - * - * Usually this would be handled by the processor noticing an - * IRQ/NMI window request, or checking RVI during evaluation of - * pending virtual interrupts. However, this setting was done - * on VMCS01 and now VMCS02 is active instead. Thus, we force L0 - * to perform pending event evaluation by requesting a KVM_REQ_EVENT. + * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI + * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can + * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit + * unconditionally. */ if (unlikely(evaluate_pending_interrupts)) kvm_make_request(KVM_REQ_EVENT, vcpu); @@ -3718,7 +3719,7 @@ static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, is_double_fault(exit_intr_info))) { vmcs12->idt_vectoring_info_field = 0; } else if (vcpu->arch.exception.injected) { - nr = vcpu->arch.exception.nr; + nr = vcpu->arch.exception.vector; idt_vectoring = nr | VECTORING_INFO_VALID_MASK; if (kvm_exception_is_soft(nr)) { @@ -3819,19 +3820,40 @@ mmio_needed: return -ENXIO; } -static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, - unsigned long exit_qual) +static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) { + struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - unsigned int nr = vcpu->arch.exception.nr; - u32 intr_info = nr | INTR_INFO_VALID_MASK; + unsigned long exit_qual; + + if (ex->has_payload) { + exit_qual = ex->payload; + } else if (ex->vector == PF_VECTOR) { + exit_qual = vcpu->arch.cr2; + } else if (ex->vector == DB_VECTOR) { + exit_qual = vcpu->arch.dr6; + exit_qual &= ~DR6_BT; + exit_qual ^= DR6_ACTIVE_LOW; + } else { + exit_qual = 0; + } - if (vcpu->arch.exception.has_error_code) { - vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code; + if (ex->has_error_code) { + /* + * Intel CPUs do not generate error codes with bits 31:16 set, + * and more importantly VMX disallows setting bits 31:16 in the + * injected error code for VM-Entry. Drop the bits to mimic + * hardware and avoid inducing failure on nested VM-Entry if L1 + * chooses to inject the exception back to L2. AMD CPUs _do_ + * generate "full" 32-bit error codes, so KVM allows userspace + * to inject exception error codes with bits 31:16 set. + */ + vmcs12->vm_exit_intr_error_code = (u16)ex->error_code; intr_info |= INTR_INFO_DELIVER_CODE_MASK; } - if (kvm_exception_is_soft(nr)) + if (kvm_exception_is_soft(ex->vector)) intr_info |= INTR_TYPE_SOFT_EXCEPTION; else intr_info |= INTR_TYPE_HARD_EXCEPTION; @@ -3844,16 +3866,39 @@ static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, } /* - * Returns true if a debug trap is pending delivery. + * Returns true if a debug trap is (likely) pending delivery. Infer the class + * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6). + * Using the payload is flawed because code breakpoints (fault-like) and data + * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e. + * this will return false positives if a to-be-injected code breakpoint #DB is + * pending (from KVM's perspective, but not "pending" across an instruction + * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it + * too is trap-like. * - * In KVM, debug traps bear an exception payload. As such, the class of a #DB - * exception may be inferred from the presence of an exception payload. + * KVM "works" despite these flaws as ICEBP isn't currently supported by the + * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the + * #DB has already happened), and MTF isn't marked pending on code breakpoints + * from the emulator (because such #DBs are fault-like and thus don't trigger + * actions that fire on instruction retire). + */ +static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex) +{ + if (!ex->pending || ex->vector != DB_VECTOR) + return 0; + + /* General Detect #DBs are always fault-like. */ + return ex->payload & ~DR6_BD; +} + +/* + * Returns true if there's a pending #DB exception that is lower priority than + * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by + * KVM, but could theoretically be injected by userspace. Note, this code is + * imperfect, see above. */ -static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu) +static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex) { - return vcpu->arch.exception.pending && - vcpu->arch.exception.nr == DB_VECTOR && - vcpu->arch.exception.payload; + return vmx_get_pending_dbg_trap(ex) & ~DR6_BT; } /* @@ -3865,9 +3910,11 @@ static inline bool vmx_pending_dbg_trap(struct kvm_vcpu *vcpu) */ static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) { - if (vmx_pending_dbg_trap(vcpu)) - vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, - vcpu->arch.exception.payload); + unsigned long pending_dbg; + + pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception); + if (pending_dbg) + vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg); } static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) @@ -3876,21 +3923,113 @@ static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) to_vmx(vcpu)->nested.preemption_timer_expired; } +static bool vmx_has_nested_events(struct kvm_vcpu *vcpu) +{ + return nested_vmx_preemption_timer_pending(vcpu) || + to_vmx(vcpu)->nested.mtf_pending; +} + +/* + * Per the Intel SDM's table "Priority Among Concurrent Events", with minor + * edits to fill in missing examples, e.g. #DB due to split-lock accesses, + * and less minor edits to splice in the priority of VMX Non-Root specific + * events, e.g. MTF and NMI/INTR-window exiting. + * + * 1 Hardware Reset and Machine Checks + * - RESET + * - Machine Check + * + * 2 Trap on Task Switch + * - T flag in TSS is set (on task switch) + * + * 3 External Hardware Interventions + * - FLUSH + * - STOPCLK + * - SMI + * - INIT + * + * 3.5 Monitor Trap Flag (MTF) VM-exit[1] + * + * 4 Traps on Previous Instruction + * - Breakpoints + * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O + * breakpoint, or #DB due to a split-lock access) + * + * 4.3 VMX-preemption timer expired VM-exit + * + * 4.6 NMI-window exiting VM-exit[2] + * + * 5 Nonmaskable Interrupts (NMI) + * + * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery + * + * 6 Maskable Hardware Interrupts + * + * 7 Code Breakpoint Fault + * + * 8 Faults from Fetching Next Instruction + * - Code-Segment Limit Violation + * - Code Page Fault + * - Control protection exception (missing ENDBRANCH at target of indirect + * call or jump) + * + * 9 Faults from Decoding Next Instruction + * - Instruction length > 15 bytes + * - Invalid Opcode + * - Coprocessor Not Available + * + *10 Faults on Executing Instruction + * - Overflow + * - Bound error + * - Invalid TSS + * - Segment Not Present + * - Stack fault + * - General Protection + * - Data Page Fault + * - Alignment Check + * - x86 FPU Floating-point exception + * - SIMD floating-point exception + * - Virtualization exception + * - Control protection exception + * + * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs), + * INIT signals, and higher priority events take priority over MTF VM exits. + * MTF VM exits take priority over debug-trap exceptions and lower priority + * events. + * + * [2] Debug-trap exceptions and higher priority events take priority over VM exits + * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption + * timer take priority over VM exits caused by the "NMI-window exiting" + * VM-execution control and lower priority events. + * + * [3] Debug-trap exceptions and higher priority events take priority over VM exits + * caused by "NMI-window exiting". VM exits caused by this control take + * priority over non-maskable interrupts (NMIs) and lower priority events. + * + * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to + * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus, + * non-maskable interrupts (NMIs) and higher priority events take priority over + * delivery of a virtual interrupt; delivery of a virtual interrupt takes + * priority over external interrupts and lower priority events. + */ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - unsigned long exit_qual; - bool block_nested_events = - vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu); - bool mtf_pending = vmx->nested.mtf_pending; struct kvm_lapic *apic = vcpu->arch.apic; - + struct vcpu_vmx *vmx = to_vmx(vcpu); /* - * Clear the MTF state. If a higher priority VM-exit is delivered first, - * this state is discarded. + * Only a pending nested run blocks a pending exception. If there is a + * previously injected event, the pending exception occurred while said + * event was being delivered and thus needs to be handled. */ - if (!block_nested_events) - vmx->nested.mtf_pending = false; + bool block_nested_exceptions = vmx->nested.nested_run_pending; + /* + * New events (not exceptions) are only recognized at instruction + * boundaries. If an event needs reinjection, then KVM is handling a + * VM-Exit that occurred _during_ instruction execution; new events are + * blocked until the instruction completes. + */ + bool block_nested_events = block_nested_exceptions || + kvm_event_needs_reinjection(vcpu); if (lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &apic->pending_events)) { @@ -3900,6 +4039,9 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) clear_bit(KVM_APIC_INIT, &apic->pending_events); if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); + + /* MTF is discarded if the vCPU is in WFS. */ + vmx->nested.mtf_pending = false; return 0; } @@ -3909,31 +4051,41 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return -EBUSY; clear_bit(KVM_APIC_SIPI, &apic->pending_events); - if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) + if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, apic->sipi_vector & 0xFFUL); - return 0; + return 0; + } + /* Fallthrough, the SIPI is completely ignored. */ } /* - * Process any exceptions that are not debug traps before MTF. + * Process exceptions that are higher priority than Monitor Trap Flag: + * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but + * could theoretically come in from userspace), and ICEBP (INT1). * - * Note that only a pending nested run can block a pending exception. - * Otherwise an injected NMI/interrupt should either be - * lost or delivered to the nested hypervisor in the IDT_VECTORING_INFO, - * while delivering the pending exception. + * TODO: SMIs have higher priority than MTF and trap-like #DBs (except + * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF + * across SMI/RSM as it should; that needs to be addressed in order to + * prioritize SMI over MTF and trap-like #DBs. */ - - if (vcpu->arch.exception.pending && !vmx_pending_dbg_trap(vcpu)) { - if (vmx->nested.nested_run_pending) + if (vcpu->arch.exception_vmexit.pending && + !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) { + if (block_nested_exceptions) return -EBUSY; - if (!nested_vmx_check_exception(vcpu, &exit_qual)) - goto no_vmexit; - nested_vmx_inject_exception_vmexit(vcpu, exit_qual); + + nested_vmx_inject_exception_vmexit(vcpu); return 0; } - if (mtf_pending) { + if (vcpu->arch.exception.pending && + !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) { + if (block_nested_exceptions) + return -EBUSY; + goto no_vmexit; + } + + if (vmx->nested.mtf_pending) { if (block_nested_events) return -EBUSY; nested_vmx_update_pending_dbg(vcpu); @@ -3941,15 +4093,20 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (vcpu->arch.exception.pending) { - if (vmx->nested.nested_run_pending) + if (vcpu->arch.exception_vmexit.pending) { + if (block_nested_exceptions) return -EBUSY; - if (!nested_vmx_check_exception(vcpu, &exit_qual)) - goto no_vmexit; - nested_vmx_inject_exception_vmexit(vcpu, exit_qual); + + nested_vmx_inject_exception_vmexit(vcpu); return 0; } + if (vcpu->arch.exception.pending) { + if (block_nested_exceptions) + return -EBUSY; + goto no_vmexit; + } + if (nested_vmx_preemption_timer_pending(vcpu)) { if (block_nested_events) return -EBUSY; @@ -4255,14 +4412,6 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL); } - - /* - * Drop what we picked up for L2 via vmx_complete_interrupts. It is - * preserved above and would only end up incorrectly in L1. - */ - vcpu->arch.nmi_injected = false; - kvm_clear_exception_queue(vcpu); - kvm_clear_interrupt_queue(vcpu); } /* @@ -4538,6 +4687,9 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + /* Pending MTF traps are discarded on VM-Exit. */ + vmx->nested.mtf_pending = false; + /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); @@ -4602,6 +4754,17 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, WARN_ON_ONCE(nested_early_check); } + /* + * Drop events/exceptions that were queued for re-injection to L2 + * (picked up via vmx_complete_interrupts()), as well as exceptions + * that were pending for L2. Note, this must NOT be hoisted above + * prepare_vmcs12(), events/exceptions queued for re-injection need to + * be captured in vmcs12 (see vmcs12_save_pending_event()). + */ + vcpu->arch.nmi_injected = false; + kvm_clear_exception_queue(vcpu); + kvm_clear_interrupt_queue(vcpu); + vmx_switch_vmcs(vcpu, &vmx->vmcs01); /* Update any VMCS fields that might have changed while L2 ran */ @@ -5030,8 +5193,8 @@ static int handle_vmxoff(struct kvm_vcpu *vcpu) free_nested(vcpu); - /* Process a latched INIT during time CPU was in VMX operation */ - kvm_make_request(KVM_REQ_EVENT, vcpu); + if (kvm_apic_has_pending_init_or_sipi(vcpu)) + kvm_make_request(KVM_REQ_EVENT, vcpu); return nested_vmx_succeed(vcpu); } @@ -5067,7 +5230,7 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) * state. It is possible that the area will stay mapped as * vmx->nested.hv_evmcs but this shouldn't be a problem. */ - if (likely(!vmx->nested.enlightened_vmcs_enabled || + if (likely(!guest_cpuid_has_evmcs(vcpu) || !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) { if (vmptr == vmx->nested.current_vmptr) nested_release_vmcs12(vcpu); @@ -6463,6 +6626,9 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, if (ret) goto error_guest_mode; + if (vmx->nested.mtf_pending) + kvm_make_request(KVM_REQ_EVENT, vcpu); + return 0; error_guest_mode: @@ -6522,8 +6688,10 @@ static u64 nested_vmx_calc_vmcs_enum_msr(void) * bit in the high half is on if the corresponding bit in the control field * may be on. See also vmx_control_verify(). */ -void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) +void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps) { + struct nested_vmx_msrs *msrs = &vmcs_conf->nested; + /* * Note that as a general rule, the high half of the MSRs (bits in * the control fields which may be 1) should be initialized by the @@ -6540,11 +6708,10 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) */ /* pin-based controls */ - rdmsr(MSR_IA32_VMX_PINBASED_CTLS, - msrs->pinbased_ctls_low, - msrs->pinbased_ctls_high); - msrs->pinbased_ctls_low |= + msrs->pinbased_ctls_low = PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; + + msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl; msrs->pinbased_ctls_high &= PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING | @@ -6555,50 +6722,47 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) PIN_BASED_VMX_PREEMPTION_TIMER; /* exit controls */ - rdmsr(MSR_IA32_VMX_EXIT_CTLS, - msrs->exit_ctls_low, - msrs->exit_ctls_high); msrs->exit_ctls_low = VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; + msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl; msrs->exit_ctls_high &= #ifdef CONFIG_X86_64 VM_EXIT_HOST_ADDR_SPACE_SIZE | #endif VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | - VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; + VM_EXIT_CLEAR_BNDCFGS; msrs->exit_ctls_high |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | - VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT; + VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | + VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; /* We support free control of debug control saving. */ msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; /* entry controls */ - rdmsr(MSR_IA32_VMX_ENTRY_CTLS, - msrs->entry_ctls_low, - msrs->entry_ctls_high); msrs->entry_ctls_low = VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; + + msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl; msrs->entry_ctls_high &= #ifdef CONFIG_X86_64 VM_ENTRY_IA32E_MODE | #endif - VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS | - VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; + VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS; msrs->entry_ctls_high |= - (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER); + (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER | + VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); /* We support free control of debug control loading. */ msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; /* cpu-based controls */ - rdmsr(MSR_IA32_VMX_PROCBASED_CTLS, - msrs->procbased_ctls_low, - msrs->procbased_ctls_high); msrs->procbased_ctls_low = CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; + + msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl; msrs->procbased_ctls_high &= CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | @@ -6632,12 +6796,9 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) * depend on CPUID bits, they are added later by * vmx_vcpu_after_set_cpuid. */ - if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) - rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2, - msrs->secondary_ctls_low, - msrs->secondary_ctls_high); - msrs->secondary_ctls_low = 0; + + msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl; msrs->secondary_ctls_high &= SECONDARY_EXEC_DESC | SECONDARY_EXEC_ENABLE_RDTSCP | @@ -6717,10 +6878,7 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; /* miscellaneous data */ - rdmsr(MSR_IA32_VMX_MISC, - msrs->misc_low, - msrs->misc_high); - msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA; + msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA; msrs->misc_low |= MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | @@ -6814,9 +6972,9 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) struct kvm_x86_nested_ops vmx_nested_ops = { .leave_nested = vmx_leave_nested, + .is_exception_vmexit = nested_vmx_is_exception_vmexit, .check_events = vmx_check_nested_events, - .handle_page_fault_workaround = nested_vmx_handle_page_fault_workaround, - .hv_timer_pending = nested_vmx_preemption_timer_pending, + .has_events = vmx_has_nested_events, .triple_fault = nested_vmx_triple_fault, .get_state = vmx_get_nested_state, .set_state = vmx_set_nested_state, diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 88b00a7359e4035835630a9f2b1f36e202a1fb64..6312c9541c3cd3bc324fdf630099fed1a1134be9 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -17,7 +17,7 @@ enum nvmx_vmentry_status { }; void vmx_leave_nested(struct kvm_vcpu *vcpu); -void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps); +void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps); void nested_vmx_hardware_unsetup(void); __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)); void nested_vmx_set_vmcs_shadowing_bitmap(void); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index c399637a3a79bf399529b9a4069a9117a9b817fb..25b70a85bef54c775afe3f8094087397ca536e85 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -68,15 +68,11 @@ static struct kvm_pmc *intel_pmc_idx_to_pmc(struct kvm_pmu *pmu, int pmc_idx) } } -/* function is called when global control register has been updated. */ -static void global_ctrl_changed(struct kvm_pmu *pmu, u64 data) +static void reprogram_counters(struct kvm_pmu *pmu, u64 diff) { int bit; - u64 diff = pmu->global_ctrl ^ data; struct kvm_pmc *pmc; - pmu->global_ctrl = data; - for_each_set_bit(bit, (unsigned long *)&diff, X86_PMC_IDX_MAX) { pmc = intel_pmc_idx_to_pmc(pmu, bit); if (pmc) @@ -397,7 +393,7 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) struct kvm_pmc *pmc; u32 msr = msr_info->index; u64 data = msr_info->data; - u64 reserved_bits; + u64 reserved_bits, diff; switch (msr) { case MSR_CORE_PERF_FIXED_CTR_CTRL: @@ -418,7 +414,9 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (pmu->global_ctrl == data) return 0; if (kvm_valid_perf_global_ctrl(pmu, data)) { - global_ctrl_changed(pmu, data); + diff = pmu->global_ctrl ^ data; + pmu->global_ctrl = data; + reprogram_counters(pmu, diff); return 0; } break; @@ -433,7 +431,9 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (pmu->pebs_enable == data) return 0; if (!(data & pmu->pebs_enable_mask)) { + diff = pmu->pebs_enable ^ data; pmu->pebs_enable = data; + reprogram_counters(pmu, diff); return 0; } break; @@ -776,20 +776,23 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu) { struct kvm_pmc *pmc = NULL; - int bit; + int bit, hw_idx; for_each_set_bit(bit, (unsigned long *)&pmu->global_ctrl, X86_PMC_IDX_MAX) { pmc = intel_pmc_idx_to_pmc(pmu, bit); if (!pmc || !pmc_speculative_in_use(pmc) || - !intel_pmc_is_enabled(pmc)) + !intel_pmc_is_enabled(pmc) || !pmc->perf_event) continue; - if (pmc->perf_event && pmc->idx != pmc->perf_event->hw.idx) { - pmu->host_cross_mapped_mask |= - BIT_ULL(pmc->perf_event->hw.idx); - } + /* + * A negative index indicates the event isn't mapped to a + * physical counter in the host, e.g. due to contention. + */ + hw_idx = pmc->perf_event->hw.idx; + if (hw_idx != pmc->idx && hw_idx > -1) + pmu->host_cross_mapped_mask |= BIT_ULL(hw_idx); } } diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c index aba8cebdc587fc271d11689ee9440b423557aecc..8f95c7c0143359cc617641eddbf422018d77cf63 100644 --- a/arch/x86/kvm/vmx/sgx.c +++ b/arch/x86/kvm/vmx/sgx.c @@ -129,7 +129,7 @@ static int sgx_inject_fault(struct kvm_vcpu *vcpu, gva_t gva, int trapnr) ex.address = gva; ex.error_code_valid = true; ex.nested_page_fault = false; - kvm_inject_page_fault(vcpu, &ex); + kvm_inject_emulated_page_fault(vcpu, &ex); } else { kvm_inject_gp(vcpu, 0); } diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 6de96b9438044e52dab2482a5e7023dbd2b2e811..8477d8bdd69c262c8feeb79569f6a68f0ea65833 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -189,13 +189,16 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) xor %ebx, %ebx .Lclear_regs: + /* Discard @regs. The register is irrelevant, it just can't be RBX. */ + pop %_ASM_AX + /* * Clear all general purpose registers except RSP and RBX to prevent * speculative use of the guest's values, even those that are reloaded * via the stack. In theory, an L1 cache miss when restoring registers * could lead to speculative execution with the guest's values. * Zeroing XORs are dirt cheap, i.e. the extra paranoia is essentially - * free. RSP and RAX are exempt as RSP is restored by hardware during + * free. RSP and RBX are exempt as RSP is restored by hardware during * VM-Exit and RBX is explicitly loaded with 0 or 1 to hold the return * value. */ @@ -216,9 +219,6 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) xor %r15d, %r15d #endif - /* "POP" @regs. */ - add $WORD_SIZE, %_ASM_SP - /* * IMPORTANT: RSB filling and SPEC_CTRL handling must be done before * the first unbalanced RET after vmexit! @@ -234,7 +234,6 @@ SYM_INNER_LABEL(vmx_vmexit, SYM_L_GLOBAL) FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\ X86_FEATURE_RSB_VMEXIT_LITE - pop %_ASM_ARG2 /* @flags */ pop %_ASM_ARG1 /* @vmx */ @@ -293,22 +292,13 @@ SYM_FUNC_START(vmread_error_trampoline) push %r10 push %r11 #endif -#ifdef CONFIG_X86_64 + /* Load @field and @fault to arg1 and arg2 respectively. */ - mov 3*WORD_SIZE(%rbp), %_ASM_ARG2 - mov 2*WORD_SIZE(%rbp), %_ASM_ARG1 -#else - /* Parameters are passed on the stack for 32-bit (see asmlinkage). */ - push 3*WORD_SIZE(%ebp) - push 2*WORD_SIZE(%ebp) -#endif + mov 3*WORD_SIZE(%_ASM_BP), %_ASM_ARG2 + mov 2*WORD_SIZE(%_ASM_BP), %_ASM_ARG1 call vmread_error -#ifndef CONFIG_X86_64 - add $8, %esp -#endif - /* Zero out @fault, which will be popped into the result register. */ _ASM_MOV $0, 3*WORD_SIZE(%_ASM_BP) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d7f8331d6f7e72c0b90d3b6596b1374bd69b374c..9dba04b6b019aca4ec263395a85df9a9d494a033 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -439,7 +439,7 @@ do { \ pr_warn_ratelimited(fmt); \ } while (0) -asmlinkage void vmread_error(unsigned long field, bool fault) +void vmread_error(unsigned long field, bool fault) { if (fault) kvm_spurious_fault(); @@ -843,8 +843,7 @@ static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr) if (!(exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS)) return true; - return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, - MSR_IA32_SPEC_CTRL); + return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr); } unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) @@ -865,7 +864,7 @@ unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx) return flags; } -static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, +static __always_inline void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx, unsigned long entry, unsigned long exit) { vm_entry_controls_clearbit(vmx, entry); @@ -923,7 +922,7 @@ skip_guest: vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr); } -static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, +static __always_inline void add_atomic_switch_msr_special(struct vcpu_vmx *vmx, unsigned long entry, unsigned long exit, unsigned long guest_val_vmcs, unsigned long host_val_vmcs, u64 guest_val, u64 host_val) @@ -1653,17 +1652,25 @@ static void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu) /* * Per the SDM, MTF takes priority over debug-trap exceptions besides - * T-bit traps. As instruction emulation is completed (i.e. at the - * instruction boundary), any #DB exception pending delivery must be a - * debug-trap. Record the pending MTF state to be delivered in + * TSS T-bit traps and ICEBP (INT1). KVM doesn't emulate T-bit traps + * or ICEBP (in the emulator proper), and skipping of ICEBP after an + * intercepted #DB deliberately avoids single-step #DB and MTF updates + * as ICEBP is higher priority than both. As instruction emulation is + * completed at this point (i.e. KVM is at the instruction boundary), + * any #DB exception pending delivery must be a debug-trap of lower + * priority than MTF. Record the pending MTF state to be delivered in * vmx_check_nested_events(). */ if (nested_cpu_has_mtf(vmcs12) && (!vcpu->arch.exception.pending || - vcpu->arch.exception.nr == DB_VECTOR)) + vcpu->arch.exception.vector == DB_VECTOR) && + (!vcpu->arch.exception_vmexit.pending || + vcpu->arch.exception_vmexit.vector == DB_VECTOR)) { vmx->nested.mtf_pending = true; - else + kvm_make_request(KVM_REQ_EVENT, vcpu); + } else { vmx->nested.mtf_pending = false; + } } static int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) @@ -1685,32 +1692,40 @@ static void vmx_clear_hlt(struct kvm_vcpu *vcpu) vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE); } -static void vmx_queue_exception(struct kvm_vcpu *vcpu) +static void vmx_inject_exception(struct kvm_vcpu *vcpu) { + struct kvm_queued_exception *ex = &vcpu->arch.exception; + u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; struct vcpu_vmx *vmx = to_vmx(vcpu); - unsigned nr = vcpu->arch.exception.nr; - bool has_error_code = vcpu->arch.exception.has_error_code; - u32 error_code = vcpu->arch.exception.error_code; - u32 intr_info = nr | INTR_INFO_VALID_MASK; - kvm_deliver_exception_payload(vcpu); + kvm_deliver_exception_payload(vcpu, ex); - if (has_error_code) { - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); + if (ex->has_error_code) { + /* + * Despite the error code being architecturally defined as 32 + * bits, and the VMCS field being 32 bits, Intel CPUs and thus + * VMX don't actually supporting setting bits 31:16. Hardware + * will (should) never provide a bogus error code, but AMD CPUs + * do generate error codes with bits 31:16 set, and so KVM's + * ABI lets userspace shove in arbitrary 32-bit values. Drop + * the upper bits to avoid VM-Fail, losing information that + * does't really exist is preferable to killing the VM. + */ + vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, (u16)ex->error_code); intr_info |= INTR_INFO_DELIVER_CODE_MASK; } if (vmx->rmode.vm86_active) { int inc_eip = 0; - if (kvm_exception_is_soft(nr)) + if (kvm_exception_is_soft(ex->vector)) inc_eip = vcpu->arch.event_exit_inst_len; - kvm_inject_realmode_interrupt(vcpu, nr, inc_eip); + kvm_inject_realmode_interrupt(vcpu, ex->vector, inc_eip); return; } WARN_ON_ONCE(vmx->emulation_required); - if (kvm_exception_is_soft(nr)) { + if (kvm_exception_is_soft(ex->vector)) { vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, vmx->vcpu.arch.event_exit_inst_len); intr_info |= INTR_TYPE_SOFT_EXCEPTION; @@ -1931,9 +1946,8 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) * sanity checking and refuse to boot. Filter all unsupported * features out. */ - if (!msr_info->host_initiated && - vmx->nested.enlightened_vmcs_enabled) - nested_evmcs_filter_control_msr(msr_info->index, + if (!msr_info->host_initiated && guest_cpuid_has_evmcs(vcpu)) + nested_evmcs_filter_control_msr(vcpu, msr_info->index, &msr_info->data); break; case MSR_IA32_RTIT_CTL: @@ -2495,6 +2509,30 @@ static bool cpu_has_sgx(void) return cpuid_eax(0) >= 0x12 && (cpuid_eax(0x12) & BIT(0)); } +/* + * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they + * can't be used due to errata where VM Exit may incorrectly clear + * IA32_PERF_GLOBAL_CTRL[34:32]. Work around the errata by using the + * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. + */ +static bool cpu_has_perf_global_ctrl_bug(void) +{ + if (boot_cpu_data.x86 == 0x6) { + switch (boot_cpu_data.x86_model) { + case INTEL_FAM6_NEHALEM_EP: /* AAK155 */ + case INTEL_FAM6_NEHALEM: /* AAP115 */ + case INTEL_FAM6_WESTMERE: /* AAT100 */ + case INTEL_FAM6_WESTMERE_EP: /* BC86,AAY89,BD102 */ + case INTEL_FAM6_NEHALEM_EX: /* BA97 */ + return true; + default: + break; + } + } + + return false; +} + static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, u32 msr, u32 *result) { @@ -2527,13 +2565,13 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, struct vmx_capability *vmx_cap) { u32 vmx_msr_low, vmx_msr_high; - u32 min, opt, min2, opt2; u32 _pin_based_exec_control = 0; u32 _cpu_based_exec_control = 0; u32 _cpu_based_2nd_exec_control = 0; u64 _cpu_based_3rd_exec_control = 0; u32 _vmexit_control = 0; u32 _vmentry_control = 0; + u64 misc_msr; int i; /* @@ -2553,64 +2591,17 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, }; memset(vmcs_conf, 0, sizeof(*vmcs_conf)); - min = CPU_BASED_HLT_EXITING | -#ifdef CONFIG_X86_64 - CPU_BASED_CR8_LOAD_EXITING | - CPU_BASED_CR8_STORE_EXITING | -#endif - CPU_BASED_CR3_LOAD_EXITING | - CPU_BASED_CR3_STORE_EXITING | - CPU_BASED_UNCOND_IO_EXITING | - CPU_BASED_MOV_DR_EXITING | - CPU_BASED_USE_TSC_OFFSETTING | - CPU_BASED_MWAIT_EXITING | - CPU_BASED_MONITOR_EXITING | - CPU_BASED_INVLPG_EXITING | - CPU_BASED_RDPMC_EXITING; - - opt = CPU_BASED_TPR_SHADOW | - CPU_BASED_USE_MSR_BITMAPS | - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS | - CPU_BASED_ACTIVATE_TERTIARY_CONTROLS; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, - &_cpu_based_exec_control) < 0) + + if (adjust_vmx_controls(KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL, + KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL, + MSR_IA32_VMX_PROCBASED_CTLS, + &_cpu_based_exec_control)) return -EIO; -#ifdef CONFIG_X86_64 - if (_cpu_based_exec_control & CPU_BASED_TPR_SHADOW) - _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & - ~CPU_BASED_CR8_STORE_EXITING; -#endif if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { - min2 = 0; - opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | - SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | - SECONDARY_EXEC_WBINVD_EXITING | - SECONDARY_EXEC_ENABLE_VPID | - SECONDARY_EXEC_ENABLE_EPT | - SECONDARY_EXEC_UNRESTRICTED_GUEST | - SECONDARY_EXEC_PAUSE_LOOP_EXITING | - SECONDARY_EXEC_DESC | - SECONDARY_EXEC_ENABLE_RDTSCP | - SECONDARY_EXEC_ENABLE_INVPCID | - SECONDARY_EXEC_APIC_REGISTER_VIRT | - SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | - SECONDARY_EXEC_SHADOW_VMCS | - SECONDARY_EXEC_XSAVES | - SECONDARY_EXEC_RDSEED_EXITING | - SECONDARY_EXEC_RDRAND_EXITING | - SECONDARY_EXEC_ENABLE_PML | - SECONDARY_EXEC_TSC_SCALING | - SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | - SECONDARY_EXEC_PT_USE_GPA | - SECONDARY_EXEC_PT_CONCEAL_VMX | - SECONDARY_EXEC_ENABLE_VMFUNC | - SECONDARY_EXEC_BUS_LOCK_DETECTION | - SECONDARY_EXEC_NOTIFY_VM_EXITING; - if (cpu_has_sgx()) - opt2 |= SECONDARY_EXEC_ENCLS_EXITING; - if (adjust_vmx_controls(min2, opt2, + if (adjust_vmx_controls(KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL, + KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL, MSR_IA32_VMX_PROCBASED_CTLS2, - &_cpu_based_2nd_exec_control) < 0) + &_cpu_based_2nd_exec_control)) return -EIO; } #ifndef CONFIG_X86_64 @@ -2628,13 +2619,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP, &vmx_cap->ept, &vmx_cap->vpid); - if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) { - /* CR3 accesses and invlpg don't need to cause VM Exits when EPT - enabled */ - _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | - CPU_BASED_CR3_STORE_EXITING | - CPU_BASED_INVLPG_EXITING); - } else if (vmx_cap->ept) { + if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) && + vmx_cap->ept) { pr_warn_once("EPT CAP should not exist if not support " "1-setting enable EPT VM-execution control\n"); @@ -2654,32 +2640,24 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, vmx_cap->vpid = 0; } - if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) { - u64 opt3 = TERTIARY_EXEC_IPI_VIRT; + if (!cpu_has_sgx()) + _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; - _cpu_based_3rd_exec_control = adjust_vmx_controls64(opt3, + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) + _cpu_based_3rd_exec_control = + adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, MSR_IA32_VMX_PROCBASED_CTLS3); - } - min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT; -#ifdef CONFIG_X86_64 - min |= VM_EXIT_HOST_ADDR_SPACE_SIZE; -#endif - opt = VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | - VM_EXIT_LOAD_IA32_PAT | - VM_EXIT_LOAD_IA32_EFER | - VM_EXIT_CLEAR_BNDCFGS | - VM_EXIT_PT_CONCEAL_PIP | - VM_EXIT_CLEAR_IA32_RTIT_CTL; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS, - &_vmexit_control) < 0) + if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_EXIT_CONTROLS, + KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS, + MSR_IA32_VMX_EXIT_CTLS, + &_vmexit_control)) return -EIO; - min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING; - opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR | - PIN_BASED_VMX_PREEMPTION_TIMER; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS, - &_pin_based_exec_control) < 0) + if (adjust_vmx_controls(KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL, + KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL, + MSR_IA32_VMX_PINBASED_CTLS, + &_pin_based_exec_control)) return -EIO; if (cpu_has_broken_vmx_preemption_timer()) @@ -2688,15 +2666,10 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)) _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR; - min = VM_ENTRY_LOAD_DEBUG_CONTROLS; - opt = VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | - VM_ENTRY_LOAD_IA32_PAT | - VM_ENTRY_LOAD_IA32_EFER | - VM_ENTRY_LOAD_BNDCFGS | - VM_ENTRY_PT_CONCEAL_PIP | - VM_ENTRY_LOAD_IA32_RTIT_CTL; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS, - &_vmentry_control) < 0) + if (adjust_vmx_controls(KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS, + KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS, + MSR_IA32_VMX_ENTRY_CTLS, + &_vmentry_control)) return -EIO; for (i = 0; i < ARRAY_SIZE(vmcs_entry_exit_pairs); i++) { @@ -2716,30 +2689,6 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, _vmexit_control &= ~x_ctrl; } - /* - * Some cpus support VM_{ENTRY,EXIT}_IA32_PERF_GLOBAL_CTRL but they - * can't be used due to an errata where VM Exit may incorrectly clear - * IA32_PERF_GLOBAL_CTRL[34:32]. Workaround the errata by using the - * MSR load mechanism to switch IA32_PERF_GLOBAL_CTRL. - */ - if (boot_cpu_data.x86 == 0x6) { - switch (boot_cpu_data.x86_model) { - case 26: /* AAK155 */ - case 30: /* AAP115 */ - case 37: /* AAT100 */ - case 44: /* BC86,AAY89,BD102 */ - case 46: /* BA97 */ - _vmentry_control &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; - _vmexit_control &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; - pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " - "does not work properly. Using workaround\n"); - break; - default: - break; - } - } - - rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ @@ -2756,6 +2705,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (((vmx_msr_high >> 18) & 15) != 6) return -EIO; + rdmsrl(MSR_IA32_VMX_MISC, misc_msr); + vmcs_conf->size = vmx_msr_high & 0x1fff; vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff; @@ -2767,11 +2718,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, vmcs_conf->cpu_based_3rd_exec_ctrl = _cpu_based_3rd_exec_control; vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; - -#if IS_ENABLED(CONFIG_HYPERV) - if (enlightened_vmcs) - evmcs_sanitize_exec_ctrls(vmcs_conf); -#endif + vmcs_conf->misc = misc_msr; return 0; } @@ -3038,10 +2985,15 @@ int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) return 0; vcpu->arch.efer = efer; +#ifdef CONFIG_X86_64 if (efer & EFER_LMA) vm_entry_controls_setbit(vmx, VM_ENTRY_IA32E_MODE); else vm_entry_controls_clearbit(vmx, VM_ENTRY_IA32E_MODE); +#else + if (KVM_BUG_ON(efer & EFER_LMA, vcpu->kvm)) + return 1; +#endif vmx_setup_uret_msrs(vmx); return 0; @@ -4328,18 +4280,37 @@ static u32 vmx_vmentry_ctrl(void) if (vmx_pt_mode_is_system()) vmentry_ctrl &= ~(VM_ENTRY_PT_CONCEAL_PIP | VM_ENTRY_LOAD_IA32_RTIT_CTL); - /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ - return vmentry_ctrl & - ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | VM_ENTRY_LOAD_IA32_EFER); + /* + * IA32e mode, and loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically. + */ + vmentry_ctrl &= ~(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | + VM_ENTRY_LOAD_IA32_EFER | + VM_ENTRY_IA32E_MODE); + + if (cpu_has_perf_global_ctrl_bug()) + vmentry_ctrl &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL; + + return vmentry_ctrl; } static u32 vmx_vmexit_ctrl(void) { u32 vmexit_ctrl = vmcs_config.vmexit_ctrl; + /* + * Not used by KVM and never set in vmcs01 or vmcs02, but emulated for + * nested virtualization and thus allowed to be set in vmcs12. + */ + vmexit_ctrl &= ~(VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER | + VM_EXIT_SAVE_VMX_PREEMPTION_TIMER); + if (vmx_pt_mode_is_system()) vmexit_ctrl &= ~(VM_EXIT_PT_CONCEAL_PIP | VM_EXIT_CLEAR_IA32_RTIT_CTL); + + if (cpu_has_perf_global_ctrl_bug()) + vmexit_ctrl &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; + /* Loading of EFER and PERF_GLOBAL_CTRL are toggled dynamically */ return vmexit_ctrl & ~(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | VM_EXIT_LOAD_IA32_EFER); @@ -4377,20 +4348,38 @@ static u32 vmx_exec_control(struct vcpu_vmx *vmx) { u32 exec_control = vmcs_config.cpu_based_exec_ctrl; + /* + * Not used by KVM, but fully supported for nesting, i.e. are allowed in + * vmcs12 and propagated to vmcs02 when set in vmcs12. + */ + exec_control &= ~(CPU_BASED_RDTSC_EXITING | + CPU_BASED_USE_IO_BITMAPS | + CPU_BASED_MONITOR_TRAP_FLAG | + CPU_BASED_PAUSE_EXITING); + + /* INTR_WINDOW_EXITING and NMI_WINDOW_EXITING are toggled dynamically */ + exec_control &= ~(CPU_BASED_INTR_WINDOW_EXITING | + CPU_BASED_NMI_WINDOW_EXITING); + if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT) exec_control &= ~CPU_BASED_MOV_DR_EXITING; - if (!cpu_need_tpr_shadow(&vmx->vcpu)) { + if (!cpu_need_tpr_shadow(&vmx->vcpu)) exec_control &= ~CPU_BASED_TPR_SHADOW; + #ifdef CONFIG_X86_64 + if (exec_control & CPU_BASED_TPR_SHADOW) + exec_control &= ~(CPU_BASED_CR8_LOAD_EXITING | + CPU_BASED_CR8_STORE_EXITING); + else exec_control |= CPU_BASED_CR8_STORE_EXITING | CPU_BASED_CR8_LOAD_EXITING; #endif - } - if (!enable_ept) - exec_control |= CPU_BASED_CR3_STORE_EXITING | - CPU_BASED_CR3_LOAD_EXITING | - CPU_BASED_INVLPG_EXITING; + /* No need to intercept CR3 access or INVPLG when using EPT. */ + if (enable_ept) + exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING | + CPU_BASED_CR3_STORE_EXITING | + CPU_BASED_INVLPG_EXITING); if (kvm_mwait_in_guest(vmx->vcpu.kvm)) exec_control &= ~(CPU_BASED_MWAIT_EXITING | CPU_BASED_MONITOR_EXITING); @@ -5156,8 +5145,10 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) * instruction. ICEBP generates a trap-like #DB, but * despite its interception control being tied to #DB, * is an instruction intercept, i.e. the VM-Exit occurs - * on the ICEBP itself. Note, skipping ICEBP also - * clears STI and MOVSS blocking. + * on the ICEBP itself. Use the inner "skip" helper to + * avoid single-step #DB and MTF updates, as ICEBP is + * higher priority. Note, skipping ICEBP still clears + * STI and MOVSS blocking. * * For all other #DBs, set vmcs.PENDING_DBG_EXCEPTIONS.BS * if single-step is enabled in RFLAGS and STI or MOVSS @@ -5639,7 +5630,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); - trace_kvm_page_fault(gpa, exit_qualification); + trace_kvm_page_fault(vcpu, gpa, exit_qualification); /* Is it a read fault? */ error_code = (exit_qualification & EPT_VIOLATION_ACC_READ) @@ -5711,7 +5702,7 @@ static bool vmx_emulation_required_with_pending_exception(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); return vmx->emulation_required && !vmx->rmode.vm86_active && - (vcpu->arch.exception.pending || vcpu->arch.exception.injected); + (kvm_is_exception_pending(vcpu) || vcpu->arch.exception.injected); } static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) @@ -7431,7 +7422,7 @@ static int __init vmx_check_processor_compat(void) if (setup_vmcs_config(&vmcs_conf, &vmx_cap) < 0) return -EIO; if (nested) - nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, vmx_cap.ept); + nested_vmx_setup_ctls_msrs(&vmcs_conf, vmx_cap.ept); if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", smp_processor_id()); @@ -8071,7 +8062,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .patch_hypercall = vmx_patch_hypercall, .inject_irq = vmx_inject_irq, .inject_nmi = vmx_inject_nmi, - .queue_exception = vmx_queue_exception, + .inject_exception = vmx_inject_exception, .cancel_injection = vmx_cancel_injection, .interrupt_allowed = vmx_interrupt_allowed, .nmi_allowed = vmx_nmi_allowed, @@ -8228,6 +8219,10 @@ static __init int hardware_setup(void) if (setup_vmcs_config(&vmcs_config, &vmx_capability) < 0) return -EIO; + if (cpu_has_perf_global_ctrl_bug()) + pr_warn_once("kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL " + "does not work properly. Using workaround\n"); + if (boot_cpu_has(X86_FEATURE_NX)) kvm_enable_efer_bits(EFER_NX); @@ -8342,11 +8337,9 @@ static __init int hardware_setup(void) if (enable_preemption_timer) { u64 use_timer_freq = 5000ULL * 1000 * 1000; - u64 vmx_msr; - rdmsrl(MSR_IA32_VMX_MISC, vmx_msr); cpu_preemption_timer_multi = - vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; + vmcs_config.misc & VMX_MISC_PREEMPTION_TIMER_RATE_MASK; if (tsc_khz) use_timer_freq = (u64)tsc_khz * 1000; @@ -8382,8 +8375,7 @@ static __init int hardware_setup(void) setup_default_sgx_lepubkeyhash(); if (nested) { - nested_vmx_setup_ctls_msrs(&vmcs_config.nested, - vmx_capability.ept); + nested_vmx_setup_ctls_msrs(&vmcs_config, vmx_capability.ept); r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); if (r) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 24d58c2ffaa3d9dd4f4a6abcb12791abbe1b2a36..a3da84f4ea45609d44f76857d0e4cf3162fe210e 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -477,29 +477,145 @@ static inline u8 vmx_get_rvi(void) return vmcs_read16(GUEST_INTR_STATUS) & 0xff; } -#define BUILD_CONTROLS_SHADOW(lname, uname, bits) \ -static inline void lname##_controls_set(struct vcpu_vmx *vmx, u##bits val) \ -{ \ - if (vmx->loaded_vmcs->controls_shadow.lname != val) { \ - vmcs_write##bits(uname, val); \ - vmx->loaded_vmcs->controls_shadow.lname = val; \ - } \ -} \ -static inline u##bits __##lname##_controls_get(struct loaded_vmcs *vmcs) \ -{ \ - return vmcs->controls_shadow.lname; \ -} \ -static inline u##bits lname##_controls_get(struct vcpu_vmx *vmx) \ -{ \ - return __##lname##_controls_get(vmx->loaded_vmcs); \ -} \ -static inline void lname##_controls_setbit(struct vcpu_vmx *vmx, u##bits val) \ -{ \ - lname##_controls_set(vmx, lname##_controls_get(vmx) | val); \ -} \ -static inline void lname##_controls_clearbit(struct vcpu_vmx *vmx, u##bits val) \ -{ \ - lname##_controls_set(vmx, lname##_controls_get(vmx) & ~val); \ +#define __KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \ + (VM_ENTRY_LOAD_DEBUG_CONTROLS) +#ifdef CONFIG_X86_64 + #define KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \ + (__KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS | \ + VM_ENTRY_IA32E_MODE) +#else + #define KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \ + __KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS +#endif +#define KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS \ + (VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_ENTRY_LOAD_IA32_PAT | \ + VM_ENTRY_LOAD_IA32_EFER | \ + VM_ENTRY_LOAD_BNDCFGS | \ + VM_ENTRY_PT_CONCEAL_PIP | \ + VM_ENTRY_LOAD_IA32_RTIT_CTL) + +#define __KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \ + (VM_EXIT_SAVE_DEBUG_CONTROLS | \ + VM_EXIT_ACK_INTR_ON_EXIT) +#ifdef CONFIG_X86_64 + #define KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \ + (__KVM_REQUIRED_VMX_VM_EXIT_CONTROLS | \ + VM_EXIT_HOST_ADDR_SPACE_SIZE) +#else + #define KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \ + __KVM_REQUIRED_VMX_VM_EXIT_CONTROLS +#endif +#define KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS \ + (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \ + VM_EXIT_SAVE_IA32_PAT | \ + VM_EXIT_LOAD_IA32_PAT | \ + VM_EXIT_SAVE_IA32_EFER | \ + VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | \ + VM_EXIT_LOAD_IA32_EFER | \ + VM_EXIT_CLEAR_BNDCFGS | \ + VM_EXIT_PT_CONCEAL_PIP | \ + VM_EXIT_CLEAR_IA32_RTIT_CTL) + +#define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \ + (PIN_BASED_EXT_INTR_MASK | \ + PIN_BASED_NMI_EXITING) +#define KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL \ + (PIN_BASED_VIRTUAL_NMIS | \ + PIN_BASED_POSTED_INTR | \ + PIN_BASED_VMX_PREEMPTION_TIMER) + +#define __KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \ + (CPU_BASED_HLT_EXITING | \ + CPU_BASED_CR3_LOAD_EXITING | \ + CPU_BASED_CR3_STORE_EXITING | \ + CPU_BASED_UNCOND_IO_EXITING | \ + CPU_BASED_MOV_DR_EXITING | \ + CPU_BASED_USE_TSC_OFFSETTING | \ + CPU_BASED_MWAIT_EXITING | \ + CPU_BASED_MONITOR_EXITING | \ + CPU_BASED_INVLPG_EXITING | \ + CPU_BASED_RDPMC_EXITING | \ + CPU_BASED_INTR_WINDOW_EXITING) + +#ifdef CONFIG_X86_64 + #define KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \ + (__KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL | \ + CPU_BASED_CR8_LOAD_EXITING | \ + CPU_BASED_CR8_STORE_EXITING) +#else + #define KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \ + __KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL +#endif + +#define KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL \ + (CPU_BASED_RDTSC_EXITING | \ + CPU_BASED_TPR_SHADOW | \ + CPU_BASED_USE_IO_BITMAPS | \ + CPU_BASED_MONITOR_TRAP_FLAG | \ + CPU_BASED_USE_MSR_BITMAPS | \ + CPU_BASED_NMI_WINDOW_EXITING | \ + CPU_BASED_PAUSE_EXITING | \ + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS | \ + CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) + +#define KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL 0 +#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL \ + (SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | \ + SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \ + SECONDARY_EXEC_WBINVD_EXITING | \ + SECONDARY_EXEC_ENABLE_VPID | \ + SECONDARY_EXEC_ENABLE_EPT | \ + SECONDARY_EXEC_UNRESTRICTED_GUEST | \ + SECONDARY_EXEC_PAUSE_LOOP_EXITING | \ + SECONDARY_EXEC_DESC | \ + SECONDARY_EXEC_ENABLE_RDTSCP | \ + SECONDARY_EXEC_ENABLE_INVPCID | \ + SECONDARY_EXEC_APIC_REGISTER_VIRT | \ + SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | \ + SECONDARY_EXEC_SHADOW_VMCS | \ + SECONDARY_EXEC_XSAVES | \ + SECONDARY_EXEC_RDSEED_EXITING | \ + SECONDARY_EXEC_RDRAND_EXITING | \ + SECONDARY_EXEC_ENABLE_PML | \ + SECONDARY_EXEC_TSC_SCALING | \ + SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \ + SECONDARY_EXEC_PT_USE_GPA | \ + SECONDARY_EXEC_PT_CONCEAL_VMX | \ + SECONDARY_EXEC_ENABLE_VMFUNC | \ + SECONDARY_EXEC_BUS_LOCK_DETECTION | \ + SECONDARY_EXEC_NOTIFY_VM_EXITING | \ + SECONDARY_EXEC_ENCLS_EXITING) + +#define KVM_REQUIRED_VMX_TERTIARY_VM_EXEC_CONTROL 0 +#define KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL \ + (TERTIARY_EXEC_IPI_VIRT) + +#define BUILD_CONTROLS_SHADOW(lname, uname, bits) \ +static inline void lname##_controls_set(struct vcpu_vmx *vmx, u##bits val) \ +{ \ + if (vmx->loaded_vmcs->controls_shadow.lname != val) { \ + vmcs_write##bits(uname, val); \ + vmx->loaded_vmcs->controls_shadow.lname = val; \ + } \ +} \ +static inline u##bits __##lname##_controls_get(struct loaded_vmcs *vmcs) \ +{ \ + return vmcs->controls_shadow.lname; \ +} \ +static inline u##bits lname##_controls_get(struct vcpu_vmx *vmx) \ +{ \ + return __##lname##_controls_get(vmx->loaded_vmcs); \ +} \ +static __always_inline void lname##_controls_setbit(struct vcpu_vmx *vmx, u##bits val) \ +{ \ + BUILD_BUG_ON(!(val & (KVM_REQUIRED_VMX_##uname | KVM_OPTIONAL_VMX_##uname))); \ + lname##_controls_set(vmx, lname##_controls_get(vmx) | val); \ +} \ +static __always_inline void lname##_controls_clearbit(struct vcpu_vmx *vmx, u##bits val) \ +{ \ + BUILD_BUG_ON(!(val & (KVM_REQUIRED_VMX_##uname | KVM_OPTIONAL_VMX_##uname))); \ + lname##_controls_set(vmx, lname##_controls_get(vmx) & ~val); \ } BUILD_CONTROLS_SHADOW(vm_entry, VM_ENTRY_CONTROLS, 32) BUILD_CONTROLS_SHADOW(vm_exit, VM_EXIT_CONTROLS, 32) @@ -626,4 +742,14 @@ static inline bool vmx_can_use_ipiv(struct kvm_vcpu *vcpu) return lapic_in_kernel(vcpu) && enable_ipiv; } +static inline bool guest_cpuid_has_evmcs(struct kvm_vcpu *vcpu) +{ + /* + * eVMCS is exposed to the guest if Hyper-V is enabled in CPUID and + * eVMCS has been explicitly enabled by userspace. + */ + return vcpu->arch.hyperv_enabled && + to_vmx(vcpu)->nested.enlightened_vmcs_enabled; +} + #endif /* __KVM_X86_VMX_H */ diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 5cfc49ddb1b442801bf058779e86f0102a7536ec..ec268df83ed67988f76a007ad65e9d77aa5d7a3f 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -10,7 +10,7 @@ #include "vmcs.h" #include "../x86.h" -asmlinkage void vmread_error(unsigned long field, bool fault); +void vmread_error(unsigned long field, bool fault); __attribute__((regparm(0))) void vmread_error_trampoline(unsigned long field, bool fault); void vmwrite_error(unsigned long field, unsigned long value); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 205ebdc2b11bcde2bb1ab11e6c2cda892f74f668..4bd5f8a751de91ffeb666e1be9c5db8ae3b65f36 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -173,8 +173,13 @@ bool __read_mostly enable_vmware_backdoor = false; module_param(enable_vmware_backdoor, bool, S_IRUGO); EXPORT_SYMBOL_GPL(enable_vmware_backdoor); -static bool __read_mostly force_emulation_prefix = false; -module_param(force_emulation_prefix, bool, S_IRUGO); +/* + * Flags to manipulate forced emulation behavior (any non-zero value will + * enable forced emulation). + */ +#define KVM_FEP_CLEAR_RFLAGS_RF BIT(1) +static int __read_mostly force_emulation_prefix; +module_param(force_emulation_prefix, int, 0644); int __read_mostly pi_inject_timer = -1; module_param(pi_inject_timer, bint, S_IRUGO | S_IWUSR); @@ -528,6 +533,7 @@ static int exception_class(int vector) #define EXCPT_TRAP 1 #define EXCPT_ABORT 2 #define EXCPT_INTERRUPT 3 +#define EXCPT_DB 4 static int exception_type(int vector) { @@ -538,8 +544,14 @@ static int exception_type(int vector) mask = 1 << vector; - /* #DB is trap, as instruction watchpoints are handled elsewhere */ - if (mask & ((1 << DB_VECTOR) | (1 << BP_VECTOR) | (1 << OF_VECTOR))) + /* + * #DBs can be trap-like or fault-like, the caller must check other CPU + * state, e.g. DR6, to determine whether a #DB is a trap or fault. + */ + if (mask & (1 << DB_VECTOR)) + return EXCPT_DB; + + if (mask & ((1 << BP_VECTOR) | (1 << OF_VECTOR))) return EXCPT_TRAP; if (mask & ((1 << DF_VECTOR) | (1 << MC_VECTOR))) @@ -549,16 +561,13 @@ static int exception_type(int vector) return EXCPT_FAULT; } -void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu) +void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, + struct kvm_queued_exception *ex) { - unsigned nr = vcpu->arch.exception.nr; - bool has_payload = vcpu->arch.exception.has_payload; - unsigned long payload = vcpu->arch.exception.payload; - - if (!has_payload) + if (!ex->has_payload) return; - switch (nr) { + switch (ex->vector) { case DB_VECTOR: /* * "Certain debug exceptions may clear bit 0-3. The @@ -583,8 +592,8 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu) * So they need to be flipped for DR6. */ vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= payload; - vcpu->arch.dr6 ^= payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= ex->payload; + vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; /* * The #DB payload is defined as compatible with the 'pending @@ -595,15 +604,30 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu) vcpu->arch.dr6 &= ~BIT(12); break; case PF_VECTOR: - vcpu->arch.cr2 = payload; + vcpu->arch.cr2 = ex->payload; break; } - vcpu->arch.exception.has_payload = false; - vcpu->arch.exception.payload = 0; + ex->has_payload = false; + ex->payload = 0; } EXPORT_SYMBOL_GPL(kvm_deliver_exception_payload); +static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vector, + bool has_error_code, u32 error_code, + bool has_payload, unsigned long payload) +{ + struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + + ex->vector = vector; + ex->injected = false; + ex->pending = true; + ex->has_error_code = has_error_code; + ex->error_code = error_code; + ex->has_payload = has_payload; + ex->payload = payload; +} + static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned nr, bool has_error, u32 error_code, bool has_payload, unsigned long payload, bool reinject) @@ -613,18 +637,31 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, kvm_make_request(KVM_REQ_EVENT, vcpu); + /* + * If the exception is destined for L2 and isn't being reinjected, + * morph it to a VM-Exit if L1 wants to intercept the exception. A + * previously injected exception is not checked because it was checked + * when it was original queued, and re-checking is incorrect if _L1_ + * injected the exception, in which case it's exempt from interception. + */ + if (!reinject && is_guest_mode(vcpu) && + kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) { + kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, + has_payload, payload); + return; + } + if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) { queue: if (reinject) { /* - * On vmentry, vcpu->arch.exception.pending is only - * true if an event injection was blocked by - * nested_run_pending. In that case, however, - * vcpu_enter_guest requests an immediate exit, - * and the guest shouldn't proceed far enough to - * need reinjection. + * On VM-Entry, an exception can be pending if and only + * if event injection was blocked by nested_run_pending. + * In that case, however, vcpu_enter_guest() requests an + * immediate exit, and the guest shouldn't proceed far + * enough to need reinjection. */ - WARN_ON_ONCE(vcpu->arch.exception.pending); + WARN_ON_ONCE(kvm_is_exception_pending(vcpu)); vcpu->arch.exception.injected = true; if (WARN_ON_ONCE(has_payload)) { /* @@ -639,17 +676,18 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, vcpu->arch.exception.injected = false; } vcpu->arch.exception.has_error_code = has_error; - vcpu->arch.exception.nr = nr; + vcpu->arch.exception.vector = nr; vcpu->arch.exception.error_code = error_code; vcpu->arch.exception.has_payload = has_payload; vcpu->arch.exception.payload = payload; if (!is_guest_mode(vcpu)) - kvm_deliver_exception_payload(vcpu); + kvm_deliver_exception_payload(vcpu, + &vcpu->arch.exception); return; } /* to check exception */ - prev_nr = vcpu->arch.exception.nr; + prev_nr = vcpu->arch.exception.vector; if (prev_nr == DF_VECTOR) { /* triple fault -> shutdown */ kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); @@ -657,25 +695,22 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, } class1 = exception_class(prev_nr); class2 = exception_class(nr); - if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) - || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { + if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY) || + (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) { /* - * Generate double fault per SDM Table 5-5. Set - * exception.pending = true so that the double fault - * can trigger a nested vmexit. + * Synthesize #DF. Clear the previously injected or pending + * exception so as not to incorrectly trigger shutdown. */ - vcpu->arch.exception.pending = true; vcpu->arch.exception.injected = false; - vcpu->arch.exception.has_error_code = true; - vcpu->arch.exception.nr = DF_VECTOR; - vcpu->arch.exception.error_code = 0; - vcpu->arch.exception.has_payload = false; - vcpu->arch.exception.payload = 0; - } else + vcpu->arch.exception.pending = false; + + kvm_queue_exception_e(vcpu, DF_VECTOR, 0); + } else { /* replace previous exception with a new one in a hope that instruction re-execution will regenerate lost exception */ goto queue; + } } void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) @@ -729,20 +764,22 @@ static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { ++vcpu->stat.pf_guest; - vcpu->arch.exception.nested_apf = - is_guest_mode(vcpu) && fault->async_page_fault; - if (vcpu->arch.exception.nested_apf) { - vcpu->arch.apf.nested_apf_token = fault->address; - kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code); - } else { + + /* + * Async #PF in L2 is always forwarded to L1 as a VM-Exit regardless of + * whether or not L1 wants to intercept "regular" #PF. + */ + if (is_guest_mode(vcpu) && fault->async_page_fault) + kvm_queue_exception_vmexit(vcpu, PF_VECTOR, + true, fault->error_code, + true, fault->address); + else kvm_queue_exception_e_p(vcpu, PF_VECTOR, fault->error_code, fault->address); - } } EXPORT_SYMBOL_GPL(kvm_inject_page_fault); -/* Returns true if the page fault was immediately morphed into a VM-Exit. */ -bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, +void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { struct kvm_mmu *fault_mmu; @@ -760,26 +797,7 @@ bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address, fault_mmu->root.hpa); - /* - * A workaround for KVM's bad exception handling. If KVM injected an - * exception into L2, and L2 encountered a #PF while vectoring the - * injected exception, manually check to see if L1 wants to intercept - * #PF, otherwise queuing the #PF will lead to #DF or a lost exception. - * In all other cases, defer the check to nested_ops->check_events(), - * which will correctly handle priority (this does not). Note, other - * exceptions, e.g. #GP, are theoretically affected, #PF is simply the - * most problematic, e.g. when L0 and L1 are both intercepting #PF for - * shadow paging. - * - * TODO: Rewrite exception handling to track injected and pending - * (VM-Exit) exceptions separately. - */ - if (unlikely(vcpu->arch.exception.injected && is_guest_mode(vcpu)) && - kvm_x86_ops.nested_ops->handle_page_fault_workaround(vcpu, fault)) - return true; - fault_mmu->inject_page_fault(vcpu, fault); - return false; } EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault); @@ -1011,15 +1029,10 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_load_host_xsave_state); -static inline u64 kvm_guest_supported_xcr0(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->user_xfeatures; -} - #ifdef CONFIG_X86_64 static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) { - return kvm_guest_supported_xcr0(vcpu) & XFEATURE_MASK_USER_DYNAMIC; + return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; } #endif @@ -1042,7 +1055,7 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) * saving. However, xcr0 bit 0 is always set, even if the * emulated CPU does not support XSAVE (see kvm_vcpu_reset()). */ - valid_bits = kvm_guest_supported_xcr0(vcpu) | XFEATURE_MASK_FP; + valid_bits = vcpu->arch.guest_supported_xcr0 | XFEATURE_MASK_FP; if (xcr0 & ~valid_bits) return 1; @@ -1070,6 +1083,7 @@ static int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) { + /* Note, #UD due to CR4.OSXSAVE=0 has priority over the intercept. */ if (static_call(kvm_x86_get_cpl)(vcpu) != 0 || __kvm_set_xcr(vcpu, kvm_rcx_read(vcpu), kvm_read_edx_eax(vcpu))) { kvm_inject_gp(vcpu, 0); @@ -1557,12 +1571,32 @@ static const u32 msr_based_features_all[] = { static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)]; static unsigned int num_msr_based_features; +/* + * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM + * does not yet virtualize. These include: + * 10 - MISC_PACKAGE_CTRLS + * 11 - ENERGY_FILTERING_CTL + * 12 - DOITM + * 18 - FB_CLEAR_CTRL + * 21 - XAPIC_DISABLE_STATUS + * 23 - OVERCLOCKING_STATUS + */ + +#define KVM_SUPPORTED_ARCH_CAP \ + (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \ + ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \ + ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ + ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ + ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO) + static u64 kvm_get_arch_capabilities(void) { u64 data = 0; - if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) + if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) { rdmsrl(MSR_IA32_ARCH_CAPABILITIES, data); + data &= KVM_SUPPORTED_ARCH_CAP; + } /* * If nx_huge_pages is enabled, KVM's shadow paging will ensure that @@ -1610,9 +1644,6 @@ static u64 kvm_get_arch_capabilities(void) */ } - /* Guests don't need to know "Fill buffer clear control" exists */ - data &= ~ARCH_CAP_FB_CLEAR_CTRL; - return data; } @@ -4828,7 +4859,7 @@ static int kvm_vcpu_ready_for_interrupt_injection(struct kvm_vcpu *vcpu) return (kvm_arch_interrupt_allowed(vcpu) && kvm_cpu_accept_dm_intr(vcpu) && !kvm_event_needs_reinjection(vcpu) && - !vcpu->arch.exception.pending); + !kvm_is_exception_pending(vcpu)); } static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, @@ -5003,25 +5034,38 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) { + struct kvm_queued_exception *ex; + process_nmi(vcpu); if (kvm_check_request(KVM_REQ_SMI, vcpu)) process_smi(vcpu); /* - * In guest mode, payload delivery should be deferred, - * so that the L1 hypervisor can intercept #PF before - * CR2 is modified (or intercept #DB before DR6 is - * modified under nVMX). Unless the per-VM capability, - * KVM_CAP_EXCEPTION_PAYLOAD, is set, we may not defer the delivery of - * an exception payload and handle after a KVM_GET_VCPU_EVENTS. Since we - * opportunistically defer the exception payload, deliver it if the - * capability hasn't been requested before processing a - * KVM_GET_VCPU_EVENTS. + * KVM's ABI only allows for one exception to be migrated. Luckily, + * the only time there can be two queued exceptions is if there's a + * non-exiting _injected_ exception, and a pending exiting exception. + * In that case, ignore the VM-Exiting exception as it's an extension + * of the injected exception. + */ + if (vcpu->arch.exception_vmexit.pending && + !vcpu->arch.exception.pending && + !vcpu->arch.exception.injected) + ex = &vcpu->arch.exception_vmexit; + else + ex = &vcpu->arch.exception; + + /* + * In guest mode, payload delivery should be deferred if the exception + * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1 + * intercepts #PF, ditto for DR6 and #DBs. If the per-VM capability, + * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not + * propagate the payload and so it cannot be safely deferred. Deliver + * the payload if the capability hasn't been requested. */ if (!vcpu->kvm->arch.exception_payload_enabled && - vcpu->arch.exception.pending && vcpu->arch.exception.has_payload) - kvm_deliver_exception_payload(vcpu); + ex->pending && ex->has_payload) + kvm_deliver_exception_payload(vcpu, ex); /* * The API doesn't provide the instruction length for software @@ -5029,26 +5073,25 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, * isn't advanced, we should expect to encounter the exception * again. */ - if (kvm_exception_is_soft(vcpu->arch.exception.nr)) { + if (kvm_exception_is_soft(ex->vector)) { events->exception.injected = 0; events->exception.pending = 0; } else { - events->exception.injected = vcpu->arch.exception.injected; - events->exception.pending = vcpu->arch.exception.pending; + events->exception.injected = ex->injected; + events->exception.pending = ex->pending; /* * For ABI compatibility, deliberately conflate * pending and injected exceptions when * KVM_CAP_EXCEPTION_PAYLOAD isn't enabled. */ if (!vcpu->kvm->arch.exception_payload_enabled) - events->exception.injected |= - vcpu->arch.exception.pending; + events->exception.injected |= ex->pending; } - events->exception.nr = vcpu->arch.exception.nr; - events->exception.has_error_code = vcpu->arch.exception.has_error_code; - events->exception.error_code = vcpu->arch.exception.error_code; - events->exception_has_payload = vcpu->arch.exception.has_payload; - events->exception_payload = vcpu->arch.exception.payload; + events->exception.nr = ex->vector; + events->exception.has_error_code = ex->has_error_code; + events->exception.error_code = ex->error_code; + events->exception_has_payload = ex->has_payload; + events->exception_payload = ex->payload; events->interrupt.injected = vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft; @@ -5118,9 +5161,22 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, return -EINVAL; process_nmi(vcpu); + + /* + * Flag that userspace is stuffing an exception, the next KVM_RUN will + * morph the exception to a VM-Exit if appropriate. Do this only for + * pending exceptions, already-injected exceptions are not subject to + * intercpetion. Note, userspace that conflates pending and injected + * is hosed, and will incorrectly convert an injected exception into a + * pending exception, which in turn may cause a spurious VM-Exit. + */ + vcpu->arch.exception_from_userspace = events->exception.pending; + + vcpu->arch.exception_vmexit.pending = false; + vcpu->arch.exception.injected = events->exception.injected; vcpu->arch.exception.pending = events->exception.pending; - vcpu->arch.exception.nr = events->exception.nr; + vcpu->arch.exception.vector = events->exception.nr; vcpu->arch.exception.has_error_code = events->exception.has_error_code; vcpu->arch.exception.error_code = events->exception.error_code; vcpu->arch.exception.has_payload = events->exception_has_payload; @@ -7244,6 +7300,7 @@ static int kvm_can_emulate_insn(struct kvm_vcpu *vcpu, int emul_type, int handle_ud(struct kvm_vcpu *vcpu) { static const char kvm_emulate_prefix[] = { __KVM_EMULATE_PREFIX }; + int fep_flags = READ_ONCE(force_emulation_prefix); int emul_type = EMULTYPE_TRAP_UD; char sig[5]; /* ud2; .ascii "kvm" */ struct x86_exception e; @@ -7251,10 +7308,12 @@ int handle_ud(struct kvm_vcpu *vcpu) if (unlikely(!kvm_can_emulate_insn(vcpu, emul_type, NULL, 0))) return 1; - if (force_emulation_prefix && + if (fep_flags && kvm_read_guest_virt(vcpu, kvm_get_linear_rip(vcpu), sig, sizeof(sig), &e) == 0 && memcmp(sig, kvm_emulate_prefix, sizeof(sig)) == 0) { + if (fep_flags & KVM_FEP_CLEAR_RFLAGS_RF) + kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) & ~X86_EFLAGS_RF); kvm_rip_write(vcpu, kvm_rip_read(vcpu) + sizeof(sig)); emul_type = EMULTYPE_TRAP_UD_FORCED; } @@ -7920,14 +7979,20 @@ static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, int r; r = kvm_get_msr_with_filter(vcpu, msr_index, pdata); + if (r < 0) + return X86EMUL_UNHANDLEABLE; - if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, - complete_emulated_rdmsr, r)) { - /* Bounce to user space */ - return X86EMUL_IO_NEEDED; + if (r) { + if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, + complete_emulated_rdmsr, r)) + return X86EMUL_IO_NEEDED; + + trace_kvm_msr_read_ex(msr_index); + return X86EMUL_PROPAGATE_FAULT; } - return r; + trace_kvm_msr_read(msr_index, *pdata); + return X86EMUL_CONTINUE; } static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, @@ -7937,14 +8002,20 @@ static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, int r; r = kvm_set_msr_with_filter(vcpu, msr_index, data); + if (r < 0) + return X86EMUL_UNHANDLEABLE; - if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, - complete_emulated_msr_access, r)) { - /* Bounce to user space */ - return X86EMUL_IO_NEEDED; + if (r) { + if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, + complete_emulated_msr_access, r)) + return X86EMUL_IO_NEEDED; + + trace_kvm_msr_write_ex(msr_index, data); + return X86EMUL_PROPAGATE_FAULT; } - return r; + trace_kvm_msr_write(msr_index, data); + return X86EMUL_CONTINUE; } static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, @@ -8148,18 +8219,17 @@ static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) } } -static bool inject_emulated_exception(struct kvm_vcpu *vcpu) +static void inject_emulated_exception(struct kvm_vcpu *vcpu) { struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; - if (ctxt->exception.vector == PF_VECTOR) - return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); - if (ctxt->exception.error_code_valid) + if (ctxt->exception.vector == PF_VECTOR) + kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); + else if (ctxt->exception.error_code_valid) kvm_queue_exception_e(vcpu, ctxt->exception.vector, ctxt->exception.error_code); else kvm_queue_exception(vcpu, ctxt->exception.vector); - return false; } static struct x86_emulate_ctxt *alloc_emulate_ctxt(struct kvm_vcpu *vcpu) @@ -8535,8 +8605,46 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction); -static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, int *r) +static bool kvm_is_code_breakpoint_inhibited(struct kvm_vcpu *vcpu) +{ + u32 shadow; + + if (kvm_get_rflags(vcpu) & X86_EFLAGS_RF) + return true; + + /* + * Intel CPUs inhibit code #DBs when MOV/POP SS blocking is active, + * but AMD CPUs do not. MOV/POP SS blocking is rare, check that first + * to avoid the relatively expensive CPUID lookup. + */ + shadow = static_call(kvm_x86_get_interrupt_shadow)(vcpu); + return (shadow & KVM_X86_SHADOW_INT_MOV_SS) && + guest_cpuid_is_intel(vcpu); +} + +static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, + int emulation_type, int *r) { + WARN_ON_ONCE(emulation_type & EMULTYPE_NO_DECODE); + + /* + * Do not check for code breakpoints if hardware has already done the + * checks, as inferred from the emulation type. On NO_DECODE and SKIP, + * the instruction has passed all exception checks, and all intercepted + * exceptions that trigger emulation have lower priority than code + * breakpoints, i.e. the fact that the intercepted exception occurred + * means any code breakpoints have already been serviced. + * + * Note, KVM needs to check for code #DBs on EMULTYPE_TRAP_UD_FORCED as + * hardware has checked the RIP of the magic prefix, but not the RIP of + * the instruction being emulated. The intent of forced emulation is + * to behave as if KVM intercepted the instruction without an exception + * and without a prefix. + */ + if (emulation_type & (EMULTYPE_NO_DECODE | EMULTYPE_SKIP | + EMULTYPE_TRAP_UD | EMULTYPE_VMWARE_GP | EMULTYPE_PF)) + return false; + if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { struct kvm_run *kvm_run = vcpu->run; @@ -8556,7 +8664,7 @@ static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, int *r) } if (unlikely(vcpu->arch.dr7 & DR7_BP_EN_MASK) && - !(kvm_get_rflags(vcpu) & X86_EFLAGS_RF)) { + !kvm_is_code_breakpoint_inhibited(vcpu)) { unsigned long eip = kvm_get_linear_rip(vcpu); u32 dr6 = kvm_vcpu_check_hw_bp(eip, 0, vcpu->arch.dr7, @@ -8658,8 +8766,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * are fault-like and are higher priority than any faults on * the code fetch itself. */ - if (!(emulation_type & EMULTYPE_SKIP) && - kvm_vcpu_check_code_breakpoint(vcpu, &r)) + if (kvm_vcpu_check_code_breakpoint(vcpu, emulation_type, &r)) return r; r = x86_decode_emulated_instruction(vcpu, emulation_type, @@ -8757,8 +8864,7 @@ restart: if (ctxt->have_exception) { r = 1; - if (inject_emulated_exception(vcpu)) - return r; + inject_emulated_exception(vcpu); } else if (vcpu->arch.pio.count) { if (!vcpu->arch.pio.in) { /* FIXME: return into emulator if single-stepping. */ @@ -8788,6 +8894,12 @@ writeback: unsigned long rflags = static_call(kvm_x86_get_rflags)(vcpu); toggle_interruptibility(vcpu, ctxt->interruptibility); vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + + /* + * Note, EXCPT_DB is assumed to be fault-like as the emulator + * only supports code breakpoints and general detect #DB, both + * of which are fault-like. + */ if (!ctxt->have_exception || exception_type(ctxt->exception.vector) == EXCPT_TRAP) { kvm_pmu_trigger_event(vcpu, PERF_COUNT_HW_INSTRUCTIONS); @@ -9649,74 +9761,155 @@ int kvm_check_nested_events(struct kvm_vcpu *vcpu) static void kvm_inject_exception(struct kvm_vcpu *vcpu) { - trace_kvm_inj_exception(vcpu->arch.exception.nr, + trace_kvm_inj_exception(vcpu->arch.exception.vector, vcpu->arch.exception.has_error_code, vcpu->arch.exception.error_code, vcpu->arch.exception.injected); if (vcpu->arch.exception.error_code && !is_protmode(vcpu)) vcpu->arch.exception.error_code = false; - static_call(kvm_x86_queue_exception)(vcpu); + static_call(kvm_x86_inject_exception)(vcpu); } -static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit) +/* + * Check for any event (interrupt or exception) that is ready to be injected, + * and if there is at least one event, inject the event with the highest + * priority. This handles both "pending" events, i.e. events that have never + * been injected into the guest, and "injected" events, i.e. events that were + * injected as part of a previous VM-Enter, but weren't successfully delivered + * and need to be re-injected. + * + * Note, this is not guaranteed to be invoked on a guest instruction boundary, + * i.e. doesn't guarantee that there's an event window in the guest. KVM must + * be able to inject exceptions in the "middle" of an instruction, and so must + * also be able to re-inject NMIs and IRQs in the middle of an instruction. + * I.e. for exceptions and re-injected events, NOT invoking this on instruction + * boundaries is necessary and correct. + * + * For simplicity, KVM uses a single path to inject all events (except events + * that are injected directly from L1 to L2) and doesn't explicitly track + * instruction boundaries for asynchronous events. However, because VM-Exits + * that can occur during instruction execution typically result in KVM skipping + * the instruction or injecting an exception, e.g. instruction and exception + * intercepts, and because pending exceptions have higher priority than pending + * interrupts, KVM still honors instruction boundaries in most scenarios. + * + * But, if a VM-Exit occurs during instruction execution, and KVM does NOT skip + * the instruction or inject an exception, then KVM can incorrecty inject a new + * asynchrounous event if the event became pending after the CPU fetched the + * instruction (in the guest). E.g. if a page fault (#PF, #NPF, EPT violation) + * occurs and is resolved by KVM, a coincident NMI, SMI, IRQ, etc... can be + * injected on the restarted instruction instead of being deferred until the + * instruction completes. + * + * In practice, this virtualization hole is unlikely to be observed by the + * guest, and even less likely to cause functional problems. To detect the + * hole, the guest would have to trigger an event on a side effect of an early + * phase of instruction execution, e.g. on the instruction fetch from memory. + * And for it to be a functional problem, the guest would need to depend on the + * ordering between that side effect, the instruction completing, _and_ the + * delivery of the asynchronous event. + */ +static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, + bool *req_immediate_exit) { + bool can_inject; int r; - bool can_inject = true; - /* try to reinject previous events if any */ + /* + * Process nested events first, as nested VM-Exit supercedes event + * re-injection. If there's an event queued for re-injection, it will + * be saved into the appropriate vmc{b,s}12 fields on nested VM-Exit. + */ + if (is_guest_mode(vcpu)) + r = kvm_check_nested_events(vcpu); + else + r = 0; - if (vcpu->arch.exception.injected) { - kvm_inject_exception(vcpu); - can_inject = false; - } /* - * Do not inject an NMI or interrupt if there is a pending - * exception. Exceptions and interrupts are recognized at - * instruction boundaries, i.e. the start of an instruction. - * Trap-like exceptions, e.g. #DB, have higher priority than - * NMIs and interrupts, i.e. traps are recognized before an - * NMI/interrupt that's pending on the same instruction. - * Fault-like exceptions, e.g. #GP and #PF, are the lowest - * priority, but are only generated (pended) during instruction - * execution, i.e. a pending fault-like exception means the - * fault occurred on the *previous* instruction and must be - * serviced prior to recognizing any new events in order to - * fully complete the previous instruction. + * Re-inject exceptions and events *especially* if immediate entry+exit + * to/from L2 is needed, as any event that has already been injected + * into L2 needs to complete its lifecycle before injecting a new event. + * + * Don't re-inject an NMI or interrupt if there is a pending exception. + * This collision arises if an exception occurred while vectoring the + * injected event, KVM intercepted said exception, and KVM ultimately + * determined the fault belongs to the guest and queues the exception + * for injection back into the guest. + * + * "Injected" interrupts can also collide with pending exceptions if + * userspace ignores the "ready for injection" flag and blindly queues + * an interrupt. In that case, prioritizing the exception is correct, + * as the exception "occurred" before the exit to userspace. Trap-like + * exceptions, e.g. most #DBs, have higher priority than interrupts. + * And while fault-like exceptions, e.g. #GP and #PF, are the lowest + * priority, they're only generated (pended) during instruction + * execution, and interrupts are recognized at instruction boundaries. + * Thus a pending fault-like exception means the fault occurred on the + * *previous* instruction and must be serviced prior to recognizing any + * new events in order to fully complete the previous instruction. */ - else if (!vcpu->arch.exception.pending) { - if (vcpu->arch.nmi_injected) { - static_call(kvm_x86_inject_nmi)(vcpu); - can_inject = false; - } else if (vcpu->arch.interrupt.injected) { - static_call(kvm_x86_inject_irq)(vcpu, true); - can_inject = false; - } - } + if (vcpu->arch.exception.injected) + kvm_inject_exception(vcpu); + else if (kvm_is_exception_pending(vcpu)) + ; /* see above */ + else if (vcpu->arch.nmi_injected) + static_call(kvm_x86_inject_nmi)(vcpu); + else if (vcpu->arch.interrupt.injected) + static_call(kvm_x86_inject_irq)(vcpu, true); + /* + * Exceptions that morph to VM-Exits are handled above, and pending + * exceptions on top of injected exceptions that do not VM-Exit should + * either morph to #DF or, sadly, override the injected exception. + */ WARN_ON_ONCE(vcpu->arch.exception.injected && vcpu->arch.exception.pending); /* - * Call check_nested_events() even if we reinjected a previous event - * in order for caller to determine if it should require immediate-exit - * from L2 to L1 due to pending L1 events which require exit - * from L2 to L1. + * Bail if immediate entry+exit to/from the guest is needed to complete + * nested VM-Enter or event re-injection so that a different pending + * event can be serviced (or if KVM needs to exit to userspace). + * + * Otherwise, continue processing events even if VM-Exit occurred. The + * VM-Exit will have cleared exceptions that were meant for L2, but + * there may now be events that can be injected into L1. */ - if (is_guest_mode(vcpu)) { - r = kvm_check_nested_events(vcpu); - if (r < 0) - goto out; - } + if (r < 0) + goto out; + + /* + * A pending exception VM-Exit should either result in nested VM-Exit + * or force an immediate re-entry and exit to/from L2, and exception + * VM-Exits cannot be injected (flag should _never_ be set). + */ + WARN_ON_ONCE(vcpu->arch.exception_vmexit.injected || + vcpu->arch.exception_vmexit.pending); + + /* + * New events, other than exceptions, cannot be injected if KVM needs + * to re-inject a previous event. See above comments on re-injecting + * for why pending exceptions get priority. + */ + can_inject = !kvm_event_needs_reinjection(vcpu); - /* try to inject new event if pending */ if (vcpu->arch.exception.pending) { - if (exception_type(vcpu->arch.exception.nr) == EXCPT_FAULT) + /* + * Fault-class exceptions, except #DBs, set RF=1 in the RFLAGS + * value pushed on the stack. Trap-like exception and all #DBs + * leave RF as-is (KVM follows Intel's behavior in this regard; + * AMD states that code breakpoint #DBs excplitly clear RF=0). + * + * Note, most versions of Intel's SDM and AMD's APM incorrectly + * describe the behavior of General Detect #DBs, which are + * fault-like. They do _not_ set RF, a la code breakpoints. + */ + if (exception_type(vcpu->arch.exception.vector) == EXCPT_FAULT) __kvm_set_rflags(vcpu, kvm_get_rflags(vcpu) | X86_EFLAGS_RF); - if (vcpu->arch.exception.nr == DB_VECTOR) { - kvm_deliver_exception_payload(vcpu); + if (vcpu->arch.exception.vector == DB_VECTOR) { + kvm_deliver_exception_payload(vcpu, &vcpu->arch.exception); if (vcpu->arch.dr7 & DR7_GD) { vcpu->arch.dr7 &= ~DR7_GD; kvm_update_dr7(vcpu); @@ -9788,11 +9981,11 @@ static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit) } if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->hv_timer_pending && - kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) + kvm_x86_ops.nested_ops->has_events && + kvm_x86_ops.nested_ops->has_events(vcpu)) *req_immediate_exit = true; - WARN_ON(vcpu->arch.exception.pending); + WARN_ON(kvm_is_exception_pending(vcpu)); return 0; out: @@ -10097,7 +10290,7 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) * When APICv gets disabled, we may still have injected interrupts * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was * still active when the interrupt got accepted. Make sure - * inject_pending_event() is called to check for that. + * kvm_check_and_inject_events() is called to check for that. */ if (!apic->apicv_active) kvm_make_request(KVM_REQ_EVENT, vcpu); @@ -10394,7 +10587,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } - r = inject_pending_event(vcpu, &req_immediate_exit); + r = kvm_check_and_inject_events(vcpu, &req_immediate_exit); if (r < 0) { r = 0; goto out; @@ -10633,10 +10826,26 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu) if (hv_timer) kvm_lapic_switch_to_hv_timer(vcpu); - if (!kvm_check_request(KVM_REQ_UNHALT, vcpu)) + /* + * If the vCPU is not runnable, a signal or another host event + * of some kind is pending; service it without changing the + * vCPU's activity state. + */ + if (!kvm_arch_vcpu_runnable(vcpu)) return 1; } + /* + * Evaluate nested events before exiting the halted state. This allows + * the halt state to be recorded properly in the VMCS12's activity + * state field (AMD does not have a similar field and a VM-Exit always + * causes a spurious wakeup from HLT). + */ + if (is_guest_mode(vcpu)) { + if (kvm_check_nested_events(vcpu) < 0) + return 0; + } + if (kvm_apic_accept_events(vcpu) < 0) return 0; switch(vcpu->arch.mp_state) { @@ -10652,16 +10861,14 @@ static inline int vcpu_block(struct kvm_vcpu *vcpu) case KVM_MP_STATE_INIT_RECEIVED: break; default: - return -EINTR; + WARN_ON_ONCE(1); + break; } return 1; } static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) { - if (is_guest_mode(vcpu)) - kvm_check_nested_events(vcpu); - return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && !vcpu->arch.apf.halted); } @@ -10810,6 +11017,7 @@ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_queued_exception *ex = &vcpu->arch.exception; struct kvm_run *kvm_run = vcpu->run; int r; @@ -10838,7 +11046,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) r = 0; goto out; } - kvm_clear_request(KVM_REQ_UNHALT, vcpu); r = -EAGAIN; if (signal_pending(current)) { r = -EINTR; @@ -10868,6 +11075,21 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } } + /* + * If userspace set a pending exception and L2 is active, convert it to + * a pending VM-Exit if L1 wants to intercept the exception. + */ + if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && + kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector, + ex->error_code)) { + kvm_queue_exception_vmexit(vcpu, ex->vector, + ex->has_error_code, ex->error_code, + ex->has_payload, ex->payload); + ex->injected = false; + ex->pending = false; + } + vcpu->arch.exception_from_userspace = false; + if (unlikely(vcpu->arch.complete_userspace_io)) { int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io; vcpu->arch.complete_userspace_io = NULL; @@ -10974,6 +11196,7 @@ static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); vcpu->arch.exception.pending = false; + vcpu->arch.exception_vmexit.pending = false; kvm_make_request(KVM_REQ_EVENT, vcpu); } @@ -11093,16 +11316,30 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, vcpu_load(vcpu); - if (!lapic_in_kernel(vcpu) && - mp_state->mp_state != KVM_MP_STATE_RUNNABLE) + switch (mp_state->mp_state) { + case KVM_MP_STATE_UNINITIALIZED: + case KVM_MP_STATE_HALTED: + case KVM_MP_STATE_AP_RESET_HOLD: + case KVM_MP_STATE_INIT_RECEIVED: + case KVM_MP_STATE_SIPI_RECEIVED: + if (!lapic_in_kernel(vcpu)) + goto out; + break; + + case KVM_MP_STATE_RUNNABLE: + break; + + default: goto out; + } /* - * KVM_MP_STATE_INIT_RECEIVED means the processor is in - * INIT state; latched init should be reported using - * KVM_SET_VCPU_EVENTS, so reject it here. + * Pending INITs are reported using KVM_SET_VCPU_EVENTS, disallow + * forcing the guest into INIT/SIPI if those events are supposed to be + * blocked. KVM prioritizes SMI over INIT, so reject INIT/SIPI state + * if an SMI is pending as well. */ - if ((kvm_vcpu_latch_init(vcpu) || vcpu->arch.smi_pending) && + if ((!kvm_apic_init_sipi_allowed(vcpu) || vcpu->arch.smi_pending) && (mp_state->mp_state == KVM_MP_STATE_SIPI_RECEIVED || mp_state->mp_state == KVM_MP_STATE_INIT_RECEIVED)) goto out; @@ -11341,7 +11578,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, if (dbg->control & (KVM_GUESTDBG_INJECT_DB | KVM_GUESTDBG_INJECT_BP)) { r = -EBUSY; - if (vcpu->arch.exception.pending) + if (kvm_is_exception_pending(vcpu)) goto out; if (dbg->control & KVM_GUESTDBG_INJECT_DB) kvm_queue_exception(vcpu, DB_VECTOR); @@ -11563,7 +11800,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.mci_ctl2_banks = kcalloc(KVM_MAX_MCE_BANKS, sizeof(u64), GFP_KERNEL_ACCOUNT); if (!vcpu->arch.mce_banks || !vcpu->arch.mci_ctl2_banks) - goto fail_free_pio_data; + goto fail_free_mce_banks; vcpu->arch.mcg_cap = KVM_MAX_MCE_BANKS; if (!zalloc_cpumask_var(&vcpu->arch.wbinvd_dirty_mask, @@ -11617,7 +11854,6 @@ free_wbinvd_dirty_mask: fail_free_mce_banks: kfree(vcpu->arch.mce_banks); kfree(vcpu->arch.mci_ctl2_banks); -fail_free_pio_data: free_page((unsigned long)vcpu->arch.pio_data); fail_free_lapic: kvm_free_lapic(vcpu); @@ -11724,8 +11960,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) struct fpstate *fpstate = vcpu->arch.guest_fpu.fpstate; /* - * To avoid have the INIT path from kvm_apic_has_events() that be - * called with loaded FPU and does not let userspace fix the state. + * All paths that lead to INIT are required to load the guest's + * FPU state (because most paths are buried in KVM_RUN). */ if (init_event) kvm_put_guest_fpu(vcpu); @@ -12054,6 +12290,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) goto out_page_track; + ret = static_call(kvm_x86_vm_init)(kvm); + if (ret) + goto out_uninit_mmu; + INIT_HLIST_HEAD(&kvm->arch.mask_notifier_list); INIT_LIST_HEAD(&kvm->arch.assigned_dev_head); atomic_set(&kvm->arch.noncoherent_dma_count, 0); @@ -12089,8 +12329,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm_hv_init_vm(kvm); kvm_xen_init_vm(kvm); - return static_call(kvm_x86_vm_init)(kvm); + return 0; +out_uninit_mmu: + kvm_mmu_uninit_vm(kvm); out_page_track: kvm_page_track_cleanup(kvm); out: @@ -12473,6 +12715,50 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, } else { kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); } + + /* + * Unconditionally flush the TLBs after enabling dirty logging. + * A flush is almost always going to be necessary (see below), + * and unconditionally flushing allows the helpers to omit + * the subtly complex checks when removing write access. + * + * Do the flush outside of mmu_lock to reduce the amount of + * time mmu_lock is held. Flushing after dropping mmu_lock is + * safe as KVM only needs to guarantee the slot is fully + * write-protected before returning to userspace, i.e. before + * userspace can consume the dirty status. + * + * Flushing outside of mmu_lock requires KVM to be careful when + * making decisions based on writable status of an SPTE, e.g. a + * !writable SPTE doesn't guarantee a CPU can't perform writes. + * + * Specifically, KVM also write-protects guest page tables to + * monitor changes when using shadow paging, and must guarantee + * no CPUs can write to those page before mmu_lock is dropped. + * Because CPUs may have stale TLB entries at this point, a + * !writable SPTE doesn't guarantee CPUs can't perform writes. + * + * KVM also allows making SPTES writable outside of mmu_lock, + * e.g. to allow dirty logging without taking mmu_lock. + * + * To handle these scenarios, KVM uses a separate software-only + * bit (MMU-writable) to track if a SPTE is !writable due to + * a guest page table being write-protected (KVM clears the + * MMU-writable flag when write-protecting for shadow paging). + * + * The use of MMU-writable is also the primary motivation for + * the unconditional flush. Because KVM must guarantee that a + * CPU doesn't contain stale, writable TLB entries for a + * !MMU-writable SPTE, KVM must flush if it encounters any + * MMU-writable SPTE regardless of whether the actual hardware + * writable bit was set. I.e. KVM is almost guaranteed to need + * to flush, while unconditionally flushing allows the "remove + * write access" helpers to ignore MMU-writable entirely. + * + * See is_writable_pte() for more details (the case involving + * access-tracked SPTEs is particularly relevant). + */ + kvm_arch_flush_remote_tlbs_memslot(kvm, new); } } @@ -12519,13 +12805,14 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (!list_empty_careful(&vcpu->async_pf.done)) return true; - if (kvm_apic_has_events(vcpu)) + if (kvm_apic_has_pending_init_or_sipi(vcpu) && + kvm_apic_init_sipi_allowed(vcpu)) return true; if (vcpu->arch.pv.pv_unhalted) return true; - if (vcpu->arch.exception.pending) + if (kvm_is_exception_pending(vcpu)) return true; if (kvm_test_request(KVM_REQ_NMI, vcpu) || @@ -12547,16 +12834,13 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) return true; if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->hv_timer_pending && - kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) + kvm_x86_ops.nested_ops->has_events && + kvm_x86_ops.nested_ops->has_events(vcpu)) return true; if (kvm_xen_has_pending_events(vcpu)) return true; - if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) - return true; - return false; } @@ -12780,7 +13064,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) { if (unlikely(!lapic_in_kernel(vcpu) || kvm_event_needs_reinjection(vcpu) || - vcpu->arch.exception.pending)) + kvm_is_exception_pending(vcpu))) return false; if (kvm_hlt_in_guest(vcpu->kvm) && !kvm_can_deliver_async_pf(vcpu)) @@ -13331,7 +13615,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_inj_virq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_page_fault); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_msr); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_cr); -EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmrun); +EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmenter); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_vmexit_inject); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intr_vmexit); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 1926d2cb8e79233f066b083e5d356428def649a7..829d3134c1eb08d290aba61cf1ecf292b4ae9466 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -82,10 +82,18 @@ static inline unsigned int __shrink_ple_window(unsigned int val, void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu); int kvm_check_nested_events(struct kvm_vcpu *vcpu); +static inline bool kvm_is_exception_pending(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.exception.pending || + vcpu->arch.exception_vmexit.pending || + kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu); +} + static inline void kvm_clear_exception_queue(struct kvm_vcpu *vcpu) { vcpu->arch.exception.pending = false; vcpu->arch.exception.injected = false; + vcpu->arch.exception_vmexit.pending = false; } static inline void kvm_queue_interrupt(struct kvm_vcpu *vcpu, u8 vector, @@ -267,11 +275,6 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) return !(kvm->arch.disabled_quirks & quirk); } -static inline bool kvm_vcpu_latch_init(struct kvm_vcpu *vcpu) -{ - return is_smm(vcpu) || static_call(kvm_x86_apic_init_signal_blocked)(vcpu); -} - void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); u64 get_kvmclock_ns(struct kvm *kvm); @@ -286,7 +289,8 @@ int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, int handle_ud(struct kvm_vcpu *vcpu); -void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu); +void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, + struct kvm_queued_exception *ex); void kvm_vcpu_mtrr_init(struct kvm_vcpu *vcpu); u8 kvm_mtrr_get_guest_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 280cb5dc7341af1a6d61f7da26729d81f4bc702d..93c628d3e3a92cc7e4792ca2f7a734126a01e838 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1065,7 +1065,6 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, del_timer(&vcpu->arch.xen.poll_timer); vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; - kvm_clear_request(KVM_REQ_UNHALT, vcpu); } vcpu->arch.xen.poll_evtchn = 0; diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index f76747862bd2e0924d82c853d1ca9987ff4a06c5..7ba5f61d7273538ca65dd6f8b904fc417d6478ef 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -65,7 +65,9 @@ ifneq ($(CONFIG_X86_CMPXCHG64),y) endif else obj-y += iomap_copy_64.o +ifneq ($(CONFIG_GENERIC_CSUM),y) lib-y += csum-partial_64.o csum-copy_64.o csum-wrappers_64.o +endif lib-y += clear_page_64.o copy_page_64.o lib-y += memmove_64.o memset_64.o lib-y += copy_user_64.o diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S index fe59b8ac4fccd401789d23146ce3a261951acc52..ecbfb4dd3b019c5731d15d095dd16067fd6288bb 100644 --- a/arch/x86/lib/clear_page_64.S +++ b/arch/x86/lib/clear_page_64.S @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ #include +#include #include /* @@ -50,3 +51,140 @@ SYM_FUNC_START(clear_page_erms) RET SYM_FUNC_END(clear_page_erms) EXPORT_SYMBOL_GPL(clear_page_erms) + +/* + * Default clear user-space. + * Input: + * rdi destination + * rcx count + * + * Output: + * rcx: uncleared bytes or 0 if successful. + */ +SYM_FUNC_START(clear_user_original) + /* + * Copy only the lower 32 bits of size as that is enough to handle the rest bytes, + * i.e., no need for a 'q' suffix and thus a REX prefix. + */ + mov %ecx,%eax + shr $3,%rcx + jz .Lrest_bytes + + # do the qwords first + .p2align 4 +.Lqwords: + movq $0,(%rdi) + lea 8(%rdi),%rdi + dec %rcx + jnz .Lqwords + +.Lrest_bytes: + and $7, %eax + jz .Lexit + + # now do the rest bytes +.Lbytes: + movb $0,(%rdi) + inc %rdi + dec %eax + jnz .Lbytes + +.Lexit: + /* + * %rax still needs to be cleared in the exception case because this function is called + * from inline asm and the compiler expects %rax to be zero when exiting the inline asm, + * in case it might reuse it somewhere. + */ + xor %eax,%eax + RET + +.Lqwords_exception: + # convert remaining qwords back into bytes to return to caller + shl $3, %rcx + and $7, %eax + add %rax,%rcx + jmp .Lexit + +.Lbytes_exception: + mov %eax,%ecx + jmp .Lexit + + _ASM_EXTABLE_UA(.Lqwords, .Lqwords_exception) + _ASM_EXTABLE_UA(.Lbytes, .Lbytes_exception) +SYM_FUNC_END(clear_user_original) +EXPORT_SYMBOL(clear_user_original) + +/* + * Alternative clear user-space when CPU feature X86_FEATURE_REP_GOOD is + * present. + * Input: + * rdi destination + * rcx count + * + * Output: + * rcx: uncleared bytes or 0 if successful. + */ +SYM_FUNC_START(clear_user_rep_good) + # call the original thing for less than a cacheline + cmp $64, %rcx + jb clear_user_original + +.Lprep: + # copy lower 32-bits for rest bytes + mov %ecx, %edx + shr $3, %rcx + jz .Lrep_good_rest_bytes + +.Lrep_good_qwords: + rep stosq + +.Lrep_good_rest_bytes: + and $7, %edx + jz .Lrep_good_exit + +.Lrep_good_bytes: + mov %edx, %ecx + rep stosb + +.Lrep_good_exit: + # see .Lexit comment above + xor %eax, %eax + RET + +.Lrep_good_qwords_exception: + # convert remaining qwords back into bytes to return to caller + shl $3, %rcx + and $7, %edx + add %rdx, %rcx + jmp .Lrep_good_exit + + _ASM_EXTABLE_UA(.Lrep_good_qwords, .Lrep_good_qwords_exception) + _ASM_EXTABLE_UA(.Lrep_good_bytes, .Lrep_good_exit) +SYM_FUNC_END(clear_user_rep_good) +EXPORT_SYMBOL(clear_user_rep_good) + +/* + * Alternative clear user-space when CPU feature X86_FEATURE_ERMS is present. + * Input: + * rdi destination + * rcx count + * + * Output: + * rcx: uncleared bytes or 0 if successful. + * + */ +SYM_FUNC_START(clear_user_erms) + # call the original thing for less than a cacheline + cmp $64, %rcx + jb clear_user_original + +.Lerms_bytes: + rep stosb + +.Lerms_exit: + xorl %eax,%eax + RET + + _ASM_EXTABLE_UA(.Lerms_bytes, .Lerms_exit) +SYM_FUNC_END(clear_user_erms) +EXPORT_SYMBOL(clear_user_erms) diff --git a/arch/x86/lib/iomem.c b/arch/x86/lib/iomem.c index 3e2f33fc33de24a77626e2c96702b79ec57fcd2c..e0411a3774d4959f1a869e93a3c8e4c7cc11c94c 100644 --- a/arch/x86/lib/iomem.c +++ b/arch/x86/lib/iomem.c @@ -1,6 +1,7 @@ #include #include #include +#include #define movs(type,to,from) \ asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory") @@ -37,6 +38,8 @@ static void string_memcpy_fromio(void *to, const volatile void __iomem *from, si n-=2; } rep_movs(to, (const void *)from, n); + /* KMSAN must treat values read from devices as initialized. */ + kmsan_unpoison_memory(to, n); } static void string_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) @@ -44,6 +47,8 @@ static void string_memcpy_toio(volatile void __iomem *to, const void *from, size if (unlikely(!n)) return; + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(from, n); /* Align any unaligned destination IO */ if (unlikely(1 & (unsigned long)to)) { movs("b", to, from); diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S index d0d7b9bc6cad394c2de1b241956336a7ed9e255e..dd8cd8831251fc62f77f00ebaeee5028a58b9f00 100644 --- a/arch/x86/lib/memcpy_64.S +++ b/arch/x86/lib/memcpy_64.S @@ -2,6 +2,7 @@ /* Copyright 2002 Andi Kleen */ #include +#include #include #include #include @@ -27,7 +28,7 @@ * Output: * rax original destination */ -SYM_FUNC_START(__memcpy) +SYM_TYPED_FUNC_START(__memcpy) ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ "jmp memcpy_erms", X86_FEATURE_ERMS diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index ad0139d254014244050e5873f09cfe231fe3d936..f1bb1861715621dd7136136f46bf223dc68e550d 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -44,7 +44,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) * called from other contexts. */ pagefault_disable(); - ret = __copy_from_user_inatomic(to, from, n); + ret = raw_copy_from_user(to, from, n); pagefault_enable(); return ret; diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 0ae6cf804197053e2bc1ab4f8bfc6eede4f13d84..6c1f8ac5e7214496d3ac953d6c64866574eefbba 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -14,46 +14,6 @@ * Zero Userspace */ -unsigned long __clear_user(void __user *addr, unsigned long size) -{ - long __d0; - might_fault(); - /* no memory constraint because it doesn't change any memory gcc knows - about */ - stac(); - asm volatile( - " testq %[size8],%[size8]\n" - " jz 4f\n" - " .align 16\n" - "0: movq $0,(%[dst])\n" - " addq $8,%[dst]\n" - " decl %%ecx ; jnz 0b\n" - "4: movq %[size1],%%rcx\n" - " testl %%ecx,%%ecx\n" - " jz 2f\n" - "1: movb $0,(%[dst])\n" - " incq %[dst]\n" - " decl %%ecx ; jnz 1b\n" - "2:\n" - - _ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1]) - _ASM_EXTABLE_UA(1b, 2b) - - : [size8] "=&c"(size), [dst] "=&D" (__d0) - : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr)); - clac(); - return size; -} -EXPORT_SYMBOL(__clear_user); - -unsigned long clear_user(void __user *to, unsigned long n) -{ - if (access_ok(to, n)) - return __clear_user(to, n); - return n; -} -EXPORT_SYMBOL(clear_user); - #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE /** * clean_cache_range - write back a cache range with CLWB diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index f8220fd2c169a4f2fc0a3db9ed0bde6b08e77fea..c80febc44cd2feed47bf3c419accb98b12c1e38b 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -4,19 +4,24 @@ KCOV_INSTRUMENT_tlb.o := n KCOV_INSTRUMENT_mem_encrypt.o := n KCOV_INSTRUMENT_mem_encrypt_amd.o := n KCOV_INSTRUMENT_mem_encrypt_identity.o := n +KCOV_INSTRUMENT_pgprot.o := n KASAN_SANITIZE_mem_encrypt.o := n KASAN_SANITIZE_mem_encrypt_amd.o := n KASAN_SANITIZE_mem_encrypt_identity.o := n +KASAN_SANITIZE_pgprot.o := n # Disable KCSAN entirely, because otherwise we get warnings that some functions # reference __initdata sections. KCSAN_SANITIZE := n +# Avoid recursion by not calling KMSAN hooks for CEA code. +KMSAN_SANITIZE_cpu_entry_area.o := n ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_mem_encrypt.o = -pg CFLAGS_REMOVE_mem_encrypt_amd.o = -pg CFLAGS_REMOVE_mem_encrypt_identity.o = -pg +CFLAGS_REMOVE_pgprot.o = -pg endif obj-y := init.o init_$(BITS).o fault.o ioremap.o extable.o mmap.o \ @@ -41,6 +46,9 @@ obj-$(CONFIG_HIGHMEM) += highmem_32.o KASAN_SANITIZE_kasan_init_$(BITS).o := n obj-$(CONFIG_KASAN) += kasan_init_$(BITS).o +KMSAN_SANITIZE_kmsan_shadow.o := n +obj-$(CONFIG_KMSAN) += kmsan_shadow.o + obj-$(CONFIG_MMIOTRACE) += mmiotrace.o mmiotrace-y := kmmio.o pf_in.o mmio-mod.o obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index fa71a5d12e872df8c2ce68049d6d58163c435388..7b0d4ab894c8bcd18c67034b0875e1a0640a724b 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -260,7 +260,7 @@ static noinline int vmalloc_fault(unsigned long address) } NOKPROBE_SYMBOL(vmalloc_fault); -void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +static void __arch_sync_kernel_mappings(unsigned long start, unsigned long end) { unsigned long addr; @@ -284,6 +284,27 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) } } +void arch_sync_kernel_mappings(unsigned long start, unsigned long end) +{ + __arch_sync_kernel_mappings(start, end); +#ifdef CONFIG_KMSAN + /* + * KMSAN maintains two additional metadata page mappings for the + * [VMALLOC_START, VMALLOC_END) range. These mappings start at + * KMSAN_VMALLOC_SHADOW_START and KMSAN_VMALLOC_ORIGIN_START and + * have to be synced together with the vmalloc memory mapping. + */ + if (start >= VMALLOC_START && end < VMALLOC_END) { + __arch_sync_kernel_mappings( + start - VMALLOC_START + KMSAN_VMALLOC_SHADOW_START, + end - VMALLOC_START + KMSAN_VMALLOC_SHADOW_START); + __arch_sync_kernel_mappings( + start - VMALLOC_START + KMSAN_VMALLOC_ORIGIN_START, + end - VMALLOC_START + KMSAN_VMALLOC_ORIGIN_START); + } +#endif +} + static bool low_pfn(unsigned long pfn) { return pfn < max_low_pfn; @@ -769,6 +790,8 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, unsigned long address, struct task_struct *tsk) { const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG; + /* This is a racy snapshot, but it's better than nothing. */ + int cpu = raw_smp_processor_id(); if (!unhandled_signal(tsk, SIGSEGV)) return; @@ -782,6 +805,14 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code, print_vma_addr(KERN_CONT " in ", regs->ip); + /* + * Dump the likely CPU where the fatal segfault happened. + * This can help identify faulty hardware. + */ + printk(KERN_CONT " likely on CPU %d (core %d, socket %d)", cpu, + topology_core_id(cpu), topology_physical_package_id(cpu)); + + printk(KERN_CONT "\n"); show_opcodes(regs, loglvl); diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index 82a042c0382480d56817dd686be79b131139607b..9121bc1b9453ab3236150e98855be90b91888f64 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -1054,7 +1054,7 @@ void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) } #ifdef CONFIG_SWAP -unsigned long max_swapfile_size(void) +unsigned long arch_max_swapfile_size(void) { unsigned long pages; diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 0fe690ebc269b529c5364d6148fd7e90e51c0a96..3f040c6e5d13adec988114a570b52e600853d328 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -90,6 +90,12 @@ DEFINE_ENTRY(pud, pud, init) DEFINE_ENTRY(pmd, pmd, init) DEFINE_ENTRY(pte, pte, init) +static inline pgprot_t prot_sethuge(pgprot_t prot) +{ + WARN_ON_ONCE(pgprot_val(prot) & _PAGE_PAT); + + return __pgprot(pgprot_val(prot) | _PAGE_PSE); +} /* * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the @@ -557,9 +563,8 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end, if (page_size_mask & (1<> PAGE_SHIFT, - __pgprot(pgprot_val(prot) | _PAGE_PSE)), + set_pmd_init(pmd, + pfn_pmd(paddr >> PAGE_SHIFT, prot_sethuge(prot)), init); spin_unlock(&init_mm.page_table_lock); paddr_last = paddr_next; @@ -644,12 +649,8 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end, if (page_size_mask & (1<> PAGE_SHIFT, - prot), + set_pud_init(pud, + pfn_pud(paddr >> PAGE_SHIFT, prot_sethuge(prot)), init); spin_unlock(&init_mm.page_table_lock); paddr_last = paddr_next; @@ -1287,7 +1288,7 @@ static void __init preallocate_vmalloc_pages(void) unsigned long addr; const char *lvl; - for (addr = VMALLOC_START; addr <= VMALLOC_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) { + for (addr = VMALLOC_START; addr <= VMEMORY_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) { pgd_t *pgd = pgd_offset_k(addr); p4d_t *p4d; pud_t *pud; diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 1ad0228f8ceb98032b6ea4e7fac0eb23e1c31d1f..78c5bc654cff5b054f1b3e63b21651e2e8163305 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -479,6 +480,8 @@ void iounmap(volatile void __iomem *addr) return; } + kmsan_iounmap_page_range((unsigned long)addr, + (unsigned long)addr + get_vm_area_size(p)); memtype_free(p->phys_addr, p->phys_addr + get_vm_area_size(p)); /* Finally remove it */ diff --git a/arch/x86/mm/kmsan_shadow.c b/arch/x86/mm/kmsan_shadow.c new file mode 100644 index 0000000000000000000000000000000000000000..bee2ec4a3bfa82bc02744b54015d73e02bc39dcc --- /dev/null +++ b/arch/x86/mm/kmsan_shadow.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * x86-specific bits of KMSAN shadow implementation. + * + * Copyright (C) 2022 Google LLC + * Author: Alexander Potapenko + */ + +#include +#include + +/* + * Addresses within the CPU entry area (including e.g. exception stacks) do not + * have struct page entries corresponding to them, so they need separate + * handling. + * arch_kmsan_get_meta_or_null() (declared in the header) maps the addresses in + * CPU entry area to addresses in cpu_entry_area_shadow/cpu_entry_area_origin. + */ +DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow); +DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin); diff --git a/arch/x86/mm/pat/cpa-test.c b/arch/x86/mm/pat/cpa-test.c index 0612a73638a81290556c9db3b82ec191ec37fb3d..423b21e80929ab12adfa742b50a931656f96d4cb 100644 --- a/arch/x86/mm/pat/cpa-test.c +++ b/arch/x86/mm/pat/cpa-test.c @@ -136,10 +136,10 @@ static int pageattr_test(void) failed += print_split(&sa); for (i = 0; i < NTEST; i++) { - unsigned long pfn = prandom_u32() % max_pfn_mapped; + unsigned long pfn = prandom_u32_max(max_pfn_mapped); addr[i] = (unsigned long)__va(pfn << PAGE_SHIFT); - len[i] = prandom_u32() % NPAGES; + len[i] = prandom_u32_max(NPAGES); len[i] = min_t(unsigned long, len[i], max_pfn_mapped - pfn - 1); if (len[i] == 0) diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index d5ef64ddd35e9f38c2a533e22d79c016fd6fcc55..66a209f7eb86da01b9c27537cb7a4183f657162b 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -62,6 +62,7 @@ static bool __read_mostly pat_bp_initialized; static bool __read_mostly pat_disabled = !IS_ENABLED(CONFIG_X86_PAT); +static bool __initdata pat_force_disabled = !IS_ENABLED(CONFIG_X86_PAT); static bool __read_mostly pat_bp_enabled; static bool __read_mostly pat_cm_initialized; @@ -86,6 +87,7 @@ void pat_disable(const char *msg_reason) static int __init nopat(char *str) { pat_disable("PAT support disabled via boot option."); + pat_force_disabled = true; return 0; } early_param("nopat", nopat); @@ -272,7 +274,7 @@ static void pat_ap_init(u64 pat) wrmsrl(MSR_IA32_CR_PAT, pat); } -void init_cache_modes(void) +void __init init_cache_modes(void) { u64 pat = 0; @@ -313,6 +315,12 @@ void init_cache_modes(void) */ pat = PAT(0, WB) | PAT(1, WT) | PAT(2, UC_MINUS) | PAT(3, UC) | PAT(4, WB) | PAT(5, WT) | PAT(6, UC_MINUS) | PAT(7, UC); + } else if (!pat_force_disabled && cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) { + /* + * Clearly PAT is enabled underneath. Allow pat_enabled() to + * reflect this. + */ + pat_bp_enabled = true; } __init_cache_modes(pat); diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 1abd5438f1269553cbb5dd7a4a55425a5abfbd60..97342c42dda8e3de10d95224013dc7b288e0cd2d 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -579,6 +579,46 @@ static inline pgprot_t static_protections(pgprot_t prot, unsigned long start, return __pgprot(pgprot_val(prot) & ~forbidden); } +/* + * Validate strict W^X semantics. + */ +static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long start, + unsigned long pfn, unsigned long npg) +{ + unsigned long end; + + /* + * 32-bit has some unfixable W+X issues, like EFI code + * and writeable data being in the same page. Disable + * detection and enforcement there. + */ + if (IS_ENABLED(CONFIG_X86_32)) + return new; + + /* Only verify when NX is supported: */ + if (!(__supported_pte_mask & _PAGE_NX)) + return new; + + if (!((pgprot_val(old) ^ pgprot_val(new)) & (_PAGE_RW | _PAGE_NX))) + return new; + + if ((pgprot_val(new) & (_PAGE_RW | _PAGE_NX)) != _PAGE_RW) + return new; + + end = start + npg * PAGE_SIZE - 1; + WARN_ONCE(1, "CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n", + (unsigned long long)pgprot_val(old), + (unsigned long long)pgprot_val(new), + start, end, pfn); + + /* + * For now, allow all permission change attempts by returning the + * attempted permissions. This can 'return old' to actively + * refuse the permission change at a later time. + */ + return new; +} + /* * Lookup the page table entry for a virtual address in a specific pgd. * Return a pointer to the entry and the level of the mapping. @@ -885,6 +925,8 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address, new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages, psize, CPA_DETECT); + new_prot = verify_rwx(old_prot, new_prot, lpaddr, old_pfn, numpages); + /* * If there is a conflict, split the large page. * @@ -1525,6 +1567,7 @@ repeat: if (level == PG_LEVEL_4K) { pte_t new_pte; + pgprot_t old_prot = pte_pgprot(old_pte); pgprot_t new_prot = pte_pgprot(old_pte); unsigned long pfn = pte_pfn(old_pte); @@ -1536,6 +1579,8 @@ repeat: new_prot = static_protections(new_prot, address, pfn, 1, 0, CPA_PROTECT); + new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1); + new_prot = pgprot_clear_protnone_bits(new_prot); /* @@ -1944,7 +1989,7 @@ int set_mce_nospec(unsigned long pfn) return rc; } -static int set_memory_present(unsigned long *addr, int numpages) +static int set_memory_p(unsigned long *addr, int numpages) { return change_page_attr_set(addr, numpages, __pgprot(_PAGE_PRESENT), 0); } @@ -1954,7 +1999,7 @@ int clear_mce_nospec(unsigned long pfn) { unsigned long addr = (unsigned long) pfn_to_kaddr(pfn); - return set_memory_present(&addr, 1); + return set_memory_p(&addr, 1); } EXPORT_SYMBOL_GPL(clear_mce_nospec); #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index a932d7712d851da5e7f17329a51e45aca2d2eee8..8525f2876fb409101e525bd3f43997fa66180c32 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -550,7 +550,7 @@ int ptep_test_and_clear_young(struct vm_area_struct *vma, return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { @@ -562,6 +562,9 @@ int pmdp_test_and_clear_young(struct vm_area_struct *vma, return ret; } +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE int pudp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pud_t *pudp) { diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c1f6c1c51d998ffafedb79e1cab167939f372aa4..99620428ad7851d7bd3bb3d4cd46d1c927d33ad8 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip) OPTIMIZER_HIDE_VAR(reg); emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip); } else { - EMIT2(0xFF, 0xE0 + reg); + EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ + if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS)) + EMIT1(0xCC); /* int3 */ } *pprog = prog; @@ -662,7 +664,7 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg, */ emit_mov_imm32(&prog, false, dst_reg, imm32_lo); } else { - /* movabsq %rax, imm64 */ + /* movabsq rax, imm64 */ EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); EMIT(imm32_lo, 4); EMIT(imm32_hi, 4); @@ -1751,34 +1753,60 @@ emit_jmp: static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args, int stack_size) { - int i; + int i, j, arg_size, nr_regs; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0; i < min(nr_args, 6); i++) - emit_stx(prog, bytes_to_bpf_size(m->arg_size[i]), - BPF_REG_FP, - i == 5 ? X86_REG_R9 : BPF_REG_1 + i, - -(stack_size - i * 8)); + for (i = 0, j = 0; i < min(nr_args, 6); i++) { + if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) { + nr_regs = (m->arg_size[i] + 7) / 8; + arg_size = 8; + } else { + nr_regs = 1; + arg_size = m->arg_size[i]; + } + + while (nr_regs) { + emit_stx(prog, bytes_to_bpf_size(arg_size), + BPF_REG_FP, + j == 5 ? X86_REG_R9 : BPF_REG_1 + j, + -(stack_size - j * 8)); + nr_regs--; + j++; + } + } } static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args, int stack_size) { - int i; + int i, j, arg_size, nr_regs; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10] * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8] */ - for (i = 0; i < min(nr_args, 6); i++) - emit_ldx(prog, bytes_to_bpf_size(m->arg_size[i]), - i == 5 ? X86_REG_R9 : BPF_REG_1 + i, - BPF_REG_FP, - -(stack_size - i * 8)); + for (i = 0, j = 0; i < min(nr_args, 6); i++) { + if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) { + nr_regs = (m->arg_size[i] + 7) / 8; + arg_size = 8; + } else { + nr_regs = 1; + arg_size = m->arg_size[i]; + } + + while (nr_regs) { + emit_ldx(prog, bytes_to_bpf_size(arg_size), + j == 5 ? X86_REG_R9 : BPF_REG_1 + j, + BPF_REG_FP, + -(stack_size - j * 8)); + nr_regs--; + j++; + } + } } static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, @@ -1810,6 +1838,9 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (p->aux->sleepable) { enter = __bpf_prog_enter_sleepable; exit = __bpf_prog_exit_sleepable; + } else if (p->type == BPF_PROG_TYPE_STRUCT_OPS) { + enter = __bpf_prog_enter_struct_ops; + exit = __bpf_prog_exit_struct_ops; } else if (p->expected_attach_type == BPF_LSM_CGROUP) { enter = __bpf_prog_enter_lsm_cgroup; exit = __bpf_prog_exit_lsm_cgroup; @@ -2013,13 +2044,14 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, - void *orig_call) + void *func_addr) { - int ret, i, nr_args = m->nr_args; + int ret, i, nr_args = m->nr_args, extra_nregs = 0; int regs_off, ip_off, args_off, stack_size = nr_args * 8, run_ctx_off; struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; + void *orig_call = func_addr; u8 **branches = NULL; u8 *prog; bool save_ret; @@ -2028,6 +2060,14 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i if (nr_args > 6) return -ENOTSUPP; + for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) { + if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) + extra_nregs += (m->arg_size[i] + 7) / 8 - 1; + } + if (nr_args + extra_nregs > 6) + return -ENOTSUPP; + stack_size += extra_nregs * 8; + /* Generated trampoline stack layout: * * RBP + 8 [ return address ] @@ -2040,7 +2080,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i * [ ... ] * RBP - regs_off [ reg_arg1 ] program's ctx pointer * - * RBP - args_off [ args count ] always + * RBP - args_off [ arg regs count ] always * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * @@ -2083,21 +2123,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */ EMIT1(0x53); /* push rbx */ - /* Store number of arguments of the traced function: - * mov rax, nr_args + /* Store number of argument registers of the traced function: + * mov rax, nr_args + extra_nregs * mov QWORD PTR [rbp - args_off], rax */ - emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args); + emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args + extra_nregs); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off); if (flags & BPF_TRAMP_F_IP_ARG) { /* Store IP address of the traced function: - * mov rax, QWORD PTR [rbp + 8] - * sub rax, X86_PATCH_SIZE + * movabsq rax, func_addr * mov QWORD PTR [rbp - ip_off], rax */ - emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8); - EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE); + emit_mov_imm64(&prog, BPF_REG_0, (long) func_addr >> 32, (u32) (long) func_addr); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off); } @@ -2209,7 +2247,7 @@ cleanup: return ret; } -static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) +static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) { u8 *jg_reloc, *prog = *pprog; int pivot, err, jg_bytes = 1; @@ -2225,12 +2263,12 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a]); err = emit_cond_near_jump(&prog, /* je func */ - (void *)progs[a], prog, + (void *)progs[a], image + (prog - buf), X86_JE); if (err) return err; - emit_indirect_jump(&prog, 2 /* rdx */, prog); + emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf)); *pprog = prog; return 0; @@ -2255,7 +2293,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) jg_reloc = prog; err = emit_bpf_dispatcher(&prog, a, a + pivot, /* emit lower_part */ - progs); + progs, image, buf); if (err) return err; @@ -2269,7 +2307,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes); err = emit_bpf_dispatcher(&prog, a + pivot + 1, /* emit upper_part */ - b, progs); + b, progs, image, buf); if (err) return err; @@ -2289,12 +2327,12 @@ static int cmp_ips(const void *a, const void *b) return 0; } -int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs) +int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs) { - u8 *prog = image; + u8 *prog = buf; sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL); - return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs); + return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf); } struct x64_jit_data { diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 6e598bd78eef3520c816c7cfb1da525af3d712fe..ebc98a68c4005b9dc61f704da788ec29d518360c 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -169,7 +169,7 @@ static void __init do_add_efi_memmap(void) } /* - * Given add_efi_memmap defaults to 0 and there there is no alternative + * Given add_efi_memmap defaults to 0 and there is no alternative * e820 mechanism for soft-reserved memory, import the full EFI memory * map if soft reservations are present and enabled. Otherwise, the * mechanism to disable the kernel's consideration of EFI_MEMORY_SP is diff --git a/arch/x86/platform/efi/efi_64.c b/arch/x86/platform/efi/efi_64.c index 1f3675453a57ab61bbae2f1074135246c8559fa0..b36596bf0fc38f4fbf907f5c19147425a2a4360d 100644 --- a/arch/x86/platform/efi/efi_64.c +++ b/arch/x86/platform/efi/efi_64.c @@ -176,7 +176,8 @@ virt_to_phys_or_null_size(void *va, unsigned long size) int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) { - unsigned long pfn, text, pf, rodata; + extern const u8 __efi64_thunk_ret_tramp[]; + unsigned long pfn, text, pf, rodata, tramp; struct page *page; unsigned npages; pgd_t *pgd = efi_mm.pgd; @@ -238,11 +239,9 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) npages = (_etext - _text) >> PAGE_SHIFT; text = __pa(_text); - pfn = text >> PAGE_SHIFT; - pf = _PAGE_ENC; - if (kernel_map_pages_in_pgd(pgd, pfn, text, npages, pf)) { - pr_err("Failed to map kernel text 1:1\n"); + if (kernel_unmap_pages_in_pgd(pgd, text, npages)) { + pr_err("Failed to unmap kernel text 1:1 mapping\n"); return 1; } @@ -256,6 +255,15 @@ int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages) return 1; } + tramp = __pa(__efi64_thunk_ret_tramp); + pfn = tramp >> PAGE_SHIFT; + + pf = _PAGE_ENC; + if (kernel_map_pages_in_pgd(pgd, pfn, tramp, 1, pf)) { + pr_err("Failed to map mixed mode return trampoline\n"); + return 1; + } + return 0; } diff --git a/arch/x86/platform/efi/efi_thunk_64.S b/arch/x86/platform/efi/efi_thunk_64.S index 4e5257a4811b28e980b9bd48f813be1448154273..c4b1144f99f6362aa4ac56a3eedc355c9722cb04 100644 --- a/arch/x86/platform/efi/efi_thunk_64.S +++ b/arch/x86/platform/efi/efi_thunk_64.S @@ -23,7 +23,6 @@ #include #include #include -#include .text .code64 @@ -73,10 +72,18 @@ STACK_FRAME_NON_STANDARD __efi64_thunk pushq %rdi /* EFI runtime service address */ lretq + // This return instruction is not needed for correctness, as it will + // never be reached. It only exists to make objtool happy, which will + // otherwise complain about unreachable instructions in the callers. + RET +SYM_FUNC_END(__efi64_thunk) + + .section ".rodata", "a", @progbits + .balign 16 +SYM_DATA_START(__efi64_thunk_ret_tramp) 1: movq 0x20(%rsp), %rsp pop %rbx pop %rbp - ANNOTATE_UNRET_SAFE ret int3 @@ -84,7 +91,7 @@ STACK_FRAME_NON_STANDARD __efi64_thunk 2: pushl $__KERNEL_CS pushl %ebp lret -SYM_FUNC_END(__efi64_thunk) +SYM_DATA_END(__efi64_thunk_ret_tramp) .bss .balign 8 diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index 31c634a228183e31d16d3c27ba5634e51796bc3f..58a200dc762d6da78808d9974aa3914186bc7def 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS) endif +ifdef CONFIG_CFI_CLANG +PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI) +endif + CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE) CFLAGS_purgatory.o += $(PURGATORY_CFLAGS) diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index 83f1b6a56449fedd03d0976a63e2e5ecd886d775..f614009d3e4e2eb944862bacb260b0da088f5c0a 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -10,6 +10,7 @@ # Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n KCSAN_SANITIZE := n +KMSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c index e2c5b296120d65a45d7a0541b999c65b406c6cf9..2925074b9a588dec46961384bbd9668e013ce43b 100644 --- a/arch/x86/tools/relocs.c +++ b/arch/x86/tools/relocs.c @@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { "^(xen_irq_disable_direct_reloc$|" "xen_save_fl_direct_reloc$|" "VDSO|" + "__kcfi_typeid_|" "__crc_)", /* diff --git a/arch/x86/um/shared/sysdep/syscalls_32.h b/arch/x86/um/shared/sysdep/syscalls_32.h index 68fd2cf526fd7e47f0dd823db12e8389b01ed6b9..f6e9f84397e792d5f3ba3102f6c10cfe7b57aae6 100644 --- a/arch/x86/um/shared/sysdep/syscalls_32.h +++ b/arch/x86/um/shared/sysdep/syscalls_32.h @@ -6,10 +6,9 @@ #include #include -typedef long syscall_handler_t(struct pt_regs); +typedef long syscall_handler_t(struct syscall_args); extern syscall_handler_t *sys_call_table[]; #define EXECUTE_SYSCALL(syscall, regs) \ - ((long (*)(struct syscall_args)) \ - (*sys_call_table[syscall]))(SYSCALL_ARGS(®s->regs)) + ((*sys_call_table[syscall]))(SYSCALL_ARGS(®s->regs)) diff --git a/arch/x86/um/tls_32.c b/arch/x86/um/tls_32.c index ac8eee093f9cd0fa112fe89e945202eb60203a84..66162eafd8e8f8770935451b555c644dc5aca688 100644 --- a/arch/x86/um/tls_32.c +++ b/arch/x86/um/tls_32.c @@ -65,9 +65,6 @@ static int get_free_idx(struct task_struct* task) struct thread_struct *t = &task->thread; int idx; - if (!t->arch.tls_array) - return GDT_ENTRY_TLS_MIN; - for (idx = 0; idx < GDT_ENTRY_TLS_ENTRIES; idx++) if (!t->arch.tls_array[idx].present) return idx + GDT_ENTRY_TLS_MIN; @@ -240,9 +237,6 @@ static int get_tls_entry(struct task_struct *task, struct user_desc *info, { struct thread_struct *t = &task->thread; - if (!t->arch.tls_array) - goto clear; - if (idx < GDT_ENTRY_TLS_MIN || idx > GDT_ENTRY_TLS_MAX) return -EINVAL; diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile index 8c0396fd0e6f691e72a36bad703cf4172f8693da..6fbe97c52c991e2f71cd917d1cbaa6b4bf9ffcf4 100644 --- a/arch/x86/um/vdso/Makefile +++ b/arch/x86/um/vdso/Makefile @@ -65,7 +65,7 @@ quiet_cmd_vdso = VDSO $@ -Wl,-T,$(filter %.lds,$^) $(filter %.o,$^) && \ sh $(srctree)/$(src)/checkundef.sh '$(NM)' '$@' -VDSO_LDFLAGS = -fPIC -shared -Wl,--hash-style=sysv +VDSO_LDFLAGS = -fPIC -shared -Wl,--hash-style=sysv -z noexecstack GCOV_PROFILE := n # diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index 85246dd9faa14cf55c947eacc227d6ad76e8a916..9b1ec5d8c99c8da127c9615f84b2fe66c6e32c72 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -92,3 +92,12 @@ config XEN_DOM0 select X86_X2APIC if XEN_PVH && X86_64 help Support running as a Xen Dom0 guest. + +config XEN_PV_MSR_SAFE + bool "Always use safe MSR accesses in PV guests" + default y + depends on XEN_PV + help + Use safe (not faulting) MSR access functions even if the MSR access + should not fault anyway. + The default can be changed by using the "xen_msr_safe" boot parameter. diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index 1c1ac418484b5fe02cf84f43f6608074dbe0e751..c1cd28e915a3a9a853d8e572cf516e10affdd8db 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -212,7 +212,7 @@ static void __init xen_hvm_guest_init(void) return; if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT)) - virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); + virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc); init_hvm_pv_info(); diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index 0ed2e487a693fa307831e3fd5ed5c8bacc390c05..f82857e488152a7377690f9d37a3a72e733c58d7 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -108,11 +108,21 @@ struct tls_descs { */ static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc); +static __read_mostly bool xen_msr_safe = IS_ENABLED(CONFIG_XEN_PV_MSR_SAFE); + +static int __init parse_xen_msr_safe(char *str) +{ + if (str) + return strtobool(str, &xen_msr_safe); + return -EINVAL; +} +early_param("xen_msr_safe", parse_xen_msr_safe); + static void __init xen_pv_init_platform(void) { /* PV guests can't operate virtio devices without grants. */ if (IS_ENABLED(CONFIG_XEN_VIRTIO)) - virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc); + virtio_set_mem_acc_cb(xen_virtio_restricted_mem_acc); populate_extra_pte(fix_to_virt(FIX_PARAVIRT_BOOTMAP)); @@ -765,6 +775,7 @@ static void xen_load_idt(const struct desc_ptr *desc) { static DEFINE_SPINLOCK(lock); static struct trap_info traps[257]; + static const struct trap_info zero = { }; unsigned out; trace_xen_cpu_load_idt(desc); @@ -774,7 +785,7 @@ static void xen_load_idt(const struct desc_ptr *desc) memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc)); out = xen_convert_trap_info(desc, traps, false); - memset(&traps[out], 0, sizeof(traps[0])); + traps[out] = zero; xen_mc_flush(); if (HYPERVISOR_set_trap_table(traps)) @@ -916,14 +927,18 @@ static void xen_write_cr4(unsigned long cr4) native_write_cr4(cr4); } -static u64 xen_read_msr_safe(unsigned int msr, int *err) +static u64 xen_do_read_msr(unsigned int msr, int *err) { - u64 val; + u64 val = 0; /* Avoid uninitialized value for safe variant. */ if (pmu_msr_read(msr, &val, err)) return val; - val = native_read_msr_safe(msr, err); + if (err) + val = native_read_msr_safe(msr, err); + else + val = native_read_msr(msr); + switch (msr) { case MSR_IA32_APICBASE: val &= ~X2APIC_ENABLE; @@ -932,23 +947,39 @@ static u64 xen_read_msr_safe(unsigned int msr, int *err) return val; } -static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high) +static void set_seg(unsigned int which, unsigned int low, unsigned int high, + int *err) { - int ret; - unsigned int which; - u64 base; + u64 base = ((u64)high << 32) | low; - ret = 0; + if (HYPERVISOR_set_segment_base(which, base) == 0) + return; + if (err) + *err = -EIO; + else + WARN(1, "Xen set_segment_base(%u, %llx) failed\n", which, base); +} + +/* + * Support write_msr_safe() and write_msr() semantics. + * With err == NULL write_msr() semantics are selected. + * Supplying an err pointer requires err to be pre-initialized with 0. + */ +static void xen_do_write_msr(unsigned int msr, unsigned int low, + unsigned int high, int *err) +{ switch (msr) { - case MSR_FS_BASE: which = SEGBASE_FS; goto set; - case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set; - case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set; - - set: - base = ((u64)high << 32) | low; - if (HYPERVISOR_set_segment_base(which, base) != 0) - ret = -EIO; + case MSR_FS_BASE: + set_seg(SEGBASE_FS, low, high, err); + break; + + case MSR_KERNEL_GS_BASE: + set_seg(SEGBASE_GS_USER, low, high, err); + break; + + case MSR_GS_BASE: + set_seg(SEGBASE_GS_KERNEL, low, high, err); break; case MSR_STAR: @@ -964,31 +995,42 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high) break; default: - if (!pmu_msr_write(msr, low, high, &ret)) - ret = native_write_msr_safe(msr, low, high); + if (!pmu_msr_write(msr, low, high, err)) { + if (err) + *err = native_write_msr_safe(msr, low, high); + else + native_write_msr(msr, low, high); + } } +} - return ret; +static u64 xen_read_msr_safe(unsigned int msr, int *err) +{ + return xen_do_read_msr(msr, err); +} + +static int xen_write_msr_safe(unsigned int msr, unsigned int low, + unsigned int high) +{ + int err = 0; + + xen_do_write_msr(msr, low, high, &err); + + return err; } static u64 xen_read_msr(unsigned int msr) { - /* - * This will silently swallow a #GP from RDMSR. It may be worth - * changing that. - */ int err; - return xen_read_msr_safe(msr, &err); + return xen_do_read_msr(msr, xen_msr_safe ? &err : NULL); } static void xen_write_msr(unsigned int msr, unsigned low, unsigned high) { - /* - * This will silently swallow a #GP from WRMSR. It may be worth - * changing that. - */ - xen_write_msr_safe(msr, low, high); + int err; + + xen_do_write_msr(msr, low, high, xen_msr_safe ? &err : NULL); } /* This is called once we have the cpu_possible_mask */ diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 21ecbe754cb2f5b531bce3aeac8516128064f761..68aff138287282b7e8e878f644c80bee5cad54a5 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -131,6 +131,10 @@ static inline uint32_t get_fam15h_addr(u32 addr) static inline bool is_amd_pmu_msr(unsigned int msr) { + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && + boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) + return false; + if ((msr >= MSR_F15H_PERF_CTL && msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) || (msr >= MSR_K7_EVNTSEL0 && @@ -140,10 +144,15 @@ static inline bool is_amd_pmu_msr(unsigned int msr) return false; } -static int is_intel_pmu_msr(u32 msr_index, int *type, int *index) +static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index) { u32 msr_index_pmc; + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL && + boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR && + boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN) + return false; + switch (msr_index) { case MSR_CORE_PERF_FIXED_CTR_CTRL: case MSR_IA32_DS_AREA: @@ -290,48 +299,52 @@ static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) return false; } +static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read, + bool *emul) +{ + int type, index; + + if (is_amd_pmu_msr(msr)) + *emul = xen_amd_pmu_emulate(msr, val, is_read); + else if (is_intel_pmu_msr(msr, &type, &index)) + *emul = xen_intel_pmu_emulate(msr, val, type, index, is_read); + else + return false; + + return true; +} + bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) { - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { - if (is_amd_pmu_msr(msr)) { - if (!xen_amd_pmu_emulate(msr, val, 1)) - *val = native_read_msr_safe(msr, err); - return true; - } - } else { - int type, index; + bool emulated; - if (is_intel_pmu_msr(msr, &type, &index)) { - if (!xen_intel_pmu_emulate(msr, val, type, index, 1)) - *val = native_read_msr_safe(msr, err); - return true; - } + if (!pmu_msr_chk_emulated(msr, val, true, &emulated)) + return false; + + if (!emulated) { + *val = err ? native_read_msr_safe(msr, err) + : native_read_msr(msr); } - return false; + return true; } bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) { uint64_t val = ((uint64_t)high << 32) | low; + bool emulated; - if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) { - if (is_amd_pmu_msr(msr)) { - if (!xen_amd_pmu_emulate(msr, &val, 0)) - *err = native_write_msr_safe(msr, low, high); - return true; - } - } else { - int type, index; + if (!pmu_msr_chk_emulated(msr, &val, false, &emulated)) + return false; - if (is_intel_pmu_msr(msr, &type, &index)) { - if (!xen_intel_pmu_emulate(msr, &val, type, index, 0)) - *err = native_write_msr_safe(msr, low, high); - return true; - } + if (!emulated) { + if (err) + *err = native_write_msr_safe(msr, low, high); + else + native_write_msr(msr, low, high); } - return false; + return true; } static unsigned long long xen_amd_read_pmc(int counter) diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index ba7af2eca755b7f08e4fe9a4c31f350defe80c2d..480be82e9b7be3c3e6856da908afa6cb8eebc4c8 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -179,7 +179,7 @@ static void __init _get_smp_config(unsigned int early) * hypercall to expand the max number of VCPUs an already * running guest has. So cap it up to X. */ if (subtract) - nr_cpu_ids = nr_cpu_ids - subtract; + set_nr_cpu_ids(nr_cpu_ids - subtract); #endif } diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index 12ac277282bab31ba8c5eba7b246f64bddecbb78..bcb0c5d2abc2fe78337eec71fab09d11d212898a 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -771,7 +771,7 @@ config HIGHMEM If unsure, say Y. -config FORCE_MAX_ZONEORDER +config ARCH_FORCE_MAX_ORDER int "Maximum zone order" default "11" help diff --git a/arch/xtensa/Makefile b/arch/xtensa/Makefile index 5097caa7bf0cfa114c761c2af685bdb5558ee008..bfd8e433ed621e89366ddaabb80bb9d0672d2b89 100644 --- a/arch/xtensa/Makefile +++ b/arch/xtensa/Makefile @@ -55,8 +55,6 @@ KBUILD_CPPFLAGS += $(patsubst %,-I$(srctree)/%include,$(vardirs) $(plfdirs)) KBUILD_DEFCONFIG := iss_defconfig -head-y := arch/xtensa/kernel/head.o - libs-y += arch/xtensa/lib/ boot := arch/xtensa/boot diff --git a/arch/xtensa/configs/audio_kc705_defconfig b/arch/xtensa/configs/audio_kc705_defconfig index 3be62da8089b8e6a821c7dec61db4e328229dd84..ef0ebcfbccf910991c13d5b72b96eb6acd693783 100644 --- a/arch/xtensa/configs/audio_kc705_defconfig +++ b/arch/xtensa/configs/audio_kc705_defconfig @@ -120,7 +120,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/xtensa/configs/cadence_csp_defconfig b/arch/xtensa/configs/cadence_csp_defconfig index fc240737b14de920e5e42157db61fa7116e948a6..2665962d247a727812265449a506503e0042c61c 100644 --- a/arch/xtensa/configs/cadence_csp_defconfig +++ b/arch/xtensa/configs/cadence_csp_defconfig @@ -100,7 +100,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/xtensa/configs/generic_kc705_defconfig b/arch/xtensa/configs/generic_kc705_defconfig index e9d6b6f6eca1151697a014a20be7b77265dd7a98..236c7f23cc10aff29a1a3c7c4cc57d1c317133ec 100644 --- a/arch/xtensa/configs/generic_kc705_defconfig +++ b/arch/xtensa/configs/generic_kc705_defconfig @@ -107,7 +107,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/xtensa/configs/nommu_kc705_defconfig b/arch/xtensa/configs/nommu_kc705_defconfig index fcb620ef3799722fc1db4a735bb2cb8e8b9fb4fe..8263da9e078d751d91a4201c867934a750435247 100644 --- a/arch/xtensa/configs/nommu_kc705_defconfig +++ b/arch/xtensa/configs/nommu_kc705_defconfig @@ -105,7 +105,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y # CONFIG_FRAME_POINTER is not set CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_VM=y diff --git a/arch/xtensa/configs/smp_lx200_defconfig b/arch/xtensa/configs/smp_lx200_defconfig index a47c85638ec1116671da373614cc705d0d7e8b46..7bdffa3a69c606f587468a66329f6166aac8749c 100644 --- a/arch/xtensa/configs/smp_lx200_defconfig +++ b/arch/xtensa/configs/smp_lx200_defconfig @@ -111,7 +111,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_VM=y CONFIG_LOCKUP_DETECTOR=y diff --git a/arch/xtensa/configs/virt_defconfig b/arch/xtensa/configs/virt_defconfig index 6d1387dfa96fcd1d4aa91b6f70773cf162b706e2..98acb7191cb773abb3a3bc2f3f826ea9a2fb6571 100644 --- a/arch/xtensa/configs/virt_defconfig +++ b/arch/xtensa/configs/virt_defconfig @@ -97,7 +97,7 @@ CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_FONTS=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y # CONFIG_SCHED_DEBUG is not set CONFIG_SCHEDSTATS=y diff --git a/arch/xtensa/configs/xip_kc705_defconfig b/arch/xtensa/configs/xip_kc705_defconfig index 062148e1713513ba167f31cfb7268039dcf5a1c1..1c3cebaaa71ba1a3dadeb55d5b8b0527ea023ccf 100644 --- a/arch/xtensa/configs/xip_kc705_defconfig +++ b/arch/xtensa/configs/xip_kc705_defconfig @@ -102,7 +102,7 @@ CONFIG_CRYPTO_LZO=y CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/xtensa/include/asm/elf.h b/arch/xtensa/include/asm/elf.h index 909a6ab4f22b43ec5ada6b5fc9312144d6da8884..ffcf1ada19c6623d772a815ab673846a57c1eeb7 100644 --- a/arch/xtensa/include/asm/elf.h +++ b/arch/xtensa/include/asm/elf.h @@ -93,6 +93,10 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #define elf_check_arch(x) ( ( (x)->e_machine == EM_XTENSA ) || \ ( (x)->e_machine == EM_XTENSA_OLD ) ) +#define ELFOSABI_XTENSA_FDPIC 65 +#define elf_check_fdpic(x) ((x)->e_ident[EI_OSABI] == ELFOSABI_XTENSA_FDPIC) +#define ELF_FDPIC_CORE_EFLAGS 0 + /* * These are used to set parameters in the core dumps. */ @@ -153,10 +157,22 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; */ #define ELF_PLAT_INIT(_r, load_addr) \ - do { _r->areg[0]=0; /*_r->areg[1]=0;*/ _r->areg[2]=0; _r->areg[3]=0; \ - _r->areg[4]=0; _r->areg[5]=0; _r->areg[6]=0; _r->areg[7]=0; \ - _r->areg[8]=0; _r->areg[9]=0; _r->areg[10]=0; _r->areg[11]=0; \ - _r->areg[12]=0; _r->areg[13]=0; _r->areg[14]=0; _r->areg[15]=0; \ + do { \ + (_r)->areg[0] = 0; /*(_r)->areg[1] = 0;*/ \ + (_r)->areg[2] = 0; (_r)->areg[3] = 0; \ + (_r)->areg[4] = 0; (_r)->areg[5] = 0; \ + (_r)->areg[6] = 0; (_r)->areg[7] = 0; \ + (_r)->areg[8] = 0; (_r)->areg[9] = 0; \ + (_r)->areg[10] = 0; (_r)->areg[11] = 0; \ + (_r)->areg[12] = 0; (_r)->areg[13] = 0; \ + (_r)->areg[14] = 0; (_r)->areg[15] = 0; \ + } while (0) + +#define ELF_FDPIC_PLAT_INIT(_r, _exec_map_addr, _interp_map_addr, dynamic_addr) \ + do { \ + (_r)->areg[4] = _exec_map_addr; \ + (_r)->areg[5] = _interp_map_addr; \ + (_r)->areg[6] = dynamic_addr; \ } while (0) typedef struct { diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h index 76bc63127c66ef02403d4b7d5c67bd28a0d10456..228e4dff5fb2d3b7fb08e59743915d8223940dc8 100644 --- a/arch/xtensa/include/asm/processor.h +++ b/arch/xtensa/include/asm/processor.h @@ -205,9 +205,12 @@ struct thread_struct { #define start_thread(regs, new_pc, new_sp) \ do { \ unsigned long syscall = (regs)->syscall; \ + unsigned long current_aregs[16]; \ + memcpy(current_aregs, (regs)->areg, sizeof(current_aregs)); \ memset((regs), 0, sizeof(*(regs))); \ (regs)->pc = (new_pc); \ (regs)->ps = USER_PS_VALUE; \ + memcpy((regs)->areg, current_aregs, sizeof(current_aregs)); \ (regs)->areg[1] = (new_sp); \ (regs)->areg[0] = 0; \ (regs)->wmask = 1; \ @@ -221,9 +224,6 @@ struct thread_struct { struct task_struct; struct mm_struct; -/* Free all resources held by a thread. */ -#define release_thread(thread) do { } while(0) - extern unsigned long __get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->pc) diff --git a/arch/xtensa/include/uapi/asm/mman.h b/arch/xtensa/include/uapi/asm/mman.h index 7966a58af472a18b726a1ecb2698ac6de17989d1..1ff0c858544fa81e7164cf602b3c2ef4f9f17df6 100644 --- a/arch/xtensa/include/uapi/asm/mman.h +++ b/arch/xtensa/include/uapi/asm/mman.h @@ -111,6 +111,8 @@ #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/arch/xtensa/include/uapi/asm/ptrace.h b/arch/xtensa/include/uapi/asm/ptrace.h index 50db3e0a634130806fbaaba36366e150c8c09203..9115e86ebc75fc052b430278cf838d01db39a4a0 100644 --- a/arch/xtensa/include/uapi/asm/ptrace.h +++ b/arch/xtensa/include/uapi/asm/ptrace.h @@ -37,6 +37,10 @@ #define PTRACE_SETXTREGS 19 #define PTRACE_GETHBPREGS 20 #define PTRACE_SETHBPREGS 21 +#define PTRACE_GETFDPIC 22 + +#define PTRACE_GETFDPIC_EXEC 0 +#define PTRACE_GETFDPIC_INTERP 1 #ifndef __ASSEMBLY__ diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile index 897c1c7410589826be38626288be77afa3dc41e9..f28b8e3d717eee087bad79ee1db79bf1d9928507 100644 --- a/arch/xtensa/kernel/Makefile +++ b/arch/xtensa/kernel/Makefile @@ -3,9 +3,9 @@ # Makefile for the Linux/Xtensa kernel. # -extra-y := head.o vmlinux.lds +extra-y := vmlinux.lds -obj-y := align.o coprocessor.o entry.o irq.o platform.o process.o \ +obj-y := head.o align.o coprocessor.o entry.o irq.o platform.o process.o \ ptrace.o setup.o signal.o stacktrace.o syscall.o time.o traps.o \ vectors.o diff --git a/arch/xtensa/kernel/syscall.c b/arch/xtensa/kernel/syscall.c index 201356faa7e6e01d19bedf684194f44f076b6409..b3c2450d6f239e6b8f534eba2969b8758d71a1fa 100644 --- a/arch/xtensa/kernel/syscall.c +++ b/arch/xtensa/kernel/syscall.c @@ -58,6 +58,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct vm_area_struct *vmm; + struct vma_iterator vmi; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -79,15 +80,20 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, else addr = PAGE_ALIGN(addr); - for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { - /* At this point: (!vmm || addr < vmm->vm_end). */ - if (TASK_SIZE - len < addr) - return -ENOMEM; - if (!vmm || addr + len <= vm_start_gap(vmm)) - return addr; + vma_iter_init(&vmi, current->mm, addr); + for_each_vma(vmi, vmm) { + /* At this point: (addr < vmm->vm_end). */ + if (addr + len <= vm_start_gap(vmm)) + break; + addr = vmm->vm_end; if (flags & MAP_SHARED) addr = COLOUR_ALIGN(addr, pgoff); } + + if (TASK_SIZE - len < addr) + return -ENOMEM; + + return addr; } #endif diff --git a/block/bdev.c b/block/bdev.c index ce05175e71cea49ef0a9133c99395026f1426c8c..d699ecdb32604e7cb3f3ddcb9b71fa9762a3ba03 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "../fs/internal.h" #include "blk.h" @@ -1069,3 +1070,25 @@ void sync_bdevs(bool wait) spin_unlock(&blockdev_superblock->s_inode_list_lock); iput(old_inode); } + +/* + * Handle STATX_DIOALIGN for block devices. + * + * Note that the inode passed to this is the inode of a block device node file, + * not the block device's internal inode. Therefore it is *not* valid to use + * I_BDEV() here; the block device has to be looked up by i_rdev instead. + */ +void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) +{ + struct block_device *bdev; + + bdev = blkdev_get_no_open(inode->i_rdev); + if (!bdev) + return; + + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + stat->dio_offset_align = bdev_logical_block_size(bdev); + stat->result_mask |= STATX_DIOALIGN; + + blkdev_put_no_open(bdev); +} diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 30b15a9a47c4f0f5c09b5864753a1f0fe7c62a68..144bca006463dbbf83e6b51f7c719e3fb3b6f3e8 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -254,17 +254,12 @@ void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, #else /* CONFIG_BFQ_CGROUP_DEBUG */ -void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, - blk_opf_t opf) { } void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { } void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { } void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, u64 io_start_time_ns, blk_opf_t opf) { } void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } -void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } -void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } -void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } #endif /* CONFIG_BFQ_CGROUP_DEBUG */ diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index c740b41fe0a49981c457d3209f39c27037dbe71f..7ea427817f7f5fdcff8630cced9e67023d548b19 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -1925,7 +1925,7 @@ static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, bfqq->service_from_backlogged = 0; bfq_clear_bfqq_softrt_update(bfqq); - bfq_add_bfqq_busy(bfqd, bfqq); + bfq_add_bfqq_busy(bfqq); /* * Expire in-service queue if preemption may be needed for @@ -2419,7 +2419,7 @@ static void bfq_remove_request(struct request_queue *q, bfqq->next_rq = NULL; if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { - bfq_del_bfqq_busy(bfqd, bfqq, false); + bfq_del_bfqq_busy(bfqq, false); /* * bfqq emptied. In normal operation, when * bfqq is empty, bfqq->entity.service and @@ -3098,7 +3098,7 @@ void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) */ if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq != bfqd->in_service_queue) - bfq_del_bfqq_busy(bfqd, bfqq, false); + bfq_del_bfqq_busy(bfqq, false); bfq_reassign_last_bfqq(bfqq, NULL); @@ -3908,7 +3908,7 @@ static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, */ bfqq->budget_timeout = jiffies; - bfq_del_bfqq_busy(bfqd, bfqq, true); + bfq_del_bfqq_busy(bfqq, true); } else { bfq_requeue_bfqq(bfqd, bfqq, true); /* @@ -5255,9 +5255,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) struct hlist_node *n; struct bfq_group *bfqg = bfqq_group(bfqq); - if (bfqq->bfqd) - bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", - bfqq, bfqq->ref); + bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); bfqq->ref--; if (bfqq->ref) @@ -5321,7 +5319,7 @@ void bfq_put_queue(struct bfq_queue *bfqq) hlist_del_init(&item->woken_list_node); } - if (bfqq->bfqd && bfqq->bfqd->last_completed_rq_bfqq == bfqq) + if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) bfqq->bfqd->last_completed_rq_bfqq = NULL; kmem_cache_free(bfq_pool, bfqq); diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index ad8e513d7e8764dfa39f6649e8363f1cd5ac642c..64ee618064ba14f5e9b5608ede0249cb4407615f 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -993,20 +993,23 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); /* ---------------- cgroups-support interface ---------------- */ void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq); -void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, - blk_opf_t opf); void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf); void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf); void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns, u64 io_start_time_ns, blk_opf_t opf); void bfqg_stats_update_dequeue(struct bfq_group *bfqg); -void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); -void bfqg_stats_update_idle_time(struct bfq_group *bfqg); void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg); -void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, struct bfq_group *bfqg); +#ifdef CONFIG_BFQ_CGROUP_DEBUG +void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq, + blk_opf_t opf); +void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); +void bfqg_stats_update_idle_time(struct bfq_group *bfqg); +void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg); +#endif + void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg); void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio); void bfq_end_wr_async(struct bfq_data *bfqd); @@ -1077,9 +1080,8 @@ void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, bool expiration); -void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, - bool expiration); -void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq); +void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration); +void bfq_add_bfqq_busy(struct bfq_queue *bfqq); /* --------------- end of interface of B-WF2Q+ ---------------- */ diff --git a/block/bfq-wf2q.c b/block/bfq-wf2q.c index 983413cdefad882725954a49da797f8c77e296cf..8fc3da4c23bb9410bc80a1d3083209da86edcbf7 100644 --- a/block/bfq-wf2q.c +++ b/block/bfq-wf2q.c @@ -1651,9 +1651,10 @@ void bfq_requeue_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, * the service tree. As a special case, it can be invoked during an * expiration. */ -void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, - bool expiration) +void bfq_del_bfqq_busy(struct bfq_queue *bfqq, bool expiration) { + struct bfq_data *bfqd = bfqq->bfqd; + bfq_log_bfqq(bfqd, bfqq, "del from busy"); bfq_clear_bfqq_busy(bfqq); @@ -1674,8 +1675,10 @@ void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, /* * Called when an inactive queue receives a new request. */ -void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) +void bfq_add_bfqq_busy(struct bfq_queue *bfqq) { + struct bfq_data *bfqd = bfqq->bfqd; + bfq_log_bfqq(bfqd, bfqq, "add to busy"); bfq_activate_bfqq(bfqd, bfqq); diff --git a/block/bio.c b/block/bio.c index 3d3a2678fea251e8cfc4426a884e1c13d69a8407..633a902468ec7f10f91f18edb5e6e083eb3f6652 100644 --- a/block/bio.c +++ b/block/bio.c @@ -567,7 +567,7 @@ EXPORT_SYMBOL(bio_alloc_bioset); * be reused by calling bio_uninit() before calling bio_init() again. * * Note that unlike bio_alloc() or bio_alloc_bioset() allocations from this - * function are not backed by a mempool can can fail. Do not use this function + * function are not backed by a mempool can fail. Do not use this function * for allocations in the file system I/O path. * * Returns: Pointer to new bio on success, NULL on failure. @@ -760,8 +760,6 @@ EXPORT_SYMBOL(bio_put); static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) { bio_set_flag(bio, BIO_CLONED); - if (bio_flagged(bio_src, BIO_THROTTLED)) - bio_set_flag(bio, BIO_THROTTLED); bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_iter = bio_src->bi_iter; @@ -869,6 +867,8 @@ static inline bool page_is_mergeable(const struct bio_vec *bv, *same_page = ((vec_end_addr & PAGE_MASK) == page_addr); if (*same_page) return true; + else if (IS_ENABLED(CONFIG_KMSAN)) + return false; return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE); } @@ -1065,9 +1065,6 @@ void __bio_add_page(struct bio *bio, struct page *page, bio->bi_iter.bi_size += len; bio->bi_vcnt++; - - if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) - bio_set_flag(bio, BIO_WORKINGSET); } EXPORT_SYMBOL_GPL(__bio_add_page); @@ -1276,9 +1273,6 @@ out: * fit into the bio, or are requested in @iter, whatever is smaller. If * MM encounters an error pinning the requested pages, it stops. Error * is returned only if 0 pages could be pinned. - * - * It's intended for direct IO, so doesn't do PSI tracking, the caller is - * responsible for setting BIO_WORKINGSET if necessary. */ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { @@ -1294,8 +1288,6 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) ret = __bio_iov_iter_get_pages(bio, iter); } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); - /* don't account direct I/O as memory stall */ - bio_clear_flag(bio, BIO_WORKINGSET); return bio->bi_vcnt ? 0 : ret; } EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); @@ -1754,7 +1746,8 @@ static int __init init_bio(void) cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead); - if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS)) + if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, + BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE)) panic("bio: can't allocate bios\n"); if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE)) diff --git a/block/blk-cgroup-fc-appid.c b/block/blk-cgroup-fc-appid.c index 760a2e1878dde5a1ff14e63b8179e8e094f47947..842e5e1c0f3cb97dafe6810dfbc527a30edb78ff 100644 --- a/block/blk-cgroup-fc-appid.c +++ b/block/blk-cgroup-fc-appid.c @@ -19,8 +19,8 @@ int blkcg_set_fc_appid(char *app_id, u64 cgrp_id, size_t app_id_len) return -EINVAL; cgrp = cgroup_get_from_id(cgrp_id); - if (!cgrp) - return -ENOENT; + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); css = cgroup_get_e_css(cgrp, &io_cgrp_subsys); if (!css) { ret = -ENOENT; diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 869af9d72bcf8cb4c9d036d9e3a25a77421f6307..6a5c849ee061bedf05aa5fe95dccfeea8b0e7b2c 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -202,19 +202,19 @@ static inline struct blkcg *blkcg_parent(struct blkcg *blkcg) /** * blkg_alloc - allocate a blkg * @blkcg: block cgroup the new blkg is associated with - * @q: request_queue the new blkg is associated with + * @disk: gendisk the new blkg is associated with * @gfp_mask: allocation mask to use * * Allocate a new blkg assocating @blkcg and @q. */ -static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, +static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, gfp_t gfp_mask) { struct blkcg_gq *blkg; int i, cpu; /* alloc and init base part */ - blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node); + blkg = kzalloc_node(sizeof(*blkg), gfp_mask, disk->queue->node); if (!blkg) return NULL; @@ -225,10 +225,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, if (!blkg->iostat_cpu) goto err_free; - if (!blk_get_queue(q)) + if (!blk_get_queue(disk->queue)) goto err_free; - blkg->q = q; + blkg->q = disk->queue; INIT_LIST_HEAD(&blkg->q_node); spin_lock_init(&blkg->async_bio_lock); bio_list_init(&blkg->async_bios); @@ -243,11 +243,11 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, struct blkcg_policy *pol = blkcg_policy[i]; struct blkg_policy_data *pd; - if (!blkcg_policy_enabled(q, pol)) + if (!blkcg_policy_enabled(disk->queue, pol)) continue; /* alloc per-policy data and attach it to blkg */ - pd = pol->pd_alloc_fn(gfp_mask, q, blkcg); + pd = pol->pd_alloc_fn(gfp_mask, disk->queue, blkcg); if (!pd) goto err_free; @@ -263,45 +263,20 @@ err_free: return NULL; } -struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, - struct request_queue *q, bool update_hint) -{ - struct blkcg_gq *blkg; - - /* - * Hint didn't match. Look up from the radix tree. Note that the - * hint can only be updated under queue_lock as otherwise @blkg - * could have already been removed from blkg_tree. The caller is - * responsible for grabbing queue_lock if @update_hint. - */ - blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); - if (blkg && blkg->q == q) { - if (update_hint) { - lockdep_assert_held(&q->queue_lock); - rcu_assign_pointer(blkcg->blkg_hint, blkg); - } - return blkg; - } - - return NULL; -} -EXPORT_SYMBOL_GPL(blkg_lookup_slowpath); - /* * If @new_blkg is %NULL, this function tries to allocate a new one as * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return. */ -static struct blkcg_gq *blkg_create(struct blkcg *blkcg, - struct request_queue *q, +static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, struct blkcg_gq *new_blkg) { struct blkcg_gq *blkg; int i, ret; - lockdep_assert_held(&q->queue_lock); + lockdep_assert_held(&disk->queue->queue_lock); /* request_queue is dying, do not create/recreate a blkg */ - if (blk_queue_dying(q)) { + if (blk_queue_dying(disk->queue)) { ret = -ENODEV; goto err_free_blkg; } @@ -314,7 +289,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, /* allocate */ if (!new_blkg) { - new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN); + new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT | __GFP_NOWARN); if (unlikely(!new_blkg)) { ret = -ENOMEM; goto err_put_css; @@ -324,7 +299,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, /* link parent */ if (blkcg_parent(blkcg)) { - blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); + blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue); if (WARN_ON_ONCE(!blkg->parent)) { ret = -ENODEV; goto err_put_css; @@ -342,10 +317,10 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, /* insert */ spin_lock(&blkcg->lock); - ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg); + ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg); if (likely(!ret)) { hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list); - list_add(&blkg->q_node, &q->blkg_list); + list_add(&blkg->q_node, &disk->queue->blkg_list); for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; @@ -374,19 +349,20 @@ err_free_blkg: /** * blkg_lookup_create - lookup blkg, try to create one if not there * @blkcg: blkcg of interest - * @q: request_queue of interest + * @disk: gendisk of interest * - * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to + * Lookup blkg for the @blkcg - @disk pair. If it doesn't exist, try to * create one. blkg creation is performed recursively from blkcg_root such * that all non-root blkg's have access to the parent blkg. This function - * should be called under RCU read lock and takes @q->queue_lock. + * should be called under RCU read lock and takes @disk->queue->queue_lock. * * Returns the blkg or the closest blkg if blkg_create() fails as it walks * down from root. */ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, - struct request_queue *q) + struct gendisk *disk) { + struct request_queue *q = disk->queue; struct blkcg_gq *blkg; unsigned long flags; @@ -397,9 +373,13 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, return blkg; spin_lock_irqsave(&q->queue_lock, flags); - blkg = __blkg_lookup(blkcg, q, true); - if (blkg) + blkg = blkg_lookup(blkcg, q); + if (blkg) { + if (blkcg != &blkcg_root && + blkg != rcu_dereference(blkcg->blkg_hint)) + rcu_assign_pointer(blkcg->blkg_hint, blkg); goto found; + } /* * Create blkgs walking down from blkcg_root to @blkcg, so that all @@ -412,7 +392,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, struct blkcg_gq *ret_blkg = q->root_blkg; while (parent) { - blkg = __blkg_lookup(parent, q, false); + blkg = blkg_lookup(parent, q); if (blkg) { /* remember closest blkg */ ret_blkg = blkg; @@ -422,7 +402,7 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, parent = blkcg_parent(parent); } - blkg = blkg_create(pos, q, NULL); + blkg = blkg_create(pos, disk, NULL); if (IS_ERR(blkg)) { blkg = ret_blkg; break; @@ -476,14 +456,9 @@ static void blkg_destroy(struct blkcg_gq *blkg) percpu_ref_kill(&blkg->refcnt); } -/** - * blkg_destroy_all - destroy all blkgs associated with a request_queue - * @q: request_queue of interest - * - * Destroy all blkgs associated with @q. - */ -static void blkg_destroy_all(struct request_queue *q) +static void blkg_destroy_all(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct blkcg_gq *blkg, *n; int count = BLKG_DESTROY_BATCH_SIZE; @@ -616,19 +591,6 @@ u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v) } EXPORT_SYMBOL_GPL(__blkg_prfill_u64); -/* Performs queue bypass and policy enabled checks then looks up blkg. */ -static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg, - const struct blkcg_policy *pol, - struct request_queue *q) -{ - WARN_ON_ONCE(!rcu_read_lock_held()); - lockdep_assert_held(&q->queue_lock); - - if (!blkcg_policy_enabled(q, pol)) - return ERR_PTR(-EOPNOTSUPP); - return __blkg_lookup(blkcg, q, true /* update_hint */); -} - /** * blkcg_conf_open_bdev - parse and open bdev for per-blkg config update * @inputp: input string pointer @@ -684,6 +646,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, __acquires(rcu) __acquires(&bdev->bd_queue->queue_lock) { struct block_device *bdev; + struct gendisk *disk; struct request_queue *q; struct blkcg_gq *blkg; int ret; @@ -691,8 +654,8 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, bdev = blkcg_conf_open_bdev(&input); if (IS_ERR(bdev)) return PTR_ERR(bdev); - - q = bdev_get_queue(bdev); + disk = bdev->bd_disk; + q = disk->queue; /* * blkcg_deactivate_policy() requires queue to be frozen, we can grab @@ -705,12 +668,12 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - blkg = blkg_lookup_check(blkcg, pol, q); - if (IS_ERR(blkg)) { - ret = PTR_ERR(blkg); + if (!blkcg_policy_enabled(q, pol)) { + ret = -EOPNOTSUPP; goto fail_unlock; } + blkg = blkg_lookup(blkcg, q); if (blkg) goto success; @@ -724,7 +687,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, struct blkcg_gq *new_blkg; parent = blkcg_parent(blkcg); - while (parent && !__blkg_lookup(parent, q, false)) { + while (parent && !blkg_lookup(parent, q)) { pos = parent; parent = blkcg_parent(parent); } @@ -733,7 +696,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, spin_unlock_irq(&q->queue_lock); rcu_read_unlock(); - new_blkg = blkg_alloc(pos, q, GFP_KERNEL); + new_blkg = blkg_alloc(pos, disk, GFP_KERNEL); if (unlikely(!new_blkg)) { ret = -ENOMEM; goto fail_exit_queue; @@ -748,17 +711,17 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, rcu_read_lock(); spin_lock_irq(&q->queue_lock); - blkg = blkg_lookup_check(pos, pol, q); - if (IS_ERR(blkg)) { - ret = PTR_ERR(blkg); + if (!blkcg_policy_enabled(q, pol)) { blkg_free(new_blkg); + ret = -EOPNOTSUPP; goto fail_preloaded; } + blkg = blkg_lookup(pos, q); if (blkg) { blkg_free(new_blkg); } else { - blkg = blkg_create(pos, q, new_blkg); + blkg = blkg_create(pos, disk, new_blkg); if (IS_ERR(blkg)) { ret = PTR_ERR(blkg); goto fail_preloaded; @@ -915,8 +878,7 @@ static void blkcg_fill_root_iostats(void) class_dev_iter_init(&iter, &block_class, NULL, &disk_type); while ((dev = class_dev_iter_next(&iter))) { struct block_device *bdev = dev_to_bdev(dev); - struct blkcg_gq *blkg = - blk_queue_root_blkg(bdev_get_queue(bdev)); + struct blkcg_gq *blkg = bdev->bd_disk->queue->root_blkg; struct blkg_iostat tmp; int cpu; unsigned long flags; @@ -1255,25 +1217,16 @@ static int blkcg_css_online(struct cgroup_subsys_state *css) return 0; } -/** - * blkcg_init_queue - initialize blkcg part of request queue - * @q: request_queue to initialize - * - * Called from blk_alloc_queue(). Responsible for initializing blkcg - * part of new request_queue @q. - * - * RETURNS: - * 0 on success, -errno on failure. - */ -int blkcg_init_queue(struct request_queue *q) +int blkcg_init_disk(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct blkcg_gq *new_blkg, *blkg; bool preloaded; int ret; INIT_LIST_HEAD(&q->blkg_list); - new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL); + new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) return -ENOMEM; @@ -1282,7 +1235,7 @@ int blkcg_init_queue(struct request_queue *q) /* Make sure the root blkg exists. */ /* spin_lock_irq can serve as RCU read-side critical section. */ spin_lock_irq(&q->queue_lock); - blkg = blkg_create(&blkcg_root, q, new_blkg); + blkg = blkg_create(&blkcg_root, disk, new_blkg); if (IS_ERR(blkg)) goto err_unlock; q->root_blkg = blkg; @@ -1291,25 +1244,26 @@ int blkcg_init_queue(struct request_queue *q) if (preloaded) radix_tree_preload_end(); - ret = blk_ioprio_init(q); + ret = blk_ioprio_init(disk); if (ret) goto err_destroy_all; - ret = blk_throtl_init(q); + ret = blk_throtl_init(disk); if (ret) - goto err_destroy_all; + goto err_ioprio_exit; - ret = blk_iolatency_init(q); - if (ret) { - blk_throtl_exit(q); - blk_ioprio_exit(q); - goto err_destroy_all; - } + ret = blk_iolatency_init(disk); + if (ret) + goto err_throtl_exit; return 0; +err_throtl_exit: + blk_throtl_exit(disk); +err_ioprio_exit: + blk_ioprio_exit(disk); err_destroy_all: - blkg_destroy_all(q); + blkg_destroy_all(disk); return ret; err_unlock: spin_unlock_irq(&q->queue_lock); @@ -1318,16 +1272,10 @@ err_unlock: return PTR_ERR(blkg); } -/** - * blkcg_exit_queue - exit and release blkcg part of request_queue - * @q: request_queue being released - * - * Called from blk_exit_queue(). Responsible for exiting blkcg part. - */ -void blkcg_exit_queue(struct request_queue *q) +void blkcg_exit_disk(struct gendisk *disk) { - blkg_destroy_all(q); - blk_throtl_exit(q); + blkg_destroy_all(disk); + blk_throtl_exit(disk); } static void blkcg_bind(struct cgroup_subsys_state *root_css) @@ -1836,13 +1784,13 @@ out: /** * blkcg_schedule_throttle - this task needs to check for throttling - * @q: the request queue IO was submitted on + * @gendisk: disk to throttle * @use_memdelay: do we charge this to memory delay for PSI * * This is called by the IO controller when we know there's delay accumulated * for the blkg for this task. We do not pass the blkg because there are places * we call this that may not have that information, the swapping code for - * instance will only have a request_queue at that point. This set's the + * instance will only have a block_device at that point. This set's the * notify_resume for the task to check and see if it requires throttling before * returning to user space. * @@ -1851,8 +1799,10 @@ out: * throttle once. If the task needs to be throttled again it'll need to be * re-set at the next time we see the task. */ -void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) +void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay) { + struct request_queue *q = disk->queue; + if (unlikely(current->flags & PF_KTHREAD)) return; @@ -1902,8 +1852,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct blkcg_gq *blkg, *ret_blkg = NULL; rcu_read_lock(); - blkg = blkg_lookup_create(css_to_blkcg(css), - bdev_get_queue(bio->bi_bdev)); + blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk); while (blkg) { if (blkg_tryget(blkg)) { ret_blkg = blkg; diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index d2724d1dd7c9bfc41944a1b7cc8e6b8306a37499..aa2b286bc825fbdcb3bc404dc106f158a6e750dd 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -178,10 +178,8 @@ struct blkcg_policy { extern struct blkcg blkcg_root; extern bool blkcg_debug_stats; -struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, - struct request_queue *q, bool update_hint); -int blkcg_init_queue(struct request_queue *q); -void blkcg_exit_queue(struct request_queue *q); +int blkcg_init_disk(struct gendisk *disk); +void blkcg_exit_disk(struct gendisk *disk); /* Blkio controller policy registration */ int blkcg_policy_register(struct blkcg_policy *pol); @@ -227,22 +225,21 @@ static inline bool bio_issue_as_root_blkg(struct bio *bio) } /** - * __blkg_lookup - internal version of blkg_lookup() + * blkg_lookup - lookup blkg for the specified blkcg - q pair * @blkcg: blkcg of interest * @q: request_queue of interest - * @update_hint: whether to update lookup hint with the result or not * - * This is internal version and shouldn't be used by policy - * implementations. Looks up blkgs for the @blkcg - @q pair regardless of - * @q's bypass state. If @update_hint is %true, the caller should be - * holding @q->queue_lock and lookup hint is updated on success. + * Lookup blkg for the @blkcg - @q pair. + + * Must be called in a RCU critical section. */ -static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, - struct request_queue *q, - bool update_hint) +static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, + struct request_queue *q) { struct blkcg_gq *blkg; + WARN_ON_ONCE(!rcu_read_lock_held()); + if (blkcg == &blkcg_root) return q->root_blkg; @@ -250,33 +247,10 @@ static inline struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, if (blkg && blkg->q == q) return blkg; - return blkg_lookup_slowpath(blkcg, q, update_hint); -} - -/** - * blkg_lookup - lookup blkg for the specified blkcg - q pair - * @blkcg: blkcg of interest - * @q: request_queue of interest - * - * Lookup blkg for the @blkcg - @q pair. This function should be called - * under RCU read lock. - */ -static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, - struct request_queue *q) -{ - WARN_ON_ONCE(!rcu_read_lock_held()); - return __blkg_lookup(blkcg, q, false); -} - -/** - * blk_queue_root_blkg - return blkg for the (blkcg_root, @q) pair - * @q: request_queue of interest - * - * Lookup blkg for @q at the root level. See also blkg_lookup(). - */ -static inline struct blkcg_gq *blk_queue_root_blkg(struct request_queue *q) -{ - return q->root_blkg; + blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id); + if (blkg && blkg->q != q) + blkg = NULL; + return blkg; } /** @@ -373,8 +347,8 @@ static inline void blkg_put(struct blkcg_gq *blkg) */ #define blkg_for_each_descendant_pre(d_blkg, pos_css, p_blkg) \ css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \ - if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \ - (p_blkg)->q, false))) + if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \ + (p_blkg)->q))) /** * blkg_for_each_descendant_post - post-order walk of a blkg's descendants @@ -388,8 +362,8 @@ static inline void blkg_put(struct blkcg_gq *blkg) */ #define blkg_for_each_descendant_post(d_blkg, pos_css, p_blkg) \ css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \ - if (((d_blkg) = __blkg_lookup(css_to_blkcg(pos_css), \ - (p_blkg)->q, false))) + if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \ + (p_blkg)->q))) bool __blkcg_punt_bio_submit(struct bio *bio); @@ -507,10 +481,8 @@ struct blkcg { }; static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; } -static inline struct blkcg_gq *blk_queue_root_blkg(struct request_queue *q) -{ return NULL; } -static inline int blkcg_init_queue(struct request_queue *q) { return 0; } -static inline void blkcg_exit_queue(struct request_queue *q) { } +static inline int blkcg_init_disk(struct gendisk *disk) { return 0; } +static inline void blkcg_exit_disk(struct gendisk *disk) { } static inline int blkcg_policy_register(struct blkcg_policy *pol) { return 0; } static inline void blkcg_policy_unregister(struct blkcg_policy *pol) { } static inline int blkcg_activate_policy(struct request_queue *q, diff --git a/block/blk-core.c b/block/blk-core.c index a0d1104c5590c16b9484c925ac387b571e1f15ed..17667159482e0309d37c6784b7f3f8b4aac1a63c 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -295,7 +294,7 @@ int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags) while (!blk_try_enter_queue(q, pm)) { if (flags & BLK_MQ_REQ_NOWAIT) - return -EBUSY; + return -EAGAIN; /* * read pair of barrier in blk_freeze_queue_start(), we need to @@ -325,7 +324,7 @@ int __bio_queue_enter(struct request_queue *q, struct bio *bio) if (test_bit(GD_DEAD, &disk->state)) goto dead; bio_wouldblock_error(bio); - return -EBUSY; + return -EAGAIN; } /* @@ -487,18 +486,15 @@ static int __init fail_make_request_debugfs(void) late_initcall(fail_make_request_debugfs); #endif /* CONFIG_FAIL_MAKE_REQUEST */ -static inline bool bio_check_ro(struct bio *bio) +static inline void bio_check_ro(struct bio *bio) { if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) { if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) - return false; + return; pr_warn("Trying to write to read-only block-device %pg\n", bio->bi_bdev); /* Older lvm-tools actually trigger this */ - return false; } - - return false; } static noinline int should_fail_bio(struct bio *bio) @@ -717,13 +713,12 @@ void submit_bio_noacct(struct bio *bio) * For a REQ_NOWAIT based request, return -EOPNOTSUPP * if queue does not support NOWAIT. */ - if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q)) + if ((bio->bi_opf & REQ_NOWAIT) && !bdev_nowait(bdev)) goto not_supported; if (should_fail_bio(bio)) goto end_io; - if (unlikely(bio_check_ro(bio))) - goto end_io; + bio_check_ro(bio); if (!bio_flagged(bio, BIO_REMAPPED)) { if (unlikely(bio_check_eod(bio))) goto end_io; @@ -814,7 +809,7 @@ EXPORT_SYMBOL(submit_bio_noacct); * * The success/failure status of the request, along with notification of * completion, is delivered asynchronously through the ->bi_end_io() callback - * in @bio. The bio must NOT be touched by thecaller until ->bi_end_io() has + * in @bio. The bio must NOT be touched by the caller until ->bi_end_io() has * been called. */ void submit_bio(struct bio *bio) @@ -829,22 +824,6 @@ void submit_bio(struct bio *bio) count_vm_events(PGPGOUT, bio_sectors(bio)); } - /* - * If we're reading data that is part of the userspace workingset, count - * submission time as memory stall. When the device is congested, or - * the submitting cgroup IO-throttled, submission can be a significant - * part of overall IO time. - */ - if (unlikely(bio_op(bio) == REQ_OP_READ && - bio_flagged(bio, BIO_WORKINGSET))) { - unsigned long pflags; - - psi_memstall_enter(&pflags); - submit_bio_noacct(bio); - psi_memstall_leave(&pflags); - return; - } - submit_bio_noacct(bio); } EXPORT_SYMBOL(submit_bio); @@ -871,6 +850,12 @@ int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags) !test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) return 0; + /* + * As the requests that require a zone lock are not plugged in the + * first place, directly accessing the plug instead of using + * blk_mq_plug() should not have any consequences during flushing for + * zoned devices. + */ blk_flush_plug(current->plug, false); if (bio_queue_enter(bio)) diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 621abd1b0e4d329655b2ea276bed7005e93522ef..ad9844c5b40cb8f491ff9f6647d1f67af007a39e 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -539,7 +539,7 @@ static int blk_crypto_fallback_init(void) if (blk_crypto_fallback_inited) return 0; - prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + get_random_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); err = bioset_init(&crypto_bio_split, 64, 0, 0); if (err) diff --git a/block/blk-flush.c b/block/blk-flush.c index d20a0c6b2c66edb451a120f26d2348c6061956c6..53202eff545efb7097e7ae8316541edf88e18b0e 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -205,7 +205,6 @@ static void blk_flush_complete_seq(struct request *rq, * flush data request completion path. Restore @rq for * normal completion and end it. */ - BUG_ON(!list_empty(&rq->queuelist)); list_del_init(&rq->flush.list); blk_flush_restore_request(rq); blk_mq_end_request(rq, error); @@ -218,7 +217,8 @@ static void blk_flush_complete_seq(struct request *rq, blk_kick_flush(q, fq, cmd_flags); } -static void flush_end_io(struct request *flush_rq, blk_status_t error) +static enum rq_end_io_ret flush_end_io(struct request *flush_rq, + blk_status_t error) { struct request_queue *q = flush_rq->q; struct list_head *running; @@ -232,7 +232,7 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error) if (!req_ref_put_and_test(flush_rq)) { fq->rq_status = error; spin_unlock_irqrestore(&fq->mq_flush_lock, flags); - return; + return RQ_END_IO_NONE; } blk_account_io_flush(flush_rq); @@ -269,6 +269,7 @@ static void flush_end_io(struct request *flush_rq, blk_status_t error) } spin_unlock_irqrestore(&fq->mq_flush_lock, flags); + return RQ_END_IO_NONE; } bool is_flush_rq(struct request *rq) @@ -354,7 +355,8 @@ static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq, blk_flush_queue_rq(flush_rq, false); } -static void mq_flush_data_end_io(struct request *rq, blk_status_t error) +static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq, + blk_status_t error) { struct request_queue *q = rq->q; struct blk_mq_hw_ctx *hctx = rq->mq_hctx; @@ -376,6 +378,7 @@ static void mq_flush_data_end_io(struct request *rq, blk_status_t error) spin_unlock_irqrestore(&fq->mq_flush_lock, flags); blk_mq_sched_restart(hctx); + return RQ_END_IO_NONE; } /** diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 7936e5f5821c73fdf20ce71d2cc44c7fee891493..495396425bade019ff8f26564b62d4949903cea9 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -664,17 +664,13 @@ static struct ioc *q_to_ioc(struct request_queue *q) return rqos_to_ioc(rq_qos_id(q, RQ_QOS_COST)); } -static const char *q_name(struct request_queue *q) -{ - if (blk_queue_registered(q)) - return kobject_name(q->kobj.parent); - else - return ""; -} - static const char __maybe_unused *ioc_name(struct ioc *ioc) { - return q_name(ioc->rqos.q); + struct gendisk *disk = ioc->rqos.q->disk; + + if (!disk) + return ""; + return disk->disk_name; } static struct ioc_gq *pd_to_iocg(struct blkg_policy_data *pd) @@ -1430,7 +1426,7 @@ static int iocg_wake_fn(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key) { struct iocg_wait *wait = container_of(wq_entry, struct iocg_wait, wait); - struct iocg_wake_ctx *ctx = (struct iocg_wake_ctx *)key; + struct iocg_wake_ctx *ctx = key; u64 cost = abs_cost_to_cost(wait->abs_cost, ctx->hw_inuse); ctx->vbudget -= cost; @@ -2640,7 +2636,7 @@ retry_lock: if (use_debt) { iocg_incur_debt(iocg, abs_cost, &now); if (iocg_kick_delay(iocg, &now)) - blkcg_schedule_throttle(rqos->q, + blkcg_schedule_throttle(rqos->q->disk, (bio->bi_opf & REQ_SWAP) == REQ_SWAP); iocg_unlock(iocg, ioc_locked, &flags); return; @@ -2741,7 +2737,7 @@ static void ioc_rqos_merge(struct rq_qos *rqos, struct request *rq, if (likely(!list_empty(&iocg->active_list))) { iocg_incur_debt(iocg, abs_cost, &now); if (iocg_kick_delay(iocg, &now)) - blkcg_schedule_throttle(rqos->q, + blkcg_schedule_throttle(rqos->q->disk, (bio->bi_opf & REQ_SWAP) == REQ_SWAP); } else { iocg_commit_bio(iocg, bio, abs_cost, cost); @@ -2832,8 +2828,9 @@ static struct rq_qos_ops ioc_rqos_ops = { .exit = ioc_rqos_exit, }; -static int blk_iocost_init(struct request_queue *q) +static int blk_iocost_init(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct ioc *ioc; struct rq_qos *rqos; int i, cpu, ret; @@ -3170,6 +3167,7 @@ static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input, size_t nbytes, loff_t off) { struct block_device *bdev; + struct gendisk *disk; struct ioc *ioc; u32 qos[NR_QOS_PARAMS]; bool enable, user; @@ -3180,12 +3178,13 @@ static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input, if (IS_ERR(bdev)) return PTR_ERR(bdev); - ioc = q_to_ioc(bdev_get_queue(bdev)); + disk = bdev->bd_disk; + ioc = q_to_ioc(disk->queue); if (!ioc) { - ret = blk_iocost_init(bdev_get_queue(bdev)); + ret = blk_iocost_init(disk); if (ret) goto err; - ioc = q_to_ioc(bdev_get_queue(bdev)); + ioc = q_to_ioc(disk->queue); } spin_lock_irq(&ioc->lock); @@ -3262,11 +3261,11 @@ static ssize_t ioc_qos_write(struct kernfs_open_file *of, char *input, spin_lock_irq(&ioc->lock); if (enable) { - blk_stat_enable_accounting(ioc->rqos.q); - blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q); + blk_stat_enable_accounting(disk->queue); + blk_queue_flag_set(QUEUE_FLAG_RQ_ALLOC_TIME, disk->queue); ioc->enabled = true; } else { - blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, ioc->rqos.q); + blk_queue_flag_clear(QUEUE_FLAG_RQ_ALLOC_TIME, disk->queue); ioc->enabled = false; } @@ -3349,7 +3348,7 @@ static ssize_t ioc_cost_model_write(struct kernfs_open_file *of, char *input, ioc = q_to_ioc(bdev_get_queue(bdev)); if (!ioc) { - ret = blk_iocost_init(bdev_get_queue(bdev)); + ret = blk_iocost_init(bdev->bd_disk); if (ret) goto err; ioc = q_to_ioc(bdev_get_queue(bdev)); diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index e285152345a20a6d6809fddf06c3f470a6a72ccf..571fa95aafe96b3552a2b9f37a8522974c07b4a8 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -292,7 +292,7 @@ static void __blkcg_iolatency_throttle(struct rq_qos *rqos, unsigned use_delay = atomic_read(&lat_to_blkg(iolat)->use_delay); if (use_delay) - blkcg_schedule_throttle(rqos->q, use_memdelay); + blkcg_schedule_throttle(rqos->q->disk, use_memdelay); /* * To avoid priority inversions we want to just take a slot if we are @@ -756,8 +756,9 @@ static void blkiolatency_enable_work_fn(struct work_struct *work) } } -int blk_iolatency_init(struct request_queue *q) +int blk_iolatency_init(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct blk_iolatency *blkiolat; struct rq_qos *rqos; int ret; diff --git a/block/blk-ioprio.c b/block/blk-ioprio.c index c00060a02c6efd6fa16b8a080bc3eb1da23ca620..8bb6b8eba4cee816804a5de3e2df15f1fa64e4e1 100644 --- a/block/blk-ioprio.c +++ b/block/blk-ioprio.c @@ -202,14 +202,14 @@ void blkcg_set_ioprio(struct bio *bio) bio->bi_ioprio = prio; } -void blk_ioprio_exit(struct request_queue *q) +void blk_ioprio_exit(struct gendisk *disk) { - blkcg_deactivate_policy(q, &ioprio_policy); + blkcg_deactivate_policy(disk->queue, &ioprio_policy); } -int blk_ioprio_init(struct request_queue *q) +int blk_ioprio_init(struct gendisk *disk) { - return blkcg_activate_policy(q, &ioprio_policy); + return blkcg_activate_policy(disk->queue, &ioprio_policy); } static int __init ioprio_init(void) diff --git a/block/blk-ioprio.h b/block/blk-ioprio.h index 5a1eb550e178c46e709a1e4cb59ffd7223a16058..b6afb8e80de05cc990e9d263b6884b44e6bc3b82 100644 --- a/block/blk-ioprio.h +++ b/block/blk-ioprio.h @@ -9,15 +9,15 @@ struct request_queue; struct bio; #ifdef CONFIG_BLK_CGROUP_IOPRIO -int blk_ioprio_init(struct request_queue *q); -void blk_ioprio_exit(struct request_queue *q); +int blk_ioprio_init(struct gendisk *disk); +void blk_ioprio_exit(struct gendisk *disk); void blkcg_set_ioprio(struct bio *bio); #else -static inline int blk_ioprio_init(struct request_queue *q) +static inline int blk_ioprio_init(struct gendisk *disk) { return 0; } -static inline void blk_ioprio_exit(struct request_queue *q) +static inline void blk_ioprio_exit(struct gendisk *disk) { } static inline void blkcg_set_ioprio(struct bio *bio) diff --git a/block/blk-lib.c b/block/blk-lib.c index 67e6dbc1ae8179594a57e89e87d6d247af358940..e59c3069e8351f7edf0d82c6a3b376a3029a994c 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -309,6 +309,11 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, struct blk_plug plug; int ret = 0; + /* make sure that "len << SECTOR_SHIFT" doesn't overflow */ + if (max_sectors > UINT_MAX >> SECTOR_SHIFT) + max_sectors = UINT_MAX >> SECTOR_SHIFT; + max_sectors &= ~bs_mask; + if (max_sectors == 0) return -EOPNOTSUPP; if ((sector | nr_sects) & bs_mask) @@ -322,10 +327,10 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp); bio->bi_iter.bi_sector = sector; - bio->bi_iter.bi_size = len; + bio->bi_iter.bi_size = len << SECTOR_SHIFT; - sector += len << SECTOR_SHIFT; - nr_sects -= len << SECTOR_SHIFT; + sector += len; + nr_sects -= len; if (!nr_sects) { ret = submit_bio_wait(bio); bio_put(bio); diff --git a/block/blk-map.c b/block/blk-map.c index 7196a6b64c80271e60a4917e95204da6b283b7cc..34735626b00f3d125563cdd3799cc9756125d694 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -158,7 +158,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); if (map_data) { - nr_pages = 1 << map_data->page_order; + nr_pages = 1U << map_data->page_order; i = map_data->offset / PAGE_SIZE; } while (len) { @@ -231,6 +231,37 @@ out_bmd: return ret; } +static void blk_mq_map_bio_put(struct bio *bio) +{ + if (bio->bi_opf & REQ_ALLOC_CACHE) { + bio_put(bio); + } else { + bio_uninit(bio); + kfree(bio); + } +} + +static struct bio *blk_rq_map_bio_alloc(struct request *rq, + unsigned int nr_vecs, gfp_t gfp_mask) +{ + struct bio *bio; + + if (rq->cmd_flags & REQ_POLLED) { + blk_opf_t opf = rq->cmd_flags | REQ_ALLOC_CACHE; + + bio = bio_alloc_bioset(NULL, nr_vecs, opf, gfp_mask, + &fs_bio_set); + if (!bio) + return NULL; + } else { + bio = bio_kmalloc(nr_vecs, gfp_mask); + if (!bio) + return NULL; + bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); + } + return bio; +} + static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { @@ -243,18 +274,24 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, if (!iov_iter_count(iter)) return -EINVAL; - bio = bio_kmalloc(nr_vecs, gfp_mask); - if (!bio) + bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); + if (bio == NULL) return -ENOMEM; - bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); while (iov_iter_count(iter)) { - struct page **pages; + struct page **pages, *stack_pages[UIO_FASTIOV]; ssize_t bytes; - size_t offs, added = 0; + size_t offs; int npages; - bytes = iov_iter_get_pages_alloc2(iter, &pages, LONG_MAX, &offs); + if (nr_vecs <= ARRAY_SIZE(stack_pages)) { + pages = stack_pages; + bytes = iov_iter_get_pages2(iter, pages, LONG_MAX, + nr_vecs, &offs); + } else { + bytes = iov_iter_get_pages_alloc2(iter, &pages, + LONG_MAX, &offs); + } if (unlikely(bytes <= 0)) { ret = bytes ? bytes : -EFAULT; goto out_unmap; @@ -280,7 +317,6 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, break; } - added += n; bytes -= n; offs = 0; } @@ -290,7 +326,8 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, */ while (j < npages) put_page(pages[j++]); - kvfree(pages); + if (pages != stack_pages) + kvfree(pages); /* couldn't stuff something into bio? */ if (bytes) { iov_iter_revert(iter, bytes); @@ -305,8 +342,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, out_unmap: bio_release_pages(bio, false); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); return ret; } @@ -512,6 +548,62 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) } EXPORT_SYMBOL(blk_rq_append_bio); +/* Prepare bio for passthrough IO given ITER_BVEC iter */ +static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter) +{ + struct request_queue *q = rq->q; + size_t nr_iter = iov_iter_count(iter); + size_t nr_segs = iter->nr_segs; + struct bio_vec *bvecs, *bvprvp = NULL; + struct queue_limits *lim = &q->limits; + unsigned int nsegs = 0, bytes = 0; + struct bio *bio; + size_t i; + + if (!nr_iter || (nr_iter >> SECTOR_SHIFT) > queue_max_hw_sectors(q)) + return -EINVAL; + if (nr_segs > queue_max_segments(q)) + return -EINVAL; + + /* no iovecs to alloc, as we already have a BVEC iterator */ + bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); + if (bio == NULL) + return -ENOMEM; + + bio_iov_bvec_set(bio, (struct iov_iter *)iter); + blk_rq_bio_prep(rq, bio, nr_segs); + + /* loop to perform a bunch of sanity checks */ + bvecs = (struct bio_vec *)iter->bvec; + for (i = 0; i < nr_segs; i++) { + struct bio_vec *bv = &bvecs[i]; + + /* + * If the queue doesn't support SG gaps and adding this + * offset would create a gap, fallback to copy. + */ + if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv->bv_offset)) { + blk_mq_map_bio_put(bio); + return -EREMOTEIO; + } + /* check full condition */ + if (nsegs >= nr_segs || bytes > UINT_MAX - bv->bv_len) + goto put_bio; + if (bytes + bv->bv_len > nr_iter) + goto put_bio; + if (bv->bv_offset + bv->bv_len > PAGE_SIZE) + goto put_bio; + + nsegs++; + bytes += bv->bv_len; + bvprvp = bv; + } + return 0; +put_bio: + blk_mq_map_bio_put(bio); + return -EINVAL; +} + /** * blk_rq_map_user_iov - map user data to a request, for passthrough requests * @q: request queue where request should be inserted @@ -531,24 +623,35 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, struct rq_map_data *map_data, const struct iov_iter *iter, gfp_t gfp_mask) { - bool copy = false; + bool copy = false, map_bvec = false; unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); struct bio *bio = NULL; struct iov_iter i; int ret = -EINVAL; - if (!iter_is_iovec(iter)) - goto fail; - if (map_data) copy = true; else if (blk_queue_may_bounce(q)) copy = true; else if (iov_iter_alignment(iter) & align) copy = true; + else if (iov_iter_is_bvec(iter)) + map_bvec = true; + else if (!iter_is_iovec(iter)) + copy = true; else if (queue_virt_boundary(q)) copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); + if (map_bvec) { + ret = blk_rq_map_user_bvec(rq, iter); + if (!ret) + return 0; + if (ret != -EREMOTEIO) + goto fail; + /* fall back to copying the data on limits mismatches */ + copy = true; + } + i = *iter; do { if (copy) @@ -586,6 +689,42 @@ int blk_rq_map_user(struct request_queue *q, struct request *rq, } EXPORT_SYMBOL(blk_rq_map_user); +int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, + void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, + bool vec, int iov_count, bool check_iter_count, int rw) +{ + int ret = 0; + + if (vec) { + struct iovec fast_iov[UIO_FASTIOV]; + struct iovec *iov = fast_iov; + struct iov_iter iter; + + ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, + UIO_FASTIOV, &iov, &iter); + if (ret < 0) + return ret; + + if (iov_count) { + /* SG_IO howto says that the shorter of the two wins */ + iov_iter_truncate(&iter, buf_len); + if (check_iter_count && !iov_iter_count(&iter)) { + kfree(iov); + return -EINVAL; + } + } + + ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, + gfp_mask); + kfree(iov); + } else if (buf_len) { + ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, + gfp_mask); + } + return ret; +} +EXPORT_SYMBOL(blk_rq_map_user_io); + /** * blk_rq_unmap_user - unmap a request with user data * @bio: start of bio list @@ -611,8 +750,7 @@ int blk_rq_unmap_user(struct bio *bio) next_bio = bio; bio = bio->bi_next; - bio_uninit(next_bio); - kfree(next_bio); + blk_mq_map_bio_put(next_bio); } return ret; diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c index 3db84d3197f111713035855ce6d761a9984c207d..9c2fce1a7b50eee5e63cb65b3cfe312b4d21ffdc 100644 --- a/block/blk-mq-cpumap.c +++ b/block/blk-mq-cpumap.c @@ -32,7 +32,7 @@ static int get_first_sibling(unsigned int cpu) return cpu; } -int blk_mq_map_queues(struct blk_mq_queue_map *qmap) +void blk_mq_map_queues(struct blk_mq_queue_map *qmap) { unsigned int *map = qmap->mq_map; unsigned int nr_queues = qmap->nr_queues; @@ -70,8 +70,6 @@ int blk_mq_map_queues(struct blk_mq_queue_map *qmap) map[cpu] = map[first_sibling]; } } - - return 0; } EXPORT_SYMBOL_GPL(blk_mq_map_queues); diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index 8559cea7f300eefb19c8b5e67eae53de69fd4746..bd942341b6382f13132366f62de12bafdbbedbcf 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -283,7 +283,9 @@ static const char *const rqf_name[] = { RQF_NAME(SPECIAL_PAYLOAD), RQF_NAME(ZONE_WRITE_LOCKED), RQF_NAME(MQ_POLL_SLEPT), + RQF_NAME(TIMED_OUT), RQF_NAME(ELV), + RQF_NAME(RESV), }; #undef RQF_NAME @@ -805,8 +807,6 @@ static const char *rq_qos_id_to_name(enum rq_qos_id id) return "latency"; case RQ_QOS_COST: return "cost"; - case RQ_QOS_IOPRIO: - return "ioprio"; } return "unknown"; } diff --git a/block/blk-mq-pci.c b/block/blk-mq-pci.c index b595a94c4d16bb93b5dc61c7880534c1ef5a528e..a90b88fd1332ce68582487643e305058f9403165 100644 --- a/block/blk-mq-pci.c +++ b/block/blk-mq-pci.c @@ -23,8 +23,8 @@ * that maps a queue to the CPUs that have irq affinity for the corresponding * vector. */ -int blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev, - int offset) +void blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev, + int offset) { const struct cpumask *mask; unsigned int queue, cpu; @@ -38,11 +38,10 @@ int blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev, qmap->mq_map[cpu] = qmap->queue_offset + queue; } - return 0; + return; fallback: WARN_ON_ONCE(qmap->nr_queues > 1); blk_mq_clear_mq_map(qmap); - return 0; } EXPORT_SYMBOL_GPL(blk_mq_pci_map_queues); diff --git a/block/blk-mq-rdma.c b/block/blk-mq-rdma.c index 14f968e58b8f7103bc81db1f42996d6853f70317..29c1f4d6eb0412308afbdac2317eeec0ac11e121 100644 --- a/block/blk-mq-rdma.c +++ b/block/blk-mq-rdma.c @@ -21,7 +21,7 @@ * @set->nr_hw_queues, or @dev does not provide an affinity mask for a * vector, we fallback to the naive mapping. */ -int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, +void blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, struct ib_device *dev, int first_vec) { const struct cpumask *mask; @@ -36,9 +36,9 @@ int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, map->mq_map[cpu] = map->queue_offset + queue; } - return 0; + return; fallback: - return blk_mq_map_queues(map); + blk_mq_map_queues(map); } EXPORT_SYMBOL_GPL(blk_mq_rdma_map_queues); diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 8e3b36d1cb574ed7dbab5da45720a3e0a3eacdde..9eb968e14d31f83286c31c3321c0599a0c714881 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -196,7 +196,7 @@ unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) * other allocations on previous queue won't be starved. */ if (bt != bt_prev) - sbitmap_queue_wake_up(bt_prev); + sbitmap_queue_wake_up(bt_prev, 1); ws = bt_wait_ptr(bt, data->hctx); } while (1); diff --git a/block/blk-mq-virtio.c b/block/blk-mq-virtio.c index 7b8a42c351026501dc8aceaf6f33d5a82a55e42f..6589f076a09635ca1ef3e68638cb7bd4708481ad 100644 --- a/block/blk-mq-virtio.c +++ b/block/blk-mq-virtio.c @@ -21,7 +21,7 @@ * that maps a queue to the CPUs that have irq affinity for the corresponding * vector. */ -int blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap, +void blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap, struct virtio_device *vdev, int first_vec) { const struct cpumask *mask; @@ -39,8 +39,9 @@ int blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap, qmap->mq_map[cpu] = qmap->queue_offset + queue; } - return 0; + return; + fallback: - return blk_mq_map_queues(qmap); + blk_mq_map_queues(qmap); } EXPORT_SYMBOL_GPL(blk_mq_virtio_map_queues); diff --git a/block/blk-mq.c b/block/blk-mq.c index 3c1e6b6d991d2de7ad831a4bc18eabef9448958b..8070b6c10e8d5ae1cf97b6f8a712b566f89d2255 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -510,25 +510,87 @@ retry: alloc_time_ns); } -struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, - blk_mq_req_flags_t flags) +static struct request *blk_mq_rq_cache_fill(struct request_queue *q, + struct blk_plug *plug, + blk_opf_t opf, + blk_mq_req_flags_t flags) { struct blk_mq_alloc_data data = { .q = q, .flags = flags, .cmd_flags = opf, - .nr_tags = 1, + .nr_tags = plug->nr_ios, + .cached_rq = &plug->cached_rq, }; struct request *rq; - int ret; - ret = blk_queue_enter(q, flags); - if (ret) - return ERR_PTR(ret); + if (blk_queue_enter(q, flags)) + return NULL; + + plug->nr_ios = 1; rq = __blk_mq_alloc_requests(&data); - if (!rq) - goto out_queue_exit; + if (unlikely(!rq)) + blk_queue_exit(q); + return rq; +} + +static struct request *blk_mq_alloc_cached_request(struct request_queue *q, + blk_opf_t opf, + blk_mq_req_flags_t flags) +{ + struct blk_plug *plug = current->plug; + struct request *rq; + + if (!plug) + return NULL; + if (rq_list_empty(plug->cached_rq)) { + if (plug->nr_ios == 1) + return NULL; + rq = blk_mq_rq_cache_fill(q, plug, opf, flags); + if (rq) + goto got_it; + return NULL; + } + rq = rq_list_peek(&plug->cached_rq); + if (!rq || rq->q != q) + return NULL; + + if (blk_mq_get_hctx_type(opf) != rq->mq_hctx->type) + return NULL; + if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) + return NULL; + + plug->cached_rq = rq_list_next(rq); +got_it: + rq->cmd_flags = opf; + INIT_LIST_HEAD(&rq->queuelist); + return rq; +} + +struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, + blk_mq_req_flags_t flags) +{ + struct request *rq; + + rq = blk_mq_alloc_cached_request(q, opf, flags); + if (!rq) { + struct blk_mq_alloc_data data = { + .q = q, + .flags = flags, + .cmd_flags = opf, + .nr_tags = 1, + }; + int ret; + + ret = blk_queue_enter(q, flags); + if (ret) + return ERR_PTR(ret); + + rq = __blk_mq_alloc_requests(&data); + if (!rq) + goto out_queue_exit; + } rq->__data_len = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; @@ -761,8 +823,10 @@ static void blk_complete_request(struct request *req) * can find how many bytes remain in the request * later. */ - req->bio = NULL; - req->__data_len = 0; + if (!req->end_io) { + req->bio = NULL; + req->__data_len = 0; + } } /** @@ -939,7 +1003,8 @@ inline void __blk_mq_end_request(struct request *rq, blk_status_t error) if (rq->end_io) { rq_qos_done(rq->q, rq); - rq->end_io(rq, error); + if (rq->end_io(rq, error) == RQ_END_IO_FREE) + blk_mq_free_request(rq); } else { blk_mq_free_request(rq); } @@ -992,6 +1057,13 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob) rq_qos_done(rq->q, rq); + /* + * If end_io handler returns NONE, then it still has + * ownership of the request. + */ + if (rq->end_io && rq->end_io(rq, 0) == RQ_END_IO_NONE) + continue; + WRITE_ONCE(rq->state, MQ_RQ_IDLE); if (!req_ref_put_and_test(rq)) continue; @@ -1093,10 +1165,12 @@ bool blk_mq_complete_request_remote(struct request *rq) WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); /* - * For a polled request, always complete locally, it's pointless - * to redirect the completion. + * For request which hctx has only one ctx mapping, + * or a polled request, always complete locally, + * it's pointless to redirect the completion. */ - if (rq->cmd_flags & REQ_POLLED) + if (rq->mq_hctx->nr_ctx == 1 || + rq->cmd_flags & REQ_POLLED) return false; if (blk_mq_complete_need_ipi(rq)) { @@ -1213,6 +1287,12 @@ void blk_execute_rq_nowait(struct request *rq, bool at_head) WARN_ON(!blk_rq_is_passthrough(rq)); blk_account_io_start(rq); + + /* + * As plugging can be enabled for passthrough requests on a zoned + * device, directly accessing the plug instead of using blk_mq_plug() + * should not have any consequences. + */ if (current->plug) blk_add_rq_to_plug(current->plug, rq); else @@ -1225,15 +1305,16 @@ struct blk_rq_wait { blk_status_t ret; }; -static void blk_end_sync_rq(struct request *rq, blk_status_t ret) +static enum rq_end_io_ret blk_end_sync_rq(struct request *rq, blk_status_t ret) { struct blk_rq_wait *wait = rq->end_io_data; wait->ret = ret; complete(&wait->done); + return RQ_END_IO_NONE; } -static bool blk_rq_is_poll(struct request *rq) +bool blk_rq_is_poll(struct request *rq) { if (!rq->mq_hctx) return false; @@ -1243,6 +1324,7 @@ static bool blk_rq_is_poll(struct request *rq) return false; return true; } +EXPORT_SYMBOL_GPL(blk_rq_is_poll); static void blk_rq_poll_completion(struct request *rq, struct completion *wait) { @@ -1463,10 +1545,12 @@ static bool blk_mq_req_expired(struct request *rq, unsigned long *next) void blk_mq_put_rq_ref(struct request *rq) { - if (is_flush_rq(rq)) - rq->end_io(rq, 0); - else if (req_ref_put_and_test(rq)) + if (is_flush_rq(rq)) { + if (rq->end_io(rq, 0) == RQ_END_IO_FREE) + blk_mq_free_request(rq); + } else if (req_ref_put_and_test(rq)) { __blk_mq_free_request(rq); + } } static bool blk_mq_check_expired(struct request *rq, void *priv) @@ -1931,7 +2015,8 @@ out: /* If we didn't flush the entire list, we could have told the driver * there was more coming, but that turned out to be a lie. */ - if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued) + if ((!list_empty(list) || errors || needs_resource || + ret == BLK_STS_DEV_RESOURCE) && q->mq_ops->commit_rqs && queued) q->mq_ops->commit_rqs(hctx); /* * Any items that need requeuing? Stuff them into hctx->dispatch, @@ -1991,7 +2076,7 @@ out: if (!needs_restart || (no_tag && list_empty_careful(&hctx->dispatch_wait.entry))) blk_mq_run_hw_queue(hctx, true); - else if (needs_restart && needs_resource) + else if (needs_resource) blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY); blk_mq_update_dispatch_busy(hctx, true); @@ -2660,6 +2745,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, list_del_init(&rq->queuelist); ret = blk_mq_request_issue_directly(rq, list_empty(list)); if (ret != BLK_STS_OK) { + errors++; if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE) { blk_mq_request_bypass_insert(rq, false, @@ -2667,7 +2753,6 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx, break; } blk_mq_end_request(rq, ret); - errors++; } else queued++; } @@ -4190,7 +4275,7 @@ static int blk_mq_alloc_set_map_and_rqs(struct blk_mq_tag_set *set) return 0; } -static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) +static void blk_mq_update_queue_map(struct blk_mq_tag_set *set) { /* * blk_mq_map_queues() and multiple .map_queues() implementations @@ -4220,10 +4305,10 @@ static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) for (i = 0; i < set->nr_maps; i++) blk_mq_clear_mq_map(&set->map[i]); - return set->ops->map_queues(set); + set->ops->map_queues(set); } else { BUG_ON(set->nr_maps > 1); - return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); + blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); } } @@ -4322,9 +4407,7 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues; } - ret = blk_mq_update_queue_map(set); - if (ret) - goto out_free_mq_map; + blk_mq_update_queue_map(set); ret = blk_mq_alloc_set_map_and_rqs(set); if (ret) @@ -4472,14 +4555,14 @@ static bool blk_mq_elv_switch_none(struct list_head *head, list_add(&qe->node, head); /* - * After elevator_switch_mq, the previous elevator_queue will be + * After elevator_switch, the previous elevator_queue will be * released by elevator_release. The reference of the io scheduler * module get by elevator_get will also be put. So we need to get * a reference of the io scheduler module here to prevent it to be * removed. */ __module_get(qe->type->elevator_owner); - elevator_switch_mq(q, NULL); + elevator_switch(q, NULL); mutex_unlock(&q->sysfs_lock); return true; @@ -4511,7 +4594,7 @@ static void blk_mq_elv_switch_back(struct list_head *head, kfree(qe); mutex_lock(&q->sysfs_lock); - elevator_switch_mq(q, t); + elevator_switch(q, t); mutex_unlock(&q->sysfs_lock); } diff --git a/block/blk-mq.h b/block/blk-mq.h index 8ca453ac243d87f6938a2b24b9a53f1b7b7cae32..0b2870839cdd6c827fd35ce51a9c9cf232a72c38 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -312,7 +312,8 @@ static inline void blk_mq_clear_mq_map(struct blk_mq_queue_map *qmap) static inline struct blk_plug *blk_mq_plug( struct bio *bio) { /* Zoned block device write operation case: do not plug the BIO */ - if (bdev_is_zoned(bio->bi_bdev) && op_is_write(bio_op(bio))) + if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && + bdev_op_is_zoned_write(bio->bi_bdev, bio_op(bio))) return NULL; /* diff --git a/block/blk-rq-qos.h b/block/blk-rq-qos.h index 08b856570ad10a2e96f423c621593e610ed1be69..1ef1f7d4bc3cbc994c512f17a9bdb388c06ff677 100644 --- a/block/blk-rq-qos.h +++ b/block/blk-rq-qos.h @@ -17,7 +17,6 @@ enum rq_qos_id { RQ_QOS_WBT, RQ_QOS_LATENCY, RQ_QOS_COST, - RQ_QOS_IOPRIO, }; struct rq_wait { diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index e1f009aba6fd2ad6c3ca2a4f395658da72a1774a..e71b3b43927c0f65dcd447f8bb46eb1c4d79a2a8 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -844,7 +844,7 @@ int blk_register_queue(struct gendisk *disk) blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q); wbt_enable_default(q); - blk_throtl_register_queue(q); + blk_throtl_register(disk); /* Now everything is ready and send out KOBJ_ADD uevent */ kobject_uevent(&q->kobj, KOBJ_ADD); diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 9f5fe62afff9284918330e187a29fc00218e1d71..847721dc2b2b8160f501bbc06bba2f055b2513de 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -329,8 +329,8 @@ static struct bio *throtl_pop_queued(struct list_head *queued, /* init a service_queue, assumes the caller zeroed it */ static void throtl_service_queue_init(struct throtl_service_queue *sq) { - INIT_LIST_HEAD(&sq->queued[0]); - INIT_LIST_HEAD(&sq->queued[1]); + INIT_LIST_HEAD(&sq->queued[READ]); + INIT_LIST_HEAD(&sq->queued[WRITE]); sq->pending_tree = RB_ROOT_CACHED; timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0); } @@ -420,24 +420,17 @@ static void tg_update_has_rules(struct throtl_grp *tg) struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq); struct throtl_data *td = tg->td; int rw; - int has_iops_limit = 0; for (rw = READ; rw <= WRITE; rw++) { - unsigned int iops_limit = tg_iops_limit(tg, rw); - - tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) || + tg->has_rules_iops[rw] = + (parent_tg && parent_tg->has_rules_iops[rw]) || (td->limit_valid[td->limit_index] && - (tg_bps_limit(tg, rw) != U64_MAX || - iops_limit != UINT_MAX)); - - if (iops_limit != UINT_MAX) - has_iops_limit = 1; + tg_iops_limit(tg, rw) != UINT_MAX); + tg->has_rules_bps[rw] = + (parent_tg && parent_tg->has_rules_bps[rw]) || + (td->limit_valid[td->limit_index] && + (tg_bps_limit(tg, rw) != U64_MAX)); } - - if (has_iops_limit) - tg->flags |= THROTL_TG_HAS_IOPS_LIMIT; - else - tg->flags &= ~THROTL_TG_HAS_IOPS_LIMIT; } static void throtl_pd_online(struct blkg_policy_data *pd) @@ -520,7 +513,6 @@ static void throtl_rb_erase(struct rb_node *n, { rb_erase_cached(n, &parent_sq->pending_tree); RB_CLEAR_NODE(n); - --parent_sq->nr_pending; } static void update_min_dispatch_time(struct throtl_service_queue *parent_sq) @@ -572,7 +564,11 @@ static void throtl_enqueue_tg(struct throtl_grp *tg) static void throtl_dequeue_tg(struct throtl_grp *tg) { if (tg->flags & THROTL_TG_PENDING) { - throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); + struct throtl_service_queue *parent_sq = + tg->service_queue.parent_sq; + + throtl_rb_erase(&tg->rb_node, parent_sq); + --parent_sq->nr_pending; tg->flags &= ~THROTL_TG_PENDING; } } @@ -639,6 +635,8 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, { tg->bytes_disp[rw] = 0; tg->io_disp[rw] = 0; + tg->carryover_bytes[rw] = 0; + tg->carryover_ios[rw] = 0; /* * Previous slice has expired. We must have trimmed it after last @@ -656,12 +654,17 @@ static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg, tg->slice_end[rw], jiffies); } -static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw) +static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw, + bool clear_carryover) { tg->bytes_disp[rw] = 0; tg->io_disp[rw] = 0; tg->slice_start[rw] = jiffies; tg->slice_end[rw] = jiffies + tg->td->throtl_slice; + if (clear_carryover) { + tg->carryover_bytes[rw] = 0; + tg->carryover_ios[rw] = 0; + } throtl_log(&tg->service_queue, "[%c] new slice start=%lu end=%lu jiffies=%lu", @@ -754,33 +757,20 @@ static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw) tg->slice_start[rw], tg->slice_end[rw], jiffies); } -static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, - u32 iops_limit, unsigned long *wait) +static unsigned int calculate_io_allowed(u32 iops_limit, + unsigned long jiffy_elapsed) { - bool rw = bio_data_dir(bio); unsigned int io_allowed; - unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; u64 tmp; - if (iops_limit == UINT_MAX) { - if (wait) - *wait = 0; - return true; - } - - jiffy_elapsed = jiffies - tg->slice_start[rw]; - - /* Round up to the next throttle slice, wait time must be nonzero */ - jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); - /* - * jiffy_elapsed_rnd should not be a big value as minimum iops can be + * jiffy_elapsed should not be a big value as minimum iops can be * 1 then at max jiffy elapsed should be equivalent of 1 second as we * will allow dispatch after 1 second and after that slice should * have been trimmed. */ - tmp = (u64)iops_limit * jiffy_elapsed_rnd; + tmp = (u64)iops_limit * jiffy_elapsed; do_div(tmp, HZ); if (tmp > UINT_MAX) @@ -788,6 +778,68 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, else io_allowed = tmp; + return io_allowed; +} + +static u64 calculate_bytes_allowed(u64 bps_limit, unsigned long jiffy_elapsed) +{ + return mul_u64_u64_div_u64(bps_limit, (u64)jiffy_elapsed, (u64)HZ); +} + +static void __tg_update_carryover(struct throtl_grp *tg, bool rw) +{ + unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; + u64 bps_limit = tg_bps_limit(tg, rw); + u32 iops_limit = tg_iops_limit(tg, rw); + + /* + * If config is updated while bios are still throttled, calculate and + * accumulate how many bytes/ios are waited across changes. And + * carryover_bytes/ios will be used to calculate new wait time under new + * configuration. + */ + if (bps_limit != U64_MAX) + tg->carryover_bytes[rw] += + calculate_bytes_allowed(bps_limit, jiffy_elapsed) - + tg->bytes_disp[rw]; + if (iops_limit != UINT_MAX) + tg->carryover_ios[rw] += + calculate_io_allowed(iops_limit, jiffy_elapsed) - + tg->io_disp[rw]; +} + +static void tg_update_carryover(struct throtl_grp *tg) +{ + if (tg->service_queue.nr_queued[READ]) + __tg_update_carryover(tg, READ); + if (tg->service_queue.nr_queued[WRITE]) + __tg_update_carryover(tg, WRITE); + + /* see comments in struct throtl_grp for meaning of these fields. */ + throtl_log(&tg->service_queue, "%s: %llu %llu %u %u\n", __func__, + tg->carryover_bytes[READ], tg->carryover_bytes[WRITE], + tg->carryover_ios[READ], tg->carryover_ios[WRITE]); +} + +static bool tg_within_iops_limit(struct throtl_grp *tg, struct bio *bio, + u32 iops_limit, unsigned long *wait) +{ + bool rw = bio_data_dir(bio); + unsigned int io_allowed; + unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; + + if (iops_limit == UINT_MAX) { + if (wait) + *wait = 0; + return true; + } + + jiffy_elapsed = jiffies - tg->slice_start[rw]; + + /* Round up to the next throttle slice, wait time must be nonzero */ + jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); + io_allowed = calculate_io_allowed(iops_limit, jiffy_elapsed_rnd) + + tg->carryover_ios[rw]; if (tg->io_disp[rw] + 1 <= io_allowed) { if (wait) *wait = 0; @@ -802,16 +854,16 @@ static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio, return false; } -static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, - u64 bps_limit, unsigned long *wait) +static bool tg_within_bps_limit(struct throtl_grp *tg, struct bio *bio, + u64 bps_limit, unsigned long *wait) { bool rw = bio_data_dir(bio); - u64 bytes_allowed, extra_bytes, tmp; + u64 bytes_allowed, extra_bytes; unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd; unsigned int bio_size = throtl_bio_data_size(bio); /* no need to throttle if this bio's bytes have been accounted */ - if (bps_limit == U64_MAX || bio_flagged(bio, BIO_THROTTLED)) { + if (bps_limit == U64_MAX || bio_flagged(bio, BIO_BPS_THROTTLED)) { if (wait) *wait = 0; return true; @@ -824,11 +876,8 @@ static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio, jiffy_elapsed_rnd = tg->td->throtl_slice; jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); - - tmp = bps_limit * jiffy_elapsed_rnd; - do_div(tmp, HZ); - bytes_allowed = tmp; - + bytes_allowed = calculate_bytes_allowed(bps_limit, jiffy_elapsed_rnd) + + tg->carryover_bytes[rw]; if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) { if (wait) *wait = 0; @@ -889,7 +938,7 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, * slice and it should be extended instead. */ if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) - throtl_start_new_slice(tg, rw); + throtl_start_new_slice(tg, rw, true); else { if (time_before(tg->slice_end[rw], jiffies + tg->td->throtl_slice)) @@ -897,8 +946,8 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, jiffies + tg->td->throtl_slice); } - if (tg_with_in_bps_limit(tg, bio, bps_limit, &bps_wait) && - tg_with_in_iops_limit(tg, bio, iops_limit, &iops_wait)) { + if (tg_within_bps_limit(tg, bio, bps_limit, &bps_wait) && + tg_within_iops_limit(tg, bio, iops_limit, &iops_wait)) { if (wait) *wait = 0; return true; @@ -921,22 +970,13 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) unsigned int bio_size = throtl_bio_data_size(bio); /* Charge the bio to the group */ - if (!bio_flagged(bio, BIO_THROTTLED)) { + if (!bio_flagged(bio, BIO_BPS_THROTTLED)) { tg->bytes_disp[rw] += bio_size; tg->last_bytes_disp[rw] += bio_size; } tg->io_disp[rw]++; tg->last_io_disp[rw]++; - - /* - * BIO_THROTTLED is used to prevent the same bio to be throttled - * more than once as a throttled bio will go through blk-throtl the - * second time when it eventually gets issued. Set it when a bio - * is being charged to a tg. - */ - if (!bio_flagged(bio, BIO_THROTTLED)) - bio_set_flag(bio, BIO_THROTTLED); } /** @@ -990,9 +1030,9 @@ static void tg_update_disptime(struct throtl_grp *tg) disptime = jiffies + min_wait; /* Update dispatch time */ - throtl_dequeue_tg(tg); + throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); tg->disptime = disptime; - throtl_enqueue_tg(tg); + tg_service_queue_add(tg); /* see throtl_add_bio_tg() */ tg->flags &= ~THROTL_TG_WAS_EMPTY; @@ -1026,6 +1066,7 @@ static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw) sq->nr_queued[rw]--; throtl_charge_bio(tg, bio); + bio_set_flag(bio, BIO_BPS_THROTTLED); /* * If our parent is another tg, we just need to transfer @bio to @@ -1101,13 +1142,13 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq) if (time_before(jiffies, tg->disptime)) break; - throtl_dequeue_tg(tg); - nr_disp += throtl_dispatch_tg(tg); sq = &tg->service_queue; - if (sq->nr_queued[0] || sq->nr_queued[1]) + if (sq->nr_queued[READ] || sq->nr_queued[WRITE]) tg_update_disptime(tg); + else + throtl_dequeue_tg(tg); if (nr_disp >= THROTL_QUANTUM) break; @@ -1321,8 +1362,8 @@ static void tg_conf_updated(struct throtl_grp *tg, bool global) * that a group's limit are dropped suddenly and we don't want to * account recently dispatched IO with new low rate. */ - throtl_start_new_slice(tg, READ); - throtl_start_new_slice(tg, WRITE); + throtl_start_new_slice(tg, READ, false); + throtl_start_new_slice(tg, WRITE, false); if (tg->flags & THROTL_TG_PENDING) { tg_update_disptime(tg); @@ -1350,6 +1391,7 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, v = U64_MAX; tg = blkg_to_tg(ctx.blkg); + tg_update_carryover(tg); if (is_u64) *(u64 *)((void *)tg + of_cft(of)->private) = v; @@ -1536,6 +1578,7 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of, return ret; tg = blkg_to_tg(ctx.blkg); + tg_update_carryover(tg); v[0] = tg->bps_conf[READ][index]; v[1] = tg->bps_conf[WRITE][index]; @@ -1673,6 +1716,41 @@ struct blkcg_policy blkcg_policy_throtl = { .pd_free_fn = throtl_pd_free, }; +void blk_throtl_cancel_bios(struct gendisk *disk) +{ + struct request_queue *q = disk->queue; + struct cgroup_subsys_state *pos_css; + struct blkcg_gq *blkg; + + spin_lock_irq(&q->queue_lock); + /* + * queue_lock is held, rcu lock is not needed here technically. + * However, rcu lock is still held to emphasize that following + * path need RCU protection and to prevent warning from lockdep. + */ + rcu_read_lock(); + blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { + struct throtl_grp *tg = blkg_to_tg(blkg); + struct throtl_service_queue *sq = &tg->service_queue; + + /* + * Set the flag to make sure throtl_pending_timer_fn() won't + * stop until all throttled bios are dispatched. + */ + blkg_to_tg(blkg)->flags |= THROTL_TG_CANCELING; + /* + * Update disptime after setting the above flag to make sure + * throtl_select_dispatch() won't exit without dispatching. + */ + tg_update_disptime(tg); + + throtl_schedule_pending_timer(sq, jiffies + 1); + } + rcu_read_unlock(); + spin_unlock_irq(&q->queue_lock); +} + +#ifdef CONFIG_BLK_DEV_THROTTLING_LOW static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg) { unsigned long rtime = jiffies, wtime = jiffies; @@ -1777,39 +1855,6 @@ static bool throtl_hierarchy_can_upgrade(struct throtl_grp *tg) return false; } -void blk_throtl_cancel_bios(struct request_queue *q) -{ - struct cgroup_subsys_state *pos_css; - struct blkcg_gq *blkg; - - spin_lock_irq(&q->queue_lock); - /* - * queue_lock is held, rcu lock is not needed here technically. - * However, rcu lock is still held to emphasize that following - * path need RCU protection and to prevent warning from lockdep. - */ - rcu_read_lock(); - blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { - struct throtl_grp *tg = blkg_to_tg(blkg); - struct throtl_service_queue *sq = &tg->service_queue; - - /* - * Set the flag to make sure throtl_pending_timer_fn() won't - * stop until all throttled bios are dispatched. - */ - blkg_to_tg(blkg)->flags |= THROTL_TG_CANCELING; - /* - * Update disptime after setting the above flag to make sure - * throtl_select_dispatch() won't exit without dispatching. - */ - tg_update_disptime(tg); - - throtl_schedule_pending_timer(sq, jiffies + 1); - } - rcu_read_unlock(); - spin_unlock_irq(&q->queue_lock); -} - static bool throtl_can_upgrade(struct throtl_data *td, struct throtl_grp *this_tg) { @@ -2005,7 +2050,6 @@ static void blk_throtl_update_idletime(struct throtl_grp *tg) tg->checked_last_finish_time = last_finish_time; } -#ifdef CONFIG_BLK_DEV_THROTTLING_LOW static void throtl_update_latency_buckets(struct throtl_data *td) { struct avg_latency_bucket avg_latency[2][LATENCY_BUCKET_SIZE]; @@ -2086,6 +2130,28 @@ static void throtl_update_latency_buckets(struct throtl_data *td) static inline void throtl_update_latency_buckets(struct throtl_data *td) { } + +static void blk_throtl_update_idletime(struct throtl_grp *tg) +{ +} + +static void throtl_downgrade_check(struct throtl_grp *tg) +{ +} + +static void throtl_upgrade_check(struct throtl_grp *tg) +{ +} + +static bool throtl_can_upgrade(struct throtl_data *td, + struct throtl_grp *this_tg) +{ + return false; +} + +static void throtl_upgrade_state(struct throtl_data *td) +{ +} #endif bool __blk_throtl_bio(struct bio *bio) @@ -2159,8 +2225,10 @@ again: qn = &tg->qnode_on_parent[rw]; sq = sq->parent_sq; tg = sq_to_tg(sq); - if (!tg) + if (!tg) { + bio_set_flag(bio, BIO_BPS_THROTTLED); goto out_unlock; + } } /* out-of-limit, queue to @tg */ @@ -2189,8 +2257,6 @@ again: } out_unlock: - bio_set_flag(bio, BIO_THROTTLED); - #ifdef CONFIG_BLK_DEV_THROTTLING_LOW if (throttled || !td->track_bio_latency) bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY; @@ -2286,8 +2352,9 @@ void blk_throtl_bio_endio(struct bio *bio) } #endif -int blk_throtl_init(struct request_queue *q) +int blk_throtl_init(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct throtl_data *td; int ret; @@ -2329,8 +2396,10 @@ int blk_throtl_init(struct request_queue *q) return ret; } -void blk_throtl_exit(struct request_queue *q) +void blk_throtl_exit(struct gendisk *disk) { + struct request_queue *q = disk->queue; + BUG_ON(!q->td); del_timer_sync(&q->td->service_queue.pending_timer); throtl_shutdown_wq(q); @@ -2340,8 +2409,9 @@ void blk_throtl_exit(struct request_queue *q) kfree(q->td); } -void blk_throtl_register_queue(struct request_queue *q) +void blk_throtl_register(struct gendisk *disk) { + struct request_queue *q = disk->queue; struct throtl_data *td; int i; diff --git a/block/blk-throttle.h b/block/blk-throttle.h index c1b60299612723ebc4cef93a0574a905d76ec780..ef4b7a4de987dddfd0dc368cd36893b899d8c3f2 100644 --- a/block/blk-throttle.h +++ b/block/blk-throttle.h @@ -55,8 +55,7 @@ struct throtl_service_queue { enum tg_state_flags { THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */ THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */ - THROTL_TG_HAS_IOPS_LIMIT = 1 << 2, /* tg has iops limit */ - THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */ + THROTL_TG_CANCELING = 1 << 2, /* starts to cancel bio */ }; enum { @@ -99,7 +98,8 @@ struct throtl_grp { unsigned int flags; /* are there any throtl rules between this group and td? */ - bool has_rules[2]; + bool has_rules_bps[2]; + bool has_rules_iops[2]; /* internally used bytes per second rate limits */ uint64_t bps[2][LIMIT_CNT]; @@ -121,6 +121,15 @@ struct throtl_grp { uint64_t last_bytes_disp[2]; unsigned int last_io_disp[2]; + /* + * The following two fields are updated when new configuration is + * submitted while some bios are still throttled, they record how many + * bytes/ios are waited already in previous configuration, and they will + * be used to calculate wait time under new configuration. + */ + uint64_t carryover_bytes[2]; + unsigned int carryover_ios[2]; + unsigned long last_check_time; unsigned long latency_target; /* us */ @@ -159,27 +168,37 @@ static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg) * Internal throttling interface */ #ifndef CONFIG_BLK_DEV_THROTTLING -static inline int blk_throtl_init(struct request_queue *q) { return 0; } -static inline void blk_throtl_exit(struct request_queue *q) { } -static inline void blk_throtl_register_queue(struct request_queue *q) { } +static inline int blk_throtl_init(struct gendisk *disk) { return 0; } +static inline void blk_throtl_exit(struct gendisk *disk) { } +static inline void blk_throtl_register(struct gendisk *disk) { } static inline bool blk_throtl_bio(struct bio *bio) { return false; } -static inline void blk_throtl_cancel_bios(struct request_queue *q) { } +static inline void blk_throtl_cancel_bios(struct gendisk *disk) { } #else /* CONFIG_BLK_DEV_THROTTLING */ -int blk_throtl_init(struct request_queue *q); -void blk_throtl_exit(struct request_queue *q); -void blk_throtl_register_queue(struct request_queue *q); +int blk_throtl_init(struct gendisk *disk); +void blk_throtl_exit(struct gendisk *disk); +void blk_throtl_register(struct gendisk *disk); bool __blk_throtl_bio(struct bio *bio); -void blk_throtl_cancel_bios(struct request_queue *q); -static inline bool blk_throtl_bio(struct bio *bio) +void blk_throtl_cancel_bios(struct gendisk *disk); + +static inline bool blk_should_throtl(struct bio *bio) { struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg); + int rw = bio_data_dir(bio); - /* no need to throttle bps any more if the bio has been throttled */ - if (bio_flagged(bio, BIO_THROTTLED) && - !(tg->flags & THROTL_TG_HAS_IOPS_LIMIT)) - return false; + /* iops limit is always counted */ + if (tg->has_rules_iops[rw]) + return true; + + if (tg->has_rules_bps[rw] && !bio_flagged(bio, BIO_BPS_THROTTLED)) + return true; + + return false; +} + +static inline bool blk_throtl_bio(struct bio *bio) +{ - if (!tg->has_rules[bio_data_dir(bio)]) + if (!blk_should_throtl(bio)) return false; return __blk_throtl_bio(bio); diff --git a/block/blk-wbt.c b/block/blk-wbt.c index a9982000b667e62f176d660186c2e66b46454445..c293e08b301ff6610c77262f7ccb259e4749c062 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -841,8 +841,11 @@ int wbt_init(struct request_queue *q) rwb->last_comp = rwb->last_issue = jiffies; rwb->win_nsec = RWB_WINDOW_NSEC; rwb->enable_state = WBT_STATE_ON_DEFAULT; - rwb->wc = 1; + rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags); rwb->rq_depth.default_depth = RWB_DEF_DEPTH; + rwb->min_lat_nsec = wbt_default_latency_nsec(q); + + wbt_queue_depth_changed(&rwb->rqos); /* * Assign rwb and add the stats callback. @@ -853,11 +856,6 @@ int wbt_init(struct request_queue *q) blk_stat_add_callback(q, rwb->cb); - rwb->min_lat_nsec = wbt_default_latency_nsec(q); - - wbt_queue_depth_changed(&rwb->rqos); - wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags)); - return 0; err_free: diff --git a/block/blk-zoned.c b/block/blk-zoned.c index a264621d490552f24c0c38f4493eb9e7b45a7d70..db829401d8d0ca377a9d871fc2c38aa58e3620da 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -63,13 +63,10 @@ bool blk_req_needs_zone_write_lock(struct request *rq) if (!rq->q->disk->seq_zones_wlock) return false; - switch (req_op(rq)) { - case REQ_OP_WRITE_ZEROES: - case REQ_OP_WRITE: + if (bdev_op_is_zoned_write(rq->q->disk->part0, req_op(rq))) return blk_rq_zone_is_seq(rq); - default: - return false; - } + + return false; } EXPORT_SYMBOL_GPL(blk_req_needs_zone_write_lock); diff --git a/block/blk.h b/block/blk.h index d7142c4d2fefb4a1bfa108fe43fa0ab36e477710..d6ea0d1a6db0ffb47941d45fd64fac7724eb74e8 100644 --- a/block/blk.h +++ b/block/blk.h @@ -88,6 +88,13 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset; phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset; + /* + * Merging adjacent physical pages may not work correctly under KMSAN + * if their metadata pages aren't adjacent. Just disable merging. + */ + if (IS_ENABLED(CONFIG_KMSAN)) + return false; + if (addr1 + vec1->bv_len != addr2) return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) @@ -270,8 +277,7 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, void blk_insert_flush(struct request *rq); -int elevator_switch_mq(struct request_queue *q, - struct elevator_type *new_e); +int elevator_switch(struct request_queue *q, struct elevator_type *new_e); void elevator_exit(struct request_queue *q); int elv_register_queue(struct request_queue *q, bool uevent); void elv_unregister_queue(struct request_queue *q); @@ -389,9 +395,9 @@ static inline struct bio *blk_queue_bounce(struct bio *bio, } #ifdef CONFIG_BLK_CGROUP_IOLATENCY -extern int blk_iolatency_init(struct request_queue *q); +int blk_iolatency_init(struct gendisk *disk); #else -static inline int blk_iolatency_init(struct request_queue *q) { return 0; } +static inline int blk_iolatency_init(struct gendisk *disk) { return 0; }; #endif #ifdef CONFIG_BLK_DEV_ZONED diff --git a/block/elevator.c b/block/elevator.c index c319765892bb90f00162cb6cac03f8d5434e64b2..bd71f0fc4e4b6725687da123cab3563a7370d18b 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -588,7 +588,7 @@ void elv_unregister(struct elevator_type *e) } EXPORT_SYMBOL_GPL(elv_unregister); -int elevator_switch_mq(struct request_queue *q, +static int elevator_switch_mq(struct request_queue *q, struct elevator_type *new_e) { int ret; @@ -723,7 +723,7 @@ void elevator_init_mq(struct request_queue *q) * need for the new one. this way we have a chance of going back to the old * one, if the new one fails init for some reason. */ -static int elevator_switch(struct request_queue *q, struct elevator_type *new_e) +int elevator_switch(struct request_queue *q, struct elevator_type *new_e) { int err; diff --git a/block/genhd.c b/block/genhd.c index d36fabf0abc1faaca05b18916f7f14f9cd1cee15..17b33c62423dfbe9233c97a6f79848d1283c7199 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -507,6 +507,13 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, */ dev_set_uevent_suppress(ddev, 0); disk_uevent(disk, KOBJ_ADD); + } else { + /* + * Even if the block_device for a hidden gendisk is not + * registered, it needs to have a valid bd_dev so that the + * freeing of the dynamic major works. + */ + disk->part0->bd_dev = MKDEV(disk->major, disk->first_minor); } disk_update_readahead(disk); @@ -602,7 +609,6 @@ void del_gendisk(struct gendisk *disk) * Prevent new I/O from crossing bio_queue_enter(). */ blk_queue_start_drain(q); - blk_mq_freeze_queue_wait(q); if (!(disk->flags & GENHD_FL_HIDDEN)) { sysfs_remove_link(&disk_to_dev(disk)->kobj, "bdi"); @@ -626,7 +632,9 @@ void del_gendisk(struct gendisk *disk) pm_runtime_set_memalloc_noio(disk_to_dev(disk), false); device_del(disk_to_dev(disk)); - blk_throtl_cancel_bios(disk->queue); + blk_mq_freeze_queue_wait(q); + + blk_throtl_cancel_bios(disk); blk_sync_queue(q); blk_flush_integrity(); @@ -1150,7 +1158,8 @@ static void disk_release(struct device *dev) !test_bit(GD_ADDED, &disk->state)) blk_mq_exit_queue(disk->queue); - blkcg_exit_queue(disk->queue); + blkcg_exit_disk(disk); + bioset_exit(&disk->bio_split); disk_release_events(disk); @@ -1363,7 +1372,7 @@ struct gendisk *__alloc_disk_node(struct request_queue *q, int node_id, if (xa_insert(&disk->part_tbl, 0, disk->part0, GFP_KERNEL)) goto out_destroy_part_tbl; - if (blkcg_init_queue(q)) + if (blkcg_init_disk(disk)) goto out_erase_part0; rand_initialize_disk(disk); diff --git a/block/opal_proto.h b/block/opal_proto.h index b486b3ec7dc41e95a203524400b06a1a5f5f610c..7152aa1f1a49e3bf177ce65f4e64990892deca57 100644 --- a/block/opal_proto.h +++ b/block/opal_proto.h @@ -39,7 +39,12 @@ enum opal_response_token { #define FIRST_TPER_SESSION_NUM 4096 #define TPER_SYNC_SUPPORTED 0x01 +/* FC_LOCKING features */ +#define LOCKING_SUPPORTED_MASK 0x01 +#define LOCKING_ENABLED_MASK 0x02 +#define LOCKED_MASK 0x04 #define MBR_ENABLED_MASK 0x10 +#define MBR_DONE_MASK 0x20 #define TINY_ATOM_DATA_MASK 0x3F #define TINY_ATOM_SIGNED 0x40 diff --git a/block/partitions/core.c b/block/partitions/core.c index fc1d70384825cd804cce322038694126dbcbc4b8..b8112f52d38800742a2e51eb4af61fa116da35d6 100644 --- a/block/partitions/core.c +++ b/block/partitions/core.c @@ -596,6 +596,9 @@ static int blk_add_partitions(struct gendisk *disk) if (disk->flags & GENHD_FL_NO_PART) return 0; + if (test_bit(GD_SUPPRESS_PART_SCAN, &disk->state)) + return 0; + state = check_partition(disk); if (!state) return 0; diff --git a/block/sed-opal.c b/block/sed-opal.c index 9700197000f206eb1e9b88695a960beecfdb8ea7..2c5327a0543a66889e3b5eb0c0ec0786e1845c19 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -74,8 +74,7 @@ struct parsed_resp { }; struct opal_dev { - bool supported; - bool mbr_enabled; + u32 flags; void *data; sec_send_recv *send_recv; @@ -280,6 +279,30 @@ static bool check_tper(const void *data) return true; } +static bool check_lcksuppt(const void *data) +{ + const struct d0_locking_features *lfeat = data; + u8 sup_feat = lfeat->supported_features; + + return !!(sup_feat & LOCKING_SUPPORTED_MASK); +} + +static bool check_lckenabled(const void *data) +{ + const struct d0_locking_features *lfeat = data; + u8 sup_feat = lfeat->supported_features; + + return !!(sup_feat & LOCKING_ENABLED_MASK); +} + +static bool check_locked(const void *data) +{ + const struct d0_locking_features *lfeat = data; + u8 sup_feat = lfeat->supported_features; + + return !!(sup_feat & LOCKED_MASK); +} + static bool check_mbrenabled(const void *data) { const struct d0_locking_features *lfeat = data; @@ -288,6 +311,14 @@ static bool check_mbrenabled(const void *data) return !!(sup_feat & MBR_ENABLED_MASK); } +static bool check_mbrdone(const void *data) +{ + const struct d0_locking_features *lfeat = data; + u8 sup_feat = lfeat->supported_features; + + return !!(sup_feat & MBR_DONE_MASK); +} + static bool check_sum(const void *data) { const struct d0_single_user_mode *sum = data; @@ -435,7 +466,7 @@ static int opal_discovery0_end(struct opal_dev *dev) u32 hlen = be32_to_cpu(hdr->length); print_buffer(dev->resp, hlen); - dev->mbr_enabled = false; + dev->flags &= OPAL_FL_SUPPORTED; if (hlen > IO_BUFFER_LENGTH - sizeof(*hdr)) { pr_debug("Discovery length overflows buffer (%zu+%u)/%u\n", @@ -461,7 +492,16 @@ static int opal_discovery0_end(struct opal_dev *dev) check_geometry(dev, body); break; case FC_LOCKING: - dev->mbr_enabled = check_mbrenabled(body->features); + if (check_lcksuppt(body->features)) + dev->flags |= OPAL_FL_LOCKING_SUPPORTED; + if (check_lckenabled(body->features)) + dev->flags |= OPAL_FL_LOCKING_ENABLED; + if (check_locked(body->features)) + dev->flags |= OPAL_FL_LOCKED; + if (check_mbrenabled(body->features)) + dev->flags |= OPAL_FL_MBR_ENABLED; + if (check_mbrdone(body->features)) + dev->flags |= OPAL_FL_MBR_DONE; break; case FC_ENTERPRISE: case FC_DATASTORE: @@ -2109,7 +2149,8 @@ static int check_opal_support(struct opal_dev *dev) mutex_lock(&dev->dev_lock); setup_opal_dev(dev); ret = opal_discovery0_step(dev); - dev->supported = !ret; + if (!ret) + dev->flags |= OPAL_FL_SUPPORTED; mutex_unlock(&dev->dev_lock); return ret; @@ -2148,6 +2189,7 @@ struct opal_dev *init_opal_dev(void *data, sec_send_recv *send_recv) INIT_LIST_HEAD(&dev->unlk_lst); mutex_init(&dev->dev_lock); + dev->flags = 0; dev->data = data; dev->send_recv = send_recv; if (check_opal_support(dev) != 0) { @@ -2528,7 +2570,7 @@ bool opal_unlock_from_suspend(struct opal_dev *dev) if (!dev) return false; - if (!dev->supported) + if (!(dev->flags & OPAL_FL_SUPPORTED)) return false; mutex_lock(&dev->dev_lock); @@ -2546,7 +2588,7 @@ bool opal_unlock_from_suspend(struct opal_dev *dev) was_failure = true; } - if (dev->mbr_enabled) { + if (dev->flags & OPAL_FL_MBR_ENABLED) { ret = __opal_set_mbr_done(dev, &suspend->unlk.session.opal_key); if (ret) pr_debug("Failed to set MBR Done in S3 resume\n"); @@ -2620,6 +2662,23 @@ static int opal_generic_read_write_table(struct opal_dev *dev, return ret; } +static int opal_get_status(struct opal_dev *dev, void __user *data) +{ + struct opal_status sts = {0}; + + /* + * check_opal_support() error is not fatal, + * !dev->supported is a valid condition + */ + if (!check_opal_support(dev)) + sts.flags = dev->flags; + if (copy_to_user(data, &sts, sizeof(sts))) { + pr_debug("Error copying status to userspace\n"); + return -EFAULT; + } + return 0; +} + int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg) { void *p; @@ -2629,12 +2688,14 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg) return -EACCES; if (!dev) return -ENOTSUPP; - if (!dev->supported) + if (!(dev->flags & OPAL_FL_SUPPORTED)) return -ENOTSUPP; - p = memdup_user(arg, _IOC_SIZE(cmd)); - if (IS_ERR(p)) - return PTR_ERR(p); + if (cmd & IOC_IN) { + p = memdup_user(arg, _IOC_SIZE(cmd)); + if (IS_ERR(p)) + return PTR_ERR(p); + } switch (cmd) { case IOC_OPAL_SAVE: @@ -2685,11 +2746,15 @@ int sed_ioctl(struct opal_dev *dev, unsigned int cmd, void __user *arg) case IOC_OPAL_GENERIC_TABLE_RW: ret = opal_generic_read_write_table(dev, p); break; + case IOC_OPAL_GET_STATUS: + ret = opal_get_status(dev, arg); + break; default: break; } - kfree(p); + if (cmd & IOC_IN) + kfree(p); return ret; } EXPORT_SYMBOL_GPL(sed_ioctl); diff --git a/certs/Kconfig b/certs/Kconfig index bf9b511573d75135182a3f15711d617c5e335e38..1f109b0708778bcc051b3840893dd1921acb92a5 100644 --- a/certs/Kconfig +++ b/certs/Kconfig @@ -43,7 +43,7 @@ config SYSTEM_TRUSTED_KEYRING bool "Provide system-wide ring of trusted keys" depends on KEYS depends on ASYMMETRIC_KEY_TYPE - depends on X509_CERTIFICATE_PARSER + depends on X509_CERTIFICATE_PARSER = y help Provide a system keyring to which trusted keys can be added. Keys in the keyring are considered to be trusted. Keys may be added at will diff --git a/crypto/Kconfig b/crypto/Kconfig index bb427a835e44a6e57f015e01bd605b923ae6c647..d779667671b23f03c57b5e887627f06da775ef2f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -15,13 +15,13 @@ source "crypto/async_tx/Kconfig" # menuconfig CRYPTO tristate "Cryptographic API" - select LIB_MEMNEQ + select CRYPTO_LIB_UTILS help This option provides the core Cryptographic API. if CRYPTO -comment "Crypto core or helper" +menu "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" @@ -219,7 +219,8 @@ config CRYPTO_AUTHENC select CRYPTO_NULL help Authenc: Combined mode wrapper for IPsec. - This is required for IPSec. + + This is required for IPSec ESP (XFRM_ESP). config CRYPTO_TEST tristate "Testing module" @@ -235,54 +236,65 @@ config CRYPTO_SIMD config CRYPTO_ENGINE tristate -comment "Public-key cryptography" +endmenu + +menu "Public-key cryptography" config CRYPTO_RSA - tristate "RSA algorithm" + tristate "RSA (Rivest-Shamir-Adleman)" select CRYPTO_AKCIPHER select CRYPTO_MANAGER select MPILIB select ASN1 help - Generic implementation of the RSA public key algorithm. + RSA (Rivest-Shamir-Adleman) public key algorithm (RFC8017) config CRYPTO_DH - tristate "Diffie-Hellman algorithm" + tristate "DH (Diffie-Hellman)" select CRYPTO_KPP select MPILIB help - Generic implementation of the Diffie-Hellman algorithm. + DH (Diffie-Hellman) key exchange algorithm config CRYPTO_DH_RFC7919_GROUPS - bool "Support for RFC 7919 FFDHE group parameters" + bool "RFC 7919 FFDHE groups" depends on CRYPTO_DH select CRYPTO_RNG_DEFAULT help - Provide support for RFC 7919 FFDHE group parameters. If unsure, say N. + FFDHE (Finite-Field-based Diffie-Hellman Ephemeral) groups + defined in RFC7919. + + Support these finite-field groups in DH key exchanges: + - ffdhe2048, ffdhe3072, ffdhe4096, ffdhe6144, ffdhe8192 + + If unsure, say N. config CRYPTO_ECC tristate select CRYPTO_RNG_DEFAULT config CRYPTO_ECDH - tristate "ECDH algorithm" + tristate "ECDH (Elliptic Curve Diffie-Hellman)" select CRYPTO_ECC select CRYPTO_KPP help - Generic implementation of the ECDH algorithm + ECDH (Elliptic Curve Diffie-Hellman) key exchange algorithm + using curves P-192, P-256, and P-384 (FIPS 186) config CRYPTO_ECDSA - tristate "ECDSA (NIST P192, P256 etc.) algorithm" + tristate "ECDSA (Elliptic Curve Digital Signature Algorithm)" select CRYPTO_ECC select CRYPTO_AKCIPHER select ASN1 help - Elliptic Curve Digital Signature Algorithm (NIST P192, P256 etc.) - is A NIST cryptographic standard algorithm. Only signature verification - is implemented. + ECDSA (Elliptic Curve Digital Signature Algorithm) (FIPS 186, + ISO/IEC 14888-3) + using curves P-192, P-256, and P-384 + + Only signature verification is implemented. config CRYPTO_ECRDSA - tristate "EC-RDSA (GOST 34.10) algorithm" + tristate "EC-RDSA (Elliptic Curve Russian Digital Signature Algorithm)" select CRYPTO_ECC select CRYPTO_AKCIPHER select CRYPTO_STREEBOG @@ -290,1725 +302,947 @@ config CRYPTO_ECRDSA select ASN1 help Elliptic Curve Russian Digital Signature Algorithm (GOST R 34.10-2012, - RFC 7091, ISO/IEC 14888-3:2018) is one of the Russian cryptographic - standard algorithms (called GOST algorithms). Only signature verification - is implemented. + RFC 7091, ISO/IEC 14888-3) + + One of the Russian cryptographic standard algorithms (called GOST + algorithms). Only signature verification is implemented. config CRYPTO_SM2 - tristate "SM2 algorithm" + tristate "SM2 (ShangMi 2)" select CRYPTO_SM3 select CRYPTO_AKCIPHER select CRYPTO_MANAGER select MPILIB select ASN1 help - Generic implementation of the SM2 public key algorithm. It was - published by State Encryption Management Bureau, China. + SM2 (ShangMi 2) public key algorithm + + Published by State Encryption Management Bureau, China, as specified by OSCCA GM/T 0003.1-2012 -- 0003.5-2012. References: - https://tools.ietf.org/html/draft-shen-sm2-ecdsa-02 + https://datatracker.ietf.org/doc/draft-shen-sm2-ecdsa/ http://www.oscca.gov.cn/sca/xxgk/2010-12/17/content_1002386.shtml http://www.gmbz.org.cn/main/bzlb.html config CRYPTO_CURVE25519 - tristate "Curve25519 algorithm" + tristate "Curve25519" select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC + help + Curve25519 elliptic curve (RFC7748) -config CRYPTO_CURVE25519_X86 - tristate "x86_64 accelerated Curve25519 scalar multiplication library" - depends on X86 && 64BIT - select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 +endmenu -comment "Authenticated Encryption with Associated Data" +menu "Block ciphers" -config CRYPTO_CCM - tristate "CCM support" - select CRYPTO_CTR - select CRYPTO_HASH - select CRYPTO_AEAD - select CRYPTO_MANAGER +config CRYPTO_AES + tristate "AES (Advanced Encryption Standard)" + select CRYPTO_ALGAPI + select CRYPTO_LIB_AES help - Support for Counter with CBC MAC. Required for IPsec. + AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) -config CRYPTO_GCM - tristate "GCM/GMAC support" - select CRYPTO_CTR - select CRYPTO_AEAD - select CRYPTO_GHASH - select CRYPTO_NULL - select CRYPTO_MANAGER - help - Support for Galois/Counter Mode (GCM) and Galois Message - Authentication Code (GMAC). Required for IPSec. + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. -config CRYPTO_CHACHA20POLY1305 - tristate "ChaCha20-Poly1305 AEAD support" - select CRYPTO_CHACHA20 - select CRYPTO_POLY1305 - select CRYPTO_AEAD - select CRYPTO_MANAGER + The AES specifies three key sizes: 128, 192 and 256 bits + +config CRYPTO_AES_TI + tristate "AES (Advanced Encryption Standard) (fixed time)" + select CRYPTO_ALGAPI + select CRYPTO_LIB_AES help - ChaCha20-Poly1305 AEAD support, RFC7539. + AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) + + This is a generic implementation of AES that attempts to eliminate + data dependent latencies as much as possible without affecting + performance too much. It is intended for use by the generic CCM + and GCM drivers, and other CTR or CMAC/XCBC based modes that rely + solely on encryption (although decryption is supported as well, but + with a more dramatic performance hit) - Support for the AEAD wrapper using the ChaCha20 stream cipher combined - with the Poly1305 authenticator. It is defined in RFC7539 for use in - IETF protocols. + Instead of using 16 lookup tables of 1 KB each, (8 for encryption and + 8 for decryption), this implementation only uses just two S-boxes of + 256 bytes each, and attempts to eliminate data dependent latencies by + prefetching the entire table into the cache at the start of each + block. Interrupts are also disabled to avoid races where cachelines + are evicted when the CPU is interrupted to do something else. -config CRYPTO_AEGIS128 - tristate "AEGIS-128 AEAD algorithm" - select CRYPTO_AEAD - select CRYPTO_AES # for AES S-box tables +config CRYPTO_ANUBIS + tristate "Anubis" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI help - Support for the AEGIS-128 dedicated AEAD algorithm. + Anubis cipher algorithm -config CRYPTO_AEGIS128_SIMD - bool "Support SIMD acceleration for AEGIS-128" - depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) - default y + Anubis is a variable key length cipher which can use keys from + 128 bits to 320 bits in length. It was evaluated as a entrant + in the NESSIE competition. -config CRYPTO_AEGIS128_AESNI_SSE2 - tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)" - depends on X86 && 64BIT - select CRYPTO_AEAD - select CRYPTO_SIMD - help - AESNI+SSE2 implementation of the AEGIS-128 dedicated AEAD algorithm. + See https://web.archive.org/web/20160606112246/http://www.larc.usp.br/~pbarreto/AnubisPage.html + for further information. -config CRYPTO_SEQIV - tristate "Sequence Number IV Generator" - select CRYPTO_AEAD - select CRYPTO_SKCIPHER - select CRYPTO_NULL - select CRYPTO_RNG_DEFAULT - select CRYPTO_MANAGER +config CRYPTO_ARIA + tristate "ARIA" + select CRYPTO_ALGAPI help - This IV generator generates an IV based on a sequence number by - xoring it with a salt. This algorithm is mainly useful for CTR + ARIA cipher algorithm (RFC5794) -config CRYPTO_ECHAINIV - tristate "Encrypted Chain IV Generator" - select CRYPTO_AEAD - select CRYPTO_NULL - select CRYPTO_RNG_DEFAULT - select CRYPTO_MANAGER - help - This IV generator generates an IV based on the encryption of - a sequence number xored with a salt. This is the default - algorithm for CBC. + ARIA is a standard encryption algorithm of the Republic of Korea. + The ARIA specifies three key sizes and rounds. + 128-bit: 12 rounds. + 192-bit: 14 rounds. + 256-bit: 16 rounds. -comment "Block modes" + See: + https://seed.kisa.or.kr/kisa/algorithm/EgovAriaInfo.do -config CRYPTO_CBC - tristate "CBC support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_BLOWFISH + tristate "Blowfish" + select CRYPTO_ALGAPI + select CRYPTO_BLOWFISH_COMMON help - CBC: Cipher Block Chaining mode - This block cipher algorithm is required for IPSec. + Blowfish cipher algorithm, by Bruce Schneier -config CRYPTO_CFB - tristate "CFB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - CFB: Cipher FeedBack mode - This block cipher algorithm is required for TPM2 Cryptography. + This is a variable key length cipher which can use keys from 32 + bits to 448 bits in length. It's fast, simple and specifically + designed for use on "large microprocessors". -config CRYPTO_CTR - tristate "CTR support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER + See https://www.schneier.com/blowfish.html for further information. + +config CRYPTO_BLOWFISH_COMMON + tristate help - CTR: Counter mode - This block cipher algorithm is required for IPSec. + Common parts of the Blowfish cipher algorithm shared by the + generic c and the assembler implementations. -config CRYPTO_CTS - tristate "CTS support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAMELLIA + tristate "Camellia" + select CRYPTO_ALGAPI help - CTS: Cipher Text Stealing - This is the Cipher Text Stealing mode as described by - Section 8 of rfc2040 and referenced by rfc3962 - (rfc3962 includes errata information in its Appendix A) or - CBC-CS3 as defined by NIST in Sp800-38A addendum from Oct 2010. - This mode is required for Kerberos gss mechanism support - for AES encryption. + Camellia cipher algorithms (ISO/IEC 18033-3) - See: https://csrc.nist.gov/publications/detail/sp/800-38a/addendum/final + Camellia is a symmetric key block cipher developed jointly + at NTT and Mitsubishi Electric Corporation. -config CRYPTO_ECB - tristate "ECB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - ECB: Electronic CodeBook mode - This is the simplest block cipher algorithm. It simply encrypts - the input block by block. + The Camellia specifies three key sizes: 128, 192 and 256 bits. -config CRYPTO_LRW - tristate "LRW support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - select CRYPTO_GF128MUL - select CRYPTO_ECB - help - LRW: Liskov Rivest Wagner, a tweakable, non malleable, non movable - narrow block cipher mode for dm-crypt. Use it with cipher - specification string aes-lrw-benbi, the key must be 256, 320 or 384. - The first 128, 192 or 256 bits in the key are used for AES and the - rest is used to tie each cipher block to its logical position. + See https://info.isl.ntt.co.jp/crypt/eng/camellia/ for further information. -config CRYPTO_OFB - tristate "OFB support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAST_COMMON + tristate help - OFB: the Output Feedback mode makes a block cipher into a synchronous - stream cipher. It generates keystream blocks, which are then XORed - with the plaintext blocks to get the ciphertext. Flipping a bit in the - ciphertext produces a flipped bit in the plaintext at the same - location. This property allows many error correcting codes to function - normally even when applied before encryption. + Common parts of the CAST cipher algorithms shared by the + generic c and the assembler implementations. -config CRYPTO_PCBC - tristate "PCBC support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAST5 + tristate "CAST5 (CAST-128)" + select CRYPTO_ALGAPI + select CRYPTO_CAST_COMMON help - PCBC: Propagating Cipher Block Chaining mode - This block cipher algorithm is required for RxRPC. + CAST5 (CAST-128) cipher algorithm (RFC2144, ISO/IEC 18033-3) -config CRYPTO_XCTR - tristate - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER +config CRYPTO_CAST6 + tristate "CAST6 (CAST-256)" + select CRYPTO_ALGAPI + select CRYPTO_CAST_COMMON help - XCTR: XOR Counter mode. This blockcipher mode is a variant of CTR mode - using XORs and little-endian addition rather than big-endian arithmetic. - XCTR mode is used to implement HCTR2. + CAST6 (CAST-256) encryption algorithm (RFC2612) -config CRYPTO_XTS - tristate "XTS support" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - select CRYPTO_ECB +config CRYPTO_DES + tristate "DES and Triple DES EDE" + select CRYPTO_ALGAPI + select CRYPTO_LIB_DES help - XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain, - key size 256, 384 or 512 bits. This implementation currently - can't handle a sectorsize which is not a multiple of 16 bytes. + DES (Data Encryption Standard)(FIPS 46-2, ISO/IEC 18033-3) and + Triple DES EDE (Encrypt/Decrypt/Encrypt) (FIPS 46-3, ISO/IEC 18033-3) + cipher algorithms -config CRYPTO_KEYWRAP - tristate "Key wrapping support" +config CRYPTO_FCRYPT + tristate "FCrypt" + select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_MANAGER help - Support for key wrapping (NIST SP800-38F / RFC3394) without - padding. + FCrypt algorithm used by RxRPC -config CRYPTO_NHPOLY1305 - tristate - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC + See https://ota.polyonymo.us/fcrypt-paper.txt -config CRYPTO_NHPOLY1305_SSE2 - tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)" - depends on X86 && 64BIT - select CRYPTO_NHPOLY1305 +config CRYPTO_KHAZAD + tristate "Khazad" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI help - SSE2 optimized implementation of the hash function used by the - Adiantum encryption mode. + Khazad cipher algorithm -config CRYPTO_NHPOLY1305_AVX2 - tristate "NHPoly1305 hash function (x86_64 AVX2 implementation)" - depends on X86 && 64BIT - select CRYPTO_NHPOLY1305 - help - AVX2 optimized implementation of the hash function used by the - Adiantum encryption mode. + Khazad was a finalist in the initial NESSIE competition. It is + an algorithm optimized for 64-bit processors with good performance + on 32-bit processors. Khazad uses an 128 bit key size. -config CRYPTO_ADIANTUM - tristate "Adiantum support" - select CRYPTO_CHACHA20 - select CRYPTO_LIB_POLY1305_GENERIC - select CRYPTO_NHPOLY1305 - select CRYPTO_MANAGER - help - Adiantum is a tweakable, length-preserving encryption mode - designed for fast and secure disk encryption, especially on - CPUs without dedicated crypto instructions. It encrypts - each sector using the XChaCha12 stream cipher, two passes of - an ε-almost-∆-universal hash function, and an invocation of - the AES-256 block cipher on a single 16-byte block. On CPUs - without AES instructions, Adiantum is much faster than - AES-XTS. + See https://web.archive.org/web/20171011071731/http://www.larc.usp.br/~pbarreto/KhazadPage.html + for further information. - Adiantum's security is provably reducible to that of its - underlying stream and block ciphers, subject to a security - bound. Unlike XTS, Adiantum is a true wide-block encryption - mode, so it actually provides an even stronger notion of - security than XTS, subject to the security bound. +config CRYPTO_SEED + tristate "SEED" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE + select CRYPTO_ALGAPI + help + SEED cipher algorithm (RFC4269, ISO/IEC 18033-3) - If unsure, say N. + SEED is a 128-bit symmetric key block cipher that has been + developed by KISA (Korea Information Security Agency) as a + national standard encryption algorithm of the Republic of Korea. + It is a 16 round block cipher with the key size of 128 bit. -config CRYPTO_HCTR2 - tristate "HCTR2 support" - select CRYPTO_XCTR - select CRYPTO_POLYVAL - select CRYPTO_MANAGER - help - HCTR2 is a length-preserving encryption mode for storage encryption that - is efficient on processors with instructions to accelerate AES and - carryless multiplication, e.g. x86 processors with AES-NI and CLMUL, and - ARM processors with the ARMv8 crypto extensions. + See https://seed.kisa.or.kr/kisa/algorithm/EgovSeedInfo.do + for further information. -config CRYPTO_ESSIV - tristate "ESSIV support for block encryption" - select CRYPTO_AUTHENC +config CRYPTO_SERPENT + tristate "Serpent" + select CRYPTO_ALGAPI help - Encrypted salt-sector initialization vector (ESSIV) is an IV - generation method that is used in some cases by fscrypt and/or - dm-crypt. It uses the hash of the block encryption key as the - symmetric key for a block encryption pass applied to the input - IV, making low entropy IV sources more suitable for block - encryption. + Serpent cipher algorithm, by Anderson, Biham & Knudsen - This driver implements a crypto API template that can be - instantiated either as an skcipher or as an AEAD (depending on the - type of the first template argument), and which defers encryption - and decryption requests to the encapsulated cipher after applying - ESSIV to the input IV. Note that in the AEAD case, it is assumed - that the keys are presented in the same format used by the authenc - template, and that the IV appears at the end of the authenticated - associated data (AAD) region (which is how dm-crypt uses it.) + Keys are allowed to be from 0 to 256 bits in length, in steps + of 8 bits. - Note that the use of ESSIV is not recommended for new deployments, - and so this only needs to be enabled when interoperability with - existing encrypted volumes of filesystems is required, or when - building for a particular system that requires it (e.g., when - the SoC in question has accelerated CBC but not XTS, making CBC - combined with ESSIV the only feasible mode for h/w accelerated - block encryption) + See https://www.cl.cam.ac.uk/~rja14/serpent.html for further information. -comment "Hash modes" +config CRYPTO_SM4 + tristate -config CRYPTO_CMAC - tristate "CMAC support" - select CRYPTO_HASH - select CRYPTO_MANAGER +config CRYPTO_SM4_GENERIC + tristate "SM4 (ShangMi 4)" + select CRYPTO_ALGAPI + select CRYPTO_SM4 help - Cipher-based Message Authentication Code (CMAC) specified by - The National Institute of Standards and Technology (NIST). + SM4 cipher algorithms (OSCCA GB/T 32907-2016, + ISO/IEC 18033-3:2010/Amd 1:2021) - https://tools.ietf.org/html/rfc4493 - http://csrc.nist.gov/publications/nistpubs/800-38B/SP_800-38B.pdf - -config CRYPTO_HMAC - tristate "HMAC support" - select CRYPTO_HASH - select CRYPTO_MANAGER - help - HMAC: Keyed-Hashing for Message Authentication (RFC2104). - This is required for IPSec. - -config CRYPTO_XCBC - tristate "XCBC support" - select CRYPTO_HASH - select CRYPTO_MANAGER - help - XCBC: Keyed-Hashing with encryption algorithm - https://www.ietf.org/rfc/rfc3566.txt - http://csrc.nist.gov/encryption/modes/proposedmodes/ - xcbc-mac/xcbc-mac-spec.pdf - -config CRYPTO_VMAC - tristate "VMAC support" - select CRYPTO_HASH - select CRYPTO_MANAGER - help - VMAC is a message authentication algorithm designed for - very high speed on 64-bit architectures. - - See also: - - -comment "Digest" - -config CRYPTO_CRC32C - tristate "CRC32c CRC algorithm" - select CRYPTO_HASH - select CRC32 - help - Castagnoli, et al Cyclic Redundancy-Check Algorithm. Used - by iSCSI for header and data digests and by others. - See Castagnoli93. Module will be crc32c. - -config CRYPTO_CRC32C_INTEL - tristate "CRC32c INTEL hardware acceleration" - depends on X86 - select CRYPTO_HASH - help - In Intel processor with SSE4.2 supported, the processor will - support CRC32C implementation using hardware accelerated CRC32 - instruction. This option will create 'crc32c-intel' module, - which will enable any routine to use the CRC32 instruction to - gain performance compared with software implementation. - Module will be crc32c-intel. - -config CRYPTO_CRC32C_VPMSUM - tristate "CRC32c CRC algorithm (powerpc64)" - depends on PPC64 && ALTIVEC - select CRYPTO_HASH - select CRC32 - help - CRC32c algorithm implemented using vector polynomial multiply-sum - (vpmsum) instructions, introduced in POWER8. Enable on POWER8 - and newer processors for improved performance. - - -config CRYPTO_CRC32C_SPARC64 - tristate "CRC32c CRC algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_HASH - select CRC32 - help - CRC32c CRC algorithm implemented using sparc64 crypto instructions, - when available. - -config CRYPTO_CRC32 - tristate "CRC32 CRC algorithm" - select CRYPTO_HASH - select CRC32 - help - CRC-32-IEEE 802.3 cyclic redundancy-check algorithm. - Shash crypto api wrappers to crc32_le function. - -config CRYPTO_CRC32_PCLMUL - tristate "CRC32 PCLMULQDQ hardware acceleration" - depends on X86 - select CRYPTO_HASH - select CRC32 - help - From Intel Westmere and AMD Bulldozer processor with SSE4.2 - and PCLMULQDQ supported, the processor will support - CRC32 PCLMULQDQ implementation using hardware accelerated PCLMULQDQ - instruction. This option will create 'crc32-pclmul' module, - which will enable any routine to use the CRC-32-IEEE 802.3 checksum - and gain better performance as compared with the table implementation. - -config CRYPTO_CRC32_MIPS - tristate "CRC32c and CRC32 CRC algorithm (MIPS)" - depends on MIPS_CRC_SUPPORT - select CRYPTO_HASH - help - CRC32c and CRC32 CRC algorithms implemented using mips crypto - instructions, when available. - -config CRYPTO_CRC32_S390 - tristate "CRC-32 algorithms" - depends on S390 - select CRYPTO_HASH - select CRC32 - help - Select this option if you want to use hardware accelerated - implementations of CRC algorithms. With this option, you - can optimize the computation of CRC-32 (IEEE 802.3 Ethernet) - and CRC-32C (Castagnoli). - - It is available with IBM z13 or later. - -config CRYPTO_XXHASH - tristate "xxHash hash algorithm" - select CRYPTO_HASH - select XXHASH - help - xxHash non-cryptographic hash algorithm. Extremely fast, working at - speeds close to RAM limits. - -config CRYPTO_BLAKE2B - tristate "BLAKE2b digest algorithm" - select CRYPTO_HASH - help - Implementation of cryptographic hash function BLAKE2b (or just BLAKE2), - optimized for 64bit platforms and can produce digests of any size - between 1 to 64. The keyed hash is also implemented. - - This module provides the following algorithms: - - - blake2b-160 - - blake2b-256 - - blake2b-384 - - blake2b-512 - - See https://blake2.net for further information. - -config CRYPTO_BLAKE2S_X86 - bool "BLAKE2s digest algorithm (x86 accelerated version)" - depends on X86 && 64BIT - select CRYPTO_LIB_BLAKE2S_GENERIC - select CRYPTO_ARCH_HAVE_LIB_BLAKE2S - -config CRYPTO_CRCT10DIF - tristate "CRCT10DIF algorithm" - select CRYPTO_HASH - help - CRC T10 Data Integrity Field computation is being cast as - a crypto transform. This allows for faster crc t10 diff - transforms to be used if they are available. - -config CRYPTO_CRCT10DIF_PCLMUL - tristate "CRCT10DIF PCLMULQDQ hardware acceleration" - depends on X86 && 64BIT && CRC_T10DIF - select CRYPTO_HASH - help - For x86_64 processors with SSE4.2 and PCLMULQDQ supported, - CRC T10 DIF PCLMULQDQ computation can be hardware - accelerated PCLMULQDQ instruction. This option will create - 'crct10dif-pclmul' module, which is faster when computing the - crct10dif checksum as compared with the generic table implementation. - -config CRYPTO_CRCT10DIF_VPMSUM - tristate "CRC32T10DIF powerpc64 hardware acceleration" - depends on PPC64 && ALTIVEC && CRC_T10DIF - select CRYPTO_HASH - help - CRC10T10DIF algorithm implemented using vector polynomial - multiply-sum (vpmsum) instructions, introduced in POWER8. Enable on - POWER8 and newer processors for improved performance. - -config CRYPTO_CRC64_ROCKSOFT - tristate "Rocksoft Model CRC64 algorithm" - depends on CRC64 - select CRYPTO_HASH - -config CRYPTO_VPMSUM_TESTER - tristate "Powerpc64 vpmsum hardware acceleration tester" - depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM - help - Stress test for CRC32c and CRC-T10DIF algorithms implemented with - POWER8 vpmsum instructions. - Unless you are testing these algorithms, you don't need this. - -config CRYPTO_GHASH - tristate "GHASH hash function" - select CRYPTO_GF128MUL - select CRYPTO_HASH - help - GHASH is the hash function used in GCM (Galois/Counter Mode). - It is not a general-purpose cryptographic hash function. - -config CRYPTO_POLYVAL - tristate - select CRYPTO_GF128MUL - select CRYPTO_HASH - help - POLYVAL is the hash function used in HCTR2. It is not a general-purpose - cryptographic hash function. - -config CRYPTO_POLYVAL_CLMUL_NI - tristate "POLYVAL hash function (CLMUL-NI accelerated)" - depends on X86 && 64BIT - select CRYPTO_POLYVAL - help - This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is - used to efficiently implement HCTR2 on x86-64 processors that support - carry-less multiplication instructions. - -config CRYPTO_POLY1305 - tristate "Poly1305 authenticator algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm, RFC7539. - - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the portable C implementation of Poly1305. - -config CRYPTO_POLY1305_X86_64 - tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)" - depends on X86 && 64BIT - select CRYPTO_LIB_POLY1305_GENERIC - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - help - Poly1305 authenticator algorithm, RFC7539. - - Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. - It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use - in IETF protocols. This is the x86_64 assembler implementation using SIMD - instructions. - -config CRYPTO_POLY1305_MIPS - tristate "Poly1305 authenticator algorithm (MIPS optimized)" - depends on MIPS - select CRYPTO_ARCH_HAVE_LIB_POLY1305 - -config CRYPTO_MD4 - tristate "MD4 digest algorithm" - select CRYPTO_HASH - help - MD4 message digest algorithm (RFC1320). - -config CRYPTO_MD5 - tristate "MD5 digest algorithm" - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321). - -config CRYPTO_MD5_OCTEON - tristate "MD5 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_MD5 - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - using OCTEON crypto instructions, when available. - -config CRYPTO_MD5_PPC - tristate "MD5 digest algorithm (PPC)" - depends on PPC - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - in PPC assembler. - -config CRYPTO_MD5_SPARC64 - tristate "MD5 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_MD5 - select CRYPTO_HASH - help - MD5 message digest algorithm (RFC1321) implemented - using sparc64 crypto instructions, when available. - -config CRYPTO_MICHAEL_MIC - tristate "Michael MIC keyed digest algorithm" - select CRYPTO_HASH - help - Michael MIC is used for message integrity protection in TKIP - (IEEE 802.11i). This algorithm is required for TKIP, but it - should not be used for other purposes because of the weakness - of the algorithm. - -config CRYPTO_RMD160 - tristate "RIPEMD-160 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-160 (ISO/IEC 10118-3:2004). - - RIPEMD-160 is a 160-bit cryptographic hash function. It is intended - to be used as a secure replacement for the 128-bit hash functions - MD4, MD5 and its predecessor RIPEMD - (not to be confused with RIPEMD-128). - - It's speed is comparable to SHA1 and there are no known attacks - against RIPEMD-160. - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - -config CRYPTO_SHA1 - tristate "SHA1 digest algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_SHA1 - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - -config CRYPTO_SHA1_SSSE3 - tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions or Advanced Vector - Extensions (AVX/AVX2) or SHA-NI(SHA Extensions New Instructions), - when available. - -config CRYPTO_SHA256_SSSE3 - tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)" - depends on X86 && 64BIT - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, or SHA-NI (SHA Extensions New - Instructions) when available. - -config CRYPTO_SHA512_SSSE3 - tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using Supplemental SSE3 (SSSE3) instructions, or Advanced Vector - Extensions version 1 (AVX1), or Advanced Vector Extensions - version 2 (AVX2) instructions, when available. - -config CRYPTO_SHA512_S390 - tristate "SHA384 and SHA512 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA512 secure hash standard. - - It is available as of z10. - -config CRYPTO_SHA1_OCTEON - tristate "SHA1 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - -config CRYPTO_SHA1_SPARC64 - tristate "SHA1 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA1 - select CRYPTO_HASH - help - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - -config CRYPTO_SHA1_PPC - tristate "SHA1 digest algorithm (powerpc)" - depends on PPC - help - This is the powerpc hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - -config CRYPTO_SHA1_PPC_SPE - tristate "SHA1 digest algorithm (PPC SPE)" - depends on PPC && SPE - help - SHA-1 secure hash standard (DFIPS 180-4) implemented - using powerpc SPE SIMD instruction set. - -config CRYPTO_SHA1_S390 - tristate "SHA1 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2). - - It is available as of z990. - -config CRYPTO_SHA256 - tristate "SHA224 and SHA256 digest algorithm" - select CRYPTO_HASH - select CRYPTO_LIB_SHA256 - help - SHA256 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 256 bit hash with 128 bits of - security against collision attacks. - - This code also includes SHA-224, a 224 bit hash with 112 bits - of security against collision attacks. - -config CRYPTO_SHA256_PPC_SPE - tristate "SHA224 and SHA256 digest algorithm (PPC SPE)" - depends on PPC && SPE - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA224 and SHA256 secure hash standard (DFIPS 180-2) - implemented using powerpc SPE SIMD instruction set. - -config CRYPTO_SHA256_OCTEON - tristate "SHA224 and SHA256 digest algorithm (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - -config CRYPTO_SHA256_SPARC64 - tristate "SHA224 and SHA256 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-256 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - -config CRYPTO_SHA256_S390 - tristate "SHA256 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA256 secure hash standard (DFIPS 180-2). - - It is available as of z9. - -config CRYPTO_SHA512 - tristate "SHA384 and SHA512 digest algorithms" - select CRYPTO_HASH - help - SHA512 secure hash standard (DFIPS 180-2). - - This version of SHA implements a 512 bit hash with 256 bits of - security against collision attacks. - - This code also includes SHA-384, a 384 bit hash with 192 bits - of security against collision attacks. - -config CRYPTO_SHA512_OCTEON - tristate "SHA384 and SHA512 digest algorithms (OCTEON)" - depends on CPU_CAVIUM_OCTEON - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using OCTEON crypto instructions, when available. - -config CRYPTO_SHA512_SPARC64 - tristate "SHA384 and SHA512 digest algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_SHA512 - select CRYPTO_HASH - help - SHA-512 secure hash standard (DFIPS 180-2) implemented - using sparc64 crypto instructions, when available. - -config CRYPTO_SHA3 - tristate "SHA3 digest algorithm" - select CRYPTO_HASH - help - SHA-3 secure hash standard (DFIPS 202). It's based on - cryptographic sponge function family called Keccak. - - References: - http://keccak.noekeon.org/ - -config CRYPTO_SHA3_256_S390 - tristate "SHA3_224 and SHA3_256 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA3_256 secure hash standard. - - It is available as of z14. - -config CRYPTO_SHA3_512_S390 - tristate "SHA3_384 and SHA3_512 digest algorithm" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of the - SHA3_512 secure hash standard. - - It is available as of z14. + SM4 (GBT.32907-2016) is a cryptographic standard issued by the + Organization of State Commercial Administration of China (OSCCA) + as an authorized cryptographic algorithms for the use within China. -config CRYPTO_SM3 - tristate + SMS4 was originally created for use in protecting wireless + networks, and is mandated in the Chinese National Standard for + Wireless LAN WAPI (Wired Authentication and Privacy Infrastructure) + (GB.15629.11-2003). -config CRYPTO_SM3_GENERIC - tristate "SM3 digest algorithm" - select CRYPTO_HASH - select CRYPTO_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. + The latest SM4 standard (GBT.32907-2016) was proposed by OSCCA and + standardized through TC 260 of the Standardization Administration + of the People's Republic of China (SAC). - References: - http://www.oscca.gov.cn/UpFile/20101222141857786.pdf - https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash + The input, output, and key of SMS4 are each 128 bits. -config CRYPTO_SM3_AVX_X86_64 - tristate "SM3 digest algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_HASH - select CRYPTO_SM3 - help - SM3 secure hash function as defined by OSCCA GM/T 0004-2012 SM3). - It is part of the Chinese Commercial Cryptography suite. This is - SM3 optimized implementation using Advanced Vector Extensions (AVX) - when available. + See https://eprint.iacr.org/2008/329.pdf for further information. If unsure, say N. -config CRYPTO_STREEBOG - tristate "Streebog Hash Function" - select CRYPTO_HASH - help - Streebog Hash Function (GOST R 34.11-2012, RFC 6986) is one of the Russian - cryptographic standard algorithms (called GOST algorithms). - This setting enables two hash algorithms with 256 and 512 bits output. - - References: - https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf - https://tools.ietf.org/html/rfc6986 - -config CRYPTO_WP512 - tristate "Whirlpool digest algorithms" - select CRYPTO_HASH - help - Whirlpool hash algorithm 512, 384 and 256-bit hashes - - Whirlpool-512 is part of the NESSIE cryptographic primitives. - Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard - - See also: - - -config CRYPTO_GHASH_CLMUL_NI_INTEL - tristate "GHASH hash function (CLMUL-NI accelerated)" - depends on X86 && 64BIT - select CRYPTO_CRYPTD - help - This is the x86_64 CLMUL-NI accelerated implementation of - GHASH, the hash function used in GCM (Galois/Counter mode). - -config CRYPTO_GHASH_S390 - tristate "GHASH hash function" - depends on S390 - select CRYPTO_HASH - help - This is the s390 hardware accelerated implementation of GHASH, - the hash function used in GCM (Galois/Counter mode). - - It is available as of z196. - -comment "Ciphers" - -config CRYPTO_AES - tristate "AES cipher algorithms" - select CRYPTO_ALGAPI - select CRYPTO_LIB_AES - help - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. - - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits - - See for more information. - -config CRYPTO_AES_TI - tristate "Fixed time AES cipher" - select CRYPTO_ALGAPI - select CRYPTO_LIB_AES - help - This is a generic implementation of AES that attempts to eliminate - data dependent latencies as much as possible without affecting - performance too much. It is intended for use by the generic CCM - and GCM drivers, and other CTR or CMAC/XCBC based modes that rely - solely on encryption (although decryption is supported as well, but - with a more dramatic performance hit) - - Instead of using 16 lookup tables of 1 KB each, (8 for encryption and - 8 for decryption), this implementation only uses just two S-boxes of - 256 bytes each, and attempts to eliminate data dependent latencies by - prefetching the entire table into the cache at the start of each - block. Interrupts are also disabled to avoid races where cachelines - are evicted when the CPU is interrupted to do something else. - -config CRYPTO_AES_NI_INTEL - tristate "AES cipher algorithms (AES-NI)" - depends on X86 - select CRYPTO_AEAD - select CRYPTO_LIB_AES +config CRYPTO_TEA + tristate "TEA, XTEA and XETA" + depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - select CRYPTO_SIMD help - Use Intel AES-NI instructions for AES algorithm. - - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. + TEA (Tiny Encryption Algorithm) cipher algorithms - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. - - The AES specifies three key sizes: 128, 192 and 256 bits + Tiny Encryption Algorithm is a simple cipher that uses + many rounds for security. It is very fast and uses + little memory. - See for more information. + Xtendend Tiny Encryption Algorithm is a modification to + the TEA algorithm to address a potential key weakness + in the TEA algorithm. - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB, CBC, LRW, XTS. The 64 bit version has additional - acceleration for CTR and XCTR. + Xtendend Encryption Tiny Algorithm is a mis-implementation + of the XTEA algorithm for compatibility purposes. -config CRYPTO_AES_SPARC64 - tristate "AES cipher algorithms (SPARC64)" - depends on SPARC64 - select CRYPTO_SKCIPHER +config CRYPTO_TWOFISH + tristate "Twofish" + select CRYPTO_ALGAPI + select CRYPTO_TWOFISH_COMMON help - Use SPARC64 crypto opcodes for AES algorithm. + Twofish cipher algorithm - AES cipher algorithms (FIPS-197). AES uses the Rijndael - algorithm. + Twofish was submitted as an AES (Advanced Encryption Standard) + candidate cipher by researchers at CounterPane Systems. It is a + 16 round block cipher supporting key sizes of 128, 192, and 256 + bits. - Rijndael appears to be consistently a very good performer in - both hardware and software across a wide range of computing - environments regardless of its use in feedback or non-feedback - modes. Its key setup time is excellent, and its key agility is - good. Rijndael's very low memory requirements make it very well - suited for restricted-space environments, in which it also - demonstrates excellent performance. Rijndael's operations are - among the easiest to defend against power and timing attacks. + See https://www.schneier.com/twofish.html for further information. - The AES specifies three key sizes: 128, 192 and 256 bits +config CRYPTO_TWOFISH_COMMON + tristate + help + Common parts of the Twofish cipher algorithm shared by the + generic c and the assembler implementations. - See for more information. +endmenu - In addition to AES cipher algorithm support, the acceleration - for some popular block cipher mode is supported too, including - ECB and CBC. +menu "Length-preserving ciphers and modes" -config CRYPTO_AES_PPC_SPE - tristate "AES cipher algorithms (PPC SPE)" - depends on PPC && SPE - select CRYPTO_SKCIPHER - help - AES cipher algorithms (FIPS-197). Additionally the acceleration - for popular block cipher modes ECB, CBC, CTR and XTS is supported. - This module should only be used for low power (router) devices - without hardware AES acceleration (e.g. caam crypto). It reduces the - size of the AES tables from 16KB to 8KB + 256 bytes and mitigates - timining attacks. Nevertheless it might be not as secure as other - architecture specific assembler implementations that work on 1KB - tables or 256 bytes S-boxes. - -config CRYPTO_AES_S390 - tristate "AES cipher algorithms" - depends on S390 - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER +config CRYPTO_ADIANTUM + tristate "Adiantum" + select CRYPTO_CHACHA20 + select CRYPTO_LIB_POLY1305_GENERIC + select CRYPTO_NHPOLY1305 + select CRYPTO_MANAGER help - This is the s390 hardware accelerated implementation of the - AES cipher algorithms (FIPS-197). + Adiantum tweakable, length-preserving encryption mode - As of z9 the ECB and CBC modes are hardware accelerated - for 128 bit keys. - As of z10 the ECB and CBC modes are hardware accelerated - for all AES key sizes. - As of z196 the CTR mode is hardware accelerated for all AES - key sizes and XTS mode is hardware accelerated for 256 and - 512 bit keys. - -config CRYPTO_ANUBIS - tristate "Anubis cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI - help - Anubis cipher algorithm. + Designed for fast and secure disk encryption, especially on + CPUs without dedicated crypto instructions. It encrypts + each sector using the XChaCha12 stream cipher, two passes of + an ε-almost-∆-universal hash function, and an invocation of + the AES-256 block cipher on a single 16-byte block. On CPUs + without AES instructions, Adiantum is much faster than + AES-XTS. - Anubis is a variable key length cipher which can use keys from - 128 bits to 320 bits in length. It was evaluated as a entrant - in the NESSIE competition. + Adiantum's security is provably reducible to that of its + underlying stream and block ciphers, subject to a security + bound. Unlike XTS, Adiantum is a true wide-block encryption + mode, so it actually provides an even stronger notion of + security than XTS, subject to the security bound. - See also: - - + If unsure, say N. config CRYPTO_ARC4 - tristate "ARC4 cipher algorithm" + tristate "ARC4 (Alleged Rivest Cipher 4)" depends on CRYPTO_USER_API_ENABLE_OBSOLETE select CRYPTO_SKCIPHER select CRYPTO_LIB_ARC4 help - ARC4 cipher algorithm. + ARC4 cipher algorithm ARC4 is a stream cipher using keys ranging from 8 bits to 2048 bits in length. This algorithm is required for driver-based WEP, but it should not be for other purposes because of the weakness of the algorithm. -config CRYPTO_BLOWFISH - tristate "Blowfish cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_BLOWFISH_COMMON +config CRYPTO_CHACHA20 + tristate "ChaCha" + select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_SKCIPHER help - Blowfish cipher algorithm, by Bruce Schneier. - - This is a variable key length cipher which can use keys from 32 - bits to 448 bits in length. It's fast, simple and specifically - designed for use on "large microprocessors". + The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms - See also: - + ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. + Bernstein and further specified in RFC7539 for use in IETF protocols. + This is the portable C implementation of ChaCha20. See + https://cr.yp.to/chacha/chacha-20080128.pdf for further information. -config CRYPTO_BLOWFISH_COMMON - tristate - help - Common parts of the Blowfish cipher algorithm shared by the - generic c and the assembler implementations. + XChaCha20 is the application of the XSalsa20 construction to ChaCha20 + rather than to Salsa20. XChaCha20 extends ChaCha20's nonce length + from 64 bits (or 96 bits using the RFC7539 convention) to 192 bits, + while provably retaining ChaCha20's security. See + https://cr.yp.to/snuffle/xsalsa-20081128.pdf for further information. - See also: - + XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly + reduced security margin but increased performance. It can be needed + in some performance-sensitive scenarios. -config CRYPTO_BLOWFISH_X86_64 - tristate "Blowfish cipher algorithm (x86_64)" - depends on X86 && 64BIT +config CRYPTO_CBC + tristate "CBC (Cipher Block Chaining)" select CRYPTO_SKCIPHER - select CRYPTO_BLOWFISH_COMMON - imply CRYPTO_CTR + select CRYPTO_MANAGER help - Blowfish cipher algorithm (x86_64), by Bruce Schneier. + CBC (Cipher Block Chaining) mode (NIST SP800-38A) - This is a variable key length cipher which can use keys from 32 - bits to 448 bits in length. It's fast, simple and specifically - designed for use on "large microprocessors". - - See also: - + This block cipher mode is required for IPSec ESP (XFRM_ESP). -config CRYPTO_CAMELLIA - tristate "Camellia cipher algorithms" - select CRYPTO_ALGAPI +config CRYPTO_CFB + tristate "CFB (Cipher Feedback)" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Camellia cipher algorithms module. - - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. + CFB (Cipher Feedback) mode (NIST SP800-38A) - The Camellia specifies three key sizes: 128, 192 and 256 bits. - - See also: - + This block cipher mode is required for TPM2 Cryptography. -config CRYPTO_CAMELLIA_X86_64 - tristate "Camellia cipher algorithm (x86_64)" - depends on X86 && 64BIT +config CRYPTO_CTR + tristate "CTR (Counter)" select CRYPTO_SKCIPHER - imply CRYPTO_CTR + select CRYPTO_MANAGER help - Camellia cipher algorithm module (x86_64). + CTR (Counter) mode (NIST SP800-38A) - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. - - The Camellia specifies three key sizes: 128, 192 and 256 bits. +config CRYPTO_CTS + tristate "CTS (Cipher Text Stealing)" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + help + CBC-CS3 variant of CTS (Cipher Text Stealing) (NIST + Addendum to SP800-38A (October 2010)) - See also: - + This mode is required for Kerberos gss mechanism support + for AES encryption. -config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)" - depends on X86 && 64BIT +config CRYPTO_ECB + tristate "ECB (Electronic Codebook)" select CRYPTO_SKCIPHER - select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_SIMD - imply CRYPTO_XTS + select CRYPTO_MANAGER help - Camellia cipher algorithm module (x86_64/AES-NI/AVX). + ECB (Electronic Codebook) mode (NIST SP800-38A) - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. +config CRYPTO_HCTR2 + tristate "HCTR2" + select CRYPTO_XCTR + select CRYPTO_POLYVAL + select CRYPTO_MANAGER + help + HCTR2 length-preserving encryption mode - The Camellia specifies three key sizes: 128, 192 and 256 bits. + A mode for storage encryption that is efficient on processors with + instructions to accelerate AES and carryless multiplication, e.g. + x86 processors with AES-NI and CLMUL, and ARM processors with the + ARMv8 crypto extensions. - See also: - + See https://eprint.iacr.org/2021/1441 -config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64 - tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)" - depends on X86 && 64BIT - select CRYPTO_CAMELLIA_AESNI_AVX_X86_64 +config CRYPTO_KEYWRAP + tristate "KW (AES Key Wrap)" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Camellia cipher algorithm module (x86_64/AES-NI/AVX2). + KW (AES Key Wrap) authenticated encryption mode (NIST SP800-38F + and RFC3394) without padding. - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. +config CRYPTO_LRW + tristate "LRW (Liskov Rivest Wagner)" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + select CRYPTO_GF128MUL + select CRYPTO_ECB + help + LRW (Liskov Rivest Wagner) mode - The Camellia specifies three key sizes: 128, 192 and 256 bits. + A tweakable, non malleable, non movable + narrow block cipher mode for dm-crypt. Use it with cipher + specification string aes-lrw-benbi, the key must be 256, 320 or 384. + The first 128, 192 or 256 bits in the key are used for AES and the + rest is used to tie each cipher block to its logical position. - See also: - + See https://people.csail.mit.edu/rivest/pubs/LRW02.pdf -config CRYPTO_CAMELLIA_SPARC64 - tristate "Camellia cipher algorithm (SPARC64)" - depends on SPARC64 - select CRYPTO_ALGAPI +config CRYPTO_OFB + tristate "OFB (Output Feedback)" select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Camellia cipher algorithm module (SPARC64). + OFB (Output Feedback) mode (NIST SP800-38A) - Camellia is a symmetric key block cipher developed jointly - at NTT and Mitsubishi Electric Corporation. + This mode makes a block cipher into a synchronous + stream cipher. It generates keystream blocks, which are then XORed + with the plaintext blocks to get the ciphertext. Flipping a bit in the + ciphertext produces a flipped bit in the plaintext at the same + location. This property allows many error correcting codes to function + normally even when applied before encryption. - The Camellia specifies three key sizes: 128, 192 and 256 bits. +config CRYPTO_PCBC + tristate "PCBC (Propagating Cipher Block Chaining)" + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + help + PCBC (Propagating Cipher Block Chaining) mode - See also: - + This block cipher mode is required for RxRPC. -config CRYPTO_CAST_COMMON +config CRYPTO_XCTR tristate + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER help - Common parts of the CAST cipher algorithms shared by the - generic c and the assembler implementations. + XCTR (XOR Counter) mode for HCTR2 -config CRYPTO_CAST5 - tristate "CAST5 (CAST-128) cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_CAST_COMMON - help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. + This blockcipher mode is a variant of CTR mode using XORs and little-endian + addition rather than big-endian arithmetic. + + XCTR mode is used to implement HCTR2. -config CRYPTO_CAST5_AVX_X86_64 - tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT +config CRYPTO_XTS + tristate "XTS (XOR Encrypt XOR with ciphertext stealing)" select CRYPTO_SKCIPHER - select CRYPTO_CAST5 - select CRYPTO_CAST_COMMON - select CRYPTO_SIMD - imply CRYPTO_CTR + select CRYPTO_MANAGER + select CRYPTO_ECB help - The CAST5 encryption algorithm (synonymous with CAST-128) is - described in RFC2144. + XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E + and IEEE 1619) - This module provides the Cast5 cipher algorithm that processes - sixteen blocks parallel using the AVX instruction set. + Use with aes-xts-plain, key size 256, 384 or 512 bits. This + implementation currently can't handle a sectorsize which is not a + multiple of 16 bytes. -config CRYPTO_CAST6 - tristate "CAST6 (CAST-256) cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_CAST_COMMON +config CRYPTO_NHPOLY1305 + tristate + select CRYPTO_HASH + select CRYPTO_LIB_POLY1305_GENERIC + +endmenu + +menu "AEAD (authenticated encryption with associated data) ciphers" + +config CRYPTO_AEGIS128 + tristate "AEGIS-128" + select CRYPTO_AEAD + select CRYPTO_AES # for AES S-box tables help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. + AEGIS-128 AEAD algorithm -config CRYPTO_CAST6_AVX_X86_64 - tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_CAST6 - select CRYPTO_CAST_COMMON - select CRYPTO_SIMD - imply CRYPTO_XTS - imply CRYPTO_CTR +config CRYPTO_AEGIS128_SIMD + bool "AEGIS-128 (arm NEON, arm64 NEON)" + depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON) + default y help - The CAST6 encryption algorithm (synonymous with CAST-256) is - described in RFC2612. + AEGIS-128 AEAD algorithm - This module provides the Cast6 cipher algorithm that processes - eight blocks parallel using the AVX instruction set. + Architecture: arm or arm64 using: + - NEON (Advanced SIMD) extension -config CRYPTO_DES - tristate "DES and Triple DES EDE cipher algorithms" - select CRYPTO_ALGAPI - select CRYPTO_LIB_DES +config CRYPTO_CHACHA20POLY1305 + tristate "ChaCha20-Poly1305" + select CRYPTO_CHACHA20 + select CRYPTO_POLY1305 + select CRYPTO_AEAD + select CRYPTO_MANAGER help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + ChaCha20 stream cipher and Poly1305 authenticator combined + mode (RFC8439) -config CRYPTO_DES_SPARC64 - tristate "DES and Triple DES EDE cipher algorithms (SPARC64)" - depends on SPARC64 - select CRYPTO_ALGAPI - select CRYPTO_LIB_DES - select CRYPTO_SKCIPHER +config CRYPTO_CCM + tristate "CCM (Counter with Cipher Block Chaining-MAC)" + select CRYPTO_CTR + select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_MANAGER help - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3), - optimized using SPARC64 crypto opcodes. + CCM (Counter with Cipher Block Chaining-Message Authentication Code) + authenticated encryption mode (NIST SP800-38C) -config CRYPTO_DES3_EDE_X86_64 - tristate "Triple DES EDE cipher algorithm (x86-64)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - imply CRYPTO_CTR +config CRYPTO_GCM + tristate "GCM (Galois/Counter Mode) and GMAC (GCM MAC)" + select CRYPTO_CTR + select CRYPTO_AEAD + select CRYPTO_GHASH + select CRYPTO_NULL + select CRYPTO_MANAGER help - Triple DES EDE (FIPS 46-3) algorithm. + GCM (Galois/Counter Mode) authenticated encryption mode and GMAC + (GCM Message Authentication Code) (NIST SP800-38D) - This module provides implementation of the Triple DES EDE cipher - algorithm that is optimized for x86-64 processors. Two versions of - algorithm are provided; regular processing one input block and - one that processes three blocks parallel. + This is required for IPSec ESP (XFRM_ESP). -config CRYPTO_DES_S390 - tristate "DES and Triple DES cipher algorithms" - depends on S390 - select CRYPTO_ALGAPI +config CRYPTO_SEQIV + tristate "Sequence Number IV Generator" + select CRYPTO_AEAD select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES + select CRYPTO_NULL + select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help - This is the s390 hardware accelerated implementation of the - DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3). + Sequence Number IV generator - As of z990 the ECB and CBC mode are hardware accelerated. - As of z196 the CTR mode is hardware accelerated. + This IV generator generates an IV based on a sequence number by + xoring it with a salt. This algorithm is mainly useful for CTR. -config CRYPTO_FCRYPT - tristate "FCrypt cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_SKCIPHER - help - FCrypt algorithm used by RxRPC. + This is required for IPsec ESP (XFRM_ESP). -config CRYPTO_KHAZAD - tristate "Khazad cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI +config CRYPTO_ECHAINIV + tristate "Encrypted Chain IV Generator" + select CRYPTO_AEAD + select CRYPTO_NULL + select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help - Khazad cipher algorithm. - - Khazad was a finalist in the initial NESSIE competition. It is - an algorithm optimized for 64-bit processors with good performance - on 32-bit processors. Khazad uses an 128 bit key size. + Encrypted Chain IV generator - See also: - + This IV generator generates an IV based on the encryption of + a sequence number xored with a salt. This is the default + algorithm for CBC. -config CRYPTO_CHACHA20 - tristate "ChaCha stream cipher algorithms" - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_SKCIPHER +config CRYPTO_ESSIV + tristate "Encrypted Salt-Sector IV Generator" + select CRYPTO_AUTHENC help - The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms. + Encrypted Salt-Sector IV generator - ChaCha20 is a 256-bit high-speed stream cipher designed by Daniel J. - Bernstein and further specified in RFC7539 for use in IETF protocols. - This is the portable C implementation of ChaCha20. See also: - + This IV generator is used in some cases by fscrypt and/or + dm-crypt. It uses the hash of the block encryption key as the + symmetric key for a block encryption pass applied to the input + IV, making low entropy IV sources more suitable for block + encryption. - XChaCha20 is the application of the XSalsa20 construction to ChaCha20 - rather than to Salsa20. XChaCha20 extends ChaCha20's nonce length - from 64 bits (or 96 bits using the RFC7539 convention) to 192 bits, - while provably retaining ChaCha20's security. See also: - + This driver implements a crypto API template that can be + instantiated either as an skcipher or as an AEAD (depending on the + type of the first template argument), and which defers encryption + and decryption requests to the encapsulated cipher after applying + ESSIV to the input IV. Note that in the AEAD case, it is assumed + that the keys are presented in the same format used by the authenc + template, and that the IV appears at the end of the authenticated + associated data (AAD) region (which is how dm-crypt uses it.) - XChaCha12 is XChaCha20 reduced to 12 rounds, with correspondingly - reduced security margin but increased performance. It can be needed - in some performance-sensitive scenarios. + Note that the use of ESSIV is not recommended for new deployments, + and so this only needs to be enabled when interoperability with + existing encrypted volumes of filesystems is required, or when + building for a particular system that requires it (e.g., when + the SoC in question has accelerated CBC but not XTS, making CBC + combined with ESSIV the only feasible mode for h/w accelerated + block encryption) -config CRYPTO_CHACHA20_X86_64 - tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - help - SSSE3, AVX2, and AVX-512VL optimized implementations of the ChaCha20, - XChaCha20, and XChaCha12 stream ciphers. +endmenu -config CRYPTO_CHACHA_MIPS - tristate "ChaCha stream cipher algorithms (MIPS 32r2 optimized)" - depends on CPU_MIPS32_R2 - select CRYPTO_SKCIPHER - select CRYPTO_ARCH_HAVE_LIB_CHACHA +menu "Hashes, digests, and MACs" -config CRYPTO_CHACHA_S390 - tristate "ChaCha20 stream cipher" - depends on S390 - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA +config CRYPTO_BLAKE2B + tristate "BLAKE2b" + select CRYPTO_HASH help - This is the s390 SIMD implementation of the ChaCha20 stream - cipher (RFC 7539). + BLAKE2b cryptographic hash function (RFC 7693) - It is available as of z13. + BLAKE2b is optimized for 64-bit platforms and can produce digests + of any size between 1 and 64 bytes. The keyed hash is also implemented. -config CRYPTO_SEED - tristate "SEED cipher algorithm" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI - help - SEED cipher algorithm (RFC4269). + This module provides the following algorithms: + - blake2b-160 + - blake2b-256 + - blake2b-384 + - blake2b-512 - SEED is a 128-bit symmetric key block cipher that has been - developed by KISA (Korea Information Security Agency) as a - national standard encryption algorithm of the Republic of Korea. - It is a 16 round block cipher with the key size of 128 bit. + Used by the btrfs filesystem. - See also: - + See https://blake2.net for further information. -config CRYPTO_ARIA - tristate "ARIA cipher algorithm" - select CRYPTO_ALGAPI +config CRYPTO_CMAC + tristate "CMAC (Cipher-based MAC)" + select CRYPTO_HASH + select CRYPTO_MANAGER help - ARIA cipher algorithm (RFC5794). - - ARIA is a standard encryption algorithm of the Republic of Korea. - The ARIA specifies three key sizes and rounds. - 128-bit: 12 rounds. - 192-bit: 14 rounds. - 256-bit: 16 rounds. + CMAC (Cipher-based Message Authentication Code) authentication + mode (NIST SP800-38B and IETF RFC4493) - See also: - +config CRYPTO_GHASH + tristate "GHASH" + select CRYPTO_GF128MUL + select CRYPTO_HASH + help + GCM GHASH function (NIST SP800-38D) -config CRYPTO_SERPENT - tristate "Serpent cipher algorithm" - select CRYPTO_ALGAPI +config CRYPTO_HMAC + tristate "HMAC (Keyed-Hash MAC)" + select CRYPTO_HASH + select CRYPTO_MANAGER help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + HMAC (Keyed-Hash Message Authentication Code) (FIPS 198 and + RFC2104) - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. + This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP). - See also: - +config CRYPTO_MD4 + tristate "MD4" + select CRYPTO_HASH + help + MD4 message digest algorithm (RFC1320) -config CRYPTO_SERPENT_SSE2_X86_64 - tristate "Serpent cipher algorithm (x86_64/SSE2)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_CTR +config CRYPTO_MD5 + tristate "MD5" + select CRYPTO_HASH help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + MD5 message digest algorithm (RFC1321) - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. +config CRYPTO_MICHAEL_MIC + tristate "Michael MIC" + select CRYPTO_HASH + help + Michael MIC (Message Integrity Code) (IEEE 802.11i) - This module provides Serpent cipher algorithm that processes eight - blocks parallel using SSE2 instruction set. + Defined by the IEEE 802.11i TKIP (Temporal Key Integrity Protocol), + known as WPA (Wif-Fi Protected Access). - See also: - + This algorithm is required for TKIP, but it should not be used for + other purposes because of the weakness of the algorithm. -config CRYPTO_SERPENT_SSE2_586 - tristate "Serpent cipher algorithm (i586/SSE2)" - depends on X86 && !64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_CTR +config CRYPTO_POLYVAL + tristate + select CRYPTO_GF128MUL + select CRYPTO_HASH help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + POLYVAL hash function for HCTR2 - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. + This is used in HCTR2. It is not a general-purpose + cryptographic hash function. - This module provides Serpent cipher algorithm that processes four - blocks parallel using SSE2 instruction set. +config CRYPTO_POLY1305 + tristate "Poly1305" + select CRYPTO_HASH + select CRYPTO_LIB_POLY1305_GENERIC + help + Poly1305 authenticator algorithm (RFC7539) - See also: - + Poly1305 is an authenticator algorithm designed by Daniel J. Bernstein. + It is used for the ChaCha20-Poly1305 AEAD, specified in RFC7539 for use + in IETF protocols. This is the portable C implementation of Poly1305. -config CRYPTO_SERPENT_AVX_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SERPENT - select CRYPTO_SIMD - imply CRYPTO_XTS - imply CRYPTO_CTR +config CRYPTO_RMD160 + tristate "RIPEMD-160" + select CRYPTO_HASH help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. + RIPEMD-160 hash function (ISO/IEC 10118-3) - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. + RIPEMD-160 is a 160-bit cryptographic hash function. It is intended + to be used as a secure replacement for the 128-bit hash functions + MD4, MD5 and its predecessor RIPEMD + (not to be confused with RIPEMD-128). - This module provides the Serpent cipher algorithm that processes - eight blocks parallel using the AVX instruction set. + Its speed is comparable to SHA-1 and there are no known attacks + against RIPEMD-160. - See also: - + Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. + See https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + for further information. -config CRYPTO_SERPENT_AVX2_X86_64 - tristate "Serpent cipher algorithm (x86_64/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SERPENT_AVX_X86_64 +config CRYPTO_SHA1 + tristate "SHA-1" + select CRYPTO_HASH + select CRYPTO_LIB_SHA1 help - Serpent cipher algorithm, by Anderson, Biham & Knudsen. - - Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. - - This module provides Serpent cipher algorithm that processes 16 - blocks parallel using AVX2 instruction set. + SHA-1 secure hash algorithm (FIPS 180, ISO/IEC 10118-3) - See also: - +config CRYPTO_SHA256 + tristate "SHA-224 and SHA-256" + select CRYPTO_HASH + select CRYPTO_LIB_SHA256 + help + SHA-224 and SHA-256 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) -config CRYPTO_SM4 - tristate + This is required for IPsec AH (XFRM_AH) and IPsec ESP (XFRM_ESP). + Used by the btrfs filesystem, Ceph, NFS, and SMB. -config CRYPTO_SM4_GENERIC - tristate "SM4 cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_SM4 +config CRYPTO_SHA512 + tristate "SHA-384 and SHA-512" + select CRYPTO_HASH help - SM4 cipher algorithms (OSCCA GB/T 32907-2016). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. + SHA-384 and SHA-512 secure hash algorithms (FIPS 180, ISO/IEC 10118-3) - SMS4 was originally created for use in protecting wireless - networks, and is mandated in the Chinese National Standard for - Wireless LAN WAPI (Wired Authentication and Privacy Infrastructure) - (GB.15629.11-2003). +config CRYPTO_SHA3 + tristate "SHA-3" + select CRYPTO_HASH + help + SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) - The latest SM4 standard (GBT.32907-2016) was proposed by OSCCA and - standardized through TC 260 of the Standardization Administration - of the People's Republic of China (SAC). +config CRYPTO_SM3 + tristate - The input, output, and key of SMS4 are each 128 bits. +config CRYPTO_SM3_GENERIC + tristate "SM3 (ShangMi 3)" + select CRYPTO_HASH + select CRYPTO_SM3 + help + SM3 (ShangMi 3) secure hash function (OSCCA GM/T 0004-2012, ISO/IEC 10118-3) - See also: + This is part of the Chinese Commercial Cryptography suite. - If unsure, say N. + References: + http://www.oscca.gov.cn/UpFile/20101222141857786.pdf + https://datatracker.ietf.org/doc/html/draft-shen-sm3-hash -config CRYPTO_SM4_AESNI_AVX_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_ALGAPI - select CRYPTO_SM4 +config CRYPTO_STREEBOG + tristate "Streebog" + select CRYPTO_HASH help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. + Streebog Hash Function (GOST R 34.11-2012, RFC 6986, ISO/IEC 10118-3) - This is SM4 optimized implementation using AES-NI/AVX/x86_64 - instruction set for block cipher. Through two affine transforms, - we can use the AES S-Box to simulate the SM4 S-Box to achieve the - effect of instruction acceleration. + This is one of the Russian cryptographic standard algorithms (called + GOST algorithms). This setting enables two hash algorithms with + 256 and 512 bits output. - If unsure, say N. + References: + https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf + https://tools.ietf.org/html/rfc6986 -config CRYPTO_SM4_AESNI_AVX2_X86_64 - tristate "SM4 cipher algorithm (x86_64/AES-NI/AVX2)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_ALGAPI - select CRYPTO_SM4 - select CRYPTO_SM4_AESNI_AVX_X86_64 +config CRYPTO_VMAC + tristate "VMAC" + select CRYPTO_HASH + select CRYPTO_MANAGER help - SM4 cipher algorithms (OSCCA GB/T 32907-2016) (x86_64/AES-NI/AVX2). - - SM4 (GBT.32907-2016) is a cryptographic standard issued by the - Organization of State Commercial Administration of China (OSCCA) - as an authorized cryptographic algorithms for the use within China. - - This is SM4 optimized implementation using AES-NI/AVX2/x86_64 - instruction set for block cipher. Through two affine transforms, - we can use the AES S-Box to simulate the SM4 S-Box to achieve the - effect of instruction acceleration. + VMAC is a message authentication algorithm designed for + very high speed on 64-bit architectures. - If unsure, say N. + See https://fastcrypto.org/vmac for further information. -config CRYPTO_TEA - tristate "TEA, XTEA and XETA cipher algorithms" - depends on CRYPTO_USER_API_ENABLE_OBSOLETE - select CRYPTO_ALGAPI +config CRYPTO_WP512 + tristate "Whirlpool" + select CRYPTO_HASH help - TEA cipher algorithm. + Whirlpool hash function (ISO/IEC 10118-3) - Tiny Encryption Algorithm is a simple cipher that uses - many rounds for security. It is very fast and uses - little memory. + 512, 384 and 256-bit hashes. - Xtendend Tiny Encryption Algorithm is a modification to - the TEA algorithm to address a potential key weakness - in the TEA algorithm. + Whirlpool-512 is part of the NESSIE cryptographic primitives. - Xtendend Encryption Tiny Algorithm is a mis-implementation - of the XTEA algorithm for compatibility purposes. + See https://web.archive.org/web/20171129084214/http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html + for further information. -config CRYPTO_TWOFISH - tristate "Twofish cipher algorithm" - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON +config CRYPTO_XCBC + tristate "XCBC-MAC (Extended Cipher Block Chaining MAC)" + select CRYPTO_HASH + select CRYPTO_MANAGER help - Twofish cipher algorithm. - - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + XCBC-MAC (Extended Cipher Block Chaining Message Authentication + Code) (RFC3566) - See also: - - -config CRYPTO_TWOFISH_COMMON - tristate +config CRYPTO_XXHASH + tristate "xxHash" + select CRYPTO_HASH + select XXHASH help - Common parts of the Twofish cipher algorithm shared by the - generic c and the assembler implementations. + xxHash non-cryptographic hash algorithm -config CRYPTO_TWOFISH_586 - tristate "Twofish cipher algorithms (i586)" - depends on (X86 || UML_X86) && !64BIT - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON - imply CRYPTO_CTR - help - Twofish cipher algorithm. + Extremely fast, working at speeds close to RAM limits. - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Used by the btrfs filesystem. - See also: - +endmenu -config CRYPTO_TWOFISH_X86_64 - tristate "Twofish cipher algorithm (x86_64)" - depends on (X86 || UML_X86) && 64BIT - select CRYPTO_ALGAPI - select CRYPTO_TWOFISH_COMMON - imply CRYPTO_CTR +menu "CRCs (cyclic redundancy checks)" + +config CRYPTO_CRC32C + tristate "CRC32c" + select CRYPTO_HASH + select CRC32 help - Twofish cipher algorithm (x86_64). + CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + A 32-bit CRC (cyclic redundancy check) with a polynomial defined + by G. Castagnoli, S. Braeuer and M. Herrman in "Optimization of Cyclic + Redundancy-Check Codes with 24 and 32 Parity Bits", IEEE Transactions + on Communications, Vol. 41, No. 6, June 1993, selected for use with + iSCSI. - See also: - + Used by btrfs, ext4, jbd2, NVMeoF/TCP, and iSCSI. -config CRYPTO_TWOFISH_X86_64_3WAY - tristate "Twofish cipher algorithm (x86_64, 3-way parallel)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_TWOFISH_COMMON - select CRYPTO_TWOFISH_X86_64 +config CRYPTO_CRC32 + tristate "CRC32" + select CRYPTO_HASH + select CRC32 help - Twofish cipher algorithm (x86_64, 3-way parallel). + CRC32 CRC algorithm (IEEE 802.3) - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Used by RoCEv2 and f2fs. - This module provides Twofish cipher algorithm that processes three - blocks parallel, utilizing resources of out-of-order CPUs better. +config CRYPTO_CRCT10DIF + tristate "CRCT10DIF" + select CRYPTO_HASH + help + CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) - See also: - + CRC algorithm used by the SCSI Block Commands standard. -config CRYPTO_TWOFISH_AVX_X86_64 - tristate "Twofish cipher algorithm (x86_64/AVX)" - depends on X86 && 64BIT - select CRYPTO_SKCIPHER - select CRYPTO_SIMD - select CRYPTO_TWOFISH_COMMON - select CRYPTO_TWOFISH_X86_64 - select CRYPTO_TWOFISH_X86_64_3WAY - imply CRYPTO_XTS +config CRYPTO_CRC64_ROCKSOFT + tristate "CRC64 based on Rocksoft Model algorithm" + depends on CRC64 + select CRYPTO_HASH help - Twofish cipher algorithm (x86_64/AVX). + CRC64 CRC algorithm based on the Rocksoft Model CRC Algorithm - Twofish was submitted as an AES (Advanced Encryption Standard) - candidate cipher by researchers at CounterPane Systems. It is a - 16 round block cipher supporting key sizes of 128, 192, and 256 - bits. + Used by the NVMe implementation of T10 DIF (BLK_DEV_INTEGRITY) - This module provides the Twofish cipher algorithm that processes - eight blocks parallel using the AVX Instruction Set. + See https://zlib.net/crc_v3.txt - See also: - +endmenu -comment "Compression" +menu "Compression" config CRYPTO_DEFLATE - tristate "Deflate compression algorithm" + tristate "Deflate" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select ZLIB_INFLATE select ZLIB_DEFLATE help - This is the Deflate algorithm (RFC1951), specified for use in - IPSec with the IPCOMP protocol (RFC3173, RFC2394). + Deflate compression algorithm (RFC1951) - You will most probably want this if using IPSec. + Used by IPSec with the IPCOMP protocol (RFC3173, RFC2394) config CRYPTO_LZO - tristate "LZO compression algorithm" + tristate "LZO" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZO_COMPRESS select LZO_DECOMPRESS help - This is the LZO algorithm. + LZO compression algorithm + + See https://www.oberhumer.com/opensource/lzo/ for further information. config CRYPTO_842 - tristate "842 compression algorithm" + tristate "842" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select 842_COMPRESS select 842_DECOMPRESS help - This is the 842 algorithm. + 842 compression algorithm by IBM + + See https://github.com/plauth/lib842 for further information. config CRYPTO_LZ4 - tristate "LZ4 compression algorithm" + tristate "LZ4" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZ4_COMPRESS select LZ4_DECOMPRESS help - This is the LZ4 algorithm. + LZ4 compression algorithm + + See https://github.com/lz4/lz4 for further information. config CRYPTO_LZ4HC - tristate "LZ4HC compression algorithm" + tristate "LZ4HC" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select LZ4HC_COMPRESS select LZ4_DECOMPRESS help - This is the LZ4 high compression mode algorithm. + LZ4 high compression mode algorithm + + See https://github.com/lz4/lz4 for further information. config CRYPTO_ZSTD - tristate "Zstd compression algorithm" + tristate "Zstd" select CRYPTO_ALGAPI select CRYPTO_ACOMP2 select ZSTD_COMPRESS select ZSTD_DECOMPRESS help - This is the zstd algorithm. + zstd compression algorithm -comment "Random Number Generation" + See https://github.com/facebook/zstd for further information. + +endmenu + +menu "Random number generation" config CRYPTO_ANSI_CPRNG - tristate "Pseudo Random Number Generation for Cryptographic modules" + tristate "ANSI PRNG (Pseudo Random Number Generator)" select CRYPTO_AES select CRYPTO_RNG help - This option enables the generic pseudo random number generator - for cryptographic modules. Uses the Algorithm specified in - ANSI X9.31 A.2.4. Note that this option must be enabled if - CRYPTO_FIPS is selected + Pseudo RNG (random number generator) (ANSI X9.31 Appendix A.2.4) + + This uses the AES cipher algorithm. + + Note that this option must be enabled if CRYPTO_FIPS is selected menuconfig CRYPTO_DRBG_MENU - tristate "NIST SP800-90A DRBG" + tristate "NIST SP800-90A DRBG (Deterministic Random Bit Generator)" help - NIST SP800-90A compliant DRBG. In the following submenu, one or - more of the DRBG types must be selected. + DRBG (Deterministic Random Bit Generator) (NIST SP800-90A) + + In the following submenu, one or more of the DRBG types must be selected. if CRYPTO_DRBG_MENU @@ -2019,17 +1253,21 @@ config CRYPTO_DRBG_HMAC select CRYPTO_SHA512 config CRYPTO_DRBG_HASH - bool "Enable Hash DRBG" + bool "Hash_DRBG" select CRYPTO_SHA256 help - Enable the Hash DRBG variant as defined in NIST SP800-90A. + Hash_DRBG variant as defined in NIST SP800-90A. + + This uses the SHA-1, SHA-256, SHA-384, or SHA-512 hash algorithms. config CRYPTO_DRBG_CTR - bool "Enable CTR DRBG" + bool "CTR_DRBG" select CRYPTO_AES select CRYPTO_CTR help - Enable the CTR DRBG variant as defined in NIST SP800-90A. + CTR_DRBG variant as defined in NIST SP800-90A. + + This uses the AES cipher algorithm with the counter block mode. config CRYPTO_DRBG tristate @@ -2040,72 +1278,90 @@ config CRYPTO_DRBG endif # if CRYPTO_DRBG_MENU config CRYPTO_JITTERENTROPY - tristate "Jitterentropy Non-Deterministic Random Number Generator" + tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" select CRYPTO_RNG help - The Jitterentropy RNG is a noise that is intended - to provide seed to another RNG. The RNG does not - perform any cryptographic whitening of the generated - random numbers. This Jitterentropy RNG registers with - the kernel crypto API and can be used by any caller. + CPU Jitter RNG (Random Number Generator) from the Jitterentropy library + + A non-physical non-deterministic ("true") RNG (e.g., an entropy source + compliant with NIST SP800-90B) intended to provide a seed to a + deterministic RNG (e.g. per NIST SP800-90C). + This RNG does not perform any cryptographic whitening of the generated + + See https://www.chronox.de/jent.html config CRYPTO_KDF800108_CTR tristate select CRYPTO_HMAC select CRYPTO_SHA256 +endmenu +menu "Userspace interface" + config CRYPTO_USER_API tristate config CRYPTO_USER_API_HASH - tristate "User-space interface for hash algorithms" + tristate "Hash algorithms" depends on NET select CRYPTO_HASH select CRYPTO_USER_API help - This option enables the user-spaces interface for hash - algorithms. + Enable the userspace interface for hash algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_SKCIPHER - tristate "User-space interface for symmetric key cipher algorithms" + tristate "Symmetric key cipher algorithms" depends on NET select CRYPTO_SKCIPHER select CRYPTO_USER_API help - This option enables the user-spaces interface for symmetric - key cipher algorithms. + Enable the userspace interface for symmetric key cipher algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_RNG - tristate "User-space interface for random number generator algorithms" + tristate "RNG (random number generator) algorithms" depends on NET select CRYPTO_RNG select CRYPTO_USER_API help - This option enables the user-spaces interface for random - number generator algorithms. + Enable the userspace interface for RNG (random number generator) + algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_RNG_CAVP bool "Enable CAVP testing of DRBG" depends on CRYPTO_USER_API_RNG && CRYPTO_DRBG help - This option enables extra API for CAVP testing via the user-space - interface: resetting of DRBG entropy, and providing Additional Data. + Enable extra APIs in the userspace interface for NIST CAVP + (Cryptographic Algorithm Validation Program) testing: + - resetting DRBG entropy + - providing Additional Data + This should only be enabled for CAVP testing. You should say no unless you know what this is. config CRYPTO_USER_API_AEAD - tristate "User-space interface for AEAD cipher algorithms" + tristate "AEAD cipher algorithms" depends on NET select CRYPTO_AEAD select CRYPTO_SKCIPHER select CRYPTO_NULL select CRYPTO_USER_API help - This option enables the user-spaces interface for AEAD - cipher algorithms. + Enable the userspace interface for AEAD cipher algorithms. + + See Documentation/crypto/userspace-if.rst and + https://www.chronox.de/libkcapi/html/index.html config CRYPTO_USER_API_ENABLE_OBSOLETE - bool "Enable obsolete cryptographic algorithms for userspace" + bool "Obsolete cryptographic algorithms" depends on CRYPTO_USER_API default y help @@ -2114,20 +1370,51 @@ config CRYPTO_USER_API_ENABLE_OBSOLETE only useful for userspace clients that still rely on them. config CRYPTO_STATS - bool "Crypto usage statistics for User-space" + bool "Crypto usage statistics" depends on CRYPTO_USER help - This option enables the gathering of crypto stats. - This will collect: - - encrypt/decrypt size and numbers of symmeric operations - - compress/decompress size and numbers of compress operations - - size and numbers of hash operations - - encrypt/decrypt/sign/verify numbers for asymmetric operations - - generate/seed numbers for rng operations + Enable the gathering of crypto stats. + + This collects data sizes, numbers of requests, and numbers + of errors processed by: + - AEAD ciphers (encrypt, decrypt) + - asymmetric key ciphers (encrypt, decrypt, verify, sign) + - symmetric key ciphers (encrypt, decrypt) + - compression algorithms (compress, decompress) + - hash algorithms (hash) + - key-agreement protocol primitives (setsecret, generate + public key, compute shared secret) + - RNG (generate, seed) + +endmenu config CRYPTO_HASH_INFO bool +if !KMSAN # avoid false positives from assembly +if ARM +source "arch/arm/crypto/Kconfig" +endif +if ARM64 +source "arch/arm64/crypto/Kconfig" +endif +if MIPS +source "arch/mips/crypto/Kconfig" +endif +if PPC +source "arch/powerpc/crypto/Kconfig" +endif +if S390 +source "arch/s390/crypto/Kconfig" +endif +if SPARC +source "arch/sparc/crypto/Kconfig" +endif +if X86 +source "arch/x86/crypto/Kconfig" +endif +endif + source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" source "certs/Kconfig" diff --git a/crypto/Makefile b/crypto/Makefile index a6f94e04e1dae78d44654d08fc62b19ea89fae7b..303b21c43df05a5e4784b355150daadbd9ec8d66 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -149,7 +149,7 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o -obj-$(CONFIG_CRYPTO_ARIA) += aria.o +obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o diff --git a/crypto/akcipher.c b/crypto/akcipher.c index f866085c8a4a38e83e94bb64efcaa80e5e7eff73..ab975a420e1e982c338010db3f1e3651b22cd9d0 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -120,6 +120,12 @@ static int akcipher_default_op(struct akcipher_request *req) return -ENOSYS; } +static int akcipher_default_set_key(struct crypto_akcipher *tfm, + const void *key, unsigned int keylen) +{ + return -ENOSYS; +} + int crypto_register_akcipher(struct akcipher_alg *alg) { struct crypto_alg *base = &alg->base; @@ -132,6 +138,8 @@ int crypto_register_akcipher(struct akcipher_alg *alg) alg->encrypt = akcipher_default_op; if (!alg->decrypt) alg->decrypt = akcipher_default_op; + if (!alg->set_priv_key) + alg->set_priv_key = akcipher_default_set_key; akcipher_prepare_alg(alg); return crypto_register_alg(base); diff --git a/crypto/algapi.c b/crypto/algapi.c index d1c99288af3e0da5ff5e6b4e12b9b2b3e8cdf67b..5c69ff8e8fa5c1dab176cd476f19907f45893dfe 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -997,77 +997,6 @@ void crypto_inc(u8 *a, unsigned int size) } EXPORT_SYMBOL_GPL(crypto_inc); -void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) -{ - int relalign = 0; - - if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - int size = sizeof(unsigned long); - int d = (((unsigned long)dst ^ (unsigned long)src1) | - ((unsigned long)dst ^ (unsigned long)src2)) & - (size - 1); - - relalign = d ? 1 << __ffs(d) : size; - - /* - * If we care about alignment, process as many bytes as - * needed to advance dst and src to values whose alignments - * equal their relative alignment. This will allow us to - * process the remainder of the input using optimal strides. - */ - while (((unsigned long)dst & (relalign - 1)) && len > 0) { - *dst++ = *src1++ ^ *src2++; - len--; - } - } - - while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u64 l = get_unaligned((u64 *)src1) ^ - get_unaligned((u64 *)src2); - put_unaligned(l, (u64 *)dst); - } else { - *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; - } - dst += 8; - src1 += 8; - src2 += 8; - len -= 8; - } - - while (len >= 4 && !(relalign & 3)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u32 l = get_unaligned((u32 *)src1) ^ - get_unaligned((u32 *)src2); - put_unaligned(l, (u32 *)dst); - } else { - *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; - } - dst += 4; - src1 += 4; - src2 += 4; - len -= 4; - } - - while (len >= 2 && !(relalign & 1)) { - if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { - u16 l = get_unaligned((u16 *)src1) ^ - get_unaligned((u16 *)src2); - put_unaligned(l, (u16 *)dst); - } else { - *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; - } - dst += 2; - src1 += 2; - src2 += 2; - len -= 2; - } - - while (len--) - *dst++ = *src1++ ^ *src2++; -} -EXPORT_SYMBOL_GPL(__crypto_xor); - unsigned int crypto_alg_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize + diff --git a/crypto/api.c b/crypto/api.c index 69508ae9345ee3423f3c9edd94aee78c2c5d0e20..64f2d365a8e94d5bcd9a199e44b465da105030a4 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -114,7 +114,7 @@ struct crypto_larval *crypto_larval_alloc(const char *name, u32 type, u32 mask) larval->alg.cra_priority = -1; larval->alg.cra_destroy = crypto_larval_destroy; - strlcpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME); + strscpy(larval->alg.cra_name, name, CRYPTO_MAX_ALG_NAME); init_completion(&larval->completion); return larval; @@ -321,7 +321,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) /* * If the internal flag is set for a cipher, require a caller to - * to invoke the cipher with the internal flag to use that cipher. + * invoke the cipher with the internal flag to use that cipher. * Also, if a caller wants to allocate a cipher that may or may * not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and * !(mask & CRYPTO_ALG_INTERNAL). diff --git a/crypto/aria.c b/crypto/aria_generic.c similarity index 86% rename from crypto/aria.c rename to crypto/aria_generic.c index ac3dffac34bbc9ca9c84e2d56dbb3a01d30d3755..4cc29b82b99d055d073e7d7489228beb1640bd76 100644 --- a/crypto/aria.c +++ b/crypto/aria_generic.c @@ -16,6 +16,14 @@ #include +static const u32 key_rc[20] = { + 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0, + 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0, + 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e, + 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0, + 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 +}; + static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, unsigned int key_len) { @@ -25,7 +33,7 @@ static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, const u32 *ck; int rkidx = 0; - ck = &key_rc[(key_len - 16) / 8][0]; + ck = &key_rc[(key_len - 16) / 2]; w0[0] = be32_to_cpu(key[0]); w0[1] = be32_to_cpu(key[1]); @@ -163,8 +171,7 @@ static void aria_set_decrypt_key(struct aria_ctx *ctx) } } -static int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) +int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { struct aria_ctx *ctx = crypto_tfm_ctx(tfm); @@ -179,6 +186,7 @@ static int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, return 0; } +EXPORT_SYMBOL_GPL(aria_set_key); static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, u32 key[][ARIA_RD_KEY_WORDS]) @@ -235,14 +243,30 @@ static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, dst[3] = cpu_to_be32(reg3); } -static void aria_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +void aria_encrypt(void *_ctx, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = (struct aria_ctx *)_ctx; + + __aria_crypt(ctx, out, in, ctx->enc_key); +} +EXPORT_SYMBOL_GPL(aria_encrypt); + +void aria_decrypt(void *_ctx, u8 *out, const u8 *in) +{ + struct aria_ctx *ctx = (struct aria_ctx *)_ctx; + + __aria_crypt(ctx, out, in, ctx->dec_key); +} +EXPORT_SYMBOL_GPL(aria_decrypt); + +static void __aria_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct aria_ctx *ctx = crypto_tfm_ctx(tfm); __aria_crypt(ctx, out, in, ctx->enc_key); } -static void aria_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +static void __aria_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct aria_ctx *ctx = crypto_tfm_ctx(tfm); @@ -263,8 +287,8 @@ static struct crypto_alg aria_alg = { .cia_min_keysize = ARIA_MIN_KEY_SIZE, .cia_max_keysize = ARIA_MAX_KEY_SIZE, .cia_setkey = aria_set_key, - .cia_encrypt = aria_encrypt, - .cia_decrypt = aria_decrypt + .cia_encrypt = __aria_encrypt, + .cia_decrypt = __aria_decrypt } } }; @@ -286,3 +310,4 @@ MODULE_DESCRIPTION("ARIA Cipher Algorithm"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Taehee Yoo "); MODULE_ALIAS_CRYPTO("aria"); +MODULE_ALIAS_CRYPTO("aria-generic"); diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c index c9d218e53bcb0a2aafc7a71d9ebd7a118a601a55..d3fbee1e03e55f1132202022366798342e044922 100644 --- a/crypto/async_tx/raid6test.c +++ b/crypto/async_tx/raid6test.c @@ -37,7 +37,7 @@ static void makedata(int disks) int i; for (i = 0; i < disks; i++) { - prandom_bytes(page_address(data[i]), PAGE_SIZE); + get_random_bytes(page_address(data[i]), PAGE_SIZE); dataptrs[i] = data[i]; dataoffs[i] = 0; } @@ -189,7 +189,7 @@ static int test(int disks, int *tests) } -static int raid6_test(void) +static int __init raid6_test(void) { int err = 0; int tests = 0; @@ -236,7 +236,7 @@ static int raid6_test(void) return 0; } -static void raid6_test_exit(void) +static void __exit raid6_test_exit(void) { } diff --git a/crypto/curve25519-generic.c b/crypto/curve25519-generic.c index bd88fd571393d16bc1815eaffa424fcbd9a6973c..d055b0784c77c1be788fa1f677bb09e602b52f95 100644 --- a/crypto/curve25519-generic.c +++ b/crypto/curve25519-generic.c @@ -72,12 +72,12 @@ static struct kpp_alg curve25519_alg = { .max_size = curve25519_max_size, }; -static int curve25519_init(void) +static int __init curve25519_init(void) { return crypto_register_kpp(&curve25519_alg); } -static void curve25519_exit(void) +static void __exit curve25519_exit(void) { crypto_unregister_kpp(&curve25519_alg); } diff --git a/crypto/dh.c b/crypto/dh.c index 4406aeb1ff6167ca15e8c5301e24c9f823a4913e..99c3b2ef7adca58c5043838961e7c2cce6d29d9c 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -893,7 +893,7 @@ static struct crypto_template crypto_ffdhe_templates[] = {}; #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */ -static int dh_init(void) +static int __init dh_init(void) { int err; @@ -911,7 +911,7 @@ static int dh_init(void) return 0; } -static void dh_exit(void) +static void __exit dh_exit(void) { crypto_unregister_templates(crypto_ffdhe_templates, ARRAY_SIZE(crypto_ffdhe_templates)); diff --git a/crypto/drbg.c b/crypto/drbg.c index 177983b6ae38b3a6d59ae79b90c61219009de002..982d4ca4526d89246b0b6c87b87fb71b79516984 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1703,7 +1703,7 @@ static int drbg_init_hash_kernel(struct drbg_state *drbg) static int drbg_fini_hash_kernel(struct drbg_state *drbg) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; if (sdesc) { crypto_free_shash(sdesc->shash.tfm); kfree_sensitive(sdesc); @@ -1715,7 +1715,7 @@ static int drbg_fini_hash_kernel(struct drbg_state *drbg) static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, const unsigned char *key) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; crypto_shash_setkey(sdesc->shash.tfm, key, drbg_statelen(drbg)); } @@ -1723,7 +1723,7 @@ static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg, static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval, const struct list_head *in) { - struct sdesc *sdesc = (struct sdesc *)drbg->priv_data; + struct sdesc *sdesc = drbg->priv_data; struct drbg_string *input = NULL; crypto_shash_init(&sdesc->shash); @@ -1818,8 +1818,7 @@ static int drbg_init_sym_kernel(struct drbg_state *drbg) static void drbg_kcapi_symsetkey(struct drbg_state *drbg, const unsigned char *key) { - struct crypto_cipher *tfm = - (struct crypto_cipher *)drbg->priv_data; + struct crypto_cipher *tfm = drbg->priv_data; crypto_cipher_setkey(tfm, key, (drbg_keylen(drbg))); } @@ -1827,8 +1826,7 @@ static void drbg_kcapi_symsetkey(struct drbg_state *drbg, static int drbg_kcapi_sym(struct drbg_state *drbg, unsigned char *outval, const struct drbg_string *in) { - struct crypto_cipher *tfm = - (struct crypto_cipher *)drbg->priv_data; + struct crypto_cipher *tfm = drbg->priv_data; /* there is only component in *in */ BUG_ON(in->len < drbg_blocklen(drbg)); diff --git a/crypto/ecdh.c b/crypto/ecdh.c index e4857d534344f56c826d6dc4893efbaf932ee72e..80afee3234fbe79c674e364f3c1befdf3c6c8887 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -200,7 +200,7 @@ static struct kpp_alg ecdh_nist_p384 = { static bool ecdh_nist_p192_registered; -static int ecdh_init(void) +static int __init ecdh_init(void) { int ret; @@ -227,7 +227,7 @@ nist_p256_error: return ret; } -static void ecdh_exit(void) +static void __exit ecdh_exit(void) { if (ecdh_nist_p192_registered) crypto_unregister_kpp(&ecdh_nist_p192); diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index b3a8a6b572ba9ed71b1224cd67101c8b2d94200a..fbd76498aba8344d57af645ce352e696d442bda4 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -332,7 +332,7 @@ static struct akcipher_alg ecdsa_nist_p192 = { }; static bool ecdsa_nist_p192_registered; -static int ecdsa_init(void) +static int __init ecdsa_init(void) { int ret; @@ -359,7 +359,7 @@ nist_p256_error: return ret; } -static void ecdsa_exit(void) +static void __exit ecdsa_exit(void) { if (ecdsa_nist_p192_registered) crypto_unregister_akcipher(&ecdsa_nist_p192); diff --git a/crypto/essiv.c b/crypto/essiv.c index 8bcc5bdcb2a95476227a4749f57d5a72c059bb27..e33369df90344545b9bffd8bfda2697190441a5a 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -543,7 +543,7 @@ static int essiv_create(struct crypto_template *tmpl, struct rtattr **tb) } /* record the driver name so we can instantiate this exact algo later */ - strlcpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, + strscpy(ictx->shash_driver_name, hash_alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME); /* Instance fields */ diff --git a/crypto/rsa.c b/crypto/rsa.c index 0e555ee4addb7db47d7868b0062b5d6263ae02cf..c50f2d2a4d0649fd01566b5438d91c7e324805dc 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -327,7 +327,7 @@ static struct akcipher_alg rsa = { }, }; -static int rsa_init(void) +static int __init rsa_init(void) { int err; @@ -344,7 +344,7 @@ static int rsa_init(void) return 0; } -static void rsa_exit(void) +static void __exit rsa_exit(void) { crypto_unregister_template(&rsa_pkcs1pad_tmpl); crypto_unregister_akcipher(&rsa); diff --git a/crypto/sm2.c b/crypto/sm2.c index f3e1592965c01e8bd693dd58ecd0e70ea67e9c93..ed9307dac3d1eeb283a57a7f10b141d8159a5a34 100644 --- a/crypto/sm2.c +++ b/crypto/sm2.c @@ -441,12 +441,12 @@ static struct akcipher_alg sm2 = { }, }; -static int sm2_init(void) +static int __init sm2_init(void) { return crypto_register_akcipher(&sm2); } -static void sm2_exit(void) +static void __exit sm2_exit(void) { crypto_unregister_akcipher(&sm2); } diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 59eb8ec3666433a0a1aefbfb6452e02e73920759..a82679b576bb4381771a0f87ec0e401ef219a68e 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -66,17 +66,6 @@ static u32 num_mb = 8; static unsigned int klen; static char *tvmem[TVMEMSIZE]; -static const char *check[] = { - "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3", - "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", - "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", - "khazad", "wp512", "wp384", "wp256", "xeta", "fcrypt", - "camellia", "seed", "rmd160", "aria", - "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384", - "sha3-512", "streebog256", "streebog512", - NULL -}; - static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 }; static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; @@ -1454,18 +1443,6 @@ static void test_cipher_speed(const char *algo, int enc, unsigned int secs, false); } -static void test_available(void) -{ - const char **name = check; - - while (*name) { - printk("alg %s ", *name); - printk(crypto_has_alg(*name, 0, 0) ? - "found\n" : "not found\n"); - name++; - } -} - static inline int tcrypt_test(const char *alg) { int ret; @@ -2228,6 +2205,13 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) NULL, 0, 16, 8, speed_template_16_24_32); break; + case 229: + test_mb_aead_speed("gcm(aria)", ENCRYPT, sec, NULL, 0, 16, 8, + speed_template_16, num_mb); + test_mb_aead_speed("gcm(aria)", DECRYPT, sec, NULL, 0, 16, 8, + speed_template_16, num_mb); + break; + case 300: if (alg) { test_hash_speed(alg, sec, generic_hash_speed_template); @@ -2648,6 +2632,17 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16); break; + case 519: + test_acipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_24_32); + test_acipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_24_32); + break; + case 600: test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32, num_mb); @@ -2860,9 +2855,17 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_8_32, num_mb); break; - case 1000: - test_available(); + case 610: + test_mb_skcipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); + test_mb_skcipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, + speed_template_16_32, num_mb); break; + } return ret; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 5349ffee6bbd494ca13f01ab4b78d681e6487476..bcd059caa1c81321ee08b45a939b855f9dff2f98 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -855,9 +855,9 @@ static int prepare_keybuf(const u8 *key, unsigned int ksize, /* Generate a random length in range [0, max_len], but prefer smaller values */ static unsigned int generate_random_length(unsigned int max_len) { - unsigned int len = prandom_u32() % (max_len + 1); + unsigned int len = prandom_u32_max(max_len + 1); - switch (prandom_u32() % 4) { + switch (prandom_u32_max(4)) { case 0: return len % 64; case 1: @@ -874,14 +874,14 @@ static void flip_random_bit(u8 *buf, size_t size) { size_t bitpos; - bitpos = prandom_u32() % (size * 8); + bitpos = prandom_u32_max(size * 8); buf[bitpos / 8] ^= 1 << (bitpos % 8); } /* Flip a random byte in the given nonempty data buffer */ static void flip_random_byte(u8 *buf, size_t size) { - buf[prandom_u32() % size] ^= 0xff; + buf[prandom_u32_max(size)] ^= 0xff; } /* Sometimes make some random changes to the given nonempty data buffer */ @@ -891,15 +891,15 @@ static void mutate_buffer(u8 *buf, size_t size) size_t i; /* Sometimes flip some bits */ - if (prandom_u32() % 4 == 0) { - num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size * 8); + if (prandom_u32_max(4) == 0) { + num_flips = min_t(size_t, 1 << prandom_u32_max(8), size * 8); for (i = 0; i < num_flips; i++) flip_random_bit(buf, size); } /* Sometimes flip some bytes */ - if (prandom_u32() % 4 == 0) { - num_flips = min_t(size_t, 1 << (prandom_u32() % 8), size); + if (prandom_u32_max(4) == 0) { + num_flips = min_t(size_t, 1 << prandom_u32_max(8), size); for (i = 0; i < num_flips; i++) flip_random_byte(buf, size); } @@ -915,11 +915,11 @@ static void generate_random_bytes(u8 *buf, size_t count) if (count == 0) return; - switch (prandom_u32() % 8) { /* Choose a generation strategy */ + switch (prandom_u32_max(8)) { /* Choose a generation strategy */ case 0: case 1: /* All the same byte, plus optional mutations */ - switch (prandom_u32() % 4) { + switch (prandom_u32_max(4)) { case 0: b = 0x00; break; @@ -927,7 +927,7 @@ static void generate_random_bytes(u8 *buf, size_t count) b = 0xff; break; default: - b = (u8)prandom_u32(); + b = get_random_u8(); break; } memset(buf, b, count); @@ -935,8 +935,8 @@ static void generate_random_bytes(u8 *buf, size_t count) break; case 2: /* Ascending or descending bytes, plus optional mutations */ - increment = (u8)prandom_u32(); - b = (u8)prandom_u32(); + increment = get_random_u8(); + b = get_random_u8(); for (i = 0; i < count; i++, b += increment) buf[i] = b; mutate_buffer(buf, count); @@ -944,7 +944,7 @@ static void generate_random_bytes(u8 *buf, size_t count) default: /* Fully random bytes */ for (i = 0; i < count; i++) - buf[i] = (u8)prandom_u32(); + buf[i] = get_random_u8(); } } @@ -959,24 +959,24 @@ static char *generate_random_sgl_divisions(struct test_sg_division *divs, unsigned int this_len; const char *flushtype_str; - if (div == &divs[max_divs - 1] || prandom_u32() % 2 == 0) + if (div == &divs[max_divs - 1] || prandom_u32_max(2) == 0) this_len = remaining; else - this_len = 1 + (prandom_u32() % remaining); + this_len = 1 + prandom_u32_max(remaining); div->proportion_of_total = this_len; - if (prandom_u32() % 4 == 0) - div->offset = (PAGE_SIZE - 128) + (prandom_u32() % 128); - else if (prandom_u32() % 2 == 0) - div->offset = prandom_u32() % 32; + if (prandom_u32_max(4) == 0) + div->offset = (PAGE_SIZE - 128) + prandom_u32_max(128); + else if (prandom_u32_max(2) == 0) + div->offset = prandom_u32_max(32); else - div->offset = prandom_u32() % PAGE_SIZE; - if (prandom_u32() % 8 == 0) + div->offset = prandom_u32_max(PAGE_SIZE); + if (prandom_u32_max(8) == 0) div->offset_relative_to_alignmask = true; div->flush_type = FLUSH_TYPE_NONE; if (gen_flushes) { - switch (prandom_u32() % 4) { + switch (prandom_u32_max(4)) { case 0: div->flush_type = FLUSH_TYPE_REIMPORT; break; @@ -988,7 +988,7 @@ static char *generate_random_sgl_divisions(struct test_sg_division *divs, if (div->flush_type != FLUSH_TYPE_NONE && !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && - prandom_u32() % 2 == 0) + prandom_u32_max(2) == 0) div->nosimd = true; switch (div->flush_type) { @@ -1035,7 +1035,7 @@ static void generate_random_testvec_config(struct testvec_config *cfg, p += scnprintf(p, end - p, "random:"); - switch (prandom_u32() % 4) { + switch (prandom_u32_max(4)) { case 0: case 1: cfg->inplace_mode = OUT_OF_PLACE; @@ -1050,12 +1050,12 @@ static void generate_random_testvec_config(struct testvec_config *cfg, break; } - if (prandom_u32() % 2 == 0) { + if (prandom_u32_max(2) == 0) { cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP; p += scnprintf(p, end - p, " may_sleep"); } - switch (prandom_u32() % 4) { + switch (prandom_u32_max(4)) { case 0: cfg->finalization_type = FINALIZATION_TYPE_FINAL; p += scnprintf(p, end - p, " use_final"); @@ -1071,7 +1071,7 @@ static void generate_random_testvec_config(struct testvec_config *cfg, } if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) && - prandom_u32() % 2 == 0) { + prandom_u32_max(2) == 0) { cfg->nosimd = true; p += scnprintf(p, end - p, " nosimd"); } @@ -1084,7 +1084,7 @@ static void generate_random_testvec_config(struct testvec_config *cfg, cfg->req_flags); p += scnprintf(p, end - p, "]"); - if (cfg->inplace_mode == OUT_OF_PLACE && prandom_u32() % 2 == 0) { + if (cfg->inplace_mode == OUT_OF_PLACE && prandom_u32_max(2) == 0) { p += scnprintf(p, end - p, " dst_divs=["); p = generate_random_sgl_divisions(cfg->dst_divs, ARRAY_SIZE(cfg->dst_divs), @@ -1093,13 +1093,13 @@ static void generate_random_testvec_config(struct testvec_config *cfg, p += scnprintf(p, end - p, "]"); } - if (prandom_u32() % 2 == 0) { - cfg->iv_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK); + if (prandom_u32_max(2) == 0) { + cfg->iv_offset = 1 + prandom_u32_max(MAX_ALGAPI_ALIGNMASK); p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset); } - if (prandom_u32() % 2 == 0) { - cfg->key_offset = 1 + (prandom_u32() % MAX_ALGAPI_ALIGNMASK); + if (prandom_u32_max(2) == 0) { + cfg->key_offset = 1 + prandom_u32_max(MAX_ALGAPI_ALIGNMASK); p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset); } @@ -1652,8 +1652,8 @@ static void generate_random_hash_testvec(struct shash_desc *desc, vec->ksize = 0; if (maxkeysize) { vec->ksize = maxkeysize; - if (prandom_u32() % 4 == 0) - vec->ksize = 1 + (prandom_u32() % maxkeysize); + if (prandom_u32_max(4) == 0) + vec->ksize = 1 + prandom_u32_max(maxkeysize); generate_random_bytes((u8 *)vec->key, vec->ksize); vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key, @@ -2218,13 +2218,13 @@ static void mutate_aead_message(struct aead_testvec *vec, bool aad_iv, const unsigned int aad_tail_size = aad_iv ? ivsize : 0; const unsigned int authsize = vec->clen - vec->plen; - if (prandom_u32() % 2 == 0 && vec->alen > aad_tail_size) { + if (prandom_u32_max(2) == 0 && vec->alen > aad_tail_size) { /* Mutate the AAD */ flip_random_bit((u8 *)vec->assoc, vec->alen - aad_tail_size); - if (prandom_u32() % 2 == 0) + if (prandom_u32_max(2) == 0) return; } - if (prandom_u32() % 2 == 0) { + if (prandom_u32_max(2) == 0) { /* Mutate auth tag (assuming it's at the end of ciphertext) */ flip_random_bit((u8 *)vec->ctext + vec->plen, authsize); } else { @@ -2249,7 +2249,7 @@ static void generate_aead_message(struct aead_request *req, const unsigned int ivsize = crypto_aead_ivsize(tfm); const unsigned int authsize = vec->clen - vec->plen; const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) && - (prefer_inauthentic || prandom_u32() % 4 == 0); + (prefer_inauthentic || prandom_u32_max(4) == 0); /* Generate the AAD. */ generate_random_bytes((u8 *)vec->assoc, vec->alen); @@ -2257,7 +2257,7 @@ static void generate_aead_message(struct aead_request *req, /* Avoid implementation-defined behavior. */ memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize); - if (inauthentic && prandom_u32() % 2 == 0) { + if (inauthentic && prandom_u32_max(2) == 0) { /* Generate a random ciphertext. */ generate_random_bytes((u8 *)vec->ctext, vec->clen); } else { @@ -2321,8 +2321,8 @@ static void generate_random_aead_testvec(struct aead_request *req, /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ vec->klen = maxkeysize; - if (prandom_u32() % 4 == 0) - vec->klen = prandom_u32() % (maxkeysize + 1); + if (prandom_u32_max(4) == 0) + vec->klen = prandom_u32_max(maxkeysize + 1); generate_random_bytes((u8 *)vec->key, vec->klen); vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen); @@ -2331,8 +2331,8 @@ static void generate_random_aead_testvec(struct aead_request *req, /* Tag length: in [0, maxauthsize], but usually choose maxauthsize */ authsize = maxauthsize; - if (prandom_u32() % 4 == 0) - authsize = prandom_u32() % (maxauthsize + 1); + if (prandom_u32_max(4) == 0) + authsize = prandom_u32_max(maxauthsize + 1); if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE) authsize = MIN_COLLISION_FREE_AUTHSIZE; if (WARN_ON(authsize > maxdatasize)) @@ -2342,7 +2342,7 @@ static void generate_random_aead_testvec(struct aead_request *req, /* AAD, plaintext, and ciphertext lengths */ total_len = generate_random_length(maxdatasize); - if (prandom_u32() % 4 == 0) + if (prandom_u32_max(4) == 0) vec->alen = 0; else vec->alen = generate_random_length(total_len); @@ -2958,8 +2958,8 @@ static void generate_random_cipher_testvec(struct skcipher_request *req, /* Key: length in [0, maxkeysize], but usually choose maxkeysize */ vec->klen = maxkeysize; - if (prandom_u32() % 4 == 0) - vec->klen = prandom_u32() % (maxkeysize + 1); + if (prandom_u32_max(4) == 0) + vec->klen = prandom_u32_max(maxkeysize + 1); generate_random_bytes((u8 *)vec->key, vec->klen); vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen); @@ -3322,7 +3322,7 @@ out: } static int test_acomp(struct crypto_acomp *tfm, - const struct comp_testvec *ctemplate, + const struct comp_testvec *ctemplate, const struct comp_testvec *dtemplate, int ctcount, int dtcount) { @@ -3417,6 +3417,21 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + crypto_init_wait(&wait); + sg_init_one(&src, input_vec, ilen); + acomp_request_set_params(req, &src, NULL, ilen, 0); + + ret = crypto_wait_req(crypto_acomp_compress(req), &wait); + if (ret) { + pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } +#endif + kfree(input_vec); acomp_request_free(req); } @@ -3478,6 +3493,20 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } +#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS + crypto_init_wait(&wait); + acomp_request_set_params(req, &src, NULL, ilen, 0); + + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); + if (ret) { + pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } +#endif + kfree(input_vec); acomp_request_free(req); } @@ -5801,8 +5830,11 @@ test_done: driver, alg, fips_enabled ? "fips" : "panic_on_fail"); } - WARN(1, "alg: self-tests for %s (%s) failed (rc=%d)", - driver, alg, rc); + pr_warn("alg: self-tests for %s using %s failed (rc=%d)", + alg, driver, rc); + WARN(rc != -ENOENT, + "alg: self-tests for %s using %s failed (rc=%d)", + alg, driver, rc); } else { if (fips_enabled) pr_info("alg: self-tests for %s (%s) passed\n", diff --git a/drivers/Makefile b/drivers/Makefile index 057857258bfd90186dd9a188ccfa35c51c18619d..bdf1c66141c9bd5e8241a32233b768587e630504 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -175,6 +175,7 @@ obj-$(CONFIG_USB4) += thunderbolt/ obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ obj-y += hwtracing/intel_th/ obj-$(CONFIG_STM) += hwtracing/stm/ +obj-$(CONFIG_HISI_PTT) += hwtracing/ptt/ obj-y += android/ obj-$(CONFIG_NVMEM) += nvmem/ obj-$(CONFIG_FPGA) += fpga/ diff --git a/drivers/accessibility/speakup/speakup_dummy.c b/drivers/accessibility/speakup/speakup_dummy.c index 34f11cd47073e86a130ba8b1c1d9f86943fecc29..56419dbb28d3c678843d84f66a3dcd06134f6dce 100644 --- a/drivers/accessibility/speakup/speakup_dummy.c +++ b/drivers/accessibility/speakup/speakup_dummy.c @@ -27,6 +27,7 @@ static struct var_t vars[] = { { INFLECTION, .u.n = {"INFLECTION %d\n", 8, 0, 16, 0, 0, NULL } }, { VOL, .u.n = {"VOL %d\n", 8, 0, 16, 0, 0, NULL } }, { TONE, .u.n = {"TONE %d\n", 8, 0, 16, 0, 0, NULL } }, + { PUNCT, .u.n = {"PUNCT %d\n", 0, 0, 3, 0, 0, NULL } }, { DIRECT, .u.n = {NULL, 0, 0, 1, 0, 0, NULL } }, V_LAST_VAR }; @@ -42,6 +43,8 @@ static struct kobj_attribute pitch_attribute = __ATTR(pitch, 0644, spk_var_show, spk_var_store); static struct kobj_attribute inflection_attribute = __ATTR(inflection, 0644, spk_var_show, spk_var_store); +static struct kobj_attribute punct_attribute = + __ATTR(punct, 0644, spk_var_show, spk_var_store); static struct kobj_attribute rate_attribute = __ATTR(rate, 0644, spk_var_show, spk_var_store); static struct kobj_attribute tone_attribute = @@ -69,6 +72,7 @@ static struct attribute *synth_attrs[] = { &caps_stop_attribute.attr, &pitch_attribute.attr, &inflection_attribute.attr, + &punct_attribute.attr, &rate_attribute.attr, &tone_attribute.attr, &vol_attribute.attr, diff --git a/drivers/accessibility/speakup/speakup_soft.c b/drivers/accessibility/speakup/speakup_soft.c index 99f1d4ac426a44d37be27a4359ae58ce74e5b275..28c8f60370cf8f9d56bd0873f1ac5c48b873d1d1 100644 --- a/drivers/accessibility/speakup/speakup_soft.c +++ b/drivers/accessibility/speakup/speakup_soft.c @@ -26,6 +26,7 @@ static int softsynth_probe(struct spk_synth *synth); static void softsynth_release(struct spk_synth *synth); static int softsynth_is_alive(struct spk_synth *synth); +static int softsynth_adjust(struct spk_synth *synth, struct st_var_header *var); static unsigned char get_index(struct spk_synth *synth); static struct miscdevice synth_device, synthu_device; @@ -33,6 +34,9 @@ static int init_pos; static int misc_registered; static struct var_t vars[] = { + /* DIRECT is put first so that module_param_named can access it easily */ + { DIRECT, .u.n = {NULL, 0, 0, 1, 0, 0, NULL } }, + { CAPS_START, .u.s = {"\x01+3p" } }, { CAPS_STOP, .u.s = {"\x01-3p" } }, { PAUSE, .u.n = {"\x01P" } }, @@ -41,10 +45,9 @@ static struct var_t vars[] = { { INFLECTION, .u.n = {"\x01%dr", 5, 0, 9, 0, 0, NULL } }, { VOL, .u.n = {"\x01%dv", 5, 0, 9, 0, 0, NULL } }, { TONE, .u.n = {"\x01%dx", 1, 0, 2, 0, 0, NULL } }, - { PUNCT, .u.n = {"\x01%db", 0, 0, 2, 0, 0, NULL } }, + { PUNCT, .u.n = {"\x01%db", 0, 0, 3, 0, 0, NULL } }, { VOICE, .u.n = {"\x01%do", 0, 0, 7, 0, 0, NULL } }, { FREQUENCY, .u.n = {"\x01%df", 5, 0, 9, 0, 0, NULL } }, - { DIRECT, .u.n = {NULL, 0, 0, 1, 0, 0, NULL } }, V_LAST_VAR }; @@ -133,7 +136,7 @@ static struct spk_synth synth_soft = { .catch_up = NULL, .flush = NULL, .is_alive = softsynth_is_alive, - .synth_adjust = NULL, + .synth_adjust = softsynth_adjust, .read_buff_add = NULL, .get_index = get_index, .indexing = { @@ -426,9 +429,32 @@ static int softsynth_is_alive(struct spk_synth *synth) return 0; } +static int softsynth_adjust(struct spk_synth *synth, struct st_var_header *var) +{ + struct st_var_header *punc_level_var; + struct var_t *var_data; + + if (var->var_id != PUNC_LEVEL) + return 0; + + /* We want to set the the speech synthesis punctuation level + * accordingly, so it properly tunes speaking A_PUNC characters */ + var_data = var->data; + if (!var_data) + return 0; + punc_level_var = spk_get_var_header(PUNCT); + if (!punc_level_var) + return 0; + spk_set_num_var(var_data->u.n.value, punc_level_var, E_SET); + + return 1; +} + module_param_named(start, synth_soft.startup, short, 0444); +module_param_named(direct, vars[0].u.n.default_val, int, 0444); MODULE_PARM_DESC(start, "Start the synthesizer once it is loaded."); +MODULE_PARM_DESC(direct, "Set the direct variable on load."); module_spk_synth(synth_soft); diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 6a96ad94bc3f84255c6b14fb96db35ee1f017688..3a14d39bf896ebd9176fcf57de139687bbabc9af 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -195,7 +195,7 @@ struct spk_synth { void (*catch_up)(struct spk_synth *synth); void (*flush)(struct spk_synth *synth); int (*is_alive)(struct spk_synth *synth); - int (*synth_adjust)(struct st_var_header *var); + int (*synth_adjust)(struct spk_synth *synth, struct st_var_header *var); void (*read_buff_add)(u_char c); unsigned char (*get_index)(struct spk_synth *synth); struct synth_indexing indexing; diff --git a/drivers/accessibility/speakup/varhandlers.c b/drivers/accessibility/speakup/varhandlers.c index 067c0da97dcbf16cea24f32cdd069ffb7f27af65..e1c9f42e39f0fae670ee00ec9ffdb8a906c85667 100644 --- a/drivers/accessibility/speakup/varhandlers.c +++ b/drivers/accessibility/speakup/varhandlers.c @@ -138,6 +138,7 @@ struct st_var_header *spk_get_var_header(enum var_id_t var_id) return NULL; return p_header; } +EXPORT_SYMBOL_GPL(spk_get_var_header); struct st_var_header *spk_var_header_by_name(const char *name) { @@ -221,15 +222,17 @@ int spk_set_num_var(int input, struct st_var_header *var, int how) *p_val = val; if (var->var_id == PUNC_LEVEL) { spk_punc_mask = spk_punc_masks[val]; - return 0; } if (var_data->u.n.multiplier != 0) val *= var_data->u.n.multiplier; val += var_data->u.n.offset; - if (var->var_id < FIRST_SYNTH_VAR || !synth) + + if (!synth) + return 0; + if (synth->synth_adjust && synth->synth_adjust(synth, var)) + return 0; + if (var->var_id < FIRST_SYNTH_VAR) return 0; - if (synth->synth_adjust) - return synth->synth_adjust(var); if (!var_data->u.n.synth_fmt) return 0; @@ -245,6 +248,7 @@ int spk_set_num_var(int input, struct st_var_header *var, int how) synth_printf("%s", cp); return 0; } +EXPORT_SYMBOL_GPL(spk_set_num_var); int spk_set_string_var(const char *page, struct st_var_header *var, int len) { diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 44ad4b6bd234602e8df16def17855071e2f00a25..473241b5193fa1dc7f20a8def9fb1bfa9c2f8130 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -27,9 +27,6 @@ menuconfig ACPI Management (APM) specification. If both ACPI and APM support are configured, ACPI is used. - The project home page for the Linux ACPI subsystem is here: - - Linux support for ACPI is based on Intel Corporation's ACPI Component Architecture (ACPI CA). For more information on the ACPI CA, see: @@ -348,7 +345,6 @@ config ACPI_CUSTOM_DSDT_FILE depends on !STANDALONE help This option supports a custom DSDT by linking it into the kernel. - See Documentation/admin-guide/acpi/dsdt-override.rst Enter the full path name to the file which includes the AmlCode or dsdt_aml_code declaration. diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c index c29e41bfcf35d1f1d78f3738a33923162dd394b4..bb9fe7984b1a00fe3dae2321dab70979909ea3bf 100644 --- a/drivers/acpi/ac.c +++ b/drivers/acpi/ac.c @@ -36,11 +36,6 @@ static int acpi_ac_add(struct acpi_device *device); static int acpi_ac_remove(struct acpi_device *device); static void acpi_ac_notify(struct acpi_device *device, u32 event); -struct acpi_ac_bl { - const char *hid; - int hrv; -}; - static const struct acpi_device_id ac_device_ids[] = { {"ACPI0003", 0}, {"", 0}, diff --git a/drivers/acpi/acpi_amba.c b/drivers/acpi/acpi_amba.c index ab8a4e0191b19c9ff5a87e8cb9c47b11cc51bc59..f5b443ab01c2a83c394745a82754058f006a72da 100644 --- a/drivers/acpi/acpi_amba.c +++ b/drivers/acpi/acpi_amba.c @@ -21,6 +21,7 @@ static const struct acpi_device_id amba_id_list[] = { {"ARMH0061", 0}, /* PL061 GPIO Device */ + {"ARMH0330", 0}, /* ARM DMA Controller DMA-330 */ {"ARMHC500", 0}, /* ARM CoreSight ETM4x */ {"ARMHC501", 0}, /* ARM CoreSight ETR */ {"ARMHC502", 0}, /* ARM CoreSight STM */ @@ -48,6 +49,7 @@ static void amba_register_dummy_clk(void) static int amba_handler_attach(struct acpi_device *adev, const struct acpi_device_id *id) { + struct acpi_device *parent = acpi_dev_parent(adev); struct amba_device *dev; struct resource_entry *rentry; struct list_head resource_list; @@ -97,8 +99,8 @@ static int amba_handler_attach(struct acpi_device *adev, * attached to it, that physical device should be the parent of * the amba device we are about to create. */ - if (adev->parent) - dev->dev.parent = acpi_get_first_physical_node(adev->parent); + if (parent) + dev->dev.parent = acpi_get_first_physical_node(parent); ACPI_COMPANION_SET(&dev->dev, adev); diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index ad245bbd965ec97c6f4179dff9dc2280f111a2af..3bbe2276cac7674fbacb83205c6d714f96863243 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -60,12 +60,6 @@ static int acpi_apd_setup(struct apd_private_data *pdata) } #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE -static int misc_check_res(struct acpi_resource *ares, void *data) -{ - struct resource res; - - return !acpi_dev_resource_memory(ares, &res); -} static int fch_misc_setup(struct apd_private_data *pdata) { @@ -82,8 +76,7 @@ static int fch_misc_setup(struct apd_private_data *pdata) return -ENOMEM; INIT_LIST_HEAD(&resource_list); - ret = acpi_dev_get_resources(adev, &resource_list, misc_check_res, - NULL); + ret = acpi_dev_get_memory_resources(adev, &resource_list); if (ret < 0) return -ENOENT; diff --git a/drivers/acpi/acpi_fpdt.c b/drivers/acpi/acpi_fpdt.c index 6922a44b3ce701fbb8347ae2ae443e3acad4ac2e..a2056c4c8cb7099d68e50e9b015fcf3e3e65c263 100644 --- a/drivers/acpi/acpi_fpdt.c +++ b/drivers/acpi/acpi_fpdt.c @@ -143,6 +143,23 @@ static const struct attribute_group boot_attr_group = { static struct kobject *fpdt_kobj; +#if defined CONFIG_X86 && defined CONFIG_PHYS_ADDR_T_64BIT +#include +static bool fpdt_address_valid(u64 address) +{ + /* + * On some systems the table contains invalid addresses + * with unsuppored high address bits set, check for this. + */ + return !(address >> boot_cpu_data.x86_phys_bits); +} +#else +static bool fpdt_address_valid(u64 address) +{ + return true; +} +#endif + static int fpdt_process_subtable(u64 address, u32 subtable_type) { struct fpdt_subtable_header *subtable_header; @@ -151,6 +168,11 @@ static int fpdt_process_subtable(u64 address, u32 subtable_type) u32 length, offset; int result; + if (!fpdt_address_valid(address)) { + pr_info(FW_BUG "invalid physical address: 0x%llx!\n", address); + return -EINVAL; + } + subtable_header = acpi_os_map_memory(address, sizeof(*subtable_header)); if (!subtable_header) return -ENOMEM; diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index c4d4d21391d7b11e8d072d8e6fd16cf067df94ac..f08ffa75f4a766fb33e75a8ea7a6c7e6ac1e28e4 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -167,10 +167,10 @@ static struct pwm_lookup byt_pwm_lookup[] = { static void byt_pwm_setup(struct lpss_private_data *pdata) { - struct acpi_device *adev = pdata->adev; + u64 uid; /* Only call pwm_add_table for the first PWM controller */ - if (!adev->pnp.unique_id || strcmp(adev->pnp.unique_id, "1")) + if (acpi_dev_uid_to_integer(pdata->adev, &uid) || uid != 1) return; pwm_add_table(byt_pwm_lookup, ARRAY_SIZE(byt_pwm_lookup)); @@ -180,14 +180,13 @@ static void byt_pwm_setup(struct lpss_private_data *pdata) static void byt_i2c_setup(struct lpss_private_data *pdata) { - const char *uid_str = acpi_device_uid(pdata->adev); acpi_handle handle = pdata->adev->handle; unsigned long long shared_host = 0; acpi_status status; - long uid = 0; + u64 uid; - /* Expected to always be true, but better safe then sorry */ - if (uid_str && !kstrtol(uid_str, 10, &uid) && uid) { + /* Expected to always be successfull, but better safe then sorry */ + if (!acpi_dev_uid_to_integer(pdata->adev, &uid) && uid) { /* Detect I2C bus shared with PUNIT and ignore its d3 status */ status = acpi_evaluate_integer(handle, "_SEM", NULL, &shared_host); if (ACPI_SUCCESS(status) && shared_host) @@ -211,10 +210,10 @@ static struct pwm_lookup bsw_pwm_lookup[] = { static void bsw_pwm_setup(struct lpss_private_data *pdata) { - struct acpi_device *adev = pdata->adev; + u64 uid; /* Only call pwm_add_table for the first PWM controller */ - if (!adev->pnp.unique_id || strcmp(adev->pnp.unique_id, "1")) + if (acpi_dev_uid_to_integer(pdata->adev, &uid) || uid != 1) return; pwm_add_table(bsw_pwm_lookup, ARRAY_SIZE(bsw_pwm_lookup)); @@ -392,13 +391,6 @@ static const struct acpi_device_id acpi_lpss_device_ids[] = { #ifdef CONFIG_X86_INTEL_LPSS -static int is_memory(struct acpi_resource *res, void *not_used) -{ - struct resource r; - - return !acpi_dev_resource_memory(res, &r); -} - /* LPSS main clock device. */ static struct platform_device *lpss_clk_dev; @@ -659,29 +651,25 @@ static int acpi_lpss_create_device(struct acpi_device *adev, return -ENOMEM; INIT_LIST_HEAD(&resource_list); - ret = acpi_dev_get_resources(adev, &resource_list, is_memory, NULL); + ret = acpi_dev_get_memory_resources(adev, &resource_list); if (ret < 0) goto err_out; - list_for_each_entry(rentry, &resource_list, node) - if (resource_type(rentry->res) == IORESOURCE_MEM) { - if (dev_desc->prv_size_override) - pdata->mmio_size = dev_desc->prv_size_override; - else - pdata->mmio_size = resource_size(rentry->res); - pdata->mmio_base = ioremap(rentry->res->start, - pdata->mmio_size); - break; - } + rentry = list_first_entry_or_null(&resource_list, struct resource_entry, node); + if (rentry) { + if (dev_desc->prv_size_override) + pdata->mmio_size = dev_desc->prv_size_override; + else + pdata->mmio_size = resource_size(rentry->res); + pdata->mmio_base = ioremap(rentry->res->start, pdata->mmio_size); + } acpi_dev_free_resource_list(&resource_list); if (!pdata->mmio_base) { /* Avoid acpi_bus_attach() instantiating a pdev for this dev. */ adev->pnp.type.platform_id = 0; - /* Skip the device, but continue the namespace scan. */ - ret = 0; - goto err_out; + goto out_free; } pdata->adev = adev; @@ -692,11 +680,8 @@ static int acpi_lpss_create_device(struct acpi_device *adev, if (dev_desc->flags & LPSS_CLK) { ret = register_device_clock(adev, pdata); - if (ret) { - /* Skip the device, but continue the namespace scan. */ - ret = 0; - goto err_out; - } + if (ret) + goto out_free; } /* @@ -708,15 +693,19 @@ static int acpi_lpss_create_device(struct acpi_device *adev, adev->driver_data = pdata; pdev = acpi_create_platform_device(adev, dev_desc->properties); - if (!IS_ERR_OR_NULL(pdev)) { - acpi_lpss_create_device_links(adev, pdev); - return 1; + if (IS_ERR_OR_NULL(pdev)) { + adev->driver_data = NULL; + ret = PTR_ERR(pdev); + goto err_out; } - ret = PTR_ERR(pdev); - adev->driver_data = NULL; + acpi_lpss_create_device_links(adev, pdev); + return 1; - err_out: +out_free: + /* Skip the device, but continue the namespace scan */ + ret = 0; +err_out: kfree(pdata); return ret; } diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c index a12b55d812096e8ff1e3ed82e5f904ee28514571..ee4ce5ba1fb2417e0b0b59a3ad8f6b1f5ec5430a 100644 --- a/drivers/acpi/acpi_pcc.c +++ b/drivers/acpi/acpi_pcc.c @@ -23,6 +23,12 @@ #include +/* + * Arbitrary retries in case the remote processor is slow to respond + * to PCC commands + */ +#define PCC_CMD_WAIT_RETRIES_NUM 500 + struct pcc_data { struct pcc_mbox_chan *pcc_chan; void __iomem *pcc_comm_addr; @@ -63,6 +69,7 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function, if (IS_ERR(data->pcc_chan)) { pr_err("Failed to find PCC channel for subspace %d\n", ctx->subspace_id); + kfree(data); return AE_NOT_FOUND; } @@ -72,6 +79,8 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function, if (!data->pcc_comm_addr) { pr_err("Failed to ioremap PCC comm region mem for %d\n", ctx->subspace_id); + pcc_mbox_free_channel(data->pcc_chan); + kfree(data); return AE_NO_MEMORY; } @@ -86,6 +95,7 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr, { int ret; struct pcc_data *data = region_context; + u64 usecs_lat; reinit_completion(&data->done); @@ -96,10 +106,22 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr, if (ret < 0) return AE_ERROR; - if (data->pcc_chan->mchan->mbox->txdone_irq) - wait_for_completion(&data->done); + if (data->pcc_chan->mchan->mbox->txdone_irq) { + /* + * pcc_chan->latency is just a Nominal value. In reality the remote + * processor could be much slower to reply. So add an arbitrary + * amount of wait on top of Nominal. + */ + usecs_lat = PCC_CMD_WAIT_RETRIES_NUM * data->pcc_chan->latency; + ret = wait_for_completion_timeout(&data->done, + usecs_to_jiffies(usecs_lat)); + if (ret == 0) { + pr_err("PCC command executed timeout!\n"); + return AE_TIME; + } + } - mbox_client_txdone(data->pcc_chan->mchan, ret); + mbox_chan_txdone(data->pcc_chan->mchan, ret); memcpy_fromio(value, data->pcc_comm_addr, data->ctx.length); diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c index de3cbf152deef1aebeac08fa784ff0272baaa36c..fe00a5783f53f48798c6784126b0d95f9909e50a 100644 --- a/drivers/acpi/acpi_platform.c +++ b/drivers/acpi/acpi_platform.c @@ -20,13 +20,13 @@ #include "internal.h" static const struct acpi_device_id forbidden_id_list[] = { + {"ACPI0009", 0}, /* IOxAPIC */ + {"ACPI000A", 0}, /* IOAPIC */ {"PNP0000", 0}, /* PIC */ {"PNP0100", 0}, /* Timer */ {"PNP0200", 0}, /* AT DMA Controller */ - {"ACPI0009", 0}, /* IOxAPIC */ - {"ACPI000A", 0}, /* IOAPIC */ {"SMB0001", 0}, /* ACPI SMBUS virtual device */ - {"", 0}, + { } }; static struct platform_device *acpi_platform_device_find_by_companion(struct acpi_device *adev) @@ -78,7 +78,7 @@ static void acpi_platform_fill_resource(struct acpi_device *adev, * If the device has parent we need to take its resources into * account as well because this device might consume part of those. */ - parent = acpi_get_first_physical_node(adev->parent); + parent = acpi_get_first_physical_node(acpi_dev_parent(adev)); if (parent && dev_is_pci(parent)) dest->parent = pci_find_resource(to_pci_dev(parent), dest); } @@ -97,6 +97,7 @@ static void acpi_platform_fill_resource(struct acpi_device *adev, struct platform_device *acpi_create_platform_device(struct acpi_device *adev, const struct property_entry *properties) { + struct acpi_device *parent = acpi_dev_parent(adev); struct platform_device *pdev = NULL; struct platform_device_info pdevinfo; struct resource_entry *rentry; @@ -113,13 +114,11 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, INIT_LIST_HEAD(&resource_list); count = acpi_dev_get_resources(adev, &resource_list, NULL, NULL); - if (count < 0) { + if (count < 0) return NULL; - } else if (count > 0) { - resources = kcalloc(count, sizeof(struct resource), - GFP_KERNEL); + if (count > 0) { + resources = kcalloc(count, sizeof(*resources), GFP_KERNEL); if (!resources) { - dev_err(&adev->dev, "No memory for resources\n"); acpi_dev_free_resource_list(&resource_list); return ERR_PTR(-ENOMEM); } @@ -137,10 +136,9 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev, * attached to it, that physical device should be the parent of the * platform device we are about to create. */ - pdevinfo.parent = adev->parent ? - acpi_get_first_physical_node(adev->parent) : NULL; + pdevinfo.parent = parent ? acpi_get_first_physical_node(parent) : NULL; pdevinfo.name = dev_name(&adev->dev); - pdevinfo.id = -1; + pdevinfo.id = PLATFORM_DEVID_NONE; pdevinfo.res = resources; pdevinfo.num_res = count; pdevinfo.fwnode = acpi_fwnode_handle(adev); diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index a7c3d11e0dac71dd5aea7fdaef52c61bf5272eb9..32953646caebf6a57bac1d367ad0d79ec104d822 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -47,9 +47,6 @@ module_param(brightness_switch_enabled, bool, 0644); static bool allow_duplicates; module_param(allow_duplicates, bool, 0644); -static int disable_backlight_sysfs_if = -1; -module_param(disable_backlight_sysfs_if, int, 0444); - #define REPORT_OUTPUT_KEY_EVENTS 0x01 #define REPORT_BRIGHTNESS_KEY_EVENTS 0x02 static int report_key_events = -1; @@ -394,14 +391,6 @@ static int video_set_bqc_offset(const struct dmi_system_id *d) return 0; } -static int video_disable_backlight_sysfs_if( - const struct dmi_system_id *d) -{ - if (disable_backlight_sysfs_if == -1) - disable_backlight_sysfs_if = 1; - return 0; -} - static int video_set_device_id_scheme(const struct dmi_system_id *d) { device_id_scheme = true; @@ -474,40 +463,6 @@ static const struct dmi_system_id video_dmi_table[] = { }, }, - /* - * Some machines have a broken acpi-video interface for brightness - * control, but still need an acpi_video_device_lcd_set_level() call - * on resume to turn the backlight power on. We Enable backlight - * control on these systems, but do not register a backlight sysfs - * as brightness control does not work. - */ - { - /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ - .callback = video_disable_backlight_sysfs_if, - .ident = "Toshiba Portege R700", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), - DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"), - }, - }, - { - /* https://bugs.freedesktop.org/show_bug.cgi?id=82634 */ - .callback = video_disable_backlight_sysfs_if, - .ident = "Toshiba Portege R830", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), - DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R830"), - }, - }, - { - /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ - .callback = video_disable_backlight_sysfs_if, - .ident = "Toshiba Satellite R830", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), - DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE R830"), - }, - }, /* * Some machine's _DOD IDs don't have bit 31(Device ID Scheme) set * but the IDs actually follow the Device ID Scheme. @@ -1770,9 +1725,6 @@ static void acpi_video_dev_register_backlight(struct acpi_video_device *device) if (result) return; - if (disable_backlight_sysfs_if > 0) - return; - name = kasprintf(GFP_KERNEL, "acpi_video%d", count); if (!name) return; @@ -2040,7 +1992,7 @@ static int acpi_video_bus_add(struct acpi_device *device) acpi_status status; status = acpi_walk_namespace(ACPI_TYPE_DEVICE, - device->parent->handle, 1, + acpi_dev_parent(device)->handle, 1, acpi_video_bus_match, NULL, device, NULL); if (status == AE_ALREADY_EXISTS) { diff --git a/drivers/acpi/apei/apei-base.c b/drivers/acpi/apei/apei-base.c index 9f49272cad392b69e973a6555029438d373e0c3e..9b52482b4ed5ee7ddec61556c9b6ae540a9b1989 100644 --- a/drivers/acpi/apei/apei-base.c +++ b/drivers/acpi/apei/apei-base.c @@ -125,12 +125,9 @@ EXPORT_SYMBOL_GPL(apei_exec_write_register); int apei_exec_write_register_value(struct apei_exec_context *ctx, struct acpi_whea_header *entry) { - int rc; - ctx->value = entry->value; - rc = apei_exec_write_register(ctx, entry); - return rc; + return apei_exec_write_register(ctx, entry); } EXPORT_SYMBOL_GPL(apei_exec_write_register_value); diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c index 45973aa6e06d487815acaf48a4c3d90dcede7a90..c23eb75866d0214fa98688e1206d106effb7cca7 100644 --- a/drivers/acpi/apei/bert.c +++ b/drivers/acpi/apei/bert.c @@ -90,6 +90,9 @@ static void __init bert_print_all(struct acpi_bert_region *region, if (skipped) pr_info(HW_ERR "Skipped %d error records\n", skipped); + + if (printed + skipped) + pr_info("Total records found: %d\n", printed + skipped); } static int __init setup_bert_disable(char *str) diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index 31b077eedb584860e14452e15aabab66481198a9..247989060e2917a419d3082963ce188fdfeb4828 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -1020,14 +1020,10 @@ static int reader_pos; static int erst_open_pstore(struct pstore_info *psi) { - int rc; - if (erst_disable) return -ENODEV; - rc = erst_get_record_id_begin(&reader_pos); - - return rc; + return erst_get_record_id_begin(&reader_pos); } static int erst_close_pstore(struct pstore_info *psi) diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index d91ad378c00d604b2c7a897fd2f9769c5e4106f2..80ad530583c9c9c602eb1d17fca4eca7fd74efab 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -985,7 +985,7 @@ static void ghes_proc_in_irq(struct irq_work *irq_work) ghes_estatus_cache_add(generic, estatus); } - if (task_work_pending && current->mm != &init_mm) { + if (task_work_pending && current->mm) { estatus_node->task_work.func = ghes_kick_task_work; estatus_node->task_work_cpu = smp_processor_id(); ret = task_work_add(current, &estatus_node->task_work, diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c index f16739ad3cc05848aef1290d09318d0152eb82c2..93d796531af307d59965e7bf62a2d70a1fa13a75 100644 --- a/drivers/acpi/arm64/dma.c +++ b/drivers/acpi/arm64/dma.c @@ -4,11 +4,12 @@ #include #include -void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) +void acpi_arch_dma_setup(struct device *dev) { int ret; u64 end, mask; - u64 dmaaddr = 0, size = 0, offset = 0; + u64 size = 0; + const struct bus_dma_region *map = NULL; /* * If @dev is expected to be DMA-capable then the bus code that created @@ -26,7 +27,19 @@ void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) else size = 1ULL << 32; - ret = acpi_dma_get_range(dev, &dmaaddr, &offset, &size); + ret = acpi_dma_get_range(dev, &map); + if (!ret && map) { + const struct bus_dma_region *r = map; + + for (end = 0; r->size; r++) { + if (r->dma_start + r->size - 1 > end) + end = r->dma_start + r->size - 1; + } + + size = end + 1; + dev->dma_range_map = map; + } + if (ret == -ENODEV) ret = iort_dma_get_ranges(dev, &size); if (!ret) { @@ -34,17 +47,10 @@ void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) * Limit coherent and dma mask based on size retrieved from * firmware. */ - end = dmaaddr + size - 1; + end = size - 1; mask = DMA_BIT_MASK(ilog2(end) + 1); dev->bus_dma_limit = end; dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask); *dev->dma_mask = min(*dev->dma_mask, mask); } - - *dma_addr = dmaaddr; - *dma_size = size; - - ret = dma_direct_set_offset(dev, dmaaddr + offset, dmaaddr, size); - - dev_dbg(dev, "dma_offset(%#08llx)%s\n", offset, ret ? " failed!" : ""); } diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index c0d20d997891c34d69c6397f3f1fcfcc96ab0220..d466c81953146b92acd6a1ac4615d7f471f8fd9c 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -456,7 +456,7 @@ out_free: Notification Handling -------------------------------------------------------------------------- */ -/** +/* * acpi_bus_notify * --------------- * Callback for all 'system-level' device notifications (values 0x00-0x7F). @@ -511,7 +511,7 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data) break; } - adev = acpi_bus_get_acpi_device(handle); + adev = acpi_get_acpi_dev(handle); if (!adev) goto err; @@ -524,14 +524,14 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data) } if (!hotplug_event) { - acpi_bus_put_acpi_device(adev); + acpi_put_acpi_dev(adev); return; } if (ACPI_SUCCESS(acpi_hotplug_schedule(adev, type))) return; - acpi_bus_put_acpi_device(adev); + acpi_put_acpi_dev(adev); err: acpi_evaluate_ost(handle, type, ost_code, NULL); @@ -802,7 +802,7 @@ static bool acpi_of_modalias(struct acpi_device *adev, str = obj->string.pointer; chr = strchr(str, ','); - strlcpy(modalias, chr ? chr + 1 : str, len); + strscpy(modalias, chr ? chr + 1 : str, len); return true; } @@ -822,7 +822,7 @@ void acpi_set_modalias(struct acpi_device *adev, const char *default_id, char *modalias, size_t len) { if (!acpi_of_modalias(adev, modalias, len)) - strlcpy(modalias, default_id, len); + strscpy(modalias, default_id, len); } EXPORT_SYMBOL_GPL(acpi_set_modalias); @@ -925,12 +925,13 @@ static const void *acpi_of_device_get_match_data(const struct device *dev) const void *acpi_device_get_match_data(const struct device *dev) { + const struct acpi_device_id *acpi_ids = dev->driver->acpi_match_table; const struct acpi_device_id *match; - if (!dev->driver->acpi_match_table) + if (!acpi_ids) return acpi_of_device_get_match_data(dev); - match = acpi_match_device(dev->driver->acpi_match_table, dev); + match = acpi_match_device(acpi_ids, dev); if (!match) return NULL; @@ -948,14 +949,13 @@ EXPORT_SYMBOL(acpi_match_device_ids); bool acpi_driver_match_device(struct device *dev, const struct device_driver *drv) { - if (!drv->acpi_match_table) - return acpi_of_match_device(ACPI_COMPANION(dev), - drv->of_match_table, - NULL); - - return __acpi_match_device(acpi_companion_match(dev), - drv->acpi_match_table, drv->of_match_table, - NULL, NULL); + const struct acpi_device_id *acpi_ids = drv->acpi_match_table; + const struct of_device_id *of_ids = drv->of_match_table; + + if (!acpi_ids) + return acpi_of_match_device(ACPI_COMPANION(dev), of_ids, NULL); + + return __acpi_match_device(acpi_companion_match(dev), acpi_ids, of_ids, NULL, NULL); } EXPORT_SYMBOL_GPL(acpi_driver_match_device); @@ -973,16 +973,13 @@ EXPORT_SYMBOL_GPL(acpi_driver_match_device); */ int acpi_bus_register_driver(struct acpi_driver *driver) { - int ret; - if (acpi_disabled) return -ENODEV; driver->drv.name = driver->name; driver->drv.bus = &acpi_bus_type; driver->drv.owner = driver->owner; - ret = driver_register(&driver->drv); - return ret; + return driver_register(&driver->drv); } EXPORT_SYMBOL(acpi_bus_register_driver); diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 1e15a9f25ae97153c780ed9a9f8996e5db76119e..093675b1a1ffbe04614e7b3a317ea835a14685ae 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -424,6 +424,9 @@ bool acpi_cpc_valid(void) struct cpc_desc *cpc_ptr; int cpu; + if (acpi_disabled) + return false; + for_each_present_cpu(cpu) { cpc_ptr = per_cpu(cpc_desc_ptr, cpu); if (!cpc_ptr) @@ -1240,6 +1243,48 @@ out_err: } EXPORT_SYMBOL_GPL(cppc_get_perf_caps); +/** + * cppc_perf_ctrs_in_pcc - Check if any perf counters are in a PCC region. + * + * CPPC has flexibility about how CPU performance counters are accessed. + * One of the choices is PCC regions, which can have a high access latency. This + * routine allows callers of cppc_get_perf_ctrs() to know this ahead of time. + * + * Return: true if any of the counters are in PCC regions, false otherwise + */ +bool cppc_perf_ctrs_in_pcc(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + struct cpc_register_resource *ref_perf_reg; + struct cpc_desc *cpc_desc; + + cpc_desc = per_cpu(cpc_desc_ptr, cpu); + + if (CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) || + CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) || + CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME])) + return true; + + + ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; + + /* + * If reference perf register is not supported then we should + * use the nominal perf value + */ + if (!CPC_SUPPORTED(ref_perf_reg)) + ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; + + if (CPC_IN_PCC(ref_perf_reg)) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc); + /** * cppc_get_perf_ctrs - Read a CPU's performance feedback counters. * @cpunum: CPU from which to read counters. diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c index 9dce1245689ca2512240b20c6e544ce09b19621c..97450f4003cc9befa407496fb13476bc67ea2dc9 100644 --- a/drivers/acpi/device_pm.c +++ b/drivers/acpi/device_pm.c @@ -75,15 +75,17 @@ static int acpi_dev_pm_explicit_get(struct acpi_device *device, int *state) int acpi_device_get_power(struct acpi_device *device, int *state) { int result = ACPI_STATE_UNKNOWN; + struct acpi_device *parent; int error; if (!device || !state) return -EINVAL; + parent = acpi_dev_parent(device); + if (!device->flags.power_manageable) { /* TBD: Non-recursive algorithm for walking up hierarchy. */ - *state = device->parent ? - device->parent->power.state : ACPI_STATE_D0; + *state = parent ? parent->power.state : ACPI_STATE_D0; goto out; } @@ -122,10 +124,10 @@ int acpi_device_get_power(struct acpi_device *device, int *state) * point, the fact that the device is in D0 implies that the parent has * to be in D0 too, except if ignore_parent is set. */ - if (!device->power.flags.ignore_parent && device->parent - && device->parent->power.state == ACPI_STATE_UNKNOWN - && result == ACPI_STATE_D0) - device->parent->power.state = ACPI_STATE_D0; + if (!device->power.flags.ignore_parent && parent && + parent->power.state == ACPI_STATE_UNKNOWN && + result == ACPI_STATE_D0) + parent->power.state = ACPI_STATE_D0; *state = result; @@ -191,13 +193,17 @@ int acpi_device_set_power(struct acpi_device *device, int state) return -ENODEV; } - if (!device->power.flags.ignore_parent && device->parent && - state < device->parent->power.state) { - acpi_handle_debug(device->handle, - "Cannot transition to %s for parent in %s\n", - acpi_power_state_string(state), - acpi_power_state_string(device->parent->power.state)); - return -ENODEV; + if (!device->power.flags.ignore_parent) { + struct acpi_device *parent; + + parent = acpi_dev_parent(device); + if (parent && state < parent->power.state) { + acpi_handle_debug(device->handle, + "Cannot transition to %s for parent in %s\n", + acpi_power_state_string(state), + acpi_power_state_string(parent->power.state)); + return -ENODEV; + } } /* @@ -497,7 +503,7 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used) acpi_handle_debug(handle, "Wake notify\n"); - adev = acpi_bus_get_acpi_device(handle); + adev = acpi_get_acpi_dev(handle); if (!adev) return; @@ -515,7 +521,7 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used) mutex_unlock(&acpi_pm_notifier_lock); - acpi_bus_put_acpi_device(adev); + acpi_put_acpi_dev(adev); } /** @@ -681,7 +687,22 @@ static int acpi_dev_pm_get_state(struct device *dev, struct acpi_device *adev, d_min = ret; wakeup = device_may_wakeup(dev) && adev->wakeup.flags.valid && adev->wakeup.sleep_state >= target_state; + } else if (device_may_wakeup(dev) && dev->power.wakeirq) { + /* + * The ACPI subsystem doesn't manage the wake bit for IRQs + * defined with ExclusiveAndWake and SharedAndWake. Instead we + * expect them to be managed via the PM subsystem. Drivers + * should call dev_pm_set_wake_irq to register an IRQ as a wake + * source. + * + * If a device has a wake IRQ attached we need to check the + * _S0W method to get the correct wake D-state. Otherwise we + * end up putting the device into D3Cold which will more than + * likely disable wake functionality. + */ + wakeup = true; } else { + /* ACPI GPE is specified in _PRW. */ wakeup = adev->wakeup.flags.valid; } @@ -1460,7 +1481,7 @@ EXPORT_SYMBOL_GPL(acpi_storage_d3); * not valid to ask for the ACPI power state of the device in that time frame. * * This function is intended to be used in a driver's probe or remove - * function. See Documentation/firmware-guide/acpi/low-power-probe.rst for + * function. See Documentation/firmware-guide/acpi/non-d0-probe.rst for * more information. */ bool acpi_dev_state_d0(struct device *dev) diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig index 1e8c7ce89bf1f66d0bba80ebb4874a9bbf68b65b..4b3fdc03e4edbf2fc674b2e8f55e0b6129dc3c80 100644 --- a/drivers/acpi/dptf/Kconfig +++ b/drivers/acpi/dptf/Kconfig @@ -11,9 +11,6 @@ menuconfig ACPI_DPTF a coordinated approach for different policies to effect the hardware state of a system. - For more information see: - - if ACPI_DPTF config DPTF_POWER diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index c95e535035a04adaed3d88b954655dbf23859dd2..9b42628cf21b328963285e2c04b1e92d912fbb1a 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -917,14 +917,10 @@ EXPORT_SYMBOL(ec_read); int ec_write(u8 addr, u8 val) { - int err; - if (!first_ec) return -ENODEV; - err = acpi_ec_write(first_ec, addr, val); - - return err; + return acpi_ec_write(first_ec, addr, val); } EXPORT_SYMBOL(ec_write); diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c index b9a9a59ddcc1081efa1c40a68f17eadd3288c6dc..52a0b303b70aab742a5090c720cb8df5ff8eedc1 100644 --- a/drivers/acpi/fan_core.c +++ b/drivers/acpi/fan_core.c @@ -19,43 +19,12 @@ #include "fan.h" -MODULE_AUTHOR("Paul Diefenbaugh"); -MODULE_DESCRIPTION("ACPI Fan Driver"); -MODULE_LICENSE("GPL"); - -static int acpi_fan_probe(struct platform_device *pdev); -static int acpi_fan_remove(struct platform_device *pdev); - static const struct acpi_device_id fan_device_ids[] = { ACPI_FAN_DEVICE_IDS, {"", 0}, }; MODULE_DEVICE_TABLE(acpi, fan_device_ids); -#ifdef CONFIG_PM_SLEEP -static int acpi_fan_suspend(struct device *dev); -static int acpi_fan_resume(struct device *dev); -static const struct dev_pm_ops acpi_fan_pm = { - .resume = acpi_fan_resume, - .freeze = acpi_fan_suspend, - .thaw = acpi_fan_resume, - .restore = acpi_fan_resume, -}; -#define FAN_PM_OPS_PTR (&acpi_fan_pm) -#else -#define FAN_PM_OPS_PTR NULL -#endif - -static struct platform_driver acpi_fan_driver = { - .probe = acpi_fan_probe, - .remove = acpi_fan_remove, - .driver = { - .name = "acpi-fan", - .acpi_match_table = fan_device_ids, - .pm = FAN_PM_OPS_PTR, - }, -}; - /* thermal cooling device callbacks */ static int fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) @@ -459,6 +428,33 @@ static int acpi_fan_resume(struct device *dev) return result; } + +static const struct dev_pm_ops acpi_fan_pm = { + .resume = acpi_fan_resume, + .freeze = acpi_fan_suspend, + .thaw = acpi_fan_resume, + .restore = acpi_fan_resume, +}; +#define FAN_PM_OPS_PTR (&acpi_fan_pm) + +#else + +#define FAN_PM_OPS_PTR NULL + #endif +static struct platform_driver acpi_fan_driver = { + .probe = acpi_fan_probe, + .remove = acpi_fan_remove, + .driver = { + .name = "acpi-fan", + .acpi_match_table = fan_device_ids, + .pm = FAN_PM_OPS_PTR, + }, +}; + module_platform_driver(acpi_fan_driver); + +MODULE_AUTHOR("Paul Diefenbaugh"); +MODULE_DESCRIPTION("ACPI Fan Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 628bf8f1813042bff00167df5a67add4594b4072..219c02df9a08c720dcca32559f47d3770b0507c2 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -102,10 +102,10 @@ struct acpi_device_bus_id { struct list_head node; }; -int acpi_device_add(struct acpi_device *device, - void (*release)(struct device *)); void acpi_init_device_object(struct acpi_device *device, acpi_handle handle, - int type); + int type, void (*release)(struct device *)); +int acpi_tie_acpi_dev(struct acpi_device *adev); +int acpi_device_add(struct acpi_device *device); int acpi_device_setup_files(struct acpi_device *dev); void acpi_device_remove_files(struct acpi_device *dev); void acpi_device_add_finalize(struct acpi_device *device); diff --git a/drivers/acpi/irq.c b/drivers/acpi/irq.c index dabe45eba055d1f28721138e832ad422650bcf81..1cc4647f78b86d70875f84615b1c131e64d7a4b5 100644 --- a/drivers/acpi/irq.c +++ b/drivers/acpi/irq.c @@ -118,12 +118,12 @@ acpi_get_irq_source_fwhandle(const struct acpi_resource_source *source, if (WARN_ON(ACPI_FAILURE(status))) return NULL; - device = acpi_bus_get_acpi_device(handle); + device = acpi_get_acpi_dev(handle); if (WARN_ON(!device)) return NULL; result = &device->fwnode; - acpi_bus_put_acpi_device(device); + acpi_put_acpi_dev(device); return result; } @@ -147,6 +147,7 @@ struct acpi_irq_parse_one_ctx { * @polarity: polarity attributes of hwirq * @polarity: polarity attributes of hwirq * @shareable: shareable attributes of hwirq + * @wake_capable: wake capable attribute of hwirq * @ctx: acpi_irq_parse_one_ctx updated by this function * * Description: @@ -156,12 +157,13 @@ struct acpi_irq_parse_one_ctx { static inline void acpi_irq_parse_one_match(struct fwnode_handle *fwnode, u32 hwirq, u8 triggering, u8 polarity, u8 shareable, + u8 wake_capable, struct acpi_irq_parse_one_ctx *ctx) { if (!fwnode) return; ctx->rc = 0; - *ctx->res_flags = acpi_dev_irq_flags(triggering, polarity, shareable); + *ctx->res_flags = acpi_dev_irq_flags(triggering, polarity, shareable, wake_capable); ctx->fwspec->fwnode = fwnode; ctx->fwspec->param[0] = hwirq; ctx->fwspec->param[1] = acpi_dev_get_irq_type(triggering, polarity); @@ -204,7 +206,7 @@ static acpi_status acpi_irq_parse_one_cb(struct acpi_resource *ares, fwnode = acpi_get_gsi_domain_id(irq->interrupts[ctx->index]); acpi_irq_parse_one_match(fwnode, irq->interrupts[ctx->index], irq->triggering, irq->polarity, - irq->shareable, ctx); + irq->shareable, irq->wake_capable, ctx); return AE_CTRL_TERMINATE; case ACPI_RESOURCE_TYPE_EXTENDED_IRQ: eirq = &ares->data.extended_irq; @@ -218,7 +220,7 @@ static acpi_status acpi_irq_parse_one_cb(struct acpi_resource *ares, eirq->interrupts[ctx->index]); acpi_irq_parse_one_match(fwnode, eirq->interrupts[ctx->index], eirq->triggering, eirq->polarity, - eirq->shareable, ctx); + eirq->shareable, eirq->wake_capable, ctx); return AE_CTRL_TERMINATE; } diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c index c3d783aca196ff5aec773b1c8734b54ef55f6429..23f49a2f4d148d2fa6baa28d4d8ae6f4e8d0ae60 100644 --- a/drivers/acpi/numa/hmat.c +++ b/drivers/acpi/numa/hmat.c @@ -9,7 +9,6 @@ */ #define pr_fmt(fmt) "acpi/hmat: " fmt -#define dev_fmt(fmt) "acpi/hmat: " fmt #include #include @@ -302,7 +301,7 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header, u8 type, mem_hier; if (hmat_loc->header.length < sizeof(*hmat_loc)) { - pr_notice("HMAT: Unexpected locality header length: %u\n", + pr_notice("Unexpected locality header length: %u\n", hmat_loc->header.length); return -EINVAL; } @@ -314,12 +313,12 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header, total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + sizeof(*inits) * ipds + sizeof(*targs) * tpds; if (hmat_loc->header.length < total_size) { - pr_notice("HMAT: Unexpected locality header length:%u, minimum required:%u\n", + pr_notice("Unexpected locality header length:%u, minimum required:%u\n", hmat_loc->header.length, total_size); return -EINVAL; } - pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", + pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", hmat_loc->flags, hmat_data_type(type), ipds, tpds, hmat_loc->entry_base_unit); @@ -363,13 +362,13 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header, u32 attrs; if (cache->header.length < sizeof(*cache)) { - pr_notice("HMAT: Unexpected cache header length: %u\n", + pr_notice("Unexpected cache header length: %u\n", cache->header.length); return -EINVAL; } attrs = cache->cache_attributes; - pr_info("HMAT: Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", + pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", cache->memory_PD, cache->cache_size, attrs, cache->number_of_SMBIOShandles); @@ -424,24 +423,24 @@ static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *heade struct memory_target *target = NULL; if (p->header.length != sizeof(*p)) { - pr_notice("HMAT: Unexpected address range header length: %u\n", + pr_notice("Unexpected address range header length: %u\n", p->header.length); return -EINVAL; } if (hmat_revision == 1) - pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", + pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", p->reserved3, p->reserved4, p->flags, p->processor_PD, p->memory_PD); else - pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", + pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", p->flags, p->processor_PD, p->memory_PD); if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || hmat_revision > 1) { target = find_mem_target(p->memory_PD); if (!target) { - pr_debug("HMAT: Memory Domain missing from SRAT\n"); + pr_debug("Memory Domain missing from SRAT\n"); return -EINVAL; } } @@ -449,7 +448,7 @@ static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *heade int p_node = pxm_to_node(p->processor_PD); if (p_node == NUMA_NO_NODE) { - pr_debug("HMAT: Invalid Processor Domain\n"); + pr_debug("Invalid Processor Domain\n"); return -EINVAL; } target->processor_pxm = p->processor_PD; @@ -840,7 +839,7 @@ static __init int hmat_init(void) case 2: break; default: - pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision); + pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision); goto out_put; } @@ -848,7 +847,7 @@ static __init int hmat_init(void) if (acpi_table_parse_entries(ACPI_SIG_HMAT, sizeof(struct acpi_table_hmat), i, hmat_parse_subtable, 0) < 0) { - pr_notice("Ignoring HMAT: Invalid table"); + pr_notice("Ignoring: Invalid table"); goto out_put; } } diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c index 9f685380913849aa9b142a749aaec993737d23f7..d4405e1ca9b9719df5e6bb8ebdde3032357722aa 100644 --- a/drivers/acpi/osi.c +++ b/drivers/acpi/osi.c @@ -44,30 +44,6 @@ osi_setup_entries[OSI_STRING_ENTRIES_MAX] __initdata = { {"Processor Device", true}, {"3.0 _SCP Extensions", true}, {"Processor Aggregator Device", true}, - /* - * Linux-Dell-Video is used by BIOS to disable RTD3 for NVidia graphics - * cards as RTD3 is not supported by drivers now. Systems with NVidia - * cards will hang without RTD3 disabled. - * - * Once NVidia drivers officially support RTD3, this _OSI strings can - * be removed if both new and old graphics cards are supported. - */ - {"Linux-Dell-Video", true}, - /* - * Linux-Lenovo-NV-HDMI-Audio is used by BIOS to power on NVidia's HDMI - * audio device which is turned off for power-saving in Windows OS. - * This power management feature observed on some Lenovo Thinkpad - * systems which will not be able to output audio via HDMI without - * a BIOS workaround. - */ - {"Linux-Lenovo-NV-HDMI-Audio", true}, - /* - * Linux-HPI-Hybrid-Graphics is used by BIOS to enable dGPU to - * output video directly to external monitors on HP Inc. mobile - * workstations as Nvidia and AMD VGA drivers provide limited - * hybrid graphics supports. - */ - {"Linux-HPI-Hybrid-Graphics", true}, }; static u32 acpi_osi_handler(acpi_string interface, u32 supported) diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index d57cf8454b93e3be6a67546a3a654458c8e71ddf..c8385ef54c370947686eaedb4a6cf24ca1d84d65 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -312,76 +312,25 @@ struct acpi_handle_node { */ struct pci_dev *acpi_get_pci_dev(acpi_handle handle) { - int dev, fn; - unsigned long long adr; - acpi_status status; - acpi_handle phandle; - struct pci_bus *pbus; - struct pci_dev *pdev = NULL; - struct acpi_handle_node *node, *tmp; - struct acpi_pci_root *root; - LIST_HEAD(device_list); - - /* - * Walk up the ACPI CA namespace until we reach a PCI root bridge. - */ - phandle = handle; - while (!acpi_is_root_bridge(phandle)) { - node = kzalloc(sizeof(struct acpi_handle_node), GFP_KERNEL); - if (!node) - goto out; - - INIT_LIST_HEAD(&node->node); - node->handle = phandle; - list_add(&node->node, &device_list); - - status = acpi_get_parent(phandle, &phandle); - if (ACPI_FAILURE(status)) - goto out; - } - - root = acpi_pci_find_root(phandle); - if (!root) - goto out; + struct acpi_device *adev = acpi_fetch_acpi_dev(handle); + struct acpi_device_physical_node *pn; + struct pci_dev *pci_dev = NULL; - pbus = root->bus; - - /* - * Now, walk back down the PCI device tree until we return to our - * original handle. Assumes that everything between the PCI root - * bridge and the device we're looking for must be a P2P bridge. - */ - list_for_each_entry(node, &device_list, node) { - acpi_handle hnd = node->handle; - status = acpi_evaluate_integer(hnd, "_ADR", NULL, &adr); - if (ACPI_FAILURE(status)) - goto out; - dev = (adr >> 16) & 0xffff; - fn = adr & 0xffff; - - pdev = pci_get_slot(pbus, PCI_DEVFN(dev, fn)); - if (!pdev || hnd == handle) - break; + if (!adev) + return NULL; - pbus = pdev->subordinate; - pci_dev_put(pdev); + mutex_lock(&adev->physical_node_lock); - /* - * This function may be called for a non-PCI device that has a - * PCI parent (eg. a disk under a PCI SATA controller). In that - * case pdev->subordinate will be NULL for the parent. - */ - if (!pbus) { - dev_dbg(&pdev->dev, "Not a PCI-to-PCI bridge\n"); - pdev = NULL; + list_for_each_entry(pn, &adev->physical_node_list, node) { + if (dev_is_pci(pn->dev)) { + pci_dev = to_pci_dev(pn->dev); break; } } -out: - list_for_each_entry_safe(node, tmp, &device_list, node) - kfree(node); - return pdev; + mutex_unlock(&adev->physical_node_lock); + + return pci_dev; } EXPORT_SYMBOL_GPL(acpi_get_pci_dev); diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 8c4a73a1351e8db15b480eaa8ff15a566f465e95..f2588aba8421e7c7057f8066cbb7778e312acbaa 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -944,13 +944,15 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle) return NULL; device = &resource->device; - acpi_init_device_object(device, handle, ACPI_BUS_TYPE_POWER); + acpi_init_device_object(device, handle, ACPI_BUS_TYPE_POWER, + acpi_release_power_resource); mutex_init(&resource->resource_lock); INIT_LIST_HEAD(&resource->list_node); INIT_LIST_HEAD(&resource->dependents); strcpy(acpi_device_name(device), ACPI_POWER_DEVICE_NAME); strcpy(acpi_device_class(device), ACPI_POWER_CLASS); device->power.state = ACPI_STATE_UNKNOWN; + device->flags.match_driver = true; /* Evaluate the object to get the system level and resource order. */ status = acpi_evaluate_object(handle, NULL, NULL, &buffer); @@ -967,8 +969,11 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle) pr_info("%s [%s]\n", acpi_device_name(device), acpi_device_bid(device)); - device->flags.match_driver = true; - result = acpi_device_add(device, acpi_release_power_resource); + result = acpi_tie_acpi_dev(device); + if (result) + goto err; + + result = acpi_device_add(device); if (result) goto err; diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 16a1663d02d460579e61c9e98ff2d1ca5d0ba0c8..acfabfe07c4fa5f16c7b1b0df90bb3f19b98189a 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -531,10 +531,27 @@ static void wait_for_freeze(void) /* No delay is needed if we are in guest */ if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) return; + /* + * Modern (>=Nehalem) Intel systems use ACPI via intel_idle, + * not this code. Assume that any Intel systems using this + * are ancient and may need the dummy wait. This also assumes + * that the motivating chipset issue was Intel-only. + */ + if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) + return; #endif - /* Dummy wait op - must do something useless after P_LVL2 read - because chipsets cannot guarantee that STPCLK# signal - gets asserted in time to freeze execution properly. */ + /* + * Dummy wait op - must do something useless after P_LVL2 read + * because chipsets cannot guarantee that STPCLK# signal gets + * asserted in time to freeze execution properly + * + * This workaround has been in place since the original ACPI + * implementation was merged, circa 2002. + * + * If a profile is pointing to this instruction, please first + * consider moving your system to a more modern idle + * mechanism. + */ inl(acpi_gbl_FADT.xpm_timer_block.address); } @@ -787,7 +804,7 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr) state = &drv->states[count]; snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i); - strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); + strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); state->exit_latency = cx->latency; state->target_residency = cx->latency * latency_factor; state->enter = acpi_idle_enter; @@ -956,7 +973,7 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, obj = pkg_elem + 9; if (obj->type == ACPI_TYPE_STRING) - strlcpy(lpi_state->desc, obj->string.pointer, + strscpy(lpi_state->desc, obj->string.pointer, ACPI_CX_DESC_LEN); lpi_state->index = state_idx; @@ -1022,7 +1039,7 @@ static bool combine_lpi_states(struct acpi_lpi_state *local, result->arch_flags = parent->arch_flags; result->index = parent->index; - strlcpy(result->desc, local->desc, ACPI_CX_DESC_LEN); + strscpy(result->desc, local->desc, ACPI_CX_DESC_LEN); strlcat(result->desc, "+", ACPI_CX_DESC_LEN); strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN); return true; @@ -1196,7 +1213,7 @@ static int acpi_processor_setup_lpi_states(struct acpi_processor *pr) state = &drv->states[i]; snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i); - strlcpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN); + strscpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN); state->exit_latency = lpi->wake_latency; state->target_residency = lpi->min_residency; if (lpi->arch_flags) diff --git a/drivers/acpi/processor_thermal.c b/drivers/acpi/processor_thermal.c index db6ac540e924a75f51b41ba7fd35922d0dfa1f58..e534fd49a67e50877cc9ecd2672720fceed467f5 100644 --- a/drivers/acpi/processor_thermal.c +++ b/drivers/acpi/processor_thermal.c @@ -151,7 +151,7 @@ void acpi_thermal_cpufreq_exit(struct cpufreq_policy *policy) unsigned int cpu; for_each_cpu(cpu, policy->related_cpus) { - struct acpi_processor *pr = per_cpu(processors, policy->cpu); + struct acpi_processor *pr = per_cpu(processors, cpu); if (pr) freq_qos_remove_request(&pr->thermal_req); diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 7b3ad8ed2f4e6c46ade0eb87ffa7754a0694c0eb..b8d9eb9a433edd2c250dd68c6cdc2955ac03d577 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -304,8 +304,10 @@ static void acpi_init_of_compatible(struct acpi_device *adev) ret = acpi_dev_get_property(adev, "compatible", ACPI_TYPE_STRING, &of_compatible); if (ret) { - if (adev->parent - && adev->parent->flags.of_compatible_ok) + struct acpi_device *parent; + + parent = acpi_dev_parent(adev); + if (parent && parent->flags.of_compatible_ok) goto out; return; @@ -370,7 +372,7 @@ static bool acpi_tie_nondev_subnodes(struct acpi_device_data *data) bool ret; status = acpi_attach_data(dn->handle, acpi_nondev_subnode_tag, dn); - if (ACPI_FAILURE(status)) { + if (ACPI_FAILURE(status) && status != AE_ALREADY_EXISTS) { acpi_handle_err(dn->handle, "Can't tag data node\n"); return false; } @@ -1043,11 +1045,10 @@ static int acpi_data_prop_read_single(const struct acpi_device_data *data, break; \ } \ if (__items[i].integer.value > _Generic(__val, \ - u8: U8_MAX, \ - u16: U16_MAX, \ - u32: U32_MAX, \ - u64: U64_MAX, \ - default: 0U)) { \ + u8 *: U8_MAX, \ + u16 *: U16_MAX, \ + u32 *: U32_MAX, \ + u64 *: U64_MAX)) { \ ret = -EOVERFLOW; \ break; \ } \ @@ -1268,10 +1269,11 @@ acpi_node_get_parent(const struct fwnode_handle *fwnode) return to_acpi_data_node(fwnode)->parent; } if (is_acpi_device_node(fwnode)) { - struct device *dev = to_acpi_device_node(fwnode)->dev.parent; + struct acpi_device *parent; - if (dev) - return acpi_fwnode_handle(to_acpi_device(dev)); + parent = acpi_dev_parent(to_acpi_device_node(fwnode)); + if (parent) + return acpi_fwnode_handle(parent); } return NULL; diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 510cdec375c4d88e9bced03c7a2b3d55f91dd895..6f9489edfb4ee5446d3b2ccc4ac0ba5e5c8f1dbf 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -336,8 +336,9 @@ EXPORT_SYMBOL_GPL(acpi_dev_resource_ext_address_space); * @triggering: Triggering type as provided by ACPI. * @polarity: Interrupt polarity as provided by ACPI. * @shareable: Whether or not the interrupt is shareable. + * @wake_capable: Wake capability as provided by ACPI. */ -unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable) +unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable, u8 wake_capable) { unsigned long flags; @@ -351,6 +352,9 @@ unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable) if (shareable == ACPI_SHARED) flags |= IORESOURCE_IRQ_SHAREABLE; + if (wake_capable == ACPI_WAKE_CAPABLE) + flags |= IORESOURCE_IRQ_WAKECAPABLE; + return flags | IORESOURCE_IRQ; } EXPORT_SYMBOL_GPL(acpi_dev_irq_flags); @@ -399,6 +403,31 @@ static const struct dmi_system_id medion_laptop[] = { { } }; +static const struct dmi_system_id asus_laptop[] = { + { + .ident = "Asus Vivobook K3402ZA", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "K3402ZA"), + }, + }, + { + .ident = "Asus Vivobook K3502ZA", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "K3502ZA"), + }, + }, + { + .ident = "Asus Vivobook S5402ZA", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "S5402ZA"), + }, + }, + { } +}; + struct irq_override_cmp { const struct dmi_system_id *system; unsigned char irq; @@ -409,6 +438,7 @@ struct irq_override_cmp { static const struct irq_override_cmp skip_override_table[] = { { medion_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 }, + { asus_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 }, }; static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity, @@ -442,7 +472,7 @@ static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity, static void acpi_dev_get_irqresource(struct resource *res, u32 gsi, u8 triggering, u8 polarity, u8 shareable, - bool check_override) + u8 wake_capable, bool check_override) { int irq, p, t; @@ -475,7 +505,7 @@ static void acpi_dev_get_irqresource(struct resource *res, u32 gsi, } } - res->flags = acpi_dev_irq_flags(triggering, polarity, shareable); + res->flags = acpi_dev_irq_flags(triggering, polarity, shareable, wake_capable); irq = acpi_register_gsi(NULL, gsi, triggering, polarity); if (irq >= 0) { res->start = irq; @@ -523,7 +553,8 @@ bool acpi_dev_resource_interrupt(struct acpi_resource *ares, int index, } acpi_dev_get_irqresource(res, irq->interrupts[index], irq->triggering, irq->polarity, - irq->shareable, true); + irq->shareable, irq->wake_capable, + true); break; case ACPI_RESOURCE_TYPE_EXTENDED_IRQ: ext_irq = &ares->data.extended_irq; @@ -534,7 +565,8 @@ bool acpi_dev_resource_interrupt(struct acpi_resource *ares, int index, if (is_gsi(ext_irq)) acpi_dev_get_irqresource(res, ext_irq->interrupts[index], ext_irq->triggering, ext_irq->polarity, - ext_irq->shareable, false); + ext_irq->shareable, ext_irq->wake_capable, + false); else irqresource_disabled(res, 0); break; @@ -690,6 +722,9 @@ static int is_memory(struct acpi_resource *ares, void *not_used) memset(&win, 0, sizeof(win)); + if (acpi_dev_filter_resource_type(ares, IORESOURCE_MEM)) + return 1; + return !(acpi_dev_resource_memory(ares, res) || acpi_dev_resource_address_space(ares, &win) || acpi_dev_resource_ext_address_space(ares, &win)); @@ -718,6 +753,23 @@ int acpi_dev_get_dma_resources(struct acpi_device *adev, struct list_head *list) } EXPORT_SYMBOL_GPL(acpi_dev_get_dma_resources); +/** + * acpi_dev_get_memory_resources - Get current memory resources of a device. + * @adev: ACPI device node to get the resources for. + * @list: Head of the resultant list of resources (must be empty). + * + * This is a helper function that locates all memory type resources of @adev + * with acpi_dev_get_resources(). + * + * The number of resources in the output list is returned on success, an error + * code reflecting the error condition is returned otherwise. + */ +int acpi_dev_get_memory_resources(struct acpi_device *adev, struct list_head *list) +{ + return acpi_dev_get_resources(adev, list, is_memory, NULL); +} +EXPORT_SYMBOL_GPL(acpi_dev_get_memory_resources); + /** * acpi_dev_filter_resource_type - Filter ACPI resource according to resource * types diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c index 4938010fcac78d821f828434a44e84389eff4539..e6a01a8df1b8150015f0f1931d338d72cc315183 100644 --- a/drivers/acpi/sbs.c +++ b/drivers/acpi/sbs.c @@ -632,7 +632,7 @@ static int acpi_sbs_add(struct acpi_device *device) mutex_init(&sbs->lock); - sbs->hc = acpi_driver_data(device->parent); + sbs->hc = acpi_driver_data(acpi_dev_parent(device)); sbs->device = device; strcpy(acpi_device_name(device), ACPI_SBS_DEVICE_NAME); strcpy(acpi_device_class(device), ACPI_SBS_CLASS); diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c index 7c62e149a7a16fa5287928511c082b8887cfcda9..340e0b61587e02c4e8218d31907f4b39876fe3a0 100644 --- a/drivers/acpi/sbshc.c +++ b/drivers/acpi/sbshc.c @@ -266,7 +266,7 @@ static int acpi_smbus_hc_add(struct acpi_device *device) mutex_init(&hc->lock); init_waitqueue_head(&hc->wait); - hc->ec = acpi_driver_data(device->parent); + hc->ec = acpi_driver_data(acpi_dev_parent(device)); hc->offset = (val >> 8) & 0xff; hc->query_bit = val & 0xff; device->driver_data = hc; diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 42cec8120f18e513d60bed04c3d1b1a389f26864..558664d169fcc3e10acf1f61f5f24134c97471a4 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "internal.h" @@ -29,8 +30,6 @@ extern struct acpi_device *acpi_root; #define ACPI_BUS_HID "LNXSYBUS" #define ACPI_BUS_DEVICE_NAME "System Bus" -#define ACPI_IS_ROOT_DEVICE(device) (!(device)->parent) - #define INVALID_ACPI_HANDLE ((acpi_handle)empty_zero_page) static const char *dummy_hid = "device"; @@ -429,7 +428,7 @@ void acpi_device_hotplug(struct acpi_device *adev, u32 src) acpi_evaluate_ost(adev->handle, src, ost_code, NULL); out: - acpi_bus_put_acpi_device(adev); + acpi_put_acpi_dev(adev); mutex_unlock(&acpi_scan_lock); unlock_device_hotplug(); } @@ -599,11 +598,22 @@ static void get_acpi_device(void *dev) acpi_dev_get(dev); } -struct acpi_device *acpi_bus_get_acpi_device(acpi_handle handle) +/** + * acpi_get_acpi_dev - Retrieve ACPI device object and reference count it. + * @handle: ACPI handle associated with the requested ACPI device object. + * + * Return a pointer to the ACPI device object associated with @handle and bump + * up that object's reference counter (under the ACPI Namespace lock), if + * present, or return NULL otherwise. + * + * The ACPI device object reference acquired by this function needs to be + * dropped via acpi_dev_put(). + */ +struct acpi_device *acpi_get_acpi_dev(acpi_handle handle) { return handle_to_device(handle, get_acpi_device); } -EXPORT_SYMBOL_GPL(acpi_bus_get_acpi_device); +EXPORT_SYMBOL_GPL(acpi_get_acpi_dev); static struct acpi_device_bus_id *acpi_device_bus_id_match(const char *dev_id) { @@ -632,7 +642,7 @@ static int acpi_device_set_name(struct acpi_device *device, return 0; } -static int acpi_tie_acpi_dev(struct acpi_device *adev) +int acpi_tie_acpi_dev(struct acpi_device *adev) { acpi_handle handle = adev->handle; acpi_status status; @@ -662,8 +672,7 @@ static void acpi_store_pld_crc(struct acpi_device *adev) ACPI_FREE(pld); } -static int __acpi_device_add(struct acpi_device *device, - void (*release)(struct device *)) +int acpi_device_add(struct acpi_device *device) { struct acpi_device_bus_id *acpi_device_bus_id; int result; @@ -719,11 +728,6 @@ static int __acpi_device_add(struct acpi_device *device, mutex_unlock(&acpi_device_lock); - if (device->parent) - device->dev.parent = &device->parent->dev; - - device->dev.bus = &acpi_bus_type; - device->dev.release = release; result = device_add(&device->dev); if (result) { dev_err(&device->dev, "Error registering device\n"); @@ -750,17 +754,6 @@ err_unlock: return result; } -int acpi_device_add(struct acpi_device *adev, void (*release)(struct device *)) -{ - int ret; - - ret = acpi_tie_acpi_dev(adev); - if (ret) - return ret; - - return __acpi_device_add(adev, release); -} - /* -------------------------------------------------------------------------- Device Enumeration -------------------------------------------------------------------------- */ @@ -805,10 +798,9 @@ static const char * const acpi_honor_dep_ids[] = { NULL }; -static struct acpi_device *acpi_bus_get_parent(acpi_handle handle) +static struct acpi_device *acpi_find_parent_acpi_dev(acpi_handle handle) { - struct acpi_device *device; - acpi_status status; + struct acpi_device *adev; /* * Fixed hardware devices do not appear in the namespace and do not @@ -819,13 +811,18 @@ static struct acpi_device *acpi_bus_get_parent(acpi_handle handle) return acpi_root; do { + acpi_status status; + status = acpi_get_parent(handle, &handle); - if (ACPI_FAILURE(status)) - return status == AE_NULL_ENTRY ? NULL : acpi_root; + if (ACPI_FAILURE(status)) { + if (status != AE_NULL_ENTRY) + return acpi_root; - device = acpi_fetch_acpi_dev(handle); - } while (!device); - return device; + return NULL; + } + adev = acpi_fetch_acpi_dev(handle); + } while (!adev); + return adev; } acpi_status @@ -1112,7 +1109,7 @@ static void acpi_device_get_busid(struct acpi_device *device) * The device's Bus ID is simply the object name. * TBD: Shouldn't this value be unique (within the ACPI namespace)? */ - if (ACPI_IS_ROOT_DEVICE(device)) { + if (!acpi_dev_parent(device)) { strcpy(device->pnp.bus_id, "ACPI"); return; } @@ -1467,25 +1464,21 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev) * acpi_dma_get_range() - Get device DMA parameters. * * @dev: device to configure - * @dma_addr: pointer device DMA address result - * @offset: pointer to the DMA offset result - * @size: pointer to DMA range size result + * @map: pointer to DMA ranges result * - * Evaluate DMA regions and return respectively DMA region start, offset - * and size in dma_addr, offset and size on parsing success; it does not - * update the passed in values on failure. + * Evaluate DMA regions and return pointer to DMA regions on + * parsing success; it does not update the passed in values on failure. * * Return 0 on success, < 0 on failure. */ -int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, - u64 *size) +int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map) { struct acpi_device *adev; LIST_HEAD(list); struct resource_entry *rentry; int ret; struct device *dma_dev = dev; - u64 len, dma_start = U64_MAX, dma_end = 0, dma_offset = 0; + struct bus_dma_region *r; /* * Walk the device tree chasing an ACPI companion with a _DMA @@ -1510,31 +1503,28 @@ int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, ret = acpi_dev_get_dma_resources(adev, &list); if (ret > 0) { + r = kcalloc(ret + 1, sizeof(*r), GFP_KERNEL); + if (!r) { + ret = -ENOMEM; + goto out; + } + list_for_each_entry(rentry, &list, node) { - if (dma_offset && rentry->offset != dma_offset) { + if (rentry->res->start >= rentry->res->end) { + kfree(r); ret = -EINVAL; - dev_warn(dma_dev, "Can't handle multiple windows with different offsets\n"); + dev_dbg(dma_dev, "Invalid DMA regions configuration\n"); goto out; } - dma_offset = rentry->offset; - /* Take lower and upper limits */ - if (rentry->res->start < dma_start) - dma_start = rentry->res->start; - if (rentry->res->end > dma_end) - dma_end = rentry->res->end; - } - - if (dma_start >= dma_end) { - ret = -EINVAL; - dev_dbg(dma_dev, "Invalid DMA regions configuration\n"); - goto out; + r->cpu_start = rentry->res->start; + r->dma_start = rentry->res->start - rentry->offset; + r->size = resource_size(rentry->res); + r->offset = rentry->offset; + r++; } - *dma_addr = dma_start - dma_offset; - len = dma_end - dma_start; - *size = max(len, len + 1); - *offset = dma_offset; + *map = r; } out: acpi_dev_free_resource_list(&list); @@ -1624,20 +1614,19 @@ int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, const u32 *input_id) { const struct iommu_ops *iommu; - u64 dma_addr = 0, size = 0; if (attr == DEV_DMA_NOT_SUPPORTED) { set_dma_ops(dev, &dma_dummy_ops); return 0; } - acpi_arch_dma_setup(dev, &dma_addr, &size); + acpi_arch_dma_setup(dev); iommu = acpi_iommu_configure_id(dev, input_id); if (PTR_ERR(iommu) == -EPROBE_DEFER) return -EPROBE_DEFER; - arch_setup_dma_ops(dev, dma_addr, size, + arch_setup_dma_ops(dev, 0, U64_MAX, iommu, attr == DEV_DMA_COHERENT); return 0; @@ -1648,7 +1637,7 @@ static void acpi_init_coherency(struct acpi_device *adev) { unsigned long long cca = 0; acpi_status status; - struct acpi_device *parent = adev->parent; + struct acpi_device *parent = acpi_dev_parent(adev); if (parent && parent->flags.cca_seen) { /* @@ -1692,7 +1681,7 @@ static int acpi_check_serial_bus_slave(struct acpi_resource *ares, void *data) static bool acpi_is_indirect_io_slave(struct acpi_device *device) { - struct acpi_device *parent = device->parent; + struct acpi_device *parent = acpi_dev_parent(device); static const struct acpi_device_id indirect_io_hosts[] = { {"HISI0191", 0}, {} @@ -1762,12 +1751,16 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device) } void acpi_init_device_object(struct acpi_device *device, acpi_handle handle, - int type) + int type, void (*release)(struct device *)) { + struct acpi_device *parent = acpi_find_parent_acpi_dev(handle); + INIT_LIST_HEAD(&device->pnp.ids); device->device_type = type; device->handle = handle; - device->parent = acpi_bus_get_parent(handle); + device->dev.parent = parent ? &parent->dev : NULL; + device->dev.release = release; + device->dev.bus = &acpi_bus_type; fwnode_init(&device->fwnode, &acpi_device_fwnode_ops); acpi_set_device_status(device, ACPI_STA_DEFAULT); acpi_device_get_busid(device); @@ -1821,7 +1814,7 @@ static int acpi_add_single_object(struct acpi_device **child, if (!device) return -ENOMEM; - acpi_init_device_object(device, handle, type); + acpi_init_device_object(device, handle, type, acpi_device_release); /* * Getting the status is delayed till here so that we can call * acpi_bus_get_status() and use its quirk handling. Note that @@ -1851,7 +1844,7 @@ static int acpi_add_single_object(struct acpi_device **child, mutex_unlock(&acpi_dep_list_lock); if (!result) - result = __acpi_device_add(device, acpi_device_release); + result = acpi_device_add(device); if (result) { acpi_device_release(&device->dev); @@ -1862,8 +1855,8 @@ static int acpi_add_single_object(struct acpi_device **child, acpi_device_add_finalize(device); acpi_handle_debug(handle, "Added as %s, parent %s\n", - dev_name(&device->dev), device->parent ? - dev_name(&device->parent->dev) : "(null)"); + dev_name(&device->dev), device->dev.parent ? + dev_name(device->dev.parent) : "(null)"); *child = device; return 0; @@ -2235,11 +2228,24 @@ ok: return 0; } -static int acpi_dev_get_first_consumer_dev_cb(struct acpi_dep_data *dep, void *data) +static int acpi_dev_get_next_consumer_dev_cb(struct acpi_dep_data *dep, void *data) { - struct acpi_device *adev; + struct acpi_device **adev_p = data; + struct acpi_device *adev = *adev_p; - adev = acpi_bus_get_acpi_device(dep->consumer); + /* + * If we're passed a 'previous' consumer device then we need to skip + * any consumers until we meet the previous one, and then NULL @data + * so the next one can be returned. + */ + if (adev) { + if (dep->consumer == adev->handle) + *adev_p = NULL; + + return 0; + } + + adev = acpi_get_acpi_dev(dep->consumer); if (adev) { *(struct acpi_device **)data = adev; return 1; @@ -2292,7 +2298,7 @@ static bool acpi_scan_clear_dep_queue(struct acpi_device *adev) static int acpi_scan_clear_dep(struct acpi_dep_data *dep, void *data) { - struct acpi_device *adev = acpi_bus_get_acpi_device(dep->consumer); + struct acpi_device *adev = acpi_get_acpi_dev(dep->consumer); if (adev) { adev->dep_unmet--; @@ -2368,25 +2374,32 @@ bool acpi_dev_ready_for_enumeration(const struct acpi_device *device) EXPORT_SYMBOL_GPL(acpi_dev_ready_for_enumeration); /** - * acpi_dev_get_first_consumer_dev - Return ACPI device dependent on @supplier + * acpi_dev_get_next_consumer_dev - Return the next adev dependent on @supplier * @supplier: Pointer to the dependee device + * @start: Pointer to the current dependent device * - * Returns the first &struct acpi_device which declares itself dependent on + * Returns the next &struct acpi_device which declares itself dependent on * @supplier via the _DEP buffer, parsed from the acpi_dep_list. * - * The caller is responsible for putting the reference to adev when it is no - * longer needed. + * If the returned adev is not passed as @start to this function, the caller is + * responsible for putting the reference to adev when it is no longer needed. */ -struct acpi_device *acpi_dev_get_first_consumer_dev(struct acpi_device *supplier) +struct acpi_device *acpi_dev_get_next_consumer_dev(struct acpi_device *supplier, + struct acpi_device *start) { - struct acpi_device *adev = NULL; + struct acpi_device *adev = start; acpi_walk_dep_device_list(supplier->handle, - acpi_dev_get_first_consumer_dev_cb, &adev); + acpi_dev_get_next_consumer_dev_cb, &adev); + + acpi_dev_put(start); + + if (adev == start) + return NULL; return adev; } -EXPORT_SYMBOL_GPL(acpi_dev_get_first_consumer_dev); +EXPORT_SYMBOL_GPL(acpi_dev_get_next_consumer_dev); /** * acpi_bus_scan - Add ACPI device node objects in a given namespace scope. diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index ad4b2987b3d6ec0bc71f4d918992576ae28edce4..0b557c0d405ef1945024374556ddaf7e831ea4a4 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -1088,6 +1088,14 @@ int __init acpi_sleep_init(void) register_sys_off_handler(SYS_OFF_MODE_POWER_OFF, SYS_OFF_PRIO_FIRMWARE, acpi_power_off, NULL); + + /* + * Windows uses S5 for reboot, so some BIOSes depend on it to + * perform proper reboot. + */ + register_sys_off_handler(SYS_OFF_MODE_RESTART_PREPARE, + SYS_OFF_PRIO_FIRMWARE, + acpi_power_off_prepare, NULL); } else { acpi_no_s5 = true; } diff --git a/drivers/acpi/sleep.h b/drivers/acpi/sleep.h index 7fe41ee489d6123c2fec04a6672ef25f1c04cbce..d960a238be4e51de7f7fdb34906e1835360ccc22 100644 --- a/drivers/acpi/sleep.h +++ b/drivers/acpi/sleep.h @@ -18,6 +18,7 @@ static inline acpi_status acpi_set_waking_vector(u32 wakeup_address) extern int acpi_s2idle_begin(void); extern int acpi_s2idle_prepare(void); extern int acpi_s2idle_prepare_late(void); +extern void acpi_s2idle_check(void); extern bool acpi_s2idle_wake(void); extern void acpi_s2idle_restore_early(void); extern void acpi_s2idle_restore(void); diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 539660ef93c7e6c0dd862341e0093a60ef26a411..40b07057983e04565d28c00dc022b97e0c847e55 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -158,7 +158,7 @@ struct acpi_thermal_flags { }; struct acpi_thermal { - struct acpi_device * device; + struct acpi_device *device; acpi_bus_id name; unsigned long temperature; unsigned long last_temperature; @@ -262,7 +262,7 @@ do { \ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) { - acpi_status status = AE_OK; + acpi_status status; unsigned long long tmp; struct acpi_handle_list devices; int valid = 0; @@ -270,8 +270,7 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) /* Critical Shutdown */ if (flag & ACPI_TRIPS_CRITICAL) { - status = acpi_evaluate_integer(tz->device->handle, - "_CRT", NULL, &tmp); + status = acpi_evaluate_integer(tz->device->handle, "_CRT", NULL, &tmp); tz->trips.critical.temperature = tmp; /* * Treat freezing temperatures as invalid as well; some @@ -284,8 +283,7 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) acpi_handle_debug(tz->device->handle, "No critical threshold\n"); } else if (tmp <= 2732) { - pr_info(FW_BUG "Invalid critical threshold (%llu)\n", - tmp); + pr_info(FW_BUG "Invalid critical threshold (%llu)\n", tmp); tz->trips.critical.flags.valid = 0; } else { tz->trips.critical.flags.valid = 1; @@ -312,8 +310,7 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) /* Critical Sleep (optional) */ if (flag & ACPI_TRIPS_HOT) { - status = acpi_evaluate_integer(tz->device->handle, - "_HOT", NULL, &tmp); + status = acpi_evaluate_integer(tz->device->handle, "_HOT", NULL, &tmp); if (ACPI_FAILURE(status)) { tz->trips.hot.flags.valid = 0; acpi_handle_debug(tz->device->handle, @@ -329,7 +326,7 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) /* Passive (optional) */ if (((flag & ACPI_TRIPS_PASSIVE) && tz->trips.passive.flags.valid) || - (flag == ACPI_TRIPS_INIT)) { + flag == ACPI_TRIPS_INIT) { valid = tz->trips.passive.flags.valid; if (psv == -1) { status = AE_SUPPORT; @@ -338,32 +335,31 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) status = AE_OK; } else { status = acpi_evaluate_integer(tz->device->handle, - "_PSV", NULL, &tmp); + "_PSV", NULL, &tmp); } - if (ACPI_FAILURE(status)) + if (ACPI_FAILURE(status)) { tz->trips.passive.flags.valid = 0; - else { + } else { tz->trips.passive.temperature = tmp; tz->trips.passive.flags.valid = 1; if (flag == ACPI_TRIPS_INIT) { - status = acpi_evaluate_integer( - tz->device->handle, "_TC1", - NULL, &tmp); + status = acpi_evaluate_integer(tz->device->handle, + "_TC1", NULL, &tmp); if (ACPI_FAILURE(status)) tz->trips.passive.flags.valid = 0; else tz->trips.passive.tc1 = tmp; - status = acpi_evaluate_integer( - tz->device->handle, "_TC2", - NULL, &tmp); + + status = acpi_evaluate_integer(tz->device->handle, + "_TC2", NULL, &tmp); if (ACPI_FAILURE(status)) tz->trips.passive.flags.valid = 0; else tz->trips.passive.tc2 = tmp; - status = acpi_evaluate_integer( - tz->device->handle, "_TSP", - NULL, &tmp); + + status = acpi_evaluate_integer(tz->device->handle, + "_TSP", NULL, &tmp); if (ACPI_FAILURE(status)) tz->trips.passive.flags.valid = 0; else @@ -374,25 +370,25 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) if ((flag & ACPI_TRIPS_DEVICES) && tz->trips.passive.flags.valid) { memset(&devices, 0, sizeof(struct acpi_handle_list)); status = acpi_evaluate_reference(tz->device->handle, "_PSL", - NULL, &devices); + NULL, &devices); if (ACPI_FAILURE(status)) { acpi_handle_info(tz->device->handle, "Invalid passive threshold\n"); tz->trips.passive.flags.valid = 0; - } - else + } else { tz->trips.passive.flags.valid = 1; + } if (memcmp(&tz->trips.passive.devices, &devices, - sizeof(struct acpi_handle_list))) { + sizeof(struct acpi_handle_list))) { memcpy(&tz->trips.passive.devices, &devices, - sizeof(struct acpi_handle_list)); + sizeof(struct acpi_handle_list)); ACPI_THERMAL_TRIPS_EXCEPTION(flag, tz, "device"); } } if ((flag & ACPI_TRIPS_PASSIVE) || (flag & ACPI_TRIPS_DEVICES)) { if (valid != tz->trips.passive.flags.valid) - ACPI_THERMAL_TRIPS_EXCEPTION(flag, tz, "state"); + ACPI_THERMAL_TRIPS_EXCEPTION(flag, tz, "state"); } /* Active (optional) */ @@ -403,29 +399,31 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) if (act == -1) break; /* disable all active trip points */ - if ((flag == ACPI_TRIPS_INIT) || ((flag & ACPI_TRIPS_ACTIVE) && - tz->trips.active[i].flags.valid)) { + if (flag == ACPI_TRIPS_INIT || ((flag & ACPI_TRIPS_ACTIVE) && + tz->trips.active[i].flags.valid)) { status = acpi_evaluate_integer(tz->device->handle, - name, NULL, &tmp); + name, NULL, &tmp); if (ACPI_FAILURE(status)) { tz->trips.active[i].flags.valid = 0; if (i == 0) break; + if (act <= 0) break; + if (i == 1) - tz->trips.active[0].temperature = - celsius_to_deci_kelvin(act); + tz->trips.active[0].temperature = celsius_to_deci_kelvin(act); else /* * Don't allow override higher than * the next higher trip point */ - tz->trips.active[i - 1].temperature = - (tz->trips.active[i - 2].temperature < + tz->trips.active[i-1].temperature = + (tz->trips.active[i-2].temperature < celsius_to_deci_kelvin(act) ? - tz->trips.active[i - 2].temperature : + tz->trips.active[i-2].temperature : celsius_to_deci_kelvin(act)); + break; } else { tz->trips.active[i].temperature = tmp; @@ -434,22 +432,22 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) } name[2] = 'L'; - if ((flag & ACPI_TRIPS_DEVICES) && tz->trips.active[i].flags.valid ) { + if ((flag & ACPI_TRIPS_DEVICES) && tz->trips.active[i].flags.valid) { memset(&devices, 0, sizeof(struct acpi_handle_list)); status = acpi_evaluate_reference(tz->device->handle, - name, NULL, &devices); + name, NULL, &devices); if (ACPI_FAILURE(status)) { acpi_handle_info(tz->device->handle, "Invalid active%d threshold\n", i); tz->trips.active[i].flags.valid = 0; - } - else + } else { tz->trips.active[i].flags.valid = 1; + } if (memcmp(&tz->trips.active[i].devices, &devices, - sizeof(struct acpi_handle_list))) { + sizeof(struct acpi_handle_list))) { memcpy(&tz->trips.active[i].devices, &devices, - sizeof(struct acpi_handle_list)); + sizeof(struct acpi_handle_list)); ACPI_THERMAL_TRIPS_EXCEPTION(flag, tz, "device"); } } @@ -464,9 +462,9 @@ static int acpi_thermal_trips_update(struct acpi_thermal *tz, int flag) if (flag & ACPI_TRIPS_DEVICES) { memset(&devices, 0, sizeof(devices)); status = acpi_evaluate_reference(tz->device->handle, "_TZD", - NULL, &devices); - if (ACPI_SUCCESS(status) - && memcmp(&tz->devices, &devices, sizeof(devices))) { + NULL, &devices); + if (ACPI_SUCCESS(status) && + memcmp(&tz->devices, &devices, sizeof(devices))) { tz->devices = devices; ACPI_THERMAL_TRIPS_EXCEPTION(flag, tz, "device"); } @@ -548,8 +546,7 @@ static int thermal_get_trip_type(struct thermal_zone_device *thermal, trip--; } - for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE && - tz->trips.active[i].flags.valid; i++) { + for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE && tz->trips.active[i].flags.valid; i++) { if (!trip) { *type = THERMAL_TRIP_ACTIVE; return 0; @@ -572,8 +569,8 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal, if (tz->trips.critical.flags.valid) { if (!trip) { *temp = deci_kelvin_to_millicelsius_with_offset( - tz->trips.critical.temperature, - tz->kelvin_offset); + tz->trips.critical.temperature, + tz->kelvin_offset); return 0; } trip--; @@ -582,8 +579,8 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal, if (tz->trips.hot.flags.valid) { if (!trip) { *temp = deci_kelvin_to_millicelsius_with_offset( - tz->trips.hot.temperature, - tz->kelvin_offset); + tz->trips.hot.temperature, + tz->kelvin_offset); return 0; } trip--; @@ -592,8 +589,8 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal, if (tz->trips.passive.flags.valid) { if (!trip) { *temp = deci_kelvin_to_millicelsius_with_offset( - tz->trips.passive.temperature, - tz->kelvin_offset); + tz->trips.passive.temperature, + tz->kelvin_offset); return 0; } trip--; @@ -603,8 +600,8 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal, tz->trips.active[i].flags.valid; i++) { if (!trip) { *temp = deci_kelvin_to_millicelsius_with_offset( - tz->trips.active[i].temperature, - tz->kelvin_offset); + tz->trips.active[i].temperature, + tz->kelvin_offset); return 0; } trip--; @@ -620,15 +617,16 @@ static int thermal_get_crit_temp(struct thermal_zone_device *thermal, if (tz->trips.critical.flags.valid) { *temperature = deci_kelvin_to_millicelsius_with_offset( - tz->trips.critical.temperature, - tz->kelvin_offset); + tz->trips.critical.temperature, + tz->kelvin_offset); return 0; - } else - return -EINVAL; + } + + return -EINVAL; } static int thermal_get_trend(struct thermal_zone_device *thermal, - int trip, enum thermal_trend *trend) + int trip, enum thermal_trend *trend) { struct acpi_thermal *tz = thermal->devdata; enum thermal_trip_type type; @@ -657,9 +655,8 @@ static int thermal_get_trend(struct thermal_zone_device *thermal, * tz->temperature has already been updated by generic thermal layer, * before this callback being invoked */ - i = (tz->trips.passive.tc1 * (tz->temperature - tz->last_temperature)) - + (tz->trips.passive.tc2 - * (tz->temperature - tz->trips.passive.temperature)); + i = tz->trips.passive.tc1 * (tz->temperature - tz->last_temperature) + + tz->trips.passive.tc2 * (tz->temperature - tz->trips.passive.temperature); if (i > 0) *trend = THERMAL_TREND_RAISING; @@ -667,6 +664,7 @@ static int thermal_get_trend(struct thermal_zone_device *thermal, *trend = THERMAL_TREND_DROPPING; else *trend = THERMAL_TREND_STABLE; + return 0; } @@ -691,8 +689,8 @@ static void acpi_thermal_zone_device_critical(struct thermal_zone_device *therma } static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, - struct thermal_cooling_device *cdev, - bool bind) + struct thermal_cooling_device *cdev, + bool bind) { struct acpi_device *device = cdev->devdata; struct acpi_thermal *tz = thermal->devdata; @@ -711,22 +709,23 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, if (tz->trips.passive.flags.valid) { trip++; - for (i = 0; i < tz->trips.passive.devices.count; - i++) { + for (i = 0; i < tz->trips.passive.devices.count; i++) { handle = tz->trips.passive.devices.handles[i]; dev = acpi_fetch_acpi_dev(handle); if (dev != device) continue; + if (bind) - result = - thermal_zone_bind_cooling_device - (thermal, trip, cdev, - THERMAL_NO_LIMIT, THERMAL_NO_LIMIT, - THERMAL_WEIGHT_DEFAULT); + result = thermal_zone_bind_cooling_device( + thermal, trip, cdev, + THERMAL_NO_LIMIT, + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); else result = - thermal_zone_unbind_cooling_device - (thermal, trip, cdev); + thermal_zone_unbind_cooling_device( + thermal, trip, cdev); + if (result) goto failed; } @@ -735,22 +734,24 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE; i++) { if (!tz->trips.active[i].flags.valid) break; + trip++; - for (j = 0; - j < tz->trips.active[i].devices.count; - j++) { + for (j = 0; j < tz->trips.active[i].devices.count; j++) { handle = tz->trips.active[i].devices.handles[j]; dev = acpi_fetch_acpi_dev(handle); if (dev != device) continue; + if (bind) - result = thermal_zone_bind_cooling_device - (thermal, trip, cdev, - THERMAL_NO_LIMIT, THERMAL_NO_LIMIT, - THERMAL_WEIGHT_DEFAULT); + result = thermal_zone_bind_cooling_device( + thermal, trip, cdev, + THERMAL_NO_LIMIT, + THERMAL_NO_LIMIT, + THERMAL_WEIGHT_DEFAULT); else - result = thermal_zone_unbind_cooling_device - (thermal, trip, cdev); + result = thermal_zone_unbind_cooling_device( + thermal, trip, cdev); + if (result) goto failed; } @@ -762,14 +763,14 @@ failed: static int acpi_thermal_bind_cooling_device(struct thermal_zone_device *thermal, - struct thermal_cooling_device *cdev) + struct thermal_cooling_device *cdev) { return acpi_thermal_cooling_device_cb(thermal, cdev, true); } static int acpi_thermal_unbind_cooling_device(struct thermal_zone_device *thermal, - struct thermal_cooling_device *cdev) + struct thermal_cooling_device *cdev) { return acpi_thermal_cooling_device_cb(thermal, cdev, false); } @@ -802,20 +803,20 @@ static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz) if (tz->trips.passive.flags.valid) trips++; - for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE && - tz->trips.active[i].flags.valid; i++, trips++); + for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE && tz->trips.active[i].flags.valid; + i++, trips++); if (tz->trips.passive.flags.valid) - tz->thermal_zone = - thermal_zone_device_register("acpitz", trips, 0, tz, - &acpi_thermal_zone_ops, NULL, - tz->trips.passive.tsp*100, - tz->polling_frequency*100); + tz->thermal_zone = thermal_zone_device_register("acpitz", trips, 0, tz, + &acpi_thermal_zone_ops, NULL, + tz->trips.passive.tsp * 100, + tz->polling_frequency * 100); else tz->thermal_zone = thermal_zone_device_register("acpitz", trips, 0, tz, - &acpi_thermal_zone_ops, NULL, - 0, tz->polling_frequency*100); + &acpi_thermal_zone_ops, NULL, + 0, tz->polling_frequency * 100); + if (IS_ERR(tz->thermal_zone)) return -ENODEV; @@ -881,7 +882,6 @@ static void acpi_thermal_notify(struct acpi_device *device, u32 event) { struct acpi_thermal *tz = acpi_driver_data(device); - if (!tz) return; @@ -893,13 +893,13 @@ static void acpi_thermal_notify(struct acpi_device *device, u32 event) acpi_thermal_trips_update(tz, ACPI_TRIPS_REFRESH_THRESHOLDS); acpi_queue_thermal_check(tz); acpi_bus_generate_netlink_event(device->pnp.device_class, - dev_name(&device->dev), event, 0); + dev_name(&device->dev), event, 0); break; case ACPI_THERMAL_NOTIFY_DEVICES: acpi_thermal_trips_update(tz, ACPI_TRIPS_REFRESH_DEVICES); acpi_queue_thermal_check(tz); acpi_bus_generate_netlink_event(device->pnp.device_class, - dev_name(&device->dev), event, 0); + dev_name(&device->dev), event, 0); break; default: acpi_handle_debug(device->handle, "Unsupported event [0x%x]\n", @@ -942,8 +942,7 @@ static void acpi_thermal_aml_dependency_fix(struct acpi_thermal *tz) static int acpi_thermal_get_info(struct acpi_thermal *tz) { - int result = 0; - + int result; if (!tz) return -EINVAL; @@ -1020,9 +1019,8 @@ static void acpi_thermal_check_fn(struct work_struct *work) static int acpi_thermal_add(struct acpi_device *device) { - int result = 0; - struct acpi_thermal *tz = NULL; - + struct acpi_thermal *tz; + int result; if (!device) return -EINVAL; @@ -1063,7 +1061,7 @@ end: static int acpi_thermal_remove(struct acpi_device *device) { - struct acpi_thermal *tz = NULL; + struct acpi_thermal *tz; if (!device || !acpi_driver_data(device)) return -EINVAL; @@ -1099,6 +1097,7 @@ static int acpi_thermal_resume(struct device *dev) for (i = 0; i < ACPI_THERMAL_MAX_ACTIVE; i++) { if (!tz->trips.active[i].flags.valid) break; + tz->trips.active[i].flags.enabled = 1; for (j = 0; j < tz->trips.active[i].devices.count; j++) { result = acpi_bus_update_power( @@ -1119,7 +1118,6 @@ static int acpi_thermal_resume(struct device *dev) #endif static int thermal_act(const struct dmi_system_id *d) { - if (act == 0) { pr_notice("%s detected: disabling all active thermal trip points\n", d->ident); @@ -1128,14 +1126,12 @@ static int thermal_act(const struct dmi_system_id *d) { return 0; } static int thermal_nocrt(const struct dmi_system_id *d) { - pr_notice("%s detected: disabling all critical thermal trip point actions.\n", d->ident); nocrt = 1; return 0; } static int thermal_tzp(const struct dmi_system_id *d) { - if (tzp == 0) { pr_notice("%s detected: enabling thermal zone polling\n", d->ident); @@ -1144,7 +1140,6 @@ static int thermal_tzp(const struct dmi_system_id *d) { return 0; } static int thermal_psv(const struct dmi_system_id *d) { - if (psv == 0) { pr_notice("%s detected: disabling all passive thermal trip points\n", d->ident); @@ -1195,7 +1190,7 @@ static const struct dmi_system_id thermal_dmi_table[] __initconst = { static int __init acpi_thermal_init(void) { - int result = 0; + int result; dmi_check_system(thermal_dmi_table); @@ -1222,8 +1217,6 @@ static void __exit acpi_thermal_exit(void) { acpi_bus_unregister_driver(&acpi_thermal_driver); destroy_workqueue(acpi_thermal_pm_queue); - - return; } module_init(acpi_thermal_init); diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index 5a7b8065e77ffd73f4d1b339571fc5e12d9d58f7..2ea14648a661f869d3a505c9851f1dded5180224 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -793,6 +793,30 @@ bool acpi_dev_hid_uid_match(struct acpi_device *adev, } EXPORT_SYMBOL(acpi_dev_hid_uid_match); +/** + * acpi_dev_uid_to_integer - treat ACPI device _UID as integer + * @adev: ACPI device to get _UID from + * @integer: output buffer for integer + * + * Considers _UID as integer and converts it to @integer. + * + * Returns 0 on success, or negative error code otherwise. + */ +int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer) +{ + const char *uid; + + if (!adev) + return -ENODEV; + + uid = acpi_device_uid(adev); + if (!uid) + return -ENODATA; + + return kstrtou64(uid, 0, integer); +} +EXPORT_SYMBOL(acpi_dev_uid_to_integer); + /** * acpi_dev_found - Detect presence of a given ACPI device in the namespace. * @hid: Hardware ID of the device. @@ -878,7 +902,7 @@ bool acpi_dev_present(const char *hid, const char *uid, s64 hrv) struct acpi_dev_match_info match = {}; struct device *dev; - strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id)); + strscpy(match.hid[0].id, hid, sizeof(match.hid[0].id)); match.uid = uid; match.hrv = hrv; @@ -911,7 +935,7 @@ acpi_dev_get_next_match_dev(struct acpi_device *adev, const char *hid, const cha struct acpi_dev_match_info match = {}; struct device *dev; - strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id)); + strscpy(match.hid[0].id, hid, sizeof(match.hid[0].id)); match.uid = uid; match.hrv = hrv; @@ -961,7 +985,7 @@ EXPORT_SYMBOL(acpi_video_backlight_string); static int __init acpi_backlight(char *str) { - strlcpy(acpi_video_backlight_string, str, + strscpy(acpi_video_backlight_string, str, sizeof(acpi_video_backlight_string)); return 1; } diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index db2474fe58ac0cbf59d4a4f9eb67613d2dd660b7..0d9064a9804cf23ca6201b3550473913eb44608e 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -610,6 +610,41 @@ static const struct dmi_system_id video_detect_dmi_table[] = { }, }, + /* + * These Toshibas have a broken acpi-video interface for brightness + * control. They also have an issue where the panel is off after + * suspend until a special firmware call is made to turn it back + * on. This is handled by the toshiba_acpi kernel module, so that + * module must be enabled for these models to work correctly. + */ + { + /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ + .callback = video_detect_force_native, + /* Toshiba Portégé R700 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"), + }, + }, + { + /* Portégé: https://bugs.freedesktop.org/show_bug.cgi?id=82634 */ + /* Satellite: https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ + .callback = video_detect_force_native, + /* Toshiba Satellite/Portégé R830 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "R830"), + }, + }, + { + .callback = video_detect_force_native, + /* Toshiba Satellite/Portégé Z830 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "Z830"), + }, + }, + /* * Desktops which falsely report a backlight and which our heuristics * for this do not catch. diff --git a/drivers/acpi/viot.c b/drivers/acpi/viot.c index 6132092dab2a57e27c0fc40da3e4513dd26bb042..ed752cbbe636259a82db18ca7e2b6447487c7776 100644 --- a/drivers/acpi/viot.c +++ b/drivers/acpi/viot.c @@ -19,7 +19,6 @@ #define pr_fmt(fmt) "ACPI: VIOT: " fmt #include -#include #include #include #include diff --git a/drivers/acpi/x86/apple.c b/drivers/acpi/x86/apple.c index c285c91a5e9cfd390421fb349e6ff0f9eb5ab7c8..8812ecd03d5525cb9880b4eaa0ca3b0f104d7466 100644 --- a/drivers/acpi/x86/apple.c +++ b/drivers/acpi/x86/apple.c @@ -8,6 +8,7 @@ #include #include #include +#include "../internal.h" /* Apple _DSM device properties GUID */ static const guid_t apple_prp_guid = diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index f9ac12b778e6a311183fd1ec7068e2bc61f608d9..5350c73564b601fffa1f31ee566ae67a50d74de0 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -17,6 +17,7 @@ #include #include +#include #include #include "../sleep.h" @@ -27,6 +28,10 @@ static bool sleep_no_lps0 __read_mostly; module_param(sleep_no_lps0, bool, 0644); MODULE_PARM_DESC(sleep_no_lps0, "Do not use the special LPS0 device interface"); +static bool prefer_microsoft_dsm_guid __read_mostly; +module_param(prefer_microsoft_dsm_guid, bool, 0644); +MODULE_PARM_DESC(prefer_microsoft_dsm_guid, "Prefer using Microsoft GUID in LPS0 device _DSM evaluation"); + static const struct acpi_device_id lps0_device_ids[] = { {"PNP0D80", }, {"", }, @@ -363,40 +368,132 @@ out: return ret; } +struct amd_lps0_hid_device_data { + const unsigned int rev_id; + const bool check_off_by_one; + const bool prefer_amd_guid; +}; + +static const struct amd_lps0_hid_device_data amd_picasso = { + .rev_id = 0, + .check_off_by_one = true, + .prefer_amd_guid = false, +}; + +static const struct amd_lps0_hid_device_data amd_cezanne = { + .rev_id = 0, + .check_off_by_one = false, + .prefer_amd_guid = false, +}; + +static const struct amd_lps0_hid_device_data amd_rembrandt = { + .rev_id = 2, + .check_off_by_one = false, + .prefer_amd_guid = true, +}; + +static const struct acpi_device_id amd_hid_ids[] = { + {"AMD0004", (kernel_ulong_t)&amd_picasso, }, + {"AMD0005", (kernel_ulong_t)&amd_picasso, }, + {"AMDI0005", (kernel_ulong_t)&amd_picasso, }, + {"AMDI0006", (kernel_ulong_t)&amd_cezanne, }, + {"AMDI0007", (kernel_ulong_t)&amd_rembrandt, }, + {} +}; + +static int lps0_prefer_microsoft(const struct dmi_system_id *id) +{ + pr_debug("Preferring Microsoft GUID.\n"); + prefer_microsoft_dsm_guid = true; + return 0; +} + +static const struct dmi_system_id s2idle_dmi_table[] __initconst = { + { + /* + * ASUS TUF Gaming A17 FA707RE + * https://bugzilla.kernel.org/show_bug.cgi?id=216101 + */ + .callback = lps0_prefer_microsoft, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ASUS TUF Gaming A17"), + }, + }, + { + /* ASUS ROG Zephyrus G14 (2022) */ + .callback = lps0_prefer_microsoft, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ROG Zephyrus G14 GA402"), + }, + }, + { + /* + * Lenovo Yoga Slim 7 Pro X 14ARH7 + * https://bugzilla.kernel.org/show_bug.cgi?id=216473 : 82V2 + * https://bugzilla.kernel.org/show_bug.cgi?id=216438 : 82TL + */ + .callback = lps0_prefer_microsoft, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "82"), + }, + }, + { + /* + * ASUSTeK COMPUTER INC. ROG Flow X13 GV301RE_GV301RE + * https://gitlab.freedesktop.org/drm/amd/-/issues/2148 + */ + .callback = lps0_prefer_microsoft, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow X13 GV301"), + }, + }, + { + /* + * ASUSTeK COMPUTER INC. ROG Flow X16 GV601RW_GV601RW + * https://gitlab.freedesktop.org/drm/amd/-/issues/2148 + */ + .callback = lps0_prefer_microsoft, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow X16 GV601"), + }, + }, + {} +}; + static int lps0_device_attach(struct acpi_device *adev, const struct acpi_device_id *not_used) { if (lps0_device_handle) return 0; + lps0_dsm_func_mask_microsoft = validate_dsm(adev->handle, + ACPI_LPS0_DSM_UUID_MICROSOFT, 0, + &lps0_dsm_guid_microsoft); if (acpi_s2idle_vendor_amd()) { - /* AMD0004, AMD0005, AMDI0005: - * - Should use rev_id 0x0 - * - function mask > 0x3: Should use AMD method, but has off by one bug - * - function mask = 0x3: Should use Microsoft method - * AMDI0006: - * - should use rev_id 0x0 - * - function mask = 0x3: Should use Microsoft method - * AMDI0007: - * - Should use rev_id 0x2 - * - Should only use AMD method - */ - const char *hid = acpi_device_hid(adev); - rev_id = strcmp(hid, "AMDI0007") ? 0 : 2; + static const struct acpi_device_id *dev_id; + const struct amd_lps0_hid_device_data *data; + + for (dev_id = &amd_hid_ids[0]; dev_id->id[0]; dev_id++) + if (acpi_dev_hid_uid_match(adev, dev_id->id, NULL)) + break; + if (dev_id->id[0]) + data = (const struct amd_lps0_hid_device_data *) dev_id->driver_data; + else + data = &amd_rembrandt; + rev_id = data->rev_id; lps0_dsm_func_mask = validate_dsm(adev->handle, ACPI_LPS0_DSM_UUID_AMD, rev_id, &lps0_dsm_guid); - lps0_dsm_func_mask_microsoft = validate_dsm(adev->handle, - ACPI_LPS0_DSM_UUID_MICROSOFT, 0, - &lps0_dsm_guid_microsoft); - if (lps0_dsm_func_mask > 0x3 && (!strcmp(hid, "AMD0004") || - !strcmp(hid, "AMD0005") || - !strcmp(hid, "AMDI0005"))) { + if (lps0_dsm_func_mask > 0x3 && data->check_off_by_one) { lps0_dsm_func_mask = (lps0_dsm_func_mask << 1) | 0x1; acpi_handle_debug(adev->handle, "_DSM UUID %s: Adjusted function mask: 0x%x\n", ACPI_LPS0_DSM_UUID_AMD, lps0_dsm_func_mask); - } else if (lps0_dsm_func_mask_microsoft > 0 && - (!strcmp(hid, "AMDI0007") || - !strcmp(hid, "AMDI0008"))) { + } else if (lps0_dsm_func_mask_microsoft > 0 && data->prefer_amd_guid && + !prefer_microsoft_dsm_guid) { lps0_dsm_func_mask_microsoft = -EINVAL; acpi_handle_debug(adev->handle, "_DSM Using AMD method\n"); } @@ -404,7 +501,8 @@ static int lps0_device_attach(struct acpi_device *adev, rev_id = 1; lps0_dsm_func_mask = validate_dsm(adev->handle, ACPI_LPS0_DSM_UUID, rev_id, &lps0_dsm_guid); - lps0_dsm_func_mask_microsoft = -EINVAL; + if (!prefer_microsoft_dsm_guid) + lps0_dsm_func_mask_microsoft = -EINVAL; } if (lps0_dsm_func_mask < 0 && lps0_dsm_func_mask_microsoft < 0) @@ -486,6 +584,19 @@ int acpi_s2idle_prepare_late(void) return 0; } +void acpi_s2idle_check(void) +{ + struct acpi_s2idle_dev_ops *handler; + + if (!lps0_device_handle || sleep_no_lps0) + return; + + list_for_each_entry(handler, &lps0_s2idle_devops_head, list_node) { + if (handler->check) + handler->check(); + } +} + void acpi_s2idle_restore_early(void) { struct acpi_s2idle_dev_ops *handler; @@ -527,26 +638,30 @@ static const struct platform_s2idle_ops acpi_s2idle_ops_lps0 = { .begin = acpi_s2idle_begin, .prepare = acpi_s2idle_prepare, .prepare_late = acpi_s2idle_prepare_late, + .check = acpi_s2idle_check, .wake = acpi_s2idle_wake, .restore_early = acpi_s2idle_restore_early, .restore = acpi_s2idle_restore, .end = acpi_s2idle_end, }; -void acpi_s2idle_setup(void) +void __init acpi_s2idle_setup(void) { + dmi_check_system(s2idle_dmi_table); acpi_scan_add_handler(&lps0_handler); s2idle_set_ops(&acpi_s2idle_ops_lps0); } int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg) { + unsigned int sleep_flags; + if (!lps0_device_handle || sleep_no_lps0) return -ENODEV; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); list_add(&arg->list_node, &lps0_s2idle_devops_head); - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return 0; } @@ -554,12 +669,14 @@ EXPORT_SYMBOL_GPL(acpi_register_lps0_dev); void acpi_unregister_lps0_dev(struct acpi_s2idle_dev_ops *arg) { + unsigned int sleep_flags; + if (!lps0_device_handle || sleep_no_lps0) return; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); list_del(&arg->list_node); - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); } EXPORT_SYMBOL_GPL(acpi_unregister_lps0_dev); diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index 664070fc83498455788c7efa76a52b90364d647e..f8a2cbdc0ce2b3e49731e88209a15baca9696cec 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -207,9 +207,26 @@ static const struct x86_cpu_id storage_d3_cpu_ids[] = { {} }; +static const struct dmi_system_id force_storage_d3_dmi[] = { + { + /* + * _ADR is ambiguous between GPP1.DEV0 and GPP1.NVME + * but .NVME is needed to get StorageD3Enable node + * https://bugzilla.kernel.org/show_bug.cgi?id=216440 + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 14 7425 2-in-1"), + } + }, + {} +}; + bool force_storage_d3(void) { - return x86_match_cpu(storage_d3_cpu_ids); + const struct dmi_system_id *dmi_id = dmi_first_match(force_storage_d3_dmi); + + return dmi_id || x86_match_cpu(storage_d3_cpu_ids); } /* @@ -351,11 +368,17 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s struct acpi_device *adev = ACPI_COMPANION(controller_parent); const struct dmi_system_id *dmi_id; long quirks = 0; + u64 uid; + int ret; *skip = false; - /* !dev_is_platform() to not match on PNP enumerated debug UARTs */ - if (!adev || !adev->pnp.unique_id || !dev_is_platform(controller_parent)) + ret = acpi_dev_uid_to_integer(adev, &uid); + if (ret) + return 0; + + /* to not match on PNP enumerated debug UARTs */ + if (!dev_is_platform(controller_parent)) return 0; dmi_id = dmi_first_match(acpi_quirk_skip_dmi_ids); @@ -363,10 +386,10 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s quirks = (unsigned long)dmi_id->driver_data; if (quirks & ACPI_QUIRK_UART1_TTY_UART2_SKIP) { - if (!strcmp(adev->pnp.unique_id, "1")) + if (uid == 1) return -ENODEV; /* Create tty cdev instead of serdev */ - if (!strcmp(adev->pnp.unique_id, "2")) + if (uid == 2) *skip = true; } diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index 32b0e0b930c1068e6e8c867cf2fdbfea3d08556c..110a535648d2e1fa9ace1878b207f862b41dfca0 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -209,6 +209,7 @@ static int amba_match(struct device *dev, struct device_driver *drv) struct amba_device *pcdev = to_amba_device(dev); struct amba_driver *pcdrv = to_amba_driver(drv); + mutex_lock(&pcdev->periphid_lock); if (!pcdev->periphid) { int ret = amba_read_periphid(pcdev); @@ -218,11 +219,14 @@ static int amba_match(struct device *dev, struct device_driver *drv) * permanent failure in reading pid and cid, simply map it to * -EPROBE_DEFER. */ - if (ret) + if (ret) { + mutex_unlock(&pcdev->periphid_lock); return -EPROBE_DEFER; + } dev_set_uevent_suppress(dev, false); kobject_uevent(&dev->kobj, KOBJ_ADD); } + mutex_unlock(&pcdev->periphid_lock); /* When driver_override is set, only bind to the matching driver */ if (pcdev->driver_override) @@ -532,6 +536,7 @@ static void amba_device_release(struct device *dev) if (d->res.parent) release_resource(&d->res); + mutex_destroy(&d->periphid_lock); kfree(d); } @@ -584,6 +589,7 @@ static void amba_device_initialize(struct amba_device *dev, const char *name) dev->dev.dma_mask = &dev->dev.coherent_dma_mask; dev->dev.dma_parms = &dev->dma_parms; dev->res.name = dev_name(&dev->dev); + mutex_init(&dev->periphid_lock); } /** diff --git a/drivers/android/binder.c b/drivers/android/binder.c index c964d7c8c3841386225d317ca34a8118c7a8a880..880224ec6abb8e62257857bbac76358e4c006b82 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1385,6 +1385,18 @@ static int binder_inc_ref_for_node(struct binder_proc *proc, } ret = binder_inc_ref_olocked(ref, strong, target_list); *rdata = ref->data; + if (ret && ref == new_ref) { + /* + * Cleanup the failed reference here as the target + * could now be dead and have already released its + * references by now. Calling on the new reference + * with strong=0 and a tmp_refs will not decrement + * the node. The new_ref gets kfree'd below. + */ + binder_cleanup_ref_olocked(new_ref); + ref = NULL; + } + binder_proc_unlock(proc); if (new_ref && ref != new_ref) /* @@ -4247,10 +4259,9 @@ static int binder_wait_for_work(struct binder_thread *thread, struct binder_proc *proc = thread->proc; int ret = 0; - freezer_do_not_count(); binder_inner_proc_lock(proc); for (;;) { - prepare_to_wait(&thread->wait, &wait, TASK_INTERRUPTIBLE); + prepare_to_wait(&thread->wait, &wait, TASK_INTERRUPTIBLE|TASK_FREEZABLE); if (binder_has_work_ilocked(thread, do_proc_work)) break; if (do_proc_work) @@ -4267,7 +4278,6 @@ static int binder_wait_for_work(struct binder_thread *thread, } finish_wait(&thread->wait, &wait); binder_inner_proc_unlock(proc); - freezer_count(); return ret; } diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 1014beb128025706ac6d98d7e09a6030acb5c23f..1c39cfce32fade507d05f4a7d25070f09fc52fc0 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -208,8 +208,8 @@ static int binder_update_page_range(struct binder_alloc *alloc, int allocate, } } - if (need_mm && mmget_not_zero(alloc->vma_vm_mm)) - mm = alloc->vma_vm_mm; + if (need_mm && mmget_not_zero(alloc->mm)) + mm = alloc->mm; if (mm) { mmap_read_lock(mm); @@ -309,35 +309,13 @@ err_no_vma: return vma ? -ENOMEM : -ESRCH; } - -static inline void binder_alloc_set_vma(struct binder_alloc *alloc, - struct vm_area_struct *vma) -{ - unsigned long vm_start = 0; - - /* - * Allow clearing the vma with holding just the read lock to allow - * munmapping downgrade of the write lock before freeing and closing the - * file using binder_alloc_vma_close(). - */ - if (vma) { - vm_start = vma->vm_start; - alloc->vma_vm_mm = vma->vm_mm; - mmap_assert_write_locked(alloc->vma_vm_mm); - } else { - mmap_assert_locked(alloc->vma_vm_mm); - } - - alloc->vma_addr = vm_start; -} - static inline struct vm_area_struct *binder_alloc_get_vma( struct binder_alloc *alloc) { struct vm_area_struct *vma = NULL; if (alloc->vma_addr) - vma = vma_lookup(alloc->vma_vm_mm, alloc->vma_addr); + vma = vma_lookup(alloc->mm, alloc->vma_addr); return vma; } @@ -402,12 +380,15 @@ static struct binder_buffer *binder_alloc_new_buf_locked( size_t size, data_offsets_size; int ret; + mmap_read_lock(alloc->mm); if (!binder_alloc_get_vma(alloc)) { + mmap_read_unlock(alloc->mm); binder_alloc_debug(BINDER_DEBUG_USER_ERROR, "%d: binder_alloc_buf, no vma\n", alloc->pid); return ERR_PTR(-ESRCH); } + mmap_read_unlock(alloc->mm); data_offsets_size = ALIGN(data_size, sizeof(void *)) + ALIGN(offsets_size, sizeof(void *)); @@ -791,8 +772,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, buffer->free = 1; binder_insert_free_buffer(alloc, buffer); alloc->free_async_space = alloc->buffer_size / 2; - binder_alloc_set_vma(alloc, vma); - mmgrab(alloc->vma_vm_mm); + alloc->vma_addr = vma->vm_start; return 0; @@ -822,7 +802,7 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) buffers = 0; mutex_lock(&alloc->mutex); BUG_ON(alloc->vma_addr && - vma_lookup(alloc->vma_vm_mm, alloc->vma_addr)); + vma_lookup(alloc->mm, alloc->vma_addr)); while ((n = rb_first(&alloc->allocated_buffers))) { buffer = rb_entry(n, struct binder_buffer, rb_node); @@ -872,8 +852,8 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) kfree(alloc->pages); } mutex_unlock(&alloc->mutex); - if (alloc->vma_vm_mm) - mmdrop(alloc->vma_vm_mm); + if (alloc->mm) + mmdrop(alloc->mm); binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE, "%s: %d buffers %d, pages %d\n", @@ -929,17 +909,25 @@ void binder_alloc_print_pages(struct seq_file *m, * Make sure the binder_alloc is fully initialized, otherwise we might * read inconsistent state. */ - if (binder_alloc_get_vma(alloc) != NULL) { - for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { - page = &alloc->pages[i]; - if (!page->page_ptr) - free++; - else if (list_empty(&page->lru)) - active++; - else - lru++; - } + + mmap_read_lock(alloc->mm); + if (binder_alloc_get_vma(alloc) == NULL) { + mmap_read_unlock(alloc->mm); + goto uninitialized; } + + mmap_read_unlock(alloc->mm); + for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) { + page = &alloc->pages[i]; + if (!page->page_ptr) + free++; + else if (list_empty(&page->lru)) + active++; + else + lru++; + } + +uninitialized: mutex_unlock(&alloc->mutex); seq_printf(m, " pages: %d:%d:%d\n", active, lru, free); seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high); @@ -974,7 +962,7 @@ int binder_alloc_get_allocated_count(struct binder_alloc *alloc) */ void binder_alloc_vma_close(struct binder_alloc *alloc) { - binder_alloc_set_vma(alloc, NULL); + alloc->vma_addr = 0; } /** @@ -1011,7 +999,7 @@ enum lru_status binder_alloc_free_page(struct list_head *item, index = page - alloc->pages; page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE; - mm = alloc->vma_vm_mm; + mm = alloc->mm; if (!mmget_not_zero(mm)) goto err_mmget; if (!mmap_read_trylock(mm)) @@ -1080,6 +1068,8 @@ static struct shrinker binder_shrinker = { void binder_alloc_init(struct binder_alloc *alloc) { alloc->pid = current->group_leader->pid; + alloc->mm = current->mm; + mmgrab(alloc->mm); mutex_init(&alloc->mutex); INIT_LIST_HEAD(&alloc->buffers); } diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h index 1e4fd37af5e03e10742b627b37a158087316c7ab..0f811ac4bcffd1f922a6bd895cd5682ad9acd08c 100644 --- a/drivers/android/binder_alloc.h +++ b/drivers/android/binder_alloc.h @@ -74,11 +74,10 @@ struct binder_lru_page { /** * struct binder_alloc - per-binder proc state for binder allocator - * @vma: vm_area_struct passed to mmap_handler - * (invarient after mmap) - * @tsk: tid for task that called init for this proc - * (invariant after init) - * @vma_vm_mm: copy of vma->vm_mm (invarient after mmap) + * @mutex: protects binder_alloc fields + * @vma_addr: vm_area_struct->vm_start passed to mmap_handler + * (invariant after mmap) + * @mm: copy of task->mm (invariant after open) * @buffer: base of per-proc address space mapped via mmap * @buffers: list of all buffers for this proc * @free_buffers: rb tree of buffers available for allocation @@ -101,7 +100,7 @@ struct binder_lru_page { struct binder_alloc { struct mutex mutex; unsigned long vma_addr; - struct mm_struct *vma_vm_mm; + struct mm_struct *mm; void __user *buffer; struct list_head buffers; struct rb_root free_buffers; @@ -109,7 +108,6 @@ struct binder_alloc { size_t free_async_space; struct binder_lru_page *pages; size_t buffer_size; - uint32_t buffer_free; int pid; size_t pages_high; bool oneway_spam_detected; diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index 588d753a7a19975a447a33c6f6d128cf54269319..09b2ce7e4c345847eb0dd263c70e352d6e0ba35e 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -39,7 +39,6 @@ #define FIRST_INODE 1 #define SECOND_INODE 2 #define INODE_OFFSET 3 -#define INTSTRLEN 21 #define BINDERFS_MAX_MINOR (1U << MINORBITS) /* Ensure that the initial ipc namespace always has devices available. */ #define BINDERFS_MAX_MINOR_CAPPED (BINDERFS_MAX_MINOR - 4) @@ -340,22 +339,10 @@ static int binderfs_show_options(struct seq_file *seq, struct dentry *root) return 0; } -static void binderfs_put_super(struct super_block *sb) -{ - struct binderfs_info *info = sb->s_fs_info; - - if (info && info->ipc_ns) - put_ipc_ns(info->ipc_ns); - - kfree(info); - sb->s_fs_info = NULL; -} - static const struct super_operations binderfs_super_ops = { .evict_inode = binderfs_evict_inode, .show_options = binderfs_show_options, .statfs = simple_statfs, - .put_super = binderfs_put_super, }; static inline bool is_binderfs_control_device(const struct dentry *dentry) @@ -785,11 +772,27 @@ static int binderfs_init_fs_context(struct fs_context *fc) return 0; } +static void binderfs_kill_super(struct super_block *sb) +{ + struct binderfs_info *info = sb->s_fs_info; + + /* + * During inode eviction struct binderfs_info is needed. + * So first wipe the super_block then free struct binderfs_info. + */ + kill_litter_super(sb); + + if (info && info->ipc_ns) + put_ipc_ns(info->ipc_ns); + + kfree(info); +} + static struct file_system_type binder_fs_type = { .name = "binder", .init_fs_context = binderfs_init_fs_context, .parameters = binderfs_fs_parameters, - .kill_sb = kill_litter_super, + .kill_sb = binderfs_kill_super, .fs_flags = FS_USERNS_MOUNT, }; diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 1c9f4fb2595dc39ef569acb298ca869628835f25..36833a8629980785593f29151927a3dd1e6c2c28 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -176,9 +176,19 @@ config AHCI_DM816 If unsure, say N. +config AHCI_DWC + tristate "Synopsys DWC AHCI SATA support" + select SATA_HOST + select MFD_SYSCON if (MIPS_BAIKAL_T1 || COMPILE_TEST) + help + This option enables support for the Synopsys DWC AHCI SATA + controller implementation. + + If unsure, say N. + config AHCI_ST tristate "ST AHCI SATA support" - depends on ARCH_STI + depends on ARCH_STI || COMPILE_TEST select SATA_HOST help This option enables support for ST AHCI SATA controller. @@ -1102,8 +1112,7 @@ config PATA_PCMCIA If unsure, say N. config PATA_PLATFORM - tristate "Generic platform device PATA support" - depends on EXPERT || PPC || HAVE_PATA_PLATFORM + tristate "Generic platform device PATA support" if HAVE_PATA_PLATFORM help This option enables support for generic directly connected ATA devices commonly found on embedded systems. @@ -1112,7 +1121,8 @@ config PATA_PLATFORM config PATA_OF_PLATFORM tristate "OpenFirmware platform device PATA support" - depends on PATA_PLATFORM && OF + depends on OF + select PATA_PLATFORM help This option enables support for generic directly connected ATA devices commonly found on embedded systems with OpenFirmware diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile index b8aebfb14e825040af4a228b475e0be3c8b028ac..34623365d9a67fc5971b9861bdb1a61aa4e3ae00 100644 --- a/drivers/ata/Makefile +++ b/drivers/ata/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_AHCI_BRCM) += ahci_brcm.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_CEVA) += ahci_ceva.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_DA850) += ahci_da850.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_DM816) += ahci_dm816.o libahci.o libahci_platform.o +obj-$(CONFIG_AHCI_DWC) += ahci_dwc.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_IMX) += ahci_imx.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_MTK) += ahci_mtk.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_MVEBU) += ahci_mvebu.o libahci.o libahci_platform.o diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index c1eca72b4575df73f32dc24ddc23d460cf5de581..639de2d75d63644d2feb09e23e4bf4ce5795321c 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -657,7 +657,7 @@ static void ahci_pci_save_initial_config(struct pci_dev *pdev, { if (pdev->vendor == PCI_VENDOR_ID_JMICRON && pdev->device == 0x2361) { dev_info(&pdev->dev, "JMB361 has only one port\n"); - hpriv->force_port_map = 1; + hpriv->saved_port_map = 1; } /* @@ -690,7 +690,7 @@ static void ahci_pci_init_controller(struct ata_host *host) mv = 2; else mv = 4; - port_mmio = __ahci_port_base(host, mv); + port_mmio = __ahci_port_base(hpriv, mv); writel(0, port_mmio + PORT_IRQ_MASK); @@ -1609,15 +1609,12 @@ static void ahci_update_initial_lpm_policy(struct ata_port *ap, goto update_policy; } -#ifdef CONFIG_ACPI - if (policy > ATA_LPM_MED_POWER && - (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0)) { + if (policy > ATA_LPM_MED_POWER && pm_suspend_default_s2idle()) { if (hpriv->cap & HOST_CAP_PART) policy = ATA_LPM_MIN_POWER_WITH_PARTIAL; else if (hpriv->cap & HOST_CAP_SSC) policy = ATA_LPM_MIN_POWER; } -#endif update_policy: if (policy >= ATA_LPM_UNKNOWN && policy <= ATA_LPM_MIN_POWER) diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index ad11a4c52fbeb9120010b872799c97c0fceb26e3..da7ee8bec165a327565703512e6583742f08ca21 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -38,7 +38,6 @@ enum { AHCI_MAX_PORTS = 32, - AHCI_MAX_CLKS = 5, AHCI_MAX_SG = 168, /* hardware max is 64K */ AHCI_DMA_BOUNDARY = 0xffffffff, AHCI_MAX_CMDS = 32, @@ -139,7 +138,7 @@ enum { PORT_IRQ_BAD_PMP = (1 << 23), /* incorrect port multiplier */ PORT_IRQ_PHYRDY = (1 << 22), /* PhyRdy changed */ - PORT_IRQ_DEV_ILCK = (1 << 7), /* device interlock */ + PORT_IRQ_DMPS = (1 << 7), /* mechanical presence status */ PORT_IRQ_CONNECT = (1 << 6), /* port connect change status */ PORT_IRQ_SG_DONE = (1 << 5), /* descriptor processed */ PORT_IRQ_UNK_FIS = (1 << 4), /* unknown FIS rx'd */ @@ -167,6 +166,8 @@ enum { PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */ PORT_CMD_FBSCP = (1 << 22), /* FBS Capable Port */ PORT_CMD_ESP = (1 << 21), /* External Sata Port */ + PORT_CMD_CPD = (1 << 20), /* Cold Presence Detection */ + PORT_CMD_MPSP = (1 << 19), /* Mechanical Presence Switch */ PORT_CMD_HPCP = (1 << 18), /* HotPlug Capable Port */ PORT_CMD_PMP = (1 << 17), /* PMP attached */ PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */ @@ -182,6 +183,10 @@ enum { PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */ PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */ + /* PORT_CMD capabilities mask */ + PORT_CMD_CAP = PORT_CMD_HPCP | PORT_CMD_MPSP | + PORT_CMD_CPD | PORT_CMD_ESP | PORT_CMD_FBSCP, + /* PORT_FBS bits */ PORT_FBS_DWE_OFFSET = 16, /* FBS device with error offset */ PORT_FBS_ADO_OFFSET = 12, /* FBS active dev optimization offset */ @@ -323,7 +328,6 @@ struct ahci_port_priv { struct ahci_host_priv { /* Input fields */ unsigned int flags; /* AHCI_HFLAG_* */ - u32 force_port_map; /* force port map */ u32 mask_port_map; /* mask out particular bits */ void __iomem * mmio; /* bus-independent mem map */ @@ -334,12 +338,15 @@ struct ahci_host_priv { u32 saved_cap; /* saved initial cap */ u32 saved_cap2; /* saved initial cap2 */ u32 saved_port_map; /* saved initial port_map */ + u32 saved_port_cap[AHCI_MAX_PORTS]; /* saved port_cap */ u32 em_loc; /* enclosure management location */ u32 em_buf_sz; /* EM buffer size in byte */ u32 em_msg_type; /* EM message type */ u32 remapped_nvme; /* NVMe remapped device count */ bool got_runtime_pm; /* Did we do pm_runtime_get? */ - struct clk *clks[AHCI_MAX_CLKS]; /* Optional */ + unsigned int n_clks; + struct clk_bulk_data *clks; /* Optional */ + unsigned int f_rsts; struct reset_control *rsts; /* Optional */ struct regulator **target_pwrs; /* Optional */ struct regulator *ahci_regulator;/* Optional */ @@ -426,10 +433,9 @@ int ahci_host_activate(struct ata_host *host, struct scsi_host_template *sht); void ahci_error_handler(struct ata_port *ap); u32 ahci_handle_port_intr(struct ata_host *host, u32 irq_masked); -static inline void __iomem *__ahci_port_base(struct ata_host *host, +static inline void __iomem *__ahci_port_base(struct ahci_host_priv *hpriv, unsigned int port_no) { - struct ahci_host_priv *hpriv = host->private_data; void __iomem *mmio = hpriv->mmio; return mmio + 0x100 + (port_no * 0x80); @@ -437,7 +443,9 @@ static inline void __iomem *__ahci_port_base(struct ata_host *host, static inline void __iomem *ahci_port_base(struct ata_port *ap) { - return __ahci_port_base(ap->host, ap->port_no); + struct ahci_host_priv *hpriv = ap->host->private_data; + + return __ahci_port_base(hpriv, ap->port_no); } static inline int ahci_nr_ports(u32 cap) diff --git a/drivers/ata/ahci_da850.c b/drivers/ata/ahci_da850.c index 052c28e250aa8d96ca2c2f90e796fa91ab480423..dc8a019b8340d33f17c103f8045c58ffc94568f2 100644 --- a/drivers/ata/ahci_da850.c +++ b/drivers/ata/ahci_da850.c @@ -163,7 +163,6 @@ static int ahci_da850_probe(struct platform_device *pdev) struct ahci_host_priv *hpriv; void __iomem *pwrdn_reg; struct resource *res; - struct clk *clk; u32 mpy; int rc; @@ -172,36 +171,28 @@ static int ahci_da850_probe(struct platform_device *pdev) return PTR_ERR(hpriv); /* - * Internally ahci_platform_get_resources() calls clk_get(dev, NULL) - * when trying to obtain the functional clock. This SATA controller - * uses two clocks for which we specify two connection ids. If we don't - * have the functional clock at this point - call clk_get() again with - * con_id = "fck". + * Internally ahci_platform_get_resources() calls the bulk clocks + * get method or falls back to using a single clk_get_optional(). + * This AHCI SATA controller uses two clocks: functional clock + * with "fck" connection id and external reference clock with + * "refclk" id. If we haven't got all of them re-try the clocks + * getting procedure with the explicitly specified ids. */ - if (!hpriv->clks[0]) { - clk = clk_get(dev, "fck"); - if (IS_ERR(clk)) - return PTR_ERR(clk); - - hpriv->clks[0] = clk; - } - - /* - * The second clock used by ahci-da850 is the external REFCLK. If we - * didn't get it from ahci_platform_get_resources(), let's try to - * specify the con_id in clk_get(). - */ - if (!hpriv->clks[1]) { - clk = clk_get(dev, "refclk"); - if (IS_ERR(clk)) { - dev_err(dev, "unable to obtain the reference clock"); - return -ENODEV; - } - - hpriv->clks[1] = clk; + if (hpriv->n_clks < 2) { + hpriv->clks = devm_kcalloc(dev, 2, sizeof(*hpriv->clks), GFP_KERNEL); + if (!hpriv->clks) + return -ENOMEM; + + hpriv->clks[0].id = "fck"; + hpriv->clks[1].id = "refclk"; + hpriv->n_clks = 2; + + rc = devm_clk_bulk_get(dev, hpriv->n_clks, hpriv->clks); + if (rc) + return rc; } - mpy = ahci_da850_calculate_mpy(clk_get_rate(hpriv->clks[1])); + mpy = ahci_da850_calculate_mpy(clk_get_rate(hpriv->clks[1].clk)); if (mpy == 0) { dev_err(dev, "invalid REFCLK multiplier value: 0x%x", mpy); return -EINVAL; diff --git a/drivers/ata/ahci_dm816.c b/drivers/ata/ahci_dm816.c index 8a92112dcd59080ac9658d221f1c0414612cf653..d26efcd20f6433d6f99d5cd50c22f4410a84fc4d 100644 --- a/drivers/ata/ahci_dm816.c +++ b/drivers/ata/ahci_dm816.c @@ -69,12 +69,12 @@ static int ahci_dm816_phy_init(struct ahci_host_priv *hpriv, struct device *dev) * keep-alive clock and the external reference clock. We need the * rate of the latter to calculate the correct value of MPY bits. */ - if (!hpriv->clks[1]) { + if (hpriv->n_clks < 2) { dev_err(dev, "reference clock not supplied\n"); return -EINVAL; } - refclk_rate = clk_get_rate(hpriv->clks[1]); + refclk_rate = clk_get_rate(hpriv->clks[1].clk); if ((refclk_rate % 100) != 0) { dev_err(dev, "reference clock rate must be divisible by 100\n"); return -EINVAL; diff --git a/drivers/ata/ahci_dwc.c b/drivers/ata/ahci_dwc.c new file mode 100644 index 0000000000000000000000000000000000000000..8fb66860db318d57dd86aab3a895427eb4707e36 --- /dev/null +++ b/drivers/ata/ahci_dwc.c @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DWC AHCI SATA Platform driver + * + * Copyright (C) 2021 BAIKAL ELECTRONICS, JSC + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ahci.h" + +#define DRV_NAME "ahci-dwc" + +#define AHCI_DWC_FBS_PMPN_MAX 15 + +/* DWC AHCI SATA controller specific registers */ +#define AHCI_DWC_HOST_OOBR 0xbc +#define AHCI_DWC_HOST_OOB_WE BIT(31) +#define AHCI_DWC_HOST_CWMIN_MASK GENMASK(30, 24) +#define AHCI_DWC_HOST_CWMAX_MASK GENMASK(23, 16) +#define AHCI_DWC_HOST_CIMIN_MASK GENMASK(15, 8) +#define AHCI_DWC_HOST_CIMAX_MASK GENMASK(7, 0) + +#define AHCI_DWC_HOST_GPCR 0xd0 +#define AHCI_DWC_HOST_GPSR 0xd4 + +#define AHCI_DWC_HOST_TIMER1MS 0xe0 +#define AHCI_DWC_HOST_TIMV_MASK GENMASK(19, 0) + +#define AHCI_DWC_HOST_GPARAM1R 0xe8 +#define AHCI_DWC_HOST_ALIGN_M BIT(31) +#define AHCI_DWC_HOST_RX_BUFFER BIT(30) +#define AHCI_DWC_HOST_PHY_DATA_MASK GENMASK(29, 28) +#define AHCI_DWC_HOST_PHY_RST BIT(27) +#define AHCI_DWC_HOST_PHY_CTRL_MASK GENMASK(26, 21) +#define AHCI_DWC_HOST_PHY_STAT_MASK GENMASK(20, 15) +#define AHCI_DWC_HOST_LATCH_M BIT(14) +#define AHCI_DWC_HOST_PHY_TYPE_MASK GENMASK(13, 11) +#define AHCI_DWC_HOST_RET_ERR BIT(10) +#define AHCI_DWC_HOST_AHB_ENDIAN_MASK GENMASK(9, 8) +#define AHCI_DWC_HOST_S_HADDR BIT(7) +#define AHCI_DWC_HOST_M_HADDR BIT(6) +#define AHCI_DWC_HOST_S_HDATA_MASK GENMASK(5, 3) +#define AHCI_DWC_HOST_M_HDATA_MASK GENMASK(2, 0) + +#define AHCI_DWC_HOST_GPARAM2R 0xec +#define AHCI_DWC_HOST_FBS_MEM_S BIT(19) +#define AHCI_DWC_HOST_FBS_PMPN_MASK GENMASK(17, 16) +#define AHCI_DWC_HOST_FBS_SUP BIT(15) +#define AHCI_DWC_HOST_DEV_CP BIT(14) +#define AHCI_DWC_HOST_DEV_MP BIT(13) +#define AHCI_DWC_HOST_ENCODE_M BIT(12) +#define AHCI_DWC_HOST_RXOOB_CLK_M BIT(11) +#define AHCI_DWC_HOST_RXOOB_M BIT(10) +#define AHCI_DWC_HOST_TXOOB_M BIT(9) +#define AHCI_DWC_HOST_RXOOB_M BIT(10) +#define AHCI_DWC_HOST_RXOOB_CLK_MASK GENMASK(8, 0) + +#define AHCI_DWC_HOST_PPARAMR 0xf0 +#define AHCI_DWC_HOST_TX_MEM_M BIT(11) +#define AHCI_DWC_HOST_TX_MEM_S BIT(10) +#define AHCI_DWC_HOST_RX_MEM_M BIT(9) +#define AHCI_DWC_HOST_RX_MEM_S BIT(8) +#define AHCI_DWC_HOST_TXFIFO_DEPTH GENMASK(7, 4) +#define AHCI_DWC_HOST_RXFIFO_DEPTH GENMASK(3, 0) + +#define AHCI_DWC_HOST_TESTR 0xf4 +#define AHCI_DWC_HOST_PSEL_MASK GENMASK(18, 16) +#define AHCI_DWC_HOST_TEST_IF BIT(0) + +#define AHCI_DWC_HOST_VERSIONR 0xf8 +#define AHCI_DWC_HOST_IDR 0xfc + +#define AHCI_DWC_PORT_DMACR 0x70 +#define AHCI_DWC_PORT_RXABL_MASK GENMASK(15, 12) +#define AHCI_DWC_PORT_TXABL_MASK GENMASK(11, 8) +#define AHCI_DWC_PORT_RXTS_MASK GENMASK(7, 4) +#define AHCI_DWC_PORT_TXTS_MASK GENMASK(3, 0) +#define AHCI_DWC_PORT_PHYCR 0x74 +#define AHCI_DWC_PORT_PHYSR 0x78 + +/* Baikal-T1 AHCI SATA specific registers */ +#define AHCI_BT1_HOST_PHYCR AHCI_DWC_HOST_GPCR +#define AHCI_BT1_HOST_MPLM_MASK GENMASK(29, 23) +#define AHCI_BT1_HOST_LOSDT_MASK GENMASK(22, 20) +#define AHCI_BT1_HOST_CRR BIT(19) +#define AHCI_BT1_HOST_CRW BIT(18) +#define AHCI_BT1_HOST_CRCD BIT(17) +#define AHCI_BT1_HOST_CRCA BIT(16) +#define AHCI_BT1_HOST_CRDI_MASK GENMASK(15, 0) + +#define AHCI_BT1_HOST_PHYSR AHCI_DWC_HOST_GPSR +#define AHCI_BT1_HOST_CRA BIT(16) +#define AHCI_BT1_HOST_CRDO_MASK GENMASK(15, 0) + +struct ahci_dwc_plat_data { + unsigned int pflags; + unsigned int hflags; + int (*init)(struct ahci_host_priv *hpriv); + int (*reinit)(struct ahci_host_priv *hpriv); + void (*clear)(struct ahci_host_priv *hpriv); +}; + +struct ahci_dwc_host_priv { + const struct ahci_dwc_plat_data *pdata; + struct platform_device *pdev; + + u32 timv; + u32 dmacr[AHCI_MAX_PORTS]; +}; + +static int ahci_bt1_init(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + int ret; + + /* APB, application and reference clocks are required */ + if (!ahci_platform_find_clk(hpriv, "pclk") || + !ahci_platform_find_clk(hpriv, "aclk") || + !ahci_platform_find_clk(hpriv, "ref")) { + dev_err(&dpriv->pdev->dev, "No system clocks specified\n"); + return -EINVAL; + } + + /* + * Fully reset the SATA AXI and ref clocks domain to ensure the state + * machine is working from scratch especially if the reference clocks + * source has been changed. + */ + ret = ahci_platform_assert_rsts(hpriv); + if (ret) { + dev_err(&dpriv->pdev->dev, "Couldn't assert the resets\n"); + return ret; + } + + ret = ahci_platform_deassert_rsts(hpriv); + if (ret) { + dev_err(&dpriv->pdev->dev, "Couldn't de-assert the resets\n"); + return ret; + } + + return 0; +} + +static struct ahci_host_priv *ahci_dwc_get_resources(struct platform_device *pdev) +{ + struct ahci_dwc_host_priv *dpriv; + struct ahci_host_priv *hpriv; + + dpriv = devm_kzalloc(&pdev->dev, sizeof(*dpriv), GFP_KERNEL); + if (!dpriv) + return ERR_PTR(-ENOMEM); + + dpriv->pdev = pdev; + dpriv->pdata = device_get_match_data(&pdev->dev); + if (!dpriv->pdata) + return ERR_PTR(-EINVAL); + + hpriv = ahci_platform_get_resources(pdev, dpriv->pdata->pflags); + if (IS_ERR(hpriv)) + return hpriv; + + hpriv->flags |= dpriv->pdata->hflags; + hpriv->plat_data = (void *)dpriv; + + return hpriv; +} + +static void ahci_dwc_check_cap(struct ahci_host_priv *hpriv) +{ + unsigned long port_map = hpriv->saved_port_map | hpriv->mask_port_map; + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + bool dev_mp, dev_cp, fbs_sup; + unsigned int fbs_pmp; + u32 param; + int i; + + param = readl(hpriv->mmio + AHCI_DWC_HOST_GPARAM2R); + dev_mp = !!(param & AHCI_DWC_HOST_DEV_MP); + dev_cp = !!(param & AHCI_DWC_HOST_DEV_CP); + fbs_sup = !!(param & AHCI_DWC_HOST_FBS_SUP); + fbs_pmp = 5 * FIELD_GET(AHCI_DWC_HOST_FBS_PMPN_MASK, param); + + if (!dev_mp && hpriv->saved_cap & HOST_CAP_MPS) { + dev_warn(&dpriv->pdev->dev, "MPS is unsupported\n"); + hpriv->saved_cap &= ~HOST_CAP_MPS; + } + + + if (fbs_sup && fbs_pmp < AHCI_DWC_FBS_PMPN_MAX) { + dev_warn(&dpriv->pdev->dev, "PMPn is limited up to %u ports\n", + fbs_pmp); + } + + for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) { + if (!dev_mp && hpriv->saved_port_cap[i] & PORT_CMD_MPSP) { + dev_warn(&dpriv->pdev->dev, "MPS incapable port %d\n", i); + hpriv->saved_port_cap[i] &= ~PORT_CMD_MPSP; + } + + if (!dev_cp && hpriv->saved_port_cap[i] & PORT_CMD_CPD) { + dev_warn(&dpriv->pdev->dev, "CPD incapable port %d\n", i); + hpriv->saved_port_cap[i] &= ~PORT_CMD_CPD; + } + + if (!fbs_sup && hpriv->saved_port_cap[i] & PORT_CMD_FBSCP) { + dev_warn(&dpriv->pdev->dev, "FBS incapable port %d\n", i); + hpriv->saved_port_cap[i] &= ~PORT_CMD_FBSCP; + } + } +} + +static void ahci_dwc_init_timer(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + unsigned long rate; + struct clk *aclk; + u32 cap, cap2; + + /* 1ms tick is generated only for the CCC or DevSleep features */ + cap = readl(hpriv->mmio + HOST_CAP); + cap2 = readl(hpriv->mmio + HOST_CAP2); + if (!(cap & HOST_CAP_CCC) && !(cap2 & HOST_CAP2_SDS)) + return; + + /* + * Tick is generated based on the AXI/AHB application clocks signal + * so we need to be sure in the clock we are going to use. + */ + aclk = ahci_platform_find_clk(hpriv, "aclk"); + if (!aclk) + return; + + /* 1ms timer interval is set as TIMV = AMBA_FREQ[MHZ] * 1000 */ + dpriv->timv = readl(hpriv->mmio + AHCI_DWC_HOST_TIMER1MS); + dpriv->timv = FIELD_GET(AHCI_DWC_HOST_TIMV_MASK, dpriv->timv); + rate = clk_get_rate(aclk) / 1000UL; + if (rate == dpriv->timv) + return; + + dev_info(&dpriv->pdev->dev, "Update CCC/DevSlp timer for Fapp %lu MHz\n", + rate / 1000UL); + dpriv->timv = FIELD_PREP(AHCI_DWC_HOST_TIMV_MASK, rate); + writel(dpriv->timv, hpriv->mmio + AHCI_DWC_HOST_TIMER1MS); +} + +static int ahci_dwc_init_dmacr(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + struct device_node *child; + void __iomem *port_mmio; + u32 port, dmacr, ts; + + /* + * Update the DMA Tx/Rx transaction sizes in accordance with the + * platform setup. Note values exceeding maximal or minimal limits will + * be automatically clamped. Also note the register isn't affected by + * the HBA global reset so we can freely initialize it once until the + * next system reset. + */ + for_each_child_of_node(dpriv->pdev->dev.of_node, child) { + if (!of_device_is_available(child)) + continue; + + if (of_property_read_u32(child, "reg", &port)) { + of_node_put(child); + return -EINVAL; + } + + port_mmio = __ahci_port_base(hpriv, port); + dmacr = readl(port_mmio + AHCI_DWC_PORT_DMACR); + + if (!of_property_read_u32(child, "snps,tx-ts-max", &ts)) { + ts = ilog2(ts); + dmacr &= ~AHCI_DWC_PORT_TXTS_MASK; + dmacr |= FIELD_PREP(AHCI_DWC_PORT_TXTS_MASK, ts); + } + + if (!of_property_read_u32(child, "snps,rx-ts-max", &ts)) { + ts = ilog2(ts); + dmacr &= ~AHCI_DWC_PORT_RXTS_MASK; + dmacr |= FIELD_PREP(AHCI_DWC_PORT_RXTS_MASK, ts); + } + + writel(dmacr, port_mmio + AHCI_DWC_PORT_DMACR); + dpriv->dmacr[port] = dmacr; + } + + return 0; +} + +static int ahci_dwc_init_host(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + int rc; + + rc = ahci_platform_enable_resources(hpriv); + if (rc) + return rc; + + if (dpriv->pdata->init) { + rc = dpriv->pdata->init(hpriv); + if (rc) + goto err_disable_resources; + } + + ahci_dwc_check_cap(hpriv); + + ahci_dwc_init_timer(hpriv); + + rc = ahci_dwc_init_dmacr(hpriv); + if (rc) + goto err_clear_platform; + + return 0; + +err_clear_platform: + if (dpriv->pdata->clear) + dpriv->pdata->clear(hpriv); + +err_disable_resources: + ahci_platform_disable_resources(hpriv); + + return rc; +} + +static int ahci_dwc_reinit_host(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + unsigned long port_map = hpriv->port_map; + void __iomem *port_mmio; + int i, rc; + + rc = ahci_platform_enable_resources(hpriv); + if (rc) + return rc; + + if (dpriv->pdata->reinit) { + rc = dpriv->pdata->reinit(hpriv); + if (rc) + goto err_disable_resources; + } + + writel(dpriv->timv, hpriv->mmio + AHCI_DWC_HOST_TIMER1MS); + + for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) { + port_mmio = __ahci_port_base(hpriv, i); + writel(dpriv->dmacr[i], port_mmio + AHCI_DWC_PORT_DMACR); + } + + return 0; + +err_disable_resources: + ahci_platform_disable_resources(hpriv); + + return rc; +} + +static void ahci_dwc_clear_host(struct ahci_host_priv *hpriv) +{ + struct ahci_dwc_host_priv *dpriv = hpriv->plat_data; + + if (dpriv->pdata->clear) + dpriv->pdata->clear(hpriv); + + ahci_platform_disable_resources(hpriv); +} + +static void ahci_dwc_stop_host(struct ata_host *host) +{ + struct ahci_host_priv *hpriv = host->private_data; + + ahci_dwc_clear_host(hpriv); +} + +static struct ata_port_operations ahci_dwc_port_ops = { + .inherits = &ahci_platform_ops, + .host_stop = ahci_dwc_stop_host, +}; + +static const struct ata_port_info ahci_dwc_port_info = { + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_dwc_port_ops, +}; + +static struct scsi_host_template ahci_dwc_scsi_info = { + AHCI_SHT(DRV_NAME), +}; + +static int ahci_dwc_probe(struct platform_device *pdev) +{ + struct ahci_host_priv *hpriv; + int rc; + + hpriv = ahci_dwc_get_resources(pdev); + if (IS_ERR(hpriv)) + return PTR_ERR(hpriv); + + rc = ahci_dwc_init_host(hpriv); + if (rc) + return rc; + + rc = ahci_platform_init_host(pdev, hpriv, &ahci_dwc_port_info, + &ahci_dwc_scsi_info); + if (rc) + goto err_clear_host; + + return 0; + +err_clear_host: + ahci_dwc_clear_host(hpriv); + + return rc; +} + +static int ahci_dwc_suspend(struct device *dev) +{ + struct ata_host *host = dev_get_drvdata(dev); + struct ahci_host_priv *hpriv = host->private_data; + int rc; + + rc = ahci_platform_suspend_host(dev); + if (rc) + return rc; + + ahci_dwc_clear_host(hpriv); + + return 0; +} + +static int ahci_dwc_resume(struct device *dev) +{ + struct ata_host *host = dev_get_drvdata(dev); + struct ahci_host_priv *hpriv = host->private_data; + int rc; + + rc = ahci_dwc_reinit_host(hpriv); + if (rc) + return rc; + + return ahci_platform_resume_host(dev); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(ahci_dwc_pm_ops, ahci_dwc_suspend, + ahci_dwc_resume); + +static struct ahci_dwc_plat_data ahci_dwc_plat = { + .pflags = AHCI_PLATFORM_GET_RESETS, +}; + +static struct ahci_dwc_plat_data ahci_bt1_plat = { + .pflags = AHCI_PLATFORM_GET_RESETS | AHCI_PLATFORM_RST_TRIGGER, + .init = ahci_bt1_init, +}; + +static const struct of_device_id ahci_dwc_of_match[] = { + { .compatible = "snps,dwc-ahci", &ahci_dwc_plat }, + { .compatible = "snps,spear-ahci", &ahci_dwc_plat }, + { .compatible = "baikal,bt1-ahci", &ahci_bt1_plat }, + {}, +}; +MODULE_DEVICE_TABLE(of, ahci_dwc_of_match); + +static struct platform_driver ahci_dwc_driver = { + .probe = ahci_dwc_probe, + .remove = ata_platform_remove_one, + .shutdown = ahci_platform_shutdown, + .driver = { + .name = DRV_NAME, + .of_match_table = ahci_dwc_of_match, + .pm = &ahci_dwc_pm_ops, + }, +}; +module_platform_driver(ahci_dwc_driver); + +MODULE_DESCRIPTION("DWC AHCI SATA platform driver"); +MODULE_AUTHOR("Serge Semin "); +MODULE_LICENSE("GPL"); diff --git a/drivers/ata/ahci_imx.c b/drivers/ata/ahci_imx.c index 79aa9f2853129f6ead661865d47bfefc84fb634d..b734e069034d2a3e4ad80e9cf3eadc426fd58e0d 100644 --- a/drivers/ata/ahci_imx.c +++ b/drivers/ata/ahci_imx.c @@ -327,7 +327,7 @@ static int read_adc_sum(void *dev, u16 rtune_ctl_reg, void __iomem * mmio) } /* SATA AHCI temperature monitor */ -static int sata_ahci_read_temperature(void *dev, int *temp) +static int __sata_ahci_read_temperature(void *dev, int *temp) { u16 mpll_test_reg, rtune_ctl_reg, dac_ctl_reg, read_sum; u32 str1, str2, str3, str4; @@ -416,6 +416,11 @@ static int sata_ahci_read_temperature(void *dev, int *temp) return 0; } +static int sata_ahci_read_temperature(struct thermal_zone_device *tz, int *temp) +{ + return __sata_ahci_read_temperature(tz->devdata, temp); +} + static ssize_t sata_ahci_show_temp(struct device *dev, struct device_attribute *da, char *buf) @@ -423,14 +428,14 @@ static ssize_t sata_ahci_show_temp(struct device *dev, unsigned int temp = 0; int err; - err = sata_ahci_read_temperature(dev, &temp); + err = __sata_ahci_read_temperature(dev, &temp); if (err < 0) return err; return sprintf(buf, "%u\n", temp); } -static const struct thermal_zone_of_device_ops fsl_sata_ahci_of_thermal_ops = { +static const struct thermal_zone_device_ops fsl_sata_ahci_of_thermal_ops = { .get_temp = sata_ahci_read_temperature, }; @@ -1131,8 +1136,8 @@ static int imx_ahci_probe(struct platform_device *pdev) ret = PTR_ERR(hwmon_dev); goto disable_clk; } - devm_thermal_zone_of_sensor_register(hwmon_dev, 0, hwmon_dev, - &fsl_sata_ahci_of_thermal_ops); + devm_thermal_of_zone_register(hwmon_dev, 0, hwmon_dev, + &fsl_sata_ahci_of_thermal_ops); dev_info(dev, "%s: sensor 'sata_ahci'\n", dev_name(hwmon_dev)); } diff --git a/drivers/ata/ahci_mtk.c b/drivers/ata/ahci_mtk.c index 1f6c85fde9830ac1334fa4991dcb009a09d6897e..c056378e3e721ed4600cc6845afcfb1dea073edf 100644 --- a/drivers/ata/ahci_mtk.c +++ b/drivers/ata/ahci_mtk.c @@ -118,8 +118,6 @@ static int mtk_ahci_parse_property(struct ahci_host_priv *hpriv, SYS_CFG_SATA_EN); } - of_property_read_u32(np, "ports-implemented", &hpriv->force_port_map); - return 0; } diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c index 28a8de5b48b979c9b733d2e35e5025357d7ee58a..8f5572a9f8f196328a6172e6db599124e3499a1d 100644 --- a/drivers/ata/ahci_platform.c +++ b/drivers/ata/ahci_platform.c @@ -56,9 +56,6 @@ static int ahci_probe(struct platform_device *pdev) if (rc) return rc; - of_property_read_u32(dev->of_node, - "ports-implemented", &hpriv->force_port_map); - if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci")) hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ; @@ -83,9 +80,7 @@ static SIMPLE_DEV_PM_OPS(ahci_pm_ops, ahci_platform_suspend, static const struct of_device_id ahci_of_match[] = { { .compatible = "generic-ahci", }, /* Keep the following compatibles for device tree compatibility */ - { .compatible = "snps,spear-ahci", }, { .compatible = "ibm,476gtr-ahci", }, - { .compatible = "snps,dwc-ahci", }, { .compatible = "hisilicon,hisi-ahci", }, { .compatible = "cavium,octeon-7130-ahci", }, { /* sentinel */ } diff --git a/drivers/ata/ahci_st.c b/drivers/ata/ahci_st.c index 7526653c843b32261b8f7f3d4e263ce01c58b9b5..5a2cac60a29adda1ff2a894b6072ee87fafe0b20 100644 --- a/drivers/ata/ahci_st.c +++ b/drivers/ata/ahci_st.c @@ -144,7 +144,6 @@ static struct scsi_host_template ahci_platform_sht = { static int st_ahci_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; struct st_ahci_drv_data *drv_data; struct ahci_host_priv *hpriv; int err; @@ -168,9 +167,6 @@ static int st_ahci_probe(struct platform_device *pdev) st_ahci_configure_oob(hpriv->mmio); - of_property_read_u32(dev->of_node, - "ports-implemented", &hpriv->force_port_map); - err = ahci_platform_init_host(pdev, hpriv, &st_ahci_port_info, &ahci_platform_sht); if (err) { diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index cf8c7fd59ada5be255cc871a6339b65992e87de2..954386a2b5002b9e37401dacbfa1852cb448cf9c 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -16,6 +16,7 @@ * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf */ +#include #include #include #include @@ -443,17 +444,28 @@ static ssize_t ahci_show_em_supported(struct device *dev, void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) { void __iomem *mmio = hpriv->mmio; - u32 cap, cap2, vers, port_map; + void __iomem *port_mmio; + unsigned long port_map; + u32 cap, cap2, vers; int i; /* make sure AHCI mode is enabled before accessing CAP */ ahci_enable_ahci(mmio); - /* Values prefixed with saved_ are written back to host after - * reset. Values without are used for driver operation. + /* + * Values prefixed with saved_ are written back to the HBA and ports + * registers after reset. Values without are used for driver operation. + */ + + /* + * Override HW-init HBA capability fields with the platform-specific + * values. The rest of the HBA capabilities are defined as Read-only + * and can't be modified in CSR anyway. */ - hpriv->saved_cap = cap = readl(mmio + HOST_CAP); - hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL); + cap = readl(mmio + HOST_CAP); + if (hpriv->saved_cap) + cap = (cap & ~(HOST_CAP_SSS | HOST_CAP_MPS)) | hpriv->saved_cap; + hpriv->saved_cap = cap; /* CAP2 register is only defined for AHCI 1.2 and later */ vers = readl(mmio + HOST_VERSION); @@ -517,15 +529,18 @@ void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) cap &= ~HOST_CAP_SXS; } - if (hpriv->force_port_map && port_map != hpriv->force_port_map) { - dev_info(dev, "forcing port_map 0x%x -> 0x%x\n", - port_map, hpriv->force_port_map); - port_map = hpriv->force_port_map; + /* Override the HBA ports mapping if the platform needs it */ + port_map = readl(mmio + HOST_PORTS_IMPL); + if (hpriv->saved_port_map && port_map != hpriv->saved_port_map) { + dev_info(dev, "forcing port_map 0x%lx -> 0x%x\n", + port_map, hpriv->saved_port_map); + port_map = hpriv->saved_port_map; + } else { hpriv->saved_port_map = port_map; } if (hpriv->mask_port_map) { - dev_warn(dev, "masking port_map 0x%x -> 0x%x\n", + dev_warn(dev, "masking port_map 0x%lx -> 0x%lx\n", port_map, port_map & hpriv->mask_port_map); port_map &= hpriv->mask_port_map; @@ -544,7 +559,7 @@ void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) */ if (map_ports > ahci_nr_ports(cap)) { dev_warn(dev, - "implemented port map (0x%x) contains more ports than nr_ports (%u), using nr_ports\n", + "implemented port map (0x%lx) contains more ports than nr_ports (%u), using nr_ports\n", port_map, ahci_nr_ports(cap)); port_map = 0; } @@ -553,16 +568,30 @@ void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) /* fabricate port_map from cap.nr_ports for < AHCI 1.3 */ if (!port_map && vers < 0x10300) { port_map = (1 << ahci_nr_ports(cap)) - 1; - dev_warn(dev, "forcing PORTS_IMPL to 0x%x\n", port_map); + dev_warn(dev, "forcing PORTS_IMPL to 0x%lx\n", port_map); /* write the fixed up value to the PI register */ hpriv->saved_port_map = port_map; } + /* + * Preserve the ports capabilities defined by the platform. Note there + * is no need in storing the rest of the P#.CMD fields since they are + * volatile. + */ + for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) { + if (hpriv->saved_port_cap[i]) + continue; + + port_mmio = __ahci_port_base(hpriv, i); + hpriv->saved_port_cap[i] = + readl(port_mmio + PORT_CMD) & PORT_CMD_CAP; + } + /* record values to use during operation */ hpriv->cap = cap; hpriv->cap2 = cap2; - hpriv->version = readl(mmio + HOST_VERSION); + hpriv->version = vers; hpriv->port_map = port_map; if (!hpriv->start_engine) @@ -588,13 +617,21 @@ EXPORT_SYMBOL_GPL(ahci_save_initial_config); static void ahci_restore_initial_config(struct ata_host *host) { struct ahci_host_priv *hpriv = host->private_data; + unsigned long port_map = hpriv->port_map; void __iomem *mmio = hpriv->mmio; + void __iomem *port_mmio; + int i; writel(hpriv->saved_cap, mmio + HOST_CAP); if (hpriv->saved_cap2) writel(hpriv->saved_cap2, mmio + HOST_CAP2); writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL); (void) readl(mmio + HOST_PORTS_IMPL); /* flush */ + + for_each_set_bit(i, &port_map, AHCI_MAX_PORTS) { + port_mmio = __ahci_port_base(hpriv, i); + writel(hpriv->saved_port_cap[i], port_mmio + PORT_CMD); + } } static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg) diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index 32495ae96567ab244fa116adc7a1ee4c70fed9f5..ddf17e2d266c363c18614a7a72b42a2b02790319 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -94,31 +94,41 @@ void ahci_platform_disable_phys(struct ahci_host_priv *hpriv) EXPORT_SYMBOL_GPL(ahci_platform_disable_phys); /** - * ahci_platform_enable_clks - Enable platform clocks + * ahci_platform_find_clk - Find platform clock * @hpriv: host private area to store config values + * @con_id: clock connection ID * - * This function enables all the clks found in hpriv->clks, starting at - * index 0. If any clk fails to enable it disables all the clks already - * enabled in reverse order, and then returns an error. + * This function returns a pointer to the clock descriptor of the clock with + * the passed ID. * * RETURNS: - * 0 on success otherwise a negative error code + * Pointer to the clock descriptor on success otherwise NULL */ -int ahci_platform_enable_clks(struct ahci_host_priv *hpriv) +struct clk *ahci_platform_find_clk(struct ahci_host_priv *hpriv, const char *con_id) { - int c, rc; + int i; - for (c = 0; c < AHCI_MAX_CLKS && hpriv->clks[c]; c++) { - rc = clk_prepare_enable(hpriv->clks[c]); - if (rc) - goto disable_unprepare_clk; + for (i = 0; i < hpriv->n_clks; i++) { + if (!strcmp(hpriv->clks[i].id, con_id)) + return hpriv->clks[i].clk; } - return 0; -disable_unprepare_clk: - while (--c >= 0) - clk_disable_unprepare(hpriv->clks[c]); - return rc; + return NULL; +} +EXPORT_SYMBOL_GPL(ahci_platform_find_clk); + +/** + * ahci_platform_enable_clks - Enable platform clocks + * @hpriv: host private area to store config values + * + * This function enables all the clks found for the AHCI device. + * + * RETURNS: + * 0 on success otherwise a negative error code + */ +int ahci_platform_enable_clks(struct ahci_host_priv *hpriv) +{ + return clk_bulk_prepare_enable(hpriv->n_clks, hpriv->clks); } EXPORT_SYMBOL_GPL(ahci_platform_enable_clks); @@ -126,19 +136,54 @@ EXPORT_SYMBOL_GPL(ahci_platform_enable_clks); * ahci_platform_disable_clks - Disable platform clocks * @hpriv: host private area to store config values * - * This function disables all the clks found in hpriv->clks, in reverse - * order of ahci_platform_enable_clks (starting at the end of the array). + * This function disables all the clocks enabled before + * (bulk-clocks-disable function is supposed to do that in reverse + * from the enabling procedure order). */ void ahci_platform_disable_clks(struct ahci_host_priv *hpriv) { - int c; - - for (c = AHCI_MAX_CLKS - 1; c >= 0; c--) - if (hpriv->clks[c]) - clk_disable_unprepare(hpriv->clks[c]); + clk_bulk_disable_unprepare(hpriv->n_clks, hpriv->clks); } EXPORT_SYMBOL_GPL(ahci_platform_disable_clks); +/** + * ahci_platform_deassert_rsts - Deassert/trigger platform resets + * @hpriv: host private area to store config values + * + * This function deasserts or triggers all the reset lines found for + * the AHCI device. + * + * RETURNS: + * 0 on success otherwise a negative error code + */ +int ahci_platform_deassert_rsts(struct ahci_host_priv *hpriv) +{ + if (hpriv->f_rsts & AHCI_PLATFORM_RST_TRIGGER) + return reset_control_reset(hpriv->rsts); + + return reset_control_deassert(hpriv->rsts); +} +EXPORT_SYMBOL_GPL(ahci_platform_deassert_rsts); + +/** + * ahci_platform_assert_rsts - Assert/rearm platform resets + * @hpriv: host private area to store config values + * + * This function asserts or rearms (for self-deasserting resets) all + * the reset controls found for the AHCI device. + * + * RETURNS: + * 0 on success otherwise a negative error code + */ +int ahci_platform_assert_rsts(struct ahci_host_priv *hpriv) +{ + if (hpriv->f_rsts & AHCI_PLATFORM_RST_TRIGGER) + return reset_control_rearm(hpriv->rsts); + + return reset_control_assert(hpriv->rsts); +} +EXPORT_SYMBOL_GPL(ahci_platform_assert_rsts); + /** * ahci_platform_enable_regulators - Enable regulators * @hpriv: host private area to store config values @@ -236,18 +281,18 @@ int ahci_platform_enable_resources(struct ahci_host_priv *hpriv) if (rc) goto disable_regulator; - rc = reset_control_deassert(hpriv->rsts); + rc = ahci_platform_deassert_rsts(hpriv); if (rc) goto disable_clks; rc = ahci_platform_enable_phys(hpriv); if (rc) - goto disable_resets; + goto disable_rsts; return 0; -disable_resets: - reset_control_assert(hpriv->rsts); +disable_rsts: + ahci_platform_assert_rsts(hpriv); disable_clks: ahci_platform_disable_clks(hpriv); @@ -274,7 +319,7 @@ void ahci_platform_disable_resources(struct ahci_host_priv *hpriv) { ahci_platform_disable_phys(hpriv); - reset_control_assert(hpriv->rsts); + ahci_platform_assert_rsts(hpriv); ahci_platform_disable_clks(hpriv); @@ -292,8 +337,6 @@ static void ahci_platform_put_resources(struct device *dev, void *res) pm_runtime_disable(dev); } - for (c = 0; c < AHCI_MAX_CLKS && hpriv->clks[c]; c++) - clk_put(hpriv->clks[c]); /* * The regulators are tied to child node device and not to the * SATA device itself. So we can't use devm for automatically @@ -363,6 +406,34 @@ static int ahci_platform_get_regulator(struct ahci_host_priv *hpriv, u32 port, return rc; } +static int ahci_platform_get_firmware(struct ahci_host_priv *hpriv, + struct device *dev) +{ + struct device_node *child; + u32 port; + + if (!of_property_read_u32(dev->of_node, "hba-cap", &hpriv->saved_cap)) + hpriv->saved_cap &= (HOST_CAP_SSS | HOST_CAP_MPS); + + of_property_read_u32(dev->of_node, + "ports-implemented", &hpriv->saved_port_map); + + for_each_child_of_node(dev->of_node, child) { + if (!of_device_is_available(child)) + continue; + + if (of_property_read_u32(child, "reg", &port)) { + of_node_put(child); + return -EINVAL; + } + + if (!of_property_read_u32(child, "hba-port-cap", &hpriv->saved_port_cap[port])) + hpriv->saved_port_cap[port] &= PORT_CMD_CAP; + } + + return 0; +} + /** * ahci_platform_get_resources - Get platform resources * @pdev: platform device to get resources for @@ -374,8 +445,8 @@ static int ahci_platform_get_regulator(struct ahci_host_priv *hpriv, u32 port, * 1) mmio registers (IORESOURCE_MEM 0, mandatory) * 2) regulator for controlling the targets power (optional) * regulator for controlling the AHCI controller (optional) - * 3) 0 - AHCI_MAX_CLKS clocks, as specified in the devs devicetree node, - * or for non devicetree enabled platforms a single clock + * 3) all clocks specified in the devicetree node, or a single + * clock for non-OF platforms (optional) * 4) resets, if flags has AHCI_PLATFORM_GET_RESETS (optional) * 5) phys (optional) * @@ -385,11 +456,10 @@ static int ahci_platform_get_regulator(struct ahci_host_priv *hpriv, u32 port, struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, unsigned int flags) { + int child_nodes, rc = -ENOMEM, enabled_ports = 0; struct device *dev = &pdev->dev; struct ahci_host_priv *hpriv; - struct clk *clk; struct device_node *child; - int i, enabled_ports = 0, rc = -ENOMEM, child_nodes; u32 mask_port_map = 0; if (!devres_open_group(dev, NULL, GFP_KERNEL)) @@ -402,32 +472,51 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, devres_add(dev, hpriv); - hpriv->mmio = devm_ioremap_resource(dev, - platform_get_resource(pdev, IORESOURCE_MEM, 0)); + /* + * If the DT provided an "ahci" named resource, use it. Otherwise, + * fallback to using the default first resource for the device node. + */ + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "ahci")) + hpriv->mmio = devm_platform_ioremap_resource_byname(pdev, "ahci"); + else + hpriv->mmio = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(hpriv->mmio)) { rc = PTR_ERR(hpriv->mmio); goto err_out; } - for (i = 0; i < AHCI_MAX_CLKS; i++) { + /* + * Bulk clocks getting procedure can fail to find any clock due to + * running on a non-OF platform or due to the clocks being defined in + * bypass of the DT firmware (like da850, spear13xx). In that case we + * fallback to getting a single clock source right from the dev clocks + * list. + */ + rc = devm_clk_bulk_get_all(dev, &hpriv->clks); + if (rc < 0) + goto err_out; + + if (rc > 0) { + /* Got clocks in bulk */ + hpriv->n_clks = rc; + } else { /* - * For now we must use clk_get(dev, NULL) for the first clock, - * because some platforms (da850, spear13xx) are not yet - * converted to use devicetree for clocks. For new platforms - * this is equivalent to of_clk_get(dev->of_node, 0). + * No clock bulk found: fallback to manually getting + * the optional clock. */ - if (i == 0) - clk = clk_get(dev, NULL); - else - clk = of_clk_get(dev->of_node, i); - - if (IS_ERR(clk)) { - rc = PTR_ERR(clk); - if (rc == -EPROBE_DEFER) - goto err_out; - break; + hpriv->clks = devm_kzalloc(dev, sizeof(*hpriv->clks), GFP_KERNEL); + if (!hpriv->clks) { + rc = -ENOMEM; + goto err_out; + } + hpriv->clks->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(hpriv->clks->clk)) { + rc = PTR_ERR(hpriv->clks->clk); + goto err_out; + } else if (hpriv->clks->clk) { + hpriv->clks->id = "ahci"; + hpriv->n_clks = 1; } - hpriv->clks[i] = clk; } hpriv->ahci_regulator = devm_regulator_get(dev, "ahci"); @@ -449,16 +538,28 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, rc = PTR_ERR(hpriv->rsts); goto err_out; } + + hpriv->f_rsts = flags & AHCI_PLATFORM_RST_TRIGGER; } - hpriv->nports = child_nodes = of_get_child_count(dev->of_node); + /* + * Too many sub-nodes most likely means having something wrong with + * the firmware. + */ + child_nodes = of_get_child_count(dev->of_node); + if (child_nodes > AHCI_MAX_PORTS) { + rc = -EINVAL; + goto err_out; + } /* * If no sub-node was found, we still need to set nports to * one in order to be able to use the * ahci_platform_[en|dis]able_[phys|regulators] functions. */ - if (!child_nodes) + if (child_nodes) + hpriv->nports = child_nodes; + else hpriv->nports = 1; hpriv->phys = devm_kcalloc(dev, hpriv->nports, sizeof(*hpriv->phys), GFP_KERNEL); @@ -540,6 +641,15 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev, if (rc == -EPROBE_DEFER) goto err_out; } + + /* + * Retrieve firmware-specific flags which then will be used to set + * the HW-init fields of HBA and its ports + */ + rc = ahci_platform_get_firmware(hpriv, dev); + if (rc) + goto err_out; + pm_runtime_enable(dev); pm_runtime_get_sync(dev); hpriv->got_runtime_pm = true; diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 826d41f341e4afc05843ff8044fcc477cd862f54..d3ce5c383f3a7c43612114cebb4312ff9395983f 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -665,33 +665,33 @@ u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev) /** * ata_build_rw_tf - Build ATA taskfile for given read/write request - * @tf: Target ATA taskfile - * @dev: ATA device @tf belongs to + * @qc: Metadata associated with the taskfile to build * @block: Block address * @n_block: Number of blocks * @tf_flags: RW/FUA etc... - * @tag: tag * @class: IO priority class * * LOCKING: * None. * - * Build ATA taskfile @tf for read/write request described by - * @block, @n_block, @tf_flags and @tag on @dev. + * Build ATA taskfile for the command @qc for read/write request described + * by @block, @n_block, @tf_flags and @class. * * RETURNS: * * 0 on success, -ERANGE if the request is too large for @dev, * -EINVAL if the request is invalid. */ -int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, - u64 block, u32 n_block, unsigned int tf_flags, - unsigned int tag, int class) +int ata_build_rw_tf(struct ata_queued_cmd *qc, u64 block, u32 n_block, + unsigned int tf_flags, int class) { + struct ata_taskfile *tf = &qc->tf; + struct ata_device *dev = qc->dev; + tf->flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE; tf->flags |= tf_flags; - if (ata_ncq_enabled(dev) && !ata_tag_internal(tag)) { + if (ata_ncq_enabled(dev)) { /* yay, NCQ */ if (!lba_48_ok(block, n_block)) return -ERANGE; @@ -704,7 +704,7 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, else tf->command = ATA_CMD_FPDMA_READ; - tf->nsect = tag << 3; + tf->nsect = qc->hw_tag << 3; tf->hob_feature = (n_block >> 8) & 0xff; tf->feature = n_block & 0xff; @@ -719,7 +719,7 @@ int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, if (tf->flags & ATA_TFLAG_FUA) tf->device |= 1 << 7; - if (dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE && + if (dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLED && class == IOPRIO_CLASS_RT) tf->hob_nsect |= ATA_PRIO_HIGH << ATA_SHIFT_PRIO; } else if (dev->flags & ATA_DFLAG_LBA) { @@ -1578,8 +1578,8 @@ static unsigned ata_exec_internal_sg(struct ata_device *dev, else ata_qc_complete(qc); - ata_dev_warn(dev, "qc timeout (cmd 0x%x)\n", - command); + ata_dev_warn(dev, "qc timeout after %u msecs (cmd 0x%x)\n", + timeout, command); } spin_unlock_irqrestore(ap->lock, flags); @@ -2171,7 +2171,7 @@ static void ata_dev_config_ncq_prio(struct ata_device *dev) return; not_supported: - dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE; + dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLED; dev->flags &= ~ATA_DFLAG_NCQ_PRIO; } @@ -3021,7 +3021,8 @@ static void sata_print_link_status(struct ata_link *link) if (sata_scr_read(link, SCR_STATUS, &sstatus)) return; - sata_scr_read(link, SCR_CONTROL, &scontrol); + if (sata_scr_read(link, SCR_CONTROL, &scontrol)) + return; if (ata_phys_link_online(link)) { tmp = (sstatus >> 4) & 0xf; @@ -3988,6 +3989,10 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { { "PIONEER DVD-RW DVR-212D", NULL, ATA_HORKAGE_NOSETXFER }, { "PIONEER DVD-RW DVR-216D", NULL, ATA_HORKAGE_NOSETXFER }, + /* These specific Pioneer models have LPM issues */ + { "PIONEER BD-RW BDR-207M", NULL, ATA_HORKAGE_NOLPM }, + { "PIONEER BD-RW BDR-205", NULL, ATA_HORKAGE_NOLPM }, + /* Crucial BX100 SSD 500GB has broken LPM support */ { "CT500BX100SSD1", NULL, ATA_HORKAGE_NOLPM }, @@ -4295,7 +4300,6 @@ static void ata_dev_xfermask(struct ata_device *dev) static unsigned int ata_dev_set_xfermode(struct ata_device *dev) { struct ata_taskfile tf; - unsigned int err_mask; /* set up set-features taskfile */ ata_dev_dbg(dev, "set features - xfer mode\n"); @@ -4317,20 +4321,20 @@ static unsigned int ata_dev_set_xfermode(struct ata_device *dev) else /* In the ancient relic department - skip all of this */ return 0; - /* On some disks, this command causes spin-up, so we need longer timeout */ - err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 15000); - - return err_mask; + /* + * On some disks, this command causes spin-up, so we need longer + * timeout. + */ + return ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, 15000); } /** - * ata_dev_set_feature - Issue SET FEATURES - SATA FEATURES + * ata_dev_set_feature - Issue SET FEATURES * @dev: Device to which command will be sent - * @enable: Whether to enable or disable the feature - * @feature: The sector count represents the feature to set + * @subcmd: The SET FEATURES subcommand to be sent + * @action: The sector count represents a subcommand specific action * - * Issue SET FEATURES - SATA FEATURES command to device @dev - * on port @ap with sector count + * Issue SET FEATURES command to device @dev on port @ap with sector count * * LOCKING: * PCI/etc. bus probe sem. @@ -4338,28 +4342,26 @@ static unsigned int ata_dev_set_xfermode(struct ata_device *dev) * RETURNS: * 0 on success, AC_ERR_* mask otherwise. */ -unsigned int ata_dev_set_feature(struct ata_device *dev, u8 enable, u8 feature) +unsigned int ata_dev_set_feature(struct ata_device *dev, u8 subcmd, u8 action) { struct ata_taskfile tf; - unsigned int err_mask; unsigned int timeout = 0; /* set up set-features taskfile */ - ata_dev_dbg(dev, "set features - SATA features\n"); + ata_dev_dbg(dev, "set features\n"); ata_tf_init(dev, &tf); tf.command = ATA_CMD_SET_FEATURES; - tf.feature = enable; + tf.feature = subcmd; tf.flags |= ATA_TFLAG_ISADDR | ATA_TFLAG_DEVICE; tf.protocol = ATA_PROT_NODATA; - tf.nsect = feature; + tf.nsect = action; - if (enable == SETFEATURES_SPINUP) + if (subcmd == SETFEATURES_SPINUP) timeout = ata_probe_timeout ? ata_probe_timeout * 1000 : SETFEATURES_SPINUP_TIMEOUT; - err_mask = ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, timeout); - return err_mask; + return ata_exec_internal(dev, &tf, NULL, DMA_NONE, NULL, 0, timeout); } EXPORT_SYMBOL_GPL(ata_dev_set_feature); diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 7c128c89b45461ac51959293932c0158fa0d0698..08e11bc312c28ede474ff5e48f2bb89281465f17 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -151,6 +151,8 @@ ata_eh_cmd_timeout_table[ATA_EH_CMD_TIMEOUT_TABLE_SIZE] = { #undef CMDS static void __ata_port_freeze(struct ata_port *ap); +static int ata_eh_set_lpm(struct ata_link *link, enum ata_lpm_policy policy, + struct ata_device **r_failed_dev); #ifdef CONFIG_PM static void ata_eh_handle_port_suspend(struct ata_port *ap); static void ata_eh_handle_port_resume(struct ata_port *ap); @@ -1086,14 +1088,11 @@ static void __ata_port_freeze(struct ata_port *ap) */ int ata_port_freeze(struct ata_port *ap) { - int nr_aborted; - WARN_ON(!ap->ops->error_handler); __ata_port_freeze(ap); - nr_aborted = ata_port_abort(ap); - return nr_aborted; + return ata_port_abort(ap); } EXPORT_SYMBOL_GPL(ata_port_freeze); @@ -1393,7 +1392,6 @@ unsigned int atapi_eh_tur(struct ata_device *dev, u8 *r_sense_key) /** * ata_eh_request_sense - perform REQUEST_SENSE_DATA_EXT * @qc: qc to perform REQUEST_SENSE_SENSE_DATA_EXT to - * @cmd: scsi command for which the sense code should be set * * Perform REQUEST_SENSE_DATA_EXT after the device reported CHECK * SENSE. This function is an EH helper. @@ -1401,9 +1399,9 @@ unsigned int atapi_eh_tur(struct ata_device *dev, u8 *r_sense_key) * LOCKING: * Kernel thread context (may sleep). */ -static void ata_eh_request_sense(struct ata_queued_cmd *qc, - struct scsi_cmnd *cmd) +static void ata_eh_request_sense(struct ata_queued_cmd *qc) { + struct scsi_cmnd *cmd = qc->scsicmd; struct ata_device *dev = qc->dev; struct ata_taskfile tf; unsigned int err_mask; @@ -1541,7 +1539,6 @@ static void ata_eh_analyze_serror(struct ata_link *link) /** * ata_eh_analyze_tf - analyze taskfile of a failed qc * @qc: qc to analyze - * @tf: Taskfile registers to analyze * * Analyze taskfile of @qc and further determine cause of * failure. This function also requests ATAPI sense data if @@ -1553,9 +1550,9 @@ static void ata_eh_analyze_serror(struct ata_link *link) * RETURNS: * Determined recovery action */ -static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc, - const struct ata_taskfile *tf) +static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc) { + const struct ata_taskfile *tf = &qc->result_tf; unsigned int tmp, action = 0; u8 stat = tf->status, err = tf->error; @@ -1579,7 +1576,7 @@ static unsigned int ata_eh_analyze_tf(struct ata_queued_cmd *qc, switch (qc->dev->class) { case ATA_DEV_ZAC: if (stat & ATA_SENSE) - ata_eh_request_sense(qc, qc->scsicmd); + ata_eh_request_sense(qc); fallthrough; case ATA_DEV_ATA: if (err & ATA_ICRC) @@ -1957,7 +1954,7 @@ static void ata_eh_link_autopsy(struct ata_link *link) qc->err_mask |= ehc->i.err_mask; /* analyze TF */ - ehc->i.action |= ata_eh_analyze_tf(qc, &qc->result_tf); + ehc->i.action |= ata_eh_analyze_tf(qc); /* DEV errors are probably spurious in case of ATA_BUS error */ if (qc->err_mask & AC_ERR_ATA_BUS) @@ -2940,6 +2937,23 @@ static int ata_eh_revalidate_and_attach(struct ata_link *link, if ((action & ATA_EH_REVALIDATE) && ata_dev_enabled(dev)) { WARN_ON(dev->class == ATA_DEV_PMP); + /* + * The link may be in a deep sleep, wake it up. + * + * If the link is in deep sleep, ata_phys_link_offline() + * will return true, causing the revalidation to fail, + * which leads to a (potentially) needless hard reset. + * + * ata_eh_recover() will later restore the link policy + * to ap->target_lpm_policy after revalidation is done. + */ + if (link->lpm_policy > ATA_LPM_MAX_POWER) { + rc = ata_eh_set_lpm(link, ATA_LPM_MAX_POWER, + r_failed_dev); + if (rc) + goto err; + } + if (ata_phys_link_offline(ata_dev_phys_link(dev))) { rc = -EIO; goto err; diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index 7a5fe41aa5ae10eda99cf10c3f8d895de4995f3c..b6806d41a8c508fd4e330e5878d9498f736b543d 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -870,7 +870,7 @@ static ssize_t ata_ncq_prio_enable_show(struct device *device, if (!dev) rc = -ENODEV; else - ncq_prio_enable = dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLE; + ncq_prio_enable = dev->flags & ATA_DFLAG_NCQ_PRIO_ENABLED; spin_unlock_irq(ap->lock); return rc ? rc : sysfs_emit(buf, "%u\n", ncq_prio_enable); @@ -905,9 +905,9 @@ static ssize_t ata_ncq_prio_enable_store(struct device *device, } if (input) - dev->flags |= ATA_DFLAG_NCQ_PRIO_ENABLE; + dev->flags |= ATA_DFLAG_NCQ_PRIO_ENABLED; else - dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLE; + dev->flags &= ~ATA_DFLAG_NCQ_PRIO_ENABLED; unlock: spin_unlock_irq(ap->lock); @@ -1018,26 +1018,25 @@ DEVICE_ATTR(sw_activity, S_IWUSR | S_IRUGO, ata_scsi_activity_show, EXPORT_SYMBOL_GPL(dev_attr_sw_activity); /** - * __ata_change_queue_depth - helper for ata_scsi_change_queue_depth - * @ap: ATA port to which the device change the queue depth + * ata_change_queue_depth - Set a device maximum queue depth + * @ap: ATA port of the target device + * @dev: target ATA device * @sdev: SCSI device to configure queue depth for * @queue_depth: new queue depth * - * libsas and libata have different approaches for associating a sdev to - * its ata_port. + * Helper to set a device maximum queue depth, usable with both libsas + * and libata. * */ -int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev, - int queue_depth) +int ata_change_queue_depth(struct ata_port *ap, struct ata_device *dev, + struct scsi_device *sdev, int queue_depth) { - struct ata_device *dev; unsigned long flags; - if (queue_depth < 1 || queue_depth == sdev->queue_depth) + if (!dev || !ata_dev_enabled(dev)) return sdev->queue_depth; - dev = ata_scsi_find_dev(ap, sdev); - if (!dev || !ata_dev_enabled(dev)) + if (queue_depth < 1 || queue_depth == sdev->queue_depth) return sdev->queue_depth; /* NCQ enabled? */ @@ -1059,7 +1058,7 @@ int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev, return scsi_change_queue_depth(sdev, queue_depth); } -EXPORT_SYMBOL_GPL(__ata_change_queue_depth); +EXPORT_SYMBOL_GPL(ata_change_queue_depth); /** * ata_scsi_change_queue_depth - SCSI callback for queue depth config @@ -1080,7 +1079,8 @@ int ata_scsi_change_queue_depth(struct scsi_device *sdev, int queue_depth) { struct ata_port *ap = ata_shost_to_port(sdev->host); - return __ata_change_queue_depth(ap, sdev, queue_depth); + return ata_change_queue_depth(ap, ata_scsi_find_dev(ap, sdev), + sdev, queue_depth); } EXPORT_SYMBOL_GPL(ata_scsi_change_queue_depth); diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 29e2f55c6faa79b9ac1bacbd5bc4cb8c4bdf2b07..e2ebb0b065e27ebdd4ba38eeeb1934e818c693f0 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1055,6 +1055,7 @@ EXPORT_SYMBOL_GPL(ata_scsi_dma_need_drain); int ata_scsi_dev_config(struct scsi_device *sdev, struct ata_device *dev) { struct request_queue *q = sdev->request_queue; + int depth = 1; if (!ata_id_has_unload(dev->id)) dev->flags |= ATA_DFLAG_NO_UNLOAD; @@ -1100,13 +1101,10 @@ int ata_scsi_dev_config(struct scsi_device *sdev, struct ata_device *dev) if (dev->flags & ATA_DFLAG_AN) set_bit(SDEV_EVT_MEDIA_CHANGE, sdev->supported_events); - if (dev->flags & ATA_DFLAG_NCQ) { - int depth; - + if (dev->flags & ATA_DFLAG_NCQ) depth = min(sdev->host->can_queue, ata_id_queue_depth(dev->id)); - depth = min(ATA_MAX_QUEUE, depth); - scsi_change_queue_depth(sdev, depth); - } + depth = min(ATA_MAX_QUEUE, depth); + scsi_change_queue_depth(sdev, depth); if (dev->flags & ATA_DFLAG_TRUSTED) sdev->security_supported = 1; @@ -1605,9 +1603,7 @@ static unsigned int ata_scsi_rw_xlat(struct ata_queued_cmd *qc) qc->flags |= ATA_QCFLAG_IO; qc->nbytes = n_block * scmd->device->sector_size; - rc = ata_build_rw_tf(&qc->tf, qc->dev, block, n_block, tf_flags, - qc->hw_tag, class); - + rc = ata_build_rw_tf(qc, block, n_block, tf_flags, class); if (likely(rc == 0)) return 0; diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index b1666adc1c3a39160fa4b4f133fc8719990999dc..7916e369e15e76675bfbb2becb43806e56a04132 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -776,7 +776,7 @@ static void atapi_send_cdb(struct ata_port *ap, struct ata_queued_cmd *qc) * @qc: Command on going * @bytes: number of bytes * - * Transfer Transfer data from/to the ATAPI device. + * Transfer data from/to the ATAPI device. * * LOCKING: * Inherited from caller. diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index 98bc8649c63ff4d8cf2eae1e5ab6f0e7ec588f75..2c5c8273af01751a9fa4e8de65cd2e97438fa8ae 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -44,9 +44,8 @@ static inline void ata_force_cbl(struct ata_port *ap) { } #endif extern u64 ata_tf_to_lba(const struct ata_taskfile *tf); extern u64 ata_tf_to_lba48(const struct ata_taskfile *tf); -extern int ata_build_rw_tf(struct ata_taskfile *tf, struct ata_device *dev, - u64 block, u32 n_block, unsigned int tf_flags, - unsigned int tag, int class); +extern int ata_build_rw_tf(struct ata_queued_cmd *qc, u64 block, u32 n_block, + unsigned int tf_flags, int class); extern u64 ata_tf_read_block(const struct ata_taskfile *tf, struct ata_device *dev); extern unsigned ata_exec_internal(struct ata_device *dev, @@ -64,7 +63,7 @@ extern int ata_dev_configure(struct ata_device *dev); extern int sata_down_spd_limit(struct ata_link *link, u32 spd_limit); extern int ata_down_xfermask_limit(struct ata_device *dev, unsigned int sel); extern unsigned int ata_dev_set_feature(struct ata_device *dev, - u8 enable, u8 feature); + u8 subcmd, u8 action); extern void ata_qc_free(struct ata_queued_cmd *qc); extern void ata_qc_issue(struct ata_queued_cmd *qc); extern void __ata_qc_complete(struct ata_queued_cmd *qc); diff --git a/drivers/ata/pata_macio.c b/drivers/ata/pata_macio.c index bfea2be2959a6557224ed8c9dd008207a532d4c4..9ccaac9e2bc319cdb02fd882d23a9e4d5b730bd9 100644 --- a/drivers/ata/pata_macio.c +++ b/drivers/ata/pata_macio.c @@ -666,8 +666,7 @@ static u8 pata_macio_bmdma_status(struct ata_port *ap) * a multi-block transfer. * * - The dbdma fifo hasn't yet finished flushing to - * to system memory when the disk interrupt occurs. - * + * system memory when the disk interrupt occurs. */ /* First check for errors */ diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c index 4fab3b2c702390f602edf4ff51fd21681afd0e5a..02425991c15909ebd8d16f366b1c65ac711dc782 100644 --- a/drivers/auxdisplay/ht16k33.c +++ b/drivers/auxdisplay/ht16k33.c @@ -775,7 +775,7 @@ static int ht16k33_probe(struct i2c_client *client) return err; } -static int ht16k33_remove(struct i2c_client *client) +static void ht16k33_remove(struct i2c_client *client) { struct ht16k33_priv *priv = i2c_get_clientdata(client); struct ht16k33_fbdev *fbdev = &priv->fbdev; @@ -796,8 +796,6 @@ static int ht16k33_remove(struct i2c_client *client) device_remove_file(&client->dev, &dev_attr_map_seg14); break; } - - return 0; } static const struct i2c_device_id ht16k33_i2c_match[] = { diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c index e465108d9998a1563a646ddb3fef6309a98337b4..135831a165149c1582e38aada4463e086121bf71 100644 --- a/drivers/auxdisplay/lcd2s.c +++ b/drivers/auxdisplay/lcd2s.c @@ -340,13 +340,12 @@ fail1: return err; } -static int lcd2s_i2c_remove(struct i2c_client *i2c) +static void lcd2s_i2c_remove(struct i2c_client *i2c) { struct lcd2s_data *lcd2s = i2c_get_clientdata(i2c); charlcd_unregister(lcd2s->charlcd); charlcd_free(lcd2s->charlcd); - return 0; } static const struct i2c_device_id lcd2s_i2c_id[] = { diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c index 0424b59b695ef0fe2d717405efbfbc7dfac2dc3a..e7d6e6657ffa0498e66f5fa4a967c6e085349a64 100644 --- a/drivers/base/arch_topology.c +++ b/drivers/base/arch_topology.c @@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void) struct cppc_perf_caps perf_caps; int cpu; - if (likely(acpi_disabled || !acpi_cpc_valid())) + if (likely(!acpi_cpc_valid())) return; raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity), @@ -724,7 +724,7 @@ const struct cpumask *cpu_clustergroup_mask(int cpu) */ if (cpumask_subset(cpu_coregroup_mask(cpu), &cpu_topology[cpu].cluster_sibling)) - return get_cpu_mask(cpu); + return topology_sibling_cpumask(cpu); return &cpu_topology[cpu].cluster_sibling; } @@ -735,7 +735,7 @@ void update_siblings_masks(unsigned int cpuid) int cpu, ret; ret = detect_cache_attributes(cpuid); - if (ret) + if (ret && ret != -ENOENT) pr_info("Early cacheinfo failed, ret = %d\n", ret); /* update core and thread sibling masks */ @@ -841,4 +841,23 @@ void __init init_cpu_topology(void) return; } } + +void store_cpu_topology(unsigned int cpuid) +{ + struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; + + if (cpuid_topo->package_id != -1) + goto topology_populated; + + cpuid_topo->thread_id = -1; + cpuid_topo->core_id = cpuid; + cpuid_topo->package_id = cpu_to_node(cpuid); + + pr_debug("CPU%u: package %d core %d thread %d\n", + cpuid, cpuid_topo->package_id, cpuid_topo->core_id, + cpuid_topo->thread_id); + +topology_populated: + update_siblings_masks(cpuid); +} #endif diff --git a/drivers/base/base.h b/drivers/base/base.h index b3a43a164dcd1d52280113fb271045693dbcf01c..b902d1ecc247f142d33150ad129e27c0565fa316 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -154,8 +154,6 @@ extern void driver_remove_groups(struct device_driver *drv, const struct attribute_group **groups); void device_driver_detach(struct device *dev); -extern char *make_class_name(const char *name, struct kobject *kobj); - extern int devres_release_all(struct device *dev); extern void device_block_probing(void); extern void device_unblock_probing(void); diff --git a/drivers/base/class.c b/drivers/base/class.c index 8feb85e186e3b72450e0c1285c85b4bedbe5eb07..64f7b9a0970f7d5d89559cbb4872079021a14fea 100644 --- a/drivers/base/class.c +++ b/drivers/base/class.c @@ -260,7 +260,7 @@ EXPORT_SYMBOL_GPL(__class_create); */ void class_destroy(struct class *cls) { - if ((cls == NULL) || (IS_ERR(cls))) + if (IS_ERR_OR_NULL(cls)) return; class_unregister(cls); diff --git a/drivers/base/core.c b/drivers/base/core.c index 753e7cca0f40edc7ee39d4bf0d3d06101341d170..d02501933467d443296e2dac60225ff8d3d2fbb6 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1625,7 +1625,7 @@ static int __init fw_devlink_setup(char *arg) } early_param("fw_devlink", fw_devlink_setup); -static bool fw_devlink_strict = true; +static bool fw_devlink_strict; static int __init fw_devlink_strict_setup(char *arg) { return strtobool(arg, &fw_devlink_strict); @@ -2509,7 +2509,7 @@ static ssize_t uevent_store(struct device *dev, struct device_attribute *attr, rc = kobject_synth_uevent(&dev->kobj, buf, count); if (rc) { - dev_err(dev, "uevent: failed to send synthetic uevent\n"); + dev_err(dev, "uevent: failed to send synthetic uevent: %d\n", rc); return rc; } @@ -4170,7 +4170,7 @@ device_create_groups_vargs(struct class *class, struct device *parent, struct device *dev = NULL; int retval = -ENODEV; - if (class == NULL || IS_ERR(class)) + if (IS_ERR_OR_NULL(class)) goto error; dev = kzalloc(sizeof(*dev), GFP_KERNEL); diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 70f79fc7153948266ac5cf00526f8dc07310ed8c..3dda62503102f9ee77637d396031c5b133482b6f 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -274,12 +274,42 @@ static int __init deferred_probe_timeout_setup(char *str) } __setup("deferred_probe_timeout=", deferred_probe_timeout_setup); +/** + * driver_deferred_probe_check_state() - Check deferred probe state + * @dev: device to check + * + * Return: + * * -ENODEV if initcalls have completed and modules are disabled. + * * -ETIMEDOUT if the deferred probe timeout was set and has expired + * and modules are enabled. + * * -EPROBE_DEFER in other cases. + * + * Drivers or subsystems can opt-in to calling this function instead of directly + * returning -EPROBE_DEFER. + */ +int driver_deferred_probe_check_state(struct device *dev) +{ + if (!IS_ENABLED(CONFIG_MODULES) && initcalls_done) { + dev_warn(dev, "ignoring dependency for device, assuming no driver\n"); + return -ENODEV; + } + + if (!driver_deferred_probe_timeout && initcalls_done) { + dev_warn(dev, "deferred probe timeout, ignoring dependency\n"); + return -ETIMEDOUT; + } + + return -EPROBE_DEFER; +} +EXPORT_SYMBOL_GPL(driver_deferred_probe_check_state); + static void deferred_probe_timeout_work_func(struct work_struct *work) { struct device_private *p; fw_devlink_drivers_done(); + driver_deferred_probe_timeout = 0; driver_deferred_probe_trigger(); flush_work(&deferred_probe_work); @@ -806,7 +836,7 @@ static int __init save_async_options(char *buf) if (strlen(buf) >= ASYNC_DRV_NAMES_MAX_LEN) pr_warn("Too long list of driver names for 'driver_async_probe'!\n"); - strlcpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN); + strscpy(async_probe_drv_names, buf, ASYNC_DRV_NAMES_MAX_LEN); async_probe_default = parse_option_str(async_probe_drv_names, "*"); return 1; @@ -881,6 +911,11 @@ static int __device_attach_driver(struct device_driver *drv, void *_data) dev_dbg(dev, "Device match requests probe deferral\n"); dev->can_match = true; driver_deferred_probe_add(dev); + /* + * Device can't match with a driver right now, so don't attempt + * to match or bind with other drivers on the bus. + */ + return ret; } else if (ret < 0) { dev_dbg(dev, "Bus failed to match device: %d\n", ret); return ret; @@ -1120,6 +1155,11 @@ static int __driver_attach(struct device *dev, void *data) dev_dbg(dev, "Device match requests probe deferral\n"); dev->can_match = true; driver_deferred_probe_add(dev); + /* + * Driver could not match with device, but may match with + * another device on the bus. + */ + return 0; } else if (ret < 0) { dev_dbg(dev, "Bus failed to match device: %d\n", ret); return ret; diff --git a/drivers/base/devcoredump.c b/drivers/base/devcoredump.c index f4d794d6bb859119f8ba617335ee3ecc788b52f1..1c06781f711484f0a195ead7de754cdcf3bdff25 100644 --- a/drivers/base/devcoredump.c +++ b/drivers/base/devcoredump.c @@ -25,6 +25,47 @@ struct devcd_entry { struct device devcd_dev; void *data; size_t datalen; + /* + * Here, mutex is required to serialize the calls to del_wk work between + * user/kernel space which happens when devcd is added with device_add() + * and that sends uevent to user space. User space reads the uevents, + * and calls to devcd_data_write() which try to modify the work which is + * not even initialized/queued from devcoredump. + * + * + * + * cpu0(X) cpu1(Y) + * + * dev_coredump() uevent sent to user space + * device_add() ======================> user space process Y reads the + * uevents writes to devcd fd + * which results into writes to + * + * devcd_data_write() + * mod_delayed_work() + * try_to_grab_pending() + * del_timer() + * debug_assert_init() + * INIT_DELAYED_WORK() + * schedule_delayed_work() + * + * + * Also, mutex alone would not be enough to avoid scheduling of + * del_wk work after it get flush from a call to devcd_free() + * mentioned as below. + * + * disabled_store() + * devcd_free() + * mutex_lock() devcd_data_write() + * flush_delayed_work() + * mutex_unlock() + * mutex_lock() + * mod_delayed_work() + * mutex_unlock() + * So, delete_work flag is required. + */ + struct mutex mutex; + bool delete_work; struct module *owner; ssize_t (*read)(char *buffer, loff_t offset, size_t count, void *data, size_t datalen); @@ -84,7 +125,12 @@ static ssize_t devcd_data_write(struct file *filp, struct kobject *kobj, struct device *dev = kobj_to_dev(kobj); struct devcd_entry *devcd = dev_to_devcd(dev); - mod_delayed_work(system_wq, &devcd->del_wk, 0); + mutex_lock(&devcd->mutex); + if (!devcd->delete_work) { + devcd->delete_work = true; + mod_delayed_work(system_wq, &devcd->del_wk, 0); + } + mutex_unlock(&devcd->mutex); return count; } @@ -112,7 +158,12 @@ static int devcd_free(struct device *dev, void *data) { struct devcd_entry *devcd = dev_to_devcd(dev); + mutex_lock(&devcd->mutex); + if (!devcd->delete_work) + devcd->delete_work = true; + flush_delayed_work(&devcd->del_wk); + mutex_unlock(&devcd->mutex); return 0; } @@ -122,6 +173,30 @@ static ssize_t disabled_show(struct class *class, struct class_attribute *attr, return sysfs_emit(buf, "%d\n", devcd_disabled); } +/* + * + * disabled_store() worker() + * class_for_each_device(&devcd_class, + * NULL, NULL, devcd_free) + * ... + * ... + * while ((dev = class_dev_iter_next(&iter)) + * devcd_del() + * device_del() + * put_device() <- last reference + * error = fn(dev, data) devcd_dev_release() + * devcd_free(dev, data) kfree(devcd) + * mutex_lock(&devcd->mutex); + * + * + * In the above diagram, It looks like disabled_store() would be racing with parallely + * running devcd_del() and result in memory abort while acquiring devcd->mutex which + * is called after kfree of devcd memory after dropping its last reference with + * put_device(). However, this will not happens as fn(dev, data) runs + * with its own reference to device via klist_node so it is not its last reference. + * so, above situation would not occur. + */ + static ssize_t disabled_store(struct class *class, struct class_attribute *attr, const char *buf, size_t count) { @@ -278,13 +353,16 @@ void dev_coredumpm(struct device *dev, struct module *owner, devcd->read = read; devcd->free = free; devcd->failing_dev = get_device(dev); + devcd->delete_work = false; + mutex_init(&devcd->mutex); device_initialize(&devcd->devcd_dev); dev_set_name(&devcd->devcd_dev, "devcd%d", atomic_inc_return(&devcd_count)); devcd->devcd_dev.class = &devcd_class; + mutex_lock(&devcd->mutex); if (device_add(&devcd->devcd_dev)) goto put_device; @@ -301,10 +379,11 @@ void dev_coredumpm(struct device *dev, struct module *owner, INIT_DELAYED_WORK(&devcd->del_wk, devcd_del); schedule_delayed_work(&devcd->del_wk, DEVCD_TIMEOUT); - + mutex_unlock(&devcd->mutex); return; put_device: put_device(&devcd->devcd_dev); + mutex_unlock(&devcd->mutex); put_module: module_put(owner); free: diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 864d0b3f566eedb24444464674bfbddd0206036f..4ab2b50ee38f49a016f9581d385eba41b925d729 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -117,7 +117,9 @@ static __always_inline struct devres * alloc_dr(dr_release_t release, if (unlikely(!dr)) return NULL; - memset(dr, 0, offsetof(struct devres, data)); + /* No need to clear memory twice */ + if (!(gfp & __GFP_ZERO)) + memset(dr, 0, offsetof(struct devres, data)); INIT_LIST_HEAD(&dr->node.entry); dr->node.release = release; diff --git a/drivers/base/driver.c b/drivers/base/driver.c index 15a75afe6b8450c948e530d8d2913e7554d1e10f..676b6275d5b53606caba09373d7480c7ca073d1a 100644 --- a/drivers/base/driver.c +++ b/drivers/base/driver.c @@ -63,6 +63,12 @@ int driver_set_override(struct device *dev, const char **override, if (len >= (PAGE_SIZE - 1)) return -EINVAL; + /* + * Compute the real length of the string in case userspace sends us a + * bunch of \0 characters like python likes to do. + */ + len = strlen(s); + if (!len) { /* Empty string passed - clear override */ device_lock(dev); diff --git a/drivers/base/firmware_loader/sysfs.c b/drivers/base/firmware_loader/sysfs.c index 77bad32c481a3ab61e163e7b43d1344748cacb44..5b66b3d1fa16b8126946d67daeaf6c711e7440d9 100644 --- a/drivers/base/firmware_loader/sysfs.c +++ b/drivers/base/firmware_loader/sysfs.c @@ -93,10 +93,9 @@ static void fw_dev_release(struct device *dev) { struct fw_sysfs *fw_sysfs = to_fw_sysfs(dev); - if (fw_sysfs->fw_upload_priv) { - free_fw_priv(fw_sysfs->fw_priv); - kfree(fw_sysfs->fw_upload_priv); - } + if (fw_sysfs->fw_upload_priv) + fw_upload_free(fw_sysfs); + kfree(fw_sysfs); } diff --git a/drivers/base/firmware_loader/sysfs.h b/drivers/base/firmware_loader/sysfs.h index 5d8ff1675c79455cdbfd9667244f8ebb44ca5668..df1d5add698f1d93e69f7f3a05d15745d6ec0a3c 100644 --- a/drivers/base/firmware_loader/sysfs.h +++ b/drivers/base/firmware_loader/sysfs.h @@ -106,12 +106,17 @@ extern struct device_attribute dev_attr_cancel; extern struct device_attribute dev_attr_remaining_size; int fw_upload_start(struct fw_sysfs *fw_sysfs); +void fw_upload_free(struct fw_sysfs *fw_sysfs); umode_t fw_upload_is_visible(struct kobject *kobj, struct attribute *attr, int n); #else static inline int fw_upload_start(struct fw_sysfs *fw_sysfs) { return 0; } + +static inline void fw_upload_free(struct fw_sysfs *fw_sysfs) +{ +} #endif #endif /* __FIRMWARE_SYSFS_H */ diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c index 87044d52322aa2ff9fd8d5654185585b9a1d9d3c..a0af8f5f13d888641201eea1cf3a9b1071922e12 100644 --- a/drivers/base/firmware_loader/sysfs_upload.c +++ b/drivers/base/firmware_loader/sysfs_upload.c @@ -264,6 +264,15 @@ int fw_upload_start(struct fw_sysfs *fw_sysfs) return 0; } +void fw_upload_free(struct fw_sysfs *fw_sysfs) +{ + struct fw_upload_priv *fw_upload_priv = fw_sysfs->fw_upload_priv; + + free_fw_priv(fw_sysfs->fw_priv); + kfree(fw_upload_priv->fw_upload); + kfree(fw_upload_priv); +} + /** * firmware_upload_register() - register for the firmware upload sysfs API * @module: kernel module of this device @@ -377,6 +386,7 @@ void firmware_upload_unregister(struct fw_upload *fw_upload) { struct fw_sysfs *fw_sysfs = fw_upload->priv; struct fw_upload_priv *fw_upload_priv = fw_sysfs->fw_upload_priv; + struct module *module = fw_upload_priv->module; mutex_lock(&fw_upload_priv->lock); if (fw_upload_priv->progress == FW_UPLOAD_PROG_IDLE) { @@ -392,6 +402,6 @@ void firmware_upload_unregister(struct fw_upload *fw_upload) unregister: device_unregister(&fw_sysfs->dev); - module_put(fw_upload_priv->module); + module_put(module); } EXPORT_SYMBOL_GPL(firmware_upload_unregister); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index bc60c9cd3230821c8b2c34bebbc06b59901b2d24..9aa0da991cfb910955ece83be4be1970eced2749 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -869,12 +869,6 @@ void remove_memory_block_devices(unsigned long start, unsigned long size) } } -/* return true if the memory block is offlined, otherwise, return false */ -bool is_memblock_offlined(struct memory_block *mem) -{ - return mem->state == MEM_OFFLINE; -} - static struct attribute *memory_root_attrs[] = { #ifdef CONFIG_ARCH_MEMORY_PROBE &dev_attr_probe.attr, diff --git a/drivers/base/node.c b/drivers/base/node.c index eb0f43784c2b3a6dc3eb8885723d182dbe846459..faf3597a96da9d7b9122bda47687038254f2c793 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -20,6 +20,7 @@ #include #include #include +#include static struct bus_type node_subsys = { .name = "node", @@ -433,6 +434,7 @@ static ssize_t node_read_meminfo(struct device *dev, "Node %d ShadowCallStack:%8lu kB\n" #endif "Node %d PageTables: %8lu kB\n" + "Node %d SecPageTables: %8lu kB\n" "Node %d NFS_Unstable: %8lu kB\n" "Node %d Bounce: %8lu kB\n" "Node %d WritebackTmp: %8lu kB\n" @@ -459,6 +461,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, node_page_state(pgdat, NR_KERNEL_SCS_KB), #endif nid, K(node_page_state(pgdat, NR_PAGETABLE)), + nid, K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)), nid, 0UL, nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)), nid, K(node_page_state(pgdat, NR_WRITEBACK_TEMP)), @@ -587,64 +590,9 @@ static const struct attribute_group *node_dev_groups[] = { NULL }; -#ifdef CONFIG_HUGETLBFS -/* - * hugetlbfs per node attributes registration interface: - * When/if hugetlb[fs] subsystem initializes [sometime after this module], - * it will register its per node attributes for all online nodes with - * memory. It will also call register_hugetlbfs_with_node(), below, to - * register its attribute registration functions with this node driver. - * Once these hooks have been initialized, the node driver will call into - * the hugetlb module to [un]register attributes for hot-plugged nodes. - */ -static node_registration_func_t __hugetlb_register_node; -static node_registration_func_t __hugetlb_unregister_node; - -static inline bool hugetlb_register_node(struct node *node) -{ - if (__hugetlb_register_node && - node_state(node->dev.id, N_MEMORY)) { - __hugetlb_register_node(node); - return true; - } - return false; -} - -static inline void hugetlb_unregister_node(struct node *node) -{ - if (__hugetlb_unregister_node) - __hugetlb_unregister_node(node); -} - -void register_hugetlbfs_with_node(node_registration_func_t doregister, - node_registration_func_t unregister) -{ - __hugetlb_register_node = doregister; - __hugetlb_unregister_node = unregister; -} -#else -static inline void hugetlb_register_node(struct node *node) {} - -static inline void hugetlb_unregister_node(struct node *node) {} -#endif - static void node_device_release(struct device *dev) { - struct node *node = to_node(dev); - -#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS) - /* - * We schedule the work only when a memory section is - * onlined/offlined on this node. When we come here, - * all the memory on this node has been offlined, - * so we won't enqueue new work to this work. - * - * The work is using node->node_work, so we should - * flush work before freeing the memory. - */ - flush_work(&node->node_work); -#endif - kfree(node); + kfree(to_node(dev)); } /* @@ -663,13 +611,13 @@ static int register_node(struct node *node, int num) node->dev.groups = node_dev_groups; error = device_register(&node->dev); - if (error) + if (error) { put_device(&node->dev); - else { + } else { hugetlb_register_node(node); - compaction_register_node(node); } + return error; } @@ -682,8 +630,8 @@ static int register_node(struct node *node, int num) */ void unregister_node(struct node *node) { + hugetlb_unregister_node(node); compaction_unregister_node(node); - hugetlb_unregister_node(node); /* no-op, if memoryless node */ node_remove_accesses(node); node_remove_caches(node); device_unregister(&node->dev); @@ -905,74 +853,8 @@ void register_memory_blocks_under_node(int nid, unsigned long start_pfn, (void *)&nid, func); return; } - -#ifdef CONFIG_HUGETLBFS -/* - * Handle per node hstate attribute [un]registration on transistions - * to/from memoryless state. - */ -static void node_hugetlb_work(struct work_struct *work) -{ - struct node *node = container_of(work, struct node, node_work); - - /* - * We only get here when a node transitions to/from memoryless state. - * We can detect which transition occurred by examining whether the - * node has memory now. hugetlb_register_node() already check this - * so we try to register the attributes. If that fails, then the - * node has transitioned to memoryless, try to unregister the - * attributes. - */ - if (!hugetlb_register_node(node)) - hugetlb_unregister_node(node); -} - -static void init_node_hugetlb_work(int nid) -{ - INIT_WORK(&node_devices[nid]->node_work, node_hugetlb_work); -} - -static int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - struct memory_notify *mnb = arg; - int nid = mnb->status_change_nid; - - switch (action) { - case MEM_ONLINE: - case MEM_OFFLINE: - /* - * offload per node hstate [un]registration to a work thread - * when transitioning to/from memoryless state. - */ - if (nid != NUMA_NO_NODE) - schedule_work(&node_devices[nid]->node_work); - break; - - case MEM_GOING_ONLINE: - case MEM_GOING_OFFLINE: - case MEM_CANCEL_ONLINE: - case MEM_CANCEL_OFFLINE: - default: - break; - } - - return NOTIFY_OK; -} -#endif /* CONFIG_HUGETLBFS */ #endif /* CONFIG_MEMORY_HOTPLUG */ -#if !defined(CONFIG_MEMORY_HOTPLUG) || !defined(CONFIG_HUGETLBFS) -static inline int node_memory_callback(struct notifier_block *self, - unsigned long action, void *arg) -{ - return NOTIFY_OK; -} - -static void init_node_hugetlb_work(int nid) { } - -#endif - int __register_one_node(int nid) { int error; @@ -991,8 +873,6 @@ int __register_one_node(int nid) } INIT_LIST_HEAD(&node_devices[nid]->access_list); - /* initialize work queue for memory hot plug */ - init_node_hugetlb_work(nid); node_init_caches(nid); return error; @@ -1063,13 +943,8 @@ static const struct attribute_group *cpu_root_attr_groups[] = { NULL, }; -#define NODE_CALLBACK_PRI 2 /* lower than SLAB */ void __init node_dev_init(void) { - static struct notifier_block node_memory_callback_nb = { - .notifier_call = node_memory_callback, - .priority = NODE_CALLBACK_PRI, - }; int ret, i; BUILD_BUG_ON(ARRAY_SIZE(node_state_attr) != NR_NODE_STATES); @@ -1079,8 +954,6 @@ void __init node_dev_init(void) if (ret) panic("%s() failed to register subsystem: %d\n", __func__, ret); - register_hotmemory_notifier(&node_memory_callback_nb); - /* * Create all node devices, which will properly link the node * to applicable memory block devices and already created cpu devices. diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 296ea673d661500fac8884fc519c8ad0d60aef83..12b044151298bdbc567d994829fe9315cedab979 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -138,6 +138,7 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode, return domain; } +EXPORT_SYMBOL_GPL(platform_msi_create_irq_domain); static int platform_msi_alloc_priv_data(struct device *dev, unsigned int nvec, irq_write_msi_msg_t write_msi_msg) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 5a2e0232862e0a7c4a4fd8ed77073e6cace3a42c..ead135c7044c677941018afd876e947b85ace12d 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2085,8 +2085,10 @@ int pm_genpd_init(struct generic_pm_domain *genpd, /* Always-on domains must be powered on at initialization. */ if ((genpd_is_always_on(genpd) || genpd_is_rpm_always_on(genpd)) && - !genpd_status_on(genpd)) + !genpd_status_on(genpd)) { + pr_err("always-on PM domain %s is not on\n", genpd->name); return -EINVAL; + } /* Multiple states but no governor doesn't make sense. */ if (!gov && genpd->state_count > 1) @@ -2733,7 +2735,7 @@ static int __genpd_dev_pm_attach(struct device *dev, struct device *base_dev, mutex_unlock(&gpd_list_lock); dev_dbg(dev, "%s() failed to find PM domain: %ld\n", __func__, PTR_ERR(pd)); - return -ENODEV; + return driver_deferred_probe_check_state(base_dev); } dev_dbg(dev, "adding to PM domain %s\n", pd->name); diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 997be3ac20a799908378518567aa432275f790d3..b52049098d4ee599c4636f4f81acad51a0c92e02 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -792,10 +792,13 @@ static int rpm_resume(struct device *dev, int rpmflags) DEFINE_WAIT(wait); if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) { - if (dev->power.runtime_status == RPM_SUSPENDING) + if (dev->power.runtime_status == RPM_SUSPENDING) { dev->power.deferred_resume = true; - else + if (rpmflags & RPM_NOWAIT) + retval = -EINPROGRESS; + } else { retval = -EINPROGRESS; + } goto out; } diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index e3befa2c1b661675b7b3cc80401cb5bc846379fe..7cc0c0cf8eaa529eb2eecd484f8a2a9fbedf4fb3 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -944,6 +944,8 @@ void pm_system_irq_wakeup(unsigned int irq_number) else irq_number = 0; + pm_pr_dbg("Triggering wakeup from IRQ %d\n", irq_number); + raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags); if (irq_number) diff --git a/drivers/base/property.c b/drivers/base/property.c index ed6f449f8e5cf1109b9a280b957cd5187ac4a866..4d6278a84868691727117e227757845c667ae06d 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -17,7 +17,7 @@ #include #include -struct fwnode_handle *dev_fwnode(struct device *dev) +struct fwnode_handle *dev_fwnode(const struct device *dev) { return IS_ENABLED(CONFIG_OF) && dev->of_node ? of_fwnode_handle(dev->of_node) : dev->fwnode; @@ -1200,7 +1200,7 @@ int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, } EXPORT_SYMBOL(fwnode_graph_parse_endpoint); -const void *device_get_match_data(struct device *dev) +const void *device_get_match_data(const struct device *dev) { return fwnode_call_ptr_op(dev_fwnode(dev), device_get_match_data, dev); } diff --git a/drivers/base/regmap/regmap-mmio.c b/drivers/base/regmap/regmap-mmio.c index 71f16be7e7177a1d716613ab9fec6c9f2ceb8286..3ccdd86a97e77609fa747499b45228de3c804cd1 100644 --- a/drivers/base/regmap/regmap-mmio.c +++ b/drivers/base/regmap/regmap-mmio.c @@ -10,13 +10,14 @@ #include #include #include +#include #include "internal.h" struct regmap_mmio_context { void __iomem *regs; unsigned int val_bytes; - bool relaxed_mmio; + bool big_endian; bool attached_clk; struct clk *clk; @@ -33,9 +34,6 @@ static int regmap_mmio_regbits_check(size_t reg_bits) case 8: case 16: case 32: -#ifdef CONFIG_64BIT - case 64: -#endif return 0; default: return -EINVAL; @@ -50,18 +48,13 @@ static int regmap_mmio_get_min_stride(size_t val_bits) case 8: /* The core treats 0 as 1 */ min_stride = 0; - return 0; + break; case 16: min_stride = 2; break; case 32: min_stride = 4; break; -#ifdef CONFIG_64BIT - case 64: - min_stride = 8; - break; -#endif default: return -EINVAL; } @@ -83,6 +76,12 @@ static void regmap_mmio_write8_relaxed(struct regmap_mmio_context *ctx, writeb_relaxed(val, ctx->regs + reg); } +static void regmap_mmio_iowrite8(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite8(val, ctx->regs + reg); +} + static void regmap_mmio_write16le(struct regmap_mmio_context *ctx, unsigned int reg, unsigned int val) @@ -97,9 +96,21 @@ static void regmap_mmio_write16le_relaxed(struct regmap_mmio_context *ctx, writew_relaxed(val, ctx->regs + reg); } +static void regmap_mmio_iowrite16le(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) +{ + iowrite16(val, ctx->regs + reg); +} + static void regmap_mmio_write16be(struct regmap_mmio_context *ctx, unsigned int reg, unsigned int val) +{ + writew(swab16(val), ctx->regs + reg); +} + +static void regmap_mmio_iowrite16be(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) { iowrite16be(val, ctx->regs + reg); } @@ -118,28 +129,24 @@ static void regmap_mmio_write32le_relaxed(struct regmap_mmio_context *ctx, writel_relaxed(val, ctx->regs + reg); } -static void regmap_mmio_write32be(struct regmap_mmio_context *ctx, - unsigned int reg, - unsigned int val) +static void regmap_mmio_iowrite32le(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) { - iowrite32be(val, ctx->regs + reg); + iowrite32(val, ctx->regs + reg); } -#ifdef CONFIG_64BIT -static void regmap_mmio_write64le(struct regmap_mmio_context *ctx, +static void regmap_mmio_write32be(struct regmap_mmio_context *ctx, unsigned int reg, unsigned int val) { - writeq(val, ctx->regs + reg); + writel(swab32(val), ctx->regs + reg); } -static void regmap_mmio_write64le_relaxed(struct regmap_mmio_context *ctx, - unsigned int reg, - unsigned int val) +static void regmap_mmio_iowrite32be(struct regmap_mmio_context *ctx, + unsigned int reg, unsigned int val) { - writeq_relaxed(val, ctx->regs + reg); + iowrite32be(val, ctx->regs + reg); } -#endif static int regmap_mmio_write(void *context, unsigned int reg, unsigned int val) { @@ -160,6 +167,83 @@ static int regmap_mmio_write(void *context, unsigned int reg, unsigned int val) return 0; } +static int regmap_mmio_noinc_write(void *context, unsigned int reg, + const void *val, size_t val_count) +{ + struct regmap_mmio_context *ctx = context; + int ret = 0; + int i; + + if (!IS_ERR(ctx->clk)) { + ret = clk_enable(ctx->clk); + if (ret < 0) + return ret; + } + + /* + * There are no native, assembly-optimized write single register + * operations for big endian, so fall back to emulation if this + * is needed. (Single bytes are fine, they are not affected by + * endianness.) + */ + if (ctx->big_endian && (ctx->val_bytes > 1)) { + switch (ctx->val_bytes) { + case 2: + { + const u16 *valp = (const u16 *)val; + for (i = 0; i < val_count; i++) + writew(swab16(valp[i]), ctx->regs + reg); + goto out_clk; + } + case 4: + { + const u32 *valp = (const u32 *)val; + for (i = 0; i < val_count; i++) + writel(swab32(valp[i]), ctx->regs + reg); + goto out_clk; + } +#ifdef CONFIG_64BIT + case 8: + { + const u64 *valp = (const u64 *)val; + for (i = 0; i < val_count; i++) + writeq(swab64(valp[i]), ctx->regs + reg); + goto out_clk; + } +#endif + default: + ret = -EINVAL; + goto out_clk; + } + } + + switch (ctx->val_bytes) { + case 1: + writesb(ctx->regs + reg, (const u8 *)val, val_count); + break; + case 2: + writesw(ctx->regs + reg, (const u16 *)val, val_count); + break; + case 4: + writesl(ctx->regs + reg, (const u32 *)val, val_count); + break; +#ifdef CONFIG_64BIT + case 8: + writesq(ctx->regs + reg, (const u64 *)val, val_count); + break; +#endif + default: + ret = -EINVAL; + break; + } + +out_clk: + if (!IS_ERR(ctx->clk)) + clk_disable(ctx->clk); + + return ret; +} + static unsigned int regmap_mmio_read8(struct regmap_mmio_context *ctx, unsigned int reg) { @@ -172,6 +256,12 @@ static unsigned int regmap_mmio_read8_relaxed(struct regmap_mmio_context *ctx, return readb_relaxed(ctx->regs + reg); } +static unsigned int regmap_mmio_ioread8(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread8(ctx->regs + reg); +} + static unsigned int regmap_mmio_read16le(struct regmap_mmio_context *ctx, unsigned int reg) { @@ -184,8 +274,20 @@ static unsigned int regmap_mmio_read16le_relaxed(struct regmap_mmio_context *ctx return readw_relaxed(ctx->regs + reg); } +static unsigned int regmap_mmio_ioread16le(struct regmap_mmio_context *ctx, + unsigned int reg) +{ + return ioread16(ctx->regs + reg); +} + static unsigned int regmap_mmio_read16be(struct regmap_mmio_context *ctx, unsigned int reg) +{ + return swab16(readw(ctx->regs + reg)); +} + +static unsigned int regmap_mmio_ioread16be(struct regmap_mmio_context *ctx, + unsigned int reg) { return ioread16be(ctx->regs + reg); } @@ -202,25 +304,23 @@ static unsigned int regmap_mmio_read32le_relaxed(struct regmap_mmio_context *ctx return readl_relaxed(ctx->regs + reg); } -static unsigned int regmap_mmio_read32be(struct regmap_mmio_context *ctx, - unsigned int reg) +static unsigned int regmap_mmio_ioread32le(struct regmap_mmio_context *ctx, + unsigned int reg) { - return ioread32be(ctx->regs + reg); + return ioread32(ctx->regs + reg); } -#ifdef CONFIG_64BIT -static unsigned int regmap_mmio_read64le(struct regmap_mmio_context *ctx, +static unsigned int regmap_mmio_read32be(struct regmap_mmio_context *ctx, unsigned int reg) { - return readq(ctx->regs + reg); + return swab32(readl(ctx->regs + reg)); } -static unsigned int regmap_mmio_read64le_relaxed(struct regmap_mmio_context *ctx, - unsigned int reg) +static unsigned int regmap_mmio_ioread32be(struct regmap_mmio_context *ctx, + unsigned int reg) { - return readq_relaxed(ctx->regs + reg); + return ioread32be(ctx->regs + reg); } -#endif static int regmap_mmio_read(void *context, unsigned int reg, unsigned int *val) { @@ -241,6 +341,71 @@ static int regmap_mmio_read(void *context, unsigned int reg, unsigned int *val) return 0; } +static int regmap_mmio_noinc_read(void *context, unsigned int reg, + void *val, size_t val_count) +{ + struct regmap_mmio_context *ctx = context; + int ret = 0; + + if (!IS_ERR(ctx->clk)) { + ret = clk_enable(ctx->clk); + if (ret < 0) + return ret; + } + + switch (ctx->val_bytes) { + case 1: + readsb(ctx->regs + reg, (u8 *)val, val_count); + break; + case 2: + readsw(ctx->regs + reg, (u16 *)val, val_count); + break; + case 4: + readsl(ctx->regs + reg, (u32 *)val, val_count); + break; +#ifdef CONFIG_64BIT + case 8: + readsq(ctx->regs + reg, (u64 *)val, val_count); + break; +#endif + default: + ret = -EINVAL; + goto out_clk; + } + + /* + * There are no native, assembly-optimized write single register + * operations for big endian, so fall back to emulation if this + * is needed. (Single bytes are fine, they are not affected by + * endianness.) + */ + if (ctx->big_endian && (ctx->val_bytes > 1)) { + switch (ctx->val_bytes) { + case 2: + swab16_array(val, val_count); + break; + case 4: + swab32_array(val, val_count); + break; +#ifdef CONFIG_64BIT + case 8: + swab64_array(val, val_count); + break; +#endif + default: + ret = -EINVAL; + break; + } + } + +out_clk: + if (!IS_ERR(ctx->clk)) + clk_disable(ctx->clk); + + return ret; +} + + static void regmap_mmio_free_context(void *context) { struct regmap_mmio_context *ctx = context; @@ -257,6 +422,8 @@ static const struct regmap_bus regmap_mmio = { .fast_io = true, .reg_write = regmap_mmio_write, .reg_read = regmap_mmio_read, + .reg_noinc_write = regmap_mmio_noinc_write, + .reg_noinc_read = regmap_mmio_noinc_read, .free_context = regmap_mmio_free_context, .val_format_endian_default = REGMAP_ENDIAN_LITTLE, }; @@ -284,13 +451,15 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, if (config->reg_stride < min_stride) return ERR_PTR(-EINVAL); + if (config->use_relaxed_mmio && config->io_port) + return ERR_PTR(-EINVAL); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return ERR_PTR(-ENOMEM); ctx->regs = regs; ctx->val_bytes = config->val_bits / 8; - ctx->relaxed_mmio = config->use_relaxed_mmio; ctx->clk = ERR_PTR(-ENODEV); switch (regmap_get_val_endian(dev, ®map_mmio, config)) { @@ -301,7 +470,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, #endif switch (config->val_bits) { case 8: - if (ctx->relaxed_mmio) { + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread8; + ctx->reg_write = regmap_mmio_iowrite8; + } else if (config->use_relaxed_mmio) { ctx->reg_read = regmap_mmio_read8_relaxed; ctx->reg_write = regmap_mmio_write8_relaxed; } else { @@ -310,7 +482,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, } break; case 16: - if (ctx->relaxed_mmio) { + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread16le; + ctx->reg_write = regmap_mmio_iowrite16le; + } else if (config->use_relaxed_mmio) { ctx->reg_read = regmap_mmio_read16le_relaxed; ctx->reg_write = regmap_mmio_write16le_relaxed; } else { @@ -319,7 +494,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, } break; case 32: - if (ctx->relaxed_mmio) { + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread32le; + ctx->reg_write = regmap_mmio_iowrite32le; + } else if (config->use_relaxed_mmio) { ctx->reg_read = regmap_mmio_read32le_relaxed; ctx->reg_write = regmap_mmio_write32le_relaxed; } else { @@ -327,17 +505,6 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, ctx->reg_write = regmap_mmio_write32le; } break; -#ifdef CONFIG_64BIT - case 64: - if (ctx->relaxed_mmio) { - ctx->reg_read = regmap_mmio_read64le_relaxed; - ctx->reg_write = regmap_mmio_write64le_relaxed; - } else { - ctx->reg_read = regmap_mmio_read64le; - ctx->reg_write = regmap_mmio_write64le; - } - break; -#endif default: ret = -EINVAL; goto err_free; @@ -347,18 +514,34 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, #ifdef __BIG_ENDIAN case REGMAP_ENDIAN_NATIVE: #endif + ctx->big_endian = true; switch (config->val_bits) { case 8: - ctx->reg_read = regmap_mmio_read8; - ctx->reg_write = regmap_mmio_write8; + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread8; + ctx->reg_write = regmap_mmio_iowrite8; + } else { + ctx->reg_read = regmap_mmio_read8; + ctx->reg_write = regmap_mmio_write8; + } break; case 16: - ctx->reg_read = regmap_mmio_read16be; - ctx->reg_write = regmap_mmio_write16be; + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread16be; + ctx->reg_write = regmap_mmio_iowrite16be; + } else { + ctx->reg_read = regmap_mmio_read16be; + ctx->reg_write = regmap_mmio_write16be; + } break; case 32: - ctx->reg_read = regmap_mmio_read32be; - ctx->reg_write = regmap_mmio_write32be; + if (config->io_port) { + ctx->reg_read = regmap_mmio_ioread32be; + ctx->reg_write = regmap_mmio_iowrite32be; + } else { + ctx->reg_read = regmap_mmio_read32be; + ctx->reg_write = regmap_mmio_write32be; + } break; default: ret = -EINVAL; diff --git a/drivers/base/regmap/regmap-spi-avmm.c b/drivers/base/regmap/regmap-spi-avmm.c index ad1da83e849fe14e93b3eb09d77ea54cb75ff65c..4c2b94b3e30be588cac339d30a21b3b01ea879ff 100644 --- a/drivers/base/regmap/regmap-spi-avmm.c +++ b/drivers/base/regmap/regmap-spi-avmm.c @@ -7,6 +7,7 @@ #include #include #include +#include /* * This driver implements the regmap operations for a generic SPI @@ -162,19 +163,12 @@ struct spi_avmm_bridge { /* bridge buffer used in translation between protocol layers */ char trans_buf[TRANS_BUF_SIZE]; char phy_buf[PHY_BUF_SIZE]; - void (*swap_words)(char *buf, unsigned int len); + void (*swap_words)(void *buf, unsigned int len); }; -static void br_swap_words_32(char *buf, unsigned int len) +static void br_swap_words_32(void *buf, unsigned int len) { - u32 *p = (u32 *)buf; - unsigned int count; - - count = len / 4; - while (count--) { - *p = swab32p(p); - p++; - } + swab32_array(buf, len / 4); } /* diff --git a/drivers/base/regmap/regmap-spi.c b/drivers/base/regmap/regmap-spi.c index 719323bc6c7f1f478587400afa9fa30d6aa5ce43..37ab23a9d0345a12991e3b273e3d009b147ce4bc 100644 --- a/drivers/base/regmap/regmap-spi.c +++ b/drivers/base/regmap/regmap-spi.c @@ -113,6 +113,7 @@ static const struct regmap_bus *regmap_get_spi_bus(struct spi_device *spi, const struct regmap_config *config) { size_t max_size = spi_max_transfer_size(spi); + size_t max_msg_size, reg_reserve_size; struct regmap_bus *bus; if (max_size != SIZE_MAX) { @@ -120,9 +121,16 @@ static const struct regmap_bus *regmap_get_spi_bus(struct spi_device *spi, if (!bus) return ERR_PTR(-ENOMEM); + max_msg_size = spi_max_message_size(spi); + reg_reserve_size = config->reg_bits / BITS_PER_BYTE + + config->pad_bits / BITS_PER_BYTE; + if (max_size + reg_reserve_size > max_msg_size) + max_size -= reg_reserve_size; + bus->free_on_exit = true; bus->max_raw_read = max_size; bus->max_raw_write = max_size; + return bus; } diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index fee221c5008c9f6dfe50982906476b920d90223f..c6d6d53e8cd3fc776567db2b70f5e6790c15f7ff 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -288,15 +288,9 @@ static void regmap_format_16_native(void *buf, unsigned int val, memcpy(buf, &v, sizeof(v)); } -static void regmap_format_24(void *buf, unsigned int val, unsigned int shift) +static void regmap_format_24_be(void *buf, unsigned int val, unsigned int shift) { - u8 *b = buf; - - val <<= shift; - - b[0] = val >> 16; - b[1] = val >> 8; - b[2] = val; + put_unaligned_be24(val << shift, buf); } static void regmap_format_32_be(void *buf, unsigned int val, unsigned int shift) @@ -380,14 +374,9 @@ static unsigned int regmap_parse_16_native(const void *buf) return v; } -static unsigned int regmap_parse_24(const void *buf) +static unsigned int regmap_parse_24_be(const void *buf) { - const u8 *b = buf; - unsigned int ret = b[2]; - ret |= ((unsigned int)b[1]) << 8; - ret |= ((unsigned int)b[0]) << 16; - - return ret; + return get_unaligned_be24(buf); } static unsigned int regmap_parse_32_be(const void *buf) @@ -991,9 +980,13 @@ struct regmap *__regmap_init(struct device *dev, break; case 24: - if (reg_endian != REGMAP_ENDIAN_BIG) + switch (reg_endian) { + case REGMAP_ENDIAN_BIG: + map->format.format_reg = regmap_format_24_be; + break; + default: goto err_hwlock; - map->format.format_reg = regmap_format_24; + } break; case 32: @@ -1064,10 +1057,14 @@ struct regmap *__regmap_init(struct device *dev, } break; case 24: - if (val_endian != REGMAP_ENDIAN_BIG) + switch (val_endian) { + case REGMAP_ENDIAN_BIG: + map->format.format_val = regmap_format_24_be; + map->format.parse_val = regmap_parse_24_be; + break; + default: goto err_hwlock; - map->format.format_val = regmap_format_24; - map->format.parse_val = regmap_parse_24; + } break; case 32: switch (val_endian) { @@ -2132,6 +2129,99 @@ int regmap_raw_write(struct regmap *map, unsigned int reg, } EXPORT_SYMBOL_GPL(regmap_raw_write); +static int regmap_noinc_readwrite(struct regmap *map, unsigned int reg, + void *val, unsigned int val_len, bool write) +{ + size_t val_bytes = map->format.val_bytes; + size_t val_count = val_len / val_bytes; + unsigned int lastval; + u8 *u8p; + u16 *u16p; + u32 *u32p; +#ifdef CONFIG_64BIT + u64 *u64p; +#endif + int ret; + int i; + + switch (val_bytes) { + case 1: + u8p = val; + if (write) + lastval = (unsigned int)u8p[val_count - 1]; + break; + case 2: + u16p = val; + if (write) + lastval = (unsigned int)u16p[val_count - 1]; + break; + case 4: + u32p = val; + if (write) + lastval = (unsigned int)u32p[val_count - 1]; + break; +#ifdef CONFIG_64BIT + case 8: + u64p = val; + if (write) + lastval = (unsigned int)u64p[val_count - 1]; + break; +#endif + default: + return -EINVAL; + } + + /* + * Update the cache with the last value we write, the rest is just + * gone down in the hardware FIFO. We can't cache FIFOs. This makes + * sure a single read from the cache will work. + */ + if (write) { + if (!map->cache_bypass && !map->defer_caching) { + ret = regcache_write(map, reg, lastval); + if (ret != 0) + return ret; + if (map->cache_only) { + map->cache_dirty = true; + return 0; + } + } + ret = map->bus->reg_noinc_write(map->bus_context, reg, val, val_count); + } else { + ret = map->bus->reg_noinc_read(map->bus_context, reg, val, val_count); + } + + if (!ret && regmap_should_log(map)) { + dev_info(map->dev, "%x %s [", reg, write ? "<=" : "=>"); + for (i = 0; i < val_count; i++) { + switch (val_bytes) { + case 1: + pr_cont("%x", u8p[i]); + break; + case 2: + pr_cont("%x", u16p[i]); + break; + case 4: + pr_cont("%x", u32p[i]); + break; +#ifdef CONFIG_64BIT + case 8: + pr_cont("%llx", u64p[i]); + break; +#endif + default: + break; + } + if (i == (val_count - 1)) + pr_cont("]\n"); + else + pr_cont(","); + } + } + + return 0; +} + /** * regmap_noinc_write(): Write data from a register without incrementing the * register number @@ -2159,9 +2249,8 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg, size_t write_len; int ret; - if (!map->write) - return -ENOTSUPP; - + if (!map->write && !(map->bus && map->bus->reg_noinc_write)) + return -EINVAL; if (val_len % map->format.val_bytes) return -EINVAL; if (!IS_ALIGNED(reg, map->reg_stride)) @@ -2176,6 +2265,15 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg, goto out_unlock; } + /* + * Use the accelerated operation if we can. The val drops the const + * typing in order to facilitate code reuse in regmap_noinc_readwrite(). + */ + if (map->bus->reg_noinc_write) { + ret = regmap_noinc_readwrite(map, reg, (void *)val, val_len, true); + goto out_unlock; + } + while (val_len) { if (map->max_raw_write && map->max_raw_write < val_len) write_len = map->max_raw_write; @@ -2350,6 +2448,10 @@ out: kfree(wval); } + + if (!ret) + trace_regmap_bulk_write(map, reg, val, val_bytes * val_count); + return ret; } EXPORT_SYMBOL_GPL(regmap_bulk_write); @@ -2946,6 +3048,22 @@ int regmap_noinc_read(struct regmap *map, unsigned int reg, goto out_unlock; } + /* Use the accelerated operation if we can */ + if (map->bus->reg_noinc_read) { + /* + * We have not defined the FIFO semantics for cache, as the + * cache is just one value deep. Should we return the last + * written value? Just avoid this by always reading the FIFO + * even when using cache. Cache only will not work. + */ + if (map->cache_only) { + ret = -EBUSY; + goto out_unlock; + } + ret = regmap_noinc_readwrite(map, reg, val, val_len, false); + goto out_unlock; + } + while (val_len) { if (map->max_raw_read && map->max_raw_read < val_len) read_len = map->max_raw_read; @@ -3095,6 +3213,9 @@ out: map->unlock(map->lock_arg); } + if (!ret) + trace_regmap_bulk_read(map, reg, val, val_bytes * val_count); + return ret; } EXPORT_SYMBOL_GPL(regmap_bulk_read); diff --git a/drivers/base/regmap/trace.h b/drivers/base/regmap/trace.h index 9abee14df9eea2d50213389ff745db92961cbac8..704e106e5dbd090904595da8fa5e7b52aeef3a14 100644 --- a/drivers/base/regmap/trace.h +++ b/drivers/base/regmap/trace.h @@ -32,9 +32,7 @@ DECLARE_EVENT_CLASS(regmap_reg, __entry->val = val; ), - TP_printk("%s reg=%x val=%x", __get_str(name), - (unsigned int)__entry->reg, - (unsigned int)__entry->val) + TP_printk("%s reg=%x val=%x", __get_str(name), __entry->reg, __entry->val) ); DEFINE_EVENT(regmap_reg, regmap_reg_write, @@ -43,7 +41,6 @@ DEFINE_EVENT(regmap_reg, regmap_reg_write, unsigned int val), TP_ARGS(map, reg, val) - ); DEFINE_EVENT(regmap_reg, regmap_reg_read, @@ -52,7 +49,6 @@ DEFINE_EVENT(regmap_reg, regmap_reg_read, unsigned int val), TP_ARGS(map, reg, val) - ); DEFINE_EVENT(regmap_reg, regmap_reg_read_cache, @@ -61,7 +57,47 @@ DEFINE_EVENT(regmap_reg, regmap_reg_read_cache, unsigned int val), TP_ARGS(map, reg, val) +); + +DECLARE_EVENT_CLASS(regmap_bulk, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len), + + TP_STRUCT__entry( + __string(name, regmap_name(map)) + __field(unsigned int, reg) + __dynamic_array(char, buf, val_len) + __field(int, val_len) + ), + + TP_fast_assign( + __assign_str(name, regmap_name(map)); + __entry->reg = reg; + __entry->val_len = val_len; + memcpy(__get_dynamic_array(buf), val, val_len); + ), + TP_printk("%s reg=%x val=%s", __get_str(name), __entry->reg, + __print_hex(__get_dynamic_array(buf), __entry->val_len)) +); + +DEFINE_EVENT(regmap_bulk, regmap_bulk_write, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len) +); + +DEFINE_EVENT(regmap_bulk, regmap_bulk_read, + + TP_PROTO(struct regmap *map, unsigned int reg, + const void *val, int val_len), + + TP_ARGS(map, reg, val, val_len) ); DECLARE_EVENT_CLASS(regmap_block, @@ -82,9 +118,7 @@ DECLARE_EVENT_CLASS(regmap_block, __entry->count = count; ), - TP_printk("%s reg=%x count=%d", __get_str(name), - (unsigned int)__entry->reg, - (int)__entry->count) + TP_printk("%s reg=%x count=%d", __get_str(name), __entry->reg, __entry->count) ); DEFINE_EVENT(regmap_block, regmap_hw_read_start, @@ -154,8 +188,7 @@ DECLARE_EVENT_CLASS(regmap_bool, __entry->flag = flag; ), - TP_printk("%s flag=%d", __get_str(name), - (int)__entry->flag) + TP_printk("%s flag=%d", __get_str(name), __entry->flag) ); DEFINE_EVENT(regmap_bool, regmap_cache_only, @@ -163,7 +196,6 @@ DEFINE_EVENT(regmap_bool, regmap_cache_only, TP_PROTO(struct regmap *map, bool flag), TP_ARGS(map, flag) - ); DEFINE_EVENT(regmap_bool, regmap_cache_bypass, @@ -171,7 +203,6 @@ DEFINE_EVENT(regmap_bool, regmap_cache_bypass, TP_PROTO(struct regmap *map, bool flag), TP_ARGS(map, flag) - ); DECLARE_EVENT_CLASS(regmap_async, @@ -203,7 +234,6 @@ DEFINE_EVENT(regmap_async, regmap_async_io_complete, TP_PROTO(struct regmap *map), TP_ARGS(map) - ); DEFINE_EVENT(regmap_async, regmap_async_complete_start, @@ -211,7 +241,6 @@ DEFINE_EVENT(regmap_async, regmap_async_complete_start, TP_PROTO(struct regmap *map), TP_ARGS(map) - ); DEFINE_EVENT(regmap_async, regmap_async_complete_done, @@ -219,7 +248,6 @@ DEFINE_EVENT(regmap_async, regmap_async_complete_done, TP_PROTO(struct regmap *map), TP_ARGS(map) - ); TRACE_EVENT(regcache_drop_region, @@ -241,8 +269,7 @@ TRACE_EVENT(regcache_drop_region, __entry->to = to; ), - TP_printk("%s %u-%u", __get_str(name), (unsigned int)__entry->from, - (unsigned int)__entry->to) + TP_printk("%s %u-%u", __get_str(name), __entry->from, __entry->to) ); #endif /* _TRACE_REGMAP_H */ diff --git a/drivers/bcma/driver_gpio.c b/drivers/bcma/driver_gpio.c index fac8ff983aec864af32eb705c738268857052085..65fb9bad1577aaea2065e96bd6d78edefb9a1689 100644 --- a/drivers/bcma/driver_gpio.c +++ b/drivers/bcma/driver_gpio.c @@ -115,7 +115,7 @@ static irqreturn_t bcma_gpio_irq_handler(int irq, void *dev_id) return IRQ_NONE; for_each_set_bit(gpio, &irqs, gc->ngpio) - generic_handle_irq(irq_find_mapping(gc->irq.domain, gpio)); + generic_handle_domain_irq_safe(gc->irq.domain, gpio); bcma_chipco_gpio_polarity(cc, irqs, val & irqs); return IRQ_HANDLED; diff --git a/drivers/bcma/driver_mips.c b/drivers/bcma/driver_mips.c index 12aca34e8db0085b0e4efbcf5fc4caa5779116f6..4f01e6b17bb97cf1b373e8cb756fb4bfb2b2f19c 100644 --- a/drivers/bcma/driver_mips.c +++ b/drivers/bcma/driver_mips.c @@ -30,7 +30,7 @@ enum bcma_boot_dev { BCMA_BOOT_DEV_NAND, }; -/* The 47162a0 hangs when reading MIPS DMP registers registers */ +/* The 47162a0 hangs when reading MIPS DMP registers */ static inline bool bcma_core_mips_bcm47162a0_quirk(struct bcma_device *dev) { return dev->bus->chipinfo.id == BCMA_CHIP_ID_BCM47162 && diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 12b3ca8f6f4a91277c76f9f357e291b78987c5b6..128722cf6c3cad69dede6a2b3f3748f0ed5e00b8 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -108,7 +108,7 @@ static ssize_t aoedisk_show_payload(struct device *dev, return sysfs_emit(page, "%lu\n", d->maxbcnt); } -static int aoedisk_debugfs_show(struct seq_file *s, void *ignored) +static int aoe_debugfs_show(struct seq_file *s, void *ignored) { struct aoedev *d; struct aoetgt **t, **te; @@ -151,11 +151,7 @@ static int aoedisk_debugfs_show(struct seq_file *s, void *ignored) return 0; } - -static int aoe_debugfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, aoedisk_debugfs_show, inode->i_private); -} +DEFINE_SHOW_ATTRIBUTE(aoe_debugfs); static DEVICE_ATTR(state, 0444, aoedisk_show_state, NULL); static DEVICE_ATTR(mac, 0444, aoedisk_show_mac, NULL); @@ -184,13 +180,6 @@ static const struct attribute_group *aoe_attr_groups[] = { NULL, }; -static const struct file_operations aoe_debugfs_fops = { - .open = aoe_debugfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - static void aoedisk_add_debugfs(struct aoedev *d) { diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 859499cd1ff8e405b2ea3c4dfd94ddc8bfa5970c..20acc4a1fd6def4eddbbf63d6d581734f4f9dee9 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -397,7 +397,7 @@ static int brd_alloc(int i) disk->minors = max_part; disk->fops = &brd_fops; disk->private_data = brd; - strlcpy(disk->disk_name, buf, DISK_NAME_LEN); + strscpy(disk->disk_name, buf, DISK_NAME_LEN); set_capacity(disk, rd_size * 2); /* diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index f15f2f0415962230c78d71383703064011292729..4d661282ff412fbc8d88ed1075aedf95fc27df7a 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1529,7 +1529,6 @@ extern int w_send_read_req(struct drbd_work *, int); extern int w_e_reissue(struct drbd_work *, int); extern int w_restart_disk_io(struct drbd_work *, int); extern int w_send_out_of_sync(struct drbd_work *, int); -extern int w_start_resync(struct drbd_work *, int); extern void resync_timer_fn(struct timer_list *t); extern void start_resync_timer_fn(struct timer_list *t); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 013d355a2033bd61f7e20650bf4e64e301ccd479..864c98e748757dd615163091f597be66bf6b4219 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -4752,7 +4752,7 @@ void notify_helper(enum drbd_notification_type type, struct drbd_genlmsghdr *dh; int err; - strlcpy(helper_info.helper_name, name, sizeof(helper_info.helper_name)); + strscpy(helper_info.helper_name, name, sizeof(helper_info.helper_name)); helper_info.helper_name_len = min(strlen(name), sizeof(helper_info.helper_name)); helper_info.helper_status = status; diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index af4c7d65490b9110a824bf4f58fd87970fa3c324..ee69d50ba4fd3bd977fd4c0da72d3a2d4c59ab32 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -781,7 +781,7 @@ static struct socket *drbd_wait_for_connect(struct drbd_connection *connection, timeo = connect_int * HZ; /* 28.5% random jitter */ - timeo += (prandom_u32() & 1) ? timeo / 7 : -timeo / 7; + timeo += prandom_u32_max(2) ? timeo / 7 : -timeo / 7; err = wait_for_completion_interruptible_timeout(&ad->door_bell, timeo); if (err <= 0) @@ -1004,7 +1004,7 @@ retry: drbd_warn(connection, "Error receiving initial packet\n"); sock_release(s); randomize: - if (prandom_u32() & 1) + if (prandom_u32_max(2)) goto retry; } } @@ -2113,9 +2113,6 @@ static int receive_DataReply(struct drbd_connection *connection, struct packet_i if (unlikely(!req)) return -EIO; - /* hlist_del(&req->collision) is done in _req_may_be_done, to avoid - * special casing it there for the various failure cases. - * still no race with drbd_fail_pending_reads */ err = recv_dless_read(peer_device, req, sector, pi->size); if (!err) req_mod(req, DATA_RECEIVED); diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h index 511f39a08de4536642cf26acc4b1ad74b2f4652d..6237fa1dcb0e9652607eb941efa119740d55b351 100644 --- a/drivers/block/drbd/drbd_req.h +++ b/drivers/block/drbd/drbd_req.h @@ -266,8 +266,6 @@ struct bio_and_error { extern void start_new_tl_epoch(struct drbd_connection *connection); extern void drbd_req_destroy(struct kref *kref); -extern void _req_may_be_done(struct drbd_request *req, - struct bio_and_error *m); extern int __req_mod(struct drbd_request *req, enum drbd_req_event what, struct bio_and_error *m); extern void complete_master_bio(struct drbd_device *device, diff --git a/drivers/block/loop.c b/drivers/block/loop.c index e3c0ba93c1a34cd4969d5e9235ded7382b11d113..ad92192c7d617368194fa22de2f9c3a2845f8e62 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -979,6 +979,11 @@ loop_set_status_from_info(struct loop_device *lo, lo->lo_offset = info->lo_offset; lo->lo_sizelimit = info->lo_sizelimit; + + /* loff_t vars have been assigned __u64 */ + if (lo->lo_offset < 0 || lo->lo_sizelimit < 0) + return -EOVERFLOW; + memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); lo->lo_file_name[LO_NAME_SIZE-1] = 0; lo->lo_flags = info->lo_flags; diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 562725d222a7b95a1cde8ed194e04d55bce336d5..815d77ba63819e27da91160b06b27fee32f4b6af 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -1397,15 +1397,15 @@ static void mtip_dump_identify(struct mtip_port *port) if (!port->identify_valid) return; - strlcpy(cbuf, (char *)(port->identify+10), 21); + strscpy(cbuf, (char *)(port->identify + 10), 21); dev_info(&port->dd->pdev->dev, "Serial No.: %s\n", cbuf); - strlcpy(cbuf, (char *)(port->identify+23), 9); + strscpy(cbuf, (char *)(port->identify + 23), 9); dev_info(&port->dd->pdev->dev, "Firmware Ver.: %s\n", cbuf); - strlcpy(cbuf, (char *)(port->identify+27), 41); + strscpy(cbuf, (char *)(port->identify + 27), 41); dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", @@ -1421,13 +1421,13 @@ static void mtip_dump_identify(struct mtip_port *port) pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); switch (revid & 0xFF) { case 0x1: - strlcpy(cbuf, "A0", 3); + strscpy(cbuf, "A0", 3); break; case 0x3: - strlcpy(cbuf, "A2", 3); + strscpy(cbuf, "A2", 3); break; default: - strlcpy(cbuf, "?", 2); + strscpy(cbuf, "?", 2); break; } dev_info(&port->dd->pdev->dev, diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 2a709daefbc465c41f6e03f38382655e9a35b875..5cffd96ef2d7decd29600be8618c8670fdaf2ddf 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -157,8 +157,6 @@ static struct dentry *nbd_dbg_dir; #define nbd_name(nbd) ((nbd)->disk->disk_name) -#define NBD_MAGIC 0x68797548 - #define NBD_DEF_BLKSIZE_BITS 10 static unsigned int nbds_max = 16; @@ -1413,10 +1411,12 @@ static int nbd_start_device_ioctl(struct nbd_device *nbd) mutex_unlock(&nbd->config_lock); ret = wait_event_interruptible(config->recv_wq, atomic_read(&config->recv_threads) == 0); - if (ret) + if (ret) { sock_shutdown(nbd); - flush_workqueue(nbd->recv_workq); + nbd_clear_que(nbd); + } + flush_workqueue(nbd->recv_workq); mutex_lock(&nbd->config_lock); nbd_bdev_reset(nbd); /* user requested, ignore socket errors */ @@ -2322,6 +2322,7 @@ static struct genl_family nbd_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = nbd_connect_genl_ops, .n_small_ops = ARRAY_SIZE(nbd_connect_genl_ops), + .resv_start_op = NBD_CMD_STATUS + 1, .maxattr = NBD_ATTR_MAX, .policy = nbd_attr_policy, .mcgrps = nbd_mcast_grps, diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index c451c477978ffe84a64833e65ede313bee3162aa..1f154f92f4c276bc355ed7f61717289e1e950079 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1528,7 +1528,7 @@ static bool should_requeue_request(struct request *rq) return false; } -static int null_map_queues(struct blk_mq_tag_set *set) +static void null_map_queues(struct blk_mq_tag_set *set) { struct nullb *nullb = set->driver_data; int i, qoff; @@ -1555,7 +1555,9 @@ static int null_map_queues(struct blk_mq_tag_set *set) } else { pr_warn("tag set has unexpected nr_hw_queues: %d\n", set->nr_hw_queues); - return -EINVAL; + WARN_ON_ONCE(true); + submit_queues = 1; + poll_queues = 0; } } @@ -1577,8 +1579,6 @@ static int null_map_queues(struct blk_mq_tag_set *set) qoff += map->nr_queues; blk_mq_map_queues(map); } - - return 0; } static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index e1d080f680edfdd257e49f8ba3d69429be9e12a0..c76e0148eada3302a04b81a886fa7cdb34df2de1 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -745,7 +745,7 @@ static int ps3vram_probe(struct ps3_system_bus_device *dev) gendisk->flags |= GENHD_FL_NO_PART; gendisk->fops = &ps3vram_fops; gendisk->private_data = dev; - strlcpy(gendisk->disk_name, DEVICE_NAME, sizeof(gendisk->disk_name)); + strscpy(gendisk->disk_name, DEVICE_NAME, sizeof(gendisk->disk_name)); set_capacity(gendisk, priv->size >> 9); blk_queue_max_segments(gendisk->queue, BLK_MAX_SEGMENTS); blk_queue_max_segment_size(gendisk->queue, BLK_MAX_SEGMENT_SIZE); diff --git a/drivers/block/rnbd/Makefile b/drivers/block/rnbd/Makefile index 5bb1a7ad1adaf5dec458680aa0b2bebf8709c478..40b31630822ce5b22c504737d076ff74b08b6156 100644 --- a/drivers/block/rnbd/Makefile +++ b/drivers/block/rnbd/Makefile @@ -6,10 +6,12 @@ rnbd-client-y := rnbd-clt.o \ rnbd-clt-sysfs.o \ rnbd-common.o +CFLAGS_rnbd-srv-trace.o = -I$(src) + rnbd-server-y := rnbd-common.o \ rnbd-srv.o \ - rnbd-srv-dev.o \ - rnbd-srv-sysfs.o + rnbd-srv-sysfs.o \ + rnbd-srv-trace.o obj-$(CONFIG_BLK_DEV_RNBD_CLIENT) += rnbd-client.o obj-$(CONFIG_BLK_DEV_RNBD_SERVER) += rnbd-server.o diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c index 04da33a22ef4f5c4c1151a459b78db09d431272b..78334da74d8bf90dcfd1f51d5ccd26f71a289a0c 100644 --- a/drivers/block/rnbd/rnbd-clt.c +++ b/drivers/block/rnbd/rnbd-clt.c @@ -1159,13 +1159,11 @@ static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct rnbd_queue *q = hctx->driver_data; struct rnbd_clt_dev *dev = q->dev; - int cnt; - cnt = rtrs_clt_rdma_cq_direct(dev->sess->rtrs, hctx->queue_num); - return cnt; + return rtrs_clt_rdma_cq_direct(dev->sess->rtrs, hctx->queue_num); } -static int rnbd_rdma_map_queues(struct blk_mq_tag_set *set) +static void rnbd_rdma_map_queues(struct blk_mq_tag_set *set) { struct rnbd_clt_session *sess = set->driver_data; @@ -1194,8 +1192,6 @@ static int rnbd_rdma_map_queues(struct blk_mq_tag_set *set) set->map[HCTX_TYPE_DEFAULT].nr_queues, set->map[HCTX_TYPE_READ].nr_queues); } - - return 0; } static struct blk_mq_ops rnbd_mq_ops = { diff --git a/drivers/block/rnbd/rnbd-srv-dev.c b/drivers/block/rnbd/rnbd-srv-dev.c deleted file mode 100644 index c63017f6e421494d9d83485147da7c44f516bf2a..0000000000000000000000000000000000000000 --- a/drivers/block/rnbd/rnbd-srv-dev.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * RDMA Network Block Driver - * - * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. - * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. - * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. - */ -#undef pr_fmt -#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt - -#include "rnbd-srv-dev.h" -#include "rnbd-log.h" - -struct rnbd_dev *rnbd_dev_open(const char *path, fmode_t flags) -{ - struct rnbd_dev *dev; - int ret; - - dev = kzalloc(sizeof(*dev), GFP_KERNEL); - if (!dev) - return ERR_PTR(-ENOMEM); - - dev->blk_open_flags = flags; - dev->bdev = blkdev_get_by_path(path, flags, THIS_MODULE); - ret = PTR_ERR_OR_ZERO(dev->bdev); - if (ret) - goto err; - - dev->blk_open_flags = flags; - - return dev; - -err: - kfree(dev); - return ERR_PTR(ret); -} - -void rnbd_dev_close(struct rnbd_dev *dev) -{ - blkdev_put(dev->bdev, dev->blk_open_flags); - kfree(dev); -} diff --git a/drivers/block/rnbd/rnbd-srv-dev.h b/drivers/block/rnbd/rnbd-srv-dev.h deleted file mode 100644 index 8407d12f70afee6f61c0a5e9255830d8501a03c7..0000000000000000000000000000000000000000 --- a/drivers/block/rnbd/rnbd-srv-dev.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * RDMA Network Block Driver - * - * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved. - * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved. - * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved. - */ -#ifndef RNBD_SRV_DEV_H -#define RNBD_SRV_DEV_H - -#include -#include "rnbd-proto.h" - -struct rnbd_dev { - struct block_device *bdev; - fmode_t blk_open_flags; -}; - -/** - * rnbd_dev_open() - Open a device - * @path: path to open - * @flags: open flags - */ -struct rnbd_dev *rnbd_dev_open(const char *path, fmode_t flags); - -/** - * rnbd_dev_close() - Close a device - */ -void rnbd_dev_close(struct rnbd_dev *dev); - -void rnbd_endio(void *priv, int error); - -static inline int rnbd_dev_get_max_segs(const struct rnbd_dev *dev) -{ - return queue_max_segments(bdev_get_queue(dev->bdev)); -} - -static inline int rnbd_dev_get_max_hw_sects(const struct rnbd_dev *dev) -{ - return queue_max_hw_sectors(bdev_get_queue(dev->bdev)); -} - -static inline int rnbd_dev_get_secure_discard(const struct rnbd_dev *dev) -{ - return bdev_max_secure_erase_sectors(dev->bdev); -} - -static inline int rnbd_dev_get_max_discard_sects(const struct rnbd_dev *dev) -{ - return bdev_max_discard_sectors(dev->bdev); -} - -static inline int rnbd_dev_get_discard_granularity(const struct rnbd_dev *dev) -{ - return bdev_get_queue(dev->bdev)->limits.discard_granularity; -} - -static inline int rnbd_dev_get_discard_alignment(const struct rnbd_dev *dev) -{ - return bdev_discard_alignment(dev->bdev); -} - -#endif /* RNBD_SRV_DEV_H */ diff --git a/drivers/block/rnbd/rnbd-srv-trace.c b/drivers/block/rnbd/rnbd-srv-trace.c new file mode 100644 index 0000000000000000000000000000000000000000..30f0895c18f5f9d349c16a392ed1d34e457220bb --- /dev/null +++ b/drivers/block/rnbd/rnbd-srv-trace.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#include "rtrs.h" +#include "rtrs-srv.h" +#include "rnbd-srv.h" +#include "rnbd-proto.h" + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "rnbd-srv-trace.h" diff --git a/drivers/block/rnbd/rnbd-srv-trace.h b/drivers/block/rnbd/rnbd-srv-trace.h new file mode 100644 index 0000000000000000000000000000000000000000..8dedf73bdd281a20bb217600d55dc9bd15914ea1 --- /dev/null +++ b/drivers/block/rnbd/rnbd-srv-trace.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rnbd_srv + +#if !defined(_TRACE_RNBD_SRV_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RNBD_SRV_H + +#include + +struct rnbd_srv_session; +struct rtrs_srv_op; + +DECLARE_EVENT_CLASS(rnbd_srv_link_class, + TP_PROTO(struct rnbd_srv_session *srv), + + TP_ARGS(srv), + + TP_STRUCT__entry( + __field(int, qdepth) + __string(sessname, srv->sessname) + ), + + TP_fast_assign( + __entry->qdepth = srv->queue_depth; + __assign_str(sessname, srv->sessname); + ), + + TP_printk("sessname: %s qdepth: %d", + __get_str(sessname), + __entry->qdepth + ) +); + +#define DEFINE_LINK_EVENT(name) \ +DEFINE_EVENT(rnbd_srv_link_class, name, \ + TP_PROTO(struct rnbd_srv_session *srv), \ + TP_ARGS(srv)) + +DEFINE_LINK_EVENT(create_sess); +DEFINE_LINK_EVENT(destroy_sess); + +TRACE_DEFINE_ENUM(RNBD_OP_READ); +TRACE_DEFINE_ENUM(RNBD_OP_WRITE); +TRACE_DEFINE_ENUM(RNBD_OP_FLUSH); +TRACE_DEFINE_ENUM(RNBD_OP_DISCARD); +TRACE_DEFINE_ENUM(RNBD_OP_SECURE_ERASE); +TRACE_DEFINE_ENUM(RNBD_F_SYNC); +TRACE_DEFINE_ENUM(RNBD_F_FUA); + +#define show_rnbd_rw_flags(x) \ + __print_flags(x, "|", \ + { RNBD_OP_READ, "READ" }, \ + { RNBD_OP_WRITE, "WRITE" }, \ + { RNBD_OP_FLUSH, "FLUSH" }, \ + { RNBD_OP_DISCARD, "DISCARD" }, \ + { RNBD_OP_SECURE_ERASE, "SECURE_ERASE" }, \ + { RNBD_F_SYNC, "SYNC" }, \ + { RNBD_F_FUA, "FUA" }) + +TRACE_EVENT(process_rdma, + TP_PROTO(struct rnbd_srv_session *srv, + const struct rnbd_msg_io *msg, + struct rtrs_srv_op *id, + u32 datalen, + size_t usrlen), + + TP_ARGS(srv, msg, id, datalen, usrlen), + + TP_STRUCT__entry( + __string(sessname, srv->sessname) + __field(u8, dir) + __field(u8, ver) + __field(u32, device_id) + __field(u64, sector) + __field(u32, flags) + __field(u32, bi_size) + __field(u16, ioprio) + __field(u32, datalen) + __field(size_t, usrlen) + ), + + TP_fast_assign( + __assign_str(sessname, srv->sessname); + __entry->dir = id->dir; + __entry->ver = srv->ver; + __entry->device_id = le32_to_cpu(msg->device_id); + __entry->sector = le64_to_cpu(msg->sector); + __entry->bi_size = le32_to_cpu(msg->bi_size); + __entry->flags = le32_to_cpu(msg->rw); + __entry->ioprio = le16_to_cpu(msg->prio); + __entry->datalen = datalen; + __entry->usrlen = usrlen; + ), + + TP_printk("I/O req: sess: %s, type: %s, ver: %d, devid: %u, sector: %llu, bsize: %u, flags: %s, ioprio: %d, datalen: %u, usrlen: %zu", + __get_str(sessname), + __print_symbolic(__entry->dir, + { READ, "READ" }, + { WRITE, "WRITE" }), + __entry->ver, + __entry->device_id, + __entry->sector, + __entry->bi_size, + show_rnbd_rw_flags(__entry->flags), + __entry->ioprio, + __entry->datalen, + __entry->usrlen + ) +); + +TRACE_EVENT(process_msg_sess_info, + TP_PROTO(struct rnbd_srv_session *srv, + const struct rnbd_msg_sess_info *msg), + + TP_ARGS(srv, msg), + + TP_STRUCT__entry( + __field(u8, proto_ver) + __field(u8, clt_ver) + __field(u8, srv_ver) + __string(sessname, srv->sessname) + ), + + TP_fast_assign( + __entry->proto_ver = srv->ver; + __entry->clt_ver = msg->ver; + __entry->srv_ver = RNBD_PROTO_VER_MAJOR; + __assign_str(sessname, srv->sessname); + ), + + TP_printk("Session %s using proto-ver %d (clt-ver: %d, srv-ver: %d)", + __get_str(sessname), + __entry->proto_ver, + __entry->clt_ver, + __entry->srv_ver + ) +); + +TRACE_DEFINE_ENUM(RNBD_ACCESS_RO); +TRACE_DEFINE_ENUM(RNBD_ACCESS_RW); +TRACE_DEFINE_ENUM(RNBD_ACCESS_MIGRATION); + +#define show_rnbd_access_mode(x) \ + __print_symbolic(x, \ + { RNBD_ACCESS_RO, "RO" }, \ + { RNBD_ACCESS_RW, "RW" }, \ + { RNBD_ACCESS_MIGRATION, "MIGRATION" }) + +TRACE_EVENT(process_msg_open, + TP_PROTO(struct rnbd_srv_session *srv, + const struct rnbd_msg_open *msg), + + TP_ARGS(srv, msg), + + TP_STRUCT__entry( + __field(u8, access_mode) + __string(sessname, srv->sessname) + __string(dev_name, msg->dev_name) + ), + + TP_fast_assign( + __entry->access_mode = msg->access_mode; + __assign_str(sessname, srv->sessname); + __assign_str(dev_name, msg->dev_name); + ), + + TP_printk("Open message received: session='%s' path='%s' access_mode=%s", + __get_str(sessname), + __get_str(dev_name), + show_rnbd_access_mode(__entry->access_mode) + ) +); + +TRACE_EVENT(process_msg_close, + TP_PROTO(struct rnbd_srv_session *srv, + const struct rnbd_msg_close *msg), + + TP_ARGS(srv, msg), + + TP_STRUCT__entry( + __field(u32, device_id) + __string(sessname, srv->sessname) + ), + + TP_fast_assign( + __entry->device_id = le32_to_cpu(msg->device_id); + __assign_str(sessname, srv->sessname); + ), + + TP_printk("Close message received: session='%s' device id='%d'", + __get_str(sessname), + __entry->device_id + ) +); + +#endif /* _TRACE_RNBD_SRV_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE rnbd-srv-trace +#include + diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 5e08da277ddf8f08cbed6e7547a99fd1a56e24da..2cfed2e58d646f5df11fc4baa6125e1f0527d0e7 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -13,7 +13,7 @@ #include #include "rnbd-srv.h" -#include "rnbd-srv-dev.h" +#include "rnbd-srv-trace.h" MODULE_DESCRIPTION("RDMA Network Block Device Server"); MODULE_LICENSE("GPL"); @@ -84,18 +84,6 @@ static inline void rnbd_put_sess_dev(struct rnbd_srv_sess_dev *sess_dev) kref_put(&sess_dev->kref, rnbd_sess_dev_release); } -void rnbd_endio(void *priv, int error) -{ - struct rnbd_io_private *rnbd_priv = priv; - struct rnbd_srv_sess_dev *sess_dev = rnbd_priv->sess_dev; - - rnbd_put_sess_dev(sess_dev); - - rtrs_srv_resp_rdma(rnbd_priv->id, error); - - kfree(priv); -} - static struct rnbd_srv_sess_dev * rnbd_get_sess_dev(int dev_id, struct rnbd_srv_session *srv_sess) { @@ -116,7 +104,13 @@ rnbd_get_sess_dev(int dev_id, struct rnbd_srv_session *srv_sess) static void rnbd_dev_bi_end_io(struct bio *bio) { - rnbd_endio(bio->bi_private, blk_status_to_errno(bio->bi_status)); + struct rnbd_io_private *rnbd_priv = bio->bi_private; + struct rnbd_srv_sess_dev *sess_dev = rnbd_priv->sess_dev; + + rnbd_put_sess_dev(sess_dev); + rtrs_srv_resp_rdma(rnbd_priv->id, blk_status_to_errno(bio->bi_status)); + + kfree(rnbd_priv); bio_put(bio); } @@ -132,6 +126,8 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, struct bio *bio; short prio; + trace_process_rdma(srv_sess, msg, id, datalen, usrlen); + priv = kmalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -149,7 +145,7 @@ static int process_rdma(struct rnbd_srv_session *srv_sess, priv->sess_dev = sess_dev; priv->id = id; - bio = bio_alloc(sess_dev->rnbd_dev->bdev, 1, + bio = bio_alloc(sess_dev->bdev, 1, rnbd_to_bio_flags(le32_to_cpu(msg->rw)), GFP_KERNEL); if (bio_add_page(bio, virt_to_page(data), datalen, offset_in_page(data)) != datalen) { @@ -223,7 +219,7 @@ void rnbd_destroy_sess_dev(struct rnbd_srv_sess_dev *sess_dev, bool keep_id) rnbd_put_sess_dev(sess_dev); wait_for_completion(&dc); /* wait for inflights to drop to zero */ - rnbd_dev_close(sess_dev->rnbd_dev); + blkdev_put(sess_dev->bdev, sess_dev->open_flags); mutex_lock(&sess_dev->dev->lock); list_del(&sess_dev->dev_list); if (sess_dev->open_flags & FMODE_WRITE) @@ -244,6 +240,8 @@ static void destroy_sess(struct rnbd_srv_session *srv_sess) if (xa_empty(&srv_sess->index_idr)) goto out; + trace_destroy_sess(srv_sess); + mutex_lock(&srv_sess->lock); xa_for_each(&srv_sess->index_idr, index, sess_dev) rnbd_srv_destroy_dev_session_sysfs(sess_dev); @@ -290,6 +288,8 @@ static int create_sess(struct rtrs_srv_sess *rtrs) rtrs_srv_set_sess_priv(rtrs, srv_sess); + trace_create_sess(srv_sess); + return 0; } @@ -332,23 +332,24 @@ void rnbd_srv_sess_dev_force_close(struct rnbd_srv_sess_dev *sess_dev, mutex_unlock(&sess->lock); } -static int process_msg_close(struct rnbd_srv_session *srv_sess, +static void process_msg_close(struct rnbd_srv_session *srv_sess, void *data, size_t datalen, const void *usr, size_t usrlen) { const struct rnbd_msg_close *close_msg = usr; struct rnbd_srv_sess_dev *sess_dev; + trace_process_msg_close(srv_sess, close_msg); + sess_dev = rnbd_get_sess_dev(le32_to_cpu(close_msg->device_id), srv_sess); if (IS_ERR(sess_dev)) - return 0; + return; rnbd_put_sess_dev(sess_dev); mutex_lock(&srv_sess->lock); rnbd_srv_destroy_dev_session_sysfs(sess_dev); mutex_unlock(&srv_sess->lock); - return 0; } static int process_msg_open(struct rnbd_srv_session *srv_sess, @@ -359,10 +360,9 @@ static int process_msg_sess_info(struct rnbd_srv_session *srv_sess, const void *msg, size_t len, void *data, size_t datalen); -static int rnbd_srv_rdma_ev(void *priv, - struct rtrs_srv_op *id, int dir, - void *data, size_t datalen, const void *usr, - size_t usrlen) +static int rnbd_srv_rdma_ev(void *priv, struct rtrs_srv_op *id, + void *data, size_t datalen, + const void *usr, size_t usrlen) { struct rnbd_srv_session *srv_sess = priv; const struct rnbd_msg_hdr *hdr = usr; @@ -378,7 +378,7 @@ static int rnbd_srv_rdma_ev(void *priv, case RNBD_MSG_IO: return process_rdma(srv_sess, id, data, datalen, usr, usrlen); case RNBD_MSG_CLOSE: - ret = process_msg_close(srv_sess, data, datalen, usr, usrlen); + process_msg_close(srv_sess, data, datalen, usr, usrlen); break; case RNBD_MSG_OPEN: ret = process_msg_open(srv_sess, usr, usrlen, data, datalen); @@ -388,11 +388,16 @@ static int rnbd_srv_rdma_ev(void *priv, datalen); break; default: - pr_warn("Received unexpected message type %d with dir %d from session %s\n", - type, dir, srv_sess->sessname); + pr_warn("Received unexpected message type %d from session %s\n", + type, srv_sess->sessname); return -EINVAL; } + /* + * Since ret is passed to rtrs to handle the failure case, we + * just return 0 at the end otherwise callers in rtrs would call + * send_io_resp_imm again to print redundant err message. + */ rtrs_srv_resp_rdma(id, ret); return 0; } @@ -504,14 +509,14 @@ static int rnbd_srv_check_update_open_perm(struct rnbd_srv_dev *srv_dev, } static struct rnbd_srv_dev * -rnbd_srv_get_or_create_srv_dev(struct rnbd_dev *rnbd_dev, +rnbd_srv_get_or_create_srv_dev(struct block_device *bdev, struct rnbd_srv_session *srv_sess, enum rnbd_access_mode access_mode) { int ret; struct rnbd_srv_dev *new_dev, *dev; - new_dev = rnbd_srv_init_srv_dev(rnbd_dev->bdev); + new_dev = rnbd_srv_init_srv_dev(bdev); if (IS_ERR(new_dev)) return new_dev; @@ -531,41 +536,32 @@ rnbd_srv_get_or_create_srv_dev(struct rnbd_dev *rnbd_dev, static void rnbd_srv_fill_msg_open_rsp(struct rnbd_msg_open_rsp *rsp, struct rnbd_srv_sess_dev *sess_dev) { - struct rnbd_dev *rnbd_dev = sess_dev->rnbd_dev; + struct block_device *bdev = sess_dev->bdev; rsp->hdr.type = cpu_to_le16(RNBD_MSG_OPEN_RSP); - rsp->device_id = - cpu_to_le32(sess_dev->device_id); - rsp->nsectors = - cpu_to_le64(get_capacity(rnbd_dev->bdev->bd_disk)); - rsp->logical_block_size = - cpu_to_le16(bdev_logical_block_size(rnbd_dev->bdev)); - rsp->physical_block_size = - cpu_to_le16(bdev_physical_block_size(rnbd_dev->bdev)); - rsp->max_segments = - cpu_to_le16(rnbd_dev_get_max_segs(rnbd_dev)); + rsp->device_id = cpu_to_le32(sess_dev->device_id); + rsp->nsectors = cpu_to_le64(bdev_nr_sectors(bdev)); + rsp->logical_block_size = cpu_to_le16(bdev_logical_block_size(bdev)); + rsp->physical_block_size = cpu_to_le16(bdev_physical_block_size(bdev)); + rsp->max_segments = cpu_to_le16(bdev_max_segments(bdev)); rsp->max_hw_sectors = - cpu_to_le32(rnbd_dev_get_max_hw_sects(rnbd_dev)); + cpu_to_le32(queue_max_hw_sectors(bdev_get_queue(bdev))); rsp->max_write_same_sectors = 0; - rsp->max_discard_sectors = - cpu_to_le32(rnbd_dev_get_max_discard_sects(rnbd_dev)); - rsp->discard_granularity = - cpu_to_le32(rnbd_dev_get_discard_granularity(rnbd_dev)); - rsp->discard_alignment = - cpu_to_le32(rnbd_dev_get_discard_alignment(rnbd_dev)); - rsp->secure_discard = - cpu_to_le16(rnbd_dev_get_secure_discard(rnbd_dev)); + rsp->max_discard_sectors = cpu_to_le32(bdev_max_discard_sectors(bdev)); + rsp->discard_granularity = cpu_to_le32(bdev_discard_granularity(bdev)); + rsp->discard_alignment = cpu_to_le32(bdev_discard_alignment(bdev)); + rsp->secure_discard = cpu_to_le16(bdev_max_secure_erase_sectors(bdev)); rsp->cache_policy = 0; - if (bdev_write_cache(rnbd_dev->bdev)) + if (bdev_write_cache(bdev)) rsp->cache_policy |= RNBD_WRITEBACK; - if (bdev_fua(rnbd_dev->bdev)) + if (bdev_fua(bdev)) rsp->cache_policy |= RNBD_FUA; } static struct rnbd_srv_sess_dev * rnbd_srv_create_set_sess_dev(struct rnbd_srv_session *srv_sess, const struct rnbd_msg_open *open_msg, - struct rnbd_dev *rnbd_dev, fmode_t open_flags, + struct block_device *bdev, fmode_t open_flags, struct rnbd_srv_dev *srv_dev) { struct rnbd_srv_sess_dev *sdev = rnbd_sess_dev_alloc(srv_sess); @@ -577,7 +573,7 @@ rnbd_srv_create_set_sess_dev(struct rnbd_srv_session *srv_sess, strscpy(sdev->pathname, open_msg->dev_name, sizeof(sdev->pathname)); - sdev->rnbd_dev = rnbd_dev; + sdev->bdev = bdev; sdev->sess = srv_sess; sdev->dev = srv_dev; sdev->open_flags = open_flags; @@ -643,9 +639,8 @@ static int process_msg_sess_info(struct rnbd_srv_session *srv_sess, struct rnbd_msg_sess_info_rsp *rsp = data; srv_sess->ver = min_t(u8, sess_info_msg->ver, RNBD_PROTO_VER_MAJOR); - pr_debug("Session %s using protocol version %d (client version: %d, server version: %d)\n", - srv_sess->sessname, srv_sess->ver, - sess_info_msg->ver, RNBD_PROTO_VER_MAJOR); + + trace_process_msg_sess_info(srv_sess, sess_info_msg); rsp->hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO_RSP); rsp->ver = srv_sess->ver; @@ -685,14 +680,13 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess, struct rnbd_srv_dev *srv_dev; struct rnbd_srv_sess_dev *srv_sess_dev; const struct rnbd_msg_open *open_msg = msg; + struct block_device *bdev; fmode_t open_flags; char *full_path; - struct rnbd_dev *rnbd_dev; struct rnbd_msg_open_rsp *rsp = data; - pr_debug("Open message received: session='%s' path='%s' access_mode=%d\n", - srv_sess->sessname, open_msg->dev_name, - open_msg->access_mode); + trace_process_msg_open(srv_sess, open_msg); + open_flags = FMODE_READ; if (open_msg->access_mode != RNBD_ACCESS_RO) open_flags |= FMODE_WRITE; @@ -725,25 +719,25 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess, goto reject; } - rnbd_dev = rnbd_dev_open(full_path, open_flags); - if (IS_ERR(rnbd_dev)) { - pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %ld\n", - full_path, srv_sess->sessname, PTR_ERR(rnbd_dev)); - ret = PTR_ERR(rnbd_dev); + bdev = blkdev_get_by_path(full_path, open_flags, THIS_MODULE); + if (IS_ERR(bdev)) { + ret = PTR_ERR(bdev); + pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %d\n", + full_path, srv_sess->sessname, ret); goto free_path; } - srv_dev = rnbd_srv_get_or_create_srv_dev(rnbd_dev, srv_sess, + srv_dev = rnbd_srv_get_or_create_srv_dev(bdev, srv_sess, open_msg->access_mode); if (IS_ERR(srv_dev)) { pr_err("Opening device '%s' on session %s failed, creating srv_dev failed, err: %ld\n", full_path, srv_sess->sessname, PTR_ERR(srv_dev)); ret = PTR_ERR(srv_dev); - goto rnbd_dev_close; + goto blkdev_put; } srv_sess_dev = rnbd_srv_create_set_sess_dev(srv_sess, open_msg, - rnbd_dev, open_flags, + bdev, open_flags, srv_dev); if (IS_ERR(srv_sess_dev)) { pr_err("Opening device '%s' on session %s failed, creating sess_dev failed, err: %ld\n", @@ -758,7 +752,7 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess, */ mutex_lock(&srv_dev->lock); if (!srv_dev->dev_kobj.state_in_sysfs) { - ret = rnbd_srv_create_dev_sysfs(srv_dev, rnbd_dev->bdev); + ret = rnbd_srv_create_dev_sysfs(srv_dev, bdev); if (ret) { mutex_unlock(&srv_dev->lock); rnbd_srv_err(srv_sess_dev, @@ -800,8 +794,8 @@ srv_dev_put: mutex_unlock(&srv_dev->lock); } rnbd_put_srv_dev(srv_dev); -rnbd_dev_close: - rnbd_dev_close(rnbd_dev); +blkdev_put: + blkdev_put(bdev, open_flags); free_path: kfree(full_path); reject: diff --git a/drivers/block/rnbd/rnbd-srv.h b/drivers/block/rnbd/rnbd-srv.h index 081bceaf4ae9e2c02b3754fff6eabe691339d64d..f5962fd31d62e469a9bd7ed786e2bd64fdcca9aa 100644 --- a/drivers/block/rnbd/rnbd-srv.h +++ b/drivers/block/rnbd/rnbd-srv.h @@ -46,7 +46,7 @@ struct rnbd_srv_dev { struct rnbd_srv_sess_dev { /* Entry inside rnbd_srv_dev struct */ struct list_head dev_list; - struct rnbd_dev *rnbd_dev; + struct block_device *bdev; struct rnbd_srv_session *sess; struct rnbd_srv_dev *dev; struct kobject kobj; diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 6a4a94b4cdf42fa1d56d79d28d73c53aef941391..2651bf41dde31fdf4fa65b27513d4b543e98d9aa 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -49,7 +49,9 @@ /* All UBLK_F_* have to be included into UBLK_F_ALL */ #define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \ | UBLK_F_URING_CMD_COMP_IN_TASK \ - | UBLK_F_NEED_GET_DATA) + | UBLK_F_NEED_GET_DATA \ + | UBLK_F_USER_RECOVERY \ + | UBLK_F_USER_RECOVERY_REISSUE) /* All UBLK_PARAM_TYPE_* should be included here */ #define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD) @@ -119,7 +121,7 @@ struct ublk_queue { unsigned long io_addr; /* mapped vm address */ unsigned int max_io_sz; - bool abort_work_pending; + bool force_abort; unsigned short nr_io_ready; /* how many ios setup */ struct ublk_device *dev; struct ublk_io ios[0]; @@ -161,6 +163,7 @@ struct ublk_device { * monitor each queue's daemon periodically */ struct delayed_work monitor_work; + struct work_struct quiesce_work; struct work_struct stop_work; }; @@ -323,6 +326,30 @@ static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id) PAGE_SIZE); } +static inline bool ublk_queue_can_use_recovery_reissue( + struct ublk_queue *ubq) +{ + if ((ubq->flags & UBLK_F_USER_RECOVERY) && + (ubq->flags & UBLK_F_USER_RECOVERY_REISSUE)) + return true; + return false; +} + +static inline bool ublk_queue_can_use_recovery( + struct ublk_queue *ubq) +{ + if (ubq->flags & UBLK_F_USER_RECOVERY) + return true; + return false; +} + +static inline bool ublk_can_use_recovery(struct ublk_device *ub) +{ + if (ub->dev_info.flags & UBLK_F_USER_RECOVERY) + return true; + return false; +} + static void ublk_free_disk(struct gendisk *disk) { struct ublk_device *ub = disk->private_data; @@ -612,13 +639,17 @@ static void ublk_complete_rq(struct request *req) * Also aborting may not be started yet, keep in mind that one failed * request may be issued by block layer again. */ -static void __ublk_fail_req(struct ublk_io *io, struct request *req) +static void __ublk_fail_req(struct ublk_queue *ubq, struct ublk_io *io, + struct request *req) { WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE); if (!(io->flags & UBLK_IO_FLAG_ABORTED)) { io->flags |= UBLK_IO_FLAG_ABORTED; - blk_mq_end_request(req, BLK_STS_IOERR); + if (ublk_queue_can_use_recovery_reissue(ubq)) + blk_mq_requeue_request(req, false); + else + blk_mq_end_request(req, BLK_STS_IOERR); } } @@ -639,22 +670,40 @@ static void ubq_complete_io_cmd(struct ublk_io *io, int res) #define UBLK_REQUEUE_DELAY_MS 3 +static inline void __ublk_abort_rq(struct ublk_queue *ubq, + struct request *rq) +{ + /* We cannot process this rq so just requeue it. */ + if (ublk_queue_can_use_recovery(ubq)) + blk_mq_requeue_request(rq, false); + else + blk_mq_end_request(rq, BLK_STS_IOERR); + + mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0); +} + static inline void __ublk_rq_task_work(struct request *req) { struct ublk_queue *ubq = req->mq_hctx->driver_data; - struct ublk_device *ub = ubq->dev; int tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; - bool task_exiting = current != ubq->ubq_daemon || ubq_daemon_is_dying(ubq); unsigned int mapped_bytes; pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n", __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags, ublk_get_iod(ubq, req->tag)->addr); - if (unlikely(task_exiting)) { - blk_mq_end_request(req, BLK_STS_IOERR); - mod_delayed_work(system_wq, &ub->monitor_work, 0); + /* + * Task is exiting if either: + * + * (1) current != ubq_daemon. + * io_uring_cmd_complete_in_task() tries to run task_work + * in a workqueue if ubq_daemon(cmd's task) is PF_EXITING. + * + * (2) current->flags & PF_EXITING. + */ + if (unlikely(current != ubq->ubq_daemon || current->flags & PF_EXITING)) { + __ublk_abort_rq(ubq, req); return; } @@ -739,13 +788,24 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, res = ublk_setup_iod(ubq, rq); if (unlikely(res != BLK_STS_OK)) return BLK_STS_IOERR; + /* With recovery feature enabled, force_abort is set in + * ublk_stop_dev() before calling del_gendisk(). We have to + * abort all requeued and new rqs here to let del_gendisk() + * move on. Besides, we cannot not call io_uring_cmd_complete_in_task() + * to avoid UAF on io_uring ctx. + * + * Note: force_abort is guaranteed to be seen because it is set + * before request queue is unqiuesced. + */ + if (ublk_queue_can_use_recovery(ubq) && unlikely(ubq->force_abort)) + return BLK_STS_IOERR; blk_mq_start_request(bd->rq); if (unlikely(ubq_daemon_is_dying(ubq))) { fail: - mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0); - return BLK_STS_IOERR; + __ublk_abort_rq(ubq, rq); + return BLK_STS_OK; } if (ublk_can_use_task_work(ubq)) { @@ -916,7 +976,7 @@ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) */ rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i); if (rq) - __ublk_fail_req(io, rq); + __ublk_fail_req(ubq, io, rq); } } ublk_put_device(ub); @@ -932,7 +992,10 @@ static void ublk_daemon_monitor_work(struct work_struct *work) struct ublk_queue *ubq = ublk_get_queue(ub, i); if (ubq_daemon_is_dying(ubq)) { - schedule_work(&ub->stop_work); + if (ublk_queue_can_use_recovery(ubq)) + schedule_work(&ub->quiesce_work); + else + schedule_work(&ub->stop_work); /* abort queue is for making forward progress */ ublk_abort_queue(ub, ubq); @@ -940,12 +1003,13 @@ static void ublk_daemon_monitor_work(struct work_struct *work) } /* - * We can't schedule monitor work after ublk_remove() is started. + * We can't schedule monitor work after ub's state is not UBLK_S_DEV_LIVE. + * after ublk_remove() or __ublk_quiesce_dev() is started. * * No need ub->mutex, monitor work are canceled after state is marked - * as DEAD, so DEAD state is observed reliably. + * as not LIVE, so new state is observed reliably. */ - if (ub->dev_info.state != UBLK_S_DEV_DEAD) + if (ub->dev_info.state == UBLK_S_DEV_LIVE) schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); } @@ -982,12 +1046,97 @@ static void ublk_cancel_dev(struct ublk_device *ub) ublk_cancel_queue(ublk_get_queue(ub, i)); } -static void ublk_stop_dev(struct ublk_device *ub) +static bool ublk_check_inflight_rq(struct request *rq, void *data) +{ + bool *idle = data; + + if (blk_mq_request_started(rq)) { + *idle = false; + return false; + } + return true; +} + +static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) { + bool idle; + + WARN_ON_ONCE(!blk_queue_quiesced(ub->ub_disk->queue)); + while (true) { + idle = true; + blk_mq_tagset_busy_iter(&ub->tag_set, + ublk_check_inflight_rq, &idle); + if (idle) + break; + msleep(UBLK_REQUEUE_DELAY_MS); + } +} + +static void __ublk_quiesce_dev(struct ublk_device *ub) +{ + pr_devel("%s: quiesce ub: dev_id %d state %s\n", + __func__, ub->dev_info.dev_id, + ub->dev_info.state == UBLK_S_DEV_LIVE ? + "LIVE" : "QUIESCED"); + blk_mq_quiesce_queue(ub->ub_disk->queue); + ublk_wait_tagset_rqs_idle(ub); + ub->dev_info.state = UBLK_S_DEV_QUIESCED; + ublk_cancel_dev(ub); + /* we are going to release task_struct of ubq_daemon and resets + * ->ubq_daemon to NULL. So in monitor_work, check on ubq_daemon causes UAF. + * Besides, monitor_work is not necessary in QUIESCED state since we have + * already scheduled quiesce_work and quiesced all ubqs. + * + * Do not let monitor_work schedule itself if state it QUIESCED. And we cancel + * it here and re-schedule it in END_USER_RECOVERY to avoid UAF. + */ + cancel_delayed_work_sync(&ub->monitor_work); +} + +static void ublk_quiesce_work_fn(struct work_struct *work) +{ + struct ublk_device *ub = + container_of(work, struct ublk_device, quiesce_work); + mutex_lock(&ub->mutex); if (ub->dev_info.state != UBLK_S_DEV_LIVE) goto unlock; + __ublk_quiesce_dev(ub); + unlock: + mutex_unlock(&ub->mutex); +} + +static void ublk_unquiesce_dev(struct ublk_device *ub) +{ + int i; + + pr_devel("%s: unquiesce ub: dev_id %d state %s\n", + __func__, ub->dev_info.dev_id, + ub->dev_info.state == UBLK_S_DEV_LIVE ? + "LIVE" : "QUIESCED"); + /* quiesce_work has run. We let requeued rqs be aborted + * before running fallback_wq. "force_abort" must be seen + * after request queue is unqiuesced. Then del_gendisk() + * can move on. + */ + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) + ublk_get_queue(ub, i)->force_abort = true; + + blk_mq_unquiesce_queue(ub->ub_disk->queue); + /* We may have requeued some rqs in ublk_quiesce_queue() */ + blk_mq_kick_requeue_list(ub->ub_disk->queue); +} +static void ublk_stop_dev(struct ublk_device *ub) +{ + mutex_lock(&ub->mutex); + if (ub->dev_info.state == UBLK_S_DEV_DEAD) + goto unlock; + if (ublk_can_use_recovery(ub)) { + if (ub->dev_info.state == UBLK_S_DEV_LIVE) + __ublk_quiesce_dev(ub); + ublk_unquiesce_dev(ub); + } del_gendisk(ub->ub_disk); ub->dev_info.state = UBLK_S_DEV_DEAD; ub->dev_info.ublksrv_pid = -1; @@ -1311,6 +1460,7 @@ static void ublk_remove(struct ublk_device *ub) { ublk_stop_dev(ub); cancel_work_sync(&ub->stop_work); + cancel_work_sync(&ub->quiesce_work); cdev_device_del(&ub->cdev, &ub->cdev_dev); put_device(&ub->cdev_dev); } @@ -1487,6 +1637,7 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd) goto out_unlock; mutex_init(&ub->mutex); spin_lock_init(&ub->mm_lock); + INIT_WORK(&ub->quiesce_work, ublk_quiesce_work_fn); INIT_WORK(&ub->stop_work, ublk_stop_work_fn); INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work); @@ -1607,6 +1758,7 @@ static int ublk_ctrl_stop_dev(struct io_uring_cmd *cmd) ublk_stop_dev(ub); cancel_work_sync(&ub->stop_work); + cancel_work_sync(&ub->quiesce_work); ublk_put_device(ub); return 0; @@ -1709,6 +1861,116 @@ static int ublk_ctrl_set_params(struct io_uring_cmd *cmd) return ret; } +static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) +{ + int i; + + WARN_ON_ONCE(!(ubq->ubq_daemon && ubq_daemon_is_dying(ubq))); + /* All old ioucmds have to be completed */ + WARN_ON_ONCE(ubq->nr_io_ready); + /* old daemon is PF_EXITING, put it now */ + put_task_struct(ubq->ubq_daemon); + /* We have to reset it to NULL, otherwise ub won't accept new FETCH_REQ */ + ubq->ubq_daemon = NULL; + + for (i = 0; i < ubq->q_depth; i++) { + struct ublk_io *io = &ubq->ios[i]; + + /* forget everything now and be ready for new FETCH_REQ */ + io->flags = 0; + io->cmd = NULL; + io->addr = 0; + } +} + +static int ublk_ctrl_start_recovery(struct io_uring_cmd *cmd) +{ + struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + struct ublk_device *ub; + int ret = -EINVAL; + int i; + + ub = ublk_get_device_from_id(header->dev_id); + if (!ub) + return ret; + + mutex_lock(&ub->mutex); + if (!ublk_can_use_recovery(ub)) + goto out_unlock; + /* + * START_RECOVERY is only allowd after: + * + * (1) UB_STATE_OPEN is not set, which means the dying process is exited + * and related io_uring ctx is freed so file struct of /dev/ublkcX is + * released. + * + * (2) UBLK_S_DEV_QUIESCED is set, which means the quiesce_work: + * (a)has quiesced request queue + * (b)has requeued every inflight rqs whose io_flags is ACTIVE + * (c)has requeued/aborted every inflight rqs whose io_flags is NOT ACTIVE + * (d)has completed/camceled all ioucmds owned by ther dying process + */ + if (test_bit(UB_STATE_OPEN, &ub->state) || + ub->dev_info.state != UBLK_S_DEV_QUIESCED) { + ret = -EBUSY; + goto out_unlock; + } + pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id); + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) + ublk_queue_reinit(ub, ublk_get_queue(ub, i)); + /* set to NULL, otherwise new ubq_daemon cannot mmap the io_cmd_buf */ + ub->mm = NULL; + ub->nr_queues_ready = 0; + init_completion(&ub->completion); + ret = 0; + out_unlock: + mutex_unlock(&ub->mutex); + ublk_put_device(ub); + return ret; +} + +static int ublk_ctrl_end_recovery(struct io_uring_cmd *cmd) +{ + struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; + int ublksrv_pid = (int)header->data[0]; + struct ublk_device *ub; + int ret = -EINVAL; + + ub = ublk_get_device_from_id(header->dev_id); + if (!ub) + return ret; + + pr_devel("%s: Waiting for new ubq_daemons(nr: %d) are ready, dev id %d...\n", + __func__, ub->dev_info.nr_hw_queues, header->dev_id); + /* wait until new ubq_daemon sending all FETCH_REQ */ + wait_for_completion_interruptible(&ub->completion); + pr_devel("%s: All new ubq_daemons(nr: %d) are ready, dev id %d\n", + __func__, ub->dev_info.nr_hw_queues, header->dev_id); + + mutex_lock(&ub->mutex); + if (!ublk_can_use_recovery(ub)) + goto out_unlock; + + if (ub->dev_info.state != UBLK_S_DEV_QUIESCED) { + ret = -EBUSY; + goto out_unlock; + } + ub->dev_info.ublksrv_pid = ublksrv_pid; + pr_devel("%s: new ublksrv_pid %d, dev id %d\n", + __func__, ublksrv_pid, header->dev_id); + blk_mq_unquiesce_queue(ub->ub_disk->queue); + pr_devel("%s: queue unquiesced, dev id %d.\n", + __func__, header->dev_id); + blk_mq_kick_requeue_list(ub->ub_disk->queue); + ub->dev_info.state = UBLK_S_DEV_LIVE; + schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); + ret = 0; + out_unlock: + mutex_unlock(&ub->mutex); + ublk_put_device(ub); + return ret; +} + static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { @@ -1750,6 +2012,12 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, case UBLK_CMD_SET_PARAMS: ret = ublk_ctrl_set_params(cmd); break; + case UBLK_CMD_START_USER_RECOVERY: + ret = ublk_ctrl_start_recovery(cmd); + break; + case UBLK_CMD_END_USER_RECOVERY: + ret = ublk_ctrl_end_recovery(cmd); + break; default: break; } diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 30255fcaf18121c5051ba156e821ee0fd33ba4aa..19da5defd7348d4c32bd0324062abec7f00f2b08 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -130,7 +130,7 @@ static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr) return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC); } -static int virtblk_setup_discard_write_zeroes(struct request *req, bool unmap) +static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap) { unsigned short segments = blk_rq_nr_discard_segments(req); unsigned short n = 0; @@ -240,6 +240,9 @@ static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, type = VIRTIO_BLK_T_WRITE_ZEROES; unmap = !(req->cmd_flags & REQ_NOUNMAP); break; + case REQ_OP_SECURE_ERASE: + type = VIRTIO_BLK_T_SECURE_ERASE; + break; case REQ_OP_DRV_IN: type = VIRTIO_BLK_T_GET_ID; break; @@ -251,8 +254,9 @@ static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev, vbr->out_hdr.type = cpu_to_virtio32(vdev, type); vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req)); - if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES) { - if (virtblk_setup_discard_write_zeroes(req, unmap)) + if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES || + type == VIRTIO_BLK_T_SECURE_ERASE) { + if (virtblk_setup_discard_write_zeroes_erase(req, unmap)) return BLK_STS_RESOURCE; } @@ -322,14 +326,14 @@ static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx, if (unlikely(status)) return status; - blk_mq_start_request(req); - vbr->sg_table.nents = virtblk_map_data(hctx, req, vbr); if (unlikely(vbr->sg_table.nents < 0)) { virtblk_cleanup_cmd(req); return BLK_STS_RESOURCE; } + blk_mq_start_request(req); + return BLK_STS_OK; } @@ -391,8 +395,7 @@ static bool virtblk_prep_rq_batch(struct request *req) } static bool virtblk_add_req_batch(struct virtio_blk_vq *vq, - struct request **rqlist, - struct request **requeue_list) + struct request **rqlist) { unsigned long flags; int err; @@ -408,7 +411,7 @@ static bool virtblk_add_req_batch(struct virtio_blk_vq *vq, if (err) { virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); - rq_list_add(requeue_list, req); + blk_mq_requeue_request(req, true); } } @@ -436,7 +439,7 @@ static void virtio_queue_rqs(struct request **rqlist) if (!next || req->mq_hctx != next->mq_hctx) { req->rq_next = NULL; - kick = virtblk_add_req_batch(vq, rqlist, &requeue_list); + kick = virtblk_add_req_batch(vq, rqlist); if (kick) virtqueue_notify(vq->vq); @@ -802,7 +805,7 @@ static const struct attribute_group *virtblk_attr_groups[] = { NULL, }; -static int virtblk_map_queues(struct blk_mq_tag_set *set) +static void virtblk_map_queues(struct blk_mq_tag_set *set) { struct virtio_blk *vblk = set->driver_data; int i, qoff; @@ -827,8 +830,6 @@ static int virtblk_map_queues(struct blk_mq_tag_set *set) else blk_mq_virtio_map_queues(&set->map[i], vblk->vdev, 0); } - - return 0; } static void virtblk_complete_batch(struct io_comp_batch *iob) @@ -889,6 +890,8 @@ static int virtblk_probe(struct virtio_device *vdev) int err, index; u32 v, blk_size, max_size, sg_elems, opt_io_size; + u32 max_discard_segs = 0; + u32 discard_granularity = 0; u16 min_io_size; u8 physical_block_exp, alignment_offset; unsigned int queue_depth; @@ -1046,27 +1049,14 @@ static int virtblk_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) { virtio_cread(vdev, struct virtio_blk_config, - discard_sector_alignment, &v); - if (v) - q->limits.discard_granularity = v << SECTOR_SHIFT; - else - q->limits.discard_granularity = blk_size; + discard_sector_alignment, &discard_granularity); virtio_cread(vdev, struct virtio_blk_config, max_discard_sectors, &v); blk_queue_max_discard_sectors(q, v ? v : UINT_MAX); virtio_cread(vdev, struct virtio_blk_config, max_discard_seg, - &v); - - /* - * max_discard_seg == 0 is out of spec but we always - * handled it. - */ - if (!v) - v = sg_elems; - blk_queue_max_discard_segments(q, - min(v, MAX_DISCARD_SEGMENTS)); + &max_discard_segs); } if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) { @@ -1075,6 +1065,85 @@ static int virtblk_probe(struct virtio_device *vdev) blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX); } + /* The discard and secure erase limits are combined since the Linux + * block layer uses the same limit for both commands. + * + * If both VIRTIO_BLK_F_SECURE_ERASE and VIRTIO_BLK_F_DISCARD features + * are negotiated, we will use the minimum between the limits. + * + * discard sector alignment is set to the minimum between discard_sector_alignment + * and secure_erase_sector_alignment. + * + * max discard sectors is set to the minimum between max_discard_seg and + * max_secure_erase_seg. + */ + if (virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) { + + virtio_cread(vdev, struct virtio_blk_config, + secure_erase_sector_alignment, &v); + + /* secure_erase_sector_alignment should not be zero, the device should set a + * valid number of sectors. + */ + if (!v) { + dev_err(&vdev->dev, + "virtio_blk: secure_erase_sector_alignment can't be 0\n"); + err = -EINVAL; + goto out_cleanup_disk; + } + + discard_granularity = min_not_zero(discard_granularity, v); + + virtio_cread(vdev, struct virtio_blk_config, + max_secure_erase_sectors, &v); + + /* max_secure_erase_sectors should not be zero, the device should set a + * valid number of sectors. + */ + if (!v) { + dev_err(&vdev->dev, + "virtio_blk: max_secure_erase_sectors can't be 0\n"); + err = -EINVAL; + goto out_cleanup_disk; + } + + blk_queue_max_secure_erase_sectors(q, v); + + virtio_cread(vdev, struct virtio_blk_config, + max_secure_erase_seg, &v); + + /* max_secure_erase_seg should not be zero, the device should set a + * valid number of segments + */ + if (!v) { + dev_err(&vdev->dev, + "virtio_blk: max_secure_erase_seg can't be 0\n"); + err = -EINVAL; + goto out_cleanup_disk; + } + + max_discard_segs = min_not_zero(max_discard_segs, v); + } + + if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD) || + virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) { + /* max_discard_seg and discard_granularity will be 0 only + * if max_discard_seg and discard_sector_alignment fields in the virtio + * config are 0 and VIRTIO_BLK_F_SECURE_ERASE feature is not negotiated. + * In this case, we use default values. + */ + if (!max_discard_segs) + max_discard_segs = sg_elems; + + blk_queue_max_discard_segments(q, + min(max_discard_segs, MAX_DISCARD_SEGMENTS)); + + if (discard_granularity) + q->limits.discard_granularity = discard_granularity << SECTOR_SHIFT; + else + q->limits.discard_granularity = blk_size; + } + virtblk_update_capacity(vblk, false); virtio_device_ready(vdev); @@ -1170,6 +1239,7 @@ static unsigned int features_legacy[] = { VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, + VIRTIO_BLK_F_SECURE_ERASE, } ; static unsigned int features[] = { @@ -1177,6 +1247,7 @@ static unsigned int features[] = { VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES, + VIRTIO_BLK_F_SECURE_ERASE, }; static struct virtio_driver virtio_blk = { diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h index bda5c815e44156da9a2ba660cf5150156d171721..a28473470e66229ac530326b79646cc7fedd315c 100644 --- a/drivers/block/xen-blkback/common.h +++ b/drivers/block/xen-blkback/common.h @@ -226,6 +226,9 @@ struct xen_vbd { sector_t size; unsigned int flush_support:1; unsigned int discard_secure:1; + /* Connect-time cached feature_persistent parameter value */ + unsigned int feature_gnt_persistent_parm:1; + /* Persistent grants feature negotiation result */ unsigned int feature_gnt_persistent:1; unsigned int overflow_max_grants:1; }; diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c index ee7ad2fb432d1b9b4ab81afc56614fe3d54a0c1c..c0227dfa468879cb3af300caba282b620f3ac824 100644 --- a/drivers/block/xen-blkback/xenbus.c +++ b/drivers/block/xen-blkback/xenbus.c @@ -907,7 +907,7 @@ again: xen_blkbk_barrier(xbt, be, be->blkif->vbd.flush_support); err = xenbus_printf(xbt, dev->nodename, "feature-persistent", "%u", - be->blkif->vbd.feature_gnt_persistent); + be->blkif->vbd.feature_gnt_persistent_parm); if (err) { xenbus_dev_fatal(dev, err, "writing %s/feature-persistent", dev->nodename); @@ -1085,7 +1085,9 @@ static int connect_ring(struct backend_info *be) return -ENOSYS; } - blkif->vbd.feature_gnt_persistent = feature_persistent && + blkif->vbd.feature_gnt_persistent_parm = feature_persistent; + blkif->vbd.feature_gnt_persistent = + blkif->vbd.feature_gnt_persistent_parm && xenbus_read_unsigned(dev->otherend, "feature-persistent", 0); blkif->vbd.overflow_max_grants = 0; diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 8e56e69fb4c4e47f6886095493902946abe72fdf..35b9bcad9db901e9f0b655cc88b97b43866e28c3 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -213,6 +213,9 @@ struct blkfront_info unsigned int feature_fua:1; unsigned int feature_discard:1; unsigned int feature_secdiscard:1; + /* Connect-time cached feature_persistent parameter */ + unsigned int feature_persistent_parm:1; + /* Persistent grants feature negotiation result */ unsigned int feature_persistent:1; unsigned int bounce:1; unsigned int discard_granularity; @@ -1756,6 +1759,12 @@ abort_transaction: return err; } +/* Enable the persistent grants feature. */ +static bool feature_persistent = true; +module_param(feature_persistent, bool, 0644); +MODULE_PARM_DESC(feature_persistent, + "Enables the persistent grants feature"); + /* Common code used when first setting up, and when resuming. */ static int talk_to_blkback(struct xenbus_device *dev, struct blkfront_info *info) @@ -1847,8 +1856,9 @@ again: message = "writing protocol"; goto abort_transaction; } + info->feature_persistent_parm = feature_persistent; err = xenbus_printf(xbt, dev->nodename, "feature-persistent", "%u", - info->feature_persistent); + info->feature_persistent_parm); if (err) dev_warn(&dev->dev, "writing persistent grants feature to xenbus"); @@ -1916,12 +1926,6 @@ static int negotiate_mq(struct blkfront_info *info) return 0; } -/* Enable the persistent grants feature. */ -static bool feature_persistent = true; -module_param(feature_persistent, bool, 0644); -MODULE_PARM_DESC(feature_persistent, - "Enables the persistent grants feature"); - /* * Entry point to this code when a new device is created. Allocate the basic * structures and the ring buffer for communication with the backend, and @@ -2281,7 +2285,7 @@ static void blkfront_gather_backend_features(struct blkfront_info *info) if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0)) blkfront_setup_discard(info); - if (feature_persistent) + if (info->feature_persistent_parm) info->feature_persistent = !!xenbus_read_unsigned(info->xbdev->otherend, "feature-persistent", 0); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 92cb929a45b79ae26090233cb83c300b5c937655..966aab902d19a6c1a4720fc78138761e267b23af 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -52,9 +52,6 @@ static unsigned int num_devices = 1; static size_t huge_class_size; static const struct block_device_operations zram_devops; -#ifdef CONFIG_ZRAM_WRITEBACK -static const struct block_device_operations zram_wb_devops; -#endif static void zram_free_page(struct zram *zram, size_t index); static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, @@ -329,8 +326,8 @@ static ssize_t idle_store(struct device *dev, if (!sysfs_streq(buf, "all")) { /* - * If it did not parse as 'all' try to treat it as an integer when - * we have memory tracking enabled. + * If it did not parse as 'all' try to treat it as an integer + * when we have memory tracking enabled. */ u64 age_sec; @@ -345,7 +342,10 @@ static ssize_t idle_store(struct device *dev, if (!init_done(zram)) goto out_unlock; - /* A cutoff_time of 0 marks everything as idle, this is the "all" behavior */ + /* + * A cutoff_time of 0 marks everything as idle, this is the + * "all" behavior. + */ mark_idle(zram, cutoff_time); rv = len; @@ -499,7 +499,7 @@ static ssize_t backing_dev_store(struct device *dev, goto out; } - strlcpy(file_name, buf, PATH_MAX); + strscpy(file_name, buf, PATH_MAX); /* ignore trailing newline */ sz = strlen(file_name); if (sz > 0 && file_name[sz - 1] == '\n') @@ -543,17 +543,6 @@ static ssize_t backing_dev_store(struct device *dev, zram->backing_dev = backing_dev; zram->bitmap = bitmap; zram->nr_pages = nr_pages; - /* - * With writeback feature, zram does asynchronous IO so it's no longer - * synchronous device so let's remove synchronous io flag. Othewise, - * upper layer(e.g., swap) could wait IO completion rather than - * (submit and return), which will cause system sluggish. - * Furthermore, when the IO function returns(e.g., swap_readpage), - * upper layer expects IO was done so it could deallocate the page - * freely but in fact, IO is going on so finally could cause - * use-after-free when the IO is really done. - */ - zram->disk->fops = &zram_wb_devops; up_write(&zram->init_lock); pr_info("setup backing device %s\n", file_name); @@ -1031,7 +1020,7 @@ static ssize_t comp_algorithm_store(struct device *dev, char compressor[ARRAY_SIZE(zram->compressor)]; size_t sz; - strlcpy(compressor, buf, sizeof(compressor)); + strscpy(compressor, buf, sizeof(compressor)); /* ignore trailing newline */ sz = strlen(compressor); if (sz > 0 && compressor[sz - 1] == '\n') @@ -1146,14 +1135,15 @@ static ssize_t bd_stat_show(struct device *dev, static ssize_t debug_stat_show(struct device *dev, struct device_attribute *attr, char *buf) { - int version = 2; + int version = 1; struct zram *zram = dev_to_zram(dev); ssize_t ret; down_read(&zram->init_lock); ret = scnprintf(buf, PAGE_SIZE, - "version: %d\n%8llu\n", + "version: %d\n%8llu %8llu\n", version, + (u64)atomic64_read(&zram->stats.writestall), (u64)atomic64_read(&zram->stats.miss_free)); up_read(&zram->init_lock); @@ -1266,6 +1256,9 @@ static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index, struct bio_vec bvec; zram_slot_unlock(zram, index); + /* A null bio means rw_page was used, we must fallback to bio */ + if (!bio) + return -EOPNOTSUPP; bvec.bv_page = page; bvec.bv_len = PAGE_SIZE; @@ -1351,7 +1344,7 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, { int ret = 0; unsigned long alloced_pages; - unsigned long handle = 0; + unsigned long handle = -ENOMEM; unsigned int comp_len = 0; void *src, *dst, *mem; struct zcomp_strm *zstrm; @@ -1369,6 +1362,7 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, } kunmap_atomic(mem); +compress_again: zstrm = zcomp_stream_get(zram->comp); src = kmap_atomic(page); ret = zcomp_compress(zstrm, src, &comp_len); @@ -1377,21 +1371,50 @@ static int __zram_bvec_write(struct zram *zram, struct bio_vec *bvec, if (unlikely(ret)) { zcomp_stream_put(zram->comp); pr_err("Compression failed! err=%d\n", ret); + zs_free(zram->mem_pool, handle); return ret; } if (comp_len >= huge_class_size) comp_len = PAGE_SIZE; - - handle = zs_malloc(zram->mem_pool, comp_len, - __GFP_KSWAPD_RECLAIM | - __GFP_NOWARN | - __GFP_HIGHMEM | - __GFP_MOVABLE); - + /* + * handle allocation has 2 paths: + * a) fast path is executed with preemption disabled (for + * per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear, + * since we can't sleep; + * b) slow path enables preemption and attempts to allocate + * the page with __GFP_DIRECT_RECLAIM bit set. we have to + * put per-cpu compression stream and, thus, to re-do + * the compression once handle is allocated. + * + * if we have a 'non-null' handle here then we are coming + * from the slow path and handle has already been allocated. + */ + if (IS_ERR((void *)handle)) + handle = zs_malloc(zram->mem_pool, comp_len, + __GFP_KSWAPD_RECLAIM | + __GFP_NOWARN | + __GFP_HIGHMEM | + __GFP_MOVABLE); if (IS_ERR((void *)handle)) { zcomp_stream_put(zram->comp); - return PTR_ERR((void *)handle); + atomic64_inc(&zram->stats.writestall); + handle = zs_malloc(zram->mem_pool, comp_len, + GFP_NOIO | __GFP_HIGHMEM | + __GFP_MOVABLE); + if (IS_ERR((void *)handle)) + return PTR_ERR((void *)handle); + + if (comp_len != PAGE_SIZE) + goto compress_again; + /* + * If the page is not compressible, you need to acquire the + * lock and execute the code below. The zcomp_stream_get() + * call is needed to disable the cpu hotplug and grab the + * zstrm buffer back. It is necessary that the dereferencing + * of the zstrm variable below occurs correctly. + */ + zstrm = zcomp_stream_get(zram->comp); } alloced_pages = zs_get_total_pages(zram->mem_pool); @@ -1689,9 +1712,6 @@ out: static void zram_reset_device(struct zram *zram) { - struct zcomp *comp; - u64 disksize; - down_write(&zram->init_lock); zram->limit_pages = 0; @@ -1701,17 +1721,15 @@ static void zram_reset_device(struct zram *zram) return; } - comp = zram->comp; - disksize = zram->disksize; - zram->disksize = 0; - set_capacity_and_notify(zram->disk, 0); part_stat_set_all(zram->disk->part0, 0); /* I/O operation under all of CPU are done so let's free */ - zram_meta_free(zram, disksize); + zram_meta_free(zram, zram->disksize); + zram->disksize = 0; memset(&zram->stats, 0, sizeof(zram->stats)); - zcomp_destroy(comp); + zcomp_destroy(zram->comp); + zram->comp = NULL; reset_bdev(zram); up_write(&zram->init_lock); @@ -1827,15 +1845,6 @@ static const struct block_device_operations zram_devops = { .owner = THIS_MODULE }; -#ifdef CONFIG_ZRAM_WRITEBACK -static const struct block_device_operations zram_wb_devops = { - .open = zram_open, - .submit_bio = zram_submit_bio, - .swap_slot_free_notify = zram_slot_free_notify, - .owner = THIS_MODULE -}; -#endif - static DEVICE_ATTR_WO(compact); static DEVICE_ATTR_RW(disksize); static DEVICE_ATTR_RO(initstate); @@ -1948,11 +1957,12 @@ static int zram_add(void) if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE) blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX); + blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue); ret = device_add_disk(NULL, zram->disk, zram_disk_groups); if (ret) goto out_cleanup_disk; - strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); + strscpy(zram->compressor, default_compressor, sizeof(zram->compressor)); zram_debugfs_register(zram); pr_info("Added device: %s\n", zram->disk->disk_name); @@ -2104,6 +2114,8 @@ static int __init zram_init(void) { int ret; + BUILD_BUG_ON(__NR_ZRAM_PAGEFLAGS > BITS_PER_LONG); + ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare", zcomp_cpu_up_prepare, zcomp_cpu_dead); if (ret < 0) diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 158c91e5485018e6a56fcea418fc953ba5c455d5..a2bda53020fddeb42d8b053abd68da8b91cf8f86 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -30,16 +30,15 @@ /* - * The lower ZRAM_FLAG_SHIFT bits of table.flags is for - * object size (excluding header), the higher bits is for - * zram_pageflags. + * ZRAM is mainly used for memory efficiency so we want to keep memory + * footprint small and thus squeeze size and zram pageflags into a flags + * member. The lower ZRAM_FLAG_SHIFT bits is for object size (excluding + * header), which cannot be larger than PAGE_SIZE (requiring PAGE_SHIFT + * bits), the higher bits are for zram_pageflags. * - * zram is mainly used for memory efficiency so we want to keep memory - * footprint small so we can squeeze size and flags into a field. - * The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header), - * the higher bits is for zram_pageflags. + * We use BUILD_BUG_ON() to make sure that zram pageflags don't overflow. */ -#define ZRAM_FLAG_SHIFT 24 +#define ZRAM_FLAG_SHIFT (PAGE_SHIFT + 1) /* Flags for zram pages (table[page_no].flags) */ enum zram_pageflags { @@ -81,6 +80,7 @@ struct zram_stats { atomic64_t huge_pages_since; /* no. of huge pages since zram set up */ atomic64_t pages_stored; /* no. of pages currently stored */ atomic_long_t max_used_pages; /* no. of maximum pages stored */ + atomic64_t writestall; /* no. of write slow paths */ atomic64_t miss_free; /* no. of missed free */ #ifdef CONFIG_ZRAM_WRITEBACK atomic64_t bd_count; /* no. of pages in backing device */ diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 818681c89db8bebda7863203005f9b2f19b68f5a..a657e9a3e96a55ac9423f1b0f43080e12102940c 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -449,6 +449,7 @@ static int btintel_version_info_tlv(struct hci_dev *hdev, case 0x17: /* TyP */ case 0x18: /* Slr */ case 0x19: /* Slr-F */ + case 0x1b: /* Mgr */ break; default: bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%x)", @@ -2330,6 +2331,7 @@ static void btintel_set_msft_opcode(struct hci_dev *hdev, u8 hw_variant) case 0x17: case 0x18: case 0x19: + case 0x1b: hci_set_msft_opcode(hdev, 0xFC1E); break; default: @@ -2439,15 +2441,20 @@ static int btintel_setup_combined(struct hci_dev *hdev) INTEL_ROM_LEGACY_NO_WBS_SUPPORT)) set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks); + if (ver.hw_variant == 0x08 && ver.fw_variant == 0x22) + set_bit(HCI_QUIRK_VALID_LE_STATES, + &hdev->quirks); err = btintel_legacy_rom_setup(hdev, &ver); break; case 0x0b: /* SfP */ - case 0x0c: /* WsP */ case 0x11: /* JfP */ case 0x12: /* ThP */ case 0x13: /* HrP */ case 0x14: /* CcP */ + set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks); + fallthrough; + case 0x0c: /* WsP */ /* Apply the device specific HCI quirks * * All Legacy bootloader devices support WBS @@ -2455,11 +2462,6 @@ static int btintel_setup_combined(struct hci_dev *hdev) set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks); - /* Valid LE States quirk for JfP/ThP familiy */ - if (ver.hw_variant == 0x11 || ver.hw_variant == 0x12) - set_bit(HCI_QUIRK_VALID_LE_STATES, - &hdev->quirks); - /* Setup MSFT Extension support */ btintel_set_msft_opcode(hdev, ver.hw_variant); @@ -2530,9 +2532,8 @@ static int btintel_setup_combined(struct hci_dev *hdev) */ set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks); - /* Valid LE States quirk for JfP/ThP familiy */ - if (ver.hw_variant == 0x11 || ver.hw_variant == 0x12) - set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks); + /* Set Valid LE States quirk */ + set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks); /* Setup MSFT Extension support */ btintel_set_msft_opcode(hdev, ver.hw_variant); @@ -2542,6 +2543,7 @@ static int btintel_setup_combined(struct hci_dev *hdev) case 0x17: case 0x18: case 0x19: + case 0x1b: /* Display version information of TLV type */ btintel_version_info_tlv(hdev, &ver_tlv); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 15caa646953893fd0b528f6f85a62bae2e68734a..271963805a3841498c3a8bde4d999c45d5ec2a7b 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -426,6 +426,8 @@ static const struct usb_device_id blacklist_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0cb8, 0xc549), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek 8852CE Bluetooth devices */ { USB_DEVICE(0x04ca, 0x4007), .driver_info = BTUSB_REALTEK | @@ -438,6 +440,8 @@ static const struct usb_device_id blacklist_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3586), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Realtek Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01), @@ -466,6 +470,9 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH | BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x0489, 0xe0e0), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, { USB_DEVICE(0x04ca, 0x3802), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH | BTUSB_VALID_LE_STATES }, @@ -478,9 +485,18 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x13d3, 0x3567), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH | BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x13d3, 0x3578), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x13d3, 0x3583), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, { USB_DEVICE(0x0489, 0xe0cd), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH | BTUSB_VALID_LE_STATES }, + { USB_DEVICE(0x0e8d, 0x0608), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH | + BTUSB_VALID_LE_STATES }, /* MediaTek MT7922A Bluetooth devices */ { USB_DEVICE(0x0489, 0xe0d8), .driver_info = BTUSB_MEDIATEK | @@ -516,19 +532,17 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK }, - /* Additional Realtek 8761B Bluetooth devices */ + /* Additional Realtek 8761BUV Bluetooth devices */ { USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, - - /* Additional Realtek 8761BU Bluetooth devices */ { USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, - - /* Additional Realtek 8761BUV Bluetooth devices */ { USB_DEVICE(0x0bda, 0x8771), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x7392, 0xc611), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional Realtek 8821AE Bluetooth devices */ { USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK }, @@ -2477,15 +2491,29 @@ static int btusb_mtk_hci_wmt_sync(struct hci_dev *hdev, set_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags); + /* WMT cmd/event doesn't follow up the generic HCI cmd/event handling, + * it needs constantly polling control pipe until the host received the + * WMT event, thus, we should require to specifically acquire PM counter + * on the USB to prevent the interface from entering auto suspended + * while WMT cmd/event in progress. + */ + err = usb_autopm_get_interface(data->intf); + if (err < 0) + goto err_free_wc; + err = __hci_cmd_send(hdev, 0xfc6f, hlen, wc); if (err < 0) { clear_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags); + usb_autopm_put_interface(data->intf); goto err_free_wc; } /* Submit control IN URB on demand to process the WMT event */ err = btusb_mtk_submit_wmt_recv_urb(hdev); + + usb_autopm_put_interface(data->intf); + if (err < 0) goto err_free_wc; diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index f537673ede17440acafc13e3d74040c989e5a733..865112e96ff9fc669258b1ef049c52e997c71575 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -493,6 +493,11 @@ static int hci_uart_tty_open(struct tty_struct *tty) BT_ERR("Can't allocate control structure"); return -ENFILE; } + if (percpu_init_rwsem(&hu->proto_lock)) { + BT_ERR("Can't allocate semaphore structure"); + kfree(hu); + return -ENOMEM; + } tty->disc_data = hu; hu->tty = tty; @@ -505,8 +510,6 @@ static int hci_uart_tty_open(struct tty_struct *tty) INIT_WORK(&hu->init_ready, hci_uart_init_work); INIT_WORK(&hu->write_work, hci_uart_write_work); - percpu_init_rwsem(&hu->proto_lock); - /* Flush any pending characters in the driver */ tty_driver_flush_buffer(tty); diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index c0e5f42ec6b7df50b5fea4e67deee6e00f39440c..f16fd79bc02b8ab6c8e984faf6bf4c5bdb42bd8f 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -310,11 +310,12 @@ int hci_uart_register_device(struct hci_uart *hu, serdev_device_set_client_ops(hu->serdev, &hci_serdev_client_ops); + if (percpu_init_rwsem(&hu->proto_lock)) + return -ENOMEM; + err = serdev_device_open(hu->serdev); if (err) - return err; - - percpu_init_rwsem(&hu->proto_lock); + goto err_rwsem; err = p->open(hu); if (err) @@ -389,6 +390,8 @@ err_alloc: p->close(hu); err_open: serdev_device_close(hu->serdev); +err_rwsem: + percpu_free_rwsem(&hu->proto_lock); return err; } EXPORT_SYMBOL_GPL(hci_uart_register_device); @@ -410,5 +413,6 @@ void hci_uart_unregister_device(struct hci_uart *hu) clear_bit(HCI_UART_PROTO_READY, &hu->flags); serdev_device_close(hu->serdev); } + percpu_free_rwsem(&hu->proto_lock); } EXPORT_SYMBOL_GPL(hci_uart_unregister_device); diff --git a/drivers/bus/hisi_lpc.c b/drivers/bus/hisi_lpc.c index 2e564803e7869ec83c5f468840e9e39f8e00998e..5b65a48f17e7cec90823b682ca2313dafbd917c7 100644 --- a/drivers/bus/hisi_lpc.c +++ b/drivers/bus/hisi_lpc.c @@ -85,7 +85,7 @@ static int wait_lpc_idle(void __iomem *mbase, unsigned int waitcnt) ndelay(LPC_NSEC_PERWAIT); } while (--waitcnt); - return -ETIME; + return -ETIMEDOUT; } /* @@ -347,7 +347,7 @@ static int hisi_lpc_acpi_xlat_io_res(struct acpi_device *adev, unsigned long sys_port; resource_size_t len = resource_size(res); - sys_port = logic_pio_trans_hwaddr(&host->fwnode, res->start, len); + sys_port = logic_pio_trans_hwaddr(acpi_fwnode_handle(host), res->start, len); if (sys_port == ~0UL) return -EFAULT; @@ -472,9 +472,7 @@ static int hisi_lpc_acpi_clear_enumerated(struct acpi_device *adev, void *not_us struct hisi_lpc_acpi_cell { const char *hid; - const char *name; - void *pdata; - size_t pdata_size; + const struct platform_device_info *pdevinfo; }; static void hisi_lpc_acpi_remove(struct device *hostdev) @@ -505,28 +503,45 @@ static int hisi_lpc_acpi_add_child(struct acpi_device *child, void *data) /* ipmi */ { .hid = "IPI0001", - .name = "hisi-lpc-ipmi", + .pdevinfo = (struct platform_device_info []) { + { + .parent = hostdev, + .fwnode = acpi_fwnode_handle(child), + .name = "hisi-lpc-ipmi", + .id = PLATFORM_DEVID_AUTO, + .res = res, + .num_res = num_res, + }, + }, }, /* 8250-compatible uart */ { .hid = "HISI1031", - .name = "serial8250", - .pdata = (struct plat_serial8250_port []) { + .pdevinfo = (struct platform_device_info []) { { - .iobase = res->start, - .uartclk = 1843200, - .iotype = UPIO_PORT, - .flags = UPF_BOOT_AUTOCONF, + .parent = hostdev, + .fwnode = acpi_fwnode_handle(child), + .name = "serial8250", + .id = PLATFORM_DEVID_AUTO, + .res = res, + .num_res = num_res, + .data = (struct plat_serial8250_port []) { + { + .iobase = res->start, + .uartclk = 1843200, + .iotype = UPIO_PORT, + .flags = UPF_BOOT_AUTOCONF, + }, + {} + }, + .size_data = 2 * sizeof(struct plat_serial8250_port), }, - {} }, - .pdata_size = 2 * - sizeof(struct plat_serial8250_port), }, {} }; - for (; cell && cell->name; cell++) { + for (; cell && cell->hid; cell++) { if (!strcmp(cell->hid, hid)) { found = true; break; @@ -540,31 +555,12 @@ static int hisi_lpc_acpi_add_child(struct acpi_device *child, void *data) return 0; } - pdev = platform_device_alloc(cell->name, PLATFORM_DEVID_AUTO); - if (!pdev) - return -ENOMEM; - - pdev->dev.parent = hostdev; - ACPI_COMPANION_SET(&pdev->dev, child); - - ret = platform_device_add_resources(pdev, res, num_res); - if (ret) - goto fail; - - ret = platform_device_add_data(pdev, cell->pdata, cell->pdata_size); - if (ret) - goto fail; - - ret = platform_device_add(pdev); - if (ret) - goto fail; + pdev = platform_device_register_full(cell->pdevinfo); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); acpi_device_set_enumerated(child); return 0; - -fail: - platform_device_put(pdev); - return ret; } /* @@ -589,11 +585,6 @@ static int hisi_lpc_acpi_probe(struct device *hostdev) return ret; } - -static const struct acpi_device_id hisi_lpc_acpi_match[] = { - {"HISI0191"}, - {} -}; #else static int hisi_lpc_acpi_probe(struct device *dev) { @@ -615,11 +606,9 @@ static void hisi_lpc_acpi_remove(struct device *hostdev) static int hisi_lpc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct acpi_device *acpi_device = ACPI_COMPANION(dev); struct logic_pio_hwaddr *range; struct hisi_lpc_dev *lpcdev; resource_size_t io_end; - struct resource *res; int ret; lpcdev = devm_kzalloc(dev, sizeof(*lpcdev), GFP_KERNEL); @@ -628,8 +617,7 @@ static int hisi_lpc_probe(struct platform_device *pdev) spin_lock_init(&lpcdev->cycle_lock); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - lpcdev->membase = devm_ioremap_resource(dev, res); + lpcdev->membase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(lpcdev->membase)) return PTR_ERR(lpcdev->membase); @@ -637,7 +625,7 @@ static int hisi_lpc_probe(struct platform_device *pdev) if (!range) return -ENOMEM; - range->fwnode = dev->fwnode; + range->fwnode = dev_fwnode(dev); range->flags = LOGIC_PIO_INDIRECT; range->size = PIO_INDIRECT_SIZE; range->hostdata = lpcdev; @@ -651,7 +639,7 @@ static int hisi_lpc_probe(struct platform_device *pdev) } /* register the LPC host PIO resources */ - if (acpi_device) + if (is_acpi_device_node(range->fwnode)) ret = hisi_lpc_acpi_probe(dev); else ret = of_platform_populate(dev->of_node, NULL, NULL, dev); @@ -672,11 +660,10 @@ static int hisi_lpc_probe(struct platform_device *pdev) static int hisi_lpc_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct acpi_device *acpi_device = ACPI_COMPANION(dev); struct hisi_lpc_dev *lpcdev = dev_get_drvdata(dev); struct logic_pio_hwaddr *range = lpcdev->io_host; - if (acpi_device) + if (is_acpi_device_node(range->fwnode)) hisi_lpc_acpi_remove(dev); else of_platform_depopulate(dev); @@ -692,11 +679,16 @@ static const struct of_device_id hisi_lpc_of_match[] = { {} }; +static const struct acpi_device_id hisi_lpc_acpi_match[] = { + {"HISI0191"}, + {} +}; + static struct platform_driver hisi_lpc_driver = { .driver = { .name = DRV_NAME, .of_match_table = hisi_lpc_of_match, - .acpi_match_table = ACPI_PTR(hisi_lpc_acpi_match), + .acpi_match_table = hisi_lpc_acpi_match, }, .probe = hisi_lpc_probe, .remove = hisi_lpc_remove, diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index f3aef77a6a4a758425af8ff9796939742a57d79f..df0fbfee7b78b093387a7c24aa223ceed17846a7 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -430,12 +430,25 @@ irqreturn_t mhi_irq_handler(int irq_number, void *dev) { struct mhi_event *mhi_event = dev; struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; - struct mhi_event_ctxt *er_ctxt = - &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct mhi_event_ctxt *er_ctxt; struct mhi_ring *ev_ring = &mhi_event->ring; - dma_addr_t ptr = le64_to_cpu(er_ctxt->rp); + dma_addr_t ptr; void *dev_rp; + /* + * If CONFIG_DEBUG_SHIRQ is set, the IRQ handler will get invoked during __free_irq() + * and by that time mhi_ctxt() would've freed. So check for the existence of mhi_ctxt + * before handling the IRQs. + */ + if (!mhi_cntrl->mhi_ctxt) { + dev_dbg(&mhi_cntrl->mhi_dev->dev, + "mhi_ctxt has been freed\n"); + return IRQ_HANDLED; + } + + er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + ptr = le64_to_cpu(er_ctxt->rp); + if (!is_valid_ring_ptr(ev_ring, ptr)) { dev_err(&mhi_cntrl->mhi_dev->dev, "Event ring rp points outside of the event ring\n"); diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index 9e545f2a5a26d29e9a9dbd649490697083ac73bb..caa4ce28cf9e553af7c8404f16d2d9013eaf2fc8 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -507,6 +507,8 @@ static const struct mhi_channel_config mhi_telit_fn990_channels[] = { MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0), MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0), MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0), + MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1), + MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1), MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2), MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3), }; @@ -841,7 +843,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct mhi_controller *mhi_cntrl; int err; - dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name); + dev_info(&pdev->dev, "MHI PCI device found: %s\n", info->name); /* mhi_pdev.mhi_cntrl must be zero-initialized */ mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL); diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c index 5dc2669432ba488b7576e5de9aa6002c8cdb5728..d51573ac525ece334737149f396f4c521500019b 100644 --- a/drivers/bus/mvebu-mbus.c +++ b/drivers/bus/mvebu-mbus.c @@ -466,18 +466,7 @@ static int mvebu_sdram_debug_show(struct seq_file *seq, void *v) struct mvebu_mbus_state *mbus = &mbus_state; return mbus->soc->show_cpu_target(mbus, seq, v); } - -static int mvebu_sdram_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, mvebu_sdram_debug_show, inode->i_private); -} - -static const struct file_operations mvebu_sdram_debug_fops = { - .open = mvebu_sdram_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(mvebu_sdram_debug); static int mvebu_devs_debug_show(struct seq_file *seq, void *v) { @@ -516,18 +505,7 @@ static int mvebu_devs_debug_show(struct seq_file *seq, void *v) return 0; } - -static int mvebu_devs_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, mvebu_devs_debug_show, inode->i_private); -} - -static const struct file_operations mvebu_devs_debug_fops = { - .open = mvebu_devs_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(mvebu_devs_debug); /* * SoC-specific functions and definitions diff --git a/drivers/char/hw_random/arm_smccc_trng.c b/drivers/char/hw_random/arm_smccc_trng.c index b24ac39a903b31a31248bbf6c6606537e102dfcf..e34c3ea692b6cc820bcba255f0a28f4103dc799e 100644 --- a/drivers/char/hw_random/arm_smccc_trng.c +++ b/drivers/char/hw_random/arm_smccc_trng.c @@ -71,8 +71,6 @@ static int smccc_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) MAX_BITS_PER_CALL); arm_smccc_1_1_invoke(ARM_SMCCC_TRNG_RND, bits, &res); - if ((int)res.a0 < 0) - return (int)res.a0; switch ((int)res.a0) { case SMCCC_RET_SUCCESS: @@ -88,6 +86,8 @@ static int smccc_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) return copied; cond_resched(); break; + default: + return -EIO; } } diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 16f227b995e8a536897f24c44ac841a03fb91a8e..cc002b0c2f0c3817e71e3a9e04c89015ff447994 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -52,7 +52,7 @@ MODULE_PARM_DESC(default_quality, static void drop_current_rng(void); static int hwrng_init(struct hwrng *rng); -static void hwrng_manage_rngd(struct hwrng *rng); +static int hwrng_fillfn(void *unused); static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, int wait); @@ -96,6 +96,15 @@ static int set_current_rng(struct hwrng *rng) drop_current_rng(); current_rng = rng; + /* if necessary, start hwrng thread */ + if (!hwrng_fill) { + hwrng_fill = kthread_run(hwrng_fillfn, NULL, "hwrng"); + if (IS_ERR(hwrng_fill)) { + pr_err("hwrng_fill thread creation failed\n"); + hwrng_fill = NULL; + } + } + return 0; } @@ -167,8 +176,6 @@ skip_init: rng->quality = 1024; current_quality = rng->quality; /* obsolete */ - hwrng_manage_rngd(rng); - return 0; } @@ -454,10 +461,6 @@ static ssize_t rng_quality_store(struct device *dev, /* the best available RNG may have changed */ ret = enable_best_rng(); - /* start/stop rngd if necessary */ - if (current_rng) - hwrng_manage_rngd(current_rng); - out: mutex_unlock(&rng_mutex); return ret ? ret : len; @@ -507,16 +510,14 @@ static int hwrng_fillfn(void *unused) rng->quality = current_quality; /* obsolete */ quality = rng->quality; mutex_unlock(&reading_mutex); - put_rng(rng); - if (!quality) - break; + if (rc <= 0) + hwrng_msleep(rng, 10000); - if (rc <= 0) { - pr_warn("hwrng: no data available\n"); - msleep_interruptible(10000); + put_rng(rng); + + if (rc <= 0) continue; - } /* If we cannot credit at least one bit of entropy, * keep track of the remainder for the next iteration @@ -533,22 +534,6 @@ static int hwrng_fillfn(void *unused) return 0; } -static void hwrng_manage_rngd(struct hwrng *rng) -{ - if (WARN_ON(!mutex_is_locked(&rng_mutex))) - return; - - if (rng->quality == 0 && hwrng_fill) - kthread_stop(hwrng_fill); - if (rng->quality > 0 && !hwrng_fill) { - hwrng_fill = kthread_run(hwrng_fillfn, NULL, "hwrng"); - if (IS_ERR(hwrng_fill)) { - pr_err("hwrng_fill thread creation failed\n"); - hwrng_fill = NULL; - } - } -} - int hwrng_register(struct hwrng *rng) { int err = -EINVAL; @@ -570,6 +555,7 @@ int hwrng_register(struct hwrng *rng) init_completion(&rng->cleanup_done); complete(&rng->cleanup_done); + init_completion(&rng->dying); if (!current_rng || (!cur_rng_set_by_user && rng->quality > current_rng->quality)) { @@ -617,6 +603,7 @@ void hwrng_unregister(struct hwrng *rng) old_rng = current_rng; list_del(&rng->list); + complete_all(&rng->dying); if (current_rng == rng) { err = enable_best_rng(); if (err) { @@ -685,6 +672,14 @@ void devm_hwrng_unregister(struct device *dev, struct hwrng *rng) } EXPORT_SYMBOL_GPL(devm_hwrng_unregister); +long hwrng_msleep(struct hwrng *rng, unsigned int msecs) +{ + unsigned long timeout = msecs_to_jiffies(msecs) + 1; + + return wait_for_completion_interruptible_timeout(&rng->dying, timeout); +} +EXPORT_SYMBOL_GPL(hwrng_msleep); + static int __init hwrng_modinit(void) { int ret; diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index b05d676ca814c9300bccc67c27d4d84990ab5e6f..a1c24148ed31461bd256b5d7def3a57027e8a01e 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -245,7 +245,7 @@ static int imx_rngc_probe(struct platform_device *pdev) if (IS_ERR(rngc->base)) return PTR_ERR(rngc->base); - rngc->clk = devm_clk_get(&pdev->dev, NULL); + rngc->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(rngc->clk)) { dev_err(&pdev->dev, "Can not get rng_clk\n"); return PTR_ERR(rngc->clk); @@ -255,27 +255,14 @@ static int imx_rngc_probe(struct platform_device *pdev) if (irq < 0) return irq; - ret = clk_prepare_enable(rngc->clk); - if (ret) - return ret; - ver_id = readl(rngc->base + RNGC_VER_ID); rng_type = ver_id >> RNGC_TYPE_SHIFT; /* * This driver supports only RNGC and RNGB. (There's a different * driver for RNGA.) */ - if (rng_type != RNGC_TYPE_RNGC && rng_type != RNGC_TYPE_RNGB) { - ret = -ENODEV; - goto err; - } - - ret = devm_request_irq(&pdev->dev, - irq, imx_rngc_irq, 0, pdev->name, (void *)rngc); - if (ret) { - dev_err(rngc->dev, "Can't get interrupt working.\n"); - goto err; - } + if (rng_type != RNGC_TYPE_RNGC && rng_type != RNGC_TYPE_RNGB) + return -ENODEV; init_completion(&rngc->rng_op_done); @@ -290,18 +277,25 @@ static int imx_rngc_probe(struct platform_device *pdev) imx_rngc_irq_mask_clear(rngc); + ret = devm_request_irq(&pdev->dev, + irq, imx_rngc_irq, 0, pdev->name, (void *)rngc); + if (ret) { + dev_err(rngc->dev, "Can't get interrupt working.\n"); + return ret; + } + if (self_test) { ret = imx_rngc_self_test(rngc); if (ret) { dev_err(rngc->dev, "self test failed\n"); - goto err; + return ret; } } - ret = hwrng_register(&rngc->rng); + ret = devm_hwrng_register(&pdev->dev, &rngc->rng); if (ret) { dev_err(&pdev->dev, "hwrng registration failed\n"); - goto err; + return ret; } dev_info(&pdev->dev, @@ -309,22 +303,6 @@ static int imx_rngc_probe(struct platform_device *pdev) rng_type == RNGC_TYPE_RNGB ? 'B' : 'C', (ver_id >> RNGC_VER_MAJ_SHIFT) & 0xff, ver_id & 0xff); return 0; - -err: - clk_disable_unprepare(rngc->clk); - - return ret; -} - -static int __exit imx_rngc_remove(struct platform_device *pdev) -{ - struct imx_rngc *rngc = platform_get_drvdata(pdev); - - hwrng_unregister(&rngc->rng); - - clk_disable_unprepare(rngc->clk); - - return 0; } static int __maybe_unused imx_rngc_suspend(struct device *dev) @@ -355,11 +333,10 @@ MODULE_DEVICE_TABLE(of, imx_rngc_dt_ids); static struct platform_driver imx_rngc_driver = { .driver = { - .name = "imx_rngc", + .name = KBUILD_MODNAME, .pm = &imx_rngc_pm_ops, .of_match_table = imx_rngc_dt_ids, }, - .remove = __exit_p(imx_rngc_remove), }; module_platform_driver_probe(imx_rngc_driver, imx_rngc_probe); diff --git a/drivers/char/ipmi/Kconfig b/drivers/char/ipmi/Kconfig index b061e6b513ed5a1902daca0ae10a087ad20a5352..39565cf74b2c9b2d2992c5cb4ee78125b4864255 100644 --- a/drivers/char/ipmi/Kconfig +++ b/drivers/char/ipmi/Kconfig @@ -119,13 +119,13 @@ config ASPEED_KCS_IPMI_BMC provides the access of KCS IO space for BMC side. config NPCM7XX_KCS_IPMI_BMC - depends on ARCH_NPCM7XX || COMPILE_TEST + depends on ARCH_NPCM || COMPILE_TEST select IPMI_KCS_BMC select REGMAP_MMIO - tristate "NPCM7xx KCS IPMI BMC driver" + tristate "NPCM KCS IPMI BMC driver" help Provides a driver for the KCS (Keyboard Controller Style) IPMI - interface found on Nuvoton NPCM7xx SOCs. + interface found on Nuvoton NPCM SOCs. The driver implements the BMC side of the KCS contorller, it provides the access of KCS IO space for BMC side. diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index db40037eb3470f1fe88f846454a50133183c597f..a0e9e80d92eeb0919a4e41289579e2764d7fa8bb 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -341,14 +341,12 @@ static int ipmb_probe(struct i2c_client *client) return 0; } -static int ipmb_remove(struct i2c_client *client) +static void ipmb_remove(struct i2c_client *client) { struct ipmb_dev *ipmb_dev = i2c_get_clientdata(client); i2c_slave_unregister(client); misc_deregister(&ipmb_dev->miscdev); - - return 0; } static const struct i2c_device_id ipmb_id[] = { diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c index ab19b4b3317ecdf082a2f36cfd835368e2353936..7c1aee5e11b7739f110f40056b39ae42d23df339 100644 --- a/drivers/char/ipmi/ipmi_ipmb.c +++ b/drivers/char/ipmi/ipmi_ipmb.c @@ -218,8 +218,8 @@ static void ipmi_ipmb_send_response(struct ipmi_ipmb_dev *iidev, { if ((msg->data[0] >> 2) & 1) { /* - * It's a response being sent, we needto return a - * response response. Fake a send msg command + * It's a response being sent, we need to return a + * response to the response. Fake a send msg command * response with channel 0. This will always be ipmb * direct. */ @@ -424,10 +424,8 @@ static void ipmi_ipmb_request_events(void *send_info) /* We don't fetch events here. */ } -static int ipmi_ipmb_remove(struct i2c_client *client) +static void ipmi_ipmb_cleanup(struct ipmi_ipmb_dev *iidev) { - struct ipmi_ipmb_dev *iidev = i2c_get_clientdata(client); - if (iidev->slave) { i2c_slave_unregister(iidev->slave); if (iidev->slave != iidev->client) @@ -436,10 +434,14 @@ static int ipmi_ipmb_remove(struct i2c_client *client) iidev->slave = NULL; iidev->client = NULL; ipmi_ipmb_stop_thread(iidev); +} - ipmi_unregister_smi(iidev->intf); +static void ipmi_ipmb_remove(struct i2c_client *client) +{ + struct ipmi_ipmb_dev *iidev = i2c_get_clientdata(client); - return 0; + ipmi_ipmb_cleanup(iidev); + ipmi_unregister_smi(iidev->intf); } static int ipmi_ipmb_probe(struct i2c_client *client) @@ -544,7 +546,7 @@ static int ipmi_ipmb_probe(struct i2c_client *client) out_err: if (slave && slave != client) i2c_unregister_device(slave); - ipmi_ipmb_remove(client); + ipmi_ipmb_cleanup(iidev); return rv; } diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 703433493c8520fc4eb031225eba37cb87e1fb9c..49a1707693c9ff6395cd8e880b4c2571418c1cd4 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -736,12 +736,6 @@ static void intf_free(struct kref *ref) kfree(intf); } -struct watcher_entry { - int intf_num; - struct ipmi_smi *intf; - struct list_head link; -}; - int ipmi_smi_watcher_register(struct ipmi_smi_watcher *watcher) { struct ipmi_smi *intf; @@ -4357,7 +4351,7 @@ static int handle_oem_get_msg_cmd(struct ipmi_smi *intf, /* * The message starts at byte 4 which follows the - * the Channel Byte in the "GET MESSAGE" command + * Channel Byte in the "GET MESSAGE" command */ recv_msg->msg.data_len = msg->rsp_size - 4; memcpy(recv_msg->msg_data, &msg->rsp[4], diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index fc742ee9c0468000bcda6d566ba323bc2f7d861b..e1072809fe318b19d9530309d12098ca11332348 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -1281,13 +1281,13 @@ static void shutdown_ssif(void *send_info) } } -static int ssif_remove(struct i2c_client *client) +static void ssif_remove(struct i2c_client *client) { struct ssif_info *ssif_info = i2c_get_clientdata(client); struct ssif_addr_info *addr_info; if (!ssif_info) - return 0; + return; /* * After this point, we won't deliver anything asychronously @@ -1303,8 +1303,6 @@ static int ssif_remove(struct i2c_client *client) } kfree(ssif_info); - - return 0; } static int read_response(struct i2c_client *client, unsigned char *resp) @@ -2100,7 +2098,7 @@ static struct platform_driver ipmi_driver = { .id_table = ssif_plat_ids }; -static int init_ipmi_ssif(void) +static int __init init_ipmi_ssif(void) { int i; int rv; @@ -2142,7 +2140,7 @@ static int init_ipmi_ssif(void) } module_init(init_ipmi_ssif); -static void cleanup_ipmi_ssif(void) +static void __exit cleanup_ipmi_ssif(void) { if (!initialized) return; diff --git a/drivers/char/ipmi/kcs_bmc_aspeed.c b/drivers/char/ipmi/kcs_bmc_aspeed.c index cdc88cde1e9aad92fe8ebae4fe3518bf2fe7de25..19c32bf50e0e9c66c6aae6eceaebccf3beace2b9 100644 --- a/drivers/char/ipmi/kcs_bmc_aspeed.c +++ b/drivers/char/ipmi/kcs_bmc_aspeed.c @@ -207,17 +207,24 @@ static void aspeed_kcs_updateb(struct kcs_bmc_device *kcs_bmc, u32 reg, u8 mask, } /* - * AST_usrGuide_KCS.pdf - * 2. Background: - * we note D for Data, and C for Cmd/Status, default rules are - * A. KCS1 / KCS2 ( D / C:X / X+4 ) - * D / C : CA0h / CA4h - * D / C : CA8h / CACh - * B. KCS3 ( D / C:XX2h / XX3h ) - * D / C : CA2h / CA3h - * D / C : CB2h / CB3h - * C. KCS4 - * D / C : CA4h / CA5h + * We note D for Data, and C for Cmd/Status, default rules are + * + * 1. Only the D address is given: + * A. KCS1/KCS2 (D/C: X/X+4) + * D/C: CA0h/CA4h + * D/C: CA8h/CACh + * B. KCS3 (D/C: XX2/XX3h) + * D/C: CA2h/CA3h + * C. KCS4 (D/C: X/X+1) + * D/C: CA4h/CA5h + * + * 2. Both the D/C addresses are given: + * A. KCS1/KCS2/KCS4 (D/C: X/Y) + * D/C: CA0h/CA1h + * D/C: CA8h/CA9h + * D/C: CA4h/CA5h + * B. KCS3 (D/C: XX2/XX3h) + * D/C: CA2h/CA3h */ static int aspeed_kcs_set_address(struct kcs_bmc_device *kcs_bmc, u32 addrs[2], int nr_addrs) { diff --git a/drivers/char/ipmi/kcs_bmc_cdev_ipmi.c b/drivers/char/ipmi/kcs_bmc_cdev_ipmi.c index 486834a962c3d9426038d2546e2b2d3e52401a9f..cf670e891966d07383e3e61e0b31af44bf380006 100644 --- a/drivers/char/ipmi/kcs_bmc_cdev_ipmi.c +++ b/drivers/char/ipmi/kcs_bmc_cdev_ipmi.c @@ -548,7 +548,7 @@ static struct kcs_bmc_driver kcs_bmc_ipmi_driver = { .ops = &kcs_bmc_ipmi_driver_ops, }; -static int kcs_bmc_ipmi_init(void) +static int __init kcs_bmc_ipmi_init(void) { kcs_bmc_register_driver(&kcs_bmc_ipmi_driver); @@ -556,7 +556,7 @@ static int kcs_bmc_ipmi_init(void) } module_init(kcs_bmc_ipmi_init); -static void kcs_bmc_ipmi_exit(void) +static void __exit kcs_bmc_ipmi_exit(void) { kcs_bmc_unregister_driver(&kcs_bmc_ipmi_driver); } diff --git a/drivers/char/ipmi/kcs_bmc_serio.c b/drivers/char/ipmi/kcs_bmc_serio.c index 7e2067628a6ceea674db045218112dee876bda85..1793358be7822b17573ad46c7288ffab92e8e908 100644 --- a/drivers/char/ipmi/kcs_bmc_serio.c +++ b/drivers/char/ipmi/kcs_bmc_serio.c @@ -140,7 +140,7 @@ static struct kcs_bmc_driver kcs_bmc_serio_driver = { .ops = &kcs_bmc_serio_driver_ops, }; -static int kcs_bmc_serio_init(void) +static int __init kcs_bmc_serio_init(void) { kcs_bmc_register_driver(&kcs_bmc_serio_driver); @@ -148,7 +148,7 @@ static int kcs_bmc_serio_init(void) } module_init(kcs_bmc_serio_init); -static void kcs_bmc_serio_exit(void) +static void __exit kcs_bmc_serio_exit(void) { kcs_bmc_unregister_driver(&kcs_bmc_serio_driver); } diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 84ca98ed1dada445f9fd8a9b4c1ee42a67cdf9c4..5611d127363e470ec8185b42dd025196b4e076b8 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -480,6 +480,11 @@ static ssize_t splice_write_null(struct pipe_inode_info *pipe, struct file *out, return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null); } +static int uring_cmd_null(struct io_uring_cmd *ioucmd, unsigned int issue_flags) +{ + return 0; +} + static ssize_t read_iter_zero(struct kiocb *iocb, struct iov_iter *iter) { size_t written = 0; @@ -663,6 +668,7 @@ static const struct file_operations null_fops = { .read_iter = read_iter_null, .write_iter = write_iter_null, .splice_write = splice_write_null, + .uring_cmd = uring_cmd_null, }; static const struct file_operations __maybe_unused port_fops = { @@ -706,8 +712,8 @@ static const struct memdev { #endif [5] = { "zero", 0666, &zero_fops, FMODE_NOWAIT }, [7] = { "full", 0666, &full_fops, 0 }, - [8] = { "random", 0666, &random_fops, 0 }, - [9] = { "urandom", 0666, &urandom_fops, 0 }, + [8] = { "random", 0666, &random_fops, FMODE_NOWAIT }, + [9] = { "urandom", 0666, &urandom_fops, FMODE_NOWAIT }, #ifdef CONFIG_PRINTK [11] = { "kmsg", 0644, &kmsg_fops, 0 }, #endif diff --git a/drivers/char/pcmcia/synclink_cs.c b/drivers/char/pcmcia/synclink_cs.c index 8fc49b038372cea30f4485cbe4f6c6ed24b3e7b7..b2735be81ab29a001660fe9e530d437b3ac86240 100644 --- a/drivers/char/pcmcia/synclink_cs.c +++ b/drivers/char/pcmcia/synclink_cs.c @@ -2274,7 +2274,8 @@ static int mgslpc_ioctl(struct tty_struct *tty, * tty pointer to tty structure * termios pointer to buffer to hold returned old termios */ -static void mgslpc_set_termios(struct tty_struct *tty, struct ktermios *old_termios) +static void mgslpc_set_termios(struct tty_struct *tty, + const struct ktermios *old_termios) { MGSLPC_INFO *info = (MGSLPC_INFO *)tty->driver_data; unsigned long flags; diff --git a/drivers/char/random.c b/drivers/char/random.c index 79d7d4e4e5828e1654aeb9d9464896c5568d0a3f..2fe28eeb2f387410442d610f6e9f6c2c43503078 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -96,8 +96,8 @@ MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression"); /* * Returns whether or not the input pool has been seeded and thus guaranteed * to supply cryptographically secure random numbers. This applies to: the - * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, - * ,u64,int,long} family of functions. + * /dev/urandom device, the get_random_bytes function, and the get_random_{u8, + * u16,u32,u64,long} family of functions. * * Returns: true if the input pool has been seeded. * false if the input pool has not been seeded. @@ -119,9 +119,9 @@ static void try_to_generate_entropy(void); /* * Wait for the input pool to be seeded and thus guaranteed to supply * cryptographically secure random numbers. This applies to: the /dev/urandom - * device, the get_random_bytes function, and the get_random_{u32,u64,int,long} - * family of functions. Using any of these functions without first calling - * this function forfeits the guarantee of security. + * device, the get_random_bytes function, and the get_random_{u8,u16,u32,u64, + * int,long} family of functions. Using any of these functions without first + * calling this function forfeits the guarantee of security. * * Returns: 0 if the input pool has been seeded. * -ERESTARTSYS if the function was interrupted by a signal. @@ -157,14 +157,15 @@ EXPORT_SYMBOL(wait_for_random_bytes); * There are a few exported interfaces for use by other drivers: * * void get_random_bytes(void *buf, size_t len) + * u8 get_random_u8() + * u16 get_random_u16() * u32 get_random_u32() * u64 get_random_u64() - * unsigned int get_random_int() * unsigned long get_random_long() * * These interfaces will return the requested number of random bytes * into the given buffer or as a return value. This is equivalent to - * a read from /dev/urandom. The u32, u64, int, and long family of + * a read from /dev/urandom. The u8, u16, u32, u64, long family of * functions may be higher performance for one-off random integers, * because they do a bit of buffering and do not invoke reseeding * until the buffer is emptied. @@ -260,25 +261,23 @@ static void crng_fast_key_erasure(u8 key[CHACHA_KEY_SIZE], } /* - * Return whether the crng seed is considered to be sufficiently old - * that a reseeding is needed. This happens if the last reseeding - * was CRNG_RESEED_INTERVAL ago, or during early boot, at an interval + * Return the interval until the next reseeding, which is normally + * CRNG_RESEED_INTERVAL, but during early boot, it is at an interval * proportional to the uptime. */ -static bool crng_has_old_seed(void) +static unsigned int crng_reseed_interval(void) { static bool early_boot = true; - unsigned long interval = CRNG_RESEED_INTERVAL; if (unlikely(READ_ONCE(early_boot))) { time64_t uptime = ktime_get_seconds(); if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2) WRITE_ONCE(early_boot, false); else - interval = max_t(unsigned int, CRNG_RESEED_START_INTERVAL, - (unsigned int)uptime / 2 * HZ); + return max_t(unsigned int, CRNG_RESEED_START_INTERVAL, + (unsigned int)uptime / 2 * HZ); } - return time_is_before_jiffies(READ_ONCE(base_crng.birth) + interval); + return CRNG_RESEED_INTERVAL; } /* @@ -320,7 +319,7 @@ static void crng_make_state(u32 chacha_state[CHACHA_STATE_WORDS], * If the base_crng is old enough, we reseed, which in turn bumps the * generation counter that we check below. */ - if (unlikely(crng_has_old_seed())) + if (unlikely(time_is_before_jiffies(READ_ONCE(base_crng.birth) + crng_reseed_interval()))) crng_reseed(); local_lock_irqsave(&crngs.lock, flags); @@ -384,11 +383,11 @@ static void _get_random_bytes(void *buf, size_t len) } /* - * This function is the exported kernel interface. It returns some - * number of good random numbers, suitable for key generation, seeding - * TCP sequence numbers, etc. In order to ensure that the randomness - * by this function is okay, the function wait_for_random_bytes() - * should be called and return 0 at least once at any point prior. + * This function is the exported kernel interface. It returns some number of + * good random numbers, suitable for key generation, seeding TCP sequence + * numbers, etc. In order to ensure that the randomness returned by this + * function is okay, the function wait_for_random_bytes() should be called and + * return 0 at least once at any point prior. */ void get_random_bytes(void *buf, size_t len) { @@ -506,8 +505,10 @@ type get_random_ ##type(void) \ } \ EXPORT_SYMBOL(get_random_ ##type); -DEFINE_BATCHED_ENTROPY(u64) +DEFINE_BATCHED_ENTROPY(u8) +DEFINE_BATCHED_ENTROPY(u16) DEFINE_BATCHED_ENTROPY(u32) +DEFINE_BATCHED_ENTROPY(u64) #ifdef CONFIG_SMP /* @@ -522,6 +523,8 @@ int __cold random_prepare_cpu(unsigned int cpu) * randomness. */ per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX; + per_cpu_ptr(&batched_entropy_u8, cpu)->position = UINT_MAX; + per_cpu_ptr(&batched_entropy_u16, cpu)->position = UINT_MAX; per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX; per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX; return 0; @@ -774,18 +777,13 @@ static int random_pm_notification(struct notifier_block *nb, unsigned long actio static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification }; /* - * The first collection of entropy occurs at system boot while interrupts - * are still turned off. Here we push in latent entropy, RDSEED, a timestamp, - * utsname(), and the command line. Depending on the above configuration knob, - * RDSEED may be considered sufficient for initialization. Note that much - * earlier setup may already have pushed entropy into the input pool by the - * time we get here. + * This is called extremely early, before time keeping functionality is + * available, but arch randomness is. Interrupts are not yet enabled. */ -int __init random_init(const char *command_line) +void __init random_init_early(const char *command_line) { - ktime_t now = ktime_get_real(); - size_t i, longs, arch_bits; unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)]; + size_t i, longs, arch_bits; #if defined(LATENT_ENTROPY_PLUGIN) static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy; @@ -805,34 +803,49 @@ int __init random_init(const char *command_line) i += longs; continue; } - entropy[0] = random_get_entropy(); - _mix_pool_bytes(entropy, sizeof(*entropy)); arch_bits -= sizeof(*entropy) * 8; ++i; } - _mix_pool_bytes(&now, sizeof(now)); - _mix_pool_bytes(utsname(), sizeof(*(utsname()))); + + _mix_pool_bytes(init_utsname(), sizeof(*(init_utsname()))); _mix_pool_bytes(command_line, strlen(command_line)); + + /* Reseed if already seeded by earlier phases. */ + if (crng_ready()) + crng_reseed(); + else if (trust_cpu) + _credit_init_bits(arch_bits); +} + +/* + * This is called a little bit after the prior function, and now there is + * access to timestamps counters. Interrupts are not yet enabled. + */ +void __init random_init(void) +{ + unsigned long entropy = random_get_entropy(); + ktime_t now = ktime_get_real(); + + _mix_pool_bytes(&now, sizeof(now)); + _mix_pool_bytes(&entropy, sizeof(entropy)); add_latent_entropy(); /* - * If we were initialized by the bootloader before jump labels are - * initialized, then we should enable the static branch here, where + * If we were initialized by the cpu or bootloader before jump labels + * are initialized, then we should enable the static branch here, where * it's guaranteed that jump labels have been initialized. */ if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY) crng_set_ready(NULL); + /* Reseed if already seeded by earlier phases. */ if (crng_ready()) crng_reseed(); - else if (trust_cpu) - _credit_init_bits(arch_bits); WARN_ON(register_pm_notifier(&pm_notifier)); - WARN(!random_get_entropy(), "Missing cycle counter and fallback timer; RNG " - "entropy collection will consequently suffer."); - return 0; + WARN(!entropy, "Missing cycle counter and fallback timer; RNG " + "entropy collection will consequently suffer."); } /* @@ -866,11 +879,11 @@ void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy) credit_init_bits(entropy); /* - * Throttle writing to once every CRNG_RESEED_INTERVAL, unless - * we're not yet initialized. + * Throttle writing to once every reseed interval, unless we're not yet + * initialized or no entropy is credited. */ - if (!kthread_should_stop() && crng_ready()) - schedule_timeout_interruptible(CRNG_RESEED_INTERVAL); + if (!kthread_should_stop() && (crng_ready() || !entropy)) + schedule_timeout_interruptible(crng_reseed_interval()); } EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); @@ -920,20 +933,23 @@ EXPORT_SYMBOL_GPL(unregister_random_vmfork_notifier); #endif struct fast_pool { - struct work_struct mix; unsigned long pool[4]; unsigned long last; unsigned int count; + struct timer_list mix; }; +static void mix_interrupt_randomness(struct timer_list *work); + static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = { #ifdef CONFIG_64BIT #define FASTMIX_PERM SIPHASH_PERMUTATION - .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 } + .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 }, #else #define FASTMIX_PERM HSIPHASH_PERMUTATION - .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 } + .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 }, #endif + .mix = __TIMER_INITIALIZER(mix_interrupt_randomness, 0) }; /* @@ -975,7 +991,7 @@ int __cold random_online_cpu(unsigned int cpu) } #endif -static void mix_interrupt_randomness(struct work_struct *work) +static void mix_interrupt_randomness(struct timer_list *work) { struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix); /* @@ -1006,7 +1022,7 @@ static void mix_interrupt_randomness(struct work_struct *work) local_irq_enable(); mix_pool_bytes(pool, sizeof(pool)); - credit_init_bits(max(1u, (count & U16_MAX) / 64)); + credit_init_bits(clamp_t(unsigned int, (count & U16_MAX) / 64, 1, sizeof(pool) * 8)); memzero_explicit(pool, sizeof(pool)); } @@ -1029,10 +1045,11 @@ void add_interrupt_randomness(int irq) if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ)) return; - if (unlikely(!fast_pool->mix.func)) - INIT_WORK(&fast_pool->mix, mix_interrupt_randomness); fast_pool->count |= MIX_INFLIGHT; - queue_work_on(raw_smp_processor_id(), system_highpri_wq, &fast_pool->mix); + if (!timer_pending(&fast_pool->mix)) { + fast_pool->mix.expires = jiffies; + add_timer_on(&fast_pool->mix, raw_smp_processor_id()); + } } EXPORT_SYMBOL_GPL(add_interrupt_randomness); @@ -1191,7 +1208,7 @@ static void __cold entropy_timer(struct timer_list *timer) */ static void __cold try_to_generate_entropy(void) { - enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 30 }; + enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 15 }; struct entropy_timer_state stack; unsigned int i, num_different = 0; unsigned long last = random_get_entropy(); @@ -1210,7 +1227,7 @@ static void __cold try_to_generate_entropy(void) timer_setup_on_stack(&stack.timer, entropy_timer, 0); while (!crng_ready() && !signal_pending(current)) { if (!timer_pending(&stack.timer)) - mod_timer(&stack.timer, jiffies + 1); + mod_timer(&stack.timer, jiffies); mix_pool_bytes(&stack.entropy, sizeof(stack.entropy)); schedule(); stack.entropy = random_get_entropy(); @@ -1347,6 +1364,11 @@ static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter) { int ret; + if (!crng_ready() && + ((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) || + (kiocb->ki_filp->f_flags & O_NONBLOCK))) + return -EAGAIN; + ret = wait_for_random_bytes(); if (ret != 0) return ret; diff --git a/drivers/char/tpm/st33zp24/i2c.c b/drivers/char/tpm/st33zp24/i2c.c index 3170d59d660c089182874f6e7bf0df8fbaa03260..a3aa411389e77478e836c82ff558951695256e7d 100644 --- a/drivers/char/tpm/st33zp24/i2c.c +++ b/drivers/char/tpm/st33zp24/i2c.c @@ -264,13 +264,11 @@ static int st33zp24_i2c_probe(struct i2c_client *client, * @param: client, the i2c_client description (TPM I2C description). * @return: 0 in case of success. */ -static int st33zp24_i2c_remove(struct i2c_client *client) +static void st33zp24_i2c_remove(struct i2c_client *client) { struct tpm_chip *chip = i2c_get_clientdata(client); st33zp24_remove(chip); - - return 0; } static const struct i2c_device_id st33zp24_i2c_id[] = { diff --git a/drivers/char/tpm/tpm_i2c_atmel.c b/drivers/char/tpm/tpm_i2c_atmel.c index d5ac85558214268df5cded8afe2f420205515d22..4be3677c14633b77e2e1e9aa221ac95a99b626a1 100644 --- a/drivers/char/tpm/tpm_i2c_atmel.c +++ b/drivers/char/tpm/tpm_i2c_atmel.c @@ -179,12 +179,11 @@ static int i2c_atmel_probe(struct i2c_client *client, return tpm_chip_register(chip); } -static int i2c_atmel_remove(struct i2c_client *client) +static void i2c_atmel_remove(struct i2c_client *client) { struct device *dev = &(client->dev); struct tpm_chip *chip = dev_get_drvdata(dev); tpm_chip_unregister(chip); - return 0; } static const struct i2c_device_id i2c_atmel_id[] = { diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c index a19d32cb4e942c0890e7043475e4d9c9d16fd457..fd3c3661e6466338286d1aa12cb39aef6eb1cfa6 100644 --- a/drivers/char/tpm/tpm_i2c_infineon.c +++ b/drivers/char/tpm/tpm_i2c_infineon.c @@ -706,15 +706,13 @@ static int tpm_tis_i2c_probe(struct i2c_client *client, return rc; } -static int tpm_tis_i2c_remove(struct i2c_client *client) +static void tpm_tis_i2c_remove(struct i2c_client *client) { struct tpm_chip *chip = tpm_dev.chip; tpm_chip_unregister(chip); release_locality(chip, tpm_dev.locality, 1); tpm_dev.client = NULL; - - return 0; } static struct i2c_driver tpm_tis_i2c_driver = { diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c index b77c18e386621fd7b2d51583decb7d152d5add50..95c37350cc8e5d1e078aa7a3902ac79b7dfcd9da 100644 --- a/drivers/char/tpm/tpm_i2c_nuvoton.c +++ b/drivers/char/tpm/tpm_i2c_nuvoton.c @@ -622,12 +622,11 @@ static int i2c_nuvoton_probe(struct i2c_client *client, return tpm_chip_register(chip); } -static int i2c_nuvoton_remove(struct i2c_client *client) +static void i2c_nuvoton_remove(struct i2c_client *client) { struct tpm_chip *chip = i2c_get_clientdata(client); tpm_chip_unregister(chip); - return 0; } static const struct i2c_device_id i2c_nuvoton_id[] = { diff --git a/drivers/char/tpm/tpm_ppi.c b/drivers/char/tpm/tpm_ppi.c index 40018a73b3cb7e93aa206b012d0afca8a72ce176..bc7b1b4501b3b9441fc7a71495e4f1e7f0973824 100644 --- a/drivers/char/tpm/tpm_ppi.c +++ b/drivers/char/tpm/tpm_ppi.c @@ -380,7 +380,7 @@ void tpm_add_ppi(struct tpm_chip *chip) TPM_PPI_FN_VERSION, NULL, ACPI_TYPE_STRING); if (obj) { - strlcpy(chip->ppi_version, obj->string.pointer, + strscpy(chip->ppi_version, obj->string.pointer, sizeof(chip->ppi_version)); ACPI_FREE(obj); } diff --git a/drivers/char/tpm/tpm_tis_i2c.c b/drivers/char/tpm/tpm_tis_i2c.c index ba0911b1d1ff31d70813f3bef25ca76080481cc5..0692510dfcab9b3b6bb8b004912364b3ff70f652 100644 --- a/drivers/char/tpm/tpm_tis_i2c.c +++ b/drivers/char/tpm/tpm_tis_i2c.c @@ -351,13 +351,12 @@ static int tpm_tis_i2c_probe(struct i2c_client *dev, NULL); } -static int tpm_tis_i2c_remove(struct i2c_client *client) +static void tpm_tis_i2c_remove(struct i2c_client *client) { struct tpm_chip *chip = i2c_get_clientdata(client); tpm_chip_unregister(chip); tpm_tis_remove(chip); - return 0; } static const struct i2c_device_id tpm_tis_i2c_id[] = { diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c index 974479a1ec5a081f547e37ccf68daf6769afa239..77cea5b31c6e4b67eaabc9ec9839170095018935 100644 --- a/drivers/char/tpm/tpm_tis_i2c_cr50.c +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -763,20 +763,18 @@ static int tpm_cr50_i2c_probe(struct i2c_client *client) * - 0: Success. * - -errno: A POSIX error code. */ -static int tpm_cr50_i2c_remove(struct i2c_client *client) +static void tpm_cr50_i2c_remove(struct i2c_client *client) { struct tpm_chip *chip = i2c_get_clientdata(client); struct device *dev = &client->dev; if (!chip) { dev_crit(dev, "Could not get client data at remove, memory corruption ahead\n"); - return 0; + return; } tpm_chip_unregister(chip); tpm_cr50_release_locality(chip, true); - - return 0; } static SIMPLE_DEV_PM_OPS(cr50_i2c_pm, tpm_pm_suspend, tpm_pm_resume); diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 48f8f4221e21d038f0c2e65f6b505bf49da995e5..d79905f3e1744f71357151d4b901c115bc7f5c55 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -249,7 +249,7 @@ config COMMON_CLK_GEMINI platform, also known as SL3516 or CS3516. config COMMON_CLK_LAN966X - bool "Generic Clock Controller driver for LAN966X SoC" + tristate "Generic Clock Controller driver for LAN966X SoC" depends on HAS_IOMEM depends on OF depends on SOC_LAN966 || COMPILE_TEST @@ -377,6 +377,15 @@ config COMMON_CLK_VC5 This driver supports the IDT VersaClock 5 and VersaClock 6 programmable clock generators. +config COMMON_CLK_VC7 + tristate "Clock driver for Renesas Versaclock 7 devices" + depends on I2C + depends on OF + select REGMAP_I2C + help + Renesas Versaclock7 is a family of configurable clock generator + and jitter attenuator ICs with fractional and integer dividers. + config COMMON_CLK_STM32MP135 def_bool COMMON_CLK && MACH_STM32MP13 help diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index d5db170d38d2d9c96085f45f6f0ca2011d46acd8..e3ca0d058a25671f2c391a1a42748358e136256b 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_RS9_PCIE) += clk-renesas-pcie.o obj-$(CONFIG_COMMON_CLK_VC5) += clk-versaclock5.o +obj-$(CONFIG_COMMON_CLK_VC7) += clk-versaclock7.o obj-$(CONFIG_COMMON_CLK_WM831X) += clk-wm831x.o obj-$(CONFIG_COMMON_CLK_XGENE) += clk-xgene.o diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c index d429ba52a71908b584e9ccca3dd8cf8f3a576798..943ea67bf135fb486d96feae259b32ee45f7743e 100644 --- a/drivers/clk/at91/clk-generated.c +++ b/drivers/clk/at91/clk-generated.c @@ -136,7 +136,6 @@ static int clk_generated_determine_rate(struct clk_hw *hw, { struct clk_generated *gck = to_clk_generated(hw); struct clk_hw *parent = NULL; - struct clk_rate_request req_parent = *req; long best_rate = -EINVAL; unsigned long min_rate, parent_rate; int best_diff = -1; @@ -192,7 +191,9 @@ static int clk_generated_determine_rate(struct clk_hw *hw, goto end; for (div = 1; div < GENERATED_MAX_DIV + 2; div++) { - req_parent.rate = req->rate * div; + struct clk_rate_request req_parent; + + clk_hw_forward_rate_request(hw, req, parent, &req_parent, req->rate * div); if (__clk_determine_rate(parent, &req_parent)) continue; clk_generated_best_diff(req, parent, req_parent.rate, div, diff --git a/drivers/clk/at91/clk-master.c b/drivers/clk/at91/clk-master.c index 164e2959c7cfbceba3f6527e156c97096c1d9001..b7cd1924de52a924e3f58efaba6b0a2ab629374f 100644 --- a/drivers/clk/at91/clk-master.c +++ b/drivers/clk/at91/clk-master.c @@ -581,7 +581,6 @@ static int clk_sama7g5_master_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_master *master = to_clk_master(hw); - struct clk_rate_request req_parent = *req; struct clk_hw *parent; long best_rate = LONG_MIN, best_diff = LONG_MIN; unsigned long parent_rate; @@ -618,11 +617,15 @@ static int clk_sama7g5_master_determine_rate(struct clk_hw *hw, goto end; for (div = 0; div < MASTER_PRES_MAX + 1; div++) { + struct clk_rate_request req_parent; + unsigned long req_rate; + if (div == MASTER_PRES_MAX) - req_parent.rate = req->rate * 3; + req_rate = req->rate * 3; else - req_parent.rate = req->rate << div; + req_rate = req->rate << div; + clk_hw_forward_rate_request(hw, req, parent, &req_parent, req_rate); if (__clk_determine_rate(parent, &req_parent)) continue; diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c index e14fa5ac734cead773b17f2239ad8f7c9522108c..5104d4025484c825e5a58857e78012d704d41638 100644 --- a/drivers/clk/at91/clk-peripheral.c +++ b/drivers/clk/at91/clk-peripheral.c @@ -269,7 +269,6 @@ static int clk_sam9x5_peripheral_determine_rate(struct clk_hw *hw, { struct clk_sam9x5_peripheral *periph = to_clk_sam9x5_peripheral(hw); struct clk_hw *parent = clk_hw_get_parent(hw); - struct clk_rate_request req_parent = *req; unsigned long parent_rate = clk_hw_get_rate(parent); unsigned long tmp_rate; long best_rate = LONG_MIN; @@ -302,8 +301,9 @@ static int clk_sam9x5_peripheral_determine_rate(struct clk_hw *hw, goto end; for (shift = 0; shift <= PERIPHERAL_MAX_SHIFT; shift++) { - req_parent.rate = req->rate << shift; + struct clk_rate_request req_parent; + clk_hw_forward_rate_request(hw, req, parent, &req_parent, req->rate << shift); if (__clk_determine_rate(parent, &req_parent)) continue; diff --git a/drivers/clk/at91/dt-compat.c b/drivers/clk/at91/dt-compat.c index 8ca8bcacf66de1f06bac0b7d3d13f28f9618af95..85a964cb2d89ee7a19a7318b3d9435cb323c00cb 100644 --- a/drivers/clk/at91/dt-compat.c +++ b/drivers/clk/at91/dt-compat.c @@ -33,8 +33,11 @@ static void __init of_sama5d2_clk_audio_pll_frac_setup(struct device_node *np) const char *name = np->name; const char *parent_name; struct regmap *regmap; + struct device_node *parent_np; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -56,8 +59,11 @@ static void __init of_sama5d2_clk_audio_pll_pad_setup(struct device_node *np) const char *name = np->name; const char *parent_name; struct regmap *regmap; + struct device_node *parent_np; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -79,8 +85,11 @@ static void __init of_sama5d2_clk_audio_pll_pmc_setup(struct device_node *np) const char *name = np->name; const char *parent_name; struct regmap *regmap; + struct device_node *parent_np; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -120,7 +129,7 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np) struct clk_hw *hw; unsigned int num_parents; const char *parent_names[GENERATED_SOURCE_MAX]; - struct device_node *gcknp; + struct device_node *gcknp, *parent_np; struct clk_range range = CLK_RANGE(0, 0); struct regmap *regmap; @@ -134,7 +143,9 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np) if (!num || num > PERIPHERAL_MAX) return; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -180,8 +191,11 @@ static void __init of_sama5d4_clk_h32mx_setup(struct device_node *np) const char *name = np->name; const char *parent_name; struct regmap *regmap; + struct device_node *parent_np; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -243,12 +257,15 @@ static void __init of_at91rm9200_clk_main_osc_setup(struct device_node *np) const char *parent_name; struct regmap *regmap; bool bypass; + struct device_node *parent_np; of_property_read_string(np, "clock-output-names", &name); bypass = of_property_read_bool(np, "atmel,osc-bypass"); parent_name = of_clk_get_parent_name(np, 0); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -268,12 +285,15 @@ static void __init of_at91sam9x5_clk_main_rc_osc_setup(struct device_node *np) u32 accuracy = 0; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; of_property_read_string(np, "clock-output-names", &name); of_property_read_u32(np, "clock-frequency", &frequency); of_property_read_u32(np, "clock-accuracy", &accuracy); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -292,11 +312,14 @@ static void __init of_at91rm9200_clk_main_setup(struct device_node *np) const char *parent_name; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -316,13 +339,16 @@ static void __init of_at91sam9x5_clk_main_setup(struct device_node *np) unsigned int num_parents; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; num_parents = of_clk_get_parent_count(np); if (num_parents == 0 || num_parents > 2) return; of_clk_parent_fill(np, parent_names, num_parents); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -373,6 +399,7 @@ of_at91_clk_master_setup(struct device_node *np, const char *name = np->name; struct clk_master_characteristics *characteristics; struct regmap *regmap; + struct device_node *parent_np; num_parents = of_clk_get_parent_count(np); if (num_parents == 0 || num_parents > MASTER_SOURCE_MAX) @@ -386,7 +413,9 @@ of_at91_clk_master_setup(struct device_node *np, if (!characteristics) return; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -433,6 +462,7 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type) const char *name; struct device_node *periphclknp; struct regmap *regmap; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); if (!parent_name) @@ -442,7 +472,9 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type) if (!num || num > PERIPHERAL_MAX) return; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -601,6 +633,7 @@ of_at91_clk_pll_setup(struct device_node *np, struct regmap *regmap; const char *parent_name; const char *name = np->name; + struct device_node *parent_np; struct clk_pll_characteristics *characteristics; if (of_property_read_u32(np, "reg", &id)) @@ -610,7 +643,9 @@ of_at91_clk_pll_setup(struct device_node *np, of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -665,12 +700,15 @@ of_at91sam9x5_clk_plldiv_setup(struct device_node *np) const char *parent_name; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -694,7 +732,7 @@ of_at91_clk_prog_setup(struct device_node *np, unsigned int num_parents; const char *parent_names[PROG_SOURCE_MAX]; const char *name; - struct device_node *progclknp; + struct device_node *progclknp, *parent_np; struct regmap *regmap; num_parents = of_clk_get_parent_count(np); @@ -707,7 +745,9 @@ of_at91_clk_prog_setup(struct device_node *np, if (!num || num > (PROG_ID_MAX + 1)) return; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -756,13 +796,16 @@ static void __init of_at91sam9260_clk_slow_setup(struct device_node *np) unsigned int num_parents; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; num_parents = of_clk_get_parent_count(np); if (num_parents != 2) return; of_clk_parent_fill(np, parent_names, num_parents); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -788,6 +831,7 @@ static void __init of_at91sam9x5_clk_smd_setup(struct device_node *np) const char *parent_names[SMD_SOURCE_MAX]; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; num_parents = of_clk_get_parent_count(np); if (num_parents == 0 || num_parents > SMD_SOURCE_MAX) @@ -797,7 +841,9 @@ static void __init of_at91sam9x5_clk_smd_setup(struct device_node *np) of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -818,7 +864,7 @@ static void __init of_at91rm9200_clk_sys_setup(struct device_node *np) u32 id; struct clk_hw *hw; const char *name; - struct device_node *sysclknp; + struct device_node *sysclknp, *parent_np; const char *parent_name; struct regmap *regmap; @@ -826,7 +872,9 @@ static void __init of_at91rm9200_clk_sys_setup(struct device_node *np) if (num > (SYSTEM_MAX_ID + 1)) return; - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -859,6 +907,7 @@ static void __init of_at91sam9x5_clk_usb_setup(struct device_node *np) const char *parent_names[USB_SOURCE_MAX]; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; num_parents = of_clk_get_parent_count(np); if (num_parents == 0 || num_parents > USB_SOURCE_MAX) @@ -868,7 +917,9 @@ static void __init of_at91sam9x5_clk_usb_setup(struct device_node *np) of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -888,6 +939,7 @@ static void __init of_at91sam9n12_clk_usb_setup(struct device_node *np) const char *parent_name; const char *name = np->name; struct regmap *regmap; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); if (!parent_name) @@ -895,7 +947,9 @@ static void __init of_at91sam9n12_clk_usb_setup(struct device_node *np) of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; @@ -915,6 +969,7 @@ static void __init of_at91rm9200_clk_usb_setup(struct device_node *np) const char *name = np->name; u32 divisors[4] = {0, 0, 0, 0}; struct regmap *regmap; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); if (!parent_name) @@ -926,7 +981,9 @@ static void __init of_at91rm9200_clk_usb_setup(struct device_node *np) of_property_read_string(np, "clock-output-names", &name); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) return; hw = at91rm9200_clk_register_usb(regmap, name, parent_name, divisors); @@ -946,12 +1003,15 @@ static void __init of_at91sam9x5_clk_utmi_setup(struct device_node *np) const char *parent_name; const char *name = np->name; struct regmap *regmap_pmc, *regmap_sfr; + struct device_node *parent_np; parent_name = of_clk_get_parent_name(np, 0); of_property_read_string(np, "clock-output-names", &name); - regmap_pmc = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap_pmc = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap_pmc)) return; diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c index cfd0f5e23b9963d23b47f54c1be38b486713fde1..84156dc52bff53f1f3a0a41e4b6d51ac5cb342ec 100644 --- a/drivers/clk/at91/sama5d2.c +++ b/drivers/clk/at91/sama5d2.c @@ -120,6 +120,16 @@ static const struct { struct clk_range r; int chg_pid; } sama5d2_gck[] = { + { .n = "flx0_gclk", .id = 19, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "flx1_gclk", .id = 20, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "flx2_gclk", .id = 21, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "flx3_gclk", .id = 22, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "flx4_gclk", .id = 23, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "uart0_gclk", .id = 24, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "uart1_gclk", .id = 25, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "uart2_gclk", .id = 26, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "uart3_gclk", .id = 27, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, + { .n = "uart4_gclk", .id = 28, .chg_pid = INT_MIN, .r = { .min = 0, .max = 27666666 }, }, { .n = "sdmmc0_gclk", .id = 31, .chg_pid = INT_MIN, }, { .n = "sdmmc1_gclk", .id = 32, .chg_pid = INT_MIN, }, { .n = "tcb0_gclk", .id = 35, .chg_pid = INT_MIN, .r = { .min = 0, .max = 83000000 }, }, diff --git a/drivers/clk/baikal-t1/Kconfig b/drivers/clk/baikal-t1/Kconfig index 03102f1094bcafc87e6e49ba87fddabda7cbe3ad..f0b186830324392845713ccc9631f9857c4006a0 100644 --- a/drivers/clk/baikal-t1/Kconfig +++ b/drivers/clk/baikal-t1/Kconfig @@ -29,7 +29,6 @@ config CLK_BT1_CCU_PLL config CLK_BT1_CCU_DIV bool "Baikal-T1 CCU Dividers support" - select RESET_CONTROLLER select MFD_SYSCON default MIPS_BAIKAL_T1 help @@ -39,4 +38,15 @@ config CLK_BT1_CCU_DIV either gateable or ungateable. Some of the CCU dividers can be as well used to reset the domains they're supplying clock to. +config CLK_BT1_CCU_RST + bool "Baikal-T1 CCU Resets support" + select RESET_CONTROLLER + select MFD_SYSCON + default MIPS_BAIKAL_T1 + help + Enable this to support the CCU reset blocks responsible for the + AXI-bus and some subsystems reset. These are mainly the + self-deasserted reset controls but there are several lines which + can be directly asserted/de-asserted (PCIe and DDR sub-domains). + endif diff --git a/drivers/clk/baikal-t1/Makefile b/drivers/clk/baikal-t1/Makefile index b3b9590b95ed95d4d73ab11425cb111f74d90961..9c3637de940705c1f65a772e10aeff09707d92ee 100644 --- a/drivers/clk/baikal-t1/Makefile +++ b/drivers/clk/baikal-t1/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CLK_BT1_CCU_PLL) += ccu-pll.o clk-ccu-pll.o obj-$(CONFIG_CLK_BT1_CCU_DIV) += ccu-div.o clk-ccu-div.o +obj-$(CONFIG_CLK_BT1_CCU_RST) += ccu-rst.o diff --git a/drivers/clk/baikal-t1/ccu-div.c b/drivers/clk/baikal-t1/ccu-div.c index 4062092d67f905a601e1d8d0856175eaeee0240c..8d5fc7158f33f9c5f8a04241cb62f1ec6ced6e5d 100644 --- a/drivers/clk/baikal-t1/ccu-div.c +++ b/drivers/clk/baikal-t1/ccu-div.c @@ -34,9 +34,9 @@ #define CCU_DIV_CTL_CLKDIV_MASK(_width) \ GENMASK((_width) + CCU_DIV_CTL_CLKDIV_FLD - 1, CCU_DIV_CTL_CLKDIV_FLD) #define CCU_DIV_CTL_LOCK_SHIFTED BIT(27) +#define CCU_DIV_CTL_GATE_REF_BUF BIT(28) #define CCU_DIV_CTL_LOCK_NORMAL BIT(31) -#define CCU_DIV_RST_DELAY_US 1 #define CCU_DIV_LOCK_CHECK_RETRIES 50 #define CCU_DIV_CLKDIV_MIN 0 @@ -170,6 +170,40 @@ static int ccu_div_gate_is_enabled(struct clk_hw *hw) return !!(val & CCU_DIV_CTL_EN); } +static int ccu_div_buf_enable(struct clk_hw *hw) +{ + struct ccu_div *div = to_ccu_div(hw); + unsigned long flags; + + spin_lock_irqsave(&div->lock, flags); + regmap_update_bits(div->sys_regs, div->reg_ctl, + CCU_DIV_CTL_GATE_REF_BUF, 0); + spin_unlock_irqrestore(&div->lock, flags); + + return 0; +} + +static void ccu_div_buf_disable(struct clk_hw *hw) +{ + struct ccu_div *div = to_ccu_div(hw); + unsigned long flags; + + spin_lock_irqsave(&div->lock, flags); + regmap_update_bits(div->sys_regs, div->reg_ctl, + CCU_DIV_CTL_GATE_REF_BUF, CCU_DIV_CTL_GATE_REF_BUF); + spin_unlock_irqrestore(&div->lock, flags); +} + +static int ccu_div_buf_is_enabled(struct clk_hw *hw) +{ + struct ccu_div *div = to_ccu_div(hw); + u32 val = 0; + + regmap_read(div->sys_regs, div->reg_ctl, &val); + + return !(val & CCU_DIV_CTL_GATE_REF_BUF); +} + static unsigned long ccu_div_var_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -288,24 +322,6 @@ static int ccu_div_fixed_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } -int ccu_div_reset_domain(struct ccu_div *div) -{ - unsigned long flags; - - if (!div || !(div->features & CCU_DIV_RESET_DOMAIN)) - return -EINVAL; - - spin_lock_irqsave(&div->lock, flags); - regmap_update_bits(div->sys_regs, div->reg_ctl, - CCU_DIV_CTL_RST, CCU_DIV_CTL_RST); - spin_unlock_irqrestore(&div->lock, flags); - - /* The next delay must be enough to cover all the resets. */ - udelay(CCU_DIV_RST_DELAY_US); - - return 0; -} - #ifdef CONFIG_DEBUG_FS struct ccu_div_dbgfs_bit { @@ -323,6 +339,7 @@ static const struct ccu_div_dbgfs_bit ccu_div_bits[] = { CCU_DIV_DBGFS_BIT_ATTR("div_en", CCU_DIV_CTL_EN), CCU_DIV_DBGFS_BIT_ATTR("div_rst", CCU_DIV_CTL_RST), CCU_DIV_DBGFS_BIT_ATTR("div_bypass", CCU_DIV_CTL_SET_CLKDIV), + CCU_DIV_DBGFS_BIT_ATTR("div_buf", CCU_DIV_CTL_GATE_REF_BUF), CCU_DIV_DBGFS_BIT_ATTR("div_lock", CCU_DIV_CTL_LOCK_NORMAL) }; @@ -441,6 +458,9 @@ static void ccu_div_var_debug_init(struct clk_hw *hw, struct dentry *dentry) continue; } + if (!strcmp("div_buf", name)) + continue; + bits[didx] = ccu_div_bits[bidx]; bits[didx].div = div; @@ -477,6 +497,21 @@ static void ccu_div_gate_debug_init(struct clk_hw *hw, struct dentry *dentry) &ccu_div_dbgfs_fixed_clkdiv_fops); } +static void ccu_div_buf_debug_init(struct clk_hw *hw, struct dentry *dentry) +{ + struct ccu_div *div = to_ccu_div(hw); + struct ccu_div_dbgfs_bit *bit; + + bit = kmalloc(sizeof(*bit), GFP_KERNEL); + if (!bit) + return; + + *bit = ccu_div_bits[3]; + bit->div = div; + debugfs_create_file_unsafe(bit->name, ccu_div_dbgfs_mode, dentry, bit, + &ccu_div_dbgfs_bit_fops); +} + static void ccu_div_fixed_debug_init(struct clk_hw *hw, struct dentry *dentry) { struct ccu_div *div = to_ccu_div(hw); @@ -489,6 +524,7 @@ static void ccu_div_fixed_debug_init(struct clk_hw *hw, struct dentry *dentry) #define ccu_div_var_debug_init NULL #define ccu_div_gate_debug_init NULL +#define ccu_div_buf_debug_init NULL #define ccu_div_fixed_debug_init NULL #endif /* !CONFIG_DEBUG_FS */ @@ -520,6 +556,13 @@ static const struct clk_ops ccu_div_gate_ops = { .debug_init = ccu_div_gate_debug_init }; +static const struct clk_ops ccu_div_buf_ops = { + .enable = ccu_div_buf_enable, + .disable = ccu_div_buf_disable, + .is_enabled = ccu_div_buf_is_enabled, + .debug_init = ccu_div_buf_debug_init +}; + static const struct clk_ops ccu_div_fixed_ops = { .recalc_rate = ccu_div_fixed_recalc_rate, .round_rate = ccu_div_fixed_round_rate, @@ -566,6 +609,8 @@ struct ccu_div *ccu_div_hw_register(const struct ccu_div_init_data *div_init) } else if (div_init->type == CCU_DIV_GATE) { hw_init.ops = &ccu_div_gate_ops; div->divider = div_init->divider; + } else if (div_init->type == CCU_DIV_BUF) { + hw_init.ops = &ccu_div_buf_ops; } else if (div_init->type == CCU_DIV_FIXED) { hw_init.ops = &ccu_div_fixed_ops; div->divider = div_init->divider; @@ -579,6 +624,7 @@ struct ccu_div *ccu_div_hw_register(const struct ccu_div_init_data *div_init) goto err_free_div; } parent_data.fw_name = div_init->parent_name; + parent_data.name = div_init->parent_name; hw_init.parent_data = &parent_data; hw_init.num_parents = 1; diff --git a/drivers/clk/baikal-t1/ccu-div.h b/drivers/clk/baikal-t1/ccu-div.h index 795665caefbdc9f3320de927f9340695f2be979f..76d8ee44d4157736e4a43cecd11bedc2c845087c 100644 --- a/drivers/clk/baikal-t1/ccu-div.h +++ b/drivers/clk/baikal-t1/ccu-div.h @@ -13,15 +13,26 @@ #include #include +/* + * CCU Divider private clock IDs + * @CCU_SYS_SATA_CLK: CCU SATA internal clock + * @CCU_SYS_XGMAC_CLK: CCU XGMAC internal clock + */ +#define CCU_SYS_SATA_CLK -1 +#define CCU_SYS_XGMAC_CLK -2 + /* * CCU Divider private flags + * @CCU_DIV_BASIC: Basic divider clock required by the kernel as early as + * possible. * @CCU_DIV_SKIP_ONE: Due to some reason divider can't be set to 1. * It can be 0 though, which is functionally the same. * @CCU_DIV_SKIP_ONE_TO_THREE: For some reason divider can't be within [1,3]. * It can be either 0 or greater than 3. * @CCU_DIV_LOCK_SHIFTED: Find lock-bit at non-standard position. - * @CCU_DIV_RESET_DOMAIN: Provide reset clock domain method. + * @CCU_DIV_RESET_DOMAIN: There is a clock domain reset handle. */ +#define CCU_DIV_BASIC BIT(0) #define CCU_DIV_SKIP_ONE BIT(1) #define CCU_DIV_SKIP_ONE_TO_THREE BIT(2) #define CCU_DIV_LOCK_SHIFTED BIT(3) @@ -31,11 +42,13 @@ * enum ccu_div_type - CCU Divider types * @CCU_DIV_VAR: Clocks gate with variable divider. * @CCU_DIV_GATE: Clocks gate with fixed divider. + * @CCU_DIV_BUF: Clock gate with no divider. * @CCU_DIV_FIXED: Ungateable clock with fixed divider. */ enum ccu_div_type { CCU_DIV_VAR, CCU_DIV_GATE, + CCU_DIV_BUF, CCU_DIV_FIXED }; @@ -105,6 +118,4 @@ struct ccu_div *ccu_div_hw_register(const struct ccu_div_init_data *init); void ccu_div_hw_unregister(struct ccu_div *div); -int ccu_div_reset_domain(struct ccu_div *div); - #endif /* __CLK_BT1_CCU_DIV_H__ */ diff --git a/drivers/clk/baikal-t1/ccu-pll.h b/drivers/clk/baikal-t1/ccu-pll.h index 76cd9132a2196adb4da163ee70a850e20762a6d7..a71bfd7b90ecc4474ba7f39537e19b94edeca4cb 100644 --- a/drivers/clk/baikal-t1/ccu-pll.h +++ b/drivers/clk/baikal-t1/ccu-pll.h @@ -13,6 +13,12 @@ #include #include +/* + * CCU PLL private flags + * @CCU_PLL_BASIC: Basic PLL required by the kernel as early as possible. + */ +#define CCU_PLL_BASIC BIT(0) + /* * struct ccu_pll_init_data - CCU PLL initialization data * @id: Clock private identifier. @@ -22,6 +28,7 @@ * @sys_regs: Baikal-T1 System Controller registers map. * @np: Pointer to the node describing the CCU PLLs. * @flags: PLL clock flags. + * @features: PLL private features. */ struct ccu_pll_init_data { unsigned int id; @@ -31,6 +38,7 @@ struct ccu_pll_init_data { struct regmap *sys_regs; struct device_node *np; unsigned long flags; + unsigned long features; }; /* diff --git a/drivers/clk/baikal-t1/ccu-rst.c b/drivers/clk/baikal-t1/ccu-rst.c new file mode 100644 index 0000000000000000000000000000000000000000..40023ea674636b0ab68810a08fe4792abad41365 --- /dev/null +++ b/drivers/clk/baikal-t1/ccu-rst.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 BAIKAL ELECTRONICS, JSC + * + * Authors: + * Serge Semin + * + * Baikal-T1 CCU Resets interface driver + */ + +#define pr_fmt(fmt) "bt1-ccu-rst: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ccu-rst.h" + +#define CCU_AXI_MAIN_BASE 0x030 +#define CCU_AXI_DDR_BASE 0x034 +#define CCU_AXI_SATA_BASE 0x038 +#define CCU_AXI_GMAC0_BASE 0x03C +#define CCU_AXI_GMAC1_BASE 0x040 +#define CCU_AXI_XGMAC_BASE 0x044 +#define CCU_AXI_PCIE_M_BASE 0x048 +#define CCU_AXI_PCIE_S_BASE 0x04C +#define CCU_AXI_USB_BASE 0x050 +#define CCU_AXI_HWA_BASE 0x054 +#define CCU_AXI_SRAM_BASE 0x058 + +#define CCU_SYS_DDR_BASE 0x02c +#define CCU_SYS_SATA_REF_BASE 0x060 +#define CCU_SYS_APB_BASE 0x064 +#define CCU_SYS_PCIE_BASE 0x144 + +#define CCU_RST_DELAY_US 1 + +#define CCU_RST_TRIG(_base, _ofs) \ + { \ + .type = CCU_RST_TRIG, \ + .base = _base, \ + .mask = BIT(_ofs), \ + } + +#define CCU_RST_DIR(_base, _ofs) \ + { \ + .type = CCU_RST_DIR, \ + .base = _base, \ + .mask = BIT(_ofs), \ + } + +struct ccu_rst_info { + enum ccu_rst_type type; + unsigned int base; + unsigned int mask; +}; + +/* + * Each AXI-bus clock divider is equipped with the corresponding clock-consumer + * domain reset (it's self-deasserted reset control). + */ +static const struct ccu_rst_info axi_rst_info[] = { + [CCU_AXI_MAIN_RST] = CCU_RST_TRIG(CCU_AXI_MAIN_BASE, 1), + [CCU_AXI_DDR_RST] = CCU_RST_TRIG(CCU_AXI_DDR_BASE, 1), + [CCU_AXI_SATA_RST] = CCU_RST_TRIG(CCU_AXI_SATA_BASE, 1), + [CCU_AXI_GMAC0_RST] = CCU_RST_TRIG(CCU_AXI_GMAC0_BASE, 1), + [CCU_AXI_GMAC1_RST] = CCU_RST_TRIG(CCU_AXI_GMAC1_BASE, 1), + [CCU_AXI_XGMAC_RST] = CCU_RST_TRIG(CCU_AXI_XGMAC_BASE, 1), + [CCU_AXI_PCIE_M_RST] = CCU_RST_TRIG(CCU_AXI_PCIE_M_BASE, 1), + [CCU_AXI_PCIE_S_RST] = CCU_RST_TRIG(CCU_AXI_PCIE_S_BASE, 1), + [CCU_AXI_USB_RST] = CCU_RST_TRIG(CCU_AXI_USB_BASE, 1), + [CCU_AXI_HWA_RST] = CCU_RST_TRIG(CCU_AXI_HWA_BASE, 1), + [CCU_AXI_SRAM_RST] = CCU_RST_TRIG(CCU_AXI_SRAM_BASE, 1), +}; + +/* + * SATA reference clock domain and APB-bus domain are connected with the + * sefl-deasserted reset control, which can be activated via the corresponding + * clock divider register. DDR and PCIe sub-domains can be reset with directly + * controlled reset signals. Resetting the DDR controller though won't end up + * well while the Linux kernel is working. + */ +static const struct ccu_rst_info sys_rst_info[] = { + [CCU_SYS_SATA_REF_RST] = CCU_RST_TRIG(CCU_SYS_SATA_REF_BASE, 1), + [CCU_SYS_APB_RST] = CCU_RST_TRIG(CCU_SYS_APB_BASE, 1), + [CCU_SYS_DDR_FULL_RST] = CCU_RST_DIR(CCU_SYS_DDR_BASE, 1), + [CCU_SYS_DDR_INIT_RST] = CCU_RST_DIR(CCU_SYS_DDR_BASE, 2), + [CCU_SYS_PCIE_PCS_PHY_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 0), + [CCU_SYS_PCIE_PIPE0_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 4), + [CCU_SYS_PCIE_CORE_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 8), + [CCU_SYS_PCIE_PWR_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 9), + [CCU_SYS_PCIE_STICKY_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 10), + [CCU_SYS_PCIE_NSTICKY_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 11), + [CCU_SYS_PCIE_HOT_RST] = CCU_RST_DIR(CCU_SYS_PCIE_BASE, 12), +}; + +static int ccu_rst_reset(struct reset_controller_dev *rcdev, unsigned long idx) +{ + struct ccu_rst *rst = to_ccu_rst(rcdev); + const struct ccu_rst_info *info = &rst->rsts_info[idx]; + + if (info->type != CCU_RST_TRIG) + return -EOPNOTSUPP; + + regmap_update_bits(rst->sys_regs, info->base, info->mask, info->mask); + + /* The next delay must be enough to cover all the resets. */ + udelay(CCU_RST_DELAY_US); + + return 0; +} + +static int ccu_rst_set(struct reset_controller_dev *rcdev, + unsigned long idx, bool high) +{ + struct ccu_rst *rst = to_ccu_rst(rcdev); + const struct ccu_rst_info *info = &rst->rsts_info[idx]; + + if (info->type != CCU_RST_DIR) + return high ? -EOPNOTSUPP : 0; + + return regmap_update_bits(rst->sys_regs, info->base, + info->mask, high ? info->mask : 0); +} + +static int ccu_rst_assert(struct reset_controller_dev *rcdev, + unsigned long idx) +{ + return ccu_rst_set(rcdev, idx, true); +} + +static int ccu_rst_deassert(struct reset_controller_dev *rcdev, + unsigned long idx) +{ + return ccu_rst_set(rcdev, idx, false); +} + +static int ccu_rst_status(struct reset_controller_dev *rcdev, + unsigned long idx) +{ + struct ccu_rst *rst = to_ccu_rst(rcdev); + const struct ccu_rst_info *info = &rst->rsts_info[idx]; + u32 val; + + if (info->type != CCU_RST_DIR) + return -EOPNOTSUPP; + + regmap_read(rst->sys_regs, info->base, &val); + + return !!(val & info->mask); +} + +static const struct reset_control_ops ccu_rst_ops = { + .reset = ccu_rst_reset, + .assert = ccu_rst_assert, + .deassert = ccu_rst_deassert, + .status = ccu_rst_status, +}; + +struct ccu_rst *ccu_rst_hw_register(const struct ccu_rst_init_data *rst_init) +{ + struct ccu_rst *rst; + int ret; + + if (!rst_init) + return ERR_PTR(-EINVAL); + + rst = kzalloc(sizeof(*rst), GFP_KERNEL); + if (!rst) + return ERR_PTR(-ENOMEM); + + rst->sys_regs = rst_init->sys_regs; + if (of_device_is_compatible(rst_init->np, "baikal,bt1-ccu-axi")) { + rst->rcdev.nr_resets = ARRAY_SIZE(axi_rst_info); + rst->rsts_info = axi_rst_info; + } else if (of_device_is_compatible(rst_init->np, "baikal,bt1-ccu-sys")) { + rst->rcdev.nr_resets = ARRAY_SIZE(sys_rst_info); + rst->rsts_info = sys_rst_info; + } else { + pr_err("Incompatible DT node '%s' specified\n", + of_node_full_name(rst_init->np)); + ret = -EINVAL; + goto err_kfree_rst; + } + + rst->rcdev.owner = THIS_MODULE; + rst->rcdev.ops = &ccu_rst_ops; + rst->rcdev.of_node = rst_init->np; + + ret = reset_controller_register(&rst->rcdev); + if (ret) { + pr_err("Couldn't register '%s' reset controller\n", + of_node_full_name(rst_init->np)); + goto err_kfree_rst; + } + + return rst; + +err_kfree_rst: + kfree(rst); + + return ERR_PTR(ret); +} + +void ccu_rst_hw_unregister(struct ccu_rst *rst) +{ + reset_controller_unregister(&rst->rcdev); + + kfree(rst); +} diff --git a/drivers/clk/baikal-t1/ccu-rst.h b/drivers/clk/baikal-t1/ccu-rst.h new file mode 100644 index 0000000000000000000000000000000000000000..d6e8b2f671f4d9a4f2be7f463ff2ffe49696ec2e --- /dev/null +++ b/drivers/clk/baikal-t1/ccu-rst.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 BAIKAL ELECTRONICS, JSC + * + * Baikal-T1 CCU Resets interface driver + */ +#ifndef __CLK_BT1_CCU_RST_H__ +#define __CLK_BT1_CCU_RST_H__ + +#include +#include +#include + +struct ccu_rst_info; + +/* + * enum ccu_rst_type - CCU Reset types + * @CCU_RST_TRIG: Self-deasserted reset signal. + * @CCU_RST_DIR: Directly controlled reset signal. + */ +enum ccu_rst_type { + CCU_RST_TRIG, + CCU_RST_DIR, +}; + +/* + * struct ccu_rst_init_data - CCU Resets initialization data + * @sys_regs: Baikal-T1 System Controller registers map. + * @np: Pointer to the node with the System CCU block. + */ +struct ccu_rst_init_data { + struct regmap *sys_regs; + struct device_node *np; +}; + +/* + * struct ccu_rst - CCU Reset descriptor + * @rcdev: Reset controller descriptor. + * @sys_regs: Baikal-T1 System Controller registers map. + * @rsts_info: Reset flag info (base address and mask). + */ +struct ccu_rst { + struct reset_controller_dev rcdev; + struct regmap *sys_regs; + const struct ccu_rst_info *rsts_info; +}; +#define to_ccu_rst(_rcdev) container_of(_rcdev, struct ccu_rst, rcdev) + +#ifdef CONFIG_CLK_BT1_CCU_RST + +struct ccu_rst *ccu_rst_hw_register(const struct ccu_rst_init_data *init); + +void ccu_rst_hw_unregister(struct ccu_rst *rst); + +#else + +static inline +struct ccu_rst *ccu_rst_hw_register(const struct ccu_rst_init_data *init) +{ + return NULL; +} + +static inline void ccu_rst_hw_unregister(struct ccu_rst *rst) {} + +#endif + +#endif /* __CLK_BT1_CCU_RST_H__ */ diff --git a/drivers/clk/baikal-t1/clk-ccu-div.c b/drivers/clk/baikal-t1/clk-ccu-div.c index f141fda12b09aa69d7361581efabccfafd9d238c..0e772e034812fddc4eee5df8a405abded00e5b9a 100644 --- a/drivers/clk/baikal-t1/clk-ccu-div.c +++ b/drivers/clk/baikal-t1/clk-ccu-div.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) "bt1-ccu-div: " fmt #include +#include #include #include #include @@ -24,9 +25,9 @@ #include #include -#include #include "ccu-div.h" +#include "ccu-rst.h" #define CCU_AXI_MAIN_BASE 0x030 #define CCU_AXI_DDR_BASE 0x034 @@ -76,6 +77,16 @@ .divider = _divider \ } +#define CCU_DIV_BUF_INFO(_id, _name, _pname, _base, _flags) \ + { \ + .id = _id, \ + .name = _name, \ + .parent_name = _pname, \ + .base = _base, \ + .type = CCU_DIV_BUF, \ + .flags = _flags \ + } + #define CCU_DIV_FIXED_INFO(_id, _name, _pname, _divider) \ { \ .id = _id, \ @@ -85,12 +96,6 @@ .divider = _divider \ } -#define CCU_DIV_RST_MAP(_rst_id, _clk_id) \ - { \ - .rst_id = _rst_id, \ - .clk_id = _clk_id \ - } - struct ccu_div_info { unsigned int id; const char *name; @@ -105,11 +110,6 @@ struct ccu_div_info { unsigned long features; }; -struct ccu_div_rst_map { - unsigned int rst_id; - unsigned int clk_id; -}; - struct ccu_div_data { struct device_node *np; struct regmap *sys_regs; @@ -118,11 +118,8 @@ struct ccu_div_data { const struct ccu_div_info *divs_info; struct ccu_div **divs; - unsigned int rst_num; - const struct ccu_div_rst_map *rst_map; - struct reset_controller_dev rcdev; + struct ccu_rst *rsts; }; -#define to_ccu_div_data(_rcdev) container_of(_rcdev, struct ccu_div_data, rcdev) /* * AXI Main Interconnect (axi_main_clk) and DDR AXI-bus (axi_ddr_clk) clocks @@ -169,33 +166,22 @@ static const struct ccu_div_info axi_info[] = { CLK_SET_RATE_GATE, CCU_DIV_RESET_DOMAIN) }; -static const struct ccu_div_rst_map axi_rst_map[] = { - CCU_DIV_RST_MAP(CCU_AXI_MAIN_RST, CCU_AXI_MAIN_CLK), - CCU_DIV_RST_MAP(CCU_AXI_DDR_RST, CCU_AXI_DDR_CLK), - CCU_DIV_RST_MAP(CCU_AXI_SATA_RST, CCU_AXI_SATA_CLK), - CCU_DIV_RST_MAP(CCU_AXI_GMAC0_RST, CCU_AXI_GMAC0_CLK), - CCU_DIV_RST_MAP(CCU_AXI_GMAC1_RST, CCU_AXI_GMAC1_CLK), - CCU_DIV_RST_MAP(CCU_AXI_XGMAC_RST, CCU_AXI_XGMAC_CLK), - CCU_DIV_RST_MAP(CCU_AXI_PCIE_M_RST, CCU_AXI_PCIE_M_CLK), - CCU_DIV_RST_MAP(CCU_AXI_PCIE_S_RST, CCU_AXI_PCIE_S_CLK), - CCU_DIV_RST_MAP(CCU_AXI_USB_RST, CCU_AXI_USB_CLK), - CCU_DIV_RST_MAP(CCU_AXI_HWA_RST, CCU_AXI_HWA_CLK), - CCU_DIV_RST_MAP(CCU_AXI_SRAM_RST, CCU_AXI_SRAM_CLK) -}; - /* * APB-bus clock is marked as critical since it's a main communication bus * for the SoC devices registers IO-operations. */ static const struct ccu_div_info sys_info[] = { - CCU_DIV_VAR_INFO(CCU_SYS_SATA_REF_CLK, "sys_sata_ref_clk", + CCU_DIV_VAR_INFO(CCU_SYS_SATA_CLK, "sys_sata_clk", "sata_clk", CCU_SYS_SATA_REF_BASE, 4, CLK_SET_RATE_GATE, CCU_DIV_SKIP_ONE | CCU_DIV_LOCK_SHIFTED | CCU_DIV_RESET_DOMAIN), + CCU_DIV_BUF_INFO(CCU_SYS_SATA_REF_CLK, "sys_sata_ref_clk", + "sys_sata_clk", CCU_SYS_SATA_REF_BASE, + CLK_SET_RATE_PARENT), CCU_DIV_VAR_INFO(CCU_SYS_APB_CLK, "sys_apb_clk", "pcie_clk", CCU_SYS_APB_BASE, 5, - CLK_IS_CRITICAL, CCU_DIV_RESET_DOMAIN), + CLK_IS_CRITICAL, CCU_DIV_BASIC | CCU_DIV_RESET_DOMAIN), CCU_DIV_GATE_INFO(CCU_SYS_GMAC0_TX_CLK, "sys_gmac0_tx_clk", "eth_clk", CCU_SYS_GMAC0_BASE, 5), CCU_DIV_FIXED_INFO(CCU_SYS_GMAC0_PTP_CLK, "sys_gmac0_ptp_clk", @@ -204,10 +190,12 @@ static const struct ccu_div_info sys_info[] = { "eth_clk", CCU_SYS_GMAC1_BASE, 5), CCU_DIV_FIXED_INFO(CCU_SYS_GMAC1_PTP_CLK, "sys_gmac1_ptp_clk", "eth_clk", 10), - CCU_DIV_GATE_INFO(CCU_SYS_XGMAC_REF_CLK, "sys_xgmac_ref_clk", - "eth_clk", CCU_SYS_XGMAC_BASE, 8), + CCU_DIV_GATE_INFO(CCU_SYS_XGMAC_CLK, "sys_xgmac_clk", + "eth_clk", CCU_SYS_XGMAC_BASE, 1), + CCU_DIV_FIXED_INFO(CCU_SYS_XGMAC_REF_CLK, "sys_xgmac_ref_clk", + "sys_xgmac_clk", 8), CCU_DIV_FIXED_INFO(CCU_SYS_XGMAC_PTP_CLK, "sys_xgmac_ptp_clk", - "eth_clk", 10), + "sys_xgmac_clk", 8), CCU_DIV_GATE_INFO(CCU_SYS_USB_CLK, "sys_usb_clk", "eth_clk", CCU_SYS_USB_BASE, 10), CCU_DIV_VAR_INFO(CCU_SYS_PVT_CLK, "sys_pvt_clk", @@ -227,74 +215,58 @@ static const struct ccu_div_info sys_info[] = { "ref_clk", 25), CCU_DIV_VAR_INFO(CCU_SYS_TIMER0_CLK, "sys_timer0_clk", "ref_clk", CCU_SYS_TIMER0_BASE, 17, - CLK_SET_RATE_GATE, 0), + CLK_SET_RATE_GATE, CCU_DIV_BASIC), CCU_DIV_VAR_INFO(CCU_SYS_TIMER1_CLK, "sys_timer1_clk", "ref_clk", CCU_SYS_TIMER1_BASE, 17, - CLK_SET_RATE_GATE, 0), + CLK_SET_RATE_GATE, CCU_DIV_BASIC), CCU_DIV_VAR_INFO(CCU_SYS_TIMER2_CLK, "sys_timer2_clk", "ref_clk", CCU_SYS_TIMER2_BASE, 17, - CLK_SET_RATE_GATE, 0), + CLK_SET_RATE_GATE, CCU_DIV_BASIC), CCU_DIV_VAR_INFO(CCU_SYS_WDT_CLK, "sys_wdt_clk", "eth_clk", CCU_SYS_WDT_BASE, 17, CLK_SET_RATE_GATE, CCU_DIV_SKIP_ONE_TO_THREE) }; -static const struct ccu_div_rst_map sys_rst_map[] = { - CCU_DIV_RST_MAP(CCU_SYS_SATA_REF_RST, CCU_SYS_SATA_REF_CLK), - CCU_DIV_RST_MAP(CCU_SYS_APB_RST, CCU_SYS_APB_CLK), -}; +static struct ccu_div_data *axi_data; +static struct ccu_div_data *sys_data; -static struct ccu_div *ccu_div_find_desc(struct ccu_div_data *data, - unsigned int clk_id) +static void ccu_div_set_data(struct ccu_div_data *data) { - struct ccu_div *div; - int idx; - - for (idx = 0; idx < data->divs_num; ++idx) { - div = data->divs[idx]; - if (div && div->id == clk_id) - return div; - } - - return ERR_PTR(-EINVAL); + struct device_node *np = data->np; + + if (of_device_is_compatible(np, "baikal,bt1-ccu-axi")) + axi_data = data; + else if (of_device_is_compatible(np, "baikal,bt1-ccu-sys")) + sys_data = data; + else + pr_err("Invalid DT node '%s' specified\n", of_node_full_name(np)); } -static int ccu_div_reset(struct reset_controller_dev *rcdev, - unsigned long rst_id) +static struct ccu_div_data *ccu_div_get_data(struct device_node *np) { - struct ccu_div_data *data = to_ccu_div_data(rcdev); - const struct ccu_div_rst_map *map; - struct ccu_div *div; - int idx, ret; + if (of_device_is_compatible(np, "baikal,bt1-ccu-axi")) + return axi_data; + else if (of_device_is_compatible(np, "baikal,bt1-ccu-sys")) + return sys_data; - for (idx = 0, map = data->rst_map; idx < data->rst_num; ++idx, ++map) { - if (map->rst_id == rst_id) - break; - } - if (idx == data->rst_num) { - pr_err("Invalid reset ID %lu specified\n", rst_id); - return -EINVAL; - } + pr_err("Invalid DT node '%s' specified\n", of_node_full_name(np)); - div = ccu_div_find_desc(data, map->clk_id); - if (IS_ERR(div)) { - pr_err("Invalid clock ID %d in mapping\n", map->clk_id); - return PTR_ERR(div); - } + return NULL; +} - ret = ccu_div_reset_domain(div); - if (ret) { - pr_err("Reset isn't supported by divider %s\n", - clk_hw_get_name(ccu_div_get_clk_hw(div))); +static struct ccu_div *ccu_div_find_desc(struct ccu_div_data *data, + unsigned int clk_id) +{ + int idx; + + for (idx = 0; idx < data->divs_num; ++idx) { + if (data->divs_info[idx].id == clk_id) + return data->divs[idx]; } - return ret; + return ERR_PTR(-EINVAL); } -static const struct reset_control_ops ccu_div_rst_ops = { - .reset = ccu_div_reset, -}; - static struct ccu_div_data *ccu_div_create_data(struct device_node *np) { struct ccu_div_data *data; @@ -308,13 +280,9 @@ static struct ccu_div_data *ccu_div_create_data(struct device_node *np) if (of_device_is_compatible(np, "baikal,bt1-ccu-axi")) { data->divs_num = ARRAY_SIZE(axi_info); data->divs_info = axi_info; - data->rst_num = ARRAY_SIZE(axi_rst_map); - data->rst_map = axi_rst_map; } else if (of_device_is_compatible(np, "baikal,bt1-ccu-sys")) { data->divs_num = ARRAY_SIZE(sys_info); data->divs_info = sys_info; - data->rst_num = ARRAY_SIZE(sys_rst_map); - data->rst_map = sys_rst_map; } else { pr_err("Incompatible DT node '%s' specified\n", of_node_full_name(np)); @@ -365,14 +333,16 @@ static struct clk_hw *ccu_div_of_clk_hw_get(struct of_phandle_args *clkspec, clk_id = clkspec->args[0]; div = ccu_div_find_desc(data, clk_id); if (IS_ERR(div)) { - pr_info("Invalid clock ID %d specified\n", clk_id); + if (div != ERR_PTR(-EPROBE_DEFER)) + pr_info("Invalid clock ID %d specified\n", clk_id); + return ERR_CAST(div); } return ccu_div_get_clk_hw(div); } -static int ccu_div_clk_register(struct ccu_div_data *data) +static int ccu_div_clk_register(struct ccu_div_data *data, bool defer) { int idx, ret; @@ -380,6 +350,13 @@ static int ccu_div_clk_register(struct ccu_div_data *data) const struct ccu_div_info *info = &data->divs_info[idx]; struct ccu_div_init_data init = {0}; + if (!!(info->features & CCU_DIV_BASIC) ^ defer) { + if (!data->divs[idx]) + data->divs[idx] = ERR_PTR(-EPROBE_DEFER); + + continue; + } + init.id = info->id; init.name = info->name; init.parent_name = info->parent_name; @@ -396,6 +373,9 @@ static int ccu_div_clk_register(struct ccu_div_data *data) init.base = info->base; init.sys_regs = data->sys_regs; init.divider = info->divider; + } else if (init.type == CCU_DIV_BUF) { + init.base = info->base; + init.sys_regs = data->sys_regs; } else { init.divider = info->divider; } @@ -409,49 +389,104 @@ static int ccu_div_clk_register(struct ccu_div_data *data) } } - ret = of_clk_add_hw_provider(data->np, ccu_div_of_clk_hw_get, data); - if (ret) { - pr_err("Couldn't register dividers '%s' clock provider\n", - of_node_full_name(data->np)); - goto err_hw_unregister; - } - return 0; err_hw_unregister: - for (--idx; idx >= 0; --idx) + for (--idx; idx >= 0; --idx) { + if (!!(data->divs_info[idx].features & CCU_DIV_BASIC) ^ defer) + continue; + ccu_div_hw_unregister(data->divs[idx]); + } return ret; } -static void ccu_div_clk_unregister(struct ccu_div_data *data) +static void ccu_div_clk_unregister(struct ccu_div_data *data, bool defer) { int idx; - of_clk_del_provider(data->np); + /* Uninstall only the clocks registered on the specfied stage */ + for (idx = 0; idx < data->divs_num; ++idx) { + if (!!(data->divs_info[idx].features & CCU_DIV_BASIC) ^ defer) + continue; - for (idx = 0; idx < data->divs_num; ++idx) ccu_div_hw_unregister(data->divs[idx]); + } } -static int ccu_div_rst_register(struct ccu_div_data *data) +static int ccu_div_of_register(struct ccu_div_data *data) { int ret; - data->rcdev.ops = &ccu_div_rst_ops; - data->rcdev.of_node = data->np; - data->rcdev.nr_resets = data->rst_num; + ret = of_clk_add_hw_provider(data->np, ccu_div_of_clk_hw_get, data); + if (ret) { + pr_err("Couldn't register dividers '%s' clock provider\n", + of_node_full_name(data->np)); + } + + return ret; +} - ret = reset_controller_register(&data->rcdev); - if (ret) +static int ccu_div_rst_register(struct ccu_div_data *data) +{ + struct ccu_rst_init_data init = {0}; + + init.sys_regs = data->sys_regs; + init.np = data->np; + + data->rsts = ccu_rst_hw_register(&init); + if (IS_ERR(data->rsts)) { pr_err("Couldn't register divider '%s' reset controller\n", of_node_full_name(data->np)); + return PTR_ERR(data->rsts); + } + + return 0; +} + +static int ccu_div_probe(struct platform_device *pdev) +{ + struct ccu_div_data *data; + int ret; + + data = ccu_div_get_data(dev_of_node(&pdev->dev)); + if (!data) + return -EINVAL; + + ret = ccu_div_clk_register(data, false); + if (ret) + return ret; + + ret = ccu_div_rst_register(data); + if (ret) + goto err_clk_unregister; + + return 0; + +err_clk_unregister: + ccu_div_clk_unregister(data, false); return ret; } -static void ccu_div_init(struct device_node *np) +static const struct of_device_id ccu_div_of_match[] = { + { .compatible = "baikal,bt1-ccu-axi" }, + { .compatible = "baikal,bt1-ccu-sys" }, + { } +}; + +static struct platform_driver ccu_div_driver = { + .probe = ccu_div_probe, + .driver = { + .name = "clk-ccu-div", + .of_match_table = ccu_div_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(ccu_div_driver); + +static __init void ccu_div_init(struct device_node *np) { struct ccu_div_data *data; int ret; @@ -464,22 +499,23 @@ static void ccu_div_init(struct device_node *np) if (ret) goto err_free_data; - ret = ccu_div_clk_register(data); + ret = ccu_div_clk_register(data, true); if (ret) goto err_free_data; - ret = ccu_div_rst_register(data); + ret = ccu_div_of_register(data); if (ret) goto err_clk_unregister; + ccu_div_set_data(data); + return; err_clk_unregister: - ccu_div_clk_unregister(data); + ccu_div_clk_unregister(data, true); err_free_data: ccu_div_free_data(data); } - -CLK_OF_DECLARE(ccu_axi, "baikal,bt1-ccu-axi", ccu_div_init); -CLK_OF_DECLARE(ccu_sys, "baikal,bt1-ccu-sys", ccu_div_init); +CLK_OF_DECLARE_DRIVER(ccu_axi, "baikal,bt1-ccu-axi", ccu_div_init); +CLK_OF_DECLARE_DRIVER(ccu_sys, "baikal,bt1-ccu-sys", ccu_div_init); diff --git a/drivers/clk/baikal-t1/clk-ccu-pll.c b/drivers/clk/baikal-t1/clk-ccu-pll.c index 2445d4b12bafa09e9413ff4750bd7511c2d2c852..fce02ce77347a5f18e3b9ce635f2d3093df601f4 100644 --- a/drivers/clk/baikal-t1/clk-ccu-pll.c +++ b/drivers/clk/baikal-t1/clk-ccu-pll.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) "bt1-ccu-pll: " fmt #include +#include #include #include #include @@ -31,13 +32,14 @@ #define CCU_PCIE_PLL_BASE 0x018 #define CCU_ETH_PLL_BASE 0x020 -#define CCU_PLL_INFO(_id, _name, _pname, _base, _flags) \ - { \ - .id = _id, \ - .name = _name, \ - .parent_name = _pname, \ - .base = _base, \ - .flags = _flags \ +#define CCU_PLL_INFO(_id, _name, _pname, _base, _flags, _features) \ + { \ + .id = _id, \ + .name = _name, \ + .parent_name = _pname, \ + .base = _base, \ + .flags = _flags, \ + .features = _features, \ } #define CCU_PLL_NUM ARRAY_SIZE(pll_info) @@ -48,6 +50,7 @@ struct ccu_pll_info { const char *parent_name; unsigned int base; unsigned long flags; + unsigned long features; }; /* @@ -61,15 +64,15 @@ struct ccu_pll_info { */ static const struct ccu_pll_info pll_info[] = { CCU_PLL_INFO(CCU_CPU_PLL, "cpu_pll", "ref_clk", CCU_CPU_PLL_BASE, - CLK_IS_CRITICAL), + CLK_IS_CRITICAL, CCU_PLL_BASIC), CCU_PLL_INFO(CCU_SATA_PLL, "sata_pll", "ref_clk", CCU_SATA_PLL_BASE, - CLK_IS_CRITICAL | CLK_SET_RATE_GATE), + CLK_IS_CRITICAL | CLK_SET_RATE_GATE, 0), CCU_PLL_INFO(CCU_DDR_PLL, "ddr_pll", "ref_clk", CCU_DDR_PLL_BASE, - CLK_IS_CRITICAL | CLK_SET_RATE_GATE), + CLK_IS_CRITICAL | CLK_SET_RATE_GATE, 0), CCU_PLL_INFO(CCU_PCIE_PLL, "pcie_pll", "ref_clk", CCU_PCIE_PLL_BASE, - CLK_IS_CRITICAL), + CLK_IS_CRITICAL, CCU_PLL_BASIC), CCU_PLL_INFO(CCU_ETH_PLL, "eth_pll", "ref_clk", CCU_ETH_PLL_BASE, - CLK_IS_CRITICAL | CLK_SET_RATE_GATE) + CLK_IS_CRITICAL | CLK_SET_RATE_GATE, 0) }; struct ccu_pll_data { @@ -78,16 +81,16 @@ struct ccu_pll_data { struct ccu_pll *plls[CCU_PLL_NUM]; }; +static struct ccu_pll_data *pll_data; + static struct ccu_pll *ccu_pll_find_desc(struct ccu_pll_data *data, unsigned int clk_id) { - struct ccu_pll *pll; int idx; for (idx = 0; idx < CCU_PLL_NUM; ++idx) { - pll = data->plls[idx]; - if (pll && pll->id == clk_id) - return pll; + if (pll_info[idx].id == clk_id) + return data->plls[idx]; } return ERR_PTR(-EINVAL); @@ -133,14 +136,16 @@ static struct clk_hw *ccu_pll_of_clk_hw_get(struct of_phandle_args *clkspec, clk_id = clkspec->args[0]; pll = ccu_pll_find_desc(data, clk_id); if (IS_ERR(pll)) { - pr_info("Invalid PLL clock ID %d specified\n", clk_id); + if (pll != ERR_PTR(-EPROBE_DEFER)) + pr_info("Invalid PLL clock ID %d specified\n", clk_id); + return ERR_CAST(pll); } return ccu_pll_get_clk_hw(pll); } -static int ccu_pll_clk_register(struct ccu_pll_data *data) +static int ccu_pll_clk_register(struct ccu_pll_data *data, bool defer) { int idx, ret; @@ -148,6 +153,14 @@ static int ccu_pll_clk_register(struct ccu_pll_data *data) const struct ccu_pll_info *info = &pll_info[idx]; struct ccu_pll_init_data init = {0}; + /* Defer non-basic PLLs allocation for the probe stage */ + if (!!(info->features & CCU_PLL_BASIC) ^ defer) { + if (!data->plls[idx]) + data->plls[idx] = ERR_PTR(-EPROBE_DEFER); + + continue; + } + init.id = info->id; init.name = info->name; init.parent_name = info->parent_name; @@ -155,6 +168,7 @@ static int ccu_pll_clk_register(struct ccu_pll_data *data) init.sys_regs = data->sys_regs; init.np = data->np; init.flags = info->flags; + init.features = info->features; data->plls[idx] = ccu_pll_hw_register(&init); if (IS_ERR(data->plls[idx])) { @@ -165,22 +179,70 @@ static int ccu_pll_clk_register(struct ccu_pll_data *data) } } + return 0; + +err_hw_unregister: + for (--idx; idx >= 0; --idx) { + if (!!(pll_info[idx].features & CCU_PLL_BASIC) ^ defer) + continue; + + ccu_pll_hw_unregister(data->plls[idx]); + } + + return ret; +} + +static void ccu_pll_clk_unregister(struct ccu_pll_data *data, bool defer) +{ + int idx; + + /* Uninstall only the clocks registered on the specfied stage */ + for (idx = 0; idx < CCU_PLL_NUM; ++idx) { + if (!!(pll_info[idx].features & CCU_PLL_BASIC) ^ defer) + continue; + + ccu_pll_hw_unregister(data->plls[idx]); + } +} + +static int ccu_pll_of_register(struct ccu_pll_data *data) +{ + int ret; + ret = of_clk_add_hw_provider(data->np, ccu_pll_of_clk_hw_get, data); if (ret) { pr_err("Couldn't register PLL provider of '%s'\n", of_node_full_name(data->np)); - goto err_hw_unregister; } - return 0; + return ret; +} -err_hw_unregister: - for (--idx; idx >= 0; --idx) - ccu_pll_hw_unregister(data->plls[idx]); +static int ccu_pll_probe(struct platform_device *pdev) +{ + struct ccu_pll_data *data = pll_data; - return ret; + if (!data) + return -EINVAL; + + return ccu_pll_clk_register(data, false); } +static const struct of_device_id ccu_pll_of_match[] = { + { .compatible = "baikal,bt1-ccu-pll" }, + { } +}; + +static struct platform_driver ccu_pll_driver = { + .probe = ccu_pll_probe, + .driver = { + .name = "clk-ccu-pll", + .of_match_table = ccu_pll_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(ccu_pll_driver); + static __init void ccu_pll_init(struct device_node *np) { struct ccu_pll_data *data; @@ -194,13 +256,22 @@ static __init void ccu_pll_init(struct device_node *np) if (ret) goto err_free_data; - ret = ccu_pll_clk_register(data); + ret = ccu_pll_clk_register(data, true); if (ret) goto err_free_data; + ret = ccu_pll_of_register(data); + if (ret) + goto err_clk_unregister; + + pll_data = data; + return; +err_clk_unregister: + ccu_pll_clk_unregister(data, true); + err_free_data: ccu_pll_free_data(data); } -CLK_OF_DECLARE(ccu_pll, "baikal,bt1-ccu-pll", ccu_pll_init); +CLK_OF_DECLARE_DRIVER(ccu_pll, "baikal,bt1-ccu-pll", ccu_pll_init); diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c index 48a1eb9f2d551cc17b45c45925232dedbf577f49..e74fe6219d14e2e0c39b65a5e9b2a9cef7c42d9a 100644 --- a/drivers/clk/bcm/clk-bcm2835.c +++ b/drivers/clk/bcm/clk-bcm2835.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -502,6 +503,8 @@ struct bcm2835_clock_data { bool low_jitter; u32 tcnt_mux; + + bool round_up; }; struct bcm2835_gate_data { @@ -966,9 +969,9 @@ static u32 bcm2835_clock_choose_div(struct clk_hw *hw, return div; } -static long bcm2835_clock_rate_from_divisor(struct bcm2835_clock *clock, - unsigned long parent_rate, - u32 div) +static unsigned long bcm2835_clock_rate_from_divisor(struct bcm2835_clock *clock, + unsigned long parent_rate, + u32 div) { const struct bcm2835_clock_data *data = clock->data; u64 temp; @@ -993,12 +996,34 @@ static long bcm2835_clock_rate_from_divisor(struct bcm2835_clock *clock, return temp; } +static unsigned long bcm2835_round_rate(unsigned long rate) +{ + unsigned long scaler; + unsigned long limit; + + limit = rate / 100000; + + scaler = 1; + while (scaler < limit) + scaler *= 10; + + /* + * If increasing a clock by less than 0.1% changes it + * from ..999.. to ..000.., round up. + */ + if ((rate + scaler - 1) / scaler % 1000 == 0) + rate = roundup(rate, scaler); + + return rate; +} + static unsigned long bcm2835_clock_get_rate(struct clk_hw *hw, unsigned long parent_rate) { struct bcm2835_clock *clock = bcm2835_clock_from_hw(hw); struct bcm2835_cprman *cprman = clock->cprman; const struct bcm2835_clock_data *data = clock->data; + unsigned long rate; u32 div; if (data->int_bits == 0 && data->frac_bits == 0) @@ -1006,7 +1031,12 @@ static unsigned long bcm2835_clock_get_rate(struct clk_hw *hw, div = cprman_read(cprman, data->div_reg); - return bcm2835_clock_rate_from_divisor(clock, parent_rate, div); + rate = bcm2835_clock_rate_from_divisor(clock, parent_rate, div); + + if (data->round_up) + rate = bcm2835_round_rate(rate); + + return rate; } static void bcm2835_clock_wait_busy(struct bcm2835_clock *clock) @@ -1784,7 +1814,7 @@ static const struct bcm2835_clk_desc clk_desc_array[] = { .load_mask = CM_PLLC_LOADPER, .hold_mask = CM_PLLC_HOLDPER, .fixed_divider = 1, - .flags = CLK_SET_RATE_PARENT), + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT), /* * PLLD is the display PLL, used to drive DSI display panels. @@ -2143,7 +2173,8 @@ static const struct bcm2835_clk_desc clk_desc_array[] = { .div_reg = CM_UARTDIV, .int_bits = 10, .frac_bits = 12, - .tcnt_mux = 28), + .tcnt_mux = 28, + .round_up = true), /* TV encoder clock. Only operating frequency is 108Mhz. */ [BCM2835_CLOCK_VEC] = REGISTER_PER_CLK( diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c index 1a098db12062ebfbc8387e39e57aa852ae8e372a..680f9d8d357c7f93f943b266db78c3025d761c77 100644 --- a/drivers/clk/bcm/clk-iproc-pll.c +++ b/drivers/clk/bcm/clk-iproc-pll.c @@ -726,6 +726,7 @@ void iproc_pll_clk_setup(struct device_node *node, const char *parent_name; struct iproc_clk *iclk_array; struct clk_hw_onecell_data *clk_data; + const char *clk_name; if (WARN_ON(!pll_ctrl) || WARN_ON(!clk_ctrl)) return; @@ -773,7 +774,12 @@ void iproc_pll_clk_setup(struct device_node *node, iclk = &iclk_array[0]; iclk->pll = pll; - init.name = node->name; + ret = of_property_read_string_index(node, "clock-output-names", + 0, &clk_name); + if (WARN_ON(ret)) + goto err_pll_register; + + init.name = clk_name; init.ops = &iproc_pll_ops; init.flags = 0; parent_name = of_clk_get_parent_name(node, 0); @@ -793,13 +799,11 @@ void iproc_pll_clk_setup(struct device_node *node, goto err_pll_register; clk_data->hws[0] = &iclk->hw; + parent_name = clk_name; /* now initialize and register all leaf clocks */ for (i = 1; i < num_clks; i++) { - const char *clk_name; - memset(&init, 0, sizeof(init)); - parent_name = node->name; ret = of_property_read_string_index(node, "clock-output-names", i, &clk_name); diff --git a/drivers/clk/bcm/clk-raspberrypi.c b/drivers/clk/bcm/clk-raspberrypi.c index 73518009a0f204d61e9e177a79d57d66fce2a611..679f4649a7efd7cfc2d759ff00832c9c0c564517 100644 --- a/drivers/clk/bcm/clk-raspberrypi.c +++ b/drivers/clk/bcm/clk-raspberrypi.c @@ -33,6 +33,7 @@ enum rpi_firmware_clk_id { RPI_FIRMWARE_EMMC2_CLK_ID, RPI_FIRMWARE_M2MC_CLK_ID, RPI_FIRMWARE_PIXEL_BVB_CLK_ID, + RPI_FIRMWARE_VEC_CLK_ID, RPI_FIRMWARE_NUM_CLK_ID, }; @@ -51,6 +52,7 @@ static char *rpi_firmware_clk_names[] = { [RPI_FIRMWARE_EMMC2_CLK_ID] = "emmc2", [RPI_FIRMWARE_M2MC_CLK_ID] = "m2mc", [RPI_FIRMWARE_PIXEL_BVB_CLK_ID] = "pixel-bvb", + [RPI_FIRMWARE_VEC_CLK_ID] = "vec", }; #define RPI_FIRMWARE_STATE_ENABLE_BIT BIT(0) @@ -129,9 +131,18 @@ raspberrypi_clk_variants[RPI_FIRMWARE_NUM_CLK_ID] = { [RPI_FIRMWARE_V3D_CLK_ID] = { .export = true, }, + [RPI_FIRMWARE_PIXEL_CLK_ID] = { + .export = true, + }, + [RPI_FIRMWARE_HEVC_CLK_ID] = { + .export = true, + }, [RPI_FIRMWARE_PIXEL_BVB_CLK_ID] = { .export = true, }, + [RPI_FIRMWARE_VEC_CLK_ID] = { + .export = true, + }, }; /* @@ -203,7 +214,7 @@ static unsigned long raspberrypi_fw_get_rate(struct clk_hw *hw, ret = raspberrypi_clock_property(rpi->firmware, data, RPI_FIRMWARE_GET_CLOCK_RATE, &val); if (ret) - return ret; + return 0; return val; } @@ -220,7 +231,7 @@ static int raspberrypi_fw_set_rate(struct clk_hw *hw, unsigned long rate, ret = raspberrypi_clock_property(rpi->firmware, data, RPI_FIRMWARE_SET_CLOCK_RATE, &_rate); if (ret) - dev_err_ratelimited(rpi->dev, "Failed to change %s frequency: %d", + dev_err_ratelimited(rpi->dev, "Failed to change %s frequency: %d\n", clk_hw_get_name(hw), ret); return ret; @@ -288,7 +299,7 @@ static struct clk_hw *raspberrypi_clk_register(struct raspberrypi_clk *rpi, RPI_FIRMWARE_GET_MIN_CLOCK_RATE, &min_rate); if (ret) { - dev_err(rpi->dev, "Failed to get clock %d min freq: %d", + dev_err(rpi->dev, "Failed to get clock %d min freq: %d\n", id, ret); return ERR_PTR(ret); } @@ -344,8 +355,13 @@ static int raspberrypi_discover_clocks(struct raspberrypi_clk *rpi, struct rpi_firmware_get_clocks_response *clks; int ret; + /* + * The firmware doesn't guarantee that the last element of + * RPI_FIRMWARE_GET_CLOCKS is zeroed. So allocate an additional + * zero element as sentinel. + */ clks = devm_kcalloc(rpi->dev, - RPI_FIRMWARE_NUM_CLK_ID, sizeof(*clks), + RPI_FIRMWARE_NUM_CLK_ID + 1, sizeof(*clks), GFP_KERNEL); if (!clks) return -ENOMEM; @@ -360,7 +376,8 @@ static int raspberrypi_discover_clocks(struct raspberrypi_clk *rpi, struct raspberrypi_clk_variant *variant; if (clks->id > RPI_FIRMWARE_NUM_CLK_ID) { - dev_err(rpi->dev, "Unknown clock id: %u", clks->id); + dev_err(rpi->dev, "Unknown clock id: %u (max: %u)\n", + clks->id, RPI_FIRMWARE_NUM_CLK_ID); return -EINVAL; } diff --git a/drivers/clk/berlin/bg2.c b/drivers/clk/berlin/bg2.c index bccdfa00fd37394aa62538356783ccd3a9eef8e3..67a9edbba29c448bbd9f58e3e06f095788d1168e 100644 --- a/drivers/clk/berlin/bg2.c +++ b/drivers/clk/berlin/bg2.c @@ -500,12 +500,15 @@ static void __init berlin2_clock_setup(struct device_node *np) int n, ret; clk_data = kzalloc(struct_size(clk_data, hws, MAX_CLKS), GFP_KERNEL); - if (!clk_data) + if (!clk_data) { + of_node_put(parent_np); return; + } clk_data->num = MAX_CLKS; hws = clk_data->hws; gbase = of_iomap(parent_np, 0); + of_node_put(parent_np); if (!gbase) return; diff --git a/drivers/clk/berlin/bg2q.c b/drivers/clk/berlin/bg2q.c index e9518d35f262e171cb1233707579f8b8102a4023..dd2784bb75b649c2f5eb072759ff4fee6a4cc47a 100644 --- a/drivers/clk/berlin/bg2q.c +++ b/drivers/clk/berlin/bg2q.c @@ -286,19 +286,23 @@ static void __init berlin2q_clock_setup(struct device_node *np) int n, ret; clk_data = kzalloc(struct_size(clk_data, hws, MAX_CLKS), GFP_KERNEL); - if (!clk_data) + if (!clk_data) { + of_node_put(parent_np); return; + } clk_data->num = MAX_CLKS; hws = clk_data->hws; gbase = of_iomap(parent_np, 0); if (!gbase) { + of_node_put(parent_np); pr_err("%pOF: Unable to map global base\n", np); return; } /* BG2Q CPU PLL is not part of global registers */ cpupll_base = of_iomap(parent_np, 1); + of_node_put(parent_np); if (!cpupll_base) { pr_err("%pOF: Unable to map cpupll base\n", np); iounmap(gbase); diff --git a/drivers/clk/clk-asm9260.c b/drivers/clk/clk-asm9260.c index bacebd457e6f34697e546b7b7ea12a1cc8960427..8b3c059e19a1279a13a5378913238ad775105cfc 100644 --- a/drivers/clk/clk-asm9260.c +++ b/drivers/clk/clk-asm9260.c @@ -80,7 +80,7 @@ struct asm9260_mux_clock { u8 mask; u32 *table; const char *name; - const char **parent_names; + const struct clk_parent_data *parent_data; u8 num_parents; unsigned long offset; unsigned long flags; @@ -232,10 +232,10 @@ static const struct asm9260_gate_data asm9260_ahb_gates[] __initconst = { HW_AHBCLKCTRL1, 16 }, }; -static const char __initdata *main_mux_p[] = { NULL, NULL }; -static const char __initdata *i2s0_mux_p[] = { NULL, NULL, "i2s0m_div"}; -static const char __initdata *i2s1_mux_p[] = { NULL, NULL, "i2s1m_div"}; -static const char __initdata *clkout_mux_p[] = { NULL, NULL, "rtc"}; +static struct clk_parent_data __initdata main_mux_p[] = { { .index = 0, }, { .name = "pll" } }; +static struct clk_parent_data __initdata i2s0_mux_p[] = { { .index = 0, }, { .name = "pll" }, { .name = "i2s0m_div"} }; +static struct clk_parent_data __initdata i2s1_mux_p[] = { { .index = 0, }, { .name = "pll" }, { .name = "i2s1m_div"} }; +static struct clk_parent_data __initdata clkout_mux_p[] = { { .index = 0, }, { .name = "pll" }, { .name = "rtc"} }; static u32 three_mux_table[] = {0, 1, 3}; static struct asm9260_mux_clock asm9260_mux_clks[] __initdata = { @@ -255,9 +255,10 @@ static struct asm9260_mux_clock asm9260_mux_clks[] __initdata = { static void __init asm9260_acc_init(struct device_node *np) { - struct clk_hw *hw; + struct clk_hw *hw, *pll_hw; struct clk_hw **hws; - const char *ref_clk, *pll_clk = "pll"; + const char *pll_clk = "pll"; + struct clk_parent_data pll_parent_data = { .index = 0 }; u32 rate; int n; @@ -274,21 +275,15 @@ static void __init asm9260_acc_init(struct device_node *np) /* register pll */ rate = (ioread32(base + HW_SYSPLLCTRL) & 0xffff) * 1000000; - /* TODO: Convert to DT parent scheme */ - ref_clk = of_clk_get_parent_name(np, 0); - hw = __clk_hw_register_fixed_rate(NULL, NULL, pll_clk, - ref_clk, NULL, NULL, 0, rate, 0, - CLK_FIXED_RATE_PARENT_ACCURACY); - - if (IS_ERR(hw)) + pll_hw = clk_hw_register_fixed_rate_parent_accuracy(NULL, pll_clk, &pll_parent_data, + 0, rate); + if (IS_ERR(pll_hw)) panic("%pOFn: can't register REFCLK. Check DT!", np); for (n = 0; n < ARRAY_SIZE(asm9260_mux_clks); n++) { const struct asm9260_mux_clock *mc = &asm9260_mux_clks[n]; - mc->parent_names[0] = ref_clk; - mc->parent_names[1] = pll_clk; - hw = clk_hw_register_mux_table(NULL, mc->name, mc->parent_names, + hw = clk_hw_register_mux_table_parent_data(NULL, mc->name, mc->parent_data, mc->num_parents, mc->flags, base + mc->offset, 0, mc->mask, 0, mc->table, &asm9260_clk_lock); } diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c index 24dab2312bc6faf572e7a7b63c187de602e6df18..9c3305bcb27ae0810a677ac9bb693bd04e53e6c4 100644 --- a/drivers/clk/clk-ast2600.c +++ b/drivers/clk/clk-ast2600.c @@ -622,7 +622,7 @@ static int aspeed_g6_clk_probe(struct platform_device *pdev) regmap_write(map, 0x308, 0x12000); /* 3x3 = 9 */ /* P-Bus (BCLK) clock divider */ - hw = clk_hw_register_divider_table(dev, "bclk", "hpll", 0, + hw = clk_hw_register_divider_table(dev, "bclk", "epll", 0, scu_g6_base + ASPEED_G6_CLK_SELECTION1, 20, 3, 0, ast2600_div_table, &aspeed_g6_clk_lock); diff --git a/drivers/clk/clk-cdce706.c b/drivers/clk/clk-cdce706.c index 5467d941ddfd0b97decc993f5127b30cf68e46a7..1449d0537674eba490687ceb497b9d81c7bf28e0 100644 --- a/drivers/clk/clk-cdce706.c +++ b/drivers/clk/clk-cdce706.c @@ -665,10 +665,9 @@ static int cdce706_probe(struct i2c_client *client) cdce); } -static int cdce706_remove(struct i2c_client *client) +static void cdce706_remove(struct i2c_client *client) { of_clk_del_provider(client->dev.of_node); - return 0; } diff --git a/drivers/clk/clk-composite.c b/drivers/clk/clk-composite.c index b9c5f904f53567c68db3aed1d955449596f25cae..edfa94641bbfed8f0a9d4e575f73c82b21598c2b 100644 --- a/drivers/clk/clk-composite.c +++ b/drivers/clk/clk-composite.c @@ -85,10 +85,11 @@ static int clk_composite_determine_rate(struct clk_hw *hw, req->best_parent_hw = NULL; if (clk_hw_get_flags(hw) & CLK_SET_RATE_NO_REPARENT) { - struct clk_rate_request tmp_req = *req; + struct clk_rate_request tmp_req; parent = clk_hw_get_parent(mux_hw); + clk_hw_forward_rate_request(hw, req, parent, &tmp_req, req->rate); ret = clk_composite_determine_rate_for_parent(rate_hw, &tmp_req, parent, @@ -104,12 +105,13 @@ static int clk_composite_determine_rate(struct clk_hw *hw, } for (i = 0; i < clk_hw_get_num_parents(mux_hw); i++) { - struct clk_rate_request tmp_req = *req; + struct clk_rate_request tmp_req; parent = clk_hw_get_parent_by_index(mux_hw, i); if (!parent) continue; + clk_hw_forward_rate_request(hw, req, parent, &tmp_req, req->rate); ret = clk_composite_determine_rate_for_parent(rate_hw, &tmp_req, parent, diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c index aa5c72bab83ea5689f115c0e641997d6a22890c7..320d39922206ad470071ab9267b7025a3fb3e735 100644 --- a/drivers/clk/clk-cs2000-cp.c +++ b/drivers/clk/clk-cs2000-cp.c @@ -557,7 +557,7 @@ static int cs2000_version_print(struct cs2000_priv *priv) return 0; } -static int cs2000_remove(struct i2c_client *client) +static void cs2000_remove(struct i2c_client *client) { struct cs2000_priv *priv = i2c_get_clientdata(client); struct device *dev = priv_to_dev(priv); @@ -566,8 +566,6 @@ static int cs2000_remove(struct i2c_client *client) of_clk_del_provider(np); clk_hw_unregister(&priv->hw); - - return 0; } static int cs2000_probe(struct i2c_client *client) diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c index f6b2bf5584867e115c0fb39185b41d0607087b22..a2c2b5203b0a95cf1cf651b1fb4e2c24d230d831 100644 --- a/drivers/clk/clk-divider.c +++ b/drivers/clk/clk-divider.c @@ -386,13 +386,13 @@ long divider_round_rate_parent(struct clk_hw *hw, struct clk_hw *parent, const struct clk_div_table *table, u8 width, unsigned long flags) { - struct clk_rate_request req = { - .rate = rate, - .best_parent_rate = *prate, - .best_parent_hw = parent, - }; + struct clk_rate_request req; int ret; + clk_hw_init_rate_request(hw, &req, rate); + req.best_parent_rate = *prate; + req.best_parent_hw = parent; + ret = divider_determine_rate(hw, &req, table, width, flags); if (ret) return ret; @@ -408,13 +408,13 @@ long divider_ro_round_rate_parent(struct clk_hw *hw, struct clk_hw *parent, const struct clk_div_table *table, u8 width, unsigned long flags, unsigned int val) { - struct clk_rate_request req = { - .rate = rate, - .best_parent_rate = *prate, - .best_parent_hw = parent, - }; + struct clk_rate_request req; int ret; + clk_hw_init_rate_request(hw, &req, rate); + req.best_parent_rate = *prate; + req.best_parent_hw = parent; + ret = divider_ro_determine_rate(hw, &req, table, width, flags, val); if (ret) return ret; diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c index ac68a6b40f0e5d0d0b238844a2fd1d6cae88353c..7d775954e26da0e53c66c27376eb978cb8710a85 100644 --- a/drivers/clk/clk-fixed-rate.c +++ b/drivers/clk/clk-fixed-rate.c @@ -49,12 +49,24 @@ const struct clk_ops clk_fixed_rate_ops = { }; EXPORT_SYMBOL_GPL(clk_fixed_rate_ops); +static void devm_clk_hw_register_fixed_rate_release(struct device *dev, void *res) +{ + struct clk_fixed_rate *fix = res; + + /* + * We can not use clk_hw_unregister_fixed_rate, since it will kfree() + * the hw, resulting in double free. Just unregister the hw and let + * devres code kfree() it. + */ + clk_hw_unregister(&fix->hw); +} + struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, struct device_node *np, const char *name, const char *parent_name, const struct clk_hw *parent_hw, const struct clk_parent_data *parent_data, unsigned long flags, unsigned long fixed_rate, unsigned long fixed_accuracy, - unsigned long clk_fixed_flags) + unsigned long clk_fixed_flags, bool devm) { struct clk_fixed_rate *fixed; struct clk_hw *hw; @@ -62,7 +74,11 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, int ret = -EINVAL; /* allocate fixed-rate clock */ - fixed = kzalloc(sizeof(*fixed), GFP_KERNEL); + if (devm) + fixed = devres_alloc(devm_clk_hw_register_fixed_rate_release, + sizeof(*fixed), GFP_KERNEL); + else + fixed = kzalloc(sizeof(*fixed), GFP_KERNEL); if (!fixed) return ERR_PTR(-ENOMEM); @@ -90,9 +106,13 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, else ret = of_clk_hw_register(np, hw); if (ret) { - kfree(fixed); + if (devm) + devres_free(fixed); + else + kfree(fixed); hw = ERR_PTR(ret); - } + } else if (devm) + devres_add(dev, fixed); return hw; } diff --git a/drivers/clk/clk-lan966x.c b/drivers/clk/clk-lan966x.c index 81cb90955d68b9d572b39b1fe89bd435c8517bb6..460e7216bfa1ee19a32f448a3c4d0431dcfba30d 100644 --- a/drivers/clk/clk-lan966x.c +++ b/drivers/clk/clk-lan966x.c @@ -286,7 +286,7 @@ static struct platform_driver lan966x_clk_driver = { .of_match_table = lan966x_clk_dt_ids, }, }; -builtin_platform_driver(lan966x_clk_driver); +module_platform_driver(lan966x_clk_driver); MODULE_AUTHOR("Kavyasree Kotagiri "); MODULE_DESCRIPTION("LAN966X clock driver"); diff --git a/drivers/clk/clk-lochnagar.c b/drivers/clk/clk-lochnagar.c index 565bcd0cdde9e907003fc2281bb52f689cbd12b3..80944bf482e9a85e0957b6ccca600677926d44f6 100644 --- a/drivers/clk/clk-lochnagar.c +++ b/drivers/clk/clk-lochnagar.c @@ -19,7 +19,7 @@ #include #include -#include +#include #define LOCHNAGAR_NUM_CLOCKS (LOCHNAGAR_SPDIF_CLKOUT + 1) diff --git a/drivers/clk/clk-nomadik.c b/drivers/clk/clk-nomadik.c index bad2677e11aeb6e87ce1c04a6fa4b738550153de..71fbe687fa7b697d9aee68132a3bf412940905d6 100644 --- a/drivers/clk/clk-nomadik.c +++ b/drivers/clk/clk-nomadik.c @@ -99,7 +99,7 @@ static void __init nomadik_src_init(void) if (!src_base) { pr_err("%s: must have src parent node with REGS (%pOFn)\n", __func__, np); - return; + goto out_put; } /* Set all timers to use the 2.4 MHz TIMCLK */ @@ -132,6 +132,9 @@ static void __init nomadik_src_init(void) } writel(val, src_base + SRC_XTALCR); register_reboot_notifier(&nomadik_clk_reboot_notifier); + +out_put: + of_node_put(np); } /** diff --git a/drivers/clk/clk-npcm7xx.c b/drivers/clk/clk-npcm7xx.c index e677bb5a784b96ac2bb71c90e2a365e0d316f46e..e319cfa51a8a3ff99a742144c98f3b916fcad83c 100644 --- a/drivers/clk/clk-npcm7xx.c +++ b/drivers/clk/clk-npcm7xx.c @@ -129,20 +129,6 @@ npcm7xx_clk_register_pll(void __iomem *pllcon, const char *name, #define NPCM7XX_SECCNT (0x68) #define NPCM7XX_CNTR25M (0x6C) -struct npcm7xx_clk_gate_data { - u32 reg; - u8 bit_idx; - const char *name; - const char *parent_name; - unsigned long flags; - /* - * If this clock is exported via DT, set onecell_idx to constant - * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for - * this specific clock. Otherwise, set to -1. - */ - int onecell_idx; -}; - struct npcm7xx_clk_mux_data { u8 shift; u8 mask; @@ -160,21 +146,6 @@ struct npcm7xx_clk_mux_data { }; -struct npcm7xx_clk_div_fixed_data { - u8 mult; - u8 div; - const char *name; - const char *parent_name; - u8 clk_divider_flags; - /* - * If this clock is exported via DT, set onecell_idx to constant - * defined in include/dt-bindings/clock/nuvoton, NPCM7XX-clock.h for - * this specific clock. Otherwise, set to -1. - */ - int onecell_idx; -}; - - struct npcm7xx_clk_div_data { u32 reg; u8 shift; diff --git a/drivers/clk/clk-oxnas.c b/drivers/clk/clk-oxnas.c index cda5e258355bc68419c117c1934df5ac7652be32..584e293156ad6041d46c3e29782da6b0fe11fea7 100644 --- a/drivers/clk/clk-oxnas.c +++ b/drivers/clk/clk-oxnas.c @@ -207,7 +207,7 @@ static const struct of_device_id oxnas_stdclk_dt_ids[] = { static int oxnas_stdclk_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; + struct device_node *np = pdev->dev.of_node, *parent_np; const struct oxnas_stdclk_data *data; struct regmap *regmap; int ret; @@ -215,7 +215,9 @@ static int oxnas_stdclk_probe(struct platform_device *pdev) data = of_device_get_match_data(&pdev->dev); - regmap = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + regmap = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap)) { dev_err(&pdev->dev, "failed to have parent regmap\n"); return PTR_ERR(regmap); diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index 88898b97a443122193c105526b236b7c8507d49f..5eddb9f0d6bdb3993b3319e30baddfc12165eb15 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -1063,8 +1063,13 @@ static void __init _clockgen_init(struct device_node *np, bool legacy); */ static void __init legacy_init_clockgen(struct device_node *np) { - if (!clockgen.node) - _clockgen_init(of_get_parent(np), true); + if (!clockgen.node) { + struct device_node *parent_np; + + parent_np = of_get_parent(np); + _clockgen_init(parent_np, true); + of_node_put(parent_np); + } } /* Legacy node */ @@ -1159,6 +1164,7 @@ static struct clk * __init create_sysclk(const char *name) sysclk = of_get_child_by_name(clockgen.node, "sysclk"); if (sysclk) { clk = sysclk_from_fixed(sysclk, name); + of_node_put(sysclk); if (!IS_ERR(clk)) return clk; } diff --git a/drivers/clk/clk-si514.c b/drivers/clk/clk-si514.c index 4481c4303534f49960f5b9cedbafd9bb7c1d4d1d..c028fa103bedabdd47891d6e5ab8eb11375ebf38 100644 --- a/drivers/clk/clk-si514.c +++ b/drivers/clk/clk-si514.c @@ -370,10 +370,9 @@ static int si514_probe(struct i2c_client *client) return 0; } -static int si514_remove(struct i2c_client *client) +static void si514_remove(struct i2c_client *client) { of_clk_del_provider(client->dev.of_node); - return 0; } static const struct i2c_device_id si514_id[] = { diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c index 4bca73212662876224c6d57932e8558a431279ae..0e528d7ba656ee82a49ed352fdf4bde2086c9a3c 100644 --- a/drivers/clk/clk-si5341.c +++ b/drivers/clk/clk-si5341.c @@ -1796,7 +1796,7 @@ cleanup: return err; } -static int si5341_remove(struct i2c_client *client) +static void si5341_remove(struct i2c_client *client) { struct clk_si5341 *data = i2c_get_clientdata(client); int i; @@ -1807,8 +1807,6 @@ static int si5341_remove(struct i2c_client *client) if (data->clk[i].vddo_reg) regulator_disable(data->clk[i].vddo_reg); } - - return 0; } static const struct i2c_device_id si5341_id[] = { diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c index b9f088c4ba2f54f458307979d0ef57f74f618e0b..9e939c98a4558535ee52a508c725e985516411bc 100644 --- a/drivers/clk/clk-si5351.c +++ b/drivers/clk/clk-si5351.c @@ -1651,11 +1651,9 @@ static int si5351_i2c_probe(struct i2c_client *client) return 0; } -static int si5351_i2c_remove(struct i2c_client *client) +static void si5351_i2c_remove(struct i2c_client *client) { of_clk_del_provider(client->dev.of_node); - - return 0; } static struct i2c_driver si5351_driver = { diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c index 1ff8f32f734dcd959d534dc6f79cdd2cb050058f..0a6d70c497267872ee843d7bc3e399d5491acb86 100644 --- a/drivers/clk/clk-si570.c +++ b/drivers/clk/clk-si570.c @@ -498,10 +498,9 @@ static int si570_probe(struct i2c_client *client) return 0; } -static int si570_remove(struct i2c_client *client) +static void si570_remove(struct i2c_client *client) { of_clk_del_provider(client->dev.of_node); - return 0; } static const struct of_device_id clk_si570_of_match[] = { diff --git a/drivers/clk/clk-tps68470.c b/drivers/clk/clk-tps68470.c index e5fbefd6ac2dd8a5ee022dcbd48e64a1002e3b8d..38f44b5b9b1b80bf5404bcafda9806a5f377ed95 100644 --- a/drivers/clk/clk-tps68470.c +++ b/drivers/clk/clk-tps68470.c @@ -200,7 +200,9 @@ static int tps68470_clk_probe(struct platform_device *pdev) .flags = CLK_SET_RATE_GATE, }; struct tps68470_clkdata *tps68470_clkdata; + struct tps68470_clk_consumer *consumer; int ret; + int i; tps68470_clkdata = devm_kzalloc(&pdev->dev, sizeof(*tps68470_clkdata), GFP_KERNEL); @@ -223,10 +225,13 @@ static int tps68470_clk_probe(struct platform_device *pdev) return ret; if (pdata) { - ret = devm_clk_hw_register_clkdev(&pdev->dev, - &tps68470_clkdata->clkout_hw, - pdata->consumer_con_id, - pdata->consumer_dev_name); + for (i = 0; i < pdata->n_consumers; i++) { + consumer = &pdata->consumers[i]; + ret = devm_clk_hw_register_clkdev(&pdev->dev, + &tps68470_clkdata->clkout_hw, + consumer->consumer_con_id, + consumer->consumer_dev_name); + } } return ret; diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c index e7be3e54b9be43d2e11e56df4929aaf3843bcab7..88689415aff9c34e3cb1d1b71f1b6f400d81b066 100644 --- a/drivers/clk/clk-versaclock5.c +++ b/drivers/clk/clk-versaclock5.c @@ -24,7 +24,7 @@ #include #include -#include +#include /* VersaClock5 registers */ #define VC5_OTP_CONTROL 0x00 @@ -153,6 +153,7 @@ enum vc5_model { IDT_VC5_5P49V5935, IDT_VC6_5P49V6901, IDT_VC6_5P49V6965, + IDT_VC6_5P49V6975, }; /* Structure to describe features of a particular VC5 model */ @@ -230,8 +231,12 @@ static unsigned char vc5_mux_get_parent(struct clk_hw *hw) container_of(hw, struct vc5_driver_data, clk_mux); const u8 mask = VC5_PRIM_SRC_SHDN_EN_XTAL | VC5_PRIM_SRC_SHDN_EN_CLKIN; unsigned int src; + int ret; + + ret = regmap_read(vc5->regmap, VC5_PRIM_SRC_SHDN, &src); + if (ret) + return 0; - regmap_read(vc5->regmap, VC5_PRIM_SRC_SHDN, &src); src &= mask; if (src == VC5_PRIM_SRC_SHDN_EN_XTAL) @@ -286,8 +291,12 @@ static unsigned long vc5_dbl_recalc_rate(struct clk_hw *hw, struct vc5_driver_data *vc5 = container_of(hw, struct vc5_driver_data, clk_mul); unsigned int premul; + int ret; + + ret = regmap_read(vc5->regmap, VC5_PRIM_SRC_SHDN, &premul); + if (ret) + return 0; - regmap_read(vc5->regmap, VC5_PRIM_SRC_SHDN, &premul); if (premul & VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ) parent_rate *= 2; @@ -315,11 +324,9 @@ static int vc5_dbl_set_rate(struct clk_hw *hw, unsigned long rate, else mask = 0; - regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN, - VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ, - mask); - - return 0; + return regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN, + VC5_PRIM_SRC_SHDN_EN_DOUBLE_XTAL_FREQ, + mask); } static const struct clk_ops vc5_dbl_ops = { @@ -334,14 +341,19 @@ static unsigned long vc5_pfd_recalc_rate(struct clk_hw *hw, struct vc5_driver_data *vc5 = container_of(hw, struct vc5_driver_data, clk_pfd); unsigned int prediv, div; + int ret; - regmap_read(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, &prediv); + ret = regmap_read(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, &prediv); + if (ret) + return 0; /* The bypass_prediv is set, PLL fed from Ref_in directly. */ if (prediv & VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV) return parent_rate; - regmap_read(vc5->regmap, VC5_REF_DIVIDER, &div); + ret = regmap_read(vc5->regmap, VC5_REF_DIVIDER, &div); + if (ret) + return 0; /* The Sel_prediv2 is set, PLL fed from prediv2 (Ref_in / 2) */ if (div & VC5_REF_DIVIDER_SEL_PREDIV2) @@ -376,15 +388,17 @@ static int vc5_pfd_set_rate(struct clk_hw *hw, unsigned long rate, struct vc5_driver_data *vc5 = container_of(hw, struct vc5_driver_data, clk_pfd); unsigned long idiv; + int ret; u8 div; /* CLKIN within range of PLL input, feed directly to PLL. */ if (parent_rate <= 50000000) { - regmap_update_bits(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, - VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV, - VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV); - regmap_update_bits(vc5->regmap, VC5_REF_DIVIDER, 0xff, 0x00); - return 0; + ret = regmap_set_bits(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, + VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV); + if (ret) + return ret; + + return regmap_update_bits(vc5->regmap, VC5_REF_DIVIDER, 0xff, 0x00); } idiv = DIV_ROUND_UP(parent_rate, rate); @@ -395,11 +409,12 @@ static int vc5_pfd_set_rate(struct clk_hw *hw, unsigned long rate, else div = VC5_REF_DIVIDER_REF_DIV(idiv); - regmap_update_bits(vc5->regmap, VC5_REF_DIVIDER, 0xff, div); - regmap_update_bits(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, - VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV, 0); + ret = regmap_update_bits(vc5->regmap, VC5_REF_DIVIDER, 0xff, div); + if (ret) + return ret; - return 0; + return regmap_clear_bits(vc5->regmap, VC5_VCO_CTRL_AND_PREDIV, + VC5_VCO_CTRL_AND_PREDIV_BYPASS_PREDIV); } static const struct clk_ops vc5_pfd_ops = { @@ -551,9 +566,12 @@ static int vc5_fod_set_rate(struct clk_hw *hw, unsigned long rate, hwdata->div_int >> 4, hwdata->div_int << 4, 0 }; + int ret; - regmap_bulk_write(vc5->regmap, VC5_OUT_DIV_FRAC(hwdata->num, 0), - data, 14); + ret = regmap_bulk_write(vc5->regmap, VC5_OUT_DIV_FRAC(hwdata->num, 0), + data, 14); + if (ret) + return ret; /* * Toggle magic bit in undocumented register for unknown reason. @@ -561,12 +579,13 @@ static int vc5_fod_set_rate(struct clk_hw *hw, unsigned long rate, * datasheet somewhat implies this is needed, but the register * and the bit is not documented. */ - regmap_update_bits(vc5->regmap, VC5_GLOBAL_REGISTER, - VC5_GLOBAL_REGISTER_GLOBAL_RESET, 0); - regmap_update_bits(vc5->regmap, VC5_GLOBAL_REGISTER, - VC5_GLOBAL_REGISTER_GLOBAL_RESET, - VC5_GLOBAL_REGISTER_GLOBAL_RESET); - return 0; + ret = regmap_clear_bits(vc5->regmap, VC5_GLOBAL_REGISTER, + VC5_GLOBAL_REGISTER_GLOBAL_RESET); + if (ret) + return ret; + + return regmap_set_bits(vc5->regmap, VC5_GLOBAL_REGISTER, + VC5_GLOBAL_REGISTER_GLOBAL_RESET); } static const struct clk_ops vc5_fod_ops = { @@ -594,10 +613,9 @@ static int vc5_clk_out_prepare(struct clk_hw *hw) * registers. */ if (vc5->chip_info->flags & VC5_HAS_BYPASS_SYNC_BIT) { - ret = regmap_update_bits(vc5->regmap, - VC5_RESERVED_X0(hwdata->num), - VC5_RESERVED_X0_BYPASS_SYNC, - VC5_RESERVED_X0_BYPASS_SYNC); + ret = regmap_set_bits(vc5->regmap, + VC5_RESERVED_X0(hwdata->num), + VC5_RESERVED_X0_BYPASS_SYNC); if (ret) return ret; } @@ -606,7 +624,10 @@ static int vc5_clk_out_prepare(struct clk_hw *hw) * If the input mux is disabled, enable it first and * select source from matching FOD. */ - regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src); + ret = regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src); + if (ret) + return ret; + if ((src & mask) == 0) { src = VC5_OUT_DIV_CONTROL_RESET | VC5_OUT_DIV_CONTROL_EN_FOD; ret = regmap_update_bits(vc5->regmap, @@ -617,18 +638,22 @@ static int vc5_clk_out_prepare(struct clk_hw *hw) } /* Enable the clock buffer */ - regmap_update_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1), - VC5_CLK_OUTPUT_CFG1_EN_CLKBUF, - VC5_CLK_OUTPUT_CFG1_EN_CLKBUF); + ret = regmap_set_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1), + VC5_CLK_OUTPUT_CFG1_EN_CLKBUF); + if (ret) + return ret; + if (hwdata->clk_output_cfg0_mask) { dev_dbg(&vc5->client->dev, "Update output %d mask 0x%0X val 0x%0X\n", hwdata->num, hwdata->clk_output_cfg0_mask, hwdata->clk_output_cfg0); - regmap_update_bits(vc5->regmap, - VC5_CLK_OUTPUT_CFG(hwdata->num, 0), - hwdata->clk_output_cfg0_mask, - hwdata->clk_output_cfg0); + ret = regmap_update_bits(vc5->regmap, + VC5_CLK_OUTPUT_CFG(hwdata->num, 0), + hwdata->clk_output_cfg0_mask, + hwdata->clk_output_cfg0); + if (ret) + return ret; } return 0; @@ -640,8 +665,8 @@ static void vc5_clk_out_unprepare(struct clk_hw *hw) struct vc5_driver_data *vc5 = hwdata->vc5; /* Disable the clock buffer */ - regmap_update_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1), - VC5_CLK_OUTPUT_CFG1_EN_CLKBUF, 0); + regmap_clear_bits(vc5->regmap, VC5_CLK_OUTPUT_CFG(hwdata->num, 1), + VC5_CLK_OUTPUT_CFG1_EN_CLKBUF); } static unsigned char vc5_clk_out_get_parent(struct clk_hw *hw) @@ -656,8 +681,12 @@ static unsigned char vc5_clk_out_get_parent(struct clk_hw *hw) const u8 extclk = VC5_OUT_DIV_CONTROL_SELB_NORM | VC5_OUT_DIV_CONTROL_SEL_EXT; unsigned int src; + int ret; + + ret = regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src); + if (ret) + return 0; - regmap_read(vc5->regmap, VC5_OUT_DIV_CONTROL(hwdata->num), &src); src &= mask; if (src == 0) /* Input mux set to DISABLED */ @@ -725,6 +754,7 @@ static int vc5_map_index_to_output(const enum vc5_model model, case IDT_VC5_5P49V5935: case IDT_VC6_5P49V6901: case IDT_VC6_5P49V6965: + case IDT_VC6_5P49V6975: default: return n; } @@ -819,22 +849,27 @@ static int vc5_update_cap_load(struct device_node *node, struct vc5_driver_data { u32 value; int mapped_value; + int ret; - if (!of_property_read_u32(node, "idt,xtal-load-femtofarads", &value)) { - mapped_value = vc5_map_cap_value(value); - if (mapped_value < 0) - return mapped_value; - - /* - * The mapped_value is really the high 6 bits of - * VC5_XTAL_X1_LOAD_CAP and VC5_XTAL_X2_LOAD_CAP, so - * shift the value 2 places. - */ - regmap_update_bits(vc5->regmap, VC5_XTAL_X1_LOAD_CAP, ~0x03, mapped_value << 2); - regmap_update_bits(vc5->regmap, VC5_XTAL_X2_LOAD_CAP, ~0x03, mapped_value << 2); - } + if (of_property_read_u32(node, "idt,xtal-load-femtofarads", &value)) + return 0; - return 0; + mapped_value = vc5_map_cap_value(value); + if (mapped_value < 0) + return mapped_value; + + /* + * The mapped_value is really the high 6 bits of + * VC5_XTAL_X1_LOAD_CAP and VC5_XTAL_X2_LOAD_CAP, so + * shift the value 2 places. + */ + ret = regmap_update_bits(vc5->regmap, VC5_XTAL_X1_LOAD_CAP, ~0x03, + mapped_value << 2); + if (ret) + return ret; + + return regmap_update_bits(vc5->regmap, VC5_XTAL_X2_LOAD_CAP, ~0x03, + mapped_value << 2); } static int vc5_update_slew(struct device_node *np_output, @@ -956,7 +991,10 @@ static int vc5_probe(struct i2c_client *client) "could not read idt,output-enable-active\n"); } - regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN, src_mask, src_val); + ret = regmap_update_bits(vc5->regmap, VC5_PRIM_SRC_SHDN, src_mask, + src_val); + if (ret) + return ret; /* Register clock input mux */ memset(&init, 0, sizeof(init)); @@ -1138,7 +1176,7 @@ err_clk: return ret; } -static int vc5_remove(struct i2c_client *client) +static void vc5_remove(struct i2c_client *client) { struct vc5_driver_data *vc5 = i2c_get_clientdata(client); @@ -1146,8 +1184,6 @@ static int vc5_remove(struct i2c_client *client) if (vc5->chip_info->flags & VC5_HAS_INTERNAL_XTAL) clk_unregister_fixed_rate(vc5->pin_xin); - - return 0; } static int __maybe_unused vc5_suspend(struct device *dev) @@ -1204,7 +1240,7 @@ static const struct vc5_chip_info idt_5p49v6901_info = { .model = IDT_VC6_5P49V6901, .clk_fod_cnt = 4, .clk_out_cnt = 5, - .flags = VC5_HAS_PFD_FREQ_DBL, + .flags = VC5_HAS_PFD_FREQ_DBL | VC5_HAS_BYPASS_SYNC_BIT, }; static const struct vc5_chip_info idt_5p49v6965_info = { @@ -1214,6 +1250,13 @@ static const struct vc5_chip_info idt_5p49v6965_info = { .flags = VC5_HAS_BYPASS_SYNC_BIT, }; +static const struct vc5_chip_info idt_5p49v6975_info = { + .model = IDT_VC6_5P49V6975, + .clk_fod_cnt = 4, + .clk_out_cnt = 5, + .flags = VC5_HAS_BYPASS_SYNC_BIT | VC5_HAS_INTERNAL_XTAL, +}; + static const struct i2c_device_id vc5_id[] = { { "5p49v5923", .driver_data = IDT_VC5_5P49V5923 }, { "5p49v5925", .driver_data = IDT_VC5_5P49V5925 }, @@ -1221,6 +1264,7 @@ static const struct i2c_device_id vc5_id[] = { { "5p49v5935", .driver_data = IDT_VC5_5P49V5935 }, { "5p49v6901", .driver_data = IDT_VC6_5P49V6901 }, { "5p49v6965", .driver_data = IDT_VC6_5P49V6965 }, + { "5p49v6975", .driver_data = IDT_VC6_5P49V6975 }, { } }; MODULE_DEVICE_TABLE(i2c, vc5_id); @@ -1232,6 +1276,7 @@ static const struct of_device_id clk_vc5_of_match[] = { { .compatible = "idt,5p49v5935", .data = &idt_5p49v5935_info }, { .compatible = "idt,5p49v6901", .data = &idt_5p49v6901_info }, { .compatible = "idt,5p49v6965", .data = &idt_5p49v6965_info }, + { .compatible = "idt,5p49v6975", .data = &idt_5p49v6975_info }, { }, }; MODULE_DEVICE_TABLE(of, clk_vc5_of_match); diff --git a/drivers/clk/clk-versaclock7.c b/drivers/clk/clk-versaclock7.c new file mode 100644 index 0000000000000000000000000000000000000000..8e4f86e852aa0963421db2c5a4e028cb9975aefa --- /dev/null +++ b/drivers/clk/clk-versaclock7.c @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common clock framework driver for the Versaclock7 family of timing devices. + * + * Copyright (c) 2022 Renesas Electronics Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * 16-bit register address: the lower 8 bits of the register address come + * from the offset addr byte and the upper 8 bits come from the page register. + */ +#define VC7_PAGE_ADDR 0xFD +#define VC7_PAGE_WINDOW 256 +#define VC7_MAX_REG 0x364 + +/* Maximum number of banks supported by VC7 */ +#define VC7_NUM_BANKS 7 + +/* Maximum number of FODs supported by VC7 */ +#define VC7_NUM_FOD 3 + +/* Maximum number of IODs supported by VC7 */ +#define VC7_NUM_IOD 4 + +/* Maximum number of outputs supported by VC7 */ +#define VC7_NUM_OUT 12 + +/* VCO valid range is 9.5 GHz to 10.7 GHz */ +#define VC7_APLL_VCO_MIN 9500000000UL +#define VC7_APLL_VCO_MAX 10700000000UL + +/* APLL denominator is fixed at 2^27 */ +#define VC7_APLL_DENOMINATOR_BITS 27 + +/* FOD 1st stage denominator is fixed 2^34 */ +#define VC7_FOD_DENOMINATOR_BITS 34 + +/* IOD can operate between 1kHz and 650MHz */ +#define VC7_IOD_RATE_MIN 1000UL +#define VC7_IOD_RATE_MAX 650000000UL +#define VC7_IOD_MIN_DIVISOR 14 +#define VC7_IOD_MAX_DIVISOR 0x1ffffff /* 25-bit */ + +#define VC7_FOD_RATE_MIN 1000UL +#define VC7_FOD_RATE_MAX 650000000UL +#define VC7_FOD_1ST_STAGE_RATE_MIN 33000000UL /* 33 MHz */ +#define VC7_FOD_1ST_STAGE_RATE_MAX 650000000UL /* 650 MHz */ +#define VC7_FOD_1ST_INT_MAX 324 +#define VC7_FOD_2ND_INT_MIN 2 +#define VC7_FOD_2ND_INT_MAX 0x1ffff /* 17-bit */ + +/* VC7 Registers */ + +#define VC7_REG_XO_CNFG 0x2C +#define VC7_REG_XO_CNFG_COUNT 4 +#define VC7_REG_XO_IB_H_DIV_SHIFT 24 +#define VC7_REG_XO_IB_H_DIV_MASK GENMASK(28, VC7_REG_XO_IB_H_DIV_SHIFT) + +#define VC7_REG_APLL_FB_DIV_FRAC 0x120 +#define VC7_REG_APLL_FB_DIV_FRAC_COUNT 4 +#define VC7_REG_APLL_FB_DIV_FRAC_MASK GENMASK(26, 0) + +#define VC7_REG_APLL_FB_DIV_INT 0x124 +#define VC7_REG_APLL_FB_DIV_INT_COUNT 2 +#define VC7_REG_APLL_FB_DIV_INT_MASK GENMASK(9, 0) + +#define VC7_REG_APLL_CNFG 0x127 +#define VC7_REG_APLL_EN_DOUBLER BIT(0) + +#define VC7_REG_OUT_BANK_CNFG(idx) (0x280 + (0x4 * (idx))) +#define VC7_REG_OUTPUT_BANK_SRC_MASK GENMASK(2, 0) + +#define VC7_REG_FOD_INT_CNFG(idx) (0x1E0 + (0x10 * (idx))) +#define VC7_REG_FOD_INT_CNFG_COUNT 8 +#define VC7_REG_FOD_1ST_INT_MASK GENMASK(8, 0) +#define VC7_REG_FOD_2ND_INT_SHIFT 9 +#define VC7_REG_FOD_2ND_INT_MASK GENMASK(25, VC7_REG_FOD_2ND_INT_SHIFT) +#define VC7_REG_FOD_FRAC_SHIFT 26 +#define VC7_REG_FOD_FRAC_MASK GENMASK_ULL(59, VC7_REG_FOD_FRAC_SHIFT) + +#define VC7_REG_IOD_INT_CNFG(idx) (0x1C0 + (0x8 * (idx))) +#define VC7_REG_IOD_INT_CNFG_COUNT 4 +#define VC7_REG_IOD_INT_MASK GENMASK(24, 0) + +#define VC7_REG_ODRV_EN(idx) (0x240 + (0x4 * (idx))) +#define VC7_REG_OUT_DIS BIT(0) + +struct vc7_driver_data; +static const struct regmap_config vc7_regmap_config; + +/* Supported Renesas VC7 models */ +enum vc7_model { + VC7_RC21008A, +}; + +struct vc7_chip_info { + const enum vc7_model model; + const unsigned int banks[VC7_NUM_BANKS]; + const unsigned int num_banks; + const unsigned int outputs[VC7_NUM_OUT]; + const unsigned int num_outputs; +}; + +/* + * Changing the APLL frequency is currently not supported. + * The APLL will consist of an opaque block between the XO and FOD/IODs and + * its frequency will be computed based on the current state of the device. + */ +struct vc7_apll_data { + struct clk *clk; + struct vc7_driver_data *vc7; + u8 xo_ib_h_div; + u8 en_doubler; + u16 apll_fb_div_int; + u32 apll_fb_div_frac; +}; + +struct vc7_fod_data { + struct clk_hw hw; + struct vc7_driver_data *vc7; + unsigned int num; + u32 fod_1st_int; + u32 fod_2nd_int; + u64 fod_frac; +}; + +struct vc7_iod_data { + struct clk_hw hw; + struct vc7_driver_data *vc7; + unsigned int num; + u32 iod_int; +}; + +struct vc7_out_data { + struct clk_hw hw; + struct vc7_driver_data *vc7; + unsigned int num; + unsigned int out_dis; +}; + +struct vc7_driver_data { + struct i2c_client *client; + struct regmap *regmap; + const struct vc7_chip_info *chip_info; + + struct clk *pin_xin; + struct vc7_apll_data clk_apll; + struct vc7_fod_data clk_fod[VC7_NUM_FOD]; + struct vc7_iod_data clk_iod[VC7_NUM_IOD]; + struct vc7_out_data clk_out[VC7_NUM_OUT]; +}; + +struct vc7_bank_src_map { + enum vc7_bank_src_type { + VC7_FOD, + VC7_IOD, + } type; + union _divider { + struct vc7_iod_data *iod; + struct vc7_fod_data *fod; + } src; +}; + +static struct clk_hw *vc7_of_clk_get(struct of_phandle_args *clkspec, + void *data) +{ + struct vc7_driver_data *vc7 = data; + unsigned int idx = clkspec->args[0]; + + if (idx >= vc7->chip_info->num_outputs) + return ERR_PTR(-EINVAL); + + return &vc7->clk_out[idx].hw; +} + +static const unsigned int RC21008A_index_to_output_mapping[] = { + 1, 2, 3, 6, 7, 8, 10, 11 +}; + +static int vc7_map_index_to_output(const enum vc7_model model, const unsigned int i) +{ + switch (model) { + case VC7_RC21008A: + return RC21008A_index_to_output_mapping[i]; + default: + return i; + } +} + +/* bank to output mapping, same across all variants */ +static const unsigned int output_bank_mapping[] = { + 0, /* Output 0 */ + 1, /* Output 1 */ + 2, /* Output 2 */ + 2, /* Output 3 */ + 3, /* Output 4 */ + 3, /* Output 5 */ + 3, /* Output 6 */ + 3, /* Output 7 */ + 4, /* Output 8 */ + 4, /* Output 9 */ + 5, /* Output 10 */ + 6 /* Output 11 */ +}; + +/** + * vc7_64_mul_64_to_128() - Multiply two u64 and return an unsigned 128-bit integer + * as an upper and lower part. + * + * @left: The left argument. + * @right: The right argument. + * @hi: The upper 64-bits of the 128-bit product. + * @lo: The lower 64-bits of the 128-bit product. + * + * From mul_64_64 in crypto/ecc.c:350 in the linux kernel, accessed in v5.17.2. + */ +static void vc7_64_mul_64_to_128(u64 left, u64 right, u64 *hi, u64 *lo) +{ + u64 a0 = left & 0xffffffffull; + u64 a1 = left >> 32; + u64 b0 = right & 0xffffffffull; + u64 b1 = right >> 32; + u64 m0 = a0 * b0; + u64 m1 = a0 * b1; + u64 m2 = a1 * b0; + u64 m3 = a1 * b1; + + m2 += (m0 >> 32); + m2 += m1; + + /* Overflow */ + if (m2 < m1) + m3 += 0x100000000ull; + + *lo = (m0 & 0xffffffffull) | (m2 << 32); + *hi = m3 + (m2 >> 32); +} + +/** + * vc7_128_div_64_to_64() - Divides a 128-bit uint by a 64-bit divisor, return a 64-bit quotient. + * + * @numhi: The uppper 64-bits of the dividend. + * @numlo: The lower 64-bits of the dividend. + * @den: The denominator (divisor). + * @r: The remainder, pass NULL if the remainder is not needed. + * + * Originally from libdivide, modified to use kernel u64/u32 types. + * + * See https://github.com/ridiculousfish/libdivide/blob/master/libdivide.h#L471. + * + * Return: The 64-bit quotient of the division. + * + * In case of overflow of division by zero, max(u64) is returned. + */ +static u64 vc7_128_div_64_to_64(u64 numhi, u64 numlo, u64 den, u64 *r) +{ + /* + * We work in base 2**32. + * A uint32 holds a single digit. A uint64 holds two digits. + * Our numerator is conceptually [num3, num2, num1, num0]. + * Our denominator is [den1, den0]. + */ + const u64 b = ((u64)1 << 32); + + /* The high and low digits of our computed quotient. */ + u32 q1, q0; + + /* The normalization shift factor */ + int shift; + + /* + * The high and low digits of our denominator (after normalizing). + * Also the low 2 digits of our numerator (after normalizing). + */ + u32 den1, den0, num1, num0; + + /* A partial remainder; */ + u64 rem; + + /* + * The estimated quotient, and its corresponding remainder (unrelated + * to true remainder). + */ + u64 qhat, rhat; + + /* Variables used to correct the estimated quotient. */ + u64 c1, c2; + + /* Check for overflow and divide by 0. */ + if (numhi >= den) { + if (r) + *r = ~0ull; + return ~0ull; + } + + /* + * Determine the normalization factor. We multiply den by this, so that + * its leading digit is at least half b. In binary this means just + * shifting left by the number of leading zeros, so that there's a 1 in + * the MSB. + * + * We also shift numer by the same amount. This cannot overflow because + * numhi < den. The expression (-shift & 63) is the same as (64 - + * shift), except it avoids the UB of shifting by 64. The funny bitwise + * 'and' ensures that numlo does not get shifted into numhi if shift is + * 0. clang 11 has an x86 codegen bug here: see LLVM bug 50118. The + * sequence below avoids it. + */ + shift = __builtin_clzll(den); + den <<= shift; + numhi <<= shift; + numhi |= (numlo >> (-shift & 63)) & (-(s64)shift >> 63); + numlo <<= shift; + + /* + * Extract the low digits of the numerator and both digits of the + * denominator. + */ + num1 = (u32)(numlo >> 32); + num0 = (u32)(numlo & 0xFFFFFFFFu); + den1 = (u32)(den >> 32); + den0 = (u32)(den & 0xFFFFFFFFu); + + /* + * We wish to compute q1 = [n3 n2 n1] / [d1 d0]. + * Estimate q1 as [n3 n2] / [d1], and then correct it. + * Note while qhat may be 2 digits, q1 is always 1 digit. + */ + qhat = div64_u64_rem(numhi, den1, &rhat); + c1 = qhat * den0; + c2 = rhat * b + num1; + if (c1 > c2) + qhat -= (c1 - c2 > den) ? 2 : 1; + q1 = (u32)qhat; + + /* Compute the true (partial) remainder. */ + rem = numhi * b + num1 - q1 * den; + + /* + * We wish to compute q0 = [rem1 rem0 n0] / [d1 d0]. + * Estimate q0 as [rem1 rem0] / [d1] and correct it. + */ + qhat = div64_u64_rem(rem, den1, &rhat); + c1 = qhat * den0; + c2 = rhat * b + num0; + if (c1 > c2) + qhat -= (c1 - c2 > den) ? 2 : 1; + q0 = (u32)qhat; + + /* Return remainder if requested. */ + if (r) + *r = (rem * b + num0 - q0 * den) >> shift; + return ((u64)q1 << 32) | q0; +} + +static int vc7_get_bank_clk(struct vc7_driver_data *vc7, + unsigned int bank_idx, + unsigned int output_bank_src, + struct vc7_bank_src_map *map) +{ + /* Mapping from Table 38 in datasheet */ + if (bank_idx == 0 || bank_idx == 1) { + switch (output_bank_src) { + case 0: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[0]; + return 0; + case 1: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[1]; + return 0; + case 4: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[0]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + default: + break; + } + } else if (bank_idx == 2) { + switch (output_bank_src) { + case 1: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[1]; + return 0; + case 4: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[0]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + default: + break; + } + } else if (bank_idx == 3) { + switch (output_bank_src) { + case 4: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[0]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + case 6: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[2]; + return 0; + default: + break; + } + } else if (bank_idx == 4) { + switch (output_bank_src) { + case 0: + /* CLKIN1 not supported in this driver */ + break; + case 2: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[2]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + case 6: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[2]; + return 0; + case 7: + /* CLKIN0 not supported in this driver */ + break; + default: + break; + } + } else if (bank_idx == 5) { + switch (output_bank_src) { + case 0: + /* CLKIN1 not supported in this driver */ + break; + case 1: + /* XIN_REFIN not supported in this driver */ + break; + case 2: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[2]; + return 0; + case 3: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[3]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + case 6: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[2]; + return 0; + case 7: + /* CLKIN0 not supported in this driver */ + break; + default: + break; + } + } else if (bank_idx == 6) { + switch (output_bank_src) { + case 0: + /* CLKIN1 not supported in this driver */ + break; + case 2: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[2]; + return 0; + case 3: + map->type = VC7_IOD, + map->src.iod = &vc7->clk_iod[3]; + return 0; + case 5: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[1]; + return 0; + case 6: + map->type = VC7_FOD, + map->src.fod = &vc7->clk_fod[2]; + return 0; + case 7: + /* CLKIN0 not supported in this driver */ + break; + default: + break; + } + } + + pr_warn("bank_src%d = %d is not supported\n", bank_idx, output_bank_src); + return -1; +} + +static int vc7_read_apll(struct vc7_driver_data *vc7) +{ + int err; + u32 val32; + u16 val16; + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_XO_CNFG, + (u32 *)&val32, + VC7_REG_XO_CNFG_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read XO_CNFG\n"); + return err; + } + + vc7->clk_apll.xo_ib_h_div = (val32 & VC7_REG_XO_IB_H_DIV_MASK) + >> VC7_REG_XO_IB_H_DIV_SHIFT; + + err = regmap_read(vc7->regmap, + VC7_REG_APLL_CNFG, + &val32); + if (err) { + dev_err(&vc7->client->dev, "failed to read APLL_CNFG\n"); + return err; + } + + vc7->clk_apll.en_doubler = val32 & VC7_REG_APLL_EN_DOUBLER; + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_APLL_FB_DIV_FRAC, + (u32 *)&val32, + VC7_REG_APLL_FB_DIV_FRAC_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read APLL_FB_DIV_FRAC\n"); + return err; + } + + vc7->clk_apll.apll_fb_div_frac = val32 & VC7_REG_APLL_FB_DIV_FRAC_MASK; + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_APLL_FB_DIV_INT, + (u16 *)&val16, + VC7_REG_APLL_FB_DIV_INT_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read APLL_FB_DIV_INT\n"); + return err; + } + + vc7->clk_apll.apll_fb_div_int = val16 & VC7_REG_APLL_FB_DIV_INT_MASK; + + return 0; +} + +static int vc7_read_fod(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + u64 val; + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_FOD_INT_CNFG(idx), + (u64 *)&val, + VC7_REG_FOD_INT_CNFG_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read FOD%d\n", idx); + return err; + } + + vc7->clk_fod[idx].fod_1st_int = (val & VC7_REG_FOD_1ST_INT_MASK); + vc7->clk_fod[idx].fod_2nd_int = + (val & VC7_REG_FOD_2ND_INT_MASK) >> VC7_REG_FOD_2ND_INT_SHIFT; + vc7->clk_fod[idx].fod_frac = (val & VC7_REG_FOD_FRAC_MASK) + >> VC7_REG_FOD_FRAC_SHIFT; + + return 0; +} + +static int vc7_write_fod(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + u64 val; + + /* + * FOD dividers are part of an atomic group where fod_1st_int, + * fod_2nd_int, and fod_frac must be written together. The new divider + * is applied when the MSB of fod_frac is written. + */ + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_FOD_INT_CNFG(idx), + (u64 *)&val, + VC7_REG_FOD_INT_CNFG_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read FOD%d\n", idx); + return err; + } + + val = u64_replace_bits(val, + vc7->clk_fod[idx].fod_1st_int, + VC7_REG_FOD_1ST_INT_MASK); + val = u64_replace_bits(val, + vc7->clk_fod[idx].fod_2nd_int, + VC7_REG_FOD_2ND_INT_MASK); + val = u64_replace_bits(val, + vc7->clk_fod[idx].fod_frac, + VC7_REG_FOD_FRAC_MASK); + + err = regmap_bulk_write(vc7->regmap, + VC7_REG_FOD_INT_CNFG(idx), + (u64 *)&val, + sizeof(u64)); + if (err) { + dev_err(&vc7->client->dev, "failed to write FOD%d\n", idx); + return err; + } + + return 0; +} + +static int vc7_read_iod(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + u32 val; + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_IOD_INT_CNFG(idx), + (u32 *)&val, + VC7_REG_IOD_INT_CNFG_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read IOD%d\n", idx); + return err; + } + + vc7->clk_iod[idx].iod_int = (val & VC7_REG_IOD_INT_MASK); + + return 0; +} + +static int vc7_write_iod(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + u32 val; + + /* + * IOD divider field is atomic and all bits must be written. + * The new divider is applied when the MSB of iod_int is written. + */ + + err = regmap_bulk_read(vc7->regmap, + VC7_REG_IOD_INT_CNFG(idx), + (u32 *)&val, + VC7_REG_IOD_INT_CNFG_COUNT); + if (err) { + dev_err(&vc7->client->dev, "failed to read IOD%d\n", idx); + return err; + } + + val = u32_replace_bits(val, + vc7->clk_iod[idx].iod_int, + VC7_REG_IOD_INT_MASK); + + err = regmap_bulk_write(vc7->regmap, + VC7_REG_IOD_INT_CNFG(idx), + (u32 *)&val, + sizeof(u32)); + if (err) { + dev_err(&vc7->client->dev, "failed to write IOD%d\n", idx); + return err; + } + + return 0; +} + +static int vc7_read_output(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + unsigned int val, out_num; + + out_num = vc7_map_index_to_output(vc7->chip_info->model, idx); + err = regmap_read(vc7->regmap, + VC7_REG_ODRV_EN(out_num), + &val); + if (err) { + dev_err(&vc7->client->dev, "failed to read ODRV_EN[%d]\n", idx); + return err; + } + + vc7->clk_out[idx].out_dis = val & VC7_REG_OUT_DIS; + + return 0; +} + +static int vc7_write_output(struct vc7_driver_data *vc7, unsigned int idx) +{ + int err; + unsigned int out_num; + + out_num = vc7_map_index_to_output(vc7->chip_info->model, idx); + err = regmap_write_bits(vc7->regmap, + VC7_REG_ODRV_EN(out_num), + VC7_REG_OUT_DIS, + vc7->clk_out[idx].out_dis); + + if (err) { + dev_err(&vc7->client->dev, "failed to write ODRV_EN[%d]\n", idx); + return err; + } + + return 0; +} + +static unsigned long vc7_get_apll_rate(struct vc7_driver_data *vc7) +{ + int err; + unsigned long xtal_rate; + u64 refin_div, apll_rate; + + xtal_rate = clk_get_rate(vc7->pin_xin); + err = vc7_read_apll(vc7); + if (err) { + dev_err(&vc7->client->dev, "unable to read apll\n"); + return err; + } + + /* 0 is bypassed, 1 is reserved */ + if (vc7->clk_apll.xo_ib_h_div < 2) + refin_div = xtal_rate; + else + refin_div = div64_u64(xtal_rate, vc7->clk_apll.xo_ib_h_div); + + if (vc7->clk_apll.en_doubler) + refin_div *= 2; + + /* divider = int + (frac / 2^27) */ + apll_rate = (refin_div * vc7->clk_apll.apll_fb_div_int) + + ((refin_div * vc7->clk_apll.apll_fb_div_frac) >> VC7_APLL_DENOMINATOR_BITS); + + pr_debug("%s - xo_ib_h_div: %u, apll_fb_div_int: %u, apll_fb_div_frac: %u\n", + __func__, vc7->clk_apll.xo_ib_h_div, vc7->clk_apll.apll_fb_div_int, + vc7->clk_apll.apll_fb_div_frac); + pr_debug("%s - refin_div: %llu, apll rate: %llu\n", + __func__, refin_div, apll_rate); + + return apll_rate; +} + +static void vc7_calc_iod_divider(unsigned long rate, unsigned long parent_rate, + u32 *divider) +{ + *divider = DIV_ROUND_UP(parent_rate, rate); + if (*divider < VC7_IOD_MIN_DIVISOR) + *divider = VC7_IOD_MIN_DIVISOR; + if (*divider > VC7_IOD_MAX_DIVISOR) + *divider = VC7_IOD_MAX_DIVISOR; +} + +static void vc7_calc_fod_1st_stage(unsigned long rate, unsigned long parent_rate, + u32 *div_int, u64 *div_frac) +{ + u64 rem; + + *div_int = (u32)div64_u64_rem(parent_rate, rate, &rem); + *div_frac = div64_u64(rem << VC7_FOD_DENOMINATOR_BITS, rate); +} + +static unsigned long vc7_calc_fod_1st_stage_rate(unsigned long parent_rate, + u32 fod_1st_int, u64 fod_frac) +{ + u64 numer, denom, hi, lo, divisor; + + numer = fod_frac; + denom = BIT_ULL(VC7_FOD_DENOMINATOR_BITS); + + if (fod_frac) { + vc7_64_mul_64_to_128(parent_rate, denom, &hi, &lo); + divisor = ((u64)fod_1st_int * denom) + numer; + return vc7_128_div_64_to_64(hi, lo, divisor, NULL); + } + + return div64_u64(parent_rate, fod_1st_int); +} + +static unsigned long vc7_calc_fod_2nd_stage_rate(unsigned long parent_rate, + u32 fod_1st_int, u32 fod_2nd_int, u64 fod_frac) +{ + unsigned long fod_1st_stage_rate; + + fod_1st_stage_rate = vc7_calc_fod_1st_stage_rate(parent_rate, fod_1st_int, fod_frac); + + if (fod_2nd_int < 2) + return fod_1st_stage_rate; + + /* + * There is a div-by-2 preceding the 2nd stage integer divider + * (not shown on block diagram) so the actual 2nd stage integer + * divisor is 2 * N. + */ + return div64_u64(fod_1st_stage_rate >> 1, fod_2nd_int); +} + +static void vc7_calc_fod_divider(unsigned long rate, unsigned long parent_rate, + u32 *fod_1st_int, u32 *fod_2nd_int, u64 *fod_frac) +{ + unsigned int allow_frac, i, best_frac_i; + unsigned long first_stage_rate; + + vc7_calc_fod_1st_stage(rate, parent_rate, fod_1st_int, fod_frac); + first_stage_rate = vc7_calc_fod_1st_stage_rate(parent_rate, *fod_1st_int, *fod_frac); + + *fod_2nd_int = 0; + + /* Do we need the second stage integer divider? */ + if (first_stage_rate < VC7_FOD_1ST_STAGE_RATE_MIN) { + allow_frac = 0; + best_frac_i = VC7_FOD_2ND_INT_MIN; + + for (i = VC7_FOD_2ND_INT_MIN; i <= VC7_FOD_2ND_INT_MAX; i++) { + /* + * 1) There is a div-by-2 preceding the 2nd stage integer divider + * (not shown on block diagram) so the actual 2nd stage integer + * divisor is 2 * N. + * 2) Attempt to find an integer solution first. This means stepping + * through each 2nd stage integer and recalculating the 1st stage + * until the 1st stage frequency is out of bounds. If no integer + * solution is found, use the best fractional solution. + */ + vc7_calc_fod_1st_stage(parent_rate, rate * 2 * i, fod_1st_int, fod_frac); + first_stage_rate = vc7_calc_fod_1st_stage_rate(parent_rate, + *fod_1st_int, + *fod_frac); + + /* Remember the first viable fractional solution */ + if (best_frac_i == VC7_FOD_2ND_INT_MIN && + first_stage_rate > VC7_FOD_1ST_STAGE_RATE_MIN) { + best_frac_i = i; + } + + /* Is the divider viable? Prefer integer solutions over fractional. */ + if (*fod_1st_int < VC7_FOD_1ST_INT_MAX && + first_stage_rate >= VC7_FOD_1ST_STAGE_RATE_MIN && + (allow_frac || *fod_frac == 0)) { + *fod_2nd_int = i; + break; + } + + /* Ran out of divisors or the 1st stage frequency is out of range */ + if (i >= VC7_FOD_2ND_INT_MAX || + first_stage_rate > VC7_FOD_1ST_STAGE_RATE_MAX) { + allow_frac = 1; + i = best_frac_i; + + /* Restore the best frac and rerun the loop for the last time */ + if (best_frac_i != VC7_FOD_2ND_INT_MIN) + i--; + + continue; + } + } + } +} + +static unsigned long vc7_fod_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct vc7_fod_data *fod = container_of(hw, struct vc7_fod_data, hw); + struct vc7_driver_data *vc7 = fod->vc7; + int err; + unsigned long fod_rate; + + err = vc7_read_fod(vc7, fod->num); + if (err) { + dev_err(&vc7->client->dev, "error reading registers for %s\n", + clk_hw_get_name(hw)); + return err; + } + + pr_debug("%s - %s: parent_rate: %lu\n", __func__, clk_hw_get_name(hw), parent_rate); + + fod_rate = vc7_calc_fod_2nd_stage_rate(parent_rate, fod->fod_1st_int, + fod->fod_2nd_int, fod->fod_frac); + + pr_debug("%s - %s: fod_1st_int: %u, fod_2nd_int: %u, fod_frac: %llu\n", + __func__, clk_hw_get_name(hw), + fod->fod_1st_int, fod->fod_2nd_int, fod->fod_frac); + pr_debug("%s - %s rate: %lu\n", __func__, clk_hw_get_name(hw), fod_rate); + + return fod_rate; +} + +static long vc7_fod_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) +{ + struct vc7_fod_data *fod = container_of(hw, struct vc7_fod_data, hw); + unsigned long fod_rate; + + pr_debug("%s - %s: requested rate: %lu, parent_rate: %lu\n", + __func__, clk_hw_get_name(hw), rate, *parent_rate); + + vc7_calc_fod_divider(rate, *parent_rate, + &fod->fod_1st_int, &fod->fod_2nd_int, &fod->fod_frac); + fod_rate = vc7_calc_fod_2nd_stage_rate(*parent_rate, fod->fod_1st_int, + fod->fod_2nd_int, fod->fod_frac); + + pr_debug("%s - %s: fod_1st_int: %u, fod_2nd_int: %u, fod_frac: %llu\n", + __func__, clk_hw_get_name(hw), + fod->fod_1st_int, fod->fod_2nd_int, fod->fod_frac); + pr_debug("%s - %s rate: %lu\n", __func__, clk_hw_get_name(hw), fod_rate); + + return fod_rate; +} + +static int vc7_fod_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) +{ + struct vc7_fod_data *fod = container_of(hw, struct vc7_fod_data, hw); + struct vc7_driver_data *vc7 = fod->vc7; + unsigned long fod_rate; + + pr_debug("%s - %s: rate: %lu, parent_rate: %lu\n", + __func__, clk_hw_get_name(hw), rate, parent_rate); + + if (rate < VC7_FOD_RATE_MIN || rate > VC7_FOD_RATE_MAX) { + dev_err(&vc7->client->dev, + "requested frequency %lu Hz for %s is out of range\n", + rate, clk_hw_get_name(hw)); + return -EINVAL; + } + + vc7_write_fod(vc7, fod->num); + + fod_rate = vc7_calc_fod_2nd_stage_rate(parent_rate, fod->fod_1st_int, + fod->fod_2nd_int, fod->fod_frac); + + pr_debug("%s - %s: fod_1st_int: %u, fod_2nd_int: %u, fod_frac: %llu\n", + __func__, clk_hw_get_name(hw), + fod->fod_1st_int, fod->fod_2nd_int, fod->fod_frac); + pr_debug("%s - %s rate: %lu\n", __func__, clk_hw_get_name(hw), fod_rate); + + return 0; +} + +static const struct clk_ops vc7_fod_ops = { + .recalc_rate = vc7_fod_recalc_rate, + .round_rate = vc7_fod_round_rate, + .set_rate = vc7_fod_set_rate, +}; + +static unsigned long vc7_iod_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + struct vc7_iod_data *iod = container_of(hw, struct vc7_iod_data, hw); + struct vc7_driver_data *vc7 = iod->vc7; + int err; + unsigned long iod_rate; + + err = vc7_read_iod(vc7, iod->num); + if (err) { + dev_err(&vc7->client->dev, "error reading registers for %s\n", + clk_hw_get_name(hw)); + return err; + } + + iod_rate = div64_u64(parent_rate, iod->iod_int); + + pr_debug("%s - %s: iod_int: %u\n", __func__, clk_hw_get_name(hw), iod->iod_int); + pr_debug("%s - %s rate: %lu\n", __func__, clk_hw_get_name(hw), iod_rate); + + return iod_rate; +} + +static long vc7_iod_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) +{ + struct vc7_iod_data *iod = container_of(hw, struct vc7_iod_data, hw); + unsigned long iod_rate; + + pr_debug("%s - %s: requested rate: %lu, parent_rate: %lu\n", + __func__, clk_hw_get_name(hw), rate, *parent_rate); + + vc7_calc_iod_divider(rate, *parent_rate, &iod->iod_int); + iod_rate = div64_u64(*parent_rate, iod->iod_int); + + pr_debug("%s - %s: iod_int: %u\n", __func__, clk_hw_get_name(hw), iod->iod_int); + pr_debug("%s - %s rate: %ld\n", __func__, clk_hw_get_name(hw), iod_rate); + + return iod_rate; +} + +static int vc7_iod_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) +{ + struct vc7_iod_data *iod = container_of(hw, struct vc7_iod_data, hw); + struct vc7_driver_data *vc7 = iod->vc7; + unsigned long iod_rate; + + pr_debug("%s - %s: rate: %lu, parent_rate: %lu\n", + __func__, clk_hw_get_name(hw), rate, parent_rate); + + if (rate < VC7_IOD_RATE_MIN || rate > VC7_IOD_RATE_MAX) { + dev_err(&vc7->client->dev, + "requested frequency %lu Hz for %s is out of range\n", + rate, clk_hw_get_name(hw)); + return -EINVAL; + } + + vc7_write_iod(vc7, iod->num); + + iod_rate = div64_u64(parent_rate, iod->iod_int); + + pr_debug("%s - %s: iod_int: %u\n", __func__, clk_hw_get_name(hw), iod->iod_int); + pr_debug("%s - %s rate: %ld\n", __func__, clk_hw_get_name(hw), iod_rate); + + return 0; +} + +static const struct clk_ops vc7_iod_ops = { + .recalc_rate = vc7_iod_recalc_rate, + .round_rate = vc7_iod_round_rate, + .set_rate = vc7_iod_set_rate, +}; + +static int vc7_clk_out_prepare(struct clk_hw *hw) +{ + struct vc7_out_data *out = container_of(hw, struct vc7_out_data, hw); + struct vc7_driver_data *vc7 = out->vc7; + int err; + + out->out_dis = 0; + + err = vc7_write_output(vc7, out->num); + if (err) { + dev_err(&vc7->client->dev, "error writing registers for %s\n", + clk_hw_get_name(hw)); + return err; + } + + pr_debug("%s - %s: clk prepared\n", __func__, clk_hw_get_name(hw)); + + return 0; +} + +static void vc7_clk_out_unprepare(struct clk_hw *hw) +{ + struct vc7_out_data *out = container_of(hw, struct vc7_out_data, hw); + struct vc7_driver_data *vc7 = out->vc7; + int err; + + out->out_dis = 1; + + err = vc7_write_output(vc7, out->num); + if (err) { + dev_err(&vc7->client->dev, "error writing registers for %s\n", + clk_hw_get_name(hw)); + return; + } + + pr_debug("%s - %s: clk unprepared\n", __func__, clk_hw_get_name(hw)); +} + +static int vc7_clk_out_is_enabled(struct clk_hw *hw) +{ + struct vc7_out_data *out = container_of(hw, struct vc7_out_data, hw); + struct vc7_driver_data *vc7 = out->vc7; + int err, is_enabled; + + err = vc7_read_output(vc7, out->num); + if (err) { + dev_err(&vc7->client->dev, "error reading registers for %s\n", + clk_hw_get_name(hw)); + return err; + } + + is_enabled = !out->out_dis; + + pr_debug("%s - %s: is_enabled=%d\n", __func__, clk_hw_get_name(hw), is_enabled); + + return is_enabled; +} + +static const struct clk_ops vc7_clk_out_ops = { + .prepare = vc7_clk_out_prepare, + .unprepare = vc7_clk_out_unprepare, + .is_enabled = vc7_clk_out_is_enabled, +}; + +static int vc7_probe(struct i2c_client *client) +{ + struct vc7_driver_data *vc7; + struct clk_init_data clk_init; + struct vc7_bank_src_map bank_src_map; + const char *node_name, *apll_name; + const char *parent_names[1]; + unsigned int i, val, bank_idx, out_num; + unsigned long apll_rate; + int ret; + + vc7 = devm_kzalloc(&client->dev, sizeof(*vc7), GFP_KERNEL); + if (!vc7) + return -ENOMEM; + + i2c_set_clientdata(client, vc7); + vc7->client = client; + vc7->chip_info = of_device_get_match_data(&client->dev); + + vc7->pin_xin = devm_clk_get(&client->dev, "xin"); + if (PTR_ERR(vc7->pin_xin) == -EPROBE_DEFER) { + return dev_err_probe(&client->dev, -EPROBE_DEFER, + "xin not specified\n"); + } + + vc7->regmap = devm_regmap_init_i2c(client, &vc7_regmap_config); + if (IS_ERR(vc7->regmap)) { + return dev_err_probe(&client->dev, PTR_ERR(vc7->regmap), + "failed to allocate register map\n"); + } + + if (of_property_read_string(client->dev.of_node, "clock-output-names", + &node_name)) + node_name = client->dev.of_node->name; + + /* Register APLL */ + apll_rate = vc7_get_apll_rate(vc7); + apll_name = kasprintf(GFP_KERNEL, "%s_apll", node_name); + vc7->clk_apll.clk = clk_register_fixed_rate(&client->dev, apll_name, + __clk_get_name(vc7->pin_xin), + 0, apll_rate); + kfree(apll_name); /* ccf made a copy of the name */ + if (IS_ERR(vc7->clk_apll.clk)) { + return dev_err_probe(&client->dev, PTR_ERR(vc7->clk_apll.clk), + "failed to register apll\n"); + } + + /* Register FODs */ + for (i = 0; i < VC7_NUM_FOD; i++) { + memset(&clk_init, 0, sizeof(clk_init)); + clk_init.name = kasprintf(GFP_KERNEL, "%s_fod%d", node_name, i); + clk_init.ops = &vc7_fod_ops; + clk_init.parent_names = parent_names; + parent_names[0] = __clk_get_name(vc7->clk_apll.clk); + clk_init.num_parents = 1; + vc7->clk_fod[i].num = i; + vc7->clk_fod[i].vc7 = vc7; + vc7->clk_fod[i].hw.init = &clk_init; + ret = devm_clk_hw_register(&client->dev, &vc7->clk_fod[i].hw); + if (ret) + goto err_clk_register; + kfree(clk_init.name); /* ccf made a copy of the name */ + } + + /* Register IODs */ + for (i = 0; i < VC7_NUM_IOD; i++) { + memset(&clk_init, 0, sizeof(clk_init)); + clk_init.name = kasprintf(GFP_KERNEL, "%s_iod%d", node_name, i); + clk_init.ops = &vc7_iod_ops; + clk_init.parent_names = parent_names; + parent_names[0] = __clk_get_name(vc7->clk_apll.clk); + clk_init.num_parents = 1; + vc7->clk_iod[i].num = i; + vc7->clk_iod[i].vc7 = vc7; + vc7->clk_iod[i].hw.init = &clk_init; + ret = devm_clk_hw_register(&client->dev, &vc7->clk_iod[i].hw); + if (ret) + goto err_clk_register; + kfree(clk_init.name); /* ccf made a copy of the name */ + } + + /* Register outputs */ + for (i = 0; i < vc7->chip_info->num_outputs; i++) { + out_num = vc7_map_index_to_output(vc7->chip_info->model, i); + + /* + * This driver does not support remapping FOD/IOD to banks. + * The device state is read and the driver is setup to match + * the device's existing mapping. + */ + bank_idx = output_bank_mapping[out_num]; + + regmap_read(vc7->regmap, VC7_REG_OUT_BANK_CNFG(bank_idx), &val); + val &= VC7_REG_OUTPUT_BANK_SRC_MASK; + + memset(&bank_src_map, 0, sizeof(bank_src_map)); + ret = vc7_get_bank_clk(vc7, bank_idx, val, &bank_src_map); + if (ret) { + dev_err_probe(&client->dev, ret, + "unable to register output %d\n", i); + return ret; + } + + switch (bank_src_map.type) { + case VC7_FOD: + parent_names[0] = clk_hw_get_name(&bank_src_map.src.fod->hw); + break; + case VC7_IOD: + parent_names[0] = clk_hw_get_name(&bank_src_map.src.iod->hw); + break; + } + + memset(&clk_init, 0, sizeof(clk_init)); + clk_init.name = kasprintf(GFP_KERNEL, "%s_out%d", node_name, i); + clk_init.ops = &vc7_clk_out_ops; + clk_init.flags = CLK_SET_RATE_PARENT; + clk_init.parent_names = parent_names; + clk_init.num_parents = 1; + vc7->clk_out[i].num = i; + vc7->clk_out[i].vc7 = vc7; + vc7->clk_out[i].hw.init = &clk_init; + ret = devm_clk_hw_register(&client->dev, &vc7->clk_out[i].hw); + if (ret) + goto err_clk_register; + kfree(clk_init.name); /* ccf made a copy of the name */ + } + + ret = of_clk_add_hw_provider(client->dev.of_node, vc7_of_clk_get, vc7); + if (ret) { + dev_err_probe(&client->dev, ret, "unable to add clk provider\n"); + goto err_clk; + } + + return ret; + +err_clk_register: + dev_err_probe(&client->dev, ret, + "unable to register %s\n", clk_init.name); + kfree(clk_init.name); /* ccf made a copy of the name */ +err_clk: + clk_unregister_fixed_rate(vc7->clk_apll.clk); + return ret; +} + +static void vc7_remove(struct i2c_client *client) +{ + struct vc7_driver_data *vc7 = i2c_get_clientdata(client); + + of_clk_del_provider(client->dev.of_node); + clk_unregister_fixed_rate(vc7->clk_apll.clk); +} + +static bool vc7_volatile_reg(struct device *dev, unsigned int reg) +{ + if (reg == VC7_PAGE_ADDR) + return false; + + return true; +} + +static const struct vc7_chip_info vc7_rc21008a_info = { + .model = VC7_RC21008A, + .num_banks = 6, + .num_outputs = 8, +}; + +static struct regmap_range_cfg vc7_range_cfg[] = { +{ + .range_min = 0, + .range_max = VC7_MAX_REG, + .selector_reg = VC7_PAGE_ADDR, + .selector_mask = 0xFF, + .selector_shift = 0, + .window_start = 0, + .window_len = VC7_PAGE_WINDOW, +}}; + +static const struct regmap_config vc7_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = VC7_MAX_REG, + .ranges = vc7_range_cfg, + .num_ranges = ARRAY_SIZE(vc7_range_cfg), + .volatile_reg = vc7_volatile_reg, + .cache_type = REGCACHE_RBTREE, + .can_multi_write = true, + .reg_format_endian = REGMAP_ENDIAN_LITTLE, + .val_format_endian = REGMAP_ENDIAN_LITTLE, +}; + +static const struct i2c_device_id vc7_i2c_id[] = { + { "rc21008a", VC7_RC21008A }, + {} +}; +MODULE_DEVICE_TABLE(i2c, vc7_i2c_id); + +static const struct of_device_id vc7_of_match[] = { + { .compatible = "renesas,rc21008a", .data = &vc7_rc21008a_info }, + {} +}; +MODULE_DEVICE_TABLE(of, vc7_of_match); + +static struct i2c_driver vc7_i2c_driver = { + .driver = { + .name = "vc7", + .of_match_table = vc7_of_match, + }, + .probe_new = vc7_probe, + .remove = vc7_remove, + .id_table = vc7_i2c_id, +}; +module_i2c_driver(vc7_i2c_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Alex Helms param.reg_clk_mask ? "enabled" : "disabled"); + } else { + return 1; } - if (!pclk->param.csr_reg) - return 1; return data & pclk->param.reg_clk_mask ? 1 : 0; } diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 7fc191c155073b4b912d1ef14f588d068df3cd14..c3c3f8c072588d2a3581e7af7cad80115ffb081d 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -536,6 +536,53 @@ static bool mux_is_better_rate(unsigned long rate, unsigned long now, return now <= rate && now > best; } +static void clk_core_init_rate_req(struct clk_core * const core, + struct clk_rate_request *req, + unsigned long rate); + +static int clk_core_round_rate_nolock(struct clk_core *core, + struct clk_rate_request *req); + +static bool clk_core_has_parent(struct clk_core *core, const struct clk_core *parent) +{ + struct clk_core *tmp; + unsigned int i; + + /* Optimize for the case where the parent is already the parent. */ + if (core->parent == parent) + return true; + + for (i = 0; i < core->num_parents; i++) { + tmp = clk_core_get_parent_by_index(core, i); + if (!tmp) + continue; + + if (tmp == parent) + return true; + } + + return false; +} + +static void +clk_core_forward_rate_req(struct clk_core *core, + const struct clk_rate_request *old_req, + struct clk_core *parent, + struct clk_rate_request *req, + unsigned long parent_rate) +{ + if (WARN_ON(!clk_core_has_parent(core, parent))) + return; + + clk_core_init_rate_req(parent, req, parent_rate); + + if (req->min_rate < old_req->min_rate) + req->min_rate = old_req->min_rate; + + if (req->max_rate > old_req->max_rate) + req->max_rate = old_req->max_rate; +} + int clk_mux_determine_rate_flags(struct clk_hw *hw, struct clk_rate_request *req, unsigned long flags) @@ -543,14 +590,20 @@ int clk_mux_determine_rate_flags(struct clk_hw *hw, struct clk_core *core = hw->core, *parent, *best_parent = NULL; int i, num_parents, ret; unsigned long best = 0; - struct clk_rate_request parent_req = *req; /* if NO_REPARENT flag set, pass through to current parent */ if (core->flags & CLK_SET_RATE_NO_REPARENT) { parent = core->parent; if (core->flags & CLK_SET_RATE_PARENT) { - ret = __clk_determine_rate(parent ? parent->hw : NULL, - &parent_req); + struct clk_rate_request parent_req; + + if (!parent) { + req->rate = 0; + return 0; + } + + clk_core_forward_rate_req(core, req, parent, &parent_req, req->rate); + ret = clk_core_round_rate_nolock(parent, &parent_req); if (ret) return ret; @@ -567,23 +620,29 @@ int clk_mux_determine_rate_flags(struct clk_hw *hw, /* find the parent that can provide the fastest rate <= rate */ num_parents = core->num_parents; for (i = 0; i < num_parents; i++) { + unsigned long parent_rate; + parent = clk_core_get_parent_by_index(core, i); if (!parent) continue; if (core->flags & CLK_SET_RATE_PARENT) { - parent_req = *req; - ret = __clk_determine_rate(parent->hw, &parent_req); + struct clk_rate_request parent_req; + + clk_core_forward_rate_req(core, req, parent, &parent_req, req->rate); + ret = clk_core_round_rate_nolock(parent, &parent_req); if (ret) continue; + + parent_rate = parent_req.rate; } else { - parent_req.rate = clk_core_get_rate_nolock(parent); + parent_rate = clk_core_get_rate_nolock(parent); } - if (mux_is_better_rate(req->rate, parent_req.rate, + if (mux_is_better_rate(req->rate, parent_rate, best, flags)) { best_parent = parent; - best = parent_req.rate; + best = parent_rate; } } @@ -625,6 +684,22 @@ static void clk_core_get_boundaries(struct clk_core *core, *max_rate = min(*max_rate, clk_user->max_rate); } +/* + * clk_hw_get_rate_range() - returns the clock rate range for a hw clk + * @hw: the hw clk we want to get the range from + * @min_rate: pointer to the variable that will hold the minimum + * @max_rate: pointer to the variable that will hold the maximum + * + * Fills the @min_rate and @max_rate variables with the minimum and + * maximum that clock can reach. + */ +void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate, + unsigned long *max_rate) +{ + clk_core_get_boundaries(hw->core, min_rate, max_rate); +} +EXPORT_SYMBOL_GPL(clk_hw_get_rate_range); + static bool clk_core_check_boundaries(struct clk_core *core, unsigned long min_rate, unsigned long max_rate) @@ -840,10 +915,9 @@ static void clk_core_unprepare(struct clk_core *core) if (core->ops->unprepare) core->ops->unprepare(core->hw); - clk_pm_runtime_put(core); - trace_clk_unprepare_complete(core); clk_core_unprepare(core->parent); + clk_pm_runtime_put(core); } static void clk_core_unprepare_lock(struct clk_core *core) @@ -1341,7 +1415,19 @@ static int clk_core_determine_round_nolock(struct clk_core *core, if (!core) return 0; - req->rate = clamp(req->rate, req->min_rate, req->max_rate); + /* + * Some clock providers hand-craft their clk_rate_requests and + * might not fill min_rate and max_rate. + * + * If it's the case, clamping the rate is equivalent to setting + * the rate to 0 which is bad. Skip the clamping but complain so + * that it gets fixed, hopefully. + */ + if (!req->min_rate && !req->max_rate) + pr_warn("%s: %s: clk_rate_request has initialized min or max rate.\n", + __func__, core->name); + else + req->rate = clamp(req->rate, req->min_rate, req->max_rate); /* * At this point, core protection will be disabled @@ -1368,13 +1454,19 @@ static int clk_core_determine_round_nolock(struct clk_core *core, } static void clk_core_init_rate_req(struct clk_core * const core, - struct clk_rate_request *req) + struct clk_rate_request *req, + unsigned long rate) { struct clk_core *parent; if (WARN_ON(!core || !req)) return; + memset(req, 0, sizeof(*req)); + + req->rate = rate; + clk_core_get_boundaries(core, &req->min_rate, &req->max_rate); + parent = core->parent; if (parent) { req->best_parent_hw = parent->hw; @@ -1385,6 +1477,51 @@ static void clk_core_init_rate_req(struct clk_core * const core, } } +/** + * clk_hw_init_rate_request - Initializes a clk_rate_request + * @hw: the clk for which we want to submit a rate request + * @req: the clk_rate_request structure we want to initialise + * @rate: the rate which is to be requested + * + * Initializes a clk_rate_request structure to submit to + * __clk_determine_rate() or similar functions. + */ +void clk_hw_init_rate_request(const struct clk_hw *hw, + struct clk_rate_request *req, + unsigned long rate) +{ + if (WARN_ON(!hw || !req)) + return; + + clk_core_init_rate_req(hw->core, req, rate); +} +EXPORT_SYMBOL_GPL(clk_hw_init_rate_request); + +/** + * clk_hw_forward_rate_request - Forwards a clk_rate_request to a clock's parent + * @hw: the original clock that got the rate request + * @old_req: the original clk_rate_request structure we want to forward + * @parent: the clk we want to forward @old_req to + * @req: the clk_rate_request structure we want to initialise + * @parent_rate: The rate which is to be requested to @parent + * + * Initializes a clk_rate_request structure to submit to a clock parent + * in __clk_determine_rate() or similar functions. + */ +void clk_hw_forward_rate_request(const struct clk_hw *hw, + const struct clk_rate_request *old_req, + const struct clk_hw *parent, + struct clk_rate_request *req, + unsigned long parent_rate) +{ + if (WARN_ON(!hw || !old_req || !parent || !req)) + return; + + clk_core_forward_rate_req(hw->core, old_req, + parent->core, req, + parent_rate); +} + static bool clk_core_can_round(struct clk_core * const core) { return core->ops->determine_rate || core->ops->round_rate; @@ -1393,6 +1530,8 @@ static bool clk_core_can_round(struct clk_core * const core) static int clk_core_round_rate_nolock(struct clk_core *core, struct clk_rate_request *req) { + int ret; + lockdep_assert_held(&prepare_lock); if (!core) { @@ -1400,12 +1539,22 @@ static int clk_core_round_rate_nolock(struct clk_core *core, return 0; } - clk_core_init_rate_req(core, req); - if (clk_core_can_round(core)) return clk_core_determine_round_nolock(core, req); - else if (core->flags & CLK_SET_RATE_PARENT) - return clk_core_round_rate_nolock(core->parent, req); + + if (core->flags & CLK_SET_RATE_PARENT) { + struct clk_rate_request parent_req; + + clk_core_forward_rate_req(core, req, core->parent, &parent_req, req->rate); + ret = clk_core_round_rate_nolock(core->parent, &parent_req); + if (ret) + return ret; + + req->best_parent_rate = parent_req.rate; + req->rate = parent_req.rate; + + return 0; + } req->rate = core->rate; return 0; @@ -1449,8 +1598,7 @@ unsigned long clk_hw_round_rate(struct clk_hw *hw, unsigned long rate) int ret; struct clk_rate_request req; - clk_core_get_boundaries(hw->core, &req.min_rate, &req.max_rate); - req.rate = rate; + clk_core_init_rate_req(hw->core, &req, rate); ret = clk_core_round_rate_nolock(hw->core, &req); if (ret) @@ -1482,8 +1630,7 @@ long clk_round_rate(struct clk *clk, unsigned long rate) if (clk->exclusive_count) clk_core_rate_unprotect(clk->core); - clk_core_get_boundaries(clk->core, &req.min_rate, &req.max_rate); - req.rate = rate; + clk_core_init_rate_req(clk->core, &req, rate); ret = clk_core_round_rate_nolock(clk->core, &req); @@ -1612,6 +1759,7 @@ static unsigned long clk_recalc(struct clk_core *core, /** * __clk_recalc_rates * @core: first clk in the subtree + * @update_req: Whether req_rate should be updated with the new rate * @msg: notification type (see include/linux/clk.h) * * Walks the subtree of clks starting with clk and recalculates rates as it @@ -1621,7 +1769,8 @@ static unsigned long clk_recalc(struct clk_core *core, * clk_recalc_rates also propagates the POST_RATE_CHANGE notification, * if necessary. */ -static void __clk_recalc_rates(struct clk_core *core, unsigned long msg) +static void __clk_recalc_rates(struct clk_core *core, bool update_req, + unsigned long msg) { unsigned long old_rate; unsigned long parent_rate = 0; @@ -1635,6 +1784,8 @@ static void __clk_recalc_rates(struct clk_core *core, unsigned long msg) parent_rate = core->parent->rate; core->rate = clk_recalc(core, parent_rate); + if (update_req) + core->req_rate = core->rate; /* * ignore NOTIFY_STOP and NOTIFY_BAD return values for POST_RATE_CHANGE @@ -1644,13 +1795,13 @@ static void __clk_recalc_rates(struct clk_core *core, unsigned long msg) __clk_notify(core, msg, old_rate, core->rate); hlist_for_each_entry(child, &core->children, child_node) - __clk_recalc_rates(child, msg); + __clk_recalc_rates(child, update_req, msg); } static unsigned long clk_core_get_rate_recalc(struct clk_core *core) { if (core && (core->flags & CLK_GET_RATE_NOCACHE)) - __clk_recalc_rates(core, 0); + __clk_recalc_rates(core, false, 0); return clk_core_get_rate_nolock(core); } @@ -1660,8 +1811,9 @@ static unsigned long clk_core_get_rate_recalc(struct clk_core *core) * @clk: the clk whose rate is being returned * * Simply returns the cached rate of the clk, unless CLK_GET_RATE_NOCACHE flag - * is set, which means a recalc_rate will be issued. - * If clk is NULL then returns 0. + * is set, which means a recalc_rate will be issued. Can be called regardless of + * the clock enabledness. If clk is NULL, or if an error occurred, then returns + * 0. */ unsigned long clk_get_rate(struct clk *clk) { @@ -1865,6 +2017,7 @@ static int __clk_set_parent(struct clk_core *core, struct clk_core *parent, flags = clk_enable_lock(); clk_reparent(core, old_parent); clk_enable_unlock(flags); + __clk_set_parent_after(core, old_parent, parent); return ret; @@ -1970,11 +2123,7 @@ static struct clk_core *clk_calc_new_rates(struct clk_core *core, if (clk_core_can_round(core)) { struct clk_rate_request req; - req.rate = rate; - req.min_rate = min_rate; - req.max_rate = max_rate; - - clk_core_init_rate_req(core, &req); + clk_core_init_rate_req(core, &req, rate); ret = clk_core_determine_round_nolock(core, &req); if (ret < 0) @@ -2173,8 +2322,7 @@ static unsigned long clk_core_req_round_rate_nolock(struct clk_core *core, if (cnt < 0) return cnt; - clk_core_get_boundaries(core, &req.min_rate, &req.max_rate); - req.rate = req_rate; + clk_core_init_rate_req(core, &req, req_rate); ret = clk_core_round_rate_nolock(core, &req); @@ -2189,7 +2337,7 @@ static int clk_core_set_rate_nolock(struct clk_core *core, { struct clk_core *top, *fail_clk; unsigned long rate; - int ret = 0; + int ret; if (!core) return 0; @@ -2325,19 +2473,15 @@ int clk_set_rate_exclusive(struct clk *clk, unsigned long rate) } EXPORT_SYMBOL_GPL(clk_set_rate_exclusive); -/** - * clk_set_rate_range - set a rate range for a clock source - * @clk: clock source - * @min: desired minimum clock rate in Hz, inclusive - * @max: desired maximum clock rate in Hz, inclusive - * - * Returns success (0) or negative errno. - */ -int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) +static int clk_set_rate_range_nolock(struct clk *clk, + unsigned long min, + unsigned long max) { int ret = 0; unsigned long old_min, old_max, rate; + lockdep_assert_held(&prepare_lock); + if (!clk) return 0; @@ -2350,8 +2494,6 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) return -EINVAL; } - clk_prepare_lock(); - if (clk->exclusive_count) clk_core_rate_unprotect(clk->core); @@ -2366,6 +2508,10 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) goto out; } + rate = clk->core->req_rate; + if (clk->core->flags & CLK_GET_RATE_NOCACHE) + rate = clk_core_get_rate_recalc(clk->core); + /* * Since the boundaries have been changed, let's give the * opportunity to the provider to adjust the clock rate based on @@ -2383,7 +2529,7 @@ int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) * - the determine_rate() callback does not really check for * this corner case when determining the rate */ - rate = clamp(clk->core->req_rate, min, max); + rate = clamp(rate, min, max); ret = clk_core_set_rate_nolock(clk->core, rate); if (ret) { /* rollback the changes */ @@ -2395,6 +2541,28 @@ out: if (clk->exclusive_count) clk_core_rate_protect(clk->core); + return ret; +} + +/** + * clk_set_rate_range - set a rate range for a clock source + * @clk: clock source + * @min: desired minimum clock rate in Hz, inclusive + * @max: desired maximum clock rate in Hz, inclusive + * + * Return: 0 for success or negative errno on failure. + */ +int clk_set_rate_range(struct clk *clk, unsigned long min, unsigned long max) +{ + int ret; + + if (!clk) + return 0; + + clk_prepare_lock(); + + ret = clk_set_rate_range_nolock(clk, min, max); + clk_prepare_unlock(); return ret; @@ -2474,7 +2642,7 @@ static void clk_core_reparent(struct clk_core *core, { clk_reparent(core, new_parent); __clk_recalc_accuracies(core); - __clk_recalc_rates(core, POST_RATE_CHANGE); + __clk_recalc_rates(core, true, POST_RATE_CHANGE); } void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent) @@ -2495,27 +2663,13 @@ void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent) * * Returns true if @parent is a possible parent for @clk, false otherwise. */ -bool clk_has_parent(struct clk *clk, struct clk *parent) +bool clk_has_parent(const struct clk *clk, const struct clk *parent) { - struct clk_core *core, *parent_core; - int i; - /* NULL clocks should be nops, so return success if either is NULL. */ if (!clk || !parent) return true; - core = clk->core; - parent_core = parent->core; - - /* Optimize for the case where the parent is already the parent. */ - if (core->parent == parent_core) - return true; - - for (i = 0; i < core->num_parents; i++) - if (!strcmp(core->parents[i].name, parent_core->name)) - return true; - - return false; + return clk_core_has_parent(clk->core, parent->core); } EXPORT_SYMBOL_GPL(clk_has_parent); @@ -2572,9 +2726,9 @@ static int clk_core_set_parent_nolock(struct clk_core *core, /* propagate rate an accuracy recalculation accordingly */ if (ret) { - __clk_recalc_rates(core, ABORT_RATE_CHANGE); + __clk_recalc_rates(core, true, ABORT_RATE_CHANGE); } else { - __clk_recalc_rates(core, POST_RATE_CHANGE); + __clk_recalc_rates(core, true, POST_RATE_CHANGE); __clk_recalc_accuracies(core); } @@ -3462,7 +3616,7 @@ static void clk_core_reparent_orphans_nolock(void) /* * We need to use __clk_set_parent_before() and _after() to - * to properly migrate any prepare/enable count of the orphan + * properly migrate any prepare/enable count of the orphan * clock. This is important for CLK_IS_CRITICAL clocks, which * are enabled during init but might not have a parent yet. */ @@ -3471,7 +3625,7 @@ static void clk_core_reparent_orphans_nolock(void) __clk_set_parent_before(orphan, parent); __clk_set_parent_after(orphan, parent, NULL); __clk_recalc_accuracies(orphan); - __clk_recalc_rates(orphan, 0); + __clk_recalc_rates(orphan, true, 0); /* * __clk_init_parent() will set the initial req_rate to @@ -3672,7 +3826,6 @@ static int __clk_core_init(struct clk_core *core) clk_core_reparent_orphans_nolock(); - kref_init(&core->ref); out: clk_pm_runtime_put(core); @@ -4348,9 +4501,10 @@ void __clk_put(struct clk *clk) } hlist_del(&clk->clks_node); - if (clk->min_rate > clk->core->req_rate || - clk->max_rate < clk->core->req_rate) - clk_core_set_rate_nolock(clk->core, clk->core->req_rate); + + /* If we had any boundaries on that clock, let's drop them. */ + if (clk->min_rate > 0 || clk->max_rate < ULONG_MAX) + clk_set_rate_range_nolock(clk, 0, ULONG_MAX); owner = clk->core->owner; kref_put(&clk->core->ref, __clk_release); @@ -4751,32 +4905,6 @@ void of_clk_del_provider(struct device_node *np) } EXPORT_SYMBOL_GPL(of_clk_del_provider); -static int devm_clk_provider_match(struct device *dev, void *res, void *data) -{ - struct device_node **np = res; - - if (WARN_ON(!np || !*np)) - return 0; - - return *np == data; -} - -/** - * devm_of_clk_del_provider() - Remove clock provider registered using devm - * @dev: Device to whose lifetime the clock provider was bound - */ -void devm_of_clk_del_provider(struct device *dev) -{ - int ret; - struct device_node *np = get_clk_provider_node(dev); - - ret = devres_release(dev, devm_of_clk_release_provider, - devm_clk_provider_match, np); - - WARN_ON(ret); -} -EXPORT_SYMBOL(devm_of_clk_del_provider); - /** * of_parse_clkspec() - Parse a DT clock specifier for a given device node * @np: device node to parse clock specifier from diff --git a/drivers/clk/clk_test.c b/drivers/clk/clk_test.c index 6731a822f4e38f93494e8e001c9078148c3cf942..f9a5c2964c65dc17e97c0b2ba3c89fc0db285182 100644 --- a/drivers/clk/clk_test.c +++ b/drivers/clk/clk_test.c @@ -108,6 +108,39 @@ static const struct clk_ops clk_dummy_single_parent_ops = { .get_parent = clk_dummy_single_get_parent, }; +struct clk_multiple_parent_ctx { + struct clk_dummy_context parents_ctx[2]; + struct clk_hw hw; + u8 current_parent; +}; + +static int clk_multiple_parents_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_multiple_parent_ctx *ctx = + container_of(hw, struct clk_multiple_parent_ctx, hw); + + if (index >= clk_hw_get_num_parents(hw)) + return -EINVAL; + + ctx->current_parent = index; + + return 0; +} + +static u8 clk_multiple_parents_mux_get_parent(struct clk_hw *hw) +{ + struct clk_multiple_parent_ctx *ctx = + container_of(hw, struct clk_multiple_parent_ctx, hw); + + return ctx->current_parent; +} + +static const struct clk_ops clk_multiple_parents_mux_ops = { + .get_parent = clk_multiple_parents_mux_get_parent, + .set_parent = clk_multiple_parents_mux_set_parent, + .determine_rate = __clk_mux_determine_rate_closest, +}; + static int clk_test_init_with_ops(struct kunit *test, const struct clk_ops *ops) { struct clk_dummy_context *ctx; @@ -160,12 +193,14 @@ static void clk_test_get_rate(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, ctx->rate); + + clk_put(clk); } /* @@ -179,7 +214,7 @@ static void clk_test_set_get_rate(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -189,6 +224,8 @@ static void clk_test_set_get_rate(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(clk); } /* @@ -202,7 +239,7 @@ static void clk_test_set_set_get_rate(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -216,6 +253,8 @@ static void clk_test_set_set_get_rate(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -226,7 +265,7 @@ static void clk_test_round_set_get_rate(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rounded_rate, set_rate; rounded_rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1); @@ -240,6 +279,8 @@ static void clk_test_round_set_get_rate(struct kunit *test) set_rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, set_rate, 0); KUNIT_EXPECT_EQ(test, rounded_rate, set_rate); + + clk_put(clk); } static struct kunit_case clk_test_cases[] = { @@ -250,6 +291,11 @@ static struct kunit_case clk_test_cases[] = { {} }; +/* + * Test suite for a basic rate clock, without any parent. + * + * These tests exercise the rate API with simple scenarios + */ static struct kunit_suite clk_test_suite = { .name = "clk-test", .init = clk_test_init, @@ -257,16 +303,132 @@ static struct kunit_suite clk_test_suite = { .test_cases = clk_test_cases, }; -struct clk_single_parent_ctx { - struct clk_dummy_context parent_ctx; - struct clk_hw hw; +static int clk_uncached_test_init(struct kunit *test) +{ + struct clk_dummy_context *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + ctx->rate = DUMMY_CLOCK_INIT_RATE; + ctx->hw.init = CLK_HW_INIT_NO_PARENT("test-clk", + &clk_dummy_rate_ops, + CLK_GET_RATE_NOCACHE); + + ret = clk_hw_register(NULL, &ctx->hw); + if (ret) + return ret; + + return 0; +} + +/* + * Test that for an uncached clock, the clock framework doesn't cache + * the rate and clk_get_rate() will return the underlying clock rate + * even if it changed. + */ +static void clk_test_uncached_get_rate(struct kunit *test) +{ + struct clk_dummy_context *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE); + + /* We change the rate behind the clock framework's back */ + ctx->rate = DUMMY_CLOCK_RATE_1; + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(clk); +} + +/* + * Test that for an uncached clock, clk_set_rate_range() will work + * properly if the rate hasn't changed. + */ +static void clk_test_uncached_set_range(struct kunit *test) +{ + struct clk_dummy_context *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(clk, + DUMMY_CLOCK_RATE_1, + DUMMY_CLOCK_RATE_2), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); +} + +/* + * Test that for an uncached clock, clk_set_rate_range() will work + * properly if the rate has changed in hardware. + * + * In this case, it means that if the rate wasn't initially in the range + * we're trying to set, but got changed at some point into the range + * without the kernel knowing about it, its rate shouldn't be affected. + */ +static void clk_test_uncached_updated_rate_set_range(struct kunit *test) +{ + struct clk_dummy_context *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + + /* We change the rate behind the clock framework's back */ + ctx->rate = DUMMY_CLOCK_RATE_1 + 1000; + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(clk, + DUMMY_CLOCK_RATE_1, + DUMMY_CLOCK_RATE_2), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + + clk_put(clk); +} + +static struct kunit_case clk_uncached_test_cases[] = { + KUNIT_CASE(clk_test_uncached_get_rate), + KUNIT_CASE(clk_test_uncached_set_range), + KUNIT_CASE(clk_test_uncached_updated_rate_set_range), + {} }; -static int clk_orphan_transparent_single_parent_mux_test_init(struct kunit *test) +/* + * Test suite for a basic, uncached, rate clock, without any parent. + * + * These tests exercise the rate API with simple scenarios + */ +static struct kunit_suite clk_uncached_test_suite = { + .name = "clk-uncached-test", + .init = clk_uncached_test_init, + .exit = clk_test_exit, + .test_cases = clk_uncached_test_cases, +}; + +static int +clk_multiple_parents_mux_test_init(struct kunit *test) { - struct clk_single_parent_ctx *ctx; - struct clk_init_data init = { }; - const char * const parents[] = { "orphan_parent" }; + struct clk_multiple_parent_ctx *ctx; + const char *parents[2] = { "parent-0", "parent-1"}; int ret; ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); @@ -274,73 +436,993 @@ static int clk_orphan_transparent_single_parent_mux_test_init(struct kunit *test return -ENOMEM; test->priv = ctx; - init.name = "test_orphan_dummy_parent"; - init.ops = &clk_dummy_single_parent_ops; - init.parent_names = parents; - init.num_parents = ARRAY_SIZE(parents); - init.flags = CLK_SET_RATE_PARENT; - ctx->hw.init = &init; + ctx->parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0", + &clk_dummy_rate_ops, + 0); + ctx->parents_ctx[0].rate = DUMMY_CLOCK_RATE_1; + ret = clk_hw_register(NULL, &ctx->parents_ctx[0].hw); + if (ret) + return ret; + + ctx->parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1", + &clk_dummy_rate_ops, + 0); + ctx->parents_ctx[1].rate = DUMMY_CLOCK_RATE_2; + ret = clk_hw_register(NULL, &ctx->parents_ctx[1].hw); + if (ret) + return ret; + ctx->current_parent = 0; + ctx->hw.init = CLK_HW_INIT_PARENTS("test-mux", parents, + &clk_multiple_parents_mux_ops, + CLK_SET_RATE_PARENT); ret = clk_hw_register(NULL, &ctx->hw); if (ret) return ret; - memset(&init, 0, sizeof(init)); - init.name = "orphan_parent"; - init.ops = &clk_dummy_rate_ops; - ctx->parent_ctx.hw.init = &init; - ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE; + return 0; +} - ret = clk_hw_register(NULL, &ctx->parent_ctx.hw); +static void +clk_multiple_parents_mux_test_exit(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + + clk_hw_unregister(&ctx->hw); + clk_hw_unregister(&ctx->parents_ctx[0].hw); + clk_hw_unregister(&ctx->parents_ctx[1].hw); +} + +/* + * Test that for a clock with multiple parents, clk_get_parent() + * actually returns the current one. + */ +static void +clk_test_multiple_parents_mux_get_parent(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL); + + KUNIT_EXPECT_TRUE(test, clk_is_match(clk_get_parent(clk), parent)); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that for a clock with a multiple parents, clk_has_parent() + * actually reports all of them as parents. + */ +static void +clk_test_multiple_parents_mux_has_parent(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + + parent = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL); + KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent)); + clk_put(parent); + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent)); + clk_put(parent); + + clk_put(clk); +} + +/* + * Test that for a clock with a multiple parents, if we set a range on + * that clock and the parent is changed, its rate after the reparenting + * is still within the range we asked for. + * + * FIXME: clk_set_parent() only does the reparenting but doesn't + * reevaluate whether the new clock rate is within its boundaries or + * not. + */ +static void +clk_test_multiple_parents_mux_set_range_set_parent_get_rate(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent1, *parent2; + unsigned long rate; + int ret; + + kunit_skip(test, "This needs to be fixed in the core."); + + parent1 = clk_hw_get_clk(&ctx->parents_ctx[0].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent1); + KUNIT_ASSERT_TRUE(test, clk_is_match(clk_get_parent(clk), parent1)); + + parent2 = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent2); + + ret = clk_set_rate(parent1, DUMMY_CLOCK_RATE_1); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate(parent2, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, + DUMMY_CLOCK_RATE_1 - 1000, + DUMMY_CLOCK_RATE_1 + 1000); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_parent(clk, parent2); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 - 1000); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + + clk_put(parent2); + clk_put(parent1); + clk_put(clk); +} + +static struct kunit_case clk_multiple_parents_mux_test_cases[] = { + KUNIT_CASE(clk_test_multiple_parents_mux_get_parent), + KUNIT_CASE(clk_test_multiple_parents_mux_has_parent), + KUNIT_CASE(clk_test_multiple_parents_mux_set_range_set_parent_get_rate), + {} +}; + +/* + * Test suite for a basic mux clock with two parents, with + * CLK_SET_RATE_PARENT on the child. + * + * These tests exercise the consumer API and check that the state of the + * child and parents are sane and consistent. + */ +static struct kunit_suite +clk_multiple_parents_mux_test_suite = { + .name = "clk-multiple-parents-mux-test", + .init = clk_multiple_parents_mux_test_init, + .exit = clk_multiple_parents_mux_test_exit, + .test_cases = clk_multiple_parents_mux_test_cases, +}; + +static int +clk_orphan_transparent_multiple_parent_mux_test_init(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx; + const char *parents[2] = { "missing-parent", "proper-parent"}; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + ctx->parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("proper-parent", + &clk_dummy_rate_ops, + 0); + ctx->parents_ctx[1].rate = DUMMY_CLOCK_INIT_RATE; + ret = clk_hw_register(NULL, &ctx->parents_ctx[1].hw); + if (ret) + return ret; + + ctx->hw.init = CLK_HW_INIT_PARENTS("test-orphan-mux", parents, + &clk_multiple_parents_mux_ops, + CLK_SET_RATE_PARENT); + ret = clk_hw_register(NULL, &ctx->hw); if (ret) return ret; return 0; } -static void clk_orphan_transparent_single_parent_mux_test_exit(struct kunit *test) +static void +clk_orphan_transparent_multiple_parent_mux_test_exit(struct kunit *test) { - struct clk_single_parent_ctx *ctx = test->priv; + struct clk_multiple_parent_ctx *ctx = test->priv; clk_hw_unregister(&ctx->hw); - clk_hw_unregister(&ctx->parent_ctx.hw); + clk_hw_unregister(&ctx->parents_ctx[1].hw); } /* - * Test that a mux-only clock, with an initial rate within a range, - * will still have the same rate after the range has been enforced. + * Test that, for a mux whose current parent hasn't been registered yet and is + * thus orphan, clk_get_parent() will return NULL. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_get_parent(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + + KUNIT_EXPECT_PTR_EQ(test, clk_get_parent(clk), NULL); + + clk_put(clk); +} + +/* + * Test that, for a mux whose current parent hasn't been registered yet, + * calling clk_set_parent() to a valid parent will properly update the + * mux parent and its orphan status. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent, *new_parent; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + new_parent = clk_get_parent(clk); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + KUNIT_EXPECT_TRUE(test, clk_is_match(parent, new_parent)); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that, for a mux that started orphan but got switched to a valid + * parent, calling clk_drop_range() on the mux won't affect the parent + * rate. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent_drop_range(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long parent_rate, new_parent_rate; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, parent_rate, 0); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_drop_range(clk); + KUNIT_ASSERT_EQ(test, ret, 0); + + new_parent_rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, new_parent_rate, 0); + KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that, for a mux that started orphan but got switched to a valid + * parent, the rate of the mux and its new parent are consistent. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent_get_rate(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long parent_rate, rate; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, parent_rate, 0); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, parent_rate, rate); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that, for a mux that started orphan but got switched to a valid + * parent, calling clk_put() on the mux won't affect the parent rate. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent_put(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk *clk, *parent; + unsigned long parent_rate, new_parent_rate; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + clk = clk_hw_get_clk(&ctx->hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, clk); + + parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, parent_rate, 0); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + clk_put(clk); + + new_parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, new_parent_rate, 0); + KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate); + + clk_put(parent); +} + +/* + * Test that, for a mux that started orphan but got switched to a valid + * parent, calling clk_set_rate_range() will affect the parent state if + * its rate is out of range. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_modified(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long rate; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that, for a mux that started orphan but got switched to a valid + * parent, calling clk_set_rate_range() won't affect the parent state if + * its rate is within range. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_untouched(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long parent_rate, new_parent_rate; + int ret; + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, parent_rate, 0); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, + DUMMY_CLOCK_INIT_RATE - 1000, + DUMMY_CLOCK_INIT_RATE + 1000); + KUNIT_ASSERT_EQ(test, ret, 0); + + new_parent_rate = clk_get_rate(parent); + KUNIT_ASSERT_GT(test, new_parent_rate, 0); + KUNIT_EXPECT_EQ(test, parent_rate, new_parent_rate); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that, for a mux whose current parent hasn't been registered yet, + * calling clk_set_rate_range() will succeed, and will be taken into + * account when rounding a rate. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_range_round_rate(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + int ret; + + ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); +} + +/* + * Test that, for a mux that started orphan, was assigned and rate and + * then got switched to a valid parent, its rate is eventually within + * range. + * + * FIXME: Even though we update the rate as part of clk_set_parent(), we + * don't evaluate whether that new rate is within range and needs to be + * adjusted. + */ +static void +clk_test_orphan_transparent_multiple_parent_mux_set_range_set_parent_get_rate(struct kunit *test) +{ + struct clk_multiple_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long rate; + int ret; + + kunit_skip(test, "This needs to be fixed in the core."); + + clk_hw_set_rate_range(hw, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + + parent = clk_hw_get_clk(&ctx->parents_ctx[1].hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, parent); + + ret = clk_set_parent(clk, parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(parent); + clk_put(clk); +} + +static struct kunit_case clk_orphan_transparent_multiple_parent_mux_test_cases[] = { + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_get_parent), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_drop_range), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_get_rate), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_put), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_modified), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_parent_set_range_untouched), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_range_round_rate), + KUNIT_CASE(clk_test_orphan_transparent_multiple_parent_mux_set_range_set_parent_get_rate), + {} +}; + +/* + * Test suite for a basic mux clock with two parents. The default parent + * isn't registered, only the second parent is. By default, the clock + * will thus be orphan. + * + * These tests exercise the behaviour of the consumer API when dealing + * with an orphan clock, and how we deal with the transition to a valid + * parent. + */ +static struct kunit_suite clk_orphan_transparent_multiple_parent_mux_test_suite = { + .name = "clk-orphan-transparent-multiple-parent-mux-test", + .init = clk_orphan_transparent_multiple_parent_mux_test_init, + .exit = clk_orphan_transparent_multiple_parent_mux_test_exit, + .test_cases = clk_orphan_transparent_multiple_parent_mux_test_cases, +}; + +struct clk_single_parent_ctx { + struct clk_dummy_context parent_ctx; + struct clk_hw hw; +}; + +static int clk_single_parent_mux_test_init(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE; + ctx->parent_ctx.hw.init = + CLK_HW_INIT_NO_PARENT("parent-clk", + &clk_dummy_rate_ops, + 0); + + ret = clk_hw_register(NULL, &ctx->parent_ctx.hw); + if (ret) + return ret; + + ctx->hw.init = CLK_HW_INIT("test-clk", "parent-clk", + &clk_dummy_single_parent_ops, + CLK_SET_RATE_PARENT); + + ret = clk_hw_register(NULL, &ctx->hw); + if (ret) + return ret; + + return 0; +} + +static void +clk_single_parent_mux_test_exit(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + + clk_hw_unregister(&ctx->hw); + clk_hw_unregister(&ctx->parent_ctx.hw); +} + +/* + * Test that for a clock with a single parent, clk_get_parent() actually + * returns the parent. + */ +static void +clk_test_single_parent_mux_get_parent(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent = clk_hw_get_clk(&ctx->parent_ctx.hw, NULL); + + KUNIT_EXPECT_TRUE(test, clk_is_match(clk_get_parent(clk), parent)); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that for a clock with a single parent, clk_has_parent() actually + * reports it as a parent. + */ +static void +clk_test_single_parent_mux_has_parent(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent = clk_hw_get_clk(&ctx->parent_ctx.hw, NULL); + + KUNIT_EXPECT_TRUE(test, clk_has_parent(clk, parent)); + + clk_put(parent); + clk_put(clk); +} + +/* + * Test that for a clock that can't modify its rate and with a single + * parent, if we set disjoints range on the parent and then the child, + * the second will return an error. + * + * FIXME: clk_set_rate_range() only considers the current clock when + * evaluating whether ranges are disjoints and not the upstream clocks + * ranges. + */ +static void +clk_test_single_parent_mux_set_range_disjoint_child_last(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + int ret; + + kunit_skip(test, "This needs to be fixed in the core."); + + parent = clk_get_parent(clk); + KUNIT_ASSERT_PTR_NE(test, parent, NULL); + + ret = clk_set_rate_range(parent, 1000, 2000); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, 3000, 4000); + KUNIT_EXPECT_LT(test, ret, 0); + + clk_put(clk); +} + +/* + * Test that for a clock that can't modify its rate and with a single + * parent, if we set disjoints range on the child and then the parent, + * the second will return an error. + * + * FIXME: clk_set_rate_range() only considers the current clock when + * evaluating whether ranges are disjoints and not the downstream clocks + * ranges. + */ +static void +clk_test_single_parent_mux_set_range_disjoint_parent_last(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + int ret; + + kunit_skip(test, "This needs to be fixed in the core."); + + parent = clk_get_parent(clk); + KUNIT_ASSERT_PTR_NE(test, parent, NULL); + + ret = clk_set_rate_range(clk, 1000, 2000); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(parent, 3000, 4000); + KUNIT_EXPECT_LT(test, ret, 0); + + clk_put(clk); +} + +/* + * Test that for a clock that can't modify its rate and with a single + * parent, if we set a range on the parent and then call + * clk_round_rate(), the boundaries of the parent are taken into + * account. + */ +static void +clk_test_single_parent_mux_set_range_round_rate_parent_only(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long rate; + int ret; + + parent = clk_get_parent(clk); + KUNIT_ASSERT_PTR_NE(test, parent, NULL); + + ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); +} + +/* + * Test that for a clock that can't modify its rate and with a single + * parent, if we set a range on the parent and a more restrictive one on + * the child, and then call clk_round_rate(), the boundaries of the + * two clocks are taken into account. + */ +static void +clk_test_single_parent_mux_set_range_round_rate_child_smaller(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long rate; + int ret; + + parent = clk_get_parent(clk); + KUNIT_ASSERT_PTR_NE(test, parent, NULL); + + ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1 + 1000, DUMMY_CLOCK_RATE_2 - 1000); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_2 + 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000); + + clk_put(clk); +} + +/* + * Test that for a clock that can't modify its rate and with a single + * parent, if we set a range on the child and a more restrictive one on + * the parent, and then call clk_round_rate(), the boundaries of the + * two clocks are taken into account. + */ +static void +clk_test_single_parent_mux_set_range_round_rate_parent_smaller(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *parent; + unsigned long rate; + int ret; + + parent = clk_get_parent(clk); + KUNIT_ASSERT_PTR_NE(test, parent, NULL); + + ret = clk_set_rate_range(parent, DUMMY_CLOCK_RATE_1 + 1000, DUMMY_CLOCK_RATE_2 - 1000); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1, DUMMY_CLOCK_RATE_2); + KUNIT_ASSERT_EQ(test, ret, 0); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_1 - 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000); + + rate = clk_round_rate(clk, DUMMY_CLOCK_RATE_2 + 1000); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1 + 1000); + KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2 - 1000); + + clk_put(clk); +} + +static struct kunit_case clk_single_parent_mux_test_cases[] = { + KUNIT_CASE(clk_test_single_parent_mux_get_parent), + KUNIT_CASE(clk_test_single_parent_mux_has_parent), + KUNIT_CASE(clk_test_single_parent_mux_set_range_disjoint_child_last), + KUNIT_CASE(clk_test_single_parent_mux_set_range_disjoint_parent_last), + KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_child_smaller), + KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_parent_only), + KUNIT_CASE(clk_test_single_parent_mux_set_range_round_rate_parent_smaller), + {} +}; + +/* + * Test suite for a basic mux clock with one parent, with + * CLK_SET_RATE_PARENT on the child. + * + * These tests exercise the consumer API and check that the state of the + * child and parent are sane and consistent. + */ +static struct kunit_suite +clk_single_parent_mux_test_suite = { + .name = "clk-single-parent-mux-test", + .init = clk_single_parent_mux_test_init, + .exit = clk_single_parent_mux_test_exit, + .test_cases = clk_single_parent_mux_test_cases, +}; + +static int clk_orphan_transparent_single_parent_mux_test_init(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx; + struct clk_init_data init = { }; + const char * const parents[] = { "orphan_parent" }; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + init.name = "test_orphan_dummy_parent"; + init.ops = &clk_dummy_single_parent_ops; + init.parent_names = parents; + init.num_parents = ARRAY_SIZE(parents); + init.flags = CLK_SET_RATE_PARENT; + ctx->hw.init = &init; + + ret = clk_hw_register(NULL, &ctx->hw); + if (ret) + return ret; + + memset(&init, 0, sizeof(init)); + init.name = "orphan_parent"; + init.ops = &clk_dummy_rate_ops; + ctx->parent_ctx.hw.init = &init; + ctx->parent_ctx.rate = DUMMY_CLOCK_INIT_RATE; + + ret = clk_hw_register(NULL, &ctx->parent_ctx.hw); + if (ret) + return ret; + + return 0; +} + +/* + * Test that a mux-only clock, with an initial rate within a range, + * will still have the same rate after the range has been enforced. + * + * See: + * https://lore.kernel.org/linux-clk/7720158d-10a7-a17b-73a4-a8615c9c6d5c@collabora.com/ + */ +static void clk_test_orphan_transparent_parent_mux_set_range(struct kunit *test) +{ + struct clk_single_parent_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate, new_rate; + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(clk, + ctx->parent_ctx.rate - 1000, + ctx->parent_ctx.rate + 1000), + 0); + + new_rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, new_rate, 0); + KUNIT_EXPECT_EQ(test, rate, new_rate); + + clk_put(clk); +} + +static struct kunit_case clk_orphan_transparent_single_parent_mux_test_cases[] = { + KUNIT_CASE(clk_test_orphan_transparent_parent_mux_set_range), + {} +}; + +/* + * Test suite for a basic mux clock with one parent. The parent is + * registered after its child. The clock will thus be an orphan when + * registered, but will no longer be when the tests run. + * + * These tests make sure a clock that used to be orphan has a sane, + * consistent, behaviour. + */ +static struct kunit_suite clk_orphan_transparent_single_parent_test_suite = { + .name = "clk-orphan-transparent-single-parent-test", + .init = clk_orphan_transparent_single_parent_mux_test_init, + .exit = clk_single_parent_mux_test_exit, + .test_cases = clk_orphan_transparent_single_parent_mux_test_cases, +}; + +struct clk_single_parent_two_lvl_ctx { + struct clk_dummy_context parent_parent_ctx; + struct clk_dummy_context parent_ctx; + struct clk_hw hw; +}; + +static int +clk_orphan_two_level_root_last_test_init(struct kunit *test) +{ + struct clk_single_parent_two_lvl_ctx *ctx; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + ctx->parent_ctx.hw.init = + CLK_HW_INIT("intermediate-parent", + "root-parent", + &clk_dummy_single_parent_ops, + CLK_SET_RATE_PARENT); + ret = clk_hw_register(NULL, &ctx->parent_ctx.hw); + if (ret) + return ret; + + ctx->hw.init = + CLK_HW_INIT("test-clk", "intermediate-parent", + &clk_dummy_single_parent_ops, + CLK_SET_RATE_PARENT); + ret = clk_hw_register(NULL, &ctx->hw); + if (ret) + return ret; + + ctx->parent_parent_ctx.rate = DUMMY_CLOCK_INIT_RATE; + ctx->parent_parent_ctx.hw.init = + CLK_HW_INIT_NO_PARENT("root-parent", + &clk_dummy_rate_ops, + 0); + ret = clk_hw_register(NULL, &ctx->parent_parent_ctx.hw); + if (ret) + return ret; + + return 0; +} + +static void +clk_orphan_two_level_root_last_test_exit(struct kunit *test) +{ + struct clk_single_parent_two_lvl_ctx *ctx = test->priv; + + clk_hw_unregister(&ctx->hw); + clk_hw_unregister(&ctx->parent_ctx.hw); + clk_hw_unregister(&ctx->parent_parent_ctx.hw); +} + +/* + * Test that, for a clock whose parent used to be orphan, clk_get_rate() + * will return the proper rate. + */ +static void +clk_orphan_two_level_root_last_test_get_rate(struct kunit *test) +{ + struct clk_single_parent_two_lvl_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + + rate = clk_get_rate(clk); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE); + + clk_put(clk); +} + +/* + * Test that, for a clock whose parent used to be orphan, + * clk_set_rate_range() won't affect its rate if it is already within + * range. + * + * See (for Exynos 4210): + * https://lore.kernel.org/linux-clk/366a0232-bb4a-c357-6aa8-636e398e05eb@samsung.com/ */ -static void clk_test_orphan_transparent_parent_mux_set_range(struct kunit *test) +static void +clk_orphan_two_level_root_last_test_set_range(struct kunit *test) { - struct clk_single_parent_ctx *ctx = test->priv; + struct clk_single_parent_two_lvl_ctx *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; - unsigned long rate, new_rate; + struct clk *clk = clk_hw_get_clk(hw, NULL); + unsigned long rate; + int ret; + + ret = clk_set_rate_range(clk, + DUMMY_CLOCK_INIT_RATE - 1000, + DUMMY_CLOCK_INIT_RATE + 1000); + KUNIT_ASSERT_EQ(test, ret, 0); rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_INIT_RATE); - KUNIT_ASSERT_EQ(test, - clk_set_rate_range(clk, - ctx->parent_ctx.rate - 1000, - ctx->parent_ctx.rate + 1000), - 0); - - new_rate = clk_get_rate(clk); - KUNIT_ASSERT_GT(test, new_rate, 0); - KUNIT_EXPECT_EQ(test, rate, new_rate); + clk_put(clk); } -static struct kunit_case clk_orphan_transparent_single_parent_mux_test_cases[] = { - KUNIT_CASE(clk_test_orphan_transparent_parent_mux_set_range), +static struct kunit_case +clk_orphan_two_level_root_last_test_cases[] = { + KUNIT_CASE(clk_orphan_two_level_root_last_test_get_rate), + KUNIT_CASE(clk_orphan_two_level_root_last_test_set_range), {} }; -static struct kunit_suite clk_orphan_transparent_single_parent_test_suite = { - .name = "clk-orphan-transparent-single-parent-test", - .init = clk_orphan_transparent_single_parent_mux_test_init, - .exit = clk_orphan_transparent_single_parent_mux_test_exit, - .test_cases = clk_orphan_transparent_single_parent_mux_test_cases, +/* + * Test suite for a basic, transparent, clock with a parent that is also + * such a clock. The parent's parent is registered last, while the + * parent and its child are registered in that order. The intermediate + * and leaf clocks will thus be orphan when registered, but the leaf + * clock itself will always have its parent and will never be + * reparented. Indeed, it's only orphan because its parent is. + * + * These tests exercise the behaviour of the consumer API when dealing + * with an orphan clock, and how we deal with the transition to a valid + * parent. + */ +static struct kunit_suite +clk_orphan_two_level_root_last_test_suite = { + .name = "clk-orphan-two-level-root-last-test", + .init = clk_orphan_two_level_root_last_test_init, + .exit = clk_orphan_two_level_root_last_test_exit, + .test_cases = clk_orphan_two_level_root_last_test_cases, }; /* @@ -352,7 +1434,7 @@ static void clk_range_test_set_range(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -365,6 +1447,8 @@ static void clk_range_test_set_range(struct kunit *test) KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -375,13 +1459,15 @@ static void clk_range_test_set_range_invalid(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); KUNIT_EXPECT_LT(test, clk_set_rate_range(clk, DUMMY_CLOCK_RATE_1 + 1000, DUMMY_CLOCK_RATE_1), 0); + + clk_put(clk); } /* @@ -420,7 +1506,7 @@ static void clk_range_test_set_range_round_rate_lower(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); long rate; KUNIT_ASSERT_EQ(test, @@ -433,6 +1519,8 @@ static void clk_range_test_set_range_round_rate_lower(struct kunit *test) KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -443,7 +1531,7 @@ static void clk_range_test_set_range_set_rate_lower(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -460,6 +1548,8 @@ static void clk_range_test_set_range_set_rate_lower(struct kunit *test) KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -472,7 +1562,7 @@ static void clk_range_test_set_range_set_round_rate_consistent_lower(struct kuni { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); long rounded; KUNIT_ASSERT_EQ(test, @@ -489,6 +1579,8 @@ static void clk_range_test_set_range_set_round_rate_consistent_lower(struct kuni 0); KUNIT_EXPECT_EQ(test, rounded, clk_get_rate(clk)); + + clk_put(clk); } /* @@ -499,7 +1591,7 @@ static void clk_range_test_set_range_round_rate_higher(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); long rate; KUNIT_ASSERT_EQ(test, @@ -512,6 +1604,8 @@ static void clk_range_test_set_range_round_rate_higher(struct kunit *test) KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -522,7 +1616,7 @@ static void clk_range_test_set_range_set_rate_higher(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -539,6 +1633,8 @@ static void clk_range_test_set_range_set_rate_higher(struct kunit *test) KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_GE(test, rate, DUMMY_CLOCK_RATE_1); KUNIT_EXPECT_LE(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -551,7 +1647,7 @@ static void clk_range_test_set_range_set_round_rate_consistent_higher(struct kun { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); long rounded; KUNIT_ASSERT_EQ(test, @@ -568,6 +1664,8 @@ static void clk_range_test_set_range_set_round_rate_consistent_higher(struct kun 0); KUNIT_EXPECT_EQ(test, rounded, clk_get_rate(clk)); + + clk_put(clk); } /* @@ -582,7 +1680,7 @@ static void clk_range_test_set_range_get_rate_raised(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -598,6 +1696,8 @@ static void clk_range_test_set_range_get_rate_raised(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(clk); } /* @@ -612,7 +1712,7 @@ static void clk_range_test_set_range_get_rate_lowered(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -628,6 +1728,8 @@ static void clk_range_test_set_range_get_rate_lowered(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } static struct kunit_case clk_range_test_cases[] = { @@ -645,6 +1747,12 @@ static struct kunit_case clk_range_test_cases[] = { {} }; +/* + * Test suite for a basic rate clock, without any parent. + * + * These tests exercise the rate range API: clk_set_rate_range(), + * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range(). + */ static struct kunit_suite clk_range_test_suite = { .name = "clk-range-test", .init = clk_test_init, @@ -664,7 +1772,7 @@ static void clk_range_test_set_range_rate_maximized(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -700,6 +1808,8 @@ static void clk_range_test_set_range_rate_maximized(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(clk); } /* @@ -714,7 +1824,7 @@ static void clk_range_test_multiple_set_range_rate_maximized(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); struct clk *user1, *user2; unsigned long rate; @@ -758,14 +1868,79 @@ static void clk_range_test_multiple_set_range_rate_maximized(struct kunit *test) clk_put(user2); clk_put(user1); + clk_put(clk); +} + +/* + * Test that if we have several subsequent calls to + * clk_set_rate_range(), across multiple users, the core will reevaluate + * whether a new rate is needed, including when a user drop its clock. + * + * With clk_dummy_maximize_rate_ops, this means that the rate will + * trail along the maximum as it evolves. + */ +static void clk_range_test_multiple_set_range_rate_put_maximized(struct kunit *test) +{ + struct clk_dummy_context *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *user1, *user2; + unsigned long rate; + + user1 = clk_hw_get_clk(hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1); + + user2 = clk_hw_get_clk(hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2); + + KUNIT_ASSERT_EQ(test, + clk_set_rate(clk, DUMMY_CLOCK_RATE_2 + 1000), + 0); + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(user1, + 0, + DUMMY_CLOCK_RATE_2), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(user2, + 0, + DUMMY_CLOCK_RATE_1), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(user2); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(user1); + clk_put(clk); } static struct kunit_case clk_range_maximize_test_cases[] = { KUNIT_CASE(clk_range_test_set_range_rate_maximized), KUNIT_CASE(clk_range_test_multiple_set_range_rate_maximized), + KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_maximized), {} }; +/* + * Test suite for a basic rate clock, without any parent. + * + * These tests exercise the rate range API: clk_set_rate_range(), + * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range(), with a + * driver that will always try to run at the highest possible rate. + */ static struct kunit_suite clk_range_maximize_test_suite = { .name = "clk-range-maximize-test", .init = clk_maximize_test_init, @@ -785,7 +1960,7 @@ static void clk_range_test_set_range_rate_minimized(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); unsigned long rate; KUNIT_ASSERT_EQ(test, @@ -821,6 +1996,8 @@ static void clk_range_test_set_range_rate_minimized(struct kunit *test) rate = clk_get_rate(clk); KUNIT_ASSERT_GT(test, rate, 0); KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(clk); } /* @@ -835,7 +2012,7 @@ static void clk_range_test_multiple_set_range_rate_minimized(struct kunit *test) { struct clk_dummy_context *ctx = test->priv; struct clk_hw *hw = &ctx->hw; - struct clk *clk = hw->clk; + struct clk *clk = clk_hw_get_clk(hw, NULL); struct clk *user1, *user2; unsigned long rate; @@ -875,14 +2052,75 @@ static void clk_range_test_multiple_set_range_rate_minimized(struct kunit *test) clk_put(user2); clk_put(user1); + clk_put(clk); +} + +/* + * Test that if we have several subsequent calls to + * clk_set_rate_range(), across multiple users, the core will reevaluate + * whether a new rate is needed, including when a user drop its clock. + * + * With clk_dummy_minimize_rate_ops, this means that the rate will + * trail along the minimum as it evolves. + */ +static void clk_range_test_multiple_set_range_rate_put_minimized(struct kunit *test) +{ + struct clk_dummy_context *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *user1, *user2; + unsigned long rate; + + user1 = clk_hw_get_clk(hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user1); + + user2 = clk_hw_get_clk(hw, NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, user2); + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(user1, + DUMMY_CLOCK_RATE_1, + ULONG_MAX), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + KUNIT_ASSERT_EQ(test, + clk_set_rate_range(user2, + DUMMY_CLOCK_RATE_2, + ULONG_MAX), + 0); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_2); + + clk_put(user2); + + rate = clk_get_rate(clk); + KUNIT_ASSERT_GT(test, rate, 0); + KUNIT_EXPECT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_put(user1); + clk_put(clk); } static struct kunit_case clk_range_minimize_test_cases[] = { KUNIT_CASE(clk_range_test_set_range_rate_minimized), KUNIT_CASE(clk_range_test_multiple_set_range_rate_minimized), + KUNIT_CASE(clk_range_test_multiple_set_range_rate_put_minimized), {} }; +/* + * Test suite for a basic rate clock, without any parent. + * + * These tests exercise the rate range API: clk_set_rate_range(), + * clk_set_min_rate(), clk_set_max_rate(), clk_drop_range(), with a + * driver that will always try to run at the lowest possible rate. + */ static struct kunit_suite clk_range_minimize_test_suite = { .name = "clk-range-minimize-test", .init = clk_minimize_test_init, @@ -890,11 +2128,284 @@ static struct kunit_suite clk_range_minimize_test_suite = { .test_cases = clk_range_minimize_test_cases, }; +struct clk_leaf_mux_ctx { + struct clk_multiple_parent_ctx mux_ctx; + struct clk_hw hw; +}; + +static int +clk_leaf_mux_set_rate_parent_test_init(struct kunit *test) +{ + struct clk_leaf_mux_ctx *ctx; + const char *top_parents[2] = { "parent-0", "parent-1" }; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + + ctx->mux_ctx.parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0", + &clk_dummy_rate_ops, + 0); + ctx->mux_ctx.parents_ctx[0].rate = DUMMY_CLOCK_RATE_1; + ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[0].hw); + if (ret) + return ret; + + ctx->mux_ctx.parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1", + &clk_dummy_rate_ops, + 0); + ctx->mux_ctx.parents_ctx[1].rate = DUMMY_CLOCK_RATE_2; + ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[1].hw); + if (ret) + return ret; + + ctx->mux_ctx.current_parent = 0; + ctx->mux_ctx.hw.init = CLK_HW_INIT_PARENTS("test-mux", top_parents, + &clk_multiple_parents_mux_ops, + 0); + ret = clk_hw_register(NULL, &ctx->mux_ctx.hw); + if (ret) + return ret; + + ctx->hw.init = CLK_HW_INIT_HW("test-clock", &ctx->mux_ctx.hw, + &clk_dummy_single_parent_ops, + CLK_SET_RATE_PARENT); + ret = clk_hw_register(NULL, &ctx->hw); + if (ret) + return ret; + + return 0; +} + +static void clk_leaf_mux_set_rate_parent_test_exit(struct kunit *test) +{ + struct clk_leaf_mux_ctx *ctx = test->priv; + + clk_hw_unregister(&ctx->hw); + clk_hw_unregister(&ctx->mux_ctx.hw); + clk_hw_unregister(&ctx->mux_ctx.parents_ctx[0].hw); + clk_hw_unregister(&ctx->mux_ctx.parents_ctx[1].hw); +} + +/* + * Test that, for a clock that will forward any rate request to its + * parent, the rate request structure returned by __clk_determine_rate + * is sane and will be what we expect. + */ +static void clk_leaf_mux_set_rate_parent_determine_rate(struct kunit *test) +{ + struct clk_leaf_mux_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk_rate_request req; + unsigned long rate; + int ret; + + rate = clk_get_rate(clk); + KUNIT_ASSERT_EQ(test, rate, DUMMY_CLOCK_RATE_1); + + clk_hw_init_rate_request(hw, &req, DUMMY_CLOCK_RATE_2); + + ret = __clk_determine_rate(hw, &req); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_EXPECT_EQ(test, req.rate, DUMMY_CLOCK_RATE_2); + KUNIT_EXPECT_EQ(test, req.best_parent_rate, DUMMY_CLOCK_RATE_2); + KUNIT_EXPECT_PTR_EQ(test, req.best_parent_hw, &ctx->mux_ctx.hw); + + clk_put(clk); +} + +static struct kunit_case clk_leaf_mux_set_rate_parent_test_cases[] = { + KUNIT_CASE(clk_leaf_mux_set_rate_parent_determine_rate), + {} +}; + +/* + * Test suite for a clock whose parent is a mux with multiple parents. + * The leaf clock has CLK_SET_RATE_PARENT, and will forward rate + * requests to the mux, which will then select which parent is the best + * fit for a given rate. + * + * These tests exercise the behaviour of muxes, and the proper selection + * of parents. + */ +static struct kunit_suite clk_leaf_mux_set_rate_parent_test_suite = { + .name = "clk-leaf-mux-set-rate-parent", + .init = clk_leaf_mux_set_rate_parent_test_init, + .exit = clk_leaf_mux_set_rate_parent_test_exit, + .test_cases = clk_leaf_mux_set_rate_parent_test_cases, +}; + +struct clk_mux_notifier_rate_change { + bool done; + unsigned long old_rate; + unsigned long new_rate; + wait_queue_head_t wq; +}; + +struct clk_mux_notifier_ctx { + struct clk_multiple_parent_ctx mux_ctx; + struct clk *clk; + struct notifier_block clk_nb; + struct clk_mux_notifier_rate_change pre_rate_change; + struct clk_mux_notifier_rate_change post_rate_change; +}; + +#define NOTIFIER_TIMEOUT_MS 100 + +static int clk_mux_notifier_callback(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct clk_notifier_data *clk_data = data; + struct clk_mux_notifier_ctx *ctx = container_of(nb, + struct clk_mux_notifier_ctx, + clk_nb); + + if (action & PRE_RATE_CHANGE) { + ctx->pre_rate_change.old_rate = clk_data->old_rate; + ctx->pre_rate_change.new_rate = clk_data->new_rate; + ctx->pre_rate_change.done = true; + wake_up_interruptible(&ctx->pre_rate_change.wq); + } + + if (action & POST_RATE_CHANGE) { + ctx->post_rate_change.old_rate = clk_data->old_rate; + ctx->post_rate_change.new_rate = clk_data->new_rate; + ctx->post_rate_change.done = true; + wake_up_interruptible(&ctx->post_rate_change.wq); + } + + return 0; +} + +static int clk_mux_notifier_test_init(struct kunit *test) +{ + struct clk_mux_notifier_ctx *ctx; + const char *top_parents[2] = { "parent-0", "parent-1" }; + int ret; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + test->priv = ctx; + ctx->clk_nb.notifier_call = clk_mux_notifier_callback; + init_waitqueue_head(&ctx->pre_rate_change.wq); + init_waitqueue_head(&ctx->post_rate_change.wq); + + ctx->mux_ctx.parents_ctx[0].hw.init = CLK_HW_INIT_NO_PARENT("parent-0", + &clk_dummy_rate_ops, + 0); + ctx->mux_ctx.parents_ctx[0].rate = DUMMY_CLOCK_RATE_1; + ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[0].hw); + if (ret) + return ret; + + ctx->mux_ctx.parents_ctx[1].hw.init = CLK_HW_INIT_NO_PARENT("parent-1", + &clk_dummy_rate_ops, + 0); + ctx->mux_ctx.parents_ctx[1].rate = DUMMY_CLOCK_RATE_2; + ret = clk_hw_register(NULL, &ctx->mux_ctx.parents_ctx[1].hw); + if (ret) + return ret; + + ctx->mux_ctx.current_parent = 0; + ctx->mux_ctx.hw.init = CLK_HW_INIT_PARENTS("test-mux", top_parents, + &clk_multiple_parents_mux_ops, + 0); + ret = clk_hw_register(NULL, &ctx->mux_ctx.hw); + if (ret) + return ret; + + ctx->clk = clk_hw_get_clk(&ctx->mux_ctx.hw, NULL); + ret = clk_notifier_register(ctx->clk, &ctx->clk_nb); + if (ret) + return ret; + + return 0; +} + +static void clk_mux_notifier_test_exit(struct kunit *test) +{ + struct clk_mux_notifier_ctx *ctx = test->priv; + struct clk *clk = ctx->clk; + + clk_notifier_unregister(clk, &ctx->clk_nb); + clk_put(clk); + + clk_hw_unregister(&ctx->mux_ctx.hw); + clk_hw_unregister(&ctx->mux_ctx.parents_ctx[0].hw); + clk_hw_unregister(&ctx->mux_ctx.parents_ctx[1].hw); +} + +/* + * Test that if the we have a notifier registered on a mux, the core + * will notify us when we switch to another parent, and with the proper + * old and new rates. + */ +static void clk_mux_notifier_set_parent_test(struct kunit *test) +{ + struct clk_mux_notifier_ctx *ctx = test->priv; + struct clk_hw *hw = &ctx->mux_ctx.hw; + struct clk *clk = clk_hw_get_clk(hw, NULL); + struct clk *new_parent = clk_hw_get_clk(&ctx->mux_ctx.parents_ctx[1].hw, NULL); + int ret; + + ret = clk_set_parent(clk, new_parent); + KUNIT_ASSERT_EQ(test, ret, 0); + + ret = wait_event_interruptible_timeout(ctx->pre_rate_change.wq, + ctx->pre_rate_change.done, + msecs_to_jiffies(NOTIFIER_TIMEOUT_MS)); + KUNIT_ASSERT_GT(test, ret, 0); + + KUNIT_EXPECT_EQ(test, ctx->pre_rate_change.old_rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_EQ(test, ctx->pre_rate_change.new_rate, DUMMY_CLOCK_RATE_2); + + ret = wait_event_interruptible_timeout(ctx->post_rate_change.wq, + ctx->post_rate_change.done, + msecs_to_jiffies(NOTIFIER_TIMEOUT_MS)); + KUNIT_ASSERT_GT(test, ret, 0); + + KUNIT_EXPECT_EQ(test, ctx->post_rate_change.old_rate, DUMMY_CLOCK_RATE_1); + KUNIT_EXPECT_EQ(test, ctx->post_rate_change.new_rate, DUMMY_CLOCK_RATE_2); + + clk_put(new_parent); + clk_put(clk); +} + +static struct kunit_case clk_mux_notifier_test_cases[] = { + KUNIT_CASE(clk_mux_notifier_set_parent_test), + {} +}; + +/* + * Test suite for a mux with multiple parents, and a notifier registered + * on the mux. + * + * These tests exercise the behaviour of notifiers. + */ +static struct kunit_suite clk_mux_notifier_test_suite = { + .name = "clk-mux-notifier", + .init = clk_mux_notifier_test_init, + .exit = clk_mux_notifier_test_exit, + .test_cases = clk_mux_notifier_test_cases, +}; + kunit_test_suites( + &clk_leaf_mux_set_rate_parent_test_suite, &clk_test_suite, + &clk_multiple_parents_mux_test_suite, + &clk_mux_notifier_test_suite, + &clk_orphan_transparent_multiple_parent_mux_test_suite, &clk_orphan_transparent_single_parent_test_suite, + &clk_orphan_two_level_root_last_test_suite, &clk_range_test_suite, &clk_range_maximize_test_suite, - &clk_range_minimize_test_suite + &clk_range_minimize_test_suite, + &clk_single_parent_mux_test_suite, + &clk_uncached_test_suite ); MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index 67f601a4102395e67ba4ef710379a5fb3518cc1b..ee37d0be6877db7070f51ad264bdc1a9da02d36a 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -165,7 +165,7 @@ vclkdev_alloc(struct clk_hw *hw, const char *con_id, const char *dev_fmt, cla->cl.clk_hw = hw; if (con_id) { - strlcpy(cla->con_id, con_id, sizeof(cla->con_id)); + strscpy(cla->con_id, con_id, sizeof(cla->con_id)); cla->cl.con_id = cla->con_id; } @@ -346,45 +346,11 @@ int clk_hw_register_clkdev(struct clk_hw *hw, const char *con_id, } EXPORT_SYMBOL(clk_hw_register_clkdev); -static void devm_clkdev_release(struct device *dev, void *res) +static void devm_clkdev_release(void *res) { - clkdev_drop(*(struct clk_lookup **)res); -} - -static int devm_clk_match_clkdev(struct device *dev, void *res, void *data) -{ - struct clk_lookup **l = res; - - return *l == data; + clkdev_drop(res); } -/** - * devm_clk_release_clkdev - Resource managed clkdev lookup release - * @dev: device this lookup is bound - * @con_id: connection ID string on device - * @dev_id: format string describing device name - * - * Drop the clkdev lookup created with devm_clk_hw_register_clkdev. - * Normally this function will not need to be called and the resource - * management code will ensure that the resource is freed. - */ -void devm_clk_release_clkdev(struct device *dev, const char *con_id, - const char *dev_id) -{ - struct clk_lookup *cl; - int rval; - - mutex_lock(&clocks_mutex); - cl = clk_find(dev_id, con_id); - mutex_unlock(&clocks_mutex); - - WARN_ON(!cl); - rval = devres_release(dev, devm_clkdev_release, - devm_clk_match_clkdev, cl); - WARN_ON(rval); -} -EXPORT_SYMBOL(devm_clk_release_clkdev); - /** * devm_clk_hw_register_clkdev - managed clk lookup registration for clk_hw * @dev: device this lookup is bound @@ -403,17 +369,13 @@ EXPORT_SYMBOL(devm_clk_release_clkdev); int devm_clk_hw_register_clkdev(struct device *dev, struct clk_hw *hw, const char *con_id, const char *dev_id) { - int rval = -ENOMEM; - struct clk_lookup **cl; - - cl = devres_alloc(devm_clkdev_release, sizeof(*cl), GFP_KERNEL); - if (cl) { - rval = do_clk_register_clkdev(hw, cl, con_id, dev_id); - if (!rval) - devres_add(dev, cl); - else - devres_free(cl); - } - return rval; + struct clk_lookup *cl; + int rval; + + rval = do_clk_register_clkdev(hw, &cl, con_id, dev_id); + if (rval) + return rval; + + return devm_add_action_or_reset(dev, devm_clkdev_release, cl); } EXPORT_SYMBOL(devm_clk_hw_register_clkdev); diff --git a/drivers/clk/davinci/Makefile b/drivers/clk/davinci/Makefile index 11178b79b48390e6ab48fb7c642de7576703c299..be6f55d37b4911526f1d9bbf60eced64da7ee801 100644 --- a/drivers/clk/davinci/Makefile +++ b/drivers/clk/davinci/Makefile @@ -8,14 +8,10 @@ obj-$(CONFIG_ARCH_DAVINCI_DA830) += pll-da830.o obj-$(CONFIG_ARCH_DAVINCI_DA850) += pll-da850.o obj-$(CONFIG_ARCH_DAVINCI_DM355) += pll-dm355.o obj-$(CONFIG_ARCH_DAVINCI_DM365) += pll-dm365.o -obj-$(CONFIG_ARCH_DAVINCI_DM644x) += pll-dm644x.o -obj-$(CONFIG_ARCH_DAVINCI_DM646x) += pll-dm646x.o obj-y += psc.o obj-$(CONFIG_ARCH_DAVINCI_DA830) += psc-da830.o obj-$(CONFIG_ARCH_DAVINCI_DA850) += psc-da850.o obj-$(CONFIG_ARCH_DAVINCI_DM355) += psc-dm355.o obj-$(CONFIG_ARCH_DAVINCI_DM365) += psc-dm365.o -obj-$(CONFIG_ARCH_DAVINCI_DM644x) += psc-dm644x.o -obj-$(CONFIG_ARCH_DAVINCI_DM646x) += psc-dm646x.o endif diff --git a/drivers/clk/davinci/da8xx-cfgchip.c b/drivers/clk/davinci/da8xx-cfgchip.c index 77d18276bfe82c24f9e3559570f8ff8820bc4fd3..4103d605e804fced5cd6bf325b6ce6674fa96a95 100644 --- a/drivers/clk/davinci/da8xx-cfgchip.c +++ b/drivers/clk/davinci/da8xx-cfgchip.c @@ -510,8 +510,7 @@ da8xx_cfgchip_register_usb0_clk48(struct device *dev, fck_clk = devm_clk_get(dev, "fck"); if (IS_ERR(fck_clk)) { - if (PTR_ERR(fck_clk) != -EPROBE_DEFER) - dev_err(dev, "Missing fck clock\n"); + dev_err_probe(dev, PTR_ERR(fck_clk), "Missing fck clock\n"); return ERR_CAST(fck_clk); } diff --git a/drivers/clk/davinci/pll-dm644x.c b/drivers/clk/davinci/pll-dm644x.c deleted file mode 100644 index 7650fadfaac86d8bb0a27c6658e33da6141983fc..0000000000000000000000000000000000000000 --- a/drivers/clk/davinci/pll-dm644x.c +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * PLL clock descriptions for TI DM644X - * - * Copyright (C) 2018 David Lechner - */ - -#include -#include -#include -#include -#include - -#include "pll.h" - -static const struct davinci_pll_clk_info dm644x_pll1_info = { - .name = "pll1", - .pllm_mask = GENMASK(4, 0), - .pllm_min = 1, - .pllm_max = 32, - .pllout_min_rate = 400000000, - .pllout_max_rate = 600000000, /* 810MHz @ 1.3V, -810 only */ - .flags = PLL_HAS_CLKMODE | PLL_HAS_POSTDIV, -}; - -SYSCLK(1, pll1_sysclk1, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(2, pll1_sysclk2, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(3, pll1_sysclk3, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(5, pll1_sysclk5, pll1_pllen, 4, SYSCLK_FIXED_DIV); - -int dm644x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip) -{ - struct clk *clk; - - davinci_pll_clk_register(dev, &dm644x_pll1_info, "ref_clk", base, cfgchip); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base); - clk_register_clkdev(clk, "pll1_sysclk1", "dm644x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk2, base); - clk_register_clkdev(clk, "pll1_sysclk2", "dm644x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk3, base); - clk_register_clkdev(clk, "pll1_sysclk3", "dm644x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk5, base); - clk_register_clkdev(clk, "pll1_sysclk5", "dm644x-psc"); - - clk = davinci_pll_auxclk_register(dev, "pll1_auxclk", base); - clk_register_clkdev(clk, "pll1_auxclk", "dm644x-psc"); - - davinci_pll_sysclkbp_clk_register(dev, "pll1_sysclkbp", base); - - return 0; -} - -static const struct davinci_pll_clk_info dm644x_pll2_info = { - .name = "pll2", - .pllm_mask = GENMASK(4, 0), - .pllm_min = 1, - .pllm_max = 32, - .pllout_min_rate = 400000000, - .pllout_max_rate = 900000000, - .flags = PLL_HAS_POSTDIV | PLL_POSTDIV_FIXED_DIV, -}; - -SYSCLK(1, pll2_sysclk1, pll2_pllen, 4, 0); -SYSCLK(2, pll2_sysclk2, pll2_pllen, 4, 0); - -int dm644x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip) -{ - davinci_pll_clk_register(dev, &dm644x_pll2_info, "oscin", base, cfgchip); - - davinci_pll_sysclk_register(dev, &pll2_sysclk1, base); - - davinci_pll_sysclk_register(dev, &pll2_sysclk2, base); - - davinci_pll_sysclkbp_clk_register(dev, "pll2_sysclkbp", base); - - return 0; -} diff --git a/drivers/clk/davinci/pll-dm646x.c b/drivers/clk/davinci/pll-dm646x.c deleted file mode 100644 index 26982970df0e6c18442f75f683afcae7e4b04e9e..0000000000000000000000000000000000000000 --- a/drivers/clk/davinci/pll-dm646x.c +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * PLL clock descriptions for TI DM646X - * - * Copyright (C) 2018 David Lechner - */ - -#include -#include -#include -#include -#include - -#include "pll.h" - -static const struct davinci_pll_clk_info dm646x_pll1_info = { - .name = "pll1", - .pllm_mask = GENMASK(4, 0), - .pllm_min = 14, - .pllm_max = 32, - .flags = PLL_HAS_CLKMODE, -}; - -SYSCLK(1, pll1_sysclk1, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(2, pll1_sysclk2, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(3, pll1_sysclk3, pll1_pllen, 4, SYSCLK_FIXED_DIV); -SYSCLK(4, pll1_sysclk4, pll1_pllen, 4, 0); -SYSCLK(5, pll1_sysclk5, pll1_pllen, 4, 0); -SYSCLK(6, pll1_sysclk6, pll1_pllen, 4, 0); -SYSCLK(8, pll1_sysclk8, pll1_pllen, 4, 0); -SYSCLK(9, pll1_sysclk9, pll1_pllen, 4, 0); - -int dm646x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip) -{ - struct clk *clk; - - davinci_pll_clk_register(dev, &dm646x_pll1_info, "ref_clk", base, cfgchip); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk1, base); - clk_register_clkdev(clk, "pll1_sysclk1", "dm646x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk2, base); - clk_register_clkdev(clk, "pll1_sysclk2", "dm646x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk3, base); - clk_register_clkdev(clk, "pll1_sysclk3", "dm646x-psc"); - clk_register_clkdev(clk, NULL, "davinci-wdt"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk4, base); - clk_register_clkdev(clk, "pll1_sysclk4", "dm646x-psc"); - - clk = davinci_pll_sysclk_register(dev, &pll1_sysclk5, base); - clk_register_clkdev(clk, "pll1_sysclk5", "dm646x-psc"); - - davinci_pll_sysclk_register(dev, &pll1_sysclk6, base); - - davinci_pll_sysclk_register(dev, &pll1_sysclk8, base); - - davinci_pll_sysclk_register(dev, &pll1_sysclk9, base); - - davinci_pll_sysclkbp_clk_register(dev, "pll1_sysclkbp", base); - - davinci_pll_auxclk_register(dev, "pll1_auxclk", base); - - return 0; -} - -static const struct davinci_pll_clk_info dm646x_pll2_info = { - .name = "pll2", - .pllm_mask = GENMASK(4, 0), - .pllm_min = 14, - .pllm_max = 32, - .flags = 0, -}; - -SYSCLK(1, pll2_sysclk1, pll2_pllen, 4, SYSCLK_ALWAYS_ENABLED); - -int dm646x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip) -{ - davinci_pll_clk_register(dev, &dm646x_pll2_info, "oscin", base, cfgchip); - - davinci_pll_sysclk_register(dev, &pll2_sysclk1, base); - - return 0; -} diff --git a/drivers/clk/davinci/pll.c b/drivers/clk/davinci/pll.c index 0d750433eb42d8b65101d24231103a40f5815aab..f862f5e2b3fcb13e140a8b29ae099aff9e851c3f 100644 --- a/drivers/clk/davinci/pll.c +++ b/drivers/clk/davinci/pll.c @@ -98,7 +98,7 @@ * @hw: clk_hw for the pll * @base: Base memory address * @pllm_min: The minimum allowable PLLM[PLLM] value - * @pllm_max: The maxiumum allowable PLLM[PLLM] value + * @pllm_max: The maximum allowable PLLM[PLLM] value * @pllm_mask: Bitmask for PLLM[PLLM] value */ struct davinci_pll_clk { @@ -889,14 +889,6 @@ static const struct platform_device_id davinci_pll_id_table[] = { #ifdef CONFIG_ARCH_DAVINCI_DM365 { .name = "dm365-pll1", .driver_data = (kernel_ulong_t)dm365_pll1_init }, { .name = "dm365-pll2", .driver_data = (kernel_ulong_t)dm365_pll2_init }, -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM644x - { .name = "dm644x-pll1", .driver_data = (kernel_ulong_t)dm644x_pll1_init }, - { .name = "dm644x-pll2", .driver_data = (kernel_ulong_t)dm644x_pll2_init }, -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM646x - { .name = "dm646x-pll1", .driver_data = (kernel_ulong_t)dm646x_pll1_init }, - { .name = "dm646x-pll2", .driver_data = (kernel_ulong_t)dm646x_pll2_init }, #endif { } }; diff --git a/drivers/clk/davinci/pll.h b/drivers/clk/davinci/pll.h index c2a453caa131112a0a56ab8fdb9da5023790c5dd..1773277bc69009a239013cea72079ea2cfd06070 100644 --- a/drivers/clk/davinci/pll.h +++ b/drivers/clk/davinci/pll.h @@ -130,11 +130,5 @@ int of_da850_pll1_init(struct device *dev, void __iomem *base, struct regmap *cf #ifdef CONFIG_ARCH_DAVINCI_DM355 int dm355_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); #endif -#ifdef CONFIG_ARCH_DAVINCI_DM644x -int dm644x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM646x -int dm646x_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); -#endif #endif /* __CLK_DAVINCI_PLL_H___ */ diff --git a/drivers/clk/davinci/psc-dm644x.c b/drivers/clk/davinci/psc-dm644x.c deleted file mode 100644 index 0cea6e0bd5f06f0af4af7ba82559f261ddcfdd14..0000000000000000000000000000000000000000 --- a/drivers/clk/davinci/psc-dm644x.c +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * PSC clock descriptions for TI DaVinci DM644x - * - * Copyright (C) 2018 David Lechner - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "psc.h" - -LPSC_CLKDEV1(vpss_master_clkdev, "master", "vpss"); -LPSC_CLKDEV1(vpss_slave_clkdev, "slave", "vpss"); -LPSC_CLKDEV2(emac_clkdev, NULL, "davinci_emac.1", - "fck", "davinci_mdio.0"); -LPSC_CLKDEV1(usb_clkdev, "usb", NULL); -LPSC_CLKDEV1(ide_clkdev, NULL, "palm_bk3710"); -LPSC_CLKDEV2(aemif_clkdev, "aemif", NULL, - NULL, "ti-aemif"); -LPSC_CLKDEV1(mmcsd_clkdev, NULL, "dm6441-mmc.0"); -LPSC_CLKDEV1(asp0_clkdev, NULL, "davinci-mcbsp"); -LPSC_CLKDEV1(i2c_clkdev, NULL, "i2c_davinci.1"); -LPSC_CLKDEV1(uart0_clkdev, NULL, "serial8250.0"); -LPSC_CLKDEV1(uart1_clkdev, NULL, "serial8250.1"); -LPSC_CLKDEV1(uart2_clkdev, NULL, "serial8250.2"); -/* REVISIT: gpio-davinci.c should be modified to drop con_id */ -LPSC_CLKDEV1(gpio_clkdev, "gpio", NULL); -LPSC_CLKDEV1(timer0_clkdev, "timer0", NULL); -LPSC_CLKDEV1(timer2_clkdev, NULL, "davinci-wdt"); - -static const struct davinci_lpsc_clk_info dm644x_psc_info[] = { - LPSC(0, 0, vpss_master, pll1_sysclk3, vpss_master_clkdev, 0), - LPSC(1, 0, vpss_slave, pll1_sysclk3, vpss_slave_clkdev, 0), - LPSC(6, 0, emac, pll1_sysclk5, emac_clkdev, 0), - LPSC(9, 0, usb, pll1_sysclk5, usb_clkdev, 0), - LPSC(10, 0, ide, pll1_sysclk5, ide_clkdev, 0), - LPSC(11, 0, vlynq, pll1_sysclk5, NULL, 0), - LPSC(14, 0, aemif, pll1_sysclk5, aemif_clkdev, 0), - LPSC(15, 0, mmcsd, pll1_sysclk5, mmcsd_clkdev, 0), - LPSC(17, 0, asp0, pll1_sysclk5, asp0_clkdev, 0), - LPSC(18, 0, i2c, pll1_auxclk, i2c_clkdev, 0), - LPSC(19, 0, uart0, pll1_auxclk, uart0_clkdev, 0), - LPSC(20, 0, uart1, pll1_auxclk, uart1_clkdev, 0), - LPSC(21, 0, uart2, pll1_auxclk, uart2_clkdev, 0), - LPSC(22, 0, spi, pll1_sysclk5, NULL, 0), - LPSC(23, 0, pwm0, pll1_auxclk, NULL, 0), - LPSC(24, 0, pwm1, pll1_auxclk, NULL, 0), - LPSC(25, 0, pwm2, pll1_auxclk, NULL, 0), - LPSC(26, 0, gpio, pll1_sysclk5, gpio_clkdev, 0), - LPSC(27, 0, timer0, pll1_auxclk, timer0_clkdev, LPSC_ALWAYS_ENABLED), - LPSC(28, 0, timer1, pll1_auxclk, NULL, 0), - /* REVISIT: why can't this be disabled? */ - LPSC(29, 0, timer2, pll1_auxclk, timer2_clkdev, LPSC_ALWAYS_ENABLED), - LPSC(31, 0, arm, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - /* REVISIT how to disable? */ - LPSC(39, 1, dsp, pll1_sysclk1, NULL, LPSC_ALWAYS_ENABLED), - /* REVISIT how to disable? */ - LPSC(40, 1, vicp, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - { } -}; - -int dm644x_psc_init(struct device *dev, void __iomem *base) -{ - return davinci_psc_register_clocks(dev, dm644x_psc_info, 41, base); -} - -static struct clk_bulk_data dm644x_psc_parent_clks[] = { - { .id = "pll1_sysclk1" }, - { .id = "pll1_sysclk2" }, - { .id = "pll1_sysclk3" }, - { .id = "pll1_sysclk5" }, - { .id = "pll1_auxclk" }, -}; - -const struct davinci_psc_init_data dm644x_psc_init_data = { - .parent_clks = dm644x_psc_parent_clks, - .num_parent_clks = ARRAY_SIZE(dm644x_psc_parent_clks), - .psc_init = &dm644x_psc_init, -}; diff --git a/drivers/clk/davinci/psc-dm646x.c b/drivers/clk/davinci/psc-dm646x.c deleted file mode 100644 index 20012dc7471a2873e700190401c44ee715b475fd..0000000000000000000000000000000000000000 --- a/drivers/clk/davinci/psc-dm646x.c +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * PSC clock descriptions for TI DaVinci DM646x - * - * Copyright (C) 2018 David Lechner - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "psc.h" - -LPSC_CLKDEV1(ide_clkdev, NULL, "palm_bk3710"); -LPSC_CLKDEV2(emac_clkdev, NULL, "davinci_emac.1", - "fck", "davinci_mdio.0"); -LPSC_CLKDEV2(aemif_clkdev, "aemif", NULL, - NULL, "ti-aemif"); -LPSC_CLKDEV1(mcasp0_clkdev, NULL, "davinci-mcasp.0"); -LPSC_CLKDEV1(mcasp1_clkdev, NULL, "davinci-mcasp.1"); -LPSC_CLKDEV1(uart0_clkdev, NULL, "serial8250.0"); -LPSC_CLKDEV1(uart1_clkdev, NULL, "serial8250.1"); -LPSC_CLKDEV1(uart2_clkdev, NULL, "serial8250.2"); -LPSC_CLKDEV1(i2c_clkdev, NULL, "i2c_davinci.1"); -/* REVISIT: gpio-davinci.c should be modified to drop con_id */ -LPSC_CLKDEV1(gpio_clkdev, "gpio", NULL); -LPSC_CLKDEV1(timer0_clkdev, "timer0", NULL); - -static const struct davinci_lpsc_clk_info dm646x_psc_info[] = { - LPSC(0, 0, arm, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - /* REVISIT how to disable? */ - LPSC(1, 0, dsp, pll1_sysclk1, NULL, LPSC_ALWAYS_ENABLED), - LPSC(4, 0, edma_cc, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - LPSC(5, 0, edma_tc0, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - LPSC(6, 0, edma_tc1, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - LPSC(7, 0, edma_tc2, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - LPSC(8, 0, edma_tc3, pll1_sysclk2, NULL, LPSC_ALWAYS_ENABLED), - LPSC(10, 0, ide, pll1_sysclk4, ide_clkdev, 0), - LPSC(14, 0, emac, pll1_sysclk3, emac_clkdev, 0), - LPSC(16, 0, vpif0, ref_clk, NULL, LPSC_ALWAYS_ENABLED), - LPSC(17, 0, vpif1, ref_clk, NULL, LPSC_ALWAYS_ENABLED), - LPSC(21, 0, aemif, pll1_sysclk3, aemif_clkdev, LPSC_ALWAYS_ENABLED), - LPSC(22, 0, mcasp0, pll1_sysclk3, mcasp0_clkdev, 0), - LPSC(23, 0, mcasp1, pll1_sysclk3, mcasp1_clkdev, 0), - LPSC(26, 0, uart0, aux_clkin, uart0_clkdev, 0), - LPSC(27, 0, uart1, aux_clkin, uart1_clkdev, 0), - LPSC(28, 0, uart2, aux_clkin, uart2_clkdev, 0), - /* REVIST: disabling hangs system */ - LPSC(29, 0, pwm0, pll1_sysclk3, NULL, LPSC_ALWAYS_ENABLED), - /* REVIST: disabling hangs system */ - LPSC(30, 0, pwm1, pll1_sysclk3, NULL, LPSC_ALWAYS_ENABLED), - LPSC(31, 0, i2c, pll1_sysclk3, i2c_clkdev, 0), - LPSC(33, 0, gpio, pll1_sysclk3, gpio_clkdev, 0), - LPSC(34, 0, timer0, pll1_sysclk3, timer0_clkdev, LPSC_ALWAYS_ENABLED), - LPSC(35, 0, timer1, pll1_sysclk3, NULL, 0), - { } -}; - -int dm646x_psc_init(struct device *dev, void __iomem *base) -{ - return davinci_psc_register_clocks(dev, dm646x_psc_info, 46, base); -} - -static struct clk_bulk_data dm646x_psc_parent_clks[] = { - { .id = "ref_clk" }, - { .id = "aux_clkin" }, - { .id = "pll1_sysclk1" }, - { .id = "pll1_sysclk2" }, - { .id = "pll1_sysclk3" }, - { .id = "pll1_sysclk4" }, - { .id = "pll1_sysclk5" }, -}; - -const struct davinci_psc_init_data dm646x_psc_init_data = { - .parent_clks = dm646x_psc_parent_clks, - .num_parent_clks = ARRAY_SIZE(dm646x_psc_parent_clks), - .psc_init = &dm646x_psc_init, -}; diff --git a/drivers/clk/davinci/psc.c b/drivers/clk/davinci/psc.c index 7387e7f6276eb40a4a24f5e354edd1fbb9f8c304..42a59dbd49c8bc70144066f075751c5f3b4544e3 100644 --- a/drivers/clk/davinci/psc.c +++ b/drivers/clk/davinci/psc.c @@ -516,12 +516,6 @@ static const struct platform_device_id davinci_psc_id_table[] = { #endif #ifdef CONFIG_ARCH_DAVINCI_DM365 { .name = "dm365-psc", .driver_data = (kernel_ulong_t)&dm365_psc_init_data }, -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM644x - { .name = "dm644x-psc", .driver_data = (kernel_ulong_t)&dm644x_psc_init_data }, -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM646x - { .name = "dm646x-psc", .driver_data = (kernel_ulong_t)&dm646x_psc_init_data }, #endif { } }; diff --git a/drivers/clk/davinci/psc.h b/drivers/clk/davinci/psc.h index 69070f834391a34fcdb650110bf8e5cb80b8d6e0..5e382b6755187ec256697cd1014fcacc9c401761 100644 --- a/drivers/clk/davinci/psc.h +++ b/drivers/clk/davinci/psc.h @@ -110,11 +110,5 @@ extern const struct davinci_psc_init_data dm355_psc_init_data; #ifdef CONFIG_ARCH_DAVINCI_DM365 extern const struct davinci_psc_init_data dm365_psc_init_data; #endif -#ifdef CONFIG_ARCH_DAVINCI_DM644x -extern const struct davinci_psc_init_data dm644x_psc_init_data; -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM646x -extern const struct davinci_psc_init_data dm646x_psc_init_data; -#endif #endif /* __CLK_DAVINCI_PSC_H__ */ diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile index 88b9b9285d22e07f925aa80e1bbe290bc57e1faa..e8aacb0ee6ac6232d4615849b37d62b99bec54b1 100644 --- a/drivers/clk/imx/Makefile +++ b/drivers/clk/imx/Makefile @@ -12,6 +12,7 @@ mxc-clk-objs += clk-fixup-div.o mxc-clk-objs += clk-fixup-mux.o mxc-clk-objs += clk-frac-pll.o mxc-clk-objs += clk-gate2.o +mxc-clk-objs += clk-gate-93.o mxc-clk-objs += clk-gate-exclusive.o mxc-clk-objs += clk-pfd.o mxc-clk-objs += clk-pfdv2.o diff --git a/drivers/clk/imx/clk-composite-93.c b/drivers/clk/imx/clk-composite-93.c index b44619aa5ca5ec7f779cba38909b36669b8e00fe..74a66b0203e4bd7fe1da0bd5f238ce8246426484 100644 --- a/drivers/clk/imx/clk-composite-93.c +++ b/drivers/clk/imx/clk-composite-93.c @@ -9,22 +9,180 @@ #include #include #include +#include #include #include "clk.h" +#define TIMEOUT_US 500U + #define CCM_DIV_SHIFT 0 #define CCM_DIV_WIDTH 8 #define CCM_MUX_SHIFT 8 #define CCM_MUX_MASK 3 #define CCM_OFF_SHIFT 24 +#define CCM_BUSY_SHIFT 28 +#define STAT_OFFSET 0x4 #define AUTHEN_OFFSET 0x30 #define TZ_NS_SHIFT 9 #define TZ_NS_MASK BIT(9) +#define WHITE_LIST_SHIFT 16 + +static int imx93_clk_composite_wait_ready(struct clk_hw *hw, void __iomem *reg) +{ + int ret; + u32 val; + + ret = readl_poll_timeout_atomic(reg + STAT_OFFSET, val, !(val & BIT(CCM_BUSY_SHIFT)), + 0, TIMEOUT_US); + if (ret) + pr_err("Slice[%s] busy timeout\n", clk_hw_get_name(hw)); + + return ret; +} + +static void imx93_clk_composite_gate_endisable(struct clk_hw *hw, int enable) +{ + struct clk_gate *gate = to_clk_gate(hw); + unsigned long flags; + u32 reg; + + if (gate->lock) + spin_lock_irqsave(gate->lock, flags); + + reg = readl(gate->reg); + + if (enable) + reg &= ~BIT(gate->bit_idx); + else + reg |= BIT(gate->bit_idx); + + writel(reg, gate->reg); + + imx93_clk_composite_wait_ready(hw, gate->reg); + + if (gate->lock) + spin_unlock_irqrestore(gate->lock, flags); +} + +static int imx93_clk_composite_gate_enable(struct clk_hw *hw) +{ + imx93_clk_composite_gate_endisable(hw, 1); + + return 0; +} + +static void imx93_clk_composite_gate_disable(struct clk_hw *hw) +{ + imx93_clk_composite_gate_endisable(hw, 0); +} + +static const struct clk_ops imx93_clk_composite_gate_ops = { + .enable = imx93_clk_composite_gate_enable, + .disable = imx93_clk_composite_gate_disable, + .is_enabled = clk_gate_is_enabled, +}; + +static unsigned long +imx93_clk_composite_divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) +{ + return clk_divider_ops.recalc_rate(hw, parent_rate); +} + +static long +imx93_clk_composite_divider_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *prate) +{ + return clk_divider_ops.round_rate(hw, rate, prate); +} + +static int +imx93_clk_composite_divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + return clk_divider_ops.determine_rate(hw, req); +} + +static int imx93_clk_composite_divider_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_divider *divider = to_clk_divider(hw); + int value; + unsigned long flags = 0; + u32 val; + int ret; + + value = divider_get_val(rate, parent_rate, divider->table, divider->width, divider->flags); + if (value < 0) + return value; + + if (divider->lock) + spin_lock_irqsave(divider->lock, flags); + + val = readl(divider->reg); + val &= ~(clk_div_mask(divider->width) << divider->shift); + val |= (u32)value << divider->shift; + writel(val, divider->reg); + + ret = imx93_clk_composite_wait_ready(hw, divider->reg); + + if (divider->lock) + spin_unlock_irqrestore(divider->lock, flags); + + return ret; +} + +static const struct clk_ops imx93_clk_composite_divider_ops = { + .recalc_rate = imx93_clk_composite_divider_recalc_rate, + .round_rate = imx93_clk_composite_divider_round_rate, + .determine_rate = imx93_clk_composite_divider_determine_rate, + .set_rate = imx93_clk_composite_divider_set_rate, +}; + +static u8 imx93_clk_composite_mux_get_parent(struct clk_hw *hw) +{ + return clk_mux_ops.get_parent(hw); +} + +static int imx93_clk_composite_mux_set_parent(struct clk_hw *hw, u8 index) +{ + struct clk_mux *mux = to_clk_mux(hw); + u32 val = clk_mux_index_to_val(mux->table, mux->flags, index); + unsigned long flags = 0; + u32 reg; + int ret; + + if (mux->lock) + spin_lock_irqsave(mux->lock, flags); + + reg = readl(mux->reg); + reg &= ~(mux->mask << mux->shift); + val = val << mux->shift; + reg |= val; + writel(reg, mux->reg); + + ret = imx93_clk_composite_wait_ready(hw, mux->reg); + + if (mux->lock) + spin_unlock_irqrestore(mux->lock, flags); + + return ret; +} + +static int +imx93_clk_composite_mux_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) +{ + return clk_mux_ops.determine_rate(hw, req); +} + +static const struct clk_ops imx93_clk_composite_mux_ops = { + .get_parent = imx93_clk_composite_mux_get_parent, + .set_parent = imx93_clk_composite_mux_set_parent, + .determine_rate = imx93_clk_composite_mux_determine_rate, +}; + struct clk_hw *imx93_clk_composite_flags(const char *name, const char * const *parent_names, - int num_parents, void __iomem *reg, + int num_parents, void __iomem *reg, u32 domain_id, unsigned long flags) { struct clk_hw *hw = ERR_PTR(-ENOMEM), *mux_hw; @@ -33,6 +191,7 @@ struct clk_hw *imx93_clk_composite_flags(const char *name, const char * const *p struct clk_gate *gate = NULL; struct clk_mux *mux = NULL; bool clk_ro = false; + u32 authen; mux = kzalloc(sizeof(*mux), GFP_KERNEL); if (!mux) @@ -55,7 +214,8 @@ struct clk_hw *imx93_clk_composite_flags(const char *name, const char * const *p div->lock = &imx_ccm_lock; div->flags = CLK_DIVIDER_ROUND_CLOSEST; - if (!(readl(reg + AUTHEN_OFFSET) & TZ_NS_MASK)) + authen = readl(reg + AUTHEN_OFFSET); + if (!(authen & TZ_NS_MASK) || !(authen & BIT(WHITE_LIST_SHIFT + domain_id))) clk_ro = true; if (clk_ro) { @@ -74,9 +234,10 @@ struct clk_hw *imx93_clk_composite_flags(const char *name, const char * const *p gate->flags = CLK_GATE_SET_TO_DISABLE; hw = clk_hw_register_composite(NULL, name, parent_names, num_parents, - mux_hw, &clk_mux_ops, div_hw, - &clk_divider_ops, gate_hw, - &clk_gate_ops, flags | CLK_SET_RATE_NO_REPARENT); + mux_hw, &imx93_clk_composite_mux_ops, div_hw, + &imx93_clk_composite_divider_ops, gate_hw, + &imx93_clk_composite_gate_ops, + flags | CLK_SET_RATE_NO_REPARENT); } if (IS_ERR(hw)) diff --git a/drivers/clk/imx/clk-gate-93.c b/drivers/clk/imx/clk-gate-93.c new file mode 100644 index 0000000000000000000000000000000000000000..ceb56b290394e4951286792309bfbca69cee89f7 --- /dev/null +++ b/drivers/clk/imx/clk-gate-93.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2022 NXP + * + * Peng Fan + */ + +#include +#include +#include +#include +#include +#include + +#include "clk.h" + +#define DIRECT_OFFSET 0x0 + +/* + * 0b000 - LPCG will be OFF in any CPU mode. + * 0b100 - LPCG will be ON in any CPU mode. + */ +#define LPM_SETTING_OFF 0x0 +#define LPM_SETTING_ON 0x4 + +#define LPM_CUR_OFFSET 0x1c + +#define AUTHEN_OFFSET 0x30 +#define CPULPM_EN BIT(2) +#define TZ_NS_SHIFT 9 +#define TZ_NS_MASK BIT(9) + +#define WHITE_LIST_SHIFT 16 + +struct imx93_clk_gate { + struct clk_hw hw; + void __iomem *reg; + u32 bit_idx; + u32 val; + u32 mask; + spinlock_t *lock; + unsigned int *share_count; +}; + +#define to_imx93_clk_gate(_hw) container_of(_hw, struct imx93_clk_gate, hw) + +static void imx93_clk_gate_do_hardware(struct clk_hw *hw, bool enable) +{ + struct imx93_clk_gate *gate = to_imx93_clk_gate(hw); + u32 val; + + val = readl(gate->reg + AUTHEN_OFFSET); + if (val & CPULPM_EN) { + val = enable ? LPM_SETTING_ON : LPM_SETTING_OFF; + writel(val, gate->reg + LPM_CUR_OFFSET); + } else { + val = readl(gate->reg + DIRECT_OFFSET); + val &= ~(gate->mask << gate->bit_idx); + if (enable) + val |= (gate->val & gate->mask) << gate->bit_idx; + writel(val, gate->reg + DIRECT_OFFSET); + } +} + +static int imx93_clk_gate_enable(struct clk_hw *hw) +{ + struct imx93_clk_gate *gate = to_imx93_clk_gate(hw); + unsigned long flags; + + spin_lock_irqsave(gate->lock, flags); + + if (gate->share_count && (*gate->share_count)++ > 0) + goto out; + + imx93_clk_gate_do_hardware(hw, true); +out: + spin_unlock_irqrestore(gate->lock, flags); + + return 0; +} + +static void imx93_clk_gate_disable(struct clk_hw *hw) +{ + struct imx93_clk_gate *gate = to_imx93_clk_gate(hw); + unsigned long flags; + + spin_lock_irqsave(gate->lock, flags); + + if (gate->share_count) { + if (WARN_ON(*gate->share_count == 0)) + goto out; + else if (--(*gate->share_count) > 0) + goto out; + } + + imx93_clk_gate_do_hardware(hw, false); +out: + spin_unlock_irqrestore(gate->lock, flags); +} + +static int imx93_clk_gate_reg_is_enabled(struct imx93_clk_gate *gate) +{ + u32 val = readl(gate->reg + AUTHEN_OFFSET); + + if (val & CPULPM_EN) { + val = readl(gate->reg + LPM_CUR_OFFSET); + if (val == LPM_SETTING_ON) + return 1; + } else { + val = readl(gate->reg); + if (((val >> gate->bit_idx) & gate->mask) == gate->val) + return 1; + } + + return 0; +} + +static int imx93_clk_gate_is_enabled(struct clk_hw *hw) +{ + struct imx93_clk_gate *gate = to_imx93_clk_gate(hw); + unsigned long flags; + int ret; + + spin_lock_irqsave(gate->lock, flags); + + ret = imx93_clk_gate_reg_is_enabled(gate); + + spin_unlock_irqrestore(gate->lock, flags); + + return ret; +} + +static void imx93_clk_gate_disable_unused(struct clk_hw *hw) +{ + struct imx93_clk_gate *gate = to_imx93_clk_gate(hw); + unsigned long flags; + + spin_lock_irqsave(gate->lock, flags); + + if (!gate->share_count || *gate->share_count == 0) + imx93_clk_gate_do_hardware(hw, false); + + spin_unlock_irqrestore(gate->lock, flags); +} + +static const struct clk_ops imx93_clk_gate_ops = { + .enable = imx93_clk_gate_enable, + .disable = imx93_clk_gate_disable, + .disable_unused = imx93_clk_gate_disable_unused, + .is_enabled = imx93_clk_gate_is_enabled, +}; + +static const struct clk_ops imx93_clk_gate_ro_ops = { + .is_enabled = imx93_clk_gate_is_enabled, +}; + +struct clk_hw *imx93_clk_gate(struct device *dev, const char *name, const char *parent_name, + unsigned long flags, void __iomem *reg, u32 bit_idx, u32 val, + u32 mask, u32 domain_id, unsigned int *share_count) +{ + struct imx93_clk_gate *gate; + struct clk_hw *hw; + struct clk_init_data init; + int ret; + u32 authen; + + gate = kzalloc(sizeof(struct imx93_clk_gate), GFP_KERNEL); + if (!gate) + return ERR_PTR(-ENOMEM); + + gate->reg = reg; + gate->lock = &imx_ccm_lock; + gate->bit_idx = bit_idx; + gate->val = val; + gate->mask = mask; + gate->share_count = share_count; + + init.name = name; + init.ops = &imx93_clk_gate_ops; + init.flags = flags | CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE; + init.parent_names = parent_name ? &parent_name : NULL; + init.num_parents = parent_name ? 1 : 0; + + gate->hw.init = &init; + hw = &gate->hw; + + authen = readl(reg + AUTHEN_OFFSET); + if (!(authen & TZ_NS_MASK) || !(authen & BIT(WHITE_LIST_SHIFT + domain_id))) + init.ops = &imx93_clk_gate_ro_ops; + + ret = clk_hw_register(dev, hw); + if (ret) { + kfree(gate); + return ERR_PTR(ret); + } + + return hw; +} +EXPORT_SYMBOL_GPL(imx93_clk_gate); diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c index fc1bd23d458340d85d4321e6f01df7cf92b646c9..598f3cf4eba49d51f41e996323d0554bd88d0ae5 100644 --- a/drivers/clk/imx/clk-imx6sx.c +++ b/drivers/clk/imx/clk-imx6sx.c @@ -280,13 +280,13 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node) hws[IMX6SX_CLK_SSI3_SEL] = imx_clk_hw_mux("ssi3_sel", base + 0x1c, 14, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); hws[IMX6SX_CLK_SSI2_SEL] = imx_clk_hw_mux("ssi2_sel", base + 0x1c, 12, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); hws[IMX6SX_CLK_SSI1_SEL] = imx_clk_hw_mux("ssi1_sel", base + 0x1c, 10, 2, ssi_sels, ARRAY_SIZE(ssi_sels)); - hws[IMX6SX_CLK_QSPI1_SEL] = imx_clk_hw_mux_flags("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_QSPI1_SEL] = imx_clk_hw_mux("qspi1_sel", base + 0x1c, 7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels)); hws[IMX6SX_CLK_PERCLK_SEL] = imx_clk_hw_mux("perclk_sel", base + 0x1c, 6, 1, perclk_sels, ARRAY_SIZE(perclk_sels)); hws[IMX6SX_CLK_VID_SEL] = imx_clk_hw_mux("vid_sel", base + 0x20, 21, 3, vid_sels, ARRAY_SIZE(vid_sels)); hws[IMX6SX_CLK_ESAI_SEL] = imx_clk_hw_mux("esai_sel", base + 0x20, 19, 2, audio_sels, ARRAY_SIZE(audio_sels)); hws[IMX6SX_CLK_CAN_SEL] = imx_clk_hw_mux("can_sel", base + 0x20, 8, 2, can_sels, ARRAY_SIZE(can_sels)); hws[IMX6SX_CLK_UART_SEL] = imx_clk_hw_mux("uart_sel", base + 0x24, 6, 1, uart_sels, ARRAY_SIZE(uart_sels)); - hws[IMX6SX_CLK_QSPI2_SEL] = imx_clk_hw_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT); + hws[IMX6SX_CLK_QSPI2_SEL] = imx_clk_hw_mux("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels)); hws[IMX6SX_CLK_SPDIF_SEL] = imx_clk_hw_mux("spdif_sel", base + 0x30, 20, 2, audio_sels, ARRAY_SIZE(audio_sels)); hws[IMX6SX_CLK_AUDIO_SEL] = imx_clk_hw_mux("audio_sel", base + 0x30, 7, 2, audio_sels, ARRAY_SIZE(audio_sels)); hws[IMX6SX_CLK_ENET_PRE_SEL] = imx_clk_hw_mux("enet_pre_sel", base + 0x34, 15, 3, enet_pre_sels, ARRAY_SIZE(enet_pre_sels)); diff --git a/drivers/clk/imx/clk-imx8mp.c b/drivers/clk/imx/clk-imx8mp.c index e89db568f5a897d8f93a825fbbc0ea0bbe5beb91..652ae58c2735fffb51c9c20b82d4c1a64c467f23 100644 --- a/drivers/clk/imx/clk-imx8mp.c +++ b/drivers/clk/imx/clk-imx8mp.c @@ -665,8 +665,8 @@ static int imx8mp_clocks_probe(struct platform_device *pdev) hws[IMX8MP_CLK_CAN1_ROOT] = imx_clk_hw_gate2("can1_root_clk", "can1", ccm_base + 0x4350, 0); hws[IMX8MP_CLK_CAN2_ROOT] = imx_clk_hw_gate2("can2_root_clk", "can2", ccm_base + 0x4360, 0); hws[IMX8MP_CLK_SDMA1_ROOT] = imx_clk_hw_gate4("sdma1_root_clk", "ipg_root", ccm_base + 0x43a0, 0); - hws[IMX8MP_CLK_ENET_QOS_ROOT] = imx_clk_hw_gate4("enet_qos_root_clk", "sim_enet_root_clk", ccm_base + 0x43b0, 0); hws[IMX8MP_CLK_SIM_ENET_ROOT] = imx_clk_hw_gate4("sim_enet_root_clk", "enet_axi", ccm_base + 0x4400, 0); + hws[IMX8MP_CLK_ENET_QOS_ROOT] = imx_clk_hw_gate4("enet_qos_root_clk", "sim_enet_root_clk", ccm_base + 0x43b0, 0); hws[IMX8MP_CLK_GPU2D_ROOT] = imx_clk_hw_gate4("gpu2d_root_clk", "gpu2d_core", ccm_base + 0x4450, 0); hws[IMX8MP_CLK_GPU3D_ROOT] = imx_clk_hw_gate4("gpu3d_root_clk", "gpu3d_core", ccm_base + 0x4460, 0); hws[IMX8MP_CLK_UART1_ROOT] = imx_clk_hw_gate4("uart1_root_clk", "uart1", ccm_base + 0x4490, 0); diff --git a/drivers/clk/imx/clk-imx93.c b/drivers/clk/imx/clk-imx93.c index f5c9fa40491c5f9f00b06b3530ad8f68969a8e50..99cff1fd108b5b7c68d25c86b39e152770ac5872 100644 --- a/drivers/clk/imx/clk-imx93.c +++ b/drivers/clk/imx/clk-imx93.c @@ -28,6 +28,11 @@ enum clk_sel { MAX_SEL }; +static u32 share_count_sai1; +static u32 share_count_sai2; +static u32 share_count_sai3; +static u32 share_count_mub; + static const char *parent_names[MAX_SEL][4] = { {"osc_24m", "sys_pll_pfd0_div2", "sys_pll_pfd1_div2", "video_pll"}, {"osc_24m", "sys_pll_pfd0_div2", "sys_pll_pfd1_div2", "sys_pll_pfd2_div2"}, @@ -146,6 +151,7 @@ static const struct imx93_clk_ccgr { char *parent_name; u32 off; unsigned long flags; + u32 *shared_count; } ccgr_array[] = { { IMX93_CLK_A55_GATE, "a55", "a55_root", 0x8000, }, /* M33 critical clk for system run */ @@ -158,8 +164,10 @@ static const struct imx93_clk_ccgr { { IMX93_CLK_WDOG5_GATE, "wdog5", "osc_24m", 0x8400, }, { IMX93_CLK_SEMA1_GATE, "sema1", "bus_aon_root", 0x8440, }, { IMX93_CLK_SEMA2_GATE, "sema2", "bus_wakeup_root", 0x8480, }, - { IMX93_CLK_MU_A_GATE, "mu_a", "bus_aon_root", 0x84c0, }, - { IMX93_CLK_MU_B_GATE, "mu_b", "bus_aon_root", 0x8500, }, + { IMX93_CLK_MU1_A_GATE, "mu1_a", "bus_aon_root", 0x84c0, CLK_IGNORE_UNUSED }, + { IMX93_CLK_MU2_A_GATE, "mu2_a", "bus_wakeup_root", 0x84c0, CLK_IGNORE_UNUSED }, + { IMX93_CLK_MU1_B_GATE, "mu1_b", "bus_aon_root", 0x8500, 0, &share_count_mub }, + { IMX93_CLK_MU2_B_GATE, "mu2_b", "bus_wakeup_root", 0x8500, 0, &share_count_mub }, { IMX93_CLK_EDMA1_GATE, "edma1", "m33_root", 0x8540, }, { IMX93_CLK_EDMA2_GATE, "edma2", "wakeup_axi_root", 0x8580, }, { IMX93_CLK_FLEXSPI1_GATE, "flexspi", "flexspi_root", 0x8640, }, @@ -210,9 +218,12 @@ static const struct imx93_clk_ccgr { { IMX93_CLK_USDHC1_GATE, "usdhc1", "usdhc1_root", 0x9380, }, { IMX93_CLK_USDHC2_GATE, "usdhc2", "usdhc2_root", 0x93c0, }, { IMX93_CLK_USDHC3_GATE, "usdhc3", "usdhc3_root", 0x9400, }, - { IMX93_CLK_SAI1_GATE, "sai1", "sai1_root", 0x9440, }, - { IMX93_CLK_SAI2_GATE, "sai2", "sai2_root", 0x9480, }, - { IMX93_CLK_SAI3_GATE, "sai3", "sai3_root", 0x94c0, }, + { IMX93_CLK_SAI1_GATE, "sai1", "sai1_root", 0x9440, 0, &share_count_sai1}, + { IMX93_CLK_SAI1_IPG, "sai1_ipg_clk", "bus_aon_root", 0x9440, 0, &share_count_sai1}, + { IMX93_CLK_SAI2_GATE, "sai2", "sai2_root", 0x9480, 0, &share_count_sai2}, + { IMX93_CLK_SAI2_IPG, "sai2_ipg_clk", "bus_wakeup_root", 0x9480, 0, &share_count_sai2}, + { IMX93_CLK_SAI3_GATE, "sai3", "sai3_root", 0x94c0, 0, &share_count_sai3}, + { IMX93_CLK_SAI3_IPG, "sai3_ipg_clk", "bus_wakeup_root", 0x94c0, 0, &share_count_sai3}, { IMX93_CLK_MIPI_CSI_GATE, "mipi_csi", "media_apb_root", 0x9580, }, { IMX93_CLK_MIPI_DSI_GATE, "mipi_dsi", "media_apb_root", 0x95c0, }, { IMX93_CLK_LVDS_GATE, "lvds", "media_ldb_root", 0x9600, }, @@ -293,16 +304,15 @@ static int imx93_clocks_probe(struct platform_device *pdev) root = &root_array[i]; clks[root->clk] = imx93_clk_composite_flags(root->name, parent_names[root->sel], - 4, base + root->off, + 4, base + root->off, 3, root->flags); } for (i = 0; i < ARRAY_SIZE(ccgr_array); i++) { ccgr = &ccgr_array[i]; - clks[ccgr->clk] = imx_clk_hw_gate4_flags(ccgr->name, - ccgr->parent_name, - base + ccgr->off, 0, - ccgr->flags); + clks[ccgr->clk] = imx93_clk_gate(NULL, ccgr->name, ccgr->parent_name, + ccgr->flags, base + ccgr->off, 0, 1, 1, 3, + ccgr->shared_count); } imx_check_clk_hws(clks, IMX93_CLK_END); @@ -332,7 +342,7 @@ static struct platform_driver imx93_clk_driver = { .driver = { .name = "imx93-ccm", .suppress_bind_attrs = true, - .of_match_table = of_match_ptr(imx93_clk_of_match), + .of_match_table = imx93_clk_of_match, }, }; module_platform_driver(imx93_clk_driver); diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index c56e406138dbe28a7dff5d1c987e22a58ba75b29..1e6870f3671f6b0b15a31a38d9d3e6d1ec54b298 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -695,7 +695,11 @@ struct clk_hw *imx_clk_scu_alloc_dev(const char *name, pr_warn("%s: failed to attached the power domain %d\n", name, ret); - platform_device_add(pdev); + ret = platform_device_add(pdev); + if (ret) { + platform_device_put(pdev); + return ERR_PTR(ret); + } /* For API backwards compatiblilty, simply return NULL for success */ return NULL; diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h index 5061a06468df8d0bc78329179b30a323ba87eb0a..dd49f90110e8b0173323aadc54a451fa394f98de 100644 --- a/drivers/clk/imx/clk.h +++ b/drivers/clk/imx/clk.h @@ -445,11 +445,16 @@ struct clk_hw *imx93_clk_composite_flags(const char *name, const char * const *parent_names, int num_parents, void __iomem *reg, + u32 domain_id, unsigned long flags); -#define imx93_clk_composite(name, parent_names, num_parents, reg) \ - imx93_clk_composite_flags(name, parent_names, num_parents, reg, \ +#define imx93_clk_composite(name, parent_names, num_parents, reg, domain_id) \ + imx93_clk_composite_flags(name, parent_names, num_parents, reg, domain_id \ CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE) +struct clk_hw *imx93_clk_gate(struct device *dev, const char *name, const char *parent_name, + unsigned long flags, void __iomem *reg, u32 bit_idx, u32 val, + u32 mask, u32 domain_id, unsigned int *share_count); + struct clk_hw *imx_clk_hw_divider_gate(const char *name, const char *parent_name, unsigned long flags, void __iomem *reg, u8 shift, u8 width, u8 clk_divider_flags, const struct clk_div_table *table, diff --git a/drivers/clk/ingenic/tcu.c b/drivers/clk/ingenic/tcu.c index 201bf6e6b6e0ffb60cd34c0ecfbd1fa31673dd37..d5544cbc5c48416d2bf8ff9aeb7c8896d4050296 100644 --- a/drivers/clk/ingenic/tcu.c +++ b/drivers/clk/ingenic/tcu.c @@ -101,15 +101,11 @@ static bool ingenic_tcu_enable_regs(struct clk_hw *hw) bool enabled = false; /* - * If the SoC has no global TCU clock, we must ungate the channel's - * clock to be able to access its registers. - * If we have a TCU clock, it will be enabled automatically as it has - * been attached to the regmap. + * According to the programming manual, a timer channel's registers can + * only be accessed when the channel's stop bit is clear. */ - if (!tcu->clk) { - enabled = !!ingenic_tcu_is_enabled(hw); - regmap_write(tcu->map, TCU_REG_TSCR, BIT(info->gate_bit)); - } + enabled = !!ingenic_tcu_is_enabled(hw); + regmap_write(tcu->map, TCU_REG_TSCR, BIT(info->gate_bit)); return enabled; } @@ -120,8 +116,7 @@ static void ingenic_tcu_disable_regs(struct clk_hw *hw) const struct ingenic_tcu_clk_info *info = tcu_clk->info; struct ingenic_tcu *tcu = tcu_clk->tcu; - if (!tcu->clk) - regmap_write(tcu->map, TCU_REG_TSSR, BIT(info->gate_bit)); + regmap_write(tcu->map, TCU_REG_TSSR, BIT(info->gate_bit)); } static u8 ingenic_tcu_get_parent(struct clk_hw *hw) diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index d5936cfb3beeee582fb885f354b1f8e839061452..843cea0c7a4408f601ab3e4c34d99dcaa65a61ba 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -259,6 +259,43 @@ config COMMON_CLK_MT6779_AUDSYS help This driver supports Mediatek MT6779 audsys clocks. +config COMMON_CLK_MT6795 + tristate "Clock driver for MediaTek MT6795" + depends on ARCH_MEDIATEK || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK + help + This driver supports MediaTek MT6795 basic clocks and clocks + required for various peripherals found on MediaTek. + +config COMMON_CLK_MT6795_MFGCFG + tristate "Clock driver for MediaTek MT6795 mfgcfg" + depends on COMMON_CLK_MT6795 + default COMMON_CLK_MT6795 + help + This driver supports MediaTek MT6795 mfgcfg clocks. + +config COMMON_CLK_MT6795_MMSYS + tristate "Clock driver for MediaTek MT6795 mmsys" + depends on COMMON_CLK_MT6795 + default COMMON_CLK_MT6795 + help + This driver supports MediaTek MT6795 mmsys clocks. + +config COMMON_CLK_MT6795_VDECSYS + tristate "Clock driver for MediaTek MT6795 VDECSYS" + depends on COMMON_CLK_MT6795 + default COMMON_CLK_MT6795 + help + This driver supports MediaTek MT6795 vdecsys clocks. + +config COMMON_CLK_MT6795_VENCSYS + tristate "Clock driver for MediaTek MT6795 VENCSYS" + depends on COMMON_CLK_MT6795 + default COMMON_CLK_MT6795 + help + This driver supports MediaTek MT6795 vencsys clocks. + config COMMON_CLK_MT6797 bool "Clock driver for MediaTek MT6797" depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST @@ -608,6 +645,56 @@ config COMMON_CLK_MT8195 help This driver supports MediaTek MT8195 clocks. +config COMMON_CLK_MT8365 + tristate "Clock driver for MediaTek MT8365" + depends on ARCH_MEDIATEK || COMPILE_TEST + select COMMON_CLK_MEDIATEK + default ARCH_MEDIATEK && ARM64 + help + This driver supports MediaTek MT8365 basic clocks. + +config COMMON_CLK_MT8365_APU + tristate "Clock driver for MediaTek MT8365 apu" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 apu clocks. + +config COMMON_CLK_MT8365_CAM + tristate "Clock driver for MediaTek MT8365 cam" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 cam clocks. + +config COMMON_CLK_MT8365_MFG + tristate "Clock driver for MediaTek MT8365 mfg" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 mfg clocks. + +config COMMON_CLK_MT8365_MMSYS + tristate "Clock driver for MediaTek MT8365 mmsys" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 mmsys clocks. + +config COMMON_CLK_MT8365_VDEC + tristate "Clock driver for MediaTek MT8365 vdec" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 vdec clocks. + +config COMMON_CLK_MT8365_VENC + tristate "Clock driver for MediaTek MT8365 venc" + depends on COMMON_CLK_MT8365 + default COMMON_CLK_MT8365 + help + This driver supports MediaTek MT8365 venc clocks. + config COMMON_CLK_MT8516 bool "Clock driver for MediaTek MT8516" depends on ARCH_MEDIATEK || COMPILE_TEST diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile index caf2ce93d6668c21b80f71434420459131279be1..ea3b732403033e2ee199b2675bf946b547fb98fd 100644 --- a/drivers/clk/mediatek/Makefile +++ b/drivers/clk/mediatek/Makefile @@ -17,6 +17,12 @@ obj-$(CONFIG_COMMON_CLK_MT6779_VDECSYS) += clk-mt6779-vdec.o obj-$(CONFIG_COMMON_CLK_MT6779_VENCSYS) += clk-mt6779-venc.o obj-$(CONFIG_COMMON_CLK_MT6779_MFGCFG) += clk-mt6779-mfg.o obj-$(CONFIG_COMMON_CLK_MT6779_AUDSYS) += clk-mt6779-aud.o +obj-$(CONFIG_COMMON_CLK_MT6795) += clk-mt6795-apmixedsys.o clk-mt6795-infracfg.o \ + clk-mt6795-pericfg.o clk-mt6795-topckgen.o +obj-$(CONFIG_COMMON_CLK_MT6795_MFGCFG) += clk-mt6795-mfg.o +obj-$(CONFIG_COMMON_CLK_MT6795_MMSYS) += clk-mt6795-mm.o +obj-$(CONFIG_COMMON_CLK_MT6795_VDECSYS) += clk-mt6795-vdecsys.o +obj-$(CONFIG_COMMON_CLK_MT6795_VENCSYS) += clk-mt6795-vencsys.o obj-$(CONFIG_COMMON_CLK_MT6797) += clk-mt6797.o obj-$(CONFIG_COMMON_CLK_MT6797_IMGSYS) += clk-mt6797-img.o obj-$(CONFIG_COMMON_CLK_MT6797_MMSYS) += clk-mt6797-mm.o @@ -97,5 +103,12 @@ obj-$(CONFIG_COMMON_CLK_MT8195) += clk-mt8195-apmixedsys.o clk-mt8195-topckgen.o clk-mt8195-venc.o clk-mt8195-vpp0.o clk-mt8195-vpp1.o \ clk-mt8195-wpe.o clk-mt8195-imp_iic_wrap.o \ clk-mt8195-apusys_pll.o +obj-$(CONFIG_COMMON_CLK_MT8365) += clk-mt8365.o +obj-$(CONFIG_COMMON_CLK_MT8365_APU) += clk-mt8365-apu.o +obj-$(CONFIG_COMMON_CLK_MT8365_CAM) += clk-mt8365-cam.o +obj-$(CONFIG_COMMON_CLK_MT8365_MFG) += clk-mt8365-mfg.o +obj-$(CONFIG_COMMON_CLK_MT8365_MMSYS) += clk-mt8365-mm.o +obj-$(CONFIG_COMMON_CLK_MT8365_VDEC) += clk-mt8365-vdec.o +obj-$(CONFIG_COMMON_CLK_MT8365_VENC) += clk-mt8365-venc.o obj-$(CONFIG_COMMON_CLK_MT8516) += clk-mt8516.o obj-$(CONFIG_COMMON_CLK_MT8516_AUDSYS) += clk-mt8516-aud.o diff --git a/drivers/clk/mediatek/clk-apmixed.c b/drivers/clk/mediatek/clk-apmixed.c index fc3d4146f4827e4504156f8e1320155249b66faa..60e34f124250ddceaf97373397718c1c235f3c90 100644 --- a/drivers/clk/mediatek/clk-apmixed.c +++ b/drivers/clk/mediatek/clk-apmixed.c @@ -70,7 +70,7 @@ static const struct clk_ops mtk_ref2usb_tx_ops = { .unprepare = mtk_ref2usb_tx_unprepare, }; -struct clk_hw * __init mtk_clk_register_ref2usb_tx(const char *name, +struct clk_hw *mtk_clk_register_ref2usb_tx(const char *name, const char *parent_name, void __iomem *reg) { struct mtk_ref2usb_tx *tx; @@ -98,5 +98,15 @@ struct clk_hw * __init mtk_clk_register_ref2usb_tx(const char *name, return &tx->hw; } +EXPORT_SYMBOL_GPL(mtk_clk_register_ref2usb_tx); + +void mtk_clk_unregister_ref2usb_tx(struct clk_hw *hw) +{ + struct mtk_ref2usb_tx *tx = to_mtk_ref2usb_tx(hw); + + clk_hw_unregister(hw); + kfree(tx); +} +EXPORT_SYMBOL_GPL(mtk_clk_unregister_ref2usb_tx); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-cpumux.c b/drivers/clk/mediatek/clk-cpumux.c index 2b5d48591738830c37b77e98b0b01224298ed0a1..25618eff6f2aa2b3193f69e6d9d6a605791c19b4 100644 --- a/drivers/clk/mediatek/clk-cpumux.c +++ b/drivers/clk/mediatek/clk-cpumux.c @@ -150,6 +150,7 @@ err: return PTR_ERR(hw); } +EXPORT_SYMBOL_GPL(mtk_clk_register_cpumuxes); void mtk_clk_unregister_cpumuxes(const struct mtk_composite *clks, int num, struct clk_hw_onecell_data *clk_data) @@ -166,5 +167,6 @@ void mtk_clk_unregister_cpumuxes(const struct mtk_composite *clks, int num, clk_data->hws[mux->id] = ERR_PTR(-ENOENT); } } +EXPORT_SYMBOL_GPL(mtk_clk_unregister_cpumuxes); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-gate.c b/drivers/clk/mediatek/clk-gate.c index 421806236228f466b665627edde942803f956a06..0c867136e49d79a05ceb0391621c529d53ba1b11 100644 --- a/drivers/clk/mediatek/clk-gate.c +++ b/drivers/clk/mediatek/clk-gate.c @@ -261,6 +261,7 @@ err: return PTR_ERR(hw); } +EXPORT_SYMBOL_GPL(mtk_clk_register_gates_with_dev); int mtk_clk_register_gates(struct device_node *node, const struct mtk_gate *clks, int num, diff --git a/drivers/clk/mediatek/clk-mt2701-bdp.c b/drivers/clk/mediatek/clk-mt2701-bdp.c index 662a8ab3fbb18a828048edf3241a12cb90d4df58..435ed4819d5633e05e3eaef111a0487ef2efc54a 100644 --- a/drivers/clk/mediatek/clk-mt2701-bdp.c +++ b/drivers/clk/mediatek/clk-mt2701-bdp.c @@ -94,33 +94,23 @@ static const struct mtk_gate bdp_clks[] = { GATE_BDP1(CLK_BDP_HDMI_MON, "hdmi_mon", "hdmi_0_pll340m", 16), }; -static const struct of_device_id of_match_clk_mt2701_bdp[] = { - { .compatible = "mediatek,mt2701-bdpsys", }, - {} +static const struct mtk_clk_desc bdp_desc = { + .clks = bdp_clks, + .num_clks = ARRAY_SIZE(bdp_clks), }; -static int clk_mt2701_bdp_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_BDP_NR); - - mtk_clk_register_gates(node, bdp_clks, ARRAY_SIZE(bdp_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt2701_bdp[] = { + { + .compatible = "mediatek,mt2701-bdpsys", + .data = &bdp_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt2701_bdp_drv = { - .probe = clk_mt2701_bdp_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2701-bdp", .of_match_table = of_match_clk_mt2701_bdp, diff --git a/drivers/clk/mediatek/clk-mt2701-img.c b/drivers/clk/mediatek/clk-mt2701-img.c index c4f3cd26df60b51b2fd744fb5a7fa261fb750f55..7e53deb7f9905d2f17153962366d6e6fdaeffc2a 100644 --- a/drivers/clk/mediatek/clk-mt2701-img.c +++ b/drivers/clk/mediatek/clk-mt2701-img.c @@ -36,33 +36,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_VENC, "img_venc", "mm_sel", 9), }; -static const struct of_device_id of_match_clk_mt2701_img[] = { - { .compatible = "mediatek,mt2701-imgsys", }, - {} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), }; -static int clk_mt2701_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt2701_img[] = { + { + .compatible = "mediatek,mt2701-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt2701_img_drv = { - .probe = clk_mt2701_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2701-img", .of_match_table = of_match_clk_mt2701_img, diff --git a/drivers/clk/mediatek/clk-mt2701-vdec.c b/drivers/clk/mediatek/clk-mt2701-vdec.c index a2f18117f27a602ef312e723799a9edb8d32ddb6..d3089da0ab62eb3f96016850af8f18eaef65da78 100644 --- a/drivers/clk/mediatek/clk-mt2701-vdec.c +++ b/drivers/clk/mediatek/clk-mt2701-vdec.c @@ -47,33 +47,23 @@ static const struct mtk_gate vdec_clks[] = { GATE_VDEC1(CLK_VDEC_LARB, "vdec_larb_cken", "mm_sel", 0), }; -static const struct of_device_id of_match_clk_mt2701_vdec[] = { - { .compatible = "mediatek,mt2701-vdecsys", }, - {} +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), }; -static int clk_mt2701_vdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VDEC_NR); - - mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt2701_vdec[] = { + { + .compatible = "mediatek,mt2701-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt2701_vdec_drv = { - .probe = clk_mt2701_vdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2701-vdec", .of_match_table = of_match_clk_mt2701_vdec, diff --git a/drivers/clk/mediatek/clk-mt2712-bdp.c b/drivers/clk/mediatek/clk-mt2712-bdp.c index 9acab43571337c9d39e3e22943a3d36ae042499d..684d03e9f6de17e4d70db7d3788152b489dc35bc 100644 --- a/drivers/clk/mediatek/clk-mt2712-bdp.c +++ b/drivers/clk/mediatek/clk-mt2712-bdp.c @@ -58,33 +58,23 @@ static const struct mtk_gate bdp_clks[] = { GATE_BDP(CLK_BDP_TVD_CBUS, "bdp_tvd_cbus", "mm_sel", 30), }; -static int clk_mt2712_bdp_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_BDP_NR_CLK); - - mtk_clk_register_gates(node, bdp_clks, ARRAY_SIZE(bdp_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc bdp_desc = { + .clks = bdp_clks, + .num_clks = ARRAY_SIZE(bdp_clks), +}; static const struct of_device_id of_match_clk_mt2712_bdp[] = { - { .compatible = "mediatek,mt2712-bdpsys", }, - {} + { + .compatible = "mediatek,mt2712-bdpsys", + .data = &bdp_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_bdp_drv = { - .probe = clk_mt2712_bdp_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-bdp", .of_match_table = of_match_clk_mt2712_bdp, diff --git a/drivers/clk/mediatek/clk-mt2712-img.c b/drivers/clk/mediatek/clk-mt2712-img.c index 5cc143e65e423dfab458fb02146e9f3e0ceb36cf..335049cdc856c6f2a97e29d7fe12b708c577264a 100644 --- a/drivers/clk/mediatek/clk-mt2712-img.c +++ b/drivers/clk/mediatek/clk-mt2712-img.c @@ -36,33 +36,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_CAM_SV2_EN, "img_cam_sv2_en", "mm_sel", 11), }; -static int clk_mt2712_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), +}; static const struct of_device_id of_match_clk_mt2712_img[] = { - { .compatible = "mediatek,mt2712-imgsys", }, - {} + { + .compatible = "mediatek,mt2712-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_img_drv = { - .probe = clk_mt2712_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-img", .of_match_table = of_match_clk_mt2712_img, diff --git a/drivers/clk/mediatek/clk-mt2712-jpgdec.c b/drivers/clk/mediatek/clk-mt2712-jpgdec.c index 31fc30370d9831c1e6fd6744a3adeb6a9e1bf573..07ba7c5e80aff3a6b39887c416425559ca78a15a 100644 --- a/drivers/clk/mediatek/clk-mt2712-jpgdec.c +++ b/drivers/clk/mediatek/clk-mt2712-jpgdec.c @@ -32,33 +32,23 @@ static const struct mtk_gate jpgdec_clks[] = { GATE_JPGDEC(CLK_JPGDEC_JPGDEC, "jpgdec_jpgdec", "jpgdec_sel", 4), }; -static int clk_mt2712_jpgdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_JPGDEC_NR_CLK); - - mtk_clk_register_gates(node, jpgdec_clks, ARRAY_SIZE(jpgdec_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc jpgdec_desc = { + .clks = jpgdec_clks, + .num_clks = ARRAY_SIZE(jpgdec_clks), +}; static const struct of_device_id of_match_clk_mt2712_jpgdec[] = { - { .compatible = "mediatek,mt2712-jpgdecsys", }, - {} + { + .compatible = "mediatek,mt2712-jpgdecsys", + .data = &jpgdec_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_jpgdec_drv = { - .probe = clk_mt2712_jpgdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-jpgdec", .of_match_table = of_match_clk_mt2712_jpgdec, diff --git a/drivers/clk/mediatek/clk-mt2712-mfg.c b/drivers/clk/mediatek/clk-mt2712-mfg.c index a4d09675bf18e0d80d8dd067603421e414a5cdd6..42f8cf3ecf4cb2e529fadc68cb19eacd49d1beb9 100644 --- a/drivers/clk/mediatek/clk-mt2712-mfg.c +++ b/drivers/clk/mediatek/clk-mt2712-mfg.c @@ -31,33 +31,23 @@ static const struct mtk_gate mfg_clks[] = { GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_sel", 0), }; -static int clk_mt2712_mfg_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_MFG_NR_CLK); - - mtk_clk_register_gates(node, mfg_clks, ARRAY_SIZE(mfg_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc mfg_desc = { + .clks = mfg_clks, + .num_clks = ARRAY_SIZE(mfg_clks), +}; static const struct of_device_id of_match_clk_mt2712_mfg[] = { - { .compatible = "mediatek,mt2712-mfgcfg", }, - {} + { + .compatible = "mediatek,mt2712-mfgcfg", + .data = &mfg_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_mfg_drv = { - .probe = clk_mt2712_mfg_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-mfg", .of_match_table = of_match_clk_mt2712_mfg, diff --git a/drivers/clk/mediatek/clk-mt2712-vdec.c b/drivers/clk/mediatek/clk-mt2712-vdec.c index af13f43dd83198b6c54878db5894f88d678c4030..6296ed5c5b5559cdb48d552242626739df044c8b 100644 --- a/drivers/clk/mediatek/clk-mt2712-vdec.c +++ b/drivers/clk/mediatek/clk-mt2712-vdec.c @@ -50,33 +50,23 @@ static const struct mtk_gate vdec_clks[] = { GATE_VDEC1(CLK_VDEC_IMGRZ_CKEN, "vdec_imgrz_cken", "vdec_sel", 1), }; -static int clk_mt2712_vdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VDEC_NR_CLK); - - mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), +}; static const struct of_device_id of_match_clk_mt2712_vdec[] = { - { .compatible = "mediatek,mt2712-vdecsys", }, - {} + { + .compatible = "mediatek,mt2712-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_vdec_drv = { - .probe = clk_mt2712_vdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-vdec", .of_match_table = of_match_clk_mt2712_vdec, diff --git a/drivers/clk/mediatek/clk-mt2712-venc.c b/drivers/clk/mediatek/clk-mt2712-venc.c index abc08a02975353c83640cf35bf3b3c5deed3a281..b9bfc35de629c6a5a64873a8884ec35d6a3bb890 100644 --- a/drivers/clk/mediatek/clk-mt2712-venc.c +++ b/drivers/clk/mediatek/clk-mt2712-venc.c @@ -33,33 +33,23 @@ static const struct mtk_gate venc_clks[] = { GATE_VENC(CLK_VENC_SMI_LARB6, "venc_smi_larb6", "jpgdec_sel", 12), }; -static int clk_mt2712_venc_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VENC_NR_CLK); - - mtk_clk_register_gates(node, venc_clks, ARRAY_SIZE(venc_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r != 0) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), +}; static const struct of_device_id of_match_clk_mt2712_venc[] = { - { .compatible = "mediatek,mt2712-vencsys", }, - {} + { + .compatible = "mediatek,mt2712-vencsys", + .data = &venc_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt2712_venc_drv = { - .probe = clk_mt2712_venc_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt2712-venc", .of_match_table = of_match_clk_mt2712_venc, diff --git a/drivers/clk/mediatek/clk-mt6765-audio.c b/drivers/clk/mediatek/clk-mt6765-audio.c index 9c6e9caad5972e04e623868b9e84fe2118a0cd81..0aa6c0d352ca56720b2f76407df16913a40478e1 100644 --- a/drivers/clk/mediatek/clk-mt6765-audio.c +++ b/drivers/clk/mediatek/clk-mt6765-audio.c @@ -64,33 +64,23 @@ static const struct mtk_gate audio_clks[] = { "audio_ck", 7), }; -static int clk_mt6765_audio_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_AUDIO_NR_CLK); - - mtk_clk_register_gates(node, audio_clks, - ARRAY_SIZE(audio_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc audio_desc = { + .clks = audio_clks, + .num_clks = ARRAY_SIZE(audio_clks), +}; static const struct of_device_id of_match_clk_mt6765_audio[] = { - { .compatible = "mediatek,mt6765-audsys", }, - {} + { + .compatible = "mediatek,mt6765-audsys", + .data = &audio_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_audio_drv = { - .probe = clk_mt6765_audio_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-audio", .of_match_table = of_match_clk_mt6765_audio, diff --git a/drivers/clk/mediatek/clk-mt6765-cam.c b/drivers/clk/mediatek/clk-mt6765-cam.c index 2586d3ac4cd45fd7d3210ab0d48262084b16f0a9..25f2bef38126e376add3a16e83a64ce5178670bb 100644 --- a/drivers/clk/mediatek/clk-mt6765-cam.c +++ b/drivers/clk/mediatek/clk-mt6765-cam.c @@ -39,32 +39,23 @@ static const struct mtk_gate cam_clks[] = { GATE_CAM(CLK_CAM_CCU, "cam_ccu", "mm_ck", 12), }; -static int clk_mt6765_cam_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_CAM_NR_CLK); - - mtk_clk_register_gates(node, cam_clks, ARRAY_SIZE(cam_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc cam_desc = { + .clks = cam_clks, + .num_clks = ARRAY_SIZE(cam_clks), +}; static const struct of_device_id of_match_clk_mt6765_cam[] = { - { .compatible = "mediatek,mt6765-camsys", }, - {} + { + .compatible = "mediatek,mt6765-camsys", + .data = &cam_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_cam_drv = { - .probe = clk_mt6765_cam_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-cam", .of_match_table = of_match_clk_mt6765_cam, diff --git a/drivers/clk/mediatek/clk-mt6765-img.c b/drivers/clk/mediatek/clk-mt6765-img.c index 8cc95b98921e1909121833f757c0ebca9b790aca..a62303ef4f41df8b0976c4aef9e8f4de9bd96e9e 100644 --- a/drivers/clk/mediatek/clk-mt6765-img.c +++ b/drivers/clk/mediatek/clk-mt6765-img.c @@ -35,32 +35,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_RSC, "img_rsc", "mm_ck", 5), }; -static int clk_mt6765_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), +}; static const struct of_device_id of_match_clk_mt6765_img[] = { - { .compatible = "mediatek,mt6765-imgsys", }, - {} + { + .compatible = "mediatek,mt6765-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_img_drv = { - .probe = clk_mt6765_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-img", .of_match_table = of_match_clk_mt6765_img, diff --git a/drivers/clk/mediatek/clk-mt6765-mipi0a.c b/drivers/clk/mediatek/clk-mt6765-mipi0a.c index c816e26a95f99de84e0d2449323291f584285042..25c829fc386618b05b74063a6dd7d74d33fead31 100644 --- a/drivers/clk/mediatek/clk-mt6765-mipi0a.c +++ b/drivers/clk/mediatek/clk-mt6765-mipi0a.c @@ -32,33 +32,23 @@ static const struct mtk_gate mipi0a_clks[] = { "mipi0a_csr_0a", "f_fseninf_ck", 1), }; -static int clk_mt6765_mipi0a_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_MIPI0A_NR_CLK); - - mtk_clk_register_gates(node, mipi0a_clks, - ARRAY_SIZE(mipi0a_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc mipi0a_desc = { + .clks = mipi0a_clks, + .num_clks = ARRAY_SIZE(mipi0a_clks), +}; static const struct of_device_id of_match_clk_mt6765_mipi0a[] = { - { .compatible = "mediatek,mt6765-mipi0a", }, - {} + { + .compatible = "mediatek,mt6765-mipi0a", + .data = &mipi0a_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_mipi0a_drv = { - .probe = clk_mt6765_mipi0a_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-mipi0a", .of_match_table = of_match_clk_mt6765_mipi0a, diff --git a/drivers/clk/mediatek/clk-mt6765-mm.c b/drivers/clk/mediatek/clk-mt6765-mm.c index ee6d3b859a6ca4a04a6bf8ffd96b6761ab3d7c4c..bda774668a3616a389a3cb9fd337fa25c30d33b4 100644 --- a/drivers/clk/mediatek/clk-mt6765-mm.c +++ b/drivers/clk/mediatek/clk-mt6765-mm.c @@ -61,32 +61,23 @@ static const struct mtk_gate mm_clks[] = { GATE_MM(CLK_MM_F26M_HRTWT, "mm_hrtwt", "f_f26m_ck", 29), }; -static int clk_mt6765_mm_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK); - - mtk_clk_register_gates(node, mm_clks, ARRAY_SIZE(mm_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc mm_desc = { + .clks = mm_clks, + .num_clks = ARRAY_SIZE(mm_clks), +}; static const struct of_device_id of_match_clk_mt6765_mm[] = { - { .compatible = "mediatek,mt6765-mmsys", }, - {} + { + .compatible = "mediatek,mt6765-mmsys", + .data = &mm_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_mm_drv = { - .probe = clk_mt6765_mm_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-mm", .of_match_table = of_match_clk_mt6765_mm, diff --git a/drivers/clk/mediatek/clk-mt6765-vcodec.c b/drivers/clk/mediatek/clk-mt6765-vcodec.c index d8045979d48a5fb11538b771a001725c6de0f34d..2bc1fbde87da90a769e5d00ba727a1c37f253983 100644 --- a/drivers/clk/mediatek/clk-mt6765-vcodec.c +++ b/drivers/clk/mediatek/clk-mt6765-vcodec.c @@ -34,33 +34,23 @@ static const struct mtk_gate venc_clks[] = { GATE_VENC(CLK_VENC_SET3_VDEC, "venc_set3_vdec", "mm_ck", 12), }; -static int clk_mt6765_vcodec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VENC_NR_CLK); - - mtk_clk_register_gates(node, venc_clks, - ARRAY_SIZE(venc_clks), clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - - if (r) - pr_err("%s(): could not register clock provider: %d\n", - __func__, r); - - return r; -} +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), +}; static const struct of_device_id of_match_clk_mt6765_vcodec[] = { - { .compatible = "mediatek,mt6765-vcodecsys", }, - {} + { + .compatible = "mediatek,mt6765-vcodecsys", + .data = &venc_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6765_vcodec_drv = { - .probe = clk_mt6765_vcodec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6765-vcodec", .of_match_table = of_match_clk_mt6765_vcodec, diff --git a/drivers/clk/mediatek/clk-mt6779-aud.c b/drivers/clk/mediatek/clk-mt6779-aud.c index 97e44abb7e8798d42e6b912430f2093fcd7cde99..6e473ae1fd903c3b02db25861515592bd9170a4a 100644 --- a/drivers/clk/mediatek/clk-mt6779-aud.c +++ b/drivers/clk/mediatek/clk-mt6779-aud.c @@ -89,26 +89,23 @@ static const struct mtk_gate audio_clks[] = { "audio_h_sel", 31), }; -static const struct of_device_id of_match_clk_mt6779_aud[] = { - { .compatible = "mediatek,mt6779-audio", }, - {} +static const struct mtk_clk_desc audio_desc = { + .clks = audio_clks, + .num_clks = ARRAY_SIZE(audio_clks), }; -static int clk_mt6779_aud_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_AUD_NR_CLK); - - mtk_clk_register_gates(node, audio_clks, ARRAY_SIZE(audio_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_aud[] = { + { + .compatible = "mediatek,mt6779-audio", + .data = &audio_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_aud_drv = { - .probe = clk_mt6779_aud_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-aud", .of_match_table = of_match_clk_mt6779_aud, diff --git a/drivers/clk/mediatek/clk-mt6779-cam.c b/drivers/clk/mediatek/clk-mt6779-cam.c index 9c5117aae14689ca01acdc1a477bbc24bd97ec79..7be3db90fa4a8a28174b9d068ee6468376c0db85 100644 --- a/drivers/clk/mediatek/clk-mt6779-cam.c +++ b/drivers/clk/mediatek/clk-mt6779-cam.c @@ -38,26 +38,23 @@ static const struct mtk_gate cam_clks[] = { GATE_CAM(CLK_CAM_FAKE_ENG, "camsys_fake_eng", "cam_sel", 14), }; -static const struct of_device_id of_match_clk_mt6779_cam[] = { - { .compatible = "mediatek,mt6779-camsys", }, - {} +static const struct mtk_clk_desc cam_desc = { + .clks = cam_clks, + .num_clks = ARRAY_SIZE(cam_clks), }; -static int clk_mt6779_cam_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_CAM_NR_CLK); - - mtk_clk_register_gates(node, cam_clks, ARRAY_SIZE(cam_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_cam[] = { + { + .compatible = "mediatek,mt6779-camsys", + .data = &cam_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_cam_drv = { - .probe = clk_mt6779_cam_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-cam", .of_match_table = of_match_clk_mt6779_cam, diff --git a/drivers/clk/mediatek/clk-mt6779-img.c b/drivers/clk/mediatek/clk-mt6779-img.c index 801271477d46ba2364cee4e2487e5dd425a2558f..9bc51fc82dbd2bb250fc419a96a6d8b7f6e61ddc 100644 --- a/drivers/clk/mediatek/clk-mt6779-img.c +++ b/drivers/clk/mediatek/clk-mt6779-img.c @@ -30,26 +30,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_WPE_A, "imgsys_wpe_a", "img_sel", 7), }; -static const struct of_device_id of_match_clk_mt6779_img[] = { - { .compatible = "mediatek,mt6779-imgsys", }, - {} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), }; -static int clk_mt6779_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_img[] = { + { + .compatible = "mediatek,mt6779-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_img_drv = { - .probe = clk_mt6779_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-img", .of_match_table = of_match_clk_mt6779_img, diff --git a/drivers/clk/mediatek/clk-mt6779-ipe.c b/drivers/clk/mediatek/clk-mt6779-ipe.c index f67814ca7dfba7abdf6f76e65562592043645f75..92e9d1ade42215087d28dbe2a022a9dff22b990a 100644 --- a/drivers/clk/mediatek/clk-mt6779-ipe.c +++ b/drivers/clk/mediatek/clk-mt6779-ipe.c @@ -32,26 +32,23 @@ static const struct mtk_gate ipe_clks[] = { GATE_IPE(CLK_IPE_DPE, "ipe_dpe", "ipe_sel", 6), }; -static const struct of_device_id of_match_clk_mt6779_ipe[] = { - { .compatible = "mediatek,mt6779-ipesys", }, - {} +static const struct mtk_clk_desc ipe_desc = { + .clks = ipe_clks, + .num_clks = ARRAY_SIZE(ipe_clks), }; -static int clk_mt6779_ipe_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IPE_NR_CLK); - - mtk_clk_register_gates(node, ipe_clks, ARRAY_SIZE(ipe_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_ipe[] = { + { + .compatible = "mediatek,mt6779-ipesys", + .data = &ipe_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_ipe_drv = { - .probe = clk_mt6779_ipe_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-ipe", .of_match_table = of_match_clk_mt6779_ipe, diff --git a/drivers/clk/mediatek/clk-mt6779-mfg.c b/drivers/clk/mediatek/clk-mt6779-mfg.c index fc7387b59758d36220c11f24ab7eea41b3abdb16..efc793a1969a21c4a0ba819372e9fc84b5c2169f 100644 --- a/drivers/clk/mediatek/clk-mt6779-mfg.c +++ b/drivers/clk/mediatek/clk-mt6779-mfg.c @@ -27,26 +27,23 @@ static const struct mtk_gate mfg_clks[] = { GATE_MFG(CLK_MFGCFG_BG3D, "mfg_bg3d", "mfg_sel", 0), }; -static int clk_mt6779_mfg_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_MFGCFG_NR_CLK); - - mtk_clk_register_gates(node, mfg_clks, ARRAY_SIZE(mfg_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc mfg_desc = { + .clks = mfg_clks, + .num_clks = ARRAY_SIZE(mfg_clks), +}; static const struct of_device_id of_match_clk_mt6779_mfg[] = { - { .compatible = "mediatek,mt6779-mfgcfg", }, - {} + { + .compatible = "mediatek,mt6779-mfgcfg", + .data = &mfg_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt6779_mfg_drv = { - .probe = clk_mt6779_mfg_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-mfg", .of_match_table = of_match_clk_mt6779_mfg, diff --git a/drivers/clk/mediatek/clk-mt6779-vdec.c b/drivers/clk/mediatek/clk-mt6779-vdec.c index 7e195b082e86754420e59da6492e7baed6ec0fb6..3209a6518d5b1e35c00ca1a5f12ba6b8d5d8cd15 100644 --- a/drivers/clk/mediatek/clk-mt6779-vdec.c +++ b/drivers/clk/mediatek/clk-mt6779-vdec.c @@ -39,26 +39,23 @@ static const struct mtk_gate vdec_clks[] = { GATE_VDEC1_I(CLK_VDEC_LARB1, "vdec_larb1_cken", "vdec_sel", 0), }; -static const struct of_device_id of_match_clk_mt6779_vdec[] = { - { .compatible = "mediatek,mt6779-vdecsys", }, - {} +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), }; -static int clk_mt6779_vdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VDEC_GCON_NR_CLK); - - mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_vdec[] = { + { + .compatible = "mediatek,mt6779-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_vdec_drv = { - .probe = clk_mt6779_vdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-vdec", .of_match_table = of_match_clk_mt6779_vdec, diff --git a/drivers/clk/mediatek/clk-mt6779-venc.c b/drivers/clk/mediatek/clk-mt6779-venc.c index 573efa87c9bdc0d79a0f61f33406e2f393642439..c25035c0f334c18173f2a2497d0261053699cdc7 100644 --- a/drivers/clk/mediatek/clk-mt6779-venc.c +++ b/drivers/clk/mediatek/clk-mt6779-venc.c @@ -30,26 +30,23 @@ static const struct mtk_gate venc_clks[] = { GATE_VENC_I(CLK_VENC_GCON_GALS, "venc_gals", "venc_sel", 28), }; -static const struct of_device_id of_match_clk_mt6779_venc[] = { - { .compatible = "mediatek,mt6779-vencsys", }, - {} +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), }; -static int clk_mt6779_venc_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VENC_GCON_NR_CLK); - - mtk_clk_register_gates(node, venc_clks, ARRAY_SIZE(venc_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct of_device_id of_match_clk_mt6779_venc[] = { + { + .compatible = "mediatek,mt6779-vencsys", + .data = &venc_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6779_venc_drv = { - .probe = clk_mt6779_venc_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6779-venc", .of_match_table = of_match_clk_mt6779_venc, diff --git a/drivers/clk/mediatek/clk-mt6795-apmixedsys.c b/drivers/clk/mediatek/clk-mt6795-apmixedsys.c new file mode 100644 index 0000000000000000000000000000000000000000..59761c72d3bc2f0d1c1d980eaba47c6b65dca4e8 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-apmixedsys.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-mtk.h" +#include "clk-pll.h" + +#define REG_REF2USB 0x8 +#define REG_AP_PLL_CON7 0x1c + #define MD1_MTCMOS_OFF BIT(0) + #define MD1_MEM_OFF BIT(1) + #define MD1_CLK_OFF BIT(4) + #define MD1_ISO_OFF BIT(8) + +#define MT6795_PLL_FMAX (3000UL * MHZ) +#define MT6795_CON0_EN BIT(0) +#define MT6795_CON0_RST_BAR BIT(24) + +#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift) { \ + .id = _id, \ + .name = _name, \ + .reg = _reg, \ + .pwr_reg = _pwr_reg, \ + .en_mask = MT6795_CON0_EN | _en_mask, \ + .flags = _flags, \ + .rst_bar_mask = MT6795_CON0_RST_BAR, \ + .fmax = MT6795_PLL_FMAX, \ + .pcwbits = _pcwbits, \ + .pd_reg = _pd_reg, \ + .pd_shift = _pd_shift, \ + .tuner_reg = _tuner_reg, \ + .pcw_reg = _pcw_reg, \ + .pcw_shift = _pcw_shift, \ + .div_table = NULL, \ + .pll_en_bit = 0, \ + } + +static const struct mtk_pll_data plls[] = { + PLL(CLK_APMIXED_ARMCA53PLL, "armca53pll", 0x200, 0x20c, 0, PLL_AO, + 21, 0x204, 24, 0x0, 0x204, 0), + PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x220, 0x22c, 0xf0000101, HAVE_RST_BAR, + 21, 0x220, 4, 0x0, 0x224, 0), + PLL(CLK_APMIXED_UNIVPLL, "univpll", 0x230, 0x23c, 0xfe000101, HAVE_RST_BAR, + 7, 0x230, 4, 0x0, 0x234, 14), + PLL(CLK_APMIXED_MMPLL, "mmpll", 0x240, 0x24c, 0, 0, 21, 0x244, 24, 0x0, 0x244, 0), + PLL(CLK_APMIXED_MSDCPLL, "msdcpll", 0x250, 0x25c, 0, 0, 21, 0x250, 4, 0x0, 0x254, 0), + PLL(CLK_APMIXED_VENCPLL, "vencpll", 0x260, 0x26c, 0, 0, 21, 0x260, 4, 0x0, 0x264, 0), + PLL(CLK_APMIXED_TVDPLL, "tvdpll", 0x270, 0x27c, 0, 0, 21, 0x270, 4, 0x0, 0x274, 0), + PLL(CLK_APMIXED_MPLL, "mpll", 0x280, 0x28c, 0, 0, 21, 0x280, 4, 0x0, 0x284, 0), + PLL(CLK_APMIXED_VCODECPLL, "vcodecpll", 0x290, 0x29c, 0, 0, 21, 0x290, 4, 0x0, 0x294, 0), + PLL(CLK_APMIXED_APLL1, "apll1", 0x2a0, 0x2b0, 0, 0, 31, 0x2a0, 4, 0x2a8, 0x2a4, 0), + PLL(CLK_APMIXED_APLL2, "apll2", 0x2b4, 0x2c4, 0, 0, 31, 0x2b4, 4, 0x2bc, 0x2b8, 0), +}; + +static void clk_mt6795_apmixed_setup_md1(void __iomem *base) +{ + void __iomem *reg = base + REG_AP_PLL_CON7; + + /* Turn on MD1 internal clock */ + writel(readl(reg) & ~MD1_CLK_OFF, reg); + + /* Unlock MD1's MTCMOS power path */ + writel(readl(reg) & ~MD1_MTCMOS_OFF, reg); + + /* Turn on ISO */ + writel(readl(reg) & ~MD1_ISO_OFF, reg); + + /* Turn on memory */ + writel(readl(reg) & ~MD1_MEM_OFF, reg); +} + +static const struct of_device_id of_match_clk_mt6795_apmixed[] = { + { .compatible = "mediatek,mt6795-apmixedsys" }, + { /* sentinel */ } +}; + +static int clk_mt6795_apmixed_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + void __iomem *base; + struct clk_hw *hw; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + if (ret) + goto free_clk_data; + + hw = mtk_clk_register_ref2usb_tx("ref2usb_tx", "clk26m", base + REG_REF2USB); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + dev_err(dev, "Failed to register ref2usb_tx: %d\n", ret); + goto unregister_plls; + } + clk_data->hws[CLK_APMIXED_REF2USB_TX] = hw; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) { + dev_err(dev, "Cannot register clock provider: %d\n", ret); + goto unregister_ref2usb; + } + + /* Setup MD1 to avoid random crashes */ + dev_dbg(dev, "Performing initial setup for MD1\n"); + clk_mt6795_apmixed_setup_md1(base); + + return 0; + +unregister_ref2usb: + mtk_clk_unregister_ref2usb_tx(clk_data->hws[CLK_APMIXED_REF2USB_TX]); +unregister_plls: + mtk_clk_unregister_plls(plls, ARRAY_SIZE(plls), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + return ret; +} + +static int clk_mt6795_apmixed_remove(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct clk_hw_onecell_data *clk_data = platform_get_drvdata(pdev); + + of_clk_del_provider(node); + mtk_clk_unregister_ref2usb_tx(clk_data->hws[CLK_APMIXED_REF2USB_TX]); + mtk_clk_unregister_plls(plls, ARRAY_SIZE(plls), clk_data); + mtk_free_clk_data(clk_data); + + return 0; +} + +static struct platform_driver clk_mt6795_apmixed_drv = { + .probe = clk_mt6795_apmixed_probe, + .remove = clk_mt6795_apmixed_remove, + .driver = { + .name = "clk-mt6795-apmixed", + .of_match_table = of_match_clk_mt6795_apmixed, + }, +}; +module_platform_driver(clk_mt6795_apmixed_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 apmixed clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-infracfg.c b/drivers/clk/mediatek/clk-mt6795-infracfg.c new file mode 100644 index 0000000000000000000000000000000000000000..df7eed6e071e3354f0aba9b757f07524335af5d0 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-infracfg.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include +#include "clk-cpumux.h" +#include "clk-gate.h" +#include "clk-mtk.h" +#include "reset.h" + +#define GATE_ICG(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &infra_cg_regs, \ + _shift, &mtk_clk_gate_ops_no_setclr) + +static const struct mtk_gate_regs infra_cg_regs = { + .set_ofs = 0x0040, + .clr_ofs = 0x0044, + .sta_ofs = 0x0048, +}; + +static const char * const ca53_c0_parents[] = { + "clk26m", + "armca53pll", + "mainpll", + "univpll" +}; + +static const char * const ca53_c1_parents[] = { + "clk26m", + "armca53pll", + "mainpll", + "univpll" +}; + +static const struct mtk_composite cpu_muxes[] = { + MUX(CLK_INFRA_CA53_C0_SEL, "infra_ca53_c0_sel", ca53_c0_parents, 0x00, 0, 2), + MUX(CLK_INFRA_CA53_C1_SEL, "infra_ca53_c1_sel", ca53_c1_parents, 0x00, 2, 2), +}; + +static const struct mtk_gate infra_gates[] = { + GATE_ICG(CLK_INFRA_DBGCLK, "infra_dbgclk", "axi_sel", 0), + GATE_ICG(CLK_INFRA_SMI, "infra_smi", "mm_sel", 1), + GATE_ICG(CLK_INFRA_AUDIO, "infra_audio", "aud_intbus_sel", 5), + GATE_ICG(CLK_INFRA_GCE, "infra_gce", "axi_sel", 6), + GATE_ICG(CLK_INFRA_L2C_SRAM, "infra_l2c_sram", "axi_sel", 7), + GATE_ICG(CLK_INFRA_M4U, "infra_m4u", "mem_sel", 8), + GATE_ICG(CLK_INFRA_MD1MCU, "infra_md1mcu", "clk26m", 9), + GATE_ICG(CLK_INFRA_MD1BUS, "infra_md1bus", "axi_sel", 10), + GATE_ICG(CLK_INFRA_MD1DBB, "infra_dbb", "axi_sel", 11), + GATE_ICG(CLK_INFRA_DEVICE_APC, "infra_devapc", "clk26m", 12), + GATE_ICG(CLK_INFRA_TRNG, "infra_trng", "axi_sel", 13), + GATE_ICG(CLK_INFRA_MD1LTE, "infra_md1lte", "axi_sel", 14), + GATE_ICG(CLK_INFRA_CPUM, "infra_cpum", "cpum_ck", 15), + GATE_ICG(CLK_INFRA_KP, "infra_kp", "axi_sel", 16), +}; + +static u16 infra_ao_rst_ofs[] = { 0x30, 0x34 }; + +static u16 infra_ao_idx_map[] = { + [MT6795_INFRA_RST0_SCPSYS_RST] = 0 * RST_NR_PER_BANK + 5, + [MT6795_INFRA_RST0_PMIC_WRAP_RST] = 0 * RST_NR_PER_BANK + 7, + [MT6795_INFRA_RST1_MIPI_DSI_RST] = 1 * RST_NR_PER_BANK + 4, + [MT6795_INFRA_RST1_MIPI_CSI_RST] = 1 * RST_NR_PER_BANK + 7, + [MT6795_INFRA_RST1_MM_IOMMU_RST] = 1 * RST_NR_PER_BANK + 15, +}; + +static const struct mtk_clk_rst_desc clk_rst_desc = { + .version = MTK_RST_SET_CLR, + .rst_bank_ofs = infra_ao_rst_ofs, + .rst_bank_nr = ARRAY_SIZE(infra_ao_rst_ofs), + .rst_idx_map = infra_ao_idx_map, + .rst_idx_map_nr = ARRAY_SIZE(infra_ao_idx_map), +}; + +static const struct of_device_id of_match_clk_mt6795_infracfg[] = { + { .compatible = "mediatek,mt6795-infracfg" }, + { /* sentinel */ } +}; + +static int clk_mt6795_infracfg_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + void __iomem *base; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_INFRA_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_register_reset_controller_with_dev(&pdev->dev, &clk_rst_desc); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_gates(node, infra_gates, ARRAY_SIZE(infra_gates), clk_data); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_cpumuxes(node, cpu_muxes, ARRAY_SIZE(cpu_muxes), clk_data); + if (ret) + goto unregister_gates; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_cpumuxes; + + return 0; + +unregister_cpumuxes: + mtk_clk_unregister_cpumuxes(cpu_muxes, ARRAY_SIZE(cpu_muxes), clk_data); +unregister_gates: + mtk_clk_unregister_gates(infra_gates, ARRAY_SIZE(infra_gates), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + return ret; +} + +static int clk_mt6795_infracfg_remove(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct clk_hw_onecell_data *clk_data = platform_get_drvdata(pdev); + + of_clk_del_provider(node); + mtk_clk_unregister_cpumuxes(cpu_muxes, ARRAY_SIZE(cpu_muxes), clk_data); + mtk_clk_unregister_gates(infra_gates, ARRAY_SIZE(infra_gates), clk_data); + mtk_free_clk_data(clk_data); + + return 0; +} + +static struct platform_driver clk_mt6795_infracfg_drv = { + .driver = { + .name = "clk-mt6795-infracfg", + .of_match_table = of_match_clk_mt6795_infracfg, + }, + .probe = clk_mt6795_infracfg_probe, + .remove = clk_mt6795_infracfg_remove, +}; +module_platform_driver(clk_mt6795_infracfg_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 infracfg clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-mfg.c b/drivers/clk/mediatek/clk-mt6795-mfg.c new file mode 100644 index 0000000000000000000000000000000000000000..ee7aab24eb24cb1c2be9abf62b2af5de949b54cc --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-mfg.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs mfg_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_MFG(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mfg_cg_regs, _shift, &mtk_clk_gate_ops_setclr) + +static const struct mtk_gate mfg_clks[] = { + GATE_MFG(CLK_MFG_BAXI, "mfg_baxi", "axi_mfg_in_sel", 0), + GATE_MFG(CLK_MFG_BMEM, "mfg_bmem", "mem_mfg_in_sel", 1), + GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_sel", 2), + GATE_MFG(CLK_MFG_B26M, "mfg_b26m", "clk26m", 3), +}; + +static const struct mtk_clk_desc mfg_desc = { + .clks = mfg_clks, + .num_clks = ARRAY_SIZE(mfg_clks), +}; + +static const struct of_device_id of_match_clk_mt6795_mfg[] = { + { .compatible = "mediatek,mt6795-mfgcfg", .data = &mfg_desc }, + { /* sentinel */ } +}; + +static struct platform_driver clk_mt6795_mfg_drv = { + .driver = { + .name = "clk-mt6795-mfg", + .of_match_table = of_match_clk_mt6795_mfg, + }, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, +}; +module_platform_driver(clk_mt6795_mfg_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 mfg clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-mm.c b/drivers/clk/mediatek/clk-mt6795-mm.c new file mode 100644 index 0000000000000000000000000000000000000000..fd73f202f2925e93b0342c4ba389718beccf1b0e --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-mm.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" + +#define GATE_MM0(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mm0_cg_regs, _shift, &mtk_clk_gate_ops_setclr) + +#define GATE_MM1(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mm1_cg_regs, _shift, &mtk_clk_gate_ops_setclr) + +static const struct mtk_gate_regs mm0_cg_regs = { + .set_ofs = 0x0104, + .clr_ofs = 0x0108, + .sta_ofs = 0x0100, +}; + +static const struct mtk_gate_regs mm1_cg_regs = { + .set_ofs = 0x0114, + .clr_ofs = 0x0118, + .sta_ofs = 0x0110, +}; + +static const struct mtk_gate mm_gates[] = { + /* MM0 */ + GATE_MM0(CLK_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 0), + GATE_MM0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 1), + GATE_MM0(CLK_MM_CAM_MDP, "mm_cam_mdp", "mm_sel", 2), + GATE_MM0(CLK_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_sel", 3), + GATE_MM0(CLK_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 4), + GATE_MM0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 5), + GATE_MM0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 6), + GATE_MM0(CLK_MM_MDP_RSZ2, "mm_mdp_rsz2", "mm_sel", 7), + GATE_MM0(CLK_MM_MDP_TDSHP0, "mm_mdp_tdshp0", "mm_sel", 8), + GATE_MM0(CLK_MM_MDP_TDSHP1, "mm_mdp_tdshp1", "mm_sel", 9), + GATE_MM0(CLK_MM_MDP_CROP, "mm_mdp_crop", "mm_sel", 10), + GATE_MM0(CLK_MM_MDP_WDMA, "mm_mdp_wdma", "mm_sel", 11), + GATE_MM0(CLK_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_sel", 12), + GATE_MM0(CLK_MM_MDP_WROT1, "mm_mdp_wrot1", "mm_sel", 13), + GATE_MM0(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_sel", 14), + GATE_MM0(CLK_MM_MUTEX_32K, "mm_mutex_32k", "clk32k", 15), + GATE_MM0(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "mm_sel", 16), + GATE_MM0(CLK_MM_DISP_OVL1, "mm_disp_ovl1", "mm_sel", 17), + GATE_MM0(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_sel", 18), + GATE_MM0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 19), + GATE_MM0(CLK_MM_DISP_RDMA2, "mm_disp_rdma2", "mm_sel", 20), + GATE_MM0(CLK_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_sel", 21), + GATE_MM0(CLK_MM_DISP_WDMA1, "mm_disp_wdma1", "mm_sel", 22), + GATE_MM0(CLK_MM_DISP_COLOR0, "mm_disp_color0", "mm_sel", 23), + GATE_MM0(CLK_MM_DISP_COLOR1, "mm_disp_color1", "mm_sel", 24), + GATE_MM0(CLK_MM_DISP_AAL, "mm_disp_aal", "mm_sel", 25), + GATE_MM0(CLK_MM_DISP_GAMMA, "mm_disp_gamma", "mm_sel", 26), + GATE_MM0(CLK_MM_DISP_UFOE, "mm_disp_ufoe", "mm_sel", 27), + GATE_MM0(CLK_MM_DISP_SPLIT0, "mm_disp_split0", "mm_sel", 28), + GATE_MM0(CLK_MM_DISP_SPLIT1, "mm_disp_split1", "mm_sel", 29), + GATE_MM0(CLK_MM_DISP_MERGE, "mm_disp_merge", "mm_sel", 30), + GATE_MM0(CLK_MM_DISP_OD, "mm_disp_od", "mm_sel", 31), + + /* MM1 */ + GATE_MM1(CLK_MM_DISP_PWM0MM, "mm_disp_pwm0mm", "mm_sel", 0), + GATE_MM1(CLK_MM_DISP_PWM026M, "mm_disp_pwm026m", "pwm_sel", 1), + GATE_MM1(CLK_MM_DISP_PWM1MM, "mm_disp_pwm1mm", "mm_sel", 2), + GATE_MM1(CLK_MM_DISP_PWM126M, "mm_disp_pwm126m", "pwm_sel", 3), + GATE_MM1(CLK_MM_DSI0_ENGINE, "mm_dsi0_engine", "mm_sel", 4), + GATE_MM1(CLK_MM_DSI0_DIGITAL, "mm_dsi0_digital", "dsi0_dig", 5), + GATE_MM1(CLK_MM_DSI1_ENGINE, "mm_dsi1_engine", "mm_sel", 6), + GATE_MM1(CLK_MM_DSI1_DIGITAL, "mm_dsi1_digital", "dsi1_dig", 7), + GATE_MM1(CLK_MM_DPI_PIXEL, "mm_dpi_pixel", "dpi0_sel", 8), + GATE_MM1(CLK_MM_DPI_ENGINE, "mm_dpi_engine", "mm_sel", 9), +}; + +static int clk_mt6795_mm_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->parent->of_node; + struct clk_hw_onecell_data *clk_data; + int ret; + + clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_gates(node, mm_gates, ARRAY_SIZE(mm_gates), clk_data); + if (ret) + goto free_clk_data; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_gates; + + platform_set_drvdata(pdev, clk_data); + + return 0; + +unregister_gates: + mtk_clk_unregister_gates(mm_gates, ARRAY_SIZE(mm_gates), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + return ret; +} + +static int clk_mt6795_mm_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->parent->of_node; + struct clk_hw_onecell_data *clk_data = platform_get_drvdata(pdev); + + of_clk_del_provider(node); + mtk_clk_unregister_gates(mm_gates, ARRAY_SIZE(mm_gates), clk_data); + mtk_free_clk_data(clk_data); + + return 0; +} + +static struct platform_driver clk_mt6795_mm_drv = { + .driver = { + .name = "clk-mt6795-mm", + }, + .probe = clk_mt6795_mm_probe, + .remove = clk_mt6795_mm_remove, +}; +module_platform_driver(clk_mt6795_mm_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 MultiMedia clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-pericfg.c b/drivers/clk/mediatek/clk-mt6795-pericfg.c new file mode 100644 index 0000000000000000000000000000000000000000..cb28d35dad59b4e304905edb97245ffaca934dec --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-pericfg.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" +#include "reset.h" + +#define GATE_PERI(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &peri_cg_regs, \ + _shift, &mtk_clk_gate_ops_setclr) + +static DEFINE_SPINLOCK(mt6795_peri_clk_lock); + +static const struct mtk_gate_regs peri_cg_regs = { + .set_ofs = 0x0008, + .clr_ofs = 0x0010, + .sta_ofs = 0x0018, +}; + +static const char * const uart_ck_sel_parents[] = { + "clk26m", + "uart_sel", +}; + +static const struct mtk_composite peri_clks[] = { + MUX(CLK_PERI_UART0_SEL, "uart0_ck_sel", uart_ck_sel_parents, 0x40c, 0, 1), + MUX(CLK_PERI_UART1_SEL, "uart1_ck_sel", uart_ck_sel_parents, 0x40c, 1, 1), + MUX(CLK_PERI_UART2_SEL, "uart2_ck_sel", uart_ck_sel_parents, 0x40c, 2, 1), + MUX(CLK_PERI_UART3_SEL, "uart3_ck_sel", uart_ck_sel_parents, 0x40c, 3, 1), +}; + +static const struct mtk_gate peri_gates[] = { + GATE_PERI(CLK_PERI_NFI, "peri_nfi", "axi_sel", 0), + GATE_PERI(CLK_PERI_THERM, "peri_therm", "axi_sel", 1), + GATE_PERI(CLK_PERI_PWM1, "peri_pwm1", "axi_sel", 2), + GATE_PERI(CLK_PERI_PWM2, "peri_pwm2", "axi_sel", 3), + GATE_PERI(CLK_PERI_PWM3, "peri_pwm3", "axi_sel", 4), + GATE_PERI(CLK_PERI_PWM4, "peri_pwm4", "axi_sel", 5), + GATE_PERI(CLK_PERI_PWM5, "peri_pwm5", "axi_sel", 6), + GATE_PERI(CLK_PERI_PWM6, "peri_pwm6", "axi_sel", 7), + GATE_PERI(CLK_PERI_PWM7, "peri_pwm7", "axi_sel", 8), + GATE_PERI(CLK_PERI_PWM, "peri_pwm", "axi_sel", 9), + GATE_PERI(CLK_PERI_USB0, "peri_usb0", "usb30_sel", 10), + GATE_PERI(CLK_PERI_USB1, "peri_usb1", "usb20_sel", 11), + GATE_PERI(CLK_PERI_AP_DMA, "peri_ap_dma", "axi_sel", 12), + GATE_PERI(CLK_PERI_MSDC30_0, "peri_msdc30_0", "msdc50_0_sel", 13), + GATE_PERI(CLK_PERI_MSDC30_1, "peri_msdc30_1", "msdc30_1_sel", 14), + GATE_PERI(CLK_PERI_MSDC30_2, "peri_msdc30_2", "msdc30_2_sel", 15), + GATE_PERI(CLK_PERI_MSDC30_3, "peri_msdc30_3", "msdc30_3_sel", 16), + GATE_PERI(CLK_PERI_NLI_ARB, "peri_nli_arb", "axi_sel", 17), + GATE_PERI(CLK_PERI_IRDA, "peri_irda", "irda_sel", 18), + GATE_PERI(CLK_PERI_UART0, "peri_uart0", "axi_sel", 19), + GATE_PERI(CLK_PERI_UART1, "peri_uart1", "axi_sel", 20), + GATE_PERI(CLK_PERI_UART2, "peri_uart2", "axi_sel", 21), + GATE_PERI(CLK_PERI_UART3, "peri_uart3", "axi_sel", 22), + GATE_PERI(CLK_PERI_I2C0, "peri_i2c0", "axi_sel", 23), + GATE_PERI(CLK_PERI_I2C1, "peri_i2c1", "axi_sel", 24), + GATE_PERI(CLK_PERI_I2C2, "peri_i2c2", "axi_sel", 25), + GATE_PERI(CLK_PERI_I2C3, "peri_i2c3", "axi_sel", 26), + GATE_PERI(CLK_PERI_I2C4, "peri_i2c4", "axi_sel", 27), + GATE_PERI(CLK_PERI_AUXADC, "peri_auxadc", "clk26m", 28), + GATE_PERI(CLK_PERI_SPI0, "peri_spi0", "spi_sel", 29), +}; + +static u16 peri_rst_ofs[] = { 0x0 }; + +static u16 peri_idx_map[] = { + [MT6795_PERI_NFI_SW_RST] = 14, + [MT6795_PERI_THERM_SW_RST] = 16, + [MT6795_PERI_MSDC1_SW_RST] = 20, +}; + +static const struct mtk_clk_rst_desc clk_rst_desc = { + .version = MTK_RST_SIMPLE, + .rst_bank_ofs = peri_rst_ofs, + .rst_bank_nr = ARRAY_SIZE(peri_rst_ofs), + .rst_idx_map = peri_idx_map, + .rst_idx_map_nr = ARRAY_SIZE(peri_idx_map), +}; + +static const struct of_device_id of_match_clk_mt6795_pericfg[] = { + { .compatible = "mediatek,mt6795-pericfg" }, + { /* sentinel */ } +}; + +static int clk_mt6795_pericfg_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + void __iomem *base; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_PERI_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_register_reset_controller_with_dev(&pdev->dev, &clk_rst_desc); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_gates(node, peri_gates, ARRAY_SIZE(peri_gates), clk_data); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_composites(peri_clks, ARRAY_SIZE(peri_clks), base, + &mt6795_peri_clk_lock, clk_data); + if (ret) + goto unregister_gates; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_composites; + + return 0; + +unregister_composites: + mtk_clk_unregister_composites(peri_clks, ARRAY_SIZE(peri_clks), clk_data); +unregister_gates: + mtk_clk_unregister_gates(peri_gates, ARRAY_SIZE(peri_gates), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + return ret; +} + +static int clk_mt6795_pericfg_remove(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct clk_hw_onecell_data *clk_data = platform_get_drvdata(pdev); + + of_clk_del_provider(node); + mtk_clk_unregister_composites(peri_clks, ARRAY_SIZE(peri_clks), clk_data); + mtk_clk_unregister_gates(peri_gates, ARRAY_SIZE(peri_gates), clk_data); + mtk_free_clk_data(clk_data); + + return 0; +} + +static struct platform_driver clk_mt6795_pericfg_drv = { + .driver = { + .name = "clk-mt6795-pericfg", + .of_match_table = of_match_clk_mt6795_pericfg, + }, + .probe = clk_mt6795_pericfg_probe, + .remove = clk_mt6795_pericfg_remove, +}; +module_platform_driver(clk_mt6795_pericfg_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 pericfg clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-topckgen.c b/drivers/clk/mediatek/clk-mt6795-topckgen.c new file mode 100644 index 0000000000000000000000000000000000000000..2948dd1aee8fa796178d472cd4cf03c13ba692ce --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-topckgen.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" +#include "clk-mux.h" + +/* + * For some clocks, we don't care what their actual rates are. And these + * clocks may change their rate on different products or different scenarios. + * So we model these clocks' rate as 0, to denote it's not an actual rate. + */ +#define DUMMY_RATE 0 + +#define TOP_MUX_GATE_NOSR(_id, _name, _parents, _reg, _shift, _width, _gate, _flags) \ + MUX_GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _reg, \ + (_reg + 0x4), (_reg + 0x8), _shift, _width, \ + _gate, 0, -1, _flags) + +#define TOP_MUX_GATE(_id, _name, _parents, _reg, _shift, _width, _gate, _flags) \ + TOP_MUX_GATE_NOSR(_id, _name, _parents, _reg, _shift, _width, \ + _gate, CLK_SET_RATE_PARENT | _flags) + +static DEFINE_SPINLOCK(mt6795_top_clk_lock); + +static const char * const aud_1_parents[] = { + "clk26m", + "apll1_ck", + "univpll2_d4", + "univpll2_d8" +}; + +static const char * const aud_2_parents[] = { + "clk26m", + "apll2_ck", + "univpll2_d4", + "univpll2_d8" +}; + +static const char * const aud_intbus_parents[] = { + "clk26m", + "syspll1_d4", + "syspll4_d2", + "univpll3_d2", + "univpll2_d8", + "dmpll_d4", + "dmpll_d8" +}; + +static const char * const audio_parents[] = { + "clk26m", + "syspll3_d4", + "syspll4_d4", + "syspll1_d16" +}; + +static const char * const axi_mfg_in_parents[] = { + "clk26m", + "axi_sel", + "dmpll_d2" +}; + +static const char * const axi_parents[] = { + "clk26m", + "syspll1_d2", + "syspll_d5", + "syspll1_d4", + "univpll_d5", + "univpll2_d2", + "dmpll_d2", + "dmpll_d4" +}; + +static const char * const camtg_parents[] = { + "clk26m", + "univpll_d26", + "univpll2_d2", + "syspll3_d2", + "syspll3_d4", + "univpll1_d4", + "dmpll_d8" +}; + +static const char * const cci400_parents[] = { + "clk26m", + "vencpll_ck", + "clk26m", + "clk26m", + "univpll_d2", + "syspll_d2", + "msdcpll_ck", + "dmpll_ck" +}; + +static const char * const ddrphycfg_parents[] = { + "clk26m", + "syspll1_d8" +}; + +static const char * const dpi0_parents[] = { + "clk26m", + "tvdpll_d2", + "tvdpll_d4", + "clk26m", + "clk26m", + "tvdpll_d8", + "tvdpll_d16" +}; + +static const char * const i2s0_m_ck_parents[] = { + "apll1_div1", + "apll2_div1" +}; + +static const char * const i2s1_m_ck_parents[] = { + "apll1_div2", + "apll2_div2" +}; + +static const char * const i2s2_m_ck_parents[] = { + "apll1_div3", + "apll2_div3" +}; + +static const char * const i2s3_m_ck_parents[] = { + "apll1_div4", + "apll2_div4" +}; + +static const char * const i2s3_b_ck_parents[] = { + "apll1_div5", + "apll2_div5" +}; + +static const char * const irda_parents[] = { + "clk26m", + "univpll2_d4", + "syspll2_d4", + "dmpll_d8", +}; + +static const char * const mem_mfg_in_parents[] = { + "clk26m", + "mmpll_ck", + "dmpll_ck" +}; + +static const char * const mem_parents[] = { + "clk26m", + "dmpll_ck" +}; + +static const char * const mfg_parents[] = { + "clk26m", + "mmpll_ck", + "dmpll_ck", + "clk26m", + "clk26m", + "clk26m", + "clk26m", + "clk26m", + "clk26m", + "syspll_d3", + "syspll1_d2", + "syspll_d5", + "univpll_d3", + "univpll1_d2", + "univpll_d5", + "univpll2_d2" +}; + +static const char * const mm_parents[] = { + "clk26m", + "vencpll_d2", + "syspll_d3", + "syspll1_d2", + "syspll_d5", + "syspll1_d4", + "univpll1_d2", + "univpll2_d2", + "dmpll_d2" +}; + +static const char * const mjc_parents[] = { + "clk26m", + "univpll_d3", + "vcodecpll_ck", + "tvdpll_445p5m", + "vencpll_d2", + "syspll_d3", + "univpll1_d2", + "syspll_d5", + "syspll1_d2", + "univpll_d5", + "univpll2_d2", + "dmpll_ck" +}; + +static const char * const msdc50_0_h_parents[] = { + "clk26m", + "syspll1_d2", + "syspll2_d2", + "syspll4_d2", + "univpll_d5", + "univpll1_d4" +}; + +static const char * const msdc50_0_parents[] = { + "clk26m", + "msdcpll_ck", + "msdcpll_d2", + "univpll1_d4", + "syspll2_d2", + "syspll_d7", + "msdcpll_d4", + "vencpll_d4", + "tvdpll_ck", + "univpll_d2", + "univpll1_d2", + "mmpll_ck" +}; + +static const char * const msdc30_1_parents[] = { + "clk26m", + "univpll2_d2", + "msdcpll_d4", + "univpll1_d4", + "syspll2_d2", + "syspll_d7", + "univpll_d7", + "vencpll_d4" +}; + +static const char * const msdc30_2_parents[] = { + "clk26m", + "univpll2_d2", + "msdcpll_d4", + "univpll1_d4", + "syspll2_d2", + "syspll_d7", + "univpll_d7", + "vencpll_d2" +}; + +static const char * const msdc30_3_parents[] = { + "clk26m", + "univpll2_d2", + "msdcpll_d4", + "univpll1_d4", + "syspll2_d2", + "syspll_d7", + "univpll_d7", + "vencpll_d4" +}; + +static const char * const pmicspi_parents[] = { + "clk26m", + "syspll1_d8", + "syspll3_d4", + "syspll1_d16", + "univpll3_d4", + "univpll_d26", + "dmpll_d8", + "dmpll_d16" +}; + +static const char * const pwm_parents[] = { + "clk26m", + "univpll2_d4", + "univpll3_d2", + "univpll1_d4" +}; + +static const char * const scam_parents[] = { + "clk26m", + "syspll3_d2", + "univpll2_d4", + "dmpll_d4" +}; + +static const char * const scp_parents[] = { + "clk26m", + "syspll1_d2", + "univpll_d5", + "syspll_d5", + "dmpll_d2", + "dmpll_d4" +}; + +static const char * const spi_parents[] = { + "clk26m", + "syspll3_d2", + "syspll1_d4", + "syspll4_d2", + "univpll3_d2", + "univpll2_d4", + "univpll1_d8" +}; + +static const char * const uart_parents[] = { + "clk26m", + "univpll2_d8" +}; + +static const char * const usb20_parents[] = { + "clk26m", + "univpll1_d8", + "univpll3_d4" +}; + +static const char * const usb30_parents[] = { + "clk26m", + "univpll3_d2", + "usb_syspll_125m", + "univpll2_d4" +}; + +static const char * const vdec_parents[] = { + "clk26m", + "vcodecpll_ck", + "tvdpll_445p5m", + "univpll_d3", + "vencpll_d2", + "syspll_d3", + "univpll1_d2", + "mmpll_d2", + "dmpll_d2", + "dmpll_d4" +}; + +static const char * const venc_parents[] = { + "clk26m", + "vcodecpll_ck", + "tvdpll_445p5m", + "univpll_d3", + "vencpll_d2", + "syspll_d3", + "univpll1_d2", + "univpll2_d2", + "dmpll_d2", + "dmpll_d4" +}; + +static const struct mtk_fixed_clk fixed_clks[] = { + FIXED_CLK(CLK_TOP_ADSYS_26M, "adsys_26m", "clk26m", 26 * MHZ), + FIXED_CLK(CLK_TOP_CLKPH_MCK_O, "clkph_mck_o", "clk26m", DUMMY_RATE), + FIXED_CLK(CLK_TOP_USB_SYSPLL_125M, "usb_syspll_125m", "clk26m", 125 * MHZ), + FIXED_CLK(CLK_TOP_DSI0_DIG, "dsi0_dig", "clk26m", DUMMY_RATE), + FIXED_CLK(CLK_TOP_DSI1_DIG, "dsi1_dig", "clk26m", DUMMY_RATE), +}; + +static const struct mtk_fixed_factor top_divs[] = { + FACTOR(CLK_TOP_ARMCA53PLL_754M, "armca53pll_754m", "clk26m", 1, 2), + FACTOR(CLK_TOP_ARMCA53PLL_502M, "armca53pll_502m", "clk26m", 1, 3), + + FACTOR(CLK_TOP_MAIN_H546M, "main_h546m", "mainpll", 1, 2), + FACTOR(CLK_TOP_MAIN_H364M, "main_h364m", "mainpll", 1, 3), + FACTOR(CLK_TOP_MAIN_H218P4M, "main_h218p4m", "mainpll", 1, 5), + FACTOR(CLK_TOP_MAIN_H156M, "main_h156m", "mainpll", 1, 7), + + FACTOR(CLK_TOP_TVDPLL_445P5M, "tvdpll_445p5m", "tvdpll", 1, 4), + FACTOR(CLK_TOP_TVDPLL_594M, "tvdpll_594m", "tvdpll", 1, 3), + + FACTOR(CLK_TOP_UNIV_624M, "univ_624m", "univpll", 1, 2), + FACTOR(CLK_TOP_UNIV_416M, "univ_416m", "univpll", 1, 3), + FACTOR(CLK_TOP_UNIV_249P6M, "univ_249p6m", "univpll", 1, 5), + FACTOR(CLK_TOP_UNIV_178P3M, "univ_178p3m", "univpll", 1, 7), + FACTOR(CLK_TOP_UNIV_48M, "univ_48m", "univpll", 1, 26), + + FACTOR(CLK_TOP_CLKRTC_EXT, "clkrtc_ext", "clk32k", 1, 1), + FACTOR(CLK_TOP_CLKRTC_INT, "clkrtc_int", "clk26m", 1, 793), + FACTOR(CLK_TOP_FPC, "fpc_ck", "clk26m", 1, 1), + + FACTOR(CLK_TOP_HDMITXPLL_D2, "hdmitxpll_d2", "clk26m", 1, 2), + FACTOR(CLK_TOP_HDMITXPLL_D3, "hdmitxpll_d3", "clk26m", 1, 3), + + FACTOR(CLK_TOP_ARMCA53PLL_D2, "armca53pll_d2", "clk26m", 1, 1), + FACTOR(CLK_TOP_ARMCA53PLL_D3, "armca53pll_d3", "clk26m", 1, 1), + + FACTOR(CLK_TOP_APLL1, "apll1_ck", "apll1", 1, 1), + FACTOR(CLK_TOP_APLL2, "apll2_ck", "apll2", 1, 1), + + FACTOR(CLK_TOP_DMPLL, "dmpll_ck", "clkph_mck_o", 1, 1), + FACTOR(CLK_TOP_DMPLL_D2, "dmpll_d2", "clkph_mck_o", 1, 2), + FACTOR(CLK_TOP_DMPLL_D4, "dmpll_d4", "clkph_mck_o", 1, 4), + FACTOR(CLK_TOP_DMPLL_D8, "dmpll_d8", "clkph_mck_o", 1, 8), + FACTOR(CLK_TOP_DMPLL_D16, "dmpll_d16", "clkph_mck_o", 1, 16), + + FACTOR(CLK_TOP_MMPLL, "mmpll_ck", "mmpll", 1, 1), + FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll", 1, 2), + + FACTOR(CLK_TOP_MSDCPLL, "msdcpll_ck", "msdcpll", 1, 1), + FACTOR(CLK_TOP_MSDCPLL_D2, "msdcpll_d2", "msdcpll", 1, 2), + FACTOR(CLK_TOP_MSDCPLL_D4, "msdcpll_d4", "msdcpll", 1, 4), + FACTOR(CLK_TOP_MSDCPLL2, "msdcpll2_ck", "msdcpll2", 1, 1), + FACTOR(CLK_TOP_MSDCPLL2_D2, "msdcpll2_d2", "msdcpll2", 1, 2), + FACTOR(CLK_TOP_MSDCPLL2_D4, "msdcpll2_d4", "msdcpll2", 1, 4), + + FACTOR(CLK_TOP_SYSPLL_D2, "syspll_d2", "main_h546m", 1, 1), + FACTOR(CLK_TOP_SYSPLL1_D2, "syspll1_d2", "main_h546m", 1, 2), + FACTOR(CLK_TOP_SYSPLL1_D4, "syspll1_d4", "main_h546m", 1, 4), + FACTOR(CLK_TOP_SYSPLL1_D8, "syspll1_d8", "main_h546m", 1, 8), + FACTOR(CLK_TOP_SYSPLL1_D16, "syspll1_d16", "main_h546m", 1, 16), + FACTOR(CLK_TOP_SYSPLL_D3, "syspll_d3", "main_h364m", 1, 1), + FACTOR(CLK_TOP_SYSPLL2_D2, "syspll2_d2", "main_h364m", 1, 2), + FACTOR(CLK_TOP_SYSPLL2_D4, "syspll2_d4", "main_h364m", 1, 4), + FACTOR(CLK_TOP_SYSPLL_D5, "syspll_d5", "main_h218p4m", 1, 1), + FACTOR(CLK_TOP_SYSPLL3_D2, "syspll3_d2", "main_h218p4m", 1, 2), + FACTOR(CLK_TOP_SYSPLL3_D4, "syspll3_d4", "main_h218p4m", 1, 4), + FACTOR(CLK_TOP_SYSPLL_D7, "syspll_d7", "main_h156m", 1, 1), + FACTOR(CLK_TOP_SYSPLL4_D2, "syspll4_d2", "main_h156m", 1, 2), + FACTOR(CLK_TOP_SYSPLL4_D4, "syspll4_d4", "main_h156m", 1, 4), + + FACTOR(CLK_TOP_TVDPLL, "tvdpll_ck", "tvdpll_594m", 1, 1), + FACTOR(CLK_TOP_TVDPLL_D2, "tvdpll_d2", "tvdpll_594m", 1, 2), + FACTOR(CLK_TOP_TVDPLL_D4, "tvdpll_d4", "tvdpll_594m", 1, 4), + FACTOR(CLK_TOP_TVDPLL_D8, "tvdpll_d8", "tvdpll_594m", 1, 8), + FACTOR(CLK_TOP_TVDPLL_D16, "tvdpll_d16", "tvdpll_594m", 1, 16), + + FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univ_624m", 1, 1), + FACTOR(CLK_TOP_UNIVPLL1_D2, "univpll1_d2", "univ_624m", 1, 2), + FACTOR(CLK_TOP_UNIVPLL1_D4, "univpll1_d4", "univ_624m", 1, 4), + FACTOR(CLK_TOP_UNIVPLL1_D8, "univpll1_d8", "univ_624m", 1, 8), + FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univ_416m", 1, 1), + FACTOR(CLK_TOP_UNIVPLL2_D2, "univpll2_d2", "univ_416m", 1, 2), + FACTOR(CLK_TOP_UNIVPLL2_D4, "univpll2_d4", "univ_416m", 1, 4), + FACTOR(CLK_TOP_UNIVPLL2_D8, "univpll2_d8", "univ_416m", 1, 8), + FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univ_249p6m", 1, 1), + FACTOR(CLK_TOP_UNIVPLL3_D2, "univpll3_d2", "univ_249p6m", 1, 2), + FACTOR(CLK_TOP_UNIVPLL3_D4, "univpll3_d4", "univ_249p6m", 1, 4), + FACTOR(CLK_TOP_UNIVPLL3_D8, "univpll3_d8", "univ_249p6m", 1, 8), + FACTOR(CLK_TOP_UNIVPLL_D7, "univpll_d7", "univ_178p3m", 1, 1), + FACTOR(CLK_TOP_UNIVPLL_D26, "univpll_d26", "univ_48m", 1, 1), + FACTOR(CLK_TOP_UNIVPLL_D52, "univpll_d52", "univ_48m", 1, 2), + + FACTOR(CLK_TOP_VCODECPLL, "vcodecpll_ck", "vcodecpll", 1, 3), + FACTOR(CLK_TOP_VCODECPLL_370P5, "vcodecpll_370p5", "vcodecpll", 1, 4), + + FACTOR(CLK_TOP_VENCPLL, "vencpll_ck", "vencpll", 1, 1), + FACTOR(CLK_TOP_VENCPLL_D2, "vencpll_d2", "vencpll", 1, 2), + FACTOR(CLK_TOP_VENCPLL_D4, "vencpll_d4", "vencpll", 1, 4), +}; + +static const struct mtk_mux top_muxes[] = { + /* CLK_CFG_0 */ + TOP_MUX_GATE_NOSR(CLK_TOP_AXI_SEL, "axi_sel", axi_parents, + 0x40, 0, 3, 7, CLK_IS_CRITICAL), + TOP_MUX_GATE_NOSR(CLK_TOP_MEM_SEL, "mem_sel", mem_parents, + 0x40, 8, 1, 15, CLK_IS_CRITICAL), + TOP_MUX_GATE(CLK_TOP_DDRPHYCFG_SEL, "ddrphycfg_sel", ddrphycfg_parents, + 0x40, 16, 1, 23, CLK_IS_CRITICAL), + TOP_MUX_GATE(CLK_TOP_MM_SEL, "mm_sel", mm_parents, 0x40, 24, 3, 31, 0), + /* CLK_CFG_1 */ + TOP_MUX_GATE(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents, 0x50, 0, 2, 7, 0), + TOP_MUX_GATE(CLK_TOP_VDEC_SEL, "vdec_sel", vdec_parents, 0x50, 8, 4, 15, 0), + TOP_MUX_GATE(CLK_TOP_VENC_SEL, "venc_sel", venc_parents, 0x50, 16, 4, 23, 0), + TOP_MUX_GATE(CLK_TOP_MFG_SEL, "mfg_sel", mfg_parents, 0x50, 24, 4, 31, 0), + /* CLK_CFG_2 */ + TOP_MUX_GATE(CLK_TOP_CAMTG_SEL, "camtg_sel", camtg_parents, 0x60, 0, 3, 7, 0), + TOP_MUX_GATE(CLK_TOP_UART_SEL, "uart_sel", uart_parents, 0x60, 8, 1, 15, 0), + TOP_MUX_GATE(CLK_TOP_SPI_SEL, "spi_sel", spi_parents, 0x60, 16, 3, 23, 0), + TOP_MUX_GATE(CLK_TOP_USB20_SEL, "usb20_sel", usb20_parents, 0x60, 24, 2, 31, 0), + /* CLK_CFG_3 */ + TOP_MUX_GATE(CLK_TOP_USB30_SEL, "usb30_sel", usb30_parents, 0x70, 0, 2, 7, 0), + TOP_MUX_GATE(CLK_TOP_MSDC50_0_H_SEL, "msdc50_0_h_sel", msdc50_0_h_parents, + 0x70, 8, 3, 15, 0), + TOP_MUX_GATE(CLK_TOP_MSDC50_0_SEL, "msdc50_0_sel", msdc50_0_parents, 0x70, 16, 4, 23, 0), + TOP_MUX_GATE(CLK_TOP_MSDC30_1_SEL, "msdc30_1_sel", msdc30_1_parents, 0x70, 24, 3, 31, 0), + /* CLK_CFG_4 */ + TOP_MUX_GATE(CLK_TOP_MSDC30_2_SEL, "msdc30_2_sel", msdc30_2_parents, 0x80, 0, 3, 7, 0), + TOP_MUX_GATE(CLK_TOP_MSDC30_3_SEL, "msdc30_3_sel", msdc30_3_parents, 0x80, 8, 3, 15, 0), + TOP_MUX_GATE(CLK_TOP_AUDIO_SEL, "audio_sel", audio_parents, 0x80, 16, 2, 23, 0), + TOP_MUX_GATE(CLK_TOP_AUD_INTBUS_SEL, "aud_intbus_sel", aud_intbus_parents, + 0x80, 24, 3, 31, 0), + /* CLK_CFG_5 */ + TOP_MUX_GATE(CLK_TOP_PMICSPI_SEL, "pmicspi_sel", pmicspi_parents, 0x90, 0, 3, 5, 0), + TOP_MUX_GATE(CLK_TOP_SCP_SEL, "scp_sel", scp_parents, 0x90, 8, 3, 15, 0), + TOP_MUX_GATE(CLK_TOP_MJC_SEL, "mjc_sel", mjc_parents, 0x90, 24, 4, 31, 0), + /* CLK_CFG_6 */ + /* + * The dpi0_sel clock should not propagate rate changes to its parent + * clock so the dpi driver can have full control over PLL and divider. + */ + TOP_MUX_GATE_NOSR(CLK_TOP_DPI0_SEL, "dpi0_sel", dpi0_parents, 0xa0, 0, 3, 7, 0), + TOP_MUX_GATE(CLK_TOP_IRDA_SEL, "irda_sel", irda_parents, 0xa0, 8, 2, 15, 0), + TOP_MUX_GATE(CLK_TOP_CCI400_SEL, "cci400_sel", cci400_parents, + 0xa0, 16, 3, 23, CLK_IS_CRITICAL), + TOP_MUX_GATE(CLK_TOP_AUD_1_SEL, "aud_1_sel", aud_1_parents, 0xa0, 24, 2, 31, 0), + /* CLK_CFG_7 */ + TOP_MUX_GATE(CLK_TOP_AUD_2_SEL, "aud_2_sel", aud_2_parents, 0xb0, 0, 2, 7, 0), + TOP_MUX_GATE(CLK_TOP_MEM_MFG_IN_SEL, "mem_mfg_in_sel", mem_mfg_in_parents, + 0xb0, 8, 2, 15, 0), + TOP_MUX_GATE(CLK_TOP_AXI_MFG_IN_SEL, "axi_mfg_in_sel", axi_mfg_in_parents, + 0xb0, 16, 2, 23, 0), + TOP_MUX_GATE(CLK_TOP_SCAM_SEL, "scam_sel", scam_parents, 0xb0, 24, 2, 31, 0), +}; + +static struct mtk_composite top_aud_divs[] = { + MUX(CLK_TOP_I2S0_M_SEL, "i2s0_m_ck_sel", i2s0_m_ck_parents, 0x120, 4, 1), + MUX(CLK_TOP_I2S1_M_SEL, "i2s1_m_ck_sel", i2s1_m_ck_parents, 0x120, 5, 1), + MUX(CLK_TOP_I2S2_M_SEL, "i2s2_m_ck_sel", i2s2_m_ck_parents, 0x120, 6, 1), + MUX(CLK_TOP_I2S3_M_SEL, "i2s3_m_ck_sel", i2s3_m_ck_parents, 0x120, 7, 1), + MUX(CLK_TOP_I2S3_B_SEL, "i2s3_b_ck_sel", i2s3_b_ck_parents, 0x120, 8, 1), + + DIV_GATE(CLK_TOP_APLL1_DIV0, "apll1_div0", "aud_1_sel", 0x12c, 8, 0x120, 4, 24), + DIV_GATE(CLK_TOP_APLL1_DIV1, "apll1_div1", "aud_1_sel", 0x12c, 9, 0x124, 8, 0), + DIV_GATE(CLK_TOP_APLL1_DIV2, "apll1_div2", "aud_1_sel", 0x12c, 10, 0x124, 8, 8), + DIV_GATE(CLK_TOP_APLL1_DIV3, "apll1_div3", "aud_1_sel", 0x12c, 11, 0x124, 8, 16), + DIV_GATE(CLK_TOP_APLL1_DIV4, "apll1_div4", "aud_1_sel", 0x12c, 12, 0x124, 8, 24), + DIV_GATE(CLK_TOP_APLL1_DIV5, "apll1_div5", "apll1_div4", 0x12c, 13, 0x12c, 4, 0), + + DIV_GATE(CLK_TOP_APLL2_DIV0, "apll2_div0", "aud_2_sel", 0x12c, 16, 0x120, 4, 28), + DIV_GATE(CLK_TOP_APLL2_DIV1, "apll2_div1", "aud_2_sel", 0x12c, 17, 0x128, 8, 0), + DIV_GATE(CLK_TOP_APLL2_DIV2, "apll2_div2", "aud_2_sel", 0x12c, 18, 0x128, 8, 8), + DIV_GATE(CLK_TOP_APLL2_DIV3, "apll2_div3", "aud_2_sel", 0x12c, 19, 0x128, 8, 16), + DIV_GATE(CLK_TOP_APLL2_DIV4, "apll2_div4", "aud_2_sel", 0x12c, 20, 0x128, 8, 24), + DIV_GATE(CLK_TOP_APLL2_DIV5, "apll2_div5", "apll2_div4", 0x12c, 21, 0x12c, 4, 4), +}; + + +static const struct of_device_id of_match_clk_mt6795_topckgen[] = { + { .compatible = "mediatek,mt6795-topckgen" }, + { /* sentinel */ } +}; + +static int clk_mt6795_topckgen_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + void __iomem *base; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks), clk_data); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); + if (ret) + goto unregister_fixed_clks; + + ret = mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes), node, + &mt6795_top_clk_lock, clk_data); + if (ret) + goto unregister_factors; + + ret = mtk_clk_register_composites(top_aud_divs, ARRAY_SIZE(top_aud_divs), base, + &mt6795_top_clk_lock, clk_data); + if (ret) + goto unregister_muxes; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_composites; + + return 0; + +unregister_composites: + mtk_clk_unregister_composites(top_aud_divs, ARRAY_SIZE(top_aud_divs), clk_data); +unregister_muxes: + mtk_clk_unregister_muxes(top_muxes, ARRAY_SIZE(top_muxes), clk_data); +unregister_factors: + mtk_clk_unregister_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); +unregister_fixed_clks: + mtk_clk_unregister_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + return ret; +} + +static int clk_mt6795_topckgen_remove(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct clk_hw_onecell_data *clk_data = platform_get_drvdata(pdev); + + of_clk_del_provider(node); + mtk_clk_unregister_composites(top_aud_divs, ARRAY_SIZE(top_aud_divs), clk_data); + mtk_clk_unregister_muxes(top_muxes, ARRAY_SIZE(top_muxes), clk_data); + mtk_clk_unregister_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); + mtk_clk_unregister_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks), clk_data); + mtk_free_clk_data(clk_data); + + return 0; +} + +static struct platform_driver clk_mt6795_topckgen_drv = { + .driver = { + .name = "clk-mt6795-topckgen", + .of_match_table = of_match_clk_mt6795_topckgen, + }, + .probe = clk_mt6795_topckgen_probe, + .remove = clk_mt6795_topckgen_remove, +}; +module_platform_driver(clk_mt6795_topckgen_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 topckgen clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-vdecsys.c b/drivers/clk/mediatek/clk-mt6795-vdecsys.c new file mode 100644 index 0000000000000000000000000000000000000000..d85d04e0d0169da106fa882fba2bd876f0ffa21c --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-vdecsys.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" + +#define GATE_VDEC(_id, _name, _parent, _regs) \ + GATE_MTK(_id, _name, _parent, _regs, 0, \ + &mtk_clk_gate_ops_setclr_inv) + +static const struct mtk_gate_regs vdec0_cg_regs = { + .set_ofs = 0x0000, + .clr_ofs = 0x0004, + .sta_ofs = 0x0000, +}; + +static const struct mtk_gate_regs vdec1_cg_regs = { + .set_ofs = 0x0008, + .clr_ofs = 0x000c, + .sta_ofs = 0x0008, +}; + +static const struct mtk_gate vdec_clks[] = { + GATE_VDEC(CLK_VDEC_CKEN, "vdec_cken", "vdec_sel", &vdec0_cg_regs), + GATE_VDEC(CLK_VDEC_LARB_CKEN, "vdec_larb_cken", "mm_sel", &vdec1_cg_regs), +}; + +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), +}; + +static const struct of_device_id of_match_clk_mt6795_vdecsys[] = { + { .compatible = "mediatek,mt6795-vdecsys", .data = &vdec_desc }, + { /* sentinel */ } +}; + +static struct platform_driver clk_mt6795_vdecsys_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt6795-vdecsys", + .of_match_table = of_match_clk_mt6795_vdecsys, + }, +}; +module_platform_driver(clk_mt6795_vdecsys_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 vdecsys clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6795-vencsys.c b/drivers/clk/mediatek/clk-mt6795-vencsys.c new file mode 100644 index 0000000000000000000000000000000000000000..de40a982ca962f0ce5db240ede5580c7c9ffd514 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt6795-vencsys.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#include +#include +#include +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs venc_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_VENC(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &venc_cg_regs, _shift, &mtk_clk_gate_ops_setclr_inv) + +static const struct mtk_gate venc_clks[] = { + GATE_VENC(CLK_VENC_LARB, "venc_larb", "venc_sel", 0), + GATE_VENC(CLK_VENC_VENC, "venc_venc", "venc_sel", 4), + GATE_VENC(CLK_VENC_JPGENC, "venc_jpgenc", "venc_sel", 8), + GATE_VENC(CLK_VENC_JPGDEC, "venc_jpgdec", "venc_sel", 12), +}; + +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), +}; + +static const struct of_device_id of_match_clk_mt6795_vencsys[] = { + { .compatible = "mediatek,mt6795-vencsys", .data = &venc_desc }, + { /* sentinel */ } +}; + +static struct platform_driver clk_mt6795_vencsys_drv = { + .driver = { + .name = "clk-mt6795-vencsys", + .of_match_table = of_match_clk_mt6795_vencsys, + }, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, +}; +module_platform_driver(clk_mt6795_vencsys_drv); + +MODULE_DESCRIPTION("MediaTek MT6795 vdecsys clocks driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt6797-img.c b/drivers/clk/mediatek/clk-mt6797-img.c index 25d17db13bac3ca4674617a4a77aa0ab6e6c2cfe..7c6a53fbb8be68211b8e23f0ad02b429e62d22fc 100644 --- a/drivers/clk/mediatek/clk-mt6797-img.c +++ b/drivers/clk/mediatek/clk-mt6797-img.c @@ -32,33 +32,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_LARB6, "img_larb6", "mm_sel", 0), }; -static const struct of_device_id of_match_clk_mt6797_img[] = { - { .compatible = "mediatek,mt6797-imgsys", }, - {} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), }; -static int clk_mt6797_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt6797_img[] = { + { + .compatible = "mediatek,mt6797-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6797_img_drv = { - .probe = clk_mt6797_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6797-img", .of_match_table = of_match_clk_mt6797_img, diff --git a/drivers/clk/mediatek/clk-mt6797-vdec.c b/drivers/clk/mediatek/clk-mt6797-vdec.c index de857894e0338fdc434e166cad8ae4c8ed68b9a4..6120fccc859f1b9fa9a7458bf43588878e3cc3cb 100644 --- a/drivers/clk/mediatek/clk-mt6797-vdec.c +++ b/drivers/clk/mediatek/clk-mt6797-vdec.c @@ -49,33 +49,23 @@ static const struct mtk_gate vdec_clks[] = { GATE_VDEC1(CLK_VDEC_LARB1_CKEN, "vdec_larb1_cken", "mm_sel", 0), }; -static const struct of_device_id of_match_clk_mt6797_vdec[] = { - { .compatible = "mediatek,mt6797-vdecsys", }, - {} +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), }; -static int clk_mt6797_vdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VDEC_NR); - - mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt6797_vdec[] = { + { + .compatible = "mediatek,mt6797-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6797_vdec_drv = { - .probe = clk_mt6797_vdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6797-vdec", .of_match_table = of_match_clk_mt6797_vdec, diff --git a/drivers/clk/mediatek/clk-mt6797-venc.c b/drivers/clk/mediatek/clk-mt6797-venc.c index 78b7ed55f9799d5bdb7acc35c5b10d862b8f7fd3..834d3834d2bbcccbf616bd20a5984ae0213a5d42 100644 --- a/drivers/clk/mediatek/clk-mt6797-venc.c +++ b/drivers/clk/mediatek/clk-mt6797-venc.c @@ -34,33 +34,23 @@ static const struct mtk_gate venc_clks[] = { GATE_VENC(CLK_VENC_3, "venc_3", "venc_sel", 12), }; -static const struct of_device_id of_match_clk_mt6797_venc[] = { - { .compatible = "mediatek,mt6797-vencsys", }, - {} +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), }; -static int clk_mt6797_venc_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - int r; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VENC_NR); - - mtk_clk_register_gates(node, venc_clks, ARRAY_SIZE(venc_clks), - clk_data); - - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); - if (r) - dev_err(&pdev->dev, - "could not register clock provider: %s: %d\n", - pdev->name, r); - - return r; -} +static const struct of_device_id of_match_clk_mt6797_venc[] = { + { + .compatible = "mediatek,mt6797-vencsys", + .data = &venc_desc, + }, { + /* sentinel */ + } +}; static struct platform_driver clk_mt6797_venc_drv = { - .probe = clk_mt6797_venc_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt6797-venc", .of_match_table = of_match_clk_mt6797_venc, diff --git a/drivers/clk/mediatek/clk-mt8183-cam.c b/drivers/clk/mediatek/clk-mt8183-cam.c index fcc598a45165c313a1c92a881a3419546e58d0b0..6907b1a6a8247cf4ea388e5ca4a5512d04b249cd 100644 --- a/drivers/clk/mediatek/clk-mt8183-cam.c +++ b/drivers/clk/mediatek/clk-mt8183-cam.c @@ -34,26 +34,23 @@ static const struct mtk_gate cam_clks[] = { GATE_CAM(CLK_CAM_CCU, "cam_ccu", "cam_sel", 12), }; -static int clk_mt8183_cam_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_CAM_NR_CLK); - - mtk_clk_register_gates(node, cam_clks, ARRAY_SIZE(cam_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc cam_desc = { + .clks = cam_clks, + .num_clks = ARRAY_SIZE(cam_clks), +}; static const struct of_device_id of_match_clk_mt8183_cam[] = { - { .compatible = "mediatek,mt8183-camsys", }, - {} + { + .compatible = "mediatek,mt8183-camsys", + .data = &cam_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_cam_drv = { - .probe = clk_mt8183_cam_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-cam", .of_match_table = of_match_clk_mt8183_cam, diff --git a/drivers/clk/mediatek/clk-mt8183-img.c b/drivers/clk/mediatek/clk-mt8183-img.c index eb2def2cf0ae54a394febdde08c65574610b4bd3..8d884425d79f27c7b124ddab0017c67bb8b8f7b0 100644 --- a/drivers/clk/mediatek/clk-mt8183-img.c +++ b/drivers/clk/mediatek/clk-mt8183-img.c @@ -34,26 +34,23 @@ static const struct mtk_gate img_clks[] = { GATE_IMG(CLK_IMG_OWE, "img_owe", "img_sel", 9), }; -static int clk_mt8183_img_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK); - - mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc img_desc = { + .clks = img_clks, + .num_clks = ARRAY_SIZE(img_clks), +}; static const struct of_device_id of_match_clk_mt8183_img[] = { - { .compatible = "mediatek,mt8183-imgsys", }, - {} + { + .compatible = "mediatek,mt8183-imgsys", + .data = &img_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_img_drv = { - .probe = clk_mt8183_img_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-img", .of_match_table = of_match_clk_mt8183_img, diff --git a/drivers/clk/mediatek/clk-mt8183-ipu0.c b/drivers/clk/mediatek/clk-mt8183-ipu0.c index b30fc9f47518e9ac3013cd76738f44e7bb761da7..953a8a33d0487feb753f57c77a108ab1dc3f0e16 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu0.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu0.c @@ -27,26 +27,23 @@ static const struct mtk_gate ipu_core0_clks[] = { GATE_IPU_CORE0(CLK_IPU_CORE0_IPU, "ipu_core0_ipu", "dsp_sel", 2), }; -static int clk_mt8183_ipu_core0_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IPU_CORE0_NR_CLK); - - mtk_clk_register_gates(node, ipu_core0_clks, ARRAY_SIZE(ipu_core0_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc ipu_core0_desc = { + .clks = ipu_core0_clks, + .num_clks = ARRAY_SIZE(ipu_core0_clks), +}; static const struct of_device_id of_match_clk_mt8183_ipu_core0[] = { - { .compatible = "mediatek,mt8183-ipu_core0", }, - {} + { + .compatible = "mediatek,mt8183-ipu_core0", + .data = &ipu_core0_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_ipu_core0_drv = { - .probe = clk_mt8183_ipu_core0_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-ipu_core0", .of_match_table = of_match_clk_mt8183_ipu_core0, diff --git a/drivers/clk/mediatek/clk-mt8183-ipu1.c b/drivers/clk/mediatek/clk-mt8183-ipu1.c index b378957e11d0f74032d3fc3e90f37fa4803db2d2..221d1226597457344b8abea1df16d40b046976bf 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu1.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu1.c @@ -27,26 +27,23 @@ static const struct mtk_gate ipu_core1_clks[] = { GATE_IPU_CORE1(CLK_IPU_CORE1_IPU, "ipu_core1_ipu", "dsp_sel", 2), }; -static int clk_mt8183_ipu_core1_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IPU_CORE1_NR_CLK); - - mtk_clk_register_gates(node, ipu_core1_clks, ARRAY_SIZE(ipu_core1_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc ipu_core1_desc = { + .clks = ipu_core1_clks, + .num_clks = ARRAY_SIZE(ipu_core1_clks), +}; static const struct of_device_id of_match_clk_mt8183_ipu_core1[] = { - { .compatible = "mediatek,mt8183-ipu_core1", }, - {} + { + .compatible = "mediatek,mt8183-ipu_core1", + .data = &ipu_core1_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_ipu_core1_drv = { - .probe = clk_mt8183_ipu_core1_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-ipu_core1", .of_match_table = of_match_clk_mt8183_ipu_core1, diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_adl.c b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c index 941b43ac8bec6cb61d164128a28c5b93fcd3e6e9..8c4fd96df821e1d631f4e92544856b32faf0e9dc 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu_adl.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c @@ -25,26 +25,23 @@ static const struct mtk_gate ipu_adl_clks[] = { GATE_IPU_ADL_I(CLK_IPU_ADL_CABGEN, "ipu_adl_cabgen", "dsp_sel", 24), }; -static int clk_mt8183_ipu_adl_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IPU_ADL_NR_CLK); - - mtk_clk_register_gates(node, ipu_adl_clks, ARRAY_SIZE(ipu_adl_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc ipu_adl_desc = { + .clks = ipu_adl_clks, + .num_clks = ARRAY_SIZE(ipu_adl_clks), +}; static const struct of_device_id of_match_clk_mt8183_ipu_adl[] = { - { .compatible = "mediatek,mt8183-ipu_adl", }, - {} + { + .compatible = "mediatek,mt8183-ipu_adl", + .data = &ipu_adl_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_ipu_adl_drv = { - .probe = clk_mt8183_ipu_adl_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-ipu_adl", .of_match_table = of_match_clk_mt8183_ipu_adl, diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_conn.c b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c index ae82c2e17110f3ed2f792974c640b0e58b6cdd35..14a4c3ff82a1bcde3d715c6e69be91596a81d287 100644 --- a/drivers/clk/mediatek/clk-mt8183-ipu_conn.c +++ b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c @@ -94,26 +94,23 @@ static const struct mtk_gate ipu_conn_clks[] = { "ipu_conn_cab3to1_slice", "dsp1_sel", 17), }; -static int clk_mt8183_ipu_conn_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_IPU_CONN_NR_CLK); - - mtk_clk_register_gates(node, ipu_conn_clks, ARRAY_SIZE(ipu_conn_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc ipu_conn_desc = { + .clks = ipu_conn_clks, + .num_clks = ARRAY_SIZE(ipu_conn_clks), +}; static const struct of_device_id of_match_clk_mt8183_ipu_conn[] = { - { .compatible = "mediatek,mt8183-ipu_conn", }, - {} + { + .compatible = "mediatek,mt8183-ipu_conn", + .data = &ipu_conn_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_ipu_conn_drv = { - .probe = clk_mt8183_ipu_conn_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-ipu_conn", .of_match_table = of_match_clk_mt8183_ipu_conn, diff --git a/drivers/clk/mediatek/clk-mt8183-mfgcfg.c b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c index d774edaf760be73b08e3a24b3155da1d465bed6e..730c9ae5ea124596cbf5f198779750f45bdfdc90 100644 --- a/drivers/clk/mediatek/clk-mt8183-mfgcfg.c +++ b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c @@ -18,36 +18,31 @@ static const struct mtk_gate_regs mfg_cg_regs = { .sta_ofs = 0x0, }; -#define GATE_MFG(_id, _name, _parent, _shift) \ - GATE_MTK(_id, _name, _parent, &mfg_cg_regs, _shift, \ - &mtk_clk_gate_ops_setclr) +#define GATE_MFG(_id, _name, _parent, _shift) \ + GATE_MTK_FLAGS(_id, _name, _parent, &mfg_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr, CLK_SET_RATE_PARENT) static const struct mtk_gate mfg_clks[] = { GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_sel", 0) }; -static int clk_mt8183_mfg_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - pm_runtime_enable(&pdev->dev); - - clk_data = mtk_alloc_clk_data(CLK_MFG_NR_CLK); - - mtk_clk_register_gates_with_dev(node, mfg_clks, ARRAY_SIZE(mfg_clks), - clk_data, &pdev->dev); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc mfg_desc = { + .clks = mfg_clks, + .num_clks = ARRAY_SIZE(mfg_clks), +}; static const struct of_device_id of_match_clk_mt8183_mfg[] = { - { .compatible = "mediatek,mt8183-mfgcfg", }, - {} + { + .compatible = "mediatek,mt8183-mfgcfg", + .data = &mfg_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_mfg_drv = { - .probe = clk_mt8183_mfg_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-mfg", .of_match_table = of_match_clk_mt8183_mfg, diff --git a/drivers/clk/mediatek/clk-mt8183-vdec.c b/drivers/clk/mediatek/clk-mt8183-vdec.c index 0548cde159d09aba75e3bc0116d7807eb14769ad..c294e50b96b74bc84f256907945ee2f10c5a7b61 100644 --- a/drivers/clk/mediatek/clk-mt8183-vdec.c +++ b/drivers/clk/mediatek/clk-mt8183-vdec.c @@ -38,26 +38,23 @@ static const struct mtk_gate vdec_clks[] = { GATE_VDEC1_I(CLK_VDEC_LARB1, "vdec_larb1", "mm_sel", 0), }; -static int clk_mt8183_vdec_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VDEC_NR_CLK); - - mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), +}; static const struct of_device_id of_match_clk_mt8183_vdec[] = { - { .compatible = "mediatek,mt8183-vdecsys", }, - {} + { + .compatible = "mediatek,mt8183-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_vdec_drv = { - .probe = clk_mt8183_vdec_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-vdec", .of_match_table = of_match_clk_mt8183_vdec, diff --git a/drivers/clk/mediatek/clk-mt8183-venc.c b/drivers/clk/mediatek/clk-mt8183-venc.c index f86ec607d87a999213c7722edc2a2e7c67e82b9d..0051c5d92fc5d1f1dd33ad42dc6f9246f81f98db 100644 --- a/drivers/clk/mediatek/clk-mt8183-venc.c +++ b/drivers/clk/mediatek/clk-mt8183-venc.c @@ -30,26 +30,23 @@ static const struct mtk_gate venc_clks[] = { "mm_sel", 8), }; -static int clk_mt8183_venc_probe(struct platform_device *pdev) -{ - struct clk_hw_onecell_data *clk_data; - struct device_node *node = pdev->dev.of_node; - - clk_data = mtk_alloc_clk_data(CLK_VENC_NR_CLK); - - mtk_clk_register_gates(node, venc_clks, ARRAY_SIZE(venc_clks), - clk_data); - - return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); -} +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), +}; static const struct of_device_id of_match_clk_mt8183_venc[] = { - { .compatible = "mediatek,mt8183-vencsys", }, - {} + { + .compatible = "mediatek,mt8183-vencsys", + .data = &venc_desc, + }, { + /* sentinel */ + } }; static struct platform_driver clk_mt8183_venc_drv = { - .probe = clk_mt8183_venc_probe, + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8183-venc", .of_match_table = of_match_clk_mt8183_venc, diff --git a/drivers/clk/mediatek/clk-mt8183.c b/drivers/clk/mediatek/clk-mt8183.c index 8512101e1189a144c7ef01c45945790ff89ddea9..1860a35a723a59a8b666c117016d464dc3868b51 100644 --- a/drivers/clk/mediatek/clk-mt8183.c +++ b/drivers/clk/mediatek/clk-mt8183.c @@ -1198,10 +1198,33 @@ static void clk_mt8183_top_init_early(struct device_node *node) CLK_OF_DECLARE_DRIVER(mt8183_topckgen, "mediatek,mt8183-topckgen", clk_mt8183_top_init_early); +/* Register mux notifier for MFG mux */ +static int clk_mt8183_reg_mfg_mux_notifier(struct device *dev, struct clk *clk) +{ + struct mtk_mux_nb *mfg_mux_nb; + int i; + + mfg_mux_nb = devm_kzalloc(dev, sizeof(*mfg_mux_nb), GFP_KERNEL); + if (!mfg_mux_nb) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(top_muxes); i++) + if (top_muxes[i].id == CLK_TOP_MUX_MFG) + break; + if (i == ARRAY_SIZE(top_muxes)) + return -EINVAL; + + mfg_mux_nb->ops = top_muxes[i].ops; + mfg_mux_nb->bypass_index = 0; /* Bypass to 26M crystal */ + + return devm_mtk_clk_mux_notifier_register(dev, clk, mfg_mux_nb); +} + static int clk_mt8183_top_probe(struct platform_device *pdev) { void __iomem *base; struct device_node *node = pdev->dev.of_node; + int ret; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) @@ -1227,6 +1250,11 @@ static int clk_mt8183_top_probe(struct platform_device *pdev) mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks), top_clk_data); + ret = clk_mt8183_reg_mfg_mux_notifier(&pdev->dev, + top_clk_data->hws[CLK_TOP_MUX_MFG]->clk); + if (ret) + return ret; + return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, top_clk_data); } diff --git a/drivers/clk/mediatek/clk-mt8192-cam.c b/drivers/clk/mediatek/clk-mt8192-cam.c index fc74cd80b4b02f773b95cea0c587bec94603f389..90b57d46eef74db8cc2c8c640a66f25fe1225903 100644 --- a/drivers/clk/mediatek/clk-mt8192-cam.c +++ b/drivers/clk/mediatek/clk-mt8192-cam.c @@ -98,6 +98,7 @@ static const struct of_device_id of_match_clk_mt8192_cam[] = { static struct platform_driver clk_mt8192_cam_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-cam", .of_match_table = of_match_clk_mt8192_cam, diff --git a/drivers/clk/mediatek/clk-mt8192-img.c b/drivers/clk/mediatek/clk-mt8192-img.c index 7ce3abe42577c52ec27b0a48c42c5a4ffebcdd35..da82d65a765015f3fa161b5ad1977d4a90d21767 100644 --- a/drivers/clk/mediatek/clk-mt8192-img.c +++ b/drivers/clk/mediatek/clk-mt8192-img.c @@ -61,6 +61,7 @@ static const struct of_device_id of_match_clk_mt8192_img[] = { static struct platform_driver clk_mt8192_img_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-img", .of_match_table = of_match_clk_mt8192_img, diff --git a/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c b/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c index 700356ac6a581615ded8dcf9f5cc5ad5d638dec7..ff8e20bb44bb0ed1dc8d6ea7bd368a47d1f528cb 100644 --- a/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c +++ b/drivers/clk/mediatek/clk-mt8192-imp_iic_wrap.c @@ -110,6 +110,7 @@ static const struct of_device_id of_match_clk_mt8192_imp_iic_wrap[] = { static struct platform_driver clk_mt8192_imp_iic_wrap_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-imp_iic_wrap", .of_match_table = of_match_clk_mt8192_imp_iic_wrap, diff --git a/drivers/clk/mediatek/clk-mt8192-ipe.c b/drivers/clk/mediatek/clk-mt8192-ipe.c index 730d91b64b3f36b7ecb6b23cdf515e02dfcfb099..0225abe4170a916a1d32270af3adb8de574505a7 100644 --- a/drivers/clk/mediatek/clk-mt8192-ipe.c +++ b/drivers/clk/mediatek/clk-mt8192-ipe.c @@ -48,6 +48,7 @@ static const struct of_device_id of_match_clk_mt8192_ipe[] = { static struct platform_driver clk_mt8192_ipe_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-ipe", .of_match_table = of_match_clk_mt8192_ipe, diff --git a/drivers/clk/mediatek/clk-mt8192-mdp.c b/drivers/clk/mediatek/clk-mt8192-mdp.c index 93c87ae2f332a3276198e3dc9240f3abb1526767..4675788d78169a04db7981e0a387d7c49212a9de 100644 --- a/drivers/clk/mediatek/clk-mt8192-mdp.c +++ b/drivers/clk/mediatek/clk-mt8192-mdp.c @@ -73,6 +73,7 @@ static const struct of_device_id of_match_clk_mt8192_mdp[] = { static struct platform_driver clk_mt8192_mdp_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-mdp", .of_match_table = of_match_clk_mt8192_mdp, diff --git a/drivers/clk/mediatek/clk-mt8192-mfg.c b/drivers/clk/mediatek/clk-mt8192-mfg.c index 3bbc7469f0e4e66617c554b959818a236e009b2f..ec5b44ffa458d797afd018660817592814bb879a 100644 --- a/drivers/clk/mediatek/clk-mt8192-mfg.c +++ b/drivers/clk/mediatek/clk-mt8192-mfg.c @@ -18,8 +18,10 @@ static const struct mtk_gate_regs mfg_cg_regs = { .sta_ofs = 0x0, }; -#define GATE_MFG(_id, _name, _parent, _shift) \ - GATE_MTK(_id, _name, _parent, &mfg_cg_regs, _shift, &mtk_clk_gate_ops_setclr) +#define GATE_MFG(_id, _name, _parent, _shift) \ + GATE_MTK_FLAGS(_id, _name, _parent, &mfg_cg_regs, \ + _shift, &mtk_clk_gate_ops_setclr, \ + CLK_SET_RATE_PARENT) static const struct mtk_gate mfg_clks[] = { GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_pll_sel", 0), @@ -41,6 +43,7 @@ static const struct of_device_id of_match_clk_mt8192_mfg[] = { static struct platform_driver clk_mt8192_mfg_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-mfg", .of_match_table = of_match_clk_mt8192_mfg, diff --git a/drivers/clk/mediatek/clk-mt8192-msdc.c b/drivers/clk/mediatek/clk-mt8192-msdc.c index 635f7a0b629acfc1fee6c2fa79ebc8d48fe9e6cb..a72e1b73fce8649e92be9d473083f06aa907cb4b 100644 --- a/drivers/clk/mediatek/clk-mt8192-msdc.c +++ b/drivers/clk/mediatek/clk-mt8192-msdc.c @@ -55,6 +55,7 @@ static const struct of_device_id of_match_clk_mt8192_msdc[] = { static struct platform_driver clk_mt8192_msdc_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-msdc", .of_match_table = of_match_clk_mt8192_msdc, diff --git a/drivers/clk/mediatek/clk-mt8192-scp_adsp.c b/drivers/clk/mediatek/clk-mt8192-scp_adsp.c index 58725d79dd13cfad4ab885822f1dffcbb2dcdfc6..18a8679108b860d2e85c9e47820245d54445ca26 100644 --- a/drivers/clk/mediatek/clk-mt8192-scp_adsp.c +++ b/drivers/clk/mediatek/clk-mt8192-scp_adsp.c @@ -41,6 +41,7 @@ static const struct of_device_id of_match_clk_mt8192_scp_adsp[] = { static struct platform_driver clk_mt8192_scp_adsp_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-scp_adsp", .of_match_table = of_match_clk_mt8192_scp_adsp, diff --git a/drivers/clk/mediatek/clk-mt8192-vdec.c b/drivers/clk/mediatek/clk-mt8192-vdec.c index b1d95cfbf22afc1e1b11952cd27edff0cde1b2ae..e149962dbbf95fffbb7a2b4c46227967846f2b25 100644 --- a/drivers/clk/mediatek/clk-mt8192-vdec.c +++ b/drivers/clk/mediatek/clk-mt8192-vdec.c @@ -85,6 +85,7 @@ static const struct of_device_id of_match_clk_mt8192_vdec[] = { static struct platform_driver clk_mt8192_vdec_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-vdec", .of_match_table = of_match_clk_mt8192_vdec, diff --git a/drivers/clk/mediatek/clk-mt8192-venc.c b/drivers/clk/mediatek/clk-mt8192-venc.c index c0d867bff09e21c545ea6bdb786b035300b4e40e..80b8bb170996bbb65c58870c5dc9eacce577367c 100644 --- a/drivers/clk/mediatek/clk-mt8192-venc.c +++ b/drivers/clk/mediatek/clk-mt8192-venc.c @@ -44,6 +44,7 @@ static const struct of_device_id of_match_clk_mt8192_venc[] = { static struct platform_driver clk_mt8192_venc_drv = { .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, .driver = { .name = "clk-mt8192-venc", .of_match_table = of_match_clk_mt8192_venc, diff --git a/drivers/clk/mediatek/clk-mt8192.c b/drivers/clk/mediatek/clk-mt8192.c index ebbd2798d9a32a593391cba1158ca19c6394e5ee..d0f2269310706544ac20916698074f017057391c 100644 --- a/drivers/clk/mediatek/clk-mt8192.c +++ b/drivers/clk/mediatek/clk-mt8192.c @@ -167,22 +167,7 @@ static const char * const mdp_parents[] = { "mmpll_d5_d2" }; -static const char * const img1_parents[] = { - "clk26m", - "univpll_d4", - "tvdpll_ck", - "mainpll_d4", - "univpll_d5", - "mmpll_d6", - "univpll_d6", - "mainpll_d6", - "mmpll_d4_d2", - "mainpll_d4_d2", - "mmpll_d6_d2", - "mmpll_d5_d2" -}; - -static const char * const img2_parents[] = { +static const char * const img_parents[] = { "clk26m", "univpll_d4", "tvdpll_ck", @@ -280,61 +265,6 @@ static const char * const camtg_parents[] = { "univpll_192m_d32" }; -static const char * const camtg2_parents[] = { - "clk26m", - "univpll_192m_d8", - "univpll_d6_d8", - "univpll_192m_d4", - "univpll_d6_d16", - "csw_f26m_d2", - "univpll_192m_d16", - "univpll_192m_d32" -}; - -static const char * const camtg3_parents[] = { - "clk26m", - "univpll_192m_d8", - "univpll_d6_d8", - "univpll_192m_d4", - "univpll_d6_d16", - "csw_f26m_d2", - "univpll_192m_d16", - "univpll_192m_d32" -}; - -static const char * const camtg4_parents[] = { - "clk26m", - "univpll_192m_d8", - "univpll_d6_d8", - "univpll_192m_d4", - "univpll_d6_d16", - "csw_f26m_d2", - "univpll_192m_d16", - "univpll_192m_d32" -}; - -static const char * const camtg5_parents[] = { - "clk26m", - "univpll_192m_d8", - "univpll_d6_d8", - "univpll_192m_d4", - "univpll_d6_d16", - "csw_f26m_d2", - "univpll_192m_d16", - "univpll_192m_d32" -}; - -static const char * const camtg6_parents[] = { - "clk26m", - "univpll_192m_d8", - "univpll_d6_d8", - "univpll_192m_d4", - "univpll_d6_d16", - "csw_f26m_d2", - "univpll_192m_d16", - "univpll_192m_d32" -}; - static const char * const uart_parents[] = { "clk26m", "univpll_d6_d8" @@ -362,15 +292,7 @@ static const char * const msdc50_0_parents[] = { "univpll_d4_d2" }; -static const char * const msdc30_1_parents[] = { - "clk26m", - "univpll_d6_d2", - "mainpll_d6_d2", - "mainpll_d7_d2", - "msdcpll_d2" -}; - -static const char * const msdc30_2_parents[] = { +static const char * const msdc30_parents[] = { "clk26m", "univpll_d6_d2", "mainpll_d6_d2", @@ -457,39 +379,6 @@ static const char * const seninf_parents[] = { "univpll_d5" }; -static const char * const seninf1_parents[] = { - "clk26m", - "univpll_d4_d4", - "univpll_d6_d2", - "univpll_d4_d2", - "univpll_d7", - "univpll_d6", - "mmpll_d6", - "univpll_d5" -}; - -static const char * const seninf2_parents[] = { - "clk26m", - "univpll_d4_d4", - "univpll_d6_d2", - "univpll_d4_d2", - "univpll_d7", - "univpll_d6", - "mmpll_d6", - "univpll_d5" -}; - -static const char * const seninf3_parents[] = { - "clk26m", - "univpll_d4_d4", - "univpll_d6_d2", - "univpll_d4_d2", - "univpll_d7", - "univpll_d6", - "mmpll_d6", - "univpll_d5" -}; - static const char * const tl_parents[] = { "clk26m", "univpll_192m_d2", @@ -649,52 +538,7 @@ static const char * const sflash_parents[] = { "univpll_d5_d8" }; -static const char * const apll_i2s0_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s1_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s2_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s3_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s4_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s5_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s6_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s7_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s8_m_parents[] = { - "aud_1_sel", - "aud_2_sel" -}; - -static const char * const apll_i2s9_m_parents[] = { +static const char * const apll_i2s_m_parents[] = { "aud_1_sel", "aud_2_sel" }; @@ -724,9 +568,9 @@ static const struct mtk_mux top_mtk_muxes[] = { MUX_GATE_CLR_SET_UPD(CLK_TOP_MDP_SEL, "mdp_sel", mdp_parents, 0x020, 0x024, 0x028, 8, 4, 15, 0x004, 5), MUX_GATE_CLR_SET_UPD(CLK_TOP_IMG1_SEL, "img1_sel", - img1_parents, 0x020, 0x024, 0x028, 16, 4, 23, 0x004, 6), + img_parents, 0x020, 0x024, 0x028, 16, 4, 23, 0x004, 6), MUX_GATE_CLR_SET_UPD(CLK_TOP_IMG2_SEL, "img2_sel", - img2_parents, 0x020, 0x024, 0x028, 24, 4, 31, 0x004, 7), + img_parents, 0x020, 0x024, 0x028, 24, 4, 31, 0x004, 7), /* CLK_CFG_2 */ MUX_GATE_CLR_SET_UPD(CLK_TOP_IPE_SEL, "ipe_sel", ipe_parents, 0x030, 0x034, 0x038, 0, 4, 7, 0x004, 8), @@ -747,16 +591,16 @@ static const struct mtk_mux top_mtk_muxes[] = { camtg_parents, 0x050, 0x054, 0x058, 24, 3, 31, 0x004, 19), /* CLK_CFG_5 */ MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG2_SEL, "camtg2_sel", - camtg2_parents, 0x060, 0x064, 0x068, 0, 3, 7, 0x004, 20), + camtg_parents, 0x060, 0x064, 0x068, 0, 3, 7, 0x004, 20), MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG3_SEL, "camtg3_sel", - camtg3_parents, 0x060, 0x064, 0x068, 8, 3, 15, 0x004, 21), + camtg_parents, 0x060, 0x064, 0x068, 8, 3, 15, 0x004, 21), MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG4_SEL, "camtg4_sel", - camtg4_parents, 0x060, 0x064, 0x068, 16, 3, 23, 0x004, 22), + camtg_parents, 0x060, 0x064, 0x068, 16, 3, 23, 0x004, 22), MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG5_SEL, "camtg5_sel", - camtg5_parents, 0x060, 0x064, 0x068, 24, 3, 31, 0x004, 23), + camtg_parents, 0x060, 0x064, 0x068, 24, 3, 31, 0x004, 23), /* CLK_CFG_6 */ MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG6_SEL, "camtg6_sel", - camtg6_parents, 0x070, 0x074, 0x078, 0, 3, 7, 0x004, 24), + camtg_parents, 0x070, 0x074, 0x078, 0, 3, 7, 0x004, 24), MUX_GATE_CLR_SET_UPD(CLK_TOP_UART_SEL, "uart_sel", uart_parents, 0x070, 0x074, 0x078, 8, 1, 15, 0x004, 25), MUX_GATE_CLR_SET_UPD(CLK_TOP_SPI_SEL, "spi_sel", @@ -767,9 +611,9 @@ static const struct mtk_mux top_mtk_muxes[] = { MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_0_SEL, "msdc50_0_sel", msdc50_0_parents, 0x080, 0x084, 0x088, 0, 3, 7, 0x004, 28), MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC30_1_SEL, "msdc30_1_sel", - msdc30_1_parents, 0x080, 0x084, 0x088, 8, 3, 15, 0x004, 29), + msdc30_parents, 0x080, 0x084, 0x088, 8, 3, 15, 0x004, 29), MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC30_2_SEL, "msdc30_2_sel", - msdc30_2_parents, 0x080, 0x084, 0x088, 16, 3, 23, 0x004, 30), + msdc30_parents, 0x080, 0x084, 0x088, 16, 3, 23, 0x004, 30), MUX_GATE_CLR_SET_UPD(CLK_TOP_AUDIO_SEL, "audio_sel", audio_parents, 0x080, 0x084, 0x088, 24, 2, 31, 0x008, 0), /* CLK_CFG_8 */ @@ -796,12 +640,12 @@ static const struct mtk_mux top_mtk_muxes[] = { MUX_GATE_CLR_SET_UPD(CLK_TOP_SENINF_SEL, "seninf_sel", seninf_parents, 0x0b0, 0x0b4, 0x0b8, 16, 3, 23, 0x008, 11), MUX_GATE_CLR_SET_UPD(CLK_TOP_SENINF1_SEL, "seninf1_sel", - seninf1_parents, 0x0b0, 0x0b4, 0x0b8, 24, 3, 31, 0x008, 12), + seninf_parents, 0x0b0, 0x0b4, 0x0b8, 24, 3, 31, 0x008, 12), /* CLK_CFG_11 */ MUX_GATE_CLR_SET_UPD(CLK_TOP_SENINF2_SEL, "seninf2_sel", - seninf2_parents, 0x0c0, 0x0c4, 0x0c8, 0, 3, 7, 0x008, 13), + seninf_parents, 0x0c0, 0x0c4, 0x0c8, 0, 3, 7, 0x008, 13), MUX_GATE_CLR_SET_UPD(CLK_TOP_SENINF3_SEL, "seninf3_sel", - seninf3_parents, 0x0c0, 0x0c4, 0x0c8, 8, 3, 15, 0x008, 14), + seninf_parents, 0x0c0, 0x0c4, 0x0c8, 8, 3, 15, 0x008, 14), MUX_GATE_CLR_SET_UPD(CLK_TOP_TL_SEL, "tl_sel", tl_parents, 0x0c0, 0x0c4, 0x0c8, 16, 2, 23, 0x008, 15), MUX_GATE_CLR_SET_UPD(CLK_TOP_DXCC_SEL, "dxcc_sel", @@ -847,16 +691,16 @@ static const struct mtk_mux top_mtk_muxes[] = { static struct mtk_composite top_muxes[] = { /* CLK_AUDDIV_0 */ - MUX(CLK_TOP_APLL_I2S0_M_SEL, "apll_i2s0_m_sel", apll_i2s0_m_parents, 0x320, 16, 1), - MUX(CLK_TOP_APLL_I2S1_M_SEL, "apll_i2s1_m_sel", apll_i2s1_m_parents, 0x320, 17, 1), - MUX(CLK_TOP_APLL_I2S2_M_SEL, "apll_i2s2_m_sel", apll_i2s2_m_parents, 0x320, 18, 1), - MUX(CLK_TOP_APLL_I2S3_M_SEL, "apll_i2s3_m_sel", apll_i2s3_m_parents, 0x320, 19, 1), - MUX(CLK_TOP_APLL_I2S4_M_SEL, "apll_i2s4_m_sel", apll_i2s4_m_parents, 0x320, 20, 1), - MUX(CLK_TOP_APLL_I2S5_M_SEL, "apll_i2s5_m_sel", apll_i2s5_m_parents, 0x320, 21, 1), - MUX(CLK_TOP_APLL_I2S6_M_SEL, "apll_i2s6_m_sel", apll_i2s6_m_parents, 0x320, 22, 1), - MUX(CLK_TOP_APLL_I2S7_M_SEL, "apll_i2s7_m_sel", apll_i2s7_m_parents, 0x320, 23, 1), - MUX(CLK_TOP_APLL_I2S8_M_SEL, "apll_i2s8_m_sel", apll_i2s8_m_parents, 0x320, 24, 1), - MUX(CLK_TOP_APLL_I2S9_M_SEL, "apll_i2s9_m_sel", apll_i2s9_m_parents, 0x320, 25, 1), + MUX(CLK_TOP_APLL_I2S0_M_SEL, "apll_i2s0_m_sel", apll_i2s_m_parents, 0x320, 16, 1), + MUX(CLK_TOP_APLL_I2S1_M_SEL, "apll_i2s1_m_sel", apll_i2s_m_parents, 0x320, 17, 1), + MUX(CLK_TOP_APLL_I2S2_M_SEL, "apll_i2s2_m_sel", apll_i2s_m_parents, 0x320, 18, 1), + MUX(CLK_TOP_APLL_I2S3_M_SEL, "apll_i2s3_m_sel", apll_i2s_m_parents, 0x320, 19, 1), + MUX(CLK_TOP_APLL_I2S4_M_SEL, "apll_i2s4_m_sel", apll_i2s_m_parents, 0x320, 20, 1), + MUX(CLK_TOP_APLL_I2S5_M_SEL, "apll_i2s5_m_sel", apll_i2s_m_parents, 0x320, 21, 1), + MUX(CLK_TOP_APLL_I2S6_M_SEL, "apll_i2s6_m_sel", apll_i2s_m_parents, 0x320, 22, 1), + MUX(CLK_TOP_APLL_I2S7_M_SEL, "apll_i2s7_m_sel", apll_i2s_m_parents, 0x320, 23, 1), + MUX(CLK_TOP_APLL_I2S8_M_SEL, "apll_i2s8_m_sel", apll_i2s_m_parents, 0x320, 24, 1), + MUX(CLK_TOP_APLL_I2S9_M_SEL, "apll_i2s9_m_sel", apll_i2s_m_parents, 0x320, 25, 1), }; static const struct mtk_composite top_adj_divs[] = { @@ -1224,6 +1068,28 @@ static void clk_mt8192_top_init_early(struct device_node *node) CLK_OF_DECLARE_DRIVER(mt8192_topckgen, "mediatek,mt8192-topckgen", clk_mt8192_top_init_early); +/* Register mux notifier for MFG mux */ +static int clk_mt8192_reg_mfg_mux_notifier(struct device *dev, struct clk *clk) +{ + struct mtk_mux_nb *mfg_mux_nb; + int i; + + mfg_mux_nb = devm_kzalloc(dev, sizeof(*mfg_mux_nb), GFP_KERNEL); + if (!mfg_mux_nb) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(top_mtk_muxes); i++) + if (top_mtk_muxes[i].id == CLK_TOP_MFG_PLL_SEL) + break; + if (i == ARRAY_SIZE(top_mtk_muxes)) + return -EINVAL; + + mfg_mux_nb->ops = top_mtk_muxes[i].ops; + mfg_mux_nb->bypass_index = 0; /* Bypass to 26M crystal */ + + return devm_mtk_clk_mux_notifier_register(dev, clk, mfg_mux_nb); +} + static int clk_mt8192_top_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; @@ -1247,6 +1113,12 @@ static int clk_mt8192_top_probe(struct platform_device *pdev) if (r) return r; + r = clk_mt8192_reg_mfg_mux_notifier(&pdev->dev, + top_clk_data->hws[CLK_TOP_MFG_PLL_SEL]->clk); + if (r) + return r; + + return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, top_clk_data); } diff --git a/drivers/clk/mediatek/clk-mt8195-infra_ao.c b/drivers/clk/mediatek/clk-mt8195-infra_ao.c index 97657f255618cabec5d502aa3414978a312fe622..fcd410461d3bb4419046dff5a24c8e10da26d7af 100644 --- a/drivers/clk/mediatek/clk-mt8195-infra_ao.c +++ b/drivers/clk/mediatek/clk-mt8195-infra_ao.c @@ -55,8 +55,12 @@ static const struct mtk_gate_regs infra_ao4_cg_regs = { #define GATE_INFRA_AO1(_id, _name, _parent, _shift) \ GATE_INFRA_AO1_FLAGS(_id, _name, _parent, _shift, 0) +#define GATE_INFRA_AO2_FLAGS(_id, _name, _parent, _shift, _flag) \ + GATE_MTK_FLAGS(_id, _name, _parent, &infra_ao2_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr, _flag) + #define GATE_INFRA_AO2(_id, _name, _parent, _shift) \ - GATE_MTK(_id, _name, _parent, &infra_ao2_cg_regs, _shift, &mtk_clk_gate_ops_setclr) + GATE_INFRA_AO2_FLAGS(_id, _name, _parent, _shift, 0) #define GATE_INFRA_AO3_FLAGS(_id, _name, _parent, _shift, _flag) \ GATE_MTK_FLAGS(_id, _name, _parent, &infra_ao3_cg_regs, _shift, \ @@ -136,8 +140,11 @@ static const struct mtk_gate infra_ao_clks[] = { GATE_INFRA_AO2(CLK_INFRA_AO_UNIPRO_SYS, "infra_ao_unipro_sys", "top_ufs", 11), GATE_INFRA_AO2(CLK_INFRA_AO_UNIPRO_TICK, "infra_ao_unipro_tick", "top_ufs_tick1us", 12), GATE_INFRA_AO2(CLK_INFRA_AO_UFS_MP_SAP_B, "infra_ao_ufs_mp_sap_b", "top_ufs_mp_sap_cfg", 13), - GATE_INFRA_AO2(CLK_INFRA_AO_PWRMCU, "infra_ao_pwrmcu", "top_pwrmcu", 15), - GATE_INFRA_AO2(CLK_INFRA_AO_PWRMCU_BUS_H, "infra_ao_pwrmcu_bus_h", "top_axi", 17), + /* pwrmcu is used by ATF for platform PM: clocks must never be disabled by the kernel */ + GATE_INFRA_AO2_FLAGS(CLK_INFRA_AO_PWRMCU, "infra_ao_pwrmcu", "top_pwrmcu", 15, + CLK_IS_CRITICAL), + GATE_INFRA_AO2_FLAGS(CLK_INFRA_AO_PWRMCU_BUS_H, "infra_ao_pwrmcu_bus_h", "top_axi", 17, + CLK_IS_CRITICAL), GATE_INFRA_AO2(CLK_INFRA_AO_APDMA_B, "infra_ao_apdma_b", "top_axi", 18), GATE_INFRA_AO2(CLK_INFRA_AO_SPI4, "infra_ao_spi4", "top_spi", 25), GATE_INFRA_AO2(CLK_INFRA_AO_SPI5, "infra_ao_spi5", "top_spi", 26), @@ -193,6 +200,9 @@ static u16 infra_ao_rst_ofs[] = { static u16 infra_ao_idx_map[] = { [MT8195_INFRA_RST0_THERM_CTRL_SWRST] = 0 * RST_NR_PER_BANK + 0, + [MT8195_INFRA_RST2_USBSIF_P1_SWRST] = 2 * RST_NR_PER_BANK + 18, + [MT8195_INFRA_RST2_PCIE_P0_SWRST] = 2 * RST_NR_PER_BANK + 26, + [MT8195_INFRA_RST2_PCIE_P1_SWRST] = 2 * RST_NR_PER_BANK + 27, [MT8195_INFRA_RST3_THERM_CTRL_PTP_SWRST] = 3 * RST_NR_PER_BANK + 5, [MT8195_INFRA_RST4_THERM_CTRL_MCU_SWRST] = 4 * RST_NR_PER_BANK + 10, }; diff --git a/drivers/clk/mediatek/clk-mt8195-mfg.c b/drivers/clk/mediatek/clk-mt8195-mfg.c index 9411c556a5a97da4cd379ed13df2883a8cf94824..c94cb71bd9b94c163580468ce7843ed9e9627831 100644 --- a/drivers/clk/mediatek/clk-mt8195-mfg.c +++ b/drivers/clk/mediatek/clk-mt8195-mfg.c @@ -17,10 +17,12 @@ static const struct mtk_gate_regs mfg_cg_regs = { }; #define GATE_MFG(_id, _name, _parent, _shift) \ - GATE_MTK(_id, _name, _parent, &mfg_cg_regs, _shift, &mtk_clk_gate_ops_setclr) + GATE_MTK_FLAGS(_id, _name, _parent, &mfg_cg_regs, \ + _shift, &mtk_clk_gate_ops_setclr, \ + CLK_SET_RATE_PARENT) static const struct mtk_gate mfg_clks[] = { - GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "top_mfg_core_tmp", 0), + GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_ck_fast_ref", 0), }; static const struct mtk_clk_desc mfg_desc = { diff --git a/drivers/clk/mediatek/clk-mt8195-topckgen.c b/drivers/clk/mediatek/clk-mt8195-topckgen.c index ec70e1f65eafb43c19fcb6c2b9cc962557f23d86..8cbab5ca2e581dad2a183e6b433736fee8542ee9 100644 --- a/drivers/clk/mediatek/clk-mt8195-topckgen.c +++ b/drivers/clk/mediatek/clk-mt8195-topckgen.c @@ -298,11 +298,14 @@ static const char * const ipu_if_parents[] = { "mmpll_d4" }; +/* + * MFG can be also parented to "univpll_d6" and "univpll_d7": + * these have been removed from the parents list to let us + * achieve GPU DVFS without any special clock handlers. + */ static const char * const mfg_parents[] = { "clk26m", - "mainpll_d5_d2", - "univpll_d6", - "univpll_d7" + "mainpll_d5_d2" }; static const char * const camtg_parents[] = { @@ -1149,11 +1152,6 @@ static const struct mtk_mux top_mtk_muxes[] = { */ }; -static struct mtk_composite top_muxes[] = { - /* CLK_MISC_CFG_3 */ - MUX(CLK_TOP_MFG_CK_FAST_REF, "mfg_ck_fast_ref", mfg_fast_parents, 0x0250, 8, 1), -}; - static const struct mtk_composite top_adj_divs[] = { DIV_GATE(CLK_TOP_APLL12_DIV0, "apll12_div0", "top_i2si1_mck", 0x0320, 0, 0x0328, 8, 0), DIV_GATE(CLK_TOP_APLL12_DIV1, "apll12_div1", "top_i2si2_mck", 0x0320, 1, 0x0328, 8, 8), @@ -1222,10 +1220,26 @@ static const struct of_device_id of_match_clk_mt8195_topck[] = { {} }; +/* Register mux notifier for MFG mux */ +static int clk_mt8195_reg_mfg_mux_notifier(struct device *dev, struct clk *clk) +{ + struct mtk_mux_nb *mfg_mux_nb; + + mfg_mux_nb = devm_kzalloc(dev, sizeof(*mfg_mux_nb), GFP_KERNEL); + if (!mfg_mux_nb) + return -ENOMEM; + + mfg_mux_nb->ops = &clk_mux_ops; + mfg_mux_nb->bypass_index = 0; /* Bypass to TOP_MFG_CORE_TMP */ + + return devm_mtk_clk_mux_notifier_register(dev, clk, mfg_mux_nb); +} + static int clk_mt8195_topck_probe(struct platform_device *pdev) { struct clk_hw_onecell_data *top_clk_data; struct device_node *node = pdev->dev.of_node; + struct clk_hw *hw; int r; void __iomem *base; @@ -1253,15 +1267,22 @@ static int clk_mt8195_topck_probe(struct platform_device *pdev) if (r) goto unregister_factors; - r = mtk_clk_register_composites(top_muxes, ARRAY_SIZE(top_muxes), base, - &mt8195_clk_lock, top_clk_data); + hw = devm_clk_hw_register_mux(&pdev->dev, "mfg_ck_fast_ref", mfg_fast_parents, + ARRAY_SIZE(mfg_fast_parents), CLK_SET_RATE_PARENT, + (base + 0x250), 8, 1, 0, &mt8195_clk_lock); + if (IS_ERR(hw)) + goto unregister_muxes; + top_clk_data->hws[CLK_TOP_MFG_CK_FAST_REF] = hw; + + r = clk_mt8195_reg_mfg_mux_notifier(&pdev->dev, + top_clk_data->hws[CLK_TOP_MFG_CK_FAST_REF]->clk); if (r) goto unregister_muxes; r = mtk_clk_register_composites(top_adj_divs, ARRAY_SIZE(top_adj_divs), base, &mt8195_clk_lock, top_clk_data); if (r) - goto unregister_composite_muxes; + goto unregister_muxes; r = mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks), top_clk_data); if (r) @@ -1279,8 +1300,6 @@ unregister_gates: mtk_clk_unregister_gates(top_clks, ARRAY_SIZE(top_clks), top_clk_data); unregister_composite_divs: mtk_clk_unregister_composites(top_adj_divs, ARRAY_SIZE(top_adj_divs), top_clk_data); -unregister_composite_muxes: - mtk_clk_unregister_composites(top_muxes, ARRAY_SIZE(top_muxes), top_clk_data); unregister_muxes: mtk_clk_unregister_muxes(top_mtk_muxes, ARRAY_SIZE(top_mtk_muxes), top_clk_data); unregister_factors: @@ -1300,7 +1319,6 @@ static int clk_mt8195_topck_remove(struct platform_device *pdev) of_clk_del_provider(node); mtk_clk_unregister_gates(top_clks, ARRAY_SIZE(top_clks), top_clk_data); mtk_clk_unregister_composites(top_adj_divs, ARRAY_SIZE(top_adj_divs), top_clk_data); - mtk_clk_unregister_composites(top_muxes, ARRAY_SIZE(top_muxes), top_clk_data); mtk_clk_unregister_muxes(top_mtk_muxes, ARRAY_SIZE(top_mtk_muxes), top_clk_data); mtk_clk_unregister_factors(top_divs, ARRAY_SIZE(top_divs), top_clk_data); mtk_clk_unregister_fixed_clks(top_fixed_clks, ARRAY_SIZE(top_fixed_clks), top_clk_data); diff --git a/drivers/clk/mediatek/clk-mt8195-vdo0.c b/drivers/clk/mediatek/clk-mt8195-vdo0.c index 261a7f76dd3cc12954e4ca166b41505a0dd0fb74..07b46bfd504068ebce7c0c3a158e0f9997400a77 100644 --- a/drivers/clk/mediatek/clk-mt8195-vdo0.c +++ b/drivers/clk/mediatek/clk-mt8195-vdo0.c @@ -37,6 +37,10 @@ static const struct mtk_gate_regs vdo0_2_cg_regs = { #define GATE_VDO0_2(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &vdo0_2_cg_regs, _shift, &mtk_clk_gate_ops_setclr) +#define GATE_VDO0_2_FLAGS(_id, _name, _parent, _shift, _flags) \ + GATE_MTK_FLAGS(_id, _name, _parent, &vdo0_2_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr, _flags) + static const struct mtk_gate vdo0_clks[] = { /* VDO0_0 */ GATE_VDO0_0(CLK_VDO0_DISP_OVL0, "vdo0_disp_ovl0", "top_vpp", 0), @@ -85,7 +89,8 @@ static const struct mtk_gate vdo0_clks[] = { /* VDO0_2 */ GATE_VDO0_2(CLK_VDO0_DSI0_DSI, "vdo0_dsi0_dsi", "top_dsi_occ", 0), GATE_VDO0_2(CLK_VDO0_DSI1_DSI, "vdo0_dsi1_dsi", "top_dsi_occ", 8), - GATE_VDO0_2(CLK_VDO0_DP_INTF0_DP_INTF, "vdo0_dp_intf0_dp_intf", "top_edp", 16), + GATE_VDO0_2_FLAGS(CLK_VDO0_DP_INTF0_DP_INTF, "vdo0_dp_intf0_dp_intf", + "top_edp", 16, CLK_SET_RATE_PARENT), }; static int clk_mt8195_vdo0_probe(struct platform_device *pdev) diff --git a/drivers/clk/mediatek/clk-mt8195-vdo1.c b/drivers/clk/mediatek/clk-mt8195-vdo1.c index 3378487d2c904c3454fa670aed46bab6a14dd1ae..835335b9d87bba06a19dcd8c5509fd7dd9535eea 100644 --- a/drivers/clk/mediatek/clk-mt8195-vdo1.c +++ b/drivers/clk/mediatek/clk-mt8195-vdo1.c @@ -34,6 +34,12 @@ static const struct mtk_gate_regs vdo1_3_cg_regs = { .sta_ofs = 0x140, }; +static const struct mtk_gate_regs vdo1_4_cg_regs = { + .set_ofs = 0x400, + .clr_ofs = 0x400, + .sta_ofs = 0x400, +}; + #define GATE_VDO1_0(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &vdo1_0_cg_regs, _shift, &mtk_clk_gate_ops_setclr) @@ -43,9 +49,16 @@ static const struct mtk_gate_regs vdo1_3_cg_regs = { #define GATE_VDO1_2(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &vdo1_2_cg_regs, _shift, &mtk_clk_gate_ops_setclr) +#define GATE_VDO1_2_FLAGS(_id, _name, _parent, _shift, _flags) \ + GATE_MTK_FLAGS(_id, _name, _parent, &vdo1_2_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr, _flags) + #define GATE_VDO1_3(_id, _name, _parent, _shift) \ GATE_MTK(_id, _name, _parent, &vdo1_3_cg_regs, _shift, &mtk_clk_gate_ops_setclr) +#define GATE_VDO1_4(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &vdo1_4_cg_regs, _shift, &mtk_clk_gate_ops_no_setclr_inv) + static const struct mtk_gate vdo1_clks[] = { /* VDO1_0 */ GATE_VDO1_0(CLK_VDO1_SMI_LARB2, "vdo1_smi_larb2", "top_vpp", 0), @@ -99,10 +112,12 @@ static const struct mtk_gate vdo1_clks[] = { GATE_VDO1_2(CLK_VDO1_DISP_MONITOR_DPI0, "vdo1_disp_monitor_dpi0", "top_vpp", 1), GATE_VDO1_2(CLK_VDO1_DPI1, "vdo1_dpi1", "top_vpp", 8), GATE_VDO1_2(CLK_VDO1_DISP_MONITOR_DPI1, "vdo1_disp_monitor_dpi1", "top_vpp", 9), - GATE_VDO1_2(CLK_VDO1_DPINTF, "vdo1_dpintf", "top_vpp", 16), + GATE_VDO1_2_FLAGS(CLK_VDO1_DPINTF, "vdo1_dpintf", "top_dp", 16, CLK_SET_RATE_PARENT), GATE_VDO1_2(CLK_VDO1_DISP_MONITOR_DPINTF, "vdo1_disp_monitor_dpintf", "top_vpp", 17), /* VDO1_3 */ GATE_VDO1_3(CLK_VDO1_26M_SLOW, "vdo1_26m_slow", "clk26m", 8), + /* VDO1_4 */ + GATE_VDO1_4(CLK_VDO1_DPI1_HDMI, "vdo1_dpi1_hdmi", "hdmi_txpll", 0), }; static int clk_mt8195_vdo1_probe(struct platform_device *pdev) diff --git a/drivers/clk/mediatek/clk-mt8365-apu.c b/drivers/clk/mediatek/clk-mt8365-apu.c new file mode 100644 index 0000000000000000000000000000000000000000..91ffe89d9721f6b17e34836314224a11183bec29 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-apu.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs apu_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_APU(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &apu_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr) + +static const struct mtk_gate apu_clks[] = { + GATE_APU(CLK_APU_AHB, "apu_ahb", "ifr_apu_axi", 5), + GATE_APU(CLK_APU_EDMA, "apu_edma", "apu_sel", 4), + GATE_APU(CLK_APU_IF_CK, "apu_if_ck", "apu_if_sel", 3), + GATE_APU(CLK_APU_JTAG, "apu_jtag", "clk26m", 2), + GATE_APU(CLK_APU_AXI, "apu_axi", "apu_sel", 1), + GATE_APU(CLK_APU_IPU_CK, "apu_ck", "apu_sel", 0), +}; + +static const struct mtk_clk_desc apu_desc = { + .clks = apu_clks, + .num_clks = ARRAY_SIZE(apu_clks), +}; + +static const struct of_device_id of_match_clk_mt8365_apu[] = { + { + .compatible = "mediatek,mt8365-apu", + .data = &apu_desc, + }, { + /* sentinel */ + } +}; + +static struct platform_driver clk_mt8365_apu_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt8365-apu", + .of_match_table = of_match_clk_mt8365_apu, + }, +}; +builtin_platform_driver(clk_mt8365_apu_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-cam.c b/drivers/clk/mediatek/clk-mt8365-cam.c new file mode 100644 index 0000000000000000000000000000000000000000..31d5b5cd6de1b174b0f5c3a67f6c7e0204bd7c31 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-cam.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs cam_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_CAM(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &cam_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr) + +static const struct mtk_gate cam_clks[] = { + GATE_CAM(CLK_CAM_LARB2, "cam_larb2", "mm_sel", 0), + GATE_CAM(CLK_CAM, "cam", "mm_sel", 6), + GATE_CAM(CLK_CAMTG, "camtg", "mm_sel", 7), + GATE_CAM(CLK_CAM_SENIF, "cam_senif", "mm_sel", 8), + GATE_CAM(CLK_CAMSV0, "camsv0", "mm_sel", 9), + GATE_CAM(CLK_CAMSV1, "camsv1", "mm_sel", 10), + GATE_CAM(CLK_CAM_FDVT, "cam_fdvt", "mm_sel", 11), + GATE_CAM(CLK_CAM_WPE, "cam_wpe", "mm_sel", 12), +}; + +static const struct mtk_clk_desc cam_desc = { + .clks = cam_clks, + .num_clks = ARRAY_SIZE(cam_clks), +}; + +static const struct of_device_id of_match_clk_mt8365_cam[] = { + { + .compatible = "mediatek,mt8365-imgsys", + .data = &cam_desc, + }, { + /* sentinel */ + } +}; + +static struct platform_driver clk_mt8365_cam_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt8365-cam", + .of_match_table = of_match_clk_mt8365_cam, + }, +}; +builtin_platform_driver(clk_mt8365_cam_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-mfg.c b/drivers/clk/mediatek/clk-mt8365-mfg.c new file mode 100644 index 0000000000000000000000000000000000000000..587b49128b035a40be2dead0ad0073bbc6919c34 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-mfg.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs mfg0_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +static const struct mtk_gate_regs mfg1_cg_regs = { + .set_ofs = 0x280, + .clr_ofs = 0x280, + .sta_ofs = 0x280, +}; + +#define GATE_MFG0(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mfg0_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr) + +#define GATE_MFG1(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mfg1_cg_regs, _shift, \ + &mtk_clk_gate_ops_no_setclr) + +static const struct mtk_gate mfg_clks[] = { + /* MFG0 */ + GATE_MFG0(CLK_MFG_BG3D, "mfg_bg3d", "mfg_sel", 0), + /* MFG1 */ + GATE_MFG1(CLK_MFG_MBIST_DIAG, "mfg_mbist_diag", "mbist_diag_sel", 24), +}; + +static const struct mtk_clk_desc mfg_desc = { + .clks = mfg_clks, + .num_clks = ARRAY_SIZE(mfg_clks), +}; + +static const struct of_device_id of_match_clk_mt8365_mfg[] = { + { + .compatible = "mediatek,mt8365-mfgcfg", + .data = &mfg_desc, + }, { + /* sentinel */ + } +}; + +static struct platform_driver clk_mt8365_mfg_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt8365-mfg", + .of_match_table = of_match_clk_mt8365_mfg, + }, +}; +builtin_platform_driver(clk_mt8365_mfg_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-mm.c b/drivers/clk/mediatek/clk-mt8365-mm.c new file mode 100644 index 0000000000000000000000000000000000000000..5c8bf18ab1f1d0872bebb35009fa8e380919c731 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-mm.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + * Copyright (c) 2022 BayLibre, SAS + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs mm0_cg_regs = { + .set_ofs = 0x104, + .clr_ofs = 0x108, + .sta_ofs = 0x100, +}; + +static const struct mtk_gate_regs mm1_cg_regs = { + .set_ofs = 0x114, + .clr_ofs = 0x118, + .sta_ofs = 0x110, +}; + +#define GATE_MM0(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mm0_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr) + +#define GATE_MM1(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &mm1_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr) + +static const struct mtk_gate mm_clks[] = { + /* MM0 */ + GATE_MM0(CLK_MM_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_sel", 0), + GATE_MM0(CLK_MM_MM_MDP_CCORR0, "mm_mdp_ccorr0", "mm_sel", 1), + GATE_MM0(CLK_MM_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 2), + GATE_MM0(CLK_MM_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 3), + GATE_MM0(CLK_MM_MM_MDP_TDSHP0, "mm_mdp_tdshp0", "mm_sel", 4), + GATE_MM0(CLK_MM_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_sel", 5), + GATE_MM0(CLK_MM_MM_MDP_WDMA0, "mm_mdp_wdma0", "mm_sel", 6), + GATE_MM0(CLK_MM_MM_DISP_OVL0, "mm_disp_ovl0", "mm_sel", 7), + GATE_MM0(CLK_MM_MM_DISP_OVL0_2L, "mm_disp_ovl0_2l", "mm_sel", 8), + GATE_MM0(CLK_MM_MM_DISP_RSZ0, "mm_disp_rsz0", "mm_sel", 9), + GATE_MM0(CLK_MM_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_sel", 10), + GATE_MM0(CLK_MM_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_sel", 11), + GATE_MM0(CLK_MM_MM_DISP_COLOR0, "mm_disp_color0", "mm_sel", 12), + GATE_MM0(CLK_MM_MM_DISP_CCORR0, "mm_disp_ccorr0", "mm_sel", 13), + GATE_MM0(CLK_MM_MM_DISP_AAL0, "mm_disp_aal0", "mm_sel", 14), + GATE_MM0(CLK_MM_MM_DISP_GAMMA0, "mm_disp_gamma0", "mm_sel", 15), + GATE_MM0(CLK_MM_MM_DISP_DITHER0, "mm_disp_dither0", "mm_sel", 16), + GATE_MM0(CLK_MM_MM_DSI0, "mm_dsi0", "mm_sel", 17), + GATE_MM0(CLK_MM_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 18), + GATE_MM0(CLK_MM_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 19), + GATE_MM0(CLK_MM_DPI0_DPI0, "mm_dpi0_dpi0", "vpll_dpix", 20), + GATE_MM0(CLK_MM_MM_FAKE, "mm_fake", "mm_sel", 21), + GATE_MM0(CLK_MM_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 22), + GATE_MM0(CLK_MM_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 23), + GATE_MM0(CLK_MM_MM_SMI_COMM0, "mm_smi_comm0", "mm_sel", 24), + GATE_MM0(CLK_MM_MM_SMI_COMM1, "mm_smi_comm1", "mm_sel", 25), + GATE_MM0(CLK_MM_MM_CAM_MDP, "mm_cam_mdp", "mm_sel", 26), + GATE_MM0(CLK_MM_MM_SMI_IMG, "mm_smi_img", "mm_sel", 27), + GATE_MM0(CLK_MM_MM_SMI_CAM, "mm_smi_cam", "mm_sel", 28), + GATE_MM0(CLK_MM_IMG_IMG_DL_RELAY, "mm_dl_relay", "mm_sel", 29), + GATE_MM0(CLK_MM_IMG_IMG_DL_ASYNC_TOP, "mm_dl_async_top", "mm_sel", 30), + GATE_MM0(CLK_MM_DSI0_DIG_DSI, "mm_dsi0_dig_dsi", "dsi0_lntc_dsick", 31), + /* MM1 */ + GATE_MM1(CLK_MM_26M_HRTWT, "mm_f26m_hrtwt", "clk26m", 0), + GATE_MM1(CLK_MM_MM_DPI0, "mm_dpi0", "mm_sel", 1), + GATE_MM1(CLK_MM_LVDSTX_PXL, "mm_flvdstx_pxl", "vpll_dpix", 2), + GATE_MM1(CLK_MM_LVDSTX_CTS, "mm_flvdstx_cts", "lvdstx_dig_cts", 3), +}; + +static int clk_mt8365_mm_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->parent->of_node; + struct clk_hw_onecell_data *clk_data; + int ret; + + clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK); + + ret = mtk_clk_register_gates_with_dev(node, mm_clks, + ARRAY_SIZE(mm_clks), clk_data, + dev); + if (ret) + goto err_free_clk_data; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto err_unregister_gates; + + return 0; + +err_unregister_gates: + mtk_clk_unregister_gates(mm_clks, ARRAY_SIZE(mm_clks), clk_data); + +err_free_clk_data: + mtk_free_clk_data(clk_data); + + return ret; +} + +static struct platform_driver clk_mt8365_mm_drv = { + .probe = clk_mt8365_mm_probe, + .driver = { + .name = "clk-mt8365-mm", + }, +}; +builtin_platform_driver(clk_mt8365_mm_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-vdec.c b/drivers/clk/mediatek/clk-mt8365-vdec.c new file mode 100644 index 0000000000000000000000000000000000000000..cdc678e8941ca654eb6929b71440843570435b04 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-vdec.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs vdec0_cg_regs = { + .set_ofs = 0x0, + .clr_ofs = 0x4, + .sta_ofs = 0x0, +}; + +static const struct mtk_gate_regs vdec1_cg_regs = { + .set_ofs = 0x8, + .clr_ofs = 0xc, + .sta_ofs = 0x8, +}; + +#define GATE_VDEC0(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &vdec0_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr_inv) + +#define GATE_VDEC1(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &vdec1_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr_inv) + +static const struct mtk_gate vdec_clks[] = { + /* VDEC0 */ + GATE_VDEC0(CLK_VDEC_VDEC, "vdec_fvdec_ck", "mm_sel", 0), + /* VDEC1 */ + GATE_VDEC1(CLK_VDEC_LARB1, "vdec_flarb1_ck", "mm_sel", 0), +}; + +static const struct mtk_clk_desc vdec_desc = { + .clks = vdec_clks, + .num_clks = ARRAY_SIZE(vdec_clks), +}; + +static const struct of_device_id of_match_clk_mt8365_vdec[] = { + { + .compatible = "mediatek,mt8365-vdecsys", + .data = &vdec_desc, + }, { + /* sentinel */ + } +}; + +static struct platform_driver clk_mt8365_vdec_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt8365-vdec", + .of_match_table = of_match_clk_mt8365_vdec, + }, +}; +builtin_platform_driver(clk_mt8365_vdec_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365-venc.c b/drivers/clk/mediatek/clk-mt8365-venc.c new file mode 100644 index 0000000000000000000000000000000000000000..0e080c22119dc63b2fb54630d7772c5ff127f805 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365-venc.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" + +static const struct mtk_gate_regs venc_cg_regs = { + .set_ofs = 0x4, + .clr_ofs = 0x8, + .sta_ofs = 0x0, +}; + +#define GATE_VENC(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &venc_cg_regs, _shift, \ + &mtk_clk_gate_ops_setclr_inv) + +static const struct mtk_gate venc_clks[] = { + /* VENC */ + GATE_VENC(CLK_VENC, "venc_fvenc_ck", "mm_sel", 4), + GATE_VENC(CLK_VENC_JPGENC, "venc_jpgenc_ck", "mm_sel", 8), +}; + +static const struct mtk_clk_desc venc_desc = { + .clks = venc_clks, + .num_clks = ARRAY_SIZE(venc_clks), +}; + +static const struct of_device_id of_match_clk_mt8365_venc[] = { + { + .compatible = "mediatek,mt8365-vencsys", + .data = &venc_desc, + }, { + /* sentinel */ + } +}; + +static struct platform_driver clk_mt8365_venc_drv = { + .probe = mtk_clk_simple_probe, + .remove = mtk_clk_simple_remove, + .driver = { + .name = "clk-mt8365-venc", + .of_match_table = of_match_clk_mt8365_venc, + }, +}; +builtin_platform_driver(clk_mt8365_venc_drv); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mt8365.c b/drivers/clk/mediatek/clk-mt8365.c new file mode 100644 index 0000000000000000000000000000000000000000..adfecb618f10255cb12483bf36047bf113392928 --- /dev/null +++ b/drivers/clk/mediatek/clk-mt8365.c @@ -0,0 +1,1155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-gate.h" +#include "clk-mtk.h" +#include "clk-mux.h" +#include "clk-pll.h" + +static DEFINE_SPINLOCK(mt8365_clk_lock); + +static const struct mtk_fixed_clk top_fixed_clks[] = { + FIXED_CLK(CLK_TOP_I2S0_BCK, "i2s0_bck", NULL, 26000000), + FIXED_CLK(CLK_TOP_DSI0_LNTC_DSICK, "dsi0_lntc_dsick", "clk26m", + 75000000), + FIXED_CLK(CLK_TOP_VPLL_DPIX, "vpll_dpix", "clk26m", 75000000), + FIXED_CLK(CLK_TOP_LVDSTX_CLKDIG_CTS, "lvdstx_dig_cts", "clk26m", + 52500000), +}; + +static const struct mtk_fixed_factor top_divs[] = { + FACTOR(CLK_TOP_SYS_26M_D2, "sys_26m_d2", "clk26m", 1, 2), + FACTOR(CLK_TOP_SYSPLL_D2, "syspll_d2", "mainpll", 1, 2), + FACTOR(CLK_TOP_SYSPLL1_D2, "syspll1_d2", "mainpll", 1, 4), + FACTOR(CLK_TOP_SYSPLL1_D4, "syspll1_d4", "mainpll", 1, 8), + FACTOR(CLK_TOP_SYSPLL1_D8, "syspll1_d8", "mainpll", 1, 16), + FACTOR(CLK_TOP_SYSPLL1_D16, "syspll1_d16", "mainpll", 1, 32), + FACTOR(CLK_TOP_SYSPLL_D3, "syspll_d3", "mainpll", 1, 3), + FACTOR(CLK_TOP_SYSPLL2_D2, "syspll2_d2", "mainpll", 1, 6), + FACTOR(CLK_TOP_SYSPLL2_D4, "syspll2_d4", "mainpll", 1, 12), + FACTOR(CLK_TOP_SYSPLL2_D8, "syspll2_d8", "mainpll", 1, 24), + FACTOR(CLK_TOP_SYSPLL_D5, "syspll_d5", "mainpll", 1, 5), + FACTOR(CLK_TOP_SYSPLL3_D2, "syspll3_d2", "mainpll", 1, 10), + FACTOR(CLK_TOP_SYSPLL3_D4, "syspll3_d4", "mainpll", 1, 20), + FACTOR(CLK_TOP_SYSPLL_D7, "syspll_d7", "mainpll", 1, 7), + FACTOR(CLK_TOP_SYSPLL4_D2, "syspll4_d2", "mainpll", 1, 14), + FACTOR(CLK_TOP_SYSPLL4_D4, "syspll4_d4", "mainpll", 1, 28), + FACTOR(CLK_TOP_UNIVPLL, "univpll", "univ_en", 1, 2), + FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll", 1, 2), + FACTOR(CLK_TOP_UNIVPLL1_D2, "univpll1_d2", "univpll", 1, 4), + FACTOR(CLK_TOP_UNIVPLL1_D4, "univpll1_d4", "univpll", 1, 8), + FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1, 3), + FACTOR(CLK_TOP_UNIVPLL2_D2, "univpll2_d2", "univpll", 1, 6), + FACTOR(CLK_TOP_UNIVPLL2_D4, "univpll2_d4", "univpll", 1, 12), + FACTOR(CLK_TOP_UNIVPLL2_D8, "univpll2_d8", "univpll", 1, 24), + FACTOR(CLK_TOP_UNIVPLL2_D32, "univpll2_d32", "univpll", 1, 96), + FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1, 5), + FACTOR(CLK_TOP_UNIVPLL3_D2, "univpll3_d2", "univpll", 1, 10), + FACTOR(CLK_TOP_UNIVPLL3_D4, "univpll3_d4", "univpll", 1, 20), + FACTOR(CLK_TOP_MMPLL, "mmpll_ck", "mmpll", 1, 1), + FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll", 1, 2), + FACTOR(CLK_TOP_MFGPLL, "mfgpll_ck", "mfgpll", 1, 1), + FACTOR(CLK_TOP_LVDSPLL_D2, "lvdspll_d2", "lvdspll", 1, 2), + FACTOR(CLK_TOP_LVDSPLL_D4, "lvdspll_d4", "lvdspll", 1, 4), + FACTOR(CLK_TOP_LVDSPLL_D8, "lvdspll_d8", "lvdspll", 1, 8), + FACTOR(CLK_TOP_LVDSPLL_D16, "lvdspll_d16", "lvdspll", 1, 16), + FACTOR(CLK_TOP_USB20_192M, "usb20_192m_ck", "usb20_en", 1, 13), + FACTOR(CLK_TOP_USB20_192M_D4, "usb20_192m_d4", "usb20_192m_ck", 1, 4), + FACTOR(CLK_TOP_USB20_192M_D8, "usb20_192m_d8", "usb20_192m_ck", 1, 8), + FACTOR(CLK_TOP_USB20_192M_D16, "usb20_192m_d16", "usb20_192m_ck", + 1, 16), + FACTOR(CLK_TOP_USB20_192M_D32, "usb20_192m_d32", "usb20_192m_ck", + 1, 32), + FACTOR(CLK_TOP_APLL1, "apll1_ck", "apll1", 1, 1), + FACTOR(CLK_TOP_APLL1_D2, "apll1_d2", "apll1_ck", 1, 2), + FACTOR(CLK_TOP_APLL1_D4, "apll1_d4", "apll1_ck", 1, 4), + FACTOR(CLK_TOP_APLL1_D8, "apll1_d8", "apll1_ck", 1, 8), + FACTOR(CLK_TOP_APLL2, "apll2_ck", "apll2", 1, 1), + FACTOR(CLK_TOP_APLL2_D2, "apll2_d2", "apll2_ck", 1, 2), + FACTOR(CLK_TOP_APLL2_D4, "apll2_d4", "apll2_ck", 1, 4), + FACTOR(CLK_TOP_APLL2_D8, "apll2_d8", "apll2_ck", 1, 8), + FACTOR(CLK_TOP_MSDCPLL, "msdcpll_ck", "msdcpll", 1, 1), + FACTOR(CLK_TOP_MSDCPLL_D2, "msdcpll_d2", "msdcpll", 1, 2), + FACTOR(CLK_TOP_DSPPLL, "dsppll_ck", "dsppll", 1, 1), + FACTOR(CLK_TOP_DSPPLL_D2, "dsppll_d2", "dsppll", 1, 2), + FACTOR(CLK_TOP_DSPPLL_D4, "dsppll_d4", "dsppll", 1, 4), + FACTOR(CLK_TOP_DSPPLL_D8, "dsppll_d8", "dsppll", 1, 8), + FACTOR(CLK_TOP_APUPLL, "apupll_ck", "apupll", 1, 1), + FACTOR(CLK_TOP_CLK26M_D52, "clk26m_d52", "clk26m", 1, 52), +}; + +static const char * const axi_parents[] = { + "clk26m", + "syspll_d7", + "syspll1_d4", + "syspll3_d2" +}; + +static const char * const mem_parents[] = { + "clk26m", + "mmpll_ck", + "syspll_d3", + "syspll1_d2" +}; + +static const char * const mm_parents[] = { + "clk26m", + "mmpll_ck", + "syspll1_d2", + "syspll_d5", + "syspll1_d4", + "univpll_d5", + "univpll1_d2", + "mmpll_d2" +}; + +static const char * const scp_parents[] = { + "clk26m", + "syspll4_d2", + "univpll2_d2", + "syspll1_d2", + "univpll1_d2", + "syspll_d3", + "univpll_d3" +}; + +static const char * const mfg_parents[] = { + "clk26m", + "mfgpll_ck", + "syspll_d3", + "univpll_d3" +}; + +static const char * const atb_parents[] = { + "clk26m", + "syspll1_d4", + "syspll1_d2" +}; + +static const char * const camtg_parents[] = { + "clk26m", + "usb20_192m_d8", + "univpll2_d8", + "usb20_192m_d4", + "univpll2_d32", + "usb20_192m_d16", + "usb20_192m_d32" +}; + +static const char * const uart_parents[] = { + "clk26m", + "univpll2_d8" +}; + +static const char * const spi_parents[] = { + "clk26m", + "univpll2_d2", + "univpll2_d4", + "univpll2_d8" +}; + +static const char * const msdc50_0_hc_parents[] = { + "clk26m", + "syspll1_d2", + "univpll1_d4", + "syspll2_d2" +}; + +static const char * const msdc50_0_parents[] = { + "clk26m", + "msdcpll_ck", + "univpll1_d2", + "syspll1_d2", + "univpll_d5", + "syspll2_d2", + "univpll1_d4", + "syspll4_d2" +}; + +static const char * const msdc50_2_parents[] = { + "clk26m", + "msdcpll_ck", + "univpll_d3", + "univpll1_d2", + "syspll1_d2", + "univpll2_d2", + "syspll2_d2", + "univpll1_d4" +}; + +static const char * const msdc30_1_parents[] = { + "clk26m", + "msdcpll_d2", + "univpll2_d2", + "syspll2_d2", + "univpll1_d4", + "syspll1_d4", + "syspll2_d4", + "univpll2_d8" +}; + +static const char * const audio_parents[] = { + "clk26m", + "syspll3_d4", + "syspll4_d4", + "syspll1_d16" +}; + +static const char * const aud_intbus_parents[] = { + "clk26m", + "syspll1_d4", + "syspll4_d2" +}; + +static const char * const aud_1_parents[] = { + "clk26m", + "apll1_ck" +}; + +static const char * const aud_2_parents[] = { + "clk26m", + "apll2_ck" +}; + +static const char * const aud_engen1_parents[] = { + "clk26m", + "apll1_d2", + "apll1_d4", + "apll1_d8" +}; + +static const char * const aud_engen2_parents[] = { + "clk26m", + "apll2_d2", + "apll2_d4", + "apll2_d8" +}; + +static const char * const aud_spdif_parents[] = { + "clk26m", + "univpll_d2" +}; + +static const char * const disp_pwm_parents[] = { + "clk26m", + "univpll2_d4" +}; + +static const char * const dxcc_parents[] = { + "clk26m", + "syspll1_d2", + "syspll1_d4", + "syspll1_d8" +}; + +static const char * const ssusb_sys_parents[] = { + "clk26m", + "univpll3_d4", + "univpll2_d4", + "univpll3_d2" +}; + +static const char * const spm_parents[] = { + "clk26m", + "syspll1_d8" +}; + +static const char * const i2c_parents[] = { + "clk26m", + "univpll3_d4", + "univpll3_d2", + "syspll1_d8", + "syspll2_d8" +}; + +static const char * const pwm_parents[] = { + "clk26m", + "univpll3_d4", + "syspll1_d8" +}; + +static const char * const senif_parents[] = { + "clk26m", + "univpll1_d4", + "univpll1_d2", + "univpll2_d2" +}; + +static const char * const aes_fde_parents[] = { + "clk26m", + "msdcpll_ck", + "univpll_d3", + "univpll2_d2", + "univpll1_d2", + "syspll1_d2" +}; + +static const char * const dpi0_parents[] = { + "clk26m", + "lvdspll_d2", + "lvdspll_d4", + "lvdspll_d8", + "lvdspll_d16" +}; + +static const char * const dsp_parents[] = { + "clk26m", + "sys_26m_d2", + "dsppll_ck", + "dsppll_d2", + "dsppll_d4", + "dsppll_d8" +}; + +static const char * const nfi2x_parents[] = { + "clk26m", + "syspll2_d2", + "syspll_d7", + "syspll_d3", + "syspll2_d4", + "msdcpll_d2", + "univpll1_d2", + "univpll_d5" +}; + +static const char * const nfiecc_parents[] = { + "clk26m", + "syspll4_d2", + "univpll2_d4", + "syspll_d7", + "univpll1_d2", + "syspll1_d2", + "univpll2_d2", + "syspll_d5" +}; + +static const char * const ecc_parents[] = { + "clk26m", + "univpll2_d2", + "univpll1_d2", + "univpll_d3", + "syspll_d2" +}; + +static const char * const eth_parents[] = { + "clk26m", + "univpll2_d8", + "syspll4_d4", + "syspll1_d8", + "syspll4_d2" +}; + +static const char * const gcpu_parents[] = { + "clk26m", + "univpll_d3", + "univpll2_d2", + "syspll_d3", + "syspll2_d2" +}; + +static const char * const gcpu_cpm_parents[] = { + "clk26m", + "univpll2_d2", + "syspll2_d2" +}; + +static const char * const apu_parents[] = { + "clk26m", + "univpll_d2", + "apupll_ck", + "mmpll_ck", + "syspll_d3", + "univpll1_d2", + "syspll1_d2", + "syspll1_d4" +}; + +static const char * const mbist_diag_parents[] = { + "clk26m", + "syspll4_d4", + "univpll2_d8" +}; + +static const char * const apll_i2s0_parents[] = { + "aud_1_sel", + "aud_2_sel" +}; + +static struct mtk_composite top_misc_mux_gates[] = { + /* CLK_CFG_11 */ + MUX_GATE(CLK_TOP_MBIST_DIAG_SEL, "mbist_diag_sel", mbist_diag_parents, + 0x0ec, 0, 2, 7), +}; + +struct mt8365_clk_audio_mux { + int id; + const char *name; + u8 shift; +}; + +static struct mt8365_clk_audio_mux top_misc_muxes[] = { + { CLK_TOP_APLL_I2S0_SEL, "apll_i2s0_sel", 11}, + { CLK_TOP_APLL_I2S1_SEL, "apll_i2s1_sel", 12}, + { CLK_TOP_APLL_I2S2_SEL, "apll_i2s2_sel", 13}, + { CLK_TOP_APLL_I2S3_SEL, "apll_i2s3_sel", 14}, + { CLK_TOP_APLL_TDMOUT_SEL, "apll_tdmout_sel", 15}, + { CLK_TOP_APLL_TDMIN_SEL, "apll_tdmin_sel", 16}, + { CLK_TOP_APLL_SPDIF_SEL, "apll_spdif_sel", 17}, +}; + +#define CLK_CFG_UPDATE 0x004 +#define CLK_CFG_UPDATE1 0x008 + +static const struct mtk_mux top_muxes[] = { + /* CLK_CFG_0 */ + MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_AXI_SEL, "axi_sel", axi_parents, + 0x040, 0x044, 0x048, 0, 2, 7, CLK_CFG_UPDATE, + 0, CLK_IS_CRITICAL), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MEM_SEL, "mem_sel", mem_parents, 0x040, + 0x044, 0x048, 8, 2, 15, CLK_CFG_UPDATE, 1), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MM_SEL, "mm_sel", mm_parents, 0x040, 0x044, + 0x048, 16, 3, 23, CLK_CFG_UPDATE, 2), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SCP_SEL, "scp_sel", scp_parents, 0x040, + 0x044, 0x048, 24, 3, 31, CLK_CFG_UPDATE, 3), + /* CLK_CFG_1 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_MFG_SEL, "mfg_sel", mfg_parents, 0x050, + 0x054, 0x058, 0, 2, 7, CLK_CFG_UPDATE, 4), + MUX_GATE_CLR_SET_UPD(CLK_TOP_ATB_SEL, "atb_sel", atb_parents, 0x050, + 0x054, 0x058, 8, 2, 15, CLK_CFG_UPDATE, 5), + MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG_SEL, "camtg_sel", camtg_parents, + 0x050, 0x054, 0x058, 16, 3, 23, CLK_CFG_UPDATE, 6), + MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG1_SEL, "camtg1_sel", camtg_parents, + 0x050, 0x054, 0x058, 24, 3, 31, CLK_CFG_UPDATE, 7), + /* CLK_CFG_2 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_UART_SEL, "uart_sel", uart_parents, 0x060, + 0x064, 0x068, 0, 1, 7, CLK_CFG_UPDATE, 8), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SPI_SEL, "spi_sel", spi_parents, 0x060, + 0x064, 0x068, 8, 2, 15, CLK_CFG_UPDATE, 9), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_0_HC_SEL, "msdc50_0_hc_sel", + msdc50_0_hc_parents, 0x060, 0x064, 0x068, 16, 2, + 23, CLK_CFG_UPDATE, 10), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC2_2_HC_SEL, "msdc2_2_hc_sel", + msdc50_0_hc_parents, 0x060, 0x064, 0x068, 24, 2, + 31, CLK_CFG_UPDATE, 11), + /* CLK_CFG_3 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_0_SEL, "msdc50_0_sel", + msdc50_0_parents, 0x070, 0x074, 0x078, 0, 3, 7, + CLK_CFG_UPDATE, 12), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC50_2_SEL, "msdc50_2_sel", + msdc50_2_parents, 0x070, 0x074, 0x078, 8, 3, 15, + CLK_CFG_UPDATE, 13), + MUX_GATE_CLR_SET_UPD(CLK_TOP_MSDC30_1_SEL, "msdc30_1_sel", + msdc30_1_parents, 0x070, 0x074, 0x078, 16, 3, 23, + CLK_CFG_UPDATE, 14), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUDIO_SEL, "audio_sel", audio_parents, + 0x070, 0x074, 0x078, 24, 2, 31, CLK_CFG_UPDATE, + 15), + /* CLK_CFG_4 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_INTBUS_SEL, "aud_intbus_sel", + aud_intbus_parents, 0x080, 0x084, 0x088, 0, 2, 7, + CLK_CFG_UPDATE, 16), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_1_SEL, "aud_1_sel", aud_1_parents, + 0x080, 0x084, 0x088, 8, 1, 15, CLK_CFG_UPDATE, 17), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_2_SEL, "aud_2_sel", aud_2_parents, + 0x080, 0x084, 0x088, 16, 1, 23, CLK_CFG_UPDATE, + 18), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_ENGEN1_SEL, "aud_engen1_sel", + aud_engen1_parents, 0x080, 0x084, 0x088, 24, 2, 31, + CLK_CFG_UPDATE, 19), + /* CLK_CFG_5 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_ENGEN2_SEL, "aud_engen2_sel", + aud_engen2_parents, 0x090, 0x094, 0x098, 0, 2, 7, + CLK_CFG_UPDATE, 20), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AUD_SPDIF_SEL, "aud_spdif_sel", + aud_spdif_parents, 0x090, 0x094, 0x098, 8, 1, 15, + CLK_CFG_UPDATE, 21), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DISP_PWM_SEL, "disp_pwm_sel", + disp_pwm_parents, 0x090, 0x094, 0x098, 16, 2, 23, + CLK_CFG_UPDATE, 22), + /* CLK_CFG_6 */ + MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_DXCC_SEL, "dxcc_sel", dxcc_parents, + 0x0a0, 0x0a4, 0x0a8, 0, 2, 7, CLK_CFG_UPDATE, + 24, CLK_IS_CRITICAL), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SSUSB_SYS_SEL, "ssusb_sys_sel", + ssusb_sys_parents, 0x0a0, 0x0a4, 0x0a8, 8, 2, 15, + CLK_CFG_UPDATE, 25), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SSUSB_XHCI_SEL, "ssusb_xhci_sel", + ssusb_sys_parents, 0x0a0, 0x0a4, 0x0a8, 16, 2, 23, + CLK_CFG_UPDATE, 26), + MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_SPM_SEL, "spm_sel", spm_parents, + 0x0a0, 0x0a4, 0x0a8, 24, 1, 31, + CLK_CFG_UPDATE, 27, CLK_IS_CRITICAL), + /* CLK_CFG_7 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_I2C_SEL, "i2c_sel", i2c_parents, 0x0b0, + 0x0b4, 0x0b8, 0, 3, 7, CLK_CFG_UPDATE, 28), + MUX_GATE_CLR_SET_UPD(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents, 0x0b0, + 0x0b4, 0x0b8, 8, 2, 15, CLK_CFG_UPDATE, 29), + MUX_GATE_CLR_SET_UPD(CLK_TOP_SENIF_SEL, "senif_sel", senif_parents, + 0x0b0, 0x0b4, 0x0b8, 16, 2, 23, CLK_CFG_UPDATE, + 30), + MUX_GATE_CLR_SET_UPD(CLK_TOP_AES_FDE_SEL, "aes_fde_sel", + aes_fde_parents, 0x0b0, 0x0b4, 0x0b8, 24, 3, 31, + CLK_CFG_UPDATE, 31), + /* CLK_CFG_8 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTM_SEL, "camtm_sel", senif_parents, + 0x0c0, 0x0c4, 0x0c8, 0, 2, 7, CLK_CFG_UPDATE1, 0), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DPI0_SEL, "dpi0_sel", dpi0_parents, 0x0c0, + 0x0c4, 0x0c8, 8, 3, 15, CLK_CFG_UPDATE1, 1), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DPI1_SEL, "dpi1_sel", dpi0_parents, 0x0c0, + 0x0c4, 0x0c8, 16, 3, 23, CLK_CFG_UPDATE1, 2), + MUX_GATE_CLR_SET_UPD(CLK_TOP_DSP_SEL, "dsp_sel", dsp_parents, 0x0c0, + 0x0c4, 0x0c8, 24, 3, 31, CLK_CFG_UPDATE1, 3), + /* CLK_CFG_9 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_NFI2X_SEL, "nfi2x_sel", nfi2x_parents, + 0x0d0, 0x0d4, 0x0d8, 0, 3, 7, CLK_CFG_UPDATE1, 4), + MUX_GATE_CLR_SET_UPD(CLK_TOP_NFIECC_SEL, "nfiecc_sel", nfiecc_parents, + 0x0d0, 0x0d4, 0x0d8, 8, 3, 15, CLK_CFG_UPDATE1, 5), + MUX_GATE_CLR_SET_UPD(CLK_TOP_ECC_SEL, "ecc_sel", ecc_parents, 0x0d0, + 0x0d4, 0x0d8, 16, 3, 23, CLK_CFG_UPDATE1, 6), + MUX_GATE_CLR_SET_UPD(CLK_TOP_ETH_SEL, "eth_sel", eth_parents, 0x0d0, + 0x0d4, 0x0d8, 24, 3, 31, CLK_CFG_UPDATE1, 7), + /* CLK_CFG_10 */ + MUX_GATE_CLR_SET_UPD(CLK_TOP_GCPU_SEL, "gcpu_sel", gcpu_parents, 0x0e0, + 0x0e4, 0x0e8, 0, 3, 7, CLK_CFG_UPDATE1, 8), + MUX_GATE_CLR_SET_UPD(CLK_TOP_GCPU_CPM_SEL, "gcpu_cpm_sel", + gcpu_cpm_parents, 0x0e0, 0x0e4, 0x0e8, 8, 2, 15, + CLK_CFG_UPDATE1, 9), + MUX_GATE_CLR_SET_UPD(CLK_TOP_APU_SEL, "apu_sel", apu_parents, 0x0e0, + 0x0e4, 0x0e8, 16, 3, 23, CLK_CFG_UPDATE1, 10), + MUX_GATE_CLR_SET_UPD(CLK_TOP_APU_IF_SEL, "apu_if_sel", apu_parents, + 0x0e0, 0x0e4, 0x0e8, 24, 3, 31, CLK_CFG_UPDATE1, + 11), +}; + +static const char * const mcu_bus_parents[] = { + "clk26m", + "armpll", + "mainpll", + "univpll_d2" +}; + +static struct mtk_composite mcu_muxes[] = { + /* bus_pll_divider_cfg */ + MUX_GATE_FLAGS(CLK_MCU_BUS_SEL, "mcu_bus_sel", mcu_bus_parents, 0x7C0, + 9, 2, -1, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL), +}; + +#define DIV_ADJ_F(_id, _name, _parent, _reg, _shift, _width, _flags) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .div_reg = _reg, \ + .div_shift = _shift, \ + .div_width = _width, \ + .clk_divider_flags = _flags, \ +} + +static const struct mtk_clk_divider top_adj_divs[] = { + DIV_ADJ_F(CLK_TOP_APLL12_CK_DIV0, "apll12_ck_div0", "apll_i2s0_sel", + 0x324, 0, 8, CLK_DIVIDER_ROUND_CLOSEST), + DIV_ADJ_F(CLK_TOP_APLL12_CK_DIV1, "apll12_ck_div1", "apll_i2s1_sel", + 0x324, 8, 8, CLK_DIVIDER_ROUND_CLOSEST), + DIV_ADJ_F(CLK_TOP_APLL12_CK_DIV2, "apll12_ck_div2", "apll_i2s2_sel", + 0x324, 16, 8, CLK_DIVIDER_ROUND_CLOSEST), + DIV_ADJ_F(CLK_TOP_APLL12_CK_DIV3, "apll12_ck_div3", "apll_i2s3_sel", + 0x324, 24, 8, CLK_DIVIDER_ROUND_CLOSEST), + DIV_ADJ_F(CLK_TOP_APLL12_CK_DIV6, "apll12_ck_div6", "apll_spdif_sel", + 0x32c, 0, 8, CLK_DIVIDER_ROUND_CLOSEST), +}; + +struct mtk_simple_gate { + int id; + const char *name; + const char *parent; + u32 reg; + u8 shift; + unsigned long gate_flags; +}; + +static const struct mtk_simple_gate top_clk_gates[] = { + { CLK_TOP_CONN_32K, "conn_32k", "clk32k", 0x0, 10, CLK_GATE_SET_TO_DISABLE }, + { CLK_TOP_CONN_26M, "conn_26m", "clk26m", 0x0, 11, CLK_GATE_SET_TO_DISABLE }, + { CLK_TOP_DSP_32K, "dsp_32k", "clk32k", 0x0, 16, CLK_GATE_SET_TO_DISABLE }, + { CLK_TOP_DSP_26M, "dsp_26m", "clk26m", 0x0, 17, CLK_GATE_SET_TO_DISABLE }, + { CLK_TOP_USB20_48M_EN, "usb20_48m_en", "usb20_192m_d4", 0x104, 8, 0 }, + { CLK_TOP_UNIVPLL_48M_EN, "univpll_48m_en", "usb20_192m_d4", 0x104, 9, 0 }, + { CLK_TOP_LVDSTX_CLKDIG_EN, "lvdstx_dig_en", "lvdstx_dig_cts", 0x104, 20, 0 }, + { CLK_TOP_VPLL_DPIX_EN, "vpll_dpix_en", "vpll_dpix", 0x104, 21, 0 }, + { CLK_TOP_SSUSB_TOP_CK_EN, "ssusb_top_ck_en", NULL, 0x104, 22, 0 }, + { CLK_TOP_SSUSB_PHY_CK_EN, "ssusb_phy_ck_en", NULL, 0x104, 23, 0 }, + { CLK_TOP_AUD_I2S0_M, "aud_i2s0_m_ck", "apll12_ck_div0", 0x320, 0, 0 }, + { CLK_TOP_AUD_I2S1_M, "aud_i2s1_m_ck", "apll12_ck_div1", 0x320, 1, 0 }, + { CLK_TOP_AUD_I2S2_M, "aud_i2s2_m_ck", "apll12_ck_div2", 0x320, 2, 0 }, + { CLK_TOP_AUD_I2S3_M, "aud_i2s3_m_ck", "apll12_ck_div3", 0x320, 3, 0 }, + { CLK_TOP_AUD_TDMOUT_M, "aud_tdmout_m_ck", "apll12_ck_div4", 0x320, 4, 0 }, + { CLK_TOP_AUD_TDMOUT_B, "aud_tdmout_b_ck", "apll12_ck_div4b", 0x320, 5, 0 }, + { CLK_TOP_AUD_TDMIN_M, "aud_tdmin_m_ck", "apll12_ck_div5", 0x320, 6, 0 }, + { CLK_TOP_AUD_TDMIN_B, "aud_tdmin_b_ck", "apll12_ck_div5b", 0x320, 7, 0 }, + { CLK_TOP_AUD_SPDIF_M, "aud_spdif_m_ck", "apll12_ck_div6", 0x320, 8, 0 }, +}; + +static const struct mtk_gate_regs ifr2_cg_regs = { + .set_ofs = 0x80, + .clr_ofs = 0x84, + .sta_ofs = 0x90, +}; + +static const struct mtk_gate_regs ifr3_cg_regs = { + .set_ofs = 0x88, + .clr_ofs = 0x8c, + .sta_ofs = 0x94, +}; + +static const struct mtk_gate_regs ifr4_cg_regs = { + .set_ofs = 0xa4, + .clr_ofs = 0xa8, + .sta_ofs = 0xac, +}; + +static const struct mtk_gate_regs ifr5_cg_regs = { + .set_ofs = 0xc0, + .clr_ofs = 0xc4, + .sta_ofs = 0xc8, +}; + +static const struct mtk_gate_regs ifr6_cg_regs = { + .set_ofs = 0xd0, + .clr_ofs = 0xd4, + .sta_ofs = 0xd8, +}; + +#define GATE_IFR2(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &ifr2_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_IFR3(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &ifr3_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_IFR4(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &ifr4_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_IFR5(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &ifr5_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +#define GATE_IFR6(_id, _name, _parent, _shift) { \ + .id = _id, \ + .name = _name, \ + .parent_name = _parent, \ + .regs = &ifr6_cg_regs, \ + .shift = _shift, \ + .ops = &mtk_clk_gate_ops_setclr, \ + } + +static const struct mtk_gate ifr_clks[] = { + /* IFR2 */ + GATE_IFR2(CLK_IFR_PMIC_TMR, "ifr_pmic_tmr", "clk26m", 0), + GATE_IFR2(CLK_IFR_PMIC_AP, "ifr_pmic_ap", "clk26m", 1), + GATE_IFR2(CLK_IFR_PMIC_MD, "ifr_pmic_md", "clk26m", 2), + GATE_IFR2(CLK_IFR_PMIC_CONN, "ifr_pmic_conn", "clk26m", 3), + GATE_IFR2(CLK_IFR_ICUSB, "ifr_icusb", "axi_sel", 8), + GATE_IFR2(CLK_IFR_GCE, "ifr_gce", "axi_sel", 9), + GATE_IFR2(CLK_IFR_THERM, "ifr_therm", "axi_sel", 10), + GATE_IFR2(CLK_IFR_PWM_HCLK, "ifr_pwm_hclk", "axi_sel", 15), + GATE_IFR2(CLK_IFR_PWM1, "ifr_pwm1", "pwm_sel", 16), + GATE_IFR2(CLK_IFR_PWM2, "ifr_pwm2", "pwm_sel", 17), + GATE_IFR2(CLK_IFR_PWM3, "ifr_pwm3", "pwm_sel", 18), + GATE_IFR2(CLK_IFR_PWM4, "ifr_pwm4", "pwm_sel", 19), + GATE_IFR2(CLK_IFR_PWM5, "ifr_pwm5", "pwm_sel", 20), + GATE_IFR2(CLK_IFR_PWM, "ifr_pwm", "pwm_sel", 21), + GATE_IFR2(CLK_IFR_UART0, "ifr_uart0", "uart_sel", 22), + GATE_IFR2(CLK_IFR_UART1, "ifr_uart1", "uart_sel", 23), + GATE_IFR2(CLK_IFR_UART2, "ifr_uart2", "uart_sel", 24), + GATE_IFR2(CLK_IFR_DSP_UART, "ifr_dsp_uart", "uart_sel", 26), + GATE_IFR2(CLK_IFR_GCE_26M, "ifr_gce_26m", "clk26m", 27), + GATE_IFR2(CLK_IFR_CQ_DMA_FPC, "ifr_cq_dma_fpc", "axi_sel", 28), + GATE_IFR2(CLK_IFR_BTIF, "ifr_btif", "axi_sel", 31), + /* IFR3 */ + GATE_IFR3(CLK_IFR_SPI0, "ifr_spi0", "spi_sel", 1), + GATE_IFR3(CLK_IFR_MSDC0_HCLK, "ifr_msdc0", "msdc50_0_hc_sel", 2), + GATE_IFR3(CLK_IFR_MSDC2_HCLK, "ifr_msdc2", "msdc2_2_hc_sel", 3), + GATE_IFR3(CLK_IFR_MSDC1_HCLK, "ifr_msdc1", "axi_sel", 4), + GATE_IFR3(CLK_IFR_DVFSRC, "ifr_dvfsrc", "clk26m", 7), + GATE_IFR3(CLK_IFR_GCPU, "ifr_gcpu", "axi_sel", 8), + GATE_IFR3(CLK_IFR_TRNG, "ifr_trng", "axi_sel", 9), + GATE_IFR3(CLK_IFR_AUXADC, "ifr_auxadc", "clk26m", 10), + GATE_IFR3(CLK_IFR_AUXADC_MD, "ifr_auxadc_md", "clk26m", 14), + GATE_IFR3(CLK_IFR_AP_DMA, "ifr_ap_dma", "axi_sel", 18), + GATE_IFR3(CLK_IFR_DEBUGSYS, "ifr_debugsys", "axi_sel", 24), + GATE_IFR3(CLK_IFR_AUDIO, "ifr_audio", "axi_sel", 25), + /* IFR4 */ + GATE_IFR4(CLK_IFR_PWM_FBCLK6, "ifr_pwm_fbclk6", "pwm_sel", 0), + GATE_IFR4(CLK_IFR_DISP_PWM, "ifr_disp_pwm", "disp_pwm_sel", 2), + GATE_IFR4(CLK_IFR_AUD_26M_BK, "ifr_aud_26m_bk", "clk26m", 4), + GATE_IFR4(CLK_IFR_CQ_DMA, "ifr_cq_dma", "axi_sel", 27), + /* IFR5 */ + GATE_IFR5(CLK_IFR_MSDC0_SF, "ifr_msdc0_sf", "msdc50_0_sel", 0), + GATE_IFR5(CLK_IFR_MSDC1_SF, "ifr_msdc1_sf", "msdc50_0_sel", 1), + GATE_IFR5(CLK_IFR_MSDC2_SF, "ifr_msdc2_sf", "msdc50_0_sel", 2), + GATE_IFR5(CLK_IFR_AP_MSDC0, "ifr_ap_msdc0", "msdc50_0_sel", 7), + GATE_IFR5(CLK_IFR_MD_MSDC0, "ifr_md_msdc0", "msdc50_0_sel", 8), + GATE_IFR5(CLK_IFR_MSDC0_SRC, "ifr_msdc0_src", "msdc50_0_sel", 9), + GATE_IFR5(CLK_IFR_MSDC1_SRC, "ifr_msdc1_src", "msdc30_1_sel", 10), + GATE_IFR5(CLK_IFR_MSDC2_SRC, "ifr_msdc2_src", "msdc50_2_sel", 11), + GATE_IFR5(CLK_IFR_PWRAP_TMR, "ifr_pwrap_tmr", "clk26m", 12), + GATE_IFR5(CLK_IFR_PWRAP_SPI, "ifr_pwrap_spi", "clk26m", 13), + GATE_IFR5(CLK_IFR_PWRAP_SYS, "ifr_pwrap_sys", "clk26m", 14), + GATE_IFR5(CLK_IFR_IRRX_26M, "ifr_irrx_26m", "clk26m", 22), + GATE_IFR5(CLK_IFR_IRRX_32K, "ifr_irrx_32k", "clk32k", 23), + GATE_IFR5(CLK_IFR_I2C0_AXI, "ifr_i2c0_axi", "i2c_sel", 24), + GATE_IFR5(CLK_IFR_I2C1_AXI, "ifr_i2c1_axi", "i2c_sel", 25), + GATE_IFR5(CLK_IFR_I2C2_AXI, "ifr_i2c2_axi", "i2c_sel", 26), + GATE_IFR5(CLK_IFR_I2C3_AXI, "ifr_i2c3_axi", "i2c_sel", 27), + GATE_IFR5(CLK_IFR_NIC_AXI, "ifr_nic_axi", "axi_sel", 28), + GATE_IFR5(CLK_IFR_NIC_SLV_AXI, "ifr_nic_slv_axi", "axi_sel", 29), + GATE_IFR5(CLK_IFR_APU_AXI, "ifr_apu_axi", "axi_sel", 30), + /* IFR6 */ + GATE_IFR6(CLK_IFR_NFIECC, "ifr_nfiecc", "nfiecc_sel", 0), + GATE_IFR6(CLK_IFR_NFI1X_BK, "ifr_nfi1x_bk", "nfi2x_sel", 1), + GATE_IFR6(CLK_IFR_NFIECC_BK, "ifr_nfiecc_bk", "nfi2x_sel", 2), + GATE_IFR6(CLK_IFR_NFI_BK, "ifr_nfi_bk", "axi_sel", 3), + GATE_IFR6(CLK_IFR_MSDC2_AP_BK, "ifr_msdc2_ap_bk", "axi_sel", 4), + GATE_IFR6(CLK_IFR_MSDC2_MD_BK, "ifr_msdc2_md_bk", "axi_sel", 5), + GATE_IFR6(CLK_IFR_MSDC2_BK, "ifr_msdc2_bk", "axi_sel", 6), + GATE_IFR6(CLK_IFR_SUSB_133_BK, "ifr_susb_133_bk", "axi_sel", 7), + GATE_IFR6(CLK_IFR_SUSB_66_BK, "ifr_susb_66_bk", "axi_sel", 8), + GATE_IFR6(CLK_IFR_SSUSB_SYS, "ifr_ssusb_sys", "ssusb_sys_sel", 9), + GATE_IFR6(CLK_IFR_SSUSB_REF, "ifr_ssusb_ref", "ssusb_sys_sel", 10), + GATE_IFR6(CLK_IFR_SSUSB_XHCI, "ifr_ssusb_xhci", "ssusb_xhci_sel", 11), +}; + +static const struct mtk_simple_gate peri_clks[] = { + { CLK_PERIAXI, "periaxi", "axi_sel", 0x20c, 31, 0 }, +}; + +#define MT8365_PLL_FMAX (3800UL * MHZ) +#define MT8365_PLL_FMIN (1500UL * MHZ) +#define CON0_MT8365_RST_BAR BIT(23) + +#define PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, _tuner_en_reg, \ + _tuner_en_bit, _pcw_reg, _pcw_shift, _div_table, \ + _rst_bar_mask, _pcw_chg_reg) { \ + .id = _id, \ + .name = _name, \ + .reg = _reg, \ + .pwr_reg = _pwr_reg, \ + .en_mask = _en_mask, \ + .flags = _flags, \ + .rst_bar_mask = _rst_bar_mask, \ + .fmax = MT8365_PLL_FMAX, \ + .fmin = MT8365_PLL_FMIN, \ + .pcwbits = _pcwbits, \ + .pcwibits = 8, \ + .pd_reg = _pd_reg, \ + .pd_shift = _pd_shift, \ + .tuner_reg = _tuner_reg, \ + .tuner_en_reg = _tuner_en_reg, \ + .tuner_en_bit = _tuner_en_bit, \ + .pcw_reg = _pcw_reg, \ + .pcw_shift = _pcw_shift, \ + .pcw_chg_reg = _pcw_chg_reg, \ + .div_table = _div_table, \ + } + +#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \ + _pd_reg, _pd_shift, _tuner_reg, \ + _tuner_en_reg, _tuner_en_bit, _pcw_reg, \ + _pcw_shift, _rst_bar_mask, _pcw_chg_reg) \ + PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, \ + _pcwbits, _pd_reg, _pd_shift, \ + _tuner_reg, _tuner_en_reg, _tuner_en_bit, \ + _pcw_reg, _pcw_shift, NULL, _rst_bar_mask, \ + _pcw_chg_reg) \ + +static const struct mtk_pll_div_table armpll_div_table[] = { + { .div = 0, .freq = MT8365_PLL_FMAX }, + { .div = 1, .freq = 1500 * MHZ }, + { .div = 2, .freq = 750 * MHZ }, + { .div = 3, .freq = 375 * MHZ }, + { .div = 4, .freq = 182500000 }, + { } /* sentinel */ +}; + +static const struct mtk_pll_div_table mfgpll_div_table[] = { + { .div = 0, .freq = MT8365_PLL_FMAX }, + { .div = 1, .freq = 1600 * MHZ }, + { .div = 2, .freq = 800 * MHZ }, + { .div = 3, .freq = 400 * MHZ }, + { .div = 4, .freq = 200 * MHZ }, + { } /* sentinel */ +}; + +static const struct mtk_pll_div_table dsppll_div_table[] = { + { .div = 0, .freq = MT8365_PLL_FMAX }, + { .div = 1, .freq = 1600 * MHZ }, + { .div = 2, .freq = 600 * MHZ }, + { .div = 3, .freq = 400 * MHZ }, + { .div = 4, .freq = 200 * MHZ }, + { } /* sentinel */ +}; + +static const struct mtk_pll_data plls[] = { + PLL_B(CLK_APMIXED_ARMPLL, "armpll", 0x030C, 0x0318, 0x00000001, PLL_AO, + 22, 0x0310, 24, 0, 0, 0, 0x0310, 0, armpll_div_table, 0, 0), + PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x0228, 0x0234, 0xFF000001, + HAVE_RST_BAR, 22, 0x022C, 24, 0, 0, 0, 0x022C, 0, + CON0_MT8365_RST_BAR, 0), + PLL(CLK_APMIXED_UNIVPLL, "univpll2", 0x0208, 0x0214, 0xFF000001, + HAVE_RST_BAR, 22, 0x020C, 24, 0, 0, 0, 0x020C, 0, + CON0_MT8365_RST_BAR, 0), + PLL_B(CLK_APMIXED_MFGPLL, "mfgpll", 0x0218, 0x0224, 0x00000001, 0, 22, + 0x021C, 24, 0, 0, 0, 0x021C, 0, mfgpll_div_table, 0, 0), + PLL(CLK_APMIXED_MSDCPLL, "msdcpll", 0x0350, 0x035C, 0x00000001, 0, 22, + 0x0354, 24, 0, 0, 0, 0x0354, 0, 0, 0), + PLL(CLK_APMIXED_MMPLL, "mmpll", 0x0330, 0x033C, 0x00000001, 0, 22, + 0x0334, 24, 0, 0, 0, 0x0334, 0, 0, 0), + PLL(CLK_APMIXED_APLL1, "apll1", 0x031C, 0x032C, 0x00000001, 0, 32, + 0x0320, 24, 0x0040, 0x000C, 0, 0x0324, 0, 0, 0x0320), + PLL(CLK_APMIXED_APLL2, "apll2", 0x0360, 0x0370, 0x00000001, 0, 32, + 0x0364, 24, 0x004C, 0x000C, 5, 0x0368, 0, 0, 0x0364), + PLL(CLK_APMIXED_LVDSPLL, "lvdspll", 0x0374, 0x0380, 0x00000001, 0, 22, + 0x0378, 24, 0, 0, 0, 0x0378, 0, 0, 0), + PLL_B(CLK_APMIXED_DSPPLL, "dsppll", 0x0390, 0x039C, 0x00000001, 0, 22, + 0x0394, 24, 0, 0, 0, 0x0394, 0, dsppll_div_table, 0, 0), + PLL(CLK_APMIXED_APUPLL, "apupll", 0x03A0, 0x03AC, 0x00000001, 0, 22, + 0x03A4, 24, 0, 0, 0, 0x03A4, 0, 0, 0), +}; + +static int clk_mt8365_apmixed_probe(struct platform_device *pdev) +{ + void __iomem *base; + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct clk_hw *hw; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_devm_alloc_clk_data(dev, CLK_APMIXED_NR_CLK); + if (!clk_data) + return -ENOMEM; + + hw = devm_clk_hw_register_gate(dev, "univ_en", "univpll2", 0, + base + 0x204, 0, 0, NULL); + if (IS_ERR(hw)) + return PTR_ERR(hw); + clk_data->hws[CLK_APMIXED_UNIV_EN] = hw; + + hw = devm_clk_hw_register_gate(dev, "usb20_en", "univ_en", 0, + base + 0x204, 1, 0, NULL); + if (IS_ERR(hw)) + return PTR_ERR(hw); + clk_data->hws[CLK_APMIXED_USB20_EN] = hw; + + ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + if (ret) + return ret; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_plls; + + return 0; + +unregister_plls: + mtk_clk_unregister_plls(plls, ARRAY_SIZE(plls), clk_data); + + return ret; +} + +static int +clk_mt8365_register_mtk_simple_gates(struct device *dev, void __iomem *base, + struct clk_hw_onecell_data *clk_data, + const struct mtk_simple_gate *gates, + unsigned int num_gates) +{ + unsigned int i; + + for (i = 0; i != num_gates; ++i) { + const struct mtk_simple_gate *gate = &gates[i]; + struct clk_hw *hw; + + hw = devm_clk_hw_register_gate(dev, gate->name, gate->parent, 0, + base + gate->reg, gate->shift, + gate->gate_flags, NULL); + if (IS_ERR(hw)) + return PTR_ERR(hw); + + clk_data->hws[gate->id] = hw; + } + + return 0; +} + +static int clk_mt8365_top_probe(struct platform_device *pdev) +{ + void __iomem *base; + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; + int ret; + int i; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_fixed_clks(top_fixed_clks, + ARRAY_SIZE(top_fixed_clks), clk_data); + if (ret) + goto free_clk_data; + + ret = mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), + clk_data); + if (ret) + goto unregister_fixed_clks; + + ret = mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes), node, + &mt8365_clk_lock, clk_data); + if (ret) + goto unregister_factors; + + ret = mtk_clk_register_composites(top_misc_mux_gates, + ARRAY_SIZE(top_misc_mux_gates), base, + &mt8365_clk_lock, clk_data); + if (ret) + goto unregister_muxes; + + for (i = 0; i != ARRAY_SIZE(top_misc_muxes); ++i) { + struct mt8365_clk_audio_mux *mux = &top_misc_muxes[i]; + struct clk_hw *hw; + + hw = devm_clk_hw_register_mux(dev, mux->name, apll_i2s0_parents, + ARRAY_SIZE(apll_i2s0_parents), + CLK_SET_RATE_PARENT, base + 0x320, + mux->shift, 1, 0, NULL); + if (IS_ERR(hw)) { + ret = PTR_ERR(hw); + goto unregister_composites; + } + + clk_data->hws[mux->id] = hw; + } + + ret = mtk_clk_register_dividers(top_adj_divs, ARRAY_SIZE(top_adj_divs), + base, &mt8365_clk_lock, clk_data); + if (ret) + goto unregister_composites; + + ret = clk_mt8365_register_mtk_simple_gates(dev, base, clk_data, + top_clk_gates, + ARRAY_SIZE(top_clk_gates)); + if (ret) + goto unregister_dividers; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_dividers; + + return 0; +unregister_dividers: + mtk_clk_unregister_dividers(top_adj_divs, ARRAY_SIZE(top_adj_divs), + clk_data); +unregister_composites: + mtk_clk_unregister_composites(top_misc_mux_gates, + ARRAY_SIZE(top_misc_mux_gates), clk_data); +unregister_muxes: + mtk_clk_unregister_muxes(top_muxes, ARRAY_SIZE(top_muxes), clk_data); +unregister_factors: + mtk_clk_unregister_factors(top_divs, ARRAY_SIZE(top_divs), clk_data); +unregister_fixed_clks: + mtk_clk_unregister_fixed_clks(top_fixed_clks, + ARRAY_SIZE(top_fixed_clks), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + + return ret; +} + +static int clk_mt8365_infra_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + int ret; + + clk_data = mtk_alloc_clk_data(CLK_IFR_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_gates(node, ifr_clks, ARRAY_SIZE(ifr_clks), + clk_data); + if (ret) + goto free_clk_data; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_gates; + + return 0; + +unregister_gates: + mtk_clk_unregister_gates(ifr_clks, ARRAY_SIZE(ifr_clks), clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + + return ret; +} + +static int clk_mt8365_peri_probe(struct platform_device *pdev) +{ + void __iomem *base; + struct clk_hw_onecell_data *clk_data; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_devm_alloc_clk_data(dev, CLK_PERI_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = clk_mt8365_register_mtk_simple_gates(dev, base, clk_data, + peri_clks, + ARRAY_SIZE(peri_clks)); + if (ret) + return ret; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + + return ret; +} + +static int clk_mt8365_mcu_probe(struct platform_device *pdev) +{ + struct clk_hw_onecell_data *clk_data; + struct device_node *node = pdev->dev.of_node; + void __iomem *base; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + clk_data = mtk_alloc_clk_data(CLK_MCU_NR_CLK); + if (!clk_data) + return -ENOMEM; + + ret = mtk_clk_register_composites(mcu_muxes, ARRAY_SIZE(mcu_muxes), + base, &mt8365_clk_lock, clk_data); + if (ret) + goto free_clk_data; + + ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + if (ret) + goto unregister_composites; + + return 0; + +unregister_composites: + mtk_clk_unregister_composites(mcu_muxes, ARRAY_SIZE(mcu_muxes), + clk_data); +free_clk_data: + mtk_free_clk_data(clk_data); + + return ret; +} + +static const struct of_device_id of_match_clk_mt8365[] = { + { + .compatible = "mediatek,mt8365-apmixedsys", + .data = clk_mt8365_apmixed_probe, + }, { + .compatible = "mediatek,mt8365-topckgen", + .data = clk_mt8365_top_probe, + }, { + .compatible = "mediatek,mt8365-infracfg", + .data = clk_mt8365_infra_probe, + }, { + .compatible = "mediatek,mt8365-pericfg", + .data = clk_mt8365_peri_probe, + }, { + .compatible = "mediatek,mt8365-mcucfg", + .data = clk_mt8365_mcu_probe, + }, { + /* sentinel */ + } +}; + +static int clk_mt8365_probe(struct platform_device *pdev) +{ + int (*clk_probe)(struct platform_device *pdev); + int ret; + + clk_probe = of_device_get_match_data(&pdev->dev); + if (!clk_probe) + return -EINVAL; + + ret = clk_probe(pdev); + if (ret) + dev_err(&pdev->dev, + "%s: could not register clock provider: %d\n", + pdev->name, ret); + + return ret; +} + +static struct platform_driver clk_mt8365_drv = { + .probe = clk_mt8365_probe, + .driver = { + .name = "clk-mt8365", + .of_match_table = of_match_clk_mt8365, + }, +}; + +static int __init clk_mt8365_init(void) +{ + return platform_driver_register(&clk_mt8365_drv); +} +arch_initcall(clk_mt8365_init); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mtk.c b/drivers/clk/mediatek/clk-mtk.c index 05a188c6211916aaaf23d9903f8515196c8385a3..d31f01d0ba1c23c19c964aea19f7a49eaba14680 100644 --- a/drivers/clk/mediatek/clk-mtk.c +++ b/drivers/clk/mediatek/clk-mtk.c @@ -18,19 +18,42 @@ #include "clk-mtk.h" #include "clk-gate.h" -struct clk_hw_onecell_data *mtk_alloc_clk_data(unsigned int clk_num) +static void mtk_init_clk_data(struct clk_hw_onecell_data *clk_data, + unsigned int clk_num) { int i; + + clk_data->num = clk_num; + + for (i = 0; i < clk_num; i++) + clk_data->hws[i] = ERR_PTR(-ENOENT); +} + +struct clk_hw_onecell_data *mtk_devm_alloc_clk_data(struct device *dev, + unsigned int clk_num) +{ struct clk_hw_onecell_data *clk_data; - clk_data = kzalloc(struct_size(clk_data, hws, clk_num), GFP_KERNEL); + clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, clk_num), + GFP_KERNEL); if (!clk_data) return NULL; - clk_data->num = clk_num; + mtk_init_clk_data(clk_data, clk_num); - for (i = 0; i < clk_num; i++) - clk_data->hws[i] = ERR_PTR(-ENOENT); + return clk_data; +} +EXPORT_SYMBOL_GPL(mtk_devm_alloc_clk_data); + +struct clk_hw_onecell_data *mtk_alloc_clk_data(unsigned int clk_num) +{ + struct clk_hw_onecell_data *clk_data; + + clk_data = kzalloc(struct_size(clk_data, hws, clk_num), GFP_KERNEL); + if (!clk_data) + return NULL; + + mtk_init_clk_data(clk_data, clk_num); return clk_data; } @@ -80,7 +103,7 @@ err: if (IS_ERR_OR_NULL(clk_data->hws[rc->id])) continue; - clk_unregister_fixed_rate(clk_data->hws[rc->id]->clk); + clk_hw_unregister_fixed_rate(clk_data->hws[rc->id]); clk_data->hws[rc->id] = ERR_PTR(-ENOENT); } @@ -102,7 +125,7 @@ void mtk_clk_unregister_fixed_clks(const struct mtk_fixed_clk *clks, int num, if (IS_ERR_OR_NULL(clk_data->hws[rc->id])) continue; - clk_unregister_fixed_rate(clk_data->hws[rc->id]->clk); + clk_hw_unregister_fixed_rate(clk_data->hws[rc->id]); clk_data->hws[rc->id] = ERR_PTR(-ENOENT); } } @@ -146,7 +169,7 @@ err: if (IS_ERR_OR_NULL(clk_data->hws[ff->id])) continue; - clk_unregister_fixed_factor(clk_data->hws[ff->id]->clk); + clk_hw_unregister_fixed_factor(clk_data->hws[ff->id]); clk_data->hws[ff->id] = ERR_PTR(-ENOENT); } @@ -168,7 +191,7 @@ void mtk_clk_unregister_factors(const struct mtk_fixed_factor *clks, int num, if (IS_ERR_OR_NULL(clk_data->hws[ff->id])) continue; - clk_unregister_fixed_factor(clk_data->hws[ff->id]->clk); + clk_hw_unregister_fixed_factor(clk_data->hws[ff->id]); clk_data->hws[ff->id] = ERR_PTR(-ENOENT); } } @@ -393,12 +416,13 @@ err: if (IS_ERR_OR_NULL(clk_data->hws[mcd->id])) continue; - mtk_clk_unregister_composite(clk_data->hws[mcd->id]); + clk_hw_unregister_divider(clk_data->hws[mcd->id]); clk_data->hws[mcd->id] = ERR_PTR(-ENOENT); } return PTR_ERR(hw); } +EXPORT_SYMBOL_GPL(mtk_clk_register_dividers); void mtk_clk_unregister_dividers(const struct mtk_clk_divider *mcds, int num, struct clk_hw_onecell_data *clk_data) @@ -414,10 +438,11 @@ void mtk_clk_unregister_dividers(const struct mtk_clk_divider *mcds, int num, if (IS_ERR_OR_NULL(clk_data->hws[mcd->id])) continue; - clk_unregister_divider(clk_data->hws[mcd->id]->clk); + clk_hw_unregister_divider(clk_data->hws[mcd->id]); clk_data->hws[mcd->id] = ERR_PTR(-ENOENT); } } +EXPORT_SYMBOL_GPL(mtk_clk_unregister_dividers); int mtk_clk_simple_probe(struct platform_device *pdev) { @@ -434,7 +459,8 @@ int mtk_clk_simple_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_gates(node, mcd->clks, mcd->num_clks, clk_data); + r = mtk_clk_register_gates_with_dev(node, mcd->clks, mcd->num_clks, + clk_data, &pdev->dev); if (r) goto free_data; @@ -459,6 +485,7 @@ free_data: mtk_free_clk_data(clk_data); return r; } +EXPORT_SYMBOL_GPL(mtk_clk_simple_probe); int mtk_clk_simple_remove(struct platform_device *pdev) { @@ -472,5 +499,6 @@ int mtk_clk_simple_remove(struct platform_device *pdev) return 0; } +EXPORT_SYMBOL_GPL(mtk_clk_simple_remove); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mtk.h b/drivers/clk/mediatek/clk-mtk.h index 1b95c484d5aa42ed81d49ce8e520989b92894de8..63ae7941aa92f10c03483f288ab352aed44637ee 100644 --- a/drivers/clk/mediatek/clk-mtk.h +++ b/drivers/clk/mediatek/clk-mtk.h @@ -184,10 +184,13 @@ void mtk_clk_unregister_dividers(const struct mtk_clk_divider *mcds, int num, struct clk_hw_onecell_data *clk_data); struct clk_hw_onecell_data *mtk_alloc_clk_data(unsigned int clk_num); +struct clk_hw_onecell_data *mtk_devm_alloc_clk_data(struct device *dev, + unsigned int clk_num); void mtk_free_clk_data(struct clk_hw_onecell_data *clk_data); struct clk_hw *mtk_clk_register_ref2usb_tx(const char *name, const char *parent_name, void __iomem *reg); +void mtk_clk_unregister_ref2usb_tx(struct clk_hw *hw); struct mtk_clk_desc { const struct mtk_gate *clks; diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c index cd5f9fd8cb98e14a38b4eb6e2f98886a9ead9b9c..ba1720b9e23101929a54b4c09955f0ec93d52342 100644 --- a/drivers/clk/mediatek/clk-mux.c +++ b/drivers/clk/mediatek/clk-mux.c @@ -4,6 +4,7 @@ * Author: Owen Chen */ +#include #include #include #include @@ -128,9 +129,18 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) return 0; } +static int mtk_clk_mux_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); + + return clk_mux_determine_rate_flags(hw, req, mux->data->flags); +} + const struct clk_ops mtk_mux_clr_set_upd_ops = { .get_parent = mtk_clk_mux_get_parent, .set_parent = mtk_clk_mux_set_parent_setclr_lock, + .determine_rate = mtk_clk_mux_determine_rate, }; EXPORT_SYMBOL_GPL(mtk_mux_clr_set_upd_ops); @@ -140,6 +150,7 @@ const struct clk_ops mtk_mux_gate_clr_set_upd_ops = { .is_enabled = mtk_clk_mux_is_enabled, .get_parent = mtk_clk_mux_get_parent, .set_parent = mtk_clk_mux_set_parent_setclr_lock, + .determine_rate = mtk_clk_mux_determine_rate, }; EXPORT_SYMBOL_GPL(mtk_mux_gate_clr_set_upd_ops); @@ -259,4 +270,41 @@ void mtk_clk_unregister_muxes(const struct mtk_mux *muxes, int num, } EXPORT_SYMBOL_GPL(mtk_clk_unregister_muxes); +/* + * This clock notifier is called when the frequency of the parent + * PLL clock is to be changed. The idea is to switch the parent to a + * stable clock, such as the main oscillator, while the PLL frequency + * stabilizes. + */ +static int mtk_clk_mux_notifier_cb(struct notifier_block *nb, + unsigned long event, void *_data) +{ + struct clk_notifier_data *data = _data; + struct clk_hw *hw = __clk_get_hw(data->clk); + struct mtk_mux_nb *mux_nb = to_mtk_mux_nb(nb); + int ret = 0; + + switch (event) { + case PRE_RATE_CHANGE: + mux_nb->original_index = mux_nb->ops->get_parent(hw); + ret = mux_nb->ops->set_parent(hw, mux_nb->bypass_index); + break; + case POST_RATE_CHANGE: + case ABORT_RATE_CHANGE: + ret = mux_nb->ops->set_parent(hw, mux_nb->original_index); + break; + } + + return notifier_from_errno(ret); +} + +int devm_mtk_clk_mux_notifier_register(struct device *dev, struct clk *clk, + struct mtk_mux_nb *mux_nb) +{ + mux_nb->nb.notifier_call = mtk_clk_mux_notifier_cb; + + return devm_clk_notifier_register(dev, clk, &mux_nb->nb); +} +EXPORT_SYMBOL_GPL(devm_mtk_clk_mux_notifier_register); + MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mediatek/clk-mux.h b/drivers/clk/mediatek/clk-mux.h index 6539c58f5d7df1f4d200ba3c911f507c613d260e..83ff420f4ebe6f49627018f6f616a820d6c7b7af 100644 --- a/drivers/clk/mediatek/clk-mux.h +++ b/drivers/clk/mediatek/clk-mux.h @@ -7,12 +7,14 @@ #ifndef __DRV_CLK_MTK_MUX_H #define __DRV_CLK_MTK_MUX_H +#include #include #include struct clk; struct clk_hw_onecell_data; struct clk_ops; +struct device; struct device_node; struct mtk_mux { @@ -89,4 +91,17 @@ int mtk_clk_register_muxes(const struct mtk_mux *muxes, void mtk_clk_unregister_muxes(const struct mtk_mux *muxes, int num, struct clk_hw_onecell_data *clk_data); +struct mtk_mux_nb { + struct notifier_block nb; + const struct clk_ops *ops; + + u8 bypass_index; /* Which parent to temporarily use */ + u8 original_index; /* Set by notifier callback */ +}; + +#define to_mtk_mux_nb(_nb) container_of(_nb, struct mtk_mux_nb, nb) + +int devm_mtk_clk_mux_notifier_register(struct device *dev, struct clk *clk, + struct mtk_mux_nb *mux_nb); + #endif /* __DRV_CLK_MTK_MUX_H */ diff --git a/drivers/clk/mediatek/reset.c b/drivers/clk/mediatek/reset.c index 179505549a7c4d6f7e4d1b90c117c1e6dc8cc6d6..290ceda84ce47dc54b5ad8b347b6189df347a371 100644 --- a/drivers/clk/mediatek/reset.c +++ b/drivers/clk/mediatek/reset.c @@ -228,5 +228,6 @@ int mtk_register_reset_controller_with_dev(struct device *dev, return 0; } +EXPORT_SYMBOL_GPL(mtk_register_reset_controller_with_dev); MODULE_LICENSE("GPL"); diff --git a/drivers/clk/meson/meson-aoclk.c b/drivers/clk/meson/meson-aoclk.c index 27cd2c1f3f6123fceb5216bffccf159d3c0d318f..434cd8f9de8262d2ce33b66623e74061a6c6f590 100644 --- a/drivers/clk/meson/meson-aoclk.c +++ b/drivers/clk/meson/meson-aoclk.c @@ -38,6 +38,7 @@ int meson_aoclkc_probe(struct platform_device *pdev) struct meson_aoclk_reset_controller *rstc; struct meson_aoclk_data *data; struct device *dev = &pdev->dev; + struct device_node *np; struct regmap *regmap; int ret, clkid; @@ -49,7 +50,9 @@ int meson_aoclkc_probe(struct platform_device *pdev) if (!rstc) return -ENOMEM; - regmap = syscon_node_to_regmap(of_get_parent(dev->of_node)); + np = of_get_parent(dev->of_node); + regmap = syscon_node_to_regmap(np); + of_node_put(np); if (IS_ERR(regmap)) { dev_err(dev, "failed to get regmap\n"); return PTR_ERR(regmap); diff --git a/drivers/clk/meson/meson-eeclk.c b/drivers/clk/meson/meson-eeclk.c index 8d5a5dab955a85c5ef7042d887def12cda1de439..0e5e6b57eb20e37c1082fa23f25f9495c44f565f 100644 --- a/drivers/clk/meson/meson-eeclk.c +++ b/drivers/clk/meson/meson-eeclk.c @@ -18,6 +18,7 @@ int meson_eeclkc_probe(struct platform_device *pdev) { const struct meson_eeclkc_data *data; struct device *dev = &pdev->dev; + struct device_node *np; struct regmap *map; int ret, i; @@ -26,7 +27,9 @@ int meson_eeclkc_probe(struct platform_device *pdev) return -EINVAL; /* Get the hhi system controller node */ - map = syscon_node_to_regmap(of_get_parent(dev->of_node)); + np = of_get_parent(dev->of_node); + map = syscon_node_to_regmap(np); + of_node_put(np); if (IS_ERR(map)) { dev_err(dev, "failed to get HHI regmap\n"); diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 8f3b7a94a66773f9e6ee2f4ca46f9d31e24b11f3..827e78fb16a84d749b9676b7c6267e7567f636ab 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -3792,12 +3792,15 @@ static void __init meson8b_clkc_init_common(struct device_node *np, struct clk_hw_onecell_data *clk_hw_onecell_data) { struct meson8b_clk_reset *rstc; + struct device_node *parent_np; const char *notifier_clk_name; struct clk *notifier_clk; struct regmap *map; int i, ret; - map = syscon_node_to_regmap(of_get_parent(np)); + parent_np = of_get_parent(np); + map = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(map)) { pr_err("failed to get HHI regmap - Trying obsolete regs\n"); return; diff --git a/drivers/clk/microchip/Kconfig b/drivers/clk/microchip/Kconfig index a5a99873c4f549942ababc2fd51043412811e2bd..b46e864b3bd87d719bb2c858405538efb3f1a30f 100644 --- a/drivers/clk/microchip/Kconfig +++ b/drivers/clk/microchip/Kconfig @@ -6,5 +6,6 @@ config COMMON_CLK_PIC32 config MCHP_CLK_MPFS bool "Clk driver for PolarFire SoC" depends on (RISCV && SOC_MICROCHIP_POLARFIRE) || COMPILE_TEST + select AUXILIARY_BUS help Supports Clock Configuration for PolarFire SoC diff --git a/drivers/clk/microchip/Makefile b/drivers/clk/microchip/Makefile index 5fa6dcf30a9aa378b8156f7403f49a98d2c1cbd6..13250e04e46cf14059da8cc3a5bbe3c443fb6fb5 100644 --- a/drivers/clk/microchip/Makefile +++ b/drivers/clk/microchip/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_COMMON_CLK_PIC32) += clk-core.o obj-$(CONFIG_PIC32MZDA) += clk-pic32mzda.o obj-$(CONFIG_MCHP_CLK_MPFS) += clk-mpfs.o +obj-$(CONFIG_MCHP_CLK_MPFS) += clk-mpfs-ccc.o diff --git a/drivers/clk/microchip/clk-mpfs-ccc.c b/drivers/clk/microchip/clk-mpfs-ccc.c new file mode 100644 index 0000000000000000000000000000000000000000..7be028dced63d352c039e40128f6da89d6b31d28 --- /dev/null +++ b/drivers/clk/microchip/clk-mpfs-ccc.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Author: Conor Dooley + * + * Copyright (C) 2022 Microchip Technology Inc. and its subsidiaries + */ +#include "asm-generic/errno-base.h" +#include +#include +#include +#include +#include + +/* address offset of control registers */ +#define MPFS_CCC_PLL_CR 0x04u +#define MPFS_CCC_REF_CR 0x08u +#define MPFS_CCC_SSCG_2_CR 0x2Cu +#define MPFS_CCC_POSTDIV01_CR 0x10u +#define MPFS_CCC_POSTDIV23_CR 0x14u + +#define MPFS_CCC_FBDIV_SHIFT 0x00u +#define MPFS_CCC_FBDIV_WIDTH 0x0Cu +#define MPFS_CCC_POSTDIV0_SHIFT 0x08u +#define MPFS_CCC_POSTDIV1_SHIFT 0x18u +#define MPFS_CCC_POSTDIV2_SHIFT MPFS_CCC_POSTDIV0_SHIFT +#define MPFS_CCC_POSTDIV3_SHIFT MPFS_CCC_POSTDIV1_SHIFT +#define MPFS_CCC_POSTDIV_WIDTH 0x06u +#define MPFS_CCC_REFCLK_SEL BIT(6) +#define MPFS_CCC_REFDIV_SHIFT 0x08u +#define MPFS_CCC_REFDIV_WIDTH 0x06u + +#define MPFS_CCC_FIXED_DIV 4 +#define MPFS_CCC_OUTPUTS_PER_PLL 4 +#define MPFS_CCC_REFS_PER_PLL 2 + +struct mpfs_ccc_data { + void __iomem **pll_base; + struct device *dev; + struct clk_hw_onecell_data hw_data; +}; + +struct mpfs_ccc_pll_hw_clock { + void __iomem *base; + const char *name; + const struct clk_parent_data *parents; + unsigned int id; + u32 reg_offset; + u32 shift; + u32 width; + u32 flags; + struct clk_hw hw; + struct clk_init_data init; +}; + +#define to_mpfs_ccc_clk(_hw) container_of(_hw, struct mpfs_ccc_pll_hw_clock, hw) + +/* + * mpfs_ccc_lock prevents anything else from writing to a fabric ccc + * while a software locked register is being written. + */ +static DEFINE_SPINLOCK(mpfs_ccc_lock); + +static const struct clk_parent_data mpfs_ccc_pll0_refs[] = { + { .fw_name = "pll0_ref0" }, + { .fw_name = "pll0_ref1" }, +}; + +static const struct clk_parent_data mpfs_ccc_pll1_refs[] = { + { .fw_name = "pll1_ref0" }, + { .fw_name = "pll1_ref1" }, +}; + +static unsigned long mpfs_ccc_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) +{ + struct mpfs_ccc_pll_hw_clock *ccc_hw = to_mpfs_ccc_clk(hw); + void __iomem *mult_addr = ccc_hw->base + ccc_hw->reg_offset; + void __iomem *ref_div_addr = ccc_hw->base + MPFS_CCC_REF_CR; + u32 mult, ref_div; + + mult = readl_relaxed(mult_addr) >> MPFS_CCC_FBDIV_SHIFT; + mult &= clk_div_mask(MPFS_CCC_FBDIV_WIDTH); + ref_div = readl_relaxed(ref_div_addr) >> MPFS_CCC_REFDIV_SHIFT; + ref_div &= clk_div_mask(MPFS_CCC_REFDIV_WIDTH); + + return prate * mult / (ref_div * MPFS_CCC_FIXED_DIV); +} + +static u8 mpfs_ccc_pll_get_parent(struct clk_hw *hw) +{ + struct mpfs_ccc_pll_hw_clock *ccc_hw = to_mpfs_ccc_clk(hw); + void __iomem *pll_cr_addr = ccc_hw->base + MPFS_CCC_PLL_CR; + + return !!(readl_relaxed(pll_cr_addr) & MPFS_CCC_REFCLK_SEL); +} + +static const struct clk_ops mpfs_ccc_pll_ops = { + .recalc_rate = mpfs_ccc_pll_recalc_rate, + .get_parent = mpfs_ccc_pll_get_parent, +}; + +#define CLK_CCC_PLL(_id, _parents, _shift, _width, _flags, _offset) { \ + .id = _id, \ + .shift = _shift, \ + .width = _width, \ + .reg_offset = _offset, \ + .flags = _flags, \ + .parents = _parents, \ +} + +static struct mpfs_ccc_pll_hw_clock mpfs_ccc_pll_clks[] = { + CLK_CCC_PLL(CLK_CCC_PLL0, mpfs_ccc_pll0_refs, MPFS_CCC_FBDIV_SHIFT, + MPFS_CCC_FBDIV_WIDTH, 0, MPFS_CCC_SSCG_2_CR), + CLK_CCC_PLL(CLK_CCC_PLL1, mpfs_ccc_pll1_refs, MPFS_CCC_FBDIV_SHIFT, + MPFS_CCC_FBDIV_WIDTH, 0, MPFS_CCC_SSCG_2_CR), +}; + +struct mpfs_ccc_out_hw_clock { + struct clk_divider divider; + struct clk_init_data init; + unsigned int id; + u32 reg_offset; +}; + +#define CLK_CCC_OUT(_id, _shift, _width, _flags, _offset) { \ + .id = _id, \ + .divider.shift = _shift, \ + .divider.width = _width, \ + .reg_offset = _offset, \ + .divider.flags = _flags, \ + .divider.lock = &mpfs_ccc_lock, \ +} + +static struct mpfs_ccc_out_hw_clock mpfs_ccc_pll0out_clks[] = { + CLK_CCC_OUT(CLK_CCC_PLL0_OUT0, MPFS_CCC_POSTDIV0_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV01_CR), + CLK_CCC_OUT(CLK_CCC_PLL0_OUT1, MPFS_CCC_POSTDIV1_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV01_CR), + CLK_CCC_OUT(CLK_CCC_PLL0_OUT2, MPFS_CCC_POSTDIV2_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV23_CR), + CLK_CCC_OUT(CLK_CCC_PLL0_OUT3, MPFS_CCC_POSTDIV3_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV23_CR), +}; + +static struct mpfs_ccc_out_hw_clock mpfs_ccc_pll1out_clks[] = { + CLK_CCC_OUT(CLK_CCC_PLL1_OUT0, MPFS_CCC_POSTDIV0_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV01_CR), + CLK_CCC_OUT(CLK_CCC_PLL1_OUT1, MPFS_CCC_POSTDIV1_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV01_CR), + CLK_CCC_OUT(CLK_CCC_PLL1_OUT2, MPFS_CCC_POSTDIV2_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV23_CR), + CLK_CCC_OUT(CLK_CCC_PLL1_OUT3, MPFS_CCC_POSTDIV3_SHIFT, MPFS_CCC_POSTDIV_WIDTH, + CLK_DIVIDER_ONE_BASED, MPFS_CCC_POSTDIV23_CR), +}; + +static struct mpfs_ccc_out_hw_clock *mpfs_ccc_pllout_clks[] = { + mpfs_ccc_pll0out_clks, mpfs_ccc_pll1out_clks +}; + +static int mpfs_ccc_register_outputs(struct device *dev, struct mpfs_ccc_out_hw_clock *out_hws, + unsigned int num_clks, struct mpfs_ccc_data *data, + struct mpfs_ccc_pll_hw_clock *parent) +{ + int ret; + + for (unsigned int i = 0; i < num_clks; i++) { + struct mpfs_ccc_out_hw_clock *out_hw = &out_hws[i]; + char *name = devm_kzalloc(dev, 23, GFP_KERNEL); + + snprintf(name, 23, "%s_out%u", parent->name, i); + out_hw->divider.hw.init = CLK_HW_INIT_HW(name, &parent->hw, &clk_divider_ops, 0); + out_hw->divider.reg = data->pll_base[i / MPFS_CCC_OUTPUTS_PER_PLL] + + out_hw->reg_offset; + + ret = devm_clk_hw_register(dev, &out_hw->divider.hw); + if (ret) + return dev_err_probe(dev, ret, "failed to register clock id: %d\n", + out_hw->id); + + data->hw_data.hws[out_hw->id] = &out_hw->divider.hw; + } + + return 0; +} + +#define CLK_HW_INIT_PARENTS_DATA_FIXED_SIZE(_name, _parents, _ops, _flags) \ + (&(struct clk_init_data) { \ + .flags = _flags, \ + .name = _name, \ + .parent_data = _parents, \ + .num_parents = MPFS_CCC_REFS_PER_PLL, \ + .ops = _ops, \ + }) + +static int mpfs_ccc_register_plls(struct device *dev, struct mpfs_ccc_pll_hw_clock *pll_hws, + unsigned int num_clks, struct mpfs_ccc_data *data) +{ + int ret; + + for (unsigned int i = 0; i < num_clks; i++) { + struct mpfs_ccc_pll_hw_clock *pll_hw = &pll_hws[i]; + char *name = devm_kzalloc(dev, 18, GFP_KERNEL); + + pll_hw->base = data->pll_base[i]; + snprintf(name, 18, "ccc%s_pll%u", strchrnul(dev->of_node->full_name, '@'), i); + pll_hw->name = (const char *)name; + pll_hw->hw.init = CLK_HW_INIT_PARENTS_DATA_FIXED_SIZE(pll_hw->name, + pll_hw->parents, + &mpfs_ccc_pll_ops, 0); + + ret = devm_clk_hw_register(dev, &pll_hw->hw); + if (ret) + return dev_err_probe(dev, ret, "failed to register ccc id: %d\n", + pll_hw->id); + + data->hw_data.hws[pll_hw->id] = &pll_hw->hw; + + ret = mpfs_ccc_register_outputs(dev, mpfs_ccc_pllout_clks[i], + MPFS_CCC_OUTPUTS_PER_PLL, data, pll_hw); + if (ret) + return ret; + } + + return 0; +} + +static int mpfs_ccc_probe(struct platform_device *pdev) +{ + struct mpfs_ccc_data *clk_data; + void __iomem *pll_base[ARRAY_SIZE(mpfs_ccc_pll_clks)]; + unsigned int num_clks; + int ret; + + num_clks = ARRAY_SIZE(mpfs_ccc_pll_clks) + ARRAY_SIZE(mpfs_ccc_pll0out_clks) + + ARRAY_SIZE(mpfs_ccc_pll1out_clks); + + clk_data = devm_kzalloc(&pdev->dev, struct_size(clk_data, hw_data.hws, num_clks), + GFP_KERNEL); + if (!clk_data) + return -ENOMEM; + + pll_base[0] = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pll_base[0])) + return PTR_ERR(pll_base[0]); + + pll_base[1] = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(pll_base[1])) + return PTR_ERR(pll_base[1]); + + clk_data->pll_base = pll_base; + clk_data->hw_data.num = num_clks; + clk_data->dev = &pdev->dev; + + ret = mpfs_ccc_register_plls(clk_data->dev, mpfs_ccc_pll_clks, + ARRAY_SIZE(mpfs_ccc_pll_clks), clk_data); + if (ret) + return ret; + + return devm_of_clk_add_hw_provider(clk_data->dev, of_clk_hw_onecell_get, + &clk_data->hw_data); +} + +static const struct of_device_id mpfs_ccc_of_match_table[] = { + { .compatible = "microchip,mpfs-ccc", }, + {} +}; +MODULE_DEVICE_TABLE(of, mpfs_ccc_of_match_table); + +static struct platform_driver mpfs_ccc_driver = { + .probe = mpfs_ccc_probe, + .driver = { + .name = "microchip-mpfs-ccc", + .of_match_table = mpfs_ccc_of_match_table, + }, +}; + +static int __init clk_ccc_init(void) +{ + return platform_driver_register(&mpfs_ccc_driver); +} +core_initcall(clk_ccc_init); + +static void __exit clk_ccc_exit(void) +{ + platform_driver_unregister(&mpfs_ccc_driver); +} +module_exit(clk_ccc_exit); + +MODULE_DESCRIPTION("Microchip PolarFire SoC Clock Conditioning Circuitry Driver"); +MODULE_AUTHOR("Conor Dooley "); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c index 070c3b89655906e5960e666f30e25f53f0f5f6b0..4f0a19db7ed749a43d08a12a46da70b80a82410d 100644 --- a/drivers/clk/microchip/clk-mpfs.c +++ b/drivers/clk/microchip/clk-mpfs.c @@ -1,14 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Daire McNamara, - * Copyright (C) 2020 Microchip Technology Inc. All rights reserved. + * PolarFire SoC MSS/core complex clock control + * + * Copyright (C) 2020-2022 Microchip Technology Inc. All rights reserved. */ +#include #include #include #include #include #include #include +#include /* address offset of control registers */ #define REG_MSSPLL_REF_CR 0x08u @@ -28,6 +31,7 @@ #define MSSPLL_FIXED_DIV 4u struct mpfs_clock_data { + struct device *dev; void __iomem *base; void __iomem *msspll_base; struct clk_hw_onecell_data hw_data; @@ -46,37 +50,18 @@ struct mpfs_msspll_hw_clock { #define to_mpfs_msspll_clk(_hw) container_of(_hw, struct mpfs_msspll_hw_clock, hw) -struct mpfs_cfg_clock { - const struct clk_div_table *table; - unsigned int id; - u32 reg_offset; - u8 shift; - u8 width; - u8 flags; -}; - struct mpfs_cfg_hw_clock { - struct mpfs_cfg_clock cfg; - void __iomem *sys_base; - struct clk_hw hw; + struct clk_divider cfg; struct clk_init_data init; -}; - -#define to_mpfs_cfg_clk(_hw) container_of(_hw, struct mpfs_cfg_hw_clock, hw) - -struct mpfs_periph_clock { unsigned int id; - u8 shift; + u32 reg_offset; }; struct mpfs_periph_hw_clock { - struct mpfs_periph_clock periph; - void __iomem *sys_base; - struct clk_hw hw; + struct clk_gate periph; + unsigned int id; }; -#define to_mpfs_periph_clk(_hw) container_of(_hw, struct mpfs_periph_hw_clock, hw) - /* * mpfs_clk_lock prevents anything else from writing to the * mpfs clk block while a software locked register is being written. @@ -126,8 +111,62 @@ static unsigned long mpfs_clk_msspll_recalc_rate(struct clk_hw *hw, unsigned lon return prate * mult / (ref_div * MSSPLL_FIXED_DIV * postdiv); } +static long mpfs_clk_msspll_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *prate) +{ + struct mpfs_msspll_hw_clock *msspll_hw = to_mpfs_msspll_clk(hw); + void __iomem *mult_addr = msspll_hw->base + msspll_hw->reg_offset; + void __iomem *ref_div_addr = msspll_hw->base + REG_MSSPLL_REF_CR; + u32 mult, ref_div; + unsigned long rate_before_ctrl; + + mult = readl_relaxed(mult_addr) >> MSSPLL_FBDIV_SHIFT; + mult &= clk_div_mask(MSSPLL_FBDIV_WIDTH); + ref_div = readl_relaxed(ref_div_addr) >> MSSPLL_REFDIV_SHIFT; + ref_div &= clk_div_mask(MSSPLL_REFDIV_WIDTH); + + rate_before_ctrl = rate * (ref_div * MSSPLL_FIXED_DIV) / mult; + + return divider_round_rate(hw, rate_before_ctrl, prate, NULL, MSSPLL_POSTDIV_WIDTH, + msspll_hw->flags); +} + +static int mpfs_clk_msspll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long prate) +{ + struct mpfs_msspll_hw_clock *msspll_hw = to_mpfs_msspll_clk(hw); + void __iomem *mult_addr = msspll_hw->base + msspll_hw->reg_offset; + void __iomem *ref_div_addr = msspll_hw->base + REG_MSSPLL_REF_CR; + void __iomem *postdiv_addr = msspll_hw->base + REG_MSSPLL_POSTDIV_CR; + u32 mult, ref_div, postdiv; + int divider_setting; + unsigned long rate_before_ctrl, flags; + + mult = readl_relaxed(mult_addr) >> MSSPLL_FBDIV_SHIFT; + mult &= clk_div_mask(MSSPLL_FBDIV_WIDTH); + ref_div = readl_relaxed(ref_div_addr) >> MSSPLL_REFDIV_SHIFT; + ref_div &= clk_div_mask(MSSPLL_REFDIV_WIDTH); + + rate_before_ctrl = rate * (ref_div * MSSPLL_FIXED_DIV) / mult; + divider_setting = divider_get_val(rate_before_ctrl, prate, NULL, MSSPLL_POSTDIV_WIDTH, + msspll_hw->flags); + + if (divider_setting < 0) + return divider_setting; + + spin_lock_irqsave(&mpfs_clk_lock, flags); + + postdiv = readl_relaxed(postdiv_addr); + postdiv &= ~(clk_div_mask(MSSPLL_POSTDIV_WIDTH) << MSSPLL_POSTDIV_SHIFT); + writel_relaxed(postdiv, postdiv_addr); + + spin_unlock_irqrestore(&mpfs_clk_lock, flags); + + return 0; +} + static const struct clk_ops mpfs_clk_msspll_ops = { .recalc_rate = mpfs_clk_msspll_recalc_rate, + .round_rate = mpfs_clk_msspll_round_rate, + .set_rate = mpfs_clk_msspll_set_rate, }; #define CLK_PLL(_id, _name, _parent, _shift, _width, _flags, _offset) { \ @@ -144,25 +183,17 @@ static struct mpfs_msspll_hw_clock mpfs_msspll_clks[] = { MSSPLL_FBDIV_WIDTH, 0, REG_MSSPLL_SSCG_2_CR), }; -static int mpfs_clk_register_msspll(struct device *dev, struct mpfs_msspll_hw_clock *msspll_hw, - void __iomem *base) -{ - msspll_hw->base = base; - - return devm_clk_hw_register(dev, &msspll_hw->hw); -} - static int mpfs_clk_register_mssplls(struct device *dev, struct mpfs_msspll_hw_clock *msspll_hws, unsigned int num_clks, struct mpfs_clock_data *data) { - void __iomem *base = data->msspll_base; unsigned int i; int ret; for (i = 0; i < num_clks; i++) { struct mpfs_msspll_hw_clock *msspll_hw = &msspll_hws[i]; - ret = mpfs_clk_register_msspll(dev, msspll_hw, base); + msspll_hw->base = data->msspll_base; + ret = devm_clk_hw_register(dev, &msspll_hw->hw); if (ret) return dev_err_probe(dev, ret, "failed to register msspll id: %d\n", CLK_MSSPLL); @@ -177,68 +208,22 @@ static int mpfs_clk_register_mssplls(struct device *dev, struct mpfs_msspll_hw_c * "CFG" clocks */ -static unsigned long mpfs_cfg_clk_recalc_rate(struct clk_hw *hw, unsigned long prate) -{ - struct mpfs_cfg_hw_clock *cfg_hw = to_mpfs_cfg_clk(hw); - struct mpfs_cfg_clock *cfg = &cfg_hw->cfg; - void __iomem *base_addr = cfg_hw->sys_base; - u32 val; - - val = readl_relaxed(base_addr + cfg->reg_offset) >> cfg->shift; - val &= clk_div_mask(cfg->width); - - return divider_recalc_rate(hw, prate, val, cfg->table, cfg->flags, cfg->width); -} - -static long mpfs_cfg_clk_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *prate) -{ - struct mpfs_cfg_hw_clock *cfg_hw = to_mpfs_cfg_clk(hw); - struct mpfs_cfg_clock *cfg = &cfg_hw->cfg; - - return divider_round_rate(hw, rate, prate, cfg->table, cfg->width, 0); -} - -static int mpfs_cfg_clk_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long prate) -{ - struct mpfs_cfg_hw_clock *cfg_hw = to_mpfs_cfg_clk(hw); - struct mpfs_cfg_clock *cfg = &cfg_hw->cfg; - void __iomem *base_addr = cfg_hw->sys_base; - unsigned long flags; - u32 val; - int divider_setting; - - divider_setting = divider_get_val(rate, prate, cfg->table, cfg->width, 0); - - if (divider_setting < 0) - return divider_setting; - - spin_lock_irqsave(&mpfs_clk_lock, flags); - val = readl_relaxed(base_addr + cfg->reg_offset); - val &= ~(clk_div_mask(cfg->width) << cfg_hw->cfg.shift); - val |= divider_setting << cfg->shift; - writel_relaxed(val, base_addr + cfg->reg_offset); - - spin_unlock_irqrestore(&mpfs_clk_lock, flags); - - return 0; -} - -static const struct clk_ops mpfs_clk_cfg_ops = { - .recalc_rate = mpfs_cfg_clk_recalc_rate, - .round_rate = mpfs_cfg_clk_round_rate, - .set_rate = mpfs_cfg_clk_set_rate, -}; - #define CLK_CFG(_id, _name, _parent, _shift, _width, _table, _flags, _offset) { \ - .cfg.id = _id, \ + .id = _id, \ .cfg.shift = _shift, \ .cfg.width = _width, \ .cfg.table = _table, \ - .cfg.reg_offset = _offset, \ + .reg_offset = _offset, \ .cfg.flags = _flags, \ - .hw.init = CLK_HW_INIT(_name, _parent, &mpfs_clk_cfg_ops, 0), \ + .cfg.hw.init = CLK_HW_INIT(_name, _parent, &clk_divider_ops, 0), \ + .cfg.lock = &mpfs_clk_lock, \ } +#define CLK_CPU_OFFSET 0u +#define CLK_AXI_OFFSET 1u +#define CLK_AHB_OFFSET 2u +#define CLK_RTCREF_OFFSET 3u + static struct mpfs_cfg_hw_clock mpfs_cfg_clks[] = { CLK_CFG(CLK_CPU, "clk_cpu", "clk_msspll", 0, 2, mpfs_div_cpu_axi_table, 0, REG_CLOCK_CONFIG_CR), @@ -247,42 +232,34 @@ static struct mpfs_cfg_hw_clock mpfs_cfg_clks[] = { CLK_CFG(CLK_AHB, "clk_ahb", "clk_msspll", 4, 2, mpfs_div_ahb_table, 0, REG_CLOCK_CONFIG_CR), { - .cfg.id = CLK_RTCREF, + .id = CLK_RTCREF, .cfg.shift = 0, .cfg.width = 12, .cfg.table = mpfs_div_rtcref_table, - .cfg.reg_offset = REG_RTC_CLOCK_CR, + .reg_offset = REG_RTC_CLOCK_CR, .cfg.flags = CLK_DIVIDER_ONE_BASED, - .hw.init = - CLK_HW_INIT_PARENTS_DATA("clk_rtcref", mpfs_ext_ref, &mpfs_clk_cfg_ops, 0), + .cfg.hw.init = + CLK_HW_INIT_PARENTS_DATA("clk_rtcref", mpfs_ext_ref, &clk_divider_ops, 0), } }; -static int mpfs_clk_register_cfg(struct device *dev, struct mpfs_cfg_hw_clock *cfg_hw, - void __iomem *sys_base) -{ - cfg_hw->sys_base = sys_base; - - return devm_clk_hw_register(dev, &cfg_hw->hw); -} - static int mpfs_clk_register_cfgs(struct device *dev, struct mpfs_cfg_hw_clock *cfg_hws, unsigned int num_clks, struct mpfs_clock_data *data) { - void __iomem *sys_base = data->base; unsigned int i, id; int ret; for (i = 0; i < num_clks; i++) { struct mpfs_cfg_hw_clock *cfg_hw = &cfg_hws[i]; - ret = mpfs_clk_register_cfg(dev, cfg_hw, sys_base); + cfg_hw->cfg.reg = data->base + cfg_hw->reg_offset; + ret = devm_clk_hw_register(dev, &cfg_hw->cfg.hw); if (ret) return dev_err_probe(dev, ret, "failed to register clock id: %d\n", - cfg_hw->cfg.id); + cfg_hw->id); - id = cfg_hw->cfg.id; - data->hw_data.hws[id] = &cfg_hw->hw; + id = cfg_hw->id; + data->hw_data.hws[id] = &cfg_hw->cfg.hw; } return 0; @@ -292,77 +269,15 @@ static int mpfs_clk_register_cfgs(struct device *dev, struct mpfs_cfg_hw_clock * * peripheral clocks - devices connected to axi or ahb buses. */ -static int mpfs_periph_clk_enable(struct clk_hw *hw) -{ - struct mpfs_periph_hw_clock *periph_hw = to_mpfs_periph_clk(hw); - struct mpfs_periph_clock *periph = &periph_hw->periph; - void __iomem *base_addr = periph_hw->sys_base; - u32 reg, val; - unsigned long flags; - - spin_lock_irqsave(&mpfs_clk_lock, flags); - - reg = readl_relaxed(base_addr + REG_SUBBLK_RESET_CR); - val = reg & ~(1u << periph->shift); - writel_relaxed(val, base_addr + REG_SUBBLK_RESET_CR); - - reg = readl_relaxed(base_addr + REG_SUBBLK_CLOCK_CR); - val = reg | (1u << periph->shift); - writel_relaxed(val, base_addr + REG_SUBBLK_CLOCK_CR); - - spin_unlock_irqrestore(&mpfs_clk_lock, flags); - - return 0; -} - -static void mpfs_periph_clk_disable(struct clk_hw *hw) -{ - struct mpfs_periph_hw_clock *periph_hw = to_mpfs_periph_clk(hw); - struct mpfs_periph_clock *periph = &periph_hw->periph; - void __iomem *base_addr = periph_hw->sys_base; - u32 reg, val; - unsigned long flags; - - spin_lock_irqsave(&mpfs_clk_lock, flags); - - reg = readl_relaxed(base_addr + REG_SUBBLK_CLOCK_CR); - val = reg & ~(1u << periph->shift); - writel_relaxed(val, base_addr + REG_SUBBLK_CLOCK_CR); - - spin_unlock_irqrestore(&mpfs_clk_lock, flags); -} - -static int mpfs_periph_clk_is_enabled(struct clk_hw *hw) -{ - struct mpfs_periph_hw_clock *periph_hw = to_mpfs_periph_clk(hw); - struct mpfs_periph_clock *periph = &periph_hw->periph; - void __iomem *base_addr = periph_hw->sys_base; - u32 reg; - - reg = readl_relaxed(base_addr + REG_SUBBLK_RESET_CR); - if ((reg & (1u << periph->shift)) == 0u) { - reg = readl_relaxed(base_addr + REG_SUBBLK_CLOCK_CR); - if (reg & (1u << periph->shift)) - return 1; - } - - return 0; -} - -static const struct clk_ops mpfs_periph_clk_ops = { - .enable = mpfs_periph_clk_enable, - .disable = mpfs_periph_clk_disable, - .is_enabled = mpfs_periph_clk_is_enabled, -}; - #define CLK_PERIPH(_id, _name, _parent, _shift, _flags) { \ - .periph.id = _id, \ - .periph.shift = _shift, \ - .hw.init = CLK_HW_INIT_HW(_name, _parent, &mpfs_periph_clk_ops, \ + .id = _id, \ + .periph.bit_idx = _shift, \ + .periph.hw.init = CLK_HW_INIT_HW(_name, _parent, &clk_gate_ops, \ _flags), \ + .periph.lock = &mpfs_clk_lock, \ } -#define PARENT_CLK(PARENT) (&mpfs_cfg_clks[CLK_##PARENT].hw) +#define PARENT_CLK(PARENT) (&mpfs_cfg_clks[CLK_##PARENT##_OFFSET].cfg.hw) /* * Critical clocks: @@ -370,6 +285,8 @@ static const struct clk_ops mpfs_periph_clk_ops = { * trap handler * - CLK_MMUART0: reserved by the hss * - CLK_DDRC: provides clock to the ddr subsystem + * - CLK_RTC: the onboard RTC's AHB bus clock must be kept running as the rtc will stop + * if the AHB interface clock is disabled * - CLK_FICx: these provide the processor side clocks to the "FIC" (Fabric InterConnect) * clock domain crossers which provide the interface to the FPGA fabric. Disabling them * causes the FPGA fabric to go into reset. @@ -394,7 +311,7 @@ static struct mpfs_periph_hw_clock mpfs_periph_clks[] = { CLK_PERIPH(CLK_CAN0, "clk_periph_can0", PARENT_CLK(AHB), 14, 0), CLK_PERIPH(CLK_CAN1, "clk_periph_can1", PARENT_CLK(AHB), 15, 0), CLK_PERIPH(CLK_USB, "clk_periph_usb", PARENT_CLK(AHB), 16, 0), - CLK_PERIPH(CLK_RTC, "clk_periph_rtc", PARENT_CLK(AHB), 18, 0), + CLK_PERIPH(CLK_RTC, "clk_periph_rtc", PARENT_CLK(AHB), 18, CLK_IS_CRITICAL), CLK_PERIPH(CLK_QSPI, "clk_periph_qspi", PARENT_CLK(AHB), 19, 0), CLK_PERIPH(CLK_GPIO0, "clk_periph_gpio0", PARENT_CLK(AHB), 20, 0), CLK_PERIPH(CLK_GPIO1, "clk_periph_gpio1", PARENT_CLK(AHB), 21, 0), @@ -408,36 +325,116 @@ static struct mpfs_periph_hw_clock mpfs_periph_clks[] = { CLK_PERIPH(CLK_CFM, "clk_periph_cfm", PARENT_CLK(AHB), 29, 0), }; -static int mpfs_clk_register_periph(struct device *dev, struct mpfs_periph_hw_clock *periph_hw, - void __iomem *sys_base) -{ - periph_hw->sys_base = sys_base; - - return devm_clk_hw_register(dev, &periph_hw->hw); -} - static int mpfs_clk_register_periphs(struct device *dev, struct mpfs_periph_hw_clock *periph_hws, int num_clks, struct mpfs_clock_data *data) { - void __iomem *sys_base = data->base; unsigned int i, id; int ret; for (i = 0; i < num_clks; i++) { struct mpfs_periph_hw_clock *periph_hw = &periph_hws[i]; - ret = mpfs_clk_register_periph(dev, periph_hw, sys_base); + periph_hw->periph.reg = data->base + REG_SUBBLK_CLOCK_CR; + ret = devm_clk_hw_register(dev, &periph_hw->periph.hw); if (ret) return dev_err_probe(dev, ret, "failed to register clock id: %d\n", - periph_hw->periph.id); + periph_hw->id); - id = periph_hws[i].periph.id; - data->hw_data.hws[id] = &periph_hw->hw; + id = periph_hws[i].id; + data->hw_data.hws[id] = &periph_hw->periph.hw; } return 0; } +/* + * Peripheral clock resets + */ + +#if IS_ENABLED(CONFIG_RESET_CONTROLLER) + +u32 mpfs_reset_read(struct device *dev) +{ + struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); + + return readl_relaxed(clock_data->base + REG_SUBBLK_RESET_CR); +} +EXPORT_SYMBOL_NS_GPL(mpfs_reset_read, MCHP_CLK_MPFS); + +void mpfs_reset_write(struct device *dev, u32 val) +{ + struct mpfs_clock_data *clock_data = dev_get_drvdata(dev->parent); + + writel_relaxed(val, clock_data->base + REG_SUBBLK_RESET_CR); +} +EXPORT_SYMBOL_NS_GPL(mpfs_reset_write, MCHP_CLK_MPFS); + +static void mpfs_reset_unregister_adev(void *_adev) +{ + struct auxiliary_device *adev = _adev; + + auxiliary_device_delete(adev); +} + +static void mpfs_reset_adev_release(struct device *dev) +{ + struct auxiliary_device *adev = to_auxiliary_dev(dev); + + auxiliary_device_uninit(adev); + + kfree(adev); +} + +static struct auxiliary_device *mpfs_reset_adev_alloc(struct mpfs_clock_data *clk_data) +{ + struct auxiliary_device *adev; + int ret; + + adev = kzalloc(sizeof(*adev), GFP_KERNEL); + if (!adev) + return ERR_PTR(-ENOMEM); + + adev->name = "reset-mpfs"; + adev->dev.parent = clk_data->dev; + adev->dev.release = mpfs_reset_adev_release; + adev->id = 666u; + + ret = auxiliary_device_init(adev); + if (ret) { + kfree(adev); + return ERR_PTR(ret); + } + + return adev; +} + +static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) +{ + struct auxiliary_device *adev; + int ret; + + adev = mpfs_reset_adev_alloc(clk_data); + if (IS_ERR(adev)) + return PTR_ERR(adev); + + ret = auxiliary_device_add(adev); + if (ret) { + auxiliary_device_uninit(adev); + return ret; + } + + return devm_add_action_or_reset(clk_data->dev, mpfs_reset_unregister_adev, adev); +} + +#else /* !CONFIG_RESET_CONTROLLER */ + +static int mpfs_reset_controller_register(struct mpfs_clock_data *clk_data) +{ + return 0; +} + +#endif /* !CONFIG_RESET_CONTROLLER */ + static int mpfs_clk_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -462,6 +459,8 @@ static int mpfs_clk_probe(struct platform_device *pdev) return PTR_ERR(clk_data->msspll_base); clk_data->hw_data.num = num_clks; + clk_data->dev = dev; + dev_set_drvdata(dev, clk_data); ret = mpfs_clk_register_mssplls(dev, mpfs_msspll_clks, ARRAY_SIZE(mpfs_msspll_clks), clk_data); @@ -481,14 +480,14 @@ static int mpfs_clk_probe(struct platform_device *pdev) if (ret) return ret; - return ret; + return mpfs_reset_controller_register(clk_data); } static const struct of_device_id mpfs_clk_of_match_table[] = { { .compatible = "microchip,mpfs-clkcfg", }, {} }; -MODULE_DEVICE_TABLE(of, mpfs_clk_match_table); +MODULE_DEVICE_TABLE(of, mpfs_clk_of_match_table); static struct platform_driver mpfs_clk_driver = { .probe = mpfs_clk_probe, @@ -511,4 +510,7 @@ static void __exit clk_mpfs_exit(void) module_exit(clk_mpfs_exit); MODULE_DESCRIPTION("Microchip PolarFire SoC Clock Driver"); -MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Padmarao Begari "); +MODULE_AUTHOR("Daire McNamara "); +MODULE_AUTHOR("Conor Dooley "); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/mmp/clk-of-pxa168.c b/drivers/clk/mmp/clk-of-pxa168.c index 48dfb18b490e72e02a4da561ed0ef17568b59989..130d1a72387970b45684ea852155145e65737a81 100644 --- a/drivers/clk/mmp/clk-of-pxa168.c +++ b/drivers/clk/mmp/clk-of-pxa168.c @@ -19,9 +19,6 @@ #include "clk.h" #include "reset.h" -#define APBC_RTC 0x28 -#define APBC_TWSI0 0x2c -#define APBC_KPC 0x30 #define APBC_UART0 0x0 #define APBC_UART1 0x4 #define APBC_GPIO 0x8 @@ -29,20 +26,40 @@ #define APBC_PWM1 0x10 #define APBC_PWM2 0x14 #define APBC_PWM3 0x18 +#define APBC_RTC 0x28 +#define APBC_TWSI0 0x2c +#define APBC_KPC 0x30 #define APBC_TIMER 0x34 +#define APBC_AIB 0x3c +#define APBC_SW_JTAG 0x40 +#define APBC_ONEWIRE 0x48 +#define APBC_TWSI1 0x6c +#define APBC_UART2 0x70 +#define APBC_AC97 0x84 #define APBC_SSP0 0x81c #define APBC_SSP1 0x820 #define APBC_SSP2 0x84c #define APBC_SSP3 0x858 #define APBC_SSP4 0x85c -#define APBC_TWSI1 0x6c -#define APBC_UART2 0x70 +#define APMU_DISP0 0x4c +#define APMU_CCIC0 0x50 #define APMU_SDH0 0x54 #define APMU_SDH1 0x58 #define APMU_USB 0x5c -#define APMU_DISP0 0x4c -#define APMU_CCIC0 0x50 #define APMU_DFC 0x60 +#define APMU_DMA 0x64 +#define APMU_BUS 0x6c +#define APMU_GC 0xcc +#define APMU_SMC 0xd4 +#define APMU_XD 0xdc +#define APMU_SDH2 0xe0 +#define APMU_SDH3 0xe4 +#define APMU_CF 0xf0 +#define APMU_MSP 0xf4 +#define APMU_CMU 0xf8 +#define APMU_FE 0xfc +#define APMU_PCIE 0x100 +#define APMU_EPD 0x104 #define MPMU_UART_PLL 0x14 struct pxa168_clk_unit { @@ -71,9 +88,12 @@ static struct mmp_param_fixed_factor_clk fixed_factor_clks[] = { {PXA168_CLK_PLL1_96, "pll1_96", "pll1_48", 1, 2, 0}, {PXA168_CLK_PLL1_192, "pll1_192", "pll1_96", 1, 2, 0}, {PXA168_CLK_PLL1_13, "pll1_13", "pll1", 1, 13, 0}, - {PXA168_CLK_PLL1_13_1_5, "pll1_13_1_5", "pll1_13", 2, 3, 0}, - {PXA168_CLK_PLL1_2_1_5, "pll1_2_1_5", "pll1_2", 2, 3, 0}, + {PXA168_CLK_PLL1_13_1_5, "pll1_13_1_5", "pll1_13", 1, 5, 0}, + {PXA168_CLK_PLL1_2_1_5, "pll1_2_1_5", "pll1_2", 1, 5, 0}, {PXA168_CLK_PLL1_3_16, "pll1_3_16", "pll1", 3, 16, 0}, + {PXA168_CLK_PLL1_2_1_10, "pll1_2_1_10", "pll1_2", 1, 10, 0}, + {PXA168_CLK_PLL1_2_3_16, "pll1_2_3_16", "pll1_2", 3, 16, 0}, + {PXA168_CLK_CLK32_2, "clk32_2", "clk32", 1, 2, 0}, }; static struct mmp_clk_factor_masks uart_factor_masks = { @@ -107,24 +127,44 @@ static void pxa168_pll_init(struct pxa168_clk_unit *pxa_unit) mmp_clk_add(unit, PXA168_CLK_UART_PLL, clk); } +static DEFINE_SPINLOCK(twsi0_lock); +static DEFINE_SPINLOCK(twsi1_lock); +static const char * const twsi_parent_names[] = {"pll1_2_1_10", "pll1_2_1_5"}; + +static DEFINE_SPINLOCK(kpc_lock); +static const char * const kpc_parent_names[] = {"clk32", "clk32_2", "pll1_24"}; + +static DEFINE_SPINLOCK(pwm0_lock); +static DEFINE_SPINLOCK(pwm1_lock); +static DEFINE_SPINLOCK(pwm2_lock); +static DEFINE_SPINLOCK(pwm3_lock); +static const char * const pwm_parent_names[] = {"pll1_48", "clk32"}; + static DEFINE_SPINLOCK(uart0_lock); static DEFINE_SPINLOCK(uart1_lock); static DEFINE_SPINLOCK(uart2_lock); -static const char *uart_parent_names[] = {"pll1_3_16", "uart_pll"}; +static const char * const uart_parent_names[] = {"pll1_2_3_16", "uart_pll"}; static DEFINE_SPINLOCK(ssp0_lock); static DEFINE_SPINLOCK(ssp1_lock); static DEFINE_SPINLOCK(ssp2_lock); static DEFINE_SPINLOCK(ssp3_lock); static DEFINE_SPINLOCK(ssp4_lock); -static const char *ssp_parent_names[] = {"pll1_96", "pll1_48", "pll1_24", "pll1_12"}; +static const char * const ssp_parent_names[] = {"pll1_96", "pll1_48", "pll1_24", "pll1_12"}; static DEFINE_SPINLOCK(timer_lock); -static const char *timer_parent_names[] = {"pll1_48", "clk32", "pll1_96", "pll1_192"}; +static const char * const timer_parent_names[] = {"pll1_48", "clk32", "pll1_96", "pll1_192"}; static DEFINE_SPINLOCK(reset_lock); static struct mmp_param_mux_clk apbc_mux_clks[] = { + {0, "twsi0_mux", twsi_parent_names, ARRAY_SIZE(twsi_parent_names), CLK_SET_RATE_PARENT, APBC_TWSI0, 4, 3, 0, &twsi0_lock}, + {0, "twsi1_mux", twsi_parent_names, ARRAY_SIZE(twsi_parent_names), CLK_SET_RATE_PARENT, APBC_TWSI1, 4, 3, 0, &twsi1_lock}, + {0, "kpc_mux", kpc_parent_names, ARRAY_SIZE(kpc_parent_names), CLK_SET_RATE_PARENT, APBC_KPC, 4, 3, 0, &kpc_lock}, + {0, "pwm0_mux", pwm_parent_names, ARRAY_SIZE(pwm_parent_names), CLK_SET_RATE_PARENT, APBC_PWM0, 4, 3, 0, &pwm0_lock}, + {0, "pwm1_mux", pwm_parent_names, ARRAY_SIZE(pwm_parent_names), CLK_SET_RATE_PARENT, APBC_PWM1, 4, 3, 0, &pwm1_lock}, + {0, "pwm2_mux", pwm_parent_names, ARRAY_SIZE(pwm_parent_names), CLK_SET_RATE_PARENT, APBC_PWM2, 4, 3, 0, &pwm2_lock}, + {0, "pwm3_mux", pwm_parent_names, ARRAY_SIZE(pwm_parent_names), CLK_SET_RATE_PARENT, APBC_PWM3, 4, 3, 0, &pwm3_lock}, {0, "uart0_mux", uart_parent_names, ARRAY_SIZE(uart_parent_names), CLK_SET_RATE_PARENT, APBC_UART0, 4, 3, 0, &uart0_lock}, {0, "uart1_mux", uart_parent_names, ARRAY_SIZE(uart_parent_names), CLK_SET_RATE_PARENT, APBC_UART1, 4, 3, 0, &uart1_lock}, {0, "uart2_mux", uart_parent_names, ARRAY_SIZE(uart_parent_names), CLK_SET_RATE_PARENT, APBC_UART2, 4, 3, 0, &uart2_lock}, @@ -137,16 +177,15 @@ static struct mmp_param_mux_clk apbc_mux_clks[] = { }; static struct mmp_param_gate_clk apbc_gate_clks[] = { - {PXA168_CLK_TWSI0, "twsi0_clk", "pll1_13_1_5", CLK_SET_RATE_PARENT, APBC_TWSI0, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_TWSI1, "twsi1_clk", "pll1_13_1_5", CLK_SET_RATE_PARENT, APBC_TWSI1, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_GPIO, "gpio_clk", "vctcxo", CLK_SET_RATE_PARENT, APBC_GPIO, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_KPC, "kpc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_KPC, 0x3, 0x3, 0x0, MMP_CLK_GATE_NEED_DELAY, NULL}, + {PXA168_CLK_TWSI0, "twsi0_clk", "twsi0_mux", CLK_SET_RATE_PARENT, APBC_TWSI0, 0x3, 0x3, 0x0, 0, &twsi0_lock}, + {PXA168_CLK_TWSI1, "twsi1_clk", "twsi1_mux", CLK_SET_RATE_PARENT, APBC_TWSI1, 0x3, 0x3, 0x0, 0, &twsi1_lock}, + {PXA168_CLK_GPIO, "gpio_clk", "vctcxo", CLK_SET_RATE_PARENT, APBC_GPIO, 0x1, 0x1, 0x0, 0, &reset_lock}, + {PXA168_CLK_KPC, "kpc_clk", "kpc_mux", CLK_SET_RATE_PARENT, APBC_KPC, 0x3, 0x3, 0x0, MMP_CLK_GATE_NEED_DELAY, &kpc_lock}, {PXA168_CLK_RTC, "rtc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_RTC, 0x83, 0x83, 0x0, MMP_CLK_GATE_NEED_DELAY, NULL}, - {PXA168_CLK_PWM0, "pwm0_clk", "pll1_48", CLK_SET_RATE_PARENT, APBC_PWM0, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_PWM1, "pwm1_clk", "pll1_48", CLK_SET_RATE_PARENT, APBC_PWM1, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_PWM2, "pwm2_clk", "pll1_48", CLK_SET_RATE_PARENT, APBC_PWM2, 0x3, 0x3, 0x0, 0, &reset_lock}, - {PXA168_CLK_PWM3, "pwm3_clk", "pll1_48", CLK_SET_RATE_PARENT, APBC_PWM3, 0x3, 0x3, 0x0, 0, &reset_lock}, - /* The gate clocks has mux parent. */ + {PXA168_CLK_PWM0, "pwm0_clk", "pwm0_mux", CLK_SET_RATE_PARENT, APBC_PWM0, 0x3, 0x3, 0x0, 0, &pwm0_lock}, + {PXA168_CLK_PWM1, "pwm1_clk", "pwm1_mux", CLK_SET_RATE_PARENT, APBC_PWM1, 0x3, 0x3, 0x0, 0, &pwm1_lock}, + {PXA168_CLK_PWM2, "pwm2_clk", "pwm2_mux", CLK_SET_RATE_PARENT, APBC_PWM2, 0x3, 0x3, 0x0, 0, &pwm2_lock}, + {PXA168_CLK_PWM3, "pwm3_clk", "pwm3_mux", CLK_SET_RATE_PARENT, APBC_PWM3, 0x3, 0x3, 0x0, 0, &pwm3_lock}, {PXA168_CLK_UART0, "uart0_clk", "uart0_mux", CLK_SET_RATE_PARENT, APBC_UART0, 0x3, 0x3, 0x0, 0, &uart0_lock}, {PXA168_CLK_UART1, "uart1_clk", "uart1_mux", CLK_SET_RATE_PARENT, APBC_UART1, 0x3, 0x3, 0x0, 0, &uart1_lock}, {PXA168_CLK_UART2, "uart2_clk", "uart2_mux", CLK_SET_RATE_PARENT, APBC_UART2, 0x3, 0x3, 0x0, 0, &uart2_lock}, @@ -170,22 +209,30 @@ static void pxa168_apb_periph_clk_init(struct pxa168_clk_unit *pxa_unit) } +static DEFINE_SPINLOCK(dfc_lock); +static const char * const dfc_parent_names[] = {"pll1_4", "pll1_8"}; + static DEFINE_SPINLOCK(sdh0_lock); static DEFINE_SPINLOCK(sdh1_lock); -static const char *sdh_parent_names[] = {"pll1_12", "pll1_13"}; +static DEFINE_SPINLOCK(sdh2_lock); +static DEFINE_SPINLOCK(sdh3_lock); +static const char * const sdh_parent_names[] = {"pll1_13", "pll1_12", "pll1_8"}; static DEFINE_SPINLOCK(usb_lock); static DEFINE_SPINLOCK(disp0_lock); -static const char *disp_parent_names[] = {"pll1_2", "pll1_12"}; +static const char * const disp_parent_names[] = {"pll1", "pll1_2"}; static DEFINE_SPINLOCK(ccic0_lock); -static const char *ccic_parent_names[] = {"pll1_2", "pll1_12"}; -static const char *ccic_phy_parent_names[] = {"pll1_6", "pll1_12"}; +static const char * const ccic_parent_names[] = {"pll1_4", "pll1_8"}; +static const char * const ccic_phy_parent_names[] = {"pll1_6", "pll1_12"}; static struct mmp_param_mux_clk apmu_mux_clks[] = { - {0, "sdh0_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH0, 6, 1, 0, &sdh0_lock}, - {0, "sdh1_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH1, 6, 1, 0, &sdh1_lock}, + {0, "dfc_mux", dfc_parent_names, ARRAY_SIZE(dfc_parent_names), CLK_SET_RATE_PARENT, APMU_DFC, 6, 1, 0, &dfc_lock}, + {0, "sdh0_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH0, 6, 2, 0, &sdh0_lock}, + {0, "sdh1_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH1, 6, 2, 0, &sdh1_lock}, + {0, "sdh2_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH2, 6, 2, 0, &sdh2_lock}, + {0, "sdh3_mux", sdh_parent_names, ARRAY_SIZE(sdh_parent_names), CLK_SET_RATE_PARENT, APMU_SDH3, 6, 2, 0, &sdh3_lock}, {0, "disp0_mux", disp_parent_names, ARRAY_SIZE(disp_parent_names), CLK_SET_RATE_PARENT, APMU_DISP0, 6, 1, 0, &disp0_lock}, {0, "ccic0_mux", ccic_parent_names, ARRAY_SIZE(ccic_parent_names), CLK_SET_RATE_PARENT, APMU_CCIC0, 6, 1, 0, &ccic0_lock}, {0, "ccic0_phy_mux", ccic_phy_parent_names, ARRAY_SIZE(ccic_phy_parent_names), CLK_SET_RATE_PARENT, APMU_CCIC0, 7, 1, 0, &ccic0_lock}, @@ -196,12 +243,16 @@ static struct mmp_param_div_clk apmu_div_clks[] = { }; static struct mmp_param_gate_clk apmu_gate_clks[] = { - {PXA168_CLK_DFC, "dfc_clk", "pll1_4", CLK_SET_RATE_PARENT, APMU_DFC, 0x19b, 0x19b, 0x0, 0, NULL}, + {PXA168_CLK_DFC, "dfc_clk", "dfc_mux", CLK_SET_RATE_PARENT, APMU_DFC, 0x19b, 0x19b, 0x0, 0, &dfc_lock}, {PXA168_CLK_USB, "usb_clk", "usb_pll", 0, APMU_USB, 0x9, 0x9, 0x0, 0, &usb_lock}, {PXA168_CLK_SPH, "sph_clk", "usb_pll", 0, APMU_USB, 0x12, 0x12, 0x0, 0, &usb_lock}, - /* The gate clocks has mux parent. */ - {PXA168_CLK_SDH0, "sdh0_clk", "sdh0_mux", CLK_SET_RATE_PARENT, APMU_SDH0, 0x1b, 0x1b, 0x0, 0, &sdh0_lock}, - {PXA168_CLK_SDH1, "sdh1_clk", "sdh1_mux", CLK_SET_RATE_PARENT, APMU_SDH1, 0x1b, 0x1b, 0x0, 0, &sdh1_lock}, + {PXA168_CLK_SDH0, "sdh0_clk", "sdh0_mux", CLK_SET_RATE_PARENT, APMU_SDH0, 0x12, 0x12, 0x0, 0, &sdh0_lock}, + {PXA168_CLK_SDH1, "sdh1_clk", "sdh1_mux", CLK_SET_RATE_PARENT, APMU_SDH1, 0x12, 0x12, 0x0, 0, &sdh1_lock}, + {PXA168_CLK_SDH2, "sdh2_clk", "sdh2_mux", CLK_SET_RATE_PARENT, APMU_SDH2, 0x12, 0x12, 0x0, 0, &sdh2_lock}, + {PXA168_CLK_SDH3, "sdh3_clk", "sdh3_mux", CLK_SET_RATE_PARENT, APMU_SDH3, 0x12, 0x12, 0x0, 0, &sdh3_lock}, + /* SDH0/1 and 2/3 AXI clocks are also gated by common bits in SDH0 and SDH2 registers */ + {PXA168_CLK_SDH01_AXI, "sdh01_axi_clk", NULL, CLK_SET_RATE_PARENT, APMU_SDH0, 0x9, 0x9, 0x0, 0, &sdh0_lock}, + {PXA168_CLK_SDH23_AXI, "sdh23_axi_clk", NULL, CLK_SET_RATE_PARENT, APMU_SDH2, 0x9, 0x9, 0x0, 0, &sdh2_lock}, {PXA168_CLK_DISP0, "disp0_clk", "disp0_mux", CLK_SET_RATE_PARENT, APMU_DISP0, 0x1b, 0x1b, 0x0, 0, &disp0_lock}, {PXA168_CLK_CCIC0, "ccic0_clk", "ccic0_mux", CLK_SET_RATE_PARENT, APMU_CCIC0, 0x1b, 0x1b, 0x0, 0, &ccic0_lock}, {PXA168_CLK_CCIC0_PHY, "ccic0_phy_clk", "ccic0_phy_mux", CLK_SET_RATE_PARENT, APMU_CCIC0, 0x24, 0x24, 0x0, 0, &ccic0_lock}, diff --git a/drivers/clk/mvebu/armada-37xx-tbg.c b/drivers/clk/mvebu/armada-37xx-tbg.c index 585a02e0b330f2a447ebf5a31e86fedeeabdd2d3..fc403ad735adeff8a07a21fe0fa00f567521848f 100644 --- a/drivers/clk/mvebu/armada-37xx-tbg.c +++ b/drivers/clk/mvebu/armada-37xx-tbg.c @@ -87,7 +87,7 @@ static int armada_3700_tbg_clock_probe(struct platform_device *pdev) struct resource *res; struct clk *parent; void __iomem *reg; - int i, ret; + int i; hw_tbg_data = devm_kzalloc(&pdev->dev, struct_size(hw_tbg_data, hws, NUM_TBG), @@ -123,9 +123,7 @@ static int armada_3700_tbg_clock_probe(struct platform_device *pdev) dev_err(dev, "Can't register TBG clock %s\n", name); } - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, hw_tbg_data); - - return ret; + return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, hw_tbg_data); } static int armada_3700_tbg_clock_remove(struct platform_device *pdev) diff --git a/drivers/clk/mvebu/dove-divider.c b/drivers/clk/mvebu/dove-divider.c index 7e35c891e1684338460d0cba223ce656c4d52c25..0a90452ee808b6bf2f1f9f815624b99720c3a81f 100644 --- a/drivers/clk/mvebu/dove-divider.c +++ b/drivers/clk/mvebu/dove-divider.c @@ -170,7 +170,7 @@ static struct clk *clk_register_dove_divider(struct device *dev, .num_parents = num_parents, }; - strlcpy(name, dc->name, sizeof(name)); + strscpy(name, dc->name, sizeof(name)); dc->hw.init = &init; dc->base = base; diff --git a/drivers/clk/nxp/clk-lpc18xx-cgu.c b/drivers/clk/nxp/clk-lpc18xx-cgu.c index f253ef1996b106fec71488ca942c4a8b325525b0..69ebf65081b81f42f68755ee55a114d88ce14ce0 100644 --- a/drivers/clk/nxp/clk-lpc18xx-cgu.c +++ b/drivers/clk/nxp/clk-lpc18xx-cgu.c @@ -606,7 +606,7 @@ static void __init lpc18xx_cgu_register_source_clks(struct device_node *np, if (IS_ERR(clk)) pr_warn("%s: failed to register irc clk\n", __func__); - /* Register crystal oscillator controlller */ + /* Register crystal oscillator controller */ parents[0] = of_clk_get_parent_name(np, 0); clk = clk_register_gate(NULL, clk_src_names[CLK_SRC_OSC], parents[0], 0, base + LPC18XX_CGU_XTAL_OSC_CTRL, diff --git a/drivers/clk/pistachio/clk.h b/drivers/clk/pistachio/clk.h index f9c31e3a0e47a44e1ff62b62780f7e2f1ab322cb..2f4ffbd98282109b8c3cd6e99ea01d30fa7894dc 100644 --- a/drivers/clk/pistachio/clk.h +++ b/drivers/clk/pistachio/clk.h @@ -31,10 +31,10 @@ struct pistachio_mux { unsigned int shift; unsigned int num_parents; const char *name; - const char **parents; + const char *const *parents; }; -#define PNAME(x) static const char *x[] __initconst +#define PNAME(x) static const char *const x[] __initconst #define MUX(_id, _name, _pnames, _reg, _shift) \ { \ diff --git a/drivers/clk/pxa/clk-pxa.c b/drivers/clk/pxa/clk-pxa.c index 03de634efc5249c6bef7d54f3ca0764c12197758..374098ebbf2b6be139212c904d0ff5adb04cbd7e 100644 --- a/drivers/clk/pxa/clk-pxa.c +++ b/drivers/clk/pxa/clk-pxa.c @@ -104,6 +104,8 @@ int __init clk_pxa_cken_init(const struct desc_clk_cken *clks, for (i = 0; i < nb_clks; i++) { pxa_clk = kzalloc(sizeof(*pxa_clk), GFP_KERNEL); + if (!pxa_clk) + return -ENOMEM; pxa_clk->is_in_low_power = clks[i].is_in_low_power; pxa_clk->lp = clks[i].lp; pxa_clk->hp = clks[i].hp; diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 1cf1ef70e3478c288427bcf26da324b5ce1ef0b2..76e6dee450d59c55e3a82d9501362331c662ef0c 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -180,6 +180,14 @@ config MSM_GCC_8660 Say Y if you want to use peripheral devices such as UART, SPI, i2c, USB, SD/eMMC, etc. +config MSM_GCC_8909 + tristate "MSM8909 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on msm8909 devices. + Say Y if you want to use devices such as UART, SPI, I2C, USB, + SD/eMMC, display, graphics, camera etc. + config MSM_GCC_8916 tristate "MSM8916 Global Clock Controller" select QCOM_GDSC @@ -445,6 +453,14 @@ config SC_GPUCC_7280 Say Y if you want to support graphics controller devices and functionality such as 3D graphics. +config SC_GPUCC_8280XP + tristate "SC8280XP Graphics Clock Controller" + select SC_GCC_8280XP + help + Support for the graphics clock controller on SC8280XP devices. + Say Y if you want to support graphics controller devices and + functionality such as 3D graphics. + config SC_LPASSCC_7280 tristate "SC7280 Low Power Audio Subsystem (LPASS) Clock Controller" select SC_GCC_7280 @@ -545,10 +561,10 @@ config QCS_Q6SSTOP_404 controller to reset the Q6SSTOP subsystem. config SDM_GCC_845 - tristate "SDM845 Global Clock Controller" + tristate "SDM845/SDM670 Global Clock Controller" select QCOM_GDSC help - Support for the global clock controller on SDM845 devices. + Support for the global clock controller on SDM845 and SDM670 devices. Say Y if you want to use peripheral devices such as UART, SPI, i2C, USB, UFS, SDDC, PCIe, etc. @@ -616,6 +632,15 @@ config SM_CAMCC_8450 Support for the camera clock controller on SM8450 devices. Say Y if you want to support camera devices and camera functionality. +config SM_DISPCC_6115 + tristate "SM6115 Display Clock Controller" + depends on SM_GCC_6115 + help + Support for the display clock controller on Qualcomm Technologies, Inc + SM6115/SM4250 devices. + Say Y if you want to support display devices and functionality such as + splash screen + config SM_DISPCC_6125 tristate "SM6125 Display Clock Controller" depends on SM_GCC_6125 @@ -643,8 +668,18 @@ config SM_DISPCC_6350 Say Y if you want to support display devices and functionality such as splash screen. +config SM_DISPCC_8450 + tristate "SM8450 Display Clock Controller" + depends on SM_GCC_8450 + help + Support for the display clock controller on Qualcomm Technologies, Inc + SM8450 devices. + Say Y if you want to support display devices and functionality such as + splash screen. + config SM_GCC_6115 tristate "SM6115 and SM4250 Global Clock Controller" + select QCOM_GDSC help Support for the global clock controller on SM6115 and SM4250 devices. Say Y if you want to use peripheral devices such as UART, SPI, @@ -665,6 +700,14 @@ config SM_GCC_6350 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, SD/UFS, PCIe etc. +config SM_GCC_6375 + tristate "SM6375 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on SM6375 devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS etc. + config SM_GCC_8150 tristate "SM8150 Global Clock Controller" help diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index fbcf04073f07db231c5c3d591c5c7425a0e5052f..e6cecf9e0436a696964e48316d467831765f460e 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_MDM_GCC_9607) += gcc-mdm9607.o obj-$(CONFIG_MDM_GCC_9615) += gcc-mdm9615.o obj-$(CONFIG_MDM_LCC_9615) += lcc-mdm9615.o obj-$(CONFIG_MSM_GCC_8660) += gcc-msm8660.o +obj-$(CONFIG_MSM_GCC_8909) += gcc-msm8909.o obj-$(CONFIG_MSM_GCC_8916) += gcc-msm8916.o obj-$(CONFIG_MSM_GCC_8939) += gcc-msm8939.o obj-$(CONFIG_MSM_GCC_8953) += gcc-msm8953.o @@ -71,6 +72,7 @@ obj-$(CONFIG_SC_GCC_8180X) += gcc-sc8180x.o obj-$(CONFIG_SC_GCC_8280XP) += gcc-sc8280xp.o obj-$(CONFIG_SC_GPUCC_7180) += gpucc-sc7180.o obj-$(CONFIG_SC_GPUCC_7280) += gpucc-sc7280.o +obj-$(CONFIG_SC_GPUCC_8280XP) += gpucc-sc8280xp.o obj-$(CONFIG_SC_LPASSCC_7280) += lpasscc-sc7280.o obj-$(CONFIG_SC_LPASS_CORECC_7180) += lpasscorecc-sc7180.o obj-$(CONFIG_SC_LPASS_CORECC_7280) += lpasscorecc-sc7280.o lpassaudiocc-sc7280.o @@ -90,12 +92,15 @@ obj-$(CONFIG_SDX_GCC_55) += gcc-sdx55.o obj-$(CONFIG_SDX_GCC_65) += gcc-sdx65.o obj-$(CONFIG_SM_CAMCC_8250) += camcc-sm8250.o obj-$(CONFIG_SM_CAMCC_8450) += camcc-sm8450.o +obj-$(CONFIG_SM_DISPCC_6115) += dispcc-sm6115.o obj-$(CONFIG_SM_DISPCC_6125) += dispcc-sm6125.o obj-$(CONFIG_SM_DISPCC_6350) += dispcc-sm6350.o obj-$(CONFIG_SM_DISPCC_8250) += dispcc-sm8250.o +obj-$(CONFIG_SM_DISPCC_8450) += dispcc-sm8450.o obj-$(CONFIG_SM_GCC_6115) += gcc-sm6115.o obj-$(CONFIG_SM_GCC_6125) += gcc-sm6125.o obj-$(CONFIG_SM_GCC_6350) += gcc-sm6350.o +obj-$(CONFIG_SM_GCC_6375) += gcc-sm6375.o obj-$(CONFIG_SM_GCC_8150) += gcc-sm8150.o obj-$(CONFIG_SM_GCC_8250) += gcc-sm8250.o obj-$(CONFIG_SM_GCC_8350) += gcc-sm8350.o diff --git a/drivers/clk/qcom/a53-pll.c b/drivers/clk/qcom/a53-pll.c index 329d2c5356d8b30429b5f80b299a57d55be285f4..f9c5e296dba2a0e16fcdfa6a99cdb483d6db1b72 100644 --- a/drivers/clk/qcom/a53-pll.c +++ b/drivers/clk/qcom/a53-pll.c @@ -127,7 +127,9 @@ static int qcom_a53pll_probe(struct platform_device *pdev) if (!init.name) return -ENOMEM; - init.parent_names = (const char *[]){ "xo" }; + init.parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", .name = "xo_board", + }; init.num_parents = 1; init.ops = &clk_pll_sr2_ops; pll->clkr.hw.init = &init; diff --git a/drivers/clk/qcom/apss-ipq-pll.c b/drivers/clk/qcom/apss-ipq-pll.c index bef7899ad0d660de94d6cd1cfd2639fd2e943ad7..a5aea27eb867bb2e695eafb93ff4693c76425de8 100644 --- a/drivers/clk/qcom/apss-ipq-pll.c +++ b/drivers/clk/qcom/apss-ipq-pll.c @@ -2,6 +2,7 @@ // Copyright (c) 2018, The Linux Foundation. All rights reserved. #include #include +#include #include #include @@ -36,12 +37,28 @@ static struct clk_alpha_pll ipq_pll = { }, }; -static const struct alpha_pll_config ipq_pll_config = { +static const struct alpha_pll_config ipq6018_pll_config = { .l = 0x37, - .config_ctl_val = 0x04141200, - .config_ctl_hi_val = 0x0, + .config_ctl_val = 0x240d4828, + .config_ctl_hi_val = 0x6, .early_output_mask = BIT(3), + .aux2_output_mask = BIT(2), + .aux_output_mask = BIT(1), .main_output_mask = BIT(0), + .test_ctl_val = 0x1c0000C0, + .test_ctl_hi_val = 0x4000, +}; + +static const struct alpha_pll_config ipq8074_pll_config = { + .l = 0x48, + .config_ctl_val = 0x200d4828, + .config_ctl_hi_val = 0x6, + .early_output_mask = BIT(3), + .aux2_output_mask = BIT(2), + .aux_output_mask = BIT(1), + .main_output_mask = BIT(0), + .test_ctl_val = 0x1c000000, + .test_ctl_hi_val = 0x4000, }; static const struct regmap_config ipq_pll_regmap_config = { @@ -54,6 +71,7 @@ static const struct regmap_config ipq_pll_regmap_config = { static int apss_ipq_pll_probe(struct platform_device *pdev) { + const struct alpha_pll_config *ipq_pll_config; struct device *dev = &pdev->dev; struct regmap *regmap; void __iomem *base; @@ -67,7 +85,11 @@ static int apss_ipq_pll_probe(struct platform_device *pdev) if (IS_ERR(regmap)) return PTR_ERR(regmap); - clk_alpha_pll_configure(&ipq_pll, regmap, &ipq_pll_config); + ipq_pll_config = of_device_get_match_data(&pdev->dev); + if (!ipq_pll_config) + return -ENODEV; + + clk_alpha_pll_configure(&ipq_pll, regmap, ipq_pll_config); ret = devm_clk_register_regmap(dev, &ipq_pll.clkr); if (ret) @@ -78,7 +100,8 @@ static int apss_ipq_pll_probe(struct platform_device *pdev) } static const struct of_device_id apss_ipq_pll_match_table[] = { - { .compatible = "qcom,ipq6018-a53pll" }, + { .compatible = "qcom,ipq6018-a53pll", .data = &ipq6018_pll_config }, + { .compatible = "qcom,ipq8074-a53pll", .data = &ipq8074_pll_config }, { } }; MODULE_DEVICE_TABLE(of, apss_ipq_pll_match_table); diff --git a/drivers/clk/qcom/apss-ipq6018.c b/drivers/clk/qcom/apss-ipq6018.c index d78ff2f310bfadc69e64bc3d3f8fbe17ace4fe86..f2f502e2d5a45ec42210f649a575c52dadf57057 100644 --- a/drivers/clk/qcom/apss-ipq6018.c +++ b/drivers/clk/qcom/apss-ipq6018.c @@ -16,7 +16,7 @@ #include "clk-regmap.h" #include "clk-branch.h" #include "clk-alpha-pll.h" -#include "clk-regmap-mux.h" +#include "clk-rcg.h" enum { P_XO, @@ -33,16 +33,15 @@ static const struct parent_map parents_apcs_alias0_clk_src_map[] = { { P_APSS_PLL_EARLY, 5 }, }; -static struct clk_regmap_mux apcs_alias0_clk_src = { - .reg = 0x0050, - .width = 3, - .shift = 7, +static struct clk_rcg2 apcs_alias0_clk_src = { + .cmd_rcgr = 0x0050, + .hid_width = 5, .parent_map = parents_apcs_alias0_clk_src_map, .clkr.hw.init = &(struct clk_init_data){ .name = "apcs_alias0_clk_src", .parent_data = parents_apcs_alias0_clk_src, - .num_parents = 2, - .ops = &clk_regmap_mux_closest_ops, + .num_parents = ARRAY_SIZE(parents_apcs_alias0_clk_src), + .ops = &clk_rcg2_mux_closest_ops, .flags = CLK_SET_RATE_PARENT, }, }; @@ -57,7 +56,7 @@ static struct clk_branch apcs_alias0_core_clk = { .parent_hws = (const struct clk_hw *[]){ &apcs_alias0_clk_src.clkr.hw }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, .ops = &clk_branch2_ops, }, }, diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index b42684703fbb1b0f81714e824d23938559d1b904..1973d79c94655225105cc341653e4b060dac0730 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -27,6 +27,7 @@ # define PLL_VOTE_FSM_RESET BIT(21) # define PLL_UPDATE BIT(22) # define PLL_UPDATE_BYPASS BIT(23) +# define PLL_FSM_LEGACY_MODE BIT(24) # define PLL_OFFLINE_ACK BIT(28) # define ALPHA_PLL_ACK_LATCH BIT(29) # define PLL_ACTIVE_FLAG BIT(30) @@ -166,6 +167,27 @@ const u8 clk_alpha_pll_regs[][PLL_OFF_MAX_REGS] = { [PLL_OFF_TEST_CTL] = 0x28, [PLL_OFF_TEST_CTL_U] = 0x2c, }, + [CLK_ALPHA_PLL_TYPE_DEFAULT_EVO] = { + [PLL_OFF_L_VAL] = 0x04, + [PLL_OFF_ALPHA_VAL] = 0x08, + [PLL_OFF_ALPHA_VAL_U] = 0x0c, + [PLL_OFF_TEST_CTL] = 0x10, + [PLL_OFF_TEST_CTL_U] = 0x14, + [PLL_OFF_USER_CTL] = 0x18, + [PLL_OFF_USER_CTL_U] = 0x1c, + [PLL_OFF_CONFIG_CTL] = 0x20, + [PLL_OFF_STATUS] = 0x24, + }, + [CLK_ALPHA_PLL_TYPE_BRAMMO_EVO] = { + [PLL_OFF_L_VAL] = 0x04, + [PLL_OFF_ALPHA_VAL] = 0x08, + [PLL_OFF_ALPHA_VAL_U] = 0x0c, + [PLL_OFF_TEST_CTL] = 0x10, + [PLL_OFF_TEST_CTL_U] = 0x14, + [PLL_OFF_USER_CTL] = 0x18, + [PLL_OFF_CONFIG_CTL] = 0x1C, + [PLL_OFF_STATUS] = 0x20, + }, }; EXPORT_SYMBOL_GPL(clk_alpha_pll_regs); @@ -1102,6 +1124,10 @@ void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, regmap_update_bits(regmap, PLL_USER_CTL(pll), mask, val); } + if (pll->flags & SUPPORTS_FSM_LEGACY_MODE) + regmap_update_bits(regmap, PLL_MODE(pll), PLL_FSM_LEGACY_MODE, + PLL_FSM_LEGACY_MODE); + regmap_update_bits(regmap, PLL_MODE(pll), PLL_UPDATE_BYPASS, PLL_UPDATE_BYPASS); @@ -2088,7 +2114,7 @@ static int alpha_pll_lucid_evo_enable(struct clk_hw *hw) return ret; } -static void alpha_pll_lucid_evo_disable(struct clk_hw *hw) +static void _alpha_pll_lucid_evo_disable(struct clk_hw *hw, bool reset) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); struct regmap *regmap = pll->clkr.regmap; @@ -2117,9 +2143,12 @@ static void alpha_pll_lucid_evo_disable(struct clk_hw *hw) /* Place the PLL mode in STANDBY */ regmap_write(regmap, PLL_OPMODE(pll), PLL_STANDBY); + + if (reset) + regmap_update_bits(regmap, PLL_MODE(pll), PLL_RESET_N, 0); } -static int alpha_pll_lucid_evo_prepare(struct clk_hw *hw) +static int _alpha_pll_lucid_evo_prepare(struct clk_hw *hw, bool reset) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); struct clk_hw *p; @@ -2139,11 +2168,31 @@ static int alpha_pll_lucid_evo_prepare(struct clk_hw *hw) if (ret) return ret; - alpha_pll_lucid_evo_disable(hw); + _alpha_pll_lucid_evo_disable(hw, reset); return 0; } +static void alpha_pll_lucid_evo_disable(struct clk_hw *hw) +{ + _alpha_pll_lucid_evo_disable(hw, false); +} + +static int alpha_pll_lucid_evo_prepare(struct clk_hw *hw) +{ + return _alpha_pll_lucid_evo_prepare(hw, false); +} + +static void alpha_pll_reset_lucid_evo_disable(struct clk_hw *hw) +{ + _alpha_pll_lucid_evo_disable(hw, true); +} + +static int alpha_pll_reset_lucid_evo_prepare(struct clk_hw *hw) +{ + return _alpha_pll_lucid_evo_prepare(hw, true); +} + static unsigned long alpha_pll_lucid_evo_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { @@ -2191,6 +2240,17 @@ const struct clk_ops clk_alpha_pll_lucid_evo_ops = { }; EXPORT_SYMBOL_GPL(clk_alpha_pll_lucid_evo_ops); +const struct clk_ops clk_alpha_pll_reset_lucid_evo_ops = { + .prepare = alpha_pll_reset_lucid_evo_prepare, + .enable = alpha_pll_lucid_evo_enable, + .disable = alpha_pll_reset_lucid_evo_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = alpha_pll_lucid_evo_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, + .set_rate = alpha_pll_lucid_5lpe_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_alpha_pll_reset_lucid_evo_ops); + void clk_rivian_evo_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, const struct alpha_pll_config *config) { diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 447efb82fe593b4df17475d7e8de664bfb358aba..f9524b3fce6b9f21efbc87ca39afc04dd233dee0 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -19,6 +19,8 @@ enum { CLK_ALPHA_PLL_TYPE_ZONDA, CLK_ALPHA_PLL_TYPE_LUCID_EVO, CLK_ALPHA_PLL_TYPE_RIVIAN_EVO, + CLK_ALPHA_PLL_TYPE_DEFAULT_EVO, + CLK_ALPHA_PLL_TYPE_BRAMMO_EVO, CLK_ALPHA_PLL_TYPE_MAX, }; @@ -70,9 +72,10 @@ struct clk_alpha_pll { const struct pll_vco *vco_table; size_t num_vco; -#define SUPPORTS_OFFLINE_REQ BIT(0) -#define SUPPORTS_FSM_MODE BIT(2) +#define SUPPORTS_OFFLINE_REQ BIT(0) +#define SUPPORTS_FSM_MODE BIT(2) #define SUPPORTS_DYNAMIC_UPDATE BIT(3) +#define SUPPORTS_FSM_LEGACY_MODE BIT(4) u8 flags; struct clk_regmap clkr; @@ -155,6 +158,7 @@ extern const struct clk_ops clk_alpha_pll_zonda_ops; #define clk_alpha_pll_postdiv_zonda_ops clk_alpha_pll_postdiv_fabia_ops extern const struct clk_ops clk_alpha_pll_lucid_evo_ops; +extern const struct clk_ops clk_alpha_pll_reset_lucid_evo_ops; extern const struct clk_ops clk_alpha_pll_fixed_lucid_evo_ops; extern const struct clk_ops clk_alpha_pll_postdiv_lucid_evo_ops; diff --git a/drivers/clk/qcom/clk-cpu-8996.c b/drivers/clk/qcom/clk-cpu-8996.c index 4a4fde8dd12dd9443bf4f7ffd286cd45408dd9f0..ee76ef958d31c6980a9418c5f63ccda49f11151c 100644 --- a/drivers/clk/qcom/clk-cpu-8996.c +++ b/drivers/clk/qcom/clk-cpu-8996.c @@ -49,6 +49,7 @@ * detect voltage droops. */ +#include #include #include #include @@ -59,9 +60,10 @@ #include "clk-alpha-pll.h" #include "clk-regmap.h" +#include "clk-regmap-mux.h" enum _pmux_input { - DIV_2_INDEX = 0, + SMUX_INDEX = 0, PLL_INDEX, ACD_INDEX, ALT_INDEX, @@ -75,6 +77,8 @@ enum _pmux_input { #define ALT_PLL_OFFSET 0x100 #define SSSCTL_OFFSET 0x160 +#define PMUX_MASK 0x3 + static const u8 prim_pll_regs[PLL_OFF_MAX_REGS] = { [PLL_OFF_L_VAL] = 0x04, [PLL_OFF_ALPHA_VAL] = 0x08, @@ -111,30 +115,90 @@ static const struct alpha_pll_config hfpll_config = { .early_output_mask = BIT(3), }; -static struct clk_alpha_pll perfcl_pll = { - .offset = PERFCL_REG_OFFSET, +static const struct clk_parent_data pll_parent[] = { + { .fw_name = "xo" }, +}; + +static struct clk_alpha_pll pwrcl_pll = { + .offset = PWRCL_REG_OFFSET, .regs = prim_pll_regs, .flags = SUPPORTS_DYNAMIC_UPDATE | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data){ - .name = "perfcl_pll", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, + .name = "pwrcl_pll", + .parent_data = pll_parent, + .num_parents = ARRAY_SIZE(pll_parent), .ops = &clk_alpha_pll_huayra_ops, }, }; -static struct clk_alpha_pll pwrcl_pll = { - .offset = PWRCL_REG_OFFSET, +static struct clk_alpha_pll perfcl_pll = { + .offset = PERFCL_REG_OFFSET, .regs = prim_pll_regs, .flags = SUPPORTS_DYNAMIC_UPDATE | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data){ - .name = "pwrcl_pll", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, + .name = "perfcl_pll", + .parent_data = pll_parent, + .num_parents = ARRAY_SIZE(pll_parent), .ops = &clk_alpha_pll_huayra_ops, }, }; +static struct clk_fixed_factor pwrcl_pll_postdiv = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "pwrcl_pll_postdiv", + .parent_data = &(const struct clk_parent_data){ + .hw = &pwrcl_pll.clkr.hw + }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_fixed_factor perfcl_pll_postdiv = { + .mult = 1, + .div = 2, + .hw.init = &(struct clk_init_data){ + .name = "perfcl_pll_postdiv", + .parent_data = &(const struct clk_parent_data){ + .hw = &perfcl_pll.clkr.hw + }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_fixed_factor perfcl_pll_acd = { + .mult = 1, + .div = 1, + .hw.init = &(struct clk_init_data){ + .name = "perfcl_pll_acd", + .parent_data = &(const struct clk_parent_data){ + .hw = &perfcl_pll.clkr.hw + }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_fixed_factor pwrcl_pll_acd = { + .mult = 1, + .div = 1, + .hw.init = &(struct clk_init_data){ + .name = "pwrcl_pll_acd", + .parent_data = &(const struct clk_parent_data){ + .hw = &pwrcl_pll.clkr.hw + }, + .num_parents = 1, + .ops = &clk_fixed_factor_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + static const struct pll_vco alt_pll_vco_modes[] = { VCO(3, 250000000, 500000000), VCO(2, 500000000, 750000000), @@ -153,93 +217,87 @@ static const struct alpha_pll_config altpll_config = { .early_output_mask = BIT(3), }; -static struct clk_alpha_pll perfcl_alt_pll = { - .offset = PERFCL_REG_OFFSET + ALT_PLL_OFFSET, +static struct clk_alpha_pll pwrcl_alt_pll = { + .offset = PWRCL_REG_OFFSET + ALT_PLL_OFFSET, .regs = alt_pll_regs, .vco_table = alt_pll_vco_modes, .num_vco = ARRAY_SIZE(alt_pll_vco_modes), .flags = SUPPORTS_OFFLINE_REQ | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data) { - .name = "perfcl_alt_pll", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, + .name = "pwrcl_alt_pll", + .parent_data = pll_parent, + .num_parents = ARRAY_SIZE(pll_parent), .ops = &clk_alpha_pll_hwfsm_ops, }, }; -static struct clk_alpha_pll pwrcl_alt_pll = { - .offset = PWRCL_REG_OFFSET + ALT_PLL_OFFSET, +static struct clk_alpha_pll perfcl_alt_pll = { + .offset = PERFCL_REG_OFFSET + ALT_PLL_OFFSET, .regs = alt_pll_regs, .vco_table = alt_pll_vco_modes, .num_vco = ARRAY_SIZE(alt_pll_vco_modes), .flags = SUPPORTS_OFFLINE_REQ | SUPPORTS_FSM_MODE, .clkr.hw.init = &(struct clk_init_data) { - .name = "pwrcl_alt_pll", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, + .name = "perfcl_alt_pll", + .parent_data = pll_parent, + .num_parents = ARRAY_SIZE(pll_parent), .ops = &clk_alpha_pll_hwfsm_ops, }, }; -struct clk_cpu_8996_mux { +struct clk_cpu_8996_pmux { u32 reg; - u8 shift; - u8 width; struct notifier_block nb; - struct clk_hw *pll; - struct clk_hw *pll_div_2; struct clk_regmap clkr; }; static int cpu_clk_notifier_cb(struct notifier_block *nb, unsigned long event, void *data); -#define to_clk_cpu_8996_mux_nb(_nb) \ - container_of(_nb, struct clk_cpu_8996_mux, nb) +#define to_clk_cpu_8996_pmux_nb(_nb) \ + container_of(_nb, struct clk_cpu_8996_pmux, nb) -static inline struct clk_cpu_8996_mux *to_clk_cpu_8996_mux_hw(struct clk_hw *hw) +static inline struct clk_cpu_8996_pmux *to_clk_cpu_8996_pmux_hw(struct clk_hw *hw) { - return container_of(to_clk_regmap(hw), struct clk_cpu_8996_mux, clkr); + return container_of(to_clk_regmap(hw), struct clk_cpu_8996_pmux, clkr); } -static u8 clk_cpu_8996_mux_get_parent(struct clk_hw *hw) +static u8 clk_cpu_8996_pmux_get_parent(struct clk_hw *hw) { struct clk_regmap *clkr = to_clk_regmap(hw); - struct clk_cpu_8996_mux *cpuclk = to_clk_cpu_8996_mux_hw(hw); - u32 mask = GENMASK(cpuclk->width - 1, 0); + struct clk_cpu_8996_pmux *cpuclk = to_clk_cpu_8996_pmux_hw(hw); u32 val; regmap_read(clkr->regmap, cpuclk->reg, &val); - val >>= cpuclk->shift; - return val & mask; + return FIELD_GET(PMUX_MASK, val); } -static int clk_cpu_8996_mux_set_parent(struct clk_hw *hw, u8 index) +static int clk_cpu_8996_pmux_set_parent(struct clk_hw *hw, u8 index) { struct clk_regmap *clkr = to_clk_regmap(hw); - struct clk_cpu_8996_mux *cpuclk = to_clk_cpu_8996_mux_hw(hw); - u32 mask = GENMASK(cpuclk->width + cpuclk->shift - 1, cpuclk->shift); + struct clk_cpu_8996_pmux *cpuclk = to_clk_cpu_8996_pmux_hw(hw); u32 val; - val = index; - val <<= cpuclk->shift; + val = FIELD_PREP(PMUX_MASK, index); - return regmap_update_bits(clkr->regmap, cpuclk->reg, mask, val); + return regmap_update_bits(clkr->regmap, cpuclk->reg, PMUX_MASK, val); } -static int clk_cpu_8996_mux_determine_rate(struct clk_hw *hw, +static int clk_cpu_8996_pmux_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { - struct clk_cpu_8996_mux *cpuclk = to_clk_cpu_8996_mux_hw(hw); - struct clk_hw *parent = cpuclk->pll; + struct clk_hw *parent; - if (cpuclk->pll_div_2 && req->rate < DIV_2_THRESHOLD) { - if (req->rate < (DIV_2_THRESHOLD / 2)) - return -EINVAL; + if (req->rate < (DIV_2_THRESHOLD / 2)) + return -EINVAL; - parent = cpuclk->pll_div_2; - } + if (req->rate < DIV_2_THRESHOLD) + parent = clk_hw_get_parent_by_index(hw, SMUX_INDEX); + else + parent = clk_hw_get_parent_by_index(hw, ACD_INDEX); + if (!parent) + return -EINVAL; req->best_parent_rate = clk_hw_round_rate(parent, req->rate); req->best_parent_hw = parent; @@ -247,83 +305,83 @@ static int clk_cpu_8996_mux_determine_rate(struct clk_hw *hw, return 0; } -static const struct clk_ops clk_cpu_8996_mux_ops = { - .set_parent = clk_cpu_8996_mux_set_parent, - .get_parent = clk_cpu_8996_mux_get_parent, - .determine_rate = clk_cpu_8996_mux_determine_rate, +static const struct clk_ops clk_cpu_8996_pmux_ops = { + .set_parent = clk_cpu_8996_pmux_set_parent, + .get_parent = clk_cpu_8996_pmux_get_parent, + .determine_rate = clk_cpu_8996_pmux_determine_rate, +}; + +static const struct clk_parent_data pwrcl_smux_parents[] = { + { .fw_name = "xo" }, + { .hw = &pwrcl_pll_postdiv.hw }, }; -static struct clk_cpu_8996_mux pwrcl_smux = { +static const struct clk_parent_data perfcl_smux_parents[] = { + { .fw_name = "xo" }, + { .hw = &perfcl_pll_postdiv.hw }, +}; + +static struct clk_regmap_mux pwrcl_smux = { .reg = PWRCL_REG_OFFSET + MUX_OFFSET, .shift = 2, .width = 2, .clkr.hw.init = &(struct clk_init_data) { .name = "pwrcl_smux", - .parent_names = (const char *[]){ - "xo", - "pwrcl_pll_main", - }, - .num_parents = 2, - .ops = &clk_cpu_8996_mux_ops, + .parent_data = pwrcl_smux_parents, + .num_parents = ARRAY_SIZE(pwrcl_smux_parents), + .ops = &clk_regmap_mux_closest_ops, .flags = CLK_SET_RATE_PARENT, }, }; -static struct clk_cpu_8996_mux perfcl_smux = { +static struct clk_regmap_mux perfcl_smux = { .reg = PERFCL_REG_OFFSET + MUX_OFFSET, .shift = 2, .width = 2, .clkr.hw.init = &(struct clk_init_data) { .name = "perfcl_smux", - .parent_names = (const char *[]){ - "xo", - "perfcl_pll_main", - }, - .num_parents = 2, - .ops = &clk_cpu_8996_mux_ops, + .parent_data = perfcl_smux_parents, + .num_parents = ARRAY_SIZE(perfcl_smux_parents), + .ops = &clk_regmap_mux_closest_ops, .flags = CLK_SET_RATE_PARENT, }, }; -static struct clk_cpu_8996_mux pwrcl_pmux = { +static const struct clk_hw *pwrcl_pmux_parents[] = { + [SMUX_INDEX] = &pwrcl_smux.clkr.hw, + [PLL_INDEX] = &pwrcl_pll.clkr.hw, + [ACD_INDEX] = &pwrcl_pll_acd.hw, + [ALT_INDEX] = &pwrcl_alt_pll.clkr.hw, +}; + +static const struct clk_hw *perfcl_pmux_parents[] = { + [SMUX_INDEX] = &perfcl_smux.clkr.hw, + [PLL_INDEX] = &perfcl_pll.clkr.hw, + [ACD_INDEX] = &perfcl_pll_acd.hw, + [ALT_INDEX] = &perfcl_alt_pll.clkr.hw, +}; + +static struct clk_cpu_8996_pmux pwrcl_pmux = { .reg = PWRCL_REG_OFFSET + MUX_OFFSET, - .shift = 0, - .width = 2, - .pll = &pwrcl_pll.clkr.hw, - .pll_div_2 = &pwrcl_smux.clkr.hw, .nb.notifier_call = cpu_clk_notifier_cb, .clkr.hw.init = &(struct clk_init_data) { .name = "pwrcl_pmux", - .parent_names = (const char *[]){ - "pwrcl_smux", - "pwrcl_pll", - "pwrcl_pll_acd", - "pwrcl_alt_pll", - }, - .num_parents = 4, - .ops = &clk_cpu_8996_mux_ops, + .parent_hws = pwrcl_pmux_parents, + .num_parents = ARRAY_SIZE(pwrcl_pmux_parents), + .ops = &clk_cpu_8996_pmux_ops, /* CPU clock is critical and should never be gated */ .flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, }, }; -static struct clk_cpu_8996_mux perfcl_pmux = { +static struct clk_cpu_8996_pmux perfcl_pmux = { .reg = PERFCL_REG_OFFSET + MUX_OFFSET, - .shift = 0, - .width = 2, - .pll = &perfcl_pll.clkr.hw, - .pll_div_2 = &perfcl_smux.clkr.hw, .nb.notifier_call = cpu_clk_notifier_cb, .clkr.hw.init = &(struct clk_init_data) { .name = "perfcl_pmux", - .parent_names = (const char *[]){ - "perfcl_smux", - "perfcl_pll", - "perfcl_pll_acd", - "perfcl_alt_pll", - }, - .num_parents = 4, - .ops = &clk_cpu_8996_mux_ops, + .parent_hws = perfcl_pmux_parents, + .num_parents = ARRAY_SIZE(perfcl_pmux_parents), + .ops = &clk_cpu_8996_pmux_ops, /* CPU clock is critical and should never be gated */ .flags = CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, }, @@ -338,15 +396,22 @@ static const struct regmap_config cpu_msm8996_regmap_config = { .val_format_endian = REGMAP_ENDIAN_LITTLE, }; +static struct clk_hw *cpu_msm8996_hw_clks[] = { + &pwrcl_pll_postdiv.hw, + &perfcl_pll_postdiv.hw, + &pwrcl_pll_acd.hw, + &perfcl_pll_acd.hw, +}; + static struct clk_regmap *cpu_msm8996_clks[] = { - &perfcl_pll.clkr, &pwrcl_pll.clkr, - &perfcl_alt_pll.clkr, + &perfcl_pll.clkr, &pwrcl_alt_pll.clkr, - &perfcl_smux.clkr, + &perfcl_alt_pll.clkr, &pwrcl_smux.clkr, - &perfcl_pmux.clkr, + &perfcl_smux.clkr, &pwrcl_pmux.clkr, + &perfcl_pmux.clkr, }; static int qcom_cpu_clk_msm8996_register_clks(struct device *dev, @@ -354,67 +419,33 @@ static int qcom_cpu_clk_msm8996_register_clks(struct device *dev, { int i, ret; - perfcl_smux.pll = clk_hw_register_fixed_factor(dev, "perfcl_pll_main", - "perfcl_pll", - CLK_SET_RATE_PARENT, - 1, 2); - if (IS_ERR(perfcl_smux.pll)) { - dev_err(dev, "Failed to initialize perfcl_pll_main\n"); - return PTR_ERR(perfcl_smux.pll); - } - - pwrcl_smux.pll = clk_hw_register_fixed_factor(dev, "pwrcl_pll_main", - "pwrcl_pll", - CLK_SET_RATE_PARENT, - 1, 2); - if (IS_ERR(pwrcl_smux.pll)) { - dev_err(dev, "Failed to initialize pwrcl_pll_main\n"); - clk_hw_unregister(perfcl_smux.pll); - return PTR_ERR(pwrcl_smux.pll); + for (i = 0; i < ARRAY_SIZE(cpu_msm8996_hw_clks); i++) { + ret = devm_clk_hw_register(dev, cpu_msm8996_hw_clks[i]); + if (ret) + return ret; } for (i = 0; i < ARRAY_SIZE(cpu_msm8996_clks); i++) { ret = devm_clk_register_regmap(dev, cpu_msm8996_clks[i]); - if (ret) { - clk_hw_unregister(perfcl_smux.pll); - clk_hw_unregister(pwrcl_smux.pll); + if (ret) return ret; - } } - clk_alpha_pll_configure(&perfcl_pll, regmap, &hfpll_config); clk_alpha_pll_configure(&pwrcl_pll, regmap, &hfpll_config); - clk_alpha_pll_configure(&perfcl_alt_pll, regmap, &altpll_config); + clk_alpha_pll_configure(&perfcl_pll, regmap, &hfpll_config); clk_alpha_pll_configure(&pwrcl_alt_pll, regmap, &altpll_config); + clk_alpha_pll_configure(&perfcl_alt_pll, regmap, &altpll_config); /* Enable alt PLLs */ clk_prepare_enable(pwrcl_alt_pll.clkr.hw.clk); clk_prepare_enable(perfcl_alt_pll.clkr.hw.clk); - clk_notifier_register(pwrcl_pmux.clkr.hw.clk, &pwrcl_pmux.nb); - clk_notifier_register(perfcl_pmux.clkr.hw.clk, &perfcl_pmux.nb); + devm_clk_notifier_register(dev, pwrcl_pmux.clkr.hw.clk, &pwrcl_pmux.nb); + devm_clk_notifier_register(dev, perfcl_pmux.clkr.hw.clk, &perfcl_pmux.nb); return ret; } -static int qcom_cpu_clk_msm8996_unregister_clks(void) -{ - int ret = 0; - - ret = clk_notifier_unregister(pwrcl_pmux.clkr.hw.clk, &pwrcl_pmux.nb); - if (ret) - return ret; - - ret = clk_notifier_unregister(perfcl_pmux.clkr.hw.clk, &perfcl_pmux.nb); - if (ret) - return ret; - - clk_hw_unregister(perfcl_smux.pll); - clk_hw_unregister(pwrcl_smux.pll); - - return 0; -} - #define CPU_AFINITY_MASK 0xFFF #define PWRCL_CPU_REG_MASK 0x3 #define PERFCL_CPU_REG_MASK 0x103 @@ -456,22 +487,22 @@ static void qcom_cpu_clk_msm8996_acd_init(void __iomem *base) static int cpu_clk_notifier_cb(struct notifier_block *nb, unsigned long event, void *data) { - struct clk_cpu_8996_mux *cpuclk = to_clk_cpu_8996_mux_nb(nb); + struct clk_cpu_8996_pmux *cpuclk = to_clk_cpu_8996_pmux_nb(nb); struct clk_notifier_data *cnd = data; int ret; switch (event) { case PRE_RATE_CHANGE: - ret = clk_cpu_8996_mux_set_parent(&cpuclk->clkr.hw, ALT_INDEX); + ret = clk_cpu_8996_pmux_set_parent(&cpuclk->clkr.hw, ALT_INDEX); qcom_cpu_clk_msm8996_acd_init(base); break; case POST_RATE_CHANGE: if (cnd->new_rate < DIV_2_THRESHOLD) - ret = clk_cpu_8996_mux_set_parent(&cpuclk->clkr.hw, - DIV_2_INDEX); + ret = clk_cpu_8996_pmux_set_parent(&cpuclk->clkr.hw, + SMUX_INDEX); else - ret = clk_cpu_8996_mux_set_parent(&cpuclk->clkr.hw, - ACD_INDEX); + ret = clk_cpu_8996_pmux_set_parent(&cpuclk->clkr.hw, + ACD_INDEX); break; default: ret = 0; @@ -513,11 +544,6 @@ static int qcom_cpu_clk_msm8996_driver_probe(struct platform_device *pdev) return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, data); } -static int qcom_cpu_clk_msm8996_driver_remove(struct platform_device *pdev) -{ - return qcom_cpu_clk_msm8996_unregister_clks(); -} - static const struct of_device_id qcom_cpu_clk_msm8996_match_table[] = { { .compatible = "qcom,msm8996-apcc" }, {} @@ -526,7 +552,6 @@ MODULE_DEVICE_TABLE(of, qcom_cpu_clk_msm8996_match_table); static struct platform_driver qcom_cpu_clk_msm8996_driver = { .probe = qcom_cpu_clk_msm8996_driver_probe, - .remove = qcom_cpu_clk_msm8996_driver_remove, .driver = { .name = "qcom-msm8996-apcc", .of_match_table = qcom_cpu_clk_msm8996_match_table, diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index 012e745794fde2ee97eaa50c0022b038b4e2b570..01581f4d2c39c4a8e27df94d11e1ef379c291c07 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -167,6 +167,7 @@ struct clk_rcg2_gfx3d { extern const struct clk_ops clk_rcg2_ops; extern const struct clk_ops clk_rcg2_floor_ops; +extern const struct clk_ops clk_rcg2_mux_closest_ops; extern const struct clk_ops clk_edp_pixel_ops; extern const struct clk_ops clk_byte_ops; extern const struct clk_ops clk_byte2_ops; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 28019edd2a5082e6c1e2a0b420ef6c05b25a2b6b..76551534f10dfe0dc09a9917a3df974b80f2bd58 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -509,6 +509,13 @@ const struct clk_ops clk_rcg2_floor_ops = { }; EXPORT_SYMBOL_GPL(clk_rcg2_floor_ops); +const struct clk_ops clk_rcg2_mux_closest_ops = { + .determine_rate = __clk_mux_determine_rate_closest, + .get_parent = clk_rcg2_get_parent, + .set_parent = clk_rcg2_set_parent, +}; +EXPORT_SYMBOL_GPL(clk_rcg2_mux_closest_ops); + struct frac_entry { int num; int den; @@ -908,6 +915,15 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, req->best_parent_hw = p2; } + clk_hw_get_rate_range(req->best_parent_hw, + &parent_req.min_rate, &parent_req.max_rate); + + if (req->min_rate > parent_req.min_rate) + parent_req.min_rate = req->min_rate; + + if (req->max_rate < parent_req.max_rate) + parent_req.max_rate = req->max_rate; + ret = __clk_determine_rate(req->best_parent_hw, &parent_req); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index c07cab6905cbf699b405596e757428e6dfc57cfe..0471bab824642feccba35873d4bcb9f566bbff74 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -195,10 +195,6 @@ static int clk_rpmh_aggregate_state_send_command(struct clk_rpmh *c, { int ret; - /* Nothing required to be done if already off or on */ - if (enable == c->state) - return 0; - c->state = enable ? c->valid_state_mask : 0; c->aggr_state = c->state | c->peer->state; c->peer->aggr_state = c->aggr_state; @@ -382,6 +378,26 @@ static const struct clk_rpmh_desc clk_rpmh_sdm845 = { .num_clks = ARRAY_SIZE(sdm845_rpmh_clocks), }; +static struct clk_hw *sdm670_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sdm845_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK2] = &sdm845_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sdm845_ln_bb_clk2_ao.hw, + [RPMH_LN_BB_CLK3] = &sdm845_ln_bb_clk3.hw, + [RPMH_LN_BB_CLK3_A] = &sdm845_ln_bb_clk3_ao.hw, + [RPMH_RF_CLK1] = &sdm845_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, + [RPMH_RF_CLK2] = &sdm845_rf_clk2.hw, + [RPMH_RF_CLK2_A] = &sdm845_rf_clk2_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, + [RPMH_CE_CLK] = &sdm845_ce.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sdm670 = { + .clks = sdm670_rpmh_clocks, + .num_clks = ARRAY_SIZE(sdm670_rpmh_clocks), +}; + DEFINE_CLK_RPMH_VRM(sdx55, rf_clk1, rf_clk1_ao, "rfclkd1", 1); DEFINE_CLK_RPMH_VRM(sdx55, rf_clk2, rf_clk2_ao, "rfclkd2", 1); DEFINE_CLK_RPMH_BCM(sdx55, qpic_clk, "QP0"); @@ -715,6 +731,7 @@ static const struct of_device_id clk_rpmh_match_table[] = { { .compatible = "qcom,sc8180x-rpmh-clk", .data = &clk_rpmh_sc8180x}, { .compatible = "qcom,sc8280xp-rpmh-clk", .data = &clk_rpmh_sc8280xp}, { .compatible = "qcom,sdm845-rpmh-clk", .data = &clk_rpmh_sdm845}, + { .compatible = "qcom,sdm670-rpmh-clk", .data = &clk_rpmh_sdm670}, { .compatible = "qcom,sdx55-rpmh-clk", .data = &clk_rpmh_sdx55}, { .compatible = "qcom,sdx65-rpmh-clk", .data = &clk_rpmh_sdx65}, { .compatible = "qcom,sm6350-rpmh-clk", .data = &clk_rpmh_sm6350}, diff --git a/drivers/clk/qcom/clk-smd-rpm.c b/drivers/clk/qcom/clk-smd-rpm.c index 10b4e6d8d10f30e06b01db7328bd4d925c05f5b2..fea50587685503e1f22844e5916d03ab07a44b95 100644 --- a/drivers/clk/qcom/clk-smd-rpm.c +++ b/drivers/clk/qcom/clk-smd-rpm.c @@ -417,6 +417,7 @@ DEFINE_CLK_SMD_RPM_BRANCH(sdm660, bi_tcxo, bi_tcxo_a, QCOM_SMD_RPM_MISC_CLK, 0, DEFINE_CLK_SMD_RPM(msm8916, pcnoc_clk, pcnoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 0); DEFINE_CLK_SMD_RPM(msm8916, snoc_clk, snoc_a_clk, QCOM_SMD_RPM_BUS_CLK, 1); DEFINE_CLK_SMD_RPM(msm8916, bimc_clk, bimc_a_clk, QCOM_SMD_RPM_MEM_CLK, 0); +DEFINE_CLK_SMD_RPM(qcs404, qpic_clk, qpic_a_clk, QCOM_SMD_RPM_QPIC_CLK, 0); DEFINE_CLK_SMD_RPM_QDSS(msm8916, qdss_clk, qdss_a_clk, QCOM_SMD_RPM_MISC_CLK, 1); DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, bb_clk1, bb_clk1_a, 1, 19200000); DEFINE_CLK_SMD_RPM_XO_BUFFER(msm8916, bb_clk2, bb_clk2_a, 2, 19200000); @@ -427,6 +428,40 @@ DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, bb_clk2_pin, bb_clk2_a_pin, 2, 192 DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, rf_clk1_pin, rf_clk1_a_pin, 4, 19200000); DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(msm8916, rf_clk2_pin, rf_clk2_a_pin, 5, 19200000); +static struct clk_smd_rpm *msm8909_clks[] = { + [RPM_SMD_PCNOC_CLK] = &msm8916_pcnoc_clk, + [RPM_SMD_PCNOC_A_CLK] = &msm8916_pcnoc_a_clk, + [RPM_SMD_SNOC_CLK] = &msm8916_snoc_clk, + [RPM_SMD_SNOC_A_CLK] = &msm8916_snoc_a_clk, + [RPM_SMD_BIMC_CLK] = &msm8916_bimc_clk, + [RPM_SMD_BIMC_A_CLK] = &msm8916_bimc_a_clk, + [RPM_SMD_QPIC_CLK] = &qcs404_qpic_clk, + [RPM_SMD_QPIC_CLK_A] = &qcs404_qpic_a_clk, + [RPM_SMD_QDSS_CLK] = &msm8916_qdss_clk, + [RPM_SMD_QDSS_A_CLK] = &msm8916_qdss_a_clk, + [RPM_SMD_BB_CLK1] = &msm8916_bb_clk1, + [RPM_SMD_BB_CLK1_A] = &msm8916_bb_clk1_a, + [RPM_SMD_BB_CLK2] = &msm8916_bb_clk2, + [RPM_SMD_BB_CLK2_A] = &msm8916_bb_clk2_a, + [RPM_SMD_RF_CLK1] = &msm8916_rf_clk1, + [RPM_SMD_RF_CLK1_A] = &msm8916_rf_clk1_a, + [RPM_SMD_RF_CLK2] = &msm8916_rf_clk2, + [RPM_SMD_RF_CLK2_A] = &msm8916_rf_clk2_a, + [RPM_SMD_BB_CLK1_PIN] = &msm8916_bb_clk1_pin, + [RPM_SMD_BB_CLK1_A_PIN] = &msm8916_bb_clk1_a_pin, + [RPM_SMD_BB_CLK2_PIN] = &msm8916_bb_clk2_pin, + [RPM_SMD_BB_CLK2_A_PIN] = &msm8916_bb_clk2_a_pin, + [RPM_SMD_RF_CLK1_PIN] = &msm8916_rf_clk1_pin, + [RPM_SMD_RF_CLK1_A_PIN] = &msm8916_rf_clk1_a_pin, + [RPM_SMD_RF_CLK2_PIN] = &msm8916_rf_clk2_pin, + [RPM_SMD_RF_CLK2_A_PIN] = &msm8916_rf_clk2_a_pin, +}; + +static const struct rpm_smd_clk_desc rpm_clk_msm8909 = { + .clks = msm8909_clks, + .num_clks = ARRAY_SIZE(msm8909_clks), +}; + static struct clk_smd_rpm *msm8916_clks[] = { [RPM_SMD_PCNOC_CLK] = &msm8916_pcnoc_clk, [RPM_SMD_PCNOC_A_CLK] = &msm8916_pcnoc_a_clk, @@ -787,7 +822,6 @@ static const struct rpm_smd_clk_desc rpm_clk_msm8996 = { }; DEFINE_CLK_SMD_RPM(qcs404, bimc_gpu_clk, bimc_gpu_a_clk, QCOM_SMD_RPM_MEM_CLK, 2); -DEFINE_CLK_SMD_RPM(qcs404, qpic_clk, qpic_a_clk, QCOM_SMD_RPM_QPIC_CLK, 0); DEFINE_CLK_SMD_RPM_XO_BUFFER_PINCTRL(qcs404, ln_bb_clk_pin, ln_bb_clk_a_pin, 8, 19200000); static struct clk_smd_rpm *qcs404_clks[] = { @@ -1085,13 +1119,54 @@ static const struct rpm_smd_clk_desc rpm_clk_sm6115 = { .num_clks = ARRAY_SIZE(sm6115_clks), }; +/* SM6375 */ +DEFINE_CLK_SMD_RPM(sm6375, mmnrt_clk, mmnrt_a_clk, QCOM_SMD_RPM_MMXI_CLK, 0); +DEFINE_CLK_SMD_RPM(sm6375, mmrt_clk, mmrt_a_clk, QCOM_SMD_RPM_MMXI_CLK, 1); +DEFINE_CLK_SMD_RPM(qcm2290, hwkm_clk, hwkm_a_clk, QCOM_SMD_RPM_HWKM_CLK, 0); +DEFINE_CLK_SMD_RPM(qcm2290, pka_clk, pka_a_clk, QCOM_SMD_RPM_PKA_CLK, 0); +DEFINE_CLK_SMD_RPM_BRANCH(sm6375, bimc_freq_log, bimc_freq_log_a, QCOM_SMD_RPM_MISC_CLK, 4, 1); +static struct clk_smd_rpm *sm6375_clks[] = { + [RPM_SMD_XO_CLK_SRC] = &sdm660_bi_tcxo, + [RPM_SMD_XO_A_CLK_SRC] = &sdm660_bi_tcxo_a, + [RPM_SMD_SNOC_CLK] = &sm6125_snoc_clk, + [RPM_SMD_SNOC_A_CLK] = &sm6125_snoc_a_clk, + [RPM_SMD_BIMC_CLK] = &msm8916_bimc_clk, + [RPM_SMD_BIMC_A_CLK] = &msm8916_bimc_a_clk, + [RPM_SMD_QDSS_CLK] = &sm6125_qdss_clk, + [RPM_SMD_QDSS_A_CLK] = &sm6125_qdss_a_clk, + [RPM_SMD_CNOC_CLK] = &sm6125_cnoc_clk, + [RPM_SMD_CNOC_A_CLK] = &sm6125_cnoc_a_clk, + [RPM_SMD_IPA_CLK] = &msm8976_ipa_clk, + [RPM_SMD_IPA_A_CLK] = &msm8976_ipa_a_clk, + [RPM_SMD_QUP_CLK] = &sm6125_qup_clk, + [RPM_SMD_QUP_A_CLK] = &sm6125_qup_a_clk, + [RPM_SMD_MMRT_CLK] = &sm6375_mmrt_clk, + [RPM_SMD_MMRT_A_CLK] = &sm6375_mmrt_a_clk, + [RPM_SMD_MMNRT_CLK] = &sm6375_mmnrt_clk, + [RPM_SMD_MMNRT_A_CLK] = &sm6375_mmnrt_a_clk, + [RPM_SMD_SNOC_PERIPH_CLK] = &sm6125_snoc_periph_clk, + [RPM_SMD_SNOC_PERIPH_A_CLK] = &sm6125_snoc_periph_a_clk, + [RPM_SMD_SNOC_LPASS_CLK] = &sm6125_snoc_lpass_clk, + [RPM_SMD_SNOC_LPASS_A_CLK] = &sm6125_snoc_lpass_a_clk, + [RPM_SMD_CE1_CLK] = &msm8992_ce1_clk, + [RPM_SMD_CE1_A_CLK] = &msm8992_ce1_a_clk, + [RPM_SMD_HWKM_CLK] = &qcm2290_hwkm_clk, + [RPM_SMD_HWKM_A_CLK] = &qcm2290_hwkm_a_clk, + [RPM_SMD_PKA_CLK] = &qcm2290_pka_clk, + [RPM_SMD_PKA_A_CLK] = &qcm2290_pka_a_clk, + [RPM_SMD_BIMC_FREQ_LOG] = &sm6375_bimc_freq_log, +}; + +static const struct rpm_smd_clk_desc rpm_clk_sm6375 = { + .clks = sm6375_clks, + .num_clks = ARRAY_SIZE(sm6375_clks), +}; + /* QCM2290 */ DEFINE_CLK_SMD_RPM_XO_BUFFER(qcm2290, ln_bb_clk2, ln_bb_clk2_a, 0x2, 19200000); DEFINE_CLK_SMD_RPM_XO_BUFFER(qcm2290, rf_clk3, rf_clk3_a, 6, 38400000); DEFINE_CLK_SMD_RPM(qcm2290, qpic_clk, qpic_a_clk, QCOM_SMD_RPM_QPIC_CLK, 0); -DEFINE_CLK_SMD_RPM(qcm2290, hwkm_clk, hwkm_a_clk, QCOM_SMD_RPM_HWKM_CLK, 0); -DEFINE_CLK_SMD_RPM(qcm2290, pka_clk, pka_a_clk, QCOM_SMD_RPM_PKA_CLK, 0); DEFINE_CLK_SMD_RPM(qcm2290, cpuss_gnoc_clk, cpuss_gnoc_a_clk, QCOM_SMD_RPM_MEM_CLK, 1); DEFINE_CLK_SMD_RPM(qcm2290, bimc_gpu_clk, bimc_gpu_a_clk, @@ -1146,6 +1221,7 @@ static const struct rpm_smd_clk_desc rpm_clk_qcm2290 = { static const struct of_device_id rpm_smd_clk_match_table[] = { { .compatible = "qcom,rpmcc-mdm9607", .data = &rpm_clk_mdm9607 }, { .compatible = "qcom,rpmcc-msm8226", .data = &rpm_clk_msm8974 }, + { .compatible = "qcom,rpmcc-msm8909", .data = &rpm_clk_msm8909 }, { .compatible = "qcom,rpmcc-msm8916", .data = &rpm_clk_msm8916 }, { .compatible = "qcom,rpmcc-msm8936", .data = &rpm_clk_msm8936 }, { .compatible = "qcom,rpmcc-msm8953", .data = &rpm_clk_msm8953 }, @@ -1160,6 +1236,7 @@ static const struct of_device_id rpm_smd_clk_match_table[] = { { .compatible = "qcom,rpmcc-sdm660", .data = &rpm_clk_sdm660 }, { .compatible = "qcom,rpmcc-sm6115", .data = &rpm_clk_sm6115 }, { .compatible = "qcom,rpmcc-sm6125", .data = &rpm_clk_sm6125 }, + { .compatible = "qcom,rpmcc-sm6375", .data = &rpm_clk_sm6375 }, { } }; MODULE_DEVICE_TABLE(of, rpm_smd_clk_match_table); diff --git a/drivers/clk/qcom/dispcc-sm6115.c b/drivers/clk/qcom/dispcc-sm6115.c new file mode 100644 index 0000000000000000000000000000000000000000..818bb8f4637c38eb57dbdf3814195db7ffee5a79 --- /dev/null +++ b/drivers/clk/qcom/dispcc-sm6115.c @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Based on dispcc-qcm2290.c + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2021, Linaro Ltd. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "gdsc.h" + +enum { + DT_BI_TCXO, + DT_SLEEP_CLK, + DT_DSI0_PHY_PLL_OUT_BYTECLK, + DT_DSI0_PHY_PLL_OUT_DSICLK, + DT_GPLL0_DISP_DIV, +}; + +enum { + P_BI_TCXO, + P_DISP_CC_PLL0_OUT_MAIN, + P_DSI0_PHY_PLL_OUT_BYTECLK, + P_DSI0_PHY_PLL_OUT_DSICLK, + P_GPLL0_OUT_MAIN, + P_SLEEP_CLK, +}; + +static const struct clk_parent_data parent_data_tcxo = { .index = DT_BI_TCXO }; + +static const struct pll_vco spark_vco[] = { + { 500000000, 1000000000, 2 }, +}; + +/* 768MHz configuration */ +static const struct alpha_pll_config disp_cc_pll0_config = { + .l = 0x28, + .alpha = 0x0, + .alpha_en_mask = BIT(24), + .vco_val = 0x2 << 20, + .vco_mask = GENMASK(21, 20), + .main_output_mask = BIT(0), + .config_ctl_val = 0x4001055B, +}; + +static struct clk_alpha_pll disp_cc_pll0 = { + .offset = 0x0, + .vco_table = spark_vco, + .num_vco = ARRAY_SIZE(spark_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_pll0", + .parent_data = &parent_data_tcxo, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_disp_cc_pll0_out_main[] = { + { 0x0, 1 }, + { } +}; +static struct clk_alpha_pll_postdiv disp_cc_pll0_out_main = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_disp_cc_pll0_out_main, + .num_post_div = ARRAY_SIZE(post_div_table_disp_cc_pll0_out_main), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_pll0_out_main", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_pll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_ops, + }, +}; + +static const struct parent_map disp_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_BYTECLK, 1 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DSI0_PHY_PLL_OUT_BYTECLK }, +}; + +static const struct parent_map disp_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_1[] = { + { .index = DT_BI_TCXO }, +}; + +static const struct parent_map disp_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .index = DT_GPLL0_DISP_DIV }, +}; + +static const struct parent_map disp_cc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_3[] = { + { .index = DT_BI_TCXO }, + { .hw = &disp_cc_pll0_out_main.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_DSICLK, 1 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_4[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DSI0_PHY_PLL_OUT_DSICLK }, +}; + +static const struct parent_map disp_cc_parent_map_5[] = { + { P_SLEEP_CLK, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_5[] = { + { .index = DT_SLEEP_CLK, }, +}; + +static struct clk_rcg2 disp_cc_mdss_byte0_clk_src = { + .cmd_rcgr = 0x20bc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + /* For set_rate and set_parent to succeed, parent(s) must be enabled */ + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE | CLK_GET_RATE_NOCACHE, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { + .reg = 0x20d4, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_div_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_ahb_clk_src = { + .cmd_rcgr = 0x2154, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_esc0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_esc0_clk_src = { + .cmd_rcgr = 0x20d8, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_esc0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_mdp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(192000000, P_DISP_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(256000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(307200000, P_DISP_CC_PLL0_OUT_MAIN, 2.5, 0, 0), + F(384000000, P_DISP_CC_PLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = { + .cmd_rcgr = 0x2074, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { + .cmd_rcgr = 0x205c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk_src", + .parent_data = disp_cc_parent_data_4, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), + /* For set_rate and set_parent to succeed, parent(s) must be enabled */ + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE | CLK_GET_RATE_NOCACHE, + .ops = &clk_pixel_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_rot_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(192000000, P_DISP_CC_PLL0_OUT_MAIN, 4, 0, 0), + F(256000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(307200000, P_DISP_CC_PLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_rot_clk_src = { + .cmd_rcgr = 0x208c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = { + .cmd_rcgr = 0x20a4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_esc0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_sleep_clk_src[] = { + F(32764, P_SLEEP_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_sleep_clk_src = { + .cmd_rcgr = 0x6050, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .freq_tbl = ftbl_disp_cc_sleep_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "disp_cc_sleep_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch disp_cc_mdss_ahb_clk = { + .halt_reg = 0x2044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_ahb_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_clk = { + .halt_reg = 0x2024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_intf_clk = { + .halt_reg = 0x2028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_byte0_intf_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_byte0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc0_clk = { + .halt_reg = 0x202c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x202c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_esc0_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_clk = { + .halt_reg = 0x2008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_lut_clk = { + .halt_reg = 0x2018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x2018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_mdp_lut_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_non_gdsc_ahb_clk = { + .halt_reg = 0x4004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_non_gdsc_ahb_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk0_clk = { + .halt_reg = 0x2004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_pclk0_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rot_clk = { + .halt_reg = 0x2010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_rot_clk", + .parent_names = (const char *[]){ + "disp_cc_mdss_rot_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_vsync_clk = { + .halt_reg = 0x2020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_mdss_vsync_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_sleep_clk = { + .halt_reg = 0x6068, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "disp_cc_sleep_clk", + .parent_hws = (const struct clk_hw*[]){ + &disp_cc_sleep_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x3000, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, +}; + +static struct gdsc *disp_cc_sm6115_gdscs[] = { + [MDSS_GDSC] = &mdss_gdsc, +}; + +static struct clk_regmap *disp_cc_sm6115_clocks[] = { + [DISP_CC_PLL0] = &disp_cc_pll0.clkr, + [DISP_CC_PLL0_OUT_MAIN] = &disp_cc_pll0_out_main.clkr, + [DISP_CC_MDSS_AHB_CLK] = &disp_cc_mdss_ahb_clk.clkr, + [DISP_CC_MDSS_AHB_CLK_SRC] = &disp_cc_mdss_ahb_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_CLK] = &disp_cc_mdss_byte0_clk.clkr, + [DISP_CC_MDSS_BYTE0_CLK_SRC] = &disp_cc_mdss_byte0_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_DIV_CLK_SRC] = &disp_cc_mdss_byte0_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_INTF_CLK] = &disp_cc_mdss_byte0_intf_clk.clkr, + [DISP_CC_MDSS_ESC0_CLK] = &disp_cc_mdss_esc0_clk.clkr, + [DISP_CC_MDSS_ESC0_CLK_SRC] = &disp_cc_mdss_esc0_clk_src.clkr, + [DISP_CC_MDSS_MDP_CLK] = &disp_cc_mdss_mdp_clk.clkr, + [DISP_CC_MDSS_MDP_CLK_SRC] = &disp_cc_mdss_mdp_clk_src.clkr, + [DISP_CC_MDSS_MDP_LUT_CLK] = &disp_cc_mdss_mdp_lut_clk.clkr, + [DISP_CC_MDSS_NON_GDSC_AHB_CLK] = &disp_cc_mdss_non_gdsc_ahb_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK] = &disp_cc_mdss_pclk0_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK_SRC] = &disp_cc_mdss_pclk0_clk_src.clkr, + [DISP_CC_MDSS_ROT_CLK] = &disp_cc_mdss_rot_clk.clkr, + [DISP_CC_MDSS_ROT_CLK_SRC] = &disp_cc_mdss_rot_clk_src.clkr, + [DISP_CC_MDSS_VSYNC_CLK] = &disp_cc_mdss_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK_SRC] = &disp_cc_mdss_vsync_clk_src.clkr, + [DISP_CC_SLEEP_CLK] = &disp_cc_sleep_clk.clkr, + [DISP_CC_SLEEP_CLK_SRC] = &disp_cc_sleep_clk_src.clkr, +}; + +static const struct regmap_config disp_cc_sm6115_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x10000, + .fast_io = true, +}; + +static const struct qcom_cc_desc disp_cc_sm6115_desc = { + .config = &disp_cc_sm6115_regmap_config, + .clks = disp_cc_sm6115_clocks, + .num_clks = ARRAY_SIZE(disp_cc_sm6115_clocks), + .gdscs = disp_cc_sm6115_gdscs, + .num_gdscs = ARRAY_SIZE(disp_cc_sm6115_gdscs), +}; + +static const struct of_device_id disp_cc_sm6115_match_table[] = { + { .compatible = "qcom,sm6115-dispcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, disp_cc_sm6115_match_table); + +static int disp_cc_sm6115_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &disp_cc_sm6115_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_alpha_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config); + + /* Keep DISP_CC_XO_CLK always-ON */ + regmap_update_bits(regmap, 0x604c, BIT(0), BIT(0)); + + ret = qcom_cc_really_probe(pdev, &disp_cc_sm6115_desc, regmap); + if (ret) { + dev_err(&pdev->dev, "Failed to register DISP CC clocks\n"); + return ret; + } + + return ret; +} + +static struct platform_driver disp_cc_sm6115_driver = { + .probe = disp_cc_sm6115_probe, + .driver = { + .name = "dispcc-sm6115", + .of_match_table = disp_cc_sm6115_match_table, + }, +}; + +module_platform_driver(disp_cc_sm6115_driver); +MODULE_DESCRIPTION("Qualcomm SM6115 Display Clock controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c new file mode 100644 index 0000000000000000000000000000000000000000..0cd7ebe90301c054003dd2561fb25a0db33a942b --- /dev/null +++ b/drivers/clk/qcom/dispcc-sm8450.c @@ -0,0 +1,1829 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Linaro Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "reset.h" +#include "gdsc.h" + +/* Need to match the order of clocks in DT binding */ +enum { + DT_BI_TCXO, + DT_BI_TCXO_AO, + DT_AHB_CLK, + DT_SLEEP_CLK, + + DT_DSI0_PHY_PLL_OUT_BYTECLK, + DT_DSI0_PHY_PLL_OUT_DSICLK, + DT_DSI1_PHY_PLL_OUT_BYTECLK, + DT_DSI1_PHY_PLL_OUT_DSICLK, + + DT_DP0_PHY_PLL_LINK_CLK, + DT_DP0_PHY_PLL_VCO_DIV_CLK, + DT_DP1_PHY_PLL_LINK_CLK, + DT_DP1_PHY_PLL_VCO_DIV_CLK, + DT_DP2_PHY_PLL_LINK_CLK, + DT_DP2_PHY_PLL_VCO_DIV_CLK, + DT_DP3_PHY_PLL_LINK_CLK, + DT_DP3_PHY_PLL_VCO_DIV_CLK, +}; + +#define DISP_CC_MISC_CMD 0xF000 + +enum { + P_BI_TCXO, + P_DISP_CC_PLL0_OUT_MAIN, + P_DISP_CC_PLL1_OUT_EVEN, + P_DISP_CC_PLL1_OUT_MAIN, + P_DP0_PHY_PLL_LINK_CLK, + P_DP0_PHY_PLL_VCO_DIV_CLK, + P_DP1_PHY_PLL_LINK_CLK, + P_DP1_PHY_PLL_VCO_DIV_CLK, + P_DP2_PHY_PLL_LINK_CLK, + P_DP2_PHY_PLL_VCO_DIV_CLK, + P_DP3_PHY_PLL_LINK_CLK, + P_DP3_PHY_PLL_VCO_DIV_CLK, + P_DSI0_PHY_PLL_OUT_BYTECLK, + P_DSI0_PHY_PLL_OUT_DSICLK, + P_DSI1_PHY_PLL_OUT_BYTECLK, + P_DSI1_PHY_PLL_OUT_DSICLK, + P_SLEEP_CLK, +}; + +static struct pll_vco lucid_evo_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static const struct alpha_pll_config disp_cc_pll0_config = { + .l = 0xD, + .alpha = 0x6492, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x32AA299C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, +}; + +static struct clk_alpha_pll disp_cc_pll0 = { + .offset = 0x0, + .vco_table = lucid_evo_vco, + .num_vco = ARRAY_SIZE(lucid_evo_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_pll0", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_reset_lucid_evo_ops, + }, + }, +}; + +static const struct alpha_pll_config disp_cc_pll1_config = { + .l = 0x1F, + .alpha = 0x4000, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00182261, + .config_ctl_hi1_val = 0x32AA299C, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, +}; + +static struct clk_alpha_pll disp_cc_pll1 = { + .offset = 0x1000, + .vco_table = lucid_evo_vco, + .num_vco = ARRAY_SIZE(lucid_evo_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID_EVO], + .clkr = { + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_pll1", + .parent_data = &(const struct clk_parent_data) { + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_reset_lucid_evo_ops, + }, + }, +}; + +static const struct parent_map disp_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_DP0_PHY_PLL_LINK_CLK, 1 }, + { P_DP0_PHY_PLL_VCO_DIV_CLK, 2 }, + { P_DP3_PHY_PLL_VCO_DIV_CLK, 3 }, + { P_DP1_PHY_PLL_VCO_DIV_CLK, 4 }, + { P_DP2_PHY_PLL_VCO_DIV_CLK, 6 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DP0_PHY_PLL_LINK_CLK }, + { .index = DT_DP0_PHY_PLL_VCO_DIV_CLK }, + { .index = DT_DP3_PHY_PLL_VCO_DIV_CLK }, + { .index = DT_DP1_PHY_PLL_VCO_DIV_CLK }, + { .index = DT_DP2_PHY_PLL_VCO_DIV_CLK }, +}; + +static const struct parent_map disp_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_1[] = { + { .index = DT_BI_TCXO }, +}; + +static const struct clk_parent_data disp_cc_parent_data_1_ao[] = { + { .index = DT_BI_TCXO_AO }, +}; + +static const struct parent_map disp_cc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_DSICLK, 1 }, + { P_DSI0_PHY_PLL_OUT_BYTECLK, 2 }, + { P_DSI1_PHY_PLL_OUT_DSICLK, 3 }, + { P_DSI1_PHY_PLL_OUT_BYTECLK, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DSI0_PHY_PLL_OUT_DSICLK }, + { .index = DT_DSI0_PHY_PLL_OUT_BYTECLK }, + { .index = DT_DSI1_PHY_PLL_OUT_DSICLK }, + { .index = DT_DSI1_PHY_PLL_OUT_BYTECLK }, +}; + +static const struct parent_map disp_cc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_DP0_PHY_PLL_LINK_CLK, 1 }, + { P_DP1_PHY_PLL_LINK_CLK, 2 }, + { P_DP2_PHY_PLL_LINK_CLK, 3 }, + { P_DP3_PHY_PLL_LINK_CLK, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_3[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DP0_PHY_PLL_LINK_CLK }, + { .index = DT_DP1_PHY_PLL_LINK_CLK }, + { .index = DT_DP2_PHY_PLL_LINK_CLK }, + { .index = DT_DP3_PHY_PLL_LINK_CLK }, +}; + +static const struct parent_map disp_cc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_DSI0_PHY_PLL_OUT_BYTECLK, 2 }, + { P_DSI1_PHY_PLL_OUT_BYTECLK, 4 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_4[] = { + { .index = DT_BI_TCXO }, + { .index = DT_DSI0_PHY_PLL_OUT_BYTECLK }, + { .index = DT_DSI1_PHY_PLL_OUT_BYTECLK }, +}; + +static const struct parent_map disp_cc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL0_OUT_MAIN, 1 }, + { P_DISP_CC_PLL1_OUT_MAIN, 4 }, + { P_DISP_CC_PLL1_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_5[] = { + { .index = DT_BI_TCXO }, + { .hw = &disp_cc_pll0.clkr.hw }, + { .hw = &disp_cc_pll1.clkr.hw }, + { .hw = &disp_cc_pll1.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_DISP_CC_PLL1_OUT_MAIN, 4 }, + { P_DISP_CC_PLL1_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_6[] = { + { .index = DT_BI_TCXO }, + { .hw = &disp_cc_pll1.clkr.hw }, + { .hw = &disp_cc_pll1.clkr.hw }, +}; + +static const struct parent_map disp_cc_parent_map_7[] = { + { P_SLEEP_CLK, 0 }, +}; + +static const struct clk_parent_data disp_cc_parent_data_7[] = { + { .index = DT_SLEEP_CLK }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_DISP_CC_PLL1_OUT_MAIN, 16, 0, 0), + F(75000000, P_DISP_CC_PLL1_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_ahb_clk_src = { + .cmd_rcgr = 0x8324, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_6, + .freq_tbl = ftbl_disp_cc_mdss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_ahb_clk_src", + .parent_data = disp_cc_parent_data_6, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_6), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_byte0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_byte0_clk_src = { + .cmd_rcgr = 0x8134, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_byte1_clk_src = { + .cmd_rcgr = 0x8150, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte1_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_byte2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx0_aux_clk_src = { + .cmd_rcgr = 0x81ec, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_aux_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_dptx0_link_clk_src[] = { + F(162000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), + F(270000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), + F(540000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), + F(810000, P_DP0_PHY_PLL_LINK_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_dptx0_link_clk_src = { + .cmd_rcgr = 0x819c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_link_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx0_pixel0_clk_src = { + .cmd_rcgr = 0x81bc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_pixel0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx0_pixel1_clk_src = { + .cmd_rcgr = 0x81d4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_pixel1_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx1_aux_clk_src = { + .cmd_rcgr = 0x8254, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_aux_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx1_link_clk_src = { + .cmd_rcgr = 0x8234, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_link_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx1_pixel0_clk_src = { + .cmd_rcgr = 0x8204, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_pixel0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx1_pixel1_clk_src = { + .cmd_rcgr = 0x821c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_pixel1_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx2_aux_clk_src = { + .cmd_rcgr = 0x82bc, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_aux_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx2_link_clk_src = { + .cmd_rcgr = 0x826c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_link_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx2_pixel0_clk_src = { + .cmd_rcgr = 0x828c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_pixel0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx2_pixel1_clk_src = { + .cmd_rcgr = 0x82a4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_pixel1_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx3_aux_clk_src = { + .cmd_rcgr = 0x8308, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_aux_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx3_link_clk_src = { + .cmd_rcgr = 0x82ec, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_3, + .freq_tbl = ftbl_disp_cc_mdss_dptx0_link_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_link_clk_src", + .parent_data = disp_cc_parent_data_3, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_3), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_dptx3_pixel0_clk_src = { + .cmd_rcgr = 0x82d4, + .mnd_width = 16, + .hid_width = 5, + .parent_map = disp_cc_parent_map_0, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_pixel0_clk_src", + .parent_data = disp_cc_parent_data_0, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_dp_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_esc0_clk_src = { + .cmd_rcgr = 0x816c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_4, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_esc0_clk_src", + .parent_data = disp_cc_parent_data_4, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_esc1_clk_src = { + .cmd_rcgr = 0x8184, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_4, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_esc1_clk_src", + .parent_data = disp_cc_parent_data_4, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_4), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_mdp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(85714286, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(100000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(150000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(172000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(200000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(325000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(375000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + F(500000000, P_DISP_CC_PLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = { + .cmd_rcgr = 0x80ec, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .freq_tbl = ftbl_disp_cc_mdss_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_mdp_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { + .cmd_rcgr = 0x80bc, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_pclk0_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_pixel_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { + .cmd_rcgr = 0x80d4, + .mnd_width = 8, + .hid_width = 5, + .parent_map = disp_cc_parent_map_2, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_pclk1_clk_src", + .parent_data = disp_cc_parent_data_2, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_2), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_pixel_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_mdss_rot_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(150000000, P_DISP_CC_PLL1_OUT_MAIN, 4, 0, 0), + F(200000000, P_DISP_CC_PLL1_OUT_MAIN, 3, 0, 0), + F(300000000, P_DISP_CC_PLL1_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_mdss_rot_clk_src = { + .cmd_rcgr = 0x8104, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_5, + .freq_tbl = ftbl_disp_cc_mdss_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_rot_clk_src", + .parent_data = disp_cc_parent_data_5, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_5), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = { + .cmd_rcgr = 0x811c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_vsync_clk_src", + .parent_data = disp_cc_parent_data_1, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_disp_cc_sleep_clk_src[] = { + F(32000, P_SLEEP_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 disp_cc_sleep_clk_src = { + .cmd_rcgr = 0xe060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_7, + .freq_tbl = ftbl_disp_cc_sleep_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_sleep_clk_src", + .parent_data = disp_cc_parent_data_7, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_7), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 disp_cc_xo_clk_src = { + .cmd_rcgr = 0xe044, + .mnd_width = 0, + .hid_width = 5, + .parent_map = disp_cc_parent_map_1, + .freq_tbl = ftbl_disp_cc_mdss_byte0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_xo_clk_src", + .parent_data = disp_cc_parent_data_1_ao, + .num_parents = ARRAY_SIZE(disp_cc_parent_data_1_ao), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { + .reg = 0x814c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = { + .reg = 0x8168, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte1_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_dptx0_link_div_clk_src = { + .reg = 0x81b4, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_link_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_dptx1_link_div_clk_src = { + .reg = 0x824c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_link_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_dptx2_link_div_clk_src = { + .reg = 0x8284, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_link_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div disp_cc_mdss_dptx3_link_div_clk_src = { + .reg = 0x8304, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_link_div_clk_src", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch disp_cc_mdss_ahb1_clk = { + .halt_reg = 0xa020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_ahb1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_ahb_clk = { + .halt_reg = 0x80a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x80a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_ahb_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_clk = { + .halt_reg = 0x8028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte0_intf_clk = { + .halt_reg = 0x802c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x802c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte0_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte1_clk = { + .halt_reg = 0x8030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_byte1_intf_clk = { + .halt_reg = 0x8034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_byte1_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_byte1_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_aux_clk = { + .halt_reg = 0x8058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_aux_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_crypto_clk = { + .halt_reg = 0x804c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x804c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_crypto_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_link_clk = { + .halt_reg = 0x8040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_link_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_link_intf_clk = { + .halt_reg = 0x8048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_pixel0_clk = { + .halt_reg = 0x8050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_pixel0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_pixel0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_pixel1_clk = { + .halt_reg = 0x8054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_pixel1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_pixel1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx0_usb_router_link_intf_clk = { + .halt_reg = 0x8044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx0_usb_router_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_aux_clk = { + .halt_reg = 0x8074, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8074, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_aux_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_crypto_clk = { + .halt_reg = 0x8070, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8070, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_crypto_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_link_clk = { + .halt_reg = 0x8064, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_link_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_link_intf_clk = { + .halt_reg = 0x806c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x806c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_pixel0_clk = { + .halt_reg = 0x805c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x805c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_pixel0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_pixel0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_pixel1_clk = { + .halt_reg = 0x8060, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8060, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_pixel1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx1_pixel1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx1_usb_router_link_intf_clk = { + .halt_reg = 0x8068, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8068, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx1_usb_router_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx0_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_aux_clk = { + .halt_reg = 0x808c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x808c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_aux_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_crypto_clk = { + .halt_reg = 0x8088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_crypto_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_link_clk = { + .halt_reg = 0x8080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_link_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_link_intf_clk = { + .halt_reg = 0x8084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_pixel0_clk = { + .halt_reg = 0x8078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_pixel0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_pixel0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx2_pixel1_clk = { + .halt_reg = 0x807c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x807c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx2_pixel1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx2_pixel1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx3_aux_clk = { + .halt_reg = 0x809c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x809c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_aux_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx3_crypto_clk = { + .halt_reg = 0x80a0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x80a0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_crypto_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx3_link_clk = { + .halt_reg = 0x8094, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8094, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_link_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_link_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx3_link_intf_clk = { + .halt_reg = 0x8098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_link_intf_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_link_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_dptx3_pixel0_clk = { + .halt_reg = 0x8090, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_dptx3_pixel0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_dptx3_pixel0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc0_clk = { + .halt_reg = 0x8038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_esc0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_esc1_clk = { + .halt_reg = 0x803c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x803c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_esc1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_esc1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp1_clk = { + .halt_reg = 0xa004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_mdp1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_clk = { + .halt_reg = 0x800c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_mdp_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_lut1_clk = { + .halt_reg = 0xa014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_mdp_lut1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_mdp_lut_clk = { + .halt_reg = 0x801c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x801c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_mdp_lut_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_non_gdsc_ahb_clk = { + .halt_reg = 0xc004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0xc004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_non_gdsc_ahb_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk0_clk = { + .halt_reg = 0x8004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_pclk0_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_pclk1_clk = { + .halt_reg = 0x8008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_pclk1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_pclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rot1_clk = { + .halt_reg = 0xa00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_rot1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_rot_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rot_clk = { + .halt_reg = 0x8014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_rot_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_rot_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_ahb_clk = { + .halt_reg = 0xc00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_rscc_ahb_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_rscc_vsync_clk = { + .halt_reg = 0xc008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xc008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_rscc_vsync_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_vsync1_clk = { + .halt_reg = 0xa01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_vsync1_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_mdss_vsync_clk = { + .halt_reg = 0x8024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_mdss_vsync_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_mdss_vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch disp_cc_sleep_clk = { + .halt_reg = 0xe078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xe078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "disp_cc_sleep_clk", + .parent_data = &(const struct clk_parent_data) { + .hw = &disp_cc_sleep_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x9000, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL | RETAIN_FF_ENABLE, +}; + +static struct gdsc mdss_int2_gdsc = { + .gdscr = 0xb000, + .pd = { + .name = "mdss_int2_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL | RETAIN_FF_ENABLE, +}; + +static struct clk_regmap *disp_cc_sm8450_clocks[] = { + [DISP_CC_MDSS_AHB1_CLK] = &disp_cc_mdss_ahb1_clk.clkr, + [DISP_CC_MDSS_AHB_CLK] = &disp_cc_mdss_ahb_clk.clkr, + [DISP_CC_MDSS_AHB_CLK_SRC] = &disp_cc_mdss_ahb_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_CLK] = &disp_cc_mdss_byte0_clk.clkr, + [DISP_CC_MDSS_BYTE0_CLK_SRC] = &disp_cc_mdss_byte0_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_DIV_CLK_SRC] = &disp_cc_mdss_byte0_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE0_INTF_CLK] = &disp_cc_mdss_byte0_intf_clk.clkr, + [DISP_CC_MDSS_BYTE1_CLK] = &disp_cc_mdss_byte1_clk.clkr, + [DISP_CC_MDSS_BYTE1_CLK_SRC] = &disp_cc_mdss_byte1_clk_src.clkr, + [DISP_CC_MDSS_BYTE1_DIV_CLK_SRC] = &disp_cc_mdss_byte1_div_clk_src.clkr, + [DISP_CC_MDSS_BYTE1_INTF_CLK] = &disp_cc_mdss_byte1_intf_clk.clkr, + [DISP_CC_MDSS_DPTX0_AUX_CLK] = &disp_cc_mdss_dptx0_aux_clk.clkr, + [DISP_CC_MDSS_DPTX0_AUX_CLK_SRC] = &disp_cc_mdss_dptx0_aux_clk_src.clkr, + [DISP_CC_MDSS_DPTX0_CRYPTO_CLK] = &disp_cc_mdss_dptx0_crypto_clk.clkr, + [DISP_CC_MDSS_DPTX0_LINK_CLK] = &disp_cc_mdss_dptx0_link_clk.clkr, + [DISP_CC_MDSS_DPTX0_LINK_CLK_SRC] = &disp_cc_mdss_dptx0_link_clk_src.clkr, + [DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC] = &disp_cc_mdss_dptx0_link_div_clk_src.clkr, + [DISP_CC_MDSS_DPTX0_LINK_INTF_CLK] = &disp_cc_mdss_dptx0_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX0_PIXEL0_CLK] = &disp_cc_mdss_dptx0_pixel0_clk.clkr, + [DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC] = &disp_cc_mdss_dptx0_pixel0_clk_src.clkr, + [DISP_CC_MDSS_DPTX0_PIXEL1_CLK] = &disp_cc_mdss_dptx0_pixel1_clk.clkr, + [DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC] = &disp_cc_mdss_dptx0_pixel1_clk_src.clkr, + [DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK] = + &disp_cc_mdss_dptx0_usb_router_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX1_AUX_CLK] = &disp_cc_mdss_dptx1_aux_clk.clkr, + [DISP_CC_MDSS_DPTX1_AUX_CLK_SRC] = &disp_cc_mdss_dptx1_aux_clk_src.clkr, + [DISP_CC_MDSS_DPTX1_CRYPTO_CLK] = &disp_cc_mdss_dptx1_crypto_clk.clkr, + [DISP_CC_MDSS_DPTX1_LINK_CLK] = &disp_cc_mdss_dptx1_link_clk.clkr, + [DISP_CC_MDSS_DPTX1_LINK_CLK_SRC] = &disp_cc_mdss_dptx1_link_clk_src.clkr, + [DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC] = &disp_cc_mdss_dptx1_link_div_clk_src.clkr, + [DISP_CC_MDSS_DPTX1_LINK_INTF_CLK] = &disp_cc_mdss_dptx1_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX1_PIXEL0_CLK] = &disp_cc_mdss_dptx1_pixel0_clk.clkr, + [DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC] = &disp_cc_mdss_dptx1_pixel0_clk_src.clkr, + [DISP_CC_MDSS_DPTX1_PIXEL1_CLK] = &disp_cc_mdss_dptx1_pixel1_clk.clkr, + [DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC] = &disp_cc_mdss_dptx1_pixel1_clk_src.clkr, + [DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK] = + &disp_cc_mdss_dptx1_usb_router_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX2_AUX_CLK] = &disp_cc_mdss_dptx2_aux_clk.clkr, + [DISP_CC_MDSS_DPTX2_AUX_CLK_SRC] = &disp_cc_mdss_dptx2_aux_clk_src.clkr, + [DISP_CC_MDSS_DPTX2_CRYPTO_CLK] = &disp_cc_mdss_dptx2_crypto_clk.clkr, + [DISP_CC_MDSS_DPTX2_LINK_CLK] = &disp_cc_mdss_dptx2_link_clk.clkr, + [DISP_CC_MDSS_DPTX2_LINK_CLK_SRC] = &disp_cc_mdss_dptx2_link_clk_src.clkr, + [DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC] = &disp_cc_mdss_dptx2_link_div_clk_src.clkr, + [DISP_CC_MDSS_DPTX2_LINK_INTF_CLK] = &disp_cc_mdss_dptx2_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX2_PIXEL0_CLK] = &disp_cc_mdss_dptx2_pixel0_clk.clkr, + [DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC] = &disp_cc_mdss_dptx2_pixel0_clk_src.clkr, + [DISP_CC_MDSS_DPTX2_PIXEL1_CLK] = &disp_cc_mdss_dptx2_pixel1_clk.clkr, + [DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC] = &disp_cc_mdss_dptx2_pixel1_clk_src.clkr, + [DISP_CC_MDSS_DPTX3_AUX_CLK] = &disp_cc_mdss_dptx3_aux_clk.clkr, + [DISP_CC_MDSS_DPTX3_AUX_CLK_SRC] = &disp_cc_mdss_dptx3_aux_clk_src.clkr, + [DISP_CC_MDSS_DPTX3_CRYPTO_CLK] = &disp_cc_mdss_dptx3_crypto_clk.clkr, + [DISP_CC_MDSS_DPTX3_LINK_CLK] = &disp_cc_mdss_dptx3_link_clk.clkr, + [DISP_CC_MDSS_DPTX3_LINK_CLK_SRC] = &disp_cc_mdss_dptx3_link_clk_src.clkr, + [DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC] = &disp_cc_mdss_dptx3_link_div_clk_src.clkr, + [DISP_CC_MDSS_DPTX3_LINK_INTF_CLK] = &disp_cc_mdss_dptx3_link_intf_clk.clkr, + [DISP_CC_MDSS_DPTX3_PIXEL0_CLK] = &disp_cc_mdss_dptx3_pixel0_clk.clkr, + [DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC] = &disp_cc_mdss_dptx3_pixel0_clk_src.clkr, + [DISP_CC_MDSS_ESC0_CLK] = &disp_cc_mdss_esc0_clk.clkr, + [DISP_CC_MDSS_ESC0_CLK_SRC] = &disp_cc_mdss_esc0_clk_src.clkr, + [DISP_CC_MDSS_ESC1_CLK] = &disp_cc_mdss_esc1_clk.clkr, + [DISP_CC_MDSS_ESC1_CLK_SRC] = &disp_cc_mdss_esc1_clk_src.clkr, + [DISP_CC_MDSS_MDP1_CLK] = &disp_cc_mdss_mdp1_clk.clkr, + [DISP_CC_MDSS_MDP_CLK] = &disp_cc_mdss_mdp_clk.clkr, + [DISP_CC_MDSS_MDP_CLK_SRC] = &disp_cc_mdss_mdp_clk_src.clkr, + [DISP_CC_MDSS_MDP_LUT1_CLK] = &disp_cc_mdss_mdp_lut1_clk.clkr, + [DISP_CC_MDSS_MDP_LUT_CLK] = &disp_cc_mdss_mdp_lut_clk.clkr, + [DISP_CC_MDSS_NON_GDSC_AHB_CLK] = &disp_cc_mdss_non_gdsc_ahb_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK] = &disp_cc_mdss_pclk0_clk.clkr, + [DISP_CC_MDSS_PCLK0_CLK_SRC] = &disp_cc_mdss_pclk0_clk_src.clkr, + [DISP_CC_MDSS_PCLK1_CLK] = &disp_cc_mdss_pclk1_clk.clkr, + [DISP_CC_MDSS_PCLK1_CLK_SRC] = &disp_cc_mdss_pclk1_clk_src.clkr, + [DISP_CC_MDSS_ROT1_CLK] = &disp_cc_mdss_rot1_clk.clkr, + [DISP_CC_MDSS_ROT_CLK] = &disp_cc_mdss_rot_clk.clkr, + [DISP_CC_MDSS_ROT_CLK_SRC] = &disp_cc_mdss_rot_clk_src.clkr, + [DISP_CC_MDSS_RSCC_AHB_CLK] = &disp_cc_mdss_rscc_ahb_clk.clkr, + [DISP_CC_MDSS_RSCC_VSYNC_CLK] = &disp_cc_mdss_rscc_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC1_CLK] = &disp_cc_mdss_vsync1_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK] = &disp_cc_mdss_vsync_clk.clkr, + [DISP_CC_MDSS_VSYNC_CLK_SRC] = &disp_cc_mdss_vsync_clk_src.clkr, + [DISP_CC_PLL0] = &disp_cc_pll0.clkr, + [DISP_CC_PLL1] = &disp_cc_pll1.clkr, + [DISP_CC_SLEEP_CLK] = &disp_cc_sleep_clk.clkr, + [DISP_CC_SLEEP_CLK_SRC] = &disp_cc_sleep_clk_src.clkr, + [DISP_CC_XO_CLK_SRC] = &disp_cc_xo_clk_src.clkr, +}; + +static const struct qcom_reset_map disp_cc_sm8450_resets[] = { + [DISP_CC_MDSS_CORE_BCR] = { 0x8000 }, + [DISP_CC_MDSS_CORE_INT2_BCR] = { 0xa000 }, + [DISP_CC_MDSS_RSCC_BCR] = { 0xc000 }, +}; + +static struct gdsc *disp_cc_sm8450_gdscs[] = { + [MDSS_GDSC] = &mdss_gdsc, + [MDSS_INT2_GDSC] = &mdss_int2_gdsc, +}; + +static const struct regmap_config disp_cc_sm8450_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x11008, + .fast_io = true, +}; + +static struct qcom_cc_desc disp_cc_sm8450_desc = { + .config = &disp_cc_sm8450_regmap_config, + .clks = disp_cc_sm8450_clocks, + .num_clks = ARRAY_SIZE(disp_cc_sm8450_clocks), + .resets = disp_cc_sm8450_resets, + .num_resets = ARRAY_SIZE(disp_cc_sm8450_resets), + .gdscs = disp_cc_sm8450_gdscs, + .num_gdscs = ARRAY_SIZE(disp_cc_sm8450_gdscs), +}; + +static const struct of_device_id disp_cc_sm8450_match_table[] = { + { .compatible = "qcom,sm8450-dispcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, disp_cc_sm8450_match_table); + +static void disp_cc_sm8450_pm_runtime_disable(void *data) +{ + pm_runtime_disable(data); +} + +static int disp_cc_sm8450_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + pm_runtime_enable(&pdev->dev); + + ret = devm_add_action_or_reset(&pdev->dev, disp_cc_sm8450_pm_runtime_disable, &pdev->dev); + if (ret) + return ret; + + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret) + return ret; + + regmap = qcom_cc_map(pdev, &disp_cc_sm8450_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_lucid_evo_pll_configure(&disp_cc_pll0, regmap, &disp_cc_pll0_config); + clk_lucid_evo_pll_configure(&disp_cc_pll1, regmap, &disp_cc_pll1_config); + + /* Enable clock gating for MDP clocks */ + regmap_update_bits(regmap, DISP_CC_MISC_CMD, 0x10, 0x10); + + /* + * Keep clocks always enabled: + * disp_cc_xo_clk + */ + regmap_update_bits(regmap, 0xe05c, BIT(0), BIT(0)); + + ret = qcom_cc_really_probe(pdev, &disp_cc_sm8450_desc, regmap); + + pm_runtime_put(&pdev->dev); + + return ret; +} + +static struct platform_driver disp_cc_sm8450_driver = { + .probe = disp_cc_sm8450_probe, + .driver = { + .name = "disp_cc-sm8450", + .of_match_table = disp_cc_sm8450_match_table, + }, +}; + +static int __init disp_cc_sm8450_init(void) +{ + return platform_driver_register(&disp_cc_sm8450_driver); +} +subsys_initcall(disp_cc_sm8450_init); + +static void __exit disp_cc_sm8450_exit(void) +{ + platform_driver_unregister(&disp_cc_sm8450_driver); +} +module_exit(disp_cc_sm8450_exit); + +MODULE_DESCRIPTION("QTI DISPCC SM8450 Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/gcc-msm8660.c b/drivers/clk/qcom/gcc-msm8660.c index 94ea2d84d1b1fa953bdef7baea8adc64d9cd9004..a9eb6a9ac4454ee68b545612d0860dde95300890 100644 --- a/drivers/clk/qcom/gcc-msm8660.c +++ b/drivers/clk/qcom/gcc-msm8660.c @@ -34,7 +34,9 @@ static struct clk_pll pll8 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll8", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -45,7 +47,9 @@ static struct clk_regmap pll8_vote = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "pll8_vote", - .parent_names = (const char *[]){ "pll8" }, + .parent_hws = (const struct clk_hw*[]){ + &pll8.clkr.hw + }, .num_parents = 1, .ops = &clk_pll_vote_ops, }, @@ -62,9 +66,9 @@ static const struct parent_map gcc_pxo_pll8_map[] = { { P_PLL8, 3 } }; -static const char * const gcc_pxo_pll8[] = { - "pxo", - "pll8_vote", +static const struct clk_parent_data gcc_pxo_pll8[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .hw = &pll8_vote.hw }, }; static const struct parent_map gcc_pxo_pll8_cxo_map[] = { @@ -73,10 +77,10 @@ static const struct parent_map gcc_pxo_pll8_cxo_map[] = { { P_CXO, 5 } }; -static const char * const gcc_pxo_pll8_cxo[] = { - "pxo", - "pll8_vote", - "cxo", +static const struct clk_parent_data gcc_pxo_pll8_cxo[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .hw = &pll8_vote.hw }, + { .fw_name = "cxo", .name = "cxo_board" }, }; static struct freq_tbl clk_tbl_gsbi_uart[] = { @@ -122,8 +126,8 @@ static struct clk_rcg gsbi1_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi1_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -138,8 +142,8 @@ static struct clk_branch gsbi1_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi1_uart_clk", - .parent_names = (const char *[]){ - "gsbi1_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi1_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -173,8 +177,8 @@ static struct clk_rcg gsbi2_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi2_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -189,8 +193,8 @@ static struct clk_branch gsbi2_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi2_uart_clk", - .parent_names = (const char *[]){ - "gsbi2_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi2_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -224,8 +228,8 @@ static struct clk_rcg gsbi3_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi3_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -240,8 +244,8 @@ static struct clk_branch gsbi3_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi3_uart_clk", - .parent_names = (const char *[]){ - "gsbi3_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi3_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -275,8 +279,8 @@ static struct clk_rcg gsbi4_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi4_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -291,8 +295,8 @@ static struct clk_branch gsbi4_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi4_uart_clk", - .parent_names = (const char *[]){ - "gsbi4_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi4_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -326,8 +330,8 @@ static struct clk_rcg gsbi5_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi5_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -342,8 +346,8 @@ static struct clk_branch gsbi5_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi5_uart_clk", - .parent_names = (const char *[]){ - "gsbi5_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi5_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -377,8 +381,8 @@ static struct clk_rcg gsbi6_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi6_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -393,8 +397,8 @@ static struct clk_branch gsbi6_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi6_uart_clk", - .parent_names = (const char *[]){ - "gsbi6_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi6_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -428,8 +432,8 @@ static struct clk_rcg gsbi7_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi7_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -444,8 +448,8 @@ static struct clk_branch gsbi7_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi7_uart_clk", - .parent_names = (const char *[]){ - "gsbi7_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi7_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -479,8 +483,8 @@ static struct clk_rcg gsbi8_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi8_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -495,7 +499,9 @@ static struct clk_branch gsbi8_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi8_uart_clk", - .parent_names = (const char *[]){ "gsbi8_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi8_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -528,8 +534,8 @@ static struct clk_rcg gsbi9_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi9_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -544,7 +550,9 @@ static struct clk_branch gsbi9_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi9_uart_clk", - .parent_names = (const char *[]){ "gsbi9_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi9_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -577,8 +585,8 @@ static struct clk_rcg gsbi10_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi10_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -593,7 +601,9 @@ static struct clk_branch gsbi10_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi10_uart_clk", - .parent_names = (const char *[]){ "gsbi10_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi10_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -626,8 +636,8 @@ static struct clk_rcg gsbi11_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi11_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -642,7 +652,9 @@ static struct clk_branch gsbi11_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi11_uart_clk", - .parent_names = (const char *[]){ "gsbi11_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi11_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -675,8 +687,8 @@ static struct clk_rcg gsbi12_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi12_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -691,7 +703,9 @@ static struct clk_branch gsbi12_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi12_uart_clk", - .parent_names = (const char *[]){ "gsbi12_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi12_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -737,8 +751,8 @@ static struct clk_rcg gsbi1_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi1_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -753,7 +767,9 @@ static struct clk_branch gsbi1_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi1_qup_clk", - .parent_names = (const char *[]){ "gsbi1_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi1_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -786,8 +802,8 @@ static struct clk_rcg gsbi2_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi2_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -802,7 +818,9 @@ static struct clk_branch gsbi2_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi2_qup_clk", - .parent_names = (const char *[]){ "gsbi2_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi2_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -835,8 +853,8 @@ static struct clk_rcg gsbi3_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi3_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -851,7 +869,9 @@ static struct clk_branch gsbi3_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi3_qup_clk", - .parent_names = (const char *[]){ "gsbi3_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi3_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -884,8 +904,8 @@ static struct clk_rcg gsbi4_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi4_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -900,7 +920,9 @@ static struct clk_branch gsbi4_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi4_qup_clk", - .parent_names = (const char *[]){ "gsbi4_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi4_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -933,8 +955,8 @@ static struct clk_rcg gsbi5_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi5_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -949,7 +971,9 @@ static struct clk_branch gsbi5_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi5_qup_clk", - .parent_names = (const char *[]){ "gsbi5_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi5_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -982,8 +1006,8 @@ static struct clk_rcg gsbi6_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi6_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -998,7 +1022,9 @@ static struct clk_branch gsbi6_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi6_qup_clk", - .parent_names = (const char *[]){ "gsbi6_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi6_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1031,8 +1057,8 @@ static struct clk_rcg gsbi7_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi7_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1047,7 +1073,9 @@ static struct clk_branch gsbi7_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi7_qup_clk", - .parent_names = (const char *[]){ "gsbi7_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi7_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1080,8 +1108,8 @@ static struct clk_rcg gsbi8_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi8_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1096,7 +1124,9 @@ static struct clk_branch gsbi8_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi8_qup_clk", - .parent_names = (const char *[]){ "gsbi8_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi8_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1129,8 +1159,8 @@ static struct clk_rcg gsbi9_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi9_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1145,7 +1175,9 @@ static struct clk_branch gsbi9_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi9_qup_clk", - .parent_names = (const char *[]){ "gsbi9_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi9_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1178,8 +1210,8 @@ static struct clk_rcg gsbi10_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi10_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1194,7 +1226,9 @@ static struct clk_branch gsbi10_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi10_qup_clk", - .parent_names = (const char *[]){ "gsbi10_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi10_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1227,8 +1261,8 @@ static struct clk_rcg gsbi11_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi11_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1243,7 +1277,9 @@ static struct clk_branch gsbi11_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi11_qup_clk", - .parent_names = (const char *[]){ "gsbi11_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi11_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1276,8 +1312,8 @@ static struct clk_rcg gsbi12_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi12_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1292,7 +1328,9 @@ static struct clk_branch gsbi12_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi12_qup_clk", - .parent_names = (const char *[]){ "gsbi12_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi12_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1338,8 +1376,8 @@ static struct clk_rcg gp0_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp0_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1354,7 +1392,9 @@ static struct clk_branch gp0_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp0_clk", - .parent_names = (const char *[]){ "gp0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp0_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1387,8 +1427,8 @@ static struct clk_rcg gp1_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp1_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1403,7 +1443,9 @@ static struct clk_branch gp1_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp1_clk", - .parent_names = (const char *[]){ "gp1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1436,8 +1478,8 @@ static struct clk_rcg gp2_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp2_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1452,7 +1494,9 @@ static struct clk_branch gp2_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp2_clk", - .parent_names = (const char *[]){ "gp2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1488,8 +1532,8 @@ static struct clk_rcg prng_src = { .clkr.hw = { .init = &(struct clk_init_data){ .name = "prng_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, }, @@ -1504,7 +1548,9 @@ static struct clk_branch prng_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "prng_clk", - .parent_names = (const char *[]){ "prng_src" }, + .parent_hws = (const struct clk_hw*[]){ + &prng_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, }, @@ -1547,8 +1593,8 @@ static struct clk_rcg sdc1_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc1_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1562,7 +1608,9 @@ static struct clk_branch sdc1_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc1_clk", - .parent_names = (const char *[]){ "sdc1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1595,8 +1643,8 @@ static struct clk_rcg sdc2_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc2_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1610,7 +1658,9 @@ static struct clk_branch sdc2_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc2_clk", - .parent_names = (const char *[]){ "sdc2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1643,8 +1693,8 @@ static struct clk_rcg sdc3_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc3_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1658,7 +1708,9 @@ static struct clk_branch sdc3_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc3_clk", - .parent_names = (const char *[]){ "sdc3_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc3_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1691,8 +1743,8 @@ static struct clk_rcg sdc4_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc4_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1706,7 +1758,9 @@ static struct clk_branch sdc4_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc4_clk", - .parent_names = (const char *[]){ "sdc4_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc4_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1739,8 +1793,8 @@ static struct clk_rcg sdc5_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc5_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1754,7 +1808,9 @@ static struct clk_branch sdc5_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc5_clk", - .parent_names = (const char *[]){ "sdc5_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc5_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1792,8 +1848,8 @@ static struct clk_rcg tsif_ref_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "tsif_ref_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1808,7 +1864,9 @@ static struct clk_branch tsif_ref_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "tsif_ref_clk", - .parent_names = (const char *[]){ "tsif_ref_src" }, + .parent_hws = (const struct clk_hw*[]){ + &tsif_ref_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1846,8 +1904,8 @@ static struct clk_rcg usb_hs1_xcvr_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_hs1_xcvr_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1862,7 +1920,9 @@ static struct clk_branch usb_hs1_xcvr_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_hs1_xcvr_clk", - .parent_names = (const char *[]){ "usb_hs1_xcvr_src" }, + .parent_hws = (const struct clk_hw*[]){ + &usb_hs1_xcvr_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1895,16 +1955,14 @@ static struct clk_rcg usb_fs1_xcvr_fs_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_fs1_xcvr_fs_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, } }; -static const char * const usb_fs1_xcvr_fs_src_p[] = { "usb_fs1_xcvr_fs_src" }; - static struct clk_branch usb_fs1_xcvr_fs_clk = { .halt_reg = 0x2fcc, .halt_bit = 15, @@ -1913,7 +1971,9 @@ static struct clk_branch usb_fs1_xcvr_fs_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_fs1_xcvr_fs_clk", - .parent_names = usb_fs1_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs1_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1928,7 +1988,9 @@ static struct clk_branch usb_fs1_system_clk = { .enable_reg = 0x296c, .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ - .parent_names = usb_fs1_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs1_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .name = "usb_fs1_system_clk", .ops = &clk_branch_ops, @@ -1962,16 +2024,14 @@ static struct clk_rcg usb_fs2_xcvr_fs_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_xcvr_fs_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, } }; -static const char * const usb_fs2_xcvr_fs_src_p[] = { "usb_fs2_xcvr_fs_src" }; - static struct clk_branch usb_fs2_xcvr_fs_clk = { .halt_reg = 0x2fcc, .halt_bit = 12, @@ -1980,7 +2040,9 @@ static struct clk_branch usb_fs2_xcvr_fs_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_xcvr_fs_clk", - .parent_names = usb_fs2_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs2_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1996,7 +2058,9 @@ static struct clk_branch usb_fs2_system_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_system_clk", - .parent_names = usb_fs2_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs2_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2703,17 +2767,6 @@ MODULE_DEVICE_TABLE(of, gcc_msm8660_match_table); static int gcc_msm8660_probe(struct platform_device *pdev) { - int ret; - struct device *dev = &pdev->dev; - - ret = qcom_cc_register_board_clk(dev, "cxo_board", "cxo", 19200000); - if (ret) - return ret; - - ret = qcom_cc_register_board_clk(dev, "pxo_board", "pxo", 27000000); - if (ret) - return ret; - return qcom_cc_probe(pdev, &gcc_msm8660_desc); } diff --git a/drivers/clk/qcom/gcc-msm8909.c b/drivers/clk/qcom/gcc-msm8909.c new file mode 100644 index 0000000000000000000000000000000000000000..2a00b11ce2cddf1ba458540b73f65f765d31b194 --- /dev/null +++ b/drivers/clk/qcom/gcc-msm8909.c @@ -0,0 +1,2731 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Kernkonzept GmbH. + * + * Based on gcc-msm8916.c: + * Copyright 2015 Linaro Limited + * adapted with data from clock-gcc-8909.c in Qualcomm's msm-3.18 release: + * Copyright (c) 2014-2016, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +/* Need to match the order of clocks in DT binding */ +enum { + DT_XO, + DT_SLEEP_CLK, + DT_DSI0PLL, + DT_DSI0PLL_BYTE, +}; + +enum { + P_XO, + P_SLEEP_CLK, + P_GPLL0, + P_GPLL1, + P_GPLL2, + P_BIMC, + P_DSI0PLL, + P_DSI0PLL_BYTE, +}; + +static const struct parent_map gcc_xo_map[] = { + { P_XO, 0 }, +}; + +static const struct clk_parent_data gcc_xo_data[] = { + { .index = DT_XO }, +}; + +static const struct clk_parent_data gcc_sleep_clk_data[] = { + { .index = DT_SLEEP_CLK }, +}; + +static struct clk_alpha_pll gpll0_early = { + .offset = 0x21000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x45000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gpll0_early", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_fixed_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv gpll0 = { + .offset = 0x21000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr.hw.init = &(struct clk_init_data) { + .name = "gpll0", + .parent_hws = (const struct clk_hw*[]) { + &gpll0_early.clkr.hw, + }, + .num_parents = 1, + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_postdiv_ro_ops, + }, +}; + +static struct clk_pll gpll1 = { + .l_reg = 0x20004, + .m_reg = 0x20008, + .n_reg = 0x2000c, + .config_reg = 0x20010, + .mode_reg = 0x20000, + .status_reg = 0x2001c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gpll1", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll1_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data) { + .name = "gpll1_vote", + .parent_hws = (const struct clk_hw*[]) { + &gpll1.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_alpha_pll gpll2_early = { + .offset = 0x25000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x45000, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data) { + .name = "gpll2_early", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_fixed_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv gpll2 = { + .offset = 0x25000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr.hw.init = &(struct clk_init_data) { + .name = "gpll2", + .parent_hws = (const struct clk_hw*[]) { + &gpll2_early.clkr.hw, + }, + .num_parents = 1, + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_postdiv_ro_ops, + }, +}; + +static struct clk_alpha_pll bimc_pll_early = { + .offset = 0x23000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x45000, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data) { + .name = "bimc_pll_early", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_fixed_ops, + }, + }, +}; + +static struct clk_alpha_pll_postdiv bimc_pll = { + .offset = 0x23000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr.hw.init = &(struct clk_init_data) { + .name = "bimc_pll", + .parent_hws = (const struct clk_hw*[]) { + &bimc_pll_early.clkr.hw, + }, + .num_parents = 1, + /* Avoid rate changes for shared clock */ + .ops = &clk_alpha_pll_postdiv_ro_ops, + }, +}; + +static const struct parent_map gcc_xo_gpll0_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, +}; + +static const struct clk_parent_data gcc_xo_gpll0_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_xo_gpll0_bimc_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_BIMC, 2 }, +}; + +static const struct clk_parent_data gcc_xo_gpll0_bimc_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &bimc_pll.clkr.hw }, +}; + +static const struct freq_tbl ftbl_apss_ahb_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 apss_ahb_clk_src = { + .cmd_rcgr = 0x46000, + .hid_width = 5, + .freq_tbl = ftbl_apss_ahb_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "apss_ahb_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 bimc_ddr_clk_src = { + .cmd_rcgr = 0x32004, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_bimc_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "bimc_ddr_clk_src", + .parent_data = gcc_xo_gpll0_bimc_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc_data), + .ops = &clk_rcg2_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_rcg2 bimc_gpu_clk_src = { + .cmd_rcgr = 0x31028, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_bimc_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "bimc_gpu_clk_src", + .parent_data = gcc_xo_gpll0_bimc_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc_data), + .ops = &clk_rcg2_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, +}; + +static const struct freq_tbl ftbl_blsp_i2c_apps_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x0200c, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup1_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x03000, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup2_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x04000, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup3_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x05000, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup4_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = { + .cmd_rcgr = 0x06000, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup5_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = { + .cmd_rcgr = 0x07000, + .hid_width = 5, + .freq_tbl = ftbl_blsp_i2c_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup6_i2c_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_blsp_spi_apps_clk_src[] = { + F(960000, P_XO, 10, 1, 2), + F(4800000, P_XO, 4, 0, 0), + F(9600000, P_XO, 2, 0, 0), + F(16000000, P_GPLL0, 10, 1, 5), + F(19200000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0, 16, 1, 2), + F(50000000, P_GPLL0, 16, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x02024, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup1_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x03014, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup2_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x04024, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup3_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x05024, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup4_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = { + .cmd_rcgr = 0x06024, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup5_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = { + .cmd_rcgr = 0x07024, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_blsp_spi_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_qup6_spi_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_blsp_uart_apps_clk_src[] = { + F(3686400, P_GPLL0, 1, 72, 15625), + F(7372800, P_GPLL0, 1, 144, 15625), + F(14745600, P_GPLL0, 1, 288, 15625), + F(16000000, P_GPLL0, 10, 1, 5), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0, 1, 3, 100), + F(25000000, P_GPLL0, 16, 1, 2), + F(32000000, P_GPLL0, 1, 1, 25), + F(40000000, P_GPLL0, 1, 1, 20), + F(46400000, P_GPLL0, 1, 29, 500), + F(48000000, P_GPLL0, 1, 3, 50), + F(51200000, P_GPLL0, 1, 8, 125), + F(56000000, P_GPLL0, 1, 7, 100), + F(58982400, P_GPLL0, 1, 1152, 15625), + F(60000000, P_GPLL0, 1, 3, 40), + { } +}; + +static struct clk_rcg2 blsp1_uart1_apps_clk_src = { + .cmd_rcgr = 0x02044, + .hid_width = 5, + .mnd_width = 16, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_uart1_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 blsp1_uart2_apps_clk_src = { + .cmd_rcgr = 0x03034, + .hid_width = 5, + .mnd_width = 16, + .freq_tbl = ftbl_blsp_uart_apps_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "blsp1_uart2_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_byte0_map[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 1 }, +}; + +static const struct clk_parent_data gcc_byte_data[] = { + { .index = DT_XO }, + { .index = DT_DSI0PLL_BYTE }, +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x4d044, + .hid_width = 5, + .parent_map = gcc_byte0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "byte0_clk_src", + .parent_data = gcc_byte_data, + .num_parents = ARRAY_SIZE(gcc_byte_data), + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT, + } +}; + +static const struct freq_tbl ftbl_camss_gp_clk_src[] = { + F(100000000, P_GPLL0, 8, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 camss_gp0_clk_src = { + .cmd_rcgr = 0x54000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_camss_gp_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "camss_gp0_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 camss_gp1_clk_src = { + .cmd_rcgr = 0x55000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_camss_gp_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "camss_gp1_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_camss_top_ahb_clk_src[] = { + F(40000000, P_GPLL0, 10, 1, 2), + F(80000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 camss_top_ahb_clk_src = { + .cmd_rcgr = 0x5a000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_camss_top_ahb_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "camss_top_ahb_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_crypto_clk_src[] = { + F(50000000, P_GPLL0, 16, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + { } +}; + +static struct clk_rcg2 crypto_clk_src = { + .cmd_rcgr = 0x16004, + .hid_width = 5, + .freq_tbl = ftbl_crypto_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "crypto_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_csi_clk_src[] = { + F(100000000, P_GPLL0, 8, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x4e020, + .hid_width = 5, + .freq_tbl = ftbl_csi_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "csi0_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_map), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x4f020, + .hid_width = 5, + .freq_tbl = ftbl_csi_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "csi1_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_csi_phytimer_clk_src[] = { + F(100000000, P_GPLL0, 8, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x4e000, + .hid_width = 5, + .freq_tbl = ftbl_csi_phytimer_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "csi0phytimer_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_esc0_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x4d05c, + .hid_width = 5, + .freq_tbl = ftbl_esc0_clk_src, + .parent_map = gcc_xo_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "esc0_clk_src", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_gfx3d_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL1, 2 }, +}; + +static const struct clk_parent_data gcc_gfx3d_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll1_vote.hw }, +}; + +static const struct freq_tbl ftbl_gfx3d_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 16, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(177780000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266670000, P_GPLL0, 3, 0, 0), + F(307200000, P_GPLL1, 4, 0, 0), + F(409600000, P_GPLL1, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gfx3d_clk_src = { + .cmd_rcgr = 0x59000, + .hid_width = 5, + .freq_tbl = ftbl_gfx3d_clk_src, + .parent_map = gcc_gfx3d_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gfx3d_clk_src", + .parent_data = gcc_gfx3d_data, + .num_parents = ARRAY_SIZE(gcc_gfx3d_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_gp_clk_src[] = { + F(150000, P_XO, 1, 1, 128), + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gp1_clk_src = { + .cmd_rcgr = 0x08004, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_gp_clk_src, + .parent_map = gcc_xo_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gp1_clk_src", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 gp2_clk_src = { + .cmd_rcgr = 0x09004, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_gp_clk_src, + .parent_map = gcc_xo_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gp2_clk_src", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 gp3_clk_src = { + .cmd_rcgr = 0x0a004, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_gp_clk_src, + .parent_map = gcc_xo_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gp3_clk_src", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_mclk_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL2, 3 }, +}; + +static const struct clk_parent_data gcc_mclk_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll2.clkr.hw }, +}; + +static const struct freq_tbl ftbl_mclk_clk_src[] = { + F(24000000, P_GPLL2, 1, 1, 33), + F(66667000, P_GPLL0, 12, 0, 0), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x52000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_mclk_clk_src, + .parent_map = gcc_mclk_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "mclk0_clk_src", + .parent_data = gcc_mclk_data, + .num_parents = ARRAY_SIZE(gcc_mclk_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x53000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_mclk_clk_src, + .parent_map = gcc_mclk_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "mclk1_clk_src", + .parent_data = gcc_mclk_data, + .num_parents = ARRAY_SIZE(gcc_mclk_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_mdp_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL1, 3 }, +}; + +static const struct clk_parent_data gcc_mdp_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll1_vote.hw }, +}; + +static const struct freq_tbl ftbl_mdp_clk_src[] = { + F(50000000, P_GPLL0, 16, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(177780000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266670000, P_GPLL0, 3, 0, 0), + F(307200000, P_GPLL1, 4, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x4d014, + .hid_width = 5, + .freq_tbl = ftbl_mdp_clk_src, + .parent_map = gcc_mdp_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "mdp_clk_src", + .parent_data = gcc_mdp_data, + .num_parents = ARRAY_SIZE(gcc_mdp_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_pclk0_map[] = { + { P_XO, 0 }, + { P_DSI0PLL, 1 }, +}; + +static const struct clk_parent_data gcc_pclk_data[] = { + { .index = DT_XO }, + { .index = DT_DSI0PLL }, +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x4d000, + .hid_width = 5, + .mnd_width = 8, + .parent_map = gcc_pclk0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "pclk0_clk_src", + .parent_data = gcc_pclk_data, + .num_parents = ARRAY_SIZE(gcc_pclk_data), + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + } +}; + +static struct clk_rcg2 pcnoc_bfdcd_clk_src = { + .cmd_rcgr = 0x27000, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_bimc_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "pcnoc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_bimc_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc_data), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_pdm2_clk_src[] = { + F(64000000, P_GPLL0, 12.5, 0, 0), + { } +}; + +static struct clk_rcg2 pdm2_clk_src = { + .cmd_rcgr = 0x44010, + .hid_width = 5, + .freq_tbl = ftbl_pdm2_clk_src, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "pdm2_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_2_apps_clk[] = { + F(144000, P_XO, 16, 3, 25), + F(400000, P_XO, 12, 1, 4), + F(20000000, P_GPLL0, 10, 1, 4), + F(25000000, P_GPLL0, 16, 1, 2), + F(50000000, P_GPLL0, 16, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(177770000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 sdcc1_apps_clk_src = { + .cmd_rcgr = 0x42004, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_gcc_sdcc1_2_apps_clk, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "sdcc1_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_floor_ops, + } +}; + +static struct clk_rcg2 sdcc2_apps_clk_src = { + .cmd_rcgr = 0x43004, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_gcc_sdcc1_2_apps_clk, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "sdcc2_apps_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_floor_ops, + } +}; + +static struct clk_rcg2 system_noc_bfdcd_clk_src = { + .cmd_rcgr = 0x26004, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_bimc_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "system_noc_bfdcd_clk_src", + .parent_data = gcc_xo_gpll0_bimc_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc_data), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = { + F(57140000, P_GPLL0, 14, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hs_system_clk_src = { + .cmd_rcgr = 0x41010, + .hid_width = 5, + .freq_tbl = ftbl_gcc_usb_hs_system_clk, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "usb_hs_system_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct parent_map gcc_vcodec0_map[] = { + { P_XO, 0 }, + { P_GPLL0, 1 }, + { P_GPLL1, 3 }, +}; + +static const struct clk_parent_data gcc_vcodec0_data[] = { + { .index = DT_XO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll1_vote.hw }, +}; + +static const struct freq_tbl ftbl_vcodec0_clk_src[] = { + F(133330000, P_GPLL0, 6, 0, 0), + F(266670000, P_GPLL0, 3, 0, 0), + F(307200000, P_GPLL1, 4, 0, 0), + { } +}; + +static struct clk_rcg2 vcodec0_clk_src = { + .cmd_rcgr = 0x4c000, + .hid_width = 5, + .mnd_width = 8, + .freq_tbl = ftbl_vcodec0_clk_src, + .parent_map = gcc_vcodec0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "vcodec0_clk_src", + .parent_data = gcc_vcodec0_data, + .num_parents = ARRAY_SIZE(gcc_vcodec0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_gcc_camss_vfe0_clk[] = { + F(50000000, P_GPLL0, 16, 0, 0), + F(80000000, P_GPLL0, 10, 0, 0), + F(100000000, P_GPLL0, 8, 0, 0), + F(133330000, P_GPLL0, 6, 0, 0), + F(160000000, P_GPLL0, 5, 0, 0), + F(177780000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 4, 0, 0), + F(266670000, P_GPLL0, 3, 0, 0), + F(320000000, P_GPLL0, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x58000, + .hid_width = 5, + .freq_tbl = ftbl_gcc_camss_vfe0_clk, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "vfe0_clk_src", + .parent_data = gcc_xo_gpll0_data, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_data), + .ops = &clk_rcg2_ops, + } +}; + +static const struct freq_tbl ftbl_vsync_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x4d02c, + .hid_width = 5, + .freq_tbl = ftbl_vsync_clk_src, + .parent_map = gcc_xo_map, + .clkr.hw.init = &(struct clk_init_data) { + .name = "vsync_clk_src", + .parent_data = gcc_xo_data, + .num_parents = ARRAY_SIZE(gcc_xo_data), + .ops = &clk_rcg2_ops, + } +}; + +static struct clk_branch gcc_apss_tcu_clk = { + .halt_reg = 0x12018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data) { + .name = "gcc_apss_tcu_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_ddr_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_blsp1_ahb_clk = { + .halt_reg = 0x01008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_blsp1_sleep_clk = { + .halt_reg = 0x01004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_sleep_clk", + .parent_data = gcc_sleep_clk_data, + .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x1300c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data) { + .name = "gcc_boot_rom_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_crypto_clk = { + .halt_reg = 0x1601c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data) { + .name = "gcc_crypto_clk", + .parent_hws = (const struct clk_hw*[]) { + &crypto_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_crypto_ahb_clk = { + .halt_reg = 0x16024, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_crypto_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_crypto_axi_clk = { + .halt_reg = 0x16020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data) { + .name = "gcc_crypto_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_gfx_tbu_clk = { + .halt_reg = 0x12010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gfx_tbu_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_ddr_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_gfx_tcu_clk = { + .halt_reg = 0x12020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gfx_tcu_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_ddr_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_gtcu_ahb_clk = { + .halt_reg = 0x12044, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gtcu_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_mdp_tbu_clk = { + .halt_reg = 0x1201c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdp_tbu_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x13004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x45004, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data) { + .name = "gcc_prng_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_smmu_cfg_clk = { + .halt_reg = 0x12038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data) { + .name = "gcc_smmu_cfg_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_venus_tbu_clk = { + .halt_reg = 0x12014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus_tbu_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_vfe_tbu_clk = { + .halt_reg = 0x1203c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x4500c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data) { + .name = "gcc_vfe_tbu_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_bimc_gfx_clk = { + .halt_reg = 0x31024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_bimc_gfx_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_gpu_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_bimc_gpu_clk = { + .halt_reg = 0x31040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_bimc_gpu_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_gpu_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { + .halt_reg = 0x02008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x02008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup1_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup1_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { + .halt_reg = 0x03010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x03010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup2_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup2_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { + .halt_reg = 0x04020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x04020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup3_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup3_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { + .halt_reg = 0x05020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x05020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup4_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup4_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { + .halt_reg = 0x06020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x06020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup5_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup5_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = { + .halt_reg = 0x07020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x07020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup6_i2c_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup6_i2c_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { + .halt_reg = 0x02004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x02004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup1_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup1_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { + .halt_reg = 0x0300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup2_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup2_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { + .halt_reg = 0x0401c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0401c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup3_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup3_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { + .halt_reg = 0x0501c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup4_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup4_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { + .halt_reg = 0x0601c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0601c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup5_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup5_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { + .halt_reg = 0x0701c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_qup6_spi_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_qup6_spi_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_uart1_apps_clk = { + .halt_reg = 0x0203c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0203c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_uart1_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_uart1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_blsp1_uart2_apps_clk = { + .halt_reg = 0x0302c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0302c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_blsp1_uart2_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &blsp1_uart2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_ahb_clk = { + .halt_reg = 0x5a014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5a014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_camss_csi0_clk = { + .halt_reg = 0x4e03c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi0_ahb_clk = { + .halt_reg = 0x4e040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi0phy_clk = { + .halt_reg = 0x4e048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0phy_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi0phytimer_clk = { + .halt_reg = 0x4e01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0phytimer_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi0phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi0pix_clk = { + .halt_reg = 0x4e058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0pix_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi0rdi_clk = { + .halt_reg = 0x4e050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi0rdi_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi1_clk = { + .halt_reg = 0x4f03c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi1_ahb_clk = { + .halt_reg = 0x4f040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi1phy_clk = { + .halt_reg = 0x4f048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1phy_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi1pix_clk = { + .halt_reg = 0x4f058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1pix_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi1rdi_clk = { + .halt_reg = 0x4f050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi1rdi_clk", + .parent_hws = (const struct clk_hw*[]) { + &csi1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_csi_vfe0_clk = { + .halt_reg = 0x58050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_csi_vfe0_clk", + .parent_hws = (const struct clk_hw*[]) { + &vfe0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_gp0_clk = { + .halt_reg = 0x54018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x54018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_gp0_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_gp0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_gp1_clk = { + .halt_reg = 0x55018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x55018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_gp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_ispif_ahb_clk = { + .halt_reg = 0x50004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x50004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_ispif_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_mclk0_clk = { + .halt_reg = 0x52018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_mclk0_clk", + .parent_hws = (const struct clk_hw*[]) { + &mclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_mclk1_clk = { + .halt_reg = 0x53018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x53018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_mclk1_clk", + .parent_hws = (const struct clk_hw*[]) { + &mclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_top_ahb_clk = { + .halt_reg = 0x56004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x56004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_top_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_vfe0_clk = { + .halt_reg = 0x58038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe0_clk", + .parent_hws = (const struct clk_hw*[]) { + &vfe0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_vfe_ahb_clk = { + .halt_reg = 0x58044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_camss_vfe_axi_clk = { + .halt_reg = 0x58048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_camss_vfe_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x08000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x08000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw*[]) { + &gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x09000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x09000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw*[]) { + &gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x0a000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x0a000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw*[]) { + &gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mdss_ahb_clk = { + .halt_reg = 0x4d07c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_mdss_axi_clk = { + .halt_reg = 0x4d080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_mdss_byte0_clk = { + .halt_reg = 0x4d094, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d094, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_byte0_clk", + .parent_hws = (const struct clk_hw*[]) { + &byte0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mdss_esc0_clk = { + .halt_reg = 0x4d098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_esc0_clk", + .parent_hws = (const struct clk_hw*[]) { + &esc0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mdss_mdp_clk = { + .halt_reg = 0x4d088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_mdp_clk", + .parent_hws = (const struct clk_hw*[]) { + &mdp_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mdss_pclk0_clk = { + .halt_reg = 0x4d084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_pclk0_clk", + .parent_hws = (const struct clk_hw*[]) { + &pclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mdss_vsync_clk = { + .halt_reg = 0x4d090, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mdss_vsync_clk", + .parent_hws = (const struct clk_hw*[]) { + &vsync_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_mss_cfg_ahb_clk = { + .halt_reg = 0x49000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mss_cfg_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_mss_q6_bimc_axi_clk = { + .halt_reg = 0x49004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x49004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_mss_q6_bimc_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &bimc_ddr_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_oxili_ahb_clk = { + .halt_reg = 0x59028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x59028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_oxili_gfx3d_clk = { + .halt_reg = 0x59020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x59020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_oxili_gfx3d_clk", + .parent_hws = (const struct clk_hw*[]) { + &gfx3d_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x4400c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4400c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_pdm2_clk", + .parent_hws = (const struct clk_hw*[]) { + &pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x44004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x44004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_pdm_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x4201c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc1_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x42018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x42018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc1_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x4301c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4301c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc2_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x43018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x43018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_sdcc2_apps_clk", + .parent_hws = (const struct clk_hw*[]) { + &sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_usb2a_phy_sleep_clk = { + .halt_reg = 0x4102c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4102c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb2a_phy_sleep_clk", + .parent_data = gcc_sleep_clk_data, + .num_parents = ARRAY_SIZE(gcc_sleep_clk_data), + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_usb_hs_ahb_clk = { + .halt_reg = 0x41008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x41008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_usb_hs_phy_cfg_ahb_clk = { + .halt_reg = 0x41030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x41030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_phy_cfg_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_usb_hs_system_clk = { + .halt_reg = 0x41004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x41004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_usb_hs_system_clk", + .parent_hws = (const struct clk_hw*[]) { + &usb_hs_system_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_venus0_ahb_clk = { + .halt_reg = 0x4c020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4c020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_ahb_clk", + .parent_hws = (const struct clk_hw*[]) { + &pcnoc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_venus0_axi_clk = { + .halt_reg = 0x4c024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4c024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_axi_clk", + .parent_hws = (const struct clk_hw*[]) { + &system_noc_bfdcd_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + } + } +}; + +static struct clk_branch gcc_venus0_core0_vcodec0_clk = { + .halt_reg = 0x4c02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4c02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_core0_vcodec0_clk", + .parent_hws = (const struct clk_hw*[]) { + &vcodec0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct clk_branch gcc_venus0_vcodec0_clk = { + .halt_reg = 0x4c01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4c01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data) { + .name = "gcc_venus0_vcodec0_clk", + .parent_hws = (const struct clk_hw*[]) { + &vcodec0_clk_src.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + } + } +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x4d078, + .cxcs = (unsigned int []) { 0x4d080, 0x4d088 }, + .cxc_count = 2, + .pd = { + .name = "mdss_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc oxili_gdsc = { + .gdscr = 0x5901c, + .cxcs = (unsigned int []) { 0x59020 }, + .cxc_count = 1, + .pd = { + .name = "oxili_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x4c018, + .cxcs = (unsigned int []) { 0x4c024, 0x4c01c }, + .cxc_count = 2, + .pd = { + .name = "venus_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_core0_gdsc = { + .gdscr = 0x4c028, + .cxcs = (unsigned int []) { 0x4c02c }, + .cxc_count = 1, + .pd = { + .name = "venus_core0_gdsc", + }, + .flags = HW_CTRL, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vfe_gdsc = { + .gdscr = 0x58034, + .cxcs = (unsigned int []) { 0x58038, 0x58048, 0x58050 }, + .cxc_count = 3, + .pd = { + .name = "vfe_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct clk_regmap *gcc_msm8909_clocks[] = { + [GPLL0_EARLY] = &gpll0_early.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL1] = &gpll1.clkr, + [GPLL1_VOTE] = &gpll1_vote, + [GPLL2_EARLY] = &gpll2_early.clkr, + [GPLL2] = &gpll2.clkr, + [BIMC_PLL_EARLY] = &bimc_pll_early.clkr, + [BIMC_PLL] = &bimc_pll.clkr, + [APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr, + [BIMC_DDR_CLK_SRC] = &bimc_ddr_clk_src.clkr, + [BIMC_GPU_CLK_SRC] = &bimc_gpu_clk_src.clkr, + [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, + [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, + [BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr, + [BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr, + [BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr, + [BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr, + [BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr, + [BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr, + [BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr, + [BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr, + [BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr, + [BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr, + [BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr, + [BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr, + [BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr, + [CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr, + [CAMSS_TOP_AHB_CLK_SRC] = &camss_top_ahb_clk_src.clkr, + [CRYPTO_CLK_SRC] = &crypto_clk_src.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, + [GP1_CLK_SRC] = &gp1_clk_src.clkr, + [GP2_CLK_SRC] = &gp2_clk_src.clkr, + [GP3_CLK_SRC] = &gp3_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr, + [PDM2_CLK_SRC] = &pdm2_clk_src.clkr, + [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, + [SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr, + [SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr, + [USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr, + [VCODEC0_CLK_SRC] = &vcodec0_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [GCC_APSS_TCU_CLK] = &gcc_apss_tcu_clk.clkr, + [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, + [GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr, + [GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr, + [GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr, + [GCC_GFX_TBU_CLK] = &gcc_gfx_tbu_clk.clkr, + [GCC_GFX_TCU_CLK] = &gcc_gfx_tcu_clk.clkr, + [GCC_GTCU_AHB_CLK] = &gcc_gtcu_ahb_clk.clkr, + [GCC_MDP_TBU_CLK] = &gcc_mdp_tbu_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_SMMU_CFG_CLK] = &gcc_smmu_cfg_clk.clkr, + [GCC_VENUS_TBU_CLK] = &gcc_venus_tbu_clk.clkr, + [GCC_VFE_TBU_CLK] = &gcc_vfe_tbu_clk.clkr, + [GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr, + [GCC_BIMC_GPU_CLK] = &gcc_bimc_gpu_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr, + [GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr, + [GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr, + [GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr, + [GCC_CAMSS_AHB_CLK] = &gcc_camss_ahb_clk.clkr, + [GCC_CAMSS_CSI0_CLK] = &gcc_camss_csi0_clk.clkr, + [GCC_CAMSS_CSI0_AHB_CLK] = &gcc_camss_csi0_ahb_clk.clkr, + [GCC_CAMSS_CSI0PHY_CLK] = &gcc_camss_csi0phy_clk.clkr, + [GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr, + [GCC_CAMSS_CSI0PIX_CLK] = &gcc_camss_csi0pix_clk.clkr, + [GCC_CAMSS_CSI0RDI_CLK] = &gcc_camss_csi0rdi_clk.clkr, + [GCC_CAMSS_CSI1_CLK] = &gcc_camss_csi1_clk.clkr, + [GCC_CAMSS_CSI1_AHB_CLK] = &gcc_camss_csi1_ahb_clk.clkr, + [GCC_CAMSS_CSI1PHY_CLK] = &gcc_camss_csi1phy_clk.clkr, + [GCC_CAMSS_CSI1PIX_CLK] = &gcc_camss_csi1pix_clk.clkr, + [GCC_CAMSS_CSI1RDI_CLK] = &gcc_camss_csi1rdi_clk.clkr, + [GCC_CAMSS_CSI_VFE0_CLK] = &gcc_camss_csi_vfe0_clk.clkr, + [GCC_CAMSS_GP0_CLK] = &gcc_camss_gp0_clk.clkr, + [GCC_CAMSS_GP1_CLK] = &gcc_camss_gp1_clk.clkr, + [GCC_CAMSS_ISPIF_AHB_CLK] = &gcc_camss_ispif_ahb_clk.clkr, + [GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr, + [GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr, + [GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr, + [GCC_CAMSS_VFE0_CLK] = &gcc_camss_vfe0_clk.clkr, + [GCC_CAMSS_VFE_AHB_CLK] = &gcc_camss_vfe_ahb_clk.clkr, + [GCC_CAMSS_VFE_AXI_CLK] = &gcc_camss_vfe_axi_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_MDSS_AHB_CLK] = &gcc_mdss_ahb_clk.clkr, + [GCC_MDSS_AXI_CLK] = &gcc_mdss_axi_clk.clkr, + [GCC_MDSS_BYTE0_CLK] = &gcc_mdss_byte0_clk.clkr, + [GCC_MDSS_ESC0_CLK] = &gcc_mdss_esc0_clk.clkr, + [GCC_MDSS_MDP_CLK] = &gcc_mdss_mdp_clk.clkr, + [GCC_MDSS_PCLK0_CLK] = &gcc_mdss_pclk0_clk.clkr, + [GCC_MDSS_VSYNC_CLK] = &gcc_mdss_vsync_clk.clkr, + [GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr, + [GCC_MSS_Q6_BIMC_AXI_CLK] = &gcc_mss_q6_bimc_axi_clk.clkr, + [GCC_OXILI_AHB_CLK] = &gcc_oxili_ahb_clk.clkr, + [GCC_OXILI_GFX3D_CLK] = &gcc_oxili_gfx3d_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_USB2A_PHY_SLEEP_CLK] = &gcc_usb2a_phy_sleep_clk.clkr, + [GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr, + [GCC_USB_HS_PHY_CFG_AHB_CLK] = &gcc_usb_hs_phy_cfg_ahb_clk.clkr, + [GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr, + [GCC_VENUS0_AHB_CLK] = &gcc_venus0_ahb_clk.clkr, + [GCC_VENUS0_AXI_CLK] = &gcc_venus0_axi_clk.clkr, + [GCC_VENUS0_CORE0_VCODEC0_CLK] = &gcc_venus0_core0_vcodec0_clk.clkr, + [GCC_VENUS0_VCODEC0_CLK] = &gcc_venus0_vcodec0_clk.clkr, +}; + +static struct gdsc *gcc_msm8909_gdscs[] = { + [MDSS_GDSC] = &mdss_gdsc, + [OXILI_GDSC] = &oxili_gdsc, + [VENUS_GDSC] = &venus_gdsc, + [VENUS_CORE0_GDSC] = &venus_core0_gdsc, + [VFE_GDSC] = &vfe_gdsc, +}; + +static const struct qcom_reset_map gcc_msm8909_resets[] = { + [GCC_AUDIO_CORE_BCR] = { 0x1c008 }, + [GCC_BLSP1_BCR] = { 0x01000 }, + [GCC_BLSP1_QUP1_BCR] = { 0x02000 }, + [GCC_BLSP1_QUP2_BCR] = { 0x03008 }, + [GCC_BLSP1_QUP3_BCR] = { 0x04018 }, + [GCC_BLSP1_QUP4_BCR] = { 0x05018 }, + [GCC_BLSP1_QUP5_BCR] = { 0x06018 }, + [GCC_BLSP1_QUP6_BCR] = { 0x07018 }, + [GCC_BLSP1_UART1_BCR] = { 0x02038 }, + [GCC_BLSP1_UART2_BCR] = { 0x03028 }, + [GCC_CAMSS_CSI0_BCR] = { 0x4e038 }, + [GCC_CAMSS_CSI0PHY_BCR] = { 0x4e044 }, + [GCC_CAMSS_CSI0PIX_BCR] = { 0x4e054 }, + [GCC_CAMSS_CSI0RDI_BCR] = { 0x4e04c }, + [GCC_CAMSS_CSI1_BCR] = { 0x4f038 }, + [GCC_CAMSS_CSI1PHY_BCR] = { 0x4f044 }, + [GCC_CAMSS_CSI1PIX_BCR] = { 0x4f054 }, + [GCC_CAMSS_CSI1RDI_BCR] = { 0x4f04c }, + [GCC_CAMSS_CSI_VFE0_BCR] = { 0x5804c }, + [GCC_CAMSS_GP0_BCR] = { 0x54014 }, + [GCC_CAMSS_GP1_BCR] = { 0x55014 }, + [GCC_CAMSS_ISPIF_BCR] = { 0x50000 }, + [GCC_CAMSS_MCLK0_BCR] = { 0x52014 }, + [GCC_CAMSS_MCLK1_BCR] = { 0x53014 }, + [GCC_CAMSS_PHY0_BCR] = { 0x4e018 }, + [GCC_CAMSS_TOP_BCR] = { 0x56000 }, + [GCC_CAMSS_TOP_AHB_BCR] = { 0x5a018 }, + [GCC_CAMSS_VFE_BCR] = { 0x58030 }, + [GCC_CRYPTO_BCR] = { 0x16000 }, + [GCC_MDSS_BCR] = { 0x4d074 }, + [GCC_OXILI_BCR] = { 0x59018 }, + [GCC_PDM_BCR] = { 0x44000 }, + [GCC_PRNG_BCR] = { 0x13000 }, + [GCC_QUSB2_PHY_BCR] = { 0x4103c }, + [GCC_SDCC1_BCR] = { 0x42000 }, + [GCC_SDCC2_BCR] = { 0x43000 }, + [GCC_ULT_AUDIO_BCR] = { 0x1c0b4 }, + [GCC_USB2A_PHY_BCR] = { 0x41028 }, + [GCC_USB2_HS_PHY_ONLY_BCR] = { .reg = 0x41034, .udelay = 15 }, + [GCC_USB_HS_BCR] = { 0x41000 }, + [GCC_VENUS0_BCR] = { 0x4c014 }, + /* Subsystem Restart */ + [GCC_MSS_RESTART] = { 0x3e000 }, +}; + +static const struct regmap_config gcc_msm8909_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x80000, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_msm8909_desc = { + .config = &gcc_msm8909_regmap_config, + .clks = gcc_msm8909_clocks, + .num_clks = ARRAY_SIZE(gcc_msm8909_clocks), + .resets = gcc_msm8909_resets, + .num_resets = ARRAY_SIZE(gcc_msm8909_resets), + .gdscs = gcc_msm8909_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_msm8909_gdscs), +}; + +static const struct of_device_id gcc_msm8909_match_table[] = { + { .compatible = "qcom,gcc-msm8909" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_msm8909_match_table); + +static int gcc_msm8909_probe(struct platform_device *pdev) +{ + return qcom_cc_probe(pdev, &gcc_msm8909_desc); +} + +static struct platform_driver gcc_msm8909_driver = { + .probe = gcc_msm8909_probe, + .driver = { + .name = "gcc-msm8909", + .of_match_table = gcc_msm8909_match_table, + }, +}; + +static int __init gcc_msm8909_init(void) +{ + return platform_driver_register(&gcc_msm8909_driver); +} +core_initcall(gcc_msm8909_init); + +static void __exit gcc_msm8909_exit(void) +{ + platform_driver_unregister(&gcc_msm8909_driver); +} +module_exit(gcc_msm8909_exit); + +MODULE_DESCRIPTION("Qualcomm GCC MSM8909 Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:gcc-msm8909"); diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c index 9a46794f6eb8adc2c85fac32419dc28285520540..0c8fe19387a73750b0c01b7ed67e3ed03e857424 100644 --- a/drivers/clk/qcom/gcc-msm8916.c +++ b/drivers/clk/qcom/gcc-msm8916.c @@ -42,14 +42,138 @@ enum { P_EXT_MCLK, }; +static struct clk_pll gpll0 = { + .l_reg = 0x21004, + .m_reg = 0x21008, + .n_reg = 0x2100c, + .config_reg = 0x21010, + .mode_reg = 0x21000, + .status_reg = 0x2101c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", .name = "xo_board", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll0_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0_vote", + .parent_hws = (const struct clk_hw*[]){ + &gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll gpll1 = { + .l_reg = 0x20004, + .m_reg = 0x20008, + .n_reg = 0x2000c, + .config_reg = 0x20010, + .mode_reg = 0x20000, + .status_reg = 0x2001c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", .name = "xo_board", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll1_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gpll1_vote", + .parent_hws = (const struct clk_hw*[]){ + &gpll1.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll gpll2 = { + .l_reg = 0x4a004, + .m_reg = 0x4a008, + .n_reg = 0x4a00c, + .config_reg = 0x4a010, + .mode_reg = 0x4a000, + .status_reg = 0x4a01c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll2", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", .name = "xo_board", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll2_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gpll2_vote", + .parent_hws = (const struct clk_hw*[]){ + &gpll2.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll bimc_pll = { + .l_reg = 0x23004, + .m_reg = 0x23008, + .n_reg = 0x2300c, + .config_reg = 0x23010, + .mode_reg = 0x23000, + .status_reg = 0x2301c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "bimc_pll", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", .name = "xo_board", + }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap bimc_pll_vote = { + .enable_reg = 0x45000, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "bimc_pll_vote", + .parent_hws = (const struct clk_hw*[]){ + &bimc_pll.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + static const struct parent_map gcc_xo_gpll0_map[] = { { P_XO, 0 }, { P_GPLL0, 1 }, }; -static const char * const gcc_xo_gpll0[] = { - "xo", - "gpll0_vote", +static const struct clk_parent_data gcc_xo_gpll0[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, }; static const struct parent_map gcc_xo_gpll0_bimc_map[] = { @@ -58,10 +182,10 @@ static const struct parent_map gcc_xo_gpll0_bimc_map[] = { { P_BIMC, 2 }, }; -static const char * const gcc_xo_gpll0_bimc[] = { - "xo", - "gpll0_vote", - "bimc_pll_vote", +static const struct clk_parent_data gcc_xo_gpll0_bimc[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &bimc_pll_vote.hw }, }; static const struct parent_map gcc_xo_gpll0a_gpll1_gpll2a_map[] = { @@ -71,11 +195,11 @@ static const struct parent_map gcc_xo_gpll0a_gpll1_gpll2a_map[] = { { P_GPLL2_AUX, 2 }, }; -static const char * const gcc_xo_gpll0a_gpll1_gpll2a[] = { - "xo", - "gpll0_vote", - "gpll1_vote", - "gpll2_vote", +static const struct clk_parent_data gcc_xo_gpll0a_gpll1_gpll2a[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll1_vote.hw }, + { .hw = &gpll2_vote.hw }, }; static const struct parent_map gcc_xo_gpll0_gpll2_map[] = { @@ -84,10 +208,10 @@ static const struct parent_map gcc_xo_gpll0_gpll2_map[] = { { P_GPLL2, 2 }, }; -static const char * const gcc_xo_gpll0_gpll2[] = { - "xo", - "gpll0_vote", - "gpll2_vote", +static const struct clk_parent_data gcc_xo_gpll0_gpll2[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll2_vote.hw }, }; static const struct parent_map gcc_xo_gpll0a_map[] = { @@ -95,9 +219,9 @@ static const struct parent_map gcc_xo_gpll0a_map[] = { { P_GPLL0_AUX, 2 }, }; -static const char * const gcc_xo_gpll0a[] = { - "xo", - "gpll0_vote", +static const struct clk_parent_data gcc_xo_gpll0a[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, }; static const struct parent_map gcc_xo_gpll0_gpll1a_sleep_map[] = { @@ -107,11 +231,11 @@ static const struct parent_map gcc_xo_gpll0_gpll1a_sleep_map[] = { { P_SLEEP_CLK, 6 }, }; -static const char * const gcc_xo_gpll0_gpll1a_sleep[] = { - "xo", - "gpll0_vote", - "gpll1_vote", - "sleep_clk", +static const struct clk_parent_data gcc_xo_gpll0_gpll1a_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll1_vote.hw }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static const struct parent_map gcc_xo_gpll0_gpll1a_map[] = { @@ -120,10 +244,10 @@ static const struct parent_map gcc_xo_gpll0_gpll1a_map[] = { { P_GPLL1_AUX, 2 }, }; -static const char * const gcc_xo_gpll0_gpll1a[] = { - "xo", - "gpll0_vote", - "gpll1_vote", +static const struct clk_parent_data gcc_xo_gpll0_gpll1a[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll1_vote.hw }, }; static const struct parent_map gcc_xo_dsibyte_map[] = { @@ -131,9 +255,9 @@ static const struct parent_map gcc_xo_dsibyte_map[] = { { P_DSI0_PHYPLL_BYTE, 2 }, }; -static const char * const gcc_xo_dsibyte[] = { - "xo", - "dsi0pllbyte", +static const struct clk_parent_data gcc_xo_dsibyte[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .fw_name = "dsi0pllbyte", .name = "dsi0pllbyte" }, }; static const struct parent_map gcc_xo_gpll0a_dsibyte_map[] = { @@ -142,10 +266,10 @@ static const struct parent_map gcc_xo_gpll0a_dsibyte_map[] = { { P_DSI0_PHYPLL_BYTE, 1 }, }; -static const char * const gcc_xo_gpll0a_dsibyte[] = { - "xo", - "gpll0_vote", - "dsi0pllbyte", +static const struct clk_parent_data gcc_xo_gpll0a_dsibyte[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .fw_name = "dsi0pllbyte", .name = "dsi0pllbyte" }, }; static const struct parent_map gcc_xo_gpll0_dsiphy_map[] = { @@ -154,10 +278,10 @@ static const struct parent_map gcc_xo_gpll0_dsiphy_map[] = { { P_DSI0_PHYPLL_DSI, 2 }, }; -static const char * const gcc_xo_gpll0_dsiphy[] = { - "xo", - "gpll0_vote", - "dsi0pll", +static const struct clk_parent_data gcc_xo_gpll0_dsiphy[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .fw_name = "dsi0pll", .name = "dsi0pll" }, }; static const struct parent_map gcc_xo_gpll0a_dsiphy_map[] = { @@ -166,10 +290,10 @@ static const struct parent_map gcc_xo_gpll0a_dsiphy_map[] = { { P_DSI0_PHYPLL_DSI, 1 }, }; -static const char * const gcc_xo_gpll0a_dsiphy[] = { - "xo", - "gpll0_vote", - "dsi0pll", +static const struct clk_parent_data gcc_xo_gpll0a_dsiphy[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .fw_name = "dsi0pll", .name = "dsi0pll" }, }; static const struct parent_map gcc_xo_gpll0a_gpll1_gpll2_map[] = { @@ -179,11 +303,11 @@ static const struct parent_map gcc_xo_gpll0a_gpll1_gpll2_map[] = { { P_GPLL2, 2 }, }; -static const char * const gcc_xo_gpll0a_gpll1_gpll2[] = { - "xo", - "gpll0_vote", - "gpll1_vote", - "gpll2_vote", +static const struct clk_parent_data gcc_xo_gpll0a_gpll1_gpll2[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll1_vote.hw }, + { .hw = &gpll2_vote.hw }, }; static const struct parent_map gcc_xo_gpll0_gpll1_sleep_map[] = { @@ -193,11 +317,11 @@ static const struct parent_map gcc_xo_gpll0_gpll1_sleep_map[] = { { P_SLEEP_CLK, 6 } }; -static const char * const gcc_xo_gpll0_gpll1_sleep[] = { - "xo", - "gpll0_vote", - "gpll1_vote", - "sleep_clk", +static const struct clk_parent_data gcc_xo_gpll0_gpll1_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll0_vote.hw }, + { .hw = &gpll1_vote.hw }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static const struct parent_map gcc_xo_gpll1_epi2s_emclk_sleep_map[] = { @@ -208,12 +332,12 @@ static const struct parent_map gcc_xo_gpll1_epi2s_emclk_sleep_map[] = { { P_SLEEP_CLK, 6 } }; -static const char * const gcc_xo_gpll1_epi2s_emclk_sleep[] = { - "xo", - "gpll1_vote", - "ext_pri_i2s", - "ext_mclk", - "sleep_clk", +static const struct clk_parent_data gcc_xo_gpll1_epi2s_emclk_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll1_vote.hw }, + { .fw_name = "ext_pri_i2s", .name = "ext_pri_i2s" }, + { .fw_name = "ext_mclk", .name = "ext_mclk" }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static const struct parent_map gcc_xo_gpll1_esi2s_emclk_sleep_map[] = { @@ -224,12 +348,12 @@ static const struct parent_map gcc_xo_gpll1_esi2s_emclk_sleep_map[] = { { P_SLEEP_CLK, 6 } }; -static const char * const gcc_xo_gpll1_esi2s_emclk_sleep[] = { - "xo", - "gpll1_vote", - "ext_sec_i2s", - "ext_mclk", - "sleep_clk", +static const struct clk_parent_data gcc_xo_gpll1_esi2s_emclk_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll1_vote.hw }, + { .fw_name = "ext_sec_i2s", .name = "ext_sec_i2s" }, + { .fw_name = "ext_mclk", .name = "ext_mclk" }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static const struct parent_map gcc_xo_sleep_map[] = { @@ -237,9 +361,9 @@ static const struct parent_map gcc_xo_sleep_map[] = { { P_SLEEP_CLK, 6 } }; -static const char * const gcc_xo_sleep[] = { - "xo", - "sleep_clk", +static const struct clk_parent_data gcc_xo_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static const struct parent_map gcc_xo_gpll1_emclk_sleep_map[] = { @@ -249,119 +373,11 @@ static const struct parent_map gcc_xo_gpll1_emclk_sleep_map[] = { { P_SLEEP_CLK, 6 } }; -static const char * const gcc_xo_gpll1_emclk_sleep[] = { - "xo", - "gpll1_vote", - "ext_mclk", - "sleep_clk", -}; - -static struct clk_pll gpll0 = { - .l_reg = 0x21004, - .m_reg = 0x21008, - .n_reg = 0x2100c, - .config_reg = 0x21010, - .mode_reg = 0x21000, - .status_reg = 0x2101c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gpll0", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap gpll0_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gpll0_vote", - .parent_names = (const char *[]){ "gpll0" }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, -}; - -static struct clk_pll gpll1 = { - .l_reg = 0x20004, - .m_reg = 0x20008, - .n_reg = 0x2000c, - .config_reg = 0x20010, - .mode_reg = 0x20000, - .status_reg = 0x2001c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gpll1", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap gpll1_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(1), - .hw.init = &(struct clk_init_data){ - .name = "gpll1_vote", - .parent_names = (const char *[]){ "gpll1" }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, -}; - -static struct clk_pll gpll2 = { - .l_reg = 0x4a004, - .m_reg = 0x4a008, - .n_reg = 0x4a00c, - .config_reg = 0x4a010, - .mode_reg = 0x4a000, - .status_reg = 0x4a01c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gpll2", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap gpll2_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(2), - .hw.init = &(struct clk_init_data){ - .name = "gpll2_vote", - .parent_names = (const char *[]){ "gpll2" }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, -}; - -static struct clk_pll bimc_pll = { - .l_reg = 0x23004, - .m_reg = 0x23008, - .n_reg = 0x2300c, - .config_reg = 0x23010, - .mode_reg = 0x23000, - .status_reg = 0x2301c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "bimc_pll", - .parent_names = (const char *[]){ "xo" }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap bimc_pll_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(3), - .hw.init = &(struct clk_init_data){ - .name = "bimc_pll_vote", - .parent_names = (const char *[]){ "bimc_pll" }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, +static const struct clk_parent_data gcc_xo_gpll1_emclk_sleep[] = { + { .fw_name = "xo", .name = "xo_board" }, + { .hw = &gpll1_vote.hw }, + { .fw_name = "ext_mclk", .name = "ext_mclk" }, + { .fw_name = "sleep_clk", .name = "sleep_clk" }, }; static struct clk_rcg2 pcnoc_bfdcd_clk_src = { @@ -370,8 +386,8 @@ static struct clk_rcg2 pcnoc_bfdcd_clk_src = { .parent_map = gcc_xo_gpll0_bimc_map, .clkr.hw.init = &(struct clk_init_data){ .name = "pcnoc_bfdcd_clk_src", - .parent_names = gcc_xo_gpll0_bimc, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_bimc, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), .ops = &clk_rcg2_ops, }, }; @@ -382,8 +398,8 @@ static struct clk_rcg2 system_noc_bfdcd_clk_src = { .parent_map = gcc_xo_gpll0_bimc_map, .clkr.hw.init = &(struct clk_init_data){ .name = "system_noc_bfdcd_clk_src", - .parent_names = gcc_xo_gpll0_bimc, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_bimc, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), .ops = &clk_rcg2_ops, }, }; @@ -402,8 +418,8 @@ static struct clk_rcg2 camss_ahb_clk_src = { .freq_tbl = ftbl_gcc_camss_ahb_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "camss_ahb_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -423,8 +439,8 @@ static struct clk_rcg2 apss_ahb_clk_src = { .freq_tbl = ftbl_apss_ahb_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "apss_ahb_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -442,8 +458,8 @@ static struct clk_rcg2 csi0_clk_src = { .freq_tbl = ftbl_gcc_camss_csi0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "csi0_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -455,8 +471,8 @@ static struct clk_rcg2 csi1_clk_src = { .freq_tbl = ftbl_gcc_camss_csi0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "csi1_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -483,8 +499,8 @@ static struct clk_rcg2 gfx3d_clk_src = { .freq_tbl = ftbl_gcc_oxili_gfx3d_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "gfx3d_clk_src", - .parent_names = gcc_xo_gpll0a_gpll1_gpll2a, - .num_parents = 4, + .parent_data = gcc_xo_gpll0a_gpll1_gpll2a, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_gpll1_gpll2a), .ops = &clk_rcg2_ops, }, }; @@ -510,8 +526,8 @@ static struct clk_rcg2 vfe0_clk_src = { .freq_tbl = ftbl_gcc_camss_vfe0_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "vfe0_clk_src", - .parent_names = gcc_xo_gpll0_gpll2, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll2), .ops = &clk_rcg2_ops, }, }; @@ -529,8 +545,8 @@ static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup1_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -558,8 +574,8 @@ static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup1_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -571,8 +587,8 @@ static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup2_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -585,8 +601,8 @@ static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup2_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -598,8 +614,8 @@ static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup3_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -612,8 +628,8 @@ static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup3_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -625,8 +641,8 @@ static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup4_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -639,8 +655,8 @@ static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup4_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -652,8 +668,8 @@ static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup5_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -666,8 +682,8 @@ static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup5_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -679,8 +695,8 @@ static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_i2c_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup6_i2c_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -693,8 +709,8 @@ static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_qup1_6_spi_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup6_spi_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -726,8 +742,8 @@ static struct clk_rcg2 blsp1_uart1_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_uart1_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -740,8 +756,8 @@ static struct clk_rcg2 blsp1_uart2_apps_clk_src = { .freq_tbl = ftbl_gcc_blsp1_uart1_6_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_uart2_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -759,8 +775,8 @@ static struct clk_rcg2 cci_clk_src = { .freq_tbl = ftbl_gcc_camss_cci_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "cci_clk_src", - .parent_names = gcc_xo_gpll0a, - .num_parents = 2, + .parent_data = gcc_xo_gpll0a, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a), .ops = &clk_rcg2_ops, }, }; @@ -792,8 +808,8 @@ static struct clk_rcg2 camss_gp0_clk_src = { .freq_tbl = ftbl_gcc_camss_gp0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "camss_gp0_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -806,8 +822,8 @@ static struct clk_rcg2 camss_gp1_clk_src = { .freq_tbl = ftbl_gcc_camss_gp0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "camss_gp1_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -826,8 +842,8 @@ static struct clk_rcg2 jpeg0_clk_src = { .freq_tbl = ftbl_gcc_camss_jpeg0_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "jpeg0_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -847,8 +863,8 @@ static struct clk_rcg2 mclk0_clk_src = { .freq_tbl = ftbl_gcc_camss_mclk0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "mclk0_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -861,8 +877,8 @@ static struct clk_rcg2 mclk1_clk_src = { .freq_tbl = ftbl_gcc_camss_mclk0_1_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "mclk1_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -880,8 +896,8 @@ static struct clk_rcg2 csi0phytimer_clk_src = { .freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "csi0phytimer_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll1a, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a), .ops = &clk_rcg2_ops, }, }; @@ -893,8 +909,8 @@ static struct clk_rcg2 csi1phytimer_clk_src = { .freq_tbl = ftbl_gcc_camss_csi0_1phytimer_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "csi1phytimer_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll1a, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a), .ops = &clk_rcg2_ops, }, }; @@ -913,8 +929,8 @@ static struct clk_rcg2 cpp_clk_src = { .freq_tbl = ftbl_gcc_camss_cpp_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "cpp_clk_src", - .parent_names = gcc_xo_gpll0_gpll2, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll2), .ops = &clk_rcg2_ops, }, }; @@ -934,8 +950,8 @@ static struct clk_rcg2 crypto_clk_src = { .freq_tbl = ftbl_gcc_crypto_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "crypto_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -975,8 +991,8 @@ static struct clk_rcg2 gp1_clk_src = { .freq_tbl = ftbl_gcc_gp1_3_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "gp1_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -989,8 +1005,8 @@ static struct clk_rcg2 gp2_clk_src = { .freq_tbl = ftbl_gcc_gp1_3_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "gp2_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1003,8 +1019,8 @@ static struct clk_rcg2 gp3_clk_src = { .freq_tbl = ftbl_gcc_gp1_3_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "gp3_clk_src", - .parent_names = gcc_xo_gpll0_gpll1a_sleep, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_gpll1a_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1015,8 +1031,8 @@ static struct clk_rcg2 byte0_clk_src = { .parent_map = gcc_xo_gpll0a_dsibyte_map, .clkr.hw.init = &(struct clk_init_data){ .name = "byte0_clk_src", - .parent_names = gcc_xo_gpll0a_dsibyte, - .num_parents = 3, + .parent_data = gcc_xo_gpll0a_dsibyte, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsibyte), .ops = &clk_byte2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1034,8 +1050,8 @@ static struct clk_rcg2 esc0_clk_src = { .freq_tbl = ftbl_gcc_mdss_esc0_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "esc0_clk_src", - .parent_names = gcc_xo_dsibyte, - .num_parents = 2, + .parent_data = gcc_xo_dsibyte, + .num_parents = ARRAY_SIZE(gcc_xo_dsibyte), .ops = &clk_rcg2_ops, }, }; @@ -1059,8 +1075,8 @@ static struct clk_rcg2 mdp_clk_src = { .freq_tbl = ftbl_gcc_mdss_mdp_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "mdp_clk_src", - .parent_names = gcc_xo_gpll0_dsiphy, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_dsiphy, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_dsiphy), .ops = &clk_rcg2_ops, }, }; @@ -1072,8 +1088,8 @@ static struct clk_rcg2 pclk0_clk_src = { .parent_map = gcc_xo_gpll0a_dsiphy_map, .clkr.hw.init = &(struct clk_init_data){ .name = "pclk0_clk_src", - .parent_names = gcc_xo_gpll0a_dsiphy, - .num_parents = 3, + .parent_data = gcc_xo_gpll0a_dsiphy, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsiphy), .ops = &clk_pixel_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1091,8 +1107,8 @@ static struct clk_rcg2 vsync_clk_src = { .freq_tbl = ftbl_gcc_mdss_vsync_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "vsync_clk_src", - .parent_names = gcc_xo_gpll0a, - .num_parents = 2, + .parent_data = gcc_xo_gpll0a, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a), .ops = &clk_rcg2_ops, }, }; @@ -1109,8 +1125,8 @@ static struct clk_rcg2 pdm2_clk_src = { .freq_tbl = ftbl_gcc_pdm2_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "pdm2_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -1134,8 +1150,8 @@ static struct clk_rcg2 sdcc1_apps_clk_src = { .freq_tbl = ftbl_gcc_sdcc1_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "sdcc1_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_floor_ops, }, }; @@ -1159,8 +1175,8 @@ static struct clk_rcg2 sdcc2_apps_clk_src = { .freq_tbl = ftbl_gcc_sdcc2_apps_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "sdcc2_apps_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_floor_ops, }, }; @@ -1179,8 +1195,8 @@ static struct clk_rcg2 apss_tcu_clk_src = { .freq_tbl = ftbl_gcc_apss_tcu_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "apss_tcu_clk_src", - .parent_names = gcc_xo_gpll0a_gpll1_gpll2, - .num_parents = 4, + .parent_data = gcc_xo_gpll0a_gpll1_gpll2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_gpll1_gpll2), .ops = &clk_rcg2_ops, }, }; @@ -1202,8 +1218,8 @@ static struct clk_rcg2 bimc_gpu_clk_src = { .freq_tbl = ftbl_gcc_bimc_gpu_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "bimc_gpu_clk_src", - .parent_names = gcc_xo_gpll0_bimc, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_bimc, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), .flags = CLK_GET_RATE_NOCACHE, .ops = &clk_rcg2_ops, }, @@ -1221,8 +1237,8 @@ static struct clk_rcg2 usb_hs_system_clk_src = { .freq_tbl = ftbl_gcc_usb_hs_system_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "usb_hs_system_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -1247,8 +1263,8 @@ static struct clk_rcg2 ultaudio_ahbfabric_clk_src = { .freq_tbl = ftbl_gcc_ultaudio_ahb_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_ahbfabric_clk_src", - .parent_names = gcc_xo_gpll0_gpll1_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll0_gpll1_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1260,8 +1276,8 @@ static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_ahbfabric_ixfabric_clk", - .parent_names = (const char *[]){ - "ultaudio_ahbfabric_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_ahbfabric_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1277,8 +1293,8 @@ static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_lpm_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_ahbfabric_ixfabric_lpm_clk", - .parent_names = (const char *[]){ - "ultaudio_ahbfabric_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_ahbfabric_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1326,8 +1342,8 @@ static struct clk_rcg2 ultaudio_lpaif_pri_i2s_clk_src = { .freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_pri_i2s_clk_src", - .parent_names = gcc_xo_gpll1_epi2s_emclk_sleep, - .num_parents = 5, + .parent_data = gcc_xo_gpll1_epi2s_emclk_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_epi2s_emclk_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1339,8 +1355,8 @@ static struct clk_branch gcc_ultaudio_lpaif_pri_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_pri_i2s_clk", - .parent_names = (const char *[]){ - "ultaudio_lpaif_pri_i2s_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_pri_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1357,8 +1373,8 @@ static struct clk_rcg2 ultaudio_lpaif_sec_i2s_clk_src = { .freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_sec_i2s_clk_src", - .parent_names = gcc_xo_gpll1_esi2s_emclk_sleep, - .num_parents = 5, + .parent_data = gcc_xo_gpll1_esi2s_emclk_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_esi2s_emclk_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1370,8 +1386,8 @@ static struct clk_branch gcc_ultaudio_lpaif_sec_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_sec_i2s_clk", - .parent_names = (const char *[]){ - "ultaudio_lpaif_sec_i2s_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_sec_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1388,8 +1404,8 @@ static struct clk_rcg2 ultaudio_lpaif_aux_i2s_clk_src = { .freq_tbl = ftbl_gcc_ultaudio_lpaif_i2s_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_aux_i2s_clk_src", - .parent_names = gcc_xo_gpll1_esi2s_emclk_sleep, - .num_parents = 5, + .parent_data = gcc_xo_gpll1_esi2s_emclk_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_esi2s_emclk_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1401,8 +1417,8 @@ static struct clk_branch gcc_ultaudio_lpaif_aux_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_aux_i2s_clk", - .parent_names = (const char *[]){ - "ultaudio_lpaif_aux_i2s_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_aux_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1423,8 +1439,8 @@ static struct clk_rcg2 ultaudio_xo_clk_src = { .freq_tbl = ftbl_gcc_ultaudio_xo_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_xo_clk_src", - .parent_names = gcc_xo_sleep, - .num_parents = 2, + .parent_data = gcc_xo_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1436,8 +1452,8 @@ static struct clk_branch gcc_ultaudio_avsync_xo_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_avsync_xo_clk", - .parent_names = (const char *[]){ - "ultaudio_xo_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_xo_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1453,8 +1469,8 @@ static struct clk_branch gcc_ultaudio_stc_xo_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_stc_xo_clk", - .parent_names = (const char *[]){ - "ultaudio_xo_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_xo_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1479,8 +1495,8 @@ static struct clk_rcg2 codec_digcodec_clk_src = { .freq_tbl = ftbl_codec_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "codec_digcodec_clk_src", - .parent_names = gcc_xo_gpll1_emclk_sleep, - .num_parents = 4, + .parent_data = gcc_xo_gpll1_emclk_sleep, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_emclk_sleep), .ops = &clk_rcg2_ops, }, }; @@ -1492,8 +1508,8 @@ static struct clk_branch gcc_codec_digcodec_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_codec_digcodec_clk", - .parent_names = (const char *[]){ - "codec_digcodec_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &codec_digcodec_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1509,8 +1525,8 @@ static struct clk_branch gcc_ultaudio_pcnoc_mport_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_pcnoc_mport_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1525,8 +1541,8 @@ static struct clk_branch gcc_ultaudio_pcnoc_sway_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_pcnoc_sway_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1549,8 +1565,8 @@ static struct clk_rcg2 vcodec0_clk_src = { .freq_tbl = ftbl_gcc_venus0_vcodec0_clk, .clkr.hw.init = &(struct clk_init_data){ .name = "vcodec0_clk_src", - .parent_names = gcc_xo_gpll0, - .num_parents = 2, + .parent_data = gcc_xo_gpll0, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0), .ops = &clk_rcg2_ops, }, }; @@ -1563,8 +1579,8 @@ static struct clk_branch gcc_blsp1_ahb_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1579,8 +1595,8 @@ static struct clk_branch gcc_blsp1_sleep_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", - .parent_names = (const char *[]){ - "sleep_clk_src", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "sleep_clk", .name = "sleep_clk_src", }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1596,8 +1612,8 @@ static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup1_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup1_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup1_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1613,8 +1629,8 @@ static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup1_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup1_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup1_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1630,8 +1646,8 @@ static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup2_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup2_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup2_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1647,8 +1663,8 @@ static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup2_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup2_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup2_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1664,8 +1680,8 @@ static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup3_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup3_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup3_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1681,8 +1697,8 @@ static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup3_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup3_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup3_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1698,8 +1714,8 @@ static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup4_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup4_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup4_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1715,8 +1731,8 @@ static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup4_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup4_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup4_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1732,8 +1748,8 @@ static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup5_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup5_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup5_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1749,8 +1765,8 @@ static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup5_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup5_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup5_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1766,8 +1782,8 @@ static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup6_i2c_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup6_i2c_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup6_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1783,8 +1799,8 @@ static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup6_spi_apps_clk", - .parent_names = (const char *[]){ - "blsp1_qup6_spi_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup6_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1800,8 +1816,8 @@ static struct clk_branch gcc_blsp1_uart1_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_uart1_apps_clk", - .parent_names = (const char *[]){ - "blsp1_uart1_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_uart1_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1817,8 +1833,8 @@ static struct clk_branch gcc_blsp1_uart2_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_uart2_apps_clk", - .parent_names = (const char *[]){ - "blsp1_uart2_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &blsp1_uart2_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1835,8 +1851,8 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "gcc_boot_rom_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1851,8 +1867,8 @@ static struct clk_branch gcc_camss_cci_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cci_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1868,8 +1884,8 @@ static struct clk_branch gcc_camss_cci_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cci_clk", - .parent_names = (const char *[]){ - "cci_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &cci_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1885,8 +1901,8 @@ static struct clk_branch gcc_camss_csi0_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1902,8 +1918,8 @@ static struct clk_branch gcc_camss_csi0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0_clk", - .parent_names = (const char *[]){ - "csi0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1919,8 +1935,8 @@ static struct clk_branch gcc_camss_csi0phy_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0phy_clk", - .parent_names = (const char *[]){ - "csi0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1936,8 +1952,8 @@ static struct clk_branch gcc_camss_csi0pix_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0pix_clk", - .parent_names = (const char *[]){ - "csi0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1953,8 +1969,8 @@ static struct clk_branch gcc_camss_csi0rdi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0rdi_clk", - .parent_names = (const char *[]){ - "csi0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1970,8 +1986,8 @@ static struct clk_branch gcc_camss_csi1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1987,8 +2003,8 @@ static struct clk_branch gcc_camss_csi1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1_clk", - .parent_names = (const char *[]){ - "csi1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2004,8 +2020,8 @@ static struct clk_branch gcc_camss_csi1phy_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1phy_clk", - .parent_names = (const char *[]){ - "csi1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2021,8 +2037,8 @@ static struct clk_branch gcc_camss_csi1pix_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1pix_clk", - .parent_names = (const char *[]){ - "csi1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2038,8 +2054,8 @@ static struct clk_branch gcc_camss_csi1rdi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1rdi_clk", - .parent_names = (const char *[]){ - "csi1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2055,8 +2071,8 @@ static struct clk_branch gcc_camss_csi_vfe0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi_vfe0_clk", - .parent_names = (const char *[]){ - "vfe0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &vfe0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2072,8 +2088,8 @@ static struct clk_branch gcc_camss_gp0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_gp0_clk", - .parent_names = (const char *[]){ - "camss_gp0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_gp0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2089,8 +2105,8 @@ static struct clk_branch gcc_camss_gp1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_gp1_clk", - .parent_names = (const char *[]){ - "camss_gp1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_gp1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2106,8 +2122,8 @@ static struct clk_branch gcc_camss_ispif_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_ispif_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2123,8 +2139,8 @@ static struct clk_branch gcc_camss_jpeg0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg0_clk", - .parent_names = (const char *[]){ - "jpeg0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &jpeg0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2140,8 +2156,8 @@ static struct clk_branch gcc_camss_jpeg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2157,8 +2173,8 @@ static struct clk_branch gcc_camss_jpeg_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg_axi_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2174,8 +2190,8 @@ static struct clk_branch gcc_camss_mclk0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_mclk0_clk", - .parent_names = (const char *[]){ - "mclk0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &mclk0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2191,8 +2207,8 @@ static struct clk_branch gcc_camss_mclk1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_mclk1_clk", - .parent_names = (const char *[]){ - "mclk1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &mclk1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2208,8 +2224,8 @@ static struct clk_branch gcc_camss_micro_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_micro_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2225,8 +2241,8 @@ static struct clk_branch gcc_camss_csi0phytimer_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0phytimer_clk", - .parent_names = (const char *[]){ - "csi0phytimer_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi0phytimer_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2242,8 +2258,8 @@ static struct clk_branch gcc_camss_csi1phytimer_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1phytimer_clk", - .parent_names = (const char *[]){ - "csi1phytimer_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &csi1phytimer_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2259,8 +2275,8 @@ static struct clk_branch gcc_camss_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2276,8 +2292,8 @@ static struct clk_branch gcc_camss_top_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_top_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2293,8 +2309,8 @@ static struct clk_branch gcc_camss_cpp_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cpp_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2310,8 +2326,8 @@ static struct clk_branch gcc_camss_cpp_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cpp_clk", - .parent_names = (const char *[]){ - "cpp_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &cpp_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2327,8 +2343,8 @@ static struct clk_branch gcc_camss_vfe0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe0_clk", - .parent_names = (const char *[]){ - "vfe0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &vfe0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2344,8 +2360,8 @@ static struct clk_branch gcc_camss_vfe_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe_ahb_clk", - .parent_names = (const char *[]){ - "camss_ahb_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2361,8 +2377,8 @@ static struct clk_branch gcc_camss_vfe_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe_axi_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2379,8 +2395,8 @@ static struct clk_branch gcc_crypto_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2397,8 +2413,8 @@ static struct clk_branch gcc_crypto_axi_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_axi_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2415,8 +2431,8 @@ static struct clk_branch gcc_crypto_clk = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_clk", - .parent_names = (const char *[]){ - "crypto_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &crypto_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2432,8 +2448,8 @@ static struct clk_branch gcc_oxili_gmem_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_gmem_clk", - .parent_names = (const char *[]){ - "gfx3d_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gfx3d_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2449,8 +2465,8 @@ static struct clk_branch gcc_gp1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp1_clk", - .parent_names = (const char *[]){ - "gp1_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gp1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2466,8 +2482,8 @@ static struct clk_branch gcc_gp2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp2_clk", - .parent_names = (const char *[]){ - "gp2_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gp2_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2483,8 +2499,8 @@ static struct clk_branch gcc_gp3_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp3_clk", - .parent_names = (const char *[]){ - "gp3_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gp3_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2500,8 +2516,8 @@ static struct clk_branch gcc_mdss_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2517,8 +2533,8 @@ static struct clk_branch gcc_mdss_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_axi_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2534,8 +2550,8 @@ static struct clk_branch gcc_mdss_byte0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_byte0_clk", - .parent_names = (const char *[]){ - "byte0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &byte0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2551,8 +2567,8 @@ static struct clk_branch gcc_mdss_esc0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_esc0_clk", - .parent_names = (const char *[]){ - "esc0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &esc0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2568,8 +2584,8 @@ static struct clk_branch gcc_mdss_mdp_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_mdp_clk", - .parent_names = (const char *[]){ - "mdp_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &mdp_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2585,8 +2601,8 @@ static struct clk_branch gcc_mdss_pclk0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_pclk0_clk", - .parent_names = (const char *[]){ - "pclk0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pclk0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2602,8 +2618,8 @@ static struct clk_branch gcc_mdss_vsync_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_vsync_clk", - .parent_names = (const char *[]){ - "vsync_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &vsync_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2619,25 +2635,8 @@ static struct clk_branch gcc_mss_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_cfg_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", - }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - -static struct clk_branch gcc_mss_q6_bimc_axi_clk = { - .halt_reg = 0x49004, - .clkr = { - .enable_reg = 0x49004, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_mss_q6_bimc_axi_clk", - .parent_names = (const char *[]){ - "bimc_ddr_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2653,8 +2652,8 @@ static struct clk_branch gcc_oxili_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2670,8 +2669,8 @@ static struct clk_branch gcc_oxili_gfx3d_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_gfx3d_clk", - .parent_names = (const char *[]){ - "gfx3d_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gfx3d_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2687,8 +2686,8 @@ static struct clk_branch gcc_pdm2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm2_clk", - .parent_names = (const char *[]){ - "pdm2_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pdm2_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2704,8 +2703,8 @@ static struct clk_branch gcc_pdm_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2722,8 +2721,8 @@ static struct clk_branch gcc_prng_ahb_clk = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "gcc_prng_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -2738,8 +2737,8 @@ static struct clk_branch gcc_sdcc1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2755,8 +2754,8 @@ static struct clk_branch gcc_sdcc1_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_apps_clk", - .parent_names = (const char *[]){ - "sdcc1_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &sdcc1_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2772,8 +2771,8 @@ static struct clk_branch gcc_sdcc2_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2789,8 +2788,8 @@ static struct clk_branch gcc_sdcc2_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_apps_clk", - .parent_names = (const char *[]){ - "sdcc2_apps_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &sdcc2_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2805,13 +2804,30 @@ static struct clk_rcg2 bimc_ddr_clk_src = { .parent_map = gcc_xo_gpll0_bimc_map, .clkr.hw.init = &(struct clk_init_data){ .name = "bimc_ddr_clk_src", - .parent_names = gcc_xo_gpll0_bimc, - .num_parents = 3, + .parent_data = gcc_xo_gpll0_bimc, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), .ops = &clk_rcg2_ops, .flags = CLK_GET_RATE_NOCACHE, }, }; +static struct clk_branch gcc_mss_q6_bimc_axi_clk = { + .halt_reg = 0x49004, + .clkr = { + .enable_reg = 0x49004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_q6_bimc_axi_clk", + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_apss_tcu_clk = { .halt_reg = 0x12018, .clkr = { @@ -2819,8 +2835,8 @@ static struct clk_branch gcc_apss_tcu_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_tcu_clk", - .parent_names = (const char *[]){ - "bimc_ddr_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -2835,8 +2851,8 @@ static struct clk_branch gcc_gfx_tcu_clk = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gcc_gfx_tcu_clk", - .parent_names = (const char *[]){ - "bimc_ddr_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -2851,8 +2867,8 @@ static struct clk_branch gcc_gtcu_ahb_clk = { .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "gcc_gtcu_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2868,8 +2884,8 @@ static struct clk_branch gcc_bimc_gfx_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_bimc_gfx_clk", - .parent_names = (const char *[]){ - "bimc_gpu_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &bimc_gpu_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2885,8 +2901,8 @@ static struct clk_branch gcc_bimc_gpu_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_bimc_gpu_clk", - .parent_names = (const char *[]){ - "bimc_gpu_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &bimc_gpu_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2902,8 +2918,8 @@ static struct clk_branch gcc_jpeg_tbu_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_jpeg_tbu_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2919,8 +2935,8 @@ static struct clk_branch gcc_mdp_tbu_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "gcc_mdp_tbu_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2936,8 +2952,8 @@ static struct clk_branch gcc_smmu_cfg_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "gcc_smmu_cfg_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2953,8 +2969,8 @@ static struct clk_branch gcc_venus_tbu_clk = { .enable_mask = BIT(5), .hw.init = &(struct clk_init_data){ .name = "gcc_venus_tbu_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2970,8 +2986,8 @@ static struct clk_branch gcc_vfe_tbu_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_vfe_tbu_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2987,8 +3003,8 @@ static struct clk_branch gcc_usb2a_phy_sleep_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb2a_phy_sleep_clk", - .parent_names = (const char *[]){ - "sleep_clk_src", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "sleep_clk", .name = "sleep_clk_src", }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3004,8 +3020,8 @@ static struct clk_branch gcc_usb_hs_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3021,8 +3037,8 @@ static struct clk_branch gcc_usb_hs_system_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_system_clk", - .parent_names = (const char *[]){ - "usb_hs_system_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &usb_hs_system_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3038,8 +3054,8 @@ static struct clk_branch gcc_venus0_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_ahb_clk", - .parent_names = (const char *[]){ - "pcnoc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3055,8 +3071,8 @@ static struct clk_branch gcc_venus0_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_axi_clk", - .parent_names = (const char *[]){ - "system_noc_bfdcd_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &system_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3072,8 +3088,8 @@ static struct clk_branch gcc_venus0_vcodec0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_vcodec0_clk", - .parent_names = (const char *[]){ - "vcodec0_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &vcodec0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c index 8e2d9fb98ad55e66a89b36925b18f8097bfb7b11..af608f1658967a4f85f5c8786f0a7373310865dc 100644 --- a/drivers/clk/qcom/gcc-msm8939.c +++ b/drivers/clk/qcom/gcc-msm8939.c @@ -614,7 +614,7 @@ static struct clk_rcg2 pcnoc_bfdcd_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "pcnoc_bfdcd_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -626,7 +626,7 @@ static struct clk_rcg2 system_noc_bfdcd_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "system_noc_bfdcd_clk_src", .parent_data = gcc_xo_gpll0_gpll6a_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -638,7 +638,7 @@ static struct clk_rcg2 bimc_ddr_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "bimc_ddr_clk_src", .parent_data = gcc_xo_gpll0_bimc_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc_parent_data), .ops = &clk_rcg2_ops, .flags = CLK_GET_RATE_NOCACHE, }, @@ -651,7 +651,7 @@ static struct clk_rcg2 system_mm_noc_bfdcd_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "system_mm_noc_bfdcd_clk_src", .parent_data = gcc_xo_gpll0_gpll6a_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll6a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -671,7 +671,7 @@ static struct clk_rcg2 camss_ahb_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "camss_ahb_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -692,7 +692,7 @@ static struct clk_rcg2 apss_ahb_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "apss_ahb_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -711,7 +711,7 @@ static struct clk_rcg2 csi0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "csi0_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -724,7 +724,7 @@ static struct clk_rcg2 csi1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "csi1_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -753,7 +753,7 @@ static struct clk_rcg2 gfx3d_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "gfx3d_clk_src", .parent_data = gcc_xo_gpll0_gpll2a_gpll3_gpll6a_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll2a_gpll3_gpll6a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -782,7 +782,7 @@ static struct clk_rcg2 vfe0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "vfe0_clk_src", .parent_data = gcc_xo_gpll0_gpll2_gpll4_parent_data, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll2_gpll4_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -801,7 +801,7 @@ static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup1_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -826,7 +826,7 @@ static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup1_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -839,7 +839,7 @@ static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup2_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -853,7 +853,7 @@ static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup2_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -866,7 +866,7 @@ static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup3_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -880,7 +880,7 @@ static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup3_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -893,7 +893,7 @@ static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup4_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -907,7 +907,7 @@ static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup4_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -920,7 +920,7 @@ static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup5_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -934,7 +934,7 @@ static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup5_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -947,7 +947,7 @@ static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup6_i2c_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -961,7 +961,7 @@ static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_qup6_spi_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -994,7 +994,7 @@ static struct clk_rcg2 blsp1_uart1_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_uart1_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1008,7 +1008,7 @@ static struct clk_rcg2 blsp1_uart2_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "blsp1_uart2_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1028,7 +1028,7 @@ static struct clk_rcg2 cci_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "cci_clk_src", .parent_data = gcc_xo_gpll0a_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1048,7 +1048,7 @@ static struct clk_rcg2 camss_gp0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "camss_gp0_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1062,7 +1062,7 @@ static struct clk_rcg2 camss_gp1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "camss_gp1_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1082,7 +1082,7 @@ static struct clk_rcg2 jpeg0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "jpeg0_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1102,7 +1102,7 @@ static struct clk_rcg2 mclk0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "mclk0_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1116,7 +1116,7 @@ static struct clk_rcg2 mclk1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "mclk1_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_gpll6_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1135,7 +1135,7 @@ static struct clk_rcg2 csi0phytimer_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "csi0phytimer_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1148,7 +1148,7 @@ static struct clk_rcg2 csi1phytimer_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "csi1phytimer_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1171,7 +1171,7 @@ static struct clk_rcg2 cpp_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "cpp_clk_src", .parent_data = gcc_xo_gpll0_gpll2_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll2_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1193,7 +1193,7 @@ static struct clk_rcg2 crypto_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "crypto_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1212,7 +1212,7 @@ static struct clk_rcg2 gp1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "gp1_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1226,7 +1226,7 @@ static struct clk_rcg2 gp2_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "gp2_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1240,7 +1240,7 @@ static struct clk_rcg2 gp3_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "gp3_clk_src", .parent_data = gcc_xo_gpll0_gpll1a_sleep_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1a_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1252,7 +1252,7 @@ static struct clk_rcg2 byte0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "byte0_clk_src", .parent_data = gcc_xo_gpll0a_dsibyte_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsibyte_parent_data), .ops = &clk_byte2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1265,7 +1265,7 @@ static struct clk_rcg2 byte1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "byte1_clk_src", .parent_data = gcc_xo_gpll0a_dsibyte_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsibyte_parent_data), .ops = &clk_byte2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1284,7 +1284,7 @@ static struct clk_rcg2 esc0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "esc0_clk_src", .parent_data = gcc_xo_dsibyte_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_dsibyte_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1297,7 +1297,7 @@ static struct clk_rcg2 esc1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "esc1_clk_src", .parent_data = gcc_xo_dsibyte_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_dsibyte_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1325,7 +1325,7 @@ static struct clk_rcg2 mdp_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "mdp_clk_src", .parent_data = gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_parent_data, - .num_parents = 6, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_dsiphy_gpll6_gpll3a_gpll0a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1338,7 +1338,7 @@ static struct clk_rcg2 pclk0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "pclk0_clk_src", .parent_data = gcc_xo_gpll0a_dsiphy_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsiphy_parent_data), .ops = &clk_pixel_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1352,7 +1352,7 @@ static struct clk_rcg2 pclk1_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "pclk1_clk_src", .parent_data = gcc_xo_gpll0a_dsiphy_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_dsiphy_parent_data), .ops = &clk_pixel_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -1371,7 +1371,7 @@ static struct clk_rcg2 vsync_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "vsync_clk_src", .parent_data = gcc_xo_gpll0a_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1390,7 +1390,7 @@ static struct clk_rcg2 pdm2_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "pdm2_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1416,7 +1416,7 @@ static struct clk_rcg2 sdcc1_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "sdcc1_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_floor_ops, }, }; @@ -1430,7 +1430,7 @@ static struct clk_rcg2 sdcc2_apps_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "sdcc2_apps_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_floor_ops, }, }; @@ -1450,7 +1450,7 @@ static struct clk_rcg2 apss_tcu_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "apss_tcu_clk_src", .parent_data = gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1473,7 +1473,7 @@ static struct clk_rcg2 bimc_gpu_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "bimc_gpu_clk_src", .parent_data = gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll5a_gpll6_bimc_parent_data), .flags = CLK_GET_RATE_NOCACHE, .ops = &clk_rcg2_ops, }, @@ -1494,7 +1494,7 @@ static struct clk_rcg2 usb_hs_system_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "usb_hs_system_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1512,7 +1512,7 @@ static struct clk_rcg2 usb_fs_system_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "usb_fs_system_clk_src", .parent_data = gcc_xo_gpll6_gpll0_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll6_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1530,7 +1530,7 @@ static struct clk_rcg2 usb_fs_ic_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "usb_fs_ic_clk_src", .parent_data = gcc_xo_gpll6_gpll0a_parent_data, - .num_parents = 3, + .num_parents = ARRAY_SIZE(gcc_xo_gpll6_gpll0a_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1556,7 +1556,7 @@ static struct clk_rcg2 ultaudio_ahbfabric_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_ahbfabric_clk_src", .parent_data = gcc_xo_gpll0_gpll1_sleep_parent_data, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_gpll1_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1568,8 +1568,8 @@ static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_ahbfabric_ixfabric_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_ahbfabric_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_ahbfabric_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1585,8 +1585,8 @@ static struct clk_branch gcc_ultaudio_ahbfabric_ixfabric_lpm_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_ahbfabric_ixfabric_lpm_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_ahbfabric_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_ahbfabric_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1635,7 +1635,7 @@ static struct clk_rcg2 ultaudio_lpaif_pri_i2s_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_pri_i2s_clk_src", .parent_data = gcc_xo_gpll1_epi2s_emclk_sleep_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_epi2s_emclk_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1647,8 +1647,8 @@ static struct clk_branch gcc_ultaudio_lpaif_pri_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_pri_i2s_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_lpaif_pri_i2s_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_pri_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1666,7 +1666,7 @@ static struct clk_rcg2 ultaudio_lpaif_sec_i2s_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_sec_i2s_clk_src", .parent_data = gcc_xo_gpll1_esi2s_emclk_sleep_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_esi2s_emclk_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1678,8 +1678,8 @@ static struct clk_branch gcc_ultaudio_lpaif_sec_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_sec_i2s_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_lpaif_sec_i2s_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_sec_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1697,7 +1697,7 @@ static struct clk_rcg2 ultaudio_lpaif_aux_i2s_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_lpaif_aux_i2s_clk_src", .parent_data = gcc_xo_gpll1_esi2s_emclk_sleep_parent_data, - .num_parents = 5, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_esi2s_emclk_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1709,8 +1709,8 @@ static struct clk_branch gcc_ultaudio_lpaif_aux_i2s_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_lpaif_aux_i2s_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_lpaif_aux_i2s_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_lpaif_aux_i2s_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1732,7 +1732,7 @@ static struct clk_rcg2 ultaudio_xo_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "ultaudio_xo_clk_src", .parent_data = gcc_xo_sleep_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1744,8 +1744,8 @@ static struct clk_branch gcc_ultaudio_avsync_xo_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_avsync_xo_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_xo_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_xo_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1761,8 +1761,8 @@ static struct clk_branch gcc_ultaudio_stc_xo_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_stc_xo_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &ultaudio_xo_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &ultaudio_xo_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1788,7 +1788,7 @@ static struct clk_rcg2 codec_digcodec_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "codec_digcodec_clk_src", .parent_data = gcc_xo_gpll1_emclk_sleep_parent_data, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gcc_xo_gpll1_emclk_sleep_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1800,8 +1800,8 @@ static struct clk_branch gcc_codec_digcodec_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_codec_digcodec_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &codec_digcodec_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &codec_digcodec_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1817,8 +1817,8 @@ static struct clk_branch gcc_ultaudio_pcnoc_mport_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_pcnoc_mport_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1833,8 +1833,8 @@ static struct clk_branch gcc_ultaudio_pcnoc_sway_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_ultaudio_pcnoc_sway_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1858,7 +1858,7 @@ static struct clk_rcg2 vcodec0_clk_src = { .clkr.hw.init = &(struct clk_init_data){ .name = "vcodec0_clk_src", .parent_data = gcc_xo_gpll0_parent_data, - .num_parents = 2, + .num_parents = ARRAY_SIZE(gcc_xo_gpll0_parent_data), .ops = &clk_rcg2_ops, }, }; @@ -1871,8 +1871,8 @@ static struct clk_branch gcc_blsp1_ahb_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -1899,8 +1899,8 @@ static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup1_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup1_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup1_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1916,8 +1916,8 @@ static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup1_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup1_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup1_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1933,8 +1933,8 @@ static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup2_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup2_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup2_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1950,8 +1950,8 @@ static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup2_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup2_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup2_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1967,8 +1967,8 @@ static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup3_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup3_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup3_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -1984,8 +1984,8 @@ static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup3_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup3_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup3_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2001,8 +2001,8 @@ static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup4_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup4_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup4_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2018,8 +2018,8 @@ static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup4_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup4_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup4_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2035,8 +2035,8 @@ static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup5_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup5_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup5_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2052,8 +2052,8 @@ static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup5_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup5_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup5_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2069,8 +2069,8 @@ static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup6_i2c_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup6_i2c_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup6_i2c_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2086,8 +2086,8 @@ static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_qup6_spi_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_qup6_spi_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_qup6_spi_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2103,8 +2103,8 @@ static struct clk_branch gcc_blsp1_uart1_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_uart1_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_uart1_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_uart1_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2120,8 +2120,8 @@ static struct clk_branch gcc_blsp1_uart2_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_uart2_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &blsp1_uart2_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &blsp1_uart2_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2138,8 +2138,8 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "gcc_boot_rom_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -2154,8 +2154,8 @@ static struct clk_branch gcc_camss_cci_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cci_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2171,8 +2171,8 @@ static struct clk_branch gcc_camss_cci_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cci_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &cci_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &cci_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2188,8 +2188,8 @@ static struct clk_branch gcc_camss_csi0_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2205,8 +2205,8 @@ static struct clk_branch gcc_camss_csi0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2222,8 +2222,8 @@ static struct clk_branch gcc_camss_csi0phy_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0phy_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2239,8 +2239,8 @@ static struct clk_branch gcc_camss_csi0pix_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0pix_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2256,8 +2256,8 @@ static struct clk_branch gcc_camss_csi0rdi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0rdi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2273,8 +2273,8 @@ static struct clk_branch gcc_camss_csi1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2290,8 +2290,8 @@ static struct clk_branch gcc_camss_csi1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2307,8 +2307,8 @@ static struct clk_branch gcc_camss_csi1phy_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1phy_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2324,8 +2324,8 @@ static struct clk_branch gcc_camss_csi1pix_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1pix_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2341,8 +2341,8 @@ static struct clk_branch gcc_camss_csi1rdi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1rdi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2358,8 +2358,8 @@ static struct clk_branch gcc_camss_csi_vfe0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi_vfe0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vfe0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vfe0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2375,8 +2375,8 @@ static struct clk_branch gcc_camss_gp0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_gp0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_gp0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_gp0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2392,8 +2392,8 @@ static struct clk_branch gcc_camss_gp1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_gp1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_gp1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_gp1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2409,8 +2409,8 @@ static struct clk_branch gcc_camss_ispif_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_ispif_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2426,8 +2426,8 @@ static struct clk_branch gcc_camss_jpeg0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &jpeg0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &jpeg0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2443,8 +2443,8 @@ static struct clk_branch gcc_camss_jpeg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2460,8 +2460,8 @@ static struct clk_branch gcc_camss_jpeg_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_jpeg_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2477,8 +2477,8 @@ static struct clk_branch gcc_camss_mclk0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_mclk0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &mclk0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &mclk0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2494,8 +2494,8 @@ static struct clk_branch gcc_camss_mclk1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_mclk1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &mclk1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &mclk1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2511,8 +2511,8 @@ static struct clk_branch gcc_camss_micro_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_micro_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2528,8 +2528,8 @@ static struct clk_branch gcc_camss_csi0phytimer_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi0phytimer_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi0phytimer_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi0phytimer_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2545,8 +2545,8 @@ static struct clk_branch gcc_camss_csi1phytimer_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_csi1phytimer_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &csi1phytimer_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &csi1phytimer_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2562,8 +2562,8 @@ static struct clk_branch gcc_camss_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2579,8 +2579,8 @@ static struct clk_branch gcc_camss_top_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_top_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2596,8 +2596,8 @@ static struct clk_branch gcc_camss_cpp_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cpp_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2613,8 +2613,8 @@ static struct clk_branch gcc_camss_cpp_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_cpp_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &cpp_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &cpp_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2630,8 +2630,8 @@ static struct clk_branch gcc_camss_vfe0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vfe0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vfe0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2647,8 +2647,8 @@ static struct clk_branch gcc_camss_vfe_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &camss_ahb_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &camss_ahb_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2664,8 +2664,8 @@ static struct clk_branch gcc_camss_vfe_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_camss_vfe_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2682,8 +2682,8 @@ static struct clk_branch gcc_crypto_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2700,8 +2700,8 @@ static struct clk_branch gcc_crypto_axi_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2718,8 +2718,8 @@ static struct clk_branch gcc_crypto_clk = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &crypto_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &crypto_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2735,8 +2735,8 @@ static struct clk_branch gcc_oxili_gmem_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_gmem_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &gfx3d_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &gfx3d_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2752,8 +2752,8 @@ static struct clk_branch gcc_gp1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &gp1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &gp1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2769,8 +2769,8 @@ static struct clk_branch gcc_gp2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp2_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &gp2_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &gp2_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2786,8 +2786,8 @@ static struct clk_branch gcc_gp3_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_gp3_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &gp3_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &gp3_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2803,8 +2803,8 @@ static struct clk_branch gcc_mdss_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2820,8 +2820,8 @@ static struct clk_branch gcc_mdss_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2837,8 +2837,8 @@ static struct clk_branch gcc_mdss_byte0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_byte0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &byte0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &byte0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2854,8 +2854,8 @@ static struct clk_branch gcc_mdss_byte1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_byte1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &byte1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &byte1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2871,8 +2871,8 @@ static struct clk_branch gcc_mdss_esc0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_esc0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &esc0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &esc0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2888,8 +2888,8 @@ static struct clk_branch gcc_mdss_esc1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_esc1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &esc1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &esc1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2905,8 +2905,8 @@ static struct clk_branch gcc_mdss_mdp_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_mdp_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &mdp_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &mdp_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2922,8 +2922,8 @@ static struct clk_branch gcc_mdss_pclk0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_pclk0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pclk0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pclk0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2939,8 +2939,8 @@ static struct clk_branch gcc_mdss_pclk1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_pclk1_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pclk1_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pclk1_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2956,8 +2956,8 @@ static struct clk_branch gcc_mdss_vsync_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mdss_vsync_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vsync_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vsync_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2973,8 +2973,8 @@ static struct clk_branch gcc_mss_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_cfg_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -2990,8 +2990,8 @@ static struct clk_branch gcc_mss_q6_bimc_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_q6_bimc_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_ddr_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3007,8 +3007,8 @@ static struct clk_branch gcc_oxili_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3024,8 +3024,8 @@ static struct clk_branch gcc_oxili_gfx3d_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_oxili_gfx3d_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &gfx3d_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &gfx3d_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3041,8 +3041,8 @@ static struct clk_branch gcc_pdm2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm2_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pdm2_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pdm2_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3058,8 +3058,8 @@ static struct clk_branch gcc_pdm_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3076,8 +3076,8 @@ static struct clk_branch gcc_prng_ahb_clk = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "gcc_prng_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -3092,8 +3092,8 @@ static struct clk_branch gcc_sdcc1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3109,8 +3109,8 @@ static struct clk_branch gcc_sdcc1_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &sdcc1_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &sdcc1_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3126,8 +3126,8 @@ static struct clk_branch gcc_sdcc2_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3143,8 +3143,8 @@ static struct clk_branch gcc_sdcc2_apps_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_apps_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &sdcc2_apps_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &sdcc2_apps_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3161,8 +3161,8 @@ static struct clk_branch gcc_apss_tcu_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_tcu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_ddr_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -3178,8 +3178,8 @@ static struct clk_branch gcc_gfx_tcu_clk = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gcc_gfx_tcu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_ddr_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -3195,8 +3195,8 @@ static struct clk_branch gcc_gfx_tbu_clk = { .enable_mask = BIT(3), .hw.init = &(struct clk_init_data){ .name = "gcc_gfx_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_ddr_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_ddr_clk_src.clkr.hw, }, .num_parents = 1, .ops = &clk_branch2_ops, @@ -3212,8 +3212,8 @@ static struct clk_branch gcc_mdp_tbu_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "gcc_mdp_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3230,8 +3230,8 @@ static struct clk_branch gcc_venus_tbu_clk = { .enable_mask = BIT(5), .hw.init = &(struct clk_init_data){ .name = "gcc_venus_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3248,8 +3248,8 @@ static struct clk_branch gcc_vfe_tbu_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_vfe_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3266,8 +3266,8 @@ static struct clk_branch gcc_jpeg_tbu_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_jpeg_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3284,8 +3284,8 @@ static struct clk_branch gcc_smmu_cfg_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "gcc_smmu_cfg_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3302,8 +3302,8 @@ static struct clk_branch gcc_gtcu_ahb_clk = { .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "gcc_gtcu_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3320,8 +3320,8 @@ static struct clk_branch gcc_cpp_tbu_clk = { .enable_mask = BIT(14), .hw.init = &(struct clk_init_data){ .name = "gcc_cpp_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3338,8 +3338,8 @@ static struct clk_branch gcc_mdp_rt_tbu_clk = { .enable_mask = BIT(15), .hw.init = &(struct clk_init_data){ .name = "gcc_mdp_rt_tbu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3355,8 +3355,8 @@ static struct clk_branch gcc_bimc_gfx_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_bimc_gfx_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_gpu_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_gpu_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3372,8 +3372,8 @@ static struct clk_branch gcc_bimc_gpu_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_bimc_gpu_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &bimc_gpu_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &bimc_gpu_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3401,8 +3401,8 @@ static struct clk_branch gcc_usb_fs_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_fs_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3418,8 +3418,8 @@ static struct clk_branch gcc_usb_fs_ic_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_fs_ic_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &usb_fs_ic_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs_ic_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3435,8 +3435,8 @@ static struct clk_branch gcc_usb_fs_system_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_fs_system_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &usb_fs_system_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs_system_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3452,8 +3452,8 @@ static struct clk_branch gcc_usb_hs_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3469,8 +3469,8 @@ static struct clk_branch gcc_usb_hs_system_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_system_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &usb_hs_system_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &usb_hs_system_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3486,8 +3486,8 @@ static struct clk_branch gcc_venus0_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_ahb_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &pcnoc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &pcnoc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3503,8 +3503,8 @@ static struct clk_branch gcc_venus0_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_axi_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &system_mm_noc_bfdcd_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &system_mm_noc_bfdcd_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3520,8 +3520,8 @@ static struct clk_branch gcc_venus0_vcodec0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_vcodec0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vcodec0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vcodec0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3537,8 +3537,8 @@ static struct clk_branch gcc_venus0_core0_vcodec0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_core0_vcodec0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vcodec0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vcodec0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, @@ -3554,8 +3554,8 @@ static struct clk_branch gcc_venus0_core1_vcodec0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_venus0_core1_vcodec0_clk", - .parent_data = &(const struct clk_parent_data){ - .hw = &vcodec0_clk_src.clkr.hw, + .parent_hws = (const struct clk_hw*[]){ + &vcodec0_clk_src.clkr.hw, }, .num_parents = 1, .flags = CLK_SET_RATE_PARENT, diff --git a/drivers/clk/qcom/gcc-msm8960.c b/drivers/clk/qcom/gcc-msm8960.c index a6e13b91e4c84f53306203eef29ab07161f079d0..9dd4e7ffa1f8f8217f5c6e18d69e7af55a5ece1b 100644 --- a/drivers/clk/qcom/gcc-msm8960.c +++ b/drivers/clk/qcom/gcc-msm8960.c @@ -35,7 +35,9 @@ static struct clk_pll pll3 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll3", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -46,7 +48,9 @@ static struct clk_regmap pll4_vote = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "pll4_vote", - .parent_names = (const char *[]){ "pll4" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pll4", .name = "pll4", + }, .num_parents = 1, .ops = &clk_pll_vote_ops, }, @@ -62,7 +66,9 @@ static struct clk_pll pll8 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll8", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -73,7 +79,9 @@ static struct clk_regmap pll8_vote = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "pll8_vote", - .parent_names = (const char *[]){ "pll8" }, + .parent_hws = (const struct clk_hw*[]){ + &pll8.clkr.hw + }, .num_parents = 1, .ops = &clk_pll_vote_ops, }, @@ -96,7 +104,9 @@ static struct hfpll_data hfpll0_data = { static struct clk_hfpll hfpll0 = { .d = &hfpll0_data, .clkr.hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .name = "hfpll0", .ops = &clk_ops_hfpll, @@ -136,7 +146,9 @@ static struct hfpll_data hfpll1_data = { static struct clk_hfpll hfpll1 = { .d = &hfpll1_data, .clkr.hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .name = "hfpll1", .ops = &clk_ops_hfpll, @@ -162,7 +174,9 @@ static struct hfpll_data hfpll2_data = { static struct clk_hfpll hfpll2 = { .d = &hfpll2_data, .clkr.hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .name = "hfpll2", .ops = &clk_ops_hfpll, @@ -188,7 +202,9 @@ static struct hfpll_data hfpll3_data = { static struct clk_hfpll hfpll3 = { .d = &hfpll3_data, .clkr.hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .name = "hfpll3", .ops = &clk_ops_hfpll, @@ -228,7 +244,9 @@ static struct hfpll_data hfpll_l2_data = { static struct clk_hfpll hfpll_l2 = { .d = &hfpll_l2_data, .clkr.hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .name = "hfpll_l2", .ops = &clk_ops_hfpll, @@ -247,7 +265,9 @@ static struct clk_pll pll14 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll14", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -258,7 +278,9 @@ static struct clk_regmap pll14_vote = { .enable_mask = BIT(14), .hw.init = &(struct clk_init_data){ .name = "pll14_vote", - .parent_names = (const char *[]){ "pll14" }, + .parent_hws = (const struct clk_hw*[]){ + &pll14.clkr.hw + }, .num_parents = 1, .ops = &clk_pll_vote_ops, }, @@ -276,9 +298,9 @@ static const struct parent_map gcc_pxo_pll8_map[] = { { P_PLL8, 3 } }; -static const char * const gcc_pxo_pll8[] = { - "pxo", - "pll8_vote", +static const struct clk_parent_data gcc_pxo_pll8[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .hw = &pll8_vote.hw }, }; static const struct parent_map gcc_pxo_pll8_cxo_map[] = { @@ -287,10 +309,10 @@ static const struct parent_map gcc_pxo_pll8_cxo_map[] = { { P_CXO, 5 } }; -static const char * const gcc_pxo_pll8_cxo[] = { - "pxo", - "pll8_vote", - "cxo", +static const struct clk_parent_data gcc_pxo_pll8_cxo[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .hw = &pll8_vote.hw }, + { .fw_name = "cxo", .name = "cxo_board" }, }; static const struct parent_map gcc_pxo_pll8_pll3_map[] = { @@ -299,10 +321,10 @@ static const struct parent_map gcc_pxo_pll8_pll3_map[] = { { P_PLL3, 6 } }; -static const char * const gcc_pxo_pll8_pll3[] = { - "pxo", - "pll8_vote", - "pll3", +static const struct clk_parent_data gcc_pxo_pll8_pll3[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .hw = &pll8_vote.hw }, + { .hw = &pll3.clkr.hw }, }; static struct freq_tbl clk_tbl_gsbi_uart[] = { @@ -348,8 +370,8 @@ static struct clk_rcg gsbi1_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi1_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -364,8 +386,8 @@ static struct clk_branch gsbi1_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi1_uart_clk", - .parent_names = (const char *[]){ - "gsbi1_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi1_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -399,8 +421,8 @@ static struct clk_rcg gsbi2_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi2_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -415,8 +437,8 @@ static struct clk_branch gsbi2_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi2_uart_clk", - .parent_names = (const char *[]){ - "gsbi2_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi2_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -450,8 +472,8 @@ static struct clk_rcg gsbi3_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi3_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -466,8 +488,8 @@ static struct clk_branch gsbi3_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi3_uart_clk", - .parent_names = (const char *[]){ - "gsbi3_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi3_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -501,8 +523,8 @@ static struct clk_rcg gsbi4_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi4_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -517,8 +539,8 @@ static struct clk_branch gsbi4_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi4_uart_clk", - .parent_names = (const char *[]){ - "gsbi4_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi4_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -552,8 +574,8 @@ static struct clk_rcg gsbi5_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi5_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -568,8 +590,8 @@ static struct clk_branch gsbi5_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi5_uart_clk", - .parent_names = (const char *[]){ - "gsbi5_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi5_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -603,8 +625,8 @@ static struct clk_rcg gsbi6_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi6_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -619,8 +641,8 @@ static struct clk_branch gsbi6_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi6_uart_clk", - .parent_names = (const char *[]){ - "gsbi6_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi6_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -654,8 +676,8 @@ static struct clk_rcg gsbi7_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi7_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -670,8 +692,8 @@ static struct clk_branch gsbi7_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi7_uart_clk", - .parent_names = (const char *[]){ - "gsbi7_uart_src", + .parent_hws = (const struct clk_hw*[]){ + &gsbi7_uart_src.clkr.hw }, .num_parents = 1, .ops = &clk_branch_ops, @@ -705,8 +727,8 @@ static struct clk_rcg gsbi8_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi8_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -721,7 +743,9 @@ static struct clk_branch gsbi8_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi8_uart_clk", - .parent_names = (const char *[]){ "gsbi8_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi8_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -754,8 +778,8 @@ static struct clk_rcg gsbi9_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi9_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -770,7 +794,9 @@ static struct clk_branch gsbi9_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi9_uart_clk", - .parent_names = (const char *[]){ "gsbi9_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi9_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -803,8 +829,8 @@ static struct clk_rcg gsbi10_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi10_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -819,7 +845,9 @@ static struct clk_branch gsbi10_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi10_uart_clk", - .parent_names = (const char *[]){ "gsbi10_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi10_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -852,8 +880,8 @@ static struct clk_rcg gsbi11_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi11_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -868,7 +896,9 @@ static struct clk_branch gsbi11_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi11_uart_clk", - .parent_names = (const char *[]){ "gsbi11_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi11_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -901,8 +931,8 @@ static struct clk_rcg gsbi12_uart_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi12_uart_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -917,7 +947,9 @@ static struct clk_branch gsbi12_uart_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi12_uart_clk", - .parent_names = (const char *[]){ "gsbi12_uart_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi12_uart_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -963,8 +995,8 @@ static struct clk_rcg gsbi1_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi1_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -979,7 +1011,9 @@ static struct clk_branch gsbi1_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi1_qup_clk", - .parent_names = (const char *[]){ "gsbi1_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi1_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1012,8 +1046,8 @@ static struct clk_rcg gsbi2_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi2_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1028,7 +1062,9 @@ static struct clk_branch gsbi2_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi2_qup_clk", - .parent_names = (const char *[]){ "gsbi2_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi2_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1061,8 +1097,8 @@ static struct clk_rcg gsbi3_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi3_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1077,7 +1113,9 @@ static struct clk_branch gsbi3_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi3_qup_clk", - .parent_names = (const char *[]){ "gsbi3_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi3_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1110,8 +1148,8 @@ static struct clk_rcg gsbi4_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi4_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1126,7 +1164,9 @@ static struct clk_branch gsbi4_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi4_qup_clk", - .parent_names = (const char *[]){ "gsbi4_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi4_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1159,8 +1199,8 @@ static struct clk_rcg gsbi5_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi5_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1175,7 +1215,9 @@ static struct clk_branch gsbi5_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi5_qup_clk", - .parent_names = (const char *[]){ "gsbi5_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi5_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1208,8 +1250,8 @@ static struct clk_rcg gsbi6_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi6_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1224,7 +1266,9 @@ static struct clk_branch gsbi6_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi6_qup_clk", - .parent_names = (const char *[]){ "gsbi6_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi6_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1257,8 +1301,8 @@ static struct clk_rcg gsbi7_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi7_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1273,7 +1317,9 @@ static struct clk_branch gsbi7_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi7_qup_clk", - .parent_names = (const char *[]){ "gsbi7_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi7_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1306,8 +1352,8 @@ static struct clk_rcg gsbi8_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi8_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1322,7 +1368,9 @@ static struct clk_branch gsbi8_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi8_qup_clk", - .parent_names = (const char *[]){ "gsbi8_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi8_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1355,8 +1403,8 @@ static struct clk_rcg gsbi9_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi9_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1371,7 +1419,9 @@ static struct clk_branch gsbi9_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi9_qup_clk", - .parent_names = (const char *[]){ "gsbi9_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi9_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1404,8 +1454,8 @@ static struct clk_rcg gsbi10_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi10_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1420,7 +1470,9 @@ static struct clk_branch gsbi10_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi10_qup_clk", - .parent_names = (const char *[]){ "gsbi10_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi10_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1453,8 +1505,8 @@ static struct clk_rcg gsbi11_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi11_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1469,7 +1521,9 @@ static struct clk_branch gsbi11_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi11_qup_clk", - .parent_names = (const char *[]){ "gsbi11_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi11_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1502,8 +1556,8 @@ static struct clk_rcg gsbi12_qup_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gsbi12_qup_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1518,7 +1572,9 @@ static struct clk_branch gsbi12_qup_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gsbi12_qup_clk", - .parent_names = (const char *[]){ "gsbi12_qup_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gsbi12_qup_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1564,8 +1620,8 @@ static struct clk_rcg gp0_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp0_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_PARENT_GATE, }, @@ -1580,7 +1636,9 @@ static struct clk_branch gp0_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp0_clk", - .parent_names = (const char *[]){ "gp0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp0_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1613,8 +1671,8 @@ static struct clk_rcg gp1_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp1_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1629,7 +1687,9 @@ static struct clk_branch gp1_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp1_clk", - .parent_names = (const char *[]){ "gp1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1662,8 +1722,8 @@ static struct clk_rcg gp2_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "gp2_src", - .parent_names = gcc_pxo_pll8_cxo, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_cxo, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_cxo), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -1678,7 +1738,9 @@ static struct clk_branch gp2_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gp2_clk", - .parent_names = (const char *[]){ "gp2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gp2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1714,8 +1776,8 @@ static struct clk_rcg prng_src = { .clkr = { .hw.init = &(struct clk_init_data){ .name = "prng_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, }, @@ -1730,7 +1792,9 @@ static struct clk_branch prng_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "prng_clk", - .parent_names = (const char *[]){ "prng_src" }, + .parent_hws = (const struct clk_hw*[]){ + &prng_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, }, @@ -1776,8 +1840,8 @@ static struct clk_rcg sdc1_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc1_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1791,7 +1855,9 @@ static struct clk_branch sdc1_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc1_clk", - .parent_names = (const char *[]){ "sdc1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1824,8 +1890,8 @@ static struct clk_rcg sdc2_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc2_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1839,7 +1905,9 @@ static struct clk_branch sdc2_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc2_clk", - .parent_names = (const char *[]){ "sdc2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1872,8 +1940,8 @@ static struct clk_rcg sdc3_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc3_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1887,7 +1955,9 @@ static struct clk_branch sdc3_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc3_clk", - .parent_names = (const char *[]){ "sdc3_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc3_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1920,8 +1990,8 @@ static struct clk_rcg sdc4_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc4_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1935,7 +2005,9 @@ static struct clk_branch sdc4_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc4_clk", - .parent_names = (const char *[]){ "sdc4_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc4_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1968,8 +2040,8 @@ static struct clk_rcg sdc5_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "sdc5_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, }, } @@ -1983,7 +2055,9 @@ static struct clk_branch sdc5_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "sdc5_clk", - .parent_names = (const char *[]){ "sdc5_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sdc5_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2021,8 +2095,8 @@ static struct clk_rcg tsif_ref_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "tsif_ref_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2037,7 +2111,9 @@ static struct clk_branch tsif_ref_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "tsif_ref_clk", - .parent_names = (const char *[]){ "tsif_ref_src" }, + .parent_hws = (const struct clk_hw*[]){ + &tsif_ref_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2075,8 +2151,8 @@ static struct clk_rcg usb_hs1_xcvr_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_hs1_xcvr_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2091,7 +2167,9 @@ static struct clk_branch usb_hs1_xcvr_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_hs1_xcvr_clk", - .parent_names = (const char *[]){ "usb_hs1_xcvr_src" }, + .parent_hws = (const struct clk_hw*[]){ + &usb_hs1_xcvr_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2124,8 +2202,8 @@ static struct clk_rcg usb_hs3_xcvr_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_hs3_xcvr_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2140,7 +2218,9 @@ static struct clk_branch usb_hs3_xcvr_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_hs3_xcvr_clk", - .parent_names = (const char *[]){ "usb_hs3_xcvr_src" }, + .parent_hws = (const struct clk_hw*[]){ + &usb_hs3_xcvr_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2173,8 +2253,8 @@ static struct clk_rcg usb_hs4_xcvr_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_hs4_xcvr_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2189,7 +2269,9 @@ static struct clk_branch usb_hs4_xcvr_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_hs4_xcvr_clk", - .parent_names = (const char *[]){ "usb_hs4_xcvr_src" }, + .parent_hws = (const struct clk_hw*[]){ + &usb_hs4_xcvr_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2222,16 +2304,14 @@ static struct clk_rcg usb_hsic_xcvr_fs_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_hsic_xcvr_fs_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, } }; -static const char * const usb_hsic_xcvr_fs_src_p[] = { "usb_hsic_xcvr_fs_src" }; - static struct clk_branch usb_hsic_xcvr_fs_clk = { .halt_reg = 0x2fc8, .halt_bit = 2, @@ -2240,7 +2320,9 @@ static struct clk_branch usb_hsic_xcvr_fs_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_hsic_xcvr_fs_clk", - .parent_names = usb_hsic_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_hsic_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2255,7 +2337,9 @@ static struct clk_branch usb_hsic_system_clk = { .enable_reg = 0x292c, .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ - .parent_names = usb_hsic_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_hsic_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .name = "usb_hsic_system_clk", .ops = &clk_branch_ops, @@ -2271,7 +2355,9 @@ static struct clk_branch usb_hsic_hsic_clk = { .enable_reg = 0x2b44, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pll14_vote" }, + .parent_hws = (const struct clk_hw*[]){ + &pll14_vote.hw + }, .num_parents = 1, .name = "usb_hsic_hsic_clk", .ops = &clk_branch_ops, @@ -2317,16 +2403,14 @@ static struct clk_rcg usb_fs1_xcvr_fs_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_fs1_xcvr_fs_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, } }; -static const char * const usb_fs1_xcvr_fs_src_p[] = { "usb_fs1_xcvr_fs_src" }; - static struct clk_branch usb_fs1_xcvr_fs_clk = { .halt_reg = 0x2fcc, .halt_bit = 15, @@ -2335,7 +2419,9 @@ static struct clk_branch usb_fs1_xcvr_fs_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_fs1_xcvr_fs_clk", - .parent_names = usb_fs1_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs1_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2350,7 +2436,9 @@ static struct clk_branch usb_fs1_system_clk = { .enable_reg = 0x296c, .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ - .parent_names = usb_fs1_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs1_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .name = "usb_fs1_system_clk", .ops = &clk_branch_ops, @@ -2384,16 +2472,14 @@ static struct clk_rcg usb_fs2_xcvr_fs_src = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_xcvr_fs_src", - .parent_names = gcc_pxo_pll8, - .num_parents = 2, + .parent_data = gcc_pxo_pll8, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, } }; -static const char * const usb_fs2_xcvr_fs_src_p[] = { "usb_fs2_xcvr_fs_src" }; - static struct clk_branch usb_fs2_xcvr_fs_clk = { .halt_reg = 0x2fcc, .halt_bit = 12, @@ -2402,7 +2488,9 @@ static struct clk_branch usb_fs2_xcvr_fs_clk = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_xcvr_fs_clk", - .parent_names = usb_fs2_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs2_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2418,7 +2506,9 @@ static struct clk_branch usb_fs2_system_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "usb_fs2_system_clk", - .parent_names = usb_fs2_xcvr_fs_src_p, + .parent_hws = (const struct clk_hw*[]){ + &usb_fs2_xcvr_fs_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2872,8 +2962,8 @@ static struct clk_rcg ce3_src = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "ce3_src", - .parent_names = gcc_pxo_pll8_pll3, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_pll3, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_pll3), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2888,7 +2978,9 @@ static struct clk_branch ce3_core_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "ce3_core_clk", - .parent_names = (const char *[]){ "ce3_src" }, + .parent_hws = (const struct clk_hw*[]){ + &ce3_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2904,7 +2996,9 @@ static struct clk_branch ce3_h_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "ce3_h_clk", - .parent_names = (const char *[]){ "ce3_src" }, + .parent_hws = (const struct clk_hw*[]){ + &ce3_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2934,8 +3028,8 @@ static struct clk_rcg sata_clk_src = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "sata_clk_src", - .parent_names = gcc_pxo_pll8_pll3, - .num_parents = 3, + .parent_data = gcc_pxo_pll8_pll3, + .num_parents = ARRAY_SIZE(gcc_pxo_pll8_pll3), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -2950,7 +3044,9 @@ static struct clk_branch sata_rxoob_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "sata_rxoob_clk", - .parent_names = (const char *[]){ "sata_clk_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sata_clk_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2966,7 +3062,9 @@ static struct clk_branch sata_pmalive_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "sata_pmalive_clk", - .parent_names = (const char *[]){ "sata_clk_src" }, + .parent_hws = (const struct clk_hw*[]){ + &sata_clk_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2982,7 +3080,9 @@ static struct clk_branch sata_phy_ref_clk = { .enable_mask = BIT(4), .hw.init = &(struct clk_init_data){ .name = "sata_phy_ref_clk", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data){ + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_branch_ops, }, diff --git a/drivers/clk/qcom/gcc-qcm2290.c b/drivers/clk/qcom/gcc-qcm2290.c index b6fa7b8e80066cc64f1609f77ac8d2629202e738..7792b8f237047bedb4fbdd53d025ab3251c310ff 100644 --- a/drivers/clk/qcom/gcc-qcm2290.c +++ b/drivers/clk/qcom/gcc-qcm2290.c @@ -54,33 +54,9 @@ static const struct pll_vco spark_vco[] = { { 750000000, 1500000000, 1 }, }; -static const u8 clk_alpha_pll_regs_offset[][PLL_OFF_MAX_REGS] = { - [CLK_ALPHA_PLL_TYPE_DEFAULT] = { - [PLL_OFF_L_VAL] = 0x04, - [PLL_OFF_ALPHA_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL_U] = 0x0c, - [PLL_OFF_TEST_CTL] = 0x10, - [PLL_OFF_TEST_CTL_U] = 0x14, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_USER_CTL_U] = 0x1C, - [PLL_OFF_CONFIG_CTL] = 0x20, - [PLL_OFF_STATUS] = 0x24, - }, - [CLK_ALPHA_PLL_TYPE_BRAMMO] = { - [PLL_OFF_L_VAL] = 0x04, - [PLL_OFF_ALPHA_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL_U] = 0x0c, - [PLL_OFF_TEST_CTL] = 0x10, - [PLL_OFF_TEST_CTL_U] = 0x14, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x1C, - [PLL_OFF_STATUS] = 0x20, - }, -}; - static struct clk_alpha_pll gpll0 = { .offset = 0x0, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(0), @@ -106,7 +82,7 @@ static struct clk_alpha_pll_postdiv gpll0_out_aux2 = { .post_div_table = post_div_table_gpll0_out_aux2, .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_aux2), .width = 4, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_aux2", .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, @@ -117,7 +93,7 @@ static struct clk_alpha_pll_postdiv gpll0_out_aux2 = { static struct clk_alpha_pll gpll1 = { .offset = 0x1000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(1), @@ -147,7 +123,7 @@ static struct clk_alpha_pll gpll10 = { .offset = 0xa000, .vco_table = spark_vco, .num_vco = ARRAY_SIZE(spark_vco), - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(10), @@ -179,7 +155,7 @@ static struct clk_alpha_pll gpll11 = { .offset = 0xb000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x79000, @@ -197,7 +173,7 @@ static struct clk_alpha_pll gpll11 = { static struct clk_alpha_pll gpll3 = { .offset = 0x3000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(3), @@ -223,7 +199,7 @@ static struct clk_alpha_pll_postdiv gpll3_out_main = { .post_div_table = post_div_table_gpll3_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll3_out_main), .width = 4, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll3.clkr.hw }, @@ -234,7 +210,7 @@ static struct clk_alpha_pll_postdiv gpll3_out_main = { static struct clk_alpha_pll gpll4 = { .offset = 0x4000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(4), @@ -251,7 +227,7 @@ static struct clk_alpha_pll gpll4 = { static struct clk_alpha_pll gpll5 = { .offset = 0x5000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(5), @@ -268,7 +244,7 @@ static struct clk_alpha_pll gpll5 = { static struct clk_alpha_pll gpll6 = { .offset = 0x6000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(6), @@ -294,7 +270,7 @@ static struct clk_alpha_pll_postdiv gpll6_out_main = { .post_div_table = post_div_table_gpll6_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll6_out_main), .width = 4, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll6_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll6.clkr.hw }, @@ -305,7 +281,7 @@ static struct clk_alpha_pll_postdiv gpll6_out_main = { static struct clk_alpha_pll gpll7 = { .offset = 0x7000, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(7), @@ -340,7 +316,7 @@ static struct clk_alpha_pll gpll8 = { .offset = 0x8000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x79000, @@ -367,7 +343,7 @@ static struct clk_alpha_pll_postdiv gpll8_out_main = { .post_div_table = post_div_table_gpll8_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll8_out_main), .width = 4, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll8_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll8.clkr.hw }, @@ -393,7 +369,7 @@ static struct clk_alpha_pll gpll9 = { .offset = 0x9000, .vco_table = brammo_vco, .num_vco = ARRAY_SIZE(brammo_vco), - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_BRAMMO], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(9), @@ -419,7 +395,7 @@ static struct clk_alpha_pll_postdiv gpll9_out_main = { .post_div_table = post_div_table_gpll9_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll9_out_main), .width = 2, - .regs = clk_alpha_pll_regs_offset[CLK_ALPHA_PLL_TYPE_BRAMMO], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll9_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll9.clkr.hw }, diff --git a/drivers/clk/qcom/gcc-sc7180.c b/drivers/clk/qcom/gcc-sc7180.c index c2ea09945c4720ae7c2d60b876dcb78f3b8c99d7..2d3980251e78ecc791548ca572a8ac8a684b35a7 100644 --- a/drivers/clk/qcom/gcc-sc7180.c +++ b/drivers/clk/qcom/gcc-sc7180.c @@ -2224,7 +2224,7 @@ static struct gdsc usb30_prim_gdsc = { .pd = { .name = "usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, }; static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c index 7ff64d4d5920d89ae34887de8ddea2c4ae25a533..8afb7575e712fd13499fc979e7a3c6d08612e5fc 100644 --- a/drivers/clk/qcom/gcc-sc7280.c +++ b/drivers/clk/qcom/gcc-sc7280.c @@ -3108,7 +3108,7 @@ static struct gdsc gcc_pcie_1_gdsc = { .pd = { .name = "gcc_pcie_1_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE, }; @@ -3126,7 +3126,7 @@ static struct gdsc gcc_usb30_prim_gdsc = { .pd = { .name = "gcc_usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE, }; @@ -3135,7 +3135,7 @@ static struct gdsc gcc_usb30_sec_gdsc = { .pd = { .name = "gcc_usb30_sec_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE, }; diff --git a/drivers/clk/qcom/gcc-sc8280xp.c b/drivers/clk/qcom/gcc-sc8280xp.c index a2f3ffcc58491bb27554e3e7004ab8a2939c0cd8..a18ed88f3b822a5375e8cc92fb33e4f280a6392d 100644 --- a/drivers/clk/qcom/gcc-sc8280xp.c +++ b/drivers/clk/qcom/gcc-sc8280xp.c @@ -6768,6 +6768,10 @@ static struct gdsc pcie_1_tunnel_gdsc = { .flags = VOTABLE, }; +/* + * The Qualcomm PCIe driver does not yet implement suspend so to keep the + * PCIe power domains always-on for now. + */ static struct gdsc pcie_2a_gdsc = { .gdscr = 0x9d004, .collapse_ctrl = 0x52128, @@ -6776,7 +6780,7 @@ static struct gdsc pcie_2a_gdsc = { .name = "pcie_2a_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = VOTABLE | ALWAYS_ON, }; static struct gdsc pcie_2b_gdsc = { @@ -6787,7 +6791,7 @@ static struct gdsc pcie_2b_gdsc = { .name = "pcie_2b_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = VOTABLE | ALWAYS_ON, }; static struct gdsc pcie_3a_gdsc = { @@ -6798,7 +6802,7 @@ static struct gdsc pcie_3a_gdsc = { .name = "pcie_3a_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = VOTABLE | ALWAYS_ON, }; static struct gdsc pcie_3b_gdsc = { @@ -6809,7 +6813,7 @@ static struct gdsc pcie_3b_gdsc = { .name = "pcie_3b_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = VOTABLE | ALWAYS_ON, }; static struct gdsc pcie_4_gdsc = { @@ -6820,7 +6824,7 @@ static struct gdsc pcie_4_gdsc = { .name = "pcie_4_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = VOTABLE, + .flags = VOTABLE | ALWAYS_ON, }; static struct gdsc ufs_card_gdsc = { @@ -6844,7 +6848,7 @@ static struct gdsc usb30_mp_gdsc = { .pd = { .name = "usb30_mp_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, }; static struct gdsc usb30_prim_gdsc = { @@ -6852,7 +6856,7 @@ static struct gdsc usb30_prim_gdsc = { .pd = { .name = "usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, }; static struct gdsc usb30_sec_gdsc = { @@ -6860,7 +6864,7 @@ static struct gdsc usb30_sec_gdsc = { .pd = { .name = "usb30_sec_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, }; static struct clk_regmap *gcc_sc8280xp_clocks[] = { diff --git a/drivers/clk/qcom/gcc-sdm660.c b/drivers/clk/qcom/gcc-sdm660.c index 9b97425008ce1227385b40cf4361901929d2de28..db918c92a522c3e10a3bedb269edb174465b754c 100644 --- a/drivers/clk/qcom/gcc-sdm660.c +++ b/drivers/clk/qcom/gcc-sdm660.c @@ -757,7 +757,7 @@ static struct clk_rcg2 sdcc1_apps_clk_src = { .name = "sdcc1_apps_clk_src", .parent_data = gcc_parent_data_xo_gpll0_gpll4_gpll0_early_div, .num_parents = ARRAY_SIZE(gcc_parent_data_xo_gpll0_gpll4_gpll0_early_div), - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_floor_ops, }, }; diff --git a/drivers/clk/qcom/gcc-sdm845.c b/drivers/clk/qcom/gcc-sdm845.c index 58aa3ec9a7fc3bb12361b20c375292cca0773835..6af08e0ca847569963c703ce03b3a0e6bd7bd74b 100644 --- a/drivers/clk/qcom/gcc-sdm845.c +++ b/drivers/clk/qcom/gcc-sdm845.c @@ -31,6 +31,7 @@ enum { P_GPLL0_OUT_EVEN, P_GPLL0_OUT_MAIN, P_GPLL4_OUT_MAIN, + P_GPLL6_OUT_MAIN, P_SLEEP_CLK, }; @@ -68,6 +69,23 @@ static struct clk_alpha_pll gpll4 = { }, }; +static struct clk_alpha_pll gpll6 = { + .offset = 0x13000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gpll6", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", .name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_fabia_ops, + }, + }, +}; + static const struct clk_div_table post_div_table_fabia_even[] = { { 0x0, 1 }, { 0x1, 2 }, @@ -194,6 +212,19 @@ static const struct clk_parent_data gcc_parent_data_10[] = { { .hw = &gpll0_out_even.clkr.hw }, }; +static const struct parent_map gcc_parent_map_11[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL6_OUT_MAIN, 2 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "bi_tcxo", .name = "bi_tcxo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { F(19200000, P_BI_TCXO, 1, 0, 0), @@ -233,6 +264,26 @@ static struct clk_rcg2 gcc_cpuss_rbcpr_clk_src = { }, }; +static const struct freq_tbl ftbl_gcc_sdm670_cpuss_rbcpr_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdm670_cpuss_rbcpr_clk_src = { + .cmd_rcgr = 0x4815c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_sdm670_cpuss_rbcpr_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_rbcpr_clk_src", + .parent_data = gcc_parent_data_8_ao, + .num_parents = ARRAY_SIZE(gcc_parent_data_8_ao), + .ops = &clk_rcg2_ops, + }, +}; + static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { F(19200000, P_BI_TCXO, 1, 0, 0), F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), @@ -656,6 +707,54 @@ static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = { .clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init, }; +static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = { + F(144000, P_BI_TCXO, 16, 3, 25), + F(400000, P_BI_TCXO, 12, 1, 4), + F(20000000, P_GPLL0_OUT_EVEN, 5, 1, 3), + F(25000000, P_GPLL0_OUT_EVEN, 6, 1, 2), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(192000000, P_GPLL6_OUT_MAIN, 2, 0, 0), + F(384000000, P_GPLL6_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { + .cmd_rcgr = 0x26028, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_11, + .freq_tbl = ftbl_gcc_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = ARRAY_SIZE(gcc_parent_data_11), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = { + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x26010, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { F(400000, P_BI_TCXO, 12, 1, 4), F(9600000, P_BI_TCXO, 2, 0, 0), @@ -705,6 +804,31 @@ static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { }, }; +static const struct freq_tbl ftbl_gcc_sdm670_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(33333333, P_GPLL0_OUT_EVEN, 9, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdm670_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdm670_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_floor_ops, + }, +}; + static const struct freq_tbl ftbl_gcc_tsif_ref_clk_src[] = { F(105495, P_BI_TCXO, 2, 1, 91), { } @@ -1283,6 +1407,28 @@ static struct clk_branch gcc_cpuss_rbcpr_clk = { }, }; +/* + * The source clock frequencies are different for SDM670; define a child clock + * pointing to the source clock that uses SDM670 frequencies. + */ +static struct clk_branch gcc_sdm670_cpuss_rbcpr_clk = { + .halt_reg = 0x48008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x48008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_rbcpr_clk", + .parent_hws = (const struct clk_hw*[]){ + &gcc_sdm670_cpuss_rbcpr_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_ddrss_gpu_axi_clk = { .halt_reg = 0x44038, .halt_check = BRANCH_VOTED, @@ -2353,6 +2499,55 @@ static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { }, }; +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x26004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x26004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_hws = (const struct clk_hw*[]){ + &gcc_sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk", + .parent_hws = (const struct clk_hw*[]){ + &gcc_sdcc1_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sdcc2_ahb_clk = { .halt_reg = 0x14008, .halt_check = BRANCH_HALT, @@ -2415,6 +2610,28 @@ static struct clk_branch gcc_sdcc4_apps_clk = { }, }; +/* + * The source clock frequencies are different for SDM670; define a child clock + * pointing to the source clock that uses SDM670 frequencies. + */ +static struct clk_branch gcc_sdm670_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_hws = (const struct clk_hw*[]){ + &gcc_sdm670_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { .halt_reg = 0x414c, .halt_check = BRANCH_HALT_VOTED, @@ -3308,6 +3525,155 @@ static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf_gdsc = { .flags = VOTABLE, }; +static struct clk_regmap *gcc_sdm670_clocks[] = { + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_APC_VS_CLK] = &gcc_apc_vs_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_AHB_CLK] = &gcc_camera_ahb_clk.clkr, + [GCC_CAMERA_AXI_CLK] = &gcc_camera_axi_clk.clkr, + [GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr, + [GCC_CE1_AHB_CLK] = &gcc_ce1_ahb_clk.clkr, + [GCC_CE1_AXI_CLK] = &gcc_ce1_axi_clk.clkr, + [GCC_CE1_CLK] = &gcc_ce1_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_RBCPR_CLK] = &gcc_sdm670_cpuss_rbcpr_clk.clkr, + [GCC_CPUSS_RBCPR_CLK_SRC] = &gcc_sdm670_cpuss_rbcpr_clk_src.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DISP_AHB_CLK] = &gcc_disp_ahb_clk.clkr, + [GCC_DISP_AXI_CLK] = &gcc_disp_axi_clk.clkr, + [GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr, + [GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr, + [GCC_DISP_XO_CLK] = &gcc_disp_xo_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_IREF_CLK] = &gcc_gpu_iref_clk.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_GPU_VS_CLK] = &gcc_gpu_vs_clk.clkr, + [GCC_MSS_AXIS2_CLK] = &gcc_mss_axis2_clk.clkr, + [GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr, + [GCC_MSS_GPLL0_DIV_CLK_SRC] = &gcc_mss_gpll0_div_clk_src.clkr, + [GCC_MSS_MFAB_AXIS_CLK] = &gcc_mss_mfab_axis_clk.clkr, + [GCC_MSS_Q6_MEMNOC_AXI_CLK] = &gcc_mss_q6_memnoc_axi_clk.clkr, + [GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr, + [GCC_MSS_VS_CLK] = &gcc_mss_vs_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_QMIP_CAMERA_AHB_CLK] = &gcc_qmip_camera_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_AHB_CLK] = &gcc_qmip_video_ahb_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr, + [GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr, + [GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr, + [GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr, + [GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdm670_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdm670_sdcc4_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, + [GCC_TSIF_INACTIVITY_TIMERS_CLK] = + &gcc_tsif_inactivity_timers_clk.clkr, + [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, + [GCC_TSIF_REF_CLK_SRC] = &gcc_tsif_ref_clk_src.clkr, + [GCC_UFS_MEM_CLKREF_CLK] = &gcc_ufs_mem_clkref_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = + &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = + &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB3_PRIM_CLKREF_CLK] = &gcc_usb3_prim_clkref_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB_PHY_CFG_AHB2PHY_CLK] = &gcc_usb_phy_cfg_ahb2phy_clk.clkr, + [GCC_VDDA_VS_CLK] = &gcc_vdda_vs_clk.clkr, + [GCC_VDDCX_VS_CLK] = &gcc_vddcx_vs_clk.clkr, + [GCC_VDDMX_VS_CLK] = &gcc_vddmx_vs_clk.clkr, + [GCC_VIDEO_AHB_CLK] = &gcc_video_ahb_clk.clkr, + [GCC_VIDEO_AXI_CLK] = &gcc_video_axi_clk.clkr, + [GCC_VIDEO_XO_CLK] = &gcc_video_xo_clk.clkr, + [GCC_VS_CTRL_AHB_CLK] = &gcc_vs_ctrl_ahb_clk.clkr, + [GCC_VS_CTRL_CLK] = &gcc_vs_ctrl_clk.clkr, + [GCC_VS_CTRL_CLK_SRC] = &gcc_vs_ctrl_clk_src.clkr, + [GCC_VSENSOR_CLK_SRC] = &gcc_vsensor_clk_src.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, + [GPLL4] = &gpll4.clkr, + [GPLL6] = &gpll6.clkr, + [GCC_CPUSS_DVM_BUS_CLK] = &gcc_cpuss_dvm_bus_clk.clkr, + [GCC_CPUSS_GNOC_CLK] = &gcc_cpuss_gnoc_clk.clkr, + [GCC_QSPI_CORE_CLK_SRC] = &gcc_qspi_core_clk_src.clkr, + [GCC_QSPI_CORE_CLK] = &gcc_qspi_core_clk.clkr, + [GCC_QSPI_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_cnoc_periph_ahb_clk.clkr, +}; + static struct clk_regmap *gcc_sdm845_clocks[] = { [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, @@ -3533,6 +3899,22 @@ static const struct qcom_reset_map gcc_sdm845_resets[] = { [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, }; +static struct gdsc *gcc_sdm670_gdscs[] = { + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [HLOS1_VOTE_AGGRE_NOC_MMU_AUDIO_TBU_GDSC] = + &hlos1_vote_aggre_noc_mmu_audio_tbu_gdsc, + [HLOS1_VOTE_AGGRE_NOC_MMU_TBU1_GDSC] = + &hlos1_vote_aggre_noc_mmu_tbu1_gdsc, + [HLOS1_VOTE_AGGRE_NOC_MMU_TBU2_GDSC] = + &hlos1_vote_aggre_noc_mmu_tbu2_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = + &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = + &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf_gdsc, +}; + static struct gdsc *gcc_sdm845_gdscs[] = { [PCIE_0_GDSC] = &pcie_0_gdsc, [PCIE_1_GDSC] = &pcie_1_gdsc, @@ -3563,6 +3945,17 @@ static const struct regmap_config gcc_sdm845_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_desc gcc_sdm670_desc = { + .config = &gcc_sdm845_regmap_config, + .clks = gcc_sdm670_clocks, + .num_clks = ARRAY_SIZE(gcc_sdm670_clocks), + /* Snapdragon 670 can function without its own exclusive resets. */ + .resets = gcc_sdm845_resets, + .num_resets = ARRAY_SIZE(gcc_sdm845_resets), + .gdscs = gcc_sdm670_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sdm670_gdscs), +}; + static const struct qcom_cc_desc gcc_sdm845_desc = { .config = &gcc_sdm845_regmap_config, .clks = gcc_sdm845_clocks, @@ -3574,7 +3967,8 @@ static const struct qcom_cc_desc gcc_sdm845_desc = { }; static const struct of_device_id gcc_sdm845_match_table[] = { - { .compatible = "qcom,gcc-sdm845" }, + { .compatible = "qcom,gcc-sdm670", .data = &gcc_sdm670_desc }, + { .compatible = "qcom,gcc-sdm845", .data = &gcc_sdm845_desc }, { } }; MODULE_DEVICE_TABLE(of, gcc_sdm845_match_table); @@ -3600,6 +3994,7 @@ static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { static int gcc_sdm845_probe(struct platform_device *pdev) { + const struct qcom_cc_desc *gcc_desc; struct regmap *regmap; int ret; @@ -3616,7 +4011,8 @@ static int gcc_sdm845_probe(struct platform_device *pdev) if (ret) return ret; - return qcom_cc_really_probe(pdev, &gcc_sdm845_desc, regmap); + gcc_desc = of_device_get_match_data(&pdev->dev); + return qcom_cc_really_probe(pdev, gcc_desc, regmap); } static struct platform_driver gcc_sdm845_driver = { diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c index 68fe9f6f0d2f3bc8428e33357d3cdaa09355307d..565f9912039fe4fdb9f91b654766bfdf5b8c9471 100644 --- a/drivers/clk/qcom/gcc-sm6115.c +++ b/drivers/clk/qcom/gcc-sm6115.c @@ -57,7 +57,7 @@ static struct clk_alpha_pll gpll0 = { .offset = 0x0, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(0), @@ -83,7 +83,7 @@ static struct clk_alpha_pll_postdiv gpll0_out_aux2 = { .post_div_table = post_div_table_gpll0_out_aux2, .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_aux2), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_aux2", .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, @@ -92,18 +92,6 @@ static struct clk_alpha_pll_postdiv gpll0_out_aux2 = { }, }; -/* listed as BRAMMO, but it doesn't really match */ -static const u8 clk_gpll9_regs[PLL_OFF_MAX_REGS] = { - [PLL_OFF_L_VAL] = 0x04, - [PLL_OFF_ALPHA_VAL] = 0x08, - [PLL_OFF_ALPHA_VAL_U] = 0x0c, - [PLL_OFF_TEST_CTL] = 0x10, - [PLL_OFF_TEST_CTL_U] = 0x14, - [PLL_OFF_USER_CTL] = 0x18, - [PLL_OFF_CONFIG_CTL] = 0x1C, - [PLL_OFF_STATUS] = 0x20, -}; - static const struct clk_div_table post_div_table_gpll0_out_main[] = { { 0x0, 1 }, { } @@ -115,7 +103,7 @@ static struct clk_alpha_pll_postdiv gpll0_out_main = { .post_div_table = post_div_table_gpll0_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, @@ -137,7 +125,7 @@ static struct clk_alpha_pll gpll10 = { .offset = 0xa000, .vco_table = gpll10_vco, .num_vco = ARRAY_SIZE(gpll10_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(10), @@ -163,7 +151,7 @@ static struct clk_alpha_pll_postdiv gpll10_out_main = { .post_div_table = post_div_table_gpll10_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll10_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll10_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll10.clkr.hw }, @@ -189,7 +177,7 @@ static struct clk_alpha_pll gpll11 = { .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), .flags = SUPPORTS_DYNAMIC_UPDATE, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(11), @@ -215,7 +203,7 @@ static struct clk_alpha_pll_postdiv gpll11_out_main = { .post_div_table = post_div_table_gpll11_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll11_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll11_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll11.clkr.hw }, @@ -229,7 +217,7 @@ static struct clk_alpha_pll gpll3 = { .offset = 0x3000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(3), @@ -248,7 +236,7 @@ static struct clk_alpha_pll gpll4 = { .offset = 0x4000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(4), @@ -274,7 +262,7 @@ static struct clk_alpha_pll_postdiv gpll4_out_main = { .post_div_table = post_div_table_gpll4_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll4_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll4.clkr.hw }, @@ -287,7 +275,7 @@ static struct clk_alpha_pll gpll6 = { .offset = 0x6000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(6), @@ -313,7 +301,7 @@ static struct clk_alpha_pll_postdiv gpll6_out_main = { .post_div_table = post_div_table_gpll6_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll6_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll6_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll6.clkr.hw }, @@ -326,7 +314,7 @@ static struct clk_alpha_pll gpll7 = { .offset = 0x7000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(7), @@ -352,7 +340,7 @@ static struct clk_alpha_pll_postdiv gpll7_out_main = { .post_div_table = post_div_table_gpll7_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll7_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll7_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll7.clkr.hw }, @@ -380,7 +368,7 @@ static struct clk_alpha_pll gpll8 = { .offset = 0x8000, .vco_table = default_vco, .num_vco = ARRAY_SIZE(default_vco), - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .flags = SUPPORTS_DYNAMIC_UPDATE, .clkr = { .enable_reg = 0x79000, @@ -407,7 +395,7 @@ static struct clk_alpha_pll_postdiv gpll8_out_main = { .post_div_table = post_div_table_gpll8_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll8_out_main), .width = 4, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll8_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll8.clkr.hw }, @@ -431,7 +419,7 @@ static struct clk_alpha_pll gpll9 = { .offset = 0x9000, .vco_table = gpll9_vco, .num_vco = ARRAY_SIZE(gpll9_vco), - .regs = clk_gpll9_regs, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO], .clkr = { .enable_reg = 0x79000, .enable_mask = BIT(9), @@ -457,7 +445,7 @@ static struct clk_alpha_pll_postdiv gpll9_out_main = { .post_div_table = post_div_table_gpll9_out_main, .num_post_div = ARRAY_SIZE(post_div_table_gpll9_out_main), .width = 2, - .regs = clk_gpll9_regs, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_BRAMMO_EVO], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll9_out_main", .parent_hws = (const struct clk_hw *[]){ &gpll9.clkr.hw }, diff --git a/drivers/clk/qcom/gcc-sm6350.c b/drivers/clk/qcom/gcc-sm6350.c index 69412400efa43da8b811004951fb94b4288abeee..9b4e4bb0596355c8c30b28ac2c532e5a3900e8bb 100644 --- a/drivers/clk/qcom/gcc-sm6350.c +++ b/drivers/clk/qcom/gcc-sm6350.c @@ -2316,7 +2316,7 @@ static struct gdsc usb30_prim_gdsc = { .pd = { .name = "usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, }; static struct gdsc ufs_phy_gdsc = { diff --git a/drivers/clk/qcom/gcc-sm6375.c b/drivers/clk/qcom/gcc-sm6375.c new file mode 100644 index 0000000000000000000000000000000000000000..89a1cc90b1453bab990b6dc87a64950537d01241 --- /dev/null +++ b/drivers/clk/qcom/gcc-sm6375.c @@ -0,0 +1,3919 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Konrad Dybcio + */ + +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "clk-regmap-phy-mux.h" +#include "gdsc.h" +#include "reset.h" + +enum { + DT_BI_TCXO, + DT_BI_TCXO_AO, + DT_SLEEP_CLK +}; + +enum { + P_BI_TCXO, + P_GPLL0_OUT_EVEN, + P_GPLL0_OUT_MAIN, + P_GPLL0_OUT_ODD, + P_GPLL10_OUT_EVEN, + P_GPLL11_OUT_EVEN, + P_GPLL11_OUT_ODD, + P_GPLL3_OUT_EVEN, + P_GPLL3_OUT_MAIN, + P_GPLL4_OUT_EVEN, + P_GPLL5_OUT_EVEN, + P_GPLL6_OUT_EVEN, + P_GPLL6_OUT_MAIN, + P_GPLL7_OUT_EVEN, + P_GPLL8_OUT_EVEN, + P_GPLL8_OUT_MAIN, + P_GPLL9_OUT_EARLY, + P_GPLL9_OUT_MAIN, + P_SLEEP_CLK, +}; + +static struct pll_vco lucid_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct pll_vco zonda_vco[] = { + { 595200000, 3600000000UL, 0 }, +}; + +static struct clk_alpha_pll gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0_out_even", + .parent_hws = (const struct clk_hw*[]){ + &gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static const struct clk_div_table post_div_table_gpll0_out_odd[] = { + { 0x3, 3 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll0_out_odd = { + .offset = 0x0, + .post_div_shift = 12, + .post_div_table = post_div_table_gpll0_out_odd, + .num_post_div = ARRAY_SIZE(post_div_table_gpll0_out_odd), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0_out_odd", + .parent_hws = (const struct clk_hw*[]){ + &gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static struct clk_alpha_pll gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gpll1", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +/* 1152MHz Configuration */ +static const struct alpha_pll_config gpll10_config = { + .l = 0x3c, + .alpha = 0x0, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329a299c, + .user_ctl_val = 0x00000001, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll gpll10 = { + .offset = 0xa000, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .flags = SUPPORTS_FSM_LEGACY_MODE, + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gpll10", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +/* 532MHz Configuration */ +static const struct alpha_pll_config gpll11_config = { + .l = 0x1b, + .alpha = 0xb555, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329a299c, + .user_ctl_val = 0x00000001, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll gpll11 = { + .offset = 0xb000, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .flags = SUPPORTS_FSM_LEGACY_MODE, + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gpll11", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll3 = { + .offset = 0x3000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gpll3", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll3_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll3_out_even = { + .offset = 0x3000, + .post_div_shift = 8, + .post_div_table = post_div_table_gpll3_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gpll3_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll3_out_even", + .parent_hws = (const struct clk_hw*[]){ + &gpll3.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static struct clk_alpha_pll gpll4 = { + .offset = 0x4000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll5 = { + .offset = 0x5000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gpll5", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll6 = { + .offset = 0x6000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gpll6", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll6_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll6_out_even = { + .offset = 0x6000, + .post_div_shift = 8, + .post_div_table = post_div_table_gpll6_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gpll6_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll6_out_even", + .parent_hws = (const struct clk_hw*[]){ + &gpll6.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static struct clk_alpha_pll gpll7 = { + .offset = 0x7000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gpll7", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +/* 400MHz Configuration */ +static const struct alpha_pll_config gpll8_config = { + .l = 0x14, + .alpha = 0xd555, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x329a299c, + .user_ctl_val = 0x00000101, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll gpll8 = { + .offset = 0x8000, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .flags = SUPPORTS_FSM_LEGACY_MODE, + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gpll8", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll8_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll8_out_even = { + .offset = 0x8000, + .post_div_shift = 8, + .post_div_table = post_div_table_gpll8_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gpll8_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll8_out_even", + .parent_hws = (const struct clk_hw*[]){ + &gpll8.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +/* 1440MHz Configuration */ +static const struct alpha_pll_config gpll9_config = { + .l = 0x4b, + .alpha = 0x0, + .config_ctl_val = 0x08200800, + .config_ctl_hi_val = 0x05022011, + .config_ctl_hi1_val = 0x08000000, + .user_ctl_val = 0x00000301, +}; + +static struct clk_alpha_pll gpll9 = { + .offset = 0x9000, + .vco_table = zonda_vco, + .num_vco = ARRAY_SIZE(zonda_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_ZONDA], + .clkr = { + .enable_reg = 0x79000, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gpll9", + .parent_data = &(const struct clk_parent_data){ + .index = DT_BI_TCXO, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_zonda_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gpll9_out_main[] = { + { 0x3, 4 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll9_out_main = { + .offset = 0x9000, + .post_div_shift = 8, + .post_div_table = post_div_table_gpll9_out_main, + .num_post_div = ARRAY_SIZE(post_div_table_gpll9_out_main), + .width = 2, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_ZONDA], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll9_out_main", + .parent_hws = (const struct clk_hw*[]){ + &gpll9.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_alpha_pll_postdiv_zonda_ops, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL6_OUT_EVEN, 4 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll6_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL0_OUT_ODD, 4 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll0_out_odd.clkr.hw }, +}; + +static const struct clk_parent_data gcc_parent_data_2_ao[] = { + { .index = DT_BI_TCXO_AO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll0_out_odd.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL9_OUT_EARLY, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL9_OUT_MAIN, 4 }, + { P_GPLL3_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll9.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll9_out_main.clkr.hw }, + { .hw = &gpll3_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL0_OUT_ODD, 4 }, + { P_GPLL4_OUT_EVEN, 5 }, + { P_GPLL3_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll0_out_odd.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll3_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL8_OUT_MAIN, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL9_OUT_MAIN, 4 }, + { P_GPLL8_OUT_EVEN, 5 }, + { P_GPLL3_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll8.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll9_out_main.clkr.hw }, + { .hw = &gpll8_out_even.clkr.hw }, + { .hw = &gpll3_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL8_OUT_MAIN, 2 }, + { P_GPLL5_OUT_EVEN, 3 }, + { P_GPLL9_OUT_MAIN, 4 }, + { P_GPLL8_OUT_EVEN, 5 }, + { P_GPLL3_OUT_MAIN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll8.clkr.hw }, + { .hw = &gpll5.clkr.hw }, + { .hw = &gpll9_out_main.clkr.hw }, + { .hw = &gpll8_out_even.clkr.hw }, + { .hw = &gpll3.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL0_OUT_ODD, 4 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll0_out_odd.clkr.hw }, + { .index = DT_SLEEP_CLK }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL4_OUT_EVEN, 5 }, + { P_GPLL3_OUT_MAIN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll3.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL9_OUT_MAIN, 4 }, + { P_GPLL8_OUT_EVEN, 5 }, + { P_GPLL3_OUT_MAIN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll9_out_main.clkr.hw }, + { .hw = &gpll8_out_even.clkr.hw }, + { .hw = &gpll3.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL8_OUT_MAIN, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL9_OUT_MAIN, 4 }, + { P_GPLL8_OUT_EVEN, 5 }, + { P_GPLL3_OUT_MAIN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll8.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll9_out_main.clkr.hw }, + { .hw = &gpll8_out_even.clkr.hw }, + { .hw = &gpll3.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL8_OUT_MAIN, 2 }, + { P_GPLL10_OUT_EVEN, 3 }, + { P_GPLL6_OUT_MAIN, 4 }, + { P_GPLL3_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll8.clkr.hw }, + { .hw = &gpll10.clkr.hw }, + { .hw = &gpll6.clkr.hw }, + { .hw = &gpll3_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 2 }, + { P_GPLL7_OUT_EVEN, 3 }, + { P_GPLL4_OUT_EVEN, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, + { .hw = &gpll7.clkr.hw }, + { .hw = &gpll4.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_13[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_13[] = { + { .index = DT_BI_TCXO }, + { .index = DT_SLEEP_CLK }, +}; + +static const struct parent_map gcc_parent_map_14[] = { + { P_BI_TCXO, 0 }, + { P_GPLL11_OUT_ODD, 2 }, + { P_GPLL11_OUT_EVEN, 3 }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpll11.clkr.hw }, + { .hw = &gpll11.clkr.hw }, +}; + +static const struct freq_tbl ftbl_gcc_camss_axi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + F(300000000, P_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_axi_clk_src = { + .cmd_rcgr = 0x5802c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_camss_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_axi_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_cci_0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_cci_0_clk_src = { + .cmd_rcgr = 0x56000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_9, + .freq_tbl = ftbl_gcc_camss_cci_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cci_0_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_cci_1_clk_src = { + .cmd_rcgr = 0x5c000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_9, + .freq_tbl = ftbl_gcc_camss_cci_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cci_1_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_csi0phytimer_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GPLL0_OUT_ODD, 2, 0, 0), + F(300000000, P_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_csi0phytimer_clk_src = { + .cmd_rcgr = 0x59000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi0phytimer_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_csi1phytimer_clk_src = { + .cmd_rcgr = 0x5901c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi1phytimer_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_csi2phytimer_clk_src = { + .cmd_rcgr = 0x59038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi2phytimer_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_csi3phytimer_clk_src = { + .cmd_rcgr = 0x59054, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_camss_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi3phytimer_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_mclk0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(24000000, P_GPLL9_OUT_MAIN, 1, 1, 15), + F(65454545, P_GPLL9_OUT_EARLY, 11, 1, 2), + { } +}; + +static struct clk_rcg2 gcc_camss_mclk0_clk_src = { + .cmd_rcgr = 0x51000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_camss_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk0_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_mclk1_clk_src = { + .cmd_rcgr = 0x5101c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_camss_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk1_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_mclk2_clk_src = { + .cmd_rcgr = 0x51038, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_camss_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk2_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_mclk3_clk_src = { + .cmd_rcgr = 0x51054, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_camss_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk3_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_mclk4_clk_src = { + .cmd_rcgr = 0x51070, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_camss_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk4_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_ope_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(171428571, P_GPLL0_OUT_MAIN, 3.5, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_ope_ahb_clk_src = { + .cmd_rcgr = 0x55024, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_10, + .freq_tbl = ftbl_gcc_camss_ope_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_ope_ahb_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_ope_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_GPLL8_OUT_EVEN, 1, 0, 0), + F(266600000, P_GPLL8_OUT_EVEN, 1, 0, 0), + F(480000000, P_GPLL8_OUT_EVEN, 1, 0, 0), + F(580000000, P_GPLL8_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_ope_clk_src = { + .cmd_rcgr = 0x55004, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_10, + .freq_tbl = ftbl_gcc_camss_ope_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_ope_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_tfe_0_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(120000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(144000000, P_GPLL9_OUT_MAIN, 2.5, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(171428571, P_GPLL0_OUT_MAIN, 3.5, 0, 0), + F(180000000, P_GPLL9_OUT_MAIN, 2, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(329142857, P_GPLL10_OUT_EVEN, 3.5, 0, 0), + F(384000000, P_GPLL10_OUT_EVEN, 3, 0, 0), + F(460800000, P_GPLL10_OUT_EVEN, 2.5, 0, 0), + F(576000000, P_GPLL10_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_tfe_0_clk_src = { + .cmd_rcgr = 0x52004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_camss_tfe_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_0_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_tfe_0_csid_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(120000000, P_GPLL0_OUT_MAIN, 5, 0, 0), + F(266571429, P_GPLL5_OUT_EVEN, 3.5, 0, 0), + F(426400000, P_GPLL3_OUT_MAIN, 2.5, 0, 0), + F(466500000, P_GPLL5_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_tfe_0_csid_clk_src = { + .cmd_rcgr = 0x52094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_camss_tfe_0_csid_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_0_csid_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_tfe_1_clk_src = { + .cmd_rcgr = 0x52024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_camss_tfe_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_1_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_tfe_1_csid_clk_src = { + .cmd_rcgr = 0x520b4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_camss_tfe_0_csid_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_1_csid_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_tfe_2_clk_src = { + .cmd_rcgr = 0x52044, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_camss_tfe_0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_2_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_camss_tfe_2_csid_clk_src = { + .cmd_rcgr = 0x520d4, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_camss_tfe_0_csid_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_2_csid_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_tfe_cphy_rx_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(256000000, P_GPLL6_OUT_MAIN, 3, 0, 0), + F(384000000, P_GPLL6_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_tfe_cphy_rx_clk_src = { + .cmd_rcgr = 0x52064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_11, + .freq_tbl = ftbl_gcc_camss_tfe_cphy_rx_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_cphy_rx_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = ARRAY_SIZE(gcc_parent_data_11), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_camss_top_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(40000000, P_GPLL0_OUT_EVEN, 7.5, 0, 0), + F(80000000, P_GPLL0_OUT_MAIN, 7.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_camss_top_ahb_clk_src = { + .cmd_rcgr = 0x58010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_camss_top_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_camss_top_ahb_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_ODD, 4, 0, 0), + F(100000000, P_GPLL0_OUT_ODD, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x2b13c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parent_data_2_ao, + .num_parents = ARRAY_SIZE(gcc_parent_data_2_ao), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_ODD, 4, 0, 0), + F(100000000, P_GPLL0_OUT_ODD, 2, 0, 0), + F(200000000, P_GPLL0_OUT_ODD, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x4d004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x4e004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x4f004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_EVEN, 5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x20010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0), + F(102400000, P_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GPLL0_OUT_EVEN, 2.5, 0, 0), + F(128000000, P_GPLL6_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x1f148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x1f278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x1f3a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x1f4d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x1f608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x1f738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x5301c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x5314c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x5327c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x533ac, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x534dc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x5360c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = { + F(144000, P_BI_TCXO, 16, 3, 25), + F(400000, P_BI_TCXO, 12, 1, 4), + F(20000000, P_GPLL0_OUT_EVEN, 5, 1, 3), + F(25000000, P_GPLL0_OUT_EVEN, 6, 1, 2), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0), + F(192000000, P_GPLL6_OUT_EVEN, 2, 0, 0), + F(384000000, P_GPLL6_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { + .cmd_rcgr = 0x38028, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = { + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x38010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GPLL7_OUT_EVEN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1e00c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_12, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = ARRAY_SIZE(gcc_parent_data_12), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_ODD, 4, 0, 0), + F(100000000, P_GPLL0_OUT_ODD, 2, 0, 0), + F(200000000, P_GPLL0_OUT_ODD, 1, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x45020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x45048, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x4507c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_unipro_core_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x45060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_unipro_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_ODD, 1, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0x1a01c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_mock_utmi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0x1a034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0x1a060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_13, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_13, + .num_parents = ARRAY_SIZE(gcc_parent_data_13), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_video_venus_clk_src[] = { + F(133000000, P_GPLL11_OUT_EVEN, 4, 0, 0), + F(240000000, P_GPLL11_OUT_EVEN, 2.5, 0, 0), + F(300000000, P_GPLL11_OUT_EVEN, 2, 0, 0), + F(384000000, P_GPLL11_OUT_EVEN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_video_venus_clk_src = { + .cmd_rcgr = 0x58060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_14, + .freq_tbl = ftbl_gcc_video_venus_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_video_venus_clk_src", + .parent_data = gcc_parent_data_14, + .num_parents = ARRAY_SIZE(gcc_parent_data_14), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_regmap_div gcc_cpuss_ahb_postdiv_clk_src = { + .reg = 0x2b154, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_cpuss_ahb_postdiv_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gcc_cpuss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0x1a04c, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_ahb2phy_csi_clk = { + .halt_reg = 0x1d004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1d004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1d004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ahb2phy_csi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ahb2phy_usb_clk = { + .halt_reg = 0x1d008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1d008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1d008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ahb2phy_usb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_bimc_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x71154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_bimc_gpu_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x23004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x23004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cam_throttle_nrt_clk = { + .halt_reg = 0x17070, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17070, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cam_throttle_nrt_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cam_throttle_rt_clk = { + .halt_reg = 0x1706c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1706c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cam_throttle_rt_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x17008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_ahb_clk", + .flags = CLK_IS_CRITICAL, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_axi_clk = { + .halt_reg = 0x58044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cci_0_clk = { + .halt_reg = 0x56018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x56018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cci_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_cci_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cci_1_clk = { + .halt_reg = 0x5c018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5c018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cci_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_cci_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cphy_0_clk = { + .halt_reg = 0x52088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x52088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cphy_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cphy_1_clk = { + .halt_reg = 0x5208c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5208c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cphy_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cphy_2_clk = { + .halt_reg = 0x52090, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x52090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cphy_2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_cphy_3_clk = { + .halt_reg = 0x520f8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x520f8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_cphy_3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi0phytimer_clk = { + .halt_reg = 0x59018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x59018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi0phytimer_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_csi0phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi1phytimer_clk = { + .halt_reg = 0x59034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x59034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi1phytimer_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_csi1phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi2phytimer_clk = { + .halt_reg = 0x59050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x59050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi2phytimer_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_csi2phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_csi3phytimer_clk = { + .halt_reg = 0x5906c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5906c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_csi3phytimer_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_csi3phytimer_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk0_clk = { + .halt_reg = 0x51018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x51018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_mclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk1_clk = { + .halt_reg = 0x51034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x51034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_mclk1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk2_clk = { + .halt_reg = 0x51050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x51050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_mclk2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk3_clk = { + .halt_reg = 0x5106c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5106c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_mclk3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_mclk4_clk = { + .halt_reg = 0x51088, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x51088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_mclk4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_mclk4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_nrt_axi_clk = { + .halt_reg = 0x58054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_nrt_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_ope_ahb_clk = { + .halt_reg = 0x5503c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5503c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_ope_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_ope_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_ope_clk = { + .halt_reg = 0x5501c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_ope_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_ope_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_rt_axi_clk = { + .halt_reg = 0x5805c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5805c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_rt_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_0_clk = { + .halt_reg = 0x5201c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_0_cphy_rx_clk = { + .halt_reg = 0x5207c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5207c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_0_cphy_rx_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_0_csid_clk = { + .halt_reg = 0x520ac, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x520ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_0_csid_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_0_csid_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_1_clk = { + .halt_reg = 0x5203c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5203c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_1_cphy_rx_clk = { + .halt_reg = 0x52080, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x52080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_1_cphy_rx_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_1_csid_clk = { + .halt_reg = 0x520cc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x520cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_1_csid_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_1_csid_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_2_clk = { + .halt_reg = 0x5205c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5205c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_2_cphy_rx_clk = { + .halt_reg = 0x52084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x52084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_2_cphy_rx_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_cphy_rx_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_tfe_2_csid_clk = { + .halt_reg = 0x520ec, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x520ec, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_tfe_2_csid_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_tfe_2_csid_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camss_top_ahb_clk = { + .halt_reg = 0x58028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x58028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camss_top_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_camss_top_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0x1a084, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1a084, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1a084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_ahb_clk = { + .halt_reg = 0x1700c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_ahb_clk", + .flags = CLK_IS_CRITICAL, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div gcc_disp_gpll0_clk_src = { + .reg = 0x17058, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_disp_gpll0_clk_src", + .parent_names = + (const char *[]){ "gpll0" }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + }, +}; + +static struct clk_branch gcc_disp_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_disp_gpll0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x17020, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0x17020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x17020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sleep_clk = { + .halt_reg = 0x17074, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17074, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x17074, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_throttle_core_clk = { + .halt_reg = 0x17064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x4d000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4d000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x4e000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4e000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x4f000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4f000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_cfg_ahb_clk = { + .halt_reg = 0x36004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x36004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x36004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_cfg_ahb_clk", + .flags = CLK_IS_CRITICAL, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x3600c, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0x3600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x36018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_throttle_core_clk = { + .halt_reg = 0x36048, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x36048, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(31), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x2000c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2000c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x20004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x20004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x20004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x20008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x20008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x21004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x21004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x17014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x17060, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17060, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x17018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_gpu_cfg_ahb_clk = { + .halt_reg = 0x36040, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x36040, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_gpu_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x17010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x1f014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x1f00c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x1f144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x1f274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x1f3a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x1f4d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x1f604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x1f734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x53014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x5300c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x53018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x53148, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x53278, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x533a8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x534d8, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x53608, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x1f004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1f004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x1f008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1f008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x53004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x53004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x53008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x53008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7900c, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x38008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x38008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x38004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x3800c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x3800c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x3800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1e008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1e004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { + .halt_reg = 0x2b06c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2b06c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_cpuss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_ufs_phy_axi_clk = { + .halt_reg = 0x45098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x45098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_usb3_prim_axi_clk = { + .halt_reg = 0x1a080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1a080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1a080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x45014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x45014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x45014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x45010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x45010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x45010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x45044, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x45044, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x45044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x45078, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x45078, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x45078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x4501c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x4501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x45018, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x45018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x45040, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x45040, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x45040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0x1a010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0x1a018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0x1a014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_mem_clkref_clk = { + .halt_reg = 0x8c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_mem_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_rx5_pcie_clkref_en_clk = { + .halt_reg = 0x8c00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_rx5_pcie_clkref_en_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_clkref_clk = { + .halt_reg = 0x8c010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0x1a054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1a054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0x1a058, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x1a058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1a058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_vcodec0_axi_clk = { + .halt_reg = 0x6e008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6e008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_vcodec0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus_ahb_clk = { + .halt_reg = 0x6e010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6e010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_venus_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_venus_ctl_axi_clk = { + .halt_reg = 0x6e004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6e004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_venus_ctl_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0x17004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x17004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_ahb_clk", + .flags = CLK_IS_CRITICAL, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x1701c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1701c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_throttle_core_clk = { + .halt_reg = 0x17068, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17068, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x79004, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_vcodec0_sys_clk = { + .halt_reg = 0x580a4, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x580a4, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x580a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_vcodec0_sys_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_video_venus_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_venus_ctl_clk = { + .halt_reg = 0x5808c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x5808c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_venus_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_video_venus_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_xo_clk = { + .halt_reg = 0x17024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x17024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_xo_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0x1a004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x45004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_top_gdsc = { + .gdscr = 0x58004, + .pd = { + .name = "camss_top_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x5807c, + .pd = { + .name = "venus_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc vcodec0_gdsc = { + .gdscr = 0x58098, + .pd = { + .name = "vcodec0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = HW_CTRL, +}; + +static struct gdsc hlos1_vote_mm_snoc_mmu_tbu_rt_gdsc = { + .gdscr = 0x7d074, + .pd = { + .name = "hlos1_vote_mm_snoc_mmu_tbu_rt_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mm_snoc_mmu_tbu_nrt_gdsc = { + .gdscr = 0x7d078, + .pd = { + .name = "hlos1_vote_mm_snoc_mmu_tbu_nrt_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu1_gdsc = { + .gdscr = 0x7d060, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu0_gdsc = { + .gdscr = 0x7d07c, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct clk_regmap *gcc_sm6375_clocks[] = { + [GCC_AHB2PHY_CSI_CLK] = &gcc_ahb2phy_csi_clk.clkr, + [GCC_AHB2PHY_USB_CLK] = &gcc_ahb2phy_usb_clk.clkr, + [GCC_BIMC_GPU_AXI_CLK] = &gcc_bimc_gpu_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAM_THROTTLE_NRT_CLK] = &gcc_cam_throttle_nrt_clk.clkr, + [GCC_CAM_THROTTLE_RT_CLK] = &gcc_cam_throttle_rt_clk.clkr, + [GCC_CAMERA_AHB_CLK] = &gcc_camera_ahb_clk.clkr, + [GCC_CAMSS_AXI_CLK] = &gcc_camss_axi_clk.clkr, + [GCC_CAMSS_AXI_CLK_SRC] = &gcc_camss_axi_clk_src.clkr, + [GCC_CAMSS_CCI_0_CLK] = &gcc_camss_cci_0_clk.clkr, + [GCC_CAMSS_CCI_0_CLK_SRC] = &gcc_camss_cci_0_clk_src.clkr, + [GCC_CAMSS_CCI_1_CLK] = &gcc_camss_cci_1_clk.clkr, + [GCC_CAMSS_CCI_1_CLK_SRC] = &gcc_camss_cci_1_clk_src.clkr, + [GCC_CAMSS_CPHY_0_CLK] = &gcc_camss_cphy_0_clk.clkr, + [GCC_CAMSS_CPHY_1_CLK] = &gcc_camss_cphy_1_clk.clkr, + [GCC_CAMSS_CPHY_2_CLK] = &gcc_camss_cphy_2_clk.clkr, + [GCC_CAMSS_CPHY_3_CLK] = &gcc_camss_cphy_3_clk.clkr, + [GCC_CAMSS_CSI0PHYTIMER_CLK] = &gcc_camss_csi0phytimer_clk.clkr, + [GCC_CAMSS_CSI0PHYTIMER_CLK_SRC] = &gcc_camss_csi0phytimer_clk_src.clkr, + [GCC_CAMSS_CSI1PHYTIMER_CLK] = &gcc_camss_csi1phytimer_clk.clkr, + [GCC_CAMSS_CSI1PHYTIMER_CLK_SRC] = &gcc_camss_csi1phytimer_clk_src.clkr, + [GCC_CAMSS_CSI2PHYTIMER_CLK] = &gcc_camss_csi2phytimer_clk.clkr, + [GCC_CAMSS_CSI2PHYTIMER_CLK_SRC] = &gcc_camss_csi2phytimer_clk_src.clkr, + [GCC_CAMSS_CSI3PHYTIMER_CLK] = &gcc_camss_csi3phytimer_clk.clkr, + [GCC_CAMSS_CSI3PHYTIMER_CLK_SRC] = &gcc_camss_csi3phytimer_clk_src.clkr, + [GCC_CAMSS_MCLK0_CLK] = &gcc_camss_mclk0_clk.clkr, + [GCC_CAMSS_MCLK0_CLK_SRC] = &gcc_camss_mclk0_clk_src.clkr, + [GCC_CAMSS_MCLK1_CLK] = &gcc_camss_mclk1_clk.clkr, + [GCC_CAMSS_MCLK1_CLK_SRC] = &gcc_camss_mclk1_clk_src.clkr, + [GCC_CAMSS_MCLK2_CLK] = &gcc_camss_mclk2_clk.clkr, + [GCC_CAMSS_MCLK2_CLK_SRC] = &gcc_camss_mclk2_clk_src.clkr, + [GCC_CAMSS_MCLK3_CLK] = &gcc_camss_mclk3_clk.clkr, + [GCC_CAMSS_MCLK3_CLK_SRC] = &gcc_camss_mclk3_clk_src.clkr, + [GCC_CAMSS_MCLK4_CLK] = &gcc_camss_mclk4_clk.clkr, + [GCC_CAMSS_MCLK4_CLK_SRC] = &gcc_camss_mclk4_clk_src.clkr, + [GCC_CAMSS_NRT_AXI_CLK] = &gcc_camss_nrt_axi_clk.clkr, + [GCC_CAMSS_OPE_AHB_CLK] = &gcc_camss_ope_ahb_clk.clkr, + [GCC_CAMSS_OPE_AHB_CLK_SRC] = &gcc_camss_ope_ahb_clk_src.clkr, + [GCC_CAMSS_OPE_CLK] = &gcc_camss_ope_clk.clkr, + [GCC_CAMSS_OPE_CLK_SRC] = &gcc_camss_ope_clk_src.clkr, + [GCC_CAMSS_RT_AXI_CLK] = &gcc_camss_rt_axi_clk.clkr, + [GCC_CAMSS_TFE_0_CLK] = &gcc_camss_tfe_0_clk.clkr, + [GCC_CAMSS_TFE_0_CLK_SRC] = &gcc_camss_tfe_0_clk_src.clkr, + [GCC_CAMSS_TFE_0_CPHY_RX_CLK] = &gcc_camss_tfe_0_cphy_rx_clk.clkr, + [GCC_CAMSS_TFE_0_CSID_CLK] = &gcc_camss_tfe_0_csid_clk.clkr, + [GCC_CAMSS_TFE_0_CSID_CLK_SRC] = &gcc_camss_tfe_0_csid_clk_src.clkr, + [GCC_CAMSS_TFE_1_CLK] = &gcc_camss_tfe_1_clk.clkr, + [GCC_CAMSS_TFE_1_CLK_SRC] = &gcc_camss_tfe_1_clk_src.clkr, + [GCC_CAMSS_TFE_1_CPHY_RX_CLK] = &gcc_camss_tfe_1_cphy_rx_clk.clkr, + [GCC_CAMSS_TFE_1_CSID_CLK] = &gcc_camss_tfe_1_csid_clk.clkr, + [GCC_CAMSS_TFE_1_CSID_CLK_SRC] = &gcc_camss_tfe_1_csid_clk_src.clkr, + [GCC_CAMSS_TFE_2_CLK] = &gcc_camss_tfe_2_clk.clkr, + [GCC_CAMSS_TFE_2_CLK_SRC] = &gcc_camss_tfe_2_clk_src.clkr, + [GCC_CAMSS_TFE_2_CPHY_RX_CLK] = &gcc_camss_tfe_2_cphy_rx_clk.clkr, + [GCC_CAMSS_TFE_2_CSID_CLK] = &gcc_camss_tfe_2_csid_clk.clkr, + [GCC_CAMSS_TFE_2_CSID_CLK_SRC] = &gcc_camss_tfe_2_csid_clk_src.clkr, + [GCC_CAMSS_TFE_CPHY_RX_CLK_SRC] = &gcc_camss_tfe_cphy_rx_clk_src.clkr, + [GCC_CAMSS_TOP_AHB_CLK] = &gcc_camss_top_ahb_clk.clkr, + [GCC_CAMSS_TOP_AHB_CLK_SRC] = &gcc_camss_top_ahb_clk_src.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_AHB_POSTDIV_CLK_SRC] = &gcc_cpuss_ahb_postdiv_clk_src.clkr, + [GCC_DISP_AHB_CLK] = &gcc_disp_ahb_clk.clkr, + [GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr, + [GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SLEEP_CLK] = &gcc_disp_sleep_clk.clkr, + [GCC_DISP_THROTTLE_CORE_CLK] = &gcc_disp_throttle_core_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPU_CFG_AHB_CLK] = &gcc_gpu_cfg_ahb_clk.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_GPU_THROTTLE_CORE_CLK] = &gcc_gpu_throttle_core_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_GPU_CFG_AHB_CLK] = &gcc_qmip_gpu_cfg_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_SYS_NOC_UFS_PHY_AXI_CLK] = &gcc_sys_noc_ufs_phy_axi_clk.clkr, + [GCC_SYS_NOC_USB3_PRIM_AXI_CLK] = &gcc_sys_noc_usb3_prim_axi_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB3_PRIM_CLKREF_CLK] = &gcc_usb3_prim_clkref_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_VCODEC0_AXI_CLK] = &gcc_vcodec0_axi_clk.clkr, + [GCC_VENUS_AHB_CLK] = &gcc_venus_ahb_clk.clkr, + [GCC_VENUS_CTL_AXI_CLK] = &gcc_venus_ctl_axi_clk.clkr, + [GCC_VIDEO_AHB_CLK] = &gcc_video_ahb_clk.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_THROTTLE_CORE_CLK] = &gcc_video_throttle_core_clk.clkr, + [GCC_VIDEO_VCODEC0_SYS_CLK] = &gcc_video_vcodec0_sys_clk.clkr, + [GCC_VIDEO_VENUS_CLK_SRC] = &gcc_video_venus_clk_src.clkr, + [GCC_VIDEO_VENUS_CTL_CLK] = &gcc_video_venus_ctl_clk.clkr, + [GCC_VIDEO_XO_CLK] = &gcc_video_xo_clk.clkr, + [GCC_UFS_MEM_CLKREF_CLK] = &gcc_ufs_mem_clkref_clk.clkr, + [GCC_RX5_PCIE_CLKREF_EN_CLK] = &gcc_rx5_pcie_clkref_en_clk.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, + [GPLL0_OUT_ODD] = &gpll0_out_odd.clkr, + [GPLL1] = &gpll1.clkr, + [GPLL10] = &gpll10.clkr, + [GPLL11] = &gpll11.clkr, + [GPLL3] = &gpll3.clkr, + [GPLL3_OUT_EVEN] = &gpll3_out_even.clkr, + [GPLL4] = &gpll4.clkr, + [GPLL5] = &gpll5.clkr, + [GPLL6] = &gpll6.clkr, + [GPLL6_OUT_EVEN] = &gpll6_out_even.clkr, + [GPLL7] = &gpll7.clkr, + [GPLL8] = &gpll8.clkr, + [GPLL8_OUT_EVEN] = &gpll8_out_even.clkr, + [GPLL9] = &gpll9.clkr, + [GPLL9_OUT_MAIN] = &gpll9_out_main.clkr, +}; + +static const struct qcom_reset_map gcc_sm6375_resets[] = { + [GCC_MMSS_BCR] = { 0x17000 }, + [GCC_USB30_PRIM_BCR] = { 0x1a000 }, + [GCC_USB3_PHY_PRIM_SP0_BCR] = { 0x1b000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x1b020 }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x1c000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x1c004 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x1d000 }, + [GCC_SDCC2_BCR] = { 0x1e000 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x1f000 }, + [GCC_PDM_BCR] = { 0x20000 }, + [GCC_GPU_BCR] = { 0x36000 }, + [GCC_SDCC1_BCR] = { 0x38000 }, + [GCC_UFS_PHY_BCR] = { 0x45000 }, + [GCC_CAMSS_TFE_BCR] = { 0x52000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x53000 }, + [GCC_CAMSS_OPE_BCR] = { 0x55000 }, + [GCC_CAMSS_TOP_BCR] = { 0x58000 }, + [GCC_VENUS_BCR] = { 0x58078 }, + [GCC_VCODEC0_BCR] = { 0x58094 }, + [GCC_VIDEO_INTERFACE_BCR] = { 0x6e000 }, +}; + + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), +}; + +static struct gdsc *gcc_sm6375_gdscs[] = { + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [CAMSS_TOP_GDSC] = &camss_top_gdsc, + [VENUS_GDSC] = &venus_gdsc, + [VCODEC0_GDSC] = &vcodec0_gdsc, + [HLOS1_VOTE_MM_SNOC_MMU_TBU_NRT_GDSC] = &hlos1_vote_mm_snoc_mmu_tbu_nrt_gdsc, + [HLOS1_VOTE_MM_SNOC_MMU_TBU_RT_GDSC] = &hlos1_vote_mm_snoc_mmu_tbu_rt_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU0_GDSC] = &hlos1_vote_turing_mmu_tbu0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU1_GDSC] = &hlos1_vote_turing_mmu_tbu1_gdsc, +}; + +static const struct regmap_config gcc_sm6375_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xc7000, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sm6375_desc = { + .config = &gcc_sm6375_regmap_config, + .clks = gcc_sm6375_clocks, + .num_clks = ARRAY_SIZE(gcc_sm6375_clocks), + .resets = gcc_sm6375_resets, + .num_resets = ARRAY_SIZE(gcc_sm6375_resets), + .gdscs = gcc_sm6375_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sm6375_gdscs), +}; + +static const struct of_device_id gcc_sm6375_match_table[] = { + { .compatible = "qcom,sm6375-gcc" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sm6375_match_table); + +static int gcc_sm6375_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sm6375_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + /* + * Keep the following clocks always on: + * GCC_CAMERA_XO_CLK, GCC_CPUSS_GNOC_CLK, GCC_DISP_XO_CLK + */ + regmap_update_bits(regmap, 0x17028, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x2b004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x1702c, BIT(0), BIT(0)); + + clk_lucid_pll_configure(&gpll10, regmap, &gpll10_config); + clk_lucid_pll_configure(&gpll11, regmap, &gpll11_config); + clk_lucid_pll_configure(&gpll8, regmap, &gpll8_config); + clk_zonda_pll_configure(&gpll9, regmap, &gpll9_config); + + return qcom_cc_really_probe(pdev, &gcc_sm6375_desc, regmap); +} + +static struct platform_driver gcc_sm6375_driver = { + .probe = gcc_sm6375_probe, + .driver = { + .name = "gcc-sm6375", + .of_match_table = gcc_sm6375_match_table, + }, +}; + +static int __init gcc_sm6375_init(void) +{ + return platform_driver_register(&gcc_sm6375_driver); +} +subsys_initcall(gcc_sm6375_init); + +static void __exit gcc_sm6375_exit(void) +{ + platform_driver_unregister(&gcc_sm6375_driver); +} +module_exit(gcc_sm6375_exit); + +MODULE_DESCRIPTION("QTI GCC SM6375 Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index d3244006c661a90eb4d00763fefc15cfa6f75142..7cf5e130e92fb59f8d99759ebe656b7bceacf5a1 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -368,6 +368,16 @@ static int _gdsc_disable(struct gdsc *sc) if (sc->pwrsts & PWRSTS_OFF) gdsc_clear_mem_on(sc); + /* + * If the GDSC supports only a Retention state, apart from ON, + * leave it in ON state. + * There is no SW control to transition the GDSC into + * Retention state. This happens in HW when the parent + * domain goes down to a Low power state + */ + if (sc->pwrsts == PWRSTS_RET_ON) + return 0; + ret = gdsc_toggle_logic(sc, GDSC_OFF); if (ret) return ret; @@ -439,11 +449,8 @@ static int gdsc_init(struct gdsc *sc) /* ...and the power-domain */ ret = gdsc_pm_runtime_get(sc); - if (ret) { - if (sc->rsupply) - regulator_disable(sc->rsupply); - return ret; - } + if (ret) + goto err_disable_supply; /* * Votable GDSCs can be ON due to Vote from other masters. @@ -452,14 +459,14 @@ static int gdsc_init(struct gdsc *sc) if (sc->flags & VOTABLE) { ret = gdsc_update_collapse_bit(sc, false); if (ret) - return ret; + goto err_put_rpm; } /* Turn on HW trigger mode if supported */ if (sc->flags & HW_CTRL) { ret = gdsc_hwctrl(sc, true); if (ret < 0) - return ret; + goto err_put_rpm; } /* @@ -486,9 +493,21 @@ static int gdsc_init(struct gdsc *sc) sc->pd.power_off = gdsc_disable; if (!sc->pd.power_on) sc->pd.power_on = gdsc_enable; - pm_genpd_init(&sc->pd, NULL, !on); + + ret = pm_genpd_init(&sc->pd, NULL, !on); + if (ret) + goto err_put_rpm; return 0; + +err_put_rpm: + if (on) + gdsc_pm_runtime_put(sc); +err_disable_supply: + if (on && sc->rsupply) + regulator_disable(sc->rsupply); + + return ret; } int gdsc_register(struct gdsc_desc *desc, diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index 5de48c9439b29af03d208bef151a61f0d024bee6..981a12c8502d9c4b353657ee96467f8883cf875a 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -49,6 +49,11 @@ struct gdsc { const u8 pwrsts; /* Powerdomain allowable state bitfields */ #define PWRSTS_OFF BIT(0) +/* + * There is no SW control to transition a GDSC into + * PWRSTS_RET. This happens in HW when the parent + * domain goes down to a low power state + */ #define PWRSTS_RET BIT(1) #define PWRSTS_ON BIT(2) #define PWRSTS_OFF_ON (PWRSTS_OFF | PWRSTS_ON) diff --git a/drivers/clk/qcom/gpucc-sc8280xp.c b/drivers/clk/qcom/gpucc-sc8280xp.c new file mode 100644 index 0000000000000000000000000000000000000000..ea1e9505c33598f1ee2d1146fbb6c8c167912815 --- /dev/null +++ b/drivers/clk/qcom/gpucc-sc8280xp.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap-divider.h" +#include "common.h" +#include "reset.h" +#include "gdsc.h" + +/* Need to match the order of clocks in DT binding */ +enum { + DT_BI_TCXO, + DT_GCC_GPU_GPLL0_CLK_SRC, + DT_GCC_GPU_GPLL0_DIV_CLK_SRC, +}; + +enum { + P_BI_TCXO, + P_GCC_GPU_GPLL0_CLK_SRC, + P_GCC_GPU_GPLL0_DIV_CLK_SRC, + P_GPU_CC_PLL0_OUT_MAIN, + P_GPU_CC_PLL1_OUT_MAIN, +}; + +static const struct clk_parent_data parent_data_tcxo = { .index = DT_BI_TCXO }; + +static const struct pll_vco lucid_5lpe_vco[] = { + { 249600000, 1800000000, 0 }, +}; + +static struct alpha_pll_config gpu_cc_pll0_config = { + .l = 0x1c, + .alpha = 0xa555, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x2a9a699c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000000, + .test_ctl_hi1_val = 0x01800000, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll gpu_cc_pll0 = { + .offset = 0x0, + .vco_table = lucid_5lpe_vco, + .num_vco = ARRAY_SIZE(lucid_5lpe_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_pll0", + .parent_data = &parent_data_tcxo, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_5lpe_ops, + }, + }, +}; + +static struct alpha_pll_config gpu_cc_pll1_config = { + .l = 0x1A, + .alpha = 0xaaa, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x00002261, + .config_ctl_hi1_val = 0x2a9a699c, + .test_ctl_val = 0x00000000, + .test_ctl_hi_val = 0x00000000, + .test_ctl_hi1_val = 0x01800000, + .user_ctl_val = 0x00000000, + .user_ctl_hi_val = 0x00000805, + .user_ctl_hi1_val = 0x00000000, +}; + +static struct clk_alpha_pll gpu_cc_pll1 = { + .offset = 0x100, + .vco_table = lucid_5lpe_vco, + .num_vco = ARRAY_SIZE(lucid_5lpe_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_pll1", + .parent_data = &parent_data_tcxo, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_5lpe_ops, + }, + }, +}; + +static const struct parent_map gpu_cc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPU_CC_PLL0_OUT_MAIN, 1 }, + { P_GPU_CC_PLL1_OUT_MAIN, 3 }, + { P_GCC_GPU_GPLL0_CLK_SRC, 5 }, + { P_GCC_GPU_GPLL0_DIV_CLK_SRC, 6 }, +}; + +static const struct clk_parent_data gpu_cc_parent_data_0[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpu_cc_pll0.clkr.hw }, + { .hw = &gpu_cc_pll1.clkr.hw }, + { .index = DT_GCC_GPU_GPLL0_CLK_SRC }, + { .index = DT_GCC_GPU_GPLL0_DIV_CLK_SRC }, +}; + +static const struct parent_map gpu_cc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GPU_CC_PLL1_OUT_MAIN, 3 }, + { P_GCC_GPU_GPLL0_CLK_SRC, 5 }, + { P_GCC_GPU_GPLL0_DIV_CLK_SRC, 6 }, +}; + +static const struct clk_parent_data gpu_cc_parent_data_1[] = { + { .index = DT_BI_TCXO }, + { .hw = &gpu_cc_pll1.clkr.hw }, + { .index = DT_GCC_GPU_GPLL0_CLK_SRC }, + { .index = DT_GCC_GPU_GPLL0_DIV_CLK_SRC }, +}; + +static const struct freq_tbl ftbl_gpu_cc_gmu_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(200000000, P_GCC_GPU_GPLL0_DIV_CLK_SRC, 1.5, 0, 0), + F(500000000, P_GPU_CC_PLL1_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gpu_cc_gmu_clk_src = { + .cmd_rcgr = 0x1120, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpu_cc_parent_map_0, + .freq_tbl = ftbl_gpu_cc_gmu_clk_src, + .clkr.hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_gmu_clk_src", + .parent_data = gpu_cc_parent_data_0, + .num_parents = ARRAY_SIZE(gpu_cc_parent_data_0), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static const struct freq_tbl ftbl_gpu_cc_hub_clk_src[] = { + F(200000000, P_GCC_GPU_GPLL0_CLK_SRC, 3, 0, 0), + F(300000000, P_GCC_GPU_GPLL0_CLK_SRC, 2, 0, 0), + F(400000000, P_GCC_GPU_GPLL0_CLK_SRC, 1.5, 0, 0), + { } +}; + +static struct clk_rcg2 gpu_cc_hub_clk_src = { + .cmd_rcgr = 0x117c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpu_cc_parent_map_1, + .freq_tbl = ftbl_gpu_cc_hub_clk_src, + .clkr.hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_hub_clk_src", + .parent_data = gpu_cc_parent_data_1, + .num_parents = ARRAY_SIZE(gpu_cc_parent_data_1), + .ops = &clk_rcg2_shared_ops, + }, +}; + +static struct clk_regmap_div gpu_cc_hub_ahb_div_clk_src = { + .reg = 0x11c0, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gpu_cc_hub_ahb_div_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gpu_cc_hub_cx_int_div_clk_src = { + .reg = 0x11bc, + .shift = 0, + .width = 4, + .clkr.hw.init = &(const struct clk_init_data) { + .name = "gpu_cc_hub_cx_int_div_clk_src", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gpu_cc_ahb_clk = { + .halt_reg = 0x1078, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1078, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_ahb_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_ahb_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_crc_ahb_clk = { + .halt_reg = 0x107c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x107c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_crc_ahb_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_ahb_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cx_gmu_clk = { + .halt_reg = 0x1098, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1098, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_cx_gmu_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_gmu_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cx_snoc_dvm_clk = { + .halt_reg = 0x108c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x108c, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_cx_snoc_dvm_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_cxo_aon_clk = { + .halt_reg = 0x1004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1004, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_cxo_aon_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_gx_gmu_clk = { + .halt_reg = 0x1064, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1064, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_gx_gmu_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_gmu_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_hlos1_vote_gpu_smmu_clk = { + .halt_reg = 0x5000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5000, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_hlos1_vote_gpu_smmu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_hub_aon_clk = { + .halt_reg = 0x1178, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1178, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_hub_aon_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_hub_cx_int_clk = { + .halt_reg = 0x1204, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1204, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_hub_cx_int_clk", + .parent_hws = (const struct clk_hw*[]){ + &gpu_cc_hub_cx_int_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gpu_cc_sleep_clk = { + .halt_reg = 0x1090, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1090, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "gpu_cc_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap *gpu_cc_sc8280xp_clocks[] = { + [GPU_CC_AHB_CLK] = &gpu_cc_ahb_clk.clkr, + [GPU_CC_CRC_AHB_CLK] = &gpu_cc_crc_ahb_clk.clkr, + [GPU_CC_CX_GMU_CLK] = &gpu_cc_cx_gmu_clk.clkr, + [GPU_CC_CX_SNOC_DVM_CLK] = &gpu_cc_cx_snoc_dvm_clk.clkr, + [GPU_CC_CXO_AON_CLK] = &gpu_cc_cxo_aon_clk.clkr, + [GPU_CC_GMU_CLK_SRC] = &gpu_cc_gmu_clk_src.clkr, + [GPU_CC_GX_GMU_CLK] = &gpu_cc_gx_gmu_clk.clkr, + [GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK] = &gpu_cc_hlos1_vote_gpu_smmu_clk.clkr, + [GPU_CC_HUB_AHB_DIV_CLK_SRC] = &gpu_cc_hub_ahb_div_clk_src.clkr, + [GPU_CC_HUB_AON_CLK] = &gpu_cc_hub_aon_clk.clkr, + [GPU_CC_HUB_CLK_SRC] = &gpu_cc_hub_clk_src.clkr, + [GPU_CC_HUB_CX_INT_CLK] = &gpu_cc_hub_cx_int_clk.clkr, + [GPU_CC_HUB_CX_INT_DIV_CLK_SRC] = &gpu_cc_hub_cx_int_div_clk_src.clkr, + [GPU_CC_PLL0] = &gpu_cc_pll0.clkr, + [GPU_CC_PLL1] = &gpu_cc_pll1.clkr, + [GPU_CC_SLEEP_CLK] = &gpu_cc_sleep_clk.clkr, +}; + +static struct gdsc cx_gdsc = { + .gdscr = 0x106c, + .gds_hw_ctrl = 0x1540, + .pd = { + .name = "cx_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE | RETAIN_FF_ENABLE, +}; + +static struct gdsc gx_gdsc = { + .gdscr = 0x100c, + .clamp_io_ctrl = 0x1508, + .pd = { + .name = "gx_gdsc", + .power_on = gdsc_gx_do_nothing_enable, + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = CLAMP_IO | RETAIN_FF_ENABLE, +}; + +static struct gdsc *gpu_cc_sc8280xp_gdscs[] = { + [GPU_CC_CX_GDSC] = &cx_gdsc, + [GPU_CC_GX_GDSC] = &gx_gdsc, +}; + +static const struct regmap_config gpu_cc_sc8280xp_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x8030, + .fast_io = true, +}; + +static struct qcom_cc_desc gpu_cc_sc8280xp_desc = { + .config = &gpu_cc_sc8280xp_regmap_config, + .clks = gpu_cc_sc8280xp_clocks, + .num_clks = ARRAY_SIZE(gpu_cc_sc8280xp_clocks), + .gdscs = gpu_cc_sc8280xp_gdscs, + .num_gdscs = ARRAY_SIZE(gpu_cc_sc8280xp_gdscs), +}; + +static int gpu_cc_sc8280xp_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &gpu_cc_sc8280xp_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_lucid_pll_configure(&gpu_cc_pll0, regmap, &gpu_cc_pll0_config); + clk_lucid_pll_configure(&gpu_cc_pll1, regmap, &gpu_cc_pll1_config); + + /* + * Keep the clocks always-ON + * GPU_CC_CB_CLK, GPU_CC_CXO_CLK + */ + regmap_update_bits(regmap, 0x1170, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x109c, BIT(0), BIT(0)); + + return qcom_cc_really_probe(pdev, &gpu_cc_sc8280xp_desc, regmap); +} + +static const struct of_device_id gpu_cc_sc8280xp_match_table[] = { + { .compatible = "qcom,sc8280xp-gpucc" }, + { } +}; +MODULE_DEVICE_TABLE(of, gpu_cc_sc8280xp_match_table); + +static struct platform_driver gpu_cc_sc8280xp_driver = { + .probe = gpu_cc_sc8280xp_probe, + .driver = { + .name = "gpu_cc-sc8280xp", + .of_match_table = gpu_cc_sc8280xp_match_table, + }, +}; +module_platform_driver(gpu_cc_sc8280xp_driver); + +MODULE_DESCRIPTION("Qualcomm SC8280XP GPU clock controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/qcom/kpss-xcc.c b/drivers/clk/qcom/kpss-xcc.c index 88d4b33ac0cc35e269ccb67d2421c558cbee2c9b..b1b370274ec48fec5b97312ff3904c48149e12ce 100644 --- a/drivers/clk/qcom/kpss-xcc.c +++ b/drivers/clk/qcom/kpss-xcc.c @@ -12,9 +12,9 @@ #include #include -static const char *aux_parents[] = { - "pll8_vote", - "pxo", +static const struct clk_parent_data aux_parents[] = { + { .fw_name = "pll8_vote", .name = "pll8_vote" }, + { .fw_name = "pxo", .name = "pxo_board" }, }; static const u32 aux_parent_map[] = { @@ -32,8 +32,8 @@ MODULE_DEVICE_TABLE(of, kpss_xcc_match_table); static int kpss_xcc_driver_probe(struct platform_device *pdev) { const struct of_device_id *id; - struct clk *clk; void __iomem *base; + struct clk_hw *hw; const char *name; id = of_match_device(kpss_xcc_match_table, &pdev->dev); @@ -55,24 +55,16 @@ static int kpss_xcc_driver_probe(struct platform_device *pdev) base += 0x28; } - clk = clk_register_mux_table(&pdev->dev, name, aux_parents, - ARRAY_SIZE(aux_parents), 0, base, 0, 0x3, - 0, aux_parent_map, NULL); + hw = devm_clk_hw_register_mux_parent_data_table(&pdev->dev, name, aux_parents, + ARRAY_SIZE(aux_parents), 0, + base, 0, 0x3, + 0, aux_parent_map, NULL); - platform_set_drvdata(pdev, clk); - - return PTR_ERR_OR_ZERO(clk); -} - -static int kpss_xcc_driver_remove(struct platform_device *pdev) -{ - clk_unregister_mux(platform_get_drvdata(pdev)); - return 0; + return PTR_ERR_OR_ZERO(hw); } static struct platform_driver kpss_xcc_driver = { .probe = kpss_xcc_driver_probe, - .remove = kpss_xcc_driver_remove, .driver = { .name = "kpss-xcc", .of_match_table = kpss_xcc_match_table, diff --git a/drivers/clk/qcom/lcc-ipq806x.c b/drivers/clk/qcom/lcc-ipq806x.c index 1a2be4aeb31d4453dba396aad427a522cf69f508..81a44a9a9abc6b1cf68f8f1333cc64ebcd208bdb 100644 --- a/drivers/clk/qcom/lcc-ipq806x.c +++ b/drivers/clk/qcom/lcc-ipq806x.c @@ -22,6 +22,7 @@ #include "clk-branch.h" #include "clk-regmap-divider.h" #include "clk-regmap-mux.h" +#include "reset.h" static struct clk_pll pll4 = { .l_reg = 0x4, @@ -33,7 +34,9 @@ static struct clk_pll pll4 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll4", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = &(const struct clk_parent_data) { + .fw_name = "pxo", .name = "pxo_board", + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -63,9 +66,9 @@ static const struct parent_map lcc_pxo_pll4_map[] = { { P_PLL4, 2 } }; -static const char * const lcc_pxo_pll4[] = { - "pxo", - "pll4_vote", +static const struct clk_parent_data lcc_pxo_pll4[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "pll4_vote", .name = "pll4_vote" }, }; static struct freq_tbl clk_tbl_aif_mi2s[] = { @@ -130,18 +133,14 @@ static struct clk_rcg mi2s_osr_src = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "mi2s_osr_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, }, }; -static const char * const lcc_mi2s_parents[] = { - "mi2s_osr_src", -}; - static struct clk_branch mi2s_osr_clk = { .halt_reg = 0x50, .halt_bit = 1, @@ -151,7 +150,9 @@ static struct clk_branch mi2s_osr_clk = { .enable_mask = BIT(17), .hw.init = &(struct clk_init_data){ .name = "mi2s_osr_clk", - .parent_names = lcc_mi2s_parents, + .parent_hws = (const struct clk_hw*[]) { + &mi2s_osr_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -166,7 +167,9 @@ static struct clk_regmap_div mi2s_div_clk = { .clkr = { .hw.init = &(struct clk_init_data){ .name = "mi2s_div_clk", - .parent_names = lcc_mi2s_parents, + .parent_hws = (const struct clk_hw*[]) { + &mi2s_osr_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_regmap_div_ops, }, @@ -182,7 +185,9 @@ static struct clk_branch mi2s_bit_div_clk = { .enable_mask = BIT(15), .hw.init = &(struct clk_init_data){ .name = "mi2s_bit_div_clk", - .parent_names = (const char *[]){ "mi2s_div_clk" }, + .parent_hws = (const struct clk_hw*[]) { + &mi2s_div_clk.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -190,6 +195,10 @@ static struct clk_branch mi2s_bit_div_clk = { }, }; +static const struct clk_parent_data lcc_mi2s_bit_div_codec_clk[] = { + { .hw = &mi2s_bit_div_clk.clkr.hw, }, + { .fw_name = "mi2s_codec", .name = "mi2s_codec_clk" }, +}; static struct clk_regmap_mux mi2s_bit_clk = { .reg = 0x48, @@ -198,11 +207,8 @@ static struct clk_regmap_mux mi2s_bit_clk = { .clkr = { .hw.init = &(struct clk_init_data){ .name = "mi2s_bit_clk", - .parent_names = (const char *[]){ - "mi2s_bit_div_clk", - "mi2s_codec_clk", - }, - .num_parents = 2, + .parent_data = lcc_mi2s_bit_div_codec_clk, + .num_parents = ARRAY_SIZE(lcc_mi2s_bit_div_codec_clk), .ops = &clk_regmap_mux_closest_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -244,8 +250,8 @@ static struct clk_rcg pcm_src = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "pcm_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -261,7 +267,9 @@ static struct clk_branch pcm_clk_out = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "pcm_clk_out", - .parent_names = (const char *[]){ "pcm_src" }, + .parent_hws = (const struct clk_hw*[]) { + &pcm_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -269,6 +277,11 @@ static struct clk_branch pcm_clk_out = { }, }; +static const struct clk_parent_data lcc_pcm_clk_out_codec_clk[] = { + { .hw = &pcm_clk_out.clkr.hw, }, + { .fw_name = "pcm_codec_clk", .name = "pcm_codec_clk" }, +}; + static struct clk_regmap_mux pcm_clk = { .reg = 0x54, .shift = 10, @@ -276,11 +289,8 @@ static struct clk_regmap_mux pcm_clk = { .clkr = { .hw.init = &(struct clk_init_data){ .name = "pcm_clk", - .parent_names = (const char *[]){ - "pcm_clk_out", - "pcm_codec_clk", - }, - .num_parents = 2, + .parent_data = lcc_pcm_clk_out_codec_clk, + .num_parents = ARRAY_SIZE(lcc_pcm_clk_out_codec_clk), .ops = &clk_regmap_mux_closest_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -324,18 +334,14 @@ static struct clk_rcg spdif_src = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "spdif_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, }, }; -static const char * const lcc_spdif_parents[] = { - "spdif_src", -}; - static struct clk_branch spdif_clk = { .halt_reg = 0xd4, .halt_bit = 1, @@ -345,7 +351,9 @@ static struct clk_branch spdif_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "spdif_clk", - .parent_names = lcc_spdif_parents, + .parent_hws = (const struct clk_hw*[]) { + &spdif_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -383,8 +391,8 @@ static struct clk_rcg ahbix_clk = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "ahbix", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_lcc_ops, }, }, @@ -405,6 +413,10 @@ static struct clk_regmap *lcc_ipq806x_clks[] = { [AHBIX_CLK] = &ahbix_clk.clkr, }; +static const struct qcom_reset_map lcc_ipq806x_resets[] = { + [LCC_PCM_RESET] = { 0x54, 13 }, +}; + static const struct regmap_config lcc_ipq806x_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -417,6 +429,8 @@ static const struct qcom_cc_desc lcc_ipq806x_desc = { .config = &lcc_ipq806x_regmap_config, .clks = lcc_ipq806x_clks, .num_clks = ARRAY_SIZE(lcc_ipq806x_clks), + .resets = lcc_ipq806x_resets, + .num_resets = ARRAY_SIZE(lcc_ipq806x_resets), }; static const struct of_device_id lcc_ipq806x_match_table[] = { diff --git a/drivers/clk/qcom/lcc-msm8960.c b/drivers/clk/qcom/lcc-msm8960.c index 84817cf2b6bdf31c08ea8b3247b5e55498bdb2af..3926184cc91b9dbdf56bebde0312cded32016cbd 100644 --- a/drivers/clk/qcom/lcc-msm8960.c +++ b/drivers/clk/qcom/lcc-msm8960.c @@ -33,7 +33,9 @@ static struct clk_pll pll4 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll4", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "pxo", .name = "pxo_board" }, + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -49,9 +51,9 @@ static const struct parent_map lcc_pxo_pll4_map[] = { { P_PLL4, 2 } }; -static const char * const lcc_pxo_pll4[] = { - "pxo", - "pll4_vote", +static const struct clk_parent_data lcc_pxo_pll4[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "pll4_vote", .name = "pll4_vote" }, }; static struct freq_tbl clk_tbl_aif_osr_492[] = { @@ -86,112 +88,7 @@ static struct freq_tbl clk_tbl_aif_osr_393[] = { { } }; -static struct clk_rcg mi2s_osr_src = { - .ns_reg = 0x48, - .md_reg = 0x4c, - .mn = { - .mnctr_en_bit = 8, - .mnctr_reset_bit = 7, - .mnctr_mode_shift = 5, - .n_val_shift = 24, - .m_val_shift = 8, - .width = 8, - }, - .p = { - .pre_div_shift = 3, - .pre_div_width = 2, - }, - .s = { - .src_sel_shift = 0, - .parent_map = lcc_pxo_pll4_map, - }, - .freq_tbl = clk_tbl_aif_osr_393, - .clkr = { - .enable_reg = 0x48, - .enable_mask = BIT(9), - .hw.init = &(struct clk_init_data){ - .name = "mi2s_osr_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, - .ops = &clk_rcg_ops, - .flags = CLK_SET_RATE_GATE, - }, - }, -}; - -static const char * const lcc_mi2s_parents[] = { - "mi2s_osr_src", -}; - -static struct clk_branch mi2s_osr_clk = { - .halt_reg = 0x50, - .halt_bit = 1, - .halt_check = BRANCH_HALT_ENABLE, - .clkr = { - .enable_reg = 0x48, - .enable_mask = BIT(17), - .hw.init = &(struct clk_init_data){ - .name = "mi2s_osr_clk", - .parent_names = lcc_mi2s_parents, - .num_parents = 1, - .ops = &clk_branch_ops, - .flags = CLK_SET_RATE_PARENT, - }, - }, -}; - -static struct clk_regmap_div mi2s_div_clk = { - .reg = 0x48, - .shift = 10, - .width = 4, - .clkr = { - .enable_reg = 0x48, - .enable_mask = BIT(15), - .hw.init = &(struct clk_init_data){ - .name = "mi2s_div_clk", - .parent_names = lcc_mi2s_parents, - .num_parents = 1, - .ops = &clk_regmap_div_ops, - }, - }, -}; - -static struct clk_branch mi2s_bit_div_clk = { - .halt_reg = 0x50, - .halt_bit = 0, - .halt_check = BRANCH_HALT_ENABLE, - .clkr = { - .enable_reg = 0x48, - .enable_mask = BIT(15), - .hw.init = &(struct clk_init_data){ - .name = "mi2s_bit_div_clk", - .parent_names = (const char *[]){ "mi2s_div_clk" }, - .num_parents = 1, - .ops = &clk_branch_ops, - .flags = CLK_SET_RATE_PARENT, - }, - }, -}; - -static struct clk_regmap_mux mi2s_bit_clk = { - .reg = 0x48, - .shift = 14, - .width = 1, - .clkr = { - .hw.init = &(struct clk_init_data){ - .name = "mi2s_bit_clk", - .parent_names = (const char *[]){ - "mi2s_bit_div_clk", - "mi2s_codec_clk", - }, - .num_parents = 2, - .ops = &clk_regmap_mux_closest_ops, - .flags = CLK_SET_RATE_PARENT, - }, - }, -}; - -#define CLK_AIF_OSR_DIV(prefix, _ns, _md, hr) \ +#define CLK_AIF_OSR_SRC(prefix, _ns, _md) \ static struct clk_rcg prefix##_osr_src = { \ .ns_reg = _ns, \ .md_reg = _md, \ @@ -217,85 +114,103 @@ static struct clk_rcg prefix##_osr_src = { \ .enable_mask = BIT(9), \ .hw.init = &(struct clk_init_data){ \ .name = #prefix "_osr_src", \ - .parent_names = lcc_pxo_pll4, \ - .num_parents = 2, \ + .parent_data = lcc_pxo_pll4, \ + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), \ .ops = &clk_rcg_ops, \ .flags = CLK_SET_RATE_GATE, \ }, \ }, \ }; \ - \ -static const char * const lcc_##prefix##_parents[] = { \ - #prefix "_osr_src", \ -}; \ - \ + +#define CLK_AIF_OSR_CLK(prefix, _ns, hr, en_bit) \ static struct clk_branch prefix##_osr_clk = { \ .halt_reg = hr, \ .halt_bit = 1, \ .halt_check = BRANCH_HALT_ENABLE, \ .clkr = { \ .enable_reg = _ns, \ - .enable_mask = BIT(21), \ + .enable_mask = BIT(en_bit), \ .hw.init = &(struct clk_init_data){ \ .name = #prefix "_osr_clk", \ - .parent_names = lcc_##prefix##_parents, \ + .parent_hws = (const struct clk_hw*[]){ \ + &prefix##_osr_src.clkr.hw, \ + }, \ .num_parents = 1, \ .ops = &clk_branch_ops, \ .flags = CLK_SET_RATE_PARENT, \ }, \ }, \ }; \ - \ + +#define CLK_AIF_OSR_DIV_CLK(prefix, _ns, _width) \ static struct clk_regmap_div prefix##_div_clk = { \ .reg = _ns, \ .shift = 10, \ - .width = 8, \ + .width = _width, \ .clkr = { \ .hw.init = &(struct clk_init_data){ \ .name = #prefix "_div_clk", \ - .parent_names = lcc_##prefix##_parents, \ + .parent_hws = (const struct clk_hw*[]){ \ + &prefix##_osr_src.clkr.hw, \ + }, \ .num_parents = 1, \ .ops = &clk_regmap_div_ops, \ }, \ }, \ }; \ - \ + +#define CLK_AIF_OSR_BIT_DIV_CLK(prefix, _ns, hr, en_bit) \ static struct clk_branch prefix##_bit_div_clk = { \ .halt_reg = hr, \ .halt_bit = 0, \ .halt_check = BRANCH_HALT_ENABLE, \ .clkr = { \ .enable_reg = _ns, \ - .enable_mask = BIT(19), \ + .enable_mask = BIT(en_bit), \ .hw.init = &(struct clk_init_data){ \ .name = #prefix "_bit_div_clk", \ - .parent_names = (const char *[]){ \ - #prefix "_div_clk" \ - }, \ + .parent_hws = (const struct clk_hw*[]){ \ + &prefix##_div_clk.clkr.hw, \ + }, \ .num_parents = 1, \ .ops = &clk_branch_ops, \ .flags = CLK_SET_RATE_PARENT, \ }, \ }, \ }; \ - \ + +#define CLK_AIF_OSR_BIT_CLK(prefix, _ns, _shift) \ static struct clk_regmap_mux prefix##_bit_clk = { \ .reg = _ns, \ - .shift = 18, \ + .shift = _shift, \ .width = 1, \ .clkr = { \ .hw.init = &(struct clk_init_data){ \ .name = #prefix "_bit_clk", \ - .parent_names = (const char *[]){ \ - #prefix "_bit_div_clk", \ - #prefix "_codec_clk", \ + .parent_data = (const struct clk_parent_data[]){ \ + { .hw = &prefix##_bit_div_clk.clkr.hw, }, \ + { .fw_name = #prefix "_codec_clk", \ + .name = #prefix "_codec_clk", }, \ }, \ .num_parents = 2, \ .ops = &clk_regmap_mux_closest_ops, \ .flags = CLK_SET_RATE_PARENT, \ }, \ }, \ -} +}; + +CLK_AIF_OSR_SRC(mi2s, 0x48, 0x4c) +CLK_AIF_OSR_CLK(mi2s, 0x48, 0x50, 17) +CLK_AIF_OSR_DIV_CLK(mi2s, 0x48, 4) +CLK_AIF_OSR_BIT_DIV_CLK(mi2s, 0x48, 0x50, 15) +CLK_AIF_OSR_BIT_CLK(mi2s, 0x48, 14) + +#define CLK_AIF_OSR_DIV(prefix, _ns, _md, hr) \ + CLK_AIF_OSR_SRC(prefix, _ns, _md) \ + CLK_AIF_OSR_CLK(prefix, _ns, hr, 21) \ + CLK_AIF_OSR_DIV_CLK(prefix, _ns, 8) \ + CLK_AIF_OSR_BIT_DIV_CLK(prefix, _ns, hr, 19) \ + CLK_AIF_OSR_BIT_CLK(prefix, _ns, 18) CLK_AIF_OSR_DIV(codec_i2s_mic, 0x60, 0x64, 0x68); CLK_AIF_OSR_DIV(spare_i2s_mic, 0x78, 0x7c, 0x80); @@ -361,8 +276,8 @@ static struct clk_rcg pcm_src = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "pcm_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, @@ -378,7 +293,9 @@ static struct clk_branch pcm_clk_out = { .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ .name = "pcm_clk_out", - .parent_names = (const char *[]){ "pcm_src" }, + .parent_hws = (const struct clk_hw*[]){ + &pcm_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -393,9 +310,9 @@ static struct clk_regmap_mux pcm_clk = { .clkr = { .hw.init = &(struct clk_init_data){ .name = "pcm_clk", - .parent_names = (const char *[]){ - "pcm_clk_out", - "pcm_codec_clk", + .parent_data = (const struct clk_parent_data[]){ + { .hw = &pcm_clk_out.clkr.hw }, + { .fw_name = "pcm_codec_clk", .name = "pcm_codec_clk" }, }, .num_parents = 2, .ops = &clk_regmap_mux_closest_ops, @@ -429,18 +346,14 @@ static struct clk_rcg slimbus_src = { .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "slimbus_src", - .parent_names = lcc_pxo_pll4, - .num_parents = 2, + .parent_data = lcc_pxo_pll4, + .num_parents = ARRAY_SIZE(lcc_pxo_pll4), .ops = &clk_rcg_ops, .flags = CLK_SET_RATE_GATE, }, }, }; -static const char * const lcc_slimbus_parents[] = { - "slimbus_src", -}; - static struct clk_branch audio_slimbus_clk = { .halt_reg = 0xd4, .halt_bit = 0, @@ -450,7 +363,9 @@ static struct clk_branch audio_slimbus_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "audio_slimbus_clk", - .parent_names = lcc_slimbus_parents, + .parent_hws = (const struct clk_hw*[]){ + &slimbus_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -467,7 +382,9 @@ static struct clk_branch sps_slimbus_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "sps_slimbus_clk", - .parent_names = lcc_slimbus_parents, + .parent_hws = (const struct clk_hw*[]){ + &slimbus_src.clkr.hw, + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, diff --git a/drivers/clk/qcom/lpassaudiocc-sc7280.c b/drivers/clk/qcom/lpassaudiocc-sc7280.c index 6ab6e5a34c7243ba64f30d3c50f7bc4a1ba1fe9f..063e0365f311928e2bb3342f943a84c85eb854de 100644 --- a/drivers/clk/qcom/lpassaudiocc-sc7280.c +++ b/drivers/clk/qcom/lpassaudiocc-sc7280.c @@ -12,6 +12,7 @@ #include #include +#include #include #include "clk-alpha-pll.h" @@ -22,6 +23,7 @@ #include "clk-regmap-mux.h" #include "common.h" #include "gdsc.h" +#include "reset.h" enum { P_BI_TCXO, @@ -38,6 +40,32 @@ static const struct pll_vco zonda_vco[] = { { 595200000UL, 3600000000UL, 0 }, }; +static struct clk_branch lpass_q6ss_ahbm_clk = { + .halt_reg = 0x901c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x901c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "lpass_q6ss_ahbm_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch lpass_q6ss_ahbs_clk = { + .halt_reg = 0x9020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x9020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "lpass_q6ss_ahbs_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + /* 1128.96MHz configuration */ static const struct alpha_pll_config lpass_audio_cc_pll_config = { .l = 0x3a, @@ -221,7 +249,7 @@ static struct clk_rcg2 lpass_aon_cc_main_rcg_clk_src = { .parent_data = lpass_aon_cc_parent_data_0, .num_parents = ARRAY_SIZE(lpass_aon_cc_parent_data_0), .flags = CLK_OPS_PARENT_ENABLE, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -614,6 +642,11 @@ static struct gdsc lpass_aon_cc_lpass_audio_hm_gdsc = { .flags = RETAIN_FF_ENABLE, }; +static struct clk_regmap *lpass_cc_sc7280_clocks[] = { + [LPASS_Q6SS_AHBM_CLK] = &lpass_q6ss_ahbm_clk.clkr, + [LPASS_Q6SS_AHBS_CLK] = &lpass_q6ss_ahbs_clk.clkr, +}; + static struct clk_regmap *lpass_aon_cc_sc7280_clocks[] = { [LPASS_AON_CC_AUDIO_HM_H_CLK] = &lpass_aon_cc_audio_hm_h_clk.clkr, [LPASS_AON_CC_VA_MEM0_CLK] = &lpass_aon_cc_va_mem0_clk.clkr, @@ -659,12 +692,30 @@ static struct regmap_config lpass_audio_cc_sc7280_regmap_config = { .fast_io = true, }; +static const struct qcom_cc_desc lpass_cc_sc7280_desc = { + .config = &lpass_audio_cc_sc7280_regmap_config, + .clks = lpass_cc_sc7280_clocks, + .num_clks = ARRAY_SIZE(lpass_cc_sc7280_clocks), +}; + static const struct qcom_cc_desc lpass_audio_cc_sc7280_desc = { .config = &lpass_audio_cc_sc7280_regmap_config, .clks = lpass_audio_cc_sc7280_clocks, .num_clks = ARRAY_SIZE(lpass_audio_cc_sc7280_clocks), }; +static const struct qcom_reset_map lpass_audio_cc_sc7280_resets[] = { + [LPASS_AUDIO_SWR_RX_CGCR] = { 0xa0, 1 }, + [LPASS_AUDIO_SWR_TX_CGCR] = { 0xa8, 1 }, + [LPASS_AUDIO_SWR_WSA_CGCR] = { 0xb0, 1 }, +}; + +static const struct qcom_cc_desc lpass_audio_cc_reset_sc7280_desc = { + .config = &lpass_audio_cc_sc7280_regmap_config, + .resets = lpass_audio_cc_sc7280_resets, + .num_resets = ARRAY_SIZE(lpass_audio_cc_sc7280_resets), +}; + static const struct of_device_id lpass_audio_cc_sc7280_match_table[] = { { .compatible = "qcom,sc7280-lpassaudiocc" }, { } @@ -741,6 +792,13 @@ static int lpass_audio_cc_sc7280_probe(struct platform_device *pdev) return ret; } + ret = qcom_cc_probe_by_index(pdev, 1, &lpass_audio_cc_reset_sc7280_desc); + if (ret) { + dev_err(&pdev->dev, "Failed to register LPASS AUDIO CC Resets\n"); + pm_runtime_disable(&pdev->dev); + return ret; + } + pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); @@ -785,6 +843,12 @@ static int lpass_aon_cc_sc7280_probe(struct platform_device *pdev) if (ret) return ret; + if (of_property_read_bool(pdev->dev.of_node, "qcom,adsp-pil-mode")) { + lpass_audio_cc_sc7280_regmap_config.name = "cc"; + desc = &lpass_cc_sc7280_desc; + return qcom_cc_probe(pdev, desc); + } + lpass_audio_cc_sc7280_regmap_config.name = "lpasscc_aon"; lpass_audio_cc_sc7280_regmap_config.max_register = 0xa0008; desc = &lpass_aon_cc_sc7280_desc; diff --git a/drivers/clk/qcom/lpasscc-sc7280.c b/drivers/clk/qcom/lpasscc-sc7280.c index b39ee1c9647bc2ef6ed3463e8900a0335736579c..5c1e17bd0d763d56dcf6cf167051865f352a85fe 100644 --- a/drivers/clk/qcom/lpasscc-sc7280.c +++ b/drivers/clk/qcom/lpasscc-sc7280.c @@ -17,32 +17,6 @@ #include "clk-branch.h" #include "common.h" -static struct clk_branch lpass_q6ss_ahbm_clk = { - .halt_reg = 0x1c, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x1c, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "lpass_q6ss_ahbm_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - -static struct clk_branch lpass_q6ss_ahbs_clk = { - .halt_reg = 0x20, - .halt_check = BRANCH_HALT_VOTED, - .clkr = { - .enable_reg = 0x20, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "lpass_q6ss_ahbs_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch lpass_top_cc_lpi_q6_axim_hs_clk = { .halt_reg = 0x0, .halt_check = BRANCH_HALT, @@ -105,17 +79,6 @@ static struct regmap_config lpass_regmap_config = { .fast_io = true, }; -static struct clk_regmap *lpass_cc_sc7280_clocks[] = { - [LPASS_Q6SS_AHBM_CLK] = &lpass_q6ss_ahbm_clk.clkr, - [LPASS_Q6SS_AHBS_CLK] = &lpass_q6ss_ahbs_clk.clkr, -}; - -static const struct qcom_cc_desc lpass_cc_sc7280_desc = { - .config = &lpass_regmap_config, - .clks = lpass_cc_sc7280_clocks, - .num_clks = ARRAY_SIZE(lpass_cc_sc7280_clocks), -}; - static struct clk_regmap *lpass_cc_top_sc7280_clocks[] = { [LPASS_TOP_CC_LPI_Q6_AXIM_HS_CLK] = &lpass_top_cc_lpi_q6_axim_hs_clk.clkr, @@ -169,13 +132,6 @@ static int lpass_cc_sc7280_probe(struct platform_device *pdev) if (ret) goto destroy_pm_clk; - lpass_regmap_config.name = "cc"; - desc = &lpass_cc_sc7280_desc; - - ret = qcom_cc_probe_by_index(pdev, 2, desc); - if (ret) - goto destroy_pm_clk; - return 0; destroy_pm_clk: diff --git a/drivers/clk/qcom/lpasscorecc-sc7280.c b/drivers/clk/qcom/lpasscorecc-sc7280.c index 1f1f1bd1b68ef3a96c8c7c0eac5716a78384e6fa..6ad19b06b1ce3cd9d056b7d3ad9c1e60e67cad05 100644 --- a/drivers/clk/qcom/lpasscorecc-sc7280.c +++ b/drivers/clk/qcom/lpasscorecc-sc7280.c @@ -190,6 +190,19 @@ static struct clk_rcg2 lpass_core_cc_ext_if1_clk_src = { }, }; +static struct clk_rcg2 lpass_core_cc_ext_mclk0_clk_src = { + .cmd_rcgr = 0x20000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = lpass_core_cc_parent_map_0, + .freq_tbl = ftbl_lpass_core_cc_ext_if0_clk_src, + .clkr.hw.init = &(const struct clk_init_data){ + .name = "lpass_core_cc_ext_mclk0_clk_src", + .parent_data = lpass_core_cc_parent_data_0, + .num_parents = ARRAY_SIZE(lpass_core_cc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; static struct clk_branch lpass_core_cc_core_clk = { .halt_reg = 0x1f000, @@ -283,6 +296,24 @@ static struct clk_branch lpass_core_cc_lpm_mem0_core_clk = { }, }; +static struct clk_branch lpass_core_cc_ext_mclk0_clk = { + .halt_reg = 0x20014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x20014, + .enable_mask = BIT(0), + .hw.init = &(const struct clk_init_data){ + .name = "lpass_core_cc_ext_mclk0_clk", + .parent_hws = (const struct clk_hw*[]){ + &lpass_core_cc_ext_mclk0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch lpass_core_cc_sysnoc_mport_core_clk = { .halt_reg = 0x23000, .halt_check = BRANCH_HALT_VOTED, @@ -326,6 +357,8 @@ static struct clk_regmap *lpass_core_cc_sc7280_clocks[] = { [LPASS_CORE_CC_LPM_CORE_CLK] = &lpass_core_cc_lpm_core_clk.clkr, [LPASS_CORE_CC_LPM_MEM0_CORE_CLK] = &lpass_core_cc_lpm_mem0_core_clk.clkr, [LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK] = &lpass_core_cc_sysnoc_mport_core_clk.clkr, + [LPASS_CORE_CC_EXT_MCLK0_CLK] = &lpass_core_cc_ext_mclk0_clk.clkr, + [LPASS_CORE_CC_EXT_MCLK0_CLK_SRC] = &lpass_core_cc_ext_mclk0_clk_src.clkr, }; static struct regmap_config lpass_core_cc_sc7280_regmap_config = { diff --git a/drivers/clk/qcom/mmcc-msm8960.c b/drivers/clk/qcom/mmcc-msm8960.c index aaaad65b64583cf133691e7326c27420bbb95ac6..6bf908a51f5309ee6f4a61a58ad61d55eac42085 100644 --- a/drivers/clk/qcom/mmcc-msm8960.c +++ b/drivers/clk/qcom/mmcc-msm8960.c @@ -41,70 +41,6 @@ enum { #define F_MN(f, s, _m, _n) { .freq = f, .src = s, .m = _m, .n = _n } -static const struct parent_map mmcc_pxo_pll8_pll2_map[] = { - { P_PXO, 0 }, - { P_PLL8, 2 }, - { P_PLL2, 1 } -}; - -static const char * const mmcc_pxo_pll8_pll2[] = { - "pxo", - "pll8_vote", - "pll2", -}; - -static const struct parent_map mmcc_pxo_pll8_pll2_pll3_map[] = { - { P_PXO, 0 }, - { P_PLL8, 2 }, - { P_PLL2, 1 }, - { P_PLL3, 3 } -}; - -static const char * const mmcc_pxo_pll8_pll2_pll15[] = { - "pxo", - "pll8_vote", - "pll2", - "pll15", -}; - -static const struct parent_map mmcc_pxo_pll8_pll2_pll15_map[] = { - { P_PXO, 0 }, - { P_PLL8, 2 }, - { P_PLL2, 1 }, - { P_PLL15, 3 } -}; - -static const char * const mmcc_pxo_pll8_pll2_pll3[] = { - "pxo", - "pll8_vote", - "pll2", - "pll3", -}; - -static const struct parent_map mmcc_pxo_dsi2_dsi1_map[] = { - { P_PXO, 0 }, - { P_DSI2_PLL_DSICLK, 1 }, - { P_DSI1_PLL_DSICLK, 3 }, -}; - -static const char * const mmcc_pxo_dsi2_dsi1[] = { - "pxo", - "dsi2pll", - "dsi1pll", -}; - -static const struct parent_map mmcc_pxo_dsi1_dsi2_byte_map[] = { - { P_PXO, 0 }, - { P_DSI1_PLL_BYTECLK, 1 }, - { P_DSI2_PLL_BYTECLK, 2 }, -}; - -static const char * const mmcc_pxo_dsi1_dsi2_byte[] = { - "pxo", - "dsi1pllbyte", - "dsi2pllbyte", -}; - static struct clk_pll pll2 = { .l_reg = 0x320, .m_reg = 0x324, @@ -115,7 +51,9 @@ static struct clk_pll pll2 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll2", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "pxo", .name = "pxo_board" }, + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -131,7 +69,9 @@ static struct clk_pll pll15 = { .status_bit = 16, .clkr.hw.init = &(struct clk_init_data){ .name = "pll15", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "pxo", .name = "pxo_board" }, + }, .num_parents = 1, .ops = &clk_pll_ops, }, @@ -151,6 +91,70 @@ static const struct pll_config pll15_config = { .main_output_mask = BIT(23), }; +static const struct parent_map mmcc_pxo_pll8_pll2_map[] = { + { P_PXO, 0 }, + { P_PLL8, 2 }, + { P_PLL2, 1 } +}; + +static const struct clk_parent_data mmcc_pxo_pll8_pll2[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "pll8_vote", .name = "pll8_vote" }, + { .hw = &pll2.clkr.hw }, +}; + +static const struct parent_map mmcc_pxo_pll8_pll2_pll3_map[] = { + { P_PXO, 0 }, + { P_PLL8, 2 }, + { P_PLL2, 1 }, + { P_PLL3, 3 } +}; + +static const struct clk_parent_data mmcc_pxo_pll8_pll2_pll15[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "pll8_vote", .name = "pll8_vote" }, + { .hw = &pll2.clkr.hw }, + { .hw = &pll15.clkr.hw }, +}; + +static const struct parent_map mmcc_pxo_pll8_pll2_pll15_map[] = { + { P_PXO, 0 }, + { P_PLL8, 2 }, + { P_PLL2, 1 }, + { P_PLL15, 3 } +}; + +static const struct clk_parent_data mmcc_pxo_pll8_pll2_pll3[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "pll8_vote", .name = "pll8_vote" }, + { .hw = &pll2.clkr.hw }, + { .fw_name = "pll3", .name = "pll3" }, +}; + +static const struct parent_map mmcc_pxo_dsi2_dsi1_map[] = { + { P_PXO, 0 }, + { P_DSI2_PLL_DSICLK, 1 }, + { P_DSI1_PLL_DSICLK, 3 }, +}; + +static const struct clk_parent_data mmcc_pxo_dsi2_dsi1[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "dsi2pll", .name = "dsi2pll" }, + { .fw_name = "dsi1pll", .name = "dsi1pll" }, +}; + +static const struct parent_map mmcc_pxo_dsi1_dsi2_byte_map[] = { + { P_PXO, 0 }, + { P_DSI1_PLL_BYTECLK, 1 }, + { P_DSI2_PLL_BYTECLK, 2 }, +}; + +static const struct clk_parent_data mmcc_pxo_dsi1_dsi2_byte[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "dsi1pllbyte", .name = "dsi1pllbyte" }, + { .fw_name = "dsi2pllbyte", .name = "dsi2pllbyte" }, +}; + static struct freq_tbl clk_tbl_cam[] = { { 6000000, P_PLL8, 4, 1, 16 }, { 8000000, P_PLL8, 4, 1, 12 }, @@ -192,8 +196,8 @@ static struct clk_rcg camclk0_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "camclk0_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -207,7 +211,9 @@ static struct clk_branch camclk0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "camclk0_clk", - .parent_names = (const char *[]){ "camclk0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &camclk0_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, }, @@ -241,8 +247,8 @@ static struct clk_rcg camclk1_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "camclk1_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -256,7 +262,9 @@ static struct clk_branch camclk1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "camclk1_clk", - .parent_names = (const char *[]){ "camclk1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &camclk1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, }, @@ -290,8 +298,8 @@ static struct clk_rcg camclk2_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "camclk2_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -305,7 +313,9 @@ static struct clk_branch camclk2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "camclk2_clk", - .parent_names = (const char *[]){ "camclk2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &camclk2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, }, @@ -345,8 +355,8 @@ static struct clk_rcg csi0_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "csi0_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -359,7 +369,9 @@ static struct clk_branch csi0_clk = { .enable_reg = 0x0040, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi0_src.clkr.hw + }, .num_parents = 1, .name = "csi0_clk", .ops = &clk_branch_ops, @@ -375,7 +387,9 @@ static struct clk_branch csi0_phy_clk = { .enable_reg = 0x0040, .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi0_src.clkr.hw + }, .num_parents = 1, .name = "csi0_phy_clk", .ops = &clk_branch_ops, @@ -409,8 +423,8 @@ static struct clk_rcg csi1_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "csi1_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -423,7 +437,9 @@ static struct clk_branch csi1_clk = { .enable_reg = 0x0024, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi1_src.clkr.hw + }, .num_parents = 1, .name = "csi1_clk", .ops = &clk_branch_ops, @@ -439,7 +455,9 @@ static struct clk_branch csi1_phy_clk = { .enable_reg = 0x0024, .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi1_src.clkr.hw + }, .num_parents = 1, .name = "csi1_phy_clk", .ops = &clk_branch_ops, @@ -473,8 +491,8 @@ static struct clk_rcg csi2_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "csi2_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -487,7 +505,9 @@ static struct clk_branch csi2_clk = { .enable_reg = 0x022c, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi2_src.clkr.hw + }, .num_parents = 1, .name = "csi2_clk", .ops = &clk_branch_ops, @@ -503,7 +523,9 @@ static struct clk_branch csi2_phy_clk = { .enable_reg = 0x022c, .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "csi2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &csi2_src.clkr.hw + }, .num_parents = 1, .name = "csi2_phy_clk", .ops = &clk_branch_ops, @@ -602,10 +624,10 @@ static const struct clk_ops clk_ops_pix_rdi = { .determine_rate = __clk_mux_determine_rate, }; -static const char * const pix_rdi_parents[] = { - "csi0_clk", - "csi1_clk", - "csi2_clk", +static const struct clk_hw *pix_rdi_parents[] = { + &csi0_clk.clkr.hw, + &csi1_clk.clkr.hw, + &csi2_clk.clkr.hw, }; static struct clk_pix_rdi csi_pix_clk = { @@ -618,8 +640,8 @@ static struct clk_pix_rdi csi_pix_clk = { .enable_mask = BIT(26), .hw.init = &(struct clk_init_data){ .name = "csi_pix_clk", - .parent_names = pix_rdi_parents, - .num_parents = 3, + .parent_hws = pix_rdi_parents, + .num_parents = ARRAY_SIZE(pix_rdi_parents), .ops = &clk_ops_pix_rdi, }, }, @@ -635,8 +657,8 @@ static struct clk_pix_rdi csi_pix1_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "csi_pix1_clk", - .parent_names = pix_rdi_parents, - .num_parents = 3, + .parent_hws = pix_rdi_parents, + .num_parents = ARRAY_SIZE(pix_rdi_parents), .ops = &clk_ops_pix_rdi, }, }, @@ -652,8 +674,8 @@ static struct clk_pix_rdi csi_rdi_clk = { .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "csi_rdi_clk", - .parent_names = pix_rdi_parents, - .num_parents = 3, + .parent_hws = pix_rdi_parents, + .num_parents = ARRAY_SIZE(pix_rdi_parents), .ops = &clk_ops_pix_rdi, }, }, @@ -669,8 +691,8 @@ static struct clk_pix_rdi csi_rdi1_clk = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "csi_rdi1_clk", - .parent_names = pix_rdi_parents, - .num_parents = 3, + .parent_hws = pix_rdi_parents, + .num_parents = ARRAY_SIZE(pix_rdi_parents), .ops = &clk_ops_pix_rdi, }, }, @@ -686,8 +708,8 @@ static struct clk_pix_rdi csi_rdi2_clk = { .enable_mask = BIT(6), .hw.init = &(struct clk_init_data){ .name = "csi_rdi2_clk", - .parent_names = pix_rdi_parents, - .num_parents = 3, + .parent_hws = pix_rdi_parents, + .num_parents = ARRAY_SIZE(pix_rdi_parents), .ops = &clk_ops_pix_rdi, }, }, @@ -725,15 +747,13 @@ static struct clk_rcg csiphytimer_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "csiphytimer_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, }; -static const char * const csixphy_timer_src[] = { "csiphytimer_src" }; - static struct clk_branch csiphy0_timer_clk = { .halt_reg = 0x01e8, .halt_bit = 17, @@ -741,7 +761,9 @@ static struct clk_branch csiphy0_timer_clk = { .enable_reg = 0x0160, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = csixphy_timer_src, + .parent_hws = (const struct clk_hw*[]){ + &csiphytimer_src.clkr.hw, + }, .num_parents = 1, .name = "csiphy0_timer_clk", .ops = &clk_branch_ops, @@ -757,7 +779,9 @@ static struct clk_branch csiphy1_timer_clk = { .enable_reg = 0x0160, .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ - .parent_names = csixphy_timer_src, + .parent_hws = (const struct clk_hw*[]){ + &csiphytimer_src.clkr.hw, + }, .num_parents = 1, .name = "csiphy1_timer_clk", .ops = &clk_branch_ops, @@ -773,7 +797,9 @@ static struct clk_branch csiphy2_timer_clk = { .enable_reg = 0x0160, .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ - .parent_names = csixphy_timer_src, + .parent_hws = (const struct clk_hw*[]){ + &csiphytimer_src.clkr.hw, + }, .num_parents = 1, .name = "csiphy2_timer_clk", .ops = &clk_branch_ops, @@ -835,8 +861,8 @@ static struct clk_dyn_rcg gfx2d0_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gfx2d0_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -850,7 +876,9 @@ static struct clk_branch gfx2d0_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gfx2d0_clk", - .parent_names = (const char *[]){ "gfx2d0_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gfx2d0_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -895,8 +923,8 @@ static struct clk_dyn_rcg gfx2d1_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gfx2d1_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -910,7 +938,9 @@ static struct clk_branch gfx2d1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gfx2d1_clk", - .parent_names = (const char *[]){ "gfx2d1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gfx2d1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -996,8 +1026,8 @@ static struct clk_dyn_rcg gfx3d_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "gfx3d_src", - .parent_names = mmcc_pxo_pll8_pll2_pll3, - .num_parents = 4, + .parent_data = mmcc_pxo_pll8_pll2_pll3, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2_pll3), .ops = &clk_dyn_rcg_ops, }, }, @@ -1005,8 +1035,8 @@ static struct clk_dyn_rcg gfx3d_src = { static const struct clk_init_data gfx3d_8064_init = { .name = "gfx3d_src", - .parent_names = mmcc_pxo_pll8_pll2_pll15, - .num_parents = 4, + .parent_data = mmcc_pxo_pll8_pll2_pll15, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2_pll15), .ops = &clk_dyn_rcg_ops, }; @@ -1018,7 +1048,9 @@ static struct clk_branch gfx3d_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gfx3d_clk", - .parent_names = (const char *[]){ "gfx3d_src" }, + .parent_hws = (const struct clk_hw*[]){ + &gfx3d_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1074,8 +1106,8 @@ static struct clk_dyn_rcg vcap_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "vcap_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -1089,7 +1121,9 @@ static struct clk_branch vcap_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "vcap_clk", - .parent_names = (const char *[]){ "vcap_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vcap_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1105,7 +1139,9 @@ static struct clk_branch vcap_npl_clk = { .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "vcap_npl_clk", - .parent_names = (const char *[]){ "vcap_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vcap_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1153,8 +1189,8 @@ static struct clk_rcg ijpeg_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "ijpeg_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -1168,7 +1204,9 @@ static struct clk_branch ijpeg_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "ijpeg_clk", - .parent_names = (const char *[]){ "ijpeg_src" }, + .parent_hws = (const struct clk_hw*[]){ + &ijpeg_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1201,8 +1239,8 @@ static struct clk_rcg jpegd_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "jpegd_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -1216,7 +1254,9 @@ static struct clk_branch jpegd_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "jpegd_clk", - .parent_names = (const char *[]){ "jpegd_src" }, + .parent_hws = (const struct clk_hw*[]){ + &jpegd_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1281,8 +1321,8 @@ static struct clk_dyn_rcg mdp_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "mdp_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -1296,7 +1336,9 @@ static struct clk_branch mdp_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "mdp_clk", - .parent_names = (const char *[]){ "mdp_src" }, + .parent_hws = (const struct clk_hw*[]){ + &mdp_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1311,7 +1353,9 @@ static struct clk_branch mdp_lut_clk = { .enable_reg = 0x016c, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "mdp_src" }, + .parent_hws = (const struct clk_hw*[]){ + &mdp_src.clkr.hw + }, .num_parents = 1, .name = "mdp_lut_clk", .ops = &clk_branch_ops, @@ -1328,7 +1372,9 @@ static struct clk_branch mdp_vsync_clk = { .enable_mask = BIT(6), .hw.init = &(struct clk_init_data){ .name = "mdp_vsync_clk", - .parent_names = (const char *[]){ "pxo" }, + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "pxo", .name = "pxo_board" }, + }, .num_parents = 1, .ops = &clk_branch_ops }, @@ -1380,8 +1426,8 @@ static struct clk_dyn_rcg rot_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "rot_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -1395,7 +1441,9 @@ static struct clk_branch rot_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "rot_clk", - .parent_names = (const char *[]){ "rot_src" }, + .parent_hws = (const struct clk_hw*[]){ + &rot_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1408,9 +1456,9 @@ static const struct parent_map mmcc_pxo_hdmi_map[] = { { P_HDMI_PLL, 3 } }; -static const char * const mmcc_pxo_hdmi[] = { - "pxo", - "hdmi_pll", +static const struct clk_parent_data mmcc_pxo_hdmi[] = { + { .fw_name = "pxo", .name = "pxo_board" }, + { .fw_name = "hdmipll", .name = "hdmi_pll" }, }; static struct freq_tbl clk_tbl_tv[] = { @@ -1443,16 +1491,14 @@ static struct clk_rcg tv_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "tv_src", - .parent_names = mmcc_pxo_hdmi, - .num_parents = 2, + .parent_data = mmcc_pxo_hdmi, + .num_parents = ARRAY_SIZE(mmcc_pxo_hdmi), .ops = &clk_rcg_bypass_ops, .flags = CLK_SET_RATE_PARENT, }, }, }; -static const char * const tv_src_name[] = { "tv_src" }; - static struct clk_branch tv_enc_clk = { .halt_reg = 0x01d4, .halt_bit = 9, @@ -1460,7 +1506,9 @@ static struct clk_branch tv_enc_clk = { .enable_reg = 0x00ec, .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "tv_enc_clk", .ops = &clk_branch_ops, @@ -1476,7 +1524,9 @@ static struct clk_branch tv_dac_clk = { .enable_reg = 0x00ec, .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "tv_dac_clk", .ops = &clk_branch_ops, @@ -1492,7 +1542,9 @@ static struct clk_branch mdp_tv_clk = { .enable_reg = 0x00ec, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "mdp_tv_clk", .ops = &clk_branch_ops, @@ -1508,7 +1560,9 @@ static struct clk_branch hdmi_tv_clk = { .enable_reg = 0x00ec, .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "hdmi_tv_clk", .ops = &clk_branch_ops, @@ -1524,7 +1578,9 @@ static struct clk_branch rgb_tv_clk = { .enable_reg = 0x0124, .enable_mask = BIT(14), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "rgb_tv_clk", .ops = &clk_branch_ops, @@ -1540,7 +1596,9 @@ static struct clk_branch npl_tv_clk = { .enable_reg = 0x0124, .enable_mask = BIT(16), .hw.init = &(struct clk_init_data){ - .parent_names = tv_src_name, + .parent_hws = (const struct clk_hw*[]){ + &tv_src.clkr.hw, + }, .num_parents = 1, .name = "npl_tv_clk", .ops = &clk_branch_ops, @@ -1556,7 +1614,9 @@ static struct clk_branch hdmi_app_clk = { .enable_reg = 0x005c, .enable_mask = BIT(11), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "pxo" }, + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "pxo", .name = "pxo_board" }, + }, .num_parents = 1, .name = "hdmi_app_clk", .ops = &clk_branch_ops, @@ -1614,8 +1674,8 @@ static struct clk_dyn_rcg vcodec_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "vcodec_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_dyn_rcg_ops, }, }, @@ -1629,7 +1689,9 @@ static struct clk_branch vcodec_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "vcodec_clk", - .parent_names = (const char *[]){ "vcodec_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vcodec_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1665,8 +1727,8 @@ static struct clk_rcg vpe_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "vpe_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -1680,7 +1742,9 @@ static struct clk_branch vpe_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "vpe_clk", - .parent_names = (const char *[]){ "vpe_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vpe_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1733,8 +1797,8 @@ static struct clk_rcg vfe_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "vfe_src", - .parent_names = mmcc_pxo_pll8_pll2, - .num_parents = 3, + .parent_data = mmcc_pxo_pll8_pll2, + .num_parents = ARRAY_SIZE(mmcc_pxo_pll8_pll2), .ops = &clk_rcg_ops, }, }, @@ -1748,7 +1812,9 @@ static struct clk_branch vfe_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "vfe_clk", - .parent_names = (const char *[]){ "vfe_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vfe_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -1763,7 +1829,9 @@ static struct clk_branch vfe_csi_clk = { .enable_reg = 0x0104, .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "vfe_src" }, + .parent_hws = (const struct clk_hw*[]){ + &vfe_src.clkr.hw + }, .num_parents = 1, .name = "vfe_csi_clk", .ops = &clk_branch_ops, @@ -2067,8 +2135,8 @@ static struct clk_rcg dsi1_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi1_src", - .parent_names = mmcc_pxo_dsi2_dsi1, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi2_dsi1, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi2_dsi1), .ops = &clk_rcg_bypass2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -2083,7 +2151,9 @@ static struct clk_branch dsi1_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi1_clk", - .parent_names = (const char *[]){ "dsi1_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi1_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2115,8 +2185,8 @@ static struct clk_rcg dsi2_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi2_src", - .parent_names = mmcc_pxo_dsi2_dsi1, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi2_dsi1, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi2_dsi1), .ops = &clk_rcg_bypass2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -2131,7 +2201,9 @@ static struct clk_branch dsi2_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi2_clk", - .parent_names = (const char *[]){ "dsi2_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi2_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2154,8 +2226,8 @@ static struct clk_rcg dsi1_byte_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi1_byte_src", - .parent_names = mmcc_pxo_dsi1_dsi2_byte, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi1_dsi2_byte, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi1_dsi2_byte), .ops = &clk_rcg_bypass2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -2170,7 +2242,9 @@ static struct clk_branch dsi1_byte_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi1_byte_clk", - .parent_names = (const char *[]){ "dsi1_byte_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi1_byte_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2193,8 +2267,8 @@ static struct clk_rcg dsi2_byte_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi2_byte_src", - .parent_names = mmcc_pxo_dsi1_dsi2_byte, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi1_dsi2_byte, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi1_dsi2_byte), .ops = &clk_rcg_bypass2_ops, .flags = CLK_SET_RATE_PARENT, }, @@ -2209,7 +2283,9 @@ static struct clk_branch dsi2_byte_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi2_byte_clk", - .parent_names = (const char *[]){ "dsi2_byte_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi2_byte_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2232,8 +2308,8 @@ static struct clk_rcg dsi1_esc_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi1_esc_src", - .parent_names = mmcc_pxo_dsi1_dsi2_byte, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi1_dsi2_byte, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi1_dsi2_byte), .ops = &clk_rcg_esc_ops, }, }, @@ -2247,7 +2323,9 @@ static struct clk_branch dsi1_esc_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi1_esc_clk", - .parent_names = (const char *[]){ "dsi1_esc_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi1_esc_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2270,8 +2348,8 @@ static struct clk_rcg dsi2_esc_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi2_esc_src", - .parent_names = mmcc_pxo_dsi1_dsi2_byte, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi1_dsi2_byte, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi1_dsi2_byte), .ops = &clk_rcg_esc_ops, }, }, @@ -2285,7 +2363,9 @@ static struct clk_branch dsi2_esc_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "dsi2_esc_clk", - .parent_names = (const char *[]){ "dsi2_esc_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi2_esc_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2317,8 +2397,8 @@ static struct clk_rcg dsi1_pixel_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi1_pixel_src", - .parent_names = mmcc_pxo_dsi2_dsi1, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi2_dsi1, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi2_dsi1), .ops = &clk_rcg_pixel_ops, }, }, @@ -2332,7 +2412,9 @@ static struct clk_branch dsi1_pixel_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "mdp_pclk1_clk", - .parent_names = (const char *[]){ "dsi1_pixel_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi1_pixel_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, @@ -2364,8 +2446,8 @@ static struct clk_rcg dsi2_pixel_src = { .enable_mask = BIT(2), .hw.init = &(struct clk_init_data){ .name = "dsi2_pixel_src", - .parent_names = mmcc_pxo_dsi2_dsi1, - .num_parents = 3, + .parent_data = mmcc_pxo_dsi2_dsi1, + .num_parents = ARRAY_SIZE(mmcc_pxo_dsi2_dsi1), .ops = &clk_rcg_pixel_ops, }, }, @@ -2379,7 +2461,9 @@ static struct clk_branch dsi2_pixel_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "mdp_pclk2_clk", - .parent_names = (const char *[]){ "dsi2_pixel_src" }, + .parent_hws = (const struct clk_hw*[]){ + &dsi2_pixel_src.clkr.hw + }, .num_parents = 1, .ops = &clk_branch_ops, .flags = CLK_SET_RATE_PARENT, diff --git a/drivers/clk/qcom/reset.c b/drivers/clk/qcom/reset.c index 819d194be8f7b64bf0e2e131a3b5d313c6a4206e..2a16adb572d2bcdc4ccceaaebb8692db6bb50aac 100644 --- a/drivers/clk/qcom/reset.c +++ b/drivers/clk/qcom/reset.c @@ -13,8 +13,10 @@ static int qcom_reset(struct reset_controller_dev *rcdev, unsigned long id) { + struct qcom_reset_controller *rst = to_qcom_reset_controller(rcdev); + rcdev->ops->assert(rcdev, id); - udelay(1); + udelay(rst->reset_map[id].udelay ?: 1); /* use 1 us as default */ rcdev->ops->deassert(rcdev, id); return 0; } diff --git a/drivers/clk/qcom/reset.h b/drivers/clk/qcom/reset.h index 2a08b5e282c77c97db32e379bf5db33364d28fd5..b8c113582072bf1bb408e931848e929dd7fa4497 100644 --- a/drivers/clk/qcom/reset.h +++ b/drivers/clk/qcom/reset.h @@ -11,6 +11,7 @@ struct qcom_reset_map { unsigned int reg; u8 bit; + u8 udelay; }; struct regmap; diff --git a/drivers/clk/renesas/r8a779f0-cpg-mssr.c b/drivers/clk/renesas/r8a779f0-cpg-mssr.c index cd80b6084eceb258181b64b4c4d10b0bb787c1c9..4baf355e26d88b0acdb2673f754cb0bb4d21a71a 100644 --- a/drivers/clk/renesas/r8a779f0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779f0-cpg-mssr.c @@ -108,7 +108,13 @@ static const struct cpg_core_clk r8a779f0_core_clks[] __initconst = { DEF_FIXED("cbfusa", R8A779F0_CLK_CBFUSA, CLK_EXTAL, 2, 1), DEF_FIXED("cpex", R8A779F0_CLK_CPEX, CLK_EXTAL, 2, 1), - DEF_GEN4_SD("sd0", R8A779F0_CLK_SD0, CLK_SDSRC, 0x870), + DEF_FIXED("sasyncrt", R8A779F0_CLK_SASYNCRT, CLK_PLL5_DIV4, 48, 1), + DEF_FIXED("sasyncperd1", R8A779F0_CLK_SASYNCPERD1, CLK_PLL5_DIV4, 3, 1), + DEF_FIXED("sasyncperd2", R8A779F0_CLK_SASYNCPERD2, R8A779F0_CLK_SASYNCPERD1, 2, 1), + DEF_FIXED("sasyncperd4", R8A779F0_CLK_SASYNCPERD4, R8A779F0_CLK_SASYNCPERD1, 4, 1), + + DEF_GEN4_SDH("sdh0", R8A779F0_CLK_SD0H, CLK_SDSRC, 0x870), + DEF_GEN4_SD("sd0", R8A779F0_CLK_SD0, R8A779F0_CLK_SD0H, 0x870), DEF_BASE("rpc", R8A779F0_CLK_RPC, CLK_TYPE_GEN4_RPC, CLK_RPCSRC), DEF_BASE("rpcd2", R8A779F0_CLK_RPCD2, CLK_TYPE_GEN4_RPCD2, R8A779F0_CLK_RPC), @@ -130,6 +136,10 @@ static const struct mssr_mod_clk r8a779f0_mod_clks[] __initconst = { DEF_MOD("i2c3", 521, R8A779F0_CLK_S0D6_PER), DEF_MOD("i2c4", 522, R8A779F0_CLK_S0D6_PER), DEF_MOD("i2c5", 523, R8A779F0_CLK_S0D6_PER), + DEF_MOD("msiof0", 618, R8A779F0_CLK_MSO), + DEF_MOD("msiof1", 619, R8A779F0_CLK_MSO), + DEF_MOD("msiof2", 620, R8A779F0_CLK_MSO), + DEF_MOD("msiof3", 621, R8A779F0_CLK_MSO), DEF_MOD("pcie0", 624, R8A779F0_CLK_S0D2), DEF_MOD("pcie1", 625, R8A779F0_CLK_S0D2), DEF_MOD("scif0", 702, R8A779F0_CLK_S0D12_PER), @@ -139,7 +149,16 @@ static const struct mssr_mod_clk r8a779f0_mod_clks[] __initconst = { DEF_MOD("sdhi0", 706, R8A779F0_CLK_SD0), DEF_MOD("sys-dmac0", 709, R8A779F0_CLK_S0D3_PER), DEF_MOD("sys-dmac1", 710, R8A779F0_CLK_S0D3_PER), + DEF_MOD("tmu0", 713, R8A779F0_CLK_SASYNCRT), + DEF_MOD("tmu1", 714, R8A779F0_CLK_SASYNCPERD2), + DEF_MOD("tmu2", 715, R8A779F0_CLK_SASYNCPERD2), + DEF_MOD("tmu3", 716, R8A779F0_CLK_SASYNCPERD2), + DEF_MOD("tmu4", 717, R8A779F0_CLK_SASYNCPERD2), DEF_MOD("wdt", 907, R8A779F0_CLK_R), + DEF_MOD("cmt0", 910, R8A779F0_CLK_R), + DEF_MOD("cmt1", 911, R8A779F0_CLK_R), + DEF_MOD("cmt2", 912, R8A779F0_CLK_R), + DEF_MOD("cmt3", 913, R8A779F0_CLK_R), DEF_MOD("pfc0", 915, R8A779F0_CLK_CL16M), DEF_MOD("tsc", 919, R8A779F0_CLK_CL16M), DEF_MOD("ufs", 1514, R8A779F0_CLK_S0D4_HSC), diff --git a/drivers/clk/renesas/r8a779g0-cpg-mssr.c b/drivers/clk/renesas/r8a779g0-cpg-mssr.c index 3fc4233b1ead87c5c15244184ff797f6f3e8b1c5..9641122133b54f9aa0cf18adefdf3bfd25db98ea 100644 --- a/drivers/clk/renesas/r8a779g0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779g0-cpg-mssr.c @@ -150,10 +150,24 @@ static const struct cpg_core_clk r8a779g0_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a779g0_mod_clks[] __initconst = { + DEF_MOD("avb0", 211, R8A779G0_CLK_S0D4_HSC), + DEF_MOD("avb1", 212, R8A779G0_CLK_S0D4_HSC), + DEF_MOD("avb2", 213, R8A779G0_CLK_S0D4_HSC), DEF_MOD("hscif0", 514, R8A779G0_CLK_S0D3_PER), DEF_MOD("hscif1", 515, R8A779G0_CLK_S0D3_PER), DEF_MOD("hscif2", 516, R8A779G0_CLK_S0D3_PER), DEF_MOD("hscif3", 517, R8A779G0_CLK_S0D3_PER), + DEF_MOD("i2c0", 518, R8A779G0_CLK_S0D6_PER), + DEF_MOD("i2c1", 519, R8A779G0_CLK_S0D6_PER), + DEF_MOD("i2c2", 520, R8A779G0_CLK_S0D6_PER), + DEF_MOD("i2c3", 521, R8A779G0_CLK_S0D6_PER), + DEF_MOD("i2c4", 522, R8A779G0_CLK_S0D6_PER), + DEF_MOD("i2c5", 523, R8A779G0_CLK_S0D6_PER), + DEF_MOD("wdt1:wdt0", 907, R8A779G0_CLK_R), + DEF_MOD("pfc0", 915, R8A779G0_CLK_CL16M), + DEF_MOD("pfc1", 916, R8A779G0_CLK_CL16M), + DEF_MOD("pfc2", 917, R8A779G0_CLK_CL16M), + DEF_MOD("pfc3", 918, R8A779G0_CLK_CL16M), }; /* diff --git a/drivers/clk/renesas/r9a07g044-cpg.c b/drivers/clk/renesas/r9a07g044-cpg.c index fd7c4eecd39870138fca5209fdb7347dc7ef1ae2..02a4fc41bb6e1175b44fab19986356bb428879da 100644 --- a/drivers/clk/renesas/r9a07g044-cpg.c +++ b/drivers/clk/renesas/r9a07g044-cpg.c @@ -414,6 +414,7 @@ static const unsigned int r9a07g044_crit_mod_clks[] __initconst = { MOD_CLK_BASE + R9A07G044_DMAC_ACLK, }; +#ifdef CONFIG_CLK_R9A07G044 const struct rzg2l_cpg_info r9a07g044_cpg_info = { /* Core Clocks */ .core_clks = core_clks.common, @@ -436,6 +437,7 @@ const struct rzg2l_cpg_info r9a07g044_cpg_info = { .has_clk_mon_regs = true, }; +#endif #ifdef CONFIG_CLK_R9A07G054 const struct rzg2l_cpg_info r9a07g054_cpg_info = { diff --git a/drivers/clk/renesas/r9a09g011-cpg.c b/drivers/clk/renesas/r9a09g011-cpg.c index b21915cf6648733b77fb7711302fbc2c9a9ffad7..fbef1b35d2547d1507d57c03d91e1eee3251c0b9 100644 --- a/drivers/clk/renesas/r9a09g011-cpg.c +++ b/drivers/clk/renesas/r9a09g011-cpg.c @@ -132,6 +132,8 @@ static const struct rzg2l_mod_clk r9a09g011_mod_clks[] __initconst = { DEF_COUPLED("eth_chi", R9A09G011_ETH0_CLK_CHI, CLK_PLL2_100, 0x40c, 8), DEF_MOD("eth_clk_gptp", R9A09G011_ETH0_GPTP_EXT, CLK_PLL2_100, 0x40c, 9), DEF_MOD("syc_cnt_clk", R9A09G011_SYC_CNT_CLK, CLK_MAIN_24, 0x41c, 12), + DEF_MOD("iic_pclk0", R9A09G011_IIC_PCLK0, CLK_SEL_E, 0x420, 12), + DEF_MOD("iic_pclk1", R9A09G011_IIC_PCLK1, CLK_SEL_E, 0x424, 12), DEF_MOD("wdt0_pclk", R9A09G011_WDT0_PCLK, CLK_SEL_E, 0x428, 12), DEF_MOD("wdt0_clk", R9A09G011_WDT0_CLK, CLK_MAIN, 0x428, 13), DEF_MOD("urt_pclk", R9A09G011_URT_PCLK, CLK_SEL_E, 0x438, 4), @@ -143,6 +145,8 @@ static const struct rzg2l_reset r9a09g011_resets[] = { DEF_RST(R9A09G011_PFC_PRESETN, 0x600, 2), DEF_RST_MON(R9A09G011_ETH0_RST_HW_N, 0x608, 11, 11), DEF_RST_MON(R9A09G011_SYC_RST_N, 0x610, 9, 13), + DEF_RST(R9A09G011_IIC_GPA_PRESETN, 0x614, 8), + DEF_RST(R9A09G011_IIC_GPB_PRESETN, 0x614, 9), DEF_RST_MON(R9A09G011_WDT0_PRESETN, 0x614, 12, 19), }; diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig index 3067bdb6e1191684a74dc5abb8f9c7f688aa18a0..345a5d2a457c2d2578549593f8e1d81218bd07b7 100644 --- a/drivers/clk/rockchip/Kconfig +++ b/drivers/clk/rockchip/Kconfig @@ -23,6 +23,13 @@ config CLK_RV110X help Build the driver for RV110x Clock Driver. +config CLK_RV1126 + bool "Rockchip RV1126 clock controller support" + depends on ARM || COMPILE_TEST + default y + help + Build the driver for RV1126 Clock Driver. + config CLK_RK3036 bool "Rockchip RK3036 clock controller support" depends on ARM || COMPILE_TEST diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile index 2b78f124737263499852b36e492b7d59986abdf0..e8543876c05603109b37bb3c73a1590093dc422c 100644 --- a/drivers/clk/rockchip/Makefile +++ b/drivers/clk/rockchip/Makefile @@ -17,6 +17,7 @@ clk-rockchip-$(CONFIG_RESET_CONTROLLER) += softrst.o obj-$(CONFIG_CLK_PX30) += clk-px30.o obj-$(CONFIG_CLK_RV110X) += clk-rv1108.o +obj-$(CONFIG_CLK_RV1126) += clk-rv1126.o obj-$(CONFIG_CLK_RK3036) += clk-rk3036.o obj-$(CONFIG_CLK_RK312X) += clk-rk3128.o obj-$(CONFIG_CLK_RK3188) += clk-rk3188.o diff --git a/drivers/clk/rockchip/clk-rv1126.c b/drivers/clk/rockchip/clk-rv1126.c new file mode 100644 index 0000000000000000000000000000000000000000..c18790f5d05b6bd00d592d471a47736e99589863 --- /dev/null +++ b/drivers/clk/rockchip/clk-rv1126.c @@ -0,0 +1,1138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Rockchip Electronics Co. Ltd. + * Author: Finley Xiao + */ + +#include +#include +#include +#include +#include +#include +#include +#include "clk.h" + +#define RV1126_GMAC_CON 0x460 +#define RV1126_GRF_IOFUNC_CON1 0x10264 +#define RV1126_GRF_SOC_STATUS0 0x10 + +#define RV1126_FRAC_MAX_PRATE 1200000000 +#define RV1126_CSIOUT_FRAC_MAX_PRATE 300000000 + +enum rv1126_pmu_plls { + gpll, +}; + +enum rv1126_plls { + apll, dpll, cpll, hpll, +}; + +static struct rockchip_pll_rate_table rv1126_pll_rates[] = { + /* _mhz, _refdiv, _fbdiv, _postdiv1, _postdiv2, _dsmpd, _frac */ + RK3036_PLL_RATE(1608000000, 1, 67, 1, 1, 1, 0), + RK3036_PLL_RATE(1600000000, 3, 200, 1, 1, 1, 0), + RK3036_PLL_RATE(1584000000, 1, 132, 2, 1, 1, 0), + RK3036_PLL_RATE(1560000000, 1, 130, 2, 1, 1, 0), + RK3036_PLL_RATE(1536000000, 1, 128, 2, 1, 1, 0), + RK3036_PLL_RATE(1512000000, 1, 126, 2, 1, 1, 0), + RK3036_PLL_RATE(1488000000, 1, 124, 2, 1, 1, 0), + RK3036_PLL_RATE(1464000000, 1, 122, 2, 1, 1, 0), + RK3036_PLL_RATE(1440000000, 1, 120, 2, 1, 1, 0), + RK3036_PLL_RATE(1416000000, 1, 118, 2, 1, 1, 0), + RK3036_PLL_RATE(1400000000, 3, 350, 2, 1, 1, 0), + RK3036_PLL_RATE(1392000000, 1, 116, 2, 1, 1, 0), + RK3036_PLL_RATE(1368000000, 1, 114, 2, 1, 1, 0), + RK3036_PLL_RATE(1344000000, 1, 112, 2, 1, 1, 0), + RK3036_PLL_RATE(1320000000, 1, 110, 2, 1, 1, 0), + RK3036_PLL_RATE(1296000000, 1, 108, 2, 1, 1, 0), + RK3036_PLL_RATE(1272000000, 1, 106, 2, 1, 1, 0), + RK3036_PLL_RATE(1248000000, 1, 104, 2, 1, 1, 0), + RK3036_PLL_RATE(1200000000, 1, 100, 2, 1, 1, 0), + RK3036_PLL_RATE(1188000000, 1, 99, 2, 1, 1, 0), + RK3036_PLL_RATE(1104000000, 1, 92, 2, 1, 1, 0), + RK3036_PLL_RATE(1100000000, 3, 275, 2, 1, 1, 0), + RK3036_PLL_RATE(1008000000, 1, 84, 2, 1, 1, 0), + RK3036_PLL_RATE(1000000000, 3, 250, 2, 1, 1, 0), + RK3036_PLL_RATE(984000000, 1, 82, 2, 1, 1, 0), + RK3036_PLL_RATE(960000000, 1, 80, 2, 1, 1, 0), + RK3036_PLL_RATE(936000000, 1, 78, 2, 1, 1, 0), + RK3036_PLL_RATE(912000000, 1, 76, 2, 1, 1, 0), + RK3036_PLL_RATE(900000000, 1, 75, 2, 1, 1, 0), + RK3036_PLL_RATE(888000000, 1, 74, 2, 1, 1, 0), + RK3036_PLL_RATE(864000000, 1, 72, 2, 1, 1, 0), + RK3036_PLL_RATE(840000000, 1, 70, 2, 1, 1, 0), + RK3036_PLL_RATE(816000000, 1, 68, 2, 1, 1, 0), + RK3036_PLL_RATE(800000000, 3, 200, 2, 1, 1, 0), + RK3036_PLL_RATE(700000000, 3, 350, 4, 1, 1, 0), + RK3036_PLL_RATE(696000000, 1, 116, 4, 1, 1, 0), + RK3036_PLL_RATE(624000000, 1, 104, 4, 1, 1, 0), + RK3036_PLL_RATE(600000000, 1, 100, 4, 1, 1, 0), + RK3036_PLL_RATE(594000000, 1, 99, 4, 1, 1, 0), + RK3036_PLL_RATE(504000000, 1, 84, 4, 1, 1, 0), + RK3036_PLL_RATE(500000000, 1, 125, 6, 1, 1, 0), + RK3036_PLL_RATE(408000000, 1, 68, 2, 2, 1, 0), + RK3036_PLL_RATE(312000000, 1, 78, 6, 1, 1, 0), + RK3036_PLL_RATE(216000000, 1, 72, 4, 2, 1, 0), + RK3036_PLL_RATE(96000000, 1, 96, 6, 4, 1, 0), + { /* sentinel */ }, +}; + +#define RV1126_DIV_ACLK_CORE_MASK 0xf +#define RV1126_DIV_ACLK_CORE_SHIFT 4 +#define RV1126_DIV_PCLK_DBG_MASK 0x7 +#define RV1126_DIV_PCLK_DBG_SHIFT 0 + +#define RV1126_CLKSEL1(_aclk_core, _pclk_dbg) \ +{ \ + .reg = RV1126_CLKSEL_CON(1), \ + .val = HIWORD_UPDATE(_aclk_core, RV1126_DIV_ACLK_CORE_MASK, \ + RV1126_DIV_ACLK_CORE_SHIFT) | \ + HIWORD_UPDATE(_pclk_dbg, RV1126_DIV_PCLK_DBG_MASK, \ + RV1126_DIV_PCLK_DBG_SHIFT), \ +} + +#define RV1126_CPUCLK_RATE(_prate, _aclk_core, _pclk_dbg) \ +{ \ + .prate = _prate, \ + .divs = { \ + RV1126_CLKSEL1(_aclk_core, _pclk_dbg), \ + }, \ +} + +static struct rockchip_cpuclk_rate_table rv1126_cpuclk_rates[] __initdata = { + RV1126_CPUCLK_RATE(1608000000, 1, 7), + RV1126_CPUCLK_RATE(1584000000, 1, 7), + RV1126_CPUCLK_RATE(1560000000, 1, 7), + RV1126_CPUCLK_RATE(1536000000, 1, 7), + RV1126_CPUCLK_RATE(1512000000, 1, 7), + RV1126_CPUCLK_RATE(1488000000, 1, 5), + RV1126_CPUCLK_RATE(1464000000, 1, 5), + RV1126_CPUCLK_RATE(1440000000, 1, 5), + RV1126_CPUCLK_RATE(1416000000, 1, 5), + RV1126_CPUCLK_RATE(1392000000, 1, 5), + RV1126_CPUCLK_RATE(1368000000, 1, 5), + RV1126_CPUCLK_RATE(1344000000, 1, 5), + RV1126_CPUCLK_RATE(1320000000, 1, 5), + RV1126_CPUCLK_RATE(1296000000, 1, 5), + RV1126_CPUCLK_RATE(1272000000, 1, 5), + RV1126_CPUCLK_RATE(1248000000, 1, 5), + RV1126_CPUCLK_RATE(1224000000, 1, 5), + RV1126_CPUCLK_RATE(1200000000, 1, 5), + RV1126_CPUCLK_RATE(1104000000, 1, 5), + RV1126_CPUCLK_RATE(1008000000, 1, 5), + RV1126_CPUCLK_RATE(912000000, 1, 5), + RV1126_CPUCLK_RATE(816000000, 1, 3), + RV1126_CPUCLK_RATE(696000000, 1, 3), + RV1126_CPUCLK_RATE(600000000, 1, 3), + RV1126_CPUCLK_RATE(408000000, 1, 1), + RV1126_CPUCLK_RATE(312000000, 1, 1), + RV1126_CPUCLK_RATE(216000000, 1, 1), + RV1126_CPUCLK_RATE(96000000, 1, 1), +}; + +static const struct rockchip_cpuclk_reg_data rv1126_cpuclk_data = { + .core_reg[0] = RV1126_CLKSEL_CON(0), + .div_core_shift[0] = 0, + .div_core_mask[0] = 0x1f, + .num_cores = 1, + .mux_core_alt = 0, + .mux_core_main = 2, + .mux_core_shift = 6, + .mux_core_mask = 0x3, +}; + +PNAME(mux_pll_p) = { "xin24m" }; +PNAME(mux_rtc32k_p) = { "clk_pmupvtm_divout", "xin32k", "clk_osc0_div32k" }; +PNAME(mux_wifi_p) = { "clk_wifi_osc0", "clk_wifi_div" }; +PNAME(mux_gpll_usb480m_cpll_xin24m_p) = { "gpll", "usb480m", "cpll", "xin24m" }; +PNAME(mux_uart1_p) = { "sclk_uart1_div", "sclk_uart1_fracdiv", "xin24m" }; +PNAME(mux_xin24m_gpll_p) = { "xin24m", "gpll" }; +PNAME(mux_gpll_xin24m_p) = { "gpll", "xin24m" }; +PNAME(mux_xin24m_32k_p) = { "xin24m", "clk_rtc32k" }; +PNAME(mux_usbphy_otg_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_otg" }; +PNAME(mux_usbphy_host_ref_p) = { "clk_ref12m", "xin_osc0_div2_usbphyref_host" }; +PNAME(mux_mipidsiphy_ref_p) = { "clk_ref24m", "xin_osc0_mipiphyref" }; +PNAME(mux_usb480m_p) = { "xin24m", "usb480m_phy", "clk_rtc32k" }; +PNAME(mux_armclk_p) = { "gpll", "cpll", "apll" }; +PNAME(mux_gpll_cpll_dpll_p) = { "gpll", "cpll", "dummy_dpll" }; +PNAME(mux_gpll_cpll_p) = { "gpll", "cpll" }; +PNAME(mux_hclk_pclk_pdbus_p) = { "gpll", "dummy_cpll" }; +PNAME(mux_gpll_cpll_usb480m_xin24m_p) = { "gpll", "cpll", "usb480m", "xin24m" }; +PNAME(mux_uart0_p) = { "sclk_uart0_div", "sclk_uart0_frac", "xin24m" }; +PNAME(mux_uart2_p) = { "sclk_uart2_div", "sclk_uart2_frac", "xin24m" }; +PNAME(mux_uart3_p) = { "sclk_uart3_div", "sclk_uart3_frac", "xin24m" }; +PNAME(mux_uart4_p) = { "sclk_uart4_div", "sclk_uart4_frac", "xin24m" }; +PNAME(mux_uart5_p) = { "sclk_uart5_div", "sclk_uart5_frac", "xin24m" }; +PNAME(mux_cpll_gpll_p) = { "cpll", "gpll" }; +PNAME(mux_i2s0_tx_p) = { "mclk_i2s0_tx_div", "mclk_i2s0_tx_fracdiv", "i2s0_mclkin", "xin12m" }; +PNAME(mux_i2s0_rx_p) = { "mclk_i2s0_rx_div", "mclk_i2s0_rx_fracdiv", "i2s0_mclkin", "xin12m" }; +PNAME(mux_i2s0_tx_out2io_p) = { "mclk_i2s0_tx", "xin12m" }; +PNAME(mux_i2s0_rx_out2io_p) = { "mclk_i2s0_rx", "xin12m" }; +PNAME(mux_i2s1_p) = { "mclk_i2s1_div", "mclk_i2s1_fracdiv", "i2s1_mclkin", "xin12m" }; +PNAME(mux_i2s1_out2io_p) = { "mclk_i2s1", "xin12m" }; +PNAME(mux_i2s2_p) = { "mclk_i2s2_div", "mclk_i2s2_fracdiv", "i2s2_mclkin", "xin12m" }; +PNAME(mux_i2s2_out2io_p) = { "mclk_i2s2", "xin12m" }; +PNAME(mux_gpll_cpll_xin24m_p) = { "gpll", "cpll", "xin24m" }; +PNAME(mux_audpwm_p) = { "sclk_audpwm_div", "sclk_audpwm_fracdiv", "xin24m" }; +PNAME(mux_usb480m_gpll_p) = { "usb480m", "gpll" }; +PNAME(clk_gmac_src_m0_p) = { "clk_gmac_div", "clk_gmac_rgmii_m0" }; +PNAME(clk_gmac_src_m1_p) = { "clk_gmac_div", "clk_gmac_rgmii_m1" }; +PNAME(mux_clk_gmac_src_p) = { "clk_gmac_src_m0", "clk_gmac_src_m1" }; +PNAME(mux_rgmii_clk_p) = { "clk_gmac_tx_div50", "clk_gmac_tx_div5", "clk_gmac_tx_src", "clk_gmac_tx_src"}; +PNAME(mux_rmii_clk_p) = { "clk_gmac_rx_div20", "clk_gmac_rx_div2" }; +PNAME(mux_gmac_tx_rx_p) = { "rgmii_mode_clk", "rmii_mode_clk" }; +PNAME(mux_dpll_gpll_p) = { "dpll", "gpll" }; + +static u32 rgmii_mux_idx[] = { 2, 3, 0, 1 }; + +static struct rockchip_pll_clock rv1126_pmu_pll_clks[] __initdata = { + [gpll] = PLL(pll_rk3328, PLL_GPLL, "gpll", mux_pll_p, + 0, RV1126_PMU_PLL_CON(0), + RV1126_PMU_MODE, 0, 3, 0, rv1126_pll_rates), +}; + +static struct rockchip_pll_clock rv1126_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3328, PLL_APLL, "apll", mux_pll_p, + 0, RV1126_PLL_CON(0), + RV1126_MODE_CON, 0, 0, 0, rv1126_pll_rates), + [dpll] = PLL(pll_rk3328, PLL_DPLL, "dpll", mux_pll_p, + 0, RV1126_PLL_CON(8), + RV1126_MODE_CON, 2, 1, 0, NULL), + [cpll] = PLL(pll_rk3328, PLL_CPLL, "cpll", mux_pll_p, + 0, RV1126_PLL_CON(16), + RV1126_MODE_CON, 4, 2, 0, rv1126_pll_rates), + [hpll] = PLL(pll_rk3328, PLL_HPLL, "hpll", mux_pll_p, + 0, RV1126_PLL_CON(24), + RV1126_MODE_CON, 6, 4, 0, rv1126_pll_rates), +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +static struct rockchip_clk_branch rv1126_rtc32k_fracmux __initdata = + MUX(CLK_RTC32K, "clk_rtc32k", mux_rtc32k_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(0), 7, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart1_fracmux __initdata = + MUX(SCLK_UART1_MUX, "sclk_uart1_mux", mux_uart1_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(4), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart0_fracmux __initdata = + MUX(SCLK_UART0_MUX, "sclk_uart0_mux", mux_uart0_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(10), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart2_fracmux __initdata = + MUX(SCLK_UART2_MUX, "sclk_uart2_mux", mux_uart2_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(12), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart3_fracmux __initdata = + MUX(SCLK_UART3_MUX, "sclk_uart3_mux", mux_uart3_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(14), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart4_fracmux __initdata = + MUX(SCLK_UART4_MUX, "sclk_uart4_mux", mux_uart4_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(16), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_uart5_fracmux __initdata = + MUX(SCLK_UART5_MUX, "sclk_uart5_mux", mux_uart5_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(18), 10, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s0_tx_fracmux __initdata = + MUX(MCLK_I2S0_TX_MUX, "mclk_i2s0_tx_mux", mux_i2s0_tx_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 0, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s0_rx_fracmux __initdata = + MUX(MCLK_I2S0_RX_MUX, "mclk_i2s0_rx_mux", mux_i2s0_rx_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(30), 2, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s1_fracmux __initdata = + MUX(MCLK_I2S1_MUX, "mclk_i2s1_mux", mux_i2s1_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(31), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_i2s2_fracmux __initdata = + MUX(MCLK_I2S2_MUX, "mclk_i2s2_mux", mux_i2s2_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(33), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_audpwm_fracmux __initdata = + MUX(SCLK_AUDPWM_MUX, "mclk_audpwm_mux", mux_audpwm_p, CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(36), 8, 2, MFLAGS); + +static struct rockchip_clk_branch rv1126_clk_pmu_branches[] __initdata = { + /* + * Clock-Architecture Diagram 2 + */ + /* PD_PMU */ + COMPOSITE_NOMUX(PCLK_PDPMU, "pclk_pdpmu", "gpll", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKSEL_CON(1), 0, 5, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 0, GFLAGS), + + COMPOSITE_FRACMUX(CLK_OSC0_DIV32K, "clk_osc0_div32k", "xin24m", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKSEL_CON(13), 0, + RV1126_PMU_CLKGATE_CON(2), 9, GFLAGS, + &rv1126_rtc32k_fracmux), + + COMPOSITE_NOMUX(CLK_WIFI_DIV, "clk_wifi_div", "gpll", 0, + RV1126_PMU_CLKSEL_CON(12), 0, 6, DFLAGS, + RV1126_PMU_CLKGATE_CON(2), 10, GFLAGS), + GATE(CLK_WIFI_OSC0, "clk_wifi_osc0", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 11, GFLAGS), + MUX(CLK_WIFI, "clk_wifi", mux_wifi_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(12), 8, 1, MFLAGS), + + GATE(PCLK_PMU, "pclk_pmu", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 1, GFLAGS), + + GATE(PCLK_UART1, "pclk_uart1", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE(SCLK_UART1_DIV, "sclk_uart1_div", mux_gpll_usb480m_cpll_xin24m_p, 0, + RV1126_PMU_CLKSEL_CON(4), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 12, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART1_FRACDIV, "sclk_uart1_fracdiv", "sclk_uart1_div", + CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(5), 0, + RV1126_PMU_CLKGATE_CON(0), 13, GFLAGS, + &rv1126_uart1_fracmux), + GATE(SCLK_UART1, "sclk_uart1", "sclk_uart1_mux", 0, + RV1126_PMU_CLKGATE_CON(0), 14, GFLAGS), + + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 5, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C0, "clk_i2c0", "gpll", 0, + RV1126_PMU_CLKSEL_CON(2), 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 6, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C2, "clk_i2c2", "gpll", 0, + RV1126_PMU_CLKSEL_CON(3), 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(0), 10, GFLAGS), + + GATE(CLK_CAPTURE_PWM0, "clk_capture_pwm0", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 2, GFLAGS), + GATE(PCLK_PWM0, "pclk_pwm0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 0, GFLAGS), + COMPOSITE(CLK_PWM0, "clk_pwm0", mux_xin24m_gpll_p, 0, + RV1126_PMU_CLKSEL_CON(6), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 1, GFLAGS), + GATE(CLK_CAPTURE_PWM1, "clk_capture_pwm1", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 5, GFLAGS), + GATE(PCLK_PWM1, "pclk_pwm1", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 3, GFLAGS), + COMPOSITE(CLK_PWM1, "clk_pwm1", mux_xin24m_gpll_p, 0, + RV1126_PMU_CLKSEL_CON(6), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 4, GFLAGS), + + GATE(PCLK_SPI0, "pclk_spi0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 11, GFLAGS), + COMPOSITE(CLK_SPI0, "clk_spi0", mux_gpll_xin24m_p, 0, + RV1126_PMU_CLKSEL_CON(9), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 12, GFLAGS), + + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(1), 9, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO0, "dbclk_gpio0", mux_xin24m_32k_p, 0, + RV1126_PMU_CLKSEL_CON(8), 15, 1, MFLAGS, + RV1126_PMU_CLKGATE_CON(1), 10, GFLAGS), + + GATE(PCLK_PMUPVTM, "pclk_pmupvtm", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(2), 6, GFLAGS), + GATE(CLK_PMUPVTM, "clk_pmupvtm", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 5, GFLAGS), + GATE(CLK_CORE_PMUPVTM, "clk_core_pmupvtm", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(2), 7, GFLAGS), + + COMPOSITE_NOMUX(CLK_REF12M, "clk_ref12m", "gpll", 0, + RV1126_PMU_CLKSEL_CON(7), 8, 7, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 15, GFLAGS), + GATE(0, "xin_osc0_usbphyref_otg", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 6, GFLAGS), + GATE(0, "xin_osc0_usbphyref_host", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 7, GFLAGS), + FACTOR(0, "xin_osc0_div2_usbphyref_otg", "xin_osc0_usbphyref_otg", 0, 1, 2), + FACTOR(0, "xin_osc0_div2_usbphyref_host", "xin_osc0_usbphyref_host", 0, 1, 2), + MUX(CLK_USBPHY_OTG_REF, "clk_usbphy_otg_ref", mux_usbphy_otg_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 6, 1, MFLAGS), + MUX(CLK_USBPHY_HOST_REF, "clk_usbphy_host_ref", mux_usbphy_host_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 7, 1, MFLAGS), + + COMPOSITE_NOMUX(CLK_REF24M, "clk_ref24m", "gpll", 0, + RV1126_PMU_CLKSEL_CON(7), 0, 6, DFLAGS, + RV1126_PMU_CLKGATE_CON(1), 14, GFLAGS), + GATE(0, "xin_osc0_mipiphyref", "xin24m", 0, + RV1126_PMU_CLKGATE_CON(1), 8, GFLAGS), + MUX(CLK_MIPIDSIPHY_REF, "clk_mipidsiphy_ref", mux_mipidsiphy_ref_p, CLK_SET_RATE_PARENT, + RV1126_PMU_CLKSEL_CON(7), 15, 1, MFLAGS), + + GATE(CLK_PMU, "clk_pmu", "xin24m", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 15, GFLAGS), + + GATE(PCLK_PMUSGRF, "pclk_pmusgrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 4, GFLAGS), + GATE(PCLK_PMUGRF, "pclk_pmugrf", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(1), 13, GFLAGS), + GATE(PCLK_PMUCRU, "pclk_pmucru", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(2), 4, GFLAGS), + GATE(PCLK_CHIPVEROTP, "pclk_chipverotp", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(2), 0, GFLAGS), + GATE(PCLK_PDPMU_NIU, "pclk_pdpmu_niu", "pclk_pdpmu", CLK_IGNORE_UNUSED, + RV1126_PMU_CLKGATE_CON(0), 2, GFLAGS), + + GATE(PCLK_SCRKEYGEN, "pclk_scrkeygen", "pclk_pdpmu", 0, + RV1126_PMU_CLKGATE_CON(0), 7, GFLAGS), +}; + +static struct rockchip_clk_branch rv1126_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 1 + */ + MUX(USB480M, "usb480m", mux_usb480m_p, CLK_SET_RATE_PARENT, + RV1126_MODE_CON, 10, 2, MFLAGS), + FACTOR(0, "xin12m", "xin24m", 0, 1, 2), + + /* + * Clock-Architecture Diagram 3 + */ + /* PD_CORE */ + COMPOSITE_NOMUX(0, "pclk_dbg", "armclk", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(1), 0, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + RV1126_CLKGATE_CON(0), 6, GFLAGS), + GATE(CLK_CORE_CPUPVTM, "clk_core_cpupvtm", "armclk", 0, + RV1126_CLKGATE_CON(0), 12, GFLAGS), + GATE(PCLK_CPUPVTM, "pclk_cpupvtm", "pclk_dbg", 0, + RV1126_CLKGATE_CON(0), 10, GFLAGS), + GATE(CLK_CPUPVTM, "clk_cpupvtm", "xin24m", 0, + RV1126_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDCORE_NIU, "hclk_pdcore_niu", "gpll", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(0), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(0), 8, GFLAGS), + + /* + * Clock-Architecture Diagram 4 + */ + /* PD_BUS */ + COMPOSITE(0, "aclk_pdbus_pre", mux_gpll_cpll_dpll_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(2), 6, 2, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 0, GFLAGS), + GATE(ACLK_PDBUS, "aclk_pdbus", "aclk_pdbus_pre", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 11, GFLAGS), + COMPOSITE(0, "hclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(2), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 1, GFLAGS), + GATE(HCLK_PDBUS, "hclk_pdbus", "hclk_pdbus_pre", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 12, GFLAGS), + COMPOSITE(0, "pclk_pdbus_pre", mux_hclk_pclk_pdbus_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(3), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(2), 2, GFLAGS), + GATE(PCLK_PDBUS, "pclk_pdbus", "pclk_pdbus_pre", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 13, GFLAGS), + /* aclk_dmac is controlled by sgrf_clkgat_con. */ + SGRF_GATE(ACLK_DMAC, "aclk_dmac", "hclk_pdbus"), + GATE(ACLK_DCF, "aclk_dcf", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 6, GFLAGS), + GATE(PCLK_DCF, "pclk_dcf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 7, GFLAGS), + GATE(PCLK_WDT, "pclk_wdt", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 14, GFLAGS), + GATE(PCLK_MAILBOX, "pclk_mailbox", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 10, GFLAGS), + + COMPOSITE(CLK_SCR1, "clk_scr1", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(3), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(4), 7, GFLAGS), + GATE(0, "clk_scr1_niu", "clk_scr1", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 14, GFLAGS), + GATE(CLK_SCR1_CORE, "clk_scr1_core", "clk_scr1", 0, + RV1126_CLKGATE_CON(4), 8, GFLAGS), + GATE(CLK_SCR1_RTC, "clk_scr1_rtc", "xin24m", 0, + RV1126_CLKGATE_CON(4), 9, GFLAGS), + GATE(CLK_SCR1_JTAG, "clk_scr1_jtag", "clk_scr1_jtag_io", 0, + RV1126_CLKGATE_CON(4), 10, GFLAGS), + + GATE(PCLK_UART0, "pclk_uart0", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 0, GFLAGS), + COMPOSITE(SCLK_UART0_DIV, "sclk_uart0_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(10), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 1, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART0_FRAC, "sclk_uart0_frac", "sclk_uart0_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(11), 0, + RV1126_CLKGATE_CON(5), 2, GFLAGS, + &rv1126_uart0_fracmux), + GATE(SCLK_UART0, "sclk_uart0", "sclk_uart0_mux", 0, + RV1126_CLKGATE_CON(5), 3, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 4, GFLAGS), + COMPOSITE(SCLK_UART2_DIV, "sclk_uart2_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(12), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 5, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART2_FRAC, "sclk_uart2_frac", "sclk_uart2_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(13), 0, + RV1126_CLKGATE_CON(5), 6, GFLAGS, + &rv1126_uart2_fracmux), + GATE(SCLK_UART2, "sclk_uart2", "sclk_uart2_mux", 0, + RV1126_CLKGATE_CON(5), 7, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 8, GFLAGS), + COMPOSITE(SCLK_UART3_DIV, "sclk_uart3_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(14), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(5), 9, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART3_FRAC, "sclk_uart3_frac", "sclk_uart3_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(15), 0, + RV1126_CLKGATE_CON(5), 10, GFLAGS, + &rv1126_uart3_fracmux), + GATE(SCLK_UART3, "sclk_uart3", "sclk_uart3_mux", 0, + RV1126_CLKGATE_CON(5), 11, GFLAGS), + GATE(PCLK_UART4, "pclk_uart4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(5), 12, GFLAGS), + COMPOSITE(SCLK_UART4_DIV, "sclk_uart4_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(16), 8, 2, MFLAGS, 0, 7, + DFLAGS, RV1126_CLKGATE_CON(5), 13, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART4_FRAC, "sclk_uart4_frac", "sclk_uart4_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(17), 0, + RV1126_CLKGATE_CON(5), 14, GFLAGS, + &rv1126_uart4_fracmux), + GATE(SCLK_UART4, "sclk_uart4", "sclk_uart4_mux", 0, + RV1126_CLKGATE_CON(5), 15, GFLAGS), + GATE(PCLK_UART5, "pclk_uart5", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 0, GFLAGS), + COMPOSITE(SCLK_UART5_DIV, "sclk_uart5_div", mux_gpll_cpll_usb480m_xin24m_p, 0, + RV1126_CLKSEL_CON(18), 8, 2, MFLAGS, 0, 7, + DFLAGS, RV1126_CLKGATE_CON(6), 1, GFLAGS), + COMPOSITE_FRACMUX(SCLK_UART5_FRAC, "sclk_uart5_frac", "sclk_uart5_div", CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(19), 0, + RV1126_CLKGATE_CON(6), 2, GFLAGS, + &rv1126_uart5_fracmux), + GATE(SCLK_UART5, "sclk_uart5", "sclk_uart5_mux", 0, + RV1126_CLKGATE_CON(6), 3, GFLAGS), + + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 10, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C1, "clk_i2c1", "gpll", 0, + RV1126_CLKSEL_CON(5), 0, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 11, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 12, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C3, "clk_i2c3", "gpll", 0, + RV1126_CLKSEL_CON(5), 8, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 13, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(3), 14, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C4, "clk_i2c4", "gpll", 0, + RV1126_CLKSEL_CON(6), 0, 7, DFLAGS, + RV1126_CLKGATE_CON(3), 15, GFLAGS), + GATE(PCLK_I2C5, "pclk_i2c5", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 0, GFLAGS), + COMPOSITE_NOMUX(CLK_I2C5, "clk_i2c5", "gpll", 0, + RV1126_CLKSEL_CON(6), 8, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 1, GFLAGS), + + GATE(PCLK_SPI1, "pclk_spi1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 2, GFLAGS), + COMPOSITE(CLK_SPI1, "clk_spi1", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(8), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 3, GFLAGS), + + GATE(CLK_CAPTURE_PWM2, "clk_capture_pwm2", "xin24m", 0, + RV1126_CLKGATE_CON(4), 6, GFLAGS), + GATE(PCLK_PWM2, "pclk_pwm2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(4), 4, GFLAGS), + COMPOSITE(CLK_PWM2, "clk_pwm2", mux_xin24m_gpll_p, 0, + RV1126_CLKSEL_CON(9), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(4), 5, GFLAGS), + + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 0, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO1, "dbclk_gpio1", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(21), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 1, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 2, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO2, "dbclk_gpio2", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(22), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 3, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 4, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO3, "dbclk_gpio3", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(23), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 5, GFLAGS), + GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 6, GFLAGS), + COMPOSITE_NODIV(DBCLK_GPIO4, "dbclk_gpio4", mux_xin24m_32k_p, 0, + RV1126_CLKSEL_CON(24), 15, 1, MFLAGS, + RV1126_CLKGATE_CON(7), 7, GFLAGS), + + GATE(PCLK_SARADC, "pclk_saradc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 4, GFLAGS), + COMPOSITE_NOMUX(CLK_SARADC, "clk_saradc", "xin24m", 0, + RV1126_CLKSEL_CON(20), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(6), 5, GFLAGS), + + GATE(PCLK_TIMER, "pclk_timer", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 7, GFLAGS), + GATE(CLK_TIMER0, "clk_timer0", "xin24m", 0, + RV1126_CLKGATE_CON(6), 8, GFLAGS), + GATE(CLK_TIMER1, "clk_timer1", "xin24m", 0, + RV1126_CLKGATE_CON(6), 9, GFLAGS), + GATE(CLK_TIMER2, "clk_timer2", "xin24m", 0, + RV1126_CLKGATE_CON(6), 10, GFLAGS), + GATE(CLK_TIMER3, "clk_timer3", "xin24m", 0, + RV1126_CLKGATE_CON(6), 11, GFLAGS), + GATE(CLK_TIMER4, "clk_timer4", "xin24m", 0, + RV1126_CLKGATE_CON(6), 12, GFLAGS), + GATE(CLK_TIMER5, "clk_timer5", "xin24m", 0, + RV1126_CLKGATE_CON(6), 13, GFLAGS), + + GATE(ACLK_SPINLOCK, "aclk_spinlock", "hclk_pdbus", 0, + RV1126_CLKGATE_CON(6), 6, GFLAGS), + + GATE(ACLK_DECOM, "aclk_decom", "aclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 11, GFLAGS), + GATE(PCLK_DECOM, "pclk_decom", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 12, GFLAGS), + COMPOSITE(DCLK_DECOM, "dclk_decom", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(25), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(7), 13, GFLAGS), + + GATE(PCLK_CAN, "pclk_can", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(7), 8, GFLAGS), + COMPOSITE(CLK_CAN, "clk_can", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(7), 9, GFLAGS), + /* pclk_otp and clk_otp are controlled by sgrf_clkgat_con. */ + SGRF_GATE(CLK_OTP, "clk_otp", "xin24m"), + SGRF_GATE(PCLK_OTP, "pclk_otp", "pclk_pdbus"), + + GATE(PCLK_NPU_TSADC, "pclk_npu_tsadc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(24), 3, GFLAGS), + COMPOSITE_NOMUX(CLK_NPU_TSADC, "clk_npu_tsadc", "xin24m", 0, + RV1126_CLKSEL_CON(71), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(24), 4, GFLAGS), + GATE(CLK_NPU_TSADCPHY, "clk_npu_tsadcphy", "clk_npu_tsadc", 0, + RV1126_CLKGATE_CON(24), 5, GFLAGS), + GATE(PCLK_CPU_TSADC, "pclk_cpu_tsadc", "pclk_pdbus", 0, + RV1126_CLKGATE_CON(24), 0, GFLAGS), + COMPOSITE_NOMUX(CLK_CPU_TSADC, "clk_cpu_tsadc", "xin24m", 0, + RV1126_CLKSEL_CON(70), 0, 11, DFLAGS, + RV1126_CLKGATE_CON(24), 1, GFLAGS), + GATE(CLK_CPU_TSADCPHY, "clk_cpu_tsadcphy", "clk_cpu_tsadc", 0, + RV1126_CLKGATE_CON(24), 2, GFLAGS), + + /* + * Clock-Architecture Diagram 6 + */ + /* PD_AUDIO */ + COMPOSITE_NOMUX(HCLK_PDAUDIO, "hclk_pdaudio", "gpll", 0, + RV1126_CLKSEL_CON(26), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(9), 0, GFLAGS), + + GATE(HCLK_I2S0, "hclk_i2s0", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(9), 4, GFLAGS), + COMPOSITE(MCLK_I2S0_TX_DIV, "mclk_i2s0_tx_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(27), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(9), 5, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S0_TX_FRACDIV, "mclk_i2s0_tx_fracdiv", "mclk_i2s0_tx_div", + CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(28), 0, + RV1126_CLKGATE_CON(9), 6, GFLAGS, + &rv1126_i2s0_tx_fracmux), + GATE(MCLK_I2S0_TX, "mclk_i2s0_tx", "mclk_i2s0_tx_mux", 0, + RV1126_CLKGATE_CON(9), 9, GFLAGS), + COMPOSITE(MCLK_I2S0_RX_DIV, "mclk_i2s0_rx_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(27), 15, 1, MFLAGS, 8, 7, DFLAGS, + RV1126_CLKGATE_CON(9), 7, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S0_RX_FRACDIV, "mclk_i2s0_rx_fracdiv", "mclk_i2s0_rx_div", + CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(29), 0, + RV1126_CLKGATE_CON(9), 8, GFLAGS, + &rv1126_i2s0_rx_fracmux), + GATE(MCLK_I2S0_RX, "mclk_i2s0_rx", "mclk_i2s0_rx_mux", 0, + RV1126_CLKGATE_CON(9), 10, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S0_TX_OUT2IO, "mclk_i2s0_tx_out2io", mux_i2s0_tx_out2io_p, 0, + RV1126_CLKSEL_CON(30), 6, 1, MFLAGS, + RV1126_CLKGATE_CON(9), 13, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S0_RX_OUT2IO, "mclk_i2s0_rx_out2io", mux_i2s0_rx_out2io_p, 0, + RV1126_CLKSEL_CON(30), 8, 1, MFLAGS, + RV1126_CLKGATE_CON(9), 14, GFLAGS), + + GATE(HCLK_I2S1, "hclk_i2s1", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 0, GFLAGS), + COMPOSITE(MCLK_I2S1_DIV, "mclk_i2s1_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(31), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 1, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S1_FRACDIV, "mclk_i2s1_fracdiv", "mclk_i2s1_div", + CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(32), 0, + RV1126_CLKGATE_CON(10), 2, GFLAGS, + &rv1126_i2s1_fracmux), + GATE(MCLK_I2S1, "mclk_i2s1", "mclk_i2s1_mux", 0, + RV1126_CLKGATE_CON(10), 3, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S1_OUT2IO, "mclk_i2s1_out2io", mux_i2s1_out2io_p, 0, + RV1126_CLKSEL_CON(31), 12, 1, MFLAGS, + RV1126_CLKGATE_CON(10), 4, GFLAGS), + GATE(HCLK_I2S2, "hclk_i2s2", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 5, GFLAGS), + COMPOSITE(MCLK_I2S2_DIV, "mclk_i2s2_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(33), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 6, GFLAGS), + COMPOSITE_FRACMUX(MCLK_I2S2_FRACDIV, "mclk_i2s2_fracdiv", "mclk_i2s2_div", + CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(34), 0, + RV1126_CLKGATE_CON(10), 7, GFLAGS, + &rv1126_i2s2_fracmux), + GATE(MCLK_I2S2, "mclk_i2s2", "mclk_i2s2_mux", 0, + RV1126_CLKGATE_CON(10), 8, GFLAGS), + COMPOSITE_NODIV(MCLK_I2S2_OUT2IO, "mclk_i2s2_out2io", mux_i2s2_out2io_p, 0, + RV1126_CLKSEL_CON(33), 10, 1, MFLAGS, + RV1126_CLKGATE_CON(10), 9, GFLAGS), + + GATE(HCLK_PDM, "hclk_pdm", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 10, GFLAGS), + COMPOSITE(MCLK_PDM, "mclk_pdm", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(35), 8, 2, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 11, GFLAGS), + + GATE(HCLK_AUDPWM, "hclk_audpwm", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(10), 12, GFLAGS), + COMPOSITE(SCLK_ADUPWM_DIV, "sclk_audpwm_div", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(36), 7, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(10), 13, GFLAGS), + COMPOSITE_FRACMUX(SCLK_AUDPWM_FRACDIV, "sclk_audpwm_fracdiv", "sclk_audpwm_div", + CLK_SET_RATE_PARENT, + RV1126_CLKSEL_CON(37), 0, + RV1126_CLKGATE_CON(10), 14, GFLAGS, + &rv1126_audpwm_fracmux), + GATE(SCLK_AUDPWM, "sclk_audpwm", "mclk_audpwm_mux", 0, + RV1126_CLKGATE_CON(10), 15, GFLAGS), + + GATE(PCLK_ACDCDIG, "pclk_acdcdig", "hclk_pdaudio", 0, + RV1126_CLKGATE_CON(11), 0, GFLAGS), + GATE(CLK_ACDCDIG_ADC, "clk_acdcdig_adc", "mclk_i2s0_rx", 0, + RV1126_CLKGATE_CON(11), 2, GFLAGS), + GATE(CLK_ACDCDIG_DAC, "clk_acdcdig_dac", "mclk_i2s0_tx", 0, + RV1126_CLKGATE_CON(11), 3, GFLAGS), + COMPOSITE(CLK_ACDCDIG_I2C, "clk_acdcdig_i2c", mux_gpll_xin24m_p, 0, + RV1126_CLKSEL_CON(72), 8, 1, MFLAGS, 0, 7, DFLAGS, + RV1126_CLKGATE_CON(11), 1, GFLAGS), + + /* + * Clock-Architecture Diagram 12 + */ + /* PD_PHP */ + COMPOSITE(ACLK_PDPHP, "aclk_pdphp", mux_gpll_cpll_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(53), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(17), 0, GFLAGS), + COMPOSITE_NOMUX(HCLK_PDPHP, "hclk_pdphp", "gpll", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(53), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(17), 1, GFLAGS), + /* PD_SDCARD */ + GATE(HCLK_PDSDMMC, "hclk_pdsdmmc", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(17), 6, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_pdsdmmc", 0, + RV1126_CLKGATE_CON(18), 4, GFLAGS), + COMPOSITE(CLK_SDMMC, "clk_sdmmc", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(55), 14, 2, MFLAGS, 0, 8, + DFLAGS, RV1126_CLKGATE_CON(18), 5, GFLAGS), + MMC(SCLK_SDMMC_DRV, "sdmmc_drv", "clk_sdmmc", RV1126_SDMMC_CON0, 1), + MMC(SCLK_SDMMC_SAMPLE, "sdmmc_sample", "clk_sdmmc", RV1126_SDMMC_CON1, 1), + + /* PD_SDIO */ + GATE(HCLK_PDSDIO, "hclk_pdsdio", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(17), 8, GFLAGS), + GATE(HCLK_SDIO, "hclk_sdio", "hclk_pdsdio", 0, + RV1126_CLKGATE_CON(18), 6, GFLAGS), + COMPOSITE(CLK_SDIO, "clk_sdio", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(56), 14, 2, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 7, GFLAGS), + MMC(SCLK_SDIO_DRV, "sdio_drv", "clk_sdio", RV1126_SDIO_CON0, 1), + MMC(SCLK_SDIO_SAMPLE, "sdio_sample", "clk_sdio", RV1126_SDIO_CON1, 1), + + /* PD_NVM */ + GATE(HCLK_PDNVM, "hclk_pdnvm", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(18), 1, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 8, GFLAGS), + COMPOSITE(CLK_EMMC, "clk_emmc", mux_gpll_cpll_xin24m_p, 0, + RV1126_CLKSEL_CON(57), 14, 2, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 9, GFLAGS), + GATE(HCLK_NANDC, "hclk_nandc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 13, GFLAGS), + COMPOSITE(CLK_NANDC, "clk_nandc", mux_gpll_cpll_p, 0, + RV1126_CLKSEL_CON(59), 15, 1, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 14, GFLAGS), + GATE(HCLK_SFC, "hclk_sfc", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 10, GFLAGS), + GATE(HCLK_SFCXIP, "hclk_sfcxip", "hclk_pdnvm", 0, + RV1126_CLKGATE_CON(18), 11, GFLAGS), + COMPOSITE(SCLK_SFC, "sclk_sfc", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(58), 15, 1, MFLAGS, 0, 8, DFLAGS, + RV1126_CLKGATE_CON(18), 12, GFLAGS), + MMC(SCLK_EMMC_DRV, "emmc_drv", "clk_emmc", RV1126_EMMC_CON0, 1), + MMC(SCLK_EMMC_SAMPLE, "emmc_sample", "clk_emmc", RV1126_EMMC_CON1, 1), + + /* PD_USB */ + GATE(ACLK_PDUSB, "aclk_pdusb", "aclk_pdphp", 0, + RV1126_CLKGATE_CON(19), 0, GFLAGS), + GATE(HCLK_PDUSB, "hclk_pdusb", "hclk_pdphp", 0, + RV1126_CLKGATE_CON(19), 1, GFLAGS), + GATE(HCLK_USBHOST, "hclk_usbhost", "hclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 4, GFLAGS), + GATE(HCLK_USBHOST_ARB, "hclk_usbhost_arb", "hclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 5, GFLAGS), + COMPOSITE(CLK_USBHOST_UTMI_OHCI, "clk_usbhost_utmi_ohci", mux_usb480m_gpll_p, 0, + RV1126_CLKSEL_CON(61), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(19), 6, GFLAGS), + GATE(ACLK_USBOTG, "aclk_usbotg", "aclk_pdusb", 0, + RV1126_CLKGATE_CON(19), 7, GFLAGS), + GATE(CLK_USBOTG_REF, "clk_usbotg_ref", "xin24m", 0, + RV1126_CLKGATE_CON(19), 8, GFLAGS), + /* PD_GMAC */ + GATE(ACLK_PDGMAC, "aclk_pdgmac", "aclk_pdphp", 0, + RV1126_CLKGATE_CON(20), 0, GFLAGS), + COMPOSITE_NOMUX(PCLK_PDGMAC, "pclk_pdgmac", "aclk_pdgmac", 0, + RV1126_CLKSEL_CON(63), 8, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 1, GFLAGS), + GATE(ACLK_GMAC, "aclk_gmac", "aclk_pdgmac", 0, + RV1126_CLKGATE_CON(20), 4, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_pdgmac", 0, + RV1126_CLKGATE_CON(20), 5, GFLAGS), + + COMPOSITE(CLK_GMAC_DIV, "clk_gmac_div", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(63), 7, 1, MFLAGS, 0, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 6, GFLAGS), + GATE(CLK_GMAC_RGMII_M0, "clk_gmac_rgmii_m0", "clk_gmac_rgmii_clkin_m0", 0, + RV1126_CLKGATE_CON(20), 12, GFLAGS), + MUX(CLK_GMAC_SRC_M0, "clk_gmac_src_m0", clk_gmac_src_m0_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 0, 1, MFLAGS), + GATE(CLK_GMAC_RGMII_M1, "clk_gmac_rgmii_m1", "clk_gmac_rgmii_clkin_m1", 0, + RV1126_CLKGATE_CON(20), 13, GFLAGS), + MUX(CLK_GMAC_SRC_M1, "clk_gmac_src_m1", clk_gmac_src_m1_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 5, 1, MFLAGS), + MUXGRF(CLK_GMAC_SRC, "clk_gmac_src", mux_clk_gmac_src_p, CLK_SET_RATE_PARENT | + CLK_SET_RATE_NO_REPARENT, + RV1126_GRF_IOFUNC_CON1, 12, 1, MFLAGS), + + GATE(CLK_GMAC_REF, "clk_gmac_ref", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 7, GFLAGS), + + GATE(CLK_GMAC_TX_SRC, "clk_gmac_tx_src", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 9, GFLAGS), + FACTOR(CLK_GMAC_TX_DIV5, "clk_gmac_tx_div5", "clk_gmac_tx_src", 0, 1, 5), + FACTOR(CLK_GMAC_TX_DIV50, "clk_gmac_tx_div50", "clk_gmac_tx_src", 0, 1, 50), + MUXTBL(RGMII_MODE_CLK, "rgmii_mode_clk", mux_rgmii_clk_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 2, 2, MFLAGS, rgmii_mux_idx), + GATE(CLK_GMAC_RX_SRC, "clk_gmac_rx_src", "clk_gmac_src", 0, + RV1126_CLKGATE_CON(20), 8, GFLAGS), + FACTOR(CLK_GMAC_RX_DIV2, "clk_gmac_rx_div2", "clk_gmac_rx_src", 0, 1, 2), + FACTOR(CLK_GMAC_RX_DIV20, "clk_gmac_rx_div20", "clk_gmac_rx_src", 0, 1, 20), + MUX(RMII_MODE_CLK, "rmii_mode_clk", mux_rmii_clk_p, CLK_SET_RATE_PARENT, + RV1126_GMAC_CON, 1, 1, MFLAGS), + MUX(CLK_GMAC_TX_RX, "clk_gmac_tx_rx", mux_gmac_tx_rx_p, CLK_SET_RATE_PARENT | + CLK_SET_RATE_NO_REPARENT, + RV1126_GMAC_CON, 4, 1, MFLAGS), + + GATE(CLK_GMAC_PTPREF, "clk_gmac_ptpref", "xin24m", 0, + RV1126_CLKGATE_CON(20), 10, GFLAGS), + COMPOSITE(CLK_GMAC_ETHERNET_OUT, "clk_gmac_ethernet_out2io", mux_cpll_gpll_p, 0, + RV1126_CLKSEL_CON(61), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(20), 11, GFLAGS), + + /* + * Clock-Architecture Diagram 15 + */ + GATE(PCLK_PDTOP, "pclk_pdtop", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 8, GFLAGS), + GATE(PCLK_DSIPHY, "pclk_dsiphy", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 4, GFLAGS), + GATE(PCLK_CSIPHY0, "pclk_csiphy0", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 2, GFLAGS), + GATE(PCLK_CSIPHY1, "pclk_csiphy1", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(23), 3, GFLAGS), + GATE(PCLK_USBPHY_HOST, "pclk_usbphy_host", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(19), 13, GFLAGS), + GATE(PCLK_USBPHY_OTG, "pclk_usbphy_otg", "pclk_pdtop", 0, + RV1126_CLKGATE_CON(19), 12, GFLAGS), + + /* + * Clock-Architecture Diagram 3 + */ + /* PD_CORE */ + COMPOSITE_NOMUX(0, "aclk_core", "armclk", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(1), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY, + RV1126_CLKGATE_CON(0), 2, GFLAGS), + GATE(0, "pclk_dbg_daplite", "pclk_dbg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 5, GFLAGS), + GATE(0, "clk_a7_jtag", "clk_jtag_ori", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 9, GFLAGS), + GATE(0, "aclk_core_niu", "aclk_core", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 3, GFLAGS), + GATE(0, "pclk_dbg_niu", "pclk_dbg", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(0), 4, GFLAGS), + /* + * Clock-Architecture Diagram 4 + */ + /* PD_BUS */ + GATE(0, "aclk_pdbus_hold_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 10, GFLAGS), + GATE(0, "aclk_pdbus_niu1", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 3, GFLAGS), + GATE(0, "hclk_pdbus_niu1", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 4, GFLAGS), + GATE(0, "pclk_pdbus_niu1", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 5, GFLAGS), + GATE(0, "aclk_pdbus_niu2", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 6, GFLAGS), + GATE(0, "hclk_pdbus_niu2", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 7, GFLAGS), + GATE(0, "aclk_pdbus_niu3", "aclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 8, GFLAGS), + GATE(0, "hclk_pdbus_niu3", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(2), 9, GFLAGS), + GATE(0, "pclk_grf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(6), 15, GFLAGS), + GATE(0, "pclk_sgrf", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(8), 4, GFLAGS), + GATE(0, "aclk_sysram", "hclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(3), 9, GFLAGS), + GATE(0, "pclk_intmux", "pclk_pdbus", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(7), 14, GFLAGS), + + /* + * Clock-Architecture Diagram 6 + */ + /* PD_AUDIO */ + GATE(0, "hclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(9), 2, GFLAGS), + GATE(0, "pclk_pdaudio_niu", "hclk_pdaudio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(9), 3, GFLAGS), + + /* + * Clock-Architecture Diagram 12 + */ + /* PD_PHP */ + GATE(0, "aclk_pdphpmid", "aclk_pdphp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 2, GFLAGS), + GATE(0, "hclk_pdphpmid", "hclk_pdphp", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 3, GFLAGS), + GATE(0, "aclk_pdphpmid_niu", "aclk_pdphpmid", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 4, GFLAGS), + GATE(0, "hclk_pdphpmid_niu", "hclk_pdphpmid", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 5, GFLAGS), + + /* PD_SDCARD */ + GATE(0, "hclk_pdsdmmc_niu", "hclk_pdsdmmc", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 7, GFLAGS), + + /* PD_SDIO */ + GATE(0, "hclk_pdsdio_niu", "hclk_pdsdio", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(17), 9, GFLAGS), + + /* PD_NVM */ + GATE(0, "hclk_pdnvm_niu", "hclk_pdnvm", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(18), 3, GFLAGS), + + /* PD_USB */ + GATE(0, "aclk_pdusb_niu", "aclk_pdusb", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(19), 2, GFLAGS), + GATE(0, "hclk_pdusb_niu", "hclk_pdusb", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(19), 3, GFLAGS), + + /* PD_GMAC */ + GATE(0, "aclk_pdgmac_niu", "aclk_pdgmac", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 2, GFLAGS), + GATE(0, "pclk_pdgmac_niu", "pclk_pdgmac", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 3, GFLAGS), + + /* + * Clock-Architecture Diagram 13 + */ + /* PD_DDR */ + COMPOSITE_NOMUX(0, "pclk_pdddr_pre", "gpll", CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(64), 0, 5, DFLAGS, + RV1126_CLKGATE_CON(21), 0, GFLAGS), + GATE(PCLK_PDDDR, "pclk_pdddr", "pclk_pdddr_pre", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 15, GFLAGS), + GATE(0, "pclk_ddr_msch", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 6, GFLAGS), + COMPOSITE_NOGATE(SCLK_DDRCLK, "sclk_ddrc", mux_dpll_gpll_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS | + CLK_DIVIDER_POWER_OF_TWO), + COMPOSITE(CLK_DDRPHY, "clk_ddrphy", mux_dpll_gpll_p, CLK_IGNORE_UNUSED, + RV1126_CLKSEL_CON(64), 15, 1, MFLAGS, 8, 5, DFLAGS, + RV1126_CLKGATE_CON(21), 8, GFLAGS), + GATE(0, "clk1x_phy", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 1, GFLAGS), + GATE(0, "clk_ddr_msch", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 10, GFLAGS), + GATE(0, "pclk_ddr_dfictl", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 2, GFLAGS), + GATE(0, "clk_ddr_dfictl", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 13, GFLAGS), + GATE(0, "pclk_ddr_standby", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 4, GFLAGS), + GATE(0, "clk_ddr_standby", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 14, GFLAGS), + GATE(0, "aclk_ddr_split", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 9, GFLAGS), + GATE(0, "pclk_ddr_grf", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 5, GFLAGS), + GATE(PCLK_DDR_MON, "pclk_ddr_mon", "pclk_pdddr", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 3, GFLAGS), + GATE(CLK_DDR_MON, "clk_ddr_mon", "clk_ddrphy", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(20), 15, GFLAGS), + GATE(TMCLK_DDR_MON, "tmclk_ddr_mon", "xin24m", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(21), 7, GFLAGS), + + /* + * Clock-Architecture Diagram 15 + */ + GATE(0, "pclk_topniu", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 9, GFLAGS), + GATE(PCLK_TOPCRU, "pclk_topcru", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 10, GFLAGS), + GATE(PCLK_TOPGRF, "pclk_topgrf", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 11, GFLAGS), + GATE(PCLK_CPUEMADET, "pclk_cpuemadet", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 12, GFLAGS), + GATE(PCLK_DDRPHY, "pclk_ddrphy", "pclk_pdtop", CLK_IGNORE_UNUSED, + RV1126_CLKGATE_CON(23), 0, GFLAGS), +}; + +static const char *const rv1126_cru_critical_clocks[] __initconst = { + "gpll", + "cpll", + "hpll", + "armclk", + "pclk_dbg", + "pclk_pdpmu", + "aclk_pdbus", + "hclk_pdbus", + "pclk_pdbus", + "aclk_pdphp", + "hclk_pdphp", + "clk_ddrphy", + "pclk_pdddr", + "pclk_pdtop", + "clk_usbhost_utmi_ohci", + "aclk_pdjpeg_niu", + "hclk_pdjpeg_niu", + "aclk_pdvdec_niu", + "hclk_pdvdec_niu", +}; + +static void __init rv1126_pmu_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru pmu region\n", __func__); + return; + } + + ctx = rockchip_clk_init(np, reg_base, CLKPMU_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip pmu clk init failed\n", __func__); + return; + } + + rockchip_clk_register_plls(ctx, rv1126_pmu_pll_clks, + ARRAY_SIZE(rv1126_pmu_pll_clks), + RV1126_GRF_SOC_STATUS0); + + rockchip_clk_register_branches(ctx, rv1126_clk_pmu_branches, + ARRAY_SIZE(rv1126_clk_pmu_branches)); + + rockchip_register_softrst(np, 2, reg_base + RV1126_PMU_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_clk_of_add_provider(np, ctx); +} + +static void __init rv1126_clk_init(struct device_node *np) +{ + struct rockchip_clk_provider *ctx; + void __iomem *reg_base; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + ctx = rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + if (IS_ERR(ctx)) { + pr_err("%s: rockchip clk init failed\n", __func__); + iounmap(reg_base); + return; + } + + rockchip_clk_register_plls(ctx, rv1126_pll_clks, + ARRAY_SIZE(rv1126_pll_clks), + RV1126_GRF_SOC_STATUS0); + + rockchip_clk_register_armclk(ctx, ARMCLK, "armclk", + mux_armclk_p, ARRAY_SIZE(mux_armclk_p), + &rv1126_cpuclk_data, rv1126_cpuclk_rates, + ARRAY_SIZE(rv1126_cpuclk_rates)); + + rockchip_clk_register_branches(ctx, rv1126_clk_branches, + ARRAY_SIZE(rv1126_clk_branches)); + + rockchip_register_softrst(np, 15, reg_base + RV1126_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); + + rockchip_register_restart_notifier(ctx, RV1126_GLB_SRST_FST, NULL); + + rockchip_clk_protect_critical(rv1126_cru_critical_clocks, + ARRAY_SIZE(rv1126_cru_critical_clocks)); + + rockchip_clk_of_add_provider(np, ctx); +} + +struct clk_rv1126_inits { + void (*inits)(struct device_node *np); +}; + +static const struct clk_rv1126_inits clk_rv1126_pmucru_init = { + .inits = rv1126_pmu_clk_init, +}; + +static const struct clk_rv1126_inits clk_rv1126_cru_init = { + .inits = rv1126_clk_init, +}; + +static const struct of_device_id clk_rv1126_match_table[] = { + { + .compatible = "rockchip,rv1126-cru", + .data = &clk_rv1126_cru_init, + }, { + .compatible = "rockchip,rv1126-pmucru", + .data = &clk_rv1126_pmucru_init, + }, + { } +}; + +static int __init clk_rv1126_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct clk_rv1126_inits *init_data; + + init_data = (struct clk_rv1126_inits *)of_device_get_match_data(&pdev->dev); + if (!init_data) + return -EINVAL; + + if (init_data->inits) + init_data->inits(np); + + return 0; +} + +static struct platform_driver clk_rv1126_driver = { + .driver = { + .name = "clk-rv1126", + .of_match_table = clk_rv1126_match_table, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver_probe(clk_rv1126_driver, clk_rv1126_probe); diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index bb8a844309bf5467b8f2935bd53a3482d9751b2c..e63d4f20b479d2d70a8148cac716dce2abbfdf73 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -40,6 +40,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, const char *const *parent_names, u8 num_parents, void __iomem *base, int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, + u32 *mux_table, int div_offset, u8 div_shift, u8 div_width, u8 div_flags, struct clk_div_table *div_table, int gate_offset, u8 gate_shift, u8 gate_flags, unsigned long flags, @@ -62,6 +63,7 @@ static struct clk *rockchip_clk_register_branch(const char *name, mux->shift = mux_shift; mux->mask = BIT(mux_width) - 1; mux->flags = mux_flags; + mux->table = mux_table; mux->lock = lock; mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops : &clk_mux_ops; @@ -270,6 +272,8 @@ static struct clk *rockchip_clk_register_frac_branch( frac_mux->shift = child->mux_shift; frac_mux->mask = BIT(child->mux_width) - 1; frac_mux->flags = child->mux_flags; + if (child->mux_table) + frac_mux->table = child->mux_table; frac_mux->lock = lock; frac_mux->hw.init = &init; @@ -444,11 +448,21 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, /* catch simple muxes */ switch (list->branch_type) { case branch_mux: - clk = clk_register_mux(NULL, list->name, - list->parent_names, list->num_parents, - flags, ctx->reg_base + list->muxdiv_offset, - list->mux_shift, list->mux_width, - list->mux_flags, &ctx->lock); + if (list->mux_table) + clk = clk_register_mux_table(NULL, list->name, + list->parent_names, list->num_parents, + flags, + ctx->reg_base + list->muxdiv_offset, + list->mux_shift, list->mux_width, + list->mux_flags, list->mux_table, + &ctx->lock); + else + clk = clk_register_mux(NULL, list->name, + list->parent_names, list->num_parents, + flags, + ctx->reg_base + list->muxdiv_offset, + list->mux_shift, list->mux_width, + list->mux_flags, &ctx->lock); break; case branch_muxgrf: clk = rockchip_clk_register_muxgrf(list->name, @@ -506,7 +520,8 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx, ctx->reg_base, list->muxdiv_offset, list->mux_shift, list->mux_width, list->mux_flags, - list->div_offset, list->div_shift, list->div_width, + list->mux_table, list->div_offset, + list->div_shift, list->div_width, list->div_flags, list->div_table, list->gate_offset, list->gate_shift, list->gate_flags, flags, &ctx->lock); diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h index 7aa45cc70287ad1ec91c38aeca26981ea029b246..ee01739e4a7cb540dc69ca15a6ed40da6d0cf7ef 100644 --- a/drivers/clk/rockchip/clk.h +++ b/drivers/clk/rockchip/clk.h @@ -79,6 +79,25 @@ struct clk; #define RV1108_EMMC_CON0 0x1e8 #define RV1108_EMMC_CON1 0x1ec +#define RV1126_PMU_MODE 0x0 +#define RV1126_PMU_PLL_CON(x) ((x) * 0x4 + 0x10) +#define RV1126_PMU_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RV1126_PMU_CLKGATE_CON(x) ((x) * 0x4 + 0x180) +#define RV1126_PMU_SOFTRST_CON(x) ((x) * 0x4 + 0x200) +#define RV1126_PLL_CON(x) ((x) * 0x4) +#define RV1126_MODE_CON 0x90 +#define RV1126_CLKSEL_CON(x) ((x) * 0x4 + 0x100) +#define RV1126_CLKGATE_CON(x) ((x) * 0x4 + 0x280) +#define RV1126_SOFTRST_CON(x) ((x) * 0x4 + 0x300) +#define RV1126_GLB_SRST_FST 0x408 +#define RV1126_GLB_SRST_SND 0x40c +#define RV1126_SDMMC_CON0 0x440 +#define RV1126_SDMMC_CON1 0x444 +#define RV1126_SDIO_CON0 0x448 +#define RV1126_SDIO_CON1 0x44c +#define RV1126_EMMC_CON0 0x450 +#define RV1126_EMMC_CON1 0x454 + #define RK2928_PLL_CON(x) ((x) * 0x4) #define RK2928_MODE_CON 0x40 #define RK2928_CLKSEL_CON(x) ((x) * 0x4 + 0x44) @@ -448,6 +467,7 @@ struct rockchip_clk_branch { u8 mux_shift; u8 mux_width; u8 mux_flags; + u32 *mux_table; int div_offset; u8 div_shift; u8 div_width; @@ -680,6 +700,22 @@ struct rockchip_clk_branch { .gate_offset = -1, \ } +#define MUXTBL(_id, cname, pnames, f, o, s, w, mf, mt) \ + { \ + .id = _id, \ + .branch_type = branch_mux, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = o, \ + .mux_shift = s, \ + .mux_width = w, \ + .mux_flags = mf, \ + .gate_offset = -1, \ + .mux_table = mt, \ + } + #define MUXGRF(_id, cname, pnames, f, o, s, w, mf) \ { \ .id = _id, \ diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c index e6d6cbf8c4e61e5d7d1c7edc653d27a0a995ed78..273f77d54dab9eb57296f3f22954785fe180ec55 100644 --- a/drivers/clk/samsung/clk-exynos-clkout.c +++ b/drivers/clk/samsung/clk-exynos-clkout.c @@ -81,19 +81,17 @@ MODULE_DEVICE_TABLE(of, exynos_clkout_ids); static int exynos_clkout_match_parent_dev(struct device *dev, u32 *mux_mask) { const struct exynos_clkout_variant *variant; - const struct of_device_id *match; if (!dev->parent) { dev_err(dev, "not instantiated from MFD\n"); return -EINVAL; } - match = of_match_device(exynos_clkout_ids, dev->parent); - if (!match) { + variant = of_device_get_match_data(dev->parent); + if (!variant) { dev_err(dev, "cannot match parent device\n"); return -EINVAL; } - variant = match->data; *mux_mask = variant->mux_mask; diff --git a/drivers/clk/samsung/clk-exynos7885.c b/drivers/clk/samsung/clk-exynos7885.c index a7b106302706781f2806b2e4f6a0a2225dc5d853..62ce6814f1411f70f2e3510ad9a037459a3619c5 100644 --- a/drivers/clk/samsung/clk-exynos7885.c +++ b/drivers/clk/samsung/clk-exynos7885.c @@ -27,6 +27,11 @@ #define CLK_CON_MUX_MUX_CLKCMU_CORE_BUS 0x1014 #define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 #define CLK_CON_MUX_MUX_CLKCMU_CORE_G3D 0x101c +#define CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS 0x1028 +#define CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD 0x102c +#define CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD 0x1030 +#define CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_SDIO 0x1034 +#define CLK_CON_MUX_MUX_CLKCMU_FSYS_USB30DRD 0x1038 #define CLK_CON_MUX_MUX_CLKCMU_PERI_BUS 0x1058 #define CLK_CON_MUX_MUX_CLKCMU_PERI_SPI0 0x105c #define CLK_CON_MUX_MUX_CLKCMU_PERI_SPI1 0x1060 @@ -39,6 +44,11 @@ #define CLK_CON_DIV_CLKCMU_CORE_BUS 0x181c #define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1820 #define CLK_CON_DIV_CLKCMU_CORE_G3D 0x1824 +#define CLK_CON_DIV_CLKCMU_FSYS_BUS 0x1844 +#define CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD 0x1848 +#define CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD 0x184c +#define CLK_CON_DIV_CLKCMU_FSYS_MMC_SDIO 0x1850 +#define CLK_CON_DIV_CLKCMU_FSYS_USB30DRD 0x1854 #define CLK_CON_DIV_CLKCMU_PERI_BUS 0x1874 #define CLK_CON_DIV_CLKCMU_PERI_SPI0 0x1878 #define CLK_CON_DIV_CLKCMU_PERI_SPI1 0x187c @@ -59,6 +69,11 @@ #define CLK_CON_GAT_GATE_CLKCMU_CORE_BUS 0x201c #define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 #define CLK_CON_GAT_GATE_CLKCMU_CORE_G3D 0x2024 +#define CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS 0x2044 +#define CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD 0x2048 +#define CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD 0x204c +#define CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_SDIO 0x2050 +#define CLK_CON_GAT_GATE_CLKCMU_FSYS_USB30DRD 0x2054 #define CLK_CON_GAT_GATE_CLKCMU_PERI_BUS 0x207c #define CLK_CON_GAT_GATE_CLKCMU_PERI_SPI0 0x2080 #define CLK_CON_GAT_GATE_CLKCMU_PERI_SPI1 0x2084 @@ -76,6 +91,11 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, CLK_CON_MUX_MUX_CLKCMU_CORE_G3D, + CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_SDIO, + CLK_CON_MUX_MUX_CLKCMU_FSYS_USB30DRD, CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, CLK_CON_MUX_MUX_CLKCMU_PERI_SPI0, CLK_CON_MUX_MUX_CLKCMU_PERI_SPI1, @@ -88,6 +108,11 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_CLKCMU_CORE_BUS, CLK_CON_DIV_CLKCMU_CORE_CCI, CLK_CON_DIV_CLKCMU_CORE_G3D, + CLK_CON_DIV_CLKCMU_FSYS_BUS, + CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD, + CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD, + CLK_CON_DIV_CLKCMU_FSYS_MMC_SDIO, + CLK_CON_DIV_CLKCMU_FSYS_USB30DRD, CLK_CON_DIV_CLKCMU_PERI_BUS, CLK_CON_DIV_CLKCMU_PERI_SPI0, CLK_CON_DIV_CLKCMU_PERI_SPI1, @@ -108,6 +133,11 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_GAT_GATE_CLKCMU_CORE_BUS, CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, CLK_CON_GAT_GATE_CLKCMU_CORE_G3D, + CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS, + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD, + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD, + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_SDIO, + CLK_CON_GAT_GATE_CLKCMU_FSYS_USB30DRD, CLK_CON_GAT_GATE_CLKCMU_PERI_BUS, CLK_CON_GAT_GATE_CLKCMU_PERI_SPI0, CLK_CON_GAT_GATE_CLKCMU_PERI_SPI1, @@ -146,6 +176,13 @@ PNAME(mout_peri_usi0_p) = { "oscclk", "dout_shared0_div4" }; PNAME(mout_peri_usi1_p) = { "oscclk", "dout_shared0_div4" }; PNAME(mout_peri_usi2_p) = { "oscclk", "dout_shared0_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_FSYS */ +PNAME(mout_fsys_bus_p) = { "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_fsys_mmc_card_p) = { "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_fsys_mmc_embd_p) = { "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_fsys_mmc_sdio_p) = { "dout_shared0_div2", "dout_shared1_div2" }; +PNAME(mout_fsys_usb30drd_p) = { "dout_shared0_div4", "dout_shared1_div4" }; + static const struct samsung_mux_clock top_mux_clks[] __initconst = { /* CORE */ MUX(CLK_MOUT_CORE_BUS, "mout_core_bus", mout_core_bus_p, @@ -174,6 +211,18 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_PERI_USI1, 0, 1), MUX(CLK_MOUT_PERI_USI2, "mout_peri_usi2", mout_peri_usi2_p, CLK_CON_MUX_MUX_CLKCMU_PERI_USI2, 0, 1), + + /* FSYS */ + MUX(CLK_MOUT_FSYS_BUS, "mout_fsys_bus", mout_fsys_bus_p, + CLK_CON_MUX_MUX_CLKCMU_FSYS_BUS, 0, 1), + MUX(CLK_MOUT_FSYS_MMC_CARD, "mout_fsys_mmc_card", mout_fsys_mmc_card_p, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_CARD, 0, 1), + MUX(CLK_MOUT_FSYS_MMC_EMBD, "mout_fsys_mmc_embd", mout_fsys_mmc_embd_p, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_EMBD, 0, 1), + MUX(CLK_MOUT_FSYS_MMC_SDIO, "mout_fsys_mmc_sdio", mout_fsys_mmc_sdio_p, + CLK_CON_MUX_MUX_CLKCMU_FSYS_MMC_SDIO, 0, 1), + MUX(CLK_MOUT_FSYS_USB30DRD, "mout_fsys_usb30drd", mout_fsys_usb30drd_p, + CLK_CON_MUX_MUX_CLKCMU_FSYS_USB30DRD, 0, 1), }; static const struct samsung_div_clock top_div_clks[] __initconst = { @@ -220,6 +269,18 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { CLK_CON_DIV_CLKCMU_PERI_USI1, 0, 4), DIV(CLK_DOUT_PERI_USI2, "dout_peri_usi2", "gout_peri_usi2", CLK_CON_DIV_CLKCMU_PERI_USI2, 0, 4), + + /* FSYS */ + DIV(CLK_DOUT_FSYS_BUS, "dout_fsys_bus", "gout_fsys_bus", + CLK_CON_DIV_CLKCMU_FSYS_BUS, 0, 4), + DIV(CLK_DOUT_FSYS_MMC_CARD, "dout_fsys_mmc_card", "gout_fsys_mmc_card", + CLK_CON_DIV_CLKCMU_FSYS_MMC_CARD, 0, 9), + DIV(CLK_DOUT_FSYS_MMC_EMBD, "dout_fsys_mmc_embd", "gout_fsys_mmc_embd", + CLK_CON_DIV_CLKCMU_FSYS_MMC_EMBD, 0, 9), + DIV(CLK_DOUT_FSYS_MMC_SDIO, "dout_fsys_mmc_sdio", "gout_fsys_mmc_sdio", + CLK_CON_DIV_CLKCMU_FSYS_MMC_SDIO, 0, 9), + DIV(CLK_DOUT_FSYS_USB30DRD, "dout_fsys_usb30drd", "gout_fsys_usb30drd", + CLK_CON_DIV_CLKCMU_FSYS_USB30DRD, 0, 4), }; static const struct samsung_gate_clock top_gate_clks[] __initconst = { @@ -250,6 +311,18 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { CLK_CON_GAT_GATE_CLKCMU_PERI_USI1, 21, 0, 0), GATE(CLK_GOUT_PERI_USI2, "gout_peri_usi2", "mout_peri_usi2", CLK_CON_GAT_GATE_CLKCMU_PERI_USI2, 21, 0, 0), + + /* FSYS */ + GATE(CLK_GOUT_FSYS_BUS, "gout_fsys_bus", "mout_fsys_bus", + CLK_CON_GAT_GATE_CLKCMU_FSYS_BUS, 21, 0, 0), + GATE(CLK_GOUT_FSYS_MMC_CARD, "gout_fsys_mmc_card", "mout_fsys_mmc_card", + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_CARD, 21, 0, 0), + GATE(CLK_GOUT_FSYS_MMC_EMBD, "gout_fsys_mmc_embd", "mout_fsys_mmc_embd", + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_EMBD, 21, 0, 0), + GATE(CLK_GOUT_FSYS_MMC_SDIO, "gout_fsys_mmc_sdio", "mout_fsys_mmc_sdio", + CLK_CON_GAT_GATE_CLKCMU_FSYS_MMC_SDIO, 21, 0, 0), + GATE(CLK_GOUT_FSYS_USB30DRD, "gout_fsys_usb30drd", "mout_fsys_usb30drd", + CLK_CON_GAT_GATE_CLKCMU_FSYS_USB30DRD, 21, 0, 0), }; static const struct samsung_cmu_info top_cmu_info __initconst = { @@ -498,13 +571,20 @@ CLK_OF_DECLARE(exynos7885_cmu_peri, "samsung,exynos7885-cmu-peri", /* ---- CMU_CORE ------------------------------------------------------------ */ /* Register Offset definitions for CMU_CORE (0x12000000) */ -#define PLL_CON0_MUX_CLKCMU_CORE_BUS_USER 0x0100 -#define PLL_CON0_MUX_CLKCMU_CORE_CCI_USER 0x0120 -#define PLL_CON0_MUX_CLKCMU_CORE_G3D_USER 0x0140 -#define CLK_CON_MUX_MUX_CLK_CORE_GIC 0x1000 -#define CLK_CON_DIV_DIV_CLK_CORE_BUSP 0x1800 -#define CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK 0x2054 -#define CLK_CON_GAT_GOUT_CORE_GIC400_CLK 0x2058 +#define PLL_CON0_MUX_CLKCMU_CORE_BUS_USER 0x0100 +#define PLL_CON0_MUX_CLKCMU_CORE_CCI_USER 0x0120 +#define PLL_CON0_MUX_CLKCMU_CORE_G3D_USER 0x0140 +#define CLK_CON_MUX_MUX_CLK_CORE_GIC 0x1000 +#define CLK_CON_DIV_DIV_CLK_CORE_BUSP 0x1800 +#define CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK 0x2054 +#define CLK_CON_GAT_GOUT_CORE_GIC400_CLK 0x2058 +#define CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_ACLK 0x215c +#define CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_GCLK 0x2160 +#define CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_PCLK 0x2164 +#define CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_ACLK_P_CORE 0x2168 +#define CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_CCLK_P_CORE 0x216c +#define CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK 0x2170 +#define CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK_P_CORE 0x2174 static const unsigned long core_clk_regs[] __initconst = { PLL_CON0_MUX_CLKCMU_CORE_BUS_USER, @@ -514,6 +594,13 @@ static const unsigned long core_clk_regs[] __initconst = { CLK_CON_DIV_DIV_CLK_CORE_BUSP, CLK_CON_GAT_GOUT_CORE_CCI_550_ACLK, CLK_CON_GAT_GOUT_CORE_GIC400_CLK, + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_ACLK, + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_GCLK, + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_PCLK, + CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_ACLK_P_CORE, + CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_CCLK_P_CORE, + CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK, + CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK_P_CORE, }; /* List of parent clocks for Muxes in CMU_CORE */ @@ -545,6 +632,27 @@ static const struct samsung_gate_clock core_gate_clks[] __initconst = { /* GIC (interrupt controller) clock must be always running */ GATE(CLK_GOUT_GIC400_CLK, "gout_gic400_clk", "mout_core_gic", CLK_CON_GAT_GOUT_CORE_GIC400_CLK, 21, CLK_IS_CRITICAL, 0), + /* + * TREX D and P Core (seems to be related to "bus traffic shaper") + * clocks must always be running + */ + GATE(CLK_GOUT_TREX_D_CORE_ACLK, "gout_trex_d_core_aclk", "mout_core_bus_user", + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_ACLK, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_D_CORE_GCLK, "gout_trex_d_core_gclk", "mout_core_g3d_user", + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_GCLK, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_D_CORE_PCLK, "gout_trex_d_core_pclk", "dout_core_busp", + CLK_CON_GAT_GOUT_CORE_TREX_D_CORE_PCLK, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_P_CORE_ACLK_P_CORE, "gout_trex_p_core_aclk_p_core", + "mout_core_bus_user", CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_ACLK_P_CORE, 21, + CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_P_CORE_CCLK_P_CORE, "gout_trex_p_core_cclk_p_core", + "mout_core_cci_user", CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_CCLK_P_CORE, 21, + CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_P_CORE_PCLK, "gout_trex_p_core_pclk", "dout_core_busp", + CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_TREX_P_CORE_PCLK_P_CORE, "gout_trex_p_core_pclk_p_core", + "dout_core_busp", CLK_CON_GAT_GOUT_CORE_TREX_P_CORE_PCLK_P_CORE, 21, + CLK_IS_CRITICAL, 0), }; static const struct samsung_cmu_info core_cmu_info __initconst = { @@ -560,6 +668,88 @@ static const struct samsung_cmu_info core_cmu_info __initconst = { .clk_name = "dout_core_bus", }; +/* ---- CMU_FSYS ------------------------------------------------------------ */ + +/* Register Offset definitions for CMU_FSYS (0x13400000) */ +#define PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER 0x0100 +#define PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER 0x0120 +#define PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER 0x0140 +#define PLL_CON0_MUX_CLKCMU_FSYS_MMC_SDIO_USER 0x0160 +#define PLL_CON0_MUX_CLKCMU_FSYS_USB30DRD_USER 0x0180 +#define CLK_CON_GAT_GOUT_FSYS_MMC_CARD_I_ACLK 0x2030 +#define CLK_CON_GAT_GOUT_FSYS_MMC_CARD_SDCLKIN 0x2034 +#define CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_I_ACLK 0x2038 +#define CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_SDCLKIN 0x203c +#define CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_I_ACLK 0x2040 +#define CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_SDCLKIN 0x2044 + +static const unsigned long fsys_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER, + PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER, + PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER, + PLL_CON0_MUX_CLKCMU_FSYS_MMC_SDIO_USER, + PLL_CON0_MUX_CLKCMU_FSYS_USB30DRD_USER, + CLK_CON_GAT_GOUT_FSYS_MMC_CARD_I_ACLK, + CLK_CON_GAT_GOUT_FSYS_MMC_CARD_SDCLKIN, + CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_I_ACLK, + CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_SDCLKIN, + CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_I_ACLK, + CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_SDCLKIN, +}; + +/* List of parent clocks for Muxes in CMU_FSYS */ +PNAME(mout_fsys_bus_user_p) = { "oscclk", "dout_fsys_bus" }; +PNAME(mout_fsys_mmc_card_user_p) = { "oscclk", "dout_fsys_mmc_card" }; +PNAME(mout_fsys_mmc_embd_user_p) = { "oscclk", "dout_fsys_mmc_embd" }; +PNAME(mout_fsys_mmc_sdio_user_p) = { "oscclk", "dout_fsys_mmc_sdio" }; +PNAME(mout_fsys_usb30drd_user_p) = { "oscclk", "dout_fsys_usb30drd" }; + +static const struct samsung_mux_clock fsys_mux_clks[] __initconst = { + MUX(CLK_MOUT_FSYS_BUS_USER, "mout_fsys_bus_user", mout_fsys_bus_user_p, + PLL_CON0_MUX_CLKCMU_FSYS_BUS_USER, 4, 1), + MUX_F(CLK_MOUT_FSYS_MMC_CARD_USER, "mout_fsys_mmc_card_user", + mout_fsys_mmc_card_user_p, PLL_CON0_MUX_CLKCMU_FSYS_MMC_CARD_USER, + 4, 1, CLK_SET_RATE_PARENT, 0), + MUX_F(CLK_MOUT_FSYS_MMC_EMBD_USER, "mout_fsys_mmc_embd_user", + mout_fsys_mmc_embd_user_p, PLL_CON0_MUX_CLKCMU_FSYS_MMC_EMBD_USER, + 4, 1, CLK_SET_RATE_PARENT, 0), + MUX_F(CLK_MOUT_FSYS_MMC_SDIO_USER, "mout_fsys_mmc_sdio_user", + mout_fsys_mmc_sdio_user_p, PLL_CON0_MUX_CLKCMU_FSYS_MMC_SDIO_USER, + 4, 1, CLK_SET_RATE_PARENT, 0), + MUX_F(CLK_MOUT_FSYS_USB30DRD_USER, "mout_fsys_usb30drd_user", + mout_fsys_usb30drd_user_p, PLL_CON0_MUX_CLKCMU_FSYS_USB30DRD_USER, + 4, 1, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_gate_clock fsys_gate_clks[] __initconst = { + GATE(CLK_GOUT_MMC_CARD_ACLK, "gout_mmc_card_aclk", "mout_fsys_bus_user", + CLK_CON_GAT_GOUT_FSYS_MMC_CARD_I_ACLK, 21, 0, 0), + GATE(CLK_GOUT_MMC_CARD_SDCLKIN, "gout_mmc_card_sdclkin", + "mout_fsys_mmc_card_user", CLK_CON_GAT_GOUT_FSYS_MMC_CARD_SDCLKIN, + 21, CLK_SET_RATE_PARENT, 0), + GATE(CLK_GOUT_MMC_EMBD_ACLK, "gout_mmc_embd_aclk", "mout_fsys_bus_user", + CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_I_ACLK, 21, 0, 0), + GATE(CLK_GOUT_MMC_EMBD_SDCLKIN, "gout_mmc_embd_sdclkin", + "mout_fsys_mmc_embd_user", CLK_CON_GAT_GOUT_FSYS_MMC_EMBD_SDCLKIN, + 21, CLK_SET_RATE_PARENT, 0), + GATE(CLK_GOUT_MMC_SDIO_ACLK, "gout_mmc_sdio_aclk", "mout_fsys_bus_user", + CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_I_ACLK, 21, 0, 0), + GATE(CLK_GOUT_MMC_SDIO_SDCLKIN, "gout_mmc_sdio_sdclkin", + "mout_fsys_mmc_sdio_user", CLK_CON_GAT_GOUT_FSYS_MMC_SDIO_SDCLKIN, + 21, CLK_SET_RATE_PARENT, 0), +}; + +static const struct samsung_cmu_info fsys_cmu_info __initconst = { + .mux_clks = fsys_mux_clks, + .nr_mux_clks = ARRAY_SIZE(fsys_mux_clks), + .gate_clks = fsys_gate_clks, + .nr_gate_clks = ARRAY_SIZE(fsys_gate_clks), + .nr_clk_ids = FSYS_NR_CLK, + .clk_regs = fsys_clk_regs, + .nr_clk_regs = ARRAY_SIZE(fsys_clk_regs), + .clk_name = "dout_fsys_bus", +}; + /* ---- platform_driver ----------------------------------------------------- */ static int __init exynos7885_cmu_probe(struct platform_device *pdev) @@ -577,6 +767,9 @@ static const struct of_device_id exynos7885_cmu_of_match[] = { { .compatible = "samsung,exynos7885-cmu-core", .data = &core_cmu_info, + }, { + .compatible = "samsung,exynos7885-cmu-fsys", + .data = &fsys_cmu_info, }, { }, }; diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c index cd9725f1dbf7ad320cc9d675262dab1acd80564c..541761e96aeb677d2abebfd86ef079ead45b15d1 100644 --- a/drivers/clk/samsung/clk-exynos850.c +++ b/drivers/clk/samsung/clk-exynos850.c @@ -30,6 +30,7 @@ #define PLL_CON0_PLL_SHARED1 0x0180 #define PLL_CON3_PLL_SHARED1 0x018c #define CLK_CON_MUX_MUX_CLKCMU_APM_BUS 0x1000 +#define CLK_CON_MUX_MUX_CLKCMU_AUD 0x1004 #define CLK_CON_MUX_MUX_CLKCMU_CORE_BUS 0x1014 #define CLK_CON_MUX_MUX_CLKCMU_CORE_CCI 0x1018 #define CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD 0x101c @@ -38,10 +39,19 @@ #define CLK_CON_MUX_MUX_CLKCMU_HSI_BUS 0x103c #define CLK_CON_MUX_MUX_CLKCMU_HSI_MMC_CARD 0x1040 #define CLK_CON_MUX_MUX_CLKCMU_HSI_USB20DRD 0x1044 +#define CLK_CON_MUX_MUX_CLKCMU_IS_BUS 0x1048 +#define CLK_CON_MUX_MUX_CLKCMU_IS_GDC 0x104c +#define CLK_CON_MUX_MUX_CLKCMU_IS_ITP 0x1050 +#define CLK_CON_MUX_MUX_CLKCMU_IS_VRA 0x1054 +#define CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_JPEG 0x1058 +#define CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_M2M 0x105c +#define CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MCSC 0x1060 +#define CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MFC 0x1064 #define CLK_CON_MUX_MUX_CLKCMU_PERI_BUS 0x1070 #define CLK_CON_MUX_MUX_CLKCMU_PERI_IP 0x1074 #define CLK_CON_MUX_MUX_CLKCMU_PERI_UART 0x1078 #define CLK_CON_DIV_CLKCMU_APM_BUS 0x180c +#define CLK_CON_DIV_CLKCMU_AUD 0x1810 #define CLK_CON_DIV_CLKCMU_CORE_BUS 0x1820 #define CLK_CON_DIV_CLKCMU_CORE_CCI 0x1824 #define CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD 0x1828 @@ -50,6 +60,14 @@ #define CLK_CON_DIV_CLKCMU_HSI_BUS 0x1848 #define CLK_CON_DIV_CLKCMU_HSI_MMC_CARD 0x184c #define CLK_CON_DIV_CLKCMU_HSI_USB20DRD 0x1850 +#define CLK_CON_DIV_CLKCMU_IS_BUS 0x1854 +#define CLK_CON_DIV_CLKCMU_IS_GDC 0x1858 +#define CLK_CON_DIV_CLKCMU_IS_ITP 0x185c +#define CLK_CON_DIV_CLKCMU_IS_VRA 0x1860 +#define CLK_CON_DIV_CLKCMU_MFCMSCL_JPEG 0x1864 +#define CLK_CON_DIV_CLKCMU_MFCMSCL_M2M 0x1868 +#define CLK_CON_DIV_CLKCMU_MFCMSCL_MCSC 0x186c +#define CLK_CON_DIV_CLKCMU_MFCMSCL_MFC 0x1870 #define CLK_CON_DIV_CLKCMU_PERI_BUS 0x187c #define CLK_CON_DIV_CLKCMU_PERI_IP 0x1880 #define CLK_CON_DIV_CLKCMU_PERI_UART 0x1884 @@ -60,6 +78,7 @@ #define CLK_CON_DIV_PLL_SHARED1_DIV3 0x189c #define CLK_CON_DIV_PLL_SHARED1_DIV4 0x18a0 #define CLK_CON_GAT_GATE_CLKCMU_APM_BUS 0x2008 +#define CLK_CON_GAT_GATE_CLKCMU_AUD 0x200c #define CLK_CON_GAT_GATE_CLKCMU_CORE_BUS 0x201c #define CLK_CON_GAT_GATE_CLKCMU_CORE_CCI 0x2020 #define CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD 0x2024 @@ -68,6 +87,14 @@ #define CLK_CON_GAT_GATE_CLKCMU_HSI_BUS 0x2044 #define CLK_CON_GAT_GATE_CLKCMU_HSI_MMC_CARD 0x2048 #define CLK_CON_GAT_GATE_CLKCMU_HSI_USB20DRD 0x204c +#define CLK_CON_GAT_GATE_CLKCMU_IS_BUS 0x2050 +#define CLK_CON_GAT_GATE_CLKCMU_IS_GDC 0x2054 +#define CLK_CON_GAT_GATE_CLKCMU_IS_ITP 0x2058 +#define CLK_CON_GAT_GATE_CLKCMU_IS_VRA 0x205c +#define CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_JPEG 0x2060 +#define CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_M2M 0x2064 +#define CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MCSC 0x2068 +#define CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MFC 0x206c #define CLK_CON_GAT_GATE_CLKCMU_PERI_BUS 0x2080 #define CLK_CON_GAT_GATE_CLKCMU_PERI_IP 0x2084 #define CLK_CON_GAT_GATE_CLKCMU_PERI_UART 0x2088 @@ -83,6 +110,7 @@ static const unsigned long top_clk_regs[] __initconst = { PLL_CON0_PLL_SHARED1, PLL_CON3_PLL_SHARED1, CLK_CON_MUX_MUX_CLKCMU_APM_BUS, + CLK_CON_MUX_MUX_CLKCMU_AUD, CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, CLK_CON_MUX_MUX_CLKCMU_CORE_CCI, CLK_CON_MUX_MUX_CLKCMU_CORE_MMC_EMBD, @@ -91,10 +119,19 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_MUX_MUX_CLKCMU_HSI_BUS, CLK_CON_MUX_MUX_CLKCMU_HSI_MMC_CARD, CLK_CON_MUX_MUX_CLKCMU_HSI_USB20DRD, + CLK_CON_MUX_MUX_CLKCMU_IS_BUS, + CLK_CON_MUX_MUX_CLKCMU_IS_GDC, + CLK_CON_MUX_MUX_CLKCMU_IS_ITP, + CLK_CON_MUX_MUX_CLKCMU_IS_VRA, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_JPEG, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_M2M, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MCSC, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MFC, CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, CLK_CON_MUX_MUX_CLKCMU_PERI_IP, CLK_CON_MUX_MUX_CLKCMU_PERI_UART, CLK_CON_DIV_CLKCMU_APM_BUS, + CLK_CON_DIV_CLKCMU_AUD, CLK_CON_DIV_CLKCMU_CORE_BUS, CLK_CON_DIV_CLKCMU_CORE_CCI, CLK_CON_DIV_CLKCMU_CORE_MMC_EMBD, @@ -103,6 +140,14 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_CLKCMU_HSI_BUS, CLK_CON_DIV_CLKCMU_HSI_MMC_CARD, CLK_CON_DIV_CLKCMU_HSI_USB20DRD, + CLK_CON_DIV_CLKCMU_IS_BUS, + CLK_CON_DIV_CLKCMU_IS_GDC, + CLK_CON_DIV_CLKCMU_IS_ITP, + CLK_CON_DIV_CLKCMU_IS_VRA, + CLK_CON_DIV_CLKCMU_MFCMSCL_JPEG, + CLK_CON_DIV_CLKCMU_MFCMSCL_M2M, + CLK_CON_DIV_CLKCMU_MFCMSCL_MCSC, + CLK_CON_DIV_CLKCMU_MFCMSCL_MFC, CLK_CON_DIV_CLKCMU_PERI_BUS, CLK_CON_DIV_CLKCMU_PERI_IP, CLK_CON_DIV_CLKCMU_PERI_UART, @@ -113,6 +158,7 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_DIV_PLL_SHARED1_DIV3, CLK_CON_DIV_PLL_SHARED1_DIV4, CLK_CON_GAT_GATE_CLKCMU_APM_BUS, + CLK_CON_GAT_GATE_CLKCMU_AUD, CLK_CON_GAT_GATE_CLKCMU_CORE_BUS, CLK_CON_GAT_GATE_CLKCMU_CORE_CCI, CLK_CON_GAT_GATE_CLKCMU_CORE_MMC_EMBD, @@ -121,6 +167,14 @@ static const unsigned long top_clk_regs[] __initconst = { CLK_CON_GAT_GATE_CLKCMU_HSI_BUS, CLK_CON_GAT_GATE_CLKCMU_HSI_MMC_CARD, CLK_CON_GAT_GATE_CLKCMU_HSI_USB20DRD, + CLK_CON_GAT_GATE_CLKCMU_IS_BUS, + CLK_CON_GAT_GATE_CLKCMU_IS_GDC, + CLK_CON_GAT_GATE_CLKCMU_IS_ITP, + CLK_CON_GAT_GATE_CLKCMU_IS_VRA, + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_JPEG, + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_M2M, + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MCSC, + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MFC, CLK_CON_GAT_GATE_CLKCMU_PERI_BUS, CLK_CON_GAT_GATE_CLKCMU_PERI_IP, CLK_CON_GAT_GATE_CLKCMU_PERI_UART, @@ -148,6 +202,9 @@ PNAME(mout_shared1_pll_p) = { "oscclk", "fout_shared1_pll" }; PNAME(mout_mmc_pll_p) = { "oscclk", "fout_mmc_pll" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_APM */ PNAME(mout_clkcmu_apm_bus_p) = { "dout_shared0_div4", "pll_shared1_div4" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_AUD */ +PNAME(mout_aud_p) = { "fout_shared1_pll", "dout_shared0_div2", + "dout_shared1_div2", "dout_shared0_div3" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_CORE */ PNAME(mout_core_bus_p) = { "dout_shared1_div2", "dout_shared0_div3", "dout_shared1_div3", "dout_shared0_div4" }; @@ -167,13 +224,30 @@ PNAME(mout_hsi_mmc_card_p) = { "oscclk", "dout_shared0_div2", "oscclk", "oscclk" }; PNAME(mout_hsi_usb20drd_p) = { "oscclk", "dout_shared0_div4", "dout_shared1_div4", "oscclk" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_IS */ +PNAME(mout_is_bus_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared1_div3" }; +PNAME(mout_is_itp_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared1_div3" }; +PNAME(mout_is_vra_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared1_div3" }; +PNAME(mout_is_gdc_p) = { "dout_shared0_div2", "dout_shared1_div2", + "dout_shared0_div3", "dout_shared1_div3" }; +/* List of parent clocks for Muxes in CMU_TOP: for CMU_MFCMSCL */ +PNAME(mout_mfcmscl_mfc_p) = { "dout_shared1_div2", "dout_shared0_div3", + "dout_shared1_div3", "dout_shared0_div4" }; +PNAME(mout_mfcmscl_m2m_p) = { "dout_shared1_div2", "dout_shared0_div3", + "dout_shared1_div3", "dout_shared0_div4" }; +PNAME(mout_mfcmscl_mcsc_p) = { "dout_shared1_div2", "dout_shared0_div3", + "dout_shared1_div3", "dout_shared0_div4" }; +PNAME(mout_mfcmscl_jpeg_p) = { "dout_shared0_div3", "dout_shared1_div3", + "dout_shared0_div4", "dout_shared1_div4" }; /* List of parent clocks for Muxes in CMU_TOP: for CMU_PERI */ PNAME(mout_peri_bus_p) = { "dout_shared0_div4", "dout_shared1_div4" }; PNAME(mout_peri_uart_p) = { "oscclk", "dout_shared0_div4", "dout_shared1_div4", "oscclk" }; PNAME(mout_peri_ip_p) = { "oscclk", "dout_shared0_div4", "dout_shared1_div4", "oscclk" }; - /* List of parent clocks for Muxes in CMU_TOP: for CMU_DPU */ PNAME(mout_dpu_p) = { "dout_shared0_div3", "dout_shared1_div3", "dout_shared0_div4", "dout_shared1_div4" }; @@ -191,6 +265,10 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(CLK_MOUT_CLKCMU_APM_BUS, "mout_clkcmu_apm_bus", mout_clkcmu_apm_bus_p, CLK_CON_MUX_MUX_CLKCMU_APM_BUS, 0, 1), + /* AUD */ + MUX(CLK_MOUT_AUD, "mout_aud", mout_aud_p, + CLK_CON_MUX_MUX_CLKCMU_AUD, 0, 2), + /* CORE */ MUX(CLK_MOUT_CORE_BUS, "mout_core_bus", mout_core_bus_p, CLK_CON_MUX_MUX_CLKCMU_CORE_BUS, 0, 2), @@ -213,6 +291,26 @@ static const struct samsung_mux_clock top_mux_clks[] __initconst = { MUX(CLK_MOUT_HSI_USB20DRD, "mout_hsi_usb20drd", mout_hsi_usb20drd_p, CLK_CON_MUX_MUX_CLKCMU_HSI_USB20DRD, 0, 2), + /* IS */ + MUX(CLK_MOUT_IS_BUS, "mout_is_bus", mout_is_bus_p, + CLK_CON_MUX_MUX_CLKCMU_IS_BUS, 0, 2), + MUX(CLK_MOUT_IS_ITP, "mout_is_itp", mout_is_itp_p, + CLK_CON_MUX_MUX_CLKCMU_IS_ITP, 0, 2), + MUX(CLK_MOUT_IS_VRA, "mout_is_vra", mout_is_vra_p, + CLK_CON_MUX_MUX_CLKCMU_IS_VRA, 0, 2), + MUX(CLK_MOUT_IS_GDC, "mout_is_gdc", mout_is_gdc_p, + CLK_CON_MUX_MUX_CLKCMU_IS_GDC, 0, 2), + + /* MFCMSCL */ + MUX(CLK_MOUT_MFCMSCL_MFC, "mout_mfcmscl_mfc", mout_mfcmscl_mfc_p, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MFC, 0, 2), + MUX(CLK_MOUT_MFCMSCL_M2M, "mout_mfcmscl_m2m", mout_mfcmscl_m2m_p, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_M2M, 0, 2), + MUX(CLK_MOUT_MFCMSCL_MCSC, "mout_mfcmscl_mcsc", mout_mfcmscl_mcsc_p, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_MCSC, 0, 2), + MUX(CLK_MOUT_MFCMSCL_JPEG, "mout_mfcmscl_jpeg", mout_mfcmscl_jpeg_p, + CLK_CON_MUX_MUX_CLKCMU_MFCMSCL_JPEG, 0, 2), + /* PERI */ MUX(CLK_MOUT_PERI_BUS, "mout_peri_bus", mout_peri_bus_p, CLK_CON_MUX_MUX_CLKCMU_PERI_BUS, 0, 1), @@ -241,6 +339,10 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(CLK_DOUT_CLKCMU_APM_BUS, "dout_clkcmu_apm_bus", "gout_clkcmu_apm_bus", CLK_CON_DIV_CLKCMU_APM_BUS, 0, 3), + /* AUD */ + DIV(CLK_DOUT_AUD, "dout_aud", "gout_aud", + CLK_CON_DIV_CLKCMU_AUD, 0, 4), + /* CORE */ DIV(CLK_DOUT_CORE_BUS, "dout_core_bus", "gout_core_bus", CLK_CON_DIV_CLKCMU_CORE_BUS, 0, 4), @@ -263,6 +365,26 @@ static const struct samsung_div_clock top_div_clks[] __initconst = { DIV(CLK_DOUT_HSI_USB20DRD, "dout_hsi_usb20drd", "gout_hsi_usb20drd", CLK_CON_DIV_CLKCMU_HSI_USB20DRD, 0, 4), + /* IS */ + DIV(CLK_DOUT_IS_BUS, "dout_is_bus", "gout_is_bus", + CLK_CON_DIV_CLKCMU_IS_BUS, 0, 4), + DIV(CLK_DOUT_IS_ITP, "dout_is_itp", "gout_is_itp", + CLK_CON_DIV_CLKCMU_IS_ITP, 0, 4), + DIV(CLK_DOUT_IS_VRA, "dout_is_vra", "gout_is_vra", + CLK_CON_DIV_CLKCMU_IS_VRA, 0, 4), + DIV(CLK_DOUT_IS_GDC, "dout_is_gdc", "gout_is_gdc", + CLK_CON_DIV_CLKCMU_IS_GDC, 0, 4), + + /* MFCMSCL */ + DIV(CLK_DOUT_MFCMSCL_MFC, "dout_mfcmscl_mfc", "gout_mfcmscl_mfc", + CLK_CON_DIV_CLKCMU_MFCMSCL_MFC, 0, 4), + DIV(CLK_DOUT_MFCMSCL_M2M, "dout_mfcmscl_m2m", "gout_mfcmscl_m2m", + CLK_CON_DIV_CLKCMU_MFCMSCL_M2M, 0, 4), + DIV(CLK_DOUT_MFCMSCL_MCSC, "dout_mfcmscl_mcsc", "gout_mfcmscl_mcsc", + CLK_CON_DIV_CLKCMU_MFCMSCL_MCSC, 0, 4), + DIV(CLK_DOUT_MFCMSCL_JPEG, "dout_mfcmscl_jpeg", "gout_mfcmscl_jpeg", + CLK_CON_DIV_CLKCMU_MFCMSCL_JPEG, 0, 4), + /* PERI */ DIV(CLK_DOUT_PERI_BUS, "dout_peri_bus", "gout_peri_bus", CLK_CON_DIV_CLKCMU_PERI_BUS, 0, 4), @@ -287,6 +409,10 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_GOUT_CLKCMU_APM_BUS, "gout_clkcmu_apm_bus", "mout_clkcmu_apm_bus", CLK_CON_GAT_GATE_CLKCMU_APM_BUS, 21, 0, 0), + /* AUD */ + GATE(CLK_GOUT_AUD, "gout_aud", "mout_aud", + CLK_CON_GAT_GATE_CLKCMU_AUD, 21, 0, 0), + /* DPU */ GATE(CLK_GOUT_DPU, "gout_dpu", "mout_dpu", CLK_CON_GAT_GATE_CLKCMU_DPU, 21, 0, 0), @@ -299,6 +425,28 @@ static const struct samsung_gate_clock top_gate_clks[] __initconst = { GATE(CLK_GOUT_HSI_USB20DRD, "gout_hsi_usb20drd", "mout_hsi_usb20drd", CLK_CON_GAT_GATE_CLKCMU_HSI_USB20DRD, 21, 0, 0), + /* IS */ + /* TODO: These clocks have to be always enabled to access CMU_IS regs */ + GATE(CLK_GOUT_IS_BUS, "gout_is_bus", "mout_is_bus", + CLK_CON_GAT_GATE_CLKCMU_IS_BUS, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_IS_ITP, "gout_is_itp", "mout_is_itp", + CLK_CON_GAT_GATE_CLKCMU_IS_ITP, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_IS_VRA, "gout_is_vra", "mout_is_vra", + CLK_CON_GAT_GATE_CLKCMU_IS_VRA, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_IS_GDC, "gout_is_gdc", "mout_is_gdc", + CLK_CON_GAT_GATE_CLKCMU_IS_GDC, 21, CLK_IS_CRITICAL, 0), + + /* MFCMSCL */ + /* TODO: These have to be always enabled to access CMU_MFCMSCL regs */ + GATE(CLK_GOUT_MFCMSCL_MFC, "gout_mfcmscl_mfc", "mout_mfcmscl_mfc", + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MFC, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_MFCMSCL_M2M, "gout_mfcmscl_m2m", "mout_mfcmscl_m2m", + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_M2M, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_MFCMSCL_MCSC, "gout_mfcmscl_mcsc", "mout_mfcmscl_mcsc", + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_MCSC, 21, CLK_IS_CRITICAL, 0), + GATE(CLK_GOUT_MFCMSCL_JPEG, "gout_mfcmscl_jpeg", "mout_mfcmscl_jpeg", + CLK_CON_GAT_GATE_CLKCMU_MFCMSCL_JPEG, 21, CLK_IS_CRITICAL, 0), + /* PERI */ GATE(CLK_GOUT_PERI_BUS, "gout_peri_bus", "mout_peri_bus", CLK_CON_GAT_GATE_CLKCMU_PERI_BUS, 21, 0, 0), @@ -463,6 +611,284 @@ static const struct samsung_cmu_info apm_cmu_info __initconst = { .clk_name = "dout_clkcmu_apm_bus", }; +/* ---- CMU_AUD ------------------------------------------------------------- */ + +#define PLL_LOCKTIME_PLL_AUD 0x0000 +#define PLL_CON0_PLL_AUD 0x0100 +#define PLL_CON3_PLL_AUD 0x010c +#define PLL_CON0_MUX_CLKCMU_AUD_CPU_USER 0x0600 +#define PLL_CON0_MUX_TICK_USB_USER 0x0610 +#define CLK_CON_MUX_MUX_CLK_AUD_CPU 0x1000 +#define CLK_CON_MUX_MUX_CLK_AUD_CPU_HCH 0x1004 +#define CLK_CON_MUX_MUX_CLK_AUD_FM 0x1008 +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF0 0x100c +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF1 0x1010 +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF2 0x1014 +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF3 0x1018 +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF4 0x101c +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF5 0x1020 +#define CLK_CON_MUX_MUX_CLK_AUD_UAIF6 0x1024 +#define CLK_CON_DIV_DIV_CLK_AUD_MCLK 0x1800 +#define CLK_CON_DIV_DIV_CLK_AUD_AUDIF 0x1804 +#define CLK_CON_DIV_DIV_CLK_AUD_BUSD 0x1808 +#define CLK_CON_DIV_DIV_CLK_AUD_BUSP 0x180c +#define CLK_CON_DIV_DIV_CLK_AUD_CNT 0x1810 +#define CLK_CON_DIV_DIV_CLK_AUD_CPU 0x1814 +#define CLK_CON_DIV_DIV_CLK_AUD_CPU_ACLK 0x1818 +#define CLK_CON_DIV_DIV_CLK_AUD_CPU_PCLKDBG 0x181c +#define CLK_CON_DIV_DIV_CLK_AUD_FM 0x1820 +#define CLK_CON_DIV_DIV_CLK_AUD_FM_SPDY 0x1824 +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF0 0x1828 +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF1 0x182c +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF2 0x1830 +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF3 0x1834 +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF4 0x1838 +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF5 0x183c +#define CLK_CON_DIV_DIV_CLK_AUD_UAIF6 0x1840 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_CNT 0x2000 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF0 0x2004 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF1 0x2008 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF2 0x200c +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF3 0x2010 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF4 0x2014 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF5 0x2018 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF6 0x201c +#define CLK_CON_GAT_GOUT_AUD_ABOX_ACLK 0x2048 +#define CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_SPDY 0x204c +#define CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_ASB 0x2050 +#define CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_CA32 0x2054 +#define CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_DAP 0x2058 +#define CLK_CON_GAT_GOUT_AUD_CODEC_MCLK 0x206c +#define CLK_CON_GAT_GOUT_AUD_TZPC_PCLK 0x2070 +#define CLK_CON_GAT_GOUT_AUD_GPIO_PCLK 0x2074 +#define CLK_CON_GAT_GOUT_AUD_PPMU_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_AUD_PPMU_PCLK 0x208c +#define CLK_CON_GAT_GOUT_AUD_SYSMMU_CLK_S1 0x20b4 +#define CLK_CON_GAT_GOUT_AUD_SYSREG_PCLK 0x20b8 +#define CLK_CON_GAT_GOUT_AUD_WDT_PCLK 0x20bc + +static const unsigned long aud_clk_regs[] __initconst = { + PLL_LOCKTIME_PLL_AUD, + PLL_CON0_PLL_AUD, + PLL_CON3_PLL_AUD, + PLL_CON0_MUX_CLKCMU_AUD_CPU_USER, + PLL_CON0_MUX_TICK_USB_USER, + CLK_CON_MUX_MUX_CLK_AUD_CPU, + CLK_CON_MUX_MUX_CLK_AUD_CPU_HCH, + CLK_CON_MUX_MUX_CLK_AUD_FM, + CLK_CON_MUX_MUX_CLK_AUD_UAIF0, + CLK_CON_MUX_MUX_CLK_AUD_UAIF1, + CLK_CON_MUX_MUX_CLK_AUD_UAIF2, + CLK_CON_MUX_MUX_CLK_AUD_UAIF3, + CLK_CON_MUX_MUX_CLK_AUD_UAIF4, + CLK_CON_MUX_MUX_CLK_AUD_UAIF5, + CLK_CON_MUX_MUX_CLK_AUD_UAIF6, + CLK_CON_DIV_DIV_CLK_AUD_MCLK, + CLK_CON_DIV_DIV_CLK_AUD_AUDIF, + CLK_CON_DIV_DIV_CLK_AUD_BUSD, + CLK_CON_DIV_DIV_CLK_AUD_BUSP, + CLK_CON_DIV_DIV_CLK_AUD_CNT, + CLK_CON_DIV_DIV_CLK_AUD_CPU, + CLK_CON_DIV_DIV_CLK_AUD_CPU_ACLK, + CLK_CON_DIV_DIV_CLK_AUD_CPU_PCLKDBG, + CLK_CON_DIV_DIV_CLK_AUD_FM, + CLK_CON_DIV_DIV_CLK_AUD_FM_SPDY, + CLK_CON_DIV_DIV_CLK_AUD_UAIF0, + CLK_CON_DIV_DIV_CLK_AUD_UAIF1, + CLK_CON_DIV_DIV_CLK_AUD_UAIF2, + CLK_CON_DIV_DIV_CLK_AUD_UAIF3, + CLK_CON_DIV_DIV_CLK_AUD_UAIF4, + CLK_CON_DIV_DIV_CLK_AUD_UAIF5, + CLK_CON_DIV_DIV_CLK_AUD_UAIF6, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_CNT, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF0, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF1, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF2, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF3, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF4, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF5, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF6, + CLK_CON_GAT_GOUT_AUD_ABOX_ACLK, + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_SPDY, + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_ASB, + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_CA32, + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_DAP, + CLK_CON_GAT_GOUT_AUD_CODEC_MCLK, + CLK_CON_GAT_GOUT_AUD_TZPC_PCLK, + CLK_CON_GAT_GOUT_AUD_GPIO_PCLK, + CLK_CON_GAT_GOUT_AUD_PPMU_ACLK, + CLK_CON_GAT_GOUT_AUD_PPMU_PCLK, + CLK_CON_GAT_GOUT_AUD_SYSMMU_CLK_S1, + CLK_CON_GAT_GOUT_AUD_SYSREG_PCLK, + CLK_CON_GAT_GOUT_AUD_WDT_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_AUD */ +PNAME(mout_aud_pll_p) = { "oscclk", "fout_aud_pll" }; +PNAME(mout_aud_cpu_user_p) = { "oscclk", "dout_aud" }; +PNAME(mout_aud_cpu_p) = { "dout_aud_cpu", "mout_aud_cpu_user" }; +PNAME(mout_aud_cpu_hch_p) = { "mout_aud_cpu", "oscclk" }; +PNAME(mout_aud_uaif0_p) = { "dout_aud_uaif0", "ioclk_audiocdclk0" }; +PNAME(mout_aud_uaif1_p) = { "dout_aud_uaif1", "ioclk_audiocdclk1" }; +PNAME(mout_aud_uaif2_p) = { "dout_aud_uaif2", "ioclk_audiocdclk2" }; +PNAME(mout_aud_uaif3_p) = { "dout_aud_uaif3", "ioclk_audiocdclk3" }; +PNAME(mout_aud_uaif4_p) = { "dout_aud_uaif4", "ioclk_audiocdclk4" }; +PNAME(mout_aud_uaif5_p) = { "dout_aud_uaif5", "ioclk_audiocdclk5" }; +PNAME(mout_aud_uaif6_p) = { "dout_aud_uaif6", "ioclk_audiocdclk6" }; +PNAME(mout_aud_tick_usb_user_p) = { "oscclk", "tick_usb" }; +PNAME(mout_aud_fm_p) = { "oscclk", "dout_aud_fm_spdy" }; + +/* + * Do not provide PLL table to PLL_AUD, as MANUAL_PLL_CTRL bit is not set + * for that PLL by default, so set_rate operation would fail. + */ +static const struct samsung_pll_clock aud_pll_clks[] __initconst = { + PLL(pll_0831x, CLK_FOUT_AUD_PLL, "fout_aud_pll", "oscclk", + PLL_LOCKTIME_PLL_AUD, PLL_CON3_PLL_AUD, NULL), +}; + +static const struct samsung_fixed_rate_clock aud_fixed_clks[] __initconst = { + FRATE(IOCLK_AUDIOCDCLK0, "ioclk_audiocdclk0", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK1, "ioclk_audiocdclk1", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK2, "ioclk_audiocdclk2", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK3, "ioclk_audiocdclk3", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK4, "ioclk_audiocdclk4", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK5, "ioclk_audiocdclk5", NULL, 0, 25000000), + FRATE(IOCLK_AUDIOCDCLK6, "ioclk_audiocdclk6", NULL, 0, 25000000), + FRATE(TICK_USB, "tick_usb", NULL, 0, 60000000), +}; + +static const struct samsung_mux_clock aud_mux_clks[] __initconst = { + MUX(CLK_MOUT_AUD_PLL, "mout_aud_pll", mout_aud_pll_p, + PLL_CON0_PLL_AUD, 4, 1), + MUX(CLK_MOUT_AUD_CPU_USER, "mout_aud_cpu_user", mout_aud_cpu_user_p, + PLL_CON0_MUX_CLKCMU_AUD_CPU_USER, 4, 1), + MUX(CLK_MOUT_AUD_TICK_USB_USER, "mout_aud_tick_usb_user", + mout_aud_tick_usb_user_p, + PLL_CON0_MUX_TICK_USB_USER, 4, 1), + MUX(CLK_MOUT_AUD_CPU, "mout_aud_cpu", mout_aud_cpu_p, + CLK_CON_MUX_MUX_CLK_AUD_CPU, 0, 1), + MUX(CLK_MOUT_AUD_CPU_HCH, "mout_aud_cpu_hch", mout_aud_cpu_hch_p, + CLK_CON_MUX_MUX_CLK_AUD_CPU_HCH, 0, 1), + MUX(CLK_MOUT_AUD_UAIF0, "mout_aud_uaif0", mout_aud_uaif0_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF0, 0, 1), + MUX(CLK_MOUT_AUD_UAIF1, "mout_aud_uaif1", mout_aud_uaif1_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF1, 0, 1), + MUX(CLK_MOUT_AUD_UAIF2, "mout_aud_uaif2", mout_aud_uaif2_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF2, 0, 1), + MUX(CLK_MOUT_AUD_UAIF3, "mout_aud_uaif3", mout_aud_uaif3_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF3, 0, 1), + MUX(CLK_MOUT_AUD_UAIF4, "mout_aud_uaif4", mout_aud_uaif4_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF4, 0, 1), + MUX(CLK_MOUT_AUD_UAIF5, "mout_aud_uaif5", mout_aud_uaif5_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF5, 0, 1), + MUX(CLK_MOUT_AUD_UAIF6, "mout_aud_uaif6", mout_aud_uaif6_p, + CLK_CON_MUX_MUX_CLK_AUD_UAIF6, 0, 1), + MUX(CLK_MOUT_AUD_FM, "mout_aud_fm", mout_aud_fm_p, + CLK_CON_MUX_MUX_CLK_AUD_FM, 0, 1), +}; + +static const struct samsung_div_clock aud_div_clks[] __initconst = { + DIV(CLK_DOUT_AUD_CPU, "dout_aud_cpu", "mout_aud_pll", + CLK_CON_DIV_DIV_CLK_AUD_CPU, 0, 4), + DIV(CLK_DOUT_AUD_BUSD, "dout_aud_busd", "mout_aud_pll", + CLK_CON_DIV_DIV_CLK_AUD_BUSD, 0, 4), + DIV(CLK_DOUT_AUD_BUSP, "dout_aud_busp", "mout_aud_pll", + CLK_CON_DIV_DIV_CLK_AUD_BUSP, 0, 4), + DIV(CLK_DOUT_AUD_AUDIF, "dout_aud_audif", "mout_aud_pll", + CLK_CON_DIV_DIV_CLK_AUD_AUDIF, 0, 9), + DIV(CLK_DOUT_AUD_CPU_ACLK, "dout_aud_cpu_aclk", "mout_aud_cpu_hch", + CLK_CON_DIV_DIV_CLK_AUD_CPU_ACLK, 0, 3), + DIV(CLK_DOUT_AUD_CPU_PCLKDBG, "dout_aud_cpu_pclkdbg", + "mout_aud_cpu_hch", + CLK_CON_DIV_DIV_CLK_AUD_CPU_PCLKDBG, 0, 3), + DIV(CLK_DOUT_AUD_MCLK, "dout_aud_mclk", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_MCLK, 0, 2), + DIV(CLK_DOUT_AUD_CNT, "dout_aud_cnt", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_CNT, 0, 10), + DIV(CLK_DOUT_AUD_UAIF0, "dout_aud_uaif0", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF0, 0, 10), + DIV(CLK_DOUT_AUD_UAIF1, "dout_aud_uaif1", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF1, 0, 10), + DIV(CLK_DOUT_AUD_UAIF2, "dout_aud_uaif2", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF2, 0, 10), + DIV(CLK_DOUT_AUD_UAIF3, "dout_aud_uaif3", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF3, 0, 10), + DIV(CLK_DOUT_AUD_UAIF4, "dout_aud_uaif4", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF4, 0, 10), + DIV(CLK_DOUT_AUD_UAIF5, "dout_aud_uaif5", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF5, 0, 10), + DIV(CLK_DOUT_AUD_UAIF6, "dout_aud_uaif6", "dout_aud_audif", + CLK_CON_DIV_DIV_CLK_AUD_UAIF6, 0, 10), + DIV(CLK_DOUT_AUD_FM_SPDY, "dout_aud_fm_spdy", "mout_aud_tick_usb_user", + CLK_CON_DIV_DIV_CLK_AUD_FM_SPDY, 0, 1), + DIV(CLK_DOUT_AUD_FM, "dout_aud_fm", "mout_aud_fm", + CLK_CON_DIV_DIV_CLK_AUD_FM, 0, 10), +}; + +static const struct samsung_gate_clock aud_gate_clks[] __initconst = { + GATE(CLK_GOUT_AUD_CA32_CCLK, "gout_aud_ca32_cclk", "mout_aud_cpu_hch", + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_CA32, 21, 0, 0), + GATE(CLK_GOUT_AUD_ASB_CCLK, "gout_aud_asb_cclk", "dout_aud_cpu_aclk", + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_ASB, 21, 0, 0), + GATE(CLK_GOUT_AUD_DAP_CCLK, "gout_aud_dap_cclk", "dout_aud_cpu_pclkdbg", + CLK_CON_GAT_GOUT_AUD_ABOX_CCLK_DAP, 21, 0, 0), + /* TODO: Should be enabled in ABOX driver (or made CLK_IS_CRITICAL) */ + GATE(CLK_GOUT_AUD_ABOX_ACLK, "gout_aud_abox_aclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_ABOX_ACLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_AUD_GPIO_PCLK, "gout_aud_gpio_pclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_GPIO_PCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_PPMU_ACLK, "gout_aud_ppmu_aclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_PPMU_ACLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_PPMU_PCLK, "gout_aud_ppmu_pclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_PPMU_PCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_SYSMMU_CLK, "gout_aud_sysmmu_clk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_SYSMMU_CLK_S1, 21, 0, 0), + GATE(CLK_GOUT_AUD_SYSREG_PCLK, "gout_aud_sysreg_pclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_SYSREG_PCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_WDT_PCLK, "gout_aud_wdt_pclk", "dout_aud_busd", + CLK_CON_GAT_GOUT_AUD_WDT_PCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_TZPC_PCLK, "gout_aud_tzpc_pclk", "dout_aud_busp", + CLK_CON_GAT_GOUT_AUD_TZPC_PCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_CODEC_MCLK, "gout_aud_codec_mclk", "dout_aud_mclk", + CLK_CON_GAT_GOUT_AUD_CODEC_MCLK, 21, 0, 0), + GATE(CLK_GOUT_AUD_CNT_BCLK, "gout_aud_cnt_bclk", "dout_aud_cnt", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_CNT, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF0_BCLK, "gout_aud_uaif0_bclk", "mout_aud_uaif0", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF0, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF1_BCLK, "gout_aud_uaif1_bclk", "mout_aud_uaif1", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF1, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF2_BCLK, "gout_aud_uaif2_bclk", "mout_aud_uaif2", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF2, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF3_BCLK, "gout_aud_uaif3_bclk", "mout_aud_uaif3", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF3, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF4_BCLK, "gout_aud_uaif4_bclk", "mout_aud_uaif4", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF4, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF5_BCLK, "gout_aud_uaif5_bclk", "mout_aud_uaif5", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF5, 21, 0, 0), + GATE(CLK_GOUT_AUD_UAIF6_BCLK, "gout_aud_uaif6_bclk", "mout_aud_uaif6", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_UAIF6, 21, 0, 0), + GATE(CLK_GOUT_AUD_SPDY_BCLK, "gout_aud_spdy_bclk", "dout_aud_fm", + CLK_CON_GAT_GOUT_AUD_ABOX_BCLK_SPDY, 21, 0, 0), +}; + +static const struct samsung_cmu_info aud_cmu_info __initconst = { + .pll_clks = aud_pll_clks, + .nr_pll_clks = ARRAY_SIZE(aud_pll_clks), + .mux_clks = aud_mux_clks, + .nr_mux_clks = ARRAY_SIZE(aud_mux_clks), + .div_clks = aud_div_clks, + .nr_div_clks = ARRAY_SIZE(aud_div_clks), + .gate_clks = aud_gate_clks, + .nr_gate_clks = ARRAY_SIZE(aud_gate_clks), + .fixed_clks = aud_fixed_clks, + .nr_fixed_clks = ARRAY_SIZE(aud_fixed_clks), + .nr_clk_ids = AUD_NR_CLK, + .clk_regs = aud_clk_regs, + .nr_clk_regs = ARRAY_SIZE(aud_clk_regs), + .clk_name = "dout_aud", +}; + /* ---- CMU_CMGP ------------------------------------------------------------ */ /* Register Offset definitions for CMU_CMGP (0x11c00000) */ @@ -599,7 +1025,7 @@ static const unsigned long hsi_clk_regs[] __initconst = { CLK_CON_GAT_GOUT_HSI_USB20DRD_TOP_BUS_CLK_EARLY, }; -/* List of parent clocks for Muxes in CMU_PERI */ +/* List of parent clocks for Muxes in CMU_HSI */ PNAME(mout_hsi_bus_user_p) = { "oscclk", "dout_hsi_bus" }; PNAME(mout_hsi_mmc_card_user_p) = { "oscclk", "dout_hsi_mmc_card" }; PNAME(mout_hsi_usb20drd_user_p) = { "oscclk", "dout_hsi_usb20drd" }; @@ -654,6 +1080,247 @@ static const struct samsung_cmu_info hsi_cmu_info __initconst = { .clk_name = "dout_hsi_bus", }; +/* ---- CMU_IS -------------------------------------------------------------- */ + +#define PLL_CON0_MUX_CLKCMU_IS_BUS_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_IS_GDC_USER 0x0610 +#define PLL_CON0_MUX_CLKCMU_IS_ITP_USER 0x0620 +#define PLL_CON0_MUX_CLKCMU_IS_VRA_USER 0x0630 +#define CLK_CON_DIV_DIV_CLK_IS_BUSP 0x1800 +#define CLK_CON_GAT_CLK_IS_CMU_IS_PCLK 0x2000 +#define CLK_CON_GAT_GOUT_IS_CSIS0_ACLK 0x2040 +#define CLK_CON_GAT_GOUT_IS_CSIS1_ACLK 0x2044 +#define CLK_CON_GAT_GOUT_IS_CSIS2_ACLK 0x2048 +#define CLK_CON_GAT_GOUT_IS_TZPC_PCLK 0x204c +#define CLK_CON_GAT_GOUT_IS_CLK_CSIS_DMA 0x2050 +#define CLK_CON_GAT_GOUT_IS_CLK_GDC 0x2054 +#define CLK_CON_GAT_GOUT_IS_CLK_IPP 0x2058 +#define CLK_CON_GAT_GOUT_IS_CLK_ITP 0x205c +#define CLK_CON_GAT_GOUT_IS_CLK_MCSC 0x2060 +#define CLK_CON_GAT_GOUT_IS_CLK_VRA 0x2064 +#define CLK_CON_GAT_GOUT_IS_PPMU_IS0_ACLK 0x2074 +#define CLK_CON_GAT_GOUT_IS_PPMU_IS0_PCLK 0x2078 +#define CLK_CON_GAT_GOUT_IS_PPMU_IS1_ACLK 0x207c +#define CLK_CON_GAT_GOUT_IS_PPMU_IS1_PCLK 0x2080 +#define CLK_CON_GAT_GOUT_IS_SYSMMU_IS0_CLK_S1 0x2098 +#define CLK_CON_GAT_GOUT_IS_SYSMMU_IS1_CLK_S1 0x209c +#define CLK_CON_GAT_GOUT_IS_SYSREG_PCLK 0x20a0 + +static const unsigned long is_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_IS_BUS_USER, + PLL_CON0_MUX_CLKCMU_IS_GDC_USER, + PLL_CON0_MUX_CLKCMU_IS_ITP_USER, + PLL_CON0_MUX_CLKCMU_IS_VRA_USER, + CLK_CON_DIV_DIV_CLK_IS_BUSP, + CLK_CON_GAT_CLK_IS_CMU_IS_PCLK, + CLK_CON_GAT_GOUT_IS_CSIS0_ACLK, + CLK_CON_GAT_GOUT_IS_CSIS1_ACLK, + CLK_CON_GAT_GOUT_IS_CSIS2_ACLK, + CLK_CON_GAT_GOUT_IS_TZPC_PCLK, + CLK_CON_GAT_GOUT_IS_CLK_CSIS_DMA, + CLK_CON_GAT_GOUT_IS_CLK_GDC, + CLK_CON_GAT_GOUT_IS_CLK_IPP, + CLK_CON_GAT_GOUT_IS_CLK_ITP, + CLK_CON_GAT_GOUT_IS_CLK_MCSC, + CLK_CON_GAT_GOUT_IS_CLK_VRA, + CLK_CON_GAT_GOUT_IS_PPMU_IS0_ACLK, + CLK_CON_GAT_GOUT_IS_PPMU_IS0_PCLK, + CLK_CON_GAT_GOUT_IS_PPMU_IS1_ACLK, + CLK_CON_GAT_GOUT_IS_PPMU_IS1_PCLK, + CLK_CON_GAT_GOUT_IS_SYSMMU_IS0_CLK_S1, + CLK_CON_GAT_GOUT_IS_SYSMMU_IS1_CLK_S1, + CLK_CON_GAT_GOUT_IS_SYSREG_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_IS */ +PNAME(mout_is_bus_user_p) = { "oscclk", "dout_is_bus" }; +PNAME(mout_is_itp_user_p) = { "oscclk", "dout_is_itp" }; +PNAME(mout_is_vra_user_p) = { "oscclk", "dout_is_vra" }; +PNAME(mout_is_gdc_user_p) = { "oscclk", "dout_is_gdc" }; + +static const struct samsung_mux_clock is_mux_clks[] __initconst = { + MUX(CLK_MOUT_IS_BUS_USER, "mout_is_bus_user", mout_is_bus_user_p, + PLL_CON0_MUX_CLKCMU_IS_BUS_USER, 4, 1), + MUX(CLK_MOUT_IS_ITP_USER, "mout_is_itp_user", mout_is_itp_user_p, + PLL_CON0_MUX_CLKCMU_IS_ITP_USER, 4, 1), + MUX(CLK_MOUT_IS_VRA_USER, "mout_is_vra_user", mout_is_vra_user_p, + PLL_CON0_MUX_CLKCMU_IS_VRA_USER, 4, 1), + MUX(CLK_MOUT_IS_GDC_USER, "mout_is_gdc_user", mout_is_gdc_user_p, + PLL_CON0_MUX_CLKCMU_IS_GDC_USER, 4, 1), +}; + +static const struct samsung_div_clock is_div_clks[] __initconst = { + DIV(CLK_DOUT_IS_BUSP, "dout_is_busp", "mout_is_bus_user", + CLK_CON_DIV_DIV_CLK_IS_BUSP, 0, 2), +}; + +static const struct samsung_gate_clock is_gate_clks[] __initconst = { + /* TODO: Should be enabled in IS driver */ + GATE(CLK_GOUT_IS_CMU_IS_PCLK, "gout_is_cmu_is_pclk", "dout_is_busp", + CLK_CON_GAT_CLK_IS_CMU_IS_PCLK, 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_IS_CSIS0_ACLK, "gout_is_csis0_aclk", "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_CSIS0_ACLK, 21, 0, 0), + GATE(CLK_GOUT_IS_CSIS1_ACLK, "gout_is_csis1_aclk", "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_CSIS1_ACLK, 21, 0, 0), + GATE(CLK_GOUT_IS_CSIS2_ACLK, "gout_is_csis2_aclk", "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_CSIS2_ACLK, 21, 0, 0), + GATE(CLK_GOUT_IS_TZPC_PCLK, "gout_is_tzpc_pclk", "dout_is_busp", + CLK_CON_GAT_GOUT_IS_TZPC_PCLK, 21, 0, 0), + GATE(CLK_GOUT_IS_CSIS_DMA_CLK, "gout_is_csis_dma_clk", + "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_CLK_CSIS_DMA, 21, 0, 0), + GATE(CLK_GOUT_IS_GDC_CLK, "gout_is_gdc_clk", "mout_is_gdc_user", + CLK_CON_GAT_GOUT_IS_CLK_GDC, 21, 0, 0), + GATE(CLK_GOUT_IS_IPP_CLK, "gout_is_ipp_clk", "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_CLK_IPP, 21, 0, 0), + GATE(CLK_GOUT_IS_ITP_CLK, "gout_is_itp_clk", "mout_is_itp_user", + CLK_CON_GAT_GOUT_IS_CLK_ITP, 21, 0, 0), + GATE(CLK_GOUT_IS_MCSC_CLK, "gout_is_mcsc_clk", "mout_is_itp_user", + CLK_CON_GAT_GOUT_IS_CLK_MCSC, 21, 0, 0), + GATE(CLK_GOUT_IS_VRA_CLK, "gout_is_vra_clk", "mout_is_vra_user", + CLK_CON_GAT_GOUT_IS_CLK_VRA, 21, 0, 0), + GATE(CLK_GOUT_IS_PPMU_IS0_ACLK, "gout_is_ppmu_is0_aclk", + "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_PPMU_IS0_ACLK, 21, 0, 0), + GATE(CLK_GOUT_IS_PPMU_IS0_PCLK, "gout_is_ppmu_is0_pclk", "dout_is_busp", + CLK_CON_GAT_GOUT_IS_PPMU_IS0_PCLK, 21, 0, 0), + GATE(CLK_GOUT_IS_PPMU_IS1_ACLK, "gout_is_ppmu_is1_aclk", + "mout_is_itp_user", + CLK_CON_GAT_GOUT_IS_PPMU_IS1_ACLK, 21, 0, 0), + GATE(CLK_GOUT_IS_PPMU_IS1_PCLK, "gout_is_ppmu_is1_pclk", "dout_is_busp", + CLK_CON_GAT_GOUT_IS_PPMU_IS1_PCLK, 21, 0, 0), + GATE(CLK_GOUT_IS_SYSMMU_IS0_CLK, "gout_is_sysmmu_is0_clk", + "mout_is_bus_user", + CLK_CON_GAT_GOUT_IS_SYSMMU_IS0_CLK_S1, 21, 0, 0), + GATE(CLK_GOUT_IS_SYSMMU_IS1_CLK, "gout_is_sysmmu_is1_clk", + "mout_is_itp_user", + CLK_CON_GAT_GOUT_IS_SYSMMU_IS1_CLK_S1, 21, 0, 0), + GATE(CLK_GOUT_IS_SYSREG_PCLK, "gout_is_sysreg_pclk", "dout_is_busp", + CLK_CON_GAT_GOUT_IS_SYSREG_PCLK, 21, 0, 0), +}; + +static const struct samsung_cmu_info is_cmu_info __initconst = { + .mux_clks = is_mux_clks, + .nr_mux_clks = ARRAY_SIZE(is_mux_clks), + .div_clks = is_div_clks, + .nr_div_clks = ARRAY_SIZE(is_div_clks), + .gate_clks = is_gate_clks, + .nr_gate_clks = ARRAY_SIZE(is_gate_clks), + .nr_clk_ids = IS_NR_CLK, + .clk_regs = is_clk_regs, + .nr_clk_regs = ARRAY_SIZE(is_clk_regs), + .clk_name = "dout_is_bus", +}; + +/* ---- CMU_MFCMSCL --------------------------------------------------------- */ + +#define PLL_CON0_MUX_CLKCMU_MFCMSCL_JPEG_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_MFCMSCL_M2M_USER 0x0610 +#define PLL_CON0_MUX_CLKCMU_MFCMSCL_MCSC_USER 0x0620 +#define PLL_CON0_MUX_CLKCMU_MFCMSCL_MFC_USER 0x0630 +#define CLK_CON_DIV_DIV_CLK_MFCMSCL_BUSP 0x1800 +#define CLK_CON_GAT_CLK_MFCMSCL_CMU_MFCMSCL_PCLK 0x2000 +#define CLK_CON_GAT_GOUT_MFCMSCL_TZPC_PCLK 0x2038 +#define CLK_CON_GAT_GOUT_MFCMSCL_JPEG_ACLK 0x203c +#define CLK_CON_GAT_GOUT_MFCMSCL_M2M_ACLK 0x2048 +#define CLK_CON_GAT_GOUT_MFCMSCL_MCSC_I_CLK 0x204c +#define CLK_CON_GAT_GOUT_MFCMSCL_MFC_ACLK 0x2050 +#define CLK_CON_GAT_GOUT_MFCMSCL_PPMU_ACLK 0x2054 +#define CLK_CON_GAT_GOUT_MFCMSCL_PPMU_PCLK 0x2058 +#define CLK_CON_GAT_GOUT_MFCMSCL_SYSMMU_CLK_S1 0x2074 +#define CLK_CON_GAT_GOUT_MFCMSCL_SYSREG_PCLK 0x2078 + +static const unsigned long mfcmscl_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_MFCMSCL_JPEG_USER, + PLL_CON0_MUX_CLKCMU_MFCMSCL_M2M_USER, + PLL_CON0_MUX_CLKCMU_MFCMSCL_MCSC_USER, + PLL_CON0_MUX_CLKCMU_MFCMSCL_MFC_USER, + CLK_CON_DIV_DIV_CLK_MFCMSCL_BUSP, + CLK_CON_GAT_CLK_MFCMSCL_CMU_MFCMSCL_PCLK, + CLK_CON_GAT_GOUT_MFCMSCL_TZPC_PCLK, + CLK_CON_GAT_GOUT_MFCMSCL_JPEG_ACLK, + CLK_CON_GAT_GOUT_MFCMSCL_M2M_ACLK, + CLK_CON_GAT_GOUT_MFCMSCL_MCSC_I_CLK, + CLK_CON_GAT_GOUT_MFCMSCL_MFC_ACLK, + CLK_CON_GAT_GOUT_MFCMSCL_PPMU_ACLK, + CLK_CON_GAT_GOUT_MFCMSCL_PPMU_PCLK, + CLK_CON_GAT_GOUT_MFCMSCL_SYSMMU_CLK_S1, + CLK_CON_GAT_GOUT_MFCMSCL_SYSREG_PCLK, +}; + +/* List of parent clocks for Muxes in CMU_MFCMSCL */ +PNAME(mout_mfcmscl_mfc_user_p) = { "oscclk", "dout_mfcmscl_mfc" }; +PNAME(mout_mfcmscl_m2m_user_p) = { "oscclk", "dout_mfcmscl_m2m" }; +PNAME(mout_mfcmscl_mcsc_user_p) = { "oscclk", "dout_mfcmscl_mcsc" }; +PNAME(mout_mfcmscl_jpeg_user_p) = { "oscclk", "dout_mfcmscl_jpeg" }; + +static const struct samsung_mux_clock mfcmscl_mux_clks[] __initconst = { + MUX(CLK_MOUT_MFCMSCL_MFC_USER, "mout_mfcmscl_mfc_user", + mout_mfcmscl_mfc_user_p, + PLL_CON0_MUX_CLKCMU_MFCMSCL_MFC_USER, 4, 1), + MUX(CLK_MOUT_MFCMSCL_M2M_USER, "mout_mfcmscl_m2m_user", + mout_mfcmscl_m2m_user_p, + PLL_CON0_MUX_CLKCMU_MFCMSCL_M2M_USER, 4, 1), + MUX(CLK_MOUT_MFCMSCL_MCSC_USER, "mout_mfcmscl_mcsc_user", + mout_mfcmscl_mcsc_user_p, + PLL_CON0_MUX_CLKCMU_MFCMSCL_MCSC_USER, 4, 1), + MUX(CLK_MOUT_MFCMSCL_JPEG_USER, "mout_mfcmscl_jpeg_user", + mout_mfcmscl_jpeg_user_p, + PLL_CON0_MUX_CLKCMU_MFCMSCL_JPEG_USER, 4, 1), +}; + +static const struct samsung_div_clock mfcmscl_div_clks[] __initconst = { + DIV(CLK_DOUT_MFCMSCL_BUSP, "dout_mfcmscl_busp", "mout_mfcmscl_mfc_user", + CLK_CON_DIV_DIV_CLK_MFCMSCL_BUSP, 0, 3), +}; + +static const struct samsung_gate_clock mfcmscl_gate_clks[] __initconst = { + /* TODO: Should be enabled in MFC driver */ + GATE(CLK_GOUT_MFCMSCL_CMU_MFCMSCL_PCLK, "gout_mfcmscl_cmu_mfcmscl_pclk", + "dout_mfcmscl_busp", CLK_CON_GAT_CLK_MFCMSCL_CMU_MFCMSCL_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_MFCMSCL_TZPC_PCLK, "gout_mfcmscl_tzpc_pclk", + "dout_mfcmscl_busp", CLK_CON_GAT_GOUT_MFCMSCL_TZPC_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_JPEG_ACLK, "gout_mfcmscl_jpeg_aclk", + "mout_mfcmscl_jpeg_user", CLK_CON_GAT_GOUT_MFCMSCL_JPEG_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_M2M_ACLK, "gout_mfcmscl_m2m_aclk", + "mout_mfcmscl_m2m_user", CLK_CON_GAT_GOUT_MFCMSCL_M2M_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_MCSC_CLK, "gout_mfcmscl_mcsc_clk", + "mout_mfcmscl_mcsc_user", CLK_CON_GAT_GOUT_MFCMSCL_MCSC_I_CLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_MFC_ACLK, "gout_mfcmscl_mfc_aclk", + "mout_mfcmscl_mfc_user", CLK_CON_GAT_GOUT_MFCMSCL_MFC_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_PPMU_ACLK, "gout_mfcmscl_ppmu_aclk", + "mout_mfcmscl_mfc_user", CLK_CON_GAT_GOUT_MFCMSCL_PPMU_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_PPMU_PCLK, "gout_mfcmscl_ppmu_pclk", + "dout_mfcmscl_busp", CLK_CON_GAT_GOUT_MFCMSCL_PPMU_PCLK, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_SYSMMU_CLK, "gout_mfcmscl_sysmmu_clk", + "mout_mfcmscl_mfc_user", CLK_CON_GAT_GOUT_MFCMSCL_SYSMMU_CLK_S1, + 21, 0, 0), + GATE(CLK_GOUT_MFCMSCL_SYSREG_PCLK, "gout_mfcmscl_sysreg_pclk", + "dout_mfcmscl_busp", CLK_CON_GAT_GOUT_MFCMSCL_SYSREG_PCLK, + 21, 0, 0), +}; + +static const struct samsung_cmu_info mfcmscl_cmu_info __initconst = { + .mux_clks = mfcmscl_mux_clks, + .nr_mux_clks = ARRAY_SIZE(mfcmscl_mux_clks), + .div_clks = mfcmscl_div_clks, + .nr_div_clks = ARRAY_SIZE(mfcmscl_div_clks), + .gate_clks = mfcmscl_gate_clks, + .nr_gate_clks = ARRAY_SIZE(mfcmscl_gate_clks), + .nr_clk_ids = MFCMSCL_NR_CLK, + .clk_regs = mfcmscl_clk_regs, + .nr_clk_regs = ARRAY_SIZE(mfcmscl_clk_regs), + .clk_name = "dout_mfcmscl_mfc", +}; + /* ---- CMU_PERI ------------------------------------------------------------ */ /* Register Offset definitions for CMU_PERI (0x10030000) */ @@ -963,7 +1630,7 @@ static const unsigned long dpu_clk_regs[] __initconst = { CLK_CON_GAT_GOUT_DPU_SYSREG_PCLK, }; -/* List of parent clocks for Muxes in CMU_CORE */ +/* List of parent clocks for Muxes in CMU_DPU */ PNAME(mout_dpu_user_p) = { "oscclk", "dout_dpu" }; static const struct samsung_mux_clock dpu_mux_clks[] __initconst = { @@ -1027,12 +1694,21 @@ static const struct of_device_id exynos850_cmu_of_match[] = { { .compatible = "samsung,exynos850-cmu-apm", .data = &apm_cmu_info, + }, { + .compatible = "samsung,exynos850-cmu-aud", + .data = &aud_cmu_info, }, { .compatible = "samsung,exynos850-cmu-cmgp", .data = &cmgp_cmu_info, }, { .compatible = "samsung,exynos850-cmu-hsi", .data = &hsi_cmu_info, + }, { + .compatible = "samsung,exynos850-cmu-is", + .data = &is_cmu_info, + }, { + .compatible = "samsung,exynos850-cmu-mfcmscl", + .data = &mfcmscl_cmu_info, }, { .compatible = "samsung,exynos850-cmu-core", .data = &core_cmu_info, diff --git a/drivers/clk/samsung/clk-exynosautov9.c b/drivers/clk/samsung/clk-exynosautov9.c index d9e1f8e4a7b4555581ba7dec49533677d0bbd41d..7b16320bba667c6330179345dcdc308861fa912f 100644 --- a/drivers/clk/samsung/clk-exynosautov9.c +++ b/drivers/clk/samsung/clk-exynosautov9.c @@ -1067,6 +1067,373 @@ static const struct samsung_cmu_info core_cmu_info __initconst = { .clk_name = "dout_clkcmu_core_bus", }; +/* ---- CMU_FSYS0 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_FSYS2 (0x17700000) */ +#define PLL_CON0_MUX_CLKCMU_FSYS0_BUS_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_FSYS0_PCIE_USER 0x0610 +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_FSYS0_CMU_FSYS0_IPCLKPORT_PCLK 0x2000 + +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_PHY_REFCLK_IN 0x2004 +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_PHY_REFCLK_IN 0x2008 +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_PHY_REFCLK_IN 0x200c +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_PHY_REFCLK_IN 0x2010 +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_PHY_REFCLK_IN 0x2014 +#define CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_PHY_REFCLK_IN 0x2018 + +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_DBI_ACLK 0x205c +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_MSTR_ACLK 0x2060 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_SLV_ACLK 0x2064 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_DBI_ACLK 0x206c +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_MSTR_ACLK 0x2070 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_SLV_ACLK 0x2074 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_PIPE_CLK 0x207c + +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_DBI_ACLK 0x2084 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_MSTR_ACLK 0x2088 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_SLV_ACLK 0x208c +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_DBI_ACLK 0x2094 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_MSTR_ACLK 0x2098 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_SLV_ACLK 0x209c +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_PIPE_CLK 0x20a4 + +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_DBI_ACLK 0x20ac +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_MSTR_ACLK 0x20b0 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_SLV_ACLK 0x20b4 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_DBI_ACLK 0x20bc +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_MSTR_ACLK 0x20c0 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_SLV_ACLK 0x20c4 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_PIPE_CLK 0x20cc + +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L0_CLK 0x20d4 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L1_CLK 0x20d8 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_4L_CLK 0x20dc +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L0_CLK 0x20e0 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L1_CLK 0x20e4 +#define CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_4L_CLK 0x20e8 + + +static const unsigned long fsys0_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_FSYS0_BUS_USER, + PLL_CON0_MUX_CLKCMU_FSYS0_PCIE_USER, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_FSYS0_CMU_FSYS0_IPCLKPORT_PCLK, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_PHY_REFCLK_IN, + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_PHY_REFCLK_IN, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_PIPE_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_PIPE_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_DBI_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_MSTR_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_SLV_ACLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_PIPE_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L0_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L1_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_4L_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L0_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L1_CLK, + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_4L_CLK, +}; + +/* List of parent clocks for Muxes in CMU_FSYS0 */ +PNAME(mout_fsys0_bus_user_p) = { "oscclk", "dout_clkcmu_fsys0_bus" }; +PNAME(mout_fsys0_pcie_user_p) = { "oscclk", "dout_clkcmu_fsys0_pcie" }; + +static const struct samsung_mux_clock fsys0_mux_clks[] __initconst = { + MUX(CLK_MOUT_FSYS0_BUS_USER, "mout_fsys0_bus_user", + mout_fsys0_bus_user_p, PLL_CON0_MUX_CLKCMU_FSYS0_BUS_USER, 4, 1), + MUX(CLK_MOUT_FSYS0_PCIE_USER, "mout_fsys0_pcie_user", + mout_fsys0_pcie_user_p, PLL_CON0_MUX_CLKCMU_FSYS0_PCIE_USER, 4, 1), +}; + +static const struct samsung_gate_clock fsys0_gate_clks[] __initconst = { + GATE(CLK_GOUT_FSYS0_BUS_PCLK, "gout_fsys0_bus_pclk", + "mout_fsys0_bus_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_FSYS0_CMU_FSYS0_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + + /* Gen3 2L0 */ + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_REFCLK, + "gout_fsys0_pcie_gen3_2l0_x1_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_REFCLK, + "gout_fsys0_pcie_gen3_2l0_x2_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_DBI_ACLK, + "gout_fsys0_pcie_gen3_2l0_x1_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_MSTR_ACLK, + "gout_fsys0_pcie_gen3_2l0_x1_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_SLV_ACLK, + "gout_fsys0_pcie_gen3_2l0_x1_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X1_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_DBI_ACLK, + "gout_fsys0_pcie_gen3_2l0_x2_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_MSTR_ACLK, + "gout_fsys0_pcie_gen3_2l0_x2_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_SLV_ACLK, + "gout_fsys0_pcie_gen3_2l0_x2_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L0_X2_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3A_2L0_CLK, + "gout_fsys0_pcie_gen3a_2l0_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L0_CLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3B_2L0_CLK, + "gout_fsys0_pcie_gen3b_2l0_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L0_CLK, + 21, 0, 0), + + /* Gen3 2L1 */ + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_REFCLK, + "gout_fsys0_pcie_gen3_2l1_x1_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_REFCLK, + "gout_fsys0_pcie_gen3_2l1_x2_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_DBI_ACLK, + "gout_fsys0_pcie_gen3_2l1_x1_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_MSTR_ACLK, + "gout_fsys0_pcie_gen3_2l1_x1_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_SLV_ACLK, + "gout_fsys0_pcie_gen3_2l1_x1_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X1_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_DBI_ACLK, + "gout_fsys0_pcie_gen3_2l1_x2_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_MSTR_ACLK, + "gout_fsys0_pcie_gen3_2l1_x2_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_SLV_ACLK, + "gout_fsys0_pcie_gen3_2l1_x2_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_2L1_X2_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3A_2L1_CLK, + "gout_fsys0_pcie_gen3a_2l1_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_2L1_CLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3B_2L1_CLK, + "gout_fsys0_pcie_gen3b_2l1_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_2L1_CLK, + 21, 0, 0), + + /* Gen3 4L */ + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_REFCLK, + "gout_fsys0_pcie_gen3_4l_x2_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_REFCLK, + "gout_fsys0_pcie_gen3_4l_x4_refclk", "mout_fsys0_pcie_user", + CLK_CON_GAT_CLK_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_PHY_REFCLK_IN, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_DBI_ACLK, + "gout_fsys0_pcie_gen3_4l_x2_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_MSTR_ACLK, + "gout_fsys0_pcie_gen3_4l_x2_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_SLV_ACLK, + "gout_fsys0_pcie_gen3_4l_x2_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X2_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_DBI_ACLK, + "gout_fsys0_pcie_gen3_4l_x4_dbi_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_DBI_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_MSTR_ACLK, + "gout_fsys0_pcie_gen3_4l_x4_mstr_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_MSTR_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_SLV_ACLK, + "gout_fsys0_pcie_gen3_4l_x4_slv_aclk", "mout_fsys0_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3_4L_X4_SLV_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3A_4L_CLK, + "gout_fsys0_pcie_gen3a_4l_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3A_4L_CLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS0_PCIE_GEN3B_4L_CLK, + "gout_fsys0_pcie_gen3b_4l_clk", "mout_fsys0_pcie_user", + CLK_CON_GAT_GOUT_BLK_FSYS0_UID_PCIE_GEN3B_4L_CLK, + 21, 0, 0), +}; + +static const struct samsung_cmu_info fsys0_cmu_info __initconst = { + .mux_clks = fsys0_mux_clks, + .nr_mux_clks = ARRAY_SIZE(fsys0_mux_clks), + .gate_clks = fsys0_gate_clks, + .nr_gate_clks = ARRAY_SIZE(fsys0_gate_clks), + .nr_clk_ids = FSYS0_NR_CLK, + .clk_regs = fsys0_clk_regs, + .nr_clk_regs = ARRAY_SIZE(fsys0_clk_regs), + .clk_name = "dout_clkcmu_fsys0_bus", +}; + +/* ---- CMU_FSYS1 ---------------------------------------------------------- */ + +/* Register Offset definitions for CMU_FSYS1 (0x17040000) */ +#define PLL_LOCKTIME_PLL_MMC 0x0000 +#define PLL_CON0_PLL_MMC 0x0100 +#define PLL_CON3_PLL_MMC 0x010c +#define PLL_CON0_MUX_CLKCMU_FSYS1_BUS_USER 0x0600 +#define PLL_CON0_MUX_CLKCMU_FSYS1_MMC_CARD_USER 0x0610 +#define PLL_CON0_MUX_CLKCMU_FSYS1_USBDRD_USER 0x0620 + +#define CLK_CON_MUX_MUX_CLK_FSYS1_MMC_CARD 0x1000 +#define CLK_CON_DIV_DIV_CLK_FSYS1_MMC_CARD 0x1800 + +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_FSYS1_CMU_FSYS1_IPCLKPORT_PCLK 0x2018 +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_MMC_CARD_IPCLKPORT_SDCLKIN 0x202c +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_MMC_CARD_IPCLKPORT_I_ACLK 0x2028 + +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB20DRD_0_REF_CLK_40 0x204c +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB20DRD_1_REF_CLK_40 0x2058 +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB30DRD_0_REF_CLK_40 0x2064 +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB30DRD_1_REF_CLK_40 0x2070 + +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB2_0_IPCLKPORT_ACLK 0x2074 +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB2_1_IPCLKPORT_ACLK 0x2078 +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB3_0_IPCLKPORT_ACLK 0x207c +#define CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB3_1_IPCLKPORT_ACLK 0x2080 + +static const unsigned long fsys1_clk_regs[] __initconst = { + PLL_CON0_MUX_CLKCMU_FSYS1_BUS_USER, +}; + +static const struct samsung_pll_clock fsys1_pll_clks[] __initconst = { + PLL(pll_0831x, FOUT_MMC_PLL, "fout_mmc_pll", "oscclk", + PLL_LOCKTIME_PLL_MMC, PLL_CON3_PLL_MMC, NULL), +}; + +/* List of parent clocks for Muxes in CMU_FSYS1 */ +PNAME(mout_fsys1_bus_user_p) = { "oscclk", "dout_clkcmu_fsys1_bus" }; +PNAME(mout_fsys1_mmc_pll_p) = { "oscclk", "fout_mmc_pll" }; +PNAME(mout_fsys1_mmc_card_user_p) = { "oscclk", "gout_clkcmu_fsys1_mmc_card" }; +PNAME(mout_fsys1_usbdrd_user_p) = { "oscclk", "dout_clkcmu_fsys1_usbdrd" }; +PNAME(mout_fsys1_mmc_card_p) = { "mout_fsys1_mmc_card_user", + "mout_fsys1_mmc_pll" }; + +static const struct samsung_mux_clock fsys1_mux_clks[] __initconst = { + MUX(CLK_MOUT_FSYS1_BUS_USER, "mout_fsys1_bus_user", + mout_fsys1_bus_user_p, PLL_CON0_MUX_CLKCMU_FSYS1_BUS_USER, 4, 1), + MUX(CLK_MOUT_FSYS1_MMC_PLL, "mout_fsys1_mmc_pll", mout_fsys1_mmc_pll_p, + PLL_CON0_PLL_MMC, 4, 1), + MUX(CLK_MOUT_FSYS1_MMC_CARD_USER, "mout_fsys1_mmc_card_user", + mout_fsys1_mmc_card_user_p, PLL_CON0_MUX_CLKCMU_FSYS1_MMC_CARD_USER, + 4, 1), + MUX(CLK_MOUT_FSYS1_USBDRD_USER, "mout_fsys1_usbdrd_user", + mout_fsys1_usbdrd_user_p, PLL_CON0_MUX_CLKCMU_FSYS1_USBDRD_USER, + 4, 1), + MUX(CLK_MOUT_FSYS1_MMC_CARD, "mout_fsys1_mmc_card", + mout_fsys1_mmc_card_p, CLK_CON_MUX_MUX_CLK_FSYS1_MMC_CARD, + 0, 1), +}; + +static const struct samsung_div_clock fsys1_div_clks[] __initconst = { + DIV(CLK_DOUT_FSYS1_MMC_CARD, "dout_fsys1_mmc_card", + "mout_fsys1_mmc_card", + CLK_CON_DIV_DIV_CLK_FSYS1_MMC_CARD, 0, 9), +}; + +static const struct samsung_gate_clock fsys1_gate_clks[] __initconst = { + GATE(CLK_GOUT_FSYS1_PCLK, "gout_fsys1_pclk", "mout_fsys1_bus_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_FSYS1_CMU_FSYS1_IPCLKPORT_PCLK, + 21, CLK_IGNORE_UNUSED, 0), + GATE(CLK_GOUT_FSYS1_MMC_CARD_SDCLKIN, "gout_fsys1_mmc_card_sdclkin", + "dout_fsys1_mmc_card", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_MMC_CARD_IPCLKPORT_SDCLKIN, + 21, CLK_SET_RATE_PARENT, 0), + GATE(CLK_GOUT_FSYS1_MMC_CARD_ACLK, "gout_fsys1_mmc_card_aclk", + "dout_fsys1_mmc_card", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_MMC_CARD_IPCLKPORT_I_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB20DRD_0_REFCLK, "gout_fsys1_usb20drd_0_refclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB20DRD_0_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB20DRD_1_REFCLK, "gout_fsys1_usb20drd_1_refclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB20DRD_1_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB30DRD_0_REFCLK, "gout_fsys1_usb30drd_0_refclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB30DRD_0_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB30DRD_1_REFCLK, "gout_fsys1_usb30drd_1_refclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_USB30DRD_1_REF_CLK_40, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB20_0_ACLK, "gout_fsys1_usb20_0_aclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB2_0_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB20_1_ACLK, "gout_fsys1_usb20_1_aclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB2_1_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB30_0_ACLK, "gout_fsys1_usb30_0_aclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB3_0_IPCLKPORT_ACLK, + 21, 0, 0), + GATE(CLK_GOUT_FSYS1_USB30_1_ACLK, "gout_fsys1_usb30_1_aclk", + "mout_fsys1_usbdrd_user", + CLK_CON_GAT_GOUT_BLK_FSYS1_UID_US_D_USB3_1_IPCLKPORT_ACLK, + 21, 0, 0), +}; + +static const struct samsung_cmu_info fsys1_cmu_info __initconst = { + .pll_clks = fsys1_pll_clks, + .nr_pll_clks = ARRAY_SIZE(fsys1_pll_clks), + .mux_clks = fsys1_mux_clks, + .nr_mux_clks = ARRAY_SIZE(fsys1_mux_clks), + .div_clks = fsys1_div_clks, + .nr_div_clks = ARRAY_SIZE(fsys1_div_clks), + .gate_clks = fsys1_gate_clks, + .nr_gate_clks = ARRAY_SIZE(fsys1_gate_clks), + .nr_clk_ids = FSYS1_NR_CLK, + .clk_regs = fsys1_clk_regs, + .nr_clk_regs = ARRAY_SIZE(fsys1_clk_regs), + .clk_name = "dout_clkcmu_fsys1_bus", +}; + /* ---- CMU_FSYS2 ---------------------------------------------------------- */ /* Register Offset definitions for CMU_FSYS2 (0x17c00000) */ @@ -1170,9 +1537,9 @@ static const struct samsung_cmu_info fsys2_cmu_info __initconst = { #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_2 0x2058 #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_3 0x205c #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_4 0x2060 -#define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_7 0x206c #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_5 0x2064 #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_6 0x2068 +#define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_7 0x206c #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_8 0x2070 #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_9 0x2074 #define CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_10 0x204c @@ -1330,6 +1697,10 @@ static const struct samsung_gate_clock peric0_gate_clks[] __initconst = { "mout_peric0_bus_user", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_0, 21, 0, 0), + GATE(CLK_GOUT_PERIC0_PCLK_1, "gout_peric0_pclk_1", + "mout_peric0_bus_user", + CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_1, + 21, 0, 0), GATE(CLK_GOUT_PERIC0_PCLK_2, "gout_peric0_pclk_2", "mout_peric0_bus_user", CLK_CON_GAT_GOUT_BLK_PERIC0_UID_PERIC0_TOP0_IPCLKPORT_PCLK_2, @@ -1418,14 +1789,14 @@ static const struct samsung_cmu_info peric0_cmu_info __initconst = { #define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_IPCLK_11 0x2020 #define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_0 0x2044 #define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_1 0x2048 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_2 0x2058 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_3 0x205c -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_4 0x2060 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_7 0x206c -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_5 0x2064 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_6 0x2068 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_8 0x2070 -#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_9 0x2074 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_2 0x2054 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_3 0x2058 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_4 0x205c +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_5 0x2060 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_6 0x2064 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_7 0x2068 +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_8 0x206c +#define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_9 0x2070 #define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_10 0x204c #define CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_11 0x2050 @@ -1463,9 +1834,9 @@ static const unsigned long peric1_clk_regs[] __initconst = { CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_2, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_3, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_4, - CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_7, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_5, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_6, + CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_7, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_8, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_9, CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_10, @@ -1581,6 +1952,10 @@ static const struct samsung_gate_clock peric1_gate_clks[] __initconst = { "mout_peric1_bus_user", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_0, 21, 0, 0), + GATE(CLK_GOUT_PERIC1_PCLK_1, "gout_peric1_pclk_1", + "mout_peric1_bus_user", + CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_1, + 21, 0, 0), GATE(CLK_GOUT_PERIC1_PCLK_2, "gout_peric1_pclk_2", "mout_peric1_bus_user", CLK_CON_GAT_GOUT_BLK_PERIC1_UID_PERIC1_TOP0_IPCLKPORT_PCLK_2, @@ -1701,6 +2076,12 @@ static const struct of_device_id exynosautov9_cmu_of_match[] = { }, { .compatible = "samsung,exynosautov9-cmu-core", .data = &core_cmu_info, + }, { + .compatible = "samsung,exynosautov9-cmu-fsys0", + .data = &fsys0_cmu_info, + }, { + .compatible = "samsung,exynosautov9-cmu-fsys1", + .data = &fsys1_cmu_info, }, { .compatible = "samsung,exynosautov9-cmu-fsys2", .data = &fsys2_cmu_info, diff --git a/drivers/clk/spear/spear3xx_clock.c b/drivers/clk/spear/spear3xx_clock.c index 41717ff707f69cef32768507077737ffc73d40ca..ba87913031567d5102cb71d8d0d1e5b0d3f49d23 100644 --- a/drivers/clk/spear/spear3xx_clock.c +++ b/drivers/clk/spear/spear3xx_clock.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/drivers/clk/spear/spear6xx_clock.c b/drivers/clk/spear/spear6xx_clock.c index 490701ac9e938ccf69dea6db1a109e3ac439f9b9..c192a9141b86694235472ac2a7da4382234f05e8 100644 --- a/drivers/clk/spear/spear6xx_clock.c +++ b/drivers/clk/spear/spear6xx_clock.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include "clk.h" diff --git a/drivers/clk/sprd/Kconfig b/drivers/clk/sprd/Kconfig index e18c80fbe804e1742b576b61a80ce2d1ce1e3963..c744bd9d2f9601c0b34cff1946ec8400d4bb735c 100644 --- a/drivers/clk/sprd/Kconfig +++ b/drivers/clk/sprd/Kconfig @@ -21,4 +21,10 @@ config SPRD_SC9863A_CLK help Support for the global clock controller on sc9863a devices. Say Y if you want to use peripheral devices on sc9863a SoC. + +config SPRD_UMS512_CLK + tristate "Support for the Spreadtrum UMS512 clocks" + help + Support for the global clock controller on ums512 devices. + Say Y if you want to use peripheral devices on ums512 SoC. endif diff --git a/drivers/clk/sprd/Makefile b/drivers/clk/sprd/Makefile index 41d90e0d7863097610e58569b7195cb26888c2b9..f25b2c3904fbe577e79a21d6926a55646e4a3629 100644 --- a/drivers/clk/sprd/Makefile +++ b/drivers/clk/sprd/Makefile @@ -11,3 +11,4 @@ clk-sprd-y += pll.o ## SoC support obj-$(CONFIG_SPRD_SC9860_CLK) += sc9860-clk.o obj-$(CONFIG_SPRD_SC9863A_CLK) += sc9863a-clk.o +obj-$(CONFIG_SPRD_UMS512_CLK) += ums512-clk.o diff --git a/drivers/clk/sprd/common.c b/drivers/clk/sprd/common.c index d620bbbcdfc88eeb77446b8f1ce267e75103a917..ce81e4087a8fce2e8d2239e6d706de7f33b24c7d 100644 --- a/drivers/clk/sprd/common.c +++ b/drivers/clk/sprd/common.c @@ -41,7 +41,7 @@ int sprd_clk_regmap_init(struct platform_device *pdev, { void __iomem *base; struct device *dev = &pdev->dev; - struct device_node *node = dev->of_node; + struct device_node *node = dev->of_node, *np; struct regmap *regmap; if (of_find_property(node, "sprd,syscon", NULL)) { @@ -50,9 +50,10 @@ int sprd_clk_regmap_init(struct platform_device *pdev, pr_err("%s: failed to get syscon regmap\n", __func__); return PTR_ERR(regmap); } - } else if (of_device_is_compatible(of_get_parent(dev->of_node), - "syscon")) { - regmap = device_node_to_regmap(of_get_parent(dev->of_node)); + } else if (of_device_is_compatible(np = of_get_parent(node), "syscon") || + (of_node_put(np), 0)) { + regmap = device_node_to_regmap(np); + of_node_put(np); if (IS_ERR(regmap)) { dev_err(dev, "failed to get regmap from its parent.\n"); return PTR_ERR(regmap); diff --git a/drivers/clk/sprd/ums512-clk.c b/drivers/clk/sprd/ums512-clk.c new file mode 100644 index 0000000000000000000000000000000000000000..fc25bdd85e4eaba5a60ecba13e65fe67c4cfe0e8 --- /dev/null +++ b/drivers/clk/sprd/ums512-clk.c @@ -0,0 +1,2202 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Unisoc UMS512 clock driver + * + * Copyright (C) 2022 Unisoc, Inc. + * Author: Xiaolong Zhang + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "composite.h" +#include "div.h" +#include "gate.h" +#include "mux.h" +#include "pll.h" + +#define UMS512_MUX_FLAG \ + (CLK_GET_RATE_NOCACHE | CLK_SET_RATE_NO_REPARENT) + +/* pll gate clock */ +/* some pll clocks configure CLK_IGNORE_UNUSED because hw dvfs does not call + * clock interface. hw dvfs can not gate the pll clock. + */ +static CLK_FIXED_FACTOR_FW_NAME(clk_26m_aud, "clk-26m-aud", "ext-26m", 1, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_13m, "clk-13m", "ext-26m", 2, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_6m5, "clk-6m5", "ext-26m", 4, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_4m3, "clk-4m3", "ext-26m", 6, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_2m, "clk-2m", "ext-26m", 13, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_1m, "clk-1m", "ext-26m", 26, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(clk_250k, "clk-250k", "ext-26m", 104, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(rco_25m, "rco-25m", "rco-100m", 4, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(rco_4m, "rco-4m", "rco-100m", 25, 1, 0); +static CLK_FIXED_FACTOR_FW_NAME(rco_2m, "rco-2m", "rco-100m", 50, 1, 0); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(isppll_gate, "isppll-gate", "ext-26m", 0x8c, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll0_gate, "dpll0-gate", "ext-26m", 0x98, + 0x1000, BIT(0), 0, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(dpll1_gate, "dpll1-gate", "ext-26m", 0x9c, + 0x1000, BIT(0), 0, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(lpll_gate, "lpll-gate", "ext-26m", 0xa0, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(twpll_gate, "twpll-gate", "ext-26m", 0xa4, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(gpll_gate, "gpll-gate", "ext-26m", 0xa8, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(rpll_gate, "rpll-gate", "ext-26m", 0xac, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(cppll_gate, "cppll-gate", "ext-26m", 0xe4, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll0_gate, "mpll0-gate", "ext-26m", 0x190, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll1_gate, "mpll1-gate", "ext-26m", 0x194, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); +static SPRD_PLL_SC_GATE_CLK_FW_NAME(mpll2_gate, "mpll2-gate", "ext-26m", 0x198, + 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0, 240); + +static struct sprd_clk_common *ums512_pmu_gate_clks[] = { + /* address base is 0x327e0000 */ + &isppll_gate.common, + &dpll0_gate.common, + &dpll1_gate.common, + &lpll_gate.common, + &twpll_gate.common, + &gpll_gate.common, + &rpll_gate.common, + &cppll_gate.common, + &mpll0_gate.common, + &mpll1_gate.common, + &mpll2_gate.common, +}; + +static struct clk_hw_onecell_data ums512_pmu_gate_hws = { + .hws = { + [CLK_26M_AUD] = &clk_26m_aud.hw, + [CLK_13M] = &clk_13m.hw, + [CLK_6M5] = &clk_6m5.hw, + [CLK_4M3] = &clk_4m3.hw, + [CLK_2M] = &clk_2m.hw, + [CLK_1M] = &clk_1m.hw, + [CLK_250K] = &clk_250k.hw, + [CLK_RCO_25M] = &rco_25m.hw, + [CLK_RCO_4M] = &rco_4m.hw, + [CLK_RCO_2M] = &rco_2m.hw, + [CLK_ISPPLL_GATE] = &isppll_gate.common.hw, + [CLK_DPLL0_GATE] = &dpll0_gate.common.hw, + [CLK_DPLL1_GATE] = &dpll1_gate.common.hw, + [CLK_LPLL_GATE] = &lpll_gate.common.hw, + [CLK_TWPLL_GATE] = &twpll_gate.common.hw, + [CLK_GPLL_GATE] = &gpll_gate.common.hw, + [CLK_RPLL_GATE] = &rpll_gate.common.hw, + [CLK_CPPLL_GATE] = &cppll_gate.common.hw, + [CLK_MPLL0_GATE] = &mpll0_gate.common.hw, + [CLK_MPLL1_GATE] = &mpll1_gate.common.hw, + [CLK_MPLL2_GATE] = &mpll2_gate.common.hw, + }, + .num = CLK_PMU_GATE_NUM, +}; + +static struct sprd_clk_desc ums512_pmu_gate_desc = { + .clk_clks = ums512_pmu_gate_clks, + .num_clk_clks = ARRAY_SIZE(ums512_pmu_gate_clks), + .hw_clks = &ums512_pmu_gate_hws, +}; + +/* pll clock at g0 */ +static const u64 itable_dpll0[7] = { 6, 0, 0, + 1173000000ULL, 1475000000ULL, + 1855000000ULL, 1866000000ULL }; + +static struct clk_bit_field f_dpll0[PLL_FACT_MAX] = { + { .shift = 18, .width = 1 }, /* lock_done */ + { .shift = 0, .width = 1 }, /* div_s */ + { .shift = 67, .width = 1 }, /* mod_en */ + { .shift = 1, .width = 1 }, /* sdm_en */ + { .shift = 0, .width = 0 }, /* refin */ + { .shift = 4, .width = 3 }, /* icp */ + { .shift = 7, .width = 11 }, /* n */ + { .shift = 55, .width = 7 }, /* nint */ + { .shift = 32, .width = 23}, /* kint */ + { .shift = 0, .width = 0 }, /* prediv */ + { .shift = 0, .width = 0 }, /* postdiv */ +}; +static SPRD_PLL_HW(dpll0, "dpll0", &dpll0_gate.common.hw, 0x4, 3, + itable_dpll0, f_dpll0, 240, 1000, 1000, 0, 0); +static CLK_FIXED_FACTOR_HW(dpll0_58m31, "dpll0-58m31", &dpll0.common.hw, + 32, 1, 0); + +static struct sprd_clk_common *ums512_g0_pll_clks[] = { + /* address base is 0x32390000 */ + &dpll0.common, +}; + +static struct clk_hw_onecell_data ums512_g0_pll_hws = { + .hws = { + [CLK_DPLL0] = &dpll0.common.hw, + [CLK_DPLL0_58M31] = &dpll0_58m31.hw, + }, + .num = CLK_ANLG_PHY_G0_NUM, +}; + +static struct sprd_clk_desc ums512_g0_pll_desc = { + .clk_clks = ums512_g0_pll_clks, + .num_clk_clks = ARRAY_SIZE(ums512_g0_pll_clks), + .hw_clks = &ums512_g0_pll_hws, +}; + +/* pll clock at g2 */ +static const u64 itable_mpll[8] = { 7, 0, + 1400000000ULL, 1600000000ULL, + 1800000000ULL, 2000000000ULL, + 2200000000ULL, 2500000000ULL }; + +static struct clk_bit_field f_mpll[PLL_FACT_MAX] = { + { .shift = 17, .width = 1 }, /* lock_done */ + { .shift = 0, .width = 1 }, /* div_s */ + { .shift = 67, .width = 1 }, /* mod_en */ + { .shift = 1, .width = 1 }, /* sdm_en */ + { .shift = 0, .width = 0 }, /* refin */ + { .shift = 2, .width = 3 }, /* icp */ + { .shift = 5, .width = 11 }, /* n */ + { .shift = 55, .width = 7 }, /* nint */ + { .shift = 32, .width = 23}, /* kint */ + { .shift = 0, .width = 0 }, /* prediv */ + { .shift = 77, .width = 1 }, /* postdiv */ +}; +static SPRD_PLL_HW(mpll1, "mpll1", &mpll1_gate.common.hw, 0x0, 3, + itable_mpll, f_mpll, 240, 1000, 1000, 1, 1200000000); +static CLK_FIXED_FACTOR_HW(mpll1_63m38, "mpll1-63m38", &mpll1.common.hw, + 32, 1, 0); + +static struct sprd_clk_common *ums512_g2_pll_clks[] = { + /* address base is 0x323B0000 */ + &mpll1.common, +}; + +static struct clk_hw_onecell_data ums512_g2_pll_hws = { + .hws = { + [CLK_MPLL1] = &mpll1.common.hw, + [CLK_MPLL1_63M38] = &mpll1_63m38.hw, + }, + .num = CLK_ANLG_PHY_G2_NUM, +}; + +static struct sprd_clk_desc ums512_g2_pll_desc = { + .clk_clks = ums512_g2_pll_clks, + .num_clk_clks = ARRAY_SIZE(ums512_g2_pll_clks), + .hw_clks = &ums512_g2_pll_hws, +}; + +/* pll at g3 */ +static const u64 itable[8] = { 7, 0, 0, + 900000000ULL, 1100000000ULL, + 1300000000ULL, 1500000000ULL, + 1600000000ULL }; + +static struct clk_bit_field f_pll[PLL_FACT_MAX] = { + { .shift = 18, .width = 1 }, /* lock_done */ + { .shift = 0, .width = 1 }, /* div_s */ + { .shift = 67, .width = 1 }, /* mod_en */ + { .shift = 1, .width = 1 }, /* sdm_en */ + { .shift = 0, .width = 0 }, /* refin */ + { .shift = 2, .width = 3 }, /* icp */ + { .shift = 5, .width = 11 }, /* n */ + { .shift = 55, .width = 7 }, /* nint */ + { .shift = 32, .width = 23}, /* kint */ + { .shift = 0, .width = 0 }, /* prediv */ + { .shift = 77, .width = 1 }, /* postdiv */ +}; + +static SPRD_PLL_FW_NAME(rpll, "rpll", "ext-26m", 0x0, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); + +static SPRD_SC_GATE_CLK_FW_NAME(audio_gate, "audio-gate", "ext-26m", 0x24, + 0x1000, BIT(1), CLK_IGNORE_UNUSED, 0); + +static struct clk_bit_field f_mpll2[PLL_FACT_MAX] = { + { .shift = 16, .width = 1 }, /* lock_done */ + { .shift = 0, .width = 1 }, /* div_s */ + { .shift = 67, .width = 1 }, /* mod_en */ + { .shift = 1, .width = 1 }, /* sdm_en */ + { .shift = 0, .width = 0 }, /* refin */ + { .shift = 2, .width = 3 }, /* icp */ + { .shift = 5, .width = 11 }, /* n */ + { .shift = 55, .width = 7 }, /* nint */ + { .shift = 32, .width = 23}, /* kint */ + { .shift = 0, .width = 0 }, /* prediv */ + { .shift = 77, .width = 1 }, /* postdiv */ +}; +static SPRD_PLL_HW(mpll0, "mpll0", &mpll0_gate.common.hw, 0x54, 3, + itable_mpll, f_mpll, 240, 1000, 1000, 1, 1200000000); +static CLK_FIXED_FACTOR_HW(mpll0_56m88, "mpll0-56m88", &mpll0.common.hw, + 32, 1, 0); + +static const u64 itable_mpll2[6] = { 5, + 1200000000ULL, 1400000000ULL, + 1600000000ULL, 1800000000ULL, + 2000000000ULL }; + +static SPRD_PLL_HW(mpll2, "mpll2", &mpll2_gate.common.hw, 0x9c, 3, + itable_mpll2, f_mpll2, 240, 1000, 1000, 1, 1000000000); +static CLK_FIXED_FACTOR_HW(mpll2_47m13, "mpll2-47m13", &mpll2.common.hw, + 32, 1, 0); + +static struct sprd_clk_common *ums512_g3_pll_clks[] = { + /* address base is 0x323c0000 */ + &rpll.common, + &audio_gate.common, + &mpll0.common, + &mpll2.common, +}; + +static struct clk_hw_onecell_data ums512_g3_pll_hws = { + .hws = { + [CLK_RPLL] = &rpll.common.hw, + [CLK_AUDIO_GATE] = &audio_gate.common.hw, + [CLK_MPLL0] = &mpll0.common.hw, + [CLK_MPLL0_56M88] = &mpll0_56m88.hw, + [CLK_MPLL2] = &mpll2.common.hw, + [CLK_MPLL2_47M13] = &mpll2_47m13.hw, + }, + .num = CLK_ANLG_PHY_G3_NUM, +}; + +static struct sprd_clk_desc ums512_g3_pll_desc = { + .clk_clks = ums512_g3_pll_clks, + .num_clk_clks = ARRAY_SIZE(ums512_g3_pll_clks), + .hw_clks = &ums512_g3_pll_hws, +}; + +/* pll clock at gc */ +static SPRD_PLL_FW_NAME(twpll, "twpll", "ext-26m", 0x0, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); +static CLK_FIXED_FACTOR_HW(twpll_768m, "twpll-768m", &twpll.common.hw, + 2, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_384m, "twpll-384m", &twpll.common.hw, + 4, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_192m, "twpll-192m", &twpll.common.hw, + 8, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_96m, "twpll-96m", &twpll.common.hw, + 16, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_48m, "twpll-48m", &twpll.common.hw, + 32, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_24m, "twpll-24m", &twpll.common.hw, + 64, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_12m, "twpll-12m", &twpll.common.hw, + 128, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_512m, "twpll-512m", &twpll.common.hw, + 3, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_256m, "twpll-256m", &twpll.common.hw, + 6, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_128m, "twpll-128m", &twpll.common.hw, + 12, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_64m, "twpll-64m", &twpll.common.hw, + 24, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_307m2, "twpll-307m2", &twpll.common.hw, + 5, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_219m4, "twpll-219m4", &twpll.common.hw, + 7, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_170m6, "twpll-170m6", &twpll.common.hw, + 9, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_153m6, "twpll-153m6", &twpll.common.hw, + 10, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_76m8, "twpll-76m8", &twpll.common.hw, + 20, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_51m2, "twpll-51m2", &twpll.common.hw, + 30, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_38m4, "twpll-38m4", &twpll.common.hw, + 40, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_19m2, "twpll-19m2", &twpll.common.hw, + 80, 1, 0); +static CLK_FIXED_FACTOR_HW(twpll_12m29, "twpll-12m29", &twpll.common.hw, + 125, 1, 0); + +static SPRD_PLL_FW_NAME(lpll, "lpll", "ext-26m", 0x18, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); +static CLK_FIXED_FACTOR_HW(lpll_614m4, "lpll-614m4", &lpll.common.hw, + 2, 1, 0); +static CLK_FIXED_FACTOR_HW(lpll_409m6, "lpll-409m6", &lpll.common.hw, + 3, 1, 0); +static CLK_FIXED_FACTOR_HW(lpll_245m76, "lpll-245m76", &lpll.common.hw, + 5, 1, 0); +static CLK_FIXED_FACTOR_HW(lpll_30m72, "lpll-30m72", &lpll.common.hw, + 40, 1, 0); + +static SPRD_PLL_FW_NAME(isppll, "isppll", "ext-26m", 0x30, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); +static CLK_FIXED_FACTOR_HW(isppll_468m, "isppll-468m", &isppll.common.hw, + 2, 1, 0); +static CLK_FIXED_FACTOR_HW(isppll_78m, "isppll-78m", &isppll.common.hw, + 12, 1, 0); + +static SPRD_PLL_HW(gpll, "gpll", &gpll_gate.common.hw, 0x48, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); +static CLK_FIXED_FACTOR_HW(gpll_40m, "gpll-40m", &gpll.common.hw, + 20, 1, 0); + +static SPRD_PLL_HW(cppll, "cppll", &cppll_gate.common.hw, 0x60, 3, + itable, f_pll, 240, 1000, 1000, 1, 750000000); +static CLK_FIXED_FACTOR_HW(cppll_39m32, "cppll-39m32", &cppll.common.hw, + 26, 1, 0); + +static struct sprd_clk_common *ums512_gc_pll_clks[] = { + /* address base is 0x323e0000 */ + &twpll.common, + &lpll.common, + &isppll.common, + &gpll.common, + &cppll.common, +}; + +static struct clk_hw_onecell_data ums512_gc_pll_hws = { + .hws = { + [CLK_TWPLL] = &twpll.common.hw, + [CLK_TWPLL_768M] = &twpll_768m.hw, + [CLK_TWPLL_384M] = &twpll_384m.hw, + [CLK_TWPLL_192M] = &twpll_192m.hw, + [CLK_TWPLL_96M] = &twpll_96m.hw, + [CLK_TWPLL_48M] = &twpll_48m.hw, + [CLK_TWPLL_24M] = &twpll_24m.hw, + [CLK_TWPLL_12M] = &twpll_12m.hw, + [CLK_TWPLL_512M] = &twpll_512m.hw, + [CLK_TWPLL_256M] = &twpll_256m.hw, + [CLK_TWPLL_128M] = &twpll_128m.hw, + [CLK_TWPLL_64M] = &twpll_64m.hw, + [CLK_TWPLL_307M2] = &twpll_307m2.hw, + [CLK_TWPLL_219M4] = &twpll_219m4.hw, + [CLK_TWPLL_170M6] = &twpll_170m6.hw, + [CLK_TWPLL_153M6] = &twpll_153m6.hw, + [CLK_TWPLL_76M8] = &twpll_76m8.hw, + [CLK_TWPLL_51M2] = &twpll_51m2.hw, + [CLK_TWPLL_38M4] = &twpll_38m4.hw, + [CLK_TWPLL_19M2] = &twpll_19m2.hw, + [CLK_TWPLL_12M29] = &twpll_12m29.hw, + [CLK_LPLL] = &lpll.common.hw, + [CLK_LPLL_614M4] = &lpll_614m4.hw, + [CLK_LPLL_409M6] = &lpll_409m6.hw, + [CLK_LPLL_245M76] = &lpll_245m76.hw, + [CLK_LPLL_30M72] = &lpll_30m72.hw, + [CLK_ISPPLL] = &isppll.common.hw, + [CLK_ISPPLL_468M] = &isppll_468m.hw, + [CLK_ISPPLL_78M] = &isppll_78m.hw, + [CLK_GPLL] = &gpll.common.hw, + [CLK_GPLL_40M] = &gpll_40m.hw, + [CLK_CPPLL] = &cppll.common.hw, + [CLK_CPPLL_39M32] = &cppll_39m32.hw, + }, + .num = CLK_ANLG_PHY_GC_NUM, +}; + +static struct sprd_clk_desc ums512_gc_pll_desc = { + .clk_clks = ums512_gc_pll_clks, + .num_clk_clks = ARRAY_SIZE(ums512_gc_pll_clks), + .hw_clks = &ums512_gc_pll_hws, +}; + +/* ap ahb gates */ +static SPRD_SC_GATE_CLK_FW_NAME(dsi_eb, "dsi-eb", "ext-26m", + 0x0, 0x1000, BIT(0), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dispc_eb, "dispc-eb", "ext-26m", + 0x0, 0x1000, BIT(1), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(vsp_eb, "vsp-eb", "ext-26m", + 0x0, 0x1000, BIT(2), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(vdma_eb, "vdma-eb", "ext-26m", + 0x0, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dma_pub_eb, "dma-pub-eb", "ext-26m", + 0x0, 0x1000, BIT(4), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dma_sec_eb, "dma-sec-eb", "ext-26m", + 0x0, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ipi_eb, "ipi-eb", "ext-26m", + 0x0, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ahb_ckg_eb, "ahb-ckg-eb", "ext-26m", + 0x0, 0x1000, BIT(7), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(bm_clk_eb, "bm-clk-eb", "ext-26m", + 0x0, 0x1000, BIT(8), CLK_IGNORE_UNUSED, 0); + +static struct sprd_clk_common *ums512_apahb_gate[] = { + /* address base is 0x20100000 */ + &dsi_eb.common, + &dispc_eb.common, + &vsp_eb.common, + &vdma_eb.common, + &dma_pub_eb.common, + &dma_sec_eb.common, + &ipi_eb.common, + &ahb_ckg_eb.common, + &bm_clk_eb.common, +}; + +static struct clk_hw_onecell_data ums512_apahb_gate_hws = { + .hws = { + [CLK_DSI_EB] = &dsi_eb.common.hw, + [CLK_DISPC_EB] = &dispc_eb.common.hw, + [CLK_VSP_EB] = &vsp_eb.common.hw, + [CLK_VDMA_EB] = &vdma_eb.common.hw, + [CLK_DMA_PUB_EB] = &dma_pub_eb.common.hw, + [CLK_DMA_SEC_EB] = &dma_sec_eb.common.hw, + [CLK_IPI_EB] = &ipi_eb.common.hw, + [CLK_AHB_CKG_EB] = &ahb_ckg_eb.common.hw, + [CLK_BM_CLK_EB] = &bm_clk_eb.common.hw, + }, + .num = CLK_AP_AHB_GATE_NUM, +}; + +static struct sprd_clk_desc ums512_apahb_gate_desc = { + .clk_clks = ums512_apahb_gate, + .num_clk_clks = ARRAY_SIZE(ums512_apahb_gate), + .hw_clks = &ums512_apahb_gate_hws, +}; + +/* ap clks */ +static const struct clk_parent_data ap_apb_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_64m.hw }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_MUX_CLK_DATA(ap_apb_clk, "ap-apb-clk", ap_apb_parents, + 0x20, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data ipi_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_64m.hw }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_MUX_CLK_DATA(ipi_clk, "ipi-clk", ipi_parents, + 0x24, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data ap_uart_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_51m2.hw }, + { .hw = &twpll_96m.hw }, +}; +static SPRD_COMP_CLK_DATA(ap_uart0_clk, "ap-uart0-clk", ap_uart_parents, + 0x28, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_uart1_clk, "ap-uart1-clk", ap_uart_parents, + 0x2c, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_uart2_clk, "ap-uart2-clk", ap_uart_parents, + 0x30, 0, 2, 8, 3, 0); + +static const struct clk_parent_data i2c_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_51m2.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_COMP_CLK_DATA(ap_i2c0_clk, "ap-i2c0-clk", i2c_parents, + 0x34, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_i2c1_clk, "ap-i2c1-clk", i2c_parents, + 0x38, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_i2c2_clk, "ap-i2c2-clk", i2c_parents, + 0x3c, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_i2c3_clk, "ap-i2c3-clk", i2c_parents, + 0x40, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_i2c4_clk, "ap-i2c4-clk", i2c_parents, + 0x44, 0, 2, 8, 3, 0); + +static const struct clk_parent_data spi_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, + { .hw = &twpll_192m.hw }, +}; +static SPRD_COMP_CLK_DATA(ap_spi0_clk, "ap-spi0-clk", spi_parents, + 0x48, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_spi1_clk, "ap-spi1-clk", spi_parents, + 0x4c, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_spi2_clk, "ap-spi2-clk", spi_parents, + 0x50, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_spi3_clk, "ap-spi3-clk", spi_parents, + 0x54, 0, 2, 8, 3, 0); + +static const struct clk_parent_data iis_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_COMP_CLK_DATA(ap_iis0_clk, "ap-iis0-clk", iis_parents, + 0x58, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_iis1_clk, "ap-iis1-clk", iis_parents, + 0x5c, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(ap_iis2_clk, "ap-iis2-clk", iis_parents, + 0x60, 0, 2, 8, 3, 0); + +static const struct clk_parent_data sim_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_51m2.hw }, + { .hw = &twpll_64m.hw }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_COMP_CLK_DATA(ap_sim_clk, "ap-sim-clk", sim_parents, + 0x64, 0, 3, 8, 3, 0); + +static const struct clk_parent_data ap_ce_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_192m.hw }, + { .hw = &twpll_256m.hw }, +}; +static SPRD_MUX_CLK_DATA(ap_ce_clk, "ap-ce-clk", ap_ce_parents, + 0x68, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data sdio_parents[] = { + { .hw = &clk_1m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &rpll.common.hw }, + { .hw = &lpll_409m6.hw }, +}; +static SPRD_MUX_CLK_DATA(sdio0_2x_clk, "sdio0-2x", sdio_parents, + 0x80, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(sdio1_2x_clk, "sdio1-2x", sdio_parents, + 0x88, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(emmc_2x_clk, "emmc-2x", sdio_parents, + 0x90, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data vsp_parents[] = { + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, +}; +static SPRD_MUX_CLK_DATA(vsp_clk, "vsp-clk", vsp_parents, + 0x98, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data dispc0_parents[] = { + { .hw = &twpll_153m6.hw }, + { .hw = &twpll_192m.hw }, + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, +}; +static SPRD_MUX_CLK_DATA(dispc0_clk, "dispc0-clk", dispc0_parents, + 0x9c, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data dispc0_dpi_parents[] = { + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, + { .hw = &twpll_192m.hw }, +}; +static SPRD_COMP_CLK_DATA(dispc0_dpi_clk, "dispc0-dpi-clk", dispc0_dpi_parents, + 0xa0, 0, 3, 8, 4, 0); + +static const struct clk_parent_data dsi_apb_parents[] = { + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, + { .hw = &twpll_192m.hw }, +}; +static SPRD_MUX_CLK_DATA(dsi_apb_clk, "dsi-apb-clk", dsi_apb_parents, + 0xa4, 0, 2, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_FW_NAME(dsi_rxesc, "dsi-rxesc", "ext-26m", + 0xa8, BIT(16), 0, 0); + +static SPRD_GATE_CLK_FW_NAME(dsi_lanebyte, "dsi-lanebyte", "ext-26m", + 0xac, BIT(16), 0, 0); + +static const struct clk_parent_data vdsp_parents[] = { + { .hw = &twpll_256m.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &twpll_512m.hw }, + { .hw = &lpll_614m4.hw }, + { .hw = &twpll_768m.hw }, + { .hw = &isppll.common.hw }, +}; +static SPRD_MUX_CLK_DATA(vdsp_clk, "vdsp-clk", vdsp_parents, + 0xb0, 0, 3, UMS512_MUX_FLAG); +static SPRD_DIV_CLK_HW(vdsp_m_clk, "vdsp-m-clk", &vdsp_clk.common.hw, + 0xb4, 8, 2, 0); + +static struct sprd_clk_common *ums512_ap_clks[] = { + /* address base is 0x20200000 */ + &ap_apb_clk.common, + &ipi_clk.common, + &ap_uart0_clk.common, + &ap_uart1_clk.common, + &ap_uart2_clk.common, + &ap_i2c0_clk.common, + &ap_i2c1_clk.common, + &ap_i2c2_clk.common, + &ap_i2c3_clk.common, + &ap_i2c4_clk.common, + &ap_spi0_clk.common, + &ap_spi1_clk.common, + &ap_spi2_clk.common, + &ap_spi3_clk.common, + &ap_iis0_clk.common, + &ap_iis1_clk.common, + &ap_iis2_clk.common, + &ap_sim_clk.common, + &ap_ce_clk.common, + &sdio0_2x_clk.common, + &sdio1_2x_clk.common, + &emmc_2x_clk.common, + &vsp_clk.common, + &dispc0_clk.common, + &dispc0_dpi_clk.common, + &dsi_apb_clk.common, + &dsi_rxesc.common, + &dsi_lanebyte.common, + &vdsp_clk.common, + &vdsp_m_clk.common, + +}; + +static struct clk_hw_onecell_data ums512_ap_clk_hws = { + .hws = { + [CLK_AP_APB] = &ap_apb_clk.common.hw, + [CLK_IPI] = &ipi_clk.common.hw, + [CLK_AP_UART0] = &ap_uart0_clk.common.hw, + [CLK_AP_UART1] = &ap_uart1_clk.common.hw, + [CLK_AP_UART2] = &ap_uart2_clk.common.hw, + [CLK_AP_I2C0] = &ap_i2c0_clk.common.hw, + [CLK_AP_I2C1] = &ap_i2c1_clk.common.hw, + [CLK_AP_I2C2] = &ap_i2c2_clk.common.hw, + [CLK_AP_I2C3] = &ap_i2c3_clk.common.hw, + [CLK_AP_I2C4] = &ap_i2c4_clk.common.hw, + [CLK_AP_SPI0] = &ap_spi0_clk.common.hw, + [CLK_AP_SPI1] = &ap_spi1_clk.common.hw, + [CLK_AP_SPI2] = &ap_spi2_clk.common.hw, + [CLK_AP_SPI3] = &ap_spi3_clk.common.hw, + [CLK_AP_IIS0] = &ap_iis0_clk.common.hw, + [CLK_AP_IIS1] = &ap_iis1_clk.common.hw, + [CLK_AP_IIS2] = &ap_iis2_clk.common.hw, + [CLK_AP_SIM] = &ap_sim_clk.common.hw, + [CLK_AP_CE] = &ap_ce_clk.common.hw, + [CLK_SDIO0_2X] = &sdio0_2x_clk.common.hw, + [CLK_SDIO1_2X] = &sdio1_2x_clk.common.hw, + [CLK_EMMC_2X] = &emmc_2x_clk.common.hw, + [CLK_VSP] = &vsp_clk.common.hw, + [CLK_DISPC0] = &dispc0_clk.common.hw, + [CLK_DISPC0_DPI] = &dispc0_dpi_clk.common.hw, + [CLK_DSI_APB] = &dsi_apb_clk.common.hw, + [CLK_DSI_RXESC] = &dsi_rxesc.common.hw, + [CLK_DSI_LANEBYTE] = &dsi_lanebyte.common.hw, + [CLK_VDSP] = &vdsp_clk.common.hw, + [CLK_VDSP_M] = &vdsp_m_clk.common.hw, + }, + .num = CLK_AP_CLK_NUM, +}; + +static struct sprd_clk_desc ums512_ap_clk_desc = { + .clk_clks = ums512_ap_clks, + .num_clk_clks = ARRAY_SIZE(ums512_ap_clks), + .hw_clks = &ums512_ap_clk_hws, +}; + +/* aon apb clks */ +static const struct clk_parent_data aon_apb_parents[] = { + { .hw = &rco_4m.hw }, + { .fw_name = "ext-4m" }, + { .hw = &clk_13m.hw }, + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_96m.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_COMP_CLK_DATA(aon_apb_clk, "aon-apb-clk", aon_apb_parents, + 0x220, 0, 3, 8, 2, 0); + + +static const struct clk_parent_data adi_parents[] = { + { .hw = &rco_4m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &rco_25m.hw }, + { .hw = &twpll_38m4.hw }, + { .hw = &twpll_51m2.hw }, +}; +static SPRD_MUX_CLK_DATA(adi_clk, "adi-clk", adi_parents, + 0x224, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data aux_parents[] = { + { .fw_name = "ext-32k" }, + { .fw_name = "ext-26m" }, + { .hw = &clk_26m_aud.hw }, + { .hw = &rco_25m.hw }, + { .hw = &cppll_39m32.hw }, + { .hw = &mpll0_56m88.hw }, + { .hw = &mpll1_63m38.hw }, + { .hw = &mpll2_47m13.hw }, + { .hw = &dpll0_58m31.hw }, + { .hw = &gpll_40m.hw }, + { .hw = &twpll_48m.hw }, +}; +static const struct clk_parent_data aux1_parents[] = { + { .fw_name = "ext-32k" }, + { .fw_name = "ext-26m" }, + { .hw = &clk_26m_aud.hw }, + { .hw = &rco_25m.hw }, + { .hw = &cppll_39m32.hw }, + { .hw = &mpll0_56m88.hw }, + { .hw = &mpll1_63m38.hw }, + { .hw = &mpll2_47m13.hw }, + { .hw = &dpll0_58m31.hw }, + { .hw = &gpll_40m.hw }, + { .hw = &twpll_19m2.hw }, + { .hw = &lpll_30m72.hw }, + { .hw = &rpll.common.hw }, + { .hw = &twpll_12m29.hw }, + +}; +static SPRD_COMP_CLK_DATA(aux0_clk, "aux0-clk", aux_parents, + 0x228, 0, 5, 8, 4, 0); +static SPRD_COMP_CLK_DATA(aux1_clk, "aux1-clk", aux1_parents, + 0x22c, 0, 5, 8, 4, 0); +static SPRD_COMP_CLK_DATA(aux2_clk, "aux2-clk", aux_parents, + 0x230, 0, 5, 8, 4, 0); +static SPRD_COMP_CLK_DATA(probe_clk, "probe-clk", aux_parents, + 0x234, 0, 5, 8, 4, 0); + +static const struct clk_parent_data pwm_parents[] = { + { .fw_name = "ext-32k" }, + { .fw_name = "ext-26m" }, + { .hw = &rco_4m.hw }, + { .hw = &rco_25m.hw }, + { .hw = &twpll_48m.hw }, +}; +static SPRD_MUX_CLK_DATA(pwm0_clk, "pwm0-clk", pwm_parents, + 0x238, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(pwm1_clk, "pwm1-clk", pwm_parents, + 0x23c, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(pwm2_clk, "pwm2-clk", pwm_parents, + 0x240, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(pwm3_clk, "pwm3-clk", pwm_parents, + 0x244, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data efuse_parents[] = { + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(efuse_clk, "efuse-clk", efuse_parents, + 0x248, 0, 1, UMS512_MUX_FLAG); + +static const struct clk_parent_data uart_parents[] = { + { .hw = &rco_4m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_51m2.hw }, + { .hw = &twpll_96m.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_MUX_CLK_DATA(uart0_clk, "uart0-clk", uart_parents, + 0x24c, 0, 3, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(uart1_clk, "uart1-clk", uart_parents, + 0x250, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data thm_parents[] = { + { .fw_name = "ext-32m" }, + { .hw = &clk_250k.hw }, +}; +static SPRD_MUX_CLK_DATA(thm0_clk, "thm0-clk", thm_parents, + 0x260, 0, 1, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(thm1_clk, "thm1-clk", thm_parents, + 0x264, 0, 1, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(thm2_clk, "thm2-clk", thm_parents, + 0x268, 0, 1, UMS512_MUX_FLAG); +static SPRD_MUX_CLK_DATA(thm3_clk, "thm3-clk", thm_parents, + 0x26c, 0, 1, UMS512_MUX_FLAG); + +static const struct clk_parent_data aon_i2c_parents[] = { + { .hw = &rco_4m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_51m2.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(aon_i2c_clk, "aon-i2c-clk", aon_i2c_parents, + 0x27c, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data aon_iis_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(aon_iis_clk, "aon-iis-clk", aon_iis_parents, + 0x280, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data scc_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_51m2.hw }, + { .hw = &twpll_96m.hw }, +}; +static SPRD_MUX_CLK_DATA(scc_clk, "scc-clk", scc_parents, + 0x284, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data apcpu_dap_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &rco_4m.hw }, + { .hw = &twpll_76m8.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(apcpu_dap_clk, "apcpu-dap-clk", apcpu_dap_parents, + 0x288, 0, 3, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_FW_NAME(apcpu_dap_mtck, "apcpu-dap-mtck", "ext-26m", + 0x28c, BIT(16), 0, 0); + +static const struct clk_parent_data apcpu_ts_parents[] = { + { .fw_name = "ext-32m" }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(apcpu_ts_clk, "apcpu-ts-clk", apcpu_ts_parents, + 0x290, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data debug_ts_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_192m.hw }, +}; +static SPRD_MUX_CLK_DATA(debug_ts_clk, "debug-ts-clk", debug_ts_parents, + 0x294, 0, 2, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_FW_NAME(dsi_test_s, "dsi-test-s", "ext-26m", + 0x298, BIT(16), 0, 0); + +static const struct clk_parent_data djtag_tck_parents[] = { + { .hw = &rco_4m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(djtag_tck_clk, "djtag-tck-clk", djtag_tck_parents, + 0x2b4, 0, 1, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_FW_NAME(djtag_tck_hw, "djtag-tck-hw", "ext-26m", + 0x2b8, BIT(16), 0, 0); + +static const struct clk_parent_data aon_tmr_parents[] = { + { .hw = &rco_4m.hw }, + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(aon_tmr_clk, "aon-tmr-clk", aon_tmr_parents, + 0x2c0, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data aon_pmu_parents[] = { + { .fw_name = "ext-32k" }, + { .hw = &rco_4m.hw }, + { .fw_name = "ext-4m" }, +}; +static SPRD_MUX_CLK_DATA(aon_pmu_clk, "aon-pmu-clk", aon_pmu_parents, + 0x2c8, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data debounce_parents[] = { + { .fw_name = "ext-32k" }, + { .hw = &rco_4m.hw }, + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(debounce_clk, "debounce-clk", debounce_parents, + 0x2cc, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data apcpu_pmu_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_76m8.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_MUX_CLK_DATA(apcpu_pmu_clk, "apcpu-pmu-clk", apcpu_pmu_parents, + 0x2d0, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data top_dvfs_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_96m.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_MUX_CLK_DATA(top_dvfs_clk, "top-dvfs-clk", top_dvfs_parents, + 0x2d8, 0, 2, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_FW_NAME(otg_utmi, "otg-utmi", "ext-26m", 0x2dc, + BIT(16), 0, 0); + +static const struct clk_parent_data otg_ref_parents[] = { + { .hw = &twpll_12m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(otg_ref_clk, "otg-ref-clk", otg_ref_parents, + 0x2e0, 0, 1, UMS512_MUX_FLAG); + +static const struct clk_parent_data cssys_parents[] = { + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_153m6.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &twpll_512m.hw }, +}; +static SPRD_COMP_CLK_DATA(cssys_clk, "cssys-clk", cssys_parents, + 0x2e4, 0, 3, 8, 2, 0); +static SPRD_DIV_CLK_HW(cssys_pub_clk, "cssys-pub-clk", &cssys_clk.common.hw, + 0x2e8, 8, 2, 0); +static SPRD_DIV_CLK_HW(cssys_apb_clk, "cssys-apb-clk", &cssys_clk.common.hw, + 0x2ec, 8, 3, 0); + +static const struct clk_parent_data ap_axi_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_256m.hw }, +}; +static SPRD_MUX_CLK_DATA(ap_axi_clk, "ap-axi-clk", ap_axi_parents, + 0x2f0, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data ap_mm_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(ap_mm_clk, "ap-mm-clk", ap_mm_parents, + 0x2f4, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data sdio2_2x_parents[] = { + { .hw = &clk_1m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &rpll.common.hw }, + { .hw = &lpll_409m6.hw }, +}; +static SPRD_MUX_CLK_DATA(sdio2_2x_clk, "sdio2-2x-clk", sdio2_2x_parents, + 0x2f8, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data analog_io_apb_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, +}; +static SPRD_COMP_CLK_DATA(analog_io_apb, "analog-io-apb", analog_io_apb_parents, + 0x300, 0, 1, 8, 2, 0); + +static const struct clk_parent_data dmc_ref_parents[] = { + { .hw = &clk_6m5.hw }, + { .hw = &clk_13m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(dmc_ref_clk, "dmc-ref-clk", dmc_ref_parents, + 0x304, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data emc_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_384m.hw }, + { .hw = &twpll_512m.hw }, + { .hw = &twpll_768m.hw }, +}; +static SPRD_MUX_CLK_DATA(emc_clk, "emc-clk", emc_parents, + 0x30c, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data usb_parents[] = { + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, + { .hw = &twpll_192m.hw }, + { .hw = &twpll_96m.hw }, + { .fw_name = "rco-100m" }, + { .hw = &twpll_128m.hw }, +}; +static SPRD_COMP_CLK_DATA(usb_clk, "usb-clk", usb_parents, + 0x310, 0, 3, 8, 2, 0); + +static const struct clk_parent_data pmu_26m_parents[] = { + { .hw = &rco_25m.hw }, + { .fw_name = "ext-26m" }, +}; +static SPRD_MUX_CLK_DATA(pmu_26m_clk, "26m-pmu-clk", pmu_26m_parents, + 0x318, 0, 1, UMS512_MUX_FLAG); + +static struct sprd_clk_common *ums512_aon_apb[] = { + /* address base is 0x32080200 */ + &aon_apb_clk.common, + &adi_clk.common, + &aux0_clk.common, + &aux1_clk.common, + &aux2_clk.common, + &probe_clk.common, + &pwm0_clk.common, + &pwm1_clk.common, + &pwm2_clk.common, + &pwm3_clk.common, + &efuse_clk.common, + &uart0_clk.common, + &uart1_clk.common, + &thm0_clk.common, + &thm1_clk.common, + &thm2_clk.common, + &thm3_clk.common, + &aon_i2c_clk.common, + &aon_iis_clk.common, + &scc_clk.common, + &apcpu_dap_clk.common, + &apcpu_dap_mtck.common, + &apcpu_ts_clk.common, + &debug_ts_clk.common, + &dsi_test_s.common, + &djtag_tck_clk.common, + &djtag_tck_hw.common, + &aon_tmr_clk.common, + &aon_pmu_clk.common, + &debounce_clk.common, + &apcpu_pmu_clk.common, + &top_dvfs_clk.common, + &otg_utmi.common, + &otg_ref_clk.common, + &cssys_clk.common, + &cssys_pub_clk.common, + &cssys_apb_clk.common, + &ap_axi_clk.common, + &ap_mm_clk.common, + &sdio2_2x_clk.common, + &analog_io_apb.common, + &dmc_ref_clk.common, + &emc_clk.common, + &usb_clk.common, + &pmu_26m_clk.common, +}; + +static struct clk_hw_onecell_data ums512_aon_apb_hws = { + .hws = { + [CLK_AON_APB] = &aon_apb_clk.common.hw, + [CLK_ADI] = &adi_clk.common.hw, + [CLK_AUX0] = &aux0_clk.common.hw, + [CLK_AUX1] = &aux1_clk.common.hw, + [CLK_AUX2] = &aux2_clk.common.hw, + [CLK_PROBE] = &probe_clk.common.hw, + [CLK_PWM0] = &pwm0_clk.common.hw, + [CLK_PWM1] = &pwm1_clk.common.hw, + [CLK_PWM2] = &pwm2_clk.common.hw, + [CLK_PWM3] = &pwm3_clk.common.hw, + [CLK_EFUSE] = &efuse_clk.common.hw, + [CLK_UART0] = &uart0_clk.common.hw, + [CLK_UART1] = &uart1_clk.common.hw, + [CLK_THM0] = &thm0_clk.common.hw, + [CLK_THM1] = &thm1_clk.common.hw, + [CLK_THM2] = &thm2_clk.common.hw, + [CLK_THM3] = &thm3_clk.common.hw, + [CLK_AON_I2C] = &aon_i2c_clk.common.hw, + [CLK_AON_IIS] = &aon_iis_clk.common.hw, + [CLK_SCC] = &scc_clk.common.hw, + [CLK_APCPU_DAP] = &apcpu_dap_clk.common.hw, + [CLK_APCPU_DAP_MTCK] = &apcpu_dap_mtck.common.hw, + [CLK_APCPU_TS] = &apcpu_ts_clk.common.hw, + [CLK_DEBUG_TS] = &debug_ts_clk.common.hw, + [CLK_DSI_TEST_S] = &dsi_test_s.common.hw, + [CLK_DJTAG_TCK] = &djtag_tck_clk.common.hw, + [CLK_DJTAG_TCK_HW] = &djtag_tck_hw.common.hw, + [CLK_AON_TMR] = &aon_tmr_clk.common.hw, + [CLK_AON_PMU] = &aon_pmu_clk.common.hw, + [CLK_DEBOUNCE] = &debounce_clk.common.hw, + [CLK_APCPU_PMU] = &apcpu_pmu_clk.common.hw, + [CLK_TOP_DVFS] = &top_dvfs_clk.common.hw, + [CLK_OTG_UTMI] = &otg_utmi.common.hw, + [CLK_OTG_REF] = &otg_ref_clk.common.hw, + [CLK_CSSYS] = &cssys_clk.common.hw, + [CLK_CSSYS_PUB] = &cssys_pub_clk.common.hw, + [CLK_CSSYS_APB] = &cssys_apb_clk.common.hw, + [CLK_AP_AXI] = &ap_axi_clk.common.hw, + [CLK_AP_MM] = &ap_mm_clk.common.hw, + [CLK_SDIO2_2X] = &sdio2_2x_clk.common.hw, + [CLK_ANALOG_IO_APB] = &analog_io_apb.common.hw, + [CLK_DMC_REF_CLK] = &dmc_ref_clk.common.hw, + [CLK_EMC] = &emc_clk.common.hw, + [CLK_USB] = &usb_clk.common.hw, + [CLK_26M_PMU] = &pmu_26m_clk.common.hw, + }, + .num = CLK_AON_APB_NUM, +}; + +static struct sprd_clk_desc ums512_aon_apb_desc = { + .clk_clks = ums512_aon_apb, + .num_clk_clks = ARRAY_SIZE(ums512_aon_apb), + .hw_clks = &ums512_aon_apb_hws, +}; + +/* aon apb gates */ +static SPRD_SC_GATE_CLK_FW_NAME(rc100m_cal_eb, "rc100m-cal-eb", "ext-26m", + 0x0, 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(djtag_tck_eb, "djtag-tck-eb", "ext-26m", + 0x0, 0x1000, BIT(2), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(djtag_eb, "djtag-eb", "ext-26m", + 0x0, 0x1000, BIT(3), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aux0_eb, "aux0-eb", "ext-26m", + 0x0, 0x1000, BIT(4), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aux1_eb, "aux1-eb", "ext-26m", + 0x0, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aux2_eb, "aux2-eb", "ext-26m", + 0x0, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(probe_eb, "probe-eb", "ext-26m", + 0x0, 0x1000, BIT(7), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(mm_eb, "mm-eb", "ext-26m", + 0x0, 0x1000, BIT(9), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(gpu_eb, "gpu-eb", "ext-26m", + 0x0, 0x1000, BIT(11), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(mspi_eb, "mspi-eb", "ext-26m", + 0x0, 0x1000, BIT(12), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(apcpu_dap_eb, "apcpu-dap-eb", "ext-26m", + 0x0, 0x1000, BIT(14), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_cssys_eb, "aon-cssys-eb", "ext-26m", + 0x0, 0x1000, BIT(15), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(cssys_apb_eb, "cssys-apb-eb", "ext-26m", + 0x0, 0x1000, BIT(16), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(cssys_pub_eb, "cssys-pub-eb", "ext-26m", + 0x0, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdphy_cfg_eb, "sdphy-cfg-eb", "ext-26m", + 0x0, 0x1000, BIT(19), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdphy_ref_eb, "sdphy-ref-eb", "ext-26m", + 0x0, 0x1000, BIT(20), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(efuse_eb, "efuse-eb", "ext-26m", + 0x4, 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(gpio_eb, "gpio-eb", "ext-26m", + 0x4, 0x1000, BIT(1), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(mbox_eb, "mbox-eb", "ext-26m", + 0x4, 0x1000, BIT(2), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(kpd_eb, "kpd-eb", "ext-26m", + 0x4, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_syst_eb, "aon-syst-eb", "ext-26m", + 0x4, 0x1000, BIT(4), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_syst_eb, "ap-syst-eb", "ext-26m", + 0x4, 0x1000, BIT(5), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_tmr_eb, "aon-tmr-eb", "ext-26m", + 0x4, 0x1000, BIT(6), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(otg_utmi_eb, "otg-utmi-eb", "ext-26m", + 0x4, 0x1000, BIT(8), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(otg_phy_eb, "otg-phy-eb", "ext-26m", + 0x4, 0x1000, BIT(9), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(splk_eb, "splk-eb", "ext-26m", + 0x4, 0x1000, BIT(10), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pin_eb, "pin-eb", "ext-26m", + 0x4, 0x1000, BIT(11), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ana_eb, "ana-eb", "ext-26m", + 0x4, 0x1000, BIT(12), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(apcpu_ts0_eb, "apcpu-ts0-eb", "ext-26m", + 0x4, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(apb_busmon_eb, "apb-busmon-eb", "ext-26m", + 0x4, 0x1000, BIT(18), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_iis_eb, "aon-iis-eb", "ext-26m", + 0x4, 0x1000, BIT(19), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(scc_eb, "scc-eb", "ext-26m", + 0x4, 0x1000, BIT(20), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(thm0_eb, "thm0-eb", "ext-26m", + 0x8, 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(thm1_eb, "thm1-eb", "ext-26m", + 0x8, 0x1000, BIT(1), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(thm2_eb, "thm2-eb", "ext-26m", + 0x8, 0x1000, BIT(2), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(asim_top_eb, "asim-top", "ext-26m", + 0x8, 0x1000, BIT(3), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c_eb, "i2c-eb", "ext-26m", + 0x8, 0x1000, BIT(7), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pmu_eb, "pmu-eb", "ext-26m", + 0x8, 0x1000, BIT(8), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(adi_eb, "adi-eb", "ext-26m", + 0x8, 0x1000, BIT(9), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(eic_eb, "eic-eb", "ext-26m", + 0x8, 0x1000, BIT(10), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc0_eb, "ap-intc0-eb", "ext-26m", + 0x8, 0x1000, BIT(11), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc1_eb, "ap-intc1-eb", "ext-26m", + 0x8, 0x1000, BIT(12), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc2_eb, "ap-intc2-eb", "ext-26m", + 0x8, 0x1000, BIT(13), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc3_eb, "ap-intc3-eb", "ext-26m", + 0x8, 0x1000, BIT(14), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc4_eb, "ap-intc4-eb", "ext-26m", + 0x8, 0x1000, BIT(15), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_intc5_eb, "ap-intc5-eb", "ext-26m", + 0x8, 0x1000, BIT(16), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(audcp_intc_eb, "audcp-intc-eb", "ext-26m", + 0x8, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr0_eb, "ap-tmr0-eb", "ext-26m", + 0x8, 0x1000, BIT(22), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr1_eb, "ap-tmr1-eb", "ext-26m", + 0x8, 0x1000, BIT(23), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr2_eb, "ap-tmr2-eb", "ext-26m", + 0x8, 0x1000, BIT(24), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pwm0_eb, "pwm0-eb", "ext-26m", + 0x8, 0x1000, BIT(25), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pwm1_eb, "pwm1-eb", "ext-26m", + 0x8, 0x1000, BIT(26), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pwm2_eb, "pwm2-eb", "ext-26m", + 0x8, 0x1000, BIT(27), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pwm3_eb, "pwm3-eb", "ext-26m", + 0x8, 0x1000, BIT(28), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_wdg_eb, "ap-wdg-eb", "ext-26m", + 0x8, 0x1000, BIT(29), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(apcpu_wdg_eb, "apcpu-wdg-eb", "ext-26m", + 0x8, 0x1000, BIT(30), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(serdes_eb, "serdes-eb", "ext-26m", + 0x8, 0x1000, BIT(31), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(arch_rtc_eb, "arch-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(0), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(kpd_rtc_eb, "kpd-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(1), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_syst_rtc_eb, "aon-syst-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(2), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_syst_rtc_eb, "ap-syst-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(3), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(aon_tmr_rtc_eb, "aon-tmr-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(4), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(eic_rtc_eb, "eic-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(5), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(eic_rtcdv5_eb, "eic-rtcdv5-eb", "ext-26m", + 0x18, 0x1000, BIT(6), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_wdg_rtc_eb, "ap-wdg-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(7), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ac_wdg_rtc_eb, "ac-wdg-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(8), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr0_rtc_eb, "ap-tmr0-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(9), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr1_rtc_eb, "ap-tmr1-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(10), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_tmr2_rtc_eb, "ap-tmr2-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(11), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dcxo_lc_rtc_eb, "dcxo-lc-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(12), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(bb_cal_rtc_eb, "bb-cal-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(13), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_emmc_rtc_eb, "ap-emmc-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(14), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_sdio0_rtc_eb, "ap-sdio0-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(15), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_sdio1_rtc_eb, "ap-sdio1-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(16), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_sdio2_rtc_eb, "ap-sdio2-rtc-eb", "ext-26m", + 0x18, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dsi_csi_test_eb, "dsi-csi-test-eb", "ext-26m", + 0x138, 0x1000, BIT(8), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(djtag_tck_en, "djtag-tck-en", "ext-26m", + 0x138, 0x1000, BIT(9), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dphy_ref_eb, "dphy-ref-eb", "ext-26m", + 0x138, 0x1000, BIT(10), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(dmc_ref_eb, "dmc-ref-eb", "ext-26m", + 0x138, 0x1000, BIT(11), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(otg_ref_eb, "otg-ref-eb", "ext-26m", + 0x138, 0x1000, BIT(12), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(tsen_eb, "tsen-eb", "ext-26m", + 0x138, 0x1000, BIT(13), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(tmr_eb, "tmr-eb", "ext-26m", + 0x138, 0x1000, BIT(14), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(rc100m_ref_eb, "rc100m-ref-eb", "ext-26m", + 0x138, 0x1000, BIT(15), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(rc100m_fdk_eb, "rc100m-fdk-eb", "ext-26m", + 0x138, 0x1000, BIT(16), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(debounce_eb, "debounce-eb", "ext-26m", + 0x138, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(det_32k_eb, "det-32k-eb", "ext-26m", + 0x138, 0x1000, BIT(18), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(top_cssys_en, "top-cssys-en", "ext-26m", + 0x13c, 0x1000, BIT(0), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(ap_axi_en, "ap-axi-en", "ext-26m", + 0x13c, 0x1000, BIT(1), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio0_2x_en, "sdio0-2x-en", "ext-26m", + 0x13c, 0x1000, BIT(2), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio0_1x_en, "sdio0-1x-en", "ext-26m", + 0x13c, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio1_2x_en, "sdio1-2x-en", "ext-26m", + 0x13c, 0x1000, BIT(4), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio1_1x_en, "sdio1-1x-en", "ext-26m", + 0x13c, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio2_2x_en, "sdio2-2x-en", "ext-26m", + 0x13c, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio2_1x_en, "sdio2-1x-en", "ext-26m", + 0x13c, 0x1000, BIT(7), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(emmc_2x_en, "emmc-2x-en", "ext-26m", + 0x13c, 0x1000, BIT(8), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(emmc_1x_en, "emmc-1x-en", "ext-26m", + 0x13c, 0x1000, BIT(9), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(pll_test_en, "pll-test-en", "ext-26m", + 0x13c, 0x1000, BIT(14), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(cphy_cfg_en, "cphy-cfg-en", "ext-26m", + 0x13c, 0x1000, BIT(15), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(debug_ts_en, "debug-ts-en", "ext-26m", + 0x13c, 0x1000, BIT(18), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(access_aud_en, "access-aud-en", + "ext-26m", 0x14c, 0x1000, BIT(0), 0, 0); + +static struct sprd_clk_common *ums512_aon_gate[] = { + /* address base is 0x327d0000 */ + &rc100m_cal_eb.common, + &djtag_tck_eb.common, + &djtag_eb.common, + &aux0_eb.common, + &aux1_eb.common, + &aux2_eb.common, + &probe_eb.common, + &mm_eb.common, + &gpu_eb.common, + &mspi_eb.common, + &apcpu_dap_eb.common, + &aon_cssys_eb.common, + &cssys_apb_eb.common, + &cssys_pub_eb.common, + &sdphy_cfg_eb.common, + &sdphy_ref_eb.common, + &efuse_eb.common, + &gpio_eb.common, + &mbox_eb.common, + &kpd_eb.common, + &aon_syst_eb.common, + &ap_syst_eb.common, + &aon_tmr_eb.common, + &otg_utmi_eb.common, + &otg_phy_eb.common, + &splk_eb.common, + &pin_eb.common, + &ana_eb.common, + &apcpu_ts0_eb.common, + &apb_busmon_eb.common, + &aon_iis_eb.common, + &scc_eb.common, + &thm0_eb.common, + &thm1_eb.common, + &thm2_eb.common, + &asim_top_eb.common, + &i2c_eb.common, + &pmu_eb.common, + &adi_eb.common, + &eic_eb.common, + &ap_intc0_eb.common, + &ap_intc1_eb.common, + &ap_intc2_eb.common, + &ap_intc3_eb.common, + &ap_intc4_eb.common, + &ap_intc5_eb.common, + &audcp_intc_eb.common, + &ap_tmr0_eb.common, + &ap_tmr1_eb.common, + &ap_tmr2_eb.common, + &pwm0_eb.common, + &pwm1_eb.common, + &pwm2_eb.common, + &pwm3_eb.common, + &ap_wdg_eb.common, + &apcpu_wdg_eb.common, + &serdes_eb.common, + &arch_rtc_eb.common, + &kpd_rtc_eb.common, + &aon_syst_rtc_eb.common, + &ap_syst_rtc_eb.common, + &aon_tmr_rtc_eb.common, + &eic_rtc_eb.common, + &eic_rtcdv5_eb.common, + &ap_wdg_rtc_eb.common, + &ac_wdg_rtc_eb.common, + &ap_tmr0_rtc_eb.common, + &ap_tmr1_rtc_eb.common, + &ap_tmr2_rtc_eb.common, + &dcxo_lc_rtc_eb.common, + &bb_cal_rtc_eb.common, + &ap_emmc_rtc_eb.common, + &ap_sdio0_rtc_eb.common, + &ap_sdio1_rtc_eb.common, + &ap_sdio2_rtc_eb.common, + &dsi_csi_test_eb.common, + &djtag_tck_en.common, + &dphy_ref_eb.common, + &dmc_ref_eb.common, + &otg_ref_eb.common, + &tsen_eb.common, + &tmr_eb.common, + &rc100m_ref_eb.common, + &rc100m_fdk_eb.common, + &debounce_eb.common, + &det_32k_eb.common, + &top_cssys_en.common, + &ap_axi_en.common, + &sdio0_2x_en.common, + &sdio0_1x_en.common, + &sdio1_2x_en.common, + &sdio1_1x_en.common, + &sdio2_2x_en.common, + &sdio2_1x_en.common, + &emmc_2x_en.common, + &emmc_1x_en.common, + &pll_test_en.common, + &cphy_cfg_en.common, + &debug_ts_en.common, + &access_aud_en.common, +}; + +static struct clk_hw_onecell_data ums512_aon_gate_hws = { + .hws = { + [CLK_RC100M_CAL_EB] = &rc100m_cal_eb.common.hw, + [CLK_DJTAG_TCK_EB] = &djtag_tck_eb.common.hw, + [CLK_DJTAG_EB] = &djtag_eb.common.hw, + [CLK_AUX0_EB] = &aux0_eb.common.hw, + [CLK_AUX1_EB] = &aux1_eb.common.hw, + [CLK_AUX2_EB] = &aux2_eb.common.hw, + [CLK_PROBE_EB] = &probe_eb.common.hw, + [CLK_MM_EB] = &mm_eb.common.hw, + [CLK_GPU_EB] = &gpu_eb.common.hw, + [CLK_MSPI_EB] = &mspi_eb.common.hw, + [CLK_APCPU_DAP_EB] = &apcpu_dap_eb.common.hw, + [CLK_AON_CSSYS_EB] = &aon_cssys_eb.common.hw, + [CLK_CSSYS_APB_EB] = &cssys_apb_eb.common.hw, + [CLK_CSSYS_PUB_EB] = &cssys_pub_eb.common.hw, + [CLK_SDPHY_CFG_EB] = &sdphy_cfg_eb.common.hw, + [CLK_SDPHY_REF_EB] = &sdphy_ref_eb.common.hw, + [CLK_EFUSE_EB] = &efuse_eb.common.hw, + [CLK_GPIO_EB] = &gpio_eb.common.hw, + [CLK_MBOX_EB] = &mbox_eb.common.hw, + [CLK_KPD_EB] = &kpd_eb.common.hw, + [CLK_AON_SYST_EB] = &aon_syst_eb.common.hw, + [CLK_AP_SYST_EB] = &ap_syst_eb.common.hw, + [CLK_AON_TMR_EB] = &aon_tmr_eb.common.hw, + [CLK_OTG_UTMI_EB] = &otg_utmi_eb.common.hw, + [CLK_OTG_PHY_EB] = &otg_phy_eb.common.hw, + [CLK_SPLK_EB] = &splk_eb.common.hw, + [CLK_PIN_EB] = &pin_eb.common.hw, + [CLK_ANA_EB] = &ana_eb.common.hw, + [CLK_APCPU_TS0_EB] = &apcpu_ts0_eb.common.hw, + [CLK_APB_BUSMON_EB] = &apb_busmon_eb.common.hw, + [CLK_AON_IIS_EB] = &aon_iis_eb.common.hw, + [CLK_SCC_EB] = &scc_eb.common.hw, + [CLK_THM0_EB] = &thm0_eb.common.hw, + [CLK_THM1_EB] = &thm1_eb.common.hw, + [CLK_THM2_EB] = &thm2_eb.common.hw, + [CLK_ASIM_TOP_EB] = &asim_top_eb.common.hw, + [CLK_I2C_EB] = &i2c_eb.common.hw, + [CLK_PMU_EB] = &pmu_eb.common.hw, + [CLK_ADI_EB] = &adi_eb.common.hw, + [CLK_EIC_EB] = &eic_eb.common.hw, + [CLK_AP_INTC0_EB] = &ap_intc0_eb.common.hw, + [CLK_AP_INTC1_EB] = &ap_intc1_eb.common.hw, + [CLK_AP_INTC2_EB] = &ap_intc2_eb.common.hw, + [CLK_AP_INTC3_EB] = &ap_intc3_eb.common.hw, + [CLK_AP_INTC4_EB] = &ap_intc4_eb.common.hw, + [CLK_AP_INTC5_EB] = &ap_intc5_eb.common.hw, + [CLK_AUDCP_INTC_EB] = &audcp_intc_eb.common.hw, + [CLK_AP_TMR0_EB] = &ap_tmr0_eb.common.hw, + [CLK_AP_TMR1_EB] = &ap_tmr1_eb.common.hw, + [CLK_AP_TMR2_EB] = &ap_tmr2_eb.common.hw, + [CLK_PWM0_EB] = &pwm0_eb.common.hw, + [CLK_PWM1_EB] = &pwm1_eb.common.hw, + [CLK_PWM2_EB] = &pwm2_eb.common.hw, + [CLK_PWM3_EB] = &pwm3_eb.common.hw, + [CLK_AP_WDG_EB] = &ap_wdg_eb.common.hw, + [CLK_APCPU_WDG_EB] = &apcpu_wdg_eb.common.hw, + [CLK_SERDES_EB] = &serdes_eb.common.hw, + [CLK_ARCH_RTC_EB] = &arch_rtc_eb.common.hw, + [CLK_KPD_RTC_EB] = &kpd_rtc_eb.common.hw, + [CLK_AON_SYST_RTC_EB] = &aon_syst_rtc_eb.common.hw, + [CLK_AP_SYST_RTC_EB] = &ap_syst_rtc_eb.common.hw, + [CLK_AON_TMR_RTC_EB] = &aon_tmr_rtc_eb.common.hw, + [CLK_EIC_RTC_EB] = &eic_rtc_eb.common.hw, + [CLK_EIC_RTCDV5_EB] = &eic_rtcdv5_eb.common.hw, + [CLK_AP_WDG_RTC_EB] = &ap_wdg_rtc_eb.common.hw, + [CLK_AC_WDG_RTC_EB] = &ac_wdg_rtc_eb.common.hw, + [CLK_AP_TMR0_RTC_EB] = &ap_tmr0_rtc_eb.common.hw, + [CLK_AP_TMR1_RTC_EB] = &ap_tmr1_rtc_eb.common.hw, + [CLK_AP_TMR2_RTC_EB] = &ap_tmr2_rtc_eb.common.hw, + [CLK_DCXO_LC_RTC_EB] = &dcxo_lc_rtc_eb.common.hw, + [CLK_BB_CAL_RTC_EB] = &bb_cal_rtc_eb.common.hw, + [CLK_AP_EMMC_RTC_EB] = &ap_emmc_rtc_eb.common.hw, + [CLK_AP_SDIO0_RTC_EB] = &ap_sdio0_rtc_eb.common.hw, + [CLK_AP_SDIO1_RTC_EB] = &ap_sdio1_rtc_eb.common.hw, + [CLK_AP_SDIO2_RTC_EB] = &ap_sdio2_rtc_eb.common.hw, + [CLK_DSI_CSI_TEST_EB] = &dsi_csi_test_eb.common.hw, + [CLK_DJTAG_TCK_EN] = &djtag_tck_en.common.hw, + [CLK_DPHY_REF_EB] = &dphy_ref_eb.common.hw, + [CLK_DMC_REF_EB] = &dmc_ref_eb.common.hw, + [CLK_OTG_REF_EB] = &otg_ref_eb.common.hw, + [CLK_TSEN_EB] = &tsen_eb.common.hw, + [CLK_TMR_EB] = &tmr_eb.common.hw, + [CLK_RC100M_REF_EB] = &rc100m_ref_eb.common.hw, + [CLK_RC100M_FDK_EB] = &rc100m_fdk_eb.common.hw, + [CLK_DEBOUNCE_EB] = &debounce_eb.common.hw, + [CLK_DET_32K_EB] = &det_32k_eb.common.hw, + [CLK_TOP_CSSYS_EB] = &top_cssys_en.common.hw, + [CLK_AP_AXI_EN] = &ap_axi_en.common.hw, + [CLK_SDIO0_2X_EN] = &sdio0_2x_en.common.hw, + [CLK_SDIO0_1X_EN] = &sdio0_1x_en.common.hw, + [CLK_SDIO1_2X_EN] = &sdio1_2x_en.common.hw, + [CLK_SDIO1_1X_EN] = &sdio1_1x_en.common.hw, + [CLK_SDIO2_2X_EN] = &sdio2_2x_en.common.hw, + [CLK_SDIO2_1X_EN] = &sdio2_1x_en.common.hw, + [CLK_EMMC_2X_EN] = &emmc_2x_en.common.hw, + [CLK_EMMC_1X_EN] = &emmc_1x_en.common.hw, + [CLK_PLL_TEST_EN] = &pll_test_en.common.hw, + [CLK_CPHY_CFG_EN] = &cphy_cfg_en.common.hw, + [CLK_DEBUG_TS_EN] = &debug_ts_en.common.hw, + [CLK_ACCESS_AUD_EN] = &access_aud_en.common.hw, + }, + .num = CLK_AON_APB_GATE_NUM, +}; + +static struct sprd_clk_desc ums512_aon_gate_desc = { + .clk_clks = ums512_aon_gate, + .num_clk_clks = ARRAY_SIZE(ums512_aon_gate), + .hw_clks = &ums512_aon_gate_hws, +}; + +/* audcp apb gates */ +/* Audcp apb clocks configure CLK_IGNORE_UNUSED because these clocks may be + * controlled by audcp sys at the same time. It may be cause an execption if + * kernel gates these clock. + */ +static SPRD_SC_GATE_CLK_HW(audcp_wdg_eb, "audcp-wdg-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(1), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_rtc_wdg_eb, "audcp-rtc-wdg-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(2), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_tmr0_eb, "audcp-tmr0-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(5), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_tmr1_eb, "audcp-tmr1-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(6), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); + +static struct sprd_clk_common *ums512_audcpapb_gate[] = { + /* address base is 0x3350d000 */ + &audcp_wdg_eb.common, + &audcp_rtc_wdg_eb.common, + &audcp_tmr0_eb.common, + &audcp_tmr1_eb.common, +}; + +static struct clk_hw_onecell_data ums512_audcpapb_gate_hws = { + .hws = { + [CLK_AUDCP_WDG_EB] = &audcp_wdg_eb.common.hw, + [CLK_AUDCP_RTC_WDG_EB] = &audcp_rtc_wdg_eb.common.hw, + [CLK_AUDCP_TMR0_EB] = &audcp_tmr0_eb.common.hw, + [CLK_AUDCP_TMR1_EB] = &audcp_tmr1_eb.common.hw, + }, + .num = CLK_AUDCP_APB_GATE_NUM, +}; + +static const struct sprd_clk_desc ums512_audcpapb_gate_desc = { + .clk_clks = ums512_audcpapb_gate, + .num_clk_clks = ARRAY_SIZE(ums512_audcpapb_gate), + .hw_clks = &ums512_audcpapb_gate_hws, +}; + +/* audcp ahb gates */ +/* Audcp aphb clocks configure CLK_IGNORE_UNUSED because these clocks may be + * controlled by audcp sys at the same time. It may be cause an execption if + * kernel gates these clock. + */ +static SPRD_SC_GATE_CLK_HW(audcp_iis0_eb, "audcp-iis0-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(0), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_iis1_eb, "audcp-iis1-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(1), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_iis2_eb, "audcp-iis2-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(2), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_uart_eb, "audcp-uart-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(4), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_dma_cp_eb, "audcp-dma-cp-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(5), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_dma_ap_eb, "audcp-dma-ap-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(6), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_src48k_eb, "audcp-src48k-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(10), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_mcdt_eb, "audcp-mcdt-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(12), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_vbcifd_eb, "audcp-vbcifd-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(13), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_vbc_eb, "audcp-vbc-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(14), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_splk_eb, "audcp-splk-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(15), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_icu_eb, "audcp-icu-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(16), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(dma_ap_ashb_eb, "dma-ap-ashb-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(17), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(dma_cp_ashb_eb, "dma-cp-ashb-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(18), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_aud_eb, "audcp-aud-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(19), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_vbc_24m_eb, "audcp-vbc-24m-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(21), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_tmr_26m_eb, "audcp-tmr-26m-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(22), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); +static SPRD_SC_GATE_CLK_HW(audcp_dvfs_ashb_eb, "audcp-dvfs-ashb-eb", + &access_aud_en.common.hw, 0x0, 0x100, BIT(23), + CLK_IGNORE_UNUSED, SPRD_GATE_NON_AON); + +static struct sprd_clk_common *ums512_audcpahb_gate[] = { + /* address base is 0x335e0000 */ + &audcp_iis0_eb.common, + &audcp_iis1_eb.common, + &audcp_iis2_eb.common, + &audcp_uart_eb.common, + &audcp_dma_cp_eb.common, + &audcp_dma_ap_eb.common, + &audcp_src48k_eb.common, + &audcp_mcdt_eb.common, + &audcp_vbcifd_eb.common, + &audcp_vbc_eb.common, + &audcp_splk_eb.common, + &audcp_icu_eb.common, + &dma_ap_ashb_eb.common, + &dma_cp_ashb_eb.common, + &audcp_aud_eb.common, + &audcp_vbc_24m_eb.common, + &audcp_tmr_26m_eb.common, + &audcp_dvfs_ashb_eb.common, +}; + +static struct clk_hw_onecell_data ums512_audcpahb_gate_hws = { + .hws = { + [CLK_AUDCP_IIS0_EB] = &audcp_iis0_eb.common.hw, + [CLK_AUDCP_IIS1_EB] = &audcp_iis1_eb.common.hw, + [CLK_AUDCP_IIS2_EB] = &audcp_iis2_eb.common.hw, + [CLK_AUDCP_UART_EB] = &audcp_uart_eb.common.hw, + [CLK_AUDCP_DMA_CP_EB] = &audcp_dma_cp_eb.common.hw, + [CLK_AUDCP_DMA_AP_EB] = &audcp_dma_ap_eb.common.hw, + [CLK_AUDCP_SRC48K_EB] = &audcp_src48k_eb.common.hw, + [CLK_AUDCP_MCDT_EB] = &audcp_mcdt_eb.common.hw, + [CLK_AUDCP_VBCIFD_EB] = &audcp_vbcifd_eb.common.hw, + [CLK_AUDCP_VBC_EB] = &audcp_vbc_eb.common.hw, + [CLK_AUDCP_SPLK_EB] = &audcp_splk_eb.common.hw, + [CLK_AUDCP_ICU_EB] = &audcp_icu_eb.common.hw, + [CLK_AUDCP_DMA_AP_ASHB_EB] = &dma_ap_ashb_eb.common.hw, + [CLK_AUDCP_DMA_CP_ASHB_EB] = &dma_cp_ashb_eb.common.hw, + [CLK_AUDCP_AUD_EB] = &audcp_aud_eb.common.hw, + [CLK_AUDCP_VBC_24M_EB] = &audcp_vbc_24m_eb.common.hw, + [CLK_AUDCP_TMR_26M_EB] = &audcp_tmr_26m_eb.common.hw, + [CLK_AUDCP_DVFS_ASHB_EB] = &audcp_dvfs_ashb_eb.common.hw, + }, + .num = CLK_AUDCP_AHB_GATE_NUM, +}; + +static const struct sprd_clk_desc ums512_audcpahb_gate_desc = { + .clk_clks = ums512_audcpahb_gate, + .num_clk_clks = ARRAY_SIZE(ums512_audcpahb_gate), + .hw_clks = &ums512_audcpahb_gate_hws, +}; + +/* gpu clocks */ +static SPRD_GATE_CLK_HW(gpu_core_gate, "gpu-core-gate", &gpu_eb.common.hw, + 0x4, BIT(0), 0, 0); + +static const struct clk_parent_data gpu_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_384m.hw }, + { .hw = &twpll_512m.hw }, + { .hw = &lpll_614m4.hw }, + { .hw = &twpll_768m.hw }, + { .hw = &gpll.common.hw }, +}; + +static SPRD_COMP_CLK_DATA(gpu_core_clk, "gpu-core-clk", gpu_parents, + 0x4, 4, 3, 8, 3, 0); + +static SPRD_GATE_CLK_HW(gpu_mem_gate, "gpu-mem-gate", &gpu_eb.common.hw, + 0x8, BIT(0), 0, 0); + +static SPRD_COMP_CLK_DATA(gpu_mem_clk, "gpu-mem-clk", gpu_parents, + 0x8, 4, 3, 8, 3, 0); + +static SPRD_GATE_CLK_HW(gpu_sys_gate, "gpu-sys-gate", &gpu_eb.common.hw, + 0xc, BIT(0), 0, 0); + +static SPRD_DIV_CLK_HW(gpu_sys_clk, "gpu-sys-clk", &gpu_eb.common.hw, + 0xc, 4, 3, 0); + +static struct sprd_clk_common *ums512_gpu_clk[] = { + /* address base is 0x60100000 */ + &gpu_core_gate.common, + &gpu_core_clk.common, + &gpu_mem_gate.common, + &gpu_mem_clk.common, + &gpu_sys_gate.common, + &gpu_sys_clk.common, +}; + +static struct clk_hw_onecell_data ums512_gpu_clk_hws = { + .hws = { + [CLK_GPU_CORE_EB] = &gpu_core_gate.common.hw, + [CLK_GPU_CORE] = &gpu_core_clk.common.hw, + [CLK_GPU_MEM_EB] = &gpu_mem_gate.common.hw, + [CLK_GPU_MEM] = &gpu_mem_clk.common.hw, + [CLK_GPU_SYS_EB] = &gpu_sys_gate.common.hw, + [CLK_GPU_SYS] = &gpu_sys_clk.common.hw, + }, + .num = CLK_GPU_CLK_NUM, +}; + +static struct sprd_clk_desc ums512_gpu_clk_desc = { + .clk_clks = ums512_gpu_clk, + .num_clk_clks = ARRAY_SIZE(ums512_gpu_clk), + .hw_clks = &ums512_gpu_clk_hws, +}; + +/* mm clocks */ +static const struct clk_parent_data mm_ahb_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_96m.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_153m6.hw }, +}; +static SPRD_MUX_CLK_DATA(mm_ahb_clk, "mm-ahb-clk", mm_ahb_parents, + 0x20, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data mm_mtx_parents[] = { + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &isppll_468m.hw }, + { .hw = &twpll_512m.hw }, +}; +static SPRD_MUX_CLK_DATA(mm_mtx_clk, "mm-mtx-clk", mm_mtx_parents, + 0x24, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data sensor_parents[] = { + { .fw_name = "ext-26m" }, + { .hw = &twpll_48m.hw }, + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_96m.hw }, +}; +static SPRD_COMP_CLK_DATA(sensor0_clk, "sensor0-clk", sensor_parents, + 0x28, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(sensor1_clk, "sensor1-clk", sensor_parents, + 0x2c, 0, 2, 8, 3, 0); +static SPRD_COMP_CLK_DATA(sensor2_clk, "sensor2-clk", sensor_parents, + 0x30, 0, 2, 8, 3, 0); + +static const struct clk_parent_data cpp_parents[] = { + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_256m.hw }, + { .hw = &twpll_384m.hw }, +}; +static SPRD_MUX_CLK_DATA(cpp_clk, "cpp-clk", cpp_parents, + 0x34, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data jpg_parents[] = { + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_128m.hw }, + { .hw = &twpll_256m.hw }, + { .hw = &twpll_384m.hw }, +}; +static SPRD_MUX_CLK_DATA(jpg_clk, "jpg-clk", jpg_parents, + 0x38, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data fd_parents[] = { + { .hw = &twpll_76m8.hw }, + { .hw = &twpll_192m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, +}; +static SPRD_MUX_CLK_DATA(fd_clk, "fd-clk", fd_parents, + 0x3c, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data dcam_if_parents[] = { + { .hw = &twpll_192m.hw }, + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &isppll_468m.hw }, +}; +static SPRD_MUX_CLK_DATA(dcam_if_clk, "dcam-if-clk", dcam_if_parents, + 0x40, 0, 3, UMS512_MUX_FLAG); + +static const struct clk_parent_data dcam_axi_parents[] = { + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &isppll_468m.hw }, +}; +static SPRD_MUX_CLK_DATA(dcam_axi_clk, "dcam-axi-clk", dcam_axi_parents, + 0x44, 0, 2, UMS512_MUX_FLAG); + +static const struct clk_parent_data isp_parents[] = { + { .hw = &twpll_256m.hw }, + { .hw = &twpll_307m2.hw }, + { .hw = &twpll_384m.hw }, + { .hw = &isppll_468m.hw }, + { .hw = &twpll_512m.hw }, +}; +static SPRD_MUX_CLK_DATA(isp_clk, "isp-clk", isp_parents, + 0x48, 0, 3, UMS512_MUX_FLAG); + +static SPRD_GATE_CLK_HW(mipi_csi0, "mipi-csi0", &mm_eb.common.hw, + 0x4c, BIT(16), CLK_IGNORE_UNUSED, 0); + +static SPRD_GATE_CLK_HW(mipi_csi1, "mipi-csi1", &mm_eb.common.hw, + 0x50, BIT(16), CLK_IGNORE_UNUSED, 0); + +static SPRD_GATE_CLK_HW(mipi_csi2, "mipi-csi2", &mm_eb.common.hw, + 0x54, BIT(16), CLK_IGNORE_UNUSED, 0); + +static struct sprd_clk_common *ums512_mm_clk[] = { + /* address base is 0x62100000 */ + &mm_ahb_clk.common, + &mm_mtx_clk.common, + &sensor0_clk.common, + &sensor1_clk.common, + &sensor2_clk.common, + &cpp_clk.common, + &jpg_clk.common, + &fd_clk.common, + &dcam_if_clk.common, + &dcam_axi_clk.common, + &isp_clk.common, + &mipi_csi0.common, + &mipi_csi1.common, + &mipi_csi2.common, +}; + +static struct clk_hw_onecell_data ums512_mm_clk_hws = { + .hws = { + [CLK_MM_AHB] = &mm_ahb_clk.common.hw, + [CLK_MM_MTX] = &mm_mtx_clk.common.hw, + [CLK_SENSOR0] = &sensor0_clk.common.hw, + [CLK_SENSOR1] = &sensor1_clk.common.hw, + [CLK_SENSOR2] = &sensor2_clk.common.hw, + [CLK_CPP] = &cpp_clk.common.hw, + [CLK_JPG] = &jpg_clk.common.hw, + [CLK_FD] = &fd_clk.common.hw, + [CLK_DCAM_IF] = &dcam_if_clk.common.hw, + [CLK_DCAM_AXI] = &dcam_axi_clk.common.hw, + [CLK_ISP] = &isp_clk.common.hw, + [CLK_MIPI_CSI0] = &mipi_csi0.common.hw, + [CLK_MIPI_CSI1] = &mipi_csi1.common.hw, + [CLK_MIPI_CSI2] = &mipi_csi2.common.hw, + }, + .num = CLK_MM_CLK_NUM, +}; + +static struct sprd_clk_desc ums512_mm_clk_desc = { + .clk_clks = ums512_mm_clk, + .num_clk_clks = ARRAY_SIZE(ums512_mm_clk), + .hw_clks = &ums512_mm_clk_hws, +}; + +/* mm gate clocks */ +static SPRD_SC_GATE_CLK_HW(mm_cpp_eb, "mm-cpp-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(0), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_jpg_eb, "mm-jpg-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(1), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_dcam_eb, "mm-dcam-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(2), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_isp_eb, "mm-isp-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_csi2_eb, "mm-csi2-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(4), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_csi1_eb, "mm-csi1-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_csi0_eb, "mm-csi0-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_ckg_eb, "mm-ckg-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(7), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_isp_ahb_eb, "mm-isp-ahb-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(8), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_dvfs_eb, "mm-dvfs-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(9), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_fd_eb, "mm-fd-eb", &mm_eb.common.hw, + 0x0, 0x1000, BIT(10), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_sensor2_en, "mm-sensor2-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(0), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_sensor1_en, "mm-sensor1-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(1), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_sensor0_en, "mm-sensor0-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(2), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_mipi_csi2_en, "mm-mipi-csi2-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_mipi_csi1_en, "mm-mipi-csi1-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(4), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_mipi_csi0_en, "mm-mipi-csi0-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_dcam_axi_en, "mm-dcam-axi-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_isp_axi_en, "mm-isp-axi-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(7), 0, 0); +static SPRD_SC_GATE_CLK_HW(mm_cphy_en, "mm-cphy-en", &mm_eb.common.hw, + 0x8, 0x1000, BIT(8), 0, 0); + +static struct sprd_clk_common *ums512_mm_gate_clk[] = { + /* address base is 0x62200000 */ + &mm_cpp_eb.common, + &mm_jpg_eb.common, + &mm_dcam_eb.common, + &mm_isp_eb.common, + &mm_csi2_eb.common, + &mm_csi1_eb.common, + &mm_csi0_eb.common, + &mm_ckg_eb.common, + &mm_isp_ahb_eb.common, + &mm_dvfs_eb.common, + &mm_fd_eb.common, + &mm_sensor2_en.common, + &mm_sensor1_en.common, + &mm_sensor0_en.common, + &mm_mipi_csi2_en.common, + &mm_mipi_csi1_en.common, + &mm_mipi_csi0_en.common, + &mm_dcam_axi_en.common, + &mm_isp_axi_en.common, + &mm_cphy_en.common, +}; + +static struct clk_hw_onecell_data ums512_mm_gate_clk_hws = { + .hws = { + [CLK_MM_CPP_EB] = &mm_cpp_eb.common.hw, + [CLK_MM_JPG_EB] = &mm_jpg_eb.common.hw, + [CLK_MM_DCAM_EB] = &mm_dcam_eb.common.hw, + [CLK_MM_ISP_EB] = &mm_isp_eb.common.hw, + [CLK_MM_CSI2_EB] = &mm_csi2_eb.common.hw, + [CLK_MM_CSI1_EB] = &mm_csi1_eb.common.hw, + [CLK_MM_CSI0_EB] = &mm_csi0_eb.common.hw, + [CLK_MM_CKG_EB] = &mm_ckg_eb.common.hw, + [CLK_ISP_AHB_EB] = &mm_isp_ahb_eb.common.hw, + [CLK_MM_DVFS_EB] = &mm_dvfs_eb.common.hw, + [CLK_MM_FD_EB] = &mm_fd_eb.common.hw, + [CLK_MM_SENSOR2_EB] = &mm_sensor2_en.common.hw, + [CLK_MM_SENSOR1_EB] = &mm_sensor1_en.common.hw, + [CLK_MM_SENSOR0_EB] = &mm_sensor0_en.common.hw, + [CLK_MM_MIPI_CSI2_EB] = &mm_mipi_csi2_en.common.hw, + [CLK_MM_MIPI_CSI1_EB] = &mm_mipi_csi1_en.common.hw, + [CLK_MM_MIPI_CSI0_EB] = &mm_mipi_csi0_en.common.hw, + [CLK_DCAM_AXI_EB] = &mm_dcam_axi_en.common.hw, + [CLK_ISP_AXI_EB] = &mm_isp_axi_en.common.hw, + [CLK_MM_CPHY_EB] = &mm_cphy_en.common.hw, + }, + .num = CLK_MM_GATE_CLK_NUM, +}; + +static struct sprd_clk_desc ums512_mm_gate_clk_desc = { + .clk_clks = ums512_mm_gate_clk, + .num_clk_clks = ARRAY_SIZE(ums512_mm_gate_clk), + .hw_clks = &ums512_mm_gate_clk_hws, +}; + +/* ap apb gates */ +static SPRD_SC_GATE_CLK_FW_NAME(sim0_eb, "sim0-eb", "ext-26m", + 0x0, 0x1000, BIT(0), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(iis0_eb, "iis0-eb", "ext-26m", + 0x0, 0x1000, BIT(1), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(iis1_eb, "iis1-eb", "ext-26m", + 0x0, 0x1000, BIT(2), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(iis2_eb, "iis2-eb", "ext-26m", + 0x0, 0x1000, BIT(3), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(apb_reg_eb, "apb-reg-eb", "ext-26m", + 0x0, 0x1000, BIT(4), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi0_eb, "spi0-eb", "ext-26m", + 0x0, 0x1000, BIT(5), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi1_eb, "spi1-eb", "ext-26m", + 0x0, 0x1000, BIT(6), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi2_eb, "spi2-eb", "ext-26m", + 0x0, 0x1000, BIT(7), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi3_eb, "spi3-eb", "ext-26m", + 0x0, 0x1000, BIT(8), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c0_eb, "i2c0-eb", "ext-26m", + 0x0, 0x1000, BIT(9), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c1_eb, "i2c1-eb", "ext-26m", + 0x0, 0x1000, BIT(10), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c2_eb, "i2c2-eb", "ext-26m", + 0x0, 0x1000, BIT(11), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c3_eb, "i2c3-eb", "ext-26m", + 0x0, 0x1000, BIT(12), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(i2c4_eb, "i2c4-eb", "ext-26m", + 0x0, 0x1000, BIT(13), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(uart0_eb, "uart0-eb", "ext-26m", + 0x0, 0x1000, BIT(14), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(uart1_eb, "uart1-eb", "ext-26m", + 0x0, 0x1000, BIT(15), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(uart2_eb, "uart2-eb", "ext-26m", + 0x0, 0x1000, BIT(16), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sim0_32k_eb, "sim0-32k-eb", "ext-26m", + 0x0, 0x1000, BIT(17), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi0_lfin_eb, "spi0-lfin-eb", "ext-26m", + 0x0, 0x1000, BIT(18), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi1_lfin_eb, "spi1-lfin-eb", "ext-26m", + 0x0, 0x1000, BIT(19), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi2_lfin_eb, "spi2-lfin-eb", "ext-26m", + 0x0, 0x1000, BIT(20), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(spi3_lfin_eb, "spi3-lfin-eb", "ext-26m", + 0x0, 0x1000, BIT(21), CLK_IGNORE_UNUSED, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio0_eb, "sdio0-eb", "ext-26m", + 0x0, 0x1000, BIT(22), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio1_eb, "sdio1-eb", "ext-26m", + 0x0, 0x1000, BIT(23), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio2_eb, "sdio2-eb", "ext-26m", + 0x0, 0x1000, BIT(24), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(emmc_eb, "emmc-eb", "ext-26m", + 0x0, 0x1000, BIT(25), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio0_32k_eb, "sdio0-32k-eb", "ext-26m", + 0x0, 0x1000, BIT(26), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio1_32k_eb, "sdio1-32k-eb", "ext-26m", + 0x0, 0x1000, BIT(27), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(sdio2_32k_eb, "sdio2-32k-eb", "ext-26m", + 0x0, 0x1000, BIT(28), 0, 0); +static SPRD_SC_GATE_CLK_FW_NAME(emmc_32k_eb, "emmc-32k-eb", "ext-26m", + 0x0, 0x1000, BIT(29), 0, 0); + +static struct sprd_clk_common *ums512_apapb_gate[] = { + /* address base is 0x71000000 */ + &sim0_eb.common, + &iis0_eb.common, + &iis1_eb.common, + &iis2_eb.common, + &apb_reg_eb.common, + &spi0_eb.common, + &spi1_eb.common, + &spi2_eb.common, + &spi3_eb.common, + &i2c0_eb.common, + &i2c1_eb.common, + &i2c2_eb.common, + &i2c3_eb.common, + &i2c4_eb.common, + &uart0_eb.common, + &uart1_eb.common, + &uart2_eb.common, + &sim0_32k_eb.common, + &spi0_lfin_eb.common, + &spi1_lfin_eb.common, + &spi2_lfin_eb.common, + &spi3_lfin_eb.common, + &sdio0_eb.common, + &sdio1_eb.common, + &sdio2_eb.common, + &emmc_eb.common, + &sdio0_32k_eb.common, + &sdio1_32k_eb.common, + &sdio2_32k_eb.common, + &emmc_32k_eb.common, +}; + +static struct clk_hw_onecell_data ums512_apapb_gate_hws = { + .hws = { + [CLK_SIM0_EB] = &sim0_eb.common.hw, + [CLK_IIS0_EB] = &iis0_eb.common.hw, + [CLK_IIS1_EB] = &iis1_eb.common.hw, + [CLK_IIS2_EB] = &iis2_eb.common.hw, + [CLK_APB_REG_EB] = &apb_reg_eb.common.hw, + [CLK_SPI0_EB] = &spi0_eb.common.hw, + [CLK_SPI1_EB] = &spi1_eb.common.hw, + [CLK_SPI2_EB] = &spi2_eb.common.hw, + [CLK_SPI3_EB] = &spi3_eb.common.hw, + [CLK_I2C0_EB] = &i2c0_eb.common.hw, + [CLK_I2C1_EB] = &i2c1_eb.common.hw, + [CLK_I2C2_EB] = &i2c2_eb.common.hw, + [CLK_I2C3_EB] = &i2c3_eb.common.hw, + [CLK_I2C4_EB] = &i2c4_eb.common.hw, + [CLK_UART0_EB] = &uart0_eb.common.hw, + [CLK_UART1_EB] = &uart1_eb.common.hw, + [CLK_UART2_EB] = &uart2_eb.common.hw, + [CLK_SIM0_32K_EB] = &sim0_32k_eb.common.hw, + [CLK_SPI0_LFIN_EB] = &spi0_lfin_eb.common.hw, + [CLK_SPI1_LFIN_EB] = &spi1_lfin_eb.common.hw, + [CLK_SPI2_LFIN_EB] = &spi2_lfin_eb.common.hw, + [CLK_SPI3_LFIN_EB] = &spi3_lfin_eb.common.hw, + [CLK_SDIO0_EB] = &sdio0_eb.common.hw, + [CLK_SDIO1_EB] = &sdio1_eb.common.hw, + [CLK_SDIO2_EB] = &sdio2_eb.common.hw, + [CLK_EMMC_EB] = &emmc_eb.common.hw, + [CLK_SDIO0_32K_EB] = &sdio0_32k_eb.common.hw, + [CLK_SDIO1_32K_EB] = &sdio1_32k_eb.common.hw, + [CLK_SDIO2_32K_EB] = &sdio2_32k_eb.common.hw, + [CLK_EMMC_32K_EB] = &emmc_32k_eb.common.hw, + }, + .num = CLK_AP_APB_GATE_NUM, +}; + +static struct sprd_clk_desc ums512_apapb_gate_desc = { + .clk_clks = ums512_apapb_gate, + .num_clk_clks = ARRAY_SIZE(ums512_apapb_gate), + .hw_clks = &ums512_apapb_gate_hws, +}; + +static const struct of_device_id sprd_ums512_clk_ids[] = { + { .compatible = "sprd,ums512-pmu-gate", /* 0x327e0000 */ + .data = &ums512_pmu_gate_desc }, + { .compatible = "sprd,ums512-g0-pll", /* 0x32390000 */ + .data = &ums512_g0_pll_desc }, + { .compatible = "sprd,ums512-g2-pll", /* 0x323b0000 */ + .data = &ums512_g2_pll_desc }, + { .compatible = "sprd,ums512-g3-pll", /* 0x323c0000 */ + .data = &ums512_g3_pll_desc }, + { .compatible = "sprd,ums512-gc-pll", /* 0x323e0000 */ + .data = &ums512_gc_pll_desc }, + { .compatible = "sprd,ums512-apahb-gate", /* 0x20100000 */ + .data = &ums512_apahb_gate_desc }, + { .compatible = "sprd,ums512-ap-clk", /* 0x20200000 */ + .data = &ums512_ap_clk_desc }, + { .compatible = "sprd,ums512-aonapb-clk", /* 0x32080200 */ + .data = &ums512_aon_apb_desc }, + { .compatible = "sprd,ums512-aon-gate", /* 0x327d0000 */ + .data = &ums512_aon_gate_desc }, + { .compatible = "sprd,ums512-audcpapb-gate", /* 0x3350d000 */ + .data = &ums512_audcpapb_gate_desc }, + { .compatible = "sprd,ums512-audcpahb-gate", /* 0x335e0000 */ + .data = &ums512_audcpahb_gate_desc }, + { .compatible = "sprd,ums512-gpu-clk", /* 0x60100000 */ + .data = &ums512_gpu_clk_desc }, + { .compatible = "sprd,ums512-mm-clk", /* 0x62100000 */ + .data = &ums512_mm_clk_desc }, + { .compatible = "sprd,ums512-mm-gate-clk", /* 0x62200000 */ + .data = &ums512_mm_gate_clk_desc }, + { .compatible = "sprd,ums512-apapb-gate", /* 0x71000000 */ + .data = &ums512_apapb_gate_desc }, + { } +}; +MODULE_DEVICE_TABLE(of, sprd_ums512_clk_ids); + +static int ums512_clk_probe(struct platform_device *pdev) +{ + const struct sprd_clk_desc *desc; + int ret; + + desc = device_get_match_data(&pdev->dev); + if (!desc) + return -ENODEV; + + ret = sprd_clk_regmap_init(pdev, desc); + if (ret) + return ret; + + return sprd_clk_probe(&pdev->dev, desc->hw_clks); +} + +static struct platform_driver ums512_clk_driver = { + .probe = ums512_clk_probe, + .driver = { + .name = "ums512-clk", + .of_match_table = sprd_ums512_clk_ids, + }, +}; +module_platform_driver(ums512_clk_driver); + +MODULE_DESCRIPTION("Unisoc UMS512 Clock Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index 582a22c049194afbb25a201dc431d142c30ac7ac..d820292a381d0a78c818775b9b316c17f2d79e78 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -987,6 +987,7 @@ static void __init st_of_quadfs_setup(struct device_node *np, const char *pll_name, *clk_parent_name; void __iomem *reg; spinlock_t *lock; + struct device_node *parent_np; /* * First check for reg property within the node to keep backward @@ -994,7 +995,9 @@ static void __init st_of_quadfs_setup(struct device_node *np, */ reg = of_iomap(np, 0); if (!reg) { - reg = of_iomap(of_get_parent(np), 0); + parent_np = of_get_parent(np); + reg = of_iomap(parent_np, 0); + of_node_put(parent_np); if (!reg) { pr_err("%s: Failed to get base address\n", __func__); return; diff --git a/drivers/clk/st/clkgen-mux.c b/drivers/clk/st/clkgen-mux.c index ee39af7a0b7215411b607940a5c52a4c989cfaf9..596e939ad905e02b5ba195c3ec16cc43bd2d6e56 100644 --- a/drivers/clk/st/clkgen-mux.c +++ b/drivers/clk/st/clkgen-mux.c @@ -56,6 +56,7 @@ static void __init st_of_clkgen_mux_setup(struct device_node *np, void __iomem *reg; const char **parents; int num_parents = 0; + struct device_node *parent_np; /* * First check for reg property within the node to keep backward @@ -63,7 +64,9 @@ static void __init st_of_clkgen_mux_setup(struct device_node *np, */ reg = of_iomap(np, 0); if (!reg) { - reg = of_iomap(of_get_parent(np), 0); + parent_np = of_get_parent(np); + reg = of_iomap(parent_np, 0); + of_node_put(parent_np); if (!reg) { pr_err("%s: Failed to get base address\n", __func__); return; diff --git a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c index 51058ba4db4d202e0108374e56964a89fc97b58c..8ef3cdeb79625b4c169fe264fee4dc408535ba20 100644 --- a/drivers/clk/sunxi-ng/ccu-sun20i-d1.c +++ b/drivers/clk/sunxi-ng/ccu-sun20i-d1.c @@ -104,6 +104,8 @@ static struct ccu_nm pll_video0_4x_clk = { .lock = BIT(28), .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .min_rate = 252000000U, + .max_rate = 2400000000U, .common = { .reg = 0x040, .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-video0-4x", osc24M, @@ -126,6 +128,8 @@ static struct ccu_nm pll_video1_4x_clk = { .lock = BIT(28), .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .min_rate = 252000000U, + .max_rate = 2400000000U, .common = { .reg = 0x048, .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-video1-4x", osc24M, @@ -175,6 +179,8 @@ static struct ccu_nm pll_audio0_4x_clk = { .m = _SUNXI_CCU_DIV(16, 6), .sdm = _SUNXI_CCU_SDM(pll_audio0_sdm_table, BIT(24), 0x178, BIT(31)), + .min_rate = 180000000U, + .max_rate = 3000000000U, .common = { .reg = 0x078, .features = CCU_FEATURE_SIGMA_DELTA_MOD, @@ -202,6 +208,8 @@ static struct ccu_nm pll_audio1_clk = { .lock = BIT(28), .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), .m = _SUNXI_CCU_DIV(1, 1), + .min_rate = 180000000U, + .max_rate = 3000000000U, .common = { .reg = 0x080, .hw.init = CLK_HW_INIT_PARENTS_DATA("pll-audio1", osc24M, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index 30056da3e0af80387fb729144afc74643ab8e24d..42568c6161814dc797079d1ee224bc7a2e6847a8 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -1191,9 +1191,13 @@ static int sun50i_h6_ccu_probe(struct platform_device *pdev) if (IS_ERR(reg)) return PTR_ERR(reg); - /* Force PLL_GPU output divider bits to 0 */ + /* + * Force PLL_GPU output divider bits to 0 and adjust + * multiplier to sensible default value of 432 MHz. + */ val = readl(reg + SUN50I_H6_PLL_GPU_REG); - val &= ~BIT(0); + val &= ~(GENMASK(15, 8) | BIT(0)); + val |= 17 << 8; writel(val, reg + SUN50I_H6_PLL_GPU_REG); /* Force GPU_CLK divider bits to 0 */ diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c index 2f6f02f00be24f7f25f3d1829bd8f335e931f9f3..b70b312e74836fef0b3aed9607adf78c7cbf0155 100644 --- a/drivers/clk/sunxi-ng/ccu-sun8i-de2.c +++ b/drivers/clk/sunxi-ng/ccu-sun8i-de2.c @@ -256,29 +256,19 @@ static int sunxi_de2_clk_probe(struct platform_device *pdev) return PTR_ERR(reg); bus_clk = devm_clk_get(&pdev->dev, "bus"); - if (IS_ERR(bus_clk)) { - ret = PTR_ERR(bus_clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Couldn't get bus clk: %d\n", ret); - return ret; - } + if (IS_ERR(bus_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(bus_clk), + "Couldn't get bus clk\n"); mod_clk = devm_clk_get(&pdev->dev, "mod"); - if (IS_ERR(mod_clk)) { - ret = PTR_ERR(mod_clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Couldn't get mod clk: %d\n", ret); - return ret; - } + if (IS_ERR(mod_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(mod_clk), + "Couldn't get mod clk\n"); rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(rstc)) { - ret = PTR_ERR(rstc); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "Couldn't get reset control: %d\n", ret); - return ret; - } + if (IS_ERR(rstc)) + return dev_err_probe(&pdev->dev, PTR_ERR(rstc), + "Couldn't get reset control\n"); /* The clocks need to be enabled for us to access the registers */ ret = clk_prepare_enable(bus_clk); diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c index f2fe0e1cc3c0bf9eddef7fa6c31c87bef7cf1a5b..1d8b1ae1619df8c37d11b03ea6d13d5d31a26360 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-de.c @@ -213,21 +213,14 @@ static int sun9i_a80_de_clk_probe(struct platform_device *pdev) return PTR_ERR(reg); bus_clk = devm_clk_get(&pdev->dev, "bus"); - if (IS_ERR(bus_clk)) { - ret = PTR_ERR(bus_clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Couldn't get bus clk: %d\n", ret); - return ret; - } + if (IS_ERR(bus_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(bus_clk), + "Couldn't get bus clk\n"); rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(rstc)) { - ret = PTR_ERR(rstc); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "Couldn't get reset control: %d\n", ret); - return ret; - } + if (IS_ERR(rstc)) + return dev_err_probe(&pdev->dev, PTR_ERR(rstc), + "Couldn't get reset control\n"); /* The bus clock needs to be enabled for us to access the registers */ ret = clk_prepare_enable(bus_clk); diff --git a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c index 575ae4ccc65fc89664c94e74098b551aa668e17b..a0fb0da8f3563fb7833a56d8162571eb16e87dec 100644 --- a/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c +++ b/drivers/clk/sunxi-ng/ccu-sun9i-a80-usb.c @@ -101,12 +101,9 @@ static int sun9i_a80_usb_clk_probe(struct platform_device *pdev) return PTR_ERR(reg); bus_clk = devm_clk_get(&pdev->dev, "bus"); - if (IS_ERR(bus_clk)) { - ret = PTR_ERR(bus_clk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Couldn't get bus clk: %d\n", ret); - return ret; - } + if (IS_ERR(bus_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(bus_clk), + "Couldn't get bus clk\n"); /* The bus clock needs to be enabled for us to access the registers */ ret = clk_prepare_enable(bus_clk); diff --git a/drivers/clk/tegra/clk-bpmp.c b/drivers/clk/tegra/clk-bpmp.c index 3748a39dae7cb442c5a29527f317a7605f83bdea..d82a71f10c2c182309653e4c1ea7230e79396af8 100644 --- a/drivers/clk/tegra/clk-bpmp.c +++ b/drivers/clk/tegra/clk-bpmp.c @@ -349,7 +349,7 @@ static int tegra_bpmp_clk_get_info(struct tegra_bpmp *bpmp, unsigned int id, if (err < 0) return err; - strlcpy(info->name, response.name, MRQ_CLK_NAME_MAXLEN); + strscpy(info->name, response.name, MRQ_CLK_NAME_MAXLEN); info->num_parents = response.num_parents; for (i = 0; i < info->num_parents; i++) diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c index ef718c4b3826712cfced14cb1d31521d99cfed2c..73303458e88667b6ab1023d5595d250afe59492c 100644 --- a/drivers/clk/tegra/clk-tegra114.c +++ b/drivers/clk/tegra/clk-tegra114.c @@ -1166,6 +1166,7 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA114_CLK_I2S3_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, { TEGRA114_CLK_I2S4_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, { TEGRA114_CLK_VIMCLK_SYNC, TEGRA114_CLK_CLK_MAX, 24000000, 0 }, + { TEGRA114_CLK_PWM, TEGRA114_CLK_PLL_P, 408000000, 0 }, /* must be the last entry */ { TEGRA114_CLK_CLK_MAX, TEGRA114_CLK_CLK_MAX, 0, 0 }, }; @@ -1317,6 +1318,7 @@ static void __init tegra114_clock_init(struct device_node *np) } pmc_base = of_iomap(node, 0); + of_node_put(node); if (!pmc_base) { pr_err("Can't map pmc registers\n"); WARN_ON(1); diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c index 934520aab6e38991ae3a12b72b55304980a6c93f..6c46592d794ec909a8257049cb9cbc398ebadbd9 100644 --- a/drivers/clk/tegra/clk-tegra124.c +++ b/drivers/clk/tegra/clk-tegra124.c @@ -1330,6 +1330,7 @@ static struct tegra_clk_init_table common_init_table[] __initdata = { { TEGRA124_CLK_I2S3_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 }, { TEGRA124_CLK_I2S4_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 }, { TEGRA124_CLK_VIMCLK_SYNC, TEGRA124_CLK_CLK_MAX, 24576000, 0 }, + { TEGRA124_CLK_PWM, TEGRA124_CLK_PLL_P, 408000000, 0 }, /* must be the last entry */ { TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0 }, }; @@ -1471,6 +1472,7 @@ static void __init tegra124_132_clock_init_pre(struct device_node *np) } pmc_base = of_iomap(node, 0); + of_node_put(node); if (!pmc_base) { pr_err("Can't map pmc registers\n"); WARN_ON(1); diff --git a/drivers/clk/tegra/clk-tegra20.c b/drivers/clk/tegra/clk-tegra20.c index be3c33441cfc41d4584006b050991f121c22db2e..422d782475532b3362cdd5f42664dd53298763b5 100644 --- a/drivers/clk/tegra/clk-tegra20.c +++ b/drivers/clk/tegra/clk-tegra20.c @@ -1044,6 +1044,7 @@ static struct tegra_clk_init_table init_table[] = { { TEGRA20_CLK_GR2D, TEGRA20_CLK_PLL_C, 300000000, 0 }, { TEGRA20_CLK_GR3D, TEGRA20_CLK_PLL_C, 300000000, 0 }, { TEGRA20_CLK_VDE, TEGRA20_CLK_PLL_C, 300000000, 0 }, + { TEGRA20_CLK_PWM, TEGRA20_CLK_PLL_P, 48000000, 0 }, /* must be the last entry */ { TEGRA20_CLK_CLK_MAX, TEGRA20_CLK_CLK_MAX, 0, 0 }, }; @@ -1131,6 +1132,7 @@ static void __init tegra20_clock_init(struct device_node *np) } pmc_base = of_iomap(node, 0); + of_node_put(node); if (!pmc_base) { pr_err("Can't map pmc registers\n"); BUG(); diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c index b9099012dc7b1d3d24accd2d1a65881fb2938103..a3488aaac3f78f040722357a197b766a5d80825f 100644 --- a/drivers/clk/tegra/clk-tegra210.c +++ b/drivers/clk/tegra/clk-tegra210.c @@ -3597,6 +3597,7 @@ static struct tegra_clk_init_table init_table[] __initdata = { { TEGRA210_CLK_VIMCLK_SYNC, TEGRA210_CLK_CLK_MAX, 24576000, 0 }, { TEGRA210_CLK_HDA, TEGRA210_CLK_PLL_P, 51000000, 0 }, { TEGRA210_CLK_HDA2CODEC_2X, TEGRA210_CLK_PLL_P, 48000000, 0 }, + { TEGRA210_CLK_PWM, TEGRA210_CLK_PLL_P, 48000000, 0 }, /* This MUST be the last entry. */ { TEGRA210_CLK_CLK_MAX, TEGRA210_CLK_CLK_MAX, 0, 0 }, }; @@ -3748,6 +3749,7 @@ static void __init tegra210_clock_init(struct device_node *np) } pmc_base = of_iomap(node, 0); + of_node_put(node); if (!pmc_base) { pr_err("Can't map pmc registers\n"); WARN_ON(1); diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index 04b4961238209d8d81bbbe8883a32e39be4b4ee2..60f1534711f1c2dc4553b1ce693ab3ac70697174 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -1237,6 +1237,7 @@ static struct tegra_clk_init_table init_table[] = { { TEGRA30_CLK_VIMCLK_SYNC, TEGRA30_CLK_CLK_MAX, 24000000, 0 }, { TEGRA30_CLK_HDA, TEGRA30_CLK_PLL_P, 102000000, 0 }, { TEGRA30_CLK_HDA2CODEC_2X, TEGRA30_CLK_PLL_P, 48000000, 0 }, + { TEGRA30_CLK_PWM, TEGRA30_CLK_PLL_P, 48000000, 0 }, /* must be the last entry */ { TEGRA30_CLK_CLK_MAX, TEGRA30_CLK_CLK_MAX, 0, 0 }, }; @@ -1320,6 +1321,7 @@ static void __init tegra30_clock_init(struct device_node *np) } pmc_base = of_iomap(node, 0); + of_node_put(node); if (!pmc_base) { pr_err("Can't map pmc registers\n"); BUG(); diff --git a/drivers/clk/ti/clk-dra7-atl.c b/drivers/clk/ti/clk-dra7-atl.c index f0f5bf68b6d23046e53acb4660d4913034028f98..ff4d6a9516813c30eb063dde75a6d64cf356014e 100644 --- a/drivers/clk/ti/clk-dra7-atl.c +++ b/drivers/clk/ti/clk-dra7-atl.c @@ -245,14 +245,16 @@ static int of_dra7_atl_clk_probe(struct platform_device *pdev) if (rc) { pr_err("%s: failed to lookup atl clock %d\n", __func__, i); - return -EINVAL; + ret = -EINVAL; + goto pm_put; } clk = of_clk_get_from_provider(&clkspec); if (IS_ERR(clk)) { pr_err("%s: failed to get atl clock %d from provider\n", __func__, i); - return PTR_ERR(clk); + ret = PTR_ERR(clk); + goto pm_put; } cdesc = to_atl_desc(__clk_get_hw(clk)); @@ -285,8 +287,9 @@ static int of_dra7_atl_clk_probe(struct platform_device *pdev) if (cdesc->enabled) atl_clk_enable(__clk_get_hw(clk)); } - pm_runtime_put_sync(cinfo->dev); +pm_put: + pm_runtime_put_sync(cinfo->dev); return ret; } diff --git a/drivers/clk/ti/clk.c b/drivers/clk/ti/clk.c index ef2a445c63a3c6cbf09ff2648da3cb81435c23b3..1dc2f15fb75b2657a48a05392485561cfa3120f1 100644 --- a/drivers/clk/ti/clk.c +++ b/drivers/clk/ti/clk.c @@ -135,15 +135,17 @@ static struct device_node *ti_find_clock_provider(struct device_node *from, continue; if (!strncmp(n, tmp, strlen(tmp))) { + of_node_get(np); found = true; break; } } - of_node_put(from); kfree(tmp); - if (found) + if (found) { + of_node_put(from); return np; + } /* Fall back to using old node name base provider name */ return of_find_node_by_name(from, name); diff --git a/drivers/clk/xilinx/Kconfig b/drivers/clk/xilinx/Kconfig index 5224114176ed60db3b4168467894c1453b481473..f205522c40ff4c4c420e46f61bfda507c95f5cc4 100644 --- a/drivers/clk/xilinx/Kconfig +++ b/drivers/clk/xilinx/Kconfig @@ -17,3 +17,15 @@ config XILINX_VCU To compile this driver as a module, choose M here: the module will be called xlnx_vcu. +config COMMON_CLK_XLNX_CLKWZRD + tristate "Xilinx Clocking Wizard" + depends on COMMON_CLK && OF + depends on HAS_IOMEM + help + Support for the Xilinx Clocking Wizard IP core clock generator. + Adds support for clocking wizard and compatible. + This driver supports the Xilinx clocking wizard programmable clock + synthesizer. The number of output is configurable in the design. + + If unsure, say N. + diff --git a/drivers/clk/xilinx/Makefile b/drivers/clk/xilinx/Makefile index dee8fd51e303281968e2c2f198d403a5ac573473..7ac1789c6b1bcaef4f632180669a63c69e1b57d1 100644 --- a/drivers/clk/xilinx/Makefile +++ b/drivers/clk/xilinx/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_XILINX_VCU) += xlnx_vcu.o +obj-$(CONFIG_COMMON_CLK_XLNX_CLKWZRD) += clk-xlnx-clock-wizard.o diff --git a/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c similarity index 94% rename from drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c rename to drivers/clk/xilinx/clk-xlnx-clock-wizard.c index 39367712ef540ab62ff6c21329a2cd7f111e32b3..eb1dfe7ecc1b43edddc1db13ca5b32d99a58e97f 100644 --- a/drivers/staging/clocking-wizard/clk-xlnx-clock-wizard.c +++ b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c @@ -2,9 +2,10 @@ /* * Xilinx 'Clocking Wizard' driver * - * Copyright (C) 2013 - 2014 Xilinx + * Copyright (C) 2013 - 2021 Xilinx * * Sören Brinkmann + * */ #include @@ -43,6 +44,8 @@ #define WZRD_DR_INIT_REG_OFFSET 0x25C #define WZRD_DR_DIV_TO_PHASE_OFFSET 4 #define WZRD_DR_BEGIN_DYNA_RECONF 0x03 +#define WZRD_DR_BEGIN_DYNA_RECONF_5_2 0x07 +#define WZRD_DR_BEGIN_DYNA_RECONF1_5_2 0x02 #define WZRD_USEC_POLL 10 #define WZRD_TIMEOUT_POLL 1000 @@ -164,7 +167,9 @@ static int clk_wzrd_dynamic_reconfig(struct clk_hw *hw, unsigned long rate, goto err_reconfig; /* Initiate reconfiguration */ - writel(WZRD_DR_BEGIN_DYNA_RECONF, + writel(WZRD_DR_BEGIN_DYNA_RECONF_5_2, + divider->base + WZRD_DR_INIT_REG_OFFSET); + writel(WZRD_DR_BEGIN_DYNA_RECONF1_5_2, divider->base + WZRD_DR_INIT_REG_OFFSET); /* Check status register */ @@ -223,7 +228,7 @@ static int clk_wzrd_dynamic_reconfig_f(struct clk_hw *hw, unsigned long rate, struct clk_wzrd_divider *divider = to_clk_wzrd_divider(hw); void __iomem *div_addr = divider->base + divider->offset; - rate_div = ((parent_rate * 1000) / rate); + rate_div = DIV_ROUND_DOWN_ULL(parent_rate * 1000, rate); clockout0_div = rate_div / 1000; pre = DIV_ROUND_CLOSEST((parent_rate * 1000), rate); @@ -245,7 +250,9 @@ static int clk_wzrd_dynamic_reconfig_f(struct clk_hw *hw, unsigned long rate, return err; /* Initiate reconfiguration */ - writel(WZRD_DR_BEGIN_DYNA_RECONF, + writel(WZRD_DR_BEGIN_DYNA_RECONF_5_2, + divider->base + WZRD_DR_INIT_REG_OFFSET); + writel(WZRD_DR_BEGIN_DYNA_RECONF1_5_2, divider->base + WZRD_DR_INIT_REG_OFFSET); /* Check status register */ @@ -441,18 +448,14 @@ static int clk_wzrd_probe(struct platform_device *pdev) } clk_wzrd->clk_in1 = devm_clk_get(&pdev->dev, "clk_in1"); - if (IS_ERR(clk_wzrd->clk_in1)) { - if (clk_wzrd->clk_in1 != ERR_PTR(-EPROBE_DEFER)) - dev_err(&pdev->dev, "clk_in1 not found\n"); - return PTR_ERR(clk_wzrd->clk_in1); - } + if (IS_ERR(clk_wzrd->clk_in1)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk_wzrd->clk_in1), + "clk_in1 not found\n"); clk_wzrd->axi_clk = devm_clk_get(&pdev->dev, "s_axi_aclk"); - if (IS_ERR(clk_wzrd->axi_clk)) { - if (clk_wzrd->axi_clk != ERR_PTR(-EPROBE_DEFER)) - dev_err(&pdev->dev, "s_axi_aclk not found\n"); - return PTR_ERR(clk_wzrd->axi_clk); - } + if (IS_ERR(clk_wzrd->axi_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk_wzrd->axi_clk), + "s_axi_aclk not found\n"); ret = clk_prepare_enable(clk_wzrd->axi_clk); if (ret) { dev_err(&pdev->dev, "enabling s_axi_aclk failed\n"); @@ -479,7 +482,7 @@ static int clk_wzrd_probe(struct platform_device *pdev) goto err_disable_clk; } - ret = of_property_read_u32(np, "nr-outputs", &nr_outputs); + ret = of_property_read_u32(np, "xlnx,nr-outputs", &nr_outputs); if (ret || nr_outputs > WZRD_NUM_OUTPUTS) { ret = -EINVAL; goto err_disable_clk; @@ -614,6 +617,8 @@ static int clk_wzrd_remove(struct platform_device *pdev) static const struct of_device_id clk_wzrd_ids[] = { { .compatible = "xlnx,clocking-wizard" }, + { .compatible = "xlnx,clocking-wizard-v5.2" }, + { .compatible = "xlnx,clocking-wizard-v6.0" }, { }, }; MODULE_DEVICE_TABLE(of, clk_wzrd_ids); diff --git a/drivers/clk/zynqmp/clkc.c b/drivers/clk/zynqmp/clkc.c index eb25303eefed478b0faf1778a8c05143c898a442..5636ff1ce552deb56fcc6251e5cb6c2aa6d55076 100644 --- a/drivers/clk/zynqmp/clkc.c +++ b/drivers/clk/zynqmp/clkc.c @@ -163,7 +163,7 @@ static int zynqmp_get_clock_name(u32 clk_id, char *clk_name) ret = zynqmp_is_valid_clock(clk_id); if (ret == 1) { - strncpy(clk_name, clock[clk_id].clk_name, MAX_NAME_LEN); + strscpy(clk_name, clock[clk_id].clk_name, MAX_NAME_LEN); return 0; } @@ -220,18 +220,22 @@ static int zynqmp_pm_clock_get_num_clocks(u32 *nclocks) * This function is used to get name of clock specified by given * clock ID. * - * Return: Returns 0 + * Return: 0 on success else error+reason */ static int zynqmp_pm_clock_get_name(u32 clock_id, struct name_resp *response) { struct zynqmp_pm_query_data qdata = {0}; u32 ret_payload[PAYLOAD_ARG_CNT]; + int ret; qdata.qid = PM_QID_CLOCK_GET_NAME; qdata.arg1 = clock_id; - zynqmp_pm_query_data(qdata, ret_payload); + ret = zynqmp_pm_query_data(qdata, ret_payload); + if (ret) + return ret; + memcpy(response, ret_payload, sizeof(*response)); return 0; @@ -710,9 +714,16 @@ static void zynqmp_get_clock_info(void) FIELD_PREP(CLK_ATTR_NODE_INDEX, i); zynqmp_pm_clock_get_name(clock[i].clk_id, &name); + + /* + * Terminate with NULL character in case name provided by firmware + * is longer and truncated due to size limit. + */ + name.name[sizeof(name.name) - 1] = '\0'; + if (!strcmp(name.name, RESERVED_CLK_NAME)) continue; - strncpy(clock[i].clk_name, name.name, MAX_NAME_LEN); + strscpy(clock[i].clk_name, name.name, MAX_NAME_LEN); } /* Get topology of all clock */ diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index 422ea79907dd038fbc4442e5d8318e0edef5afac..33a3b2a226595de9ae7be6fa48e573e0bf680e52 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -113,17 +113,20 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw, static void zynqmp_get_divider2_val(struct clk_hw *hw, unsigned long rate, struct zynqmp_clk_divider *divider, - int *bestdiv) + u32 *bestdiv) { int div1; int div2; long error = LONG_MAX; unsigned long div1_prate; struct clk_hw *div1_parent_hw; + struct zynqmp_clk_divider *pdivider; struct clk_hw *div2_parent_hw = clk_hw_get_parent(hw); - struct zynqmp_clk_divider *pdivider = - to_zynqmp_clk_divider(div2_parent_hw); + if (!div2_parent_hw) + return; + + pdivider = to_zynqmp_clk_divider(div2_parent_hw); if (!pdivider) return; diff --git a/drivers/clk/zynqmp/pll.c b/drivers/clk/zynqmp/pll.c index 91a6b4cc910eb15b5bc098b9074d92cd7c322532..0d3e1377b092ca149885bd0c7d5c92a4edd06916 100644 --- a/drivers/clk/zynqmp/pll.c +++ b/drivers/clk/zynqmp/pll.c @@ -102,26 +102,25 @@ static long zynqmp_pll_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *prate) { u32 fbdiv; - long rate_div, f; + u32 mult, div; - /* Enable the fractional mode if needed */ - rate_div = (rate * FRAC_DIV) / *prate; - f = rate_div % FRAC_DIV; - if (f) { - if (rate > PS_PLL_VCO_MAX) { - fbdiv = rate / PS_PLL_VCO_MAX; - rate = rate / (fbdiv + 1); - } - if (rate < PS_PLL_VCO_MIN) { - fbdiv = DIV_ROUND_UP(PS_PLL_VCO_MIN, rate); - rate = rate * fbdiv; - } - return rate; + /* Let rate fall inside the range PS_PLL_VCO_MIN ~ PS_PLL_VCO_MAX */ + if (rate > PS_PLL_VCO_MAX) { + div = DIV_ROUND_UP(rate, PS_PLL_VCO_MAX); + rate = rate / div; + } + if (rate < PS_PLL_VCO_MIN) { + mult = DIV_ROUND_UP(PS_PLL_VCO_MIN, rate); + rate = rate * mult; } fbdiv = DIV_ROUND_CLOSEST(rate, *prate); - fbdiv = clamp_t(u32, fbdiv, PLL_FBDIV_MIN, PLL_FBDIV_MAX); - return *prate * fbdiv; + if (fbdiv < PLL_FBDIV_MIN || fbdiv > PLL_FBDIV_MAX) { + fbdiv = clamp_t(u32, fbdiv, PLL_FBDIV_MIN, PLL_FBDIV_MAX); + rate = *prate * fbdiv; + } + + return rate; } /** diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 4f2bb7315b67ae9fb8e0e9ac8d63b5fafe8ff2fe..4469e7f555e9793192e6d150a04ac55cf60bf583 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -434,7 +434,7 @@ config ATMEL_TCB_CLKSRC config CLKSRC_EXYNOS_MCT bool "Exynos multi core timer driver" if COMPILE_TEST depends on ARM || ARM64 - depends on ARCH_EXYNOS || COMPILE_TEST + depends on ARCH_ARTPEC || ARCH_EXYNOS || COMPILE_TEST help Support for Multi Core Timer controller on Exynos SoCs. diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 9ab8221ee3c65915367a8225d25266cef83291c3..a7ff77550e1737e291ef07ffc5ef1ebc83509d99 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -44,8 +44,8 @@ #define CNTACR_RWVT BIT(4) #define CNTACR_RWPT BIT(5) -#define CNTVCT_LO 0x00 -#define CNTPCT_LO 0x08 +#define CNTPCT_LO 0x00 +#define CNTVCT_LO 0x08 #define CNTFRQ 0x10 #define CNTP_CVAL_LO 0x20 #define CNTP_CTL 0x2c @@ -473,6 +473,8 @@ static const struct arch_timer_erratum_workaround ool_workarounds[] = { .desc = "ARM erratum 858921", .read_cntpct_el0 = arm64_858921_read_cntpct_el0, .read_cntvct_el0 = arm64_858921_read_cntvct_el0, + .set_next_event_phys = erratum_set_next_event_phys, + .set_next_event_virt = erratum_set_next_event_virt, }, #endif #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1 diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index f29c812b70c99e5231da68a380398b3544810417..bfd60093ee1c934c91a4a5d1dec1f583ba0f12fc 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -33,7 +33,7 @@ #define EXYNOS4_MCT_G_INT_ENB EXYNOS4_MCTREG(0x248) #define EXYNOS4_MCT_G_WSTAT EXYNOS4_MCTREG(0x24C) #define _EXYNOS4_MCT_L_BASE EXYNOS4_MCTREG(0x300) -#define EXYNOS4_MCT_L_BASE(x) (_EXYNOS4_MCT_L_BASE + (0x100 * x)) +#define EXYNOS4_MCT_L_BASE(x) (_EXYNOS4_MCT_L_BASE + (0x100 * (x))) #define EXYNOS4_MCT_L_MASK (0xffffff00) #define MCT_L_TCNTB_OFFSET (0x00) @@ -66,6 +66,8 @@ #define MCT_L0_IRQ 4 /* Max number of IRQ as per DT binding document */ #define MCT_NR_IRQS 20 +/* Max number of local timers */ +#define MCT_NR_LOCAL (MCT_NR_IRQS - MCT_L0_IRQ) enum { MCT_INT_SPI, @@ -233,9 +235,16 @@ static cycles_t exynos4_read_current_timer(void) } #endif -static int __init exynos4_clocksource_init(void) +static int __init exynos4_clocksource_init(bool frc_shared) { - exynos4_mct_frc_start(); + /* + * When the frc is shared, the main processer should have already + * turned it on and we shouldn't be writing to TCON. + */ + if (frc_shared) + mct_frc.resume = NULL; + else + exynos4_mct_frc_start(); #if defined(CONFIG_ARM) exynos4_delay_timer.read_current_timer = &exynos4_read_current_timer; @@ -449,7 +458,6 @@ static int exynos4_mct_starting_cpu(unsigned int cpu) per_cpu_ptr(&percpu_mct_tick, cpu); struct clock_event_device *evt = &mevt->evt; - mevt->base = EXYNOS4_MCT_L_BASE(cpu); snprintf(mevt->name, sizeof(mevt->name), "mct_tick%d", cpu); evt->name = mevt->name; @@ -520,8 +528,17 @@ static int __init exynos4_timer_resources(struct device_node *np) return 0; } +/** + * exynos4_timer_interrupts - initialize MCT interrupts + * @np: device node for MCT + * @int_type: interrupt type, MCT_INT_PPI or MCT_INT_SPI + * @local_idx: array mapping CPU numbers to local timer indices + * @nr_local: size of @local_idx array + */ static int __init exynos4_timer_interrupts(struct device_node *np, - unsigned int int_type) + unsigned int int_type, + const u32 *local_idx, + size_t nr_local) { int nr_irqs, i, err, cpu; @@ -554,13 +571,21 @@ static int __init exynos4_timer_interrupts(struct device_node *np, } else { for_each_possible_cpu(cpu) { int mct_irq; + unsigned int irq_idx; struct mct_clock_event_device *pcpu_mevt = per_cpu_ptr(&percpu_mct_tick, cpu); + if (cpu >= nr_local) { + err = -EINVAL; + goto out_irq; + } + + irq_idx = MCT_L0_IRQ + local_idx[cpu]; + pcpu_mevt->evt.irq = -1; - if (MCT_L0_IRQ + cpu >= ARRAY_SIZE(mct_irqs)) + if (irq_idx >= ARRAY_SIZE(mct_irqs)) break; - mct_irq = mct_irqs[MCT_L0_IRQ + cpu]; + mct_irq = mct_irqs[irq_idx]; irq_set_status_flags(mct_irq, IRQ_NOAUTOEN); if (request_irq(mct_irq, @@ -576,6 +601,17 @@ static int __init exynos4_timer_interrupts(struct device_node *np, } } + for_each_possible_cpu(cpu) { + struct mct_clock_event_device *mevt = per_cpu_ptr(&percpu_mct_tick, cpu); + + if (cpu >= nr_local) { + err = -EINVAL; + goto out_irq; + } + + mevt->base = EXYNOS4_MCT_L_BASE(local_idx[cpu]); + } + /* Install hotplug callbacks which configure the timer on this CPU */ err = cpuhp_setup_state(CPUHP_AP_EXYNOS4_MCT_TIMER_STARTING, "clockevents/exynos4/mct_timer:starting", @@ -605,20 +641,49 @@ out_irq: static int __init mct_init_dt(struct device_node *np, unsigned int int_type) { + bool frc_shared = of_property_read_bool(np, "samsung,frc-shared"); + u32 local_idx[MCT_NR_LOCAL] = {0}; + int nr_local; int ret; + nr_local = of_property_count_u32_elems(np, "samsung,local-timers"); + if (nr_local == 0) + return -EINVAL; + if (nr_local > 0) { + if (nr_local > ARRAY_SIZE(local_idx)) + return -EINVAL; + + ret = of_property_read_u32_array(np, "samsung,local-timers", + local_idx, nr_local); + if (ret) + return ret; + } else { + int i; + + nr_local = ARRAY_SIZE(local_idx); + for (i = 0; i < nr_local; i++) + local_idx[i] = i; + } + ret = exynos4_timer_resources(np); if (ret) return ret; - ret = exynos4_timer_interrupts(np, int_type); + ret = exynos4_timer_interrupts(np, int_type, local_idx, nr_local); if (ret) return ret; - ret = exynos4_clocksource_init(); + ret = exynos4_clocksource_init(frc_shared); if (ret) return ret; + /* + * When the FRC is shared with a main processor, this secondary + * processor cannot use the global comparator. + */ + if (frc_shared) + return ret; + return exynos4_clockevent_init(); } diff --git a/drivers/clocksource/renesas-ostm.c b/drivers/clocksource/renesas-ostm.c index 21d1392637b8dfc2e52f6b224adfd2cb9fac7f29..8da972dc171365bc88f7a5390155f999d9804d0e 100644 --- a/drivers/clocksource/renesas-ostm.c +++ b/drivers/clocksource/renesas-ostm.c @@ -224,7 +224,7 @@ err_free: TIMER_OF_DECLARE(ostm, "renesas,ostm", ostm_init); -#ifdef CONFIG_ARCH_R9A07G044 +#ifdef CONFIG_ARCH_RZG2L static int __init ostm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/drivers/clocksource/timer-gxp.c b/drivers/clocksource/timer-gxp.c index 8b38b32123880aa0782fe2a6b7cbd12c3ebcbb4d..fe4fa8d7b3f1322d259f5c49a13261efe6b81dfa 100644 --- a/drivers/clocksource/timer-gxp.c +++ b/drivers/clocksource/timer-gxp.c @@ -171,6 +171,7 @@ static int gxp_timer_probe(struct platform_device *pdev) { struct platform_device *gxp_watchdog_device; struct device *dev = &pdev->dev; + int ret; if (!gxp_timer) { pr_err("Gxp Timer not initialized, cannot create watchdog"); @@ -187,7 +188,11 @@ static int gxp_timer_probe(struct platform_device *pdev) gxp_watchdog_device->dev.platform_data = gxp_timer->counter; gxp_watchdog_device->dev.parent = dev; - return platform_device_add(gxp_watchdog_device); + ret = platform_device_add(gxp_watchdog_device); + if (ret) + platform_device_put(gxp_watchdog_device); + + return ret; } static const struct of_device_id gxp_timer_of_match[] = { diff --git a/drivers/clocksource/timer-imx-sysctr.c b/drivers/clocksource/timer-imx-sysctr.c index 523e37662a6e8e505c20d802e8a1a41ff9c5ac0b..5a7a951c4efcdf187e73e2190b27dcea943f97bc 100644 --- a/drivers/clocksource/timer-imx-sysctr.c +++ b/drivers/clocksource/timer-imx-sysctr.c @@ -134,8 +134,10 @@ static int __init sysctr_timer_init(struct device_node *np) if (ret) return ret; - /* system counter clock is divided by 3 internally */ - to_sysctr.of_clk.rate /= SYS_CTR_CLK_DIV; + if (!of_property_read_bool(np, "nxp,no-divider")) { + /* system counter clock is divided by 3 internally */ + to_sysctr.of_clk.rate /= SYS_CTR_CLK_DIV; + } sys_ctr_base = timer_of_base(&to_sysctr); cmpcr = readl(sys_ctr_base + CMPCR); diff --git a/drivers/clocksource/timer-sun4i.c b/drivers/clocksource/timer-sun4i.c index 94dc6e42e983d8f6bcb447e89880841d65bf91d5..e5a70aa1deb4633e7525953871d069895de0f7d7 100644 --- a/drivers/clocksource/timer-sun4i.c +++ b/drivers/clocksource/timer-sun4i.c @@ -26,6 +26,7 @@ #define TIMER_IRQ_EN_REG 0x00 #define TIMER_IRQ_EN(val) BIT(val) #define TIMER_IRQ_ST_REG 0x04 +#define TIMER_IRQ_CLEAR(val) BIT(val) #define TIMER_CTL_REG(val) (0x10 * val + 0x10) #define TIMER_CTL_ENABLE BIT(0) #define TIMER_CTL_RELOAD BIT(1) @@ -123,7 +124,7 @@ static int sun4i_clkevt_next_event(unsigned long evt, static void sun4i_timer_clear_interrupt(void __iomem *base) { - writel(TIMER_IRQ_EN(0), base + TIMER_IRQ_ST_REG); + writel(TIMER_IRQ_CLEAR(0), base + TIMER_IRQ_ST_REG); } static irqreturn_t sun4i_timer_interrupt(int irq, void *dev_id) diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c index 469f7c91564b664ffeb0317d5e9b1f29579c30ac..cad29ded3a48fd182334e8a218acca55795e1697 100644 --- a/drivers/clocksource/timer-ti-dm.c +++ b/drivers/clocksource/timer-ti-dm.c @@ -33,6 +33,116 @@ #include +/* + * timer errata flags + * + * Errata i103/i767 impacts all OMAP3/4/5 devices including AM33xx. This + * errata prevents us from using posted mode on these devices, unless the + * timer counter register is never read. For more details please refer to + * the OMAP3/4/5 errata documents. + */ +#define OMAP_TIMER_ERRATA_I103_I767 0x80000000 + +/* posted mode types */ +#define OMAP_TIMER_NONPOSTED 0x00 +#define OMAP_TIMER_POSTED 0x01 + +/* register offsets with the write pending bit encoded */ +#define WPSHIFT 16 + +#define OMAP_TIMER_WAKEUP_EN_REG (_OMAP_TIMER_WAKEUP_EN_OFFSET \ + | (WP_NONE << WPSHIFT)) + +#define OMAP_TIMER_CTRL_REG (_OMAP_TIMER_CTRL_OFFSET \ + | (WP_TCLR << WPSHIFT)) + +#define OMAP_TIMER_COUNTER_REG (_OMAP_TIMER_COUNTER_OFFSET \ + | (WP_TCRR << WPSHIFT)) + +#define OMAP_TIMER_LOAD_REG (_OMAP_TIMER_LOAD_OFFSET \ + | (WP_TLDR << WPSHIFT)) + +#define OMAP_TIMER_TRIGGER_REG (_OMAP_TIMER_TRIGGER_OFFSET \ + | (WP_TTGR << WPSHIFT)) + +#define OMAP_TIMER_WRITE_PEND_REG (_OMAP_TIMER_WRITE_PEND_OFFSET \ + | (WP_NONE << WPSHIFT)) + +#define OMAP_TIMER_MATCH_REG (_OMAP_TIMER_MATCH_OFFSET \ + | (WP_TMAR << WPSHIFT)) + +#define OMAP_TIMER_CAPTURE_REG (_OMAP_TIMER_CAPTURE_OFFSET \ + | (WP_NONE << WPSHIFT)) + +#define OMAP_TIMER_IF_CTRL_REG (_OMAP_TIMER_IF_CTRL_OFFSET \ + | (WP_NONE << WPSHIFT)) + +#define OMAP_TIMER_CAPTURE2_REG (_OMAP_TIMER_CAPTURE2_OFFSET \ + | (WP_NONE << WPSHIFT)) + +#define OMAP_TIMER_TICK_POS_REG (_OMAP_TIMER_TICK_POS_OFFSET \ + | (WP_TPIR << WPSHIFT)) + +#define OMAP_TIMER_TICK_NEG_REG (_OMAP_TIMER_TICK_NEG_OFFSET \ + | (WP_TNIR << WPSHIFT)) + +#define OMAP_TIMER_TICK_COUNT_REG (_OMAP_TIMER_TICK_COUNT_OFFSET \ + | (WP_TCVR << WPSHIFT)) + +#define OMAP_TIMER_TICK_INT_MASK_SET_REG \ + (_OMAP_TIMER_TICK_INT_MASK_SET_OFFSET | (WP_TOCR << WPSHIFT)) + +#define OMAP_TIMER_TICK_INT_MASK_COUNT_REG \ + (_OMAP_TIMER_TICK_INT_MASK_COUNT_OFFSET | (WP_TOWR << WPSHIFT)) + +struct timer_regs { + u32 ocp_cfg; + u32 tidr; + u32 tier; + u32 twer; + u32 tclr; + u32 tcrr; + u32 tldr; + u32 ttrg; + u32 twps; + u32 tmar; + u32 tcar1; + u32 tsicr; + u32 tcar2; + u32 tpir; + u32 tnir; + u32 tcvr; + u32 tocr; + u32 towr; +}; + +struct dmtimer { + struct omap_dm_timer cookie; + int id; + int irq; + struct clk *fclk; + + void __iomem *io_base; + int irq_stat; /* TISR/IRQSTATUS interrupt status */ + int irq_ena; /* irq enable */ + int irq_dis; /* irq disable, only on v2 ip */ + void __iomem *pend; /* write pending */ + void __iomem *func_base; /* function register base */ + + atomic_t enabled; + unsigned long rate; + unsigned reserved:1; + unsigned posted:1; + unsigned omap1:1; + struct timer_regs context; + int revision; + u32 capability; + u32 errata; + struct platform_device *pdev; + struct list_head node; + struct notifier_block nb; +}; + static u32 omap_reserved_systimers; static LIST_HEAD(omap_timer_list); static DEFINE_SPINLOCK(dm_timer_lock); @@ -44,27 +154,56 @@ enum { REQUEST_BY_NODE, }; -static inline u32 __omap_dm_timer_read(struct omap_dm_timer *timer, u32 reg, - int posted) +/** + * dmtimer_read - read timer registers in posted and non-posted mode + * @timer: timer pointer over which read operation to perform + * @reg: lowest byte holds the register offset + * + * The posted mode bit is encoded in reg. Note that in posted mode, write + * pending bit must be checked. Otherwise a read of a non completed write + * will produce an error. + */ +static inline u32 dmtimer_read(struct dmtimer *timer, u32 reg) { - if (posted) - while (readl_relaxed(timer->pend) & (reg >> WPSHIFT)) + u16 wp, offset; + + wp = reg >> WPSHIFT; + offset = reg & 0xff; + + /* Wait for a possible write pending bit in posted mode */ + if (wp && timer->posted) + while (readl_relaxed(timer->pend) & wp) cpu_relax(); - return readl_relaxed(timer->func_base + (reg & 0xff)); + return readl_relaxed(timer->func_base + offset); } -static inline void __omap_dm_timer_write(struct omap_dm_timer *timer, - u32 reg, u32 val, int posted) +/** + * dmtimer_write - write timer registers in posted and non-posted mode + * @timer: timer pointer over which write operation is to perform + * @reg: lowest byte holds the register offset + * @value: data to write into the register + * + * The posted mode bit is encoded in reg. Note that in posted mode, the write + * pending bit must be checked. Otherwise a write on a register which has a + * pending write will be lost. + */ +static inline void dmtimer_write(struct dmtimer *timer, u32 reg, u32 val) { - if (posted) - while (readl_relaxed(timer->pend) & (reg >> WPSHIFT)) + u16 wp, offset; + + wp = reg >> WPSHIFT; + offset = reg & 0xff; + + /* Wait for a possible write pending bit in posted mode */ + if (wp && timer->posted) + while (readl_relaxed(timer->pend) & wp) cpu_relax(); - writel_relaxed(val, timer->func_base + (reg & 0xff)); + writel_relaxed(val, timer->func_base + offset); } -static inline void __omap_dm_timer_init_regs(struct omap_dm_timer *timer) +static inline void __omap_dm_timer_init_regs(struct dmtimer *timer) { u32 tidr; @@ -72,16 +211,16 @@ static inline void __omap_dm_timer_init_regs(struct omap_dm_timer *timer) tidr = readl_relaxed(timer->io_base); if (!(tidr >> 16)) { timer->revision = 1; - timer->irq_stat = timer->io_base + OMAP_TIMER_V1_STAT_OFFSET; - timer->irq_ena = timer->io_base + OMAP_TIMER_V1_INT_EN_OFFSET; - timer->irq_dis = timer->io_base + OMAP_TIMER_V1_INT_EN_OFFSET; + timer->irq_stat = OMAP_TIMER_V1_STAT_OFFSET; + timer->irq_ena = OMAP_TIMER_V1_INT_EN_OFFSET; + timer->irq_dis = OMAP_TIMER_V1_INT_EN_OFFSET; timer->pend = timer->io_base + _OMAP_TIMER_WRITE_PEND_OFFSET; timer->func_base = timer->io_base; } else { timer->revision = 2; - timer->irq_stat = timer->io_base + OMAP_TIMER_V2_IRQSTATUS; - timer->irq_ena = timer->io_base + OMAP_TIMER_V2_IRQENABLE_SET; - timer->irq_dis = timer->io_base + OMAP_TIMER_V2_IRQENABLE_CLR; + timer->irq_stat = OMAP_TIMER_V2_IRQSTATUS - OMAP_TIMER_V2_FUNC_OFFSET; + timer->irq_ena = OMAP_TIMER_V2_IRQENABLE_SET - OMAP_TIMER_V2_FUNC_OFFSET; + timer->irq_dis = OMAP_TIMER_V2_IRQENABLE_CLR - OMAP_TIMER_V2_FUNC_OFFSET; timer->pend = timer->io_base + _OMAP_TIMER_WRITE_PEND_OFFSET + OMAP_TIMER_V2_FUNC_OFFSET; @@ -99,35 +238,34 @@ static inline void __omap_dm_timer_init_regs(struct omap_dm_timer *timer) * complete. Enabling this feature can improve performance for writing to the * timer registers. */ -static inline void __omap_dm_timer_enable_posted(struct omap_dm_timer *timer) +static inline void __omap_dm_timer_enable_posted(struct dmtimer *timer) { if (timer->posted) return; if (timer->errata & OMAP_TIMER_ERRATA_I103_I767) { timer->posted = OMAP_TIMER_NONPOSTED; - __omap_dm_timer_write(timer, OMAP_TIMER_IF_CTRL_REG, 0, 0); + dmtimer_write(timer, OMAP_TIMER_IF_CTRL_REG, 0); return; } - __omap_dm_timer_write(timer, OMAP_TIMER_IF_CTRL_REG, - OMAP_TIMER_CTRL_POSTED, 0); + dmtimer_write(timer, OMAP_TIMER_IF_CTRL_REG, OMAP_TIMER_CTRL_POSTED); timer->context.tsicr = OMAP_TIMER_CTRL_POSTED; timer->posted = OMAP_TIMER_POSTED; } -static inline void __omap_dm_timer_stop(struct omap_dm_timer *timer, - int posted, unsigned long rate) +static inline void __omap_dm_timer_stop(struct dmtimer *timer, + unsigned long rate) { u32 l; - l = __omap_dm_timer_read(timer, OMAP_TIMER_CTRL_REG, posted); + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); if (l & OMAP_TIMER_CTRL_ST) { l &= ~0x1; - __omap_dm_timer_write(timer, OMAP_TIMER_CTRL_REG, l, posted); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, l); #ifdef CONFIG_ARCH_OMAP2PLUS /* Readback to make sure write has completed */ - __omap_dm_timer_read(timer, OMAP_TIMER_CTRL_REG, posted); + dmtimer_read(timer, OMAP_TIMER_CTRL_REG); /* * Wait for functional clock period x 3.5 to make sure that * timer is stopped @@ -137,104 +275,59 @@ static inline void __omap_dm_timer_stop(struct omap_dm_timer *timer, } /* Ack possibly pending interrupt */ - writel_relaxed(OMAP_TIMER_INT_OVERFLOW, timer->irq_stat); + dmtimer_write(timer, timer->irq_stat, OMAP_TIMER_INT_OVERFLOW); } -static inline void __omap_dm_timer_int_enable(struct omap_dm_timer *timer, - unsigned int value) +static inline void __omap_dm_timer_int_enable(struct dmtimer *timer, + unsigned int value) { - writel_relaxed(value, timer->irq_ena); - __omap_dm_timer_write(timer, OMAP_TIMER_WAKEUP_EN_REG, value, 0); + dmtimer_write(timer, timer->irq_ena, value); + dmtimer_write(timer, OMAP_TIMER_WAKEUP_EN_REG, value); } static inline unsigned int -__omap_dm_timer_read_counter(struct omap_dm_timer *timer, int posted) +__omap_dm_timer_read_counter(struct dmtimer *timer) { - return __omap_dm_timer_read(timer, OMAP_TIMER_COUNTER_REG, posted); + return dmtimer_read(timer, OMAP_TIMER_COUNTER_REG); } -static inline void __omap_dm_timer_write_status(struct omap_dm_timer *timer, +static inline void __omap_dm_timer_write_status(struct dmtimer *timer, unsigned int value) { - writel_relaxed(value, timer->irq_stat); + dmtimer_write(timer, timer->irq_stat, value); } -/** - * omap_dm_timer_read_reg - read timer registers in posted and non-posted mode - * @timer: timer pointer over which read operation to perform - * @reg: lowest byte holds the register offset - * - * The posted mode bit is encoded in reg. Note that in posted mode write - * pending bit must be checked. Otherwise a read of a non completed write - * will produce an error. - */ -static inline u32 omap_dm_timer_read_reg(struct omap_dm_timer *timer, u32 reg) +static void omap_timer_restore_context(struct dmtimer *timer) { - WARN_ON((reg & 0xff) < _OMAP_TIMER_WAKEUP_EN_OFFSET); - return __omap_dm_timer_read(timer, reg, timer->posted); + dmtimer_write(timer, OMAP_TIMER_OCP_CFG_OFFSET, timer->context.ocp_cfg); + + dmtimer_write(timer, OMAP_TIMER_WAKEUP_EN_REG, timer->context.twer); + dmtimer_write(timer, OMAP_TIMER_COUNTER_REG, timer->context.tcrr); + dmtimer_write(timer, OMAP_TIMER_LOAD_REG, timer->context.tldr); + dmtimer_write(timer, OMAP_TIMER_MATCH_REG, timer->context.tmar); + dmtimer_write(timer, OMAP_TIMER_IF_CTRL_REG, timer->context.tsicr); + dmtimer_write(timer, timer->irq_ena, timer->context.tier); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, timer->context.tclr); } -/** - * omap_dm_timer_write_reg - write timer registers in posted and non-posted mode - * @timer: timer pointer over which write operation is to perform - * @reg: lowest byte holds the register offset - * @value: data to write into the register - * - * The posted mode bit is encoded in reg. Note that in posted mode the write - * pending bit must be checked. Otherwise a write on a register which has a - * pending write will be lost. - */ -static void omap_dm_timer_write_reg(struct omap_dm_timer *timer, u32 reg, - u32 value) +static void omap_timer_save_context(struct dmtimer *timer) { - WARN_ON((reg & 0xff) < _OMAP_TIMER_WAKEUP_EN_OFFSET); - __omap_dm_timer_write(timer, reg, value, timer->posted); -} - -static void omap_timer_restore_context(struct omap_dm_timer *timer) -{ - __omap_dm_timer_write(timer, OMAP_TIMER_OCP_CFG_OFFSET, - timer->context.ocp_cfg, 0); - - omap_dm_timer_write_reg(timer, OMAP_TIMER_WAKEUP_EN_REG, - timer->context.twer); - omap_dm_timer_write_reg(timer, OMAP_TIMER_COUNTER_REG, - timer->context.tcrr); - omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, - timer->context.tldr); - omap_dm_timer_write_reg(timer, OMAP_TIMER_MATCH_REG, - timer->context.tmar); - omap_dm_timer_write_reg(timer, OMAP_TIMER_IF_CTRL_REG, - timer->context.tsicr); - writel_relaxed(timer->context.tier, timer->irq_ena); - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, - timer->context.tclr); -} - -static void omap_timer_save_context(struct omap_dm_timer *timer) -{ - timer->context.ocp_cfg = - __omap_dm_timer_read(timer, OMAP_TIMER_OCP_CFG_OFFSET, 0); - - timer->context.tclr = - omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); - timer->context.twer = - omap_dm_timer_read_reg(timer, OMAP_TIMER_WAKEUP_EN_REG); - timer->context.tldr = - omap_dm_timer_read_reg(timer, OMAP_TIMER_LOAD_REG); - timer->context.tmar = - omap_dm_timer_read_reg(timer, OMAP_TIMER_MATCH_REG); - timer->context.tier = readl_relaxed(timer->irq_ena); - timer->context.tsicr = - omap_dm_timer_read_reg(timer, OMAP_TIMER_IF_CTRL_REG); + timer->context.ocp_cfg = dmtimer_read(timer, OMAP_TIMER_OCP_CFG_OFFSET); + + timer->context.tclr = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); + timer->context.twer = dmtimer_read(timer, OMAP_TIMER_WAKEUP_EN_REG); + timer->context.tldr = dmtimer_read(timer, OMAP_TIMER_LOAD_REG); + timer->context.tmar = dmtimer_read(timer, OMAP_TIMER_MATCH_REG); + timer->context.tier = dmtimer_read(timer, timer->irq_ena); + timer->context.tsicr = dmtimer_read(timer, OMAP_TIMER_IF_CTRL_REG); } static int omap_timer_context_notifier(struct notifier_block *nb, unsigned long cmd, void *v) { - struct omap_dm_timer *timer; + struct dmtimer *timer; - timer = container_of(nb, struct omap_dm_timer, nb); + timer = container_of(nb, struct dmtimer, nb); switch (cmd) { case CPU_CLUSTER_PM_ENTER: @@ -256,18 +349,17 @@ static int omap_timer_context_notifier(struct notifier_block *nb, return NOTIFY_OK; } -static int omap_dm_timer_reset(struct omap_dm_timer *timer) +static int omap_dm_timer_reset(struct dmtimer *timer) { u32 l, timeout = 100000; if (timer->revision != 1) return -EINVAL; - omap_dm_timer_write_reg(timer, OMAP_TIMER_IF_CTRL_REG, 0x06); + dmtimer_write(timer, OMAP_TIMER_IF_CTRL_REG, 0x06); do { - l = __omap_dm_timer_read(timer, - OMAP_TIMER_V1_SYS_STAT_OFFSET, 0); + l = dmtimer_read(timer, OMAP_TIMER_V1_SYS_STAT_OFFSET); } while (!l && timeout--); if (!timeout) { @@ -276,22 +368,38 @@ static int omap_dm_timer_reset(struct omap_dm_timer *timer) } /* Configure timer for smart-idle mode */ - l = __omap_dm_timer_read(timer, OMAP_TIMER_OCP_CFG_OFFSET, 0); + l = dmtimer_read(timer, OMAP_TIMER_OCP_CFG_OFFSET); l |= 0x2 << 0x3; - __omap_dm_timer_write(timer, OMAP_TIMER_OCP_CFG_OFFSET, l, 0); + dmtimer_write(timer, OMAP_TIMER_OCP_CFG_OFFSET, l); timer->posted = 0; return 0; } -static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source) +/* + * Functions exposed to PWM and remoteproc drivers via platform_data. + * Do not use these in the driver, these will get deprecated and will + * will be replaced by Linux generic framework functions such as + * chained interrupts and clock framework. + */ +static struct dmtimer *to_dmtimer(struct omap_dm_timer *cookie) +{ + if (!cookie) + return NULL; + + return container_of(cookie, struct dmtimer, cookie); +} + +static int omap_dm_timer_set_source(struct omap_dm_timer *cookie, int source) { int ret; const char *parent_name; struct clk *parent; struct dmtimer_platform_data *pdata; + struct dmtimer *timer; + timer = to_dmtimer(cookie); if (unlikely(!timer) || IS_ERR(timer->fclk)) return -EINVAL; @@ -316,7 +424,7 @@ static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source) * use the clock framework to set the parent clock. To be removed * once OMAP1 migrated to using clock framework for dmtimers */ - if (pdata && pdata->set_timer_src) + if (timer->omap1 && pdata && pdata->set_timer_src) return pdata->set_timer_src(timer->pdev, source); #if defined(CONFIG_COMMON_CLK) @@ -341,44 +449,44 @@ static int omap_dm_timer_set_source(struct omap_dm_timer *timer, int source) return ret; } -static void omap_dm_timer_enable(struct omap_dm_timer *timer) +static void omap_dm_timer_enable(struct omap_dm_timer *cookie) { - pm_runtime_get_sync(&timer->pdev->dev); + struct dmtimer *timer = to_dmtimer(cookie); + struct device *dev = &timer->pdev->dev; + int rc; + + rc = pm_runtime_resume_and_get(dev); + if (rc) + dev_err(dev, "could not enable timer\n"); } -static void omap_dm_timer_disable(struct omap_dm_timer *timer) +static void omap_dm_timer_disable(struct omap_dm_timer *cookie) { - pm_runtime_put_sync(&timer->pdev->dev); + struct dmtimer *timer = to_dmtimer(cookie); + struct device *dev = &timer->pdev->dev; + + pm_runtime_put_sync(dev); } -static int omap_dm_timer_prepare(struct omap_dm_timer *timer) +static int omap_dm_timer_prepare(struct dmtimer *timer) { + struct device *dev = &timer->pdev->dev; int rc; - /* - * FIXME: OMAP1 devices do not use the clock framework for dmtimers so - * do not call clk_get() for these devices. - */ - if (!(timer->capability & OMAP_TIMER_NEEDS_RESET)) { - timer->fclk = clk_get(&timer->pdev->dev, "fck"); - if (WARN_ON_ONCE(IS_ERR(timer->fclk))) { - dev_err(&timer->pdev->dev, ": No fclk handle.\n"); - return -EINVAL; - } - } - - omap_dm_timer_enable(timer); + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; if (timer->capability & OMAP_TIMER_NEEDS_RESET) { rc = omap_dm_timer_reset(timer); if (rc) { - omap_dm_timer_disable(timer); + pm_runtime_put_sync(dev); return rc; } } __omap_dm_timer_enable_posted(timer); - omap_dm_timer_disable(timer); + pm_runtime_put_sync(dev); return 0; } @@ -388,19 +496,9 @@ static inline u32 omap_dm_timer_reserved_systimer(int id) return (omap_reserved_systimers & (1 << (id - 1))) ? 1 : 0; } -int omap_dm_timer_reserve_systimer(int id) +static struct dmtimer *_omap_dm_timer_request(int req_type, void *data) { - if (omap_dm_timer_reserved_systimer(id)) - return -ENODEV; - - omap_reserved_systimers |= (1 << (id - 1)); - - return 0; -} - -static struct omap_dm_timer *_omap_dm_timer_request(int req_type, void *data) -{ - struct omap_dm_timer *timer = NULL, *t; + struct dmtimer *timer = NULL, *t; struct device_node *np = NULL; unsigned long flags; u32 cap = 0; @@ -484,11 +582,19 @@ found: static struct omap_dm_timer *omap_dm_timer_request(void) { - return _omap_dm_timer_request(REQUEST_ANY, NULL); + struct dmtimer *timer; + + timer = _omap_dm_timer_request(REQUEST_ANY, NULL); + if (!timer) + return NULL; + + return &timer->cookie; } static struct omap_dm_timer *omap_dm_timer_request_specific(int id) { + struct dmtimer *timer; + /* Requesting timer by ID is not supported when device tree is used */ if (of_have_populated_dt()) { pr_warn("%s: Please use omap_dm_timer_request_by_node()\n", @@ -496,21 +602,11 @@ static struct omap_dm_timer *omap_dm_timer_request_specific(int id) return NULL; } - return _omap_dm_timer_request(REQUEST_BY_ID, &id); -} + timer = _omap_dm_timer_request(REQUEST_BY_ID, &id); + if (!timer) + return NULL; -/** - * omap_dm_timer_request_by_cap - Request a timer by capability - * @cap: Bit mask of capabilities to match - * - * Find a timer based upon capabilities bit mask. Callers of this function - * should use the definitions found in the plat/dmtimer.h file under the - * comment "timer capabilities used in hwmod database". Returns pointer to - * timer handle on success and a NULL pointer on failure. - */ -struct omap_dm_timer *omap_dm_timer_request_by_cap(u32 cap) -{ - return _omap_dm_timer_request(REQUEST_BY_CAP, &cap); + return &timer->cookie; } /** @@ -522,26 +618,34 @@ struct omap_dm_timer *omap_dm_timer_request_by_cap(u32 cap) */ static struct omap_dm_timer *omap_dm_timer_request_by_node(struct device_node *np) { + struct dmtimer *timer; + if (!np) return NULL; - return _omap_dm_timer_request(REQUEST_BY_NODE, np); + timer = _omap_dm_timer_request(REQUEST_BY_NODE, np); + if (!timer) + return NULL; + + return &timer->cookie; } -static int omap_dm_timer_free(struct omap_dm_timer *timer) +static int omap_dm_timer_free(struct omap_dm_timer *cookie) { + struct dmtimer *timer; + + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - clk_put(timer->fclk); - WARN_ON(!timer->reserved); timer->reserved = 0; return 0; } -int omap_dm_timer_get_irq(struct omap_dm_timer *timer) +int omap_dm_timer_get_irq(struct omap_dm_timer *cookie) { + struct dmtimer *timer = to_dmtimer(cookie); if (timer) return timer->irq; return -EINVAL; @@ -550,7 +654,7 @@ int omap_dm_timer_get_irq(struct omap_dm_timer *timer) #if defined(CONFIG_ARCH_OMAP1) #include -static struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *timer) +static struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *cookie) { return NULL; } @@ -562,7 +666,7 @@ static struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *timer) __u32 omap_dm_timer_modify_idlect_mask(__u32 inputmask) { int i = 0; - struct omap_dm_timer *timer = NULL; + struct dmtimer *timer = NULL; unsigned long flags; /* If ARMXOR cannot be idled this function call is unnecessary */ @@ -574,7 +678,7 @@ __u32 omap_dm_timer_modify_idlect_mask(__u32 inputmask) list_for_each_entry(timer, &omap_timer_list, node) { u32 l; - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); if (l & OMAP_TIMER_CTRL_ST) { if (((omap_readl(MOD_CONF_CTRL_1) >> (i * 2)) & 0x03) == 0) inputmask &= ~(1 << 1); @@ -590,8 +694,10 @@ __u32 omap_dm_timer_modify_idlect_mask(__u32 inputmask) #else -static struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *timer) +static struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *cookie) { + struct dmtimer *timer = to_dmtimer(cookie); + if (timer && !IS_ERR(timer->fclk)) return timer->fclk; return NULL; @@ -606,95 +712,125 @@ __u32 omap_dm_timer_modify_idlect_mask(__u32 inputmask) #endif -int omap_dm_timer_trigger(struct omap_dm_timer *timer) -{ - if (unlikely(!timer || !atomic_read(&timer->enabled))) { - pr_err("%s: timer not available or enabled.\n", __func__); - return -EINVAL; - } - - omap_dm_timer_write_reg(timer, OMAP_TIMER_TRIGGER_REG, 0); - return 0; -} - -static int omap_dm_timer_start(struct omap_dm_timer *timer) +static int omap_dm_timer_start(struct omap_dm_timer *cookie) { + struct dmtimer *timer; + struct device *dev; + int rc; u32 l; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); + dev = &timer->pdev->dev; + + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); if (!(l & OMAP_TIMER_CTRL_ST)) { l |= OMAP_TIMER_CTRL_ST; - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, l); } return 0; } -static int omap_dm_timer_stop(struct omap_dm_timer *timer) +static int omap_dm_timer_stop(struct omap_dm_timer *cookie) { + struct dmtimer *timer; + struct device *dev; unsigned long rate = 0; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - if (!(timer->capability & OMAP_TIMER_NEEDS_RESET)) + dev = &timer->pdev->dev; + + if (!timer->omap1) rate = clk_get_rate(timer->fclk); - __omap_dm_timer_stop(timer, timer->posted, rate); + __omap_dm_timer_stop(timer, rate); + + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_set_load(struct omap_dm_timer *timer, +static int omap_dm_timer_set_load(struct omap_dm_timer *cookie, unsigned int load) { + struct dmtimer *timer; + struct device *dev; + int rc; + + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); - omap_dm_timer_write_reg(timer, OMAP_TIMER_LOAD_REG, load); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + + dmtimer_write(timer, OMAP_TIMER_LOAD_REG, load); + + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_set_match(struct omap_dm_timer *timer, int enable, +static int omap_dm_timer_set_match(struct omap_dm_timer *cookie, int enable, unsigned int match) { + struct dmtimer *timer; + struct device *dev; + int rc; u32 l; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); if (enable) l |= OMAP_TIMER_CTRL_CE; else l &= ~OMAP_TIMER_CTRL_CE; - omap_dm_timer_write_reg(timer, OMAP_TIMER_MATCH_REG, match); - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); + dmtimer_write(timer, OMAP_TIMER_MATCH_REG, match); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, l); + + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_set_pwm(struct omap_dm_timer *timer, int def_on, +static int omap_dm_timer_set_pwm(struct omap_dm_timer *cookie, int def_on, int toggle, int trigger, int autoreload) { + struct dmtimer *timer; + struct device *dev; + int rc; u32 l; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); l &= ~(OMAP_TIMER_CTRL_GPOCFG | OMAP_TIMER_CTRL_SCPWM | OMAP_TIMER_CTRL_PT | (0x03 << 10) | OMAP_TIMER_CTRL_AR); if (def_on) @@ -704,57 +840,86 @@ static int omap_dm_timer_set_pwm(struct omap_dm_timer *timer, int def_on, l |= trigger << 10; if (autoreload) l |= OMAP_TIMER_CTRL_AR; - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, l); + + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_get_pwm_status(struct omap_dm_timer *timer) +static int omap_dm_timer_get_pwm_status(struct omap_dm_timer *cookie) { + struct dmtimer *timer; + struct device *dev; + int rc; u32 l; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); - omap_dm_timer_disable(timer); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); + + pm_runtime_put_sync(dev); return l; } -static int omap_dm_timer_set_prescaler(struct omap_dm_timer *timer, - int prescaler) +static int omap_dm_timer_set_prescaler(struct omap_dm_timer *cookie, + int prescaler) { + struct dmtimer *timer; + struct device *dev; + int rc; u32 l; + timer = to_dmtimer(cookie); if (unlikely(!timer) || prescaler < -1 || prescaler > 7) return -EINVAL; - omap_dm_timer_enable(timer); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + + l = dmtimer_read(timer, OMAP_TIMER_CTRL_REG); l &= ~(OMAP_TIMER_CTRL_PRE | (0x07 << 2)); if (prescaler >= 0) { l |= OMAP_TIMER_CTRL_PRE; l |= prescaler << 2; } - omap_dm_timer_write_reg(timer, OMAP_TIMER_CTRL_REG, l); + dmtimer_write(timer, OMAP_TIMER_CTRL_REG, l); + + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static int omap_dm_timer_set_int_enable(struct omap_dm_timer *timer, +static int omap_dm_timer_set_int_enable(struct omap_dm_timer *cookie, unsigned int value) { + struct dmtimer *timer; + struct device *dev; + int rc; + + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; + __omap_dm_timer_int_enable(timer, value); - omap_dm_timer_disable(timer); + pm_runtime_put_sync(dev); + return 0; } @@ -765,42 +930,55 @@ static int omap_dm_timer_set_int_enable(struct omap_dm_timer *timer, * * Disables the specified timer interrupts for a timer. */ -static int omap_dm_timer_set_int_disable(struct omap_dm_timer *timer, u32 mask) +static int omap_dm_timer_set_int_disable(struct omap_dm_timer *cookie, u32 mask) { + struct dmtimer *timer; + struct device *dev; u32 l = mask; + int rc; + timer = to_dmtimer(cookie); if (unlikely(!timer)) return -EINVAL; - omap_dm_timer_enable(timer); + dev = &timer->pdev->dev; + rc = pm_runtime_resume_and_get(dev); + if (rc) + return rc; if (timer->revision == 1) - l = readl_relaxed(timer->irq_ena) & ~mask; + l = dmtimer_read(timer, timer->irq_ena) & ~mask; + + dmtimer_write(timer, timer->irq_dis, l); + l = dmtimer_read(timer, OMAP_TIMER_WAKEUP_EN_REG) & ~mask; + dmtimer_write(timer, OMAP_TIMER_WAKEUP_EN_REG, l); - writel_relaxed(l, timer->irq_dis); - l = omap_dm_timer_read_reg(timer, OMAP_TIMER_WAKEUP_EN_REG) & ~mask; - omap_dm_timer_write_reg(timer, OMAP_TIMER_WAKEUP_EN_REG, l); + pm_runtime_put_sync(dev); - omap_dm_timer_disable(timer); return 0; } -static unsigned int omap_dm_timer_read_status(struct omap_dm_timer *timer) +static unsigned int omap_dm_timer_read_status(struct omap_dm_timer *cookie) { + struct dmtimer *timer; unsigned int l; + timer = to_dmtimer(cookie); if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not available or enabled.\n", __func__); return 0; } - l = readl_relaxed(timer->irq_stat); + l = dmtimer_read(timer, timer->irq_stat); return l; } -static int omap_dm_timer_write_status(struct omap_dm_timer *timer, unsigned int value) +static int omap_dm_timer_write_status(struct omap_dm_timer *cookie, unsigned int value) { + struct dmtimer *timer; + + timer = to_dmtimer(cookie); if (unlikely(!timer || !atomic_read(&timer->enabled))) return -EINVAL; @@ -809,49 +987,39 @@ static int omap_dm_timer_write_status(struct omap_dm_timer *timer, unsigned int return 0; } -static unsigned int omap_dm_timer_read_counter(struct omap_dm_timer *timer) +static unsigned int omap_dm_timer_read_counter(struct omap_dm_timer *cookie) { + struct dmtimer *timer; + + timer = to_dmtimer(cookie); if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not iavailable or enabled.\n", __func__); return 0; } - return __omap_dm_timer_read_counter(timer, timer->posted); + return __omap_dm_timer_read_counter(timer); } -static int omap_dm_timer_write_counter(struct omap_dm_timer *timer, unsigned int value) +static int omap_dm_timer_write_counter(struct omap_dm_timer *cookie, unsigned int value) { + struct dmtimer *timer; + + timer = to_dmtimer(cookie); if (unlikely(!timer || !atomic_read(&timer->enabled))) { pr_err("%s: timer not available or enabled.\n", __func__); return -EINVAL; } - omap_dm_timer_write_reg(timer, OMAP_TIMER_COUNTER_REG, value); + dmtimer_write(timer, OMAP_TIMER_COUNTER_REG, value); /* Save the context */ timer->context.tcrr = value; return 0; } -int omap_dm_timers_active(void) -{ - struct omap_dm_timer *timer; - - list_for_each_entry(timer, &omap_timer_list, node) { - if (!timer->reserved) - continue; - - if (omap_dm_timer_read_reg(timer, OMAP_TIMER_CTRL_REG) & - OMAP_TIMER_CTRL_ST) { - return 1; - } - } - return 0; -} - static int __maybe_unused omap_dm_timer_runtime_suspend(struct device *dev) { - struct omap_dm_timer *timer = dev_get_drvdata(dev); + struct dmtimer *timer = dev_get_drvdata(dev); atomic_set(&timer->enabled, 0); @@ -865,7 +1033,7 @@ static int __maybe_unused omap_dm_timer_runtime_suspend(struct device *dev) static int __maybe_unused omap_dm_timer_runtime_resume(struct device *dev) { - struct omap_dm_timer *timer = dev_get_drvdata(dev); + struct dmtimer *timer = dev_get_drvdata(dev); if (!(timer->capability & OMAP_TIMER_ALWON) && timer->func_base) omap_timer_restore_context(timer); @@ -892,7 +1060,7 @@ static const struct of_device_id omap_timer_match[]; static int omap_dm_timer_probe(struct platform_device *pdev) { unsigned long flags; - struct omap_dm_timer *timer; + struct dmtimer *timer; struct device *dev = &pdev->dev; const struct dmtimer_platform_data *pdata; int ret; @@ -916,7 +1084,6 @@ static int omap_dm_timer_probe(struct platform_device *pdev) if (timer->irq < 0) return timer->irq; - timer->fclk = ERR_PTR(-ENODEV); timer->io_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(timer->io_base)) return PTR_ERR(timer->io_base); @@ -938,6 +1105,17 @@ static int omap_dm_timer_probe(struct platform_device *pdev) timer->reserved = omap_dm_timer_reserved_systimer(timer->id); } + timer->omap1 = timer->capability & OMAP_TIMER_NEEDS_RESET; + + /* OMAP1 devices do not yet use the clock framework for dmtimers */ + if (!timer->omap1) { + timer->fclk = devm_clk_get(dev, "fck"); + if (IS_ERR(timer->fclk)) + return PTR_ERR(timer->fclk); + } else { + timer->fclk = ERR_PTR(-ENODEV); + } + if (!(timer->capability & OMAP_TIMER_ALWON)) { timer->nb.notifier_call = omap_timer_context_notifier; cpu_pm_register_notifier(&timer->nb); @@ -950,11 +1128,11 @@ static int omap_dm_timer_probe(struct platform_device *pdev) pm_runtime_enable(dev); if (!timer->reserved) { - ret = pm_runtime_get_sync(dev); - if (ret < 0) { + ret = pm_runtime_resume_and_get(dev); + if (ret) { dev_err(dev, "%s: pm_runtime_get_sync failed!\n", __func__); - goto err_get_sync; + goto err_disable; } __omap_dm_timer_init_regs(timer); pm_runtime_put(dev); @@ -969,8 +1147,7 @@ static int omap_dm_timer_probe(struct platform_device *pdev) return 0; -err_get_sync: - pm_runtime_put_noidle(dev); +err_disable: pm_runtime_disable(dev); return ret; } @@ -985,7 +1162,7 @@ err_get_sync: */ static int omap_dm_timer_remove(struct platform_device *pdev) { - struct omap_dm_timer *timer; + struct dmtimer *timer; unsigned long flags; int ret = -EINVAL; diff --git a/drivers/comedi/comedi_fops.c b/drivers/comedi/comedi_fops.c index 55a0cae04b8d163c01bb19356c7bcf900656aefd..e2114bcf815ad8d9dd6c9f48d4f0780a0a996738 100644 --- a/drivers/comedi/comedi_fops.c +++ b/drivers/comedi/comedi_fops.c @@ -396,7 +396,7 @@ static ssize_t max_read_buffer_kb_show(struct device *csdev, mutex_unlock(&dev->mutex); comedi_dev_put(dev); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t max_read_buffer_kb_store(struct device *csdev, @@ -452,7 +452,7 @@ static ssize_t read_buffer_kb_show(struct device *csdev, mutex_unlock(&dev->mutex); comedi_dev_put(dev); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t read_buffer_kb_store(struct device *csdev, @@ -509,7 +509,7 @@ static ssize_t max_write_buffer_kb_show(struct device *csdev, mutex_unlock(&dev->mutex); comedi_dev_put(dev); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t max_write_buffer_kb_store(struct device *csdev, @@ -565,7 +565,7 @@ static ssize_t write_buffer_kb_show(struct device *csdev, mutex_unlock(&dev->mutex); comedi_dev_put(dev); - return snprintf(buf, PAGE_SIZE, "%u\n", size); + return sysfs_emit(buf, "%u\n", size); } static ssize_t write_buffer_kb_store(struct device *csdev, diff --git a/drivers/counter/104-quad-8.c b/drivers/counter/104-quad-8.c index 62c2b7ac4339fb4ff36d633cd9df11ea84106651..77a863b7eefeac1a3fbd1d005e57eb508aadb604 100644 --- a/drivers/counter/104-quad-8.c +++ b/drivers/counter/104-quad-8.c @@ -28,7 +28,8 @@ module_param_hw_array(base, uint, ioport, &num_quad8, 0); MODULE_PARM_DESC(base, "ACCES 104-QUAD-8 base addresses"); static unsigned int irq[max_num_isa_dev(QUAD8_EXTENT)]; -module_param_hw_array(irq, uint, irq, NULL, 0); +static unsigned int num_irq; +module_param_hw_array(irq, uint, irq, &num_irq, 0); MODULE_PARM_DESC(irq, "ACCES 104-QUAD-8 interrupt line numbers"); #define QUAD8_NUM_COUNTERS 8 @@ -449,6 +450,9 @@ static int quad8_events_configure(struct counter_device *counter) return -EINVAL; } + /* Enable IRQ line */ + irq_enabled |= BIT(event_node->channel); + /* Skip configuration if it is the same as previously set */ if (priv->irq_trigger[event_node->channel] == next_irq_trigger) continue; @@ -462,9 +466,6 @@ static int quad8_events_configure(struct counter_device *counter) priv->irq_trigger[event_node->channel] << 3; iowrite8(QUAD8_CTR_IOR | ior_cfg, &priv->reg->channel[event_node->channel].control); - - /* Enable IRQ line */ - irq_enabled |= BIT(event_node->channel); } iowrite8(irq_enabled, &priv->reg->index_interrupt); @@ -549,6 +550,32 @@ static int quad8_index_polarity_set(struct counter_device *counter, return 0; } +static int quad8_polarity_read(struct counter_device *counter, + struct counter_signal *signal, + enum counter_signal_polarity *polarity) +{ + int err; + u32 index_polarity; + + err = quad8_index_polarity_get(counter, signal, &index_polarity); + if (err) + return err; + + *polarity = (index_polarity) ? COUNTER_SIGNAL_POLARITY_POSITIVE : + COUNTER_SIGNAL_POLARITY_NEGATIVE; + + return 0; +} + +static int quad8_polarity_write(struct counter_device *counter, + struct counter_signal *signal, + enum counter_signal_polarity polarity) +{ + const u32 pol = (polarity == COUNTER_SIGNAL_POLARITY_POSITIVE) ? 1 : 0; + + return quad8_index_polarity_set(counter, signal, pol); +} + static const char *const quad8_synchronous_modes[] = { "non-synchronous", "synchronous" @@ -977,6 +1004,13 @@ static struct counter_comp quad8_signal_ext[] = { quad8_signal_fck_prescaler_write) }; +static const enum counter_signal_polarity quad8_polarities[] = { + COUNTER_SIGNAL_POLARITY_POSITIVE, + COUNTER_SIGNAL_POLARITY_NEGATIVE, +}; + +static DEFINE_COUNTER_AVAILABLE(quad8_polarity_available, quad8_polarities); + static DEFINE_COUNTER_ENUM(quad8_index_pol_enum, quad8_index_polarity_modes); static DEFINE_COUNTER_ENUM(quad8_synch_mode_enum, quad8_synchronous_modes); @@ -984,6 +1018,8 @@ static struct counter_comp quad8_index_ext[] = { COUNTER_COMP_SIGNAL_ENUM("index_polarity", quad8_index_polarity_get, quad8_index_polarity_set, quad8_index_pol_enum), + COUNTER_COMP_POLARITY(quad8_polarity_read, quad8_polarity_write, + quad8_polarity_available), COUNTER_COMP_SIGNAL_ENUM("synchronous_mode", quad8_synchronous_mode_get, quad8_synchronous_mode_set, quad8_synch_mode_enum), @@ -1236,8 +1272,9 @@ static struct isa_driver quad8_driver = { } }; -module_isa_driver(quad8_driver, num_quad8); +module_isa_driver_with_irq(quad8_driver, num_quad8, num_irq); MODULE_AUTHOR("William Breathitt Gray "); MODULE_DESCRIPTION("ACCES 104-QUAD-8 driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/Kconfig b/drivers/counter/Kconfig index 5edd155f19114a28da828e64953b38dbc53be297..d388bf26f4dc3dcf64e1a635a108c2915ec38164 100644 --- a/drivers/counter/Kconfig +++ b/drivers/counter/Kconfig @@ -101,4 +101,19 @@ config INTEL_QEP To compile this driver as a module, choose M here: the module will be called intel-qep. +config TI_ECAP_CAPTURE + tristate "TI eCAP capture driver" + depends on ARCH_OMAP2PLUS || ARCH_DAVINCI_DA8XX || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST + depends on HAS_IOMEM + select REGMAP_MMIO + help + Select this option to enable the Texas Instruments Enhanced Capture + (eCAP) driver in input mode. + + It can be used to timestamp events (falling/rising edges) detected + on ECAP input signal. + + To compile this driver as a module, choose M here: the module + will be called ti-ecap-capture. + endif # COUNTER diff --git a/drivers/counter/Makefile b/drivers/counter/Makefile index 8fde6c100ebc43782ff2192c8bc47c89f5c02e1b..b9a369e0d4fc7a884eac54c2e4a36bb2dfdf2ba7 100644 --- a/drivers/counter/Makefile +++ b/drivers/counter/Makefile @@ -14,3 +14,4 @@ obj-$(CONFIG_TI_EQEP) += ti-eqep.o obj-$(CONFIG_FTM_QUADDEC) += ftm-quaddec.o obj-$(CONFIG_MICROCHIP_TCB_CAPTURE) += microchip-tcb-capture.o obj-$(CONFIG_INTEL_QEP) += intel-qep.o +obj-$(CONFIG_TI_ECAP_CAPTURE) += ti-ecap-capture.o diff --git a/drivers/counter/counter-chrdev.c b/drivers/counter/counter-chrdev.c index 69d340be9c93f7ffbaf04b7315512520099ca852..80acdf62794a3a16adbed9964c48d53bf47defde 100644 --- a/drivers/counter/counter-chrdev.c +++ b/drivers/counter/counter-chrdev.c @@ -40,7 +40,11 @@ struct counter_comp_node { a.signal_u32_read == b.signal_u32_read || \ a.device_u64_read == b.device_u64_read || \ a.count_u64_read == b.count_u64_read || \ - a.signal_u64_read == b.signal_u64_read) + a.signal_u64_read == b.signal_u64_read || \ + a.signal_array_u32_read == b.signal_array_u32_read || \ + a.device_array_u64_read == b.device_array_u64_read || \ + a.count_array_u64_read == b.count_array_u64_read || \ + a.signal_array_u64_read == b.signal_array_u64_read) #define counter_comp_read_is_set(comp) \ (comp.action_read || \ @@ -52,7 +56,11 @@ struct counter_comp_node { comp.signal_u32_read || \ comp.device_u64_read || \ comp.count_u64_read || \ - comp.signal_u64_read) + comp.signal_u64_read || \ + comp.signal_array_u32_read || \ + comp.device_array_u64_read || \ + comp.count_array_u64_read || \ + comp.signal_array_u64_read) static ssize_t counter_chrdev_read(struct file *filp, char __user *buf, size_t len, loff_t *f_ps) @@ -228,6 +236,31 @@ static int counter_disable_events(struct counter_device *const counter) return err; } +static int counter_get_ext(const struct counter_comp *const ext, + const size_t num_ext, const size_t component_id, + size_t *const ext_idx, size_t *const id) +{ + struct counter_array *element; + + *id = 0; + for (*ext_idx = 0; *ext_idx < num_ext; (*ext_idx)++) { + if (*id == component_id) + return 0; + + if (ext->type == COUNTER_COMP_ARRAY) { + element = ext->priv; + + if (component_id - *id < element->length) + return 0; + + *id += element->length; + } else + (*id)++; + } + + return -EINVAL; +} + static int counter_add_watch(struct counter_device *const counter, const unsigned long arg) { @@ -237,6 +270,7 @@ static int counter_add_watch(struct counter_device *const counter, size_t parent, id; struct counter_comp *ext; size_t num_ext; + size_t ext_idx, ext_id; int err = 0; if (copy_from_user(&watch, uwatch, sizeof(watch))) @@ -314,11 +348,11 @@ static int counter_add_watch(struct counter_device *const counter, comp_node.comp.priv = counter->counts[parent].synapses + id; break; case COUNTER_COMPONENT_EXTENSION: - if (id >= num_ext) - return -EINVAL; - id = array_index_nospec(id, num_ext); + err = counter_get_ext(ext, num_ext, id, &ext_idx, &ext_id); + if (err < 0) + return err; - comp_node.comp = ext[id]; + comp_node.comp = ext[ext_idx]; break; default: return -EINVAL; @@ -451,14 +485,56 @@ void counter_chrdev_remove(struct counter_device *const counter) kfifo_free(&counter->events); } +static int counter_get_array_data(struct counter_device *const counter, + const enum counter_scope scope, + void *const parent, + const struct counter_comp *const comp, + const size_t idx, u64 *const value) +{ + const struct counter_array *const element = comp->priv; + u32 value_u32 = 0; + int ret; + + switch (element->type) { + case COUNTER_COMP_SIGNAL_POLARITY: + if (scope != COUNTER_SCOPE_SIGNAL) + return -EINVAL; + ret = comp->signal_array_u32_read(counter, parent, idx, + &value_u32); + *value = value_u32; + return ret; + case COUNTER_COMP_U64: + switch (scope) { + case COUNTER_SCOPE_DEVICE: + return comp->device_array_u64_read(counter, idx, value); + case COUNTER_SCOPE_SIGNAL: + return comp->signal_array_u64_read(counter, parent, idx, + value); + case COUNTER_SCOPE_COUNT: + return comp->count_array_u64_read(counter, parent, idx, + value); + default: + return -EINVAL; + } + default: + return -EINVAL; + } +} + static int counter_get_data(struct counter_device *const counter, const struct counter_comp_node *const comp_node, u64 *const value) { const struct counter_comp *const comp = &comp_node->comp; - void *const parent = comp_node->parent; + const enum counter_scope scope = comp_node->component.scope; + const size_t id = comp_node->component.id; + struct counter_signal *const signal = comp_node->parent; + struct counter_count *const count = comp_node->parent; u8 value_u8 = 0; u32 value_u32 = 0; + const struct counter_comp *ext; + size_t num_ext; + size_t ext_idx, ext_id; int ret; if (comp_node->component.type == COUNTER_COMPONENT_NONE) @@ -467,15 +543,15 @@ static int counter_get_data(struct counter_device *const counter, switch (comp->type) { case COUNTER_COMP_U8: case COUNTER_COMP_BOOL: - switch (comp_node->component.scope) { + switch (scope) { case COUNTER_SCOPE_DEVICE: ret = comp->device_u8_read(counter, &value_u8); break; case COUNTER_SCOPE_SIGNAL: - ret = comp->signal_u8_read(counter, parent, &value_u8); + ret = comp->signal_u8_read(counter, signal, &value_u8); break; case COUNTER_SCOPE_COUNT: - ret = comp->count_u8_read(counter, parent, &value_u8); + ret = comp->count_u8_read(counter, count, &value_u8); break; default: return -EINVAL; @@ -487,16 +563,17 @@ static int counter_get_data(struct counter_device *const counter, case COUNTER_COMP_ENUM: case COUNTER_COMP_COUNT_DIRECTION: case COUNTER_COMP_COUNT_MODE: - switch (comp_node->component.scope) { + case COUNTER_COMP_SIGNAL_POLARITY: + switch (scope) { case COUNTER_SCOPE_DEVICE: ret = comp->device_u32_read(counter, &value_u32); break; case COUNTER_SCOPE_SIGNAL: - ret = comp->signal_u32_read(counter, parent, + ret = comp->signal_u32_read(counter, signal, &value_u32); break; case COUNTER_SCOPE_COUNT: - ret = comp->count_u32_read(counter, parent, &value_u32); + ret = comp->count_u32_read(counter, count, &value_u32); break; default: return -EINVAL; @@ -504,21 +581,43 @@ static int counter_get_data(struct counter_device *const counter, *value = value_u32; return ret; case COUNTER_COMP_U64: - switch (comp_node->component.scope) { + switch (scope) { case COUNTER_SCOPE_DEVICE: return comp->device_u64_read(counter, value); case COUNTER_SCOPE_SIGNAL: - return comp->signal_u64_read(counter, parent, value); + return comp->signal_u64_read(counter, signal, value); case COUNTER_SCOPE_COUNT: - return comp->count_u64_read(counter, parent, value); + return comp->count_u64_read(counter, count, value); default: return -EINVAL; } case COUNTER_COMP_SYNAPSE_ACTION: - ret = comp->action_read(counter, parent, comp->priv, - &value_u32); + ret = comp->action_read(counter, count, comp->priv, &value_u32); *value = value_u32; return ret; + case COUNTER_COMP_ARRAY: + switch (scope) { + case COUNTER_SCOPE_DEVICE: + ext = counter->ext; + num_ext = counter->num_ext; + break; + case COUNTER_SCOPE_SIGNAL: + ext = signal->ext; + num_ext = signal->num_ext; + break; + case COUNTER_SCOPE_COUNT: + ext = count->ext; + num_ext = count->num_ext; + break; + default: + return -EINVAL; + } + ret = counter_get_ext(ext, num_ext, id, &ext_idx, &ext_id); + if (ret < 0) + return ret; + + return counter_get_array_data(counter, scope, comp_node->parent, + comp, id - ext_id, value); default: return -EINVAL; } @@ -574,4 +673,4 @@ exit_early: if (copied) wake_up_poll(&counter->events_wait, EPOLLIN); } -EXPORT_SYMBOL_GPL(counter_push_event); +EXPORT_SYMBOL_NS_GPL(counter_push_event, COUNTER); diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c index 938651f9e9e09221b62eecc4e0da40081ffe8c87..09c77afb33ca84e79c077c87659252c64840929a 100644 --- a/drivers/counter/counter-core.c +++ b/drivers/counter/counter-core.c @@ -73,7 +73,7 @@ void *counter_priv(const struct counter_device *const counter) return &ch->privdata; } -EXPORT_SYMBOL_GPL(counter_priv); +EXPORT_SYMBOL_NS_GPL(counter_priv, COUNTER); /** * counter_alloc - allocate a counter_device @@ -133,13 +133,13 @@ err_ida_alloc: return NULL; } -EXPORT_SYMBOL_GPL(counter_alloc); +EXPORT_SYMBOL_NS_GPL(counter_alloc, COUNTER); void counter_put(struct counter_device *counter) { put_device(&counter->dev); } -EXPORT_SYMBOL_GPL(counter_put); +EXPORT_SYMBOL_NS_GPL(counter_put, COUNTER); /** * counter_add - complete registration of a counter @@ -166,7 +166,7 @@ int counter_add(struct counter_device *counter) /* implies device_add(dev) */ return cdev_device_add(&counter->chrdev, dev); } -EXPORT_SYMBOL_GPL(counter_add); +EXPORT_SYMBOL_NS_GPL(counter_add, COUNTER); /** * counter_unregister - unregister Counter from the system @@ -188,7 +188,7 @@ void counter_unregister(struct counter_device *const counter) mutex_unlock(&counter->ops_exist_lock); } -EXPORT_SYMBOL_GPL(counter_unregister); +EXPORT_SYMBOL_NS_GPL(counter_unregister, COUNTER); static void devm_counter_release(void *counter) { @@ -223,7 +223,7 @@ struct counter_device *devm_counter_alloc(struct device *dev, size_t sizeof_priv return counter; } -EXPORT_SYMBOL_GPL(devm_counter_alloc); +EXPORT_SYMBOL_NS_GPL(devm_counter_alloc, COUNTER); /** * devm_counter_add - complete registration of a counter @@ -244,7 +244,7 @@ int devm_counter_add(struct device *dev, return devm_add_action_or_reset(dev, devm_counter_release, counter); } -EXPORT_SYMBOL_GPL(devm_counter_add); +EXPORT_SYMBOL_NS_GPL(devm_counter_add, COUNTER); #define COUNTER_DEV_MAX 256 diff --git a/drivers/counter/counter-sysfs.c b/drivers/counter/counter-sysfs.c index 04eac41dad33e9227647cbd54a28cdf2468a67f7..b9efe66f9f8d86fb8e337a3d7ca285d0d9e39bbc 100644 --- a/drivers/counter/counter-sysfs.c +++ b/drivers/counter/counter-sysfs.c @@ -91,6 +91,11 @@ static const char *const counter_count_mode_str[] = { [COUNTER_COUNT_MODE_MODULO_N] = "modulo-n" }; +static const char *const counter_signal_polarity_str[] = { + [COUNTER_SIGNAL_POLARITY_POSITIVE] = "positive", + [COUNTER_SIGNAL_POLARITY_NEGATIVE] = "negative" +}; + static ssize_t counter_comp_u8_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -201,6 +206,8 @@ static ssize_t counter_comp_u32_show(struct device *dev, return sysfs_emit(buf, "%s\n", counter_count_direction_str[data]); case COUNTER_COMP_COUNT_MODE: return sysfs_emit(buf, "%s\n", counter_count_mode_str[data]); + case COUNTER_COMP_SIGNAL_POLARITY: + return sysfs_emit(buf, "%s\n", counter_signal_polarity_str[data]); default: return sysfs_emit(buf, "%u\n", (unsigned int)data); } @@ -252,6 +259,10 @@ static ssize_t counter_comp_u32_store(struct device *dev, err = counter_find_enum(&data, avail->enums, avail->num_items, buf, counter_count_mode_str); break; + case COUNTER_COMP_SIGNAL_POLARITY: + err = counter_find_enum(&data, avail->enums, avail->num_items, + buf, counter_signal_polarity_str); + break; default: err = kstrtou32(buf, 0, &data); break; @@ -341,6 +352,124 @@ static ssize_t counter_comp_u64_store(struct device *dev, return len; } +static ssize_t counter_comp_array_u32_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + const struct counter_attribute *const a = to_counter_attribute(attr); + struct counter_device *const counter = counter_from_dev(dev); + const struct counter_array *const element = a->comp.priv; + int err; + u32 data = 0; + + if (a->scope != COUNTER_SCOPE_SIGNAL || + element->type != COUNTER_COMP_SIGNAL_POLARITY) + return -EINVAL; + + err = a->comp.signal_array_u32_read(counter, a->parent, element->idx, + &data); + if (err < 0) + return err; + + return sysfs_emit(buf, "%s\n", counter_signal_polarity_str[data]); +} + +static ssize_t counter_comp_array_u32_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + const struct counter_attribute *const a = to_counter_attribute(attr); + struct counter_device *const counter = counter_from_dev(dev); + const struct counter_array *const element = a->comp.priv; + int err; + u32 data = 0; + + if (element->type != COUNTER_COMP_SIGNAL_POLARITY || + a->scope != COUNTER_SCOPE_SIGNAL) + return -EINVAL; + + err = counter_find_enum(&data, element->avail->enums, + element->avail->num_items, buf, + counter_signal_polarity_str); + if (err < 0) + return err; + + err = a->comp.signal_array_u32_write(counter, a->parent, element->idx, + data); + if (err < 0) + return err; + + return len; +} + +static ssize_t counter_comp_array_u64_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + const struct counter_attribute *const a = to_counter_attribute(attr); + struct counter_device *const counter = counter_from_dev(dev); + const struct counter_array *const element = a->comp.priv; + int err; + u64 data = 0; + + switch (a->scope) { + case COUNTER_SCOPE_DEVICE: + err = a->comp.device_array_u64_read(counter, element->idx, + &data); + break; + case COUNTER_SCOPE_SIGNAL: + err = a->comp.signal_array_u64_read(counter, a->parent, + element->idx, &data); + break; + case COUNTER_SCOPE_COUNT: + err = a->comp.count_array_u64_read(counter, a->parent, + element->idx, &data); + break; + default: + return -EINVAL; + } + if (err < 0) + return err; + + return sysfs_emit(buf, "%llu\n", (unsigned long long)data); +} + +static ssize_t counter_comp_array_u64_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + const struct counter_attribute *const a = to_counter_attribute(attr); + struct counter_device *const counter = counter_from_dev(dev); + const struct counter_array *const element = a->comp.priv; + int err; + u64 data = 0; + + err = kstrtou64(buf, 0, &data); + if (err < 0) + return err; + + switch (a->scope) { + case COUNTER_SCOPE_DEVICE: + err = a->comp.device_array_u64_write(counter, element->idx, + data); + break; + case COUNTER_SCOPE_SIGNAL: + err = a->comp.signal_array_u64_write(counter, a->parent, + element->idx, data); + break; + case COUNTER_SCOPE_COUNT: + err = a->comp.count_array_u64_write(counter, a->parent, + element->idx, data); + break; + default: + return -EINVAL; + } + if (err < 0) + return err; + + return len; +} + static ssize_t enums_available_show(const u32 *const enums, const size_t num_enums, const char *const strs[], char *buf) @@ -435,6 +564,7 @@ static int counter_attr_create(struct device *const dev, const enum counter_scope scope, void *const parent) { + const struct counter_array *const array = comp->priv; struct counter_attribute *counter_attr; struct device_attribute *dev_attr; @@ -469,6 +599,7 @@ static int counter_attr_create(struct device *const dev, case COUNTER_COMP_ENUM: case COUNTER_COMP_COUNT_DIRECTION: case COUNTER_COMP_COUNT_MODE: + case COUNTER_COMP_SIGNAL_POLARITY: if (comp->device_u32_read) { dev_attr->attr.mode |= 0444; dev_attr->show = counter_comp_u32_show; @@ -488,6 +619,32 @@ static int counter_attr_create(struct device *const dev, dev_attr->store = counter_comp_u64_store; } break; + case COUNTER_COMP_ARRAY: + switch (array->type) { + case COUNTER_COMP_SIGNAL_POLARITY: + if (comp->signal_array_u32_read) { + dev_attr->attr.mode |= 0444; + dev_attr->show = counter_comp_array_u32_show; + } + if (comp->signal_array_u32_write) { + dev_attr->attr.mode |= 0200; + dev_attr->store = counter_comp_array_u32_store; + } + break; + case COUNTER_COMP_U64: + if (comp->device_array_u64_read) { + dev_attr->attr.mode |= 0444; + dev_attr->show = counter_comp_array_u64_show; + } + if (comp->device_array_u64_write) { + dev_attr->attr.mode |= 0200; + dev_attr->store = counter_comp_array_u64_store; + } + break; + default: + return -EINVAL; + } + break; default: return -EINVAL; } @@ -580,6 +737,95 @@ static int counter_comp_id_attr_create(struct device *const dev, return 0; } +static int counter_ext_attrs_create(struct device *const dev, + struct counter_attribute_group *const group, + const struct counter_comp *const ext, + const enum counter_scope scope, + void *const parent, const size_t id) +{ + int err; + + /* Create main extension attribute */ + err = counter_attr_create(dev, group, ext, scope, parent); + if (err < 0) + return err; + + /* Create extension id attribute */ + return counter_comp_id_attr_create(dev, group, ext->name, id); +} + +static int counter_array_attrs_create(struct device *const dev, + struct counter_attribute_group *const group, + const struct counter_comp *const comp, + const enum counter_scope scope, + void *const parent, const size_t id) +{ + const struct counter_array *const array = comp->priv; + struct counter_comp ext = *comp; + struct counter_array *element; + size_t idx; + int err; + + /* Create an attribute for each array element */ + for (idx = 0; idx < array->length; idx++) { + /* Generate array element attribute name */ + ext.name = devm_kasprintf(dev, GFP_KERNEL, "%s%zu", comp->name, + idx); + if (!ext.name) + return -ENOMEM; + + /* Allocate and configure array element */ + element = devm_kzalloc(dev, sizeof(*element), GFP_KERNEL); + if (!element) + return -ENOMEM; + element->type = array->type; + element->avail = array->avail; + element->idx = idx; + ext.priv = element; + + /* Create all attributes associated with the array element */ + err = counter_ext_attrs_create(dev, group, &ext, scope, parent, + id + idx); + if (err < 0) + return err; + } + + return 0; +} + +static int counter_sysfs_exts_add(struct device *const dev, + struct counter_attribute_group *const group, + const struct counter_comp *const exts, + const size_t num_ext, + const enum counter_scope scope, + void *const parent) +{ + size_t i; + const struct counter_comp *ext; + int err; + size_t id = 0; + const struct counter_array *array; + + /* Create attributes for each extension */ + for (i = 0; i < num_ext; i++) { + ext = &exts[i]; + if (ext->type == COUNTER_COMP_ARRAY) { + err = counter_array_attrs_create(dev, group, ext, scope, + parent, id); + array = ext->priv; + id += array->length; + } else { + err = counter_ext_attrs_create(dev, group, ext, scope, + parent, id); + id++; + } + if (err < 0) + return err; + } + + return 0; +} + static struct counter_comp counter_signal_comp = { .type = COUNTER_COMP_SIGNAL_LEVEL, .name = "signal", @@ -593,8 +839,6 @@ static int counter_signal_attrs_create(struct counter_device *const counter, struct device *const dev = &counter->dev; int err; struct counter_comp comp; - size_t i; - struct counter_comp *ext; /* Create main Signal attribute */ comp = counter_signal_comp; @@ -608,21 +852,9 @@ static int counter_signal_attrs_create(struct counter_device *const counter, if (err < 0) return err; - /* Create an attribute for each extension */ - for (i = 0; i < signal->num_ext; i++) { - ext = &signal->ext[i]; - - err = counter_attr_create(dev, cattr_group, ext, scope, signal); - if (err < 0) - return err; - - err = counter_comp_id_attr_create(dev, cattr_group, ext->name, - i); - if (err < 0) - return err; - } - - return 0; + /* Add Signal extensions */ + return counter_sysfs_exts_add(dev, cattr_group, signal->ext, + signal->num_ext, scope, signal); } static int counter_sysfs_signals_add(struct counter_device *const counter, @@ -707,8 +939,6 @@ static int counter_count_attrs_create(struct counter_device *const counter, struct device *const dev = &counter->dev; int err; struct counter_comp comp; - size_t i; - struct counter_comp *ext; /* Create main Count attribute */ comp = counter_count_comp; @@ -731,21 +961,9 @@ static int counter_count_attrs_create(struct counter_device *const counter, if (err < 0) return err; - /* Create an attribute for each extension */ - for (i = 0; i < count->num_ext; i++) { - ext = &count->ext[i]; - - err = counter_attr_create(dev, cattr_group, ext, scope, count); - if (err < 0) - return err; - - err = counter_comp_id_attr_create(dev, cattr_group, ext->name, - i); - if (err < 0) - return err; - } - - return 0; + /* Add Count extensions */ + return counter_sysfs_exts_add(dev, cattr_group, count->ext, + count->num_ext, scope, count); } static int counter_sysfs_counts_add(struct counter_device *const counter, @@ -838,8 +1056,6 @@ static int counter_sysfs_attr_add(struct counter_device *const counter, const enum counter_scope scope = COUNTER_SCOPE_DEVICE; struct device *const dev = &counter->dev; int err; - size_t i; - struct counter_comp *ext; /* Add Signals sysfs attributes */ err = counter_sysfs_signals_add(counter, cattr_group); @@ -876,19 +1092,9 @@ static int counter_sysfs_attr_add(struct counter_device *const counter, if (err < 0) return err; - /* Create an attribute for each extension */ - for (i = 0; i < counter->num_ext; i++) { - ext = &counter->ext[i]; - - err = counter_attr_create(dev, cattr_group, ext, scope, NULL); - if (err < 0) - return err; - - err = counter_comp_id_attr_create(dev, cattr_group, ext->name, - i); - if (err < 0) - return err; - } + /* Add device extensions */ + return counter_sysfs_exts_add(dev, cattr_group, counter->ext, + counter->num_ext, scope, NULL); return 0; } diff --git a/drivers/counter/ftm-quaddec.c b/drivers/counter/ftm-quaddec.c index 2a58582a9df4250cccc3d04c1281b779d12c89ed..aea6622a9b1383c03aeb424e7fe318fd21f23e44 100644 --- a/drivers/counter/ftm-quaddec.c +++ b/drivers/counter/ftm-quaddec.c @@ -325,3 +325,4 @@ module_platform_driver(ftm_quaddec_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Kjeld Flarup "); MODULE_AUTHOR("Patrick Havelange "); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/intel-qep.c b/drivers/counter/intel-qep.c index 47a6a9dfc9e8c191d8194dc11e5e979e7a5b1d06..af5942e66f7d0c584a61d936498a0d43fd465a7e 100644 --- a/drivers/counter/intel-qep.c +++ b/drivers/counter/intel-qep.c @@ -523,3 +523,4 @@ MODULE_AUTHOR("Jarkko Nikula "); MODULE_AUTHOR("Raymond Tan "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Intel Quadrature Encoder Peripheral driver"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/interrupt-cnt.c b/drivers/counter/interrupt-cnt.c index 3b13f56bbb115cc4a51aba2c97edd53aef4d7276..229473855c5b38abf9900046d184a32ce6943c5e 100644 --- a/drivers/counter/interrupt-cnt.c +++ b/drivers/counter/interrupt-cnt.c @@ -139,12 +139,23 @@ static int interrupt_cnt_signal_read(struct counter_device *counter, return 0; } +static int interrupt_cnt_watch_validate(struct counter_device *counter, + const struct counter_watch *watch) +{ + if (watch->channel != 0 || + watch->event != COUNTER_EVENT_CHANGE_OF_STATE) + return -EINVAL; + + return 0; +} + static const struct counter_ops interrupt_cnt_ops = { .action_read = interrupt_cnt_action_read, .count_read = interrupt_cnt_read, .count_write = interrupt_cnt_write, .function_read = interrupt_cnt_function_read, .signal_read = interrupt_cnt_signal_read, + .watch_validate = interrupt_cnt_watch_validate, }; static int interrupt_cnt_probe(struct platform_device *pdev) @@ -242,3 +253,4 @@ MODULE_ALIAS("platform:interrupt-counter"); MODULE_AUTHOR("Oleksij Rempel "); MODULE_DESCRIPTION("Interrupt counter driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/microchip-tcb-capture.c b/drivers/counter/microchip-tcb-capture.c index 00844445143b65d2e753e0dc8f6c7b77b40caebb..f9dee15d9777631a24ac3a7a6a8e34c295b11fcd 100644 --- a/drivers/counter/microchip-tcb-capture.c +++ b/drivers/counter/microchip-tcb-capture.c @@ -394,3 +394,4 @@ module_platform_driver(mchp_tc_driver); MODULE_AUTHOR("Kamel Bouhara "); MODULE_DESCRIPTION("Microchip TCB Capture driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/stm32-lptimer-cnt.c b/drivers/counter/stm32-lptimer-cnt.c index 68031d93ce895e6a69773ae84c2aeee409c0f3c6..d6b80b6dfc28796a1a710c93f97689defef65b4d 100644 --- a/drivers/counter/stm32-lptimer-cnt.c +++ b/drivers/counter/stm32-lptimer-cnt.c @@ -520,3 +520,4 @@ MODULE_AUTHOR("Fabrice Gasnier "); MODULE_ALIAS("platform:stm32-lptimer-counter"); MODULE_DESCRIPTION("STMicroelectronics STM32 LPTIM counter driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/stm32-timer-cnt.c b/drivers/counter/stm32-timer-cnt.c index 5779ae7c73cfc5362ed103d64ef4a3b3325ef39f..9bf20a5d6bda8f9d4f47821708588a922bea36f8 100644 --- a/drivers/counter/stm32-timer-cnt.c +++ b/drivers/counter/stm32-timer-cnt.c @@ -417,3 +417,4 @@ MODULE_AUTHOR("Benjamin Gaignard "); MODULE_ALIAS("platform:stm32-timer-counter"); MODULE_DESCRIPTION("STMicroelectronics STM32 TIMER counter driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/ti-ecap-capture.c b/drivers/counter/ti-ecap-capture.c new file mode 100644 index 0000000000000000000000000000000000000000..af10de30aba51d7b4c6d37bc8bef53e22725787b --- /dev/null +++ b/drivers/counter/ti-ecap-capture.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * ECAP Capture driver + * + * Copyright (C) 2022 Julien Panis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ECAP_DRV_NAME "ecap" + +/* ECAP event IDs */ +#define ECAP_CEVT1 0 +#define ECAP_CEVT2 1 +#define ECAP_CEVT3 2 +#define ECAP_CEVT4 3 +#define ECAP_CNTOVF 4 + +#define ECAP_CEVT_LAST ECAP_CEVT4 +#define ECAP_NB_CEVT (ECAP_CEVT_LAST + 1) + +#define ECAP_EVT_LAST ECAP_CNTOVF +#define ECAP_NB_EVT (ECAP_EVT_LAST + 1) + +/* Registers */ +#define ECAP_TSCNT_REG 0x00 + +#define ECAP_CAP_REG(i) (((i) << 2) + 0x08) + +#define ECAP_ECCTL_REG 0x28 +#define ECAP_CAPPOL_BIT(i) BIT((i) << 1) +#define ECAP_EV_MODE_MASK GENMASK(7, 0) +#define ECAP_CAPLDEN_BIT BIT(8) +#define ECAP_CONT_ONESHT_BIT BIT(16) +#define ECAP_STOPVALUE_MASK GENMASK(18, 17) +#define ECAP_TSCNTSTP_BIT BIT(20) +#define ECAP_SYNCO_DIS_MASK GENMASK(23, 22) +#define ECAP_CAP_APWM_BIT BIT(25) +#define ECAP_ECCTL_EN_MASK (ECAP_CAPLDEN_BIT | ECAP_TSCNTSTP_BIT) +#define ECAP_ECCTL_CFG_MASK (ECAP_SYNCO_DIS_MASK | ECAP_STOPVALUE_MASK \ + | ECAP_ECCTL_EN_MASK | ECAP_CAP_APWM_BIT \ + | ECAP_CONT_ONESHT_BIT) + +#define ECAP_ECINT_EN_FLG_REG 0x2c +#define ECAP_EVT_EN_MASK GENMASK(ECAP_NB_EVT, ECAP_NB_CEVT) +#define ECAP_EVT_FLG_BIT(i) BIT((i) + 17) + +#define ECAP_ECINT_CLR_FRC_REG 0x30 +#define ECAP_INT_CLR_BIT BIT(0) +#define ECAP_EVT_CLR_BIT(i) BIT((i) + 1) +#define ECAP_EVT_CLR_MASK GENMASK(ECAP_NB_EVT, 0) + +#define ECAP_PID_REG 0x5c + +/* ECAP signals */ +#define ECAP_CLOCK_SIG 0 +#define ECAP_INPUT_SIG 1 + +static const struct regmap_config ecap_cnt_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = ECAP_PID_REG, +}; + +/** + * struct ecap_cnt_dev - device private data structure + * @enabled: device state + * @lock: synchronization lock to prevent I/O race conditions + * @clk: device clock + * @regmap: device register map + * @nb_ovf: number of overflows since capture start + * @pm_ctx: device context for PM operations + * @pm_ctx.ev_mode: event mode bits + * @pm_ctx.time_cntr: timestamp counter value + */ +struct ecap_cnt_dev { + bool enabled; + struct mutex lock; + struct clk *clk; + struct regmap *regmap; + atomic_t nb_ovf; + struct { + u8 ev_mode; + u32 time_cntr; + } pm_ctx; +}; + +static u8 ecap_cnt_capture_get_evmode(struct counter_device *counter) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + unsigned int regval; + + pm_runtime_get_sync(counter->parent); + regmap_read(ecap_dev->regmap, ECAP_ECCTL_REG, ®val); + pm_runtime_put_sync(counter->parent); + + return regval; +} + +static void ecap_cnt_capture_set_evmode(struct counter_device *counter, u8 ev_mode) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + pm_runtime_get_sync(counter->parent); + regmap_update_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_EV_MODE_MASK, ev_mode); + pm_runtime_put_sync(counter->parent); +} + +static void ecap_cnt_capture_enable(struct counter_device *counter) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + pm_runtime_get_sync(counter->parent); + + /* Enable interrupts on events */ + regmap_update_bits(ecap_dev->regmap, ECAP_ECINT_EN_FLG_REG, + ECAP_EVT_EN_MASK, ECAP_EVT_EN_MASK); + + /* Run counter */ + regmap_update_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_ECCTL_CFG_MASK, + ECAP_SYNCO_DIS_MASK | ECAP_STOPVALUE_MASK | ECAP_ECCTL_EN_MASK); +} + +static void ecap_cnt_capture_disable(struct counter_device *counter) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + /* Stop counter */ + regmap_update_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_ECCTL_EN_MASK, 0); + + /* Disable interrupts on events */ + regmap_update_bits(ecap_dev->regmap, ECAP_ECINT_EN_FLG_REG, ECAP_EVT_EN_MASK, 0); + + pm_runtime_put_sync(counter->parent); +} + +static u32 ecap_cnt_count_get_val(struct counter_device *counter, unsigned int reg) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + unsigned int regval; + + pm_runtime_get_sync(counter->parent); + regmap_read(ecap_dev->regmap, reg, ®val); + pm_runtime_put_sync(counter->parent); + + return regval; +} + +static void ecap_cnt_count_set_val(struct counter_device *counter, unsigned int reg, u32 val) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + pm_runtime_get_sync(counter->parent); + regmap_write(ecap_dev->regmap, reg, val); + pm_runtime_put_sync(counter->parent); +} + +static int ecap_cnt_count_read(struct counter_device *counter, + struct counter_count *count, u64 *val) +{ + *val = ecap_cnt_count_get_val(counter, ECAP_TSCNT_REG); + + return 0; +} + +static int ecap_cnt_count_write(struct counter_device *counter, + struct counter_count *count, u64 val) +{ + if (val > U32_MAX) + return -ERANGE; + + ecap_cnt_count_set_val(counter, ECAP_TSCNT_REG, val); + + return 0; +} + +static int ecap_cnt_function_read(struct counter_device *counter, + struct counter_count *count, + enum counter_function *function) +{ + *function = COUNTER_FUNCTION_INCREASE; + + return 0; +} + +static int ecap_cnt_action_read(struct counter_device *counter, + struct counter_count *count, + struct counter_synapse *synapse, + enum counter_synapse_action *action) +{ + *action = (synapse->signal->id == ECAP_CLOCK_SIG) ? + COUNTER_SYNAPSE_ACTION_RISING_EDGE : + COUNTER_SYNAPSE_ACTION_NONE; + + return 0; +} + +static int ecap_cnt_watch_validate(struct counter_device *counter, + const struct counter_watch *watch) +{ + if (watch->channel > ECAP_CEVT_LAST) + return -EINVAL; + + switch (watch->event) { + case COUNTER_EVENT_CAPTURE: + case COUNTER_EVENT_OVERFLOW: + return 0; + default: + return -EINVAL; + } +} + +static int ecap_cnt_clk_get_freq(struct counter_device *counter, + struct counter_signal *signal, u64 *freq) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + *freq = clk_get_rate(ecap_dev->clk); + + return 0; +} + +static int ecap_cnt_pol_read(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, enum counter_signal_polarity *pol) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + int bitval; + + pm_runtime_get_sync(counter->parent); + bitval = regmap_test_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_CAPPOL_BIT(idx)); + pm_runtime_put_sync(counter->parent); + + *pol = bitval ? COUNTER_SIGNAL_POLARITY_NEGATIVE : COUNTER_SIGNAL_POLARITY_POSITIVE; + + return 0; +} + +static int ecap_cnt_pol_write(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, enum counter_signal_polarity pol) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + pm_runtime_get_sync(counter->parent); + if (pol == COUNTER_SIGNAL_POLARITY_NEGATIVE) + regmap_set_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_CAPPOL_BIT(idx)); + else + regmap_clear_bits(ecap_dev->regmap, ECAP_ECCTL_REG, ECAP_CAPPOL_BIT(idx)); + pm_runtime_put_sync(counter->parent); + + return 0; +} + +static int ecap_cnt_cap_read(struct counter_device *counter, + struct counter_count *count, + size_t idx, u64 *cap) +{ + *cap = ecap_cnt_count_get_val(counter, ECAP_CAP_REG(idx)); + + return 0; +} + +static int ecap_cnt_cap_write(struct counter_device *counter, + struct counter_count *count, + size_t idx, u64 cap) +{ + if (cap > U32_MAX) + return -ERANGE; + + ecap_cnt_count_set_val(counter, ECAP_CAP_REG(idx), cap); + + return 0; +} + +static int ecap_cnt_nb_ovf_read(struct counter_device *counter, + struct counter_count *count, u64 *val) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + *val = atomic_read(&ecap_dev->nb_ovf); + + return 0; +} + +static int ecap_cnt_nb_ovf_write(struct counter_device *counter, + struct counter_count *count, u64 val) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + if (val > U32_MAX) + return -ERANGE; + + atomic_set(&ecap_dev->nb_ovf, val); + + return 0; +} + +static int ecap_cnt_ceiling_read(struct counter_device *counter, + struct counter_count *count, u64 *val) +{ + *val = U32_MAX; + + return 0; +} + +static int ecap_cnt_enable_read(struct counter_device *counter, + struct counter_count *count, u8 *enable) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + *enable = ecap_dev->enabled; + + return 0; +} + +static int ecap_cnt_enable_write(struct counter_device *counter, + struct counter_count *count, u8 enable) +{ + struct ecap_cnt_dev *ecap_dev = counter_priv(counter); + + mutex_lock(&ecap_dev->lock); + + if (enable == ecap_dev->enabled) + goto out; + + if (enable) + ecap_cnt_capture_enable(counter); + else + ecap_cnt_capture_disable(counter); + ecap_dev->enabled = enable; + +out: + mutex_unlock(&ecap_dev->lock); + + return 0; +} + +static const struct counter_ops ecap_cnt_ops = { + .count_read = ecap_cnt_count_read, + .count_write = ecap_cnt_count_write, + .function_read = ecap_cnt_function_read, + .action_read = ecap_cnt_action_read, + .watch_validate = ecap_cnt_watch_validate, +}; + +static const enum counter_function ecap_cnt_functions[] = { + COUNTER_FUNCTION_INCREASE, +}; + +static const enum counter_synapse_action ecap_cnt_clock_actions[] = { + COUNTER_SYNAPSE_ACTION_RISING_EDGE, +}; + +static const enum counter_synapse_action ecap_cnt_input_actions[] = { + COUNTER_SYNAPSE_ACTION_NONE, +}; + +static struct counter_comp ecap_cnt_clock_ext[] = { + COUNTER_COMP_SIGNAL_U64("frequency", ecap_cnt_clk_get_freq, NULL), +}; + +static const enum counter_signal_polarity ecap_cnt_pol_avail[] = { + COUNTER_SIGNAL_POLARITY_POSITIVE, + COUNTER_SIGNAL_POLARITY_NEGATIVE, +}; + +static DEFINE_COUNTER_ARRAY_POLARITY(ecap_cnt_pol_array, ecap_cnt_pol_avail, ECAP_NB_CEVT); + +static struct counter_comp ecap_cnt_signal_ext[] = { + COUNTER_COMP_ARRAY_POLARITY(ecap_cnt_pol_read, ecap_cnt_pol_write, ecap_cnt_pol_array), +}; + +static struct counter_signal ecap_cnt_signals[] = { + { + .id = ECAP_CLOCK_SIG, + .name = "Clock Signal", + .ext = ecap_cnt_clock_ext, + .num_ext = ARRAY_SIZE(ecap_cnt_clock_ext), + }, + { + .id = ECAP_INPUT_SIG, + .name = "Input Signal", + .ext = ecap_cnt_signal_ext, + .num_ext = ARRAY_SIZE(ecap_cnt_signal_ext), + }, +}; + +static struct counter_synapse ecap_cnt_synapses[] = { + { + .actions_list = ecap_cnt_clock_actions, + .num_actions = ARRAY_SIZE(ecap_cnt_clock_actions), + .signal = &ecap_cnt_signals[ECAP_CLOCK_SIG], + }, + { + .actions_list = ecap_cnt_input_actions, + .num_actions = ARRAY_SIZE(ecap_cnt_input_actions), + .signal = &ecap_cnt_signals[ECAP_INPUT_SIG], + }, +}; + +static DEFINE_COUNTER_ARRAY_CAPTURE(ecap_cnt_cap_array, ECAP_NB_CEVT); + +static struct counter_comp ecap_cnt_count_ext[] = { + COUNTER_COMP_ARRAY_CAPTURE(ecap_cnt_cap_read, ecap_cnt_cap_write, ecap_cnt_cap_array), + COUNTER_COMP_COUNT_U64("num_overflows", ecap_cnt_nb_ovf_read, ecap_cnt_nb_ovf_write), + COUNTER_COMP_CEILING(ecap_cnt_ceiling_read, NULL), + COUNTER_COMP_ENABLE(ecap_cnt_enable_read, ecap_cnt_enable_write), +}; + +static struct counter_count ecap_cnt_counts[] = { + { + .name = "Timestamp Counter", + .functions_list = ecap_cnt_functions, + .num_functions = ARRAY_SIZE(ecap_cnt_functions), + .synapses = ecap_cnt_synapses, + .num_synapses = ARRAY_SIZE(ecap_cnt_synapses), + .ext = ecap_cnt_count_ext, + .num_ext = ARRAY_SIZE(ecap_cnt_count_ext), + }, +}; + +static irqreturn_t ecap_cnt_isr(int irq, void *dev_id) +{ + struct counter_device *counter_dev = dev_id; + struct ecap_cnt_dev *ecap_dev = counter_priv(counter_dev); + unsigned int clr = 0; + unsigned int flg; + int i; + + regmap_read(ecap_dev->regmap, ECAP_ECINT_EN_FLG_REG, &flg); + + /* Check capture events */ + for (i = 0 ; i < ECAP_NB_CEVT ; i++) { + if (flg & ECAP_EVT_FLG_BIT(i)) { + counter_push_event(counter_dev, COUNTER_EVENT_CAPTURE, i); + clr |= ECAP_EVT_CLR_BIT(i); + } + } + + /* Check counter overflow */ + if (flg & ECAP_EVT_FLG_BIT(ECAP_CNTOVF)) { + atomic_inc(&ecap_dev->nb_ovf); + for (i = 0 ; i < ECAP_NB_CEVT ; i++) + counter_push_event(counter_dev, COUNTER_EVENT_OVERFLOW, i); + clr |= ECAP_EVT_CLR_BIT(ECAP_CNTOVF); + } + + clr |= ECAP_INT_CLR_BIT; + regmap_update_bits(ecap_dev->regmap, ECAP_ECINT_CLR_FRC_REG, ECAP_EVT_CLR_MASK, clr); + + return IRQ_HANDLED; +} + +static void ecap_cnt_pm_disable(void *dev) +{ + pm_runtime_disable(dev); +} + +static int ecap_cnt_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ecap_cnt_dev *ecap_dev; + struct counter_device *counter_dev; + void __iomem *mmio_base; + unsigned long clk_rate; + int ret; + + counter_dev = devm_counter_alloc(dev, sizeof(*ecap_dev)); + if (IS_ERR(counter_dev)) + return PTR_ERR(counter_dev); + + counter_dev->name = ECAP_DRV_NAME; + counter_dev->parent = dev; + counter_dev->ops = &ecap_cnt_ops; + counter_dev->signals = ecap_cnt_signals; + counter_dev->num_signals = ARRAY_SIZE(ecap_cnt_signals); + counter_dev->counts = ecap_cnt_counts; + counter_dev->num_counts = ARRAY_SIZE(ecap_cnt_counts); + + ecap_dev = counter_priv(counter_dev); + + mutex_init(&ecap_dev->lock); + + ecap_dev->clk = devm_clk_get_enabled(dev, "fck"); + if (IS_ERR(ecap_dev->clk)) + return dev_err_probe(dev, PTR_ERR(ecap_dev->clk), "failed to get clock\n"); + + clk_rate = clk_get_rate(ecap_dev->clk); + if (!clk_rate) { + dev_err(dev, "failed to get clock rate\n"); + return -EINVAL; + } + + mmio_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mmio_base)) + return PTR_ERR(mmio_base); + + ecap_dev->regmap = devm_regmap_init_mmio(dev, mmio_base, &ecap_cnt_regmap_config); + if (IS_ERR(ecap_dev->regmap)) + return dev_err_probe(dev, PTR_ERR(ecap_dev->regmap), "failed to init regmap\n"); + + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to get irq\n"); + + ret = devm_request_irq(dev, ret, ecap_cnt_isr, 0, pdev->name, counter_dev); + if (ret) + return dev_err_probe(dev, ret, "failed to request irq\n"); + + platform_set_drvdata(pdev, counter_dev); + + pm_runtime_enable(dev); + + /* Register a cleanup callback to care for disabling PM */ + ret = devm_add_action_or_reset(dev, ecap_cnt_pm_disable, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to add pm disable action\n"); + + ret = devm_counter_add(dev, counter_dev); + if (ret) + return dev_err_probe(dev, ret, "failed to add counter\n"); + + return 0; +} + +static int ecap_cnt_remove(struct platform_device *pdev) +{ + struct counter_device *counter_dev = platform_get_drvdata(pdev); + struct ecap_cnt_dev *ecap_dev = counter_priv(counter_dev); + + if (ecap_dev->enabled) + ecap_cnt_capture_disable(counter_dev); + + return 0; +} + +static int ecap_cnt_suspend(struct device *dev) +{ + struct counter_device *counter_dev = dev_get_drvdata(dev); + struct ecap_cnt_dev *ecap_dev = counter_priv(counter_dev); + + /* If eCAP is running, stop capture then save timestamp counter */ + if (ecap_dev->enabled) { + /* + * Disabling capture has the following effects: + * - interrupts are disabled + * - loading of capture registers is disabled + * - timebase counter is stopped + */ + ecap_cnt_capture_disable(counter_dev); + ecap_dev->pm_ctx.time_cntr = ecap_cnt_count_get_val(counter_dev, ECAP_TSCNT_REG); + } + + ecap_dev->pm_ctx.ev_mode = ecap_cnt_capture_get_evmode(counter_dev); + + clk_disable(ecap_dev->clk); + + return 0; +} + +static int ecap_cnt_resume(struct device *dev) +{ + struct counter_device *counter_dev = dev_get_drvdata(dev); + struct ecap_cnt_dev *ecap_dev = counter_priv(counter_dev); + + clk_enable(ecap_dev->clk); + + ecap_cnt_capture_set_evmode(counter_dev, ecap_dev->pm_ctx.ev_mode); + + /* If eCAP was running, restore timestamp counter then run capture */ + if (ecap_dev->enabled) { + ecap_cnt_count_set_val(counter_dev, ECAP_TSCNT_REG, ecap_dev->pm_ctx.time_cntr); + ecap_cnt_capture_enable(counter_dev); + } + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(ecap_cnt_pm_ops, ecap_cnt_suspend, ecap_cnt_resume); + +static const struct of_device_id ecap_cnt_of_match[] = { + { .compatible = "ti,am62-ecap-capture" }, + {}, +}; +MODULE_DEVICE_TABLE(of, ecap_cnt_of_match); + +static struct platform_driver ecap_cnt_driver = { + .probe = ecap_cnt_probe, + .remove = ecap_cnt_remove, + .driver = { + .name = "ecap-capture", + .of_match_table = ecap_cnt_of_match, + .pm = pm_sleep_ptr(&ecap_cnt_pm_ops), + }, +}; +module_platform_driver(ecap_cnt_driver); + +MODULE_DESCRIPTION("ECAP Capture driver"); +MODULE_AUTHOR("Julien Panis "); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/counter/ti-eqep.c b/drivers/counter/ti-eqep.c index 0489d26eb47c7b9b1de013ace96adab2a597049c..b0f24cf3e891dedd8295bc43c79e4f1403e1bd72 100644 --- a/drivers/counter/ti-eqep.c +++ b/drivers/counter/ti-eqep.c @@ -456,3 +456,4 @@ module_platform_driver(ti_eqep_driver); MODULE_AUTHOR("David Lechner "); MODULE_DESCRIPTION("TI eQEP counter driver"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(COUNTER); diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index 954749afb5fe9cbb83848223a5de4ec905c22378..82e5de1f6f8c959f7519bd53a3bdf4dd000a74db 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -153,7 +153,7 @@ config ARM_OMAP2PLUS_CPUFREQ config ARM_QCOM_CPUFREQ_NVMEM tristate "Qualcomm nvmem based CPUFreq" depends on ARCH_QCOM - depends on QCOM_QFPROM + depends on NVMEM_QCOM_QFPROM depends on QCOM_SMEM select PM_OPP help diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 55516043b656bd0d43cd506e7a09424c78de00dd..310779b07daf131015db699d1db232684d25b7f8 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -51,6 +51,21 @@ config X86_AMD_PSTATE If in doubt, say N. +config X86_AMD_PSTATE_UT + tristate "selftest for AMD Processor P-State driver" + depends on X86 && ACPI_PROCESSOR + default n + help + This kernel module is used for testing. It's safe to say M here. + + It can also be built-in without X86_AMD_PSTATE enabled. + Currently, only tests for amd-pstate are supported. If X86_AMD_PSTATE + is set disabled, it can tell the users test can only run on amd-pstate + driver, please set X86_AMD_PSTATE enabled. + In the future, comparison tests will be added. It can set amd-pstate + disabled and set acpi-cpufreq enabled to run test cases, then compare + the test results. + config X86_ACPI_CPUFREQ tristate "ACPI Processor P-States driver" depends on ACPI_PROCESSOR diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 285de70af87749781b4c717f7247c4ce981b4899..49b98c62c5af59cc6e6236dc6399ad1e28516361 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -30,6 +30,7 @@ amd_pstate-y := amd-pstate.o amd-pstate-trace.o obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o obj-$(CONFIG_X86_AMD_PSTATE) += amd_pstate.o +obj-$(CONFIG_X86_AMD_PSTATE_UT) += amd-pstate-ut.o obj-$(CONFIG_X86_POWERNOW_K8) += powernow-k8.o obj-$(CONFIG_X86_PCC_CPUFREQ) += pcc-cpufreq.o obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c new file mode 100644 index 0000000000000000000000000000000000000000..e4a5b4d90f833c0143382992da4187a23b310e81 --- /dev/null +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-1.0-or-later +/* + * AMD Processor P-state Frequency Driver Unit Test + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Author: Meng Li + * + * The AMD P-State Unit Test is a test module for testing the amd-pstate + * driver. 1) It can help all users to verify their processor support + * (SBIOS/Firmware or Hardware). 2) Kernel can have a basic function + * test to avoid the kernel regression during the update. 3) We can + * introduce more functional or performance tests to align the result + * together, it will benefit power and performance scale optimization. + * + * This driver implements basic framework with plans to enhance it with + * additional test cases to improve the depth and coverage of the test. + * + * See Documentation/admin-guide/pm/amd-pstate.rst Unit Tests for + * amd-pstate to get more detail. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include + +#include + +/* + * Abbreviations: + * amd_pstate_ut: used as a shortform for AMD P-State unit test. + * It helps to keep variable names smaller, simpler + */ +enum amd_pstate_ut_result { + AMD_PSTATE_UT_RESULT_PASS, + AMD_PSTATE_UT_RESULT_FAIL, +}; + +struct amd_pstate_ut_struct { + const char *name; + void (*func)(u32 index); + enum amd_pstate_ut_result result; +}; + +/* + * Kernel module for testing the AMD P-State unit test + */ +static void amd_pstate_ut_acpi_cpc_valid(u32 index); +static void amd_pstate_ut_check_enabled(u32 index); +static void amd_pstate_ut_check_perf(u32 index); +static void amd_pstate_ut_check_freq(u32 index); + +static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = { + {"amd_pstate_ut_acpi_cpc_valid", amd_pstate_ut_acpi_cpc_valid }, + {"amd_pstate_ut_check_enabled", amd_pstate_ut_check_enabled }, + {"amd_pstate_ut_check_perf", amd_pstate_ut_check_perf }, + {"amd_pstate_ut_check_freq", amd_pstate_ut_check_freq } +}; + +static bool get_shared_mem(void) +{ + bool result = false; + char path[] = "/sys/module/amd_pstate/parameters/shared_mem"; + char buf[5] = {0}; + struct file *filp = NULL; + loff_t pos = 0; + ssize_t ret; + + if (!boot_cpu_has(X86_FEATURE_CPPC)) { + filp = filp_open(path, O_RDONLY, 0); + if (IS_ERR(filp)) + pr_err("%s unable to open %s file!\n", __func__, path); + else { + ret = kernel_read(filp, &buf, sizeof(buf), &pos); + if (ret < 0) + pr_err("%s read %s file fail ret=%ld!\n", + __func__, path, (long)ret); + filp_close(filp, NULL); + } + + if ('Y' == *buf) + result = true; + } + + return result; +} + +/* + * check the _CPC object is present in SBIOS. + */ +static void amd_pstate_ut_acpi_cpc_valid(u32 index) +{ + if (acpi_cpc_valid()) + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + else { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s the _CPC object is not present in SBIOS!\n", __func__); + } +} + +static void amd_pstate_ut_pstate_enable(u32 index) +{ + int ret = 0; + u64 cppc_enable = 0; + + ret = rdmsrl_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable); + if (ret) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s rdmsrl_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret); + return; + } + if (cppc_enable) + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + else { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s amd pstate must be enabled!\n", __func__); + } +} + +/* + * check if amd pstate is enabled + */ +static void amd_pstate_ut_check_enabled(u32 index) +{ + if (get_shared_mem()) + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + else + amd_pstate_ut_pstate_enable(index); +} + +/* + * check if performance values are reasonable. + * highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0 + */ +static void amd_pstate_ut_check_perf(u32 index) +{ + int cpu = 0, ret = 0; + u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0; + u64 cap1 = 0; + struct cppc_perf_caps cppc_perf; + struct cpufreq_policy *policy = NULL; + struct amd_cpudata *cpudata = NULL; + + highest_perf = amd_get_highest_perf(); + + for_each_possible_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) + break; + cpudata = policy->driver_data; + + if (get_shared_mem()) { + ret = cppc_get_perf_caps(cpu, &cppc_perf); + if (ret) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret); + return; + } + + nominal_perf = cppc_perf.nominal_perf; + lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; + lowest_perf = cppc_perf.lowest_perf; + } else { + ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); + if (ret) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret); + return; + } + + nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1); + lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1); + lowest_perf = AMD_CPPC_LOWEST_PERF(cap1); + } + + if ((highest_perf != READ_ONCE(cpudata->highest_perf)) || + (nominal_perf != READ_ONCE(cpudata->nominal_perf)) || + (lowest_nonlinear_perf != READ_ONCE(cpudata->lowest_nonlinear_perf)) || + (lowest_perf != READ_ONCE(cpudata->lowest_perf))) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d highest=%d %d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n", + __func__, cpu, highest_perf, cpudata->highest_perf, + nominal_perf, cpudata->nominal_perf, + lowest_nonlinear_perf, cpudata->lowest_nonlinear_perf, + lowest_perf, cpudata->lowest_perf); + return; + } + + if (!((highest_perf >= nominal_perf) && + (nominal_perf > lowest_nonlinear_perf) && + (lowest_nonlinear_perf > lowest_perf) && + (lowest_perf > 0))) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n", + __func__, cpu, highest_perf, nominal_perf, + lowest_nonlinear_perf, lowest_perf); + return; + } + } + + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; +} + +/* + * Check if frequency values are reasonable. + * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0 + * check max freq when set support boost mode. + */ +static void amd_pstate_ut_check_freq(u32 index) +{ + int cpu = 0; + struct cpufreq_policy *policy = NULL; + struct amd_cpudata *cpudata = NULL; + + for_each_possible_cpu(cpu) { + policy = cpufreq_cpu_get(cpu); + if (!policy) + break; + cpudata = policy->driver_data; + + if (!((cpudata->max_freq >= cpudata->nominal_freq) && + (cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) && + (cpudata->lowest_nonlinear_freq > cpudata->min_freq) && + (cpudata->min_freq > 0))) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n", + __func__, cpu, cpudata->max_freq, cpudata->nominal_freq, + cpudata->lowest_nonlinear_freq, cpudata->min_freq); + return; + } + + if (cpudata->min_freq != policy->min) { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d cpudata_min_freq=%d policy_min=%d, they should be equal!\n", + __func__, cpu, cpudata->min_freq, policy->min); + return; + } + + if (cpudata->boost_supported) { + if ((policy->max == cpudata->max_freq) || + (policy->max == cpudata->nominal_freq)) + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; + else { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n", + __func__, cpu, policy->max, cpudata->max_freq, + cpudata->nominal_freq); + return; + } + } else { + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL; + pr_err("%s cpu%d must support boost!\n", __func__, cpu); + return; + } + } + + amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS; +} + +static int __init amd_pstate_ut_init(void) +{ + u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); + + for (i = 0; i < arr_size; i++) { + amd_pstate_ut_cases[i].func(i); + switch (amd_pstate_ut_cases[i].result) { + case AMD_PSTATE_UT_RESULT_PASS: + pr_info("%-4d %-20s\t success!\n", i+1, amd_pstate_ut_cases[i].name); + break; + case AMD_PSTATE_UT_RESULT_FAIL: + default: + pr_info("%-4d %-20s\t fail!\n", i+1, amd_pstate_ut_cases[i].name); + break; + } + } + + return 0; +} + +static void __exit amd_pstate_ut_exit(void) +{ +} + +module_init(amd_pstate_ut_init); +module_exit(amd_pstate_ut_exit); + +MODULE_AUTHOR("Meng Li "); +MODULE_DESCRIPTION("AMD P-state driver Test module"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 9ac75c1cde9c22fc3bcee6885521477ea1352625..ace7d50cf2ac426d9d6e8969ff6ad5df54f2fecb 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -46,8 +47,8 @@ #include #include "amd-pstate-trace.h" -#define AMD_PSTATE_TRANSITION_LATENCY 0x20000 -#define AMD_PSTATE_TRANSITION_DELAY 500 +#define AMD_PSTATE_TRANSITION_LATENCY 20000 +#define AMD_PSTATE_TRANSITION_DELAY 1000 /* * TODO: We need more time to fine tune processors with shared memory solution @@ -65,65 +66,6 @@ MODULE_PARM_DESC(shared_mem, static struct cpufreq_driver amd_pstate_driver; -/** - * struct amd_aperf_mperf - * @aperf: actual performance frequency clock count - * @mperf: maximum performance frequency clock count - * @tsc: time stamp counter - */ -struct amd_aperf_mperf { - u64 aperf; - u64 mperf; - u64 tsc; -}; - -/** - * struct amd_cpudata - private CPU data for AMD P-State - * @cpu: CPU number - * @req: constraint request to apply - * @cppc_req_cached: cached performance request hints - * @highest_perf: the maximum performance an individual processor may reach, - * assuming ideal conditions - * @nominal_perf: the maximum sustained performance level of the processor, - * assuming ideal operating conditions - * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power - * savings are achieved - * @lowest_perf: the absolute lowest performance level of the processor - * @max_freq: the frequency that mapped to highest_perf - * @min_freq: the frequency that mapped to lowest_perf - * @nominal_freq: the frequency that mapped to nominal_perf - * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf - * @cur: Difference of Aperf/Mperf/tsc count between last and current sample - * @prev: Last Aperf/Mperf/tsc count value read from register - * @freq: current cpu frequency value - * @boost_supported: check whether the Processor or SBIOS supports boost mode - * - * The amd_cpudata is key private data for each CPU thread in AMD P-State, and - * represents all the attributes and goals that AMD P-State requests at runtime. - */ -struct amd_cpudata { - int cpu; - - struct freq_qos_request req[2]; - u64 cppc_req_cached; - - u32 highest_perf; - u32 nominal_perf; - u32 lowest_nonlinear_perf; - u32 lowest_perf; - - u32 max_freq; - u32 min_freq; - u32 nominal_freq; - u32 lowest_nonlinear_freq; - - struct amd_aperf_mperf cur; - struct amd_aperf_mperf prev; - - u64 freq; - bool boost_supported; -}; - static inline int pstate_enable(bool enable) { return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); @@ -152,6 +94,7 @@ static inline int amd_pstate_enable(bool enable) static int pstate_init_perf(struct amd_cpudata *cpudata) { u64 cap1; + u32 highest_perf; int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &cap1); @@ -163,7 +106,11 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) * * CPPC entry doesn't indicate the highest performance in some ASICs. */ - WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); + highest_perf = amd_get_highest_perf(); + if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) + highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); + + WRITE_ONCE(cpudata->highest_perf, highest_perf); WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); @@ -175,12 +122,17 @@ static int pstate_init_perf(struct amd_cpudata *cpudata) static int cppc_init_perf(struct amd_cpudata *cpudata) { struct cppc_perf_caps cppc_perf; + u32 highest_perf; int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); if (ret) return ret; - WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf()); + highest_perf = amd_get_highest_perf(); + if (highest_perf > cppc_perf.highest_perf) + highest_perf = cppc_perf.highest_perf; + + WRITE_ONCE(cpudata->highest_perf, highest_perf); WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); WRITE_ONCE(cpudata->lowest_nonlinear_perf, @@ -269,6 +221,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, u64 prev = READ_ONCE(cpudata->cppc_req_cached); u64 value = prev; + des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); value &= ~AMD_CPPC_MIN_PERF(~0L); value |= AMD_CPPC_MIN_PERF(min_perf); @@ -312,7 +265,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy, return -ENODEV; cap_perf = READ_ONCE(cpudata->highest_perf); - min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); + min_perf = READ_ONCE(cpudata->lowest_perf); max_perf = cap_perf; freqs.old = policy->cur; @@ -357,8 +310,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu, if (max_perf < min_perf) max_perf = min_perf; - des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); - amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); } @@ -555,9 +506,7 @@ free_cpudata1: static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) { - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; freq_qos_remove_request(&cpudata->req[1]); freq_qos_remove_request(&cpudata->req[0]); @@ -599,9 +548,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, char *buf) { int max_freq; - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; max_freq = amd_get_max_freq(cpudata); if (max_freq < 0) @@ -614,9 +561,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli char *buf) { int freq; - struct amd_cpudata *cpudata; - - cpudata = policy->driver_data; + struct amd_cpudata *cpudata = policy->driver_data; freq = amd_get_lowest_nonlinear_freq(cpudata); if (freq < 0) @@ -662,7 +607,7 @@ static struct cpufreq_driver amd_pstate_driver = { .resume = amd_pstate_cpu_resume, .set_boost = amd_pstate_set_boost, .name = "amd-pstate", - .attr = amd_pstate_attr, + .attr = amd_pstate_attr, }; static int __init amd_pstate_init(void) @@ -673,7 +618,7 @@ static int __init amd_pstate_init(void) return -ENODEV; if (!acpi_cpc_valid()) { - pr_debug("the _CPC object is not present in SBIOS\n"); + pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); return -ENODEV; } diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c index f7c23fa468f0b89246b23af3b8263152cfaf235d..39221a9a187a79d2cb3c25e7bb6c0c8d39fa88e1 100644 --- a/drivers/cpufreq/bmips-cpufreq.c +++ b/drivers/cpufreq/bmips-cpufreq.c @@ -156,7 +156,7 @@ static struct cpufreq_driver bmips_cpufreq_driver = { .name = BMIPS_CPUFREQ_PREFIX, }; -static int __init bmips_cpufreq_probe(void) +static int __init bmips_cpufreq_driver_init(void) { struct cpufreq_compat *cc; struct device_node *np; @@ -176,7 +176,13 @@ static int __init bmips_cpufreq_probe(void) return cpufreq_register_driver(&bmips_cpufreq_driver); } -device_initcall(bmips_cpufreq_probe); +module_init(bmips_cpufreq_driver_init); + +static void __exit bmips_cpufreq_driver_exit(void) +{ + cpufreq_unregister_driver(&bmips_cpufreq_driver); +} +module_exit(bmips_cpufreq_driver_exit); MODULE_AUTHOR("Markus Mayer "); MODULE_DESCRIPTION("CPUfreq driver for Broadcom BMIPS SoCs"); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 24eaf0ec344d55027beda78e77a7436db7dc4b95..432dfb4e8027e96b908c420cd40f0d0eb911bfef 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -63,7 +63,15 @@ static struct cppc_workaround_oem_info wa_info[] = { static struct cpufreq_driver cppc_cpufreq_driver; +static enum { + FIE_UNSET = -1, + FIE_ENABLED, + FIE_DISABLED +} fie_disabled = FIE_UNSET; + #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE +module_param(fie_disabled, int, 0444); +MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); /* Frequency invariance support */ struct cppc_freq_invariance { @@ -158,7 +166,7 @@ static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) struct cppc_freq_invariance *cppc_fi; int cpu, ret; - if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) + if (fie_disabled) return; for_each_cpu(cpu, policy->cpus) { @@ -199,7 +207,7 @@ static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) struct cppc_freq_invariance *cppc_fi; int cpu; - if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) + if (fie_disabled) return; /* policy->cpus will be empty here, use related_cpus instead */ @@ -229,7 +237,15 @@ static void __init cppc_freq_invariance_init(void) }; int ret; - if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) + if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) { + fie_disabled = FIE_ENABLED; + if (cppc_perf_ctrs_in_pcc()) { + pr_info("FIE not enabled on systems with registers in PCC\n"); + fie_disabled = FIE_DISABLED; + } + } + + if (fie_disabled) return; kworker_fie = kthread_create_worker(0, "cppc_fie"); @@ -247,7 +263,7 @@ static void __init cppc_freq_invariance_init(void) static void cppc_freq_invariance_exit(void) { - if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate) + if (fie_disabled) return; kthread_destroy_worker(kworker_fie); @@ -936,6 +952,7 @@ static void cppc_check_hisi_workaround(void) wa_info[i].oem_revision == tbl->oem_revision) { /* Overwrite the get() callback */ cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate; + fie_disabled = FIE_DISABLED; break; } } @@ -947,7 +964,7 @@ static int __init cppc_cpufreq_init(void) { int ret; - if ((acpi_disabled) || !acpi_cpc_valid()) + if (!acpi_cpc_valid()) return -ENODEV; cppc_check_hisi_workaround(); diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 2c96de3f2d83ccd1ccc57839f9c447513ba54da7..6ac3800db4508229bf2d102b0449ae852d503b7c 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -146,6 +146,7 @@ static const struct of_device_id blocklist[] __initconst = { { .compatible = "qcom,sc8180x", }, { .compatible = "qcom,sc8280xp", }, { .compatible = "qcom,sdm845", }, + { .compatible = "qcom,sm6115", }, { .compatible = "qcom,sm6350", }, { .compatible = "qcom,sm8150", }, { .compatible = "qcom,sm8250", }, diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7820c4e7428934ddd9a31e4fa5e9117cb31574f2..69b3d61852ac61268453f7144a74b552943444ee 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -532,7 +532,7 @@ static unsigned int __resolve_freq(struct cpufreq_policy *policy, target_freq = clamp_val(target_freq, policy->min, policy->max); - if (!cpufreq_driver->target_index) + if (!policy->freq_table) return target_freq; idx = cpufreq_frequency_table_target(policy, target_freq, relation); diff --git a/drivers/cpufreq/highbank-cpufreq.c b/drivers/cpufreq/highbank-cpufreq.c index ac57cddc5f2fe5f057942bbe2e146f6ac6e226c6..a458647011434a35a9d1b64e91c4152a7ec42f88 100644 --- a/drivers/cpufreq/highbank-cpufreq.c +++ b/drivers/cpufreq/highbank-cpufreq.c @@ -55,7 +55,7 @@ static struct notifier_block hb_cpufreq_clk_nb = { .notifier_call = hb_cpufreq_clk_notify, }; -static int hb_cpufreq_driver_init(void) +static int __init hb_cpufreq_driver_init(void) { struct platform_device_info devinfo = { .name = "cpufreq-dt", }; struct device *cpu_dev; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 57cdb36798854a941d1258b611c6e57bceddafcb..fc3ebeb0bbe59ae7c0d20a158540268490f796d4 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2416,6 +2416,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = { X86_MATCH(SKYLAKE_X, core_funcs), X86_MATCH(COMETLAKE, core_funcs), X86_MATCH(ICELAKE_X, core_funcs), + X86_MATCH(TIGERLAKE, core_funcs), {} }; MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids); diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index d5ef3c66c762587afea6f41c46a652747bcecf2a..833589bc95e40d5e749a17652d310a8120d985f4 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,8 @@ struct qcom_cpufreq_data { struct cpufreq_policy *policy; bool per_core_dcvs; + + struct freq_qos_request throttle_freq_req; }; static unsigned long cpu_hw_rate, xo_rate; @@ -316,14 +319,16 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) if (IS_ERR(opp)) { dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); } else { - throttled_freq = freq_hz / HZ_PER_KHZ; - - /* Update thermal pressure (the boost frequencies are accepted) */ - arch_update_thermal_pressure(policy->related_cpus, throttled_freq); - dev_pm_opp_put(opp); } + throttled_freq = freq_hz / HZ_PER_KHZ; + + freq_qos_update_request(&data->throttle_freq_req, throttled_freq); + + /* Update thermal pressure (the boost frequencies are accepted) */ + arch_update_thermal_pressure(policy->related_cpus, throttled_freq); + /* * In the unlikely case policy is unregistered do not enable * polling or h/w interrupt @@ -413,6 +418,14 @@ static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) if (data->throttle_irq < 0) return data->throttle_irq; + ret = freq_qos_add_request(&policy->constraints, + &data->throttle_freq_req, FREQ_QOS_MAX, + FREQ_QOS_MAX_DEFAULT_VALUE); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to add freq constraint (%d)\n", ret); + return ret; + } + data->cancel_throttle = false; data->policy = policy; @@ -479,6 +492,7 @@ static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) if (data->throttle_irq <= 0) return; + freq_qos_remove_request(&data->throttle_freq_req); free_irq(data->throttle_irq, data); } diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c index a67df90848c2e45bf65258a19f4643e37f031578..1a63aeea87112c2d856b16b13aa9a7ddaec1b762 100644 --- a/drivers/cpufreq/sti-cpufreq.c +++ b/drivers/cpufreq/sti-cpufreq.c @@ -252,7 +252,7 @@ static int sti_cpufreq_fetch_syscon_registers(void) return 0; } -static int sti_cpufreq_init(void) +static int __init sti_cpufreq_init(void) { int ret; diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 1216046cf4c2ee2dd1ff2965f673d9399c52300b..c2004cae3f021dc881f618f87c091b7c0551973f 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -38,14 +38,6 @@ /* cpufreq transisition latency */ #define TEGRA_CPUFREQ_TRANSITION_LATENCY (300 * 1000) /* unit in nanoseconds */ -enum cluster { - CLUSTER0, - CLUSTER1, - CLUSTER2, - CLUSTER3, - MAX_CLUSTERS, -}; - struct tegra_cpu_ctr { u32 cpu; u32 coreclk_cnt, last_coreclk_cnt; @@ -67,12 +59,12 @@ struct tegra_cpufreq_ops { struct tegra_cpufreq_soc { struct tegra_cpufreq_ops *ops; int maxcpus_per_cluster; + unsigned int num_clusters; phys_addr_t actmon_cntr_base; }; struct tegra194_cpufreq_data { void __iomem *regs; - size_t num_clusters; struct cpufreq_frequency_table **tables; const struct tegra_cpufreq_soc *soc; }; @@ -166,6 +158,14 @@ static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = { .ops = &tegra234_cpufreq_ops, .actmon_cntr_base = 0x9000, .maxcpus_per_cluster = 4, + .num_clusters = 3, +}; + +static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = { + .ops = &tegra234_cpufreq_ops, + .actmon_cntr_base = 0x4000, + .maxcpus_per_cluster = 8, + .num_clusters = 1, }; static void tegra194_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid) @@ -314,11 +314,7 @@ static void tegra194_get_cpu_ndiv_sysreg(void *ndiv) static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv) { - int ret; - - ret = smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); - - return ret; + return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true); } static void tegra194_set_cpu_ndiv_sysreg(void *data) @@ -382,7 +378,7 @@ static int tegra194_cpufreq_init(struct cpufreq_policy *policy) data->soc->ops->get_cpu_cluster_id(policy->cpu, NULL, &clusterid); - if (clusterid >= data->num_clusters || !data->tables[clusterid]) + if (clusterid >= data->soc->num_clusters || !data->tables[clusterid]) return -EINVAL; start_cpu = rounddown(policy->cpu, maxcpus_per_cluster); @@ -433,6 +429,7 @@ static struct tegra_cpufreq_ops tegra194_cpufreq_ops = { static const struct tegra_cpufreq_soc tegra194_cpufreq_soc = { .ops = &tegra194_cpufreq_ops, .maxcpus_per_cluster = 2, + .num_clusters = 4, }; static void tegra194_cpufreq_free_resources(void) @@ -525,15 +522,14 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev) soc = of_device_get_match_data(&pdev->dev); - if (soc->ops && soc->maxcpus_per_cluster) { + if (soc->ops && soc->maxcpus_per_cluster && soc->num_clusters) { data->soc = soc; } else { dev_err(&pdev->dev, "soc data missing\n"); return -EINVAL; } - data->num_clusters = MAX_CLUSTERS; - data->tables = devm_kcalloc(&pdev->dev, data->num_clusters, + data->tables = devm_kcalloc(&pdev->dev, data->soc->num_clusters, sizeof(*data->tables), GFP_KERNEL); if (!data->tables) return -ENOMEM; @@ -558,7 +554,7 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev) goto put_bpmp; } - for (i = 0; i < data->num_clusters; i++) { + for (i = 0; i < data->soc->num_clusters; i++) { data->tables[i] = init_freq_table(pdev, bpmp, i); if (IS_ERR(data->tables[i])) { err = PTR_ERR(data->tables[i]); @@ -590,6 +586,7 @@ static int tegra194_cpufreq_remove(struct platform_device *pdev) static const struct of_device_id tegra194_cpufreq_of_match[] = { { .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc }, { .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc }, + { .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc }, { /* sentinel */ } }; diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c index df85a77d476b4b0c9e7d6d52d3876cbdf5a95bd7..f64180dd2005b30416d20e7586fa65e9b9b8746a 100644 --- a/drivers/cpufreq/ti-cpufreq.c +++ b/drivers/cpufreq/ti-cpufreq.c @@ -398,7 +398,7 @@ fail_put_node: return ret; } -static int ti_cpufreq_init(void) +static int __init ti_cpufreq_init(void) { const struct of_device_id *match; diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c index 74068742cef361f4ada8e4259ce5e84961665e6f..9acde71558d5101a1d98daeb8e16c185ff8ca98b 100644 --- a/drivers/cpuidle/coupled.c +++ b/drivers/cpuidle/coupled.c @@ -54,7 +54,7 @@ * variable is not locked. It is only written from the cpu that * it stores (or by the on/offlining cpu if that cpu is offline), * and only read after all the cpus are ready for the coupled idle - * state are are no longer updating it. + * state are no longer updating it. * * Three atomic counters are used. alive_count tracks the number * of cpus in the coupled set that are currently or soon will be diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index c32c600b3cf87c24be0002511ac955d41581efbc..0b5461b3d7dd4236b59d78ec801cf91da2b3e6e0 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -233,8 +233,8 @@ static inline void add_powernv_state(int index, const char *name, unsigned int exit_latency, u64 psscr_val, u64 psscr_mask) { - strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN); - strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN); + strscpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN); + strscpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN); powernv_states[index].flags = flags; powernv_states[index].target_residency = target_residency; powernv_states[index].exit_latency = exit_latency; diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index 3db4fca1172b4355877426a8a5f1b4aa95f942b7..821984947ed9bff93ebd169307af64a49aea7515 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -124,10 +124,8 @@ static bool psci_pd_try_set_osi_mode(void) return false; ret = psci_set_osi_mode(true); - if (ret) { - pr_warn("failed to enable OSI mode: %d\n", ret); + if (ret) return false; - } return true; } diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c index 862a2876f1c9df7483a3d33639d7f5ce342d2198..05fe2902df9a7858bf6ba18d8b71c97c0a112fb9 100644 --- a/drivers/cpuidle/cpuidle-riscv-sbi.c +++ b/drivers/cpuidle/cpuidle-riscv-sbi.c @@ -97,8 +97,13 @@ static int sbi_cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, int idx) { u32 *states = __this_cpu_read(sbi_cpuidle_data.states); + u32 state = states[idx]; - return CPU_PM_CPU_IDLE_ENTER_PARAM(sbi_suspend, idx, states[idx]); + if (state & SBI_HSM_SUSP_NON_RET_BIT) + return CPU_PM_CPU_IDLE_ENTER_PARAM(sbi_suspend, idx, state); + else + return CPU_PM_CPU_IDLE_ENTER_RETENTION_PARAM(sbi_suspend, + idx, state); } static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index 29acaf48e575818933c0176f576e18adefc0e5d3..0d0f9751ff8fabf400dae1779e2c8cf79285873f 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -63,12 +63,11 @@ int cpuidle_switch_governor(struct cpuidle_governor *gov) cpuidle_curr_governor = gov; - if (gov) { - list_for_each_entry(dev, &cpuidle_detected_devices, device_list) - cpuidle_enable_device(dev); - cpuidle_install_idle_handler(); - printk(KERN_INFO "cpuidle: using governor %s\n", gov->name); - } + list_for_each_entry(dev, &cpuidle_detected_devices, device_list) + cpuidle_enable_device(dev); + + cpuidle_install_idle_handler(); + pr_info("cpuidle: using governor %s\n", gov->name); return 0; } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 3e6aa319920b708375b56eb4210ab4bc54b078cb..55e75fbb658ee1b5fd46a10fef0dde1e6dd3125d 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -802,9 +802,7 @@ source "drivers/crypto/amlogic/Kconfig" config CRYPTO_DEV_SA2UL tristate "Support for TI security accelerator" depends on ARCH_K3 || COMPILE_TEST - select ARM64_CRYPTO select CRYPTO_AES - select CRYPTO_AES_ARM64 select CRYPTO_ALGAPI select CRYPTO_AUTHENC select CRYPTO_SHA1 @@ -818,5 +816,6 @@ config CRYPTO_DEV_SA2UL acceleration for cryptographic algorithms on these devices. source "drivers/crypto/keembay/Kconfig" +source "drivers/crypto/aspeed/Kconfig" endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index f81703a86b98e0acd47c8b65dc1f147a1ca91624..116de173a66c595cfb536af3c83a3f8b492146da 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_ALLWINNER) += allwinner/ +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed/ obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 44b8fc4b786dc8c724373c415d3e8ecc409e1753..006e40133c284ba252727a8b25652b0877a2aaed 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -235,7 +235,7 @@ static struct sun4i_ss_alg_template ss_algs[] = { #endif }; -static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) +static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v) { unsigned int i; @@ -266,19 +266,7 @@ static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) } return 0; } - -static int sun4i_ss_dbgfs_open(struct inode *inode, struct file *file) -{ - return single_open(file, sun4i_ss_dbgfs_read, inode->i_private); -} - -static const struct file_operations sun4i_ss_debugfs_fops = { - .owner = THIS_MODULE, - .open = sun4i_ss_dbgfs_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(sun4i_ss_debugfs); /* * Power management strategy: The device is suspended unless a TFM exists for diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c index 19cd2e52f89d40076bde59538e678d127bbfdf28..c4b0a8b588429b7f62fdfb37d2710c75ace7d9e6 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-trng.c @@ -54,11 +54,9 @@ static int sun8i_ce_trng_read(struct hwrng *rng, void *data, size_t max, bool wa goto err_dst; } - err = pm_runtime_get_sync(ce->dev); - if (err < 0) { - pm_runtime_put_noidle(ce->dev); + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) goto err_pm; - } mutex_lock(&ce->rnglock); chan = &ce->chanlist[flow]; diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c index e79514fce731fc900e3525e0a50351d1543d14ee..af017a087ebf08694e2e289522da995b6f0aeb20 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -177,7 +177,7 @@ static int meson_cipher(struct skcipher_request *areq) if (areq->src == areq->dst) { nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); - if (nr_sgs < 0) { + if (!nr_sgs) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); err = -EINVAL; goto theend; @@ -186,14 +186,14 @@ static int meson_cipher(struct skcipher_request *areq) } else { nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); - if (nr_sgs < 0 || nr_sgs > MAXDESC - 3) { + if (!nr_sgs || nr_sgs > MAXDESC - 3) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgs); err = -EINVAL; goto theend; } nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); - if (nr_sgd < 0 || nr_sgd > MAXDESC - 3) { + if (!nr_sgd || nr_sgd > MAXDESC - 3) { dev_err(mc->dev, "Invalid SG count %d\n", nr_sgd); err = -EINVAL; goto theend; diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..ae2710ae8d8f9e1aa3adc9dc6a2523cbe0076ee8 --- /dev/null +++ b/drivers/crypto/aspeed/Kconfig @@ -0,0 +1,48 @@ +config CRYPTO_DEV_ASPEED + tristate "Support for Aspeed cryptographic engine driver" + depends on ARCH_ASPEED || COMPILE_TEST + select CRYPTO_ENGINE + help + Hash and Crypto Engine (HACE) is designed to accelerate the + throughput of hash data digest, encryption and decryption. + + Select y here to have support for the cryptographic driver + available on Aspeed SoC. + +config CRYPTO_DEV_ASPEED_DEBUG + bool "Enable Aspeed crypto debug messages" + depends on CRYPTO_DEV_ASPEED + help + Print Aspeed crypto debugging messages if you use this + option to ask for those messages. + Avoid enabling this option for production build to + minimize driver timing. + +config CRYPTO_DEV_ASPEED_HACE_HASH + bool "Enable Aspeed Hash & Crypto Engine (HACE) hash" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select CRYPTO_HMAC + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + hash driver. + Supports multiple message digest standards, including + SHA-1, SHA-224, SHA-256, SHA-384, SHA-512, and so on. + +config CRYPTO_DEV_ASPEED_HACE_CRYPTO + bool "Enable Aspeed Hash & Crypto Engine (HACE) crypto" + depends on CRYPTO_DEV_ASPEED + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CFB + select CRYPTO_OFB + select CRYPTO_CTR + help + Select here to enable Aspeed Hash & Crypto Engine (HACE) + crypto driver. + Supports AES/DES symmetric-key encryption and decryption + with ECB/CBC/CFB/OFB/CTR options. diff --git a/drivers/crypto/aspeed/Makefile b/drivers/crypto/aspeed/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a0ed40ddaad120dd461568abba5e5200c8de3908 --- /dev/null +++ b/drivers/crypto/aspeed/Makefile @@ -0,0 +1,7 @@ +hace-hash-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH) := aspeed-hace-hash.o +hace-crypto-$(CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO) := aspeed-hace-crypto.o + +obj-$(CONFIG_CRYPTO_DEV_ASPEED) += aspeed_crypto.o +aspeed_crypto-objs := aspeed-hace.o \ + $(hace-hash-y) \ + $(hace-crypto-y) diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c new file mode 100644 index 0000000000000000000000000000000000000000..ef73b0028b4d773d0834ba0bf65ea7bfb499cd74 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -0,0 +1,1133 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO_DEBUG +#define CIPHER_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define CIPHER_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +static int aspeed_crypto_do_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + + if (rctx->enc_cmd & HACE_CMD_ENCRYPT) + err = crypto_skcipher_encrypt(&rctx->fallback_req); + else + err = crypto_skcipher_decrypt(&rctx->fallback_req); + + return err; +} + +static bool aspeed_crypto_need_fallback(struct skcipher_request *areq) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(areq); + + if (areq->cryptlen == 0) + return true; + + if ((rctx->enc_cmd & HACE_CMD_DES_SELECT) && + !IS_ALIGNED(areq->cryptlen, DES_BLOCK_SIZE)) + return true; + + if ((!(rctx->enc_cmd & HACE_CMD_DES_SELECT)) && + !IS_ALIGNED(areq->cryptlen, AES_BLOCK_SIZE)) + return true; + + return false; +} + +static int aspeed_hace_crypto_handle_queue(struct aspeed_hace_dev *hace_dev, + struct skcipher_request *req) +{ + if (hace_dev->version == AST2500_VERSION && + aspeed_crypto_need_fallback(req)) { + CIPHER_DBG(hace_dev, "SW fallback\n"); + return aspeed_crypto_do_fallback(req); + } + + return crypto_transfer_skcipher_request_to_engine( + hace_dev->crypt_engine_crypto, req); +} + +static int aspeed_crypto_do_request(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = skcipher_request_cast(areq); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct aspeed_engine_crypto *crypto_engine; + int rc; + + crypto_engine = &hace_dev->crypto_engine; + crypto_engine->req = req; + crypto_engine->flags |= CRYPTO_FLAGS_BUSY; + + rc = ctx->start(hace_dev); + + if (rc != -EINPROGRESS) + return -EIO; + + return 0; +} + +static int aspeed_sk_complete(struct aspeed_hace_dev *hace_dev, int err) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(req->iv, crypto_engine->cipher_ctx + + DES_KEY_SIZE, DES_KEY_SIZE); + else + memcpy(req->iv, crypto_engine->cipher_ctx, + AES_BLOCK_SIZE); + } + + crypto_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_skcipher_request(hace_dev->crypt_engine_crypto, req, + err); + + return err; +} + +static int aspeed_sk_transfer_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct device *dev = hace_dev->dev; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, rctx->dst_nents, DMA_FROM_DEVICE); + } + + return aspeed_sk_complete(hace_dev, 0); +} + +static int aspeed_sk_transfer(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *out_sg; + int nbytes = 0; + int rc = 0; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + out_sg = req->dst; + + /* Copy output buffer to dst scatter-gather lists */ + nbytes = sg_copy_from_buffer(out_sg, rctx->dst_nents, + crypto_engine->cipher_addr, req->cryptlen); + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + rc = -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_out_sg", rctx->dst_nents, + "cipher addr", crypto_engine->cipher_addr); + + return aspeed_sk_complete(hace_dev, rc); +} + +static int aspeed_sk_start(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *in_sg; + int nbytes; + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + in_sg = req->src; + + nbytes = sg_copy_to_buffer(in_sg, rctx->src_nents, + crypto_engine->cipher_addr, req->cryptlen); + + CIPHER_DBG(hace_dev, "%s:%d, %s:%d, %s:%d, %s:%p\n", + "nbytes", nbytes, "req->cryptlen", req->cryptlen, + "nb_in_sg", rctx->src_nents, + "cipher addr", crypto_engine->cipher_addr); + + if (!nbytes) { + dev_warn(hace_dev->dev, "invalid sg copy, %s:0x%x, %s:0x%x\n", + "nbytes", nbytes, "cryptlen", req->cryptlen); + return -EINVAL; + } + + crypto_engine->resume = aspeed_sk_transfer; + + /* Trigger engines */ + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_SRC); + ast_hace_write(hace_dev, crypto_engine->cipher_dma_addr, + ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; +} + +static int aspeed_sk_start_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_sg_list *src_list, *dst_list; + dma_addr_t src_dma_addr, dst_dma_addr; + struct aspeed_cipher_reqctx *rctx; + struct skcipher_request *req; + struct scatterlist *s; + int src_sg_len; + int dst_sg_len; + int total, i; + int rc; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + + rctx->enc_cmd |= HACE_CMD_DES_SG_CTRL | HACE_CMD_SRC_SG_CTRL | + HACE_CMD_AES_KEY_HW_EXP | HACE_CMD_MBUS_REQ_SYNC_EN; + + /* BIDIRECTIONAL */ + if (req->dst == req->src) { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_BIDIRECTIONAL); + dst_sg_len = src_sg_len; + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + } else { + src_sg_len = dma_map_sg(hace_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (!src_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + return -EINVAL; + } + + dst_sg_len = dma_map_sg(hace_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (!dst_sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() dst error\n"); + rc = -EINVAL; + goto free_req_src; + } + } + + src_list = (struct aspeed_sg_list *)crypto_engine->cipher_addr; + src_dma_addr = crypto_engine->cipher_dma_addr; + total = req->cryptlen; + + for_each_sg(req->src, s, src_sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (total > len) + total -= len; + else { + /* last sg list */ + len = total; + len |= BIT(31); + total = 0; + } + + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + if (req->dst == req->src) { + dst_list = src_list; + dst_dma_addr = src_dma_addr; + + } else { + dst_list = (struct aspeed_sg_list *)crypto_engine->dst_sg_addr; + dst_dma_addr = crypto_engine->dst_sg_dma_addr; + total = req->cryptlen; + + for_each_sg(req->dst, s, dst_sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (total > len) + total -= len; + else { + /* last sg list */ + len = total; + len |= BIT(31); + total = 0; + } + + dst_list[i].phy_addr = cpu_to_le32(phy_addr); + dst_list[i].len = cpu_to_le32(len); + + } + + dst_list[dst_sg_len].phy_addr = 0; + dst_list[dst_sg_len].len = 0; + } + + if (total != 0) { + rc = -EINVAL; + goto free_req; + } + + crypto_engine->resume = aspeed_sk_transfer_sg; + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + /* Trigger engines */ + ast_hace_write(hace_dev, src_dma_addr, ASPEED_HACE_SRC); + ast_hace_write(hace_dev, dst_dma_addr, ASPEED_HACE_DEST); + ast_hace_write(hace_dev, req->cryptlen, ASPEED_HACE_DATA_LEN); + ast_hace_write(hace_dev, rctx->enc_cmd, ASPEED_HACE_CMD); + + return -EINPROGRESS; + +free_req: + if (req->dst == req->src) { + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_BIDIRECTIONAL); + + } else { + dma_unmap_sg(hace_dev->dev, req->dst, rctx->dst_nents, + DMA_TO_DEVICE); + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, + DMA_TO_DEVICE); + } + + return rc; + +free_req_src: + dma_unmap_sg(hace_dev->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + + return rc; +} + +static int aspeed_hace_skcipher_trigger(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_cipher_reqctx *rctx; + struct crypto_skcipher *cipher; + struct aspeed_cipher_ctx *ctx; + struct skcipher_request *req; + + CIPHER_DBG(hace_dev, "\n"); + + req = crypto_engine->req; + rctx = skcipher_request_ctx(req); + cipher = crypto_skcipher_reqtfm(req); + ctx = crypto_skcipher_ctx(cipher); + + /* enable interrupt */ + rctx->enc_cmd |= HACE_CMD_ISR_EN; + + rctx->dst_nents = sg_nents(req->dst); + rctx->src_nents = sg_nents(req->src); + + ast_hace_write(hace_dev, crypto_engine->cipher_ctx_dma, + ASPEED_HACE_CONTEXT); + + if (rctx->enc_cmd & HACE_CMD_IV_REQUIRE) { + if (rctx->enc_cmd & HACE_CMD_DES_SELECT) + memcpy(crypto_engine->cipher_ctx + DES_BLOCK_SIZE, + req->iv, DES_BLOCK_SIZE); + else + memcpy(crypto_engine->cipher_ctx, req->iv, + AES_BLOCK_SIZE); + } + + if (hace_dev->version == AST2600_VERSION) { + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, ctx->key_len); + + return aspeed_sk_start_sg(hace_dev); + } + + memcpy(crypto_engine->cipher_ctx + 16, ctx->key, AES_MAX_KEYLENGTH); + + return aspeed_sk_start(hace_dev); +} + +static int aspeed_des_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + CIPHER_DBG(hace_dev, "\n"); + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) + return -EINVAL; + } + + rctx->enc_cmd = cmd | HACE_CMD_DES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_DES | HACE_CMD_CONTEXT_LOAD_ENABLE | + HACE_CMD_CONTEXT_SAVE_ENABLE; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_des_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + int rc; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", keylen); + + if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) { + dev_warn(hace_dev->dev, "invalid keylen: %d bits\n", keylen); + return -EINVAL; + } + + if (keylen == DES_KEY_SIZE) { + rc = crypto_des_verify_key(tfm, key); + if (rc) + return rc; + + } else if (keylen == DES3_EDE_KEY_SIZE) { + rc = crypto_des3_ede_verify_key(tfm, key); + if (rc) + return rc; + } + + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_tdes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_tdes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_TRIPLE_DES); +} + +static int aspeed_des_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_des_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB | + HACE_CMD_SINGLE_DES); +} + +static int aspeed_aes_crypt(struct skcipher_request *req, u32 cmd) +{ + struct aspeed_cipher_reqctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + u32 crypto_alg = cmd & HACE_CMD_OP_MODE_MASK; + + if (crypto_alg == HACE_CMD_CBC || crypto_alg == HACE_CMD_ECB) { + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + } + + CIPHER_DBG(hace_dev, "%s\n", + (cmd & HACE_CMD_ENCRYPT) ? "encrypt" : "decrypt"); + + cmd |= HACE_CMD_AES_SELECT | HACE_CMD_RI_WO_DATA_ENABLE | + HACE_CMD_CONTEXT_LOAD_ENABLE | HACE_CMD_CONTEXT_SAVE_ENABLE; + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + cmd |= HACE_CMD_AES128; + break; + case AES_KEYSIZE_192: + cmd |= HACE_CMD_AES192; + break; + case AES_KEYSIZE_256: + cmd |= HACE_CMD_AES256; + break; + default: + return -EINVAL; + } + + rctx->enc_cmd = cmd; + + return aspeed_hace_crypto_handle_queue(hace_dev, req); +} + +static int aspeed_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + struct crypto_aes_ctx gen_aes_key; + + CIPHER_DBG(hace_dev, "keylen: %d bits\n", (keylen * 8)); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + + if (ctx->hace_dev->version == AST2500_VERSION) { + aes_expandkey(&gen_aes_key, key, keylen); + memcpy(ctx->key, gen_aes_key.key_enc, AES_MAX_KEYLENGTH); + + } else { + memcpy(ctx->key, key, keylen); + } + + ctx->key_len = keylen; + + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, cipher->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen); +} + +static int aspeed_aes_ctr_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_ctr_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR); +} + +static int aspeed_aes_ofb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB); +} + +static int aspeed_aes_ofb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB); +} + +static int aspeed_aes_cfb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB); +} + +static int aspeed_aes_cfb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB); +} + +static int aspeed_aes_cbc_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_cbc_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CBC); +} + +static int aspeed_aes_ecb_decrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_ECB); +} + +static int aspeed_aes_ecb_encrypt(struct skcipher_request *req) +{ + return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_ECB); +} + +static int aspeed_crypto_cra_init(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + const char *name = crypto_tfm_alg_name(&tfm->base); + struct aspeed_hace_alg *crypto_alg; + + + crypto_alg = container_of(alg, struct aspeed_hace_alg, alg.skcipher); + ctx->hace_dev = crypto_alg->hace_dev; + ctx->start = aspeed_hace_skcipher_trigger; + + CIPHER_DBG(ctx->hace_dev, "%s\n", name); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + dev_err(ctx->hace_dev->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + crypto_skcipher_set_reqsize(tfm, sizeof(struct aspeed_cipher_reqctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); + + ctx->enginectx.op.do_one_request = aspeed_crypto_do_request; + ctx->enginectx.op.prepare_request = NULL; + ctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void aspeed_crypto_cra_exit(struct crypto_skcipher *tfm) +{ + struct aspeed_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct aspeed_hace_dev *hace_dev = ctx->hace_dev; + + CIPHER_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(&tfm->base)); + crypto_free_skcipher(ctx->fallback_tfm); +} + +static struct aspeed_hace_alg aspeed_crypto_algs[] = { + { + .alg.skcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ecb_encrypt, + .decrypt = aspeed_aes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "aspeed-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_cbc_encrypt, + .decrypt = aspeed_aes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "aspeed-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_cfb_encrypt, + .decrypt = aspeed_aes_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "aspeed-cfb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ofb_encrypt, + .decrypt = aspeed_aes_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(aes)", + .cra_driver_name = "aspeed-ofb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ecb_encrypt, + .decrypt = aspeed_des_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "aspeed-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_cbc_encrypt, + .decrypt = aspeed_des_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "aspeed-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_cfb_encrypt, + .decrypt = aspeed_des_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(des)", + .cra_driver_name = "aspeed-cfb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ofb_encrypt, + .decrypt = aspeed_des_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(des)", + .cra_driver_name = "aspeed-ofb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ecb_encrypt, + .decrypt = aspeed_tdes_ecb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "aspeed-ecb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_cbc_encrypt, + .decrypt = aspeed_tdes_cbc_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "aspeed-cbc-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_cfb_encrypt, + .decrypt = aspeed_tdes_cfb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "cfb(des3_ede)", + .cra_driver_name = "aspeed-cfb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ofb_encrypt, + .decrypt = aspeed_tdes_ofb_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ofb(des3_ede)", + .cra_driver_name = "aspeed-ofb-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, +}; + +static struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { + { + .alg.skcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aspeed_aes_setkey, + .encrypt = aspeed_aes_ctr_encrypt, + .decrypt = aspeed_aes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "aspeed-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_des_ctr_encrypt, + .decrypt = aspeed_des_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des)", + .cra_driver_name = "aspeed-ctr-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + { + .alg.skcipher = { + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = aspeed_des_setkey, + .encrypt = aspeed_tdes_ctr_encrypt, + .decrypt = aspeed_tdes_ctr_decrypt, + .init = aspeed_crypto_cra_init, + .exit = aspeed_crypto_cra_exit, + .base = { + .cra_name = "ctr(des3_ede)", + .cra_driver_name = "aspeed-ctr-tdes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_module = THIS_MODULE, + } + } + }, + +}; + +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) + crypto_unregister_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) + crypto_unregister_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); +} + +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + CIPHER_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs); i++) { + aspeed_crypto_algs[i].hace_dev = hace_dev; + rc = crypto_register_skcipher(&aspeed_crypto_algs[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs[i].alg.skcipher.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_crypto_algs_g6); i++) { + aspeed_crypto_algs_g6[i].hace_dev = hace_dev; + rc = crypto_register_skcipher(&aspeed_crypto_algs_g6[i].alg.skcipher); + if (rc) { + CIPHER_DBG(hace_dev, "Failed to register %s\n", + aspeed_crypto_algs_g6[i].alg.skcipher.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace-hash.c b/drivers/crypto/aspeed/aspeed-hace-hash.c new file mode 100644 index 0000000000000000000000000000000000000000..935135229ebd42c9828c72306ff1ff0f9541fd60 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace-hash.c @@ -0,0 +1,1391 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define AHASH_DBG(h, fmt, ...) \ + dev_info((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define AHASH_DBG(h, fmt, ...) \ + dev_dbg((h)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* Initialization Vectors for SHA-family */ +static const __be32 sha1_iv[8] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), 0, 0, 0 +}; + +static const __be32 sha224_iv[8] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 sha256_iv[8] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +static const __be64 sha384_iv[8] = { + cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1), + cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3), + cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5), + cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7) +}; + +static const __be64 sha512_iv[8] = { + cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1), + cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3), + cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5), + cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7) +}; + +static const __be32 sha512_224_iv[16] = { + cpu_to_be32(0xC8373D8CUL), cpu_to_be32(0xA24D5419UL), + cpu_to_be32(0x6699E173UL), cpu_to_be32(0xD6D4DC89UL), + cpu_to_be32(0xAEB7FA1DUL), cpu_to_be32(0x829CFF32UL), + cpu_to_be32(0x14D59D67UL), cpu_to_be32(0xCF9F2F58UL), + cpu_to_be32(0x692B6D0FUL), cpu_to_be32(0xA84DD47BUL), + cpu_to_be32(0x736FE377UL), cpu_to_be32(0x4289C404UL), + cpu_to_be32(0xA8859D3FUL), cpu_to_be32(0xC8361D6AUL), + cpu_to_be32(0xADE61211UL), cpu_to_be32(0xA192D691UL) +}; + +static const __be32 sha512_256_iv[16] = { + cpu_to_be32(0x94213122UL), cpu_to_be32(0x2CF72BFCUL), + cpu_to_be32(0xA35F559FUL), cpu_to_be32(0xC2644CC8UL), + cpu_to_be32(0x6BB89323UL), cpu_to_be32(0x51B1536FUL), + cpu_to_be32(0x19773896UL), cpu_to_be32(0xBDEA4059UL), + cpu_to_be32(0xE23E2896UL), cpu_to_be32(0xE3FF8EA8UL), + cpu_to_be32(0x251E5EBEUL), cpu_to_be32(0x92398653UL), + cpu_to_be32(0xFC99012BUL), cpu_to_be32(0xAAB8852CUL), + cpu_to_be32(0xDC2DB70EUL), cpu_to_be32(0xA22CC581UL) +}; + +/* The purpose of this padding is to ensure that the padded message is a + * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). + * The bit "1" is appended at the end of the message followed by + * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or + * 128 bits block (SHA384/SHA512) equals to the message length in bits + * is appended. + * + * For SHA1/SHA224/SHA256, padlen is calculated as followed: + * - if message length < 56 bytes then padlen = 56 - message length + * - else padlen = 64 + 56 - message length + * + * For SHA384/SHA512, padlen is calculated as followed: + * - if message length < 112 bytes then padlen = 112 - message length + * - else padlen = 128 + 112 - message length + */ +static void aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev, + struct aspeed_sham_reqctx *rctx) +{ + unsigned int index, padlen; + __be64 bits[2]; + + AHASH_DBG(hace_dev, "rctx flags:0x%x\n", (u32)rctx->flags); + + switch (rctx->flags & SHA_FLAGS_MASK) { + case SHA_FLAGS_SHA1: + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + bits[0] = cpu_to_be64(rctx->digcnt[0] << 3); + index = rctx->bufcnt & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); + *(rctx->buffer + rctx->bufcnt) = 0x80; + memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); + memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 8); + rctx->bufcnt += padlen + 8; + break; + default: + bits[1] = cpu_to_be64(rctx->digcnt[0] << 3); + bits[0] = cpu_to_be64(rctx->digcnt[1] << 3 | + rctx->digcnt[0] >> 61); + index = rctx->bufcnt & 0x7f; + padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); + *(rctx->buffer + rctx->bufcnt) = 0x80; + memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1); + memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 16); + rctx->bufcnt += padlen + 16; + break; + } +} + +/* + * Prepare DMA buffer before hardware engine + * processing. + */ +static int aspeed_ahash_dma_prepare(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + int length, remain; + + length = rctx->total + rctx->bufcnt; + remain = length % rctx->block_size; + + AHASH_DBG(hace_dev, "length:0x%x, remain:0x%x\n", length, remain); + + if (rctx->bufcnt) + memcpy(hash_engine->ahash_src_addr, rctx->buffer, rctx->bufcnt); + + if (rctx->total + rctx->bufcnt < ASPEED_CRYPTO_SRC_DMA_BUF_LEN) { + scatterwalk_map_and_copy(hash_engine->ahash_src_addr + + rctx->bufcnt, rctx->src_sg, + rctx->offset, rctx->total - remain, 0); + rctx->offset += rctx->total - remain; + + } else { + dev_warn(hace_dev->dev, "Hash data length is too large\n"); + return -EINVAL; + } + + scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, + rctx->offset, remain, 0); + + rctx->bufcnt = remain; + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + return -ENOMEM; + } + + hash_engine->src_length = length - remain; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; +} + +/* + * Prepare DMA buffer as SG list buffer before + * hardware engine processing. + */ +static int aspeed_ahash_dma_prepare_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct aspeed_sg_list *src_list; + struct scatterlist *s; + int length, remain, sg_len, i; + int rc = 0; + + remain = (rctx->total + rctx->bufcnt) % rctx->block_size; + length = rctx->total + rctx->bufcnt - remain; + + AHASH_DBG(hace_dev, "%s:0x%x, %s:%zu, %s:0x%x, %s:0x%x\n", + "rctx total", rctx->total, "bufcnt", rctx->bufcnt, + "length", length, "remain", remain); + + sg_len = dma_map_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + if (!sg_len) { + dev_warn(hace_dev->dev, "dma_map_sg() src error\n"); + rc = -ENOMEM; + goto end; + } + + src_list = (struct aspeed_sg_list *)hash_engine->ahash_src_addr; + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto free_src_sg; + } + + if (rctx->bufcnt != 0) { + u32 phy_addr; + u32 len; + + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + phy_addr = rctx->buffer_dma_addr; + len = rctx->bufcnt; + length -= len; + + /* Last sg list */ + if (length == 0) + len |= HASH_SG_LAST_LIST; + + src_list[0].phy_addr = cpu_to_le32(phy_addr); + src_list[0].len = cpu_to_le32(len); + src_list++; + } + + if (length != 0) { + for_each_sg(rctx->src_sg, s, sg_len, i) { + u32 phy_addr = sg_dma_address(s); + u32 len = sg_dma_len(s); + + if (length > len) + length -= len; + else { + /* Last sg list */ + len = length; + len |= HASH_SG_LAST_LIST; + length = 0; + } + + src_list[i].phy_addr = cpu_to_le32(phy_addr); + src_list[i].len = cpu_to_le32(len); + } + } + + if (length != 0) { + rc = -EINVAL; + goto free_rctx_buffer; + } + + rctx->offset = rctx->total - remain; + hash_engine->src_length = rctx->total + rctx->bufcnt - remain; + hash_engine->src_dma = hash_engine->ahash_src_dma_addr; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return 0; + +free_rctx_buffer: + if (rctx->bufcnt != 0) + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +free_src_sg: + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); +end: + return rc; +} + +static int aspeed_ahash_complete(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + + AHASH_DBG(hace_dev, "\n"); + + hash_engine->flags &= ~CRYPTO_FLAGS_BUSY; + + crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, 0); + + return 0; +} + +/* + * Copy digest to the corresponding request result. + * This function will be called at final() stage. + */ +static int aspeed_ahash_transfer(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); + + memcpy(req->result, rctx->digest, rctx->digsize); + + return aspeed_ahash_complete(hace_dev); +} + +/* + * Trigger hardware engines to do the math. + */ +static int aspeed_hace_ahash_trigger(struct aspeed_hace_dev *hace_dev, + aspeed_hace_fn_t resume) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "src_dma:%pad, digest_dma:%pad, length:%zu\n", + &hash_engine->src_dma, &hash_engine->digest_dma, + hash_engine->src_length); + + rctx->cmd |= HASH_CMD_INT_ENABLE; + hash_engine->resume = resume; + + ast_hace_write(hace_dev, hash_engine->src_dma, ASPEED_HACE_HASH_SRC); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_DIGEST_BUFF); + ast_hace_write(hace_dev, hash_engine->digest_dma, + ASPEED_HACE_HASH_KEY_BUFF); + ast_hace_write(hace_dev, hash_engine->src_length, + ASPEED_HACE_HASH_DATA_LEN); + + /* Memory barrier to ensure all data setup before engine starts */ + mb(); + + ast_hace_write(hace_dev, rctx->cmd, ASPEED_HACE_HASH_CMD); + + return -EINPROGRESS; +} + +/* + * HMAC resume aims to do the second pass produces + * the final HMAC code derived from the inner hash + * result and the outer key. + */ +static int aspeed_ahash_hmac_resume(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + int rc = 0; + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, DMA_TO_DEVICE); + + /* o key pad + hash sum 1 */ + memcpy(rctx->buffer, bctx->opad, rctx->block_size); + memcpy(rctx->buffer + rctx->block_size, rctx->digest, rctx->digsize); + + rctx->bufcnt = rctx->block_size + rctx->digsize; + rctx->digcnt[0] = rctx->block_size + rctx->digsize; + + aspeed_ahash_fill_padding(hace_dev, rctx); + memcpy(rctx->digest, rctx->sha_iv, rctx->ivsize); + + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto end; + } + + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + hash_engine->src_dma = rctx->buffer_dma_addr; + hash_engine->src_length = rctx->bufcnt; + hash_engine->digest_dma = rctx->digest_dma_addr; + + return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); + +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +end: + return rc; +} + +static int aspeed_ahash_req_final(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + int rc = 0; + + AHASH_DBG(hace_dev, "\n"); + + aspeed_ahash_fill_padding(hace_dev, rctx); + + rctx->digest_dma_addr = dma_map_single(hace_dev->dev, + rctx->digest, + SHA512_DIGEST_SIZE, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx digest error\n"); + rc = -ENOMEM; + goto end; + } + + rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, + rctx->buffer, + rctx->block_size * 2, + DMA_TO_DEVICE); + if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) { + dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n"); + rc = -ENOMEM; + goto free_rctx_digest; + } + + hash_engine->src_dma = rctx->buffer_dma_addr; + hash_engine->src_length = rctx->bufcnt; + hash_engine->digest_dma = rctx->digest_dma_addr; + + if (rctx->flags & SHA_FLAGS_HMAC) + return aspeed_hace_ahash_trigger(hace_dev, + aspeed_ahash_hmac_resume); + + return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer); + +free_rctx_digest: + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); +end: + return rc; +} + +static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents, + DMA_TO_DEVICE); + + if (rctx->bufcnt != 0) + dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr, + rctx->block_size * 2, + DMA_TO_DEVICE); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, rctx->offset, + rctx->total - rctx->offset, 0); + + rctx->bufcnt = rctx->total - rctx->offset; + rctx->cmd &= ~HASH_CMD_HASH_SRC_SG_CTRL; + + if (rctx->flags & SHA_FLAGS_FINUP) + return aspeed_ahash_req_final(hace_dev); + + return aspeed_ahash_complete(hace_dev); +} + +static int aspeed_ahash_update_resume(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + AHASH_DBG(hace_dev, "\n"); + + dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr, + SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + if (rctx->flags & SHA_FLAGS_FINUP) + return aspeed_ahash_req_final(hace_dev); + + return aspeed_ahash_complete(hace_dev); +} + +static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev) +{ + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + struct ahash_request *req = hash_engine->req; + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + aspeed_hace_fn_t resume; + int ret; + + AHASH_DBG(hace_dev, "\n"); + + if (hace_dev->version == AST2600_VERSION) { + rctx->cmd |= HASH_CMD_HASH_SRC_SG_CTRL; + resume = aspeed_ahash_update_resume_sg; + + } else { + resume = aspeed_ahash_update_resume; + } + + ret = hash_engine->dma_prepare(hace_dev); + if (ret) + return ret; + + return aspeed_hace_ahash_trigger(hace_dev, resume); +} + +static int aspeed_hace_hash_handle_queue(struct aspeed_hace_dev *hace_dev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine( + hace_dev->crypt_engine_hash, req); +} + +static int aspeed_ahash_do_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + int ret = 0; + + hash_engine = &hace_dev->hash_engine; + hash_engine->flags |= CRYPTO_FLAGS_BUSY; + + if (rctx->op == SHA_OP_UPDATE) + ret = aspeed_ahash_req_update(hace_dev); + else if (rctx->op == SHA_OP_FINAL) + ret = aspeed_ahash_req_final(hace_dev); + + if (ret != -EINPROGRESS) + return ret; + + return 0; +} + +static int aspeed_ahash_prepare_request(struct crypto_engine *engine, + void *areq) +{ + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_engine_hash *hash_engine; + + hash_engine = &hace_dev->hash_engine; + hash_engine->req = req; + + if (hace_dev->version == AST2600_VERSION) + hash_engine->dma_prepare = aspeed_ahash_dma_prepare_sg; + else + hash_engine->dma_prepare = aspeed_ahash_dma_prepare; + + return 0; +} + +static int aspeed_sham_update(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->total = req->nbytes; + rctx->src_sg = req->src; + rctx->offset = 0; + rctx->src_nents = sg_nents(req->src); + rctx->op = SHA_OP_UPDATE; + + rctx->digcnt[0] += rctx->total; + if (rctx->digcnt[0] < rctx->total) + rctx->digcnt[1]++; + + if (rctx->bufcnt + rctx->total < rctx->block_size) { + scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, + rctx->src_sg, rctx->offset, + rctx->total, 0); + rctx->bufcnt += rctx->total; + + return 0; + } + + return aspeed_hace_hash_handle_queue(hace_dev, req); +} + +static int aspeed_sham_shash_digest(struct crypto_shash *tfm, u32 flags, + const u8 *data, unsigned int len, u8 *out) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + + return crypto_shash_digest(shash, data, len, out); +} + +static int aspeed_sham_final(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "req->nbytes:%d, rctx->total:%d\n", + req->nbytes, rctx->total); + rctx->op = SHA_OP_FINAL; + + return aspeed_hace_hash_handle_queue(hace_dev, req); +} + +static int aspeed_sham_finup(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + int rc1, rc2; + + AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes); + + rctx->flags |= SHA_FLAGS_FINUP; + + rc1 = aspeed_sham_update(req); + if (rc1 == -EINPROGRESS || rc1 == -EBUSY) + return rc1; + + /* + * final() has to be always called to cleanup resources + * even if update() failed, except EINPROGRESS + */ + rc2 = aspeed_sham_final(req); + + return rc1 ? : rc2; +} + +static int aspeed_sham_init(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + AHASH_DBG(hace_dev, "%s: digest size:%d\n", + crypto_tfm_alg_name(&tfm->base), + crypto_ahash_digestsize(tfm)); + + rctx->cmd = HASH_CMD_ACC_MODE; + rctx->flags = 0; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA1 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA1; + rctx->digsize = SHA1_DIGEST_SIZE; + rctx->block_size = SHA1_BLOCK_SIZE; + rctx->sha_iv = sha1_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha1_iv, rctx->ivsize); + break; + case SHA224_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA224 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA224; + rctx->digsize = SHA224_DIGEST_SIZE; + rctx->block_size = SHA224_BLOCK_SIZE; + rctx->sha_iv = sha224_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha224_iv, rctx->ivsize); + break; + case SHA256_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA256 | HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA256; + rctx->digsize = SHA256_DIGEST_SIZE; + rctx->block_size = SHA256_BLOCK_SIZE; + rctx->sha_iv = sha256_iv; + rctx->ivsize = 32; + memcpy(rctx->digest, sha256_iv, rctx->ivsize); + break; + case SHA384_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA384 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA384; + rctx->digsize = SHA384_DIGEST_SIZE; + rctx->block_size = SHA384_BLOCK_SIZE; + rctx->sha_iv = (const __be32 *)sha384_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha384_iv, rctx->ivsize); + break; + case SHA512_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512; + rctx->digsize = SHA512_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = (const __be32 *)sha512_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_iv, rctx->ivsize); + break; + default: + dev_warn(tctx->hace_dev->dev, "digest size %d not support\n", + crypto_ahash_digestsize(tfm)); + return -EINVAL; + } + + rctx->bufcnt = 0; + rctx->total = 0; + rctx->digcnt[0] = 0; + rctx->digcnt[1] = 0; + + /* HMAC init */ + if (tctx->flags & SHA_FLAGS_HMAC) { + rctx->digcnt[0] = rctx->block_size; + rctx->bufcnt = rctx->block_size; + memcpy(rctx->buffer, bctx->ipad, rctx->block_size); + rctx->flags |= SHA_FLAGS_HMAC; + } + + return 0; +} + +static int aspeed_sha512s_init(struct ahash_request *req) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + AHASH_DBG(hace_dev, "digest size: %d\n", crypto_ahash_digestsize(tfm)); + + rctx->cmd = HASH_CMD_ACC_MODE; + rctx->flags = 0; + + switch (crypto_ahash_digestsize(tfm)) { + case SHA224_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512_224 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512_224; + rctx->digsize = SHA224_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = sha512_224_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_224_iv, rctx->ivsize); + break; + case SHA256_DIGEST_SIZE: + rctx->cmd |= HASH_CMD_SHA512_SER | HASH_CMD_SHA512_256 | + HASH_CMD_SHA_SWAP; + rctx->flags |= SHA_FLAGS_SHA512_256; + rctx->digsize = SHA256_DIGEST_SIZE; + rctx->block_size = SHA512_BLOCK_SIZE; + rctx->sha_iv = sha512_256_iv; + rctx->ivsize = 64; + memcpy(rctx->digest, sha512_256_iv, rctx->ivsize); + break; + default: + dev_warn(tctx->hace_dev->dev, "digest size %d not support\n", + crypto_ahash_digestsize(tfm)); + return -EINVAL; + } + + rctx->bufcnt = 0; + rctx->total = 0; + rctx->digcnt[0] = 0; + rctx->digcnt[1] = 0; + + /* HMAC init */ + if (tctx->flags & SHA_FLAGS_HMAC) { + rctx->digcnt[0] = rctx->block_size; + rctx->bufcnt = rctx->block_size; + memcpy(rctx->buffer, bctx->ipad, rctx->block_size); + rctx->flags |= SHA_FLAGS_HMAC; + } + + return 0; +} + +static int aspeed_sham_digest(struct ahash_request *req) +{ + return aspeed_sham_init(req) ? : aspeed_sham_finup(req); +} + +static int aspeed_sham_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + int ds = crypto_shash_digestsize(bctx->shash); + int bs = crypto_shash_blocksize(bctx->shash); + int err = 0; + int i; + + AHASH_DBG(hace_dev, "%s: keylen:%d\n", crypto_tfm_alg_name(&tfm->base), + keylen); + + if (keylen > bs) { + err = aspeed_sham_shash_digest(bctx->shash, + crypto_shash_get_flags(bctx->shash), + key, keylen, bctx->ipad); + if (err) + return err; + keylen = ds; + + } else { + memcpy(bctx->ipad, key, keylen); + } + + memset(bctx->ipad + keylen, 0, bs - keylen); + memcpy(bctx->opad, bctx->ipad, bs); + + for (i = 0; i < bs; i++) { + bctx->ipad[i] ^= HMAC_IPAD_VALUE; + bctx->opad[i] ^= HMAC_OPAD_VALUE; + } + + return err; +} + +static int aspeed_sham_cra_init(struct crypto_tfm *tfm) +{ + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); + struct aspeed_hace_alg *ast_alg; + + ast_alg = container_of(alg, struct aspeed_hace_alg, alg.ahash); + tctx->hace_dev = ast_alg->hace_dev; + tctx->flags = 0; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct aspeed_sham_reqctx)); + + if (ast_alg->alg_base) { + /* hmac related */ + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + tctx->flags |= SHA_FLAGS_HMAC; + bctx->shash = crypto_alloc_shash(ast_alg->alg_base, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(bctx->shash)) { + dev_warn(ast_alg->hace_dev->dev, + "base driver '%s' could not be loaded.\n", + ast_alg->alg_base); + return PTR_ERR(bctx->shash); + } + } + + tctx->enginectx.op.do_one_request = aspeed_ahash_do_request; + tctx->enginectx.op.prepare_request = aspeed_ahash_prepare_request; + tctx->enginectx.op.unprepare_request = NULL; + + return 0; +} + +static void aspeed_sham_cra_exit(struct crypto_tfm *tfm) +{ + struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm); + struct aspeed_hace_dev *hace_dev = tctx->hace_dev; + + AHASH_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(tfm)); + + if (tctx->flags & SHA_FLAGS_HMAC) { + struct aspeed_sha_hmac_ctx *bctx = tctx->base; + + crypto_free_shash(bctx->shash); + } +} + +static int aspeed_sham_export(struct ahash_request *req, void *out) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int aspeed_sham_import(struct ahash_request *req, const void *in) +{ + struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req); + + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static struct aspeed_hace_alg aspeed_ahash_algs[] = { + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "aspeed-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "aspeed-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "aspeed-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha1", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "aspeed-hmac-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha224", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "aspeed-hmac-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha256", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "aspeed-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, +}; + +static struct aspeed_hace_alg aspeed_ahash_algs_g6[] = { + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "aspeed-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "aspeed-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512_224", + .cra_driver_name = "aspeed-sha512_224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "sha512_256", + .cra_driver_name = "aspeed-sha512_256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha384", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "aspeed-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512", + .alg.ahash = { + .init = aspeed_sham_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "aspeed-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512_224", + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512_224)", + .cra_driver_name = "aspeed-hmac-sha512_224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, + { + .alg_base = "sha512_256", + .alg.ahash = { + .init = aspeed_sha512s_init, + .update = aspeed_sham_update, + .final = aspeed_sham_final, + .finup = aspeed_sham_finup, + .digest = aspeed_sham_digest, + .setkey = aspeed_sham_setkey, + .export = aspeed_sham_export, + .import = aspeed_sham_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct aspeed_sham_reqctx), + .base = { + .cra_name = "hmac(sha512_256)", + .cra_driver_name = "aspeed-hmac-sha512_256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aspeed_sham_ctx) + + sizeof(struct aspeed_sha_hmac_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aspeed_sham_cra_init, + .cra_exit = aspeed_sham_cra_exit, + } + } + }, + }, +}; + +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) + crypto_unregister_ahash(&aspeed_ahash_algs[i].alg.ahash); + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) + crypto_unregister_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); +} + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev) +{ + int rc, i; + + AHASH_DBG(hace_dev, "\n"); + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs); i++) { + aspeed_ahash_algs[i].hace_dev = hace_dev; + rc = crypto_register_ahash(&aspeed_ahash_algs[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs[i].alg.ahash.halg.base.cra_name); + } + } + + if (hace_dev->version != AST2600_VERSION) + return; + + for (i = 0; i < ARRAY_SIZE(aspeed_ahash_algs_g6); i++) { + aspeed_ahash_algs_g6[i].hace_dev = hace_dev; + rc = crypto_register_ahash(&aspeed_ahash_algs_g6[i].alg.ahash); + if (rc) { + AHASH_DBG(hace_dev, "Failed to register %s\n", + aspeed_ahash_algs_g6[i].alg.ahash.halg.base.cra_name); + } + } +} diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c new file mode 100644 index 0000000000000000000000000000000000000000..656cb92c8bb61312324b979eee7162ac495b9c52 --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (c) 2021 Aspeed Technology Inc. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "aspeed-hace.h" + +#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG +#define HACE_DBG(d, fmt, ...) \ + dev_info((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#else +#define HACE_DBG(d, fmt, ...) \ + dev_dbg((d)->dev, "%s() " fmt, __func__, ##__VA_ARGS__) +#endif + +/* HACE interrupt service routine */ +static irqreturn_t aspeed_hace_irq(int irq, void *dev) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)dev; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + u32 sts; + + sts = ast_hace_read(hace_dev, ASPEED_HACE_STS); + ast_hace_write(hace_dev, sts, ASPEED_HACE_STS); + + HACE_DBG(hace_dev, "irq status: 0x%x\n", sts); + + if (sts & HACE_HASH_ISR) { + if (hash_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&hash_engine->done_task); + else + dev_warn(hace_dev->dev, "HASH no active requests.\n"); + } + + if (sts & HACE_CRYPTO_ISR) { + if (crypto_engine->flags & CRYPTO_FLAGS_BUSY) + tasklet_schedule(&crypto_engine->done_task); + else + dev_warn(hace_dev->dev, "CRYPTO no active requests.\n"); + } + + return IRQ_HANDLED; +} + +static void aspeed_hace_crypto_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + + crypto_engine->resume(hace_dev); +} + +static void aspeed_hace_hash_done_task(unsigned long data) +{ + struct aspeed_hace_dev *hace_dev = (struct aspeed_hace_dev *)data; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + hash_engine->resume(hace_dev); +} + +static void aspeed_hace_register(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_register_hace_hash_algs(hace_dev); +#endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_register_hace_crypto_algs(hace_dev); +#endif +} + +static void aspeed_hace_unregister(struct aspeed_hace_dev *hace_dev) +{ +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_HASH + aspeed_unregister_hace_hash_algs(hace_dev); +#endif +#ifdef CONFIG_CRYPTO_DEV_ASPEED_HACE_CRYPTO + aspeed_unregister_hace_crypto_algs(hace_dev); +#endif +} + +static const struct of_device_id aspeed_hace_of_matches[] = { + { .compatible = "aspeed,ast2500-hace", .data = (void *)5, }, + { .compatible = "aspeed,ast2600-hace", .data = (void *)6, }, + {}, +}; + +static int aspeed_hace_probe(struct platform_device *pdev) +{ + struct aspeed_engine_crypto *crypto_engine; + const struct of_device_id *hace_dev_id; + struct aspeed_engine_hash *hash_engine; + struct aspeed_hace_dev *hace_dev; + struct resource *res; + int rc; + + hace_dev = devm_kzalloc(&pdev->dev, sizeof(struct aspeed_hace_dev), + GFP_KERNEL); + if (!hace_dev) + return -ENOMEM; + + hace_dev_id = of_match_device(aspeed_hace_of_matches, &pdev->dev); + if (!hace_dev_id) { + dev_err(&pdev->dev, "Failed to match hace dev id\n"); + return -EINVAL; + } + + hace_dev->dev = &pdev->dev; + hace_dev->version = (unsigned long)hace_dev_id->data; + hash_engine = &hace_dev->hash_engine; + crypto_engine = &hace_dev->crypto_engine; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + platform_set_drvdata(pdev, hace_dev); + + hace_dev->regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(hace_dev->regs)) + return PTR_ERR(hace_dev->regs); + + /* Get irq number and register it */ + hace_dev->irq = platform_get_irq(pdev, 0); + if (hace_dev->irq < 0) + return -ENXIO; + + rc = devm_request_irq(&pdev->dev, hace_dev->irq, aspeed_hace_irq, 0, + dev_name(&pdev->dev), hace_dev); + if (rc) { + dev_err(&pdev->dev, "Failed to request interrupt\n"); + return rc; + } + + /* Get clk and enable it */ + hace_dev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hace_dev->clk)) { + dev_err(&pdev->dev, "Failed to get clk\n"); + return -ENODEV; + } + + rc = clk_prepare_enable(hace_dev->clk); + if (rc) { + dev_err(&pdev->dev, "Failed to enable clock 0x%x\n", rc); + return rc; + } + + /* Initialize crypto hardware engine structure for hash */ + hace_dev->crypt_engine_hash = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_hash) { + rc = -ENOMEM; + goto clk_exit; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_hash); + if (rc) + goto err_engine_hash_start; + + tasklet_init(&hash_engine->done_task, aspeed_hace_hash_done_task, + (unsigned long)hace_dev); + + /* Initialize crypto hardware engine structure for crypto */ + hace_dev->crypt_engine_crypto = crypto_engine_alloc_init(hace_dev->dev, + true); + if (!hace_dev->crypt_engine_crypto) { + rc = -ENOMEM; + goto err_engine_hash_start; + } + + rc = crypto_engine_start(hace_dev->crypt_engine_crypto); + if (rc) + goto err_engine_crypto_start; + + tasklet_init(&crypto_engine->done_task, aspeed_hace_crypto_done_task, + (unsigned long)hace_dev); + + /* Allocate DMA buffer for hash engine input used */ + hash_engine->ahash_src_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_HASH_SRC_DMA_BUF_LEN, + &hash_engine->ahash_src_dma_addr, + GFP_KERNEL); + if (!hash_engine->ahash_src_addr) { + dev_err(&pdev->dev, "Failed to allocate dma buffer\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine context used */ + crypto_engine->cipher_ctx = + dmam_alloc_coherent(&pdev->dev, + PAGE_SIZE, + &crypto_engine->cipher_ctx_dma, + GFP_KERNEL); + if (!crypto_engine->cipher_ctx) { + dev_err(&pdev->dev, "Failed to allocate cipher ctx dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine input used */ + crypto_engine->cipher_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_SRC_DMA_BUF_LEN, + &crypto_engine->cipher_dma_addr, + GFP_KERNEL); + if (!crypto_engine->cipher_addr) { + dev_err(&pdev->dev, "Failed to allocate cipher addr dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + + /* Allocate DMA buffer for crypto engine output used */ + if (hace_dev->version == AST2600_VERSION) { + crypto_engine->dst_sg_addr = + dmam_alloc_coherent(&pdev->dev, + ASPEED_CRYPTO_DST_DMA_BUF_LEN, + &crypto_engine->dst_sg_dma_addr, + GFP_KERNEL); + if (!crypto_engine->dst_sg_addr) { + dev_err(&pdev->dev, "Failed to allocate dst_sg dma\n"); + rc = -ENOMEM; + goto err_engine_crypto_start; + } + } + + aspeed_hace_register(hace_dev); + + dev_info(&pdev->dev, "Aspeed Crypto Accelerator successfully registered\n"); + + return 0; + +err_engine_crypto_start: + crypto_engine_exit(hace_dev->crypt_engine_crypto); +err_engine_hash_start: + crypto_engine_exit(hace_dev->crypt_engine_hash); +clk_exit: + clk_disable_unprepare(hace_dev->clk); + + return rc; +} + +static int aspeed_hace_remove(struct platform_device *pdev) +{ + struct aspeed_hace_dev *hace_dev = platform_get_drvdata(pdev); + struct aspeed_engine_crypto *crypto_engine = &hace_dev->crypto_engine; + struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine; + + aspeed_hace_unregister(hace_dev); + + crypto_engine_exit(hace_dev->crypt_engine_hash); + crypto_engine_exit(hace_dev->crypt_engine_crypto); + + tasklet_kill(&hash_engine->done_task); + tasklet_kill(&crypto_engine->done_task); + + clk_disable_unprepare(hace_dev->clk); + + return 0; +} + +MODULE_DEVICE_TABLE(of, aspeed_hace_of_matches); + +static struct platform_driver aspeed_hace_driver = { + .probe = aspeed_hace_probe, + .remove = aspeed_hace_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = aspeed_hace_of_matches, + }, +}; + +module_platform_driver(aspeed_hace_driver); + +MODULE_AUTHOR("Neal Liu "); +MODULE_DESCRIPTION("Aspeed HACE driver Crypto Accelerator"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/aspeed/aspeed-hace.h b/drivers/crypto/aspeed/aspeed-hace.h new file mode 100644 index 0000000000000000000000000000000000000000..f2cde23b56aeaca453af67b0063654be1630a31a --- /dev/null +++ b/drivers/crypto/aspeed/aspeed-hace.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __ASPEED_HACE_H__ +#define __ASPEED_HACE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/***************************** + * * + * HACE register definitions * + * * + * ***************************/ +#define ASPEED_HACE_SRC 0x00 /* Crypto Data Source Base Address Register */ +#define ASPEED_HACE_DEST 0x04 /* Crypto Data Destination Base Address Register */ +#define ASPEED_HACE_CONTEXT 0x08 /* Crypto Context Buffer Base Address Register */ +#define ASPEED_HACE_DATA_LEN 0x0C /* Crypto Data Length Register */ +#define ASPEED_HACE_CMD 0x10 /* Crypto Engine Command Register */ + +/* G5 */ +#define ASPEED_HACE_TAG 0x18 /* HACE Tag Register */ +/* G6 */ +#define ASPEED_HACE_GCM_ADD_LEN 0x14 /* Crypto AES-GCM Additional Data Length Register */ +#define ASPEED_HACE_GCM_TAG_BASE_ADDR 0x18 /* Crypto AES-GCM Tag Write Buff Base Address Reg */ + +#define ASPEED_HACE_STS 0x1C /* HACE Status Register */ + +#define ASPEED_HACE_HASH_SRC 0x20 /* Hash Data Source Base Address Register */ +#define ASPEED_HACE_HASH_DIGEST_BUFF 0x24 /* Hash Digest Write Buffer Base Address Register */ +#define ASPEED_HACE_HASH_KEY_BUFF 0x28 /* Hash HMAC Key Buffer Base Address Register */ +#define ASPEED_HACE_HASH_DATA_LEN 0x2C /* Hash Data Length Register */ +#define ASPEED_HACE_HASH_CMD 0x30 /* Hash Engine Command Register */ + +/* crypto cmd */ +#define HACE_CMD_SINGLE_DES 0 +#define HACE_CMD_TRIPLE_DES BIT(17) +#define HACE_CMD_AES_SELECT 0 +#define HACE_CMD_DES_SELECT BIT(16) +#define HACE_CMD_ISR_EN BIT(12) +#define HACE_CMD_CONTEXT_SAVE_ENABLE (0) +#define HACE_CMD_CONTEXT_SAVE_DISABLE BIT(9) +#define HACE_CMD_AES (0) +#define HACE_CMD_DES (0) +#define HACE_CMD_RC4 BIT(8) +#define HACE_CMD_DECRYPT (0) +#define HACE_CMD_ENCRYPT BIT(7) + +#define HACE_CMD_ECB (0x0 << 4) +#define HACE_CMD_CBC (0x1 << 4) +#define HACE_CMD_CFB (0x2 << 4) +#define HACE_CMD_OFB (0x3 << 4) +#define HACE_CMD_CTR (0x4 << 4) +#define HACE_CMD_OP_MODE_MASK (0x7 << 4) + +#define HACE_CMD_AES128 (0x0 << 2) +#define HACE_CMD_AES192 (0x1 << 2) +#define HACE_CMD_AES256 (0x2 << 2) +#define HACE_CMD_OP_CASCADE (0x3) +#define HACE_CMD_OP_INDEPENDENT (0x1) + +/* G5 */ +#define HACE_CMD_RI_WO_DATA_ENABLE (0) +#define HACE_CMD_RI_WO_DATA_DISABLE BIT(11) +#define HACE_CMD_CONTEXT_LOAD_ENABLE (0) +#define HACE_CMD_CONTEXT_LOAD_DISABLE BIT(10) +/* G6 */ +#define HACE_CMD_AES_KEY_FROM_OTP BIT(24) +#define HACE_CMD_GHASH_TAG_XOR_EN BIT(23) +#define HACE_CMD_GHASH_PAD_LEN_INV BIT(22) +#define HACE_CMD_GCM_TAG_ADDR_SEL BIT(21) +#define HACE_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HACE_CMD_DES_SG_CTRL BIT(19) +#define HACE_CMD_SRC_SG_CTRL BIT(18) +#define HACE_CMD_CTR_IV_AES_96 (0x1 << 14) +#define HACE_CMD_CTR_IV_DES_32 (0x1 << 14) +#define HACE_CMD_CTR_IV_AES_64 (0x2 << 14) +#define HACE_CMD_CTR_IV_AES_32 (0x3 << 14) +#define HACE_CMD_AES_KEY_HW_EXP BIT(13) +#define HACE_CMD_GCM (0x5 << 4) + +/* interrupt status reg */ +#define HACE_CRYPTO_ISR BIT(12) +#define HACE_HASH_ISR BIT(9) +#define HACE_HASH_BUSY BIT(0) + +/* hash cmd reg */ +#define HASH_CMD_MBUS_REQ_SYNC_EN BIT(20) +#define HASH_CMD_HASH_SRC_SG_CTRL BIT(18) +#define HASH_CMD_SHA512_224 (0x3 << 10) +#define HASH_CMD_SHA512_256 (0x2 << 10) +#define HASH_CMD_SHA384 (0x1 << 10) +#define HASH_CMD_SHA512 (0) +#define HASH_CMD_INT_ENABLE BIT(9) +#define HASH_CMD_HMAC (0x1 << 7) +#define HASH_CMD_ACC_MODE (0x2 << 7) +#define HASH_CMD_HMAC_KEY (0x3 << 7) +#define HASH_CMD_SHA1 (0x2 << 4) +#define HASH_CMD_SHA224 (0x4 << 4) +#define HASH_CMD_SHA256 (0x5 << 4) +#define HASH_CMD_SHA512_SER (0x6 << 4) +#define HASH_CMD_SHA_SWAP (0x2 << 2) + +#define HASH_SG_LAST_LIST BIT(31) + +#define CRYPTO_FLAGS_BUSY BIT(1) + +#define SHA_OP_UPDATE 1 +#define SHA_OP_FINAL 2 + +#define SHA_FLAGS_SHA1 BIT(0) +#define SHA_FLAGS_SHA224 BIT(1) +#define SHA_FLAGS_SHA256 BIT(2) +#define SHA_FLAGS_SHA384 BIT(3) +#define SHA_FLAGS_SHA512 BIT(4) +#define SHA_FLAGS_SHA512_224 BIT(5) +#define SHA_FLAGS_SHA512_256 BIT(6) +#define SHA_FLAGS_HMAC BIT(8) +#define SHA_FLAGS_FINUP BIT(9) +#define SHA_FLAGS_MASK (0xff) + +#define ASPEED_CRYPTO_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_DST_DMA_BUF_LEN 0xa000 +#define ASPEED_CRYPTO_GCM_TAG_OFFSET 0x9ff0 +#define ASPEED_HASH_SRC_DMA_BUF_LEN 0xa000 +#define ASPEED_HASH_QUEUE_LENGTH 50 + +#define HACE_CMD_IV_REQUIRE (HACE_CMD_CBC | HACE_CMD_CFB | \ + HACE_CMD_OFB | HACE_CMD_CTR) + +struct aspeed_hace_dev; + +typedef int (*aspeed_hace_fn_t)(struct aspeed_hace_dev *); + +struct aspeed_sg_list { + __le32 len; + __le32 phy_addr; +}; + +struct aspeed_engine_hash { + struct tasklet_struct done_task; + unsigned long flags; + struct ahash_request *req; + + /* input buffer */ + void *ahash_src_addr; + dma_addr_t ahash_src_dma_addr; + + dma_addr_t src_dma; + dma_addr_t digest_dma; + + size_t src_length; + + /* callback func */ + aspeed_hace_fn_t resume; + aspeed_hace_fn_t dma_prepare; +}; + +struct aspeed_sha_hmac_ctx { + struct crypto_shash *shash; + u8 ipad[SHA512_BLOCK_SIZE]; + u8 opad[SHA512_BLOCK_SIZE]; +}; + +struct aspeed_sham_ctx { + struct crypto_engine_ctx enginectx; + + struct aspeed_hace_dev *hace_dev; + unsigned long flags; /* hmac flag */ + + struct aspeed_sha_hmac_ctx base[0]; +}; + +struct aspeed_sham_reqctx { + unsigned long flags; /* final update flag should no use*/ + unsigned long op; /* final or update */ + u32 cmd; /* trigger cmd */ + + /* walk state */ + struct scatterlist *src_sg; + int src_nents; + unsigned int offset; /* offset in current sg */ + unsigned int total; /* per update length */ + + size_t digsize; + size_t block_size; + size_t ivsize; + const __be32 *sha_iv; + + /* remain data buffer */ + u8 buffer[SHA512_BLOCK_SIZE * 2]; + dma_addr_t buffer_dma_addr; + size_t bufcnt; /* buffer counter */ + + /* output buffer */ + u8 digest[SHA512_DIGEST_SIZE] __aligned(64); + dma_addr_t digest_dma_addr; + u64 digcnt[2]; +}; + +struct aspeed_engine_crypto { + struct tasklet_struct done_task; + unsigned long flags; + struct skcipher_request *req; + + /* context buffer */ + void *cipher_ctx; + dma_addr_t cipher_ctx_dma; + + /* input buffer, could be single/scatter-gather lists */ + void *cipher_addr; + dma_addr_t cipher_dma_addr; + + /* output buffer, only used in scatter-gather lists */ + void *dst_sg_addr; + dma_addr_t dst_sg_dma_addr; + + /* callback func */ + aspeed_hace_fn_t resume; +}; + +struct aspeed_cipher_ctx { + struct crypto_engine_ctx enginectx; + + struct aspeed_hace_dev *hace_dev; + int key_len; + u8 key[AES_MAX_KEYLENGTH]; + + /* callback func */ + aspeed_hace_fn_t start; + + struct crypto_skcipher *fallback_tfm; +}; + +struct aspeed_cipher_reqctx { + int enc_cmd; + int src_nents; + int dst_nents; + + struct skcipher_request fallback_req; /* keep at the end */ +}; + +struct aspeed_hace_dev { + void __iomem *regs; + struct device *dev; + int irq; + struct clk *clk; + unsigned long version; + + struct crypto_engine *crypt_engine_hash; + struct crypto_engine *crypt_engine_crypto; + + struct aspeed_engine_hash hash_engine; + struct aspeed_engine_crypto crypto_engine; +}; + +struct aspeed_hace_alg { + struct aspeed_hace_dev *hace_dev; + + const char *alg_base; + + union { + struct skcipher_alg skcipher; + struct ahash_alg ahash; + } alg; +}; + +enum aspeed_version { + AST2500_VERSION = 5, + AST2600_VERSION +}; + +#define ast_hace_write(hace, val, offset) \ + writel((val), (hace)->regs + (offset)) +#define ast_hace_read(hace, offset) \ + readl((hace)->regs + (offset)) + +void aspeed_register_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_hash_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_register_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); +void aspeed_unregister_hace_crypto_algs(struct aspeed_hace_dev *hace_dev); + +#endif diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index a4b13d326cfc69ee8f21e6657f30fd87dd773a5e..82bf15d4956144710d95124602d0a574d396b9f3 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -343,7 +343,7 @@ static int atmel_ecc_probe(struct i2c_client *client, return ret; } -static int atmel_ecc_remove(struct i2c_client *client) +static void atmel_ecc_remove(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); @@ -358,7 +358,7 @@ static int atmel_ecc_remove(struct i2c_client *client) * accessing the freed memory. */ dev_emerg(&client->dev, "Device is busy, expect memory corruption.\n"); - return 0; + return; } crypto_unregister_kpp(&atmel_ecdh_nist_p256); @@ -366,8 +366,6 @@ static int atmel_ecc_remove(struct i2c_client *client) spin_lock(&driver_data.i2c_list_lock); list_del(&i2c_priv->i2c_client_list_node); spin_unlock(&driver_data.i2c_list_lock); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index e4087bdd24759960b21aa3b102edb2b49849578a..a84b657598c6e854053c83141634484af0e1220a 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -116,18 +116,16 @@ static int atmel_sha204a_probe(struct i2c_client *client, return ret; } -static int atmel_sha204a_remove(struct i2c_client *client) +static void atmel_sha204a_remove(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); if (atomic_read(&i2c_priv->tfm_count)) { dev_emerg(&client->dev, "Device is busy, will remove it anyhow\n"); - return 0; + return; } kfree((void *)i2c_priv->hwrng.priv); - - return 0; } static const struct of_device_id atmel_sha204a_dt_ids[] = { diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 9ad188cffd0d71273c70ead63abbedbd903d5d6f..51c66afbe677c0116e60e82e6748536fbc0a18d4 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -1712,7 +1712,7 @@ static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) cipher_len = regk_crypto_key_256; break; default: - pr_err("%s: Invalid key length %d!\n", + pr_err("%s: Invalid key length %zu!\n", MODULE_NAME, ctx->key_length); return -EINVAL; } @@ -2091,7 +2091,7 @@ static void artpec6_crypto_task(unsigned long data) return; } - spin_lock_bh(&ac->queue_lock); + spin_lock(&ac->queue_lock); list_for_each_entry_safe(req, n, &ac->pending, list) { struct artpec6_crypto_dma_descriptors *dma = req->dma; @@ -2128,7 +2128,7 @@ static void artpec6_crypto_task(unsigned long data) artpec6_crypto_process_queue(ac, &complete_in_progress); - spin_unlock_bh(&ac->queue_lock); + spin_unlock(&ac->queue_lock); /* Perform the completion callbacks without holding the queue lock * to allow new request submissions from the callbacks. diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 053315e260c2267b4a47cf1c15d732fcbec6db30..c8c799428fe0b1c6d187f9b4f122c8874ad5af3e 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -1928,7 +1928,7 @@ static int ahash_enqueue(struct ahash_request *req) /* SPU2 hardware does not compute hash of zero length data */ if ((rctx->is_final == 1) && (rctx->total_todo == 0) && (iproc_priv.spu.spu_type == SPU_TYPE_SPU2)) { - alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); + alg_name = crypto_ahash_alg_name(tfm); flow_log("Doing %sfinal %s zero-len hash request in software\n", rctx->is_final ? "" : "non-", alg_name); err = do_shash((unsigned char *)alg_name, req->result, @@ -2029,7 +2029,7 @@ static int ahash_init(struct ahash_request *req) * supported by the hardware, we need to handle it in software * by calling synchronous hash functions. */ - alg_name = crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); + alg_name = crypto_ahash_alg_name(tfm); hash = crypto_alloc_shash(alg_name, 0, 0); if (IS_ERR(hash)) { ret = PTR_ERR(hash); diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 71281a3bdbdc07dca64ff201344b73a9e504426e..d6d87332140acbe0870c4cfa41d6a19e9c8b18ac 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -231,7 +231,7 @@ struct iproc_ctx_s { /* * shash descriptor - needed to perform incremental hashing in - * in software, when hw doesn't support it. + * software, when hw doesn't support it. */ struct shash_desc *shash; diff --git a/drivers/crypto/cavium/cpt/cpt_hw_types.h b/drivers/crypto/cavium/cpt/cpt_hw_types.h index 8ec6edc69f3febd361d7e98ca2bf0d4a967e6890..ae4791a8ec4a8d01763c7d115f440adcd2bdf5c5 100644 --- a/drivers/crypto/cavium/cpt/cpt_hw_types.h +++ b/drivers/crypto/cavium/cpt/cpt_hw_types.h @@ -396,7 +396,7 @@ union cptx_vqx_misc_ena_w1s { * Word0 * reserved_20_63:44 [63:20] Reserved. * dbell_cnt:20 [19:0](R/W/H) Number of instruction queue 64-bit words to add - * to the CPT instruction doorbell count. Readback value is the the + * to the CPT instruction doorbell count. Readback value is the * current number of pending doorbell requests. If counter overflows * CPT()_VQ()_MISC_INT[DBELL_DOVF] is set. To reset the count back to * zero, write one to clear CPT()_VQ()_MISC_INT_ENA_W1C[DBELL_DOVF], diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 8c32d0eb8fcf265d4609f22526cb95fb7c277628..6872ac3440010f1f4029a6adcf78464230db35b1 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -253,6 +253,7 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) const struct firmware *fw_entry; struct device *dev = &cpt->pdev->dev; struct ucode_header *ucode; + unsigned int code_length; struct microcode *mcode; int j, ret = 0; @@ -263,11 +264,12 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) ucode = (struct ucode_header *)fw_entry->data; mcode = &cpt->mcode[cpt->next_mc_idx]; memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); - mcode->code_size = ntohl(ucode->code_length) * 2; - if (!mcode->code_size) { + code_length = ntohl(ucode->code_length); + if (code_length == 0 || code_length >= INT_MAX / 2) { ret = -EINVAL; goto fw_release; } + mcode->code_size = code_length * 2; mcode->is_ae = is_ae; mcode->core_mask = 0ULL; diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c index 7df71fcebe8f27481adf59ed3a3694a080133c98..1046a746d36f551c5ddf19726b8bac234c7bd969 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -198,22 +198,16 @@ static int zip_decompress(const u8 *src, unsigned int slen, /* Legacy Compress framework start */ int zip_alloc_comp_ctx_deflate(struct crypto_tfm *tfm) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_ctx_init(zip_ctx, 0); - - return ret; + return zip_ctx_init(zip_ctx, 0); } int zip_alloc_comp_ctx_lzs(struct crypto_tfm *tfm) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_ctx_init(zip_ctx, 1); - - return ret; + return zip_ctx_init(zip_ctx, 1); } void zip_free_comp_ctx(struct crypto_tfm *tfm) @@ -227,24 +221,18 @@ int zip_comp_compress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_compress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_compress(src, slen, dst, dlen, zip_ctx); } int zip_comp_decompress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { - int ret; struct zip_kernel_ctx *zip_ctx = crypto_tfm_ctx(tfm); - ret = zip_decompress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_decompress(src, slen, dst, dlen, zip_ctx); } /* Legacy compress framework end */ /* SCOMP framework start */ @@ -298,22 +286,16 @@ int zip_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { - int ret; struct zip_kernel_ctx *zip_ctx = ctx; - ret = zip_compress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_compress(src, slen, dst, dlen, zip_ctx); } int zip_scomp_decompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { - int ret; struct zip_kernel_ctx *zip_ctx = ctx; - ret = zip_decompress(src, slen, dst, dlen, zip_ctx); - - return ret; + return zip_decompress(src, slen, dst, dlen, zip_ctx); } /* SCOMP framework end */ diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index ec97daf0fcb7cde5b959997edf480b377e71fd17..278636ed251a5349c57faec8af89ff6048dfcf3d 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -64,7 +64,6 @@ static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt) struct ccp_des3_req_ctx *rctx = skcipher_request_ctx(req); struct scatterlist *iv_sg = NULL; unsigned int iv_len = 0; - int ret; if (!ctx->u.des3.key_len) return -EINVAL; @@ -100,9 +99,7 @@ static int ccp_des3_crypt(struct skcipher_request *req, bool encrypt) rctx->cmd.u.des3.src_len = req->cryptlen; rctx->cmd.u.des3.dst = req->dst; - ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); - - return ret; + return ccp_crypto_enqueue_request(&req->base, &rctx->cmd); } static int ccp_des3_encrypt(struct skcipher_request *req) diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 7d4b4ad1db1f3ae30f8f7fecd9182eae6dcdc46c..9f753cb4f5f18af014e8c040e33162cdb454d6ba 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -641,6 +641,10 @@ static void ccp_dma_release(struct ccp_device *ccp) for (i = 0; i < ccp->cmd_q_count; i++) { chan = ccp->ccp_dma_chan + i; dma_chan = &chan->dma_chan; + + if (dma_chan->client_count) + dma_release_channel(dma_chan); + tasklet_kill(&chan->cleanup_tasklet); list_del_rcu(&dma_chan->device_node); } @@ -766,8 +770,8 @@ void ccp_dmaengine_unregister(struct ccp_device *ccp) if (!dmaengine) return; - dma_async_device_unregister(dma_dev); ccp_dma_release(ccp); + dma_async_device_unregister(dma_dev); kmem_cache_destroy(ccp->dma_desc_cache); kmem_cache_destroy(ccp->dma_cmd_cache); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 9f588c9728f8b20e9f63ca284ee1e37993ff31ed..06fc7156c04f3d5e5b960c81a9fffec07396b2ef 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -211,18 +211,24 @@ static int sev_read_init_ex_file(void) if (IS_ERR(fp)) { int ret = PTR_ERR(fp); - dev_err(sev->dev, - "SEV: could not open %s for read, error %d\n", - init_ex_path, ret); + if (ret == -ENOENT) { + dev_info(sev->dev, + "SEV: %s does not exist and will be created later.\n", + init_ex_path); + ret = 0; + } else { + dev_err(sev->dev, + "SEV: could not open %s for read, error %d\n", + init_ex_path, ret); + } return ret; } nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL); if (nread != NV_LENGTH) { - dev_err(sev->dev, - "SEV: failed to read %u bytes to non volatile memory area, ret %ld\n", + dev_info(sev->dev, + "SEV: could not read %u bytes to non volatile memory area, ret %ld\n", NV_LENGTH, nread); - return -EIO; } dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread); @@ -231,7 +237,7 @@ static int sev_read_init_ex_file(void) return 0; } -static void sev_write_init_ex_file(void) +static int sev_write_init_ex_file(void) { struct sev_device *sev = psp_master->sev_data; struct file *fp; @@ -241,14 +247,16 @@ static void sev_write_init_ex_file(void) lockdep_assert_held(&sev_cmd_mutex); if (!sev_init_ex_buffer) - return; + return 0; fp = open_file_as_root(init_ex_path, O_CREAT | O_WRONLY, 0600); if (IS_ERR(fp)) { + int ret = PTR_ERR(fp); + dev_err(sev->dev, - "SEV: could not open file for write, error %ld\n", - PTR_ERR(fp)); - return; + "SEV: could not open file for write, error %d\n", + ret); + return ret; } nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset); @@ -259,18 +267,20 @@ static void sev_write_init_ex_file(void) dev_err(sev->dev, "SEV: failed to write %u bytes to non volatile memory area, ret %ld\n", NV_LENGTH, nwrite); - return; + return -EIO; } dev_dbg(sev->dev, "SEV: write successful to NV file\n"); + + return 0; } -static void sev_write_init_ex_file_if_required(int cmd_id) +static int sev_write_init_ex_file_if_required(int cmd_id) { lockdep_assert_held(&sev_cmd_mutex); if (!sev_init_ex_buffer) - return; + return 0; /* * Only a few platform commands modify the SPI/NV area, but none of the @@ -285,10 +295,10 @@ static void sev_write_init_ex_file_if_required(int cmd_id) case SEV_CMD_PEK_GEN: break; default: - return; + return 0; } - sev_write_init_ex_file(); + return sev_write_init_ex_file(); } static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) @@ -361,7 +371,7 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) cmd, reg & PSP_CMDRESP_ERR_MASK); ret = -EIO; } else { - sev_write_init_ex_file_if_required(cmd); + ret = sev_write_init_ex_file_if_required(cmd); } print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data, @@ -410,17 +420,12 @@ static int __sev_init_locked(int *error) static int __sev_init_ex_locked(int *error) { struct sev_data_init_ex data; - int ret; memset(&data, 0, sizeof(data)); data.length = sizeof(data); data.nv_address = __psp_pa(sev_init_ex_buffer); data.nv_len = NV_LENGTH; - ret = sev_read_init_ex_file(); - if (ret) - return ret; - if (sev_es_tmr) { /* * Do not include the encryption mask on the physical @@ -439,7 +444,7 @@ static int __sev_platform_init_locked(int *error) { struct psp_device *psp = psp_master; struct sev_device *sev; - int rc, psp_ret = -1; + int rc = 0, psp_ret = -1; int (*init_function)(int *error); if (!psp || !psp->sev_data) @@ -450,8 +455,15 @@ static int __sev_platform_init_locked(int *error) if (sev->state == SEV_STATE_INIT) return 0; - init_function = sev_init_ex_buffer ? __sev_init_ex_locked : - __sev_init_locked; + if (sev_init_ex_buffer) { + init_function = __sev_init_ex_locked; + rc = sev_read_init_ex_file(); + if (rc) + return rc; + } else { + init_function = __sev_init_locked; + } + rc = init_function(&psp_ret); if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) { /* @@ -744,6 +756,11 @@ static int sev_update_firmware(struct device *dev) struct page *p; u64 data_size; + if (!sev_version_greater_or_equal(0, 15)) { + dev_dbg(dev, "DOWNLOAD_FIRMWARE not supported\n"); + return -1; + } + if (sev_get_firmware(dev, &firmware) == -ENOENT) { dev_dbg(dev, "No SEV firmware file present\n"); return -1; @@ -776,6 +793,14 @@ static int sev_update_firmware(struct device *dev) data->len = firmware->size; ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); + + /* + * A quirk for fixing the committed TCB version, when upgrading from + * earlier firmware version than 1.50. + */ + if (!ret && !sev_version_greater_or_equal(1, 50)) + ret = sev_do_cmd(SEV_CMD_DOWNLOAD_FIRMWARE, data, &error); + if (ret) dev_dbg(dev, "Failed to update SEV firmware: %#x\n", error); else @@ -1285,8 +1310,7 @@ void sev_pci_init(void) if (sev_get_api_version()) goto err; - if (sev_version_greater_or_equal(0, 15) && - sev_update_firmware(sev->dev) == 0) + if (sev_update_firmware(sev->dev) == 0) sev_get_api_version(); /* If an init_ex_path is provided rely on INIT_EX for PSP initialization diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 6140e492732265b0af70b6b00c53c6dd7e82d774..9efd88f871d1232fdc5708c312de012347175152 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -274,7 +274,7 @@ static int cc_map_sg(struct device *dev, struct scatterlist *sg, } ret = dma_map_sg(dev, sg, *nents, direction); - if (dma_mapping_error(dev, ret)) { + if (!ret) { *nents = 0; dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret); return -ENOMEM; diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 9a0558ed82f904c419850fbade118133c81cd9cd..9f0b94c8e03dd03f51fd5c9a0aa6ebee22bb1f59 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -22,7 +22,8 @@ enum { HPRE_CLUSTER0, HPRE_CLUSTER1, HPRE_CLUSTER2, - HPRE_CLUSTER3 + HPRE_CLUSTER3, + HPRE_CLUSTERS_NUM_MAX }; enum hpre_ctrl_dbgfs_file { @@ -42,9 +43,6 @@ enum hpre_dfx_dbgfs_file { HPRE_DFX_FILE_NUM }; -#define HPRE_CLUSTERS_NUM_V2 (HPRE_CLUSTER3 + 1) -#define HPRE_CLUSTERS_NUM_V3 1 -#define HPRE_CLUSTERS_NUM_MAX HPRE_CLUSTERS_NUM_V2 #define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM_MAX - 1) struct hpre_debugfs_file { @@ -105,5 +103,5 @@ struct hpre_sqe { struct hisi_qp *hpre_create_qp(u8 type); int hpre_algs_register(struct hisi_qm *qm); void hpre_algs_unregister(struct hisi_qm *qm); - +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg); #endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index 3ba6f15deafc661819c9954c2a494dac4947656e..ef02dadd6217098fe963416973e82a00db3970c4 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -51,6 +51,12 @@ struct hpre_ctx; #define HPRE_ECC_HW256_KSZ_B 32 #define HPRE_ECC_HW384_KSZ_B 48 +/* capability register mask of driver */ +#define HPRE_DRV_RSA_MASK_CAP BIT(0) +#define HPRE_DRV_DH_MASK_CAP BIT(1) +#define HPRE_DRV_ECDH_MASK_CAP BIT(2) +#define HPRE_DRV_X25519_MASK_CAP BIT(5) + typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); struct hpre_rsa_ctx { @@ -147,7 +153,7 @@ static int hpre_alloc_req_id(struct hpre_ctx *ctx) int id; spin_lock_irqsave(&ctx->req_lock, flags); - id = idr_alloc(&ctx->req_idr, NULL, 0, QM_Q_DEPTH, GFP_ATOMIC); + id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); spin_unlock_irqrestore(&ctx->req_lock, flags); return id; @@ -488,7 +494,7 @@ static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) qp->qp_ctx = ctx; qp->req_cb = hpre_alg_cb; - ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH); + ret = hpre_ctx_set(ctx, qp, qp->sq_depth); if (ret) hpre_stop_qp_and_put(qp); @@ -2002,55 +2008,53 @@ static struct kpp_alg dh = { }, }; -static struct kpp_alg ecdh_nist_p192 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p192_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p192", - .cra_driver_name = "hpre-ecdh-nist-p192", - .cra_module = THIS_MODULE, - }, -}; - -static struct kpp_alg ecdh_nist_p256 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p256_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p256", - .cra_driver_name = "hpre-ecdh-nist-p256", - .cra_module = THIS_MODULE, - }, -}; - -static struct kpp_alg ecdh_nist_p384 = { - .set_secret = hpre_ecdh_set_secret, - .generate_public_key = hpre_ecdh_compute_value, - .compute_shared_secret = hpre_ecdh_compute_value, - .max_size = hpre_ecdh_max_size, - .init = hpre_ecdh_nist_p384_init_tfm, - .exit = hpre_ecdh_exit_tfm, - .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, - .base = { - .cra_ctxsize = sizeof(struct hpre_ctx), - .cra_priority = HPRE_CRYPTO_ALG_PRI, - .cra_name = "ecdh-nist-p384", - .cra_driver_name = "hpre-ecdh-nist-p384", - .cra_module = THIS_MODULE, - }, +static struct kpp_alg ecdh_curves[] = { + { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p192_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p192", + .cra_driver_name = "hpre-ecdh-nist-p192", + .cra_module = THIS_MODULE, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p256_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p256", + .cra_driver_name = "hpre-ecdh-nist-p256", + .cra_module = THIS_MODULE, + }, + }, { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p384_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "hpre-ecdh-nist-p384", + .cra_module = THIS_MODULE, + }, + } }; static struct kpp_alg curve25519_alg = { @@ -2070,78 +2074,144 @@ static struct kpp_alg curve25519_alg = { }, }; - -static int hpre_register_ecdh(void) +static int hpre_register_rsa(struct hisi_qm *qm) { int ret; - ret = crypto_register_kpp(&ecdh_nist_p192); - if (ret) - return ret; + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return 0; - ret = crypto_register_kpp(&ecdh_nist_p256); + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); if (ret) - goto unregister_ecdh_p192; + dev_err(&qm->pdev->dev, "failed to register rsa (%d)!\n", ret); - ret = crypto_register_kpp(&ecdh_nist_p384); + return ret; +} + +static void hpre_unregister_rsa(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_RSA_MASK_CAP)) + return; + + crypto_unregister_akcipher(&rsa); +} + +static int hpre_register_dh(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&dh); if (ret) - goto unregister_ecdh_p256; + dev_err(&qm->pdev->dev, "failed to register dh (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_dh(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_DH_MASK_CAP)) + return; + + crypto_unregister_kpp(&dh); +} + +static int hpre_register_ecdh(struct hisi_qm *qm) +{ + int ret, i; + + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return 0; + + for (i = 0; i < ARRAY_SIZE(ecdh_curves); i++) { + ret = crypto_register_kpp(&ecdh_curves[i]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to register %s (%d)!\n", + ecdh_curves[i].base.cra_name, ret); + goto unreg_kpp; + } + } return 0; -unregister_ecdh_p256: - crypto_unregister_kpp(&ecdh_nist_p256); -unregister_ecdh_p192: - crypto_unregister_kpp(&ecdh_nist_p192); +unreg_kpp: + for (--i; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); + return ret; } -static void hpre_unregister_ecdh(void) +static void hpre_unregister_ecdh(struct hisi_qm *qm) { - crypto_unregister_kpp(&ecdh_nist_p384); - crypto_unregister_kpp(&ecdh_nist_p256); - crypto_unregister_kpp(&ecdh_nist_p192); + int i; + + if (!hpre_check_alg_support(qm, HPRE_DRV_ECDH_MASK_CAP)) + return; + + for (i = ARRAY_SIZE(ecdh_curves) - 1; i >= 0; --i) + crypto_unregister_kpp(&ecdh_curves[i]); +} + +static int hpre_register_x25519(struct hisi_qm *qm) +{ + int ret; + + if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) + return 0; + + ret = crypto_register_kpp(&curve25519_alg); + if (ret) + dev_err(&qm->pdev->dev, "failed to register x25519 (%d)!\n", ret); + + return ret; +} + +static void hpre_unregister_x25519(struct hisi_qm *qm) +{ + if (!hpre_check_alg_support(qm, HPRE_DRV_X25519_MASK_CAP)) + return; + + crypto_unregister_kpp(&curve25519_alg); } int hpre_algs_register(struct hisi_qm *qm) { int ret; - rsa.base.cra_flags = 0; - ret = crypto_register_akcipher(&rsa); + ret = hpre_register_rsa(qm); if (ret) return ret; - ret = crypto_register_kpp(&dh); + ret = hpre_register_dh(qm); if (ret) goto unreg_rsa; - if (qm->ver >= QM_HW_V3) { - ret = hpre_register_ecdh(); - if (ret) - goto unreg_dh; - ret = crypto_register_kpp(&curve25519_alg); - if (ret) - goto unreg_ecdh; - } - return 0; + ret = hpre_register_ecdh(qm); + if (ret) + goto unreg_dh; + + ret = hpre_register_x25519(qm); + if (ret) + goto unreg_ecdh; + + return ret; unreg_ecdh: - hpre_unregister_ecdh(); + hpre_unregister_ecdh(qm); unreg_dh: - crypto_unregister_kpp(&dh); + hpre_unregister_dh(qm); unreg_rsa: - crypto_unregister_akcipher(&rsa); + hpre_unregister_rsa(qm); return ret; } void hpre_algs_unregister(struct hisi_qm *qm) { - if (qm->ver >= QM_HW_V3) { - crypto_unregister_kpp(&curve25519_alg); - hpre_unregister_ecdh(); - } - - crypto_unregister_kpp(&dh); - crypto_unregister_akcipher(&rsa); + hpre_unregister_x25519(qm); + hpre_unregister_ecdh(qm); + hpre_unregister_dh(qm); + hpre_unregister_rsa(qm); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 9d529df0eab9b1a7cc3349d33a289738dc7f2eac..471e5ca720f5781551cffe39744deb0c800d15ac 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -53,9 +53,7 @@ #define HPRE_CORE_IS_SCHD_OFFSET 0x90 #define HPRE_RAS_CE_ENB 0x301410 -#define HPRE_HAC_RAS_CE_ENABLE (BIT(0) | BIT(22) | BIT(23)) #define HPRE_RAS_NFE_ENB 0x301414 -#define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c #define HPRE_HAC_RAS_FE_ENABLE 0 @@ -79,8 +77,6 @@ #define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0) #define HPRE_QM_VFG_AX_MASK GENMASK(7, 0) #define HPRE_BD_USR_MASK GENMASK(1, 0) -#define HPRE_CLUSTER_CORE_MASK_V2 GENMASK(3, 0) -#define HPRE_CLUSTER_CORE_MASK_V3 GENMASK(7, 0) #define HPRE_PREFETCH_CFG 0x301130 #define HPRE_SVA_PREFTCH_DFX 0x30115C #define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) @@ -122,6 +118,8 @@ #define HPRE_DFX_COMMON2_LEN 0xE #define HPRE_DFX_CORE_LEN 0x43 +#define HPRE_DEV_ALG_MAX_LEN 256 + static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -137,6 +135,38 @@ struct hpre_hw_error { const char *msg; }; +struct hpre_dev_alg { + u32 alg_msk; + const char *alg; +}; + +static const struct hpre_dev_alg hpre_dev_algs[] = { + { + .alg_msk = BIT(0), + .alg = "rsa\n" + }, { + .alg_msk = BIT(1), + .alg = "dh\n" + }, { + .alg_msk = BIT(2), + .alg = "ecdh\n" + }, { + .alg_msk = BIT(3), + .alg = "ecdsa\n" + }, { + .alg_msk = BIT(4), + .alg = "sm2\n" + }, { + .alg_msk = BIT(5), + .alg = "x25519\n" + }, { + .alg_msk = BIT(6), + .alg = "x448\n" + }, { + /* sentinel */ + } +}; + static struct hisi_qm_list hpre_devices = { .register_to_crypto = hpre_algs_register, .unregister_from_crypto = hpre_algs_unregister, @@ -147,6 +177,62 @@ static const char * const hpre_debug_file_name[] = { [HPRE_CLUSTER_CTRL] = "cluster_ctrl", }; +enum hpre_cap_type { + HPRE_QM_NFE_MASK_CAP, + HPRE_QM_RESET_MASK_CAP, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, + HPRE_QM_CE_MASK_CAP, + HPRE_NFE_MASK_CAP, + HPRE_RESET_MASK_CAP, + HPRE_OOO_SHUTDOWN_MASK_CAP, + HPRE_CE_MASK_CAP, + HPRE_CLUSTER_NUM_CAP, + HPRE_CORE_TYPE_NUM_CAP, + HPRE_CORE_NUM_CAP, + HPRE_CLUSTER_CORE_NUM_CAP, + HPRE_CORE_ENABLE_BITMAP_CAP, + HPRE_DRV_ALG_BITMAP_CAP, + HPRE_DEV_ALG_BITMAP_CAP, + HPRE_CORE1_ALG_BITMAP_CAP, + HPRE_CORE2_ALG_BITMAP_CAP, + HPRE_CORE3_ALG_BITMAP_CAP, + HPRE_CORE4_ALG_BITMAP_CAP, + HPRE_CORE5_ALG_BITMAP_CAP, + HPRE_CORE6_ALG_BITMAP_CAP, + HPRE_CORE7_ALG_BITMAP_CAP, + HPRE_CORE8_ALG_BITMAP_CAP, + HPRE_CORE9_ALG_BITMAP_CAP, + HPRE_CORE10_ALG_BITMAP_CAP +}; + +static const struct hisi_qm_cap_info hpre_basic_info[] = { + {HPRE_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C37, 0x7C37}, + {HPRE_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC37, 0x6C37}, + {HPRE_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C37}, + {HPRE_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xFFFFFE}, + {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFFFE}, + {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFFFE}, + {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, + {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2}, + {HPRE_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x0, 0x8, 0xA}, + {HPRE_CLUSTER_CORE_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x0, 0x2, 0xA}, + {HPRE_CORE_ENABLE_BITMAP_CAP, 0x3140, 0, GENMASK(31, 0), 0x0, 0xF, 0x3FF}, + {HPRE_DRV_ALG_BITMAP_CAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x03, 0x27}, + {HPRE_DEV_ALG_BITMAP_CAP, 0x3148, 0, GENMASK(31, 0), 0x0, 0x03, 0x7F}, + {HPRE_CORE1_ALG_BITMAP_CAP, 0x314c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE2_ALG_BITMAP_CAP, 0x3150, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE3_ALG_BITMAP_CAP, 0x3154, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE4_ALG_BITMAP_CAP, 0x3158, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE5_ALG_BITMAP_CAP, 0x315c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE6_ALG_BITMAP_CAP, 0x3160, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE7_ALG_BITMAP_CAP, 0x3164, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE8_ALG_BITMAP_CAP, 0x3168, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F}, + {HPRE_CORE9_ALG_BITMAP_CAP, 0x316c, 0, GENMASK(31, 0), 0x0, 0x10, 0x10}, + {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} +}; + static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(0), @@ -262,6 +348,46 @@ static struct dfx_diff_registers hpre_diff_regs[] = { }, }; +bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DRV_ALG_BITMAP_CAP, qm->cap_ver); + if (alg & cap_val) + return true; + + return false; +} + +static int hpre_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u32 alg_msk; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, HPRE_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); + + for (i = 0; i < ARRAY_SIZE(hpre_dev_algs); i++) + if (alg_msk & hpre_dev_algs[i].alg_msk) + strcat(algs, hpre_dev_algs[i].alg); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static int hpre_diff_regs_show(struct seq_file *s, void *unused) { struct hisi_qm *qm = s->private; @@ -330,14 +456,12 @@ MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); static inline int hpre_cluster_num(struct hisi_qm *qm) { - return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : - HPRE_CLUSTERS_NUM_V2; + return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CLUSTER_NUM_CAP, qm->cap_ver); } static inline int hpre_cluster_core_mask(struct hisi_qm *qm) { - return (qm->ver >= QM_HW_V3) ? - HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2; + return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CORE_ENABLE_BITMAP_CAP, qm->cap_ver); } struct hisi_qp *hpre_create_qp(u8 type) @@ -457,7 +581,7 @@ static void hpre_open_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -478,7 +602,7 @@ static void hpre_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); @@ -630,7 +754,8 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); if (enable) { val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - val2 = HPRE_HAC_RAS_NFE_ENABLE; + val2 = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; val2 = 0x0; @@ -644,21 +769,30 @@ static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hpre_hw_error_disable(struct hisi_qm *qm) { - /* disable hpre hw error interrupts */ - writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); + u32 ce, nfe; + + ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + /* disable hpre hw error interrupts */ + writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_INT_MASK); /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { + u32 ce, nfe; + + ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + /* clear HPRE hw error source if having */ - writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT); + writel(ce | nfe | HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_HAC_SOURCE_INT); /* configure error type */ - writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); - writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB); + writel(ce, qm->io_base + HPRE_RAS_CE_ENB); + writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ @@ -708,7 +842,7 @@ static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file) return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT); } -static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) +static void hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) { struct hisi_qm *qm = hpre_file_to_qm(file); int cluster_index = file->index - HPRE_CLUSTER_CTRL; @@ -716,8 +850,6 @@ static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val) HPRE_CLSTR_ADDR_INTRVL; writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY); - - return 0; } static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf, @@ -792,9 +924,7 @@ static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf, goto err_input; break; case HPRE_CLUSTER_CTRL: - ret = hpre_cluster_inqry_write(file, val); - if (ret) - goto err_input; + hpre_cluster_inqry_write(file, val); break; default: ret = -EINVAL; @@ -1006,15 +1136,13 @@ static void hpre_debugfs_exit(struct hisi_qm *qm) static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + if (pdev->revision == QM_HW_V1) { pci_warn(pdev, "HPRE version 1 is not supported!\n"); return -EINVAL; } - if (pdev->revision >= QM_HW_V3) - qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2"; - else - qm->algs = "rsa\ndh"; qm->mode = uacce_mode; qm->pdev = pdev; qm->ver = pdev->revision; @@ -1030,7 +1158,19 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qm_list = &hpre_devices; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(pdev, "Failed to init hpre qm configures!\n"); + return ret; + } + + ret = hpre_set_qm_algs(qm); + if (ret) { + pci_err(pdev, "Failed to set hpre algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static int hpre_show_last_regs_init(struct hisi_qm *qm) @@ -1129,7 +1269,11 @@ static u32 hpre_get_hw_err_status(struct hisi_qm *qm) static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); + nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); } static void hpre_open_axi_master_ooo(struct hisi_qm *qm) @@ -1147,14 +1291,20 @@ static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; - err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | - HPRE_OOO_ECC_2BIT_ERR; - err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE; + err_info->fe = HPRE_HAC_RAS_FE_ENABLE; + err_info->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver); + err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR; + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, + HPRE_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HPRE_WR_MSI_PORT; err_info->acpi_rst = "HRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT; } static const struct hisi_qm_err_ini hpre_err_ini = { diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index ad83c194d6648ed907937b43543d5570ef1d5310..8b387de69d229b7cb6059a6bc5094e277a4be16b 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -22,20 +22,17 @@ #define QM_VF_AEQ_INT_MASK 0x4 #define QM_VF_EQ_INT_SOURCE 0x8 #define QM_VF_EQ_INT_MASK 0xc -#define QM_IRQ_NUM_V1 1 -#define QM_IRQ_NUM_PF_V2 4 -#define QM_IRQ_NUM_VF_V2 2 -#define QM_IRQ_NUM_VF_V3 3 -#define QM_EQ_EVENT_IRQ_VECTOR 0 -#define QM_AEQ_EVENT_IRQ_VECTOR 1 -#define QM_CMD_EVENT_IRQ_VECTOR 2 -#define QM_ABNORMAL_EVENT_IRQ_VECTOR 3 +#define QM_IRQ_VECTOR_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_MASK GENMASK(15, 0) +#define QM_IRQ_TYPE_SHIFT 16 +#define QM_ABN_IRQ_TYPE_MASK GENMASK(7, 0) /* mailbox */ #define QM_MB_PING_ALL_VFS 0xffff #define QM_MB_CMD_DATA_SHIFT 32 #define QM_MB_CMD_DATA_MASK GENMASK(31, 0) +#define QM_MB_STATUS_MASK GENMASK(12, 9) /* sqc shift */ #define QM_SQ_HOP_NUM_SHIFT 0 @@ -77,6 +74,9 @@ #define QM_EQ_OVERFLOW 1 #define QM_CQE_ERROR 2 +#define QM_XQ_DEPTH_SHIFT 16 +#define QM_XQ_DEPTH_MASK GENMASK(15, 0) + #define QM_DOORBELL_CMD_SQ 0 #define QM_DOORBELL_CMD_CQ 1 #define QM_DOORBELL_CMD_EQ 2 @@ -86,11 +86,7 @@ #define QM_DB_CMD_SHIFT_V1 16 #define QM_DB_INDEX_SHIFT_V1 32 #define QM_DB_PRIORITY_SHIFT_V1 48 -#define QM_QUE_ISO_CFG_V 0x0030 #define QM_PAGE_SIZE 0x0034 -#define QM_QUE_ISO_EN 0x100154 -#define QM_CAPBILITY 0x100158 -#define QM_QP_NUN_MASK GENMASK(10, 0) #define QM_QP_DB_INTERVAL 0x10000 #define QM_MEM_START_INIT 0x100040 @@ -126,7 +122,6 @@ #define QM_DFX_CNT_CLR_CE 0x100118 #define QM_ABNORMAL_INT_SOURCE 0x100000 -#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(14, 0) #define QM_ABNORMAL_INT_MASK 0x100004 #define QM_ABNORMAL_INT_MASK_VALUE 0x7fff #define QM_ABNORMAL_INT_STATUS 0x100008 @@ -144,8 +139,10 @@ #define QM_RAS_NFE_ENABLE 0x1000f4 #define QM_RAS_CE_THRESHOLD 0x1000f8 #define QM_RAS_CE_TIMES_PER_IRQ 1 -#define QM_RAS_MSI_INT_SEL 0x1040f4 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 +#define QM_ECC_MBIT BIT(2) +#define QM_DB_TIMEOUT BIT(10) +#define QM_OF_FIFO_OF BIT(11) #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 @@ -205,6 +202,8 @@ #define MAX_WAIT_COUNTS 1000 #define QM_CACHE_WB_START 0x204 #define QM_CACHE_WB_DONE 0x208 +#define QM_FUNC_CAPS_REG 0x3100 +#define QM_CAPBILITY_VERSION GENMASK(7, 0) #define PCI_BAR_2 2 #define PCI_BAR_4 4 @@ -221,7 +220,6 @@ #define WAIT_PERIOD 20 #define REMOVE_WAIT_DELAY 10 #define QM_SQE_ADDR_MASK GENMASK(7, 0) -#define QM_EQ_DEPTH (1024 * 2) #define QM_DRIVER_REMOVING 0 #define QM_RST_SCHED 1 @@ -270,8 +268,8 @@ ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \ ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) -#define QM_MK_CQC_DW3_V2(cqe_sz) \ - ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) +#define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \ + ((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT)) #define QM_MK_SQC_W13(priority, orders, alg_type) \ (((priority) << QM_SQ_PRIORITY_SHIFT) | \ @@ -284,8 +282,8 @@ ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \ ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) -#define QM_MK_SQC_DW3_V2(sqe_sz) \ - ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) +#define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \ + ((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT)) #define INIT_QC_COMMON(qc, base, pasid) do { \ (qc)->head = 0; \ @@ -329,6 +327,48 @@ enum qm_mb_cmd { QM_VF_GET_QOS, }; +enum qm_basic_type { + QM_TOTAL_QP_NUM_CAP = 0x0, + QM_FUNC_MAX_QP_CAP, + QM_XEQ_DEPTH_CAP, + QM_QP_DEPTH_CAP, + QM_EQ_IRQ_TYPE_CAP, + QM_AEQ_IRQ_TYPE_CAP, + QM_ABN_IRQ_TYPE_CAP, + QM_PF2VF_IRQ_TYPE_CAP, + QM_PF_IRQ_NUM_CAP, + QM_VF_IRQ_NUM_CAP, +}; + +static const struct hisi_qm_cap_info qm_cap_info_comm[] = { + {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, + {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, + {QM_SUPPORT_STOP_QP, 0x3100, 0, BIT(9), 0x0, 0x0, 0x1}, + {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, + {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_pf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1}, +}; + +static const struct hisi_qm_cap_info qm_cap_info_vf[] = { + {QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0}, +}; + +static const struct hisi_qm_cap_info qm_basic_info[] = { + {QM_TOTAL_QP_NUM_CAP, 0x100158, 0, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_FUNC_MAX_QP_CAP, 0x100158, 11, GENMASK(10, 0), 0x1000, 0x400, 0x400}, + {QM_XEQ_DEPTH_CAP, 0x3104, 0, GENMASK(15, 0), 0x800, 0x4000800, 0x4000800}, + {QM_QP_DEPTH_CAP, 0x3108, 0, GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400}, + {QM_EQ_IRQ_TYPE_CAP, 0x310c, 0, GENMASK(31, 0), 0x10000, 0x10000, 0x10000}, + {QM_AEQ_IRQ_TYPE_CAP, 0x3110, 0, GENMASK(31, 0), 0x0, 0x10001, 0x10001}, + {QM_ABN_IRQ_TYPE_CAP, 0x3114, 0, GENMASK(31, 0), 0x0, 0x10003, 0x10003}, + {QM_PF2VF_IRQ_TYPE_CAP, 0x3118, 0, GENMASK(31, 0), 0x0, 0x0, 0x10002}, + {QM_PF_IRQ_NUM_CAP, 0x311c, 16, GENMASK(15, 0), 0x1, 0x4, 0x4}, + {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, +}; + struct qm_cqe { __le32 rsvd0; __le16 cmd_id; @@ -421,15 +461,11 @@ struct hisi_qm_hw_ops { int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number); void (*qm_db)(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority); - u32 (*get_irq_num)(struct hisi_qm *qm); int (*debug_init)(struct hisi_qm *qm); - void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe); + void (*hw_error_init)(struct hisi_qm *qm); void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); - int (*stop_qp)(struct hisi_qp *qp); int (*set_msi)(struct hisi_qm *qm, bool set); - int (*ping_all_vfs)(struct hisi_qm *qm, u64 cmd); - int (*ping_pf)(struct hisi_qm *qm, u64 cmd); }; struct qm_dfx_item { @@ -533,6 +569,8 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { {50100, 100000, 19} }; +static void qm_irqs_unregister(struct hisi_qm *qm); + static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) { enum qm_state curr = atomic_read(&qm->status.flags); @@ -623,22 +661,17 @@ static u32 qm_get_dev_err_status(struct hisi_qm *qm) } /* Check if the error causes the master ooo block */ -static int qm_check_dev_error(struct hisi_qm *qm) +static bool qm_check_dev_error(struct hisi_qm *qm) { u32 val, dev_val; if (qm->fun_type == QM_HW_VF) - return 0; + return false; - val = qm_get_hw_error_status(qm); - dev_val = qm_get_dev_err_status(qm); + val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; + dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; - if (qm->ver < QM_HW_V3) - return (val & QM_ECC_MBIT) || - (dev_val & qm->err_info.ecc_2bits_mask); - - return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || - (dev_val & (~qm->err_info.dev_ce_mask)); + return val || dev_val; } static int qm_wait_reset_finish(struct hisi_qm *qm) @@ -728,8 +761,12 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) { + int ret; + u32 val; + if (unlikely(hisi_qm_wait_mb_ready(qm))) { dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); + ret = -EBUSY; goto mb_busy; } @@ -737,6 +774,14 @@ static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) if (unlikely(hisi_qm_wait_mb_ready(qm))) { dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); + ret = -ETIMEDOUT; + goto mb_busy; + } + + val = readl(qm->io_base + QM_MB_CMD_SEND_BASE); + if (val & QM_MB_STATUS_MASK) { + dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n"); + ret = -EIO; goto mb_busy; } @@ -744,7 +789,7 @@ static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) mb_busy: atomic64_inc(&qm->debug.dfx.mb_err_cnt); - return -EBUSY; + return ret; } int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, @@ -828,25 +873,52 @@ static int qm_dev_mem_reset(struct hisi_qm *qm) POLL_TIMEOUT); } -static u32 qm_get_irq_num_v1(struct hisi_qm *qm) +/** + * hisi_qm_get_hw_info() - Get device information. + * @qm: The qm which want to get information. + * @info_table: Array for storing device information. + * @index: Index in info_table. + * @is_read: Whether read from reg, 0: not support read from reg. + * + * This function returns device information the caller needs. + */ +u32 hisi_qm_get_hw_info(struct hisi_qm *qm, + const struct hisi_qm_cap_info *info_table, + u32 index, bool is_read) { - return QM_IRQ_NUM_V1; + u32 val; + + switch (qm->ver) { + case QM_HW_V1: + return info_table[index].v1_val; + case QM_HW_V2: + return info_table[index].v2_val; + default: + if (!is_read) + return info_table[index].v3_val; + + val = readl(qm->io_base + info_table[index].offset); + return (val >> info_table[index].shift) & info_table[index].mask; + } } +EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); -static u32 qm_get_irq_num_v2(struct hisi_qm *qm) +static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, + u16 *high_bits, enum qm_basic_type type) { - if (qm->fun_type == QM_HW_PF) - return QM_IRQ_NUM_PF_V2; - else - return QM_IRQ_NUM_VF_V2; + u32 depth; + + depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); + *high_bits = depth & QM_XQ_DEPTH_MASK; + *low_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; } -static u32 qm_get_irq_num_v3(struct hisi_qm *qm) +static u32 qm_get_irq_num(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) - return QM_IRQ_NUM_PF_V2; + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); - return QM_IRQ_NUM_VF_V3; + return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); } static int qm_pm_get_sync(struct hisi_qm *qm) @@ -854,7 +926,7 @@ static int qm_pm_get_sync(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; int ret; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return 0; ret = pm_runtime_resume_and_get(dev); @@ -870,7 +942,7 @@ static void qm_pm_put_sync(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_mark_last_busy(dev); @@ -879,7 +951,7 @@ static void qm_pm_put_sync(struct hisi_qm *qm) static void qm_cq_head_update(struct hisi_qp *qp) { - if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { + if (qp->qp_status.cq_head == qp->cq_depth - 1) { qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase; qp->qp_status.cq_head = 0; } else { @@ -911,6 +983,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) { struct hisi_qm *qm = poll_data->qm; struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; + u16 eq_depth = qm->eq_depth; int eqe_num = 0; u16 cqn; @@ -919,7 +992,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) poll_data->qp_finish_id[eqe_num] = cqn; eqe_num++; - if (qm->status.eq_head == QM_EQ_DEPTH - 1) { + if (qm->status.eq_head == eq_depth - 1) { qm->status.eqc_phase = !qm->status.eqc_phase; eqe = qm->eqe; qm->status.eq_head = 0; @@ -928,7 +1001,7 @@ static int qm_get_complete_eqe_num(struct hisi_qm_poll_data *poll_data) qm->status.eq_head++; } - if (eqe_num == (QM_EQ_DEPTH >> 1) - 1) + if (eqe_num == (eq_depth >> 1) - 1) break; } @@ -1068,6 +1141,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u16 aeq_depth = qm->aeq_depth; u32 type, qp_id; while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { @@ -1092,7 +1166,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) break; } - if (qm->status.aeq_head == QM_Q_DEPTH - 1) { + if (qm->status.aeq_head == aeq_depth - 1) { qm->status.aeqc_phase = !qm->status.aeqc_phase; aeqe = qm->aeqe; qm->status.aeq_head = 0; @@ -1118,24 +1192,6 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) return IRQ_WAKE_THREAD; } -static void qm_irq_unregister(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - - free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - - if (qm->ver > QM_HW_V1) { - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); - - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); - } - - if (qm->ver > QM_HW_V2) - free_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), qm); -} - static void qm_init_qp_status(struct hisi_qp *qp) { struct hisi_qp_status *qp_status = &qp->qp_status; @@ -1151,7 +1207,7 @@ static void qm_init_prefetch(struct hisi_qm *qm) struct device *dev = &qm->pdev->dev; u32 page_type = 0x0; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; switch (PAGE_SIZE) { @@ -1270,7 +1326,7 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, } break; case SHAPER_VFT: - if (qm->ver >= QM_HW_V3) { + if (factor) { tmp = factor->cir_b | (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | @@ -1288,10 +1344,13 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, u32 fun_num, u32 base, u32 number) { - struct qm_shaper_factor *factor = &qm->factor[fun_num]; + struct qm_shaper_factor *factor = NULL; unsigned int val; int ret; + if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + factor = &qm->factor[fun_num]; + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, val & BIT(0), POLL_PERIOD, POLL_TIMEOUT); @@ -1349,7 +1408,7 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, } /* init default shaper qos val */ - if (qm->ver >= QM_HW_V3) { + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { ret = qm_shaper_init_vft(qm, fun_num); if (ret) goto back_sqc_cqc; @@ -1357,11 +1416,9 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, return 0; back_sqc_cqc: - for (i = SQC_VFT; i <= CQC_VFT; i++) { - ret = qm_set_vft_common(qm, i, fun_num, 0, 0); - if (ret) - return ret; - } + for (i = SQC_VFT; i <= CQC_VFT; i++) + qm_set_vft_common(qm, i, fun_num, 0, 0); + return ret; } @@ -1857,39 +1914,19 @@ static void qm_ctx_free(struct hisi_qm *qm, size_t ctx_size, kfree(ctx_addr); } -static int dump_show(struct hisi_qm *qm, void *info, +static void dump_show(struct hisi_qm *qm, void *info, unsigned int info_size, char *info_name) { struct device *dev = &qm->pdev->dev; - u8 *info_buf, *info_curr = info; + u8 *info_curr = info; u32 i; #define BYTE_PER_DW 4 - info_buf = kzalloc(info_size, GFP_KERNEL); - if (!info_buf) - return -ENOMEM; - - for (i = 0; i < info_size; i++, info_curr++) { - if (i % BYTE_PER_DW == 0) - info_buf[i + 3UL] = *info_curr; - else if (i % BYTE_PER_DW == 1) - info_buf[i + 1UL] = *info_curr; - else if (i % BYTE_PER_DW == 2) - info_buf[i - 1] = *info_curr; - else if (i % BYTE_PER_DW == 3) - info_buf[i - 3] = *info_curr; - } - dev_info(dev, "%s DUMP\n", info_name); - for (i = 0; i < info_size; i += BYTE_PER_DW) { + for (i = 0; i < info_size; i += BYTE_PER_DW, info_curr += BYTE_PER_DW) { pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW, - info_buf[i], info_buf[i + 1UL], - info_buf[i + 2UL], info_buf[i + 3UL]); + *(info_curr + 3), *(info_curr + 2), *(info_curr + 1), *(info_curr)); } - - kfree(info_buf); - - return 0; } static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id) @@ -1929,23 +1966,18 @@ static int qm_sqc_dump(struct hisi_qm *qm, const char *s) if (qm->sqc) { sqc_curr = qm->sqc + qp_id; - ret = dump_show(qm, sqc_curr, sizeof(*sqc), - "SOFT SQC"); - if (ret) - dev_info(dev, "Show soft sqc failed!\n"); + dump_show(qm, sqc_curr, sizeof(*sqc), "SOFT SQC"); } up_read(&qm->qps_lock); - goto err_free_ctx; + goto free_ctx; } - ret = dump_show(qm, sqc, sizeof(*sqc), "SQC"); - if (ret) - dev_info(dev, "Show hw sqc failed!\n"); + dump_show(qm, sqc, sizeof(*sqc), "SQC"); -err_free_ctx: +free_ctx: qm_ctx_free(qm, sizeof(*sqc), sqc, &sqc_dma); - return ret; + return 0; } static int qm_cqc_dump(struct hisi_qm *qm, const char *s) @@ -1975,23 +2007,18 @@ static int qm_cqc_dump(struct hisi_qm *qm, const char *s) if (qm->cqc) { cqc_curr = qm->cqc + qp_id; - ret = dump_show(qm, cqc_curr, sizeof(*cqc), - "SOFT CQC"); - if (ret) - dev_info(dev, "Show soft cqc failed!\n"); + dump_show(qm, cqc_curr, sizeof(*cqc), "SOFT CQC"); } up_read(&qm->qps_lock); - goto err_free_ctx; + goto free_ctx; } - ret = dump_show(qm, cqc, sizeof(*cqc), "CQC"); - if (ret) - dev_info(dev, "Show hw cqc failed!\n"); + dump_show(qm, cqc, sizeof(*cqc), "CQC"); -err_free_ctx: +free_ctx: qm_ctx_free(qm, sizeof(*cqc), cqc, &cqc_dma); - return ret; + return 0; } static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size, @@ -2015,9 +2042,7 @@ static int qm_eqc_aeqc_dump(struct hisi_qm *qm, char *s, size_t size, if (ret) goto err_free_ctx; - ret = dump_show(qm, xeqc, size, name); - if (ret) - dev_info(dev, "Show hw %s failed!\n", name); + dump_show(qm, xeqc, size, name); err_free_ctx: qm_ctx_free(qm, size, xeqc, &xeqc_dma); @@ -2025,7 +2050,7 @@ err_free_ctx: } static int q_dump_param_parse(struct hisi_qm *qm, char *s, - u32 *e_id, u32 *q_id) + u32 *e_id, u32 *q_id, u16 q_depth) { struct device *dev = &qm->pdev->dev; unsigned int qp_num = qm->qp_num; @@ -2051,8 +2076,8 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, } ret = kstrtou32(presult, 0, e_id); - if (ret || *e_id >= QM_Q_DEPTH) { - dev_err(dev, "Please input sqe num (0-%d)", QM_Q_DEPTH - 1); + if (ret || *e_id >= q_depth) { + dev_err(dev, "Please input sqe num (0-%u)", q_depth - 1); return -EINVAL; } @@ -2066,54 +2091,49 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, static int qm_sq_dump(struct hisi_qm *qm, char *s) { - struct device *dev = &qm->pdev->dev; + u16 sq_depth = qm->qp_array->cq_depth; void *sqe, *sqe_curr; struct hisi_qp *qp; u32 qp_id, sqe_id; int ret; - ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id); + ret = q_dump_param_parse(qm, s, &sqe_id, &qp_id, sq_depth); if (ret) return ret; - sqe = kzalloc(qm->sqe_size * QM_Q_DEPTH, GFP_KERNEL); + sqe = kzalloc(qm->sqe_size * sq_depth, GFP_KERNEL); if (!sqe) return -ENOMEM; qp = &qm->qp_array[qp_id]; - memcpy(sqe, qp->sqe, qm->sqe_size * QM_Q_DEPTH); + memcpy(sqe, qp->sqe, qm->sqe_size * sq_depth); sqe_curr = sqe + (u32)(sqe_id * qm->sqe_size); memset(sqe_curr + qm->debug.sqe_mask_offset, QM_SQE_ADDR_MASK, qm->debug.sqe_mask_len); - ret = dump_show(qm, sqe_curr, qm->sqe_size, "SQE"); - if (ret) - dev_info(dev, "Show sqe failed!\n"); + dump_show(qm, sqe_curr, qm->sqe_size, "SQE"); kfree(sqe); - return ret; + return 0; } static int qm_cq_dump(struct hisi_qm *qm, char *s) { - struct device *dev = &qm->pdev->dev; struct qm_cqe *cqe_curr; struct hisi_qp *qp; u32 qp_id, cqe_id; int ret; - ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id); + ret = q_dump_param_parse(qm, s, &cqe_id, &qp_id, qm->qp_array->cq_depth); if (ret) return ret; qp = &qm->qp_array[qp_id]; cqe_curr = qp->cqe + cqe_id; - ret = dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE"); - if (ret) - dev_info(dev, "Show cqe failed!\n"); + dump_show(qm, cqe_curr, sizeof(struct qm_cqe), "CQE"); - return ret; + return 0; } static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, @@ -2131,11 +2151,11 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, if (ret) return -EINVAL; - if (!strcmp(name, "EQE") && xeqe_id >= QM_EQ_DEPTH) { - dev_err(dev, "Please input eqe num (0-%d)", QM_EQ_DEPTH - 1); + if (!strcmp(name, "EQE") && xeqe_id >= qm->eq_depth) { + dev_err(dev, "Please input eqe num (0-%u)", qm->eq_depth - 1); return -EINVAL; - } else if (!strcmp(name, "AEQE") && xeqe_id >= QM_Q_DEPTH) { - dev_err(dev, "Please input aeqe num (0-%d)", QM_Q_DEPTH - 1); + } else if (!strcmp(name, "AEQE") && xeqe_id >= qm->aeq_depth) { + dev_err(dev, "Please input aeqe num (0-%u)", qm->eq_depth - 1); return -EINVAL; } @@ -2150,9 +2170,7 @@ static int qm_eq_aeq_dump(struct hisi_qm *qm, const char *s, goto err_unlock; } - ret = dump_show(qm, xeqe, size, name); - if (ret) - dev_info(dev, "Show %s failed!\n", name); + dump_show(qm, xeqe, size, name); err_unlock: up_read(&qm->qps_lock); @@ -2245,8 +2263,10 @@ static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer, return ret; /* Judge if the instance is being reset. */ - if (unlikely(atomic_read(&qm->status.flags) == QM_STOP)) - return 0; + if (unlikely(atomic_read(&qm->status.flags) == QM_STOP)) { + ret = 0; + goto put_dfx_access; + } if (count > QM_DBG_WRITE_LEN) { ret = -ENOSPC; @@ -2300,58 +2320,65 @@ static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir, file->debug = &qm->debug; } -static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v1(struct hisi_qm *qm) { writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_cfg(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_cfg(struct hisi_qm *qm) { - qm->error_mask = ce | nfe | fe; + struct hisi_qm_err_info *err_info = &qm->err_info; + + qm->error_mask = err_info->nfe | err_info->ce | err_info->fe; /* clear QM hw residual error source */ - writel(QM_ABNORMAL_INT_SOURCE_CLR, - qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); /* configure error type */ - writel(ce, qm->io_base + QM_RAS_CE_ENABLE); + writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE); writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); - writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE); - writel(fe, qm->io_base + QM_RAS_FE_ENABLE); + writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE); } -static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v2(struct hisi_qm *qm) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; + u32 irq_unmask; - qm_hw_error_cfg(qm, ce, nfe, fe); + qm_hw_error_cfg(qm); + irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v2(struct hisi_qm *qm) { - writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + u32 irq_mask = qm->error_mask; + + irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_init_v3(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_init_v3(struct hisi_qm *qm) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; + u32 irq_unmask; - qm_hw_error_cfg(qm, ce, nfe, fe); + qm_hw_error_cfg(qm); /* enable close master ooo when hardware error happened */ - writel(nfe & (~QM_DB_RANDOM_INVALID), qm->io_base + QM_OOO_SHUTDOWN_SEL); + writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); + irq_unmask = ~qm->error_mask; irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); } static void qm_hw_error_uninit_v3(struct hisi_qm *qm) { - writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + u32 irq_mask = qm->error_mask; + + irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); /* disable close master ooo when hardware error happened */ writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); @@ -2396,7 +2423,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { - u32 error_status, tmp, val; + u32 error_status, tmp; /* read err sts */ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); @@ -2407,17 +2434,11 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE; - /* ce error does not need to be reset */ - if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) { - writel(error_status, qm->io_base + - QM_ABNORMAL_INT_SOURCE); - writel(qm->err_info.nfe, - qm->io_base + QM_RAS_NFE_ENABLE); - return ACC_ERR_RECOVERED; - } + if (error_status & qm->err_info.qm_reset_mask) + return ACC_ERR_NEED_RESET; - return ACC_ERR_NEED_RESET; + writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); } return ACC_ERR_RECOVERED; @@ -2493,7 +2514,7 @@ static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) u64 val; u32 i; - if (!qm->vfs_num || qm->ver < QM_HW_V3) + if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return 0; while (true) { @@ -2756,7 +2777,6 @@ static int qm_set_msi_v3(struct hisi_qm *qm, bool set) static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, - .get_irq_num = qm_get_irq_num_v1, .hw_error_init = qm_hw_error_init_v1, .set_msi = qm_set_msi, }; @@ -2764,7 +2784,6 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v2, .hw_error_init = qm_hw_error_init_v2, .hw_error_uninit = qm_hw_error_uninit_v2, .hw_error_handle = qm_hw_error_handle_v2, @@ -2774,14 +2793,10 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v3, .hw_error_init = qm_hw_error_init_v3, .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, - .stop_qp = qm_stop_qp, .set_msi = qm_set_msi_v3, - .ping_all_vfs = qm_ping_all_vfs, - .ping_pf = qm_ping_pf, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2789,7 +2804,7 @@ static void *qm_get_avail_sqe(struct hisi_qp *qp) struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH - 1)) + if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1)) return NULL; return qp->sqe + sq_tail * qp->qm->sqe_size; @@ -2830,7 +2845,7 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp = &qm->qp_array[qp_id]; hisi_qm_unset_hw_reset(qp); - memset(qp->cqe, 0, sizeof(struct qm_cqe) * QM_Q_DEPTH); + memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth); qp->event_cb = NULL; qp->req_cb = NULL; @@ -2911,9 +2926,9 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); if (ver == QM_HW_V1) { sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); - sqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); + sqc->w8 = cpu_to_le16(qp->sq_depth - 1); } else { - sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size)); + sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); sqc->w8 = 0; /* rand_qc */ } sqc->cq_num = cpu_to_le16(qp_id); @@ -2954,9 +2969,9 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) if (ver == QM_HW_V1) { cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE)); - cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); + cqc->w8 = cpu_to_le16(qp->cq_depth - 1); } else { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); cqc->w8 = 0; /* rand_qc */ } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -3043,13 +3058,14 @@ static void qp_stop_fail_cb(struct hisi_qp *qp) { int qp_used = atomic_read(&qp->qp_status.used); u16 cur_tail = qp->qp_status.sq_tail; - u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH; + u16 sq_depth = qp->sq_depth; + u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth; struct hisi_qm *qm = qp->qm; u16 pos; int i; for (i = 0; i < qp_used; i++) { - pos = (i + cur_head) % QM_Q_DEPTH; + pos = (i + cur_head) % sq_depth; qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); atomic_dec(&qp->qp_status.used); } @@ -3078,8 +3094,8 @@ static int qm_drain_qp(struct hisi_qp *qp) return 0; /* Kunpeng930 supports drain qp by device */ - if (qm->ops->stop_qp) { - ret = qm->ops->stop_qp(qp); + if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { + ret = qm_stop_qp(qp); if (ret) dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id); return ret; @@ -3197,7 +3213,7 @@ int hisi_qp_send(struct hisi_qp *qp, const void *msg) { struct hisi_qp_status *qp_status = &qp->qp_status; u16 sq_tail = qp_status->sq_tail; - u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH; + u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; void *sqe = qm_get_avail_sqe(qp); if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || @@ -3286,7 +3302,6 @@ static void hisi_qm_uacce_put_queue(struct uacce_queue *q) { struct hisi_qp *qp = q->priv; - hisi_qm_cache_wb(qp->qm); hisi_qm_release_qp(qp); } @@ -3310,7 +3325,7 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q, if (qm->ver == QM_HW_V1) { if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR) return -EINVAL; - } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) { + } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE)) return -EINVAL; @@ -3387,6 +3402,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, unsigned long arg) { struct hisi_qp *qp = q->priv; + struct hisi_qp_info qp_info; struct hisi_qp_ctx qp_ctx; if (cmd == UACCE_CMD_QM_SET_QP_CTX) { @@ -3403,11 +3419,25 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, if (copy_to_user((void __user *)arg, &qp_ctx, sizeof(struct hisi_qp_ctx))) return -EFAULT; - } else { - return -EINVAL; + + return 0; + } else if (cmd == UACCE_CMD_QM_SET_QP_INFO) { + if (copy_from_user(&qp_info, (void __user *)arg, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + qp_info.sqe_size = qp->qm->sqe_size; + qp_info.sq_depth = qp->sq_depth; + qp_info.cq_depth = qp->cq_depth; + + if (copy_to_user((void __user *)arg, &qp_info, + sizeof(struct hisi_qp_info))) + return -EFAULT; + + return 0; } - return 0; + return -EINVAL; } static const struct uacce_ops uacce_qm_ops = { @@ -3427,6 +3457,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) struct uacce_device *uacce; unsigned long mmio_page_nr; unsigned long dus_page_nr; + u16 sq_depth, cq_depth; struct uacce_interface interface = { .flags = UACCE_DEV_SVA, .ops = &uacce_qm_ops, @@ -3453,7 +3484,6 @@ static int qm_alloc_uacce(struct hisi_qm *qm) uacce->is_vf = pdev->is_virtfn; uacce->priv = qm; - uacce->algs = qm->algs; if (qm->ver == QM_HW_V1) uacce->api_ver = HISI_QM_API_VER_BASE; @@ -3464,15 +3494,17 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) mmio_page_nr = QM_DOORBELL_PAGE_NR; - else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) + else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) mmio_page_nr = QM_DOORBELL_PAGE_NR + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; else mmio_page_nr = qm->db_interval / PAGE_SIZE; + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + /* Add one more page for device or qp status */ - dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH + PAGE_SIZE) >> + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + + sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> PAGE_SHIFT; uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; @@ -3577,10 +3609,11 @@ static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) kfree(qm->qp_array); } -static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) +static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, + u16 sq_depth, u16 cq_depth) { struct device *dev = &qm->pdev->dev; - size_t off = qm->sqe_size * QM_Q_DEPTH; + size_t off = qm->sqe_size * sq_depth; struct hisi_qp *qp; int ret = -ENOMEM; @@ -3600,6 +3633,8 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) qp->cqe = qp->qdma.va + off; qp->cqe_dma = qp->qdma.dma + off; qp->qdma.size = dma_size; + qp->sq_depth = sq_depth; + qp->cq_depth = cq_depth; qp->qm = qm; qp->qp_id = id; @@ -3626,7 +3661,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; qm->misc_ctl = false; - if (qm->fun_type == QM_HW_PF && qm->ver > QM_HW_V2) { + if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev))) dev_info(&pdev->dev, "_PS0 and _PR0 are not defined"); } @@ -3636,7 +3671,7 @@ static void qm_cmd_uninit(struct hisi_qm *qm) { u32 val; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return; val = readl(qm->io_base + QM_IFC_INT_MASK); @@ -3648,7 +3683,7 @@ static void qm_cmd_init(struct hisi_qm *qm) { u32 val; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) return; /* Clear communication interrupt source */ @@ -3664,7 +3699,7 @@ static void qm_put_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; - if (qm->use_db_isolation) + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) iounmap(qm->db_io_base); iounmap(qm->io_base); @@ -3714,7 +3749,9 @@ static void hisi_qm_memory_uninit(struct hisi_qm *qm) } idr_destroy(&qm->qp_idr); - kfree(qm->factor); + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); } /** @@ -3740,7 +3777,7 @@ void hisi_qm_uninit(struct hisi_qm *qm) hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock); - qm_irq_unregister(qm); + qm_irqs_unregister(qm); hisi_qm_pci_uninit(qm); if (qm->use_sva) { uacce_remove(qm->uacce); @@ -3841,7 +3878,7 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); - eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc->dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), DMA_TO_DEVICE); @@ -3870,7 +3907,7 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); - aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + aeqc->dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc), DMA_TO_DEVICE); @@ -4136,14 +4173,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, static void qm_hw_error_init(struct hisi_qm *qm) { - struct hisi_qm_err_info *err_info = &qm->err_info; - if (!qm->ops->hw_error_init) { dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); return; } - qm->ops->hw_error_init(qm, err_info->ce, err_info->nfe, err_info->fe); + qm->ops->hw_error_init(qm); } static void qm_hw_error_uninit(struct hisi_qm *qm) @@ -4497,12 +4532,10 @@ static int qm_vf_read_qos(struct hisi_qm *qm) qm->mb_qos = 0; /* vf ping pf to get function qos */ - if (qm->ops->ping_pf) { - ret = qm->ops->ping_pf(qm, QM_VF_GET_QOS); - if (ret) { - pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); - return ret; - } + ret = qm_ping_pf(qm, QM_VF_GET_QOS); + if (ret) { + pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); + return ret; } while (true) { @@ -4674,14 +4707,14 @@ static const struct file_operations qm_algqos_fops = { * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. * @qm: The qm for which we want to add debugfs files. * - * Create function qos debugfs files. + * Create function qos debugfs files, VF ping PF to get function qos. */ static void hisi_qm_set_algqos_init(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, qm, &qm_algqos_fops); - else + else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, qm, &qm_algqos_fops); } @@ -4729,7 +4762,7 @@ void hisi_qm_debug_init(struct hisi_qm *qm) &qm_atomic64_ops); } - if (qm->ver >= QM_HW_V3) + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) hisi_qm_set_algqos_init(qm); } EXPORT_SYMBOL_GPL(hisi_qm_debug_init); @@ -4768,6 +4801,14 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm) } EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); +static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) +{ + int i; + + for (i = 1; i <= total_func; i++) + qm->factor[i].func_qos = QM_QOS_MAX_VAL; +} + /** * hisi_qm_sriov_enable() - enable virtual functions * @pdev: the PCIe device @@ -4794,7 +4835,17 @@ int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs) goto err_put_sync; } - num_vfs = min_t(int, max_vfs, total_vfs); + if (max_vfs > total_vfs) { + pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs); + ret = -ERANGE; + goto err_put_sync; + } + + num_vfs = max_vfs; + + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + hisi_qm_init_vf_qos(qm, num_vfs); + ret = qm_vf_q_assign(qm, num_vfs); if (ret) { pci_err(pdev, "Can't assign queues for VF!\n"); @@ -4830,7 +4881,6 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); - int total_vfs = pci_sriov_get_totalvfs(qm->pdev); int ret; if (pci_vfs_assigned(pdev)) { @@ -4845,8 +4895,7 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); - /* clear vf function shaper configure array */ - memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); + ret = qm_clear_vft_config(qm); if (ret) return ret; @@ -4891,17 +4940,11 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) if (qm->err_ini->log_dev_hw_err) qm->err_ini->log_dev_hw_err(qm, err_sts); - /* ce error does not need to be reset */ - if ((err_sts | qm->err_info.dev_ce_mask) == - qm->err_info.dev_ce_mask) { - if (qm->err_ini->clear_dev_hw_err_status) - qm->err_ini->clear_dev_hw_err_status(qm, - err_sts); + if (err_sts & qm->err_info.dev_reset_mask) + return ACC_ERR_NEED_RESET; - return ACC_ERR_RECOVERED; - } - - return ACC_ERR_NEED_RESET; + if (qm->err_ini->clear_dev_hw_err_status) + qm->err_ini->clear_dev_hw_err_status(qm, err_sts); } return ACC_ERR_RECOVERED; @@ -5070,8 +5113,8 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, return 0; /* Kunpeng930 supports to notify VFs to stop before PF reset */ - if (qm->ops->ping_all_vfs) { - ret = qm->ops->ping_all_vfs(qm, cmd); + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, cmd); if (ret) pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); } else { @@ -5262,8 +5305,8 @@ static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) } /* Kunpeng930 supports to notify VFs to start after PF reset. */ - if (qm->ops->ping_all_vfs) { - ret = qm->ops->ping_all_vfs(qm, cmd); + if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { + ret = qm_ping_all_vfs(qm, cmd); if (ret) pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); } else { @@ -5466,8 +5509,6 @@ pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) if (pdev->is_virtfn) return PCI_ERS_RESULT_RECOVERED; - pci_aer_clear_nonfatal_status(pdev); - /* reset pcie device controller */ ret = qm_controller_reset(qm); if (ret) { @@ -5599,51 +5640,6 @@ static irqreturn_t qm_abnormal_irq(int irq, void *data) return IRQ_HANDLED; } -static int qm_irq_register(struct hisi_qm *qm) -{ - struct pci_dev *pdev = qm->pdev; - int ret; - - ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), - qm_irq, 0, qm->dev_name, qm); - if (ret) - return ret; - - if (qm->ver > QM_HW_V1) { - ret = request_threaded_irq(pci_irq_vector(pdev, - QM_AEQ_EVENT_IRQ_VECTOR), - qm_aeq_irq, qm_aeq_thread, - 0, qm->dev_name, qm); - if (ret) - goto err_aeq_irq; - - if (qm->fun_type == QM_HW_PF) { - ret = request_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), - qm_abnormal_irq, 0, qm->dev_name, qm); - if (ret) - goto err_abonormal_irq; - } - } - - if (qm->ver > QM_HW_V2) { - ret = request_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), - qm_mb_cmd_irq, 0, qm->dev_name, qm); - if (ret) - goto err_mb_cmd_irq; - } - - return 0; - -err_mb_cmd_irq: - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); -err_abonormal_irq: - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); -err_aeq_irq: - free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - return ret; -} /** * hisi_qm_dev_shutdown() - Shutdown device. @@ -5711,7 +5707,7 @@ err_prepare: hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); out: pci_save_state(pdev); - ret = qm->ops->ping_pf(qm, cmd); + ret = qm_ping_pf(qm, cmd); if (ret) dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); } @@ -5729,7 +5725,7 @@ static void qm_pf_reset_vf_done(struct hisi_qm *qm) cmd = QM_VF_START_FAIL; } - ret = qm->ops->ping_pf(qm, cmd); + ret = qm_ping_pf(qm, cmd); if (ret) dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); @@ -5924,21 +5920,193 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); +static void qm_unregister_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + if (qm->fun_type == QM_HW_VF) + return; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_abnormal_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + if (qm->fun_type == QM_HW_VF) + return 0; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_mb_cmd_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_aeq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), qm_aeq_irq, + qm_aeq_thread, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_unregister_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + free_irq(pci_irq_vector(pdev, irq_vector), qm); +} + +static int qm_register_eq_irq(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 irq_vector, val; + int ret; + + val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) + return 0; + + irq_vector = val & QM_IRQ_VECTOR_MASK; + ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_irq, 0, qm->dev_name, qm); + if (ret) + dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret); + + return ret; +} + +static void qm_irqs_unregister(struct hisi_qm *qm) +{ + qm_unregister_mb_cmd_irq(qm); + qm_unregister_abnormal_irq(qm); + qm_unregister_aeq_irq(qm); + qm_unregister_eq_irq(qm); +} + +static int qm_irqs_register(struct hisi_qm *qm) +{ + int ret; + + ret = qm_register_eq_irq(qm); + if (ret) + return ret; + + ret = qm_register_aeq_irq(qm); + if (ret) + goto free_eq_irq; + + ret = qm_register_abnormal_irq(qm); + if (ret) + goto free_aeq_irq; + + ret = qm_register_mb_cmd_irq(qm); + if (ret) + goto free_abnormal_irq; + + return 0; + +free_abnormal_irq: + qm_unregister_abnormal_irq(qm); +free_aeq_irq: + qm_unregister_aeq_irq(qm); +free_eq_irq: + qm_unregister_eq_irq(qm); + return ret; +} + static int qm_get_qp_num(struct hisi_qm *qm) { - if (qm->ver == QM_HW_V1) - qm->ctrl_qp_num = QM_QNUM_V1; - else if (qm->ver == QM_HW_V2) - qm->ctrl_qp_num = QM_QNUM_V2; - else - qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) & - QM_QP_NUN_MASK; + bool is_db_isolation; - if (qm->use_db_isolation) - qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >> - QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK; - else - qm->max_qp_num = qm->ctrl_qp_num; + /* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */ + if (qm->fun_type == QM_HW_VF) { + if (qm->ver != QM_HW_V1) + /* v2 starts to support get vft by mailbox */ + return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); + + return 0; + } + + is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); + qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, + QM_FUNC_MAX_QP_CAP, is_db_isolation); /* check if qp number is valid */ if (qm->qp_num > qm->max_qp_num) { @@ -5950,6 +6118,39 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; } +static void qm_get_hw_caps(struct hisi_qm *qm) +{ + const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? + qm_cap_info_pf : qm_cap_info_vf; + u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : + ARRAY_SIZE(qm_cap_info_vf); + u32 val, i; + + /* Doorbell isolate register is a independent register. */ + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); + if (val) + set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); + + if (qm->ver >= QM_HW_V3) { + val = readl(qm->io_base + QM_FUNC_CAPS_REG); + qm->cap_ver = val & QM_CAPBILITY_VERSION; + } + + /* Get PF/VF common capbility */ + for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) { + val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); + if (val) + set_bit(qm_cap_info_comm[i].type, &qm->caps); + } + + /* Get PF/VF different capbility */ + for (i = 0; i < size; i++) { + val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); + if (val) + set_bit(cap_info[i].type, &qm->caps); + } +} + static int qm_get_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -5969,16 +6170,8 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } - if (qm->ver > QM_HW_V2) { - if (qm->fun_type == QM_HW_PF) - qm->use_db_isolation = readl(qm->io_base + - QM_QUE_ISO_EN) & BIT(0); - else - qm->use_db_isolation = readl(qm->io_base + - QM_QUE_ISO_CFG_V) & BIT(0); - } - - if (qm->use_db_isolation) { + qm_get_hw_caps(qm); + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { qm->db_interval = QM_QP_DB_INTERVAL; qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); qm->db_io_base = ioremap(qm->db_phys_base, @@ -5993,16 +6186,14 @@ static int qm_get_pci_res(struct hisi_qm *qm) qm->db_interval = 0; } - if (qm->fun_type == QM_HW_PF) { - ret = qm_get_qp_num(qm); - if (ret) - goto err_db_ioremap; - } + ret = qm_get_qp_num(qm); + if (ret) + goto err_db_ioremap; return 0; err_db_ioremap: - if (qm->use_db_isolation) + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) iounmap(qm->db_io_base); err_ioremap: iounmap(qm->io_base); @@ -6033,11 +6224,7 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) goto err_get_pci_res; pci_set_master(pdev); - if (!qm->ops->get_irq_num) { - ret = -EOPNOTSUPP; - goto err_get_pci_res; - } - num_vec = qm->ops->get_irq_num(qm); + num_vec = qm_get_irq_num(qm); ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to enable MSI vectors!\n"); @@ -6080,6 +6267,7 @@ static int hisi_qm_init_work(struct hisi_qm *qm) static int hisi_qp_alloc_memory(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; + u16 sq_depth, cq_depth; size_t qp_dma_size; int i, ret; @@ -6093,13 +6281,14 @@ static int hisi_qp_alloc_memory(struct hisi_qm *qm) return -ENOMEM; } + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + /* one more page for device or qp statuses */ - qp_dma_size = qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH; + qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; for (i = 0; i < qm->qp_num; i++) { qm->poll_data[i].qm = qm; - ret = hisi_qp_memory_init(qm, qp_dma_size, i); + ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); if (ret) goto err_init_qp_mem; @@ -6116,15 +6305,18 @@ err_init_qp_mem: static int hisi_qm_memory_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - int ret, total_func, i; + int ret, total_func; size_t off = 0; - total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; - qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); - if (!qm->factor) - return -ENOMEM; - for (i = 0; i < total_func; i++) - qm->factor[i].func_qos = QM_QOS_MAX_VAL; + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { + total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; + qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); + if (!qm->factor) + return -ENOMEM; + + /* Only the PF value needs to be initialized */ + qm->factor[0].func_qos = QM_QOS_MAX_VAL; + } #define QM_INIT_BUF(qm, type, num) do { \ (qm)->type = ((qm)->qdma.va + (off)); \ @@ -6133,20 +6325,21 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) } while (0) idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + + qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + + QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, GFP_ATOMIC); dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); if (!qm->qdma.va) { - ret = -ENOMEM; - goto err_alloc_qdma; + ret = -ENOMEM; + goto err_destroy_idr; } - QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); - QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, eqe, qm->eq_depth); + QM_INIT_BUF(qm, aeqe, qm->aeq_depth); QM_INIT_BUF(qm, sqc, qm->qp_num); QM_INIT_BUF(qm, cqc, qm->qp_num); @@ -6158,8 +6351,10 @@ static int hisi_qm_memory_init(struct hisi_qm *qm) err_alloc_qp_array: dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); -err_alloc_qdma: - kfree(qm->factor); +err_destroy_idr: + idr_destroy(&qm->qp_idr); + if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) + kfree(qm->factor); return ret; } @@ -6202,17 +6397,10 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) return ret; - ret = qm_irq_register(qm); + ret = qm_irqs_register(qm); if (ret) goto err_pci_init; - if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { - /* v2 starts to support get vft by mailbox */ - ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); - if (ret) - goto err_irq_register; - } - if (qm->fun_type == QM_HW_PF) { qm_disable_clock_gate(qm); ret = qm_dev_mem_reset(qm); @@ -6251,7 +6439,7 @@ err_alloc_uacce: qm->uacce = NULL; } err_irq_register: - qm_irq_unregister(qm); + qm_irqs_unregister(qm); err_pci_init: hisi_qm_pci_uninit(qm); return ret; @@ -6302,7 +6490,7 @@ void hisi_qm_pm_init(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY); @@ -6321,7 +6509,7 @@ void hisi_qm_pm_uninit(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - if (qm->fun_type == QM_HW_VF || qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) return; pm_runtime_get_noresume(dev); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index d2a0bc93e75253ebd36d5ad1b13e2e8b99ae62ce..3e57fc04b377038146bc23fd9612623e37d5a5c3 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -17,6 +17,7 @@ struct sec_alg_res { dma_addr_t a_ivin_dma; u8 *out_mac; dma_addr_t out_mac_dma; + u16 depth; }; /* Cipher request of SEC private */ @@ -115,9 +116,9 @@ struct sec_cipher_ctx { /* SEC queue context which defines queue's relatives */ struct sec_qp_ctx { struct hisi_qp *qp; - struct sec_req *req_list[QM_Q_DEPTH]; + struct sec_req **req_list; struct idr req_idr; - struct sec_alg_res res[QM_Q_DEPTH]; + struct sec_alg_res *res; struct sec_ctx *ctx; spinlock_t req_lock; struct list_head backlog; @@ -191,8 +192,37 @@ struct sec_dev { bool iommu_used; }; +enum sec_cap_type { + SEC_QM_NFE_MASK_CAP = 0x0, + SEC_QM_RESET_MASK_CAP, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, + SEC_QM_CE_MASK_CAP, + SEC_NFE_MASK_CAP, + SEC_RESET_MASK_CAP, + SEC_OOO_SHUTDOWN_MASK_CAP, + SEC_CE_MASK_CAP, + SEC_CLUSTER_NUM_CAP, + SEC_CORE_TYPE_NUM_CAP, + SEC_CORE_NUM_CAP, + SEC_CORES_PER_CLUSTER_NUM_CAP, + SEC_CORE_ENABLE_BITMAP, + SEC_DRV_ALG_BITMAP_LOW, + SEC_DRV_ALG_BITMAP_HIGH, + SEC_DEV_ALG_BITMAP_LOW, + SEC_DEV_ALG_BITMAP_HIGH, + SEC_CORE1_ALG_BITMAP_LOW, + SEC_CORE1_ALG_BITMAP_HIGH, + SEC_CORE2_ALG_BITMAP_LOW, + SEC_CORE2_ALG_BITMAP_HIGH, + SEC_CORE3_ALG_BITMAP_LOW, + SEC_CORE3_ALG_BITMAP_HIGH, + SEC_CORE4_ALG_BITMAP_LOW, + SEC_CORE4_ALG_BITMAP_HIGH, +}; + void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(struct hisi_qm *qm); void sec_unregister_from_crypto(struct hisi_qm *qm); +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 77c9f13cf69ac67368f622b4b9e9ff416fc0b100..84ae8ddd1a131b658594ff70d3c2043ce78ac8cb 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -59,14 +59,14 @@ #define SEC_ICV_MASK 0x000E #define SEC_SQE_LEN_RATE_MASK 0x3 -#define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) +#define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) #define SEC_SGL_SGE_NR 128 #define SEC_CIPHER_AUTH 0xfe #define SEC_AUTH_CIPHER 0x1 #define SEC_MAX_MAC_LEN 64 #define SEC_MAX_AAD_LEN 65535 #define SEC_MAX_CCM_AAD_LEN 65279 -#define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) +#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) #define SEC_PBUF_SZ 512 #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ @@ -74,11 +74,11 @@ #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ SEC_MAX_MAC_LEN * 2) #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) -#define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) -#define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ - SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) -#define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ - SEC_PBUF_LEFT_SZ) +#define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) +#define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ + SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) +#define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ + SEC_PBUF_LEFT_SZ(depth)) #define SEC_SQE_LEN_RATE 4 #define SEC_SQE_CFLAG 2 @@ -104,6 +104,16 @@ #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 +struct sec_skcipher { + u64 alg_msk; + struct skcipher_alg alg; +}; + +struct sec_aead { + u64 alg_msk; + struct aead_alg alg; +}; + /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) { @@ -128,9 +138,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) int req_id; spin_lock_bh(&qp_ctx->req_lock); - - req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, - 0, QM_Q_DEPTH, GFP_ATOMIC); + req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); spin_unlock_bh(&qp_ctx->req_lock); if (unlikely(req_id < 0)) { dev_err(req->ctx->dev, "alloc req id fail!\n"); @@ -148,7 +156,7 @@ static void sec_free_req_id(struct sec_req *req) struct sec_qp_ctx *qp_ctx = req->qp_ctx; int req_id = req->req_id; - if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { + if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { dev_err(req->ctx->dev, "free request id invalid!\n"); return; } @@ -300,14 +308,15 @@ static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) /* Get DMA memory resources */ static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), &res->c_ivin_dma, GFP_KERNEL); if (!res->c_ivin) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; } @@ -318,20 +327,21 @@ static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) { if (res->c_ivin) - dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), res->c_ivin, res->c_ivin_dma); } static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), &res->a_ivin_dma, GFP_KERNEL); if (!res->a_ivin) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; } @@ -342,20 +352,21 @@ static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) { if (res->a_ivin) - dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), res->a_ivin, res->a_ivin_dma); } static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; int i; - res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, &res->out_mac_dma, GFP_KERNEL); if (!res->out_mac) return -ENOMEM; - for (i = 1; i < QM_Q_DEPTH; i++) { + for (i = 1; i < q_depth; i++) { res[i].out_mac_dma = res->out_mac_dma + i * (SEC_MAX_MAC_LEN << 1); res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); @@ -367,14 +378,14 @@ static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) { if (res->out_mac) - dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, + dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, res->out_mac, res->out_mac_dma); } static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) { if (res->pbuf) - dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, + dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), res->pbuf, res->pbuf_dma); } @@ -384,10 +395,12 @@ static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) */ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) { + u16 q_depth = res->depth; + int size = SEC_PBUF_PAGE_NUM(q_depth); int pbuf_page_offset; int i, j, k; - res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, + res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), &res->pbuf_dma, GFP_KERNEL); if (!res->pbuf) return -ENOMEM; @@ -400,11 +413,11 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) * So we need SEC_PBUF_PAGE_NUM numbers of PAGE * for the SEC_TOTAL_PBUF_SZ */ - for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { + for (i = 0; i <= size; i++) { pbuf_page_offset = PAGE_SIZE * i; for (j = 0; j < SEC_PBUF_NUM; j++) { k = i * SEC_PBUF_NUM + j; - if (k == QM_Q_DEPTH) + if (k == q_depth) break; res[k].pbuf = res->pbuf + j * SEC_PBUF_PKG + pbuf_page_offset; @@ -470,36 +483,29 @@ static void sec_alg_resource_free(struct sec_ctx *ctx, sec_free_mac_resource(dev, qp_ctx->res); } -static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, - int qp_ctx_id, int alg_type) +static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) { + u16 q_depth = qp_ctx->qp->sq_depth; struct device *dev = ctx->dev; - struct sec_qp_ctx *qp_ctx; - struct hisi_qp *qp; int ret = -ENOMEM; - qp_ctx = &ctx->qp_ctx[qp_ctx_id]; - qp = ctx->qps[qp_ctx_id]; - qp->req_type = 0; - qp->qp_ctx = qp_ctx; - qp_ctx->qp = qp; - qp_ctx->ctx = ctx; - - qp->req_cb = sec_req_cb; + qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL); + if (!qp_ctx->req_list) + return ret; - spin_lock_init(&qp_ctx->req_lock); - idr_init(&qp_ctx->req_idr); - INIT_LIST_HEAD(&qp_ctx->backlog); + qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL); + if (!qp_ctx->res) + goto err_free_req_list; + qp_ctx->res->depth = q_depth; - qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, - SEC_SGL_SGE_NR); + qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); if (IS_ERR(qp_ctx->c_in_pool)) { dev_err(dev, "fail to create sgl pool for input!\n"); - goto err_destroy_idr; + goto err_free_res; } - qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, - SEC_SGL_SGE_NR); + qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); if (IS_ERR(qp_ctx->c_out_pool)) { dev_err(dev, "fail to create sgl pool for output!\n"); goto err_free_c_in_pool; @@ -509,34 +515,72 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, if (ret) goto err_free_c_out_pool; - ret = hisi_qm_start_qp(qp, 0); - if (ret < 0) - goto err_queue_free; - return 0; -err_queue_free: - sec_alg_resource_free(ctx, qp_ctx); err_free_c_out_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); -err_destroy_idr: - idr_destroy(&qp_ctx->req_idr); +err_free_res: + kfree(qp_ctx->res); +err_free_req_list: + kfree(qp_ctx->req_list); return ret; } -static void sec_release_qp_ctx(struct sec_ctx *ctx, - struct sec_qp_ctx *qp_ctx) +static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) { struct device *dev = ctx->dev; - hisi_qm_stop_qp(qp_ctx->qp); sec_alg_resource_free(ctx, qp_ctx); - hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); + kfree(qp_ctx->res); + kfree(qp_ctx->req_list); +} + +static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, + int qp_ctx_id, int alg_type) +{ + struct sec_qp_ctx *qp_ctx; + struct hisi_qp *qp; + int ret; + qp_ctx = &ctx->qp_ctx[qp_ctx_id]; + qp = ctx->qps[qp_ctx_id]; + qp->req_type = 0; + qp->qp_ctx = qp_ctx; + qp_ctx->qp = qp; + qp_ctx->ctx = ctx; + + qp->req_cb = sec_req_cb; + + spin_lock_init(&qp_ctx->req_lock); + idr_init(&qp_ctx->req_idr); + INIT_LIST_HEAD(&qp_ctx->backlog); + + ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx); + if (ret) + goto err_destroy_idr; + + ret = hisi_qm_start_qp(qp, 0); + if (ret < 0) + goto err_resource_free; + + return 0; + +err_resource_free: + sec_free_qp_ctx_resource(ctx, qp_ctx); +err_destroy_idr: + idr_destroy(&qp_ctx->req_idr); + return ret; +} + +static void sec_release_qp_ctx(struct sec_ctx *ctx, + struct sec_qp_ctx *qp_ctx) +{ + hisi_qm_stop_qp(qp_ctx->qp); + sec_free_qp_ctx_resource(ctx, qp_ctx); idr_destroy(&qp_ctx->req_idr); } @@ -559,7 +603,7 @@ static int sec_ctx_base_init(struct sec_ctx *ctx) ctx->pbuf_supported = ctx->sec->iommu_used; /* Half of queue depth is taken as fake requests limit in the queue. */ - ctx->fake_req_limit = QM_Q_DEPTH >> 1; + ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1; ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), GFP_KERNEL); if (!ctx->qp_ctx) { @@ -1679,7 +1723,6 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) aead_req->out_mac, authsize, a_req->cryptlen + a_req->assoclen); - if (unlikely(sz != authsize)) { dev_err(c->dev, "copy out mac err!\n"); err = -EINVAL; @@ -1966,7 +2009,6 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) return sec_aead_ctx_init(tfm, "sha512"); } - static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, struct sec_req *sreq) { @@ -2126,67 +2168,80 @@ static int sec_skcipher_decrypt(struct skcipher_request *sk_req) .min_keysize = sec_min_key_size,\ .max_keysize = sec_max_key_size,\ .ivsize = iv_size,\ -}, +} #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ max_key_size, blk_size, iv_size) \ SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) -static struct skcipher_alg sec_skciphers[] = { - SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - AES_BLOCK_SIZE, 0) - - SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, - SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, - SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, - DES3_EDE_BLOCK_SIZE, 0) - - SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, - SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, - DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, - SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - AES_BLOCK_SIZE, AES_BLOCK_SIZE) -}; - -static struct skcipher_alg sec_skciphers_v3[] = { - SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, - AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) - - SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, - AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, - SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) +static struct sec_skcipher sec_skciphers[] = { + { + .alg_msk = BIT(0), + .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(1), + .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(2), + .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(3), + .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(4), + .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(5), + .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE, + AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(12), + .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(13), + .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(14), + .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, + SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(15), + .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(16), + .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE, + AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(23), + .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), + }, + { + .alg_msk = BIT(24), + .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, + DES3_EDE_BLOCK_SIZE), + }, }; static int aead_iv_demension_check(struct aead_request *aead_req) @@ -2380,90 +2435,135 @@ static int sec_aead_decrypt(struct aead_request *a_req) .maxauthsize = max_authsize,\ } -static struct aead_alg sec_aeads[] = { - SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", - sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), +static struct sec_aead sec_aeads[] = { + { + .alg_msk = BIT(6), + .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(7), + .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(17), + .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(18), + .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, + AES_BLOCK_SIZE), + }, + { + .alg_msk = BIT(43), + .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, + sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + }, + { + .alg_msk = BIT(44), + .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, + sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + }, + { + .alg_msk = BIT(45), + .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, + sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + }, +}; - SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", - sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), +static void sec_unregister_skcipher(u64 alg_mask, int end) +{ + int i; - SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", - sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, - sec_aead_ctx_exit, AES_BLOCK_SIZE, - AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + for (i = 0; i < end; i++) + if (sec_skciphers[i].alg_msk & alg_mask) + crypto_unregister_skcipher(&sec_skciphers[i].alg); +} - SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - AES_BLOCK_SIZE, AES_BLOCK_SIZE), +static int sec_register_skcipher(u64 alg_mask) +{ + int i, ret, count; - SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - SEC_AIV_SIZE, AES_BLOCK_SIZE) -}; + count = ARRAY_SIZE(sec_skciphers); -static struct aead_alg sec_aeads_v3[] = { - SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - AES_BLOCK_SIZE, AES_BLOCK_SIZE), + for (i = 0; i < count; i++) { + if (!(sec_skciphers[i].alg_msk & alg_mask)) + continue; - SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, - sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, - SEC_AIV_SIZE, AES_BLOCK_SIZE) -}; + ret = crypto_register_skcipher(&sec_skciphers[i].alg); + if (ret) + goto err; + } + + return 0; + +err: + sec_unregister_skcipher(alg_mask, i); + + return ret; +} + +static void sec_unregister_aead(u64 alg_mask, int end) +{ + int i; + + for (i = 0; i < end; i++) + if (sec_aeads[i].alg_msk & alg_mask) + crypto_unregister_aead(&sec_aeads[i].alg); +} + +static int sec_register_aead(u64 alg_mask) +{ + int i, ret, count; + + count = ARRAY_SIZE(sec_aeads); + + for (i = 0; i < count; i++) { + if (!(sec_aeads[i].alg_msk & alg_mask)) + continue; + + ret = crypto_register_aead(&sec_aeads[i].alg); + if (ret) + goto err; + } + + return 0; + +err: + sec_unregister_aead(alg_mask, i); + + return ret; +} int sec_register_to_crypto(struct hisi_qm *qm) { + u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); int ret; - /* To avoid repeat register */ - ret = crypto_register_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + ret = sec_register_skcipher(alg_mask); if (ret) return ret; - if (qm->ver > QM_HW_V2) { - ret = crypto_register_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); - if (ret) - goto reg_skcipher_fail; - } - - ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + ret = sec_register_aead(alg_mask); if (ret) - goto reg_aead_fail; - if (qm->ver > QM_HW_V2) { - ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3)); - if (ret) - goto reg_aead_v3_fail; - } - return ret; + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); -reg_aead_v3_fail: - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); -reg_aead_fail: - if (qm->ver > QM_HW_V2) - crypto_unregister_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); -reg_skcipher_fail: - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(struct hisi_qm *qm) { - if (qm->ver > QM_HW_V2) - crypto_unregister_aeads(sec_aeads_v3, - ARRAY_SIZE(sec_aeads_v3)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); - if (qm->ver > QM_HW_V2) - crypto_unregister_skciphers(sec_skciphers_v3, - ARRAY_SIZE(sec_skciphers_v3)); - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); + sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2c0be91c0b094a6f4160ee412c388d54b69d4914..3705412bac5f19a1cf11cbbffa3a8b2efa647257 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -27,7 +27,6 @@ #define SEC_BD_ERR_CHK_EN3 0xffffbfff #define SEC_SQE_SIZE 128 -#define SEC_SQ_SIZE (SEC_SQE_SIZE * QM_Q_DEPTH) #define SEC_PF_DEF_Q_NUM 256 #define SEC_PF_DEF_Q_BASE 0 #define SEC_CTX_Q_NUM_DEF 2 @@ -42,16 +41,11 @@ #define SEC_ECC_NUM 16 #define SEC_ECC_MASH 0xFF #define SEC_CORE_INT_DISABLE 0x0 -#define SEC_CORE_INT_ENABLE 0x7c1ff -#define SEC_CORE_INT_CLEAR 0x7c1ff -#define SEC_SAA_ENABLE 0x17f #define SEC_RAS_CE_REG 0x301050 #define SEC_RAS_FE_REG 0x301054 #define SEC_RAS_NFE_REG 0x301058 -#define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 -#define SEC_RAS_NFE_ENB_MSK 0x7c177 #define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 #define SEC_MEM_START_INIT_REG 0x301100 @@ -119,6 +113,16 @@ #define SEC_DFX_COMMON1_LEN 0x45 #define SEC_DFX_COMMON2_LEN 0xBA +#define SEC_ALG_BITMAP_SHIFT 32 + +#define SEC_CIPHER_BITMAP (GENMASK_ULL(5, 0) | GENMASK_ULL(16, 12) | \ + GENMASK(24, 21)) +#define SEC_DIGEST_BITMAP (GENMASK_ULL(11, 8) | GENMASK_ULL(20, 19) | \ + GENMASK_ULL(42, 25)) +#define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ + GENMASK_ULL(45, 43)) +#define SEC_DEV_ALG_MAX_LEN 256 + struct sec_hw_error { u32 int_msk; const char *msg; @@ -129,6 +133,11 @@ struct sec_dfx_item { u32 offset; }; +struct sec_dev_alg { + u64 alg_msk; + const char *algs; +}; + static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; @@ -137,6 +146,46 @@ static struct hisi_qm_list sec_devices = { .unregister_from_crypto = sec_unregister_from_crypto, }; +static const struct hisi_qm_cap_info sec_basic_info[] = { + {SEC_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C77, 0x7C77}, + {SEC_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC77, 0x6C77}, + {SEC_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {SEC_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {SEC_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x177, 0x60177}, + {SEC_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x177, 0x177}, + {SEC_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x4, 0x177}, + {SEC_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x88, 0xC088}, + {SEC_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, + {SEC_CORE_ENABLE_BITMAP, 0x3140, 32, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, + {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x187F0FF}, + {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, + {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE1_ALG_BITMAP_LOW, 0x3154, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE1_ALG_BITMAP_HIGH, 0x3158, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE2_ALG_BITMAP_LOW, 0x315c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE2_ALG_BITMAP_HIGH, 0x3160, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE3_ALG_BITMAP_LOW, 0x3164, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE3_ALG_BITMAP_HIGH, 0x3168, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE4_ALG_BITMAP_LOW, 0x316c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, +}; + +static const struct sec_dev_alg sec_dev_algs[] = { { + .alg_msk = SEC_CIPHER_BITMAP, + .algs = "cipher\n", + }, { + .alg_msk = SEC_DIGEST_BITMAP, + .algs = "digest\n", + }, { + .alg_msk = SEC_AEAD_BITMAP, + .algs = "aead\n", + }, +}; + static const struct sec_hw_error sec_hw_errors[] = { { .int_msk = BIT(0), @@ -339,6 +388,16 @@ struct hisi_qp **sec_create_qps(void) return NULL; } +u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) +{ + u32 cap_val_h, cap_val_l; + + cap_val_h = hisi_qm_get_hw_info(qm, sec_basic_info, high, qm->cap_ver); + cap_val_l = hisi_qm_get_hw_info(qm, sec_basic_info, low, qm->cap_ver); + + return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; +} + static const struct kernel_param_ops sec_uacce_mode_ops = { .set = uacce_mode_set, .get = param_get_int, @@ -415,7 +474,7 @@ static void sec_open_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -435,7 +494,7 @@ static void sec_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); @@ -506,7 +565,8 @@ static int sec_engine_init(struct hisi_qm *qm) writel(SEC_SINGLE_PORT_MAX_TRANS, qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS); - writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG); + reg = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CORE_ENABLE_BITMAP, qm->cap_ver); + writel(reg, qm->io_base + SEC_SAA_EN_REG); if (qm->ver < QM_HW_V3) { /* HW V2 enable sm4 extra mode, as ctr/ecb */ @@ -576,7 +636,8 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + SEC_CONTROL_REG); if (enable) { val1 |= SEC_AXI_SHUTDOWN_ENABLE; - val2 = SEC_RAS_NFE_ENB_MSK; + val2 = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= SEC_AXI_SHUTDOWN_DISABLE; val2 = 0x0; @@ -590,25 +651,30 @@ static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void sec_hw_error_enable(struct hisi_qm *qm) { + u32 ce, nfe; + if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); pci_info(qm->pdev, "V1 not support hw error handle\n"); return; } + ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_CE_MASK_CAP, qm->cap_ver); + nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + /* clear SEC hw error source if having */ - writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); + writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_SOURCE); /* enable RAS int */ - writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); + writel(ce, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); - writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); + writel(nfe, qm->io_base + SEC_RAS_NFE_REG); /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ sec_master_ooo_ctrl(qm, true); /* enable SEC hw error interrupts */ - writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); + writel(ce | nfe | SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) @@ -939,7 +1005,11 @@ static u32 sec_get_hw_err_status(struct hisi_qm *qm) static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); + nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + SEC_RAS_NFE_REG); } static void sec_open_axi_master_ooo(struct hisi_qm *qm) @@ -955,14 +1025,20 @@ static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; + err_info->fe = SEC_RAS_FE_ENB_MSK; + err_info->ce = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_QM_NFE_MASK_CAP, qm->cap_ver); err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC; - err_info->dev_ce_mask = SEC_RAS_CE_ENB_MSK; + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, sec_basic_info, + SEC_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = BIT(0); err_info->acpi_rst = "SRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT; } static const struct hisi_qm_err_ini sec_err_ini = { @@ -1001,11 +1077,41 @@ static int sec_pf_probe_init(struct sec_dev *sec) return ret; } +static int sec_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u64 alg_mask; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, SEC_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); + + for (i = 0; i < ARRAY_SIZE(sec_dev_algs); i++) + if (alg_mask & sec_dev_algs[i].alg_msk) + strcat(algs, sec_dev_algs[i].algs); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; - qm->algs = "cipher\ndigest\naead"; qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; @@ -1028,7 +1134,19 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = SEC_QUEUE_NUM_V1 - SEC_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init sec qm configures!\n"); + return ret; + } + + ret = sec_set_qm_algs(qm); + if (ret) { + pci_err(qm->pdev, "Failed to set sec algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static void sec_qm_uninit(struct hisi_qm *qm) diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index 3dfd3bac5a33552a7fe5e85a290a085db1b3c027..f2e6da3240aeb3d183d88e90fe93366ae647a232 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -81,7 +81,8 @@ struct hisi_zip_sqe { u32 rsvd1[4]; }; -int zip_create_qps(struct hisi_qp **qps, int ctx_num, int node); +int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); #endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c index ad35434a3fdb784b95890f6d66c78cbe341fe9ec..6608971d10cdc18ef73bcfc9a0af89d825314710 100644 --- a/drivers/crypto/hisilicon/zip/zip_crypto.c +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -39,6 +39,9 @@ #define HZIP_ALG_PRIORITY 300 #define HZIP_SGL_SGE_NR 10 +#define HZIP_ALG_ZLIB GENMASK(1, 0) +#define HZIP_ALG_GZIP GENMASK(3, 2) + static const u8 zlib_head[HZIP_ZLIB_HEAD_SIZE] = {0x78, 0x9c}; static const u8 gzip_head[HZIP_GZIP_HEAD_SIZE] = { 0x1f, 0x8b, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x03 @@ -123,19 +126,19 @@ static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) if (ret || n == 0 || n > HISI_ACC_SGL_SGE_NR_MAX) return -EINVAL; - return param_set_int(val, kp); + return param_set_ushort(val, kp); } static const struct kernel_param_ops sgl_sge_nr_ops = { .set = sgl_sge_nr_set, - .get = param_get_int, + .get = param_get_ushort, }; static u16 sgl_sge_nr = HZIP_SGL_SGE_NR; module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); -static u16 get_extra_field_size(const u8 *start) +static u32 get_extra_field_size(const u8 *start) { return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN; } @@ -167,7 +170,7 @@ static u32 __get_gzip_head_size(const u8 *src) return size; } -static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl) +static u32 __maybe_unused get_gzip_head_size(struct scatterlist *sgl) { char buf[HZIP_GZIP_HEAD_BUF]; @@ -183,7 +186,7 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) int ret; ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size); - if (ret != head_size) { + if (unlikely(ret != head_size)) { pr_err("the head size of buffer is wrong (%d)!\n", ret); return -ENOMEM; } @@ -193,11 +196,11 @@ static int add_comp_head(struct scatterlist *dst, u8 req_type) static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type) { - if (!acomp_req->src || !acomp_req->slen) + if (unlikely(!acomp_req->src || !acomp_req->slen)) return -EINVAL; - if (req_type == HZIP_ALG_TYPE_GZIP && - acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT) + if (unlikely(req_type == HZIP_ALG_TYPE_GZIP && + acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)) return -EINVAL; switch (req_type) { @@ -230,6 +233,8 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, } set_bit(req_id, req_q->req_bitmap); + write_unlock(&req_q->req_lock); + req_cache = q + req_id; req_cache->req_id = req_id; req_cache->req = req; @@ -242,8 +247,6 @@ static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req, req_cache->dskip = 0; } - write_unlock(&req_q->req_lock); - return req_cache; } @@ -254,7 +257,6 @@ static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx, write_lock(&req_q->req_lock); clear_bit(req->req_id, req_q->req_bitmap); - memset(req, 0, sizeof(struct hisi_zip_req)); write_unlock(&req_q->req_lock); } @@ -339,7 +341,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, struct hisi_zip_sqe zip_sqe; int ret; - if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen) + if (unlikely(!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)) return -EINVAL; req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, @@ -365,7 +367,7 @@ static int hisi_zip_do_work(struct hisi_zip_req *req, /* send command to start a task */ atomic64_inc(&dfx->send_cnt); ret = hisi_qp_send(qp, &zip_sqe); - if (ret < 0) { + if (unlikely(ret < 0)) { atomic64_inc(&dfx->send_busy_cnt); ret = -EAGAIN; dev_dbg_ratelimited(dev, "failed to send request!\n"); @@ -417,7 +419,7 @@ static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) atomic64_inc(&dfx->recv_cnt); status = ops->get_status(sqe); - if (status != 0 && status != HZIP_NC_ERR) { + if (unlikely(status != 0 && status != HZIP_NC_ERR)) { dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n", (qp->alg_type == 0) ? "" : "de", qp->qp_id, status, sqe->produced); @@ -450,7 +452,7 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) /* let's output compression head now */ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type); - if (head_size < 0) { + if (unlikely(head_size < 0)) { dev_err_ratelimited(dev, "failed to add comp head (%d)!\n", head_size); return head_size; @@ -461,7 +463,7 @@ static int hisi_zip_acompress(struct acomp_req *acomp_req) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { + if (unlikely(ret != -EINPROGRESS)) { dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); } @@ -478,7 +480,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) int head_size, ret; head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type); - if (head_size < 0) { + if (unlikely(head_size < 0)) { dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n", head_size); return head_size; @@ -489,7 +491,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return PTR_ERR(req); ret = hisi_zip_do_work(req, qp_ctx); - if (ret != -EINPROGRESS) { + if (unlikely(ret != -EINPROGRESS)) { dev_info_ratelimited(dev, "failed to do decompress (%d)!\n", ret); hisi_zip_remove_req(qp_ctx, req); @@ -498,7 +500,7 @@ static int hisi_zip_adecompress(struct acomp_req *acomp_req) return ret; } -static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, +static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, int alg_type, int req_type) { struct device *dev = &qp->qm->pdev->dev; @@ -506,7 +508,7 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, qp->req_type = req_type; qp->alg_type = alg_type; - qp->qp_ctx = ctx; + qp->qp_ctx = qp_ctx; ret = hisi_qm_start_qp(qp, 0); if (ret < 0) { @@ -514,15 +516,15 @@ static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx, return ret; } - ctx->qp = qp; + qp_ctx->qp = qp; return 0; } -static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) +static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(ctx->qp); - hisi_qm_free_qps(&ctx->qp, 1); + hisi_qm_stop_qp(qp_ctx->qp); + hisi_qm_free_qps(&qp_ctx->qp, 1); } static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = { @@ -594,18 +596,19 @@ static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) { int i; - for (i = 1; i >= 0; i--) + for (i = 0; i < HZIP_CTX_Q_NUM; i++) hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); } static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) { + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; struct hisi_zip_req_q *req_q; int i, ret; for (i = 0; i < HZIP_CTX_Q_NUM; i++) { req_q = &ctx->qp_ctx[i].req_q; - req_q->size = QM_Q_DEPTH; + req_q->size = q_depth; req_q->req_bitmap = bitmap_zalloc(req_q->size, GFP_KERNEL); if (!req_q->req_bitmap) { @@ -613,7 +616,7 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) if (i == 0) return ret; - goto err_free_loop0; + goto err_free_comp_q; } rwlock_init(&req_q->req_lock); @@ -622,19 +625,19 @@ static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) if (!req_q->q) { ret = -ENOMEM; if (i == 0) - goto err_free_bitmap; + goto err_free_comp_bitmap; else - goto err_free_loop1; + goto err_free_decomp_bitmap; } } return 0; -err_free_loop1: +err_free_decomp_bitmap: bitmap_free(ctx->qp_ctx[HZIP_QPC_DECOMP].req_q.req_bitmap); -err_free_loop0: +err_free_comp_q: kfree(ctx->qp_ctx[HZIP_QPC_COMP].req_q.q); -err_free_bitmap: +err_free_comp_bitmap: bitmap_free(ctx->qp_ctx[HZIP_QPC_COMP].req_q.req_bitmap); return ret; } @@ -651,6 +654,7 @@ static void hisi_zip_release_req_q(struct hisi_zip_ctx *ctx) static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) { + u16 q_depth = ctx->qp_ctx[0].qp->sq_depth; struct hisi_zip_qp_ctx *tmp; struct device *dev; int i; @@ -658,7 +662,7 @@ static int hisi_zip_create_sgl_pool(struct hisi_zip_ctx *ctx) for (i = 0; i < HZIP_CTX_Q_NUM; i++) { tmp = &ctx->qp_ctx[i]; dev = &tmp->qp->qm->pdev->dev; - tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH << 1, + tmp->sgl_pool = hisi_acc_create_sgl_pool(dev, q_depth << 1, sgl_sge_nr); if (IS_ERR(tmp->sgl_pool)) { if (i == 1) @@ -755,6 +759,28 @@ static struct acomp_alg hisi_zip_acomp_zlib = { } }; +static int hisi_zip_register_zlib(struct hisi_qm *qm) +{ + int ret; + + if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) + return 0; + + ret = crypto_register_acomp(&hisi_zip_acomp_zlib); + if (ret) + dev_err(&qm->pdev->dev, "failed to register to zlib (%d)!\n", ret); + + return ret; +} + +static void hisi_zip_unregister_zlib(struct hisi_qm *qm) +{ + if (!hisi_zip_alg_support(qm, HZIP_ALG_ZLIB)) + return; + + crypto_unregister_acomp(&hisi_zip_acomp_zlib); +} + static struct acomp_alg hisi_zip_acomp_gzip = { .init = hisi_zip_acomp_init, .exit = hisi_zip_acomp_exit, @@ -769,27 +795,45 @@ static struct acomp_alg hisi_zip_acomp_gzip = { } }; -int hisi_zip_register_to_crypto(struct hisi_qm *qm) +static int hisi_zip_register_gzip(struct hisi_qm *qm) { int ret; - ret = crypto_register_acomp(&hisi_zip_acomp_zlib); - if (ret) { - pr_err("failed to register to zlib (%d)!\n", ret); - return ret; - } + if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) + return 0; ret = crypto_register_acomp(&hisi_zip_acomp_gzip); - if (ret) { - pr_err("failed to register to gzip (%d)!\n", ret); - crypto_unregister_acomp(&hisi_zip_acomp_zlib); - } + if (ret) + dev_err(&qm->pdev->dev, "failed to register to gzip (%d)!\n", ret); return ret; } -void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) +static void hisi_zip_unregister_gzip(struct hisi_qm *qm) { + if (!hisi_zip_alg_support(qm, HZIP_ALG_GZIP)) + return; + crypto_unregister_acomp(&hisi_zip_acomp_gzip); - crypto_unregister_acomp(&hisi_zip_acomp_zlib); +} + +int hisi_zip_register_to_crypto(struct hisi_qm *qm) +{ + int ret = 0; + + ret = hisi_zip_register_zlib(qm); + if (ret) + return ret; + + ret = hisi_zip_register_gzip(qm); + if (ret) + hisi_zip_unregister_zlib(qm); + + return ret; +} + +void hisi_zip_unregister_from_crypto(struct hisi_qm *qm) +{ + hisi_zip_unregister_zlib(qm); + hisi_zip_unregister_gzip(qm); } diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index c3303d99acac7b679d7428aaf226ee03930bd192..c863435e8c75aadc5c57c2d98900b251b6f99fe1 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -20,18 +20,6 @@ #define HZIP_QUEUE_NUM_V1 4096 #define HZIP_CLOCK_GATE_CTRL 0x301004 -#define COMP0_ENABLE BIT(0) -#define COMP1_ENABLE BIT(1) -#define DECOMP0_ENABLE BIT(2) -#define DECOMP1_ENABLE BIT(3) -#define DECOMP2_ENABLE BIT(4) -#define DECOMP3_ENABLE BIT(5) -#define DECOMP4_ENABLE BIT(6) -#define DECOMP5_ENABLE BIT(7) -#define HZIP_ALL_COMP_DECOMP_EN (COMP0_ENABLE | COMP1_ENABLE | \ - DECOMP0_ENABLE | DECOMP1_ENABLE | \ - DECOMP2_ENABLE | DECOMP3_ENABLE | \ - DECOMP4_ENABLE | DECOMP5_ENABLE) #define HZIP_DECOMP_CHECK_ENABLE BIT(16) #define HZIP_FSM_MAX_CNT 0x301008 @@ -69,20 +57,14 @@ #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 #define HZIP_CORE_INT_RAS_CE_ENB 0x301160 -#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_CORE_INT_RAS_FE_ENB_MASK 0x0 #define HZIP_OOO_SHUTDOWN_SEL 0x30120C -#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 #define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0) -#define HZIP_COMP_CORE_NUM 2 -#define HZIP_DECOMP_CORE_NUM 6 -#define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \ - HZIP_DECOMP_CORE_NUM) #define HZIP_SQE_SIZE 128 -#define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH) #define HZIP_PF_DEF_Q_NUM 64 #define HZIP_PF_DEF_Q_BASE 0 @@ -92,6 +74,12 @@ #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) +#define HZIP_DEV_ALG_MAX_LEN 256 +#define HZIP_ALG_ZLIB_BIT GENMASK(1, 0) +#define HZIP_ALG_GZIP_BIT GENMASK(3, 2) +#define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) +#define HZIP_ALG_LZ77_BIT GENMASK(7, 6) + #define HZIP_BUF_SIZE 22 #define HZIP_SQE_MASK_OFFSET 64 #define HZIP_SQE_MASK_LEN 48 @@ -132,6 +120,26 @@ struct zip_dfx_item { u32 offset; }; +struct zip_dev_alg { + u32 alg_msk; + const char *algs; +}; + +static const struct zip_dev_alg zip_dev_algs[] = { { + .alg_msk = HZIP_ALG_ZLIB_BIT, + .algs = "zlib\n", + }, { + .alg_msk = HZIP_ALG_GZIP_BIT, + .algs = "gzip\n", + }, { + .alg_msk = HZIP_ALG_DEFLATE_BIT, + .algs = "deflate\n", + }, { + .alg_msk = HZIP_ALG_LZ77_BIT, + .algs = "lz77_zstd\n", + }, +}; + static struct hisi_qm_list zip_devices = { .register_to_crypto = hisi_zip_register_to_crypto, .unregister_from_crypto = hisi_zip_unregister_from_crypto, @@ -187,6 +195,58 @@ struct hisi_zip_ctrl { struct ctrl_debug_file files[HZIP_DEBUG_FILE_NUM]; }; +enum zip_cap_type { + ZIP_QM_NFE_MASK_CAP = 0x0, + ZIP_QM_RESET_MASK_CAP, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, + ZIP_QM_CE_MASK_CAP, + ZIP_NFE_MASK_CAP, + ZIP_RESET_MASK_CAP, + ZIP_OOO_SHUTDOWN_MASK_CAP, + ZIP_CE_MASK_CAP, + ZIP_CLUSTER_NUM_CAP, + ZIP_CORE_TYPE_NUM_CAP, + ZIP_CORE_NUM_CAP, + ZIP_CLUSTER_COMP_NUM_CAP, + ZIP_CLUSTER_DECOMP_NUM_CAP, + ZIP_DECOMP_ENABLE_BITMAP, + ZIP_COMP_ENABLE_BITMAP, + ZIP_DRV_ALG_BITMAP, + ZIP_DEV_ALG_BITMAP, + ZIP_CORE1_ALG_BITMAP, + ZIP_CORE2_ALG_BITMAP, + ZIP_CORE3_ALG_BITMAP, + ZIP_CORE4_ALG_BITMAP, + ZIP_CORE5_ALG_BITMAP, + ZIP_CAP_MAX +}; + +static struct hisi_qm_cap_info zip_basic_cap_info[] = { + {ZIP_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C57, 0x7C77}, + {ZIP_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC57, 0x6C77}, + {ZIP_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C77}, + {ZIP_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8}, + {ZIP_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x1FFE}, + {ZIP_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x7FE, 0x7FE}, + {ZIP_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x2, 0x7FE}, + {ZIP_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, + {ZIP_CLUSTER_NUM_CAP, 0x313C, 28, GENMASK(3, 0), 0x1, 0x1, 0x1}, + {ZIP_CORE_TYPE_NUM_CAP, 0x313C, 24, GENMASK(3, 0), 0x2, 0x2, 0x2}, + {ZIP_CORE_NUM_CAP, 0x313C, 16, GENMASK(7, 0), 0x8, 0x8, 0x5}, + {ZIP_CLUSTER_COMP_NUM_CAP, 0x313C, 8, GENMASK(7, 0), 0x2, 0x2, 0x2}, + {ZIP_CLUSTER_DECOMP_NUM_CAP, 0x313C, 0, GENMASK(7, 0), 0x6, 0x6, 0x3}, + {ZIP_DECOMP_ENABLE_BITMAP, 0x3140, 16, GENMASK(15, 0), 0xFC, 0xFC, 0x1C}, + {ZIP_COMP_ENABLE_BITMAP, 0x3140, 0, GENMASK(15, 0), 0x3, 0x3, 0x3}, + {ZIP_DRV_ALG_BITMAP, 0x3144, 0, GENMASK(31, 0), 0xF, 0xF, 0xF}, + {ZIP_DEV_ALG_BITMAP, 0x3148, 0, GENMASK(31, 0), 0xF, 0xF, 0xFF}, + {ZIP_CORE1_ALG_BITMAP, 0x314C, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE2_ALG_BITMAP, 0x3150, 0, GENMASK(31, 0), 0x5, 0x5, 0xD5}, + {ZIP_CORE3_ALG_BITMAP, 0x3154, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE4_ALG_BITMAP, 0x3158, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CORE5_ALG_BITMAP, 0x315C, 0, GENMASK(31, 0), 0xA, 0xA, 0x2A}, + {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} +}; + enum { HZIP_COMP_CORE0, HZIP_COMP_CORE1, @@ -343,12 +403,52 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); } +bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) +{ + u32 cap_val; + + cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DRV_ALG_BITMAP, qm->cap_ver); + if ((alg & cap_val) == alg) + return true; + + return false; +} + +static int hisi_zip_set_qm_algs(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + u32 alg_mask; + int i; + + if (!qm->use_sva) + return 0; + + algs = devm_kzalloc(dev, HZIP_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + alg_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); + + for (i = 0; i < ARRAY_SIZE(zip_dev_algs); i++) + if (alg_mask & zip_dev_algs[i].alg_msk) + strcat(algs, zip_dev_algs[i].algs); + + ptr = strrchr(algs, '\n'); + if (ptr) + *ptr = '\0'; + + qm->uacce->algs = algs; + + return 0; +} + static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) { u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; /* Enable prefetch */ @@ -368,7 +468,7 @@ static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) u32 val; int ret; - if (qm->ver < QM_HW_V3) + if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) return; val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); @@ -401,6 +501,7 @@ static void hisi_zip_enable_clock_gate(struct hisi_qm *qm) static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) { void __iomem *base = qm->io_base; + u32 dcomp_bm, comp_bm; /* qm user domain */ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); @@ -438,8 +539,11 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - writel(HZIP_DECOMP_CHECK_ENABLE | HZIP_ALL_COMP_DECOMP_EN, - base + HZIP_CLOCK_GATE_CTRL); + dcomp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_DECOMP_ENABLE_BITMAP, qm->cap_ver); + comp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_COMP_ENABLE_BITMAP, qm->cap_ver); + writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL); /* enable sqc,cqc writeback */ writel(SQC_CACHE_ENABLE | CQC_CACHE_ENABLE | SQC_CACHE_WB_ENABLE | @@ -458,7 +562,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); if (enable) { val1 |= HZIP_AXI_SHUTDOWN_ENABLE; - val2 = HZIP_CORE_INT_RAS_NFE_ENABLE; + val2 = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); } else { val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; val2 = 0x0; @@ -472,6 +577,8 @@ static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { + u32 nfe, ce; + if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -479,17 +586,17 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) return; } + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + /* clear ZIP hw error source if having */ - writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); + writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(HZIP_CORE_INT_RAS_CE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); - writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); - writel(HZIP_CORE_INT_RAS_NFE_ENABLE, - qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + writel(ce, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); + writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); - /* enable ZIP block master OOO when nfe occurs on Kunpeng930 */ hisi_zip_master_ooo_ctrl(qm, true); /* enable ZIP hw error interrupts */ @@ -498,10 +605,13 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { + u32 nfe, ce; + /* disable ZIP hw error interrupts */ - writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CE_MASK_CAP, qm->cap_ver); + writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); - /* disable ZIP block master OOO when nfe occurs on Kunpeng930 */ hisi_zip_master_ooo_ctrl(qm, false); } @@ -586,8 +696,9 @@ static ssize_t hisi_zip_ctrl_debug_write(struct file *filp, return len; tbuf[len] = '\0'; - if (kstrtoul(tbuf, 0, &val)) - return -EFAULT; + ret = kstrtoul(tbuf, 0, &val); + if (ret) + return ret; ret = hisi_qm_get_dfx_access(qm); if (ret) @@ -651,18 +762,23 @@ DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs); static int hisi_zip_core_debug_init(struct hisi_qm *qm) { + u32 zip_core_num, zip_comp_core_num; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; struct dentry *tmp_d; char buf[HZIP_BUF_SIZE]; int i; - for (i = 0; i < HZIP_CORE_NUM; i++) { - if (i < HZIP_COMP_CORE_NUM) + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, + qm->cap_ver); + + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "comp_core%d", i); else scnprintf(buf, sizeof(buf), "decomp_core%d", - i - HZIP_COMP_CORE_NUM); + i - zip_comp_core_num); regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); if (!regset) @@ -675,7 +791,7 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) tmp_d = debugfs_create_dir(buf, qm->debug.debug_root); debugfs_create_file("regs", 0444, tmp_d, regset, - &hisi_zip_regs_fops); + &hisi_zip_regs_fops); } return 0; @@ -795,10 +911,13 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u32 zip_core_num; int i, j, idx; - debug->last_words = kcalloc(core_dfx_regs_num * HZIP_CORE_NUM + - com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + + debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, + sizeof(unsigned int), GFP_KERNEL); if (!debug->last_words) return -ENOMEM; @@ -807,7 +926,7 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) debug->last_words[i] = readl_relaxed(io_base); } - for (i = 0; i < HZIP_CORE_NUM; i++) { + for (i = 0; i < zip_core_num; i++) { io_base = qm->io_base + core_offsets[i]; for (j = 0; j < core_dfx_regs_num; j++) { idx = com_dfx_regs_num + i * core_dfx_regs_num + j; @@ -834,6 +953,7 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) { int core_dfx_regs_num = ARRAY_SIZE(hzip_dump_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); + u32 zip_core_num, zip_comp_core_num; struct qm_debug *debug = &qm->debug; char buf[HZIP_BUF_SIZE]; void __iomem *base; @@ -847,15 +967,18 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) val = readl_relaxed(qm->io_base + hzip_com_dfx_regs[i].offset); if (debug->last_words[i] != val) pci_info(qm->pdev, "com_dfx: %s \t= 0x%08x => 0x%08x\n", - hzip_com_dfx_regs[i].name, debug->last_words[i], val); + hzip_com_dfx_regs[i].name, debug->last_words[i], val); } - for (i = 0; i < HZIP_CORE_NUM; i++) { - if (i < HZIP_COMP_CORE_NUM) + zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, + qm->cap_ver); + for (i = 0; i < zip_core_num; i++) { + if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "Comp_core-%d", i); else scnprintf(buf, sizeof(buf), "Decomp_core-%d", - i - HZIP_COMP_CORE_NUM); + i - zip_comp_core_num); base = qm->io_base + core_offsets[i]; pci_info(qm->pdev, "==>%s:\n", buf); @@ -865,7 +988,8 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) val = readl_relaxed(base + hzip_dump_dfx_regs[j].offset); if (debug->last_words[idx] != val) pci_info(qm->pdev, "%s \t= 0x%08x => 0x%08x\n", - hzip_dump_dfx_regs[j].name, debug->last_words[idx], val); + hzip_dump_dfx_regs[j].name, + debug->last_words[idx], val); } } } @@ -900,7 +1024,11 @@ static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm) static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { + u32 nfe; + writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE); + nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); } static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) @@ -934,16 +1062,21 @@ static void hisi_zip_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; - err_info->ce = QM_BASE_CE; - err_info->fe = 0; + err_info->fe = HZIP_CORE_INT_RAS_FE_ENB_MASK; + err_info->ce = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_QM_CE_MASK_CAP, qm->cap_ver); + err_info->nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_NFE_MASK_CAP, qm->cap_ver); err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC; - err_info->dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE; + err_info->qm_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->dev_shutdown_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver); + err_info->qm_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_QM_RESET_MASK_CAP, qm->cap_ver); + err_info->dev_reset_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + ZIP_RESET_MASK_CAP, qm->cap_ver); err_info->msi_wr_port = HZIP_WR_PORT; err_info->acpi_rst = "ZRST"; - err_info->nfe = QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT; - - if (qm->ver >= QM_HW_V3) - err_info->nfe |= QM_ACC_DO_TASK_TIMEOUT; } static const struct hisi_qm_err_ini hisi_zip_err_ini = { @@ -976,7 +1109,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) qm->err_ini = &hisi_zip_err_ini; qm->err_ini->err_info_init(qm); - hisi_zip_set_user_domain_and_cache(qm); + ret = hisi_zip_set_user_domain_and_cache(qm); + if (ret) + return ret; + hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -990,12 +1126,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; - if (pdev->revision >= QM_HW_V3) - qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd"; - else - qm->algs = "zlib\ngzip"; qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -1019,7 +1153,19 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + ret = hisi_qm_init(qm); + if (ret) { + pci_err(qm->pdev, "Failed to init zip qm configures!\n"); + return ret; + } + + ret = hisi_zip_set_qm_algs(qm); + if (ret) { + pci_err(qm->pdev, "Failed to set zip algs!\n"); + hisi_qm_uninit(qm); + } + + return ret; } static void hisi_zip_qm_uninit(struct hisi_qm *qm) diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index d68ef16650d47da360eefaeb044e833c755af2bd..32a37e3850c585ace08ccd660448e5ed59dfe04f 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -63,7 +63,6 @@ struct safexcel_cipher_ctx { u32 hash_alg; u32 state_sz; - struct crypto_cipher *hkaes; struct crypto_aead *fback; }; @@ -642,10 +641,16 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin safexcel_complete(priv, ring); if (src == dst) { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } /* @@ -737,23 +742,29 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, max(totlen_src, totlen_dst)); return -EINVAL; } - dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_map_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { if (unlikely(totlen_src && (sreq->nr_src <= 0))) { dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", totlen_src); return -EINVAL; } - dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); + + if (sreq->nr_src > 0) + dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", totlen_dst); - dma_unmap_sg(priv->dev, src, sreq->nr_src, - DMA_TO_DEVICE); - return -EINVAL; + ret = -EINVAL; + goto unmap; } - dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + + if (sreq->nr_dst > 0) + dma_map_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); @@ -883,12 +894,18 @@ rdesc_rollback: cdesc_rollback: for (i = 0; i < n_cdesc; i++) safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); - +unmap: if (src == dst) { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); + if (sreq->nr_src > 0) + dma_unmap_sg(priv->dev, src, sreq->nr_src, + DMA_TO_DEVICE); + if (sreq->nr_dst > 0) + dma_unmap_sg(priv->dev, dst, sreq->nr_dst, + DMA_FROM_DEVICE); } return ret; @@ -2589,15 +2606,8 @@ static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, ctx->key_len = len; /* Compute hash key by encrypting zeroes with cipher key */ - crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->hkaes, key, len); - if (ret) - return ret; - memset(hashkey, 0, AES_BLOCK_SIZE); - crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); + aes_encrypt(&aes, (u8 *)hashkey, (u8 *)hashkey); if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { @@ -2626,15 +2636,11 @@ static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) ctx->xcm = EIP197_XCM_MODE_GCM; ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ - ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); - return PTR_ERR_OR_ZERO(ctx->hkaes); + return 0; } static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) { - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_cipher(ctx->hkaes); safexcel_aead_cra_exit(tfm); } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index bc60b5802256453399c33ebacb5a5b325d915955..103fc551d2af902262ef9fef45a5c95e87b92a21 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -30,7 +30,7 @@ struct safexcel_ahash_ctx { bool fb_init_done; bool fb_do_setkey; - struct crypto_cipher *kaes; + struct crypto_aes_ctx *aes; struct crypto_ahash *fback; struct crypto_shash *shpre; struct shash_desc *shdesc; @@ -383,7 +383,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, u32 x; x = ipad[i] ^ ipad[i + 4]; - cache[i] ^= swab(x); + cache[i] ^= swab32(x); } } cache_len = AES_BLOCK_SIZE; @@ -821,10 +821,10 @@ static int safexcel_ahash_final(struct ahash_request *areq) u32 *result = (void *)areq->result; /* K3 */ - result[i] = swab(ctx->base.ipad.word[i + 4]); + result[i] = swab32(ctx->base.ipad.word[i + 4]); } areq->result[0] ^= 0x80; // 10- padding - crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result); + aes_encrypt(ctx->aes, areq->result, areq->result); return 0; } else if (unlikely(req->hmac && (req->len == req->block_sz) && @@ -2083,37 +2083,26 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int len) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct crypto_aes_ctx aes; u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)]; int ret, i; - ret = aes_expandkey(&aes, key, len); + ret = aes_expandkey(ctx->aes, key, len); if (ret) return ret; /* precompute the XCBC key material */ - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, key, len); - if (ret) - return ret; - - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, - "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp, - "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE, - "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); + aes_encrypt(ctx->aes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); + aes_encrypt(ctx->aes, (u8 *)key_tmp, + "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2"); + aes_encrypt(ctx->aes, (u8 *)key_tmp + AES_BLOCK_SIZE, + "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3"); for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++) - ctx->base.ipad.word[i] = swab(key_tmp[i]); - - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, - (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, - AES_MIN_KEY_SIZE); + ctx->base.ipad.word[i] = swab32(key_tmp[i]); + + ret = aes_expandkey(ctx->aes, + (u8 *)key_tmp + 2 * AES_BLOCK_SIZE, + AES_MIN_KEY_SIZE); if (ret) return ret; @@ -2121,7 +2110,6 @@ static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key, ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE; ctx->cbcmac = false; - memzero_explicit(&aes, sizeof(aes)); return 0; } @@ -2130,15 +2118,15 @@ static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); safexcel_ahash_cra_init(tfm); - ctx->kaes = crypto_alloc_cipher("aes", 0, 0); - return PTR_ERR_OR_ZERO(ctx->kaes); + ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); + return PTR_ERR_OR_ZERO(ctx->aes); } static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_cipher(ctx->kaes); + kfree(ctx->aes); safexcel_ahash_cra_exit(tfm); } @@ -2178,31 +2166,23 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int len) { struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct crypto_aes_ctx aes; __be64 consts[4]; u64 _const[2]; u8 msb_mask, gfmask; int ret, i; - ret = aes_expandkey(&aes, key, len); + /* precompute the CMAC key material */ + ret = aes_expandkey(ctx->aes, key, len); if (ret) return ret; for (i = 0; i < len / sizeof(u32); i++) - ctx->base.ipad.word[i + 8] = swab(aes.key_enc[i]); - - /* precompute the CMAC key material */ - crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK); - crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - ret = crypto_cipher_setkey(ctx->kaes, key, len); - if (ret) - return ret; + ctx->base.ipad.word[i + 8] = swab32(ctx->aes->key_enc[i]); /* code below borrowed from crypto/cmac.c */ /* encrypt the zero block */ memset(consts, 0, AES_BLOCK_SIZE); - crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts); + aes_encrypt(ctx->aes, (u8 *)consts, (u8 *)consts); gfmask = 0x87; _const[0] = be64_to_cpu(consts[1]); @@ -2234,7 +2214,6 @@ static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, } ctx->cbcmac = false; - memzero_explicit(&aes, sizeof(aes)); return 0; } diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index 7942b48dd55a8a75869c147de45d7e987ea962b0..1cd62f9c3e3a6143caeb1c34891829f1127a254b 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -42,7 +42,7 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS config CRYPTO_DEV_KEEMBAY_OCS_ECC tristate "Support for Intel Keem Bay OCS ECC HW acceleration" depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF || COMPILE_TEST + depends on OF depends on HAS_IOMEM select CRYPTO_ECDH select CRYPTO_ENGINE @@ -64,7 +64,7 @@ config CRYPTO_DEV_KEEMBAY_OCS_HCU select CRYPTO_ENGINE depends on HAS_IOMEM depends on ARCH_KEEMBAY || COMPILE_TEST - depends on OF || COMPILE_TEST + depends on OF help Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash Control Unit (HCU) hardware acceleration for use with Crypto API. diff --git a/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h index b8bdb9f134f31958e40df90d826011afc771d345..205eacac4a348bc995ed99468af1d8d2b19c2107 100644 --- a/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h +++ b/drivers/crypto/marvell/octeontx/otx_cpt_hw_types.h @@ -403,7 +403,7 @@ union otx_cptx_pf_exe_bist_status { * big-endian format in memory. * iqb_ldwb:1 [7:7](R/W) Instruction load don't write back. * 0 = The hardware issues NCB transient load (LDT) towards the cache, - * which if the line hits and is is dirty will cause the line to be + * which if the line hits and is dirty will cause the line to be * written back before being replaced. * 1 = The hardware issues NCB LDWB read-and-invalidate command towards * the cache when fetching the last word of instructions; as a result the diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c index 40b482198ebc566c73f19bcd1fdce00ea82ab6ba..df9c2b8747e6dd0c15c1ceaad085c93a4fe31daf 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c @@ -97,7 +97,7 @@ static int dev_supports_eng_type(struct otx_cpt_eng_grps *eng_grps, static void set_ucode_filename(struct otx_cpt_ucode *ucode, const char *filename) { - strlcpy(ucode->filename, filename, OTX_CPT_UCODE_NAME_LENGTH); + strscpy(ucode->filename, filename, OTX_CPT_UCODE_NAME_LENGTH); } static char *get_eng_type_str(int eng_type) @@ -138,7 +138,7 @@ static int get_ucode_type(struct otx_cpt_ucode_hdr *ucode_hdr, int *ucode_type) u32 i, val = 0; u8 nn; - strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); + strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); for (i = 0; i < strlen(tmp_ver_str); i++) tmp_ver_str[i] = tolower(tmp_ver_str[i]); @@ -286,6 +286,7 @@ static int process_tar_file(struct device *dev, struct tar_ucode_info_t *tar_info; struct otx_cpt_ucode_hdr *ucode_hdr; int ucode_type, ucode_size; + unsigned int code_length; /* * If size is less than microcode header size then don't report @@ -303,7 +304,13 @@ static int process_tar_file(struct device *dev, if (get_ucode_type(ucode_hdr, &ucode_type)) return 0; - ucode_size = ntohl(ucode_hdr->code_length) * 2; + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Invalid code_length %u\n", code_length); + return -EINVAL; + } + + ucode_size = code_length * 2; if (!ucode_size || (size < round_up(ucode_size, 16) + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { dev_err(dev, "Ucode %s invalid size\n", filename); @@ -886,6 +893,7 @@ static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode, { struct otx_cpt_ucode_hdr *ucode_hdr; const struct firmware *fw; + unsigned int code_length; int ret; set_ucode_filename(ucode, ucode_filename); @@ -896,7 +904,13 @@ static int ucode_load(struct device *dev, struct otx_cpt_ucode *ucode, ucode_hdr = (struct otx_cpt_ucode_hdr *) fw->data; memcpy(ucode->ver_str, ucode_hdr->ver_str, OTX_CPT_UCODE_VER_STR_SZ); ucode->ver_num = ucode_hdr->ver_num; - ucode->size = ntohl(ucode_hdr->code_length) * 2; + code_length = ntohl(ucode_hdr->code_length); + if (code_length >= INT_MAX / 2) { + dev_err(dev, "Ucode invalid code_length %u\n", code_length); + ret = -EINVAL; + goto release_fw; + } + ucode->size = code_length * 2; if (!ucode->size || (fw->size < round_up(ucode->size, 16) + sizeof(struct otx_cpt_ucode_hdr) + OTX_CPT_UCODE_SIGN_LEN)) { dev_err(dev, "Ucode %s invalid size\n", ucode_filename); @@ -1328,7 +1342,7 @@ static ssize_t ucode_load_store(struct device *dev, eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr); err_msg = "Invalid engine group format"; - strlcpy(tmp_buf, buf, OTX_CPT_UCODE_NAME_LENGTH); + strscpy(tmp_buf, buf, OTX_CPT_UCODE_NAME_LENGTH); start = tmp_buf; has_se = has_ie = has_ae = false; diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 36d72e35ebeb6aa9af979fe503dca617b5089418..88a41d1ca5f6445abce0ada23886f9746aff2f31 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -661,7 +661,7 @@ static ssize_t vf_type_show(struct device *dev, msg = "Invalid"; } - return scnprintf(buf, PAGE_SIZE, "%s\n", msg); + return sysfs_emit(buf, "%s\n", msg); } static ssize_t vf_engine_group_show(struct device *dev, @@ -670,7 +670,7 @@ static ssize_t vf_engine_group_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", cptvf->vfgrp); + return sysfs_emit(buf, "%d\n", cptvf->vfgrp); } static ssize_t vf_engine_group_store(struct device *dev, @@ -706,7 +706,7 @@ static ssize_t vf_coalesc_time_wait_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", + return sysfs_emit(buf, "%d\n", cptvf_read_vq_done_timewait(cptvf)); } @@ -716,7 +716,7 @@ static ssize_t vf_coalesc_num_wait_show(struct device *dev, { struct otx_cptvf *cptvf = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", + return sysfs_emit(buf, "%d\n", cptvf_read_vq_done_numwait(cptvf)); } diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c index 5663787c7a62a7f1e962975f5f19c226650e88c8..90fdafb7c4685bd1a6473f66acdf887161a4d53a 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_mbox.c @@ -159,12 +159,10 @@ static int cptvf_send_msg_to_pf_timeout(struct otx_cptvf *cptvf, int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_READY; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -174,13 +172,11 @@ int otx_cptvf_check_pf_ready(struct otx_cptvf *cptvf) int otx_cptvf_send_vq_size_msg(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_QLEN; mbx.data = cptvf->qsize; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -208,14 +204,12 @@ int otx_cptvf_send_vf_to_grp_msg(struct otx_cptvf *cptvf, int group) int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VQ_PRIORITY; /* Convey group of the VF */ mbx.data = cptvf->priority; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -224,12 +218,10 @@ int otx_cptvf_send_vf_priority_msg(struct otx_cptvf *cptvf) int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VF_UP; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } /* @@ -238,10 +230,8 @@ int otx_cptvf_send_vf_up(struct otx_cptvf *cptvf) int otx_cptvf_send_vf_down(struct otx_cptvf *cptvf) { struct otx_cpt_mbox mbx = {}; - int ret; mbx.msg = OTX_CPT_MSG_VF_DOWN; - ret = cptvf_send_msg_to_pf_timeout(cptvf, &mbx); - return ret; + return cptvf_send_msg_to_pf_timeout(cptvf, &mbx); } diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index f10050fead1641db4840fb3e20689e74ea3125f1..1577986677f60d25f4170d56a8f5b140d215b98b 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -68,7 +68,7 @@ static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp) static void set_ucode_filename(struct otx2_cpt_ucode *ucode, const char *filename) { - strlcpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); + strscpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); } static char *get_eng_type_str(int eng_type) @@ -126,7 +126,7 @@ static int get_ucode_type(struct device *dev, int i, val = 0; u8 nn; - strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); + strscpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); for (i = 0; i < strlen(tmp_ver_str); i++) tmp_ver_str[i] = tolower(tmp_ver_str[i]); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index 02cb9e44afd81b90a3f42e9ef739bc4a58e99c94..75c403f2b1d98503dabd12ba31de9023d84747df 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -191,7 +191,6 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) struct otx2_mbox *mbox = &cptvf->pfvf_mbox; struct pci_dev *pdev = cptvf->pdev; struct mbox_msghdr *req; - int ret; req = (struct mbox_msghdr *) otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), @@ -204,7 +203,5 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) req->sig = OTX2_MBOX_REQ_SIG; req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); - ret = otx2_cpt_send_mbox_msg(mbox, pdev); - - return ret; + return otx2_cpt_send_mbox_msg(mbox, pdev); } diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3b0bf6fea491a3dc35e37937e42f56ce79c1816f..31e24df18877f52a07b6d55c36a2b1d78da1cd2d 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1494,7 +1494,7 @@ static void n2_unregister_algs(void) * * So we have to back-translate, going through the 'intr' and 'ino' * property tables of the n2cp MDESC node, matching it with the OF - * 'interrupts' property entries, in order to to figure out which + * 'interrupts' property entries, in order to figure out which * devino goes to which already-translated IRQ. */ static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c index 3793885f928dd33e8fa25b6f9fa2ee5aa1ef3ced..c843f4c6f684d4b19c1cd7b9842105a2108b1b32 100644 --- a/drivers/crypto/nx/nx-aes-ccm.c +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -134,7 +134,6 @@ static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, unsigned int cryptlen, u8 *b0) { unsigned int l, lp, m = authsize; - int rc; memcpy(b0, iv, 16); @@ -148,9 +147,7 @@ static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize, if (assoclen) *b0 |= 64; - rc = set_msg_len(b0 + 16 - l, cryptlen, l); - - return rc; + return set_msg_len(b0 + 16 - l, cryptlen, l); } static int generate_pat(u8 *iv, diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index e61b3e13db3bdcd071365bc63439ac9e9d409aa3..1931e5b37f2bd16d84b493a369aaaac74f0c53d1 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -251,13 +251,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -ENOMEM; INIT_LIST_HEAD(&key_val->list); - strlcpy(key_val->key, key, sizeof(key_val->key)); + strscpy(key_val->key, key, sizeof(key_val->key)); if (type == ADF_DEC) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "%ld", (*((long *)val))); } else if (type == ADF_STR) { - strlcpy(key_val->val, (char *)val, sizeof(key_val->val)); + strscpy(key_val->val, (char *)val, sizeof(key_val->val)); } else if (type == ADF_HEX) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "0x%lx", (unsigned long)val); @@ -315,7 +315,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) if (!sec) return -ENOMEM; - strlcpy(sec->name, name, sizeof(sec->name)); + strscpy(sec->name, name, sizeof(sec->name)); INIT_LIST_HEAD(&sec->param_head); down_write(&cfg->lock); list_add_tail(&sec->list, &cfg->sec_list); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index e8ac932bbaab6aeb799d48609df46a604029f56d..82b69e1f725ba278d76fdd414cb7b67e7d450073 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -16,6 +16,9 @@ #include "adf_cfg_common.h" #include "adf_cfg_user.h" +#define ADF_CFG_MAX_SECTION 512 +#define ADF_CFG_MAX_KEY_VAL 256 + #define DEVICE_NAME "qat_adf_ctl" static DEFINE_MUTEX(adf_ctl_lock); @@ -137,10 +140,11 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, struct adf_user_cfg_key_val key_val; struct adf_user_cfg_key_val *params_head; struct adf_user_cfg_section section, *section_head; + int i, j; section_head = ctl_data->config_section; - while (section_head) { + for (i = 0; section_head && i < ADF_CFG_MAX_SECTION; i++) { if (copy_from_user(§ion, (void __user *)section_head, sizeof(*section_head))) { dev_err(&GET_DEV(accel_dev), @@ -156,7 +160,7 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, params_head = section.params; - while (params_head) { + for (j = 0; params_head && j < ADF_CFG_MAX_KEY_VAL; j++) { if (copy_from_user(&key_val, (void __user *)params_head, sizeof(key_val))) { dev_err(&GET_DEV(accel_dev), @@ -363,7 +367,7 @@ static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, dev_info.num_logical_accel = hw_data->num_logical_accel; dev_info.banks_per_accel = hw_data->num_banks / hw_data->num_logical_accel; - strlcpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); + strscpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); dev_info.instance_id = hw_data->instance_id; dev_info.type = hw_data->dev_class->type; dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h index 43b8f864806bdfc28c024fa23382a6c97c9b8a64..4fb4b3df5a1885ac11c16a3bf3e9a2af999caf03 100644 --- a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -107,7 +107,7 @@ do { \ * Timeout is in cycles. Clock speed may vary across products but this * value should be a few milli-seconds. */ -#define ADF_SSM_WDT_DEFAULT_VALUE 0x200000 +#define ADF_SSM_WDT_DEFAULT_VALUE 0x7000000ULL #define ADF_SSM_WDT_PKE_DEFAULT_VALUE 0x8000000 #define ADF_SSMWDTL_OFFSET 0x54 #define ADF_SSMWDTH_OFFSET 0x5C diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index e69e5907f5950e60e2b74d9ba03ee6acc4a11ae3..08bca1c506c0efb44a995fb95f74b843afc99791 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -96,7 +96,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) if (!ring_debug) return -ENOMEM; - strlcpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); snprintf(entry_name, sizeof(entry_name), "ring_%02d", ring->ring_number); diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 4b36869bf460bb750b6b91a36b73ab6b34e62bae..69482abdb8b936c5c8ad194d9263f0bc136b2871 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -86,7 +86,8 @@ ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ ICP_QAT_CSS_SIGNATURE_LEN(handle)) -#define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 +#define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 +#define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index fb45fa83841c532875fccc38d1f8ace85b06ae1d..cad9c58caab13504ab29e66fa2e0bd5eaefc94f9 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -673,11 +673,14 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, dma_addr_t blpout = qat_req->buf.bloutp; size_t sz = qat_req->buf.sz; size_t sz_out = qat_req->buf.sz_out; + int bl_dma_dir; int i; + bl_dma_dir = blp != blpout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + for (i = 0; i < bl->num_bufs; i++) dma_unmap_single(dev, bl->bufers[i].addr, - bl->bufers[i].len, DMA_BIDIRECTIONAL); + bl->bufers[i].len, bl_dma_dir); dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); @@ -691,7 +694,7 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, for (i = bufless; i < blout->num_bufs; i++) { dma_unmap_single(dev, blout->bufers[i].addr, blout->bufers[i].len, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); } dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); @@ -716,6 +719,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct scatterlist *sg; size_t sz_out, sz = struct_size(bufl, bufers, n); int node = dev_to_node(&GET_DEV(inst->accel_dev)); + int bufl_dma_dir; if (unlikely(!n)) return -EINVAL; @@ -733,6 +737,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, qat_req->buf.sgl_src_valid = true; } + bufl_dma_dir = sgl != sglout ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + for_each_sg(sgl, sg, n, i) bufl->bufers[i].addr = DMA_MAPPING_ERROR; @@ -744,7 +750,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, - DMA_BIDIRECTIONAL); + bufl_dma_dir); bufl->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) goto err_in; @@ -787,7 +793,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(dev, bufers[y].addr))) goto err_out; bufers[y].len = sg->length; @@ -817,7 +823,7 @@ err_out: if (!dma_mapping_error(dev, buflout->bufers[i].addr)) dma_unmap_single(dev, buflout->bufers[i].addr, buflout->bufers[i].len, - DMA_BIDIRECTIONAL); + DMA_FROM_DEVICE); if (!qat_req->buf.sgl_dst_valid) kfree(buflout); @@ -831,7 +837,7 @@ err_in: if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, - DMA_BIDIRECTIONAL); + bufl_dma_dir); if (!qat_req->buf.sgl_src_valid) kfree(bufl); diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 095ed2a404d2f8aa36d01639764e1313fe62f284..94a26702aeae113762adb19dbe2e7909af4d393e 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -332,14 +332,14 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->in.dh.in_tab[n_input_params] = 0; qat_req->out.dh.out_tab[1] = 0; /* Mapping in.in.b or in.in_g2.xa is the same */ - qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, - sizeof(qat_req->in.dh.in.b), + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh, + sizeof(struct qat_dh_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, - sizeof(qat_req->out.dh.r), + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh, + sizeof(struct qat_dh_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -729,14 +729,14 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, - sizeof(qat_req->in.rsa.enc.m), + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, + sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, - sizeof(qat_req->out.rsa.enc.c), + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, + sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -875,14 +875,14 @@ static int qat_rsa_dec(struct akcipher_request *req) else qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; - qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, - sizeof(qat_req->in.rsa.dec.c), + qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa, + sizeof(struct qat_rsa_input_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; - qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, - sizeof(qat_req->out.rsa.dec.m), + qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa, + sizeof(struct qat_rsa_output_params), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 0fe5a474aa4525f2a00cb76056b6e1cf701a90e6..b7f7869ef8b2f35bef5ff3793ffc06184d655f53 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1367,6 +1367,48 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, } } +static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + unsigned int fw_type) +{ + char *fw_type_name = fw_type ? "MMP" : "AE"; + unsigned int css_dword_size = sizeof(u32); + + if (handle->chip_info->fw_auth) { + struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; + unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); + + if ((css_hdr->header_len * css_dword_size) != header_len) + goto err; + if ((css_hdr->size * css_dword_size) != size) + goto err; + if (fw_type != css_hdr->fw_type) + goto err; + if (size <= header_len) + goto err; + size -= header_len; + } + + if (fw_type == CSS_AE_FIRMWARE) { + if (size < sizeof(struct icp_qat_simg_ae_mode *) + + ICP_QAT_SIMG_AE_INIT_SEQ_LEN) + goto err; + if (size > ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) + goto err; + } else if (fw_type == CSS_MMP_FIRMWARE) { + if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) + goto err; + } else { + pr_err("QAT: Unsupported firmware type\n"); + return -EINVAL; + } + return 0; + +err: + pr_err("QAT: Invalid %s firmware image\n", fw_type_name); + return -EINVAL; +} + static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, char *image, unsigned int size, struct icp_qat_fw_auth_desc **desc) @@ -1379,7 +1421,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_simg_ae_mode *simg_ae_mode; struct icp_firml_dram_desc img_desc; - if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return -EINVAL; } @@ -1547,6 +1589,11 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_fw_auth_desc *desc = NULL; int status = 0; + int ret; + + ret = qat_uclo_check_image(handle, addr_ptr, mem_size, CSS_MMP_FIRMWARE); + if (ret) + return ret; if (handle->chip_info->fw_auth) { status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); @@ -2018,8 +2065,15 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) struct icp_qat_fw_auth_desc *desc = NULL; struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; + int ret; for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { + ret = qat_uclo_check_image(handle, simg_hdr[i].simg_buf, + simg_hdr[i].simg_len, + CSS_AE_FIRMWARE); + if (ret) + return ret; + if (qat_uclo_map_auth_fw(handle, (char *)simg_hdr[i].simg_buf, (unsigned int) diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 97a530171f07a8caa618379942ebc12749c6ae79..6eb4d2e356297c1182f88382ff0757aee26fbf90 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -450,8 +450,8 @@ qce_aead_async_req_handle(struct crypto_async_request *async_req) if (ret) return ret; dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); - if (dst_nents < 0) { - ret = dst_nents; + if (!dst_nents) { + ret = -EIO; goto error_free; } diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 59159f5e64e522df14f7643f64e1dddfee51f464..37bafd7aeb79df4fb650e5c95e2604798a5cf653 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -97,14 +97,16 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) } ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); - if (ret < 0) - return ret; + if (!ret) + return -EIO; sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); - if (ret < 0) + if (!ret) { + ret = -EIO; goto error_unmap_src; + } ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents, &rctx->result_sg, 1, qce_ahash_done, async_req); diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index 3d27cd5210ef50a943a66ba0af7a13ef411da836..5b493fdc1e747fd3ec238f2ddeaee2acc445bdc3 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -124,15 +124,15 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req) rctx->dst_sg = rctx->dst_tbl.sgl; dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); - if (dst_nents < 0) { - ret = dst_nents; + if (!dst_nents) { + ret = -EIO; goto error_free; } if (diff_dst) { src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); - if (src_nents < 0) { - ret = src_nents; + if (!src_nents) { + ret = -EIO; goto error_unmap_dst; } rctx->src_sg = req->src; diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 031b5f701a0a35b40316be71ae41eeb02f06dcff..72dd1a4ebac4452731a69502b19d075f90573ffa 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -201,15 +202,13 @@ static int qcom_rng_remove(struct platform_device *pdev) return 0; } -#if IS_ENABLED(CONFIG_ACPI) -static const struct acpi_device_id qcom_rng_acpi_match[] = { +static const struct acpi_device_id __maybe_unused qcom_rng_acpi_match[] = { { .id = "QCOM8160", .driver_data = 1 }, {} }; MODULE_DEVICE_TABLE(acpi, qcom_rng_acpi_match); -#endif -static const struct of_device_id qcom_rng_of_match[] = { +static const struct of_device_id __maybe_unused qcom_rng_of_match[] = { { .compatible = "qcom,prng", .data = (void *)0}, { .compatible = "qcom,prng-ee", .data = (void *)1}, {} diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 457084b344c170733e2c1331dc00a0ecb8e297ce..7ab20fb95166e73da4c3dbaac2fad3d5949becbb 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -26,10 +26,10 @@ #include #include #include -#include #include #include #include +#include #define SHA_BUFFER_LEN PAGE_SIZE #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE @@ -196,7 +196,7 @@ struct sahara_dev { void __iomem *regs_base; struct clk *clk_ipg; struct clk *clk_ahb; - struct mutex queue_mutex; + spinlock_t queue_spinlock; struct task_struct *kthread; struct completion dma_completion; @@ -487,13 +487,13 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); - if (ret != dev->nb_in_sg) { + if (!ret) { dev_err(dev->device, "couldn't map in sg\n"); goto unmap_in; } ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, DMA_FROM_DEVICE); - if (ret != dev->nb_out_sg) { + if (!ret) { dev_err(dev->device, "couldn't map out sg\n"); goto unmap_out; } @@ -642,9 +642,9 @@ static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) rctx->mode = mode; - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); err = crypto_enqueue_request(&dev->queue, &req->base); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); wake_up_process(dev->kthread); @@ -1043,10 +1043,10 @@ static int sahara_queue_manage(void *data) do { __set_current_state(TASK_INTERRUPTIBLE); - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); if (backlog) backlog->complete(backlog, -EINPROGRESS); @@ -1092,9 +1092,9 @@ static int sahara_sha_enqueue(struct ahash_request *req, int last) rctx->first = 1; } - mutex_lock(&dev->queue_mutex); + spin_lock_bh(&dev->queue_spinlock); ret = crypto_enqueue_request(&dev->queue, &req->base); - mutex_unlock(&dev->queue_mutex); + spin_unlock_bh(&dev->queue_spinlock); wake_up_process(dev->kthread); @@ -1449,7 +1449,7 @@ static int sahara_probe(struct platform_device *pdev) crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); - mutex_init(&dev->queue_mutex); + spin_lock_init(&dev->queue_spinlock); dev_ptr = dev; diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 2a60d0525cdec07bc62adab3799f11566a9f65a9..168195672e2e10bed4a18d13991de26ef6536125 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -56,6 +56,10 @@ static void virtio_crypto_akcipher_finalize_req( struct virtio_crypto_akcipher_request *vc_akcipher_req, struct akcipher_request *req, int err) { + kfree(vc_akcipher_req->src_buf); + kfree(vc_akcipher_req->dst_buf); + vc_akcipher_req->src_buf = NULL; + vc_akcipher_req->dst_buf = NULL; virtcrypto_clear_request(&vc_akcipher_req->base); crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err); diff --git a/drivers/dax/hmem/device.c b/drivers/dax/hmem/device.c index cb6401c9e9a4fcb69c501bdfd90cb82d79bba71d..97086fab698e3db9d72a255725e0006a0df244ef 100644 --- a/drivers/dax/hmem/device.c +++ b/drivers/dax/hmem/device.c @@ -15,6 +15,7 @@ void hmem_register_device(int target_nid, struct resource *r) .start = r->start, .end = r->end, .flags = IORESOURCE_MEM, + .desc = IORES_DESC_SOFT_RESERVED, }; struct platform_device *pdev; struct memregion_info info; @@ -47,7 +48,7 @@ void hmem_register_device(int target_nid, struct resource *r) rc = platform_device_add_data(pdev, &info, sizeof(info)); if (rc < 0) { pr_err("hmem memregion_info allocation failure for %pr\n", &res); - goto out_pdev; + goto out_resource; } rc = platform_device_add_resources(pdev, &res, 1); @@ -65,7 +66,7 @@ void hmem_register_device(int target_nid, struct resource *r) return; out_resource: - put_device(&pdev->dev); + platform_device_put(pdev); out_pdev: memregion_free(id); } diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index a37622060fffa3a5ba821ee90263d7dfbd3db91d..4852a2dbdb278e7f9f2e2e7d2e9752a73e0b148d 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -11,9 +11,17 @@ #include #include #include +#include #include "dax-private.h" #include "bus.h" +/* + * Default abstract distance assigned to the NUMA node onlined + * by DAX/kmem if the low level platform driver didn't initialize + * one for this NUMA node. + */ +#define MEMTIER_DEFAULT_DAX_ADISTANCE (MEMTIER_ADISTANCE_DRAM * 5) + /* Memory resource name used for add_memory_driver_managed(). */ static const char *kmem_name; /* Set if any memory will remain added when the driver will be unloaded. */ @@ -41,6 +49,7 @@ struct dax_kmem_data { struct resource *res[]; }; +static struct memory_dev_type *dax_slowmem_type; static int dev_dax_kmem_probe(struct dev_dax *dev_dax) { struct device *dev = &dev_dax->dev; @@ -79,11 +88,13 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) return -EINVAL; } + init_node_memory_type(numa_node, dax_slowmem_type); + + rc = -ENOMEM; data = kzalloc(struct_size(data, res, dev_dax->nr_range), GFP_KERNEL); if (!data) - return -ENOMEM; + goto err_dax_kmem_data; - rc = -ENOMEM; data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); if (!data->res_name) goto err_res_name; @@ -155,6 +166,8 @@ err_reg_mgid: kfree(data->res_name); err_res_name: kfree(data); +err_dax_kmem_data: + clear_node_memory_type(numa_node, dax_slowmem_type); return rc; } @@ -162,6 +175,7 @@ err_res_name: static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { int i, success = 0; + int node = dev_dax->target_node; struct device *dev = &dev_dax->dev; struct dax_kmem_data *data = dev_get_drvdata(dev); @@ -198,6 +212,14 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax) kfree(data->res_name); kfree(data); dev_set_drvdata(dev, NULL); + /* + * Clear the memtype association on successful unplug. + * If not, we have memory blocks left which can be + * offlined/onlined later. We need to keep memory_dev_type + * for that. This implies this reference will be around + * till next reboot. + */ + clear_node_memory_type(node, dax_slowmem_type); } } #else @@ -228,9 +250,22 @@ static int __init dax_kmem_init(void) if (!kmem_name) return -ENOMEM; + dax_slowmem_type = alloc_memory_type(MEMTIER_DEFAULT_DAX_ADISTANCE); + if (IS_ERR(dax_slowmem_type)) { + rc = PTR_ERR(dax_slowmem_type); + goto err_dax_slowmem_type; + } + rc = dax_driver_register(&device_dax_kmem_driver); if (rc) - kfree_const(kmem_name); + goto error_dax_driver; + + return rc; + +error_dax_driver: + destroy_memory_type(dax_slowmem_type); +err_dax_slowmem_type: + kfree_const(kmem_name); return rc; } @@ -239,6 +274,7 @@ static void __exit dax_kmem_exit(void) dax_driver_unregister(&device_dax_kmem_driver); if (!any_hotremove_failed) kfree_const(kmem_name); + destroy_memory_type(dax_slowmem_type); } MODULE_AUTHOR("Intel Corporation"); diff --git a/drivers/dax/super.c b/drivers/dax/super.c index 9b5e2a5eb0ae68ae2c1e22431fc020ad41f72e1e..da4438f3188c8d93a5eac42c4999e4a4aa20a7ca 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -363,7 +363,7 @@ static void dax_free_inode(struct inode *inode) { struct dax_device *dax_dev = to_dax_dev(inode); if (inode->i_rdev) - ida_simple_remove(&dax_minor_ida, iminor(inode)); + ida_free(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); } @@ -445,7 +445,7 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) if (WARN_ON_ONCE(ops && !ops->zero_page_range)) return ERR_PTR(-EINVAL); - minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL); + minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL); if (minor < 0) return ERR_PTR(-ENOMEM); @@ -459,7 +459,7 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops) return dax_dev; err_dev: - ida_simple_remove(&dax_minor_ida, minor); + ida_free(&dax_minor_ida, minor); return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL_GPL(alloc_dax); diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c index 9a88faaf8b27ffbd44fa2c55d8b603aa21e34317..39ac069cabc756c93bd38b2f8c11aff42ed6bd7e 100644 --- a/drivers/devfreq/event/rockchip-dfi.c +++ b/drivers/devfreq/event/rockchip-dfi.c @@ -189,10 +189,9 @@ static int rockchip_dfi_probe(struct platform_device *pdev) return PTR_ERR(data->regs); data->clk = devm_clk_get(dev, "pclk_ddr_mon"); - if (IS_ERR(data->clk)) { - dev_err(dev, "Cannot get the clk dmc_clk\n"); - return PTR_ERR(data->clk); - } + if (IS_ERR(data->clk)) + return dev_err_probe(dev, PTR_ERR(data->clk), + "Cannot get the clk pclk_ddr_mon\n"); /* try to find the optional reference to the pmu syscon */ node = of_parse_phandle(np, "rockchip,pmu", 0); diff --git a/drivers/devfreq/mtk-cci-devfreq.c b/drivers/devfreq/mtk-cci-devfreq.c index 71abb3fbd0420135046b2ca54d593120a5dcf95c..e5458ada5197a1d5a40d889268205d6c029e2f01 100644 --- a/drivers/devfreq/mtk-cci-devfreq.c +++ b/drivers/devfreq/mtk-cci-devfreq.c @@ -291,9 +291,13 @@ static int mtk_ccifreq_probe(struct platform_device *pdev) } drv->sram_reg = devm_regulator_get_optional(dev, "sram"); - if (IS_ERR(drv->sram_reg)) + if (IS_ERR(drv->sram_reg)) { + ret = PTR_ERR(drv->sram_reg); + if (ret == -EPROBE_DEFER) + goto out_free_resources; + drv->sram_reg = NULL; - else { + } else { ret = regulator_enable(drv->sram_reg); if (ret) { dev_err(dev, "failed to enable sram regulator\n"); diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 7663c4e784b6df83e48ab6df5d6d273943cce327..c40d72d318fd05611397151201ddce90adef8dd8 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -53,7 +53,7 @@ static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN); spin_unlock(&dmabuf->name_lock); - return dynamic_dname(dentry, buffer, buflen, "/%s:%s", + return dynamic_dname(buffer, buflen, "/%s:%s", dentry->d_name.name, ret > 0 ? name : ""); } @@ -531,11 +531,11 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) * value. */ inode->i_ino = atomic64_add_return(1, &dmabuf_inode); + flags &= O_ACCMODE | O_NONBLOCK; file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf", flags, &dma_buf_fops); if (IS_ERR(file)) goto err_alloc_file; - file->f_flags = flags & (O_ACCMODE | O_NONBLOCK); file->private_data = dmabuf; file->f_path.dentry->d_fsdata = dmabuf; diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c index 205acb2c744ded50d42d2138f288cdce9ef97967..e3885c90a3acba1b7cebbf895038893d097aa223 100644 --- a/drivers/dma-buf/dma-resv.c +++ b/drivers/dma-buf/dma-resv.c @@ -295,7 +295,8 @@ void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence, enum dma_resv_usage old_usage; dma_resv_list_entry(fobj, i, obj, &old, &old_usage); - if ((old->context == fence->context && old_usage >= usage) || + if ((old->context == fence->context && old_usage >= usage && + dma_fence_is_later(fence, old)) || dma_fence_is_signaled(old)) { dma_resv_list_set(fobj, i, fence, usage); dma_fence_put(old); diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index bf11d32205f38aa282e86209e5e01a2a3e6efcc8..2bcdb935a3ac4d2ad6195cf780aec60fad83e39a 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -213,7 +213,7 @@ static long udmabuf_create(struct miscdevice *device, memfd = fget(list[i].memfd); if (!memfd) goto err; - mapping = file_inode(memfd)->i_mapping; + mapping = memfd->f_mapping; if (!shmem_mapping(mapping) && !is_file_hugepages(memfd)) goto err; seals = memfd_fcntl(memfd, F_GET_SEALS, 0); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index a06d2a7627aa6d20a6e905cc2a2bf23a037061b5..7524b62a8870a24990d7a8a001c201872ca94160 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -180,7 +180,7 @@ config DMA_SUN6I config DW_AXI_DMAC tristate "Synopsys DesignWare AXI DMA support" - depends on OF || COMPILE_TEST + depends on OF depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 487a01aa207dfaaaa81e075b8f41971748320c1e..eea8bd33b4b7385d0fd21dad5583588c7de3556f 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2367,7 +2367,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, INIT_LIST_HEAD(&dmadev->channels); /* - * Register as many many memcpy as we have physical channels, + * Register as many memcpy as we have physical channels, * we won't always be able to use all but the code will have * to cope with that situation. */ diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index d1f74a3aa999d77319990e41f85686a700d6356b..317ca76ccafd2b755a279cf5c64a83147a4aee8d 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -12,8 +12,9 @@ #include #include #include -#include +#include #include +#include #include "dmaengine.h" @@ -95,7 +96,9 @@ struct admac_data { struct dma_device dma; struct device *dev; __iomem void *base; + struct reset_control *rstc; + int irq; int irq_index; int nchannels; struct admac_chan channels[]; @@ -724,18 +727,17 @@ static int admac_probe(struct platform_device *pdev) if (irq < 0) return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n"); - - err = devm_request_irq(&pdev->dev, irq, admac_interrupt, - 0, dev_name(&pdev->dev), ad); - if (err) - return dev_err_probe(&pdev->dev, err, - "unable to register interrupt\n"); + ad->irq = irq; ad->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ad->base)) return dev_err_probe(&pdev->dev, PTR_ERR(ad->base), "unable to obtain MMIO resource\n"); + ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL); + if (IS_ERR(ad->rstc)) + return PTR_ERR(ad->rstc); + dma = &ad->dma; dma_cap_set(DMA_PRIVATE, dma->cap_mask); @@ -774,17 +776,38 @@ static int admac_probe(struct platform_device *pdev) tasklet_setup(&adchan->tasklet, admac_chan_tasklet); } - err = dma_async_device_register(&ad->dma); + err = reset_control_reset(ad->rstc); if (err) - return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + return dev_err_probe(&pdev->dev, err, + "unable to trigger reset\n"); + + err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad); + if (err) { + dev_err_probe(&pdev->dev, err, + "unable to register interrupt\n"); + goto free_reset; + } + + err = dma_async_device_register(&ad->dma); + if (err) { + dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + goto free_irq; + } err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad); if (err) { dma_async_device_unregister(&ad->dma); - return dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + goto free_irq; } return 0; + +free_irq: + free_irq(ad->irq, ad); +free_reset: + reset_control_rearm(ad->rstc); + return err; } static int admac_remove(struct platform_device *pdev) @@ -793,6 +816,8 @@ static int admac_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&ad->dma); + free_irq(ad->irq, ad); + reset_control_rearm(ad->rstc); return 0; } diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index b102d8eb5d83d5268185fde99692ea8ae6e8c231..d6c9781cd46afe9b43e0c63c46741d9bde5c8335 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1470,10 +1470,7 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, bool initd; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - if (!txstate) + if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&atchan->lock, flags); diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 9fe2ae794316967caf8c36a3d38d0cb2a0bf6ba2..ffe621695e472b6b6d96e2a06a85ff514ee65651 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -312,7 +312,7 @@ static unsigned long dmatest_random(void) { unsigned long buf; - prandom_bytes(&buf, sizeof(buf)); + get_random_bytes(&buf, sizeof(buf)); return buf; } diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 07f7564796637e01dfe2833a7f3c9a726b3c30e9..c54b24ff5206a39c93c2422b54f79864456f796a 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -682,15 +681,12 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) if (chan->status != EDMA_ST_IDLE) return -EBUSY; - pm_runtime_get(chan->dw->chip->dev); - return 0; } static void dw_edma_free_chan_resources(struct dma_chan *dchan) { unsigned long timeout = jiffies + msecs_to_jiffies(5000); - struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int ret; while (time_before(jiffies, timeout)) { @@ -703,8 +699,6 @@ static void dw_edma_free_chan_resources(struct dma_chan *dchan) cpu_relax(); } - - pm_runtime_put(chan->dw->chip->dev); } static int dw_edma_channel_setup(struct dw_edma *dw, bool write, @@ -977,9 +971,6 @@ int dw_edma_probe(struct dw_edma_chip *chip) if (err) goto err_irq_free; - /* Power management */ - pm_runtime_enable(dev); - /* Turn debugfs on */ dw_edma_v0_core_debugfs_on(dw); @@ -1009,9 +1000,6 @@ int dw_edma_remove(struct dw_edma_chip *chip) for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); - /* Power management */ - pm_runtime_disable(dev); - /* Deregister eDMA device */ dma_async_device_unregister(&dw->wr_edma); list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 43817ced3a3e1e092570161ae8832883d1a4846d..c1350a36fddd995c3144176b91affb5f87350ff5 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2019 HiSilicon Limited. */ +/* Copyright(c) 2019-2022 HiSilicon Limited. */ + #include #include #include @@ -9,32 +10,87 @@ #include #include "virt-dma.h" -#define HISI_DMA_SQ_BASE_L 0x0 -#define HISI_DMA_SQ_BASE_H 0x4 -#define HISI_DMA_SQ_DEPTH 0x8 -#define HISI_DMA_SQ_TAIL_PTR 0xc -#define HISI_DMA_CQ_BASE_L 0x10 -#define HISI_DMA_CQ_BASE_H 0x14 -#define HISI_DMA_CQ_DEPTH 0x18 -#define HISI_DMA_CQ_HEAD_PTR 0x1c -#define HISI_DMA_CTRL0 0x20 -#define HISI_DMA_CTRL0_QUEUE_EN_S 0 -#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4 -#define HISI_DMA_CTRL1 0x24 -#define HISI_DMA_CTRL1_QUEUE_RESET_S 0 -#define HISI_DMA_Q_FSM_STS 0x30 -#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0) -#define HISI_DMA_INT_STS 0x40 -#define HISI_DMA_INT_STS_MASK GENMASK(12, 0) -#define HISI_DMA_INT_MSK 0x44 -#define HISI_DMA_MODE 0x217c -#define HISI_DMA_OFFSET 0x100 - -#define HISI_DMA_MSI_NUM 32 -#define HISI_DMA_CHAN_NUM 30 -#define HISI_DMA_Q_DEPTH_VAL 1024 - -#define PCI_BAR_2 2 +/* HiSilicon DMA register common field define */ +#define HISI_DMA_Q_SQ_BASE_L 0x0 +#define HISI_DMA_Q_SQ_BASE_H 0x4 +#define HISI_DMA_Q_SQ_DEPTH 0x8 +#define HISI_DMA_Q_SQ_TAIL_PTR 0xc +#define HISI_DMA_Q_CQ_BASE_L 0x10 +#define HISI_DMA_Q_CQ_BASE_H 0x14 +#define HISI_DMA_Q_CQ_DEPTH 0x18 +#define HISI_DMA_Q_CQ_HEAD_PTR 0x1c +#define HISI_DMA_Q_CTRL0 0x20 +#define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0) +#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4) +#define HISI_DMA_Q_CTRL1 0x24 +#define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0) +#define HISI_DMA_Q_FSM_STS 0x30 +#define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0) +#define HISI_DMA_Q_ERR_INT_NUM0 0x84 +#define HISI_DMA_Q_ERR_INT_NUM1 0x88 +#define HISI_DMA_Q_ERR_INT_NUM2 0x8c + +/* HiSilicon IP08 DMA register and field define */ +#define HISI_DMA_HIP08_MODE 0x217C +#define HISI_DMA_HIP08_Q_BASE 0x0 +#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2) +#define HISI_DMA_HIP08_Q_INT_STS 0x40 +#define HISI_DMA_HIP08_Q_INT_MSK 0x44 +#define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0) +#define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48 +#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24) + +/* HiSilicon IP09 DMA register and field define */ +#define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00 +#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0) +#define HISI_DMA_HIP09_Q_BASE 0x2000 +#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28) +#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26) +#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27) +#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2) +#define HISI_DMA_HIP09_Q_INT_STS 0x40 +#define HISI_DMA_HIP09_Q_INT_MSK 0x44 +#define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1 +#define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48 +#define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C +#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1) +#define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \ + (port_id) * 0x20) +#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16) + +#define HISI_DMA_HIP09_MAX_PORT_NUM 16 + +#define HISI_DMA_HIP08_MSI_NUM 32 +#define HISI_DMA_HIP08_CHAN_NUM 30 +#define HISI_DMA_HIP09_MSI_NUM 4 +#define HISI_DMA_HIP09_CHAN_NUM 4 +#define HISI_DMA_REVISION_HIP08B 0x21 +#define HISI_DMA_REVISION_HIP09A 0x30 + +#define HISI_DMA_Q_OFFSET 0x100 +#define HISI_DMA_Q_DEPTH_VAL 1024 + +#define PCI_BAR_2 2 + +#define HISI_DMA_POLL_Q_STS_DELAY_US 10 +#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000 + +#define HISI_DMA_MAX_DIR_NAME_LEN 128 + +/* + * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they + * have the same pci device id but different pci revision. + * Unfortunately, they have different register layouts, so two layout + * enumerations are defined. + */ +enum hisi_dma_reg_layout { + HISI_DMA_REG_LAYOUT_INVALID = 0, + HISI_DMA_REG_LAYOUT_HIP08, + HISI_DMA_REG_LAYOUT_HIP09 +}; enum hisi_dma_mode { EP = 0, @@ -105,9 +161,162 @@ struct hisi_dma_dev { struct dma_device dma_dev; u32 chan_num; u32 chan_depth; + enum hisi_dma_reg_layout reg_layout; + void __iomem *queue_base; /* queue region start of register */ struct hisi_dma_chan chan[]; }; +#ifdef CONFIG_DEBUG_FS + +static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = { + {"DMA_QUEUE_SQ_DEPTH ", 0x0008ull}, + {"DMA_QUEUE_SQ_TAIL_PTR ", 0x000Cull}, + {"DMA_QUEUE_CQ_DEPTH ", 0x0018ull}, + {"DMA_QUEUE_CQ_HEAD_PTR ", 0x001Cull}, + {"DMA_QUEUE_CTRL0 ", 0x0020ull}, + {"DMA_QUEUE_CTRL1 ", 0x0024ull}, + {"DMA_QUEUE_FSM_STS ", 0x0030ull}, + {"DMA_QUEUE_SQ_STS ", 0x0034ull}, + {"DMA_QUEUE_CQ_TAIL_PTR ", 0x003Cull}, + {"DMA_QUEUE_INT_STS ", 0x0040ull}, + {"DMA_QUEUE_INT_MSK ", 0x0044ull}, + {"DMA_QUEUE_INT_RO ", 0x006Cull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = { + {"DMA_QUEUE_BYTE_CNT ", 0x0038ull}, + {"DMA_ERR_INT_NUM6 ", 0x0048ull}, + {"DMA_QUEUE_DESP0 ", 0x0050ull}, + {"DMA_QUEUE_DESP1 ", 0x0054ull}, + {"DMA_QUEUE_DESP2 ", 0x0058ull}, + {"DMA_QUEUE_DESP3 ", 0x005Cull}, + {"DMA_QUEUE_DESP4 ", 0x0074ull}, + {"DMA_QUEUE_DESP5 ", 0x0078ull}, + {"DMA_QUEUE_DESP6 ", 0x007Cull}, + {"DMA_QUEUE_DESP7 ", 0x0080ull}, + {"DMA_ERR_INT_NUM0 ", 0x0084ull}, + {"DMA_ERR_INT_NUM1 ", 0x0088ull}, + {"DMA_ERR_INT_NUM2 ", 0x008Cull}, + {"DMA_ERR_INT_NUM3 ", 0x0090ull}, + {"DMA_ERR_INT_NUM4 ", 0x0094ull}, + {"DMA_ERR_INT_NUM5 ", 0x0098ull}, + {"DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = { + {"DMA_QUEUE_ERR_INT_STS ", 0x0048ull}, + {"DMA_QUEUE_ERR_INT_MSK ", 0x004Cull}, + {"DFX_SQ_READ_ERR_PTR ", 0x0068ull}, + {"DFX_DMA_ERR_INT_NUM0 ", 0x0084ull}, + {"DFX_DMA_ERR_INT_NUM1 ", 0x0088ull}, + {"DFX_DMA_ERR_INT_NUM2 ", 0x008Cull}, + {"DFX_DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = { + {"DMA_ECC_ERR_ADDR ", 0x2004ull}, + {"DMA_ECC_ECC_CNT ", 0x2014ull}, + {"COMMON_AND_CH_ERR_STS ", 0x2030ull}, + {"LOCAL_CPL_ID_STS_0 ", 0x20E0ull}, + {"LOCAL_CPL_ID_STS_1 ", 0x20E4ull}, + {"LOCAL_CPL_ID_STS_2 ", 0x20E8ull}, + {"LOCAL_CPL_ID_STS_3 ", 0x20ECull}, + {"LOCAL_TLP_NUM ", 0x2158ull}, + {"SQCQ_TLP_NUM ", 0x2164ull}, + {"CPL_NUM ", 0x2168ull}, + {"INF_BACK_PRESS_STS ", 0x2170ull}, + {"DMA_CH_RAS_LEVEL ", 0x2184ull}, + {"DMA_CM_RAS_LEVEL ", 0x2188ull}, + {"DMA_CH_ERR_STS ", 0x2190ull}, + {"DMA_CH_DONE_STS ", 0x2194ull}, + {"DMA_SQ_TAG_STS_0 ", 0x21A0ull}, + {"DMA_SQ_TAG_STS_1 ", 0x21A4ull}, + {"DMA_SQ_TAG_STS_2 ", 0x21A8ull}, + {"DMA_SQ_TAG_STS_3 ", 0x21ACull}, + {"LOCAL_P_ID_STS_0 ", 0x21B0ull}, + {"LOCAL_P_ID_STS_1 ", 0x21B4ull}, + {"LOCAL_P_ID_STS_2 ", 0x21B8ull}, + {"LOCAL_P_ID_STS_3 ", 0x21BCull}, + {"DMA_PREBUFF_INFO_0 ", 0x2200ull}, + {"DMA_CM_TABLE_INFO_0 ", 0x2220ull}, + {"DMA_CM_CE_RO ", 0x2244ull}, + {"DMA_CM_NFE_RO ", 0x2248ull}, + {"DMA_CM_FE_RO ", 0x224Cull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = { + {"COMMON_AND_CH_ERR_STS ", 0x0030ull}, + {"DMA_PORT_IDLE_STS ", 0x0150ull}, + {"DMA_CH_RAS_LEVEL ", 0x0184ull}, + {"DMA_CM_RAS_LEVEL ", 0x0188ull}, + {"DMA_CM_CE_RO ", 0x0244ull}, + {"DMA_CM_NFE_RO ", 0x0248ull}, + {"DMA_CM_FE_RO ", 0x024Cull}, + {"DFX_INF_BACK_PRESS_STS0 ", 0x1A40ull}, + {"DFX_INF_BACK_PRESS_STS1 ", 0x1A44ull}, + {"DFX_INF_BACK_PRESS_STS2 ", 0x1A48ull}, + {"DFX_DMA_WRR_DISABLE ", 0x1A4Cull}, + {"DFX_PA_REQ_TLP_NUM ", 0x1C00ull}, + {"DFX_PA_BACK_TLP_NUM ", 0x1C04ull}, + {"DFX_PA_RETRY_TLP_NUM ", 0x1C08ull}, + {"DFX_LOCAL_NP_TLP_NUM ", 0x1C0Cull}, + {"DFX_LOCAL_CPL_HEAD_TLP_NUM ", 0x1C10ull}, + {"DFX_LOCAL_CPL_DATA_TLP_NUM ", 0x1C14ull}, + {"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM ", 0x1C18ull}, + {"DFX_LOCAL_P_HEAD_TLP_NUM ", 0x1C1Cull}, + {"DFX_LOCAL_P_ACK_TLP_NUM ", 0x1C20ull}, + {"DFX_BUF_ALOC_PORT_REQ_NUM ", 0x1C24ull}, + {"DFX_BUF_ALOC_PORT_RESULT_NUM ", 0x1C28ull}, + {"DFX_BUF_FAIL_SIZE_NUM ", 0x1C2Cull}, + {"DFX_BUF_ALOC_SIZE_NUM ", 0x1C30ull}, + {"DFX_BUF_NP_RELEASE_SIZE_NUM ", 0x1C34ull}, + {"DFX_BUF_P_RELEASE_SIZE_NUM ", 0x1C38ull}, + {"DFX_BUF_PORT_RELEASE_SIZE_NUM ", 0x1C3Cull}, + {"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR ", 0x1CA8ull}, + {"DFX_DMA_PREBUF_MEM0_ECC_CNT ", 0x1CACull}, + {"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR ", 0x1CB0ull}, + {"DFX_DMA_LOC_NP_OSTB_ECC_CNT ", 0x1CB4ull}, + {"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR ", 0x1CC0ull}, + {"DFX_DMA_PREBUF_MEM1_ECC_CNT ", 0x1CC4ull}, + {"DMA_CH_DONE_STS ", 0x02E0ull}, + {"DMA_CH_ERR_STS ", 0x0320ull}, +}; +#endif /* CONFIG_DEBUG_FS*/ + +static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_REG_LAYOUT_HIP08; + else if (pdev->revision >= HISI_DMA_REVISION_HIP09A) + return HISI_DMA_REG_LAYOUT_HIP09; + + return HISI_DMA_REG_LAYOUT_INVALID; +} + +static u32 hisi_dma_get_chan_num(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_CHAN_NUM; + + return HISI_DMA_HIP09_CHAN_NUM; +} + +static u32 hisi_dma_get_msi_num(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_MSI_NUM; + + return HISI_DMA_HIP09_MSI_NUM; +} + +static u32 hisi_dma_get_queue_base(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_Q_BASE; + + return HISI_DMA_HIP09_Q_BASE; +} + static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c) { return container_of(c, struct hisi_dma_chan, vc.chan); @@ -121,7 +330,7 @@ static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd) static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index, u32 val) { - writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET); + writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET); } static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) @@ -129,70 +338,103 @@ static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) u32 tmp; tmp = readl_relaxed(addr); - tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos); + tmp = val ? tmp | pos : tmp & ~pos; writel_relaxed(tmp, addr); } static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index, bool pause) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause); + addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause); } static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index, bool enable) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable); + addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable); } static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index, - HISI_DMA_INT_STS_MASK); + void __iomem *q_base = hdma_dev->queue_base; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, + qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); + else { + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, + qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, + qp_index, + HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); + } } static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - void __iomem *base = hdma_dev->base; - - hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index, - HISI_DMA_INT_STS_MASK); - hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0); + void __iomem *q_base = hdma_dev->queue_base; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS, + qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, + qp_index, 0); + } else { + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS, + qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS, + qp_index, + HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, + qp_index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, + qp_index, 0); + } } static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1); + addr = hdma_dev->queue_base + + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1); } static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index) { - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0); - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0); + void __iomem *q_base = hdma_dev->queue_base; + + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); } -static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan) +static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan, + bool disable) { struct hisi_dma_dev *hdma_dev = chan->hdma_dev; u32 index = chan->qp_num, tmp; + void __iomem *addr; int ret; hisi_dma_pause_dma(hdma_dev, index, true); hisi_dma_enable_dma(hdma_dev, index, false); hisi_dma_mask_irq(hdma_dev, index); - ret = readl_relaxed_poll_timeout(hdma_dev->base + - HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, - FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000); + addr = hdma_dev->queue_base + + HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET; + + ret = readl_relaxed_poll_timeout(addr, tmp, + FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN, + HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); if (ret) { dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n"); WARN_ON(1); @@ -201,12 +443,15 @@ static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan) hisi_dma_do_reset(hdma_dev, index); hisi_dma_reset_qp_point(hdma_dev, index); hisi_dma_pause_dma(hdma_dev, index, false); - hisi_dma_enable_dma(hdma_dev, index, true); - hisi_dma_unmask_irq(hdma_dev, index); - ret = readl_relaxed_poll_timeout(hdma_dev->base + - HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, - FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000); + if (!disable) { + hisi_dma_enable_dma(hdma_dev, index, true); + hisi_dma_unmask_irq(hdma_dev, index); + } + + ret = readl_relaxed_poll_timeout(addr, tmp, + FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE, + HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); if (ret) { dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n"); WARN_ON(1); @@ -218,7 +463,7 @@ static void hisi_dma_free_chan_resources(struct dma_chan *c) struct hisi_dma_chan *chan = to_hisi_dma_chan(c); struct hisi_dma_dev *hdma_dev = chan->hdma_dev; - hisi_dma_reset_hw_chan(chan); + hisi_dma_reset_or_disable_hw_chan(chan, false); vchan_free_chan_resources(&chan->vc); memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth); @@ -267,7 +512,6 @@ static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) vd = vchan_next_desc(&chan->vc); if (!vd) { - dev_err(&hdma_dev->pdev->dev, "no issued task!\n"); chan->desc = NULL; return; } @@ -288,8 +532,8 @@ static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth; /* update sq_tail to trigger a new task */ - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num, - chan->sq_tail); + hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR, + chan->qp_num, chan->sq_tail); } static void hisi_dma_issue_pending(struct dma_chan *c) @@ -299,7 +543,7 @@ static void hisi_dma_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc)) + if (vchan_issue_pending(&chan->vc) && !chan->desc) hisi_dma_start_transfer(chan); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -363,26 +607,86 @@ static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev) static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index) { struct hisi_dma_chan *chan = &hdma_dev->chan[index]; + void __iomem *q_base = hdma_dev->queue_base; u32 hw_depth = hdma_dev->chan_depth - 1; - void __iomem *base = hdma_dev->base; + void __iomem *addr; + u32 tmp; /* set sq, cq base */ - hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index, lower_32_bits(chan->sq_dma)); - hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index, upper_32_bits(chan->sq_dma)); - hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index, lower_32_bits(chan->cq_dma)); - hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index, upper_32_bits(chan->cq_dma)); /* set sq, cq depth */ - hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth); - hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth); + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth); /* init sq tail and cq head */ - hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0); - hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); + + /* init error interrupt stats */ + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6, + index, 0); + /* + * init SQ/CQ direction selecting register. + * "0" is to local side and "1" is to remote side. + */ + addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0); + + /* + * 0 - Continue to next descriptor if error occurs. + * 1 - Abort the DMA queue if error occurs. + */ + hisi_dma_update_bit(addr, + HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0); + } else { + addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; + + /* + * init SQ/CQ direction selecting register. + * "0" is to local side and "1" is to remote side. + */ + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0); + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0); + + /* + * 0 - Continue to next descriptor if error occurs. + * 1 - Abort the DMA queue if error occurs. + */ + + tmp = readl_relaxed(addr); + tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN; + writel_relaxed(tmp, addr); + + /* + * 0 - dma should process FLR whith CPU. + * 1 - dma not process FLR, only cpu process FLR. + */ + addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0); + + addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1); + } } static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) @@ -394,7 +698,7 @@ static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]); + hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true); } static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev) @@ -426,24 +730,23 @@ static irqreturn_t hisi_dma_irq(int irq, void *data) struct hisi_dma_dev *hdma_dev = chan->hdma_dev; struct hisi_dma_desc *desc; struct hisi_dma_cqe *cqe; + void __iomem *q_base; spin_lock(&chan->vc.lock); desc = chan->desc; cqe = chan->cq + chan->cq_head; + q_base = hdma_dev->queue_base; if (desc) { + chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth; + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, + chan->qp_num, chan->cq_head); if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) { - chan->cq_head = (chan->cq_head + 1) % - hdma_dev->chan_depth; - hisi_dma_chan_write(hdma_dev->base, - HISI_DMA_CQ_HEAD_PTR, chan->qp_num, - chan->cq_head); vchan_cookie_complete(&desc->vd); + hisi_dma_start_transfer(chan); } else { dev_err(&hdma_dev->pdev->dev, "task error!\n"); } - - chan->desc = NULL; } spin_unlock(&chan->vc.lock); @@ -497,16 +800,169 @@ static void hisi_dma_disable_hw_channels(void *data) static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev, enum hisi_dma_mode mode) { - writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE); + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + writel_relaxed(mode == RC ? 1 : 0, + hdma_dev->base + HISI_DMA_HIP08_MODE); } +static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev) +{ + void __iomem *addr; + int i; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) { + for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) { + addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i); + hisi_dma_update_bit(addr, + HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1); + } + } +} + +static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev) +{ + struct dma_device *dma_dev; + + dma_dev = &hdma_dev->dma_dev; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; + dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; + dma_dev->device_tx_status = hisi_dma_tx_status; + dma_dev->device_issue_pending = hisi_dma_issue_pending; + dma_dev->device_terminate_all = hisi_dma_terminate_all; + dma_dev->device_synchronize = hisi_dma_synchronize; + dma_dev->directions = BIT(DMA_MEM_TO_MEM); + dma_dev->dev = &hdma_dev->pdev->dev; + INIT_LIST_HEAD(&dma_dev->channels); +} + +/* --- debugfs implementation --- */ +#ifdef CONFIG_DEBUG_FS +#include +static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev, + u32 *regs_sz) +{ + struct device *dev = &hdma_dev->pdev->dev; + struct debugfs_reg32 *regs; + u32 regs_sz_comm; + + regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs); + else + *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs); + + regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32), + GFP_KERNEL); + if (!regs) + return NULL; + memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs)); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs, + sizeof(hisi_dma_hip08_chan_regs)); + else + memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs, + sizeof(hisi_dma_hip09_chan_regs)); + + return regs; +} + +static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev) +{ + char dir_name[HISI_DMA_MAX_DIR_NAME_LEN]; + struct debugfs_regset32 *regsets; + struct debugfs_reg32 *regs; + struct dentry *chan_dir; + struct device *dev; + u32 regs_sz; + int ret; + int i; + + dev = &hdma_dev->pdev->dev; + + regsets = devm_kcalloc(dev, hdma_dev->chan_num, + sizeof(*regsets), GFP_KERNEL); + if (!regsets) + return -ENOMEM; + + regs = hisi_dma_get_ch_regs(hdma_dev, ®s_sz); + if (!regs) + return -ENOMEM; + + for (i = 0; i < hdma_dev->chan_num; i++) { + regsets[i].regs = regs; + regsets[i].nregs = regs_sz; + regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET; + regsets[i].dev = dev; + + memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN); + ret = sprintf(dir_name, "channel%d", i); + if (ret < 0) + return ret; + + chan_dir = debugfs_create_dir(dir_name, + hdma_dev->dma_dev.dbg_dev_root); + debugfs_create_regset32("regs", 0444, chan_dir, ®sets[i]); + } + + return 0; +} + +static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) +{ + struct debugfs_regset32 *regset; + struct device *dev; + int ret; + + dev = &hdma_dev->pdev->dev; + + if (hdma_dev->dma_dev.dbg_dev_root == NULL) + return; + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + regset->regs = hisi_dma_hip08_comm_regs; + regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs); + } else { + regset->regs = hisi_dma_hip09_comm_regs; + regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs); + } + regset->base = hdma_dev->base; + regset->dev = dev; + + debugfs_create_regset32("regs", 0444, + hdma_dev->dma_dev.dbg_dev_root, regset); + + ret = hisi_dma_create_chan_dir(hdma_dev); + if (ret < 0) + dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n"); +} +#else +static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { } +#endif /* CONFIG_DEBUG_FS*/ +/* --- debugfs implementation --- */ + static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + enum hisi_dma_reg_layout reg_layout; struct device *dev = &pdev->dev; struct hisi_dma_dev *hdma_dev; struct dma_device *dma_dev; + u32 chan_num; + u32 msi_num; int ret; + reg_layout = hisi_dma_get_reg_layout(pdev); + if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) { + dev_err(dev, "unsupported device!\n"); + return -EINVAL; + } + ret = pcim_enable_device(pdev); if (ret) { dev_err(dev, "failed to enable device mem!\n"); @@ -523,40 +979,37 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; - hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL); + chan_num = hisi_dma_get_chan_num(pdev); + hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), + GFP_KERNEL); if (!hdma_dev) return -EINVAL; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; - hdma_dev->chan_num = HISI_DMA_CHAN_NUM; hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL; + hdma_dev->chan_num = chan_num; + hdma_dev->reg_layout = reg_layout; + hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev); pci_set_drvdata(pdev, hdma_dev); pci_set_master(pdev); + msi_num = hisi_dma_get_msi_num(pdev); + /* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */ - ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM, - PCI_IRQ_MSI); + ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to allocate MSI vectors!\n"); return ret; } - dma_dev = &hdma_dev->dma_dev; - dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); - dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; - dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; - dma_dev->device_tx_status = hisi_dma_tx_status; - dma_dev->device_issue_pending = hisi_dma_issue_pending; - dma_dev->device_terminate_all = hisi_dma_terminate_all; - dma_dev->device_synchronize = hisi_dma_synchronize; - dma_dev->directions = BIT(DMA_MEM_TO_MEM); - dma_dev->dev = dev; - INIT_LIST_HEAD(&dma_dev->channels); + hisi_dma_init_dma_dev(hdma_dev); hisi_dma_set_mode(hdma_dev, RC); + hisi_dma_init_hw(hdma_dev); + ret = hisi_dma_enable_hw_channels(hdma_dev); if (ret < 0) { dev_err(dev, "failed to enable hw channel!\n"); @@ -568,11 +1021,16 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; + dma_dev = &hdma_dev->dma_dev; ret = dmaenginem_async_device_register(dma_dev); - if (ret < 0) + if (ret < 0) { dev_err(dev, "failed to register device!\n"); + return ret; + } + + hisi_dma_create_debugfs(hdma_dev); - return ret; + return 0; } static const struct pci_device_id hisi_dma_pci_tbl[] = { diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index 92caae55aece30e0c3244026c45037d7c9e5d4cf..af5a2e252c2527ed53a3f7dee452e1f7ed579a56 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -16,12 +16,20 @@ * port 3, and so on. */ +#include #include +#include #include #include #include +#include +#include #include +#include +#include #include +#include +#include #include "hsu.h" diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h index 9e5956345748d19e08721b98831519c51131fd12..3bca577b98a1f8584302a70c20772336349db4ee 100644 --- a/drivers/dma/hsu/hsu.h +++ b/drivers/dma/hsu/hsu.h @@ -10,7 +10,11 @@ #ifndef __DMA_HSU_H__ #define __DMA_HSU_H__ -#include +#include +#include +#include +#include + #include #include "../virt-dma.h" @@ -36,11 +40,11 @@ /* Bits in HSU_CH_SR */ #define HSU_CH_SR_DESCTO(x) BIT(8 + (x)) -#define HSU_CH_SR_DESCTO_ANY (BIT(11) | BIT(10) | BIT(9) | BIT(8)) +#define HSU_CH_SR_DESCTO_ANY GENMASK(11, 8) #define HSU_CH_SR_CHE BIT(15) #define HSU_CH_SR_DESCE(x) BIT(16 + (x)) -#define HSU_CH_SR_DESCE_ANY (BIT(19) | BIT(18) | BIT(17) | BIT(16)) -#define HSU_CH_SR_CDESC_ANY (BIT(31) | BIT(30)) +#define HSU_CH_SR_DESCE_ANY GENMASK(19, 16) +#define HSU_CH_SR_CDESC_ANY GENMASK(31, 30) /* Bits in HSU_CH_CR */ #define HSU_CH_CR_CHA BIT(0) diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index 6a2df3dd78d0b9e6cabb4ce3ff0d8d38647cf568..0fcc0c0c22fc5edfa128705fd9fef08ad75d4fe7 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -26,29 +27,32 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; - u32 dmaisr; - u32 status; + unsigned long dmaisr; unsigned short i; + u32 status; int ret = 0; int err; dmaisr = readl(chip->regs + HSU_PCI_DMAISR); - for (i = 0; i < chip->hsu->nr_channels; i++) { - if (dmaisr & 0x1) { - err = hsu_dma_get_status(chip, i, &status); - if (err > 0) - ret |= 1; - else if (err == 0) - ret |= hsu_dma_do_irq(chip, i, status); - } - dmaisr >>= 1; + for_each_set_bit(i, &dmaisr, chip->hsu->nr_channels) { + err = hsu_dma_get_status(chip, i, &status); + if (err > 0) + ret |= 1; + else if (err == 0) + ret |= hsu_dma_do_irq(chip, i, status); } return IRQ_RETVAL(ret); } +static void hsu_pci_dma_remove(void *chip) +{ + hsu_dma_remove(chip); +} + static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct device *dev = &pdev->dev; struct hsu_dma_chip *chip; int ret; @@ -87,9 +91,13 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; - ret = request_irq(chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip); + ret = devm_add_action_or_reset(dev, hsu_pci_dma_remove, chip); if (ret) - goto err_register_irq; + return ret; + + ret = devm_request_irq(dev, chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip); + if (ret) + return ret; /* * On Intel Tangier B0 and Anniedale the interrupt line, disregarding @@ -105,18 +113,6 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, chip); return 0; - -err_register_irq: - hsu_dma_remove(chip); - return ret; -} - -static void hsu_pci_remove(struct pci_dev *pdev) -{ - struct hsu_dma_chip *chip = pci_get_drvdata(pdev); - - free_irq(chip->irq, chip); - hsu_dma_remove(chip); } static const struct pci_device_id hsu_pci_id_table[] = { @@ -130,7 +126,6 @@ static struct pci_driver hsu_pci_driver = { .name = "hsu_dma_pci", .id_table = hsu_pci_id_table, .probe = hsu_pci_probe, - .remove = hsu_pci_remove, }; module_pci_driver(hsu_pci_driver); diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 5a8cc52c1abfd16938aa3d98ebfc28ebe323045a..2c1e6f6daa6286089b872d488e88d5a730588af0 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -196,6 +196,7 @@ int idxd_wq_enable(struct idxd_wq *wq) } wq->state = IDXD_WQ_ENABLED; + set_bit(wq->id, idxd->wq_enable_map); dev_dbg(dev, "WQ %d enabled\n", wq->id); return 0; } @@ -223,6 +224,7 @@ int idxd_wq_disable(struct idxd_wq *wq, bool reset_config) if (reset_config) idxd_wq_disable_cleanup(wq); + clear_bit(wq->id, idxd->wq_enable_map); wq->state = IDXD_WQ_DISABLED; dev_dbg(dev, "WQ %d disabled\n", wq->id); return 0; @@ -258,7 +260,6 @@ void idxd_wq_reset(struct idxd_wq *wq) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL); idxd_wq_disable_cleanup(wq); - wq->state = IDXD_WQ_DISABLED; } int idxd_wq_map_portal(struct idxd_wq *wq) @@ -378,17 +379,20 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) struct idxd_device *idxd = wq->idxd; lockdep_assert_held(&wq->wq_lock); + wq->state = IDXD_WQ_DISABLED; memset(wq->wqcfg, 0, idxd->wqcfg_size); wq->type = IDXD_WQT_NONE; wq->threshold = 0; wq->priority = 0; - wq->ats_dis = 0; wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; clear_bit(WQ_FLAG_DEDICATED, &wq->flags); clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); + clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); memset(wq->name, 0, WQ_NAME_SIZE); wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; wq->max_batch_size = WQ_DEFAULT_MAX_BATCH; + if (wq->opcap_bmap) + bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); } static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq) @@ -705,6 +709,8 @@ static void idxd_groups_clear_state(struct idxd_device *idxd) group->tc_a = -1; group->tc_b = -1; } + group->desc_progress_limit = 0; + group->batch_progress_limit = 0; } } @@ -761,10 +767,10 @@ static void idxd_group_config_write(struct idxd_group *group) /* setup GRPFLAGS */ grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); - iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); - dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", + iowrite64(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n", group->id, grpcfg_offset, - ioread32(idxd->reg_base + grpcfg_offset)); + ioread64(idxd->reg_base + grpcfg_offset)); } static int idxd_groups_config_write(struct idxd_device *idxd) @@ -807,7 +813,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq) struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; u32 wq_offset; - int i; + int i, n; if (!wq->group) return 0; @@ -859,12 +865,23 @@ static int idxd_wq_config_write(struct idxd_wq *wq) wq->wqcfg->bof = 1; if (idxd->hw.wq_cap.wq_ats_support) - wq->wqcfg->wq_ats_disable = wq->ats_dis; + wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); /* bytes 12-15 */ wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); + /* bytes 32-63 */ + if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) { + memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8); + for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) { + int pos = n % BITS_PER_LONG_LONG; + int idx = n / BITS_PER_LONG_LONG; + + wq->wqcfg->op_config[idx] |= BIT(pos); + } + } + dev_dbg(dev, "WQ %d CFGs\n", wq->id); for (i = 0; i < WQCFG_STRIDES(idxd); i++) { wq_offset = WQCFG_OFFSET(idxd, wq->id, i); @@ -914,6 +931,9 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) group->grpcfg.flags.rdbufs_allowed = group->rdbufs_allowed; else group->grpcfg.flags.rdbufs_allowed = idxd->max_rdbufs; + + group->grpcfg.flags.desc_progress_limit = group->desc_progress_limit; + group->grpcfg.flags.batch_progress_limit = group->batch_progress_limit; } } @@ -1096,8 +1116,8 @@ static void idxd_group_load_config(struct idxd_group *group) } grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); - group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset); - dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", + group->grpcfg.flags.bits = ioread64(idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n", group->id, grpcfg_offset, group->grpcfg.flags.bits); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index fed0dfc1eaa83766ed6ae1bc43de994bdba1122d..1196ab342f0113d658a75322506b44a8ff43043a 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include "registers.h" @@ -95,6 +96,8 @@ struct idxd_group { u8 rdbufs_reserved; int tc_a; int tc_b; + int desc_progress_limit; + int batch_progress_limit; }; struct idxd_pmu { @@ -132,6 +135,7 @@ enum idxd_wq_state { enum idxd_wq_flag { WQ_FLAG_DEDICATED = 0, WQ_FLAG_BLOCK_ON_FAULT, + WQ_FLAG_ATS_DISABLE, }; enum idxd_wq_type { @@ -194,6 +198,8 @@ struct idxd_wq { enum idxd_wq_state state; unsigned long flags; union wqcfg *wqcfg; + unsigned long *opcap_bmap; + struct dsa_hw_desc **hw_descs; int num_descs; union { @@ -208,7 +214,6 @@ struct idxd_wq { char name[WQ_NAME_SIZE + 1]; u64 max_xfer_bytes; u32 max_batch_size; - bool ats_dis; }; struct idxd_engine { @@ -299,6 +304,7 @@ struct idxd_device { int rdbuf_limit; int nr_rdbufs; /* non-reserved read buffers */ unsigned int wqcfg_size; + unsigned long *wq_enable_map; union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; @@ -308,6 +314,8 @@ struct idxd_device { struct work_struct work; struct idxd_pmu *idxd_pmu; + + unsigned long *opcap_bmap; }; /* IDXD software descriptor */ diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index aa3478257ddb54f73204bc80694f56e8fe288d70..2b18d512cbfc9bad1ab74b22a77f01b80c164b04 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -151,6 +151,12 @@ static int idxd_setup_wqs(struct idxd_device *idxd) if (!idxd->wqs) return -ENOMEM; + idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); + if (!idxd->wq_enable_map) { + kfree(idxd->wqs); + return -ENOMEM; + } + for (i = 0; i < idxd->max_wqs; i++) { wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); if (!wq) { @@ -185,6 +191,16 @@ static int idxd_setup_wqs(struct idxd_device *idxd) rc = -ENOMEM; goto err; } + + if (idxd->hw.wq_cap.op_config) { + wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); + if (!wq->opcap_bmap) { + put_device(conf_dev); + rc = -ENOMEM; + goto err; + } + bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); + } idxd->wqs[i] = wq; } @@ -369,6 +385,19 @@ static void idxd_read_table_offsets(struct idxd_device *idxd) dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); } +static void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) +{ + int i, j, nr; + + for (i = 0, nr = 0; i < count; i++) { + for (j = 0; j < BITS_PER_LONG_LONG; j++) { + if (val[i] & BIT(j)) + set_bit(nr, bmap); + nr++; + } + } +} + static void idxd_read_caps(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -427,6 +456,7 @@ static void idxd_read_caps(struct idxd_device *idxd) IDXD_OPCAP_OFFSET + i * sizeof(u64)); dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); } + multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); } static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) @@ -448,6 +478,12 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d if (idxd->id < 0) return NULL; + idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); + if (!idxd->opcap_bmap) { + ida_free(&idxd_ida, idxd->id); + return NULL; + } + device_initialize(conf_dev); conf_dev->parent = dev; conf_dev->bus = &dsa_bus_type; diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 743ead5ebc5796f3e853858ddd106d580b71f733..aa314ebec58783699fd8cd651fb0d5f54b37e25d 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -17,12 +17,6 @@ enum irq_work_type { IRQ_WORK_PROCESS_FAULT, }; -struct idxd_fault { - struct work_struct work; - u64 addr; - struct idxd_device *idxd; -}; - struct idxd_resubmit { struct work_struct work; struct idxd_desc *desc; @@ -49,11 +43,12 @@ static void idxd_device_reinit(struct work_struct *work) goto out; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; + if (test_bit(i, idxd->wq_enable_map)) { + struct idxd_wq *wq = idxd->wqs[i]; - if (wq->state == IDXD_WQ_ENABLED) { rc = idxd_wq_enable(wq); if (rc < 0) { + clear_bit(i, idxd->wq_enable_map); dev_warn(dev, "Unable to re-enable wq %s\n", dev_name(wq_confdev(wq))); } @@ -324,13 +319,11 @@ halt: idxd->state = IDXD_DEV_HALTED; idxd_wqs_quiesce(idxd); idxd_wqs_unmap_portal(idxd); - spin_lock(&idxd->dev_lock); idxd_device_clear_state(idxd); dev_err(&idxd->pdev->dev, "idxd halted, need %s.\n", gensts.reset_type == IDXD_DEVICE_RESET_FLR ? "FLR" : "system reset"); - spin_unlock(&idxd->dev_lock); return -ENXIO; } } diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 02449aa9c454f14e5c8b74d5dc06600dc0dd88a0..fe3b8d04f9db165d5df454a31e0a6b398f9451e4 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -54,7 +54,8 @@ union wq_cap_reg { u64 priority:1; u64 occupancy:1; u64 occupancy_int:1; - u64 rsvd3:10; + u64 op_config:1; + u64 rsvd3:9; }; u64 bits; } __packed; @@ -67,7 +68,8 @@ union group_cap_reg { u64 total_rdbufs:8; /* formerly total_tokens */ u64 rdbuf_ctrl:1; /* formerly token_en */ u64 rdbuf_limit:1; /* formerly token_limit */ - u64 rsvd:46; + u64 progress_limit:1; /* descriptor and batch descriptor */ + u64 rsvd:45; }; u64 bits; } __packed; @@ -90,6 +92,8 @@ struct opcap { u64 bits[4]; }; +#define IDXD_MAX_OPCAP_BITS 256U + #define IDXD_OPCAP_OFFSET 0x40 #define IDXD_TABLE_OFFSET 0x60 @@ -285,16 +289,20 @@ union msix_perm { union group_flags { struct { - u32 tc_a:3; - u32 tc_b:3; - u32 rsvd:1; - u32 use_rdbuf_limit:1; - u32 rdbufs_reserved:8; - u32 rsvd2:4; - u32 rdbufs_allowed:8; - u32 rsvd3:4; + u64 tc_a:3; + u64 tc_b:3; + u64 rsvd:1; + u64 use_rdbuf_limit:1; + u64 rdbufs_reserved:8; + u64 rsvd2:4; + u64 rdbufs_allowed:8; + u64 rsvd3:4; + u64 desc_progress_limit:2; + u64 rsvd4:2; + u64 batch_progress_limit:2; + u64 rsvd5:26; }; - u32 bits; + u64 bits; } __packed; struct grpcfg { @@ -348,8 +356,11 @@ union wqcfg { /* bytes 28-31 */ u32 rsvd8; + + /* bytes 32-63 */ + u64 op_config[4]; }; - u32 bits[8]; + u32 bits[16]; } __packed; #define WQCFG_PASID_IDX 2 diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 3f262a57441b47bda8a124462f48254b633d25d9..bdaccf9e04363befea2ea0a595c12760a3346117 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -443,6 +443,67 @@ static struct device_attribute dev_attr_group_traffic_class_b = __ATTR(traffic_class_b, 0644, group_traffic_class_b_show, group_traffic_class_b_store); +static ssize_t group_desc_progress_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%d\n", group->desc_progress_limit); +} + +static ssize_t group_desc_progress_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_group *group = confdev_to_group(dev); + int val, rc; + + rc = kstrtoint(buf, 10, &val); + if (rc < 0) + return -EINVAL; + + if (val & ~GENMASK(1, 0)) + return -EINVAL; + + group->desc_progress_limit = val; + return count; +} + +static struct device_attribute dev_attr_group_desc_progress_limit = + __ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show, + group_desc_progress_limit_store); + +static ssize_t group_batch_progress_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%d\n", group->batch_progress_limit); +} + +static ssize_t group_batch_progress_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_group *group = confdev_to_group(dev); + int val, rc; + + rc = kstrtoint(buf, 10, &val); + if (rc < 0) + return -EINVAL; + + if (val & ~GENMASK(1, 0)) + return -EINVAL; + + group->batch_progress_limit = val; + return count; +} + +static struct device_attribute dev_attr_group_batch_progress_limit = + __ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show, + group_batch_progress_limit_store); static struct attribute *idxd_group_attributes[] = { &dev_attr_group_work_queues.attr, &dev_attr_group_engines.attr, @@ -454,11 +515,35 @@ static struct attribute *idxd_group_attributes[] = { &dev_attr_group_read_buffers_reserved.attr, &dev_attr_group_traffic_class_a.attr, &dev_attr_group_traffic_class_b.attr, + &dev_attr_group_desc_progress_limit.attr, + &dev_attr_group_batch_progress_limit.attr, NULL, }; +static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + return (attr == &dev_attr_group_desc_progress_limit.attr || + attr == &dev_attr_group_batch_progress_limit.attr) && + !idxd->hw.group_cap.progress_limit; +} + +static umode_t idxd_group_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_group *group = confdev_to_group(dev); + struct idxd_device *idxd = group->idxd; + + if (idxd_group_attr_progress_limit_invisible(attr, idxd)) + return 0; + + return attr->mode; +} + static const struct attribute_group idxd_group_attribute_group = { .attrs = idxd_group_attributes, + .is_visible = idxd_group_attr_visible, }; static const struct attribute_group *idxd_group_attribute_groups[] = { @@ -973,7 +1058,7 @@ static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute * { struct idxd_wq *wq = confdev_to_wq(dev); - return sysfs_emit(buf, "%u\n", wq->ats_dis); + return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags)); } static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, @@ -994,7 +1079,10 @@ static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute if (rc < 0) return rc; - wq->ats_dis = ats_dis; + if (ats_dis) + set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); + else + clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); return count; } @@ -1055,6 +1143,68 @@ static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attrib static struct device_attribute dev_attr_wq_enqcmds_retries = __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store); +static ssize_t wq_op_config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap); +} + +static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask) +{ + int bit; + + /* + * The OPCAP is defined as 256 bits that represents each operation the device + * supports per bit. Iterate through all the bits and check if the input mask + * is set for bits that are not set in the OPCAP for the device. If no OPCAP + * bit is set and input mask has the bit set, then return error. + */ + for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) { + if (!test_bit(bit, idxd->opcap_bmap)) + return -EINVAL; + } + + return 0; +} + +static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + unsigned long *opmask; + int rc; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); + if (!opmask) + return -ENOMEM; + + rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS); + if (rc < 0) + goto err; + + rc = idxd_verify_supported_opcap(idxd, opmask); + if (rc < 0) + goto err; + + bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS); + + bitmap_free(opmask); + return count; + +err: + bitmap_free(opmask); + return rc; +} + +static struct device_attribute dev_attr_wq_op_config = + __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1072,11 +1222,33 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_ats_disable.attr, &dev_attr_wq_occupancy.attr, &dev_attr_wq_enqcmds_retries.attr, + &dev_attr_wq_op_config.attr, NULL, }; +static bool idxd_wq_attr_op_config_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + return attr == &dev_attr_wq_op_config.attr && + !idxd->hw.wq_cap.op_config; +} + +static umode_t idxd_wq_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + + if (idxd_wq_attr_op_config_invisible(attr, idxd)) + return 0; + + return attr->mode; +} + static const struct attribute_group idxd_wq_attribute_group = { .attrs = idxd_wq_attributes, + .is_visible = idxd_wq_attr_visible, }; static const struct attribute_group *idxd_wq_attribute_groups[] = { @@ -1088,6 +1260,7 @@ static void idxd_conf_wq_release(struct device *dev) { struct idxd_wq *wq = confdev_to_wq(dev); + bitmap_free(wq->opcap_bmap); kfree(wq->wqcfg); kfree(wq); } @@ -1177,14 +1350,8 @@ static ssize_t op_cap_show(struct device *dev, struct device_attribute *attr, char *buf) { struct idxd_device *idxd = confdev_to_idxd(dev); - int i, rc = 0; - - for (i = 0; i < 4; i++) - rc += sysfs_emit_at(buf, rc, "%#llx ", idxd->hw.opcap.bits[i]); - rc--; - rc += sysfs_emit_at(buf, rc, "\n"); - return rc; + return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap); } static DEVICE_ATTR_RO(op_cap); @@ -1405,9 +1572,11 @@ static void idxd_conf_device_release(struct device *dev) struct idxd_device *idxd = confdev_to_idxd(dev); kfree(idxd->groups); + bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); kfree(idxd->engines); ida_free(&idxd_ida, idxd->id); + bitmap_free(idxd->opcap_bmap); kfree(idxd); } diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 37ff4ec7db76fd77f46a7d40de7f0013e52206f3..e2070df6cad287bda75386961060339eb1b15d30 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -656,7 +656,7 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) if (active - i == 0) { dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n", __func__); - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } /* microsecond delay by sysfs variable per pending descriptor */ @@ -682,7 +682,7 @@ static void ioat_cleanup(struct ioatdma_chan *ioat_chan) if (chanerr & (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) { - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); ioat_eh(ioat_chan); } } @@ -879,7 +879,7 @@ static void check_active(struct ioatdma_chan *ioat_chan) } if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan) diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 140cfe3782fbb3d8ad3942db04d9f100c61d13c7..35e06b3826034a99416a9139b0da39cba9c433e0 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -196,10 +196,8 @@ extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; extern int ioat_pending_level; -extern int ioat_ring_alloc_order; extern struct kobj_type ioat_ktype; extern struct kmem_cache *ioat_cache; -extern int ioat_ring_max_alloc_order; extern struct kmem_cache *ioat_sed_cache; static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 994fc4d2aca42e6a6a0c34995877a3f97137a948..dc147cc2436e9e35566fffba413ba539baa9e05f 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -670,7 +670,7 @@ static enum dma_status mxs_dma_tx_status(struct dma_chan *chan, return mxs_chan->status; } -static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma) +static int mxs_dma_init(struct mxs_dma_engine *mxs_dma) { int ret; @@ -741,7 +741,7 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, ofdma->of_node); } -static int __init mxs_dma_probe(struct platform_device *pdev) +static int mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; const struct mxs_dma_type *dma_type; @@ -839,10 +839,7 @@ static struct platform_driver mxs_dma_driver = { .name = "mxs-dma", .of_match_table = mxs_dma_dt_ids, }, + .probe = mxs_dma_probe, }; -static int __init mxs_dma_module_init(void) -{ - return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe); -} -subsys_initcall(mxs_dma_module_init); +builtin_platform_driver(mxs_dma_driver); diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 09915a5cba3ea6bda18d6125eefb8a68284e4ac3..0d9257fbdfb0d90d902761406de722eff92b1888 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2752,7 +2752,6 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( return NULL; pch->cyclic = true; - desc->txd.flags = flags; return &desc->txd; } @@ -2804,8 +2803,6 @@ pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, desc->bytes_requested = len; - desc->txd.flags = flags; - return &desc->txd; } @@ -2889,7 +2886,6 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } /* Return the last desc in the chain */ - desc->txd.flags = flg; return &desc->txd; } diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 8f0c9c4e2efda0820e48db5a5c19bd8906d84f3c..3f56514bbef8f9d3178b2a67fdaf2fb6e0bb530d 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -1150,9 +1150,9 @@ static void gpi_ev_tasklet(unsigned long data) { struct gpii *gpii = (struct gpii *)data; - read_lock_bh(&gpii->pm_lock); + read_lock(&gpii->pm_lock); if (!REG_ACCESS_VALID(gpii->pm_state)) { - read_unlock_bh(&gpii->pm_lock); + read_unlock(&gpii->pm_lock); dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n", TO_GPI_PM_STR(gpii->pm_state)); return; @@ -1163,7 +1163,7 @@ static void gpi_ev_tasklet(unsigned long data) /* enable IEOB, switching back to interrupts */ gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1); - read_unlock_bh(&gpii->pm_lock); + read_unlock(&gpii->pm_lock); } /* marks all pending events for the channel as stale */ @@ -2288,6 +2288,7 @@ static int gpi_probe(struct platform_device *pdev) static const struct of_device_id gpi_of_match[] = { { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 }, + { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 }, { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 }, { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 }, { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 }, diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c index facdacf8aede6a0f5fc2138edc2c39843addf69d..d56caf1681ffb3a3a37f52edb109b48b00891045 100644 --- a/drivers/dma/qcom/qcom_adm.c +++ b/drivers/dma/qcom/qcom_adm.c @@ -379,13 +379,13 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, if (blk_size < 0) { dev_err(adev->dev, "invalid burst value: %d\n", burst); - return ERR_PTR(-EINVAL); + return NULL; } crci = achan->crci & 0xf; if (!crci || achan->crci > 0x1f) { dev_err(adev->dev, "invalid crci value\n"); - return ERR_PTR(-EINVAL); + return NULL; } } @@ -403,8 +403,10 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, } async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT); - if (!async_desc) - return ERR_PTR(-ENOMEM); + if (!async_desc) { + dev_err(adev->dev, "not enough memory for async_desc struct\n"); + return NULL; + } async_desc->mux = achan->mux ? ADM_CRCI_CTL_MUX_SEL : 0; async_desc->crci = crci; @@ -414,8 +416,10 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, sizeof(*cple) + 2 * ADM_DESC_ALIGN; async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT); - if (!async_desc->cpl) + if (!async_desc->cpl) { + dev_err(adev->dev, "not enough memory for cpl struct\n"); goto free; + } async_desc->adev = adev; @@ -437,8 +441,10 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl, async_desc->dma_len, DMA_TO_DEVICE); - if (dma_mapping_error(adev->dev, async_desc->dma_addr)) + if (dma_mapping_error(adev->dev, async_desc->dma_addr)) { + dev_err(adev->dev, "dma mapping error for cpl\n"); goto free; + } cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl); @@ -454,7 +460,7 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, free: kfree(async_desc); - return ERR_PTR(-ENOMEM); + return NULL; } /** @@ -494,7 +500,7 @@ static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) spin_lock_irqsave(&achan->vc.lock, flag); memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); - if (cfg->peripheral_size == sizeof(config)) + if (cfg->peripheral_size == sizeof(*config)) achan->crci = config->crci; spin_unlock_irqrestore(&achan->vc.lock, flag); diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c index f6ed7e88978131c76efc318ee7cf0ecc66fe62f4..a09eeb545f7d5880c62f7f9cd9e17a4da3c19ceb 100644 --- a/drivers/dma/s3c24xx-dma.c +++ b/drivers/dma/s3c24xx-dma.c @@ -1094,7 +1094,7 @@ static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma, INIT_LIST_HEAD(&dmadev->channels); /* - * Register as many many memcpy as we have physical channels, + * Register as many memcpy as we have physical channels, * we won't always be able to use all but the code will have * to cope with that situation. */ diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index 4f8b8498c5c62acef1abad81f95c453cb9a4e2fd..6b524eb6bcf3a016d84085248626d21fa8fe7c02 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -405,10 +405,8 @@ static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) chan = &pdma->chans[i]; irq = platform_get_irq(pdev, i * 2); - if (irq < 0) { - dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i); + if (irq < 0) return -EINVAL; - } r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, dev_name(&pdev->dev), (void *)chan); @@ -420,10 +418,8 @@ static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) chan->txirq = irq; irq = platform_get_irq(pdev, (i * 2) + 1); - if (irq < 0) { - dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i); + if (irq < 0) return -EINVAL; - } r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, dev_name(&pdev->dev), (void *)chan); diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 13d12d660cc2005756830ed19b96181444b7dd01..641d689d17ff189b485ca54673812aeb7976292a 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -103,8 +103,8 @@ struct rcar_dmac_desc_page { struct list_head node; union { - struct rcar_dmac_desc descs[0]; - struct rcar_dmac_xfer_chunk chunks[0]; + DECLARE_FLEX_ARRAY(struct rcar_dmac_desc, descs); + DECLARE_FLEX_ARRAY(struct rcar_dmac_xfer_chunk, chunks); }; }; diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index adb25a11c70fe00ae170461b9704bf16f1244c91..4891a1767e5aad6b13c28433faba3e4ec32fe330 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -9,6 +9,7 @@ * Pierre-Yves Mordret */ +#include #include #include #include @@ -32,8 +33,10 @@ #define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */ #define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */ +#define STM32_DMA_ISR(n) (((n) & 4) ? STM32_DMA_HISR : STM32_DMA_LISR) #define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */ #define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */ +#define STM32_DMA_IFCR(n) (((n) & 4) ? STM32_DMA_HIFCR : STM32_DMA_LIFCR) #define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */ #define STM32_DMA_HTI BIT(4) /* Half Transfer Interrupt */ #define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */ @@ -43,23 +46,22 @@ | STM32_DMA_TEI \ | STM32_DMA_DMEI \ | STM32_DMA_FEI) +/* + * If (chan->id % 4) is 2 or 3, left shift the mask by 16 bits; + * if (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. + */ +#define STM32_DMA_FLAGS_SHIFT(n) ({ typeof(n) (_n) = (n); \ + (((_n) & 2) << 3) | (((_n) & 1) * 6); }) /* DMA Stream x Configuration Register */ #define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */ -#define STM32_DMA_SCR_REQ(n) ((n & 0x7) << 25) +#define STM32_DMA_SCR_REQ_MASK GENMASK(27, 25) #define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23) -#define STM32_DMA_SCR_MBURST(n) ((n & 0x3) << 23) #define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21) -#define STM32_DMA_SCR_PBURST(n) ((n & 0x3) << 21) #define STM32_DMA_SCR_PL_MASK GENMASK(17, 16) -#define STM32_DMA_SCR_PL(n) ((n & 0x3) << 16) #define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13) -#define STM32_DMA_SCR_MSIZE(n) ((n & 0x3) << 13) #define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11) -#define STM32_DMA_SCR_PSIZE(n) ((n & 0x3) << 11) -#define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11) #define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6) -#define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6) #define STM32_DMA_SCR_TRBUFF BIT(20) /* Bufferable transfer for USART/UART */ #define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */ #define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */ @@ -96,7 +98,6 @@ /* DMA stream x FIFO control register */ #define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x)) #define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0) -#define STM32_DMA_SFCR_FTH(n) (n & STM32_DMA_SFCR_FTH_MASK) #define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */ #define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */ #define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \ @@ -137,11 +138,9 @@ /* DMA Features */ #define STM32_DMA_THRESHOLD_FTR_MASK GENMASK(1, 0) -#define STM32_DMA_THRESHOLD_FTR_GET(n) ((n) & STM32_DMA_THRESHOLD_FTR_MASK) #define STM32_DMA_DIRECT_MODE_MASK BIT(2) -#define STM32_DMA_DIRECT_MODE_GET(n) (((n) & STM32_DMA_DIRECT_MODE_MASK) >> 2) #define STM32_DMA_ALT_ACK_MODE_MASK BIT(4) -#define STM32_DMA_ALT_ACK_MODE_GET(n) (((n) & STM32_DMA_ALT_ACK_MODE_MASK) >> 4) +#define STM32_DMA_MDMA_STREAM_ID_MASK GENMASK(19, 16) enum stm32_dma_width { STM32_DMA_BYTE, @@ -195,6 +194,19 @@ struct stm32_dma_desc { struct stm32_dma_sg_req sg_req[]; }; +/** + * struct stm32_dma_mdma_config - STM32 DMA MDMA configuration + * @stream_id: DMA request to trigger STM32 MDMA transfer + * @ifcr: DMA interrupt flag clear register address, + * used by STM32 MDMA to clear DMA Transfer Complete flag + * @tcf: DMA Transfer Complete flag + */ +struct stm32_dma_mdma_config { + u32 stream_id; + u32 ifcr; + u32 tcf; +}; + struct stm32_dma_chan { struct virt_dma_chan vchan; bool config_init; @@ -209,6 +221,8 @@ struct stm32_dma_chan { u32 mem_burst; u32 mem_width; enum dma_status status; + bool trig_mdma; + struct stm32_dma_mdma_config mdma_config; }; struct stm32_dma_device { @@ -388,6 +402,13 @@ static int stm32_dma_slave_config(struct dma_chan *c, memcpy(&chan->dma_sconfig, config, sizeof(*config)); + /* Check if user is requesting DMA to trigger STM32 MDMA */ + if (config->peripheral_size) { + config->peripheral_config = &chan->mdma_config; + config->peripheral_size = sizeof(chan->mdma_config); + chan->trig_mdma = true; + } + chan->config_init = true; return 0; @@ -401,17 +422,10 @@ static u32 stm32_dma_irq_status(struct stm32_dma_chan *chan) /* * Read "flags" from DMA_xISR register corresponding to the selected * DMA channel at the correct bit offset inside that register. - * - * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. - * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. */ - if (chan->id & 4) - dma_isr = stm32_dma_read(dmadev, STM32_DMA_HISR); - else - dma_isr = stm32_dma_read(dmadev, STM32_DMA_LISR); - - flags = dma_isr >> (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); + dma_isr = stm32_dma_read(dmadev, STM32_DMA_ISR(chan->id)); + flags = dma_isr >> STM32_DMA_FLAGS_SHIFT(chan->id); return flags & STM32_DMA_MASKI; } @@ -424,17 +438,11 @@ static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags) /* * Write "flags" to the DMA_xIFCR register corresponding to the selected * DMA channel at the correct bit offset inside that register. - * - * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. - * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. */ flags &= STM32_DMA_MASKI; - dma_ifcr = flags << (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); + dma_ifcr = flags << STM32_DMA_FLAGS_SHIFT(chan->id); - if (chan->id & 4) - stm32_dma_write(dmadev, STM32_DMA_HIFCR, dma_ifcr); - else - stm32_dma_write(dmadev, STM32_DMA_LIFCR, dma_ifcr); + stm32_dma_write(dmadev, STM32_DMA_IFCR(chan->id), dma_ifcr); } static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) @@ -576,6 +584,10 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) sg_req = &chan->desc->sg_req[chan->next_sg]; reg = &sg_req->chan_reg; + /* When DMA triggers STM32 MDMA, DMA Transfer Complete is managed by STM32 MDMA */ + if (chan->trig_mdma && chan->dma_sconfig.direction != DMA_MEM_TO_DEV) + reg->dma_scr &= ~STM32_DMA_SCR_TCIE; + reg->dma_scr &= ~STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); @@ -725,6 +737,8 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) if (chan->desc->cyclic) { vchan_cyclic_callback(&chan->desc->vdesc); + if (chan->trig_mdma) + return; stm32_dma_sg_inc(chan); /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */ if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))) @@ -861,7 +875,8 @@ static int stm32_dma_resume(struct dma_chan *c) sg_req = &chan->desc->sg_req[chan->next_sg - 1]; ndtr = sg_req->chan_reg.dma_sndtr; - offset = (ndtr - chan_reg.dma_sndtr) << STM32_DMA_SCR_PSIZE_GET(chan_reg.dma_scr); + offset = (ndtr - chan_reg.dma_sndtr); + offset <<= FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, chan_reg.dma_scr); spar = sg_req->chan_reg.dma_spar; sm0ar = sg_req->chan_reg.dma_sm0ar; sm1ar = sg_req->chan_reg.dma_sm1ar; @@ -973,16 +988,16 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (src_burst_size < 0) return src_burst_size; - dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_DEV) | - STM32_DMA_SCR_PSIZE(dst_bus_width) | - STM32_DMA_SCR_MSIZE(src_bus_width) | - STM32_DMA_SCR_PBURST(dst_burst_size) | - STM32_DMA_SCR_MBURST(src_burst_size); + dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_DEV) | + FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, dst_bus_width) | + FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, src_bus_width) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dst_burst_size) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, src_burst_size); /* Set FIFO threshold */ chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE) - chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth); + chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); /* Set peripheral address */ chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr; @@ -1030,16 +1045,16 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (dst_burst_size < 0) return dst_burst_size; - dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_DEV_TO_MEM) | - STM32_DMA_SCR_PSIZE(src_bus_width) | - STM32_DMA_SCR_MSIZE(dst_bus_width) | - STM32_DMA_SCR_PBURST(src_burst_size) | - STM32_DMA_SCR_MBURST(dst_burst_size); + dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_DEV_TO_MEM) | + FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, src_bus_width) | + FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, dst_bus_width) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, src_burst_size) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dst_burst_size); /* Set FIFO threshold */ chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE) - chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth); + chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); /* Set peripheral address */ chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr; @@ -1099,6 +1114,10 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( else chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; + /* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */ + if (chan->trig_mdma && sg_len > 1) + chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; + for_each_sg(sgl, sg, sg_len, i) { ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, sg_dma_len(sg), @@ -1120,6 +1139,8 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg); desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg); + if (chan->trig_mdma) + desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg); desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; } @@ -1207,8 +1228,11 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr; desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr; + if (chan->trig_mdma) + desc->sg_req[i].chan_reg.dma_sm1ar += period_len; desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; - buf_addr += period_len; + if (!chan->trig_mdma) + buf_addr += period_len; } desc->num_sgs = num_periods; @@ -1247,16 +1271,15 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); desc->sg_req[i].chan_reg.dma_scr = - STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_MEM) | - STM32_DMA_SCR_PBURST(dma_burst) | - STM32_DMA_SCR_MBURST(dma_burst) | + FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_MEM) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dma_burst) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dma_burst) | STM32_DMA_SCR_MINC | STM32_DMA_SCR_PINC | STM32_DMA_SCR_TCIE | STM32_DMA_SCR_TEIE; desc->sg_req[i].chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK; - desc->sg_req[i].chan_reg.dma_sfcr |= - STM32_DMA_SFCR_FTH(threshold); + desc->sg_req[i].chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, threshold); desc->sg_req[i].chan_reg.dma_spar = src + offset; desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset; desc->sg_req[i].chan_reg.dma_sndtr = xfer_count; @@ -1275,7 +1298,7 @@ static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan) struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); - width = STM32_DMA_SCR_PSIZE_GET(dma_scr); + width = FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, dma_scr); ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); return ndtr << width; @@ -1481,16 +1504,17 @@ static void stm32_dma_set_config(struct stm32_dma_chan *chan, stm32_dma_clear_reg(&chan->chan_reg); chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK; - chan->chan_reg.dma_scr |= STM32_DMA_SCR_REQ(cfg->request_line); + chan->chan_reg.dma_scr |= FIELD_PREP(STM32_DMA_SCR_REQ_MASK, cfg->request_line); /* Enable Interrupts */ chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE; - chan->threshold = STM32_DMA_THRESHOLD_FTR_GET(cfg->features); - if (STM32_DMA_DIRECT_MODE_GET(cfg->features)) + chan->threshold = FIELD_GET(STM32_DMA_THRESHOLD_FTR_MASK, cfg->features); + if (FIELD_GET(STM32_DMA_DIRECT_MODE_MASK, cfg->features)) chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE; - if (STM32_DMA_ALT_ACK_MODE_GET(cfg->features)) + if (FIELD_GET(STM32_DMA_ALT_ACK_MODE_MASK, cfg->features)) chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF; + chan->mdma_config.stream_id = FIELD_GET(STM32_DMA_MDMA_STREAM_ID_MASK, cfg->features); } static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec, @@ -1630,6 +1654,12 @@ static int stm32_dma_probe(struct platform_device *pdev) chan->id = i; chan->vchan.desc_free = stm32_dma_desc_free; vchan_init(&chan->vchan, dd); + + chan->mdma_config.ifcr = res->start; + chan->mdma_config.ifcr += STM32_DMA_IFCR(chan->id); + + chan->mdma_config.tcf = STM32_DMA_TCI; + chan->mdma_config.tcf <<= STM32_DMA_FLAGS_SHIFT(chan->id); } ret = dma_async_device_register(dd); diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32-dmamux.c index eee0c5aa5fb57fae088a350252ea44b5e3c69003..ee3cbbf510065acc5d5dbd58d3d7acdb4c5eae3d 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32-dmamux.c @@ -39,13 +39,13 @@ struct stm32_dmamux_data { u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ spinlock_t lock; /* Protects register access */ - unsigned long *dma_inuse; /* Used DMA channel */ + DECLARE_BITMAP(dma_inuse, STM32_DMAMUX_MAX_DMA_REQUESTS); /* Used DMA channel */ u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register * in suspend */ u32 dma_reqs[]; /* Number of DMA Request per DMA masters. * [0] holds number of DMA Masters. - * To be kept at very end end of this structure + * To be kept at very end of this structure */ }; @@ -147,7 +147,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, mux->request = dma_spec->args[0]; /* craft DMA spec */ - dma_spec->args[3] = dma_spec->args[2]; + dma_spec->args[3] = dma_spec->args[2] | mux->chan_id << 16; dma_spec->args[2] = dma_spec->args[1]; dma_spec->args[1] = 0; dma_spec->args[0] = mux->chan_id - min; @@ -229,12 +229,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev) stm32_dmamux->dma_requests = dma_req; stm32_dmamux->dma_reqs[0] = count; - stm32_dmamux->dma_inuse = devm_kcalloc(&pdev->dev, - BITS_TO_LONGS(dma_req), - sizeof(unsigned long), - GFP_KERNEL); - if (!stm32_dmamux->dma_inuse) - return -ENOMEM; if (device_property_read_u32(&pdev->dev, "dma-requests", &stm32_dmamux->dmamux_requests)) { diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index b11927ed4367d49e3d7d663e179723c69aaeb597..e28acbcb53f4c701f70542e359830d42368580d8 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -199,6 +199,7 @@ struct stm32_mdma_chan_config { u32 transfer_config; u32 mask_addr; u32 mask_data; + bool m2m_hw; /* True when MDMA is triggered by STM32 DMA */ }; struct stm32_mdma_hwdesc { @@ -227,6 +228,12 @@ struct stm32_mdma_desc { struct stm32_mdma_desc_node node[]; }; +struct stm32_mdma_dma_config { + u32 request; /* STM32 DMA channel stream id, triggering MDMA */ + u32 cmar; /* STM32 DMA interrupt flag clear register address */ + u32 cmdr; /* STM32 DMA Transfer Complete flag */ +}; + struct stm32_mdma_chan { struct virt_dma_chan vchan; struct dma_pool *desc_pool; @@ -539,13 +546,23 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, dst_addr = chan->dma_config.dst_addr; /* Set device data size */ + if (chan_config->m2m_hw) + dst_addr_width = stm32_mdma_get_max_width(dst_addr, buf_len, + STM32_MDMA_MAX_BUF_LEN); dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); if (dst_bus_width < 0) return dst_bus_width; ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK; ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width); + if (chan_config->m2m_hw) { + ctcr &= ~STM32_MDMA_CTCR_DINCOS_MASK; + ctcr |= STM32_MDMA_CTCR_DINCOS(dst_bus_width); + } /* Set device burst value */ + if (chan_config->m2m_hw) + dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width; + dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, dst_maxburst, dst_addr_width); @@ -588,13 +605,24 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, src_addr = chan->dma_config.src_addr; /* Set device data size */ + if (chan_config->m2m_hw) + src_addr_width = stm32_mdma_get_max_width(src_addr, buf_len, + STM32_MDMA_MAX_BUF_LEN); + src_bus_width = stm32_mdma_get_width(chan, src_addr_width); if (src_bus_width < 0) return src_bus_width; ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK; ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width); + if (chan_config->m2m_hw) { + ctcr &= ~STM32_MDMA_CTCR_SINCOS_MASK; + ctcr |= STM32_MDMA_CTCR_SINCOS(src_bus_width); + } /* Set device burst value */ + if (chan_config->m2m_hw) + src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width; + src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, src_maxburst, src_addr_width); @@ -702,11 +730,15 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, { struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); struct dma_slave_config *dma_config = &chan->dma_config; + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct scatterlist *sg; dma_addr_t src_addr, dst_addr; - u32 ccr, ctcr, ctbr; + u32 m2m_hw_period, ccr, ctcr, ctbr; int i, ret = 0; + if (chan_config->m2m_hw) + m2m_hw_period = sg_dma_len(sgl); + for_each_sg(sgl, sg, sg_len, i) { if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) { dev_err(chan2dev(chan), "Invalid block len\n"); @@ -716,6 +748,8 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, if (direction == DMA_MEM_TO_DEV) { src_addr = sg_dma_address(sg); dst_addr = dma_config->dst_addr; + if (chan_config->m2m_hw && (i & 1)) + dst_addr += m2m_hw_period; ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, &ctbr, src_addr, sg_dma_len(sg)); @@ -723,6 +757,8 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, src_addr); } else { src_addr = dma_config->src_addr; + if (chan_config->m2m_hw && (i & 1)) + src_addr += m2m_hw_period; dst_addr = sg_dma_address(sg); ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, &ctbr, dst_addr, @@ -755,6 +791,7 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, unsigned long flags, void *context) { struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct stm32_mdma_desc *desc; int i, ret; @@ -777,6 +814,21 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, if (ret < 0) goto xfer_setup_err; + /* + * In case of M2M HW transfer triggered by STM32 DMA, we do not have to clear the + * transfer complete flag by hardware in order to let the CPU rearm the STM32 DMA + * with the next sg element and update some data in dmaengine framework. + */ + if (chan_config->m2m_hw && direction == DMA_MEM_TO_DEV) { + struct stm32_mdma_hwdesc *hwdesc; + + for (i = 0; i < sg_len; i++) { + hwdesc = desc->node[i].hwdesc; + hwdesc->cmar = 0; + hwdesc->cmdr = 0; + } + } + desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -798,6 +850,7 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); struct dma_slave_config *dma_config = &chan->dma_config; + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct stm32_mdma_desc *desc; dma_addr_t src_addr, dst_addr; u32 ccr, ctcr, ctbr, count; @@ -858,8 +911,12 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, if (direction == DMA_MEM_TO_DEV) { src_addr = buf_addr + i * period_len; dst_addr = dma_config->dst_addr; + if (chan_config->m2m_hw && (i & 1)) + dst_addr += period_len; } else { src_addr = dma_config->src_addr; + if (chan_config->m2m_hw && (i & 1)) + src_addr += period_len; dst_addr = buf_addr + i * period_len; } @@ -1244,6 +1301,17 @@ static int stm32_mdma_slave_config(struct dma_chan *c, memcpy(&chan->dma_config, config, sizeof(*config)); + /* Check if user is requesting STM32 DMA to trigger MDMA */ + if (config->peripheral_size) { + struct stm32_mdma_dma_config *mdma_config; + + mdma_config = (struct stm32_mdma_dma_config *)chan->dma_config.peripheral_config; + chan->chan_config.request = mdma_config->request; + chan->chan_config.mask_addr = mdma_config->cmar; + chan->chan_config.mask_data = mdma_config->cmdr; + chan->chan_config.m2m_hw = true; + } + return 0; } diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 4cbca80ee16e606a94d3bb2d0e6917d36c01a1f4..fa06d7e6d8e3837637a28e4a3741ddc8bea3873c 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -352,12 +352,6 @@ static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i, edma_modify(ecc, offset + (i << 2), and, or); } -static inline void edma_or_array(struct edma_cc *ecc, int offset, int i, - unsigned or) -{ - edma_or(ecc, offset + (i << 2), or); -} - static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j, unsigned or) { @@ -370,11 +364,6 @@ static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i, edma_write(ecc, offset + ((i * 2 + j) << 2), val); } -static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset) -{ - return edma_read(ecc, EDMA_SHADOW0 + offset); -} - static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc, int offset, int i) { @@ -393,36 +382,12 @@ static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset, edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val); } -static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset, - int param_no) -{ - return edma_read(ecc, EDMA_PARM + offset + (param_no << 5)); -} - -static inline void edma_param_write(struct edma_cc *ecc, int offset, - int param_no, unsigned val) -{ - edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val); -} - static inline void edma_param_modify(struct edma_cc *ecc, int offset, int param_no, unsigned and, unsigned or) { edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or); } -static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no, - unsigned and) -{ - edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and); -} - -static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no, - unsigned or) -{ - edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or); -} - static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no, int priority) { @@ -743,11 +708,6 @@ static void edma_free_channel(struct edma_chan *echan) edma_setup_interrupt(echan, false); } -static inline struct edma_cc *to_edma_cc(struct dma_device *d) -{ - return container_of(d, struct edma_cc, dma_slave); -} - static inline struct edma_chan *to_edma_chan(struct dma_chan *c) { return container_of(c, struct edma_chan, vchan.chan); diff --git a/drivers/dma/ti/k3-psil-j7200.c b/drivers/dma/ti/k3-psil-j7200.c index 5ea63ea74822347f15d1b66cf6350f23f1b9146e..e3feff8699912e6be315b984577d447dd13f8430 100644 --- a/drivers/dma/ti/k3-psil-j7200.c +++ b/drivers/dma/ti/k3-psil-j7200.c @@ -143,6 +143,57 @@ static struct psil_ep j7200_src_ep_map[] = { /* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ static struct psil_ep j7200_dst_ep_map[] = { + /* PDMA_MCASP - McASP0-2 */ + PSIL_PDMA_MCASP(0xc400), + PSIL_PDMA_MCASP(0xc401), + PSIL_PDMA_MCASP(0xc402), + /* PDMA_SPI_G0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0xc600), + PSIL_PDMA_XY_PKT(0xc601), + PSIL_PDMA_XY_PKT(0xc602), + PSIL_PDMA_XY_PKT(0xc603), + PSIL_PDMA_XY_PKT(0xc604), + PSIL_PDMA_XY_PKT(0xc605), + PSIL_PDMA_XY_PKT(0xc606), + PSIL_PDMA_XY_PKT(0xc607), + PSIL_PDMA_XY_PKT(0xc608), + PSIL_PDMA_XY_PKT(0xc609), + PSIL_PDMA_XY_PKT(0xc60a), + PSIL_PDMA_XY_PKT(0xc60b), + PSIL_PDMA_XY_PKT(0xc60c), + PSIL_PDMA_XY_PKT(0xc60d), + PSIL_PDMA_XY_PKT(0xc60e), + PSIL_PDMA_XY_PKT(0xc60f), + /* PDMA_SPI_G1 - SPI4-7 */ + PSIL_PDMA_XY_PKT(0xc610), + PSIL_PDMA_XY_PKT(0xc611), + PSIL_PDMA_XY_PKT(0xc612), + PSIL_PDMA_XY_PKT(0xc613), + PSIL_PDMA_XY_PKT(0xc614), + PSIL_PDMA_XY_PKT(0xc615), + PSIL_PDMA_XY_PKT(0xc616), + PSIL_PDMA_XY_PKT(0xc617), + PSIL_PDMA_XY_PKT(0xc618), + PSIL_PDMA_XY_PKT(0xc619), + PSIL_PDMA_XY_PKT(0xc61a), + PSIL_PDMA_XY_PKT(0xc61b), + PSIL_PDMA_XY_PKT(0xc61c), + PSIL_PDMA_XY_PKT(0xc61d), + PSIL_PDMA_XY_PKT(0xc61e), + PSIL_PDMA_XY_PKT(0xc61f), + /* PDMA_USART_G0 - UART0-1 */ + PSIL_PDMA_XY_PKT(0xc700), + PSIL_PDMA_XY_PKT(0xc701), + /* PDMA_USART_G1 - UART2-3 */ + PSIL_PDMA_XY_PKT(0xc702), + PSIL_PDMA_XY_PKT(0xc703), + /* PDMA_USART_G2 - UART4-9 */ + PSIL_PDMA_XY_PKT(0xc704), + PSIL_PDMA_XY_PKT(0xc705), + PSIL_PDMA_XY_PKT(0xc706), + PSIL_PDMA_XY_PKT(0xc707), + PSIL_PDMA_XY_PKT(0xc708), + PSIL_PDMA_XY_PKT(0xc709), /* CPSW5 */ PSIL_ETHERNET(0xca00), PSIL_ETHERNET(0xca01), @@ -161,6 +212,22 @@ static struct psil_ep j7200_dst_ep_map[] = { PSIL_ETHERNET(0xf005), PSIL_ETHERNET(0xf006), PSIL_ETHERNET(0xf007), + /* MCU_PDMA_MISC_G0 - SPI0 */ + PSIL_PDMA_XY_PKT(0xf100), + PSIL_PDMA_XY_PKT(0xf101), + PSIL_PDMA_XY_PKT(0xf102), + PSIL_PDMA_XY_PKT(0xf103), + /* MCU_PDMA_MISC_G1 - SPI1-2 */ + PSIL_PDMA_XY_PKT(0xf200), + PSIL_PDMA_XY_PKT(0xf201), + PSIL_PDMA_XY_PKT(0xf202), + PSIL_PDMA_XY_PKT(0xf203), + PSIL_PDMA_XY_PKT(0xf204), + PSIL_PDMA_XY_PKT(0xf205), + PSIL_PDMA_XY_PKT(0xf206), + PSIL_PDMA_XY_PKT(0xf207), + /* MCU_PDMA_MISC_G2 - UART0 */ + PSIL_PDMA_XY_PKT(0xf300), /* SA2UL */ PSIL_SA2UL(0xf500, 1), PSIL_SA2UL(0xf501, 1), diff --git a/drivers/dma/ti/k3-psil-j721e.c b/drivers/dma/ti/k3-psil-j721e.c index 34e3fc565a37866ef2fa6afff43734d55d1ed762..e7c83d668bb66c130e390d36185fe60b964ce24b 100644 --- a/drivers/dma/ti/k3-psil-j721e.c +++ b/drivers/dma/ti/k3-psil-j721e.c @@ -266,6 +266,69 @@ static struct psil_ep j721e_dst_ep_map[] = { PSIL_ETHERNET(0xc205), PSIL_ETHERNET(0xc206), PSIL_ETHERNET(0xc207), + /* PDMA6 (PSIL_PDMA_MCASP_G0) - McASP0-2 */ + PSIL_PDMA_MCASP(0xc400), + PSIL_PDMA_MCASP(0xc401), + PSIL_PDMA_MCASP(0xc402), + /* PDMA7 (PSIL_PDMA_MCASP_G1) - McASP3-11 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + PSIL_PDMA_MCASP(0xc503), + PSIL_PDMA_MCASP(0xc504), + PSIL_PDMA_MCASP(0xc505), + PSIL_PDMA_MCASP(0xc506), + PSIL_PDMA_MCASP(0xc507), + PSIL_PDMA_MCASP(0xc508), + /* PDMA8 (PDMA_MISC_G0) - SPI0-1 */ + PSIL_PDMA_XY_PKT(0xc600), + PSIL_PDMA_XY_PKT(0xc601), + PSIL_PDMA_XY_PKT(0xc602), + PSIL_PDMA_XY_PKT(0xc603), + PSIL_PDMA_XY_PKT(0xc604), + PSIL_PDMA_XY_PKT(0xc605), + PSIL_PDMA_XY_PKT(0xc606), + PSIL_PDMA_XY_PKT(0xc607), + /* PDMA9 (PDMA_MISC_G1) - SPI2-3 */ + PSIL_PDMA_XY_PKT(0xc60c), + PSIL_PDMA_XY_PKT(0xc60d), + PSIL_PDMA_XY_PKT(0xc60e), + PSIL_PDMA_XY_PKT(0xc60f), + PSIL_PDMA_XY_PKT(0xc610), + PSIL_PDMA_XY_PKT(0xc611), + PSIL_PDMA_XY_PKT(0xc612), + PSIL_PDMA_XY_PKT(0xc613), + /* PDMA10 (PDMA_MISC_G2) - SPI4-5 */ + PSIL_PDMA_XY_PKT(0xc618), + PSIL_PDMA_XY_PKT(0xc619), + PSIL_PDMA_XY_PKT(0xc61a), + PSIL_PDMA_XY_PKT(0xc61b), + PSIL_PDMA_XY_PKT(0xc61c), + PSIL_PDMA_XY_PKT(0xc61d), + PSIL_PDMA_XY_PKT(0xc61e), + PSIL_PDMA_XY_PKT(0xc61f), + /* PDMA11 (PDMA_MISC_G3) */ + PSIL_PDMA_XY_PKT(0xc624), + PSIL_PDMA_XY_PKT(0xc625), + PSIL_PDMA_XY_PKT(0xc626), + PSIL_PDMA_XY_PKT(0xc627), + PSIL_PDMA_XY_PKT(0xc628), + PSIL_PDMA_XY_PKT(0xc629), + PSIL_PDMA_XY_PKT(0xc630), + PSIL_PDMA_XY_PKT(0xc63a), + /* PDMA13 (PDMA_USART_G0) - UART0-1 */ + PSIL_PDMA_XY_PKT(0xc700), + PSIL_PDMA_XY_PKT(0xc701), + /* PDMA14 (PDMA_USART_G1) - UART2-3 */ + PSIL_PDMA_XY_PKT(0xc702), + PSIL_PDMA_XY_PKT(0xc703), + /* PDMA15 (PDMA_USART_G2) - UART4-9 */ + PSIL_PDMA_XY_PKT(0xc704), + PSIL_PDMA_XY_PKT(0xc705), + PSIL_PDMA_XY_PKT(0xc706), + PSIL_PDMA_XY_PKT(0xc707), + PSIL_PDMA_XY_PKT(0xc708), + PSIL_PDMA_XY_PKT(0xc709), /* CPSW9 */ PSIL_ETHERNET(0xca00), PSIL_ETHERNET(0xca01), @@ -284,6 +347,22 @@ static struct psil_ep j721e_dst_ep_map[] = { PSIL_ETHERNET(0xf005), PSIL_ETHERNET(0xf006), PSIL_ETHERNET(0xf007), + /* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */ + PSIL_PDMA_XY_PKT(0xf100), + PSIL_PDMA_XY_PKT(0xf101), + PSIL_PDMA_XY_PKT(0xf102), + PSIL_PDMA_XY_PKT(0xf103), + /* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */ + PSIL_PDMA_XY_PKT(0xf200), + PSIL_PDMA_XY_PKT(0xf201), + PSIL_PDMA_XY_PKT(0xf202), + PSIL_PDMA_XY_PKT(0xf203), + PSIL_PDMA_XY_PKT(0xf204), + PSIL_PDMA_XY_PKT(0xf205), + PSIL_PDMA_XY_PKT(0xf206), + PSIL_PDMA_XY_PKT(0xf207), + /* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */ + PSIL_PDMA_XY_PKT(0xf300), /* SA2UL */ PSIL_SA2UL(0xf500, 1), PSIL_SA2UL(0xf501, 1), diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index d4f1e4e9603a400308e514350a2f3ec330c9cbaf..85e00701473cb121ab668c97a2b5c875e082615a 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -31,14 +31,14 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property) } pdev = of_find_device_by_node(udma_node); + if (np != udma_node) + of_node_put(udma_node); + if (!pdev) { pr_debug("UDMA device not found\n"); return ERR_PTR(-EPROBE_DEFER); } - if (np != udma_node) - of_node_put(udma_node); - ud = platform_get_drvdata(pdev); if (!ud) { pr_debug("UDMA has not been probed\n"); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 2f0d2c68c93c6152c433d47b4c4d5321feeec53b..7b5081989b3d6f76b6657b901362fc219ffd7396 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -263,6 +263,7 @@ struct udma_chan_config { enum udma_tp_level channel_tpl; /* Channel Throughput Level */ u32 tr_trigger_type; + unsigned long tx_flags; /* PKDMA mapped channel */ int mapped_channel_id; @@ -300,8 +301,6 @@ struct udma_chan { struct udma_tx_drain tx_drain; - u32 bcnt; /* number of bytes completed since the start of the channel */ - /* Channel configuration parameters */ struct udma_chan_config config; @@ -757,6 +756,20 @@ static void udma_reset_rings(struct udma_chan *uc) } } +static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val) +{ + if (uc->desc->dir == DMA_DEV_TO_MEM) { + udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); + udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + } else { + udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); + udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); + if (!uc->bchan) + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + } +} + static void udma_reset_counters(struct udma_chan *uc) { u32 val; @@ -790,8 +803,6 @@ static void udma_reset_counters(struct udma_chan *uc) val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); } - - uc->bcnt = 0; } static int udma_reset_chan(struct udma_chan *uc, bool hard) @@ -1045,9 +1056,14 @@ static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) { u32 peer_bcnt, bcnt; - /* Only TX towards PDMA is affected */ + /* + * Only TX towards PDMA is affected. + * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer + * completion calculation, consumer must ensure that there is no stale + * data in DMA fabric in this case. + */ if (uc->config.ep_type == PSIL_EP_NATIVE || - uc->config.dir != DMA_MEM_TO_DEV) + uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT)) return true; peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); @@ -1115,7 +1131,7 @@ static void udma_check_tx_completion(struct work_struct *work) if (uc->desc) { struct udma_desc *d = uc->desc; - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); break; @@ -1168,7 +1184,7 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data) vchan_cyclic_callback(&d->vd); } else { if (udma_is_desc_really_done(uc, d)) { - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); } else { @@ -1204,7 +1220,7 @@ static irqreturn_t udma_udma_irq_handler(int irq, void *data) vchan_cyclic_callback(&d->vd); } else { /* TODO: figure out the real amount of data */ - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); } @@ -3408,6 +3424,8 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!burst) burst = 1; + uc->config.tx_flags = tx_flags; + if (uc->config.pkt_mode) d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, context); @@ -3809,7 +3827,6 @@ static enum dma_status udma_tx_status(struct dma_chan *chan, bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); } - bcnt -= uc->bcnt; if (bcnt && !(bcnt % uc->desc->residue)) residue = 0; else diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 6276934d4d2be9e3345c20921c5cdfce941f4a79..8cd4e69dc7b4c24b334c450562c2a88925b0f452 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -3040,9 +3040,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Request and map I/O memory */ xdev->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(xdev->regs)) - return PTR_ERR(xdev->regs); - + if (IS_ERR(xdev->regs)) { + err = PTR_ERR(xdev->regs); + goto disable_clks; + } /* Retrieve the DMA engine properties from the device tree */ xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; @@ -3070,7 +3071,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) if (err < 0) { dev_err(xdev->dev, "missing xlnx,num-fstores property\n"); - return err; + goto disable_clks; } err = of_property_read_u32(node, "xlnx,flush-fsync", @@ -3090,7 +3091,11 @@ static int xilinx_dma_probe(struct platform_device *pdev) xdev->ext_addr = false; /* Set the dma mask bits */ - dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); + err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); + if (err < 0) { + dev_err(xdev->dev, "DMA mask error %d\n", err); + goto disable_clks; + } /* Initialize the DMA engine */ xdev->common.dev = &pdev->dev; @@ -3137,7 +3142,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) for_each_child_of_node(node, child) { err = xilinx_dma_child_probe(xdev, child); if (err < 0) - goto disable_clks; + goto error; } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { @@ -3172,12 +3177,12 @@ static int xilinx_dma_probe(struct platform_device *pdev) return 0; -disable_clks: - xdma_disable_allclks(xdev); error: for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); +disable_clks: + xdma_disable_allclks(xdev); return err; } diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index dc299ab3681803ce094cafaaba1d7fa92168baf4..21472a5d76368057601c425addf3c0373f2b6909 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -795,6 +795,17 @@ static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan) return 0; } +/** + * zynqmp_dma_synchronize - Synchronizes the termination of a transfers to the current context. + * @dchan: DMA channel pointer + */ +static void zynqmp_dma_synchronize(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + tasklet_kill(&chan->tasklet); +} + /** * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction * @dchan: DMA channel @@ -849,7 +860,7 @@ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( zynqmp_dma_desc_config_eod(chan, desc); async_tx_ack(&first->async_tx); - first->async_tx.flags = flags; + first->async_tx.flags = (enum dma_ctrl_flags)flags; return &first->async_tx; } @@ -1057,6 +1068,7 @@ static int zynqmp_dma_probe(struct platform_device *pdev) p = &zdev->common; p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy; p->device_terminate_all = zynqmp_dma_device_terminate_all; + p->device_synchronize = zynqmp_dma_synchronize; p->device_issue_pending = zynqmp_dma_issue_pending; p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources; p->device_free_chan_resources = zynqmp_dma_free_chan_resources; diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 17562cf1fe973715fc7fdbd5b7b298a3c1c81d4d..456602d373b7b1e71c438734aa6f7a032529ff47 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -473,7 +473,7 @@ config EDAC_ALTERA_SDMMC config EDAC_SIFIVE bool "Sifive platform EDAC driver" - depends on EDAC=y && SIFIVE_L2 + depends on EDAC=y && SIFIVE_CCACHE help Support for error detection and correction on the SiFive SoCs. diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index eb58644bb0190630af007f2ad10363b3bb5cde6c..6faeb2ab39601b561924aa6ec04ace126b963489 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -103,7 +103,6 @@ static void edac_mc_dump_dimm(struct dimm_info *dimm) edac_dbg(4, " dimm->label = '%s'\n", dimm->label); edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages); edac_dbg(4, " dimm->grain = %d\n", dimm->grain); - edac_dbg(4, " dimm->nr_pages = 0x%x\n", dimm->nr_pages); } static void edac_mc_dump_csrow(struct csrow_info *csrow) diff --git a/drivers/edac/edac_module.h b/drivers/edac/edac_module.h index 96f6de0c8ff6f221bbdbdbc47d8e40b8fb58c9c2..50ed9f2425bb5274a61d33901dac6c5541d7bbf0 100644 --- a/drivers/edac/edac_module.h +++ b/drivers/edac/edac_module.h @@ -28,13 +28,9 @@ void edac_mc_sysfs_exit(void); extern int edac_create_sysfs_mci_device(struct mem_ctl_info *mci, const struct attribute_group **groups); extern void edac_remove_sysfs_mci_device(struct mem_ctl_info *mci); -extern int edac_get_log_ue(void); -extern int edac_get_log_ce(void); -extern int edac_get_panic_on_ue(void); extern int edac_mc_get_log_ue(void); extern int edac_mc_get_log_ce(void); extern int edac_mc_get_panic_on_ue(void); -extern int edac_get_poll_msec(void); extern unsigned int edac_mc_get_poll_msec(void); unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf, diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 6cf50ee0b77c5f7213a5ba5a6aa6c54c8ba51aa7..a22ea053f8e1cdbd92d507128258edc450fcb409 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -74,31 +74,47 @@ static struct list_head *i10nm_edac_list; static struct res_config *res_cfg; static int retry_rd_err_log; +static int decoding_via_mca; +static bool mem_cfg_2lm; static u32 offsets_scrub_icx[] = {0x22c60, 0x22c54, 0x22c5c, 0x22c58, 0x22c28, 0x20ed8}; static u32 offsets_scrub_spr[] = {0x22c60, 0x22c54, 0x22f08, 0x22c58, 0x22c28, 0x20ed8}; +static u32 offsets_scrub_spr_hbm0[] = {0x2860, 0x2854, 0x2b08, 0x2858, 0x2828, 0x0ed8}; +static u32 offsets_scrub_spr_hbm1[] = {0x2c60, 0x2c54, 0x2f08, 0x2c58, 0x2c28, 0x0fa8}; static u32 offsets_demand_icx[] = {0x22e54, 0x22e60, 0x22e64, 0x22e58, 0x22e5c, 0x20ee0}; static u32 offsets_demand_spr[] = {0x22e54, 0x22e60, 0x22f10, 0x22e58, 0x22e5c, 0x20ee0}; +static u32 offsets_demand2_spr[] = {0x22c70, 0x22d80, 0x22f18, 0x22d58, 0x22c64, 0x20f10}; +static u32 offsets_demand_spr_hbm0[] = {0x2a54, 0x2a60, 0x2b10, 0x2a58, 0x2a5c, 0x0ee0}; +static u32 offsets_demand_spr_hbm1[] = {0x2e54, 0x2e60, 0x2f10, 0x2e58, 0x2e5c, 0x0fb0}; -static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable) +static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable, + u32 *offsets_scrub, u32 *offsets_demand, + u32 *offsets_demand2) { - u32 s, d; + u32 s, d, d2; - if (!imc->mbase) - return; - - s = I10NM_GET_REG32(imc, chan, res_cfg->offsets_scrub[0]); - d = I10NM_GET_REG32(imc, chan, res_cfg->offsets_demand[0]); + s = I10NM_GET_REG32(imc, chan, offsets_scrub[0]); + d = I10NM_GET_REG32(imc, chan, offsets_demand[0]); + if (offsets_demand2) + d2 = I10NM_GET_REG32(imc, chan, offsets_demand2[0]); if (enable) { /* Save default configurations */ imc->chan[chan].retry_rd_err_log_s = s; imc->chan[chan].retry_rd_err_log_d = d; + if (offsets_demand2) + imc->chan[chan].retry_rd_err_log_d2 = d2; s &= ~RETRY_RD_ERR_LOG_NOOVER_UC; s |= RETRY_RD_ERR_LOG_EN; d &= ~RETRY_RD_ERR_LOG_NOOVER_UC; d |= RETRY_RD_ERR_LOG_EN; + + if (offsets_demand2) { + d2 &= ~RETRY_RD_ERR_LOG_UC; + d2 |= RETRY_RD_ERR_LOG_NOOVER; + d2 |= RETRY_RD_ERR_LOG_EN; + } } else { /* Restore default configurations */ if (imc->chan[chan].retry_rd_err_log_s & RETRY_RD_ERR_LOG_UC) @@ -113,23 +129,55 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable d |= RETRY_RD_ERR_LOG_NOOVER; if (!(imc->chan[chan].retry_rd_err_log_d & RETRY_RD_ERR_LOG_EN)) d &= ~RETRY_RD_ERR_LOG_EN; + + if (offsets_demand2) { + if (imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_UC) + d2 |= RETRY_RD_ERR_LOG_UC; + if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_NOOVER)) + d2 &= ~RETRY_RD_ERR_LOG_NOOVER; + if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_EN)) + d2 &= ~RETRY_RD_ERR_LOG_EN; + } } - I10NM_SET_REG32(imc, chan, res_cfg->offsets_scrub[0], s); - I10NM_SET_REG32(imc, chan, res_cfg->offsets_demand[0], d); + I10NM_SET_REG32(imc, chan, offsets_scrub[0], s); + I10NM_SET_REG32(imc, chan, offsets_demand[0], d); + if (offsets_demand2) + I10NM_SET_REG32(imc, chan, offsets_demand2[0], d2); } static void enable_retry_rd_err_log(bool enable) { + struct skx_imc *imc; struct skx_dev *d; int i, j; edac_dbg(2, "\n"); list_for_each_entry(d, i10nm_edac_list, list) - for (i = 0; i < I10NM_NUM_IMC; i++) - for (j = 0; j < I10NM_NUM_CHANNELS; j++) - __enable_retry_rd_err_log(&d->imc[i], j, enable); + for (i = 0; i < I10NM_NUM_IMC; i++) { + imc = &d->imc[i]; + if (!imc->mbase) + continue; + + for (j = 0; j < I10NM_NUM_CHANNELS; j++) { + if (imc->hbm_mc) { + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub_hbm0, + res_cfg->offsets_demand_hbm0, + NULL); + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub_hbm1, + res_cfg->offsets_demand_hbm1, + NULL); + } else { + __enable_retry_rd_err_log(imc, j, enable, + res_cfg->offsets_scrub, + res_cfg->offsets_demand, + res_cfg->offsets_demand2); + } + } + } } static void show_retry_rd_err_log(struct decoded_addr *res, char *msg, @@ -138,14 +186,33 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg, struct skx_imc *imc = &res->dev->imc[res->imc]; u32 log0, log1, log2, log3, log4; u32 corr0, corr1, corr2, corr3; + u32 lxg0, lxg1, lxg3, lxg4; + u32 *xffsets = NULL; u64 log2a, log5; + u64 lxg2a, lxg5; u32 *offsets; - int n; + int n, pch; if (!imc->mbase) return; - offsets = scrub_err ? res_cfg->offsets_scrub : res_cfg->offsets_demand; + if (imc->hbm_mc) { + pch = res->cs & 1; + + if (pch) + offsets = scrub_err ? res_cfg->offsets_scrub_hbm1 : + res_cfg->offsets_demand_hbm1; + else + offsets = scrub_err ? res_cfg->offsets_scrub_hbm0 : + res_cfg->offsets_demand_hbm0; + } else { + if (scrub_err) { + offsets = res_cfg->offsets_scrub; + } else { + offsets = res_cfg->offsets_demand; + xffsets = res_cfg->offsets_demand2; + } + } log0 = I10NM_GET_REG32(imc, res->channel, offsets[0]); log1 = I10NM_GET_REG32(imc, res->channel, offsets[1]); @@ -153,20 +220,52 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg, log4 = I10NM_GET_REG32(imc, res->channel, offsets[4]); log5 = I10NM_GET_REG64(imc, res->channel, offsets[5]); + if (xffsets) { + lxg0 = I10NM_GET_REG32(imc, res->channel, xffsets[0]); + lxg1 = I10NM_GET_REG32(imc, res->channel, xffsets[1]); + lxg3 = I10NM_GET_REG32(imc, res->channel, xffsets[3]); + lxg4 = I10NM_GET_REG32(imc, res->channel, xffsets[4]); + lxg5 = I10NM_GET_REG64(imc, res->channel, xffsets[5]); + } + if (res_cfg->type == SPR) { log2a = I10NM_GET_REG64(imc, res->channel, offsets[2]); - n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.16llx %.8x %.8x %.16llx]", + n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.16llx %.8x %.8x %.16llx", log0, log1, log2a, log3, log4, log5); + + if (len - n > 0) { + if (xffsets) { + lxg2a = I10NM_GET_REG64(imc, res->channel, xffsets[2]); + n += snprintf(msg + n, len - n, " %.8x %.8x %.16llx %.8x %.8x %.16llx]", + lxg0, lxg1, lxg2a, lxg3, lxg4, lxg5); + } else { + n += snprintf(msg + n, len - n, "]"); + } + } } else { log2 = I10NM_GET_REG32(imc, res->channel, offsets[2]); n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.8x %.8x %.8x %.16llx]", log0, log1, log2, log3, log4, log5); } - corr0 = I10NM_GET_REG32(imc, res->channel, 0x22c18); - corr1 = I10NM_GET_REG32(imc, res->channel, 0x22c1c); - corr2 = I10NM_GET_REG32(imc, res->channel, 0x22c20); - corr3 = I10NM_GET_REG32(imc, res->channel, 0x22c24); + if (imc->hbm_mc) { + if (pch) { + corr0 = I10NM_GET_REG32(imc, res->channel, 0x2c18); + corr1 = I10NM_GET_REG32(imc, res->channel, 0x2c1c); + corr2 = I10NM_GET_REG32(imc, res->channel, 0x2c20); + corr3 = I10NM_GET_REG32(imc, res->channel, 0x2c24); + } else { + corr0 = I10NM_GET_REG32(imc, res->channel, 0x2818); + corr1 = I10NM_GET_REG32(imc, res->channel, 0x281c); + corr2 = I10NM_GET_REG32(imc, res->channel, 0x2820); + corr3 = I10NM_GET_REG32(imc, res->channel, 0x2824); + } + } else { + corr0 = I10NM_GET_REG32(imc, res->channel, 0x22c18); + corr1 = I10NM_GET_REG32(imc, res->channel, 0x22c1c); + corr2 = I10NM_GET_REG32(imc, res->channel, 0x22c20); + corr3 = I10NM_GET_REG32(imc, res->channel, 0x22c24); + } if (len - n > 0) snprintf(msg + n, len - n, @@ -177,9 +276,16 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg, corr3 & 0xffff, corr3 >> 16); /* Clear status bits */ - if (retry_rd_err_log == 2 && (log0 & RETRY_RD_ERR_LOG_OVER_UC_V)) { - log0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V; - I10NM_SET_REG32(imc, res->channel, offsets[0], log0); + if (retry_rd_err_log == 2) { + if (log0 & RETRY_RD_ERR_LOG_OVER_UC_V) { + log0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V; + I10NM_SET_REG32(imc, res->channel, offsets[0], log0); + } + + if (xffsets && (lxg0 & RETRY_RD_ERR_LOG_OVER_UC_V)) { + lxg0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V; + I10NM_SET_REG32(imc, res->channel, xffsets[0], lxg0); + } } } @@ -231,6 +337,103 @@ static bool i10nm_check_2lm(struct res_config *cfg) return false; } +/* + * Check whether the error comes from DDRT by ICX/Tremont model specific error code. + * Refer to SDM vol3B 16.11.3 Intel IMC MC error codes for IA32_MCi_STATUS. + */ +static bool i10nm_mscod_is_ddrt(u32 mscod) +{ + switch (mscod) { + case 0x0106: case 0x0107: + case 0x0800: case 0x0804: + case 0x0806 ... 0x0808: + case 0x080a ... 0x080e: + case 0x0810: case 0x0811: + case 0x0816: case 0x081e: + case 0x081f: + return true; + } + + return false; +} + +static bool i10nm_mc_decode_available(struct mce *mce) +{ + u8 bank; + + if (!decoding_via_mca || mem_cfg_2lm) + return false; + + if ((mce->status & (MCI_STATUS_MISCV | MCI_STATUS_ADDRV)) + != (MCI_STATUS_MISCV | MCI_STATUS_ADDRV)) + return false; + + bank = mce->bank; + + switch (res_cfg->type) { + case I10NM: + if (bank < 13 || bank > 26) + return false; + + /* DDRT errors can't be decoded from MCA bank registers */ + if (MCI_MISC_ECC_MODE(mce->misc) == MCI_MISC_ECC_DDRT) + return false; + + if (i10nm_mscod_is_ddrt(MCI_STATUS_MSCOD(mce->status))) + return false; + + /* Check whether one of {13,14,17,18,21,22,25,26} */ + return ((bank - 13) & BIT(1)) == 0; + default: + return false; + } +} + +static bool i10nm_mc_decode(struct decoded_addr *res) +{ + struct mce *m = res->mce; + struct skx_dev *d; + u8 bank; + + if (!i10nm_mc_decode_available(m)) + return false; + + list_for_each_entry(d, i10nm_edac_list, list) { + if (d->imc[0].src_id == m->socketid) { + res->socket = m->socketid; + res->dev = d; + break; + } + } + + switch (res_cfg->type) { + case I10NM: + bank = m->bank - 13; + res->imc = bank / 4; + res->channel = bank % 2; + break; + default: + return false; + } + + if (!res->dev) { + skx_printk(KERN_ERR, "No device for src_id %d imc %d\n", + m->socketid, res->imc); + return false; + } + + res->column = GET_BITFIELD(m->misc, 9, 18) << 2; + res->row = GET_BITFIELD(m->misc, 19, 39); + res->bank_group = GET_BITFIELD(m->misc, 40, 41); + res->bank_address = GET_BITFIELD(m->misc, 42, 43); + res->bank_group |= GET_BITFIELD(m->misc, 44, 44) << 2; + res->rank = GET_BITFIELD(m->misc, 56, 58); + res->dimm = res->rank >> 2; + res->rank = res->rank % 4; + + return true; +} + static int i10nm_get_ddr_munits(void) { struct pci_dev *mdev; @@ -420,7 +623,12 @@ static struct res_config spr_cfg = { .sad_all_devfn = PCI_DEVFN(10, 0), .sad_all_offset = 0x300, .offsets_scrub = offsets_scrub_spr, + .offsets_scrub_hbm0 = offsets_scrub_spr_hbm0, + .offsets_scrub_hbm1 = offsets_scrub_spr_hbm1, .offsets_demand = offsets_demand_spr, + .offsets_demand2 = offsets_demand2_spr, + .offsets_demand_hbm0 = offsets_demand_spr_hbm0, + .offsets_demand_hbm1 = offsets_demand_spr_hbm1, }; static const struct x86_cpu_id i10nm_cpuids[] = { @@ -574,7 +782,8 @@ static int __init i10nm_init(void) return -ENODEV; } - skx_set_mem_cfg(i10nm_check_2lm(cfg)); + mem_cfg_2lm = i10nm_check_2lm(cfg); + skx_set_mem_cfg(mem_cfg_2lm); rc = i10nm_get_ddr_munits(); @@ -626,9 +835,11 @@ static int __init i10nm_init(void) setup_i10nm_debug(); if (retry_rd_err_log && res_cfg->offsets_scrub && res_cfg->offsets_demand) { - skx_set_decode(NULL, show_retry_rd_err_log); + skx_set_decode(i10nm_mc_decode, show_retry_rd_err_log); if (retry_rd_err_log == 2) enable_retry_rd_err_log(true); + } else { + skx_set_decode(i10nm_mc_decode, NULL); } i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION); @@ -658,6 +869,34 @@ static void __exit i10nm_exit(void) module_init(i10nm_init); module_exit(i10nm_exit); +static int set_decoding_via_mca(const char *buf, const struct kernel_param *kp) +{ + unsigned long val; + int ret; + + ret = kstrtoul(buf, 0, &val); + + if (ret || val > 1) + return -EINVAL; + + if (val && mem_cfg_2lm) { + i10nm_printk(KERN_NOTICE, "Decoding errors via MCA banks for 2LM isn't supported yet\n"); + return -EIO; + } + + ret = param_set_int(buf, kp); + + return ret; +} + +static const struct kernel_param_ops decoding_via_mca_param_ops = { + .set = set_decoding_via_mca, + .get = param_get_int, +}; + +module_param_cb(decoding_via_mca, &decoding_via_mca_param_ops, &decoding_via_mca, 0644); +MODULE_PARM_DESC(decoding_via_mca, "decoding_via_mca: 0=off(default), 1=enable"); + module_param(retry_rd_err_log, int, 0444); MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)"); diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c index 4f28b8c8d3789fc79091ef843ccbe5d11bb39112..61adaa872ba7bba8c15be82c658455c95bff1c2f 100644 --- a/drivers/edac/i7300_edac.c +++ b/drivers/edac/i7300_edac.c @@ -1193,7 +1193,7 @@ static int __init i7300_init(void) } /** - * i7300_init() - Unregisters the driver + * i7300_exit() - Unregisters the driver */ static void __exit i7300_exit(void) { diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 9a9ff5ad611acf5ed490211f9170285b4935f9f3..9ef13570f2e54062044fd2d69dfb6b8878c207c0 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -20,11 +20,15 @@ * 0c08: Xeon E3-1200 v3 Processor DRAM Controller * 1918: Xeon E3-1200 v5 Skylake Host Bridge/DRAM Registers * 5918: Xeon E3-1200 Xeon E3-1200 v6/7th Gen Core Processor Host Bridge/DRAM Registers + * 190f: 6th Gen Core Dual-Core Processor Host Bridge/DRAM Registers + * 191f: 6th Gen Core Quad-Core Processor Host Bridge/DRAM Registers * 3e..: 8th/9th Gen Core Processor Host Bridge/DRAM Registers * * Based on Intel specification: * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf * http://www.intel.com/content/www/us/en/processors/xeon/xeon-e3-1200-family-vol-2-datasheet.html + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/desktop-6th-gen-core-family-datasheet-vol-2.pdf + * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v6-vol-2-datasheet.pdf * https://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-h-processor-lines-datasheet-vol-2.html * https://www.intel.com/content/www/us/en/products/docs/processors/core/8th-gen-core-family-datasheet-vol-2.html * @@ -53,15 +57,17 @@ #define ie31200_printk(level, fmt, arg...) \ edac_printk(level, "ie31200", fmt, ##arg) -#define PCI_DEVICE_ID_INTEL_IE31200_HB_1 0x0108 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_2 0x010c -#define PCI_DEVICE_ID_INTEL_IE31200_HB_3 0x0150 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_4 0x0158 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_5 0x015c -#define PCI_DEVICE_ID_INTEL_IE31200_HB_6 0x0c04 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_7 0x0c08 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_8 0x1918 -#define PCI_DEVICE_ID_INTEL_IE31200_HB_9 0x5918 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_1 0x0108 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_2 0x010c +#define PCI_DEVICE_ID_INTEL_IE31200_HB_3 0x0150 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_4 0x0158 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_5 0x015c +#define PCI_DEVICE_ID_INTEL_IE31200_HB_6 0x0c04 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_7 0x0c08 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_8 0x190F +#define PCI_DEVICE_ID_INTEL_IE31200_HB_9 0x1918 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_10 0x191F +#define PCI_DEVICE_ID_INTEL_IE31200_HB_11 0x5918 /* Coffee Lake-S */ #define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK 0x3e00 @@ -80,6 +86,8 @@ #define DEVICE_ID_SKYLAKE_OR_LATER(did) \ (((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_8) || \ ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_9) || \ + ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_10) || \ + ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_11) || \ (((did) & PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK) == \ PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK)) @@ -577,6 +585,8 @@ static const struct pci_device_id ie31200_pci_tbl[] = { { PCI_VEND_DEV(INTEL, IE31200_HB_7), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, { PCI_VEND_DEV(INTEL, IE31200_HB_8), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, { PCI_VEND_DEV(INTEL, IE31200_HB_9), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, + { PCI_VEND_DEV(INTEL, IE31200_HB_10), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, + { PCI_VEND_DEV(INTEL, IE31200_HB_11), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_1), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_2), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, { PCI_VEND_DEV(INTEL, IE31200_HB_CFL_3), PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 }, diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c index 0bc670778c9920c72b2316b221be8f4098cf342d..046969b4e82ef4759130e52a039a7547602e2981 100644 --- a/drivers/edac/ppc4xx_edac.c +++ b/drivers/edac/ppc4xx_edac.c @@ -178,11 +178,6 @@ struct ppc4xx_ecc_status { u32 wmirq; }; -/* Function Prototypes */ - -static int ppc4xx_edac_probe(struct platform_device *device); -static int ppc4xx_edac_remove(struct platform_device *device); - /* Global Variables */ /* @@ -197,15 +192,6 @@ static const struct of_device_id ppc4xx_edac_match[] = { }; MODULE_DEVICE_TABLE(of, ppc4xx_edac_match); -static struct platform_driver ppc4xx_edac_driver = { - .probe = ppc4xx_edac_probe, - .remove = ppc4xx_edac_remove, - .driver = { - .name = PPC4XX_EDAC_MODULE_NAME, - .of_match_table = ppc4xx_edac_match, - }, -}; - /* * TODO: The row and channel parameters likely need to be dynamically * set based on the aforementioned variant controller realizations. @@ -1391,6 +1377,15 @@ ppc4xx_edac_opstate_init(void) EDAC_OPSTATE_UNKNOWN_STR))); } +static struct platform_driver ppc4xx_edac_driver = { + .probe = ppc4xx_edac_probe, + .remove = ppc4xx_edac_remove, + .driver = { + .name = PPC4XX_EDAC_MODULE_NAME, + .of_match_table = ppc4xx_edac_match, + }, +}; + /** * ppc4xx_edac_init - driver/module insertion entry point * diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index 9678ab97c7ac3b92898705e175fdcacf43e11691..8e39370fdb5cd6a84469e8032a50442676ef103c 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -335,6 +335,12 @@ struct sbridge_info { struct sbridge_channel { u32 ranks; u32 dimms; + struct dimm { + u32 rowbits; + u32 colbits; + u32 bank_xor_enable; + u32 amap_fine; + } dimm[MAX_DIMMS]; }; struct pci_id_descr { @@ -1603,7 +1609,7 @@ static int __populate_dimms(struct mem_ctl_info *mci, banks = 8; for (i = 0; i < channels; i++) { - u32 mtr; + u32 mtr, amap = 0; int max_dimms_per_channel; @@ -1615,6 +1621,7 @@ static int __populate_dimms(struct mem_ctl_info *mci, max_dimms_per_channel = ARRAY_SIZE(mtr_regs); if (!pvt->pci_tad[i]) continue; + pci_read_config_dword(pvt->pci_tad[i], 0x8c, &amap); } for (j = 0; j < max_dimms_per_channel; j++) { @@ -1627,6 +1634,7 @@ static int __populate_dimms(struct mem_ctl_info *mci, mtr_regs[j], &mtr); } edac_dbg(4, "Channel #%d MTR%d = %x\n", i, j, mtr); + if (IS_DIMM_PRESENT(mtr)) { if (!IS_ECC_ENABLED(pvt->info.mcmtr)) { sbridge_printk(KERN_ERR, "CPU SrcID #%d, Ha #%d, Channel #%d has DIMMs, but ECC is disabled\n", @@ -1661,6 +1669,11 @@ static int __populate_dimms(struct mem_ctl_info *mci, dimm->dtype = pvt->info.get_width(pvt, mtr); dimm->mtype = mtype; dimm->edac_mode = mode; + pvt->channel[i].dimm[j].rowbits = order_base_2(rows); + pvt->channel[i].dimm[j].colbits = order_base_2(cols); + pvt->channel[i].dimm[j].bank_xor_enable = + GET_BITFIELD(pvt->info.mcmtr, 9, 9); + pvt->channel[i].dimm[j].amap_fine = GET_BITFIELD(amap, 0, 0); snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u", pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom, i, j); @@ -1922,6 +1935,99 @@ static struct mem_ctl_info *get_mci_for_node_id(u8 node_id, u8 ha) return NULL; } +static u8 sb_close_row[] = { + 15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33 +}; + +static u8 sb_close_column[] = { + 3, 4, 5, 14, 19, 23, 24, 25, 26, 27 +}; + +static u8 sb_open_row[] = { + 14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33 +}; + +static u8 sb_open_column[] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +}; + +static u8 sb_open_fine_column[] = { + 3, 4, 5, 7, 8, 9, 10, 11, 12, 13 +}; + +static int sb_bits(u64 addr, int nbits, u8 *bits) +{ + int i, res = 0; + + for (i = 0; i < nbits; i++) + res |= ((addr >> bits[i]) & 1) << i; + return res; +} + +static int sb_bank_bits(u64 addr, int b0, int b1, int do_xor, int x0, int x1) +{ + int ret = GET_BITFIELD(addr, b0, b0) | (GET_BITFIELD(addr, b1, b1) << 1); + + if (do_xor) + ret ^= GET_BITFIELD(addr, x0, x0) | (GET_BITFIELD(addr, x1, x1) << 1); + + return ret; +} + +static bool sb_decode_ddr4(struct mem_ctl_info *mci, int ch, u8 rank, + u64 rank_addr, char *msg) +{ + int dimmno = 0; + int row, col, bank_address, bank_group; + struct sbridge_pvt *pvt; + u32 bg0 = 0, rowbits = 0, colbits = 0; + u32 amap_fine = 0, bank_xor_enable = 0; + + dimmno = (rank < 12) ? rank / 4 : 2; + pvt = mci->pvt_info; + amap_fine = pvt->channel[ch].dimm[dimmno].amap_fine; + bg0 = amap_fine ? 6 : 13; + rowbits = pvt->channel[ch].dimm[dimmno].rowbits; + colbits = pvt->channel[ch].dimm[dimmno].colbits; + bank_xor_enable = pvt->channel[ch].dimm[dimmno].bank_xor_enable; + + if (pvt->is_lockstep) { + pr_warn_once("LockStep row/column decode is not supported yet!\n"); + msg[0] = '\0'; + return false; + } + + if (pvt->is_close_pg) { + row = sb_bits(rank_addr, rowbits, sb_close_row); + col = sb_bits(rank_addr, colbits, sb_close_column); + col |= 0x400; /* C10 is autoprecharge, always set */ + bank_address = sb_bank_bits(rank_addr, 8, 9, bank_xor_enable, 22, 28); + bank_group = sb_bank_bits(rank_addr, 6, 7, bank_xor_enable, 20, 21); + } else { + row = sb_bits(rank_addr, rowbits, sb_open_row); + if (amap_fine) + col = sb_bits(rank_addr, colbits, sb_open_fine_column); + else + col = sb_bits(rank_addr, colbits, sb_open_column); + bank_address = sb_bank_bits(rank_addr, 18, 19, bank_xor_enable, 22, 23); + bank_group = sb_bank_bits(rank_addr, bg0, 17, bank_xor_enable, 20, 21); + } + + row &= (1u << rowbits) - 1; + + sprintf(msg, "row:0x%x col:0x%x bank_addr:%d bank_group:%d", + row, col, bank_address, bank_group); + return true; +} + +static bool sb_decode_ddr3(struct mem_ctl_info *mci, int ch, u8 rank, + u64 rank_addr, char *msg) +{ + pr_warn_once("DDR3 row/column decode not support yet!\n"); + msg[0] = '\0'; + return false; +} + static int get_memory_error_data(struct mem_ctl_info *mci, u64 addr, u8 *socket, u8 *ha, @@ -1937,12 +2043,13 @@ static int get_memory_error_data(struct mem_ctl_info *mci, int interleave_mode, shiftup = 0; unsigned int sad_interleave[MAX_INTERLEAVE]; u32 reg, dram_rule; - u8 ch_way, sck_way, pkg, sad_ha = 0; + u8 ch_way, sck_way, pkg, sad_ha = 0, rankid = 0; u32 tad_offset; u32 rir_way; u32 mb, gb; u64 ch_addr, offset, limit = 0, prv = 0; - + u64 rank_addr; + enum mem_type mtype; /* * Step 0) Check if the address is at special memory ranges @@ -2226,6 +2333,28 @@ static int get_memory_error_data(struct mem_ctl_info *mci, pci_read_config_dword(pvt->pci_tad[base_ch], rir_offset[n_rir][idx], ®); *rank = RIR_RNK_TGT(pvt->info.type, reg); + if (pvt->info.type == BROADWELL) { + if (pvt->is_close_pg) + shiftup = 6; + else + shiftup = 13; + + rank_addr = ch_addr >> shiftup; + rank_addr /= (1 << rir_way); + rank_addr <<= shiftup; + rank_addr |= ch_addr & GENMASK_ULL(shiftup - 1, 0); + rank_addr -= RIR_OFFSET(pvt->info.type, reg); + + mtype = pvt->info.get_memory_type(pvt); + rankid = *rank; + if (mtype == MEM_DDR4 || mtype == MEM_RDDR4) + sb_decode_ddr4(mci, base_ch, rankid, rank_addr, msg); + else + sb_decode_ddr3(mci, base_ch, rankid, rank_addr, msg); + } else { + msg[0] = '\0'; + } + edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n", n_rir, ch_addr, @@ -2950,7 +3079,7 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci, struct mem_ctl_info *new_mci; struct sbridge_pvt *pvt = mci->pvt_info; enum hw_event_mc_err_type tp_event; - char *optype, msg[256]; + char *optype, msg[256], msg_full[512]; bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0); bool overflow = GET_BITFIELD(m->status, 62, 62); bool uncorrected_error = GET_BITFIELD(m->status, 61, 61); @@ -3089,18 +3218,17 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci, */ if (!pvt->is_lockstep && !pvt->is_cur_addr_mirrored && !pvt->is_close_pg) channel = first_channel; - - snprintf(msg, sizeof(msg), - "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d", + snprintf(msg_full, sizeof(msg_full), + "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d %s", overflow ? " OVERFLOW" : "", (uncorrected_error && recoverable) ? " recoverable" : "", area_type, mscod, errcode, socket, ha, channel_mask, - rank); + rank, msg); - edac_dbg(0, "%s\n", msg); + edac_dbg(0, "%s\n", msg_full); /* FIXME: need support for channel mask */ @@ -3111,7 +3239,7 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci, edac_mc_handle_error(tp_event, mci, core_err_cnt, m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0, channel, dimm, -1, - optype, msg); + optype, msg_full); return; err_parsing: edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, diff --git a/drivers/edac/sifive_edac.c b/drivers/edac/sifive_edac.c index ee800aec7d47926f38a5caec2b156647f098fe93..b844e2626fd50686687af90c3af194ecde5a71f5 100644 --- a/drivers/edac/sifive_edac.c +++ b/drivers/edac/sifive_edac.c @@ -2,7 +2,7 @@ /* * SiFive Platform EDAC Driver * - * Copyright (C) 2018-2019 SiFive, Inc. + * Copyright (C) 2018-2022 SiFive, Inc. * * This driver is partially based on octeon_edac-pc.c * @@ -10,7 +10,7 @@ #include #include #include "edac_module.h" -#include +#include #define DRVNAME "sifive_edac" @@ -32,9 +32,9 @@ int ecc_err_event(struct notifier_block *this, unsigned long event, void *ptr) p = container_of(this, struct sifive_edac_priv, notifier); - if (event == SIFIVE_L2_ERR_TYPE_UE) + if (event == SIFIVE_CCACHE_ERR_TYPE_UE) edac_device_handle_ue(p->dci, 0, 0, msg); - else if (event == SIFIVE_L2_ERR_TYPE_CE) + else if (event == SIFIVE_CCACHE_ERR_TYPE_CE) edac_device_handle_ce(p->dci, 0, 0, msg); return NOTIFY_OK; @@ -67,7 +67,7 @@ static int ecc_register(struct platform_device *pdev) goto err; } - register_sifive_l2_error_notifier(&p->notifier); + register_sifive_ccache_error_notifier(&p->notifier); return 0; @@ -81,7 +81,7 @@ static int ecc_unregister(struct platform_device *pdev) { struct sifive_edac_priv *p = platform_get_drvdata(pdev); - unregister_sifive_l2_error_notifier(&p->notifier); + unregister_sifive_ccache_error_notifier(&p->notifier); edac_device_del_device(&pdev->dev); edac_device_free_ctl_info(p->dci); diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c index 1abc020d49ab64a6654ab7b9b80f5140e17532d1..7e2762f62eec1e49ce7310bd2070e3420503ff4b 100644 --- a/drivers/edac/skx_base.c +++ b/drivers/edac/skx_base.c @@ -714,8 +714,13 @@ static int __init skx_init(void) skx_set_decode(skx_decode, skx_show_retry_rd_err_log); - if (nvdimm_count && skx_adxl_get() == -ENODEV) - skx_printk(KERN_NOTICE, "Only decoding DDR4 address!\n"); + if (nvdimm_count && skx_adxl_get() != -ENODEV) { + skx_set_decode(NULL, skx_show_retry_rd_err_log); + } else { + if (nvdimm_count) + skx_printk(KERN_NOTICE, "Only decoding DDR4 address!\n"); + skx_set_decode(skx_decode, skx_show_retry_rd_err_log); + } /* Ensure that the OPSTATE is set correctly for POLL or NMI */ opstate_init(); diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 19c17c5198c5f9d8bf9f97feae2d4c3b5669115a..f0f8e98f6efb289f0c415be7885f7473e65b8ca3 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -27,9 +27,11 @@ static const char * const component_names[] = { [INDEX_MEMCTRL] = "MemoryControllerId", [INDEX_CHANNEL] = "ChannelId", [INDEX_DIMM] = "DimmSlotId", + [INDEX_CS] = "ChipSelect", [INDEX_NM_MEMCTRL] = "NmMemoryControllerId", [INDEX_NM_CHANNEL] = "NmChannelId", [INDEX_NM_DIMM] = "NmDimmSlotId", + [INDEX_NM_CS] = "NmChipSelect", }; static int component_indices[ARRAY_SIZE(component_names)]; @@ -40,7 +42,7 @@ static char *adxl_msg; static unsigned long adxl_nm_bitmap; static char skx_msg[MSG_SIZE]; -static skx_decode_f skx_decode; +static skx_decode_f driver_decode; static skx_show_retry_log_f skx_show_retry_rd_err_log; static u64 skx_tolm, skx_tohm; static LIST_HEAD(dev_edac_list); @@ -139,10 +141,13 @@ static bool skx_adxl_decode(struct decoded_addr *res, bool error_in_1st_level_me (int)adxl_values[component_indices[INDEX_NM_CHANNEL]] : -1; res->dimm = (adxl_nm_bitmap & BIT_NM_DIMM) ? (int)adxl_values[component_indices[INDEX_NM_DIMM]] : -1; + res->cs = (adxl_nm_bitmap & BIT_NM_CS) ? + (int)adxl_values[component_indices[INDEX_NM_CS]] : -1; } else { res->imc = (int)adxl_values[component_indices[INDEX_MEMCTRL]]; res->channel = (int)adxl_values[component_indices[INDEX_CHANNEL]]; res->dimm = (int)adxl_values[component_indices[INDEX_DIMM]]; + res->cs = (int)adxl_values[component_indices[INDEX_CS]]; } if (res->imc > NUM_IMC - 1 || res->imc < 0) { @@ -173,6 +178,8 @@ static bool skx_adxl_decode(struct decoded_addr *res, bool error_in_1st_level_me break; } + res->decoded_by_adxl = true; + return true; } @@ -183,7 +190,7 @@ void skx_set_mem_cfg(bool mem_cfg_2lm) void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log) { - skx_decode = decode; + driver_decode = decode; skx_show_retry_rd_err_log = show_retry_log; } @@ -591,19 +598,19 @@ static void skx_mce_output_error(struct mem_ctl_info *mci, break; } } - if (adxl_component_count) { + if (res->decoded_by_adxl) { len = snprintf(skx_msg, MSG_SIZE, "%s%s err_code:0x%04x:0x%04x %s", overflow ? " OVERFLOW" : "", (uncorrected_error && recoverable) ? " recoverable" : "", mscod, errcode, adxl_msg); } else { len = snprintf(skx_msg, MSG_SIZE, - "%s%s err_code:0x%04x:0x%04x socket:%d imc:%d rank:%d bg:%d ba:%d row:0x%x col:0x%x", + "%s%s err_code:0x%04x:0x%04x ProcessorSocketId:0x%x MemoryControllerId:0x%x PhysicalRankId:0x%x Row:0x%x Column:0x%x Bank:0x%x BankGroup:0x%x", overflow ? " OVERFLOW" : "", (uncorrected_error && recoverable) ? " recoverable" : "", mscod, errcode, res->socket, res->imc, res->rank, - res->bank_group, res->bank_address, res->row, res->column); + res->row, res->column, res->bank_address, res->bank_group); } if (skx_show_retry_rd_err_log) @@ -649,13 +656,14 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, return NOTIFY_DONE; memset(&res, 0, sizeof(res)); + res.mce = mce; res.addr = mce->addr; - if (adxl_component_count) { - if (!skx_adxl_decode(&res, skx_error_in_1st_level_mem(mce))) + /* Try driver decoder first */ + if (!(driver_decode && driver_decode(&res))) { + /* Then try firmware decoder (ACPI DSM methods) */ + if (!(adxl_component_count && skx_adxl_decode(&res, skx_error_in_1st_level_mem(mce)))) return NOTIFY_DONE; - } else if (!skx_decode || !skx_decode(&res)) { - return NOTIFY_DONE; } mci = res.dev->imc[res.imc].mci; diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 03ac067a80b9f779fe2ef726672f222ba4403be7..0cbadd3d2cd39044f438dfa6bc6431ac72826229 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -10,6 +10,7 @@ #define _SKX_COMM_EDAC_H #include +#include #define MSG_SIZE 1024 @@ -52,6 +53,9 @@ #define IS_DIMM_PRESENT(r) GET_BITFIELD(r, 15, 15) #define IS_NVDIMM_PRESENT(r, i) GET_BITFIELD(r, i, i) +#define MCI_MISC_ECC_MODE(m) (((m) >> 59) & 15) +#define MCI_MISC_ECC_DDRT 8 /* read from DDRT */ + /* * Each cpu socket contains some pci devices that provide global * information, and also some that are local to each of the two @@ -82,6 +86,7 @@ struct skx_dev { struct pci_dev *edev; u32 retry_rd_err_log_s; u32 retry_rd_err_log_d; + u32 retry_rd_err_log_d2; struct skx_dimm { u8 close_pg; u8 bank_xor_enable; @@ -108,18 +113,22 @@ enum { INDEX_MEMCTRL, INDEX_CHANNEL, INDEX_DIMM, + INDEX_CS, INDEX_NM_FIRST, INDEX_NM_MEMCTRL = INDEX_NM_FIRST, INDEX_NM_CHANNEL, INDEX_NM_DIMM, + INDEX_NM_CS, INDEX_MAX }; #define BIT_NM_MEMCTRL BIT_ULL(INDEX_NM_MEMCTRL) #define BIT_NM_CHANNEL BIT_ULL(INDEX_NM_CHANNEL) #define BIT_NM_DIMM BIT_ULL(INDEX_NM_DIMM) +#define BIT_NM_CS BIT_ULL(INDEX_NM_CS) struct decoded_addr { + struct mce *mce; struct skx_dev *dev; u64 addr; int socket; @@ -129,6 +138,7 @@ struct decoded_addr { int sktways; int chanways; int dimm; + int cs; int rank; int channel_rank; u64 rank_address; @@ -136,6 +146,7 @@ struct decoded_addr { int column; int bank_address; int bank_group; + bool decoded_by_adxl; }; struct res_config { @@ -154,7 +165,12 @@ struct res_config { int sad_all_offset; /* Offsets of retry_rd_err_log registers */ u32 *offsets_scrub; + u32 *offsets_scrub_hbm0; + u32 *offsets_scrub_hbm1; u32 *offsets_demand; + u32 *offsets_demand2; + u32 *offsets_demand_hbm0; + u32 *offsets_demand_hbm1; }; typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci, diff --git a/drivers/edac/wq.c b/drivers/edac/wq.c index d021d287eaec60cc935370ab07ba5ab02c577e94..ad3f516627c5e0932dcc99508d3a2b3824609d56 100644 --- a/drivers/edac/wq.c +++ b/drivers/edac/wq.c @@ -37,7 +37,6 @@ int edac_workqueue_setup(void) void edac_workqueue_teardown(void) { - flush_workqueue(wq); destroy_workqueue(wq); wq = NULL; } diff --git a/drivers/extcon/Kconfig b/drivers/extcon/Kconfig index dca7cecb37e3435c3ac0951324e30c2305568234..290186e44e6bdbe34d1f41805bca894e38c7e741 100644 --- a/drivers/extcon/Kconfig +++ b/drivers/extcon/Kconfig @@ -183,7 +183,7 @@ config EXTCON_USBC_CROS_EC config EXTCON_USBC_TUSB320 tristate "TI TUSB320 USB-C extcon support" - depends on I2C + depends on I2C && TYPEC select REGMAP_I2C help Say Y here to enable support for USB Type C cable detection extcon diff --git a/drivers/extcon/extcon-rt8973a.c b/drivers/extcon/extcon-rt8973a.c index 02ba770acb2791ac76acb72a0a0e44e16401db5b..e6e448f6ea2f66c225fc659b6ae253b0dbbfd7f2 100644 --- a/drivers/extcon/extcon-rt8973a.c +++ b/drivers/extcon/extcon-rt8973a.c @@ -646,13 +646,11 @@ static int rt8973a_muic_i2c_probe(struct i2c_client *i2c, return 0; } -static int rt8973a_muic_i2c_remove(struct i2c_client *i2c) +static void rt8973a_muic_i2c_remove(struct i2c_client *i2c) { struct rt8973a_muic_info *info = i2c_get_clientdata(i2c); regmap_del_irq_chip(info->irq, info->irq_data); - - return 0; } static const struct of_device_id rt8973a_dt_match[] = { diff --git a/drivers/extcon/extcon-usbc-tusb320.c b/drivers/extcon/extcon-usbc-tusb320.c index 6ba3d89b106d059ddd37c8b3a48b6314847e4f5e..41041ff0fadbbbf2b9730c3e73a3701670a725cb 100644 --- a/drivers/extcon/extcon-usbc-tusb320.c +++ b/drivers/extcon/extcon-usbc-tusb320.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 -/** +/* * drivers/extcon/extcon-tusb320.c - TUSB320 extcon driver * * Copyright (C) 2020 National Instruments Corporation * Author: Michael Auchter */ +#include #include #include #include @@ -13,6 +14,24 @@ #include #include #include +#include + +#define TUSB320_REG8 0x8 +#define TUSB320_REG8_CURRENT_MODE_ADVERTISE GENMASK(7, 6) +#define TUSB320_REG8_CURRENT_MODE_ADVERTISE_USB 0x0 +#define TUSB320_REG8_CURRENT_MODE_ADVERTISE_15A 0x1 +#define TUSB320_REG8_CURRENT_MODE_ADVERTISE_30A 0x2 +#define TUSB320_REG8_CURRENT_MODE_DETECT GENMASK(5, 4) +#define TUSB320_REG8_CURRENT_MODE_DETECT_DEF 0x0 +#define TUSB320_REG8_CURRENT_MODE_DETECT_MED 0x1 +#define TUSB320_REG8_CURRENT_MODE_DETECT_ACC 0x2 +#define TUSB320_REG8_CURRENT_MODE_DETECT_HI 0x3 +#define TUSB320_REG8_ACCESSORY_CONNECTED GENMASK(3, 2) +#define TUSB320_REG8_ACCESSORY_CONNECTED_NONE 0x0 +#define TUSB320_REG8_ACCESSORY_CONNECTED_AUDIO 0x4 +#define TUSB320_REG8_ACCESSORY_CONNECTED_ACC 0x5 +#define TUSB320_REG8_ACCESSORY_CONNECTED_DEBUG 0x6 +#define TUSB320_REG8_ACTIVE_CABLE_DETECTION BIT(0) #define TUSB320_REG9 0x9 #define TUSB320_REG9_ATTACHED_STATE_SHIFT 6 @@ -55,6 +74,10 @@ struct tusb320_priv { struct extcon_dev *edev; struct tusb320_ops *ops; enum tusb320_attached_state state; + struct typec_port *port; + struct typec_capability cap; + enum typec_port_type port_type; + enum typec_pwr_opmode pwr_opmode; }; static const char * const tusb_attached_states[] = { @@ -184,19 +207,47 @@ static struct tusb320_ops tusb320l_ops = { .get_revision = tusb320l_get_revision, }; -static irqreturn_t tusb320_irq_handler(int irq, void *dev_id) +static int tusb320_set_adv_pwr_mode(struct tusb320_priv *priv) { - struct tusb320_priv *priv = dev_id; - int state, polarity; - unsigned reg; + u8 mode; + + if (priv->pwr_opmode == TYPEC_PWR_MODE_USB) + mode = TUSB320_REG8_CURRENT_MODE_ADVERTISE_USB; + else if (priv->pwr_opmode == TYPEC_PWR_MODE_1_5A) + mode = TUSB320_REG8_CURRENT_MODE_ADVERTISE_15A; + else if (priv->pwr_opmode == TYPEC_PWR_MODE_3_0A) + mode = TUSB320_REG8_CURRENT_MODE_ADVERTISE_30A; + else /* No other mode is supported. */ + return -EINVAL; - if (regmap_read(priv->regmap, TUSB320_REG9, ®)) { - dev_err(priv->dev, "error during i2c read!\n"); - return IRQ_NONE; - } + return regmap_write_bits(priv->regmap, TUSB320_REG8, + TUSB320_REG8_CURRENT_MODE_ADVERTISE, + FIELD_PREP(TUSB320_REG8_CURRENT_MODE_ADVERTISE, + mode)); +} - if (!(reg & TUSB320_REG9_INTERRUPT_STATUS)) - return IRQ_NONE; +static int tusb320_port_type_set(struct typec_port *port, + enum typec_port_type type) +{ + struct tusb320_priv *priv = typec_get_drvdata(port); + + if (type == TYPEC_PORT_SRC) + return priv->ops->set_mode(priv, TUSB320_MODE_DFP); + else if (type == TYPEC_PORT_SNK) + return priv->ops->set_mode(priv, TUSB320_MODE_UFP); + else if (type == TYPEC_PORT_DRP) + return priv->ops->set_mode(priv, TUSB320_MODE_DRP); + else + return priv->ops->set_mode(priv, TUSB320_MODE_PORT); +} + +static const struct typec_operations tusb320_typec_ops = { + .port_type_set = tusb320_port_type_set, +}; + +static void tusb320_extcon_irq_handler(struct tusb320_priv *priv, u8 reg) +{ + int state, polarity; state = (reg >> TUSB320_REG9_ATTACHED_STATE_SHIFT) & TUSB320_REG9_ATTACHED_STATE_MASK; @@ -219,6 +270,64 @@ static irqreturn_t tusb320_irq_handler(int irq, void *dev_id) extcon_sync(priv->edev, EXTCON_USB_HOST); priv->state = state; +} + +static void tusb320_typec_irq_handler(struct tusb320_priv *priv, u8 reg9) +{ + struct typec_port *port = priv->port; + struct device *dev = priv->dev; + u8 mode, role, state; + int ret, reg8; + bool ori; + + ori = reg9 & TUSB320_REG9_CABLE_DIRECTION; + typec_set_orientation(port, ori ? TYPEC_ORIENTATION_REVERSE : + TYPEC_ORIENTATION_NORMAL); + + state = (reg9 >> TUSB320_REG9_ATTACHED_STATE_SHIFT) & + TUSB320_REG9_ATTACHED_STATE_MASK; + if (state == TUSB320_ATTACHED_STATE_DFP) + role = TYPEC_SOURCE; + else + role = TYPEC_SINK; + + typec_set_vconn_role(port, role); + typec_set_pwr_role(port, role); + typec_set_data_role(port, role == TYPEC_SOURCE ? + TYPEC_HOST : TYPEC_DEVICE); + + ret = regmap_read(priv->regmap, TUSB320_REG8, ®8); + if (ret) { + dev_err(dev, "error during reg8 i2c read, ret=%d!\n", ret); + return; + } + + mode = FIELD_GET(TUSB320_REG8_CURRENT_MODE_DETECT, reg8); + if (mode == TUSB320_REG8_CURRENT_MODE_DETECT_DEF) + typec_set_pwr_opmode(port, TYPEC_PWR_MODE_USB); + else if (mode == TUSB320_REG8_CURRENT_MODE_DETECT_MED) + typec_set_pwr_opmode(port, TYPEC_PWR_MODE_1_5A); + else if (mode == TUSB320_REG8_CURRENT_MODE_DETECT_HI) + typec_set_pwr_opmode(port, TYPEC_PWR_MODE_3_0A); + else /* Charge through accessory */ + typec_set_pwr_opmode(port, TYPEC_PWR_MODE_USB); +} + +static irqreturn_t tusb320_irq_handler(int irq, void *dev_id) +{ + struct tusb320_priv *priv = dev_id; + unsigned int reg; + + if (regmap_read(priv->regmap, TUSB320_REG9, ®)) { + dev_err(priv->dev, "error during i2c read!\n"); + return IRQ_NONE; + } + + if (!(reg & TUSB320_REG9_INTERRUPT_STATUS)) + return IRQ_NONE; + + tusb320_extcon_irq_handler(priv, reg); + tusb320_typec_irq_handler(priv, reg); regmap_write(priv->regmap, TUSB320_REG9, reg); @@ -230,8 +339,84 @@ static const struct regmap_config tusb320_regmap_config = { .val_bits = 8, }; -static int tusb320_extcon_probe(struct i2c_client *client, - const struct i2c_device_id *id) +static int tusb320_extcon_probe(struct tusb320_priv *priv) +{ + int ret; + + priv->edev = devm_extcon_dev_allocate(priv->dev, tusb320_extcon_cable); + if (IS_ERR(priv->edev)) { + dev_err(priv->dev, "failed to allocate extcon device\n"); + return PTR_ERR(priv->edev); + } + + ret = devm_extcon_dev_register(priv->dev, priv->edev); + if (ret < 0) { + dev_err(priv->dev, "failed to register extcon device\n"); + return ret; + } + + extcon_set_property_capability(priv->edev, EXTCON_USB, + EXTCON_PROP_USB_TYPEC_POLARITY); + extcon_set_property_capability(priv->edev, EXTCON_USB_HOST, + EXTCON_PROP_USB_TYPEC_POLARITY); + + return 0; +} + +static int tusb320_typec_probe(struct i2c_client *client, + struct tusb320_priv *priv) +{ + struct fwnode_handle *connector; + const char *cap_str; + int ret; + + /* The Type-C connector is optional, for backward compatibility. */ + connector = device_get_named_child_node(&client->dev, "connector"); + if (!connector) + return 0; + + /* Type-C connector found. */ + ret = typec_get_fw_cap(&priv->cap, connector); + if (ret) + return ret; + + priv->port_type = priv->cap.type; + + /* This goes into register 0x8 field CURRENT_MODE_ADVERTISE */ + ret = fwnode_property_read_string(connector, "typec-power-opmode", &cap_str); + if (ret) + return ret; + + ret = typec_find_pwr_opmode(cap_str); + if (ret < 0) + return ret; + if (ret == TYPEC_PWR_MODE_PD) + return -EINVAL; + + priv->pwr_opmode = ret; + + /* Initialize the hardware with the devicetree settings. */ + ret = tusb320_set_adv_pwr_mode(priv); + if (ret) + return ret; + + priv->cap.revision = USB_TYPEC_REV_1_1; + priv->cap.accessory[0] = TYPEC_ACCESSORY_AUDIO; + priv->cap.accessory[1] = TYPEC_ACCESSORY_DEBUG; + priv->cap.orientation_aware = true; + priv->cap.driver_data = priv; + priv->cap.ops = &tusb320_typec_ops; + priv->cap.fwnode = connector; + + priv->port = typec_register_port(&client->dev, &priv->cap); + if (IS_ERR(priv->port)) + return PTR_ERR(priv->port); + + return 0; +} + +static int tusb320_probe(struct i2c_client *client, + const struct i2c_device_id *id) { struct tusb320_priv *priv; const void *match_data; @@ -257,12 +442,6 @@ static int tusb320_extcon_probe(struct i2c_client *client, priv->ops = (struct tusb320_ops*)match_data; - priv->edev = devm_extcon_dev_allocate(priv->dev, tusb320_extcon_cable); - if (IS_ERR(priv->edev)) { - dev_err(priv->dev, "failed to allocate extcon device\n"); - return PTR_ERR(priv->edev); - } - if (priv->ops->get_revision) { ret = priv->ops->get_revision(priv, &revision); if (ret) @@ -272,16 +451,13 @@ static int tusb320_extcon_probe(struct i2c_client *client, dev_info(priv->dev, "chip revision %d\n", revision); } - ret = devm_extcon_dev_register(priv->dev, priv->edev); - if (ret < 0) { - dev_err(priv->dev, "failed to register extcon device\n"); + ret = tusb320_extcon_probe(priv); + if (ret) return ret; - } - extcon_set_property_capability(priv->edev, EXTCON_USB, - EXTCON_PROP_USB_TYPEC_POLARITY); - extcon_set_property_capability(priv->edev, EXTCON_USB_HOST, - EXTCON_PROP_USB_TYPEC_POLARITY); + ret = tusb320_typec_probe(client, priv); + if (ret) + return ret; /* update initial state */ tusb320_irq_handler(client->irq, priv); @@ -313,7 +489,7 @@ static const struct of_device_id tusb320_extcon_dt_match[] = { MODULE_DEVICE_TABLE(of, tusb320_extcon_dt_match); static struct i2c_driver tusb320_extcon_driver = { - .probe = tusb320_extcon_probe, + .probe = tusb320_probe, .driver = { .name = "extcon-tusb320", .of_match_table = tusb320_extcon_dt_match, diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c index 641a91819088050759f995bbf346449a28f6a53b..99d43948061260377130e43d0918ca19c5301557 100644 --- a/drivers/firmware/arm_ffa/bus.c +++ b/drivers/firmware/arm_ffa/bus.c @@ -167,7 +167,8 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev) return valid; } -struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id) +struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id, + const struct ffa_ops *ops) { int ret; struct device *dev; @@ -183,6 +184,7 @@ struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id) dev_set_name(&ffa_dev->dev, "arm-ffa-%04x", vm_id); ffa_dev->vm_id = vm_id; + ffa_dev->ops = ops; uuid_copy(&ffa_dev->uuid, uuid); ret = device_register(&ffa_dev->dev); diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index ec731e9e942b8825efdcd6038b513ac8154c09f3..d5e86ef40b8960b2ab9dde32ffb2d36dbba4ec65 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -163,6 +163,7 @@ struct ffa_drv_info { struct mutex tx_lock; /* lock to protect Tx buffer */ void *rx_buffer; void *tx_buffer; + bool mem_ops_native; }; static struct ffa_drv_info *drv_info; @@ -263,18 +264,24 @@ static int ffa_rxtx_unmap(u16 vm_id) return 0; } +#define PARTITION_INFO_GET_RETURN_COUNT_ONLY BIT(0) + /* buffer must be sizeof(struct ffa_partition_info) * num_partitions */ static int __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, struct ffa_partition_info *buffer, int num_partitions) { - int count; + int idx, count, flags = 0, sz, buf_sz; ffa_value_t partition_info; + if (!buffer || !num_partitions) /* Just get the count for now */ + flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY; + mutex_lock(&drv_info->rx_lock); invoke_ffa_fn((ffa_value_t){ .a0 = FFA_PARTITION_INFO_GET, .a1 = uuid0, .a2 = uuid1, .a3 = uuid2, .a4 = uuid3, + .a5 = flags, }, &partition_info); if (partition_info.a0 == FFA_ERROR) { @@ -284,8 +291,19 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, count = partition_info.a2; + if (drv_info->version > FFA_VERSION_1_0) { + buf_sz = sz = partition_info.a3; + if (sz > sizeof(*buffer)) + buf_sz = sizeof(*buffer); + } else { + /* FFA_VERSION_1_0 lacks size in the response */ + buf_sz = sz = 8; + } + if (buffer && count <= num_partitions) - memcpy(buffer, drv_info->rx_buffer, sizeof(*buffer) * count); + for (idx = 0; idx < count; idx++) + memcpy(buffer + idx, drv_info->rx_buffer + idx * sz, + buf_sz); ffa_rx_release(); @@ -571,6 +589,39 @@ static int ffa_memory_reclaim(u64 g_handle, u32 flags) return 0; } +static int ffa_features(u32 func_feat_id, u32 input_props, + u32 *if_props_1, u32 *if_props_2) +{ + ffa_value_t id; + + if (!ARM_SMCCC_IS_FAST_CALL(func_feat_id) && input_props) { + pr_err("%s: Invalid Parameters: %x, %x", __func__, + func_feat_id, input_props); + return ffa_to_linux_errno(FFA_RET_INVALID_PARAMETERS); + } + + invoke_ffa_fn((ffa_value_t){ + .a0 = FFA_FEATURES, .a1 = func_feat_id, .a2 = input_props, + }, &id); + + if (id.a0 == FFA_ERROR) + return ffa_to_linux_errno((int)id.a2); + + if (if_props_1) + *if_props_1 = id.a2; + if (if_props_2) + *if_props_2 = id.a3; + + return 0; +} + +static void ffa_set_up_mem_ops_native_flag(void) +{ + if (!ffa_features(FFA_FN_NATIVE(MEM_LEND), 0, NULL, NULL) || + !ffa_features(FFA_FN_NATIVE(MEM_SHARE), 0, NULL, NULL)) + drv_info->mem_ops_native = true; +} + static u32 ffa_api_version_get(void) { return drv_info->version; @@ -597,11 +648,19 @@ static int ffa_partition_info_get(const char *uuid_str, return 0; } -static void ffa_mode_32bit_set(struct ffa_device *dev) +static void _ffa_mode_32bit_set(struct ffa_device *dev) { dev->mode_32bit = true; } +static void ffa_mode_32bit_set(struct ffa_device *dev) +{ + if (drv_info->version > FFA_VERSION_1_0) + return; + + _ffa_mode_32bit_set(dev); +} + static int ffa_sync_send_receive(struct ffa_device *dev, struct ffa_send_direct_data *data) { @@ -609,17 +668,15 @@ static int ffa_sync_send_receive(struct ffa_device *dev, dev->mode_32bit, data); } -static int -ffa_memory_share(struct ffa_device *dev, struct ffa_mem_ops_args *args) +static int ffa_memory_share(struct ffa_mem_ops_args *args) { - if (dev->mode_32bit) - return ffa_memory_ops(FFA_MEM_SHARE, args); + if (drv_info->mem_ops_native) + return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args); - return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args); + return ffa_memory_ops(FFA_MEM_SHARE, args); } -static int -ffa_memory_lend(struct ffa_device *dev, struct ffa_mem_ops_args *args) +static int ffa_memory_lend(struct ffa_mem_ops_args *args) { /* Note that upon a successful MEM_LEND request the caller * must ensure that the memory region specified is not accessed @@ -628,36 +685,47 @@ ffa_memory_lend(struct ffa_device *dev, struct ffa_mem_ops_args *args) * however on systems without a hypervisor the responsibility * falls to the calling kernel driver to prevent access. */ - if (dev->mode_32bit) - return ffa_memory_ops(FFA_MEM_LEND, args); + if (drv_info->mem_ops_native) + return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args); - return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args); + return ffa_memory_ops(FFA_MEM_LEND, args); } -static const struct ffa_dev_ops ffa_ops = { +static const struct ffa_info_ops ffa_drv_info_ops = { .api_version_get = ffa_api_version_get, .partition_info_get = ffa_partition_info_get, +}; + +static const struct ffa_msg_ops ffa_drv_msg_ops = { .mode_32bit_set = ffa_mode_32bit_set, .sync_send_receive = ffa_sync_send_receive, +}; + +static const struct ffa_mem_ops ffa_drv_mem_ops = { .memory_reclaim = ffa_memory_reclaim, .memory_share = ffa_memory_share, .memory_lend = ffa_memory_lend, }; -const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev) -{ - if (ffa_device_is_valid(dev)) - return &ffa_ops; - - return NULL; -} -EXPORT_SYMBOL_GPL(ffa_dev_ops_get); +static const struct ffa_ops ffa_drv_ops = { + .info_ops = &ffa_drv_info_ops, + .msg_ops = &ffa_drv_msg_ops, + .mem_ops = &ffa_drv_mem_ops, +}; void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid) { int count, idx; struct ffa_partition_info *pbuf, *tpbuf; + /* + * FF-A v1.1 provides UUID for each partition as part of the discovery + * API, the discovered UUID must be populated in the device's UUID and + * there is no need to copy the same from the driver table. + */ + if (drv_info->version > FFA_VERSION_1_0) + return; + count = ffa_partition_probe(uuid, &pbuf); if (count <= 0) return; @@ -671,6 +739,7 @@ void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid) static void ffa_setup_partitions(void) { int count, idx; + uuid_t uuid; struct ffa_device *ffa_dev; struct ffa_partition_info *pbuf, *tpbuf; @@ -681,19 +750,24 @@ static void ffa_setup_partitions(void) } for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) { - /* Note that the &uuid_null parameter will require + import_uuid(&uuid, (u8 *)tpbuf->uuid); + + /* Note that if the UUID will be uuid_null, that will require * ffa_device_match() to find the UUID of this partition id - * with help of ffa_device_match_uuid(). Once the FF-A spec - * is updated to provide correct UUID here for each partition - * as part of the discovery API, we need to pass the - * discovered UUID here instead. + * with help of ffa_device_match_uuid(). FF-A v1.1 and above + * provides UUID here for each partition as part of the + * discovery API and the same is passed. */ - ffa_dev = ffa_device_register(&uuid_null, tpbuf->id); + ffa_dev = ffa_device_register(&uuid, tpbuf->id, &ffa_drv_ops); if (!ffa_dev) { pr_err("%s: failed to register partition ID 0x%x\n", __func__, tpbuf->id); continue; } + + if (drv_info->version > FFA_VERSION_1_0 && + !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC)) + _ffa_mode_32bit_set(ffa_dev); } kfree(pbuf); } @@ -751,6 +825,8 @@ static int __init ffa_init(void) ffa_setup_partitions(); + ffa_set_up_mem_ops_native_flag(); + return 0; free_pages: if (drv_info->tx_buffer) diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c index 3ed7ae0d6781e560a69ed79935acabb1c8266235..96060bf90a24abd63c36a9f702e6c633c01083a1 100644 --- a/drivers/firmware/arm_scmi/clock.c +++ b/drivers/firmware/arm_scmi/clock.c @@ -450,9 +450,13 @@ static int scmi_clock_count_get(const struct scmi_protocol_handle *ph) static const struct scmi_clock_info * scmi_clock_info_get(const struct scmi_protocol_handle *ph, u32 clk_id) { + struct scmi_clock_info *clk; struct clock_info *ci = ph->get_priv(ph); - struct scmi_clock_info *clk = ci->clk + clk_id; + if (clk_id >= ci->num_clocks) + return NULL; + + clk = ci->clk + clk_id; if (!clk->name[0]) return NULL; diff --git a/drivers/firmware/arm_scmi/optee.c b/drivers/firmware/arm_scmi/optee.c index 8abace56b95885da4ccb9d52488083996c234e9e..f42dad997ac9a50a947191f55d19a05a1c3d08ba 100644 --- a/drivers/firmware/arm_scmi/optee.c +++ b/drivers/firmware/arm_scmi/optee.c @@ -106,6 +106,7 @@ enum scmi_optee_pta_cmd { * @channel_id: OP-TEE channel ID used for this transport * @tee_session: TEE session identifier * @caps: OP-TEE SCMI channel capabilities + * @rx_len: Response size * @mu: Mutex protection on channel access * @cinfo: SCMI channel information * @shmem: Virtual base address of the shared memory diff --git a/drivers/firmware/arm_scmi/reset.c b/drivers/firmware/arm_scmi/reset.c index 673f3eb498f43483c1ece184c98d8c4b3ce5aef0..e9afa8cab730949ae65b1199283e94cbbe205f78 100644 --- a/drivers/firmware/arm_scmi/reset.c +++ b/drivers/firmware/arm_scmi/reset.c @@ -166,9 +166,13 @@ static int scmi_domain_reset(const struct scmi_protocol_handle *ph, u32 domain, struct scmi_xfer *t; struct scmi_msg_reset_domain_reset *dom; struct scmi_reset_info *pi = ph->get_priv(ph); - struct reset_dom_info *rdom = pi->dom_info + domain; + struct reset_dom_info *rdom; - if (rdom->async_reset) + if (domain >= pi->num_domains) + return -EINVAL; + + rdom = pi->dom_info + domain; + if (rdom->async_reset && flags & AUTONOMOUS_RESET) flags |= ASYNCHRONOUS_RESET; ret = ph->xops->xfer_get_init(ph, RESET, sizeof(*dom), 0, &t); @@ -180,7 +184,7 @@ static int scmi_domain_reset(const struct scmi_protocol_handle *ph, u32 domain, dom->flags = cpu_to_le32(flags); dom->reset_state = cpu_to_le32(state); - if (rdom->async_reset) + if (flags & ASYNCHRONOUS_RESET) ret = ph->xops->do_xfer_with_response(ph, t); else ret = ph->xops->do_xfer(ph, t); diff --git a/drivers/firmware/arm_scmi/scmi_pm_domain.c b/drivers/firmware/arm_scmi/scmi_pm_domain.c index 581d34c9576954d0b4563090e2d7e08f5ec550da..0e05a79de82d8ad6e080ada9f12f87f2715dc7ad 100644 --- a/drivers/firmware/arm_scmi/scmi_pm_domain.c +++ b/drivers/firmware/arm_scmi/scmi_pm_domain.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -53,27 +52,6 @@ static int scmi_pd_power_off(struct generic_pm_domain *domain) return scmi_pd_power(domain, false); } -static int scmi_pd_attach_dev(struct generic_pm_domain *pd, struct device *dev) -{ - int ret; - - ret = pm_clk_create(dev); - if (ret) - return ret; - - ret = of_pm_clk_add_clks(dev); - if (ret >= 0) - return 0; - - pm_clk_destroy(dev); - return ret; -} - -static void scmi_pd_detach_dev(struct generic_pm_domain *pd, struct device *dev) -{ - pm_clk_destroy(dev); -} - static int scmi_pm_domain_probe(struct scmi_device *sdev) { int num_domains, i; @@ -124,10 +102,6 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev) scmi_pd->genpd.name = scmi_pd->name; scmi_pd->genpd.power_off = scmi_pd_power_off; scmi_pd->genpd.power_on = scmi_pd_power_on; - scmi_pd->genpd.attach_dev = scmi_pd_attach_dev; - scmi_pd->genpd.detach_dev = scmi_pd_detach_dev; - scmi_pd->genpd.flags = GENPD_FLAG_PM_CLK | - GENPD_FLAG_ACTIVE_WAKEUP; pm_genpd_init(&scmi_pd->genpd, NULL, state == SCMI_POWER_STATE_GENERIC_OFF); @@ -138,9 +112,28 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev) scmi_pd_data->domains = domains; scmi_pd_data->num_domains = num_domains; + dev_set_drvdata(dev, scmi_pd_data); + return of_genpd_add_provider_onecell(np, scmi_pd_data); } +static void scmi_pm_domain_remove(struct scmi_device *sdev) +{ + int i; + struct genpd_onecell_data *scmi_pd_data; + struct device *dev = &sdev->dev; + struct device_node *np = dev->of_node; + + of_genpd_del_provider(np); + + scmi_pd_data = dev_get_drvdata(dev); + for (i = 0; i < scmi_pd_data->num_domains; i++) { + if (!scmi_pd_data->domains[i]) + continue; + pm_genpd_remove(scmi_pd_data->domains[i]); + } +} + static const struct scmi_device_id scmi_id_table[] = { { SCMI_PROTOCOL_POWER, "genpd" }, { }, @@ -150,6 +143,7 @@ MODULE_DEVICE_TABLE(scmi, scmi_id_table); static struct scmi_driver scmi_power_domain_driver = { .name = "scmi-power-domain", .probe = scmi_pm_domain_probe, + .remove = scmi_pm_domain_remove, .id_table = scmi_id_table, }; module_scmi_driver(scmi_power_domain_driver); diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c index 7288c61178380813cb63fb83e588180c3308ab93..0b5853fa9d874f2d0800a5fa2aa20a45904e0f66 100644 --- a/drivers/firmware/arm_scmi/sensors.c +++ b/drivers/firmware/arm_scmi/sensors.c @@ -762,6 +762,10 @@ static int scmi_sensor_config_get(const struct scmi_protocol_handle *ph, { int ret; struct scmi_xfer *t; + struct sensors_info *si = ph->get_priv(ph); + + if (sensor_id >= si->num_sensors) + return -EINVAL; ret = ph->xops->xfer_get_init(ph, SENSOR_CONFIG_GET, sizeof(__le32), sizeof(__le32), &t); @@ -771,7 +775,6 @@ static int scmi_sensor_config_get(const struct scmi_protocol_handle *ph, put_unaligned_le32(sensor_id, t->tx.buf); ret = ph->xops->do_xfer(ph, t); if (!ret) { - struct sensors_info *si = ph->get_priv(ph); struct scmi_sensor_info *s = si->sensors + sensor_id; *sensor_config = get_unaligned_le64(t->rx.buf); @@ -788,6 +791,10 @@ static int scmi_sensor_config_set(const struct scmi_protocol_handle *ph, int ret; struct scmi_xfer *t; struct scmi_msg_sensor_config_set *msg; + struct sensors_info *si = ph->get_priv(ph); + + if (sensor_id >= si->num_sensors) + return -EINVAL; ret = ph->xops->xfer_get_init(ph, SENSOR_CONFIG_SET, sizeof(*msg), 0, &t); @@ -800,7 +807,6 @@ static int scmi_sensor_config_set(const struct scmi_protocol_handle *ph, ret = ph->xops->do_xfer(ph, t); if (!ret) { - struct sensors_info *si = ph->get_priv(ph); struct scmi_sensor_info *s = si->sensors + sensor_id; s->sensor_config = sensor_config; @@ -831,8 +837,11 @@ static int scmi_sensor_reading_get(const struct scmi_protocol_handle *ph, int ret; struct scmi_xfer *t; struct scmi_msg_sensor_reading_get *sensor; + struct scmi_sensor_info *s; struct sensors_info *si = ph->get_priv(ph); - struct scmi_sensor_info *s = si->sensors + sensor_id; + + if (sensor_id >= si->num_sensors) + return -EINVAL; ret = ph->xops->xfer_get_init(ph, SENSOR_READING_GET, sizeof(*sensor), 0, &t); @@ -841,6 +850,7 @@ static int scmi_sensor_reading_get(const struct scmi_protocol_handle *ph, sensor = t->tx.buf; sensor->id = cpu_to_le32(sensor_id); + s = si->sensors + sensor_id; if (s->async) { sensor->flags = cpu_to_le32(SENSOR_READ_ASYNC); ret = ph->xops->do_xfer_with_response(ph, t); @@ -895,9 +905,13 @@ scmi_sensor_reading_get_timestamped(const struct scmi_protocol_handle *ph, int ret; struct scmi_xfer *t; struct scmi_msg_sensor_reading_get *sensor; + struct scmi_sensor_info *s; struct sensors_info *si = ph->get_priv(ph); - struct scmi_sensor_info *s = si->sensors + sensor_id; + if (sensor_id >= si->num_sensors) + return -EINVAL; + + s = si->sensors + sensor_id; if (!count || !readings || (!s->num_axis && count > 1) || (s->num_axis && count > s->num_axis)) return -EINVAL; @@ -948,6 +962,9 @@ scmi_sensor_info_get(const struct scmi_protocol_handle *ph, u32 sensor_id) { struct sensors_info *si = ph->get_priv(ph); + if (sensor_id >= si->num_sensors) + return NULL; + return si->sensors + sensor_id; } diff --git a/drivers/firmware/dmi_scan.c b/drivers/firmware/dmi_scan.c index f191a1f901ac70cd7924ec7e62db054f9ff84e85..015c95a825d315e02b586ece662d8c43cd484428 100644 --- a/drivers/firmware/dmi_scan.c +++ b/drivers/firmware/dmi_scan.c @@ -567,8 +567,13 @@ static int __init dmi_present(const u8 *buf) { u32 smbios_ver; + /* + * The size of this structure is 31 bytes, but we also accept value + * 30 due to a mistake in SMBIOS specification version 2.1. + */ if (memcmp(buf, "_SM_", 4) == 0 && - buf[5] < 32 && dmi_checksum(buf, buf[5])) { + buf[5] >= 30 && buf[5] <= 32 && + dmi_checksum(buf, buf[5])) { smbios_ver = get_unaligned_be16(buf + 6); smbios_entry_point_size = buf[5]; memcpy(smbios_entry_point, buf, smbios_entry_point_size); @@ -629,8 +634,9 @@ static int __init dmi_present(const u8 *buf) static int __init dmi_smbios3_present(const u8 *buf) { if (memcmp(buf, "_SM3_", 5) == 0 && - buf[6] < 32 && dmi_checksum(buf, buf[6])) { - dmi_ver = get_unaligned_be32(buf + 6) & 0xFFFFFF; + buf[6] >= 24 && buf[6] <= 32 && + dmi_checksum(buf, buf[6])) { + dmi_ver = get_unaligned_be24(buf + 7); dmi_num = 0; /* No longer specified */ dmi_len = get_unaligned_le32(buf + 12); dmi_base = get_unaligned_le64(buf + 16); diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 6cb7384ad2ac7062882197d4f0d6a912cbbad5c2..5b79a4a4a88d85c9cbaa4b98cc4a814d1ec0d74d 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -105,9 +105,50 @@ config EFI_RUNTIME_WRAPPERS config EFI_GENERIC_STUB bool +config EFI_ZBOOT + bool "Enable the generic EFI decompressor" + depends on EFI_GENERIC_STUB && !ARM + select HAVE_KERNEL_GZIP + select HAVE_KERNEL_LZ4 + select HAVE_KERNEL_LZMA + select HAVE_KERNEL_LZO + select HAVE_KERNEL_XZ + select HAVE_KERNEL_ZSTD + help + Create the bootable image as an EFI application that carries the + actual kernel image in compressed form, and decompresses it into + memory before executing it via LoadImage/StartImage EFI boot service + calls. For compatibility with non-EFI loaders, the payload can be + decompressed and executed by the loader as well, provided that the + loader implements the decompression algorithm and that non-EFI boot + is supported by the encapsulated image. (The compression algorithm + used is described in the zboot image header) + +config EFI_ZBOOT_SIGNED + def_bool y + depends on EFI_ZBOOT_SIGNING_CERT != "" + depends on EFI_ZBOOT_SIGNING_KEY != "" + +config EFI_ZBOOT_SIGNING + bool "Sign the EFI decompressor for UEFI secure boot" + depends on EFI_ZBOOT + help + Use the 'sbsign' command line tool (which must exist on the host + path) to sign both the EFI decompressor PE/COFF image, as well as the + encapsulated PE/COFF image, which is subsequently compressed and + wrapped by the former image. + +config EFI_ZBOOT_SIGNING_CERT + string "Certificate to use for signing the compressed EFI boot image" + depends on EFI_ZBOOT_SIGNING + +config EFI_ZBOOT_SIGNING_KEY + string "Private key to use for signing the compressed EFI boot image" + depends on EFI_ZBOOT_SIGNING + config EFI_ARMSTUB_DTB_LOADER bool "Enable the DTB loader" - depends on EFI_GENERIC_STUB && !RISCV + depends on EFI_GENERIC_STUB && !RISCV && !LOONGARCH default y help Select this config option to add support for the dtb= command @@ -124,7 +165,7 @@ config EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER bool "Enable the command line initrd loader" if !X86 depends on EFI_STUB && (EFI_GENERIC_STUB || X86) default y if X86 - depends on !RISCV + depends on !RISCV && !LOONGARCH help Select this config option to add support for the initrd= command line parameter, allowing an initrd that resides on the same volume diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c index 4dde8edd53b62225a54e2fe0da5df2f30bde050f..3e8d4b51a8140c16720eef8f08d311b024b1a830 100644 --- a/drivers/firmware/efi/capsule-loader.c +++ b/drivers/firmware/efi/capsule-loader.c @@ -242,29 +242,6 @@ failed: return ret; } -/** - * efi_capsule_flush - called by file close or file flush - * @file: file pointer - * @id: not used - * - * If a capsule is being partially uploaded then calling this function - * will be treated as upload termination and will free those completed - * buffer pages and -ECANCELED will be returned. - **/ -static int efi_capsule_flush(struct file *file, fl_owner_t id) -{ - int ret = 0; - struct capsule_info *cap_info = file->private_data; - - if (cap_info->index > 0) { - pr_err("capsule upload not complete\n"); - efi_free_all_buff_pages(cap_info); - ret = -ECANCELED; - } - - return ret; -} - /** * efi_capsule_release - called by file close * @inode: not used @@ -277,6 +254,13 @@ static int efi_capsule_release(struct inode *inode, struct file *file) { struct capsule_info *cap_info = file->private_data; + if (cap_info->index > 0 && + (cap_info->header.headersize == 0 || + cap_info->count < cap_info->total_size)) { + pr_err("capsule upload not complete\n"); + efi_free_all_buff_pages(cap_info); + } + kfree(cap_info->pages); kfree(cap_info->phys); kfree(file->private_data); @@ -324,7 +308,6 @@ static const struct file_operations efi_capsule_fops = { .owner = THIS_MODULE, .open = efi_capsule_open, .write = efi_capsule_write, - .flush = efi_capsule_flush, .release = efi_capsule_release, .llseek = no_llseek, }; diff --git a/drivers/firmware/efi/dev-path-parser.c b/drivers/firmware/efi/dev-path-parser.c index eb9c65f978419951329b9e02431c975bcceee052..f80d87c199c3c35adfc73e752b1016a3e3bb749f 100644 --- a/drivers/firmware/efi/dev-path-parser.c +++ b/drivers/firmware/efi/dev-path-parser.c @@ -15,9 +15,11 @@ static long __init parse_acpi_path(const struct efi_dev_path *node, struct device *parent, struct device **child) { - char hid[ACPI_ID_LEN], uid[11]; /* UINT_MAX + null byte */ struct acpi_device *adev; struct device *phys_dev; + char hid[ACPI_ID_LEN]; + u64 uid; + int ret; if (node->header.length != 12) return -EINVAL; @@ -27,12 +29,12 @@ static long __init parse_acpi_path(const struct efi_dev_path *node, 'A' + ((node->acpi.hid >> 5) & 0x1f) - 1, 'A' + ((node->acpi.hid >> 0) & 0x1f) - 1, node->acpi.hid >> 16); - sprintf(uid, "%u", node->acpi.uid); for_each_acpi_dev_match(adev, hid, NULL, -1) { - if (adev->pnp.unique_id && !strcmp(adev->pnp.unique_id, uid)) + ret = acpi_dev_uid_to_integer(adev, &uid); + if (ret == 0 && node->acpi.uid == uid) break; - if (!adev->pnp.unique_id && node->acpi.uid == 0) + if (ret == -ENODATA && node->acpi.uid == 0) break; } if (!adev) diff --git a/drivers/firmware/efi/efi-init.c b/drivers/firmware/efi/efi-init.c index 3928dbff76d0410a0744d355227f5f6596115f70..2fd770b499a35301f2c0c04f09cd379fa11dcf30 100644 --- a/drivers/firmware/efi/efi-init.c +++ b/drivers/firmware/efi/efi-init.c @@ -51,34 +51,10 @@ static phys_addr_t __init efi_to_phys(unsigned long addr) return addr; } -static __initdata unsigned long screen_info_table = EFI_INVALID_TABLE_ADDR; -static __initdata unsigned long cpu_state_table = EFI_INVALID_TABLE_ADDR; - -static const efi_config_table_type_t arch_tables[] __initconst = { - {LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID, &screen_info_table}, - {LINUX_EFI_ARM_CPU_STATE_TABLE_GUID, &cpu_state_table}, - {} -}; +extern __weak const efi_config_table_type_t efi_arch_tables[]; static void __init init_screen_info(void) { - struct screen_info *si; - - if (IS_ENABLED(CONFIG_ARM) && - screen_info_table != EFI_INVALID_TABLE_ADDR) { - si = early_memremap_ro(screen_info_table, sizeof(*si)); - if (!si) { - pr_err("Could not map screen_info config table\n"); - return; - } - screen_info = *si; - early_memunmap(si, sizeof(*si)); - - /* dummycon on ARM needs non-zero values for columns/lines */ - screen_info.orig_video_cols = 80; - screen_info.orig_video_lines = 25; - } - if (screen_info.orig_video_isVGA == VIDEO_TYPE_EFI && memblock_is_map_memory(screen_info.lfb_base)) memblock_mark_nomap(screen_info.lfb_base, screen_info.lfb_size); @@ -119,8 +95,7 @@ static int __init uefi_init(u64 efi_system_table) goto out; } retval = efi_config_parse_tables(config_tables, systab->nr_tables, - IS_ENABLED(CONFIG_ARM) ? arch_tables - : NULL); + efi_arch_tables); early_memunmap(config_tables, table_size); out: @@ -248,36 +223,4 @@ void __init efi_init(void) PAGE_ALIGN(data.size + (data.phys_map & ~PAGE_MASK))); init_screen_info(); - -#ifdef CONFIG_ARM - /* ARM does not permit early mappings to persist across paging_init() */ - efi_memmap_unmap(); - - if (cpu_state_table != EFI_INVALID_TABLE_ADDR) { - struct efi_arm_entry_state *state; - bool dump_state = true; - - state = early_memremap_ro(cpu_state_table, - sizeof(struct efi_arm_entry_state)); - if (state == NULL) { - pr_warn("Unable to map CPU entry state table.\n"); - return; - } - - if ((state->sctlr_before_ebs & 1) == 0) - pr_warn(FW_BUG "EFI stub was entered with MMU and Dcache disabled, please fix your firmware!\n"); - else if ((state->sctlr_after_ebs & 1) == 0) - pr_warn(FW_BUG "ExitBootServices() returned with MMU and Dcache disabled, please fix your firmware!\n"); - else - dump_state = false; - - if (dump_state || efi_enabled(EFI_DBG)) { - pr_info("CPSR at EFI stub entry : 0x%08x\n", state->cpsr_before_ebs); - pr_info("SCTLR at EFI stub entry : 0x%08x\n", state->sctlr_before_ebs); - pr_info("CPSR after ExitBootServices() : 0x%08x\n", state->cpsr_after_ebs); - pr_info("SCTLR after ExitBootServices(): 0x%08x\n", state->sctlr_after_ebs); - } - early_memunmap(state, sizeof(struct efi_arm_entry_state)); - } -#endif } diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index e4080ad96089abd7f84745dd8461c548bcbb7685..9624735f15757e8e38b3eff805c8f620c25871b0 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -55,9 +56,10 @@ EXPORT_SYMBOL(efi); unsigned long __ro_after_init efi_rng_seed = EFI_INVALID_TABLE_ADDR; static unsigned long __initdata mem_reserve = EFI_INVALID_TABLE_ADDR; static unsigned long __initdata rt_prop = EFI_INVALID_TABLE_ADDR; +static unsigned long __initdata initrd = EFI_INVALID_TABLE_ADDR; struct mm_struct efi_mm = { - .mm_rb = RB_ROOT, + .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, efi_mm.mmap_lock), .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), .write_protect_seq = SEQCNT_ZERO(efi_mm.write_protect_seq), @@ -532,6 +534,7 @@ static const efi_config_table_type_t common_tables[] __initconst = { {LINUX_EFI_TPM_EVENT_LOG_GUID, &efi.tpm_log, "TPMEventLog" }, {LINUX_EFI_TPM_FINAL_LOG_GUID, &efi.tpm_final_log, "TPMFinalLog" }, {LINUX_EFI_MEMRESERVE_TABLE_GUID, &mem_reserve, "MEMRESERVE" }, + {LINUX_EFI_INITRD_MEDIA_GUID, &initrd, "INITRD" }, {EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" }, #ifdef CONFIG_EFI_RCI2_TABLE {DELLEMC_EFI_RCI2_TABLE_GUID, &rci2_table_phys }, @@ -674,6 +677,18 @@ int __init efi_config_parse_tables(const efi_config_table_t *config_tables, } } + if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && + initrd != EFI_INVALID_TABLE_ADDR && phys_initrd_size == 0) { + struct linux_efi_initrd *tbl; + + tbl = early_memremap(initrd, sizeof(*tbl)); + if (tbl) { + phys_initrd_start = tbl->base; + phys_initrd_size = tbl->size; + early_memunmap(tbl, sizeof(*tbl)); + } + } + return 0; } diff --git a/drivers/firmware/efi/efibc.c b/drivers/firmware/efi/efibc.c index 8ced7af8e56d28d0dffe40e84506e6a88dd60b54..4f9fb086eab7b0e22252d22e59e5aae55865322d 100644 --- a/drivers/firmware/efi/efibc.c +++ b/drivers/firmware/efi/efibc.c @@ -48,6 +48,9 @@ static int efibc_reboot_notifier_call(struct notifier_block *notifier, return NOTIFY_DONE; wdata = kmalloc(MAX_DATA_LEN * sizeof(efi_char16_t), GFP_KERNEL); + if (!wdata) + return NOTIFY_DONE; + for (l = 0; l < MAX_DATA_LEN - 1 && str[l] != '\0'; l++) wdata[l] = str[l]; wdata[l] = L'\0'; diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index d0537573501e9353446bd45dcf6bf7da026551d1..b1601aad7e1a8d66eb1888454bb866bc7c5d8c4c 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -26,8 +26,10 @@ cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ $(call cc-option,-mno-single-pic-base) cflags-$(CONFIG_RISCV) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ -fpic +cflags-$(CONFIG_LOONGARCH) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ + -fpie -cflags-$(CONFIG_EFI_GENERIC_STUB) += -I$(srctree)/scripts/dtc/libfdt +cflags-$(CONFIG_EFI_PARAMS_FROM_FDT) += -I$(srctree)/scripts/dtc/libfdt KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ -include $(srctree)/include/linux/hidden.h \ @@ -37,8 +39,17 @@ KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ $(call cc-option,-fno-addrsig) \ -D__DISABLE_EXPORTS +# +# struct randomization only makes sense for Linux internal types, which the EFI +# stub code never touches, so let's turn off struct randomization for the stub +# altogether +# +KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS), $(KBUILD_CFLAGS)) + # remove SCS flags from all objects in this directory KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) +# disable CFI +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS)) # disable LTO KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) @@ -46,6 +57,7 @@ GCOV_PROFILE := n # Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n KCSAN_SANITIZE := n +KMSAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y @@ -57,21 +69,32 @@ lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ alignedmem.o relocate.o vsprintf.o -# include the stub's generic dependencies from lib/ when building for ARM/arm64 -efi-deps-y := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c fdt_empty_tree.c fdt_sw.c +# include the stub's libfdt dependencies from lib/ when needed +libfdt-deps := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c \ + fdt_empty_tree.c fdt_sw.c + +lib-$(CONFIG_EFI_PARAMS_FROM_FDT) += fdt.o \ + $(patsubst %.c,lib-%.o,$(libfdt-deps)) $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) -lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o fdt.o string.o \ - $(patsubst %.c,lib-%.o,$(efi-deps-y)) +lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o lib-$(CONFIG_ARM) += arm32-stub.o lib-$(CONFIG_ARM64) += arm64-stub.o lib-$(CONFIG_X86) += x86-stub.o lib-$(CONFIG_RISCV) += riscv-stub.o +lib-$(CONFIG_LOONGARCH) += loongarch-stub.o + CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) +zboot-obj-$(CONFIG_RISCV) := lib-clz_ctz.o lib-ashldi3.o +lib-$(CONFIG_EFI_ZBOOT) += zboot.o $(zboot-obj-y) + +extra-y := $(lib-y) +lib-y := $(patsubst %.o,%.stub.o,$(lib-y)) + # Even when -mbranch-protection=none is set, Clang will generate a # .note.gnu.property for code-less object files (like lib/ctype.c), # so work around this by explicitly removing the unwanted section. @@ -111,9 +134,6 @@ STUBCOPY_RELOC-$(CONFIG_ARM) := R_ARM_ABS # a verification pass to see if any absolute relocations exist in any of the # object files. # -extra-y := $(lib-y) -lib-y := $(patsubst %.o,%.stub.o,$(lib-y)) - STUBCOPY_FLAGS-$(CONFIG_ARM64) += --prefix-alloc-sections=.init \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS @@ -125,6 +145,12 @@ STUBCOPY_FLAGS-$(CONFIG_RISCV) += --prefix-alloc-sections=.init \ --prefix-symbols=__efistub_ STUBCOPY_RELOC-$(CONFIG_RISCV) := R_RISCV_HI20 +# For LoongArch, keep all the symbols in .init section and make sure that no +# absolute symbols references exist. +STUBCOPY_FLAGS-$(CONFIG_LOONGARCH) += --prefix-alloc-sections=.init \ + --prefix-symbols=__efistub_ +STUBCOPY_RELOC-$(CONFIG_LOONGARCH) := R_LARCH_MARK_LA + $(obj)/%.stub.o: $(obj)/%.o FORCE $(call if_changed,stubcopy) diff --git a/drivers/firmware/efi/libstub/Makefile.zboot b/drivers/firmware/efi/libstub/Makefile.zboot new file mode 100644 index 0000000000000000000000000000000000000000..35f234ad8738d16e01e837472cb5e81ef300e6bc --- /dev/null +++ b/drivers/firmware/efi/libstub/Makefile.zboot @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: GPL-2.0 + +# to be include'd by arch/$(ARCH)/boot/Makefile after setting +# EFI_ZBOOT_PAYLOAD, EFI_ZBOOT_BFD_TARGET and EFI_ZBOOT_MACH_TYPE + +comp-type-$(CONFIG_KERNEL_GZIP) := gzip +comp-type-$(CONFIG_KERNEL_LZ4) := lz4 +comp-type-$(CONFIG_KERNEL_LZMA) := lzma +comp-type-$(CONFIG_KERNEL_LZO) := lzo +comp-type-$(CONFIG_KERNEL_XZ) := xzkern +comp-type-$(CONFIG_KERNEL_ZSTD) := zstd22 + +# in GZIP, the appended le32 carrying the uncompressed size is part of the +# format, but in other cases, we just append it at the end for convenience, +# causing the original tools to complain when checking image integrity. +# So disregard it when calculating the payload size in the zimage header. +zboot-method-y := $(comp-type-y)_with_size +zboot-size-len-y := 4 + +zboot-method-$(CONFIG_KERNEL_GZIP) := gzip +zboot-size-len-$(CONFIG_KERNEL_GZIP) := 0 + +quiet_cmd_sbsign = SBSIGN $@ + cmd_sbsign = sbsign --out $@ $< \ + --key $(CONFIG_EFI_ZBOOT_SIGNING_KEY) \ + --cert $(CONFIG_EFI_ZBOOT_SIGNING_CERT) + +$(obj)/$(EFI_ZBOOT_PAYLOAD).signed: $(obj)/$(EFI_ZBOOT_PAYLOAD) FORCE + $(call if_changed,sbsign) + +ZBOOT_PAYLOAD-y := $(EFI_ZBOOT_PAYLOAD) +ZBOOT_PAYLOAD-$(CONFIG_EFI_ZBOOT_SIGNED) := $(EFI_ZBOOT_PAYLOAD).signed + +$(obj)/vmlinuz: $(obj)/$(ZBOOT_PAYLOAD-y) FORCE + $(call if_changed,$(zboot-method-y)) + +OBJCOPYFLAGS_vmlinuz.o := -I binary -O $(EFI_ZBOOT_BFD_TARGET) \ + --rename-section .data=.gzdata,load,alloc,readonly,contents +$(obj)/vmlinuz.o: $(obj)/vmlinuz FORCE + $(call if_changed,objcopy) + +AFLAGS_zboot-header.o += -DMACHINE_TYPE=IMAGE_FILE_MACHINE_$(EFI_ZBOOT_MACH_TYPE) \ + -DZBOOT_EFI_PATH="\"$(realpath $(obj)/vmlinuz.efi.elf)\"" \ + -DZBOOT_SIZE_LEN=$(zboot-size-len-y) \ + -DCOMP_TYPE="\"$(comp-type-y)\"" + +$(obj)/zboot-header.o: $(srctree)/drivers/firmware/efi/libstub/zboot-header.S FORCE + $(call if_changed_rule,as_o_S) + +ZBOOT_DEPS := $(obj)/zboot-header.o $(objtree)/drivers/firmware/efi/libstub/lib.a + +LDFLAGS_vmlinuz.efi.elf := -T $(srctree)/drivers/firmware/efi/libstub/zboot.lds +$(obj)/vmlinuz.efi.elf: $(obj)/vmlinuz.o $(ZBOOT_DEPS) FORCE + $(call if_changed,ld) + +ZBOOT_EFI-y := vmlinuz.efi +ZBOOT_EFI-$(CONFIG_EFI_ZBOOT_SIGNED) := vmlinuz.efi.unsigned + +OBJCOPYFLAGS_$(ZBOOT_EFI-y) := -O binary +$(obj)/$(ZBOOT_EFI-y): $(obj)/vmlinuz.efi.elf FORCE + $(call if_changed,objcopy) + +targets += zboot-header.o vmlinuz vmlinuz.o vmlinuz.efi.elf vmlinuz.efi + +ifneq ($(CONFIG_EFI_ZBOOT_SIGNED),) +$(obj)/vmlinuz.efi: $(obj)/vmlinuz.efi.unsigned FORCE + $(call if_changed,sbsign) +endif + +targets += $(EFI_ZBOOT_PAYLOAD).signed vmlinuz.efi.unsigned diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 577173ee1f83d4eebca0100e2941cd47e3ae34b2..259e4b852d63276d7732f462c54984ad155ab84a 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -19,12 +19,20 @@ efi_status_t check_platform_features(void) { u64 tg; + /* + * If we have 48 bits of VA space for TTBR0 mappings, we can map the + * UEFI runtime regions 1:1 and so calling SetVirtualAddressMap() is + * unnecessary. + */ + if (VA_BITS_MIN >= 48) + efi_novamap = true; + /* UEFI mandates support for 4 KB granularity, no need to check */ if (IS_ENABLED(CONFIG_ARM64_4K_PAGES)) return EFI_SUCCESS; - tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_TGRAN_SHIFT) & 0xf; - if (tg < ID_AA64MMFR0_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_TGRAN_SUPPORTED_MAX) { + tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf; + if (tg < ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX) { if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) efi_err("This 64 KB granular kernel is not supported by your CPU\n"); else @@ -42,26 +50,17 @@ efi_status_t check_platform_features(void) */ static bool check_image_region(u64 base, u64 size) { - unsigned long map_size, desc_size, buff_size; - efi_memory_desc_t *memory_map; - struct efi_boot_memmap map; + struct efi_boot_memmap *map; efi_status_t status; bool ret = false; int map_offset; - map.map = &memory_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = NULL; - map.key_ptr = NULL; - map.buff_size = &buff_size; - - status = efi_get_memory_map(&map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) return false; - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE; /* @@ -74,7 +73,7 @@ static bool check_image_region(u64 base, u64 size) } } - efi_bs_call(free_pool, memory_map); + efi_bs_call(free_pool, map); return ret; } diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c index 3d972061c1b0d30b9d07b450daf51a4a4f32fb01..0c493521b25b8fbc870c93c15645a707c9681be0 100644 --- a/drivers/firmware/efi/libstub/efi-stub-helper.c +++ b/drivers/firmware/efi/libstub/efi-stub-helper.c @@ -218,7 +218,7 @@ efi_status_t efi_parse_options(char const *cmdline) efi_noinitrd = true; } else if (!strcmp(param, "efi") && val) { efi_nochunk = parse_option_str(val, "nochunk"); - efi_novamap = parse_option_str(val, "novamap"); + efi_novamap |= parse_option_str(val, "novamap"); efi_nosoftreserve = IS_ENABLED(CONFIG_EFI_SOFT_RESERVE) && parse_option_str(val, "nosoftreserve"); @@ -310,7 +310,7 @@ bool efi_load_option_unpack(efi_load_option_unpacked_t *dest, * * Detect this case and extract OptionalData. */ -void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_size) +void efi_apply_loadoptions_quirk(const void **load_options, u32 *load_options_size) { const efi_load_option_t *load_option = *load_options; efi_load_option_unpacked_t load_option_unpacked; @@ -334,6 +334,85 @@ void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_si *load_options_size = load_option_unpacked.optional_data_size; } +enum efistub_event { + EFISTUB_EVT_INITRD, + EFISTUB_EVT_LOAD_OPTIONS, + EFISTUB_EVT_COUNT, +}; + +#define STR_WITH_SIZE(s) sizeof(s), s + +static const struct { + u32 pcr_index; + u32 event_id; + u32 event_data_len; + u8 event_data[52]; +} events[] = { + [EFISTUB_EVT_INITRD] = { + 9, + INITRD_EVENT_TAG_ID, + STR_WITH_SIZE("Linux initrd") + }, + [EFISTUB_EVT_LOAD_OPTIONS] = { + 9, + LOAD_OPTIONS_EVENT_TAG_ID, + STR_WITH_SIZE("LOADED_IMAGE::LoadOptions") + }, +}; + +static efi_status_t efi_measure_tagged_event(unsigned long load_addr, + unsigned long load_size, + enum efistub_event event) +{ + efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID; + efi_tcg2_protocol_t *tcg2 = NULL; + efi_status_t status; + + efi_bs_call(locate_protocol, &tcg2_guid, NULL, (void **)&tcg2); + if (tcg2) { + struct efi_measured_event { + efi_tcg2_event_t event_data; + efi_tcg2_tagged_event_t tagged_event; + u8 tagged_event_data[]; + } *evt; + int size = sizeof(*evt) + events[event].event_data_len; + + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, + (void **)&evt); + if (status != EFI_SUCCESS) + goto fail; + + evt->event_data = (struct efi_tcg2_event){ + .event_size = size, + .event_header.header_size = sizeof(evt->event_data.event_header), + .event_header.header_version = EFI_TCG2_EVENT_HEADER_VERSION, + .event_header.pcr_index = events[event].pcr_index, + .event_header.event_type = EV_EVENT_TAG, + }; + + evt->tagged_event = (struct efi_tcg2_tagged_event){ + .tagged_event_id = events[event].event_id, + .tagged_event_data_size = events[event].event_data_len, + }; + + memcpy(evt->tagged_event_data, events[event].event_data, + events[event].event_data_len); + + status = efi_call_proto(tcg2, hash_log_extend_event, 0, + load_addr, load_size, &evt->event_data); + efi_bs_call(free_pool, evt); + + if (status != EFI_SUCCESS) + goto fail; + return EFI_SUCCESS; + } + + return EFI_UNSUPPORTED; +fail: + efi_warn("Failed to measure data for event %d: 0x%lx\n", event, status); + return status; +} + /* * Convert the unicode UEFI command line to ASCII to pass to kernel. * Size of memory allocated return in *cmd_line_len. @@ -341,21 +420,26 @@ void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_si */ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) { - const u16 *s2; - unsigned long cmdline_addr = 0; - int options_chars = efi_table_attr(image, load_options_size); - const u16 *options = efi_table_attr(image, load_options); + const efi_char16_t *options = efi_table_attr(image, load_options); + u32 options_size = efi_table_attr(image, load_options_size); int options_bytes = 0, safe_options_bytes = 0; /* UTF-8 bytes */ + unsigned long cmdline_addr = 0; + const efi_char16_t *s2; bool in_quote = false; efi_status_t status; + u32 options_chars; + + if (options_size > 0) + efi_measure_tagged_event((unsigned long)options, options_size, + EFISTUB_EVT_LOAD_OPTIONS); - efi_apply_loadoptions_quirk((const void **)&options, &options_chars); - options_chars /= sizeof(*options); + efi_apply_loadoptions_quirk((const void **)&options, &options_size); + options_chars = options_size / sizeof(efi_char16_t); if (options) { s2 = options; while (options_bytes < COMMAND_LINE_SIZE && options_chars--) { - u16 c = *s2++; + efi_char16_t c = *s2++; if (c < 0x80) { if (c == L'\0' || c == L'\n') @@ -419,7 +503,6 @@ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) /** * efi_exit_boot_services() - Exit boot services * @handle: handle of the exiting image - * @map: pointer to receive the memory map * @priv: argument to be passed to @priv_func * @priv_func: function to process the memory map before exiting boot services * @@ -432,26 +515,26 @@ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) * * Return: status code */ -efi_status_t efi_exit_boot_services(void *handle, - struct efi_boot_memmap *map, - void *priv, +efi_status_t efi_exit_boot_services(void *handle, void *priv, efi_exit_boot_map_processing priv_func) { + struct efi_boot_memmap *map; efi_status_t status; - status = efi_get_memory_map(map); - + status = efi_get_memory_map(&map, true); if (status != EFI_SUCCESS) - goto fail; + return status; status = priv_func(map, priv); - if (status != EFI_SUCCESS) - goto free_map; + if (status != EFI_SUCCESS) { + efi_bs_call(free_pool, map); + return status; + } if (efi_disable_pci_dma) efi_pci_disable_bridge_busmaster(); - status = efi_bs_call(exit_boot_services, handle, *map->key_ptr); + status = efi_bs_call(exit_boot_services, handle, map->map_key); if (status == EFI_INVALID_PARAMETER) { /* @@ -467,35 +550,26 @@ efi_status_t efi_exit_boot_services(void *handle, * buffer should account for any changes in the map so the call * to get_memory_map() is expected to succeed here. */ - *map->map_size = *map->buff_size; + map->map_size = map->buff_size; status = efi_bs_call(get_memory_map, - map->map_size, - *map->map, - map->key_ptr, - map->desc_size, - map->desc_ver); + &map->map_size, + &map->map, + &map->map_key, + &map->desc_size, + &map->desc_ver); /* exit_boot_services() was called, thus cannot free */ if (status != EFI_SUCCESS) - goto fail; + return status; status = priv_func(map, priv); /* exit_boot_services() was called, thus cannot free */ if (status != EFI_SUCCESS) - goto fail; + return status; - status = efi_bs_call(exit_boot_services, handle, *map->key_ptr); + status = efi_bs_call(exit_boot_services, handle, map->map_key); } - /* exit_boot_services() was called, thus cannot free */ - if (status != EFI_SUCCESS) - goto fail; - - return EFI_SUCCESS; - -free_map: - efi_bs_call(free_pool, *map->map); -fail: return status; } @@ -560,20 +634,16 @@ static const struct { * * %EFI_SUCCESS if the initrd was loaded successfully, in which * case @load_addr and @load_size are assigned accordingly * * %EFI_NOT_FOUND if no LoadFile2 protocol exists on the initrd device path - * * %EFI_INVALID_PARAMETER if load_addr == NULL or load_size == NULL * * %EFI_OUT_OF_RESOURCES if memory allocation failed * * %EFI_LOAD_ERROR in all other cases */ static -efi_status_t efi_load_initrd_dev_path(unsigned long *load_addr, - unsigned long *load_size, +efi_status_t efi_load_initrd_dev_path(struct linux_efi_initrd *initrd, unsigned long max) { efi_guid_t lf2_proto_guid = EFI_LOAD_FILE2_PROTOCOL_GUID; efi_device_path_protocol_t *dp; efi_load_file2_protocol_t *lf2; - unsigned long initrd_addr; - unsigned long initrd_size; efi_handle_t handle; efi_status_t status; @@ -587,124 +657,98 @@ efi_status_t efi_load_initrd_dev_path(unsigned long *load_addr, if (status != EFI_SUCCESS) return status; - status = efi_call_proto(lf2, load_file, dp, false, &initrd_size, NULL); + initrd->size = 0; + status = efi_call_proto(lf2, load_file, dp, false, &initrd->size, NULL); if (status != EFI_BUFFER_TOO_SMALL) return EFI_LOAD_ERROR; - status = efi_allocate_pages(initrd_size, &initrd_addr, max); + status = efi_allocate_pages(initrd->size, &initrd->base, max); if (status != EFI_SUCCESS) return status; - status = efi_call_proto(lf2, load_file, dp, false, &initrd_size, - (void *)initrd_addr); + status = efi_call_proto(lf2, load_file, dp, false, &initrd->size, + (void *)initrd->base); if (status != EFI_SUCCESS) { - efi_free(initrd_size, initrd_addr); + efi_free(initrd->size, initrd->base); return EFI_LOAD_ERROR; } - - *load_addr = initrd_addr; - *load_size = initrd_size; return EFI_SUCCESS; } static efi_status_t efi_load_initrd_cmdline(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, + struct linux_efi_initrd *initrd, unsigned long soft_limit, unsigned long hard_limit) { if (!IS_ENABLED(CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER) || - (IS_ENABLED(CONFIG_X86) && (!efi_is_native() || image == NULL))) { - *load_addr = *load_size = 0; - return EFI_SUCCESS; - } + (IS_ENABLED(CONFIG_X86) && (!efi_is_native() || image == NULL))) + return EFI_UNSUPPORTED; return handle_cmdline_files(image, L"initrd=", sizeof(L"initrd=") - 2, soft_limit, hard_limit, - load_addr, load_size); -} - -static const struct { - efi_tcg2_event_t event_data; - efi_tcg2_tagged_event_t tagged_event; - u8 tagged_event_data[]; -} initrd_tcg2_event = { - { - sizeof(initrd_tcg2_event) + sizeof("Linux initrd"), - { - sizeof(initrd_tcg2_event.event_data.event_header), - EFI_TCG2_EVENT_HEADER_VERSION, - 9, - EV_EVENT_TAG, - }, - }, - { - INITRD_EVENT_TAG_ID, - sizeof("Linux initrd"), - }, - { "Linux initrd" }, -}; - -static void efi_measure_initrd(unsigned long load_addr, unsigned long load_size) -{ - efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID; - efi_tcg2_protocol_t *tcg2 = NULL; - efi_status_t status; - - efi_bs_call(locate_protocol, &tcg2_guid, NULL, (void **)&tcg2); - if (tcg2) { - status = efi_call_proto(tcg2, hash_log_extend_event, - 0, load_addr, load_size, - &initrd_tcg2_event.event_data); - if (status != EFI_SUCCESS) - efi_warn("Failed to measure initrd data: 0x%lx\n", - status); - else - efi_info("Measured initrd data into PCR %d\n", - initrd_tcg2_event.event_data.event_header.pcr_index); - } + &initrd->base, &initrd->size); } /** * efi_load_initrd() - Load initial RAM disk * @image: EFI loaded image protocol - * @load_addr: pointer to loaded initrd - * @load_size: size of loaded initrd * @soft_limit: preferred address for loading the initrd * @hard_limit: upper limit address for loading the initrd * * Return: status code */ efi_status_t efi_load_initrd(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, unsigned long soft_limit, - unsigned long hard_limit) + unsigned long hard_limit, + const struct linux_efi_initrd **out) { - efi_status_t status; + efi_guid_t tbl_guid = LINUX_EFI_INITRD_MEDIA_GUID; + efi_status_t status = EFI_SUCCESS; + struct linux_efi_initrd initrd, *tbl; - if (efi_noinitrd) { - *load_addr = *load_size = 0; - status = EFI_SUCCESS; - } else { - status = efi_load_initrd_dev_path(load_addr, load_size, hard_limit); - if (status == EFI_SUCCESS) { - efi_info("Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path\n"); - if (*load_size > 0) - efi_measure_initrd(*load_addr, *load_size); - } else if (status == EFI_NOT_FOUND) { - status = efi_load_initrd_cmdline(image, load_addr, load_size, - soft_limit, hard_limit); - if (status == EFI_SUCCESS && *load_size > 0) - efi_info("Loaded initrd from command line option\n"); - } - if (status != EFI_SUCCESS) { - efi_err("Failed to load initrd: 0x%lx\n", status); - *load_addr = *load_size = 0; - } + if (!IS_ENABLED(CONFIG_BLK_DEV_INITRD) || efi_noinitrd) + return EFI_SUCCESS; + + status = efi_load_initrd_dev_path(&initrd, hard_limit); + if (status == EFI_SUCCESS) { + efi_info("Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path\n"); + if (initrd.size > 0 && + efi_measure_tagged_event(initrd.base, initrd.size, + EFISTUB_EVT_INITRD) == EFI_SUCCESS) + efi_info("Measured initrd data into PCR 9\n"); + } else if (status == EFI_NOT_FOUND) { + status = efi_load_initrd_cmdline(image, &initrd, soft_limit, + hard_limit); + /* command line loader disabled or no initrd= passed? */ + if (status == EFI_UNSUPPORTED || status == EFI_NOT_READY) + return EFI_SUCCESS; + if (status == EFI_SUCCESS) + efi_info("Loaded initrd from command line option\n"); } + if (status != EFI_SUCCESS) + goto failed; + + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, sizeof(initrd), + (void **)&tbl); + if (status != EFI_SUCCESS) + goto free_initrd; + + *tbl = initrd; + status = efi_bs_call(install_configuration_table, &tbl_guid, tbl); + if (status != EFI_SUCCESS) + goto free_tbl; + + if (out) + *out = tbl; + return EFI_SUCCESS; +free_tbl: + efi_bs_call(free_pool, tbl); +free_initrd: + efi_free(initrd.size, initrd.base); +failed: + efi_err("Failed to load initrd: 0x%lx\n", status); return status; } diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c index f515394cce6e23295e93f018a57b4f1aa4c59a37..cf474f0dd261a9234fdf64283fc2b221bb6aa029 100644 --- a/drivers/firmware/efi/libstub/efi-stub.c +++ b/drivers/firmware/efi/libstub/efi-stub.c @@ -10,7 +10,6 @@ */ #include -#include #include #include "efistub.h" @@ -40,16 +39,22 @@ #ifdef CONFIG_ARM64 # define EFI_RT_VIRTUAL_LIMIT DEFAULT_MAP_WINDOW_64 -#elif defined(CONFIG_RISCV) +#elif defined(CONFIG_RISCV) || defined(CONFIG_LOONGARCH) # define EFI_RT_VIRTUAL_LIMIT TASK_SIZE_MIN -#else +#else /* Only if TASK_SIZE is a constant */ # define EFI_RT_VIRTUAL_LIMIT TASK_SIZE #endif -static u64 virtmap_base = EFI_RT_VIRTUAL_BASE; -static bool flat_va_mapping; +/* + * Some architectures map the EFI regions into the kernel's linear map using a + * fixed offset. + */ +#ifndef EFI_RT_VIRTUAL_OFFSET +#define EFI_RT_VIRTUAL_OFFSET 0 +#endif -const efi_system_table_t *efi_system_table; +static u64 virtmap_base = EFI_RT_VIRTUAL_BASE; +static bool flat_va_mapping = (EFI_RT_VIRTUAL_OFFSET != 0); static struct screen_info *setup_graphics(void) { @@ -124,16 +129,11 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, unsigned long image_addr; unsigned long image_size = 0; /* addr/point and size pairs for memory management*/ - unsigned long initrd_addr = 0; - unsigned long initrd_size = 0; - unsigned long fdt_addr = 0; /* Original DTB */ - unsigned long fdt_size = 0; char *cmdline_ptr = NULL; int cmdline_size = 0; efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID; unsigned long reserve_addr = 0; unsigned long reserve_size = 0; - enum efi_secureboot_mode secure_boot; struct screen_info *si; efi_properties_table_t *prop_tbl; @@ -154,8 +154,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, * information about the running image, such as size and the command * line. */ - status = efi_system_table->boottime->handle_protocol(handle, - &loaded_image_proto, (void *)&image); + status = efi_bs_call(handle_protocol, handle, &loaded_image_proto, + (void *)&image); if (status != EFI_SUCCESS) { efi_err("Failed to get loaded image protocol\n"); goto fail; @@ -209,40 +209,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, /* Ask the firmware to clear memory on unclean shutdown */ efi_enable_reset_attack_mitigation(); - secure_boot = efi_get_secureboot(); - - /* - * Unauthenticated device tree data is a security hazard, so ignore - * 'dtb=' unless UEFI Secure Boot is disabled. We assume that secure - * boot is enabled if we can't determine its state. - */ - if (!IS_ENABLED(CONFIG_EFI_ARMSTUB_DTB_LOADER) || - secure_boot != efi_secureboot_mode_disabled) { - if (strstr(cmdline_ptr, "dtb=")) - efi_err("Ignoring DTB from command line.\n"); - } else { - status = efi_load_dtb(image, &fdt_addr, &fdt_size); - - if (status != EFI_SUCCESS) { - efi_err("Failed to load device tree!\n"); - goto fail_free_image; - } - } - - if (fdt_addr) { - efi_info("Using DTB from command line\n"); - } else { - /* Look for a device tree configuration table entry. */ - fdt_addr = (uintptr_t)get_fdt(&fdt_size); - if (fdt_addr) - efi_info("Using DTB from configuration table\n"); - } - - if (!fdt_addr) - efi_info("Generating empty DTB\n"); - - efi_load_initrd(image, &initrd_addr, &initrd_size, ULONG_MAX, - efi_get_max_initrd_addr(image_addr)); + efi_load_initrd(image, ULONG_MAX, efi_get_max_initrd_addr(image_addr), + NULL); efi_random_get_seed(); @@ -254,8 +222,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, * The easiest way to achieve that is to simply use a 1:1 mapping. */ prop_tbl = get_efi_config_table(EFI_PROPERTIES_TABLE_GUID); - flat_va_mapping = prop_tbl && - (prop_tbl->memory_protection_attribute & + flat_va_mapping |= prop_tbl && + (prop_tbl->memory_protection_attribute & EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA); /* force efi_novamap if SetVirtualAddressMap() is unsupported */ @@ -284,25 +252,8 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, install_memreserve_table(); - status = allocate_new_fdt_and_exit_boot(handle, &fdt_addr, - initrd_addr, initrd_size, - cmdline_ptr, fdt_addr, fdt_size); - if (status != EFI_SUCCESS) - goto fail_free_initrd; - - if (IS_ENABLED(CONFIG_ARM)) - efi_handle_post_ebs_state(); - - efi_enter_kernel(image_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); - /* not reached */ - -fail_free_initrd: - efi_err("Failed to update FDT and exit boot services\n"); + status = efi_boot_kernel(handle, image, image_addr, cmdline_ptr); - efi_free(initrd_size, initrd_addr); - efi_free(fdt_size, fdt_addr); - -fail_free_image: efi_free(image_size, image_addr); efi_free(reserve_size, reserve_addr); fail_free_screeninfo: @@ -313,6 +264,35 @@ fail: return status; } +/* + * efi_allocate_virtmap() - create a pool allocation for the virtmap + * + * Create an allocation that is of sufficient size to hold all the memory + * descriptors that will be passed to SetVirtualAddressMap() to inform the + * firmware about the virtual mapping that will be used under the OS to call + * into the firmware. + */ +efi_status_t efi_alloc_virtmap(efi_memory_desc_t **virtmap, + unsigned long *desc_size, u32 *desc_ver) +{ + unsigned long size, mmap_key; + efi_status_t status; + + /* + * Use the size of the current memory map as an upper bound for the + * size of the buffer we need to pass to SetVirtualAddressMap() to + * cover all EFI_MEMORY_RUNTIME regions. + */ + size = 0; + status = efi_bs_call(get_memory_map, &size, NULL, &mmap_key, desc_size, + desc_ver); + if (status != EFI_BUFFER_TOO_SMALL) + return EFI_LOAD_ERROR; + + return efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, + (void **)virtmap); +} + /* * efi_get_virtmap() - create a virtual mapping for the EFI memory map * @@ -328,6 +308,8 @@ void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, efi_memory_desc_t *in, *out = runtime_map; int l; + *count = 0; + for (l = 0; l < map_size; l += desc_size) { u64 paddr, size; @@ -338,7 +320,7 @@ void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, paddr = in->phys_addr; size = in->num_pages * EFI_PAGE_SIZE; - in->virt_addr = in->phys_addr; + in->virt_addr = in->phys_addr + EFI_RT_VIRTUAL_OFFSET; if (efi_novamap) { continue; } diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index b0ae0a454404b83944ec75465fe0fd890b45f95e..a30fb5d8ef05ae9c781c70737ffcba2bf7d03059 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -160,16 +160,24 @@ void efi_set_u64_split(u64 data, u32 *lo, u32 *hi) */ #define EFI_MMAP_NR_SLACK_SLOTS 8 -struct efi_boot_memmap { - efi_memory_desc_t **map; - unsigned long *map_size; - unsigned long *desc_size; - u32 *desc_ver; - unsigned long *key_ptr; - unsigned long *buff_size; +typedef struct efi_generic_dev_path efi_device_path_protocol_t; + +union efi_device_path_to_text_protocol { + struct { + efi_char16_t *(__efiapi *convert_device_node_to_text)( + const efi_device_path_protocol_t *, + bool, bool); + efi_char16_t *(__efiapi *convert_device_path_to_text)( + const efi_device_path_protocol_t *, + bool, bool); + }; + struct { + u32 convert_device_node_to_text; + u32 convert_device_path_to_text; + } mixed_mode; }; -typedef struct efi_generic_dev_path efi_device_path_protocol_t; +typedef union efi_device_path_to_text_protocol efi_device_path_to_text_protocol_t; typedef void *efi_event_t; /* Note that notifications won't work in mixed mode */ @@ -254,13 +262,17 @@ union efi_boot_services { efi_handle_t *); efi_status_t (__efiapi *install_configuration_table)(efi_guid_t *, void *); - void *load_image; - void *start_image; + efi_status_t (__efiapi *load_image)(bool, efi_handle_t, + efi_device_path_protocol_t *, + void *, unsigned long, + efi_handle_t *); + efi_status_t (__efiapi *start_image)(efi_handle_t, unsigned long *, + efi_char16_t **); efi_status_t __noreturn (__efiapi *exit)(efi_handle_t, efi_status_t, unsigned long, efi_char16_t *); - void *unload_image; + efi_status_t (__efiapi *unload_image)(efi_handle_t); efi_status_t (__efiapi *exit_boot_services)(efi_handle_t, unsigned long); void *get_next_monotonic_count; @@ -277,11 +289,11 @@ union efi_boot_services { void *locate_handle_buffer; efi_status_t (__efiapi *locate_protocol)(efi_guid_t *, void *, void **); - void *install_multiple_protocol_interfaces; - void *uninstall_multiple_protocol_interfaces; + efi_status_t (__efiapi *install_multiple_protocol_interfaces)(efi_handle_t *, ...); + efi_status_t (__efiapi *uninstall_multiple_protocol_interfaces)(efi_handle_t, ...); void *calculate_crc32; - void *copy_mem; - void *set_mem; + void (__efiapi *copy_mem)(void *, const void *, unsigned long); + void (__efiapi *set_mem)(void *, unsigned long, unsigned char); void *create_event_ex; }; struct { @@ -741,6 +753,7 @@ union apple_properties_protocol { typedef u32 efi_tcg2_event_log_format; #define INITRD_EVENT_TAG_ID 0x8F3B22ECU +#define LOAD_OPTIONS_EVENT_TAG_ID 0x8F3B22EDU #define EV_EVENT_TAG 0x00000006U #define EFI_TCG2_EVENT_HEADER_VERSION 0x1 @@ -840,7 +853,7 @@ typedef struct { u16 file_path_list_length; const efi_char16_t *description; const efi_device_path_protocol_t *file_path_list; - size_t optional_data_size; + u32 optional_data_size; const void *optional_data; } efi_load_option_unpacked_t; @@ -850,20 +863,16 @@ typedef efi_status_t (*efi_exit_boot_map_processing)( struct efi_boot_memmap *map, void *priv); -efi_status_t efi_exit_boot_services(void *handle, - struct efi_boot_memmap *map, - void *priv, +efi_status_t efi_exit_boot_services(void *handle, void *priv, efi_exit_boot_map_processing priv_func); -efi_status_t allocate_new_fdt_and_exit_boot(void *handle, - unsigned long *new_fdt_addr, - u64 initrd_addr, u64 initrd_size, - char *cmdline_ptr, - unsigned long fdt_addr, - unsigned long fdt_size); +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr); void *get_fdt(unsigned long *fdt_size); +efi_status_t efi_alloc_virtmap(efi_memory_desc_t **virtmap, + unsigned long *desc_size, u32 *desc_ver); void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, unsigned long desc_size, efi_memory_desc_t *runtime_map, int *count); @@ -885,11 +894,12 @@ __printf(1, 2) int efi_printk(char const *fmt, ...); void efi_free(unsigned long size, unsigned long addr); -void efi_apply_loadoptions_quirk(const void **load_options, int *load_options_size); +void efi_apply_loadoptions_quirk(const void **load_options, u32 *load_options_size); char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len); -efi_status_t efi_get_memory_map(struct efi_boot_memmap *map); +efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, + bool install_cfg_tbl); efi_status_t efi_allocate_pages(unsigned long size, unsigned long *addr, unsigned long max); @@ -932,10 +942,9 @@ static inline efi_status_t efi_load_dtb(efi_loaded_image_t *image, } efi_status_t efi_load_initrd(efi_loaded_image_t *image, - unsigned long *load_addr, - unsigned long *load_size, unsigned long soft_limit, - unsigned long hard_limit); + unsigned long hard_limit, + const struct linux_efi_initrd **out); /* * This function handles the architcture specific differences between arm and * arm64 regarding where the kernel image must be loaded and any memory that diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index fe567be0f118beb18ec061850e17b3c8ae77945a..4f4d98e51fbfd99f8a354b62ccdfa38089202092 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -28,8 +28,7 @@ static void fdt_update_cell_size(void *fdt) } static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size, - void *fdt, int new_fdt_size, char *cmdline_ptr, - u64 initrd_addr, u64 initrd_size) + void *fdt, int new_fdt_size, char *cmdline_ptr) { int node, num_rsv; int status; @@ -93,21 +92,6 @@ static efi_status_t update_fdt(void *orig_fdt, unsigned long orig_fdt_size, goto fdt_set_fail; } - /* Set initrd address/end in device tree, if present */ - if (initrd_size != 0) { - u64 initrd_image_end; - u64 initrd_image_start = cpu_to_fdt64(initrd_addr); - - status = fdt_setprop_var(fdt, node, "linux,initrd-start", initrd_image_start); - if (status) - goto fdt_set_fail; - - initrd_image_end = cpu_to_fdt64(initrd_addr + initrd_size); - status = fdt_setprop_var(fdt, node, "linux,initrd-end", initrd_image_end); - if (status) - goto fdt_set_fail; - } - /* Add FDT entries for EFI runtime services in chosen node. */ node = fdt_subnode_offset(fdt, 0, "chosen"); fdt_val64 = cpu_to_fdt64((u64)(unsigned long)efi_system_table); @@ -170,25 +154,25 @@ static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map) if (node < 0) return EFI_LOAD_ERROR; - fdt_val64 = cpu_to_fdt64((unsigned long)*map->map); + fdt_val64 = cpu_to_fdt64((unsigned long)map->map); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-start", fdt_val64); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->map_size); + fdt_val32 = cpu_to_fdt32(map->map_size); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-size", fdt_val32); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->desc_size); + fdt_val32 = cpu_to_fdt32(map->desc_size); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-desc-size", fdt_val32); if (err) return EFI_LOAD_ERROR; - fdt_val32 = cpu_to_fdt32(*map->desc_ver); + fdt_val32 = cpu_to_fdt32(map->desc_ver); err = fdt_setprop_inplace_var(fdt, node, "linux,uefi-mmap-desc-ver", fdt_val32); if (err) @@ -198,22 +182,25 @@ static efi_status_t update_fdt_memmap(void *fdt, struct efi_boot_memmap *map) } struct exit_boot_struct { + struct efi_boot_memmap *boot_memmap; efi_memory_desc_t *runtime_map; - int *runtime_entry_count; + int runtime_entry_count; void *new_fdt_addr; }; -static efi_status_t exit_boot_func(struct efi_boot_memmap *map, - void *priv) +static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) { struct exit_boot_struct *p = priv; + + p->boot_memmap = map; + /* * Update the memory map with virtual addresses. The function will also * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME * entries so that we can pass it straight to SetVirtualAddressMap() */ - efi_get_virtmap(*map->map, *map->map_size, *map->desc_size, - p->runtime_map, p->runtime_entry_count); + efi_get_virtmap(map->map, map->map_size, map->desc_size, + p->runtime_map, &p->runtime_entry_count); return update_fdt_memmap(p->new_fdt_addr, map); } @@ -223,86 +210,86 @@ static efi_status_t exit_boot_func(struct efi_boot_memmap *map, #endif /* - * Allocate memory for a new FDT, then add EFI, commandline, and - * initrd related fields to the FDT. This routine increases the - * FDT allocation size until the allocated memory is large - * enough. EFI allocations are in EFI_PAGE_SIZE granules, - * which are fixed at 4K bytes, so in most cases the first - * allocation should succeed. - * EFI boot services are exited at the end of this function. - * There must be no allocations between the get_memory_map() - * call and the exit_boot_services() call, so the exiting of - * boot services is very tightly tied to the creation of the FDT - * with the final memory map in it. + * Allocate memory for a new FDT, then add EFI and commandline related fields + * to the FDT. This routine increases the FDT allocation size until the + * allocated memory is large enough. EFI allocations are in EFI_PAGE_SIZE + * granules, which are fixed at 4K bytes, so in most cases the first allocation + * should succeed. EFI boot services are exited at the end of this function. + * There must be no allocations between the get_memory_map() call and the + * exit_boot_services() call, so the exiting of boot services is very tightly + * tied to the creation of the FDT with the final memory map in it. */ - +static efi_status_t allocate_new_fdt_and_exit_boot(void *handle, + efi_loaded_image_t *image, unsigned long *new_fdt_addr, - u64 initrd_addr, u64 initrd_size, - char *cmdline_ptr, - unsigned long fdt_addr, - unsigned long fdt_size) + char *cmdline_ptr) { - unsigned long map_size, desc_size, buff_size; + unsigned long desc_size; u32 desc_ver; - unsigned long mmap_key; - efi_memory_desc_t *memory_map, *runtime_map; efi_status_t status; - int runtime_entry_count; - struct efi_boot_memmap map; struct exit_boot_struct priv; + unsigned long fdt_addr = 0; + unsigned long fdt_size = 0; - map.map = &runtime_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = &desc_ver; - map.key_ptr = &mmap_key; - map.buff_size = &buff_size; + if (!efi_novamap) { + status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, + &desc_ver); + if (status != EFI_SUCCESS) { + efi_err("Unable to retrieve UEFI memory map.\n"); + return status; + } + } /* - * Get a copy of the current memory map that we will use to prepare - * the input for SetVirtualAddressMap(). We don't have to worry about - * subsequent allocations adding entries, since they could not affect - * the number of EFI_MEMORY_RUNTIME regions. + * Unauthenticated device tree data is a security hazard, so ignore + * 'dtb=' unless UEFI Secure Boot is disabled. We assume that secure + * boot is enabled if we can't determine its state. */ - status = efi_get_memory_map(&map); - if (status != EFI_SUCCESS) { - efi_err("Unable to retrieve UEFI memory map.\n"); - return status; + if (!IS_ENABLED(CONFIG_EFI_ARMSTUB_DTB_LOADER) || + efi_get_secureboot() != efi_secureboot_mode_disabled) { + if (strstr(cmdline_ptr, "dtb=")) + efi_err("Ignoring DTB from command line.\n"); + } else { + status = efi_load_dtb(image, &fdt_addr, &fdt_size); + + if (status != EFI_SUCCESS && status != EFI_NOT_READY) { + efi_err("Failed to load device tree!\n"); + goto fail; + } } + if (fdt_addr) { + efi_info("Using DTB from command line\n"); + } else { + /* Look for a device tree configuration table entry. */ + fdt_addr = (uintptr_t)get_fdt(&fdt_size); + if (fdt_addr) + efi_info("Using DTB from configuration table\n"); + } + + if (!fdt_addr) + efi_info("Generating empty DTB\n"); + efi_info("Exiting boot services...\n"); - map.map = &memory_map; status = efi_allocate_pages(MAX_FDT_SIZE, new_fdt_addr, ULONG_MAX); if (status != EFI_SUCCESS) { efi_err("Unable to allocate memory for new device tree.\n"); goto fail; } - /* - * Now that we have done our final memory allocation (and free) - * we can get the memory map key needed for exit_boot_services(). - */ - status = efi_get_memory_map(&map); - if (status != EFI_SUCCESS) - goto fail_free_new_fdt; - status = update_fdt((void *)fdt_addr, fdt_size, - (void *)*new_fdt_addr, MAX_FDT_SIZE, cmdline_ptr, - initrd_addr, initrd_size); + (void *)*new_fdt_addr, MAX_FDT_SIZE, cmdline_ptr); if (status != EFI_SUCCESS) { efi_err("Unable to construct new device tree.\n"); goto fail_free_new_fdt; } - runtime_entry_count = 0; - priv.runtime_map = runtime_map; - priv.runtime_entry_count = &runtime_entry_count; - priv.new_fdt_addr = (void *)*new_fdt_addr; + priv.new_fdt_addr = (void *)*new_fdt_addr; - status = efi_exit_boot_services(handle, &map, &priv, exit_boot_func); + status = efi_exit_boot_services(handle, &priv, exit_boot_func); if (status == EFI_SUCCESS) { efi_set_virtual_address_map_t *svam; @@ -312,8 +299,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, /* Install the new virtual address map */ svam = efi_system_table->runtime->set_virtual_address_map; - status = svam(runtime_entry_count * desc_size, desc_size, - desc_ver, runtime_map); + status = svam(priv.runtime_entry_count * desc_size, desc_size, + desc_ver, priv.runtime_map); /* * We are beyond the point of no return here, so if the call to @@ -321,6 +308,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, * incoming kernel but proceed normally otherwise. */ if (status != EFI_SUCCESS) { + efi_memory_desc_t *p; int l; /* @@ -329,8 +317,9 @@ efi_status_t allocate_new_fdt_and_exit_boot(void *handle, * the incoming kernel that no virtual translation has * been installed. */ - for (l = 0; l < map_size; l += desc_size) { - efi_memory_desc_t *p = (void *)memory_map + l; + for (l = 0; l < priv.boot_memmap->map_size; + l += priv.boot_memmap->desc_size) { + p = (void *)priv.boot_memmap->map + l; if (p->attribute & EFI_MEMORY_RUNTIME) p->virt_addr = 0; @@ -345,11 +334,33 @@ fail_free_new_fdt: efi_free(MAX_FDT_SIZE, *new_fdt_addr); fail: - efi_system_table->boottime->free_pool(runtime_map); + efi_free(fdt_size, fdt_addr); + + efi_bs_call(free_pool, priv.runtime_map); return EFI_LOAD_ERROR; } +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr) +{ + unsigned long fdt_addr; + efi_status_t status; + + status = allocate_new_fdt_and_exit_boot(handle, image, &fdt_addr, + cmdline_ptr); + if (status != EFI_SUCCESS) { + efi_err("Failed to update FDT and exit boot services\n"); + return status; + } + + if (IS_ENABLED(CONFIG_ARM)) + efi_handle_post_ebs_state(); + + efi_enter_kernel(kernel_addr, fdt_addr, fdt_totalsize((void *)fdt_addr)); + /* not reached */ +} + void *get_fdt(unsigned long *fdt_size) { void *fdt; diff --git a/drivers/firmware/efi/libstub/file.c b/drivers/firmware/efi/libstub/file.c index dd95f330fe6e173ef8bed69f2b59b7e2cbaddb57..f756c61396e9a82c57bd6a1e9f291475777280bd 100644 --- a/drivers/firmware/efi/libstub/file.c +++ b/drivers/firmware/efi/libstub/file.c @@ -66,10 +66,28 @@ static efi_status_t efi_open_file(efi_file_protocol_t *volume, static efi_status_t efi_open_volume(efi_loaded_image_t *image, efi_file_protocol_t **fh) { + struct efi_vendor_dev_path *dp = image->file_path; + efi_guid_t li_proto = LOADED_IMAGE_PROTOCOL_GUID; efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID; efi_simple_file_system_protocol_t *io; efi_status_t status; + // If we are using EFI zboot, we should look for the file system + // protocol on the parent image's handle instead + if (IS_ENABLED(CONFIG_EFI_ZBOOT) && + image->parent_handle != NULL && + dp != NULL && + dp->header.type == EFI_DEV_MEDIA && + dp->header.sub_type == EFI_DEV_MEDIA_VENDOR && + !efi_guidcmp(dp->vendorguid, LINUX_EFI_ZBOOT_MEDIA_GUID)) { + status = efi_bs_call(handle_protocol, image->parent_handle, + &li_proto, (void *)&image); + if (status != EFI_SUCCESS) { + efi_err("Failed to locate parent image handle\n"); + return status; + } + } + status = efi_bs_call(handle_protocol, image->device_handle, &fs_proto, (void **)&io); if (status != EFI_SUCCESS) { @@ -136,7 +154,7 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, unsigned long *load_size) { const efi_char16_t *cmdline = image->load_options; - int cmdline_len = image->load_options_size; + u32 cmdline_len = image->load_options_size; unsigned long efi_chunk_size = ULONG_MAX; efi_file_protocol_t *volume = NULL; efi_file_protocol_t *file; @@ -238,6 +256,9 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, if (volume) volume->close(volume); + + if (*load_size == 0) + return EFI_NOT_READY; return EFI_SUCCESS; err_close_file: diff --git a/drivers/firmware/efi/libstub/intrinsics.c b/drivers/firmware/efi/libstub/intrinsics.c new file mode 100644 index 0000000000000000000000000000000000000000..a04ab39292b62d2bf53d69e461967befc4dd1c0f --- /dev/null +++ b/drivers/firmware/efi/libstub/intrinsics.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include "efistub.h" + +#ifdef CONFIG_KASAN +#undef memcpy +#undef memmove +#undef memset +void *__memcpy(void *__dest, const void *__src, size_t __n) __alias(memcpy); +void *__memmove(void *__dest, const void *__src, size_t count) __alias(memmove); +void *__memset(void *s, int c, size_t count) __alias(memset); +#endif + +void *memcpy(void *dst, const void *src, size_t len) +{ + efi_bs_call(copy_mem, dst, src, len); + return dst; +} + +extern void *memmove(void *dst, const void *src, size_t len) __alias(memcpy); + +void *memset(void *dst, int c, size_t len) +{ + efi_bs_call(set_mem, dst, len, c & U8_MAX); + return dst; +} diff --git a/drivers/firmware/efi/libstub/loongarch-stub.c b/drivers/firmware/efi/libstub/loongarch-stub.c new file mode 100644 index 0000000000000000000000000000000000000000..32329f2a92f951965c50dc64e37f94757db984f8 --- /dev/null +++ b/drivers/firmware/efi/libstub/loongarch-stub.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Yun Liu + * Huacai Chen + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include +#include +#include "efistub.h" + +typedef void __noreturn (*kernel_entry_t)(bool efi, unsigned long cmdline, + unsigned long systab); + +extern int kernel_asize; +extern int kernel_fsize; +extern int kernel_offset; +extern kernel_entry_t kernel_entry; + +efi_status_t check_platform_features(void) +{ + return EFI_SUCCESS; +} + +efi_status_t handle_kernel_image(unsigned long *image_addr, + unsigned long *image_size, + unsigned long *reserve_addr, + unsigned long *reserve_size, + efi_loaded_image_t *image, + efi_handle_t image_handle) +{ + efi_status_t status; + unsigned long kernel_addr = 0; + + kernel_addr = (unsigned long)&kernel_offset - kernel_offset; + + status = efi_relocate_kernel(&kernel_addr, kernel_fsize, kernel_asize, + PHYSADDR(VMLINUX_LOAD_ADDRESS), SZ_2M, 0x0); + + *image_addr = kernel_addr; + *image_size = kernel_asize; + + return status; +} + +struct exit_boot_struct { + efi_memory_desc_t *runtime_map; + int runtime_entry_count; +}; + +static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) +{ + struct exit_boot_struct *p = priv; + + /* + * Update the memory map with virtual addresses. The function will also + * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME + * entries so that we can pass it straight to SetVirtualAddressMap() + */ + efi_get_virtmap(map->map, map->map_size, map->desc_size, + p->runtime_map, &p->runtime_entry_count); + + return EFI_SUCCESS; +} + +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr) +{ + kernel_entry_t real_kernel_entry; + struct exit_boot_struct priv; + unsigned long desc_size; + efi_status_t status; + u32 desc_ver; + + status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, &desc_ver); + if (status != EFI_SUCCESS) { + efi_err("Unable to retrieve UEFI memory map.\n"); + return status; + } + + efi_info("Exiting boot services\n"); + + efi_novamap = false; + status = efi_exit_boot_services(handle, &priv, exit_boot_func); + if (status != EFI_SUCCESS) + return status; + + /* Install the new virtual address map */ + efi_rt_call(set_virtual_address_map, + priv.runtime_entry_count * desc_size, desc_size, + desc_ver, priv.runtime_map); + + /* Config Direct Mapping */ + csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); + csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); + + real_kernel_entry = (kernel_entry_t) + ((unsigned long)&kernel_entry - kernel_addr + VMLINUX_LOAD_ADDRESS); + + real_kernel_entry(true, (unsigned long)cmdline_ptr, + (unsigned long)efi_system_table); +} diff --git a/drivers/firmware/efi/libstub/mem.c b/drivers/firmware/efi/libstub/mem.c index feef8d4be113b653d8216347ed81e7beaf55396e..45841ef55a9f6ba284ec7f4bd34fc24a6b91451e 100644 --- a/drivers/firmware/efi/libstub/mem.c +++ b/drivers/firmware/efi/libstub/mem.c @@ -5,71 +5,66 @@ #include "efistub.h" -static inline bool mmap_has_headroom(unsigned long buff_size, - unsigned long map_size, - unsigned long desc_size) -{ - unsigned long slack = buff_size - map_size; - - return slack / desc_size >= EFI_MMAP_NR_SLACK_SLOTS; -} - /** * efi_get_memory_map() - get memory map - * @map: on return pointer to memory map + * @map: pointer to memory map pointer to which to assign the + * newly allocated memory map + * @install_cfg_tbl: whether or not to install the boot memory map as a + * configuration table * * Retrieve the UEFI memory map. The allocated memory leaves room for * up to EFI_MMAP_NR_SLACK_SLOTS additional memory map entries. * * Return: status code */ -efi_status_t efi_get_memory_map(struct efi_boot_memmap *map) +efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, + bool install_cfg_tbl) { - efi_memory_desc_t *m = NULL; + int memtype = install_cfg_tbl ? EFI_ACPI_RECLAIM_MEMORY + : EFI_LOADER_DATA; + efi_guid_t tbl_guid = LINUX_EFI_BOOT_MEMMAP_GUID; + struct efi_boot_memmap *m, tmp; efi_status_t status; - unsigned long key; - u32 desc_version; - - *map->desc_size = sizeof(*m); - *map->map_size = *map->desc_size * 32; - *map->buff_size = *map->map_size; -again: - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, - *map->map_size, (void **)&m); + unsigned long size; + + tmp.map_size = 0; + status = efi_bs_call(get_memory_map, &tmp.map_size, NULL, &tmp.map_key, + &tmp.desc_size, &tmp.desc_ver); + if (status != EFI_BUFFER_TOO_SMALL) + return EFI_LOAD_ERROR; + + size = tmp.map_size + tmp.desc_size * EFI_MMAP_NR_SLACK_SLOTS; + status = efi_bs_call(allocate_pool, memtype, sizeof(*m) + size, + (void **)&m); if (status != EFI_SUCCESS) - goto fail; - - *map->desc_size = 0; - key = 0; - status = efi_bs_call(get_memory_map, map->map_size, m, - &key, map->desc_size, &desc_version); - if (status == EFI_BUFFER_TOO_SMALL || - !mmap_has_headroom(*map->buff_size, *map->map_size, - *map->desc_size)) { - efi_bs_call(free_pool, m); + return status; + + if (install_cfg_tbl) { /* - * Make sure there is some entries of headroom so that the - * buffer can be reused for a new map after allocations are - * no longer permitted. Its unlikely that the map will grow to - * exceed this headroom once we are ready to trigger - * ExitBootServices() + * Installing a configuration table might allocate memory, and + * this may modify the memory map. This means we should install + * the configuration table first, and re-install or delete it + * as needed. */ - *map->map_size += *map->desc_size * EFI_MMAP_NR_SLACK_SLOTS; - *map->buff_size = *map->map_size; - goto again; + status = efi_bs_call(install_configuration_table, &tbl_guid, m); + if (status != EFI_SUCCESS) + goto free_map; } - if (status == EFI_SUCCESS) { - if (map->key_ptr) - *map->key_ptr = key; - if (map->desc_ver) - *map->desc_ver = desc_version; - } else { - efi_bs_call(free_pool, m); - } + m->buff_size = m->map_size = size; + status = efi_bs_call(get_memory_map, &m->map_size, m->map, &m->map_key, + &m->desc_size, &m->desc_ver); + if (status != EFI_SUCCESS) + goto uninstall_table; + + *map = m; + return EFI_SUCCESS; -fail: - *map->map = m; +uninstall_table: + if (install_cfg_tbl) + efi_bs_call(install_configuration_table, &tbl_guid, NULL); +free_map: + efi_bs_call(free_pool, m); return status; } diff --git a/drivers/firmware/efi/libstub/randomalloc.c b/drivers/firmware/efi/libstub/randomalloc.c index 715f374791542bb477bdbb9e613862c6c5b861c7..9fb5869896be7340c9bb7f4141c4cf93c9a4bbca 100644 --- a/drivers/firmware/efi/libstub/randomalloc.c +++ b/drivers/firmware/efi/libstub/randomalloc.c @@ -55,22 +55,13 @@ efi_status_t efi_random_alloc(unsigned long size, unsigned long *addr, unsigned long random_seed) { - unsigned long map_size, desc_size, total_slots = 0, target_slot; + unsigned long total_slots = 0, target_slot; unsigned long total_mirrored_slots = 0; - unsigned long buff_size; + struct efi_boot_memmap *map; efi_status_t status; - efi_memory_desc_t *memory_map; int map_offset; - struct efi_boot_memmap map; - map.map = &memory_map; - map.map_size = &map_size; - map.desc_size = &desc_size; - map.desc_ver = NULL; - map.key_ptr = NULL; - map.buff_size = &buff_size; - - status = efi_get_memory_map(&map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) return status; @@ -80,8 +71,8 @@ efi_status_t efi_random_alloc(unsigned long size, size = round_up(size, EFI_ALLOC_ALIGN); /* count the suitable slots in each memory map entry */ - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; unsigned long slots; slots = get_entry_num_slots(md, size, ilog2(align)); @@ -109,8 +100,8 @@ efi_status_t efi_random_alloc(unsigned long size, * to calculate the randomly chosen address, and allocate it directly * using EFI_ALLOCATE_ADDRESS. */ - for (map_offset = 0; map_offset < map_size; map_offset += desc_size) { - efi_memory_desc_t *md = (void *)memory_map + map_offset; + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; efi_physical_addr_t target; unsigned long pages; @@ -133,7 +124,7 @@ efi_status_t efi_random_alloc(unsigned long size, break; } - efi_bs_call(free_pool, memory_map); + efi_bs_call(free_pool, map); return status; } diff --git a/drivers/firmware/efi/libstub/relocate.c b/drivers/firmware/efi/libstub/relocate.c index 8ee9eb2b90392d993431bba9972923cc4c293683..bf6fbd5d22a1a53af196be2a15f5593468f78da4 100644 --- a/drivers/firmware/efi/libstub/relocate.c +++ b/drivers/firmware/efi/libstub/relocate.c @@ -23,21 +23,12 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, unsigned long *addr, unsigned long min) { - unsigned long map_size, desc_size, buff_size; - efi_memory_desc_t *map; + struct efi_boot_memmap *map; efi_status_t status; unsigned long nr_pages; int i; - struct efi_boot_memmap boot_map; - boot_map.map = ↦ - boot_map.map_size = &map_size; - boot_map.desc_size = &desc_size; - boot_map.desc_ver = NULL; - boot_map.key_ptr = NULL; - boot_map.buff_size = &buff_size; - - status = efi_get_memory_map(&boot_map); + status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) goto fail; @@ -52,12 +43,12 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, size = round_up(size, EFI_ALLOC_ALIGN); nr_pages = size / EFI_PAGE_SIZE; - for (i = 0; i < map_size / desc_size; i++) { + for (i = 0; i < map->map_size / map->desc_size; i++) { efi_memory_desc_t *desc; - unsigned long m = (unsigned long)map; + unsigned long m = (unsigned long)map->map; u64 start, end; - desc = efi_early_memdesc_ptr(m, desc_size, i); + desc = efi_early_memdesc_ptr(m, map->desc_size, i); if (desc->type != EFI_CONVENTIONAL_MEMORY) continue; @@ -87,7 +78,7 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, } } - if (i == map_size / desc_size) + if (i == map->map_size / map->desc_size) status = EFI_NOT_FOUND; efi_bs_call(free_pool, map); diff --git a/drivers/firmware/efi/libstub/secureboot.c b/drivers/firmware/efi/libstub/secureboot.c index 8a18930f3eb69561a0dcc199b398b8cd09fd55df..516f4f0069bd2fcfecc37481351e3f576fc5944b 100644 --- a/drivers/firmware/efi/libstub/secureboot.c +++ b/drivers/firmware/efi/libstub/secureboot.c @@ -14,7 +14,7 @@ /* SHIM variables */ static const efi_guid_t shim_guid = EFI_SHIM_LOCK_GUID; -static const efi_char16_t shim_MokSBState_name[] = L"MokSBState"; +static const efi_char16_t shim_MokSBState_name[] = L"MokSBStateRT"; static efi_status_t get_var(efi_char16_t *name, efi_guid_t *vendor, u32 *attr, unsigned long *data_size, void *data) @@ -43,8 +43,8 @@ enum efi_secureboot_mode efi_get_secureboot(void) /* * See if a user has put the shim into insecure mode. If so, and if the - * variable doesn't have the runtime attribute set, we might as well - * honor that. + * variable doesn't have the non-volatile attribute set, we might as + * well honor that. */ size = sizeof(moksbstate); status = get_efi_var(shim_MokSBState_name, &shim_guid, @@ -53,7 +53,7 @@ enum efi_secureboot_mode efi_get_secureboot(void) /* If it fails, we don't care why. Default to secure */ if (status != EFI_SUCCESS) goto secure_boot_enabled; - if (!(attr & EFI_VARIABLE_RUNTIME_ACCESS) && moksbstate == 1) + if (!(attr & EFI_VARIABLE_NON_VOLATILE) && moksbstate == 1) return efi_secureboot_mode_disabled; secure_boot_enabled: diff --git a/drivers/firmware/efi/libstub/systable.c b/drivers/firmware/efi/libstub/systable.c new file mode 100644 index 0000000000000000000000000000000000000000..91d016b02f8ca873a3ecf7e48878edec6888d8fb --- /dev/null +++ b/drivers/firmware/efi/libstub/systable.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include "efistub.h" + +const efi_system_table_t *efi_system_table; diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 05ae8bcc9d671e46bb350d0193c866c54f22bb3d..b9ce6393e35313c9d714c126b370010bc304610e 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -220,7 +220,6 @@ adjust_memory_range_protection(unsigned long start, unsigned long size) unsigned long end, next; unsigned long rounded_start, rounded_end; unsigned long unprotect_start, unprotect_size; - int has_system_memory = 0; if (efi_dxe_table == NULL) return; @@ -517,6 +516,13 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, hdr->ramdisk_image = 0; hdr->ramdisk_size = 0; + /* + * Disregard any setup data that was provided by the bootloader: + * setup_data could be pointing anywhere, and we have no way of + * authenticating or validating the payload. + */ + hdr->setup_data = 0; + efi_stub_entry(handle, sys_table_arg, boot_params); /* not reached */ @@ -716,32 +722,22 @@ static efi_status_t exit_boot_func(struct efi_boot_memmap *map, efi_set_u64_split((unsigned long)efi_system_table, &p->efi->efi_systab, &p->efi->efi_systab_hi); - p->efi->efi_memdesc_size = *map->desc_size; - p->efi->efi_memdesc_version = *map->desc_ver; - efi_set_u64_split((unsigned long)*map->map, + p->efi->efi_memdesc_size = map->desc_size; + p->efi->efi_memdesc_version = map->desc_ver; + efi_set_u64_split((unsigned long)map->map, &p->efi->efi_memmap, &p->efi->efi_memmap_hi); - p->efi->efi_memmap_size = *map->map_size; + p->efi->efi_memmap_size = map->map_size; return EFI_SUCCESS; } static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) { - unsigned long map_sz, key, desc_size, buff_size; - efi_memory_desc_t *mem_map; struct setup_data *e820ext = NULL; __u32 e820ext_size = 0; efi_status_t status; - __u32 desc_version; - struct efi_boot_memmap map; struct exit_boot_struct priv; - map.map = &mem_map; - map.map_size = &map_sz; - map.desc_size = &desc_size; - map.desc_ver = &desc_version; - map.key_ptr = &key; - map.buff_size = &buff_size; priv.boot_params = boot_params; priv.efi = &boot_params->efi_info; @@ -750,7 +746,7 @@ static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) return status; /* Might as well exit boot services now */ - status = efi_exit_boot_services(handle, &map, &priv, exit_boot_func); + status = efi_exit_boot_services(handle, &priv, exit_boot_func); if (status != EFI_SUCCESS) return status; @@ -776,7 +772,7 @@ unsigned long efi_main(efi_handle_t handle, unsigned long bzimage_addr = (unsigned long)startup_32; unsigned long buffer_start, buffer_end; struct setup_header *hdr = &boot_params->hdr; - unsigned long addr, size; + const struct linux_efi_initrd *initrd = NULL; efi_status_t status; efi_system_table = sys_table_arg; @@ -871,17 +867,18 @@ unsigned long efi_main(efi_handle_t handle, * arguments will be processed only if image is not NULL, which will be * the case only if we were loaded via the PE entry point. */ - status = efi_load_initrd(image, &addr, &size, hdr->initrd_addr_max, - ULONG_MAX); + status = efi_load_initrd(image, hdr->initrd_addr_max, ULONG_MAX, + &initrd); if (status != EFI_SUCCESS) goto fail; - if (size > 0) { - efi_set_u64_split(addr, &hdr->ramdisk_image, + if (initrd && initrd->size > 0) { + efi_set_u64_split(initrd->base, &hdr->ramdisk_image, &boot_params->ext_ramdisk_image); - efi_set_u64_split(size, &hdr->ramdisk_size, + efi_set_u64_split(initrd->size, &hdr->ramdisk_size, &boot_params->ext_ramdisk_size); } + /* * If the boot loader gave us a value for secure_boot then we use that, * otherwise we ask the BIOS. diff --git a/drivers/firmware/efi/libstub/zboot-header.S b/drivers/firmware/efi/libstub/zboot-header.S new file mode 100644 index 0000000000000000000000000000000000000000..9e6fe061ab07a00859789f832ca0f2f18c032ae7 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot-header.S @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +#ifdef CONFIG_64BIT + .set .Lextra_characteristics, 0x0 + .set .Lpe_opt_magic, PE_OPT_MAGIC_PE32PLUS +#else + .set .Lextra_characteristics, IMAGE_FILE_32BIT_MACHINE + .set .Lpe_opt_magic, PE_OPT_MAGIC_PE32 +#endif + + .section ".head", "a" + .globl __efistub_efi_zboot_header +__efistub_efi_zboot_header: +.Ldoshdr: + .long MZ_MAGIC + .ascii "zimg" // image type + .long __efistub__gzdata_start - .Ldoshdr // payload offset + .long __efistub__gzdata_size - ZBOOT_SIZE_LEN // payload size + .long 0, 0 // reserved + .asciz COMP_TYPE // compression type + .org .Ldoshdr + 0x3c + .long .Lpehdr - .Ldoshdr // PE header offset + +.Lpehdr: + .long PE_MAGIC + .short MACHINE_TYPE + .short .Lsection_count + .long 0 + .long 0 + .long 0 + .short .Lsection_table - .Loptional_header + .short IMAGE_FILE_DEBUG_STRIPPED | \ + IMAGE_FILE_EXECUTABLE_IMAGE | \ + IMAGE_FILE_LINE_NUMS_STRIPPED |\ + .Lextra_characteristics + +.Loptional_header: + .short .Lpe_opt_magic + .byte 0, 0 + .long _etext - .Lefi_header_end + .long __data_size + .long 0 + .long __efistub_efi_zboot_entry - .Ldoshdr + .long .Lefi_header_end - .Ldoshdr + +#ifdef CONFIG_64BIT + .quad 0 +#else + .long _etext - .Ldoshdr, 0x0 +#endif + .long 4096 + .long 512 + .short 0, 0 + .short LINUX_EFISTUB_MAJOR_VERSION // MajorImageVersion + .short LINUX_EFISTUB_MINOR_VERSION // MinorImageVersion + .short 0, 0 + .long 0 + .long _end - .Ldoshdr + + .long .Lefi_header_end - .Ldoshdr + .long 0 + .short IMAGE_SUBSYSTEM_EFI_APPLICATION + .short 0 +#ifdef CONFIG_64BIT + .quad 0, 0, 0, 0 +#else + .long 0, 0, 0, 0 +#endif + .long 0 + .long (.Lsection_table - .) / 8 + + .quad 0 // ExportTable + .quad 0 // ImportTable + .quad 0 // ResourceTable + .quad 0 // ExceptionTable + .quad 0 // CertificationTable + .quad 0 // BaseRelocationTable +#ifdef CONFIG_DEBUG_EFI + .long .Lefi_debug_table - .Ldoshdr // DebugTable + .long .Lefi_debug_table_size +#endif + +.Lsection_table: + .ascii ".text\0\0\0" + .long _etext - .Lefi_header_end + .long .Lefi_header_end - .Ldoshdr + .long _etext - .Lefi_header_end + .long .Lefi_header_end - .Ldoshdr + + .long 0, 0 + .short 0, 0 + .long IMAGE_SCN_CNT_CODE | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_EXECUTE + + .ascii ".data\0\0\0" + .long __data_size + .long _etext - .Ldoshdr + .long __data_rawsize + .long _etext - .Ldoshdr + + .long 0, 0 + .short 0, 0 + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_WRITE + + .set .Lsection_count, (. - .Lsection_table) / 40 + +#ifdef CONFIG_DEBUG_EFI + .section ".rodata", "a" + .align 2 +.Lefi_debug_table: + // EFI_IMAGE_DEBUG_DIRECTORY_ENTRY + .long 0 // Characteristics + .long 0 // TimeDateStamp + .short 0 // MajorVersion + .short 0 // MinorVersion + .long IMAGE_DEBUG_TYPE_CODEVIEW // Type + .long .Lefi_debug_entry_size // SizeOfData + .long 0 // RVA + .long .Lefi_debug_entry - .Ldoshdr // FileOffset + + .set .Lefi_debug_table_size, . - .Lefi_debug_table + .previous + +.Lefi_debug_entry: + // EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY + .ascii "NB10" // Signature + .long 0 // Unknown + .long 0 // Unknown2 + .long 0 // Unknown3 + + .asciz ZBOOT_EFI_PATH + + .set .Lefi_debug_entry_size, . - .Lefi_debug_entry +#endif + + .p2align 12 +.Lefi_header_end: + diff --git a/drivers/firmware/efi/libstub/zboot.c b/drivers/firmware/efi/libstub/zboot.c new file mode 100644 index 0000000000000000000000000000000000000000..ea72c8f27da6cd19eb3752633427cbf9fd3599ac --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +#include "efistub.h" + +static unsigned char zboot_heap[SZ_256K] __aligned(64); +static unsigned long free_mem_ptr, free_mem_end_ptr; + +#define STATIC static +#if defined(CONFIG_KERNEL_GZIP) +#include "../../../../lib/decompress_inflate.c" +#elif defined(CONFIG_KERNEL_LZ4) +#include "../../../../lib/decompress_unlz4.c" +#elif defined(CONFIG_KERNEL_LZMA) +#include "../../../../lib/decompress_unlzma.c" +#elif defined(CONFIG_KERNEL_LZO) +#include "../../../../lib/decompress_unlzo.c" +#elif defined(CONFIG_KERNEL_XZ) +#undef memcpy +#define memcpy memcpy +#undef memmove +#define memmove memmove +#include "../../../../lib/decompress_unxz.c" +#elif defined(CONFIG_KERNEL_ZSTD) +#include "../../../../lib/decompress_unzstd.c" +#endif + +extern char efi_zboot_header[]; +extern char _gzdata_start[], _gzdata_end[]; + +static void log(efi_char16_t str[]) +{ + efi_call_proto(efi_table_attr(efi_system_table, con_out), + output_string, L"EFI decompressor: "); + efi_call_proto(efi_table_attr(efi_system_table, con_out), + output_string, str); + efi_call_proto(efi_table_attr(efi_system_table, con_out), + output_string, L"\n"); +} + +static void error(char *x) +{ + log(L"error() called from decompressor library\n"); +} + +// Local version to avoid pulling in memcmp() +static bool guids_eq(const efi_guid_t *a, const efi_guid_t *b) +{ + const u32 *l = (u32 *)a; + const u32 *r = (u32 *)b; + + return l[0] == r[0] && l[1] == r[1] && l[2] == r[2] && l[3] == r[3]; +} + +static efi_status_t __efiapi +load_file(efi_load_file_protocol_t *this, efi_device_path_protocol_t *rem, + bool boot_policy, unsigned long *bufsize, void *buffer) +{ + unsigned long compressed_size = _gzdata_end - _gzdata_start; + struct efi_vendor_dev_path *vendor_dp; + bool decompress = false; + unsigned long size; + int ret; + + if (rem == NULL || bufsize == NULL) + return EFI_INVALID_PARAMETER; + + if (boot_policy) + return EFI_UNSUPPORTED; + + // Look for our vendor media device node in the remaining file path + if (rem->type == EFI_DEV_MEDIA && + rem->sub_type == EFI_DEV_MEDIA_VENDOR) { + vendor_dp = container_of(rem, struct efi_vendor_dev_path, header); + if (!guids_eq(&vendor_dp->vendorguid, &LINUX_EFI_ZBOOT_MEDIA_GUID)) + return EFI_NOT_FOUND; + + decompress = true; + rem = (void *)(vendor_dp + 1); + } + + if (rem->type != EFI_DEV_END_PATH || + rem->sub_type != EFI_DEV_END_ENTIRE) + return EFI_NOT_FOUND; + + // The uncompressed size of the payload is appended to the raw bit + // stream, and may therefore appear misaligned in memory + size = decompress ? get_unaligned_le32(_gzdata_end - 4) + : compressed_size; + if (buffer == NULL || *bufsize < size) { + *bufsize = size; + return EFI_BUFFER_TOO_SMALL; + } + + if (decompress) { + ret = __decompress(_gzdata_start, compressed_size, NULL, NULL, + buffer, size, NULL, error); + if (ret < 0) { + log(L"Decompression failed"); + return EFI_DEVICE_ERROR; + } + } else { + memcpy(buffer, _gzdata_start, compressed_size); + } + + return EFI_SUCCESS; +} + +// Return the length in bytes of the device path up to the first end node. +static int device_path_length(const efi_device_path_protocol_t *dp) +{ + int len = 0; + + while (dp->type != EFI_DEV_END_PATH) { + len += dp->length; + dp = (void *)((u8 *)dp + dp->length); + } + return len; +} + +static void append_rel_offset_node(efi_device_path_protocol_t **dp, + unsigned long start, unsigned long end) +{ + struct efi_rel_offset_dev_path *rodp = (void *)*dp; + + rodp->header.type = EFI_DEV_MEDIA; + rodp->header.sub_type = EFI_DEV_MEDIA_REL_OFFSET; + rodp->header.length = sizeof(struct efi_rel_offset_dev_path); + rodp->reserved = 0; + rodp->starting_offset = start; + rodp->ending_offset = end; + + *dp = (void *)(rodp + 1); +} + +static void append_ven_media_node(efi_device_path_protocol_t **dp, + efi_guid_t *guid) +{ + struct efi_vendor_dev_path *vmdp = (void *)*dp; + + vmdp->header.type = EFI_DEV_MEDIA; + vmdp->header.sub_type = EFI_DEV_MEDIA_VENDOR; + vmdp->header.length = sizeof(struct efi_vendor_dev_path); + vmdp->vendorguid = *guid; + + *dp = (void *)(vmdp + 1); +} + +static void append_end_node(efi_device_path_protocol_t **dp) +{ + (*dp)->type = EFI_DEV_END_PATH; + (*dp)->sub_type = EFI_DEV_END_ENTIRE; + (*dp)->length = sizeof(struct efi_generic_dev_path); + + ++*dp; +} + +asmlinkage efi_status_t __efiapi +efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab) +{ + struct efi_mem_mapped_dev_path mmdp = { + .header.type = EFI_DEV_HW, + .header.sub_type = EFI_DEV_MEM_MAPPED, + .header.length = sizeof(struct efi_mem_mapped_dev_path) + }; + efi_device_path_protocol_t *parent_dp, *dpp, *lf2_dp, *li_dp; + efi_load_file2_protocol_t zboot_load_file2; + efi_loaded_image_t *parent, *child; + unsigned long exit_data_size; + efi_handle_t child_handle; + efi_handle_t zboot_handle; + efi_char16_t *exit_data; + efi_status_t status; + void *dp_alloc; + int dp_len; + + WRITE_ONCE(efi_system_table, systab); + + free_mem_ptr = (unsigned long)&zboot_heap; + free_mem_end_ptr = free_mem_ptr + sizeof(zboot_heap); + + exit_data = NULL; + exit_data_size = 0; + + status = efi_bs_call(handle_protocol, handle, + &LOADED_IMAGE_PROTOCOL_GUID, (void **)&parent); + if (status != EFI_SUCCESS) { + log(L"Failed to locate parent's loaded image protocol"); + return status; + } + + status = efi_bs_call(handle_protocol, handle, + &LOADED_IMAGE_DEVICE_PATH_PROTOCOL_GUID, + (void **)&parent_dp); + if (status != EFI_SUCCESS || parent_dp == NULL) { + // Create a MemoryMapped() device path node to describe + // the parent image if no device path was provided. + mmdp.memory_type = parent->image_code_type; + mmdp.starting_addr = (unsigned long)parent->image_base; + mmdp.ending_addr = (unsigned long)parent->image_base + + parent->image_size - 1; + parent_dp = &mmdp.header; + dp_len = sizeof(mmdp); + } else { + dp_len = device_path_length(parent_dp); + } + + // Allocate some pool memory for device path protocol data + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, + 2 * (dp_len + sizeof(struct efi_rel_offset_dev_path) + + sizeof(struct efi_generic_dev_path)) + + sizeof(struct efi_vendor_dev_path), + (void **)&dp_alloc); + if (status != EFI_SUCCESS) { + log(L"Failed to allocate device path pool memory"); + return status; + } + + // Create a device path describing the compressed payload in this image + // <...parent_dp...>/Offset(, ) + lf2_dp = memcpy(dp_alloc, parent_dp, dp_len); + dpp = (void *)((u8 *)lf2_dp + dp_len); + append_rel_offset_node(&dpp, + (unsigned long)(_gzdata_start - efi_zboot_header), + (unsigned long)(_gzdata_end - efi_zboot_header - 1)); + append_end_node(&dpp); + + // Create a device path describing the decompressed payload in this image + // <...parent_dp...>/Offset(, )/VenMedia(ZBOOT_MEDIA_GUID) + dp_len += sizeof(struct efi_rel_offset_dev_path); + li_dp = memcpy(dpp, lf2_dp, dp_len); + dpp = (void *)((u8 *)li_dp + dp_len); + append_ven_media_node(&dpp, &LINUX_EFI_ZBOOT_MEDIA_GUID); + append_end_node(&dpp); + + zboot_handle = NULL; + zboot_load_file2.load_file = load_file; + status = efi_bs_call(install_multiple_protocol_interfaces, + &zboot_handle, + &EFI_DEVICE_PATH_PROTOCOL_GUID, lf2_dp, + &EFI_LOAD_FILE2_PROTOCOL_GUID, &zboot_load_file2, + NULL); + if (status != EFI_SUCCESS) { + log(L"Failed to install LoadFile2 protocol and device path"); + goto free_dpalloc; + } + + status = efi_bs_call(load_image, false, handle, li_dp, NULL, 0, + &child_handle); + if (status != EFI_SUCCESS) { + log(L"Failed to load image"); + goto uninstall_lf2; + } + + status = efi_bs_call(handle_protocol, child_handle, + &LOADED_IMAGE_PROTOCOL_GUID, (void **)&child); + if (status != EFI_SUCCESS) { + log(L"Failed to locate child's loaded image protocol"); + goto unload_image; + } + + // Copy the kernel command line + child->load_options = parent->load_options; + child->load_options_size = parent->load_options_size; + + status = efi_bs_call(start_image, child_handle, &exit_data_size, + &exit_data); + if (status != EFI_SUCCESS) { + log(L"StartImage() returned with error"); + if (exit_data_size > 0) + log(exit_data); + + // If StartImage() returns EFI_SECURITY_VIOLATION, the image is + // not unloaded so we need to do it by hand. + if (status == EFI_SECURITY_VIOLATION) +unload_image: + efi_bs_call(unload_image, child_handle); + } + +uninstall_lf2: + efi_bs_call(uninstall_multiple_protocol_interfaces, + zboot_handle, + &EFI_DEVICE_PATH_PROTOCOL_GUID, lf2_dp, + &EFI_LOAD_FILE2_PROTOCOL_GUID, &zboot_load_file2, + NULL); + +free_dpalloc: + efi_bs_call(free_pool, dp_alloc); + + efi_bs_call(exit, handle, status, exit_data_size, exit_data); + + // Free ExitData in case Exit() returned with a failure code, + // but return the original status code. + log(L"Exit() returned with failure code"); + if (exit_data != NULL) + efi_bs_call(free_pool, exit_data); + return status; +} diff --git a/drivers/firmware/efi/libstub/zboot.lds b/drivers/firmware/efi/libstub/zboot.lds new file mode 100644 index 0000000000000000000000000000000000000000..87a62765bafdbd59732fab2b0d15550141ce7107 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot.lds @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +ENTRY(__efistub_efi_zboot_header); + +SECTIONS +{ + .head : ALIGN(4096) { + *(.head) + } + + .text : { + *(.text* .init.text*) + } + + .rodata : ALIGN(8) { + __efistub__gzdata_start = .; + *(.gzdata) + __efistub__gzdata_end = .; + *(.rodata* .init.rodata* .srodata*) + _etext = ALIGN(4096); + . = _etext; + } + + .data : ALIGN(4096) { + *(.data* .init.data*) + _edata = ALIGN(512); + . = _edata; + } + + .bss : { + *(.bss* .init.bss*) + _end = ALIGN(512); + . = _end; + } + + /DISCARD/ : { + *(.modinfo .init.modinfo) + } +} + +PROVIDE(__efistub__gzdata_size = ABSOLUTE(. - __efistub__gzdata_start)); + +PROVIDE(__data_rawsize = ABSOLUTE(_edata - _etext)); +PROVIDE(__data_size = ABSOLUTE(_end - _etext)); diff --git a/drivers/firmware/google/gsmi.c b/drivers/firmware/google/gsmi.c index adaa492c3d2dfae33af1de749511a9b65c9ecfd4..4e2575dfeb908ff50d37a1c6a5d7be4f6313ba5b 100644 --- a/drivers/firmware/google/gsmi.c +++ b/drivers/firmware/google/gsmi.c @@ -681,6 +681,15 @@ static struct notifier_block gsmi_die_notifier = { static int gsmi_panic_callback(struct notifier_block *nb, unsigned long reason, void *arg) { + + /* + * Panic callbacks are executed with all other CPUs stopped, + * so we must not attempt to spin waiting for gsmi_dev.lock + * to be released. + */ + if (spin_is_locked(&gsmi_dev.lock)) + return NOTIFY_DONE; + gsmi_shutdown_reason(GSMI_SHUTDOWN_PANIC); return NOTIFY_DONE; } diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index cfb448eabdaa22e367c5f9159cdc9ddfd782332f..e7bcfca4159f603317660d8525e6eafd4488ec94 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -163,6 +164,8 @@ int psci_set_osi_mode(bool enable) PSCI_1_0_SUSPEND_MODE_PC; err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, suspend_mode, 0, 0); + if (err < 0) + pr_warn("failed to set %s mode: %d\n", enable ? "OSI" : "PC", err); return psci_to_linux_errno(err); } @@ -274,7 +277,7 @@ static void set_conduit(enum arm_smccc_conduit conduit) psci_conduit = conduit; } -static int get_set_conduit_method(struct device_node *np) +static int get_set_conduit_method(const struct device_node *np) { const char *method; @@ -324,17 +327,130 @@ static void psci_sys_poweroff(void) invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); } -static int __init psci_features(u32 psci_func_id) +static int psci_features(u32 psci_func_id) { return invoke_psci_fn(PSCI_1_0_FN_PSCI_FEATURES, psci_func_id, 0, 0); } +#ifdef CONFIG_DEBUG_FS + +#define PSCI_ID(ver, _name) \ + { .fn = PSCI_##ver##_FN_##_name, .name = #_name, } +#define PSCI_ID_NATIVE(ver, _name) \ + { .fn = PSCI_FN_NATIVE(ver, _name), .name = #_name, } + +/* A table of all optional functions */ +static const struct { + u32 fn; + const char *name; +} psci_fn_ids[] = { + PSCI_ID_NATIVE(0_2, MIGRATE), + PSCI_ID(0_2, MIGRATE_INFO_TYPE), + PSCI_ID_NATIVE(0_2, MIGRATE_INFO_UP_CPU), + PSCI_ID(1_0, CPU_FREEZE), + PSCI_ID_NATIVE(1_0, CPU_DEFAULT_SUSPEND), + PSCI_ID_NATIVE(1_0, NODE_HW_STATE), + PSCI_ID_NATIVE(1_0, SYSTEM_SUSPEND), + PSCI_ID(1_0, SET_SUSPEND_MODE), + PSCI_ID_NATIVE(1_0, STAT_RESIDENCY), + PSCI_ID_NATIVE(1_0, STAT_COUNT), + PSCI_ID_NATIVE(1_1, SYSTEM_RESET2), + PSCI_ID(1_1, MEM_PROTECT), + PSCI_ID_NATIVE(1_1, MEM_PROTECT_CHECK_RANGE), +}; + +static int psci_debugfs_read(struct seq_file *s, void *data) +{ + int feature, type, i; + u32 ver; + + ver = psci_ops.get_version(); + seq_printf(s, "PSCIv%d.%d\n", + PSCI_VERSION_MAJOR(ver), + PSCI_VERSION_MINOR(ver)); + + /* PSCI_FEATURES is available only starting from 1.0 */ + if (PSCI_VERSION_MAJOR(ver) < 1) + return 0; + + feature = psci_features(ARM_SMCCC_VERSION_FUNC_ID); + if (feature != PSCI_RET_NOT_SUPPORTED) { + ver = invoke_psci_fn(ARM_SMCCC_VERSION_FUNC_ID, 0, 0, 0); + seq_printf(s, "SMC Calling Convention v%d.%d\n", + PSCI_VERSION_MAJOR(ver), + PSCI_VERSION_MINOR(ver)); + } else { + seq_puts(s, "SMC Calling Convention v1.0 is assumed\n"); + } + + feature = psci_features(PSCI_FN_NATIVE(0_2, CPU_SUSPEND)); + if (feature < 0) { + seq_printf(s, "PSCI_FEATURES(CPU_SUSPEND) error (%d)\n", feature); + } else { + seq_printf(s, "OSI is %ssupported\n", + (feature & BIT(0)) ? "" : "not "); + seq_printf(s, "%s StateID format is used\n", + (feature & BIT(1)) ? "Extended" : "Original"); + } + + type = psci_ops.migrate_info_type(); + if (type == PSCI_0_2_TOS_UP_MIGRATE || + type == PSCI_0_2_TOS_UP_NO_MIGRATE) { + unsigned long cpuid; + + seq_printf(s, "Trusted OS %smigrate capable\n", + type == PSCI_0_2_TOS_UP_NO_MIGRATE ? "not " : ""); + cpuid = psci_migrate_info_up_cpu(); + seq_printf(s, "Trusted OS resident on physical CPU 0x%lx (#%d)\n", + cpuid, resident_cpu); + } else if (type == PSCI_0_2_TOS_MP) { + seq_puts(s, "Trusted OS migration not required\n"); + } else { + if (type != PSCI_RET_NOT_SUPPORTED) + seq_printf(s, "MIGRATE_INFO_TYPE returned unknown type (%d)\n", type); + } + + for (i = 0; i < ARRAY_SIZE(psci_fn_ids); i++) { + feature = psci_features(psci_fn_ids[i].fn); + if (feature == PSCI_RET_NOT_SUPPORTED) + continue; + if (feature < 0) + seq_printf(s, "PSCI_FEATURES(%s) error (%d)\n", + psci_fn_ids[i].name, feature); + else + seq_printf(s, "%s is supported\n", psci_fn_ids[i].name); + } + + return 0; +} + +static int psci_debugfs_open(struct inode *inode, struct file *f) +{ + return single_open(f, psci_debugfs_read, NULL); +} + +static const struct file_operations psci_debugfs_ops = { + .owner = THIS_MODULE, + .open = psci_debugfs_open, + .release = single_release, + .read = seq_read, + .llseek = seq_lseek +}; + +static int __init psci_debugfs_init(void) +{ + return PTR_ERR_OR_ZERO(debugfs_create_file("psci", 0444, NULL, NULL, + &psci_debugfs_ops)); +} +late_initcall(psci_debugfs_init) +#endif + #ifdef CONFIG_CPU_IDLE static int psci_suspend_finisher(unsigned long state) { u32 power_state = state; - phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume)); + phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume); return psci_ops.cpu_suspend(power_state, pa_cpu_resume); } @@ -359,7 +475,7 @@ int psci_cpu_suspend_enter(u32 state) static int psci_system_suspend(unsigned long unused) { - phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume)); + phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume); return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND), pa_cpu_resume, 0, 0); @@ -528,7 +644,7 @@ typedef int (*psci_initcall_t)(const struct device_node *); * * Probe based on PSCI PSCI_VERSION function */ -static int __init psci_0_2_init(struct device_node *np) +static int __init psci_0_2_init(const struct device_node *np) { int err; @@ -549,7 +665,7 @@ static int __init psci_0_2_init(struct device_node *np) /* * PSCI < v0.2 get PSCI Function IDs via DT. */ -static int __init psci_0_1_init(struct device_node *np) +static int __init psci_0_1_init(const struct device_node *np) { u32 id; int err; @@ -585,7 +701,7 @@ static int __init psci_0_1_init(struct device_node *np) return 0; } -static int __init psci_1_0_init(struct device_node *np) +static int __init psci_1_0_init(const struct device_node *np) { int err; diff --git a/drivers/firmware/qcom_scm.h b/drivers/firmware/qcom_scm.h index 0d51eef2472f02c5dd22232fa8fa441cf2467b8c..db3d08a012090f41933e71d007b36afe4d39654c 100644 --- a/drivers/firmware/qcom_scm.h +++ b/drivers/firmware/qcom_scm.h @@ -129,8 +129,6 @@ extern int scm_legacy_call(struct device *dev, const struct qcom_scm_desc *desc, #define QCOM_SCM_SMMU_CONFIG_ERRATA1 0x03 #define QCOM_SCM_SMMU_CONFIG_ERRATA1_CLIENT_ALL 0x02 -extern void __qcom_scm_init(void); - /* common error codes */ #define QCOM_SCM_V2_EBUSY -12 #define QCOM_SCM_ENOMEM -5 diff --git a/drivers/firmware/tegra/bpmp-debugfs.c b/drivers/firmware/tegra/bpmp-debugfs.c index 0c440afd522479bc62a0c81d284bdc4986806ece..9d3874cdaaeefc567e1be8b31d3915a74ad709c9 100644 --- a/drivers/firmware/tegra/bpmp-debugfs.c +++ b/drivers/firmware/tegra/bpmp-debugfs.c @@ -377,18 +377,11 @@ static ssize_t bpmp_debug_store(struct file *file, const char __user *buf, if (!filename) return -ENOENT; - databuf = kmalloc(count, GFP_KERNEL); - if (!databuf) - return -ENOMEM; - - if (copy_from_user(databuf, buf, count)) { - err = -EFAULT; - goto free_ret; - } + databuf = memdup_user(buf, count); + if (IS_ERR(databuf)) + return PTR_ERR(databuf); err = mrq_debug_write(bpmp, filename, databuf, count); - -free_ret: kfree(databuf); return err ?: count; diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index d1f652802181e5ac9415b41bc608d3c654fc67ff..ff5cabe70a2b2478eef8b742149546d03cc57daf 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -1311,6 +1311,37 @@ int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, id, 0, payload); } +/** + * zynqmp_pm_set_sd_config - PM call to set value of SD config registers + * @node: SD node ID + * @config: The config type of SD registers + * @value: Value to be set + * + * Return: Returns 0 on success or error value on failure. + */ +int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value) +{ + return zynqmp_pm_invoke_fn(PM_IOCTL, node, IOCTL_SET_SD_CONFIG, + config, value, NULL); +} +EXPORT_SYMBOL_GPL(zynqmp_pm_set_sd_config); + +/** + * zynqmp_pm_set_gem_config - PM call to set value of GEM config registers + * @node: GEM node ID + * @config: The config type of GEM registers + * @value: Value to be set + * + * Return: Returns 0 on success or error value on failure. + */ +int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config, + u32 value) +{ + return zynqmp_pm_invoke_fn(PM_IOCTL, node, IOCTL_SET_GEM_CONFIG, + config, value, NULL); +} +EXPORT_SYMBOL_GPL(zynqmp_pm_set_gem_config); + /** * struct zynqmp_pm_shutdown_scope - Struct for shutdown scope * @subtype: Shutdown subtype diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index fd1fa55c911308221a1deb41a57cf061e0bcec73..0914e7328b1a53664d233b5f2bd52576b09d397b 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -77,12 +77,18 @@ static void cci_pci_free_irq(struct pci_dev *pcidev) #define PCIE_DEVICE_ID_INTEL_PAC_D5005 0x0B2B #define PCIE_DEVICE_ID_SILICOM_PAC_N5010 0x1000 #define PCIE_DEVICE_ID_SILICOM_PAC_N5011 0x1001 +#define PCIE_DEVICE_ID_INTEL_DFL 0xbcce +/* PCI Subdevice ID for PCIE_DEVICE_ID_INTEL_DFL */ +#define PCIE_SUBDEVICE_ID_INTEL_N6000 0x1770 +#define PCIE_SUBDEVICE_ID_INTEL_N6001 0x1771 +#define PCIE_SUBDEVICE_ID_INTEL_C6100 0x17d4 /* VF Device */ #define PCIE_DEVICE_ID_VF_INT_5_X 0xBCBF #define PCIE_DEVICE_ID_VF_INT_6_X 0xBCC1 #define PCIE_DEVICE_ID_VF_DSC_1_X 0x09C5 #define PCIE_DEVICE_ID_INTEL_PAC_D5005_VF 0x0B2C +#define PCIE_DEVICE_ID_INTEL_DFL_VF 0xbccf static struct pci_device_id cci_pcie_id_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_5_X),}, @@ -96,6 +102,18 @@ static struct pci_device_id cci_pcie_id_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_PAC_D5005_VF),}, {PCI_DEVICE(PCI_VENDOR_ID_SILICOM_DENMARK, PCIE_DEVICE_ID_SILICOM_PAC_N5010),}, {PCI_DEVICE(PCI_VENDOR_ID_SILICOM_DENMARK, PCIE_DEVICE_ID_SILICOM_PAC_N5011),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_N6000),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL_VF, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_N6000),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_N6001),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL_VF, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_N6001),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_C6100),}, + {PCI_DEVICE_SUB(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_INTEL_DFL_VF, + PCI_VENDOR_ID_INTEL, PCIE_SUBDEVICE_ID_INTEL_C6100),}, {0,} }; MODULE_DEVICE_TABLE(pci, cci_pcie_id_tbl); diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index 5498bc337f8b2425afcb68f0b7d7cb47e21d47a6..b9aae85ba9308ea1969e69300ebaea1b27fabdc1 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -1866,7 +1866,7 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, return -EINVAL; fds = memdup_user((void __user *)(arg + sizeof(hdr)), - hdr.count * sizeof(s32)); + array_size(hdr.count, sizeof(s32))); if (IS_ERR(fds)) return PTR_ERR(fds); diff --git a/drivers/fpga/intel-m10-bmc-sec-update.c b/drivers/fpga/intel-m10-bmc-sec-update.c index 72c677c910de3bda260d18248b10695169860bd3..79d48852825ef47b35d2400cd1b91c874f6a5677 100644 --- a/drivers/fpga/intel-m10-bmc-sec-update.c +++ b/drivers/fpga/intel-m10-bmc-sec-update.c @@ -148,10 +148,6 @@ static ssize_t flash_count_show(struct device *dev, stride = regmap_get_reg_stride(sec->m10bmc->regmap); num_bits = FLASH_COUNT_SIZE * 8; - flash_buf = kmalloc(FLASH_COUNT_SIZE, GFP_KERNEL); - if (!flash_buf) - return -ENOMEM; - if (FLASH_COUNT_SIZE % stride) { dev_err(sec->dev, "FLASH_COUNT_SIZE (0x%x) not aligned to stride (0x%x)\n", @@ -160,6 +156,10 @@ static ssize_t flash_count_show(struct device *dev, return -EINVAL; } + flash_buf = kmalloc(FLASH_COUNT_SIZE, GFP_KERNEL); + if (!flash_buf) + return -ENOMEM; + ret = regmap_bulk_read(sec->m10bmc->regmap, STAGING_FLASH_COUNT, flash_buf, FLASH_COUNT_SIZE / stride); if (ret) { @@ -605,6 +605,9 @@ static const struct platform_device_id intel_m10bmc_sec_ids[] = { { .name = "n3000bmc-sec-update", }, + { + .name = "d5005bmc-sec-update", + }, { } }; MODULE_DEVICE_TABLE(platform, intel_m10bmc_sec_ids); diff --git a/drivers/fpga/microchip-spi.c b/drivers/fpga/microchip-spi.c index bd284c7b8dc93a589b388dc154c3a1d2ed380158..7436976ea9048a50f5836279bde6cc8d0a5f870b 100644 --- a/drivers/fpga/microchip-spi.c +++ b/drivers/fpga/microchip-spi.c @@ -395,4 +395,5 @@ static struct spi_driver mpf_driver = { module_spi_driver(mpf_driver); MODULE_DESCRIPTION("Microchip Polarfire SPI FPGA Manager"); +MODULE_AUTHOR("Ivan Bornyakov "); MODULE_LICENSE("GPL"); diff --git a/drivers/fsi/fsi-core.c b/drivers/fsi/fsi-core.c index 3a7b78e3670118d32bacc5cce1454a65177ba864..694e80c06665dfa2d2ba7d91ed23cd36849a770c 100644 --- a/drivers/fsi/fsi-core.c +++ b/drivers/fsi/fsi-core.c @@ -392,8 +392,8 @@ int fsi_slave_write(struct fsi_slave *slave, uint32_t addr, } EXPORT_SYMBOL_GPL(fsi_slave_write); -extern int fsi_slave_claim_range(struct fsi_slave *slave, - uint32_t addr, uint32_t size) +int fsi_slave_claim_range(struct fsi_slave *slave, + uint32_t addr, uint32_t size) { if (addr + size < addr) return -EINVAL; @@ -406,8 +406,8 @@ extern int fsi_slave_claim_range(struct fsi_slave *slave, } EXPORT_SYMBOL_GPL(fsi_slave_claim_range); -extern void fsi_slave_release_range(struct fsi_slave *slave, - uint32_t addr, uint32_t size) +void fsi_slave_release_range(struct fsi_slave *slave, + uint32_t addr, uint32_t size) { } EXPORT_SYMBOL_GPL(fsi_slave_release_range); @@ -1314,6 +1314,9 @@ int fsi_master_register(struct fsi_master *master) mutex_init(&master->scan_lock); master->idx = ida_simple_get(&master_ida, 0, INT_MAX, GFP_KERNEL); + if (master->idx < 0) + return master->idx; + dev_set_name(&master->dev, "fsi%d", master->idx); master->dev.class = &fsi_master_class; diff --git a/drivers/fsi/fsi-master-ast-cf.c b/drivers/fsi/fsi-master-ast-cf.c index 24292acdbaf84d5421dd4555e942dbee01307a00..5f608ef8b53cadb76a2a737d0fb5829915600a04 100644 --- a/drivers/fsi/fsi-master-ast-cf.c +++ b/drivers/fsi/fsi-master-ast-cf.c @@ -1324,12 +1324,14 @@ static int fsi_master_acf_probe(struct platform_device *pdev) } master->cvic = devm_of_iomap(&pdev->dev, np, 0, NULL); if (IS_ERR(master->cvic)) { + of_node_put(np); rc = PTR_ERR(master->cvic); dev_err(&pdev->dev, "Error %d mapping CVIC\n", rc); goto err_free; } rc = of_property_read_u32(np, "copro-sw-interrupts", &master->cvic_sw_irq); + of_node_put(np); if (rc) { dev_err(&pdev->dev, "Can't find coprocessor SW interrupt\n"); goto err_free; diff --git a/drivers/fsi/fsi-master.h b/drivers/fsi/fsi-master.h index cd6bee5e12a712639d4466647cea5cad65a23f43..4762315a46ba36078ba5f627b632bf7d28f5a373 100644 --- a/drivers/fsi/fsi-master.h +++ b/drivers/fsi/fsi-master.h @@ -51,7 +51,7 @@ #define FSI_MMODE_CRS1SHFT 8 /* Clk rate selection 1 shift */ #define FSI_MMODE_CRS1MASK 0x3ff /* Clk rate selection 1 mask */ -/* MRESB: Reset brindge */ +/* MRESB: Reset bridge */ #define FSI_MRESB_RST_GEN 0x80000000 /* General reset */ #define FSI_MRESB_RST_ERR 0x40000000 /* Error Reset */ diff --git a/drivers/fsi/fsi-occ.c b/drivers/fsi/fsi-occ.c index c9cc75fbdfb9d4aef75f1137a75ccfdd8c620000..abdd37d5507fefb187a2367ca07d34307c986cac 100644 --- a/drivers/fsi/fsi-occ.c +++ b/drivers/fsi/fsi-occ.c @@ -44,6 +44,7 @@ struct occ { struct device *sbefifo; char name[32]; int idx; + bool platform_hwmon; u8 sequence_number; void *buffer; void *client_buffer; @@ -94,6 +95,7 @@ static int occ_open(struct inode *inode, struct file *file) client->occ = occ; mutex_init(&client->lock); file->private_data = client; + get_device(occ->dev); /* We allocate a 1-page buffer, make sure it all fits */ BUILD_BUG_ON((OCC_CMD_DATA_BYTES + 3) > PAGE_SIZE); @@ -197,6 +199,7 @@ static int occ_release(struct inode *inode, struct file *file) { struct occ_client *client = file->private_data; + put_device(client->occ->dev); free_page((unsigned long)client->buffer); kfree(client); @@ -246,7 +249,7 @@ static int occ_verify_checksum(struct occ *occ, struct occ_response *resp, if (checksum != checksum_resp) { dev_err(occ->dev, "Bad checksum: %04x!=%04x\n", checksum, checksum_resp); - return -EBADMSG; + return -EBADE; } return 0; @@ -493,12 +496,19 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len, for (i = 1; i < req_len - 2; ++i) checksum += byte_request[i]; - mutex_lock(&occ->occ_lock); + rc = mutex_lock_interruptible(&occ->occ_lock); + if (rc) + return rc; occ->client_buffer = response; occ->client_buffer_size = user_resp_len; occ->client_response_size = 0; + if (!occ->buffer) { + rc = -ENOENT; + goto done; + } + /* * Get a sequence number and update the counter. Avoid a sequence * number of 0 which would pass the response check below even if the @@ -575,8 +585,11 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len, dev_dbg(dev, "resp_status=%02x resp_data_len=%d\n", resp->return_status, resp_data_length); - occ->client_response_size = resp_data_length + 7; rc = occ_verify_checksum(occ, resp, resp_data_length); + if (rc) + goto done; + + occ->client_response_size = resp_data_length + 7; done: *resp_len = occ->client_response_size; @@ -586,7 +599,7 @@ int fsi_occ_submit(struct device *dev, const void *request, size_t req_len, } EXPORT_SYMBOL_GPL(fsi_occ_submit); -static int occ_unregister_child(struct device *dev, void *data) +static int occ_unregister_platform_child(struct device *dev, void *data) { struct platform_device *hwmon_dev = to_platform_device(dev); @@ -595,12 +608,25 @@ static int occ_unregister_child(struct device *dev, void *data) return 0; } +static int occ_unregister_of_child(struct device *dev, void *data) +{ + struct platform_device *hwmon_dev = to_platform_device(dev); + + of_device_unregister(hwmon_dev); + if (dev->of_node) + of_node_clear_flag(dev->of_node, OF_POPULATED); + + return 0; +} + static int occ_probe(struct platform_device *pdev) { int rc; u32 reg; + char child_name[32]; struct occ *occ; - struct platform_device *hwmon_dev; + struct platform_device *hwmon_dev = NULL; + struct device_node *hwmon_node; struct device *dev = &pdev->dev; struct platform_device_info hwmon_dev_info = { .parent = dev, @@ -659,10 +685,20 @@ static int occ_probe(struct platform_device *pdev) return rc; } - hwmon_dev_info.id = occ->idx; - hwmon_dev = platform_device_register_full(&hwmon_dev_info); - if (IS_ERR(hwmon_dev)) - dev_warn(dev, "failed to create hwmon device\n"); + hwmon_node = of_get_child_by_name(dev->of_node, hwmon_dev_info.name); + if (hwmon_node) { + snprintf(child_name, sizeof(child_name), "%s.%d", hwmon_dev_info.name, occ->idx); + hwmon_dev = of_platform_device_create(hwmon_node, child_name, dev); + of_node_put(hwmon_node); + } + + if (!hwmon_dev) { + occ->platform_hwmon = true; + hwmon_dev_info.id = occ->idx; + hwmon_dev = platform_device_register_full(&hwmon_dev_info); + if (IS_ERR(hwmon_dev)) + dev_warn(dev, "failed to create hwmon device\n"); + } return 0; } @@ -671,11 +707,17 @@ static int occ_remove(struct platform_device *pdev) { struct occ *occ = platform_get_drvdata(pdev); - kvfree(occ->buffer); - misc_deregister(&occ->mdev); - device_for_each_child(&pdev->dev, NULL, occ_unregister_child); + mutex_lock(&occ->occ_lock); + kvfree(occ->buffer); + occ->buffer = NULL; + mutex_unlock(&occ->occ_lock); + + if (occ->platform_hwmon) + device_for_each_child(&pdev->dev, NULL, occ_unregister_platform_child); + else + device_for_each_child(&pdev->dev, NULL, occ_unregister_of_child); ida_simple_remove(&occ_ida, occ->idx); diff --git a/drivers/fsi/fsi-sbefifo.c b/drivers/fsi/fsi-sbefifo.c index f52a912cdf16bc0c6eed48f6df06681351f6ef73..5f93a53846aad78cc5149429c449dba2af638192 100644 --- a/drivers/fsi/fsi-sbefifo.c +++ b/drivers/fsi/fsi-sbefifo.c @@ -477,7 +477,8 @@ static int sbefifo_wait(struct sbefifo *sbefifo, bool up, if (!ready) { sysfs_notify(&sbefifo->dev.kobj, NULL, dev_attr_timeout.attr.name); sbefifo->timed_out = true; - dev_err(dev, "%s FIFO Timeout ! status=%08x\n", up ? "UP" : "DOWN", sts); + dev_err(dev, "%s FIFO Timeout (%u ms)! status=%08x\n", + up ? "UP" : "DOWN", jiffies_to_msecs(timeout), sts); return -ETIMEDOUT; } dev_vdbg(dev, "End of wait status: %08x\n", sts); @@ -497,8 +498,8 @@ static int sbefifo_send_command(struct sbefifo *sbefifo, u32 status; int rc; - dev_vdbg(dev, "sending command (%zd words, cmd=%04x)\n", - cmd_len, be32_to_cpu(command[1])); + dev_dbg(dev, "sending command (%zd words, cmd=%04x)\n", + cmd_len, be32_to_cpu(command[1])); /* As long as there's something to send */ timeout = msecs_to_jiffies(SBEFIFO_TIMEOUT_START_CMD); @@ -551,21 +552,23 @@ static int sbefifo_read_response(struct sbefifo *sbefifo, struct iov_iter *respo size_t len; int rc; - dev_vdbg(dev, "reading response, buflen = %zd\n", iov_iter_count(response)); + dev_dbg(dev, "reading response, buflen = %zd\n", iov_iter_count(response)); timeout = msecs_to_jiffies(sbefifo->timeout_start_rsp_ms); for (;;) { /* Grab FIFO status (this will handle parity errors) */ rc = sbefifo_wait(sbefifo, false, &status, timeout); - if (rc < 0) + if (rc < 0) { + dev_dbg(dev, "timeout waiting (%u ms)\n", jiffies_to_msecs(timeout)); return rc; + } timeout = msecs_to_jiffies(SBEFIFO_TIMEOUT_IN_RSP); /* Decode status */ len = sbefifo_populated(status); eot_set = sbefifo_eot_set(status); - dev_vdbg(dev, " chunk size %zd eot_set=0x%x\n", len, eot_set); + dev_dbg(dev, " chunk size %zd eot_set=0x%x\n", len, eot_set); /* Go through the chunk */ while(len--) { diff --git a/drivers/gnss/core.c b/drivers/gnss/core.c index e6f94501cb28c29cd9f6cd7fa95fa8feb8c62dfa..1e82b7967570a8f38b2411e76686ea93b4c84d40 100644 --- a/drivers/gnss/core.c +++ b/drivers/gnss/core.c @@ -217,7 +217,7 @@ static void gnss_device_release(struct device *dev) kfree(gdev->write_buf); kfifo_free(&gdev->read_fifo); - ida_simple_remove(&gnss_minors, gdev->id); + ida_free(&gnss_minors, gdev->id); kfree(gdev); } @@ -232,7 +232,7 @@ struct gnss_device *gnss_allocate_device(struct device *parent) if (!gdev) return NULL; - id = ida_simple_get(&gnss_minors, 0, GNSS_MINORS, GFP_KERNEL); + id = ida_alloc_max(&gnss_minors, GNSS_MINORS - 1, GFP_KERNEL); if (id < 0) { kfree(gdev); return NULL; diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 0642f579196f241b01676a715a953775252afffc..a01af11806164c42680e2eb2a917fecca9a9f6f5 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -341,6 +341,10 @@ config GPIO_ICH If unsure, say N. +config GPIO_IMX_SCU + def_bool y + depends on IMX_SCU + config GPIO_IOP tristate "Intel IOP GPIO" depends on ARCH_IOP32X || COMPILE_TEST @@ -874,10 +878,11 @@ config GPIO_104_IDI_48 module parameter. config GPIO_F7188X - tristate "F71869, F71869A, F71882FG, F71889F and F81866 GPIO support" + tristate "Fintek and Nuvoton Super-I/O GPIO support" help This option enables support for GPIOs found on Fintek Super-I/O chips F71869, F71869A, F71882FG, F71889F and F81866. + As well as Nuvoton Super-I/O chip NCT6116D. To compile this driver as a module, choose M here: the module will be called f7188x-gpio. @@ -985,20 +990,6 @@ endmenu menu "I2C GPIO expanders" depends on I2C -config GPIO_ADP5588 - tristate "ADP5588 I2C GPIO expander" - help - This option enables support for 18 GPIOs found - on Analog Devices ADP5588 GPIO Expanders. - -config GPIO_ADP5588_IRQ - bool "Interrupt controller support for ADP5588" - depends on GPIO_ADP5588=y - select GPIOLIB_IRQCHIP - help - Say yes here to enable the adp5588 to be used as an interrupt - controller. It requires the driver to be built in the kernel. - config GPIO_ADNP tristate "Avionic Design N-bit GPIO expander" depends on OF_GPIO diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index a0985d30f51bb1f1945e2c4ecbfb281202aae204..29e3beb6548cb0361979722158179c884b4bab5e 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -25,7 +25,6 @@ obj-$(CONFIG_GPIO_74X164) += gpio-74x164.o obj-$(CONFIG_GPIO_74XX_MMIO) += gpio-74xx-mmio.o obj-$(CONFIG_GPIO_ADNP) += gpio-adnp.o obj-$(CONFIG_GPIO_ADP5520) += gpio-adp5520.o -obj-$(CONFIG_GPIO_ADP5588) += gpio-adp5588.o obj-$(CONFIG_GPIO_AGGREGATOR) += gpio-aggregator.o obj-$(CONFIG_GPIO_ALTERA_A10SR) += gpio-altera-a10sr.o obj-$(CONFIG_GPIO_ALTERA) += gpio-altera.o @@ -70,6 +69,7 @@ obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o obj-$(CONFIG_GPIO_I8255) += gpio-i8255.o obj-$(CONFIG_GPIO_ICH) += gpio-ich.o obj-$(CONFIG_GPIO_IDT3243X) += gpio-idt3243x.o +obj-$(CONFIG_GPIO_IMX_SCU) += gpio-imx-scu.o obj-$(CONFIG_GPIO_IOP) += gpio-iop.o obj-$(CONFIG_GPIO_IT87) += gpio-it87.o obj-$(CONFIG_GPIO_IXP4XX) += gpio-ixp4xx.o diff --git a/drivers/gpio/gpio-104-dio-48e.c b/drivers/gpio/gpio-104-dio-48e.c index a415518707596a9a11f38c352dd253e3c08ac188..7b8829c8e4231313ef98a9310e55c69b55c9ec92 100644 --- a/drivers/gpio/gpio-104-dio-48e.c +++ b/drivers/gpio/gpio-104-dio-48e.c @@ -34,7 +34,8 @@ module_param_hw_array(base, uint, ioport, &num_dio48e, 0); MODULE_PARM_DESC(base, "ACCES 104-DIO-48E base addresses"); static unsigned int irq[MAX_NUM_DIO48E]; -module_param_hw_array(irq, uint, irq, NULL, 0); +static unsigned int num_irq; +module_param_hw_array(irq, uint, irq, &num_irq, 0); MODULE_PARM_DESC(irq, "ACCES 104-DIO-48E interrupt line numbers"); #define DIO48E_NUM_PPI 2 @@ -164,6 +165,7 @@ static void dio48e_irq_mask(struct irq_data *data) dio48egpio->irq_mask &= ~BIT(0); else dio48egpio->irq_mask &= ~BIT(1); + gpiochip_disable_irq(chip, offset); if (!dio48egpio->irq_mask) /* disable interrupts */ @@ -191,6 +193,7 @@ static void dio48e_irq_unmask(struct irq_data *data) iowrite8(0x00, &dio48egpio->reg->enable_interrupt); } + gpiochip_enable_irq(chip, offset); if (offset == 19) dio48egpio->irq_mask |= BIT(0); else @@ -213,12 +216,14 @@ static int dio48e_irq_set_type(struct irq_data *data, unsigned int flow_type) return 0; } -static struct irq_chip dio48e_irqchip = { +static const struct irq_chip dio48e_irqchip = { .name = "104-dio-48e", .irq_ack = dio48e_irq_ack, .irq_mask = dio48e_irq_mask, .irq_unmask = dio48e_irq_unmask, - .irq_set_type = dio48e_irq_set_type + .irq_set_type = dio48e_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static irqreturn_t dio48e_irq_handler(int irq, void *dev_id) @@ -322,7 +327,7 @@ static int dio48e_probe(struct device *dev, unsigned int id) dio48egpio->chip.set_multiple = dio48e_gpio_set_multiple; girq = &dio48egpio->chip.irq; - girq->chip = &dio48e_irqchip; + gpio_irq_chip_set_chip(girq, &dio48e_irqchip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; @@ -358,7 +363,7 @@ static struct isa_driver dio48e_driver = { .name = "104-dio-48e" }, }; -module_isa_driver(dio48e_driver, num_dio48e); +module_isa_driver_with_irq(dio48e_driver, num_dio48e, num_irq); MODULE_AUTHOR("William Breathitt Gray "); MODULE_DESCRIPTION("ACCES 104-DIO-48E GPIO driver"); diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c index 40be76efeed78601ec6c04e0666311019fcbc0fe..c5e231fde1affa3377864cbd8cead383068c4a06 100644 --- a/drivers/gpio/gpio-104-idi-48.c +++ b/drivers/gpio/gpio-104-idi-48.c @@ -34,7 +34,8 @@ module_param_hw_array(base, uint, ioport, &num_idi_48, 0); MODULE_PARM_DESC(base, "ACCES 104-IDI-48 base addresses"); static unsigned int irq[MAX_NUM_IDI_48]; -module_param_hw_array(irq, uint, irq, NULL, 0); +static unsigned int num_irq; +module_param_hw_array(irq, uint, irq, &num_irq, 0); MODULE_PARM_DESC(irq, "ACCES 104-IDI-48 interrupt line numbers"); /** @@ -113,6 +114,7 @@ static void idi_48_irq_mask(struct irq_data *data) spin_lock_irqsave(&idi48gpio->lock, flags); idi48gpio->irq_mask[boundary] &= ~mask; + gpiochip_disable_irq(chip, offset); /* Exit early if there are still input lines with IRQ unmasked */ if (idi48gpio->irq_mask[boundary]) @@ -140,6 +142,7 @@ static void idi_48_irq_unmask(struct irq_data *data) prev_irq_mask = idi48gpio->irq_mask[boundary]; + gpiochip_enable_irq(chip, offset); idi48gpio->irq_mask[boundary] |= mask; /* Exit early if IRQ was already unmasked for this boundary */ @@ -164,12 +167,14 @@ static int idi_48_irq_set_type(struct irq_data *data, unsigned int flow_type) return 0; } -static struct irq_chip idi_48_irqchip = { +static const struct irq_chip idi_48_irqchip = { .name = "104-idi-48", .irq_ack = idi_48_irq_ack, .irq_mask = idi_48_irq_mask, .irq_unmask = idi_48_irq_unmask, - .irq_set_type = idi_48_irq_set_type + .irq_set_type = idi_48_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static irqreturn_t idi_48_irq_handler(int irq, void *dev_id) @@ -267,7 +272,7 @@ static int idi_48_probe(struct device *dev, unsigned int id) idi48gpio->chip.get_multiple = idi_48_gpio_get_multiple; girq = &idi48gpio->chip.irq; - girq->chip = &idi_48_irqchip; + gpio_irq_chip_set_chip(girq, &idi_48_irqchip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; @@ -300,7 +305,7 @@ static struct isa_driver idi_48_driver = { .name = "104-idi-48" }, }; -module_isa_driver(idi_48_driver, num_idi_48); +module_isa_driver_with_irq(idi_48_driver, num_idi_48, num_irq); MODULE_AUTHOR("William Breathitt Gray "); MODULE_DESCRIPTION("ACCES 104-IDI-48 GPIO driver"); diff --git a/drivers/gpio/gpio-104-idio-16.c b/drivers/gpio/gpio-104-idio-16.c index 65a5f581d981ca111c8053446f4e2c80bee34621..718bd54e2a25511da21604e0b3942d98fda278b0 100644 --- a/drivers/gpio/gpio-104-idio-16.c +++ b/drivers/gpio/gpio-104-idio-16.c @@ -30,7 +30,8 @@ module_param_hw_array(base, uint, ioport, &num_idio_16, 0); MODULE_PARM_DESC(base, "ACCES 104-IDIO-16 base addresses"); static unsigned int irq[MAX_NUM_IDIO_16]; -module_param_hw_array(irq, uint, irq, NULL, 0); +static unsigned int num_irq; +module_param_hw_array(irq, uint, irq, &num_irq, 0); MODULE_PARM_DESC(irq, "ACCES 104-IDIO-16 interrupt line numbers"); /** @@ -174,10 +175,11 @@ static void idio_16_irq_mask(struct irq_data *data) { struct gpio_chip *chip = irq_data_get_irq_chip_data(data); struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip); - const unsigned long mask = BIT(irqd_to_hwirq(data)); + const unsigned long offset = irqd_to_hwirq(data); unsigned long flags; - idio16gpio->irq_mask &= ~mask; + idio16gpio->irq_mask &= ~BIT(offset); + gpiochip_disable_irq(chip, offset); if (!idio16gpio->irq_mask) { raw_spin_lock_irqsave(&idio16gpio->lock, flags); @@ -192,11 +194,12 @@ static void idio_16_irq_unmask(struct irq_data *data) { struct gpio_chip *chip = irq_data_get_irq_chip_data(data); struct idio_16_gpio *const idio16gpio = gpiochip_get_data(chip); - const unsigned long mask = BIT(irqd_to_hwirq(data)); + const unsigned long offset = irqd_to_hwirq(data); const unsigned long prev_irq_mask = idio16gpio->irq_mask; unsigned long flags; - idio16gpio->irq_mask |= mask; + gpiochip_enable_irq(chip, offset); + idio16gpio->irq_mask |= BIT(offset); if (!prev_irq_mask) { raw_spin_lock_irqsave(&idio16gpio->lock, flags); @@ -217,12 +220,14 @@ static int idio_16_irq_set_type(struct irq_data *data, unsigned int flow_type) return 0; } -static struct irq_chip idio_16_irqchip = { +static const struct irq_chip idio_16_irqchip = { .name = "104-idio-16", .irq_ack = idio_16_irq_ack, .irq_mask = idio_16_irq_mask, .irq_unmask = idio_16_irq_unmask, - .irq_set_type = idio_16_irq_set_type + .irq_set_type = idio_16_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static irqreturn_t idio_16_irq_handler(int irq, void *dev_id) @@ -299,7 +304,7 @@ static int idio_16_probe(struct device *dev, unsigned int id) idio16gpio->out_state = 0xFFFF; girq = &idio16gpio->chip.irq; - girq->chip = &idio_16_irqchip; + gpio_irq_chip_set_chip(girq, &idio_16_irqchip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; @@ -333,7 +338,7 @@ static struct isa_driver idio_16_driver = { }, }; -module_isa_driver(idio_16_driver, num_idio_16); +module_isa_driver_with_irq(idio_16_driver, num_idio_16, num_irq); MODULE_AUTHOR("William Breathitt Gray "); MODULE_DESCRIPTION("ACCES 104-IDIO-16 GPIO driver"); diff --git a/drivers/gpio/gpio-adp5588.c b/drivers/gpio/gpio-adp5588.c deleted file mode 100644 index d49f12560cde2db4ada4ad8a4783c923af395fe0..0000000000000000000000000000000000000000 --- a/drivers/gpio/gpio-adp5588.c +++ /dev/null @@ -1,448 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * GPIO Chip driver for Analog Devices - * ADP5588/ADP5587 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2009-2010 Analog Devices Inc. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -/* - * Early pre 4.0 Silicon required to delay readout by at least 25ms, - * since the Event Counter Register updated 25ms after the interrupt - * asserted. - */ -#define WA_DELAYED_READOUT_REVID(rev) ((rev) < 4) - -struct adp5588_gpio { - struct i2c_client *client; - struct gpio_chip gpio_chip; - struct mutex lock; /* protect cached dir, dat_out */ - /* protect serialized access to the interrupt controller bus */ - struct mutex irq_lock; - uint8_t dat_out[3]; - uint8_t dir[3]; - uint8_t int_lvl_low[3]; - uint8_t int_lvl_high[3]; - uint8_t int_en[3]; - uint8_t irq_mask[3]; - uint8_t int_input_en[3]; -}; - -static int adp5588_gpio_read(struct i2c_client *client, u8 reg) -{ - int ret = i2c_smbus_read_byte_data(client, reg); - - if (ret < 0) - dev_err(&client->dev, "Read Error\n"); - - return ret; -} - -static int adp5588_gpio_write(struct i2c_client *client, u8 reg, u8 val) -{ - int ret = i2c_smbus_write_byte_data(client, reg, val); - - if (ret < 0) - dev_err(&client->dev, "Write Error\n"); - - return ret; -} - -static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off) -{ - struct adp5588_gpio *dev = gpiochip_get_data(chip); - unsigned bank = ADP5588_BANK(off); - unsigned bit = ADP5588_BIT(off); - int val; - - mutex_lock(&dev->lock); - - if (dev->dir[bank] & bit) - val = dev->dat_out[bank]; - else - val = adp5588_gpio_read(dev->client, GPIO_DAT_STAT1 + bank); - - mutex_unlock(&dev->lock); - - return !!(val & bit); -} - -static void adp5588_gpio_set_value(struct gpio_chip *chip, - unsigned off, int val) -{ - unsigned bank, bit; - struct adp5588_gpio *dev = gpiochip_get_data(chip); - - bank = ADP5588_BANK(off); - bit = ADP5588_BIT(off); - - mutex_lock(&dev->lock); - if (val) - dev->dat_out[bank] |= bit; - else - dev->dat_out[bank] &= ~bit; - - adp5588_gpio_write(dev->client, GPIO_DAT_OUT1 + bank, - dev->dat_out[bank]); - mutex_unlock(&dev->lock); -} - -static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off) -{ - int ret; - unsigned bank; - struct adp5588_gpio *dev = gpiochip_get_data(chip); - - bank = ADP5588_BANK(off); - - mutex_lock(&dev->lock); - dev->dir[bank] &= ~ADP5588_BIT(off); - ret = adp5588_gpio_write(dev->client, GPIO_DIR1 + bank, dev->dir[bank]); - mutex_unlock(&dev->lock); - - return ret; -} - -static int adp5588_gpio_direction_output(struct gpio_chip *chip, - unsigned off, int val) -{ - int ret; - unsigned bank, bit; - struct adp5588_gpio *dev = gpiochip_get_data(chip); - - bank = ADP5588_BANK(off); - bit = ADP5588_BIT(off); - - mutex_lock(&dev->lock); - dev->dir[bank] |= bit; - - if (val) - dev->dat_out[bank] |= bit; - else - dev->dat_out[bank] &= ~bit; - - ret = adp5588_gpio_write(dev->client, GPIO_DAT_OUT1 + bank, - dev->dat_out[bank]); - ret |= adp5588_gpio_write(dev->client, GPIO_DIR1 + bank, - dev->dir[bank]); - mutex_unlock(&dev->lock); - - return ret; -} - -#ifdef CONFIG_GPIO_ADP5588_IRQ - -static void adp5588_irq_bus_lock(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct adp5588_gpio *dev = gpiochip_get_data(gc); - - mutex_lock(&dev->irq_lock); -} - - /* - * genirq core code can issue chip->mask/unmask from atomic context. - * This doesn't work for slow busses where an access needs to sleep. - * bus_sync_unlock() is therefore called outside the atomic context, - * syncs the current irq mask state with the slow external controller - * and unlocks the bus. - */ - -static void adp5588_irq_bus_sync_unlock(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct adp5588_gpio *dev = gpiochip_get_data(gc); - int i; - - for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) { - if (dev->int_input_en[i]) { - mutex_lock(&dev->lock); - dev->dir[i] &= ~dev->int_input_en[i]; - dev->int_input_en[i] = 0; - adp5588_gpio_write(dev->client, GPIO_DIR1 + i, - dev->dir[i]); - mutex_unlock(&dev->lock); - } - - if (dev->int_en[i] ^ dev->irq_mask[i]) { - dev->int_en[i] = dev->irq_mask[i]; - adp5588_gpio_write(dev->client, GPI_EM1 + i, - dev->int_en[i]); - } - } - - mutex_unlock(&dev->irq_lock); -} - -static void adp5588_irq_mask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct adp5588_gpio *dev = gpiochip_get_data(gc); - - dev->irq_mask[ADP5588_BANK(d->hwirq)] &= ~ADP5588_BIT(d->hwirq); -} - -static void adp5588_irq_unmask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct adp5588_gpio *dev = gpiochip_get_data(gc); - - dev->irq_mask[ADP5588_BANK(d->hwirq)] |= ADP5588_BIT(d->hwirq); -} - -static int adp5588_irq_set_type(struct irq_data *d, unsigned int type) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct adp5588_gpio *dev = gpiochip_get_data(gc); - uint16_t gpio = d->hwirq; - unsigned bank, bit; - - bank = ADP5588_BANK(gpio); - bit = ADP5588_BIT(gpio); - - dev->int_lvl_low[bank] &= ~bit; - dev->int_lvl_high[bank] &= ~bit; - - if (type & IRQ_TYPE_EDGE_BOTH || type & IRQ_TYPE_LEVEL_HIGH) - dev->int_lvl_high[bank] |= bit; - - if (type & IRQ_TYPE_EDGE_BOTH || type & IRQ_TYPE_LEVEL_LOW) - dev->int_lvl_low[bank] |= bit; - - dev->int_input_en[bank] |= bit; - - return 0; -} - -static struct irq_chip adp5588_irq_chip = { - .name = "adp5588", - .irq_mask = adp5588_irq_mask, - .irq_unmask = adp5588_irq_unmask, - .irq_bus_lock = adp5588_irq_bus_lock, - .irq_bus_sync_unlock = adp5588_irq_bus_sync_unlock, - .irq_set_type = adp5588_irq_set_type, -}; - -static irqreturn_t adp5588_irq_handler(int irq, void *devid) -{ - struct adp5588_gpio *dev = devid; - int status = adp5588_gpio_read(dev->client, INT_STAT); - - if (status & ADP5588_KE_INT) { - int ev_cnt = adp5588_gpio_read(dev->client, KEY_LCK_EC_STAT); - - if (ev_cnt > 0) { - int i; - - for (i = 0; i < (ev_cnt & ADP5588_KEC); i++) { - int key = adp5588_gpio_read(dev->client, - Key_EVENTA + i); - /* GPIN events begin at 97, - * bit 7 indicates logic level - */ - int gpio = (key & 0x7f) - 97; - int lvl = key & (1 << 7); - int bank = ADP5588_BANK(gpio); - int bit = ADP5588_BIT(gpio); - - if ((lvl && dev->int_lvl_high[bank] & bit) || - (!lvl && dev->int_lvl_low[bank] & bit)) - handle_nested_irq(irq_find_mapping( - dev->gpio_chip.irq.domain, gpio)); - } - } - } - - adp5588_gpio_write(dev->client, INT_STAT, status); /* Status is W1C */ - - return IRQ_HANDLED; -} - - -static int adp5588_irq_init_hw(struct gpio_chip *gc) -{ - struct adp5588_gpio *dev = gpiochip_get_data(gc); - /* Enable IRQs after registering chip */ - adp5588_gpio_write(dev->client, CFG, - ADP5588_AUTO_INC | ADP5588_INT_CFG | ADP5588_KE_IEN); - - return 0; -} - -static int adp5588_irq_setup(struct adp5588_gpio *dev) -{ - struct i2c_client *client = dev->client; - int ret; - struct adp5588_gpio_platform_data *pdata = - dev_get_platdata(&client->dev); - struct gpio_irq_chip *girq; - - adp5588_gpio_write(client, CFG, ADP5588_AUTO_INC); - adp5588_gpio_write(client, INT_STAT, -1); /* status is W1C */ - - mutex_init(&dev->irq_lock); - - ret = devm_request_threaded_irq(&client->dev, client->irq, - NULL, adp5588_irq_handler, IRQF_ONESHOT - | IRQF_TRIGGER_FALLING | IRQF_SHARED, - dev_name(&client->dev), dev); - if (ret) { - dev_err(&client->dev, "failed to request irq %d\n", - client->irq); - return ret; - } - - /* This will be registered in the call to devm_gpiochip_add_data() */ - girq = &dev->gpio_chip.irq; - girq->chip = &adp5588_irq_chip; - /* This will let us handle the parent IRQ in the driver */ - girq->parent_handler = NULL; - girq->num_parents = 0; - girq->parents = NULL; - girq->first = pdata ? pdata->irq_base : 0; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - girq->init_hw = adp5588_irq_init_hw; - girq->threaded = true; - - return 0; -} - -#else -static int adp5588_irq_setup(struct adp5588_gpio *dev) -{ - struct i2c_client *client = dev->client; - dev_warn(&client->dev, "interrupt support not compiled in\n"); - - return 0; -} - -#endif /* CONFIG_GPIO_ADP5588_IRQ */ - -static int adp5588_gpio_probe(struct i2c_client *client) -{ - struct adp5588_gpio_platform_data *pdata = - dev_get_platdata(&client->dev); - struct adp5588_gpio *dev; - struct gpio_chip *gc; - int ret, i, revid; - unsigned int pullup_dis_mask = 0; - - if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_BYTE_DATA)) { - dev_err(&client->dev, "SMBUS Byte Data not Supported\n"); - return -EIO; - } - - dev = devm_kzalloc(&client->dev, sizeof(*dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - dev->client = client; - - gc = &dev->gpio_chip; - gc->direction_input = adp5588_gpio_direction_input; - gc->direction_output = adp5588_gpio_direction_output; - gc->get = adp5588_gpio_get_value; - gc->set = adp5588_gpio_set_value; - gc->can_sleep = true; - gc->base = -1; - gc->parent = &client->dev; - - if (pdata) { - gc->base = pdata->gpio_start; - gc->names = pdata->names; - pullup_dis_mask = pdata->pullup_dis_mask; - } - - gc->ngpio = ADP5588_MAXGPIO; - gc->label = client->name; - gc->owner = THIS_MODULE; - - mutex_init(&dev->lock); - - ret = adp5588_gpio_read(dev->client, DEV_ID); - if (ret < 0) - return ret; - - revid = ret & ADP5588_DEVICE_ID_MASK; - - for (i = 0, ret = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) { - dev->dat_out[i] = adp5588_gpio_read(client, GPIO_DAT_OUT1 + i); - dev->dir[i] = adp5588_gpio_read(client, GPIO_DIR1 + i); - ret |= adp5588_gpio_write(client, KP_GPIO1 + i, 0); - ret |= adp5588_gpio_write(client, GPIO_PULL1 + i, - (pullup_dis_mask >> (8 * i)) & 0xFF); - ret |= adp5588_gpio_write(client, GPIO_INT_EN1 + i, 0); - if (ret) - return ret; - } - - if (client->irq) { - if (WA_DELAYED_READOUT_REVID(revid)) { - dev_warn(&client->dev, "GPIO int not supported\n"); - } else { - ret = adp5588_irq_setup(dev); - if (ret) - return ret; - } - } - - ret = devm_gpiochip_add_data(&client->dev, &dev->gpio_chip, dev); - if (ret) - return ret; - - i2c_set_clientdata(client, dev); - - return 0; -} - -static int adp5588_gpio_remove(struct i2c_client *client) -{ - struct adp5588_gpio *dev = i2c_get_clientdata(client); - - if (dev->client->irq) - free_irq(dev->client->irq, dev); - - return 0; -} - -static const struct i2c_device_id adp5588_gpio_id[] = { - { "adp5588-gpio" }, - {} -}; -MODULE_DEVICE_TABLE(i2c, adp5588_gpio_id); - -static const struct of_device_id adp5588_gpio_of_id[] = { - { .compatible = "adi,adp5588-gpio" }, - {} -}; -MODULE_DEVICE_TABLE(of, adp5588_gpio_of_id); - -static struct i2c_driver adp5588_gpio_driver = { - .driver = { - .name = "adp5588-gpio", - .of_match_table = adp5588_gpio_of_id, - }, - .probe_new = adp5588_gpio_probe, - .remove = adp5588_gpio_remove, - .id_table = adp5588_gpio_id, -}; - -module_i2c_driver(adp5588_gpio_driver); - -MODULE_AUTHOR("Michael Hennerich "); -MODULE_DESCRIPTION("GPIO ADP5588 Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpio-exar.c b/drivers/gpio/gpio-exar.c index d37de78247a655c4e2e2cee1afee1caa2bc4c2a9..482f678c893e5ebed813508e4e6d4d030be33971 100644 --- a/drivers/gpio/gpio-exar.c +++ b/drivers/gpio/gpio-exar.c @@ -21,6 +21,12 @@ #define EXAR_OFFSET_MPIOLVL_HI 0x96 #define EXAR_OFFSET_MPIOSEL_HI 0x99 +/* + * The Device Configuration and UART Configuration Registers + * for each UART channel take 1KB of memory address space. + */ +#define EXAR_UART_CHANNEL_SIZE 0x400 + #define DRIVER_NAME "gpio_exar" static DEFINE_IDA(ida_index); @@ -31,26 +37,39 @@ struct exar_gpio_chip { int index; char name[20]; unsigned int first_pin; + /* + * The offset to the cascaded device's (if existing) + * Device Configuration Registers. + */ + unsigned int cascaded_offset; }; static unsigned int exar_offset_to_sel_addr(struct exar_gpio_chip *exar_gpio, unsigned int offset) { - return (offset + exar_gpio->first_pin) / 8 ? EXAR_OFFSET_MPIOSEL_HI - : EXAR_OFFSET_MPIOSEL_LO; + unsigned int pin = exar_gpio->first_pin + (offset % 16); + unsigned int cascaded = offset / 16; + unsigned int addr = pin / 8 ? EXAR_OFFSET_MPIOSEL_HI : EXAR_OFFSET_MPIOSEL_LO; + + return addr + (cascaded ? exar_gpio->cascaded_offset : 0); } static unsigned int exar_offset_to_lvl_addr(struct exar_gpio_chip *exar_gpio, unsigned int offset) { - return (offset + exar_gpio->first_pin) / 8 ? EXAR_OFFSET_MPIOLVL_HI - : EXAR_OFFSET_MPIOLVL_LO; + unsigned int pin = exar_gpio->first_pin + (offset % 16); + unsigned int cascaded = offset / 16; + unsigned int addr = pin / 8 ? EXAR_OFFSET_MPIOLVL_HI : EXAR_OFFSET_MPIOLVL_LO; + + return addr + (cascaded ? exar_gpio->cascaded_offset : 0); } static unsigned int exar_offset_to_bit(struct exar_gpio_chip *exar_gpio, unsigned int offset) { - return (offset + exar_gpio->first_pin) % 8; + unsigned int pin = exar_gpio->first_pin + (offset % 16); + + return pin % 8; } static int exar_get_direction(struct gpio_chip *chip, unsigned int offset) @@ -153,6 +172,17 @@ static int gpio_exar_probe(struct platform_device *pdev) if (!exar_gpio) return -ENOMEM; + /* + * If cascaded, secondary xr17v354 or xr17v358 have the same amount + * of MPIOs as their primaries and the last 4 bits of the primary's + * PCI Device ID is the number of its UART channels. + */ + if (pcidev->device & GENMASK(15, 12)) { + ngpios += ngpios; + exar_gpio->cascaded_offset = (pcidev->device & GENMASK(3, 0)) * + EXAR_UART_CHANNEL_SIZE; + } + /* * We don't need to check the return values of mmio regmap operations (unless * the regmap has a clock attached which is not the case here). diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c index 18a3147f5a42a8f6b1c49ddc69bb2ebf2588f2e9..9effa7769bef5bcdea5865b964f78f261da40dfe 100644 --- a/drivers/gpio/gpio-f7188x.c +++ b/drivers/gpio/gpio-f7188x.c @@ -1,12 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * GPIO driver for Fintek Super-I/O F71869, F71869A, F71882, F71889 and F81866 + * GPIO driver for Fintek and Nuvoton Super-I/O chips * * Copyright (C) 2010-2013 LaCie * * Author: Simon Guinot */ +#define DRVNAME "gpio-f7188x" +#define pr_fmt(fmt) DRVNAME ": " fmt + #include #include #include @@ -14,30 +17,41 @@ #include #include -#define DRVNAME "gpio-f7188x" - /* * Super-I/O registers */ #define SIO_LDSEL 0x07 /* Logical device select */ #define SIO_DEVID 0x20 /* Device ID (2 bytes) */ -#define SIO_DEVREV 0x22 /* Device revision */ -#define SIO_MANID 0x23 /* Fintek ID (2 bytes) */ -#define SIO_LD_GPIO 0x06 /* GPIO logical device */ #define SIO_UNLOCK_KEY 0x87 /* Key to enable Super-I/O */ #define SIO_LOCK_KEY 0xAA /* Key to disable Super-I/O */ -#define SIO_FINTEK_ID 0x1934 /* Manufacturer ID */ +/* + * Fintek devices. + */ +#define SIO_FINTEK_DEVREV 0x22 /* Fintek Device revision */ +#define SIO_FINTEK_MANID 0x23 /* Fintek ID (2 bytes) */ + +#define SIO_FINTEK_ID 0x1934 /* Manufacturer ID */ + #define SIO_F71869_ID 0x0814 /* F71869 chipset ID */ #define SIO_F71869A_ID 0x1007 /* F71869A chipset ID */ #define SIO_F71882_ID 0x0541 /* F71882 chipset ID */ #define SIO_F71889_ID 0x0909 /* F71889 chipset ID */ #define SIO_F71889A_ID 0x1005 /* F71889A chipset ID */ #define SIO_F81866_ID 0x1010 /* F81866 chipset ID */ -#define SIO_F81804_ID 0x1502 /* F81804 chipset ID, same for f81966 */ +#define SIO_F81804_ID 0x1502 /* F81804 chipset ID, same for F81966 */ #define SIO_F81865_ID 0x0704 /* F81865 chipset ID */ +#define SIO_LD_GPIO_FINTEK 0x06 /* GPIO logical device */ + +/* + * Nuvoton devices. + */ +#define SIO_NCT6116D_ID 0xD283 /* NCT6116D chipset ID */ + +#define SIO_LD_GPIO_NUVOTON 0x07 /* GPIO logical device */ + enum chips { f71869, @@ -48,6 +62,7 @@ enum chips { f81866, f81804, f81865, + nct6116d, }; static const char * const f7188x_names[] = { @@ -59,10 +74,12 @@ static const char * const f7188x_names[] = { "f81866", "f81804", "f81865", + "nct6116d", }; struct f7188x_sio { int addr; + int device; enum chips type; }; @@ -110,7 +127,7 @@ static inline int superio_enter(int base) { /* Don't step on other drivers' I/O space by accident. */ if (!request_muxed_region(base, 2, DRVNAME)) { - pr_err(DRVNAME "I/O address 0x%04x already in use\n", base); + pr_err("I/O address 0x%04x already in use\n", base); return -EBUSY; } @@ -146,10 +163,10 @@ static void f7188x_gpio_set(struct gpio_chip *chip, unsigned offset, int value); static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset, unsigned long config); -#define F7188X_GPIO_BANK(_base, _ngpio, _regbase) \ +#define F7188X_GPIO_BANK(_base, _ngpio, _regbase, _label) \ { \ .chip = { \ - .label = DRVNAME, \ + .label = _label, \ .owner = THIS_MODULE, \ .get_direction = f7188x_gpio_get_direction, \ .direction_input = f7188x_gpio_direction_in, \ @@ -164,94 +181,108 @@ static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset, .regbase = _regbase, \ } -#define gpio_dir(base) (base + 0) -#define gpio_data_out(base) (base + 1) -#define gpio_data_in(base) (base + 2) +#define f7188x_gpio_dir(base) ((base) + 0) +#define f7188x_gpio_data_out(base) ((base) + 1) +#define f7188x_gpio_data_in(base) ((base) + 2) /* Output mode register (0:open drain 1:push-pull). */ -#define gpio_out_mode(base) (base + 3) +#define f7188x_gpio_out_mode(base) ((base) + 3) + +#define f7188x_gpio_dir_invert(type) ((type) == nct6116d) +#define f7188x_gpio_data_single(type) ((type) == nct6116d) static struct f7188x_gpio_bank f71869_gpio_bank[] = { - F7188X_GPIO_BANK(0, 6, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 5, 0xA0), - F7188X_GPIO_BANK(60, 6, 0x90), + F7188X_GPIO_BANK(0, 6, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 6, 0x90, DRVNAME "-6"), }; static struct f7188x_gpio_bank f71869a_gpio_bank[] = { - F7188X_GPIO_BANK(0, 6, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 5, 0xA0), - F7188X_GPIO_BANK(60, 8, 0x90), - F7188X_GPIO_BANK(70, 8, 0x80), + F7188X_GPIO_BANK(0, 6, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"), + F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"), }; static struct f7188x_gpio_bank f71882_gpio_bank[] = { - F7188X_GPIO_BANK(0, 8, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 4, 0xC0), - F7188X_GPIO_BANK(40, 4, 0xB0), + F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 4, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 4, 0xB0, DRVNAME "-4"), }; static struct f7188x_gpio_bank f71889a_gpio_bank[] = { - F7188X_GPIO_BANK(0, 7, 0xF0), - F7188X_GPIO_BANK(10, 7, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 5, 0xA0), - F7188X_GPIO_BANK(60, 8, 0x90), - F7188X_GPIO_BANK(70, 8, 0x80), + F7188X_GPIO_BANK(0, 7, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 7, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"), + F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"), }; static struct f7188x_gpio_bank f71889_gpio_bank[] = { - F7188X_GPIO_BANK(0, 7, 0xF0), - F7188X_GPIO_BANK(10, 7, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 5, 0xA0), - F7188X_GPIO_BANK(60, 8, 0x90), - F7188X_GPIO_BANK(70, 8, 0x80), + F7188X_GPIO_BANK(0, 7, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 7, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"), + F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"), }; static struct f7188x_gpio_bank f81866_gpio_bank[] = { - F7188X_GPIO_BANK(0, 8, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 8, 0xA0), - F7188X_GPIO_BANK(60, 8, 0x90), - F7188X_GPIO_BANK(70, 8, 0x80), - F7188X_GPIO_BANK(80, 8, 0x88), + F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"), + F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"), + F7188X_GPIO_BANK(80, 8, 0x88, DRVNAME "-8"), }; static struct f7188x_gpio_bank f81804_gpio_bank[] = { - F7188X_GPIO_BANK(0, 8, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(50, 8, 0xA0), - F7188X_GPIO_BANK(60, 8, 0x90), - F7188X_GPIO_BANK(70, 8, 0x80), - F7188X_GPIO_BANK(90, 8, 0x98), + F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-3"), + F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-4"), + F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-5"), + F7188X_GPIO_BANK(90, 8, 0x98, DRVNAME "-6"), }; static struct f7188x_gpio_bank f81865_gpio_bank[] = { - F7188X_GPIO_BANK(0, 8, 0xF0), - F7188X_GPIO_BANK(10, 8, 0xE0), - F7188X_GPIO_BANK(20, 8, 0xD0), - F7188X_GPIO_BANK(30, 8, 0xC0), - F7188X_GPIO_BANK(40, 8, 0xB0), - F7188X_GPIO_BANK(50, 8, 0xA0), - F7188X_GPIO_BANK(60, 5, 0x90), + F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 5, 0x90, DRVNAME "-6"), +}; + +static struct f7188x_gpio_bank nct6116d_gpio_bank[] = { + F7188X_GPIO_BANK(0, 8, 0xE0, DRVNAME "-0"), + F7188X_GPIO_BANK(10, 8, 0xE4, DRVNAME "-1"), + F7188X_GPIO_BANK(20, 8, 0xE8, DRVNAME "-2"), + F7188X_GPIO_BANK(30, 8, 0xEC, DRVNAME "-3"), + F7188X_GPIO_BANK(40, 8, 0xF0, DRVNAME "-4"), + F7188X_GPIO_BANK(50, 8, 0xF4, DRVNAME "-5"), + F7188X_GPIO_BANK(60, 8, 0xF8, DRVNAME "-6"), + F7188X_GPIO_BANK(70, 1, 0xFC, DRVNAME "-7"), }; static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset) @@ -264,13 +295,16 @@ static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset) err = superio_enter(sio->addr); if (err) return err; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); + dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase)); superio_exit(sio->addr); - if (dir & 1 << offset) + if (f7188x_gpio_dir_invert(sio->type)) + dir = ~dir; + + if (dir & BIT(offset)) return GPIO_LINE_DIRECTION_OUT; return GPIO_LINE_DIRECTION_IN; @@ -286,11 +320,15 @@ static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset) err = superio_enter(sio->addr); if (err) return err; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); - dir &= ~BIT(offset); - superio_outb(sio->addr, gpio_dir(bank->regbase), dir); + dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase)); + + if (f7188x_gpio_dir_invert(sio->type)) + dir |= BIT(offset); + else + dir &= ~BIT(offset); + superio_outb(sio->addr, f7188x_gpio_dir(bank->regbase), dir); superio_exit(sio->addr); @@ -307,14 +345,14 @@ static int f7188x_gpio_get(struct gpio_chip *chip, unsigned offset) err = superio_enter(sio->addr); if (err) return err; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); + dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase)); dir = !!(dir & BIT(offset)); - if (dir) - data = superio_inb(sio->addr, gpio_data_out(bank->regbase)); + if (f7188x_gpio_data_single(sio->type) || dir) + data = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase)); else - data = superio_inb(sio->addr, gpio_data_in(bank->regbase)); + data = superio_inb(sio->addr, f7188x_gpio_data_in(bank->regbase)); superio_exit(sio->addr); @@ -332,18 +370,21 @@ static int f7188x_gpio_direction_out(struct gpio_chip *chip, err = superio_enter(sio->addr); if (err) return err; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - data_out = superio_inb(sio->addr, gpio_data_out(bank->regbase)); + data_out = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase)); if (value) data_out |= BIT(offset); else data_out &= ~BIT(offset); - superio_outb(sio->addr, gpio_data_out(bank->regbase), data_out); + superio_outb(sio->addr, f7188x_gpio_data_out(bank->regbase), data_out); - dir = superio_inb(sio->addr, gpio_dir(bank->regbase)); - dir |= BIT(offset); - superio_outb(sio->addr, gpio_dir(bank->regbase), dir); + dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase)); + if (f7188x_gpio_dir_invert(sio->type)) + dir &= ~BIT(offset); + else + dir |= BIT(offset); + superio_outb(sio->addr, f7188x_gpio_dir(bank->regbase), dir); superio_exit(sio->addr); @@ -360,14 +401,14 @@ static void f7188x_gpio_set(struct gpio_chip *chip, unsigned offset, int value) err = superio_enter(sio->addr); if (err) return; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - data_out = superio_inb(sio->addr, gpio_data_out(bank->regbase)); + data_out = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase)); if (value) data_out |= BIT(offset); else data_out &= ~BIT(offset); - superio_outb(sio->addr, gpio_data_out(bank->regbase), data_out); + superio_outb(sio->addr, f7188x_gpio_data_out(bank->regbase), data_out); superio_exit(sio->addr); } @@ -388,14 +429,14 @@ static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset, err = superio_enter(sio->addr); if (err) return err; - superio_select(sio->addr, SIO_LD_GPIO); + superio_select(sio->addr, sio->device); - data = superio_inb(sio->addr, gpio_out_mode(bank->regbase)); + data = superio_inb(sio->addr, f7188x_gpio_out_mode(bank->regbase)); if (param == PIN_CONFIG_DRIVE_OPEN_DRAIN) data &= ~BIT(offset); else data |= BIT(offset); - superio_outb(sio->addr, gpio_out_mode(bank->regbase), data); + superio_outb(sio->addr, f7188x_gpio_out_mode(bank->regbase), data); superio_exit(sio->addr); return 0; @@ -449,6 +490,10 @@ static int f7188x_gpio_probe(struct platform_device *pdev) data->nr_bank = ARRAY_SIZE(f81865_gpio_bank); data->bank = f81865_gpio_bank; break; + case nct6116d: + data->nr_bank = ARRAY_SIZE(nct6116d_gpio_bank); + data->bank = nct6116d_gpio_bank; + break; default: return -ENODEV; } @@ -479,18 +524,15 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio) { int err; u16 devid; + u16 manid; err = superio_enter(addr); if (err) return err; err = -ENODEV; - devid = superio_inw(addr, SIO_MANID); - if (devid != SIO_FINTEK_ID) { - pr_debug(DRVNAME ": Not a Fintek device at 0x%08x\n", addr); - goto err; - } + sio->device = SIO_LD_GPIO_FINTEK; devid = superio_inw(addr, SIO_DEVID); switch (devid) { case SIO_F71869_ID: @@ -517,17 +559,30 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio) case SIO_F81865_ID: sio->type = f81865; break; + case SIO_NCT6116D_ID: + sio->device = SIO_LD_GPIO_NUVOTON; + sio->type = nct6116d; + break; default: - pr_info(DRVNAME ": Unsupported Fintek device 0x%04x\n", devid); + pr_info("Unsupported Fintek device 0x%04x\n", devid); goto err; } + + /* double check manufacturer where possible */ + if (sio->type != nct6116d) { + manid = superio_inw(addr, SIO_FINTEK_MANID); + if (manid != SIO_FINTEK_ID) { + pr_debug("Not a Fintek device at 0x%08x\n", addr); + goto err; + } + } + sio->addr = addr; err = 0; - pr_info(DRVNAME ": Found %s at %#x, revision %d\n", - f7188x_names[sio->type], - (unsigned int) addr, - (int) superio_inb(addr, SIO_DEVREV)); + pr_info("Found %s at %#x\n", f7188x_names[sio->type], (unsigned int)addr); + if (sio->type != nct6116d) + pr_info(" revision %d\n", superio_inb(addr, SIO_FINTEK_DEVREV)); err: superio_exit(addr); @@ -548,13 +603,13 @@ f7188x_gpio_device_add(const struct f7188x_sio *sio) err = platform_device_add_data(f7188x_gpio_pdev, sio, sizeof(*sio)); if (err) { - pr_err(DRVNAME "Platform data allocation failed\n"); + pr_err("Platform data allocation failed\n"); goto err; } err = platform_device_add(f7188x_gpio_pdev); if (err) { - pr_err(DRVNAME "Device addition failed\n"); + pr_err("Device addition failed\n"); goto err; } diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c index f422c3e129a0c4cc1b34c03c746608e387e013e5..f77a965f5780d8d9817f2546df480e3341cb3b68 100644 --- a/drivers/gpio/gpio-ftgpio010.c +++ b/drivers/gpio/gpio-ftgpio010.c @@ -41,14 +41,12 @@ * struct ftgpio_gpio - Gemini GPIO state container * @dev: containing device for this instance * @gc: gpiochip for this instance - * @irq: irqchip for this instance * @base: remapped I/O-memory base * @clk: silicon clock */ struct ftgpio_gpio { struct device *dev; struct gpio_chip gc; - struct irq_chip irq; void __iomem *base; struct clk *clk; }; @@ -70,6 +68,7 @@ static void ftgpio_gpio_mask_irq(struct irq_data *d) val = readl(g->base + GPIO_INT_EN); val &= ~BIT(irqd_to_hwirq(d)); writel(val, g->base + GPIO_INT_EN); + gpiochip_disable_irq(gc, irqd_to_hwirq(d)); } static void ftgpio_gpio_unmask_irq(struct irq_data *d) @@ -78,6 +77,7 @@ static void ftgpio_gpio_unmask_irq(struct irq_data *d) struct ftgpio_gpio *g = gpiochip_get_data(gc); u32 val; + gpiochip_enable_irq(gc, irqd_to_hwirq(d)); val = readl(g->base + GPIO_INT_EN); val |= BIT(irqd_to_hwirq(d)); writel(val, g->base + GPIO_INT_EN); @@ -221,6 +221,16 @@ static int ftgpio_gpio_set_config(struct gpio_chip *gc, unsigned int offset, return 0; } +static const struct irq_chip ftgpio_irq_chip = { + .name = "FTGPIO010", + .irq_ack = ftgpio_gpio_ack_irq, + .irq_mask = ftgpio_gpio_mask_irq, + .irq_unmask = ftgpio_gpio_unmask_irq, + .irq_set_type = ftgpio_gpio_set_irq_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int ftgpio_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -277,14 +287,8 @@ static int ftgpio_gpio_probe(struct platform_device *pdev) if (!IS_ERR(g->clk)) g->gc.set_config = ftgpio_gpio_set_config; - g->irq.name = "FTGPIO010"; - g->irq.irq_ack = ftgpio_gpio_ack_irq; - g->irq.irq_mask = ftgpio_gpio_mask_irq; - g->irq.irq_unmask = ftgpio_gpio_unmask_irq; - g->irq.irq_set_type = ftgpio_gpio_set_irq_type; - girq = &g->gc.irq; - girq->chip = &g->irq; + gpio_irq_chip_set_chip(girq, &ftgpio_irq_chip); girq->parent_handler = ftgpio_gpio_irq_handler; girq->num_parents = 1; girq->parents = devm_kcalloc(dev, 1, sizeof(*girq->parents), diff --git a/drivers/gpio/gpio-imx-scu.c b/drivers/gpio/gpio-imx-scu.c new file mode 100644 index 0000000000000000000000000000000000000000..17be21b8f3b70cb9448264db5db6e06382f066f5 --- /dev/null +++ b/drivers/gpio/gpio-imx-scu.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2021~2022 NXP + * + * The driver exports a standard gpiochip interface + * to control the PIN resources on SCU domain. + */ + +#include +#include +#include +#include +#include + +struct scu_gpio_priv { + struct gpio_chip chip; + struct mutex lock; + struct device *dev; + struct imx_sc_ipc *handle; +}; + +static unsigned int scu_rsrc_arr[] = { + IMX_SC_R_BOARD_R0, + IMX_SC_R_BOARD_R1, + IMX_SC_R_BOARD_R2, + IMX_SC_R_BOARD_R3, + IMX_SC_R_BOARD_R4, + IMX_SC_R_BOARD_R5, + IMX_SC_R_BOARD_R6, + IMX_SC_R_BOARD_R7, +}; + +static int imx_scu_gpio_get(struct gpio_chip *chip, unsigned int offset) +{ + struct scu_gpio_priv *priv = gpiochip_get_data(chip); + int level; + int err; + + if (offset >= chip->ngpio) + return -EINVAL; + + mutex_lock(&priv->lock); + + /* to read PIN state via scu api */ + err = imx_sc_misc_get_control(priv->handle, + scu_rsrc_arr[offset], 0, &level); + mutex_unlock(&priv->lock); + + if (err) { + dev_err(priv->dev, "SCU get failed: %d\n", err); + return err; + } + + return level; +} + +static void imx_scu_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) +{ + struct scu_gpio_priv *priv = gpiochip_get_data(chip); + int err; + + if (offset >= chip->ngpio) + return; + + mutex_lock(&priv->lock); + + /* to set PIN output level via scu api */ + err = imx_sc_misc_set_control(priv->handle, + scu_rsrc_arr[offset], 0, value); + mutex_unlock(&priv->lock); + + if (err) + dev_err(priv->dev, "SCU set (%d) failed: %d\n", + scu_rsrc_arr[offset], err); +} + +static int imx_scu_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + if (offset >= chip->ngpio) + return -EINVAL; + + return GPIO_LINE_DIRECTION_OUT; +} + +static int imx_scu_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct scu_gpio_priv *priv; + struct gpio_chip *gc; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + ret = imx_scu_get_handle(&priv->handle); + if (ret) + return ret; + + priv->dev = dev; + mutex_init(&priv->lock); + + gc = &priv->chip; + gc->base = -1; + gc->parent = dev; + gc->ngpio = sizeof(scu_rsrc_arr)/sizeof(unsigned int); + gc->label = dev_name(dev); + gc->get = imx_scu_gpio_get; + gc->set = imx_scu_gpio_set; + gc->get_direction = imx_scu_gpio_get_direction; + + platform_set_drvdata(pdev, priv); + + return devm_gpiochip_add_data(dev, gc, priv); +} + +static const struct of_device_id imx_scu_gpio_dt_ids[] = { + { .compatible = "fsl,imx8qxp-sc-gpio" }, + { /* sentinel */ } +}; + +static struct platform_driver imx_scu_gpio_driver = { + .driver = { + .name = "gpio-imx-scu", + .of_match_table = imx_scu_gpio_dt_ids, + }, + .probe = imx_scu_gpio_probe, +}; + +static int __init _imx_scu_gpio_init(void) +{ + return platform_driver_register(&imx_scu_gpio_driver); +} + +subsys_initcall_sync(_imx_scu_gpio_init); + +MODULE_AUTHOR("Shenwei Wang "); +MODULE_DESCRIPTION("NXP GPIO over IMX SCU API"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c index 312309be0287db567c0a5c838078969fd4b19b1a..56656fb519f8553e783998188473c88db9a6e1b1 100644 --- a/drivers/gpio/gpio-ixp4xx.c +++ b/drivers/gpio/gpio-ixp4xx.c @@ -63,6 +63,14 @@ static void ixp4xx_gpio_irq_ack(struct irq_data *d) __raw_writel(BIT(d->hwirq), g->base + IXP4XX_REG_GPIS); } +static void ixp4xx_gpio_mask_irq(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + + irq_chip_mask_parent(d); + gpiochip_disable_irq(gc, d->hwirq); +} + static void ixp4xx_gpio_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); @@ -72,6 +80,7 @@ static void ixp4xx_gpio_irq_unmask(struct irq_data *d) if (!(g->irq_edge & BIT(d->hwirq))) ixp4xx_gpio_irq_ack(d); + gpiochip_enable_irq(gc, d->hwirq); irq_chip_unmask_parent(d); } @@ -149,12 +158,14 @@ static int ixp4xx_gpio_irq_set_type(struct irq_data *d, unsigned int type) return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH); } -static struct irq_chip ixp4xx_gpio_irqchip = { +static const struct irq_chip ixp4xx_gpio_irqchip = { .name = "IXP4GPIO", .irq_ack = ixp4xx_gpio_irq_ack, - .irq_mask = irq_chip_mask_parent, + .irq_mask = ixp4xx_gpio_mask_irq, .irq_unmask = ixp4xx_gpio_irq_unmask, .irq_set_type = ixp4xx_gpio_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int ixp4xx_gpio_child_to_parent_hwirq(struct gpio_chip *gc, @@ -263,7 +274,7 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev) g->gc.owner = THIS_MODULE; girq = &g->gc.irq; - girq->chip = &ixp4xx_gpio_irqchip; + gpio_irq_chip_set_chip(girq, &ixp4xx_gpio_irqchip); girq->fwnode = g->fwnode; girq->parent_domain = parent; girq->child_to_parent_hwirq = ixp4xx_gpio_child_to_parent_hwirq; diff --git a/drivers/gpio/gpio-max7300.c b/drivers/gpio/gpio-max7300.c index b2b547dd6e84648487997eef93d8d4feb4df06b5..43da381a4d7e7ed322a25fdc5304b9014d0345ef 100644 --- a/drivers/gpio/gpio-max7300.c +++ b/drivers/gpio/gpio-max7300.c @@ -48,11 +48,9 @@ static int max7300_probe(struct i2c_client *client, return __max730x_probe(ts); } -static int max7300_remove(struct i2c_client *client) +static void max7300_remove(struct i2c_client *client) { __max730x_remove(&client->dev); - - return 0; } static const struct i2c_device_id max7300_id[] = { diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c index 64cb060d9d7533195042102aa38fda72dcfeb2d5..77a41151c921bd790bab6a99363408f385ff864d 100644 --- a/drivers/gpio/gpio-mlxbf2.c +++ b/drivers/gpio/gpio-mlxbf2.c @@ -273,10 +273,8 @@ static irqreturn_t mlxbf2_gpio_irq_handler(int irq, void *ptr) pending = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_CAUSE_EVTEN0); writel(pending, gs->gpio_io + YU_GPIO_CAUSE_OR_CLRCAUSE); - for_each_set_bit(level, &pending, gc->ngpio) { - int gpio_irq = irq_find_mapping(gc->irq.domain, level); - generic_handle_irq(gpio_irq); - } + for_each_set_bit(level, &pending, gc->ngpio) + generic_handle_domain_irq_safe(gc->irq.domain, level); return IRQ_RETVAL(pending); } diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c index 8943cea92764204e8c037b759009ea6d33edf7a5..523dfd17dd922548ebabbd1cd63f7bbf033b73e2 100644 --- a/drivers/gpio/gpio-mockup.c +++ b/drivers/gpio/gpio-mockup.c @@ -373,6 +373,13 @@ static void gpio_mockup_debugfs_setup(struct device *dev, } } +static void gpio_mockup_debugfs_cleanup(void *data) +{ + struct gpio_mockup_chip *chip = data; + + debugfs_remove_recursive(chip->dbg_dir); +} + static void gpio_mockup_dispose_mappings(void *data) { struct gpio_mockup_chip *chip = data; @@ -455,7 +462,7 @@ static int gpio_mockup_probe(struct platform_device *pdev) gpio_mockup_debugfs_setup(dev, chip); - return 0; + return devm_add_action_or_reset(dev, gpio_mockup_debugfs_cleanup, chip); } static const struct of_device_id gpio_mockup_of_match[] = { @@ -526,8 +533,10 @@ static int __init gpio_mockup_register_chip(int idx) } fwnode = fwnode_create_software_node(properties, NULL); - if (IS_ERR(fwnode)) + if (IS_ERR(fwnode)) { + kfree_strarray(line_names, ngpio); return PTR_ERR(fwnode); + } pdevinfo.name = "gpio-mockup"; pdevinfo.id = idx; @@ -590,9 +599,9 @@ static int __init gpio_mockup_init(void) static void __exit gpio_mockup_exit(void) { + gpio_mockup_unregister_pdevs(); debugfs_remove_recursive(gpio_mockup_dbg_dir); platform_driver_unregister(&gpio_mockup_driver); - gpio_mockup_unregister_pdevs(); } module_init(gpio_mockup_init); diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c index 15049822937a43a19c98d04562c6235b67724332..3eb08cd1fdc0818a89458b5ad72a4761b2b31a9c 100644 --- a/drivers/gpio/gpio-mpc8xxx.c +++ b/drivers/gpio/gpio-mpc8xxx.c @@ -169,6 +169,7 @@ static int mpc8xxx_irq_set_type(struct irq_data *d, unsigned int flow_type) switch (flow_type) { case IRQ_TYPE_EDGE_FALLING: + case IRQ_TYPE_LEVEL_LOW: raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); gc->write_reg(mpc8xxx_gc->regs + GPIO_ICR, gc->read_reg(mpc8xxx_gc->regs + GPIO_ICR) diff --git a/drivers/gpio/gpio-mt7621.c b/drivers/gpio/gpio-mt7621.c index d8a26e503ca5d2fe7d99ce2745b484520a2973c0..93facbebb80efadbdd3fb4500e0db14936287f1a 100644 --- a/drivers/gpio/gpio-mt7621.c +++ b/drivers/gpio/gpio-mt7621.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -112,6 +111,8 @@ mediatek_gpio_irq_unmask(struct irq_data *d) unsigned long flags; u32 rise, fall, high, low; + gpiochip_enable_irq(gc, d->hwirq); + spin_lock_irqsave(&rg->lock, flags); rise = mtk_gpio_r32(rg, GPIO_REG_REDGE); fall = mtk_gpio_r32(rg, GPIO_REG_FEDGE); @@ -143,6 +144,8 @@ mediatek_gpio_irq_mask(struct irq_data *d) mtk_gpio_w32(rg, GPIO_REG_HLVL, high & ~BIT(pin)); mtk_gpio_w32(rg, GPIO_REG_LLVL, low & ~BIT(pin)); spin_unlock_irqrestore(&rg->lock, flags); + + gpiochip_disable_irq(gc, d->hwirq); } static int @@ -204,6 +207,16 @@ mediatek_gpio_xlate(struct gpio_chip *chip, return gpio % MTK_BANK_WIDTH; } +static const struct irq_chip mt7621_irq_chip = { + .name = "mt7621-gpio", + .irq_mask_ack = mediatek_gpio_irq_mask, + .irq_mask = mediatek_gpio_irq_mask, + .irq_unmask = mediatek_gpio_irq_unmask, + .irq_set_type = mediatek_gpio_irq_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int mediatek_gpio_bank_probe(struct device *dev, int bank) { @@ -238,11 +251,6 @@ mediatek_gpio_bank_probe(struct device *dev, int bank) return -ENOMEM; rg->chip.offset = bank * MTK_BANK_WIDTH; - rg->irq_chip.name = dev_name(dev); - rg->irq_chip.irq_unmask = mediatek_gpio_irq_unmask; - rg->irq_chip.irq_mask = mediatek_gpio_irq_mask; - rg->irq_chip.irq_mask_ack = mediatek_gpio_irq_mask; - rg->irq_chip.irq_set_type = mediatek_gpio_irq_type; if (mtk->gpio_irq) { struct gpio_irq_chip *girq; @@ -262,7 +270,7 @@ mediatek_gpio_bank_probe(struct device *dev, int bank) } girq = &rg->chip.irq; - girq->chip = &rg->irq_chip; + gpio_irq_chip_set_chip(girq, &mt7621_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; @@ -290,7 +298,6 @@ static int mediatek_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; struct mtk *mtk; int i; int ret; @@ -303,7 +310,10 @@ mediatek_gpio_probe(struct platform_device *pdev) if (IS_ERR(mtk->base)) return PTR_ERR(mtk->base); - mtk->gpio_irq = irq_of_parse_and_map(np, 0); + mtk->gpio_irq = platform_get_irq(pdev, 0); + if (mtk->gpio_irq < 0) + return mtk->gpio_irq; + mtk->dev = dev; platform_set_drvdata(pdev, mtk); diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index aa126ab80f0ccc8e41f04aeec0e21ccef633628e..1bb317b8dcceac60e0b5040e76b0e3496b2adbf0 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -790,8 +790,12 @@ static int mvebu_pwm_probe(struct platform_device *pdev, u32 offset; u32 set; - if (of_device_is_compatible(mvchip->chip.of_node, - "marvell,armada-370-gpio")) { + if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { + int ret = of_property_read_u32(dev->of_node, + "marvell,pwm-offset", &offset); + if (ret < 0) + return 0; + } else { /* * There are only two sets of PWM configuration registers for * all the GPIO lines on those SoCs which this driver reserves @@ -801,13 +805,6 @@ static int mvebu_pwm_probe(struct platform_device *pdev, if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm")) return 0; offset = 0; - } else if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { - int ret = of_property_read_u32(dev->of_node, - "marvell,pwm-offset", &offset); - if (ret < 0) - return 0; - } else { - return 0; } if (IS_ERR(mvchip->clk)) diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index ecd7d169470b068ddbbc9685ae33a1f0a860cd32..ebe1943b85dd99f77a7c8883c2759ef666abc703 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -66,6 +66,7 @@ #define PCA_LATCH_INT (PCA_PCAL | PCA_INT) #define PCA953X_TYPE BIT(12) #define PCA957X_TYPE BIT(13) +#define PCAL653X_TYPE BIT(14) #define PCA_TYPE_MASK GENMASK(15, 12) #define PCA_CHIP_TYPE(x) ((x) & PCA_TYPE_MASK) @@ -89,8 +90,10 @@ static const struct i2c_device_id pca953x_id[] = { { "pca9575", 16 | PCA957X_TYPE | PCA_INT, }, { "pca9698", 40 | PCA953X_TYPE, }, + { "pcal6408", 8 | PCA953X_TYPE | PCA_LATCH_INT, }, { "pcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, { "pcal6524", 24 | PCA953X_TYPE | PCA_LATCH_INT, }, + { "pcal6534", 34 | PCAL653X_TYPE | PCA_LATCH_INT, }, { "pcal9535", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, { "pcal9554b", 8 | PCA953X_TYPE | PCA_LATCH_INT, }, { "pcal9555a", 16 | PCA953X_TYPE | PCA_LATCH_INT, }, @@ -211,6 +214,10 @@ struct pca953x_chip { struct regulator *regulator; const struct pca953x_reg_config *regs; + + u8 (*recalc_addr)(struct pca953x_chip *chip, int reg, int off); + bool (*check_reg)(struct pca953x_chip *chip, unsigned int reg, + u32 checkbank); }; static int pca953x_bank_shift(struct pca953x_chip *chip) @@ -288,18 +295,67 @@ static bool pca953x_check_register(struct pca953x_chip *chip, unsigned int reg, return true; } +/* + * Unfortunately, whilst the PCAL6534 chip (and compatibles) broadly follow the + * same register layout as the PCAL6524, the spacing of the registers has been + * fundamentally altered by compacting them and thus does not obey the same + * rules, including being able to use bit shifting to determine bank. These + * chips hence need special handling here. + */ +static bool pcal6534_check_register(struct pca953x_chip *chip, unsigned int reg, + u32 checkbank) +{ + int bank; + int offset; + + if (reg >= 0x30) { + /* + * Reserved block between 14h and 2Fh does not align on + * expected bank boundaries like other devices. + */ + int temp = reg - 0x30; + + bank = temp / NBANK(chip); + offset = temp - (bank * NBANK(chip)); + bank += 8; + } else if (reg >= 0x54) { + /* + * Handle lack of reserved registers after output port + * configuration register to form a bank. + */ + int temp = reg - 0x54; + + bank = temp / NBANK(chip); + offset = temp - (bank * NBANK(chip)); + bank += 16; + } else { + bank = reg / NBANK(chip); + offset = reg - (bank * NBANK(chip)); + } + + /* Register is not in the matching bank. */ + if (!(BIT(bank) & checkbank)) + return false; + + /* Register is not within allowed range of bank. */ + if (offset >= NBANK(chip)) + return false; + + return true; +} + static bool pca953x_readable_register(struct device *dev, unsigned int reg) { struct pca953x_chip *chip = dev_get_drvdata(dev); u32 bank; - if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE) { - bank = PCA953x_BANK_INPUT | PCA953x_BANK_OUTPUT | - PCA953x_BANK_POLARITY | PCA953x_BANK_CONFIG; - } else { + if (PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE) { bank = PCA957x_BANK_INPUT | PCA957x_BANK_OUTPUT | PCA957x_BANK_POLARITY | PCA957x_BANK_CONFIG | PCA957x_BANK_BUSHOLD; + } else { + bank = PCA953x_BANK_INPUT | PCA953x_BANK_OUTPUT | + PCA953x_BANK_POLARITY | PCA953x_BANK_CONFIG; } if (chip->driver_data & PCA_PCAL) { @@ -308,7 +364,7 @@ static bool pca953x_readable_register(struct device *dev, unsigned int reg) PCAL9xxx_BANK_IRQ_STAT; } - return pca953x_check_register(chip, reg, bank); + return chip->check_reg(chip, reg, bank); } static bool pca953x_writeable_register(struct device *dev, unsigned int reg) @@ -316,19 +372,19 @@ static bool pca953x_writeable_register(struct device *dev, unsigned int reg) struct pca953x_chip *chip = dev_get_drvdata(dev); u32 bank; - if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE) { - bank = PCA953x_BANK_OUTPUT | PCA953x_BANK_POLARITY | - PCA953x_BANK_CONFIG; - } else { + if (PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE) { bank = PCA957x_BANK_OUTPUT | PCA957x_BANK_POLARITY | PCA957x_BANK_CONFIG | PCA957x_BANK_BUSHOLD; + } else { + bank = PCA953x_BANK_OUTPUT | PCA953x_BANK_POLARITY | + PCA953x_BANK_CONFIG; } if (chip->driver_data & PCA_PCAL) bank |= PCAL9xxx_BANK_IN_LATCH | PCAL9xxx_BANK_PULL_EN | PCAL9xxx_BANK_PULL_SEL | PCAL9xxx_BANK_IRQ_MASK; - return pca953x_check_register(chip, reg, bank); + return chip->check_reg(chip, reg, bank); } static bool pca953x_volatile_register(struct device *dev, unsigned int reg) @@ -336,15 +392,15 @@ static bool pca953x_volatile_register(struct device *dev, unsigned int reg) struct pca953x_chip *chip = dev_get_drvdata(dev); u32 bank; - if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE) - bank = PCA953x_BANK_INPUT; - else + if (PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE) bank = PCA957x_BANK_INPUT; + else + bank = PCA953x_BANK_INPUT; if (chip->driver_data & PCA_PCAL) bank |= PCAL9xxx_BANK_IRQ_STAT; - return pca953x_check_register(chip, reg, bank); + return chip->check_reg(chip, reg, bank); } static const struct regmap_config pca953x_i2c_regmap = { @@ -389,9 +445,42 @@ static u8 pca953x_recalc_addr(struct pca953x_chip *chip, int reg, int off) return regaddr; } +/* + * The PCAL6534 and compatible chips have altered bank alignment that doesn't + * fit within the bit shifting scheme used for other devices. + */ +static u8 pcal6534_recalc_addr(struct pca953x_chip *chip, int reg, int off) +{ + int addr; + int pinctrl; + + addr = (reg & PCAL_GPIO_MASK) * NBANK(chip); + + switch (reg) { + case PCAL953X_OUT_STRENGTH: + case PCAL953X_IN_LATCH: + case PCAL953X_PULL_EN: + case PCAL953X_PULL_SEL: + case PCAL953X_INT_MASK: + case PCAL953X_INT_STAT: + case PCAL953X_OUT_CONF: + pinctrl = ((reg & PCAL_PINCTRL_MASK) >> 1) + 0x20; + break; + case PCAL6524_INT_EDGE: + case PCAL6524_INT_CLR: + case PCAL6524_IN_STATUS: + case PCAL6524_OUT_INDCONF: + case PCAL6524_DEBOUNCE: + pinctrl = ((reg & PCAL_PINCTRL_MASK) >> 1) + 0x1c; + break; + } + + return pinctrl + addr + (off / BANK_SZ); +} + static int pca953x_write_regs(struct pca953x_chip *chip, int reg, unsigned long *val) { - u8 regaddr = pca953x_recalc_addr(chip, reg, 0); + u8 regaddr = chip->recalc_addr(chip, reg, 0); u8 value[MAX_BANK]; int i, ret; @@ -409,7 +498,7 @@ static int pca953x_write_regs(struct pca953x_chip *chip, int reg, unsigned long static int pca953x_read_regs(struct pca953x_chip *chip, int reg, unsigned long *val) { - u8 regaddr = pca953x_recalc_addr(chip, reg, 0); + u8 regaddr = chip->recalc_addr(chip, reg, 0); u8 value[MAX_BANK]; int i, ret; @@ -428,7 +517,7 @@ static int pca953x_read_regs(struct pca953x_chip *chip, int reg, unsigned long * static int pca953x_gpio_direction_input(struct gpio_chip *gc, unsigned off) { struct pca953x_chip *chip = gpiochip_get_data(gc); - u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off); + u8 dirreg = chip->recalc_addr(chip, chip->regs->direction, off); u8 bit = BIT(off % BANK_SZ); int ret; @@ -442,8 +531,8 @@ static int pca953x_gpio_direction_output(struct gpio_chip *gc, unsigned off, int val) { struct pca953x_chip *chip = gpiochip_get_data(gc); - u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off); - u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off); + u8 dirreg = chip->recalc_addr(chip, chip->regs->direction, off); + u8 outreg = chip->recalc_addr(chip, chip->regs->output, off); u8 bit = BIT(off % BANK_SZ); int ret; @@ -463,7 +552,7 @@ exit: static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off) { struct pca953x_chip *chip = gpiochip_get_data(gc); - u8 inreg = pca953x_recalc_addr(chip, chip->regs->input, off); + u8 inreg = chip->recalc_addr(chip, chip->regs->input, off); u8 bit = BIT(off % BANK_SZ); u32 reg_val; int ret; @@ -480,7 +569,7 @@ static int pca953x_gpio_get_value(struct gpio_chip *gc, unsigned off) static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val) { struct pca953x_chip *chip = gpiochip_get_data(gc); - u8 outreg = pca953x_recalc_addr(chip, chip->regs->output, off); + u8 outreg = chip->recalc_addr(chip, chip->regs->output, off); u8 bit = BIT(off % BANK_SZ); mutex_lock(&chip->i2c_lock); @@ -491,7 +580,7 @@ static void pca953x_gpio_set_value(struct gpio_chip *gc, unsigned off, int val) static int pca953x_gpio_get_direction(struct gpio_chip *gc, unsigned off) { struct pca953x_chip *chip = gpiochip_get_data(gc); - u8 dirreg = pca953x_recalc_addr(chip, chip->regs->direction, off); + u8 dirreg = chip->recalc_addr(chip, chip->regs->direction, off); u8 bit = BIT(off % BANK_SZ); u32 reg_val; int ret; @@ -548,8 +637,10 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, unsigned int offset, unsigned long config) { - u8 pull_en_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_EN, offset); - u8 pull_sel_reg = pca953x_recalc_addr(chip, PCAL953X_PULL_SEL, offset); + enum pin_config_param param = pinconf_to_config_param(config); + + u8 pull_en_reg = chip->recalc_addr(chip, PCAL953X_PULL_EN, offset); + u8 pull_sel_reg = chip->recalc_addr(chip, PCAL953X_PULL_SEL, offset); u8 bit = BIT(offset % BANK_SZ); int ret; @@ -563,9 +654,9 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, mutex_lock(&chip->i2c_lock); /* Configure pull-up/pull-down */ - if (config == PIN_CONFIG_BIAS_PULL_UP) + if (param == PIN_CONFIG_BIAS_PULL_UP) ret = regmap_write_bits(chip->regmap, pull_sel_reg, bit, bit); - else if (config == PIN_CONFIG_BIAS_PULL_DOWN) + else if (param == PIN_CONFIG_BIAS_PULL_DOWN) ret = regmap_write_bits(chip->regmap, pull_sel_reg, bit, 0); else ret = 0; @@ -573,7 +664,7 @@ static int pca953x_gpio_set_pull_up_down(struct pca953x_chip *chip, goto exit; /* Disable/Enable pull-up/pull-down */ - if (config == PIN_CONFIG_BIAS_DISABLE) + if (param == PIN_CONFIG_BIAS_DISABLE) ret = regmap_write_bits(chip->regmap, pull_en_reg, bit, 0); else ret = regmap_write_bits(chip->regmap, pull_en_reg, bit, bit); @@ -912,13 +1003,13 @@ static int device_pca95xx_init(struct pca953x_chip *chip, u32 invert) u8 regaddr; int ret; - regaddr = pca953x_recalc_addr(chip, chip->regs->output, 0); + regaddr = chip->recalc_addr(chip, chip->regs->output, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) goto out; - regaddr = pca953x_recalc_addr(chip, chip->regs->direction, 0); + regaddr = chip->recalc_addr(chip, chip->regs->direction, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) @@ -1037,6 +1128,14 @@ static int pca953x_probe(struct i2c_client *client, regmap_config = &pca953x_i2c_regmap; } + if (PCA_CHIP_TYPE(chip->driver_data) == PCAL653X_TYPE) { + chip->recalc_addr = pcal6534_recalc_addr; + chip->check_reg = pcal6534_check_register; + } else { + chip->recalc_addr = pca953x_recalc_addr; + chip->check_reg = pca953x_check_register; + } + chip->regmap = devm_regmap_init_i2c(client, regmap_config); if (IS_ERR(chip->regmap)) { ret = PTR_ERR(chip->regmap); @@ -1068,13 +1167,12 @@ static int pca953x_probe(struct i2c_client *client, /* initialize cached registers from their original values. * we can't share this chip with another i2c master. */ - - if (PCA_CHIP_TYPE(chip->driver_data) == PCA953X_TYPE) { - chip->regs = &pca953x_regs; - ret = device_pca95xx_init(chip, invert); - } else { + if (PCA_CHIP_TYPE(chip->driver_data) == PCA957X_TYPE) { chip->regs = &pca957x_regs; ret = device_pca957x_init(chip, invert); + } else { + chip->regs = &pca953x_regs; + ret = device_pca95xx_init(chip, invert); } if (ret) goto err_exit; @@ -1101,24 +1199,17 @@ err_exit: return ret; } -static int pca953x_remove(struct i2c_client *client) +static void pca953x_remove(struct i2c_client *client) { struct pca953x_platform_data *pdata = dev_get_platdata(&client->dev); struct pca953x_chip *chip = i2c_get_clientdata(client); - int ret; if (pdata && pdata->teardown) { - ret = pdata->teardown(client, chip->gpio_chip.base, - chip->gpio_chip.ngpio, pdata->context); - if (ret < 0) - dev_err(&client->dev, "teardown failed, %d\n", ret); - } else { - ret = 0; + pdata->teardown(client, chip->gpio_chip.base, + chip->gpio_chip.ngpio, pdata->context); } regulator_disable(chip->regulator); - - return ret; } #ifdef CONFIG_PM_SLEEP @@ -1132,14 +1223,14 @@ static int pca953x_regcache_sync(struct device *dev) * The ordering between direction and output is important, * sync these registers first and only then sync the rest. */ - regaddr = pca953x_recalc_addr(chip, chip->regs->direction, 0); + regaddr = chip->recalc_addr(chip, chip->regs->direction, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) { dev_err(dev, "Failed to sync GPIO dir registers: %d\n", ret); return ret; } - regaddr = pca953x_recalc_addr(chip, chip->regs->output, 0); + regaddr = chip->recalc_addr(chip, chip->regs->output, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) { dev_err(dev, "Failed to sync GPIO out registers: %d\n", ret); @@ -1148,7 +1239,7 @@ static int pca953x_regcache_sync(struct device *dev) #ifdef CONFIG_GPIO_PCA953X_IRQ if (chip->driver_data & PCA_PCAL) { - regaddr = pca953x_recalc_addr(chip, PCAL953X_IN_LATCH, 0); + regaddr = chip->recalc_addr(chip, PCAL953X_IN_LATCH, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) { @@ -1157,7 +1248,7 @@ static int pca953x_regcache_sync(struct device *dev) return ret; } - regaddr = pca953x_recalc_addr(chip, PCAL953X_INT_MASK, 0); + regaddr = chip->recalc_addr(chip, PCAL953X_INT_MASK, 0); ret = regcache_sync_region(chip->regmap, regaddr, regaddr + NBANK(chip) - 1); if (ret) { @@ -1175,7 +1266,9 @@ static int pca953x_suspend(struct device *dev) { struct pca953x_chip *chip = dev_get_drvdata(dev); + mutex_lock(&chip->i2c_lock); regcache_cache_only(chip->regmap, true); + mutex_unlock(&chip->i2c_lock); if (atomic_read(&chip->wakeup_path)) device_set_wakeup_path(dev); @@ -1198,13 +1291,17 @@ static int pca953x_resume(struct device *dev) } } + mutex_lock(&chip->i2c_lock); regcache_cache_only(chip->regmap, false); regcache_mark_dirty(chip->regmap); ret = pca953x_regcache_sync(dev); - if (ret) + if (ret) { + mutex_unlock(&chip->i2c_lock); return ret; + } ret = regcache_sync(chip->regmap); + mutex_unlock(&chip->i2c_lock); if (ret) { dev_err(dev, "Failed to restore register map: %d\n", ret); return ret; @@ -1215,6 +1312,7 @@ static int pca953x_resume(struct device *dev) #endif /* convenience to stop overlong match-table lines */ +#define OF_653X(__nrgpio, __int) ((void *)(__nrgpio | PCAL653X_TYPE | __int)) #define OF_953X(__nrgpio, __int) (void *)(__nrgpio | PCA953X_TYPE | __int) #define OF_957X(__nrgpio, __int) (void *)(__nrgpio | PCA957X_TYPE | __int) @@ -1237,8 +1335,10 @@ static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "nxp,pca9575", .data = OF_957X(16, PCA_INT), }, { .compatible = "nxp,pca9698", .data = OF_953X(40, 0), }, + { .compatible = "nxp,pcal6408", .data = OF_953X(8, PCA_LATCH_INT), }, { .compatible = "nxp,pcal6416", .data = OF_953X(16, PCA_LATCH_INT), }, { .compatible = "nxp,pcal6524", .data = OF_953X(24, PCA_LATCH_INT), }, + { .compatible = "nxp,pcal6534", .data = OF_653X(34, PCA_LATCH_INT), }, { .compatible = "nxp,pcal9535", .data = OF_953X(16, PCA_LATCH_INT), }, { .compatible = "nxp,pcal9554b", .data = OF_953X( 8, PCA_LATCH_INT), }, { .compatible = "nxp,pcal9555a", .data = OF_953X(16, PCA_LATCH_INT), }, diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index 59cc27e4de511b18236952908a2f1684a7826744..e98ea47d7237985e51dda9016c2193f4ae58fc05 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -399,7 +399,7 @@ fail: return status; } -static int pcf857x_remove(struct i2c_client *client) +static void pcf857x_remove(struct i2c_client *client) { struct pcf857x_platform_data *pdata = dev_get_platdata(&client->dev); struct pcf857x *gpio = i2c_get_clientdata(client); @@ -407,8 +407,6 @@ static int pcf857x_remove(struct i2c_client *client) if (pdata && pdata->teardown) pdata->teardown(client, gpio->chip.base, gpio->chip.ngpio, pdata->context); - - return 0; } static void pcf857x_shutdown(struct i2c_client *client) diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c index c7fbfa3ae43b920eca9e6a9c0d6b3696e64ac851..1198ab0305d037bcd787c091c9e183e489406db2 100644 --- a/drivers/gpio/gpio-pxa.c +++ b/drivers/gpio/gpio-pxa.c @@ -661,24 +661,17 @@ static int pxa_gpio_probe(struct platform_device *pdev) if (IS_ERR(gpio_reg_base)) return PTR_ERR(gpio_reg_base); - clk = clk_get(&pdev->dev, NULL); + clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(clk)) { dev_err(&pdev->dev, "Error %ld to get gpio clock\n", PTR_ERR(clk)); return PTR_ERR(clk); } - ret = clk_prepare_enable(clk); - if (ret) { - clk_put(clk); - return ret; - } /* Initialize GPIO chips */ ret = pxa_init_gpio_chip(pchip, pxa_last_gpio + 1, gpio_reg_base); - if (ret) { - clk_put(clk); + if (ret) return ret; - } /* clear all GPIO edge detects */ for_each_gpio_bank(gpio, c, pchip) { diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c index 63dcf42f7c206e0b9f05ba49426b7f806c34835c..d6418f89d3f63d6029e127d4f774507c2ebbe0cb 100644 --- a/drivers/gpio/gpio-realtek-otto.c +++ b/drivers/gpio/gpio-realtek-otto.c @@ -46,10 +46,20 @@ * @lock: Lock for accessing the IRQ registers and values * @intr_mask: Mask for interrupts lines * @intr_type: Interrupt type selection + * @bank_read: Read a bank setting as a single 32-bit value + * @bank_write: Write a bank setting as a single 32-bit value + * @imr_line_pos: Bit shift of an IRQ line's IMR value. + * + * The DIR, DATA, and ISR registers consist of four 8-bit port values, packed + * into a single 32-bit register. Use @bank_read (@bank_write) to get (assign) + * a value from (to) these registers. The IMR register consists of four 16-bit + * port values, packed into two 32-bit registers. Use @imr_line_pos to get the + * bit shift of the 2-bit field for a line's IMR settings. Shifts larger than + * 32 overflow into the second register. * * Because the interrupt mask register (IMR) combines the function of IRQ type * selection and masking, two extra values are stored. @intr_mask is used to - * mask/unmask the interrupts for a GPIO port, and @intr_type is used to store + * mask/unmask the interrupts for a GPIO line, and @intr_type is used to store * the selected interrupt types. The logical AND of these values is written to * IMR on changes. */ @@ -59,10 +69,11 @@ struct realtek_gpio_ctrl { void __iomem *cpumask_base; struct cpumask cpu_irq_maskable; raw_spinlock_t lock; - u16 intr_mask[REALTEK_GPIO_PORTS_PER_BANK]; - u16 intr_type[REALTEK_GPIO_PORTS_PER_BANK]; - unsigned int (*port_offset_u8)(unsigned int port); - unsigned int (*port_offset_u16)(unsigned int port); + u8 intr_mask[REALTEK_GPIO_MAX]; + u8 intr_type[REALTEK_GPIO_MAX]; + u32 (*bank_read)(void __iomem *reg); + void (*bank_write)(void __iomem *reg, u32 value); + unsigned int (*line_imr_pos)(unsigned int line); }; /* Expand with more flags as devices with other quirks are added */ @@ -101,14 +112,22 @@ static struct realtek_gpio_ctrl *irq_data_to_ctrl(struct irq_data *data) * port. The two interrupt mask registers store two bits per GPIO, so use u16 * values. */ -static unsigned int realtek_gpio_port_offset_u8(unsigned int port) +static u32 realtek_gpio_bank_read_swapped(void __iomem *reg) { - return port; + return ioread32be(reg); } -static unsigned int realtek_gpio_port_offset_u16(unsigned int port) +static void realtek_gpio_bank_write_swapped(void __iomem *reg, u32 value) { - return 2 * port; + iowrite32be(value, reg); +} + +static unsigned int realtek_gpio_line_imr_pos_swapped(unsigned int line) +{ + unsigned int port_pin = line % 8; + unsigned int port = line / 8; + + return 2 * (8 * (port ^ 1) + port_pin); } /* @@ -119,66 +138,67 @@ static unsigned int realtek_gpio_port_offset_u16(unsigned int port) * per GPIO, so use u16 values. The first register contains ports 1 and 0, the * second ports 3 and 2. */ -static unsigned int realtek_gpio_port_offset_u8_rev(unsigned int port) +static u32 realtek_gpio_bank_read(void __iomem *reg) { - return 3 - port; + return ioread32(reg); } -static unsigned int realtek_gpio_port_offset_u16_rev(unsigned int port) +static void realtek_gpio_bank_write(void __iomem *reg, u32 value) { - return 2 * (port ^ 1); + iowrite32(value, reg); } -static void realtek_gpio_write_imr(struct realtek_gpio_ctrl *ctrl, - unsigned int port, u16 irq_type, u16 irq_mask) +static unsigned int realtek_gpio_line_imr_pos(unsigned int line) { - iowrite16(irq_type & irq_mask, - ctrl->base + REALTEK_GPIO_REG_IMR + ctrl->port_offset_u16(port)); + return 2 * line; } -static void realtek_gpio_clear_isr(struct realtek_gpio_ctrl *ctrl, - unsigned int port, u8 mask) +static void realtek_gpio_clear_isr(struct realtek_gpio_ctrl *ctrl, u32 mask) { - iowrite8(mask, ctrl->base + REALTEK_GPIO_REG_ISR + ctrl->port_offset_u8(port)); + ctrl->bank_write(ctrl->base + REALTEK_GPIO_REG_ISR, mask); } -static u8 realtek_gpio_read_isr(struct realtek_gpio_ctrl *ctrl, unsigned int port) +static u32 realtek_gpio_read_isr(struct realtek_gpio_ctrl *ctrl) { - return ioread8(ctrl->base + REALTEK_GPIO_REG_ISR + ctrl->port_offset_u8(port)); + return ctrl->bank_read(ctrl->base + REALTEK_GPIO_REG_ISR); } -/* Set the rising and falling edge mask bits for a GPIO port pin */ -static u16 realtek_gpio_imr_bits(unsigned int pin, u16 value) +/* Set the rising and falling edge mask bits for a GPIO pin */ +static void realtek_gpio_update_line_imr(struct realtek_gpio_ctrl *ctrl, unsigned int line) { - return (value & REALTEK_GPIO_IMR_LINE_MASK) << 2 * pin; + void __iomem *reg = ctrl->base + REALTEK_GPIO_REG_IMR; + unsigned int line_shift = ctrl->line_imr_pos(line); + unsigned int shift = line_shift % 32; + u32 irq_type = ctrl->intr_type[line]; + u32 irq_mask = ctrl->intr_mask[line]; + u32 reg_val; + + reg += 4 * (line_shift / 32); + reg_val = ioread32(reg); + reg_val &= ~(REALTEK_GPIO_IMR_LINE_MASK << shift); + reg_val |= (irq_type & irq_mask & REALTEK_GPIO_IMR_LINE_MASK) << shift; + iowrite32(reg_val, reg); } static void realtek_gpio_irq_ack(struct irq_data *data) { struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); irq_hw_number_t line = irqd_to_hwirq(data); - unsigned int port = line / 8; - unsigned int port_pin = line % 8; - realtek_gpio_clear_isr(ctrl, port, BIT(port_pin)); + realtek_gpio_clear_isr(ctrl, BIT(line)); } static void realtek_gpio_irq_unmask(struct irq_data *data) { struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); unsigned int line = irqd_to_hwirq(data); - unsigned int port = line / 8; - unsigned int port_pin = line % 8; unsigned long flags; - u16 m; gpiochip_enable_irq(&ctrl->gc, line); raw_spin_lock_irqsave(&ctrl->lock, flags); - m = ctrl->intr_mask[port]; - m |= realtek_gpio_imr_bits(port_pin, REALTEK_GPIO_IMR_LINE_MASK); - ctrl->intr_mask[port] = m; - realtek_gpio_write_imr(ctrl, port, ctrl->intr_type[port], m); + ctrl->intr_mask[line] = REALTEK_GPIO_IMR_LINE_MASK; + realtek_gpio_update_line_imr(ctrl, line); raw_spin_unlock_irqrestore(&ctrl->lock, flags); } @@ -186,16 +206,11 @@ static void realtek_gpio_irq_mask(struct irq_data *data) { struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); unsigned int line = irqd_to_hwirq(data); - unsigned int port = line / 8; - unsigned int port_pin = line % 8; unsigned long flags; - u16 m; raw_spin_lock_irqsave(&ctrl->lock, flags); - m = ctrl->intr_mask[port]; - m &= ~realtek_gpio_imr_bits(port_pin, REALTEK_GPIO_IMR_LINE_MASK); - ctrl->intr_mask[port] = m; - realtek_gpio_write_imr(ctrl, port, ctrl->intr_type[port], m); + ctrl->intr_mask[line] = 0; + realtek_gpio_update_line_imr(ctrl, line); raw_spin_unlock_irqrestore(&ctrl->lock, flags); gpiochip_disable_irq(&ctrl->gc, line); @@ -205,10 +220,8 @@ static int realtek_gpio_irq_set_type(struct irq_data *data, unsigned int flow_ty { struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); unsigned int line = irqd_to_hwirq(data); - unsigned int port = line / 8; - unsigned int port_pin = line % 8; unsigned long flags; - u16 type, t; + u8 type; switch (flow_type & IRQ_TYPE_SENSE_MASK) { case IRQ_TYPE_EDGE_FALLING: @@ -227,11 +240,8 @@ static int realtek_gpio_irq_set_type(struct irq_data *data, unsigned int flow_ty irq_set_handler_locked(data, handle_edge_irq); raw_spin_lock_irqsave(&ctrl->lock, flags); - t = ctrl->intr_type[port]; - t &= ~realtek_gpio_imr_bits(port_pin, REALTEK_GPIO_IMR_LINE_MASK); - t |= realtek_gpio_imr_bits(port_pin, type); - ctrl->intr_type[port] = t; - realtek_gpio_write_imr(ctrl, port, t, ctrl->intr_mask[port]); + ctrl->intr_type[line] = type; + realtek_gpio_update_line_imr(ctrl, line); raw_spin_unlock_irqrestore(&ctrl->lock, flags); return 0; @@ -242,28 +252,21 @@ static void realtek_gpio_irq_handler(struct irq_desc *desc) struct gpio_chip *gc = irq_desc_get_handler_data(desc); struct realtek_gpio_ctrl *ctrl = gpiochip_get_data(gc); struct irq_chip *irq_chip = irq_desc_get_chip(desc); - unsigned int lines_done; - unsigned int port_pin_count; unsigned long status; int offset; chained_irq_enter(irq_chip, desc); - for (lines_done = 0; lines_done < gc->ngpio; lines_done += 8) { - status = realtek_gpio_read_isr(ctrl, lines_done / 8); - port_pin_count = min(gc->ngpio - lines_done, 8U); - for_each_set_bit(offset, &status, port_pin_count) - generic_handle_domain_irq(gc->irq.domain, offset + lines_done); - } + status = realtek_gpio_read_isr(ctrl); + for_each_set_bit(offset, &status, gc->ngpio) + generic_handle_domain_irq(gc->irq.domain, offset); chained_irq_exit(irq_chip, desc); } -static inline void __iomem *realtek_gpio_irq_cpu_mask(struct realtek_gpio_ctrl *ctrl, - unsigned int port, int cpu) +static inline void __iomem *realtek_gpio_irq_cpu_mask(struct realtek_gpio_ctrl *ctrl, int cpu) { - return ctrl->cpumask_base + ctrl->port_offset_u8(port) + - REALTEK_GPIO_PORTS_PER_BANK * cpu; + return ctrl->cpumask_base + REALTEK_GPIO_PORTS_PER_BANK * cpu; } static int realtek_gpio_irq_set_affinity(struct irq_data *data, @@ -271,12 +274,10 @@ static int realtek_gpio_irq_set_affinity(struct irq_data *data, { struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); unsigned int line = irqd_to_hwirq(data); - unsigned int port = line / 8; - unsigned int port_pin = line % 8; void __iomem *irq_cpu_mask; unsigned long flags; int cpu; - u8 v; + u32 v; if (!ctrl->cpumask_base) return -ENXIO; @@ -284,15 +285,15 @@ static int realtek_gpio_irq_set_affinity(struct irq_data *data, raw_spin_lock_irqsave(&ctrl->lock, flags); for_each_cpu(cpu, &ctrl->cpu_irq_maskable) { - irq_cpu_mask = realtek_gpio_irq_cpu_mask(ctrl, port, cpu); - v = ioread8(irq_cpu_mask); + irq_cpu_mask = realtek_gpio_irq_cpu_mask(ctrl, cpu); + v = ctrl->bank_read(irq_cpu_mask); if (cpumask_test_cpu(cpu, dest)) - v |= BIT(port_pin); + v |= BIT(line); else - v &= ~BIT(port_pin); + v &= ~BIT(line); - iowrite8(v, irq_cpu_mask); + ctrl->bank_write(irq_cpu_mask, v); } raw_spin_unlock_irqrestore(&ctrl->lock, flags); @@ -305,16 +306,17 @@ static int realtek_gpio_irq_set_affinity(struct irq_data *data, static int realtek_gpio_irq_init(struct gpio_chip *gc) { struct realtek_gpio_ctrl *ctrl = gpiochip_get_data(gc); - unsigned int port; + u32 mask_all = GENMASK(gc->ngpio - 1, 0); + unsigned int line; int cpu; - for (port = 0; (port * 8) < gc->ngpio; port++) { - realtek_gpio_write_imr(ctrl, port, 0, 0); - realtek_gpio_clear_isr(ctrl, port, GENMASK(7, 0)); + for (line = 0; line < gc->ngpio; line++) + realtek_gpio_update_line_imr(ctrl, line); - for_each_cpu(cpu, &ctrl->cpu_irq_maskable) - iowrite8(GENMASK(7, 0), realtek_gpio_irq_cpu_mask(ctrl, port, cpu)); - } + realtek_gpio_clear_isr(ctrl, mask_all); + + for_each_cpu(cpu, &ctrl->cpu_irq_maskable) + ctrl->bank_write(realtek_gpio_irq_cpu_mask(ctrl, cpu), mask_all); return 0; } @@ -387,12 +389,14 @@ static int realtek_gpio_probe(struct platform_device *pdev) if (dev_flags & GPIO_PORTS_REVERSED) { bgpio_flags = 0; - ctrl->port_offset_u8 = realtek_gpio_port_offset_u8_rev; - ctrl->port_offset_u16 = realtek_gpio_port_offset_u16_rev; + ctrl->bank_read = realtek_gpio_bank_read; + ctrl->bank_write = realtek_gpio_bank_write; + ctrl->line_imr_pos = realtek_gpio_line_imr_pos; } else { bgpio_flags = BGPIOF_BIG_ENDIAN_BYTE_ORDER; - ctrl->port_offset_u8 = realtek_gpio_port_offset_u8; - ctrl->port_offset_u16 = realtek_gpio_port_offset_u16; + ctrl->bank_read = realtek_gpio_bank_read_swapped; + ctrl->bank_write = realtek_gpio_bank_write_swapped; + ctrl->line_imr_pos = realtek_gpio_line_imr_pos_swapped; } err = bgpio_init(&ctrl->gc, dev, 4, diff --git a/drivers/gpio/gpio-rockchip.c b/drivers/gpio/gpio-rockchip.c index f91e876fd9690cef5aa83775a2504402e6cfb42f..870910bb9dd35e32990f3a11a94f310b2376f597 100644 --- a/drivers/gpio/gpio-rockchip.c +++ b/drivers/gpio/gpio-rockchip.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -156,6 +157,12 @@ static int rockchip_gpio_set_direction(struct gpio_chip *chip, unsigned long flags; u32 data = input ? 0 : 1; + + if (input) + pinctrl_gpio_direction_input(bank->pin_base + offset); + else + pinctrl_gpio_direction_output(bank->pin_base + offset); + raw_spin_lock_irqsave(&bank->slock, flags); rockchip_gpio_writel_bit(bank, offset, data, bank->gpio_regs->port_ddr); raw_spin_unlock_irqrestore(&bank->slock, flags); @@ -325,26 +332,15 @@ static void rockchip_irq_demux(struct irq_desc *desc) { struct irq_chip *chip = irq_desc_get_chip(desc); struct rockchip_pin_bank *bank = irq_desc_get_handler_data(desc); - u32 pend; + unsigned long pending; + unsigned int irq; dev_dbg(bank->dev, "got irq for bank %s\n", bank->name); chained_irq_enter(chip, desc); - pend = readl_relaxed(bank->reg_base + bank->gpio_regs->int_status); - - while (pend) { - unsigned int irq, virq; - - irq = __ffs(pend); - pend &= ~BIT(irq); - virq = irq_find_mapping(bank->domain, irq); - - if (!virq) { - dev_err(bank->dev, "unmapped irq %d\n", irq); - continue; - } - + pending = readl_relaxed(bank->reg_base + bank->gpio_regs->int_status); + for_each_set_bit(irq, &pending, 32) { dev_dbg(bank->dev, "handling irq %d\n", irq); /* @@ -378,7 +374,7 @@ static void rockchip_irq_demux(struct irq_desc *desc) } while ((data & BIT(irq)) != (data_old & BIT(irq))); } - generic_handle_irq(virq); + generic_handle_domain_irq(bank->domain, irq); } chained_irq_exit(chip, desc); @@ -419,11 +415,11 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) goto out; } else { bank->toggle_edge_mode |= mask; - level |= mask; + level &= ~mask; /* * Determine gpio state. If 1 next interrupt should be - * falling otherwise rising. + * low otherwise high. */ data = readl(bank->reg_base + bank->gpio_regs->ext_port); if (data & mask) diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c index 443fe975bf1392e8bb2c8de2c340a463ec73333d..e62ee7e56908f9125ccb6deb21130a5d9043fbde 100644 --- a/drivers/gpio/gpio-tc3589x.c +++ b/drivers/gpio/gpio-tc3589x.c @@ -230,6 +230,7 @@ static void tc3589x_gpio_irq_mask(struct irq_data *d) tc3589x_gpio->regs[REG_IE][regoffset] &= ~mask; tc3589x_gpio->regs[REG_DIRECT][regoffset] |= mask; + gpiochip_disable_irq(gc, offset); } static void tc3589x_gpio_irq_unmask(struct irq_data *d) @@ -240,17 +241,20 @@ static void tc3589x_gpio_irq_unmask(struct irq_data *d) int regoffset = offset / 8; int mask = BIT(offset % 8); + gpiochip_enable_irq(gc, offset); tc3589x_gpio->regs[REG_IE][regoffset] |= mask; tc3589x_gpio->regs[REG_DIRECT][regoffset] &= ~mask; } -static struct irq_chip tc3589x_gpio_irq_chip = { +static const struct irq_chip tc3589x_gpio_irq_chip = { .name = "tc3589x-gpio", .irq_bus_lock = tc3589x_gpio_irq_lock, .irq_bus_sync_unlock = tc3589x_gpio_irq_sync_unlock, .irq_mask = tc3589x_gpio_irq_mask, .irq_unmask = tc3589x_gpio_irq_unmask, .irq_set_type = tc3589x_gpio_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static irqreturn_t tc3589x_gpio_irq(int irq, void *dev) @@ -321,7 +325,7 @@ static int tc3589x_gpio_probe(struct platform_device *pdev) tc3589x_gpio->chip.base = -1; girq = &tc3589x_gpio->chip.irq; - girq->chip = &tc3589x_gpio_irq_chip; + gpio_irq_chip_set_chip(girq, &tc3589x_gpio_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; diff --git a/drivers/gpio/gpio-tpic2810.c b/drivers/gpio/gpio-tpic2810.c index a09b1e69b07220d7f7dafd947e209e2bf6137958..d642c35cb97c0ce61bd40cc2a3c97ce7507e31c3 100644 --- a/drivers/gpio/gpio-tpic2810.c +++ b/drivers/gpio/gpio-tpic2810.c @@ -126,13 +126,11 @@ static int tpic2810_probe(struct i2c_client *client, return 0; } -static int tpic2810_remove(struct i2c_client *client) +static void tpic2810_remove(struct i2c_client *client) { struct tpic2810 *gpio = i2c_get_clientdata(client); gpiochip_remove(&gpio->chip); - - return 0; } static const struct i2c_device_id tpic2810_id_table[] = { diff --git a/drivers/gpio/gpio-tqmx86.c b/drivers/gpio/gpio-tqmx86.c index fa4bc7481f9a600b57d8ab6851ac9052c19cb49f..e739dcea61b231ff30387c12fef35ce59272d548 100644 --- a/drivers/gpio/gpio-tqmx86.c +++ b/drivers/gpio/gpio-tqmx86.c @@ -307,6 +307,8 @@ static int tqmx86_gpio_probe(struct platform_device *pdev) girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_simple_irq; girq->init_valid_mask = tqmx86_init_irq_valid_mask; + + irq_domain_set_pm_device(girq->domain, dev); } ret = devm_gpiochip_add_data(dev, chip, gpio); @@ -315,8 +317,6 @@ static int tqmx86_gpio_probe(struct platform_device *pdev) goto out_pm_dis; } - irq_domain_set_pm_device(girq->domain, dev); - dev_info(dev, "GPIO functionality initialized with %d pins\n", chip->ngpio); diff --git a/drivers/gpio/gpio-twl4030.c b/drivers/gpio/gpio-twl4030.c index 5046e51af8df27c77e4b6c71b55ec6e07fdf4391..c1bb2c3ca6f298308bd51d58d6f6e93b942978cf 100644 --- a/drivers/gpio/gpio-twl4030.c +++ b/drivers/gpio/gpio-twl4030.c @@ -465,8 +465,6 @@ static int gpio_twl4030_debounce(u32 debounce, u8 mmc_cd) REG_GPIO_DEBEN1, 3); } -static int gpio_twl4030_remove(struct platform_device *pdev); - static struct twl4030_gpio_platform_data *of_gpio_twl4030(struct device *dev, struct twl4030_gpio_platform_data *pdata) { @@ -494,6 +492,18 @@ static struct twl4030_gpio_platform_data *of_gpio_twl4030(struct device *dev, return omap_twl_info; } +/* Cannot use as gpio_twl4030_probe() calls us */ +static int gpio_twl4030_remove(struct platform_device *pdev) +{ + struct gpio_twl4030_priv *priv = platform_get_drvdata(pdev); + + gpiochip_remove(&priv->gpio_chip); + + /* REVISIT no support yet for deregistering all the IRQs */ + WARN_ON(!is_module()); + return 0; +} + static int gpio_twl4030_probe(struct platform_device *pdev) { struct twl4030_gpio_platform_data *pdata = dev_get_platdata(&pdev->dev); @@ -590,18 +600,6 @@ out: return ret; } -/* Cannot use as gpio_twl4030_probe() calls us */ -static int gpio_twl4030_remove(struct platform_device *pdev) -{ - struct gpio_twl4030_priv *priv = platform_get_drvdata(pdev); - - gpiochip_remove(&priv->gpio_chip); - - /* REVISIT no support yet for deregistering all the IRQs */ - WARN_ON(!is_module()); - return 0; -} - static const struct of_device_id twl_gpio_match[] = { { .compatible = "ti,twl4030-gpio", }, { }, diff --git a/drivers/gpio/gpio-ucb1400.c b/drivers/gpio/gpio-ucb1400.c index 386e69300332be316d79265ad879eb151328d7d1..676adf1f198ae3a6e9148b1f5cba31296e4e9b8c 100644 --- a/drivers/gpio/gpio-ucb1400.c +++ b/drivers/gpio/gpio-ucb1400.c @@ -7,6 +7,7 @@ #include #include +#include static int ucb1400_gpio_dir_in(struct gpio_chip *gc, unsigned off) { diff --git a/drivers/gpio/gpio-ws16c48.c b/drivers/gpio/gpio-ws16c48.c index b098f2dc196b311c7176086d76f8c3206c069a96..e73885a4dc32871ed634dc9548b1f63fd3ca5ed8 100644 --- a/drivers/gpio/gpio-ws16c48.c +++ b/drivers/gpio/gpio-ws16c48.c @@ -27,7 +27,8 @@ module_param_hw_array(base, uint, ioport, &num_ws16c48, 0); MODULE_PARM_DESC(base, "WinSystems WS16C48 base addresses"); static unsigned int irq[MAX_NUM_WS16C48]; -module_param_hw_array(irq, uint, irq, NULL, 0); +static unsigned int num_irq; +module_param_hw_array(irq, uint, irq, &num_irq, 0); MODULE_PARM_DESC(irq, "WinSystems WS16C48 interrupt line numbers"); /** @@ -265,6 +266,7 @@ static void ws16c48_irq_mask(struct irq_data *data) raw_spin_lock_irqsave(&ws16c48gpio->lock, flags); ws16c48gpio->irq_mask &= ~mask; + gpiochip_disable_irq(chip, offset); port_state = ws16c48gpio->irq_mask >> (8 * port); /* Select Register Page 2; Unlock all I/O ports */ @@ -295,6 +297,7 @@ static void ws16c48_irq_unmask(struct irq_data *data) raw_spin_lock_irqsave(&ws16c48gpio->lock, flags); + gpiochip_enable_irq(chip, offset); ws16c48gpio->irq_mask |= mask; port_state = ws16c48gpio->irq_mask >> (8 * port); @@ -356,12 +359,14 @@ static int ws16c48_irq_set_type(struct irq_data *data, unsigned flow_type) return 0; } -static struct irq_chip ws16c48_irqchip = { +static const struct irq_chip ws16c48_irqchip = { .name = "ws16c48", .irq_ack = ws16c48_irq_ack, .irq_mask = ws16c48_irq_mask, .irq_unmask = ws16c48_irq_unmask, - .irq_set_type = ws16c48_irq_set_type + .irq_set_type = ws16c48_irq_set_type, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static irqreturn_t ws16c48_irq_handler(int irq, void *dev_id) @@ -463,7 +468,7 @@ static int ws16c48_probe(struct device *dev, unsigned int id) ws16c48gpio->chip.set_multiple = ws16c48_gpio_set_multiple; girq = &ws16c48gpio->chip.irq; - girq->chip = &ws16c48_irqchip; + gpio_irq_chip_set_chip(girq, &ws16c48_irqchip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; @@ -497,7 +502,7 @@ static struct isa_driver ws16c48_driver = { }, }; -module_isa_driver(ws16c48_driver, num_ws16c48); +module_isa_driver_with_irq(ws16c48_driver, num_ws16c48, num_irq); MODULE_AUTHOR("William Breathitt Gray "); MODULE_DESCRIPTION("WinSystems WS16C48 GPIO driver"); diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 9be1376f9a627f493f61351edf44a00fd334e3a1..a7d2358736fe761bfdd08bc30f9c8b611f80fdb2 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -32,9 +32,16 @@ MODULE_PARM_DESC(ignore_wake, "controller@pin combos on which to ignore the ACPI wake flag " "ignore_wake=controller@pin[,controller@pin[,...]]"); +static char *ignore_interrupt; +module_param(ignore_interrupt, charp, 0444); +MODULE_PARM_DESC(ignore_interrupt, + "controller@pin combos on which to ignore interrupt " + "ignore_interrupt=controller@pin[,controller@pin[,...]]"); + struct acpi_gpiolib_dmi_quirk { bool no_edge_events_on_boot; char *ignore_wake; + char *ignore_interrupt; }; /** @@ -317,14 +324,15 @@ static struct gpio_desc *acpi_request_own_gpiod(struct gpio_chip *chip, return desc; } -static bool acpi_gpio_in_ignore_list(const char *controller_in, unsigned int pin_in) +static bool acpi_gpio_in_ignore_list(const char *ignore_list, const char *controller_in, + unsigned int pin_in) { const char *controller, *pin_str; unsigned int pin; char *endp; int len; - controller = ignore_wake; + controller = ignore_list; while (controller) { pin_str = strchr(controller, '@'); if (!pin_str) @@ -348,7 +356,7 @@ static bool acpi_gpio_in_ignore_list(const char *controller_in, unsigned int pin return false; err: - pr_err_once("Error: Invalid value for gpiolib_acpi.ignore_wake: %s\n", ignore_wake); + pr_err_once("Error: Invalid value for gpiolib_acpi.ignore_...: %s\n", ignore_list); return false; } @@ -360,7 +368,7 @@ static bool acpi_gpio_irq_is_wake(struct device *parent, if (agpio->wake_capable != ACPI_WAKE_CAPABLE) return false; - if (acpi_gpio_in_ignore_list(dev_name(parent), pin)) { + if (acpi_gpio_in_ignore_list(ignore_wake, dev_name(parent), pin)) { dev_info(parent, "Ignoring wakeup on pin %u\n", pin); return false; } @@ -427,6 +435,11 @@ static acpi_status acpi_gpiochip_alloc_event(struct acpi_resource *ares, goto fail_unlock_irq; } + if (acpi_gpio_in_ignore_list(ignore_interrupt, dev_name(chip->parent), pin)) { + dev_info(chip->parent, "Ignoring interrupt on pin %u\n", pin); + return AE_OK; + } + event = kzalloc(sizeof(*event), GFP_KERNEL); if (!event) goto fail_unlock_irq; @@ -741,6 +754,7 @@ static int acpi_populate_gpio_lookup(struct acpi_resource *ares, void *data) lookup->info.pin_config = agpio->pin_config; lookup->info.debounce = agpio->debounce_timeout; lookup->info.gpioint = gpioint; + lookup->info.wake_capable = agpio->wake_capable == ACPI_WAKE_CAPABLE; /* * Polarity and triggering are only specified for GpioInt @@ -987,10 +1001,11 @@ struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode, } /** - * acpi_dev_gpio_irq_get_by() - Find GpioInt and translate it to Linux IRQ number + * acpi_dev_gpio_irq_wake_get_by() - Find GpioInt and translate it to Linux IRQ number * @adev: pointer to a ACPI device to get IRQ from * @name: optional name of GpioInt resource * @index: index of GpioInt resource (starting from %0) + * @wake_capable: Set to true if the IRQ is wake capable * * If the device has one or more GpioInt resources, this function can be * used to translate from the GPIO offset in the resource to the Linux IRQ @@ -1002,9 +1017,13 @@ struct gpio_desc *acpi_node_get_gpiod(struct fwnode_handle *fwnode, * The function takes optional @name parameter. If the resource has a property * name, then only those will be taken into account. * + * The GPIO is considered wake capable if the GpioInt resource specifies + * SharedAndWake or ExclusiveAndWake. + * * Return: Linux IRQ number (> %0) on success, negative errno on failure. */ -int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, int index) +int acpi_dev_gpio_irq_wake_get_by(struct acpi_device *adev, const char *name, int index, + bool *wake_capable) { int idx, i; unsigned int irq_flags; @@ -1061,13 +1080,16 @@ int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, int ind dev_dbg(&adev->dev, "IRQ %d already in use\n", irq); } + if (wake_capable) + *wake_capable = info.wake_capable; + return irq; } } return -ENOENT; } -EXPORT_SYMBOL_GPL(acpi_dev_gpio_irq_get_by); +EXPORT_SYMBOL_GPL(acpi_dev_gpio_irq_wake_get_by); static acpi_status acpi_gpio_adr_space_handler(u32 function, acpi_physical_address address, @@ -1563,6 +1585,20 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] __initconst = { .ignore_wake = "INT33FF:01@0", }, }, + { + /* + * Interrupt storm caused from edge triggered floating pin + * Found in BIOS UX325UAZ.300 + * https://bugzilla.kernel.org/show_bug.cgi?id=216208 + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX325UAZ_UM325UAZ"), + }, + .driver_data = &(struct acpi_gpiolib_dmi_quirk) { + .ignore_interrupt = "AMDI0030:00@18", + }, + }, {} /* Terminating entry */ }; @@ -1585,6 +1621,9 @@ static int __init acpi_gpio_setup_params(void) if (ignore_wake == NULL && quirk && quirk->ignore_wake) ignore_wake = quirk->ignore_wake; + if (ignore_interrupt == NULL && quirk && quirk->ignore_interrupt) + ignore_interrupt = quirk->ignore_interrupt; + return 0; } diff --git a/drivers/gpio/gpiolib-acpi.h b/drivers/gpio/gpiolib-acpi.h index e476558d947136d12d62abd29c26ca6b9b2d741b..1ac6816839dbce7544843dcaa2b47e5826f041c8 100644 --- a/drivers/gpio/gpiolib-acpi.h +++ b/drivers/gpio/gpiolib-acpi.h @@ -18,6 +18,7 @@ struct acpi_device; * @pin_config: pin bias as provided by ACPI * @polarity: interrupt polarity as provided by ACPI * @triggering: triggering type as provided by ACPI + * @wake_capable: wake capability as provided by ACPI * @debounce: debounce timeout as provided by ACPI * @quirks: Linux specific quirks as provided by struct acpi_gpio_mapping */ @@ -28,6 +29,7 @@ struct acpi_gpio_info { int pin_config; int polarity; int triggering; + bool wake_capable; unsigned int debounce; unsigned int quirks; }; diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index f8041d4898d1916ab11428b37954e04788a1fc1b..0cb6b468f364f99c6ff4ed0e68b2718c52e74201 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -1497,6 +1497,21 @@ static int linereq_release(struct inode *inode, struct file *file) return 0; } +#ifdef CONFIG_PROC_FS +static void linereq_show_fdinfo(struct seq_file *out, struct file *file) +{ + struct linereq *lr = file->private_data; + struct device *dev = &lr->gdev->dev; + u16 i; + + seq_printf(out, "gpio-chip:\t%s\n", dev_name(dev)); + + for (i = 0; i < lr->num_lines; i++) + seq_printf(out, "gpio-line:\t%d\n", + gpio_chip_hwgpio(lr->lines[i].desc)); +} +#endif + static const struct file_operations line_fileops = { .release = linereq_release, .read = linereq_read, @@ -1507,6 +1522,9 @@ static const struct file_operations line_fileops = { #ifdef CONFIG_COMPAT .compat_ioctl = linereq_ioctl_compat, #endif +#ifdef CONFIG_PROC_FS + .show_fdinfo = linereq_show_fdinfo, +#endif }; static int linereq_create(struct gpio_device *gdev, void __user *ip) @@ -1986,7 +2004,6 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) ret = -ENODEV; goto out_free_le; } - le->irq = irq; if (eflags & GPIOEVENT_REQUEST_RISING_EDGE) irqflags |= test_bit(FLAG_ACTIVE_LOW, &desc->flags) ? @@ -2000,7 +2017,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) init_waitqueue_head(&le->wait); /* Request a thread to read the events */ - ret = request_threaded_irq(le->irq, + ret = request_threaded_irq(irq, lineevent_irq_handler, lineevent_irq_thread, irqflags, @@ -2009,6 +2026,8 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) if (ret) goto out_free_le; + le->irq = irq; + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); if (fd < 0) { ret = fd; diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index a037b50bef330d9ef774435316615b36f8bd8b15..0e4e1291604d67da9fcfd3f9f1c4da86f963eb90 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -289,6 +289,36 @@ int of_get_named_gpio_flags(const struct device_node *np, const char *list_name, } EXPORT_SYMBOL_GPL(of_get_named_gpio_flags); +/* Converts gpio_lookup_flags into bitmask of GPIO_* values */ +static unsigned long of_convert_gpio_flags(enum of_gpio_flags flags) +{ + unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT; + + if (flags & OF_GPIO_ACTIVE_LOW) + lflags |= GPIO_ACTIVE_LOW; + + if (flags & OF_GPIO_SINGLE_ENDED) { + if (flags & OF_GPIO_OPEN_DRAIN) + lflags |= GPIO_OPEN_DRAIN; + else + lflags |= GPIO_OPEN_SOURCE; + } + + if (flags & OF_GPIO_TRANSITORY) + lflags |= GPIO_TRANSITORY; + + if (flags & OF_GPIO_PULL_UP) + lflags |= GPIO_PULL_UP; + + if (flags & OF_GPIO_PULL_DOWN) + lflags |= GPIO_PULL_DOWN; + + if (flags & OF_GPIO_PULL_DISABLE) + lflags |= GPIO_PULL_DISABLE; + + return lflags; +} + /** * gpiod_get_from_of_node() - obtain a GPIO from an OF node * @node: handle of the OF node @@ -308,26 +338,14 @@ struct gpio_desc *gpiod_get_from_of_node(const struct device_node *node, enum gpiod_flags dflags, const char *label) { - unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT; + unsigned long lflags; struct gpio_desc *desc; - enum of_gpio_flags flags; - bool active_low = false; - bool single_ended = false; - bool open_drain = false; - bool transitory = false; + enum of_gpio_flags of_flags; int ret; - desc = of_get_named_gpiod_flags(node, propname, - index, &flags); - - if (!desc || IS_ERR(desc)) { + desc = of_get_named_gpiod_flags(node, propname, index, &of_flags); + if (!desc || IS_ERR(desc)) return desc; - } - - active_low = flags & OF_GPIO_ACTIVE_LOW; - single_ended = flags & OF_GPIO_SINGLE_ENDED; - open_drain = flags & OF_GPIO_OPEN_DRAIN; - transitory = flags & OF_GPIO_TRANSITORY; ret = gpiod_request(desc, label); if (ret == -EBUSY && (dflags & GPIOD_FLAGS_BIT_NONEXCLUSIVE)) @@ -335,27 +353,7 @@ struct gpio_desc *gpiod_get_from_of_node(const struct device_node *node, if (ret) return ERR_PTR(ret); - if (active_low) - lflags |= GPIO_ACTIVE_LOW; - - if (single_ended) { - if (open_drain) - lflags |= GPIO_OPEN_DRAIN; - else - lflags |= GPIO_OPEN_SOURCE; - } - - if (transitory) - lflags |= GPIO_TRANSITORY; - - if (flags & OF_GPIO_PULL_UP) - lflags |= GPIO_PULL_UP; - - if (flags & OF_GPIO_PULL_DOWN) - lflags |= GPIO_PULL_DOWN; - - if (flags & OF_GPIO_PULL_DISABLE) - lflags |= GPIO_PULL_DISABLE; + lflags = of_convert_gpio_flags(of_flags); ret = gpiod_configure_flags(desc, propname, lflags, dflags); if (ret < 0) { @@ -372,12 +370,12 @@ EXPORT_SYMBOL_GPL(gpiod_get_from_of_node); * properties should be named "foo-gpios" so we have this special kludge for * them. */ -static struct gpio_desc *of_find_spi_gpio(struct device *dev, const char *con_id, +static struct gpio_desc *of_find_spi_gpio(struct device_node *np, + const char *con_id, + unsigned int idx, enum of_gpio_flags *of_flags) { char prop_name[32]; /* 32 is max size of property name */ - const struct device_node *np = dev->of_node; - struct gpio_desc *desc; /* * Hopefully the compiler stubs the rest of the function if this @@ -393,8 +391,7 @@ static struct gpio_desc *of_find_spi_gpio(struct device *dev, const char *con_id /* Will be "gpio-sck", "gpio-mosi" or "gpio-miso" */ snprintf(prop_name, sizeof(prop_name), "%s-%s", "gpio", con_id); - desc = of_get_named_gpiod_flags(np, prop_name, 0, of_flags); - return desc; + return of_get_named_gpiod_flags(np, prop_name, idx, of_flags); } /* @@ -402,13 +399,11 @@ static struct gpio_desc *of_find_spi_gpio(struct device *dev, const char *con_id * lines rather than "cs-gpios" like all other SPI hardware. Account for this * with a special quirk. */ -static struct gpio_desc *of_find_spi_cs_gpio(struct device *dev, +static struct gpio_desc *of_find_spi_cs_gpio(struct device_node *np, const char *con_id, unsigned int idx, - unsigned long *flags) + enum of_gpio_flags *of_flags) { - const struct device_node *np = dev->of_node; - if (!IS_ENABLED(CONFIG_SPI_MASTER)) return ERR_PTR(-ENOENT); @@ -426,7 +421,7 @@ static struct gpio_desc *of_find_spi_cs_gpio(struct device *dev, * uses just "gpios" so translate to that when "cs-gpios" is * requested. */ - return of_find_gpio(dev, NULL, idx, flags); + return of_get_named_gpiod_flags(np, "gpios", idx, of_flags); } /* @@ -434,7 +429,9 @@ static struct gpio_desc *of_find_spi_cs_gpio(struct device *dev, * properties should be named "foo-gpios" so we have this special kludge for * them. */ -static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char *con_id, +static struct gpio_desc *of_find_regulator_gpio(struct device_node *np, + const char *con_id, + unsigned int idx, enum of_gpio_flags *of_flags) { /* These are the connection IDs we accept as legacy GPIO phandles */ @@ -443,8 +440,6 @@ static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char * "wlf,ldo1ena", /* WM8994 */ "wlf,ldo2ena", /* WM8994 */ }; - const struct device_node *np = dev->of_node; - struct gpio_desc *desc; int i; if (!IS_ENABLED(CONFIG_REGULATOR)) @@ -457,12 +452,12 @@ static struct gpio_desc *of_find_regulator_gpio(struct device *dev, const char * if (i < 0) return ERR_PTR(-ENOENT); - desc = of_get_named_gpiod_flags(np, con_id, 0, of_flags); - return desc; + return of_get_named_gpiod_flags(np, con_id, idx, of_flags); } -static struct gpio_desc *of_find_arizona_gpio(struct device *dev, +static struct gpio_desc *of_find_arizona_gpio(struct device_node *np, const char *con_id, + unsigned int idx, enum of_gpio_flags *of_flags) { if (!IS_ENABLED(CONFIG_MFD_ARIZONA)) @@ -471,17 +466,18 @@ static struct gpio_desc *of_find_arizona_gpio(struct device *dev, if (!con_id || strcmp(con_id, "wlf,reset")) return ERR_PTR(-ENOENT); - return of_get_named_gpiod_flags(dev->of_node, con_id, 0, of_flags); + return of_get_named_gpiod_flags(np, con_id, idx, of_flags); } -static struct gpio_desc *of_find_usb_gpio(struct device *dev, +static struct gpio_desc *of_find_usb_gpio(struct device_node *np, const char *con_id, + unsigned int idx, enum of_gpio_flags *of_flags) { /* - * Currently this USB quirk is only for the Fairchild FUSB302 host which is using - * an undocumented DT GPIO line named "fcs,int_n" without the compulsory "-gpios" - * suffix. + * Currently this USB quirk is only for the Fairchild FUSB302 host + * which is using an undocumented DT GPIO line named "fcs,int_n" + * without the compulsory "-gpios" suffix. */ if (!IS_ENABLED(CONFIG_TYPEC_FUSB302)) return ERR_PTR(-ENOENT); @@ -489,14 +485,28 @@ static struct gpio_desc *of_find_usb_gpio(struct device *dev, if (!con_id || strcmp(con_id, "fcs,int_n")) return ERR_PTR(-ENOENT); - return of_get_named_gpiod_flags(dev->of_node, con_id, 0, of_flags); + return of_get_named_gpiod_flags(np, con_id, idx, of_flags); } +typedef struct gpio_desc *(*of_find_gpio_quirk)(struct device_node *np, + const char *con_id, + unsigned int idx, + enum of_gpio_flags *of_flags); +static const of_find_gpio_quirk of_find_gpio_quirks[] = { + of_find_spi_gpio, + of_find_spi_cs_gpio, + of_find_regulator_gpio, + of_find_arizona_gpio, + of_find_usb_gpio, + NULL +}; + struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, unsigned int idx, unsigned long *flags) { char prop_name[32]; /* 32 is max size of property name */ enum of_gpio_flags of_flags; + const of_find_gpio_quirk *q; struct gpio_desc *desc; unsigned int i; @@ -516,51 +526,14 @@ struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, break; } - if (gpiod_not_found(desc)) { - /* Special handling for SPI GPIOs if used */ - desc = of_find_spi_gpio(dev, con_id, &of_flags); - } - - if (gpiod_not_found(desc)) { - /* This quirk looks up flags and all */ - desc = of_find_spi_cs_gpio(dev, con_id, idx, flags); - if (!IS_ERR(desc)) - return desc; - } - - if (gpiod_not_found(desc)) { - /* Special handling for regulator GPIOs if used */ - desc = of_find_regulator_gpio(dev, con_id, &of_flags); - } - - if (gpiod_not_found(desc)) - desc = of_find_arizona_gpio(dev, con_id, &of_flags); - - if (gpiod_not_found(desc)) - desc = of_find_usb_gpio(dev, con_id, &of_flags); + /* Properly named GPIO was not found, try workarounds */ + for (q = of_find_gpio_quirks; gpiod_not_found(desc) && *q; q++) + desc = (*q)(dev->of_node, con_id, idx, &of_flags); if (IS_ERR(desc)) return desc; - if (of_flags & OF_GPIO_ACTIVE_LOW) - *flags |= GPIO_ACTIVE_LOW; - - if (of_flags & OF_GPIO_SINGLE_ENDED) { - if (of_flags & OF_GPIO_OPEN_DRAIN) - *flags |= GPIO_OPEN_DRAIN; - else - *flags |= GPIO_OPEN_SOURCE; - } - - if (of_flags & OF_GPIO_TRANSITORY) - *flags |= GPIO_TRANSITORY; - - if (of_flags & OF_GPIO_PULL_UP) - *flags |= GPIO_PULL_UP; - if (of_flags & OF_GPIO_PULL_DOWN) - *flags |= GPIO_PULL_DOWN; - if (of_flags & OF_GPIO_PULL_DISABLE) - *flags |= GPIO_PULL_DISABLE; + *flags = of_convert_gpio_flags(of_flags); return desc; } @@ -618,16 +591,7 @@ static struct gpio_desc *of_parse_own_gpio(struct device_node *np, if (IS_ERR(desc)) return desc; - if (xlate_flags & OF_GPIO_ACTIVE_LOW) - *lflags |= GPIO_ACTIVE_LOW; - if (xlate_flags & OF_GPIO_TRANSITORY) - *lflags |= GPIO_TRANSITORY; - if (xlate_flags & OF_GPIO_PULL_UP) - *lflags |= GPIO_PULL_UP; - if (xlate_flags & OF_GPIO_PULL_DOWN) - *lflags |= GPIO_PULL_DOWN; - if (xlate_flags & OF_GPIO_PULL_DISABLE) - *lflags |= GPIO_PULL_DISABLE; + *lflags = of_convert_gpio_flags(xlate_flags); if (of_property_read_bool(np, "input")) *dflags |= GPIOD_IN; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index cc9c0a12259e15f0c5d73b0ee42986d931587605..4756ea08894f652566bcb7bef868560efebc683c 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -3798,6 +3798,72 @@ static int platform_gpio_count(struct device *dev, const char *con_id) return count; } +/** + * fwnode_get_named_gpiod - obtain a GPIO from firmware node + * @fwnode: handle of the firmware node + * @propname: name of the firmware property representing the GPIO + * @index: index of the GPIO to obtain for the consumer + * @dflags: GPIO initialization flags + * @label: label to attach to the requested GPIO + * + * This function can be used for drivers that get their configuration + * from opaque firmware. + * + * The function properly finds the corresponding GPIO using whatever is the + * underlying firmware interface and then makes sure that the GPIO + * descriptor is requested before it is returned to the caller. + * + * Returns: + * On successful request the GPIO pin is configured in accordance with + * provided @dflags. + * + * In case of error an ERR_PTR() is returned. + */ +static struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, + const char *propname, int index, + enum gpiod_flags dflags, + const char *label) +{ + unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT; + struct gpio_desc *desc = ERR_PTR(-ENODEV); + int ret; + + if (is_of_node(fwnode)) { + desc = gpiod_get_from_of_node(to_of_node(fwnode), + propname, index, + dflags, + label); + return desc; + } else if (is_acpi_node(fwnode)) { + struct acpi_gpio_info info; + + desc = acpi_node_get_gpiod(fwnode, propname, index, &info); + if (IS_ERR(desc)) + return desc; + + acpi_gpio_update_gpiod_flags(&dflags, &info); + acpi_gpio_update_gpiod_lookup_flags(&lflags, &info); + } else { + return ERR_PTR(-EINVAL); + } + + /* Currently only ACPI takes this path */ + ret = gpiod_request(desc, label); + if (ret) + return ERR_PTR(ret); + + ret = gpiod_configure_flags(desc, propname, lflags, dflags); + if (ret < 0) { + gpiod_put(desc); + return ERR_PTR(ret); + } + + blocking_notifier_call_chain(&desc->gdev->notifier, + GPIOLINE_CHANGED_REQUESTED, desc); + + return desc; +} + /** * fwnode_gpiod_get_index - obtain a GPIO from firmware node * @fwnode: handle of the firmware node @@ -4063,72 +4129,6 @@ struct gpio_desc *__must_check gpiod_get_index(struct device *dev, } EXPORT_SYMBOL_GPL(gpiod_get_index); -/** - * fwnode_get_named_gpiod - obtain a GPIO from firmware node - * @fwnode: handle of the firmware node - * @propname: name of the firmware property representing the GPIO - * @index: index of the GPIO to obtain for the consumer - * @dflags: GPIO initialization flags - * @label: label to attach to the requested GPIO - * - * This function can be used for drivers that get their configuration - * from opaque firmware. - * - * The function properly finds the corresponding GPIO using whatever is the - * underlying firmware interface and then makes sure that the GPIO - * descriptor is requested before it is returned to the caller. - * - * Returns: - * On successful request the GPIO pin is configured in accordance with - * provided @dflags. - * - * In case of error an ERR_PTR() is returned. - */ -struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, - const char *propname, int index, - enum gpiod_flags dflags, - const char *label) -{ - unsigned long lflags = GPIO_LOOKUP_FLAGS_DEFAULT; - struct gpio_desc *desc = ERR_PTR(-ENODEV); - int ret; - - if (is_of_node(fwnode)) { - desc = gpiod_get_from_of_node(to_of_node(fwnode), - propname, index, - dflags, - label); - return desc; - } else if (is_acpi_node(fwnode)) { - struct acpi_gpio_info info; - - desc = acpi_node_get_gpiod(fwnode, propname, index, &info); - if (IS_ERR(desc)) - return desc; - - acpi_gpio_update_gpiod_flags(&dflags, &info); - acpi_gpio_update_gpiod_lookup_flags(&lflags, &info); - } else - return ERR_PTR(-EINVAL); - - /* Currently only ACPI takes this path */ - ret = gpiod_request(desc, label); - if (ret) - return ERR_PTR(ret); - - ret = gpiod_configure_flags(desc, propname, lflags, dflags); - if (ret < 0) { - gpiod_put(desc); - return ERR_PTR(ret); - } - - blocking_notifier_call_chain(&desc->gdev->notifier, - GPIOLINE_CHANGED_REQUESTED, desc); - - return desc; -} -EXPORT_SYMBOL_GPL(fwnode_get_named_gpiod); - /** * gpiod_get_index_optional - obtain an optional GPIO from a multi-index GPIO * function diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 198ba846d34bf7b5f2c143a9b443ef912354852d..34f5a092c99e72f3a3b8744fa5b71f80de96e9b6 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -51,6 +51,18 @@ config DRM_DEBUG_MM If in doubt, say "N". +config DRM_USE_DYNAMIC_DEBUG + bool "use dynamic debug to implement drm.debug" + default y + depends on DRM + depends on DYNAMIC_DEBUG || DYNAMIC_DEBUG_CORE + depends on JUMP_LABEL + help + Use dynamic-debug to avoid drm_debug_enabled() runtime overheads. + Due to callsite counts in DRM drivers (~4k in amdgpu) and 56 + bytes per callsite, the .data costs can be substantial, and + are therefore configurable. + config DRM_KUNIT_TEST tristate "KUnit tests for DRM" if !KUNIT_ALL_TESTS depends on DRM && KUNIT diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 6ad98d3ceff76d6fa259ed5fbe1bedc11cb22648..6e55c47288e4232773b7aa5c35aeaee10e85bfe4 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -3,6 +3,8 @@ # Makefile for the drm device driver. This driver provides support for the # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. +CFLAGS-$(CONFIG_DRM_USE_DYNAMIC_DEBUG) += -DDYNAMIC_DEBUG_MODULE + drm-y := \ drm_aperture.o \ drm_atomic.o \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 79bb6fd830949dc7caeddd5b83cfe7bcbbb43c93..ae9371b172e3a53acc567b6ad24427b28bb041f8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -885,6 +885,7 @@ struct amdgpu_device { u64 fence_context; unsigned num_rings; struct amdgpu_ring *rings[AMDGPU_MAX_RINGS]; + struct dma_fence __rcu *gang_submit; bool ib_pool_ready; struct amdgpu_sa_manager ib_pools[AMDGPU_IB_POOL_MAX]; struct amdgpu_sched gpu_sched[AMDGPU_HW_IP_NUM][AMDGPU_RING_PRIO_MAX]; @@ -1294,6 +1295,8 @@ u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, u32 reg); void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, u32 reg, u32 v); +struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, + struct dma_fence *gang); /* atpx handler */ #if defined(CONFIG_VGA_SWITCHEROO) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c index 55402d238919369e177a21d1505cc2ab922df585..b14800ac179ee1054fa9d827c03c764119192d2f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* * Copyright 2012 Advanced Micro Devices, Inc. * @@ -849,6 +850,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev) if (amdgpu_device_has_dc_support(adev)) { #if defined(CONFIG_DRM_AMD_DC) struct amdgpu_display_manager *dm = &adev->dm; + if (dm->backlight_dev[0]) atif->bd = dm->backlight_dev[0]; #endif @@ -863,6 +865,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev) if ((enc->devices & (ATOM_DEVICE_LCD_SUPPORT)) && enc->enc_priv) { struct amdgpu_encoder_atom_dig *dig = enc->enc_priv; + if (dig->bl_dev) { atif->bd = dig->bl_dev; break; @@ -919,9 +922,9 @@ static bool amdgpu_atif_pci_probe_handle(struct pci_dev *pdev) return false; status = acpi_get_handle(dhandle, "ATIF", &atif_handle); - if (ACPI_FAILURE(status)) { + if (ACPI_FAILURE(status)) return false; - } + amdgpu_acpi_priv.atif.handle = atif_handle; acpi_get_name(amdgpu_acpi_priv.atif.handle, ACPI_FULL_PATHNAME, &buffer); DRM_DEBUG_DRIVER("Found ATIF handle %s\n", acpi_method_name); @@ -954,9 +957,9 @@ static bool amdgpu_atcs_pci_probe_handle(struct pci_dev *pdev) return false; status = acpi_get_handle(dhandle, "ATCS", &atcs_handle); - if (ACPI_FAILURE(status)) { + if (ACPI_FAILURE(status)) return false; - } + amdgpu_acpi_priv.atcs.handle = atcs_handle; acpi_get_name(amdgpu_acpi_priv.atcs.handle, ACPI_FULL_PATHNAME, &buffer); DRM_DEBUG_DRIVER("Found ATCS handle %s\n", acpi_method_name); @@ -1050,6 +1053,10 @@ bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev) { if (adev->flags & AMD_IS_APU) return false; + + if (amdgpu_sriov_vf(adev)) + return false; + return pm_suspend_target_state != PM_SUSPEND_TO_IDLE; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 091415a4abf02b203c019f36e504168062bbfb20..03bbfaa51cbcb85cb4e85d9516f7432e89c673ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* * Copyright 2014 Advanced Micro Devices, Inc. * @@ -74,9 +75,6 @@ void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) return; adev->kfd.dev = kgd2kfd_probe(adev, vf); - - if (adev->kfd.dev) - amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; } /** @@ -130,6 +128,7 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work) kfd.reset_work); struct amdgpu_reset_context reset_context; + memset(&reset_context, 0, sizeof(reset_context)); reset_context.method = AMD_RESET_METHOD_NONE; @@ -199,6 +198,8 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev, adev_to_drm(adev), &gpu_resources); + amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; + INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work); } } @@ -208,6 +209,7 @@ void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev) if (adev->kfd.dev) { kgd2kfd_device_exit(adev->kfd.dev); adev->kfd.dev = NULL; + amdgpu_amdkfd_total_mem_size -= adev->gmc.real_vram_size; } } @@ -684,6 +686,7 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, ib->length_dw = ib_len; /* This works for NO_HWS. TODO: need to handle without knowing VMID */ job->vmid = vmid; + job->num_ibs = 1; ret = amdgpu_ib_schedule(ring, 1, ib, job, &f); @@ -753,11 +756,7 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev, bo { struct ras_err_data err_data = {0, 0, 0, NULL}; - /* CPU MCA will handle page retirement if connected_to_cpu is 1 */ - if (!adev->gmc.xgmi.connected_to_cpu) - amdgpu_umc_poison_handler(adev, &err_data, reset); - else if (reset) - amdgpu_amdkfd_gpu_reset(adev); + amdgpu_umc_poison_handler(adev, &err_data, reset); } bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 2170db83e41d95dcde7e2e069f1171bcbda7d061..978d3970b5cc49f9b86bd5d123fde63e4c8c1fcb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* * Copyright 2014-2018 Advanced Micro Devices, Inc. * @@ -297,7 +298,7 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, */ replacement = dma_fence_get_stub(); dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, - replacement, DMA_RESV_USAGE_READ); + replacement, DMA_RESV_USAGE_BOOKKEEP); dma_fence_put(replacement); return 0; } @@ -1390,8 +1391,9 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); if (ret) goto reserve_shared_fail; - amdgpu_bo_fence(vm->root.bo, - &vm->process_info->eviction_fence->base, true); + dma_resv_add_fence(vm->root.bo->tbo.base.resv, + &vm->process_info->eviction_fence->base, + DMA_RESV_USAGE_BOOKKEEP); amdgpu_bo_unreserve(vm->root.bo); /* Update process info */ @@ -1612,6 +1614,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev) uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); size_t available; + spin_lock(&kfd_mem_limit.mem_limit_lock); available = adev->gmc.real_vram_size - adev->kfd.vram_used_aligned @@ -1987,9 +1990,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( } if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count) - amdgpu_bo_fence(bo, - &avm->process_info->eviction_fence->base, - true); + dma_resv_add_fence(bo->tbo.base.resv, + &avm->process_info->eviction_fence->base, + DMA_RESV_USAGE_BOOKKEEP); ret = unreserve_bo_and_vms(&ctx, false, false); goto out; @@ -2216,7 +2219,7 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, { if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { *mem = *adev->gmc.vm_fault_info; - mb(); + mb(); /* make sure read happened */ atomic_set(&adev->gmc.vm_fault_info_updated, 0); } return 0; @@ -2758,15 +2761,18 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) if (mem->bo->tbo.pin_count) continue; - amdgpu_bo_fence(mem->bo, - &process_info->eviction_fence->base, true); + dma_resv_add_fence(mem->bo->tbo.base.resv, + &process_info->eviction_fence->base, + DMA_RESV_USAGE_BOOKKEEP); } /* Attach eviction fence to PD / PT BOs */ list_for_each_entry(peer_vm, &process_info->vm_list_head, vm_list_node) { struct amdgpu_bo *bo = peer_vm->root.bo; - amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); + dma_resv_add_fence(bo->tbo.base.resv, + &process_info->eviction_fence->base, + DMA_RESV_USAGE_BOOKKEEP); } validate_map_fail: @@ -2820,7 +2826,9 @@ int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1); if (ret) goto reserve_shared_fail; - amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true); + dma_resv_add_fence(gws_bo->tbo.base.resv, + &process_info->eviction_fence->base, + DMA_RESV_USAGE_BOOKKEEP); amdgpu_bo_unreserve(gws_bo); mutex_unlock(&(*mem)->process_info->lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index b7933c2ce765c950ca8703e95f7e577ddf5be63d..491d4846fc02c8130c879348d282973acebc5900 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -1674,10 +1674,12 @@ amdgpu_connector_add(struct amdgpu_device *adev, adev->mode_info.dither_property, AMDGPU_FMT_DITHER_DISABLE); - if (amdgpu_audio != 0) + if (amdgpu_audio != 0) { drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.audio_property, AMDGPU_AUDIO_AUTO); + amdgpu_connector->audio = AMDGPU_AUDIO_AUTO; + } subpixel_order = SubPixelHorizontalRGB; connector->interlace_allowed = true; @@ -1799,6 +1801,7 @@ amdgpu_connector_add(struct amdgpu_device *adev, drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.audio_property, AMDGPU_AUDIO_AUTO); + amdgpu_connector->audio = AMDGPU_AUDIO_AUTO; } drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.dither_property, @@ -1852,6 +1855,7 @@ amdgpu_connector_add(struct amdgpu_device *adev, drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.audio_property, AMDGPU_AUDIO_AUTO); + amdgpu_connector->audio = AMDGPU_AUDIO_AUTO; } drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.dither_property, @@ -1902,6 +1906,7 @@ amdgpu_connector_add(struct amdgpu_device *adev, drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.audio_property, AMDGPU_AUDIO_AUTO); + amdgpu_connector->audio = AMDGPU_AUDIO_AUTO; } drm_object_attach_property(&amdgpu_connector->base.base, adev->mode_info.dither_property, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index b7bae833c804b02b05478fc509fa6d2eb3807341..1bbd39b3b0fc4aed0bb7959fa4bf175dfe64417b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -39,9 +39,82 @@ #include "amdgpu_gem.h" #include "amdgpu_ras.h" -static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p, - struct drm_amdgpu_cs_chunk_fence *data, - uint32_t *offset) +static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, + struct amdgpu_device *adev, + struct drm_file *filp, + union drm_amdgpu_cs *cs) +{ + struct amdgpu_fpriv *fpriv = filp->driver_priv; + + if (cs->in.num_chunks == 0) + return -EINVAL; + + memset(p, 0, sizeof(*p)); + p->adev = adev; + p->filp = filp; + + p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); + if (!p->ctx) + return -EINVAL; + + if (atomic_read(&p->ctx->guilty)) { + amdgpu_ctx_put(p->ctx); + return -ECANCELED; + } + return 0; +} + +static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p, + struct drm_amdgpu_cs_chunk_ib *chunk_ib) +{ + struct drm_sched_entity *entity; + unsigned int i; + int r; + + r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type, + chunk_ib->ip_instance, + chunk_ib->ring, &entity); + if (r) + return r; + + /* + * Abort if there is no run queue associated with this entity. + * Possibly because of disabled HW IP. + */ + if (entity->rq == NULL) + return -EINVAL; + + /* Check if we can add this IB to some existing job */ + for (i = 0; i < p->gang_size; ++i) + if (p->entities[i] == entity) + return i; + + /* If not increase the gang size if possible */ + if (i == AMDGPU_CS_GANG_SIZE) + return -EINVAL; + + p->entities[i] = entity; + p->gang_size = i + 1; + return i; +} + +static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p, + struct drm_amdgpu_cs_chunk_ib *chunk_ib, + unsigned int *num_ibs) +{ + int r; + + r = amdgpu_cs_job_idx(p, chunk_ib); + if (r < 0) + return r; + + ++(num_ibs[r]); + return 0; +} + +static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p, + struct drm_amdgpu_cs_chunk_fence *data, + uint32_t *offset) { struct drm_gem_object *gobj; struct amdgpu_bo *bo; @@ -80,11 +153,11 @@ error_unref: return r; } -static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, - struct drm_amdgpu_bo_list_in *data) +static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p, + struct drm_amdgpu_bo_list_in *data) { + struct drm_amdgpu_bo_list_entry *info; int r; - struct drm_amdgpu_bo_list_entry *info = NULL; r = amdgpu_bo_create_list_entry_array(data, &info); if (r) @@ -104,38 +177,25 @@ error_free: return r; } -static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) +/* Copy the data from userspace and go over it the first time */ +static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, + union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { }; struct amdgpu_vm *vm = &fpriv->vm; uint64_t *chunk_array_user; uint64_t *chunk_array; - unsigned size, num_ibs = 0; uint32_t uf_offset = 0; - int i; + unsigned int size; int ret; + int i; - if (cs->in.num_chunks == 0) - return -EINVAL; - - chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); + chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), + GFP_KERNEL); if (!chunk_array) return -ENOMEM; - p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); - if (!p->ctx) { - ret = -EINVAL; - goto free_chunk; - } - - mutex_lock(&p->ctx->lock); - - /* skip guilty context job */ - if (atomic_read(&p->ctx->guilty) == 1) { - ret = -ECANCELED; - goto free_chunk; - } - /* get chunks */ chunk_array_user = u64_to_user_ptr(cs->in.chunks); if (copy_from_user(chunk_array, chunk_array_user, @@ -170,7 +230,8 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs size = p->chunks[i].length_dw; cdata = u64_to_user_ptr(user_chunk.chunk_data); - p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL); + p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), + GFP_KERNEL); if (p->chunks[i].kdata == NULL) { ret = -ENOMEM; i--; @@ -182,36 +243,35 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs goto free_partial_kdata; } + /* Assume the worst on the following checks */ + ret = -EINVAL; switch (p->chunks[i].chunk_id) { case AMDGPU_CHUNK_ID_IB: - ++num_ibs; + if (size < sizeof(struct drm_amdgpu_cs_chunk_ib)) + goto free_partial_kdata; + + ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs); + if (ret) + goto free_partial_kdata; break; case AMDGPU_CHUNK_ID_FENCE: - size = sizeof(struct drm_amdgpu_cs_chunk_fence); - if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { - ret = -EINVAL; + if (size < sizeof(struct drm_amdgpu_cs_chunk_fence)) goto free_partial_kdata; - } - ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata, - &uf_offset); + ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata, + &uf_offset); if (ret) goto free_partial_kdata; - break; case AMDGPU_CHUNK_ID_BO_HANDLES: - size = sizeof(struct drm_amdgpu_bo_list_in); - if (p->chunks[i].length_dw * sizeof(uint32_t) < size) { - ret = -EINVAL; + if (size < sizeof(struct drm_amdgpu_bo_list_in)) goto free_partial_kdata; - } - ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata); + ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata); if (ret) goto free_partial_kdata; - break; case AMDGPU_CHUNK_ID_DEPENDENCIES: @@ -223,22 +283,32 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs break; default: - ret = -EINVAL; goto free_partial_kdata; } } - ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm); - if (ret) - goto free_all_kdata; + if (!p->gang_size) + return -EINVAL; + + for (i = 0; i < p->gang_size; ++i) { + ret = amdgpu_job_alloc(p->adev, num_ibs[i], &p->jobs[i], vm); + if (ret) + goto free_all_kdata; + + ret = drm_sched_job_init(&p->jobs[i]->base, p->entities[i], + &fpriv->vm); + if (ret) + goto free_all_kdata; + } + p->gang_leader = p->jobs[p->gang_size - 1]; - if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) { + if (p->ctx->vram_lost_counter != p->gang_leader->vram_lost_counter) { ret = -ECANCELED; goto free_all_kdata; } if (p->uf_entry.tv.bo) - p->job->uf_addr = uf_offset; + p->gang_leader->uf_addr = uf_offset; kvfree(chunk_array); /* Use this opportunity to fill in task info for the vm */ @@ -260,943 +330,880 @@ free_chunk: return ret; } -/* Convert microseconds to bytes. */ -static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) +static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk, + unsigned int *ce_preempt, + unsigned int *de_preempt) { - if (us <= 0 || !adev->mm_stats.log2_max_MBps) - return 0; + struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata; + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_ring *ring; + struct amdgpu_job *job; + struct amdgpu_ib *ib; + int r; - /* Since accum_us is incremented by a million per second, just - * multiply it by the number of MB/s to get the number of bytes. - */ - return us << adev->mm_stats.log2_max_MBps; -} + r = amdgpu_cs_job_idx(p, chunk_ib); + if (r < 0) + return r; -static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) -{ - if (!adev->mm_stats.log2_max_MBps) - return 0; + job = p->jobs[r]; + ring = amdgpu_job_ring(job); + ib = &job->ibs[job->num_ibs++]; - return bytes >> adev->mm_stats.log2_max_MBps; -} + /* MM engine doesn't support user fences */ + if (p->uf_entry.tv.bo && ring->funcs->no_user_fence) + return -EINVAL; -/* Returns how many bytes TTM can move right now. If no bytes can be moved, - * it returns 0. If it returns non-zero, it's OK to move at least one buffer, - * which means it can go over the threshold once. If that happens, the driver - * will be in debt and no other buffer migrations can be done until that debt - * is repaid. - * - * This approach allows moving a buffer of any size (it's important to allow - * that). - * - * The currency is simply time in microseconds and it increases as the clock - * ticks. The accumulated microseconds (us) are converted to bytes and - * returned. - */ -static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, - u64 *max_bytes, - u64 *max_vis_bytes) -{ - s64 time_us, increment_us; - u64 free_vram, total_vram, used_vram; - /* Allow a maximum of 200 accumulated ms. This is basically per-IB - * throttling. - * - * It means that in order to get full max MBps, at least 5 IBs per - * second must be submitted and not more than 200ms apart from each - * other. - */ - const s64 us_upper_bound = 200000; + if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && + chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { + if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) + (*ce_preempt)++; + else + (*de_preempt)++; - if (!adev->mm_stats.log2_max_MBps) { - *max_bytes = 0; - *max_vis_bytes = 0; - return; + /* Each GFX command submit allows only 1 IB max + * preemptible for CE & DE */ + if (*ce_preempt > 1 || *de_preempt > 1) + return -EINVAL; } - total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); - used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); - free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; + if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) + job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; - spin_lock(&adev->mm_stats.lock); + r = amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ? + chunk_ib->ib_bytes : 0, + AMDGPU_IB_POOL_DELAYED, ib); + if (r) { + DRM_ERROR("Failed to get ib !\n"); + return r; + } - /* Increase the amount of accumulated us. */ - time_us = ktime_to_us(ktime_get()); - increment_us = time_us - adev->mm_stats.last_update_us; - adev->mm_stats.last_update_us = time_us; - adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, - us_upper_bound); + ib->gpu_addr = chunk_ib->va_start; + ib->length_dw = chunk_ib->ib_bytes / 4; + ib->flags = chunk_ib->flags; + return 0; +} - /* This prevents the short period of low performance when the VRAM - * usage is low and the driver is in debt or doesn't have enough - * accumulated us to fill VRAM quickly. - * - * The situation can occur in these cases: - * - a lot of VRAM is freed by userspace - * - the presence of a big buffer causes a lot of evictions - * (solution: split buffers into smaller ones) - * - * If 128 MB or 1/8th of VRAM is free, start filling it now by setting - * accum_us to a positive number. - */ - if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { - s64 min_us; +static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata; + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + unsigned num_deps; + int i, r; - /* Be more aggressive on dGPUs. Try to fill a portion of free - * VRAM now. - */ - if (!(adev->flags & AMD_IS_APU)) - min_us = bytes_to_us(adev, free_vram / 4); - else - min_us = 0; /* Reset accum_us on APUs. */ + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_dep); - adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); - } + for (i = 0; i < num_deps; ++i) { + struct amdgpu_ctx *ctx; + struct drm_sched_entity *entity; + struct dma_fence *fence; - /* This is set to 0 if the driver is in debt to disallow (optional) - * buffer moves. - */ - *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); + ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); + if (ctx == NULL) + return -EINVAL; - /* Do the same for visible VRAM if half of it is free */ - if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { - u64 total_vis_vram = adev->gmc.visible_vram_size; - u64 used_vis_vram = - amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); + r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, + deps[i].ip_instance, + deps[i].ring, &entity); + if (r) { + amdgpu_ctx_put(ctx); + return r; + } - if (used_vis_vram < total_vis_vram) { - u64 free_vis_vram = total_vis_vram - used_vis_vram; - adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + - increment_us, us_upper_bound); + fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); + amdgpu_ctx_put(ctx); - if (free_vis_vram >= total_vis_vram / 2) - adev->mm_stats.accum_us_vis = - max(bytes_to_us(adev, free_vis_vram / 2), - adev->mm_stats.accum_us_vis); - } + if (IS_ERR(fence)) + return PTR_ERR(fence); + else if (!fence) + continue; - *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); - } else { - *max_vis_bytes = 0; - } + if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { + struct drm_sched_fence *s_fence; + struct dma_fence *old = fence; - spin_unlock(&adev->mm_stats.lock); -} + s_fence = to_drm_sched_fence(fence); + fence = dma_fence_get(&s_fence->scheduled); + dma_fence_put(old); + } -/* Report how many bytes have really been moved for the last command - * submission. This can result in a debt that can stop buffer migrations - * temporarily. - */ -void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, - u64 num_vis_bytes) -{ - spin_lock(&adev->mm_stats.lock); - adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); - adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); - spin_unlock(&adev->mm_stats.lock); + r = amdgpu_sync_fence(&p->gang_leader->sync, fence); + dma_fence_put(fence); + if (r) + return r; + } + return 0; } -static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) +static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p, + uint32_t handle, u64 point, + u64 flags) { - struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct amdgpu_cs_parser *p = param; - struct ttm_operation_ctx ctx = { - .interruptible = true, - .no_wait_gpu = false, - .resv = bo->tbo.base.resv - }; - uint32_t domain; + struct dma_fence *fence; int r; - if (bo->tbo.pin_count) - return 0; + r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); + if (r) { + DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", + handle, point, r); + return r; + } - /* Don't move this buffer if we have depleted our allowance - * to move it. Don't move anything if the threshold is zero. - */ - if (p->bytes_moved < p->bytes_moved_threshold && - (!bo->tbo.base.dma_buf || - list_empty(&bo->tbo.base.dma_buf->attachments))) { - if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && - (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { - /* And don't move a CPU_ACCESS_REQUIRED BO to limited - * visible VRAM if we've depleted our allowance to do - * that. - */ - if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) - domain = bo->preferred_domains; - else - domain = bo->allowed_domains; - } else { - domain = bo->preferred_domains; - } - } else { - domain = bo->allowed_domains; - } - -retry: - amdgpu_bo_placement_from_domain(bo, domain); - r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); - - p->bytes_moved += ctx.bytes_moved; - if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && - amdgpu_bo_in_cpu_visible_vram(bo)) - p->bytes_moved_vis += ctx.bytes_moved; - - if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { - domain = bo->allowed_domains; - goto retry; - } + r = amdgpu_sync_fence(&p->gang_leader->sync, fence); + dma_fence_put(fence); return r; } -static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, - struct list_head *validated) +static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) { - struct ttm_operation_ctx ctx = { true, false }; - struct amdgpu_bo_list_entry *lobj; - int r; - - list_for_each_entry(lobj, validated, tv.head) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); - struct mm_struct *usermm; - - usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); - if (usermm && usermm != current->mm) - return -EPERM; - - if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && - lobj->user_invalidated && lobj->user_pages) { - amdgpu_bo_placement_from_domain(bo, - AMDGPU_GEM_DOMAIN_CPU); - r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); - if (r) - return r; - - amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, - lobj->user_pages); - } + struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; + unsigned num_deps; + int i, r; - r = amdgpu_cs_bo_validate(p, bo); + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_sem); + for (i = 0; i < num_deps; ++i) { + r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0); if (r) return r; - - kvfree(lobj->user_pages); - lobj->user_pages = NULL; } + return 0; } -static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, - union drm_amdgpu_cs *cs) +static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) { - struct amdgpu_fpriv *fpriv = p->filp->driver_priv; - struct amdgpu_vm *vm = &fpriv->vm; - struct amdgpu_bo_list_entry *e; - struct list_head duplicates; - struct amdgpu_bo *gds; - struct amdgpu_bo *gws; - struct amdgpu_bo *oa; - int r; - - INIT_LIST_HEAD(&p->validated); - - /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ - if (cs->in.bo_list_handle) { - if (p->bo_list) - return -EINVAL; + struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; + unsigned num_deps; + int i, r; - r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, - &p->bo_list); - if (r) - return r; - } else if (!p->bo_list) { - /* Create a empty bo_list when no handle is provided */ - r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, - &p->bo_list); + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_syncobj); + for (i = 0; i < num_deps; ++i) { + r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle, + syncobj_deps[i].point, + syncobj_deps[i].flags); if (r) return r; } - mutex_lock(&p->bo_list->bo_list_mutex); - - /* One for TTM and one for the CS job */ - amdgpu_bo_list_for_each_entry(e, p->bo_list) - e->tv.num_shared = 2; - - amdgpu_bo_list_get_list(p->bo_list, &p->validated); + return 0; +} - INIT_LIST_HEAD(&duplicates); - amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); +static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; + unsigned num_deps; + int i; - if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) - list_add(&p->uf_entry.tv.head, &p->validated); + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_sem); - /* Get userptr backing pages. If pages are updated after registered - * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do - * amdgpu_ttm_backend_bind() to flush and invalidate new pages - */ - amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); - bool userpage_invalidated = false; - int i; + if (p->post_deps) + return -EINVAL; - e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, - sizeof(struct page *), - GFP_KERNEL | __GFP_ZERO); - if (!e->user_pages) { - DRM_ERROR("kvmalloc_array failure\n"); - r = -ENOMEM; - goto out_free_user_pages; - } + p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), + GFP_KERNEL); + p->num_post_deps = 0; - r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); - if (r) { - kvfree(e->user_pages); - e->user_pages = NULL; - goto out_free_user_pages; - } + if (!p->post_deps) + return -ENOMEM; - for (i = 0; i < bo->tbo.ttm->num_pages; i++) { - if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { - userpage_invalidated = true; - break; - } - } - e->user_invalidated = userpage_invalidated; - } - r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, - &duplicates); - if (unlikely(r != 0)) { - if (r != -ERESTARTSYS) - DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); - goto out_free_user_pages; + for (i = 0; i < num_deps; ++i) { + p->post_deps[i].syncobj = + drm_syncobj_find(p->filp, deps[i].handle); + if (!p->post_deps[i].syncobj) + return -EINVAL; + p->post_deps[i].chain = NULL; + p->post_deps[i].point = 0; + p->num_post_deps++; } - amdgpu_bo_list_for_each_entry(e, p->bo_list) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); - - e->bo_va = amdgpu_vm_bo_find(vm, bo); - } + return 0; +} - /* Move fence waiting after getting reservation lock of - * PD root. Then there is no need on a ctx mutex lock. - */ - r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entity); - if (unlikely(r != 0)) { - if (r != -ERESTARTSYS) - DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n"); - goto error_validate; - } +static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p, + struct amdgpu_cs_chunk *chunk) +{ + struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; + unsigned num_deps; + int i; - amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, - &p->bytes_moved_vis_threshold); - p->bytes_moved = 0; - p->bytes_moved_vis = 0; + num_deps = chunk->length_dw * 4 / + sizeof(struct drm_amdgpu_cs_chunk_syncobj); - r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, - amdgpu_cs_bo_validate, p); - if (r) { - DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); - goto error_validate; - } + if (p->post_deps) + return -EINVAL; - r = amdgpu_cs_list_validate(p, &duplicates); - if (r) - goto error_validate; + p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), + GFP_KERNEL); + p->num_post_deps = 0; - r = amdgpu_cs_list_validate(p, &p->validated); - if (r) - goto error_validate; + if (!p->post_deps) + return -ENOMEM; - amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, - p->bytes_moved_vis); + for (i = 0; i < num_deps; ++i) { + struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; - gds = p->bo_list->gds_obj; - gws = p->bo_list->gws_obj; - oa = p->bo_list->oa_obj; + dep->chain = NULL; + if (syncobj_deps[i].point) { + dep->chain = dma_fence_chain_alloc(); + if (!dep->chain) + return -ENOMEM; + } - if (gds) { - p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; - p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; - } - if (gws) { - p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; - p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; - } - if (oa) { - p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; - p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; + dep->syncobj = drm_syncobj_find(p->filp, + syncobj_deps[i].handle); + if (!dep->syncobj) { + dma_fence_chain_free(dep->chain); + return -EINVAL; + } + dep->point = syncobj_deps[i].point; + p->num_post_deps++; } - if (!r && p->uf_entry.tv.bo) { - struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); + return 0; +} - r = amdgpu_ttm_alloc_gart(&uf->tbo); - p->job->uf_addr += amdgpu_bo_gpu_offset(uf); - } +static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p) +{ + unsigned int ce_preempt = 0, de_preempt = 0; + int i, r; -error_validate: - if (r) - ttm_eu_backoff_reservation(&p->ticket, &p->validated); + for (i = 0; i < p->nchunks; ++i) { + struct amdgpu_cs_chunk *chunk; -out_free_user_pages: - if (r) { - amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + chunk = &p->chunks[i]; - if (!e->user_pages) - continue; - amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); - kvfree(e->user_pages); - e->user_pages = NULL; + switch (chunk->chunk_id) { + case AMDGPU_CHUNK_ID_IB: + r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_DEPENDENCIES: + case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: + r = amdgpu_cs_p2_dependencies(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_IN: + r = amdgpu_cs_p2_syncobj_in(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: + r = amdgpu_cs_p2_syncobj_out(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: + r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk); + if (r) + return r; + break; + case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: + r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk); + if (r) + return r; + break; } - mutex_unlock(&p->bo_list->bo_list_mutex); } - return r; + + return 0; } -static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) +/* Convert microseconds to bytes. */ +static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) { - struct amdgpu_fpriv *fpriv = p->filp->driver_priv; - struct amdgpu_bo_list_entry *e; - int r; + if (us <= 0 || !adev->mm_stats.log2_max_MBps) + return 0; - list_for_each_entry(e, &p->validated, tv.head) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); - struct dma_resv *resv = bo->tbo.base.resv; - enum amdgpu_sync_mode sync_mode; + /* Since accum_us is incremented by a million per second, just + * multiply it by the number of MB/s to get the number of bytes. + */ + return us << adev->mm_stats.log2_max_MBps; +} - sync_mode = amdgpu_bo_explicit_sync(bo) ? - AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; - r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode, - &fpriv->vm); - if (r) - return r; - } - return 0; +static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) +{ + if (!adev->mm_stats.log2_max_MBps) + return 0; + + return bytes >> adev->mm_stats.log2_max_MBps; } -/** - * amdgpu_cs_parser_fini() - clean parser states - * @parser: parser structure holding parsing context. - * @error: error number - * @backoff: indicator to backoff the reservation +/* Returns how many bytes TTM can move right now. If no bytes can be moved, + * it returns 0. If it returns non-zero, it's OK to move at least one buffer, + * which means it can go over the threshold once. If that happens, the driver + * will be in debt and no other buffer migrations can be done until that debt + * is repaid. + * + * This approach allows moving a buffer of any size (it's important to allow + * that). * - * If error is set then unvalidate buffer, otherwise just free memory - * used by parsing context. - **/ -static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, - bool backoff) + * The currency is simply time in microseconds and it increases as the clock + * ticks. The accumulated microseconds (us) are converted to bytes and + * returned. + */ +static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, + u64 *max_bytes, + u64 *max_vis_bytes) { - unsigned i; + s64 time_us, increment_us; + u64 free_vram, total_vram, used_vram; + /* Allow a maximum of 200 accumulated ms. This is basically per-IB + * throttling. + * + * It means that in order to get full max MBps, at least 5 IBs per + * second must be submitted and not more than 200ms apart from each + * other. + */ + const s64 us_upper_bound = 200000; - if (error && backoff) { - ttm_eu_backoff_reservation(&parser->ticket, - &parser->validated); - mutex_unlock(&parser->bo_list->bo_list_mutex); + if (!adev->mm_stats.log2_max_MBps) { + *max_bytes = 0; + *max_vis_bytes = 0; + return; } - for (i = 0; i < parser->num_post_deps; i++) { - drm_syncobj_put(parser->post_deps[i].syncobj); - kfree(parser->post_deps[i].chain); - } - kfree(parser->post_deps); + total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); + used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); + free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; - dma_fence_put(parser->fence); + spin_lock(&adev->mm_stats.lock); - if (parser->ctx) { - mutex_unlock(&parser->ctx->lock); - amdgpu_ctx_put(parser->ctx); + /* Increase the amount of accumulated us. */ + time_us = ktime_to_us(ktime_get()); + increment_us = time_us - adev->mm_stats.last_update_us; + adev->mm_stats.last_update_us = time_us; + adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, + us_upper_bound); + + /* This prevents the short period of low performance when the VRAM + * usage is low and the driver is in debt or doesn't have enough + * accumulated us to fill VRAM quickly. + * + * The situation can occur in these cases: + * - a lot of VRAM is freed by userspace + * - the presence of a big buffer causes a lot of evictions + * (solution: split buffers into smaller ones) + * + * If 128 MB or 1/8th of VRAM is free, start filling it now by setting + * accum_us to a positive number. + */ + if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { + s64 min_us; + + /* Be more aggressive on dGPUs. Try to fill a portion of free + * VRAM now. + */ + if (!(adev->flags & AMD_IS_APU)) + min_us = bytes_to_us(adev, free_vram / 4); + else + min_us = 0; /* Reset accum_us on APUs. */ + + adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); } - if (parser->bo_list) - amdgpu_bo_list_put(parser->bo_list); - for (i = 0; i < parser->nchunks; i++) - kvfree(parser->chunks[i].kdata); - kvfree(parser->chunks); - if (parser->job) - amdgpu_job_free(parser->job); - if (parser->uf_entry.tv.bo) { - struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); + /* This is set to 0 if the driver is in debt to disallow (optional) + * buffer moves. + */ + *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); - amdgpu_bo_unref(&uf); + /* Do the same for visible VRAM if half of it is free */ + if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { + u64 total_vis_vram = adev->gmc.visible_vram_size; + u64 used_vis_vram = + amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); + + if (used_vis_vram < total_vis_vram) { + u64 free_vis_vram = total_vis_vram - used_vis_vram; + adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + + increment_us, us_upper_bound); + + if (free_vis_vram >= total_vis_vram / 2) + adev->mm_stats.accum_us_vis = + max(bytes_to_us(adev, free_vis_vram / 2), + adev->mm_stats.accum_us_vis); + } + + *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); + } else { + *max_vis_bytes = 0; } + + spin_unlock(&adev->mm_stats.lock); } -static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) +/* Report how many bytes have really been moved for the last command + * submission. This can result in a debt that can stop buffer migrations + * temporarily. + */ +void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, + u64 num_vis_bytes) { - struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); - struct amdgpu_fpriv *fpriv = p->filp->driver_priv; - struct amdgpu_device *adev = p->adev; - struct amdgpu_vm *vm = &fpriv->vm; - struct amdgpu_bo_list_entry *e; - struct amdgpu_bo_va *bo_va; - struct amdgpu_bo *bo; - int r; + spin_lock(&adev->mm_stats.lock); + adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); + adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); + spin_unlock(&adev->mm_stats.lock); +} - /* Only for UVD/VCE VM emulation */ - if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) { - unsigned i, j; - - for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) { - struct drm_amdgpu_cs_chunk_ib *chunk_ib; - struct amdgpu_bo_va_mapping *m; - struct amdgpu_bo *aobj = NULL; - struct amdgpu_cs_chunk *chunk; - uint64_t offset, va_start; - struct amdgpu_ib *ib; - uint8_t *kptr; - - chunk = &p->chunks[i]; - ib = &p->job->ibs[j]; - chunk_ib = chunk->kdata; - - if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) - continue; +static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_cs_parser *p = param; + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + .resv = bo->tbo.base.resv + }; + uint32_t domain; + int r; - va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK; - r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); - if (r) { - DRM_ERROR("IB va_start is invalid\n"); - return r; - } + if (bo->tbo.pin_count) + return 0; - if ((va_start + chunk_ib->ib_bytes) > - (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { - DRM_ERROR("IB va_start+ib_bytes is invalid\n"); - return -EINVAL; - } + /* Don't move this buffer if we have depleted our allowance + * to move it. Don't move anything if the threshold is zero. + */ + if (p->bytes_moved < p->bytes_moved_threshold && + (!bo->tbo.base.dma_buf || + list_empty(&bo->tbo.base.dma_buf->attachments))) { + if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { + /* And don't move a CPU_ACCESS_REQUIRED BO to limited + * visible VRAM if we've depleted our allowance to do + * that. + */ + if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) + domain = bo->preferred_domains; + else + domain = bo->allowed_domains; + } else { + domain = bo->preferred_domains; + } + } else { + domain = bo->allowed_domains; + } - /* the IB should be reserved at this point */ - r = amdgpu_bo_kmap(aobj, (void **)&kptr); - if (r) { - return r; - } +retry: + amdgpu_bo_placement_from_domain(bo, domain); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); - offset = m->start * AMDGPU_GPU_PAGE_SIZE; - kptr += va_start - offset; - - if (ring->funcs->parse_cs) { - memcpy(ib->ptr, kptr, chunk_ib->ib_bytes); - amdgpu_bo_kunmap(aobj); - - r = amdgpu_ring_parse_cs(ring, p, p->job, ib); - if (r) - return r; - } else { - ib->ptr = (uint32_t *)kptr; - r = amdgpu_ring_patch_cs_in_place(ring, p, p->job, ib); - amdgpu_bo_kunmap(aobj); - if (r) - return r; - } + p->bytes_moved += ctx.bytes_moved; + if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && + amdgpu_bo_in_cpu_visible_vram(bo)) + p->bytes_moved_vis += ctx.bytes_moved; - j++; - } + if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { + domain = bo->allowed_domains; + goto retry; } - if (!p->job->vm) - return amdgpu_cs_sync_rings(p); + return r; +} +static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p, + struct list_head *validated) +{ + struct ttm_operation_ctx ctx = { true, false }; + struct amdgpu_bo_list_entry *lobj; + int r; - r = amdgpu_vm_clear_freed(adev, vm, NULL); - if (r) - return r; + list_for_each_entry(lobj, validated, tv.head) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(lobj->tv.bo); + struct mm_struct *usermm; - r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); - if (r) - return r; + usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); + if (usermm && usermm != current->mm) + return -EPERM; - r = amdgpu_sync_fence(&p->job->sync, fpriv->prt_va->last_pt_update); - if (r) - return r; + if (amdgpu_ttm_tt_is_userptr(bo->tbo.ttm) && + lobj->user_invalidated && lobj->user_pages) { + amdgpu_bo_placement_from_domain(bo, + AMDGPU_GEM_DOMAIN_CPU); + r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); + if (r) + return r; - if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { - bo_va = fpriv->csa_va; - BUG_ON(!bo_va); - r = amdgpu_vm_bo_update(adev, bo_va, false); - if (r) - return r; + amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, + lobj->user_pages); + } - r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); + r = amdgpu_cs_bo_validate(p, bo); if (r) return r; + + kvfree(lobj->user_pages); + lobj->user_pages = NULL; } + return 0; +} - amdgpu_bo_list_for_each_entry(e, p->bo_list) { - /* ignore duplicates */ - bo = ttm_to_amdgpu_bo(e->tv.bo); - if (!bo) - continue; +static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, + union drm_amdgpu_cs *cs) +{ + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_list_entry *e; + struct list_head duplicates; + unsigned int i; + int r; - bo_va = e->bo_va; - if (bo_va == NULL) - continue; + INIT_LIST_HEAD(&p->validated); - r = amdgpu_vm_bo_update(adev, bo_va, false); + /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ + if (cs->in.bo_list_handle) { + if (p->bo_list) + return -EINVAL; + + r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, + &p->bo_list); if (r) return r; - - r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update); + } else if (!p->bo_list) { + /* Create a empty bo_list when no handle is provided */ + r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, + &p->bo_list); if (r) return r; } - r = amdgpu_vm_handle_moved(adev, vm); - if (r) - return r; + mutex_lock(&p->bo_list->bo_list_mutex); - r = amdgpu_vm_update_pdes(adev, vm, false); - if (r) - return r; + /* One for TTM and one for the CS job */ + amdgpu_bo_list_for_each_entry(e, p->bo_list) + e->tv.num_shared = 2; - r = amdgpu_sync_fence(&p->job->sync, vm->last_update); - if (r) - return r; + amdgpu_bo_list_get_list(p->bo_list, &p->validated); - p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); + INIT_LIST_HEAD(&duplicates); + amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd); - if (amdgpu_vm_debug) { - /* Invalidate all BOs to test for userspace bugs */ - amdgpu_bo_list_for_each_entry(e, p->bo_list) { - struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + if (p->uf_entry.tv.bo && !ttm_to_amdgpu_bo(p->uf_entry.tv.bo)->parent) + list_add(&p->uf_entry.tv.head, &p->validated); - /* ignore duplicates */ - if (!bo) - continue; + /* Get userptr backing pages. If pages are updated after registered + * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do + * amdgpu_ttm_backend_bind() to flush and invalidate new pages + */ + amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + bool userpage_invalidated = false; + int i; - amdgpu_vm_bo_invalidate(adev, bo, false); + e->user_pages = kvmalloc_array(bo->tbo.ttm->num_pages, + sizeof(struct page *), + GFP_KERNEL | __GFP_ZERO); + if (!e->user_pages) { + DRM_ERROR("kvmalloc_array failure\n"); + r = -ENOMEM; + goto out_free_user_pages; } - } - - return amdgpu_cs_sync_rings(p); -} - -static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, - struct amdgpu_cs_parser *parser) -{ - struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; - struct amdgpu_vm *vm = &fpriv->vm; - int r, ce_preempt = 0, de_preempt = 0; - struct amdgpu_ring *ring; - int i, j; - - for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) { - struct amdgpu_cs_chunk *chunk; - struct amdgpu_ib *ib; - struct drm_amdgpu_cs_chunk_ib *chunk_ib; - struct drm_sched_entity *entity; - chunk = &parser->chunks[i]; - ib = &parser->job->ibs[j]; - chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata; - - if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB) - continue; + r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages); + if (r) { + kvfree(e->user_pages); + e->user_pages = NULL; + goto out_free_user_pages; + } - if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && - (amdgpu_mcbp || amdgpu_sriov_vf(adev))) { - if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { - if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) - ce_preempt++; - else - de_preempt++; + for (i = 0; i < bo->tbo.ttm->num_pages; i++) { + if (bo->tbo.ttm->pages[i] != e->user_pages[i]) { + userpage_invalidated = true; + break; } - - /* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */ - if (ce_preempt > 1 || de_preempt > 1) - return -EINVAL; } + e->user_invalidated = userpage_invalidated; + } - r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type, - chunk_ib->ip_instance, chunk_ib->ring, - &entity); - if (r) - return r; - - if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) - parser->job->preamble_status |= - AMDGPU_PREAMBLE_IB_PRESENT; - - if (parser->entity && parser->entity != entity) - return -EINVAL; - - /* Return if there is no run queue associated with this entity. - * Possibly because of disabled HW IP*/ - if (entity->rq == NULL) - return -EINVAL; - - parser->entity = entity; - - ring = to_amdgpu_ring(entity->rq->sched); - r = amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ? - chunk_ib->ib_bytes : 0, - AMDGPU_IB_POOL_DELAYED, ib); - if (r) { - DRM_ERROR("Failed to get ib !\n"); - return r; - } + r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, + &duplicates); + if (unlikely(r != 0)) { + if (r != -ERESTARTSYS) + DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); + goto out_free_user_pages; + } - ib->gpu_addr = chunk_ib->va_start; - ib->length_dw = chunk_ib->ib_bytes / 4; - ib->flags = chunk_ib->flags; + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); - j++; + e->bo_va = amdgpu_vm_bo_find(vm, bo); } - /* MM engine doesn't support user fences */ - ring = to_amdgpu_ring(parser->entity->rq->sched); - if (parser->job->uf_addr && ring->funcs->no_user_fence) - return -EINVAL; + amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, + &p->bytes_moved_vis_threshold); + p->bytes_moved = 0; + p->bytes_moved_vis = 0; - return 0; -} + r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm, + amdgpu_cs_bo_validate, p); + if (r) { + DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n"); + goto error_validate; + } -static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p, - struct amdgpu_cs_chunk *chunk) -{ - struct amdgpu_fpriv *fpriv = p->filp->driver_priv; - unsigned num_deps; - int i, r; - struct drm_amdgpu_cs_chunk_dep *deps; + r = amdgpu_cs_list_validate(p, &duplicates); + if (r) + goto error_validate; - deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata; - num_deps = chunk->length_dw * 4 / - sizeof(struct drm_amdgpu_cs_chunk_dep); + r = amdgpu_cs_list_validate(p, &p->validated); + if (r) + goto error_validate; - for (i = 0; i < num_deps; ++i) { - struct amdgpu_ctx *ctx; - struct drm_sched_entity *entity; - struct dma_fence *fence; + if (p->uf_entry.tv.bo) { + struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo); - ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); - if (ctx == NULL) - return -EINVAL; + r = amdgpu_ttm_alloc_gart(&uf->tbo); + if (r) + goto error_validate; - r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, - deps[i].ip_instance, - deps[i].ring, &entity); - if (r) { - amdgpu_ctx_put(ctx); - return r; - } + p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(uf); + } - fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); - amdgpu_ctx_put(ctx); + amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, + p->bytes_moved_vis); - if (IS_ERR(fence)) - return PTR_ERR(fence); - else if (!fence) - continue; + for (i = 0; i < p->gang_size; ++i) + amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj, + p->bo_list->gws_obj, + p->bo_list->oa_obj); + return 0; - if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { - struct drm_sched_fence *s_fence; - struct dma_fence *old = fence; +error_validate: + ttm_eu_backoff_reservation(&p->ticket, &p->validated); - s_fence = to_drm_sched_fence(fence); - fence = dma_fence_get(&s_fence->scheduled); - dma_fence_put(old); - } +out_free_user_pages: + amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); - r = amdgpu_sync_fence(&p->job->sync, fence); - dma_fence_put(fence); - if (r) - return r; + if (!e->user_pages) + continue; + amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); + kvfree(e->user_pages); + e->user_pages = NULL; } - return 0; + return r; } -static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p, - uint32_t handle, u64 point, - u64 flags) +static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p) { - struct dma_fence *fence; - int r; + int i, j; - r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); - if (r) { - DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n", - handle, point, r); - return r; - } + if (!trace_amdgpu_cs_enabled()) + return; - r = amdgpu_sync_fence(&p->job->sync, fence); - dma_fence_put(fence); + for (i = 0; i < p->gang_size; ++i) { + struct amdgpu_job *job = p->jobs[i]; - return r; + for (j = 0; j < job->num_ibs; ++j) + trace_amdgpu_cs(p, job, &job->ibs[j]); + } } -static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p, - struct amdgpu_cs_chunk *chunk) +static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p, + struct amdgpu_job *job) { - struct drm_amdgpu_cs_chunk_sem *deps; - unsigned num_deps; - int i, r; + struct amdgpu_ring *ring = amdgpu_job_ring(job); + unsigned int i; + int r; - deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; - num_deps = chunk->length_dw * 4 / - sizeof(struct drm_amdgpu_cs_chunk_sem); - for (i = 0; i < num_deps; ++i) { - r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle, - 0, 0); - if (r) + /* Only for UVD/VCE VM emulation */ + if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place) + return 0; + + for (i = 0; i < job->num_ibs; ++i) { + struct amdgpu_ib *ib = &job->ibs[i]; + struct amdgpu_bo_va_mapping *m; + struct amdgpu_bo *aobj; + uint64_t va_start; + uint8_t *kptr; + + va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK; + r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); + if (r) { + DRM_ERROR("IB va_start is invalid\n"); return r; + } + + if ((va_start + ib->length_dw * 4) > + (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { + DRM_ERROR("IB va_start+ib_bytes is invalid\n"); + return -EINVAL; + } + + /* the IB should be reserved at this point */ + r = amdgpu_bo_kmap(aobj, (void **)&kptr); + if (r) { + return r; + } + + kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE); + + if (ring->funcs->parse_cs) { + memcpy(ib->ptr, kptr, ib->length_dw * 4); + amdgpu_bo_kunmap(aobj); + + r = amdgpu_ring_parse_cs(ring, p, job, ib); + if (r) + return r; + } else { + ib->ptr = (uint32_t *)kptr; + r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib); + amdgpu_bo_kunmap(aobj); + if (r) + return r; + } } return 0; } - -static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p, - struct amdgpu_cs_chunk *chunk) +static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p) { - struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; - unsigned num_deps; - int i, r; + unsigned int i; + int r; - syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; - num_deps = chunk->length_dw * 4 / - sizeof(struct drm_amdgpu_cs_chunk_syncobj); - for (i = 0; i < num_deps; ++i) { - r = amdgpu_syncobj_lookup_and_add_to_sync(p, - syncobj_deps[i].handle, - syncobj_deps[i].point, - syncobj_deps[i].flags); + for (i = 0; i < p->gang_size; ++i) { + r = amdgpu_cs_patch_ibs(p, p->jobs[i]); if (r) return r; } - return 0; } -static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p, - struct amdgpu_cs_chunk *chunk) +static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) { - struct drm_amdgpu_cs_chunk_sem *deps; - unsigned num_deps; - int i; - - deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata; - num_deps = chunk->length_dw * 4 / - sizeof(struct drm_amdgpu_cs_chunk_sem); + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_job *job = p->gang_leader; + struct amdgpu_device *adev = p->adev; + struct amdgpu_vm *vm = &fpriv->vm; + struct amdgpu_bo_list_entry *e; + struct amdgpu_bo_va *bo_va; + struct amdgpu_bo *bo; + unsigned int i; + int r; - if (p->post_deps) - return -EINVAL; + r = amdgpu_vm_clear_freed(adev, vm, NULL); + if (r) + return r; - p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), - GFP_KERNEL); - p->num_post_deps = 0; + r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); + if (r) + return r; - if (!p->post_deps) - return -ENOMEM; + r = amdgpu_sync_fence(&job->sync, fpriv->prt_va->last_pt_update); + if (r) + return r; + if (fpriv->csa_va) { + bo_va = fpriv->csa_va; + BUG_ON(!bo_va); + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + return r; - for (i = 0; i < num_deps; ++i) { - p->post_deps[i].syncobj = - drm_syncobj_find(p->filp, deps[i].handle); - if (!p->post_deps[i].syncobj) - return -EINVAL; - p->post_deps[i].chain = NULL; - p->post_deps[i].point = 0; - p->num_post_deps++; + r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update); + if (r) + return r; } - return 0; -} + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + /* ignore duplicates */ + bo = ttm_to_amdgpu_bo(e->tv.bo); + if (!bo) + continue; + bo_va = e->bo_va; + if (bo_va == NULL) + continue; -static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p, - struct amdgpu_cs_chunk *chunk) -{ - struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps; - unsigned num_deps; - int i; + r = amdgpu_vm_bo_update(adev, bo_va, false); + if (r) + return r; - syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata; - num_deps = chunk->length_dw * 4 / - sizeof(struct drm_amdgpu_cs_chunk_syncobj); + r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update); + if (r) + return r; + } - if (p->post_deps) - return -EINVAL; + r = amdgpu_vm_handle_moved(adev, vm); + if (r) + return r; - p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps), - GFP_KERNEL); - p->num_post_deps = 0; + r = amdgpu_vm_update_pdes(adev, vm, false); + if (r) + return r; - if (!p->post_deps) - return -ENOMEM; + r = amdgpu_sync_fence(&job->sync, vm->last_update); + if (r) + return r; - for (i = 0; i < num_deps; ++i) { - struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; + for (i = 0; i < p->gang_size; ++i) { + job = p->jobs[i]; - dep->chain = NULL; - if (syncobj_deps[i].point) { - dep->chain = dma_fence_chain_alloc(); - if (!dep->chain) - return -ENOMEM; - } + if (!job->vm) + continue; - dep->syncobj = drm_syncobj_find(p->filp, - syncobj_deps[i].handle); - if (!dep->syncobj) { - dma_fence_chain_free(dep->chain); - return -EINVAL; + job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); + } + + if (amdgpu_vm_debug) { + /* Invalidate all BOs to test for userspace bugs */ + amdgpu_bo_list_for_each_entry(e, p->bo_list) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + + /* ignore duplicates */ + if (!bo) + continue; + + amdgpu_vm_bo_invalidate(adev, bo, false); } - dep->point = syncobj_deps[i].point; - p->num_post_deps++; } return 0; } -static int amdgpu_cs_dependencies(struct amdgpu_device *adev, - struct amdgpu_cs_parser *p) +static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) { - int i, r; - - /* TODO: Investigate why we still need the context lock */ - mutex_unlock(&p->ctx->lock); + struct amdgpu_fpriv *fpriv = p->filp->driver_priv; + struct amdgpu_job *leader = p->gang_leader; + struct amdgpu_bo_list_entry *e; + unsigned int i; + int r; - for (i = 0; i < p->nchunks; ++i) { - struct amdgpu_cs_chunk *chunk; + list_for_each_entry(e, &p->validated, tv.head) { + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); + struct dma_resv *resv = bo->tbo.base.resv; + enum amdgpu_sync_mode sync_mode; - chunk = &p->chunks[i]; + sync_mode = amdgpu_bo_explicit_sync(bo) ? + AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; + r = amdgpu_sync_resv(p->adev, &leader->sync, resv, sync_mode, + &fpriv->vm); + if (r) + return r; + } - switch (chunk->chunk_id) { - case AMDGPU_CHUNK_ID_DEPENDENCIES: - case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: - r = amdgpu_cs_process_fence_dep(p, chunk); - if (r) - goto out; - break; - case AMDGPU_CHUNK_ID_SYNCOBJ_IN: - r = amdgpu_cs_process_syncobj_in_dep(p, chunk); - if (r) - goto out; - break; - case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: - r = amdgpu_cs_process_syncobj_out_dep(p, chunk); - if (r) - goto out; - break; - case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: - r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk); - if (r) - goto out; - break; - case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: - r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk); - if (r) - goto out; - break; - } + for (i = 0; i < p->gang_size - 1; ++i) { + r = amdgpu_sync_clone(&leader->sync, &p->jobs[i]->sync); + if (r) + return r; } -out: - mutex_lock(&p->ctx->lock); + r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_size - 1]); + if (r && r != -ERESTARTSYS) + DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n"); + return r; } @@ -1221,20 +1228,28 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs) { struct amdgpu_fpriv *fpriv = p->filp->driver_priv; - struct drm_sched_entity *entity = p->entity; + struct amdgpu_job *leader = p->gang_leader; struct amdgpu_bo_list_entry *e; - struct amdgpu_job *job; + unsigned int i; uint64_t seq; int r; - job = p->job; - p->job = NULL; + for (i = 0; i < p->gang_size; ++i) + drm_sched_job_arm(&p->jobs[i]->base); - r = drm_sched_job_init(&job->base, entity, &fpriv->vm); - if (r) - goto error_unlock; + for (i = 0; i < (p->gang_size - 1); ++i) { + struct dma_fence *fence; + + fence = &p->jobs[i]->base.s_fence->scheduled; + r = amdgpu_sync_fence(&leader->sync, fence); + if (r) + goto error_cleanup; + } - drm_sched_job_arm(&job->base); + if (p->gang_size > 1) { + for (i = 0; i < p->gang_size; ++i) + amdgpu_job_set_gang_leader(p->jobs[i], leader); + } /* No memory allocation is allowed while holding the notifier lock. * The lock is held until amdgpu_cs_submit is finished and fence is @@ -1245,6 +1260,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, /* If userptr are invalidated after amdgpu_cs_parser_bos(), return * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. */ + r = 0; amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo); @@ -1252,67 +1268,96 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, } if (r) { r = -EAGAIN; - goto error_abort; + goto error_unlock; } - p->fence = dma_fence_get(&job->base.s_fence->finished); + p->fence = dma_fence_get(&leader->base.s_fence->finished); + list_for_each_entry(e, &p->validated, tv.head) { + + /* Everybody except for the gang leader uses READ */ + for (i = 0; i < (p->gang_size - 1); ++i) { + dma_resv_add_fence(e->tv.bo->base.resv, + &p->jobs[i]->base.s_fence->finished, + DMA_RESV_USAGE_READ); + } + + /* The gang leader is remembered as writer */ + e->tv.num_shared = 0; + } - seq = amdgpu_ctx_add_fence(p->ctx, entity, p->fence); + seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_size - 1], + p->fence); amdgpu_cs_post_dependencies(p); - if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && + if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && !p->ctx->preamble_presented) { - job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; + leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; p->ctx->preamble_presented = true; } cs->out.handle = seq; - job->uf_sequence = seq; + leader->uf_sequence = seq; - amdgpu_job_free_resources(job); - - trace_amdgpu_cs_ioctl(job); amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket); - drm_sched_entity_push_job(&job->base); + for (i = 0; i < p->gang_size; ++i) { + amdgpu_job_free_resources(p->jobs[i]); + trace_amdgpu_cs_ioctl(p->jobs[i]); + drm_sched_entity_push_job(&p->jobs[i]->base); + p->jobs[i] = NULL; + } amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); - - /* Make sure all BOs are remembered as writers */ - amdgpu_bo_list_for_each_entry(e, p->bo_list) - e->tv.num_shared = 0; - ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence); + mutex_unlock(&p->adev->notifier_lock); mutex_unlock(&p->bo_list->bo_list_mutex); - return 0; -error_abort: - drm_sched_job_cleanup(&job->base); +error_unlock: mutex_unlock(&p->adev->notifier_lock); -error_unlock: - amdgpu_job_free(job); +error_cleanup: + for (i = 0; i < p->gang_size; ++i) + drm_sched_job_cleanup(&p->jobs[i]->base); return r; } -static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser) +/* Cleanup the parser structure */ +static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser) { - int i; + unsigned i; - if (!trace_amdgpu_cs_enabled()) - return; + for (i = 0; i < parser->num_post_deps; i++) { + drm_syncobj_put(parser->post_deps[i].syncobj); + kfree(parser->post_deps[i].chain); + } + kfree(parser->post_deps); + + dma_fence_put(parser->fence); + + if (parser->ctx) + amdgpu_ctx_put(parser->ctx); + if (parser->bo_list) + amdgpu_bo_list_put(parser->bo_list); + + for (i = 0; i < parser->nchunks; i++) + kvfree(parser->chunks[i].kdata); + kvfree(parser->chunks); + for (i = 0; i < parser->gang_size; ++i) { + if (parser->jobs[i]) + amdgpu_job_free(parser->jobs[i]); + } + if (parser->uf_entry.tv.bo) { + struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo); - for (i = 0; i < parser->job->num_ibs; i++) - trace_amdgpu_cs(parser, i); + amdgpu_bo_unref(&uf); + } } int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct amdgpu_device *adev = drm_to_adev(dev); - union drm_amdgpu_cs *cs = data; - struct amdgpu_cs_parser parser = {}; - bool reserved_buffers = false; + struct amdgpu_cs_parser parser; int r; if (amdgpu_ras_intr_triggered()) @@ -1321,25 +1366,20 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (!adev->accel_working) return -EBUSY; - parser.adev = adev; - parser.filp = filp; - - r = amdgpu_cs_parser_init(&parser, data); + r = amdgpu_cs_parser_init(&parser, adev, filp, data); if (r) { if (printk_ratelimit()) DRM_ERROR("Failed to initialize parser %d!\n", r); - goto out; + return r; } - r = amdgpu_cs_ib_fill(adev, &parser); + r = amdgpu_cs_pass1(&parser, data); if (r) - goto out; + goto error_fini; - r = amdgpu_cs_dependencies(adev, &parser); - if (r) { - DRM_ERROR("Failed in the dependencies handling %d!\n", r); - goto out; - } + r = amdgpu_cs_pass2(&parser); + if (r) + goto error_fini; r = amdgpu_cs_parser_bos(&parser, data); if (r) { @@ -1347,22 +1387,36 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) DRM_ERROR("Not enough memory for command submission!\n"); else if (r != -ERESTARTSYS && r != -EAGAIN) DRM_ERROR("Failed to process the buffer list %d!\n", r); - goto out; + goto error_fini; } - reserved_buffers = true; + r = amdgpu_cs_patch_jobs(&parser); + if (r) + goto error_backoff; + + r = amdgpu_cs_vm_handling(&parser); + if (r) + goto error_backoff; + + r = amdgpu_cs_sync_rings(&parser); + if (r) + goto error_backoff; trace_amdgpu_cs_ibs(&parser); - r = amdgpu_cs_vm_handling(&parser); + r = amdgpu_cs_submit(&parser, data); if (r) - goto out; + goto error_backoff; - r = amdgpu_cs_submit(&parser, cs); + amdgpu_cs_parser_fini(&parser); + return 0; -out: - amdgpu_cs_parser_fini(&parser, r, reserved_buffers); +error_backoff: + ttm_eu_backoff_reservation(&parser.ticket, &parser.validated); + mutex_unlock(&parser.bo_list->bo_list_mutex); +error_fini: + amdgpu_cs_parser_fini(&parser); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h index 30ecc4917f811d8666fb2823a5c16666428e452c..cbaa19b2b8a30b45bc11fc37135937a31cf9c77c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h @@ -27,6 +27,8 @@ #include "amdgpu_bo_list.h" #include "amdgpu_ring.h" +#define AMDGPU_CS_GANG_SIZE 4 + struct amdgpu_bo_va_mapping; struct amdgpu_cs_chunk { @@ -50,9 +52,11 @@ struct amdgpu_cs_parser { unsigned nchunks; struct amdgpu_cs_chunk *chunks; - /* scheduler job object */ - struct amdgpu_job *job; - struct drm_sched_entity *entity; + /* scheduler job objects */ + unsigned int gang_size; + struct drm_sched_entity *entities[AMDGPU_CS_GANG_SIZE]; + struct amdgpu_job *jobs[AMDGPU_CS_GANG_SIZE]; + struct amdgpu_job *gang_leader; /* buffer objects */ struct ww_acquire_ctx ticket; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 3ea48385fab3ae97bb2a7549ae405d7c72a8c979..f6d9d5da53cd2f137a470774d66d628248d38f03 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -315,7 +315,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, kref_init(&ctx->refcount); ctx->mgr = mgr; spin_lock_init(&ctx->ring_lock); - mutex_init(&ctx->lock); ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter); ctx->reset_counter_query = ctx->reset_counter; @@ -407,7 +406,6 @@ static void amdgpu_ctx_fini(struct kref *ref) drm_dev_exit(idx); } - mutex_destroy(&ctx->lock); kfree(ctx); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h index cc7c8afff4144ccac981315a4e2657974e2d3ba3..0fa0e56daf67e9167526268494a9c81840fbd7ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h @@ -53,7 +53,6 @@ struct amdgpu_ctx { bool preamble_presented; int32_t init_priority; int32_t override_priority; - struct mutex lock; atomic_t guilty; unsigned long ras_counter_ce; unsigned long ras_counter_ue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 62b26f0e37b071a1d12ecfd09b476990e73eade4..ab8f970b284918b408f37ca79b493bc9384f8a9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2365,8 +2365,16 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } adev->ip_blocks[i].status.sw = true; - /* need to do gmc hw init early so we can allocate gpu mem */ - if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { + if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { + /* need to do common hw init early so everything is set up for gmc */ + r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); + if (r) { + DRM_ERROR("hw_init %d failed %d\n", i, r); + goto init_failed; + } + adev->ip_blocks[i].status.hw = true; + } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { + /* need to do gmc hw init early so we can allocate gpu mem */ /* Try to reserve bad pages early */ if (amdgpu_sriov_vf(adev)) amdgpu_virt_exchange_data(adev); @@ -2451,19 +2459,21 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) */ if (adev->gmc.xgmi.num_physical_nodes > 1) { if (amdgpu_xgmi_add_device(adev) == 0) { - struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); + if (!amdgpu_sriov_vf(adev)) { + struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); - if (!hive->reset_domain || - !amdgpu_reset_get_reset_domain(hive->reset_domain)) { - r = -ENOENT; + if (!hive->reset_domain || + !amdgpu_reset_get_reset_domain(hive->reset_domain)) { + r = -ENOENT; + amdgpu_put_xgmi_hive(hive); + goto init_failed; + } + + /* Drop the early temporary reset domain we created for device */ + amdgpu_reset_put_reset_domain(adev->reset_domain); + adev->reset_domain = hive->reset_domain; amdgpu_put_xgmi_hive(hive); - goto init_failed; } - - /* Drop the early temporary reset domain we created for device */ - amdgpu_reset_put_reset_domain(adev->reset_domain); - adev->reset_domain = hive->reset_domain; - amdgpu_put_xgmi_hive(hive); } } @@ -3052,8 +3062,8 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) int i, r; static enum amd_ip_block_type ip_order[] = { - AMD_IP_BLOCK_TYPE_GMC, AMD_IP_BLOCK_TYPE_COMMON, + AMD_IP_BLOCK_TYPE_GMC, AMD_IP_BLOCK_TYPE_PSP, AMD_IP_BLOCK_TYPE_IH, }; @@ -3144,7 +3154,8 @@ static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) continue; if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || - adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { + adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || + (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { r = adev->ip_blocks[i].version->funcs->resume(adev); if (r) { @@ -3501,6 +3512,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->gmc.gart_size = 512 * 1024 * 1024; adev->accel_working = false; adev->num_rings = 0; + RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); adev->mman.buffer_funcs = NULL; adev->mman.buffer_funcs_ring = NULL; adev->vm_manager.vm_pte_funcs = NULL; @@ -3982,6 +3994,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) release_firmware(adev->firmware.gpu_info_fw); adev->firmware.gpu_info_fw = NULL; adev->accel_working = false; + dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); amdgpu_reset_fini(adev); @@ -4057,12 +4070,20 @@ static void amdgpu_device_evict_resources(struct amdgpu_device *adev) int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) { struct amdgpu_device *adev = drm_to_adev(dev); + int r = 0; if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; adev->in_suspend = true; + if (amdgpu_sriov_vf(adev)) { + amdgpu_virt_fini_data_exchange(adev); + r = amdgpu_virt_request_full_gpu(adev, false); + if (r) + return r; + } + if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3)) DRM_WARN("smart shift update failed\n"); @@ -4086,6 +4107,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) amdgpu_device_ip_suspend_phase2(adev); + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_release_full_gpu(adev, false); + return 0; } @@ -4104,6 +4128,12 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) struct amdgpu_device *adev = drm_to_adev(dev); int r = 0; + if (amdgpu_sriov_vf(adev)) { + r = amdgpu_virt_request_full_gpu(adev, true); + if (r) + return r; + } + if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) return 0; @@ -4118,6 +4148,13 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) } r = amdgpu_device_ip_resume(adev); + + /* no matter what r is, always need to properly release full GPU */ + if (amdgpu_sriov_vf(adev)) { + amdgpu_virt_init_data_exchange(adev); + amdgpu_virt_release_full_gpu(adev, true); + } + if (r) { dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); return r; @@ -4739,6 +4776,7 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle, struct amdgpu_device *tmp_adev = NULL; bool need_full_reset, skip_hw_reset, vram_lost = false; int r = 0; + bool gpu_reset_for_dev_remove = 0; /* Try reset handler method first */ tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, @@ -4758,6 +4796,10 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle, test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); + gpu_reset_for_dev_remove = + test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) && + test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); + /* * ASIC reset has to be done on all XGMI hive nodes ASAP * to allow proper links negotiation in FW (within 1 sec) @@ -4802,6 +4844,18 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle, amdgpu_ras_intr_cleared(); } + /* Since the mode1 reset affects base ip blocks, the + * phase1 ip blocks need to be resumed. Otherwise there + * will be a BIOS signature error and the psp bootloader + * can't load kdb on the next amdgpu install. + */ + if (gpu_reset_for_dev_remove) { + list_for_each_entry(tmp_adev, device_list_handle, reset_list) + amdgpu_device_ip_resume_phase1(tmp_adev); + + goto end; + } + list_for_each_entry(tmp_adev, device_list_handle, reset_list) { if (need_full_reset) { /* post card */ @@ -5124,6 +5178,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, bool need_emergency_restart = false; bool audio_suspended = false; int tmp_vram_lost_counter; + bool gpu_reset_for_dev_remove = false; + + gpu_reset_for_dev_remove = + test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) && + test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); /* * Special case: RAS triggered and full reset isn't supported @@ -5159,8 +5218,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, */ INIT_LIST_HEAD(&device_list); if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) { - list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { list_add_tail(&tmp_adev->reset_list, &device_list); + if (gpu_reset_for_dev_remove && adev->shutdown) + tmp_adev->shutdown = true; + } if (!list_is_first(&adev->reset_list, &device_list)) list_rotate_to_front(&adev->reset_list, &device_list); device_list_handle = &device_list; @@ -5243,6 +5305,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, retry: /* Rest of adevs pre asic reset from XGMI hive. */ list_for_each_entry(tmp_adev, device_list_handle, reset_list) { + if (gpu_reset_for_dev_remove) { + /* Workaroud for ASICs need to disable SMC first */ + amdgpu_device_smu_fini_early(tmp_adev); + } r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); /*TODO Should we stop ?*/ if (r) { @@ -5276,6 +5342,9 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ adev->asic_reset_res = 0; goto retry; } + + if (!r && gpu_reset_for_dev_remove) + goto recover_end; } skip_hw_reset: @@ -5349,6 +5418,7 @@ skip_sched_resume: amdgpu_device_unset_mp1_state(tmp_adev); } +recover_end: tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, reset_list); amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); @@ -5531,9 +5601,9 @@ bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); resource_size_t aper_limit = adev->gmc.aper_base + adev->gmc.aper_size - 1; - bool p2p_access = !adev->gmc.xgmi.connected_to_cpu && - !(pci_p2pdma_distance_many(adev->pdev, - &peer_adev->dev, 1, true) < 0); + bool p2p_access = + !adev->gmc.xgmi.connected_to_cpu && + !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size && adev->gmc.real_vram_size == adev->gmc.visible_vram_size && @@ -5917,3 +5987,36 @@ void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, (void)RREG32(data); spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); } + +/** + * amdgpu_device_switch_gang - switch to a new gang + * @adev: amdgpu_device pointer + * @gang: the gang to switch to + * + * Try to switch to a new gang. + * Returns: NULL if we switched to the new gang or a reference to the current + * gang leader. + */ +struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, + struct dma_fence *gang) +{ + struct dma_fence *old = NULL; + + do { + dma_fence_put(old); + rcu_read_lock(); + old = dma_fence_get_rcu_safe(&adev->gang_submit); + rcu_read_unlock(); + + if (old == gang) + break; + + if (!dma_fence_is_signaled(old)) + return old; + + } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, + old, gang) != old); + + dma_fence_put(old); + return NULL; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 9fa2a5ceb77dabb033d145a9b7b956601812b283..3993e61349141b0e58e4dd4944ea9adec90eacb6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -229,7 +229,7 @@ static int amdgpu_discovery_read_binary_from_file(struct amdgpu_device *adev, ui return r; } - memcpy((u8 *)binary, (u8 *)fw->data, adev->mman.discovery_tmr_size); + memcpy((u8 *)binary, (u8 *)fw->data, fw->size); release_firmware(fw); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index c20922a5af9fc4d410a34bd12b7b06bbcb455363..1a06b8d724f391c411659b1d84977024361285e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -1101,6 +1101,7 @@ static int amdgpu_display_gem_fb_verify_and_init(struct drm_device *dev, goto err; ret = drm_framebuffer_init(dev, &rfb->base, &amdgpu_fb_funcs); + if (ret) goto err; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index 782cbca37538fd1d99b4d6dfd4d7ca812523ebb3..7bd8e33b14be5a478d71017040263f8905d92104 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -58,7 +58,7 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); int r; - if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0) + if (pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0) attach->peer2peer = false; r = pm_runtime_get_sync(adev_to_drm(adev)->dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 728a0933ea6fb56969bb32f0e10f3f52e7a91e75..3c9fecdd6b2f322fc7f1dbbb28aecf91739c84f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include "amdgpu.h" #include "amdgpu_irq.h" @@ -102,9 +104,10 @@ * - 3.46.0 - To enable hot plug amdgpu tests in libdrm * - 3.47.0 - Add AMDGPU_GEM_CREATE_DISCARDABLE and AMDGPU_VM_NOALLOC flags * - 3.48.0 - Add IP discovery version info to HW INFO + * 3.49.0 - Add gang submit into CS IOCTL */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 48 +#define KMS_DRIVER_MINOR 49 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit; @@ -185,6 +188,18 @@ int amdgpu_vcnfw_log; static void amdgpu_drv_delayed_reset_work_handler(struct work_struct *work); +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + struct amdgpu_mgpu_info mgpu_info = { .mutex = __MUTEX_INITIALIZER(mgpu_info.mutex), .delayed_reset_work = __DELAYED_WORK_INITIALIZER( @@ -2186,6 +2201,37 @@ amdgpu_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(dev->dev); } + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)) { + bool need_to_reset_gpu = false; + + if (adev->gmc.xgmi.num_physical_nodes > 1) { + struct amdgpu_hive_info *hive; + + hive = amdgpu_get_xgmi_hive(adev); + if (hive->device_remove_count == 0) + need_to_reset_gpu = true; + hive->device_remove_count++; + amdgpu_put_xgmi_hive(hive); + } else { + need_to_reset_gpu = true; + } + + /* Workaround for ASICs need to reset SMU. + * Called only when the first device is removed. + */ + if (need_to_reset_gpu) { + struct amdgpu_reset_context reset_context; + + adev->shutdown = true; + memset(&reset_context, 0, sizeof(reset_context)); + reset_context.method = AMD_RESET_METHOD_NONE; + reset_context.reset_req_dev = adev; + set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); + set_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context.flags); + amdgpu_device_gpu_recover(adev, NULL, &reset_context); + } + } + amdgpu_driver_unload_kms(dev); drm_dev_unplug(dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 8adeb7469f1e5715c0cb6a086468c11dc37b4871..d0d99ed607ddd4f7710a412b6592825395d9f8fb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -400,7 +400,6 @@ unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring) /* We are not protected by ring lock when reading the last sequence * but it's ok to report slightly wrong fence count here. */ - amdgpu_fence_process(ring); emitted = 0x100000000ull; emitted -= atomic_read(&ring->fence_drv.last_seq); emitted += READ_ONCE(ring->fence_drv.sync_seq); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index ceb91469958aa5aa5b9e9e0adfa8e32bf1adcbe6..9546adc8a76f6ccf48aa8823ffe9c0df8886cf9c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -23,6 +23,7 @@ * */ +#include #include "amdgpu.h" #include "amdgpu_gfx.h" #include "amdgpu_rlc.h" @@ -865,3 +866,142 @@ int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev) } return amdgpu_num_kcq; } + +void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, + uint32_t ucode_id) +{ + const struct gfx_firmware_header_v1_0 *cp_hdr; + const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0; + struct amdgpu_firmware_info *info = NULL; + const struct firmware *ucode_fw; + unsigned int fw_size; + + switch (ucode_id) { + case AMDGPU_UCODE_ID_CP_PFP: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.pfp_fw->data; + adev->gfx.pfp_fw_version = + le32_to_cpu(cp_hdr->header.ucode_version); + adev->gfx.pfp_feature_version = + le32_to_cpu(cp_hdr->ucode_feature_version); + ucode_fw = adev->gfx.pfp_fw; + fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_RS64_PFP: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.pfp_fw->data; + adev->gfx.pfp_fw_version = + le32_to_cpu(cp_hdr_v2_0->header.ucode_version); + adev->gfx.pfp_feature_version = + le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); + ucode_fw = adev->gfx.pfp_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK: + case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.pfp_fw->data; + ucode_fw = adev->gfx.pfp_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_ME: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.me_fw->data; + adev->gfx.me_fw_version = + le32_to_cpu(cp_hdr->header.ucode_version); + adev->gfx.me_feature_version = + le32_to_cpu(cp_hdr->ucode_feature_version); + ucode_fw = adev->gfx.me_fw; + fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_RS64_ME: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.me_fw->data; + adev->gfx.me_fw_version = + le32_to_cpu(cp_hdr_v2_0->header.ucode_version); + adev->gfx.me_feature_version = + le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); + ucode_fw = adev->gfx.me_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK: + case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.me_fw->data; + ucode_fw = adev->gfx.me_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_CE: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.ce_fw->data; + adev->gfx.ce_fw_version = + le32_to_cpu(cp_hdr->header.ucode_version); + adev->gfx.ce_feature_version = + le32_to_cpu(cp_hdr->ucode_feature_version); + ucode_fw = adev->gfx.ce_fw; + fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_MEC1: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.mec_fw->data; + adev->gfx.mec_fw_version = + le32_to_cpu(cp_hdr->header.ucode_version); + adev->gfx.mec_feature_version = + le32_to_cpu(cp_hdr->ucode_feature_version); + ucode_fw = adev->gfx.mec_fw; + fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - + le32_to_cpu(cp_hdr->jt_size) * 4; + break; + case AMDGPU_UCODE_ID_CP_MEC1_JT: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.mec_fw->data; + ucode_fw = adev->gfx.mec_fw; + fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; + break; + case AMDGPU_UCODE_ID_CP_MEC2: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.mec2_fw->data; + adev->gfx.mec2_fw_version = + le32_to_cpu(cp_hdr->header.ucode_version); + adev->gfx.mec2_feature_version = + le32_to_cpu(cp_hdr->ucode_feature_version); + ucode_fw = adev->gfx.mec2_fw; + fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - + le32_to_cpu(cp_hdr->jt_size) * 4; + break; + case AMDGPU_UCODE_ID_CP_MEC2_JT: + cp_hdr = (const struct gfx_firmware_header_v1_0 *) + adev->gfx.mec2_fw->data; + ucode_fw = adev->gfx.mec2_fw; + fw_size = le32_to_cpu(cp_hdr->jt_size) * 4; + break; + case AMDGPU_UCODE_ID_CP_RS64_MEC: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.mec_fw->data; + adev->gfx.mec_fw_version = + le32_to_cpu(cp_hdr_v2_0->header.ucode_version); + adev->gfx.mec_feature_version = + le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); + ucode_fw = adev->gfx.mec_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes); + break; + case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK: + case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK: + case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK: + case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK: + cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *) + adev->gfx.mec_fw->data; + ucode_fw = adev->gfx.mec_fw; + fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes); + break; + default: + break; + } + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + info = &adev->firmware.ucode[ucode_id]; + info->ucode_id = ucode_id; + info->fw = ucode_fw; + adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE); + } +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index 8abdf41d0f832c78cfb5db6e68d4bcebf9a1ed8d..832b3807f1d6bdba507372334acc6b1c849515e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -304,6 +304,10 @@ struct amdgpu_gfx { uint32_t rlc_srlg_feature_version; uint32_t rlc_srls_fw_version; uint32_t rlc_srls_feature_version; + uint32_t rlcp_ucode_version; + uint32_t rlcp_ucode_feature_version; + uint32_t rlcv_ucode_version; + uint32_t rlcv_ucode_feature_version; uint32_t mec_feature_version; uint32_t mec2_feature_version; bool mec_fw_write_wait; @@ -422,4 +426,6 @@ int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev, uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg); void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v); int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev); +void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, uint32_t ucode_id); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index aebc384531ac8fc100209dae588abc5408b4d614..34233a74248c2ffe5087e78d472b908690c3fc01 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -572,45 +572,15 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) { struct amdgpu_gmc *gmc = &adev->gmc; - - switch (adev->ip_versions[GC_HWIP][0]) { - case IP_VERSION(9, 0, 1): - case IP_VERSION(9, 3, 0): - case IP_VERSION(9, 4, 0): - case IP_VERSION(9, 4, 1): - case IP_VERSION(9, 4, 2): - case IP_VERSION(10, 3, 3): - case IP_VERSION(10, 3, 4): - case IP_VERSION(10, 3, 5): - case IP_VERSION(10, 3, 6): - case IP_VERSION(10, 3, 7): - /* - * noretry = 0 will cause kfd page fault tests fail - * for some ASICs, so set default to 1 for these ASICs. - */ - if (amdgpu_noretry == -1) - gmc->noretry = 1; - else - gmc->noretry = amdgpu_noretry; - break; - default: - /* Raven currently has issues with noretry - * regardless of what we decide for other - * asics, we should leave raven with - * noretry = 0 until we root cause the - * issues. - * - * default this to 0 for now, but we may want - * to change this in the future for certain - * GPUs as it can increase performance in - * certain cases. - */ - if (amdgpu_noretry == -1) - gmc->noretry = 0; - else - gmc->noretry = amdgpu_noretry; - break; - } + uint32_t gc_ver = adev->ip_versions[GC_HWIP][0]; + bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) || + gc_ver == IP_VERSION(9, 3, 0) || + gc_ver == IP_VERSION(9, 4, 0) || + gc_ver == IP_VERSION(9, 4, 1) || + gc_ver == IP_VERSION(9, 4, 2) || + gc_ver >= IP_VERSION(10, 3, 0)); + + gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; } void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 1062b7ed74ec13ea3f72f02c522b1851bc2006b5..46c99331d7f126a98d31631f1f03fac89c89d048 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -105,7 +105,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, */ (*job)->base.sched = &adev->rings[0]->sched; (*job)->vm = vm; - (*job)->num_ibs = num_ibs; amdgpu_sync_create(&(*job)->sync); amdgpu_sync_create(&(*job)->sched_sync); @@ -125,6 +124,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, if (r) return r; + (*job)->num_ibs = 1; r = amdgpu_ib_get(adev, NULL, size, pool_type, &(*job)->ibs[0]); if (r) kfree(*job); @@ -132,6 +132,23 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, return r; } +void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds, + struct amdgpu_bo *gws, struct amdgpu_bo *oa) +{ + if (gds) { + job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT; + job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT; + } + if (gws) { + job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT; + job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT; + } + if (oa) { + job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT; + job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT; + } +} + void amdgpu_job_free_resources(struct amdgpu_job *job) { struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched); @@ -156,11 +173,29 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job) dma_fence_put(&job->hw_fence); } +void amdgpu_job_set_gang_leader(struct amdgpu_job *job, + struct amdgpu_job *leader) +{ + struct dma_fence *fence = &leader->base.s_fence->scheduled; + + WARN_ON(job->gang_submit); + + /* + * Don't add a reference when we are the gang leader to avoid circle + * dependency. + */ + if (job != leader) + dma_fence_get(fence); + job->gang_submit = fence; +} + void amdgpu_job_free(struct amdgpu_job *job) { amdgpu_job_free_resources(job); amdgpu_sync_free(&job->sync); amdgpu_sync_free(&job->sched_sync); + if (job->gang_submit != &job->base.s_fence->scheduled) + dma_fence_put(job->gang_submit); if (!job->hw_fence.ops) kfree(job); @@ -230,12 +265,16 @@ static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job, fence = amdgpu_sync_get_fence(&job->sync); } + if (!fence && job->gang_submit) + fence = amdgpu_device_switch_gang(ring->adev, job->gang_submit); + return fence; } static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) { struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched); + struct amdgpu_device *adev = ring->adev; struct dma_fence *fence = NULL, *finished; struct amdgpu_job *job; int r = 0; @@ -247,8 +286,10 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) trace_amdgpu_sched_run_job(job); - if (job->vram_lost_counter != atomic_read(&ring->adev->vram_lost_counter)) - dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */ + /* Skip job if VRAM is lost and never resubmit gangs */ + if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter) || + (job->job_run_counter && job->gang_submit)) + dma_fence_set_error(finished, -ECANCELED); if (finished->error < 0) { DRM_INFO("Skip scheduling IBs!\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h index babc0af751c2f267ca639185b7a618443bc9af4b..ab7b150e5d50df8941d953fd5ecbf8ea2087370d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h @@ -50,6 +50,7 @@ struct amdgpu_job { struct amdgpu_sync sync; struct amdgpu_sync sched_sync; struct dma_fence hw_fence; + struct dma_fence *gang_submit; uint32_t preamble_status; uint32_t preemption_status; bool vm_needs_flush; @@ -72,11 +73,20 @@ struct amdgpu_job { struct amdgpu_ib ibs[]; }; +static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job) +{ + return to_amdgpu_ring(job->base.entity->rq->sched); +} + int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, struct amdgpu_job **job, struct amdgpu_vm *vm); int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, enum amdgpu_ib_pool_type pool, struct amdgpu_job **job); +void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds, + struct amdgpu_bo *gws, struct amdgpu_bo *oa); void amdgpu_job_free_resources(struct amdgpu_job *job); +void amdgpu_job_set_gang_leader(struct amdgpu_job *job, + struct amdgpu_job *leader); void amdgpu_job_free(struct amdgpu_job *job); int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, void *owner, struct dma_fence **f); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 77668c3dae5bfdeb98799e38daca83b12f00bb2d..fe23e09eec985ca291f57edfc9c183cc8928182c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -247,6 +247,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->ver = adev->gfx.rlc_srls_fw_version; fw_info->feature = adev->gfx.rlc_srls_feature_version; break; + case AMDGPU_INFO_FW_GFX_RLCP: + fw_info->ver = adev->gfx.rlcp_ucode_version; + fw_info->feature = adev->gfx.rlcp_ucode_feature_version; + break; + case AMDGPU_INFO_FW_GFX_RLCV: + fw_info->ver = adev->gfx.rlcv_ucode_version; + fw_info->feature = adev->gfx.rlcv_ucode_feature_version; + break; case AMDGPU_INFO_FW_GFX_MEC: if (query_fw->index == 0) { fw_info->ver = adev->gfx.mec_fw_version; @@ -328,6 +336,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->ver = adev->psp.cap_fw_version; fw_info->feature = adev->psp.cap_feature_version; break; + case AMDGPU_INFO_FW_MES_KIQ: + fw_info->ver = adev->mes.ucode_fw_version[0]; + fw_info->feature = 0; + break; + case AMDGPU_INFO_FW_MES: + fw_info->ver = adev->mes.ucode_fw_version[1]; + fw_info->feature = 0; + break; default: return -EINVAL; } @@ -1469,6 +1485,22 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused) seq_printf(m, "RLC SRLS feature version: %u, firmware version: 0x%08x\n", fw_info.feature, fw_info.ver); + /* RLCP */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCP; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLCP feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* RLCV */ + query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCV; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "RLCV feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + /* MEC */ query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC; query_fw.index = 0; @@ -1581,6 +1613,22 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused) fw_info.feature, fw_info.ver); } + /* MES_KIQ */ + query_fw.fw_type = AMDGPU_INFO_FW_MES_KIQ; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MES_KIQ feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + + /* MES */ + query_fw.fw_type = AMDGPU_INFO_FW_MES; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "MES feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + seq_printf(m, "VBIOS version: %s\n", ctx->vbios_version); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index 7b46f6bf41875fea422aea1576927c56da89a9be..ad980f4b66e19143899169cc83e19d0e7c1efdf7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -222,6 +222,8 @@ struct mes_add_queue_input { uint64_t tba_addr; uint64_t tma_addr; uint32_t is_kfd_process; + uint32_t is_aql_queue; + uint32_t queue_size; }; struct mes_remove_queue_input { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e6a9b9fc9e0bb1fb5996a80ed5b447dc5efbbd3e..2e8f6cd7a72935fb817a80b76c15c7c44df6aeac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -688,13 +688,16 @@ int amdgpu_bo_create_vm(struct amdgpu_device *adev, * num of amdgpu_vm_pt entries. */ BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); - bp->destroy = &amdgpu_bo_vm_destroy; r = amdgpu_bo_create(adev, bp, &bo_ptr); if (r) return r; *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); INIT_LIST_HEAD(&(*vmbo_ptr)->shadow_list); + /* Set destroy callback to amdgpu_bo_vm_destroy after vmbo->shadow_list + * is initialized. + */ + bo_ptr->tbo.destroy = &amdgpu_bo_vm_destroy; return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index cfcaf890a6a1224eb02b7ff9a8f44db4c7b4d40b..effa7df3ddbfa47aafd26dc6875f71e56d439167 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -511,6 +511,11 @@ static int psp_sw_fini(void *handle) kfree(cmd); cmd = NULL; + if (psp->km_ring.ring_mem) + amdgpu_bo_free_kernel(&adev->firmware.rbuf, + &psp->km_ring.ring_mem_mc_addr, + (void **)&psp->km_ring.ring_mem); + amdgpu_bo_free_kernel(&psp->fw_pri_bo, &psp->fw_pri_mc_addr, &psp->fw_pri_buf); amdgpu_bo_free_kernel(&psp->fence_buf_bo, @@ -766,7 +771,7 @@ static int psp_tmr_init(struct psp_context *psp) } pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL; - ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_SIZE(psp->adev), + ret = amdgpu_bo_create_kernel(psp->adev, tmr_size, PSP_TMR_ALIGNMENT, AMDGPU_GEM_DOMAIN_VRAM, &psp->tmr_bo, &psp->tmr_mc_addr, pptr); @@ -2055,6 +2060,15 @@ static int psp_hw_start(struct psp_context *psp) } } + if ((is_psp_fw_valid(psp->ras_drv)) && + (psp->funcs->bootloader_load_ras_drv != NULL)) { + ret = psp_bootloader_load_ras_drv(psp); + if (ret) { + DRM_ERROR("PSP load ras_drv failed!\n"); + return ret; + } + } + if ((is_psp_fw_valid(psp->sos)) && (psp->funcs->bootloader_load_sos != NULL)) { ret = psp_bootloader_load_sos(psp); @@ -3040,6 +3054,12 @@ static int parse_sos_bin_descriptor(struct psp_context *psp, psp->dbg_drv.size_bytes = le32_to_cpu(desc->size_bytes); psp->dbg_drv.start_addr = ucode_start_addr; break; + case PSP_FW_TYPE_PSP_RAS_DRV: + psp->ras_drv.fw_version = le32_to_cpu(desc->fw_version); + psp->ras_drv.feature_version = le32_to_cpu(desc->fw_version); + psp->ras_drv.size_bytes = le32_to_cpu(desc->size_bytes); + psp->ras_drv.start_addr = ucode_start_addr; + break; default: dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type); break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index c32b74bd970fc53ecb0dc6c3bc833cdf4ac0bc50..58ce3ebb446cf802ab8fa8fcea7d793e4af60313 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -36,6 +36,7 @@ #define PSP_CMD_BUFFER_SIZE 0x1000 #define PSP_1_MEG 0x100000 #define PSP_TMR_SIZE(adev) ((adev)->asic_type == CHIP_ALDEBARAN ? 0x800000 : 0x400000) +#define PSP_TMR_ALIGNMENT 0x100000 #define PSP_FW_NAME_LEN 0x24 enum psp_shared_mem_size { @@ -71,6 +72,7 @@ enum psp_bootloader_cmd { PSP_BL__LOAD_SOCDRV = 0xB0000, PSP_BL__LOAD_DBGDRV = 0xC0000, PSP_BL__LOAD_INTFDRV = 0xD0000, + PSP_BL__LOAD_RASDRV = 0xE0000, PSP_BL__DRAM_LONG_TRAIN = 0x100000, PSP_BL__DRAM_SHORT_TRAIN = 0x200000, PSP_BL__LOAD_TOS_SPL_TABLE = 0x10000000, @@ -114,6 +116,7 @@ struct psp_funcs int (*bootloader_load_soc_drv)(struct psp_context *psp); int (*bootloader_load_intf_drv)(struct psp_context *psp); int (*bootloader_load_dbg_drv)(struct psp_context *psp); + int (*bootloader_load_ras_drv)(struct psp_context *psp); int (*bootloader_load_sos)(struct psp_context *psp); int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type); int (*ring_create)(struct psp_context *psp, @@ -323,6 +326,7 @@ struct psp_context struct psp_bin_desc soc_drv; struct psp_bin_desc intf_drv; struct psp_bin_desc dbg_drv; + struct psp_bin_desc ras_drv; /* tmr buffer */ struct amdgpu_bo *tmr_bo; @@ -403,6 +407,9 @@ struct amdgpu_psp_funcs { ((psp)->funcs->bootloader_load_intf_drv ? (psp)->funcs->bootloader_load_intf_drv((psp)) : 0) #define psp_bootloader_load_dbg_drv(psp) \ ((psp)->funcs->bootloader_load_dbg_drv ? (psp)->funcs->bootloader_load_dbg_drv((psp)) : 0) +#define psp_bootloader_load_ras_drv(psp) \ + ((psp)->funcs->bootloader_load_ras_drv ? \ + (psp)->funcs->bootloader_load_ras_drv((psp)) : 0) #define psp_bootloader_load_sos(psp) \ ((psp)->funcs->bootloader_load_sos ? (psp)->funcs->bootloader_load_sos((psp)) : 0) #define psp_smu_reload_quirk(psp) \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index ab9ba5a9c33dbebbe146918ca5e4a5775f37b5dc..2dad7aa9a03b94737dfc4948bc82b5eaac3991b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -1811,7 +1811,8 @@ static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev) amdgpu_ras_query_error_status(adev, &info); if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) && - adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) { + adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) && + adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) { if (amdgpu_ras_reset_error_status(adev, info.head.block)) dev_warn(adev->dev, "Failed to reset error counter and error status"); } @@ -2719,7 +2720,8 @@ int amdgpu_ras_pre_fini(struct amdgpu_device *adev) /* Need disable ras on all IPs here before ip [hw/sw]fini */ - amdgpu_ras_disable_all_features(adev, 0); + if (con->features) + amdgpu_ras_disable_all_features(adev, 0); amdgpu_ras_recovery_fini(adev); return 0; } @@ -2832,11 +2834,8 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, struct mce *m = (struct mce *)data; struct amdgpu_device *adev = NULL; uint32_t gpu_id = 0; - uint32_t umc_inst = 0; - uint32_t ch_inst, channel_index = 0; + uint32_t umc_inst = 0, ch_inst = 0; struct ras_err_data err_data = {0, 0, 0, NULL}; - struct eeprom_table_record err_rec; - uint64_t retired_page; /* * If the error was generated in UMC_V2, which belongs to GPU UMCs, @@ -2875,21 +2874,22 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", umc_inst, ch_inst); + err_data.err_addr = + kcalloc(adev->umc.max_ras_err_cnt_per_query, + sizeof(struct eeprom_table_record), GFP_KERNEL); + if (!err_data.err_addr) { + dev_warn(adev->dev, + "Failed to alloc memory for umc error record in mca notifier!\n"); + return NOTIFY_DONE; + } + /* * Translate UMC channel address to Physical address */ - channel_index = - adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num - + ch_inst]; - - retired_page = ADDR_OF_8KB_BLOCK(m->addr) | - ADDR_OF_256B_BLOCK(channel_index) | - OFFSET_IN_256B_BLOCK(m->addr); - - memset(&err_rec, 0x0, sizeof(struct eeprom_table_record)); - err_data.err_addr = &err_rec; - amdgpu_umc_fill_error_record(&err_data, m->addr, - retired_page, channel_index, umc_inst); + if (adev->umc.ras && + adev->umc.ras->convert_ras_error_address) + adev->umc.ras->convert_ras_error_address(adev, + &err_data, m->addr, ch_inst, umc_inst); if (amdgpu_bad_page_threshold != 0) { amdgpu_ras_add_bad_pages(adev, err_data.err_addr, @@ -2897,6 +2897,7 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, amdgpu_ras_save_bad_pages(adev); } + kfree(err_data.err_addr); return NOTIFY_OK; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index c4283987bb1e89e59670aeb0c31391f84946f886..84c241b9a2a1335978a9e71bf5e5bfa376f1caae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -38,6 +38,7 @@ #define EEPROM_I2C_MADDR_ARCTURUS_D342 0x0 #define EEPROM_I2C_MADDR_SIENNA_CICHLID 0x0 #define EEPROM_I2C_MADDR_ALDEBARAN 0x0 +#define EEPROM_I2C_MADDR_SMU_13_0_0 (0x54UL << 16) /* * The 2 macros bellow represent the actual size in bytes that @@ -156,6 +157,15 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, return false; } + switch (adev->ip_versions[MP1_HWIP][0]) { + case IP_VERSION(13, 0, 0): + control->i2c_address = EEPROM_I2C_MADDR_SMU_13_0_0; + break; + + default: + break; + } + return true; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h index f71b83c425908e970f35b9bb54333fa8deee86aa..f5318fedf2f0460fd287b837a8b2d4a209f0248c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h @@ -31,6 +31,7 @@ enum AMDGPU_RESET_FLAGS { AMDGPU_NEED_FULL_RESET = 0, AMDGPU_SKIP_HW_RESET = 1, AMDGPU_SKIP_MODE2_RESET = 2, + AMDGPU_RESET_FOR_DEVICE_REMOVE = 3, }; struct amdgpu_reset_context { @@ -112,7 +113,8 @@ static inline bool amdgpu_reset_get_reset_domain(struct amdgpu_reset_domain *dom static inline void amdgpu_reset_put_reset_domain(struct amdgpu_reset_domain *domain) { - kref_put(&domain->refcount, amdgpu_reset_destroy_reset_domain); + if (domain) + kref_put(&domain->refcount, amdgpu_reset_destroy_reset_domain); } static inline bool amdgpu_reset_domain_schedule(struct amdgpu_reset_domain *domain, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c index 6373bfb47d55d7b366b46095c25682e8b3f68717..012b72d00e0400d0c0f7bd3f927d8d2ff9076a07 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c @@ -272,3 +272,275 @@ void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev) &adev->gfx.rlc.cp_table_gpu_addr, (void **)&adev->gfx.rlc.cp_table_ptr); } + +static int amdgpu_gfx_rlc_init_microcode_v2_0(struct amdgpu_device *adev) +{ + const struct common_firmware_header *common_hdr; + const struct rlc_firmware_header_v2_0 *rlc_hdr; + struct amdgpu_firmware_info *info; + unsigned int *tmp; + unsigned int i; + + rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; + + adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); + adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); + adev->gfx.rlc.save_and_restore_offset = + le32_to_cpu(rlc_hdr->save_and_restore_offset); + adev->gfx.rlc.clear_state_descriptor_offset = + le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); + adev->gfx.rlc.avail_scratch_ram_locations = + le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); + adev->gfx.rlc.reg_restore_list_size = + le32_to_cpu(rlc_hdr->reg_restore_list_size); + adev->gfx.rlc.reg_list_format_start = + le32_to_cpu(rlc_hdr->reg_list_format_start); + adev->gfx.rlc.reg_list_format_separate_start = + le32_to_cpu(rlc_hdr->reg_list_format_separate_start); + adev->gfx.rlc.starting_offsets_start = + le32_to_cpu(rlc_hdr->starting_offsets_start); + adev->gfx.rlc.reg_list_format_size_bytes = + le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); + adev->gfx.rlc.reg_list_size_bytes = + le32_to_cpu(rlc_hdr->reg_list_size_bytes); + adev->gfx.rlc.register_list_format = + kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + + adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); + if (!adev->gfx.rlc.register_list_format) { + dev_err(adev->dev, "failed to allocate memory for rlc register_list_format\n"); + return -ENOMEM; + } + + tmp = (unsigned int *)((uintptr_t)rlc_hdr + + le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); + for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++) + adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); + + adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; + + tmp = (unsigned int *)((uintptr_t)rlc_hdr + + le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); + for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++) + adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_G; + info->fw = adev->gfx.rlc_fw; + if (info->fw) { + common_hdr = (const struct common_firmware_header *)info->fw->data; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(common_hdr->ucode_size_bytes), PAGE_SIZE); + } + } + + return 0; +} + +static void amdgpu_gfx_rlc_init_microcode_v2_1(struct amdgpu_device *adev) +{ + const struct rlc_firmware_header_v2_1 *rlc_hdr; + struct amdgpu_firmware_info *info; + + rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; + adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); + adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); + adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); + adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); + adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); + adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); + adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); + adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); + adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); + adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); + adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); + adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); + adev->gfx.rlc.reg_list_format_direct_reg_list_length = + le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + if (adev->gfx.rlc.save_restore_list_cntl_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.save_restore_list_gpm_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.save_restore_list_srm_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); + } + } +} + +static void amdgpu_gfx_rlc_init_microcode_v2_2(struct amdgpu_device *adev) +{ + const struct rlc_firmware_header_v2_2 *rlc_hdr; + struct amdgpu_firmware_info *info; + + rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; + adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes); + adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes); + adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes); + adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + if (adev->gfx.rlc.rlc_iram_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.rlc_dram_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE); + } + } +} + +static void amdgpu_gfx_rlc_init_microcode_v2_3(struct amdgpu_device *adev) +{ + const struct rlc_firmware_header_v2_3 *rlc_hdr; + struct amdgpu_firmware_info *info; + + rlc_hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; + adev->gfx.rlcp_ucode_version = le32_to_cpu(rlc_hdr->rlcp_ucode_version); + adev->gfx.rlcp_ucode_feature_version = le32_to_cpu(rlc_hdr->rlcp_ucode_feature_version); + adev->gfx.rlc.rlcp_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcp_ucode_size_bytes); + adev->gfx.rlc.rlcp_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcp_ucode_offset_bytes); + + adev->gfx.rlcv_ucode_version = le32_to_cpu(rlc_hdr->rlcv_ucode_version); + adev->gfx.rlcv_ucode_feature_version = le32_to_cpu(rlc_hdr->rlcv_ucode_feature_version); + adev->gfx.rlc.rlcv_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcv_ucode_size_bytes); + adev->gfx.rlc.rlcv_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcv_ucode_offset_bytes); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + if (adev->gfx.rlc.rlcp_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_P]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_P; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlcp_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.rlcv_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_V]; + info->ucode_id = AMDGPU_UCODE_ID_RLC_V; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.rlcv_ucode_size_bytes, PAGE_SIZE); + } + } +} + +static void amdgpu_gfx_rlc_init_microcode_v2_4(struct amdgpu_device *adev) +{ + const struct rlc_firmware_header_v2_4 *rlc_hdr; + struct amdgpu_firmware_info *info; + + rlc_hdr = (const struct rlc_firmware_header_v2_4 *)adev->gfx.rlc_fw->data; + adev->gfx.rlc.global_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->global_tap_delays_ucode_size_bytes); + adev->gfx.rlc.global_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->global_tap_delays_ucode_offset_bytes); + adev->gfx.rlc.se0_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_size_bytes); + adev->gfx.rlc.se0_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_offset_bytes); + adev->gfx.rlc.se1_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_size_bytes); + adev->gfx.rlc.se1_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_offset_bytes); + adev->gfx.rlc.se2_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_size_bytes); + adev->gfx.rlc.se2_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_offset_bytes); + adev->gfx.rlc.se3_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_size_bytes); + adev->gfx.rlc.se3_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_offset_bytes); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + if (adev->gfx.rlc.global_tap_delays_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS]; + info->ucode_id = AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.global_tap_delays_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.se0_tap_delays_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE0_TAP_DELAYS]; + info->ucode_id = AMDGPU_UCODE_ID_SE0_TAP_DELAYS; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.se0_tap_delays_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.se1_tap_delays_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE1_TAP_DELAYS]; + info->ucode_id = AMDGPU_UCODE_ID_SE1_TAP_DELAYS; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.se1_tap_delays_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.se2_tap_delays_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE2_TAP_DELAYS]; + info->ucode_id = AMDGPU_UCODE_ID_SE2_TAP_DELAYS; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.se2_tap_delays_ucode_size_bytes, PAGE_SIZE); + } + + if (adev->gfx.rlc.se3_tap_delays_ucode_size_bytes) { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE3_TAP_DELAYS]; + info->ucode_id = AMDGPU_UCODE_ID_SE3_TAP_DELAYS; + info->fw = adev->gfx.rlc_fw; + adev->firmware.fw_size += + ALIGN(adev->gfx.rlc.se3_tap_delays_ucode_size_bytes, PAGE_SIZE); + } + } +} + +int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev, + uint16_t version_major, + uint16_t version_minor) +{ + int err; + + if (version_major < 2) { + /* only support rlc_hdr v2.x and onwards */ + dev_err(adev->dev, "unsupported rlc fw hdr\n"); + return -EINVAL; + } + + /* is_rlc_v2_1 is still used in APU code path */ + if (version_major == 2 && version_minor == 1) + adev->gfx.rlc.is_rlc_v2_1 = true; + + if (version_minor >= 0) { + err = amdgpu_gfx_rlc_init_microcode_v2_0(adev); + if (err) { + dev_err(adev->dev, "fail to init rlc v2_0 microcode\n"); + return err; + } + } + if (version_minor >= 1) + amdgpu_gfx_rlc_init_microcode_v2_1(adev); + if (version_minor >= 2) + amdgpu_gfx_rlc_init_microcode_v2_2(adev); + if (version_minor == 3) + amdgpu_gfx_rlc_init_microcode_v2_3(adev); + if (version_minor == 4) + amdgpu_gfx_rlc_init_microcode_v2_4(adev); + + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h index 03ac36b2c2cfa90802d3b42500382db9e566cf65..23f060db9255ce92479f112f02d00b520f8a1a77 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h @@ -267,5 +267,7 @@ int amdgpu_gfx_rlc_init_csb(struct amdgpu_device *adev); int amdgpu_gfx_rlc_init_cpt(struct amdgpu_device *adev); void amdgpu_gfx_rlc_setup_cp_table(struct amdgpu_device *adev); void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev); - +int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev, + uint16_t version_major, + uint16_t version_minor); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 42c1f050542f6e54766726c3878bc5574ac63f59..ea5278f094c0814b7519124ef8578b8e901fc57e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -21,6 +21,7 @@ * */ +#include #include "amdgpu.h" #include "amdgpu_sdma.h" #include "amdgpu_ras.h" @@ -150,3 +151,158 @@ int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev, amdgpu_ras_interrupt_dispatch(adev, &ih_data); return 0; } + +static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) +{ + int err = 0; + uint16_t version_major; + const struct common_firmware_header *header = NULL; + const struct sdma_firmware_header_v1_0 *hdr; + const struct sdma_firmware_header_v2_0 *hdr_v2; + + err = amdgpu_ucode_validate(sdma_inst->fw); + if (err) + return err; + + header = (const struct common_firmware_header *) + sdma_inst->fw->data; + version_major = le16_to_cpu(header->header_version_major); + + switch (version_major) { + case 1: + hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; + sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); + sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); + break; + case 2: + hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data; + sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version); + sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version); + break; + default: + return -EINVAL; + } + + if (sdma_inst->feature_version >= 20) + sdma_inst->burst_nop = true; + + return 0; +} + +void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev, + bool duplicate) +{ + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) { + release_firmware(adev->sdma.instance[i].fw); + if (duplicate) + break; + } + + memset((void *)adev->sdma.instance, 0, + sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); +} + +int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, + char *fw_name, u32 instance, + bool duplicate) +{ + struct amdgpu_firmware_info *info = NULL; + const struct common_firmware_header *header = NULL; + int err = 0, i; + const struct sdma_firmware_header_v2_0 *sdma_hdr; + uint16_t version_major; + + err = request_firmware(&adev->sdma.instance[instance].fw, fw_name, adev->dev); + if (err) + goto out; + + header = (const struct common_firmware_header *) + adev->sdma.instance[instance].fw->data; + version_major = le16_to_cpu(header->header_version_major); + + if ((duplicate && instance) || (!duplicate && version_major > 1)) { + err = -EINVAL; + goto out; + } + + err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]); + if (err) + goto out; + + if (duplicate) { + for (i = 1; i < adev->sdma.num_instances; i++) + memcpy((void *)&adev->sdma.instance[i], + (void *)&adev->sdma.instance[0], + sizeof(struct amdgpu_sdma_instance)); + } + + if (amdgpu_sriov_vf(adev)) + return 0; + + DRM_DEBUG("psp_load == '%s'\n", + adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + switch (version_major) { + case 1: + for (i = 0; i < adev->sdma.num_instances; i++) { + if (!duplicate && (instance != i)) + continue; + else { + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; + info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; + info->fw = adev->sdma.instance[i].fw; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); + } + } + break; + case 2: + sdma_hdr = (const struct sdma_firmware_header_v2_0 *) + adev->sdma.instance[0].fw->data; + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0]; + info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0; + info->fw = adev->sdma.instance[0].fw; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE); + info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1]; + info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1; + info->fw = adev->sdma.instance[0].fw; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE); + break; + default: + err = -EINVAL; + } + } + +out: + if (err) { + DRM_ERROR("SDMA: Failed to init firmware \"%s\"\n", fw_name); + amdgpu_sdma_destroy_inst_ctx(adev, duplicate); + } + return err; +} + +void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev) +{ + struct amdgpu_ring *sdma; + int i; + + for (i = 0; i < adev->sdma.num_instances; i++) { + if (adev->sdma.has_page_queue) { + sdma = &adev->sdma.instance[i].page; + if (adev->mman.buffer_funcs_ring == sdma) { + amdgpu_ttm_set_buffer_funcs_status(adev, false); + break; + } + } + sdma = &adev->sdma.instance[i].ring; + if (adev->mman.buffer_funcs_ring == sdma) { + amdgpu_ttm_set_buffer_funcs_status(adev, false); + break; + } + } +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h index 53ac3ebae8d61b825cebdc9de36a3fd25728b6b4..7d99205c2e018bfe4268e48647efed1a63423323 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h @@ -124,4 +124,10 @@ int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev, int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry); +int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, + char *fw_name, u32 instance, bool duplicate); +void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev, + bool duplicate); +void amdgpu_sdma_unset_buffer_funcs_helper(struct amdgpu_device *adev); + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 504af1b93bfa84e728127b7e1ee6afc035fe4156..090e66a1b2842e9ba6fff460ead0ff689e939c96 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: MIT /* * Copyright 2014 Advanced Micro Devices, Inc. * All Rights Reserved. @@ -315,6 +316,7 @@ struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) struct hlist_node *tmp; struct dma_fence *f; int i; + hash_for_each_safe(sync->fences, i, tmp, e, node) { f = e->fence; @@ -392,7 +394,7 @@ void amdgpu_sync_free(struct amdgpu_sync *sync) { struct amdgpu_sync_entry *e; struct hlist_node *tmp; - unsigned i; + unsigned int i; hash_for_each_safe(sync->fences, i, tmp, e, node) { hash_del(&e->node); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 06dfcf297a8dbe3805aa69688feef574235fa357..5e6ddc7e101c6aa1c4f2412530716f8509eef39c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -140,8 +140,10 @@ TRACE_EVENT(amdgpu_bo_create, ); TRACE_EVENT(amdgpu_cs, - TP_PROTO(struct amdgpu_cs_parser *p, int i), - TP_ARGS(p, i), + TP_PROTO(struct amdgpu_cs_parser *p, + struct amdgpu_job *job, + struct amdgpu_ib *ib), + TP_ARGS(p, job, ib), TP_STRUCT__entry( __field(struct amdgpu_bo_list *, bo_list) __field(u32, ring) @@ -151,10 +153,10 @@ TRACE_EVENT(amdgpu_cs, TP_fast_assign( __entry->bo_list = p->bo_list; - __entry->ring = to_amdgpu_ring(p->entity->rq->sched)->idx; - __entry->dw = p->job->ibs[i].length_dw; + __entry->ring = to_amdgpu_ring(job->base.sched)->idx; + __entry->dw = ib->length_dw; __entry->fences = amdgpu_fence_count_emitted( - to_amdgpu_ring(p->entity->rq->sched)); + to_amdgpu_ring(job->base.sched)); ), TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u", __entry->bo_list, __entry->ring, __entry->dw, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index b1c455329023ad7fcb9ed18692b039e412e188de..dc262d2c2925ef964a9cd33b20b5b802b68e6c6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -424,8 +424,9 @@ error: static bool amdgpu_mem_visible(struct amdgpu_device *adev, struct ttm_resource *mem) { - uint64_t mem_size = (u64)mem->num_pages << PAGE_SHIFT; + u64 mem_size = (u64)mem->num_pages << PAGE_SHIFT; struct amdgpu_res_cursor cursor; + u64 end; if (mem->mem_type == TTM_PL_SYSTEM || mem->mem_type == TTM_PL_TT) @@ -434,12 +435,18 @@ static bool amdgpu_mem_visible(struct amdgpu_device *adev, return false; amdgpu_res_first(mem, 0, mem_size, &cursor); + end = cursor.start + cursor.size; + while (cursor.remaining) { + amdgpu_res_next(&cursor, cursor.size); - /* ttm_resource_ioremap only supports contiguous memory */ - if (cursor.size != mem_size) - return false; + /* ttm_resource_ioremap only supports contiguous memory */ + if (end != cursor.start) + return false; + + end = cursor.start + cursor.size; + } - return cursor.start + cursor.size <= adev->gmc.visible_vram_size; + return end <= adev->gmc.visible_vram_size; } /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index 939c8614f0e3378cdb0075f196aa97f9108fa7f6..dd0bc649a57d107dc7482b17bcd42a38908b7686 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -164,70 +164,138 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr) } else if (version_major == 2) { const struct rlc_firmware_header_v2_0 *rlc_hdr = container_of(hdr, struct rlc_firmware_header_v2_0, header); + const struct rlc_firmware_header_v2_1 *rlc_hdr_v2_1 = + container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0); + const struct rlc_firmware_header_v2_2 *rlc_hdr_v2_2 = + container_of(rlc_hdr_v2_1, struct rlc_firmware_header_v2_2, v2_1); + const struct rlc_firmware_header_v2_3 *rlc_hdr_v2_3 = + container_of(rlc_hdr_v2_2, struct rlc_firmware_header_v2_3, v2_2); + const struct rlc_firmware_header_v2_4 *rlc_hdr_v2_4 = + container_of(rlc_hdr_v2_3, struct rlc_firmware_header_v2_4, v2_3); - DRM_DEBUG("ucode_feature_version: %u\n", - le32_to_cpu(rlc_hdr->ucode_feature_version)); - DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(rlc_hdr->jt_offset)); - DRM_DEBUG("jt_size: %u\n", le32_to_cpu(rlc_hdr->jt_size)); - DRM_DEBUG("save_and_restore_offset: %u\n", - le32_to_cpu(rlc_hdr->save_and_restore_offset)); - DRM_DEBUG("clear_state_descriptor_offset: %u\n", - le32_to_cpu(rlc_hdr->clear_state_descriptor_offset)); - DRM_DEBUG("avail_scratch_ram_locations: %u\n", - le32_to_cpu(rlc_hdr->avail_scratch_ram_locations)); - DRM_DEBUG("reg_restore_list_size: %u\n", - le32_to_cpu(rlc_hdr->reg_restore_list_size)); - DRM_DEBUG("reg_list_format_start: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_start)); - DRM_DEBUG("reg_list_format_separate_start: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_separate_start)); - DRM_DEBUG("starting_offsets_start: %u\n", - le32_to_cpu(rlc_hdr->starting_offsets_start)); - DRM_DEBUG("reg_list_format_size_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_size_bytes)); - DRM_DEBUG("reg_list_format_array_offset_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); - DRM_DEBUG("reg_list_size_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_size_bytes)); - DRM_DEBUG("reg_list_array_offset_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); - DRM_DEBUG("reg_list_format_separate_size_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_separate_size_bytes)); - DRM_DEBUG("reg_list_format_separate_array_offset_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes)); - DRM_DEBUG("reg_list_separate_size_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes)); - DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n", - le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes)); - if (version_minor == 1) { - const struct rlc_firmware_header_v2_1 *v2_1 = - container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0); + switch (version_minor) { + case 0: + /* rlc_hdr v2_0 */ + DRM_DEBUG("ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr->ucode_feature_version)); + DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(rlc_hdr->jt_offset)); + DRM_DEBUG("jt_size: %u\n", le32_to_cpu(rlc_hdr->jt_size)); + DRM_DEBUG("save_and_restore_offset: %u\n", + le32_to_cpu(rlc_hdr->save_and_restore_offset)); + DRM_DEBUG("clear_state_descriptor_offset: %u\n", + le32_to_cpu(rlc_hdr->clear_state_descriptor_offset)); + DRM_DEBUG("avail_scratch_ram_locations: %u\n", + le32_to_cpu(rlc_hdr->avail_scratch_ram_locations)); + DRM_DEBUG("reg_restore_list_size: %u\n", + le32_to_cpu(rlc_hdr->reg_restore_list_size)); + DRM_DEBUG("reg_list_format_start: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_start)); + DRM_DEBUG("reg_list_format_separate_start: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_start)); + DRM_DEBUG("starting_offsets_start: %u\n", + le32_to_cpu(rlc_hdr->starting_offsets_start)); + DRM_DEBUG("reg_list_format_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_size_bytes)); + DRM_DEBUG("reg_list_format_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); + DRM_DEBUG("reg_list_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_size_bytes)); + DRM_DEBUG("reg_list_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); + DRM_DEBUG("reg_list_format_separate_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_size_bytes)); + DRM_DEBUG("reg_list_format_separate_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes)); + DRM_DEBUG("reg_list_separate_size_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes)); + DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes)); + break; + case 1: + /* rlc_hdr v2_1 */ DRM_DEBUG("reg_list_format_direct_reg_list_length: %u\n", - le32_to_cpu(v2_1->reg_list_format_direct_reg_list_length)); + le32_to_cpu(rlc_hdr_v2_1->reg_list_format_direct_reg_list_length)); DRM_DEBUG("save_restore_list_cntl_ucode_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_cntl_ucode_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_ucode_ver)); DRM_DEBUG("save_restore_list_cntl_feature_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_cntl_feature_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_feature_ver)); DRM_DEBUG("save_restore_list_cntl_size_bytes %u\n", - le32_to_cpu(v2_1->save_restore_list_cntl_size_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_size_bytes)); DRM_DEBUG("save_restore_list_cntl_offset_bytes: %u\n", - le32_to_cpu(v2_1->save_restore_list_cntl_offset_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_offset_bytes)); DRM_DEBUG("save_restore_list_gpm_ucode_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_gpm_ucode_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_ucode_ver)); DRM_DEBUG("save_restore_list_gpm_feature_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_gpm_feature_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_feature_ver)); DRM_DEBUG("save_restore_list_gpm_size_bytes %u\n", - le32_to_cpu(v2_1->save_restore_list_gpm_size_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_size_bytes)); DRM_DEBUG("save_restore_list_gpm_offset_bytes: %u\n", - le32_to_cpu(v2_1->save_restore_list_gpm_offset_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_offset_bytes)); DRM_DEBUG("save_restore_list_srm_ucode_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_srm_ucode_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_ucode_ver)); DRM_DEBUG("save_restore_list_srm_feature_ver: %u\n", - le32_to_cpu(v2_1->save_restore_list_srm_feature_ver)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_feature_ver)); DRM_DEBUG("save_restore_list_srm_size_bytes %u\n", - le32_to_cpu(v2_1->save_restore_list_srm_size_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_size_bytes)); DRM_DEBUG("save_restore_list_srm_offset_bytes: %u\n", - le32_to_cpu(v2_1->save_restore_list_srm_offset_bytes)); + le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_offset_bytes)); + break; + case 2: + /* rlc_hdr v2_2 */ + DRM_DEBUG("rlc_iram_ucode_size_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_2->rlc_iram_ucode_size_bytes)); + DRM_DEBUG("rlc_iram_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_2->rlc_iram_ucode_offset_bytes)); + DRM_DEBUG("rlc_dram_ucode_size_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_2->rlc_dram_ucode_size_bytes)); + DRM_DEBUG("rlc_dram_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_2->rlc_dram_ucode_offset_bytes)); + break; + case 3: + /* rlc_hdr v2_3 */ + DRM_DEBUG("rlcp_ucode_version: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_version)); + DRM_DEBUG("rlcp_ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_feature_version)); + DRM_DEBUG("rlcp_ucode_size_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_size_bytes)); + DRM_DEBUG("rlcp_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_offset_bytes)); + DRM_DEBUG("rlcv_ucode_version: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_version)); + DRM_DEBUG("rlcv_ucode_feature_version: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_feature_version)); + DRM_DEBUG("rlcv_ucode_size_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_size_bytes)); + DRM_DEBUG("rlcv_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_offset_bytes)); + break; + case 4: + /* rlc_hdr v2_4 */ + DRM_DEBUG("global_tap_delays_ucode_size_bytes :%u\n", + le32_to_cpu(rlc_hdr_v2_4->global_tap_delays_ucode_size_bytes)); + DRM_DEBUG("global_tap_delays_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_4->global_tap_delays_ucode_offset_bytes)); + DRM_DEBUG("se0_tap_delays_ucode_size_bytes :%u\n", + le32_to_cpu(rlc_hdr_v2_4->se0_tap_delays_ucode_size_bytes)); + DRM_DEBUG("se0_tap_delays_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_4->se0_tap_delays_ucode_offset_bytes)); + DRM_DEBUG("se1_tap_delays_ucode_size_bytes :%u\n", + le32_to_cpu(rlc_hdr_v2_4->se1_tap_delays_ucode_size_bytes)); + DRM_DEBUG("se1_tap_delays_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_4->se1_tap_delays_ucode_offset_bytes)); + DRM_DEBUG("se2_tap_delays_ucode_size_bytes :%u\n", + le32_to_cpu(rlc_hdr_v2_4->se2_tap_delays_ucode_size_bytes)); + DRM_DEBUG("se2_tap_delays_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_4->se2_tap_delays_ucode_offset_bytes)); + DRM_DEBUG("se3_tap_delays_ucode_size_bytes :%u\n", + le32_to_cpu(rlc_hdr_v2_4->se3_tap_delays_ucode_size_bytes)); + DRM_DEBUG("se3_tap_delays_ucode_offset_bytes: %u\n", + le32_to_cpu(rlc_hdr_v2_4->se3_tap_delays_ucode_offset_bytes)); + break; + default: + DRM_ERROR("Unknown RLC v2 ucode: v2.%u\n", version_minor); + break; } } else { DRM_ERROR("Unknown RLC ucode version: %u.%u\n", version_major, version_minor); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 96b6cf4c4d54f863488a4999058b06efaa182fe1..1c36235b4539c7c6497011fc21dab8e8aafa1bcf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -124,6 +124,7 @@ enum psp_fw_type { PSP_FW_TYPE_PSP_SOC_DRV, PSP_FW_TYPE_PSP_INTF_DRV, PSP_FW_TYPE_PSP_DBG_DRV, + PSP_FW_TYPE_PSP_RAS_DRV, }; /* version_major=2, version_minor=0 */ @@ -260,8 +261,12 @@ struct rlc_firmware_header_v2_2 { /* version_major=2, version_minor=3 */ struct rlc_firmware_header_v2_3 { struct rlc_firmware_header_v2_2 v2_2; + uint32_t rlcp_ucode_version; + uint32_t rlcp_ucode_feature_version; uint32_t rlcp_ucode_size_bytes; uint32_t rlcp_ucode_offset_bytes; + uint32_t rlcv_ucode_version; + uint32_t rlcv_ucode_feature_version; uint32_t rlcv_ucode_size_bytes; uint32_t rlcv_ucode_offset_bytes; }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h index 3629d8f292ef9272e4f352493d1613c5aae08fce..e46439274f3a0911b0f3b36e1d6689826b2aac98 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h @@ -51,6 +51,9 @@ struct amdgpu_umc_ras { struct amdgpu_ras_block_object ras_block; void (*err_cnt_init)(struct amdgpu_device *adev); bool (*query_ras_poison_mode)(struct amdgpu_device *adev); + void (*convert_ras_error_address)(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst); void (*ecc_info_query_ras_error_count)(struct amdgpu_device *adev, void *ras_error_status); void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index f36e4f08db6d4d6110cbbf7585601c09e7627f2b..0b52af415b282f809898c7a21a4414f86bd3e772 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -191,7 +191,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev) fw_name = FIRMWARE_VCN4_0_2; if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) && (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)) - adev->vcn.indirect_sram = false; + adev->vcn.indirect_sram = true; break; case IP_VERSION(4, 0, 4): fw_name = FIRMWARE_VCN4_0_4; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h index 80b7a6cfd026910ad508a6b5d5dfc41379628b61..253ea6b159df97de3890f5b93cf37742de4a8f33 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h @@ -161,7 +161,8 @@ #define AMDGPU_VCN_SW_RING_FLAG (1 << 9) #define AMDGPU_VCN_FW_LOGGING_FLAG (1 << 10) #define AMDGPU_VCN_SMU_VERSION_INFO_FLAG (1 << 11) -#define AMDGPU_VCN_VF_RB_SETUP_FLAG (1 << 12) +#define AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG (1 << 11) +#define AMDGPU_VCN_VF_RB_SETUP_FLAG (1 << 14) #define AMDGPU_VCN_IB_FLAG_DECODE_BUFFER 0x00000001 #define AMDGPU_VCN_CMD_FLAG_MSG_BUFFER 0x00000001 @@ -171,6 +172,9 @@ #define VCN_CODEC_DISABLE_MASK_HEVC (1 << 2) #define VCN_CODEC_DISABLE_MASK_H264 (1 << 3) +#define AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU (0) +#define AMDGPU_VCN_SMU_DPM_INTERFACE_APU (1) + enum fw_queue_mode { FW_QUEUE_RING_RESET = 1, FW_QUEUE_DPG_HOLD_OFF = 2, @@ -335,7 +339,9 @@ struct amdgpu_vcn4_fw_shared { struct amdgpu_fw_shared_unified_queue_struct sq; uint8_t pad1[8]; struct amdgpu_fw_shared_fw_logging fw_log; + uint8_t pad2[20]; struct amdgpu_fw_shared_rb_setup rb_setup; + struct amdgpu_fw_shared_smu_interface_info smu_dpm_interface; }; struct amdgpu_vcn_fwlog { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 59cac347baa38ebe0fa7dd908753a9219963a41a..83b0c5d86e4802c4606be3cce157f8b5a83a7dc4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -183,10 +183,12 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) struct amdgpu_bo *bo = vm_bo->bo; vm_bo->moved = true; + spin_lock(&vm_bo->vm->status_lock); if (bo->tbo.type == ttm_bo_type_kernel) list_move(&vm_bo->vm_status, &vm->evicted); else list_move_tail(&vm_bo->vm_status, &vm->evicted); + spin_unlock(&vm_bo->vm->status_lock); } /** * amdgpu_vm_bo_moved - vm_bo is moved @@ -198,7 +200,9 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) */ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) { + spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->moved); + spin_unlock(&vm_bo->vm->status_lock); } /** @@ -211,7 +215,9 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) */ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) { + spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->idle); + spin_unlock(&vm_bo->vm->status_lock); vm_bo->moved = false; } @@ -225,9 +231,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) */ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) { - spin_lock(&vm_bo->vm->invalidated_lock); + spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); - spin_unlock(&vm_bo->vm->invalidated_lock); + spin_unlock(&vm_bo->vm->status_lock); } /** @@ -240,10 +246,13 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) */ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) { - if (vm_bo->bo->parent) + if (vm_bo->bo->parent) { + spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); - else + spin_unlock(&vm_bo->vm->status_lock); + } else { amdgpu_vm_bo_idle(vm_bo); + } } /** @@ -256,9 +265,9 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) */ static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) { - spin_lock(&vm_bo->vm->invalidated_lock); + spin_lock(&vm_bo->vm->status_lock); list_move(&vm_bo->vm_status, &vm_bo->vm->done); - spin_unlock(&vm_bo->vm->invalidated_lock); + spin_unlock(&vm_bo->vm->status_lock); } /** @@ -363,12 +372,20 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, int (*validate)(void *p, struct amdgpu_bo *bo), void *param) { - struct amdgpu_vm_bo_base *bo_base, *tmp; + struct amdgpu_vm_bo_base *bo_base; + struct amdgpu_bo *shadow; + struct amdgpu_bo *bo; int r; - list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { - struct amdgpu_bo *bo = bo_base->bo; - struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo); + spin_lock(&vm->status_lock); + while (!list_empty(&vm->evicted)) { + bo_base = list_first_entry(&vm->evicted, + struct amdgpu_vm_bo_base, + vm_status); + spin_unlock(&vm->status_lock); + + bo = bo_base->bo; + shadow = amdgpu_bo_shadowed(bo); r = validate(param, bo); if (r) @@ -385,7 +402,9 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, vm->update_funcs->map_table(to_amdgpu_bo_vm(bo)); amdgpu_vm_bo_relocated(bo_base); } + spin_lock(&vm->status_lock); } + spin_unlock(&vm->status_lock); amdgpu_vm_eviction_lock(vm); vm->evicting = false; @@ -406,13 +425,18 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, */ bool amdgpu_vm_ready(struct amdgpu_vm *vm) { + bool empty; bool ret; amdgpu_vm_eviction_lock(vm); ret = !vm->evicting; amdgpu_vm_eviction_unlock(vm); - return ret && list_empty(&vm->evicted); + spin_lock(&vm->status_lock); + empty = list_empty(&vm->evicted); + spin_unlock(&vm->status_lock); + + return ret && empty; } /** @@ -680,9 +704,14 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, struct amdgpu_vm_update_params params; struct amdgpu_vm_bo_base *entry; bool flush_tlb_needed = false; + LIST_HEAD(relocated); int r, idx; - if (list_empty(&vm->relocated)) + spin_lock(&vm->status_lock); + list_splice_init(&vm->relocated, &relocated); + spin_unlock(&vm->status_lock); + + if (list_empty(&relocated)) return 0; if (!drm_dev_enter(adev_to_drm(adev), &idx)) @@ -697,7 +726,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (r) goto error; - list_for_each_entry(entry, &vm->relocated, vm_status) { + list_for_each_entry(entry, &relocated, vm_status) { /* vm_flush_needed after updating moved PDEs */ flush_tlb_needed |= entry->moved; @@ -713,9 +742,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev, if (flush_tlb_needed) atomic64_inc(&vm->tlb_seq); - while (!list_empty(&vm->relocated)) { - entry = list_first_entry(&vm->relocated, - struct amdgpu_vm_bo_base, + while (!list_empty(&relocated)) { + entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base, vm_status); amdgpu_vm_bo_idle(entry); } @@ -912,6 +940,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, { struct amdgpu_bo_va *bo_va, *tmp; + spin_lock(&vm->status_lock); list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { if (!bo_va->base.bo) continue; @@ -936,7 +965,6 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, amdgpu_bo_get_memory(bo_va->base.bo, vram_mem, gtt_mem, cpu_mem); } - spin_lock(&vm->invalidated_lock); list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { if (!bo_va->base.bo) continue; @@ -949,7 +977,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem, amdgpu_bo_get_memory(bo_va->base.bo, vram_mem, gtt_mem, cpu_mem); } - spin_unlock(&vm->invalidated_lock); + spin_unlock(&vm->status_lock); } /** * amdgpu_vm_bo_update - update all BO mappings in the vm page table @@ -1278,24 +1306,29 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, int amdgpu_vm_handle_moved(struct amdgpu_device *adev, struct amdgpu_vm *vm) { - struct amdgpu_bo_va *bo_va, *tmp; + struct amdgpu_bo_va *bo_va; struct dma_resv *resv; bool clear; int r; - list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { + spin_lock(&vm->status_lock); + while (!list_empty(&vm->moved)) { + bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va, + base.vm_status); + spin_unlock(&vm->status_lock); + /* Per VM BOs never need to bo cleared in the page tables */ r = amdgpu_vm_bo_update(adev, bo_va, false); if (r) return r; + spin_lock(&vm->status_lock); } - spin_lock(&vm->invalidated_lock); while (!list_empty(&vm->invalidated)) { bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, base.vm_status); resv = bo_va->base.bo->tbo.base.resv; - spin_unlock(&vm->invalidated_lock); + spin_unlock(&vm->status_lock); /* Try to reserve the BO to avoid clearing its ptes */ if (!amdgpu_vm_debug && dma_resv_trylock(resv)) @@ -1310,9 +1343,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev, if (!clear) dma_resv_unlock(resv); - spin_lock(&vm->invalidated_lock); + spin_lock(&vm->status_lock); } - spin_unlock(&vm->invalidated_lock); + spin_unlock(&vm->status_lock); return 0; } @@ -1387,7 +1420,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv && !bo_va->base.moved) { - list_move(&bo_va->base.vm_status, &vm->moved); + amdgpu_vm_bo_moved(&bo_va->base); } trace_amdgpu_vm_bo_map(bo_va, mapping); } @@ -1763,9 +1796,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev, } } - spin_lock(&vm->invalidated_lock); + spin_lock(&vm->status_lock); list_del(&bo_va->base.vm_status); - spin_unlock(&vm->invalidated_lock); + spin_unlock(&vm->status_lock); list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_del(&mapping->list); @@ -2019,9 +2052,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) INIT_LIST_HEAD(&vm->moved); INIT_LIST_HEAD(&vm->idle); INIT_LIST_HEAD(&vm->invalidated); - spin_lock_init(&vm->invalidated_lock); + spin_lock_init(&vm->status_lock); INIT_LIST_HEAD(&vm->freed); INIT_LIST_HEAD(&vm->done); + INIT_LIST_HEAD(&vm->pt_freed); + INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work); /* create scheduler entities for page table updates */ r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL, @@ -2223,6 +2258,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); + flush_work(&vm->pt_free_work); + root = amdgpu_bo_ref(vm->root.bo); amdgpu_bo_reserve(root, true); amdgpu_vm_set_pasid(adev, vm, 0); @@ -2484,8 +2521,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, /* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags = AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | - AMDGPU_PTE_TF; + flags = AMDGPU_PTE_SNOOPED | AMDGPU_PTE_PRT; value = 0; } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) { /* Redirect the access to the dummy page */ @@ -2548,6 +2584,7 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) unsigned int total_done_objs = 0; unsigned int id = 0; + spin_lock(&vm->status_lock); seq_puts(m, "\tIdle BOs:\n"); list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { if (!bo_va->base.bo) @@ -2585,7 +2622,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) id = 0; seq_puts(m, "\tInvalidated BOs:\n"); - spin_lock(&vm->invalidated_lock); list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { if (!bo_va->base.bo) continue; @@ -2600,7 +2636,7 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) continue; total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); } - spin_unlock(&vm->invalidated_lock); + spin_unlock(&vm->status_lock); total_done_objs = id; seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 9ecb7f663e1967d57ee9f8e9802bfb6eaa13feb9..83acb7bd80feb5a710e39f76715df04379fda7fa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -254,6 +254,9 @@ struct amdgpu_vm { bool evicting; unsigned int saved_flags; + /* Lock to protect vm_bo add/del/move on all lists of vm */ + spinlock_t status_lock; + /* BOs who needs a validation */ struct list_head evicted; @@ -268,7 +271,6 @@ struct amdgpu_vm { /* regular invalidated BOs, but not yet updated in the PT */ struct list_head invalidated; - spinlock_t invalidated_lock; /* BO mappings freed, but not yet updated in the PT */ struct list_head freed; @@ -276,6 +278,10 @@ struct amdgpu_vm { /* BOs which are invalidated, has been updated in the PTs */ struct list_head done; + /* PT BOs scheduled to free and fill with zero if vm_resv is not hold */ + struct list_head pt_freed; + struct work_struct pt_free_work; + /* contains the page directory */ struct amdgpu_vm_bo_base root; struct dma_fence *last_update; @@ -471,6 +477,7 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, uint64_t start, uint64_t end, uint64_t dst, uint64_t flags); +void amdgpu_vm_pt_free_work(struct work_struct *work); #if defined(CONFIG_DEBUG_FS) void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index 88de9f0d4728ac10ff64166622318faeccbc7c1a..358b91243e37b74b1cda4a4097fef2c392428db1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -637,10 +637,34 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) } ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); entry->bo->vm_bo = NULL; + + spin_lock(&entry->vm->status_lock); list_del(&entry->vm_status); + spin_unlock(&entry->vm->status_lock); amdgpu_bo_unref(&entry->bo); } +void amdgpu_vm_pt_free_work(struct work_struct *work) +{ + struct amdgpu_vm_bo_base *entry, *next; + struct amdgpu_vm *vm; + LIST_HEAD(pt_freed); + + vm = container_of(work, struct amdgpu_vm, pt_free_work); + + spin_lock(&vm->status_lock); + list_splice_init(&vm->pt_freed, &pt_freed); + spin_unlock(&vm->status_lock); + + /* flush_work in amdgpu_vm_fini ensure vm->root.bo is valid. */ + amdgpu_bo_reserve(vm->root.bo, true); + + list_for_each_entry_safe(entry, next, &pt_freed, vm_status) + amdgpu_vm_pt_free(entry); + + amdgpu_bo_unreserve(vm->root.bo); +} + /** * amdgpu_vm_pt_free_dfs - free PD/PT levels * @@ -652,11 +676,24 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) */ static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev, struct amdgpu_vm *vm, - struct amdgpu_vm_pt_cursor *start) + struct amdgpu_vm_pt_cursor *start, + bool unlocked) { struct amdgpu_vm_pt_cursor cursor; struct amdgpu_vm_bo_base *entry; + if (unlocked) { + spin_lock(&vm->status_lock); + for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) + list_move(&entry->vm_status, &vm->pt_freed); + + if (start) + list_move(&start->entry->vm_status, &vm->pt_freed); + spin_unlock(&vm->status_lock); + schedule_work(&vm->pt_free_work); + return; + } + for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) amdgpu_vm_pt_free(entry); @@ -673,7 +710,7 @@ static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev, */ void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm) { - amdgpu_vm_pt_free_dfs(adev, vm, NULL); + amdgpu_vm_pt_free_dfs(adev, vm, NULL, false); } /** @@ -966,7 +1003,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, if (cursor.entry->bo) { params->table_freed = true; amdgpu_vm_pt_free_dfs(adev, params->vm, - &cursor); + &cursor, + params->unlocked); } amdgpu_vm_pt_next(adev, &cursor); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c index 1fd3cbca20a2907f1aaf985ce8a5e8fa56a22ee3..2b0669c464f636442589a96f91fed29dae28a600 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c @@ -112,7 +112,8 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p, swap(p->vm->last_unlocked, tmp); dma_fence_put(tmp); } else { - amdgpu_bo_fence(p->vm->root.bo, f, true); + dma_resv_add_fence(p->vm->root.bo->tbo.base.resv, f, + DMA_RESV_USAGE_BOOKKEEP); } if (fence && !p->immediate) @@ -211,12 +212,15 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p, int r; /* Wait for PD/PT moves to be completed */ - dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, - DMA_RESV_USAGE_KERNEL, fence) { + dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL); + dma_resv_for_each_fence_unlocked(&cursor, fence) { r = amdgpu_sync_fence(&p->job->sync, fence); - if (r) + if (r) { + dma_resv_iter_end(&cursor); return r; + } } + dma_resv_iter_end(&cursor); do { ndw = p->num_dw_left; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index d3b483aa81f834fc337ff6746b32c023222cce34..47159e9a08848c72125bb9a6cc81b7c316d5e426 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -391,13 +391,21 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) goto pro_end; } + /** + * Only init hive->reset_domain for none SRIOV configuration. For SRIOV, + * Host driver decide how to reset the GPU either through FLR or chain reset. + * Guest side will get individual notifications from the host for the FLR + * if necessary. + */ + if (!amdgpu_sriov_vf(adev)) { /** * Avoid recreating reset domain when hive is reconstructed for the case - * of reset the devices in the XGMI hive during probe for SRIOV + * of reset the devices in the XGMI hive during probe for passthrough GPU * See https://www.spinics.net/lists/amd-gfx/msg58836.html */ - if (adev->reset_domain->type != XGMI_HIVE) { - hive->reset_domain = amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive"); + if (adev->reset_domain->type != XGMI_HIVE) { + hive->reset_domain = + amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive"); if (!hive->reset_domain) { dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n"); ret = -ENOMEM; @@ -406,9 +414,10 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) hive = NULL; goto pro_end; } - } else { - amdgpu_reset_get_reset_domain(adev->reset_domain); - hive->reset_domain = adev->reset_domain; + } else { + amdgpu_reset_get_reset_domain(adev->reset_domain); + hive->reset_domain = adev->reset_domain; + } } hive->hive_id = adev->gmc.xgmi.hive_id; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h index 552e6fb55aa8a0fef601aaf131529bf16ebc2954..30dcc1681b4e94745b1cae76f04c7334f18aee19 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h @@ -43,6 +43,7 @@ struct amdgpu_hive_info { } pstate; struct amdgpu_reset_domain *reset_domain; + uint32_t device_remove_count; }; struct amdgpu_pcs_ras_field { diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 5647f13b98d49c814bdd34f3a67a8b00e118394c..cbca9866645c5ddaee97cdfa81990f67e83000d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -309,14 +309,10 @@ static void cik_sdma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq */ static void cik_sdma_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; u32 rb_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index e4dde41f2f68f97947281e5cc77b67c073ca577c..af94ac580d3e1f6c5eb98c0d155a9c06246b81fa 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -3943,56 +3943,6 @@ static void gfx_v10_0_check_fw_write_wait(struct amdgpu_device *adev) DRM_WARN_ONCE("CP firmware version too old, please update!"); } - -static void gfx_v10_0_init_rlc_ext_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_1 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); - adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); - adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); - adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); - adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); - adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); - adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); - adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); - adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); - adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); - adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); - adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); - adev->gfx.rlc.reg_list_format_direct_reg_list_length = - le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); -} - -static void gfx_v10_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_2 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes); - adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes); - adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes); - adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes); -} - -static void gfx_v10_0_init_tap_delays_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_4 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_4 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc.global_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->global_tap_delays_ucode_size_bytes); - adev->gfx.rlc.global_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->global_tap_delays_ucode_offset_bytes); - adev->gfx.rlc.se0_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_size_bytes); - adev->gfx.rlc.se0_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_offset_bytes); - adev->gfx.rlc.se1_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_size_bytes); - adev->gfx.rlc.se1_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_offset_bytes); - adev->gfx.rlc.se2_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_size_bytes); - adev->gfx.rlc.se2_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_offset_bytes); - adev->gfx.rlc.se3_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_size_bytes); - adev->gfx.rlc.se3_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_offset_bytes); -} - static bool gfx_v10_0_navi10_gfxoff_should_enable(struct amdgpu_device *adev) { bool ret = false; @@ -4028,12 +3978,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) char fw_name[40]; char *wks = ""; int err; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; - const struct gfx_firmware_header_v1_0 *cp_hdr; const struct rlc_firmware_header_v2_0 *rlc_hdr; - unsigned int *tmp = NULL; - unsigned int i = 0; uint16_t version_major; uint16_t version_minor; @@ -4091,9 +4036,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) err = amdgpu_ucode_validate(adev->gfx.pfp_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; - adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me%s.bin", chip_name, wks); err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); @@ -4102,9 +4045,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) err = amdgpu_ucode_validate(adev->gfx.me_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; - adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce%s.bin", chip_name, wks); err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); @@ -4113,69 +4054,27 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) err = amdgpu_ucode_validate(adev->gfx.ce_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; - adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE); if (!amdgpu_sriov_vf(adev)) { snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); if (err) goto out; + /* don't check this. There are apparently firmwares in the wild with + * incorrect size in the header + */ err = amdgpu_ucode_validate(adev->gfx.rlc_fw); + if (err) + dev_dbg(adev->dev, + "gfx10: amdgpu_ucode_validate() failed \"%s\"\n", + fw_name); rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; version_major = le16_to_cpu(rlc_hdr->header.header_version_major); version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); - - adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); - adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); - adev->gfx.rlc.save_and_restore_offset = - le32_to_cpu(rlc_hdr->save_and_restore_offset); - adev->gfx.rlc.clear_state_descriptor_offset = - le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); - adev->gfx.rlc.avail_scratch_ram_locations = - le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); - adev->gfx.rlc.reg_restore_list_size = - le32_to_cpu(rlc_hdr->reg_restore_list_size); - adev->gfx.rlc.reg_list_format_start = - le32_to_cpu(rlc_hdr->reg_list_format_start); - adev->gfx.rlc.reg_list_format_separate_start = - le32_to_cpu(rlc_hdr->reg_list_format_separate_start); - adev->gfx.rlc.starting_offsets_start = - le32_to_cpu(rlc_hdr->starting_offsets_start); - adev->gfx.rlc.reg_list_format_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); - adev->gfx.rlc.reg_list_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_size_bytes); - adev->gfx.rlc.register_list_format = - kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + - adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); - if (!adev->gfx.rlc.register_list_format) { - err = -ENOMEM; + err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); + if (err) goto out; - } - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); - for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++) - adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); - - adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); - for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++) - adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); - - if (version_major == 2) { - if (version_minor >= 1) - gfx_v10_0_init_rlc_ext_microcode(adev); - if (version_minor >= 2) - gfx_v10_0_init_rlc_iram_dram_microcode(adev); - if (version_minor == 4) { - gfx_v10_0_init_tap_delays_microcode(adev); - } - } } snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec%s.bin", chip_name, wks); @@ -4185,9 +4084,8 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) err = amdgpu_ucode_validate(adev->gfx.mec_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; - adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2%s.bin", chip_name, wks); err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); @@ -4195,164 +4093,18 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev) err = amdgpu_ucode_validate(adev->gfx.mec2_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *) - adev->gfx.mec2_fw->data; - adev->gfx.mec2_fw_version = - le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.mec2_feature_version = - le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT); } else { err = 0; adev->gfx.mec2_fw = NULL; } - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; - info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; - info->ucode_id = AMDGPU_UCODE_ID_CP_ME; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; - info->ucode_id = AMDGPU_UCODE_ID_CP_CE; - info->fw = adev->gfx.ce_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_G; - info->fw = adev->gfx.rlc_fw; - if (info->fw) { - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - } - if (adev->gfx.rlc.save_restore_list_cntl_size_bytes && - adev->gfx.rlc.save_restore_list_gpm_size_bytes && - adev->gfx.rlc.save_restore_list_srm_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); - - if (adev->gfx.rlc.rlc_iram_ucode_size_bytes && - adev->gfx.rlc.rlc_dram_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE); - } - - } - - if (adev->gfx.rlc.global_tap_delays_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS]; - info->ucode_id = AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.global_tap_delays_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.se0_tap_delays_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE0_TAP_DELAYS]; - info->ucode_id = AMDGPU_UCODE_ID_SE0_TAP_DELAYS; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.se0_tap_delays_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.se1_tap_delays_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE1_TAP_DELAYS]; - info->ucode_id = AMDGPU_UCODE_ID_SE1_TAP_DELAYS; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.se1_tap_delays_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.se2_tap_delays_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE2_TAP_DELAYS]; - info->ucode_id = AMDGPU_UCODE_ID_SE2_TAP_DELAYS; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.se2_tap_delays_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.se3_tap_delays_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE3_TAP_DELAYS]; - info->ucode_id = AMDGPU_UCODE_ID_SE3_TAP_DELAYS; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.se3_tap_delays_ucode_size_bytes, PAGE_SIZE); - } - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes) - - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; - info->fw = adev->gfx.mec_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - if (adev->gfx.mec2_fw) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; - info->fw = adev->gfx.mec2_fw; - header = (const struct common_firmware_header *)info->fw->data; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes) - - le32_to_cpu(cp_hdr->jt_size) * 4, - PAGE_SIZE); - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; - info->fw = adev->gfx.mec2_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, - PAGE_SIZE); - } - } - gfx_v10_0_check_fw_write_wait(adev); out: if (err) { dev_err(adev->dev, - "gfx10: Failed to load firmware \"%s\"\n", + "gfx10: Failed to init firmware \"%s\"\n", fw_name); release_firmware(adev->gfx.pfp_fw); adev->gfx.pfp_fw = NULL; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index fa718318568e2b22dd5f95d14e81cb5b070b43d0..251109723ab63ef1942aaceb46e8282f5f76f919 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -438,61 +438,12 @@ static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) kfree(adev->gfx.rlc.register_list_format); } -static void gfx_v11_0_init_rlc_ext_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_1 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); - adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); - adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); - adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); - adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); - adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); - adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); - adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); - adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); - adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); - adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); - adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); - adev->gfx.rlc.reg_list_format_direct_reg_list_length = - le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); -} - -static void gfx_v11_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_2 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes); - adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes); - adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes); - adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes); -} - -static void gfx_v11_0_init_rlcp_rlcv_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_3 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc.rlcp_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcp_ucode_size_bytes); - adev->gfx.rlc.rlcp_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcp_ucode_offset_bytes); - adev->gfx.rlc.rlcv_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcv_ucode_size_bytes); - adev->gfx.rlc.rlcv_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcv_ucode_offset_bytes); -} - static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) { char fw_name[40]; char ucode_prefix[30]; int err; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; - const struct gfx_firmware_header_v1_0 *cp_hdr; - const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0; const struct rlc_firmware_header_v2_0 *rlc_hdr; - unsigned int *tmp = NULL; - unsigned int i = 0; uint16_t version_major; uint16_t version_minor; @@ -513,14 +464,11 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) adev->gfx.pfp_fw->data, 2, 0); if (adev->gfx.rs64_enable) { dev_info(adev->dev, "CP RS64 enable\n"); - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data; - adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version); - adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); - + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); } else { - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; - adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); } snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix); @@ -531,14 +479,11 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) if (err) goto out; if (adev->gfx.rs64_enable) { - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data; - adev->gfx.me_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version); - adev->gfx.me_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); - + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); } else { - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; - adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); } if (!amdgpu_sriov_vf(adev)) { @@ -547,58 +492,14 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) if (err) goto out; err = amdgpu_ucode_validate(adev->gfx.rlc_fw); + if (err) + goto out; rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; version_major = le16_to_cpu(rlc_hdr->header.header_version_major); version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); - - adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); - adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); - adev->gfx.rlc.save_and_restore_offset = - le32_to_cpu(rlc_hdr->save_and_restore_offset); - adev->gfx.rlc.clear_state_descriptor_offset = - le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); - adev->gfx.rlc.avail_scratch_ram_locations = - le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); - adev->gfx.rlc.reg_restore_list_size = - le32_to_cpu(rlc_hdr->reg_restore_list_size); - adev->gfx.rlc.reg_list_format_start = - le32_to_cpu(rlc_hdr->reg_list_format_start); - adev->gfx.rlc.reg_list_format_separate_start = - le32_to_cpu(rlc_hdr->reg_list_format_separate_start); - adev->gfx.rlc.starting_offsets_start = - le32_to_cpu(rlc_hdr->starting_offsets_start); - adev->gfx.rlc.reg_list_format_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); - adev->gfx.rlc.reg_list_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_size_bytes); - adev->gfx.rlc.register_list_format = - kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + - adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); - if (!adev->gfx.rlc.register_list_format) { - err = -ENOMEM; + err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); + if (err) goto out; - } - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); - for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++) - adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); - - adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); - for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++) - adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); - - if (version_major == 2) { - if (version_minor >= 1) - gfx_v11_0_init_rlc_ext_microcode(adev); - if (version_minor >= 2) - gfx_v11_0_init_rlc_iram_dram_microcode(adev); - if (version_minor == 3) - gfx_v11_0_init_rlcp_rlcv_microcode(adev); - } } snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix); @@ -609,190 +510,23 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) if (err) goto out; if (adev->gfx.rs64_enable) { - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; - adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version); - adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version); - + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); } else { - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; - adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); } /* only one MEC for gfx 11.0.0. */ adev->gfx.mec2_fw = NULL; - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - if (adev->gfx.rs64_enable) { - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data; - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data; - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK]; - info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE); - } else { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; - info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; - info->ucode_id = AMDGPU_UCODE_ID_CP_ME; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes) - - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; - info->fw = adev->gfx.mec_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - } - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_G; - info->fw = adev->gfx.rlc_fw; - if (info->fw) { - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - } - if (adev->gfx.rlc.save_restore_list_gpm_size_bytes && - adev->gfx.rlc.save_restore_list_srm_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.rlc_iram_ucode_size_bytes && - adev->gfx.rlc.rlc_dram_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.rlcp_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_P]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_P; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlcp_ucode_size_bytes, PAGE_SIZE); - } - - if (adev->gfx.rlc.rlcv_ucode_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_V]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_V; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.rlcv_ucode_size_bytes, PAGE_SIZE); - } - } - out: if (err) { dev_err(adev->dev, - "gfx11: Failed to load firmware \"%s\"\n", + "gfx11: Failed to init firmware \"%s\"\n", fw_name); release_firmware(adev->gfx.pfp_fw); adev->gfx.pfp_fw = NULL; @@ -5240,6 +4974,8 @@ static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) { u32 reg, data; + amdgpu_gfx_off_ctrl(adev, false); + reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); if (amdgpu_sriov_is_pp_one_vf(adev)) data = RREG32_NO_KIQ(reg); @@ -5253,6 +4989,8 @@ static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); else WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); + + amdgpu_gfx_off_ctrl(adev, true); } static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 1d6d3a852a0b3d158f340b48503cb06c8c546f12..0320be4a5fc6c3c3c55b9c69641eedd4b0ac9cbe 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1091,27 +1091,6 @@ static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) kfree(adev->gfx.rlc.register_list_format); } -static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) -{ - const struct rlc_firmware_header_v2_1 *rlc_hdr; - - rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; - adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); - adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); - adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); - adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); - adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); - adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); - adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); - adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); - adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); - adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); - adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); - adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); - adev->gfx.rlc.reg_list_format_direct_reg_list_length = - le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); -} - static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) { adev->gfx.me_fw_write_wait = false; @@ -1273,9 +1252,6 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, { char fw_name[30]; int err; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; - const struct gfx_firmware_header_v1_0 *cp_hdr; snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); @@ -1284,9 +1260,7 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, err = amdgpu_ucode_validate(adev->gfx.pfp_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; - adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); @@ -1295,9 +1269,7 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, err = amdgpu_ucode_validate(adev->gfx.me_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; - adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); @@ -1306,37 +1278,12 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, err = amdgpu_ucode_validate(adev->gfx.ce_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; - adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; - info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; - info->fw = adev->gfx.pfp_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; - info->ucode_id = AMDGPU_UCODE_ID_CP_ME; - info->fw = adev->gfx.me_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; - info->ucode_id = AMDGPU_UCODE_ID_CP_CE; - info->fw = adev->gfx.ce_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - } + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE); out: if (err) { dev_err(adev->dev, - "gfx9: Failed to load firmware \"%s\"\n", + "gfx9: Failed to init firmware \"%s\"\n", fw_name); release_firmware(adev->gfx.pfp_fw); adev->gfx.pfp_fw = NULL; @@ -1353,11 +1300,7 @@ static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, { char fw_name[30]; int err; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; const struct rlc_firmware_header_v2_0 *rlc_hdr; - unsigned int *tmp = NULL; - unsigned int i = 0; uint16_t version_major; uint16_t version_minor; uint32_t smu_version; @@ -1386,92 +1329,17 @@ static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, if (err) goto out; err = amdgpu_ucode_validate(adev->gfx.rlc_fw); + if (err) + goto out; rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; version_major = le16_to_cpu(rlc_hdr->header.header_version_major); version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); - if (version_major == 2 && version_minor == 1) - adev->gfx.rlc.is_rlc_v2_1 = true; - - adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); - adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); - adev->gfx.rlc.save_and_restore_offset = - le32_to_cpu(rlc_hdr->save_and_restore_offset); - adev->gfx.rlc.clear_state_descriptor_offset = - le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); - adev->gfx.rlc.avail_scratch_ram_locations = - le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); - adev->gfx.rlc.reg_restore_list_size = - le32_to_cpu(rlc_hdr->reg_restore_list_size); - adev->gfx.rlc.reg_list_format_start = - le32_to_cpu(rlc_hdr->reg_list_format_start); - adev->gfx.rlc.reg_list_format_separate_start = - le32_to_cpu(rlc_hdr->reg_list_format_separate_start); - adev->gfx.rlc.starting_offsets_start = - le32_to_cpu(rlc_hdr->starting_offsets_start); - adev->gfx.rlc.reg_list_format_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); - adev->gfx.rlc.reg_list_size_bytes = - le32_to_cpu(rlc_hdr->reg_list_size_bytes); - adev->gfx.rlc.register_list_format = - kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + - adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); - if (!adev->gfx.rlc.register_list_format) { - err = -ENOMEM; - goto out; - } - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); - for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) - adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); - - adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; - - tmp = (unsigned int *)((uintptr_t)rlc_hdr + - le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); - for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) - adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); - - if (adev->gfx.rlc.is_rlc_v2_1) - gfx_v9_0_init_rlc_ext_microcode(adev); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_G; - info->fw = adev->gfx.rlc_fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - - if (adev->gfx.rlc.is_rlc_v2_1 && - adev->gfx.rlc.save_restore_list_cntl_size_bytes && - adev->gfx.rlc.save_restore_list_gpm_size_bytes && - adev->gfx.rlc.save_restore_list_srm_size_bytes) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; - info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; - info->fw = adev->gfx.rlc_fw; - adev->firmware.fw_size += - ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); - } - } - + err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); out: if (err) { dev_err(adev->dev, - "gfx9: Failed to load firmware \"%s\"\n", + "gfx9: Failed to init firmware \"%s\"\n", fw_name); release_firmware(adev->gfx.rlc_fw); adev->gfx.rlc_fw = NULL; @@ -1494,9 +1362,6 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, { char fw_name[30]; int err; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; - const struct gfx_firmware_header_v1_0 *cp_hdr; if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec.bin", chip_name); @@ -1509,10 +1374,8 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, err = amdgpu_ucode_validate(adev->gfx.mec_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; - adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); - + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) @@ -1525,12 +1388,8 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, err = amdgpu_ucode_validate(adev->gfx.mec2_fw); if (err) goto out; - cp_hdr = (const struct gfx_firmware_header_v1_0 *) - adev->gfx.mec2_fw->data; - adev->gfx.mec2_fw_version = - le32_to_cpu(cp_hdr->header.ucode_version); - adev->gfx.mec2_feature_version = - le32_to_cpu(cp_hdr->ucode_feature_version); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2); + amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT); } else { err = 0; adev->gfx.mec2_fw = NULL; @@ -1540,49 +1399,12 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; } - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; - info->fw = adev->gfx.mec_fw; - header = (const struct common_firmware_header *)info->fw->data; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; - info->fw = adev->gfx.mec_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - if (adev->gfx.mec2_fw) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; - info->fw = adev->gfx.mec2_fw; - header = (const struct common_firmware_header *)info->fw->data; - cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); - - /* TODO: Determine if MEC2 JT FW loading can be removed - for all GFX V9 asic and above */ - if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; - info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; - info->fw = adev->gfx.mec2_fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, - PAGE_SIZE); - } - } - } - out: gfx_v9_0_check_if_need_gfxoff(adev); gfx_v9_0_check_fw_write_wait(adev); if (err) { dev_err(adev->dev, - "gfx9: Failed to load firmware \"%s\"\n", + "gfx9: Failed to init firmware \"%s\"\n", fw_name); release_firmware(adev->gfx.mec_fw); adev->gfx.mec_fw = NULL; @@ -5607,7 +5429,7 @@ static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne BUG_ON(offset > ring->buf_mask); BUG_ON(ring->ring[offset] != 0x55aa55aa); - cur = (ring->wptr & ring->buf_mask) - 1; + cur = (ring->wptr - 1) & ring->buf_mask; if (likely(cur > offset)) ring->ring[offset] = cur - offset; else diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 4603653916f5a551854a784c75da1817731f4bf9..67ca16a8027c7d9afe83e30a916e8a9c0984e0a0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -1103,10 +1103,13 @@ static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, *flags |= AMDGPU_PDE_BFS(0x9); } else if (level == AMDGPU_VM_PDB0) { - if (*flags & AMDGPU_PDE_PTE) + if (*flags & AMDGPU_PDE_PTE) { *flags &= ~AMDGPU_PDE_PTE; - else + if (!(*flags & AMDGPU_PTE_VALID)) + *addr |= 1 << PAGE_SHIFT; + } else { *flags |= AMDGPU_PTE_TF; + } } } diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c index 536dafb57ee0b9248b4be4efe8c1bcbc04c1ba7c..fc69c1a29e2393a361071b37cb3b75e9c0c06081 100644 --- a/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c @@ -22,6 +22,7 @@ */ #include "amdgpu.h" #include "amdgpu_imu.h" +#include "imu_v11_0_3.h" #include "gc/gc_11_0_3_offset.h" #include "gc/gc_11_0_3_sh_mask.h" diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index b64cd46a159a948c4eb37bf3fcbf9200c73df825..5cec6b259b7f764bc0d361d60ff90ca592be4d57 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -187,6 +187,19 @@ static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes, mes_add_queue_pkt.is_kfd_process = input->is_kfd_process; mes_add_queue_pkt.trap_en = 1; + /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */ + mes_add_queue_pkt.is_aql_queue = input->is_aql_queue; + mes_add_queue_pkt.gds_size = input->queue_size; + + if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 4) && + (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) && + (adev->ip_versions[GC_HWIP][0] <= IP_VERSION(11, 0, 3)))) + mes_add_queue_pkt.trap_en = 1; + + /* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */ + mes_add_queue_pkt.is_aql_queue = input->is_aql_queue; + mes_add_queue_pkt.gds_size = input->queue_size; + return mes_v11_0_submit_pkt_and_poll_completion(mes, &mes_add_queue_pkt, sizeof(mes_add_queue_pkt), offsetof(union MESAPI__ADD_QUEUE, api_status)); diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c index b465baa267628229aaee75ec7f8826232390c59b..aa761ff3a5faec4e383b93f7467d13b27974bce6 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c @@ -380,6 +380,7 @@ static void nbio_v2_3_enable_aspm(struct amdgpu_device *adev, WREG32_PCIE(smnPCIE_LC_CNTL, data); } +#ifdef CONFIG_PCIEASPM static void nbio_v2_3_program_ltr(struct amdgpu_device *adev) { uint32_t def, data; @@ -401,9 +402,11 @@ static void nbio_v2_3_program_ltr(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); } +#endif static void nbio_v2_3_program_aspm(struct amdgpu_device *adev) { +#ifdef CONFIG_PCIEASPM uint32_t def, data; def = data = RREG32_PCIE(smnPCIE_LC_CNTL); @@ -459,7 +462,10 @@ static void nbio_v2_3_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL6, data); - nbio_v2_3_program_ltr(adev); + /* Don't bother about LTR if LTR is not enabled + * in the path */ + if (adev->pdev->ltr_path) + nbio_v2_3_program_ltr(adev); def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3); data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; @@ -483,6 +489,7 @@ static void nbio_v2_3_program_aspm(struct amdgpu_device *adev) data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL3, data); +#endif } static void nbio_v2_3_apply_lc_spc_mode_wa(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c index f7f6ddebd3e49b6ec55f411fda96cf4d35959f33..37615a77287bc00e52dab28d57d25b9df21cc57c 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c @@ -282,6 +282,7 @@ static void nbio_v6_1_init_registers(struct amdgpu_device *adev) mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL) << 2; } +#ifdef CONFIG_PCIEASPM static void nbio_v6_1_program_ltr(struct amdgpu_device *adev) { uint32_t def, data; @@ -303,9 +304,11 @@ static void nbio_v6_1_program_ltr(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); } +#endif static void nbio_v6_1_program_aspm(struct amdgpu_device *adev) { +#ifdef CONFIG_PCIEASPM uint32_t def, data; def = data = RREG32_PCIE(smnPCIE_LC_CNTL); @@ -361,7 +364,10 @@ static void nbio_v6_1_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL6, data); - nbio_v6_1_program_ltr(adev); + /* Don't bother about LTR if LTR is not enabled + * in the path */ + if (adev->pdev->ltr_path) + nbio_v6_1_program_ltr(adev); def = data = RREG32_PCIE(smnRCC_BIF_STRAP3); data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; @@ -385,6 +391,7 @@ static void nbio_v6_1_program_aspm(struct amdgpu_device *adev) data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL3, data); +#endif } const struct amdgpu_nbio_funcs nbio_v6_1_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index 11848d1e238b6b7c3724b339d8c955bc1685d664..19455a72593916989de322cb038ce42b4d7482c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -673,6 +673,7 @@ struct amdgpu_nbio_ras nbio_v7_4_ras = { }; +#ifdef CONFIG_PCIEASPM static void nbio_v7_4_program_ltr(struct amdgpu_device *adev) { uint32_t def, data; @@ -694,9 +695,11 @@ static void nbio_v7_4_program_ltr(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); } +#endif static void nbio_v7_4_program_aspm(struct amdgpu_device *adev) { +#ifdef CONFIG_PCIEASPM uint32_t def, data; if (adev->ip_versions[NBIO_HWIP][0] == IP_VERSION(7, 4, 4)) @@ -755,7 +758,10 @@ static void nbio_v7_4_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL6, data); - nbio_v7_4_program_ltr(adev); + /* Don't bother about LTR if LTR is not enabled + * in the path */ + if (adev->pdev->ltr_path) + nbio_v7_4_program_ltr(adev); def = data = RREG32_PCIE(smnRCC_BIF_STRAP3); data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; @@ -779,6 +785,7 @@ static void nbio_v7_4_program_aspm(struct amdgpu_device *adev) data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; if (def != data) WREG32_PCIE(smnPCIE_LC_CNTL3, data); +#endif } const struct amdgpu_nbio_funcs nbio_v7_4_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_7.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_7.c index f30bc826a87819b3670f868c188436ce4f467f1d..def89379b51a57b13f1ca14c61899b68ad5ccb45 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_7.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_7.c @@ -28,6 +28,14 @@ #include "nbio/nbio_7_7_0_sh_mask.h" #include +static void nbio_v7_7_remap_hdp_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); +} + static u32 nbio_v7_7_get_rev_id(struct amdgpu_device *adev) { u32 tmp; @@ -336,4 +344,5 @@ const struct amdgpu_nbio_funcs nbio_v7_7_funcs = { .get_clockgating_state = nbio_v7_7_get_clockgating_state, .ih_control = nbio_v7_7_ih_control, .init_registers = nbio_v7_7_init_registers, + .remap_hdp_registers = nbio_v7_7_remap_hdp_registers, }; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c index 5b5b1ef0c2b1c5796c079467db14dffcb94430ad..21d822b1d5896d1f810cbfd1b1f9579509aba895 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c @@ -224,6 +224,12 @@ static int psp_v13_0_bootloader_load_dbg_drv(struct psp_context *psp) return psp_v13_0_bootloader_load_component(psp, &psp->dbg_drv, PSP_BL__LOAD_DBGDRV); } +static int psp_v13_0_bootloader_load_ras_drv(struct psp_context *psp) +{ + return psp_v13_0_bootloader_load_component(psp, &psp->ras_drv, PSP_BL__LOAD_RASDRV); +} + + static int psp_v13_0_bootloader_load_sos(struct psp_context *psp) { int ret; @@ -720,6 +726,7 @@ static const struct psp_funcs psp_v13_0_funcs = { .bootloader_load_soc_drv = psp_v13_0_bootloader_load_soc_drv, .bootloader_load_intf_drv = psp_v13_0_bootloader_load_intf_drv, .bootloader_load_dbg_drv = psp_v13_0_bootloader_load_dbg_drv, + .bootloader_load_ras_drv = psp_v13_0_bootloader_load_ras_drv, .bootloader_load_sos = psp_v13_0_bootloader_load_sos, .ring_init = psp_v13_0_ring_init, .ring_create = psp_v13_0_ring_create, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 6bdffdc1c0b92593d26f9529c29b662393fc374d..c52d246a1d965c3dc4a0cca79e0b32dc0a72adb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -342,14 +342,10 @@ static void sdma_v2_4_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v2_4_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; u32 rb_cntl, ib_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 2584fa3cb13e7ad5f9aefae8f670026a9bcc5613..486d9b5c1b9e75777b28131720486b45c2ac9e1e 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -516,14 +516,10 @@ static void sdma_v3_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v3_0_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; u32 rb_cntl, ib_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32(mmSDMA0_GFX_RB_CNTL + sdma_offsets[i]); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index 0cf9d3b486b277ab6119b0dbac086a4a0f459fab..298fa11702e755875aa093f8bb60985afa6bb360 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -561,44 +561,6 @@ static void sdma_v4_0_setup_ulv(struct amdgpu_device *adev) } } -static int sdma_v4_0_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) -{ - int err = 0; - const struct sdma_firmware_header_v1_0 *hdr; - - err = amdgpu_ucode_validate(sdma_inst->fw); - if (err) - return err; - - hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; - sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); - sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); - - if (sdma_inst->feature_version >= 20) - sdma_inst->burst_nop = true; - - return 0; -} - -static void sdma_v4_0_destroy_inst_ctx(struct amdgpu_device *adev) -{ - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - release_firmware(adev->sdma.instance[i].fw); - adev->sdma.instance[i].fw = NULL; - - /* arcturus shares the same FW memory across - all SDMA isntances */ - if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) || - adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0)) - break; - } - - memset((void *)adev->sdma.instance, 0, - sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); -} - /** * sdma_v4_0_init_microcode - load ucode images from disk * @@ -615,9 +577,7 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev) { const char *chip_name; char fw_name[30]; - int err = 0, i; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; + int ret, i; DRM_DEBUG("\n"); @@ -656,58 +616,25 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev) BUG(); } - snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); - - err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[0]); - if (err) - goto out; - - for (i = 1; i < adev->sdma.num_instances; i++) { + for (i = 0; i < adev->sdma.num_instances; i++) { + if (i == 0) + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); + else + snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i); if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) || adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0)) { /* Acturus & Aldebaran will leverage the same FW memory for every SDMA instance */ - memcpy((void *)&adev->sdma.instance[i], - (void *)&adev->sdma.instance[0], - sizeof(struct amdgpu_sdma_instance)); - } - else { - snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i); - - err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[i]); - if (err) - goto out; - } - } - - DRM_DEBUG("psp_load == '%s'\n", - adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - for (i = 0; i < adev->sdma.num_instances; i++) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; - info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; - info->fw = adev->sdma.instance[i].fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); + ret = amdgpu_sdma_init_microcode(adev, fw_name, 0, true); + break; + } else { + ret = amdgpu_sdma_init_microcode(adev, fw_name, i, false); + if (ret) + return ret; } } -out: - if (err) { - DRM_ERROR("sdma_v4_0: Failed to load firmware \"%s\"\n", fw_name); - sdma_v4_0_destroy_inst_ctx(adev); - } - return err; + return ret; } /** @@ -988,18 +915,12 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v4_0_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES]; u32 rb_cntl, ib_cntl; - int i, unset = 0; - - for (i = 0; i < adev->sdma.num_instances; i++) { - sdma[i] = &adev->sdma.instance[i].ring; + int i; - if ((adev->mman.buffer_funcs_ring == sdma[i]) && unset != 1) { - amdgpu_ttm_set_buffer_funcs_status(adev, false); - unset = 1; - } + amdgpu_sdma_unset_buffer_funcs_helper(adev); + for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL); rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_GFX_RB_CNTL, RB_ENABLE, 0); WREG32_SDMA(i, mmSDMA0_GFX_RB_CNTL, rb_cntl); @@ -1030,20 +951,12 @@ static void sdma_v4_0_rlc_stop(struct amdgpu_device *adev) */ static void sdma_v4_0_page_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma[AMDGPU_MAX_SDMA_INSTANCES]; u32 rb_cntl, ib_cntl; int i; - bool unset = false; - for (i = 0; i < adev->sdma.num_instances; i++) { - sdma[i] = &adev->sdma.instance[i].page; - - if ((adev->mman.buffer_funcs_ring == sdma[i]) && - (!unset)) { - amdgpu_ttm_set_buffer_funcs_status(adev, false); - unset = true; - } + amdgpu_sdma_unset_buffer_funcs_helper(adev); + for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_CNTL); rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_PAGE_RB_CNTL, RB_ENABLE, 0); @@ -1504,6 +1417,11 @@ static int sdma_v4_0_start(struct amdgpu_device *adev) WREG32_SDMA(i, mmSDMA0_CNTL, temp); if (!amdgpu_sriov_vf(adev)) { + ring = &adev->sdma.instance[i].ring; + adev->nbio.funcs->sdma_doorbell_range(adev, i, + ring->use_doorbell, ring->doorbell_index, + adev->doorbell_index.sdma_doorbell_range); + /* unhalt engine */ temp = RREG32_SDMA(i, mmSDMA0_F32_CNTL); temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0); @@ -1995,7 +1913,11 @@ static int sdma_v4_0_sw_fini(void *handle) amdgpu_ring_fini(&adev->sdma.instance[i].page); } - sdma_v4_0_destroy_inst_ctx(adev); + if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 0) || + adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0)) + amdgpu_sdma_destroy_inst_ctx(adev, true); + else + amdgpu_sdma_destroy_inst_ctx(adev, false); return 0; } @@ -2018,8 +1940,11 @@ static int sdma_v4_0_hw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; int i; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev)) { + /* disable the scheduler for SDMA */ + amdgpu_sdma_unset_buffer_funcs_helper(adev); return 0; + } for (i = 0; i < adev->sdma.num_instances; i++) { amdgpu_irq_put(adev, &adev->sdma.ecc_irq, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index a019ac92edb73140310bc544a09e2509774ca06d..d4d9f196db834e1ee69c74f2c66a451fbf5deef4 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -240,10 +240,7 @@ static int sdma_v5_0_init_microcode(struct amdgpu_device *adev) { const char *chip_name; char fw_name[40]; - int err = 0, i; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; - const struct sdma_firmware_header_v1_0 *hdr; + int ret, i; if (amdgpu_sriov_vf(adev) && (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(5, 0, 5))) return 0; @@ -272,38 +269,12 @@ static int sdma_v5_0_init_microcode(struct amdgpu_device *adev) snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name); else snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name); - err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); - if (err) - goto out; - err = amdgpu_ucode_validate(adev->sdma.instance[i].fw); - if (err) - goto out; - hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data; - adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version); - adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); - if (adev->sdma.instance[i].feature_version >= 20) - adev->sdma.instance[i].burst_nop = true; - DRM_DEBUG("psp_load == '%s'\n", - adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; - info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; - info->fw = adev->sdma.instance[i].fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - } + ret = amdgpu_sdma_init_microcode(adev, fw_name, i, false); + if (ret) + return ret; } -out: - if (err) { - DRM_ERROR("sdma_v5_0: Failed to load firmware \"%s\"\n", fw_name); - for (i = 0; i < adev->sdma.num_instances; i++) { - release_firmware(adev->sdma.instance[i].fw); - adev->sdma.instance[i].fw = NULL; - } - } - return err; + + return ret; } static unsigned sdma_v5_0_ring_init_cond_exec(struct amdgpu_ring *ring) @@ -613,14 +584,10 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v5_0_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; u32 rb_cntl, ib_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_0_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); @@ -1465,12 +1432,10 @@ static int sdma_v5_0_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; int i; - for (i = 0; i < adev->sdma.num_instances; i++) { - release_firmware(adev->sdma.instance[i].fw); - adev->sdma.instance[i].fw = NULL; - + for (i = 0; i < adev->sdma.num_instances; i++) amdgpu_ring_fini(&adev->sdma.instance[i].ring); - } + + amdgpu_sdma_destroy_inst_ctx(adev, false); return 0; } @@ -1491,8 +1456,11 @@ static int sdma_v5_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev)) { + /* disable the scheduler for SDMA */ + amdgpu_sdma_unset_buffer_funcs_helper(adev); return 0; + } sdma_v5_0_ctx_switch_enable(adev, false); sdma_v5_0_enable(adev, false); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 95689ef4be102115c19166f168a680e7a42c4a1c..809eca54fc617f2c7ac5057495e2fbc046d0be53 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -89,33 +89,6 @@ static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u3 return base + internal_offset; } -static int sdma_v5_2_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) -{ - int err = 0; - const struct sdma_firmware_header_v1_0 *hdr; - - err = amdgpu_ucode_validate(sdma_inst->fw); - if (err) - return err; - - hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; - sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); - sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); - - if (sdma_inst->feature_version >= 20) - sdma_inst->burst_nop = true; - - return 0; -} - -static void sdma_v5_2_destroy_inst_ctx(struct amdgpu_device *adev) -{ - release_firmware(adev->sdma.instance[0].fw); - - memset((void *)adev->sdma.instance, 0, - sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); -} - /** * sdma_v5_2_init_microcode - load ucode images from disk * @@ -132,9 +105,6 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev) { const char *chip_name; char fw_name[40]; - int err = 0, i; - struct amdgpu_firmware_info *info = NULL; - const struct common_firmware_header *header = NULL; DRM_DEBUG("\n"); @@ -169,42 +139,7 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev) snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", chip_name); - err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v5_2_init_inst_ctx(&adev->sdma.instance[0]); - if (err) - goto out; - - for (i = 1; i < adev->sdma.num_instances; i++) - memcpy((void *)&adev->sdma.instance[i], - (void *)&adev->sdma.instance[0], - sizeof(struct amdgpu_sdma_instance)); - - if (amdgpu_sriov_vf(adev) && (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(5, 2, 0))) - return 0; - - DRM_DEBUG("psp_load == '%s'\n", - adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - for (i = 0; i < adev->sdma.num_instances; i++) { - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; - info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; - info->fw = adev->sdma.instance[i].fw; - header = (const struct common_firmware_header *)info->fw->data; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); - } - } - -out: - if (err) { - DRM_ERROR("sdma_v5_2: Failed to load firmware \"%s\"\n", fw_name); - sdma_v5_2_destroy_inst_ctx(adev); - } - return err; + return amdgpu_sdma_init_microcode(adev, fw_name, 0, true); } static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring) @@ -479,18 +414,10 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v5_2_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; - struct amdgpu_ring *sdma2 = &adev->sdma.instance[2].ring; - struct amdgpu_ring *sdma3 = &adev->sdma.instance[3].ring; u32 rb_cntl, ib_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1) || - (adev->mman.buffer_funcs_ring == sdma2) || - (adev->mman.buffer_funcs_ring == sdma3)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32_SOC15_IP(GC, sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_GFX_RB_CNTL)); @@ -1406,7 +1333,7 @@ static int sdma_v5_2_sw_fini(void *handle) for (i = 0; i < adev->sdma.num_instances; i++) amdgpu_ring_fini(&adev->sdma.instance[i].ring); - sdma_v5_2_destroy_inst_ctx(adev); + amdgpu_sdma_destroy_inst_ctx(adev, true); return 0; } @@ -1422,8 +1349,11 @@ static int sdma_v5_2_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev)) { + /* disable the scheduler for SDMA */ + amdgpu_sdma_unset_buffer_funcs_helper(adev); return 0; + } sdma_v5_2_ctx_switch_enable(adev, false); sdma_v5_2_enable(adev, false); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 7ae572a08cb37e9b28860d3fce374848571c4904..da3beb0bf2fa2e0c040064e9446248b5bd387b62 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -78,33 +78,6 @@ static u32 sdma_v6_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u3 return base + internal_offset; } -static int sdma_v6_0_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) -{ - int err = 0; - const struct sdma_firmware_header_v2_0 *hdr; - - err = amdgpu_ucode_validate(sdma_inst->fw); - if (err) - return err; - - hdr = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data; - sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); - sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); - - if (sdma_inst->feature_version >= 20) - sdma_inst->burst_nop = true; - - return 0; -} - -static void sdma_v6_0_destroy_inst_ctx(struct amdgpu_device *adev) -{ - release_firmware(adev->sdma.instance[0].fw); - - memset((void*)adev->sdma.instance, 0, - sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); -} - /** * sdma_v6_0_init_microcode - load ucode images from disk * @@ -114,16 +87,10 @@ static void sdma_v6_0_destroy_inst_ctx(struct amdgpu_device *adev) * the driver (not loaded into hw). * Returns 0 on success, error on failure. */ - -// emulation only, won't work on real chip -// sdma 6.0.0 real chip need to use PSP to load firmware static int sdma_v6_0_init_microcode(struct amdgpu_device *adev) { char fw_name[30]; char ucode_prefix[30]; - int err = 0, i; - struct amdgpu_firmware_info *info = NULL; - const struct sdma_firmware_header_v2_0 *sdma_hdr; DRM_DEBUG("\n"); @@ -131,43 +98,7 @@ static int sdma_v6_0_init_microcode(struct amdgpu_device *adev) snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", ucode_prefix); - err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v6_0_init_inst_ctx(&adev->sdma.instance[0]); - if (err) - goto out; - - for (i = 1; i < adev->sdma.num_instances; i++) { - memcpy((void*)&adev->sdma.instance[i], - (void*)&adev->sdma.instance[0], - sizeof(struct amdgpu_sdma_instance)); - } - - DRM_DEBUG("psp_load == '%s'\n", - adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); - - if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - sdma_hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data; - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0]; - info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0; - info->fw = adev->sdma.instance[0].fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE); - info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1]; - info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1; - info->fw = adev->sdma.instance[0].fw; - adev->firmware.fw_size += - ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE); - } - -out: - if (err) { - DRM_ERROR("sdma_v6_0: Failed to load firmware \"%s\"\n", fw_name); - sdma_v6_0_destroy_inst_ctx(adev); - } - return err; + return amdgpu_sdma_init_microcode(adev, fw_name, 0, true); } static unsigned sdma_v6_0_ring_init_cond_exec(struct amdgpu_ring *ring) @@ -467,14 +398,10 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se */ static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *sdma0 = &adev->sdma.instance[0].ring; - struct amdgpu_ring *sdma1 = &adev->sdma.instance[1].ring; u32 rb_cntl, ib_cntl; int i; - if ((adev->mman.buffer_funcs_ring == sdma0) || - (adev->mman.buffer_funcs_ring == sdma1)) - amdgpu_ttm_set_buffer_funcs_status(adev, false); + amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { rb_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_CNTL)); @@ -484,9 +411,6 @@ static void sdma_v6_0_gfx_stop(struct amdgpu_device *adev) ib_cntl = REG_SET_FIELD(ib_cntl, SDMA0_QUEUE0_IB_CNTL, IB_ENABLE, 0); WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_IB_CNTL), ib_cntl); } - - sdma0->sched.ready = false; - sdma1->sched.ready = false; } /** @@ -915,7 +839,8 @@ static int sdma_v6_0_mqd_init(struct amdgpu_device *adev, void *mqd, m->sdmax_rlcx_rb_cntl = order_base_2(prop->queue_size / 4) << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT | 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | - 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; + 4 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | + 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT; m->sdmax_rlcx_rb_base = lower_32_bits(prop->hqd_base_gpu_addr >> 8); m->sdmax_rlcx_rb_base_hi = upper_32_bits(prop->hqd_base_gpu_addr >> 8); @@ -1370,27 +1295,27 @@ static int sdma_v6_0_sw_fini(void *handle) for (i = 0; i < adev->sdma.num_instances; i++) amdgpu_ring_fini(&adev->sdma.instance[i].ring); - sdma_v6_0_destroy_inst_ctx(adev); + amdgpu_sdma_destroy_inst_ctx(adev, true); return 0; } static int sdma_v6_0_hw_init(void *handle) { - int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - r = sdma_v6_0_start(adev); - - return r; + return sdma_v6_0_start(adev); } static int sdma_v6_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev)) { + /* disable the scheduler for SDMA */ + amdgpu_sdma_unset_buffer_funcs_helper(adev); return 0; + } sdma_v6_0_ctx_switch_enable(adev, false); sdma_v6_0_enable(adev, false); diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index f675111ace20c2c858180511247f4b05d57b44af..4d5e718540aa99a838b3c7858458c070a3375a58 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -116,15 +116,14 @@ static void si_dma_stop(struct amdgpu_device *adev) u32 rb_cntl; unsigned i; + amdgpu_sdma_unset_buffer_funcs_helper(adev); + for (i = 0; i < adev->sdma.num_instances; i++) { ring = &adev->sdma.instance[i].ring; /* dma0 */ rb_cntl = RREG32(DMA_RB_CNTL + sdma_offsets[i]); rb_cntl &= ~DMA_RB_ENABLE; WREG32(DMA_RB_CNTL + sdma_offsets[i], rb_cntl); - - if (adev->mman.buffer_funcs_ring == ring) - amdgpu_ttm_set_buffer_funcs_status(adev, false); } } diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index fde6154f200963eb2aa81a039349dc987e1f93c0..183024d7c184e3d2be2b4c1386a827f500661bb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -1211,25 +1211,6 @@ static int soc15_common_sw_fini(void *handle) return 0; } -static void soc15_doorbell_range_init(struct amdgpu_device *adev) -{ - int i; - struct amdgpu_ring *ring; - - /* sdma/ih doorbell range are programed by hypervisor */ - if (!amdgpu_sriov_vf(adev)) { - for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; - adev->nbio.funcs->sdma_doorbell_range(adev, i, - ring->use_doorbell, ring->doorbell_index, - adev->doorbell_index.sdma_doorbell_range); - } - - adev->nbio.funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, - adev->irq.ih.doorbell_index); - } -} - static int soc15_common_hw_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -1249,12 +1230,6 @@ static int soc15_common_hw_init(void *handle) /* enable the doorbell aperture */ soc15_enable_doorbell_aperture(adev, true); - /* HW doorbell routing policy: doorbell writing not - * in SDMA/IH/MM/ACV range will be routed to CP. So - * we need to init SDMA/IH/MM/ACV doorbell range prior - * to CP ip block init and ring test. - */ - soc15_doorbell_range_init(adev); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index a26c5723c46e27a555d5103b438d598fa4d88934..795706b3b092f2cd832762d80abc0fb1774a46d4 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -421,6 +421,7 @@ static bool soc21_need_full_reset(struct amdgpu_device *adev) { switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(11, 0, 0): + return amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC); case IP_VERSION(11, 0, 2): return false; default: @@ -628,6 +629,8 @@ static int soc21_common_early_init(void *handle) AMD_CG_SUPPORT_JPEG_MGCG; adev->pg_flags = AMD_PG_SUPPORT_GFX_PG | + AMD_PG_SUPPORT_VCN | + AMD_PG_SUPPORT_VCN_DPG | AMD_PG_SUPPORT_JPEG; adev->external_rev_id = adev->rev_id + 0x1; break; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c index 939cb203f7ad53de64d2b1245d6d2873971ccdf7..f17d297b594bc0da1385347f7d5301e29eb2d8a0 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_1.c @@ -327,10 +327,9 @@ static void umc_v6_1_query_error_address(struct amdgpu_device *adev, return; } - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4); /* the lowest lsb bits should be ignored */ @@ -343,10 +342,7 @@ static void umc_v6_1_query_error_address(struct amdgpu_device *adev, ADDR_OF_256B_BLOCK(channel_index) | OFFSET_IN_256B_BLOCK(err_addr); - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) - == 1) - amdgpu_umc_fill_error_record(err_data, err_addr, + amdgpu_umc_fill_error_record(err_data, err_addr, retired_page, channel_index, umc_inst); } diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c index bf7524f16b669c767fc432bc2ba17bd0849ac935..5d5d031c9e7d09d60dcca406b32932cd36d42628 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c @@ -187,20 +187,51 @@ static void umc_v6_7_ecc_info_query_ras_error_count(struct amdgpu_device *adev, } } +static void umc_v6_7_convert_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst) +{ + uint32_t channel_index; + uint64_t soc_pa, retired_page, column; + + channel_index = + adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; + /* translate umc channel address to soc pa, 3 parts are included */ + soc_pa = ADDR_OF_8KB_BLOCK(err_addr) | + ADDR_OF_256B_BLOCK(channel_index) | + OFFSET_IN_256B_BLOCK(err_addr); + + /* The umc channel bits are not original values, they are hashed */ + SET_CHANNEL_HASH(channel_index, soc_pa); + + /* clear [C4 C3 C2] in soc physical address */ + soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT); + + /* loop for all possibilities of [C4 C3 C2] */ + for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) { + retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT); + dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); + amdgpu_umc_fill_error_record(err_data, err_addr, + retired_page, channel_index, umc_inst); + + /* shift R14 bit */ + retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT); + dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); + amdgpu_umc_fill_error_record(err_data, err_addr, + retired_page, channel_index, umc_inst); + } +} + static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev, struct ras_err_data *err_data, uint32_t ch_inst, uint32_t umc_inst) { - uint64_t mc_umc_status, err_addr, soc_pa, retired_page, column; - uint32_t channel_index; + uint64_t mc_umc_status, err_addr; uint32_t eccinfo_table_idx; struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst; - channel_index = - adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; - mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status; if (mc_umc_status == 0) @@ -209,42 +240,15 @@ static void umc_v6_7_ecc_info_query_error_address(struct amdgpu_device *adev, if (!err_data->err_addr) return; - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { err_addr = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_addr; err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); - /* translate umc channel address to soc pa, 3 parts are included */ - soc_pa = ADDR_OF_8KB_BLOCK(err_addr) | - ADDR_OF_256B_BLOCK(channel_index) | - OFFSET_IN_256B_BLOCK(err_addr); - - /* The umc channel bits are not original values, they are hashed */ - SET_CHANNEL_HASH(channel_index, soc_pa); - - /* clear [C4 C3 C2] in soc physical address */ - soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT); - - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) - == 1) { - /* loop for all possibilities of [C4 C3 C2] */ - for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) { - retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT); - dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); - - /* shift R14 bit */ - retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT); - dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); - } - } + umc_v6_7_convert_error_address(adev, err_data, err_addr, + ch_inst, umc_inst); } } @@ -452,14 +456,11 @@ static void umc_v6_7_query_ras_error_count(struct amdgpu_device *adev, static void umc_v6_7_query_error_address(struct amdgpu_device *adev, struct ras_err_data *err_data, - uint32_t umc_reg_offset, - uint32_t ch_inst, + uint32_t umc_reg_offset, uint32_t ch_inst, uint32_t umc_inst) { uint32_t mc_umc_status_addr; - uint32_t channel_index; - uint64_t mc_umc_status, mc_umc_addrt0; - uint64_t err_addr, soc_pa, retired_page, column; + uint64_t mc_umc_status = 0, mc_umc_addrt0, err_addr; mc_umc_status_addr = SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0); @@ -477,45 +478,15 @@ static void umc_v6_7_query_error_address(struct amdgpu_device *adev, return; } - channel_index = - adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; - - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { - + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4); - err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); + err_addr = + REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); - /* translate umc channel address to soc pa, 3 parts are included */ - soc_pa = ADDR_OF_8KB_BLOCK(err_addr) | - ADDR_OF_256B_BLOCK(channel_index) | - OFFSET_IN_256B_BLOCK(err_addr); - - /* The umc channel bits are not original values, they are hashed */ - SET_CHANNEL_HASH(channel_index, soc_pa); - - /* clear [C4 C3 C2] in soc physical address */ - soc_pa &= ~(0x7ULL << UMC_V6_7_PA_C2_BIT); - - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) - == 1) { - /* loop for all possibilities of [C4 C3 C2] */ - for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) { - retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT); - dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); - - /* shift R14 bit */ - retired_page ^= (0x1ULL << UMC_V6_7_PA_R14_BIT); - dev_info(adev->dev, "Error Address(PA): 0x%llx\n", retired_page); - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); - } - } + umc_v6_7_convert_error_address(adev, err_data, err_addr, + ch_inst, umc_inst); } /* clear umc status */ @@ -540,8 +511,7 @@ static void umc_v6_7_query_ras_error_address(struct amdgpu_device *adev, ch_inst); umc_v6_7_query_error_address(adev, err_data, - umc_reg_offset, - ch_inst, + umc_reg_offset, ch_inst, umc_inst); } } @@ -583,4 +553,5 @@ struct amdgpu_umc_ras umc_v6_7_ras = { .query_ras_poison_mode = umc_v6_7_query_ras_poison_mode, .ecc_info_query_ras_error_count = umc_v6_7_ecc_info_query_ras_error_count, .ecc_info_query_ras_error_address = umc_v6_7_ecc_info_query_ras_error_address, + .convert_ras_error_address = umc_v6_7_convert_error_address, }; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c index 36a2053f2e8b94b688bff03f9f363db95db5d7c5..91235df54e22bb45da27847319144e264ef95d5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c @@ -101,22 +101,16 @@ static void umc_v8_10_query_correctable_error_count(struct amdgpu_device *adev, uint32_t umc_reg_offset, unsigned long *error_count) { - uint32_t ecc_err_cnt, ecc_err_cnt_addr; uint64_t mc_umc_status; uint32_t mc_umc_status_addr; /* UMC 8_10 registers */ - ecc_err_cnt_addr = - SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccErrCnt); mc_umc_status_addr = SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0); - ecc_err_cnt = RREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4); - *error_count += - (REG_GET_FIELD(ecc_err_cnt, UMCCH0_0_GeccErrCnt, GeccErrCnt) - - UMC_V8_10_CE_CNT_INIT); - - /* Check for SRAM correctable error, MCUMC_STATUS is a 64 bit register */ + /* Rely on MCUMC_STATUS for correctable error counter + * MCUMC_STATUS is a 64 bit register + */ mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4); if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1) @@ -214,7 +208,10 @@ static void umc_v8_10_query_error_address(struct amdgpu_device *adev, { uint64_t mc_umc_status_addr; uint64_t mc_umc_status, err_addr; - uint32_t channel_index; + uint64_t mc_umc_addrt0, na_err_addr_base; + uint64_t na_err_addr, retired_page_addr; + uint32_t channel_index, addr_lsb, col = 0; + int ret = 0; mc_umc_status_addr = SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0); @@ -235,13 +232,10 @@ static void umc_v8_10_query_error_address(struct amdgpu_device *adev, umc_inst * adev->umc.channel_inst_num + ch_inst]; - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrV) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { - uint32_t addr_lsb; - uint64_t mc_umc_addrt0; + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { mc_umc_addrt0 = SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0); err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4); @@ -249,32 +243,24 @@ static void umc_v8_10_query_error_address(struct amdgpu_device *adev, /* the lowest lsb bits should be ignored */ addr_lsb = REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, AddrLsb); - err_addr &= ~((0x1ULL << addr_lsb) - 1); - - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { - uint64_t na_err_addr_base = err_addr & ~(0x3ULL << UMC_V8_10_NA_C5_BIT); - uint64_t na_err_addr, retired_page_addr; - uint32_t col = 0; - int ret = 0; - - /* loop for all possibilities of [C6 C5] in normal address. */ - for (col = 0; col < UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM; col++) { - na_err_addr = na_err_addr_base | (col << UMC_V8_10_NA_C5_BIT); - - /* Mapping normal error address to retired soc physical address. */ - ret = umc_v8_10_swizzle_mode_na_to_pa(adev, channel_index, - na_err_addr, &retired_page_addr); - if (ret) { - dev_err(adev->dev, "Failed to map pa from umc na.\n"); - break; - } - dev_info(adev->dev, "Error Address(PA): 0x%llx\n", - retired_page_addr); - amdgpu_umc_fill_error_record(err_data, na_err_addr, - retired_page_addr, channel_index, umc_inst); + na_err_addr_base = err_addr & ~(0x3ULL << UMC_V8_10_NA_C5_BIT); + + /* loop for all possibilities of [C6 C5] in normal address. */ + for (col = 0; col < UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM; col++) { + na_err_addr = na_err_addr_base | (col << UMC_V8_10_NA_C5_BIT); + + /* Mapping normal error address to retired soc physical address. */ + ret = umc_v8_10_swizzle_mode_na_to_pa(adev, channel_index, + na_err_addr, &retired_page_addr); + if (ret) { + dev_err(adev->dev, "Failed to map pa from umc na.\n"); + break; } + dev_info(adev->dev, "Error Address(PA): 0x%llx\n", + retired_page_addr); + amdgpu_umc_fill_error_record(err_data, na_err_addr, + retired_page_addr, channel_index, umc_inst); } } @@ -344,6 +330,31 @@ static void umc_v8_10_err_cnt_init(struct amdgpu_device *adev) } } +static uint32_t umc_v8_10_query_ras_poison_mode_per_channel( + struct amdgpu_device *adev, + uint32_t umc_reg_offset) +{ + uint32_t ecc_ctrl_addr, ecc_ctrl; + + ecc_ctrl_addr = + SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccCtrl); + ecc_ctrl = RREG32_PCIE((ecc_ctrl_addr + + umc_reg_offset) * 4); + + return REG_GET_FIELD(ecc_ctrl, UMCCH0_0_GeccCtrl, UCFatalEn); +} + +static bool umc_v8_10_query_ras_poison_mode(struct amdgpu_device *adev) +{ + uint32_t umc_reg_offset = 0; + + /* Enabling fatal error in umc node0 instance0 channel0 will be + * considered as fatal error mode + */ + umc_reg_offset = get_umc_v8_10_reg_offset(adev, 0, 0, 0); + return !umc_v8_10_query_ras_poison_mode_per_channel(adev, umc_reg_offset); +} + const struct amdgpu_ras_block_hw_ops umc_v8_10_ras_hw_ops = { .query_ras_error_count = umc_v8_10_query_ras_error_count, .query_ras_error_address = umc_v8_10_query_ras_error_address, @@ -354,4 +365,5 @@ struct amdgpu_umc_ras umc_v8_10_ras = { .hw_ops = &umc_v8_10_ras_hw_ops, }, .err_cnt_init = umc_v8_10_err_cnt_init, + .query_ras_poison_mode = umc_v8_10_query_ras_poison_mode, }; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c index f35253e0eaa6d62c7477b283eeb121c31b5b03e1..b717fdaa46e45281db06b6f273c5d56a72afe507 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_7.c @@ -108,20 +108,35 @@ static void umc_v8_7_ecc_info_query_ras_error_count(struct amdgpu_device *adev, } } +static void umc_v8_7_convert_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst) +{ + uint64_t retired_page; + uint32_t channel_index; + + channel_index = + adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; + + /* translate umc channel address to soc pa, 3 parts are included */ + retired_page = ADDR_OF_4KB_BLOCK(err_addr) | + ADDR_OF_256B_BLOCK(channel_index) | + OFFSET_IN_256B_BLOCK(err_addr); + + amdgpu_umc_fill_error_record(err_data, err_addr, + retired_page, channel_index, umc_inst); +} + static void umc_v8_7_ecc_info_query_error_address(struct amdgpu_device *adev, struct ras_err_data *err_data, uint32_t ch_inst, uint32_t umc_inst) { - uint64_t mc_umc_status, err_addr, retired_page; - uint32_t channel_index; + uint64_t mc_umc_status, err_addr; uint32_t eccinfo_table_idx; struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); eccinfo_table_idx = umc_inst * adev->umc.channel_inst_num + ch_inst; - channel_index = - adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; - mc_umc_status = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_status; if (mc_umc_status == 0) @@ -130,24 +145,15 @@ static void umc_v8_7_ecc_info_query_error_address(struct amdgpu_device *adev, if (!err_data->err_addr) return; - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { err_addr = ras->umc_ecc.ecc[eccinfo_table_idx].mca_umc_addr; err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); - /* translate umc channel address to soc pa, 3 parts are included */ - retired_page = ADDR_OF_4KB_BLOCK(err_addr) | - ADDR_OF_256B_BLOCK(channel_index) | - OFFSET_IN_256B_BLOCK(err_addr); - - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) - == 1) - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); + umc_v8_7_convert_error_address(adev, err_data, err_addr, + ch_inst, umc_inst); } } @@ -324,14 +330,12 @@ static void umc_v8_7_query_error_address(struct amdgpu_device *adev, uint32_t umc_inst) { uint32_t lsb, mc_umc_status_addr; - uint64_t mc_umc_status, err_addr, retired_page, mc_umc_addrt0; - uint32_t channel_index = adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst]; + uint64_t mc_umc_status, err_addr, mc_umc_addrt0; mc_umc_status_addr = SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_STATUST0); mc_umc_addrt0 = SOC15_REG_OFFSET(UMC, 0, mmMCA_UMC_UMC0_MCUMC_ADDRT0); - mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4); if (mc_umc_status == 0) @@ -343,10 +347,9 @@ static void umc_v8_7_query_error_address(struct amdgpu_device *adev, return; } - /* calculate error address if ue/ce error is detected */ + /* calculate error address if ue error is detected */ if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 && - (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 || - REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) { + REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1) { err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4); /* the lowest lsb bits should be ignored */ @@ -354,16 +357,8 @@ static void umc_v8_7_query_error_address(struct amdgpu_device *adev, err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); err_addr &= ~((0x1ULL << lsb) - 1); - /* translate umc channel address to soc pa, 3 parts are included */ - retired_page = ADDR_OF_4KB_BLOCK(err_addr) | - ADDR_OF_256B_BLOCK(channel_index) | - OFFSET_IN_256B_BLOCK(err_addr); - - /* we only save ue error information currently, ce is skipped */ - if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) - == 1) - amdgpu_umc_fill_error_record(err_data, err_addr, - retired_page, channel_index, umc_inst); + umc_v8_7_convert_error_address(adev, err_data, err_addr, + ch_inst, umc_inst); } /* clear umc status */ diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index 39405f0db824100c708b26ce71447a0f4d3bbf90..9c8b5fd9903720253c9464380864a3a16ec26a64 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -1761,21 +1761,23 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = { .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, }; -static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p) +static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p, + struct amdgpu_job *job) { struct drm_gpu_scheduler **scheds; /* The create msg must be in the first IB submitted */ - if (atomic_read(&p->entity->fence_seq)) + if (atomic_read(&job->base.entity->fence_seq)) return -EINVAL; scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC] [AMDGPU_RING_PRIO_DEFAULT].sched; - drm_sched_entity_modify_sched(p->entity, scheds, 1); + drm_sched_entity_modify_sched(job->base.entity, scheds, 1); return 0; } -static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr) +static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, + uint64_t addr) { struct ttm_operation_ctx ctx = { false, false }; struct amdgpu_bo_va_mapping *map; @@ -1846,7 +1848,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr) if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11) continue; - r = vcn_v3_0_limit_sched(p); + r = vcn_v3_0_limit_sched(p, job); if (r) goto out; } @@ -1860,7 +1862,7 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_job *job, struct amdgpu_ib *ib) { - struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); + struct amdgpu_ring *ring = amdgpu_job_ring(job); uint32_t msg_lo = 0, msg_hi = 0; unsigned i; int r; @@ -1879,7 +1881,8 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, msg_hi = val; } else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) && val == 0) { - r = vcn_v3_0_dec_msg(p, ((u64)msg_hi) << 32 | msg_lo); + r = vcn_v3_0_dec_msg(p, job, + ((u64)msg_hi) << 32 | msg_lo); if (r) return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index 09c89faa8c277fee188b188ae4614aa546d5ca8e..897a5ce9c9da6f6f38e025ee0c0238bc9e8a9af3 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -150,6 +150,10 @@ static int vcn_v4_0_sw_init(void *handle) fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); fw_shared->sq.is_enabled = 1; + fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG); + fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ? + AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU; + if (amdgpu_sriov_vf(adev)) fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); @@ -1591,21 +1595,23 @@ static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring) } } -static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p) +static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p, + struct amdgpu_job *job) { struct drm_gpu_scheduler **scheds; /* The create msg must be in the first IB submitted */ - if (atomic_read(&p->entity->fence_seq)) + if (atomic_read(&job->base.entity->fence_seq)) return -EINVAL; scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC] [AMDGPU_RING_PRIO_0].sched; - drm_sched_entity_modify_sched(p->entity, scheds, 1); + drm_sched_entity_modify_sched(job->base.entity, scheds, 1); return 0; } -static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr) +static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, + uint64_t addr) { struct ttm_operation_ctx ctx = { false, false }; struct amdgpu_bo_va_mapping *map; @@ -1676,7 +1682,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr) if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11) continue; - r = vcn_v4_0_limit_sched(p); + r = vcn_v4_0_limit_sched(p, job); if (r) goto out; } @@ -1689,32 +1695,34 @@ out: #define RADEON_VCN_ENGINE_TYPE_DECODE (0x00000003) static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, - struct amdgpu_job *job, - struct amdgpu_ib *ib) + struct amdgpu_job *job, + struct amdgpu_ib *ib) { - struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched); - struct amdgpu_vcn_decode_buffer *decode_buffer = NULL; + struct amdgpu_ring *ring = amdgpu_job_ring(job); + struct amdgpu_vcn_decode_buffer *decode_buffer; + uint64_t addr; uint32_t val; - int r = 0; /* The first instance can decode anything */ if (!ring->me) - return r; + return 0; /* unified queue ib header has 8 double words. */ if (ib->length_dw < 8) - return r; + return 0; val = amdgpu_ib_get_value(ib, 6); //RADEON_VCN_ENGINE_TYPE + if (val != RADEON_VCN_ENGINE_TYPE_DECODE) + return 0; - if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { - decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10]; + decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10]; - if (decode_buffer->valid_buf_flag & 0x1) - r = vcn_v4_0_dec_msg(p, ((u64)decode_buffer->msg_buffer_address_hi) << 32 | - decode_buffer->msg_buffer_address_lo); - } - return r; + if (!(decode_buffer->valid_buf_flag & 0x1)) + return 0; + + addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 | + decode_buffer->msg_buffer_address_lo; + return vcn_v4_0_dec_msg(p, job, addr); } static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = { diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index 03b7066471f9ad251d4337350ced36882a10f582..1e83db0c5438d0c07b27bf1540ad6b4bdf2bc61e 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -289,6 +289,10 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) } } + if (!amdgpu_sriov_vf(adev)) + adev->nbio.funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, + adev->irq.ih.doorbell_index); + pci_set_master(adev->pdev); /* enable interrupts */ diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c index 2022ffbb8dba55e6522e56e689d582c87dc6e543..59dfca093155c6a1cf5bd58b8f62a31e2adf4738 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -340,6 +340,10 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev) } } + if (!amdgpu_sriov_vf(adev)) + adev->nbio.funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, + adev->irq.ih.doorbell_index); + pci_set_master(adev->pdev); /* enable interrupts */ diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index 60a81649cf1287775607cf59168ae532da789a9a..c7118843db051ccaa86435a6a938f3541c985ceb 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -742,7 +742,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = { 0xbf88fffe, 0x877aff7f, 0x04000000, 0x8f7a857a, 0x886d7a6d, 0xb97b02dc, - 0x8f7b997b, 0xb97a2a05, + 0x8f7b997b, 0xb97a3a05, 0x807a817a, 0xbf0d997b, 0xbf850002, 0x8f7a897a, 0xbf820001, 0x8f7a8a7a, @@ -819,7 +819,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = { 0xbefe037c, 0xbefc0370, 0xf4611c7a, 0xf8000000, 0x80708470, 0xbefc037e, - 0xb9702a05, 0x80708170, + 0xb9703a05, 0x80708170, 0xbf0d9973, 0xbf850002, 0x8f708970, 0xbf820001, 0x8f708a70, 0xb97a1e06, @@ -1069,7 +1069,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = { 0xb9f9f816, 0x876f7bff, 0xfffff800, 0x906f8b6f, 0xb9efa2c3, 0xb9f3f801, - 0xb96e2a05, 0x806e816e, + 0xb96e3a05, 0x806e816e, 0xbf0d9972, 0xbf850002, 0x8f6e896e, 0xbf820001, 0x8f6e8a6e, 0xb96f1e06, @@ -2114,7 +2114,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0x007a0000, 0x7e000280, 0xbefe037a, 0xbeff037b, 0xb97b02dc, 0x8f7b997b, - 0xb97a2a05, 0x807a817a, + 0xb97a3a05, 0x807a817a, 0xbf0d997b, 0xbf850002, 0x8f7a897a, 0xbf820001, 0x8f7a8a7a, 0xb97b1e06, @@ -2157,7 +2157,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0x01000000, 0xe0704100, 0x705d0100, 0xe0704200, 0x705d0200, 0xe0704300, - 0x705d0300, 0xb9702a05, + 0x705d0300, 0xb9703a05, 0x80708170, 0xbf0d9973, 0xbf850002, 0x8f708970, 0xbf820001, 0x8f708a70, @@ -2189,7 +2189,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xbefe03ff, 0x0000ffff, 0xbeff0380, 0xe0704000, 0x705d0200, 0xbefe03c1, - 0xb9702a05, 0x80708170, + 0xb9703a05, 0x80708170, 0xbf0d9973, 0xbf850002, 0x8f708970, 0xbf820001, 0x8f708a70, 0xb97a1e06, @@ -2475,7 +2475,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xb9ef4803, 0x876f7bff, 0xfffff800, 0x906f8b6f, 0xb9efa2c3, 0xb9f3f801, - 0xb96e2a05, 0x806e816e, + 0xb96e3a05, 0x806e816e, 0xbf0d9972, 0xbf850002, 0x8f6e896e, 0xbf820001, 0x8f6e8a6e, 0xb96f1e06, @@ -2494,438 +2494,441 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0x00000000, }; - static const uint32_t cwsr_trap_gfx11_hex[] = { - 0xbfa00001, 0xbfa0021b, + 0xbfa00001, 0xbfa0021e, 0xb0804006, 0xb8f8f802, - 0x91788678, 0xb8fbf803, - 0x8b6eff78, 0x00002000, - 0xbfa10009, 0x8b6eff6d, - 0x00ff0000, 0xbfa2001e, - 0x8b6eff7b, 0x00000400, - 0xbfa20041, 0xbf830010, - 0xb8fbf803, 0xbfa0fffa, - 0x8b6eff7b, 0x00000900, - 0xbfa20015, 0x8b6eff7b, - 0x000071ff, 0xbfa10008, - 0x8b6fff7b, 0x00007080, - 0xbfa10001, 0xbeee1287, - 0xb8eff801, 0x846e8c6e, - 0x8b6e6f6e, 0xbfa2000a, + 0x9178ff78, 0x00020006, + 0xb8fbf803, 0xbf0d9f6d, + 0xbfa20006, 0x8b6eff78, + 0x00002000, 0xbfa10009, 0x8b6eff6d, 0x00ff0000, - 0xbfa20007, 0xb8eef801, - 0x8b6eff6e, 0x00000800, - 0xbfa20003, 0x8b6eff7b, - 0x00000400, 0xbfa20026, - 0xbefa4d82, 0xbf89fc07, - 0x84fa887a, 0xf4005bbd, - 0xf8000010, 0xbf89fc07, - 0x846e976e, 0x9177ff77, - 0x00800000, 0x8c776e77, - 0xf4045bbd, 0xf8000000, - 0xbf89fc07, 0xf4045ebd, - 0xf8000008, 0xbf89fc07, - 0x8bee6e6e, 0xbfa10001, - 0xbe80486e, 0x8b6eff6d, - 0x01ff0000, 0xbfa20005, - 0x8c78ff78, 0x00002000, - 0x80ec886c, 0x82ed806d, - 0xbfa00005, 0x8b6eff6d, - 0x01000000, 0xbfa20002, - 0x806c846c, 0x826d806d, - 0x8b6dff6d, 0x0000ffff, - 0x8bfe7e7e, 0x8bea6a6a, - 0xb978f802, 0xbe804a6c, - 0x8b6dff6d, 0x0000ffff, - 0xbefa0080, 0xb97a0283, - 0xbeee007e, 0xbeef007f, - 0xbefe0180, 0xbefe4d84, - 0xbf89fc07, 0x8b7aff7f, - 0x04000000, 0x847a857a, - 0x8c6d7a6d, 0xbefa007e, - 0x8b7bff7f, 0x0000ffff, - 0xbefe00c1, 0xbeff00c1, - 0xdca6c000, 0x007a0000, - 0x7e000280, 0xbefe007a, - 0xbeff007b, 0xb8fb02dc, - 0x847b997b, 0xb8fa3b05, - 0x807a817a, 0xbf0d997b, - 0xbfa20002, 0x847a897a, - 0xbfa00001, 0x847a8a7a, - 0xb8fb1e06, 0x847b8a7b, - 0x807a7b7a, 0x8b7bff7f, - 0x0000ffff, 0x807aff7a, - 0x00000200, 0x807a7e7a, - 0x827b807b, 0xd7610000, - 0x00010870, 0xd7610000, - 0x00010a71, 0xd7610000, - 0x00010c72, 0xd7610000, - 0x00010e73, 0xd7610000, - 0x00011074, 0xd7610000, - 0x00011275, 0xd7610000, - 0x00011476, 0xd7610000, - 0x00011677, 0xd7610000, - 0x00011a79, 0xd7610000, - 0x00011c7e, 0xd7610000, - 0x00011e7f, 0xbefe00ff, - 0x00003fff, 0xbeff0080, - 0xdca6c040, 0x007a0000, - 0xd760007a, 0x00011d00, - 0xd760007b, 0x00011f00, + 0xbfa2001e, 0x8b6eff7b, + 0x00000400, 0xbfa20041, + 0xbf830010, 0xb8fbf803, + 0xbfa0fffa, 0x8b6eff7b, + 0x00000900, 0xbfa20015, + 0x8b6eff7b, 0x000071ff, + 0xbfa10008, 0x8b6fff7b, + 0x00007080, 0xbfa10001, + 0xbeee1287, 0xb8eff801, + 0x846e8c6e, 0x8b6e6f6e, + 0xbfa2000a, 0x8b6eff6d, + 0x00ff0000, 0xbfa20007, + 0xb8eef801, 0x8b6eff6e, + 0x00000800, 0xbfa20003, + 0x8b6eff7b, 0x00000400, + 0xbfa20026, 0xbefa4d82, + 0xbf89fc07, 0x84fa887a, + 0xf4005bbd, 0xf8000010, + 0xbf89fc07, 0x846e976e, + 0x9177ff77, 0x00800000, + 0x8c776e77, 0xf4045bbd, + 0xf8000000, 0xbf89fc07, + 0xf4045ebd, 0xf8000008, + 0xbf89fc07, 0x8bee6e6e, + 0xbfa10001, 0xbe80486e, + 0x8b6eff6d, 0x01ff0000, + 0xbfa20005, 0x8c78ff78, + 0x00002000, 0x80ec886c, + 0x82ed806d, 0xbfa00005, + 0x8b6eff6d, 0x01000000, + 0xbfa20002, 0x806c846c, + 0x826d806d, 0x8b6dff6d, + 0x0000ffff, 0x8bfe7e7e, + 0x8bea6a6a, 0xb978f802, + 0xbe804a6c, 0x8b6dff6d, + 0x0000ffff, 0xbefa0080, + 0xb97a0283, 0xbeee007e, + 0xbeef007f, 0xbefe0180, + 0xbefe4d84, 0xbf89fc07, + 0x8b7aff7f, 0x04000000, + 0x847a857a, 0x8c6d7a6d, + 0xbefa007e, 0x8b7bff7f, + 0x0000ffff, 0xbefe00c1, + 0xbeff00c1, 0xdca6c000, + 0x007a0000, 0x7e000280, 0xbefe007a, 0xbeff007b, - 0xbef4007e, 0x8b75ff7f, - 0x0000ffff, 0x8c75ff75, - 0x00040000, 0xbef60080, - 0xbef700ff, 0x10807fac, - 0xbef1007d, 0xbef00080, - 0xb8f302dc, 0x84739973, - 0xbefe00c1, 0x857d9973, - 0x8b7d817d, 0xbf06817d, - 0xbfa20002, 0xbeff0080, - 0xbfa00002, 0xbeff00c1, - 0xbfa00009, 0xbef600ff, - 0x01000000, 0xe0685080, - 0x701d0100, 0xe0685100, - 0x701d0200, 0xe0685180, - 0x701d0300, 0xbfa00008, + 0xb8fb02dc, 0x847b997b, + 0xb8fa3b05, 0x807a817a, + 0xbf0d997b, 0xbfa20002, + 0x847a897a, 0xbfa00001, + 0x847a8a7a, 0xb8fb1e06, + 0x847b8a7b, 0x807a7b7a, + 0x8b7bff7f, 0x0000ffff, + 0x807aff7a, 0x00000200, + 0x807a7e7a, 0x827b807b, + 0xd7610000, 0x00010870, + 0xd7610000, 0x00010a71, + 0xd7610000, 0x00010c72, + 0xd7610000, 0x00010e73, + 0xd7610000, 0x00011074, + 0xd7610000, 0x00011275, + 0xd7610000, 0x00011476, + 0xd7610000, 0x00011677, + 0xd7610000, 0x00011a79, + 0xd7610000, 0x00011c7e, + 0xd7610000, 0x00011e7f, + 0xbefe00ff, 0x00003fff, + 0xbeff0080, 0xdca6c040, + 0x007a0000, 0xd760007a, + 0x00011d00, 0xd760007b, + 0x00011f00, 0xbefe007a, + 0xbeff007b, 0xbef4007e, + 0x8b75ff7f, 0x0000ffff, + 0x8c75ff75, 0x00040000, + 0xbef60080, 0xbef700ff, + 0x10807fac, 0xbef1007d, + 0xbef00080, 0xb8f302dc, + 0x84739973, 0xbefe00c1, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00002, + 0xbeff00c1, 0xbfa00009, 0xbef600ff, 0x01000000, - 0xe0685100, 0x701d0100, - 0xe0685200, 0x701d0200, - 0xe0685300, 0x701d0300, + 0xe0685080, 0x701d0100, + 0xe0685100, 0x701d0200, + 0xe0685180, 0x701d0300, + 0xbfa00008, 0xbef600ff, + 0x01000000, 0xe0685100, + 0x701d0100, 0xe0685200, + 0x701d0200, 0xe0685300, + 0x701d0300, 0xb8f03b05, + 0x80708170, 0xbf0d9973, + 0xbfa20002, 0x84708970, + 0xbfa00001, 0x84708a70, + 0xb8fa1e06, 0x847a8a7a, + 0x80707a70, 0x8070ff70, + 0x00000200, 0xbef600ff, + 0x01000000, 0x7e000280, + 0x7e020280, 0x7e040280, + 0xbefd0080, 0xd7610002, + 0x0000fa71, 0x807d817d, + 0xd7610002, 0x0000fa6c, + 0x807d817d, 0x917aff6d, + 0x80000000, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xd7610002, 0x0000fa6e, + 0x807d817d, 0xd7610002, + 0x0000fa6f, 0x807d817d, + 0xd7610002, 0x0000fa78, + 0x807d817d, 0xb8faf803, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xd7610002, + 0x0000fa7b, 0x807d817d, + 0xb8f1f801, 0xd7610002, + 0x0000fa71, 0x807d817d, + 0xb8f1f814, 0xd7610002, + 0x0000fa71, 0x807d817d, + 0xb8f1f815, 0xd7610002, + 0x0000fa71, 0x807d817d, + 0xbefe00ff, 0x0000ffff, + 0xbeff0080, 0xe0685000, + 0x701d0200, 0xbefe00c1, 0xb8f03b05, 0x80708170, 0xbf0d9973, 0xbfa20002, 0x84708970, 0xbfa00001, 0x84708a70, 0xb8fa1e06, 0x847a8a7a, 0x80707a70, - 0x8070ff70, 0x00000200, 0xbef600ff, 0x01000000, - 0x7e000280, 0x7e020280, - 0x7e040280, 0xbefd0080, - 0xd7610002, 0x0000fa71, - 0x807d817d, 0xd7610002, - 0x0000fa6c, 0x807d817d, - 0x917aff6d, 0x80000000, - 0xd7610002, 0x0000fa7a, - 0x807d817d, 0xd7610002, - 0x0000fa6e, 0x807d817d, - 0xd7610002, 0x0000fa6f, - 0x807d817d, 0xd7610002, - 0x0000fa78, 0x807d817d, - 0xb8faf803, 0xd7610002, - 0x0000fa7a, 0x807d817d, - 0xd7610002, 0x0000fa7b, - 0x807d817d, 0xb8f1f801, - 0xd7610002, 0x0000fa71, - 0x807d817d, 0xb8f1f814, - 0xd7610002, 0x0000fa71, - 0x807d817d, 0xb8f1f815, - 0xd7610002, 0x0000fa71, - 0x807d817d, 0xbefe00ff, - 0x0000ffff, 0xbeff0080, - 0xe0685000, 0x701d0200, - 0xbefe00c1, 0xb8f03b05, - 0x80708170, 0xbf0d9973, - 0xbfa20002, 0x84708970, - 0xbfa00001, 0x84708a70, - 0xb8fa1e06, 0x847a8a7a, - 0x80707a70, 0xbef600ff, - 0x01000000, 0xbef90080, - 0xbefd0080, 0xbf800000, - 0xbe804100, 0xbe824102, - 0xbe844104, 0xbe864106, - 0xbe884108, 0xbe8a410a, - 0xbe8c410c, 0xbe8e410e, - 0xd7610002, 0x0000f200, - 0x80798179, 0xd7610002, - 0x0000f201, 0x80798179, - 0xd7610002, 0x0000f202, - 0x80798179, 0xd7610002, - 0x0000f203, 0x80798179, - 0xd7610002, 0x0000f204, + 0xbef90080, 0xbefd0080, + 0xbf800000, 0xbe804100, + 0xbe824102, 0xbe844104, + 0xbe864106, 0xbe884108, + 0xbe8a410a, 0xbe8c410c, + 0xbe8e410e, 0xd7610002, + 0x0000f200, 0x80798179, + 0xd7610002, 0x0000f201, 0x80798179, 0xd7610002, - 0x0000f205, 0x80798179, - 0xd7610002, 0x0000f206, + 0x0000f202, 0x80798179, + 0xd7610002, 0x0000f203, 0x80798179, 0xd7610002, - 0x0000f207, 0x80798179, - 0xd7610002, 0x0000f208, + 0x0000f204, 0x80798179, + 0xd7610002, 0x0000f205, 0x80798179, 0xd7610002, - 0x0000f209, 0x80798179, - 0xd7610002, 0x0000f20a, + 0x0000f206, 0x80798179, + 0xd7610002, 0x0000f207, 0x80798179, 0xd7610002, - 0x0000f20b, 0x80798179, - 0xd7610002, 0x0000f20c, + 0x0000f208, 0x80798179, + 0xd7610002, 0x0000f209, 0x80798179, 0xd7610002, - 0x0000f20d, 0x80798179, - 0xd7610002, 0x0000f20e, + 0x0000f20a, 0x80798179, + 0xd7610002, 0x0000f20b, 0x80798179, 0xd7610002, - 0x0000f20f, 0x80798179, - 0xbf06a079, 0xbfa10006, - 0xe0685000, 0x701d0200, - 0x8070ff70, 0x00000080, - 0xbef90080, 0x7e040280, - 0x807d907d, 0xbf0aff7d, - 0x00000060, 0xbfa2ffbc, - 0xbe804100, 0xbe824102, - 0xbe844104, 0xbe864106, - 0xbe884108, 0xbe8a410a, - 0xd7610002, 0x0000f200, + 0x0000f20c, 0x80798179, + 0xd7610002, 0x0000f20d, 0x80798179, 0xd7610002, - 0x0000f201, 0x80798179, - 0xd7610002, 0x0000f202, + 0x0000f20e, 0x80798179, + 0xd7610002, 0x0000f20f, + 0x80798179, 0xbf06a079, + 0xbfa10006, 0xe0685000, + 0x701d0200, 0x8070ff70, + 0x00000080, 0xbef90080, + 0x7e040280, 0x807d907d, + 0xbf0aff7d, 0x00000060, + 0xbfa2ffbc, 0xbe804100, + 0xbe824102, 0xbe844104, + 0xbe864106, 0xbe884108, + 0xbe8a410a, 0xd7610002, + 0x0000f200, 0x80798179, + 0xd7610002, 0x0000f201, 0x80798179, 0xd7610002, - 0x0000f203, 0x80798179, - 0xd7610002, 0x0000f204, + 0x0000f202, 0x80798179, + 0xd7610002, 0x0000f203, 0x80798179, 0xd7610002, - 0x0000f205, 0x80798179, - 0xd7610002, 0x0000f206, + 0x0000f204, 0x80798179, + 0xd7610002, 0x0000f205, 0x80798179, 0xd7610002, - 0x0000f207, 0x80798179, - 0xd7610002, 0x0000f208, + 0x0000f206, 0x80798179, + 0xd7610002, 0x0000f207, 0x80798179, 0xd7610002, - 0x0000f209, 0x80798179, - 0xd7610002, 0x0000f20a, + 0x0000f208, 0x80798179, + 0xd7610002, 0x0000f209, 0x80798179, 0xd7610002, - 0x0000f20b, 0x80798179, - 0xe0685000, 0x701d0200, + 0x0000f20a, 0x80798179, + 0xd7610002, 0x0000f20b, + 0x80798179, 0xe0685000, + 0x701d0200, 0xbefe00c1, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00001, + 0xbeff00c1, 0xb8fb4306, + 0x8b7bc17b, 0xbfa10044, + 0xbfbd0000, 0x8b7aff6d, + 0x80000000, 0xbfa10040, + 0x847b867b, 0x847b827b, + 0xbef6007b, 0xb8f03b05, + 0x80708170, 0xbf0d9973, + 0xbfa20002, 0x84708970, + 0xbfa00001, 0x84708a70, + 0xb8fa1e06, 0x847a8a7a, + 0x80707a70, 0x8070ff70, + 0x00000200, 0x8070ff70, + 0x00000080, 0xbef600ff, + 0x01000000, 0xd71f0000, + 0x000100c1, 0xd7200000, + 0x000200c1, 0x16000084, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbefd0080, + 0xbfa20012, 0xbe8300ff, + 0x00000080, 0xbf800000, + 0xbf800000, 0xbf800000, + 0xd8d80000, 0x01000000, + 0xbf890000, 0xe0685000, + 0x701d0100, 0x807d037d, + 0x80700370, 0xd5250000, + 0x0001ff00, 0x00000080, + 0xbf0a7b7d, 0xbfa2fff4, + 0xbfa00011, 0xbe8300ff, + 0x00000100, 0xbf800000, + 0xbf800000, 0xbf800000, + 0xd8d80000, 0x01000000, + 0xbf890000, 0xe0685000, + 0x701d0100, 0x807d037d, + 0x80700370, 0xd5250000, + 0x0001ff00, 0x00000100, + 0xbf0a7b7d, 0xbfa2fff4, 0xbefe00c1, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbfa20002, 0xbeff0080, - 0xbfa00001, 0xbeff00c1, - 0xb8fb4306, 0x8b7bc17b, - 0xbfa10044, 0xbfbd0000, - 0x8b7aff6d, 0x80000000, - 0xbfa10040, 0x847b867b, - 0x847b827b, 0xbef6007b, - 0xb8f03b05, 0x80708170, - 0xbf0d9973, 0xbfa20002, - 0x84708970, 0xbfa00001, - 0x84708a70, 0xb8fa1e06, - 0x847a8a7a, 0x80707a70, - 0x8070ff70, 0x00000200, - 0x8070ff70, 0x00000080, - 0xbef600ff, 0x01000000, - 0xd71f0000, 0x000100c1, - 0xd7200000, 0x000200c1, - 0x16000084, 0x857d9973, + 0xbfa20004, 0xbef000ff, + 0x00000200, 0xbeff0080, + 0xbfa00003, 0xbef000ff, + 0x00000400, 0xbeff00c1, + 0xb8fb3b05, 0x807b817b, + 0x847b827b, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbefd0080, 0xbfa20012, - 0xbe8300ff, 0x00000080, - 0xbf800000, 0xbf800000, - 0xbf800000, 0xd8d80000, - 0x01000000, 0xbf890000, - 0xe0685000, 0x701d0100, - 0x807d037d, 0x80700370, - 0xd5250000, 0x0001ff00, - 0x00000080, 0xbf0a7b7d, - 0xbfa2fff4, 0xbfa00011, - 0xbe8300ff, 0x00000100, - 0xbf800000, 0xbf800000, - 0xbf800000, 0xd8d80000, - 0x01000000, 0xbf890000, - 0xe0685000, 0x701d0100, - 0x807d037d, 0x80700370, - 0xd5250000, 0x0001ff00, - 0x00000100, 0xbf0a7b7d, - 0xbfa2fff4, 0xbefe00c1, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20004, - 0xbef000ff, 0x00000200, - 0xbeff0080, 0xbfa00003, - 0xbef000ff, 0x00000400, - 0xbeff00c1, 0xb8fb3b05, - 0x807b817b, 0x847b827b, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20017, + 0xbfa20017, 0xbef600ff, + 0x01000000, 0xbefd0084, + 0xbf0a7b7d, 0xbfa10037, + 0x7e008700, 0x7e028701, + 0x7e048702, 0x7e068703, + 0xe0685000, 0x701d0000, + 0xe0685080, 0x701d0100, + 0xe0685100, 0x701d0200, + 0xe0685180, 0x701d0300, + 0x807d847d, 0x8070ff70, + 0x00000200, 0xbf0a7b7d, + 0xbfa2ffef, 0xbfa00025, 0xbef600ff, 0x01000000, 0xbefd0084, 0xbf0a7b7d, - 0xbfa10037, 0x7e008700, + 0xbfa10011, 0x7e008700, 0x7e028701, 0x7e048702, 0x7e068703, 0xe0685000, - 0x701d0000, 0xe0685080, - 0x701d0100, 0xe0685100, - 0x701d0200, 0xe0685180, + 0x701d0000, 0xe0685100, + 0x701d0100, 0xe0685200, + 0x701d0200, 0xe0685300, 0x701d0300, 0x807d847d, - 0x8070ff70, 0x00000200, + 0x8070ff70, 0x00000400, 0xbf0a7b7d, 0xbfa2ffef, - 0xbfa00025, 0xbef600ff, - 0x01000000, 0xbefd0084, - 0xbf0a7b7d, 0xbfa10011, - 0x7e008700, 0x7e028701, - 0x7e048702, 0x7e068703, + 0xb8fb1e06, 0x8b7bc17b, + 0xbfa1000c, 0x847b837b, + 0x807b7d7b, 0xbefe00c1, + 0xbeff0080, 0x7e008700, 0xe0685000, 0x701d0000, - 0xe0685100, 0x701d0100, - 0xe0685200, 0x701d0200, - 0xe0685300, 0x701d0300, - 0x807d847d, 0x8070ff70, - 0x00000400, 0xbf0a7b7d, - 0xbfa2ffef, 0xb8fb1e06, - 0x8b7bc17b, 0xbfa1000c, - 0x847b837b, 0x807b7d7b, - 0xbefe00c1, 0xbeff0080, - 0x7e008700, 0xe0685000, - 0x701d0000, 0x807d817d, - 0x8070ff70, 0x00000080, - 0xbf0a7b7d, 0xbfa2fff8, - 0xbfa00141, 0xbef4007e, - 0x8b75ff7f, 0x0000ffff, - 0x8c75ff75, 0x00040000, - 0xbef60080, 0xbef700ff, - 0x10807fac, 0xb8f202dc, - 0x84729972, 0x8b6eff7f, - 0x04000000, 0xbfa1003a, + 0x807d817d, 0x8070ff70, + 0x00000080, 0xbf0a7b7d, + 0xbfa2fff8, 0xbfa00146, + 0xbef4007e, 0x8b75ff7f, + 0x0000ffff, 0x8c75ff75, + 0x00040000, 0xbef60080, + 0xbef700ff, 0x10807fac, + 0xb8f202dc, 0x84729972, + 0x8b6eff7f, 0x04000000, + 0xbfa1003a, 0xbefe00c1, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00001, + 0xbeff00c1, 0xb8ef4306, + 0x8b6fc16f, 0xbfa1002f, + 0x846f866f, 0x846f826f, + 0xbef6006f, 0xb8f83b05, + 0x80788178, 0xbf0d9972, + 0xbfa20002, 0x84788978, + 0xbfa00001, 0x84788a78, + 0xb8ee1e06, 0x846e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0x8078ff78, + 0x00000080, 0xbef600ff, + 0x01000000, 0x857d9972, + 0x8b7d817d, 0xbf06817d, + 0xbefd0080, 0xbfa2000c, + 0xe0500000, 0x781d0000, + 0xbf8903f7, 0xdac00000, + 0x00000000, 0x807dff7d, + 0x00000080, 0x8078ff78, + 0x00000080, 0xbf0a6f7d, + 0xbfa2fff5, 0xbfa0000b, + 0xe0500000, 0x781d0000, + 0xbf8903f7, 0xdac00000, + 0x00000000, 0x807dff7d, + 0x00000100, 0x8078ff78, + 0x00000100, 0xbf0a6f7d, + 0xbfa2fff5, 0xbef80080, 0xbefe00c1, 0x857d9972, 0x8b7d817d, 0xbf06817d, 0xbfa20002, 0xbeff0080, 0xbfa00001, 0xbeff00c1, - 0xb8ef4306, 0x8b6fc16f, - 0xbfa1002f, 0x846f866f, - 0x846f826f, 0xbef6006f, - 0xb8f83b05, 0x80788178, - 0xbf0d9972, 0xbfa20002, - 0x84788978, 0xbfa00001, - 0x84788a78, 0xb8ee1e06, - 0x846e8a6e, 0x80786e78, + 0xb8ef3b05, 0x806f816f, + 0x846f826f, 0x857d9972, + 0x8b7d817d, 0xbf06817d, + 0xbfa20024, 0xbef600ff, + 0x01000000, 0xbeee0078, 0x8078ff78, 0x00000200, - 0x8078ff78, 0x00000080, - 0xbef600ff, 0x01000000, - 0x857d9972, 0x8b7d817d, - 0xbf06817d, 0xbefd0080, - 0xbfa2000c, 0xe0500000, - 0x781d0000, 0xbf8903f7, - 0xdac00000, 0x00000000, - 0x807dff7d, 0x00000080, - 0x8078ff78, 0x00000080, - 0xbf0a6f7d, 0xbfa2fff5, - 0xbfa0000b, 0xe0500000, - 0x781d0000, 0xbf8903f7, - 0xdac00000, 0x00000000, - 0x807dff7d, 0x00000100, - 0x8078ff78, 0x00000100, - 0xbf0a6f7d, 0xbfa2fff5, - 0xbef80080, 0xbefe00c1, - 0x857d9972, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00001, - 0xbeff00c1, 0xb8ef3b05, - 0x806f816f, 0x846f826f, - 0x857d9972, 0x8b7d817d, - 0xbf06817d, 0xbfa20024, - 0xbef600ff, 0x01000000, - 0xbeee0078, 0x8078ff78, - 0x00000200, 0xbefd0084, - 0xbf0a6f7d, 0xbfa10050, + 0xbefd0084, 0xbf0a6f7d, + 0xbfa10050, 0xe0505000, + 0x781d0000, 0xe0505080, + 0x781d0100, 0xe0505100, + 0x781d0200, 0xe0505180, + 0x781d0300, 0xbf8903f7, + 0x7e008500, 0x7e028501, + 0x7e048502, 0x7e068503, + 0x807d847d, 0x8078ff78, + 0x00000200, 0xbf0a6f7d, + 0xbfa2ffee, 0xe0505000, + 0x6e1d0000, 0xe0505080, + 0x6e1d0100, 0xe0505100, + 0x6e1d0200, 0xe0505180, + 0x6e1d0300, 0xbf8903f7, + 0xbfa00034, 0xbef600ff, + 0x01000000, 0xbeee0078, + 0x8078ff78, 0x00000400, + 0xbefd0084, 0xbf0a6f7d, + 0xbfa10012, 0xe0505000, + 0x781d0000, 0xe0505100, + 0x781d0100, 0xe0505200, + 0x781d0200, 0xe0505300, + 0x781d0300, 0xbf8903f7, + 0x7e008500, 0x7e028501, + 0x7e048502, 0x7e068503, + 0x807d847d, 0x8078ff78, + 0x00000400, 0xbf0a6f7d, + 0xbfa2ffee, 0xb8ef1e06, + 0x8b6fc16f, 0xbfa1000e, + 0x846f836f, 0x806f7d6f, + 0xbefe00c1, 0xbeff0080, 0xe0505000, 0x781d0000, - 0xe0505080, 0x781d0100, - 0xe0505100, 0x781d0200, - 0xe0505180, 0x781d0300, 0xbf8903f7, 0x7e008500, - 0x7e028501, 0x7e048502, - 0x7e068503, 0x807d847d, - 0x8078ff78, 0x00000200, - 0xbf0a6f7d, 0xbfa2ffee, + 0x807d817d, 0x8078ff78, + 0x00000080, 0xbf0a6f7d, + 0xbfa2fff7, 0xbeff00c1, 0xe0505000, 0x6e1d0000, - 0xe0505080, 0x6e1d0100, - 0xe0505100, 0x6e1d0200, - 0xe0505180, 0x6e1d0300, - 0xbf8903f7, 0xbfa00034, - 0xbef600ff, 0x01000000, - 0xbeee0078, 0x8078ff78, - 0x00000400, 0xbefd0084, - 0xbf0a6f7d, 0xbfa10012, - 0xe0505000, 0x781d0000, - 0xe0505100, 0x781d0100, - 0xe0505200, 0x781d0200, - 0xe0505300, 0x781d0300, - 0xbf8903f7, 0x7e008500, - 0x7e028501, 0x7e048502, - 0x7e068503, 0x807d847d, - 0x8078ff78, 0x00000400, - 0xbf0a6f7d, 0xbfa2ffee, - 0xb8ef1e06, 0x8b6fc16f, - 0xbfa1000e, 0x846f836f, - 0x806f7d6f, 0xbefe00c1, - 0xbeff0080, 0xe0505000, - 0x781d0000, 0xbf8903f7, - 0x7e008500, 0x807d817d, - 0x8078ff78, 0x00000080, - 0xbf0a6f7d, 0xbfa2fff7, - 0xbeff00c1, 0xe0505000, - 0x6e1d0000, 0xe0505100, - 0x6e1d0100, 0xe0505200, - 0x6e1d0200, 0xe0505300, - 0x6e1d0300, 0xbf8903f7, + 0xe0505100, 0x6e1d0100, + 0xe0505200, 0x6e1d0200, + 0xe0505300, 0x6e1d0300, + 0xbf8903f7, 0xb8f83b05, + 0x80788178, 0xbf0d9972, + 0xbfa20002, 0x84788978, + 0xbfa00001, 0x84788a78, + 0xb8ee1e06, 0x846e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0x80f8ff78, + 0x00000050, 0xbef600ff, + 0x01000000, 0xbefd00ff, + 0x0000006c, 0x80f89078, + 0xf428403a, 0xf0000000, + 0xbf89fc07, 0x80fd847d, + 0xbf800000, 0xbe804300, + 0xbe824302, 0x80f8a078, + 0xf42c403a, 0xf0000000, + 0xbf89fc07, 0x80fd887d, + 0xbf800000, 0xbe804300, + 0xbe824302, 0xbe844304, + 0xbe864306, 0x80f8c078, + 0xf430403a, 0xf0000000, + 0xbf89fc07, 0x80fd907d, + 0xbf800000, 0xbe804300, + 0xbe824302, 0xbe844304, + 0xbe864306, 0xbe884308, + 0xbe8a430a, 0xbe8c430c, + 0xbe8e430e, 0xbf06807d, + 0xbfa1fff0, 0xb980f801, + 0x00000000, 0xbfbd0000, 0xb8f83b05, 0x80788178, 0xbf0d9972, 0xbfa20002, 0x84788978, 0xbfa00001, 0x84788a78, 0xb8ee1e06, 0x846e8a6e, 0x80786e78, 0x8078ff78, 0x00000200, - 0x80f8ff78, 0x00000050, 0xbef600ff, 0x01000000, - 0xbefd00ff, 0x0000006c, - 0x80f89078, 0xf428403a, - 0xf0000000, 0xbf89fc07, - 0x80fd847d, 0xbf800000, - 0xbe804300, 0xbe824302, - 0x80f8a078, 0xf42c403a, - 0xf0000000, 0xbf89fc07, - 0x80fd887d, 0xbf800000, - 0xbe804300, 0xbe824302, - 0xbe844304, 0xbe864306, - 0x80f8c078, 0xf430403a, - 0xf0000000, 0xbf89fc07, - 0x80fd907d, 0xbf800000, - 0xbe804300, 0xbe824302, - 0xbe844304, 0xbe864306, - 0xbe884308, 0xbe8a430a, - 0xbe8c430c, 0xbe8e430e, - 0xbf06807d, 0xbfa1fff0, - 0xb980f801, 0x00000000, - 0xbfbd0000, 0xb8f83b05, - 0x80788178, 0xbf0d9972, - 0xbfa20002, 0x84788978, - 0xbfa00001, 0x84788a78, - 0xb8ee1e06, 0x846e8a6e, - 0x80786e78, 0x8078ff78, - 0x00000200, 0xbef600ff, - 0x01000000, 0xf4205bfa, + 0xf4205bfa, 0xf0000000, + 0x80788478, 0xf4205b3a, 0xf0000000, 0x80788478, - 0xf4205b3a, 0xf0000000, - 0x80788478, 0xf4205b7a, + 0xf4205b7a, 0xf0000000, + 0x80788478, 0xf4205c3a, 0xf0000000, 0x80788478, - 0xf4205c3a, 0xf0000000, - 0x80788478, 0xf4205c7a, + 0xf4205c7a, 0xf0000000, + 0x80788478, 0xf4205eba, 0xf0000000, 0x80788478, - 0xf4205eba, 0xf0000000, - 0x80788478, 0xf4205efa, + 0xf4205efa, 0xf0000000, + 0x80788478, 0xf4205e7a, 0xf0000000, 0x80788478, - 0xf4205e7a, 0xf0000000, - 0x80788478, 0xf4205cfa, + 0xf4205cfa, 0xf0000000, + 0x80788478, 0xf4205bba, 0xf0000000, 0x80788478, + 0xbf89fc07, 0xb96ef814, 0xf4205bba, 0xf0000000, 0x80788478, 0xbf89fc07, - 0xb96ef814, 0xf4205bba, - 0xf0000000, 0x80788478, - 0xbf89fc07, 0xb96ef815, - 0xbefd006f, 0xbefe0070, - 0xbeff0071, 0x8b6f7bff, - 0x000003ff, 0xb96f4803, - 0x8b6f7bff, 0xfffff800, - 0x856f8b6f, 0xb96fa2c3, - 0xb973f801, 0xb8ee3b05, - 0x806e816e, 0xbf0d9972, - 0xbfa20002, 0x846e896e, - 0xbfa00001, 0x846e8a6e, - 0xb8ef1e06, 0x846f8a6f, - 0x806e6f6e, 0x806eff6e, - 0x00000200, 0x806e746e, - 0x826f8075, 0x8b6fff6f, - 0x0000ffff, 0xf4085c37, - 0xf8000050, 0xf4085d37, - 0xf8000060, 0xf4005e77, - 0xf8000074, 0xbf89fc07, - 0x8b6dff6d, 0x0000ffff, - 0x8bfe7e7e, 0x8bea6a6a, + 0xb96ef815, 0xbefd006f, + 0xbefe0070, 0xbeff0071, + 0x8b6f7bff, 0x000003ff, + 0xb96f4803, 0x8b6f7bff, + 0xfffff800, 0x856f8b6f, + 0xb96fa2c3, 0xb973f801, + 0xb8ee3b05, 0x806e816e, + 0xbf0d9972, 0xbfa20002, + 0x846e896e, 0xbfa00001, + 0x846e8a6e, 0xb8ef1e06, + 0x846f8a6f, 0x806e6f6e, + 0x806eff6e, 0x00000200, + 0x806e746e, 0x826f8075, + 0x8b6fff6f, 0x0000ffff, + 0xf4085c37, 0xf8000050, + 0xf4085d37, 0xf8000060, + 0xf4005e77, 0xf8000074, + 0xbf89fc07, 0x8b6dff6d, + 0x0000ffff, 0x8bfe7e7e, + 0x8bea6a6a, 0xb8eef802, + 0xbf0d866e, 0xbfa20002, + 0xb97af802, 0xbe80486c, 0xb97af802, 0xbe804a6c, 0xbfb00000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm index 250ab007399bd7402413812c2354650d2e09c366..0f81670f6f9c6b9f11bfc880d8184565a5e5e04d 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm @@ -43,12 +43,14 @@ #define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID) #define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO) #define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO) +#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO) var SINGLE_STEP_MISSED_WORKAROUND = 1 //workaround for lost MODE.DEBUG_EN exception when SAVECTX raised var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006 var SQ_WAVE_STATUS_HALT_MASK = 0x2000 var SQ_WAVE_STATUS_ECC_ERR_MASK = 0x20000 +var SQ_WAVE_STATUS_TRAP_EN_SHIFT = 6 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9 @@ -183,6 +185,13 @@ L_SKIP_RESTORE: s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS) +#if SW_SA_TRAP + // If ttmp1[31] is set then trap may occur early. + // Spin wait until SAVECTX exception is raised. + s_bitcmp1_b32 s_save_pc_hi, 31 + s_cbranch_scc1 L_CHECK_SAVE +#endif + s_and_b32 ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK s_cbranch_scc0 L_NOT_HALTED @@ -1061,8 +1070,20 @@ L_RESTORE_HWREG: s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 + +#if SW_SA_TRAP + // If traps are enabled then return to the shader with PRIV=0. + // Otherwise retain PRIV=1 for subsequent context save requests. + s_getreg_b32 s_restore_tmp, hwreg(HW_REG_STATUS) + s_bitcmp1_b32 s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT + s_cbranch_scc1 L_RETURN_WITHOUT_PRIV + s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu + s_setpc_b64 [s_restore_pc_lo, s_restore_pc_hi] +L_RETURN_WITHOUT_PRIV: +#endif + s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu s_rfe_b64 s_restore_pc_lo //Return to the main shader program and resume execution L_END_PGM: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 84da1a9ce37cdec7646d7fed69559400f4f74b2b..5feaba6a77de48a1e521d4488b954d5a9c174e4f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1584,6 +1584,8 @@ static int kfd_ioctl_smi_events(struct file *filep, return kfd_smi_event_open(pdd->dev, &args->anon_fd); } +#if IS_ENABLED(CONFIG_HSA_AMD_SVM) + static int kfd_ioctl_set_xnack_mode(struct file *filep, struct kfd_process *p, void *data) { @@ -1594,22 +1596,29 @@ static int kfd_ioctl_set_xnack_mode(struct file *filep, if (args->xnack_enabled >= 0) { if (!list_empty(&p->pqm.queues)) { pr_debug("Process has user queues running\n"); - mutex_unlock(&p->mutex); - return -EBUSY; + r = -EBUSY; + goto out_unlock; } - if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) + + if (p->xnack_enabled == args->xnack_enabled) + goto out_unlock; + + if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) { r = -EPERM; - else - p->xnack_enabled = args->xnack_enabled; + goto out_unlock; + } + + r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled); } else { args->xnack_enabled = p->xnack_enabled; } + +out_unlock: mutex_unlock(&p->mutex); return r; } -#if IS_ENABLED(CONFIG_HSA_AMD_SVM) static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) { struct kfd_ioctl_svm_args *args = data; @@ -1629,6 +1638,11 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) return r; } #else +static int kfd_ioctl_set_xnack_mode(struct file *filep, + struct kfd_process *p, void *data) +{ + return -EPERM; +} static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data) { return -EPERM; @@ -2153,6 +2167,12 @@ static int criu_restore_devices(struct kfd_process *p, ret = PTR_ERR(pdd); goto exit; } + + if (!pdd->doorbell_index && + kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) { + ret = -ENOMEM; + goto exit; + } } /* diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 24b414cff3ec0a99ead0e50be09b8e0804d6f041..cd5f8b219bf94668e7edbe2d9ccc78515be8be35 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -2284,7 +2284,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image, /* Fill in Subtype: IO_LINKS * Only direct links are added here which is Link from GPU to - * to its NUMA node. Indirect links are added by userspace. + * its NUMA node. Indirect links are added by userspace. */ sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + cache_mem_filled); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index e83725a281068b74ffa7d5f5870826329ce7e8e3..ecb4c3abc629739e18a7940761e146129dc654cf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -205,6 +205,8 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, } queue_input.is_kfd_process = 1; + queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL); + queue_input.queue_size = q->properties.queue_size >> 2; queue_input.paging = false; queue_input.tba_addr = qpd->tba_addr; @@ -1240,6 +1242,24 @@ static void init_interrupts(struct device_queue_manager *dqm) dqm->dev->kfd2kgd->init_interrupts(dqm->dev->adev, i); } +static void init_sdma_bitmaps(struct device_queue_manager *dqm) +{ + unsigned int num_sdma_queues = + min_t(unsigned int, sizeof(dqm->sdma_bitmap)*8, + get_num_sdma_queues(dqm)); + unsigned int num_xgmi_sdma_queues = + min_t(unsigned int, sizeof(dqm->xgmi_sdma_bitmap)*8, + get_num_xgmi_sdma_queues(dqm)); + + if (num_sdma_queues) + dqm->sdma_bitmap = GENMASK_ULL(num_sdma_queues-1, 0); + if (num_xgmi_sdma_queues) + dqm->xgmi_sdma_bitmap = GENMASK_ULL(num_xgmi_sdma_queues-1, 0); + + dqm->sdma_bitmap &= ~get_reserved_sdma_queues_bitmap(dqm); + pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap); +} + static int initialize_nocpsch(struct device_queue_manager *dqm) { int pipe, queue; @@ -1268,11 +1288,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm) memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid)); - dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm)); - dqm->sdma_bitmap &= ~(get_reserved_sdma_queues_bitmap(dqm)); - pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap); - - dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm)); + init_sdma_bitmaps(dqm); return 0; } @@ -1450,9 +1466,6 @@ static int set_sched_resources(struct device_queue_manager *dqm) static int initialize_cpsch(struct device_queue_manager *dqm) { - uint64_t num_sdma_queues; - uint64_t num_xgmi_sdma_queues; - pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm)); mutex_init(&dqm->lock_hidden); @@ -1461,24 +1474,10 @@ static int initialize_cpsch(struct device_queue_manager *dqm) dqm->active_cp_queue_count = 0; dqm->gws_queue_count = 0; dqm->active_runlist = false; - - num_sdma_queues = get_num_sdma_queues(dqm); - if (num_sdma_queues >= BITS_PER_TYPE(dqm->sdma_bitmap)) - dqm->sdma_bitmap = ULLONG_MAX; - else - dqm->sdma_bitmap = (BIT_ULL(num_sdma_queues) - 1); - - dqm->sdma_bitmap &= ~(get_reserved_sdma_queues_bitmap(dqm)); - pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap); - - num_xgmi_sdma_queues = get_num_xgmi_sdma_queues(dqm); - if (num_xgmi_sdma_queues >= BITS_PER_TYPE(dqm->xgmi_sdma_bitmap)) - dqm->xgmi_sdma_bitmap = ULLONG_MAX; - else - dqm->xgmi_sdma_bitmap = (BIT_ULL(num_xgmi_sdma_queues) - 1); - INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); + init_sdma_bitmaps(dqm); + return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index b33798f89ef09517ab110657f275e9a22f6bb74f..cd4e61bf04939f93fd6387580db8f53961460b82 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -303,6 +303,9 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, unsigned int *doorbell_inde if (r > 0) *doorbell_index = r; + if (r < 0) + pr_err("Failed to allocate process doorbells\n"); + return r; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c index a6fcbeeb7428597c19bc88740daf4a30e7fbad7d..0d53f60674226b6abe0b1ec4d1377051a60e91ff 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c @@ -350,11 +350,11 @@ static void event_interrupt_wq_v11(struct kfd_dev *dev, print_sq_intr_info_inst(context_id0, context_id1); sq_int_priv = REG_GET_FIELD(context_id0, SQ_INTERRUPT_WORD_WAVE_CTXID0, PRIV); - if (sq_int_priv /*&& (kfd_set_dbg_ev_from_interrupt(dev, pasid, + /*if (sq_int_priv && (kfd_set_dbg_ev_from_interrupt(dev, pasid, KFD_CTXID0_DOORBELL_ID(context_id0), KFD_CTXID0_TRAP_CODE(context_id0), - NULL, 0))*/) - return; + NULL, 0))) + return;*/ break; case SQ_INTERRUPT_WORD_ENCODING_ERROR: print_sq_intr_info_error(context_id0, context_id1); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index b059a77b6081d8c003127420bb08f72238e410f1..2797029bd50015f421a1ff74f03ddf37564ccc8f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -223,7 +223,7 @@ svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn) page = pfn_to_page(pfn); svm_range_bo_ref(prange->svm_bo); page->zone_device_data = prange->svm_bo; - lock_page(page); + zone_device_page_init(page); } static void @@ -322,12 +322,13 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange, for (i = j = 0; i < npages; i++) { struct page *spage; + dst[i] = cursor.start + (j << PAGE_SHIFT); + migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); + svm_migrate_get_vram_page(prange, migrate->dst[i]); + migrate->dst[i] = migrate_pfn(migrate->dst[i]); + spage = migrate_pfn_to_page(migrate->src[i]); if (spage && !is_zone_device_page(spage)) { - dst[i] = cursor.start + (j << PAGE_SHIFT); - migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]); - svm_migrate_get_vram_page(prange, migrate->dst[i]); - migrate->dst[i] = migrate_pfn(migrate->dst[i]); src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE, DMA_TO_DEVICE); r = dma_mapping_error(dev, src[i]); @@ -409,7 +410,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange, uint64_t npages = (end - start) >> PAGE_SHIFT; struct kfd_process_device *pdd; struct dma_fence *mfence = NULL; - struct migrate_vma migrate; + struct migrate_vma migrate = { 0 }; unsigned long cpages = 0; dma_addr_t *scratch; void *buf; @@ -522,9 +523,6 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc, pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms, prange->start, prange->last, best_loc); - /* FIXME: workaround for page locking bug with invalid pages */ - svm_range_prefault(prange, mm, SVM_ADEV_PGMAP_OWNER(adev)); - start = prange->start << PAGE_SHIFT; end = (prange->last + 1) << PAGE_SHIFT; @@ -668,7 +666,7 @@ out_oom: static long svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, struct vm_area_struct *vma, uint64_t start, uint64_t end, - uint32_t trigger) + uint32_t trigger, struct page *fault_page) { struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); uint64_t npages = (end - start) >> PAGE_SHIFT; @@ -676,7 +674,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, unsigned long cpages = 0; struct kfd_process_device *pdd; struct dma_fence *mfence = NULL; - struct migrate_vma migrate; + struct migrate_vma migrate = { 0 }; dma_addr_t *scratch; void *buf; int r = -ENOMEM; @@ -699,6 +697,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange, migrate.src = buf; migrate.dst = migrate.src + npages; + migrate.fault_page = fault_page; scratch = (dma_addr_t *)(migrate.dst + npages); kfd_smi_event_migration_start(adev->kfd.dev, p->lead_thread->pid, @@ -766,7 +765,7 @@ out: * 0 - OK, otherwise error code */ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm, - uint32_t trigger) + uint32_t trigger, struct page *fault_page) { struct amdgpu_device *adev; struct vm_area_struct *vma; @@ -807,7 +806,8 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm, } next = min(vma->vm_end, end); - r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger); + r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next, trigger, + fault_page); if (r < 0) { pr_debug("failed %ld to migrate prange %p\n", r, prange); break; @@ -851,7 +851,7 @@ svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc, pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc); do { - r = svm_migrate_vram_to_ram(prange, mm, trigger); + r = svm_migrate_vram_to_ram(prange, mm, trigger, NULL); if (r) return r; } while (prange->actual_loc && --retries); @@ -886,7 +886,7 @@ svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc, static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) { unsigned long addr = vmf->address; - struct vm_area_struct *vma; + struct svm_range_bo *svm_bo; enum svm_work_list_ops op; struct svm_range *parent; struct svm_range *prange; @@ -894,29 +894,42 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) struct mm_struct *mm; int r = 0; - vma = vmf->vma; - mm = vma->vm_mm; + svm_bo = vmf->page->zone_device_data; + if (!svm_bo) { + pr_debug("failed get device page at addr 0x%lx\n", addr); + return VM_FAULT_SIGBUS; + } + if (!mmget_not_zero(svm_bo->eviction_fence->mm)) { + pr_debug("addr 0x%lx of process mm is destroyed\n", addr); + return VM_FAULT_SIGBUS; + } + + mm = svm_bo->eviction_fence->mm; + if (mm != vmf->vma->vm_mm) + pr_debug("addr 0x%lx is COW mapping in child process\n", addr); - p = kfd_lookup_process_by_mm(vma->vm_mm); + p = kfd_lookup_process_by_mm(mm); if (!p) { pr_debug("failed find process at fault address 0x%lx\n", addr); - return VM_FAULT_SIGBUS; + r = VM_FAULT_SIGBUS; + goto out_mmput; } if (READ_ONCE(p->svms.faulting_task) == current) { pr_debug("skipping ram migration\n"); - kfd_unref_process(p); - return 0; + r = 0; + goto out_unref_process; } - addr >>= PAGE_SHIFT; + pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr); + addr >>= PAGE_SHIFT; mutex_lock(&p->svms.lock); prange = svm_range_from_addr(&p->svms, addr, &parent); if (!prange) { - pr_debug("cannot find svm range at 0x%lx\n", addr); + pr_debug("failed get range svms 0x%p addr 0x%lx\n", &p->svms, addr); r = -EFAULT; - goto out; + goto out_unlock_svms; } mutex_lock(&parent->migrate_mutex); @@ -938,10 +951,12 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) goto out_unlock_prange; } - r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU); + r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm, + KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU, + vmf->page); if (r) - pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r, - prange, prange->start, prange->last); + pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n", + r, prange->svms, prange, prange->start, prange->last); /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */ if (p->xnack_enabled && parent == prange) @@ -955,9 +970,12 @@ out_unlock_prange: if (prange != parent) mutex_unlock(&prange->migrate_mutex); mutex_unlock(&parent->migrate_mutex); -out: +out_unlock_svms: mutex_unlock(&p->svms.lock); +out_unref_process: kfd_unref_process(p); +out_mmput: + mmput(mm); pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h index b3f0754b32faa9a0682cf21449225bf13185d242..a5d7e6d2226469a4684cea39e9158836b29c0d71 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.h @@ -43,7 +43,7 @@ enum MIGRATION_COPY_DIR { int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc, struct mm_struct *mm, uint32_t trigger); int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm, - uint32_t trigger); + uint32_t trigger, struct page *fault_page); unsigned long svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c index b8e14c2cc2956e7c867e3e81245067175cecf491..4f6390f3236ef17100a856ea8163c4f4a02bb3f2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c @@ -126,6 +126,10 @@ static void init_mqd(struct mqd_manager *mm, void **mqd, m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF; m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | 0x55 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; @@ -177,14 +181,6 @@ static int load_mqd(struct mqd_manager *mm, void *mqd, return r; } -static int hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd, - uint32_t pipe_id, uint32_t queue_id, - struct queue_properties *p, struct mm_struct *mms) -{ - return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, mqd, pipe_id, - queue_id, p->doorbell_off); -} - static void update_mqd(struct mqd_manager *mm, void *mqd, struct queue_properties *q, struct mqd_update_info *minfo) @@ -256,31 +252,6 @@ static uint32_t read_doorbell_id(void *mqd) return m->queue_doorbell_id0; } -static int destroy_mqd(struct mqd_manager *mm, void *mqd, - enum kfd_preempt_type type, - unsigned int timeout, uint32_t pipe_id, - uint32_t queue_id) -{ - return mm->dev->kfd2kgd->hqd_destroy - (mm->dev->adev, mqd, type, timeout, - pipe_id, queue_id); -} - -static void free_mqd(struct mqd_manager *mm, void *mqd, - struct kfd_mem_obj *mqd_mem_obj) -{ - kfd_gtt_sa_free(mm->dev, mqd_mem_obj); -} - -static bool is_occupied(struct mqd_manager *mm, void *mqd, - uint64_t queue_address, uint32_t pipe_id, - uint32_t queue_id) -{ - return mm->dev->kfd2kgd->hqd_is_occupied( - mm->dev->adev, queue_address, - pipe_id, queue_id); -} - static int get_wave_state(struct mqd_manager *mm, void *mqd, void __user *ctl_stack, u32 *ctl_stack_used_size, @@ -349,15 +320,6 @@ static void init_mqd_sdma(struct mqd_manager *mm, void **mqd, mm->update_mqd(mm, m, q, NULL); } -static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, - uint32_t pipe_id, uint32_t queue_id, - struct queue_properties *p, struct mm_struct *mms) -{ - return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd, - (uint32_t __user *)p->write_ptr, - mms); -} - #define SDMA_RLC_DUMMY_DEFAULT 0xf static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, @@ -371,7 +333,8 @@ static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, << SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT | q->vmid << SDMA0_QUEUE0_RB_CNTL__RB_VMID__SHIFT | 1 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | - 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; + 6 << SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT | + 1 << SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT; m->sdmax_rlcx_rb_base = lower_32_bits(q->queue_address >> 8); m->sdmax_rlcx_rb_base_hi = upper_32_bits(q->queue_address >> 8); @@ -389,25 +352,6 @@ static void update_mqd_sdma(struct mqd_manager *mm, void *mqd, q->is_active = QUEUE_IS_ACTIVE(*q); } -/* - * * preempt type here is ignored because there is only one way - * * to preempt sdma queue - */ -static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, - enum kfd_preempt_type type, - unsigned int timeout, uint32_t pipe_id, - uint32_t queue_id) -{ - return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout); -} - -static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, - uint64_t queue_address, uint32_t pipe_id, - uint32_t queue_id) -{ - return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd); -} - #if defined(CONFIG_DEBUG_FS) static int debugfs_show_mqd(struct seq_file *m, void *data) @@ -445,11 +389,11 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type, pr_debug("%s@%i\n", __func__, __LINE__); mqd->allocate_mqd = allocate_mqd; mqd->init_mqd = init_mqd; - mqd->free_mqd = free_mqd; + mqd->free_mqd = kfd_free_mqd_cp; mqd->load_mqd = load_mqd; mqd->update_mqd = update_mqd; - mqd->destroy_mqd = destroy_mqd; - mqd->is_occupied = is_occupied; + mqd->destroy_mqd = kfd_destroy_mqd_cp; + mqd->is_occupied = kfd_is_occupied_cp; mqd->mqd_size = sizeof(struct v11_compute_mqd); mqd->get_wave_state = get_wave_state; #if defined(CONFIG_DEBUG_FS) @@ -462,10 +406,10 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type, mqd->allocate_mqd = allocate_hiq_mqd; mqd->init_mqd = init_mqd_hiq; mqd->free_mqd = free_mqd_hiq_sdma; - mqd->load_mqd = hiq_load_mqd_kiq; + mqd->load_mqd = kfd_hiq_load_mqd_kiq; mqd->update_mqd = update_mqd; - mqd->destroy_mqd = destroy_mqd; - mqd->is_occupied = is_occupied; + mqd->destroy_mqd = kfd_destroy_mqd_cp; + mqd->is_occupied = kfd_is_occupied_cp; mqd->mqd_size = sizeof(struct v11_compute_mqd); #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; @@ -476,11 +420,11 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type, case KFD_MQD_TYPE_DIQ: mqd->allocate_mqd = allocate_mqd; mqd->init_mqd = init_mqd_hiq; - mqd->free_mqd = free_mqd; + mqd->free_mqd = kfd_free_mqd_cp; mqd->load_mqd = load_mqd; mqd->update_mqd = update_mqd; - mqd->destroy_mqd = destroy_mqd; - mqd->is_occupied = is_occupied; + mqd->destroy_mqd = kfd_destroy_mqd_cp; + mqd->is_occupied = kfd_is_occupied_cp; mqd->mqd_size = sizeof(struct v11_compute_mqd); #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd; @@ -491,10 +435,10 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type, mqd->allocate_mqd = allocate_sdma_mqd; mqd->init_mqd = init_mqd_sdma; mqd->free_mqd = free_mqd_hiq_sdma; - mqd->load_mqd = load_mqd_sdma; + mqd->load_mqd = kfd_load_mqd_sdma; mqd->update_mqd = update_mqd_sdma; - mqd->destroy_mqd = destroy_mqd_sdma; - mqd->is_occupied = is_occupied_sdma; + mqd->destroy_mqd = kfd_destroy_mqd_sdma; + mqd->is_occupied = kfd_is_occupied_sdma; mqd->mqd_size = sizeof(struct v11_sdma_mqd); #if defined(CONFIG_DEBUG_FS) mqd->debugfs_show_mqd = debugfs_show_mqd_sdma; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 6e3e7f54381b33dadd940c7a1f3dcff3ed32a04a..5137476ec18e67d521d0ef590eb6cd8f4de94fd1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -857,6 +857,13 @@ int kfd_criu_restore_queue(struct kfd_process *p, ret = -EINVAL; goto exit; } + + if (!pdd->doorbell_index && + kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) { + ret = -ENOMEM; + goto exit; + } + /* data stored in this order: mqd, ctl_stack */ mqd = q_extra_data; ctl_stack = mqd + q_data->mqd_size; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 11074cc8c333b274484929dddb0752725e4af24b..64fdf63093a00d40ce1672bef4a20c89fd48537c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -278,7 +278,7 @@ static void svm_range_free(struct svm_range *prange, bool update_mem_usage) svm_range_free_dma_mappings(prange); if (update_mem_usage && !p->xnack_enabled) { - pr_debug("unreserve mem limit: %lld\n", size); + pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size); amdgpu_amdkfd_unreserve_mem_limit(NULL, size, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); } @@ -2913,13 +2913,15 @@ retry_write_locked: */ if (prange->actual_loc) r = svm_migrate_vram_to_ram(prange, mm, - KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU); + KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, + NULL); else r = 0; } } else { r = svm_migrate_vram_to_ram(prange, mm, - KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU); + KFD_MIGRATE_TRIGGER_PAGEFAULT_GPU, + NULL); } if (r) { pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n", @@ -2956,6 +2958,64 @@ out: return r; } +int +svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled) +{ + struct svm_range *prange, *pchild; + uint64_t reserved_size = 0; + uint64_t size; + int r = 0; + + pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled); + + mutex_lock(&p->svms.lock); + + list_for_each_entry(prange, &p->svms.list, list) { + svm_range_lock(prange); + list_for_each_entry(pchild, &prange->child_list, child_list) { + size = (pchild->last - pchild->start + 1) << PAGE_SHIFT; + if (xnack_enabled) { + amdgpu_amdkfd_unreserve_mem_limit(NULL, size, + KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + } else { + r = amdgpu_amdkfd_reserve_mem_limit(NULL, size, + KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + if (r) + goto out_unlock; + reserved_size += size; + } + } + + size = (prange->last - prange->start + 1) << PAGE_SHIFT; + if (xnack_enabled) { + amdgpu_amdkfd_unreserve_mem_limit(NULL, size, + KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + } else { + r = amdgpu_amdkfd_reserve_mem_limit(NULL, size, + KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + if (r) + goto out_unlock; + reserved_size += size; + } +out_unlock: + svm_range_unlock(prange); + if (r) + break; + } + + if (r) + amdgpu_amdkfd_unreserve_mem_limit(NULL, reserved_size, + KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + else + /* Change xnack mode must be inside svms lock, to avoid race with + * svm_range_deferred_list_work unreserve memory in parallel. + */ + p->xnack_enabled = xnack_enabled; + + mutex_unlock(&p->svms.lock); + return r; +} + void svm_range_list_fini(struct kfd_process *p) { struct svm_range *prange; @@ -3181,28 +3241,6 @@ out: return best_loc; } -/* FIXME: This is a workaround for page locking bug when some pages are - * invalid during migration to VRAM - */ -void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm, - void *owner) -{ - struct hmm_range *hmm_range; - int r; - - if (prange->validated_once) - return; - - r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL, - prange->start << PAGE_SHIFT, - prange->npages, &hmm_range, - false, true, owner); - if (!r) { - amdgpu_hmm_range_get_pages_done(hmm_range); - prange->validated_once = true; - } -} - /* svm_range_trigger_migration - start page migration if prefetch loc changed * @mm: current process mm_struct * @prange: svm range structure @@ -3242,7 +3280,8 @@ svm_range_trigger_migration(struct mm_struct *mm, struct svm_range *prange, return 0; if (!best_loc) { - r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PREFETCH); + r = svm_migrate_vram_to_ram(prange, mm, + KFD_MIGRATE_TRIGGER_PREFETCH, NULL); *migrated = !r; return r; } @@ -3303,7 +3342,7 @@ static void svm_range_evict_svm_bo_worker(struct work_struct *work) mutex_lock(&prange->migrate_mutex); do { r = svm_migrate_vram_to_ram(prange, mm, - KFD_MIGRATE_TRIGGER_TTM_EVICTION); + KFD_MIGRATE_TRIGGER_TTM_EVICTION, NULL); } while (!r && prange->actual_loc && --retries); if (!r && prange->actual_loc) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h index cfac13ad06ef0f70e2983bceeb85b59c0427e2fb..7a33b93f9df6fbb80a7628946f678bc48368f1a2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h @@ -181,8 +181,6 @@ void schedule_deferred_list_work(struct svm_range_list *svms); void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr, unsigned long offset, unsigned long npages); void svm_range_free_dma_mappings(struct svm_range *prange); -void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm, - void *owner); int svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges, uint64_t *svm_priv_data_size); int kfd_criu_checkpoint_svm(struct kfd_process *p, @@ -205,6 +203,7 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s void svm_range_bo_unref_async(struct svm_range_bo *svm_bo); void svm_range_set_max_pages(struct amdgpu_device *adev); +int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled); #else diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 413d8c6d592ff8f3674a293e6a5f83075251ff0c..6925e0280dbe62110616321145f8363d89926d06 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -28,7 +28,6 @@ config DRM_AMD_DC_SI bool "AMD DC support for Southern Islands ASICs" depends on DRM_AMDGPU_SI depends on DRM_AMD_DC - default n help Choose this option to enable new AMD DC support for SI asics by default. This includes Tahiti, Pitcairn, Cape Verde, Oland. @@ -43,7 +42,6 @@ config DEBUG_KERNEL_DC config DRM_AMD_SECURE_DISPLAY bool "Enable secure display support" - default n depends on DEBUG_FS depends on DRM_AMD_DC_DCN help diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 7b19f444624c3d12aa046334ab2eef27f043a2c5..c053cb79cd063eb3f9eab2e5a6fa2541f7d0ddc9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -100,8 +100,6 @@ #include "soc15_common.h" #include "vega10_ip_offset.h" -#include "soc15_common.h" - #include "gc/gc_11_0_0_offset.h" #include "gc/gc_11_0_0_sh_mask.h" @@ -1112,7 +1110,8 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev) hw_params.fb[i] = &fb_info->fb[i]; switch (adev->ip_versions[DCE_HWIP][0]) { - case IP_VERSION(3, 1, 3): /* Only for this asic hw internal rev B0 */ + case IP_VERSION(3, 1, 3): + case IP_VERSION(3, 1, 4): hw_params.dpia_supported = true; hw_params.disable_dpia = adev->dm.dc->debug.dpia_debug.bits.disable_dpia; break; @@ -1298,13 +1297,21 @@ static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct if (hpd_rx_offload_wq[i].wq == NULL) { DRM_ERROR("create amdgpu_dm_hpd_rx_offload_wq fail!"); - return NULL; + goto out_err; } spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); } return hpd_rx_offload_wq; + +out_err: + for (i = 0; i < max_caps; i++) { + if (hpd_rx_offload_wq[i].wq) + destroy_workqueue(hpd_rx_offload_wq[i].wq); + } + kfree(hpd_rx_offload_wq); + return NULL; } struct amdgpu_stutter_quirk { @@ -4742,7 +4749,7 @@ fill_dc_plane_info_and_addr(struct amdgpu_device *adev, plane_info->visible = true; plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE; - plane_info->layer_index = 0; + plane_info->layer_index = plane_state->normalized_zpos; ret = fill_plane_color_attributes(plane_state, plane_info->format, &plane_info->color_space); @@ -4810,7 +4817,7 @@ static int fill_dc_plane_attributes(struct amdgpu_device *adev, dc_plane_state->global_alpha = plane_info.global_alpha; dc_plane_state->global_alpha_value = plane_info.global_alpha_value; dc_plane_state->dcc = plane_info.dcc; - dc_plane_state->layer_index = plane_info.layer_index; // Always returns 0 + dc_plane_state->layer_index = plane_info.layer_index; dc_plane_state->flip_int_enabled = true; /* @@ -7379,11 +7386,6 @@ static void update_freesync_state_on_stream( &vrr_infopacket, pack_sdp_v1_3); - new_crtc_state->freesync_timing_changed |= - (memcmp(&acrtc->dm_irq_params.vrr_params.adjust, - &vrr_params.adjust, - sizeof(vrr_params.adjust)) != 0); - new_crtc_state->freesync_vrr_info_changed |= (memcmp(&new_crtc_state->vrr_infopacket, &vrr_infopacket, @@ -7392,7 +7394,6 @@ static void update_freesync_state_on_stream( acrtc->dm_irq_params.vrr_params = vrr_params; new_crtc_state->vrr_infopacket = vrr_infopacket; - new_stream->adjust = acrtc->dm_irq_params.vrr_params.adjust; new_stream->vrr_infopacket = vrr_infopacket; if (new_crtc_state->freesync_vrr_info_changed) @@ -7455,10 +7456,6 @@ static void update_stream_irq_parameters( new_stream, &config, &vrr_params); - new_crtc_state->freesync_timing_changed |= - (memcmp(&acrtc->dm_irq_params.vrr_params.adjust, - &vrr_params.adjust, sizeof(vrr_params.adjust)) != 0); - new_crtc_state->freesync_config = config; /* Copy state for access from DM IRQ handler */ acrtc->dm_irq_params.freesync_config = config; @@ -7482,15 +7479,15 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, * We also need vupdate irq for the actual core vblank handling * at end of vblank. */ - dm_set_vupdate_irq(new_state->base.crtc, true); - drm_crtc_vblank_get(new_state->base.crtc); + WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, true) != 0); + WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0); DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n", __func__, new_state->base.crtc->base.id); } else if (old_vrr_active && !new_vrr_active) { /* Transition VRR active -> inactive: * Allow vblank irq disable again for fixed refresh rate. */ - dm_set_vupdate_irq(new_state->base.crtc, false); + WARN_ON(dm_set_vupdate_irq(new_state->base.crtc, false) != 0); drm_crtc_vblank_put(new_state->base.crtc); DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n", __func__, new_state->base.crtc->base.id); @@ -8246,23 +8243,6 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) mutex_unlock(&dm->dc_lock); } - /* Count number of newly disabled CRTCs for dropping PM refs later. */ - for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - if (old_crtc_state->active && !new_crtc_state->active) - crtc_disable_count++; - - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); - - /* For freesync config update on crtc state and params for irq */ - update_stream_irq_parameters(dm, dm_new_crtc_state); - - /* Handle vrr on->off / off->on transitions */ - amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, - dm_new_crtc_state); - } - /** * Enable interrupts for CRTCs that are newly enabled or went through * a modeset. It was intentionally deferred until after the front end @@ -8272,16 +8252,29 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); #ifdef CONFIG_DEBUG_FS - bool configure_crc = false; enum amdgpu_dm_pipe_crc_source cur_crc_src; #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) - struct crc_rd_work *crc_rd_wrk = dm->crc_rd_wrk; + struct crc_rd_work *crc_rd_wrk; +#endif +#endif + /* Count number of newly disabled CRTCs for dropping PM refs later. */ + if (old_crtc_state->active && !new_crtc_state->active) + crtc_disable_count++; + + dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); + dm_old_crtc_state = to_dm_crtc_state(old_crtc_state); + + /* For freesync config update on crtc state and params for irq */ + update_stream_irq_parameters(dm, dm_new_crtc_state); + +#ifdef CONFIG_DEBUG_FS +#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) + crc_rd_wrk = dm->crc_rd_wrk; #endif spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); cur_crc_src = acrtc->dm_irq_params.crc_src; spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); #endif - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); if (new_crtc_state->active && (!old_crtc_state->active || @@ -8289,16 +8282,19 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) dc_stream_retain(dm_new_crtc_state->stream); acrtc->dm_irq_params.stream = dm_new_crtc_state->stream; manage_dm_interrupts(adev, acrtc, true); + } + /* Handle vrr on->off / off->on transitions */ + amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state); #ifdef CONFIG_DEBUG_FS + if (new_crtc_state->active && + (!old_crtc_state->active || + drm_atomic_crtc_needs_modeset(new_crtc_state))) { /** * Frontend may have changed so reapply the CRC capture * settings for the stream. */ - dm_new_crtc_state = to_dm_crtc_state(new_crtc_state); - if (amdgpu_dm_is_valid_crc_source(cur_crc_src)) { - configure_crc = true; #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) if (amdgpu_dm_crc_window_is_activated(crtc)) { spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); @@ -8310,14 +8306,12 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } #endif - } - - if (configure_crc) if (amdgpu_dm_crtc_configure_crc_source( crtc, dm_new_crtc_state, cur_crc_src)) DRM_DEBUG_DRIVER("Failed to configure crc source"); -#endif + } } +#endif } for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) @@ -9396,10 +9390,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } } } - if (!pre_validate_dsc(state, &dm_state, vars)) { - ret = -EINVAL; - goto fail; - } } #endif for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { @@ -9473,6 +9463,14 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } } + /* + * DC consults the zpos (layer_index in DC terminology) to determine the + * hw plane on which to enable the hw cursor (see + * `dcn10_can_pipe_disable_cursor`). By now, all modified planes are in + * atomic state, so call drm helper to normalize zpos. + */ + drm_atomic_normalize_zpos(dev, state); + /* Remove exiting planes if they are modified */ for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { ret = dm_update_plane_state(dc, state, plane, @@ -9525,6 +9523,15 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev, } } +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (dc_resource_is_dsc_encoding_supported(dc)) { + if (!pre_validate_dsc(state, &dm_state, vars)) { + ret = -EINVAL; + goto fail; + } + } +#endif + /* Run this here since we want to validate the streams we created */ ret = drm_atomic_helper_check_planes(dev, state); if (ret) { diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index b44faaad9b0b5ca217e337ea671094535bd4af8d..b5ce15c43bcc1667ea79fbc8da5247f364738280 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -681,7 +681,6 @@ struct dm_crtc_state { int crc_skip_count; - bool freesync_timing_changed; bool freesync_vrr_info_changed; bool dsc_force_changed; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index b8077fcd4651e1dfeca417041f30c1b5d5f3ac81..f0b01c8dc4a6b819731f666da64acefc079a0935 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -880,8 +880,17 @@ void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigne void dm_helpers_init_panel_settings( struct dc_context *ctx, - struct dc_panel_config *panel_config) -{ + struct dc_panel_config *panel_config, + struct dc_sink *sink) +{ + // Extra Panel Power Sequence + panel_config->pps.extra_t3_ms = sink->edid_caps.panel_patch.extra_t3_ms; + panel_config->pps.extra_t7_ms = sink->edid_caps.panel_patch.extra_t7_ms; + panel_config->pps.extra_delay_backlight_off = sink->edid_caps.panel_patch.extra_delay_backlight_off; + panel_config->pps.extra_post_t7_ms = 0; + panel_config->pps.extra_pre_t11_ms = 0; + panel_config->pps.extra_t12_ms = sink->edid_caps.panel_patch.extra_t12_ms; + panel_config->pps.extra_post_OUI_ms = 0; // Feature DSC panel_config->dsc.disable_dsc_edp = false; panel_config->dsc.force_dsc_edp_policy = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c index c8da18e45b0e865a371e75b0646a6c6fc15cc38b..26291db0a3cf6c4142b957ac2198aca5891cc9eb 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c @@ -60,11 +60,15 @@ static bool link_supports_psrsu(struct dc_link *link) */ void amdgpu_dm_set_psr_caps(struct dc_link *link) { - if (!(link->connector_signal & SIGNAL_TYPE_EDP)) + if (!(link->connector_signal & SIGNAL_TYPE_EDP)) { + link->psr_settings.psr_feature_enabled = false; return; + } - if (link->type == dc_connection_none) + if (link->type == dc_connection_none) { + link->psr_settings.psr_feature_enabled = false; return; + } if (link->dpcd_caps.psr_info.psr_version == 0) { link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; @@ -170,7 +174,13 @@ bool amdgpu_dm_psr_enable(struct dc_stream_state *stream) &stream, 1, ¶ms); - power_opt |= psr_power_opt_z10_static_screen; + /* + * Only enable static-screen optimizations for PSR1. For PSR SU, this + * causes vstartup interrupt issues, used by amdgpu_dm to send vblank + * events. + */ + if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1) + power_opt |= psr_power_opt_z10_static_screen; return dc_link_set_psr_allow_active(link, &psr_enable, false, false, &power_opt); } diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index ead4da11a992420632dc5f8be2bc01463f60e588..ee0456b5e14e443d030bcdea5e8f8b39ea6d0a03 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -24,6 +24,7 @@ */ #include "dm_services.h" +#include "core_types.h" #include "ObjectID.h" #include "atomfirmware.h" @@ -50,13 +51,6 @@ #define LAST_RECORD_TYPE 0xff #define SMU9_SYSPLL0_ID 0 -struct i2c_id_config_access { - uint8_t bfI2C_LineMux:4; - uint8_t bfHW_EngineID:3; - uint8_t bfHW_Capable:1; - uint8_t ucAccess; -}; - static enum bp_result get_gpio_i2c_info(struct bios_parser *bp, struct atom_i2c_record *record, struct graphics_object_i2c_info *info); @@ -849,6 +843,8 @@ static enum bp_result get_ss_info_v4_1( disp_cntl_tbl->dvi_ss_rate_10hz * 10; if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_HDMI: ss_info->spread_spectrum_percentage = @@ -857,6 +853,8 @@ static enum bp_result get_ss_info_v4_1( disp_cntl_tbl->hdmi_ss_rate_10hz * 10; if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; /* TODO LVDS not support anymore? */ case AS_SIGNAL_TYPE_DISPLAY_PORT: @@ -866,6 +864,8 @@ static enum bp_result get_ss_info_v4_1( disp_cntl_tbl->dp_ss_rate_10hz * 10; if (disp_cntl_tbl->dp_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_GPU_PLL: /* atom_firmware: DAL only get data from dce_info table. @@ -879,13 +879,15 @@ static enum bp_result get_ss_info_v4_1( DATA_TABLES(smu_info)); if (!smu_info) return BP_RESULT_BADBIOSTABLE; - + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info->gpuclk_ss_percentage); ss_info->spread_spectrum_percentage = smu_info->waflclk_ss_percentage; ss_info->spread_spectrum_range = smu_info->gpuclk_ss_rate_10hz * 10; if (smu_info->waflclk_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_XGMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; default: result = BP_RESULT_UNSUPPORTED; @@ -922,6 +924,7 @@ static enum bp_result get_ss_info_v4_2( if (!smu_info) return BP_RESULT_BADBIOSTABLE; + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info->gpuclk_ss_percentage); ss_info->type.STEP_AND_DELAY_INFO = false; ss_info->spread_percentage_divider = 1000; /* BIOS no longer uses target clock. Always enable for now */ @@ -935,6 +938,8 @@ static enum bp_result get_ss_info_v4_2( disp_cntl_tbl->dvi_ss_rate_10hz * 10; if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_HDMI: ss_info->spread_spectrum_percentage = @@ -943,6 +948,8 @@ static enum bp_result get_ss_info_v4_2( disp_cntl_tbl->hdmi_ss_rate_10hz * 10; if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; /* TODO LVDS not support anymore? */ case AS_SIGNAL_TYPE_DISPLAY_PORT: @@ -952,6 +959,8 @@ static enum bp_result get_ss_info_v4_2( smu_info->gpuclk_ss_rate_10hz * 10; if (smu_info->gpuclk_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_GPU_PLL: /* atom_firmware: DAL only get data from dce_info table. @@ -1000,6 +1009,8 @@ static enum bp_result get_ss_info_v4_5( disp_cntl_tbl->dvi_ss_rate_10hz * 10; if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_HDMI: ss_info->spread_spectrum_percentage = @@ -1008,6 +1019,8 @@ static enum bp_result get_ss_info_v4_5( disp_cntl_tbl->hdmi_ss_rate_10hz * 10; if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_DISPLAY_PORT: ss_info->spread_spectrum_percentage = @@ -1016,6 +1029,8 @@ static enum bp_result get_ss_info_v4_5( disp_cntl_tbl->dp_ss_rate_10hz * 10; if (disp_cntl_tbl->dp_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE) ss_info->type.CENTER_MODE = true; + + DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage); break; case AS_SIGNAL_TYPE_GPU_PLL: /* atom_smu_info_v4_0 does not have fields for SS for SMU Display PLL anymore. @@ -1353,7 +1368,7 @@ static enum bp_result bios_parser_get_lttpr_interop( default: break; } - + DC_LOG_BIOS("DCE_INFO_CAPS_VBIOS_LTTPR_TRANSPARENT_ENABLE: %d tbl_revision.major = %d tbl_revision.minor = %d\n", *dce_caps, tbl_revision.major, tbl_revision.minor); return result; } @@ -1369,6 +1384,7 @@ static enum bp_result bios_parser_get_lttpr_caps( if (!DATA_TABLES(dce_info)) return BP_RESULT_UNSUPPORTED; + *dce_caps = 0; header = GET_IMAGE(struct atom_common_table_header, DATA_TABLES(dce_info)); get_atom_data_table_revision(header, &tbl_revision); @@ -1402,7 +1418,11 @@ static enum bp_result bios_parser_get_lttpr_caps( default: break; } - + DC_LOG_BIOS("DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE: %d tbl_revision.major = %d tbl_revision.minor = %d\n", *dce_caps, tbl_revision.major, tbl_revision.minor); + if (dcb->ctx->dc->config.force_bios_enable_lttpr && *dce_caps == 0) { + *dce_caps = 1; + DC_LOG_BIOS("DCE_INFO_CAPS_VBIOS_LTTPR_TRANSPARENT_ENABLE: forced enabled"); + } return result; } @@ -1840,7 +1860,7 @@ static enum bp_result get_firmware_info_v3_2( /* Vega12 */ smu_info_v3_2 = GET_IMAGE(struct atom_smu_info_v3_2, DATA_TABLES(smu_info)); - + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_2->gpuclk_ss_percentage); if (!smu_info_v3_2) return BP_RESULT_BADBIOSTABLE; @@ -1849,7 +1869,7 @@ static enum bp_result get_firmware_info_v3_2( /* Vega20 */ smu_info_v3_3 = GET_IMAGE(struct atom_smu_info_v3_3, DATA_TABLES(smu_info)); - + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_3->gpuclk_ss_percentage); if (!smu_info_v3_3) return BP_RESULT_BADBIOSTABLE; @@ -1991,7 +2011,7 @@ static enum bp_result get_firmware_info_v3_4( if (!smu_info_v3_5) return BP_RESULT_BADBIOSTABLE; - + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_5->gpuclk_ss_percentage); info->default_engine_clk = smu_info_v3_5->bootup_dcefclk_10khz * 10; break; @@ -2397,6 +2417,7 @@ static enum bp_result get_integrated_info_v11( info_v11 = GET_IMAGE(struct atom_integrated_system_info_v1_11, DATA_TABLES(integratedsysteminfo)); + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v11->gpuclk_ss_percentage); if (info_v11 == NULL) return BP_RESULT_BADBIOSTABLE; @@ -2611,6 +2632,7 @@ static enum bp_result get_integrated_info_v2_1( info_v2_1 = GET_IMAGE(struct atom_integrated_system_info_v2_1, DATA_TABLES(integratedsysteminfo)); + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v2_1->gpuclk_ss_percentage); if (info_v2_1 == NULL) return BP_RESULT_BADBIOSTABLE; @@ -2772,6 +2794,8 @@ static enum bp_result get_integrated_info_v2_2( info_v2_2 = GET_IMAGE(struct atom_integrated_system_info_v2_2, DATA_TABLES(integratedsysteminfo)); + DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v2_2->gpuclk_ss_percentage); + if (info_v2_2 == NULL) return BP_RESULT_BADBIOSTABLE; @@ -2923,6 +2947,27 @@ static enum bp_result construct_integrated_info( default: return result; } + if (result == BP_RESULT_OK) { + + DC_LOG_BIOS("edp1:\n" + "\tedp_pwr_on_off_delay = %d\n" + "\tedp_pwr_on_vary_bl_to_blon = %d\n" + "\tedp_pwr_down_bloff_to_vary_bloff = %d\n" + "\tedp_bootup_bl_level = %d\n", + info->edp1_info.edp_pwr_on_off_delay, + info->edp1_info.edp_pwr_on_vary_bl_to_blon, + info->edp1_info.edp_pwr_down_bloff_to_vary_bloff, + info->edp1_info.edp_bootup_bl_level); + DC_LOG_BIOS("edp2:\n" + "\tedp_pwr_on_off_delayv = %d\n" + "\tedp_pwr_on_vary_bl_to_blon = %d\n" + "\tedp_pwr_down_bloff_to_vary_bloff = %d\n" + "\tedp_bootup_bl_level = %d\n", + info->edp2_info.edp_pwr_on_off_delay, + info->edp2_info.edp_pwr_on_vary_bl_to_blon, + info->edp2_info.edp_pwr_down_bloff_to_vary_bloff, + info->edp2_info.edp_bootup_bl_level); + } } if (result != BP_RESULT_OK) @@ -2948,13 +2993,22 @@ static enum bp_result construct_integrated_info( info->ext_disp_conn_info.path[i].ext_encoder_obj_id.id, info->ext_disp_conn_info.path[i].caps ); + if (info->ext_disp_conn_info.path[i].caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) + DC_LOG_BIOS("BIOS EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN on path %d\n", i); + else if (bp->base.ctx->dc->config.force_bios_fixed_vs) { + info->ext_disp_conn_info.path[i].caps |= EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN; + DC_LOG_BIOS("driver forced EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN on path %d\n", i); + } } - // Log the Checksum and Voltage Swing DC_LOG_BIOS("Integrated info table CHECKSUM: %d\n" "Integrated info table FIX_DP_VOLTAGE_SWING: %d\n", info->ext_disp_conn_info.checksum, info->ext_disp_conn_info.fixdpvoltageswing); + if (bp->base.ctx->dc->config.force_bios_fixed_vs && info->ext_disp_conn_info.fixdpvoltageswing == 0) { + info->ext_disp_conn_info.fixdpvoltageswing = bp->base.ctx->dc->config.force_bios_fixed_vs & 0xF; + DC_LOG_BIOS("driver forced fixdpvoltageswing = %d\n", info->ext_disp_conn_info.fixdpvoltageswing); + } } /* Sort voltage table from low to high*/ for (i = 1; i < NUMBER_OF_DISP_CLK_VOLTAGE; ++i) { @@ -3300,6 +3354,7 @@ static enum bp_result bios_get_board_layout_info( struct bios_parser *bp; static enum bp_result record_result; + unsigned int max_slots; const unsigned int slot_index_to_vbios_id[MAX_BOARD_SLOTS] = { GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1, @@ -3316,8 +3371,14 @@ static enum bp_result bios_get_board_layout_info( } board_layout_info->num_of_slots = 0; + max_slots = MAX_BOARD_SLOTS; + + // Assume single slot on v1_5 + if (bp->object_info_tbl.revision.minor == 5) { + max_slots = 1; + } - for (i = 0; i < MAX_BOARD_SLOTS; ++i) { + for (i = 0; i < max_slots; ++i) { record_result = get_bracket_layout_record(dcb, slot_index_to_vbios_id[i], &board_layout_info->slots[i]); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c index 0d30d1d9d67e9c38f2cd668870d5142affff2d40..650f3b4b562e90eea549935db4ed52a542c0c1bb 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c @@ -179,7 +179,7 @@ void dcn20_update_clocks_update_dentist(struct clk_mgr_internal *clk_mgr, struct } else if (dispclk_wdivider == 127 && current_dispclk_wdivider != 127) { REG_UPDATE(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, 126); - REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 100); + REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000); for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; struct dccg *dccg = clk_mgr->base.ctx->dc->res_pool->dccg; @@ -206,7 +206,7 @@ void dcn20_update_clocks_update_dentist(struct clk_mgr_internal *clk_mgr, struct REG_UPDATE(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, dispclk_wdivider); - REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 1000); + REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_DONE, 1, 50, 2000); REG_UPDATE(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_WDIVIDER, dppclk_wdivider); REG_WAIT(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_CHG_DONE, 1, 5, 100); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c index d43258e3cd4fe528435547784736f2eaf37de31e..c1eaf571407a37e56e4f687eb2be82fcbce2f823 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c @@ -104,7 +104,7 @@ static int dcn31_get_active_display_cnt_wa( return display_count; } -static void dcn31_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) +static void dcn31_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable) { struct dc *dc = clk_mgr_base->ctx->dc; int i; @@ -115,9 +115,10 @@ static void dcn31_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) if (pipe->top_pipe || pipe->prev_odm_pipe) continue; if (pipe->stream && (pipe->stream->dpms_off || dc_is_virtual_signal(pipe->stream->signal))) { - if (disable) + if (disable) { pipe->stream_res.tg->funcs->immediate_disable_crtc(pipe->stream_res.tg); - else + reset_sync_context_for_pipe(dc, context, i); + } else pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); } } @@ -216,11 +217,11 @@ void dcn31_update_clocks(struct clk_mgr *clk_mgr_base, } if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { - dcn31_disable_otg_wa(clk_mgr_base, true); + dcn31_disable_otg_wa(clk_mgr_base, context, true); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; dcn31_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); - dcn31_disable_otg_wa(clk_mgr_base, false); + dcn31_disable_otg_wa(clk_mgr_base, context, false); update_dispclk = true; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c index 171c38fac6a301608df829681d03657e5c826b2d..1131c6d73f6cdd2bbfe30a18107428148f34054e 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c @@ -126,7 +126,7 @@ static int dcn314_get_active_display_cnt_wa( return display_count; } -static void dcn314_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) +static void dcn314_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable) { struct dc *dc = clk_mgr_base->ctx->dc; int i; @@ -136,12 +136,21 @@ static void dcn314_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) if (pipe->top_pipe || pipe->prev_odm_pipe) continue; - if (pipe->stream && (pipe->stream->dpms_off || pipe->plane_state == NULL || - dc_is_virtual_signal(pipe->stream->signal))) { - if (disable) + if (pipe->stream && (pipe->stream->dpms_off || dc_is_virtual_signal(pipe->stream->signal))) { + struct stream_encoder *stream_enc = pipe->stream_res.stream_enc; + + if (disable) { + if (stream_enc && stream_enc->funcs->disable_fifo) + pipe->stream_res.stream_enc->funcs->disable_fifo(stream_enc); + pipe->stream_res.tg->funcs->immediate_disable_crtc(pipe->stream_res.tg); - else + reset_sync_context_for_pipe(dc, context, i); + } else { pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); + + if (stream_enc && stream_enc->funcs->enable_fifo) + pipe->stream_res.stream_enc->funcs->enable_fifo(stream_enc); + } } } } @@ -240,11 +249,11 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base, } if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { - dcn314_disable_otg_wa(clk_mgr_base, true); + dcn314_disable_otg_wa(clk_mgr_base, context, true); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; dcn314_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); - dcn314_disable_otg_wa(clk_mgr_base, false); + dcn314_disable_otg_wa(clk_mgr_base, context, false); update_dispclk = true; } @@ -677,6 +686,8 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl } ASSERT(bw_params->clk_table.entries[i-1].dcfclk_mhz); bw_params->vram_type = bios_info->memory_type; + + bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4; bw_params->num_channels = bios_info->ma_channel_number ? bios_info->ma_channel_number : 4; for (i = 0; i < WM_SET_COUNT; i++) { diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.c index 897105d1c111ef26b7737fe0082b78df97a9d41e..ef0795b14a1fd0e2666661771aeab0f10f7da8d8 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_smu.c @@ -339,29 +339,24 @@ void dcn314_smu_set_zstate_support(struct clk_mgr_internal *clk_mgr, enum dcn_zs if (!clk_mgr->smu_present) return; - if (!clk_mgr->base.ctx->dc->debug.enable_z9_disable_interface && - (support == DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY)) - support = DCN_ZSTATE_SUPPORT_DISALLOW; - - // Arg[15:0] = 8/9/0 for Z8/Z9/disallow -> existing bits // Arg[16] = Disallow Z9 -> new bit switch (support) { case DCN_ZSTATE_SUPPORT_ALLOW: msg_id = VBIOSSMC_MSG_AllowZstatesEntry; - param = 9; + param = (1 << 10) | (1 << 9) | (1 << 8); break; case DCN_ZSTATE_SUPPORT_DISALLOW: msg_id = VBIOSSMC_MSG_AllowZstatesEntry; - param = 8; + param = 0; break; case DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY: msg_id = VBIOSSMC_MSG_AllowZstatesEntry; - param = 0x00010008; + param = (1 << 10); break; default: //DCN_ZSTATE_SUPPORT_UNKNOWN diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index 14071aef5eab966217b333e6725b1cd34984f21d..893991a0eb9711d2a9fd4701834c4d3d0473ba61 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -51,6 +51,9 @@ #define TO_CLK_MGR_DCN315(clk_mgr)\ container_of(clk_mgr, struct clk_mgr_dcn315, base) +#define UNSUPPORTED_DCFCLK 10000000 +#define MIN_DPP_DISP_CLK 100000 + static int dcn315_get_active_display_cnt_wa( struct dc *dc, struct dc_state *context) @@ -84,7 +87,7 @@ static int dcn315_get_active_display_cnt_wa( return display_count; } -static void dcn315_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) +static void dcn315_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable) { struct dc *dc = clk_mgr_base->ctx->dc; int i; @@ -96,9 +99,10 @@ static void dcn315_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) continue; if (pipe->stream && (pipe->stream->dpms_off || pipe->plane_state == NULL || dc_is_virtual_signal(pipe->stream->signal))) { - if (disable) + if (disable) { pipe->stream_res.tg->funcs->immediate_disable_crtc(pipe->stream_res.tg); - else + reset_sync_context_for_pipe(dc, context, i); + } else pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); } } @@ -151,6 +155,9 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, } } + /* Lock pstate by requesting unsupported dcfclk if change is unsupported */ + if (!new_clocks->p_state_change_support) + new_clocks->dcfclk_khz = UNSUPPORTED_DCFCLK; if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) { clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz; dcn315_smu_set_hard_min_dcfclk(clk_mgr, clk_mgr_base->clks.dcfclk_khz); @@ -164,10 +171,10 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, // workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow. if (!IS_DIAG_DC(dc->ctx->dce_environment)) { - if (new_clocks->dppclk_khz < 100000) - new_clocks->dppclk_khz = 100000; - if (new_clocks->dispclk_khz < 100000) - new_clocks->dispclk_khz = 100000; + if (new_clocks->dppclk_khz < MIN_DPP_DISP_CLK) + new_clocks->dppclk_khz = MIN_DPP_DISP_CLK; + if (new_clocks->dispclk_khz < MIN_DPP_DISP_CLK) + new_clocks->dispclk_khz = MIN_DPP_DISP_CLK; } if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) { @@ -180,12 +187,12 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base, if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { /* No need to apply the w/a if we haven't taken over from bios yet */ if (clk_mgr_base->clks.dispclk_khz) - dcn315_disable_otg_wa(clk_mgr_base, true); + dcn315_disable_otg_wa(clk_mgr_base, context, true); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; dcn315_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); if (clk_mgr_base->clks.dispclk_khz) - dcn315_disable_otg_wa(clk_mgr_base, false); + dcn315_disable_otg_wa(clk_mgr_base, context, false); update_dispclk = true; } @@ -280,7 +287,7 @@ static struct wm_table ddr5_wm_table = { { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, - .pstate_latency_us = 64.0, + .pstate_latency_us = 129.0, .sr_exit_time_us = 11.5, .sr_enter_plus_exit_time_us = 14.5, .valid = true, @@ -288,7 +295,7 @@ static struct wm_table ddr5_wm_table = { { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, - .pstate_latency_us = 64.0, + .pstate_latency_us = 129.0, .sr_exit_time_us = 11.5, .sr_enter_plus_exit_time_us = 14.5, .valid = true, @@ -296,7 +303,7 @@ static struct wm_table ddr5_wm_table = { { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, - .pstate_latency_us = 64.0, + .pstate_latency_us = 129.0, .sr_exit_time_us = 11.5, .sr_enter_plus_exit_time_us = 14.5, .valid = true, @@ -304,7 +311,7 @@ static struct wm_table ddr5_wm_table = { { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, - .pstate_latency_us = 64.0, + .pstate_latency_us = 129.0, .sr_exit_time_us = 11.5, .sr_enter_plus_exit_time_us = 14.5, .valid = true, @@ -561,8 +568,7 @@ static void dcn315_clk_mgr_helper_populate_bw_params( ASSERT(bw_params->clk_table.entries[i-1].dcfclk_mhz); bw_params->vram_type = bios_info->memory_type; bw_params->num_channels = bios_info->ma_channel_number; - if (!bw_params->num_channels) - bw_params->num_channels = 2; + bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4; for (i = 0; i < WM_SET_COUNT; i++) { bw_params->wm_table.entries[i].wm_inst = i; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c index 0cd3d2eb7ac700f26f1bec4166f737114cfe475f..187f5b27fdc80dec1545a7c75d5622dd04d4e368 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c @@ -112,7 +112,7 @@ static int dcn316_get_active_display_cnt_wa( return display_count; } -static void dcn316_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) +static void dcn316_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable) { struct dc *dc = clk_mgr_base->ctx->dc; int i; @@ -124,9 +124,10 @@ static void dcn316_disable_otg_wa(struct clk_mgr *clk_mgr_base, bool disable) continue; if (pipe->stream && (pipe->stream->dpms_off || pipe->plane_state == NULL || dc_is_virtual_signal(pipe->stream->signal))) { - if (disable) + if (disable) { pipe->stream_res.tg->funcs->immediate_disable_crtc(pipe->stream_res.tg); - else + reset_sync_context_for_pipe(dc, context, i); + } else pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); } } @@ -221,11 +222,11 @@ static void dcn316_update_clocks(struct clk_mgr *clk_mgr_base, } if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) { - dcn316_disable_otg_wa(clk_mgr_base, true); + dcn316_disable_otg_wa(clk_mgr_base, context, true); clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz; dcn316_smu_set_dispclk(clk_mgr, clk_mgr_base->clks.dispclk_khz); - dcn316_disable_otg_wa(clk_mgr_base, false); + dcn316_disable_otg_wa(clk_mgr_base, context, false); update_dispclk = true; } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index c6785969eb1aa7dd9a63b1a2895ad8830310d4e4..1c612ccf1944aecac044b67fa9234d907ed7b75d 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -156,12 +156,14 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) { struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); unsigned int num_levels; + struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk; memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks)); clk_mgr_base->clks.p_state_change_support = true; clk_mgr_base->clks.prev_p_state_change_support = true; clk_mgr_base->clks.fclk_prev_p_state_change_support = true; clk_mgr->smu_present = false; + clk_mgr->dpm_present = false; if (!clk_mgr_base->bw_params) return; @@ -178,22 +180,29 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) /* DCFCLK */ dcn32_init_single_clock(clk_mgr, PPCLK_DCFCLK, &clk_mgr_base->bw_params->clk_table.entries[0].dcfclk_mhz, - &num_levels); + &num_entries_per_clk->num_dcfclk_levels); /* SOCCLK */ dcn32_init_single_clock(clk_mgr, PPCLK_SOCCLK, &clk_mgr_base->bw_params->clk_table.entries[0].socclk_mhz, - &num_levels); + &num_entries_per_clk->num_socclk_levels); + /* DTBCLK */ if (!clk_mgr->base.ctx->dc->debug.disable_dtb_ref_clk_switch) dcn32_init_single_clock(clk_mgr, PPCLK_DTBCLK, &clk_mgr_base->bw_params->clk_table.entries[0].dtbclk_mhz, - &num_levels); + &num_entries_per_clk->num_dtbclk_levels); /* DISPCLK */ dcn32_init_single_clock(clk_mgr, PPCLK_DISPCLK, &clk_mgr_base->bw_params->clk_table.entries[0].dispclk_mhz, - &num_levels); + &num_entries_per_clk->num_dispclk_levels); + num_levels = num_entries_per_clk->num_dispclk_levels; + + if (num_entries_per_clk->num_dcfclk_levels && + num_entries_per_clk->num_dtbclk_levels && + num_entries_per_clk->num_dispclk_levels) + clk_mgr->dpm_present = true; if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) { unsigned int i; @@ -325,6 +334,21 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base, if (enter_display_off == safe_to_lower) dcn30_smu_set_num_of_displays(clk_mgr, display_count); + clk_mgr_base->clks.fclk_prev_p_state_change_support = clk_mgr_base->clks.fclk_p_state_change_support; + + total_plane_count = clk_mgr_helper_get_active_plane_cnt(dc, context); + fclk_p_state_change_support = new_clocks->fclk_p_state_change_support || (total_plane_count == 0); + + if (should_update_pstate_support(safe_to_lower, fclk_p_state_change_support, clk_mgr_base->clks.fclk_p_state_change_support)) { + clk_mgr_base->clks.fclk_p_state_change_support = fclk_p_state_change_support; + + /* To enable FCLK P-state switching, send FCLK_PSTATE_SUPPORTED message to PMFW */ + if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && clk_mgr_base->clks.fclk_p_state_change_support) { + /* Handle the code for sending a message to PMFW that FCLK P-state change is supported */ + dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_SUPPORTED); + } + } + if (dc->debug.force_min_dcfclk_mhz > 0) new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ? new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000); @@ -344,7 +368,6 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base, clk_mgr_base->clks.socclk_khz = new_clocks->socclk_khz; clk_mgr_base->clks.prev_p_state_change_support = clk_mgr_base->clks.p_state_change_support; - clk_mgr_base->clks.fclk_prev_p_state_change_support = clk_mgr_base->clks.fclk_p_state_change_support; clk_mgr_base->clks.prev_num_ways = clk_mgr_base->clks.num_ways; if (clk_mgr_base->clks.num_ways != new_clocks->num_ways && @@ -353,27 +376,25 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base, dcn32_smu_send_cab_for_uclk_message(clk_mgr, clk_mgr_base->clks.num_ways); } - total_plane_count = clk_mgr_helper_get_active_plane_cnt(dc, context); + p_state_change_support = new_clocks->p_state_change_support || (total_plane_count == 0); - fclk_p_state_change_support = new_clocks->fclk_p_state_change_support || (total_plane_count == 0); if (should_update_pstate_support(safe_to_lower, p_state_change_support, clk_mgr_base->clks.p_state_change_support)) { clk_mgr_base->clks.p_state_change_support = p_state_change_support; /* to disable P-State switching, set UCLK min = max */ if (!clk_mgr_base->clks.p_state_change_support) dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, - clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz); + clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz); } - if (should_update_pstate_support(safe_to_lower, fclk_p_state_change_support, clk_mgr_base->clks.fclk_p_state_change_support) && - clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21) { - clk_mgr_base->clks.fclk_p_state_change_support = fclk_p_state_change_support; + /* Always update saved value, even if new value not set due to P-State switching unsupported. Also check safe_to_lower for FCLK */ + if (safe_to_lower && (clk_mgr_base->clks.fclk_p_state_change_support != clk_mgr_base->clks.fclk_prev_p_state_change_support)) { + update_fclk = true; + } - /* To disable FCLK P-state switching, send FCLK_PSTATE_NOTSUPPORTED message to PMFW */ - if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && !clk_mgr_base->clks.fclk_p_state_change_support) { - /* Handle code for sending a message to PMFW that FCLK P-state change is not supported */ - dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_NOTSUPPORTED); - } + if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && !clk_mgr_base->clks.fclk_p_state_change_support && update_fclk) { + /* Handle code for sending a message to PMFW that FCLK P-state change is not supported */ + dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_NOTSUPPORTED); } /* Always update saved value, even if new value not set due to P-State switching unsupported */ @@ -382,21 +403,11 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base, update_uclk = true; } - /* Always update saved value, even if new value not set due to P-State switching unsupported. Also check safe_to_lower for FCLK */ - if (safe_to_lower && (clk_mgr_base->clks.fclk_p_state_change_support != clk_mgr_base->clks.fclk_prev_p_state_change_support)) { - update_fclk = true; - } - /* set UCLK to requested value if P-State switching is supported, or to re-enable P-State switching */ if (clk_mgr_base->clks.p_state_change_support && (update_uclk || !clk_mgr_base->clks.prev_p_state_change_support)) dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz)); - if (clk_mgr_base->ctx->dce_version != DCN_VERSION_3_21 && clk_mgr_base->clks.fclk_p_state_change_support && update_fclk) { - /* Handle the code for sending a message to PMFW that FCLK P-state change is supported */ - dcn32_smu_send_fclk_pstate_message(clk_mgr, FCLK_PSTATE_SUPPORTED); - } - if (clk_mgr_base->clks.num_ways != new_clocks->num_ways && clk_mgr_base->clks.num_ways > new_clocks->num_ways) { clk_mgr_base->clks.num_ways = new_clocks->num_ways; @@ -624,7 +635,7 @@ static void dcn32_set_hard_min_memclk(struct clk_mgr *clk_mgr_base, bool current khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz)); else dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, - clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz); + clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz); } else { dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz); @@ -640,24 +651,42 @@ static void dcn32_set_hard_max_memclk(struct clk_mgr *clk_mgr_base) return; dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, - clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz); + clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries_per_clk.num_memclk_levels - 1].memclk_mhz); } /* Get current memclk states, update bounding box */ static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base) { struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); + struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk; unsigned int num_levels; if (!clk_mgr->smu_present) return; - /* Refresh memclk states */ + /* Refresh memclk and fclk states */ dcn32_init_single_clock(clk_mgr, PPCLK_UCLK, &clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz, - &num_levels); + &num_entries_per_clk->num_memclk_levels); + + dcn32_init_single_clock(clk_mgr, PPCLK_FCLK, + &clk_mgr_base->bw_params->clk_table.entries[0].fclk_mhz, + &num_entries_per_clk->num_fclk_levels); + + if (num_entries_per_clk->num_memclk_levels >= num_entries_per_clk->num_fclk_levels) { + num_levels = num_entries_per_clk->num_memclk_levels; + } else { + num_levels = num_entries_per_clk->num_fclk_levels; + } + clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1; + if (clk_mgr->dpm_present && !num_levels) + clk_mgr->dpm_present = false; + + if (!clk_mgr->dpm_present) + dcn32_patch_dpm_table(clk_mgr_base->bw_params); + DC_FP_START(); /* Refresh bounding box */ clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box( diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 9860bf38c54785684b3b0c0647e72af1ade47074..997ab031f816da0186ce3cf32fb21f0c33ac2cc4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -401,6 +401,9 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc, { int i; + if (memcmp(adjust, &stream->adjust, sizeof(struct dc_crtc_timing_adjust)) == 0) + return true; + stream->adjust.v_total_max = adjust->v_total_max; stream->adjust.v_total_mid = adjust->v_total_mid; stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num; @@ -1181,11 +1184,7 @@ static void disable_vbios_mode_if_required( pipe->stream_res.pix_clk_params.requested_pix_clk_100hz; if (pix_clk_100hz != requested_pix_clk_100hz) { - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(dc->current_state, - pipe, TX_OFF_SYMCLK_OFF); - else - core_link_disable_stream(pipe); + core_link_disable_stream(pipe); pipe->stream->dpms_off = false; } } @@ -1202,7 +1201,7 @@ static void wait_for_no_pipes_pending(struct dc *dc, struct dc_state *context) int count = 0; struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; - if (!pipe->plane_state) + if (!pipe->plane_state || pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) continue; /* Timeout 100 ms */ @@ -1735,10 +1734,20 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c int i, k, l; struct dc_stream_state *dc_streams[MAX_STREAMS] = {0}; struct dc_state *old_state; + bool subvp_prev_use = false; dc_z10_restore(dc); dc_allow_idle_optimizations(dc, false); + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + + /* Check old context for SubVP */ + subvp_prev_use |= (old_pipe->stream && old_pipe->stream->mall_stream_config.type == SUBVP_PHANTOM); + if (subvp_prev_use) + break; + } + for (i = 0; i < context->stream_count; i++) dc_streams[i] = context->streams[i]; @@ -1751,6 +1760,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c context->stream_count == 0) dc->hwss.prepare_bandwidth(dc, context); + if (dc->debug.enable_double_buffered_dsc_pg_support) + dc->hwss.update_dsc_pg(dc, context, false); + disable_dangling_plane(dc, context); /* re-program planes for existing stream, in case we need to * free up plane resource for later use @@ -1775,6 +1787,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe); } + if (dc->hwss.subvp_pipe_control_lock) + dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use); + result = dc->hwss.apply_ctx_to_hw(dc, context); if (result != DC_OK) { @@ -1792,6 +1807,12 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c dc->hwss.interdependent_update_lock(dc, context, false); dc->hwss.post_unlock_program_front_end(dc, context); } + + if (dc->hwss.commit_subvp_config) + dc->hwss.commit_subvp_config(dc, context); + if (dc->hwss.subvp_pipe_control_lock) + dc->hwss.subvp_pipe_control_lock(dc, context, false, true, NULL, subvp_prev_use); + for (i = 0; i < context->stream_count; i++) { const struct dc_link *link = context->streams[i]->link; @@ -1841,6 +1862,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c dc->hwss.optimize_bandwidth(dc, context); } + if (dc->debug.enable_double_buffered_dsc_pg_support) + dc->hwss.update_dsc_pg(dc, context, true); + if (dc->ctx->dce_version >= DCE_VERSION_MAX) TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk); else @@ -2004,6 +2028,9 @@ void dc_post_update_surfaces_to_stream(struct dc *dc) dc->hwss.optimize_bandwidth(dc, context); + if (dc->debug.enable_double_buffered_dsc_pg_support) + dc->hwss.update_dsc_pg(dc, context, true); + dc->optimized_required = false; dc->wm_optimized_required = false; } @@ -2323,9 +2350,13 @@ static enum surface_update_type det_surface_update(const struct dc *dc, type = get_scaling_info_update_type(u); elevate_update_type(&overall_type, type); - if (u->flip_addr) + if (u->flip_addr) { update_flags->bits.addr_update = 1; - + if (u->flip_addr->address.tmz_surface != u->surface->address.tmz_surface) { + update_flags->bits.tmz_changed = 1; + elevate_update_type(&overall_type, UPDATE_TYPE_FULL); + } + } if (u->in_transfer_func) update_flags->bits.in_transfer_func_change = 1; @@ -2760,11 +2791,8 @@ static void copy_stream_update_to_stream(struct dc *dc, if (update->abm_level) stream->abm_level = *update->abm_level; - if (update->periodic_interrupt0) - stream->periodic_interrupt0 = *update->periodic_interrupt0; - - if (update->periodic_interrupt1) - stream->periodic_interrupt1 = *update->periodic_interrupt1; + if (update->periodic_interrupt) + stream->periodic_interrupt = *update->periodic_interrupt; if (update->gamut_remap) stream->gamut_remap_matrix = *update->gamut_remap; @@ -2849,16 +2877,6 @@ static void copy_stream_update_to_stream(struct dc *dc, } } -void dc_reset_state(struct dc *dc, struct dc_state *context) -{ - dc_resource_state_destruct(context); - - /* clear the structure, but don't reset the reference count */ - memset(context, 0, offsetof(struct dc_state, refcount)); - - init_state(dc, context); -} - static bool update_planes_and_stream_state(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, @@ -2928,6 +2946,12 @@ static bool update_planes_and_stream_state(struct dc *dc, dc_resource_state_copy_construct( dc->current_state, context); + /* For each full update, remove all existing phantom pipes first. + * Ensures that we have enough pipes for newly added MPO planes + */ + if (dc->res_pool->funcs->remove_phantom_pipes) + dc->res_pool->funcs->remove_phantom_pipes(dc, context); + /*remove old surfaces from context */ if (!dc_rem_all_planes_for_stream(dc, stream, context)) { @@ -2994,13 +3018,8 @@ static void commit_planes_do_stream_update(struct dc *dc, if (!pipe_ctx->top_pipe && !pipe_ctx->prev_odm_pipe && pipe_ctx->stream == stream) { - if (stream_update->periodic_interrupt0 && - dc->hwss.setup_periodic_interrupt) - dc->hwss.setup_periodic_interrupt(dc, pipe_ctx, VLINE0); - - if (stream_update->periodic_interrupt1 && - dc->hwss.setup_periodic_interrupt) - dc->hwss.setup_periodic_interrupt(dc, pipe_ctx, VLINE1); + if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt) + dc->hwss.setup_periodic_interrupt(dc, pipe_ctx); if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) || stream_update->vrr_infopacket || @@ -3068,11 +3087,7 @@ static void commit_planes_do_stream_update(struct dc *dc, if (stream_update->dpms_off) { if (*stream_update->dpms_off) { - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(dc->current_state, - pipe_ctx, TX_OFF_SYMCLK_ON); - else - core_link_disable_stream(pipe_ctx); + core_link_disable_stream(pipe_ctx); /* for dpms, keep acquired resources*/ if (pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio); @@ -3082,12 +3097,7 @@ static void commit_planes_do_stream_update(struct dc *dc, } else { if (get_seamless_boot_stream_count(context) == 0) dc->hwss.prepare_bandwidth(dc, dc->current_state); - - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(dc->current_state, - pipe_ctx, TX_ON_SYMCLK_ON); - else - core_link_enable_stream(dc->current_state, pipe_ctx); + core_link_enable_stream(dc->current_state, pipe_ctx); } } @@ -3218,6 +3228,9 @@ static void commit_planes_for_stream(struct dc *dc, if (get_seamless_boot_stream_count(context) == 0) dc->hwss.prepare_bandwidth(dc, context); + if (dc->debug.enable_double_buffered_dsc_pg_support) + dc->hwss.update_dsc_pg(dc, context, false); + context_clock_trace(dc, context); } @@ -3346,8 +3359,14 @@ static void commit_planes_for_stream(struct dc *dc, /* Since phantom pipe programming is moved to post_unlock_program_front_end, * move the SubVP lock to after the phantom pipes have been setup */ - if (dc->hwss.subvp_pipe_control_lock) - dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use); + if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) { + if (dc->hwss.subvp_pipe_control_lock) + dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use); + } else { + if (dc->hwss.subvp_pipe_control_lock) + dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use); + } + return; } @@ -3507,6 +3526,9 @@ static void commit_planes_for_stream(struct dc *dc, if (update_type != UPDATE_TYPE_FAST) dc->hwss.post_unlock_program_front_end(dc, context); + if (update_type != UPDATE_TYPE_FAST) + if (dc->hwss.commit_subvp_config) + dc->hwss.commit_subvp_config(dc, context); if (update_type != UPDATE_TYPE_FAST) if (dc->hwss.commit_subvp_config) @@ -3541,11 +3563,91 @@ static void commit_planes_for_stream(struct dc *dc, } } +/* Determines if the incoming context requires a applying transition state with unnecessary + * pipe splitting and ODM disabled, due to hardware limitations. In a case where + * the OPP associated with an MPCC might change due to plane additions, this function + * returns true. + */ +static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc, + struct dc_stream_state *stream, + int surface_count, + bool *is_plane_addition) +{ + + struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream); + bool force_minimal_pipe_splitting = false; + uint32_t i; + + *is_plane_addition = false; + + if (cur_stream_status && + dc->current_state->stream_count > 0 && + dc->debug.pipe_split_policy != MPC_SPLIT_AVOID) { + /* determine if minimal transition is required due to MPC*/ + if (surface_count > 0) { + if (cur_stream_status->plane_count > surface_count) { + force_minimal_pipe_splitting = true; + } else if (cur_stream_status->plane_count < surface_count) { + force_minimal_pipe_splitting = true; + *is_plane_addition = true; + } + } + } + + if (cur_stream_status && + dc->current_state->stream_count == 1 && + dc->debug.enable_single_display_2to1_odm_policy) { + /* determine if minimal transition is required due to dynamic ODM*/ + if (surface_count > 0) { + if (cur_stream_status->plane_count > 2 && cur_stream_status->plane_count > surface_count) { + force_minimal_pipe_splitting = true; + } else if (surface_count > 2 && cur_stream_status->plane_count < surface_count) { + force_minimal_pipe_splitting = true; + *is_plane_addition = true; + } + } + } + + /* For SubVP pipe split case when adding MPO video + * we need to add a minimal transition. In this case + * there will be 2 streams (1 main stream, 1 phantom + * stream). + */ + if (cur_stream_status && + dc->current_state->stream_count == 2 && + stream->mall_stream_config.type == SUBVP_MAIN) { + bool is_pipe_split = false; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream && + (dc->current_state->res_ctx.pipe_ctx[i].bottom_pipe || + dc->current_state->res_ctx.pipe_ctx[i].next_odm_pipe)) { + is_pipe_split = true; + break; + } + } + + /* determine if minimal transition is required due to SubVP*/ + if (surface_count > 0 && is_pipe_split) { + if (cur_stream_status->plane_count > surface_count) { + force_minimal_pipe_splitting = true; + } else if (cur_stream_status->plane_count < surface_count) { + force_minimal_pipe_splitting = true; + *is_plane_addition = true; + } + } + } + + return force_minimal_pipe_splitting; +} + static bool commit_minimal_transition_state(struct dc *dc, struct dc_state *transition_base_context) { struct dc_state *transition_context = dc_create_state(dc); - enum pipe_split_policy tmp_policy; + enum pipe_split_policy tmp_mpc_policy; + bool temp_dynamic_odm_policy; + bool temp_subvp_policy; enum dc_status ret = DC_ERROR_UNEXPECTED; unsigned int i, j; @@ -3553,10 +3655,16 @@ static bool commit_minimal_transition_state(struct dc *dc, return false; if (!dc->config.is_vmin_only_asic) { - tmp_policy = dc->debug.pipe_split_policy; + tmp_mpc_policy = dc->debug.pipe_split_policy; dc->debug.pipe_split_policy = MPC_SPLIT_AVOID; } + temp_dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy; + dc->debug.enable_single_display_2to1_odm_policy = false; + + temp_subvp_policy = dc->debug.force_disable_subvp; + dc->debug.force_disable_subvp = true; + dc_resource_state_copy_construct(transition_base_context, transition_context); //commit minimal state @@ -3577,20 +3685,23 @@ static bool commit_minimal_transition_state(struct dc *dc, ret = dc_commit_state_no_check(dc, transition_context); } - //always release as dc_commit_state_no_check retains in good case + /*always release as dc_commit_state_no_check retains in good case*/ dc_release_state(transition_context); - //restore previous pipe split policy + /*restore previous pipe split and odm policy*/ if (!dc->config.is_vmin_only_asic) - dc->debug.pipe_split_policy = tmp_policy; + dc->debug.pipe_split_policy = tmp_mpc_policy; + + dc->debug.enable_single_display_2to1_odm_policy = temp_dynamic_odm_policy; + dc->debug.force_disable_subvp = temp_subvp_policy; if (ret != DC_OK) { - //this should never happen + /*this should never happen*/ BREAK_TO_DEBUGGER(); return false; } - //force full surface update + /*force full surface update*/ for (i = 0; i < dc->current_state->stream_count; i++) { for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) { dc->current_state->stream_status[i].plane_states[j]->update_flags.raw = 0xFFFFFFFF; @@ -3613,22 +3724,14 @@ bool dc_update_planes_and_stream(struct dc *dc, * cause underflow. Apply stream configuration with minimal pipe * split first to avoid unsupported transitions for active pipes. */ - bool force_minimal_pipe_splitting = false; - bool is_plane_addition = false; + bool force_minimal_pipe_splitting; + bool is_plane_addition; - struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream); - - if (cur_stream_status && - dc->current_state->stream_count > 0 && - dc->debug.pipe_split_policy != MPC_SPLIT_AVOID) { - /* determine if minimal transition is required */ - if (cur_stream_status->plane_count > surface_count) { - force_minimal_pipe_splitting = true; - } else if (cur_stream_status->plane_count < surface_count) { - force_minimal_pipe_splitting = true; - is_plane_addition = true; - } - } + force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes( + dc, + stream, + surface_count, + &is_plane_addition); /* on plane addition, minimal state is the current one */ if (force_minimal_pipe_splitting && is_plane_addition && @@ -3645,7 +3748,7 @@ bool dc_update_planes_and_stream(struct dc *dc, &context)) return false; - /* on plane addition, minimal state is the new one */ + /* on plane removal, minimal state is the new one */ if (force_minimal_pipe_splitting && !is_plane_addition) { if (!commit_minimal_transition_state(dc, context)) { dc_release_state(context); @@ -4032,7 +4135,7 @@ struct dc_sink *dc_link_add_remote_sink( * Treat device as no EDID device if EDID * parsing fails */ - if (edid_status != EDID_OK) { + if (edid_status != EDID_OK && edid_status != EDID_PARTIAL_VALID) { dc_sink->dc_edid.length = 0; dm_error("Bad EDID, status%d!\n", edid_status); } @@ -4553,6 +4656,37 @@ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc, return DC_OK; } +/** + ***************************************************************************** + * Function: dc_process_dmub_dpia_hpd_int_enable + * + * @brief + * Submits dpia hpd int enable command to dmub via inbox message + * + * @param + * [in] dc: dc structure + * [in] hpd_int_enable: 1 for hpd int enable, 0 to disable + * + * @return + * None + ***************************************************************************** + */ +void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc, + uint32_t hpd_int_enable) +{ + union dmub_rb_cmd cmd = {0}; + struct dc_dmub_srv *dmub_srv = dc->ctx->dmub_srv; + + cmd.dpia_hpd_int_enable.header.type = DMUB_CMD__DPIA_HPD_INT_ENABLE; + cmd.dpia_hpd_int_enable.enable = hpd_int_enable; + + dc_dmub_srv_cmd_queue(dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dmub_srv); + dc_dmub_srv_wait_idle(dmub_srv); + + DC_LOG_DEBUG("%s: hpd_int_enable(%d)\n", __func__, hpd_int_enable); +} + /** * dc_disable_accelerated_mode - disable accelerated mode * @dc: dc structure diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c index 9dd705b985b95a7d3dd4a958f49c4f37fdd8b7b2..7c2e3b8dc26ad40cb315a0951a193296289b8238 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c @@ -417,8 +417,8 @@ void get_subvp_visual_confirm_color( for (i = 0; i < dc->res_pool->pipe_count; i++) { struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe->stream && pipe->stream->mall_stream_config.paired_stream && - pipe->stream->mall_stream_config.type == SUBVP_MAIN) { + if (pipe->stream && pipe->stream->mall_stream_config.paired_stream && + pipe->stream->mall_stream_config.type == SUBVP_MAIN) { /* SubVP enable - red */ color->color_r_cr = color_value; enable_subvp = true; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 4ab27e23133784d5b3b0b3be4dfdf7b264e7670c..d7b1ace6328a0a1c92b9a323f114a6bf5b3319dc 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -832,8 +832,9 @@ static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason LINK_INFO("link=%d, mst branch is now Connected\n", link->link_index); - apply_dpia_mst_dsc_always_on_wa(link); link->type = dc_connection_mst_branch; + apply_dpia_mst_dsc_always_on_wa(link); + dm_helpers_dp_update_branch_info(link->ctx, link); if (dm_helpers_dp_mst_start_top_mgr(link->ctx, link, (reason == DETECT_REASON_BOOT || reason == DETECT_REASON_RESUMEFROMS3S4))) { @@ -847,20 +848,13 @@ static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason bool reset_cur_dp_mst_topology(struct dc_link *link) { - bool result = false; DC_LOGGER_INIT(link->ctx->logger); LINK_INFO("link=%d, mst branch is now Disconnected\n", link->link_index); revert_dpia_mst_dsc_always_on_wa(link); - result = dm_helpers_dp_mst_stop_top_mgr(link->ctx, link); - - link->mst_stream_alloc_table.stream_count = 0; - memset(link->mst_stream_alloc_table.stream_allocations, - 0, - sizeof(link->mst_stream_alloc_table.stream_allocations)); - return result; + return dm_helpers_dp_mst_stop_top_mgr(link->ctx, link); } static bool should_prepare_phy_clocks_for_link_verification(const struct dc *dc, @@ -1313,8 +1307,11 @@ static bool detect_link_and_local_sink(struct dc_link *link, } if (link->connector_signal == SIGNAL_TYPE_EDP) { - // Init dc_panel_config - dm_helpers_init_panel_settings(dc_ctx, &link->panel_config); + /* Init dc_panel_config by HW config */ + if (dc_ctx->dc->res_pool->funcs->get_panel_config_defaults) + dc_ctx->dc->res_pool->funcs->get_panel_config_defaults(&link->panel_config); + /* Pickup base DM settings */ + dm_helpers_init_panel_settings(dc_ctx, &link->panel_config, sink); // Override dc_panel_config if system has specific settings dm_helpers_override_panel_settings(dc_ctx, &link->panel_config); } @@ -1983,7 +1980,7 @@ static enum dc_status enable_link_dp(struct dc_state *state, int i; bool apply_seamless_boot_optimization = false; uint32_t bl_oled_enable_delay = 50; // in ms - const uint32_t post_oui_delay = 30; // 30ms + uint32_t post_oui_delay = 30; // 30ms /* Reduce link bandwidth between failed link training attempts. */ bool do_fallback = false; @@ -2030,8 +2027,10 @@ static enum dc_status enable_link_dp(struct dc_state *state, // during mode switch we do DP_SET_POWER off then on, and OUI is lost dpcd_set_source_specific_data(link); - if (link->dpcd_sink_ext_caps.raw != 0) + if (link->dpcd_sink_ext_caps.raw != 0) { + post_oui_delay += link->panel_config.pps.extra_post_OUI_ms; msleep(post_oui_delay); + } // similarly, mode switch can cause loss of cable ID dpcd_write_cable_id_to_dprx(link); @@ -2643,9 +2642,8 @@ static void disable_link(struct dc_link *link, const struct link_resource *link_ dp_set_fec_ready(link, link_res, false); } } - } else { - if (signal != SIGNAL_TYPE_VIRTUAL) - link->link_enc->funcs->disable_output(link->link_enc, signal); + } else if (signal != SIGNAL_TYPE_VIRTUAL) { + link->dc->hwss.disable_link_output(link, link_res, signal); } if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { @@ -2667,6 +2665,7 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx) bool is_over_340mhz = false; bool is_vga_mode = (stream->timing.h_addressable == 640) && (stream->timing.v_addressable == 480); + struct dc *dc = pipe_ctx->stream->ctx->dc; if (stream->phy_pix_clk == 0) stream->phy_pix_clk = stream->timing.pix_clk_100hz / 10; @@ -2706,11 +2705,12 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx) if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422) display_color_depth = COLOR_DEPTH_888; - link->link_enc->funcs->enable_tmds_output( - link->link_enc, + dc->hwss.enable_tmds_link_output( + link, + &pipe_ctx->link_res, + pipe_ctx->stream->signal, pipe_ctx->clock_source->id, display_color_depth, - pipe_ctx->stream->signal, stream->phy_pix_clk); if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) @@ -2721,15 +2721,16 @@ static void enable_link_lvds(struct pipe_ctx *pipe_ctx) { struct dc_stream_state *stream = pipe_ctx->stream; struct dc_link *link = stream->link; + struct dc *dc = stream->ctx->dc; if (stream->phy_pix_clk == 0) stream->phy_pix_clk = stream->timing.pix_clk_100hz / 10; memset(&stream->link->cur_link_settings, 0, sizeof(struct dc_link_settings)); - - link->link_enc->funcs->enable_lvds_output( - link->link_enc, + dc->hwss.enable_lvds_link_output( + link, + &pipe_ctx->link_res, pipe_ctx->clock_source->id, stream->phy_pix_clk); @@ -3145,7 +3146,7 @@ bool dc_link_set_psr_allow_active(struct dc_link *link, const bool *allow_active if (!dc_get_edp_link_panel_inst(dc, link, &panel_inst)) return false; - if (allow_active && link->type == dc_connection_none) { + if ((allow_active != NULL) && (*allow_active == true) && (link->type == dc_connection_none)) { // Don't enter PSR if panel is not connected return false; } @@ -3377,8 +3378,8 @@ bool dc_link_setup_psr(struct dc_link *link, case FAMILY_YELLOW_CARP: case AMDGPU_FAMILY_GC_10_3_6: case AMDGPU_FAMILY_GC_11_0_1: - if(!dc->debug.disable_z10) - psr_context->psr_level.bits.SKIP_CRTC_DISABLE = false; + if (dc->debug.disable_z10) + psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true; break; default: psr_context->psr_level.bits.SKIP_CRTC_DISABLE = true; @@ -3567,6 +3568,35 @@ static void update_mst_stream_alloc_table( work_table[i]; } +static void remove_stream_from_alloc_table( + struct dc_link *link, + struct stream_encoder *dio_stream_enc, + struct hpo_dp_stream_encoder *hpo_dp_stream_enc) +{ + int i = 0; + struct link_mst_stream_allocation_table *table = + &link->mst_stream_alloc_table; + + if (hpo_dp_stream_enc) { + for (; i < table->stream_count; i++) + if (hpo_dp_stream_enc == table->stream_allocations[i].hpo_dp_stream_enc) + break; + } else { + for (; i < table->stream_count; i++) + if (dio_stream_enc == table->stream_allocations[i].stream_enc) + break; + } + + if (i < table->stream_count) { + i++; + for (; i < table->stream_count; i++) + table->stream_allocations[i-1] = table->stream_allocations[i]; + memset(&table->stream_allocations[table->stream_count-1], 0, + sizeof(struct link_mst_stream_allocation)); + table->stream_count--; + } +} + static void dc_log_vcp_x_y(const struct dc_link *link, struct fixed31_32 avg_time_slots_per_mtp) { const uint32_t VCP_Y_PRECISION = 1000; @@ -3984,26 +4014,32 @@ static enum dc_status deallocate_mst_payload(struct pipe_ctx *pipe_ctx) &empty_link_settings, avg_time_slots_per_mtp); - /* TODO: which component is responsible for remove payload table? */ if (mst_mode) { + /* when link is in mst mode, reply on mst manager to remove + * payload + */ if (dm_helpers_dp_mst_write_payload_allocation_table( stream->ctx, stream, &proposed_table, - false)) { + false)) update_mst_stream_alloc_table( - link, - pipe_ctx->stream_res.stream_enc, - pipe_ctx->stream_res.hpo_dp_stream_enc, - &proposed_table); - } - else { - DC_LOG_WARNING("Failed to update" - "MST allocation table for" - "pipe idx:%d\n", - pipe_ctx->pipe_idx); - } + link, + pipe_ctx->stream_res.stream_enc, + pipe_ctx->stream_res.hpo_dp_stream_enc, + &proposed_table); + else + DC_LOG_WARNING("Failed to update" + "MST allocation table for" + "pipe idx:%d\n", + pipe_ctx->pipe_idx); + } else { + /* when link is no longer in mst mode (mst hub unplugged), + * remove payload with default dc logic + */ + remove_stream_from_alloc_table(link, pipe_ctx->stream_res.stream_enc, + pipe_ctx->stream_res.hpo_dp_stream_enc); } DC_LOG_MST("%s" @@ -4299,6 +4335,19 @@ void core_link_enable_stream( if (pipe_ctx->stream->dpms_off) return; + /* Have to setup DSC before DIG FE and BE are connected (which happens before the + * link training). This is to make sure the bandwidth sent to DIG BE won't be + * bigger than what the link and/or DIG BE can handle. VBID[6]/CompressedStream_flag + * will be automatically set at a later time when the video is enabled + * (DP_VID_STREAM_EN = 1). + */ + if (pipe_ctx->stream->timing.flags.DSC) { + if (dc_is_dp_signal(pipe_ctx->stream->signal) || + dc_is_virtual_signal(pipe_ctx->stream->signal)) + dp_set_dsc_enable(pipe_ctx, true); + + } + status = enable_link(state, pipe_ctx); if (status != DC_OK) { diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c index d01d2eeed81351e3b008cc981a28684d43c311ef..651231387043d1a9bf9d99e60695ade321cbdbd1 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c @@ -35,6 +35,8 @@ #include "dc_link_ddc.h" #include "dce/dce_aux.h" #include "dmub/inc/dmub_cmd.h" +#include "link_dpcd.h" +#include "include/dal_asic_id.h" #define DC_LOGGER_INIT(logger) @@ -683,6 +685,21 @@ bool dc_link_aux_try_to_configure_timeout(struct ddc_service *ddc, bool result = false; struct ddc *ddc_pin = ddc->ddc_pin; + if ((ddc->link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) && + !ddc->link->dc->debug.disable_fixed_vs_aux_timeout_wa && + ASICREV_IS_YELLOW_CARP(ddc->ctx->asic_id.hw_internal_rev)) { + /* Fixed VS workaround for AUX timeout */ + const uint32_t fixed_vs_address = 0xF004F; + const uint8_t fixed_vs_data[4] = {0x1, 0x22, 0x63, 0xc}; + + core_link_write_dpcd(ddc->link, + fixed_vs_address, + fixed_vs_data, + sizeof(fixed_vs_data)); + + timeout = 3072; + } + /* Do not try to access nonexistent DDC pin. */ if (ddc->link->ep_type != DISPLAY_ENDPOINT_PHY) return true; @@ -691,6 +708,7 @@ bool dc_link_aux_try_to_configure_timeout(struct ddc_service *ddc, ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout(ddc, timeout); result = true; } + return result; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index e2413d2908c999d3872fc484bbbde87ba585ab7d..1254d38f1778ae2e3bd5acfe3a23e8d78e43c4b4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -526,9 +526,9 @@ uint8_t dc_dp_initialize_scrambling_data_symbols( return disable_scrabled_data_symbols; } -static inline bool is_repeater(struct dc_link *link, uint32_t offset) +static inline bool is_repeater(const struct link_training_settings *lt_settings, uint32_t offset) { - return (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (offset != 0); + return (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (offset != 0); } static void dpcd_set_lt_pattern_and_lane_settings( @@ -545,7 +545,7 @@ static void dpcd_set_lt_pattern_and_lane_settings( bool edp_workaround = false; /* TODO link_prop.INTERNAL */ dpcd_base_lt_offset = DP_TRAINING_PATTERN_SET; - if (is_repeater(link, offset)) + if (is_repeater(lt_settings, offset)) dpcd_base_lt_offset = DP_TRAINING_PATTERN_SET_PHY_REPEATER1 + ((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1)); @@ -561,7 +561,7 @@ static void dpcd_set_lt_pattern_and_lane_settings( dpcd_lt_buffer[DP_TRAINING_PATTERN_SET - DP_TRAINING_PATTERN_SET] = dpcd_pattern.raw; - if (is_repeater(link, offset)) { + if (is_repeater(lt_settings, offset)) { DC_LOG_HW_LINK_TRAINING("%s\n LTTPR Repeater ID: %d\n 0x%X pattern = %x\n", __func__, offset, @@ -584,7 +584,7 @@ static void dpcd_set_lt_pattern_and_lane_settings( lt_settings->dpcd_lane_settings, size_in_bytes); - if (is_repeater(link, offset)) { + if (is_repeater(lt_settings, offset)) { if (dp_get_link_encoding_format(<_settings->link_settings) == DP_128b_132b_ENCODING) DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n" @@ -873,7 +873,7 @@ enum dc_status dp_get_lane_status_and_lane_adjust( uint32_t lane; enum dc_status status; - if (is_repeater(link, offset)) { + if (is_repeater(link_training_setting, offset)) { lane01_status_address = DP_LANE0_1_STATUS_PHY_REPEATER1 + ((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1)); @@ -906,7 +906,7 @@ enum dc_status dp_get_lane_status_and_lane_adjust( ln_align->raw = dpcd_buf[2]; - if (is_repeater(link, offset)) { + if (is_repeater(link_training_setting, offset)) { DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n" " 0x%X Lane01Status = %x\n 0x%X Lane23Status = %x\n ", __func__, @@ -944,6 +944,23 @@ enum dc_status dp_get_lane_status_and_lane_adjust( return status; } +static enum dc_status dpcd_128b_132b_set_lane_settings( + struct dc_link *link, + const struct link_training_settings *link_training_setting) +{ + enum dc_status status = core_link_write_dpcd(link, + DP_TRAINING_LANE0_SET, + (uint8_t *)(link_training_setting->dpcd_lane_settings), + sizeof(link_training_setting->dpcd_lane_settings)); + + DC_LOG_HW_LINK_TRAINING("%s:\n 0x%X TX_FFE_PRESET_VALUE = %x\n", + __func__, + DP_TRAINING_LANE0_SET, + link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE); + return status; +} + + enum dc_status dpcd_set_lane_settings( struct dc_link *link, const struct link_training_settings *link_training_setting, @@ -954,7 +971,7 @@ enum dc_status dpcd_set_lane_settings( lane0_set_address = DP_TRAINING_LANE0_SET; - if (is_repeater(link, offset)) + if (is_repeater(link_training_setting, offset)) lane0_set_address = DP_TRAINING_LANE0_SET_PHY_REPEATER1 + ((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1)); @@ -963,17 +980,7 @@ enum dc_status dpcd_set_lane_settings( (uint8_t *)(link_training_setting->dpcd_lane_settings), link_training_setting->link_settings.lane_count); - if (is_repeater(link, offset)) { - if (dp_get_link_encoding_format(&link_training_setting->link_settings) == - DP_128b_132b_ENCODING) - DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n" - " 0x%X TX_FFE_PRESET_VALUE = %x\n", - __func__, - offset, - lane0_set_address, - link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE); - else if (dp_get_link_encoding_format(&link_training_setting->link_settings) == - DP_8b_10b_ENCODING) + if (is_repeater(link_training_setting, offset)) { DC_LOG_HW_LINK_TRAINING("%s\n LTTPR Repeater ID: %d\n" " 0x%X VS set = %x PE set = %x max VS Reached = %x max PE Reached = %x\n", __func__, @@ -985,14 +992,6 @@ enum dc_status dpcd_set_lane_settings( link_training_setting->dpcd_lane_settings[0].bits.MAX_PRE_EMPHASIS_REACHED); } else { - if (dp_get_link_encoding_format(&link_training_setting->link_settings) == - DP_128b_132b_ENCODING) - DC_LOG_HW_LINK_TRAINING("%s:\n 0x%X TX_FFE_PRESET_VALUE = %x\n", - __func__, - lane0_set_address, - link_training_setting->dpcd_lane_settings[0].tx_ffe.PRESET_VALUE); - else if (dp_get_link_encoding_format(&link_training_setting->link_settings) == - DP_8b_10b_ENCODING) DC_LOG_HW_LINK_TRAINING("%s\n 0x%X VS set = %x PE set = %x max VS Reached = %x max PE Reached = %x\n", __func__, lane0_set_address, @@ -1172,7 +1171,7 @@ static enum link_training_result perform_channel_equalization_sequence( /* Note: also check that TPS4 is a supported feature*/ tr_pattern = lt_settings->pattern_for_eq; - if (is_repeater(link, offset) && dp_get_link_encoding_format(<_settings->link_settings) == DP_8b_10b_ENCODING) + if (is_repeater(lt_settings, offset) && dp_get_link_encoding_format(<_settings->link_settings) == DP_8b_10b_ENCODING) tr_pattern = DP_TRAINING_PATTERN_SEQUENCE_4; dp_set_hw_training_pattern(link, link_res, tr_pattern, offset); @@ -1198,7 +1197,7 @@ static enum link_training_result perform_channel_equalization_sequence( /* 3. wait for receiver to lock-on*/ wait_time_microsec = lt_settings->eq_pattern_time; - if (is_repeater(link, offset)) + if (is_repeater(lt_settings, offset)) wait_time_microsec = dp_translate_training_aux_read_interval( link->dpcd_caps.lttpr_caps.aux_rd_interval[offset - 1]); @@ -1469,7 +1468,6 @@ static inline void decide_8b_10b_training_settings( */ lt_settings->link_settings.link_spread = link->dp_ss_off ? LINK_SPREAD_DISABLED : LINK_SPREAD_05_DOWNSPREAD_30KHZ; - lt_settings->lttpr_mode = link->lttpr_mode; lt_settings->cr_pattern_time = get_cr_training_aux_rd_interval(link, link_setting); lt_settings->eq_pattern_time = get_eq_training_aux_rd_interval(link, link_setting); lt_settings->pattern_for_cr = decide_cr_training_pattern(link_setting); @@ -1478,6 +1476,7 @@ static inline void decide_8b_10b_training_settings( lt_settings->should_set_fec_ready = true; lt_settings->disallow_per_lane_settings = true; lt_settings->always_match_dpcd_with_hw_lane_settings = true; + lt_settings->lttpr_mode = dp_decide_8b_10b_lttpr_mode(link); dp_hw_to_dpcd_lane_settings(lt_settings, lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings); } @@ -1501,9 +1500,8 @@ static inline void decide_128b_132b_training_settings(struct dc_link *link, lt_settings->cds_pattern_time = 2500; lt_settings->cds_wait_time_limit = (dp_convert_to_count( link->dpcd_caps.lttpr_caps.phy_repeater_cnt) + 1) * 20000; - lt_settings->lttpr_mode = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) ? - LTTPR_MODE_NON_TRANSPARENT : LTTPR_MODE_TRANSPARENT; lt_settings->disallow_per_lane_settings = true; + lt_settings->lttpr_mode = dp_decide_128b_132b_lttpr_mode(link); dp_hw_to_dpcd_lane_settings(lt_settings, lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings); } @@ -1543,7 +1541,7 @@ static void override_training_settings( lt_settings->ffe_preset = overrides->ffe_preset; /* Override HW lane settings with BIOS forced values if present */ if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN && - link->lttpr_mode == LTTPR_MODE_TRANSPARENT) { + lt_settings->lttpr_mode == LTTPR_MODE_TRANSPARENT) { lt_settings->voltage_swing = &link->bios_forced_drive_settings.VOLTAGE_SWING; lt_settings->pre_emphasis = &link->bios_forced_drive_settings.PRE_EMPHASIS; lt_settings->always_match_dpcd_with_hw_lane_settings = false; @@ -1584,6 +1582,15 @@ static void override_training_settings( if (link->preferred_training_settings.fec_enable != NULL) lt_settings->should_set_fec_ready = *link->preferred_training_settings.fec_enable; + + #if defined(CONFIG_DRM_AMD_DC_DCN) + /* Check DP tunnel LTTPR mode debug option. */ + if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA && link->dc->debug.dpia_debug.bits.force_non_lttpr) + lt_settings->lttpr_mode = LTTPR_MODE_NON_LTTPR; + +#endif + dp_get_lttpr_mode_override(link, <_settings->lttpr_mode); + } uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count) @@ -1649,7 +1656,7 @@ static enum dc_status configure_lttpr_mode_non_transparent( link->dpcd_caps.lttpr_caps.mode = repeater_mode; } - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { DC_LOG_HW_LINK_TRAINING("%s\n Set LTTPR to Non Transparent Mode\n", __func__); @@ -2015,7 +2022,7 @@ static enum link_training_result dp_perform_128b_132b_channel_eq_done_sequence( result = DP_128b_132b_LT_FAILED; } else { dp_set_hw_lane_settings(link, link_res, lt_settings, DPRX); - dpcd_set_lane_settings(link, lt_settings, DPRX); + dpcd_128b_132b_set_lane_settings(link, lt_settings); } loop_count++; } @@ -2099,7 +2106,7 @@ static enum link_training_result dp_perform_8b_10b_link_training( /* 1. set link rate, lane count and spread. */ dpcd_set_link_settings(link, lt_settings); - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { /* 2. perform link training (set link training done * to false is done as well) @@ -2216,7 +2223,7 @@ static enum link_training_result perform_fixed_vs_pe_nontransparent_training_seq link->vendor_specific_lttpr_link_rate_wa = target_rate; - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { /* 2. perform link training (set link training done * to false is done as well) @@ -2288,7 +2295,7 @@ static enum link_training_result dp_perform_fixed_vs_pe_training_sequence( ASSERT(dp_get_link_encoding_format(<_settings->link_settings) == DP_8b_10b_ENCODING); - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { status = perform_fixed_vs_pe_nontransparent_training_sequence(link, link_res, lt_settings); return status; } @@ -2635,6 +2642,7 @@ enum link_training_result dc_link_dp_perform_link_training( link, link_settings, <_settings); + override_training_settings( link, &link->preferred_training_settings, @@ -2652,7 +2660,7 @@ enum link_training_result dc_link_dp_perform_link_training( * Per DP specs starting from here, DPTX device shall not issue * Non-LT AUX transactions inside training mode. */ - if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) + if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN && encoding == DP_8b_10b_ENCODING) status = dp_perform_fixed_vs_pe_training_sequence(link, link_res, <_settings); else if (encoding == DP_8b_10b_ENCODING) status = dp_perform_8b_10b_link_training(link, link_res, <_settings); @@ -2758,8 +2766,14 @@ bool perform_link_training_with_retries( skip_video_pattern); /* Transmit idle pattern once training successful. */ - if (status == LINK_TRAINING_SUCCESS && !is_link_bw_low) + if (status == LINK_TRAINING_SUCCESS && !is_link_bw_low) { dp_set_hw_test_pattern(link, &pipe_ctx->link_res, DP_TEST_PATTERN_VIDEO_MODE, NULL, 0); + /* Update verified link settings to current one + * Because DPIA LT might fallback to lower link setting. + */ + link->verified_link_cap.link_rate = link->cur_link_settings.link_rate; + link->verified_link_cap.lane_count = link->cur_link_settings.lane_count; + } } else { status = dc_link_dp_perform_link_training(link, &pipe_ctx->link_res, @@ -3080,7 +3094,7 @@ struct dc_link_settings dp_get_max_link_cap(struct dc_link *link) * account for lttpr repeaters cap * notes: repeaters do not snoop in the DPRX Capabilities addresses (3.6.3). */ - if (link->lttpr_mode != LTTPR_MODE_NON_LTTPR) { + if (dp_is_lttpr_present(link)) { if (link->dpcd_caps.lttpr_caps.max_lane_count < max_link_cap.lane_count) max_link_cap.lane_count = link->dpcd_caps.lttpr_caps.max_lane_count; lttpr_max_link_rate = get_lttpr_max_link_rate(link); @@ -3234,7 +3248,7 @@ static bool dp_verify_link_cap( cur_link_settings = max_link_settings; /* Grant extended timeout request */ - if ((link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (link->dpcd_caps.lttpr_caps.max_ext_timeout > 0)) { + if (dp_is_lttpr_present(link) && link->dpcd_caps.lttpr_caps.max_ext_timeout > 0) { uint8_t grant = link->dpcd_caps.lttpr_caps.max_ext_timeout & 0x80; core_link_write_dpcd(link, DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT, &grant, sizeof(grant)); @@ -4095,8 +4109,13 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link) &dpcd_lane_adjustment[0].raw, sizeof(dpcd_lane_adjustment)); + /* prepare link training settings */ + link_training_settings.link_settings = link->cur_link_settings; + + link_training_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link->cur_link_settings); + if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) && - link->lttpr_mode == LTTPR_MODE_TRANSPARENT) + link_training_settings.lttpr_mode == LTTPR_MODE_TRANSPARENT) dp_fixed_vs_pe_read_lane_adjust( link, link_training_settings.dpcd_lane_settings); @@ -4203,9 +4222,6 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link) test_pattern_size); } - /* prepare link training settings */ - link_training_settings.link_settings = link->cur_link_settings; - for (lane = 0; lane < (unsigned int)(link->cur_link_settings.lane_count); lane++) { @@ -4518,25 +4534,15 @@ void dc_link_dp_handle_link_loss(struct dc_link *link) for (i = 0; i < MAX_PIPES; i++) { pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i]; if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off && - pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) { - if (link->dc->hwss.update_phy_state) - link->dc->hwss.update_phy_state(link->dc->current_state, - pipe_ctx, TX_OFF_SYMCLK_OFF); - else - core_link_disable_stream(pipe_ctx); - } + pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) + core_link_disable_stream(pipe_ctx); } for (i = 0; i < MAX_PIPES; i++) { pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i]; if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off && - pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) { - if (link->dc->hwss.update_phy_state) - link->dc->hwss.update_phy_state(link->dc->current_state, - pipe_ctx, TX_ON_SYMCLK_ON); - else - core_link_enable_stream(link->dc->current_state, pipe_ctx); - } + pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) + core_link_enable_stream(link->dc->current_state, pipe_ctx); } } @@ -5025,125 +5031,151 @@ static bool dpcd_read_sink_ext_caps(struct dc_link *link) return true; } -/* Logic to determine LTTPR mode */ -static void determine_lttpr_mode(struct dc_link *link) +bool dp_retrieve_lttpr_cap(struct dc_link *link) { - bool allow_lttpr_non_transparent_mode = 0; - bool vbios_lttpr_enable = link->dc->caps.vbios_lttpr_enable; + uint8_t lttpr_dpcd_data[8]; + enum dc_status status = DC_ERROR_UNEXPECTED; + bool is_lttpr_present = false; + + /* Logic to determine LTTPR support*/ bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware; - if (link->ctx->dc->debug.lttpr_mode_override != 0) { - link->lttpr_mode = link->ctx->dc->debug.lttpr_mode_override; - return; - } + if (!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support) + return false; - if ((link->dc->config.allow_lttpr_non_transparent_mode.bits.DP2_0 && - link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED)) { - allow_lttpr_non_transparent_mode = 1; - } else if (link->dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A && - !link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED) { - allow_lttpr_non_transparent_mode = 1; + /* By reading LTTPR capability, RX assumes that we will enable + * LTTPR extended aux timeout if LTTPR is present. + */ + status = core_link_read_dpcd(link, + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, + lttpr_dpcd_data, + sizeof(lttpr_dpcd_data)); + + link->dpcd_caps.lttpr_caps.revision.raw = + lttpr_dpcd_data[DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.max_link_rate = + lttpr_dpcd_data[DP_MAX_LINK_RATE_PHY_REPEATER - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.phy_repeater_cnt = + lttpr_dpcd_data[DP_PHY_REPEATER_CNT - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.max_lane_count = + lttpr_dpcd_data[DP_MAX_LANE_COUNT_PHY_REPEATER - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.mode = + lttpr_dpcd_data[DP_PHY_REPEATER_MODE - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.max_ext_timeout = + lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + link->dpcd_caps.lttpr_caps.main_link_channel_coding.raw = + lttpr_dpcd_data[DP_MAIN_LINK_CHANNEL_CODING_PHY_REPEATER - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + link->dpcd_caps.lttpr_caps.supported_128b_132b_rates.raw = + lttpr_dpcd_data[DP_PHY_REPEATER_128B132B_RATES - + DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; + + /* If this chip cap is set, at least one retimer must exist in the chain + * Override count to 1 if we receive a known bad count (0 or an invalid value) + */ + if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN && + (dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) == 0)) { + ASSERT(0); + link->dpcd_caps.lttpr_caps.phy_repeater_cnt = 0x80; + DC_LOG_DC("lttpr_caps forced phy_repeater_cnt = %d\n", link->dpcd_caps.lttpr_caps.phy_repeater_cnt); } - link->lttpr_mode = LTTPR_MODE_NON_LTTPR; - if (vbios_lttpr_enable && vbios_lttpr_interop) - link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT; - else if (!vbios_lttpr_enable && vbios_lttpr_interop) { - if (allow_lttpr_non_transparent_mode) - link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT; - else - link->lttpr_mode = LTTPR_MODE_TRANSPARENT; - } else if (!vbios_lttpr_enable && !vbios_lttpr_interop) { - if (!allow_lttpr_non_transparent_mode || !link->dc->caps.extended_aux_timeout_support) - link->lttpr_mode = LTTPR_MODE_NON_LTTPR; - else - link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT; - } + /* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */ + is_lttpr_present = dp_is_lttpr_present(link); -#if defined(CONFIG_DRM_AMD_DC_DCN) - /* Check DP tunnel LTTPR mode debug option. */ - if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA && - link->dc->debug.dpia_debug.bits.force_non_lttpr) - link->lttpr_mode = LTTPR_MODE_NON_LTTPR; -#endif + if (is_lttpr_present) + CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: "); + + DC_LOG_DC("is_lttpr_present = %d\n", is_lttpr_present); + return is_lttpr_present; } -bool dp_retrieve_lttpr_cap(struct dc_link *link) +bool dp_is_lttpr_present(struct dc_link *link) { - uint8_t lttpr_dpcd_data[8]; - enum dc_status status = DC_ERROR_UNEXPECTED; - bool is_lttpr_present = false; + return (dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) != 0 && + link->dpcd_caps.lttpr_caps.max_lane_count > 0 && + link->dpcd_caps.lttpr_caps.max_lane_count <= 4 && + link->dpcd_caps.lttpr_caps.revision.raw >= 0x14); +} - memset(lttpr_dpcd_data, '\0', sizeof(lttpr_dpcd_data)); +enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link, struct dc_link_settings *link_setting) +{ + enum dp_link_encoding encoding = dp_get_link_encoding_format(link_setting); - /* Logic to determine LTTPR mode*/ - determine_lttpr_mode(link); + if (encoding == DP_8b_10b_ENCODING) + return dp_decide_8b_10b_lttpr_mode(link); + else if (encoding == DP_128b_132b_ENCODING) + return dp_decide_128b_132b_lttpr_mode(link); - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT || link->lttpr_mode == LTTPR_MODE_TRANSPARENT) { - if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) && - !link->dc->debug.disable_fixed_vs_aux_timeout_wa) { - /* Fixed VS workaround for AUX timeout */ - const uint32_t fixed_vs_address = 0xF004F; - const uint8_t fixed_vs_data[4] = {0x1, 0x22, 0x63, 0xc}; + ASSERT(0); + return LTTPR_MODE_NON_LTTPR; +} - core_link_write_dpcd( - link, - fixed_vs_address, - fixed_vs_data, - sizeof(fixed_vs_data)); +void dp_get_lttpr_mode_override(struct dc_link *link, enum lttpr_mode *override) +{ + if (!dp_is_lttpr_present(link)) + return; + + if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_TRANSPARENT) { + *override = LTTPR_MODE_TRANSPARENT; + } else if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_NON_TRANSPARENT) { + *override = LTTPR_MODE_NON_TRANSPARENT; + } else if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_NON_LTTPR) { + *override = LTTPR_MODE_NON_LTTPR; + } + DC_LOG_DC("lttpr_mode_override chose LTTPR_MODE = %d\n", (uint8_t)(*override)); +} + +enum lttpr_mode dp_decide_8b_10b_lttpr_mode(struct dc_link *link) +{ + bool is_lttpr_present = dp_is_lttpr_present(link); + bool vbios_lttpr_force_non_transparent = link->dc->caps.vbios_lttpr_enable; + bool vbios_lttpr_aware = link->dc->caps.vbios_lttpr_aware; + + if (!is_lttpr_present) + return LTTPR_MODE_NON_LTTPR; + + if (vbios_lttpr_aware) { + if (vbios_lttpr_force_non_transparent) { + DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT due to VBIOS DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE set to 1.\n"); + return LTTPR_MODE_NON_TRANSPARENT; + } else { + DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT by default due to VBIOS not set DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE set to 1.\n"); + return LTTPR_MODE_TRANSPARENT; } + } - /* By reading LTTPR capability, RX assumes that we will enable - * LTTPR extended aux timeout if LTTPR is present. - */ - status = core_link_read_dpcd( - link, - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, - lttpr_dpcd_data, - sizeof(lttpr_dpcd_data)); - - link->dpcd_caps.lttpr_caps.revision.raw = - lttpr_dpcd_data[DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.max_link_rate = - lttpr_dpcd_data[DP_MAX_LINK_RATE_PHY_REPEATER - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.phy_repeater_cnt = - lttpr_dpcd_data[DP_PHY_REPEATER_CNT - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.max_lane_count = - lttpr_dpcd_data[DP_MAX_LANE_COUNT_PHY_REPEATER - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.mode = - lttpr_dpcd_data[DP_PHY_REPEATER_MODE - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.max_ext_timeout = - lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - link->dpcd_caps.lttpr_caps.main_link_channel_coding.raw = - lttpr_dpcd_data[DP_MAIN_LINK_CHANNEL_CODING_PHY_REPEATER - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - link->dpcd_caps.lttpr_caps.supported_128b_132b_rates.raw = - lttpr_dpcd_data[DP_PHY_REPEATER_128B132B_RATES - - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV]; - - /* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */ - is_lttpr_present = (link->dpcd_caps.lttpr_caps.max_lane_count > 0 && - link->dpcd_caps.lttpr_caps.max_lane_count <= 4 && - link->dpcd_caps.lttpr_caps.revision.raw >= 0x14); - if (is_lttpr_present) { - CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: "); - configure_lttpr_mode_transparent(link); - } else - link->lttpr_mode = LTTPR_MODE_NON_LTTPR; + if (link->dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A && + link->dc->caps.extended_aux_timeout_support) { + DC_LOG_DC("chose LTTPR_MODE_NON_TRANSPARENT by default and dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A set to 1.\n"); + return LTTPR_MODE_NON_TRANSPARENT; } - return is_lttpr_present; + + DC_LOG_DC("chose LTTPR_MODE_NON_LTTPR.\n"); + return LTTPR_MODE_NON_LTTPR; +} + +enum lttpr_mode dp_decide_128b_132b_lttpr_mode(struct dc_link *link) +{ + enum lttpr_mode mode = LTTPR_MODE_NON_LTTPR; + + if (dp_is_lttpr_present(link)) + mode = LTTPR_MODE_NON_TRANSPARENT; + + DC_LOG_DC("128b_132b chose LTTPR_MODE %d.\n", mode); + return mode; } static bool get_usbc_cable_id(struct dc_link *link, union dp_cable_id *cable_id) @@ -5161,9 +5193,10 @@ static bool get_usbc_cable_id(struct dc_link *link, union dp_cable_id *cable_id) cmd.cable_id.data.input.phy_inst = resource_transmitter_to_phy_idx( link->dc, link->link_enc->transmitter); if (dc_dmub_srv_cmd_with_reply_data(link->ctx->dmub_srv, &cmd) && - cmd.cable_id.header.ret_status == 1) + cmd.cable_id.header.ret_status == 1) { cable_id->raw = cmd.cable_id.data.output_raw; - + DC_LOG_DC("usbc_cable_id = %d.\n", cable_id->raw); + } return cmd.cable_id.header.ret_status == 1; } @@ -5205,13 +5238,17 @@ static enum dc_status wa_try_to_wake_dprx(struct dc_link *link, uint64_t timeout uint64_t current_ts = 0; uint64_t time_taken_ms = 0; enum dc_connection_type type = dc_connection_none; + bool lttpr_present; + bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware; - determine_lttpr_mode(link); + lttpr_present = dp_is_lttpr_present(link) || + (!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support); + DC_LOG_DC("lttpr_present = %d.\n", lttpr_present ? 1 : 0); /* Issue an AUX read to test DPRX responsiveness. If LTTPR is supported the first read is expected to * be to determine LTTPR capabilities. Otherwise trying to read power state should be an innocuous AUX read. */ - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT || link->lttpr_mode == LTTPR_MODE_TRANSPARENT) + if (lttpr_present) status = core_link_read_dpcd( link, DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, @@ -5341,6 +5378,10 @@ static bool retrieve_link_cap(struct dc_link *link) } is_lttpr_present = dp_retrieve_lttpr_cap(link); + + if (is_lttpr_present) + configure_lttpr_mode_transparent(link); + /* Read DP tunneling information. */ status = dpcd_get_tunneling_device_data(link); @@ -5770,7 +5811,7 @@ void detect_edp_sink_caps(struct dc_link *link) * Per VESA eDP spec, "The DPCD revision for eDP v1.4 is 13h" */ if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_13 && - (link->dc->debug.optimize_edp_link_rate || + (link->panel_config.ilr.optimize_edp_link_rate || link->reported_link_cap.link_rate == LINK_RATE_UNKNOWN)) { // Read DPCD 00010h - 0001Fh 16 bytes at one shot core_link_read_dpcd(link, DP_SUPPORTED_LINK_RATES, @@ -6092,7 +6133,7 @@ bool dc_link_dp_set_test_pattern( /* Set DPCD Lane Settings before running test pattern */ if (p_link_settings != NULL) { if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) && - link->lttpr_mode == LTTPR_MODE_TRANSPARENT) { + p_link_settings->lttpr_mode == LTTPR_MODE_TRANSPARENT) { dp_fixed_vs_pe_set_retimer_lane_settings( link, p_link_settings->dpcd_lane_settings, @@ -6719,7 +6760,7 @@ bool is_edp_ilr_optimization_required(struct dc_link *link, struct dc_crtc_timin ASSERT(link || crtc_timing); // invalid input if (link->dpcd_caps.edp_supported_link_rates_count == 0 || - !link->dc->debug.optimize_edp_link_rate) + !link->panel_config.ilr.optimize_edp_link_rate) return false; @@ -7069,68 +7110,16 @@ void dp_enable_link_phy( enum clock_source_id clock_source, const struct dc_link_settings *link_settings) { - struct dc *dc = link->ctx->dc; - struct dmcu *dmcu = dc->res_pool->dmcu; - struct pipe_ctx *pipes = - link->dc->current_state->res_ctx.pipe_ctx; - struct clock_source *dp_cs = - link->dc->res_pool->dp_clock_source; - const struct link_hwss *link_hwss = get_link_hwss(link, link_res); - unsigned int i; - - if (link->connector_signal == SIGNAL_TYPE_EDP) { - if (!link->dc->config.edp_no_power_sequencing) - link->dc->hwss.edp_power_control(link, true); - link->dc->hwss.edp_wait_for_hpd_ready(link, true); - } - - /* If the current pixel clock source is not DTO(happens after - * switching from HDMI passive dongle to DP on the same connector), - * switch the pixel clock source to DTO. - */ - for (i = 0; i < MAX_PIPES; i++) { - if (pipes[i].stream != NULL && - pipes[i].stream->link == link) { - if (pipes[i].clock_source != NULL && - pipes[i].clock_source->id != CLOCK_SOURCE_ID_DP_DTO) { - pipes[i].clock_source = dp_cs; - pipes[i].stream_res.pix_clk_params.requested_pix_clk_100hz = - pipes[i].stream->timing.pix_clk_100hz; - pipes[i].clock_source->funcs->program_pix_clk( - pipes[i].clock_source, - &pipes[i].stream_res.pix_clk_params, - dp_get_link_encoding_format(link_settings), - &pipes[i].pll_settings); - } - } - } - link->cur_link_settings = *link_settings; - - if (dp_get_link_encoding_format(link_settings) == DP_8b_10b_ENCODING) { - if (dc->clk_mgr->funcs->notify_link_rate_change) - dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link); - } - - if (dmcu != NULL && dmcu->funcs->lock_phy) - dmcu->funcs->lock_phy(dmcu); - - if (link_hwss->ext.enable_dp_link_output) - link_hwss->ext.enable_dp_link_output(link, link_res, signal, - clock_source, link_settings); - - if (dmcu != NULL && dmcu->funcs->unlock_phy) - dmcu->funcs->unlock_phy(dmcu); - - dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY); + link->dc->hwss.enable_dp_link_output(link, link_res, signal, + clock_source, link_settings); dp_receiver_power_ctrl(link, true); } void edp_add_delay_for_T9(struct dc_link *link) { - if (link->local_sink && - link->local_sink->edid_caps.panel_patch.extra_delay_backlight_off > 0) - udelay(link->local_sink->edid_caps.panel_patch.extra_delay_backlight_off * 1000); + if (link && link->panel_config.pps.extra_delay_backlight_off > 0) + udelay(link->panel_config.pps.extra_delay_backlight_off * 1000); } bool edp_receiver_ready_T9(struct dc_link *link) @@ -7186,9 +7175,8 @@ bool edp_receiver_ready_T7(struct dc_link *link) } while (time_taken_in_ns < 50 * 1000000); //MAx T7 is 50ms } - if (link->local_sink && - link->local_sink->edid_caps.panel_patch.extra_t7_ms > 0) - udelay(link->local_sink->edid_caps.panel_patch.extra_t7_ms * 1000); + if (link && link->panel_config.pps.extra_t7_ms > 0) + udelay(link->panel_config.pps.extra_t7_ms * 1000); return result; } @@ -7197,29 +7185,11 @@ void dp_disable_link_phy(struct dc_link *link, const struct link_resource *link_ enum signal_type signal) { struct dc *dc = link->ctx->dc; - struct dmcu *dmcu = dc->res_pool->dmcu; - const struct link_hwss *link_hwss = get_link_hwss(link, link_res); if (!link->wa_flags.dp_keep_receiver_powered) dp_receiver_power_ctrl(link, false); - if (signal == SIGNAL_TYPE_EDP) { - if (link->dc->hwss.edp_backlight_control) - link->dc->hwss.edp_backlight_control(link, false); - if (link_hwss->ext.disable_dp_link_output) - link_hwss->ext.disable_dp_link_output(link, link_res, signal); - link->dc->hwss.edp_power_control(link, false); - } else { - if (dmcu != NULL && dmcu->funcs->lock_phy) - dmcu->funcs->lock_phy(dmcu); - if (link_hwss->ext.disable_dp_link_output) - link_hwss->ext.disable_dp_link_output(link, link_res, signal); - if (dmcu != NULL && dmcu->funcs->unlock_phy) - dmcu->funcs->unlock_phy(dmcu); - } - - dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY); - + dc->hwss.disable_link_output(link, link_res, signal); /* Clear current link setting.*/ memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings)); @@ -7285,7 +7255,7 @@ void dp_set_hw_lane_settings( { const struct link_hwss *link_hwss = get_link_hwss(link, link_res); - if ((link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && !is_immediate_downstream(link, offset)) + if ((link_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && !is_immediate_downstream(link, offset)) return; if (link_hwss->ext.set_dp_lane_settings) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c index 468e39589ed8cc2e8218d0b86731381690f1dc7b..74e36b34d3f733b554195742321d1fe4ca9b4574 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c @@ -115,12 +115,14 @@ static enum link_training_result dpia_configure_link( DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) configuring\n - LTTPR mode(%d)\n", __func__, link->link_id.enum_id - ENUM_ID_1, - link->lttpr_mode); + lt_settings->lttpr_mode); dp_decide_training_settings(link, link_setting, lt_settings); + dp_get_lttpr_mode_override(link, <_settings->lttpr_mode); + status = dpcd_configure_channel_coding(link, lt_settings); if (status != DC_OK && link->is_hpd_pending) return LINK_TRAINING_ABORT; @@ -178,7 +180,7 @@ static uint8_t dpia_build_set_config_data(enum dpia_set_config_type type, switch (type) { case DPIA_SET_CFG_SET_LINK: - data.set_link.mode = link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT ? 1 : 0; + data.set_link.mode = lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT ? 1 : 0; break; case DPIA_SET_CFG_SET_PHY_TEST_MODE: break; @@ -553,7 +555,7 @@ static enum link_training_result dpia_training_cr_phase( { enum link_training_result result = LINK_TRAINING_CR_FAIL_LANE0; - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) result = dpia_training_cr_non_transparent(link, link_res, lt_settings, hop); else result = dpia_training_cr_transparent(link, link_res, lt_settings); @@ -830,7 +832,7 @@ static enum link_training_result dpia_training_eq_phase( { enum link_training_result result; - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) result = dpia_training_eq_non_transparent(link, link_res, lt_settings, hop); else result = dpia_training_eq_transparent(link, link_res, lt_settings); @@ -870,13 +872,14 @@ static enum dc_status dpcd_clear_lt_pattern(struct dc_link *link, uint32_t hop) * @param hop The Hop in display path. DPRX = 0. */ static enum link_training_result dpia_training_end(struct dc_link *link, + struct link_training_settings *lt_settings, uint32_t hop) { enum link_training_result result = LINK_TRAINING_SUCCESS; uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */ enum dc_status status; - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { + if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) { repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt); if (hop == repeater_cnt) { /* DPTX-to-DPIA */ @@ -916,7 +919,7 @@ static enum link_training_result dpia_training_end(struct dc_link *link, link->link_id.enum_id - ENUM_ID_1, hop, result, - link->lttpr_mode); + lt_settings->lttpr_mode); return result; } @@ -928,7 +931,9 @@ static enum link_training_result dpia_training_end(struct dc_link *link, * @param link DPIA link being trained. * @param hop The Hop in display path. DPRX = 0. */ -static void dpia_training_abort(struct dc_link *link, uint32_t hop) +static void dpia_training_abort(struct dc_link *link, + struct link_training_settings *lt_settings, + uint32_t hop) { uint8_t data = 0; uint32_t dpcd_tps_offset = DP_TRAINING_PATTERN_SET; @@ -936,7 +941,7 @@ static void dpia_training_abort(struct dc_link *link, uint32_t hop) DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) aborting\n - LTTPR mode(%d)\n - HPD(%d)\n", __func__, link->link_id.enum_id - ENUM_ID_1, - link->lttpr_mode, + lt_settings->lttpr_mode, link->is_hpd_pending); /* Abandon clean-up if sink unplugged. */ @@ -964,12 +969,16 @@ enum link_training_result dc_link_dpia_perform_link_training( uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */ int8_t repeater_id; /* Current hop. */ + struct dc_link_settings link_settings = *link_setting; // non-const copy to pass in + + lt_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link_settings); + /* Configure link as prescribed in link_setting and set LTTPR mode. */ result = dpia_configure_link(link, link_res, link_setting, <_settings); if (result != LINK_TRAINING_SUCCESS) return result; - if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) + if (lt_settings.lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt); /* Train each hop in turn starting with the one closest to DPTX. @@ -987,7 +996,7 @@ enum link_training_result dc_link_dpia_perform_link_training( break; /* Stop training hop. */ - result = dpia_training_end(link, repeater_id); + result = dpia_training_end(link, <_settings, repeater_id); if (result != LINK_TRAINING_SUCCESS) break; } @@ -1001,9 +1010,9 @@ enum link_training_result dc_link_dpia_perform_link_training( msleep(5); result = dp_check_link_loss_status(link, <_settings); } else if (result == LINK_TRAINING_ABORT) { - dpia_training_abort(link, repeater_id); + dpia_training_abort(link, <_settings, repeater_id); } else { - dpia_training_end(link, repeater_id); + dpia_training_end(link, <_settings, repeater_id); } return result; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 29f27e3fe3ac8e6809e709005b181393ba047e8f..fd8db482e56f9dd6c18f46107a32ecc39842ebd4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1747,7 +1747,6 @@ bool dc_remove_plane_from_context( for (i = 0; i < stream_status->plane_count; i++) { if (stream_status->plane_states[i] == plane_state) { - dc_plane_state_release(stream_status->plane_states[i]); break; } @@ -3581,6 +3580,23 @@ void check_syncd_pipes_for_disabled_master_pipe(struct dc *dc, } } +void reset_sync_context_for_pipe(const struct dc *dc, + struct dc_state *context, + uint8_t pipe_idx) +{ + int i; + struct pipe_ctx *pipe_ctx_reset; + + /* reset the otg sync context for the pipe and its slave pipes if any */ + for (i = 0; i < dc->res_pool->pipe_count; i++) { + pipe_ctx_reset = &context->res_ctx.pipe_ctx[i]; + + if (((GET_PIPE_SYNCD_FROM_PIPE(pipe_ctx_reset) == pipe_idx) && + IS_PIPE_SYNCD_VALID(pipe_ctx_reset)) || (i == pipe_idx)) + SET_PIPE_SYNCD_TO_PIPE(pipe_ctx_reset, i); + } +} + uint8_t resource_transmitter_to_phy_idx(const struct dc *dc, enum transmitter transmitter) { /* TODO - get transmitter to phy idx mapping from DMUB */ @@ -3645,3 +3661,77 @@ const struct link_hwss *get_link_hwss(const struct dc_link *link, else return get_virtual_link_hwss(); } + +bool is_h_timing_divisible_by_2(struct dc_stream_state *stream) +{ + bool divisible = false; + uint16_t h_blank_start = 0; + uint16_t h_blank_end = 0; + + if (stream) { + h_blank_start = stream->timing.h_total - stream->timing.h_front_porch; + h_blank_end = h_blank_start - stream->timing.h_addressable; + + /* HTOTAL, Hblank start/end, and Hsync start/end all must be + * divisible by 2 in order for the horizontal timing params + * to be considered divisible by 2. Hsync start is always 0. + */ + divisible = (stream->timing.h_total % 2 == 0) && + (h_blank_start % 2 == 0) && + (h_blank_end % 2 == 0) && + (stream->timing.h_sync_width % 2 == 0); + } + return divisible; +} + +bool dc_resource_acquire_secondary_pipe_for_mpc_odm( + const struct dc *dc, + struct dc_state *state, + struct pipe_ctx *pri_pipe, + struct pipe_ctx *sec_pipe, + bool odm) +{ + int pipe_idx = sec_pipe->pipe_idx; + struct pipe_ctx *sec_top, *sec_bottom, *sec_next, *sec_prev; + const struct resource_pool *pool = dc->res_pool; + + sec_top = sec_pipe->top_pipe; + sec_bottom = sec_pipe->bottom_pipe; + sec_next = sec_pipe->next_odm_pipe; + sec_prev = sec_pipe->prev_odm_pipe; + + *sec_pipe = *pri_pipe; + + sec_pipe->top_pipe = sec_top; + sec_pipe->bottom_pipe = sec_bottom; + sec_pipe->next_odm_pipe = sec_next; + sec_pipe->prev_odm_pipe = sec_prev; + + sec_pipe->pipe_idx = pipe_idx; + sec_pipe->plane_res.mi = pool->mis[pipe_idx]; + sec_pipe->plane_res.hubp = pool->hubps[pipe_idx]; + sec_pipe->plane_res.ipp = pool->ipps[pipe_idx]; + sec_pipe->plane_res.xfm = pool->transforms[pipe_idx]; + sec_pipe->plane_res.dpp = pool->dpps[pipe_idx]; + sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst; + sec_pipe->stream_res.dsc = NULL; + if (odm) { + if (!sec_pipe->top_pipe) + sec_pipe->stream_res.opp = pool->opps[pipe_idx]; + else + sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp; + if (sec_pipe->stream->timing.flags.DSC == 1) { +#if defined(CONFIG_DRM_AMD_DC_DCN) + dcn20_acquire_dsc(dc, &state->res_ctx, &sec_pipe->stream_res.dsc, pipe_idx); +#endif + ASSERT(sec_pipe->stream_res.dsc); + if (sec_pipe->stream_res.dsc == NULL) + return false; + } +#if defined(CONFIG_DRM_AMD_DC_DCN) + dcn20_build_mapped_resource(dc, state, sec_pipe->stream); +#endif + } + + return true; +} \ No newline at end of file diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index f62d50901d92e8ea225d98575583481599c49bbc..38d71b5c1f2d511274192acedfd9300a308ec3dc 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -30,6 +30,7 @@ #include "resource.h" #include "ipp.h" #include "timing_generator.h" +#include "dc_dmub_srv.h" #define DC_LOGGER dc->ctx->logger @@ -275,6 +276,8 @@ static void program_cursor_attributes( } dc->hwss.set_cursor_attribute(pipe_ctx); + + dc_send_update_cursor_info_to_dmu(pipe_ctx, i); if (dc->hwss.set_cursor_sdr_white_level) dc->hwss.set_cursor_sdr_white_level(pipe_ctx); } @@ -329,7 +332,7 @@ bool dc_stream_set_cursor_attributes( dc = stream->ctx->dc; - if (attributes->height * attributes->width * 4 > 16384) + if (dc->debug.allow_sw_cursor_fallback && attributes->height * attributes->width * 4 > 16384) if (stream->mall_stream_config.type == SUBVP_MAIN) return false; @@ -381,6 +384,8 @@ static void program_cursor_position( } dc->hwss.set_cursor_position(pipe_ctx); + + dc_send_update_cursor_info_to_dmu(pipe_ctx, i); } if (pipe_to_program) @@ -521,7 +526,7 @@ bool dc_stream_remove_writeback(struct dc *dc, /* remove writeback info for disabled writeback pipes from stream */ for (i = 0, j = 0; i < stream->num_wb_info; i++) { if (stream->writeback_info[i].wb_enabled) { - if (i != j) + if (j < i) /* trim the array */ stream->writeback_info[j] = stream->writeback_info[i]; j++; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 75dbc665f4355cc6b118cd70f0ab8d4b9676540a..bfc5474c0f4c90fdcae2fc44665ef31583ee752a 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -47,7 +47,7 @@ struct aux_payload; struct set_config_cmd_payload; struct dmub_notification; -#define DC_VER "3.2.201" +#define DC_VER "3.2.207" #define MAX_SURFACES 3 #define MAX_PLANES 6 @@ -406,6 +406,9 @@ struct dc_config { bool ignore_dpref_ss; bool enable_mipi_converter_optimization; bool use_default_clock_table; + bool force_bios_enable_lttpr; + uint8_t force_bios_fixed_vs; + }; enum visual_confirm { @@ -818,7 +821,6 @@ struct dc_debug_options { /* Enable dmub aux for legacy ddc */ bool enable_dmub_aux_for_legacy_ddc; bool disable_fams; - bool optimize_edp_link_rate; /* eDP ILR */ /* FEC/PSR1 sequence enable delay in 100us */ uint8_t fec_enable_delay_in100us; bool enable_driver_sequence_debug; @@ -830,6 +832,10 @@ struct dc_debug_options { bool disable_fixed_vs_aux_timeout_wa; bool force_disable_subvp; bool force_subvp_mclk_switch; + bool allow_sw_cursor_fallback; + unsigned int force_subvp_num_ways; + unsigned int force_mall_ss_num_ways; + bool alloc_extra_way_for_cursor; bool force_usr_allow; /* uses value at boot and disables switch */ bool disable_dtb_ref_clk_switch; @@ -843,6 +849,7 @@ struct dc_debug_options { bool use_legacy_soc_bb_mechanism; bool exit_idle_opt_for_cursor_updates; bool enable_single_display_2to1_odm_policy; + bool enable_double_buffered_dsc_pg_support; bool enable_dp_dig_pixel_rate_div_policy; enum lttpr_mode lttpr_mode_override; }; @@ -1114,6 +1121,7 @@ union surface_update_flags { uint32_t clock_change:1; uint32_t stereo_format_change:1; uint32_t lut_3d:1; + uint32_t tmz_changed:1; uint32_t full_update:1; } bits; @@ -1183,6 +1191,8 @@ struct dc_plane_state { enum dc_irq_source irq_source; struct kref refcount; struct tg_color visual_confirm_color; + + bool is_statically_allocated; }; struct dc_plane_info { @@ -1602,6 +1612,9 @@ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc, uint8_t mst_alloc_slots, uint8_t *mst_slots_in_use); +void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc, + uint32_t hpd_int_enable); + /******************************************************************************* * DSC Interfaces ******************************************************************************/ diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 7b765efe08253b01d1a2751f30c0586a376c1dfc..0541e87e4f38954dd9728e8767b4b8a140174698 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -30,6 +30,7 @@ #include "dc_hw_types.h" #include "core_types.h" #include "../basics/conversion.h" +#include "cursor_reg_cache.h" #define CTX dc_dmub_srv->ctx #define DC_LOGGER CTX->logger @@ -450,44 +451,42 @@ static void populate_subvp_cmd_drr_info(struct dc *dc, struct dc_crtc_timing *main_timing = &subvp_pipe->stream->timing; struct dc_crtc_timing *phantom_timing = &subvp_pipe->stream->mall_stream_config.paired_stream->timing; struct dc_crtc_timing *drr_timing = &vblank_pipe->stream->timing; - int16_t drr_frame_us = 0; - int16_t min_drr_supported_us = 0; - int16_t max_drr_supported_us = 0; - int16_t max_drr_vblank_us = 0; - int16_t max_drr_mallregion_us = 0; - int16_t mall_region_us = 0; - int16_t prefetch_us = 0; - int16_t subvp_active_us = 0; - int16_t drr_active_us = 0; - int16_t min_vtotal_supported = 0; - int16_t max_vtotal_supported = 0; + uint16_t drr_frame_us = 0; + uint16_t min_drr_supported_us = 0; + uint16_t max_drr_supported_us = 0; + uint16_t max_drr_vblank_us = 0; + uint16_t max_drr_mallregion_us = 0; + uint16_t mall_region_us = 0; + uint16_t prefetch_us = 0; + uint16_t subvp_active_us = 0; + uint16_t drr_active_us = 0; + uint16_t min_vtotal_supported = 0; + uint16_t max_vtotal_supported = 0; pipe_data->pipe_config.vblank_data.drr_info.drr_in_use = true; pipe_data->pipe_config.vblank_data.drr_info.use_ramping = false; // for now don't use ramping pipe_data->pipe_config.vblank_data.drr_info.drr_window_size_ms = 4; // hardcode 4ms DRR window for now - drr_frame_us = div64_s64(drr_timing->v_total * drr_timing->h_total, - (int64_t)(drr_timing->pix_clk_100hz * 100) * 1000000); + drr_frame_us = div64_u64(((uint64_t)drr_timing->v_total * drr_timing->h_total * 1000000), + (((uint64_t)drr_timing->pix_clk_100hz * 100))); // P-State allow width and FW delays already included phantom_timing->v_addressable - mall_region_us = div64_s64(phantom_timing->v_addressable * phantom_timing->h_total, - (int64_t)(phantom_timing->pix_clk_100hz * 100) * 1000000); + mall_region_us = div64_u64(((uint64_t)phantom_timing->v_addressable * phantom_timing->h_total * 1000000), + (((uint64_t)phantom_timing->pix_clk_100hz * 100))); min_drr_supported_us = drr_frame_us + mall_region_us + SUBVP_DRR_MARGIN_US; - min_vtotal_supported = div64_s64(drr_timing->pix_clk_100hz * 100 * - (div64_s64((int64_t)min_drr_supported_us, 1000000)), - (int64_t)drr_timing->h_total); - - prefetch_us = div64_s64((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total, - (int64_t)(phantom_timing->pix_clk_100hz * 100) * 1000000 + - dc->caps.subvp_prefetch_end_to_mall_start_us); - subvp_active_us = div64_s64(main_timing->v_addressable * main_timing->h_total, - (int64_t)(main_timing->pix_clk_100hz * 100) * 1000000); - drr_active_us = div64_s64(drr_timing->v_addressable * drr_timing->h_total, - (int64_t)(drr_timing->pix_clk_100hz * 100) * 1000000); - max_drr_vblank_us = div64_s64((int64_t)(subvp_active_us - prefetch_us - drr_active_us), 2) + drr_active_us; + min_vtotal_supported = div64_u64(((uint64_t)drr_timing->pix_clk_100hz * 100 * min_drr_supported_us), + (((uint64_t)drr_timing->h_total * 1000000))); + + prefetch_us = div64_u64(((uint64_t)(phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total * 1000000), + (((uint64_t)phantom_timing->pix_clk_100hz * 100) + dc->caps.subvp_prefetch_end_to_mall_start_us)); + subvp_active_us = div64_u64(((uint64_t)main_timing->v_addressable * main_timing->h_total * 1000000), + (((uint64_t)main_timing->pix_clk_100hz * 100))); + drr_active_us = div64_u64(((uint64_t)drr_timing->v_addressable * drr_timing->h_total * 1000000), + (((uint64_t)drr_timing->pix_clk_100hz * 100))); + max_drr_vblank_us = div64_u64((subvp_active_us - prefetch_us - drr_active_us), 2) + drr_active_us; max_drr_mallregion_us = subvp_active_us - prefetch_us - mall_region_us; max_drr_supported_us = max_drr_vblank_us > max_drr_mallregion_us ? max_drr_vblank_us : max_drr_mallregion_us; - max_vtotal_supported = div64_s64(drr_timing->pix_clk_100hz * 100 * (div64_s64((int64_t)max_drr_supported_us, 1000000)), - (int64_t)drr_timing->h_total); + max_vtotal_supported = div64_u64(((uint64_t)drr_timing->pix_clk_100hz * 100 * max_drr_supported_us), + (((uint64_t)drr_timing->h_total * 1000000))); pipe_data->pipe_config.vblank_data.drr_info.min_vtotal_supported = min_vtotal_supported; pipe_data->pipe_config.vblank_data.drr_info.max_vtotal_supported = max_vtotal_supported; @@ -581,10 +580,12 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc, struct dc_crtc_timing *phantom_timing1 = &subvp_pipes[1]->stream->mall_stream_config.paired_stream->timing; struct dmub_cmd_fw_assisted_mclk_switch_pipe_data_v2 *pipe_data = NULL; - subvp0_prefetch_us = div64_s64((phantom_timing0->v_total - phantom_timing0->v_front_porch) * phantom_timing0->h_total, - (int64_t)(phantom_timing0->pix_clk_100hz * 100) * 1000000 + dc->caps.subvp_prefetch_end_to_mall_start_us); - subvp1_prefetch_us = div64_s64((phantom_timing1->v_total - phantom_timing1->v_front_porch) * phantom_timing1->h_total, - (int64_t)(phantom_timing1->pix_clk_100hz * 100) * 1000000 + dc->caps.subvp_prefetch_end_to_mall_start_us); + subvp0_prefetch_us = div64_u64(((uint64_t)(phantom_timing0->v_total - phantom_timing0->v_front_porch) * + (uint64_t)phantom_timing0->h_total * 1000000), + (((uint64_t)phantom_timing0->pix_clk_100hz * 100) + dc->caps.subvp_prefetch_end_to_mall_start_us)); + subvp1_prefetch_us = div64_u64(((uint64_t)(phantom_timing1->v_total - phantom_timing1->v_front_porch) * + (uint64_t)phantom_timing1->h_total * 1000000), + (((uint64_t)phantom_timing1->pix_clk_100hz * 100) + dc->caps.subvp_prefetch_end_to_mall_start_us)); // Whichever SubVP PIPE has the smaller prefetch (including the prefetch end to mall start time) // should increase it's prefetch time to match the other @@ -592,16 +593,17 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc, pipe_data = &cmd->fw_assisted_mclk_switch_v2.config_data.pipe_data[1]; prefetch_delta_us = subvp0_prefetch_us - subvp1_prefetch_us; pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines = - div64_s64(((div64_s64((int64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us), 1000000)) * - (phantom_timing1->pix_clk_100hz * 100) + phantom_timing1->h_total - 1), - (int64_t)phantom_timing1->h_total); + div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) * + ((uint64_t)phantom_timing1->pix_clk_100hz * 100) + ((uint64_t)phantom_timing1->h_total * 1000000 - 1)), + ((uint64_t)phantom_timing1->h_total * 1000000)); + } else if (subvp1_prefetch_us > subvp0_prefetch_us) { pipe_data = &cmd->fw_assisted_mclk_switch_v2.config_data.pipe_data[0]; prefetch_delta_us = subvp1_prefetch_us - subvp0_prefetch_us; pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines = - div64_s64(((div64_s64((int64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us), 1000000)) * - (phantom_timing0->pix_clk_100hz * 100) + phantom_timing0->h_total - 1), - (int64_t)phantom_timing0->h_total); + div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) * + ((uint64_t)phantom_timing0->pix_clk_100hz * 100) + ((uint64_t)phantom_timing0->h_total * 1000000 - 1)), + ((uint64_t)phantom_timing0->h_total * 1000000)); } } @@ -668,19 +670,33 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc, // Round up pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines = - div64_s64(((div64_s64((int64_t)dc->caps.subvp_prefetch_end_to_mall_start_us, 1000000)) * - (phantom_timing->pix_clk_100hz * 100) + phantom_timing->h_total - 1), - (int64_t)phantom_timing->h_total); + div64_u64(((uint64_t)dc->caps.subvp_prefetch_end_to_mall_start_us * ((uint64_t)phantom_timing->pix_clk_100hz * 100) + + ((uint64_t)phantom_timing->h_total * 1000000 - 1)), ((uint64_t)phantom_timing->h_total * 1000000)); pipe_data->pipe_config.subvp_data.processing_delay_lines = - div64_s64(((div64_s64((int64_t)dc->caps.subvp_fw_processing_delay_us, 1000000)) * - (phantom_timing->pix_clk_100hz * 100) + phantom_timing->h_total - 1), - (int64_t)phantom_timing->h_total); + div64_u64(((uint64_t)(dc->caps.subvp_fw_processing_delay_us) * ((uint64_t)phantom_timing->pix_clk_100hz * 100) + + ((uint64_t)phantom_timing->h_total * 1000000 - 1)), ((uint64_t)phantom_timing->h_total * 1000000)); + + if (subvp_pipe->bottom_pipe) { + pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->bottom_pipe->pipe_idx; + } else if (subvp_pipe->next_odm_pipe) { + pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->next_odm_pipe->pipe_idx; + } else { + pipe_data->pipe_config.subvp_data.main_split_pipe_index = 0; + } + // Find phantom pipe index based on phantom stream for (j = 0; j < dc->res_pool->pipe_count; j++) { struct pipe_ctx *phantom_pipe = &context->res_ctx.pipe_ctx[j]; if (phantom_pipe->stream == subvp_pipe->stream->mall_stream_config.paired_stream) { pipe_data->pipe_config.subvp_data.phantom_pipe_index = phantom_pipe->pipe_idx; + if (phantom_pipe->bottom_pipe) { + pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->bottom_pipe->pipe_idx; + } else if (phantom_pipe->next_odm_pipe) { + pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->next_odm_pipe->pipe_idx; + } else { + pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = 0; + } break; } } @@ -725,7 +741,9 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc, if (!pipe->stream) continue; - if (pipe->plane_state && !pipe->top_pipe && + /* For SubVP pipe count, only count the top most (ODM / MPC) pipe + */ + if (pipe->plane_state && !pipe->top_pipe && !pipe->prev_odm_pipe && pipe->stream->mall_stream_config.type == SUBVP_MAIN) subvp_pipes[subvp_count++] = pipe; } @@ -738,7 +756,12 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc, if (!pipe->stream) continue; + /* When populating subvp cmd info, only pass in the top most (ODM / MPC) pipe. + * Any ODM or MPC splits being used in SubVP will be handled internally in + * populate_subvp_cmd_pipe_info + */ if (pipe->plane_state && pipe->stream->mall_stream_config.paired_stream && + !pipe->top_pipe && !pipe->prev_odm_pipe && pipe->stream->mall_stream_config.type == SUBVP_MAIN) { populate_subvp_cmd_pipe_info(dc, context, &cmd, pipe, cmd_pipe_index++); } else if (pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_NONE) { @@ -758,7 +781,7 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc, // Store the original watermark value for this SubVP config so we can lower it when the // MCLK switch starts wm_val_refclk = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns * - dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000 / 1000; + (dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000) / 1000; cmd.fw_assisted_mclk_switch_v2.config_data.watermark_a_cache = wm_val_refclk < 0xFFFF ? wm_val_refclk : 0xFFFF; } @@ -858,3 +881,147 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv) diag_data.is_cw0_enabled, diag_data.is_cw6_enabled); } + +static bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx) +{ + if (pipe_ctx->plane_state != NULL) { + if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE) + return false; + } + + if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 || + pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1) && + pipe_ctx->stream->ctx->dce_version >= DCN_VERSION_3_1) + return true; + + return false; +} + +static void dc_build_cursor_update_payload0( + struct pipe_ctx *pipe_ctx, uint8_t p_idx, + struct dmub_cmd_update_cursor_payload0 *payload) +{ + struct hubp *hubp = pipe_ctx->plane_res.hubp; + unsigned int panel_inst = 0; + + if (!dc_get_edp_link_panel_inst(hubp->ctx->dc, + pipe_ctx->stream->link, &panel_inst)) + return; + + /* Payload: Cursor Rect is built from position & attribute + * x & y are obtained from postion + */ + payload->cursor_rect.x = hubp->cur_rect.x; + payload->cursor_rect.y = hubp->cur_rect.y; + /* w & h are obtained from attribute */ + payload->cursor_rect.width = hubp->cur_rect.w; + payload->cursor_rect.height = hubp->cur_rect.h; + + payload->enable = hubp->pos.cur_ctl.bits.cur_enable; + payload->pipe_idx = p_idx; + payload->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1; + payload->panel_inst = panel_inst; +} + +static void dc_send_cmd_to_dmu(struct dc_dmub_srv *dmub_srv, + union dmub_rb_cmd *cmd) +{ + dc_dmub_srv_cmd_queue(dmub_srv, cmd); + dc_dmub_srv_cmd_execute(dmub_srv); + dc_dmub_srv_wait_idle(dmub_srv); +} + +static void dc_build_cursor_position_update_payload0( + struct dmub_cmd_update_cursor_payload0 *pl, const uint8_t p_idx, + const struct hubp *hubp, const struct dpp *dpp) +{ + /* Hubp */ + pl->position_cfg.pHubp.cur_ctl.raw = hubp->pos.cur_ctl.raw; + pl->position_cfg.pHubp.position.raw = hubp->pos.position.raw; + pl->position_cfg.pHubp.hot_spot.raw = hubp->pos.hot_spot.raw; + pl->position_cfg.pHubp.dst_offset.raw = hubp->pos.dst_offset.raw; + + /* dpp */ + pl->position_cfg.pDpp.cur0_ctl.raw = dpp->pos.cur0_ctl.raw; + pl->position_cfg.pipe_idx = p_idx; +} + +static void dc_build_cursor_attribute_update_payload1( + struct dmub_cursor_attributes_cfg *pl_A, const uint8_t p_idx, + const struct hubp *hubp, const struct dpp *dpp) +{ + /* Hubp */ + pl_A->aHubp.SURFACE_ADDR_HIGH = hubp->att.SURFACE_ADDR_HIGH; + pl_A->aHubp.SURFACE_ADDR = hubp->att.SURFACE_ADDR; + pl_A->aHubp.cur_ctl.raw = hubp->att.cur_ctl.raw; + pl_A->aHubp.size.raw = hubp->att.size.raw; + pl_A->aHubp.settings.raw = hubp->att.settings.raw; + + /* dpp */ + pl_A->aDpp.cur0_ctl.raw = dpp->att.cur0_ctl.raw; +} + +/** + * *************************************************************************************** + * dc_send_update_cursor_info_to_dmu: Populate the DMCUB Cursor update info command + * + * This function would store the cursor related information and pass it into dmub + * + * @param [in] pCtx: pipe context + * @param [in] pipe_idx: pipe index + * + * @return: void + * + * *************************************************************************************** + */ + +void dc_send_update_cursor_info_to_dmu( + struct pipe_ctx *pCtx, uint8_t pipe_idx) +{ + union dmub_rb_cmd cmd = { 0 }; + union dmub_cmd_update_cursor_info_data *update_cursor_info = + &cmd.update_cursor_info.update_cursor_info_data; + + if (!dc_dmub_should_update_cursor_data(pCtx)) + return; + /* + * Since we use multi_cmd_pending for dmub command, the 2nd command is + * only assigned to store cursor attributes info. + * 1st command can view as 2 parts, 1st is for PSR/Replay data, the other + * is to store cursor position info. + * + * Command heaer type must be the same type if using multi_cmd_pending. + * Besides, while process 2nd command in DMU, the sub type is useless. + * So it's meanless to pass the sub type header with different type. + */ + + { + /* Build Payload#0 Header */ + cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO; + cmd.update_cursor_info.header.payload_bytes = + sizeof(cmd.update_cursor_info.update_cursor_info_data); + cmd.update_cursor_info.header.multi_cmd_pending = 1; /* To combine multi dmu cmd, 1st cmd */ + + /* Prepare Payload */ + dc_build_cursor_update_payload0(pCtx, pipe_idx, &update_cursor_info->payload0); + + dc_build_cursor_position_update_payload0(&update_cursor_info->payload0, pipe_idx, + pCtx->plane_res.hubp, pCtx->plane_res.dpp); + /* Send update_curosr_info to queue */ + dc_dmub_srv_cmd_queue(pCtx->stream->ctx->dmub_srv, &cmd); + } + { + /* Build Payload#1 Header */ + memset(update_cursor_info, 0, sizeof(union dmub_cmd_update_cursor_info_data)); + cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO; + cmd.update_cursor_info.header.payload_bytes = sizeof(struct cursor_attributes_cfg); + cmd.update_cursor_info.header.multi_cmd_pending = 0; /* Indicate it's the last command. */ + + dc_build_cursor_attribute_update_payload1( + &cmd.update_cursor_info.update_cursor_info_data.payload1.attribute_cfg, + pipe_idx, pCtx->plane_res.hubp, pCtx->plane_res.dpp); + + /* Combine 2nd cmds update_curosr_info to DMU */ + dc_send_cmd_to_dmu(pCtx->stream->ctx->dmub_srv, &cmd); + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h index 9f5b47b9a83d038cfeb958f5d2f6a5d1d32493e5..d34f5563df2ec374f6c1b0543afc61ddc42b6c28 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h @@ -85,6 +85,8 @@ void dc_dmub_srv_send_inbox0_cmd(struct dc_dmub_srv *dmub_srv, union dmub_inbox0 bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv, struct dmub_diagnostic_data *dmub_oca); +void dc_dmub_setup_subvp_dmub_command(struct dc *dc, struct dc_state *context, bool enable); void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv); +void dc_send_update_cursor_info_to_dmu(struct pipe_ctx *pCtx, uint8_t pipe_idx); #endif /* _DMUB_DC_SRV_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index 3f64b30926927b8d45917bd1c428dd090f90163f..caf0c7af2d0b9984c5d747d23a7919a2ff55d287 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -117,11 +117,31 @@ struct psr_settings { * Add a struct dc_panel_config under dc_link */ struct dc_panel_config { + // extra panel power sequence parameters + struct pps { + unsigned int extra_t3_ms; + unsigned int extra_t7_ms; + unsigned int extra_delay_backlight_off; + unsigned int extra_post_t7_ms; + unsigned int extra_pre_t11_ms; + unsigned int extra_t12_ms; + unsigned int extra_post_OUI_ms; + } pps; + // ABM + struct varib { + unsigned int varibright_feature_enable; + unsigned int def_varibright_level; + unsigned int abm_config_setting; + } varib; // edp DSC struct dsc { bool disable_dsc_edp; unsigned int force_dsc_edp_policy; } dsc; + /* eDP ILR */ + struct ilr { + bool optimize_edp_link_rate; /* eDP ILR */ + } ilr; }; /* * A link contains one or more sinks and their connected status. @@ -141,7 +161,6 @@ struct dc_link { bool link_state_valid; bool aux_access_disabled; bool sync_lt_in_progress; - enum lttpr_mode lttpr_mode; bool is_internal_display; /* TODO: Rename. Flag an endpoint as having a programmable mapping to a @@ -244,7 +263,7 @@ struct dc_link { struct gpio *hpd_gpio; enum dc_link_fec_state fec_state; struct dc_panel_config panel_config; - enum phy_state phy_state; + struct phy_state phy_state; }; const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link); diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 9fcf9dc5bce495d643dad2fa24998c4cc470868a..9e6025c98db91f8afe71c732e9c82d64ee3a95b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -212,8 +212,7 @@ struct dc_stream_state { /* DMCU info */ unsigned int abm_level; - struct periodic_interrupt_config periodic_interrupt0; - struct periodic_interrupt_config periodic_interrupt1; + struct periodic_interrupt_config periodic_interrupt; /* from core_stream struct */ struct dc_context *ctx; @@ -281,8 +280,7 @@ struct dc_stream_update { struct dc_info_packet *hdr_static_metadata; unsigned int *abm_level; - struct periodic_interrupt_config *periodic_interrupt0; - struct periodic_interrupt_config *periodic_interrupt1; + struct periodic_interrupt_config *periodic_interrupt; struct dc_info_packet *vrr_infopacket; struct dc_info_packet *vsc_infopacket; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c index bdb6bac8dd974f65c0e6f710af7450f4acbb9e2d..c94a966c6612bd6e36dd2c88a012836b79b3676b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c @@ -300,7 +300,7 @@ static void set_high_bit_rate_capable( AZ_REG_WRITE(AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR, value); } -/* set video latency in in ms/2+1 */ +/* set video latency in ms/2+1 */ static void set_video_latency( struct audio *audio, int latency_in_ms) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c index 32782ef9ef778781e4d67803256ef93a9a2040c1..140297c8ff555bf3b5e2268733337bfbd078cc08 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c @@ -942,10 +942,6 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, case AUX_RET_ERROR_ENGINE_ACQUIRE: case AUX_RET_ERROR_UNKNOWN: default: - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION, - LOG_FLAG_I2cAux_DceAux, - "dce_aux_transfer_with_retries: Failure: operation_result=%d", - (int)operation_result); goto fail; } } @@ -953,14 +949,11 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc, fail: DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, LOG_FLAG_Error_I2cAux, - "dce_aux_transfer_with_retries: FAILURE"); + "%s: Failure: operation_result=%d", + __func__, + (int)operation_result); if (!payload_reply) payload->reply = NULL; - DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR, - WPP_BIT_FLAG_DC_ERROR, - "AUX transaction failed. Result: %d", - operation_result); - return false; } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index 0df06740ec391e71940d9ed5c2534e46703dc2ff..bec5e9f787fc04169f0cbbe3d5ae04578cc56e21 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -393,17 +393,18 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, if (copy_settings_data->dsc_enable_status && link->dpcd_caps.sink_dev_id == DP_DEVICE_ID_38EC11 && !memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1, - sizeof(link->dpcd_caps.sink_dev_id_str))) + sizeof(DP_SINK_DEVICE_STR_ID_1))) link->psr_settings.force_ffu_mode = 1; else link->psr_settings.force_ffu_mode = 0; copy_settings_data->force_ffu_mode = link->psr_settings.force_ffu_mode; if (link->fec_state == dc_link_fec_enabled && + link->dpcd_caps.sink_dev_id == DP_DEVICE_ID_38EC11 && (!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1, - sizeof(link->dpcd_caps.sink_dev_id_str)) || + sizeof(DP_SINK_DEVICE_STR_ID_1)) || !memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_2, - sizeof(link->dpcd_caps.sink_dev_id_str)))) + sizeof(DP_SINK_DEVICE_STR_ID_2)))) copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 1; else copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index fe346e96c2d12478350fda3907aa0eb6d497190d..d260eaa1509ed15f182265398db78b68656e302b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -722,7 +722,6 @@ void dce110_edp_wait_for_hpd_ready( struct dc_context *ctx = link->ctx; struct graphics_object_id connector = link->link_enc->connector; struct gpio *hpd; - struct dc_sink *sink = link->local_sink; bool edp_hpd_high = false; uint32_t time_elapsed = 0; uint32_t timeout = power_up ? @@ -755,9 +754,9 @@ void dce110_edp_wait_for_hpd_ready( return; } - if (sink != NULL) { - if (sink->edid_caps.panel_patch.extra_t3_ms > 0) { - int extra_t3_in_ms = sink->edid_caps.panel_patch.extra_t3_ms; + if (link != NULL) { + if (link->panel_config.pps.extra_t3_ms > 0) { + int extra_t3_in_ms = link->panel_config.pps.extra_t3_ms; msleep(extra_t3_in_ms); } @@ -842,7 +841,7 @@ void dce110_edp_power_control( /* add time defined by a patch, if any (usually patch extra_t12_ms is 0) */ if (link->local_sink != NULL) remaining_min_edp_poweroff_time_ms += - link->local_sink->edid_caps.panel_patch.extra_t12_ms; + link->panel_config.pps.extra_t12_ms; /* Adjust remaining_min_edp_poweroff_time_ms if this is not the first time. */ if (dp_trace_get_edp_poweroff_timestamp(link) != 0) { @@ -946,7 +945,7 @@ void dce110_edp_wait_for_T12( current_ts, dp_trace_get_edp_poweroff_timestamp(link)), 1000000); - t12_duration += link->local_sink->edid_caps.panel_patch.extra_t12_ms; // Add extra T12 + t12_duration += link->panel_config.pps.extra_t12_ms; // Add extra T12 if (time_since_edp_poweroff_ms < t12_duration) msleep(t12_duration - time_since_edp_poweroff_ms); @@ -965,6 +964,8 @@ void dce110_edp_backlight_control( struct dc_context *ctx = link->ctx; struct bp_transmitter_control cntl = { 0 }; uint8_t panel_instance; + unsigned int pre_T11_delay = OLED_PRE_T11_DELAY; + unsigned int post_T7_delay = OLED_POST_T7_DELAY; if (dal_graphics_object_id_get_connector_id(link->link_enc->connector) != CONNECTOR_ID_EDP) { @@ -1043,8 +1044,10 @@ void dce110_edp_backlight_control( link_transmitter_control(ctx->dc_bios, &cntl); - if (enable && link->dpcd_sink_ext_caps.bits.oled) - msleep(OLED_POST_T7_DELAY); + if (enable && link->dpcd_sink_ext_caps.bits.oled) { + post_T7_delay += link->panel_config.pps.extra_post_t7_ms; + msleep(post_T7_delay); + } if (link->dpcd_sink_ext_caps.bits.oled || link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 || @@ -1066,8 +1069,10 @@ void dce110_edp_backlight_control( DC_LOG_DC("edp_receiver_ready_T9 skipped\n"); } - if (!enable && link->dpcd_sink_ext_caps.bits.oled) - msleep(OLED_PRE_T11_DELAY); + if (!enable && link->dpcd_sink_ext_caps.bits.oled) { + pre_T11_delay += link->panel_config.pps.extra_pre_t11_ms; + msleep(pre_T11_delay); + } } void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx) @@ -1441,6 +1446,14 @@ static enum dc_status dce110_enable_stream_timing( return DC_ERROR_UNEXPECTED; } + if (dc_is_hdmi_tmds_signal(stream->signal)) { + stream->link->phy_state.symclk_ref_cnts.otg = 1; + if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF) + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF; + else + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON; + } + pipe_ctx->stream_res.tg->funcs->program_timing( pipe_ctx->stream_res.tg, &stream->timing, @@ -1577,25 +1590,8 @@ static enum dc_status apply_single_controller_ctx_to_hw( if (dc_is_dp_signal(pipe_ctx->stream->signal)) dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_CONNECT_DIG_FE_OTG); - /* Have to setup DSC before DIG FE and BE are connected (which happens before the - * link training). This is to make sure the bandwidth sent to DIG BE won't be - * bigger than what the link and/or DIG BE can handle. VBID[6]/CompressedStream_flag - * will be automatically set at a later time when the video is enabled - * (DP_VID_STREAM_EN = 1). - */ - if (pipe_ctx->stream->timing.flags.DSC) { - if (dc_is_dp_signal(pipe_ctx->stream->signal) || - dc_is_virtual_signal(pipe_ctx->stream->signal)) - dp_set_dsc_enable(pipe_ctx, true); - - } - - if (!stream->dpms_off) { - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(context, pipe_ctx, TX_ON_SYMCLK_ON); - else - core_link_enable_stream(context, pipe_ctx); - } + if (!stream->dpms_off) + core_link_enable_stream(context, pipe_ctx); /* DCN3.1 FPGA Workaround * Need to enable HPO DP Stream Encoder before setting OTG master enable. @@ -2131,6 +2127,7 @@ static void dce110_reset_hw_ctx_wrap( BREAK_TO_DEBUGGER(); } pipe_ctx_old->stream_res.tg->funcs->disable_crtc(pipe_ctx_old->stream_res.tg); + pipe_ctx_old->stream->link->phy_state.symclk_ref_cnts.otg = 0; pipe_ctx_old->plane_res.mi->funcs->free_mem_input( pipe_ctx_old->plane_res.mi, dc->current_state->stream_count); @@ -3009,6 +3006,124 @@ void dce110_set_pipe(struct pipe_ctx *pipe_ctx) abm->funcs->set_pipe(abm, otg_inst, panel_cntl->inst); } +void dce110_enable_lvds_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum clock_source_id clock_source, + uint32_t pixel_clock) +{ + link->link_enc->funcs->enable_lvds_output( + link->link_enc, + clock_source, + pixel_clock); + link->phy_state.symclk_state = SYMCLK_ON_TX_ON; +} + +void dce110_enable_tmds_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + uint32_t pixel_clock) +{ + link->link_enc->funcs->enable_tmds_output( + link->link_enc, + clock_source, + color_depth, + signal, + pixel_clock); + link->phy_state.symclk_state = SYMCLK_ON_TX_ON; +} + +void dce110_enable_dp_link_output( + struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + const struct dc_link_settings *link_settings) +{ + struct dc *dc = link->ctx->dc; + struct dmcu *dmcu = dc->res_pool->dmcu; + struct pipe_ctx *pipes = + link->dc->current_state->res_ctx.pipe_ctx; + struct clock_source *dp_cs = + link->dc->res_pool->dp_clock_source; + const struct link_hwss *link_hwss = get_link_hwss(link, link_res); + unsigned int i; + + + if (link->connector_signal == SIGNAL_TYPE_EDP) { + if (!link->dc->config.edp_no_power_sequencing) + link->dc->hwss.edp_power_control(link, true); + link->dc->hwss.edp_wait_for_hpd_ready(link, true); + } + + /* If the current pixel clock source is not DTO(happens after + * switching from HDMI passive dongle to DP on the same connector), + * switch the pixel clock source to DTO. + */ + + for (i = 0; i < MAX_PIPES; i++) { + if (pipes[i].stream != NULL && + pipes[i].stream->link == link) { + if (pipes[i].clock_source != NULL && + pipes[i].clock_source->id != CLOCK_SOURCE_ID_DP_DTO) { + pipes[i].clock_source = dp_cs; + pipes[i].stream_res.pix_clk_params.requested_pix_clk_100hz = + pipes[i].stream->timing.pix_clk_100hz; + pipes[i].clock_source->funcs->program_pix_clk( + pipes[i].clock_source, + &pipes[i].stream_res.pix_clk_params, + dp_get_link_encoding_format(link_settings), + &pipes[i].pll_settings); + } + } + } + + if (dp_get_link_encoding_format(link_settings) == DP_8b_10b_ENCODING) { + if (dc->clk_mgr->funcs->notify_link_rate_change) + dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link); + } + + if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->lock_phy(dmcu); + + if (link_hwss->ext.enable_dp_link_output) + link_hwss->ext.enable_dp_link_output(link, link_res, signal, + clock_source, link_settings); + + link->phy_state.symclk_state = SYMCLK_ON_TX_ON; + + if (dmcu != NULL && dmcu->funcs->unlock_phy) + dmcu->funcs->unlock_phy(dmcu); + + dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY); +} + +void dce110_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal) +{ + struct dc *dc = link->ctx->dc; + const struct link_hwss *link_hwss = get_link_hwss(link, link_res); + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (signal == SIGNAL_TYPE_EDP && + link->dc->hwss.edp_backlight_control) + link->dc->hwss.edp_backlight_control(link, false); + else if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->lock_phy(dmcu); + + link_hwss->disable_link_output(link, link_res, signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + + if (signal == SIGNAL_TYPE_EDP && + link->dc->hwss.edp_backlight_control) + link->dc->hwss.edp_power_control(link, false); + else if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->unlock_phy(dmcu); + dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY); +} + static const struct hw_sequencer_funcs dce110_funcs = { .program_gamut_remap = program_gamut_remap, .program_output_csc = program_output_csc, @@ -3048,6 +3163,10 @@ static const struct hw_sequencer_funcs dce110_funcs = { .set_backlight_level = dce110_set_backlight_level, .set_abm_immediate_disable = dce110_set_abm_immediate_disable, .set_pipe = dce110_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, }; static const struct hwseq_private_funcs dce110_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h index b6f3843d3d05c7e505a3f6d128a6aad5f751f529..758f4b3b0087f58baee53d8865c9115779203d56 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h @@ -90,6 +90,24 @@ bool dce110_set_backlight_level(struct pipe_ctx *pipe_ctx, uint32_t frame_ramp); void dce110_set_abm_immediate_disable(struct pipe_ctx *pipe_ctx); void dce110_set_pipe(struct pipe_ctx *pipe_ctx); - +void dce110_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal); +void dce110_enable_lvds_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum clock_source_id clock_source, + uint32_t pixel_clock); +void dce110_enable_tmds_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + uint32_t pixel_clock); +void dce110_enable_dp_link_output( + struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + const struct dc_link_settings *link_settings); #endif /* __DC_HWSS_DCE110_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c index 897f412f539e6502ac16e06dcbe3b1540212afef..b9765b3899e191cb4a04d6506d756f8542ceb2a2 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c @@ -469,6 +469,7 @@ void dpp1_set_cursor_position( REG_UPDATE(CURSOR0_CONTROL, CUR0_ENABLE, cur_en); + dpp_base->pos.cur0_ctl.bits.cur0_enable = cur_en; } void dpp1_cnv_set_optional_cursor_attributes( diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index f26e08032da0e1dd01140e9c9b741338f13e2bd5..11e4c4e46947348af7121425aa362153292aa75c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -899,6 +899,14 @@ enum dc_status dcn10_enable_stream_timing( return DC_ERROR_UNEXPECTED; } + if (dc_is_hdmi_tmds_signal(stream->signal)) { + stream->link->phy_state.symclk_ref_cnts.otg = 1; + if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF) + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF; + else + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON; + } + pipe_ctx->stream_res.tg->funcs->program_timing( pipe_ctx->stream_res.tg, &stream->timing, @@ -1017,6 +1025,7 @@ static void dcn10_reset_back_end_for_pipe( if (pipe_ctx->stream_res.tg->funcs->set_drr) pipe_ctx->stream_res.tg->funcs->set_drr( pipe_ctx->stream_res.tg, NULL); + pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0; } for (i = 0; i < dc->res_pool->pipe_count; i++) @@ -2235,6 +2244,9 @@ void dcn10_enable_timing_synchronization( DC_SYNC_INFO("Setting up OTG reset trigger\n"); for (i = 1; i < group_size; i++) { + if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM) + continue; + opp = grouped_pipes[i]->stream_res.opp; tg = grouped_pipes[i]->stream_res.tg; tg->funcs->get_otg_active_size(tg, &width, &height); @@ -2245,13 +2257,21 @@ void dcn10_enable_timing_synchronization( for (i = 0; i < group_size; i++) { if (grouped_pipes[i]->stream == NULL) continue; + + if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM) + continue; + grouped_pipes[i]->stream->vblank_synchronized = false; } - for (i = 1; i < group_size; i++) + for (i = 1; i < group_size; i++) { + if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM) + continue; + grouped_pipes[i]->stream_res.tg->funcs->enable_reset_trigger( grouped_pipes[i]->stream_res.tg, grouped_pipes[0]->stream_res.tg->inst); + } DC_SYNC_INFO("Waiting for trigger\n"); @@ -2259,12 +2279,21 @@ void dcn10_enable_timing_synchronization( * synchronized. Look at last pipe programmed to reset. */ - wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[1]->stream_res.tg); - for (i = 1; i < group_size; i++) + if (grouped_pipes[1]->stream && grouped_pipes[1]->stream->mall_stream_config.type != SUBVP_PHANTOM) + wait_for_reset_trigger_to_occur(dc_ctx, grouped_pipes[1]->stream_res.tg); + + for (i = 1; i < group_size; i++) { + if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM) + continue; + grouped_pipes[i]->stream_res.tg->funcs->disable_reset_trigger( grouped_pipes[i]->stream_res.tg); + } for (i = 1; i < group_size; i++) { + if (grouped_pipes[i]->stream && grouped_pipes[i]->stream->mall_stream_config.type == SUBVP_PHANTOM) + continue; + opp = grouped_pipes[i]->stream_res.opp; tg = grouped_pipes[i]->stream_res.tg; tg->funcs->get_otg_active_size(tg, &width, &height); @@ -2996,6 +3025,7 @@ void dcn10_prepare_bandwidth( { struct dce_hwseq *hws = dc->hwseq; struct hubbub *hubbub = dc->res_pool->hubbub; + int min_fclk_khz, min_dcfclk_khz, socclk_khz; if (dc->debug.sanity_checks) hws->funcs.verify_allow_pstate_change_high(dc); @@ -3018,8 +3048,11 @@ void dcn10_prepare_bandwidth( if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) { DC_FP_START(); - dcn_bw_notify_pplib_of_wm_ranges(dc); + dcn_get_soc_clks( + dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz); DC_FP_END(); + dcn_bw_notify_pplib_of_wm_ranges( + dc, min_fclk_khz, min_dcfclk_khz, socclk_khz); } if (dc->debug.sanity_checks) @@ -3032,6 +3065,7 @@ void dcn10_optimize_bandwidth( { struct dce_hwseq *hws = dc->hwseq; struct hubbub *hubbub = dc->res_pool->hubbub; + int min_fclk_khz, min_dcfclk_khz, socclk_khz; if (dc->debug.sanity_checks) hws->funcs.verify_allow_pstate_change_high(dc); @@ -3055,8 +3089,11 @@ void dcn10_optimize_bandwidth( if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) { DC_FP_START(); - dcn_bw_notify_pplib_of_wm_ranges(dc); + dcn_get_soc_clks( + dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz); DC_FP_END(); + dcn_bw_notify_pplib_of_wm_ranges( + dc, min_fclk_khz, min_dcfclk_khz, socclk_khz); } if (dc->debug.sanity_checks) @@ -3335,127 +3372,6 @@ static bool dcn10_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx) return false; } -static bool dcn10_dmub_should_update_cursor_data( - struct pipe_ctx *pipe_ctx, - struct dc_debug_options *debug) -{ - if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE) - return false; - - if (dcn10_can_pipe_disable_cursor(pipe_ctx)) - return false; - - if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 || pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1) - && pipe_ctx->stream->ctx->dce_version >= DCN_VERSION_3_1) - return true; - - return false; -} - -static void dcn10_dmub_update_cursor_data( - struct pipe_ctx *pipe_ctx, - struct hubp *hubp, - const struct dc_cursor_mi_param *param, - const struct dc_cursor_position *cur_pos, - const struct dc_cursor_attributes *cur_attr) -{ - union dmub_rb_cmd cmd; - struct dmub_cmd_update_cursor_info_data *update_cursor_info; - const struct dc_cursor_position *pos; - const struct dc_cursor_attributes *attr; - int src_x_offset = 0; - int src_y_offset = 0; - int x_hotspot = 0; - int cursor_height = 0; - int cursor_width = 0; - uint32_t cur_en = 0; - unsigned int panel_inst = 0; - - struct dc_debug_options *debug = &hubp->ctx->dc->debug; - - if (!dcn10_dmub_should_update_cursor_data(pipe_ctx, debug)) - return; - /** - * if cur_pos == NULL means the caller is from cursor_set_attribute - * then driver use previous cursor position data - * if cur_attr == NULL means the caller is from cursor_set_position - * then driver use previous cursor attribute - * if cur_pos or cur_attr is not NULL then update it - */ - if (cur_pos != NULL) - pos = cur_pos; - else - pos = &hubp->curs_pos; - - if (cur_attr != NULL) - attr = cur_attr; - else - attr = &hubp->curs_attr; - - if (!dc_get_edp_link_panel_inst(hubp->ctx->dc, pipe_ctx->stream->link, &panel_inst)) - return; - - src_x_offset = pos->x - pos->x_hotspot - param->viewport.x; - src_y_offset = pos->y - pos->y_hotspot - param->viewport.y; - x_hotspot = pos->x_hotspot; - cursor_height = (int)attr->height; - cursor_width = (int)attr->width; - cur_en = pos->enable ? 1:0; - - // Rotated cursor width/height and hotspots tweaks for offset calculation - if (param->rotation == ROTATION_ANGLE_90 || param->rotation == ROTATION_ANGLE_270) { - swap(cursor_height, cursor_width); - if (param->rotation == ROTATION_ANGLE_90) { - src_x_offset = pos->x - pos->y_hotspot - param->viewport.x; - src_y_offset = pos->y - pos->x_hotspot - param->viewport.y; - } - } else if (param->rotation == ROTATION_ANGLE_180) { - src_x_offset = pos->x - param->viewport.x; - src_y_offset = pos->y - param->viewport.y; - } - - if (param->mirror) { - x_hotspot = param->viewport.width - x_hotspot; - src_x_offset = param->viewport.x + param->viewport.width - src_x_offset; - } - - if (src_x_offset >= (int)param->viewport.width) - cur_en = 0; /* not visible beyond right edge*/ - - if (src_x_offset + cursor_width <= 0) - cur_en = 0; /* not visible beyond left edge*/ - - if (src_y_offset >= (int)param->viewport.height) - cur_en = 0; /* not visible beyond bottom edge*/ - - if (src_y_offset + cursor_height <= 0) - cur_en = 0; /* not visible beyond top edge*/ - - // Cursor bitmaps have different hotspot values - // There's a possibility that the above logic returns a negative value, so we clamp them to 0 - if (src_x_offset < 0) - src_x_offset = 0; - if (src_y_offset < 0) - src_y_offset = 0; - - memset(&cmd, 0x0, sizeof(cmd)); - cmd.update_cursor_info.header.type = DMUB_CMD__UPDATE_CURSOR_INFO; - cmd.update_cursor_info.header.payload_bytes = - sizeof(cmd.update_cursor_info.update_cursor_info_data); - update_cursor_info = &cmd.update_cursor_info.update_cursor_info_data; - update_cursor_info->cursor_rect.x = src_x_offset + param->viewport.x; - update_cursor_info->cursor_rect.y = src_y_offset + param->viewport.y; - update_cursor_info->cursor_rect.width = attr->width; - update_cursor_info->cursor_rect.height = attr->height; - update_cursor_info->enable = cur_en; - update_cursor_info->pipe_idx = pipe_ctx->pipe_idx; - update_cursor_info->cmd_version = DMUB_CMD_PSR_CONTROL_VERSION_1; - update_cursor_info->panel_inst = panel_inst; - dc_dmub_srv_cmd_queue(pipe_ctx->stream->ctx->dmub_srv, &cmd); - dc_dmub_srv_cmd_execute(pipe_ctx->stream->ctx->dmub_srv); - dc_dmub_srv_wait_idle(pipe_ctx->stream->ctx->dmub_srv); -} - void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) { struct dc_cursor_position pos_cpy = pipe_ctx->stream->cursor_position; @@ -3690,7 +3606,6 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) pipe_ctx->plane_res.scl_data.viewport.height - pos_cpy.y; } - dcn10_dmub_update_cursor_data(pipe_ctx, hubp, ¶m, &pos_cpy, NULL); hubp->funcs->set_cursor_position(hubp, &pos_cpy, ¶m); dpp->funcs->set_cursor_position(dpp, &pos_cpy, ¶m, hubp->curs_attr.width, hubp->curs_attr.height); } @@ -3698,25 +3613,6 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx) void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx) { struct dc_cursor_attributes *attributes = &pipe_ctx->stream->cursor_attributes; - struct dc_cursor_mi_param param = { 0 }; - - /** - * If enter PSR without cursor attribute update - * the cursor attribute of dmub_restore_plane - * are initial value. call dmub to exit PSR and - * restore plane then update cursor attribute to - * avoid override with initial value - */ - if (pipe_ctx->plane_state != NULL) { - param.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10; - param.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz; - param.viewport = pipe_ctx->plane_res.scl_data.viewport; - param.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz; - param.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert; - param.rotation = pipe_ctx->plane_state->rotation; - param.mirror = pipe_ctx->plane_state->horizontal_mirror; - dcn10_dmub_update_cursor_data(pipe_ctx, pipe_ctx->plane_res.hubp, ¶m, NULL, attributes); - } pipe_ctx->plane_res.hubp->funcs->set_cursor_attributes( pipe_ctx->plane_res.hubp, attributes); @@ -3801,81 +3697,56 @@ void dcn10_calc_vupdate_position( uint32_t *start_line, uint32_t *end_line) { - const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing; - int vline_int_offset_from_vupdate = - pipe_ctx->stream->periodic_interrupt0.lines_offset; - int vupdate_offset_from_vsync = dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx); - int start_position; - - if (vline_int_offset_from_vupdate > 0) - vline_int_offset_from_vupdate--; - else if (vline_int_offset_from_vupdate < 0) - vline_int_offset_from_vupdate++; - - start_position = vline_int_offset_from_vupdate + vupdate_offset_from_vsync; + const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing; + int vupdate_pos = dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx); - if (start_position >= 0) - *start_line = start_position; + if (vupdate_pos >= 0) + *start_line = vupdate_pos - ((vupdate_pos / timing->v_total) * timing->v_total); else - *start_line = dc_crtc_timing->v_total + start_position - 1; - - *end_line = *start_line + 2; - - if (*end_line >= dc_crtc_timing->v_total) - *end_line = 2; + *start_line = vupdate_pos + ((-vupdate_pos / timing->v_total) + 1) * timing->v_total - 1; + *end_line = (*start_line + 2) % timing->v_total; } static void dcn10_cal_vline_position( struct dc *dc, struct pipe_ctx *pipe_ctx, - enum vline_select vline, uint32_t *start_line, uint32_t *end_line) { - enum vertical_interrupt_ref_point ref_point = INVALID_POINT; + const struct dc_crtc_timing *timing = &pipe_ctx->stream->timing; + int vline_pos = pipe_ctx->stream->periodic_interrupt.lines_offset; - if (vline == VLINE0) - ref_point = pipe_ctx->stream->periodic_interrupt0.ref_point; - else if (vline == VLINE1) - ref_point = pipe_ctx->stream->periodic_interrupt1.ref_point; + if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_UPDATE) { + if (vline_pos > 0) + vline_pos--; + else if (vline_pos < 0) + vline_pos++; - switch (ref_point) { - case START_V_UPDATE: - dcn10_calc_vupdate_position( - dc, - pipe_ctx, - start_line, - end_line); - break; - case START_V_SYNC: - // Suppose to do nothing because vsync is 0; - break; - default: + vline_pos += dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx); + if (vline_pos >= 0) + *start_line = vline_pos - ((vline_pos / timing->v_total) * timing->v_total); + else + *start_line = vline_pos + ((-vline_pos / timing->v_total) + 1) * timing->v_total - 1; + *end_line = (*start_line + 2) % timing->v_total; + } else if (pipe_ctx->stream->periodic_interrupt.ref_point == START_V_SYNC) { + // vsync is line 0 so start_line is just the requested line offset + *start_line = vline_pos; + *end_line = (*start_line + 2) % timing->v_total; + } else ASSERT(0); - break; - } } void dcn10_setup_periodic_interrupt( struct dc *dc, - struct pipe_ctx *pipe_ctx, - enum vline_select vline) + struct pipe_ctx *pipe_ctx) { struct timing_generator *tg = pipe_ctx->stream_res.tg; + uint32_t start_line = 0; + uint32_t end_line = 0; - if (vline == VLINE0) { - uint32_t start_line = 0; - uint32_t end_line = 0; - - dcn10_cal_vline_position(dc, pipe_ctx, vline, &start_line, &end_line); + dcn10_cal_vline_position(dc, pipe_ctx, &start_line, &end_line); - tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line); - - } else if (vline == VLINE1) { - pipe_ctx->stream_res.tg->funcs->setup_vertical_interrupt1( - tg, - pipe_ctx->stream->periodic_interrupt1.lines_offset); - } + tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line); } void dcn10_setup_vupdate_interrupt(struct dc *dc, struct pipe_ctx *pipe_ctx) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h index 9ae07c77fdc01d5f008194c286e3a29a16a7b0f4..0ef7bf7ddb75e918472f92577837de7907bdebfb 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h @@ -175,8 +175,7 @@ void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx); void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx); void dcn10_setup_periodic_interrupt( struct dc *dc, - struct pipe_ctx *pipe_ctx, - enum vline_select vline); + struct pipe_ctx *pipe_ctx); enum dc_status dcn10_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32_t clk_khz, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c index 10e613ec7d24f5e5c25f2dc50cc12b4bc6d15632..f2371c948822546fefac766dfb2abfea8ec0386a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c @@ -82,6 +82,10 @@ static const struct hw_sequencer_funcs dcn10_funcs = { .set_backlight_level = dce110_set_backlight_level, .set_abm_immediate_disable = dce110_set_abm_immediate_disable, .set_pipe = dce110_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .get_dcc_en_bits = dcn10_get_dcc_en_bits, .update_visual_confirm_color = dcn10_update_visual_confirm_color, }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index 294827906c69800d249fb822eb4b13fbdfc25899..33d7802187900866958df55bfd39f5627b83302f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -207,10 +207,7 @@ void optc1_program_timing( /* In case of V_TOTAL_CONTROL is on, make sure OTG_V_TOTAL_MAX and * OTG_V_TOTAL_MIN are equal to V_TOTAL. */ - REG_SET(OTG_V_TOTAL_MAX, 0, - OTG_V_TOTAL_MAX, v_total); - REG_SET(OTG_V_TOTAL_MIN, 0, - OTG_V_TOTAL_MIN, v_total); + optc->funcs->set_vtotal_min_max(optc, v_total, v_total); /* v_sync_start = 0, v_sync_end = v_sync_width */ v_sync_end = patched_crtc_timing.v_sync_width; @@ -649,13 +646,6 @@ uint32_t optc1_get_vblank_counter(struct timing_generator *optc) void optc1_lock(struct timing_generator *optc) { struct optc *optc1 = DCN10TG_FROM_TG(optc); - uint32_t regval = 0; - - regval = REG_READ(OTG_CONTROL); - - /* otg is not running, do not need to be locked */ - if ((regval & 0x1) == 0x0) - return; REG_SET(OTG_GLOBAL_CONTROL0, 0, OTG_MASTER_UPDATE_LOCK_SEL, optc->inst); @@ -663,12 +653,10 @@ void optc1_lock(struct timing_generator *optc) OTG_MASTER_UPDATE_LOCK, 1); /* Should be fast, status does not update on maximus */ - if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS) { - + if (optc->ctx->dce_environment != DCE_ENV_FPGA_MAXIMUS) REG_WAIT(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, 1, 1, 10); - } } void optc1_unlock(struct timing_generator *optc) @@ -679,16 +667,6 @@ void optc1_unlock(struct timing_generator *optc) OTG_MASTER_UPDATE_LOCK, 0); } -bool optc1_is_locked(struct timing_generator *optc) -{ - struct optc *optc1 = DCN10TG_FROM_TG(optc); - uint32_t locked; - - REG_GET(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, &locked); - - return (locked == 1); -} - void optc1_get_position(struct timing_generator *optc, struct crtc_position *position) { @@ -941,11 +919,7 @@ void optc1_set_drr( } - REG_SET(OTG_V_TOTAL_MAX, 0, - OTG_V_TOTAL_MAX, params->vertical_total_max - 1); - - REG_SET(OTG_V_TOTAL_MIN, 0, - OTG_V_TOTAL_MIN, params->vertical_total_min - 1); + optc->funcs->set_vtotal_min_max(optc, params->vertical_total_min - 1, params->vertical_total_max - 1); REG_UPDATE_5(OTG_V_TOTAL_CONTROL, OTG_V_TOTAL_MIN_SEL, 1, @@ -964,11 +938,7 @@ void optc1_set_drr( OTG_V_TOTAL_MAX_SEL, 0, OTG_FORCE_LOCK_ON_EVENT, 0); - REG_SET(OTG_V_TOTAL_MIN, 0, - OTG_V_TOTAL_MIN, 0); - - REG_SET(OTG_V_TOTAL_MAX, 0, - OTG_V_TOTAL_MAX, 0); + optc->funcs->set_vtotal_min_max(optc, 0, 0); } } @@ -1393,6 +1363,12 @@ void optc1_read_otg_state(struct optc *optc1, REG_GET(OPTC_INPUT_GLOBAL_CONTROL, OPTC_UNDERFLOW_OCCURRED_STATUS, &s->underflow_occurred_status); + REG_GET(OTG_VERTICAL_INTERRUPT1_CONTROL, + OTG_VERTICAL_INTERRUPT1_INT_ENABLE, &s->vertical_interrupt1_en); + + REG_GET(OTG_VERTICAL_INTERRUPT1_POSITION, + OTG_VERTICAL_INTERRUPT1_LINE_START, &s->vertical_interrupt1_line); + REG_GET(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG_VERTICAL_INTERRUPT2_INT_ENABLE, &s->vertical_interrupt2_en); @@ -1577,11 +1553,11 @@ static const struct timing_generator_funcs dcn10_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc1_lock, - .is_locked = optc1_is_locked, .unlock = optc1_unlock, .enable_optc_clock = optc1_enable_optc_clock, .set_drr = optc1_set_drr, .get_last_used_drr_vtotal = NULL, + .set_vtotal_min_max = optc1_set_vtotal_min_max, .set_static_screen_control = optc1_set_static_screen_control, .set_test_pattern = optc1_set_test_pattern, .program_stereo = optc1_program_stereo, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h index 3fe5882ed01886aa224a055d642adfdb469a8c71..88ac5f6f4c96cc4ba9b3d3b9f3b7342afa6581a7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h @@ -583,6 +583,8 @@ struct dcn_otg_state { uint32_t underflow_occurred_status; uint32_t otg_enabled; uint32_t blank_enabled; + uint32_t vertical_interrupt1_en; + uint32_t vertical_interrupt1_line; uint32_t vertical_interrupt2_en; uint32_t vertical_interrupt2_line; }; @@ -652,7 +654,6 @@ void optc1_set_blank(struct timing_generator *optc, bool enable_blanking); bool optc1_is_blanked(struct timing_generator *optc); -bool optc1_is_locked(struct timing_generator *optc); void optc1_program_blank_color( struct timing_generator *optc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c index 831080b9eb873aac719beaff5dfc2930469c99ab..56d30baf12df29fc741e318ced1551ce452dcc15 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c @@ -1336,6 +1336,21 @@ static noinline void dcn10_resource_construct_fp( } } +static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks) +{ + int i; + + if (clks->num_levels == 0) + return false; + + for (i = 0; i < clks->num_levels; i++) + /* Ensure that the result is sane */ + if (clks->data[i].clocks_in_khz == 0) + return false; + + return true; +} + static bool dcn10_resource_construct( uint8_t num_virtual_links, struct dc *dc, @@ -1345,6 +1360,9 @@ static bool dcn10_resource_construct( int j; struct dc_context *ctx = dc->ctx; uint32_t pipe_fuses = read_pipe_fuses(ctx); + struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0}; + int min_fclk_khz, min_dcfclk_khz, socclk_khz; + bool res; ctx->dc_bios->regs = &bios_regs; @@ -1523,15 +1541,53 @@ static bool dcn10_resource_construct( && pool->base.pp_smu->rv_funcs.set_pme_wa_enable != NULL) dc->debug.az_endpoint_mute_only = false; - DC_FP_START(); - if (!dc->debug.disable_pplib_clock_request) - dcn_bw_update_from_pplib(dc); + + if (!dc->debug.disable_pplib_clock_request) { + /* + * TODO: This is not the proper way to obtain + * fabric_and_dram_bandwidth, should be min(fclk, memclk). + */ + res = dm_pp_get_clock_levels_by_type_with_voltage( + ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks); + + DC_FP_START(); + + if (res) + res = verify_clock_values(&fclks); + + if (res) + dcn_bw_update_from_pplib_fclks(dc, &fclks); + else + BREAK_TO_DEBUGGER(); + + DC_FP_END(); + + res = dm_pp_get_clock_levels_by_type_with_voltage( + ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks); + + DC_FP_START(); + + if (res) + res = verify_clock_values(&dcfclks); + + if (res) + dcn_bw_update_from_pplib_dcfclks(dc, &dcfclks); + else + BREAK_TO_DEBUGGER(); + + DC_FP_END(); + } + dcn_bw_sync_calcs_and_dml(dc); if (!dc->debug.disable_pplib_wm_range) { dc->res_pool = &pool->base; - dcn_bw_notify_pplib_of_wm_ranges(dc); + DC_FP_START(); + dcn_get_soc_clks( + dc, &min_fclk_khz, &min_dcfclk_khz, &socclk_khz); + DC_FP_END(); + dcn_bw_notify_pplib_of_wm_ranges( + dc, min_fclk_khz, min_dcfclk_khz, socclk_khz); } - DC_FP_END(); { struct irq_service_init_data init_data; diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h index 2b9d3e63191b227b60df1a92d6e20406637ae04b..915a20461c77cbce6f604f726486a7d2b25d05a2 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h @@ -274,6 +274,7 @@ struct dccg_registers { uint32_t DSCCLK2_DTO_PARAM; uint32_t DPSTREAMCLK_ROOT_GATE_DISABLE; uint32_t DPSTREAMCLK_GATE_DISABLE; + uint32_t DCCG_GATE_DISABLE_CNTL; uint32_t DCCG_GATE_DISABLE_CNTL2; uint32_t DCCG_GATE_DISABLE_CNTL3; uint32_t HDMISTREAMCLK0_DTO_PARAM; diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h index cd2671161ef1601b515c4fe1aa8bdb1a4c57b479..7ce64a3c1b02b4364e3bc9387550e76e06a81e60 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dsc.h @@ -445,226 +445,6 @@ type DSCRM_DSC_FORWARD_EN; \ type DSCRM_DSC_OPP_PIPE_SOURCE -#define DSC_REG_LIST_DCN314(id) \ - SRI(DSC_TOP_CONTROL, DSC_TOP, id),\ - SRI(DSC_DEBUG_CONTROL, DSC_TOP, id),\ - SRI(DSCC_CONFIG0, DSCC, id),\ - SRI(DSCC_CONFIG1, DSCC, id),\ - SRI(DSCC_STATUS, DSCC, id),\ - SRI(DSCC_INTERRUPT_CONTROL_STATUS, DSCC, id),\ - SRI(DSCC_PPS_CONFIG0, DSCC, id),\ - SRI(DSCC_PPS_CONFIG1, DSCC, id),\ - SRI(DSCC_PPS_CONFIG2, DSCC, id),\ - SRI(DSCC_PPS_CONFIG3, DSCC, id),\ - SRI(DSCC_PPS_CONFIG4, DSCC, id),\ - SRI(DSCC_PPS_CONFIG5, DSCC, id),\ - SRI(DSCC_PPS_CONFIG6, DSCC, id),\ - SRI(DSCC_PPS_CONFIG7, DSCC, id),\ - SRI(DSCC_PPS_CONFIG8, DSCC, id),\ - SRI(DSCC_PPS_CONFIG9, DSCC, id),\ - SRI(DSCC_PPS_CONFIG10, DSCC, id),\ - SRI(DSCC_PPS_CONFIG11, DSCC, id),\ - SRI(DSCC_PPS_CONFIG12, DSCC, id),\ - SRI(DSCC_PPS_CONFIG13, DSCC, id),\ - SRI(DSCC_PPS_CONFIG14, DSCC, id),\ - SRI(DSCC_PPS_CONFIG15, DSCC, id),\ - SRI(DSCC_PPS_CONFIG16, DSCC, id),\ - SRI(DSCC_PPS_CONFIG17, DSCC, id),\ - SRI(DSCC_PPS_CONFIG18, DSCC, id),\ - SRI(DSCC_PPS_CONFIG19, DSCC, id),\ - SRI(DSCC_PPS_CONFIG20, DSCC, id),\ - SRI(DSCC_PPS_CONFIG21, DSCC, id),\ - SRI(DSCC_PPS_CONFIG22, DSCC, id),\ - SRI(DSCC_MEM_POWER_CONTROL, DSCC, id),\ - SRI(DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC, id),\ - SRI(DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC, id),\ - SRI(DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC, id),\ - SRI(DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC, id),\ - SRI(DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC, id),\ - SRI(DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC, id),\ - SRI(DSCC_MAX_ABS_ERROR0, DSCC, id),\ - SRI(DSCC_MAX_ABS_ERROR1, DSCC, id),\ - SRI(DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),\ - SRI(DSCCIF_CONFIG0, DSCCIF, id),\ - SRI(DSCCIF_CONFIG1, DSCCIF, id),\ - SRI(DSCRM_DSC_FORWARD_CONFIG, DSCRM, id) - -#define DSC_REG_LIST_SH_MASK_DCN314(mask_sh)\ - DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_CLOCK_EN, mask_sh), \ - DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DISPCLK_R_GATE_DIS, mask_sh), \ - DSC_SF(DSC_TOP0_DSC_TOP_CONTROL, DSC_DSCCLK_R_GATE_DIS, mask_sh), \ - DSC_SF(DSC_TOP0_DSC_DEBUG_CONTROL, DSC_DBG_EN, mask_sh), \ - DSC_SF(DSC_TOP0_DSC_DEBUG_CONTROL, DSC_TEST_CLOCK_MUX_SEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_CONFIG0, NUMBER_OF_SLICES_PER_LINE, mask_sh), \ - DSC_SF(DSCC0_DSCC_CONFIG0, ALTERNATE_ICH_ENCODING_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_CONFIG0, NUMBER_OF_SLICES_IN_VERTICAL_DIRECTION, mask_sh), \ - DSC_SF(DSCC0_DSCC_CONFIG1, DSCC_RATE_CONTROL_BUFFER_MODEL_SIZE, mask_sh), \ - /*DSC_SF(DSCC0_DSCC_CONFIG1, DSCC_DISABLE_ICH, mask_sh),*/ \ - DSC_SF(DSCC0_DSCC_STATUS, DSCC_DOUBLE_BUFFER_REG_UPDATE_PENDING, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER0_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER1_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER2_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_BUFFER3_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL0_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL1_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL2_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_INTERRUPT_CONTROL_STATUS, DSCC_RATE_CONTROL_BUFFER_MODEL3_OVERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG0, DSC_VERSION_MINOR, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG0, DSC_VERSION_MAJOR, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG0, PPS_IDENTIFIER, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG0, LINEBUF_DEPTH, mask_sh), \ - DSC2_SF(DSCC0, DSCC_PPS_CONFIG0__BITS_PER_COMPONENT, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, BITS_PER_PIXEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, VBR_ENABLE, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, SIMPLE_422, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, CONVERT_RGB, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, BLOCK_PRED_ENABLE, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, NATIVE_422, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, NATIVE_420, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG1, CHUNK_SIZE, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG2, PIC_WIDTH, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG2, PIC_HEIGHT, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG3, SLICE_WIDTH, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG3, SLICE_HEIGHT, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG4, INITIAL_XMIT_DELAY, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG4, INITIAL_DEC_DELAY, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG5, INITIAL_SCALE_VALUE, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG5, SCALE_INCREMENT_INTERVAL, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG6, SCALE_DECREMENT_INTERVAL, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG6, FIRST_LINE_BPG_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG6, SECOND_LINE_BPG_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG7, NFL_BPG_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG7, SLICE_BPG_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG8, NSL_BPG_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG8, SECOND_LINE_OFFSET_ADJ, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG9, INITIAL_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG9, FINAL_OFFSET, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG10, FLATNESS_MIN_QP, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG10, FLATNESS_MAX_QP, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG10, RC_MODEL_SIZE, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_EDGE_FACTOR, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_QUANT_INCR_LIMIT0, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_QUANT_INCR_LIMIT1, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_TGT_OFFSET_LO, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG11, RC_TGT_OFFSET_HI, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH0, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH1, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH2, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG12, RC_BUF_THRESH3, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH4, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH5, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH6, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG13, RC_BUF_THRESH7, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH8, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH9, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH10, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG14, RC_BUF_THRESH11, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RC_BUF_THRESH12, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RC_BUF_THRESH13, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_MIN_QP0, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_MAX_QP0, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG15, RANGE_BPG_OFFSET0, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MIN_QP1, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MAX_QP1, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_BPG_OFFSET1, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MIN_QP2, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_MAX_QP2, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG16, RANGE_BPG_OFFSET2, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MIN_QP3, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MAX_QP3, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_BPG_OFFSET3, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MIN_QP4, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_MAX_QP4, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG17, RANGE_BPG_OFFSET4, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MIN_QP5, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MAX_QP5, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_BPG_OFFSET5, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MIN_QP6, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_MAX_QP6, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG18, RANGE_BPG_OFFSET6, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MIN_QP7, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MAX_QP7, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_BPG_OFFSET7, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MIN_QP8, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_MAX_QP8, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG19, RANGE_BPG_OFFSET8, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MIN_QP9, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MAX_QP9, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_BPG_OFFSET9, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MIN_QP10, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_MAX_QP10, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG20, RANGE_BPG_OFFSET10, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MIN_QP11, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MAX_QP11, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_BPG_OFFSET11, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MIN_QP12, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_MAX_QP12, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG21, RANGE_BPG_OFFSET12, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MIN_QP13, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MAX_QP13, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_BPG_OFFSET13, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MIN_QP14, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_MAX_QP14, mask_sh), \ - DSC_SF(DSCC0_DSCC_PPS_CONFIG22, RANGE_BPG_OFFSET14, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_DEFAULT_MEM_LOW_POWER_STATE, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_FORCE, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_DIS, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_MEM_PWR_STATE, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_FORCE, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_DIS, mask_sh), \ - DSC_SF(DSCC0_DSCC_MEM_POWER_CONTROL, DSCC_NATIVE_422_MEM_PWR_STATE, mask_sh), \ - DSC_SF(DSCC0_DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC_R_Y_SQUARED_ERROR_LOWER, mask_sh), \ - DSC_SF(DSCC0_DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC_R_Y_SQUARED_ERROR_UPPER, mask_sh), \ - DSC_SF(DSCC0_DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC_G_CB_SQUARED_ERROR_LOWER, mask_sh), \ - DSC_SF(DSCC0_DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC_G_CB_SQUARED_ERROR_UPPER, mask_sh), \ - DSC_SF(DSCC0_DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC_B_CR_SQUARED_ERROR_LOWER, mask_sh), \ - DSC_SF(DSCC0_DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC_B_CR_SQUARED_ERROR_UPPER, mask_sh), \ - DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR0, DSCC_R_Y_MAX_ABS_ERROR, mask_sh), \ - DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR0, DSCC_G_CB_MAX_ABS_ERROR, mask_sh), \ - DSC_SF(DSCC0_DSCC_MAX_ABS_ERROR1, DSCC_B_CR_MAX_ABS_ERROR, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCC0_DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_RECOVERY_EN, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_OCCURRED_INT_EN, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_INTERFACE_UNDERFLOW_OCCURRED_STATUS, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG0, INPUT_PIXEL_FORMAT, mask_sh), \ - DSC2_SF(DSCCIF0, DSCCIF_CONFIG0__BITS_PER_COMPONENT, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG0, DOUBLE_BUFFER_REG_UPDATE_PENDING, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG1, PIC_WIDTH, mask_sh), \ - DSC_SF(DSCCIF0_DSCCIF_CONFIG1, PIC_HEIGHT, mask_sh), \ - DSC_SF(DSCRM0_DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_FORWARD_EN, mask_sh), \ - DSC_SF(DSCRM0_DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_OPP_PIPE_SOURCE, mask_sh) - - struct dcn20_dsc_registers { uint32_t DSC_TOP_CONTROL; uint32_t DSC_DEBUG_CONTROL; diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c index b1ec0e6f7f5877948c6b562cccea1ef65850245e..4996d2810edb8e1b12183976428adc63cb0ee59c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c @@ -617,6 +617,17 @@ void hubp2_cursor_set_attributes( CURSOR0_DST_Y_OFFSET, 0, /* used to shift the cursor chunk request deadline */ CURSOR0_CHUNK_HDL_ADJUST, 3); + + hubp->att.SURFACE_ADDR_HIGH = attr->address.high_part; + hubp->att.SURFACE_ADDR = attr->address.low_part; + hubp->att.size.bits.width = attr->width; + hubp->att.size.bits.height = attr->height; + hubp->att.cur_ctl.bits.mode = attr->color_format; + hubp->att.cur_ctl.bits.pitch = hw_pitch; + hubp->att.cur_ctl.bits.line_per_chunk = lpc; + hubp->att.cur_ctl.bits.cur_2x_magnify = attr->attribute_flags.bits.ENABLE_MAGNIFICATION; + hubp->att.settings.bits.dst_y_offset = 0; + hubp->att.settings.bits.chunk_hdl_adjust = 3; } void hubp2_dmdata_set_attributes( @@ -1033,6 +1044,25 @@ void hubp2_cursor_set_position( REG_SET(CURSOR_DST_OFFSET, 0, CURSOR_DST_X_OFFSET, dst_x_offset); /* TODO Handle surface pixel formats other than 4:4:4 */ + /* Cursor Position Register Config */ + hubp->pos.cur_ctl.bits.cur_enable = cur_en; + hubp->pos.position.bits.x_pos = pos->x; + hubp->pos.position.bits.y_pos = pos->y; + hubp->pos.hot_spot.bits.x_hot = x_hotspot; + hubp->pos.hot_spot.bits.y_hot = y_hotspot; + hubp->pos.dst_offset.bits.dst_x_offset = dst_x_offset; + /* Cursor Rectangle Cache + * Cursor bitmaps have different hotspot values + * There's a possibility that the above logic returns a negative value, + * so we clamp them to 0 + */ + if (src_x_offset < 0) + src_x_offset = 0; + if (src_y_offset < 0) + src_y_offset = 0; + /* Save necessary cursor info x, y position. w, h is saved in attribute func. */ + hubp->cur_rect.x = src_x_offset + param->viewport.x; + hubp->cur_rect.y = src_y_offset + param->viewport.y; } void hubp2_clk_cntl(struct hubp *hubp, bool enable) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 6271caca4d9a868546e0e12243e576146d16a70e..d732b6f031a12a28e5cc859e6a6d9828a1636b93 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -706,6 +706,14 @@ enum dc_status dcn20_enable_stream_timing( return DC_ERROR_UNEXPECTED; } + if (dc_is_hdmi_tmds_signal(stream->signal)) { + stream->link->phy_state.symclk_ref_cnts.otg = 1; + if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF) + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF; + else + stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON; + } + if (dc->hwseq->funcs.PLAT_58856_wa && (!dc_is_dp_signal(stream->signal))) dc->hwseq->funcs.PLAT_58856_wa(context, pipe_ctx); @@ -1565,6 +1573,7 @@ static void dcn20_update_dchubp_dpp( /* Any updates are handled in dc interface, just need * to apply existing for plane enable / opp change */ if (pipe_ctx->update_flags.bits.enable || pipe_ctx->update_flags.bits.opp_changed + || pipe_ctx->update_flags.bits.plane_changed || pipe_ctx->stream->update_flags.bits.gamut_remap || pipe_ctx->stream->update_flags.bits.out_csc) { /* dpp/cm gamut remap*/ @@ -1851,24 +1860,6 @@ void dcn20_post_unlock_program_front_end( } } - for (i = 0; i < dc->res_pool->pipe_count; i++) { - struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; - struct pipe_ctx *mpcc_pipe; - - if (pipe->vtp_locked) { - dc->hwseq->funcs.wait_for_blank_complete(pipe->stream_res.opp); - pipe->plane_res.hubp->funcs->set_blank(pipe->plane_res.hubp, true); - pipe->vtp_locked = false; - - for (mpcc_pipe = pipe->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe) - mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, true); - - for (i = 0; i < dc->res_pool->pipe_count; i++) - if (context->res_ctx.pipe_ctx[i].update_flags.bits.disable) - dc->hwss.disable_plane(dc, &dc->current_state->res_ctx.pipe_ctx[i]); - } - } - for (i = 0; i < dc->res_pool->pipe_count; i++) { struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; @@ -1898,8 +1889,14 @@ void dcn20_post_unlock_program_front_end( * can underflow due to HUBP_VTG_SEL programming if done in the regular front end * programming sequence). */ - if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) + while (pipe) { + if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) { + if (dc->hwss.update_phantom_vp_position) + dc->hwss.update_phantom_vp_position(dc, context, pipe); dcn20_program_pipe(dc, pipe, context); + } + pipe = pipe->bottom_pipe; + } } } @@ -2003,6 +2000,10 @@ void dcn20_optimize_bandwidth( context->bw_ctx.bw.dcn.clk.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000) dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->dc_mode_softmax_memclk); + /* increase compbuf size */ + if (hubbub->funcs->program_compbuf_size) + hubbub->funcs->program_compbuf_size(hubbub, context->bw_ctx.bw.dcn.compbuf_size_kb, true); + dc->clk_mgr->funcs->update_clocks( dc->clk_mgr, context, @@ -2018,9 +2019,6 @@ void dcn20_optimize_bandwidth( pipe_ctx->dlg_regs.optimized_min_dst_y_next_start); } } - /* increase compbuf size */ - if (hubbub->funcs->program_compbuf_size) - hubbub->funcs->program_compbuf_size(hubbub, context->bw_ctx.bw.dcn.compbuf_size_kb, true); } bool dcn20_update_bandwidth( @@ -2346,7 +2344,9 @@ static void dcn20_reset_back_end_for_pipe( struct dc_state *context) { int i; - struct dc_link *link; + struct dc_link *link = pipe_ctx->stream->link; + const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); + DC_LOGGER_INIT(dc->ctx->logger); if (pipe_ctx->stream_res.stream_enc == NULL) { pipe_ctx->stream = NULL; @@ -2354,19 +2354,15 @@ static void dcn20_reset_back_end_for_pipe( } if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) { - link = pipe_ctx->stream->link; /* DPMS may already disable or */ /* dpms_off status is incorrect due to fastboot * feature. When system resume from S4 with second * screen only, the dpms_off would be true but * VBIOS lit up eDP, so check link status too. */ - if (!pipe_ctx->stream->dpms_off || link->link_status.link_active) { - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(dc->current_state, pipe_ctx, TX_OFF_SYMCLK_OFF); - else - core_link_disable_stream(pipe_ctx); - } else if (pipe_ctx->stream_res.audio) + if (!pipe_ctx->stream->dpms_off || link->link_status.link_active) + core_link_disable_stream(pipe_ctx); + else if (pipe_ctx->stream_res.audio) dc->hwss.disable_audio_stream(pipe_ctx); /* free acquired resources */ @@ -2406,6 +2402,16 @@ static void dcn20_reset_back_end_for_pipe( if (pipe_ctx->stream_res.tg->funcs->set_drr) pipe_ctx->stream_res.tg->funcs->set_drr( pipe_ctx->stream_res.tg, NULL); + /* TODO - convert symclk_ref_cnts for otg to a bit map to solve + * the case where the same symclk is shared across multiple otg + * instances + */ + link->phy_state.symclk_ref_cnts.otg = 0; + if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) { + link_hwss->disable_link_output(link, + &pipe_ctx->link_res, pipe_ctx->stream->signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + } } for (i = 0; i < dc->res_pool->pipe_count; i++) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c index 91e4885b743e837ca9464b5d0fca74e961c98ba2..7c5817c426faa78c55da31ac1eacf83c315fb737 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c @@ -96,6 +96,10 @@ static const struct hw_sequencer_funcs dcn20_funcs = { #ifndef TRIM_FSFT .optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft, #endif + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, .get_dcc_en_bits = dcn10_get_dcc_en_bits, .update_visual_confirm_color = dcn20_update_visual_confirm_color diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c index 694260c10a015ed353cb7bfacae01186dc85ab1f..ccd91792991b0848539927307aebc0353a8b08cc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c @@ -215,7 +215,8 @@ void mmhubbub2_config_mcif_irq(struct mcif_wb *mcif_wb, REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, params->sw_overrun_int_en); REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, params->vce_int_en); - REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, params->vce_slice_int_en); + if (mcif_wb20->mcif_wb_mask->MCIF_WB_BUFMGR_VCE_SLICE_INT_EN) + REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, params->vce_slice_int_en); } void mmhubbub2_enable_mcif(struct mcif_wb *mcif_wb) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c index 0340fdd3f5fbb982335a211e8a0bd4368724cff5..a08c335b738386e53aee90ea88b71abcb7d2304a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_optc.c @@ -529,6 +529,7 @@ static struct timing_generator_funcs dcn20_tg_funcs = { .enable_optc_clock = optc1_enable_optc_clock, .set_drr = optc1_set_drr, .get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal, + .set_vtotal_min_max = optc1_set_vtotal_min_max, .set_static_screen_control = optc1_set_static_screen_control, .program_stereo = optc1_program_stereo, .is_stereo_left_eye = optc1_is_stereo_left_eye, diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c index 05b3fba9ccce8f89721cab7fbd50088ab1e5d435..61bcfa03c4e7c5f6e070586f9c7165232960b8b8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c @@ -82,7 +82,7 @@ static bool patch_address_for_sbs_tb_stereo( return false; } -static void gpu_addr_to_uma(struct dce_hwseq *hwseq, +static bool gpu_addr_to_uma(struct dce_hwseq *hwseq, PHYSICAL_ADDRESS_LOC *addr) { bool is_in_uma; @@ -98,6 +98,7 @@ static void gpu_addr_to_uma(struct dce_hwseq *hwseq, } else { is_in_uma = false; } + return is_in_uma; } static void plane_address_in_gpu_space_to_uma(struct dce_hwseq *hwseq, diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c index 1826dd7f3da142b74cb381f200170f1bfadaff37..9c16633e473a0ff87de4589e60a7e88adbd7aa2f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c @@ -86,6 +86,10 @@ static const struct hw_sequencer_funcs dcn201_funcs = { .set_backlight_level = dce110_set_backlight_level, .set_abm_immediate_disable = dce110_set_abm_immediate_disable, .set_pipe = dce110_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, .update_visual_confirm_color = dcn20_update_visual_confirm_color, }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c index 5752271f22dfedda223a7feabdbaeb0b37047505..c5e200d09038fba2cf7cfc1eb3ceba438ed33fa9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hubbub.c @@ -67,15 +67,9 @@ static uint32_t convert_and_clamp( void dcn21_dchvm_init(struct hubbub *hubbub) { struct dcn20_hubbub *hubbub1 = TO_DCN20_HUBBUB(hubbub); - uint32_t riommu_active, prefetch_done; + uint32_t riommu_active; int i; - REG_GET(DCHVM_RIOMMU_STAT0, HOSTVM_PREFETCH_DONE, &prefetch_done); - - if (prefetch_done) { - hubbub->riommu_active = true; - return; - } //Init DCHVM block REG_UPDATE(DCHVM_CTRL0, HOSTVM_INIT_REQ, 1); diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c index b270f0b194dcb69905f587c7365fd846f5b90dcd..fe1a8e2e08ef1bf689ce33c3f99a810c68802a02 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c @@ -99,6 +99,10 @@ static const struct hw_sequencer_funcs dcn21_funcs = { #ifndef TRIM_FSFT .optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft, #endif + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .is_abm_supported = dcn21_is_abm_supported, .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, .get_dcc_en_bits = dcn10_get_dcc_en_bits, diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 7cb35bb1c0f153608bb3fe47f371807d48029bc5..887081472c0d8e3a0927c51f142e5e48ccf0dc19 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -657,7 +657,6 @@ static const struct dc_debug_options debug_defaults_drv = { .usbc_combo_phy_reset_wa = true, .dmub_command_table = true, .use_max_lb = true, - .optimize_edp_link_rate = true }; static const struct dc_debug_options debug_defaults_diags = { @@ -677,6 +676,12 @@ static const struct dc_debug_options debug_defaults_diags = { .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .ilr = { + .optimize_edp_link_rate = true, + }, +}; + enum dcn20_clk_src_array_id { DCN20_CLK_SRC_PLL0, DCN20_CLK_SRC_PLL1, @@ -1367,6 +1372,11 @@ static struct panel_cntl *dcn21_panel_cntl_create(const struct panel_cntl_init_d return &panel_cntl->base; } +static void dcn21_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + #define CTX ctx #define REG(reg_name) \ @@ -1408,6 +1418,7 @@ static const struct resource_funcs dcn21_res_pool_funcs = { .set_mcif_arb_params = dcn20_set_mcif_arb_params, .find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link, .update_bw_bounding_box = dcn21_update_bw_bounding_box, + .get_panel_config_defaults = dcn21_get_panel_config_defaults, }; static bool dcn21_resource_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c index 4a668d6563dfd6aff3dd329d0bcb27f36bf6c9a3..e5b7ef7422b833ea5b0301f024784cb3e906accc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp.c @@ -372,6 +372,10 @@ void dpp3_set_cursor_attributes( REG_UPDATE(CURSOR0_COLOR1, CUR0_COLOR1, 0xFFFFFFFF); } + + dpp_base->att.cur0_ctl.bits.expansion_mode = 0; + dpp_base->att.cur0_ctl.bits.cur0_rom_en = cur_rom_en; + dpp_base->att.cur0_ctl.bits.mode = color_format; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index fb59fed8f425474d6b967899be38a8a844688178..8c504571126499467c23c0bd98f972490fee75a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -939,13 +939,32 @@ bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, s void dcn30_hardware_release(struct dc *dc) { + bool subvp_in_use = false; + uint32_t i; + dc_dmub_srv_p_state_delegate(dc, false, NULL); + dc_dmub_setup_subvp_dmub_command(dc, dc->current_state, false); + + /* SubVP treated the same way as FPO. If driver disable and + * we are using a SubVP config, disable and force on DCN side + * to prevent P-State hang on driver enable. + */ + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + if (!pipe->stream) + continue; + + if (pipe->stream->mall_stream_config.type == SUBVP_MAIN) { + subvp_in_use = true; + break; + } + } /* If pstate unsupported, or still supported * by firmware, force it supported by dcn */ if (dc->current_state) - if ((!dc->clk_mgr->clks.p_state_change_support || + if ((!dc->clk_mgr->clks.p_state_change_support || subvp_in_use || dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) && dc->res_pool->hubbub->funcs->force_pstate_change_control) dc->res_pool->hubbub->funcs->force_pstate_change_control( diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c index 4c06e6e1ba4a6f654e79c1e74d1ade4e52ce4b93..3216d10c58ba71d31bfd992e7f28e30769a5194b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c @@ -100,6 +100,10 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .hardware_release = dcn30_hardware_release, .set_pipe = dcn21_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, .get_dcc_en_bits = dcn10_get_dcc_en_bits, .update_visual_confirm_color = dcn20_update_visual_confirm_color, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h index f2580e65196ccd145891ad61cdba7b29896494ff..7446e54bf5aacbbb8970d8df3f8ae7fac50c7115 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h @@ -227,11 +227,7 @@ SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\ SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\ - SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\ - SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\ - SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\ - SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\ SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\ @@ -363,11 +359,7 @@ SF(MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\ SF(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\ SF(MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\ SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\ SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c index 1782b9c26cf4bb3e334c9e65b5eb2c4b9121ed5c..892d3c4d01a1ecd4742bd1ee61b0bd31e3cbdc0b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c @@ -319,13 +319,13 @@ static struct timing_generator_funcs dcn30_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc3_lock, - .is_locked = optc1_is_locked, .unlock = optc1_unlock, .lock_doublebuffer_enable = optc3_lock_doublebuffer_enable, .lock_doublebuffer_disable = optc3_lock_doublebuffer_disable, .enable_optc_clock = optc1_enable_optc_clock, .set_drr = optc1_set_drr, .get_last_used_drr_vtotal = optc2_get_last_used_drr_vtotal, + .set_vtotal_min_max = optc3_set_vtotal_min_max, .set_static_screen_control = optc1_set_static_screen_control, .program_stereo = optc1_program_stereo, .is_stereo_left_eye = optc1_is_stereo_left_eye, @@ -366,4 +366,3 @@ void dcn30_timing_generator_init(struct optc *optc1) optc1->min_h_sync_width = 4; optc1->min_v_sync_width = 1; } - diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 64320e0ca446375af56d5cb5652eb0763479e16d..020f512e9690e0c10bb109a855ada9f0e100716f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -724,7 +724,8 @@ static const struct dc_debug_options debug_defaults_drv = { .dwb_fi_phase = -1, // -1 = disable, .dmub_command_table = true, .disable_psr = false, - .use_max_lb = true + .use_max_lb = true, + .exit_idle_opt_for_cursor_updates = true }; static const struct dc_debug_options debug_defaults_diags = { @@ -1654,6 +1655,9 @@ noinline bool dcn30_internal_validate_bw( if (!pipes) return false; + context->bw_ctx.dml.vba.maxMpcComb = 0; + context->bw_ctx.dml.vba.VoltageLevel = 0; + context->bw_ctx.dml.vba.DRAMClockChangeSupport[0][0] = dm_dram_clock_change_vactive; dc->res_pool->funcs->update_soc_for_wm_a(dc, context); pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate); @@ -1872,6 +1876,7 @@ noinline bool dcn30_internal_validate_bw( if (repopulate_pipes) pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate); + context->bw_ctx.dml.vba.VoltageLevel = vlevel; *vlevel_out = vlevel; *pipe_cnt_out = pipe_cnt; @@ -1916,7 +1921,7 @@ static int get_refresh_rate(struct dc_state *context) */ #define V_SCALE (10000 / MAX_STRETCHED_V_BLANK) -int get_frame_rate_at_max_stretch_100hz(struct dc_state *context) +static int get_frame_rate_at_max_stretch_100hz(struct dc_state *context) { struct dc_crtc_timing *timing = NULL; uint32_t sec_per_100_lines; @@ -1946,7 +1951,7 @@ int get_frame_rate_at_max_stretch_100hz(struct dc_state *context) return scaled_refresh_rate; } -bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context) +static bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context) { int refresh_rate_max_stretch_100hz; int min_refresh_100hz; diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c index 3d42a1a337ec1189fac5488380cf2022556c729a..6192851c59ed8950653ba4668177417eb2cf1a3d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c @@ -99,6 +99,10 @@ static const struct hw_sequencer_funcs dcn301_funcs = { .set_backlight_level = dcn21_set_backlight_level, .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, .get_dcc_en_bits = dcn10_get_dcc_en_bits, .optimize_pwr_state = dcn21_optimize_pwr_state, diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c index db172677d61342a8c54c25dbed7766568abdcc0c..f04595b750abcd0b108be9518b536f9b5b0e32db 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c @@ -634,7 +634,7 @@ static const struct dcn20_vmid_mask vmid_masks = { DCN20_VMID_MASK_SH_LIST(_MASK) }; -static const struct resource_caps res_cap_dcn301 = { +static struct resource_caps res_cap_dcn301 = { .num_timing_generator = 4, .num_opp = 4, .num_video_plane = 4, @@ -700,6 +700,7 @@ static const struct dc_debug_options debug_defaults_drv = { .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, .use_max_lb = false, + .exit_idle_opt_for_cursor_updates = true }; static const struct dc_debug_options debug_defaults_diags = { @@ -851,7 +852,7 @@ static struct hubbub *dcn301_hubbub_create(struct dc_context *ctx) vmid->masks = &vmid_masks; } - hubbub3->num_vmid = res_cap_dcn301.num_vmid; + hubbub3->num_vmid = res_cap_dcn301.num_vmid; return &hubbub3->base; } @@ -1429,6 +1430,8 @@ static bool dcn301_resource_construct( ctx->dc_bios->regs = &bios_regs; + if (dc->ctx->asic_id.chip_id == DEVICE_ID_VGH_1435) + res_cap_dcn301.num_pll = 2; pool->base.res_cap = &res_cap_dcn301; pool->base.funcs = &dcn301_res_pool_funcs; diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index 4fab537e822ff83c65729141f695fb3c5e26da13..b925b6ddde5a34d6e5afc0efc8d448c9e211ace0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -93,7 +93,8 @@ static const struct dc_debug_options debug_defaults_drv = { .underflow_assert_delay_us = 0xFFFFFFFF, .dwb_fi_phase = -1, // -1 = disable, .dmub_command_table = true, - .use_max_lb = true + .use_max_lb = true, + .exit_idle_opt_for_cursor_updates = true }; static const struct dc_debug_options debug_defaults_diags = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c index d97076648acba46ea4c53ddb170a59ff43601d7d..527d5c902878549b031d24fb9e17fc3bf4f3df12 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c @@ -77,6 +77,7 @@ static const struct dc_debug_options debug_defaults_drv = { .underflow_assert_delay_us = 0xFFFFFFFF, .dwb_fi_phase = -1, // -1 = disable, .dmub_command_table = true, + .exit_idle_opt_for_cursor_updates = true, .disable_idle_power_optimizations = false, }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c index 23621ff08c9053c781e93ddcb9904522f5301b07..814f401db3b34bcdc853e2ec64ffba82bae639c8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c @@ -150,9 +150,9 @@ static void dcn31_hpo_dp_stream_enc_dp_blank( * 10us*5000=50ms. This covers 41.7ms of minimum 24 Hz mode + * a little more because we may not trust delay accuracy. */ - //REG_WAIT(DP_SYM32_ENC_VID_STREAM_CONTROL, - // VID_STREAM_STATUS, 0, - // 10, 5000); + REG_WAIT(DP_SYM32_ENC_VID_STREAM_CONTROL, + VID_STREAM_STATUS, 0, + 10, 5000); /* Disable SDP tranmission */ REG_UPDATE(DP_SYM32_ENC_SDP_CONTROL, @@ -197,7 +197,7 @@ static void dcn31_hpo_dp_stream_enc_set_stream_attribute( uint32_t h_back_porch; uint32_t h_width; uint32_t v_height; - unsigned long long v_freq; + uint64_t v_freq; uint8_t misc0 = 0; uint8_t misc1 = 0; uint8_t hsp; @@ -360,7 +360,7 @@ static void dcn31_hpo_dp_stream_enc_set_stream_attribute( v_height = hw_crtc_timing.v_border_top + hw_crtc_timing.v_addressable + hw_crtc_timing.v_border_bottom; hsp = hw_crtc_timing.flags.HSYNC_POSITIVE_POLARITY ? 0 : 0x80; vsp = hw_crtc_timing.flags.VSYNC_POSITIVE_POLARITY ? 0 : 0x80; - v_freq = hw_crtc_timing.pix_clk_100hz * 100; + v_freq = (uint64_t)hw_crtc_timing.pix_clk_100hz * 100; /* MSA Packet Mapping to 32-bit Link Symbols - DP2 spec, section 2.7.4.1 * @@ -436,32 +436,28 @@ static void dcn31_hpo_dp_stream_enc_update_dp_info_packets( { struct dcn31_hpo_dp_stream_encoder *enc3 = DCN3_1_HPO_DP_STREAM_ENC_FROM_HPO_STREAM_ENC(enc); uint32_t dmdata_packet_enabled = 0; - bool sdp_stream_enable = false; - if (info_frame->vsc.valid) { + if (info_frame->vsc.valid) enc->vpg->funcs->update_generic_info_packet( enc->vpg, 0, /* packetIndex */ &info_frame->vsc, true); - sdp_stream_enable = true; - } - if (info_frame->spd.valid) { + + if (info_frame->spd.valid) enc->vpg->funcs->update_generic_info_packet( enc->vpg, 2, /* packetIndex */ &info_frame->spd, true); - sdp_stream_enable = true; - } - if (info_frame->hdrsmd.valid) { + + if (info_frame->hdrsmd.valid) enc->vpg->funcs->update_generic_info_packet( enc->vpg, 3, /* packetIndex */ &info_frame->hdrsmd, true); - sdp_stream_enable = true; - } + /* enable/disable transmission of packet(s). * If enabled, packet transmission begins on the next frame */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c index 51c5f3685470a3aa158a122850095406a3f524fa..6360dc9502e7039db13eea60e4e1c96f996bfd44 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c @@ -876,7 +876,7 @@ static bool hubbub31_get_dcc_compression_cap(struct hubbub *hubbub, return true; } -static int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub, +int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub, struct dcn_hubbub_phys_addr_config *pa_config) { struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub); diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h index e3a654bf04e8655238ef908526b1f52b2b917d34..70c60de448ac31b99c3e95986faa71b06c6cfc41 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h @@ -122,6 +122,8 @@ HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_PIPE, mask_sh), \ HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_INTERRUPT_STATUS, mask_sh) +int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub, + struct dcn_hubbub_phys_addr_config *pa_config); void hubbub31_construct(struct dcn20_hubbub *hubbub3, struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c index 8d83b611507afa348d9c03a2629ea495d03c1f88..bdf101547484a1d543a33f8a7b1af8395657b439 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c @@ -535,11 +535,11 @@ static void dcn31_reset_back_end_for_pipe( pipe_ctx->stream_res.tg, OPTC_DSC_DISABLED, 0, 0); pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg); - pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false); if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass) pipe_ctx->stream_res.tg->funcs->set_odm_bypass( pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing); + pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0; if (pipe_ctx->stream_res.tg->funcs->set_drr) pipe_ctx->stream_res.tg->funcs->set_drr( @@ -553,12 +553,9 @@ static void dcn31_reset_back_end_for_pipe( * screen only, the dpms_off would be true but * VBIOS lit up eDP, so check link status too. */ - if (!pipe_ctx->stream->dpms_off || link->link_status.link_active) { - if (dc->hwss.update_phy_state) - dc->hwss.update_phy_state(dc->current_state, pipe_ctx, TX_OFF_SYMCLK_OFF); - else - core_link_disable_stream(pipe_ctx); - } else if (pipe_ctx->stream_res.audio) + if (!pipe_ctx->stream->dpms_off || link->link_status.link_active) + core_link_disable_stream(pipe_ctx); + else if (pipe_ctx->stream_res.audio) dc->hwss.disable_audio_stream(pipe_ctx); /* free acquired resources */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c index e708f07fe75af1b40d45c66a841a194f017fad4d..3a32810bbe382d45a01fdfc0ab80d367c8aee14c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c @@ -100,6 +100,10 @@ static const struct hw_sequencer_funcs dcn31_funcs = { .set_backlight_level = dcn21_set_backlight_level, .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .z10_restore = dcn31_z10_restore, .z10_save_init = dcn31_z10_save_init, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_optc.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_optc.c index 2f7404a9747902d532a444c948269e335f3b8708..63a677c8ee27269abd081bb8d9b4b062e1d86d16 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_optc.c @@ -201,7 +201,6 @@ void optc31_set_drr( // Setup manual flow control for EOF via TRIG_A optc->funcs->setup_manual_trigger(optc); - } else { REG_UPDATE_4(OTG_V_TOTAL_CONTROL, OTG_SET_V_TOTAL_MIN_MASK, 0, @@ -260,7 +259,6 @@ static struct timing_generator_funcs dcn31_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc3_lock, - .is_locked = optc1_is_locked, .unlock = optc1_unlock, .lock_doublebuffer_enable = optc3_lock_doublebuffer_enable, .lock_doublebuffer_disable = optc3_lock_doublebuffer_disable, diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c index 8745132d6374cff1e5f0dadd07c4b90f2dc96788..fddc21a5a04c4da962648844057592a7d42e01e4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c @@ -888,7 +888,6 @@ static const struct dc_debug_options debug_defaults_drv = { } }, .disable_z10 = true, - .optimize_edp_link_rate = true, .enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/ .dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE, }; @@ -911,6 +910,12 @@ static const struct dc_debug_options debug_defaults_diags = { .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .ilr = { + .optimize_edp_link_rate = true, + }, +}; + static void dcn31_dpp_destroy(struct dpp **dpp) { kfree(TO_DCN20_DPP(*dpp)); @@ -1803,6 +1808,11 @@ validate_out: return out; } +static void dcn31_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static struct dc_cap_funcs cap_funcs = { .get_dcc_compression_cap = dcn20_get_dcc_compression_cap }; @@ -1829,6 +1839,7 @@ static struct resource_funcs dcn31_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn31_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn31_get_panel_config_defaults, }; static struct clock_source *dcn30_clock_source_create( diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c index 232cc15979dda16fbc600ff4b4d554d1b8bd8c56..1bd7e0f327d8128d183ac40ac5fea9683d1abf1b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c @@ -45,6 +45,48 @@ #define DC_LOGGER \ dccg->ctx->logger +static void dccg314_get_pixel_rate_div( + struct dccg *dccg, + uint32_t otg_inst, + enum pixel_rate_div *k1, + enum pixel_rate_div *k2) +{ + struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); + uint32_t val_k1 = PIXEL_RATE_DIV_NA, val_k2 = PIXEL_RATE_DIV_NA; + + *k1 = PIXEL_RATE_DIV_NA; + *k2 = PIXEL_RATE_DIV_NA; + + switch (otg_inst) { + case 0: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG0_PIXEL_RATE_DIVK1, &val_k1, + OTG0_PIXEL_RATE_DIVK2, &val_k2); + break; + case 1: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG1_PIXEL_RATE_DIVK1, &val_k1, + OTG1_PIXEL_RATE_DIVK2, &val_k2); + break; + case 2: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG2_PIXEL_RATE_DIVK1, &val_k1, + OTG2_PIXEL_RATE_DIVK2, &val_k2); + break; + case 3: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG3_PIXEL_RATE_DIVK1, &val_k1, + OTG3_PIXEL_RATE_DIVK2, &val_k2); + break; + default: + BREAK_TO_DEBUGGER(); + return; + } + + *k1 = (enum pixel_rate_div)val_k1; + *k2 = (enum pixel_rate_div)val_k2; +} + static void dccg314_set_pixel_rate_div( struct dccg *dccg, uint32_t otg_inst, @@ -52,6 +94,11 @@ static void dccg314_set_pixel_rate_div( enum pixel_rate_div k2) { struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); + enum pixel_rate_div cur_k1 = PIXEL_RATE_DIV_NA, cur_k2 = PIXEL_RATE_DIV_NA; + + dccg314_get_pixel_rate_div(dccg, otg_inst, &cur_k1, &cur_k2); + if (k1 == PIXEL_RATE_DIV_NA || k2 == PIXEL_RATE_DIV_NA || (k1 == cur_k1 && k2 == cur_k2)) + return; switch (otg_inst) { case 0: @@ -137,7 +184,7 @@ static void dccg314_set_dtbclk_p_src( } /* Controls the generation of pixel valid for OTG in (OTG -> HPO case) */ -void dccg314_set_dtbclk_dto( +static void dccg314_set_dtbclk_dto( struct dccg *dccg, const struct dtbclk_dto_params *params) { @@ -181,7 +228,7 @@ void dccg314_set_dtbclk_dto( } } -void dccg314_set_dpstreamclk( +static void dccg314_set_dpstreamclk( struct dccg *dccg, enum streamclk_source src, int otg_inst, @@ -220,7 +267,7 @@ void dccg314_set_dpstreamclk( } } -void dccg314_set_valid_pixel_rate( +static void dccg314_set_valid_pixel_rate( struct dccg *dccg, int ref_dtbclk_khz, int otg_inst, diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h index 9a4a9efc0203156ccfebbaba1563642a0a023cd6..6a35986307af19384ffdecc08fdaa9df50391745 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h @@ -63,34 +63,28 @@ DCCG_SRII(PHASE, DTBCLK_DTO, 3),\ SR(DCCG_AUDIO_DTBCLK_DTO_MODULO),\ SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),\ + SR(DCCG_AUDIO_DTO_SOURCE),\ + SR(DENTIST_DISPCLK_CNTL),\ + SR(DSCCLK0_DTO_PARAM),\ + SR(DSCCLK1_DTO_PARAM),\ + SR(DSCCLK2_DTO_PARAM),\ + SR(DSCCLK_DTO_CTRL),\ + SR(DCCG_GATE_DISABLE_CNTL2),\ + SR(DCCG_GATE_DISABLE_CNTL3),\ + SR(HDMISTREAMCLK0_DTO_PARAM),\ SR(OTG_PIXEL_RATE_DIV),\ SR(DTBCLK_P_CNTL),\ SR(DCCG_AUDIO_DTO_SOURCE) - -#define DCCG_MASK_SH_LIST_DCN314(mask_sh) \ - DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\ +#define DCCG_MASK_SH_LIST_DCN314_COMMON(mask_sh) \ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 0, mask_sh),\ - DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 1, mask_sh),\ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 1, mask_sh),\ - DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 2, mask_sh),\ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 2, mask_sh),\ - DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 3, mask_sh),\ DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 3, mask_sh),\ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\ DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\ DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\ DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\ - DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\ - DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\ - DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\ - DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_SRC_SEL, mask_sh),\ - DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_EN, mask_sh),\ - DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_SRC_SEL, mask_sh),\ - DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_EN, mask_sh),\ - DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_SRC_SEL, mask_sh),\ - DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_EN, mask_sh),\ - DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_SRC_SEL, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_EN, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_EN, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_EN, mask_sh),\ @@ -100,7 +94,6 @@ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_SRC_SEL, mask_sh),\ DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_SRC_SEL, mask_sh),\ DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, mask_sh),\ - DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_DTO_FORCE_DIS, mask_sh),\ DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_SRC_SEL, mask_sh),\ DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_SRC_SEL, mask_sh),\ @@ -148,7 +141,48 @@ DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_SRC_SEL, mask_sh),\ DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_EN, mask_sh),\ DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\ - DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh) + DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\ + DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_MODE, mask_sh),\ + DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, mask_sh),\ + DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_MODULO, mask_sh),\ + DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, mask_sh),\ + DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_MODULO, mask_sh),\ + DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_PHASE, mask_sh),\ + DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_MODULO, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE0_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE1_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE2_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE0_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE1_GATE_DISABLE, mask_sh),\ + DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_PHASE, mask_sh),\ + DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_MODULO, mask_sh) + +#define DCCG_MASK_SH_LIST_DCN314(mask_sh) \ + DCCG_MASK_SH_LIST_DCN314_COMMON(mask_sh),\ + DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\ + DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 1, mask_sh),\ + DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 2, mask_sh),\ + DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 3, mask_sh),\ + DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\ + DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\ + DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\ + DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_SRC_SEL, mask_sh),\ + DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_EN, mask_sh),\ + DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_SRC_SEL, mask_sh),\ + DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_EN, mask_sh),\ + DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_SRC_SEL, mask_sh),\ + DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_EN, mask_sh),\ + DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_SRC_SEL, mask_sh),\ + DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_DTO_FORCE_DIS, mask_sh),\ + DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_DTO_ENABLE, mask_sh),\ + DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_DTO_ENABLE, mask_sh),\ + DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_DTO_ENABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_GATE_DISABLE, mask_sh),\ + DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_GATE_DISABLE, mask_sh) struct dccg *dccg314_create( struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c index e3351ddc566cf6c32d305837d4912243678b2ef2..7e773bf7b895f05b7d5c715ddc6d2cf7aefa0b8f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c @@ -56,7 +56,8 @@ static void enc314_enable_fifo(struct stream_encoder *enc) /* TODO: Confirm if we need to wait for DIG_SYMCLK_FE_ON */ REG_WAIT(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, 1, 10, 5000); - REG_UPDATE_2(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1, DIG_FIFO_READ_START_LEVEL, 0x7); + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7); + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1); REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 10, 5000); REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0); REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 10, 5000); @@ -67,8 +68,7 @@ static void enc314_disable_fifo(struct stream_encoder *enc) { struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); - REG_UPDATE_2(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, 0, - DIG_FIFO_READ_START_LEVEL, 0); + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, 0); } static void enc314_dp_set_odm_combine( @@ -81,7 +81,7 @@ static void enc314_dp_set_odm_combine( } /* setup stream encoder in dvi mode */ -void enc314_stream_encoder_dvi_set_stream_attribute( +static void enc314_stream_encoder_dvi_set_stream_attribute( struct stream_encoder *enc, struct dc_crtc_timing *crtc_timing, bool is_dual_link) @@ -262,6 +262,16 @@ static bool is_two_pixels_per_containter(const struct dc_crtc_timing *timing) return two_pix; } +static void enc314_stream_encoder_dp_blank( + struct dc_link *link, + struct stream_encoder *enc) +{ + /* New to DCN314 - disable the FIFO before VID stream disable. */ + enc314_disable_fifo(enc); + + enc1_stream_encoder_dp_blank(link, enc); +} + static void enc314_stream_encoder_dp_unblank( struct dc_link *link, struct stream_encoder *enc, @@ -317,15 +327,11 @@ static void enc314_stream_encoder_dp_unblank( /* switch DP encoder to CRTC data, but reset it the fifo first. It may happen * that it overflows during mode transition, and sometimes doesn't recover. */ - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7); REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 1); udelay(10); REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 0); - /* DIG Resync FIFO now needs to be explicitly enabled. */ - enc314_enable_fifo(enc); - /* wait 100us for DIG/DP logic to prime * (i.e. a few video lines) */ @@ -341,6 +347,12 @@ static void enc314_stream_encoder_dp_unblank( REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true); + /* + * DIG Resync FIFO now needs to be explicitly enabled. + * This should come after DP_VID_STREAM_ENABLE per HW docs. + */ + enc314_enable_fifo(enc); + dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM); } @@ -409,7 +421,7 @@ static const struct stream_encoder_funcs dcn314_str_enc_funcs = { .stop_dp_info_packets = enc1_stream_encoder_stop_dp_info_packets, .dp_blank = - enc1_stream_encoder_dp_blank, + enc314_stream_encoder_dp_blank, .dp_unblank = enc314_stream_encoder_dp_unblank, .audio_mute_control = enc3_audio_mute_control, diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c index 39931d48f3851cebb7c18a9a73f9ba05f6b00ae1..588c1c71241fac112b27902e60b7233a4a459e42 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c @@ -343,12 +343,14 @@ unsigned int dcn314_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsig { struct dc_stream_state *stream = pipe_ctx->stream; unsigned int odm_combine_factor = 0; - struct dc *dc = pipe_ctx->stream->ctx->dc; bool two_pix_per_container = false; two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing); odm_combine_factor = get_odm_config(pipe_ctx, NULL); + if (pipe_ctx->stream->signal == SIGNAL_TYPE_VIRTUAL) + return odm_combine_factor; + if (is_dp_128b_132b_signal(pipe_ctx)) { *k2_div = PIXEL_RATE_DIV_BY_1; } else if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal) || dc_is_dvi_signal(pipe_ctx->stream->signal)) { @@ -364,7 +366,7 @@ unsigned int dcn314_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsig } else { *k1_div = PIXEL_RATE_DIV_BY_1; *k2_div = PIXEL_RATE_DIV_BY_4; - if ((odm_combine_factor == 2) || dc->debug.enable_dp_dig_pixel_rate_div_policy) + if (odm_combine_factor == 2) *k2_div = PIXEL_RATE_DIV_BY_2; } } @@ -384,21 +386,10 @@ void dcn314_set_pixels_per_cycle(struct pipe_ctx *pipe_ctx) return; odm_combine_factor = get_odm_config(pipe_ctx, NULL); - if (optc2_is_two_pixels_per_containter(&pipe_ctx->stream->timing) || odm_combine_factor > 1 - || dcn314_is_dp_dig_pixel_rate_div_policy(pipe_ctx)) + if (optc2_is_two_pixels_per_containter(&pipe_ctx->stream->timing) || odm_combine_factor > 1) pix_per_cycle = 2; if (pipe_ctx->stream_res.stream_enc->funcs->set_input_mode) pipe_ctx->stream_res.stream_enc->funcs->set_input_mode(pipe_ctx->stream_res.stream_enc, pix_per_cycle); } - -bool dcn314_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx) -{ - struct dc *dc = pipe_ctx->stream->ctx->dc; - - if (dc_is_dp_signal(pipe_ctx->stream->signal) && !is_dp_128b_132b_signal(pipe_ctx) && - dc->debug.enable_dp_dig_pixel_rate_div_policy) - return true; - return false; -} diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h index d014580592aca6aa8286beeb42e0d7a70f5e5211..244280298212c5c97bd9147c4fd51e9650515665 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.h @@ -41,6 +41,4 @@ unsigned int dcn314_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsig void dcn314_set_pixels_per_cycle(struct pipe_ctx *pipe_ctx); -bool dcn314_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx); - #endif /* __DC_HWSS_DCN314_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c index fcf67eb3478f07e60e361bdaaee35098c4ad145f..5b6c2d94ec71d90e28c5efa4889190bf5f9a458a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c @@ -102,6 +102,10 @@ static const struct hw_sequencer_funcs dcn314_funcs = { .set_backlight_level = dcn21_set_backlight_level, .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dce110_disable_link_output, .z10_restore = dcn31_z10_restore, .z10_save_init = dcn31_z10_save_init, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, @@ -146,7 +150,6 @@ static const struct hwseq_private_funcs dcn314_private_funcs = { .setup_hpo_hw_control = dcn31_setup_hpo_hw_control, .calculate_dccg_k1_k2_values = dcn314_calculate_dccg_k1_k2_values, .set_pixels_per_cycle = dcn314_set_pixels_per_cycle, - .is_dp_dig_pixel_rate_div_policy = dcn314_is_dp_dig_pixel_rate_div_policy, }; void dcn314_hw_sequencer_construct(struct dc *dc) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c index 38aa28ec6b1305f49f84fe929b55b63482fc800d..47eb162f1a7587a6e32bd61fc0620db5e2bab926 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c @@ -150,7 +150,7 @@ static bool optc314_disable_crtc(struct timing_generator *optc) return true; } -void optc314_phantom_crtc_post_enable(struct timing_generator *optc) +static void optc314_phantom_crtc_post_enable(struct timing_generator *optc) { struct optc *optc1 = DCN10TG_FROM_TG(optc); diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c index 49b7e256d4ea53172a142ebb8fee34ce7be10d24..d0ad72caead289dd11b5d9ced58c27d1a097b84f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c @@ -87,6 +87,9 @@ #define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH__SHIFT 0x10 #define DCHUBBUB_DEBUG_CTRL_0__DET_DEPTH_MASK 0x01FF0000L +#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE__SHIFT 0x0 +#define DSCC0_DSCC_CONFIG0__ICH_RESET_AT_END_OF_LINE_MASK 0x0000000FL + #include "reg_helper.h" #include "dce/dmub_abm.h" #include "dce/dmub_psr.h" @@ -579,7 +582,7 @@ static const struct dcn30_mmhubbub_mask mcif_wb30_mask = { #define dsc_regsDCN314(id)\ [id] = {\ - DSC_REG_LIST_DCN314(id)\ + DSC_REG_LIST_DCN20(id)\ } static const struct dcn20_dsc_registers dsc_regs[] = { @@ -590,11 +593,11 @@ static const struct dcn20_dsc_registers dsc_regs[] = { }; static const struct dcn20_dsc_shift dsc_shift = { - DSC_REG_LIST_SH_MASK_DCN314(__SHIFT) + DSC_REG_LIST_SH_MASK_DCN20(__SHIFT) }; static const struct dcn20_dsc_mask dsc_mask = { - DSC_REG_LIST_SH_MASK_DCN314(_MASK) + DSC_REG_LIST_SH_MASK_DCN20(_MASK) }; static const struct dcn30_mpc_registers mpc_regs = { @@ -844,7 +847,7 @@ static const struct resource_caps res_cap_dcn314 = { .num_ddc = 5, .num_vmid = 16, .num_mpc_3dlut = 2, - .num_dsc = 4, + .num_dsc = 3, }; static const struct dc_plane_cap plane_cap = { @@ -878,7 +881,8 @@ static const struct dc_plane_cap plane_cap = { }; static const struct dc_debug_options debug_defaults_drv = { - .disable_z10 = true, /*hw not support it*/ + .disable_z10 = false, + .enable_z9_disable_interface = true, .disable_dmcu = true, .force_abm_enable = false, .timing_trace = false, @@ -911,7 +915,6 @@ static const struct dc_debug_options debug_defaults_drv = { .afmt = true, } }, - .optimize_edp_link_rate = true, .seamless_boot_odm_combine = true }; @@ -933,6 +936,12 @@ static const struct dc_debug_options debug_defaults_diags = { .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .ilr = { + .optimize_edp_link_rate = true, + }, +}; + static void dcn31_dpp_destroy(struct dpp **dpp) { kfree(TO_DCN20_DPP(*dpp)); @@ -1672,6 +1681,11 @@ static void dcn314_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *b DC_FP_END(); } +static void dcn314_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static struct resource_funcs dcn314_res_pool_funcs = { .destroy = dcn314_destroy_resource_pool, .link_enc_create = dcn31_link_encoder_create, @@ -1694,6 +1708,7 @@ static struct resource_funcs dcn314_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn314_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn314_get_panel_config_defaults, }; static struct clock_source *dcn30_clock_source_create( diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c index eebb42c9ddd605de10481ed00199587d6ba882be..58746c437554f5f2f10edaf399b8a587ba8d309a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c @@ -885,7 +885,6 @@ static const struct dc_debug_options debug_defaults_drv = { .afmt = true, } }, - .optimize_edp_link_rate = true, .psr_power_use_phy_fsm = 0, }; @@ -907,6 +906,12 @@ static const struct dc_debug_options debug_defaults_diags = { .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .ilr = { + .optimize_edp_link_rate = true, + }, +}; + static void dcn31_dpp_destroy(struct dpp **dpp) { kfree(TO_DCN20_DPP(*dpp)); @@ -1708,6 +1713,11 @@ static int dcn315_populate_dml_pipes_from_context( return pipe_cnt; } +static void dcn315_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static struct dc_cap_funcs cap_funcs = { .get_dcc_compression_cap = dcn20_get_dcc_compression_cap }; @@ -1721,7 +1731,7 @@ static struct resource_funcs dcn315_res_pool_funcs = { .panel_cntl_create = dcn31_panel_cntl_create, .validate_bandwidth = dcn31_validate_bandwidth, .calculate_wm_and_dlg = dcn31_calculate_wm_and_dlg, - .update_soc_for_wm_a = dcn31_update_soc_for_wm_a, + .update_soc_for_wm_a = dcn315_update_soc_for_wm_a, .populate_dml_pipes = dcn315_populate_dml_pipes_from_context, .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer, .add_stream_to_ctx = dcn30_add_stream_to_ctx, @@ -1734,6 +1744,7 @@ static struct resource_funcs dcn315_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn315_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn315_get_panel_config_defaults, }; static bool dcn315_resource_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c index f4b52a35ad84fbbf43f31af534ac91933566d25f..6b40a11ac83a92a8085eb64c05ded39f11bab205 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c @@ -885,7 +885,6 @@ static const struct dc_debug_options debug_defaults_drv = { .afmt = true, } }, - .optimize_edp_link_rate = true, }; static const struct dc_debug_options debug_defaults_diags = { @@ -906,6 +905,12 @@ static const struct dc_debug_options debug_defaults_diags = { .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .ilr = { + .optimize_edp_link_rate = true, + }, +}; + static void dcn31_dpp_destroy(struct dpp **dpp) { kfree(TO_DCN20_DPP(*dpp)); @@ -1710,6 +1715,11 @@ static int dcn316_populate_dml_pipes_from_context( return pipe_cnt; } +static void dcn316_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static struct dc_cap_funcs cap_funcs = { .get_dcc_compression_cap = dcn20_get_dcc_compression_cap }; @@ -1736,6 +1746,7 @@ static struct resource_funcs dcn316_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn316_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn316_get_panel_config_defaults, }; static bool dcn316_resource_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c index 0d5e8a441512105428100a34f7a4cc2deac3840f..e4daed44ef5f9038c81f31c5da5d0263e2acb060 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c @@ -42,6 +42,48 @@ #define DC_LOGGER \ dccg->ctx->logger +static void dccg32_get_pixel_rate_div( + struct dccg *dccg, + uint32_t otg_inst, + enum pixel_rate_div *k1, + enum pixel_rate_div *k2) +{ + struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); + uint32_t val_k1 = PIXEL_RATE_DIV_NA, val_k2 = PIXEL_RATE_DIV_NA; + + *k1 = PIXEL_RATE_DIV_NA; + *k2 = PIXEL_RATE_DIV_NA; + + switch (otg_inst) { + case 0: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG0_PIXEL_RATE_DIVK1, &val_k1, + OTG0_PIXEL_RATE_DIVK2, &val_k2); + break; + case 1: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG1_PIXEL_RATE_DIVK1, &val_k1, + OTG1_PIXEL_RATE_DIVK2, &val_k2); + break; + case 2: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG2_PIXEL_RATE_DIVK1, &val_k1, + OTG2_PIXEL_RATE_DIVK2, &val_k2); + break; + case 3: + REG_GET_2(OTG_PIXEL_RATE_DIV, + OTG3_PIXEL_RATE_DIVK1, &val_k1, + OTG3_PIXEL_RATE_DIVK2, &val_k2); + break; + default: + BREAK_TO_DEBUGGER(); + return; + } + + *k1 = (enum pixel_rate_div)val_k1; + *k2 = (enum pixel_rate_div)val_k2; +} + static void dccg32_set_pixel_rate_div( struct dccg *dccg, uint32_t otg_inst, @@ -50,6 +92,17 @@ static void dccg32_set_pixel_rate_div( { struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); + enum pixel_rate_div cur_k1 = PIXEL_RATE_DIV_NA, cur_k2 = PIXEL_RATE_DIV_NA; + + // Don't program 0xF into the register field. Not valid since + // K1 / K2 field is only 1 / 2 bits wide + if (k1 == PIXEL_RATE_DIV_NA || k2 == PIXEL_RATE_DIV_NA) + return; + + dccg32_get_pixel_rate_div(dccg, otg_inst, &cur_k1, &cur_k2); + if (k1 == cur_k1 && k2 == cur_k2) + return; + switch (otg_inst) { case 0: REG_UPDATE_2(OTG_PIXEL_RATE_DIV, @@ -133,7 +186,7 @@ static void dccg32_set_dtbclk_p_src( } /* Controls the generation of pixel valid for OTG in (OTG -> HPO case) */ -void dccg32_set_dtbclk_dto( +static void dccg32_set_dtbclk_dto( struct dccg *dccg, const struct dtbclk_dto_params *params) { @@ -208,7 +261,7 @@ static void dccg32_get_dccg_ref_freq(struct dccg *dccg, return; } -void dccg32_set_dpstreamclk( +static void dccg32_set_dpstreamclk( struct dccg *dccg, enum streamclk_source src, int otg_inst, @@ -245,7 +298,7 @@ void dccg32_set_dpstreamclk( } } -void dccg32_otg_add_pixel(struct dccg *dccg, +static void dccg32_otg_add_pixel(struct dccg *dccg, uint32_t otg_inst) { struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); @@ -254,7 +307,7 @@ void dccg32_otg_add_pixel(struct dccg *dccg, OTG_ADD_PIXEL[otg_inst], 1); } -void dccg32_otg_drop_pixel(struct dccg *dccg, +static void dccg32_otg_drop_pixel(struct dccg *dccg, uint32_t otg_inst) { struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg); diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c index d6855d4f749b102b1cb2c5ef29bef70eedc2b54a..076969d928afaa6720060ddde44e71b26e542150 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c @@ -118,7 +118,7 @@ void dcn32_link_encoder_enable_dp_output( } } -bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc) +static bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc) { struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc); uint32_t dp_alt_mode_disable = 0; @@ -133,7 +133,7 @@ bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc) return is_usb_c_alt_mode; } -void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, +static void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, struct dc_link_settings *link_settings) { struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc); @@ -150,12 +150,6 @@ void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc, } -void enc32_set_dig_output_mode(struct link_encoder *enc, uint8_t pix_per_container) -{ - struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, pix_per_container); -} - static const struct link_encoder_funcs dcn32_link_enc_funcs = { .read_state = link_enc2_read_state, .validate_output_with_stream = @@ -186,7 +180,6 @@ static const struct link_encoder_funcs dcn32_link_enc_funcs = { .is_in_alt_mode = dcn32_link_encoder_is_in_alt_mode, .get_max_link_cap = dcn32_link_encoder_get_max_link_cap, .set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux, - .set_dig_output_mode = enc32_set_dig_output_mode, }; void dcn32_link_encoder_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h index 749a1e8cb81132eb1b83f7c056d80ea87252d451..bbcfce06bec0164bed0112afac99101907d317e3 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.h @@ -53,8 +53,4 @@ void dcn32_link_encoder_enable_dp_output( const struct dc_link_settings *link_settings, enum clock_source_id clock_source); -void enc32_set_dig_output_mode( - struct link_encoder *enc, - uint8_t pix_per_container); - #endif /* __DC_LINK_ENCODER__DCN32_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c index 38a48983f663b6250390e8947828edf1b005b954..d19fc93dbc75dfda2a528748e00dd7d2a38d3e62 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c @@ -60,7 +60,7 @@ static void enc32_dp_set_odm_combine( } /* setup stream encoder in dvi mode */ -void enc32_stream_encoder_dvi_set_stream_attribute( +static void enc32_stream_encoder_dvi_set_stream_attribute( struct stream_encoder *enc, struct dc_crtc_timing *crtc_timing, bool is_dual_link) @@ -243,6 +243,39 @@ static bool is_two_pixels_per_containter(const struct dc_crtc_timing *timing) return two_pix; } +static bool is_h_timing_divisible_by_2(const struct dc_crtc_timing *timing) +{ + /* math borrowed from function of same name in inc/resource + * checks if h_timing is divisible by 2 + */ + + bool divisible = false; + uint16_t h_blank_start = 0; + uint16_t h_blank_end = 0; + + if (timing) { + h_blank_start = timing->h_total - timing->h_front_porch; + h_blank_end = h_blank_start - timing->h_addressable; + + /* HTOTAL, Hblank start/end, and Hsync start/end all must be + * divisible by 2 in order for the horizontal timing params + * to be considered divisible by 2. Hsync start is always 0. + */ + divisible = (timing->h_total % 2 == 0) && + (h_blank_start % 2 == 0) && + (h_blank_end % 2 == 0) && + (timing->h_sync_width % 2 == 0); + } + return divisible; +} + +static bool is_dp_dig_pixel_rate_div_policy(struct dc *dc, const struct dc_crtc_timing *timing) +{ + /* should be functionally the same as dcn32_is_dp_dig_pixel_rate_div_policy for DP encoders*/ + return is_h_timing_divisible_by_2(timing) && + dc->debug.enable_dp_dig_pixel_rate_div_policy; +} + static void enc32_stream_encoder_dp_unblank( struct dc_link *link, struct stream_encoder *enc, @@ -259,7 +292,7 @@ static void enc32_stream_encoder_dp_unblank( /* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */ if (is_two_pixels_per_containter(¶m->timing) || param->opp_cnt > 1 - || dc->debug.enable_dp_dig_pixel_rate_div_policy) { + || is_dp_dig_pixel_rate_div_policy(dc, ¶m->timing)) { /*this logic should be the same in get_pixel_clock_parameters() */ n_multiply = 1; } @@ -355,7 +388,7 @@ static void enc32_dp_set_dsc_config(struct stream_encoder *enc, { struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); - REG_UPDATE(DP_DSC_CNTL, DP_DSC_MODE, dsc_mode); + REG_UPDATE(DP_DSC_CNTL, DP_DSC_MODE, dsc_mode == OPTC_DSC_DISABLED ? 0 : 1); } /* this function read dsc related register fields to be logged later in dcn10_log_hw_state @@ -378,24 +411,6 @@ static void enc32_read_state(struct stream_encoder *enc, struct enc_state *s) } } -static void enc32_stream_encoder_reset_fifo(struct stream_encoder *enc) -{ - struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); - uint32_t fifo_enabled; - - REG_GET(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, &fifo_enabled); - - if (fifo_enabled == 0) { - /* reset DIG resync FIFO */ - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1); - /* TODO: fix timeout when wait for DIG_FIFO_RESET_DONE */ - //REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 1, 100); - udelay(1); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0); - REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 1, 100); - } -} - static void enc32_set_dig_input_mode(struct stream_encoder *enc, unsigned int pix_per_container) { struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); @@ -425,8 +440,6 @@ static const struct stream_encoder_funcs dcn32_str_enc_funcs = { enc3_stream_encoder_update_dp_info_packets, .stop_dp_info_packets = enc1_stream_encoder_stop_dp_info_packets, - .reset_fifo = - enc32_stream_encoder_reset_fifo, .dp_blank = enc1_stream_encoder_dp_blank, .dp_unblank = diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h index 250d9a341cf661e3ab91e7c89d83baaaaed27a79..ecd041a446d2c9489dbbc2bab55a3249094f6357 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.h @@ -71,7 +71,9 @@ SRI(DP_MSE_RATE_UPDATE, DP, id), \ SRI(DP_PIXEL_FORMAT, DP, id), \ SRI(DP_SEC_CNTL, DP, id), \ + SRI(DP_SEC_CNTL1, DP, id), \ SRI(DP_SEC_CNTL2, DP, id), \ + SRI(DP_SEC_CNTL5, DP, id), \ SRI(DP_SEC_CNTL6, DP, id), \ SRI(DP_STEER_FIFO, DP, id), \ SRI(DP_VID_M, DP, id), \ @@ -93,7 +95,7 @@ SRI(DIG_FIFO_CTRL0, DIG, id) -#define SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh)\ +#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\ SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_ENCODING, mask_sh),\ SE_SF(DP0_DP_PIXEL_FORMAT, DP_COMPONENT_DEPTH, mask_sh),\ SE_SF(DP0_DP_PIXEL_FORMAT, DP_PIXEL_PER_CYCLE_PROCESSING_MODE, mask_sh),\ @@ -106,6 +108,7 @@ SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_CONT, mask_sh),\ SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_GC_SEND, mask_sh),\ SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_NULL_SEND, mask_sh),\ + SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, mask_sh),\ SE_SF(DIG0_HDMI_INFOFRAME_CONTROL0, HDMI_AUDIO_INFO_SEND, mask_sh),\ SE_SF(DIG0_HDMI_INFOFRAME_CONTROL1, HDMI_AUDIO_INFO_LINE, mask_sh),\ SE_SF(DIG0_HDMI_GC, HDMI_GC_AVMUTE, mask_sh),\ @@ -244,15 +247,6 @@ SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, mask_sh),\ SE_SF(DIG0_DIG_FIFO_CTRL0, DIG_FIFO_OUTPUT_PIXEL_MODE, mask_sh) -#if defined(CONFIG_DRM_AMD_DC_HDCP) -#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\ - SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh),\ - SE_SF(DIG0_HDMI_VBI_PACKET_CONTROL, HDMI_ACP_SEND, mask_sh) -#else -#define SE_COMMON_MASK_SH_LIST_DCN32(mask_sh)\ - SE_COMMON_MASK_SH_LIST_DCN32_BASE(mask_sh) -#endif - void dcn32_dio_stream_encoder_construct( struct dcn10_stream_encoder *enc1, struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c index f349cbe2a0f0c7f1a1830d9d8ff2fd80903f7e84..dcf12a0b031c78fa658ee1261ae066d037e13636 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c @@ -31,7 +31,7 @@ #include "dcn30/dcn30_cm_common.h" /* Compute the maximum number of lines that we can fit in the line buffer */ -void dscl32_calc_lb_num_partitions( +static void dscl32_calc_lb_num_partitions( const struct scaler_data *scl_data, enum lb_memory_config lb_config, int *num_part_y, diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.h index 9db1323e19337a596fee50abf89a366188d93c7f..176b1537d2a13a40a45ca99867cdf7dfe2705964 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hpo_dp_link_encoder.h @@ -47,6 +47,7 @@ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL1, mask_sh),\ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL2, mask_sh),\ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_CONFIG, TP_PRBS_SEL3, mask_sh),\ + SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_TP_SQ_PULSE, TP_SQ_PULSE_WIDTH, mask_sh),\ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0, SAT_STREAM_SOURCE, mask_sh),\ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_SAT_VC0, SAT_SLOT_COUNT, mask_sh),\ SE_SF(DP_DPHY_SYM320_DP_DPHY_SYM32_VC_RATE_CNTL0, STREAM_VC_RATE_X, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c index 99eb239bbc7bba0fb6f5f14f4280f9868df3a9ba..9fbb72369c10e4afd60054d16ab4d16e095d1337 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c @@ -68,7 +68,7 @@ static void dcn32_init_crb(struct hubbub *hubbub) REG_UPDATE(DCHUBBUB_DEBUG_CTRL_0, DET_DEPTH, 0x47F); } -static void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte) +void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte) { struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub); @@ -98,9 +98,13 @@ static void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigne default: break; } - /* Should never be hit, if it is we have an erroneous hw config*/ - ASSERT(hubbub2->det0_size + hubbub2->det1_size + hubbub2->det2_size - + hubbub2->det3_size + hubbub2->compbuf_size_segments <= hubbub2->crb_size_segs); + if (hubbub2->det0_size + hubbub2->det1_size + hubbub2->det2_size + + hubbub2->det3_size + hubbub2->compbuf_size_segments > hubbub2->crb_size_segs) { + /* This may happen during seamless transition from ODM 2:1 to ODM4:1 */ + DC_LOG_WARNING("CRB Config Warning: DET size (%d,%d,%d,%d) + Compbuf size (%d) > CRB segments (%d)\n", + hubbub2->det0_size, hubbub2->det1_size, hubbub2->det2_size, hubbub2->det3_size, + hubbub2->compbuf_size_segments, hubbub2->crb_size_segs); + } } static void dcn32_program_compbuf_size(struct hubbub *hubbub, unsigned int compbuf_size_kb, bool safe_to_increase) @@ -140,7 +144,7 @@ static uint32_t convert_and_clamp( return ret_val; } -static bool hubbub32_program_urgent_watermarks( +bool hubbub32_program_urgent_watermarks( struct hubbub *hubbub, struct dcn_watermark_set *watermarks, unsigned int refclk_mhz, @@ -330,7 +334,7 @@ static bool hubbub32_program_urgent_watermarks( return wm_pending; } -static bool hubbub32_program_stutter_watermarks( +bool hubbub32_program_stutter_watermarks( struct hubbub *hubbub, struct dcn_watermark_set *watermarks, unsigned int refclk_mhz, @@ -476,7 +480,7 @@ static bool hubbub32_program_stutter_watermarks( } -static bool hubbub32_program_pstate_watermarks( +bool hubbub32_program_pstate_watermarks( struct hubbub *hubbub, struct dcn_watermark_set *watermarks, unsigned int refclk_mhz, @@ -629,7 +633,7 @@ static bool hubbub32_program_pstate_watermarks( } -static bool hubbub32_program_usr_watermarks( +bool hubbub32_program_usr_watermarks( struct hubbub *hubbub, struct dcn_watermark_set *watermarks, unsigned int refclk_mhz, @@ -769,7 +773,7 @@ static bool hubbub32_program_watermarks( } /* Copy values from WM set A to all other sets */ -void hubbub32_init_watermarks(struct hubbub *hubbub) +static void hubbub32_init_watermarks(struct hubbub *hubbub) { struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub); uint32_t reg; @@ -820,7 +824,7 @@ void hubbub32_init_watermarks(struct hubbub *hubbub) REG_WRITE(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, reg); } -void hubbub32_wm_read_state(struct hubbub *hubbub, +static void hubbub32_wm_read_state(struct hubbub *hubbub, struct dcn_hubbub_wm *wm) { struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub); @@ -932,6 +936,7 @@ static const struct hubbub_funcs hubbub32_funcs = { .program_watermarks = hubbub32_program_watermarks, .allow_self_refresh_control = hubbub1_allow_self_refresh_control, .is_allow_self_refresh_enabled = hubbub1_is_allow_self_refresh_enabled, + .verify_allow_pstate_change_high = hubbub1_verify_allow_pstate_change_high, .force_wm_propagate_to_pipes = hubbub32_force_wm_propagate_to_pipes, .force_pstate_change_control = hubbub3_force_pstate_change_control, .init_watermarks = hubbub32_init_watermarks, diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h index 3bae6e558971b6537b21663b7dfe0d016b4e50ef..cda94e0e31bfbba5f006a40eba244b90131ad809 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h @@ -161,6 +161,35 @@ HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_PIPE, mask_sh), \ HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_INTERRUPT_STATUS, mask_sh) +bool hubbub32_program_urgent_watermarks( + struct hubbub *hubbub, + struct dcn_watermark_set *watermarks, + unsigned int refclk_mhz, + bool safe_to_lower); + +bool hubbub32_program_stutter_watermarks( + struct hubbub *hubbub, + struct dcn_watermark_set *watermarks, + unsigned int refclk_mhz, + bool safe_to_lower); + +bool hubbub32_program_pstate_watermarks( + struct hubbub *hubbub, + struct dcn_watermark_set *watermarks, + unsigned int refclk_mhz, + bool safe_to_lower); + +bool hubbub32_program_usr_watermarks( + struct hubbub *hubbub, + struct dcn_watermark_set *watermarks, + unsigned int refclk_mhz, + bool safe_to_lower); + +void hubbub32_force_usr_retraining_allow(struct hubbub *hubbub, bool allow); + +void hubbub32_force_wm_propagate_to_pipes(struct hubbub *hubbub); + +void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte); void hubbub32_construct(struct dcn20_hubbub *hubbub2, struct dc_context *ctx, diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c index 6ec1c52535b9b0a8dda30720c517450e38869fa3..ac1c6458dd55a8f6702b0b011e861e27cd14e8be 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubp.c @@ -79,6 +79,8 @@ void hubp32_phantom_hubp_post_enable(struct hubp *hubp) uint32_t reg_val; struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp); + /* For phantom pipe enable, disable GSL */ + REG_UPDATE(DCSURF_FLIP_CONTROL2, SURFACE_GSL_ENABLE, 0); REG_UPDATE(DCHUBP_CNTL, HUBP_BLANK_EN, 1); reg_val = REG_READ(DCHUBP_CNTL); if (reg_val) { @@ -103,6 +105,11 @@ void hubp32_cursor_set_attributes( enum cursor_lines_per_chunk lpc = hubp2_get_lines_per_chunk( attr->width, attr->color_format); + //Round cursor width up to next multiple of 64 + uint32_t cursor_width = ((attr->width + 63) / 64) * 64; + uint32_t cursor_height = attr->height; + uint32_t cursor_size = cursor_width * cursor_height; + hubp->curs_attr = *attr; REG_UPDATE(CURSOR_SURFACE_ADDRESS_HIGH, @@ -126,7 +133,24 @@ void hubp32_cursor_set_attributes( /* used to shift the cursor chunk request deadline */ CURSOR0_CHUNK_HDL_ADJUST, 3); - if (attr->width * attr->height * 4 > 16384) + switch (attr->color_format) { + case CURSOR_MODE_MONO: + cursor_size /= 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cursor_size *= 4; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + default: + cursor_size *= 8; + break; + } + + if (cursor_size > 16384) REG_UPDATE(DCHUBP_MALL_CONFIG, USE_MALL_FOR_CURSOR, true); else REG_UPDATE(DCHUBP_MALL_CONFIG, USE_MALL_FOR_CURSOR, false); @@ -157,12 +181,12 @@ static struct hubp_funcs dcn32_hubp_funcs = { .hubp_init = hubp3_init, .set_unbounded_requesting = hubp31_set_unbounded_requesting, .hubp_soft_reset = hubp31_soft_reset, + .hubp_set_flip_int = hubp1_set_flip_int, .hubp_in_blank = hubp1_in_blank, .hubp_update_force_pstate_disallow = hubp32_update_force_pstate_disallow, .phantom_hubp_post_enable = hubp32_phantom_hubp_post_enable, .hubp_update_mall_sel = hubp32_update_mall_sel, - .hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering, - .hubp_set_flip_int = hubp1_set_flip_int + .hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering }; bool hubp32_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c index 769171ab8ef6419afd66e0ba5704cf65d0bfc061..cf5bd9713f54f4c4890a960c7a88cc8d49fef939 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c @@ -49,6 +49,7 @@ #include "dcn20/dcn20_optc.h" #include "dmub_subvp_state.h" #include "dce/dmub_hw_lock_mgr.h" +#include "dcn32_resource.h" #include "dc_link_dp.h" #include "dmub/inc/dmub_subvp_state.h" @@ -198,42 +199,6 @@ static bool dcn32_check_no_memory_request_for_cab(struct dc *dc) return false; } -/* This function takes in the start address and surface size to be cached in CAB - * and calculates the total number of cache lines required to store the surface. - * The number of cache lines used for each surface is calculated independently of - * one another. For example, if there is a primary surface(1), meta surface(2), and - * cursor(3), this function should be called 3 times to calculate the number of cache - * lines used for each of those surfaces. - */ -static uint32_t dcn32_cache_lines_for_surface(struct dc *dc, uint32_t surface_size, uint64_t start_address) -{ - uint32_t lines_used = 1; - uint32_t num_cached_bytes = 0; - uint32_t remaining_size = 0; - uint32_t cache_line_size = dc->caps.cache_line_size; - uint32_t remainder = 0; - - /* 1. Calculate surface size minus the number of bytes stored - * in the first cache line (all bytes in first cache line might - * not be fully used). - */ - div_u64_rem(start_address, cache_line_size, &remainder); - num_cached_bytes = cache_line_size - remainder; - remaining_size = surface_size - num_cached_bytes; - - /* 2. Calculate number of cache lines that will be fully used with - * the remaining number of bytes to be stored. - */ - lines_used += (remaining_size / cache_line_size); - - /* 3. Check if we need an extra line due to the remaining size not being - * a multiple of CACHE_LINE_SIZE. - */ - if (remaining_size % cache_line_size > 0) - lines_used++; - - return lines_used; -} /* This function loops through every surface that needs to be cached in CAB for SS, * and calculates the total number of ways required to store all surfaces (primary, @@ -241,94 +206,115 @@ static uint32_t dcn32_cache_lines_for_surface(struct dc *dc, uint32_t surface_si */ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *ctx) { - uint8_t i, j; + int i, j; struct dc_stream_state *stream = NULL; struct dc_plane_state *plane = NULL; - uint32_t surface_size = 0; uint32_t cursor_size = 0; - uint32_t cache_lines_used = 0; uint32_t total_lines = 0; uint32_t lines_per_way = 0; - uint32_t num_ways = 0; - uint32_t prev_addr_low = 0; + uint8_t num_ways = 0; + uint8_t bytes_per_pixel = 0; + uint8_t cursor_bpp = 0; + uint16_t mblk_width = 0; + uint16_t mblk_height = 0; + uint16_t mall_alloc_width_blk_aligned = 0; + uint16_t mall_alloc_height_blk_aligned = 0; + uint16_t num_mblks = 0; + uint32_t bytes_in_mall = 0; + uint32_t cache_lines_used = 0; + uint32_t cache_lines_per_plane = 0; - for (i = 0; i < ctx->stream_count; i++) { - stream = ctx->streams[i]; + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - // Don't include PSR surface in the total surface size for CAB allocation - if (stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED) + if (!pipe->stream || !pipe->plane_state || + pipe->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED || + pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) continue; - if (ctx->stream_status[i].plane_count == 0) - continue; + bytes_per_pixel = pipe->plane_state->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4; + mblk_width = DCN3_2_MBLK_WIDTH; + mblk_height = bytes_per_pixel == 4 ? DCN3_2_MBLK_HEIGHT_4BPE : DCN3_2_MBLK_HEIGHT_8BPE; - // For each stream, loop through each plane to calculate the number of cache - // lines required to store the surface in CAB - for (j = 0; j < ctx->stream_status[i].plane_count; j++) { - plane = ctx->stream_status[i].plane_states[j]; + /* full_vp_width_blk_aligned = FLOOR(vp_x_start + full_vp_width + blk_width - 1, blk_width) - + * FLOOR(vp_x_start, blk_width) + * + * mall_alloc_width_blk_aligned_l/c = full_vp_width_blk_aligned_l/c + */ + mall_alloc_width_blk_aligned = ((pipe->plane_res.scl_data.viewport.x + + pipe->plane_res.scl_data.viewport.width + mblk_width - 1) / mblk_width * mblk_width) - + (pipe->plane_res.scl_data.viewport.x / mblk_width * mblk_width); + + /* full_vp_height_blk_aligned = FLOOR(vp_y_start + full_vp_height + blk_height - 1, blk_height) - + * FLOOR(vp_y_start, blk_height) + * + * mall_alloc_height_blk_aligned_l/c = full_vp_height_blk_aligned_l/c + */ + mall_alloc_height_blk_aligned = ((pipe->plane_res.scl_data.viewport.y + + pipe->plane_res.scl_data.viewport.height + mblk_height - 1) / mblk_height * mblk_height) - + (pipe->plane_res.scl_data.viewport.y / mblk_height * mblk_height); - // Calculate total surface size - if (prev_addr_low != plane->address.grph.addr.u.low_part) { - /* if plane address are different from prev FB, then userspace allocated separate FBs*/ - surface_size += plane->plane_size.surface_pitch * - plane->plane_size.surface_size.height * - (plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4); + num_mblks = ((mall_alloc_width_blk_aligned + mblk_width - 1) / mblk_width) * + ((mall_alloc_height_blk_aligned + mblk_height - 1) / mblk_height); - prev_addr_low = plane->address.grph.addr.u.low_part; - } else { - /* We have the same fb for all the planes. - * Xorg always creates one giant fb that holds all surfaces, - * so allocating it once is sufficient. - * */ - continue; - } - // Convert surface size + starting address to number of cache lines required - // (alignment accounted for) - cache_lines_used += dcn32_cache_lines_for_surface(dc, surface_size, - plane->address.grph.addr.quad_part); - - if (plane->address.grph.meta_addr.quad_part) { - // Meta surface - cache_lines_used += dcn32_cache_lines_for_surface(dc, surface_size, - plane->address.grph.meta_addr.quad_part); - } - } + /* For DCC: + * meta_num_mblk = CEILING(full_mblk_width_ub_l*full_mblk_height_ub_l*Bpe/256/mblk_bytes, 1) + */ + if (pipe->plane_state->dcc.enable) + num_mblks += (mall_alloc_width_blk_aligned * mall_alloc_width_blk_aligned * bytes_per_pixel + + (256 * DCN3_2_MALL_MBLK_SIZE_BYTES) - 1) / (256 * DCN3_2_MALL_MBLK_SIZE_BYTES); - // Include cursor size for CAB allocation - for (j = 0; j < dc->res_pool->pipe_count; j++) { - struct pipe_ctx *pipe = &ctx->res_ctx.pipe_ctx[j]; - struct hubp *hubp = pipe->plane_res.hubp; + bytes_in_mall = num_mblks * DCN3_2_MALL_MBLK_SIZE_BYTES; - if (pipe->stream && pipe->plane_state && hubp) - /* Find the cursor plane and use the exact size instead of - * using the max for calculation - */ - if (hubp->curs_attr.width > 0) { - cursor_size = hubp->curs_attr.width * hubp->curs_attr.height; - break; - } - } + /* (cache lines used is total bytes / cache_line size. Add +2 for worst case alignment + * (MALL is 64-byte aligned) + */ + cache_lines_per_plane = bytes_in_mall / dc->caps.cache_line_size + 2; + cache_lines_used += cache_lines_per_plane; + } - switch (stream->cursor_attributes.color_format) { - case CURSOR_MODE_MONO: - cursor_size /= 2; - break; - case CURSOR_MODE_COLOR_1BIT_AND: - case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: - case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: - cursor_size *= 4; - break; + // Include cursor size for CAB allocation + for (j = 0; j < dc->res_pool->pipe_count; j++) { + struct pipe_ctx *pipe = &ctx->res_ctx.pipe_ctx[j]; + struct hubp *hubp = pipe->plane_res.hubp; - case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: - case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: - cursor_size *= 8; - break; - } + if (pipe->stream && pipe->plane_state && hubp) + /* Find the cursor plane and use the exact size instead of + using the max for calculation */ - if (stream->cursor_position.enable && plane->address.grph.cursor_cache_addr.quad_part) { - cache_lines_used += dcn32_cache_lines_for_surface(dc, cursor_size, - plane->address.grph.cursor_cache_addr.quad_part); - } + if (hubp->curs_attr.width > 0) { + // Round cursor width to next multiple of 64 + cursor_size = (((hubp->curs_attr.width + 63) / 64) * 64) * hubp->curs_attr.height; + + switch (pipe->stream->cursor_attributes.color_format) { + case CURSOR_MODE_MONO: + cursor_size /= 2; + cursor_bpp = 4; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cursor_size *= 4; + cursor_bpp = 4; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + cursor_size *= 8; + cursor_bpp = 8; + break; + } + + if (pipe->stream->cursor_position.enable && !dc->debug.alloc_extra_way_for_cursor && + cursor_size > 16384) { + /* cursor_num_mblk = CEILING(num_cursors*cursor_width*cursor_width*cursor_Bpe/mblk_bytes, 1) + */ + cache_lines_used += (((hubp->curs_attr.width * hubp->curs_attr.height * cursor_bpp + + DCN3_2_MALL_MBLK_SIZE_BYTES - 1) / DCN3_2_MALL_MBLK_SIZE_BYTES) * + DCN3_2_MALL_MBLK_SIZE_BYTES) / dc->caps.cache_line_size + 2; + } + break; + } } // Convert number of cache lines required to number of ways @@ -345,8 +331,8 @@ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *c plane = ctx->stream_status[i].plane_states[j]; if (stream->cursor_position.enable && plane && - !plane->address.grph.cursor_cache_addr.quad_part && - cursor_size > 16384) { + dc->debug.alloc_extra_way_for_cursor && + cursor_size > 16384) { /* Cursor caching is not supported since it won't be on the same line. * So we need an extra line to accommodate it. With large cursors and a single 4k monitor * this case triggers corruption. If we're at the edge, then dont trigger display refresh @@ -358,7 +344,9 @@ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *c } } } - + if (dc->debug.force_mall_ss_num_ways > 0) { + num_ways = dc->debug.force_mall_ss_num_ways; + } return num_ways; } @@ -367,7 +355,7 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable) union dmub_rb_cmd cmd; uint8_t ways, i; int j; - bool stereo_in_use = false; + bool mall_ss_unsupported = false; struct dc_plane_state *plane = NULL; if (!dc->ctx->dmub_srv) @@ -398,22 +386,23 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable) */ ways = dcn32_calculate_cab_allocation(dc, dc->current_state); - /* MALL not supported with Stereo3D. If any plane is using stereo, - * don't try to enter MALL. + /* MALL not supported with Stereo3D or TMZ surface. If any plane is using stereo, + * or TMZ surface, don't try to enter MALL. */ for (i = 0; i < dc->current_state->stream_count; i++) { for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) { plane = dc->current_state->stream_status[i].plane_states[j]; - if (plane->address.type == PLN_ADDR_TYPE_GRPH_STEREO) { - stereo_in_use = true; + if (plane->address.type == PLN_ADDR_TYPE_GRPH_STEREO || + plane->address.tmz_surface) { + mall_ss_unsupported = true; break; } } - if (stereo_in_use) + if (mall_ss_unsupported) break; } - if (ways <= dc->caps.cache_num_ways && !stereo_in_use) { + if (ways <= dc->caps.cache_num_ways && !mall_ss_unsupported) { memset(&cmd, 0, sizeof(cmd)); cmd.cab.header.type = DMUB_CMD__CAB_FOR_SS; cmd.cab.header.sub_type = DMUB_CMD__CAB_DCN_SS_FIT_IN_CAB; @@ -451,7 +440,6 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable) */ void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context) { -/* int i; bool enable_subvp = false; @@ -469,7 +457,6 @@ void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context) } } dc_dmub_setup_subvp_dmub_command(dc, context, enable_subvp); -*/ } /* Sub-Viewport DMUB lock needs to be acquired by driver whenever SubVP is active and: @@ -642,10 +629,9 @@ bool dcn32_set_input_transfer_func(struct dc *dc, params = &dpp_base->degamma_params; } - result = dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); + dpp_base->funcs->dpp_program_gamcor_lut(dpp_base, params); - if (result && - pipe_ctx->stream_res.opp && + if (pipe_ctx->stream_res.opp && pipe_ctx->stream_res.opp->ctx && hws->funcs.set_mcm_luts) result = hws->funcs.set_mcm_luts(pipe_ctx, plane_state); @@ -741,7 +727,29 @@ void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context) struct hubp *hubp = pipe->plane_res.hubp; if (pipe->stream && pipe->plane_state && hubp && hubp->funcs->hubp_update_mall_sel) { - if (hubp->curs_attr.width * hubp->curs_attr.height * 4 > 16384) + //Round cursor width up to next multiple of 64 + int cursor_width = ((hubp->curs_attr.width + 63) / 64) * 64; + int cursor_height = hubp->curs_attr.height; + int cursor_size = cursor_width * cursor_height; + + switch (hubp->curs_attr.color_format) { + case CURSOR_MODE_MONO: + cursor_size /= 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cursor_size *= 4; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + default: + cursor_size *= 8; + break; + } + + if (cursor_size > 16384) cache_cursor = true; if (pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) { @@ -751,7 +759,8 @@ void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context) hubp->funcs->hubp_update_mall_sel(hubp, num_ways <= dc->caps.cache_num_ways && pipe->stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED && - pipe->plane_state->address.type != PLN_ADDR_TYPE_GRPH_STEREO ? 2 : 0, + pipe->plane_state->address.type != PLN_ADDR_TYPE_GRPH_STEREO && + !pipe->plane_state->address.tmz_surface ? 2 : 0, cache_cursor); } } @@ -861,6 +870,7 @@ void dcn32_init_hw(struct dc *dc) if (link->link_enc->funcs->is_dig_enabled && link->link_enc->funcs->is_dig_enabled(link->link_enc)) { link->link_status.link_active = true; + link->phy_state.symclk_state = SYMCLK_ON_TX_ON; if (link->link_enc->funcs->fec_is_active && link->link_enc->funcs->fec_is_active(link->link_enc)) link->fec_state = dc_link_fec_enabled; @@ -979,6 +989,10 @@ void dcn32_init_hw(struct dc *dc) dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv->dmub); dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr; } + + /* Enable support for ODM and windowed MPO if policy flag is set */ + if (dc->debug.enable_single_display_2to1_odm_policy) + dc->config.enable_windowed_mpo_odm = true; } static int calc_mpc_flow_ctrl_cnt(const struct dc_stream_state *stream, @@ -1133,23 +1147,25 @@ void dcn32_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx * true); } - // Don't program pixel clock after link is already enabled -/* if (false == pipe_ctx->clock_source->funcs->program_pix_clk( - pipe_ctx->clock_source, - &pipe_ctx->stream_res.pix_clk_params, - &pipe_ctx->pll_settings)) { - BREAK_TO_DEBUGGER(); - }*/ + if (pipe_ctx->stream_res.dsc) { + struct pipe_ctx *current_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx]; - if (pipe_ctx->stream_res.dsc) update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC); + + /* Check if no longer using pipe for ODM, then need to disconnect DSC for that pipe */ + if (!pipe_ctx->next_odm_pipe && current_pipe_ctx->next_odm_pipe && + current_pipe_ctx->next_odm_pipe->stream_res.dsc) { + struct display_stream_compressor *dsc = current_pipe_ctx->next_odm_pipe->stream_res.dsc; + /* disconnect DSC block from stream */ + dsc->funcs->dsc_disconnect(dsc); + } + } } unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsigned int *k1_div, unsigned int *k2_div) { struct dc_stream_state *stream = pipe_ctx->stream; unsigned int odm_combine_factor = 0; - struct dc *dc = pipe_ctx->stream->ctx->dc; bool two_pix_per_container = false; // For phantom pipes, use the same programming as the main pipes @@ -1159,6 +1175,9 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing); odm_combine_factor = get_odm_config(pipe_ctx, NULL); + if (pipe_ctx->stream->signal == SIGNAL_TYPE_VIRTUAL) + return odm_combine_factor; + if (is_dp_128b_132b_signal(pipe_ctx)) { *k2_div = PIXEL_RATE_DIV_BY_1; } else if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal) || dc_is_dvi_signal(pipe_ctx->stream->signal)) { @@ -1174,7 +1193,7 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign } else { *k1_div = PIXEL_RATE_DIV_BY_1; *k2_div = PIXEL_RATE_DIV_BY_4; - if ((odm_combine_factor == 2) || dc->debug.enable_dp_dig_pixel_rate_div_policy) + if ((odm_combine_factor == 2) || dcn32_is_dp_dig_pixel_rate_div_policy(pipe_ctx)) *k2_div = PIXEL_RATE_DIV_BY_2; } } @@ -1211,7 +1230,6 @@ void dcn32_unblank_stream(struct pipe_ctx *pipe_ctx, struct dc_link *link = stream->link; struct dce_hwseq *hws = link->dc->hwseq; struct pipe_ctx *odm_pipe; - struct dc *dc = pipe_ctx->stream->ctx->dc; uint32_t pix_per_cycle = 1; params.opp_cnt = 1; @@ -1230,7 +1248,7 @@ void dcn32_unblank_stream(struct pipe_ctx *pipe_ctx, pipe_ctx->stream_res.tg->inst); } else if (dc_is_dp_signal(pipe_ctx->stream->signal)) { if (optc2_is_two_pixels_per_containter(&stream->timing) || params.opp_cnt > 1 - || dc->debug.enable_dp_dig_pixel_rate_div_policy) { + || dcn32_is_dp_dig_pixel_rate_div_policy(pipe_ctx)) { params.timing.pix_clk_100hz /= 2; pix_per_cycle = 2; } @@ -1247,35 +1265,163 @@ bool dcn32_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx) { struct dc *dc = pipe_ctx->stream->ctx->dc; + if (!is_h_timing_divisible_by_2(pipe_ctx->stream)) + return false; + if (dc_is_dp_signal(pipe_ctx->stream->signal) && !is_dp_128b_132b_signal(pipe_ctx) && dc->debug.enable_dp_dig_pixel_rate_div_policy) return true; return false; } -void dcn32_update_phy_state(struct dc_state *state, struct pipe_ctx *pipe_ctx, - enum phy_state target_state) +static void apply_symclk_on_tx_off_wa(struct dc_link *link) { - enum phy_state current_state = pipe_ctx->stream->link->phy_state; - - if (target_state == TX_OFF_SYMCLK_OFF) { - core_link_disable_stream(pipe_ctx); - pipe_ctx->stream->link->phy_state = TX_OFF_SYMCLK_OFF; - } else if (target_state == TX_ON_SYMCLK_ON) { - core_link_enable_stream(state, pipe_ctx); - pipe_ctx->stream->link->phy_state = TX_ON_SYMCLK_ON; - } else if (target_state == TX_OFF_SYMCLK_ON) { - if (current_state == TX_ON_SYMCLK_ON) { - core_link_disable_stream(pipe_ctx); - pipe_ctx->stream->link->phy_state = TX_OFF_SYMCLK_OFF; + /* There are use cases where SYMCLK is referenced by OTG. For instance + * for TMDS signal, OTG relies SYMCLK even if TX video output is off. + * However current link interface will power off PHY when disabling link + * output. This will turn off SYMCLK generated by PHY. The workaround is + * to identify such case where SYMCLK is still in use by OTG when we + * power off PHY. When this is detected, we will temporarily power PHY + * back on and move PHY's SYMCLK state to SYMCLK_ON_TX_OFF by calling + * program_pix_clk interface. When OTG is disabled, we will then power + * off PHY by calling disable link output again. + * + * In future dcn generations, we plan to rework transmitter control + * interface so that we could have an option to set SYMCLK ON TX OFF + * state in one step without this workaround + */ + + struct dc *dc = link->ctx->dc; + struct pipe_ctx *pipe_ctx = NULL; + uint8_t i; + + if (link->phy_state.symclk_ref_cnts.otg > 0) { + for (i = 0; i < MAX_PIPES; i++) { + pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe_ctx->stream && pipe_ctx->stream->link == link && pipe_ctx->top_pipe == NULL) { + pipe_ctx->clock_source->funcs->program_pix_clk( + pipe_ctx->clock_source, + &pipe_ctx->stream_res.pix_clk_params, + dp_get_link_encoding_format(&pipe_ctx->link_config.dp_link_settings), + &pipe_ctx->pll_settings); + link->phy_state.symclk_state = SYMCLK_ON_TX_OFF; + break; + } } + } +} - pipe_ctx->clock_source->funcs->program_pix_clk( - pipe_ctx->clock_source, - &pipe_ctx->stream_res.pix_clk_params, - dp_get_link_encoding_format(&pipe_ctx->link_config.dp_link_settings), - &pipe_ctx->pll_settings); - pipe_ctx->stream->link->phy_state = TX_OFF_SYMCLK_ON; - } else +void dcn32_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal) +{ + struct dc *dc = link->ctx->dc; + const struct link_hwss *link_hwss = get_link_hwss(link, link_res); + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (signal == SIGNAL_TYPE_EDP && + link->dc->hwss.edp_backlight_control) + link->dc->hwss.edp_backlight_control(link, false); + else if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->lock_phy(dmcu); + + link_hwss->disable_link_output(link, link_res, signal); + link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF; + + if (signal == SIGNAL_TYPE_EDP && + link->dc->hwss.edp_backlight_control) + link->dc->hwss.edp_power_control(link, false); + else if (dmcu != NULL && dmcu->funcs->lock_phy) + dmcu->funcs->unlock_phy(dmcu); + + dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY); + + apply_symclk_on_tx_off_wa(link); +} + +/* For SubVP the main pipe can have a viewport position change + * without a full update. In this case we must also update the + * viewport positions for the phantom pipe accordingly. + */ +void dcn32_update_phantom_vp_position(struct dc *dc, + struct dc_state *context, + struct pipe_ctx *phantom_pipe) +{ + uint32_t i; + struct dc_plane_state *phantom_plane = phantom_pipe->plane_state; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_MAIN && + pipe->stream->mall_stream_config.paired_stream == phantom_pipe->stream) { + if (pipe->plane_state && pipe->plane_state->update_flags.bits.position_change) { + + phantom_plane->src_rect.x = pipe->plane_state->src_rect.x; + phantom_plane->src_rect.y = pipe->plane_state->src_rect.y; + phantom_plane->clip_rect.x = pipe->plane_state->clip_rect.x; + phantom_plane->dst_rect.x = pipe->plane_state->dst_rect.x; + phantom_plane->dst_rect.y = pipe->plane_state->dst_rect.y; + + phantom_pipe->plane_state->update_flags.bits.position_change = 1; + resource_build_scaling_params(phantom_pipe); + return; + } + } + } +} + +bool dcn32_dsc_pg_status( + struct dce_hwseq *hws, + unsigned int dsc_inst) +{ + uint32_t pwr_status = 0; + + switch (dsc_inst) { + case 0: /* DSC0 */ + REG_GET(DOMAIN16_PG_STATUS, + DOMAIN_PGFSM_PWR_STATUS, &pwr_status); + break; + case 1: /* DSC1 */ + + REG_GET(DOMAIN17_PG_STATUS, + DOMAIN_PGFSM_PWR_STATUS, &pwr_status); + break; + case 2: /* DSC2 */ + REG_GET(DOMAIN18_PG_STATUS, + DOMAIN_PGFSM_PWR_STATUS, &pwr_status); + break; + case 3: /* DSC3 */ + REG_GET(DOMAIN19_PG_STATUS, + DOMAIN_PGFSM_PWR_STATUS, &pwr_status); + break; + default: BREAK_TO_DEBUGGER(); + break; + } + + return pwr_status == 0; +} + +void dcn32_update_dsc_pg(struct dc *dc, + struct dc_state *context, + bool safe_to_disable) +{ + struct dce_hwseq *hws = dc->hwseq; + int i; + + for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) { + struct display_stream_compressor *dsc = dc->res_pool->dscs[i]; + bool is_dsc_ungated = hws->funcs.dsc_pg_status(hws, dsc->inst); + + if (context->res_ctx.is_dsc_acquired[i]) { + if (!is_dsc_ungated) { + hws->funcs.dsc_pg_control(hws, dsc->inst, true); + } + } else if (safe_to_disable) { + if (is_dsc_ungated) { + hws->funcs.dsc_pg_control(hws, dsc->inst, false); + } + } + } } diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h index 221e31144d501b0487da5f5b6f81c3f7161b09fb..ac3657a5b9eabb69c15ef69cc67b0cab763892db 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h @@ -84,7 +84,20 @@ void dcn32_unblank_stream(struct pipe_ctx *pipe_ctx, bool dcn32_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx); -void dcn32_update_phy_state(struct dc_state *state, struct pipe_ctx *pipe_ctx, - enum phy_state target_state); +void dcn32_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal); + +void dcn32_update_phantom_vp_position(struct dc *dc, + struct dc_state *context, + struct pipe_ctx *phantom_pipe); + +bool dcn32_dsc_pg_status( + struct dce_hwseq *hws, + unsigned int dsc_inst); + +void dcn32_update_dsc_pg(struct dc *dc, + struct dc_state *context, + bool safe_to_disable); #endif /* __DC_HWSS_DCN32_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c index 28d2202181332488886039574f67c4a775ed0cae..45a949ba6f3f3c97898d0f9f2f1c289876da2ae5 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c @@ -99,12 +99,17 @@ static const struct hw_sequencer_funcs dcn32_funcs = { .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .hardware_release = dcn30_hardware_release, .set_pipe = dcn21_set_pipe, + .enable_lvds_link_output = dce110_enable_lvds_link_output, + .enable_tmds_link_output = dce110_enable_tmds_link_output, + .enable_dp_link_output = dce110_enable_dp_link_output, + .disable_link_output = dcn32_disable_link_output, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, .get_dcc_en_bits = dcn10_get_dcc_en_bits, .commit_subvp_config = dcn32_commit_subvp_config, .subvp_pipe_control_lock = dcn32_subvp_pipe_control_lock, .update_visual_confirm_color = dcn20_update_visual_confirm_color, - .update_phy_state = dcn32_update_phy_state, + .update_phantom_vp_position = dcn32_update_phantom_vp_position, + .update_dsc_pg = dcn32_update_dsc_pg, }; static const struct hwseq_private_funcs dcn32_private_funcs = { @@ -134,6 +139,7 @@ static const struct hwseq_private_funcs dcn32_private_funcs = { .program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree, .update_odm = dcn32_update_odm, .dsc_pg_control = dcn32_dsc_pg_control, + .dsc_pg_status = dcn32_dsc_pg_status, .set_hdr_multiplier = dcn10_set_hdr_multiplier, .verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high, .wait_for_blank_complete = dcn20_wait_for_blank_complete, diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c index adf93cc8359cc16a5b637f8623c384c0a0d814ef..41b0baf8e18373671830007a9c9cafa55cc21bcb 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c @@ -100,7 +100,7 @@ static void mmhubbub32_warmup_mcif(struct mcif_wb *mcif_wb, REG_UPDATE(MMHUBBUB_WARMUP_CONTROL_STATUS, MMHUBBUB_WARMUP_EN, false); } -void mmhubbub32_config_mcif_buf(struct mcif_wb *mcif_wb, +static void mmhubbub32_config_mcif_buf(struct mcif_wb *mcif_wb, struct mcif_buf_params *params, unsigned int dest_height) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h index 22355051f5f73fadc3850f3e52b1384e7eb1b794..e460cf8d9041f7e2e905de46e26060c7cb0b8233 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h @@ -90,7 +90,6 @@ SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, mask_sh),\ SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, mask_sh),\ SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_ADDR_FENCE_EN, mask_sh),\ - SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_VCE_INT_STATUS, mask_sh),\ SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_INT_STATUS, mask_sh),\ SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS, mask_sh),\ SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_CUR_BUF, mask_sh),\ @@ -101,7 +100,6 @@ SF(MCIF_WB_BUF_PITCH, MCIF_WB_BUF_CHROMA_PITCH, mask_sh),\ SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_ACTIVE, mask_sh),\ SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_SW_LOCKED, mask_sh),\ - SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_VCE_LOCKED, mask_sh),\ SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_OVERFLOW, mask_sh),\ SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_DISABLE, mask_sh),\ SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_MODE, mask_sh),\ @@ -116,7 +114,6 @@ SF(MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_C_OVERRUN, mask_sh),\ SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_ACTIVE, mask_sh),\ SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_SW_LOCKED, mask_sh),\ - SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_VCE_LOCKED, mask_sh),\ SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_OVERFLOW, mask_sh),\ SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_DISABLE, mask_sh),\ SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_MODE, mask_sh),\ @@ -131,7 +128,6 @@ SF(MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_C_OVERRUN, mask_sh),\ SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_ACTIVE, mask_sh),\ SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_SW_LOCKED, mask_sh),\ - SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_VCE_LOCKED, mask_sh),\ SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_OVERFLOW, mask_sh),\ SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_DISABLE, mask_sh),\ SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_MODE, mask_sh),\ @@ -146,7 +142,6 @@ SF(MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_C_OVERRUN, mask_sh),\ SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_ACTIVE, mask_sh),\ SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_SW_LOCKED, mask_sh),\ - SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_VCE_LOCKED, mask_sh),\ SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_OVERFLOW, mask_sh),\ SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_DISABLE, mask_sh),\ SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_MODE, mask_sh),\ @@ -172,11 +167,6 @@ SF(MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\ SF(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\ SF(MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\ - SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\ SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\ SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c index 357bd2461bc984fdb40bc7fc254d78d67ec61da2..4edd0655965b806dc35e60d69d1c182c55fbe657 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c @@ -701,7 +701,7 @@ static void mpc32_power_on_shaper_3dlut( } -bool mpc32_program_shaper( +static bool mpc32_program_shaper( struct mpc *mpc, const struct pwl_params *params, uint32_t mpcc_id) @@ -726,7 +726,7 @@ bool mpc32_program_shaper( else next_mode = LUT_RAM_A; - mpc32_configure_shaper_lut(mpc, next_mode == LUT_RAM_A ? true:false, mpcc_id); + mpc32_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id); if (next_mode == LUT_RAM_A) mpc32_program_shaper_luta_settings(mpc, params, mpcc_id); @@ -897,7 +897,7 @@ static void mpc32_set_3dlut_mode( } -bool mpc32_program_3dlut( +static bool mpc32_program_3dlut( struct mpc *mpc, const struct tetrahedral_params *params, int mpcc_id) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c index 1fad7b48bd5beb51d42459856d72ee5385fd064e..2b33eeb213e2a9a0ecc4e323bbb5308328ebff1b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c @@ -151,12 +151,12 @@ static bool optc32_disable_crtc(struct timing_generator *optc) /* CRTC disabled, so disable clock. */ REG_WAIT(OTG_CLOCK_CONTROL, OTG_BUSY, 0, - 1, 100000); + 1, 150000); return true; } -void optc32_phantom_crtc_post_enable(struct timing_generator *optc) +static void optc32_phantom_crtc_post_enable(struct timing_generator *optc) { struct optc *optc1 = DCN10TG_FROM_TG(optc); @@ -190,7 +190,7 @@ static void optc32_set_odm_bypass(struct timing_generator *optc, optc1->opp_count = 1; } -void optc32_setup_manual_trigger(struct timing_generator *optc) +static void optc32_setup_manual_trigger(struct timing_generator *optc) { struct optc *optc1 = DCN10TG_FROM_TG(optc); struct dc *dc = optc->ctx->dc; @@ -215,7 +215,7 @@ void optc32_setup_manual_trigger(struct timing_generator *optc) } } -void optc32_set_drr( +static void optc32_set_drr( struct timing_generator *optc, const struct drr_params *params) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index ef0a6d468a1079904bf74268cb621a515deb647a..a88dd7b3d1c10e01774301b74076ecc413634881 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -90,29 +90,6 @@ #include "dcn20/dcn20_vmid.h" #include "dml/dcn32/dcn32_fpu.h" -#define DCN_BASE__INST0_SEG1 0x000000C0 -#define DCN_BASE__INST0_SEG2 0x000034C0 -#define DCN_BASE__INST0_SEG3 0x00009000 -#define NBIO_BASE__INST0_SEG1 0x00000014 - -#define MAX_INSTANCE 6 -#define MAX_SEGMENT 6 - -struct IP_BASE_INSTANCE { - unsigned int segment[MAX_SEGMENT]; -}; - -struct IP_BASE { - struct IP_BASE_INSTANCE instance[MAX_INSTANCE]; -}; - -static const struct IP_BASE DCN_BASE = { { { { 0x00000012, 0x000000C0, 0x000034C0, 0x00009000, 0x02403C00, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } } } }; - #define DC_LOGGER_INIT(logger) enum dcn32_clk_src_array_id { @@ -152,6 +129,13 @@ enum dcn32_clk_src_array_id { REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ reg ## block ## id ## _ ## reg_name +#define SR_ARR_I2C(reg_name, id) \ + REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name + +#define SRI_ARR_I2C(reg_name, block, id)\ + REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ + reg ## block ## id ## _ ## reg_name + #define SRI_ARR_ALPHABET(reg_name, block, index, id)\ REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ reg ## block ## id ## _ ## reg_name @@ -461,22 +445,17 @@ static const struct dcn20_dsc_mask dsc_mask = { }; static struct dcn30_mpc_registers mpc_regs; -#define dcn_mpc_regs_init()\ - ( \ - MPC_REG_LIST_DCN3_0_RI(0),\ - MPC_REG_LIST_DCN3_0_RI(1),\ - MPC_REG_LIST_DCN3_0_RI(2),\ - MPC_REG_LIST_DCN3_0_RI(3),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\ - MPC_MCM_REG_LIST_DCN32_RI(0),\ - MPC_MCM_REG_LIST_DCN32_RI(1),\ - MPC_MCM_REG_LIST_DCN32_RI(2),\ - MPC_MCM_REG_LIST_DCN32_RI(3),\ - MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)\ - ) + +#define dcn_mpc_regs_init() \ + MPC_REG_LIST_DCN3_2_RI(0),\ + MPC_REG_LIST_DCN3_2_RI(1),\ + MPC_REG_LIST_DCN3_2_RI(2),\ + MPC_REG_LIST_DCN3_2_RI(3),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\ + MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0) static const struct dcn30_mpc_shift mpc_shift = { MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT) @@ -739,7 +718,12 @@ static const struct dc_debug_options debug_defaults_drv = { .force_disable_subvp = false, .exit_idle_opt_for_cursor_updates = true, .enable_single_display_2to1_odm_policy = true, + + /* Must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/ + .enable_double_buffered_dsc_pg_support = true, .enable_dp_dig_pixel_rate_div_policy = 1, + .allow_sw_cursor_fallback = false, + .alloc_extra_way_for_cursor = true, }; static const struct dc_debug_options debug_defaults_diags = { @@ -792,7 +776,7 @@ static struct dce_aux *dcn32_aux_engine_create( #define i2c_inst_regs_init(id)\ I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id) -static struct dce_i2c_registers i2c_hw_regs[6]; +static struct dce_i2c_registers i2c_hw_regs[5]; static const struct dce_i2c_shift i2c_shifts = { I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT) @@ -916,10 +900,10 @@ static struct hubp *dcn32_hubp_create( #undef REG_STRUCT #define REG_STRUCT hubp_regs - hubp_regs_init(0), - hubp_regs_init(1), - hubp_regs_init(2), - hubp_regs_init(3); + hubp_regs_init(0), + hubp_regs_init(1), + hubp_regs_init(2), + hubp_regs_init(3); if (hubp32_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask)) @@ -1696,6 +1680,8 @@ static void dcn32_enable_phantom_plane(struct dc *dc, phantom_plane->clip_rect.y = 0; phantom_plane->clip_rect.height = phantom_stream->timing.v_addressable; + phantom_plane->is_phantom = true; + dc_add_plane_to_context(dc, phantom_stream, phantom_plane, context); curr_pipe = curr_pipe->bottom_pipe; @@ -1765,6 +1751,10 @@ bool dcn32_remove_phantom_pipes(struct dc *dc, struct dc_state *context) pipe->stream->mall_stream_config.type = SUBVP_NONE; pipe->stream->mall_stream_config.paired_stream = NULL; } + + if (pipe->plane_state) { + pipe->plane_state->is_phantom = false; + } } return removed_pipe; } @@ -1814,14 +1804,39 @@ bool dcn32_validate_bandwidth(struct dc *dc, int vlevel = 0; int pipe_cnt = 0; display_e2e_pipe_params_st *pipes = kzalloc(dc->res_pool->pipe_count * sizeof(display_e2e_pipe_params_st), GFP_KERNEL); + struct mall_temp_config mall_temp_config; + + /* To handle Freesync properly, setting FreeSync DML parameters + * to its default state for the first stage of validation + */ + context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching = false; + context->bw_ctx.dml.soc.dram_clock_change_requirement_final = true; + DC_LOGGER_INIT(dc->ctx->logger); + /* For fast validation, there are situations where a shallow copy of + * of the dc->current_state is created for the validation. In this case + * we want to save and restore the mall config because we always + * teardown subvp at the beginning of validation (and don't attempt + * to add it back if it's fast validation). If we don't restore the + * subvp config in cases of fast validation + shallow copy of the + * dc->current_state, the dc->current_state will have a partially + * removed subvp state when we did not intend to remove it. + */ + if (fast_validate) { + memset(&mall_temp_config, 0, sizeof(mall_temp_config)); + dcn32_save_mall_state(dc, context, &mall_temp_config); + } + BW_VAL_TRACE_COUNT(); DC_FP_START(); out = dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate); DC_FP_END(); + if (fast_validate) + dcn32_restore_mall_state(dc, context, &mall_temp_config); + if (pipe_cnt == 0) goto validate_out; @@ -1856,12 +1871,6 @@ validate_out: return out; } - -static bool is_dual_plane(enum surface_pixel_format format) -{ - return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA; -} - int dcn32_populate_dml_pipes_from_context( struct dc *dc, struct dc_state *context, display_e2e_pipe_params_st *pipes, @@ -1870,12 +1879,37 @@ int dcn32_populate_dml_pipes_from_context( int i, pipe_cnt; struct resource_context *res_ctx = &context->res_ctx; struct pipe_ctx *pipe; - bool subvp_in_use = false, is_pipe_split_expected[MAX_PIPES]; - int plane_count = 0; + bool subvp_in_use = false; + uint8_t is_pipe_split_expected[MAX_PIPES] = {0}; struct dc_crtc_timing *timing; dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate); + /* Determine whether we will apply ODM 2to1 policy: + * Applies to single display and where the number of planes is less than 3. + * For 3 plane case ( 2 MPO planes ), we will not set the policy for the MPO pipes. + * + * Apply pipe split policy first so we can predict the pipe split correctly + * (dcn32_predict_pipe_split). + */ + for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { + if (!res_ctx->pipe_ctx[i].stream) + continue; + pipe = &res_ctx->pipe_ctx[i]; + timing = &pipe->stream->timing; + + pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal; + if (context->stream_count == 1 && + context->stream_status[0].plane_count <= 1 && + !dc_is_hdmi_signal(res_ctx->pipe_ctx[i].stream->signal) && + is_h_timing_divisible_by_2(res_ctx->pipe_ctx[i].stream) && + pipe->stream->timing.pix_clk_100hz * 100 > DCN3_2_VMIN_DISPCLK_HZ && + dc->debug.enable_single_display_2to1_odm_policy) { + pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_2to1; + } + pipe_cnt++; + } + for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { if (!res_ctx->pipe_ctx[i].stream) @@ -1928,57 +1962,18 @@ int dcn32_populate_dml_pipes_from_context( } } - /* Calculate the number of planes we have so we can determine - * whether to apply ODM 2to1 policy or not - */ - if (pipe->stream && !pipe->prev_odm_pipe && - (!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state)) - ++plane_count; - DC_FP_START(); - is_pipe_split_expected[i] = dcn32_predict_pipe_split(context, pipes[i].pipe, i); + is_pipe_split_expected[i] = dcn32_predict_pipe_split(context, &pipes[pipe_cnt]); DC_FP_END(); pipe_cnt++; } - /* Determine whether we will apply ODM 2to1 policy - * Applies to single display and where the number of planes is less than 3 - * For 3 plane case ( 2 MPO planes ), we will not set the policy for the MPO pipes - */ - for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { - if (!res_ctx->pipe_ctx[i].stream) - continue; - pipe = &res_ctx->pipe_ctx[i]; - timing = &pipe->stream->timing; - - pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal; - if (context->stream_count == 1 && !dc_is_hdmi_signal(res_ctx->pipe_ctx[i].stream->signal)) { - if (dc->debug.enable_single_display_2to1_odm_policy) { - if (!((plane_count > 2) && pipe->top_pipe)) - pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_2to1; - } - } - pipe_cnt++; - } - /* For DET allocation, we don't want to use DML policy (not optimal for utilizing all * the DET available for each pipe). Use the DET override input to maintain our driver * policy. */ - if (pipe_cnt == 1 && !is_pipe_split_expected[0]) { - pipes[0].pipe.src.det_size_override = DCN3_2_MAX_DET_SIZE; - if (pipe->plane_state && !dc->debug.disable_z9_mpc) { - if (!is_dual_plane(pipe->plane_state->format)) { - pipes[0].pipe.src.det_size_override = DCN3_2_DEFAULT_DET_SIZE; - pipes[0].pipe.src.unbounded_req_mode = true; - if (pipe->plane_state->src_rect.width >= 5120 && - pipe->plane_state->src_rect.height >= 2880) - pipes[0].pipe.src.det_size_override = 320; // 5K or higher - } - } - } else - dcn32_determine_det_override(context, pipes, is_pipe_split_expected, dc->res_pool->pipe_count); + dcn32_set_det_allocations(dc, context, pipes); // In general cases we want to keep the dram clock change requirement // (prefer configs that support MCLK switch). Only override to false @@ -2108,7 +2103,8 @@ static bool dcn32_resource_construct( dc->caps.max_downscale_ratio = 600; dc->caps.i2c_speed_in_khz = 100; dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/ - dc->caps.max_cursor_size = 256; + /* TODO: Bring max_cursor_size back to 256 after subvp cursor corruption is fixed*/ + dc->caps.max_cursor_size = 64; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; dc->caps.mall_size_per_mem_channel = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h index 60d8fad16eee5ff9f475d2cd79cd3c0aa755be7b..f76120e67c16a9c633c26a6aa1e40646706b141e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h @@ -28,8 +28,16 @@ #include "core_types.h" +#define DCN3_2_DEFAULT_DET_SIZE 256 +#define DCN3_2_MAX_DET_SIZE 1152 +#define DCN3_2_MIN_DET_SIZE 128 +#define DCN3_2_MIN_COMPBUF_SIZE_KB 128 #define DCN3_2_DET_SEG_SIZE 64 #define DCN3_2_MALL_MBLK_SIZE_BYTES 65536 // 64 * 1024 +#define DCN3_2_MBLK_WIDTH 128 +#define DCN3_2_MBLK_HEIGHT_4BPE 128 +#define DCN3_2_MBLK_HEIGHT_8BPE 64 +#define DCN3_2_VMIN_DISPCLK_HZ 717000000 #define TO_DCN32_RES_POOL(pool)\ container_of(pool, struct dcn32_resource_pool, base) @@ -37,6 +45,17 @@ extern struct _vcs_dpi_ip_params_st dcn3_2_ip; extern struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc; +/* Temp struct used to save and restore MALL config + * during validation. + * + * TODO: Move MALL config into dc_state instead of stream struct + * to avoid needing to save/restore. + */ +struct mall_temp_config { + struct mall_stream_config mall_stream_config[MAX_PIPES]; + bool is_phantom_plane[MAX_PIPES]; +}; + struct dcn32_resource_pool { struct resource_pool base; }; @@ -100,14 +119,28 @@ bool dcn32_subvp_in_use(struct dc *dc, bool dcn32_mpo_in_use(struct dc_state *context); +bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context); + struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer( struct dc_state *state, const struct resource_pool *pool, struct dc_stream_state *stream, struct pipe_ctx *head_pipe); -void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_params_st *pipes, - bool *is_pipe_split_expected, int pipe_cnt); +void dcn32_determine_det_override(struct dc *dc, + struct dc_state *context, + display_e2e_pipe_params_st *pipes); + +void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context, + display_e2e_pipe_params_st *pipes); + +void dcn32_save_mall_state(struct dc *dc, + struct dc_state *context, + struct mall_temp_config *temp_config); + +void dcn32_restore_mall_state(struct dc *dc, + struct dc_state *context, + struct mall_temp_config *temp_config); /* definitions for run time init of reg offsets */ @@ -222,7 +255,8 @@ void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_par SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id), \ SRI_ARR(DP_MSE_RATE_CNTL, DP, id), SRI_ARR(DP_MSE_RATE_UPDATE, DP, id), \ SRI_ARR(DP_PIXEL_FORMAT, DP, id), SRI_ARR(DP_SEC_CNTL, DP, id), \ - SRI_ARR(DP_SEC_CNTL2, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id), \ + SRI_ARR(DP_SEC_CNTL1, DP, id), SRI_ARR(DP_SEC_CNTL2, DP, id), \ + SRI_ARR(DP_SEC_CNTL5, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id), \ SRI_ARR(DP_STEER_FIFO, DP, id), SRI_ARR(DP_VID_M, DP, id), \ SRI_ARR(DP_VID_N, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id), \ SRI_ARR(DP_VID_TIMING, DP, id), SRI_ARR(DP_SEC_AUD_N, DP, id), \ @@ -735,75 +769,6 @@ void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_par #define MPC_DWB_MUX_REG_LIST_DCN3_0_RI(inst) \ SRII_DWB(DWB_MUX, MUX, MPC_DWB, inst) -#define MPC_MCM_REG_LIST_DCN32_RI(inst) \ - ( \ - SRII(MPCC_MCM_SHAPER_CONTROL, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_2_3, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_4_5, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_6_7, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_8_9, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_10_11, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_12_13, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_14_15, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_16_17, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_18_19, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_20_21, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_22_23, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_24_25, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_26_27, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_28_29, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_30_31, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMA_REGION_32_33, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_0_1, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_2_3, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_4_5, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_6_7, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_8_9, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_10_11, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_12_13, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_14_15, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_16_17, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_18_19, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_20_21, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_22_23, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_24_25, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_26_27, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_28_29, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_30_31, MPCC_MCM, inst), \ - SRII(MPCC_MCM_SHAPER_RAMB_REGION_32_33, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_MODE, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_INDEX, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_DATA, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_OUT_NORM_FACTOR, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM, inst), \ - SRII(MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM, inst), \ - SRII(MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM, inst) \ - ) - #define MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst) \ ( \ SRII(MUX, MPC_OUT, inst), VUPDATE_SRII(CUR, VUPDATE_LOCK_SET, inst) \ @@ -887,6 +852,149 @@ void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_par SRII(MPCC_OGAM_LUT_CONTROL, MPCC_OGAM, inst) \ ) +#define MPC_REG_LIST_DCN3_2_RI(inst) \ + MPC_REG_LIST_DCN3_0_RI(inst),\ + SRII(MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC, inst),\ + SRII(MPCC_MCM_SHAPER_CONTROL, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_2_3, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_4_5, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_6_7, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_8_9, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_10_11, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_12_13, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_14_15, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_16_17, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_18_19, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_20_21, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_22_23, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_24_25, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_26_27, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_28_29, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_30_31, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMA_REGION_32_33, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_0_1, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_2_3, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_4_5, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_6_7, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_8_9, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_10_11, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_12_13, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_14_15, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_16_17, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_18_19, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_20_21, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_22_23, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_24_25, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_26_27, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_28_29, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_30_31, MPCC_MCM, inst),\ + SRII(MPCC_MCM_SHAPER_RAMB_REGION_32_33, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_MODE, MPCC_MCM, inst), /*TODO: may need to add other 3DLUT regs*/\ + SRII(MPCC_MCM_3DLUT_INDEX, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_DATA, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_OUT_NORM_FACTOR, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_CONTROL, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_LUT_INDEX, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_LUT_DATA, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_2_3, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_4_5, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_6_7, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_8_9, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_10_11, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_12_13, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_14_15, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_16_17, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_18_19, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_20_21, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_22_23, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_24_25, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_26_27, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_28_29, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_30_31, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMA_REGION_32_33, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_B, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_G, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_R, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_0_1, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_2_3, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_4_5, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_6_7, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_8_9, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_10_11, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_12_13, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_14_15, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_16_17, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_18_19, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_20_21, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_22_23, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_24_25, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_26_27, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_28_29, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_30_31, MPCC_MCM, inst),\ + SRII(MPCC_MCM_1DLUT_RAMB_REGION_32_33, MPCC_MCM, inst),\ + SRII(MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM, inst) + /* OPTC */ #define OPTC_COMMON_REG_LIST_DCN3_2_RI(inst) \ @@ -1121,6 +1229,7 @@ void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_par SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D), SR(DCHUBBUB_DET0_CTRL), \ SR(DCHUBBUB_DET1_CTRL), SR(DCHUBBUB_DET2_CTRL), SR(DCHUBBUB_DET3_CTRL), \ SR(DCHUBBUB_COMPBUF_CTRL), SR(COMPBUF_RESERVED_SPACE), \ + SR(DCHUBBUB_DEBUG_CTRL_0), \ SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL), \ SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A), \ SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B), \ @@ -1175,18 +1284,19 @@ void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_par #define I2C_HW_ENGINE_COMMON_REG_LIST_RI(id) \ ( \ - SRI_ARR(SETUP, DC_I2C_DDC, id), SRI_ARR(SPEED, DC_I2C_DDC, id), \ - SRI_ARR(HW_STATUS, DC_I2C_DDC, id), SR_ARR(DC_I2C_ARBITRATION, id), \ - SR_ARR(DC_I2C_CONTROL, id), SR_ARR(DC_I2C_SW_STATUS, id), \ - SR_ARR(DC_I2C_TRANSACTION0, id), SR_ARR(DC_I2C_TRANSACTION1, id), \ - SR_ARR(DC_I2C_TRANSACTION2, id), SR_ARR(DC_I2C_TRANSACTION3, id), \ - SR_ARR(DC_I2C_DATA, id), SR_ARR(MICROSECOND_TIME_BASE_DIV, id) \ + SRI_ARR_I2C(SETUP, DC_I2C_DDC, id), SRI_ARR_I2C(SPEED, DC_I2C_DDC, id), \ + SRI_ARR_I2C(HW_STATUS, DC_I2C_DDC, id), \ + SR_ARR_I2C(DC_I2C_ARBITRATION, id), \ + SR_ARR_I2C(DC_I2C_CONTROL, id), SR_ARR_I2C(DC_I2C_SW_STATUS, id), \ + SR_ARR_I2C(DC_I2C_TRANSACTION0, id), SR_ARR_I2C(DC_I2C_TRANSACTION1, id),\ + SR_ARR_I2C(DC_I2C_TRANSACTION2, id), SR_ARR_I2C(DC_I2C_TRANSACTION3, id),\ + SR_ARR_I2C(DC_I2C_DATA, id), SR_ARR_I2C(MICROSECOND_TIME_BASE_DIV, id) \ ) #define I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id) \ ( \ - I2C_HW_ENGINE_COMMON_REG_LIST_RI(id), SR_ARR(DIO_MEM_PWR_CTRL, id), \ - SR_ARR(DIO_MEM_PWR_STATUS, id) \ + I2C_HW_ENGINE_COMMON_REG_LIST_RI(id), SR_ARR_I2C(DIO_MEM_PWR_CTRL, id), \ + SR_ARR_I2C(DIO_MEM_PWR_STATUS, id) \ ) #endif /* _DCN32_RESOURCE_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c index ab918fe38f6a57f34dc247937ea2464827124dca..d51d0c40ae5bca01e88141a7928d176223a3fa82 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c @@ -28,6 +28,11 @@ #include "dcn20/dcn20_resource.h" #include "dml/dcn32/display_mode_vba_util_32.h" +static bool is_dual_plane(enum surface_pixel_format format) +{ + return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA; +} + /** * ******************************************************************************************** * dcn32_helper_calculate_num_ways_for_subvp: Calculate number of ways needed for SubVP @@ -46,7 +51,6 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_state *context) { uint32_t num_ways = 0; - uint32_t mall_region_pixels = 0; uint32_t bytes_per_pixel = 0; uint32_t cache_lines_used = 0; uint32_t lines_per_way = 0; @@ -54,28 +58,77 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_stat uint32_t bytes_in_mall = 0; uint32_t num_mblks = 0; uint32_t cache_lines_per_plane = 0; - uint32_t i = 0; + uint32_t i = 0, j = 0; + uint16_t mblk_width = 0; + uint16_t mblk_height = 0; + uint32_t full_vp_width_blk_aligned = 0; + uint32_t full_vp_height_blk_aligned = 0; + uint32_t mall_alloc_width_blk_aligned = 0; + uint32_t mall_alloc_height_blk_aligned = 0; + uint16_t full_vp_height = 0; + bool subvp_in_use = false; for (i = 0; i < dc->res_pool->pipe_count; i++) { struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; - // Find the phantom pipes - if (pipe->stream && pipe->plane_state && !pipe->top_pipe && + /* Find the phantom pipes. + * - For pipe split case we need to loop through the bottom and next ODM + * pipes or only half the viewport size is counted + */ + if (pipe->stream && pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) { + struct pipe_ctx *main_pipe = NULL; + + subvp_in_use = true; + /* Get full viewport height from main pipe (required for MBLK calculation) */ + for (j = 0; j < dc->res_pool->pipe_count; j++) { + main_pipe = &context->res_ctx.pipe_ctx[j]; + if (main_pipe->stream == pipe->stream->mall_stream_config.paired_stream) { + full_vp_height = main_pipe->plane_res.scl_data.viewport.height; + break; + } + } + bytes_per_pixel = pipe->plane_state->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4; - mall_region_pixels = pipe->plane_state->plane_size.surface_pitch * pipe->stream->timing.v_addressable; + mblk_width = DCN3_2_MBLK_WIDTH; + mblk_height = bytes_per_pixel == 4 ? DCN3_2_MBLK_HEIGHT_4BPE : DCN3_2_MBLK_HEIGHT_8BPE; + + /* full_vp_width_blk_aligned = FLOOR(vp_x_start + full_vp_width + blk_width - 1, blk_width) - + * FLOOR(vp_x_start, blk_width) + */ + full_vp_width_blk_aligned = ((pipe->plane_res.scl_data.viewport.x + + pipe->plane_res.scl_data.viewport.width + mblk_width - 1) / mblk_width * mblk_width) + + (pipe->plane_res.scl_data.viewport.x / mblk_width * mblk_width); + + /* full_vp_height_blk_aligned = FLOOR(vp_y_start + full_vp_height + blk_height - 1, blk_height) - + * FLOOR(vp_y_start, blk_height) + */ + full_vp_height_blk_aligned = ((pipe->plane_res.scl_data.viewport.y + + full_vp_height + mblk_height - 1) / mblk_height * mblk_height) + + (pipe->plane_res.scl_data.viewport.y / mblk_height * mblk_height); + + /* mall_alloc_width_blk_aligned_l/c = full_vp_width_blk_aligned_l/c */ + mall_alloc_width_blk_aligned = full_vp_width_blk_aligned; - // For bytes required in MALL, calculate based on number of MBlks required - num_mblks = (mall_region_pixels * bytes_per_pixel + - DCN3_2_MALL_MBLK_SIZE_BYTES - 1) / DCN3_2_MALL_MBLK_SIZE_BYTES; + /* mall_alloc_height_blk_aligned_l/c = CEILING(sub_vp_height_l/c - 1, blk_height_l/c) + blk_height_l/c */ + mall_alloc_height_blk_aligned = (pipe->stream->timing.v_addressable - 1 + mblk_height - 1) / + mblk_height * mblk_height + mblk_height; + + /* full_mblk_width_ub_l/c = mall_alloc_width_blk_aligned_l/c; + * full_mblk_height_ub_l/c = mall_alloc_height_blk_aligned_l/c; + * num_mblk_l/c = (full_mblk_width_ub_l/c / mblk_width_l/c) * (full_mblk_height_ub_l/c / mblk_height_l/c); + * (Should be divisible, but round up if not) + */ + num_mblks = ((mall_alloc_width_blk_aligned + mblk_width - 1) / mblk_width) * + ((mall_alloc_height_blk_aligned + mblk_height - 1) / mblk_height); bytes_in_mall = num_mblks * DCN3_2_MALL_MBLK_SIZE_BYTES; // cache lines used is total bytes / cache_line size. Add +2 for worst case alignment // (MALL is 64-byte aligned) cache_lines_per_plane = bytes_in_mall / dc->caps.cache_line_size + 2; - // For DCC we must cache the meat surface, so double cache lines required + /* For DCC divide by 256 */ if (pipe->plane_state->dcc.enable) - cache_lines_per_plane *= 2; + cache_lines_per_plane = cache_lines_per_plane + (cache_lines_per_plane / 256) + 1; cache_lines_used += cache_lines_per_plane; } } @@ -86,6 +139,9 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_stat if (cache_lines_used % lines_per_way > 0) num_ways++; + if (subvp_in_use && dc->debug.force_subvp_num_ways > 0) + num_ways = dc->debug.force_subvp_num_ways; + return num_ways; } @@ -177,36 +233,221 @@ bool dcn32_mpo_in_use(struct dc_state *context) return false; } -void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_params_st *pipes, - bool *is_pipe_split_expected, int pipe_cnt) + +bool dcn32_any_surfaces_rotated(struct dc *dc, struct dc_state *context) +{ + uint32_t i; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (!pipe->stream) + continue; + + if (pipe->plane_state && pipe->plane_state->rotation != ROTATION_ANGLE_0) + return true; + } + return false; +} + +/** + * ******************************************************************************************* + * dcn32_determine_det_override: Determine DET allocation for each pipe + * + * This function determines how much DET to allocate for each pipe. The total number of + * DET segments will be split equally among each of the streams, and after that the DET + * segments per stream will be split equally among the planes for the given stream. + * + * If there is a plane that's driven by more than 1 pipe (i.e. pipe split), then the + * number of DET for that given plane will be split among the pipes driving that plane. + * + * + * High level algorithm: + * 1. Split total DET among number of streams + * 2. For each stream, split DET among the planes + * 3. For each plane, check if there is a pipe split. If yes, split the DET allocation + * among those pipes. + * 4. Assign the DET override to the DML pipes. + * + * @param [in]: dc: Current DC state + * @param [in]: context: New DC state to be programmed + * @param [in]: pipes: Array of DML pipes + * + * @return: void + * + * ******************************************************************************************* + */ +void dcn32_determine_det_override(struct dc *dc, + struct dc_state *context, + display_e2e_pipe_params_st *pipes) { - int i, j, count, stream_segments, pipe_segments[MAX_PIPES]; + uint32_t i, j, k; + uint8_t pipe_plane_count, stream_segments, plane_segments, pipe_segments[MAX_PIPES] = {0}; + uint8_t pipe_counted[MAX_PIPES] = {0}; + uint8_t pipe_cnt = 0; + struct dc_plane_state *current_plane = NULL; + uint8_t stream_count = 0; - if (context->stream_count > 0) { - stream_segments = 18 / context->stream_count; + for (i = 0; i < context->stream_count; i++) { + /* Don't count SubVP streams for DET allocation */ + if (context->streams[i]->mall_stream_config.type != SUBVP_PHANTOM) { + stream_count++; + } + } + + if (stream_count > 0) { + stream_segments = 18 / stream_count; for (i = 0; i < context->stream_count; i++) { - count = 0; - for (j = 0; j < pipe_cnt; j++) { - if (context->res_ctx.pipe_ctx[j].stream == context->streams[i]) { - count++; - if (is_pipe_split_expected[j]) - count++; + if (context->streams[i]->mall_stream_config.type == SUBVP_PHANTOM) + continue; + if (context->stream_status[i].plane_count > 0) + plane_segments = stream_segments / context->stream_status[i].plane_count; + else + plane_segments = stream_segments; + for (j = 0; j < dc->res_pool->pipe_count; j++) { + pipe_plane_count = 0; + if (context->res_ctx.pipe_ctx[j].stream == context->streams[i] && + pipe_counted[j] != 1) { + /* Note: pipe_plane_count indicates the number of pipes to be used for a + * given plane. e.g. pipe_plane_count = 1 means single pipe (i.e. not split), + * pipe_plane_count = 2 means 2:1 split, etc. + */ + pipe_plane_count++; + pipe_counted[j] = 1; + current_plane = context->res_ctx.pipe_ctx[j].plane_state; + for (k = 0; k < dc->res_pool->pipe_count; k++) { + if (k != j && context->res_ctx.pipe_ctx[k].stream == context->streams[i] && + context->res_ctx.pipe_ctx[k].plane_state == current_plane) { + pipe_plane_count++; + pipe_counted[k] = 1; + } + } + + pipe_segments[j] = plane_segments / pipe_plane_count; + for (k = 0; k < dc->res_pool->pipe_count; k++) { + if (k != j && context->res_ctx.pipe_ctx[k].stream == context->streams[i] && + context->res_ctx.pipe_ctx[k].plane_state == current_plane) { + pipe_segments[k] = plane_segments / pipe_plane_count; + } + } } } - pipe_segments[i] = stream_segments / count; } - for (i = 0; i < pipe_cnt; i++) { - pipes[i].pipe.src.det_size_override = 0; - for (j = 0; j < context->stream_count; j++) { - if (context->res_ctx.pipe_ctx[i].stream == context->streams[j]) { - pipes[i].pipe.src.det_size_override = pipe_segments[j] * DCN3_2_DET_SEG_SIZE; - break; - } - } + for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { + if (!context->res_ctx.pipe_ctx[i].stream) + continue; + pipes[pipe_cnt].pipe.src.det_size_override = pipe_segments[i] * DCN3_2_DET_SEG_SIZE; + pipe_cnt++; } } else { - for (i = 0; i < pipe_cnt; i++) + for (i = 0; i < dc->res_pool->pipe_count; i++) pipes[i].pipe.src.det_size_override = 4 * DCN3_2_DET_SEG_SIZE; //DCN3_2_DEFAULT_DET_SIZE } } + +void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context, + display_e2e_pipe_params_st *pipes) +{ + int i, pipe_cnt; + struct resource_context *res_ctx = &context->res_ctx; + struct pipe_ctx *pipe; + + for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) { + + if (!res_ctx->pipe_ctx[i].stream) + continue; + + pipe = &res_ctx->pipe_ctx[i]; + pipe_cnt++; + } + + /* For DET allocation, we don't want to use DML policy (not optimal for utilizing all + * the DET available for each pipe). Use the DET override input to maintain our driver + * policy. + */ + if (pipe_cnt == 1) { + pipes[0].pipe.src.det_size_override = DCN3_2_MAX_DET_SIZE; + if (pipe->plane_state && !dc->debug.disable_z9_mpc && pipe->plane_state->tiling_info.gfx9.swizzle != DC_SW_LINEAR) { + if (!is_dual_plane(pipe->plane_state->format)) { + pipes[0].pipe.src.det_size_override = DCN3_2_DEFAULT_DET_SIZE; + pipes[0].pipe.src.unbounded_req_mode = true; + if (pipe->plane_state->src_rect.width >= 5120 && + pipe->plane_state->src_rect.height >= 2880) + pipes[0].pipe.src.det_size_override = 320; // 5K or higher + } + } + } else + dcn32_determine_det_override(dc, context, pipes); +} + +/** + * ******************************************************************************************* + * dcn32_save_mall_state: Save MALL (SubVP) state for fast validation cases + * + * This function saves the MALL (SubVP) case for fast validation cases. For fast validation, + * there are situations where a shallow copy of the dc->current_state is created for the + * validation. In this case we want to save and restore the mall config because we always + * teardown subvp at the beginning of validation (and don't attempt to add it back if it's + * fast validation). If we don't restore the subvp config in cases of fast validation + + * shallow copy of the dc->current_state, the dc->current_state will have a partially + * removed subvp state when we did not intend to remove it. + * + * NOTE: This function ONLY works if the streams are not moved to a different pipe in the + * validation. We don't expect this to happen in fast_validation=1 cases. + * + * @param [in]: dc: Current DC state + * @param [in]: context: New DC state to be programmed + * @param [out]: temp_config: struct used to cache the existing MALL state + * + * @return: void + * + * ******************************************************************************************* + */ +void dcn32_save_mall_state(struct dc *dc, + struct dc_state *context, + struct mall_temp_config *temp_config) +{ + uint32_t i; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (pipe->stream) + temp_config->mall_stream_config[i] = pipe->stream->mall_stream_config; + + if (pipe->plane_state) + temp_config->is_phantom_plane[i] = pipe->plane_state->is_phantom; + } +} + +/** + * ******************************************************************************************* + * dcn32_restore_mall_state: Restore MALL (SubVP) state for fast validation cases + * + * Restore the MALL state based on the previously saved state from dcn32_save_mall_state + * + * @param [in]: dc: Current DC state + * @param [in/out]: context: New DC state to be programmed, restore MALL state into here + * @param [in]: temp_config: struct that has the cached MALL state + * + * @return: void + * + * ******************************************************************************************* + */ +void dcn32_restore_mall_state(struct dc *dc, + struct dc_state *context, + struct mall_temp_config *temp_config) +{ + uint32_t i; + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (pipe->stream) + pipe->stream->mall_stream_config = temp_config->mall_stream_config[i]; + + if (pipe->plane_state) + pipe->plane_state->is_phantom = temp_config->is_phantom_plane[i]; + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c index 49682a31ecbd7b18ab820d59f8217f146c976647..fa9b6603cfd37e4d890a831d61d8c2707bc40b29 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_dio_link_encoder.c @@ -91,7 +91,6 @@ static const struct link_encoder_funcs dcn321_link_enc_funcs = { .is_in_alt_mode = dcn20_link_encoder_is_in_alt_mode, .get_max_link_cap = dcn20_link_encoder_get_max_link_cap, .set_dio_phy_mux = dcn31_link_encoder_set_dio_phy_mux, - .set_dig_output_mode = enc32_set_dig_output_mode, }; void dcn321_link_encoder_construct( diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c index a93dc00ebfb5b17c2171c295e318d05b0686e4d9..61087f2385a96ccc351bbad7e107b8d6525f31e1 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c @@ -93,34 +93,7 @@ #include "vm_helper.h" #include "dcn20/dcn20_vmid.h" -#define DCN_BASE__INST0_SEG1 0x000000C0 -#define DCN_BASE__INST0_SEG2 0x000034C0 -#define DCN_BASE__INST0_SEG3 0x00009000 -#define NBIO_BASE__INST0_SEG1 0x00000014 - -#define MAX_INSTANCE 8 -#define MAX_SEGMENT 6 - -struct IP_BASE_INSTANCE { - unsigned int segment[MAX_SEGMENT]; -}; - -struct IP_BASE { - struct IP_BASE_INSTANCE instance[MAX_INSTANCE]; -}; - -static const struct IP_BASE DCN_BASE = { { { { 0x00000012, 0x000000C0, 0x000034C0, 0x00009000, 0x02403C00, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } }, - { { 0, 0, 0, 0, 0, 0 } } } }; - #define DC_LOGGER_INIT(logger) -#define fixed16_to_double(x) (((double)x) / ((double) (1 << 16))) -#define fixed16_to_double_to_cpu(x) fixed16_to_double(le32_to_cpu(x)) enum dcn321_clk_src_array_id { DCN321_CLK_SRC_PLL0, @@ -159,6 +132,13 @@ enum dcn321_clk_src_array_id { REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ reg ## block ## id ## _ ## reg_name +#define SR_ARR_I2C(reg_name, id) \ + REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name + +#define SRI_ARR_I2C(reg_name, block, id)\ + REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ + reg ## block ## id ## _ ## reg_name + #define SRI_ARR_ALPHABET(reg_name, block, index, id)\ REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \ reg ## block ## id ## _ ## reg_name @@ -466,21 +446,15 @@ static const struct dcn20_dsc_mask dsc_mask = { static struct dcn30_mpc_registers mpc_regs; #define dcn_mpc_regs_init()\ - ( \ - MPC_REG_LIST_DCN3_0_RI(0),\ - MPC_REG_LIST_DCN3_0_RI(1),\ - MPC_REG_LIST_DCN3_0_RI(2),\ - MPC_REG_LIST_DCN3_0_RI(3),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\ - MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\ - MPC_MCM_REG_LIST_DCN32_RI(0),\ - MPC_MCM_REG_LIST_DCN32_RI(1),\ - MPC_MCM_REG_LIST_DCN32_RI(2),\ - MPC_MCM_REG_LIST_DCN32_RI(3),\ - MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)\ - ) + MPC_REG_LIST_DCN3_2_RI(0),\ + MPC_REG_LIST_DCN3_2_RI(1),\ + MPC_REG_LIST_DCN3_2_RI(2),\ + MPC_REG_LIST_DCN3_2_RI(3),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\ + MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\ + MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0) static const struct dcn30_mpc_shift mpc_shift = { MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT) @@ -742,7 +716,12 @@ static const struct dc_debug_options debug_defaults_drv = { .force_disable_subvp = false, .exit_idle_opt_for_cursor_updates = true, .enable_single_display_2to1_odm_policy = true, + + /*must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/ + .enable_double_buffered_dsc_pg_support = true, .enable_dp_dig_pixel_rate_div_policy = 1, + .allow_sw_cursor_fallback = false, + .alloc_extra_way_for_cursor = true, }; static const struct dc_debug_options debug_defaults_diags = { @@ -796,7 +775,7 @@ static struct dce_aux *dcn321_aux_engine_create( #define i2c_inst_regs_init(id)\ I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id) -static struct dce_i2c_registers i2c_hw_regs[6]; +static struct dce_i2c_registers i2c_hw_regs[5]; static const struct dce_i2c_shift i2c_shifts = { I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT) @@ -818,11 +797,11 @@ static struct dce_i2c_hw *dcn321_i2c_hw_create( #undef REG_STRUCT #define REG_STRUCT i2c_hw_regs - i2c_inst_regs_init(1), - i2c_inst_regs_init(2), - i2c_inst_regs_init(3), - i2c_inst_regs_init(4), - i2c_inst_regs_init(5); + i2c_inst_regs_init(1), + i2c_inst_regs_init(2), + i2c_inst_regs_init(3), + i2c_inst_regs_init(4), + i2c_inst_regs_init(5); dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst, &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks); @@ -920,10 +899,10 @@ static struct hubp *dcn321_hubp_create( #undef REG_STRUCT #define REG_STRUCT hubp_regs - hubp_regs_init(0), - hubp_regs_init(1), - hubp_regs_init(2), - hubp_regs_init(3); + hubp_regs_init(0), + hubp_regs_init(1), + hubp_regs_init(2), + hubp_regs_init(3); if (hubp32_construct(hubp2, ctx, inst, &hubp_regs[inst], &hubp_shift, &hubp_mask)) @@ -1625,7 +1604,7 @@ static struct resource_funcs dcn321_res_pool_funcs = { .validate_bandwidth = dcn32_validate_bandwidth, .calculate_wm_and_dlg = dcn32_calculate_wm_and_dlg, .populate_dml_pipes = dcn32_populate_dml_pipes_from_context, - .acquire_idle_pipe_for_layer = dcn20_acquire_idle_pipe_for_layer, + .acquire_idle_pipe_for_head_pipe_in_layer = dcn32_acquire_idle_pipe_for_head_pipe_in_layer, .add_stream_to_ctx = dcn30_add_stream_to_ctx, .add_dsc_to_stream_resource = dcn20_add_dsc_to_stream_resource, .remove_stream_from_ctx = dcn20_remove_stream_from_ctx, @@ -1668,14 +1647,14 @@ static bool dcn321_resource_construct( #undef REG_STRUCT #define REG_STRUCT abm_regs - abm_regs_init(0), - abm_regs_init(1), - abm_regs_init(2), - abm_regs_init(3); + abm_regs_init(0), + abm_regs_init(1), + abm_regs_init(2), + abm_regs_init(3); #undef REG_STRUCT #define REG_STRUCT dccg_regs - dccg_regs_init(); + dccg_regs_init(); ctx->dc_bios->regs = &bios_regs; @@ -1713,7 +1692,8 @@ static bool dcn321_resource_construct( dc->caps.max_downscale_ratio = 600; dc->caps.i2c_speed_in_khz = 100; dc->caps.i2c_speed_in_khz_hdcp = 100; /*1.4 w/a applied by default*/ - dc->caps.max_cursor_size = 256; + /* TODO: Bring max cursor size back to 256 after subvp cursor corruption is fixed*/ + dc->caps.max_cursor_size = 64; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; dc->caps.mall_size_per_mem_channel = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h index e93187c066482eff950587c396f370a3f432f1b8..e3e5c39895a3ad20cbae84446e29cfbefeaf0e60 100644 --- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h +++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h @@ -173,7 +173,8 @@ void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigne (result == 0x0) void dm_helpers_init_panel_settings( struct dc_context *ctx, - struct dc_panel_config *config); + struct dc_panel_config *config, + struct dc_sink *sink); void dm_helpers_override_panel_settings( struct dc_context *ctx, struct dc_panel_config *config); diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index 86a3b5bfd699b2c9b5a15ea048be1b2fbe6f9990..d70838edba801a128d1e52ef97c2d7cce1610713 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -34,7 +34,7 @@ dml_ccflags := -mhard-float -maltivec endif ifdef CONFIG_CC_IS_GCC -ifeq ($(call cc-ifversion, -lt, 0701, y), y) +ifneq ($(call gcc-min-version, 70100),y) IS_OLD_GCC = 1 endif endif @@ -70,6 +70,8 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_mode_vba_30.o := $(dml_ccflags) $(fram CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/display_rq_dlg_calc_30.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_mode_vba_31.o := $(dml_ccflags) $(frame_warn_flag) CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/display_rq_dlg_calc_31.o := $(dml_ccflags) +CFLAGS_$(AMDDALPATH)/dc/dml/dcn314/display_mode_vba_314.o := $(dml_ccflags) $(frame_warn_flag) +CFLAGS_$(AMDDALPATH)/dc/dml/dcn314/display_rq_dlg_calc_314.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn314/dcn314_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/dcn30_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/dcn32_fpu.o := $(dml_ccflags) @@ -123,6 +125,7 @@ DML += dcn20/display_rq_dlg_calc_20v2.o dcn20/display_mode_vba_20v2.o DML += dcn21/display_rq_dlg_calc_21.o dcn21/display_mode_vba_21.o DML += dcn30/dcn30_fpu.o dcn30/display_mode_vba_30.o dcn30/display_rq_dlg_calc_30.o DML += dcn31/display_mode_vba_31.o dcn31/display_rq_dlg_calc_31.o +DML += dcn314/display_mode_vba_314.o dcn314/display_rq_dlg_calc_314.o DML += dcn32/display_mode_vba_32.o dcn32/display_rq_dlg_calc_32.o dcn32/display_mode_vba_util_32.o DML += dcn31/dcn31_fpu.o DML += dcn32/dcn32_fpu.o diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c index 41284e2633250449fd4b7cb916d6f2bff0d4af1a..288d22a16cf2541f7bb47d4b1beed152044b152e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c +++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c @@ -526,10 +526,10 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v) } if (v->max_swath_height_c[k] > 0.0) { v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->max_swath_height_c[k]; - } - v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k]; - if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) { - v->rounded_up_max_swath_size_bytes_c =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256; + v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k]; + if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) { + v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256; + } } if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) { v->swath_height_yper_state[i][j][k] = v->max_swath_height_y[k]; @@ -552,14 +552,14 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v) v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0 / 3.0 / v->byte_per_pixel_in_dety[k] / (v->swath_width_yper_state[i][j][k] / 2.0); } v->effective_lb_latency_hiding_source_lines_luma =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] /dcn_bw_max2(v->h_ratio[k], 1.0)), 1.0)) - (v->vtaps[k] - 1.0); - v->effective_lb_latency_hiding_source_lines_chroma =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0 /dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0); v->effective_detlb_lines_luma =dcn_bw_floor2(v->lines_in_det_luma +dcn_bw_min2(v->lines_in_det_luma * v->required_dispclk[i][j] * v->byte_per_pixel_in_dety[k] * v->pscl_factor[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_yper_state[i][j][k]); - v->effective_detlb_lines_chroma =dcn_bw_floor2(v->lines_in_det_chroma +dcn_bw_min2(v->lines_in_det_chroma * v->required_dispclk[i][j] * v->byte_per_pixel_in_detc[k] * v->pscl_factor_chroma[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_cper_state[i][j][k]); if (v->byte_per_pixel_in_detc[k] == 0.0) { v->urgent_latency_support_us_per_state[i][j][k] = v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]); } else { - v->urgent_latency_support_us_per_state[i][j][k] =dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0 *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k])); + v->effective_lb_latency_hiding_source_lines_chroma = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0 / dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0); + v->effective_detlb_lines_chroma = dcn_bw_floor2(v->lines_in_det_chroma + dcn_bw_min2(v->lines_in_det_chroma * v->required_dispclk[i][j] * v->byte_per_pixel_in_detc[k] * v->pscl_factor_chroma[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_cper_state[i][j][k]); + v->urgent_latency_support_us_per_state[i][j][k] = dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0 * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k])); } } } @@ -1146,10 +1146,10 @@ void display_pipe_configuration(struct dcn_bw_internal_vars *v) } if (v->maximum_swath_height_c > 0.0) { v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pix_detc, 2.0) / v->maximum_swath_height_c; - } - v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c; - if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) { - v->rounded_up_max_swath_size_bytes_c =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256; + v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c; + if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) { + v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256; + } } if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) { v->swath_height_y[k] = v->maximum_swath_height_y; diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c index 07d18e78de49d7ebee9d19c4210f7a12d716203b..cac72413a0977ec6f428e9a780bacc3db5492932 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c +++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c @@ -23,6 +23,7 @@ * */ +#include "os_types.h" #include "dcn_calc_math.h" #define isNaN(number) ((number) != (number)) @@ -69,8 +70,8 @@ float dcn_bw_max2(const float arg1, const float arg2) float dcn_bw_floor2(const float arg, const float significance) { - if (significance == 0) - return 0; + ASSERT(significance != 0); + return ((int) (arg / significance)) * significance; } float dcn_bw_floor(const float arg) @@ -80,17 +81,14 @@ float dcn_bw_floor(const float arg) float dcn_bw_ceil(const float arg) { - float flr = dcn_bw_floor2(arg, 1); - - return flr + 0.00001 >= arg ? arg : flr + 1; + return (int) (arg + 0.99999); } float dcn_bw_ceil2(const float arg, const float significance) { - float flr = dcn_bw_floor2(arg, significance); - if (significance == 0) - return 0; - return flr + 0.00001 >= arg ? arg : flr + significance; + ASSERT(significance != 0); + + return ((int) (arg / significance + 0.99999)) * significance; } float dcn_bw_max3(float v1, float v2, float v3) diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c index d46adc849d2aab799a9b1cd9f9446f31b1f3372a..e73f089c84bb6c7e519333bfa2e99170a090b6d0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c @@ -1444,81 +1444,67 @@ unsigned int dcn_find_dcfclk_suits_all( return dcf_clk; } -static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks) +void dcn_bw_update_from_pplib_fclks( + struct dc *dc, + struct dm_pp_clock_levels_with_voltage *fclks) { - int i; - - if (clks->num_levels == 0) - return false; - - for (i = 0; i < clks->num_levels; i++) - /* Ensure that the result is sane */ - if (clks->data[i].clocks_in_khz == 0) - return false; + unsigned vmin0p65_idx, vmid0p72_idx, vnom0p8_idx, vmax0p9_idx; - return true; + ASSERT(fclks->num_levels); + + vmin0p65_idx = 0; + vmid0p72_idx = fclks->num_levels - + (fclks->num_levels > 2 ? 3 : (fclks->num_levels > 1 ? 2 : 1)); + vnom0p8_idx = fclks->num_levels - (fclks->num_levels > 1 ? 2 : 1); + vmax0p9_idx = fclks->num_levels - 1; + + dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = + 32 * (fclks->data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = + dc->dcn_soc->number_of_channels * + (fclks->data[vmid0p72_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = + dc->dcn_soc->number_of_channels * + (fclks->data[vnom0p8_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = + dc->dcn_soc->number_of_channels * + (fclks->data[vmax0p9_idx].clocks_in_khz / 1000.0) + * ddr4_dram_factor_single_Channel / 1000.0; } -void dcn_bw_update_from_pplib(struct dc *dc) +void dcn_bw_update_from_pplib_dcfclks( + struct dc *dc, + struct dm_pp_clock_levels_with_voltage *dcfclks) { - struct dc_context *ctx = dc->ctx; - struct dm_pp_clock_levels_with_voltage fclks = {0}, dcfclks = {0}; - bool res; - unsigned vmin0p65_idx, vmid0p72_idx, vnom0p8_idx, vmax0p9_idx; - - /* TODO: This is not the proper way to obtain fabric_and_dram_bandwidth, should be min(fclk, memclk) */ - res = dm_pp_get_clock_levels_by_type_with_voltage( - ctx, DM_PP_CLOCK_TYPE_FCLK, &fclks); - - if (res) - res = verify_clock_values(&fclks); - - if (res) { - ASSERT(fclks.num_levels); - - vmin0p65_idx = 0; - vmid0p72_idx = fclks.num_levels - - (fclks.num_levels > 2 ? 3 : (fclks.num_levels > 1 ? 2 : 1)); - vnom0p8_idx = fclks.num_levels - (fclks.num_levels > 1 ? 2 : 1); - vmax0p9_idx = fclks.num_levels - 1; - - dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = - 32 * (fclks.data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = - dc->dcn_soc->number_of_channels * - (fclks.data[vmid0p72_idx].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = - dc->dcn_soc->number_of_channels * - (fclks.data[vnom0p8_idx].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = - dc->dcn_soc->number_of_channels * - (fclks.data[vmax0p9_idx].clocks_in_khz / 1000.0) - * ddr4_dram_factor_single_Channel / 1000.0; - } else - BREAK_TO_DEBUGGER(); - - res = dm_pp_get_clock_levels_by_type_with_voltage( - ctx, DM_PP_CLOCK_TYPE_DCFCLK, &dcfclks); - - if (res) - res = verify_clock_values(&dcfclks); + if (dcfclks->num_levels >= 3) { + dc->dcn_soc->dcfclkv_min0p65 = dcfclks->data[0].clocks_in_khz / 1000.0; + dc->dcn_soc->dcfclkv_mid0p72 = dcfclks->data[dcfclks->num_levels - 3].clocks_in_khz / 1000.0; + dc->dcn_soc->dcfclkv_nom0p8 = dcfclks->data[dcfclks->num_levels - 2].clocks_in_khz / 1000.0; + dc->dcn_soc->dcfclkv_max0p9 = dcfclks->data[dcfclks->num_levels - 1].clocks_in_khz / 1000.0; + } +} - if (res && dcfclks.num_levels >= 3) { - dc->dcn_soc->dcfclkv_min0p65 = dcfclks.data[0].clocks_in_khz / 1000.0; - dc->dcn_soc->dcfclkv_mid0p72 = dcfclks.data[dcfclks.num_levels - 3].clocks_in_khz / 1000.0; - dc->dcn_soc->dcfclkv_nom0p8 = dcfclks.data[dcfclks.num_levels - 2].clocks_in_khz / 1000.0; - dc->dcn_soc->dcfclkv_max0p9 = dcfclks.data[dcfclks.num_levels - 1].clocks_in_khz / 1000.0; - } else - BREAK_TO_DEBUGGER(); +void dcn_get_soc_clks( + struct dc *dc, + int *min_fclk_khz, + int *min_dcfclk_khz, + int *socclk_khz) +{ + *min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32; + *min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000; + *socclk_khz = dc->dcn_soc->socclk * 1000; } -void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc) +void dcn_bw_notify_pplib_of_wm_ranges( + struct dc *dc, + int min_fclk_khz, + int min_dcfclk_khz, + int socclk_khz) { struct pp_smu_funcs_rv *pp = NULL; struct pp_smu_wm_range_sets ranges = {0}; - int min_fclk_khz, min_dcfclk_khz, socclk_khz; const int overdrive = 5000000; /* 5 GHz to cover Overdrive */ if (dc->res_pool->pp_smu) @@ -1526,10 +1512,6 @@ void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc) if (!pp || !pp->set_wm_ranges) return; - min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32; - min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000; - socclk_khz = dc->dcn_soc->socclk * 1000; - /* Now notify PPLib/SMU about which Watermarks sets they should select * depending on DPM state they are in. And update BW MGR GFX Engine and * Memory clock member variables for Watermarks calculations for each diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index 63bbdf8b8678b4bbc2db99dbbf4d9eafc87c2f3c..edd098c7eb927c953077232cbddffd844c89a094 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -4478,17 +4478,17 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode locals->EffectiveLBLatencyHidingSourceLinesLuma), locals->SwathHeightYPerState[i][j][k]); - locals->EffectiveDETLBLinesChroma = dml_floor(locals->LinesInDETChroma + dml_min( - locals->LinesInDETChroma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETC[k] * - locals->PSCL_FACTOR_CHROMA[k] / locals->ReturnBWPerState[i][0], - locals->EffectiveLBLatencyHidingSourceLinesChroma), - locals->SwathHeightCPerState[i][j][k]); if (locals->BytePerPixelInDETC[k] == 0) { locals->UrgentLatencySupportUsPerState[i][j][k] = locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k]) / locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] * dml_ceil(locals->BytePerPixelInDETY[k], 1) / (locals->ReturnBWPerState[i][0] / locals->NoOfDPP[i][j][k]); } else { + locals->EffectiveDETLBLinesChroma = dml_floor(locals->LinesInDETChroma + dml_min( + locals->LinesInDETChroma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETC[k] * + locals->PSCL_FACTOR_CHROMA[k] / locals->ReturnBWPerState[i][0], + locals->EffectiveLBLatencyHidingSourceLinesChroma), + locals->SwathHeightCPerState[i][j][k]); locals->UrgentLatencySupportUsPerState[i][j][k] = dml_min( locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k]) / locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] * diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c index 8a7485e21d530db56b4d374ba7bbab56fbcd7784..1d84ae50311d9b265da9d5460ee0b03ae8d56d37 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c @@ -806,10 +806,12 @@ static bool CalculatePrefetchSchedule( if (myPipe->SourceScan == dm_horz) { *swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockWidth256BytesY) + myPipe->BlockWidth256BytesY; - *swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC; + if (myPipe->BlockWidth256BytesC > 0) + *swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC; } else { *swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockHeight256BytesY) + myPipe->BlockHeight256BytesY; - *swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC; + if (myPipe->BlockWidth256BytesC > 0) + *swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC; } prefetch_bw_oto = (PrefetchSourceLinesY * *swath_width_luma_ub * dml_ceil(BytePerPixelDETY, 1) + PrefetchSourceLinesC * *swath_width_chroma_ub * dml_ceil(BytePerPixelDETC, 2)) / Tsw_oto; @@ -2634,7 +2636,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman &mode_lib->vba.SrcActiveDrainRate, &mode_lib->vba.TInitXFill, &mode_lib->vba.TslvChk); - locals->XFCRemoteSurfaceFlipLatency[k] = + locals->XFCRemoteSurfaceFlipLatency[k] = dml_floor( mode_lib->vba.XFCRemoteSurfaceFlipDelay / (mode_lib->vba.HTotal[k] diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index b7fa003ffe061b89ab214b0ff5d79ca834ff3b1e..479e2c1a13018615d66003916767a7ebcae098f6 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -6322,10 +6322,6 @@ static void CalculateSwathWidth( for (k = 0; k < NumberOfActivePlanes; ++k) { enum odm_combine_mode MainPlaneODMCombine = 0; - surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]); - surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]); - surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); - surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); if (SourceScan[k] != dm_vert) { SwathWidthSingleDPPY[k] = ViewportWidth[k]; @@ -6365,8 +6361,6 @@ static void CalculateSwathWidth( surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]); surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]); - surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); - surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); if (SourceScan[k] != dm_vert) { MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k]; @@ -6374,6 +6368,7 @@ static void CalculateSwathWidth( swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (long) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]); if (BytePerPixC[k] > 0) { + surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); swath_width_chroma_ub[k] = dml_min(surface_width_ub_c, (long) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]); } else { @@ -6385,6 +6380,7 @@ static void CalculateSwathWidth( swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (long) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]); if (BytePerPixC[k] > 0) { + surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); swath_width_chroma_ub[k] = dml_min(surface_height_ub_c, (long) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]); } else { @@ -6497,7 +6493,7 @@ static double CalculateUrgentLatency( return ret; } -static void UseMinimumDCFCLK( +static noinline_for_stack void UseMinimumDCFCLK( struct display_mode_lib *mode_lib, struct vba_vars_st *v, int MaxPrefetchMode, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c index 241d28d0b7fb1005fa815611a5905705264592d1..422f17aefd4a6cc4695468d84781a15c30911fd6 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c @@ -379,6 +379,11 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param dcn3_01_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + if ((int)(dcn3_01_soc.dram_clock_change_latency_us * 1000) + != dc->debug.dram_clock_change_latency_ns + && dc->debug.dram_clock_change_latency_ns) { + dcn3_01_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000.0; + } dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30); } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c index 0e62eb823e343fd983eef1a5e814cf2517d9ec72..7dd0845d1bd9f6318df2f851bd9a21704bee1239 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c @@ -291,6 +291,8 @@ static struct _vcs_dpi_soc_bounding_box_st dcn3_15_soc = { .do_urgent_latency_adjustment = false, .urgent_latency_adjustment_fabric_clock_component_us = 0, .urgent_latency_adjustment_fabric_clock_reference_mhz = 0, + .num_chans = 4, + .dummy_pstate_latency_us = 10.0 }; struct _vcs_dpi_ip_params_st dcn3_16_ip = { @@ -458,13 +460,30 @@ void dcn31_update_soc_for_wm_a(struct dc *dc, struct dc_state *context) } } +void dcn315_update_soc_for_wm_a(struct dc *dc, struct dc_state *context) +{ + dc_assert_fp_enabled(); + + if (dc->clk_mgr->bw_params->wm_table.entries[WM_A].valid) { + /* For 315 pstate change is only supported if possible in vactive */ + if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[context->bw_ctx.dml.vba.VoltageLevel][context->bw_ctx.dml.vba.maxMpcComb] != dm_dram_clock_change_vactive) + context->bw_ctx.dml.soc.dram_clock_change_latency_us = context->bw_ctx.dml.soc.dummy_pstate_latency_us; + else + context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.entries[WM_A].pstate_latency_us; + context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = + dc->clk_mgr->bw_params->wm_table.entries[WM_A].sr_enter_plus_exit_time_us; + context->bw_ctx.dml.soc.sr_exit_time_us = + dc->clk_mgr->bw_params->wm_table.entries[WM_A].sr_exit_time_us; + } +} + void dcn31_calculate_wm_and_dlg_fp( struct dc *dc, struct dc_state *context, display_e2e_pipe_params_st *pipes, int pipe_cnt, int vlevel) { - int i, pipe_idx; + int i, pipe_idx, active_dpp_count = 0; double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb]; dc_assert_fp_enabled(); @@ -485,72 +504,6 @@ void dcn31_calculate_wm_and_dlg_fp( pipes[0].clks_cfg.dcfclk_mhz = dcfclk; pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz; -#if 0 // TODO - /* Set B: - * TODO - */ - if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].valid) { - if (vlevel == 0) { - pipes[0].clks_cfg.voltage = 1; - pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dcfclk_mhz; - } - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.pstate_latency_us; - context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_enter_plus_exit_time_us; - context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_exit_time_us; - } - context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - - pipes[0].clks_cfg.voltage = vlevel; - pipes[0].clks_cfg.dcfclk_mhz = dcfclk; - - /* Set C: - * TODO - */ - if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) { - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.pstate_latency_us; - context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us; - context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us; - } - context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - - /* Set D: - * TODO - */ - if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].valid) { - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us; - context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us; - context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us; - } - context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_z8_ns = get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - context->bw_ctx.bw.dcn.watermarks.d.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; -#endif - /* Set A: * All clocks min required * @@ -567,16 +520,17 @@ void dcn31_calculate_wm_and_dlg_fp( context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000; - /* TODO: remove: */ context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a; context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a; context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a; - /* end remove*/ for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) { if (!context->res_ctx.pipe_ctx[i].stream) continue; + if (context->res_ctx.pipe_ctx[i].plane_state) + active_dpp_count++; + pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt); pipes[pipe_idx].clks_cfg.dppclk_mhz = get_dppclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx); @@ -593,6 +547,9 @@ void dcn31_calculate_wm_and_dlg_fp( } dcn20_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel); + /* For 31x apu pstate change is only supported if possible in vactive or if there are no active dpps */ + context->bw_ctx.bw.dcn.clk.p_state_change_support = + context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_dram_clock_change_vactive || !active_dpp_count; } void dcn31_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) @@ -667,6 +624,12 @@ void dcn31_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params dcn3_1_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + if ((int)(dcn3_1_soc.dram_clock_change_latency_us * 1000) + != dc->debug.dram_clock_change_latency_ns + && dc->debug.dram_clock_change_latency_ns) { + dcn3_1_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000; + } + if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) dml_init_instance(&dc->dml, &dcn3_1_soc, &dcn3_1_ip, DML_PROJECT_DCN31); else @@ -682,7 +645,11 @@ void dcn315_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param dcn3_15_ip.max_num_otg = dc->res_pool->res_cap->num_timing_generator; dcn3_15_ip.max_num_dpp = dc->res_pool->pipe_count; - dcn3_15_soc.num_chans = bw_params->num_channels; + + if (bw_params->num_channels > 0) + dcn3_15_soc.num_chans = bw_params->num_channels; + if (bw_params->dram_channel_width_bytes > 0) + dcn3_15_soc.dram_channel_width_bytes = bw_params->dram_channel_width_bytes; ASSERT(clk_table->num_entries); @@ -721,8 +688,14 @@ void dcn315_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param */ dcn3_15_soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2; + if ((int)(dcn3_15_soc.dram_clock_change_latency_us * 1000) + != dc->debug.dram_clock_change_latency_ns + && dc->debug.dram_clock_change_latency_ns) { + dcn3_15_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000; + } + if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) - dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN31); + dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN315); else dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN31_FPGA); } @@ -813,6 +786,11 @@ void dcn316_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param dcn3_16_soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2; } + if ((int)(dcn3_16_soc.dram_clock_change_latency_us * 1000) + != dc->debug.dram_clock_change_latency_ns + && dc->debug.dram_clock_change_latency_ns) { + dcn3_16_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000; + } if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) dml_init_instance(&dc->dml, &dcn3_16_soc, &dcn3_16_ip, DML_PROJECT_DCN31); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h index 4372f17b55d4e79787dccc28b6e73b16490f0b99..fd58b2561ec9e28874d082c82582cda47181188f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.h @@ -35,6 +35,7 @@ void dcn31_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes, int pipe_cnt); void dcn31_update_soc_for_wm_a(struct dc *dc, struct dc_state *context); +void dcn315_update_soc_for_wm_a(struct dc *dc, struct dc_state *context); void dcn31_calculate_wm_and_dlg_fp( struct dc *dc, struct dc_state *context, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c index d63b4209b14c080538fb2905129e18354f163dba..b612edb144172130a86a0a93bba1421cf6750aa7 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c @@ -43,6 +43,8 @@ #define BPP_BLENDED_PIPE 0xffffffff #define DCN31_MAX_DSC_IMAGE_WIDTH 5184 #define DCN31_MAX_FMT_420_BUFFER_WIDTH 4096 +#define DCN3_15_MIN_COMPBUF_SIZE_KB 128 +#define DCN3_15_MAX_DET_SIZE 384 // For DML-C changes that hasn't been propagated to VBA yet //#define __DML_VBA_ALLOW_DELTA__ @@ -251,33 +253,13 @@ static void CalculateRowBandwidth( static void CalculateFlipSchedule( struct display_mode_lib *mode_lib, + unsigned int k, double HostVMInefficiencyFactor, double UrgentExtraLatency, double UrgentLatency, - unsigned int GPUVMMaxPageTableLevels, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - bool GPUVMEnable, - double HostVMMinPageSize, double PDEAndMetaPTEBytesPerFrame, double MetaRowBytes, - double DPTEBytesPerRow, - double BandwidthAvailableForImmediateFlip, - unsigned int TotImmediateFlipBytes, - enum source_format_class SourcePixelFormat, - double LineTime, - double VRatio, - double VRatioChroma, - double Tno_bw, - bool DCCEnable, - unsigned int dpte_row_height, - unsigned int meta_row_height, - unsigned int dpte_row_height_chroma, - unsigned int meta_row_height_chroma, - double *DestinationLinesToRequestVMInImmediateFlip, - double *DestinationLinesToRequestRowInImmediateFlip, - double *final_flip_bw, - bool *ImmediateFlipSupportedForPipe); + double DPTEBytesPerRow); static double CalculateWriteBackDelay( enum source_format_class WritebackPixelFormat, double WritebackHRatio, @@ -311,64 +293,28 @@ static void CalculateVupdateAndDynamicMetadataParameters( static void CalculateWatermarksAndDRAMSpeedChangeSupport( struct display_mode_lib *mode_lib, unsigned int PrefetchMode, - unsigned int NumberOfActivePlanes, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizedVBlank, - unsigned int dpte_group_bytes[], - unsigned int MetaChunkSize, double UrgentLatency, double ExtraLatency, - double WritebackLatency, - double WritebackChunkSize, double SOCCLK, - double DRAMClockChangeLatency, - double SRExitTime, - double SREnterPlusExitTime, - double SRExitZ8Time, - double SREnterPlusExitZ8Time, double DCFCLKDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int vtaps[], - unsigned int VTAPsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerPlane[], double BytePerPixelDETY[], double BytePerPixelDETC[], - double DSTXAfterScaler[], - double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, int unsigned CompressedBufferSizeInkByte, enum clock_change_support *DRAMClockChangeSupport, - double *UrgentWatermark, - double *WritebackUrgentWatermark, - double *DRAMClockChangeWatermark, - double *WritebackDRAMClockChangeWatermark, double *StutterExitWatermark, double *StutterEnterPlusExitWatermark, double *Z8StutterExitWatermark, - double *Z8StutterEnterPlusExitWatermark, - double *MinActiveDRAMClockChangeLatencySupported); + double *Z8StutterEnterPlusExitWatermark); static void CalculateDCFCLKDeepSleep( struct display_mode_lib *mode_lib, @@ -1107,10 +1053,10 @@ static bool CalculatePrefetchSchedule( bytes_pp = myPipe->BytePerPixelY + myPipe->BytePerPixelC; /*rev 99*/ prefetch_bw_pr = dml_min(1, bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane); - max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime; + max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime; prefetch_sw_bytes = PrefetchSourceLinesY * swath_width_luma_ub * myPipe->BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * myPipe->BytePerPixelC; prefetch_bw_oto = dml_max(bytes_pp * myPipe->PixelClock / myPipe->DPPPerPlane, prefetch_sw_bytes / (dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime)); - prefetch_bw_oto = dml_max(prefetch_bw_pr, prefetch_sw_bytes / max_Tsw); + prefetch_bw_oto = dml_max(prefetch_bw_pr, prefetch_sw_bytes / max_Tsw); min_Lsw = dml_max(1, dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) / max_vratio_pre); Lsw_oto = dml_ceil(4 * dml_max(prefetch_sw_bytes / prefetch_bw_oto / LineTime, min_Lsw), 1) / 4; @@ -2904,33 +2850,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman for (k = 0; k < v->NumberOfActivePlanes; ++k) { CalculateFlipSchedule( mode_lib, + k, HostVMInefficiencyFactor, v->UrgentExtraLatency, v->UrgentLatency, - v->GPUVMMaxPageTableLevels, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->GPUVMEnable, - v->HostVMMinPageSize, v->PDEAndMetaPTEBytesFrame[k], v->MetaRowByte[k], - v->PixelPTEBytesPerRow[k], - v->BandwidthAvailableForImmediateFlip, - v->TotImmediateFlipBytes, - v->SourcePixelFormat[k], - v->HTotal[k] / v->PixelClock[k], - v->VRatio[k], - v->VRatioChroma[k], - v->Tno_bw[k], - v->DCCEnable[k], - v->dpte_row_height[k], - v->meta_row_height[k], - v->dpte_row_height_chroma[k], - v->meta_row_height_chroma[k], - &v->DestinationLinesToRequestVMInImmediateFlip[k], - &v->DestinationLinesToRequestRowInImmediateFlip[k], - &v->final_flip_bw[k], - &v->ImmediateFlipSupportedForPipe[k]); + v->PixelPTEBytesPerRow[k]); } v->total_dcn_read_bw_with_flip = 0.0; @@ -3017,64 +2943,28 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman CalculateWatermarksAndDRAMSpeedChangeSupport( mode_lib, PrefetchMode, - v->NumberOfActivePlanes, - v->MaxLineBufferLines, - v->LineBufferSize, - v->WritebackInterfaceBufferSize, v->DCFCLK, v->ReturnBW, - v->SynchronizedVBlank, - v->dpte_group_bytes, - v->MetaChunkSize, v->UrgentLatency, v->UrgentExtraLatency, - v->WritebackLatency, - v->WritebackChunkSize, v->SOCCLK, - v->DRAMClockChangeLatency, - v->SRExitTime, - v->SREnterPlusExitTime, - v->SRExitZ8Time, - v->SREnterPlusExitZ8Time, v->DCFCLKDeepSleep, v->DETBufferSizeY, v->DETBufferSizeC, v->SwathHeightY, v->SwathHeightC, - v->LBBitPerPixel, v->SwathWidthY, v->SwathWidthC, - v->HRatio, - v->HRatioChroma, - v->vtaps, - v->VTAPsChroma, - v->VRatio, - v->VRatioChroma, - v->HTotal, - v->PixelClock, - v->BlendingAndTiming, v->DPPPerPlane, v->BytePerPixelDETY, v->BytePerPixelDETC, - v->DSTXAfterScaler, - v->DSTYAfterScaler, - v->WritebackEnable, - v->WritebackPixelFormat, - v->WritebackDestinationWidth, - v->WritebackDestinationHeight, - v->WritebackSourceHeight, v->UnboundedRequestEnabled, v->CompressedBufferSizeInkByte, &DRAMClockChangeSupport, - &v->UrgentWatermark, - &v->WritebackUrgentWatermark, - &v->DRAMClockChangeWatermark, - &v->WritebackDRAMClockChangeWatermark, &v->StutterExitWatermark, &v->StutterEnterPlusExitWatermark, &v->Z8StutterExitWatermark, - &v->Z8StutterEnterPlusExitWatermark, - &v->MinActiveDRAMClockChangeLatencySupported); + &v->Z8StutterEnterPlusExitWatermark); for (k = 0; k < v->NumberOfActivePlanes; ++k) { if (v->WritebackEnable[k] == true) { @@ -3598,61 +3488,43 @@ static void CalculateRowBandwidth( static void CalculateFlipSchedule( struct display_mode_lib *mode_lib, + unsigned int k, double HostVMInefficiencyFactor, double UrgentExtraLatency, double UrgentLatency, - unsigned int GPUVMMaxPageTableLevels, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - bool GPUVMEnable, - double HostVMMinPageSize, double PDEAndMetaPTEBytesPerFrame, double MetaRowBytes, - double DPTEBytesPerRow, - double BandwidthAvailableForImmediateFlip, - unsigned int TotImmediateFlipBytes, - enum source_format_class SourcePixelFormat, - double LineTime, - double VRatio, - double VRatioChroma, - double Tno_bw, - bool DCCEnable, - unsigned int dpte_row_height, - unsigned int meta_row_height, - unsigned int dpte_row_height_chroma, - unsigned int meta_row_height_chroma, - double *DestinationLinesToRequestVMInImmediateFlip, - double *DestinationLinesToRequestRowInImmediateFlip, - double *final_flip_bw, - bool *ImmediateFlipSupportedForPipe) + double DPTEBytesPerRow) { + struct vba_vars_st *v = &mode_lib->vba; double min_row_time = 0.0; unsigned int HostVMDynamicLevelsTrips; double TimeForFetchingMetaPTEImmediateFlip; double TimeForFetchingRowInVBlankImmediateFlip; double ImmediateFlipBW; + double LineTime = v->HTotal[k] / v->PixelClock[k]; - if (GPUVMEnable == true && HostVMEnable == true) { - HostVMDynamicLevelsTrips = HostVMMaxNonCachedPageTableLevels; + if (v->GPUVMEnable == true && v->HostVMEnable == true) { + HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; } else { HostVMDynamicLevelsTrips = 0; } - if (GPUVMEnable == true || DCCEnable == true) { - ImmediateFlipBW = (PDEAndMetaPTEBytesPerFrame + MetaRowBytes + DPTEBytesPerRow) * BandwidthAvailableForImmediateFlip / TotImmediateFlipBytes; + if (v->GPUVMEnable == true || v->DCCEnable[k] == true) { + ImmediateFlipBW = (PDEAndMetaPTEBytesPerFrame + MetaRowBytes + DPTEBytesPerRow) * v->BandwidthAvailableForImmediateFlip / v->TotImmediateFlipBytes; } - if (GPUVMEnable == true) { + if (v->GPUVMEnable == true) { TimeForFetchingMetaPTEImmediateFlip = dml_max3( - Tno_bw + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / ImmediateFlipBW, - UrgentExtraLatency + UrgentLatency * (GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1), + v->Tno_bw[k] + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / ImmediateFlipBW, + UrgentExtraLatency + UrgentLatency * (v->GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1), LineTime / 4.0); } else { TimeForFetchingMetaPTEImmediateFlip = 0; } - *DestinationLinesToRequestVMInImmediateFlip = dml_ceil(4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime), 1) / 4.0; - if ((GPUVMEnable == true || DCCEnable == true)) { + v->DestinationLinesToRequestVMInImmediateFlip[k] = dml_ceil(4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime), 1) / 4.0; + if ((v->GPUVMEnable == true || v->DCCEnable[k] == true)) { TimeForFetchingRowInVBlankImmediateFlip = dml_max3( (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / ImmediateFlipBW, UrgentLatency * (HostVMDynamicLevelsTrips + 1), @@ -3661,54 +3533,54 @@ static void CalculateFlipSchedule( TimeForFetchingRowInVBlankImmediateFlip = 0; } - *DestinationLinesToRequestRowInImmediateFlip = dml_ceil(4.0 * (TimeForFetchingRowInVBlankImmediateFlip / LineTime), 1) / 4.0; + v->DestinationLinesToRequestRowInImmediateFlip[k] = dml_ceil(4.0 * (TimeForFetchingRowInVBlankImmediateFlip / LineTime), 1) / 4.0; - if (GPUVMEnable == true) { - *final_flip_bw = dml_max( - PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / (*DestinationLinesToRequestVMInImmediateFlip * LineTime), - (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (*DestinationLinesToRequestRowInImmediateFlip * LineTime)); - } else if ((GPUVMEnable == true || DCCEnable == true)) { - *final_flip_bw = (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (*DestinationLinesToRequestRowInImmediateFlip * LineTime); + if (v->GPUVMEnable == true) { + v->final_flip_bw[k] = dml_max( + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / (v->DestinationLinesToRequestVMInImmediateFlip[k] * LineTime), + (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (v->DestinationLinesToRequestRowInImmediateFlip[k] * LineTime)); + } else if ((v->GPUVMEnable == true || v->DCCEnable[k] == true)) { + v->final_flip_bw[k] = (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (v->DestinationLinesToRequestRowInImmediateFlip[k] * LineTime); } else { - *final_flip_bw = 0; + v->final_flip_bw[k] = 0; } - if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10 || SourcePixelFormat == dm_rgbe_alpha) { - if (GPUVMEnable == true && DCCEnable != true) { - min_row_time = dml_min(dpte_row_height * LineTime / VRatio, dpte_row_height_chroma * LineTime / VRatioChroma); - } else if (GPUVMEnable != true && DCCEnable == true) { - min_row_time = dml_min(meta_row_height * LineTime / VRatio, meta_row_height_chroma * LineTime / VRatioChroma); + if (v->SourcePixelFormat[k] == dm_420_8 || v->SourcePixelFormat[k] == dm_420_10 || v->SourcePixelFormat[k] == dm_rgbe_alpha) { + if (v->GPUVMEnable == true && v->DCCEnable[k] != true) { + min_row_time = dml_min(v->dpte_row_height[k] * LineTime / v->VRatio[k], v->dpte_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); + } else if (v->GPUVMEnable != true && v->DCCEnable[k] == true) { + min_row_time = dml_min(v->meta_row_height[k] * LineTime / v->VRatio[k], v->meta_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); } else { min_row_time = dml_min4( - dpte_row_height * LineTime / VRatio, - meta_row_height * LineTime / VRatio, - dpte_row_height_chroma * LineTime / VRatioChroma, - meta_row_height_chroma * LineTime / VRatioChroma); + v->dpte_row_height[k] * LineTime / v->VRatio[k], + v->meta_row_height[k] * LineTime / v->VRatio[k], + v->dpte_row_height_chroma[k] * LineTime / v->VRatioChroma[k], + v->meta_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); } } else { - if (GPUVMEnable == true && DCCEnable != true) { - min_row_time = dpte_row_height * LineTime / VRatio; - } else if (GPUVMEnable != true && DCCEnable == true) { - min_row_time = meta_row_height * LineTime / VRatio; + if (v->GPUVMEnable == true && v->DCCEnable[k] != true) { + min_row_time = v->dpte_row_height[k] * LineTime / v->VRatio[k]; + } else if (v->GPUVMEnable != true && v->DCCEnable[k] == true) { + min_row_time = v->meta_row_height[k] * LineTime / v->VRatio[k]; } else { - min_row_time = dml_min(dpte_row_height * LineTime / VRatio, meta_row_height * LineTime / VRatio); + min_row_time = dml_min(v->dpte_row_height[k] * LineTime / v->VRatio[k], v->meta_row_height[k] * LineTime / v->VRatio[k]); } } - if (*DestinationLinesToRequestVMInImmediateFlip >= 32 || *DestinationLinesToRequestRowInImmediateFlip >= 16 + if (v->DestinationLinesToRequestVMInImmediateFlip[k] >= 32 || v->DestinationLinesToRequestRowInImmediateFlip[k] >= 16 || TimeForFetchingMetaPTEImmediateFlip + 2 * TimeForFetchingRowInVBlankImmediateFlip > min_row_time) { - *ImmediateFlipSupportedForPipe = false; + v->ImmediateFlipSupportedForPipe[k] = false; } else { - *ImmediateFlipSupportedForPipe = true; + v->ImmediateFlipSupportedForPipe[k] = true; } #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: DestinationLinesToRequestVMInImmediateFlip = %f\n", __func__, *DestinationLinesToRequestVMInImmediateFlip); - dml_print("DML::%s: DestinationLinesToRequestRowInImmediateFlip = %f\n", __func__, *DestinationLinesToRequestRowInImmediateFlip); + dml_print("DML::%s: DestinationLinesToRequestVMInImmediateFlip = %f\n", __func__, v->DestinationLinesToRequestVMInImmediateFlip[k]); + dml_print("DML::%s: DestinationLinesToRequestRowInImmediateFlip = %f\n", __func__, v->DestinationLinesToRequestRowInImmediateFlip[k]); dml_print("DML::%s: TimeForFetchingMetaPTEImmediateFlip = %f\n", __func__, TimeForFetchingMetaPTEImmediateFlip); dml_print("DML::%s: TimeForFetchingRowInVBlankImmediateFlip = %f\n", __func__, TimeForFetchingRowInVBlankImmediateFlip); dml_print("DML::%s: min_row_time = %f\n", __func__, min_row_time); - dml_print("DML::%s: ImmediateFlipSupportedForPipe = %d\n", __func__, *ImmediateFlipSupportedForPipe); + dml_print("DML::%s: ImmediateFlipSupportedForPipe = %d\n", __func__, v->ImmediateFlipSupportedForPipe[k]); #endif } @@ -3905,6 +3777,17 @@ static noinline void CalculatePrefetchSchedulePerPlane( &v->VReadyOffsetPix[k]); } +static void PatchDETBufferSizeInKByte(unsigned int NumberOfActivePlanes, int NoOfDPPThisState[], unsigned int config_return_buffer_size_in_kbytes, unsigned int *DETBufferSizeInKByte) +{ + int i, total_pipes = 0; + for (i = 0; i < NumberOfActivePlanes; i++) + total_pipes += NoOfDPPThisState[i]; + *DETBufferSizeInKByte = ((config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB) / 64 / total_pipes) * 64; + if (*DETBufferSizeInKByte > DCN3_15_MAX_DET_SIZE) + *DETBufferSizeInKByte = DCN3_15_MAX_DET_SIZE; +} + + void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib) { struct vba_vars_st *v = &mode_lib->vba; @@ -4663,6 +4546,8 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->ODMCombineEnableThisState[k] = v->ODMCombineEnablePerState[i][k]; } + if (v->NumberOfActivePlanes > 1 && mode_lib->project == DML_PROJECT_DCN315) + PatchDETBufferSizeInKByte(v->NumberOfActivePlanes, v->NoOfDPPThisState, v->ip.config_return_buffer_size_in_kbytes, &v->DETBufferSizeInKByte[0]); CalculateSwathAndDETConfiguration( false, v->NumberOfActivePlanes, @@ -5300,33 +5185,13 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (k = 0; k < v->NumberOfActivePlanes; k++) { CalculateFlipSchedule( mode_lib, + k, HostVMInefficiencyFactor, v->ExtraLatency, v->UrgLatency[i], - v->GPUVMMaxPageTableLevels, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->GPUVMEnable, - v->HostVMMinPageSize, v->PDEAndMetaPTEBytesPerFrame[i][j][k], v->MetaRowBytes[i][j][k], - v->DPTEBytesPerRow[i][j][k], - v->BandwidthAvailableForImmediateFlip, - v->TotImmediateFlipBytes, - v->SourcePixelFormat[k], - v->HTotal[k] / v->PixelClock[k], - v->VRatio[k], - v->VRatioChroma[k], - v->Tno_bw[k], - v->DCCEnable[k], - v->dpte_row_height[k], - v->meta_row_height[k], - v->dpte_row_height_chroma[k], - v->meta_row_height_chroma[k], - &v->DestinationLinesToRequestVMInImmediateFlip[k], - &v->DestinationLinesToRequestRowInImmediateFlip[k], - &v->final_flip_bw[k], - &v->ImmediateFlipSupportedForPipe[k]); + v->DPTEBytesPerRow[i][j][k]); } v->total_dcn_read_bw_with_flip = 0.0; for (k = 0; k < v->NumberOfActivePlanes; k++) { @@ -5384,64 +5249,28 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l CalculateWatermarksAndDRAMSpeedChangeSupport( mode_lib, v->PrefetchModePerState[i][j], - v->NumberOfActivePlanes, - v->MaxLineBufferLines, - v->LineBufferSize, - v->WritebackInterfaceBufferSize, v->DCFCLKState[i][j], v->ReturnBWPerState[i][j], - v->SynchronizedVBlank, - v->dpte_group_bytes, - v->MetaChunkSize, v->UrgLatency[i], v->ExtraLatency, - v->WritebackLatency, - v->WritebackChunkSize, v->SOCCLKPerState[i], - v->DRAMClockChangeLatency, - v->SRExitTime, - v->SREnterPlusExitTime, - v->SRExitZ8Time, - v->SREnterPlusExitZ8Time, v->ProjectedDCFCLKDeepSleep[i][j], v->DETBufferSizeYThisState, v->DETBufferSizeCThisState, v->SwathHeightYThisState, v->SwathHeightCThisState, - v->LBBitPerPixel, v->SwathWidthYThisState, v->SwathWidthCThisState, - v->HRatio, - v->HRatioChroma, - v->vtaps, - v->VTAPsChroma, - v->VRatio, - v->VRatioChroma, - v->HTotal, - v->PixelClock, - v->BlendingAndTiming, v->NoOfDPPThisState, v->BytePerPixelInDETY, v->BytePerPixelInDETC, - v->DSTXAfterScaler, - v->DSTYAfterScaler, - v->WritebackEnable, - v->WritebackPixelFormat, - v->WritebackDestinationWidth, - v->WritebackDestinationHeight, - v->WritebackSourceHeight, UnboundedRequestEnabledThisState, CompressedBufferSizeInkByteThisState, &v->DRAMClockChangeSupport[i][j], - &v->UrgentWatermark, - &v->WritebackUrgentWatermark, - &v->DRAMClockChangeWatermark, - &v->WritebackDRAMClockChangeWatermark, &dummy, &dummy, &dummy, - &dummy, - &v->MinActiveDRAMClockChangeLatencySupported); + &dummy); } } @@ -5566,64 +5395,28 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l static void CalculateWatermarksAndDRAMSpeedChangeSupport( struct display_mode_lib *mode_lib, unsigned int PrefetchMode, - unsigned int NumberOfActivePlanes, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizedVBlank, - unsigned int dpte_group_bytes[], - unsigned int MetaChunkSize, double UrgentLatency, double ExtraLatency, - double WritebackLatency, - double WritebackChunkSize, double SOCCLK, - double DRAMClockChangeLatency, - double SRExitTime, - double SREnterPlusExitTime, - double SRExitZ8Time, - double SREnterPlusExitZ8Time, double DCFCLKDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int vtaps[], - unsigned int VTAPsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerPlane[], double BytePerPixelDETY[], double BytePerPixelDETC[], - double DSTXAfterScaler[], - double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, int unsigned CompressedBufferSizeInkByte, enum clock_change_support *DRAMClockChangeSupport, - double *UrgentWatermark, - double *WritebackUrgentWatermark, - double *DRAMClockChangeWatermark, - double *WritebackDRAMClockChangeWatermark, double *StutterExitWatermark, double *StutterEnterPlusExitWatermark, double *Z8StutterExitWatermark, - double *Z8StutterEnterPlusExitWatermark, - double *MinActiveDRAMClockChangeLatencySupported) + double *Z8StutterEnterPlusExitWatermark) { struct vba_vars_st *v = &mode_lib->vba; double EffectiveLBLatencyHidingY; @@ -5643,103 +5436,103 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( double TotalPixelBW = 0.0; int k, j; - *UrgentWatermark = UrgentLatency + ExtraLatency; + v->UrgentWatermark = UrgentLatency + ExtraLatency; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: UrgentLatency = %f\n", __func__, UrgentLatency); dml_print("DML::%s: ExtraLatency = %f\n", __func__, ExtraLatency); - dml_print("DML::%s: UrgentWatermark = %f\n", __func__, *UrgentWatermark); + dml_print("DML::%s: UrgentWatermark = %f\n", __func__, v->UrgentWatermark); #endif - *DRAMClockChangeWatermark = DRAMClockChangeLatency + *UrgentWatermark; + v->DRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->UrgentWatermark; #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: DRAMClockChangeLatency = %f\n", __func__, DRAMClockChangeLatency); - dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, *DRAMClockChangeWatermark); + dml_print("DML::%s: v->DRAMClockChangeLatency = %f\n", __func__, v->DRAMClockChangeLatency); + dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, v->DRAMClockChangeWatermark); #endif v->TotalActiveWriteback = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (WritebackEnable[k] == true) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (v->WritebackEnable[k] == true) { v->TotalActiveWriteback = v->TotalActiveWriteback + 1; } } if (v->TotalActiveWriteback <= 1) { - *WritebackUrgentWatermark = WritebackLatency; + v->WritebackUrgentWatermark = v->WritebackLatency; } else { - *WritebackUrgentWatermark = WritebackLatency + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->WritebackUrgentWatermark = v->WritebackLatency + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; } if (v->TotalActiveWriteback <= 1) { - *WritebackDRAMClockChangeWatermark = DRAMClockChangeLatency + WritebackLatency; + v->WritebackDRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->WritebackLatency; } else { - *WritebackDRAMClockChangeWatermark = DRAMClockChangeLatency + WritebackLatency + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->WritebackDRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->WritebackLatency + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; } - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { TotalPixelBW = TotalPixelBW - + DPPPerPlane[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k] + SwathWidthC[k] * BytePerPixelDETC[k] * VRatioChroma[k]) - / (HTotal[k] / PixelClock[k]); + + DPPPerPlane[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k] + SwathWidthC[k] * BytePerPixelDETC[k] * v->VRatioChroma[k]) + / (v->HTotal[k] / v->PixelClock[k]); } - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { double EffectiveDETBufferSizeY = DETBufferSizeY[k]; v->LBLatencyHidingSourceLinesY = dml_min( - (double) MaxLineBufferLines, - dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(HRatio[k], 1.0)), 1)) - (vtaps[k] - 1); + (double) v->MaxLineBufferLines, + dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1); v->LBLatencyHidingSourceLinesC = dml_min( - (double) MaxLineBufferLines, - dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(HRatioChroma[k], 1.0)), 1)) - (VTAPsChroma[k] - 1); + (double) v->MaxLineBufferLines, + dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1); - EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / VRatio[k] * (HTotal[k] / PixelClock[k]); + EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]); - EffectiveLBLatencyHidingC = v->LBLatencyHidingSourceLinesC / VRatioChroma[k] * (HTotal[k] / PixelClock[k]); + EffectiveLBLatencyHidingC = v->LBLatencyHidingSourceLinesC / v->VRatioChroma[k] * (v->HTotal[k] / v->PixelClock[k]); if (UnboundedRequestEnabled) { EffectiveDETBufferSizeY = EffectiveDETBufferSizeY - + CompressedBufferSizeInkByte * 1024 * SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k] / (HTotal[k] / PixelClock[k]) / TotalPixelBW; + + CompressedBufferSizeInkByte * 1024 * SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k] / (v->HTotal[k] / v->PixelClock[k]) / TotalPixelBW; } LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k]; LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]); - FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (HTotal[k] / PixelClock[k]) / VRatio[k]; + FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k]; if (BytePerPixelDETC[k] > 0) { LinesInDETC = v->DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k]; LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]); - FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (HTotal[k] / PixelClock[k]) / VRatioChroma[k]; + FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (v->HTotal[k] / v->PixelClock[k]) / v->VRatioChroma[k]; } else { LinesInDETC = 0; FullDETBufferingTimeC = 999999; } ActiveDRAMClockChangeLatencyMarginY = EffectiveLBLatencyHidingY + FullDETBufferingTimeY - - ((double) DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] / PixelClock[k] - *UrgentWatermark - *DRAMClockChangeWatermark; + - ((double) v->DSTXAfterScaler[k] / v->HTotal[k] + v->DSTYAfterScaler[k]) * v->HTotal[k] / v->PixelClock[k] - v->UrgentWatermark - v->DRAMClockChangeWatermark; - if (NumberOfActivePlanes > 1) { + if (v->NumberOfActivePlanes > 1) { ActiveDRAMClockChangeLatencyMarginY = ActiveDRAMClockChangeLatencyMarginY - - (1 - 1.0 / NumberOfActivePlanes) * SwathHeightY[k] * HTotal[k] / PixelClock[k] / VRatio[k]; + - (1 - 1.0 / v->NumberOfActivePlanes) * SwathHeightY[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatio[k]; } if (BytePerPixelDETC[k] > 0) { ActiveDRAMClockChangeLatencyMarginC = EffectiveLBLatencyHidingC + FullDETBufferingTimeC - - ((double) DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] / PixelClock[k] - *UrgentWatermark - *DRAMClockChangeWatermark; + - ((double) v->DSTXAfterScaler[k] / v->HTotal[k] + v->DSTYAfterScaler[k]) * v->HTotal[k] / v->PixelClock[k] - v->UrgentWatermark - v->DRAMClockChangeWatermark; - if (NumberOfActivePlanes > 1) { + if (v->NumberOfActivePlanes > 1) { ActiveDRAMClockChangeLatencyMarginC = ActiveDRAMClockChangeLatencyMarginC - - (1 - 1.0 / NumberOfActivePlanes) * SwathHeightC[k] * HTotal[k] / PixelClock[k] / VRatioChroma[k]; + - (1 - 1.0 / v->NumberOfActivePlanes) * SwathHeightC[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatioChroma[k]; } v->ActiveDRAMClockChangeLatencyMargin[k] = dml_min(ActiveDRAMClockChangeLatencyMarginY, ActiveDRAMClockChangeLatencyMarginC); } else { v->ActiveDRAMClockChangeLatencyMargin[k] = ActiveDRAMClockChangeLatencyMarginY; } - if (WritebackEnable[k] == true) { - WritebackDRAMClockChangeLatencyHiding = WritebackInterfaceBufferSize * 1024 - / (WritebackDestinationWidth[k] * WritebackDestinationHeight[k] / (WritebackSourceHeight[k] * HTotal[k] / PixelClock[k]) * 4); - if (WritebackPixelFormat[k] == dm_444_64) { + if (v->WritebackEnable[k] == true) { + WritebackDRAMClockChangeLatencyHiding = v->WritebackInterfaceBufferSize * 1024 + / (v->WritebackDestinationWidth[k] * v->WritebackDestinationHeight[k] / (v->WritebackSourceHeight[k] * v->HTotal[k] / v->PixelClock[k]) * 4); + if (v->WritebackPixelFormat[k] == dm_444_64) { WritebackDRAMClockChangeLatencyHiding = WritebackDRAMClockChangeLatencyHiding / 2; } WritebackDRAMClockChangeLatencyMargin = WritebackDRAMClockChangeLatencyHiding - v->WritebackDRAMClockChangeWatermark; @@ -5749,14 +5542,14 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( v->MinActiveDRAMClockChangeMargin = 999999; PlaneWithMinActiveDRAMClockChangeMargin = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { if (v->ActiveDRAMClockChangeLatencyMargin[k] < v->MinActiveDRAMClockChangeMargin) { v->MinActiveDRAMClockChangeMargin = v->ActiveDRAMClockChangeLatencyMargin[k]; - if (BlendingAndTiming[k] == k) { + if (v->BlendingAndTiming[k] == k) { PlaneWithMinActiveDRAMClockChangeMargin = k; } else { - for (j = 0; j < NumberOfActivePlanes; ++j) { - if (BlendingAndTiming[k] == j) { + for (j = 0; j < v->NumberOfActivePlanes; ++j) { + if (v->BlendingAndTiming[k] == j) { PlaneWithMinActiveDRAMClockChangeMargin = j; } } @@ -5764,11 +5557,11 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( } } - *MinActiveDRAMClockChangeLatencySupported = v->MinActiveDRAMClockChangeMargin + DRAMClockChangeLatency; + v->MinActiveDRAMClockChangeLatencySupported = v->MinActiveDRAMClockChangeMargin + v->DRAMClockChangeLatency ; SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank = 999999; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (!((k == PlaneWithMinActiveDRAMClockChangeMargin) && (BlendingAndTiming[k] == k)) && !(BlendingAndTiming[k] == PlaneWithMinActiveDRAMClockChangeMargin) + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (!((k == PlaneWithMinActiveDRAMClockChangeMargin) && (v->BlendingAndTiming[k] == k)) && !(v->BlendingAndTiming[k] == PlaneWithMinActiveDRAMClockChangeMargin) && v->ActiveDRAMClockChangeLatencyMargin[k] < SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank) { SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank = v->ActiveDRAMClockChangeLatencyMargin[k]; } @@ -5776,25 +5569,25 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( v->TotalNumberOfActiveOTG = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (BlendingAndTiming[k] == k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (v->BlendingAndTiming[k] == k) { v->TotalNumberOfActiveOTG = v->TotalNumberOfActiveOTG + 1; } } if (v->MinActiveDRAMClockChangeMargin > 0 && PrefetchMode == 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vactive; - } else if ((SynchronizedVBlank == true || v->TotalNumberOfActiveOTG == 1 + } else if ((v->SynchronizedVBlank == true || v->TotalNumberOfActiveOTG == 1 || SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank > 0) && PrefetchMode == 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vblank; } else { *DRAMClockChangeSupport = dm_dram_clock_change_unsupported; } - *StutterExitWatermark = SRExitTime + ExtraLatency + 10 / DCFCLKDeepSleep; - *StutterEnterPlusExitWatermark = (SREnterPlusExitTime + ExtraLatency + 10 / DCFCLKDeepSleep); - *Z8StutterExitWatermark = SRExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; - *Z8StutterEnterPlusExitWatermark = SREnterPlusExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; + *StutterExitWatermark = v->SRExitTime + ExtraLatency + 10 / DCFCLKDeepSleep; + *StutterEnterPlusExitWatermark = (v->SREnterPlusExitTime + ExtraLatency + 10 / DCFCLKDeepSleep); + *Z8StutterExitWatermark = v->SRExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; + *Z8StutterEnterPlusExitWatermark = v->SREnterPlusExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: StutterExitWatermark = %f\n", __func__, *StutterExitWatermark); @@ -6933,8 +6726,6 @@ static void CalculateSwathWidth( { int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]); int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]); - int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); - int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l); @@ -6945,6 +6736,8 @@ static void CalculateSwathWidth( MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k]; swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]); if (BytePerPixC[k] > 0) { + int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); + swath_width_chroma_ub[k] = dml_min( surface_width_ub_c, (int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]); @@ -6956,6 +6749,8 @@ static void CalculateSwathWidth( MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k]; swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]); if (BytePerPixC[k] > 0) { + int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); + swath_width_chroma_ub[k] = dml_min( surface_height_ub_c, (int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c index 34a5d0f87b5f9e4dce39ba14b04b5a3425e2ef47..cf420ad2b8dcd577b2e1fb28c33f8dcc7b9453d2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c @@ -194,6 +194,9 @@ void dcn314_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p dcn3_14_ip.max_num_otg = dc->res_pool->res_cap->num_timing_generator; dcn3_14_ip.max_num_dpp = dc->res_pool->pipe_count; + if (bw_params->dram_channel_width_bytes > 0) + dcn3_14_soc.dram_channel_width_bytes = bw_params->dram_channel_width_bytes; + if (bw_params->num_channels > 0) dcn3_14_soc.num_chans = bw_params->num_channels; @@ -261,8 +264,13 @@ void dcn314_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p dc->dml.soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2; } + if ((int)(dcn3_14_soc.dram_clock_change_latency_us * 1000) + != dc->debug.dram_clock_change_latency_ns + && dc->debug.dram_clock_change_latency_ns) { + dcn3_14_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000; + } if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) - dml_init_instance(&dc->dml, &dcn3_14_soc, &dcn3_14_ip, DML_PROJECT_DCN31); + dml_init_instance(&dc->dml, &dcn3_14_soc, &dcn3_14_ip, DML_PROJECT_DCN314); else dml_init_instance(&dc->dml, &dcn3_14_soc, &dcn3_14_ip, DML_PROJECT_DCN31_FPGA); } @@ -315,6 +323,8 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0; pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0; pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch; + pipes[pipe_cnt].pipe.dest.vblank_nom = + dcn3_14_ip.VBlankNomDefaultUS / (timing->h_total / (timing->pix_clk_100hz / 10000.0)); pipes[pipe_cnt].pipe.src.dcc_rate = 3; pipes[pipe_cnt].dout.dsc_input_bpc = 0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c index fc4d7474c111e056552444220bf7378d429ba1b3..0d12fd079cd6180df439887be39b9c7676db264f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c @@ -61,7 +61,7 @@ // fudge factor for min dcfclk calclation #define __DML_MIN_DCFCLK_FACTOR__ 1.15 -struct { +typedef struct { double DPPCLK; double DISPCLK; double PixelClock; @@ -265,33 +265,13 @@ static void CalculateRowBandwidth( static void CalculateFlipSchedule( struct display_mode_lib *mode_lib, + unsigned int k, double HostVMInefficiencyFactor, double UrgentExtraLatency, double UrgentLatency, - unsigned int GPUVMMaxPageTableLevels, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - bool GPUVMEnable, - double HostVMMinPageSize, double PDEAndMetaPTEBytesPerFrame, double MetaRowBytes, - double DPTEBytesPerRow, - double BandwidthAvailableForImmediateFlip, - unsigned int TotImmediateFlipBytes, - enum source_format_class SourcePixelFormat, - double LineTime, - double VRatio, - double VRatioChroma, - double Tno_bw, - bool DCCEnable, - unsigned int dpte_row_height, - unsigned int meta_row_height, - unsigned int dpte_row_height_chroma, - unsigned int meta_row_height_chroma, - double *DestinationLinesToRequestVMInImmediateFlip, - double *DestinationLinesToRequestRowInImmediateFlip, - double *final_flip_bw, - bool *ImmediateFlipSupportedForPipe); + double DPTEBytesPerRow); static double CalculateWriteBackDelay( enum source_format_class WritebackPixelFormat, double WritebackHRatio, @@ -325,64 +305,28 @@ static void CalculateVupdateAndDynamicMetadataParameters( static void CalculateWatermarksAndDRAMSpeedChangeSupport( struct display_mode_lib *mode_lib, unsigned int PrefetchMode, - unsigned int NumberOfActivePlanes, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizedVBlank, - unsigned int dpte_group_bytes[], - unsigned int MetaChunkSize, double UrgentLatency, double ExtraLatency, - double WritebackLatency, - double WritebackChunkSize, double SOCCLK, - double DRAMClockChangeLatency, - double SRExitTime, - double SREnterPlusExitTime, - double SRExitZ8Time, - double SREnterPlusExitZ8Time, double DCFCLKDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int vtaps[], - unsigned int VTAPsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerPlane[], double BytePerPixelDETY[], double BytePerPixelDETC[], - double DSTXAfterScaler[], - double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, unsigned int CompressedBufferSizeInkByte, enum clock_change_support *DRAMClockChangeSupport, - double *UrgentWatermark, - double *WritebackUrgentWatermark, - double *DRAMClockChangeWatermark, - double *WritebackDRAMClockChangeWatermark, double *StutterExitWatermark, double *StutterEnterPlusExitWatermark, double *Z8StutterExitWatermark, - double *Z8StutterEnterPlusExitWatermark, - double *MinActiveDRAMClockChangeLatencySupported); + double *Z8StutterEnterPlusExitWatermark); static void CalculateDCFCLKDeepSleep( struct display_mode_lib *mode_lib, @@ -1362,7 +1306,7 @@ static bool CalculatePrefetchSchedule( // - ((NumberOfCursors > 0 || GPUVMEnable || DCCEnable) ? - ((GPUVMEnable || myPipe->DCCEnable) ? (*DestinationLinesToRequestVMInVBlank + 2 * *DestinationLinesToRequestRowInVBlank) : 0.0); // TODO: Did someone else add this?? #else - LinesToRequestPrefetchPixelData = *DestinationLinesForPrefetch - *DestinationLinesToRequestVMInVBlank - 2 * *DestinationLinesToRequestRowInVBlank; + LinesToRequestPrefetchPixelData = *DestinationLinesForPrefetch - *DestinationLinesToRequestVMInVBlank - 2 * *DestinationLinesToRequestRowInVBlank; #endif #ifdef __DML_VBA_DEBUG__ @@ -1599,7 +1543,7 @@ static void CalculateDCCConfiguration( int segment_order_vert_contiguous_luma; int segment_order_vert_contiguous_chroma; - enum { + typedef enum { REQ_256Bytes, REQ_128BytesNonContiguous, REQ_128BytesContiguous, REQ_NA } RequestType; RequestType RequestLuma; @@ -2928,33 +2872,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman for (k = 0; k < v->NumberOfActivePlanes; ++k) { CalculateFlipSchedule( mode_lib, + k, HostVMInefficiencyFactor, v->UrgentExtraLatency, v->UrgentLatency, - v->GPUVMMaxPageTableLevels, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->GPUVMEnable, - v->HostVMMinPageSize, v->PDEAndMetaPTEBytesFrame[k], v->MetaRowByte[k], - v->PixelPTEBytesPerRow[k], - v->BandwidthAvailableForImmediateFlip, - v->TotImmediateFlipBytes, - v->SourcePixelFormat[k], - v->HTotal[k] / v->PixelClock[k], - v->VRatio[k], - v->VRatioChroma[k], - v->Tno_bw[k], - v->DCCEnable[k], - v->dpte_row_height[k], - v->meta_row_height[k], - v->dpte_row_height_chroma[k], - v->meta_row_height_chroma[k], - &v->DestinationLinesToRequestVMInImmediateFlip[k], - &v->DestinationLinesToRequestRowInImmediateFlip[k], - &v->final_flip_bw[k], - &v->ImmediateFlipSupportedForPipe[k]); + v->PixelPTEBytesPerRow[k]); } v->total_dcn_read_bw_with_flip = 0.0; @@ -3041,64 +2965,28 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman CalculateWatermarksAndDRAMSpeedChangeSupport( mode_lib, PrefetchMode, - v->NumberOfActivePlanes, - v->MaxLineBufferLines, - v->LineBufferSize, - v->WritebackInterfaceBufferSize, v->DCFCLK, v->ReturnBW, - v->SynchronizedVBlank, - v->dpte_group_bytes, - v->MetaChunkSize, v->UrgentLatency, v->UrgentExtraLatency, - v->WritebackLatency, - v->WritebackChunkSize, v->SOCCLK, - v->DRAMClockChangeLatency, - v->SRExitTime, - v->SREnterPlusExitTime, - v->SRExitZ8Time, - v->SREnterPlusExitZ8Time, v->DCFCLKDeepSleep, v->DETBufferSizeY, v->DETBufferSizeC, v->SwathHeightY, v->SwathHeightC, - v->LBBitPerPixel, v->SwathWidthY, v->SwathWidthC, - v->HRatio, - v->HRatioChroma, - v->vtaps, - v->VTAPsChroma, - v->VRatio, - v->VRatioChroma, - v->HTotal, - v->PixelClock, - v->BlendingAndTiming, v->DPPPerPlane, v->BytePerPixelDETY, v->BytePerPixelDETC, - v->DSTXAfterScaler, - v->DSTYAfterScaler, - v->WritebackEnable, - v->WritebackPixelFormat, - v->WritebackDestinationWidth, - v->WritebackDestinationHeight, - v->WritebackSourceHeight, v->UnboundedRequestEnabled, v->CompressedBufferSizeInkByte, &DRAMClockChangeSupport, - &v->UrgentWatermark, - &v->WritebackUrgentWatermark, - &v->DRAMClockChangeWatermark, - &v->WritebackDRAMClockChangeWatermark, &v->StutterExitWatermark, &v->StutterEnterPlusExitWatermark, &v->Z8StutterExitWatermark, - &v->Z8StutterEnterPlusExitWatermark, - &v->MinActiveDRAMClockChangeLatencySupported); + &v->Z8StutterEnterPlusExitWatermark); for (k = 0; k < v->NumberOfActivePlanes; ++k) { if (v->WritebackEnable[k] == true) { @@ -3710,61 +3598,43 @@ static void CalculateRowBandwidth( static void CalculateFlipSchedule( struct display_mode_lib *mode_lib, + unsigned int k, double HostVMInefficiencyFactor, double UrgentExtraLatency, double UrgentLatency, - unsigned int GPUVMMaxPageTableLevels, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - bool GPUVMEnable, - double HostVMMinPageSize, double PDEAndMetaPTEBytesPerFrame, double MetaRowBytes, - double DPTEBytesPerRow, - double BandwidthAvailableForImmediateFlip, - unsigned int TotImmediateFlipBytes, - enum source_format_class SourcePixelFormat, - double LineTime, - double VRatio, - double VRatioChroma, - double Tno_bw, - bool DCCEnable, - unsigned int dpte_row_height, - unsigned int meta_row_height, - unsigned int dpte_row_height_chroma, - unsigned int meta_row_height_chroma, - double *DestinationLinesToRequestVMInImmediateFlip, - double *DestinationLinesToRequestRowInImmediateFlip, - double *final_flip_bw, - bool *ImmediateFlipSupportedForPipe) + double DPTEBytesPerRow) { + struct vba_vars_st *v = &mode_lib->vba; double min_row_time = 0.0; unsigned int HostVMDynamicLevelsTrips; double TimeForFetchingMetaPTEImmediateFlip; double TimeForFetchingRowInVBlankImmediateFlip; double ImmediateFlipBW; + double LineTime = v->HTotal[k] / v->PixelClock[k]; - if (GPUVMEnable == true && HostVMEnable == true) { - HostVMDynamicLevelsTrips = HostVMMaxNonCachedPageTableLevels; + if (v->GPUVMEnable == true && v->HostVMEnable == true) { + HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; } else { HostVMDynamicLevelsTrips = 0; } - if (GPUVMEnable == true || DCCEnable == true) { - ImmediateFlipBW = (PDEAndMetaPTEBytesPerFrame + MetaRowBytes + DPTEBytesPerRow) * BandwidthAvailableForImmediateFlip / TotImmediateFlipBytes; + if (v->GPUVMEnable == true || v->DCCEnable[k] == true) { + ImmediateFlipBW = (PDEAndMetaPTEBytesPerFrame + MetaRowBytes + DPTEBytesPerRow) * v->BandwidthAvailableForImmediateFlip / v->TotImmediateFlipBytes; } - if (GPUVMEnable == true) { + if (v->GPUVMEnable == true) { TimeForFetchingMetaPTEImmediateFlip = dml_max3( - Tno_bw + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / ImmediateFlipBW, - UrgentExtraLatency + UrgentLatency * (GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1), + v->Tno_bw[k] + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / ImmediateFlipBW, + UrgentExtraLatency + UrgentLatency * (v->GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1), LineTime / 4.0); } else { TimeForFetchingMetaPTEImmediateFlip = 0; } - *DestinationLinesToRequestVMInImmediateFlip = dml_ceil(4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime), 1) / 4.0; - if ((GPUVMEnable == true || DCCEnable == true)) { + v->DestinationLinesToRequestVMInImmediateFlip[k] = dml_ceil(4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime), 1) / 4.0; + if ((v->GPUVMEnable == true || v->DCCEnable[k] == true)) { TimeForFetchingRowInVBlankImmediateFlip = dml_max3( (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / ImmediateFlipBW, UrgentLatency * (HostVMDynamicLevelsTrips + 1), @@ -3773,54 +3643,54 @@ static void CalculateFlipSchedule( TimeForFetchingRowInVBlankImmediateFlip = 0; } - *DestinationLinesToRequestRowInImmediateFlip = dml_ceil(4.0 * (TimeForFetchingRowInVBlankImmediateFlip / LineTime), 1) / 4.0; + v->DestinationLinesToRequestRowInImmediateFlip[k] = dml_ceil(4.0 * (TimeForFetchingRowInVBlankImmediateFlip / LineTime), 1) / 4.0; - if (GPUVMEnable == true) { - *final_flip_bw = dml_max( - PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / (*DestinationLinesToRequestVMInImmediateFlip * LineTime), - (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (*DestinationLinesToRequestRowInImmediateFlip * LineTime)); - } else if ((GPUVMEnable == true || DCCEnable == true)) { - *final_flip_bw = (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (*DestinationLinesToRequestRowInImmediateFlip * LineTime); + if (v->GPUVMEnable == true) { + v->final_flip_bw[k] = dml_max( + PDEAndMetaPTEBytesPerFrame * HostVMInefficiencyFactor / (v->DestinationLinesToRequestVMInImmediateFlip[k] * LineTime), + (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (v->DestinationLinesToRequestRowInImmediateFlip[k] * LineTime)); + } else if ((v->GPUVMEnable == true || v->DCCEnable[k] == true)) { + v->final_flip_bw[k] = (MetaRowBytes + DPTEBytesPerRow * HostVMInefficiencyFactor) / (v->DestinationLinesToRequestRowInImmediateFlip[k] * LineTime); } else { - *final_flip_bw = 0; + v->final_flip_bw[k] = 0; } - if (SourcePixelFormat == dm_420_8 || SourcePixelFormat == dm_420_10 || SourcePixelFormat == dm_rgbe_alpha) { - if (GPUVMEnable == true && DCCEnable != true) { - min_row_time = dml_min(dpte_row_height * LineTime / VRatio, dpte_row_height_chroma * LineTime / VRatioChroma); - } else if (GPUVMEnable != true && DCCEnable == true) { - min_row_time = dml_min(meta_row_height * LineTime / VRatio, meta_row_height_chroma * LineTime / VRatioChroma); + if (v->SourcePixelFormat[k] == dm_420_8 || v->SourcePixelFormat[k] == dm_420_10 || v->SourcePixelFormat[k] == dm_rgbe_alpha) { + if (v->GPUVMEnable == true && v->DCCEnable[k] != true) { + min_row_time = dml_min(v->dpte_row_height[k] * LineTime / v->VRatio[k], v->dpte_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); + } else if (v->GPUVMEnable != true && v->DCCEnable[k] == true) { + min_row_time = dml_min(v->meta_row_height[k] * LineTime / v->VRatio[k], v->meta_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); } else { min_row_time = dml_min4( - dpte_row_height * LineTime / VRatio, - meta_row_height * LineTime / VRatio, - dpte_row_height_chroma * LineTime / VRatioChroma, - meta_row_height_chroma * LineTime / VRatioChroma); + v->dpte_row_height[k] * LineTime / v->VRatio[k], + v->meta_row_height[k] * LineTime / v->VRatio[k], + v->dpte_row_height_chroma[k] * LineTime / v->VRatioChroma[k], + v->meta_row_height_chroma[k] * LineTime / v->VRatioChroma[k]); } } else { - if (GPUVMEnable == true && DCCEnable != true) { - min_row_time = dpte_row_height * LineTime / VRatio; - } else if (GPUVMEnable != true && DCCEnable == true) { - min_row_time = meta_row_height * LineTime / VRatio; + if (v->GPUVMEnable == true && v->DCCEnable[k] != true) { + min_row_time = v->dpte_row_height[k] * LineTime / v->VRatio[k]; + } else if (v->GPUVMEnable != true && v->DCCEnable[k] == true) { + min_row_time = v->meta_row_height[k] * LineTime / v->VRatio[k]; } else { - min_row_time = dml_min(dpte_row_height * LineTime / VRatio, meta_row_height * LineTime / VRatio); + min_row_time = dml_min(v->dpte_row_height[k] * LineTime / v->VRatio[k], v->meta_row_height[k] * LineTime / v->VRatio[k]); } } - if (*DestinationLinesToRequestVMInImmediateFlip >= 32 || *DestinationLinesToRequestRowInImmediateFlip >= 16 + if (v->DestinationLinesToRequestVMInImmediateFlip[k] >= 32 || v->DestinationLinesToRequestRowInImmediateFlip[k] >= 16 || TimeForFetchingMetaPTEImmediateFlip + 2 * TimeForFetchingRowInVBlankImmediateFlip > min_row_time) { - *ImmediateFlipSupportedForPipe = false; + v->ImmediateFlipSupportedForPipe[k] = false; } else { - *ImmediateFlipSupportedForPipe = true; + v->ImmediateFlipSupportedForPipe[k] = true; } #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: DestinationLinesToRequestVMInImmediateFlip = %f\n", __func__, *DestinationLinesToRequestVMInImmediateFlip); - dml_print("DML::%s: DestinationLinesToRequestRowInImmediateFlip = %f\n", __func__, *DestinationLinesToRequestRowInImmediateFlip); + dml_print("DML::%s: DestinationLinesToRequestVMInImmediateFlip = %f\n", __func__, v->DestinationLinesToRequestVMInImmediateFlip[k]); + dml_print("DML::%s: DestinationLinesToRequestRowInImmediateFlip = %f\n", __func__, v->DestinationLinesToRequestRowInImmediateFlip[k]); dml_print("DML::%s: TimeForFetchingMetaPTEImmediateFlip = %f\n", __func__, TimeForFetchingMetaPTEImmediateFlip); dml_print("DML::%s: TimeForFetchingRowInVBlankImmediateFlip = %f\n", __func__, TimeForFetchingRowInVBlankImmediateFlip); dml_print("DML::%s: min_row_time = %f\n", __func__, min_row_time); - dml_print("DML::%s: ImmediateFlipSupportedForPipe = %d\n", __func__, *ImmediateFlipSupportedForPipe); + dml_print("DML::%s: ImmediateFlipSupportedForPipe = %d\n", __func__, v->ImmediateFlipSupportedForPipe[k]); #endif } @@ -4071,9 +3941,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_ v->SourceFormatPixelAndScanSupport = true; for (k = 0; k < v->NumberOfActivePlanes; k++) { - if ((v->SurfaceTiling[k] == dm_sw_linear && (!(v->SourceScan[k] != dm_vert) || v->DCCEnable[k] == true)) - || ((v->SurfaceTiling[k] == dm_sw_64kb_d || v->SurfaceTiling[k] == dm_sw_64kb_d_t - || v->SurfaceTiling[k] == dm_sw_64kb_d_x) && !(v->SourcePixelFormat[k] == dm_444_64))) { + if (v->SurfaceTiling[k] == dm_sw_linear && (!(v->SourceScan[k] != dm_vert) || v->DCCEnable[k] == true)) { v->SourceFormatPixelAndScanSupport = false; } } @@ -5414,33 +5282,13 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_ for (k = 0; k < v->NumberOfActivePlanes; k++) { CalculateFlipSchedule( mode_lib, + k, HostVMInefficiencyFactor, v->ExtraLatency, v->UrgLatency[i], - v->GPUVMMaxPageTableLevels, - v->HostVMEnable, - v->HostVMMaxNonCachedPageTableLevels, - v->GPUVMEnable, - v->HostVMMinPageSize, v->PDEAndMetaPTEBytesPerFrame[i][j][k], v->MetaRowBytes[i][j][k], - v->DPTEBytesPerRow[i][j][k], - v->BandwidthAvailableForImmediateFlip, - v->TotImmediateFlipBytes, - v->SourcePixelFormat[k], - v->HTotal[k] / v->PixelClock[k], - v->VRatio[k], - v->VRatioChroma[k], - v->Tno_bw[k], - v->DCCEnable[k], - v->dpte_row_height[k], - v->meta_row_height[k], - v->dpte_row_height_chroma[k], - v->meta_row_height_chroma[k], - &v->DestinationLinesToRequestVMInImmediateFlip[k], - &v->DestinationLinesToRequestRowInImmediateFlip[k], - &v->final_flip_bw[k], - &v->ImmediateFlipSupportedForPipe[k]); + v->DPTEBytesPerRow[i][j][k]); } v->total_dcn_read_bw_with_flip = 0.0; for (k = 0; k < v->NumberOfActivePlanes; k++) { @@ -5498,64 +5346,28 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_ CalculateWatermarksAndDRAMSpeedChangeSupport( mode_lib, v->PrefetchModePerState[i][j], - v->NumberOfActivePlanes, - v->MaxLineBufferLines, - v->LineBufferSize, - v->WritebackInterfaceBufferSize, v->DCFCLKState[i][j], v->ReturnBWPerState[i][j], - v->SynchronizedVBlank, - v->dpte_group_bytes, - v->MetaChunkSize, v->UrgLatency[i], v->ExtraLatency, - v->WritebackLatency, - v->WritebackChunkSize, v->SOCCLKPerState[i], - v->DRAMClockChangeLatency, - v->SRExitTime, - v->SREnterPlusExitTime, - v->SRExitZ8Time, - v->SREnterPlusExitZ8Time, v->ProjectedDCFCLKDeepSleep[i][j], v->DETBufferSizeYThisState, v->DETBufferSizeCThisState, v->SwathHeightYThisState, v->SwathHeightCThisState, - v->LBBitPerPixel, v->SwathWidthYThisState, v->SwathWidthCThisState, - v->HRatio, - v->HRatioChroma, - v->vtaps, - v->VTAPsChroma, - v->VRatio, - v->VRatioChroma, - v->HTotal, - v->PixelClock, - v->BlendingAndTiming, v->NoOfDPPThisState, v->BytePerPixelInDETY, v->BytePerPixelInDETC, - v->DSTXAfterScaler, - v->DSTYAfterScaler, - v->WritebackEnable, - v->WritebackPixelFormat, - v->WritebackDestinationWidth, - v->WritebackDestinationHeight, - v->WritebackSourceHeight, UnboundedRequestEnabledThisState, CompressedBufferSizeInkByteThisState, &v->DRAMClockChangeSupport[i][j], - &v->UrgentWatermark, - &v->WritebackUrgentWatermark, - &v->DRAMClockChangeWatermark, - &v->WritebackDRAMClockChangeWatermark, - &dummy, &dummy, &dummy, &dummy, - &v->MinActiveDRAMClockChangeLatencySupported); + &dummy); } } @@ -5681,64 +5493,28 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_ static void CalculateWatermarksAndDRAMSpeedChangeSupport( struct display_mode_lib *mode_lib, unsigned int PrefetchMode, - unsigned int NumberOfActivePlanes, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizedVBlank, - unsigned int dpte_group_bytes[], - unsigned int MetaChunkSize, double UrgentLatency, double ExtraLatency, - double WritebackLatency, - double WritebackChunkSize, double SOCCLK, - double DRAMClockChangeLatency, - double SRExitTime, - double SREnterPlusExitTime, - double SRExitZ8Time, - double SREnterPlusExitZ8Time, double DCFCLKDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int vtaps[], - unsigned int VTAPsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerPlane[], double BytePerPixelDETY[], double BytePerPixelDETC[], - double DSTXAfterScaler[], - double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, unsigned int CompressedBufferSizeInkByte, enum clock_change_support *DRAMClockChangeSupport, - double *UrgentWatermark, - double *WritebackUrgentWatermark, - double *DRAMClockChangeWatermark, - double *WritebackDRAMClockChangeWatermark, double *StutterExitWatermark, double *StutterEnterPlusExitWatermark, double *Z8StutterExitWatermark, - double *Z8StutterEnterPlusExitWatermark, - double *MinActiveDRAMClockChangeLatencySupported) + double *Z8StutterEnterPlusExitWatermark) { struct vba_vars_st *v = &mode_lib->vba; double EffectiveLBLatencyHidingY; @@ -5758,103 +5534,103 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( double TotalPixelBW = 0.0; int k, j; - *UrgentWatermark = UrgentLatency + ExtraLatency; + v->UrgentWatermark = UrgentLatency + ExtraLatency; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: UrgentLatency = %f\n", __func__, UrgentLatency); dml_print("DML::%s: ExtraLatency = %f\n", __func__, ExtraLatency); - dml_print("DML::%s: UrgentWatermark = %f\n", __func__, *UrgentWatermark); + dml_print("DML::%s: UrgentWatermark = %f\n", __func__, v->UrgentWatermark); #endif - *DRAMClockChangeWatermark = DRAMClockChangeLatency + *UrgentWatermark; + v->DRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->UrgentWatermark; #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: DRAMClockChangeLatency = %f\n", __func__, DRAMClockChangeLatency); - dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, *DRAMClockChangeWatermark); + dml_print("DML::%s: v->DRAMClockChangeLatency = %f\n", __func__, v->DRAMClockChangeLatency); + dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, v->DRAMClockChangeWatermark); #endif v->TotalActiveWriteback = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (WritebackEnable[k] == true) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (v->WritebackEnable[k] == true) { v->TotalActiveWriteback = v->TotalActiveWriteback + 1; } } if (v->TotalActiveWriteback <= 1) { - *WritebackUrgentWatermark = WritebackLatency; + v->WritebackUrgentWatermark = v->WritebackLatency; } else { - *WritebackUrgentWatermark = WritebackLatency + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->WritebackUrgentWatermark = v->WritebackLatency + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; } if (v->TotalActiveWriteback <= 1) { - *WritebackDRAMClockChangeWatermark = DRAMClockChangeLatency + WritebackLatency; + v->WritebackDRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->WritebackLatency; } else { - *WritebackDRAMClockChangeWatermark = DRAMClockChangeLatency + WritebackLatency + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->WritebackDRAMClockChangeWatermark = v->DRAMClockChangeLatency + v->WritebackLatency + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; } - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { TotalPixelBW = TotalPixelBW - + DPPPerPlane[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k] + SwathWidthC[k] * BytePerPixelDETC[k] * VRatioChroma[k]) - / (HTotal[k] / PixelClock[k]); + + DPPPerPlane[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k] + SwathWidthC[k] * BytePerPixelDETC[k] * v->VRatioChroma[k]) + / (v->HTotal[k] / v->PixelClock[k]); } - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { double EffectiveDETBufferSizeY = DETBufferSizeY[k]; v->LBLatencyHidingSourceLinesY = dml_min( - (double) MaxLineBufferLines, - dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(HRatio[k], 1.0)), 1)) - (vtaps[k] - 1); + (double) v->MaxLineBufferLines, + dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1); v->LBLatencyHidingSourceLinesC = dml_min( - (double) MaxLineBufferLines, - dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(HRatioChroma[k], 1.0)), 1)) - (VTAPsChroma[k] - 1); + (double) v->MaxLineBufferLines, + dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1); - EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / VRatio[k] * (HTotal[k] / PixelClock[k]); + EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]); - EffectiveLBLatencyHidingC = v->LBLatencyHidingSourceLinesC / VRatioChroma[k] * (HTotal[k] / PixelClock[k]); + EffectiveLBLatencyHidingC = v->LBLatencyHidingSourceLinesC / v->VRatioChroma[k] * (v->HTotal[k] / v->PixelClock[k]); if (UnboundedRequestEnabled) { EffectiveDETBufferSizeY = EffectiveDETBufferSizeY - + CompressedBufferSizeInkByte * 1024 * SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k] / (HTotal[k] / PixelClock[k]) / TotalPixelBW; + + CompressedBufferSizeInkByte * 1024 * SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k] / (v->HTotal[k] / v->PixelClock[k]) / TotalPixelBW; } LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k]; LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]); - FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (HTotal[k] / PixelClock[k]) / VRatio[k]; + FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k]; if (BytePerPixelDETC[k] > 0) { LinesInDETC = v->DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k]; LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]); - FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (HTotal[k] / PixelClock[k]) / VRatioChroma[k]; + FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (v->HTotal[k] / v->PixelClock[k]) / v->VRatioChroma[k]; } else { LinesInDETC = 0; FullDETBufferingTimeC = 999999; } ActiveDRAMClockChangeLatencyMarginY = EffectiveLBLatencyHidingY + FullDETBufferingTimeY - - ((double) DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] / PixelClock[k] - *UrgentWatermark - *DRAMClockChangeWatermark; + - ((double) v->DSTXAfterScaler[k] / v->HTotal[k] + v->DSTYAfterScaler[k]) * v->HTotal[k] / v->PixelClock[k] - v->UrgentWatermark - v->DRAMClockChangeWatermark; - if (NumberOfActivePlanes > 1) { + if (v->NumberOfActivePlanes > 1) { ActiveDRAMClockChangeLatencyMarginY = ActiveDRAMClockChangeLatencyMarginY - - (1 - 1.0 / NumberOfActivePlanes) * SwathHeightY[k] * HTotal[k] / PixelClock[k] / VRatio[k]; + - (1 - 1.0 / v->NumberOfActivePlanes) * SwathHeightY[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatio[k]; } if (BytePerPixelDETC[k] > 0) { ActiveDRAMClockChangeLatencyMarginC = EffectiveLBLatencyHidingC + FullDETBufferingTimeC - - ((double) DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] / PixelClock[k] - *UrgentWatermark - *DRAMClockChangeWatermark; + - ((double) v->DSTXAfterScaler[k] / v->HTotal[k] + v->DSTYAfterScaler[k]) * v->HTotal[k] / v->PixelClock[k] - v->UrgentWatermark - v->DRAMClockChangeWatermark; - if (NumberOfActivePlanes > 1) { + if (v->NumberOfActivePlanes > 1) { ActiveDRAMClockChangeLatencyMarginC = ActiveDRAMClockChangeLatencyMarginC - - (1 - 1.0 / NumberOfActivePlanes) * SwathHeightC[k] * HTotal[k] / PixelClock[k] / VRatioChroma[k]; + - (1 - 1.0 / v->NumberOfActivePlanes) * SwathHeightC[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatioChroma[k]; } v->ActiveDRAMClockChangeLatencyMargin[k] = dml_min(ActiveDRAMClockChangeLatencyMarginY, ActiveDRAMClockChangeLatencyMarginC); } else { v->ActiveDRAMClockChangeLatencyMargin[k] = ActiveDRAMClockChangeLatencyMarginY; } - if (WritebackEnable[k] == true) { - WritebackDRAMClockChangeLatencyHiding = WritebackInterfaceBufferSize * 1024 - / (WritebackDestinationWidth[k] * WritebackDestinationHeight[k] / (WritebackSourceHeight[k] * HTotal[k] / PixelClock[k]) * 4); - if (WritebackPixelFormat[k] == dm_444_64) { + if (v->WritebackEnable[k] == true) { + WritebackDRAMClockChangeLatencyHiding = v->WritebackInterfaceBufferSize * 1024 + / (v->WritebackDestinationWidth[k] * v->WritebackDestinationHeight[k] / (v->WritebackSourceHeight[k] * v->HTotal[k] / v->PixelClock[k]) * 4); + if (v->WritebackPixelFormat[k] == dm_444_64) { WritebackDRAMClockChangeLatencyHiding = WritebackDRAMClockChangeLatencyHiding / 2; } WritebackDRAMClockChangeLatencyMargin = WritebackDRAMClockChangeLatencyHiding - v->WritebackDRAMClockChangeWatermark; @@ -5864,14 +5640,14 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( v->MinActiveDRAMClockChangeMargin = 999999; PlaneWithMinActiveDRAMClockChangeMargin = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { if (v->ActiveDRAMClockChangeLatencyMargin[k] < v->MinActiveDRAMClockChangeMargin) { v->MinActiveDRAMClockChangeMargin = v->ActiveDRAMClockChangeLatencyMargin[k]; - if (BlendingAndTiming[k] == k) { + if (v->BlendingAndTiming[k] == k) { PlaneWithMinActiveDRAMClockChangeMargin = k; } else { - for (j = 0; j < NumberOfActivePlanes; ++j) { - if (BlendingAndTiming[k] == j) { + for (j = 0; j < v->NumberOfActivePlanes; ++j) { + if (v->BlendingAndTiming[k] == j) { PlaneWithMinActiveDRAMClockChangeMargin = j; } } @@ -5879,11 +5655,11 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( } } - *MinActiveDRAMClockChangeLatencySupported = v->MinActiveDRAMClockChangeMargin + DRAMClockChangeLatency; + v->MinActiveDRAMClockChangeLatencySupported = v->MinActiveDRAMClockChangeMargin + v->DRAMClockChangeLatency ; SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank = 999999; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (!((k == PlaneWithMinActiveDRAMClockChangeMargin) && (BlendingAndTiming[k] == k)) && !(BlendingAndTiming[k] == PlaneWithMinActiveDRAMClockChangeMargin) + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (!((k == PlaneWithMinActiveDRAMClockChangeMargin) && (v->BlendingAndTiming[k] == k)) && !(v->BlendingAndTiming[k] == PlaneWithMinActiveDRAMClockChangeMargin) && v->ActiveDRAMClockChangeLatencyMargin[k] < SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank) { SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank = v->ActiveDRAMClockChangeLatencyMargin[k]; } @@ -5891,25 +5667,25 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( v->TotalNumberOfActiveOTG = 0; - for (k = 0; k < NumberOfActivePlanes; ++k) { - if (BlendingAndTiming[k] == k) { + for (k = 0; k < v->NumberOfActivePlanes; ++k) { + if (v->BlendingAndTiming[k] == k) { v->TotalNumberOfActiveOTG = v->TotalNumberOfActiveOTG + 1; } } if (v->MinActiveDRAMClockChangeMargin > 0 && PrefetchMode == 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vactive; - } else if ((SynchronizedVBlank == true || v->TotalNumberOfActiveOTG == 1 + } else if ((v->SynchronizedVBlank == true || v->TotalNumberOfActiveOTG == 1 || SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank > 0) && PrefetchMode == 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vblank; } else { *DRAMClockChangeSupport = dm_dram_clock_change_unsupported; } - *StutterExitWatermark = SRExitTime + ExtraLatency + 10 / DCFCLKDeepSleep; - *StutterEnterPlusExitWatermark = (SREnterPlusExitTime + ExtraLatency + 10 / DCFCLKDeepSleep); - *Z8StutterExitWatermark = SRExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; - *Z8StutterEnterPlusExitWatermark = SREnterPlusExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; + *StutterExitWatermark = v->SRExitTime + ExtraLatency + 10 / DCFCLKDeepSleep; + *StutterEnterPlusExitWatermark = (v->SREnterPlusExitTime + ExtraLatency + 10 / DCFCLKDeepSleep); + *Z8StutterExitWatermark = v->SRExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; + *Z8StutterEnterPlusExitWatermark = v->SREnterPlusExitZ8Time + ExtraLatency + 10 / DCFCLKDeepSleep; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: StutterExitWatermark = %f\n", __func__, *StutterExitWatermark); @@ -7049,8 +6825,6 @@ static void CalculateSwathWidth( { int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]); int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]); - int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); - int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l); @@ -7061,6 +6835,8 @@ static void CalculateSwathWidth( MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k]; swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]); if (BytePerPixC[k] > 0) { + int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); + swath_width_chroma_ub[k] = dml_min( surface_width_ub_c, (int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]); @@ -7072,6 +6848,8 @@ static void CalculateSwathWidth( MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k]; swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]); if (BytePerPixC[k] > 0) { + int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); + swath_width_chroma_ub[k] = dml_min( surface_height_ub_c, (int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]); @@ -7157,12 +6935,13 @@ static double CalculateExtraLatencyBytes( HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1); else HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2); - else + } else { HostVMDynamicLevels = 0; + } ret = ReorderingBytes + (TotalNumberOfActiveDPP * PixelChunkSizeInKByte + TotalNumberOfDCCActiveDPP * MetaChunkSize) * 1024.0; - if (GPUVMEnable == true) + if (GPUVMEnable == true) { for (k = 0; k < NumberOfActivePlanes; ++k) ret = ret + NumberOfDPP[k] * dpte_group_bytes[k] * (1 + 8 * HostVMDynamicLevels) * HostVMInefficiencyFactor; } @@ -7406,7 +7185,7 @@ static unsigned int CalculateMaxVStartup( double line_time_us = HTotal / PixelClock; unsigned int vblank_actual = VTotal - VActive; unsigned int vblank_nom_default_in_line = dml_floor(VBlankNomDefaultUS / line_time_us, 1.0); - unsigned int vblank_nom_input = dml_min(VBlankNom, vblank_nom_default_in_line); + unsigned int vblank_nom_input = VBlankNom; //dml_min(VBlankNom, vblank_nom_default_in_line); unsigned int vblank_avail = vblank_nom_input == 0 ? vblank_nom_default_in_line : vblank_nom_input; vblank_size = (unsigned int) dml_min(vblank_actual, vblank_avail); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index 7f6c977c498149f244c2c1596ca244dc2927a64c..819de0f110126eed5059298b5bd50d78e57a0539 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -121,8 +121,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc = { }, }, .num_states = 1, - .sr_exit_time_us = 20.16, - .sr_enter_plus_exit_time_us = 27.13, + .sr_exit_time_us = 42.97, + .sr_enter_plus_exit_time_us = 49.94, .sr_exit_z8_time_us = 285.0, .sr_enter_plus_exit_z8_time_us = 320, .writeback_latency_us = 12.0, @@ -243,18 +243,61 @@ void dcn32_build_wm_range_table_fpu(struct clk_mgr_internal *clk_mgr) clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF; } +/* + * Finds dummy_latency_index when MCLK switching using firmware based + * vblank stretch is enabled. This function will iterate through the + * table of dummy pstate latencies until the lowest value that allows + * dm_allow_self_refresh_and_mclk_switch to happen is found + */ +int dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc, + struct dc_state *context, + display_e2e_pipe_params_st *pipes, + int pipe_cnt, + int vlevel) +{ + const int max_latency_table_entries = 4; + const struct vba_vars_st *vba = &context->bw_ctx.dml.vba; + int dummy_latency_index = 0; + + dc_assert_fp_enabled(); + + while (dummy_latency_index < max_latency_table_entries) { + context->bw_ctx.dml.soc.dram_clock_change_latency_us = + dc->clk_mgr->bw_params->dummy_pstate_table[dummy_latency_index].dummy_pstate_latency_us; + dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false); + + if (vlevel < context->bw_ctx.dml.vba.soc.num_states && + vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] != dm_dram_clock_change_unsupported) + break; + + dummy_latency_index++; + } + + if (dummy_latency_index == max_latency_table_entries) { + ASSERT(dummy_latency_index != max_latency_table_entries); + /* If the execution gets here, it means dummy p_states are + * not possible. This should never happen and would mean + * something is severely wrong. + * Here we reset dummy_latency_index to 3, because it is + * better to have underflows than system crashes. + */ + dummy_latency_index = max_latency_table_entries - 1; + } + + return dummy_latency_index; +} + /** * dcn32_helper_populate_phantom_dlg_params - Get DLG params for phantom pipes * and populate pipe_ctx with those params. - * - * This function must be called AFTER the phantom pipes are added to context - * and run through DML (so that the DLG params for the phantom pipes can be - * populated), and BEFORE we program the timing for the phantom pipes. - * * @dc: [in] current dc state * @context: [in] new dc state * @pipes: [in] DML pipe params array * @pipe_cnt: [in] DML pipe count + * + * This function must be called AFTER the phantom pipes are added to context + * and run through DML (so that the DLG params for the phantom pipes can be + * populated), and BEFORE we program the timing for the phantom pipes. */ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc, struct dc_state *context, @@ -286,41 +329,88 @@ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc, } } -bool dcn32_predict_pipe_split(struct dc_state *context, display_pipe_params_st pipe, int index) +/** + * dcn32_predict_pipe_split - Predict if pipe split will occur for a given DML pipe + * @context: [in] New DC state to be programmed + * @pipe_e2e: [in] DML pipe end to end context + * + * This function takes in a DML pipe (pipe_e2e) and predicts if pipe split is required (both + * ODM and MPC). For pipe split, ODM combine is determined by the ODM mode, and MPC combine is + * determined by DPPClk requirements + * + * This function follows the same policy as DML: + * - Check for ODM combine requirements / policy first + * - MPC combine is only chosen if there is no ODM combine requirements / policy in place, and + * MPC is required + * + * Return: Number of splits expected (1 for 2:1 split, 3 for 4:1 split, 0 for no splits). + */ +uint8_t dcn32_predict_pipe_split(struct dc_state *context, + display_e2e_pipe_params_st *pipe_e2e) { double pscl_throughput; double pscl_throughput_chroma; double dpp_clk_single_dpp, clock; double clk_frequency = 0.0; double vco_speed = context->bw_ctx.dml.soc.dispclk_dppclk_vco_speed_mhz; + bool total_available_pipes_support = false; + uint32_t number_of_dpp = 0; + enum odm_combine_mode odm_mode = dm_odm_combine_mode_disabled; + double req_dispclk_per_surface = 0; + uint8_t num_splits = 0; dc_assert_fp_enabled(); - dml32_CalculateSinglePipeDPPCLKAndSCLThroughput(pipe.scale_ratio_depth.hscl_ratio, - pipe.scale_ratio_depth.hscl_ratio_c, - pipe.scale_ratio_depth.vscl_ratio, - pipe.scale_ratio_depth.vscl_ratio_c, - context->bw_ctx.dml.ip.max_dchub_pscl_bw_pix_per_clk, - context->bw_ctx.dml.ip.max_pscl_lb_bw_pix_per_clk, - pipe.dest.pixel_rate_mhz, - pipe.src.source_format, - pipe.scale_taps.htaps, - pipe.scale_taps.htaps_c, - pipe.scale_taps.vtaps, - pipe.scale_taps.vtaps_c, - /* Output */ - &pscl_throughput, &pscl_throughput_chroma, - &dpp_clk_single_dpp); + dml32_CalculateODMMode(context->bw_ctx.dml.ip.maximum_pixels_per_line_per_dsc_unit, + pipe_e2e->pipe.dest.hactive, + pipe_e2e->dout.output_format, + pipe_e2e->dout.output_type, + pipe_e2e->pipe.dest.odm_combine_policy, + context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dispclk_mhz, + context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dispclk_mhz, + pipe_e2e->dout.dsc_enable != 0, + 0, /* TotalNumberOfActiveDPP can be 0 since we're predicting pipe split requirement */ + context->bw_ctx.dml.ip.max_num_dpp, + pipe_e2e->pipe.dest.pixel_rate_mhz, + context->bw_ctx.dml.soc.dcn_downspread_percent, + context->bw_ctx.dml.ip.dispclk_ramp_margin_percent, + context->bw_ctx.dml.soc.dispclk_dppclk_vco_speed_mhz, + pipe_e2e->dout.dsc_slices, + /* Output */ + &total_available_pipes_support, + &number_of_dpp, + &odm_mode, + &req_dispclk_per_surface); + + dml32_CalculateSinglePipeDPPCLKAndSCLThroughput(pipe_e2e->pipe.scale_ratio_depth.hscl_ratio, + pipe_e2e->pipe.scale_ratio_depth.hscl_ratio_c, + pipe_e2e->pipe.scale_ratio_depth.vscl_ratio, + pipe_e2e->pipe.scale_ratio_depth.vscl_ratio_c, + context->bw_ctx.dml.ip.max_dchub_pscl_bw_pix_per_clk, + context->bw_ctx.dml.ip.max_pscl_lb_bw_pix_per_clk, + pipe_e2e->pipe.dest.pixel_rate_mhz, + pipe_e2e->pipe.src.source_format, + pipe_e2e->pipe.scale_taps.htaps, + pipe_e2e->pipe.scale_taps.htaps_c, + pipe_e2e->pipe.scale_taps.vtaps, + pipe_e2e->pipe.scale_taps.vtaps_c, + /* Output */ + &pscl_throughput, &pscl_throughput_chroma, + &dpp_clk_single_dpp); clock = dpp_clk_single_dpp * (1 + context->bw_ctx.dml.soc.dcn_downspread_percent / 100); if (clock > 0) - clk_frequency = vco_speed * 4.0 / ((int)(vco_speed * 4.0)); + clk_frequency = vco_speed * 4.0 / ((int)(vco_speed * 4.0) / clock); - if (clk_frequency > context->bw_ctx.dml.soc.clock_limits[index].dppclk_mhz) - return true; - else - return false; + if (odm_mode == dm_odm_combine_mode_2to1) + num_splits = 1; + else if (odm_mode == dm_odm_combine_mode_4to1) + num_splits = 3; + else if (clk_frequency > context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dppclk_mhz) + num_splits = 1; + + return num_splits; } static float calculate_net_bw_in_kbytes_sec(struct _vcs_dpi_voltage_scaling_st *entry) @@ -409,7 +499,14 @@ void insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scaling_st *table, } /** - * dcn32_set_phantom_stream_timing: Set timing params for the phantom stream + * dcn32_set_phantom_stream_timing - Set timing params for the phantom stream + * @dc: current dc state + * @context: new dc state + * @ref_pipe: Main pipe for the phantom stream + * @phantom_stream: target phantom stream state + * @pipes: DML pipe params + * @pipe_cnt: number of DML pipes + * @dc_pipe_idx: DC pipe index for the main pipe (i.e. ref_pipe) * * Set timing params of the phantom stream based on calculated output from DML. * This function first gets the DML pipe index using the DC pipe index, then @@ -422,13 +519,6 @@ void insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scaling_st *table, * that separately. * * - Set phantom backporch = vstartup of main pipe - * - * @dc: current dc state - * @context: new dc state - * @ref_pipe: Main pipe for the phantom stream - * @pipes: DML pipe params - * @pipe_cnt: number of DML pipes - * @dc_pipe_idx: DC pipe index for the main pipe (i.e. ref_pipe) */ void dcn32_set_phantom_stream_timing(struct dc *dc, struct dc_state *context, @@ -497,16 +587,14 @@ void dcn32_set_phantom_stream_timing(struct dc *dc, } /** - * dcn32_get_num_free_pipes: Calculate number of free pipes + * dcn32_get_num_free_pipes - Calculate number of free pipes + * @dc: current dc state + * @context: new dc state * * This function assumes that a "used" pipe is a pipe that has * both a stream and a plane assigned to it. * - * @dc: current dc state - * @context: new dc state - * - * Return: - * Number of free pipes available in the context + * Return: Number of free pipes available in the context */ static unsigned int dcn32_get_num_free_pipes(struct dc *dc, struct dc_state *context) { @@ -530,7 +618,10 @@ static unsigned int dcn32_get_num_free_pipes(struct dc *dc, struct dc_state *con } /** - * dcn32_assign_subvp_pipe: Function to decide which pipe will use Sub-VP. + * dcn32_assign_subvp_pipe - Function to decide which pipe will use Sub-VP. + * @dc: current dc state + * @context: new dc state + * @index: [out] dc pipe index for the pipe chosen to have phantom pipes assigned * * We enter this function if we are Sub-VP capable (i.e. enough pipes available) * and regular P-State switching (i.e. VACTIVE/VBLANK) is not supported, or if @@ -544,12 +635,7 @@ static unsigned int dcn32_get_num_free_pipes(struct dc *dc, struct dc_state *con * for determining which should be the SubVP pipe (need a way to determine if a pipe / plane doesn't * support MCLK switching naturally [i.e. ACTIVE or VBLANK]). * - * @param dc: current dc state - * @param context: new dc state - * @param index: [out] dc pipe index for the pipe chosen to have phantom pipes assigned - * - * Return: - * True if a valid pipe assignment was found for Sub-VP. Otherwise false. + * Return: True if a valid pipe assignment was found for Sub-VP. Otherwise false. */ static bool dcn32_assign_subvp_pipe(struct dc *dc, struct dc_state *context, @@ -579,9 +665,10 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc, * - Not able to switch in vactive naturally (switching in active means the * DET provides enough buffer to hide the P-State switch latency -- trying * to combine this with SubVP can cause issues with the scheduling). + * - Not TMZ surface */ if (pipe->plane_state && !pipe->top_pipe && - pipe->stream->mall_stream_config.type == SUBVP_NONE && refresh_rate < 120 && + pipe->stream->mall_stream_config.type == SUBVP_NONE && refresh_rate < 120 && !pipe->plane_state->address.tmz_surface && vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] <= 0) { while (pipe) { num_pipes++; @@ -615,7 +702,9 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc, } /** - * dcn32_enough_pipes_for_subvp: Function to check if there are "enough" pipes for SubVP. + * dcn32_enough_pipes_for_subvp - Function to check if there are "enough" pipes for SubVP. + * @dc: current dc state + * @context: new dc state * * This function returns true if there are enough free pipes * to create the required phantom pipes for any given stream @@ -627,9 +716,6 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc, * pipe which can be used as the phantom pipe for the non pipe * split pipe. * - * @dc: current dc state - * @context: new dc state - * * Return: * True if there are enough free pipes to assign phantom pipes to at least one * stream that does not already have phantom pipes assigned. Otherwise false. @@ -668,7 +754,9 @@ static bool dcn32_enough_pipes_for_subvp(struct dc *dc, struct dc_state *context } /** - * subvp_subvp_schedulable: Determine if SubVP + SubVP config is schedulable + * subvp_subvp_schedulable - Determine if SubVP + SubVP config is schedulable + * @dc: current dc state + * @context: new dc state * * High level algorithm: * 1. Find longest microschedule length (in us) between the two SubVP pipes @@ -676,11 +764,7 @@ static bool dcn32_enough_pipes_for_subvp(struct dc *dc, struct dc_state *context * pipes still allows for the maximum microschedule to fit in the active * region for both pipes. * - * @dc: current dc state - * @context: new dc state - * - * Return: - * bool - True if the SubVP + SubVP config is schedulable, false otherwise + * Return: True if the SubVP + SubVP config is schedulable, false otherwise */ static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context) { @@ -740,7 +824,10 @@ static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context) } /** - * subvp_drr_schedulable: Determine if SubVP + DRR config is schedulable + * subvp_drr_schedulable - Determine if SubVP + DRR config is schedulable + * @dc: current dc state + * @context: new dc state + * @drr_pipe: DRR pipe_ctx for the SubVP + DRR config * * High level algorithm: * 1. Get timing for SubVP pipe, phantom pipe, and DRR pipe @@ -749,12 +836,7 @@ static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context) * 3.If (SubVP Active - Prefetch > Stretched DRR frame + max(MALL region, Stretched DRR frame)) * then report the configuration as supported * - * @dc: current dc state - * @context: new dc state - * @drr_pipe: DRR pipe_ctx for the SubVP + DRR config - * - * Return: - * bool - True if the SubVP + DRR config is schedulable, false otherwise + * Return: True if the SubVP + DRR config is schedulable, false otherwise */ static bool subvp_drr_schedulable(struct dc *dc, struct dc_state *context, struct pipe_ctx *drr_pipe) { @@ -818,7 +900,9 @@ static bool subvp_drr_schedulable(struct dc *dc, struct dc_state *context, struc /** - * subvp_vblank_schedulable: Determine if SubVP + VBLANK config is schedulable + * subvp_vblank_schedulable - Determine if SubVP + VBLANK config is schedulable + * @dc: current dc state + * @context: new dc state * * High level algorithm: * 1. Get timing for SubVP pipe, phantom pipe, and VBLANK pipe @@ -826,11 +910,7 @@ static bool subvp_drr_schedulable(struct dc *dc, struct dc_state *context, struc * then report the configuration as supported * 3. If the VBLANK display is DRR, then take the DRR static schedulability path * - * @dc: current dc state - * @context: new dc state - * - * Return: - * bool - True if the SubVP + VBLANK/DRR config is schedulable, false otherwise + * Return: True if the SubVP + VBLANK/DRR config is schedulable, false otherwise */ static bool subvp_vblank_schedulable(struct dc *dc, struct dc_state *context) { @@ -907,20 +987,18 @@ static bool subvp_vblank_schedulable(struct dc *dc, struct dc_state *context) } /** - * subvp_validate_static_schedulability: Check which SubVP case is calculated and handle - * static analysis based on the case. + * subvp_validate_static_schedulability - Check which SubVP case is calculated + * and handle static analysis based on the case. + * @dc: current dc state + * @context: new dc state + * @vlevel: Voltage level calculated by DML * * Three cases: * 1. SubVP + SubVP * 2. SubVP + VBLANK (DRR checked internally) * 3. SubVP + VACTIVE (currently unsupported) * - * @dc: current dc state - * @context: new dc state - * @vlevel: Voltage level calculated by DML - * - * Return: - * bool - True if statically schedulable, false otherwise + * Return: True if statically schedulable, false otherwise */ static bool subvp_validate_static_schedulability(struct dc *dc, struct dc_state *context, @@ -1019,12 +1097,15 @@ static void dcn32_full_validate_bw_helper(struct dc *dc, * 5. (Config doesn't support MCLK in VACTIVE/VBLANK || dc->debug.force_subvp_mclk_switch) */ if (!dc->debug.force_disable_subvp && dcn32_all_pipes_have_stream_and_plane(dc, context) && - !dcn32_mpo_in_use(context) && (*vlevel == context->bw_ctx.dml.soc.num_states || + !dcn32_mpo_in_use(context) && !dcn32_any_surfaces_rotated(dc, context) && + (*vlevel == context->bw_ctx.dml.soc.num_states || vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported || dc->debug.force_subvp_mclk_switch)) { dcn32_merge_pipes_for_subvp(dc, context); - // to re-initialize viewport after the pipe merge + memset(merge, 0, MAX_PIPES * sizeof(bool)); + + /* to re-initialize viewport after the pipe merge */ for (i = 0; i < dc->res_pool->pipe_count; i++) { struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i]; @@ -1109,17 +1190,23 @@ static void dcn32_full_validate_bw_helper(struct dc *dc, vba->VoltageLevel = *vlevel; } } else { - // only call dcn20_validate_apply_pipe_split_flags if we found a supported config - memset(split, 0, MAX_PIPES * sizeof(int)); - memset(merge, 0, MAX_PIPES * sizeof(bool)); - *vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge); - vba->VoltageLevel = *vlevel; - // Most populate phantom DLG params before programming hardware / timing for phantom pipe DC_FP_START(); dcn32_helper_populate_phantom_dlg_params(dc, context, pipes, *pipe_cnt); DC_FP_END(); + /* Call validate_apply_pipe_split flags after calling DML getters for + * phantom dlg params, or some of the VBA params indicating pipe split + * can be overwritten by the getters. + * + * When setting up SubVP config, all pipes are merged before attempting to + * add phantom pipes. If pipe split (ODM / MPC) is required, both the main + * and phantom pipes will be split in the regular pipe splitting sequence. + */ + memset(split, 0, MAX_PIPES * sizeof(int)); + memset(merge, 0, MAX_PIPES * sizeof(bool)); + *vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge); + vba->VoltageLevel = *vlevel; // Note: We can't apply the phantom pipes to hardware at this time. We have to wait // until driver has acquired the DMCUB lock to do it safely. } @@ -1487,6 +1574,33 @@ bool dcn32_internal_validate_bw(struct dc *dc, if (pipe->next_odm_pipe) pipe->next_odm_pipe->prev_odm_pipe = pipe->prev_odm_pipe; + /*2:1ODM+MPC Split MPO to Single Pipe + MPC Split MPO*/ + if (pipe->bottom_pipe) { + if (pipe->bottom_pipe->prev_odm_pipe || pipe->bottom_pipe->next_odm_pipe) { + /*MPC split rules will handle this case*/ + pipe->bottom_pipe->top_pipe = NULL; + } else { + /* when merging an ODM pipes, the bottom MPC pipe must now point to + * the previous ODM pipe and its associated stream assets + */ + if (pipe->prev_odm_pipe->bottom_pipe) { + /* 3 plane MPO*/ + pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe->bottom_pipe; + pipe->prev_odm_pipe->bottom_pipe->bottom_pipe = pipe->bottom_pipe; + } else { + /* 2 plane MPO*/ + pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe; + pipe->prev_odm_pipe->bottom_pipe = pipe->bottom_pipe; + } + + memcpy(&pipe->bottom_pipe->stream_res, &pipe->bottom_pipe->top_pipe->stream_res, sizeof(struct stream_resource)); + } + } + + if (pipe->top_pipe) { + pipe->top_pipe->bottom_pipe = NULL; + } + pipe->bottom_pipe = NULL; pipe->next_odm_pipe = NULL; pipe->plane_state = NULL; @@ -1619,8 +1733,20 @@ bool dcn32_internal_validate_bw(struct dc *dc, goto validate_fail; } - if (repopulate_pipes) + if (repopulate_pipes) { pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate); + + /* repopulate_pipes = 1 means the pipes were either split or merged. In this case + * we have to re-calculate the DET allocation and run through DML once more to + * ensure all the params are calculated correctly. We do not need to run the + * pipe split check again after this call (pipes are already split / merged). + * */ + if (!fast_validate) { + context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final = + dm_prefetch_support_uclk_fclk_and_stutter_if_possible; + vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt); + } + } *vlevel_out = vlevel; *pipe_cnt_out = pipe_cnt; @@ -1643,6 +1769,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context, int i, pipe_idx, vlevel_temp = 0; double dcfclk = dcn3_2_soc.clock_limits[0].dcfclk_mhz; double dcfclk_from_validation = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb]; + double dcfclk_from_fw_based_mclk_switching = dcfclk_from_validation; bool pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] != dm_dram_clock_change_unsupported; unsigned int dummy_latency_index = 0; @@ -1666,7 +1793,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context, dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch(dc, context); if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) { - dummy_latency_index = dcn30_find_dummy_latency_index_for_fw_based_mclk_switch(dc, + dummy_latency_index = dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(dc, context, pipes, pipe_cnt, vlevel); /* After calling dcn30_find_dummy_latency_index_for_fw_based_mclk_switch @@ -1678,7 +1805,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context, dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us; dcn32_internal_validate_bw(dc, context, pipes, &pipe_cnt, &vlevel, false); maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb; - dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb]; + dcfclk_from_fw_based_mclk_switching = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb]; pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] != dm_dram_clock_change_unsupported; } @@ -1764,6 +1891,10 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context, pipes[0].clks_cfg.dcfclk_mhz = dcfclk_from_validation; pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz; + if (context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) { + pipes[0].clks_cfg.dcfclk_mhz = dcfclk_from_fw_based_mclk_switching; + } + if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) { min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed; min_dram_speed_mts_margin = 160; @@ -1906,6 +2037,45 @@ static void remove_entry_from_table_at_index(struct _vcs_dpi_voltage_scaling_st memset(&table[--(*num_entries)], 0, sizeof(struct _vcs_dpi_voltage_scaling_st)); } +void dcn32_patch_dpm_table(struct clk_bw_params *bw_params) +{ + int i; + unsigned int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, + max_phyclk_mhz = 0, max_dtbclk_mhz = 0, max_fclk_mhz = 0, max_uclk_mhz = 0; + + for (i = 0; i < MAX_NUM_DPM_LVL; i++) { + if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz) + max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz; + if (bw_params->clk_table.entries[i].fclk_mhz > max_fclk_mhz) + max_fclk_mhz = bw_params->clk_table.entries[i].fclk_mhz; + if (bw_params->clk_table.entries[i].memclk_mhz > max_uclk_mhz) + max_uclk_mhz = bw_params->clk_table.entries[i].memclk_mhz; + if (bw_params->clk_table.entries[i].dispclk_mhz > max_dispclk_mhz) + max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz; + if (bw_params->clk_table.entries[i].dppclk_mhz > max_dppclk_mhz) + max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz; + if (bw_params->clk_table.entries[i].phyclk_mhz > max_phyclk_mhz) + max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz; + if (bw_params->clk_table.entries[i].dtbclk_mhz > max_dtbclk_mhz) + max_dtbclk_mhz = bw_params->clk_table.entries[i].dtbclk_mhz; + } + + /* Scan through clock values we currently have and if they are 0, + * then populate it with dcn3_2_soc.clock_limits[] value. + * + * Do it for DCFCLK, DISPCLK, DTBCLK and UCLK as any of those being + * 0, will cause it to skip building the clock table. + */ + if (max_dcfclk_mhz == 0) + bw_params->clk_table.entries[0].dcfclk_mhz = dcn3_2_soc.clock_limits[0].dcfclk_mhz; + if (max_dispclk_mhz == 0) + bw_params->clk_table.entries[0].dispclk_mhz = dcn3_2_soc.clock_limits[0].dispclk_mhz; + if (max_dtbclk_mhz == 0) + bw_params->clk_table.entries[0].dtbclk_mhz = dcn3_2_soc.clock_limits[0].dtbclk_mhz; + if (max_uclk_mhz == 0) + bw_params->clk_table.entries[0].memclk_mhz = dcn3_2_soc.clock_limits[0].dram_speed_mts / 16; +} + static int build_synthetic_soc_states(struct clk_bw_params *bw_params, struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries) { @@ -2098,7 +2268,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, return 0; } -/** +/* * dcn32_update_bw_bounding_box * * This would override some dcn3_2 ip_or_soc initial parameters hardcoded from @@ -2140,6 +2310,7 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa if ((int)(dcn3_2_soc.urgent_latency_us * 1000) != dc->bb_overrides.urgent_latency_ns && dc->bb_overrides.urgent_latency_ns) { dcn3_2_soc.urgent_latency_us = dc->bb_overrides.urgent_latency_ns / 1000.0; + dcn3_2_soc.urgent_latency_pixel_data_only_us = dc->bb_overrides.urgent_latency_ns / 1000.0; } if ((int)(dcn3_2_soc.dram_clock_change_latency_us * 1000) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h index 3ed06ab855beddeedb02b1ab4c8a460d3581338a..3a3dc2ce4c7390ae8fa38a5734ef7305d53a9630 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h @@ -29,11 +29,6 @@ #include "clk_mgr_internal.h" -#define DCN3_2_DEFAULT_DET_SIZE 256 -#define DCN3_2_MAX_DET_SIZE 1152 -#define DCN3_2_MIN_DET_SIZE 128 -#define DCN3_2_MIN_COMPBUF_SIZE_KB 128 - void dcn32_build_wm_range_table_fpu(struct clk_mgr_internal *clk_mgr); void dcn32_helper_populate_phantom_dlg_params(struct dc *dc, @@ -41,9 +36,8 @@ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc, display_e2e_pipe_params_st *pipes, int pipe_cnt); -bool dcn32_predict_pipe_split(struct dc_state *context, - display_pipe_params_st pipe, - int index); +uint8_t dcn32_predict_pipe_split(struct dc_state *context, + display_e2e_pipe_params_st *pipe_e2e); void insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries, @@ -71,4 +65,12 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context, void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_params); +int dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc, + struct dc_state *context, + display_e2e_pipe_params_st *pipes, + int pipe_cnt, + int vlevel); + +void dcn32_patch_dpm_table(struct clk_bw_params *bw_params); + #endif diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index d8014bfbc3fe77a9eb0d6510a5392c48baa540fa..5b91660a6496b410b48249f1a16d61af4991ac0e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -733,6 +733,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman mode_lib->vba.FCLKChangeLatency, v->UrgentLatency, mode_lib->vba.SREnterPlusExitTime); + memset(&v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe, 0, sizeof(DmlPipe)); + v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.Dppclk = mode_lib->vba.DPPCLK[k]; v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.Dispclk = mode_lib->vba.DISPCLK; v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.PixelClock = mode_lib->vba.PixelClock[k]; @@ -755,30 +757,18 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.BytePerPixelY = v->BytePerPixelY[k]; v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.BytePerPixelC = v->BytePerPixelC[k]; v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe.ProgressiveToInterlaceUnitInOPP = mode_lib->vba.ProgressiveToInterlaceUnitInOPP; - v->ErrorResult[k] = dml32_CalculatePrefetchSchedule(v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.HostVMInefficiencyFactor, - &v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe, v->DSCDelay[k], - mode_lib->vba.DPPCLKDelaySubtotal + mode_lib->vba.DPPCLKDelayCNVCFormater, - mode_lib->vba.DPPCLKDelaySCL, - mode_lib->vba.DPPCLKDelaySCLLBOnly, - mode_lib->vba.DPPCLKDelayCNVCCursor, - mode_lib->vba.DISPCLKDelaySubtotal, - (unsigned int) (v->SwathWidthY[k] / mode_lib->vba.HRatio[k]), - mode_lib->vba.OutputFormat[k], - mode_lib->vba.MaxInterDCNTileRepeaters, + v->ErrorResult[k] = dml32_CalculatePrefetchSchedule( + v, + k, + v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.HostVMInefficiencyFactor, + &v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe, + v->DSCDelay[k], + (unsigned int) (v->SwathWidthY[k] / v->HRatio[k]), dml_min(v->VStartupLines, v->MaxVStartupLines[k]), v->MaxVStartupLines[k], - mode_lib->vba.GPUVMMaxPageTableLevels, - mode_lib->vba.GPUVMEnable, - mode_lib->vba.HostVMEnable, - mode_lib->vba.HostVMMaxNonCachedPageTableLevels, - mode_lib->vba.HostVMMinPageSize, - mode_lib->vba.DynamicMetadataEnable[k], - mode_lib->vba.DynamicMetadataVMEnabled, - mode_lib->vba.DynamicMetadataLinesBeforeActiveRequired[k], - mode_lib->vba.DynamicMetadataTransmittedBytes[k], v->UrgentLatency, v->UrgentExtraLatency, - mode_lib->vba.TCalc, + v->TCalc, v->PDEAndMetaPTEBytesFrame[k], v->MetaRowByte[k], v->PixelPTEBytesPerRow[k], @@ -792,8 +782,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->MaxNumSwathC[k], v->swath_width_luma_ub[k], v->swath_width_chroma_ub[k], - mode_lib->vba.SwathHeightY[k], - mode_lib->vba.SwathHeightC[k], + v->SwathHeightY[k], + v->SwathHeightC[k], TWait, /* Output */ &v->DSTXAfterScaler[k], @@ -1163,58 +1153,28 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.mmSOCParameters.SMNLatency = mode_lib->vba.SMNLatency; dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( - mode_lib->vba.USRRetrainingRequiredFinal, - mode_lib->vba.UsesMALLForPStateChange, - mode_lib->vba.PrefetchModePerState[mode_lib->vba.VoltageLevel][mode_lib->vba.maxMpcComb], - mode_lib->vba.NumberOfActiveSurfaces, - mode_lib->vba.MaxLineBufferLines, - mode_lib->vba.LineBufferSizeFinal, - mode_lib->vba.WritebackInterfaceBufferSize, - mode_lib->vba.DCFCLK, - mode_lib->vba.ReturnBW, - mode_lib->vba.SynchronizeTimingsFinal, - mode_lib->vba.SynchronizeDRRDisplaysForUCLKPStateChangeFinal, - mode_lib->vba.DRRDisplay, - v->dpte_group_bytes, - v->meta_row_height, - v->meta_row_height_chroma, + v, + v->PrefetchModePerState[v->VoltageLevel][v->maxMpcComb], + v->DCFCLK, + v->ReturnBW, v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.mmSOCParameters, - mode_lib->vba.WritebackChunkSize, - mode_lib->vba.SOCCLK, + v->SOCCLK, v->DCFCLKDeepSleep, - mode_lib->vba.DETBufferSizeY, - mode_lib->vba.DETBufferSizeC, - mode_lib->vba.SwathHeightY, - mode_lib->vba.SwathHeightC, - mode_lib->vba.LBBitPerPixel, + v->DETBufferSizeY, + v->DETBufferSizeC, + v->SwathHeightY, + v->SwathHeightC, v->SwathWidthY, v->SwathWidthC, - mode_lib->vba.HRatio, - mode_lib->vba.HRatioChroma, - mode_lib->vba.vtaps, - mode_lib->vba.VTAPsChroma, - mode_lib->vba.VRatio, - mode_lib->vba.VRatioChroma, - mode_lib->vba.HTotal, - mode_lib->vba.VTotal, - mode_lib->vba.VActive, - mode_lib->vba.PixelClock, - mode_lib->vba.BlendingAndTiming, - mode_lib->vba.DPPPerPlane, + v->DPPPerPlane, v->BytePerPixelDETY, v->BytePerPixelDETC, v->DSTXAfterScaler, v->DSTYAfterScaler, - mode_lib->vba.WritebackEnable, - mode_lib->vba.WritebackPixelFormat, - mode_lib->vba.WritebackDestinationWidth, - mode_lib->vba.WritebackDestinationHeight, - mode_lib->vba.WritebackSourceHeight, v->UnboundedRequestEnabled, v->CompressedBufferSizeInkByte, /* Output */ - &v->Watermark, &v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.dummy_dramchange_support, v->MaxActiveDRAMClockChangeLatencySupported, v->SubViewportLinesNeededInMALL, @@ -1806,10 +1766,10 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l &mode_lib->vba.Read256BlockHeightC[k], &mode_lib->vba.Read256BlockWidthY[k], &mode_lib->vba.Read256BlockWidthC[k], - &mode_lib->vba.MicroTileHeightY[k], - &mode_lib->vba.MicroTileHeightC[k], - &mode_lib->vba.MicroTileWidthY[k], - &mode_lib->vba.MicroTileWidthC[k]); + &mode_lib->vba.MacroTileHeightY[k], + &mode_lib->vba.MacroTileHeightC[k], + &mode_lib->vba.MacroTileWidthY[k], + &mode_lib->vba.MacroTileWidthC[k]); } /*Bandwidth Support Check*/ @@ -2034,6 +1994,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l dml32_CalculateODMMode( mode_lib->vba.MaximumPixelsPerLinePerDSCUnit, mode_lib->vba.HActive[k], + mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.ODMUse[k], mode_lib->vba.MaxDispclk[i], @@ -2056,6 +2017,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l dml32_CalculateODMMode( mode_lib->vba.MaximumPixelsPerLinePerDSCUnit, mode_lib->vba.HActive[k], + mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.ODMUse[k], mode_lib->vba.MaxDispclk[i], @@ -2292,9 +2254,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) { if (!(mode_lib->vba.DSCInputBitPerComponent[k] == 12.0 || mode_lib->vba.DSCInputBitPerComponent[k] == 10.0 - || mode_lib->vba.DSCInputBitPerComponent[k] == 8.0 - || mode_lib->vba.DSCInputBitPerComponent[k] > - mode_lib->vba.MaximumDSCBitsPerComponent)) { + || mode_lib->vba.DSCInputBitPerComponent[k] == 8.0) + || mode_lib->vba.DSCInputBitPerComponent[k] > mode_lib->vba.MaximumDSCBitsPerComponent) { mode_lib->vba.NonsupportedDSCInputBPC = true; } } @@ -2370,16 +2331,15 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l if (mode_lib->vba.OutputMultistreamId[k] == k && mode_lib->vba.ForcedOutputLinkBPP[k] == 0) mode_lib->vba.BPPForMultistreamNotIndicated = true; for (j = 0; j < mode_lib->vba.NumberOfActiveSurfaces; ++j) { - if (mode_lib->vba.OutputMultistreamId[k] == j && mode_lib->vba.OutputMultistreamEn[k] + if (mode_lib->vba.OutputMultistreamId[k] == j && mode_lib->vba.ForcedOutputLinkBPP[k] == 0) mode_lib->vba.BPPForMultistreamNotIndicated = true; } } if ((mode_lib->vba.Output[k] == dm_edp || mode_lib->vba.Output[k] == dm_hdmi)) { - if (mode_lib->vba.OutputMultistreamId[k] == k && mode_lib->vba.OutputMultistreamEn[k]) + if (mode_lib->vba.OutputMultistreamEn[k] == true && mode_lib->vba.OutputMultistreamId[k] == k) mode_lib->vba.MultistreamWithHDMIOreDP = true; - for (j = 0; j < mode_lib->vba.NumberOfActiveSurfaces; ++j) { if (mode_lib->vba.OutputMultistreamEn[k] == true && mode_lib->vba.OutputMultistreamId[k] == j) mode_lib->vba.MultistreamWithHDMIOreDP = true; @@ -2518,8 +2478,6 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k]); } - m = 0; - for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) { for (m = 0; m <= mode_lib->vba.NumberOfActiveSurfaces - 1; m++) { for (j = 0; j <= mode_lib->vba.NumberOfActiveSurfaces - 1; j++) { @@ -2661,10 +2619,10 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.Read256BlockWidthC, mode_lib->vba.Read256BlockHeightY, mode_lib->vba.Read256BlockHeightC, - mode_lib->vba.MicroTileWidthY, - mode_lib->vba.MicroTileWidthC, - mode_lib->vba.MicroTileHeightY, - mode_lib->vba.MicroTileHeightC, + mode_lib->vba.MacroTileWidthY, + mode_lib->vba.MacroTileWidthC, + mode_lib->vba.MacroTileHeightY, + mode_lib->vba.MacroTileHeightC, /* Output */ mode_lib->vba.SurfaceSizeInMALL, @@ -2711,10 +2669,10 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeight256BytesY = mode_lib->vba.Read256BlockHeightY[k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockWidth256BytesC = mode_lib->vba.Read256BlockWidthC[k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeight256BytesC = mode_lib->vba.Read256BlockHeightC[k]; - v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockWidthY = mode_lib->vba.MicroTileWidthY[k]; - v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeightY = mode_lib->vba.MicroTileHeightY[k]; - v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockWidthC = mode_lib->vba.MicroTileWidthC[k]; - v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeightC = mode_lib->vba.MicroTileHeightC[k]; + v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockWidthY = mode_lib->vba.MacroTileWidthY[k]; + v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeightY = mode_lib->vba.MacroTileHeightY[k]; + v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockWidthC = mode_lib->vba.MacroTileWidthC[k]; + v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].BlockHeightC = mode_lib->vba.MacroTileHeightC[k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].InterlaceEnable = mode_lib->vba.Interlace[k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].HTotal = mode_lib->vba.HTotal[k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.SurfParameters[k].DCCEnable = mode_lib->vba.DCCEnable[k]; @@ -2896,8 +2854,6 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l } } - m = 0; - //Calculate Return BW for (i = 0; i < (int) v->soc.num_states; ++i) { for (j = 0; j <= 1; ++j) { @@ -3260,63 +3216,47 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.NoTimeForPrefetch[i][j][k] = dml32_CalculatePrefetchSchedule( + v, + k, v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.HostVMInefficiencyFactor, &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe, - mode_lib->vba.DSCDelayPerState[i][k], - mode_lib->vba.DPPCLKDelaySubtotal + - mode_lib->vba.DPPCLKDelayCNVCFormater, - mode_lib->vba.DPPCLKDelaySCL, - mode_lib->vba.DPPCLKDelaySCLLBOnly, - mode_lib->vba.DPPCLKDelayCNVCCursor, - mode_lib->vba.DISPCLKDelaySubtotal, - mode_lib->vba.SwathWidthYThisState[k] / - mode_lib->vba.HRatio[k], - mode_lib->vba.OutputFormat[k], - mode_lib->vba.MaxInterDCNTileRepeaters, - dml_min(mode_lib->vba.MaxVStartup, - mode_lib->vba.MaximumVStartup[i][j][k]), - mode_lib->vba.MaximumVStartup[i][j][k], - mode_lib->vba.GPUVMMaxPageTableLevels, - mode_lib->vba.GPUVMEnable, mode_lib->vba.HostVMEnable, - mode_lib->vba.HostVMMaxNonCachedPageTableLevels, - mode_lib->vba.HostVMMinPageSize, - mode_lib->vba.DynamicMetadataEnable[k], - mode_lib->vba.DynamicMetadataVMEnabled, - mode_lib->vba.DynamicMetadataLinesBeforeActiveRequired[k], - mode_lib->vba.DynamicMetadataTransmittedBytes[k], - mode_lib->vba.UrgLatency[i], - mode_lib->vba.ExtraLatency, - mode_lib->vba.TimeCalc, - mode_lib->vba.PDEAndMetaPTEBytesPerFrame[i][j][k], - mode_lib->vba.MetaRowBytes[i][j][k], - mode_lib->vba.DPTEBytesPerRow[i][j][k], - mode_lib->vba.PrefetchLinesY[i][j][k], - mode_lib->vba.SwathWidthYThisState[k], - mode_lib->vba.PrefillY[k], - mode_lib->vba.MaxNumSwY[k], - mode_lib->vba.PrefetchLinesC[i][j][k], - mode_lib->vba.SwathWidthCThisState[k], - mode_lib->vba.PrefillC[k], - mode_lib->vba.MaxNumSwC[k], - mode_lib->vba.swath_width_luma_ub_this_state[k], - mode_lib->vba.swath_width_chroma_ub_this_state[k], - mode_lib->vba.SwathHeightYThisState[k], - mode_lib->vba.SwathHeightCThisState[k], mode_lib->vba.TWait, + v->DSCDelayPerState[i][k], + v->SwathWidthYThisState[k] / v->HRatio[k], + dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]), + v->MaximumVStartup[i][j][k], + v->UrgLatency[i], + v->ExtraLatency, + v->TimeCalc, + v->PDEAndMetaPTEBytesPerFrame[i][j][k], + v->MetaRowBytes[i][j][k], + v->DPTEBytesPerRow[i][j][k], + v->PrefetchLinesY[i][j][k], + v->SwathWidthYThisState[k], + v->PrefillY[k], + v->MaxNumSwY[k], + v->PrefetchLinesC[i][j][k], + v->SwathWidthCThisState[k], + v->PrefillC[k], + v->MaxNumSwC[k], + v->swath_width_luma_ub_this_state[k], + v->swath_width_chroma_ub_this_state[k], + v->SwathHeightYThisState[k], + v->SwathHeightCThisState[k], v->TWait, /* Output */ &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTXAfterScaler[k], &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTYAfterScaler[k], - &mode_lib->vba.LineTimesForPrefetch[k], - &mode_lib->vba.PrefetchBW[k], - &mode_lib->vba.LinesForMetaPTE[k], - &mode_lib->vba.LinesForMetaAndDPTERow[k], - &mode_lib->vba.VRatioPreY[i][j][k], - &mode_lib->vba.VRatioPreC[i][j][k], - &mode_lib->vba.RequiredPrefetchPixelDataBWLuma[0][0][k], - &mode_lib->vba.RequiredPrefetchPixelDataBWChroma[0][0][k], - &mode_lib->vba.NoTimeForDynamicMetadata[i][j][k], - &mode_lib->vba.Tno_bw[k], - &mode_lib->vba.prefetch_vmrow_bw[k], + &v->LineTimesForPrefetch[k], + &v->PrefetchBW[k], + &v->LinesForMetaPTE[k], + &v->LinesForMetaAndDPTERow[k], + &v->VRatioPreY[i][j][k], + &v->VRatioPreC[i][j][k], + &v->RequiredPrefetchPixelDataBWLuma[0][0][k], + &v->RequiredPrefetchPixelDataBWChroma[0][0][k], + &v->NoTimeForDynamicMetadata[i][j][k], + &v->Tno_bw[k], + &v->prefetch_vmrow_bw[k], &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single[0], // double *Tdmdl_vm &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single[1], // double *Tdmdl &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single[2], // double *TSetup @@ -3559,62 +3499,32 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l { dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( - mode_lib->vba.USRRetrainingRequiredFinal, - mode_lib->vba.UsesMALLForPStateChange, - mode_lib->vba.PrefetchModePerState[i][j], - mode_lib->vba.NumberOfActiveSurfaces, - mode_lib->vba.MaxLineBufferLines, - mode_lib->vba.LineBufferSizeFinal, - mode_lib->vba.WritebackInterfaceBufferSize, - mode_lib->vba.DCFCLKState[i][j], - mode_lib->vba.ReturnBWPerState[i][j], - mode_lib->vba.SynchronizeTimingsFinal, - mode_lib->vba.SynchronizeDRRDisplaysForUCLKPStateChangeFinal, - mode_lib->vba.DRRDisplay, - mode_lib->vba.dpte_group_bytes, - mode_lib->vba.meta_row_height, - mode_lib->vba.meta_row_height_chroma, + v, + v->PrefetchModePerState[i][j], + v->DCFCLKState[i][j], + v->ReturnBWPerState[i][j], v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.mSOCParameters, - mode_lib->vba.WritebackChunkSize, - mode_lib->vba.SOCCLKPerState[i], - mode_lib->vba.ProjectedDCFCLKDeepSleep[i][j], - mode_lib->vba.DETBufferSizeYThisState, - mode_lib->vba.DETBufferSizeCThisState, - mode_lib->vba.SwathHeightYThisState, - mode_lib->vba.SwathHeightCThisState, - mode_lib->vba.LBBitPerPixel, - mode_lib->vba.SwathWidthYThisState, // 24 - mode_lib->vba.SwathWidthCThisState, - mode_lib->vba.HRatio, - mode_lib->vba.HRatioChroma, - mode_lib->vba.vtaps, - mode_lib->vba.VTAPsChroma, - mode_lib->vba.VRatio, - mode_lib->vba.VRatioChroma, - mode_lib->vba.HTotal, - mode_lib->vba.VTotal, - mode_lib->vba.VActive, - mode_lib->vba.PixelClock, - mode_lib->vba.BlendingAndTiming, - mode_lib->vba.NoOfDPPThisState, - mode_lib->vba.BytePerPixelInDETY, - mode_lib->vba.BytePerPixelInDETC, + v->SOCCLKPerState[i], + v->ProjectedDCFCLKDeepSleep[i][j], + v->DETBufferSizeYThisState, + v->DETBufferSizeCThisState, + v->SwathHeightYThisState, + v->SwathHeightCThisState, + v->SwathWidthYThisState, // 24 + v->SwathWidthCThisState, + v->NoOfDPPThisState, + v->BytePerPixelInDETY, + v->BytePerPixelInDETC, v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTXAfterScaler, v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTYAfterScaler, - mode_lib->vba.WritebackEnable, - mode_lib->vba.WritebackPixelFormat, - mode_lib->vba.WritebackDestinationWidth, - mode_lib->vba.WritebackDestinationHeight, - mode_lib->vba.WritebackSourceHeight, - mode_lib->vba.UnboundedRequestEnabledThisState, - mode_lib->vba.CompressedBufferSizeInkByteThisState, + v->UnboundedRequestEnabledThisState, + v->CompressedBufferSizeInkByteThisState, /* Output */ - &mode_lib->vba.Watermark, // Store the values in vba - &mode_lib->vba.DRAMClockChangeSupport[i][j], + &v->DRAMClockChangeSupport[i][j], &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single2[0], // double *MaxActiveDRAMClockChangeLatencySupported &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_integer[0], // Long SubViewportLinesNeededInMALL[] - &mode_lib->vba.FCLKChangeSupport[i][j], + &v->FCLKChangeSupport[i][j], &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single2[1], // double *MinActiveFCLKChangeLatencySupported &mode_lib->vba.USRRetrainingSupport[i][j], mode_lib->vba.ActiveDRAMClockChangeLatencyMarginPerState[i][j]); @@ -3704,11 +3614,10 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.ModeIsSupported = mode_lib->vba.ModeSupport[i][0] == true || mode_lib->vba.ModeSupport[i][1] == true; - if (mode_lib->vba.ModeSupport[i][0] == true) { + if (mode_lib->vba.ModeSupport[i][0] == true) MaximumMPCCombine = 0; - } else { + else MaximumMPCCombine = 1; - } } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index dc501ee7d01a0336fbba72277336695a8c04a6d9..ad66e241f9ae21fb5a0b54e81ba7a9b7143b418d 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -27,6 +27,8 @@ #include "display_mode_vba_32.h" #include "../display_mode_lib.h" +#define DCN32_MAX_FMT_420_BUFFER_WIDTH 4096 + unsigned int dml32_dscceComputeDelay( unsigned int bpc, double BPP, @@ -719,8 +721,8 @@ void dml32_CalculateSwathWidth( unsigned int surface_width_ub_l; unsigned int surface_height_ub_l; - unsigned int surface_width_ub_c; - unsigned int surface_height_ub_c; + unsigned int surface_width_ub_c = 0; + unsigned int surface_height_ub_c = 0; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: ForceSingleDPP = %d\n", __func__, ForceSingleDPP); @@ -784,21 +786,6 @@ void dml32_CalculateSwathWidth( surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]); surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]); - surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); - surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); - -#ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l); - dml_print("DML::%s: k=%d surface_height_ub_l=%0d\n", __func__, k, surface_height_ub_l); - dml_print("DML::%s: k=%d surface_width_ub_c=%0d\n", __func__, k, surface_width_ub_c); - dml_print("DML::%s: k=%d surface_height_ub_c=%0d\n", __func__, k, surface_height_ub_c); - dml_print("DML::%s: k=%d Read256BytesBlockWidthY=%0d\n", __func__, k, Read256BytesBlockWidthY[k]); - dml_print("DML::%s: k=%d Read256BytesBlockHeightY=%0d\n", __func__, k, Read256BytesBlockHeightY[k]); - dml_print("DML::%s: k=%d Read256BytesBlockWidthC=%0d\n", __func__, k, Read256BytesBlockWidthC[k]); - dml_print("DML::%s: k=%d Read256BytesBlockHeightC=%0d\n", __func__, k, Read256BytesBlockHeightC[k]); - dml_print("DML::%s: k=%d ViewportStationary=%0d\n", __func__, k, ViewportStationary[k]); - dml_print("DML::%s: k=%d DPPPerSurface=%0d\n", __func__, k, DPPPerSurface[k]); -#endif if (!IsVertical(SourceRotation[k])) { MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k]; @@ -818,6 +805,7 @@ void dml32_CalculateSwathWidth( Read256BytesBlockWidthY[k]); } if (BytePerPixC[k] > 0) { + surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]); if (ViewportStationary[k] && DPPPerSurface[k] == 1) { swath_width_chroma_ub[k] = dml_min(surface_width_ub_c, dml_floor(ViewportXStartC[k] + SwathWidthC[k] + @@ -848,6 +836,7 @@ void dml32_CalculateSwathWidth( Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]); } if (BytePerPixC[k] > 0) { + surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]); if (ViewportStationary[k] && DPPPerSurface[k] == 1) { swath_width_chroma_ub[k] = dml_min(surface_height_ub_c, dml_floor(ViewportYStartC[k] + SwathWidthC[k] + @@ -866,6 +855,16 @@ void dml32_CalculateSwathWidth( } #ifdef __DML_VBA_DEBUG__ + dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l); + dml_print("DML::%s: k=%d surface_height_ub_l=%0d\n", __func__, k, surface_height_ub_l); + dml_print("DML::%s: k=%d surface_width_ub_c=%0d\n", __func__, k, surface_width_ub_c); + dml_print("DML::%s: k=%d surface_height_ub_c=%0d\n", __func__, k, surface_height_ub_c); + dml_print("DML::%s: k=%d Read256BytesBlockWidthY=%0d\n", __func__, k, Read256BytesBlockWidthY[k]); + dml_print("DML::%s: k=%d Read256BytesBlockHeightY=%0d\n", __func__, k, Read256BytesBlockHeightY[k]); + dml_print("DML::%s: k=%d Read256BytesBlockWidthC=%0d\n", __func__, k, Read256BytesBlockWidthC[k]); + dml_print("DML::%s: k=%d Read256BytesBlockHeightC=%0d\n", __func__, k, Read256BytesBlockHeightC[k]); + dml_print("DML::%s: k=%d ViewportStationary=%0d\n", __func__, k, ViewportStationary[k]); + dml_print("DML::%s: k=%d DPPPerSurface=%0d\n", __func__, k, DPPPerSurface[k]); dml_print("DML::%s: k=%d swath_width_luma_ub=%0d\n", __func__, k, swath_width_luma_ub[k]); dml_print("DML::%s: k=%d swath_width_chroma_ub=%0d\n", __func__, k, swath_width_chroma_ub[k]); dml_print("DML::%s: k=%d MaximumSwathHeightY=%0d\n", __func__, k, MaximumSwathHeightY[k]); @@ -1182,6 +1181,7 @@ void dml32_CalculateDETBufferSize( void dml32_CalculateODMMode( unsigned int MaximumPixelsPerLinePerDSCUnit, unsigned int HActive, + enum output_format_class OutFormat, enum output_encoder_class Output, enum odm_combine_policy ODMUse, double StateDispclk, @@ -1256,6 +1256,29 @@ void dml32_CalculateODMMode( else *TotalAvailablePipesSupport = false; } + if (OutFormat == dm_420 && HActive > DCN32_MAX_FMT_420_BUFFER_WIDTH && + ODMUse != dm_odm_combine_policy_4to1) { + if (HActive > DCN32_MAX_FMT_420_BUFFER_WIDTH * 4) { + *ODMMode = dm_odm_combine_mode_disabled; + *NumberOfDPP = 0; + *TotalAvailablePipesSupport = false; + } else if (HActive > DCN32_MAX_FMT_420_BUFFER_WIDTH * 2 || + *ODMMode == dm_odm_combine_mode_4to1) { + *ODMMode = dm_odm_combine_mode_4to1; + *RequiredDISPCLKPerSurface = SurfaceRequiredDISPCLKWithODMCombineFourToOne; + *NumberOfDPP = 4; + } else { + *ODMMode = dm_odm_combine_mode_2to1; + *RequiredDISPCLKPerSurface = SurfaceRequiredDISPCLKWithODMCombineTwoToOne; + *NumberOfDPP = 2; + } + } + if (Output == dm_hdmi && OutFormat == dm_420 && + HActive > DCN32_MAX_FMT_420_BUFFER_WIDTH) { + *ODMMode = dm_odm_combine_mode_disabled; + *NumberOfDPP = 0; + *TotalAvailablePipesSupport = false; + } } double dml32_CalculateRequiredDispclk( @@ -1873,7 +1896,7 @@ void dml32_CalculateSurfaceSizeInMall( if (UseMALLForStaticScreen[k] == dm_use_mall_static_screen_enable) TotalSurfaceSizeInMALL = TotalSurfaceSizeInMALL + SurfaceSizeInMALL[k]; } - *ExceededMALLSize = (TotalSurfaceSizeInMALL <= MALLAllocatedForDCN * 1024 * 1024 ? false : true); + *ExceededMALLSize = (TotalSurfaceSizeInMALL > MALLAllocatedForDCN * 1024 * 1024); } // CalculateSurfaceSizeInMall void dml32_CalculateVMRowAndSwath( @@ -3366,28 +3389,14 @@ double dml32_CalculateExtraLatency( } // CalculateExtraLatency bool dml32_CalculatePrefetchSchedule( + struct vba_vars_st *v, + unsigned int k, double HostVMInefficiencyFactor, DmlPipe *myPipe, unsigned int DSCDelay, - double DPPCLKDelaySubtotalPlusCNVCFormater, - double DPPCLKDelaySCL, - double DPPCLKDelaySCLLBOnly, - double DPPCLKDelayCNVCCursor, - double DISPCLKDelaySubtotal, unsigned int DPP_RECOUT_WIDTH, - enum output_format_class OutputFormat, - unsigned int MaxInterDCNTileRepeaters, unsigned int VStartup, unsigned int MaxVStartup, - unsigned int GPUVMPageTableLevels, - bool GPUVMEnable, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - double HostVMMinPageSize, - bool DynamicMetadataEnable, - bool DynamicMetadataVMEnabled, - int DynamicMetadataLinesBeforeActiveRequired, - unsigned int DynamicMetadataTransmittedBytes, double UrgentLatency, double UrgentExtraLatency, double TCalc, @@ -3428,6 +3437,7 @@ bool dml32_CalculatePrefetchSchedule( double *VUpdateWidthPix, double *VReadyOffsetPix) { + double DPPCLKDelaySubtotalPlusCNVCFormater = v->DPPCLKDelaySubtotal + v->DPPCLKDelayCNVCFormater; bool MyError = false; unsigned int DPPCycles, DISPCLKCycles; double DSTTotalPixelsAfterScaler; @@ -3464,27 +3474,27 @@ bool dml32_CalculatePrefetchSchedule( double Tsw_est1 = 0; double Tsw_est3 = 0; - if (GPUVMEnable == true && HostVMEnable == true) - HostVMDynamicLevelsTrips = HostVMMaxNonCachedPageTableLevels; + if (v->GPUVMEnable == true && v->HostVMEnable == true) + HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; else HostVMDynamicLevelsTrips = 0; #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: GPUVMEnable = %d\n", __func__, GPUVMEnable); - dml_print("DML::%s: GPUVMPageTableLevels = %d\n", __func__, GPUVMPageTableLevels); + dml_print("DML::%s: v->GPUVMEnable = %d\n", __func__, v->GPUVMEnable); + dml_print("DML::%s: v->GPUVMMaxPageTableLevels = %d\n", __func__, v->GPUVMMaxPageTableLevels); dml_print("DML::%s: DCCEnable = %d\n", __func__, myPipe->DCCEnable); - dml_print("DML::%s: HostVMEnable=%d HostVMInefficiencyFactor=%f\n", - __func__, HostVMEnable, HostVMInefficiencyFactor); + dml_print("DML::%s: v->HostVMEnable=%d HostVMInefficiencyFactor=%f\n", + __func__, v->HostVMEnable, HostVMInefficiencyFactor); #endif dml32_CalculateVUpdateAndDynamicMetadataParameters( - MaxInterDCNTileRepeaters, + v->MaxInterDCNTileRepeaters, myPipe->Dppclk, myPipe->Dispclk, myPipe->DCFClkDeepSleep, myPipe->PixelClock, myPipe->HTotal, myPipe->VBlank, - DynamicMetadataTransmittedBytes, - DynamicMetadataLinesBeforeActiveRequired, + v->DynamicMetadataTransmittedBytes[k], + v->DynamicMetadataLinesBeforeActiveRequired[k], myPipe->InterlaceEnable, myPipe->ProgressiveToInterlaceUnitInOPP, TSetup, @@ -3499,19 +3509,19 @@ bool dml32_CalculatePrefetchSchedule( LineTime = myPipe->HTotal / myPipe->PixelClock; trip_to_mem = UrgentLatency; - Tvm_trips = UrgentExtraLatency + trip_to_mem * (GPUVMPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1); + Tvm_trips = UrgentExtraLatency + trip_to_mem * (v->GPUVMMaxPageTableLevels * (HostVMDynamicLevelsTrips + 1) - 1); - if (DynamicMetadataVMEnabled == true) + if (v->DynamicMetadataVMEnabled == true) *Tdmdl = TWait + Tvm_trips + trip_to_mem; else *Tdmdl = TWait + UrgentExtraLatency; #ifdef __DML_VBA_ALLOW_DELTA__ - if (DynamicMetadataEnable == false) + if (v->DynamicMetadataEnable[k] == false) *Tdmdl = 0.0; #endif - if (DynamicMetadataEnable == true) { + if (v->DynamicMetadataEnable[k] == true) { if (VStartup * LineTime < *TSetup + *Tdmdl + Tdmbf + Tdmec + Tdmsks) { *NotEnoughTimeForDynamicMetadata = true; #ifdef __DML_VBA_DEBUG__ @@ -3531,17 +3541,17 @@ bool dml32_CalculatePrefetchSchedule( *NotEnoughTimeForDynamicMetadata = false; } - *Tdmdl_vm = (DynamicMetadataEnable == true && DynamicMetadataVMEnabled == true && - GPUVMEnable == true ? TWait + Tvm_trips : 0); + *Tdmdl_vm = (v->DynamicMetadataEnable[k] == true && v->DynamicMetadataVMEnabled == true && + v->GPUVMEnable == true ? TWait + Tvm_trips : 0); if (myPipe->ScalerEnabled) - DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL; + DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL; else - DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly; + DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly; - DPPCycles = DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor; + DPPCycles = DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor; - DISPCLKCycles = DISPCLKDelaySubtotal; + DISPCLKCycles = v->DISPCLKDelaySubtotal; if (myPipe->Dppclk == 0.0 || myPipe->Dispclk == 0.0) return true; @@ -3567,7 +3577,7 @@ bool dml32_CalculatePrefetchSchedule( dml_print("DML::%s: DSTXAfterScaler: %d\n", __func__, *DSTXAfterScaler); #endif - if (OutputFormat == dm_420 || (myPipe->InterlaceEnable && myPipe->ProgressiveToInterlaceUnitInOPP)) + if (v->OutputFormat[k] == dm_420 || (myPipe->InterlaceEnable && myPipe->ProgressiveToInterlaceUnitInOPP)) *DSTYAfterScaler = 1; else *DSTYAfterScaler = 0; @@ -3584,13 +3594,13 @@ bool dml32_CalculatePrefetchSchedule( Tr0_trips = trip_to_mem * (HostVMDynamicLevelsTrips + 1); - if (GPUVMEnable == true) { + if (v->GPUVMEnable == true) { Tvm_trips_rounded = dml_ceil(4.0 * Tvm_trips / LineTime, 1.0) / 4.0 * LineTime; Tr0_trips_rounded = dml_ceil(4.0 * Tr0_trips / LineTime, 1.0) / 4.0 * LineTime; - if (GPUVMPageTableLevels >= 3) { + if (v->GPUVMMaxPageTableLevels >= 3) { *Tno_bw = UrgentExtraLatency + trip_to_mem * - (double) ((GPUVMPageTableLevels - 2) * (HostVMDynamicLevelsTrips + 1) - 1); - } else if (GPUVMPageTableLevels == 1 && myPipe->DCCEnable != true) { + (double) ((v->GPUVMMaxPageTableLevels - 2) * (HostVMDynamicLevelsTrips + 1) - 1); + } else if (v->GPUVMMaxPageTableLevels == 1 && myPipe->DCCEnable != true) { Tr0_trips_rounded = dml_ceil(4.0 * UrgentExtraLatency / LineTime, 1.0) / 4.0 * LineTime; // VBA_ERROR *Tno_bw = UrgentExtraLatency; @@ -3625,7 +3635,7 @@ bool dml32_CalculatePrefetchSchedule( min_Lsw = dml_max(min_Lsw, 1.0); Lsw_oto = dml_ceil(4.0 * dml_max(prefetch_sw_bytes / prefetch_bw_oto / LineTime, min_Lsw), 1.0) / 4.0; - if (GPUVMEnable == true) { + if (v->GPUVMEnable == true) { Tvm_oto = dml_max3( Tvm_trips, *Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_oto, @@ -3633,7 +3643,7 @@ bool dml32_CalculatePrefetchSchedule( } else Tvm_oto = LineTime / 4.0; - if ((GPUVMEnable == true || myPipe->DCCEnable == true)) { + if ((v->GPUVMEnable == true || myPipe->DCCEnable == true)) { Tr0_oto = dml_max4( Tr0_trips, (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_oto, @@ -3836,7 +3846,7 @@ bool dml32_CalculatePrefetchSchedule( #endif if (prefetch_bw_equ > 0) { - if (GPUVMEnable == true) { + if (v->GPUVMEnable == true) { Tvm_equ = dml_max3(*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_equ, Tvm_trips, LineTime / 4); @@ -3844,7 +3854,7 @@ bool dml32_CalculatePrefetchSchedule( Tvm_equ = LineTime / 4; } - if ((GPUVMEnable == true || myPipe->DCCEnable == true)) { + if ((v->GPUVMEnable == true || myPipe->DCCEnable == true)) { Tr0_equ = dml_max4((MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_equ, Tr0_trips, (LineTime - Tvm_equ) / 2, LineTime / 4); @@ -4209,58 +4219,28 @@ void dml32_CalculateFlipSchedule( } // CalculateFlipSchedule void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( - bool USRRetrainingRequiredFinal, - enum dm_use_mall_for_pstate_change_mode UseMALLForPStateChange[], + struct vba_vars_st *v, unsigned int PrefetchMode, - unsigned int NumberOfActiveSurfaces, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizeTimingsFinal, - bool SynchronizeDRRDisplaysForUCLKPStateChangeFinal, - bool DRRDisplay[], - unsigned int dpte_group_bytes[], - unsigned int meta_row_height[], - unsigned int meta_row_height_chroma[], SOCParametersList mmSOCParameters, - unsigned int WritebackChunkSize, double SOCCLK, double DCFClkDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int VTaps[], - unsigned int VTapsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - unsigned int VTotal[], - unsigned int VActive[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerSurface[], double BytePerPixelDETY[], double BytePerPixelDETC[], double DSTXAfterScaler[], double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, unsigned int CompressedBufferSizeInkByte, /* Output */ - Watermarks *Watermark, enum clock_change_support *DRAMClockChangeSupport, double MaxActiveDRAMClockChangeLatencySupported[], unsigned int SubViewportLinesNeededInMALL[], @@ -4302,136 +4282,136 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( unsigned int LBLatencyHidingSourceLinesY[DC__NUM_DPP__MAX]; unsigned int LBLatencyHidingSourceLinesC[DC__NUM_DPP__MAX]; - Watermark->UrgentWatermark = mmSOCParameters.UrgentLatency + mmSOCParameters.ExtraLatency; - Watermark->USRRetrainingWatermark = mmSOCParameters.UrgentLatency + mmSOCParameters.ExtraLatency + v->Watermark.UrgentWatermark = mmSOCParameters.UrgentLatency + mmSOCParameters.ExtraLatency; + v->Watermark.USRRetrainingWatermark = mmSOCParameters.UrgentLatency + mmSOCParameters.ExtraLatency + mmSOCParameters.USRRetrainingLatency + mmSOCParameters.SMNLatency; - Watermark->DRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency + Watermark->UrgentWatermark; - Watermark->FCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency + Watermark->UrgentWatermark; - Watermark->StutterExitWatermark = mmSOCParameters.SRExitTime + mmSOCParameters.ExtraLatency + v->Watermark.DRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency + v->Watermark.UrgentWatermark; + v->Watermark.FCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency + v->Watermark.UrgentWatermark; + v->Watermark.StutterExitWatermark = mmSOCParameters.SRExitTime + mmSOCParameters.ExtraLatency + 10 / DCFClkDeepSleep; - Watermark->StutterEnterPlusExitWatermark = mmSOCParameters.SREnterPlusExitTime + mmSOCParameters.ExtraLatency + v->Watermark.StutterEnterPlusExitWatermark = mmSOCParameters.SREnterPlusExitTime + mmSOCParameters.ExtraLatency + 10 / DCFClkDeepSleep; - Watermark->Z8StutterExitWatermark = mmSOCParameters.SRExitZ8Time + mmSOCParameters.ExtraLatency + v->Watermark.Z8StutterExitWatermark = mmSOCParameters.SRExitZ8Time + mmSOCParameters.ExtraLatency + 10 / DCFClkDeepSleep; - Watermark->Z8StutterEnterPlusExitWatermark = mmSOCParameters.SREnterPlusExitZ8Time + v->Watermark.Z8StutterEnterPlusExitWatermark = mmSOCParameters.SREnterPlusExitZ8Time + mmSOCParameters.ExtraLatency + 10 / DCFClkDeepSleep; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: UrgentLatency = %f\n", __func__, mmSOCParameters.UrgentLatency); dml_print("DML::%s: ExtraLatency = %f\n", __func__, mmSOCParameters.ExtraLatency); dml_print("DML::%s: DRAMClockChangeLatency = %f\n", __func__, mmSOCParameters.DRAMClockChangeLatency); - dml_print("DML::%s: UrgentWatermark = %f\n", __func__, Watermark->UrgentWatermark); - dml_print("DML::%s: USRRetrainingWatermark = %f\n", __func__, Watermark->USRRetrainingWatermark); - dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, Watermark->DRAMClockChangeWatermark); - dml_print("DML::%s: FCLKChangeWatermark = %f\n", __func__, Watermark->FCLKChangeWatermark); - dml_print("DML::%s: StutterExitWatermark = %f\n", __func__, Watermark->StutterExitWatermark); - dml_print("DML::%s: StutterEnterPlusExitWatermark = %f\n", __func__, Watermark->StutterEnterPlusExitWatermark); - dml_print("DML::%s: Z8StutterExitWatermark = %f\n", __func__, Watermark->Z8StutterExitWatermark); + dml_print("DML::%s: UrgentWatermark = %f\n", __func__, v->Watermark.UrgentWatermark); + dml_print("DML::%s: USRRetrainingWatermark = %f\n", __func__, v->Watermark.USRRetrainingWatermark); + dml_print("DML::%s: DRAMClockChangeWatermark = %f\n", __func__, v->Watermark.DRAMClockChangeWatermark); + dml_print("DML::%s: FCLKChangeWatermark = %f\n", __func__, v->Watermark.FCLKChangeWatermark); + dml_print("DML::%s: StutterExitWatermark = %f\n", __func__, v->Watermark.StutterExitWatermark); + dml_print("DML::%s: StutterEnterPlusExitWatermark = %f\n", __func__, v->Watermark.StutterEnterPlusExitWatermark); + dml_print("DML::%s: Z8StutterExitWatermark = %f\n", __func__, v->Watermark.Z8StutterExitWatermark); dml_print("DML::%s: Z8StutterEnterPlusExitWatermark = %f\n", - __func__, Watermark->Z8StutterEnterPlusExitWatermark); + __func__, v->Watermark.Z8StutterEnterPlusExitWatermark); #endif TotalActiveWriteback = 0; - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - if (WritebackEnable[k] == true) + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + if (v->WritebackEnable[k] == true) TotalActiveWriteback = TotalActiveWriteback + 1; } if (TotalActiveWriteback <= 1) { - Watermark->WritebackUrgentWatermark = mmSOCParameters.WritebackLatency; + v->Watermark.WritebackUrgentWatermark = mmSOCParameters.WritebackLatency; } else { - Watermark->WritebackUrgentWatermark = mmSOCParameters.WritebackLatency - + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->Watermark.WritebackUrgentWatermark = mmSOCParameters.WritebackLatency + + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; } - if (USRRetrainingRequiredFinal) - Watermark->WritebackUrgentWatermark = Watermark->WritebackUrgentWatermark + if (v->USRRetrainingRequiredFinal) + v->Watermark.WritebackUrgentWatermark = v->Watermark.WritebackUrgentWatermark + mmSOCParameters.USRRetrainingLatency; if (TotalActiveWriteback <= 1) { - Watermark->WritebackDRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency + v->Watermark.WritebackDRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency + mmSOCParameters.WritebackLatency; - Watermark->WritebackFCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency + v->Watermark.WritebackFCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency + mmSOCParameters.WritebackLatency; } else { - Watermark->WritebackDRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency - + mmSOCParameters.WritebackLatency + WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; - Watermark->WritebackFCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency - + mmSOCParameters.WritebackLatency + WritebackChunkSize * 1024 / 32 / SOCCLK; + v->Watermark.WritebackDRAMClockChangeWatermark = mmSOCParameters.DRAMClockChangeLatency + + mmSOCParameters.WritebackLatency + v->WritebackChunkSize * 1024.0 / 32.0 / SOCCLK; + v->Watermark.WritebackFCLKChangeWatermark = mmSOCParameters.FCLKChangeLatency + + mmSOCParameters.WritebackLatency + v->WritebackChunkSize * 1024 / 32 / SOCCLK; } - if (USRRetrainingRequiredFinal) - Watermark->WritebackDRAMClockChangeWatermark = Watermark->WritebackDRAMClockChangeWatermark + if (v->USRRetrainingRequiredFinal) + v->Watermark.WritebackDRAMClockChangeWatermark = v->Watermark.WritebackDRAMClockChangeWatermark + mmSOCParameters.USRRetrainingLatency; - if (USRRetrainingRequiredFinal) - Watermark->WritebackFCLKChangeWatermark = Watermark->WritebackFCLKChangeWatermark + if (v->USRRetrainingRequiredFinal) + v->Watermark.WritebackFCLKChangeWatermark = v->Watermark.WritebackFCLKChangeWatermark + mmSOCParameters.USRRetrainingLatency; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: WritebackDRAMClockChangeWatermark = %f\n", - __func__, Watermark->WritebackDRAMClockChangeWatermark); - dml_print("DML::%s: WritebackFCLKChangeWatermark = %f\n", __func__, Watermark->WritebackFCLKChangeWatermark); - dml_print("DML::%s: WritebackUrgentWatermark = %f\n", __func__, Watermark->WritebackUrgentWatermark); - dml_print("DML::%s: USRRetrainingRequiredFinal = %d\n", __func__, USRRetrainingRequiredFinal); + __func__, v->Watermark.WritebackDRAMClockChangeWatermark); + dml_print("DML::%s: WritebackFCLKChangeWatermark = %f\n", __func__, v->Watermark.WritebackFCLKChangeWatermark); + dml_print("DML::%s: WritebackUrgentWatermark = %f\n", __func__, v->Watermark.WritebackUrgentWatermark); + dml_print("DML::%s: v->USRRetrainingRequiredFinal = %d\n", __func__, v->USRRetrainingRequiredFinal); dml_print("DML::%s: USRRetrainingLatency = %f\n", __func__, mmSOCParameters.USRRetrainingLatency); #endif - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - TotalPixelBW = TotalPixelBW + DPPPerSurface[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k] + - SwathWidthC[k] * BytePerPixelDETC[k] * VRatioChroma[k]) / (HTotal[k] / PixelClock[k]); + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + TotalPixelBW = TotalPixelBW + DPPPerSurface[k] * (SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k] + + SwathWidthC[k] * BytePerPixelDETC[k] * v->VRatioChroma[k]) / (v->HTotal[k] / v->PixelClock[k]); } - for (k = 0; k < NumberOfActiveSurfaces; ++k) { + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { - LBLatencyHidingSourceLinesY[k] = dml_min((double) MaxLineBufferLines, dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(HRatio[k], 1.0)), 1)) - (VTaps[k] - 1); - LBLatencyHidingSourceLinesC[k] = dml_min((double) MaxLineBufferLines, dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(HRatioChroma[k], 1.0)), 1)) - (VTapsChroma[k] - 1); + LBLatencyHidingSourceLinesY[k] = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1); + LBLatencyHidingSourceLinesC[k] = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1); #ifdef __DML_VBA_DEBUG__ - dml_print("DML::%s: k=%d, MaxLineBufferLines = %d\n", __func__, k, MaxLineBufferLines); - dml_print("DML::%s: k=%d, LineBufferSize = %d\n", __func__, k, LineBufferSize); - dml_print("DML::%s: k=%d, LBBitPerPixel = %d\n", __func__, k, LBBitPerPixel[k]); - dml_print("DML::%s: k=%d, HRatio = %f\n", __func__, k, HRatio[k]); - dml_print("DML::%s: k=%d, VTaps = %d\n", __func__, k, VTaps[k]); + dml_print("DML::%s: k=%d, v->MaxLineBufferLines = %d\n", __func__, k, v->MaxLineBufferLines); + dml_print("DML::%s: k=%d, v->LineBufferSizeFinal = %d\n", __func__, k, v->LineBufferSizeFinal); + dml_print("DML::%s: k=%d, v->LBBitPerPixel = %d\n", __func__, k, v->LBBitPerPixel[k]); + dml_print("DML::%s: k=%d, v->HRatio = %f\n", __func__, k, v->HRatio[k]); + dml_print("DML::%s: k=%d, v->vtaps = %d\n", __func__, k, v->vtaps[k]); #endif - EffectiveLBLatencyHidingY = LBLatencyHidingSourceLinesY[k] / VRatio[k] * (HTotal[k] / PixelClock[k]); - EffectiveLBLatencyHidingC = LBLatencyHidingSourceLinesC[k] / VRatioChroma[k] * (HTotal[k] / PixelClock[k]); + EffectiveLBLatencyHidingY = LBLatencyHidingSourceLinesY[k] / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]); + EffectiveLBLatencyHidingC = LBLatencyHidingSourceLinesC[k] / v->VRatioChroma[k] * (v->HTotal[k] / v->PixelClock[k]); EffectiveDETBufferSizeY = DETBufferSizeY[k]; if (UnboundedRequestEnabled) { EffectiveDETBufferSizeY = EffectiveDETBufferSizeY + CompressedBufferSizeInkByte * 1024 - * (SwathWidthY[k] * BytePerPixelDETY[k] * VRatio[k]) - / (HTotal[k] / PixelClock[k]) / TotalPixelBW; + * (SwathWidthY[k] * BytePerPixelDETY[k] * v->VRatio[k]) + / (v->HTotal[k] / v->PixelClock[k]) / TotalPixelBW; } LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k]; LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]); - FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (HTotal[k] / PixelClock[k]) / VRatio[k]; + FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k]; ActiveClockChangeLatencyHidingY = EffectiveLBLatencyHidingY + FullDETBufferingTimeY - - (DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] / PixelClock[k]; + - (DSTXAfterScaler[k] / v->HTotal[k] + DSTYAfterScaler[k]) * v->HTotal[k] / v->PixelClock[k]; - if (NumberOfActiveSurfaces > 1) { + if (v->NumberOfActiveSurfaces > 1) { ActiveClockChangeLatencyHidingY = ActiveClockChangeLatencyHidingY - - (1 - 1 / NumberOfActiveSurfaces) * SwathHeightY[k] * HTotal[k] - / PixelClock[k] / VRatio[k]; + - (1 - 1 / v->NumberOfActiveSurfaces) * SwathHeightY[k] * v->HTotal[k] + / v->PixelClock[k] / v->VRatio[k]; } if (BytePerPixelDETC[k] > 0) { LinesInDETC[k] = DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k]; LinesInDETCRoundedDownToSwath[k] = dml_floor(LinesInDETC[k], SwathHeightC[k]); - FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath[k] * (HTotal[k] / PixelClock[k]) - / VRatioChroma[k]; + FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) + / v->VRatioChroma[k]; ActiveClockChangeLatencyHidingC = EffectiveLBLatencyHidingC + FullDETBufferingTimeC - - (DSTXAfterScaler[k] / HTotal[k] + DSTYAfterScaler[k]) * HTotal[k] - / PixelClock[k]; - if (NumberOfActiveSurfaces > 1) { + - (DSTXAfterScaler[k] / v->HTotal[k] + DSTYAfterScaler[k]) * v->HTotal[k] + / v->PixelClock[k]; + if (v->NumberOfActiveSurfaces > 1) { ActiveClockChangeLatencyHidingC = ActiveClockChangeLatencyHidingC - - (1 - 1 / NumberOfActiveSurfaces) * SwathHeightC[k] * HTotal[k] - / PixelClock[k] / VRatioChroma[k]; + - (1 - 1 / v->NumberOfActiveSurfaces) * SwathHeightC[k] * v->HTotal[k] + / v->PixelClock[k] / v->VRatioChroma[k]; } ActiveClockChangeLatencyHiding = dml_min(ActiveClockChangeLatencyHidingY, ActiveClockChangeLatencyHidingC); @@ -4439,24 +4419,24 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( ActiveClockChangeLatencyHiding = ActiveClockChangeLatencyHidingY; } - ActiveDRAMClockChangeLatencyMargin[k] = ActiveClockChangeLatencyHiding - Watermark->UrgentWatermark - - Watermark->DRAMClockChangeWatermark; - ActiveFCLKChangeLatencyMargin[k] = ActiveClockChangeLatencyHiding - Watermark->UrgentWatermark - - Watermark->FCLKChangeWatermark; - USRRetrainingLatencyMargin[k] = ActiveClockChangeLatencyHiding - Watermark->USRRetrainingWatermark; - - if (WritebackEnable[k]) { - WritebackLatencyHiding = WritebackInterfaceBufferSize * 1024 - / (WritebackDestinationWidth[k] * WritebackDestinationHeight[k] - / (WritebackSourceHeight[k] * HTotal[k] / PixelClock[k]) * 4); - if (WritebackPixelFormat[k] == dm_444_64) + ActiveDRAMClockChangeLatencyMargin[k] = ActiveClockChangeLatencyHiding - v->Watermark.UrgentWatermark + - v->Watermark.DRAMClockChangeWatermark; + ActiveFCLKChangeLatencyMargin[k] = ActiveClockChangeLatencyHiding - v->Watermark.UrgentWatermark + - v->Watermark.FCLKChangeWatermark; + USRRetrainingLatencyMargin[k] = ActiveClockChangeLatencyHiding - v->Watermark.USRRetrainingWatermark; + + if (v->WritebackEnable[k]) { + WritebackLatencyHiding = v->WritebackInterfaceBufferSize * 1024 + / (v->WritebackDestinationWidth[k] * v->WritebackDestinationHeight[k] + / (v->WritebackSourceHeight[k] * v->HTotal[k] / v->PixelClock[k]) * 4); + if (v->WritebackPixelFormat[k] == dm_444_64) WritebackLatencyHiding = WritebackLatencyHiding / 2; WritebackDRAMClockChangeLatencyMargin = WritebackLatencyHiding - - Watermark->WritebackDRAMClockChangeWatermark; + - v->Watermark.WritebackDRAMClockChangeWatermark; WritebackFCLKChangeLatencyMargin = WritebackLatencyHiding - - Watermark->WritebackFCLKChangeWatermark; + - v->Watermark.WritebackFCLKChangeWatermark; ActiveDRAMClockChangeLatencyMargin[k] = dml_min(ActiveDRAMClockChangeLatencyMargin[k], WritebackFCLKChangeLatencyMargin); @@ -4464,22 +4444,22 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( WritebackDRAMClockChangeLatencyMargin); } MaxActiveDRAMClockChangeLatencySupported[k] = - (UseMALLForPStateChange[k] == dm_use_mall_pstate_change_phantom_pipe) ? + (v->UsesMALLForPStateChange[k] == dm_use_mall_pstate_change_phantom_pipe) ? 0 : (ActiveDRAMClockChangeLatencyMargin[k] + mmSOCParameters.DRAMClockChangeLatency); } - for (i = 0; i < NumberOfActiveSurfaces; ++i) { - for (j = 0; j < NumberOfActiveSurfaces; ++j) { + for (i = 0; i < v->NumberOfActiveSurfaces; ++i) { + for (j = 0; j < v->NumberOfActiveSurfaces; ++j) { if (i == j || - (BlendingAndTiming[i] == i && BlendingAndTiming[j] == i) || - (BlendingAndTiming[j] == j && BlendingAndTiming[i] == j) || - (BlendingAndTiming[i] == BlendingAndTiming[j] && BlendingAndTiming[i] != i) || - (SynchronizeTimingsFinal && PixelClock[i] == PixelClock[j] && - HTotal[i] == HTotal[j] && VTotal[i] == VTotal[j] && - VActive[i] == VActive[j]) || (SynchronizeDRRDisplaysForUCLKPStateChangeFinal && - (DRRDisplay[i] || DRRDisplay[j]))) { + (v->BlendingAndTiming[i] == i && v->BlendingAndTiming[j] == i) || + (v->BlendingAndTiming[j] == j && v->BlendingAndTiming[i] == j) || + (v->BlendingAndTiming[i] == v->BlendingAndTiming[j] && v->BlendingAndTiming[i] != i) || + (v->SynchronizeTimingsFinal && v->PixelClock[i] == v->PixelClock[j] && + v->HTotal[i] == v->HTotal[j] && v->VTotal[i] == v->VTotal[j] && + v->VActive[i] == v->VActive[j]) || (v->SynchronizeDRRDisplaysForUCLKPStateChangeFinal && + (v->DRRDisplay[i] || v->DRRDisplay[j]))) { SynchronizedSurfaces[i][j] = true; } else { SynchronizedSurfaces[i][j] = false; @@ -4487,8 +4467,8 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( } } - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - if ((UseMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + if ((v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && (!FoundFirstSurfaceWithMinActiveFCLKChangeMargin || ActiveFCLKChangeLatencyMargin[k] < MinActiveFCLKChangeMargin)) { FoundFirstSurfaceWithMinActiveFCLKChangeMargin = true; @@ -4500,9 +4480,9 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( *MinActiveFCLKChangeLatencySupported = MinActiveFCLKChangeMargin + mmSOCParameters.FCLKChangeLatency; SameTimingForFCLKChange = true; - for (k = 0; k < NumberOfActiveSurfaces; ++k) { + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { if (!SynchronizedSurfaces[k][SurfaceWithMinActiveFCLKChangeMargin]) { - if ((UseMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && + if ((v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && (SameTimingForFCLKChange || ActiveFCLKChangeLatencyMargin[k] < SecondMinActiveFCLKChangeMarginOneDisplayInVBLank)) { @@ -4522,17 +4502,17 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( } *USRRetrainingSupport = true; - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - if ((UseMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + if ((v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe) && (USRRetrainingLatencyMargin[k] < 0)) { *USRRetrainingSupport = false; } } - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - if (UseMALLForPStateChange[k] != dm_use_mall_pstate_change_full_frame && - UseMALLForPStateChange[k] != dm_use_mall_pstate_change_sub_viewport && - UseMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe && + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + if (v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_full_frame && + v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_sub_viewport && + v->UsesMALLForPStateChange[k] != dm_use_mall_pstate_change_phantom_pipe && ActiveDRAMClockChangeLatencyMargin[k] < 0) { if (PrefetchMode > 0) { DRAMClockChangeSupportNumber = 2; @@ -4546,10 +4526,10 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( } } - for (k = 0; k < NumberOfActiveSurfaces; ++k) { - if (UseMALLForPStateChange[k] == dm_use_mall_pstate_change_full_frame) + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { + if (v->UsesMALLForPStateChange[k] == dm_use_mall_pstate_change_full_frame) DRAMClockChangeMethod = 1; - else if (UseMALLForPStateChange[k] == dm_use_mall_pstate_change_sub_viewport) + else if (v->UsesMALLForPStateChange[k] == dm_use_mall_pstate_change_sub_viewport) DRAMClockChangeMethod = 2; } @@ -4576,16 +4556,16 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( *DRAMClockChangeSupport = dm_dram_clock_change_unsupported; } - for (k = 0; k < NumberOfActiveSurfaces; ++k) { + for (k = 0; k < v->NumberOfActiveSurfaces; ++k) { unsigned int dst_y_pstate; unsigned int src_y_pstate_l; unsigned int src_y_pstate_c; unsigned int src_y_ahead_l, src_y_ahead_c, sub_vp_lines_l, sub_vp_lines_c; - dst_y_pstate = dml_ceil((mmSOCParameters.DRAMClockChangeLatency + mmSOCParameters.UrgentLatency) / (HTotal[k] / PixelClock[k]), 1); - src_y_pstate_l = dml_ceil(dst_y_pstate * VRatio[k], SwathHeightY[k]); + dst_y_pstate = dml_ceil((mmSOCParameters.DRAMClockChangeLatency + mmSOCParameters.UrgentLatency) / (v->HTotal[k] / v->PixelClock[k]), 1); + src_y_pstate_l = dml_ceil(dst_y_pstate * v->VRatio[k], SwathHeightY[k]); src_y_ahead_l = dml_floor(DETBufferSizeY[k] / BytePerPixelDETY[k] / SwathWidthY[k], SwathHeightY[k]) + LBLatencyHidingSourceLinesY[k]; - sub_vp_lines_l = src_y_pstate_l + src_y_ahead_l + meta_row_height[k]; + sub_vp_lines_l = src_y_pstate_l + src_y_ahead_l + v->meta_row_height[k]; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: k=%d, DETBufferSizeY = %d\n", __func__, k, DETBufferSizeY[k]); @@ -4596,21 +4576,21 @@ dml_print("DML::%s: k=%d, LBLatencyHidingSourceLinesY = %d\n", __func__, k, LBL dml_print("DML::%s: k=%d, dst_y_pstate = %d\n", __func__, k, dst_y_pstate); dml_print("DML::%s: k=%d, src_y_pstate_l = %d\n", __func__, k, src_y_pstate_l); dml_print("DML::%s: k=%d, src_y_ahead_l = %d\n", __func__, k, src_y_ahead_l); -dml_print("DML::%s: k=%d, meta_row_height = %d\n", __func__, k, meta_row_height[k]); +dml_print("DML::%s: k=%d, v->meta_row_height = %d\n", __func__, k, v->meta_row_height[k]); dml_print("DML::%s: k=%d, sub_vp_lines_l = %d\n", __func__, k, sub_vp_lines_l); #endif SubViewportLinesNeededInMALL[k] = sub_vp_lines_l; if (BytePerPixelDETC[k] > 0) { - src_y_pstate_c = dml_ceil(dst_y_pstate * VRatioChroma[k], SwathHeightC[k]); + src_y_pstate_c = dml_ceil(dst_y_pstate * v->VRatioChroma[k], SwathHeightC[k]); src_y_ahead_c = dml_floor(DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k], SwathHeightC[k]) + LBLatencyHidingSourceLinesC[k]; - sub_vp_lines_c = src_y_pstate_c + src_y_ahead_c + meta_row_height_chroma[k]; + sub_vp_lines_c = src_y_pstate_c + src_y_ahead_c + v->meta_row_height_chroma[k]; SubViewportLinesNeededInMALL[k] = dml_max(sub_vp_lines_l, sub_vp_lines_c); #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: k=%d, src_y_pstate_c = %d\n", __func__, k, src_y_pstate_c); dml_print("DML::%s: k=%d, src_y_ahead_c = %d\n", __func__, k, src_y_ahead_c); -dml_print("DML::%s: k=%d, meta_row_height_chroma = %d\n", __func__, k, meta_row_height_chroma[k]); +dml_print("DML::%s: k=%d, v->meta_row_height_chroma = %d\n", __func__, k, v->meta_row_height_chroma[k]); dml_print("DML::%s: k=%d, sub_vp_lines_c = %d\n", __func__, k, sub_vp_lines_c); #endif } @@ -4663,10 +4643,6 @@ void dml32_CalculateMinAndMaxPrefetchMode( } else if (AllowForPStateChangeOrStutterInVBlankFinal == dm_prefetch_support_uclk_fclk_and_stutter) { *MinPrefetchMode = 0; *MaxPrefetchMode = 0; - } else if (AllowForPStateChangeOrStutterInVBlankFinal == - dm_prefetch_support_uclk_fclk_and_stutter_if_possible) { - *MinPrefetchMode = 0; - *MaxPrefetchMode = 3; } else { *MinPrefetchMode = 0; *MaxPrefetchMode = 3; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index 626f6605e2d5b6504eabab072550e062d1fa3b31..55cead0d423747331cf617f0f9c2373977ba21a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -30,6 +30,7 @@ #include "os_types.h" #include "../dc_features.h" #include "../display_mode_structs.h" +#include "dml/display_mode_vba.h" unsigned int dml32_dscceComputeDelay( unsigned int bpc, @@ -215,6 +216,7 @@ void dml32_CalculateDETBufferSize( void dml32_CalculateODMMode( unsigned int MaximumPixelsPerLinePerDSCUnit, unsigned int HActive, + enum output_format_class OutFormat, enum output_encoder_class Output, enum odm_combine_policy ODMUse, double StateDispclk, @@ -713,28 +715,14 @@ double dml32_CalculateExtraLatency( unsigned int HostVMMaxNonCachedPageTableLevels); bool dml32_CalculatePrefetchSchedule( + struct vba_vars_st *v, + unsigned int k, double HostVMInefficiencyFactor, DmlPipe *myPipe, unsigned int DSCDelay, - double DPPCLKDelaySubtotalPlusCNVCFormater, - double DPPCLKDelaySCL, - double DPPCLKDelaySCLLBOnly, - double DPPCLKDelayCNVCCursor, - double DISPCLKDelaySubtotal, unsigned int DPP_RECOUT_WIDTH, - enum output_format_class OutputFormat, - unsigned int MaxInterDCNTileRepeaters, unsigned int VStartup, unsigned int MaxVStartup, - unsigned int GPUVMPageTableLevels, - bool GPUVMEnable, - bool HostVMEnable, - unsigned int HostVMMaxNonCachedPageTableLevels, - double HostVMMinPageSize, - bool DynamicMetadataEnable, - bool DynamicMetadataVMEnabled, - int DynamicMetadataLinesBeforeActiveRequired, - unsigned int DynamicMetadataTransmittedBytes, double UrgentLatency, double UrgentExtraLatency, double TCalc, @@ -808,58 +796,28 @@ void dml32_CalculateFlipSchedule( bool *ImmediateFlipSupportedForPipe); void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( - bool USRRetrainingRequiredFinal, - enum dm_use_mall_for_pstate_change_mode UseMALLForPStateChange[], + struct vba_vars_st *v, unsigned int PrefetchMode, - unsigned int NumberOfActiveSurfaces, - unsigned int MaxLineBufferLines, - unsigned int LineBufferSize, - unsigned int WritebackInterfaceBufferSize, double DCFCLK, double ReturnBW, - bool SynchronizeTimingsFinal, - bool SynchronizeDRRDisplaysForUCLKPStateChangeFinal, - bool DRRDisplay[], - unsigned int dpte_group_bytes[], - unsigned int meta_row_height[], - unsigned int meta_row_height_chroma[], SOCParametersList mmSOCParameters, - unsigned int WritebackChunkSize, double SOCCLK, double DCFClkDeepSleep, unsigned int DETBufferSizeY[], unsigned int DETBufferSizeC[], unsigned int SwathHeightY[], unsigned int SwathHeightC[], - unsigned int LBBitPerPixel[], double SwathWidthY[], double SwathWidthC[], - double HRatio[], - double HRatioChroma[], - unsigned int VTaps[], - unsigned int VTapsChroma[], - double VRatio[], - double VRatioChroma[], - unsigned int HTotal[], - unsigned int VTotal[], - unsigned int VActive[], - double PixelClock[], - unsigned int BlendingAndTiming[], unsigned int DPPPerSurface[], double BytePerPixelDETY[], double BytePerPixelDETC[], double DSTXAfterScaler[], double DSTYAfterScaler[], - bool WritebackEnable[], - enum source_format_class WritebackPixelFormat[], - double WritebackDestinationWidth[], - double WritebackDestinationHeight[], - double WritebackSourceHeight[], bool UnboundedRequestEnabled, unsigned int CompressedBufferSizeInkByte, /* Output */ - Watermarks *Watermark, enum clock_change_support *DRAMClockChangeSupport, double MaxActiveDRAMClockChangeLatencySupported[], unsigned int SubViewportLinesNeededInMALL[], diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index 7ebf25e87933f72bf0ed2d5b971721e31018def9..dd90f241e906527f376cb6e53acaedfefd3e7054 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -489,6 +489,7 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p if ((int)(dcn3_21_soc.urgent_latency_us * 1000) != dc->bb_overrides.urgent_latency_ns && dc->bb_overrides.urgent_latency_ns) { dcn3_21_soc.urgent_latency_us = dc->bb_overrides.urgent_latency_ns / 1000.0; + dcn3_21_soc.urgent_latency_pixel_data_only_us = dc->bb_overrides.urgent_latency_ns / 1000.0; } if ((int)(dcn3_21_soc.dram_clock_change_latency_us * 1000) diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c index 5d27ff0ebb5fa17e5c2ea7e4cce14af62191f9eb..4125d3d111d150cc910137be19c333ce6b882ad1 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c @@ -35,6 +35,8 @@ #include "dcn30/display_rq_dlg_calc_30.h" #include "dcn31/display_mode_vba_31.h" #include "dcn31/display_rq_dlg_calc_31.h" +#include "dcn314/display_mode_vba_314.h" +#include "dcn314/display_rq_dlg_calc_314.h" #include "dcn32/display_mode_vba_32.h" #include "dcn32/display_rq_dlg_calc_32.h" #include "dml_logger.h" @@ -74,6 +76,13 @@ const struct dml_funcs dml31_funcs = { .rq_dlg_get_rq_reg = dml31_rq_dlg_get_rq_reg }; +const struct dml_funcs dml314_funcs = { + .validate = dml314_ModeSupportAndSystemConfigurationFull, + .recalculate = dml314_recalculate, + .rq_dlg_get_dlg_reg = dml314_rq_dlg_get_dlg_reg, + .rq_dlg_get_rq_reg = dml314_rq_dlg_get_rq_reg +}; + const struct dml_funcs dml32_funcs = { .validate = dml32_ModeSupportAndSystemConfigurationFull, .recalculate = dml32_recalculate, @@ -105,8 +114,12 @@ void dml_init_instance(struct display_mode_lib *lib, break; case DML_PROJECT_DCN31: case DML_PROJECT_DCN31_FPGA: + case DML_PROJECT_DCN315: lib->funcs = dml31_funcs; break; + case DML_PROJECT_DCN314: + lib->funcs = dml314_funcs; + break; case DML_PROJECT_DCN32: lib->funcs = dml32_funcs; break; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h index 2bdd6ed22611d266223ca35f59a14e0d5cd99767..3d643d50c3eb59e69e99c4865dfb06803c1404fe 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h @@ -40,7 +40,9 @@ enum dml_project { DML_PROJECT_DCN21, DML_PROJECT_DCN30, DML_PROJECT_DCN31, + DML_PROJECT_DCN315, DML_PROJECT_DCN31_FPGA, + DML_PROJECT_DCN314, DML_PROJECT_DCN32, }; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index c596187a1e096ce6ede544754653985f209c5fb4..f33a8879b05ad66f87072b4efd634de020b15b35 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -510,6 +510,7 @@ struct _vcs_dpi_display_pipe_dest_params_st { unsigned int htotal; unsigned int vtotal; unsigned int vfront_porch; + unsigned int vblank_nom; unsigned int vactive; unsigned int hactive; unsigned int vstartup_start; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index 503e7d984ff03a82cc6dd676552a4bdc930d1e59..03924aed8d5c77bc856b4a8c0200435957b45959 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -597,6 +597,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) mode_lib->vba.HTotal[mode_lib->vba.NumberOfActivePlanes] = dst->htotal; mode_lib->vba.VTotal[mode_lib->vba.NumberOfActivePlanes] = dst->vtotal; mode_lib->vba.VFrontPorch[mode_lib->vba.NumberOfActivePlanes] = dst->vfront_porch; + mode_lib->vba.VBlankNom[mode_lib->vba.NumberOfActivePlanes] = dst->vblank_nom; mode_lib->vba.DCCFractionOfZeroSizeRequestsLuma[mode_lib->vba.NumberOfActivePlanes] = src->dcc_fraction_of_zs_req_luma; mode_lib->vba.DCCFractionOfZeroSizeRequestsChroma[mode_lib->vba.NumberOfActivePlanes] = src->dcc_fraction_of_zs_req_chroma; mode_lib->vba.DCCEnable[mode_lib->vba.NumberOfActivePlanes] = diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h index da8acf59ccac1ef1bb85d13bcf159c8394330bef..630f3395e90a087ff454384b391b4fa31390f04e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h @@ -652,10 +652,10 @@ struct vba_vars_st { unsigned int OutputTypeAndRatePerState[DC__VOLTAGE_STATES][DC__NUM_DPP__MAX]; double RequiredDISPCLKPerSurface[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX]; - unsigned int MicroTileHeightY[DC__NUM_DPP__MAX]; - unsigned int MicroTileHeightC[DC__NUM_DPP__MAX]; - unsigned int MicroTileWidthY[DC__NUM_DPP__MAX]; - unsigned int MicroTileWidthC[DC__NUM_DPP__MAX]; + unsigned int MacroTileHeightY[DC__NUM_DPP__MAX]; + unsigned int MacroTileHeightC[DC__NUM_DPP__MAX]; + unsigned int MacroTileWidthY[DC__NUM_DPP__MAX]; + unsigned int MacroTileWidthC[DC__NUM_DPP__MAX]; bool ImmediateFlipRequiredFinal; bool DCCProgrammingAssumesScanDirectionUnknownFinal; bool EnoughWritebackUnits; @@ -801,8 +801,6 @@ struct vba_vars_st { double PSCL_FACTOR[DC__NUM_DPP__MAX]; double PSCL_FACTOR_CHROMA[DC__NUM_DPP__MAX]; double MaximumVStartup[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX]; - unsigned int MacroTileWidthY[DC__NUM_DPP__MAX]; - unsigned int MacroTileWidthC[DC__NUM_DPP__MAX]; double AlignedDCCMetaPitch[DC__NUM_DPP__MAX]; double AlignedYPitch[DC__NUM_DPP__MAX]; double AlignedCPitch[DC__NUM_DPP__MAX]; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h index 479d7d83220c2bf4d01e96bc8132a220d5c148e1..072bd05396059448350c387b6e7cb769774fee3c 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h @@ -76,14 +76,9 @@ static inline double dml_floor(double a, double granularity) static inline double dml_round(double a) { - double round_pt = 0.5; - double ceil = dml_ceil(a, 1); - double floor = dml_floor(a, 1); + const double round_pt = 0.5; - if (a - floor >= round_pt) - return ceil; - else - return floor; + return dml_floor(a + round_pt, 1); } /* float diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index 8919a2092ac508683e44b79f415777a8a1568baa..9498105c98ab39a35063ec4b23f256888b681196 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -39,6 +39,8 @@ #include "panel_cntl.h" #define MAX_CLOCK_SOURCES 7 +#define MAX_SVP_PHANTOM_STREAMS 2 +#define MAX_SVP_PHANTOM_PLANES 2 void enable_surface_flip_reporting(struct dc_plane_state *plane_state, uint32_t controller_id); @@ -232,6 +234,7 @@ struct resource_funcs { unsigned int index); bool (*remove_phantom_pipes)(struct dc *dc, struct dc_state *context); + void (*get_panel_config_defaults)(struct dc_panel_config *panel_config); }; struct audio_support{ @@ -438,7 +441,6 @@ struct pipe_ctx { union pipe_update_flags update_flags; struct dwbc *dwbc; struct mcif_wb *mcif_wb; - bool vtp_locked; }; /* Data used for dynamic link encoder assignment. @@ -492,6 +494,8 @@ struct dcn_bw_output { struct dcn_watermark_set watermarks; struct dcn_bw_writeback bw_writeback; int compbuf_size_kb; + unsigned int legacy_svp_drr_stream_index; + bool legacy_svp_drr_stream_index_valid; }; union bw_output { diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h index 6682d9e181c6e328c6343bb61c173cbc38ef3e74..b304d450b038a4175bfb07253b002a568356e91a 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h +++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h @@ -194,6 +194,11 @@ enum dc_status dpcd_configure_lttpr_mode( enum dp_link_encoding dp_get_link_encoding_format(const struct dc_link_settings *link_settings); bool dp_retrieve_lttpr_cap(struct dc_link *link); +bool dp_is_lttpr_present(struct dc_link *link); +enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link, struct dc_link_settings *link_setting); +void dp_get_lttpr_mode_override(struct dc_link *link, enum lttpr_mode *override); +enum lttpr_mode dp_decide_8b_10b_lttpr_mode(struct dc_link *link); +enum lttpr_mode dp_decide_128b_132b_lttpr_mode(struct dc_link *link); bool dpcd_write_128b_132b_sst_payload_allocation_table( const struct dc_stream_state *stream, struct dc_link *link, diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h index 806f3041db1419ca505c5b842768f22f1da289c3..9e4ddc985240655fe574522387a39efe4a88c6cc 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h +++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h @@ -628,8 +628,23 @@ unsigned int dcn_find_dcfclk_suits_all( const struct dc *dc, struct dc_clocks *clocks); -void dcn_bw_update_from_pplib(struct dc *dc); -void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc); +void dcn_get_soc_clks( + struct dc *dc, + int *min_fclk_khz, + int *min_dcfclk_khz, + int *socclk_khz); + +void dcn_bw_update_from_pplib_fclks( + struct dc *dc, + struct dm_pp_clock_levels_with_voltage *fclks); +void dcn_bw_update_from_pplib_dcfclks( + struct dc *dc, + struct dm_pp_clock_levels_with_voltage *dcfclks); +void dcn_bw_notify_pplib_of_wm_ranges( + struct dc *dc, + int min_fclk_khz, + int min_dcfclk_khz, + int socclk_khz); void dcn_bw_sync_calcs_and_dml(struct dc *dc); enum source_macro_tile_size swizzle_mode_to_macro_tile_size(enum swizzle_mode_values sw_mode); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h index 5d2b028e5dadf9564494ac924444e0235c7146b7..591ab1389e3b3d5a9bfe6ff42c03a08f1e851c00 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h @@ -95,10 +95,23 @@ struct clk_limit_table_entry { unsigned int wck_ratio; }; +struct clk_limit_num_entries { + unsigned int num_dcfclk_levels; + unsigned int num_fclk_levels; + unsigned int num_memclk_levels; + unsigned int num_socclk_levels; + unsigned int num_dtbclk_levels; + unsigned int num_dispclk_levels; + unsigned int num_dppclk_levels; + unsigned int num_phyclk_levels; + unsigned int num_phyclk_d18_levels; +}; + /* This table is contiguous */ struct clk_limit_table { struct clk_limit_table_entry entries[MAX_NUM_DPM_LVL]; - unsigned int num_entries; + struct clk_limit_num_entries num_entries_per_clk; + unsigned int num_entries; /* highest populated dpm level for back compatibility */ }; struct wm_range_table_entry { @@ -214,6 +227,7 @@ struct dummy_pstate_entry { struct clk_bw_params { unsigned int vram_type; unsigned int num_channels; + unsigned int dram_channel_width_bytes; unsigned int dispclk_vco_khz; unsigned int dc_mode_softmax_memclk; struct clk_limit_table clk_table; diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h index 68c2ed434d2c4fb9ce5644c9789359d6bcb21a32..cff5fd55a0ad73e684d7ebe0c97ef043b8e2b857 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h @@ -340,6 +340,8 @@ struct clk_mgr_internal { bool smu_present; void *wm_range_table; long long wm_range_table_addr; + + bool dpm_present; }; struct clk_mgr_internal_funcs { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/cursor_reg_cache.h b/drivers/gpu/drm/amd/display/dc/inc/hw/cursor_reg_cache.h new file mode 100644 index 0000000000000000000000000000000000000000..45645f9fd86c42597e593d338551bb036dba4c66 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/cursor_reg_cache.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: MIT */ +/* Copyright © 2022 Advanced Micro Devices, Inc. All rights reserved. */ + +#ifndef __DAL_CURSOR_CACHE_H__ +#define __DAL_CURSOR_CACHE_H__ + +union reg_cursor_control_cfg { + struct { + uint32_t cur_enable: 1; + uint32_t reser0: 3; + uint32_t cur_2x_magnify: 1; + uint32_t reser1: 3; + uint32_t mode: 3; + uint32_t reser2: 5; + uint32_t pitch: 2; + uint32_t reser3: 6; + uint32_t line_per_chunk: 5; + uint32_t reser4: 3; + } bits; + uint32_t raw; +}; +struct cursor_position_cache_hubp { + union reg_cursor_control_cfg cur_ctl; + union reg_position_cfg { + struct { + uint32_t x_pos: 16; + uint32_t y_pos: 16; + } bits; + uint32_t raw; + } position; + union reg_hot_spot_cfg { + struct { + uint32_t x_hot: 16; + uint32_t y_hot: 16; + } bits; + uint32_t raw; + } hot_spot; + union reg_dst_offset_cfg { + struct { + uint32_t dst_x_offset: 13; + uint32_t reserved: 19; + } bits; + uint32_t raw; + } dst_offset; +}; + +struct cursor_attribute_cache_hubp { + uint32_t SURFACE_ADDR_HIGH; + uint32_t SURFACE_ADDR; + union reg_cursor_control_cfg cur_ctl; + union reg_cursor_size_cfg { + struct { + uint32_t width: 16; + uint32_t height: 16; + } bits; + uint32_t raw; + } size; + union reg_cursor_settings_cfg { + struct { + uint32_t dst_y_offset: 8; + uint32_t chunk_hdl_adjust: 2; + uint32_t reserved: 22; + } bits; + uint32_t raw; + } settings; +}; + +struct cursor_rect { + uint32_t x; + uint32_t y; + uint32_t w; + uint32_t h; +}; + +union reg_cur0_control_cfg { + struct { + uint32_t cur0_enable: 1; + uint32_t expansion_mode: 1; + uint32_t reser0: 1; + uint32_t cur0_rom_en: 1; + uint32_t mode: 3; + uint32_t reserved: 25; + } bits; + uint32_t raw; +}; +struct cursor_position_cache_dpp { + union reg_cur0_control_cfg cur0_ctl; +}; + +struct cursor_attribute_cache_dpp { + union reg_cur0_control_cfg cur0_ctl; +}; + +struct cursor_attributes_cfg { + struct cursor_attribute_cache_hubp aHubp; + struct cursor_attribute_cache_dpp aDpp; +}; + +#endif diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h index 3ef7faa920528df154890487fda76531f5d8c02f..dcb80c4747b04cd281736684bf358303d4f550aa 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dpp.h @@ -28,6 +28,7 @@ #define __DAL_DPP_H__ #include "transform.h" +#include "cursor_reg_cache.h" union defer_reg_writes { struct { @@ -58,6 +59,9 @@ struct dpp { struct pwl_params shaper_params; bool cm_bypass_mode; + + struct cursor_position_cache_dpp pos; + struct cursor_attribute_cache_dpp att; }; struct dpp_input_csc_matrix { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h index 44c4578193a348aaffb2fd0149fb9a796f13bb76..d5ea7545583e87f09b0f89afbe8c2d59d00d2122 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h @@ -27,6 +27,7 @@ #define __DAL_HUBP_H__ #include "mem_input.h" +#include "cursor_reg_cache.h" #define OPP_ID_INVALID 0xf #define MAX_TTU 0xffffff @@ -65,6 +66,10 @@ struct hubp { struct dc_cursor_attributes curs_attr; struct dc_cursor_position curs_pos; bool power_gated; + + struct cursor_position_cache_hubp pos; + struct cursor_attribute_cache_hubp att; + struct cursor_rect cur_rect; }; struct surface_flip_registers { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h index 437b64e8737732f11ec17d5dbe768d3aedc33328..cd2be729846b4684beab93c3e7aeca0db0cc2495 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h @@ -268,10 +268,18 @@ enum dc_lut_mode { LUT_RAM_B }; -enum phy_state { - TX_OFF_SYMCLK_OFF, - TX_ON_SYMCLK_ON, - TX_OFF_SYMCLK_ON +enum symclk_state { + SYMCLK_OFF_TX_OFF, + SYMCLK_ON_TX_ON, + SYMCLK_ON_TX_OFF, +}; + +struct phy_state { + struct { + uint8_t otg : 1; + uint8_t reserved : 7; + } symclk_ref_cnts; + enum symclk_state symclk_state; }; /** diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index 72eef7a5ed83a5f10b5e5b41c7782199dbbd59a0..25a1df45b2641a8763b98f6d991755a27a564246 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -209,7 +209,6 @@ struct timing_generator_funcs { void (*set_blank)(struct timing_generator *tg, bool enable_blanking); bool (*is_blanked)(struct timing_generator *tg); - bool (*is_locked)(struct timing_generator *tg); void (*set_overscan_blank_color) (struct timing_generator *tg, const struct tg_color *color); void (*set_blank_color)(struct timing_generator *tg, const struct tg_color *color); void (*set_colors)(struct timing_generator *tg, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 52b4350c9cd8aec5727e6cb8debb4038c706270b..d04b68dad413b5298552f6f71746066c871c09bd 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -32,11 +32,6 @@ #include "inc/hw/link_encoder.h" #include "core_status.h" -enum vline_select { - VLINE0, - VLINE1 -}; - struct pipe_ctx; struct dc_state; struct dc_stream_status; @@ -48,6 +43,7 @@ struct dc_phy_addr_space_config; struct dc_virtual_addr_space_config; struct dpp; struct dce_hwseq; +struct link_resource; struct hw_sequencer_funcs { void (*hardware_release)(struct dc *dc); @@ -88,6 +84,7 @@ struct hw_sequencer_funcs { struct pipe_ctx *pipe_ctx, bool enableTripleBuffer); void (*update_pending_status)(struct pipe_ctx *pipe_ctx); void (*power_down)(struct dc *dc); + void (*update_dsc_pg)(struct dc *dc, struct dc_state *context, bool safe_to_disable); /* Pipe Lock Related */ void (*pipe_control_lock)(struct dc *dc, @@ -116,8 +113,7 @@ struct hw_sequencer_funcs { int group_index, int group_size, struct pipe_ctx *grouped_pipes[]); void (*setup_periodic_interrupt)(struct dc *dc, - struct pipe_ctx *pipe_ctx, - enum vline_select vline); + struct pipe_ctx *pipe_ctx); void (*set_drr)(struct pipe_ctx **pipe_ctx, int num_pipes, struct dc_crtc_timing_adjust adjust); void (*set_static_screen_control)(struct pipe_ctx **pipe_ctx, @@ -218,6 +214,25 @@ struct hw_sequencer_funcs { void (*set_pipe)(struct pipe_ctx *pipe_ctx); + void (*enable_dp_link_output)(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + const struct dc_link_settings *link_settings); + void (*enable_tmds_link_output)(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + uint32_t pixel_clock); + void (*enable_lvds_link_output)(struct dc_link *link, + const struct link_resource *link_res, + enum clock_source_id clock_source, + uint32_t pixel_clock); + void (*disable_link_output)(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal); + void (*get_dcc_en_bits)(struct dc *dc, int *dcc_en_bits); /* Idle Optimization Related */ @@ -245,7 +260,9 @@ struct hw_sequencer_funcs { struct tg_color *color, int mpcc_id); - void (*update_phy_state)(struct dc_state *state, struct pipe_ctx *pipe_ctx, enum phy_state target_state); + void (*update_phantom_vp_position)(struct dc *dc, + struct dc_state *context, + struct pipe_ctx *phantom_pipe); void (*commit_subvp_config)(struct dc *dc, struct dc_state *context); void (*subvp_pipe_control_lock)(struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h index 1cdea0efe5c1ac23b8f606e06daa5abc3c67bfd4..a4d61bb724b67cd7065bee030564d7b6f61638dd 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h @@ -124,6 +124,8 @@ struct hwseq_private_funcs { void (*dsc_pg_control)(struct dce_hwseq *hws, unsigned int dsc_inst, bool power_on); + bool (*dsc_pg_status)(struct dce_hwseq *hws, + unsigned int dsc_inst); void (*update_odm)(struct dc *dc, struct dc_state *context, struct pipe_ctx *pipe_ctx); void (*program_all_writeback_pipes_in_tree)(struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h index 3482a877b6af567477d0eca3d1596daaf73bd209..89964c980b8712107415f802614fa4312a764504 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h +++ b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h @@ -55,9 +55,6 @@ struct link_hwss_ext { enum signal_type signal, enum clock_source_id clock_source, const struct dc_link_settings *link_settings); - void (*disable_dp_link_output)(struct dc_link *link, - const struct link_resource *link_res, - enum signal_type signal); void (*set_dp_link_test_pattern)(struct dc_link *link, const struct link_resource *link_res, struct encoder_set_dp_phy_pattern_param *tp_params); @@ -79,6 +76,9 @@ struct link_hwss { void (*setup_stream_encoder)(struct pipe_ctx *pipe_ctx); void (*reset_stream_encoder)(struct pipe_ctx *pipe_ctx); void (*setup_stream_attribute)(struct pipe_ctx *pipe_ctx); + void (*disable_link_output)(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal); }; #endif /* __DC_LINK_HWSS_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h index 58158764adc0a6f94c326f10f734e9a094abf93e..5040836f404d05ed4d393481694b3553026c81be 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/resource.h +++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h @@ -219,9 +219,21 @@ void check_syncd_pipes_for_disabled_master_pipe(struct dc *dc, struct dc_state *context, uint8_t disabled_master_pipe_idx); +void reset_sync_context_for_pipe(const struct dc *dc, + struct dc_state *context, + uint8_t pipe_idx); + uint8_t resource_transmitter_to_phy_idx(const struct dc *dc, enum transmitter transmitter); const struct link_hwss *get_link_hwss(const struct dc_link *link, const struct link_resource *link_res); +bool is_h_timing_divisible_by_2(struct dc_stream_state *stream); + +bool dc_resource_acquire_secondary_pipe_for_mpc_odm( + const struct dc *dc, + struct dc_state *state, + struct pipe_ctx *pri_pipe, + struct pipe_ctx *sec_pipe, + bool odm); #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_RESOURCE_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c index 5e92019539c823268ed049a73e378e7048dace92..4227adbc646a893a8a9b09a723dab16a1560804f 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c @@ -130,7 +130,7 @@ void enable_dio_dp_link_output(struct dc_link *link, dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY); } -void disable_dio_dp_link_output(struct dc_link *link, +void disable_dio_link_output(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal) { @@ -174,10 +174,10 @@ static const struct link_hwss dio_link_hwss = { .setup_stream_encoder = setup_dio_stream_encoder, .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, + .disable_link_output = disable_dio_link_output, .ext = { .set_throttled_vcp_size = set_dio_throttled_vcp_size, .enable_dp_link_output = enable_dio_dp_link_output, - .disable_dp_link_output = disable_dio_dp_link_output, .set_dp_link_test_pattern = set_dio_dp_link_test_pattern, .set_dp_lane_settings = set_dio_dp_lane_settings, .update_stream_allocation_table = update_dio_stream_allocation_table, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h index 08f22b32df48dca5d5c024b2e288937f0498a04e..126d37f847a157c0c84bca902245c346e14dc95f 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h @@ -40,7 +40,7 @@ void enable_dio_dp_link_output(struct dc_link *link, enum signal_type signal, enum clock_source_id clock_source, const struct dc_link_settings *link_settings); -void disable_dio_dp_link_output(struct dc_link *link, +void disable_dio_link_output(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal); void set_dio_dp_link_test_pattern(struct dc_link *link, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c index 89d4e8159138cc6de760f0a52cecd8315e0a8b17..64f7ea6a9aa3241a8f5748ca2b10b7f85c33227f 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c @@ -56,10 +56,10 @@ static const struct link_hwss dpia_link_hwss = { .setup_stream_encoder = setup_dio_stream_encoder, .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, + .disable_link_output = disable_dio_link_output, .ext = { .set_throttled_vcp_size = set_dio_throttled_vcp_size, .enable_dp_link_output = enable_dio_dp_link_output, - .disable_dp_link_output = disable_dio_dp_link_output, .set_dp_link_test_pattern = set_dio_dp_link_test_pattern, .set_dp_lane_settings = set_dio_dp_lane_settings, .update_stream_allocation_table = update_dpia_stream_allocation_table, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c index 226af06278ce41498dc511e843908afc8ea19a28..153a88381f2c76571f85d95e9a32ace94682ca96 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c @@ -111,7 +111,7 @@ static void setup_hpo_dp_stream_encoder(struct pipe_ctx *pipe_ctx) enum phyd32clk_clock_source phyd32clk = get_phyd32clk_src(pipe_ctx->stream->link); dto_params.otg_inst = tg->inst; - dto_params.pixclk_khz = pipe_ctx->stream->phy_pix_clk; + dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10; dto_params.num_odm_segments = get_odm_segment_count(pipe_ctx); dto_params.timing = &pipe_ctx->stream->timing; dto_params.ref_dtbclk_khz = dc->clk_mgr->funcs->get_dtb_ref_clk_frequency(dc->clk_mgr); @@ -266,11 +266,11 @@ static const struct link_hwss hpo_dp_link_hwss = { .setup_stream_encoder = setup_hpo_dp_stream_encoder, .reset_stream_encoder = reset_hpo_dp_stream_encoder, .setup_stream_attribute = setup_hpo_dp_stream_attribute, + .disable_link_output = disable_hpo_dp_link_output, .ext = { .set_throttled_vcp_size = set_hpo_dp_throttled_vcp_size, .set_hblank_min_symbol_width = set_hpo_dp_hblank_min_symbol_width, .enable_dp_link_output = enable_hpo_dp_link_output, - .disable_dp_link_output = disable_hpo_dp_link_output, .set_dp_link_test_pattern = set_hpo_dp_link_test_pattern, .set_dp_lane_settings = set_hpo_dp_lane_settings, .update_stream_allocation_table = update_hpo_dp_stream_allocation_table, diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c index 501173ce270ed7db4dbc9afa0a30ea2762633d11..4f7f99156897b86566c446d543ac7ea953825741 100644 --- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c +++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c @@ -36,10 +36,18 @@ void virtual_setup_stream_attribute(struct pipe_ctx *pipe_ctx) void virtual_reset_stream_encoder(struct pipe_ctx *pipe_ctx) { } + +static void virtual_disable_link_output(struct dc_link *link, + const struct link_resource *link_res, + enum signal_type signal) +{ +} + static const struct link_hwss virtual_link_hwss = { .setup_stream_encoder = virtual_setup_stream_encoder, .reset_stream_encoder = virtual_reset_stream_encoder, .setup_stream_attribute = virtual_setup_stream_attribute, + .disable_link_output = virtual_disable_link_output, }; const struct link_hwss *get_virtual_link_hwss(void) diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h index f34c45b19fcb2c04c078637668fc00283518eca6..eb5b7eb292ef30afe2757bad0bd6c63797ebe512 100644 --- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h +++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h @@ -248,6 +248,7 @@ struct dmub_srv_hw_params { bool disable_dpia; bool usb4_cm_version; bool fw_in_system_memory; + bool dpia_hpd_int_enable_supported; }; /** diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 7cddbc431b57bd9d75ad0bc2d34242be861e8f71..7a8f61517424c6d9588beb55a67681073bd61591 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -400,8 +400,9 @@ union dmub_fw_boot_options { uint32_t diag_env: 1; /* 1 if diagnostic environment */ uint32_t gpint_scratch8: 1; /* 1 if GPINT is in scratch8*/ uint32_t usb4_cm_version: 1; /**< 1 CM support */ + uint32_t dpia_hpd_int_enable_supported: 1; /* 1 if dpia hpd int enable supported */ - uint32_t reserved : 17; /**< reserved */ + uint32_t reserved : 16; /**< reserved */ } bits; /**< boot bits */ uint32_t all; /**< 32-bit access to bits */ }; @@ -728,6 +729,12 @@ enum dmub_cmd_type { /** * Command type used for all VBIOS interface commands. */ + + /** + * Command type used to set DPIA HPD interrupt state + */ + DMUB_CMD__DPIA_HPD_INT_ENABLE = 86, + DMUB_CMD__VBIOS = 128, }; @@ -998,7 +1005,8 @@ struct dmub_cmd_fw_assisted_mclk_switch_pipe_data_v2 { uint8_t scale_factor_numerator; uint8_t scale_factor_denominator; uint8_t is_drr; - uint8_t pad[2]; + uint8_t main_split_pipe_index; + uint8_t phantom_split_pipe_index; } subvp_data; struct { @@ -1254,6 +1262,14 @@ struct dmub_rb_cmd_set_mst_alloc_slots { struct dmub_cmd_mst_alloc_slots_control_data mst_slots_control; /* mst slots control */ }; +/** + * DMUB command structure for DPIA HPD int enable control. + */ +struct dmub_rb_cmd_dpia_hpd_int_enable { + struct dmub_cmd_header header; /* header */ + uint32_t enable; /* dpia hpd interrupt enable */ +}; + /** * struct dmub_rb_cmd_dpphy_init - DPPHY init. */ @@ -2083,7 +2099,99 @@ struct dmub_rb_cmd_update_dirty_rect { /** * Data passed from driver to FW in a DMUB_CMD__UPDATE_CURSOR_INFO command. */ -struct dmub_cmd_update_cursor_info_data { +union dmub_reg_cursor_control_cfg { + struct { + uint32_t cur_enable: 1; + uint32_t reser0: 3; + uint32_t cur_2x_magnify: 1; + uint32_t reser1: 3; + uint32_t mode: 3; + uint32_t reser2: 5; + uint32_t pitch: 2; + uint32_t reser3: 6; + uint32_t line_per_chunk: 5; + uint32_t reser4: 3; + } bits; + uint32_t raw; +}; +struct dmub_cursor_position_cache_hubp { + union dmub_reg_cursor_control_cfg cur_ctl; + union dmub_reg_position_cfg { + struct { + uint32_t cur_x_pos: 16; + uint32_t cur_y_pos: 16; + } bits; + uint32_t raw; + } position; + union dmub_reg_hot_spot_cfg { + struct { + uint32_t hot_x: 16; + uint32_t hot_y: 16; + } bits; + uint32_t raw; + } hot_spot; + union dmub_reg_dst_offset_cfg { + struct { + uint32_t dst_x_offset: 13; + uint32_t reserved: 19; + } bits; + uint32_t raw; + } dst_offset; +}; + +union dmub_reg_cur0_control_cfg { + struct { + uint32_t cur0_enable: 1; + uint32_t expansion_mode: 1; + uint32_t reser0: 1; + uint32_t cur0_rom_en: 1; + uint32_t mode: 3; + uint32_t reserved: 25; + } bits; + uint32_t raw; +}; +struct dmub_cursor_position_cache_dpp { + union dmub_reg_cur0_control_cfg cur0_ctl; +}; +struct dmub_cursor_position_cfg { + struct dmub_cursor_position_cache_hubp pHubp; + struct dmub_cursor_position_cache_dpp pDpp; + uint8_t pipe_idx; + /* + * Padding is required. To be 4 Bytes Aligned. + */ + uint8_t padding[3]; +}; + +struct dmub_cursor_attribute_cache_hubp { + uint32_t SURFACE_ADDR_HIGH; + uint32_t SURFACE_ADDR; + union dmub_reg_cursor_control_cfg cur_ctl; + union dmub_reg_cursor_size_cfg { + struct { + uint32_t width: 16; + uint32_t height: 16; + } bits; + uint32_t raw; + } size; + union dmub_reg_cursor_settings_cfg { + struct { + uint32_t dst_y_offset: 8; + uint32_t chunk_hdl_adjust: 2; + uint32_t reserved: 22; + } bits; + uint32_t raw; + } settings; +}; +struct dmub_cursor_attribute_cache_dpp { + union dmub_reg_cur0_control_cfg cur0_ctl; +}; +struct dmub_cursor_attributes_cfg { + struct dmub_cursor_attribute_cache_hubp aHubp; + struct dmub_cursor_attribute_cache_dpp aDpp; +}; + +struct dmub_cmd_update_cursor_payload0 { /** * Cursor dirty rects. */ @@ -2110,6 +2218,20 @@ struct dmub_cmd_update_cursor_info_data { * Currently the support is only for 0 or 1 */ uint8_t panel_inst; + /** + * Cursor Position Register. + * Registers contains Hubp & Dpp modules + */ + struct dmub_cursor_position_cfg position_cfg; +}; + +struct dmub_cmd_update_cursor_payload1 { + struct dmub_cursor_attributes_cfg attribute_cfg; +}; + +union dmub_cmd_update_cursor_info_data { + struct dmub_cmd_update_cursor_payload0 payload0; + struct dmub_cmd_update_cursor_payload1 payload1; }; /** * Definition of a DMUB_CMD__UPDATE_CURSOR_INFO command. @@ -2122,7 +2244,7 @@ struct dmub_rb_cmd_update_cursor_info { /** * Data passed from driver to FW in a DMUB_CMD__UPDATE_CURSOR_INFO command. */ - struct dmub_cmd_update_cursor_info_data update_cursor_info_data; + union dmub_cmd_update_cursor_info_data update_cursor_info_data; }; /** @@ -2819,11 +2941,7 @@ struct dmub_rb_cmd_get_visual_confirm_color { struct dmub_optc_state { uint32_t v_total_max; uint32_t v_total_min; - uint32_t v_total_mid; - uint32_t v_total_mid_frame_num; uint32_t tg_inst; - uint32_t enable_manual_trigger; - uint32_t clear_force_vsync; }; struct dmub_rb_cmd_drr_update { @@ -3229,6 +3347,10 @@ union dmub_rb_cmd { * Definition of a DMUB_CMD__QUERY_HPD_STATE command. */ struct dmub_rb_cmd_query_hpd_state query_hpd; + /** + * Definition of a DMUB_CMD__DPIA_HPD_INT_ENABLE command. + */ + struct dmub_rb_cmd_dpia_hpd_int_enable dpia_hpd_int_enable; }; /** diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c index c7bd7e216710944ab3786a17a31ed68a5c4c66b7..c90b9ee42e126d67b225eed32b759e0287c8e704 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn31.c @@ -350,6 +350,7 @@ void dmub_dcn31_enable_dmub_boot_options(struct dmub_srv *dmub, const struct dmu boot_options.bits.dpia_supported = params->dpia_supported; boot_options.bits.enable_dpia = params->disable_dpia ? 0 : 1; boot_options.bits.usb4_cm_version = params->usb4_cm_version; + boot_options.bits.dpia_hpd_int_enable_supported = params->dpia_hpd_int_enable_supported; boot_options.bits.power_optimization = params->power_optimization; boot_options.bits.sel_mux_phy_c_d_phy_f_g = (dmub->asic == DMUB_ASIC_DCN31B) ? 1 : 0; diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h index 9f3558c0ef110c4a524058f36116a9fb759a585c..c3089c673975b9548355ee0303c6f2afeae454f1 100644 --- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h +++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h @@ -215,6 +215,7 @@ enum { #define DEVICE_ID_NV_143F 0x143F #define FAMILY_VGH 144 #define DEVICE_ID_VGH_163F 0x163F +#define DEVICE_ID_VGH_1435 0x1435 #define VANGOGH_A0 0x01 #define VANGOGH_UNKNOWN 0xFF diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h index 05096c644a60b00d5effebc1c3a024e10285d229..a7ba5bd8dc16ad0a3d16deb75309b5c706648f93 100644 --- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h +++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h @@ -128,8 +128,8 @@ struct av_sync_data { uint8_t aud_del_ins3;/* DPCD 0002Dh */ }; -static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3, 0}; -static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5, 0}; +static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3}; +static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5}; static const u8 DP_SINK_BRANCH_DEV_NAME_7580[] = "7580\x80u"; diff --git a/drivers/gpu/drm/amd/display/include/link_service_types.h b/drivers/gpu/drm/amd/display/include/link_service_types.h index d76ab72baf0ca136b81bff097e9dc187914ee4b5..d1e91d31d1519a9f27007e48f7e552e1bd38a58d 100644 --- a/drivers/gpu/drm/amd/display/include/link_service_types.h +++ b/drivers/gpu/drm/amd/display/include/link_service_types.h @@ -83,6 +83,7 @@ enum link_training_result { }; enum lttpr_mode { + LTTPR_MODE_UNKNOWN, LTTPR_MODE_NON_LTTPR, LTTPR_MODE_TRANSPARENT, LTTPR_MODE_NON_TRANSPARENT, diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c index 859ffd8725c5caa76882461072fc1a75b725b42c..447a0ec9cbe21cd79407c5add9abbaba31d2dff5 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c @@ -1600,6 +1600,7 @@ static void interpolate_user_regamma(uint32_t hw_points_num, struct fixed31_32 lut2; struct fixed31_32 delta_lut; struct fixed31_32 delta_index; + const struct fixed31_32 one = dc_fixpt_from_int(1); i = 0; /* fixed_pt library has problems handling too small values */ @@ -1628,6 +1629,9 @@ static void interpolate_user_regamma(uint32_t hw_points_num, } else hw_x = coordinates_x[i].x; + if (dc_fixpt_le(one, hw_x)) + hw_x = one; + norm_x = dc_fixpt_mul(norm_factor, hw_x); index = dc_fixpt_floor(norm_x); if (index < 0 || index > 255) @@ -1688,7 +1692,7 @@ static void apply_degamma_for_user_regamma(struct pwl_float_data_ex *rgb_regamma struct pwl_float_data_ex *rgb = rgb_regamma; const struct hw_x_point *coord_x = coordinates_x; - build_coefficients(&coeff, true); + build_coefficients(&coeff, TRANSFER_FUNCTION_SRGB); i = 0; while (i != hw_points_num + 1) { diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h index 1115dfc6ae1f62b3cab864d4e2adcd83b44d32ab..3973110f149cf17a75e8f3bcb843fc6f9fb13750 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h @@ -9800,18 +9800,118 @@ // addressBlock: gc_pwrdec // base address: 0x3c000 +#define mmCGTS_RD_CTRL_REG 0x5004 +#define mmCGTS_RD_CTRL_REG_BASE_IDX 1 +#define mmCGTS_RD_REG 0x5005 +#define mmCGTS_RD_REG_BASE_IDX 1 #define mmCGTS_TCC_DISABLE 0x5006 #define mmCGTS_TCC_DISABLE_BASE_IDX 1 #define mmCGTS_USER_TCC_DISABLE 0x5007 #define mmCGTS_USER_TCC_DISABLE_BASE_IDX 1 +#define mmCGTS_STATUS_REG 0x5008 +#define mmCGTS_STATUS_REG_BASE_IDX 1 +#define mmCGTT_SPI_CGTSSM_CLK_CTRL 0x5009 +#define mmCGTT_SPI_CGTSSM_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SPI_PS_CLK_CTRL 0x507d +#define mmCGTT_SPI_PS_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SPIS_CLK_CTRL 0x507e +#define mmCGTT_SPIS_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SPI_CLK_CTRL 0x5080 +#define mmCGTT_SPI_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_PC_CLK_CTRL 0x5081 +#define mmCGTT_PC_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_BCI_CLK_CTRL 0x5082 +#define mmCGTT_BCI_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_VGT_CLK_CTRL 0x5084 +#define mmCGTT_VGT_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_IA_CLK_CTRL 0x5085 +#define mmCGTT_IA_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_WD_CLK_CTRL 0x5086 +#define mmCGTT_WD_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_GS_NGG_CLK_CTRL 0x5087 +#define mmCGTT_GS_NGG_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_PA_CLK_CTRL 0x5088 +#define mmCGTT_PA_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SC_CLK_CTRL0 0x5089 +#define mmCGTT_SC_CLK_CTRL0_BASE_IDX 1 +#define mmCGTT_SC_CLK_CTRL1 0x508a +#define mmCGTT_SC_CLK_CTRL1_BASE_IDX 1 +#define mmCGTT_SC_CLK_CTRL2 0x508b +#define mmCGTT_SC_CLK_CTRL2_BASE_IDX 1 +#define mmCGTT_SQ_CLK_CTRL 0x508c +#define mmCGTT_SQ_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SQG_CLK_CTRL 0x508d +#define mmCGTT_SQG_CLK_CTRL_BASE_IDX 1 #define mmSQ_ALU_CLK_CTRL 0x508e #define mmSQ_ALU_CLK_CTRL_BASE_IDX 1 #define mmSQ_TEX_CLK_CTRL 0x508f #define mmSQ_TEX_CLK_CTRL_BASE_IDX 1 #define mmSQ_LDS_CLK_CTRL 0x5090 #define mmSQ_LDS_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_SX_CLK_CTRL0 0x5094 +#define mmCGTT_SX_CLK_CTRL0_BASE_IDX 1 +#define mmCGTT_SX_CLK_CTRL1 0x5095 +#define mmCGTT_SX_CLK_CTRL1_BASE_IDX 1 +#define mmCGTT_SX_CLK_CTRL2 0x5096 +#define mmCGTT_SX_CLK_CTRL2_BASE_IDX 1 +#define mmCGTT_SX_CLK_CTRL3 0x5097 +#define mmCGTT_SX_CLK_CTRL3_BASE_IDX 1 +#define mmCGTT_SX_CLK_CTRL4 0x5098 +#define mmCGTT_SX_CLK_CTRL4_BASE_IDX 1 +#define mmTD_CGTT_CTRL 0x509c +#define mmTD_CGTT_CTRL_BASE_IDX 1 +#define mmTA_CGTT_CTRL 0x509d +#define mmTA_CGTT_CTRL_BASE_IDX 1 +#define mmCGTT_TCPI_CLK_CTRL 0x5109 +#define mmCGTT_TCPI_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_GDS_CLK_CTRL 0x50a0 +#define mmCGTT_GDS_CLK_CTRL_BASE_IDX 1 +#define mmDB_CGTT_CLK_CTRL_0 0x50a4 +#define mmDB_CGTT_CLK_CTRL_0_BASE_IDX 1 +#define mmCB_CGTT_SCLK_CTRL 0x50a8 +#define mmCB_CGTT_SCLK_CTRL_BASE_IDX 1 +#define mmGL2C_CGTT_SCLK_CTRL 0x50fc +#define mmGL2C_CGTT_SCLK_CTRL_BASE_IDX 1 +#define mmGL2A_CGTT_SCLK_CTRL 0x50ac +#define mmGL2A_CGTT_SCLK_CTRL_BASE_IDX 1 +#define mmGL2A_CGTT_SCLK_CTRL_1 0x50ad +#define mmGL2A_CGTT_SCLK_CTRL_1_BASE_IDX 1 +#define mmCGTT_CP_CLK_CTRL 0x50b0 +#define mmCGTT_CP_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_CPF_CLK_CTRL 0x50b1 +#define mmCGTT_CPF_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_CPC_CLK_CTRL 0x50b2 +#define mmCGTT_CPC_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_RLC_CLK_CTRL 0x50b5 +#define mmCGTT_RLC_CLK_CTRL_BASE_IDX 1 #define mmRLC_GFX_RM_CNTL 0x50b6 #define mmRLC_GFX_RM_CNTL_BASE_IDX 1 +#define mmRMI_CGTT_SCLK_CTRL 0x50c0 +#define mmRMI_CGTT_SCLK_CTRL_BASE_IDX 1 +#define mmCGTT_TCPF_CLK_CTRL 0x5111 +#define mmCGTT_TCPF_CLK_CTRL_BASE_IDX 1 +#define mmGCR_CGTT_SCLK_CTRL 0x50c2 +#define mmGCR_CGTT_SCLK_CTRL_BASE_IDX 1 +#define mmUTCL1_CGTT_CLK_CTRL 0x50c3 +#define mmUTCL1_CGTT_CLK_CTRL_BASE_IDX 1 +#define mmGCEA_CGTT_CLK_CTRL 0x50c4 +#define mmGCEA_CGTT_CLK_CTRL_BASE_IDX 1 +#define mmSE_CAC_CGTT_CLK_CTRL 0x50d0 +#define mmSE_CAC_CGTT_CLK_CTRL_BASE_IDX 1 +#define mmGC_CAC_CGTT_CLK_CTRL 0x50d8 +#define mmGC_CAC_CGTT_CLK_CTRL_BASE_IDX 1 +#define mmGRBM_CGTT_CLK_CNTL 0x50e0 +#define mmGRBM_CGTT_CLK_CNTL_BASE_IDX 1 +#define mmGUS_CGTT_CLK_CTRL 0x50f4 +#define mmGUS_CGTT_CLK_CTRL_BASE_IDX 1 +#define mmCGTT_PH_CLK_CTRL0 0x50f8 +#define mmCGTT_PH_CLK_CTRL0_BASE_IDX 1 +#define mmCGTT_PH_CLK_CTRL1 0x50f9 +#define mmCGTT_PH_CLK_CTRL1_BASE_IDX 1 +#define mmCGTT_PH_CLK_CTRL2 0x50fa +#define mmCGTT_PH_CLK_CTRL2_BASE_IDX 1 +#define mmCGTT_PH_CLK_CTRL3 0x50fb +#define mmCGTT_PH_CLK_CTRL3_BASE_IDX 1 // addressBlock: gc_hypdec diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h index 83faa276523f39dedf871bd8b9d7bc548a7b9302..d4e8ff22ecb8d4cc5bdd222f81d6dab9417d1296 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h @@ -34547,6 +34547,14 @@ // addressBlock: gc_pwrdec +//CGTS_RD_CTRL_REG +#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT 0x0 +#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT 0x4 +#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK 0x0000000FL +#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK 0x000000F0L +//CGTS_RD_REG +#define CGTS_RD_REG__READ_DATA__SHIFT 0x0 +#define CGTS_RD_REG__READ_DATA_MASK 0xFFFFFFFFL //CGTS_TCC_DISABLE #define CGTS_TCC_DISABLE__HI_TCC_DISABLE__SHIFT 0x8 #define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10 @@ -34557,6 +34565,485 @@ #define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10 #define CGTS_USER_TCC_DISABLE__HI_TCC_DISABLE_MASK 0x0000FF00L #define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK 0xFFFF0000L +//CGTS_STATUS_REG +#define CGTS_STATUS_REG__SA0_QUAD0_MGCG_ENABLED__SHIFT 0x0 +#define CGTS_STATUS_REG__SA0_QUAD0_CG_STATUS__SHIFT 0x1 +#define CGTS_STATUS_REG__SA1_QUAD0_MGCG_ENABLED__SHIFT 0x8 +#define CGTS_STATUS_REG__SA1_QUAD0_CG_STATUS__SHIFT 0x9 +#define CGTS_STATUS_REG__SA0_QUAD0_MGCG_ENABLED_MASK 0x00000001L +#define CGTS_STATUS_REG__SA0_QUAD0_CG_STATUS_MASK 0x00000006L +#define CGTS_STATUS_REG__SA1_QUAD0_MGCG_ENABLED_MASK 0x00000100L +#define CGTS_STATUS_REG__SA1_QUAD0_CG_STATUS_MASK 0x00000600L +//CGTT_SPI_CGTSSM_CLK_CTRL +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP3_OVERRIDE_MASK 0x08000000L +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000L +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000L +#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000L +//CGTT_SPI_PS_CLK_CTRL +#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x10 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE__SHIFT 0x18 +#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE__SHIFT 0x19 +#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE__SHIFT 0x1a +#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b +#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c +#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d +#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e +#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00010000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE_MASK 0x01000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE_MASK 0x02000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE_MASK 0x04000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE_MASK 0x08000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000L +#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000L +#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_SPIS_CLK_CTRL +#define CGTT_SPIS_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x10 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE__SHIFT 0x18 +#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE__SHIFT 0x19 +#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE__SHIFT 0x1a +#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b +#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c +#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d +#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e +#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SPIS_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00010000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE_MASK 0x01000000L +#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE_MASK 0x02000000L +#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE_MASK 0x04000000L +#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE_MASK 0x08000000L +#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000L +#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000L +#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000L +#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_SPI_CLK_CTRL +#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x10 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_SPI_CLK_CTRL__GRP6_OVERRIDE__SHIFT 0x18 +#define CGTT_SPI_CLK_CTRL__GRP5_OVERRIDE__SHIFT 0x19 +#define CGTT_SPI_CLK_CTRL__GRP4_OVERRIDE__SHIFT 0x1a +#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b +#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c +#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d +#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e +#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00010000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_SPI_CLK_CTRL__GRP6_OVERRIDE_MASK 0x01000000L +#define CGTT_SPI_CLK_CTRL__GRP5_OVERRIDE_MASK 0x02000000L +#define CGTT_SPI_CLK_CTRL__GRP4_OVERRIDE_MASK 0x04000000L +#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK 0x08000000L +#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000L +#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000L +#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000L +#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_PC_CLK_CTRL +#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE__SHIFT 0x11 +#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE__SHIFT 0xd +#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE__SHIFT 0xe +#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b +#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c +#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d +#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e +#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE_MASK 0x00020000L +#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE_MASK 0x00002000L +#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE_MASK 0x00004000L +#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK 0x08000000L +#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000L +#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000L +#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000L +//CGTT_BCI_CLK_CTRL +#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT 0xc +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT 0x18 +#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT 0x19 +#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT 0x1a +#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b +#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c +#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d +#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e +#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_BCI_CLK_CTRL__RESERVED_MASK 0x0000F000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK 0x01000000L +#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK 0x02000000L +#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK 0x04000000L +#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK 0x08000000L +#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000L +#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000L +#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000L +#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_VGT_CLK_CTRL +#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT 0xf +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT 0x1c +#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1d +#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK 0x00008000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK 0x10000000L +#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK 0x20000000L +#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_IA_CLK_CTRL +#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT 0x19 +#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK 0x02000000L +#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L +#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_WD_CLK_CTRL +#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT 0xf +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1d +#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT 0x1e +#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK 0x00008000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK 0x20000000L +#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK 0x40000000L +#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_GS_NGG_CLK_CTRL +#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE__SHIFT 0xf +#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE__SHIFT 0x10 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT 0x1c +#define CGTT_GS_NGG_CLK_CTRL__GS1_OVERRIDE__SHIFT 0x1d +#define CGTT_GS_NGG_CLK_CTRL__GS0_OVERRIDE__SHIFT 0x1e +#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE_MASK 0x00008000L +#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE_MASK 0x00010000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE_MASK 0x10000000L +#define CGTT_GS_NGG_CLK_CTRL__GS1_OVERRIDE_MASK 0x20000000L +#define CGTT_GS_NGG_CLK_CTRL__GS0_OVERRIDE_MASK 0x40000000L +#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_PA_CLK_CTRL +#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT 0x1d +#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT 0x1e +#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT 0x1f +#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK 0x20000000L +#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK 0x40000000L +#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK 0x80000000L +//CGTT_SC_CLK_CTRL0 +#define CGTT_SC_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE__SHIFT 0x10 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT 0x17 +#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE__SHIFT 0x18 +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x19 +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x1a +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x1b +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x1c +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1d +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1e +#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT 0x1f +#define CGTT_SC_CLK_CTRL0__ON_DELAY_MASK 0x0000000FL +#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE_MASK 0x00010000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK 0x00800000L +#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE_MASK 0x01000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x02000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x04000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x08000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x10000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x20000000L +#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x40000000L +#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK 0x80000000L +//CGTT_SC_CLK_CTRL1 +#define CGTT_SC_CLK_CTRL1__ON_DELAY__SHIFT 0x0 +#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0__SHIFT 0x10 +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT 0x11 +#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT 0x12 +#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE__SHIFT 0x13 +#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE__SHIFT 0x14 +#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT 0x15 +#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT 0x16 +#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE__SHIFT 0x17 +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0__SHIFT 0x18 +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT 0x19 +#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT 0x1a +#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE__SHIFT 0x1b +#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT 0x1c +#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT 0x1d +#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE__SHIFT 0x1e +#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE__SHIFT 0x1f +#define CGTT_SC_CLK_CTRL1__ON_DELAY_MASK 0x0000000FL +#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0_MASK 0x00010000L +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK 0x00020000L +#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK 0x00040000L +#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE_MASK 0x00080000L +#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE_MASK 0x00100000L +#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK 0x00200000L +#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK 0x00400000L +#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE_MASK 0x00800000L +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0_MASK 0x01000000L +#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK 0x02000000L +#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK 0x04000000L +#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE_MASK 0x08000000L +#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK 0x10000000L +#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK 0x20000000L +#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE_MASK 0x40000000L +#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE_MASK 0x80000000L +//CGTT_SC_CLK_CTRL2 +#define CGTT_SC_CLK_CTRL2__ON_DELAY__SHIFT 0x0 +#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SC_CLK_CTRL2__DBR_CLK_OVERRIDE__SHIFT 0x1a +#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE__SHIFT 0x1b +#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE__SHIFT 0x1c +#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE__SHIFT 0x1d +#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE__SHIFT 0x1e +#define CGTT_SC_CLK_CTRL2__ON_DELAY_MASK 0x0000000FL +#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SC_CLK_CTRL2__DBR_CLK_OVERRIDE_MASK 0x04000000L +#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE_MASK 0x08000000L +#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE_MASK 0x10000000L +#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE_MASK 0x20000000L +#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE_MASK 0x40000000L +//CGTT_SQ_CLK_CTRL +#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L +#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L +//CGTT_SQG_CLK_CTRL +#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT 0x1c +#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT 0x1d +#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK 0x10000000L +#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK 0x20000000L +#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000L +#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000L //SQ_ALU_CLK_CTRL #define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT 0x0 #define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT 0x10 @@ -34572,12 +35059,982 @@ #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT 0x10 #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0_MASK 0x0000FFFFL #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1_MASK 0xFFFF0000L +//CGTT_SX_CLK_CTRL0 +#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK 0x0000000FL +#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SX_CLK_CTRL0__RESERVED_MASK 0x0000F000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_SX_CLK_CTRL1 +#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK 0x0000000FL +#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SX_CLK_CTRL1__RESERVED_MASK 0x0000F000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_SX_CLK_CTRL2 +#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT 0xd +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK 0x0000000FL +#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SX_CLK_CTRL2__RESERVED_MASK 0x0000E000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_SX_CLK_CTRL3 +#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT 0xd +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK 0x0000000FL +#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SX_CLK_CTRL3__RESERVED_MASK 0x0000E000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_SX_CLK_CTRL4 +#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK 0x0000000FL +#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_SX_CLK_CTRL4__RESERVED_MASK 0x0000F000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK 0x80000000L +//TD_CGTT_CTRL +#define TD_CGTT_CTRL__ON_DELAY__SHIFT 0x0 +#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TD_CGTT_CTRL__ON_DELAY_MASK 0x0000000FL +#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//TA_CGTT_CTRL +#define TA_CGTT_CTRL__ON_DELAY__SHIFT 0x0 +#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TA_CGTT_CTRL__ON_DELAY_MASK 0x0000000FL +#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_TCPI_CLK_CTRL +#define CGTT_TCPI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_TCPI_CLK_CTRL__SPARE__SHIFT 0xc +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0xf +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x10 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x17 +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x18 +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x19 +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1a +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1b +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1c +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1d +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1e +#define CGTT_TCPI_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_TCPI_CLK_CTRL__SPARE_MASK 0x00007000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00008000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00010000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x00800000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x01000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x02000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x04000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x08000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x10000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x20000000L +#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x40000000L +//CGTT_GDS_CLK_CTRL +#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_GDS_CLK_CTRL__UNUSED__SHIFT 0xc +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_GDS_CLK_CTRL__UNUSED_MASK 0x0000F000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//DB_CGTT_CLK_CTRL_0 +#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT 0x0 +#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT 0x4 +#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT 0xc +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT 0x18 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT 0x19 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT 0x1a +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT 0x1b +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT 0x1c +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT 0x1d +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT 0x1e +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT 0x1f +#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK 0x0000000FL +#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK 0x0000F000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK 0x01000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK 0x02000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK 0x04000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK 0x08000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK 0x10000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK 0x20000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK 0x40000000L +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK 0x80000000L +//CB_CGTT_SCLK_CTRL +#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//GL2C_CGTT_SCLK_CTRL +#define GL2C_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GL2C_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define GL2C_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GL2C_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//GL2A_CGTT_SCLK_CTRL +#define GL2A_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GL2A_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define GL2A_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GL2A_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//GL2A_CGTT_SCLK_CTRL_1 +#define GL2A_CGTT_SCLK_CTRL_1__ON_DELAY__SHIFT 0x0 +#define GL2A_CGTT_SCLK_CTRL_1__OFF_HYSTERESIS__SHIFT 0x4 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE7__SHIFT 0x18 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE6__SHIFT 0x19 +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE5__SHIFT 0x1a +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE4__SHIFT 0x1b +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE3__SHIFT 0x1c +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE2__SHIFT 0x1d +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE1__SHIFT 0x1e +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE0__SHIFT 0x1f +#define GL2A_CGTT_SCLK_CTRL_1__ON_DELAY_MASK 0x0000000FL +#define GL2A_CGTT_SCLK_CTRL_1__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE7_MASK 0x01000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE6_MASK 0x02000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE5_MASK 0x04000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE4_MASK 0x08000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE3_MASK 0x10000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE2_MASK 0x20000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE1_MASK 0x40000000L +#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_CP_CLK_CTRL +#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT 0xf +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1d +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK 0x00008000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x20000000L +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L +//CGTT_CPF_CLK_CTRL +#define CGTT_CPF_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE__SHIFT 0xf +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1a +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT__SHIFT 0x1b +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP__SHIFT 0x1c +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX__SHIFT 0x1d +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_CPF_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE_MASK 0x00008000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x04000000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT_MASK 0x08000000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP_MASK 0x10000000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX_MASK 0x20000000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L +//CGTT_CPC_CLK_CTRL +#define CGTT_CPC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT 0xf +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT 0x1d +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_CPC_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK 0x00008000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK 0x20000000L +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L +//CGTT_RLC_CLK_CTRL +#define CGTT_RLC_CLK_CTRL__RESERVED__SHIFT 0x0 +#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_RLC_CLK_CTRL__RESERVED_MASK 0x0000000FL +#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L //RLC_GFX_RM_CNTL #define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID__SHIFT 0x0 #define RLC_GFX_RM_CNTL__RESERVED__SHIFT 0x1 #define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID_MASK 0x00000001L #define RLC_GFX_RM_CNTL__RESERVED_MASK 0xFFFFFFFEL - +//RMI_CGTT_SCLK_CTRL +#define RMI_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define RMI_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//CGTT_TCPF_CLK_CTRL +#define CGTT_TCPF_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_TCPF_CLK_CTRL__SPARE__SHIFT 0xc +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0xf +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x10 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x11 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x12 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x13 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x14 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x15 +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x16 +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x17 +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x18 +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x19 +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1a +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1b +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1c +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1d +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1e +#define CGTT_TCPF_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_TCPF_CLK_CTRL__SPARE_MASK 0x00007000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00008000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00010000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00020000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00040000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00080000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00100000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00200000L +#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00400000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x00800000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x01000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x02000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x04000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x08000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x10000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x20000000L +#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x40000000L +//GCR_CGTT_SCLK_CTRL +#define GCR_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GCR_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define GCR_CGTT_SCLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GCR_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//UTCL1_CGTT_CLK_CTRL +#define UTCL1_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define UTCL1_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define UTCL1_CGTT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define UTCL1_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x04000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000L +#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000L +//GCEA_CGTT_CLK_CTRL +#define GCEA_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GCEA_CGTT_CLK_CTRL__SPARE0__SHIFT 0xc +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT 0x14 +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT 0x15 +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT 0x16 +#define GCEA_CGTT_CLK_CTRL__SPARE1__SHIFT 0x17 +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT 0x1c +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT 0x1d +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT 0x1e +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT 0x1f +#define GCEA_CGTT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GCEA_CGTT_CLK_CTRL__SPARE0_MASK 0x000FF000L +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK 0x00100000L +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK 0x00200000L +#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK 0x00400000L +#define GCEA_CGTT_CLK_CTRL__SPARE1_MASK 0x0F800000L +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK 0x10000000L +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK 0x20000000L +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK 0x40000000L +#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK 0x80000000L +//SE_CAC_CGTT_CLK_CTRL +#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L +//GC_CAC_CGTT_CLK_CTRL +#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000L +//GRBM_CGTT_CLK_CNTL +#define GRBM_CGTT_CLK_CNTL__ON_DELAY__SHIFT 0x0 +#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS__SHIFT 0x4 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0__SHIFT 0x17 +#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define GRBM_CGTT_CLK_CNTL__ON_DELAY_MASK 0x0000000FL +#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0_MASK 0x00800000L +#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN_MASK 0x40000000L +//GUS_CGTT_CLK_CTRL +#define GUS_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GUS_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GUS_CGTT_CLK_CTRL__SPARE0__SHIFT 0xc +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_DRAM__SHIFT 0x13 +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT 0x14 +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT 0x15 +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT 0x16 +#define GUS_CGTT_CLK_CTRL__SPARE1__SHIFT 0x17 +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_DRAM__SHIFT 0x1b +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT 0x1c +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT 0x1d +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT 0x1e +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT 0x1f +#define GUS_CGTT_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define GUS_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define GUS_CGTT_CLK_CTRL__SPARE0_MASK 0x0007F000L +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_DRAM_MASK 0x00080000L +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK 0x00100000L +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK 0x00200000L +#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK 0x00400000L +#define GUS_CGTT_CLK_CTRL__SPARE1_MASK 0x07800000L +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_DRAM_MASK 0x08000000L +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK 0x10000000L +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK 0x20000000L +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK 0x40000000L +#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK 0x80000000L +//CGTT_PH_CLK_CTRL0 +#define CGTT_PH_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE7__SHIFT 0x10 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE6__SHIFT 0x11 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT 0x12 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT 0x13 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT 0x14 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT 0x15 +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT 0x16 +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE__SHIFT 0x1e +#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT 0x1f +#define CGTT_PH_CLK_CTRL0__ON_DELAY_MASK 0x0000000FL +#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE7_MASK 0x00010000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE6_MASK 0x00020000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK 0x00040000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK 0x00080000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK 0x00100000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK 0x00200000L +#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK 0x00400000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE_MASK 0x40000000L +#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE_MASK 0x80000000L +//CGTT_PH_CLK_CTRL1 +#define CGTT_PH_CLK_CTRL1__ON_DELAY__SHIFT 0x0 +#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_PH_CLK_CTRL1__ON_DELAY_MASK 0x0000000FL +#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1_MASK 0x40000000L +//CGTT_PH_CLK_CTRL2 +#define CGTT_PH_CLK_CTRL2__ON_DELAY__SHIFT 0x0 +#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_PH_CLK_CTRL2__ON_DELAY_MASK 0x0000000FL +#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1_MASK 0x40000000L +//CGTT_PH_CLK_CTRL3 +#define CGTT_PH_CLK_CTRL3__ON_DELAY__SHIFT 0x0 +#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_PH_CLK_CTRL3__ON_DELAY_MASK 0x0000000FL +#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7_MASK 0x01000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6_MASK 0x02000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5_MASK 0x04000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4_MASK 0x08000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3_MASK 0x10000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2_MASK 0x20000000L +#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1_MASK 0x40000000L // addressBlock: gc_hypdec //CP_HYP_PFP_UCODE_ADDR diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h index d8632ccf349442be1d809a41c02a5362ab7a9d17..c488d4a50cf46af76daf410d55101c1e63270f4f 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h @@ -4409,6 +4409,10 @@ #define mmVMSHAREDPF0_MC_VM_XGMI_LFB_SIZE_BASE_IDX 1 #define mmVMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL 0x0af9 #define mmVMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL_BASE_IDX 1 +#define mmMC_VM_XGMI_LFB_CNTL 0x0823 +#define mmMC_VM_XGMI_LFB_CNTL_BASE_IDX 0 +#define mmMC_VM_XGMI_LFB_SIZE 0x0824 +#define mmMC_VM_XGMI_LFB_SIZE_BASE_IDX 0 // addressBlock: mmhub_utcl2_vmsharedvcdec diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h index 111a71b434e233185616f6822c262ae899732122..2969fbf282b7d066c5d34b406ea182950ff5c1ca 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h @@ -26728,6 +26728,14 @@ //VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL #define VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE__SHIFT 0x0 #define VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE_MASK 0x00000001L +//MC_VM_XGMI_LFB_CNTL +#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT 0x0 +#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT 0x3 +#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK 0x00000007L +#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK 0x00000038L +//MC_VM_XGMI_LFB_SIZE +#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT 0x0 +#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK 0x0000FFFFL // addressBlock: mmhub_utcl2_vmsharedvcdec diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_offset.h index b798cf5a2c39c8178133a8031fe892098bb114ac..38adde3cae5ac57f0d2ab655ad840b9453f1630f 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_offset.h +++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_offset.h @@ -29,5 +29,7 @@ #define regMCA_UMC_UMC0_MCUMC_STATUST0_BASE_IDX 2 #define regMCA_UMC_UMC0_MCUMC_ADDRT0 0x03c4 #define regMCA_UMC_UMC0_MCUMC_ADDRT0_BASE_IDX 2 +#define regUMCCH0_0_GeccCtrl 0x0053 +#define regUMCCH0_0_GeccCtrl_BASE_IDX 2 #endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_sh_mask.h index bd99b431247f3e4bb9248606aa93b021796f7a00..4dbec524f9434c1f0b320305350813e63197f0a8 100644 --- a/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_sh_mask.h +++ b/drivers/gpu/drm/amd/include/asic_reg/umc/umc_8_10_0_sh_mask.h @@ -90,5 +90,8 @@ #define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr__SHIFT 0x0 #define MCA_UMC_UMC0_MCUMC_ADDRT0__Reserved__SHIFT 0x38 #define MCA_UMC_UMC0_MCUMC_ADDRT0__ErrorAddr_MASK 0x00FFFFFFFFFFFFFFL +//UMCCH0_0_GeccCtrl +#define UMCCH0_0_GeccCtrl__UCFatalEn__SHIFT 0xd +#define UMCCH0_0_GeccCtrl__UCFatalEn_MASK 0x00002000L #endif diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index 7e3231c2191ca39c0a891d85b9d3d8d002e15ee9..a40ead44778af4386b68aabadff85e7744453597 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -824,4 +824,62 @@ struct gpu_metrics_v2_2 { uint64_t indep_throttle_status; }; +struct gpu_metrics_v2_3 { + struct metrics_table_header common_header; + + /* Temperature */ + uint16_t temperature_gfx; // gfx temperature on APUs + uint16_t temperature_soc; // soc temperature on APUs + uint16_t temperature_core[8]; // CPU core temperature on APUs + uint16_t temperature_l3[2]; + + /* Utilization */ + uint16_t average_gfx_activity; + uint16_t average_mm_activity; // UVD or VCN + + /* Driver attached timestamp (in ns) */ + uint64_t system_clock_counter; + + /* Power/Energy */ + uint16_t average_socket_power; // dGPU + APU power on A + A platform + uint16_t average_cpu_power; + uint16_t average_soc_power; + uint16_t average_gfx_power; + uint16_t average_core_power[8]; // CPU core power on APUs + + /* Average clocks */ + uint16_t average_gfxclk_frequency; + uint16_t average_socclk_frequency; + uint16_t average_uclk_frequency; + uint16_t average_fclk_frequency; + uint16_t average_vclk_frequency; + uint16_t average_dclk_frequency; + + /* Current clocks */ + uint16_t current_gfxclk; + uint16_t current_socclk; + uint16_t current_uclk; + uint16_t current_fclk; + uint16_t current_vclk; + uint16_t current_dclk; + uint16_t current_coreclk[8]; // CPU core clocks + uint16_t current_l3clk[2]; + + /* Throttle status (ASIC dependent) */ + uint32_t throttle_status; + + /* Fans */ + uint16_t fan_pwm; + + uint16_t padding[3]; + + /* Throttle status (ASIC independent) */ + uint64_t indep_throttle_status; + + /* Average Temperature */ + uint16_t average_temperature_gfx; // average gfx temperature on APUs + uint16_t average_temperature_soc; // average soc temperature on APUs + uint16_t average_temperature_core[8]; // average CPU core temperature on APUs + uint16_t average_temperature_l3[2]; +}; #endif diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h index 50bfa513cb35a618d281aff3b8e42fa14e79e844..7e85cdc5bd34ea842120ecfe11b5bf319725be1b 100644 --- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h +++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h @@ -269,7 +269,8 @@ union MESAPI__ADD_QUEUE { uint32_t map_kiq_utility_queue : 1; uint32_t is_kfd_process : 1; uint32_t trap_en : 1; - uint32_t reserved : 21; + uint32_t is_aql_queue : 1; + uint32_t reserved : 20; }; struct MES_API_STATUS api_status; uint64_t tma_addr; diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 5e318b3f6c0fe0d3099a5318d933b30561ae5769..948cc75376f8bc0607719a5253082a697e77be8f 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -3405,9 +3405,6 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) { - if (adev->pm.dpm_enabled == 0) - return; - if (adev->pm.int_hwmon_dev) hwmon_device_unregister(adev->pm.int_hwmon_dev); diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c index 8fd0782a2b206712ae4782efbb7bd157a551546e..f5e08b60f66ef9fa2d84bd2a86d712a839928e36 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/kv_dpm.c @@ -1384,13 +1384,16 @@ static int kv_dpm_enable(struct amdgpu_device *adev) static void kv_dpm_disable(struct amdgpu_device *adev) { struct kv_power_info *pi = kv_get_pi(adev); + int err; amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, AMDGPU_THERMAL_IRQ_LOW_TO_HIGH); amdgpu_irq_put(adev, &adev->pm.dpm.thermal.irq, AMDGPU_THERMAL_IRQ_HIGH_TO_LOW); - amdgpu_kv_smc_bapm_enable(adev, false); + err = amdgpu_kv_smc_bapm_enable(adev, false); + if (err) + DRM_ERROR("amdgpu_kv_smc_bapm_enable failed\n"); if (adev->asic_type == CHIP_MULLINS) kv_enable_nb_dpm(adev, false); diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index 1eb4e613b27a535e0aafcfba5e6296c67bfd5ed6..ec055858eb95abc6b9c120b805ab2573f8847763 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -1485,6 +1485,7 @@ static int pp_get_prv_buffer_details(void *handle, void **addr, size_t *size) { struct pp_hwmgr *hwmgr = handle; struct amdgpu_device *adev = hwmgr->adev; + int err; if (!addr || !size) return -EINVAL; @@ -1492,7 +1493,9 @@ static int pp_get_prv_buffer_details(void *handle, void **addr, size_t *size) *addr = NULL; *size = 0; if (adev->pm.smu_prv_buffer) { - amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr); + err = amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr); + if (err) + return err; *size = adev->pm.smu_prv_buffer_size; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index e4fcbf8a7eb5c887eae31a66323f4db33e291b61..7ef7e81525a30648746287e70336f548707799c8 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -3603,7 +3603,7 @@ static int smu7_get_pp_table_entry_callback_func_v1(struct pp_hwmgr *hwmgr, return -EINVAL); PP_ASSERT_WITH_CODE( - (smu7_power_state->performance_level_count <= + (smu7_power_state->performance_level_count < hwmgr->platform_descriptor.hardwareActivityPerformanceLevels), "Performance levels exceeds Driver limit!", return -EINVAL); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 99bfe5efe1710157fddbd076a98f06a6a62700b5..c8c9fb827bda1119448664d063cc1c991806bc08 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -3155,7 +3155,7 @@ static int vega10_get_pp_table_entry_callback_func(struct pp_hwmgr *hwmgr, return -1); PP_ASSERT_WITH_CODE( - (vega10_ps->performance_level_count <= + (vega10_ps->performance_level_count < hwmgr->platform_descriptor. hardwareActivityPerformanceLevels), "Performance levels exceeds Driver limit!", diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index dad3e3741a4e8a5b2852cc7ae700eb3a55274005..190af79f3236fdb097b6c9e521ad8aac76bbbbe4 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -67,22 +67,21 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { - uint32_t current_rpm; - uint32_t percent = 0; - - if (hwmgr->thermal_controller.fanInfo.bNoFan) - return 0; + struct amdgpu_device *adev = hwmgr->adev; + uint32_t duty100, duty; + uint64_t tmp64; - if (vega10_get_current_rpm(hwmgr, ¤t_rpm)) - return -1; + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), + CG_THERMAL_STATUS, FDO_PWM_DUTY); - if (hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM != 0) - percent = current_rpm * 255 / - hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM; + if (!duty100) + return -EINVAL; - *speed = MIN(percent, 255); + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); return 0; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c index 1e79baab753efa0d87cd4b2665477abb21eac934..bd54fbd393b97792d21983977d52d1185f215779 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c @@ -195,7 +195,6 @@ static int init_powerplay_table_information( struct phm_ppt_v3_information *pptable_information = (struct phm_ppt_v3_information *)hwmgr->pptable; uint32_t disable_power_control = 0; - int result; hwmgr->thermal_controller.ucType = powerplay_table->ucThermalControllerType; pptable_information->uc_thermal_controller_type = powerplay_table->ucThermalControllerType; @@ -257,9 +256,7 @@ static int init_powerplay_table_information( if (pptable_information->smc_pptable == NULL) return -ENOMEM; - result = append_vbios_pptable(hwmgr, (pptable_information->smc_pptable)); - - return result; + return append_vbios_pptable(hwmgr, (pptable_information->smc_pptable)); } static int vega12_pp_tables_initialize(struct pp_hwmgr *hwmgr) diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c index 45214a364baa9f9c0d0cfaf36e4bd79d349e4b0e..e7ed2a7adf8f7b98c8ec3dbe8f6534371e706524 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c @@ -2567,15 +2567,13 @@ static uint8_t polaris10_get_memory_modile_index(struct pp_hwmgr *hwmgr) static int polaris10_initialize_mc_reg_table(struct pp_hwmgr *hwmgr) { - int result; struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smu_backend); pp_atomctrl_mc_reg_table *mc_reg_table = &smu_data->mc_reg_table; uint8_t module_index = polaris10_get_memory_modile_index(hwmgr); memset(mc_reg_table, 0, sizeof(pp_atomctrl_mc_reg_table)); - result = atomctrl_initialize_mc_reg_table_v2_2(hwmgr, module_index, mc_reg_table); - return result; + return atomctrl_initialize_mc_reg_table_v2_2(hwmgr, module_index, mc_reg_table); } static bool polaris10_is_dpm_running(struct pp_hwmgr *hwmgr) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 7ed4d4265797e4d5d0ef8fdb3449a25aa4ae2c54..74996a8fb67122de87e8f5238e92635e293fa8b0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -369,6 +369,17 @@ static void sienna_cichlid_check_bxco_support(struct smu_context *smu) smu_baco->platform_support = (val & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK) ? true : false; + + /* + * Disable BACO entry/exit completely on below SKUs to + * avoid hardware intermittent failures. + */ + if (((adev->pdev->device == 0x73A1) && + (adev->pdev->revision == 0x00)) || + ((adev->pdev->device == 0x73BF) && + (adev->pdev->revision == 0xCF))) + smu_baco->platform_support = false; + } } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 847990145dcd9ee36e45c0dbe2d05426db0c5d04..cb10c7e31264676d6678133f5001fd684f1bfb24 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -223,14 +223,13 @@ static int vangogh_tables_init(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; struct smu_table *tables = smu_table->tables; - struct amdgpu_device *adev = smu->adev; uint32_t if_version; + uint32_t smu_version; uint32_t ret = 0; - ret = smu_cmn_get_smc_version(smu, &if_version, NULL); + ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); if (ret) { - dev_err(adev->dev, "Failed to get smu if version!\n"); - goto err0_out; + return ret; } SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t), @@ -255,7 +254,10 @@ static int vangogh_tables_init(struct smu_context *smu) goto err0_out; smu_table->metrics_time = 0; - smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_2); + if (smu_version >= 0x043F3E00) + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_3); + else + smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_2); smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); if (!smu_table->gpu_metrics_table) goto err1_out; @@ -1648,6 +1650,63 @@ static int vangogh_set_watermarks_table(struct smu_context *smu, return 0; } +static ssize_t vangogh_get_legacy_gpu_metrics_v2_3(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v2_3 *gpu_metrics = + (struct gpu_metrics_v2_3 *)smu_table->gpu_metrics_table; + SmuMetrics_legacy_t metrics; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, &metrics, true); + if (ret) + return ret; + + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 3); + + gpu_metrics->temperature_gfx = metrics.GfxTemperature; + gpu_metrics->temperature_soc = metrics.SocTemperature; + memcpy(&gpu_metrics->temperature_core[0], + &metrics.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->temperature_l3[0] = metrics.L3Temperature[0]; + + gpu_metrics->average_gfx_activity = metrics.GfxActivity; + gpu_metrics->average_mm_activity = metrics.UvdActivity; + + gpu_metrics->average_socket_power = metrics.CurrentSocketPower; + gpu_metrics->average_cpu_power = metrics.Power[0]; + gpu_metrics->average_soc_power = metrics.Power[1]; + gpu_metrics->average_gfx_power = metrics.Power[2]; + memcpy(&gpu_metrics->average_core_power[0], + &metrics.CorePower[0], + sizeof(uint16_t) * 4); + + gpu_metrics->average_gfxclk_frequency = metrics.GfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.SocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.MemclkFrequency; + gpu_metrics->average_fclk_frequency = metrics.MemclkFrequency; + gpu_metrics->average_vclk_frequency = metrics.VclkFrequency; + gpu_metrics->average_dclk_frequency = metrics.DclkFrequency; + + memcpy(&gpu_metrics->current_coreclk[0], + &metrics.CoreFrequency[0], + sizeof(uint16_t) * 4); + gpu_metrics->current_l3clk[0] = metrics.L3Frequency[0]; + + gpu_metrics->throttle_status = metrics.ThrottlerStatus; + gpu_metrics->indep_throttle_status = + smu_cmn_get_indep_throttler_status(metrics.ThrottlerStatus, + vangogh_throttler_map); + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v2_3); +} + static ssize_t vangogh_get_legacy_gpu_metrics(struct smu_context *smu, void **table) { @@ -1705,6 +1764,77 @@ static ssize_t vangogh_get_legacy_gpu_metrics(struct smu_context *smu, return sizeof(struct gpu_metrics_v2_2); } +static ssize_t vangogh_get_gpu_metrics_v2_3(struct smu_context *smu, + void **table) +{ + struct smu_table_context *smu_table = &smu->smu_table; + struct gpu_metrics_v2_3 *gpu_metrics = + (struct gpu_metrics_v2_3 *)smu_table->gpu_metrics_table; + SmuMetrics_t metrics; + int ret = 0; + + ret = smu_cmn_get_metrics_table(smu, &metrics, true); + if (ret) + return ret; + + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 3); + + gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; + gpu_metrics->temperature_soc = metrics.Current.SocTemperature; + memcpy(&gpu_metrics->temperature_core[0], + &metrics.Current.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0]; + + gpu_metrics->average_temperature_gfx = metrics.Average.GfxTemperature; + gpu_metrics->average_temperature_soc = metrics.Average.SocTemperature; + memcpy(&gpu_metrics->average_temperature_core[0], + &metrics.Average.CoreTemperature[0], + sizeof(uint16_t) * 4); + gpu_metrics->average_temperature_l3[0] = metrics.Average.L3Temperature[0]; + + gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity; + gpu_metrics->average_mm_activity = metrics.Current.UvdActivity; + + gpu_metrics->average_socket_power = metrics.Current.CurrentSocketPower; + gpu_metrics->average_cpu_power = metrics.Current.Power[0]; + gpu_metrics->average_soc_power = metrics.Current.Power[1]; + gpu_metrics->average_gfx_power = metrics.Current.Power[2]; + memcpy(&gpu_metrics->average_core_power[0], + &metrics.Average.CorePower[0], + sizeof(uint16_t) * 4); + + gpu_metrics->average_gfxclk_frequency = metrics.Average.GfxclkFrequency; + gpu_metrics->average_socclk_frequency = metrics.Average.SocclkFrequency; + gpu_metrics->average_uclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_fclk_frequency = metrics.Average.MemclkFrequency; + gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; + gpu_metrics->average_dclk_frequency = metrics.Average.DclkFrequency; + + gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; + gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; + gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; + gpu_metrics->current_vclk = metrics.Current.VclkFrequency; + gpu_metrics->current_dclk = metrics.Current.DclkFrequency; + + memcpy(&gpu_metrics->current_coreclk[0], + &metrics.Current.CoreFrequency[0], + sizeof(uint16_t) * 4); + gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0]; + + gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; + gpu_metrics->indep_throttle_status = + smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus, + vangogh_throttler_map); + + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + + *table = (void *)gpu_metrics; + + return sizeof(struct gpu_metrics_v2_3); +} + static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, void **table) { @@ -1772,20 +1902,26 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, static ssize_t vangogh_common_get_gpu_metrics(struct smu_context *smu, void **table) { - struct amdgpu_device *adev = smu->adev; uint32_t if_version; + uint32_t smu_version; int ret = 0; - ret = smu_cmn_get_smc_version(smu, &if_version, NULL); + ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version); if (ret) { - dev_err(adev->dev, "Failed to get smu if version!\n"); return ret; } - if (if_version < 0x3) - ret = vangogh_get_legacy_gpu_metrics(smu, table); - else - ret = vangogh_get_gpu_metrics(smu, table); + if (smu_version >= 0x043F3E00) { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table); + else + ret = vangogh_get_gpu_metrics_v2_3(smu, table); + } else { + if (if_version < 0x3) + ret = vangogh_get_legacy_gpu_metrics(smu, table); + else + ret = vangogh_get_gpu_metrics(smu, table); + } return ret; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index 6e4a052dc53dea76e285c44c13bd2ceb93eb0d0f..93fffdbab4f070f71bcb59e0d3e680e824b646ee 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -210,7 +210,8 @@ int smu_v13_0_init_pptable_microcode(struct smu_context *smu) if (!adev->scpm_enabled) return 0; - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 7)) + if ((adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 7)) || + (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0))) return 0; /* override pptable_id from driver parameter */ @@ -219,27 +220,6 @@ int smu_v13_0_init_pptable_microcode(struct smu_context *smu) dev_info(adev->dev, "override pptable id %d\n", pptable_id); } else { pptable_id = smu->smu_table.boot_values.pp_table_id; - - /* - * Temporary solution for SMU V13.0.0 with SCPM enabled: - * - use vbios carried pptable when pptable_id is 3664, 3715 or 3795 - * - use 36831 soft pptable when pptable_id is 3683 - */ - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0)) { - switch (pptable_id) { - case 3664: - case 3715: - case 3795: - pptable_id = 0; - break; - case 3683: - pptable_id = 36831; - break; - default: - dev_err(adev->dev, "Unsupported pptable id %d\n", pptable_id); - return -EINVAL; - } - } } /* "pptable_id == 0" means vbios carries the pptable. */ @@ -475,28 +455,8 @@ int smu_v13_0_setup_pptable(struct smu_context *smu) } else { pptable_id = smu->smu_table.boot_values.pp_table_id; - /* - * Temporary solution for SMU V13.0.0 with SCPM disabled: - * - use 3664, 3683 or 3715 on request - * - use 3664 when pptable_id is 0 - * TODO: drop these when the pptable carried in vbios is ready. - */ - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0)) { - switch (pptable_id) { - case 0: - pptable_id = 3664; - break; - case 3664: - case 3683: - case 3715: - break; - default: - dev_err(adev->dev, "Unsupported pptable id %d\n", pptable_id); - return -EINVAL; - } - } else if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10)) { + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10)) pptable_id = 6666; - } } /* force using vbios pptable in sriov mode */ diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 7db2fd9ea74ad75001f77d3c2319a318660e6990..1d454485e0d91fc0cb21a183a1c0372402c3a4b7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -239,82 +239,47 @@ smu_v13_0_0_get_allowed_feature_mask(struct smu_context *smu, uint32_t *feature_mask, uint32_t num) { struct amdgpu_device *adev = smu->adev; + u32 smu_version; if (num > 2) return -EINVAL; - memset(feature_mask, 0, sizeof(uint32_t) * num); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_DATA_READ_BIT); + memset(feature_mask, 0xff, sizeof(uint32_t) * num); - if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT); + if (!(adev->pm.pp_feature & PP_SCLK_DPM_MASK)) { + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT); + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_IMU_BIT); } - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MM_DPM_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_VCN_BIT); - - if ((adev->pg_flags & AMD_PG_SUPPORT_ATHUB) && - (adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); - - if (adev->pm.pp_feature & PP_SOCCLK_DPM_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); + if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) || + !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB)) + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT); -#if 0 - if (adev->pm.pp_feature & PP_GFXOFF_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFXOFF_BIT); -#endif - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_THROTTLERS_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FAN_CONTROL_BIT); + if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)) + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DF_CSTATE_BIT); - - if (adev->pm.pp_feature & PP_MCLK_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_UCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); + /* PMFW 78.58 contains a critical fix for gfxoff feature */ + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if ((smu_version < 0x004e3a00) || + !(adev->pm.pp_feature & PP_GFXOFF_MASK)) + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT); + + if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) { + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT); + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT); + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT); } - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MEM_TEMP_READ_BIT); + if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)) + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - if (adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_GFXCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_SOCCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOC_MPCLK_DS_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_BACO_MPCLK_DS_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_FCLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_DCN_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_DCFCLK_BIT); - - if (adev->pm.pp_feature & PP_PCIE_DPM_MASK) { - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_LINK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_LCLK_BIT); + if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) { + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT); + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT); } - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_BACO_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_MP0CLK_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_DSTATE_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_OUT_OF_BAND_MONITOR_BIT); - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOC_CG_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_FCLK_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_CTF_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_UCLK_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VR0HOT_BIT); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT); - - if (adev->pm.pp_feature & PP_ULV_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_ULV_BIT); + if (!(adev->pm.pp_feature & PP_ULV_MASK)) + *(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT); return 0; } @@ -410,58 +375,11 @@ static int smu_v13_0_0_setup_pptable(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; struct amdgpu_device *adev = smu->adev; - uint32_t pptable_id; int ret = 0; - /* - * With SCPM enabled, the pptable used will be signed. It cannot - * be used directly by driver. To get the raw pptable, we need to - * rely on the combo pptable(and its revelant SMU message). - */ - if (adev->scpm_enabled) { - ret = smu_v13_0_0_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - } else { - /* override pptable_id from driver parameter */ - if (amdgpu_smu_pptable_id >= 0) { - pptable_id = amdgpu_smu_pptable_id; - dev_info(adev->dev, "override pptable id %d\n", pptable_id); - } else { - pptable_id = smu_table->boot_values.pp_table_id; - } - - /* - * Temporary solution for SMU V13.0.0 with SCPM disabled: - * - use vbios carried pptable when pptable_id is 3664, 3715 or 3795 - * - use soft pptable when pptable_id is 3683 - */ - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0)) { - switch (pptable_id) { - case 3664: - case 3715: - case 3795: - pptable_id = 0; - break; - case 3683: - break; - default: - dev_err(adev->dev, "Unsupported pptable id %d\n", pptable_id); - return -EINVAL; - } - } - - /* force using vbios pptable in sriov mode */ - if ((amdgpu_sriov_vf(adev) || !pptable_id) && (amdgpu_emu_mode != 1)) - ret = smu_v13_0_0_get_pptable_from_pmfw(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size); - else - ret = smu_v13_0_get_pptable_from_firmware(smu, - &smu_table->power_play_table, - &smu_table->power_play_table_size, - pptable_id); - } + ret = smu_v13_0_0_get_pptable_from_pmfw(smu, + &smu_table->power_play_table, + &smu_table->power_play_table_size); if (ret) return ret; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 15e4298c7cc83938bab42e7e60675fcf9199e6f3..e4f8f90ac5aa0454f2c73da4c4d57db2c9ff4416 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -969,6 +969,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) case METRICS_VERSION(2, 2): structure_size = sizeof(struct gpu_metrics_v2_2); break; + case METRICS_VERSION(2, 3): + structure_size = sizeof(struct gpu_metrics_v2_3); + break; default: return; } diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c index 0f0950c111960aafb6f36486ea65c9e9df5db3f1..78b72739e5c3e8c4b2b89045d5dac826a5c34762 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c @@ -1346,7 +1346,7 @@ uninit_regulators: return ret; } -static int adv7511_remove(struct i2c_client *i2c) +static void adv7511_remove(struct i2c_client *i2c) { struct adv7511 *adv7511 = i2c_get_clientdata(i2c); @@ -1362,8 +1362,6 @@ static int adv7511_remove(struct i2c_client *i2c) i2c_unregister_device(adv7511->i2c_packet); i2c_unregister_device(adv7511->i2c_edid); - - return 0; } static const struct i2c_device_id adv7511_i2c_ids[] = { diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c index ae3d6e9a606c933f41e5258afd3615d6f91be3bb..660a548579299c1f3802f71c52ec8dfbcf261b1a 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c +++ b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c @@ -787,7 +787,7 @@ err_unregister_i2c: return err; } -static int anx6345_i2c_remove(struct i2c_client *client) +static void anx6345_i2c_remove(struct i2c_client *client) { struct anx6345 *anx6345 = i2c_get_clientdata(client); @@ -798,8 +798,6 @@ static int anx6345_i2c_remove(struct i2c_client *client) kfree(anx6345->edid); mutex_destroy(&anx6345->lock); - - return 0; } static const struct i2c_device_id anx6345_id[] = { diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c index d2fc8676fab606ec69e58673105f0644cfe7f792..5997049fde5be3d16728acd62d271e12c14d7b3c 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c +++ b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c @@ -1357,7 +1357,7 @@ err_unregister_i2c: return err; } -static int anx78xx_i2c_remove(struct i2c_client *client) +static void anx78xx_i2c_remove(struct i2c_client *client) { struct anx78xx *anx78xx = i2c_get_clientdata(client); @@ -1366,8 +1366,6 @@ static int anx78xx_i2c_remove(struct i2c_client *client) unregister_i2c_dummy_clients(anx78xx); kfree(anx78xx->edid); - - return 0; } static const struct i2c_device_id anx78xx_id[] = { diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c index 8aadcc0aa90b084606cd9cd4f683111444f45ccd..df9370e0ff2306c73d829a44b63eb5b0c1876850 100644 --- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c +++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c @@ -1864,12 +1864,6 @@ EXPORT_SYMBOL_GPL(analogix_dp_remove); int analogix_dp_suspend(struct analogix_dp_device *dp) { clk_disable_unprepare(dp->clock); - - if (dp->plat_data->panel) { - if (drm_panel_unprepare(dp->plat_data->panel)) - DRM_ERROR("failed to turnoff the panel\n"); - } - return 0; } EXPORT_SYMBOL_GPL(analogix_dp_suspend); @@ -1884,13 +1878,6 @@ int analogix_dp_resume(struct analogix_dp_device *dp) return ret; } - if (dp->plat_data->panel) { - if (drm_panel_prepare(dp->plat_data->panel)) { - DRM_ERROR("failed to setup the panel\n"); - return -EBUSY; - } - } - return 0; } EXPORT_SYMBOL_GPL(analogix_dp_resume); diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c index 0c323b5a1c99122ab531573dd95b15fa63786f50..b0ff1ecb80a50ba95e2be6b5455c527a083efcd0 100644 --- a/drivers/gpu/drm/bridge/analogix/anx7625.c +++ b/drivers/gpu/drm/bridge/analogix/anx7625.c @@ -2715,7 +2715,7 @@ free_hdcp_wq: return ret; } -static int anx7625_i2c_remove(struct i2c_client *client) +static void anx7625_i2c_remove(struct i2c_client *client) { struct anx7625_data *platform = i2c_get_clientdata(client); @@ -2735,8 +2735,6 @@ static int anx7625_i2c_remove(struct i2c_client *client) if (platform->pdata.audio_en) anx7625_unregister_audio(platform); - - return 0; } static const struct i2c_device_id anx7625_id[] = { diff --git a/drivers/gpu/drm/bridge/chrontel-ch7033.c b/drivers/gpu/drm/bridge/chrontel-ch7033.c index ba060277c3fdfbf933f5cb090e37f92412c104e3..b94f39a86846c28e5da673ce364f2e45bfc3771e 100644 --- a/drivers/gpu/drm/bridge/chrontel-ch7033.c +++ b/drivers/gpu/drm/bridge/chrontel-ch7033.c @@ -583,14 +583,12 @@ static int ch7033_probe(struct i2c_client *client, return 0; } -static int ch7033_remove(struct i2c_client *client) +static void ch7033_remove(struct i2c_client *client) { struct device *dev = &client->dev; struct ch7033_priv *priv = dev_get_drvdata(dev); drm_bridge_remove(&priv->bridge); - - return 0; } static const struct of_device_id ch7033_dt_ids[] = { diff --git a/drivers/gpu/drm/bridge/cros-ec-anx7688.c b/drivers/gpu/drm/bridge/cros-ec-anx7688.c index 0f6d907432e384a65eed5b8d484d4bce042f2a9e..fa91bdeddef069c95ced5c68ac86136d714b5907 100644 --- a/drivers/gpu/drm/bridge/cros-ec-anx7688.c +++ b/drivers/gpu/drm/bridge/cros-ec-anx7688.c @@ -159,13 +159,11 @@ static int cros_ec_anx7688_bridge_probe(struct i2c_client *client) return 0; } -static int cros_ec_anx7688_bridge_remove(struct i2c_client *client) +static void cros_ec_anx7688_bridge_remove(struct i2c_client *client) { struct cros_ec_anx7688 *anx7688 = i2c_get_clientdata(client); drm_bridge_remove(&anx7688->bridge); - - return 0; } static const struct of_device_id cros_ec_anx7688_bridge_match_table[] = { diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c index a4302492cf8dfc037f70acb5b211d2bdf726b520..1fd09911fe8cf82e4355fd2de405e7b45f0e7180 100644 --- a/drivers/gpu/drm/bridge/ite-it6505.c +++ b/drivers/gpu/drm/bridge/ite-it6505.c @@ -3363,7 +3363,7 @@ static int it6505_i2c_probe(struct i2c_client *client, return 0; } -static int it6505_i2c_remove(struct i2c_client *client) +static void it6505_i2c_remove(struct i2c_client *client) { struct it6505 *it6505 = i2c_get_clientdata(client); @@ -3371,8 +3371,6 @@ static int it6505_i2c_remove(struct i2c_client *client) drm_dp_aux_unregister(&it6505->aux); it6505_debugfs_remove(it6505); it6505_poweroff(it6505); - - return 0; } static const struct i2c_device_id it6505_id[] = { diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c index 44278d54d35daa9ec5b8d4f8e08daa55eddfbaef..4f6f1deba28c6f8765d1a42e277c83ad090ed086 100644 --- a/drivers/gpu/drm/bridge/ite-it66121.c +++ b/drivers/gpu/drm/bridge/ite-it66121.c @@ -1623,15 +1623,13 @@ static int it66121_probe(struct i2c_client *client, return 0; } -static int it66121_remove(struct i2c_client *client) +static void it66121_remove(struct i2c_client *client) { struct it66121_ctx *ctx = i2c_get_clientdata(client); ite66121_power_off(ctx); drm_bridge_remove(&ctx->bridge); mutex_destroy(&ctx->lock); - - return 0; } static const struct of_device_id it66121_dt_match[] = { diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c index 3e870d45f8812bd29cdd03ca9b4cddb2a9cf84a2..a98efef0ba0e032af0f8918f6cf8b156aeef138a 100644 --- a/drivers/gpu/drm/bridge/lontium-lt8912b.c +++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c @@ -731,7 +731,7 @@ err_dt_parse: return ret; } -static int lt8912_remove(struct i2c_client *client) +static void lt8912_remove(struct i2c_client *client) { struct lt8912 *lt = i2c_get_clientdata(client); @@ -739,7 +739,6 @@ static int lt8912_remove(struct i2c_client *client) drm_bridge_remove(<->bridge); lt8912_free_i2c(lt); lt8912_put_dt(lt); - return 0; } static const struct of_device_id lt8912_dt_match[] = { diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c index 9a3e90427d12a74d1c605d2fcc4a107de0c04d6a..933ca028d612d431d39713ffedc4a8a53339031a 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9211.c +++ b/drivers/gpu/drm/bridge/lontium-lt9211.c @@ -766,13 +766,11 @@ static int lt9211_probe(struct i2c_client *client, return ret; } -static int lt9211_remove(struct i2c_client *client) +static void lt9211_remove(struct i2c_client *client) { struct lt9211 *ctx = i2c_get_clientdata(client); drm_bridge_remove(&ctx->bridge); - - return 0; } static struct i2c_device_id lt9211_id[] = { diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c index 5fccacc159f0038ae89a01eb28d9dba65fd095cf..7c0a99173b39fec5a00a42923ad2a0cf92a1392f 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9611.c +++ b/drivers/gpu/drm/bridge/lontium-lt9611.c @@ -1217,7 +1217,7 @@ err_of_put: return ret; } -static int lt9611_remove(struct i2c_client *client) +static void lt9611_remove(struct i2c_client *client) { struct lt9611 *lt9611 = i2c_get_clientdata(client); @@ -1229,8 +1229,6 @@ static int lt9611_remove(struct i2c_client *client) of_node_put(lt9611->dsi1_node); of_node_put(lt9611->dsi0_node); - - return 0; } static struct i2c_device_id lt9611_id[] = { diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c index fdf12d4c6416021102be5cc71ebccefd30e5985c..fa1ee6264d92113c82dad90d852f4f32d5c31451 100644 --- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c +++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c @@ -978,7 +978,7 @@ err_of_put: return ret; } -static int lt9611uxc_remove(struct i2c_client *client) +static void lt9611uxc_remove(struct i2c_client *client) { struct lt9611uxc *lt9611uxc = i2c_get_clientdata(client); @@ -993,8 +993,6 @@ static int lt9611uxc_remove(struct i2c_client *client) of_node_put(lt9611uxc->dsi1_node); of_node_put(lt9611uxc->dsi0_node); - - return 0; } static struct i2c_device_id lt9611uxc_id[] = { diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c index 72248a565579ecfad1940b6c1caca42ad826114b..97359f807bfc3ba31817879576de6a8013d3dd6f 100644 --- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c +++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c @@ -357,11 +357,9 @@ static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c, return ge_b850v3_register(); } -static int stdp4028_ge_b850v3_fw_remove(struct i2c_client *stdp4028_i2c) +static void stdp4028_ge_b850v3_fw_remove(struct i2c_client *stdp4028_i2c) { ge_b850v3_lvds_remove(); - - return 0; } static const struct i2c_device_id stdp4028_ge_b850v3_fw_i2c_table[] = { @@ -407,11 +405,9 @@ static int stdp2690_ge_b850v3_fw_probe(struct i2c_client *stdp2690_i2c, return ge_b850v3_register(); } -static int stdp2690_ge_b850v3_fw_remove(struct i2c_client *stdp2690_i2c) +static void stdp2690_ge_b850v3_fw_remove(struct i2c_client *stdp2690_i2c) { ge_b850v3_lvds_remove(); - - return 0; } static const struct i2c_device_id stdp2690_ge_b850v3_fw_i2c_table[] = { diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c index 1ab91f4e057b15cecca09157f3991f33a1cded85..0851101a8c72430aace124f1e61d1702a7fff975 100644 --- a/drivers/gpu/drm/bridge/nxp-ptn3460.c +++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c @@ -315,13 +315,11 @@ static int ptn3460_probe(struct i2c_client *client, return 0; } -static int ptn3460_remove(struct i2c_client *client) +static void ptn3460_remove(struct i2c_client *client) { struct ptn3460_bridge *ptn_bridge = i2c_get_clientdata(client); drm_bridge_remove(&ptn_bridge->bridge); - - return 0; } static const struct i2c_device_id ptn3460_i2c_table[] = { diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c index b5750e5f71d7e9aaafc30f3ec4f0049b19f3bad1..309de802863d8f315bc566dff697944e3c66fc1b 100644 --- a/drivers/gpu/drm/bridge/parade-ps8622.c +++ b/drivers/gpu/drm/bridge/parade-ps8622.c @@ -520,14 +520,12 @@ static int ps8622_probe(struct i2c_client *client, return 0; } -static int ps8622_remove(struct i2c_client *client) +static void ps8622_remove(struct i2c_client *client) { struct ps8622_bridge *ps8622 = i2c_get_clientdata(client); backlight_device_unregister(ps8622->bl); drm_bridge_remove(&ps8622->bridge); - - return 0; } static const struct i2c_device_id ps8622_i2c_table[] = { diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c index 7ab38d734ad67661c8882e975eed04b8365e4acc..878fb7d3732badeaf7928e86d70d54f582997d38 100644 --- a/drivers/gpu/drm/bridge/sii902x.c +++ b/drivers/gpu/drm/bridge/sii902x.c @@ -1145,7 +1145,7 @@ static int sii902x_probe(struct i2c_client *client, return ret; } -static int sii902x_remove(struct i2c_client *client) +static void sii902x_remove(struct i2c_client *client) { struct sii902x *sii902x = i2c_get_clientdata(client); @@ -1154,8 +1154,6 @@ static int sii902x_remove(struct i2c_client *client) drm_bridge_remove(&sii902x->bridge); regulator_bulk_disable(ARRAY_SIZE(sii902x->supplies), sii902x->supplies); - - return 0; } static const struct of_device_id sii902x_dt_ids[] = { diff --git a/drivers/gpu/drm/bridge/sii9234.c b/drivers/gpu/drm/bridge/sii9234.c index 15c98a7bd81c408a511615f88e0089ff6169027a..5b3061d4b5c3dee0be8824e61f8265377e2397de 100644 --- a/drivers/gpu/drm/bridge/sii9234.c +++ b/drivers/gpu/drm/bridge/sii9234.c @@ -936,14 +936,12 @@ static int sii9234_probe(struct i2c_client *client, return 0; } -static int sii9234_remove(struct i2c_client *client) +static void sii9234_remove(struct i2c_client *client) { struct sii9234 *ctx = i2c_get_clientdata(client); sii9234_cable_out(ctx); drm_bridge_remove(&ctx->bridge); - - return 0; } static const struct of_device_id sii9234_dt_match[] = { diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c index ab0bce4a988c530efe611fdce630a91e3074244c..511982a1cedb1cc9b5a46db3a865295d2692f048 100644 --- a/drivers/gpu/drm/bridge/sil-sii8620.c +++ b/drivers/gpu/drm/bridge/sil-sii8620.c @@ -2346,7 +2346,7 @@ static int sii8620_probe(struct i2c_client *client, return 0; } -static int sii8620_remove(struct i2c_client *client) +static void sii8620_remove(struct i2c_client *client) { struct sii8620 *ctx = i2c_get_clientdata(client); @@ -2360,8 +2360,6 @@ static int sii8620_remove(struct i2c_client *client) sii8620_cable_out(ctx); } drm_bridge_remove(&ctx->bridge); - - return 0; } static const struct of_device_id sii8620_dt_match[] = { diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c index 89e060b273efda9c8ab6af5782b74ec0cdd073b1..2a58eb271f7017084924411e4894ef770d64d5ef 100644 --- a/drivers/gpu/drm/bridge/tc358767.c +++ b/drivers/gpu/drm/bridge/tc358767.c @@ -2184,13 +2184,11 @@ static int tc_probe(struct i2c_client *client, const struct i2c_device_id *id) return 0; } -static int tc_remove(struct i2c_client *client) +static void tc_remove(struct i2c_client *client) { struct tc_data *tc = i2c_get_clientdata(client); drm_bridge_remove(&tc->bridge); - - return 0; } static const struct i2c_device_id tc358767_i2c_ids[] = { diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c index fd585bf925fe3630dea000ab9cb3fb8500a63f29..4c4b77ce8aba3257ecf20724f5e0b50da1b4b1b0 100644 --- a/drivers/gpu/drm/bridge/tc358768.c +++ b/drivers/gpu/drm/bridge/tc358768.c @@ -1072,13 +1072,11 @@ static int tc358768_i2c_probe(struct i2c_client *client, return mipi_dsi_host_register(&priv->dsi_host); } -static int tc358768_i2c_remove(struct i2c_client *client) +static void tc358768_i2c_remove(struct i2c_client *client) { struct tc358768_priv *priv = i2c_get_clientdata(client); mipi_dsi_host_unregister(&priv->dsi_host); - - return 0; } static struct i2c_driver tc358768_driver = { diff --git a/drivers/gpu/drm/bridge/tc358775.c b/drivers/gpu/drm/bridge/tc358775.c index a5f5eae1e80f8a54c27f0fa6d6d4fc4c260e534e..3ceb0e9f9bdc5b78583f0fec694c930c5e9c1a32 100644 --- a/drivers/gpu/drm/bridge/tc358775.c +++ b/drivers/gpu/drm/bridge/tc358775.c @@ -704,13 +704,11 @@ err_bridge_remove: return ret; } -static int tc_remove(struct i2c_client *client) +static void tc_remove(struct i2c_client *client) { struct tc_data *tc = i2c_get_clientdata(client); drm_bridge_remove(&tc->bridge); - - return 0; } static const struct i2c_device_id tc358775_i2c_ids[] = { diff --git a/drivers/gpu/drm/bridge/ti-dlpc3433.c b/drivers/gpu/drm/bridge/ti-dlpc3433.c index cef454862b67f44ea77e3bb9a86e843cc91c3653..186a9e2ff24dc76e37354c4214ffb4976eac0847 100644 --- a/drivers/gpu/drm/bridge/ti-dlpc3433.c +++ b/drivers/gpu/drm/bridge/ti-dlpc3433.c @@ -379,14 +379,12 @@ err_remove_bridge: return ret; } -static int dlpc3433_remove(struct i2c_client *client) +static void dlpc3433_remove(struct i2c_client *client) { struct dlpc *dlpc = i2c_get_clientdata(client); drm_bridge_remove(&dlpc->bridge); of_node_put(dlpc->host_node); - - return 0; } static const struct i2c_device_id dlpc3433_id[] = { diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c index 14e7aa77e7584d48f406f8ec14c41cac7a0f0a57..7ba9467fff1291fa48ef90fa9b17605b47f27055 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c @@ -708,13 +708,11 @@ err_remove_bridge: return ret; } -static int sn65dsi83_remove(struct i2c_client *client) +static void sn65dsi83_remove(struct i2c_client *client) { struct sn65dsi83 *ctx = i2c_get_clientdata(client); drm_bridge_remove(&ctx->bridge); - - return 0; } static struct i2c_device_id sn65dsi83_id[] = { diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c index 401fe61217c7393a9659f15295d776ac84c05edf..b9635abbad16d7fee27930fbe6f11ab2f7eee182 100644 --- a/drivers/gpu/drm/bridge/ti-tfp410.c +++ b/drivers/gpu/drm/bridge/ti-tfp410.c @@ -394,11 +394,9 @@ static int tfp410_i2c_probe(struct i2c_client *client, return tfp410_init(&client->dev, true); } -static int tfp410_i2c_remove(struct i2c_client *client) +static void tfp410_i2c_remove(struct i2c_client *client) { tfp410_fini(&client->dev); - - return 0; } static const struct i2c_device_id tfp410_i2c_ids[] = { diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c index 9f055d9710eab91b63ec42f10d334ec380c169ad..16565a0a5da6d98d1001cfd3acb9cd547e73b105 100644 --- a/drivers/gpu/drm/display/drm_dp_helper.c +++ b/drivers/gpu/drm/display/drm_dp_helper.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -40,6 +41,18 @@ #include "drm_dp_helper_internal.h" +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + struct dp_aux_backlight { struct backlight_device *base; struct drm_dp_aux *aux; diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c index 1f0a270ac98478b054e5b2185813ba2fd5ea6217..f5fb22e0d0337301af71dabba54ac91cd8f0e766 100644 --- a/drivers/gpu/drm/drm_crtc_helper.c +++ b/drivers/gpu/drm/drm_crtc_helper.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,18 @@ #include "drm_crtc_helper_internal.h" +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + /** * DOC: overview * diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c index 493922069c9041194a0d9ab3519f92e9e2fd4e85..01ee3febb81315d1afbbc172c94d2d56b337b559 100644 --- a/drivers/gpu/drm/drm_debugfs.c +++ b/drivers/gpu/drm/drm_debugfs.c @@ -377,8 +377,8 @@ static int vrr_range_show(struct seq_file *m, void *data) if (connector->status != connector_status_connected) return -ENODEV; - seq_printf(m, "Min: %u\n", (u8)connector->display_info.monitor_range.min_vfreq); - seq_printf(m, "Max: %u\n", (u8)connector->display_info.monitor_range.max_vfreq); + seq_printf(m, "Min: %u\n", connector->display_info.monitor_range.min_vfreq); + seq_printf(m, "Max: %u\n", connector->display_info.monitor_range.max_vfreq); return 0; } diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 73c614efc13281a9deb9be990f5fbab3fe2ab2bf..47465b9765f1a483b0249e6e204fd83459c86618 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -6112,12 +6112,14 @@ static void drm_parse_cea_ext(struct drm_connector *connector, } static -void get_monitor_range(const struct detailed_timing *timing, - void *info_monitor_range) +void get_monitor_range(const struct detailed_timing *timing, void *c) { - struct drm_monitor_range_info *monitor_range = info_monitor_range; + struct detailed_mode_closure *closure = c; + struct drm_display_info *info = &closure->connector->display_info; + struct drm_monitor_range_info *monitor_range = &info->monitor_range; const struct detailed_non_pixel *data = &timing->data.other_data; const struct detailed_data_monitor_range *range = &data->data.range; + const struct edid *edid = closure->drm_edid->edid; if (!is_display_descriptor(timing, EDID_DETAIL_MONITOR_RANGE)) return; @@ -6136,12 +6138,23 @@ void get_monitor_range(const struct detailed_timing *timing, monitor_range->min_vfreq = range->min_vfreq; monitor_range->max_vfreq = range->max_vfreq; + + if (edid->revision >= 4) { + if (data->pad2 & DRM_EDID_RANGE_OFFSET_MIN_VFREQ) + monitor_range->min_vfreq += 255; + if (data->pad2 & DRM_EDID_RANGE_OFFSET_MAX_VFREQ) + monitor_range->max_vfreq += 255; + } } static void drm_get_monitor_range(struct drm_connector *connector, const struct drm_edid *drm_edid) { - struct drm_display_info *info = &connector->display_info; + const struct drm_display_info *info = &connector->display_info; + struct detailed_mode_closure closure = { + .connector = connector, + .drm_edid = drm_edid, + }; if (drm_edid->edid->revision < 4) return; @@ -6149,8 +6162,7 @@ static void drm_get_monitor_range(struct drm_connector *connector, if (!(drm_edid->edid->features & DRM_EDID_FEATURE_CONTINUOUS_FREQ)) return; - drm_for_each_detailed_block(drm_edid, get_monitor_range, - &info->monitor_range); + drm_for_each_detailed_block(drm_edid, get_monitor_range, &closure); DRM_DEBUG_KMS("Supported Monitor Refresh rate range is %d Hz - %d Hz\n", info->monitor_range.min_vfreq, diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index dbee4863e4f71e1fb13daec56f5cf66901b1b083..b8db675e7fb5e6a113603a730124f1348c2e5751 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -165,24 +165,10 @@ void drm_gem_private_object_init(struct drm_device *dev, obj->resv = &obj->_resv; drm_vma_node_reset(&obj->vma_node); + INIT_LIST_HEAD(&obj->lru_node); } EXPORT_SYMBOL(drm_gem_private_object_init); -static void -drm_gem_remove_prime_handles(struct drm_gem_object *obj, struct drm_file *filp) -{ - /* - * Note: obj->dma_buf can't disappear as long as we still hold a - * handle reference in obj->handle_count. - */ - mutex_lock(&filp->prime.lock); - if (obj->dma_buf) { - drm_prime_remove_buf_handle_locked(&filp->prime, - obj->dma_buf); - } - mutex_unlock(&filp->prime.lock); -} - /** * drm_gem_object_handle_free - release resources bound to userspace handles * @obj: GEM object to clean up. @@ -253,7 +239,7 @@ drm_gem_object_release_handle(int id, void *ptr, void *data) if (obj->funcs->close) obj->funcs->close(obj, file_priv); - drm_gem_remove_prime_handles(obj, file_priv); + drm_prime_remove_buf_handle(&file_priv->prime, id); drm_vma_node_revoke(&obj->vma_node, file_priv); drm_gem_object_handle_put_unlocked(obj); @@ -951,6 +937,7 @@ drm_gem_object_release(struct drm_gem_object *obj) dma_resv_fini(&obj->_resv); drm_gem_free_mmap_offset(obj); + drm_gem_lru_remove(obj); } EXPORT_SYMBOL(drm_gem_object_release); @@ -1298,3 +1285,171 @@ drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, ww_acquire_fini(acquire_ctx); } EXPORT_SYMBOL(drm_gem_unlock_reservations); + +/** + * drm_gem_lru_init - initialize a LRU + * + * @lru: The LRU to initialize + * @lock: The lock protecting the LRU + */ +void +drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock) +{ + lru->lock = lock; + lru->count = 0; + INIT_LIST_HEAD(&lru->list); +} +EXPORT_SYMBOL(drm_gem_lru_init); + +static void +drm_gem_lru_remove_locked(struct drm_gem_object *obj) +{ + obj->lru->count -= obj->size >> PAGE_SHIFT; + WARN_ON(obj->lru->count < 0); + list_del(&obj->lru_node); + obj->lru = NULL; +} + +/** + * drm_gem_lru_remove - remove object from whatever LRU it is in + * + * If the object is currently in any LRU, remove it. + * + * @obj: The GEM object to remove from current LRU + */ +void +drm_gem_lru_remove(struct drm_gem_object *obj) +{ + struct drm_gem_lru *lru = obj->lru; + + if (!lru) + return; + + mutex_lock(lru->lock); + drm_gem_lru_remove_locked(obj); + mutex_unlock(lru->lock); +} +EXPORT_SYMBOL(drm_gem_lru_remove); + +static void +drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj) +{ + lockdep_assert_held_once(lru->lock); + + if (obj->lru) + drm_gem_lru_remove_locked(obj); + + lru->count += obj->size >> PAGE_SHIFT; + list_add_tail(&obj->lru_node, &lru->list); + obj->lru = lru; +} + +/** + * drm_gem_lru_move_tail - move the object to the tail of the LRU + * + * If the object is already in this LRU it will be moved to the + * tail. Otherwise it will be removed from whichever other LRU + * it is in (if any) and moved into this LRU. + * + * @lru: The LRU to move the object into. + * @obj: The GEM object to move into this LRU + */ +void +drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj) +{ + mutex_lock(lru->lock); + drm_gem_lru_move_tail_locked(lru, obj); + mutex_unlock(lru->lock); +} +EXPORT_SYMBOL(drm_gem_lru_move_tail); + +/** + * drm_gem_lru_scan - helper to implement shrinker.scan_objects + * + * If the shrink callback succeeds, it is expected that the driver + * move the object out of this LRU. + * + * If the LRU possibly contain active buffers, it is the responsibility + * of the shrink callback to check for this (ie. dma_resv_test_signaled()) + * or if necessary block until the buffer becomes idle. + * + * @lru: The LRU to scan + * @nr_to_scan: The number of pages to try to reclaim + * @shrink: Callback to try to shrink/reclaim the object. + */ +unsigned long +drm_gem_lru_scan(struct drm_gem_lru *lru, unsigned nr_to_scan, + bool (*shrink)(struct drm_gem_object *obj)) +{ + struct drm_gem_lru still_in_lru; + struct drm_gem_object *obj; + unsigned freed = 0; + + drm_gem_lru_init(&still_in_lru, lru->lock); + + mutex_lock(lru->lock); + + while (freed < nr_to_scan) { + obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node); + + if (!obj) + break; + + drm_gem_lru_move_tail_locked(&still_in_lru, obj); + + /* + * If it's in the process of being freed, gem_object->free() + * may be blocked on lock waiting to remove it. So just + * skip it. + */ + if (!kref_get_unless_zero(&obj->refcount)) + continue; + + /* + * Now that we own a reference, we can drop the lock for the + * rest of the loop body, to reduce contention with other + * code paths that need the LRU lock + */ + mutex_unlock(lru->lock); + + /* + * Note that this still needs to be trylock, since we can + * hit shrinker in response to trying to get backing pages + * for this obj (ie. while it's lock is already held) + */ + if (!dma_resv_trylock(obj->resv)) + goto tail; + + if (shrink(obj)) { + freed += obj->size >> PAGE_SHIFT; + + /* + * If we succeeded in releasing the object's backing + * pages, we expect the driver to have moved the object + * out of this LRU + */ + WARN_ON(obj->lru == &still_in_lru); + WARN_ON(obj->lru == lru); + } + + dma_resv_unlock(obj->resv); + +tail: + drm_gem_object_put(obj); + mutex_lock(lru->lock); + } + + /* + * Move objects we've skipped over out of the temporary still_in_lru + * back into this LRU + */ + list_for_each_entry (obj, &still_in_lru.list, lru_node) + obj->lru = lru; + list_splice_tail(&still_in_lru.list, &lru->list); + lru->count += still_in_lru.count; + + mutex_unlock(lru->lock); + + return freed; +} +EXPORT_SYMBOL(drm_gem_lru_scan); diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 1fbbc19f1ac097b040e6ebcbff34b2fdeba8c9d1..7bb98e6a446d08311e9714536f7397bfc7511c7b 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -74,8 +74,8 @@ int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data, void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv); void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv); -void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv, - struct dma_buf *dma_buf); +void drm_prime_remove_buf_handle(struct drm_prime_file_private *prime_fpriv, + uint32_t handle); /* drm_drv.c */ struct drm_minor *drm_minor_acquire(unsigned int minor_id); diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index ef50c4e2e50909348c55d45e583124c335036c83..20e109a802ae87091a4d6777a20163df00916a15 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -190,29 +190,33 @@ static int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpri return -ENOENT; } -void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv, - struct dma_buf *dma_buf) +void drm_prime_remove_buf_handle(struct drm_prime_file_private *prime_fpriv, + uint32_t handle) { struct rb_node *rb; - rb = prime_fpriv->dmabufs.rb_node; + mutex_lock(&prime_fpriv->lock); + + rb = prime_fpriv->handles.rb_node; while (rb) { struct drm_prime_member *member; - member = rb_entry(rb, struct drm_prime_member, dmabuf_rb); - if (member->dma_buf == dma_buf) { + member = rb_entry(rb, struct drm_prime_member, handle_rb); + if (member->handle == handle) { rb_erase(&member->handle_rb, &prime_fpriv->handles); rb_erase(&member->dmabuf_rb, &prime_fpriv->dmabufs); - dma_buf_put(dma_buf); + dma_buf_put(member->dma_buf); kfree(member); - return; - } else if (member->dma_buf < dma_buf) { + break; + } else if (member->handle < handle) { rb = rb->rb_right; } else { rb = rb->rb_left; } } + + mutex_unlock(&prime_fpriv->lock); } void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv) diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c index f783d4963d4be23901e27ddc9afecbd6ee4dbb0e..5b93c11895bb1e6336c016b1cf27919e5cbae62f 100644 --- a/drivers/gpu/drm/drm_print.c +++ b/drivers/gpu/drm/drm_print.c @@ -23,14 +23,13 @@ * Rob Clark */ -#define DEBUG /* for pr_debug() */ - #include #include #include #include #include +#include #include #include @@ -40,7 +39,7 @@ * __drm_debug: Enable debug output. * Bitmask of DRM_UT_x. See include/drm/drm_print.h for details. */ -unsigned int __drm_debug; +unsigned long __drm_debug; EXPORT_SYMBOL(__drm_debug); MODULE_PARM_DESC(debug, "Enable debug output, where each bit enables a debug category.\n" @@ -52,7 +51,30 @@ MODULE_PARM_DESC(debug, "Enable debug output, where each bit enables a debug cat "\t\tBit 5 (0x20) will enable VBL messages (vblank code)\n" "\t\tBit 7 (0x80) will enable LEASE messages (leasing code)\n" "\t\tBit 8 (0x100) will enable DP messages (displayport code)"); -module_param_named(debug, __drm_debug, int, 0600); + +#if !defined(CONFIG_DRM_USE_DYNAMIC_DEBUG) +module_param_named(debug, __drm_debug, ulong, 0600); +#else +/* classnames must match vals of enum drm_debug_category */ +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + +static struct ddebug_class_param drm_debug_bitmap = { + .bits = &__drm_debug, + .flags = "p", + .map = &drm_debug_classes, +}; +module_param_cb(debug, ¶m_ops_dyndbg_classes, &drm_debug_bitmap, 0600); +#endif void __drm_puts_coredump(struct drm_printer *p, const char *str) { @@ -162,7 +184,8 @@ EXPORT_SYMBOL(__drm_printfn_info); void __drm_printfn_debug(struct drm_printer *p, struct va_format *vaf) { - pr_debug("%s %pV", p->prefix, vaf); + /* pr_debug callsite decorations are unhelpful here */ + printk(KERN_DEBUG "%s %pV", p->prefix, vaf); } EXPORT_SYMBOL(__drm_printfn_debug); @@ -256,15 +279,16 @@ void drm_dev_printk(const struct device *dev, const char *level, } EXPORT_SYMBOL(drm_dev_printk); -void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, - const char *format, ...) +void __drm_dev_dbg(struct _ddebug *desc, const struct device *dev, + enum drm_debug_category category, const char *format, ...) { struct va_format vaf; va_list args; - if (!drm_debug_enabled(category)) + if (!__drm_debug_enabled(category)) return; + /* we know we are printing for either syslog, tracefs, or both */ va_start(args, format); vaf.fmt = format; vaf.va = &args; @@ -278,14 +302,14 @@ void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, va_end(args); } -EXPORT_SYMBOL(drm_dev_dbg); +EXPORT_SYMBOL(__drm_dev_dbg); -void __drm_dbg(enum drm_debug_category category, const char *format, ...) +void ___drm_dbg(struct _ddebug *desc, enum drm_debug_category category, const char *format, ...) { struct va_format vaf; va_list args; - if (!drm_debug_enabled(category)) + if (!__drm_debug_enabled(category)) return; va_start(args, format); @@ -297,7 +321,7 @@ void __drm_dbg(enum drm_debug_category category, const char *format, ...) va_end(args); } -EXPORT_SYMBOL(__drm_dbg); +EXPORT_SYMBOL(___drm_dbg); void __drm_err(const char *format, ...) { diff --git a/drivers/gpu/drm/exynos/exynos_drm_dma.c b/drivers/gpu/drm/exynos/exynos_drm_dma.c index bf33c3084cb41fe760fcf69cebce89b0a9d8c4fb..a971590b81323021cd697b8a71bc4686e2c40ca4 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dma.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dma.c @@ -4,7 +4,6 @@ // Author: Inki Dae // Author: Andrzej Hajda -#include #include #include #include diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c index 10b0036f8a2e2d49f10daa2843018e3edf8c6e8a..b7c11bdce2c89a9fabb991a26f203416f1d15fb0 100644 --- a/drivers/gpu/drm/exynos/exynos_hdmi.c +++ b/drivers/gpu/drm/exynos/exynos_hdmi.c @@ -893,7 +893,7 @@ static int hdmi_get_modes(struct drm_connector *connector) if (!edid) return -ENODEV; - hdata->dvi_mode = !drm_detect_hdmi_monitor(edid); + hdata->dvi_mode = !connector->display_info.is_hdmi; DRM_DEV_DEBUG_KMS(hdata->dev, "%s : width[%d] x height[%d]\n", (hdata->dvi_mode ? "dvi monitor" : "hdmi monitor"), edid->width_cm, edid->height_cm); @@ -922,8 +922,8 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock) return -EINVAL; } -static int hdmi_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) +static enum drm_mode_status hdmi_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) { struct hdmi_context *hdata = connector_to_hdmi(connector); int ret; diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c index 65260a658684a32b3c4a6a9059d3eab70480a1dd..8d333db813b7d180a5d668f29b29a826e7d362fd 100644 --- a/drivers/gpu/drm/exynos/exynos_mixer.c +++ b/drivers/gpu/drm/exynos/exynos_mixer.c @@ -1045,7 +1045,7 @@ static void mixer_atomic_disable(struct exynos_drm_crtc *crtc) clear_bit(MXR_BIT_POWERED, &ctx->flags); } -static int mixer_mode_valid(struct exynos_drm_crtc *crtc, +static enum drm_mode_status mixer_mode_valid(struct exynos_drm_crtc *crtc, const struct drm_display_mode *mode) { struct mixer_context *ctx = crtc->ctx; diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index dffe37490206d241cda861e2d5000ff0cd5a6094..4b7627a7263788b939025dd298699e2565b1fa6e 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -112,12 +112,12 @@ static void psb_gem_free_object(struct drm_gem_object *obj) { struct psb_gem_object *pobj = to_psb_gem_object(obj); - drm_gem_object_release(obj); - /* Undo the mmap pin if we are destroying the object */ if (pobj->mmapping) psb_gem_unpin(pobj); + drm_gem_object_release(obj); + WARN_ON(pobj->in_gart && !pobj->stolen); release_resource(&pobj->resource); diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c index b0ea911b27de15bd7ac5bc9de0c2399045d9fc86..fe7b8436f87a36101a3ae8031f271491d7b0de8a 100644 --- a/drivers/gpu/drm/gma500/gma_display.c +++ b/drivers/gpu/drm/gma500/gma_display.c @@ -532,15 +532,18 @@ int gma_crtc_page_flip(struct drm_crtc *crtc, WARN_ON(drm_crtc_vblank_get(crtc) != 0); gma_crtc->page_flip_event = event; + spin_unlock_irqrestore(&dev->event_lock, flags); /* Call this locked if we want an event at vblank interrupt. */ ret = crtc_funcs->mode_set_base(crtc, crtc->x, crtc->y, old_fb); if (ret) { - gma_crtc->page_flip_event = NULL; - drm_crtc_vblank_put(crtc); + spin_lock_irqsave(&dev->event_lock, flags); + if (gma_crtc->page_flip_event) { + gma_crtc->page_flip_event = NULL; + drm_crtc_vblank_put(crtc); + } + spin_unlock_irqrestore(&dev->event_lock, flags); } - - spin_unlock_irqrestore(&dev->event_lock, flags); } else { ret = crtc_funcs->mode_set_base(crtc, crtc->x, crtc->y, old_fb); } diff --git a/drivers/gpu/drm/hisilicon/hibmc/Kconfig b/drivers/gpu/drm/hisilicon/hibmc/Kconfig index 073adfe438dddc4775b69cce4617c735059c71a7..4e41c144a2902a20eed70c8c669f59392f1681b5 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/Kconfig +++ b/drivers/gpu/drm/hisilicon/hibmc/Kconfig @@ -2,6 +2,7 @@ config DRM_HISI_HIBMC tristate "DRM Support for Hisilicon Hibmc" depends on DRM && PCI && (ARM64 || COMPILE_TEST) + depends on MMU select DRM_KMS_HELPER select DRM_VRAM_HELPER select DRM_TTM diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c index 6d11e7938c8370f4616a2e6c726c9258e1926d6f..ca127ff797f756ad1b32b318edccf5233d2bbc95 100644 --- a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c +++ b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c @@ -23,9 +23,6 @@ #define DRIVER_MAJOR 1 #define DRIVER_MINOR 0 -#define PCI_VENDOR_ID_MICROSOFT 0x1414 -#define PCI_DEVICE_ID_HYPERV_VIDEO 0x5353 - DEFINE_DRM_GEM_FOPS(hv_fops); static struct drm_driver hyperv_driver = { @@ -133,7 +130,6 @@ static int hyperv_vmbus_probe(struct hv_device *hdev, } ret = hyperv_setup_vram(hv, hdev); - if (ret) goto err_vmbus_close; @@ -146,22 +142,22 @@ static int hyperv_vmbus_probe(struct hv_device *hdev, if (ret) drm_warn(dev, "Failed to update vram location.\n"); - hv->dirt_needed = true; - ret = hyperv_mode_config_init(hv); if (ret) - goto err_vmbus_close; + goto err_free_mmio; ret = drm_dev_register(dev, 0); if (ret) { drm_err(dev, "Failed to register drm driver.\n"); - goto err_vmbus_close; + goto err_free_mmio; } drm_fbdev_generic_setup(dev, 0); return 0; +err_free_mmio: + vmbus_free_mmio(hv->mem->start, hv->fb_size); err_vmbus_close: vmbus_close(hdev->channel); err_hv_set_drv_data: diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c index 76a182a9a7654736c175c7eac9d3b814c815bb2a..013a7829182df80748b55082abc8f0c30a5aaebe 100644 --- a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c +++ b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c @@ -208,7 +208,7 @@ static inline int hyperv_sendpacket(struct hv_device *hdev, struct synthvid_msg VM_PKT_DATA_INBAND, 0); if (ret) - drm_err(&hv->dev, "Unable to send packet via vmbus\n"); + drm_err_ratelimited(&hv->dev, "Unable to send packet via vmbus; error %d\n", ret); return ret; } diff --git a/drivers/gpu/drm/i2c/ch7006_drv.c b/drivers/gpu/drm/i2c/ch7006_drv.c index b91e48d2190d216e1f73363443b30ca5ed66bba2..578b738859b9e5241a617e23ec7c656892083986 100644 --- a/drivers/gpu/drm/i2c/ch7006_drv.c +++ b/drivers/gpu/drm/i2c/ch7006_drv.c @@ -417,11 +417,9 @@ fail: return -ENODEV; } -static int ch7006_remove(struct i2c_client *client) +static void ch7006_remove(struct i2c_client *client) { ch7006_dbg(client, "\n"); - - return 0; } static int ch7006_resume(struct device *dev) diff --git a/drivers/gpu/drm/i2c/sil164_drv.c b/drivers/gpu/drm/i2c/sil164_drv.c index 741886b5441973b19816a229a4a317dd0c14bf2a..1bc0b5de44995dea77bcfad216a920e12b2c04d1 100644 --- a/drivers/gpu/drm/i2c/sil164_drv.c +++ b/drivers/gpu/drm/i2c/sil164_drv.c @@ -370,12 +370,6 @@ sil164_probe(struct i2c_client *client, const struct i2c_device_id *id) return 0; } -static int -sil164_remove(struct i2c_client *client) -{ - return 0; -} - static struct i2c_client * sil164_detect_slave(struct i2c_client *client) { @@ -427,7 +421,6 @@ MODULE_DEVICE_TABLE(i2c, sil164_ids); static struct drm_i2c_encoder_driver sil164_driver = { .i2c_driver = { .probe = sil164_probe, - .remove = sil164_remove, .driver = { .name = "sil164", }, diff --git a/drivers/gpu/drm/i2c/tda9950.c b/drivers/gpu/drm/i2c/tda9950.c index 5b03fdd1eaa4057a20103a1093e6ba6691b0d27c..9ed54e7ccff2a5ec7e5be6ff7e869cc1c52478ad 100644 --- a/drivers/gpu/drm/i2c/tda9950.c +++ b/drivers/gpu/drm/i2c/tda9950.c @@ -478,14 +478,12 @@ static int tda9950_probe(struct i2c_client *client, return 0; } -static int tda9950_remove(struct i2c_client *client) +static void tda9950_remove(struct i2c_client *client) { struct tda9950_priv *priv = i2c_get_clientdata(client); cec_notifier_cec_adap_unregister(priv->notify, priv->adap); cec_unregister_adapter(priv->adap); - - return 0; } static struct i2c_device_id tda9950_ids[] = { diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c index f8eb6f69be05757b54e359c5e8a9e72a0dccb177..d444e7fffb544b681137655db2a6c6b6fa259676 100644 --- a/drivers/gpu/drm/i2c/tda998x_drv.c +++ b/drivers/gpu/drm/i2c/tda998x_drv.c @@ -2076,11 +2076,10 @@ tda998x_probe(struct i2c_client *client, const struct i2c_device_id *id) return ret; } -static int tda998x_remove(struct i2c_client *client) +static void tda998x_remove(struct i2c_client *client) { component_del(&client->dev, &tda998x_ops); tda998x_destroy(&client->dev); - return 0; } #ifdef CONFIG_OF diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 522ef9b4aff329625086f8ef5cf9ed9053a2f716..a26edcdadc217906834b0a7ebc44e0ceca5bb4c1 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -123,6 +123,7 @@ gt-y += \ gt/intel_ring.o \ gt/intel_ring_submission.o \ gt/intel_rps.o \ + gt/intel_sa_media.o \ gt/intel_sseu.o \ gt/intel_sseu_debugfs.o \ gt/intel_timeline.o \ @@ -257,7 +258,8 @@ i915-y += \ display/intel_vga.o \ display/i9xx_plane.o \ display/skl_scaler.o \ - display/skl_universal_plane.o + display/skl_universal_plane.o \ + display/skl_watermark.o i915-$(CONFIG_ACPI) += \ display/intel_acpi.o \ display/intel_opregion.o diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index 82ad8fe7440c055d86cbdbf8408bea822d18db13..e3e3d27ffb53fda78ff0aec31b977d3001993ca1 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -1169,7 +1169,7 @@ intel_dp_hotplug(struct intel_encoder *encoder, static bool ibx_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin]; + u32 bit = dev_priv->display.hotplug.pch_hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, SDEISR) & bit; } @@ -1223,7 +1223,7 @@ static bool gm45_digital_port_connected(struct intel_encoder *encoder) static bool ilk_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; + u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, DEISR) & bit; } diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c index 5fbd2ae958692009a8066fc67df2e23ad1851ebb..2b73f5ff0d02b6c730c7fe84bad63f01f50c0e7f 100644 --- a/drivers/gpu/drm/i915/display/g4x_hdmi.c +++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c @@ -120,7 +120,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder, pipe_config->hw.adjusted_mode.flags |= flags; if ((tmp & SDVO_COLOR_FORMAT_MASK) == HDMI_COLOR_FORMAT_12bpc) - dotclock = pipe_config->port_clock * 2 / 3; + dotclock = DIV_ROUND_CLOSEST(pipe_config->port_clock * 2, 3); else dotclock = pipe_config->port_clock; diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c index 861dcd2eb890a0f4beadff4bbe9248da2bf25e79..a5be4af792cb0b7a7388074d18de86f31afa7e3a 100644 --- a/drivers/gpu/drm/i915/display/hsw_ips.c +++ b/drivers/gpu/drm/i915/display/hsw_ips.c @@ -202,7 +202,7 @@ bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state) * Should measure whether using a lower cdclk w/o IPS */ if (IS_BROADWELL(i915) && - crtc_state->pixel_rate > i915->max_cdclk_freq * 95 / 100) + crtc_state->pixel_rate > i915->display.cdclk.max_cdclk_freq * 95 / 100) return false; return true; diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c index 0f35f2facdfc8291638639db09fa029c53b5ed77..5afbe3e98ee8e8ccb1861d245d7b43f256900b88 100644 --- a/drivers/gpu/drm/i915/display/i9xx_plane.c +++ b/drivers/gpu/drm/i915/display/i9xx_plane.c @@ -125,7 +125,7 @@ static struct intel_fbc *i9xx_plane_fbc(struct drm_i915_private *dev_priv, enum i9xx_plane_id i9xx_plane) { if (i9xx_plane_has_fbc(dev_priv, i9xx_plane)) - return dev_priv->fbc[INTEL_FBC_A]; + return dev_priv->display.fbc[INTEL_FBC_A]; else return NULL; } diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index 5dcfa7feffa9ff924cd48cc36198c51610ef3a28..ed4d93942dbd20c254aaf32c79e94d767adc0459 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -33,6 +33,7 @@ #include "icl_dsi_regs.h" #include "intel_atomic.h" #include "intel_backlight.h" +#include "intel_backlight_regs.h" #include "intel_combo_phy.h" #include "intel_combo_phy_regs.h" #include "intel_connector.h" @@ -641,13 +642,13 @@ static void gen11_dsi_gate_clocks(struct intel_encoder *encoder) u32 tmp; enum phy phy; - mutex_lock(&dev_priv->dpll.lock); + mutex_lock(&dev_priv->display.dpll.lock); tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) tmp |= ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp); - mutex_unlock(&dev_priv->dpll.lock); + mutex_unlock(&dev_priv->display.dpll.lock); } static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) @@ -657,13 +658,13 @@ static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) u32 tmp; enum phy phy; - mutex_lock(&dev_priv->dpll.lock); + mutex_lock(&dev_priv->display.dpll.lock); tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) tmp &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp); - mutex_unlock(&dev_priv->dpll.lock); + mutex_unlock(&dev_priv->display.dpll.lock); } static bool gen11_dsi_is_clock_enabled(struct intel_encoder *encoder) @@ -693,7 +694,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder, enum phy phy; u32 val; - mutex_lock(&dev_priv->dpll.lock); + mutex_lock(&dev_priv->display.dpll.lock); val = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) { @@ -709,7 +710,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder, intel_de_posting_read(dev_priv, ICL_DPCLKA_CFGCR0); - mutex_unlock(&dev_priv->dpll.lock); + mutex_unlock(&dev_priv->display.dpll.lock); } static void @@ -1629,6 +1630,8 @@ static int gen11_dsi_dsc_compute_config(struct intel_encoder *encoder, /* FIXME: initialize from VBT */ vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST; + vdsc_cfg->pic_height = crtc_state->hw.adjusted_mode.crtc_vdisplay; + ret = intel_dsc_compute_params(crtc_state); if (ret) return ret; @@ -2070,8 +2073,11 @@ void icl_dsi_init(struct drm_i915_private *dev_priv) else intel_dsi->ports = BIT(port); - intel_dsi->dcs_backlight_ports = intel_connector->panel.vbt.dsi.bl_ports; - intel_dsi->dcs_cabc_ports = intel_connector->panel.vbt.dsi.cabc_ports; + if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports)) + intel_connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports; + + if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports)) + intel_connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports; for_each_dsi_port(port, intel_dsi->ports) { struct intel_dsi_host *host; diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c index b94973b5633f2ffd5f3651cbb662e7abdb5ce661..18f0a5ae3bacdc0170f26d312e432448bfdda268 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic.c +++ b/drivers/gpu/drm/i915/display/intel_atomic.c @@ -62,9 +62,9 @@ int intel_digital_connector_atomic_get_property(struct drm_connector *connector, struct intel_digital_connector_state *intel_conn_state = to_intel_digital_connector_state(state); - if (property == dev_priv->force_audio_property) + if (property == dev_priv->display.properties.force_audio) *val = intel_conn_state->force_audio; - else if (property == dev_priv->broadcast_rgb_property) + else if (property == dev_priv->display.properties.broadcast_rgb) *val = intel_conn_state->broadcast_rgb; else { drm_dbg_atomic(&dev_priv->drm, @@ -95,12 +95,12 @@ int intel_digital_connector_atomic_set_property(struct drm_connector *connector, struct intel_digital_connector_state *intel_conn_state = to_intel_digital_connector_state(state); - if (property == dev_priv->force_audio_property) { + if (property == dev_priv->display.properties.force_audio) { intel_conn_state->force_audio = val; return 0; } - if (property == dev_priv->broadcast_rgb_property) { + if (property == dev_priv->display.properties.broadcast_rgb) { intel_conn_state->broadcast_rgb = val; return 0; } diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index dd876dbbaa394d2f4a1c56d500ae1d91e63af46d..aaa6708256d586a92350bc28186a145f37b0990b 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -42,9 +42,9 @@ #include "intel_display_types.h" #include "intel_fb.h" #include "intel_fb_pin.h" -#include "intel_pm.h" #include "intel_sprite.h" #include "skl_scaler.h" +#include "skl_watermark.h" static void intel_plane_state_reset(struct intel_plane_state *plane_state, struct intel_plane *plane) diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c index 6c9ee905f132b2e2e8412e3f497979895c4ffd56..aacbc6da84efa76e8ef1c9889adb646eb1a12855 100644 --- a/drivers/gpu/drm/i915/display/intel_audio.c +++ b/drivers/gpu/drm/i915/display/intel_audio.c @@ -393,7 +393,7 @@ hsw_dp_audio_config_update(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct i915_audio_component *acomp = dev_priv->audio.component; + struct i915_audio_component *acomp = dev_priv->display.audio.component; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum port port = encoder->port; const struct dp_aud_n_m *nm; @@ -441,7 +441,7 @@ hsw_hdmi_audio_config_update(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct i915_audio_component *acomp = dev_priv->audio.component; + struct i915_audio_component *acomp = dev_priv->display.audio.component; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; enum port port = encoder->port; int n, rate; @@ -496,7 +496,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder, enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder; u32 tmp; - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); /* Disable timestamps */ tmp = intel_de_read(dev_priv, HSW_AUD_CFG(cpu_transcoder)); @@ -514,7 +514,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder, tmp &= ~AUDIO_OUTPUT_ENABLE(cpu_transcoder); intel_de_write(dev_priv, HSW_AUD_PIN_ELD_CP_VLD, tmp); - mutex_unlock(&dev_priv->audio.mutex); + mutex_unlock(&dev_priv->display.audio.mutex); } static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder, @@ -532,7 +532,7 @@ static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder, h_total = crtc_state->hw.adjusted_mode.crtc_htotal; pixel_clk = crtc_state->hw.adjusted_mode.crtc_clock; vdsc_bpp = crtc_state->dsc.compressed_bpp; - cdclk = i915->cdclk.hw.cdclk; + cdclk = i915->display.cdclk.hw.cdclk; /* fec= 0.972261, using rounding multiplier of 1000000 */ fec_coeff = 972261; link_clk = crtc_state->port_clock; @@ -639,7 +639,7 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder, u32 tmp; int len, i; - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); /* Enable Audio WA for 4k DSC usecases */ if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP)) @@ -677,7 +677,7 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder, /* Enable timestamps */ hsw_audio_config_update(encoder, crtc_state); - mutex_unlock(&dev_priv->audio.mutex); + mutex_unlock(&dev_priv->display.audio.mutex); } static void ilk_audio_codec_disable(struct intel_encoder *encoder, @@ -814,7 +814,7 @@ void intel_audio_codec_enable(struct intel_encoder *encoder, const struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct i915_audio_component *acomp = dev_priv->audio.component; + struct i915_audio_component *acomp = dev_priv->display.audio.component; struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_connector *connector = conn_state->connector; const struct drm_display_mode *adjusted_mode = @@ -838,17 +838,17 @@ void intel_audio_codec_enable(struct intel_encoder *encoder, connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2; - if (dev_priv->audio.funcs) - dev_priv->audio.funcs->audio_codec_enable(encoder, - crtc_state, - conn_state); + if (dev_priv->display.funcs.audio) + dev_priv->display.funcs.audio->audio_codec_enable(encoder, + crtc_state, + conn_state); - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); encoder->audio_connector = connector; /* referred in audio callbacks */ - dev_priv->audio.encoder_map[pipe] = encoder; - mutex_unlock(&dev_priv->audio.mutex); + dev_priv->display.audio.encoder_map[pipe] = encoder; + mutex_unlock(&dev_priv->display.audio.mutex); if (acomp && acomp->base.audio_ops && acomp->base.audio_ops->pin_eld_notify) { @@ -878,7 +878,7 @@ void intel_audio_codec_disable(struct intel_encoder *encoder, const struct drm_connector_state *old_conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct i915_audio_component *acomp = dev_priv->audio.component; + struct i915_audio_component *acomp = dev_priv->display.audio.component; struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); struct drm_connector *connector = old_conn_state->connector; enum port port = encoder->port; @@ -891,15 +891,15 @@ void intel_audio_codec_disable(struct intel_encoder *encoder, connector->base.id, connector->name, encoder->base.base.id, encoder->base.name, pipe_name(pipe)); - if (dev_priv->audio.funcs) - dev_priv->audio.funcs->audio_codec_disable(encoder, - old_crtc_state, - old_conn_state); + if (dev_priv->display.funcs.audio) + dev_priv->display.funcs.audio->audio_codec_disable(encoder, + old_crtc_state, + old_conn_state); - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); encoder->audio_connector = NULL; - dev_priv->audio.encoder_map[pipe] = NULL; - mutex_unlock(&dev_priv->audio.mutex); + dev_priv->display.audio.encoder_map[pipe] = NULL; + mutex_unlock(&dev_priv->display.audio.mutex); if (acomp && acomp->base.audio_ops && acomp->base.audio_ops->pin_eld_notify) { @@ -935,13 +935,13 @@ static const struct intel_audio_funcs hsw_audio_funcs = { void intel_audio_hooks_init(struct drm_i915_private *dev_priv) { if (IS_G4X(dev_priv)) { - dev_priv->audio.funcs = &g4x_audio_funcs; + dev_priv->display.funcs.audio = &g4x_audio_funcs; } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - dev_priv->audio.funcs = &ilk_audio_funcs; + dev_priv->display.funcs.audio = &ilk_audio_funcs; } else if (IS_HASWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 8) { - dev_priv->audio.funcs = &hsw_audio_funcs; + dev_priv->display.funcs.audio = &hsw_audio_funcs; } else if (HAS_PCH_SPLIT(dev_priv)) { - dev_priv->audio.funcs = &ilk_audio_funcs; + dev_priv->display.funcs.audio = &ilk_audio_funcs; } } @@ -971,7 +971,7 @@ void intel_audio_cdclk_change_post(struct drm_i915_private *i915) struct aud_ts_cdclk_m_n aud_ts; if (DISPLAY_VER(i915) >= 13) { - get_aud_ts_cdclk_m_n(i915->cdclk.hw.ref, i915->cdclk.hw.cdclk, &aud_ts); + get_aud_ts_cdclk_m_n(i915->display.cdclk.hw.ref, i915->display.cdclk.hw.cdclk, &aud_ts); intel_de_write(i915, AUD_TS_CDCLK_N, aud_ts.n); intel_de_write(i915, AUD_TS_CDCLK_M, aud_ts.m | AUD_TS_CDCLK_M_EN); @@ -1046,13 +1046,13 @@ static unsigned long i915_audio_component_get_power(struct device *kdev) ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO_PLAYBACK); - if (dev_priv->audio.power_refcount++ == 0) { + if (dev_priv->display.audio.power_refcount++ == 0) { if (DISPLAY_VER(dev_priv) >= 9) { intel_de_write(dev_priv, AUD_FREQ_CNTRL, - dev_priv->audio.freq_cntrl); + dev_priv->display.audio.freq_cntrl); drm_dbg_kms(&dev_priv->drm, "restored AUD_FREQ_CNTRL to 0x%x\n", - dev_priv->audio.freq_cntrl); + dev_priv->display.audio.freq_cntrl); } /* Force CDCLK to 2*BCLK as long as we need audio powered. */ @@ -1073,7 +1073,7 @@ static void i915_audio_component_put_power(struct device *kdev, struct drm_i915_private *dev_priv = kdev_to_i915(kdev); /* Stop forcing CDCLK to 2*BCLK if no need for audio to be powered. */ - if (--dev_priv->audio.power_refcount == 0) + if (--dev_priv->display.audio.power_refcount == 0) if (IS_GEMINILAKE(dev_priv)) glk_force_audio_cdclk(dev_priv, false); @@ -1119,7 +1119,7 @@ static int i915_audio_component_get_cdclk_freq(struct device *kdev) if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_DDI(dev_priv))) return -ENODEV; - return dev_priv->cdclk.hw.cdclk; + return dev_priv->display.cdclk.hw.cdclk; } /* @@ -1140,10 +1140,10 @@ static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv, /* MST */ if (pipe >= 0) { if (drm_WARN_ON(&dev_priv->drm, - pipe >= ARRAY_SIZE(dev_priv->audio.encoder_map))) + pipe >= ARRAY_SIZE(dev_priv->display.audio.encoder_map))) return NULL; - encoder = dev_priv->audio.encoder_map[pipe]; + encoder = dev_priv->display.audio.encoder_map[pipe]; /* * when bootup, audio driver may not know it is * MST or not. So it will poll all the port & pipe @@ -1159,7 +1159,7 @@ static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv, return NULL; for_each_pipe(dev_priv, pipe) { - encoder = dev_priv->audio.encoder_map[pipe]; + encoder = dev_priv->display.audio.encoder_map[pipe]; if (encoder == NULL) continue; @@ -1177,7 +1177,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, int pipe, int rate) { struct drm_i915_private *dev_priv = kdev_to_i915(kdev); - struct i915_audio_component *acomp = dev_priv->audio.component; + struct i915_audio_component *acomp = dev_priv->display.audio.component; struct intel_encoder *encoder; struct intel_crtc *crtc; unsigned long cookie; @@ -1187,7 +1187,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, return 0; cookie = i915_audio_component_get_power(kdev); - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); /* 1. get the pipe */ encoder = get_saved_enc(dev_priv, port, pipe); @@ -1206,7 +1206,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, hsw_audio_config_update(encoder, crtc->config); unlock: - mutex_unlock(&dev_priv->audio.mutex); + mutex_unlock(&dev_priv->display.audio.mutex); i915_audio_component_put_power(kdev, cookie); return err; } @@ -1220,13 +1220,13 @@ static int i915_audio_component_get_eld(struct device *kdev, int port, const u8 *eld; int ret = -EINVAL; - mutex_lock(&dev_priv->audio.mutex); + mutex_lock(&dev_priv->display.audio.mutex); intel_encoder = get_saved_enc(dev_priv, port, pipe); if (!intel_encoder) { drm_dbg_kms(&dev_priv->drm, "Not valid for port %c\n", port_name(port)); - mutex_unlock(&dev_priv->audio.mutex); + mutex_unlock(&dev_priv->display.audio.mutex); return ret; } @@ -1238,7 +1238,7 @@ static int i915_audio_component_get_eld(struct device *kdev, int port, memcpy(buf, eld, min(max_bytes, ret)); } - mutex_unlock(&dev_priv->audio.mutex); + mutex_unlock(&dev_priv->display.audio.mutex); return ret; } @@ -1273,7 +1273,7 @@ static int i915_audio_component_bind(struct device *i915_kdev, BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS); for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++) acomp->aud_sample_rate[i] = 0; - dev_priv->audio.component = acomp; + dev_priv->display.audio.component = acomp; drm_modeset_unlock_all(&dev_priv->drm); return 0; @@ -1288,14 +1288,14 @@ static void i915_audio_component_unbind(struct device *i915_kdev, drm_modeset_lock_all(&dev_priv->drm); acomp->base.ops = NULL; acomp->base.dev = NULL; - dev_priv->audio.component = NULL; + dev_priv->display.audio.component = NULL; drm_modeset_unlock_all(&dev_priv->drm); device_link_remove(hda_kdev, i915_kdev); - if (dev_priv->audio.power_refcount) + if (dev_priv->display.audio.power_refcount) drm_err(&dev_priv->drm, "audio power refcount %d after unbind\n", - dev_priv->audio.power_refcount); + dev_priv->display.audio.power_refcount); } static const struct component_ops i915_audio_component_bind_ops = { @@ -1359,13 +1359,13 @@ static void i915_audio_component_init(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "use AUD_FREQ_CNTRL of 0x%x (init value 0x%x)\n", aud_freq, aud_freq_init); - dev_priv->audio.freq_cntrl = aud_freq; + dev_priv->display.audio.freq_cntrl = aud_freq; } /* init with current cdclk */ intel_audio_cdclk_change_post(dev_priv); - dev_priv->audio.component_registered = true; + dev_priv->display.audio.component_registered = true; } /** @@ -1377,11 +1377,11 @@ static void i915_audio_component_init(struct drm_i915_private *dev_priv) */ static void i915_audio_component_cleanup(struct drm_i915_private *dev_priv) { - if (!dev_priv->audio.component_registered) + if (!dev_priv->display.audio.component_registered) return; component_del(dev_priv->drm.dev, &i915_audio_component_bind_ops); - dev_priv->audio.component_registered = false; + dev_priv->display.audio.component_registered = false; } /** @@ -1403,7 +1403,7 @@ void intel_audio_init(struct drm_i915_private *dev_priv) */ void intel_audio_deinit(struct drm_i915_private *dev_priv) { - if ((dev_priv)->audio.lpe.platdev != NULL) + if (dev_priv->display.audio.lpe.platdev != NULL) intel_lpe_audio_teardown(dev_priv); else i915_audio_component_cleanup(dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c index 84639e9df5c9c9caf0bd327a9085a3f5998544d7..beba39a38c87cd6e8da09258bc491e0ca7f935cc 100644 --- a/drivers/gpu/drm/i915/display/intel_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_backlight.c @@ -11,6 +11,7 @@ #include #include "intel_backlight.h" +#include "intel_backlight_regs.h" #include "intel_connector.h" #include "intel_de.h" #include "intel_display_types.h" @@ -18,6 +19,8 @@ #include "intel_dsi_dcs_backlight.h" #include "intel_panel.h" #include "intel_pci_config.h" +#include "intel_pps.h" +#include "intel_quirks.h" /** * scale - scale values from one range to another @@ -88,7 +91,7 @@ u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 val) return val; if (dev_priv->params.invert_brightness > 0 || - dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) { + intel_has_quirk(dev_priv, QUIRK_INVERT_BRIGHTNESS)) { return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min; } @@ -128,7 +131,7 @@ u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val) panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0); if (dev_priv->params.invert_brightness > 0 || - (dev_priv->params.invert_brightness == 0 && dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS)) + (dev_priv->params.invert_brightness == 0 && intel_has_quirk(dev_priv, QUIRK_INVERT_BRIGHTNESS))) val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min); return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max, @@ -305,7 +308,7 @@ void intel_backlight_set_acpi(const struct drm_connector_state *conn_state, if (!panel->backlight.present || !conn_state->crtc) return; - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); @@ -321,7 +324,7 @@ void intel_backlight_set_acpi(const struct drm_connector_state *conn_state, if (panel->backlight.enabled) intel_panel_actually_set_backlight(conn_state, hw_level); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); } static void lpt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level) @@ -465,14 +468,14 @@ void intel_backlight_disable(const struct drm_connector_state *old_conn_state) return; } - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); if (panel->backlight.device) panel->backlight.device->props.power = FB_BLANK_POWERDOWN; panel->backlight.enabled = false; panel->backlight.funcs->disable(old_conn_state, 0); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); } static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state, @@ -815,11 +818,11 @@ void intel_backlight_enable(const struct intel_crtc_state *crtc_state, drm_dbg_kms(&dev_priv->drm, "pipe %c\n", pipe_name(pipe)); - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); __intel_backlight_enable(crtc_state, conn_state); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); } #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE) @@ -829,12 +832,12 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector) struct intel_panel *panel = &connector->panel; u32 val = 0; - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); if (panel->backlight.enabled) val = panel->backlight.funcs->get(connector, intel_connector_get_pipe(connector)); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); drm_dbg_kms(&dev_priv->drm, "get backlight PWM = %d\n", val); return val; @@ -862,7 +865,7 @@ static void intel_panel_set_backlight(const struct drm_connector_state *conn_sta if (!panel->backlight.present) return; - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0); @@ -872,7 +875,7 @@ static void intel_panel_set_backlight(const struct drm_connector_state *conn_sta if (panel->backlight.enabled) intel_panel_actually_set_backlight(conn_state, hw_level); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); } static int intel_backlight_device_update_status(struct backlight_device *bd) @@ -978,26 +981,24 @@ int intel_backlight_device_register(struct intel_connector *connector) if (!name) return -ENOMEM; - bd = backlight_device_register(name, connector->base.kdev, connector, - &intel_backlight_device_ops, &props); - - /* - * Using the same name independent of the drm device or connector - * prevents registration of multiple backlight devices in the - * driver. However, we need to use the default name for backward - * compatibility. Use unique names for subsequent backlight devices as a - * fallback when the default name already exists. - */ - if (IS_ERR(bd) && PTR_ERR(bd) == -EEXIST) { + bd = backlight_device_get_by_name(name); + if (bd) { + put_device(&bd->dev); + /* + * Using the same name independent of the drm device or connector + * prevents registration of multiple backlight devices in the + * driver. However, we need to use the default name for backward + * compatibility. Use unique names for subsequent backlight devices as a + * fallback when the default name already exists. + */ kfree(name); name = kasprintf(GFP_KERNEL, "card%d-%s-backlight", i915->drm.primary->index, connector->base.name); if (!name) return -ENOMEM; - - bd = backlight_device_register(name, connector->base.kdev, connector, - &intel_backlight_device_ops, &props); } + bd = backlight_device_register(name, connector->base.kdev, connector, + &intel_backlight_device_ops, &props); if (IS_ERR(bd)) { drm_err(&i915->drm, @@ -1120,7 +1121,7 @@ static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) if (IS_PINEVIEW(dev_priv)) clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); else - clock = KHz(dev_priv->cdclk.hw.cdclk); + clock = KHz(dev_priv->display.cdclk.hw.cdclk); return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 32); } @@ -1138,7 +1139,7 @@ static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz) if (IS_G4X(dev_priv)) clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq); else - clock = KHz(dev_priv->cdclk.hw.cdclk); + clock = KHz(dev_priv->display.cdclk.hw.cdclk); return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 128); } @@ -1598,11 +1599,11 @@ void intel_backlight_update(struct intel_atomic_state *state, if (!panel->backlight.present) return; - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); if (!panel->backlight.enabled) __intel_backlight_enable(crtc_state, conn_state); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); } int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe) @@ -1612,7 +1613,7 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe) int ret; if (!connector->panel.vbt.backlight.present) { - if (dev_priv->quirks & QUIRK_BACKLIGHT_PRESENT) { + if (intel_has_quirk(dev_priv, QUIRK_BACKLIGHT_PRESENT)) { drm_dbg_kms(&dev_priv->drm, "no backlight present per VBT, but present per quirk\n"); } else { @@ -1627,9 +1628,9 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe) return -ENODEV; /* set level and max in panel struct */ - mutex_lock(&dev_priv->backlight_lock); + mutex_lock(&dev_priv->display.backlight.lock); ret = panel->backlight.funcs->setup(connector, pipe); - mutex_unlock(&dev_priv->backlight_lock); + mutex_unlock(&dev_priv->display.backlight.lock); if (ret) { drm_dbg_kms(&dev_priv->drm, @@ -1780,9 +1781,13 @@ void intel_backlight_init_funcs(struct intel_panel *panel) panel->backlight.pwm_funcs = &i9xx_pwm_funcs; } - if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP && - intel_dp_aux_init_backlight_funcs(connector) == 0) - return; + if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) { + if (intel_dp_aux_init_backlight_funcs(connector) == 0) + return; + + if (!intel_has_quirk(dev_priv, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK)) + connector->panel.backlight.power = intel_pps_backlight_power; + } /* We're using a standard PWM backlight interface */ panel->backlight.funcs = &pwm_bl_funcs; diff --git a/drivers/gpu/drm/i915/display/intel_backlight_regs.h b/drivers/gpu/drm/i915/display/intel_backlight_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..50c1210f6d5de9508033494706f27025fc745956 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_backlight_regs.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_BACKLIGHT_REGS_H__ +#define __INTEL_BACKLIGHT_REGS_H__ + +#include "i915_reg_defs.h" + +#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250) +#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350) +#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ + _VLV_BLC_PWM_CTL2_B) + +#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254) +#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354) +#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ + _VLV_BLC_PWM_CTL_B) + +#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260) +#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360) +#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ + _VLV_BLC_HIST_CTL_B) + +/* Backlight control */ +#define BLC_PWM_CTL2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */ +#define BLM_PWM_ENABLE (1 << 31) +#define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ +#define BLM_PIPE_SELECT (1 << 29) +#define BLM_PIPE_SELECT_IVB (3 << 29) +#define BLM_PIPE_A (0 << 29) +#define BLM_PIPE_B (1 << 29) +#define BLM_PIPE_C (2 << 29) /* ivb + */ +#define BLM_TRANSCODER_A BLM_PIPE_A /* hsw */ +#define BLM_TRANSCODER_B BLM_PIPE_B +#define BLM_TRANSCODER_C BLM_PIPE_C +#define BLM_TRANSCODER_EDP (3 << 29) +#define BLM_PIPE(pipe) ((pipe) << 29) +#define BLM_POLARITY_I965 (1 << 28) /* gen4 only */ +#define BLM_PHASE_IN_INTERUPT_STATUS (1 << 26) +#define BLM_PHASE_IN_ENABLE (1 << 25) +#define BLM_PHASE_IN_INTERUPT_ENABL (1 << 24) +#define BLM_PHASE_IN_TIME_BASE_SHIFT (16) +#define BLM_PHASE_IN_TIME_BASE_MASK (0xff << 16) +#define BLM_PHASE_IN_COUNT_SHIFT (8) +#define BLM_PHASE_IN_COUNT_MASK (0xff << 8) +#define BLM_PHASE_IN_INCR_SHIFT (0) +#define BLM_PHASE_IN_INCR_MASK (0xff << 0) +#define BLC_PWM_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254) +/* + * This is the most significant 15 bits of the number of backlight cycles in a + * complete cycle of the modulated backlight control. + * + * The actual value is this field multiplied by two. + */ +#define BACKLIGHT_MODULATION_FREQ_SHIFT (17) +#define BACKLIGHT_MODULATION_FREQ_MASK (0x7fff << 17) +#define BLM_LEGACY_MODE (1 << 16) /* gen2 only */ +/* + * This is the number of cycles out of the backlight modulation cycle for which + * the backlight is on. + * + * This field must be no greater than the number of cycles in the complete + * backlight modulation cycle. + */ +#define BACKLIGHT_DUTY_CYCLE_SHIFT (0) +#define BACKLIGHT_DUTY_CYCLE_MASK (0xffff) +#define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) +#define BLM_POLARITY_PNV (1 << 0) /* pnv only */ + +#define BLC_HIST_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260) +#define BLM_HISTOGRAM_ENABLE (1 << 31) + +/* New registers for PCH-split platforms. Safe where new bits show up, the + * register layout machtes with gen4 BLC_PWM_CTL[12]. */ +#define BLC_PWM_CPU_CTL2 _MMIO(0x48250) +#define BLC_PWM_CPU_CTL _MMIO(0x48254) + +#define HSW_BLC_PWM2_CTL _MMIO(0x48350) + +/* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is + * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */ +#define BLC_PWM_PCH_CTL1 _MMIO(0xc8250) +#define BLM_PCH_PWM_ENABLE (1 << 31) +#define BLM_PCH_OVERRIDE_ENABLE (1 << 30) +#define BLM_PCH_POLARITY (1 << 29) +#define BLC_PWM_PCH_CTL2 _MMIO(0xc8254) + +/* BXT backlight register definition. */ +#define _BXT_BLC_PWM_CTL1 0xC8250 +#define BXT_BLC_PWM_ENABLE (1 << 31) +#define BXT_BLC_PWM_POLARITY (1 << 29) +#define _BXT_BLC_PWM_FREQ1 0xC8254 +#define _BXT_BLC_PWM_DUTY1 0xC8258 + +#define _BXT_BLC_PWM_CTL2 0xC8350 +#define _BXT_BLC_PWM_FREQ2 0xC8354 +#define _BXT_BLC_PWM_DUTY2 0xC8358 + +#define BXT_BLC_PWM_CTL(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2) +#define BXT_BLC_PWM_FREQ(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2) +#define BXT_BLC_PWM_DUTY(controller) _MMIO_PIPE(controller, \ + _BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2) + +/* Utility pin */ +#define UTIL_PIN_CTL _MMIO(0x48400) +#define UTIL_PIN_ENABLE (1 << 31) +#define UTIL_PIN_PIPE_MASK (3 << 29) +#define UTIL_PIN_PIPE(x) ((x) << 29) +#define UTIL_PIN_MODE_MASK (0xf << 24) +#define UTIL_PIN_MODE_DATA (0 << 24) +#define UTIL_PIN_MODE_PWM (1 << 24) +#define UTIL_PIN_MODE_VBLANK (4 << 24) +#define UTIL_PIN_MODE_VSYNC (5 << 24) +#define UTIL_PIN_MODE_EYE_LEVEL (8 << 24) +#define UTIL_PIN_OUTPUT_DATA (1 << 23) +#define UTIL_PIN_POLARITY (1 << 22) +#define UTIL_PIN_DIRECTION_INPUT (1 << 19) +#define UTIL_PIN_INPUT_DATA (1 << 16) + +#endif /* __INTEL_BACKLIGHT_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 51dde5bfd9565a2f780851d992c26ec48614ef39..28bdb936cd1fc12aaeec8d8723d8fc8a84dec1de 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -135,18 +135,6 @@ static u32 raw_block_offset(const void *bdb, enum bdb_block_id section_id) return block - bdb; } -/* size of the block excluding the header */ -static u32 raw_block_size(const void *bdb, enum bdb_block_id section_id) -{ - const void *block; - - block = find_raw_section(bdb, section_id); - if (!block) - return 0; - - return get_blocksize(block); -} - struct bdb_block_entry { struct list_head node; enum bdb_block_id section_id; @@ -159,7 +147,7 @@ find_section(struct drm_i915_private *i915, { struct bdb_block_entry *entry; - list_for_each_entry(entry, &i915->vbt.bdb_blocks, node) { + list_for_each_entry(entry, &i915->display.vbt.bdb_blocks, node) { if (entry->section_id == section_id) return entry->data + 3; } @@ -231,9 +219,14 @@ static bool validate_lfp_data_ptrs(const void *bdb, { int fp_timing_size, dvo_timing_size, panel_pnp_id_size, panel_name_size; int data_block_size, lfp_data_size; + const void *data_block; int i; - data_block_size = raw_block_size(bdb, BDB_LVDS_LFP_DATA); + data_block = find_raw_section(bdb, BDB_LVDS_LFP_DATA); + if (!data_block) + return false; + + data_block_size = get_blocksize(data_block); if (data_block_size == 0) return false; @@ -261,21 +254,6 @@ static bool validate_lfp_data_ptrs(const void *bdb, if (16 * lfp_data_size > data_block_size) return false; - /* - * Except for vlv/chv machines all real VBTs seem to have 6 - * unaccounted bytes in the fp_timing table. And it doesn't - * appear to be a really intentional hole as the fp_timing - * 0xffff terminator is always within those 6 missing bytes. - */ - if (fp_timing_size + dvo_timing_size + panel_pnp_id_size != lfp_data_size && - fp_timing_size + 6 + dvo_timing_size + panel_pnp_id_size != lfp_data_size) - return false; - - if (ptrs->ptr[0].fp_timing.offset + fp_timing_size > ptrs->ptr[0].dvo_timing.offset || - ptrs->ptr[0].dvo_timing.offset + dvo_timing_size != ptrs->ptr[0].panel_pnp_id.offset || - ptrs->ptr[0].panel_pnp_id.offset + panel_pnp_id_size != lfp_data_size) - return false; - /* make sure the table entries have uniform size */ for (i = 1; i < 16; i++) { if (ptrs->ptr[i].fp_timing.table_size != fp_timing_size || @@ -289,6 +267,23 @@ static bool validate_lfp_data_ptrs(const void *bdb, return false; } + /* + * Except for vlv/chv machines all real VBTs seem to have 6 + * unaccounted bytes in the fp_timing table. And it doesn't + * appear to be a really intentional hole as the fp_timing + * 0xffff terminator is always within those 6 missing bytes. + */ + if (fp_timing_size + 6 + dvo_timing_size + panel_pnp_id_size == lfp_data_size) + fp_timing_size += 6; + + if (fp_timing_size + dvo_timing_size + panel_pnp_id_size != lfp_data_size) + return false; + + if (ptrs->ptr[0].fp_timing.offset + fp_timing_size != ptrs->ptr[0].dvo_timing.offset || + ptrs->ptr[0].dvo_timing.offset + dvo_timing_size != ptrs->ptr[0].panel_pnp_id.offset || + ptrs->ptr[0].panel_pnp_id.offset + panel_pnp_id_size != lfp_data_size) + return false; + /* make sure the tables fit inside the data block */ for (i = 0; i < 16; i++) { if (ptrs->ptr[i].fp_timing.offset + fp_timing_size > data_block_size || @@ -300,6 +295,15 @@ static bool validate_lfp_data_ptrs(const void *bdb, if (ptrs->panel_name.offset + 16 * panel_name_size > data_block_size) return false; + /* make sure fp_timing terminators are present at expected locations */ + for (i = 0; i < 16; i++) { + const u16 *t = data_block + ptrs->ptr[i].fp_timing.offset + + fp_timing_size - 2; + + if (*t != 0xffff) + return false; + } + return true; } @@ -333,18 +337,6 @@ static bool fixup_lfp_data_ptrs(const void *bdb, void *ptrs_block) return validate_lfp_data_ptrs(bdb, ptrs); } -static const void *find_fp_timing_terminator(const u8 *data, int size) -{ - int i; - - for (i = 0; i < size - 1; i++) { - if (data[i] == 0xff && data[i+1] == 0xff) - return &data[i]; - } - - return NULL; -} - static int make_lfp_data_ptr(struct lvds_lfp_data_ptr_table *table, int table_size, int total_size) { @@ -368,11 +360,22 @@ static void next_lfp_data_ptr(struct lvds_lfp_data_ptr_table *next, static void *generate_lfp_data_ptrs(struct drm_i915_private *i915, const void *bdb) { - int i, size, table_size, block_size, offset; - const void *t0, *t1, *block; + int i, size, table_size, block_size, offset, fp_timing_size; struct bdb_lvds_lfp_data_ptrs *ptrs; + const void *block; void *ptrs_block; + /* + * The hardcoded fp_timing_size is only valid for + * modernish VBTs. All older VBTs definitely should + * include block 41 and thus we don't need to + * generate one. + */ + if (i915->display.vbt.version < 155) + return NULL; + + fp_timing_size = 38; + block = find_raw_section(bdb, BDB_LVDS_LFP_DATA); if (!block) return NULL; @@ -381,17 +384,8 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915, block_size = get_blocksize(block); - size = block_size; - t0 = find_fp_timing_terminator(block, size); - if (!t0) - return NULL; - - size -= t0 - block - 2; - t1 = find_fp_timing_terminator(t0 + 2, size); - if (!t1) - return NULL; - - size = t1 - t0; + size = fp_timing_size + sizeof(struct lvds_dvo_timing) + + sizeof(struct lvds_pnp_id); if (size * 16 > block_size) return NULL; @@ -409,7 +403,7 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915, table_size = sizeof(struct lvds_dvo_timing); size = make_lfp_data_ptr(&ptrs->ptr[0].dvo_timing, table_size, size); - table_size = t0 - block + 2; + table_size = fp_timing_size; size = make_lfp_data_ptr(&ptrs->ptr[0].fp_timing, table_size, size); if (ptrs->ptr[0].fp_timing.table_size) @@ -424,14 +418,14 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915, return NULL; } - size = t1 - t0; + size = fp_timing_size + sizeof(struct lvds_dvo_timing) + + sizeof(struct lvds_pnp_id); for (i = 1; i < 16; i++) { next_lfp_data_ptr(&ptrs->ptr[i].fp_timing, &ptrs->ptr[i-1].fp_timing, size); next_lfp_data_ptr(&ptrs->ptr[i].dvo_timing, &ptrs->ptr[i-1].dvo_timing, size); next_lfp_data_ptr(&ptrs->ptr[i].panel_pnp_id, &ptrs->ptr[i-1].panel_pnp_id, size); } - size = t1 - t0; table_size = sizeof(struct lvds_lfp_panel_name); if (16 * (size + table_size) <= block_size) { @@ -479,6 +473,13 @@ init_bdb_block(struct drm_i915_private *i915, block_size = get_blocksize(block); + /* + * Version number and new block size are considered + * part of the header for MIPI sequenece block v3+. + */ + if (section_id == BDB_MIPI_SEQUENCE && *(const u8 *)block >= 3) + block_size += 5; + entry = kzalloc(struct_size(entry, data, max(min_size, block_size) + 3), GFP_KERNEL); if (!entry) { @@ -501,7 +502,7 @@ init_bdb_block(struct drm_i915_private *i915, return; } - list_add_tail(&entry->node, &i915->vbt.bdb_blocks); + list_add_tail(&entry->node, &i915->display.vbt.bdb_blocks); } static void init_bdb_blocks(struct drm_i915_private *i915, @@ -604,6 +605,19 @@ get_lfp_data_tail(const struct bdb_lvds_lfp_data *data, return NULL; } +static void dump_pnp_id(struct drm_i915_private *i915, + const struct lvds_pnp_id *pnp_id, + const char *name) +{ + u16 mfg_name = be16_to_cpu((__force __be16)pnp_id->mfg_name); + char vend[4]; + + drm_dbg_kms(&i915->drm, "%s PNPID mfg: %s (0x%x), prod: %u, serial: %u, week: %d, year: %d\n", + name, drm_edid_decode_mfg_id(mfg_name, vend), + pnp_id->mfg_name, pnp_id->product_code, pnp_id->serial, + pnp_id->mfg_week, pnp_id->mfg_year + 1990); +} + static int opregion_get_panel_type(struct drm_i915_private *i915, const struct intel_bios_encoder_data *devdata, const struct edid *edid) @@ -655,6 +669,8 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915, edid_id_nodate.mfg_week = 0; edid_id_nodate.mfg_year = 0; + dump_pnp_id(i915, edid_id, "EDID"); + ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS); if (!ptrs) return -1; @@ -861,6 +877,7 @@ parse_lfp_data(struct drm_i915_private *i915, const struct bdb_lvds_lfp_data *data; const struct bdb_lvds_lfp_data_tail *tail; const struct bdb_lvds_lfp_data_ptrs *ptrs; + const struct lvds_pnp_id *pnp_id; int panel_type = panel->vbt.panel_type; ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS); @@ -874,11 +891,18 @@ parse_lfp_data(struct drm_i915_private *i915, if (!panel->vbt.lfp_lvds_vbt_mode) parse_lfp_panel_dtd(i915, panel, data, ptrs); + pnp_id = get_lvds_pnp_id(data, ptrs, panel_type); + dump_pnp_id(i915, pnp_id, "Panel"); + tail = get_lfp_data_tail(data, ptrs); if (!tail) return; - if (i915->vbt.version >= 188) { + drm_dbg_kms(&i915->drm, "Panel name: %.*s\n", + (int)sizeof(tail->panel_name[0].name), + tail->panel_name[panel_type].name); + + if (i915->display.vbt.version >= 188) { panel->vbt.seamless_drrs_min_refresh_rate = tail->seamless_drrs_min_refresh_rate[panel_type]; drm_dbg_kms(&i915->drm, @@ -904,7 +928,7 @@ parse_generic_dtd(struct drm_i915_private *i915, * first on VBT >= 229, but still fall back to trying the old LFP * block if that fails. */ - if (i915->vbt.version < 229) + if (i915->display.vbt.version < 229) return; generic_dtd = find_section(i915, BDB_GENERIC_DTD); @@ -1008,12 +1032,12 @@ parse_lfp_backlight(struct drm_i915_private *i915, } panel->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI; - if (i915->vbt.version >= 191) { + if (i915->display.vbt.version >= 191) { size_t exp_size; - if (i915->vbt.version >= 236) + if (i915->display.vbt.version >= 236) exp_size = sizeof(struct bdb_lfp_backlight_data); - else if (i915->vbt.version >= 234) + else if (i915->display.vbt.version >= 234) exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_234; else exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_191; @@ -1030,14 +1054,14 @@ parse_lfp_backlight(struct drm_i915_private *i915, panel->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz; panel->vbt.backlight.active_low_pwm = entry->active_low_pwm; - if (i915->vbt.version >= 234) { + if (i915->display.vbt.version >= 234) { u16 min_level; bool scale; level = backlight_data->brightness_level[panel_type].level; min_level = backlight_data->brightness_min_level[panel_type].level; - if (i915->vbt.version >= 236) + if (i915->display.vbt.version >= 236) scale = backlight_data->brightness_precision_bits[panel_type] == 16; else scale = level > 255; @@ -1134,37 +1158,37 @@ parse_general_features(struct drm_i915_private *i915) if (!general) return; - i915->vbt.int_tv_support = general->int_tv_support; + i915->display.vbt.int_tv_support = general->int_tv_support; /* int_crt_support can't be trusted on earlier platforms */ - if (i915->vbt.version >= 155 && + if (i915->display.vbt.version >= 155 && (HAS_DDI(i915) || IS_VALLEYVIEW(i915))) - i915->vbt.int_crt_support = general->int_crt_support; - i915->vbt.lvds_use_ssc = general->enable_ssc; - i915->vbt.lvds_ssc_freq = + i915->display.vbt.int_crt_support = general->int_crt_support; + i915->display.vbt.lvds_use_ssc = general->enable_ssc; + i915->display.vbt.lvds_ssc_freq = intel_bios_ssc_frequency(i915, general->ssc_freq); - i915->vbt.display_clock_mode = general->display_clock_mode; - i915->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted; - if (i915->vbt.version >= 181) { - i915->vbt.orientation = general->rotate_180 ? + i915->display.vbt.display_clock_mode = general->display_clock_mode; + i915->display.vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted; + if (i915->display.vbt.version >= 181) { + i915->display.vbt.orientation = general->rotate_180 ? DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP : DRM_MODE_PANEL_ORIENTATION_NORMAL; } else { - i915->vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; + i915->display.vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; } - if (i915->vbt.version >= 249 && general->afc_startup_config) { - i915->vbt.override_afc_startup = true; - i915->vbt.override_afc_startup_val = general->afc_startup_config == 0x1 ? 0x0 : 0x7; + if (i915->display.vbt.version >= 249 && general->afc_startup_config) { + i915->display.vbt.override_afc_startup = true; + i915->display.vbt.override_afc_startup_val = general->afc_startup_config == 0x1 ? 0x0 : 0x7; } drm_dbg_kms(&i915->drm, "BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n", - i915->vbt.int_tv_support, - i915->vbt.int_crt_support, - i915->vbt.lvds_use_ssc, - i915->vbt.lvds_ssc_freq, - i915->vbt.display_clock_mode, - i915->vbt.fdi_rx_polarity_inverted); + i915->display.vbt.int_tv_support, + i915->display.vbt.int_crt_support, + i915->display.vbt.lvds_use_ssc, + i915->display.vbt.lvds_ssc_freq, + i915->display.vbt.display_clock_mode, + i915->display.vbt.fdi_rx_polarity_inverted); } static const struct child_device_config * @@ -1190,7 +1214,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *i915) return; } - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; if (child->slave_addr != SLAVE_ADDR1 && @@ -1214,7 +1238,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *i915) child->slave_addr, (child->dvo_port == DEVICE_PORT_DVOB) ? "SDVOB" : "SDVOC"); - mapping = &i915->vbt.sdvo_mappings[child->dvo_port - 1]; + mapping = &i915->display.vbt.sdvo_mappings[child->dvo_port - 1]; if (!mapping->initialized) { mapping->dvo_port = child->dvo_port; mapping->slave_addr = child->slave_addr; @@ -1265,7 +1289,7 @@ parse_driver_features(struct drm_i915_private *i915) * interpretation, but real world VBTs seem to. */ if (driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS) - i915->vbt.int_lvds_support = 0; + i915->display.vbt.int_lvds_support = 0; } else { /* * FIXME it's not clear which BDB version has the LVDS config @@ -1278,10 +1302,10 @@ parse_driver_features(struct drm_i915_private *i915) * in the wild with the bits correctly populated. Version * 108 (on i85x) does not have the bits correctly populated. */ - if (i915->vbt.version >= 134 && + if (i915->display.vbt.version >= 134 && driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS && driver->lvds_config != BDB_DRIVER_FEATURE_INT_SDVO_LVDS) - i915->vbt.int_lvds_support = 0; + i915->display.vbt.int_lvds_support = 0; } } @@ -1295,7 +1319,7 @@ parse_panel_driver_features(struct drm_i915_private *i915, if (!driver) return; - if (i915->vbt.version < 228) { + if (i915->display.vbt.version < 228) { drm_dbg_kms(&i915->drm, "DRRS State Enabled:%d\n", driver->drrs_enabled); /* @@ -1328,7 +1352,7 @@ parse_power_conservation_features(struct drm_i915_private *i915, panel->vbt.vrr = true; /* matches Windows behaviour */ - if (i915->vbt.version < 228) + if (i915->display.vbt.version < 228) return; power = find_section(i915, BDB_LFP_POWER); @@ -1354,10 +1378,10 @@ parse_power_conservation_features(struct drm_i915_private *i915, panel->vbt.drrs_type = DRRS_TYPE_NONE; } - if (i915->vbt.version >= 232) + if (i915->display.vbt.version >= 232) panel->vbt.edp.hobl = panel_bool(power->hobl, panel_type); - if (i915->vbt.version >= 233) + if (i915->display.vbt.version >= 233) panel->vbt.vrr = panel_bool(power->vrr_feature_enabled, panel_type); } @@ -1393,7 +1417,7 @@ parse_edp(struct drm_i915_private *i915, panel->vbt.edp.pps = *edp_pps; - if (i915->vbt.version >= 224) { + if (i915->display.vbt.version >= 224) { panel->vbt.edp.rate = edp->edp_fast_link_training_rate[panel_type] * 20; } else { @@ -1472,7 +1496,7 @@ parse_edp(struct drm_i915_private *i915, break; } - if (i915->vbt.version >= 173) { + if (i915->display.vbt.version >= 173) { u8 vswing; /* Don't read from VBT if module parameter has valid value*/ @@ -1488,7 +1512,7 @@ parse_edp(struct drm_i915_private *i915, panel->vbt.edp.drrs_msa_timing_delay = panel_bits(edp->sdrrs_msa_timing_delay, panel_type, 2); - if (i915->vbt.version >= 244) + if (i915->display.vbt.version >= 244) panel->vbt.edp.max_link_rate = edp->edp_max_port_link_rate[panel_type] * 20; } @@ -1520,7 +1544,7 @@ parse_psr(struct drm_i915_private *i915, * New psr options 0=500us, 1=100us, 2=2500us, 3=0us * Old decimal value is wake up time in multiples of 100 us. */ - if (i915->vbt.version >= 205 && + if (i915->display.vbt.version >= 205 && (DISPLAY_VER(i915) >= 9 && !IS_BROXTON(i915))) { switch (psr_table->tp1_wakeup_time) { case 0: @@ -1566,7 +1590,7 @@ parse_psr(struct drm_i915_private *i915, panel->vbt.psr.tp2_tp3_wakeup_time_us = psr_table->tp2_tp3_wakeup_time * 100; } - if (i915->vbt.version >= 226) { + if (i915->display.vbt.version >= 226) { u32 wakeup_time = psr->psr2_tp2_tp3_wakeup_time; wakeup_time = panel_bits(wakeup_time, panel_type, 2); @@ -1596,7 +1620,9 @@ static void parse_dsi_backlight_ports(struct drm_i915_private *i915, struct intel_panel *panel, enum port port) { - if (!panel->vbt.dsi.config->dual_link || i915->vbt.version < 197) { + enum port port_bc = DISPLAY_VER(i915) >= 11 ? PORT_B : PORT_C; + + if (!panel->vbt.dsi.config->dual_link || i915->display.vbt.version < 197) { panel->vbt.dsi.bl_ports = BIT(port); if (panel->vbt.dsi.config->cabc_supported) panel->vbt.dsi.cabc_ports = BIT(port); @@ -1609,11 +1635,11 @@ static void parse_dsi_backlight_ports(struct drm_i915_private *i915, panel->vbt.dsi.bl_ports = BIT(PORT_A); break; case DL_DCS_PORT_C: - panel->vbt.dsi.bl_ports = BIT(PORT_C); + panel->vbt.dsi.bl_ports = BIT(port_bc); break; default: case DL_DCS_PORT_A_AND_C: - panel->vbt.dsi.bl_ports = BIT(PORT_A) | BIT(PORT_C); + panel->vbt.dsi.bl_ports = BIT(PORT_A) | BIT(port_bc); break; } @@ -1625,12 +1651,12 @@ static void parse_dsi_backlight_ports(struct drm_i915_private *i915, panel->vbt.dsi.cabc_ports = BIT(PORT_A); break; case DL_DCS_PORT_C: - panel->vbt.dsi.cabc_ports = BIT(PORT_C); + panel->vbt.dsi.cabc_ports = BIT(port_bc); break; default: case DL_DCS_PORT_A_AND_C: panel->vbt.dsi.cabc_ports = - BIT(PORT_A) | BIT(PORT_C); + BIT(PORT_A) | BIT(port_bc); break; } } @@ -2051,7 +2077,7 @@ parse_compression_parameters(struct drm_i915_private *i915) u16 block_size; int index; - if (i915->vbt.version < 198) + if (i915->display.vbt.version < 198) return; params = find_section(i915, BDB_COMPRESSION_PARAMETERS); @@ -2071,7 +2097,7 @@ parse_compression_parameters(struct drm_i915_private *i915) } } - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; if (!child->compression_enable) @@ -2205,7 +2231,7 @@ static enum port get_port_by_ddc_pin(struct drm_i915_private *i915, u8 ddc_pin) return PORT_NONE; for_each_port(port) { - devdata = i915->vbt.ports[port]; + devdata = i915->display.vbt.ports[port]; if (devdata && ddc_pin == devdata->child.ddc_pin) return port; @@ -2254,7 +2280,7 @@ static void sanitize_ddc_pin(struct intel_bios_encoder_data *devdata, * there are real machines (eg. Asrock B250M-HDV) where VBT has both * port A and port E with the same AUX ch and we must pick port E :( */ - child = &i915->vbt.ports[p]->child; + child = &i915->display.vbt.ports[p]->child; child->device_type &= ~DEVICE_TYPE_TMDS_DVI_SIGNALING; child->device_type |= DEVICE_TYPE_NOT_HDMI_OUTPUT; @@ -2271,7 +2297,7 @@ static enum port get_port_by_aux_ch(struct drm_i915_private *i915, u8 aux_ch) return PORT_NONE; for_each_port(port) { - devdata = i915->vbt.ports[port]; + devdata = i915->display.vbt.ports[port]; if (devdata && aux_ch == devdata->child.aux_channel) return port; @@ -2306,7 +2332,7 @@ static void sanitize_aux_ch(struct intel_bios_encoder_data *devdata, * there are real machines (eg. Asrock B250M-HDV) where VBT has both * port A and port E with the same AUX ch and we must pick port E :( */ - child = &i915->vbt.ports[p]->child; + child = &i915->display.vbt.ports[p]->child; child->device_type &= ~DEVICE_TYPE_DISPLAYPORT_OUTPUT; child->aux_channel = 0; @@ -2418,7 +2444,7 @@ static enum port dvo_port_to_port(struct drm_i915_private *i915, [PORT_TC4] = { DVO_PORT_HDMII, DVO_PORT_DPI, -1 }, }; - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) return __dvo_port_to_port(ARRAY_SIZE(xelpd_port_mapping), ARRAY_SIZE(xelpd_port_mapping[0]), xelpd_port_mapping, @@ -2480,15 +2506,23 @@ static int parse_bdb_216_dp_max_link_rate(const int vbt_max_link_rate) static int _intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata) { - if (!devdata || devdata->i915->vbt.version < 216) + if (!devdata || devdata->i915->display.vbt.version < 216) return 0; - if (devdata->i915->vbt.version >= 230) + if (devdata->i915->display.vbt.version >= 230) return parse_bdb_230_dp_max_link_rate(devdata->child.dp_max_link_rate); else return parse_bdb_216_dp_max_link_rate(devdata->child.dp_max_link_rate); } +static int _intel_bios_dp_max_lane_count(const struct intel_bios_encoder_data *devdata) +{ + if (!devdata || devdata->i915->display.vbt.version < 244) + return 0; + + return devdata->child.dp_max_lane_count + 1; +} + static void sanitize_device_type(struct intel_bios_encoder_data *devdata, enum port port) { @@ -2544,7 +2578,7 @@ intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata) static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata) { - if (!devdata || devdata->i915->vbt.version < 158) + if (!devdata || devdata->i915->display.vbt.version < 158) return -1; return devdata->child.hdmi_level_shifter_value; @@ -2552,7 +2586,7 @@ static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *de static int _intel_bios_max_tmds_clock(const struct intel_bios_encoder_data *devdata) { - if (!devdata || devdata->i915->vbt.version < 204) + if (!devdata || devdata->i915->display.vbt.version < 204) return 0; switch (devdata->child.hdmi_max_data_rate) { @@ -2661,7 +2695,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata) return; } - if (i915->vbt.ports[port]) { + if (i915->display.vbt.ports[port]) { drm_dbg_kms(&i915->drm, "More than one child device for port %c in VBT, using the first.\n", port_name(port)); @@ -2676,7 +2710,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata) if (intel_bios_encoder_supports_dp(devdata)) sanitize_aux_ch(devdata, port); - i915->vbt.ports[port] = devdata; + i915->display.vbt.ports[port] = devdata; } static bool has_ddi_port_info(struct drm_i915_private *i915) @@ -2692,12 +2726,12 @@ static void parse_ddi_ports(struct drm_i915_private *i915) if (!has_ddi_port_info(i915)) return; - list_for_each_entry(devdata, &i915->vbt.display_devices, node) + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) parse_ddi_port(devdata); for_each_port(port) { - if (i915->vbt.ports[port]) - print_ddi_port(i915->vbt.ports[port], port); + if (i915->display.vbt.ports[port]) + print_ddi_port(i915->display.vbt.ports[port], port); } } @@ -2730,33 +2764,33 @@ parse_general_definitions(struct drm_i915_private *i915) bus_pin = defs->crt_ddc_gmbus_pin; drm_dbg_kms(&i915->drm, "crt_ddc_bus_pin: %d\n", bus_pin); if (intel_gmbus_is_valid_pin(i915, bus_pin)) - i915->vbt.crt_ddc_pin = bus_pin; + i915->display.vbt.crt_ddc_pin = bus_pin; - if (i915->vbt.version < 106) { + if (i915->display.vbt.version < 106) { expected_size = 22; - } else if (i915->vbt.version < 111) { + } else if (i915->display.vbt.version < 111) { expected_size = 27; - } else if (i915->vbt.version < 195) { + } else if (i915->display.vbt.version < 195) { expected_size = LEGACY_CHILD_DEVICE_CONFIG_SIZE; - } else if (i915->vbt.version == 195) { + } else if (i915->display.vbt.version == 195) { expected_size = 37; - } else if (i915->vbt.version <= 215) { + } else if (i915->display.vbt.version <= 215) { expected_size = 38; - } else if (i915->vbt.version <= 237) { + } else if (i915->display.vbt.version <= 237) { expected_size = 39; } else { expected_size = sizeof(*child); BUILD_BUG_ON(sizeof(*child) < 39); drm_dbg(&i915->drm, "Expected child device config size for VBT version %u not known; assuming %u\n", - i915->vbt.version, expected_size); + i915->display.vbt.version, expected_size); } /* Flag an error for unexpected size, but continue anyway. */ if (defs->child_dev_size != expected_size) drm_err(&i915->drm, "Unexpected child device config size %u (expected %u for VBT version %u)\n", - defs->child_dev_size, expected_size, i915->vbt.version); + defs->child_dev_size, expected_size, i915->display.vbt.version); /* The legacy sized child device config is the minimum we need. */ if (defs->child_dev_size < LEGACY_CHILD_DEVICE_CONFIG_SIZE) { @@ -2792,10 +2826,10 @@ parse_general_definitions(struct drm_i915_private *i915) memcpy(&devdata->child, child, min_t(size_t, defs->child_dev_size, sizeof(*child))); - list_add_tail(&devdata->node, &i915->vbt.display_devices); + list_add_tail(&devdata->node, &i915->display.vbt.display_devices); } - if (list_empty(&i915->vbt.display_devices)) + if (list_empty(&i915->display.vbt.display_devices)) drm_dbg_kms(&i915->drm, "no child dev is parsed from VBT\n"); } @@ -2804,25 +2838,25 @@ parse_general_definitions(struct drm_i915_private *i915) static void init_vbt_defaults(struct drm_i915_private *i915) { - i915->vbt.crt_ddc_pin = GMBUS_PIN_VGADDC; + i915->display.vbt.crt_ddc_pin = GMBUS_PIN_VGADDC; /* general features */ - i915->vbt.int_tv_support = 1; - i915->vbt.int_crt_support = 1; + i915->display.vbt.int_tv_support = 1; + i915->display.vbt.int_crt_support = 1; /* driver features */ - i915->vbt.int_lvds_support = 1; + i915->display.vbt.int_lvds_support = 1; /* Default to using SSC */ - i915->vbt.lvds_use_ssc = 1; + i915->display.vbt.lvds_use_ssc = 1; /* * Core/SandyBridge/IvyBridge use alternative (120MHz) reference * clock for LVDS. */ - i915->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(i915, - !HAS_PCH_SPLIT(i915)); + i915->display.vbt.lvds_ssc_freq = intel_bios_ssc_frequency(i915, + !HAS_PCH_SPLIT(i915)); drm_dbg_kms(&i915->drm, "Set default to SSC at %d kHz\n", - i915->vbt.lvds_ssc_freq); + i915->display.vbt.lvds_ssc_freq); } /* Common defaults which may be overridden by VBT. */ @@ -2883,7 +2917,7 @@ init_vbt_missing_defaults(struct drm_i915_private *i915) if (port == PORT_A) child->device_type |= DEVICE_TYPE_INTERNAL_CONNECTOR; - list_add_tail(&devdata->node, &i915->vbt.display_devices); + list_add_tail(&devdata->node, &i915->display.vbt.display_devices); drm_dbg_kms(&i915->drm, "Generating default VBT child device with type 0x04%x on port %c\n", @@ -2891,7 +2925,7 @@ init_vbt_missing_defaults(struct drm_i915_private *i915) } /* Bypass some minimum baseline VBT version checks */ - i915->vbt.version = 155; + i915->display.vbt.version = 155; } static const struct bdb_header *get_bdb_header(const struct vbt_header *vbt) @@ -3078,12 +3112,12 @@ err_unmap_oprom: */ void intel_bios_init(struct drm_i915_private *i915) { - const struct vbt_header *vbt = i915->opregion.vbt; + const struct vbt_header *vbt = i915->display.opregion.vbt; struct vbt_header *oprom_vbt = NULL; const struct bdb_header *bdb; - INIT_LIST_HEAD(&i915->vbt.display_devices); - INIT_LIST_HEAD(&i915->vbt.bdb_blocks); + INIT_LIST_HEAD(&i915->display.vbt.display_devices); + INIT_LIST_HEAD(&i915->display.vbt.bdb_blocks); if (!HAS_DISPLAY(i915)) { drm_dbg_kms(&i915->drm, @@ -3111,11 +3145,11 @@ void intel_bios_init(struct drm_i915_private *i915) goto out; bdb = get_bdb_header(vbt); - i915->vbt.version = bdb->version; + i915->display.vbt.version = bdb->version; drm_dbg_kms(&i915->drm, "VBT signature \"%.*s\", BDB version %d\n", - (int)sizeof(vbt->signature), vbt->signature, i915->vbt.version); + (int)sizeof(vbt->signature), vbt->signature, i915->display.vbt.version); init_bdb_blocks(i915, bdb); @@ -3172,13 +3206,13 @@ void intel_bios_driver_remove(struct drm_i915_private *i915) struct intel_bios_encoder_data *devdata, *nd; struct bdb_block_entry *entry, *ne; - list_for_each_entry_safe(devdata, nd, &i915->vbt.display_devices, node) { + list_for_each_entry_safe(devdata, nd, &i915->display.vbt.display_devices, node) { list_del(&devdata->node); kfree(devdata->dsc); kfree(devdata); } - list_for_each_entry_safe(entry, ne, &i915->vbt.bdb_blocks, node) { + list_for_each_entry_safe(entry, ne, &i915->display.vbt.bdb_blocks, node) { list_del(&entry->node); kfree(entry); } @@ -3212,13 +3246,13 @@ bool intel_bios_is_tv_present(struct drm_i915_private *i915) const struct intel_bios_encoder_data *devdata; const struct child_device_config *child; - if (!i915->vbt.int_tv_support) + if (!i915->display.vbt.int_tv_support) return false; - if (list_empty(&i915->vbt.display_devices)) + if (list_empty(&i915->display.vbt.display_devices)) return true; - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; /* @@ -3255,10 +3289,10 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin) const struct intel_bios_encoder_data *devdata; const struct child_device_config *child; - if (list_empty(&i915->vbt.display_devices)) + if (list_empty(&i915->display.vbt.display_devices)) return true; - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; /* If the device type is not LFP, continue. @@ -3285,7 +3319,7 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin) * additional data. Trust that if the VBT was written into * the OpRegion then they have validated the LVDS's existence. */ - if (i915->opregion.vbt) + if (i915->display.opregion.vbt) return true; } @@ -3304,7 +3338,7 @@ bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port) if (WARN_ON(!has_ddi_port_info(i915))) return true; - return i915->vbt.ports[port]; + return i915->display.vbt.ports[port]; } /** @@ -3364,7 +3398,7 @@ bool intel_bios_is_dsi_present(struct drm_i915_private *i915, const struct child_device_config *child; u8 dvo_port; - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT)) @@ -3463,7 +3497,7 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder, const struct intel_bios_encoder_data *devdata; const struct child_device_config *child; - list_for_each_entry(devdata, &i915->vbt.display_devices, node) { + list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) { child = &devdata->child; if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT)) @@ -3494,7 +3528,7 @@ bool intel_bios_is_port_hpd_inverted(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port]; if (drm_WARN_ON_ONCE(&i915->drm, !IS_GEMINILAKE(i915) && !IS_BROXTON(i915))) @@ -3514,7 +3548,7 @@ bool intel_bios_is_lspcon_present(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port]; return HAS_LSPCON(i915) && devdata && devdata->child.lspcon; } @@ -3530,7 +3564,7 @@ bool intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port]; return devdata && devdata->child.lane_reversal; } @@ -3538,7 +3572,7 @@ intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915, enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, enum port port) { - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port]; enum aux_ch aux_ch; if (!devdata || !devdata->child.aux_channel) { @@ -3576,7 +3610,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, aux_ch = AUX_CH_C; break; case DP_AUX_D: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_D_XELPD; else if (IS_ALDERLAKE_S(i915)) aux_ch = AUX_CH_USBC3; @@ -3586,7 +3620,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, aux_ch = AUX_CH_D; break; case DP_AUX_E: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_E_XELPD; else if (IS_ALDERLAKE_S(i915)) aux_ch = AUX_CH_USBC4; @@ -3594,25 +3628,25 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, aux_ch = AUX_CH_E; break; case DP_AUX_F: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_USBC1; else aux_ch = AUX_CH_F; break; case DP_AUX_G: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_USBC2; else aux_ch = AUX_CH_G; break; case DP_AUX_H: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_USBC3; else aux_ch = AUX_CH_H; break; case DP_AUX_I: - if (DISPLAY_VER(i915) == 13) + if (DISPLAY_VER(i915) >= 13) aux_ch = AUX_CH_USBC4; else aux_ch = AUX_CH_I; @@ -3632,7 +3666,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915, int intel_bios_max_tmds_clock(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port]; return _intel_bios_max_tmds_clock(devdata); } @@ -3641,14 +3675,14 @@ int intel_bios_max_tmds_clock(struct intel_encoder *encoder) int intel_bios_hdmi_level_shift(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port]; return _intel_bios_hdmi_level_shift(devdata); } int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devdata) { - if (!devdata || devdata->i915->vbt.version < 196 || !devdata->child.iboost) + if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost) return 0; return translate_iboost(devdata->child.dp_iboost_level); @@ -3656,7 +3690,7 @@ int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devd int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *devdata) { - if (!devdata || devdata->i915->vbt.version < 196 || !devdata->child.iboost) + if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost) return 0; return translate_iboost(devdata->child.hdmi_iboost_level); @@ -3665,15 +3699,23 @@ int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *de int intel_bios_dp_max_link_rate(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port]; return _intel_bios_dp_max_link_rate(devdata); } +int intel_bios_dp_max_lane_count(struct intel_encoder *encoder) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port]; + + return _intel_bios_dp_max_lane_count(devdata); +} + int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder) { struct drm_i915_private *i915 = to_i915(encoder->base.dev); - const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port]; + const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port]; if (!devdata || !devdata->child.ddc_pin) return 0; @@ -3683,16 +3725,16 @@ int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder) bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata) { - return devdata->i915->vbt.version >= 195 && devdata->child.dp_usb_type_c; + return devdata->i915->display.vbt.version >= 195 && devdata->child.dp_usb_type_c; } bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devdata) { - return devdata->i915->vbt.version >= 209 && devdata->child.tbt; + return devdata->i915->display.vbt.version >= 209 && devdata->child.tbt; } const struct intel_bios_encoder_data * intel_bios_encoder_data_lookup(struct drm_i915_private *i915, enum port port) { - return i915->vbt.ports[port]; + return i915->display.vbt.ports[port]; } diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h index e47582b0de0a17cdcad296bf08683f040605aab2..e375405a7828413360e4f9b6dded44b1dd92b841 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.h +++ b/drivers/gpu/drm/i915/display/intel_bios.h @@ -258,6 +258,7 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder, int intel_bios_max_tmds_clock(struct intel_encoder *encoder); int intel_bios_hdmi_level_shift(struct intel_encoder *encoder); int intel_bios_dp_max_link_rate(struct intel_encoder *encoder); +int intel_bios_dp_max_lane_count(struct intel_encoder *encoder); int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder); bool intel_bios_port_supports_typec_usb(struct drm_i915_private *i915, enum port port); bool intel_bios_port_supports_tbt(struct drm_i915_private *i915, enum port port); diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index 79269d2c476b2949048eb6e8f3233ded3bff60de..4ace026b29bd79ebb18894605889209bcff217be 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -5,15 +5,17 @@ #include +#include "i915_drv.h" #include "i915_reg.h" #include "i915_utils.h" #include "intel_atomic.h" #include "intel_bw.h" #include "intel_cdclk.h" +#include "intel_display_core.h" #include "intel_display_types.h" +#include "skl_watermark.h" #include "intel_mchbar_regs.h" #include "intel_pcode.h" -#include "intel_pm.h" /* Parameters for Qclk Geyserville (QGV) */ struct intel_qgv_point { @@ -137,6 +139,42 @@ int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv, return 0; } +static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv, + struct intel_qgv_point *sp, int point) +{ + u32 val, val2; + u16 dclk; + + val = intel_uncore_read(&dev_priv->uncore, + MTL_MEM_SS_INFO_QGV_POINT_LOW(point)); + val2 = intel_uncore_read(&dev_priv->uncore, + MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)); + dclk = REG_FIELD_GET(MTL_DCLK_MASK, val); + sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000); + sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val); + sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val); + + sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2); + sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2); + + sp->t_rc = sp->t_rp + sp->t_ras; + + return 0; +} + +static int +intel_read_qgv_point_info(struct drm_i915_private *dev_priv, + struct intel_qgv_point *sp, + int point) +{ + if (DISPLAY_VER(dev_priv) >= 14) + return mtl_read_qgv_point_info(dev_priv, sp, point); + else if (IS_DG1(dev_priv)) + return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point); + else + return icl_pcode_read_qgv_point_info(dev_priv, sp, point); +} + static int icl_get_qgv_points(struct drm_i915_private *dev_priv, struct intel_qgv_info *qi, bool is_y_tile) @@ -147,7 +185,32 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, qi->num_points = dram_info->num_qgv_points; qi->num_psf_points = dram_info->num_psf_gv_points; - if (DISPLAY_VER(dev_priv) >= 12) + if (DISPLAY_VER(dev_priv) >= 14) { + switch (dram_info->type) { + case INTEL_DRAM_DDR4: + qi->t_bl = 4; + qi->max_numchannels = 2; + qi->channel_width = 64; + qi->deinterleave = 2; + break; + case INTEL_DRAM_DDR5: + qi->t_bl = 8; + qi->max_numchannels = 4; + qi->channel_width = 32; + qi->deinterleave = 2; + break; + case INTEL_DRAM_LPDDR4: + case INTEL_DRAM_LPDDR5: + qi->t_bl = 16; + qi->max_numchannels = 8; + qi->channel_width = 16; + qi->deinterleave = 4; + break; + default: + MISSING_CASE(dram_info->type); + return -EINVAL; + } + } else if (DISPLAY_VER(dev_priv) >= 12) { switch (dram_info->type) { case INTEL_DRAM_DDR4: qi->t_bl = is_y_tile ? 8 : 4; @@ -181,7 +244,7 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, qi->max_numchannels = 1; break; } - else if (DISPLAY_VER(dev_priv) == 11) { + } else if (DISPLAY_VER(dev_priv) == 11) { qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8; qi->max_numchannels = 1; } @@ -193,11 +256,7 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv, for (i = 0; i < qi->num_points; i++) { struct intel_qgv_point *sp = &qi->points[i]; - if (IS_DG1(dev_priv)) - ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i); - else - ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i); - + ret = intel_read_qgv_point_info(dev_priv, sp, i); if (ret) return ret; @@ -284,6 +343,13 @@ static const struct intel_sa_info adlp_sa_info = { .derating = 20, }; +static const struct intel_sa_info mtl_sa_info = { + .deburst = 32, + .deprogbwlimit = 38, /* GB/s */ + .displayrtids = 256, + .derating = 20, +}; + static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa) { struct intel_qgv_info qi = {}; @@ -292,7 +358,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel int ipqdepth, ipqdepthpch = 16; int dclk_max; int maxdebw; - int num_groups = ARRAY_SIZE(dev_priv->max_bw); + int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); int i, ret; ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); @@ -308,7 +374,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2); for (i = 0; i < num_groups; i++) { - struct intel_bw_info *bi = &dev_priv->max_bw[i]; + struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; int clpchgroup; int j; @@ -346,9 +412,9 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel * as it will fail and pointless anyway. */ if (qi.num_points == 1) - dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; + dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; else - dev_priv->sagv_status = I915_SAGV_ENABLED; + dev_priv->display.sagv.status = I915_SAGV_ENABLED; return 0; } @@ -363,7 +429,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel int dclk_max; int maxdebw, peakbw; int clperchgroup; - int num_groups = ARRAY_SIZE(dev_priv->max_bw); + int num_groups = ARRAY_SIZE(dev_priv->display.bw.max); int i, ret; ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile); @@ -399,20 +465,22 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave; for (i = 0; i < num_groups; i++) { - struct intel_bw_info *bi = &dev_priv->max_bw[i]; + struct intel_bw_info *bi = &dev_priv->display.bw.max[i]; struct intel_bw_info *bi_next; int clpchgroup; int j; - if (i < num_groups - 1) - bi_next = &dev_priv->max_bw[i + 1]; - clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i; - if (i < num_groups - 1 && clpchgroup < clperchgroup) - bi_next->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1; - else - bi_next->num_planes = 0; + if (i < num_groups - 1) { + bi_next = &dev_priv->display.bw.max[i + 1]; + + if (clpchgroup < clperchgroup) + bi_next->num_planes = (ipqdepth - clpchgroup) / + clpchgroup + 1; + else + bi_next->num_planes = 0; + } bi->num_qgv_points = qi.num_points; bi->num_psf_gv_points = qi.num_psf_points; @@ -456,9 +524,9 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel * as it will fail and pointless anyway. */ if (qi.num_points == 1) - dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; + dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED; else - dev_priv->sagv_status = I915_SAGV_ENABLED; + dev_priv->display.sagv.status = I915_SAGV_ENABLED; return 0; } @@ -466,7 +534,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel static void dg2_get_bw_info(struct drm_i915_private *i915) { unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000; - int num_groups = ARRAY_SIZE(i915->max_bw); + int num_groups = ARRAY_SIZE(i915->display.bw.max); int i; /* @@ -477,7 +545,7 @@ static void dg2_get_bw_info(struct drm_i915_private *i915) * whereas DG2-G11 platforms have 38 GB/s. */ for (i = 0; i < num_groups; i++) { - struct intel_bw_info *bi = &i915->max_bw[i]; + struct intel_bw_info *bi = &i915->display.bw.max[i]; bi->num_planes = 1; /* Need only one dummy QGV point per group */ @@ -485,7 +553,7 @@ static void dg2_get_bw_info(struct drm_i915_private *i915) bi->deratedbw[0] = deratedbw; } - i915->sagv_status = I915_SAGV_NOT_CONTROLLED; + i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; } static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, @@ -498,9 +566,9 @@ static unsigned int icl_max_bw(struct drm_i915_private *dev_priv, */ num_planes = max(1, num_planes); - for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) { + for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) { const struct intel_bw_info *bi = - &dev_priv->max_bw[i]; + &dev_priv->display.bw.max[i]; /* * Pcode will not expose all QGV points when @@ -526,9 +594,9 @@ static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv, */ num_planes = max(1, num_planes); - for (i = ARRAY_SIZE(dev_priv->max_bw) - 1; i >= 0; i--) { + for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) { const struct intel_bw_info *bi = - &dev_priv->max_bw[i]; + &dev_priv->display.bw.max[i]; /* * Pcode will not expose all QGV points when @@ -541,14 +609,14 @@ static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv, return bi->deratedbw[qgv_point]; } - return dev_priv->max_bw[0].deratedbw[qgv_point]; + return dev_priv->display.bw.max[0].deratedbw[qgv_point]; } static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv, int psf_gv_point) { const struct intel_bw_info *bi = - &dev_priv->max_bw[0]; + &dev_priv->display.bw.max[0]; return bi->psf_bw[psf_gv_point]; } @@ -558,7 +626,9 @@ void intel_bw_init_hw(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - if (IS_DG2(dev_priv)) + if (DISPLAY_VER(dev_priv) >= 14) + tgl_get_bw_info(dev_priv, &mtl_sa_info); + else if (IS_DG2(dev_priv)) dg2_get_bw_info(dev_priv); else if (IS_ALDERLAKE_P(dev_priv)) tgl_get_bw_info(dev_priv, &adlp_sa_info); @@ -667,7 +737,7 @@ intel_atomic_get_old_bw_state(struct intel_atomic_state *state) struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj); return to_intel_bw_state(bw_state); } @@ -678,7 +748,7 @@ intel_atomic_get_new_bw_state(struct intel_atomic_state *state) struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj); return to_intel_bw_state(bw_state); } @@ -689,7 +759,7 @@ intel_atomic_get_bw_state(struct intel_atomic_state *state) struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_global_state *bw_state; - bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj); + bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj); if (IS_ERR(bw_state)) return ERR_CAST(bw_state); @@ -896,8 +966,8 @@ int intel_bw_calc_min_cdclk(struct intel_atomic_state *state, static u16 icl_qgv_points_mask(struct drm_i915_private *i915) { - unsigned int num_psf_gv_points = i915->max_bw[0].num_psf_gv_points; - unsigned int num_qgv_points = i915->max_bw[0].num_qgv_points; + unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points; + unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points; u16 qgv_points = 0, psf_points = 0; /* @@ -970,8 +1040,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) int i, ret; u16 qgv_points = 0, psf_points = 0; unsigned int max_bw_point = 0, max_bw = 0; - unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points; - unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points; + unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points; + unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points; bool changed = false; /* FIXME earlier gens need some checks too */ @@ -1126,7 +1196,7 @@ int intel_bw_init(struct drm_i915_private *dev_priv) if (!state) return -ENOMEM; - intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj, + intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj, &state->base, &intel_bw_funcs); return 0; diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 6e80162632ddf6ad2069e0a2ced1abd4afdd6b3c..ed05070b7307239c754675750850ffb70951b2af 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -79,26 +79,26 @@ struct intel_cdclk_funcs { void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_config *cdclk_config) { - dev_priv->cdclk_funcs->get_cdclk(dev_priv, cdclk_config); + dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config); } static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv, const struct intel_cdclk_config *cdclk_config, enum pipe pipe) { - dev_priv->cdclk_funcs->set_cdclk(dev_priv, cdclk_config, pipe); + dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe); } static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv, struct intel_cdclk_state *cdclk_config) { - return dev_priv->cdclk_funcs->modeset_calc_cdclk(cdclk_config); + return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config); } static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk) { - return dev_priv->cdclk_funcs->calc_voltage_level(cdclk); + return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk); } static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv, @@ -548,7 +548,7 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) else default_credits = PFI_CREDIT(8); - if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) { + if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) { /* CHV suggested value is 31 or 63 */ if (IS_CHERRYVIEW(dev_priv)) credits = PFI_CREDIT_63; @@ -1026,7 +1026,7 @@ static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco) if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5)) drm_err(&dev_priv->drm, "DPLL0 not locked\n"); - dev_priv->cdclk.hw.vco = vco; + dev_priv->display.cdclk.hw.vco = vco; /* We'll want to keep using the current vco from now on. */ skl_set_preferred_cdclk_vco(dev_priv, vco); @@ -1040,7 +1040,7 @@ static void skl_dpll0_disable(struct drm_i915_private *dev_priv) if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1)) drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n"); - dev_priv->cdclk.hw.vco = 0; + dev_priv->display.cdclk.hw.vco = 0; } static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv, @@ -1049,7 +1049,7 @@ static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv, switch (cdclk) { default: drm_WARN_ON(&dev_priv->drm, - cdclk != dev_priv->cdclk.hw.bypass); + cdclk != dev_priv->display.cdclk.hw.bypass); drm_WARN_ON(&dev_priv->drm, vco != 0); fallthrough; case 308571: @@ -1098,13 +1098,13 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco); - if (dev_priv->cdclk.hw.vco != 0 && - dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != 0 && + dev_priv->display.cdclk.hw.vco != vco) skl_dpll0_disable(dev_priv); cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL); - if (dev_priv->cdclk.hw.vco != vco) { + if (dev_priv->display.cdclk.hw.vco != vco) { /* Wa Display #1183: skl,kbl,cfl */ cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK); cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk); @@ -1116,7 +1116,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl); intel_de_posting_read(dev_priv, CDCLK_CTL); - if (dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != vco) skl_dpll0_enable(dev_priv, vco); /* Wa Display #1183: skl,kbl,cfl */ @@ -1151,11 +1151,11 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) goto sanitize; intel_update_cdclk(dev_priv); - intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK"); /* Is PLL enabled and locked ? */ - if (dev_priv->cdclk.hw.vco == 0 || - dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) + if (dev_priv->display.cdclk.hw.vco == 0 || + dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass) goto sanitize; /* DPLL okay; verify the cdclock @@ -1166,7 +1166,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) */ cdctl = intel_de_read(dev_priv, CDCLK_CTL); expected = (cdctl & CDCLK_FREQ_SEL_MASK) | - skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk); + skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk); if (cdctl == expected) /* All well; nothing to sanitize */ return; @@ -1175,9 +1175,9 @@ sanitize: drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n"); /* force cdclk programming */ - dev_priv->cdclk.hw.cdclk = 0; + dev_priv->display.cdclk.hw.cdclk = 0; /* force full PLL disable + enable */ - dev_priv->cdclk.hw.vco = -1; + dev_priv->display.cdclk.hw.vco = -1; } static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv) @@ -1186,19 +1186,19 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv) skl_sanitize_cdclk(dev_priv); - if (dev_priv->cdclk.hw.cdclk != 0 && - dev_priv->cdclk.hw.vco != 0) { + if (dev_priv->display.cdclk.hw.cdclk != 0 && + dev_priv->display.cdclk.hw.vco != 0) { /* * Use the current vco as our initial * guess as to what the preferred vco is. */ if (dev_priv->skl_preferred_vco_freq == 0) skl_set_preferred_cdclk_vco(dev_priv, - dev_priv->cdclk.hw.vco); + dev_priv->display.cdclk.hw.vco); return; } - cdclk_config = dev_priv->cdclk.hw; + cdclk_config = dev_priv->display.cdclk.hw; cdclk_config.vco = dev_priv->skl_preferred_vco_freq; if (cdclk_config.vco == 0) @@ -1211,7 +1211,7 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv) static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv) { - struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw; + struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw; cdclk_config.cdclk = cdclk_config.bypass; cdclk_config.vco = 0; @@ -1352,35 +1352,35 @@ static const struct intel_cdclk_vals dg2_cdclk_table[] = { static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk) { - const struct intel_cdclk_vals *table = dev_priv->cdclk.table; + const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table; int i; for (i = 0; table[i].refclk; i++) - if (table[i].refclk == dev_priv->cdclk.hw.ref && + if (table[i].refclk == dev_priv->display.cdclk.hw.ref && table[i].cdclk >= min_cdclk) return table[i].cdclk; drm_WARN(&dev_priv->drm, 1, "Cannot satisfy minimum cdclk %d with refclk %u\n", - min_cdclk, dev_priv->cdclk.hw.ref); + min_cdclk, dev_priv->display.cdclk.hw.ref); return 0; } static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk) { - const struct intel_cdclk_vals *table = dev_priv->cdclk.table; + const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table; int i; - if (cdclk == dev_priv->cdclk.hw.bypass) + if (cdclk == dev_priv->display.cdclk.hw.bypass) return 0; for (i = 0; table[i].refclk; i++) - if (table[i].refclk == dev_priv->cdclk.hw.ref && + if (table[i].refclk == dev_priv->display.cdclk.hw.ref && table[i].cdclk == cdclk) - return dev_priv->cdclk.hw.ref * table[i].ratio; + return dev_priv->display.cdclk.hw.ref * table[i].ratio; drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n", - cdclk, dev_priv->cdclk.hw.ref); + cdclk, dev_priv->display.cdclk.hw.ref); return 0; } @@ -1554,12 +1554,12 @@ static void bxt_de_pll_disable(struct drm_i915_private *dev_priv) BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1)) drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n"); - dev_priv->cdclk.hw.vco = 0; + dev_priv->display.cdclk.hw.vco = 0; } static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco) { - int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); + int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref); intel_de_rmw(dev_priv, BXT_DE_PLL_CTL, BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio)); @@ -1571,7 +1571,7 @@ static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco) BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1)) drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n"); - dev_priv->cdclk.hw.vco = vco; + dev_priv->display.cdclk.hw.vco = vco; } static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv) @@ -1583,12 +1583,12 @@ static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv) if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1)) drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n"); - dev_priv->cdclk.hw.vco = 0; + dev_priv->display.cdclk.hw.vco = 0; } static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) { - int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); + int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref); u32 val; val = ICL_CDCLK_PLL_RATIO(ratio); @@ -1601,12 +1601,12 @@ static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco) if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1)) drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n"); - dev_priv->cdclk.hw.vco = vco; + dev_priv->display.cdclk.hw.vco = vco; } static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco) { - int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref); + int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref); u32 val; /* Write PLL ratio without disabling */ @@ -1625,7 +1625,7 @@ static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco) val &= ~BXT_DE_PLL_FREQ_REQ; intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val); - dev_priv->cdclk.hw.vco = vco; + dev_priv->display.cdclk.hw.vco = vco; } static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) @@ -1655,7 +1655,7 @@ static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv, switch (DIV_ROUND_CLOSEST(vco, cdclk)) { default: drm_WARN_ON(&dev_priv->drm, - cdclk != dev_priv->cdclk.hw.bypass); + cdclk != dev_priv->display.cdclk.hw.bypass); drm_WARN_ON(&dev_priv->drm, vco != 0); fallthrough; case 2: @@ -1672,19 +1672,19 @@ static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv, static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv, int cdclk) { - const struct intel_cdclk_vals *table = dev_priv->cdclk.table; + const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table; int i; - if (cdclk == dev_priv->cdclk.hw.bypass) + if (cdclk == dev_priv->display.cdclk.hw.bypass) return 0; for (i = 0; table[i].refclk; i++) - if (table[i].refclk == dev_priv->cdclk.hw.ref && + if (table[i].refclk == dev_priv->display.cdclk.hw.ref && table[i].cdclk == cdclk) return table[i].waveform; drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n", - cdclk, dev_priv->cdclk.hw.ref); + cdclk, dev_priv->display.cdclk.hw.ref); return 0xffff; } @@ -1721,22 +1721,22 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, return; } - if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) { - if (dev_priv->cdclk.hw.vco != vco) + if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0) { + if (dev_priv->display.cdclk.hw.vco != vco) adlp_cdclk_pll_crawl(dev_priv, vco); } else if (DISPLAY_VER(dev_priv) >= 11) { - if (dev_priv->cdclk.hw.vco != 0 && - dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != 0 && + dev_priv->display.cdclk.hw.vco != vco) icl_cdclk_pll_disable(dev_priv); - if (dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != vco) icl_cdclk_pll_enable(dev_priv, vco); } else { - if (dev_priv->cdclk.hw.vco != 0 && - dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != 0 && + dev_priv->display.cdclk.hw.vco != vco) bxt_de_pll_disable(dev_priv); - if (dev_priv->cdclk.hw.vco != vco) + if (dev_priv->display.cdclk.hw.vco != vco) bxt_de_pll_enable(dev_priv, vco); } @@ -1803,7 +1803,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, * Can't read out the voltage level :( * Let's just assume everything is as expected. */ - dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level; + dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level; } static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) @@ -1812,10 +1812,10 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) int cdclk, clock, vco; intel_update_cdclk(dev_priv); - intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK"); - if (dev_priv->cdclk.hw.vco == 0 || - dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass) + if (dev_priv->display.cdclk.hw.vco == 0 || + dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass) goto sanitize; /* DPLL okay; verify the cdclock @@ -1833,32 +1833,32 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE); /* Make sure this is a legal cdclk value for the platform */ - cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk); - if (cdclk != dev_priv->cdclk.hw.cdclk) + cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk); + if (cdclk != dev_priv->display.cdclk.hw.cdclk) goto sanitize; /* Make sure the VCO is correct for the cdclk */ vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk); - if (vco != dev_priv->cdclk.hw.vco) + if (vco != dev_priv->display.cdclk.hw.vco) goto sanitize; expected = skl_cdclk_decimal(cdclk); /* Figure out what CD2X divider we should be using for this cdclk */ if (has_cdclk_squasher(dev_priv)) - clock = dev_priv->cdclk.hw.vco / 2; + clock = dev_priv->display.cdclk.hw.vco / 2; else - clock = dev_priv->cdclk.hw.cdclk; + clock = dev_priv->display.cdclk.hw.cdclk; expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock, - dev_priv->cdclk.hw.vco); + dev_priv->display.cdclk.hw.vco); /* * Disable SSA Precharge when CD clock frequency < 500 MHz, * enable otherwise. */ if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) && - dev_priv->cdclk.hw.cdclk >= 500000) + dev_priv->display.cdclk.hw.cdclk >= 500000) expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE; if (cdctl == expected) @@ -1869,10 +1869,10 @@ sanitize: drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n"); /* force cdclk programming */ - dev_priv->cdclk.hw.cdclk = 0; + dev_priv->display.cdclk.hw.cdclk = 0; /* force full PLL disable + enable */ - dev_priv->cdclk.hw.vco = -1; + dev_priv->display.cdclk.hw.vco = -1; } static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv) @@ -1881,11 +1881,11 @@ static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv) bxt_sanitize_cdclk(dev_priv); - if (dev_priv->cdclk.hw.cdclk != 0 && - dev_priv->cdclk.hw.vco != 0) + if (dev_priv->display.cdclk.hw.cdclk != 0 && + dev_priv->display.cdclk.hw.vco != 0) return; - cdclk_config = dev_priv->cdclk.hw; + cdclk_config = dev_priv->display.cdclk.hw; /* * FIXME: @@ -1902,7 +1902,7 @@ static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv) static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv) { - struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw; + struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw; cdclk_config.cdclk = cdclk_config.bypass; cdclk_config.vco = 0; @@ -1916,7 +1916,7 @@ static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv) * intel_cdclk_init_hw - Initialize CDCLK hardware * @i915: i915 device * - * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and + * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and * sanitizing the state of the hardware if needed. This is generally done only * during the display core initialization sequence, after which the DMC will * take care of turning CDCLK off/on as needed. @@ -2077,10 +2077,10 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, { struct intel_encoder *encoder; - if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config)) + if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config)) return; - if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk)) return; intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to"); @@ -2098,12 +2098,12 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, * functions use cdclk. Not all platforms/ports do, * but we'll lock them all for simplicity. */ - mutex_lock(&dev_priv->gmbus_mutex); + mutex_lock(&dev_priv->display.gmbus.mutex); for_each_intel_dp(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); mutex_lock_nest_lock(&intel_dp->aux.hw_mutex, - &dev_priv->gmbus_mutex); + &dev_priv->display.gmbus.mutex); } intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe); @@ -2113,7 +2113,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, mutex_unlock(&intel_dp->aux.hw_mutex); } - mutex_unlock(&dev_priv->gmbus_mutex); + mutex_unlock(&dev_priv->display.gmbus.mutex); for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); @@ -2124,9 +2124,9 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, intel_audio_cdclk_change_post(dev_priv); if (drm_WARN(&dev_priv->drm, - intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config), + intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config), "cdclk state doesn't match!\n")) { - intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "[hw state]"); + intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]"); intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]"); } } @@ -2300,7 +2300,7 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate); /* - * HACK. Currently for TGL platforms we calculate + * HACK. Currently for TGL/DG2 platforms we calculate * min_cdclk initially based on pixel_rate divided * by 2, accounting for also plane requirements, * however in some cases the lowest possible CDCLK @@ -2308,14 +2308,14 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state) * Explicitly stating here that this seems to be currently * rather a Hack, than final solution. */ - if (IS_TIGERLAKE(dev_priv)) { + if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) { /* * Clamp to max_cdclk_freq in case pixel rate is higher, * in order not to break an 8K, but still leave W/A at place. */ min_cdclk = max_t(int, min_cdclk, min_t(int, crtc_state->pixel_rate, - dev_priv->max_cdclk_freq)); + dev_priv->display.cdclk.max_cdclk_freq)); } return min_cdclk; @@ -2368,10 +2368,10 @@ static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state) for_each_pipe(dev_priv, pipe) min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk); - if (min_cdclk > dev_priv->max_cdclk_freq) { + if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) { drm_dbg_kms(&dev_priv->drm, "required cdclk (%d kHz) exceeds max (%d kHz)\n", - min_cdclk, dev_priv->max_cdclk_freq); + min_cdclk, dev_priv->display.cdclk.max_cdclk_freq); return -EINVAL; } @@ -2643,7 +2643,7 @@ intel_atomic_get_cdclk_state(struct intel_atomic_state *state) struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_global_state *cdclk_state; - cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj); + cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj); if (IS_ERR(cdclk_state)) return ERR_CAST(cdclk_state); @@ -2693,7 +2693,7 @@ int intel_cdclk_init(struct drm_i915_private *dev_priv) if (!cdclk_state) return -ENOMEM; - intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj, + intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj, &cdclk_state->base, &intel_cdclk_funcs); return 0; @@ -2799,7 +2799,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) { - int max_cdclk_freq = dev_priv->max_cdclk_freq; + int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq; if (DISPLAY_VER(dev_priv) >= 10) return 2 * max_cdclk_freq; @@ -2825,19 +2825,19 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) void intel_update_max_cdclk(struct drm_i915_private *dev_priv) { if (IS_JSL_EHL(dev_priv)) { - if (dev_priv->cdclk.hw.ref == 24000) - dev_priv->max_cdclk_freq = 552000; + if (dev_priv->display.cdclk.hw.ref == 24000) + dev_priv->display.cdclk.max_cdclk_freq = 552000; else - dev_priv->max_cdclk_freq = 556800; + dev_priv->display.cdclk.max_cdclk_freq = 556800; } else if (DISPLAY_VER(dev_priv) >= 11) { - if (dev_priv->cdclk.hw.ref == 24000) - dev_priv->max_cdclk_freq = 648000; + if (dev_priv->display.cdclk.hw.ref == 24000) + dev_priv->display.cdclk.max_cdclk_freq = 648000; else - dev_priv->max_cdclk_freq = 652800; + dev_priv->display.cdclk.max_cdclk_freq = 652800; } else if (IS_GEMINILAKE(dev_priv)) { - dev_priv->max_cdclk_freq = 316800; + dev_priv->display.cdclk.max_cdclk_freq = 316800; } else if (IS_BROXTON(dev_priv)) { - dev_priv->max_cdclk_freq = 624000; + dev_priv->display.cdclk.max_cdclk_freq = 624000; } else if (DISPLAY_VER(dev_priv) == 9) { u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; int max_cdclk, vco; @@ -2859,7 +2859,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) else max_cdclk = 308571; - dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco); + dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco); } else if (IS_BROADWELL(dev_priv)) { /* * FIXME with extra cooling we can allow @@ -2868,26 +2868,26 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) * available? PCI ID, VTB, something else? */ if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT) - dev_priv->max_cdclk_freq = 450000; + dev_priv->display.cdclk.max_cdclk_freq = 450000; else if (IS_BDW_ULX(dev_priv)) - dev_priv->max_cdclk_freq = 450000; + dev_priv->display.cdclk.max_cdclk_freq = 450000; else if (IS_BDW_ULT(dev_priv)) - dev_priv->max_cdclk_freq = 540000; + dev_priv->display.cdclk.max_cdclk_freq = 540000; else - dev_priv->max_cdclk_freq = 675000; + dev_priv->display.cdclk.max_cdclk_freq = 675000; } else if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->max_cdclk_freq = 320000; + dev_priv->display.cdclk.max_cdclk_freq = 320000; } else if (IS_VALLEYVIEW(dev_priv)) { - dev_priv->max_cdclk_freq = 400000; + dev_priv->display.cdclk.max_cdclk_freq = 400000; } else { /* otherwise assume cdclk is fixed */ - dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk; + dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk; } dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv); drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n", - dev_priv->max_cdclk_freq); + dev_priv->display.cdclk.max_cdclk_freq); drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n", dev_priv->max_dotclk_freq); @@ -2901,7 +2901,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) */ void intel_update_cdclk(struct drm_i915_private *dev_priv) { - intel_cdclk_get_cdclk(dev_priv, &dev_priv->cdclk.hw); + intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw); /* * 9:0 CMBUS [sic] CDCLK frequency (cdfreq): @@ -2911,7 +2911,7 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv) */ if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) intel_de_write(dev_priv, GMBUSFREQ_VLV, - DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000)); + DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000)); } static int dg1_rawclk(struct drm_i915_private *dev_priv) @@ -3036,6 +3036,13 @@ u32 intel_read_rawclk(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1) freq = dg1_rawclk(dev_priv); + else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP) + /* + * MTL always uses a 38.4 MHz rawclk. The bspec tells us + * "RAWCLK_FREQ defaults to the values for 38.4 and does + * not need to be programmed." + */ + freq = 38400; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) freq = cnp_rawclk(dev_priv); else if (HAS_PCH_SPLIT(dev_priv)) @@ -3187,78 +3194,78 @@ static const struct intel_cdclk_funcs i830_cdclk_funcs = { void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) { if (IS_DG2(dev_priv)) { - dev_priv->cdclk_funcs = &tgl_cdclk_funcs; - dev_priv->cdclk.table = dg2_cdclk_table; + dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs; + dev_priv->display.cdclk.table = dg2_cdclk_table; } else if (IS_ALDERLAKE_P(dev_priv)) { - dev_priv->cdclk_funcs = &tgl_cdclk_funcs; + dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs; /* Wa_22011320316:adl-p[a0] */ if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) - dev_priv->cdclk.table = adlp_a_step_cdclk_table; + dev_priv->display.cdclk.table = adlp_a_step_cdclk_table; else - dev_priv->cdclk.table = adlp_cdclk_table; + dev_priv->display.cdclk.table = adlp_cdclk_table; } else if (IS_ROCKETLAKE(dev_priv)) { - dev_priv->cdclk_funcs = &tgl_cdclk_funcs; - dev_priv->cdclk.table = rkl_cdclk_table; + dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs; + dev_priv->display.cdclk.table = rkl_cdclk_table; } else if (DISPLAY_VER(dev_priv) >= 12) { - dev_priv->cdclk_funcs = &tgl_cdclk_funcs; - dev_priv->cdclk.table = icl_cdclk_table; + dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs; + dev_priv->display.cdclk.table = icl_cdclk_table; } else if (IS_JSL_EHL(dev_priv)) { - dev_priv->cdclk_funcs = &ehl_cdclk_funcs; - dev_priv->cdclk.table = icl_cdclk_table; + dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs; + dev_priv->display.cdclk.table = icl_cdclk_table; } else if (DISPLAY_VER(dev_priv) >= 11) { - dev_priv->cdclk_funcs = &icl_cdclk_funcs; - dev_priv->cdclk.table = icl_cdclk_table; + dev_priv->display.funcs.cdclk = &icl_cdclk_funcs; + dev_priv->display.cdclk.table = icl_cdclk_table; } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) { - dev_priv->cdclk_funcs = &bxt_cdclk_funcs; + dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs; if (IS_GEMINILAKE(dev_priv)) - dev_priv->cdclk.table = glk_cdclk_table; + dev_priv->display.cdclk.table = glk_cdclk_table; else - dev_priv->cdclk.table = bxt_cdclk_table; + dev_priv->display.cdclk.table = bxt_cdclk_table; } else if (DISPLAY_VER(dev_priv) == 9) { - dev_priv->cdclk_funcs = &skl_cdclk_funcs; + dev_priv->display.funcs.cdclk = &skl_cdclk_funcs; } else if (IS_BROADWELL(dev_priv)) { - dev_priv->cdclk_funcs = &bdw_cdclk_funcs; + dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs; } else if (IS_HASWELL(dev_priv)) { - dev_priv->cdclk_funcs = &hsw_cdclk_funcs; + dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs; } else if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->cdclk_funcs = &chv_cdclk_funcs; + dev_priv->display.funcs.cdclk = &chv_cdclk_funcs; } else if (IS_VALLEYVIEW(dev_priv)) { - dev_priv->cdclk_funcs = &vlv_cdclk_funcs; + dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs; } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) { - dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs; } else if (IS_IRONLAKE(dev_priv)) { - dev_priv->cdclk_funcs = &ilk_cdclk_funcs; + dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs; } else if (IS_GM45(dev_priv)) { - dev_priv->cdclk_funcs = &gm45_cdclk_funcs; + dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs; } else if (IS_G45(dev_priv)) { - dev_priv->cdclk_funcs = &g33_cdclk_funcs; + dev_priv->display.funcs.cdclk = &g33_cdclk_funcs; } else if (IS_I965GM(dev_priv)) { - dev_priv->cdclk_funcs = &i965gm_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs; } else if (IS_I965G(dev_priv)) { - dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs; } else if (IS_PINEVIEW(dev_priv)) { - dev_priv->cdclk_funcs = &pnv_cdclk_funcs; + dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs; } else if (IS_G33(dev_priv)) { - dev_priv->cdclk_funcs = &g33_cdclk_funcs; + dev_priv->display.funcs.cdclk = &g33_cdclk_funcs; } else if (IS_I945GM(dev_priv)) { - dev_priv->cdclk_funcs = &i945gm_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs; } else if (IS_I945G(dev_priv)) { - dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs; + dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs; } else if (IS_I915GM(dev_priv)) { - dev_priv->cdclk_funcs = &i915gm_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs; } else if (IS_I915G(dev_priv)) { - dev_priv->cdclk_funcs = &i915g_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs; } else if (IS_I865G(dev_priv)) { - dev_priv->cdclk_funcs = &i865g_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs; } else if (IS_I85X(dev_priv)) { - dev_priv->cdclk_funcs = &i85x_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs; } else if (IS_I845G(dev_priv)) { - dev_priv->cdclk_funcs = &i845g_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs; } else if (IS_I830(dev_priv)) { - dev_priv->cdclk_funcs = &i830_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i830_cdclk_funcs; } - if (drm_WARN(&dev_priv->drm, !dev_priv->cdclk_funcs, + if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk, "Unknown platform. Assuming i830\n")) - dev_priv->cdclk_funcs = &i830_cdclk_funcs; + dev_priv->display.funcs.cdclk = &i830_cdclk_funcs; } diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h index b535cf6a7d9e7b5349d3f8af486aa5625c4c1477..c674879a84a58c765d1cfccc44b17e0f3eb2e240 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.h +++ b/drivers/gpu/drm/i915/display/intel_cdclk.h @@ -77,9 +77,9 @@ intel_atomic_get_cdclk_state(struct intel_atomic_state *state); #define to_intel_cdclk_state(x) container_of((x), struct intel_cdclk_state, base) #define intel_atomic_get_old_cdclk_state(state) \ - to_intel_cdclk_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->cdclk.obj)) + to_intel_cdclk_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.cdclk.obj)) #define intel_atomic_get_new_cdclk_state(state) \ - to_intel_cdclk_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->cdclk.obj)) + to_intel_cdclk_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->display.cdclk.obj)) int intel_cdclk_init(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 9583d17e858d95e9c9052fe1caf35cd889fe7001..6bda4274eae928fbbbc3333af2505d9ba20eb5b2 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -26,6 +26,7 @@ #include "intel_de.h" #include "intel_display_types.h" #include "intel_dpll.h" +#include "intel_dsb.h" #include "vlv_dsi_pll.h" struct intel_color_funcs { @@ -1167,22 +1168,22 @@ void intel_color_load_luts(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->color_funcs->load_luts(crtc_state); + dev_priv->display.funcs.color->load_luts(crtc_state); } void intel_color_commit_noarm(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - if (dev_priv->color_funcs->color_commit_noarm) - dev_priv->color_funcs->color_commit_noarm(crtc_state); + if (dev_priv->display.funcs.color->color_commit_noarm) + dev_priv->display.funcs.color->color_commit_noarm(crtc_state); } void intel_color_commit_arm(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->color_funcs->color_commit_arm(crtc_state); + dev_priv->display.funcs.color->color_commit_arm(crtc_state); } static bool intel_can_preload_luts(const struct intel_crtc_state *new_crtc_state) @@ -1238,15 +1239,15 @@ int intel_color_check(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - return dev_priv->color_funcs->color_check(crtc_state); + return dev_priv->display.funcs.color->color_check(crtc_state); } void intel_color_get_config(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - if (dev_priv->color_funcs->read_luts) - dev_priv->color_funcs->read_luts(crtc_state); + if (dev_priv->display.funcs.color->read_luts) + dev_priv->display.funcs.color->read_luts(crtc_state); } static bool need_plane_update(struct intel_plane *plane, @@ -2225,28 +2226,28 @@ void intel_color_init(struct intel_crtc *crtc) if (HAS_GMCH(dev_priv)) { if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->color_funcs = &chv_color_funcs; + dev_priv->display.funcs.color = &chv_color_funcs; } else if (DISPLAY_VER(dev_priv) >= 4) { - dev_priv->color_funcs = &i965_color_funcs; + dev_priv->display.funcs.color = &i965_color_funcs; } else { - dev_priv->color_funcs = &i9xx_color_funcs; + dev_priv->display.funcs.color = &i9xx_color_funcs; } } else { if (DISPLAY_VER(dev_priv) >= 11) - dev_priv->color_funcs = &icl_color_funcs; + dev_priv->display.funcs.color = &icl_color_funcs; else if (DISPLAY_VER(dev_priv) == 10) - dev_priv->color_funcs = &glk_color_funcs; + dev_priv->display.funcs.color = &glk_color_funcs; else if (DISPLAY_VER(dev_priv) == 9) - dev_priv->color_funcs = &skl_color_funcs; + dev_priv->display.funcs.color = &skl_color_funcs; else if (DISPLAY_VER(dev_priv) == 8) - dev_priv->color_funcs = &bdw_color_funcs; + dev_priv->display.funcs.color = &bdw_color_funcs; else if (DISPLAY_VER(dev_priv) == 7) { if (IS_HASWELL(dev_priv)) - dev_priv->color_funcs = &hsw_color_funcs; + dev_priv->display.funcs.color = &hsw_color_funcs; else - dev_priv->color_funcs = &ivb_color_funcs; + dev_priv->display.funcs.color = &ivb_color_funcs; } else - dev_priv->color_funcs = &ilk_color_funcs; + dev_priv->display.funcs.color = &ilk_color_funcs; } drm_crtc_enable_color_mgmt(&crtc->base, diff --git a/drivers/gpu/drm/i915/display/intel_connector.c b/drivers/gpu/drm/i915/display/intel_connector.c index 1dcc268927a25eda32394785e6bd7bf86b910d6c..6d5cbeb8df4daf21e00f3f6d03f9c0fa5db5f90e 100644 --- a/drivers/gpu/drm/i915/display/intel_connector.c +++ b/drivers/gpu/drm/i915/display/intel_connector.c @@ -229,7 +229,7 @@ intel_attach_force_audio_property(struct drm_connector *connector) struct drm_i915_private *dev_priv = to_i915(dev); struct drm_property *prop; - prop = dev_priv->force_audio_property; + prop = dev_priv->display.properties.force_audio; if (prop == NULL) { prop = drm_property_create_enum(dev, 0, "audio", @@ -238,7 +238,7 @@ intel_attach_force_audio_property(struct drm_connector *connector) if (prop == NULL) return; - dev_priv->force_audio_property = prop; + dev_priv->display.properties.force_audio = prop; } drm_object_attach_property(&connector->base, prop, 0); } @@ -256,7 +256,7 @@ intel_attach_broadcast_rgb_property(struct drm_connector *connector) struct drm_i915_private *dev_priv = to_i915(dev); struct drm_property *prop; - prop = dev_priv->broadcast_rgb_property; + prop = dev_priv->display.properties.broadcast_rgb; if (prop == NULL) { prop = drm_property_create_enum(dev, DRM_MODE_PROP_ENUM, "Broadcast RGB", @@ -265,7 +265,7 @@ intel_attach_broadcast_rgb_property(struct drm_connector *connector) if (prop == NULL) return; - dev_priv->broadcast_rgb_property = prop; + dev_priv->display.properties.broadcast_rgb = prop; } drm_object_attach_property(&connector->base, prop, 0); diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c index 6a3893c8ff22efbef85ee0996c90f20b2728a879..4a8ff2f976085beb972692203a4f8d1f4a000c3a 100644 --- a/drivers/gpu/drm/i915/display/intel_crt.c +++ b/drivers/gpu/drm/i915/display/intel_crt.c @@ -46,6 +46,7 @@ #include "intel_gmbus.h" #include "intel_hotplug.h" #include "intel_pch_display.h" +#include "intel_pch_refclk.h" /* Here's the desired hotplug mode */ #define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_PERIOD_128 | \ @@ -444,6 +445,8 @@ static int hsw_crt_compute_config(struct intel_encoder *encoder, /* FDI must always be 2.7 GHz */ pipe_config->port_clock = 135000 * 2; + adjusted_mode->crtc_clock = lpt_iclkip(pipe_config); + return 0; } @@ -643,9 +646,7 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector) struct i2c_adapter *i2c; bool ret = false; - BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG); - - i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); + i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->display.vbt.crt_ddc_pin); edid = intel_crt_get_edid(connector, i2c); if (edid) { @@ -931,7 +932,7 @@ static int intel_crt_get_modes(struct drm_connector *connector) wakeref = intel_display_power_get(dev_priv, intel_encoder->power_domain); - i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin); + i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->display.vbt.crt_ddc_pin); ret = intel_crt_ddc_get_modes(connector, i2c); if (ret || !IS_G4X(dev_priv)) goto out; @@ -1110,8 +1111,8 @@ void intel_crt_init(struct drm_i915_private *dev_priv) u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT | FDI_RX_LINK_REVERSAL_OVERRIDE; - dev_priv->fdi_rx_config = intel_de_read(dev_priv, - FDI_RX_CTL(PIPE_A)) & fdi_config; + dev_priv->display.fdi.rx_config = intel_de_read(dev_priv, + FDI_RX_CTL(PIPE_A)) & fdi_config; } intel_crt_reset(&crt->base.base); diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c index 4ca6e9493ff2f427cbb5d106cb168ad153431e94..e9212f69c360b521180d58a873652c0c8d47ec6d 100644 --- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c +++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c @@ -134,8 +134,8 @@ static void intel_dump_plane_state(const struct intel_plane_state *plane_state) plane->base.base.id, plane->base.name, fb->base.id, fb->width, fb->height, &fb->format->format, fb->modifier, str_yes_no(plane_state->uapi.visible)); - drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n", - plane_state->hw.rotation, plane_state->scaler_id); + drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d, scaling_filter: %d\n", + plane_state->hw.rotation, plane_state->scaler_id, plane_state->hw.scaling_filter); if (plane_state->uapi.visible) drm_dbg_kms(&i915->drm, "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n", @@ -262,10 +262,11 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config, if (DISPLAY_VER(i915) >= 9) drm_dbg_kms(&i915->drm, - "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n", + "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d\n", crtc->num_scalers, pipe_config->scaler_state.scaler_users, - pipe_config->scaler_state.scaler_id); + pipe_config->scaler_state.scaler_id, + pipe_config->hw.scaling_filter); if (HAS_GMCH(i915)) drm_dbg_kms(&i915->drm, diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 16ac560eab4986ca55c598cccd95194563a485bb..87899e89b3a7df0fd9e9d722e3f75fd19c750d75 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -19,9 +19,9 @@ #include "intel_fb.h" #include "intel_fb_pin.h" #include "intel_frontbuffer.h" -#include "intel_pm.h" #include "intel_psr.h" #include "intel_sprite.h" +#include "skl_watermark.h" /* Cursor formats */ static const u32 intel_cursor_formats[] = { diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index 2330604b0bccc1023e0ac61145ca77c351369bed..da8472cdc135709d83fc645b23e96f8d13a8c11e 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -57,6 +57,7 @@ #include "intel_lspcon.h" #include "intel_pps.h" #include "intel_psr.h" +#include "intel_quirks.h" #include "intel_snps_phy.h" #include "intel_sprite.h" #include "intel_tc.h" @@ -323,28 +324,6 @@ static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, } } -int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config) -{ - int dotclock; - - if (intel_crtc_has_dp_encoder(pipe_config)) - dotclock = intel_dotclock_calculate(pipe_config->port_clock, - &pipe_config->dp_m_n); - else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp > 24) - dotclock = pipe_config->port_clock * 24 / pipe_config->pipe_bpp; - else - dotclock = pipe_config->port_clock; - - if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 && - !intel_crtc_has_dp_encoder(pipe_config)) - dotclock *= 2; - - if (pipe_config->pixel_multiplier) - dotclock /= pipe_config->pixel_multiplier; - - return dotclock; -} - static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) { /* CRT dotclock is determined via other means */ @@ -631,7 +610,7 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl); - if (dev_priv->quirks & QUIRK_INCREASE_DDI_DISABLED_TIME && + if (intel_has_quirk(dev_priv, QUIRK_INCREASE_DDI_DISABLED_TIME) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { drm_dbg_kms(&dev_priv->drm, "Quirk Increase DDI disabled time\n"); @@ -1425,7 +1404,7 @@ hsw_set_signal_levels(struct intel_encoder *encoder, static void _icl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_sel_mask, u32 clk_sel, u32 clk_off) { - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, reg, clk_sel_mask, clk_sel); @@ -1435,17 +1414,17 @@ static void _icl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg, */ intel_de_rmw(i915, reg, clk_off, 0); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); } static void _icl_ddi_disable_clock(struct drm_i915_private *i915, i915_reg_t reg, u32 clk_off) { - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, reg, 0, clk_off); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); } static bool _icl_ddi_is_clock_enabled(struct drm_i915_private *i915, i915_reg_t reg, @@ -1720,12 +1699,12 @@ static void icl_ddi_tc_enable_clock(struct intel_encoder *encoder, intel_de_write(i915, DDI_CLK_SEL(port), icl_pll_to_ddi_clk_sel(encoder, crtc_state)); - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, ICL_DPCLKA_CFGCR0, ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port), 0); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); } static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder) @@ -1734,12 +1713,12 @@ static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder) enum tc_port tc_port = intel_port_to_tc(i915, encoder->port); enum port port = encoder->port; - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, ICL_DPCLKA_CFGCR0, 0, ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port)); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); intel_de_write(i915, DDI_CLK_SEL(port), DDI_CLK_SEL_NONE); } @@ -1824,7 +1803,7 @@ static void skl_ddi_enable_clock(struct intel_encoder *encoder, if (drm_WARN_ON(&i915->drm, !pll)) return; - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, DPLL_CTRL2, DPLL_CTRL2_DDI_CLK_OFF(port) | @@ -1832,7 +1811,7 @@ static void skl_ddi_enable_clock(struct intel_encoder *encoder, DPLL_CTRL2_DDI_CLK_SEL(pll->info->id, port) | DPLL_CTRL2_DDI_SEL_OVERRIDE(port)); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); } static void skl_ddi_disable_clock(struct intel_encoder *encoder) @@ -1840,12 +1819,12 @@ static void skl_ddi_disable_clock(struct intel_encoder *encoder) struct drm_i915_private *i915 = to_i915(encoder->base.dev); enum port port = encoder->port; - mutex_lock(&i915->dpll.lock); + mutex_lock(&i915->display.dpll.lock); intel_de_rmw(i915, DPLL_CTRL2, 0, DPLL_CTRL2_DDI_CLK_OFF(port)); - mutex_unlock(&i915->dpll.lock); + mutex_unlock(&i915->display.dpll.lock); } static bool skl_ddi_is_clock_enabled(struct intel_encoder *encoder) @@ -2691,10 +2670,14 @@ static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state, dig_port->set_infoframes(encoder, false, old_crtc_state, old_conn_state); - intel_ddi_disable_pipe_clock(old_crtc_state); + if (DISPLAY_VER(dev_priv) < 12) + intel_ddi_disable_pipe_clock(old_crtc_state); intel_disable_ddi_buf(encoder, old_crtc_state); + if (DISPLAY_VER(dev_priv) >= 12) + intel_ddi_disable_pipe_clock(old_crtc_state); + intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain, fetch_and_zero(&dig_port->ddi_io_wakeref)); @@ -2862,6 +2845,8 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, struct intel_digital_port *dig_port = enc_to_dig_port(encoder); struct drm_connector *connector = conn_state->connector; enum port port = encoder->port; + enum phy phy = intel_port_to_phy(dev_priv, port); + u32 buf_ctl; if (!intel_hdmi_handle_sink_scrambling(encoder, connector, crtc_state->hdmi_high_tmds_clock_ratio, @@ -2919,8 +2904,12 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, * On ADL_P the PHY link rate and lane count must be programmed but * these are both 0 for HDMI. */ - intel_de_write(dev_priv, DDI_BUF_CTL(port), - dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE); + buf_ctl = dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE; + if (IS_ALDERLAKE_P(dev_priv) && intel_phy_is_tc(dev_priv, phy)) { + drm_WARN_ON(&dev_priv->drm, !intel_tc_port_in_legacy_mode(dig_port)); + buf_ctl |= DDI_BUF_CTL_TC_PHY_OWNERSHIP; + } + intel_de_write(dev_priv, DDI_BUF_CTL(port), buf_ctl); intel_audio_codec_enable(encoder, crtc_state, conn_state); } @@ -3611,10 +3600,22 @@ static void intel_ddi_sync_state(struct intel_encoder *encoder, static bool intel_ddi_initial_fastset_check(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) { - if (intel_crtc_has_dp_encoder(crtc_state)) - return intel_dp_initial_fastset_check(encoder, crtc_state); + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(i915, encoder->port); + bool fastset = true; - return true; + if (intel_phy_is_tc(i915, phy)) { + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] Forcing full modeset to compute TC port DPLLs\n", + encoder->base.base.id, encoder->base.name); + crtc_state->uapi.mode_changed = true; + fastset = false; + } + + if (intel_crtc_has_dp_encoder(crtc_state) && + !intel_dp_initial_fastset_check(encoder, crtc_state)) + fastset = false; + + return fastset; } static enum intel_output_type @@ -4028,7 +4029,7 @@ intel_ddi_hotplug(struct intel_encoder *encoder, static bool lpt_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin]; + u32 bit = dev_priv->display.hotplug.pch_hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, SDEISR) & bit; } @@ -4036,7 +4037,7 @@ static bool lpt_digital_port_connected(struct intel_encoder *encoder) static bool hsw_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; + u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, DEISR) & bit; } @@ -4044,7 +4045,7 @@ static bool hsw_digital_port_connected(struct intel_encoder *encoder) static bool bdw_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; + u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, GEN8_DE_PORT_ISR) & bit; } diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 40fbf8a296e2331ac19a6bba57d76275dcbd6599..461c62c884133a567a8ee1d6e67d6deb60d0ed64 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -91,6 +91,7 @@ #include "intel_dmc.h" #include "intel_dp_link_training.h" #include "intel_dpt.h" +#include "intel_dsb.h" #include "intel_fbc.h" #include "intel_fbdev.h" #include "intel_fdi.h" @@ -117,6 +118,7 @@ #include "i9xx_plane.h" #include "skl_scaler.h" #include "skl_universal_plane.h" +#include "skl_watermark.h" #include "vlv_dsi.h" #include "vlv_dsi_pll.h" #include "vlv_dsi_regs.h" @@ -163,16 +165,16 @@ static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state); */ void intel_update_watermarks(struct drm_i915_private *dev_priv) { - if (dev_priv->wm_disp->update_wm) - dev_priv->wm_disp->update_wm(dev_priv); + if (dev_priv->display.funcs.wm->update_wm) + dev_priv->display.funcs.wm->update_wm(dev_priv); } static int intel_compute_pipe_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (dev_priv->wm_disp->compute_pipe_wm) - return dev_priv->wm_disp->compute_pipe_wm(state, crtc); + if (dev_priv->display.funcs.wm->compute_pipe_wm) + return dev_priv->display.funcs.wm->compute_pipe_wm(state, crtc); return 0; } @@ -180,20 +182,20 @@ static int intel_compute_intermediate_wm(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (!dev_priv->wm_disp->compute_intermediate_wm) + if (!dev_priv->display.funcs.wm->compute_intermediate_wm) return 0; if (drm_WARN_ON(&dev_priv->drm, - !dev_priv->wm_disp->compute_pipe_wm)) + !dev_priv->display.funcs.wm->compute_pipe_wm)) return 0; - return dev_priv->wm_disp->compute_intermediate_wm(state, crtc); + return dev_priv->display.funcs.wm->compute_intermediate_wm(state, crtc); } static bool intel_initial_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (dev_priv->wm_disp->initial_watermarks) { - dev_priv->wm_disp->initial_watermarks(state, crtc); + if (dev_priv->display.funcs.wm->initial_watermarks) { + dev_priv->display.funcs.wm->initial_watermarks(state, crtc); return true; } return false; @@ -203,23 +205,23 @@ static void intel_atomic_update_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (dev_priv->wm_disp->atomic_update_watermarks) - dev_priv->wm_disp->atomic_update_watermarks(state, crtc); + if (dev_priv->display.funcs.wm->atomic_update_watermarks) + dev_priv->display.funcs.wm->atomic_update_watermarks(state, crtc); } static void intel_optimize_watermarks(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (dev_priv->wm_disp->optimize_watermarks) - dev_priv->wm_disp->optimize_watermarks(state, crtc); + if (dev_priv->display.funcs.wm->optimize_watermarks) + dev_priv->display.funcs.wm->optimize_watermarks(state, crtc); } static int intel_compute_global_watermarks(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - if (dev_priv->wm_disp->compute_global_watermarks) - return dev_priv->wm_disp->compute_global_watermarks(state); + if (dev_priv->display.funcs.wm->compute_global_watermarks) + return dev_priv->display.funcs.wm->compute_global_watermarks(state); return 0; } @@ -618,7 +620,10 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state) if (!IS_I830(dev_priv)) val &= ~PIPECONF_ENABLE; - if (DISPLAY_VER(dev_priv) >= 12) + if (DISPLAY_VER(dev_priv) >= 14) + intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder), + FECSTALL_DIS_DPTSTREAM_DPTTG, 0); + else if (DISPLAY_VER(dev_priv) >= 12) intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder), FECSTALL_DIS_DPTSTREAM_DPTTG, 0); @@ -1486,7 +1491,7 @@ static void intel_encoders_update_prepare(struct intel_atomic_state *state) * Make sure the DPLL state is up-to-date for fastset TypeC ports after non-blocking commits. * TODO: Update the DPLL state for all cases in the encoder->update_prepare() hook. */ - if (i915->dpll.mgr) { + if (i915->display.dpll.mgr) { for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { if (intel_crtc_needs_modeset(new_crtc_state)) continue; @@ -1838,7 +1843,9 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder); + enum transcoder transcoder = crtc_state->cpu_transcoder; + i915_reg_t reg = DISPLAY_VER(dev_priv) >= 14 ? MTL_CHICKEN_TRANS(transcoder) : + CHICKEN_TRANS(transcoder); u32 val; val = intel_de_read(dev_priv, reg); @@ -2080,22 +2087,20 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy) { if (phy == PHY_NONE) return false; - else if (IS_DG2(dev_priv)) - /* - * DG2 outputs labelled as "combo PHY" in the bspec use - * SNPS PHYs with completely different programming, - * hence we always return false here. - */ - return false; else if (IS_ALDERLAKE_S(dev_priv)) return phy <= PHY_E; else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) return phy <= PHY_D; else if (IS_JSL_EHL(dev_priv)) return phy <= PHY_C; - else if (DISPLAY_VER(dev_priv) >= 11) + else if (IS_ALDERLAKE_P(dev_priv) || IS_DISPLAY_VER(dev_priv, 11, 12)) return phy <= PHY_B; else + /* + * DG2 outputs labelled as "combo PHY" in the bspec use + * SNPS PHYs with completely different programming, + * hence we always return false here. + */ return false; } @@ -2401,7 +2406,7 @@ static void i9xx_crtc_disable(struct intel_atomic_state *state, if (DISPLAY_VER(dev_priv) != 2) intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false); - if (!dev_priv->wm_disp->initial_watermarks) + if (!dev_priv->display.funcs.wm->initial_watermarks) intel_update_watermarks(dev_priv); /* clock the pipe down to 640x480@60 to potentially save power */ @@ -2660,7 +2665,7 @@ static int intel_crtc_compute_pipe_mode(struct intel_crtc_state *crtc_state) intel_mode_from_crtc_timings(pipe_mode, pipe_mode); if (DISPLAY_VER(i915) < 4) { - clock_limit = i915->max_cdclk_freq * 9 / 10; + clock_limit = i915->display.cdclk.max_cdclk_freq * 9 / 10; /* * Enable double wide mode when the dot clock @@ -2692,6 +2697,10 @@ static int intel_crtc_compute_config(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); int ret; + ret = intel_dpll_crtc_compute_clock(state, crtc); + if (ret) + return ret; + ret = intel_crtc_compute_pipe_src(crtc_state); if (ret) return ret; @@ -2718,19 +2727,11 @@ intel_reduce_m_n_ratio(u32 *num, u32 *den) } } -static void compute_m_n(unsigned int m, unsigned int n, - u32 *ret_m, u32 *ret_n, - bool constant_n) +static void compute_m_n(u32 *ret_m, u32 *ret_n, + u32 m, u32 n, u32 constant_n) { - /* - * Several DP dongles in particular seem to be fussy about - * too large link M/N values. Give N value as 0x8000 that - * should be acceptable by specific devices. 0x8000 is the - * specified fixed N value for asynchronous clock mode, - * which the devices expect also in synchronous clock mode. - */ if (constant_n) - *ret_n = DP_LINK_CONSTANT_N_VALUE; + *ret_n = constant_n; else *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX); @@ -2742,22 +2743,28 @@ void intel_link_compute_m_n(u16 bits_per_pixel, int nlanes, int pixel_clock, int link_clock, struct intel_link_m_n *m_n, - bool constant_n, bool fec_enable) + bool fec_enable) { u32 data_clock = bits_per_pixel * pixel_clock; if (fec_enable) data_clock = intel_dp_mode_to_fec_clock(data_clock); + /* + * Windows/BIOS uses fixed M/N values always. Follow suit. + * + * Also several DP dongles in particular seem to be fussy + * about too large link M/N values. Presumably the 20bit + * value used by Windows/BIOS is acceptable to everyone. + */ m_n->tu = 64; - compute_m_n(data_clock, - link_clock * nlanes * 8, - &m_n->data_m, &m_n->data_n, - constant_n); + compute_m_n(&m_n->data_m, &m_n->data_n, + data_clock, link_clock * nlanes * 8, + 0x8000000); - compute_m_n(pixel_clock, link_clock, - &m_n->link_m, &m_n->link_n, - constant_n); + compute_m_n(&m_n->link_m, &m_n->link_n, + pixel_clock, link_clock, + 0x80000); } static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv) @@ -2773,12 +2780,12 @@ static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv) PCH_DREF_CONTROL) & DREF_SSC1_ENABLE; - if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) { + if (dev_priv->display.vbt.lvds_use_ssc != bios_lvds_use_ssc) { drm_dbg_kms(&dev_priv->drm, "SSC %s by BIOS, overriding VBT which says %s\n", str_enabled_disabled(bios_lvds_use_ssc), - str_enabled_disabled(dev_priv->vbt.lvds_use_ssc)); - dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc; + str_enabled_disabled(dev_priv->display.vbt.lvds_use_ssc)); + dev_priv->display.vbt.lvds_use_ssc = bios_lvds_use_ssc; } } } @@ -4126,7 +4133,9 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, } if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) { - tmp = intel_de_read(dev_priv, CHICKEN_TRANS(pipe_config->cpu_transcoder)); + tmp = intel_de_read(dev_priv, DISPLAY_VER(dev_priv) >= 14 ? + MTL_CHICKEN_TRANS(pipe_config->cpu_transcoder) : + CHICKEN_TRANS(pipe_config->cpu_transcoder)); pipe_config->framestart_delay = REG_FIELD_GET(HSW_FRAME_START_DELAY_MASK, tmp) + 1; } else { @@ -4145,7 +4154,7 @@ bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state) struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *i915 = to_i915(crtc->base.dev); - if (!i915->display->get_pipe_config(crtc, crtc_state)) + if (!i915->display.funcs.display->get_pipe_config(crtc, crtc_state)) return false; crtc_state->hw.active = true; @@ -4374,7 +4383,7 @@ static int i9xx_pll_refclk(struct drm_device *dev, u32 dpll = pipe_config->dpll_hw_state.dpll; if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN) - return dev_priv->vbt.lvds_ssc_freq; + return dev_priv->display.vbt.lvds_ssc_freq; else if (HAS_PCH_SPLIT(dev_priv)) return 120000; else if (DISPLAY_VER(dev_priv) != 2) @@ -4492,7 +4501,31 @@ int intel_dotclock_calculate(int link_freq, if (!m_n->link_n) return 0; - return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n); + return DIV_ROUND_UP_ULL(mul_u32_u32(m_n->link_m, link_freq), + m_n->link_n); +} + +int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config) +{ + int dotclock; + + if (intel_crtc_has_dp_encoder(pipe_config)) + dotclock = intel_dotclock_calculate(pipe_config->port_clock, + &pipe_config->dp_m_n); + else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp > 24) + dotclock = DIV_ROUND_CLOSEST(pipe_config->port_clock * 24, + pipe_config->pipe_bpp); + else + dotclock = pipe_config->port_clock; + + if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 && + !intel_crtc_has_dp_encoder(pipe_config)) + dotclock *= 2; + + if (pipe_config->pixel_multiplier) + dotclock /= pipe_config->pixel_multiplier; + + return dotclock; } /* Returns the currently programmed mode of the given encoder. */ @@ -4753,7 +4786,7 @@ static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state) /* Display WA #1135: BXT:ALL GLK:ALL */ if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) && - dev_priv->ipc_enabled) + skl_watermark_ipc_enabled(dev_priv)) linetime_wm /= 2; return min(linetime_wm, 0x1ff); @@ -4799,10 +4832,6 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, crtc_state->update_wm_post = true; if (mode_changed) { - ret = intel_dpll_crtc_compute_clock(state, crtc); - if (ret) - return ret; - ret = intel_dpll_crtc_get_shared_dpll(state, crtc); if (ret) return ret; @@ -5366,47 +5395,15 @@ bool intel_fuzzy_clock_check(int clock1, int clock2) return false; } -static bool -intel_compare_m_n(unsigned int m, unsigned int n, - unsigned int m2, unsigned int n2, - bool exact) -{ - if (m == m2 && n == n2) - return true; - - if (exact || !m || !n || !m2 || !n2) - return false; - - BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX); - - if (n > n2) { - while (n > n2) { - m2 <<= 1; - n2 <<= 1; - } - } else if (n < n2) { - while (n < n2) { - m <<= 1; - n <<= 1; - } - } - - if (n != n2) - return false; - - return intel_fuzzy_clock_check(m, m2); -} - static bool intel_compare_link_m_n(const struct intel_link_m_n *m_n, - const struct intel_link_m_n *m2_n2, - bool exact) + const struct intel_link_m_n *m2_n2) { return m_n->tu == m2_n2->tu && - intel_compare_m_n(m_n->data_m, m_n->data_n, - m2_n2->data_m, m2_n2->data_n, exact) && - intel_compare_m_n(m_n->link_m, m_n->link_n, - m2_n2->link_m, m2_n2->link_n, exact); + m_n->data_m == m2_n2->data_m && + m_n->data_n == m2_n2->data_n && + m_n->link_m == m2_n2->link_m && + m_n->link_n == m2_n2->link_n; } static bool @@ -5600,8 +5597,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, #define PIPE_CONF_CHECK_M_N(name) do { \ if (!intel_compare_link_m_n(¤t_config->name, \ - &pipe_config->name,\ - !fastset)) { \ + &pipe_config->name)) { \ pipe_config_mismatch(fastset, crtc, __stringify(name), \ "(expected tu %i data %i/%i link %i/%i, " \ "found tu %i, data %i/%i link %i/%i)", \ @@ -5648,9 +5644,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, */ #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \ if (!intel_compare_link_m_n(¤t_config->name, \ - &pipe_config->name, !fastset) && \ + &pipe_config->name) && \ !intel_compare_link_m_n(¤t_config->alt_name, \ - &pipe_config->name, !fastset)) { \ + &pipe_config->name)) { \ pipe_config_mismatch(fastset, crtc, __stringify(name), \ "(expected tu %i data %i/%i link %i/%i, " \ "or tu %i data %i/%i link %i/%i, " \ @@ -5685,16 +5681,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, } \ } while (0) -#define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \ - if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \ - pipe_config_mismatch(fastset, crtc, __stringify(name), \ - "(expected %i, found %i)", \ - current_config->name, \ - pipe_config->name); \ - ret = false; \ - } \ -} while (0) - #define PIPE_CONF_CHECK_INFOFRAME(name) do { \ if (!intel_compare_infoframe(¤t_config->infoframes.name, \ &pipe_config->infoframes.name)) { \ @@ -5750,8 +5736,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_I(lane_count); PIPE_CONF_CHECK_X(lane_lat_optim_mask); - if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) { - PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); + if (HAS_DOUBLE_BUFFERED_M_N(dev_priv)) { + if (!fastset || !pipe_config->seamless_m_n) + PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2); } else { PIPE_CONF_CHECK_M_N(dp_m_n); PIPE_CONF_CHECK_M_N(dp_m2_n2); @@ -5813,7 +5800,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_RECT(pch_pfit.dst); PIPE_CONF_CHECK_I(scaler_state.scaler_id); - PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate); + PIPE_CONF_CHECK_I(pixel_rate); PIPE_CONF_CHECK_X(gamma_mode); if (IS_CHERRYVIEW(dev_priv)) @@ -5840,7 +5827,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_BOOL(double_wide); - if (dev_priv->dpll.mgr) { + if (dev_priv->display.dpll.mgr) { PIPE_CONF_CHECK_P(shared_dpll); PIPE_CONF_CHECK_X(dpll_hw_state.dpll); @@ -5883,9 +5870,11 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5) PIPE_CONF_CHECK_I(pipe_bpp); - PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock); - PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock); - PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock); + if (!fastset || !pipe_config->seamless_m_n) { + PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_clock); + PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_clock); + } + PIPE_CONF_CHECK_I(port_clock); PIPE_CONF_CHECK_I(min_voltage_level); @@ -5927,7 +5916,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE #undef PIPE_CONF_CHECK_P #undef PIPE_CONF_CHECK_FLAGS -#undef PIPE_CONF_CHECK_CLOCK_FUZZY #undef PIPE_CONF_CHECK_COLOR_LUT #undef PIPE_CONF_CHECK_TIMINGS #undef PIPE_CONF_CHECK_RECT @@ -6049,20 +6037,6 @@ void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state) } } -static void intel_modeset_clear_plls(struct intel_atomic_state *state) -{ - struct intel_crtc_state *new_crtc_state; - struct intel_crtc *crtc; - int i; - - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - if (!intel_crtc_needs_modeset(new_crtc_state)) - continue; - - intel_release_shared_dplls(state, crtc); - } -} - /* * This implements the workaround described in the "notes" section of the mode * set sequence documentation. When going from no pipes or single pipe to @@ -6163,23 +6137,6 @@ static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_sta new_crtc_state->update_pipe = true; } -static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) -{ - /* - * If we're not doing the full modeset we want to - * keep the current M/N values as they may be - * sufficiently different to the computed values - * to cause problems. - * - * FIXME: should really copy more fuzzy state here - */ - new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n; - new_crtc_state->dp_m_n = old_crtc_state->dp_m_n; - new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2; - new_crtc_state->has_drrs = old_crtc_state->has_drrs; -} - static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state, struct intel_crtc *crtc, u8 plane_ids_mask) @@ -6836,9 +6793,11 @@ static int intel_atomic_check(struct drm_device *dev, if (!intel_crtc_needs_modeset(new_crtc_state)) continue; - ret = intel_modeset_pipe_config_late(state, crtc); - if (ret) - goto fail; + if (new_crtc_state->hw.enable) { + ret = intel_modeset_pipe_config_late(state, crtc); + if (ret) + goto fail; + } intel_crtc_check_fastset(old_crtc_state, new_crtc_state); } @@ -6889,15 +6848,12 @@ static int intel_atomic_check(struct drm_device *dev, for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (intel_crtc_needs_modeset(new_crtc_state)) { - any_ms = true; + if (!intel_crtc_needs_modeset(new_crtc_state)) continue; - } - if (!new_crtc_state->update_pipe) - continue; + any_ms = true; - intel_crtc_copy_fastset(old_crtc_state, new_crtc_state); + intel_release_shared_dplls(state, crtc); } if (any_ms && !check_digital_port_conflicts(state)) { @@ -6938,8 +6894,6 @@ static int intel_atomic_check(struct drm_device *dev, ret = intel_modeset_calc_cdclk(state); if (ret) return ret; - - intel_modeset_clear_plls(state); } ret = intel_atomic_check_crtcs(state); @@ -7058,6 +7012,10 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state, if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) hsw_set_linetime_wm(new_crtc_state); + + if (new_crtc_state->seamless_m_n) + intel_cpu_transcoder_set_m1_n1(crtc, new_crtc_state->cpu_transcoder, + &new_crtc_state->dp_m_n); } static void commit_pipe_pre_planes(struct intel_atomic_state *state, @@ -7120,7 +7078,7 @@ static void intel_enable_crtc(struct intel_atomic_state *state, intel_crtc_update_active_timings(new_crtc_state); - dev_priv->display->crtc_enable(state, crtc); + dev_priv->display.funcs.display->crtc_enable(state, crtc); if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) return; @@ -7199,7 +7157,7 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state, */ intel_crtc_disable_pipe_crc(crtc); - dev_priv->display->crtc_disable(state, crtc); + dev_priv->display.funcs.display->crtc_disable(state, crtc); crtc->active = false; intel_fbc_disable(crtc); intel_disable_shared_dpll(old_crtc_state); @@ -7410,7 +7368,7 @@ static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv) struct intel_atomic_state *state, *next; struct llist_node *freed; - freed = llist_del_all(&dev_priv->atomic_helper.free_list); + freed = llist_del_all(&dev_priv->display.atomic_helper.free_list); llist_for_each_entry_safe(state, next, freed, freed) drm_atomic_state_put(&state->base); } @@ -7418,7 +7376,7 @@ static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv) static void intel_atomic_helper_free_state_worker(struct work_struct *work) { struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), atomic_helper.free_work); + container_of(work, typeof(*dev_priv), display.atomic_helper.free_work); intel_atomic_helper_free_state(dev_priv); } @@ -7588,7 +7546,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) } /* Now enable the clocks, plane, pipe, and connectors that we set up. */ - dev_priv->display->commit_modeset_enables(state); + dev_priv->display.funcs.display->commit_modeset_enables(state); intel_encoders_update_complete(state); @@ -7711,7 +7669,7 @@ intel_atomic_commit_ready(struct i915_sw_fence *fence, case FENCE_FREE: { struct intel_atomic_helper *helper = - &to_i915(state->base.dev)->atomic_helper; + &to_i915(state->base.dev)->display.atomic_helper; if (llist_add(&state->freed, &helper->free_list)) schedule_work(&helper->free_work); @@ -7814,12 +7772,12 @@ static int intel_atomic_commit(struct drm_device *dev, i915_sw_fence_commit(&state->commit_ready); if (nonblock && state->modeset) { - queue_work(dev_priv->modeset_wq, &state->base.commit_work); + queue_work(dev_priv->display.wq.modeset, &state->base.commit_work); } else if (nonblock) { - queue_work(dev_priv->flip_wq, &state->base.commit_work); + queue_work(dev_priv->display.wq.flip, &state->base.commit_work); } else { if (state->modeset) - flush_workqueue(dev_priv->modeset_wq); + flush_workqueue(dev_priv->display.wq.modeset); intel_atomic_commit_tail(state); } @@ -7925,7 +7883,7 @@ static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv) if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES) return false; - if (!dev_priv->vbt.int_crt_support) + if (!dev_priv->display.vbt.int_crt_support) return false; return true; @@ -8060,7 +8018,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { bool has_edp, has_port; - if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support) + if (IS_VALLEYVIEW(dev_priv) && dev_priv->display.vbt.int_crt_support) intel_crt_init(dev_priv); /* @@ -8172,6 +8130,17 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv) drm_helper_move_panel_connectors_to_head(&dev_priv->drm); } +static int max_dotclock(struct drm_i915_private *i915) +{ + int max_dotclock = i915->max_dotclk_freq; + + /* icl+ might use bigjoiner */ + if (DISPLAY_VER(i915) >= 11) + max_dotclock *= 2; + + return max_dotclock; +} + static enum drm_mode_status intel_mode_valid(struct drm_device *dev, const struct drm_display_mode *mode) @@ -8209,6 +8178,13 @@ intel_mode_valid(struct drm_device *dev, DRM_MODE_FLAG_CLKDIV2)) return MODE_BAD; + /* + * Reject clearly excessive dotclocks early to + * avoid having to worry about huge integers later. + */ + if (mode->clock > max_dotclock(dev_priv)) + return MODE_CLOCK_HIGH; + /* Transcoder timing limits */ if (DISPLAY_VER(dev_priv) >= 11) { hdisplay_max = 16384; @@ -8319,7 +8295,7 @@ static const struct drm_mode_config_funcs intel_mode_funcs = { .atomic_state_free = intel_atomic_state_free, }; -static const struct drm_i915_display_funcs skl_display_funcs = { +static const struct intel_display_funcs skl_display_funcs = { .get_pipe_config = hsw_get_pipe_config, .crtc_enable = hsw_crtc_enable, .crtc_disable = hsw_crtc_disable, @@ -8327,7 +8303,7 @@ static const struct drm_i915_display_funcs skl_display_funcs = { .get_initial_plane_config = skl_get_initial_plane_config, }; -static const struct drm_i915_display_funcs ddi_display_funcs = { +static const struct intel_display_funcs ddi_display_funcs = { .get_pipe_config = hsw_get_pipe_config, .crtc_enable = hsw_crtc_enable, .crtc_disable = hsw_crtc_disable, @@ -8335,7 +8311,7 @@ static const struct drm_i915_display_funcs ddi_display_funcs = { .get_initial_plane_config = i9xx_get_initial_plane_config, }; -static const struct drm_i915_display_funcs pch_split_display_funcs = { +static const struct intel_display_funcs pch_split_display_funcs = { .get_pipe_config = ilk_get_pipe_config, .crtc_enable = ilk_crtc_enable, .crtc_disable = ilk_crtc_disable, @@ -8343,7 +8319,7 @@ static const struct drm_i915_display_funcs pch_split_display_funcs = { .get_initial_plane_config = i9xx_get_initial_plane_config, }; -static const struct drm_i915_display_funcs vlv_display_funcs = { +static const struct intel_display_funcs vlv_display_funcs = { .get_pipe_config = i9xx_get_pipe_config, .crtc_enable = valleyview_crtc_enable, .crtc_disable = i9xx_crtc_disable, @@ -8351,7 +8327,7 @@ static const struct drm_i915_display_funcs vlv_display_funcs = { .get_initial_plane_config = i9xx_get_initial_plane_config, }; -static const struct drm_i915_display_funcs i9xx_display_funcs = { +static const struct intel_display_funcs i9xx_display_funcs = { .get_pipe_config = i9xx_get_pipe_config, .crtc_enable = i9xx_crtc_enable, .crtc_disable = i9xx_crtc_disable, @@ -8374,16 +8350,16 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv) intel_dpll_init_clock_hook(dev_priv); if (DISPLAY_VER(dev_priv) >= 9) { - dev_priv->display = &skl_display_funcs; + dev_priv->display.funcs.display = &skl_display_funcs; } else if (HAS_DDI(dev_priv)) { - dev_priv->display = &ddi_display_funcs; + dev_priv->display.funcs.display = &ddi_display_funcs; } else if (HAS_PCH_SPLIT(dev_priv)) { - dev_priv->display = &pch_split_display_funcs; + dev_priv->display.funcs.display = &pch_split_display_funcs; } else if (IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv)) { - dev_priv->display = &vlv_display_funcs; + dev_priv->display.funcs.display = &vlv_display_funcs; } else { - dev_priv->display = &i9xx_display_funcs; + dev_priv->display.funcs.display = &i9xx_display_funcs; } intel_fdi_init_hook(dev_priv); @@ -8396,11 +8372,11 @@ void intel_modeset_init_hw(struct drm_i915_private *i915) if (!HAS_DISPLAY(i915)) return; - cdclk_state = to_intel_cdclk_state(i915->cdclk.obj.state); + cdclk_state = to_intel_cdclk_state(i915->display.cdclk.obj.state); intel_update_cdclk(i915); - intel_cdclk_dump_config(i915, &i915->cdclk.hw, "Current CDCLK"); - cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw; + intel_cdclk_dump_config(i915, &i915->display.cdclk.hw, "Current CDCLK"); + cdclk_state->logical = cdclk_state->actual = i915->display.cdclk.hw; } static int sanitize_watermarks_add_affected(struct drm_atomic_state *state) @@ -8456,7 +8432,7 @@ static void sanitize_watermarks(struct drm_i915_private *dev_priv) int i; /* Only supported on platforms that use atomic watermark design */ - if (!dev_priv->wm_disp->optimize_watermarks) + if (!dev_priv->display.funcs.wm->optimize_watermarks) return; state = drm_atomic_state_alloc(&dev_priv->drm); @@ -8688,11 +8664,9 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915) intel_dmc_ucode_init(i915); - i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0); - i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI | - WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE); - - i915->window2_delay = 0; /* No DSB so no window2 delay */ + i915->display.wq.modeset = alloc_ordered_workqueue("i915_modeset", 0); + i915->display.wq.flip = alloc_workqueue("i915_flip", WQ_HIGHPRI | + WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE); intel_mode_config_init(i915); @@ -8708,8 +8682,8 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915) if (ret) goto cleanup_vga_client_pw_domain_dmc; - init_llist_head(&i915->atomic_helper.free_list); - INIT_WORK(&i915->atomic_helper.free_work, + init_llist_head(&i915->display.atomic_helper.free_list); + INIT_WORK(&i915->display.atomic_helper.free_work, intel_atomic_helper_free_state_worker); intel_init_quirks(i915); @@ -8769,7 +8743,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915) intel_hdcp_component_init(i915); - if (i915->max_cdclk_freq == 0) + if (i915->display.cdclk.max_cdclk_freq == 0) intel_update_max_cdclk(i915); /* @@ -8833,7 +8807,7 @@ int intel_modeset_init(struct drm_i915_private *i915) intel_hpd_init(i915); intel_hpd_poll_disable(i915); - intel_init_ipc(i915); + skl_watermark_ipc_init(i915); return 0; } @@ -8964,7 +8938,7 @@ void intel_display_resume(struct drm_device *dev) if (!ret) ret = __intel_display_resume(i915, state, &ctx); - intel_enable_ipc(i915); + skl_watermark_ipc_update(i915); drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); @@ -8999,11 +8973,18 @@ void intel_modeset_driver_remove(struct drm_i915_private *i915) if (!HAS_DISPLAY(i915)) return; - flush_workqueue(i915->flip_wq); - flush_workqueue(i915->modeset_wq); + flush_workqueue(i915->display.wq.flip); + flush_workqueue(i915->display.wq.modeset); + + flush_work(&i915->display.atomic_helper.free_work); + drm_WARN_ON(&i915->drm, !llist_empty(&i915->display.atomic_helper.free_list)); - flush_work(&i915->atomic_helper.free_work); - drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list)); + /* + * MST topology needs to be suspended so we don't have any calls to + * fbdev after it's finalized. MST will be destroyed later as part of + * drm_mode_config_cleanup() + */ + intel_dp_mst_suspend(i915); } /* part #2: call after irq uninstall */ @@ -9018,13 +8999,6 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915) */ intel_hpd_poll_fini(i915); - /* - * MST topology needs to be suspended so we don't have any calls to - * fbdev after it's finalized. MST will be destroyed later as part of - * drm_mode_config_cleanup() - */ - intel_dp_mst_suspend(i915); - /* poll work can call into fbdev, hence clean that up afterwards */ intel_fbdev_fini(i915); @@ -9041,8 +9015,8 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915) intel_gmbus_teardown(i915); - destroy_workqueue(i915->flip_wq); - destroy_workqueue(i915->modeset_wq); + destroy_workqueue(i915->display.wq.flip); + destroy_workqueue(i915->display.wq.modeset); intel_fbc_cleanup(i915); } diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index cef251025d7a3b6098f66be74a1cd444b492c212..884e8e67b17c72ba90c4753507e5cae9fda11589 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -375,7 +375,7 @@ enum hpd_pin { #define for_each_pipe(__dev_priv, __p) \ for ((__p) = 0; (__p) < I915_MAX_PIPES; (__p)++) \ - for_each_if(INTEL_INFO(__dev_priv)->display.pipe_mask & BIT(__p)) + for_each_if(RUNTIME_INFO(__dev_priv)->pipe_mask & BIT(__p)) #define for_each_pipe_masked(__dev_priv, __p, __mask) \ for_each_pipe(__dev_priv, __p) \ @@ -383,7 +383,7 @@ enum hpd_pin { #define for_each_cpu_transcoder(__dev_priv, __t) \ for ((__t) = 0; (__t) < I915_MAX_TRANSCODERS; (__t)++) \ - for_each_if (INTEL_INFO(__dev_priv)->display.cpu_transcoder_mask & BIT(__t)) + for_each_if (RUNTIME_INFO(__dev_priv)->cpu_transcoder_mask & BIT(__t)) #define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \ for_each_cpu_transcoder(__dev_priv, __t) \ @@ -547,7 +547,7 @@ u8 intel_calc_active_pipes(struct intel_atomic_state *state, void intel_link_compute_m_n(u16 bpp, int nlanes, int pixel_clock, int link_clock, struct intel_link_m_n *m_n, - bool constant_n, bool fec_enable); + bool fec_enable); u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv, u32 pixel_format, u64 modifier); enum drm_mode_status diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h new file mode 100644 index 0000000000000000000000000000000000000000..96cf994b0ad1fedb94adb372ab1322903e0b9010 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_display_core.h @@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_DISPLAY_CORE_H__ +#define __INTEL_DISPLAY_CORE_H__ + +#include +#include +#include +#include +#include +#include + +#include + +#include "intel_cdclk.h" +#include "intel_display.h" +#include "intel_display_power.h" +#include "intel_dmc.h" +#include "intel_dpll_mgr.h" +#include "intel_fbc.h" +#include "intel_global_state.h" +#include "intel_gmbus.h" +#include "intel_opregion.h" +#include "intel_pm_types.h" + +struct drm_i915_private; +struct drm_property; +struct i915_audio_component; +struct i915_hdcp_comp_master; +struct intel_atomic_state; +struct intel_audio_funcs; +struct intel_bios_encoder_data; +struct intel_cdclk_funcs; +struct intel_cdclk_vals; +struct intel_color_funcs; +struct intel_crtc; +struct intel_crtc_state; +struct intel_dpll_funcs; +struct intel_dpll_mgr; +struct intel_fbdev; +struct intel_fdi_funcs; +struct intel_hotplug_funcs; +struct intel_initial_plane_config; +struct intel_overlay; + +/* Amount of SAGV/QGV points, BSpec precisely defines this */ +#define I915_NUM_QGV_POINTS 8 + +/* Amount of PSF GV points, BSpec precisely defines this */ +#define I915_NUM_PSF_GV_POINTS 3 + +struct intel_display_funcs { + /* + * Returns the active state of the crtc, and if the crtc is active, + * fills out the pipe-config with the hw state. + */ + bool (*get_pipe_config)(struct intel_crtc *, + struct intel_crtc_state *); + void (*get_initial_plane_config)(struct intel_crtc *, + struct intel_initial_plane_config *); + void (*crtc_enable)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*crtc_disable)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*commit_modeset_enables)(struct intel_atomic_state *state); +}; + +/* functions used for watermark calcs for display. */ +struct intel_wm_funcs { + /* update_wm is for legacy wm management */ + void (*update_wm)(struct drm_i915_private *dev_priv); + int (*compute_pipe_wm)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + int (*compute_intermediate_wm)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*initial_watermarks)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*atomic_update_watermarks)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*optimize_watermarks)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + int (*compute_global_watermarks)(struct intel_atomic_state *state); +}; + +struct intel_audio { + /* hda/i915 audio component */ + struct i915_audio_component *component; + bool component_registered; + /* mutex for audio/video sync */ + struct mutex mutex; + int power_refcount; + u32 freq_cntrl; + + /* Used to save the pipe-to-encoder mapping for audio */ + struct intel_encoder *encoder_map[I915_MAX_PIPES]; + + /* necessary resource sharing with HDMI LPE audio driver. */ + struct { + struct platform_device *platdev; + int irq; + } lpe; +}; + +/* + * dpll and cdclk state is protected by connection_mutex dpll.lock serializes + * intel_{prepare,enable,disable}_shared_dpll. Must be global rather than per + * dpll, because on some platforms plls share registers. + */ +struct intel_dpll { + struct mutex lock; + + int num_shared_dpll; + struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; + const struct intel_dpll_mgr *mgr; + + struct { + int nssc; + int ssc; + } ref_clks; +}; + +struct intel_frontbuffer_tracking { + spinlock_t lock; + + /* + * Tracking bits for delayed frontbuffer flushing du to gpu activity or + * scheduled flips. + */ + unsigned busy_bits; + unsigned flip_bits; +}; + +struct intel_hotplug { + struct delayed_work hotplug_work; + + const u32 *hpd, *pch_hpd; + + struct { + unsigned long last_jiffies; + int count; + enum { + HPD_ENABLED = 0, + HPD_DISABLED = 1, + HPD_MARK_DISABLED = 2 + } state; + } stats[HPD_NUM_PINS]; + u32 event_bits; + u32 retry_bits; + struct delayed_work reenable_work; + + u32 long_port_mask; + u32 short_port_mask; + struct work_struct dig_port_work; + + struct work_struct poll_init_work; + bool poll_enabled; + + unsigned int hpd_storm_threshold; + /* Whether or not to count short HPD IRQs in HPD storms */ + u8 hpd_short_storm_enabled; + + /* + * if we get a HPD irq from DP and a HPD irq from non-DP + * the non-DP HPD could block the workqueue on a mode config + * mutex getting, that userspace may have taken. However + * userspace is waiting on the DP workqueue to run which is + * blocked behind the non-DP one. + */ + struct workqueue_struct *dp_wq; +}; + +struct intel_vbt_data { + /* bdb version */ + u16 version; + + /* Feature bits */ + unsigned int int_tv_support:1; + unsigned int int_crt_support:1; + unsigned int lvds_use_ssc:1; + unsigned int int_lvds_support:1; + unsigned int display_clock_mode:1; + unsigned int fdi_rx_polarity_inverted:1; + int lvds_ssc_freq; + enum drm_panel_orientation orientation; + + bool override_afc_startup; + u8 override_afc_startup_val; + + int crt_ddc_pin; + + struct list_head display_devices; + struct list_head bdb_blocks; + + struct intel_bios_encoder_data *ports[I915_MAX_PORTS]; /* Non-NULL if port present. */ + struct sdvo_device_mapping { + u8 initialized; + u8 dvo_port; + u8 slave_addr; + u8 dvo_wiring; + u8 i2c_pin; + u8 ddc_pin; + } sdvo_mappings[2]; +}; + +struct intel_wm { + /* + * Raw watermark latency values: + * in 0.1us units for WM0, + * in 0.5us units for WM1+. + */ + /* primary */ + u16 pri_latency[5]; + /* sprite */ + u16 spr_latency[5]; + /* cursor */ + u16 cur_latency[5]; + /* + * Raw watermark memory latency values + * for SKL for all 8 levels + * in 1us units. + */ + u16 skl_latency[8]; + + /* current hardware state */ + union { + struct ilk_wm_values hw; + struct vlv_wm_values vlv; + struct g4x_wm_values g4x; + }; + + u8 max_level; + + /* + * Should be held around atomic WM register writing; also + * protects * intel_crtc->wm.active and + * crtc_state->wm.need_postvbl_update. + */ + struct mutex wm_mutex; + + bool ipc_enabled; +}; + +struct intel_display { + /* Display functions */ + struct { + /* Top level crtc-ish functions */ + const struct intel_display_funcs *display; + + /* Display CDCLK functions */ + const struct intel_cdclk_funcs *cdclk; + + /* Display pll funcs */ + const struct intel_dpll_funcs *dpll; + + /* irq display functions */ + const struct intel_hotplug_funcs *hotplug; + + /* pm display functions */ + const struct intel_wm_funcs *wm; + + /* fdi display functions */ + const struct intel_fdi_funcs *fdi; + + /* Display internal color functions */ + const struct intel_color_funcs *color; + + /* Display internal audio functions */ + const struct intel_audio_funcs *audio; + } funcs; + + /* Grouping using anonymous structs. Keep sorted. */ + struct intel_atomic_helper { + struct llist_head free_list; + struct work_struct free_work; + } atomic_helper; + + struct { + /* backlight registers and fields in struct intel_panel */ + struct mutex lock; + } backlight; + + struct { + struct intel_global_obj obj; + + struct intel_bw_info { + /* for each QGV point */ + unsigned int deratedbw[I915_NUM_QGV_POINTS]; + /* for each PSF GV point */ + unsigned int psf_bw[I915_NUM_PSF_GV_POINTS]; + u8 num_qgv_points; + u8 num_psf_gv_points; + u8 num_planes; + } max[6]; + } bw; + + struct { + /* The current hardware cdclk configuration */ + struct intel_cdclk_config hw; + + /* cdclk, divider, and ratio table from bspec */ + const struct intel_cdclk_vals *table; + + struct intel_global_obj obj; + + unsigned int max_cdclk_freq; + } cdclk; + + struct { + /* The current hardware dbuf configuration */ + u8 enabled_slices; + + struct intel_global_obj obj; + } dbuf; + + struct { + /* VLV/CHV/BXT/GLK DSI MMIO register base address */ + u32 mmio_base; + } dsi; + + struct { + /* list of fbdev register on this device */ + struct intel_fbdev *fbdev; + struct work_struct suspend_work; + } fbdev; + + struct { + unsigned int pll_freq; + u32 rx_config; + } fdi; + + struct { + /* + * Base address of where the gmbus and gpio blocks are located + * (either on PCH or on SoC for platforms without PCH). + */ + u32 mmio_base; + + /* + * gmbus.mutex protects against concurrent usage of the single + * hw gmbus controller on different i2c buses. + */ + struct mutex mutex; + + struct intel_gmbus *bus[GMBUS_NUM_PINS]; + + wait_queue_head_t wait_queue; + } gmbus; + + struct { + struct i915_hdcp_comp_master *master; + bool comp_added; + + /* Mutex to protect the above hdcp component related values. */ + struct mutex comp_mutex; + } hdcp; + + struct { + struct i915_power_domains domains; + + /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */ + u32 chv_phy_control; + + /* perform PHY state sanity checks? */ + bool chv_phy_assert[2]; + } power; + + struct { + u32 mmio_base; + + /* protects panel power sequencer state */ + struct mutex mutex; + } pps; + + struct { + struct drm_property *broadcast_rgb; + struct drm_property *force_audio; + } properties; + + struct { + unsigned long mask; + } quirks; + + struct { + enum { + I915_SAGV_UNKNOWN = 0, + I915_SAGV_DISABLED, + I915_SAGV_ENABLED, + I915_SAGV_NOT_CONTROLLED + } status; + + u32 block_time_us; + } sagv; + + struct { + /* ordered wq for modesets */ + struct workqueue_struct *modeset; + + /* unbound hipri wq for page flips/plane updates */ + struct workqueue_struct *flip; + } wq; + + /* Grouping using named structs. Keep sorted. */ + struct intel_audio audio; + struct intel_dmc dmc; + struct intel_dpll dpll; + struct intel_fbc *fbc[I915_MAX_FBCS]; + struct intel_frontbuffer_tracking fb_tracking; + struct intel_hotplug hotplug; + struct intel_opregion opregion; + struct intel_overlay *overlay; + struct intel_vbt_data vbt; + struct intel_wm wm; +}; + +#endif /* __INTEL_DISPLAY_CORE_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index 6c3954479047520f587b2a6a0d8dd54cd63ec0df..7c7253a2541cff3ec9715f0ccd2a1073b7b338bb 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -26,6 +26,7 @@ #include "intel_pm.h" #include "intel_psr.h" #include "intel_sprite.h" +#include "skl_watermark.h" static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node) { @@ -37,10 +38,10 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused) struct drm_i915_private *dev_priv = node_to_i915(m->private); seq_printf(m, "FB tracking busy bits: 0x%08x\n", - dev_priv->fb_tracking.busy_bits); + dev_priv->display.fb_tracking.busy_bits); seq_printf(m, "FB tracking flip bits: 0x%08x\n", - dev_priv->fb_tracking.flip_bits); + dev_priv->display.fb_tracking.flip_bits); return 0; } @@ -103,7 +104,8 @@ static int i915_sr_status(struct seq_file *m, void *unused) static int i915_opregion(struct seq_file *m, void *unused) { - struct intel_opregion *opregion = &node_to_i915(m->private)->opregion; + struct drm_i915_private *i915 = node_to_i915(m->private); + struct intel_opregion *opregion = &i915->display.opregion; if (opregion->header) seq_write(m, opregion->header, OPREGION_SIZE); @@ -113,7 +115,8 @@ static int i915_opregion(struct seq_file *m, void *unused) static int i915_vbt(struct seq_file *m, void *unused) { - struct intel_opregion *opregion = &node_to_i915(m->private)->opregion; + struct drm_i915_private *i915 = node_to_i915(m->private); + struct intel_opregion *opregion = &i915->display.opregion; if (opregion->vbt) seq_write(m, opregion->vbt, opregion->vbt_size); @@ -129,7 +132,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) struct drm_framebuffer *drm_fb; #ifdef CONFIG_DRM_FBDEV_EMULATION - fbdev_fb = intel_fbdev_framebuffer(dev_priv->fbdev); + fbdev_fb = intel_fbdev_framebuffer(dev_priv->display.fbdev.fbdev); if (fbdev_fb) { seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ", fbdev_fb->base.width, @@ -722,10 +725,11 @@ static void intel_scaler_info(struct seq_file *m, struct intel_crtc *crtc) /* Not all platformas have a scaler */ if (num_scalers) { - seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d", + seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d scaling_filter=%d", num_scalers, crtc_state->scaler_state.scaler_users, - crtc_state->scaler_state.scaler_id); + crtc_state->scaler_state.scaler_id, + crtc_state->hw.scaling_filter); for (i = 0; i < num_scalers; i++) { const struct intel_scaler *sc = @@ -932,11 +936,11 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) drm_modeset_lock_all(dev); seq_printf(m, "PLL refclks: non-SSC: %d kHz, SSC: %d kHz\n", - dev_priv->dpll.ref_clks.nssc, - dev_priv->dpll.ref_clks.ssc); + dev_priv->display.dpll.ref_clks.nssc, + dev_priv->display.dpll.ref_clks.ssc); - for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i]; + for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) { + struct intel_shared_dpll *pll = &dev_priv->display.dpll.shared_dplls[i]; seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name, pll->info->id); @@ -979,58 +983,6 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) return 0; } -static int i915_ipc_status_show(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = m->private; - - seq_printf(m, "Isochronous Priority Control: %s\n", - str_yes_no(dev_priv->ipc_enabled)); - return 0; -} - -static int i915_ipc_status_open(struct inode *inode, struct file *file) -{ - struct drm_i915_private *dev_priv = inode->i_private; - - if (!HAS_IPC(dev_priv)) - return -ENODEV; - - return single_open(file, i915_ipc_status_show, dev_priv); -} - -static ssize_t i915_ipc_status_write(struct file *file, const char __user *ubuf, - size_t len, loff_t *offp) -{ - struct seq_file *m = file->private_data; - struct drm_i915_private *dev_priv = m->private; - intel_wakeref_t wakeref; - bool enable; - int ret; - - ret = kstrtobool_from_user(ubuf, len, &enable); - if (ret < 0) - return ret; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - if (!dev_priv->ipc_enabled && enable) - drm_info(&dev_priv->drm, - "Enabling IPC: WM will be proper only after next commit\n"); - dev_priv->ipc_enabled = enable; - intel_enable_ipc(dev_priv); - } - - return len; -} - -static const struct file_operations i915_ipc_status_fops = { - .owner = THIS_MODULE, - .open = i915_ipc_status_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - .write = i915_ipc_status_write -}; - static int i915_ddb_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -1427,9 +1379,9 @@ static int pri_wm_latency_show(struct seq_file *m, void *data) const u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.pri_latency; + latencies = dev_priv->display.wm.pri_latency; wm_latency_show(m, latencies); @@ -1442,9 +1394,9 @@ static int spr_wm_latency_show(struct seq_file *m, void *data) const u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.spr_latency; + latencies = dev_priv->display.wm.spr_latency; wm_latency_show(m, latencies); @@ -1457,9 +1409,9 @@ static int cur_wm_latency_show(struct seq_file *m, void *data) const u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.cur_latency; + latencies = dev_priv->display.wm.cur_latency; wm_latency_show(m, latencies); @@ -1550,9 +1502,9 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf, u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.pri_latency; + latencies = dev_priv->display.wm.pri_latency; return wm_latency_write(file, ubuf, len, offp, latencies); } @@ -1565,9 +1517,9 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf, u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.spr_latency; + latencies = dev_priv->display.wm.spr_latency; return wm_latency_write(file, ubuf, len, offp, latencies); } @@ -1580,9 +1532,9 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf, u16 *latencies; if (DISPLAY_VER(dev_priv) >= 9) - latencies = dev_priv->wm.skl_latency; + latencies = dev_priv->display.wm.skl_latency; else - latencies = dev_priv->wm.cur_latency; + latencies = dev_priv->display.wm.cur_latency; return wm_latency_write(file, ubuf, len, offp, latencies); } @@ -1617,14 +1569,14 @@ static const struct file_operations i915_cur_wm_latency_fops = { static int i915_hpd_storm_ctl_show(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = m->private; - struct i915_hotplug *hotplug = &dev_priv->hotplug; + struct intel_hotplug *hotplug = &dev_priv->display.hotplug; /* Synchronize with everything first in case there's been an HPD * storm, but we haven't finished handling it in the kernel yet */ intel_synchronize_irq(dev_priv); - flush_work(&dev_priv->hotplug.dig_port_work); - flush_delayed_work(&dev_priv->hotplug.hotplug_work); + flush_work(&dev_priv->display.hotplug.dig_port_work); + flush_delayed_work(&dev_priv->display.hotplug.hotplug_work); seq_printf(m, "Threshold: %d\n", hotplug->hpd_storm_threshold); seq_printf(m, "Detected: %s\n", @@ -1639,7 +1591,7 @@ static ssize_t i915_hpd_storm_ctl_write(struct file *file, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - struct i915_hotplug *hotplug = &dev_priv->hotplug; + struct intel_hotplug *hotplug = &dev_priv->display.hotplug; unsigned int new_threshold; int i; char *newline; @@ -1678,7 +1630,7 @@ static ssize_t i915_hpd_storm_ctl_write(struct file *file, spin_unlock_irq(&dev_priv->irq_lock); /* Re-enable hpd immediately if we were in an irq storm */ - flush_delayed_work(&dev_priv->hotplug.reenable_work); + flush_delayed_work(&dev_priv->display.hotplug.reenable_work); return len; } @@ -1702,7 +1654,7 @@ static int i915_hpd_short_storm_ctl_show(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = m->private; seq_printf(m, "Enabled: %s\n", - str_yes_no(dev_priv->hotplug.hpd_short_storm_enabled)); + str_yes_no(dev_priv->display.hotplug.hpd_short_storm_enabled)); return 0; } @@ -1720,7 +1672,7 @@ static ssize_t i915_hpd_short_storm_ctl_write(struct file *file, { struct seq_file *m = file->private_data; struct drm_i915_private *dev_priv = m->private; - struct i915_hotplug *hotplug = &dev_priv->hotplug; + struct intel_hotplug *hotplug = &dev_priv->display.hotplug; char *newline; char tmp[16]; int i; @@ -1756,7 +1708,7 @@ static ssize_t i915_hpd_short_storm_ctl_write(struct file *file, spin_unlock_irq(&dev_priv->irq_lock); /* Re-enable hpd immediately if we were in an irq storm */ - flush_delayed_work(&dev_priv->hotplug.reenable_work); + flush_delayed_work(&dev_priv->display.hotplug.reenable_work); return len; } @@ -1907,7 +1859,6 @@ static const struct { {"i915_dp_test_active", &i915_displayport_test_active_fops}, {"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops}, {"i915_hpd_short_storm_ctl", &i915_hpd_short_storm_ctl_fops}, - {"i915_ipc_status", &i915_ipc_status_fops}, {"i915_drrs_ctl", &i915_drrs_ctl_fops}, {"i915_edp_psr_debug", &i915_edp_psr_debug_fops}, }; @@ -1931,6 +1882,7 @@ void intel_display_debugfs_register(struct drm_i915_private *i915) intel_dmc_debugfs_register(i915); intel_fbc_debugfs_register(i915); + skl_watermark_ipc_debugfs_register(i915); } static int i915_panel_show(struct seq_file *m, void *data) @@ -2137,7 +2089,7 @@ static const struct file_operations i915_dsc_fec_support_fops = { .write = i915_dsc_fec_support_write }; -static int i915_dsc_bpp_show(struct seq_file *m, void *data) +static int i915_dsc_bpc_show(struct seq_file *m, void *data) { struct drm_connector *connector = m->private; struct drm_device *dev = connector->dev; @@ -2160,14 +2112,14 @@ static int i915_dsc_bpp_show(struct seq_file *m, void *data) } crtc_state = to_intel_crtc_state(crtc->state); - seq_printf(m, "Compressed_BPP: %d\n", crtc_state->dsc.compressed_bpp); + seq_printf(m, "Input_BPC: %d\n", crtc_state->dsc.config.bits_per_component); out: drm_modeset_unlock(&dev->mode_config.connection_mutex); return ret; } -static ssize_t i915_dsc_bpp_write(struct file *file, +static ssize_t i915_dsc_bpc_write(struct file *file, const char __user *ubuf, size_t len, loff_t *offp) { @@ -2175,33 +2127,32 @@ static ssize_t i915_dsc_bpp_write(struct file *file, ((struct seq_file *)file->private_data)->private; struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector)); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - int dsc_bpp = 0; + int dsc_bpc = 0; int ret; - ret = kstrtoint_from_user(ubuf, len, 0, &dsc_bpp); + ret = kstrtoint_from_user(ubuf, len, 0, &dsc_bpc); if (ret < 0) return ret; - intel_dp->force_dsc_bpp = dsc_bpp; + intel_dp->force_dsc_bpc = dsc_bpc; *offp += len; return len; } -static int i915_dsc_bpp_open(struct inode *inode, +static int i915_dsc_bpc_open(struct inode *inode, struct file *file) { - return single_open(file, i915_dsc_bpp_show, - inode->i_private); + return single_open(file, i915_dsc_bpc_show, inode->i_private); } -static const struct file_operations i915_dsc_bpp_fops = { +static const struct file_operations i915_dsc_bpc_fops = { .owner = THIS_MODULE, - .open = i915_dsc_bpp_open, + .open = i915_dsc_bpc_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, - .write = i915_dsc_bpp_write + .write = i915_dsc_bpc_write }; /* @@ -2271,8 +2222,8 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector) debugfs_create_file("i915_dsc_fec_support", 0644, root, connector, &i915_dsc_fec_support_fops); - debugfs_create_file("i915_dsc_bpp", 0644, root, - connector, &i915_dsc_bpp_fops); + debugfs_create_file("i915_dsc_bpc", 0644, root, + connector, &i915_dsc_bpc_fops); } if (connector->connector_type == DRM_MODE_CONNECTOR_DSI || diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 589af257edebc67a12daadde4b97817e7650cadd..1e608b9e5055932f74ff1785d8e17abeb877dbf6 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -7,6 +7,7 @@ #include "i915_drv.h" #include "i915_irq.h" +#include "intel_backlight_regs.h" #include "intel_cdclk.h" #include "intel_combo_phy.h" #include "intel_de.h" @@ -18,8 +19,8 @@ #include "intel_mchbar_regs.h" #include "intel_pch_refclk.h" #include "intel_pcode.h" -#include "intel_pm.h" #include "intel_snps_phy.h" +#include "skl_watermark.h" #include "vlv_sideband.h" #define for_each_power_domain_well(__dev_priv, __power_well, __domain) \ @@ -243,7 +244,7 @@ bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv, struct i915_power_domains *power_domains; bool ret; - power_domains = &dev_priv->power_domains; + power_domains = &dev_priv->display.power.domains; mutex_lock(&power_domains->lock); ret = __intel_display_power_is_enabled(dev_priv, domain); @@ -268,7 +269,7 @@ sanitize_target_dc_state(struct drm_i915_private *dev_priv, if (target_dc_state != states[i]) continue; - if (dev_priv->dmc.allowed_dc_mask & target_dc_state) + if (dev_priv->display.dmc.allowed_dc_mask & target_dc_state) break; target_dc_state = states[i + 1]; @@ -291,7 +292,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, { struct i915_power_well *power_well; bool dc_off_enabled; - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; mutex_lock(&power_domains->lock); power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF); @@ -301,7 +302,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, state = sanitize_target_dc_state(dev_priv, state); - if (state == dev_priv->dmc.target_dc_state) + if (state == dev_priv->display.dmc.target_dc_state) goto unlock; dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well); @@ -312,7 +313,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, if (!dc_off_enabled) intel_power_well_enable(dev_priv, power_well); - dev_priv->dmc.target_dc_state = state; + dev_priv->display.dmc.target_dc_state = state; if (!dc_off_enabled) intel_power_well_disable(dev_priv, power_well); @@ -339,7 +340,7 @@ assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains) { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); return !drm_WARN_ON(&i915->drm, bitmap_intersects(power_domains->async_put_domains[0].bits, @@ -352,7 +353,7 @@ __async_put_domains_state_ok(struct i915_power_domains *power_domains) { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); struct intel_power_domain_mask async_put_mask; enum intel_display_power_domain domain; bool err = false; @@ -375,7 +376,7 @@ static void print_power_domains(struct i915_power_domains *power_domains, { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); enum intel_display_power_domain domain; drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM)); @@ -390,7 +391,7 @@ print_async_put_domains_state(struct i915_power_domains *power_domains) { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); drm_dbg(&i915->drm, "async_put_wakeref %u\n", power_domains->async_put_wakeref); @@ -445,7 +446,7 @@ static bool intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct intel_power_domain_mask async_put_mask; bool ret = false; @@ -474,7 +475,7 @@ static void __intel_display_power_get_domain(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *power_well; if (intel_display_power_grab_async_put_ref(dev_priv, domain)) @@ -501,7 +502,7 @@ __intel_display_power_get_domain(struct drm_i915_private *dev_priv, intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); mutex_lock(&power_domains->lock); @@ -527,7 +528,7 @@ intel_wakeref_t intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; intel_wakeref_t wakeref; bool is_enabled; @@ -563,7 +564,7 @@ __intel_display_power_put_domain(struct drm_i915_private *dev_priv, const char *name = intel_display_power_domain_str(domain); struct intel_power_domain_mask async_put_mask; - power_domains = &dev_priv->power_domains; + power_domains = &dev_priv->display.power.domains; drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain], "Use count on domain %s is already zero\n", @@ -583,7 +584,7 @@ __intel_display_power_put_domain(struct drm_i915_private *dev_priv, static void __intel_display_power_put(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; mutex_lock(&power_domains->lock); __intel_display_power_put_domain(dev_priv, domain); @@ -596,7 +597,7 @@ queue_async_put_domains_work(struct i915_power_domains *power_domains, { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref); power_domains->async_put_wakeref = wakeref; drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq, @@ -610,7 +611,7 @@ release_async_put_domains(struct i915_power_domains *power_domains, { struct drm_i915_private *dev_priv = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; enum intel_display_power_domain domain; intel_wakeref_t wakeref; @@ -637,8 +638,8 @@ intel_display_power_put_async_work(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, - power_domains.async_put_work.work); - struct i915_power_domains *power_domains = &dev_priv->power_domains; + display.power.domains.async_put_work.work); + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm); intel_wakeref_t old_work_wakeref = 0; @@ -698,7 +699,7 @@ void __intel_display_power_put_async(struct drm_i915_private *i915, enum intel_display_power_domain domain, intel_wakeref_t wakeref) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; struct intel_runtime_pm *rpm = &i915->runtime_pm; intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm); @@ -746,7 +747,7 @@ out_verify: */ void intel_display_power_flush_work(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; struct intel_power_domain_mask async_put_mask; intel_wakeref_t work_wakeref; @@ -779,7 +780,7 @@ out_verify: static void intel_display_power_flush_work_sync(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; intel_display_power_flush_work(i915); cancel_delayed_work_sync(&power_domains->async_put_work); @@ -908,7 +909,7 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv, return 0; if (IS_DG2(dev_priv)) - max_dc = 0; + max_dc = 1; else if (IS_DG1(dev_priv)) max_dc = 3; else if (DISPLAY_VER(dev_priv) >= 12) @@ -976,15 +977,15 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv, */ int intel_power_domains_init(struct drm_i915_private *dev_priv) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; dev_priv->params.disable_power_well = sanitize_disable_power_well_option(dev_priv, dev_priv->params.disable_power_well); - dev_priv->dmc.allowed_dc_mask = + dev_priv->display.dmc.allowed_dc_mask = get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc); - dev_priv->dmc.target_dc_state = + dev_priv->display.dmc.target_dc_state = sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6); mutex_init(&power_domains->lock); @@ -1003,12 +1004,12 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) */ void intel_power_domains_cleanup(struct drm_i915_private *dev_priv) { - intel_display_power_map_cleanup(&dev_priv->power_domains); + intel_display_power_map_cleanup(&dev_priv->display.power.domains); } static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *power_well; mutex_lock(&power_domains->lock); @@ -1037,7 +1038,7 @@ static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv, void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv, u8 req_slices) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; u8 slice_mask = INTEL_INFO(dev_priv)->display.dbuf.slice_mask; enum dbuf_slice slice; @@ -1060,14 +1061,14 @@ void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv, for_each_dbuf_slice(dev_priv, slice) gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice)); - dev_priv->dbuf.enabled_slices = req_slices; + dev_priv->display.dbuf.enabled_slices = req_slices; mutex_unlock(&power_domains->lock); } static void gen9_dbuf_enable(struct drm_i915_private *dev_priv) { - dev_priv->dbuf.enabled_slices = + dev_priv->display.dbuf.enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv); /* @@ -1075,7 +1076,7 @@ static void gen9_dbuf_enable(struct drm_i915_private *dev_priv) * figure out later which slices we have and what we need. */ gen9_dbuf_slices_update(dev_priv, BIT(DBUF_S1) | - dev_priv->dbuf.enabled_slices); + dev_priv->display.dbuf.enabled_slices); } static void gen9_dbuf_disable(struct drm_i915_private *dev_priv) @@ -1101,7 +1102,7 @@ static void icl_mbus_init(struct drm_i915_private *dev_priv) unsigned long abox_regs = INTEL_INFO(dev_priv)->display.abox_mask; u32 mask, val, i; - if (IS_ALDERLAKE_P(dev_priv)) + if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14) return; mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK | @@ -1309,7 +1310,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); intel_update_cdclk(dev_priv); - intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK"); + intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK"); } /* @@ -1381,6 +1382,9 @@ static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv, reset_bits = RESET_PCH_HANDSHAKE_ENABLE; } + if (DISPLAY_VER(dev_priv) >= 14) + reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN; + val = intel_de_read(dev_priv, reg); if (enable) @@ -1394,7 +1398,7 @@ static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv, static void skl_display_core_init(struct drm_i915_private *dev_priv, bool resume) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); @@ -1426,13 +1430,14 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv, static void skl_display_core_uninit(struct drm_i915_private *dev_priv) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; if (!HAS_DISPLAY(dev_priv)) return; gen9_disable_dc_states(dev_priv); + /* TODO: disable DMC program */ gen9_dbuf_disable(dev_priv); @@ -1459,7 +1464,7 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv) static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); @@ -1493,13 +1498,14 @@ static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume static void bxt_display_core_uninit(struct drm_i915_private *dev_priv) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; if (!HAS_DISPLAY(dev_priv)) return; gen9_disable_dc_states(dev_priv); + /* TODO: disable DMC program */ gen9_dbuf_disable(dev_priv); @@ -1601,7 +1607,7 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) static void icl_display_core_init(struct drm_i915_private *dev_priv, bool resume) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; u32 val; @@ -1668,13 +1674,14 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv, static void icl_display_core_uninit(struct drm_i915_private *dev_priv) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; struct i915_power_well *well; if (!HAS_DISPLAY(dev_priv)) return; gen9_disable_dc_states(dev_priv); + intel_dmc_disable_program(dev_priv); /* 1. Disable all display engine functions -> aready done */ @@ -1712,7 +1719,7 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) * power well state and lane status to reconstruct the * expected initial value. */ - dev_priv->chv_phy_control = + dev_priv->display.power.chv_phy_control = PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) | PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) | PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) | @@ -1734,27 +1741,27 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) if (mask == 0xf) mask = 0x0; else - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0); - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0); mask = (status & DPLL_PORTC_READY_MASK) >> 4; if (mask == 0xf) mask = 0x0; else - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1); - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1); - dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0); + dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0); - dev_priv->chv_phy_assert[DPIO_PHY0] = false; + dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false; } else { - dev_priv->chv_phy_assert[DPIO_PHY0] = true; + dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true; } if (intel_power_well_is_enabled(dev_priv, cmn_d)) { @@ -1766,21 +1773,21 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv) if (mask == 0xf) mask = 0x0; else - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0); - dev_priv->chv_phy_control |= + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0); - dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1); + dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1); - dev_priv->chv_phy_assert[DPIO_PHY1] = false; + dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false; } else { - dev_priv->chv_phy_assert[DPIO_PHY1] = true; + dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true; } drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n", - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); /* Defer application of initial phy_control to enabling the powerwell */ } @@ -1864,7 +1871,7 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv); */ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; power_domains->initializing = true; @@ -1905,8 +1912,8 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume) /* Disable power support if the user asked so. */ if (!i915->params.disable_power_well) { drm_WARN_ON(&i915->drm, power_domains->disable_wakeref); - i915->power_domains.disable_wakeref = intel_display_power_get(i915, - POWER_DOMAIN_INIT); + i915->display.power.domains.disable_wakeref = intel_display_power_get(i915, + POWER_DOMAIN_INIT); } intel_power_domains_sync_hw(i915); @@ -1927,12 +1934,12 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume) void intel_power_domains_driver_remove(struct drm_i915_private *i915) { intel_wakeref_t wakeref __maybe_unused = - fetch_and_zero(&i915->power_domains.init_wakeref); + fetch_and_zero(&i915->display.power.domains.init_wakeref); /* Remove the refcount we took to keep power well support disabled. */ if (!i915->params.disable_power_well) intel_display_power_put(i915, POWER_DOMAIN_INIT, - fetch_and_zero(&i915->power_domains.disable_wakeref)); + fetch_and_zero(&i915->display.power.domains.disable_wakeref)); intel_display_power_flush_work_sync(i915); @@ -1954,7 +1961,7 @@ void intel_power_domains_driver_remove(struct drm_i915_private *i915) */ void intel_power_domains_sanitize_state(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; struct i915_power_well *power_well; mutex_lock(&power_domains->lock); @@ -1988,7 +1995,7 @@ void intel_power_domains_sanitize_state(struct drm_i915_private *i915) void intel_power_domains_enable(struct drm_i915_private *i915) { intel_wakeref_t wakeref __maybe_unused = - fetch_and_zero(&i915->power_domains.init_wakeref); + fetch_and_zero(&i915->display.power.domains.init_wakeref); intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref); intel_power_domains_verify_state(i915); @@ -2003,7 +2010,7 @@ void intel_power_domains_enable(struct drm_i915_private *i915) */ void intel_power_domains_disable(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; drm_WARN_ON(&i915->drm, power_domains->init_wakeref); power_domains->init_wakeref = @@ -2026,7 +2033,7 @@ void intel_power_domains_disable(struct drm_i915_private *i915) void intel_power_domains_suspend(struct drm_i915_private *i915, enum i915_drm_suspend_mode suspend_mode) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; intel_wakeref_t wakeref __maybe_unused = fetch_and_zero(&power_domains->init_wakeref); @@ -2039,7 +2046,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, * resources as required and also enable deeper system power states * that would be blocked if the firmware was inactive. */ - if (!(i915->dmc.allowed_dc_mask & DC_STATE_EN_DC9) && + if (!(i915->display.dmc.allowed_dc_mask & DC_STATE_EN_DC9) && suspend_mode == I915_DRM_SUSPEND_IDLE && intel_dmc_has_payload(i915)) { intel_display_power_flush_work(i915); @@ -2053,7 +2060,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, */ if (!i915->params.disable_power_well) intel_display_power_put(i915, POWER_DOMAIN_INIT, - fetch_and_zero(&i915->power_domains.disable_wakeref)); + fetch_and_zero(&i915->display.power.domains.disable_wakeref)); intel_display_power_flush_work(i915); intel_power_domains_verify_state(i915); @@ -2080,7 +2087,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915, */ void intel_power_domains_resume(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; if (power_domains->display_core_suspended) { intel_power_domains_init_hw(i915, true); @@ -2098,7 +2105,7 @@ void intel_power_domains_resume(struct drm_i915_private *i915) static void intel_power_domains_dump_info(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; struct i915_power_well *power_well; for_each_power_well(i915, power_well) { @@ -2126,7 +2133,7 @@ static void intel_power_domains_dump_info(struct drm_i915_private *i915) */ static void intel_power_domains_verify_state(struct drm_i915_private *i915) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; struct i915_power_well *power_well; bool dump_domain_info; @@ -2232,10 +2239,10 @@ void intel_display_power_resume(struct drm_i915_private *i915) bxt_disable_dc9(i915); icl_display_core_init(i915, true); if (intel_dmc_has_payload(i915)) { - if (i915->dmc.allowed_dc_mask & + if (i915->display.dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC6) skl_enable_dc6(i915); - else if (i915->dmc.allowed_dc_mask & + else if (i915->display.dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5) gen9_enable_dc5(i915); } @@ -2243,7 +2250,7 @@ void intel_display_power_resume(struct drm_i915_private *i915) bxt_disable_dc9(i915); bxt_display_core_init(i915, true); if (intel_dmc_has_payload(i915) && - (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) + (i915->display.dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)) gen9_enable_dc5(i915); } else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) { hsw_disable_pc8(i915); @@ -2252,7 +2259,7 @@ void intel_display_power_resume(struct drm_i915_private *i915) void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m) { - struct i915_power_domains *power_domains = &i915->power_domains; + struct i915_power_domains *power_domains = &i915->display.power.domains; int i; mutex_lock(&power_domains->lock); diff --git a/drivers/gpu/drm/i915/display/intel_display_power_map.c b/drivers/gpu/drm/i915/display/intel_display_power_map.c index 97b367f39f35d2014b6cd93754ac45c6a9681e96..dc04afc6cc8ff98ed3aed36b449f84cadfa2b86a 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_map.c +++ b/drivers/gpu/drm/i915/display/intel_display_power_map.c @@ -1350,6 +1350,117 @@ static const struct i915_power_well_desc_list xelpd_power_wells[] = { I915_PW_DESCRIPTORS(xelpd_power_wells_main), }; +/* + * MTL is based on XELPD power domains with the exception of power gating for: + * - DDI_IO (moved to PLL logic) + * - AUX and AUX_IO functionality and register access for USBC1-4 (PICA always-on) + */ +#define XELPDP_PW_2_POWER_DOMAINS \ + XELPD_PW_B_POWER_DOMAINS, \ + XELPD_PW_C_POWER_DOMAINS, \ + XELPD_PW_D_POWER_DOMAINS, \ + POWER_DOMAIN_AUDIO_PLAYBACK, \ + POWER_DOMAIN_VGA, \ + POWER_DOMAIN_PORT_DDI_LANES_TC1, \ + POWER_DOMAIN_PORT_DDI_LANES_TC2, \ + POWER_DOMAIN_PORT_DDI_LANES_TC3, \ + POWER_DOMAIN_PORT_DDI_LANES_TC4 + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_pw_2, + XELPDP_PW_2_POWER_DOMAINS, + POWER_DOMAIN_INIT); + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_dc_off, + XELPDP_PW_2_POWER_DOMAINS, + POWER_DOMAIN_AUDIO_MMIO, + POWER_DOMAIN_MODESET, + POWER_DOMAIN_AUX_A, + POWER_DOMAIN_AUX_B, + POWER_DOMAIN_INIT); + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc1, + POWER_DOMAIN_AUX_USBC1, + POWER_DOMAIN_AUX_TBT1); + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc2, + POWER_DOMAIN_AUX_USBC2, + POWER_DOMAIN_AUX_TBT2); + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc3, + POWER_DOMAIN_AUX_USBC3, + POWER_DOMAIN_AUX_TBT3); + +I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc4, + POWER_DOMAIN_AUX_USBC4, + POWER_DOMAIN_AUX_TBT4); + +static const struct i915_power_well_desc xelpdp_power_wells_main[] = { + { + .instances = &I915_PW_INSTANCES( + I915_PW("DC_off", &xelpdp_pwdoms_dc_off, + .id = SKL_DISP_DC_OFF), + ), + .ops = &gen9_dc_off_power_well_ops, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("PW_2", &xelpdp_pwdoms_pw_2, + .hsw.idx = ICL_PW_CTL_IDX_PW_2, + .id = SKL_DISP_PW_2), + ), + .ops = &hsw_power_well_ops, + .has_vga = true, + .has_fuses = true, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("PW_A", &xelpd_pwdoms_pw_a, + .hsw.idx = XELPD_PW_CTL_IDX_PW_A), + ), + .ops = &hsw_power_well_ops, + .irq_pipe_mask = BIT(PIPE_A), + .has_fuses = true, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("PW_B", &xelpd_pwdoms_pw_b, + .hsw.idx = XELPD_PW_CTL_IDX_PW_B), + ), + .ops = &hsw_power_well_ops, + .irq_pipe_mask = BIT(PIPE_B), + .has_fuses = true, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("PW_C", &xelpd_pwdoms_pw_c, + .hsw.idx = XELPD_PW_CTL_IDX_PW_C), + ), + .ops = &hsw_power_well_ops, + .irq_pipe_mask = BIT(PIPE_C), + .has_fuses = true, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("PW_D", &xelpd_pwdoms_pw_d, + .hsw.idx = XELPD_PW_CTL_IDX_PW_D), + ), + .ops = &hsw_power_well_ops, + .irq_pipe_mask = BIT(PIPE_D), + .has_fuses = true, + }, { + .instances = &I915_PW_INSTANCES( + I915_PW("AUX_A", &icl_pwdoms_aux_a, .xelpdp.aux_ch = AUX_CH_A), + I915_PW("AUX_B", &icl_pwdoms_aux_b, .xelpdp.aux_ch = AUX_CH_B), + I915_PW("AUX_TC1", &xelpdp_pwdoms_aux_tc1, .xelpdp.aux_ch = AUX_CH_USBC1), + I915_PW("AUX_TC2", &xelpdp_pwdoms_aux_tc2, .xelpdp.aux_ch = AUX_CH_USBC2), + I915_PW("AUX_TC3", &xelpdp_pwdoms_aux_tc3, .xelpdp.aux_ch = AUX_CH_USBC3), + I915_PW("AUX_TC4", &xelpdp_pwdoms_aux_tc4, .xelpdp.aux_ch = AUX_CH_USBC4), + ), + .ops = &xelpdp_aux_power_well_ops, + }, +}; + +static const struct i915_power_well_desc_list xelpdp_power_wells[] = { + I915_PW_DESCRIPTORS(i9xx_power_wells_always_on), + I915_PW_DESCRIPTORS(icl_power_wells_pw_1), + I915_PW_DESCRIPTORS(xelpdp_power_wells_main), +}; + static void init_power_well_domains(const struct i915_power_well_instance *inst, struct i915_power_well *power_well) { @@ -1388,7 +1499,7 @@ __set_power_wells(struct i915_power_domains *power_domains, { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); u64 power_well_ids = 0; const struct i915_power_well_desc_list *desc_list; const struct i915_power_well_desc *desc; @@ -1447,7 +1558,7 @@ int intel_display_power_map_init(struct i915_power_domains *power_domains) { struct drm_i915_private *i915 = container_of(power_domains, struct drm_i915_private, - power_domains); + display.power.domains); /* * The enabling order will be from lower to higher indexed wells, * the disabling order is reversed. @@ -1457,7 +1568,9 @@ int intel_display_power_map_init(struct i915_power_domains *power_domains) return 0; } - if (DISPLAY_VER(i915) >= 13) + if (DISPLAY_VER(i915) >= 14) + return set_power_wells(power_domains, xelpdp_power_wells); + else if (DISPLAY_VER(i915) >= 13) return set_power_wells(power_domains, xelpd_power_wells); else if (IS_DG1(i915)) return set_power_wells(power_domains, dg1_power_wells); diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c index 91cfd5890f468f450f44fb922565822c400d4457..df7ee4969ef174f672158c3e13c2be38aa51d7d7 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.c +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c @@ -5,6 +5,7 @@ #include "i915_drv.h" #include "i915_irq.h" +#include "intel_backlight_regs.h" #include "intel_combo_phy.h" #include "intel_combo_phy_regs.h" #include "intel_crt.h" @@ -16,10 +17,10 @@ #include "intel_dpll.h" #include "intel_hotplug.h" #include "intel_pcode.h" -#include "intel_pm.h" #include "intel_pps.h" #include "intel_tc.h" #include "intel_vga.h" +#include "skl_watermark.h" #include "vlv_sideband.h" #include "vlv_sideband_reg.h" @@ -84,7 +85,7 @@ lookup_power_well(struct drm_i915_private *i915, drm_WARN(&i915->drm, 1, "Power well %d not defined for this platform\n", power_well_id); - return &i915->power_domains.power_wells[0]; + return &i915->display.power.domains.power_wells[0]; } void intel_power_well_enable(struct drm_i915_private *i915, @@ -710,8 +711,8 @@ void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "Resetting DC state tracking from %02x to %02x\n", - dev_priv->dmc.dc_state, val); - dev_priv->dmc.dc_state = val; + dev_priv->display.dmc.dc_state, val); + dev_priv->display.dmc.dc_state = val; } /** @@ -746,8 +747,8 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) return; if (drm_WARN_ON_ONCE(&dev_priv->drm, - state & ~dev_priv->dmc.allowed_dc_mask)) - state &= dev_priv->dmc.allowed_dc_mask; + state & ~dev_priv->display.dmc.allowed_dc_mask)) + state &= dev_priv->display.dmc.allowed_dc_mask; val = intel_de_read(dev_priv, DC_STATE_EN); mask = gen9_dc_mask(dev_priv); @@ -755,16 +756,16 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) val & mask, state); /* Check if DMC is ignoring our DC state requests */ - if ((val & mask) != dev_priv->dmc.dc_state) + if ((val & mask) != dev_priv->display.dmc.dc_state) drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n", - dev_priv->dmc.dc_state, val & mask); + dev_priv->display.dmc.dc_state, val & mask); val &= ~mask; val |= state; gen9_write_dc_state(dev_priv, val); - dev_priv->dmc.dc_state = val & mask; + dev_priv->display.dmc.dc_state = val & mask; } static void tgl_enable_dc3co(struct drm_i915_private *dev_priv) @@ -945,7 +946,7 @@ static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv, static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv) { u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv); - u8 enabled_dbuf_slices = dev_priv->dbuf.enabled_slices; + u8 enabled_dbuf_slices = dev_priv->display.dbuf.enabled_slices; drm_WARN(&dev_priv->drm, hw_enabled_dbuf_slices != enabled_dbuf_slices, @@ -958,7 +959,7 @@ void gen9_disable_dc_states(struct drm_i915_private *dev_priv) { struct intel_cdclk_config cdclk_config = {}; - if (dev_priv->dmc.target_dc_state == DC_STATE_EN_DC3CO) { + if (dev_priv->display.dmc.target_dc_state == DC_STATE_EN_DC3CO) { tgl_disable_dc3co(dev_priv); return; } @@ -971,7 +972,7 @@ void gen9_disable_dc_states(struct drm_i915_private *dev_priv) intel_cdclk_get_cdclk(dev_priv, &cdclk_config); /* Can't read out voltage_level so can't use intel_cdclk_changed() */ drm_WARN_ON(&dev_priv->drm, - intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, + intel_cdclk_needs_modeset(&dev_priv->display.cdclk.hw, &cdclk_config)); gen9_assert_dbuf_enabled(dev_priv); @@ -1000,7 +1001,7 @@ static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv, if (!intel_dmc_has_payload(dev_priv)) return; - switch (dev_priv->dmc.target_dc_state) { + switch (dev_priv->display.dmc.target_dc_state) { case DC_STATE_EN_DC3CO: tgl_enable_dc3co(dev_priv); break; @@ -1156,10 +1157,10 @@ static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) * (and never recovering) in this case. intel_dsi_post_disable() will * clear it when we turn off the display. */ - val = intel_de_read(dev_priv, DSPCLK_GATE_D); + val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv)); val &= DPOUNIT_CLOCK_GATE_DISABLE; val |= VRHUNIT_CLOCK_GATE_DISABLE; - intel_de_write(dev_priv, DSPCLK_GATE_D, val); + intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val); /* * Disable trickle feed and enable pnd deadline calculation @@ -1207,7 +1208,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv) * During driver initialization/resume we can avoid restoring the * part of the HW/SW state that will be inited anyway explicitly. */ - if (dev_priv->power_domains.initializing) + if (dev_priv->display.power.domains.initializing) return; intel_hpd_init(dev_priv); @@ -1302,7 +1303,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC); struct i915_power_well *cmn_d = lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D); - u32 phy_control = dev_priv->chv_phy_control; + u32 phy_control = dev_priv->display.power.chv_phy_control; u32 phy_status = 0; u32 phy_status_mask = 0xffffffff; @@ -1313,7 +1314,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) * reset (ie. the power well has been disabled at * least once). */ - if (!dev_priv->chv_phy_assert[DPIO_PHY0]) + if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY0]) phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) | PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) | PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) | @@ -1321,7 +1322,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) | PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1)); - if (!dev_priv->chv_phy_assert[DPIO_PHY1]) + if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY1]) phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) | PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) | PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1)); @@ -1397,7 +1398,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv) drm_err(&dev_priv->drm, "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n", intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask, - phy_status, dev_priv->chv_phy_control); + phy_status, dev_priv->display.power.chv_phy_control); } #undef BITS_SET @@ -1457,13 +1458,13 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, vlv_dpio_put(dev_priv); - dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy); + dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy); intel_de_write(dev_priv, DISPLAY_PHY_CONTROL, - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); drm_dbg_kms(&dev_priv->drm, "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n", - phy, dev_priv->chv_phy_control); + phy, dev_priv->display.power.chv_phy_control); assert_chv_phy_status(dev_priv); } @@ -1487,18 +1488,18 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, assert_pll_disabled(dev_priv, PIPE_C); } - dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy); + dev_priv->display.power.chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy); intel_de_write(dev_priv, DISPLAY_PHY_CONTROL, - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); vlv_set_power_well(dev_priv, power_well, false); drm_dbg_kms(&dev_priv->drm, "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n", - phy, dev_priv->chv_phy_control); + phy, dev_priv->display.power.chv_phy_control); /* PHY is fully reset now, so we can enable the PHY state asserts */ - dev_priv->chv_phy_assert[phy] = true; + dev_priv->display.power.chv_phy_assert[phy] = true; assert_chv_phy_status(dev_priv); } @@ -1516,7 +1517,7 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi * reset (ie. the power well has been disabled at * least once). */ - if (!dev_priv->chv_phy_assert[phy]) + if (!dev_priv->display.power.chv_phy_assert[phy]) return; if (ch == DPIO_CH0) @@ -1570,27 +1571,27 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, enum dpio_channel ch, bool override) { - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; bool was_override; mutex_lock(&power_domains->lock); - was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + was_override = dev_priv->display.power.chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); if (override == was_override) goto out; if (override) - dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); else - dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); intel_de_write(dev_priv, DISPLAY_PHY_CONTROL, - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); drm_dbg_kms(&dev_priv->drm, "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n", - phy, ch, dev_priv->chv_phy_control); + phy, ch, dev_priv->display.power.chv_phy_control); assert_chv_phy_status(dev_priv); @@ -1604,26 +1605,26 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder, bool override, unsigned int mask) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct i915_power_domains *power_domains = &dev_priv->power_domains; + struct i915_power_domains *power_domains = &dev_priv->display.power.domains; enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder)); enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder)); mutex_lock(&power_domains->lock); - dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch); - dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch); + dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch); + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch); if (override) - dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); else - dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); + dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch); intel_de_write(dev_priv, DISPLAY_PHY_CONTROL, - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); drm_dbg_kms(&dev_priv->drm, "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n", - phy, ch, mask, dev_priv->chv_phy_control); + phy, ch, mask, dev_priv->display.power.chv_phy_control); assert_chv_phy_status(dev_priv); @@ -1701,7 +1702,7 @@ static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { intel_de_write(dev_priv, DISPLAY_PHY_CONTROL, - dev_priv->chv_phy_control); + dev_priv->display.power.chv_phy_control); } static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, @@ -1797,6 +1798,43 @@ tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv, return intel_power_well_refcount(power_well); } +static void xelpdp_aux_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch; + + intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch), + XELPDP_DP_AUX_CH_CTL_POWER_REQUEST, + XELPDP_DP_AUX_CH_CTL_POWER_REQUEST); + + /* + * The power status flag cannot be used to determine whether aux + * power wells have finished powering up. Instead we're + * expected to just wait a fixed 600us after raising the request + * bit. + */ + usleep_range(600, 1200); +} + +static void xelpdp_aux_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch; + + intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch), + XELPDP_DP_AUX_CH_CTL_POWER_REQUEST, + 0); + usleep_range(10, 30); +} + +static bool xelpdp_aux_power_well_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch; + + return intel_de_read(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch)) & + XELPDP_DP_AUX_CH_CTL_POWER_STATUS; +} const struct i915_power_well_ops i9xx_always_on_power_well_ops = { .sync_hw = i9xx_power_well_sync_hw_noop, @@ -1910,3 +1948,10 @@ const struct i915_power_well_ops tgl_tc_cold_off_ops = { .disable = tgl_tc_cold_off_power_well_disable, .is_enabled = tgl_tc_cold_off_power_well_is_enabled, }; + +const struct i915_power_well_ops xelpdp_aux_power_well_ops = { + .sync_hw = i9xx_power_well_sync_hw_noop, + .enable = xelpdp_aux_power_well_enable, + .disable = xelpdp_aux_power_well_disable, + .is_enabled = xelpdp_aux_power_well_enabled, +}; diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.h b/drivers/gpu/drm/i915/display/intel_display_power_well.h index d0624642dcb62500c209c17bde20912a24caf1ea..e13b521e322a99ee7693141f4cd5213d08e3500f 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.h +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.h @@ -14,15 +14,15 @@ struct drm_i915_private; struct i915_power_well; #define for_each_power_well(__dev_priv, __power_well) \ - for ((__power_well) = (__dev_priv)->power_domains.power_wells; \ - (__power_well) - (__dev_priv)->power_domains.power_wells < \ - (__dev_priv)->power_domains.power_well_count; \ + for ((__power_well) = (__dev_priv)->display.power.domains.power_wells; \ + (__power_well) - (__dev_priv)->display.power.domains.power_wells < \ + (__dev_priv)->display.power.domains.power_well_count; \ (__power_well)++) #define for_each_power_well_reverse(__dev_priv, __power_well) \ - for ((__power_well) = (__dev_priv)->power_domains.power_wells + \ - (__dev_priv)->power_domains.power_well_count - 1; \ - (__power_well) - (__dev_priv)->power_domains.power_wells >= 0; \ + for ((__power_well) = (__dev_priv)->display.power.domains.power_wells + \ + (__dev_priv)->display.power.domains.power_well_count - 1; \ + (__power_well) - (__dev_priv)->display.power.domains.power_wells >= 0; \ (__power_well)--) /* @@ -80,6 +80,9 @@ struct i915_power_well_instance { */ u8 idx; } hsw; + struct { + u8 aux_ch; + } xelpdp; }; }; @@ -169,5 +172,6 @@ extern const struct i915_power_well_ops vlv_dpio_power_well_ops; extern const struct i915_power_well_ops icl_aux_power_well_ops; extern const struct i915_power_well_ops icl_ddi_power_well_ops; extern const struct i915_power_well_ops tgl_tc_cold_off_ops; +extern const struct i915_power_well_ops xelpdp_aux_power_well_ops; #endif diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 01977cd237eb0a306707edb06e17360e80494406..298d00a11f47316d7450a90ff2d593d1cd2d0e5d 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1130,6 +1130,7 @@ struct intel_crtc_state { /* m2_n2 for eDP downclock */ struct intel_link_m_n dp_m2_n2; bool has_drrs; + bool seamless_m_n; /* PSR is supported but might not be enabled due the lack of enabled planes */ bool has_psr; @@ -1712,7 +1713,7 @@ struct intel_dp { /* Display stream compression testing */ bool force_dsc_en; - int force_dsc_bpp; + int force_dsc_bpc; bool hobl_failed; bool hobl_active; diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c index fa9ef591b8853420179d55b307a16c47711b475b..e52ecc0738a60613871e3e59067947027bd9271f 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.c +++ b/drivers/gpu/drm/i915/display/intel_dmc.c @@ -52,8 +52,8 @@ #define DISPLAY_VER12_DMC_MAX_FW_SIZE ICL_DMC_MAX_FW_SIZE -#define DG2_DMC_PATH DMC_PATH(dg2, 2, 06) -#define DG2_DMC_VERSION_REQUIRED DMC_VERSION(2, 06) +#define DG2_DMC_PATH DMC_PATH(dg2, 2, 07) +#define DG2_DMC_VERSION_REQUIRED DMC_VERSION(2, 07) MODULE_FIRMWARE(DG2_DMC_PATH); #define ADLP_DMC_PATH DMC_PATH(adlp, 2, 16) @@ -250,7 +250,7 @@ struct stepping_info { static bool has_dmc_id_fw(struct drm_i915_private *i915, int dmc_id) { - return i915->dmc.dmc_info[dmc_id].payload; + return i915->display.dmc.dmc_info[dmc_id].payload; } bool intel_dmc_has_payload(struct drm_i915_private *i915) @@ -277,6 +277,17 @@ static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv) intel_de_posting_read(dev_priv, DC_STATE_DEBUG); } +static void disable_event_handler(struct drm_i915_private *i915, + i915_reg_t ctl_reg, i915_reg_t htp_reg) +{ + intel_de_write(i915, ctl_reg, + REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK, + DMC_EVT_CTL_TYPE_EDGE_0_1) | + REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK, + DMC_EVT_CTL_EVENT_ID_FALSE)); + intel_de_write(i915, htp_reg, 0); +} + static void disable_flip_queue_event(struct drm_i915_private *i915, i915_reg_t ctl_reg, i915_reg_t htp_reg) @@ -299,12 +310,7 @@ disable_flip_queue_event(struct drm_i915_private *i915, return; } - intel_de_write(i915, ctl_reg, - REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK, - DMC_EVT_CTL_TYPE_EDGE_0_1) | - REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK, - DMC_EVT_CTL_EVENT_ID_FALSE)); - intel_de_write(i915, htp_reg, 0); + disable_event_handler(i915, ctl_reg, htp_reg); } static bool @@ -356,6 +362,51 @@ disable_all_flip_queue_events(struct drm_i915_private *i915) } } +static void disable_all_event_handlers(struct drm_i915_private *i915) +{ + int id; + + /* TODO: disable the event handlers on pre-GEN12 platforms as well */ + if (DISPLAY_VER(i915) < 12) + return; + + for (id = DMC_FW_MAIN; id < DMC_FW_MAX; id++) { + int handler; + + if (!has_dmc_id_fw(i915, id)) + continue; + + for (handler = 0; handler < DMC_EVENT_HANDLER_COUNT_GEN12; handler++) + disable_event_handler(i915, + DMC_EVT_CTL(i915, id, handler), + DMC_EVT_HTP(i915, id, handler)); + } +} + +static void pipedmc_clock_gating_wa(struct drm_i915_private *i915, bool enable) +{ + enum pipe pipe; + + if (DISPLAY_VER(i915) != 13) + return; + + /* + * Wa_16015201720:adl-p,dg2 + * The WA requires clock gating to be disabled all the time + * for pipe A and B. + * For pipe C and D clock gating needs to be disabled only + * during initializing the firmware. + */ + if (enable) + for (pipe = PIPE_A; pipe <= PIPE_D; pipe++) + intel_de_rmw(i915, CLKGATE_DIS_PSL_EXT(pipe), + 0, PIPEDMC_GATING_DIS); + else + for (pipe = PIPE_C; pipe <= PIPE_D; pipe++) + intel_de_rmw(i915, CLKGATE_DIS_PSL_EXT(pipe), + PIPEDMC_GATING_DIS, 0); +} + /** * intel_dmc_load_program() - write the firmware from memory to register. * @dev_priv: i915 drm device. @@ -366,12 +417,16 @@ disable_all_flip_queue_events(struct drm_i915_private *i915) */ void intel_dmc_load_program(struct drm_i915_private *dev_priv) { - struct intel_dmc *dmc = &dev_priv->dmc; + struct intel_dmc *dmc = &dev_priv->display.dmc; u32 id, i; if (!intel_dmc_has_payload(dev_priv)) return; + pipedmc_clock_gating_wa(dev_priv, true); + + disable_all_event_handlers(dev_priv); + assert_rpm_wakelock_held(&dev_priv->runtime_pm); preempt_disable(); @@ -393,7 +448,7 @@ void intel_dmc_load_program(struct drm_i915_private *dev_priv) } } - dev_priv->dmc.dc_state = 0; + dev_priv->display.dmc.dc_state = 0; gen9_set_dc_state_debugmask(dev_priv); @@ -403,12 +458,31 @@ void intel_dmc_load_program(struct drm_i915_private *dev_priv) * here. */ disable_all_flip_queue_events(dev_priv); + + pipedmc_clock_gating_wa(dev_priv, false); +} + +/** + * intel_dmc_disable_program() - disable the firmware + * @i915: i915 drm device + * + * Disable all event handlers in the firmware, making sure the firmware is + * inactive after the display is uninitialized. + */ +void intel_dmc_disable_program(struct drm_i915_private *i915) +{ + if (!intel_dmc_has_payload(i915)) + return; + + pipedmc_clock_gating_wa(i915, true); + disable_all_event_handlers(i915); + pipedmc_clock_gating_wa(i915, false); } void assert_dmc_loaded(struct drm_i915_private *i915) { drm_WARN_ONCE(&i915->drm, - !intel_de_read(i915, DMC_PROGRAM(i915->dmc.dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)), + !intel_de_read(i915, DMC_PROGRAM(i915->display.dmc.dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)), "DMC program storage start is NULL\n"); drm_WARN_ONCE(&i915->drm, !intel_de_read(i915, DMC_SSP_BASE), "DMC SSP Base Not fine\n"); @@ -445,7 +519,7 @@ static void dmc_set_fw_offset(struct intel_dmc *dmc, { unsigned int i, id; - struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc); for (i = 0; i < num_entries; i++) { id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id; @@ -473,7 +547,7 @@ static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc, const u32 *mmioaddr, u32 mmio_count, int header_ver, u8 dmc_id) { - struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc); u32 start_range, end_range; int i; @@ -511,7 +585,7 @@ static u32 parse_dmc_fw_header(struct intel_dmc *dmc, const struct intel_dmc_header_base *dmc_header, size_t rem_size, u8 dmc_id) { - struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc); struct dmc_fw_info *dmc_info = &dmc->dmc_info[dmc_id]; unsigned int header_len_bytes, dmc_header_size, payload_size, i; const u32 *mmioaddr, *mmiodata; @@ -622,7 +696,7 @@ parse_dmc_fw_package(struct intel_dmc *dmc, const struct stepping_info *si, size_t rem_size) { - struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc); u32 package_size = sizeof(struct intel_package_header); u32 num_entries, max_entries; const struct intel_fw_info *fw_info; @@ -676,7 +750,7 @@ static u32 parse_dmc_fw_css(struct intel_dmc *dmc, struct intel_css_header *css_header, size_t rem_size) { - struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc); + struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc); if (rem_size < sizeof(struct intel_css_header)) { drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n"); @@ -713,7 +787,7 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv, struct intel_css_header *css_header; struct intel_package_header *package_header; struct intel_dmc_header_base *dmc_header; - struct intel_dmc *dmc = &dev_priv->dmc; + struct intel_dmc *dmc = &dev_priv->display.dmc; struct stepping_info display_info = { '*', '*'}; const struct stepping_info *si = intel_get_stepping_info(dev_priv, &display_info); u32 readcount = 0; @@ -740,7 +814,7 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv, readcount += r; for (id = 0; id < DMC_FW_MAX; id++) { - if (!dev_priv->dmc.dmc_info[id].present) + if (!dev_priv->display.dmc.dmc_info[id].present) continue; offset = readcount + dmc->dmc_info[id].dmc_offset * 4; @@ -756,15 +830,15 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv, static void intel_dmc_runtime_pm_get(struct drm_i915_private *dev_priv) { - drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref); - dev_priv->dmc.wakeref = + drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref); + dev_priv->display.dmc.wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); } static void intel_dmc_runtime_pm_put(struct drm_i915_private *dev_priv) { intel_wakeref_t wakeref __maybe_unused = - fetch_and_zero(&dev_priv->dmc.wakeref); + fetch_and_zero(&dev_priv->display.dmc.wakeref); intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref); } @@ -775,10 +849,10 @@ static void dmc_load_work_fn(struct work_struct *work) struct intel_dmc *dmc; const struct firmware *fw = NULL; - dev_priv = container_of(work, typeof(*dev_priv), dmc.work); - dmc = &dev_priv->dmc; + dev_priv = container_of(work, typeof(*dev_priv), display.dmc.work); + dmc = &dev_priv->display.dmc; - request_firmware(&fw, dev_priv->dmc.fw_path, dev_priv->drm.dev); + request_firmware(&fw, dev_priv->display.dmc.fw_path, dev_priv->drm.dev); parse_dmc_fw(dev_priv, fw); if (intel_dmc_has_payload(dev_priv)) { @@ -787,7 +861,7 @@ static void dmc_load_work_fn(struct work_struct *work) drm_info(&dev_priv->drm, "Finished loading DMC firmware %s (v%u.%u)\n", - dev_priv->dmc.fw_path, DMC_VERSION_MAJOR(dmc->version), + dev_priv->display.dmc.fw_path, DMC_VERSION_MAJOR(dmc->version), DMC_VERSION_MINOR(dmc->version)); } else { drm_notice(&dev_priv->drm, @@ -810,9 +884,9 @@ static void dmc_load_work_fn(struct work_struct *work) */ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv) { - struct intel_dmc *dmc = &dev_priv->dmc; + struct intel_dmc *dmc = &dev_priv->display.dmc; - INIT_WORK(&dev_priv->dmc.work, dmc_load_work_fn); + INIT_WORK(&dev_priv->display.dmc.work, dmc_load_work_fn); if (!HAS_DMC(dev_priv)) return; @@ -895,7 +969,7 @@ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv) } drm_dbg_kms(&dev_priv->drm, "Loading %s\n", dmc->fw_path); - schedule_work(&dev_priv->dmc.work); + schedule_work(&dev_priv->display.dmc.work); } /** @@ -911,7 +985,7 @@ void intel_dmc_ucode_suspend(struct drm_i915_private *dev_priv) if (!HAS_DMC(dev_priv)) return; - flush_work(&dev_priv->dmc.work); + flush_work(&dev_priv->display.dmc.work); /* Drop the reference held in case DMC isn't loaded. */ if (!intel_dmc_has_payload(dev_priv)) @@ -953,16 +1027,16 @@ void intel_dmc_ucode_fini(struct drm_i915_private *dev_priv) return; intel_dmc_ucode_suspend(dev_priv); - drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref); + drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref); for (id = 0; id < DMC_FW_MAX; id++) - kfree(dev_priv->dmc.dmc_info[id].payload); + kfree(dev_priv->display.dmc.dmc_info[id].payload); } void intel_dmc_print_error_state(struct drm_i915_error_state_buf *m, struct drm_i915_private *i915) { - struct intel_dmc *dmc = &i915->dmc; + struct intel_dmc *dmc = &i915->display.dmc; if (!HAS_DMC(i915)) return; @@ -984,7 +1058,7 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused) if (!HAS_DMC(i915)) return -ENODEV; - dmc = &i915->dmc; + dmc = &i915->display.dmc; wakeref = intel_runtime_pm_get(&i915->runtime_pm); diff --git a/drivers/gpu/drm/i915/display/intel_dmc.h b/drivers/gpu/drm/i915/display/intel_dmc.h index 41091aee3b47b3b28726b1b48652f18589a0a37f..67e03315ef999417f908ba9057d59535897d0ba3 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc.h +++ b/drivers/gpu/drm/i915/display/intel_dmc.h @@ -47,6 +47,7 @@ struct intel_dmc { void intel_dmc_ucode_init(struct drm_i915_private *i915); void intel_dmc_load_program(struct drm_i915_private *i915); +void intel_dmc_disable_program(struct drm_i915_private *i915); void intel_dmc_ucode_fini(struct drm_i915_private *i915); void intel_dmc_ucode_suspend(struct drm_i915_private *i915); void intel_dmc_ucode_resume(struct drm_i915_private *i915); diff --git a/drivers/gpu/drm/i915/display/intel_dmc_regs.h b/drivers/gpu/drm/i915/display/intel_dmc_regs.h index 238620b559662d119d7b43e715e00c2f2dc7779c..5e5e41644ddfdb0e787ec9393672572d794da2c9 100644 --- a/drivers/gpu/drm/i915/display/intel_dmc_regs.h +++ b/drivers/gpu/drm/i915/display/intel_dmc_regs.h @@ -28,6 +28,8 @@ #define _DMC_REG(i915, dmc_id, reg) \ ((reg) - __DMC_REG_MMIO_BASE + _DMC_REG_MMIO_BASE(i915, dmc_id)) +#define DMC_EVENT_HANDLER_COUNT_GEN12 8 + #define _DMC_EVT_HTP_0 0x8f004 #define DMC_EVT_HTP(i915, dmc_id, handler) \ diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index a4e113253df3fe14148d49e107996003c2fd4595..c9be61d2348e0a12dc599691d6f781bb51801fa5 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -286,11 +286,22 @@ static int intel_dp_max_common_rate(struct intel_dp *intel_dp) return intel_dp_common_rate(intel_dp, intel_dp->num_common_rates - 1); } +static int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port) +{ + int vbt_max_lanes = intel_bios_dp_max_lane_count(&dig_port->base); + int max_lanes = dig_port->max_lanes; + + if (vbt_max_lanes) + max_lanes = min(max_lanes, vbt_max_lanes); + + return max_lanes; +} + /* Theoretical max between source and sink */ static int intel_dp_max_common_lane_count(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - int source_max = dig_port->max_lanes; + int source_max = intel_dp_max_source_lane_count(dig_port); int sink_max = intel_dp->max_sink_lane_count; int fia_max = intel_tc_port_fia_max_lane_count(dig_port); int lttpr_max = drm_dp_lttpr_max_lane_count(intel_dp->lttpr_common_caps); @@ -389,23 +400,13 @@ static int dg2_max_source_rate(struct intel_dp *intel_dp) return intel_dp_is_edp(intel_dp) ? 810000 : 1350000; } -static bool is_low_voltage_sku(struct drm_i915_private *i915, enum phy phy) -{ - u32 voltage; - - voltage = intel_de_read(i915, ICL_PORT_COMP_DW3(phy)) & VOLTAGE_INFO_MASK; - - return voltage == VOLTAGE_INFO_0_85V; -} - static int icl_max_source_rate(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); - if (intel_phy_is_combo(dev_priv, phy) && - (is_low_voltage_sku(dev_priv, phy) || !intel_dp_is_edp(intel_dp))) + if (intel_phy_is_combo(dev_priv, phy) && !intel_dp_is_edp(intel_dp)) return 540000; return 810000; @@ -413,23 +414,7 @@ static int icl_max_source_rate(struct intel_dp *intel_dp) static int ehl_max_source_rate(struct intel_dp *intel_dp) { - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - enum phy phy = intel_port_to_phy(dev_priv, dig_port->base.port); - - if (intel_dp_is_edp(intel_dp) || is_low_voltage_sku(dev_priv, phy)) - return 540000; - - return 810000; -} - -static int dg1_max_source_rate(struct intel_dp *intel_dp) -{ - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - enum phy phy = intel_port_to_phy(i915, dig_port->base.port); - - if (intel_phy_is_combo(i915, phy) && is_low_voltage_sku(i915, phy)) + if (intel_dp_is_edp(intel_dp)) return 540000; return 810000; @@ -491,7 +476,7 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) max_rate = dg2_max_source_rate(intel_dp); else if (IS_ALDERLAKE_P(dev_priv) || IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) - max_rate = dg1_max_source_rate(intel_dp); + max_rate = 810000; else if (IS_JSL_EHL(dev_priv)) max_rate = ehl_max_source_rate(intel_dp); else @@ -720,7 +705,7 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, if (bigjoiner) { u32 max_bpp_bigjoiner = - i915->max_cdclk_freq * 48 / + i915->display.cdclk.max_cdclk_freq * 48 / intel_dp_mode_to_fec_clock(mode_clock); bits_per_pixel = min(bits_per_pixel, max_bpp_bigjoiner); @@ -1312,21 +1297,45 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, } } +static bool has_seamless_m_n(struct intel_connector *connector) +{ + struct drm_i915_private *i915 = to_i915(connector->base.dev); + + /* + * Seamless M/N reprogramming only implemented + * for BDW+ double buffered M/N registers so far. + */ + return HAS_DOUBLE_BUFFERED_M_N(i915) && + intel_panel_drrs_type(connector) == DRRS_TYPE_SEAMLESS; +} + +static int intel_dp_mode_clock(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct intel_connector *connector = to_intel_connector(conn_state->connector); + const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; + + /* FIXME a bit of a mess wrt clock vs. crtc_clock */ + if (has_seamless_m_n(connector)) + return intel_panel_highest_mode(connector, adjusted_mode)->clock; + else + return adjusted_mode->crtc_clock; +} + /* Optimize link config in order: max bpp, min clock, min lanes */ static int intel_dp_compute_link_config_wide(struct intel_dp *intel_dp, struct intel_crtc_state *pipe_config, + const struct drm_connector_state *conn_state, const struct link_config_limits *limits) { - struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - int bpp, i, lane_count; + int bpp, i, lane_count, clock = intel_dp_mode_clock(pipe_config, conn_state); int mode_rate, link_rate, link_avail; for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) { int output_bpp = intel_dp_output_bpp(pipe_config->output_format, bpp); - mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock, - output_bpp); + mode_rate = intel_dp_link_required(clock, output_bpp); for (i = 0; i < intel_dp->num_common_rates; i++) { link_rate = intel_dp_common_rate(intel_dp, i); @@ -1377,7 +1386,18 @@ static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 max_req_bpc) return 0; } -#define DSC_SUPPORTED_VERSION_MIN 1 +static int intel_dp_source_dsc_version_minor(struct intel_dp *intel_dp) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + return DISPLAY_VER(i915) >= 14 ? 2 : 1; +} + +static int intel_dp_sink_dsc_version_minor(struct intel_dp *intel_dp) +{ + return (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & DP_DSC_MINOR_MASK) >> + DP_DSC_MINOR_SHIFT; +} static int intel_dp_dsc_compute_params(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) @@ -1395,6 +1415,7 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder, * DP_DSC_RC_BUF_SIZE for this. */ vdsc_cfg->rc_model_size = DSC_RC_MODEL_SIZE_CONST; + vdsc_cfg->pic_height = crtc_state->hw.adjusted_mode.crtc_vdisplay; /* * Slice Height of 8 works for all currently available panels. So start @@ -1416,9 +1437,8 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder, (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & DP_DSC_MAJOR_MASK) >> DP_DSC_MAJOR_SHIFT; vdsc_cfg->dsc_version_minor = - min(DSC_SUPPORTED_VERSION_MIN, - (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & - DP_DSC_MINOR_MASK) >> DP_DSC_MINOR_SHIFT); + min(intel_dp_source_dsc_version_minor(intel_dp), + intel_dp_sink_dsc_version_minor(intel_dp)); vdsc_cfg->convert_rgb = intel_dp->dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] & DP_DSC_RGB; @@ -1464,6 +1484,11 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, pipe_bpp = intel_dp_dsc_compute_bpp(intel_dp, conn_state->max_requested_bpc); + if (intel_dp->force_dsc_bpc) { + pipe_bpp = intel_dp->force_dsc_bpc * 3; + drm_dbg_kms(&dev_priv->drm, "Input DSC BPP forced to %d", pipe_bpp); + } + /* Min Input BPC for ICL+ is 8 */ if (pipe_bpp < 8 * 3) { drm_dbg_kms(&dev_priv->drm, @@ -1515,28 +1540,12 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, pipe_config->dsc.slice_count = dsc_dp_slice_count; } - /* As of today we support DSC for only RGB */ - if (intel_dp->force_dsc_bpp) { - if (intel_dp->force_dsc_bpp >= 8 && - intel_dp->force_dsc_bpp < pipe_bpp) { - drm_dbg_kms(&dev_priv->drm, - "DSC BPP forced to %d", - intel_dp->force_dsc_bpp); - pipe_config->dsc.compressed_bpp = - intel_dp->force_dsc_bpp; - } else { - drm_dbg_kms(&dev_priv->drm, - "Invalid DSC BPP %d", - intel_dp->force_dsc_bpp); - } - } - /* * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate * is greater than the maximum Cdclock and if slice count is even * then we need to use 2 VDSC instances. */ - if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq || + if (adjusted_mode->crtc_clock > dev_priv->display.cdclk.max_cdclk_freq || pipe_config->bigjoiner_pipes) { if (pipe_config->dsc.slice_count < 2) { drm_dbg_kms(&dev_priv->drm, @@ -1626,7 +1635,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, * Optimize for slow and wide for everything, because there are some * eDP 1.3 and 1.4 panels don't work well with fast and narrow. */ - ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); + ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, conn_state, &limits); if (ret || joiner_needs_dsc || intel_dp->force_dsc_en) { drm_dbg_kms(&i915->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n", @@ -1869,8 +1878,7 @@ intel_dp_compute_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, static bool cpu_transcoder_has_drrs(struct drm_i915_private *i915, enum transcoder cpu_transcoder) { - /* M1/N1 is double buffered */ - if (DISPLAY_VER(i915) >= 9 || IS_BROADWELL(i915)) + if (HAS_DOUBLE_BUFFERED_M_N(i915)) return true; return intel_cpu_transcoder_has_m2_n2(i915, cpu_transcoder); @@ -1908,13 +1916,16 @@ static bool can_enable_drrs(struct intel_connector *connector, static void intel_dp_drrs_compute_config(struct intel_connector *connector, struct intel_crtc_state *pipe_config, - int output_bpp, bool constant_n) + int output_bpp) { struct drm_i915_private *i915 = to_i915(connector->base.dev); const struct drm_display_mode *downclock_mode = intel_panel_downclock_mode(connector, &pipe_config->hw.adjusted_mode); int pixel_clock; + if (has_seamless_m_n(connector)) + pipe_config->seamless_m_n = true; + if (!can_enable_drrs(connector, pipe_config, downclock_mode)) { if (intel_cpu_transcoder_has_m2_n2(i915, pipe_config->cpu_transcoder)) intel_zero_m_n(&pipe_config->dp_m2_n2); @@ -1932,7 +1943,7 @@ intel_dp_drrs_compute_config(struct intel_connector *connector, intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock, pipe_config->port_clock, &pipe_config->dp_m2_n2, - constant_n, pipe_config->fec_enable); + pipe_config->fec_enable); /* FIXME: abstract this better */ if (pipe_config->splitter.enable) @@ -2007,7 +2018,6 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); const struct drm_display_mode *fixed_mode; struct intel_connector *connector = intel_dp->attached_connector; - bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N); int ret = 0, output_bpp; if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && encoder->port != PORT_A) @@ -2086,7 +2096,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, adjusted_mode->crtc_clock, pipe_config->port_clock, &pipe_config->dp_m_n, - constant_n, pipe_config->fec_enable); + pipe_config->fec_enable); /* FIXME: abstract this better */ if (pipe_config->splitter.enable) @@ -2097,8 +2107,7 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_vrr_compute_config(pipe_config, conn_state); intel_psr_compute_config(intel_dp, pipe_config, conn_state); - intel_dp_drrs_compute_config(connector, pipe_config, - output_bpp, constant_n); + intel_dp_drrs_compute_config(connector, pipe_config, output_bpp); intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state); intel_dp_compute_hdr_metadata_infoframe_sdp(intel_dp, pipe_config, conn_state); @@ -5032,9 +5041,9 @@ static void intel_dp_oob_hotplug_event(struct drm_connector *connector) struct drm_i915_private *i915 = to_i915(connector->dev); spin_lock_irq(&i915->irq_lock); - i915->hotplug.event_bits |= BIT(encoder->hpd_pin); + i915->display.hotplug.event_bits |= BIT(encoder->hpd_pin); spin_unlock_irq(&i915->irq_lock); - queue_delayed_work(system_wq, &i915->hotplug.hotplug_work, 0); + queue_delayed_work(system_wq, &i915->display.hotplug.hotplug_work, 0); } static const struct drm_connector_funcs intel_dp_connector_funcs = { @@ -5192,7 +5201,7 @@ intel_edp_add_properties(struct intel_dp *intel_dp) return; drm_connector_set_panel_orientation_with_quirk(&connector->base, - i915->vbt.orientation, + i915->display.vbt.orientation, fixed_mode->hdisplay, fixed_mode->vdisplay); } @@ -5302,8 +5311,6 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, intel_panel_init(intel_connector); - if (!(dev_priv->quirks & QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK)) - intel_connector->panel.backlight.power = intel_pps_backlight_power; intel_backlight_setup(intel_connector, pipe); intel_edp_add_properties(intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c index 2bc11937455579096c7d327c90f2aa499c0dbb85..48c375c65a418c30cfc6c16752efbef29783fec8 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c @@ -42,7 +42,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp) bool done; #define C (((status = intel_uncore_read_notrace(&i915->uncore, ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0) - done = wait_event_timeout(i915->gmbus_wait_queue, C, + done = wait_event_timeout(i915->display.gmbus.wait_queue, C, msecs_to_jiffies_timeout(timeout_ms)); /* just trace the final value */ @@ -86,7 +86,7 @@ static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index) * divide by 2000 and use that */ if (dig_port->aux_ch == AUX_CH_A) - freq = dev_priv->cdclk.hw.cdclk; + freq = dev_priv->display.cdclk.hw.cdclk; else freq = RUNTIME_INFO(dev_priv)->rawclk_freq; return DIV_ROUND_CLOSEST(freq, 2000); @@ -150,6 +150,7 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, u32 unused) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); u32 ret; /* @@ -170,6 +171,13 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp, if (intel_tc_port_in_tbt_alt_mode(dig_port)) ret |= DP_AUX_CH_CTL_TBT_IO; + /* + * Power request bit is already set during aux power well enable. + * Preserve the bit across aux transactions. + */ + if (DISPLAY_VER(i915) >= 14) + ret |= XELPDP_DP_AUX_CH_CTL_POWER_REQUEST; + return ret; } @@ -629,6 +637,46 @@ static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index) } } +static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; + + switch (aux_ch) { + case AUX_CH_A: + case AUX_CH_B: + case AUX_CH_USBC1: + case AUX_CH_USBC2: + case AUX_CH_USBC3: + case AUX_CH_USBC4: + return XELPDP_DP_AUX_CH_CTL(aux_ch); + default: + MISSING_CASE(aux_ch); + return XELPDP_DP_AUX_CH_CTL(AUX_CH_A); + } +} + +static i915_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + enum aux_ch aux_ch = dig_port->aux_ch; + + switch (aux_ch) { + case AUX_CH_A: + case AUX_CH_B: + case AUX_CH_USBC1: + case AUX_CH_USBC2: + case AUX_CH_USBC3: + case AUX_CH_USBC4: + return XELPDP_DP_AUX_CH_DATA(aux_ch, index); + default: + MISSING_CASE(aux_ch); + return XELPDP_DP_AUX_CH_DATA(AUX_CH_A, index); + } +} + void intel_dp_aux_fini(struct intel_dp *intel_dp) { if (cpu_latency_qos_request_active(&intel_dp->pm_qos)) @@ -644,7 +692,10 @@ void intel_dp_aux_init(struct intel_dp *intel_dp) struct intel_encoder *encoder = &dig_port->base; enum aux_ch aux_ch = dig_port->aux_ch; - if (DISPLAY_VER(dev_priv) >= 12) { + if (DISPLAY_VER(dev_priv) >= 14) { + intel_dp->aux_ch_ctl_reg = xelpdp_aux_ctl_reg; + intel_dp->aux_ch_data_reg = xelpdp_aux_data_reg; + } else if (DISPLAY_VER(dev_priv) >= 12) { intel_dp->aux_ch_ctl_reg = tgl_aux_ctl_reg; intel_dp->aux_ch_data_reg = tgl_aux_data_reg; } else if (DISPLAY_VER(dev_priv) >= 9) { diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c index a7640dbcf00ec7a60a0da6f79ebda53579cf414a..88689124c013d1f84b857b3a093ae61d21423903 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c @@ -17,6 +17,7 @@ #include "intel_dp.h" #include "intel_dp_hdcp.h" #include "intel_hdcp.h" +#include "intel_hdcp_regs.h" static unsigned int transcoder_to_stream_enc_status(enum transcoder cpu_transcoder) { diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 9feaf1a589f382cb5cba18eb5441a72e06361b5e..3d3efcf02011e7d4b89ff2b82ee0cab6256a5c40 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -37,17 +37,6 @@ static void intel_dp_reset_lttpr_count(struct intel_dp *intel_dp) DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV] = 0; } -static const char *intel_dp_phy_name(enum drm_dp_phy dp_phy, - char *buf, size_t buf_size) -{ - if (dp_phy == DP_PHY_DPRX) - snprintf(buf, buf_size, "DPRX"); - else - snprintf(buf, buf_size, "LTTPR %d", dp_phy - DP_PHY_LTTPR1 + 1); - - return buf; -} - static u8 *intel_dp_lttpr_phy_caps(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy) { @@ -60,20 +49,19 @@ static void intel_dp_read_lttpr_phy_caps(struct intel_dp *intel_dp, { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; u8 *phy_caps = intel_dp_lttpr_phy_caps(intel_dp, dp_phy); - char phy_name[10]; - - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)); if (drm_dp_read_lttpr_phy_caps(&intel_dp->aux, dpcd, dp_phy, phy_caps) < 0) { drm_dbg_kms(&dp_to_i915(intel_dp)->drm, "[ENCODER:%d:%s][%s] failed to read the PHY caps\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return; } drm_dbg_kms(&dp_to_i915(intel_dp)->drm, "[ENCODER:%d:%s][%s] PHY capabilities: %*ph\n", - encoder->base.base.id, encoder->base.name, phy_name, + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy), (int)sizeof(intel_dp->lttpr_phy_caps[0]), phy_caps); } @@ -423,14 +411,13 @@ intel_dp_get_adjust_train(struct intel_dp *intel_dp, { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; struct drm_i915_private *i915 = to_i915(encoder->base.dev); - char phy_name[10]; int lane; if (intel_dp_is_uhbr(crtc_state)) { drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, " "TX FFE request: " TRAIN_REQ_FMT "\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), crtc_state->lane_count, TRAIN_REQ_TX_FFE_ARGS(link_status)); } else { @@ -438,7 +425,7 @@ intel_dp_get_adjust_train(struct intel_dp *intel_dp, "vswing request: " TRAIN_REQ_FMT ", " "pre-emphasis request: " TRAIN_REQ_FMT "\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), crtc_state->lane_count, TRAIN_REQ_VSWING_ARGS(link_status), TRAIN_REQ_PREEMPH_ARGS(link_status)); @@ -503,13 +490,12 @@ intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; struct drm_i915_private *i915 = to_i915(encoder->base.dev); u8 train_pat = intel_dp_training_pattern_symbol(dp_train_pat); - char phy_name[10]; if (train_pat != DP_TRAINING_PATTERN_DISABLE) drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Using DP training pattern TPS%c\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), dp_training_pattern_name(train_pat)); intel_dp->set_link_train(intel_dp, crtc_state, dp_train_pat); @@ -546,13 +532,12 @@ void intel_dp_set_signal_levels(struct intel_dp *intel_dp, { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; struct drm_i915_private *i915 = to_i915(encoder->base.dev); - char phy_name[10]; if (intel_dp_is_uhbr(crtc_state)) { drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, " "TX FFE presets: " TRAIN_SET_FMT "\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), crtc_state->lane_count, TRAIN_SET_TX_FFE_ARGS(intel_dp->train_set)); } else { @@ -560,7 +545,7 @@ void intel_dp_set_signal_levels(struct intel_dp *intel_dp, "vswing levels: " TRAIN_SET_FMT ", " "pre-emphasis levels: " TRAIN_SET_FMT "\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), crtc_state->lane_count, TRAIN_SET_VSWING_ARGS(intel_dp->train_set), TRAIN_SET_PREEMPH_ARGS(intel_dp->train_set)); @@ -671,6 +656,28 @@ intel_dp_prepare_link_train(struct intel_dp *intel_dp, intel_dp_compute_rate(intel_dp, crtc_state->port_clock, &link_bw, &rate_select); + /* + * WaEdpLinkRateDataReload + * + * Parade PS8461E MUX (used on varius TGL+ laptops) needs + * to snoop the link rates reported by the sink when we + * use LINK_RATE_SET in order to operate in jitter cleaning + * mode (as opposed to redriver mode). Unfortunately it + * loses track of the snooped link rates when powered down, + * so we need to make it re-snoop often. Without this high + * link rates are not stable. + */ + if (!link_bw) { + struct intel_connector *connector = intel_dp->attached_connector; + __le16 sink_rates[DP_MAX_SUPPORTED_RATES]; + + drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] Reloading eDP link rates\n", + connector->base.base.id, connector->base.name); + + drm_dp_dpcd_read(&intel_dp->aux, DP_SUPPORTED_LINK_RATES, + sink_rates, sizeof(sink_rates)); + } + if (link_bw) drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] Using LINK_BW_SET value %02x\n", @@ -732,12 +739,11 @@ intel_dp_dump_link_status(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy, { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; struct drm_i915_private *i915 = to_i915(encoder->base.dev); - char phy_name[10]; drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x\n", encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), link_status[0], link_status[1], link_status[2], link_status[3], link_status[4], link_status[5]); } @@ -757,21 +763,19 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, int voltage_tries, cr_tries, max_cr_tries; u8 link_status[DP_LINK_STATUS_SIZE]; bool max_vswing_reached = false; - char phy_name[10]; int delay_us; delay_us = drm_dp_read_clock_recovery_delay(&intel_dp->aux, intel_dp->dpcd, dp_phy, intel_dp_is_uhbr(crtc_state)); - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)); - /* clock recovery */ if (!intel_dp_reset_link_train(intel_dp, crtc_state, dp_phy, DP_TRAINING_PATTERN_1 | DP_LINK_SCRAMBLING_DISABLE)) { drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to enable link training\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return false; } @@ -795,14 +799,16 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, dp_phy, link_status) < 0) { drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to get link status\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return false; } if (drm_dp_clock_recovery_ok(link_status, crtc_state->lane_count)) { drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Clock recovery OK\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return true; } @@ -810,7 +816,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Same voltage tried 5 times\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return false; } @@ -818,7 +825,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Max Voltage Swing reached\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return false; } @@ -828,7 +836,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) { drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to update link training\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); return false; } @@ -846,7 +855,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp, intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed clock recovery %d times, giving up!\n", - encoder->base.base.id, encoder->base.name, phy_name, max_cr_tries); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy), max_cr_tries); return false; } @@ -924,15 +934,12 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, u32 training_pattern; u8 link_status[DP_LINK_STATUS_SIZE]; bool channel_eq = false; - char phy_name[10]; int delay_us; delay_us = drm_dp_read_channel_eq_delay(&intel_dp->aux, intel_dp->dpcd, dp_phy, intel_dp_is_uhbr(crtc_state)); - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)); - training_pattern = intel_dp_training_pattern(intel_dp, crtc_state, dp_phy); /* Scrambling is disabled for TPS2/3 and enabled for TPS4 */ if (training_pattern != DP_TRAINING_PATTERN_4) @@ -944,7 +951,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to start channel equalization\n", encoder->base.base.id, encoder->base.name, - phy_name); + drm_dp_phy_name(dp_phy)); return false; } @@ -955,7 +962,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, link_status) < 0) { drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to get link status\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); break; } @@ -966,7 +974,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Clock recovery check failed, cannot " "continue channel equalization\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); break; } @@ -975,7 +984,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, channel_eq = true; drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Channel EQ done. DP Training successful\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); break; } @@ -985,7 +995,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) { drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to update link training\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); break; } } @@ -995,7 +1006,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp, intel_dp_dump_link_status(intel_dp, dp_phy, link_status); drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] Channel equalization failed 5 times\n", - encoder->base.base.id, encoder->base.name, phy_name); + encoder->base.base.id, encoder->base.name, + drm_dp_phy_name(dp_phy)); } return channel_eq; @@ -1070,7 +1082,6 @@ intel_dp_link_train_phy(struct intel_dp *intel_dp, { struct intel_connector *connector = intel_dp->attached_connector; struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; - char phy_name[10]; bool ret = false; if (!intel_dp_link_training_clock_recovery(intel_dp, crtc_state, dp_phy)) @@ -1086,7 +1097,7 @@ out: "[CONNECTOR:%d:%s][ENCODER:%d:%s][%s] Link Training %s at link rate = %d, lane count = %d\n", connector->base.base.id, connector->base.name, encoder->base.base.id, encoder->base.name, - intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)), + drm_dp_phy_name(dp_phy), ret ? "passed" : "failed", crtc_state->port_clock, crtc_state->lane_count); diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index 7713c19042f34338f1b684d75802bfaf27f0c3fc..03604a37931c68c79e9ec2e776ae884efb8501f7 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -58,7 +58,6 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, struct drm_i915_private *i915 = to_i915(connector->base.dev); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; - bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N); int bpp, slots = -EINVAL; mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr); @@ -100,7 +99,7 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, adjusted_mode->crtc_clock, crtc_state->port_clock, &crtc_state->dp_m_n, - constant_n, crtc_state->fec_enable); + crtc_state->fec_enable); crtc_state->dp_m_n.tu = slots; return 0; @@ -566,7 +565,10 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state, drm_dp_add_payload_part2(&intel_dp->mst_mgr, &state->base, drm_atomic_get_mst_payload_state(mst_state, connector->port)); - if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable) + if (DISPLAY_VER(dev_priv) >= 14 && pipe_config->fec_enable) + intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(trans), 0, + FECSTALL_DIS_DPTSTREAM_DPTTG); + else if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable) intel_de_rmw(dev_priv, CHICKEN_TRANS(trans), 0, FECSTALL_DIS_DPTSTREAM_DPTTG); diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c index cc6abe761f5e3cf6d1e966b8dc78bbf1f5966c75..8732b8722ed7ab67a43b8eb9561877cc5fbaa4ec 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c @@ -484,7 +484,7 @@ void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) enum dpio_phy rcomp_phy = phy_info->rcomp_phy; bool was_enabled; - lockdep_assert_held(&dev_priv->power_domains.lock); + lockdep_assert_held(&dev_priv->display.power.domains.lock); was_enabled = true; if (rcomp_phy != -1) diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index 5262f16b45acfd5d9eef11c5c63f4b44d7eea7a5..b15ba78d64d627bb61f92e90f84e55af74f573ce 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -938,12 +938,25 @@ static int hsw_crtc_compute_clock(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); struct intel_encoder *encoder = intel_get_crtc_new_encoder(state, crtc_state); + int ret; if (DISPLAY_VER(dev_priv) < 11 && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) return 0; - return intel_compute_shared_dplls(state, crtc, encoder); + ret = intel_compute_shared_dplls(state, crtc, encoder); + if (ret) + return ret; + + /* FIXME this is a mess */ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + return 0; + + /* CRT dotclock is determined via other means */ + if (!crtc_state->has_pch_encoder) + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + + return 0; } static int hsw_crtc_get_shared_dpll(struct intel_atomic_state *state, @@ -969,8 +982,15 @@ static int dg2_crtc_compute_clock(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); struct intel_encoder *encoder = intel_get_crtc_new_encoder(state, crtc_state); + int ret; + + ret = intel_mpllb_calc_state(crtc_state, encoder); + if (ret) + return ret; + + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); - return intel_mpllb_calc_state(crtc_state, encoder); + return 0; } static bool ilk_needs_fb_cb_tune(const struct dpll *dpll, int factor) @@ -991,7 +1011,7 @@ static void ilk_update_pll_dividers(struct intel_crtc_state *crtc_state, factor = 21; if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if ((intel_panel_use_ssc(dev_priv) && - dev_priv->vbt.lvds_ssc_freq == 100000) || + dev_priv->display.vbt.lvds_ssc_freq == 100000) || (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev_priv))) factor = 25; @@ -1096,6 +1116,7 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state, intel_atomic_get_new_crtc_state(state, crtc); const struct intel_limit *limit; int refclk = 120000; + int ret; /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */ if (!crtc_state->has_pch_encoder) @@ -1105,8 +1126,8 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state, if (intel_panel_use_ssc(dev_priv)) { drm_dbg_kms(&dev_priv->drm, "using SSC reference clock of %d kHz\n", - dev_priv->vbt.lvds_ssc_freq); - refclk = dev_priv->vbt.lvds_ssc_freq; + dev_priv->display.vbt.lvds_ssc_freq); + refclk = dev_priv->display.vbt.lvds_ssc_freq; } if (intel_is_dual_link_lvds(dev_priv)) { @@ -1132,7 +1153,14 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state, ilk_compute_dpll(crtc_state, &crtc_state->dpll, &crtc_state->dpll); - return intel_compute_shared_dplls(state, crtc, NULL); + ret = intel_compute_shared_dplls(state, crtc, NULL); + if (ret) + return ret; + + crtc_state->port_clock = crtc_state->dpll.dot; + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + + return ret; } static int ilk_crtc_get_shared_dpll(struct intel_atomic_state *state, @@ -1198,6 +1226,13 @@ static int chv_crtc_compute_clock(struct intel_atomic_state *state, chv_compute_dpll(crtc_state); + /* FIXME this is a mess */ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + return 0; + + crtc_state->port_clock = crtc_state->dpll.dot; + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1217,6 +1252,13 @@ static int vlv_crtc_compute_clock(struct intel_atomic_state *state, vlv_compute_dpll(crtc_state); + /* FIXME this is a mess */ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + return 0; + + crtc_state->port_clock = crtc_state->dpll.dot; + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1231,7 +1273,7 @@ static int g4x_crtc_compute_clock(struct intel_atomic_state *state, if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { - refclk = dev_priv->vbt.lvds_ssc_freq; + refclk = dev_priv->display.vbt.lvds_ssc_freq; drm_dbg_kms(&dev_priv->drm, "using SSC reference clock of %d kHz\n", refclk); @@ -1259,6 +1301,11 @@ static int g4x_crtc_compute_clock(struct intel_atomic_state *state, i9xx_compute_dpll(crtc_state, &crtc_state->dpll, &crtc_state->dpll); + crtc_state->port_clock = crtc_state->dpll.dot; + /* FIXME this is a mess */ + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_TVOUT)) + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1273,7 +1320,7 @@ static int pnv_crtc_compute_clock(struct intel_atomic_state *state, if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { - refclk = dev_priv->vbt.lvds_ssc_freq; + refclk = dev_priv->display.vbt.lvds_ssc_freq; drm_dbg_kms(&dev_priv->drm, "using SSC reference clock of %d kHz\n", refclk); @@ -1292,6 +1339,9 @@ static int pnv_crtc_compute_clock(struct intel_atomic_state *state, i9xx_compute_dpll(crtc_state, &crtc_state->dpll, &crtc_state->dpll); + crtc_state->port_clock = crtc_state->dpll.dot; + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1306,7 +1356,7 @@ static int i9xx_crtc_compute_clock(struct intel_atomic_state *state, if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { - refclk = dev_priv->vbt.lvds_ssc_freq; + refclk = dev_priv->display.vbt.lvds_ssc_freq; drm_dbg_kms(&dev_priv->drm, "using SSC reference clock of %d kHz\n", refclk); @@ -1325,6 +1375,11 @@ static int i9xx_crtc_compute_clock(struct intel_atomic_state *state, i9xx_compute_dpll(crtc_state, &crtc_state->dpll, &crtc_state->dpll); + crtc_state->port_clock = crtc_state->dpll.dot; + /* FIXME this is a mess */ + if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_TVOUT)) + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1339,7 +1394,7 @@ static int i8xx_crtc_compute_clock(struct intel_atomic_state *state, if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) { if (intel_panel_use_ssc(dev_priv)) { - refclk = dev_priv->vbt.lvds_ssc_freq; + refclk = dev_priv->display.vbt.lvds_ssc_freq; drm_dbg_kms(&dev_priv->drm, "using SSC reference clock of %d kHz\n", refclk); @@ -1360,6 +1415,9 @@ static int i8xx_crtc_compute_clock(struct intel_atomic_state *state, i8xx_compute_dpll(crtc_state, &crtc_state->dpll, &crtc_state->dpll); + crtc_state->port_clock = crtc_state->dpll.dot; + crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state); + return 0; } @@ -1411,16 +1469,13 @@ int intel_dpll_crtc_compute_clock(struct intel_atomic_state *state, drm_WARN_ON(&i915->drm, !intel_crtc_needs_modeset(crtc_state)); - if (drm_WARN_ON(&i915->drm, crtc_state->shared_dpll)) - return 0; - memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); if (!crtc_state->hw.enable) return 0; - ret = i915->dpll_funcs->crtc_compute_clock(state, crtc); + ret = i915->display.funcs.dpll->crtc_compute_clock(state, crtc); if (ret) { drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] Couldn't calculate DPLL settings\n", crtc->base.base.id, crtc->base.name); @@ -1439,17 +1494,15 @@ int intel_dpll_crtc_get_shared_dpll(struct intel_atomic_state *state, int ret; drm_WARN_ON(&i915->drm, !intel_crtc_needs_modeset(crtc_state)); + drm_WARN_ON(&i915->drm, !crtc_state->hw.enable && crtc_state->shared_dpll); - if (drm_WARN_ON(&i915->drm, crtc_state->shared_dpll)) - return 0; - - if (!crtc_state->hw.enable) + if (!crtc_state->hw.enable || crtc_state->shared_dpll) return 0; - if (!i915->dpll_funcs->crtc_get_shared_dpll) + if (!i915->display.funcs.dpll->crtc_get_shared_dpll) return 0; - ret = i915->dpll_funcs->crtc_get_shared_dpll(state, crtc); + ret = i915->display.funcs.dpll->crtc_get_shared_dpll(state, crtc); if (ret) { drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] Couldn't get a shared DPLL\n", crtc->base.base.id, crtc->base.name); @@ -1463,23 +1516,23 @@ void intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv) { if (IS_DG2(dev_priv)) - dev_priv->dpll_funcs = &dg2_dpll_funcs; + dev_priv->display.funcs.dpll = &dg2_dpll_funcs; else if (DISPLAY_VER(dev_priv) >= 9 || HAS_DDI(dev_priv)) - dev_priv->dpll_funcs = &hsw_dpll_funcs; + dev_priv->display.funcs.dpll = &hsw_dpll_funcs; else if (HAS_PCH_SPLIT(dev_priv)) - dev_priv->dpll_funcs = &ilk_dpll_funcs; + dev_priv->display.funcs.dpll = &ilk_dpll_funcs; else if (IS_CHERRYVIEW(dev_priv)) - dev_priv->dpll_funcs = &chv_dpll_funcs; + dev_priv->display.funcs.dpll = &chv_dpll_funcs; else if (IS_VALLEYVIEW(dev_priv)) - dev_priv->dpll_funcs = &vlv_dpll_funcs; + dev_priv->display.funcs.dpll = &vlv_dpll_funcs; else if (IS_G4X(dev_priv)) - dev_priv->dpll_funcs = &g4x_dpll_funcs; + dev_priv->display.funcs.dpll = &g4x_dpll_funcs; else if (IS_PINEVIEW(dev_priv)) - dev_priv->dpll_funcs = &pnv_dpll_funcs; + dev_priv->display.funcs.dpll = &pnv_dpll_funcs; else if (DISPLAY_VER(dev_priv) != 2) - dev_priv->dpll_funcs = &i9xx_dpll_funcs; + dev_priv->display.funcs.dpll = &i9xx_dpll_funcs; else - dev_priv->dpll_funcs = &i8xx_dpll_funcs; + dev_priv->display.funcs.dpll = &i8xx_dpll_funcs; } static bool i9xx_has_pps(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index 118598c9a8092692202e18e39830c31460aec479..e5fb66a5dd0257d4bcf0c1ecc256019f2f240145 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -113,8 +113,8 @@ intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv, enum intel_dpll_id i; /* Copy shared dpll state */ - for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i]; + for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) { + struct intel_shared_dpll *pll = &dev_priv->display.dpll.shared_dplls[i]; shared_dpll[i] = pll->state; } @@ -149,7 +149,7 @@ struct intel_shared_dpll * intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv, enum intel_dpll_id id) { - return &dev_priv->dpll.shared_dplls[id]; + return &dev_priv->display.dpll.shared_dplls[id]; } /** @@ -164,11 +164,11 @@ enum intel_dpll_id intel_get_shared_dpll_id(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - long pll_idx = pll - dev_priv->dpll.shared_dplls; + long pll_idx = pll - dev_priv->display.dpll.shared_dplls; if (drm_WARN_ON(&dev_priv->drm, pll_idx < 0 || - pll_idx >= dev_priv->dpll.num_shared_dpll)) + pll_idx >= dev_priv->display.dpll.num_shared_dpll)) return -1; return pll_idx; @@ -245,7 +245,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) if (drm_WARN_ON(&dev_priv->drm, pll == NULL)) return; - mutex_lock(&dev_priv->dpll.lock); + mutex_lock(&dev_priv->display.dpll.lock); old_mask = pll->active_mask; if (drm_WARN_ON(&dev_priv->drm, !(pll->state.pipe_mask & pipe_mask)) || @@ -271,7 +271,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) pll->on = true; out: - mutex_unlock(&dev_priv->dpll.lock); + mutex_unlock(&dev_priv->display.dpll.lock); } /** @@ -294,7 +294,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) if (pll == NULL) return; - mutex_lock(&dev_priv->dpll.lock); + mutex_lock(&dev_priv->display.dpll.lock); if (drm_WARN(&dev_priv->drm, !(pll->active_mask & pipe_mask), "%s not used by [CRTC:%d:%s]\n", pll->info->name, crtc->base.base.id, crtc->base.name)) @@ -317,7 +317,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) pll->on = false; out: - mutex_unlock(&dev_priv->dpll.lock); + mutex_unlock(&dev_priv->display.dpll.lock); } static struct intel_shared_dpll * @@ -336,7 +336,7 @@ intel_find_shared_dpll(struct intel_atomic_state *state, drm_WARN_ON(&dev_priv->drm, dpll_mask & ~(BIT(I915_NUM_PLLS) - 1)); for_each_set_bit(i, &dpll_mask, I915_NUM_PLLS) { - pll = &dev_priv->dpll.shared_dplls[i]; + pll = &dev_priv->display.dpll.shared_dplls[i]; /* Only want to check enabled timings first */ if (shared_dpll[i].pipe_mask == 0) { @@ -436,9 +436,9 @@ void intel_shared_dpll_swap_state(struct intel_atomic_state *state) if (!state->dpll_set) return; - for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { + for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) { struct intel_shared_dpll *pll = - &dev_priv->dpll.shared_dplls[i]; + &dev_priv->display.dpll.shared_dplls[i]; swap(pll->state, shared_dpll[i]); } @@ -537,7 +537,7 @@ static int ibx_get_dpll(struct intel_atomic_state *state, if (HAS_PCH_IBX(dev_priv)) { /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */ i = (enum intel_dpll_id) crtc->pipe; - pll = &dev_priv->dpll.shared_dplls[i]; + pll = &dev_priv->display.dpll.shared_dplls[i]; drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] using pre-allocated %s\n", @@ -905,37 +905,6 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */, *r2_out = best.r2; } -static int -hsw_ddi_wrpll_compute_dpll(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - unsigned int p, n2, r2; - - hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p); - - crtc_state->dpll_hw_state.wrpll = - WRPLL_PLL_ENABLE | WRPLL_REF_LCPLL | - WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) | - WRPLL_DIVIDER_POST(p); - - return 0; -} - -static struct intel_shared_dpll * -hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - return intel_find_shared_dpll(state, crtc, - &crtc_state->dpll_hw_state, - BIT(DPLL_ID_WRPLL2) | - BIT(DPLL_ID_WRPLL1)); -} - static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, const struct intel_shared_dpll *pll, const struct intel_dpll_hw_state *pll_state) @@ -948,7 +917,7 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, case WRPLL_REF_SPECIAL_HSW: /* Muxed-SSC for BDW, non-SSC for non-ULT HSW. */ if (IS_HASWELL(dev_priv) && !IS_HSW_ULT(dev_priv)) { - refclk = dev_priv->dpll.ref_clks.nssc; + refclk = dev_priv->display.dpll.ref_clks.nssc; break; } fallthrough; @@ -958,7 +927,7 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, * code only cares about 5% accuracy, and spread is a max of * 0.5% downspread. */ - refclk = dev_priv->dpll.ref_clks.ssc; + refclk = dev_priv->display.dpll.ref_clks.ssc; break; case WRPLL_REF_LCPLL: refclk = 2700000; @@ -976,6 +945,41 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, return (refclk * n / 10) / (p * r) * 2; } +static int +hsw_ddi_wrpll_compute_dpll(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + unsigned int p, n2, r2; + + hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p); + + crtc_state->dpll_hw_state.wrpll = + WRPLL_PLL_ENABLE | WRPLL_REF_LCPLL | + WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) | + WRPLL_DIVIDER_POST(p); + + crtc_state->port_clock = hsw_ddi_wrpll_get_freq(i915, NULL, + &crtc_state->dpll_hw_state); + + return 0; +} + +static struct intel_shared_dpll * +hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + return intel_find_shared_dpll(state, crtc, + &crtc_state->dpll_hw_state, + BIT(DPLL_ID_WRPLL2) | + BIT(DPLL_ID_WRPLL1)); +} + static int hsw_ddi_lcpll_compute_dpll(struct intel_crtc_state *crtc_state) { @@ -1145,12 +1149,12 @@ static int hsw_get_dpll(struct intel_atomic_state *state, static void hsw_update_dpll_ref_clks(struct drm_i915_private *i915) { - i915->dpll.ref_clks.ssc = 135000; + i915->display.dpll.ref_clks.ssc = 135000; /* Non-SSC is only used on non-ULT HSW. */ if (intel_de_read(i915, FUSE_STRAP3) & HSW_REF_CLK_SELECT) - i915->dpll.ref_clks.nssc = 24000; + i915->display.dpll.ref_clks.nssc = 24000; else - i915->dpll.ref_clks.nssc = 135000; + i915->display.dpll.ref_clks.nssc = 135000; } static void hsw_dump_hw_state(struct drm_i915_private *dev_priv, @@ -1618,48 +1622,11 @@ skip_remaining_dividers: return 0; } -static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - struct skl_wrpll_params wrpll_params = {}; - u32 ctrl1, cfgcr1, cfgcr2; - int ret; - - /* - * See comment in intel_dpll_hw_state to understand why we always use 0 - * as the DPLL id in this function. - */ - ctrl1 = DPLL_CTRL1_OVERRIDE(0); - - ctrl1 |= DPLL_CTRL1_HDMI_MODE(0); - - ret = skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000, - i915->dpll.ref_clks.nssc, &wrpll_params); - if (ret) - return ret; - - cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE | - DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) | - wrpll_params.dco_integer; - - cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) | - DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) | - DPLL_CFGCR2_KDIV(wrpll_params.kdiv) | - DPLL_CFGCR2_PDIV(wrpll_params.pdiv) | - wrpll_params.central_freq; - - crtc_state->dpll_hw_state.ctrl1 = ctrl1; - crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; - crtc_state->dpll_hw_state.cfgcr2 = cfgcr2; - - return 0; -} - static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915, const struct intel_shared_dpll *pll, const struct intel_dpll_hw_state *pll_state) { - int ref_clock = i915->dpll.ref_clks.nssc; + int ref_clock = i915->display.dpll.ref_clks.nssc; u32 p0, p1, p2, dco_freq; p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK; @@ -1726,6 +1693,46 @@ static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915, return dco_freq / (p0 * p1 * p2 * 5); } +static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + struct skl_wrpll_params wrpll_params = {}; + u32 ctrl1, cfgcr1, cfgcr2; + int ret; + + /* + * See comment in intel_dpll_hw_state to understand why we always use 0 + * as the DPLL id in this function. + */ + ctrl1 = DPLL_CTRL1_OVERRIDE(0); + + ctrl1 |= DPLL_CTRL1_HDMI_MODE(0); + + ret = skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000, + i915->display.dpll.ref_clks.nssc, &wrpll_params); + if (ret) + return ret; + + cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE | + DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) | + wrpll_params.dco_integer; + + cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) | + DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) | + DPLL_CFGCR2_KDIV(wrpll_params.kdiv) | + DPLL_CFGCR2_PDIV(wrpll_params.pdiv) | + wrpll_params.central_freq; + + crtc_state->dpll_hw_state.ctrl1 = ctrl1; + crtc_state->dpll_hw_state.cfgcr1 = cfgcr1; + crtc_state->dpll_hw_state.cfgcr2 = cfgcr2; + + crtc_state->port_clock = skl_ddi_wrpll_get_freq(i915, NULL, + &crtc_state->dpll_hw_state); + + return 0; +} + static int skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) { @@ -1858,7 +1865,7 @@ static int skl_ddi_pll_get_freq(struct drm_i915_private *i915, static void skl_update_dpll_ref_clks(struct drm_i915_private *i915) { /* No SSC ref */ - i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; + i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref; } static void skl_dump_hw_state(struct drm_i915_private *dev_priv, @@ -2171,7 +2178,7 @@ static void bxt_ddi_dp_pll_dividers(struct intel_crtc_state *crtc_state, } } - chv_calc_dpll_params(i915->dpll.ref_clks.nssc, clk_div); + chv_calc_dpll_params(i915->display.dpll.ref_clks.nssc, clk_div); drm_WARN_ON(&i915->drm, clk_div->vco == 0 || clk_div->dot != crtc_state->port_clock); @@ -2245,6 +2252,23 @@ static int bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state, return 0; } +static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll, + const struct intel_dpll_hw_state *pll_state) +{ + struct dpll clock; + + clock.m1 = 2; + clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, pll_state->pll0) << 22; + if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE) + clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, pll_state->pll2); + clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, pll_state->pll1); + clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, pll_state->ebb0); + clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, pll_state->ebb0); + + return chv_calc_dpll_params(i915->display.dpll.ref_clks.nssc, &clock); +} + static int bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) { @@ -2258,28 +2282,20 @@ bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) static int bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); struct dpll clk_div = {}; + int ret; bxt_ddi_hdmi_pll_dividers(crtc_state, &clk_div); - return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div); -} - -static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915, - const struct intel_shared_dpll *pll, - const struct intel_dpll_hw_state *pll_state) -{ - struct dpll clock; + ret = bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div); + if (ret) + return ret; - clock.m1 = 2; - clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, pll_state->pll0) << 22; - if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE) - clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, pll_state->pll2); - clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, pll_state->pll1); - clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, pll_state->ebb0); - clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, pll_state->ebb0); + crtc_state->port_clock = bxt_ddi_pll_get_freq(i915, NULL, + &crtc_state->dpll_hw_state); - return chv_calc_dpll_params(i915->dpll.ref_clks.nssc, &clock); + return 0; } static int bxt_compute_dpll(struct intel_atomic_state *state, @@ -2324,8 +2340,8 @@ static int bxt_get_dpll(struct intel_atomic_state *state, static void bxt_update_dpll_ref_clks(struct drm_i915_private *i915) { - i915->dpll.ref_clks.ssc = 100000; - i915->dpll.ref_clks.nssc = 100000; + i915->display.dpll.ref_clks.ssc = 100000; + i915->display.dpll.ref_clks.nssc = 100000; /* DSI non-SSC ref 19.2MHz */ } @@ -2468,7 +2484,7 @@ ehl_combo_pll_div_frac_wa_needed(struct drm_i915_private *i915) return ((IS_PLATFORM(i915, INTEL_ELKHARTLAKE) && IS_JSL_EHL_DISPLAY_STEP(i915, STEP_B0, STEP_FOREVER)) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) && - i915->dpll.ref_clks.nssc == 38400; + i915->display.dpll.ref_clks.nssc == 38400; } struct icl_combo_pll_params { @@ -2562,7 +2578,7 @@ static int icl_calc_dp_combo_pll(struct intel_crtc_state *crtc_state, { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); const struct icl_combo_pll_params *params = - dev_priv->dpll.ref_clks.nssc == 24000 ? + dev_priv->display.dpll.ref_clks.nssc == 24000 ? icl_dp_combo_pll_24MHz_values : icl_dp_combo_pll_19_2MHz_values; int clock = crtc_state->port_clock; @@ -2585,9 +2601,9 @@ static int icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); if (DISPLAY_VER(dev_priv) >= 12) { - switch (dev_priv->dpll.ref_clks.nssc) { + switch (dev_priv->display.dpll.ref_clks.nssc) { default: - MISSING_CASE(dev_priv->dpll.ref_clks.nssc); + MISSING_CASE(dev_priv->display.dpll.ref_clks.nssc); fallthrough; case 19200: case 38400: @@ -2598,9 +2614,9 @@ static int icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, break; } } else { - switch (dev_priv->dpll.ref_clks.nssc) { + switch (dev_priv->display.dpll.ref_clks.nssc) { default: - MISSING_CASE(dev_priv->dpll.ref_clks.nssc); + MISSING_CASE(dev_priv->display.dpll.ref_clks.nssc); fallthrough; case 19200: case 38400: @@ -2630,7 +2646,7 @@ static int icl_ddi_tbt_pll_get_freq(struct drm_i915_private *i915, static int icl_wrpll_ref_clock(struct drm_i915_private *i915) { - int ref_clock = i915->dpll.ref_clks.nssc; + int ref_clock = i915->display.dpll.ref_clks.nssc; /* * For ICL+, the spec states: if reference frequency is 38.4, @@ -2769,8 +2785,8 @@ static void icl_calc_dpll_state(struct drm_i915_private *i915, else pll_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400; - if (i915->vbt.override_afc_startup) - pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->vbt.override_afc_startup_val); + if (i915->display.vbt.override_afc_startup) + pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->display.vbt.override_afc_startup_val); } static int icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc, @@ -2857,7 +2873,7 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, struct intel_dpll_hw_state *pll_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - int refclk_khz = dev_priv->dpll.ref_clks.nssc; + int refclk_khz = dev_priv->display.dpll.ref_clks.nssc; int clock = crtc_state->port_clock; u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac; u32 iref_ndiv, iref_trim, iref_pulse_w; @@ -2965,8 +2981,8 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, DKL_PLL_DIV0_PROP_COEFF(prop_coeff) | DKL_PLL_DIV0_FBPREDIV(m1div) | DKL_PLL_DIV0_FBDIV_INT(m2div_int); - if (dev_priv->vbt.override_afc_startup) { - u8 val = dev_priv->vbt.override_afc_startup_val; + if (dev_priv->display.vbt.override_afc_startup) { + u8 val = dev_priv->display.vbt.override_afc_startup_val; pll_state->mg_pll_div0 |= DKL_PLL_DIV0_AFC_STARTUP(val); } @@ -3063,7 +3079,7 @@ static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *dev_priv, u32 m1, m2_int, m2_frac, div1, div2, ref_clock; u64 tmp; - ref_clock = dev_priv->dpll.ref_clks.nssc; + ref_clock = dev_priv->display.dpll.ref_clks.nssc; if (DISPLAY_VER(dev_priv) >= 12) { m1 = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK; @@ -3197,6 +3213,12 @@ static int icl_compute_combo_phy_dpll(struct intel_atomic_state *state, icl_calc_dpll_state(dev_priv, &pll_params, &port_dpll->hw_state); + /* this is mainly for the fastset check */ + icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT); + + crtc_state->port_clock = icl_ddi_combo_pll_get_freq(dev_priv, NULL, + &port_dpll->hw_state); + return 0; } @@ -3282,6 +3304,12 @@ static int icl_compute_tc_phy_dplls(struct intel_atomic_state *state, if (ret) return ret; + /* this is mainly for the fastset check */ + icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_MG_PHY); + + crtc_state->port_clock = icl_ddi_mg_pll_get_freq(dev_priv, NULL, + &port_dpll->hw_state); + return 0; } @@ -3440,7 +3468,7 @@ static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv, hw_state->mg_pll_tdc_coldst_bias = intel_de_read(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port)); - if (dev_priv->dpll.ref_clks.nssc == 38400) { + if (dev_priv->display.dpll.ref_clks.nssc == 38400) { hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART; hw_state->mg_pll_bias_mask = 0; } else { @@ -3502,7 +3530,7 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv, hw_state->mg_pll_div0 = intel_de_read(dev_priv, DKL_PLL_DIV0(tc_port)); val = DKL_PLL_DIV0_MASK; - if (dev_priv->vbt.override_afc_startup) + if (dev_priv->display.vbt.override_afc_startup) val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; hw_state->mg_pll_div0 &= val; @@ -3566,7 +3594,7 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv, TGL_DPLL_CFGCR0(id)); hw_state->cfgcr1 = intel_de_read(dev_priv, TGL_DPLL_CFGCR1(id)); - if (dev_priv->vbt.override_afc_startup) { + if (dev_priv->display.vbt.override_afc_startup) { hw_state->div0 = intel_de_read(dev_priv, TGL_DPLL0_DIV0(id)); hw_state->div0 &= TGL_DPLL0_DIV0_AFC_STARTUP_MASK; } @@ -3638,9 +3666,9 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv, intel_de_write(dev_priv, cfgcr0_reg, hw_state->cfgcr0); intel_de_write(dev_priv, cfgcr1_reg, hw_state->cfgcr1); - drm_WARN_ON_ONCE(&dev_priv->drm, dev_priv->vbt.override_afc_startup && + drm_WARN_ON_ONCE(&dev_priv->drm, dev_priv->display.vbt.override_afc_startup && !i915_mmio_reg_valid(div0_reg)); - if (dev_priv->vbt.override_afc_startup && + if (dev_priv->display.vbt.override_afc_startup && i915_mmio_reg_valid(div0_reg)) intel_de_rmw(dev_priv, div0_reg, TGL_DPLL0_DIV0_AFC_STARTUP_MASK, hw_state->div0); @@ -3732,7 +3760,7 @@ static void dkl_pll_write(struct drm_i915_private *dev_priv, intel_de_write(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), val); val = DKL_PLL_DIV0_MASK; - if (dev_priv->vbt.override_afc_startup) + if (dev_priv->display.vbt.override_afc_startup) val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; intel_de_rmw(dev_priv, DKL_PLL_DIV0(tc_port), val, hw_state->mg_pll_div0); @@ -3967,7 +3995,7 @@ static void mg_pll_disable(struct drm_i915_private *dev_priv, static void icl_update_dpll_ref_clks(struct drm_i915_private *i915) { /* No SSC ref */ - i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; + i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref; } static void icl_dump_hw_state(struct drm_i915_private *dev_priv, @@ -4192,22 +4220,24 @@ void intel_shared_dpll_init(struct drm_i915_private *dev_priv) dpll_mgr = &pch_pll_mgr; if (!dpll_mgr) { - dev_priv->dpll.num_shared_dpll = 0; + dev_priv->display.dpll.num_shared_dpll = 0; return; } dpll_info = dpll_mgr->dpll_info; for (i = 0; dpll_info[i].name; i++) { + if (drm_WARN_ON(&dev_priv->drm, + i >= ARRAY_SIZE(dev_priv->display.dpll.shared_dplls))) + break; + drm_WARN_ON(&dev_priv->drm, i != dpll_info[i].id); - dev_priv->dpll.shared_dplls[i].info = &dpll_info[i]; + dev_priv->display.dpll.shared_dplls[i].info = &dpll_info[i]; } - dev_priv->dpll.mgr = dpll_mgr; - dev_priv->dpll.num_shared_dpll = i; - mutex_init(&dev_priv->dpll.lock); - - BUG_ON(dev_priv->dpll.num_shared_dpll > I915_NUM_PLLS); + dev_priv->display.dpll.mgr = dpll_mgr; + dev_priv->display.dpll.num_shared_dpll = i; + mutex_init(&dev_priv->display.dpll.lock); } /** @@ -4229,7 +4259,7 @@ int intel_compute_shared_dplls(struct intel_atomic_state *state, struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr; if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr)) return -EINVAL; @@ -4262,7 +4292,7 @@ int intel_reserve_shared_dplls(struct intel_atomic_state *state, struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr; if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr)) return -EINVAL; @@ -4285,7 +4315,7 @@ void intel_release_shared_dplls(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr; /* * FIXME: this function is called for every platform having a @@ -4314,7 +4344,7 @@ void intel_update_active_dpll(struct intel_atomic_state *state, struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr; if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr)) return; @@ -4385,16 +4415,16 @@ static void readout_dpll_hw_state(struct drm_i915_private *i915, void intel_dpll_update_ref_clks(struct drm_i915_private *i915) { - if (i915->dpll.mgr && i915->dpll.mgr->update_ref_clks) - i915->dpll.mgr->update_ref_clks(i915); + if (i915->display.dpll.mgr && i915->display.dpll.mgr->update_ref_clks) + i915->display.dpll.mgr->update_ref_clks(i915); } void intel_dpll_readout_hw_state(struct drm_i915_private *i915) { int i; - for (i = 0; i < i915->dpll.num_shared_dpll; i++) - readout_dpll_hw_state(i915, &i915->dpll.shared_dplls[i]); + for (i = 0; i < i915->display.dpll.num_shared_dpll; i++) + readout_dpll_hw_state(i915, &i915->display.dpll.shared_dplls[i]); } static void sanitize_dpll_state(struct drm_i915_private *i915, @@ -4420,8 +4450,8 @@ void intel_dpll_sanitize_state(struct drm_i915_private *i915) { int i; - for (i = 0; i < i915->dpll.num_shared_dpll; i++) - sanitize_dpll_state(i915, &i915->dpll.shared_dplls[i]); + for (i = 0; i < i915->display.dpll.num_shared_dpll; i++) + sanitize_dpll_state(i915, &i915->display.dpll.shared_dplls[i]); } /** @@ -4434,8 +4464,8 @@ void intel_dpll_sanitize_state(struct drm_i915_private *i915) void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { - if (dev_priv->dpll.mgr) { - dev_priv->dpll.mgr->dump_hw_state(dev_priv, hw_state); + if (dev_priv->display.dpll.mgr) { + dev_priv->display.dpll.mgr->dump_hw_state(dev_priv, hw_state); } else { /* fallback for platforms that don't use the shared dpll * infrastructure @@ -4533,7 +4563,7 @@ void intel_shared_dpll_verify_disabled(struct drm_i915_private *i915) { int i; - for (i = 0; i < i915->dpll.num_shared_dpll; i++) - verify_single_dpll_state(i915, &i915->dpll.shared_dplls[i], + for (i = 0; i < i915->display.dpll.num_shared_dpll; i++) + verify_single_dpll_state(i915, &i915->display.dpll.shared_dplls[i], NULL, NULL); } diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index c4affcb216fdd639f79fee99b096358fc40c8847..fc9c3e41c33305437dadeb89b87b0b83d3b5298e 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -9,6 +9,36 @@ #include "i915_drv.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dsb.h" + +struct i915_vma; + +enum dsb_id { + INVALID_DSB = -1, + DSB1, + DSB2, + DSB3, + MAX_DSB_PER_PIPE +}; + +struct intel_dsb { + enum dsb_id id; + u32 *cmd_buf; + struct i915_vma *vma; + + /* + * free_pos will point the first free entry position + * and help in calculating tail of command buffer. + */ + int free_pos; + + /* + * ins_start_offset will help to store start address of the dsb + * instuction and help in identifying the batch of auto-increment + * register. + */ + u32 ins_start_offset; +}; #define DSB_BUF_SIZE (2 * PAGE_SIZE) diff --git a/drivers/gpu/drm/i915/display/intel_dsb.h b/drivers/gpu/drm/i915/display/intel_dsb.h index 6cb9c580cdcaf6dcdc43190e0656008a65b915bb..74dd2b3343bb37b9342ff570003a02a918787ffd 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.h +++ b/drivers/gpu/drm/i915/display/intel_dsb.h @@ -11,34 +11,6 @@ #include "i915_reg_defs.h" struct intel_crtc_state; -struct i915_vma; - -enum dsb_id { - INVALID_DSB = -1, - DSB1, - DSB2, - DSB3, - MAX_DSB_PER_PIPE -}; - -struct intel_dsb { - enum dsb_id id; - u32 *cmd_buf; - struct i915_vma *vma; - - /* - * free_pos will point the first free entry position - * and help in calculating tail of command buffer. - */ - int free_pos; - - /* - * ins_start_offset will help to store start address of the dsb - * instuction and help in identifying the batch of auto-increment - * register. - */ - u32 ins_start_offset; -}; void intel_dsb_prepare(struct intel_crtc_state *crtc_state); void intel_dsb_cleanup(struct intel_crtc_state *crtc_state); diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c index 35e121cd226c5c06f4fac660903c8dc81e52a600..5efdd471ac2b500d597760d74e1cca66836cc61c 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.c +++ b/drivers/gpu/drm/i915/display/intel_dsi.c @@ -106,7 +106,7 @@ intel_dsi_get_panel_orientation(struct intel_connector *connector) if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) return orientation; - orientation = dev_priv->vbt.orientation; + orientation = dev_priv->display.vbt.orientation; if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) return orientation; diff --git a/drivers/gpu/drm/i915/display/intel_dsi.h b/drivers/gpu/drm/i915/display/intel_dsi.h index eafef0a87fea79385d5745a8c71f3db7ebbf7746..ce80bd8be5194c30f05a7fdd275a1a3f3f75922f 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.h +++ b/drivers/gpu/drm/i915/display/intel_dsi.h @@ -89,9 +89,6 @@ struct intel_dsi { u8 escape_clk_div; u8 dual_link; - u16 dcs_backlight_ports; - u16 dcs_cabc_ports; - /* RGB or BGR */ bool bgr_enabled; diff --git a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c index 1bc7118c56a2a58ca25e52ecc1032f85665387ed..20e466d843ce07359da03e4b10ddb8f83afe24a5 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c @@ -53,7 +53,7 @@ static u32 dcs_get_backlight(struct intel_connector *connector, enum pipe unused enum port port; size_t len = panel->backlight.max > U8_MAX ? 2 : 1; - for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + for_each_dsi_port(port, panel->vbt.dsi.bl_ports) { dsi_device = intel_dsi->dsi_hosts[port]->device; mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_DISPLAY_BRIGHTNESS, &data, len); @@ -80,7 +80,7 @@ static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32 data[1] = level; } - for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + for_each_dsi_port(port, panel->vbt.dsi.bl_ports) { dsi_device = intel_dsi->dsi_hosts[port]->device; mode_flags = dsi_device->mode_flags; dsi_device->mode_flags &= ~MIPI_DSI_MODE_LPM; @@ -93,12 +93,13 @@ static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32 static void dcs_disable_backlight(const struct drm_connector_state *conn_state, u32 level) { struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(conn_state->best_encoder)); + struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel; struct mipi_dsi_device *dsi_device; enum port port; dcs_set_backlight(conn_state, 0); - for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) { + for_each_dsi_port(port, panel->vbt.dsi.cabc_ports) { u8 cabc = POWER_SAVE_OFF; dsi_device = intel_dsi->dsi_hosts[port]->device; @@ -106,7 +107,7 @@ static void dcs_disable_backlight(const struct drm_connector_state *conn_state, &cabc, sizeof(cabc)); } - for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + for_each_dsi_port(port, panel->vbt.dsi.bl_ports) { u8 ctrl = 0; dsi_device = intel_dsi->dsi_hosts[port]->device; @@ -127,10 +128,11 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state, u32 level) { struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(conn_state->best_encoder)); + struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel; struct mipi_dsi_device *dsi_device; enum port port; - for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) { + for_each_dsi_port(port, panel->vbt.dsi.bl_ports) { u8 ctrl = 0; dsi_device = intel_dsi->dsi_hosts[port]->device; @@ -146,7 +148,7 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state, &ctrl, sizeof(ctrl)); } - for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) { + for_each_dsi_port(port, panel->vbt.dsi.cabc_ports) { u8 cabc = POWER_SAVE_MEDIUM; dsi_device = intel_dsi->dsi_hosts[port]->device; diff --git a/drivers/gpu/drm/i915/display/intel_dvo_dev.h b/drivers/gpu/drm/i915/display/intel_dvo_dev.h index d96c3cc46e50876c07982734a856cc6adfabd7f8..50205f064d932552e3dd3cfc9dedd7e0b767ffa1 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo_dev.h +++ b/drivers/gpu/drm/i915/display/intel_dvo_dev.h @@ -75,8 +75,8 @@ struct intel_dvo_dev_ops { * * \return MODE_OK if the mode is valid, or another MODE_* otherwise. */ - int (*mode_valid)(struct intel_dvo_device *dvo, - struct drm_display_mode *mode); + enum drm_mode_status (*mode_valid)(struct intel_dvo_device *dvo, + struct drm_display_mode *mode); /* * Callback for preparing mode changes on an output diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c index c86e5d4ee016fd1e5a98155169f413d136891450..1dddd6abd77b5c4dae80141e3b231dbd9647a495 100644 --- a/drivers/gpu/drm/i915/display/intel_fb_pin.c +++ b/drivers/gpu/drm/i915/display/intel_fb_pin.c @@ -26,10 +26,17 @@ intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, struct drm_device *dev = fb->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj = intel_fb_obj(fb); + struct i915_gem_ww_ctx ww; struct i915_vma *vma; u32 alignment; int ret; + /* + * We are not syncing against the binding (and potential migrations) + * below, so this vm must never be async. + */ + GEM_WARN_ON(vm->bind_async_flags); + if (WARN_ON(!i915_gem_object_is_framebuffer(obj))) return ERR_PTR(-EINVAL); @@ -37,29 +44,48 @@ intel_pin_fb_obj_dpt(struct drm_framebuffer *fb, atomic_inc(&dev_priv->gpu_error.pending_fb_pin); - ret = i915_gem_object_lock_interruptible(obj, NULL); - if (!ret) { + for_i915_gem_ww(&ww, ret, true) { + ret = i915_gem_object_lock(obj, &ww); + if (ret) + continue; + + if (HAS_LMEM(dev_priv)) { + unsigned int flags = obj->flags; + + /* + * For this type of buffer we need to able to read from the CPU + * the clear color value found in the buffer, hence we need to + * ensure it is always in the mappable part of lmem, if this is + * a small-bar device. + */ + if (intel_fb_rc_ccs_cc_plane(fb) >= 0) + flags &= ~I915_BO_ALLOC_GPU_ONLY; + ret = __i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0, + flags); + if (ret) + continue; + } + ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); - i915_gem_object_unlock(obj); - } - if (ret) { - vma = ERR_PTR(ret); - goto err; - } + if (ret) + continue; - vma = i915_vma_instance(obj, vm, view); - if (IS_ERR(vma)) - goto err; + vma = i915_vma_instance(obj, vm, view); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + continue; + } - if (i915_vma_misplaced(vma, 0, alignment, 0)) { - ret = i915_vma_unbind_unlocked(vma); - if (ret) { - vma = ERR_PTR(ret); - goto err; + if (i915_vma_misplaced(vma, 0, alignment, 0)) { + ret = i915_vma_unbind(vma); + if (ret) + continue; } - } - ret = i915_vma_pin(vma, 0, alignment, PIN_GLOBAL); + ret = i915_vma_pin_ww(vma, &ww, 0, alignment, PIN_GLOBAL); + if (ret) + continue; + } if (ret) { vma = ERR_PTR(ret); goto err; diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 16537830ccf013353065188b92539f4e7e151a7d..f38175304928083620a7decaff246a0edd2e546e 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -55,11 +55,11 @@ #define for_each_fbc_id(__dev_priv, __fbc_id) \ for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \ - for_each_if(INTEL_INFO(__dev_priv)->display.fbc_mask & BIT(__fbc_id)) + for_each_if(RUNTIME_INFO(__dev_priv)->fbc_mask & BIT(__fbc_id)) #define for_each_intel_fbc(__dev_priv, __fbc, __fbc_id) \ for_each_fbc_id((__dev_priv), (__fbc_id)) \ - for_each_if((__fbc) = (__dev_priv)->fbc[(__fbc_id)]) + for_each_if((__fbc) = (__dev_priv)->display.fbc[(__fbc_id)]) struct intel_fbc_funcs { void (*activate)(struct intel_fbc *fbc); @@ -1098,6 +1098,12 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state, return 0; } + /* Wa_14016291713 */ + if (IS_DISPLAY_VER(i915, 12, 13) && crtc_state->has_psr) { + plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)"; + return 0; + } + if (!pixel_format_is_valid(plane_state)) { plane_state->no_fbc_reason = "pixel format not supported"; return 0; @@ -1704,17 +1710,17 @@ void intel_fbc_init(struct drm_i915_private *i915) enum intel_fbc_id fbc_id; if (!drm_mm_initialized(&i915->mm.stolen)) - mkwrite_device_info(i915)->display.fbc_mask = 0; + RUNTIME_INFO(i915)->fbc_mask = 0; if (need_fbc_vtd_wa(i915)) - mkwrite_device_info(i915)->display.fbc_mask = 0; + RUNTIME_INFO(i915)->fbc_mask = 0; i915->params.enable_fbc = intel_sanitize_fbc_option(i915); drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n", i915->params.enable_fbc); for_each_fbc_id(i915, fbc_id) - i915->fbc[fbc_id] = intel_fbc_create(i915, fbc_id); + i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id); } /** @@ -1834,7 +1840,7 @@ void intel_fbc_debugfs_register(struct drm_i915_private *i915) struct drm_minor *minor = i915->drm.primary; struct intel_fbc *fbc; - fbc = i915->fbc[INTEL_FBC_A]; + fbc = i915->display.fbc[INTEL_FBC_A]; if (fbc) intel_fbc_debugfs_add(fbc, minor->debugfs_root); } diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h index db60143295ec6584ea54832035a2b02873f0ccf0..4adb98afe6fffc752cd0832ed9652c346ac65a8a 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.h +++ b/drivers/gpu/drm/i915/display/intel_fbc.h @@ -19,6 +19,7 @@ struct intel_plane_state; enum intel_fbc_id { INTEL_FBC_A, + INTEL_FBC_B, I915_MAX_FBCS, }; diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index 1922f62d04c0bed8d0977fe66288699d7d52aa6c..112aa0447a0dc1ea4ca10b40471bf7d1171e7204 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -210,6 +210,12 @@ static int intelfb_create(struct drm_fb_helper *helper, struct drm_i915_gem_object *obj; int ret; + mutex_lock(&ifbdev->hpd_lock); + ret = ifbdev->hpd_suspended ? -EAGAIN : 0; + mutex_unlock(&ifbdev->hpd_lock); + if (ret) + return ret; + if (intel_fb && (sizes->fb_width > intel_fb->base.width || sizes->fb_height > intel_fb->base.height)) { @@ -500,7 +506,7 @@ static void intel_fbdev_suspend_worker(struct work_struct *work) { intel_fbdev_set_suspend(&container_of(work, struct drm_i915_private, - fbdev_suspend_work)->drm, + display.fbdev.suspend_work)->drm, FBINFO_STATE_RUNNING, true); } @@ -530,8 +536,8 @@ int intel_fbdev_init(struct drm_device *dev) return ret; } - dev_priv->fbdev = ifbdev; - INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker); + dev_priv->display.fbdev.fbdev = ifbdev; + INIT_WORK(&dev_priv->display.fbdev.suspend_work, intel_fbdev_suspend_worker); return 0; } @@ -548,7 +554,7 @@ static void intel_fbdev_initial_config(void *data, async_cookie_t cookie) void intel_fbdev_initial_config_async(struct drm_device *dev) { - struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev; if (!ifbdev) return; @@ -568,12 +574,13 @@ static void intel_fbdev_sync(struct intel_fbdev *ifbdev) void intel_fbdev_unregister(struct drm_i915_private *dev_priv) { - struct intel_fbdev *ifbdev = dev_priv->fbdev; + struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev; if (!ifbdev) return; - cancel_work_sync(&dev_priv->fbdev_suspend_work); + intel_fbdev_set_suspend(&dev_priv->drm, FBINFO_STATE_SUSPENDED, true); + if (!current_is_async()) intel_fbdev_sync(ifbdev); @@ -582,7 +589,7 @@ void intel_fbdev_unregister(struct drm_i915_private *dev_priv) void intel_fbdev_fini(struct drm_i915_private *dev_priv) { - struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->fbdev); + struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->display.fbdev.fbdev); if (!ifbdev) return; @@ -596,7 +603,7 @@ void intel_fbdev_fini(struct drm_i915_private *dev_priv) */ static void intel_fbdev_hpd_set_suspend(struct drm_i915_private *i915, int state) { - struct intel_fbdev *ifbdev = i915->fbdev; + struct intel_fbdev *ifbdev = i915->display.fbdev.fbdev; bool send_hpd = false; mutex_lock(&ifbdev->hpd_lock); @@ -614,11 +621,11 @@ static void intel_fbdev_hpd_set_suspend(struct drm_i915_private *i915, int state void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous) { struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_fbdev *ifbdev = dev_priv->fbdev; + struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev; struct fb_info *info; if (!ifbdev || !ifbdev->vma) - return; + goto set_suspend; info = ifbdev->helper.fbdev; @@ -631,7 +638,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous * ourselves, so only flush outstanding work upon suspend! */ if (state != FBINFO_STATE_RUNNING) - flush_work(&dev_priv->fbdev_suspend_work); + flush_work(&dev_priv->display.fbdev.suspend_work); console_lock(); } else { @@ -645,7 +652,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous /* Don't block our own workqueue as this can * be run in parallel with other i915.ko tasks. */ - schedule_work(&dev_priv->fbdev_suspend_work); + schedule_work(&dev_priv->display.fbdev.suspend_work); return; } } @@ -661,12 +668,13 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous drm_fb_helper_set_suspend(&ifbdev->helper, state); console_unlock(); +set_suspend: intel_fbdev_hpd_set_suspend(dev_priv, state); } void intel_fbdev_output_poll_changed(struct drm_device *dev) { - struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev; bool send_hpd; if (!ifbdev) @@ -685,7 +693,7 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev) void intel_fbdev_restore_mode(struct drm_device *dev) { - struct intel_fbdev *ifbdev = to_i915(dev)->fbdev; + struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev; if (!ifbdev) return; diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c index 67d2484afbaa73c3bdecc34f85091268afba05dc..7f47e5c85c81734f846a2094afc64faa194808f5 100644 --- a/drivers/gpu/drm/i915/display/intel_fdi.c +++ b/drivers/gpu/drm/i915/display/intel_fdi.c @@ -113,7 +113,7 @@ void intel_fdi_link_train(struct intel_crtc *crtc, { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - dev_priv->fdi_funcs->fdi_link_train(crtc, crtc_state); + dev_priv->display.funcs.fdi->fdi_link_train(crtc, crtc_state); } /* units of 100MHz */ @@ -210,14 +210,14 @@ void intel_fdi_pll_freq_update(struct drm_i915_private *i915) u32 fdi_pll_clk = intel_de_read(i915, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK; - i915->fdi_pll_freq = (fdi_pll_clk + 2) * 10000; + i915->display.fdi.pll_freq = (fdi_pll_clk + 2) * 10000; } else if (IS_SANDYBRIDGE(i915) || IS_IVYBRIDGE(i915)) { - i915->fdi_pll_freq = 270000; + i915->display.fdi.pll_freq = 270000; } else { return; } - drm_dbg(&i915->drm, "FDI PLL freq=%d\n", i915->fdi_pll_freq); + drm_dbg(&i915->drm, "FDI PLL freq=%d\n", i915->display.fdi.pll_freq); } int intel_fdi_link_freq(struct drm_i915_private *i915, @@ -226,7 +226,7 @@ int intel_fdi_link_freq(struct drm_i915_private *i915, if (HAS_DDI(i915)) return pipe_config->port_clock; /* SPLL */ else - return i915->fdi_pll_freq; + return i915->display.fdi.pll_freq; } int ilk_fdi_compute_config(struct intel_crtc *crtc, @@ -256,7 +256,7 @@ retry: pipe_config->fdi_lanes = lane; intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock, - link_bw, &pipe_config->fdi_m_n, false, false); + link_bw, &pipe_config->fdi_m_n, false); ret = ilk_check_fdi_lanes(dev, crtc->pipe, pipe_config); if (ret == -EDEADLK) @@ -789,7 +789,7 @@ void hsw_fdi_link_train(struct intel_encoder *encoder, FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2) | FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90); /* Enable the PCH Receiver FDI PLL */ - rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE | + rx_ctl_val = dev_priv->display.fdi.rx_config | FDI_RX_ENHANCE_FRAME_ENABLE | FDI_RX_PLL_ENABLE | FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes); intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), rx_ctl_val); @@ -1066,11 +1066,11 @@ void intel_fdi_init_hook(struct drm_i915_private *dev_priv) { if (IS_IRONLAKE(dev_priv)) { - dev_priv->fdi_funcs = &ilk_funcs; + dev_priv->display.funcs.fdi = &ilk_funcs; } else if (IS_SANDYBRIDGE(dev_priv)) { - dev_priv->fdi_funcs = &gen6_funcs; + dev_priv->display.funcs.fdi = &gen6_funcs; } else if (IS_IVYBRIDGE(dev_priv)) { /* FIXME: detect B0+ stepping and use auto training */ - dev_priv->fdi_funcs = &ivb_funcs; + dev_priv->display.funcs.fdi = &ivb_funcs; } } diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 791248f812aa7906a398a282d10b833229bddbca..d80e3e8a9b01cde94993a4a9e62dd6923ea88635 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -81,9 +81,9 @@ static void frontbuffer_flush(struct drm_i915_private *i915, enum fb_op_origin origin) { /* Delay flushing when rings are still busy.*/ - spin_lock(&i915->fb_tracking.lock); - frontbuffer_bits &= ~i915->fb_tracking.busy_bits; - spin_unlock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); + frontbuffer_bits &= ~i915->display.fb_tracking.busy_bits; + spin_unlock(&i915->display.fb_tracking.lock); if (!frontbuffer_bits) return; @@ -111,11 +111,11 @@ static void frontbuffer_flush(struct drm_i915_private *i915, void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915, unsigned frontbuffer_bits) { - spin_lock(&i915->fb_tracking.lock); - i915->fb_tracking.flip_bits |= frontbuffer_bits; + spin_lock(&i915->display.fb_tracking.lock); + i915->display.fb_tracking.flip_bits |= frontbuffer_bits; /* Remove stale busy bits due to the old buffer. */ - i915->fb_tracking.busy_bits &= ~frontbuffer_bits; - spin_unlock(&i915->fb_tracking.lock); + i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; + spin_unlock(&i915->display.fb_tracking.lock); } /** @@ -131,11 +131,11 @@ void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915, void intel_frontbuffer_flip_complete(struct drm_i915_private *i915, unsigned frontbuffer_bits) { - spin_lock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); /* Mask any cancelled flips. */ - frontbuffer_bits &= i915->fb_tracking.flip_bits; - i915->fb_tracking.flip_bits &= ~frontbuffer_bits; - spin_unlock(&i915->fb_tracking.lock); + frontbuffer_bits &= i915->display.fb_tracking.flip_bits; + i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits; + spin_unlock(&i915->display.fb_tracking.lock); if (frontbuffer_bits) frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP); @@ -155,10 +155,10 @@ void intel_frontbuffer_flip_complete(struct drm_i915_private *i915, void intel_frontbuffer_flip(struct drm_i915_private *i915, unsigned frontbuffer_bits) { - spin_lock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); /* Remove stale busy bits due to the old buffer. */ - i915->fb_tracking.busy_bits &= ~frontbuffer_bits; - spin_unlock(&i915->fb_tracking.lock); + i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; + spin_unlock(&i915->display.fb_tracking.lock); frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP); } @@ -170,10 +170,10 @@ void __intel_fb_invalidate(struct intel_frontbuffer *front, struct drm_i915_private *i915 = to_i915(front->obj->base.dev); if (origin == ORIGIN_CS) { - spin_lock(&i915->fb_tracking.lock); - i915->fb_tracking.busy_bits |= frontbuffer_bits; - i915->fb_tracking.flip_bits &= ~frontbuffer_bits; - spin_unlock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); + i915->display.fb_tracking.busy_bits |= frontbuffer_bits; + i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits; + spin_unlock(&i915->display.fb_tracking.lock); } trace_intel_frontbuffer_invalidate(frontbuffer_bits, origin); @@ -191,11 +191,11 @@ void __intel_fb_flush(struct intel_frontbuffer *front, struct drm_i915_private *i915 = to_i915(front->obj->base.dev); if (origin == ORIGIN_CS) { - spin_lock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); /* Filter out new bits since rendering started. */ - frontbuffer_bits &= i915->fb_tracking.busy_bits; - i915->fb_tracking.busy_bits &= ~frontbuffer_bits; - spin_unlock(&i915->fb_tracking.lock); + frontbuffer_bits &= i915->display.fb_tracking.busy_bits; + i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits; + spin_unlock(&i915->display.fb_tracking.lock); } if (frontbuffer_bits) @@ -221,7 +221,7 @@ static void frontbuffer_retire(struct i915_active *ref) } static void frontbuffer_release(struct kref *ref) - __releases(&to_i915(front->obj->base.dev)->fb_tracking.lock) + __releases(&to_i915(front->obj->base.dev)->display.fb_tracking.lock) { struct intel_frontbuffer *front = container_of(ref, typeof(*front), ref); @@ -238,7 +238,7 @@ static void frontbuffer_release(struct kref *ref) spin_unlock(&obj->vma.lock); RCU_INIT_POINTER(obj->frontbuffer, NULL); - spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock); + spin_unlock(&to_i915(obj->base.dev)->display.fb_tracking.lock); i915_active_fini(&front->write); @@ -268,7 +268,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj) frontbuffer_retire, I915_ACTIVE_RETIRE_SLEEPS); - spin_lock(&i915->fb_tracking.lock); + spin_lock(&i915->display.fb_tracking.lock); if (rcu_access_pointer(obj->frontbuffer)) { kfree(front); front = rcu_dereference_protected(obj->frontbuffer, true); @@ -277,7 +277,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj) i915_gem_object_get(obj); rcu_assign_pointer(obj->frontbuffer, front); } - spin_unlock(&i915->fb_tracking.lock); + spin_unlock(&i915->display.fb_tracking.lock); return front; } @@ -286,7 +286,7 @@ void intel_frontbuffer_put(struct intel_frontbuffer *front) { kref_put_lock(&front->ref, frontbuffer_release, - &to_i915(front->obj->base.dev)->fb_tracking.lock); + &to_i915(front->obj->base.dev)->display.fb_tracking.lock); } /** @@ -311,6 +311,8 @@ void intel_frontbuffer_track(struct intel_frontbuffer *old, */ BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > BITS_PER_TYPE(atomic_t)); + BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32); + BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE); if (old) { drm_WARN_ON(old->obj->base.dev, diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.h b/drivers/gpu/drm/i915/display/intel_frontbuffer.h index ff0c37b079aa7bbbca054b541aff788478b92c23..3c474ed937fb2c4352639b65a443e31647ec5401 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.h +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.h @@ -25,6 +25,7 @@ #define __INTEL_FRONTBUFFER_H__ #include +#include #include #include "gem/i915_gem_object_types.h" @@ -48,6 +49,23 @@ struct intel_frontbuffer { struct rcu_head rcu; }; +/* + * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is + * considered to be the frontbuffer for the given plane interface-wise. This + * doesn't mean that the hw necessarily already scans it out, but that any + * rendering (by the cpu or gpu) will land in the frontbuffer eventually. + * + * We have one bit per pipe and per scanout plane type. + */ +#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 +#define INTEL_FRONTBUFFER(pipe, plane_id) \ + BIT((plane_id) + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)); +#define INTEL_FRONTBUFFER_OVERLAY(pipe) \ + BIT(INTEL_FRONTBUFFER_BITS_PER_PIPE - 1 + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) +#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ + GENMASK(INTEL_FRONTBUFFER_BITS_PER_PIPE * ((pipe) + 1) - 1, \ + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) + void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915, unsigned frontbuffer_bits); void intel_frontbuffer_flip_complete(struct drm_i915_private *i915, diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index a6ba7fb723397557465b4b2fc2e1976828ff7d40..74443f57f62dfe3530dc17df6437dc2e9f4159b5 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -37,6 +37,7 @@ #include "intel_de.h" #include "intel_display_types.h" #include "intel_gmbus.h" +#include "intel_gmbus_regs.h" struct intel_gmbus { struct i2c_adapter adapter; @@ -45,7 +46,7 @@ struct intel_gmbus { u32 reg0; i915_reg_t gpio_reg; struct i2c_algo_bit_data bit_algo; - struct drm_i915_private *dev_priv; + struct drm_i915_private *i915; }; struct gmbus_pin { @@ -116,6 +117,18 @@ static const struct gmbus_pin gmbus_pins_dg2[] = { [GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ }, }; +static const struct gmbus_pin gmbus_pins_mtp[] = { + [GMBUS_PIN_1_BXT] = { "dpa", GPIOB }, + [GMBUS_PIN_2_BXT] = { "dpb", GPIOC }, + [GMBUS_PIN_3_BXT] = { "dpc", GPIOD }, + [GMBUS_PIN_4_CNP] = { "dpd", GPIOE }, + [GMBUS_PIN_5_MTP] = { "dpe", GPIOF }, + [GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ }, + [GMBUS_PIN_10_TC2_ICP] = { "tc2", GPIOK }, + [GMBUS_PIN_11_TC3_ICP] = { "tc3", GPIOL }, + [GMBUS_PIN_12_TC4_ICP] = { "tc4", GPIOM }, +}; + static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *i915, unsigned int pin) { @@ -128,6 +141,9 @@ static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *i915, } else if (INTEL_PCH_TYPE(i915) >= PCH_DG1) { pins = gmbus_pins_dg1; size = ARRAY_SIZE(gmbus_pins_dg1); + } else if (INTEL_PCH_TYPE(i915) >= PCH_MTP) { + pins = gmbus_pins_mtp; + size = ARRAY_SIZE(gmbus_pins_mtp); } else if (INTEL_PCH_TYPE(i915) >= PCH_ICP) { pins = gmbus_pins_icp; size = ARRAY_SIZE(gmbus_pins_icp); @@ -170,55 +186,55 @@ to_intel_gmbus(struct i2c_adapter *i2c) } void -intel_gmbus_reset(struct drm_i915_private *dev_priv) +intel_gmbus_reset(struct drm_i915_private *i915) { - intel_de_write(dev_priv, GMBUS0, 0); - intel_de_write(dev_priv, GMBUS4, 0); + intel_de_write(i915, GMBUS0(i915), 0); + intel_de_write(i915, GMBUS4(i915), 0); } -static void pnv_gmbus_clock_gating(struct drm_i915_private *dev_priv, +static void pnv_gmbus_clock_gating(struct drm_i915_private *i915, bool enable) { u32 val; /* When using bit bashing for I2C, this bit needs to be set to 1 */ - val = intel_de_read(dev_priv, DSPCLK_GATE_D); + val = intel_de_read(i915, DSPCLK_GATE_D(i915)); if (!enable) val |= PNV_GMBUSUNIT_CLOCK_GATE_DISABLE; else val &= ~PNV_GMBUSUNIT_CLOCK_GATE_DISABLE; - intel_de_write(dev_priv, DSPCLK_GATE_D, val); + intel_de_write(i915, DSPCLK_GATE_D(i915), val); } -static void pch_gmbus_clock_gating(struct drm_i915_private *dev_priv, +static void pch_gmbus_clock_gating(struct drm_i915_private *i915, bool enable) { u32 val; - val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D); + val = intel_de_read(i915, SOUTH_DSPCLK_GATE_D); if (!enable) val |= PCH_GMBUSUNIT_CLOCK_GATE_DISABLE; else val &= ~PCH_GMBUSUNIT_CLOCK_GATE_DISABLE; - intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val); + intel_de_write(i915, SOUTH_DSPCLK_GATE_D, val); } -static void bxt_gmbus_clock_gating(struct drm_i915_private *dev_priv, +static void bxt_gmbus_clock_gating(struct drm_i915_private *i915, bool enable) { u32 val; - val = intel_de_read(dev_priv, GEN9_CLKGATE_DIS_4); + val = intel_de_read(i915, GEN9_CLKGATE_DIS_4); if (!enable) val |= BXT_GMBUS_GATING_DIS; else val &= ~BXT_GMBUS_GATING_DIS; - intel_de_write(dev_priv, GEN9_CLKGATE_DIS_4, val); + intel_de_write(i915, GEN9_CLKGATE_DIS_4, val); } static u32 get_reserved(struct intel_gmbus *bus) { - struct drm_i915_private *i915 = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; struct intel_uncore *uncore = &i915->uncore; u32 reserved = 0; @@ -234,7 +250,7 @@ static u32 get_reserved(struct intel_gmbus *bus) static int get_clock(void *data) { struct intel_gmbus *bus = data; - struct intel_uncore *uncore = &bus->dev_priv->uncore; + struct intel_uncore *uncore = &bus->i915->uncore; u32 reserved = get_reserved(bus); intel_uncore_write_notrace(uncore, @@ -249,7 +265,7 @@ static int get_clock(void *data) static int get_data(void *data) { struct intel_gmbus *bus = data; - struct intel_uncore *uncore = &bus->dev_priv->uncore; + struct intel_uncore *uncore = &bus->i915->uncore; u32 reserved = get_reserved(bus); intel_uncore_write_notrace(uncore, @@ -264,7 +280,7 @@ static int get_data(void *data) static void set_clock(void *data, int state_high) { struct intel_gmbus *bus = data; - struct intel_uncore *uncore = &bus->dev_priv->uncore; + struct intel_uncore *uncore = &bus->i915->uncore; u32 reserved = get_reserved(bus); u32 clock_bits; @@ -283,7 +299,7 @@ static void set_clock(void *data, int state_high) static void set_data(void *data, int state_high) { struct intel_gmbus *bus = data; - struct intel_uncore *uncore = &bus->dev_priv->uncore; + struct intel_uncore *uncore = &bus->i915->uncore; u32 reserved = get_reserved(bus); u32 data_bits; @@ -301,12 +317,12 @@ static int intel_gpio_pre_xfer(struct i2c_adapter *adapter) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; - intel_gmbus_reset(dev_priv); + intel_gmbus_reset(i915); - if (IS_PINEVIEW(dev_priv)) - pnv_gmbus_clock_gating(dev_priv, false); + if (IS_PINEVIEW(i915)) + pnv_gmbus_clock_gating(i915, false); set_data(bus, 1); set_clock(bus, 1); @@ -318,13 +334,13 @@ static void intel_gpio_post_xfer(struct i2c_adapter *adapter) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; set_data(bus, 1); set_clock(bus, 1); - if (IS_PINEVIEW(dev_priv)) - pnv_gmbus_clock_gating(dev_priv, true); + if (IS_PINEVIEW(i915)) + pnv_gmbus_clock_gating(i915, true); } static void @@ -356,7 +372,7 @@ static bool has_gmbus_irq(struct drm_i915_private *i915) return HAS_GMBUS_IRQ(i915) && intel_irqs_enabled(i915); } -static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en) +static int gmbus_wait(struct drm_i915_private *i915, u32 status, u32 irq_en) { DEFINE_WAIT(wait); u32 gmbus2; @@ -366,21 +382,21 @@ static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en) * we also need to check for NAKs besides the hw ready/idle signal, we * need to wake up periodically and check that ourselves. */ - if (!has_gmbus_irq(dev_priv)) + if (!has_gmbus_irq(i915)) irq_en = 0; - add_wait_queue(&dev_priv->gmbus_wait_queue, &wait); - intel_de_write_fw(dev_priv, GMBUS4, irq_en); + add_wait_queue(&i915->display.gmbus.wait_queue, &wait); + intel_de_write_fw(i915, GMBUS4(i915), irq_en); status |= GMBUS_SATOER; - ret = wait_for_us((gmbus2 = intel_de_read_fw(dev_priv, GMBUS2)) & status, + ret = wait_for_us((gmbus2 = intel_de_read_fw(i915, GMBUS2(i915))) & status, 2); if (ret) - ret = wait_for((gmbus2 = intel_de_read_fw(dev_priv, GMBUS2)) & status, + ret = wait_for((gmbus2 = intel_de_read_fw(i915, GMBUS2(i915))) & status, 50); - intel_de_write_fw(dev_priv, GMBUS4, 0); - remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait); + intel_de_write_fw(i915, GMBUS4(i915), 0); + remove_wait_queue(&i915->display.gmbus.wait_queue, &wait); if (gmbus2 & GMBUS_SATOER) return -ENXIO; @@ -389,7 +405,7 @@ static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en) } static int -gmbus_wait_idle(struct drm_i915_private *dev_priv) +gmbus_wait_idle(struct drm_i915_private *i915) { DEFINE_WAIT(wait); u32 irq_enable; @@ -397,35 +413,35 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv) /* Important: The hw handles only the first bit, so set only one! */ irq_enable = 0; - if (has_gmbus_irq(dev_priv)) + if (has_gmbus_irq(i915)) irq_enable = GMBUS_IDLE_EN; - add_wait_queue(&dev_priv->gmbus_wait_queue, &wait); - intel_de_write_fw(dev_priv, GMBUS4, irq_enable); + add_wait_queue(&i915->display.gmbus.wait_queue, &wait); + intel_de_write_fw(i915, GMBUS4(i915), irq_enable); - ret = intel_wait_for_register_fw(&dev_priv->uncore, - GMBUS2, GMBUS_ACTIVE, 0, + ret = intel_wait_for_register_fw(&i915->uncore, + GMBUS2(i915), GMBUS_ACTIVE, 0, 10); - intel_de_write_fw(dev_priv, GMBUS4, 0); - remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait); + intel_de_write_fw(i915, GMBUS4(i915), 0); + remove_wait_queue(&i915->display.gmbus.wait_queue, &wait); return ret; } -static unsigned int gmbus_max_xfer_size(struct drm_i915_private *dev_priv) +static unsigned int gmbus_max_xfer_size(struct drm_i915_private *i915) { - return DISPLAY_VER(dev_priv) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX : + return DISPLAY_VER(i915) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX : GMBUS_BYTE_COUNT_MAX; } static int -gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, +gmbus_xfer_read_chunk(struct drm_i915_private *i915, unsigned short addr, u8 *buf, unsigned int len, u32 gmbus0_reg, u32 gmbus1_index) { unsigned int size = len; - bool burst_read = len > gmbus_max_xfer_size(dev_priv); + bool burst_read = len > gmbus_max_xfer_size(i915); bool extra_byte_added = false; if (burst_read) { @@ -438,21 +454,21 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, len++; } size = len % 256 + 256; - intel_de_write_fw(dev_priv, GMBUS0, + intel_de_write_fw(i915, GMBUS0(i915), gmbus0_reg | GMBUS_BYTE_CNT_OVERRIDE); } - intel_de_write_fw(dev_priv, GMBUS1, + intel_de_write_fw(i915, GMBUS1(i915), gmbus1_index | GMBUS_CYCLE_WAIT | (size << GMBUS_BYTE_COUNT_SHIFT) | (addr << GMBUS_SLAVE_ADDR_SHIFT) | GMBUS_SLAVE_READ | GMBUS_SW_RDY); while (len) { int ret; u32 val, loop = 0; - ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN); + ret = gmbus_wait(i915, GMBUS_HW_RDY, GMBUS_HW_RDY_EN); if (ret) return ret; - val = intel_de_read_fw(dev_priv, GMBUS3); + val = intel_de_read_fw(i915, GMBUS3(i915)); do { if (extra_byte_added && len == 1) break; @@ -463,7 +479,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, if (burst_read && len == size - 4) /* Reset the override bit */ - intel_de_write_fw(dev_priv, GMBUS0, gmbus0_reg); + intel_de_write_fw(i915, GMBUS0(i915), gmbus0_reg); } return 0; @@ -480,7 +496,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv, #define INTEL_GMBUS_BURST_READ_MAX_LEN 767U static int -gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg, +gmbus_xfer_read(struct drm_i915_private *i915, struct i2c_msg *msg, u32 gmbus0_reg, u32 gmbus1_index) { u8 *buf = msg->buf; @@ -489,12 +505,12 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg, int ret; do { - if (HAS_GMBUS_BURST_READ(dev_priv)) + if (HAS_GMBUS_BURST_READ(i915)) len = min(rx_size, INTEL_GMBUS_BURST_READ_MAX_LEN); else - len = min(rx_size, gmbus_max_xfer_size(dev_priv)); + len = min(rx_size, gmbus_max_xfer_size(i915)); - ret = gmbus_xfer_read_chunk(dev_priv, msg->addr, buf, len, + ret = gmbus_xfer_read_chunk(i915, msg->addr, buf, len, gmbus0_reg, gmbus1_index); if (ret) return ret; @@ -507,7 +523,7 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg, } static int -gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, +gmbus_xfer_write_chunk(struct drm_i915_private *i915, unsigned short addr, u8 *buf, unsigned int len, u32 gmbus1_index) { @@ -520,8 +536,8 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, len -= 1; } - intel_de_write_fw(dev_priv, GMBUS3, val); - intel_de_write_fw(dev_priv, GMBUS1, + intel_de_write_fw(i915, GMBUS3(i915), val); + intel_de_write_fw(i915, GMBUS1(i915), gmbus1_index | GMBUS_CYCLE_WAIT | (chunk_size << GMBUS_BYTE_COUNT_SHIFT) | (addr << GMBUS_SLAVE_ADDR_SHIFT) | GMBUS_SLAVE_WRITE | GMBUS_SW_RDY); while (len) { int ret; @@ -531,9 +547,9 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, val |= *buf++ << (8 * loop); } while (--len && ++loop < 4); - intel_de_write_fw(dev_priv, GMBUS3, val); + intel_de_write_fw(i915, GMBUS3(i915), val); - ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN); + ret = gmbus_wait(i915, GMBUS_HW_RDY, GMBUS_HW_RDY_EN); if (ret) return ret; } @@ -542,7 +558,7 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv, } static int -gmbus_xfer_write(struct drm_i915_private *dev_priv, struct i2c_msg *msg, +gmbus_xfer_write(struct drm_i915_private *i915, struct i2c_msg *msg, u32 gmbus1_index) { u8 *buf = msg->buf; @@ -551,9 +567,9 @@ gmbus_xfer_write(struct drm_i915_private *dev_priv, struct i2c_msg *msg, int ret; do { - len = min(tx_size, gmbus_max_xfer_size(dev_priv)); + len = min(tx_size, gmbus_max_xfer_size(i915)); - ret = gmbus_xfer_write_chunk(dev_priv, msg->addr, buf, len, + ret = gmbus_xfer_write_chunk(i915, msg->addr, buf, len, gmbus1_index); if (ret) return ret; @@ -580,7 +596,7 @@ gmbus_is_index_xfer(struct i2c_msg *msgs, int i, int num) } static int -gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs, +gmbus_index_xfer(struct drm_i915_private *i915, struct i2c_msg *msgs, u32 gmbus0_reg) { u32 gmbus1_index = 0; @@ -596,17 +612,17 @@ gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs, /* GMBUS5 holds 16-bit index */ if (gmbus5) - intel_de_write_fw(dev_priv, GMBUS5, gmbus5); + intel_de_write_fw(i915, GMBUS5(i915), gmbus5); if (msgs[1].flags & I2C_M_RD) - ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus0_reg, + ret = gmbus_xfer_read(i915, &msgs[1], gmbus0_reg, gmbus1_index); else - ret = gmbus_xfer_write(dev_priv, &msgs[1], gmbus1_index); + ret = gmbus_xfer_write(i915, &msgs[1], gmbus1_index); /* Clear GMBUS5 after each index transfer */ if (gmbus5) - intel_de_write_fw(dev_priv, GMBUS5, 0); + intel_de_write_fw(i915, GMBUS5(i915), 0); return ret; } @@ -616,34 +632,34 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num, u32 gmbus0_source) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; int i = 0, inc, try = 0; int ret = 0; /* Display WA #0868: skl,bxt,kbl,cfl,glk */ - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - bxt_gmbus_clock_gating(dev_priv, false); - else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv)) - pch_gmbus_clock_gating(dev_priv, false); + if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) + bxt_gmbus_clock_gating(i915, false); + else if (HAS_PCH_SPT(i915) || HAS_PCH_CNP(i915)) + pch_gmbus_clock_gating(i915, false); retry: - intel_de_write_fw(dev_priv, GMBUS0, gmbus0_source | bus->reg0); + intel_de_write_fw(i915, GMBUS0(i915), gmbus0_source | bus->reg0); for (; i < num; i += inc) { inc = 1; if (gmbus_is_index_xfer(msgs, i, num)) { - ret = gmbus_index_xfer(dev_priv, &msgs[i], + ret = gmbus_index_xfer(i915, &msgs[i], gmbus0_source | bus->reg0); inc = 2; /* an index transmission is two msgs */ } else if (msgs[i].flags & I2C_M_RD) { - ret = gmbus_xfer_read(dev_priv, &msgs[i], + ret = gmbus_xfer_read(i915, &msgs[i], gmbus0_source | bus->reg0, 0); } else { - ret = gmbus_xfer_write(dev_priv, &msgs[i], 0); + ret = gmbus_xfer_write(i915, &msgs[i], 0); } if (!ret) - ret = gmbus_wait(dev_priv, + ret = gmbus_wait(i915, GMBUS_HW_WAIT_PHASE, GMBUS_HW_WAIT_EN); if (ret == -ETIMEDOUT) goto timeout; @@ -655,19 +671,19 @@ retry: * a STOP on the very first cycle. To simplify the code we * unconditionally generate the STOP condition with an additional gmbus * cycle. */ - intel_de_write_fw(dev_priv, GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY); + intel_de_write_fw(i915, GMBUS1(i915), GMBUS_CYCLE_STOP | GMBUS_SW_RDY); /* Mark the GMBUS interface as disabled after waiting for idle. * We will re-enable it at the start of the next xfer, * till then let it sleep. */ - if (gmbus_wait_idle(dev_priv)) { - drm_dbg_kms(&dev_priv->drm, + if (gmbus_wait_idle(i915)) { + drm_dbg_kms(&i915->drm, "GMBUS [%s] timed out waiting for idle\n", adapter->name); ret = -ETIMEDOUT; } - intel_de_write_fw(dev_priv, GMBUS0, 0); + intel_de_write_fw(i915, GMBUS0(i915), 0); ret = ret ?: i; goto out; @@ -686,8 +702,8 @@ clear_err: * it's slow responding and only answers on the 2nd retry. */ ret = -ENXIO; - if (gmbus_wait_idle(dev_priv)) { - drm_dbg_kms(&dev_priv->drm, + if (gmbus_wait_idle(i915)) { + drm_dbg_kms(&i915->drm, "GMBUS [%s] timed out after NAK\n", adapter->name); ret = -ETIMEDOUT; @@ -697,11 +713,11 @@ clear_err: * of resetting the GMBUS controller and so clearing the * BUS_ERROR raised by the slave's NAK. */ - intel_de_write_fw(dev_priv, GMBUS1, GMBUS_SW_CLR_INT); - intel_de_write_fw(dev_priv, GMBUS1, 0); - intel_de_write_fw(dev_priv, GMBUS0, 0); + intel_de_write_fw(i915, GMBUS1(i915), GMBUS_SW_CLR_INT); + intel_de_write_fw(i915, GMBUS1(i915), 0); + intel_de_write_fw(i915, GMBUS0(i915), 0); - drm_dbg_kms(&dev_priv->drm, "GMBUS [%s] NAK for addr: %04x %c(%d)\n", + drm_dbg_kms(&i915->drm, "GMBUS [%s] NAK for addr: %04x %c(%d)\n", adapter->name, msgs[i].addr, (msgs[i].flags & I2C_M_RD) ? 'r' : 'w', msgs[i].len); @@ -712,7 +728,7 @@ clear_err: * drm_do_probe_ddc_edid, which bails out on the first -ENXIO. */ if (ret == -ENXIO && i == 0 && try++ == 0) { - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "GMBUS [%s] NAK on first message, retry\n", adapter->name); goto retry; @@ -721,10 +737,10 @@ clear_err: goto out; timeout: - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "GMBUS [%s] timed out, falling back to bit banging on pin %d\n", bus->adapter.name, bus->reg0 & 0xff); - intel_de_write_fw(dev_priv, GMBUS0, 0); + intel_de_write_fw(i915, GMBUS0(i915), 0); /* * Hardware may not support GMBUS over these pins? Try GPIO bitbanging @@ -734,10 +750,10 @@ timeout: out: /* Display WA #0868: skl,bxt,kbl,cfl,glk */ - if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - bxt_gmbus_clock_gating(dev_priv, true); - else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv)) - pch_gmbus_clock_gating(dev_priv, true); + if (IS_GEMINILAKE(i915) || IS_BROXTON(i915)) + bxt_gmbus_clock_gating(i915, true); + else if (HAS_PCH_SPT(i915) || HAS_PCH_CNP(i915)) + pch_gmbus_clock_gating(i915, true); return ret; } @@ -746,11 +762,11 @@ static int gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; intel_wakeref_t wakeref; int ret; - wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); + wakeref = intel_display_power_get(i915, POWER_DOMAIN_GMBUS); if (bus->force_bit) { ret = i2c_bit_algo.master_xfer(adapter, msgs, num); @@ -762,7 +778,7 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) bus->force_bit |= GMBUS_FORCE_BIT_RETRY; } - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); + intel_display_power_put(i915, POWER_DOMAIN_GMBUS, wakeref); return ret; } @@ -770,7 +786,7 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num) int intel_gmbus_output_aksv(struct i2c_adapter *adapter) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; u8 cmd = DRM_HDCP_DDC_AKSV; u8 buf[DRM_HDCP_KSV_LEN] = { 0 }; struct i2c_msg msgs[] = { @@ -790,8 +806,8 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter) intel_wakeref_t wakeref; int ret; - wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); - mutex_lock(&dev_priv->gmbus_mutex); + wakeref = intel_display_power_get(i915, POWER_DOMAIN_GMBUS); + mutex_lock(&i915->display.gmbus.mutex); /* * In order to output Aksv to the receiver, use an indexed write to @@ -800,8 +816,8 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter) */ ret = do_gmbus_xfer(adapter, msgs, ARRAY_SIZE(msgs), GMBUS_AKSV_SELECT); - mutex_unlock(&dev_priv->gmbus_mutex); - intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref); + mutex_unlock(&i915->display.gmbus.mutex); + intel_display_power_put(i915, POWER_DOMAIN_GMBUS, wakeref); return ret; } @@ -824,27 +840,27 @@ static void gmbus_lock_bus(struct i2c_adapter *adapter, unsigned int flags) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; - mutex_lock(&dev_priv->gmbus_mutex); + mutex_lock(&i915->display.gmbus.mutex); } static int gmbus_trylock_bus(struct i2c_adapter *adapter, unsigned int flags) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; - return mutex_trylock(&dev_priv->gmbus_mutex); + return mutex_trylock(&i915->display.gmbus.mutex); } static void gmbus_unlock_bus(struct i2c_adapter *adapter, unsigned int flags) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; - mutex_unlock(&dev_priv->gmbus_mutex); + mutex_unlock(&i915->display.gmbus.mutex); } static const struct i2c_lock_operations gmbus_lock_ops = { @@ -855,31 +871,31 @@ static const struct i2c_lock_operations gmbus_lock_ops = { /** * intel_gmbus_setup - instantiate all Intel i2c GMBuses - * @dev_priv: i915 device private + * @i915: i915 device private */ -int intel_gmbus_setup(struct drm_i915_private *dev_priv) +int intel_gmbus_setup(struct drm_i915_private *i915) { - struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); + struct pci_dev *pdev = to_pci_dev(i915->drm.dev); unsigned int pin; int ret; - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE; - else if (!HAS_GMCH(dev_priv)) + if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) + i915->display.gmbus.mmio_base = VLV_DISPLAY_BASE; + else if (!HAS_GMCH(i915)) /* * Broxton uses the same PCH offsets for South Display Engine, * even though it doesn't have a PCH. */ - dev_priv->gpio_mmio_base = PCH_DISPLAY_BASE; + i915->display.gmbus.mmio_base = PCH_DISPLAY_BASE; - mutex_init(&dev_priv->gmbus_mutex); - init_waitqueue_head(&dev_priv->gmbus_wait_queue); + mutex_init(&i915->display.gmbus.mutex); + init_waitqueue_head(&i915->display.gmbus.wait_queue); - for (pin = 0; pin < ARRAY_SIZE(dev_priv->gmbus); pin++) { + for (pin = 0; pin < ARRAY_SIZE(i915->display.gmbus.bus); pin++) { const struct gmbus_pin *gmbus_pin; struct intel_gmbus *bus; - gmbus_pin = get_gmbus_pin(dev_priv, pin); + gmbus_pin = get_gmbus_pin(i915, pin); if (!gmbus_pin) continue; @@ -896,7 +912,7 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv) "i915 gmbus %s", gmbus_pin->name); bus->adapter.dev.parent = &pdev->dev; - bus->dev_priv = dev_priv; + bus->i915 = i915; bus->adapter.algo = &gmbus_algorithm; bus->adapter.lock_ops = &gmbus_lock_ops; @@ -911,10 +927,10 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv) bus->reg0 = pin | GMBUS_RATE_100KHZ; /* gmbus seems to be broken on i830 */ - if (IS_I830(dev_priv)) + if (IS_I830(i915)) bus->force_bit = 1; - intel_gpio_setup(bus, GPIO(gmbus_pin->gpio)); + intel_gpio_setup(bus, GPIO(i915, gmbus_pin->gpio)); ret = i2c_add_adapter(&bus->adapter); if (ret) { @@ -922,43 +938,43 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv) goto err; } - dev_priv->gmbus[pin] = bus; + i915->display.gmbus.bus[pin] = bus; } - intel_gmbus_reset(dev_priv); + intel_gmbus_reset(i915); return 0; err: - intel_gmbus_teardown(dev_priv); + intel_gmbus_teardown(i915); return ret; } -struct i2c_adapter *intel_gmbus_get_adapter(struct drm_i915_private *dev_priv, +struct i2c_adapter *intel_gmbus_get_adapter(struct drm_i915_private *i915, unsigned int pin) { - if (drm_WARN_ON(&dev_priv->drm, pin >= ARRAY_SIZE(dev_priv->gmbus) || - !dev_priv->gmbus[pin])) + if (drm_WARN_ON(&i915->drm, pin >= ARRAY_SIZE(i915->display.gmbus.bus) || + !i915->display.gmbus.bus[pin])) return NULL; - return &dev_priv->gmbus[pin]->adapter; + return &i915->display.gmbus.bus[pin]->adapter; } void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit) { struct intel_gmbus *bus = to_intel_gmbus(adapter); - struct drm_i915_private *dev_priv = bus->dev_priv; + struct drm_i915_private *i915 = bus->i915; - mutex_lock(&dev_priv->gmbus_mutex); + mutex_lock(&i915->display.gmbus.mutex); bus->force_bit += force_bit ? 1 : -1; - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "%sabling bit-banging on %s. force bit now %d\n", force_bit ? "en" : "dis", adapter->name, bus->force_bit); - mutex_unlock(&dev_priv->gmbus_mutex); + mutex_unlock(&i915->display.gmbus.mutex); } bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) @@ -968,20 +984,20 @@ bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter) return bus->force_bit; } -void intel_gmbus_teardown(struct drm_i915_private *dev_priv) +void intel_gmbus_teardown(struct drm_i915_private *i915) { unsigned int pin; - for (pin = 0; pin < ARRAY_SIZE(dev_priv->gmbus); pin++) { + for (pin = 0; pin < ARRAY_SIZE(i915->display.gmbus.bus); pin++) { struct intel_gmbus *bus; - bus = dev_priv->gmbus[pin]; + bus = i915->display.gmbus.bus[pin]; if (!bus) continue; i2c_del_adapter(&bus->adapter); kfree(bus); - dev_priv->gmbus[pin] = NULL; + i915->display.gmbus.bus[pin] = NULL; } } diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.h b/drivers/gpu/drm/i915/display/intel_gmbus.h index 8edc2e99cf5379f67d39c21acd45af2f0983868d..20f704bd4e7004fab13e6aa097f4a09a99355e93 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.h +++ b/drivers/gpu/drm/i915/display/intel_gmbus.h @@ -24,6 +24,7 @@ struct i2c_adapter; #define GMBUS_PIN_2_BXT 2 #define GMBUS_PIN_3_BXT 3 #define GMBUS_PIN_4_CNP 4 +#define GMBUS_PIN_5_MTP 5 #define GMBUS_PIN_9_TC1_ICP 9 #define GMBUS_PIN_10_TC2_ICP 10 #define GMBUS_PIN_11_TC3_ICP 11 diff --git a/drivers/gpu/drm/i915/display/intel_gmbus_regs.h b/drivers/gpu/drm/i915/display/intel_gmbus_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..53aacbda983c49705957c905a371002b9258eb5d --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_gmbus_regs.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_GMBUS_REGS_H__ +#define __INTEL_GMBUS_REGS_H__ + +#include "i915_reg_defs.h" + +#define GMBUS_MMIO_BASE(__i915) ((__i915)->display.gmbus.mmio_base) + +#define GPIO(__i915, gpio) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5010 + 4 * (gpio)) +#define GPIO_CLOCK_DIR_MASK (1 << 0) +#define GPIO_CLOCK_DIR_IN (0 << 1) +#define GPIO_CLOCK_DIR_OUT (1 << 1) +#define GPIO_CLOCK_VAL_MASK (1 << 2) +#define GPIO_CLOCK_VAL_OUT (1 << 3) +#define GPIO_CLOCK_VAL_IN (1 << 4) +#define GPIO_CLOCK_PULLUP_DISABLE (1 << 5) +#define GPIO_DATA_DIR_MASK (1 << 8) +#define GPIO_DATA_DIR_IN (0 << 9) +#define GPIO_DATA_DIR_OUT (1 << 9) +#define GPIO_DATA_VAL_MASK (1 << 10) +#define GPIO_DATA_VAL_OUT (1 << 11) +#define GPIO_DATA_VAL_IN (1 << 12) +#define GPIO_DATA_PULLUP_DISABLE (1 << 13) + +/* clock/port select */ +#define GMBUS0(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5100) +#define GMBUS_AKSV_SELECT (1 << 11) +#define GMBUS_RATE_100KHZ (0 << 8) +#define GMBUS_RATE_50KHZ (1 << 8) +#define GMBUS_RATE_400KHZ (2 << 8) /* reserved on Pineview */ +#define GMBUS_RATE_1MHZ (3 << 8) /* reserved on Pineview */ +#define GMBUS_HOLD_EXT (1 << 7) /* 300ns hold time, rsvd on Pineview */ +#define GMBUS_BYTE_CNT_OVERRIDE (1 << 6) + +/* command/status */ +#define GMBUS1(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5104) +#define GMBUS_SW_CLR_INT (1 << 31) +#define GMBUS_SW_RDY (1 << 30) +#define GMBUS_ENT (1 << 29) /* enable timeout */ +#define GMBUS_CYCLE_NONE (0 << 25) +#define GMBUS_CYCLE_WAIT (1 << 25) +#define GMBUS_CYCLE_INDEX (2 << 25) +#define GMBUS_CYCLE_STOP (4 << 25) +#define GMBUS_BYTE_COUNT_SHIFT 16 +#define GMBUS_BYTE_COUNT_MAX 256U +#define GEN9_GMBUS_BYTE_COUNT_MAX 511U +#define GMBUS_SLAVE_INDEX_SHIFT 8 +#define GMBUS_SLAVE_ADDR_SHIFT 1 +#define GMBUS_SLAVE_READ (1 << 0) +#define GMBUS_SLAVE_WRITE (0 << 0) + +/* status */ +#define GMBUS2(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5108) +#define GMBUS_INUSE (1 << 15) +#define GMBUS_HW_WAIT_PHASE (1 << 14) +#define GMBUS_STALL_TIMEOUT (1 << 13) +#define GMBUS_INT (1 << 12) +#define GMBUS_HW_RDY (1 << 11) +#define GMBUS_SATOER (1 << 10) +#define GMBUS_ACTIVE (1 << 9) + +/* data buffer bytes 3-0 */ +#define GMBUS3(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x510c) + +/* interrupt mask (Pineview+) */ +#define GMBUS4(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5110) +#define GMBUS_SLAVE_TIMEOUT_EN (1 << 4) +#define GMBUS_NAK_EN (1 << 3) +#define GMBUS_IDLE_EN (1 << 2) +#define GMBUS_HW_WAIT_EN (1 << 1) +#define GMBUS_HW_RDY_EN (1 << 0) + +/* byte index */ +#define GMBUS5(__i915) _MMIO(GMBUS_MMIO_BASE(__i915) + 0x5120) +#define GMBUS_2BYTE_INDEX_EN (1 << 31) + +#endif /* __INTEL_GMBUS_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index 7dbc9f0bb24f9e37c25d1f1a799dacf7271e1b3b..6406fd487ee5248febf1ad18fa4a732b59dbb772 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -23,6 +23,7 @@ #include "intel_display_power_well.h" #include "intel_display_types.h" #include "intel_hdcp.h" +#include "intel_hdcp_regs.h" #include "intel_pcode.h" #define KEY_LOAD_TRIES 5 @@ -209,12 +210,12 @@ bool intel_hdcp2_capable(struct intel_connector *connector) return false; /* MEI interface is solid */ - mutex_lock(&dev_priv->hdcp_comp_mutex); - if (!dev_priv->hdcp_comp_added || !dev_priv->hdcp_master) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + if (!dev_priv->display.hdcp.comp_added || !dev_priv->display.hdcp.master) { + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return false; } - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); /* Sink's capability for HDCP2.2 */ hdcp->shim->hdcp_2_2_capable(dig_port, &capable); @@ -1131,8 +1132,8 @@ static void intel_hdcp_prop_work(struct work_struct *work) bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port) { - return INTEL_INFO(dev_priv)->display.has_hdcp && - (DISPLAY_VER(dev_priv) >= 12 || port < PORT_E); + return RUNTIME_INFO(dev_priv)->has_hdcp && + (DISPLAY_VER(dev_priv) >= 12 || port < PORT_E); } static int @@ -1145,11 +1146,11 @@ hdcp2_prepare_ake_init(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1157,7 +1158,7 @@ hdcp2_prepare_ake_init(struct intel_connector *connector, if (ret) drm_dbg_kms(&dev_priv->drm, "Prepare_ake_init failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1175,11 +1176,11 @@ hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1189,7 +1190,7 @@ hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Verify rx_cert failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1203,18 +1204,18 @@ static int hdcp2_verify_hprime(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } ret = comp->ops->verify_hprime(comp->mei_dev, data, rx_hprime); if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Verify hprime failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1229,11 +1230,11 @@ hdcp2_store_pairing_info(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1241,7 +1242,7 @@ hdcp2_store_pairing_info(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Store pairing info failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1256,11 +1257,11 @@ hdcp2_prepare_lc_init(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1268,7 +1269,7 @@ hdcp2_prepare_lc_init(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Prepare lc_init failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1283,11 +1284,11 @@ hdcp2_verify_lprime(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1295,7 +1296,7 @@ hdcp2_verify_lprime(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Verify L_Prime failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1309,11 +1310,11 @@ static int hdcp2_prepare_skey(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1321,7 +1322,7 @@ static int hdcp2_prepare_skey(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Get session key failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1338,11 +1339,11 @@ hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1352,7 +1353,7 @@ hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector, if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Verify rep topology failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1367,18 +1368,18 @@ hdcp2_verify_mprime(struct intel_connector *connector, struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } ret = comp->ops->verify_mprime(comp->mei_dev, data, stream_ready); if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Verify mprime failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1391,11 +1392,11 @@ static int hdcp2_authenticate_port(struct intel_connector *connector) struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } @@ -1403,7 +1404,7 @@ static int hdcp2_authenticate_port(struct intel_connector *connector) if (ret < 0) drm_dbg_kms(&dev_priv->drm, "Enable hdcp auth failed. %d\n", ret); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -1415,17 +1416,17 @@ static int hdcp2_close_mei_session(struct intel_connector *connector) struct i915_hdcp_comp_master *comp; int ret; - mutex_lock(&dev_priv->hdcp_comp_mutex); - comp = dev_priv->hdcp_master; + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + comp = dev_priv->display.hdcp.master; if (!comp || !comp->ops) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return -EINVAL; } ret = comp->ops->close_hdcp_session(comp->mei_dev, &dig_port->hdcp_port_data); - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return ret; } @@ -2143,10 +2144,10 @@ static int i915_hdcp_component_bind(struct device *i915_kdev, struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev); drm_dbg(&dev_priv->drm, "I915 HDCP comp bind\n"); - mutex_lock(&dev_priv->hdcp_comp_mutex); - dev_priv->hdcp_master = (struct i915_hdcp_comp_master *)data; - dev_priv->hdcp_master->mei_dev = mei_kdev; - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + dev_priv->display.hdcp.master = (struct i915_hdcp_comp_master *)data; + dev_priv->display.hdcp.master->mei_dev = mei_kdev; + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return 0; } @@ -2157,9 +2158,9 @@ static void i915_hdcp_component_unbind(struct device *i915_kdev, struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev); drm_dbg(&dev_priv->drm, "I915 HDCP comp unbind\n"); - mutex_lock(&dev_priv->hdcp_comp_mutex); - dev_priv->hdcp_master = NULL; - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + dev_priv->display.hdcp.master = NULL; + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); } static const struct component_ops i915_hdcp_component_ops = { @@ -2250,19 +2251,19 @@ void intel_hdcp_component_init(struct drm_i915_private *dev_priv) if (!is_hdcp2_supported(dev_priv)) return; - mutex_lock(&dev_priv->hdcp_comp_mutex); - drm_WARN_ON(&dev_priv->drm, dev_priv->hdcp_comp_added); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + drm_WARN_ON(&dev_priv->drm, dev_priv->display.hdcp.comp_added); - dev_priv->hdcp_comp_added = true; - mutex_unlock(&dev_priv->hdcp_comp_mutex); + dev_priv->display.hdcp.comp_added = true; + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); ret = component_add_typed(dev_priv->drm.dev, &i915_hdcp_component_ops, I915_COMPONENT_HDCP); if (ret < 0) { drm_dbg_kms(&dev_priv->drm, "Failed at component add(%d)\n", ret); - mutex_lock(&dev_priv->hdcp_comp_mutex); - dev_priv->hdcp_comp_added = false; - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + dev_priv->display.hdcp.comp_added = false; + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return; } } @@ -2475,14 +2476,14 @@ void intel_hdcp_update_pipe(struct intel_atomic_state *state, void intel_hdcp_component_fini(struct drm_i915_private *dev_priv) { - mutex_lock(&dev_priv->hdcp_comp_mutex); - if (!dev_priv->hdcp_comp_added) { - mutex_unlock(&dev_priv->hdcp_comp_mutex); + mutex_lock(&dev_priv->display.hdcp.comp_mutex); + if (!dev_priv->display.hdcp.comp_added) { + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); return; } - dev_priv->hdcp_comp_added = false; - mutex_unlock(&dev_priv->hdcp_comp_mutex); + dev_priv->display.hdcp.comp_added = false; + mutex_unlock(&dev_priv->display.hdcp.comp_mutex); component_del(dev_priv->drm.dev, &i915_hdcp_component_ops); } diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_regs.h b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..2a3733e8966c1b029be1d9c9493be20e7b5c2862 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_HDCP_REGS_H__ +#define __INTEL_HDCP_REGS_H__ + +#include "i915_reg_defs.h" + +/* HDCP Key Registers */ +#define HDCP_KEY_CONF _MMIO(0x66c00) +#define HDCP_AKSV_SEND_TRIGGER REG_BIT(31) +#define HDCP_CLEAR_KEYS_TRIGGER REG_BIT(30) +#define HDCP_KEY_LOAD_TRIGGER REG_BIT(8) +#define HDCP_KEY_STATUS _MMIO(0x66c04) +#define HDCP_FUSE_IN_PROGRESS REG_BIT(7) +#define HDCP_FUSE_ERROR REG_BIT(6) +#define HDCP_FUSE_DONE REG_BIT(5) +#define HDCP_KEY_LOAD_STATUS REG_BIT(1) +#define HDCP_KEY_LOAD_DONE REG_BIT(0) +#define HDCP_AKSV_LO _MMIO(0x66c10) +#define HDCP_AKSV_HI _MMIO(0x66c14) + +/* HDCP Repeater Registers */ +#define HDCP_REP_CTL _MMIO(0x66d00) +#define HDCP_TRANSA_REP_PRESENT REG_BIT(31) +#define HDCP_TRANSB_REP_PRESENT REG_BIT(30) +#define HDCP_TRANSC_REP_PRESENT REG_BIT(29) +#define HDCP_TRANSD_REP_PRESENT REG_BIT(28) +#define HDCP_DDIB_REP_PRESENT REG_BIT(30) +#define HDCP_DDIA_REP_PRESENT REG_BIT(29) +#define HDCP_DDIC_REP_PRESENT REG_BIT(28) +#define HDCP_DDID_REP_PRESENT REG_BIT(27) +#define HDCP_DDIF_REP_PRESENT REG_BIT(26) +#define HDCP_DDIE_REP_PRESENT REG_BIT(25) +#define HDCP_TRANSA_SHA1_M0 (1 << 20) +#define HDCP_TRANSB_SHA1_M0 (2 << 20) +#define HDCP_TRANSC_SHA1_M0 (3 << 20) +#define HDCP_TRANSD_SHA1_M0 (4 << 20) +#define HDCP_DDIB_SHA1_M0 (1 << 20) +#define HDCP_DDIA_SHA1_M0 (2 << 20) +#define HDCP_DDIC_SHA1_M0 (3 << 20) +#define HDCP_DDID_SHA1_M0 (4 << 20) +#define HDCP_DDIF_SHA1_M0 (5 << 20) +#define HDCP_DDIE_SHA1_M0 (6 << 20) /* Bspec says 5? */ +#define HDCP_SHA1_BUSY REG_BIT(16) +#define HDCP_SHA1_READY REG_BIT(17) +#define HDCP_SHA1_COMPLETE REG_BIT(18) +#define HDCP_SHA1_V_MATCH REG_BIT(19) +#define HDCP_SHA1_TEXT_32 (1 << 1) +#define HDCP_SHA1_COMPLETE_HASH (2 << 1) +#define HDCP_SHA1_TEXT_24 (4 << 1) +#define HDCP_SHA1_TEXT_16 (5 << 1) +#define HDCP_SHA1_TEXT_8 (6 << 1) +#define HDCP_SHA1_TEXT_0 (7 << 1) +#define HDCP_SHA_V_PRIME_H0 _MMIO(0x66d04) +#define HDCP_SHA_V_PRIME_H1 _MMIO(0x66d08) +#define HDCP_SHA_V_PRIME_H2 _MMIO(0x66d0C) +#define HDCP_SHA_V_PRIME_H3 _MMIO(0x66d10) +#define HDCP_SHA_V_PRIME_H4 _MMIO(0x66d14) +#define HDCP_SHA_V_PRIME(h) _MMIO((0x66d04 + (h) * 4)) +#define HDCP_SHA_TEXT _MMIO(0x66d18) + +/* HDCP Auth Registers */ +#define _PORTA_HDCP_AUTHENC 0x66800 +#define _PORTB_HDCP_AUTHENC 0x66500 +#define _PORTC_HDCP_AUTHENC 0x66600 +#define _PORTD_HDCP_AUTHENC 0x66700 +#define _PORTE_HDCP_AUTHENC 0x66A00 +#define _PORTF_HDCP_AUTHENC 0x66900 +#define _PORT_HDCP_AUTHENC(port, x) _MMIO(_PICK(port, \ + _PORTA_HDCP_AUTHENC, \ + _PORTB_HDCP_AUTHENC, \ + _PORTC_HDCP_AUTHENC, \ + _PORTD_HDCP_AUTHENC, \ + _PORTE_HDCP_AUTHENC, \ + _PORTF_HDCP_AUTHENC) + (x)) +#define PORT_HDCP_CONF(port) _PORT_HDCP_AUTHENC(port, 0x0) +#define _TRANSA_HDCP_CONF 0x66400 +#define _TRANSB_HDCP_CONF 0x66500 +#define TRANS_HDCP_CONF(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_CONF, \ + _TRANSB_HDCP_CONF) +#define HDCP_CONF(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_CONF(trans) : \ + PORT_HDCP_CONF(port)) + +#define HDCP_CONF_CAPTURE_AN REG_BIT(0) +#define HDCP_CONF_AUTH_AND_ENC (REG_BIT(1) | REG_BIT(0)) +#define PORT_HDCP_ANINIT(port) _PORT_HDCP_AUTHENC(port, 0x4) +#define _TRANSA_HDCP_ANINIT 0x66404 +#define _TRANSB_HDCP_ANINIT 0x66504 +#define TRANS_HDCP_ANINIT(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP_ANINIT, \ + _TRANSB_HDCP_ANINIT) +#define HDCP_ANINIT(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_ANINIT(trans) : \ + PORT_HDCP_ANINIT(port)) + +#define PORT_HDCP_ANLO(port) _PORT_HDCP_AUTHENC(port, 0x8) +#define _TRANSA_HDCP_ANLO 0x66408 +#define _TRANSB_HDCP_ANLO 0x66508 +#define TRANS_HDCP_ANLO(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANLO, \ + _TRANSB_HDCP_ANLO) +#define HDCP_ANLO(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_ANLO(trans) : \ + PORT_HDCP_ANLO(port)) + +#define PORT_HDCP_ANHI(port) _PORT_HDCP_AUTHENC(port, 0xC) +#define _TRANSA_HDCP_ANHI 0x6640C +#define _TRANSB_HDCP_ANHI 0x6650C +#define TRANS_HDCP_ANHI(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANHI, \ + _TRANSB_HDCP_ANHI) +#define HDCP_ANHI(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_ANHI(trans) : \ + PORT_HDCP_ANHI(port)) + +#define PORT_HDCP_BKSVLO(port) _PORT_HDCP_AUTHENC(port, 0x10) +#define _TRANSA_HDCP_BKSVLO 0x66410 +#define _TRANSB_HDCP_BKSVLO 0x66510 +#define TRANS_HDCP_BKSVLO(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP_BKSVLO, \ + _TRANSB_HDCP_BKSVLO) +#define HDCP_BKSVLO(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_BKSVLO(trans) : \ + PORT_HDCP_BKSVLO(port)) + +#define PORT_HDCP_BKSVHI(port) _PORT_HDCP_AUTHENC(port, 0x14) +#define _TRANSA_HDCP_BKSVHI 0x66414 +#define _TRANSB_HDCP_BKSVHI 0x66514 +#define TRANS_HDCP_BKSVHI(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP_BKSVHI, \ + _TRANSB_HDCP_BKSVHI) +#define HDCP_BKSVHI(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_BKSVHI(trans) : \ + PORT_HDCP_BKSVHI(port)) + +#define PORT_HDCP_RPRIME(port) _PORT_HDCP_AUTHENC(port, 0x18) +#define _TRANSA_HDCP_RPRIME 0x66418 +#define _TRANSB_HDCP_RPRIME 0x66518 +#define TRANS_HDCP_RPRIME(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP_RPRIME, \ + _TRANSB_HDCP_RPRIME) +#define HDCP_RPRIME(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_RPRIME(trans) : \ + PORT_HDCP_RPRIME(port)) + +#define PORT_HDCP_STATUS(port) _PORT_HDCP_AUTHENC(port, 0x1C) +#define _TRANSA_HDCP_STATUS 0x6641C +#define _TRANSB_HDCP_STATUS 0x6651C +#define TRANS_HDCP_STATUS(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP_STATUS, \ + _TRANSB_HDCP_STATUS) +#define HDCP_STATUS(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP_STATUS(trans) : \ + PORT_HDCP_STATUS(port)) + +#define HDCP_STATUS_STREAM_A_ENC REG_BIT(31) +#define HDCP_STATUS_STREAM_B_ENC REG_BIT(30) +#define HDCP_STATUS_STREAM_C_ENC REG_BIT(29) +#define HDCP_STATUS_STREAM_D_ENC REG_BIT(28) +#define HDCP_STATUS_AUTH REG_BIT(21) +#define HDCP_STATUS_ENC REG_BIT(20) +#define HDCP_STATUS_RI_MATCH REG_BIT(19) +#define HDCP_STATUS_R0_READY REG_BIT(18) +#define HDCP_STATUS_AN_READY REG_BIT(17) +#define HDCP_STATUS_CIPHER REG_BIT(16) +#define HDCP_STATUS_FRAME_CNT(x) (((x) >> 8) & 0xff) + +/* HDCP2.2 Registers */ +#define _PORTA_HDCP2_BASE 0x66800 +#define _PORTB_HDCP2_BASE 0x66500 +#define _PORTC_HDCP2_BASE 0x66600 +#define _PORTD_HDCP2_BASE 0x66700 +#define _PORTE_HDCP2_BASE 0x66A00 +#define _PORTF_HDCP2_BASE 0x66900 +#define _PORT_HDCP2_BASE(port, x) _MMIO(_PICK((port), \ + _PORTA_HDCP2_BASE, \ + _PORTB_HDCP2_BASE, \ + _PORTC_HDCP2_BASE, \ + _PORTD_HDCP2_BASE, \ + _PORTE_HDCP2_BASE, \ + _PORTF_HDCP2_BASE) + (x)) + +#define PORT_HDCP2_AUTH(port) _PORT_HDCP2_BASE(port, 0x98) +#define _TRANSA_HDCP2_AUTH 0x66498 +#define _TRANSB_HDCP2_AUTH 0x66598 +#define TRANS_HDCP2_AUTH(trans) _MMIO_TRANS(trans, _TRANSA_HDCP2_AUTH, \ + _TRANSB_HDCP2_AUTH) +#define AUTH_LINK_AUTHENTICATED REG_BIT(31) +#define AUTH_LINK_TYPE REG_BIT(30) +#define AUTH_FORCE_CLR_INPUTCTR REG_BIT(19) +#define AUTH_CLR_KEYS REG_BIT(18) +#define HDCP2_AUTH(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP2_AUTH(trans) : \ + PORT_HDCP2_AUTH(port)) + +#define PORT_HDCP2_CTL(port) _PORT_HDCP2_BASE(port, 0xB0) +#define _TRANSA_HDCP2_CTL 0x664B0 +#define _TRANSB_HDCP2_CTL 0x665B0 +#define TRANS_HDCP2_CTL(trans) _MMIO_TRANS(trans, _TRANSA_HDCP2_CTL, \ + _TRANSB_HDCP2_CTL) +#define CTL_LINK_ENCRYPTION_REQ REG_BIT(31) +#define HDCP2_CTL(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP2_CTL(trans) : \ + PORT_HDCP2_CTL(port)) + +#define PORT_HDCP2_STATUS(port) _PORT_HDCP2_BASE(port, 0xB4) +#define _TRANSA_HDCP2_STATUS 0x664B4 +#define _TRANSB_HDCP2_STATUS 0x665B4 +#define TRANS_HDCP2_STATUS(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP2_STATUS, \ + _TRANSB_HDCP2_STATUS) +#define LINK_TYPE_STATUS REG_BIT(22) +#define LINK_AUTH_STATUS REG_BIT(21) +#define LINK_ENCRYPTION_STATUS REG_BIT(20) +#define HDCP2_STATUS(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP2_STATUS(trans) : \ + PORT_HDCP2_STATUS(port)) + +#define _PIPEA_HDCP2_STREAM_STATUS 0x668C0 +#define _PIPEB_HDCP2_STREAM_STATUS 0x665C0 +#define _PIPEC_HDCP2_STREAM_STATUS 0x666C0 +#define _PIPED_HDCP2_STREAM_STATUS 0x667C0 +#define PIPE_HDCP2_STREAM_STATUS(pipe) _MMIO(_PICK((pipe), \ + _PIPEA_HDCP2_STREAM_STATUS, \ + _PIPEB_HDCP2_STREAM_STATUS, \ + _PIPEC_HDCP2_STREAM_STATUS, \ + _PIPED_HDCP2_STREAM_STATUS)) + +#define _TRANSA_HDCP2_STREAM_STATUS 0x664C0 +#define _TRANSB_HDCP2_STREAM_STATUS 0x665C0 +#define TRANS_HDCP2_STREAM_STATUS(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP2_STREAM_STATUS, \ + _TRANSB_HDCP2_STREAM_STATUS) +#define STREAM_ENCRYPTION_STATUS REG_BIT(31) +#define STREAM_TYPE_STATUS REG_BIT(30) +#define HDCP2_STREAM_STATUS(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP2_STREAM_STATUS(trans) : \ + PIPE_HDCP2_STREAM_STATUS(pipe)) + +#define _PORTA_HDCP2_AUTH_STREAM 0x66F00 +#define _PORTB_HDCP2_AUTH_STREAM 0x66F04 +#define PORT_HDCP2_AUTH_STREAM(port) _MMIO_PORT(port, \ + _PORTA_HDCP2_AUTH_STREAM, \ + _PORTB_HDCP2_AUTH_STREAM) +#define _TRANSA_HDCP2_AUTH_STREAM 0x66F00 +#define _TRANSB_HDCP2_AUTH_STREAM 0x66F04 +#define TRANS_HDCP2_AUTH_STREAM(trans) _MMIO_TRANS(trans, \ + _TRANSA_HDCP2_AUTH_STREAM, \ + _TRANSB_HDCP2_AUTH_STREAM) +#define AUTH_STREAM_TYPE REG_BIT(31) +#define HDCP2_AUTH_STREAM(dev_priv, trans, port) \ + (GRAPHICS_VER(dev_priv) >= 12 ? \ + TRANS_HDCP2_AUTH_STREAM(trans) : \ + PORT_HDCP2_AUTH_STREAM(port)) + +#endif /* __INTEL_HDCP_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index ebd91aa69dd20cc70302ac11e88180d1a2ea8a19..7816b2a33feeb76b0ec805f53642c2c43aa13c79 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -50,6 +50,7 @@ #include "intel_dp.h" #include "intel_gmbus.h" #include "intel_hdcp.h" +#include "intel_hdcp_regs.h" #include "intel_hdmi.h" #include "intel_lspcon.h" #include "intel_panel.h" @@ -1891,7 +1892,7 @@ int intel_hdmi_tmds_clock(int clock, int bpc, bool ycbcr420_output) * 1.5x for 12bpc * 1.25x for 10bpc */ - return clock * bpc / 8; + return DIV_ROUND_CLOSEST(clock * bpc, 8); } static bool intel_hdmi_source_bpc_possible(struct drm_i915_private *i915, int bpc) @@ -2001,6 +2002,15 @@ intel_hdmi_mode_valid(struct drm_connector *connector, clock *= 2; } + /* + * HDMI2.1 requires higher resolution modes like 8k60, 4K120 to be + * enumerated only if FRL is supported. Current platforms do not support + * FRL so prune the higher resolution modes that require doctclock more + * than 600MHz. + */ + if (clock > 600000) + return MODE_CLOCK_HIGH; + ycbcr_420_only = drm_mode_is_420_only(&connector->display_info, mode); status = intel_hdmi_mode_clock_valid(connector, clock, has_hdmi_sink, ycbcr_420_only); diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c index 5f8b4f481cff9a51dd1b2ed2a9c2f5286fbe2519..f7a2f485b177ce9e04aa2bb888d4488e686b1096 100644 --- a/drivers/gpu/drm/i915/display/intel_hotplug.c +++ b/drivers/gpu/drm/i915/display/intel_hotplug.c @@ -119,13 +119,13 @@ intel_connector_hpd_pin(struct intel_connector *connector) * responsible for further action. * * The number of IRQs that are allowed within @HPD_STORM_DETECT_PERIOD is - * stored in @dev_priv->hotplug.hpd_storm_threshold which defaults to + * stored in @dev_priv->display.hotplug.hpd_storm_threshold which defaults to * @HPD_STORM_DEFAULT_THRESHOLD. Long IRQs count as +10 to this threshold, and * short IRQs count as +1. If this threshold is exceeded, it's considered an * IRQ storm and the IRQ state is set to @HPD_MARK_DISABLED. * * By default, most systems will only count long IRQs towards - * &dev_priv->hotplug.hpd_storm_threshold. However, some older systems also + * &dev_priv->display.hotplug.hpd_storm_threshold. However, some older systems also * suffer from short IRQ storms and must also track these. Because short IRQ * storms are naturally caused by sideband interactions with DP MST devices, * short IRQ detection is only enabled for systems without DP MST support. @@ -140,7 +140,7 @@ intel_connector_hpd_pin(struct intel_connector *connector) static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, enum hpd_pin pin, bool long_hpd) { - struct i915_hotplug *hpd = &dev_priv->hotplug; + struct intel_hotplug *hpd = &dev_priv->display.hotplug; unsigned long start = hpd->stats[pin].last_jiffies; unsigned long end = start + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD); const int increment = long_hpd ? 10 : 1; @@ -148,7 +148,7 @@ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, bool storm = false; if (!threshold || - (!long_hpd && !dev_priv->hotplug.hpd_short_storm_enabled)) + (!long_hpd && !dev_priv->display.hotplug.hpd_short_storm_enabled)) return false; if (!time_in_range(jiffies, start, end)) { @@ -191,7 +191,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) pin = intel_connector_hpd_pin(connector); if (pin == HPD_NONE || - dev_priv->hotplug.stats[pin].state != HPD_MARK_DISABLED) + dev_priv->display.hotplug.stats[pin].state != HPD_MARK_DISABLED) continue; drm_info(&dev_priv->drm, @@ -199,7 +199,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) "switching from hotplug detection to polling\n", connector->base.name); - dev_priv->hotplug.stats[pin].state = HPD_DISABLED; + dev_priv->display.hotplug.stats[pin].state = HPD_DISABLED; connector->base.polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; hpd_disabled = true; @@ -209,7 +209,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) /* Enable polling and queue hotplug re-enabling. */ if (hpd_disabled) { drm_kms_helper_poll_enable(dev); - mod_delayed_work(system_wq, &dev_priv->hotplug.reenable_work, + mod_delayed_work(system_wq, &dev_priv->display.hotplug.reenable_work, msecs_to_jiffies(HPD_STORM_REENABLE_DELAY)); } } @@ -218,7 +218,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, typeof(*dev_priv), - hotplug.reenable_work.work); + display.hotplug.reenable_work.work); struct drm_device *dev = &dev_priv->drm; struct drm_connector_list_iter conn_iter; struct intel_connector *connector; @@ -233,7 +233,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) for_each_intel_connector_iter(connector, &conn_iter) { pin = intel_connector_hpd_pin(connector); if (pin == HPD_NONE || - dev_priv->hotplug.stats[pin].state != HPD_DISABLED) + dev_priv->display.hotplug.stats[pin].state != HPD_DISABLED) continue; if (connector->base.polled != connector->polled) @@ -245,8 +245,8 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) drm_connector_list_iter_end(&conn_iter); for_each_hpd_pin(pin) { - if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED) - dev_priv->hotplug.stats[pin].state = HPD_ENABLED; + if (dev_priv->display.hotplug.stats[pin].state == HPD_DISABLED) + dev_priv->display.hotplug.stats[pin].state = HPD_ENABLED; } intel_hpd_irq_setup(dev_priv); @@ -297,16 +297,16 @@ static bool intel_encoder_has_hpd_pulse(struct intel_encoder *encoder) static void i915_digport_work_func(struct work_struct *work) { struct drm_i915_private *dev_priv = - container_of(work, struct drm_i915_private, hotplug.dig_port_work); + container_of(work, struct drm_i915_private, display.hotplug.dig_port_work); u32 long_port_mask, short_port_mask; struct intel_encoder *encoder; u32 old_bits = 0; spin_lock_irq(&dev_priv->irq_lock); - long_port_mask = dev_priv->hotplug.long_port_mask; - dev_priv->hotplug.long_port_mask = 0; - short_port_mask = dev_priv->hotplug.short_port_mask; - dev_priv->hotplug.short_port_mask = 0; + long_port_mask = dev_priv->display.hotplug.long_port_mask; + dev_priv->display.hotplug.long_port_mask = 0; + short_port_mask = dev_priv->display.hotplug.short_port_mask; + dev_priv->display.hotplug.short_port_mask = 0; spin_unlock_irq(&dev_priv->irq_lock); for_each_intel_encoder(&dev_priv->drm, encoder) { @@ -335,9 +335,9 @@ static void i915_digport_work_func(struct work_struct *work) if (old_bits) { spin_lock_irq(&dev_priv->irq_lock); - dev_priv->hotplug.event_bits |= old_bits; + dev_priv->display.hotplug.event_bits |= old_bits; spin_unlock_irq(&dev_priv->irq_lock); - queue_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work, 0); + queue_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work, 0); } } @@ -353,10 +353,10 @@ void intel_hpd_trigger_irq(struct intel_digital_port *dig_port) struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); spin_lock_irq(&i915->irq_lock); - i915->hotplug.short_port_mask |= BIT(dig_port->base.port); + i915->display.hotplug.short_port_mask |= BIT(dig_port->base.port); spin_unlock_irq(&i915->irq_lock); - queue_work(i915->hotplug.dp_wq, &i915->hotplug.dig_port_work); + queue_work(i915->display.hotplug.dp_wq, &i915->display.hotplug.dig_port_work); } /* @@ -366,7 +366,7 @@ static void i915_hotplug_work_func(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, - hotplug.hotplug_work.work); + display.hotplug.hotplug_work.work); struct drm_device *dev = &dev_priv->drm; struct drm_connector_list_iter conn_iter; struct intel_connector *connector; @@ -379,10 +379,10 @@ static void i915_hotplug_work_func(struct work_struct *work) spin_lock_irq(&dev_priv->irq_lock); - hpd_event_bits = dev_priv->hotplug.event_bits; - dev_priv->hotplug.event_bits = 0; - hpd_retry_bits = dev_priv->hotplug.retry_bits; - dev_priv->hotplug.retry_bits = 0; + hpd_event_bits = dev_priv->display.hotplug.event_bits; + dev_priv->display.hotplug.event_bits = 0; + hpd_retry_bits = dev_priv->display.hotplug.retry_bits; + dev_priv->display.hotplug.retry_bits = 0; /* Enable polling for connectors which had HPD IRQ storms */ intel_hpd_irq_storm_switch_to_polling(dev_priv); @@ -435,10 +435,10 @@ static void i915_hotplug_work_func(struct work_struct *work) retry &= ~changed; if (retry) { spin_lock_irq(&dev_priv->irq_lock); - dev_priv->hotplug.retry_bits |= retry; + dev_priv->display.hotplug.retry_bits |= retry; spin_unlock_irq(&dev_priv->irq_lock); - mod_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work, + mod_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work, msecs_to_jiffies(HPD_RETRY_DELAY)); } } @@ -502,10 +502,10 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, if (long_hpd) { long_hpd_pulse_mask |= BIT(pin); - dev_priv->hotplug.long_port_mask |= BIT(port); + dev_priv->display.hotplug.long_port_mask |= BIT(port); } else { short_hpd_pulse_mask |= BIT(pin); - dev_priv->hotplug.short_port_mask |= BIT(port); + dev_priv->display.hotplug.short_port_mask |= BIT(port); } } @@ -516,7 +516,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, if (!(BIT(pin) & pin_mask)) continue; - if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED) { + if (dev_priv->display.hotplug.stats[pin].state == HPD_DISABLED) { /* * On GMCH platforms the interrupt mask bits only * prevent irq generation, not the setting of the @@ -529,7 +529,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, continue; } - if (dev_priv->hotplug.stats[pin].state != HPD_ENABLED) + if (dev_priv->display.hotplug.stats[pin].state != HPD_ENABLED) continue; /* @@ -540,13 +540,13 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, if (((short_hpd_pulse_mask | long_hpd_pulse_mask) & BIT(pin))) { long_hpd = long_hpd_pulse_mask & BIT(pin); } else { - dev_priv->hotplug.event_bits |= BIT(pin); + dev_priv->display.hotplug.event_bits |= BIT(pin); long_hpd = true; queue_hp = true; } if (intel_hpd_irq_storm_detect(dev_priv, pin, long_hpd)) { - dev_priv->hotplug.event_bits &= ~BIT(pin); + dev_priv->display.hotplug.event_bits &= ~BIT(pin); storm_detected = true; queue_hp = true; } @@ -567,9 +567,9 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, * deadlock. */ if (queue_dig) - queue_work(dev_priv->hotplug.dp_wq, &dev_priv->hotplug.dig_port_work); + queue_work(dev_priv->display.hotplug.dp_wq, &dev_priv->display.hotplug.dig_port_work); if (queue_hp) - queue_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work, 0); + queue_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work, 0); } /** @@ -594,8 +594,8 @@ void intel_hpd_init(struct drm_i915_private *dev_priv) return; for_each_hpd_pin(i) { - dev_priv->hotplug.stats[i].count = 0; - dev_priv->hotplug.stats[i].state = HPD_ENABLED; + dev_priv->display.hotplug.stats[i].count = 0; + dev_priv->display.hotplug.stats[i].state = HPD_ENABLED; } /* @@ -611,7 +611,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work) { struct drm_i915_private *dev_priv = container_of(work, struct drm_i915_private, - hotplug.poll_init_work); + display.hotplug.poll_init_work); struct drm_device *dev = &dev_priv->drm; struct drm_connector_list_iter conn_iter; struct intel_connector *connector; @@ -619,7 +619,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work) mutex_lock(&dev->mode_config.mutex); - enabled = READ_ONCE(dev_priv->hotplug.poll_enabled); + enabled = READ_ONCE(dev_priv->display.hotplug.poll_enabled); drm_connector_list_iter_begin(dev, &conn_iter); for_each_intel_connector_iter(connector, &conn_iter) { @@ -672,7 +672,7 @@ void intel_hpd_poll_enable(struct drm_i915_private *dev_priv) !INTEL_DISPLAY_ENABLED(dev_priv)) return; - WRITE_ONCE(dev_priv->hotplug.poll_enabled, true); + WRITE_ONCE(dev_priv->display.hotplug.poll_enabled, true); /* * We might already be holding dev->mode_config.mutex, so do this in a @@ -680,7 +680,7 @@ void intel_hpd_poll_enable(struct drm_i915_private *dev_priv) * As well, there's no issue if we race here since we always reschedule * this worker anyway */ - schedule_work(&dev_priv->hotplug.poll_init_work); + schedule_work(&dev_priv->display.hotplug.poll_init_work); } /** @@ -707,17 +707,17 @@ void intel_hpd_poll_disable(struct drm_i915_private *dev_priv) if (!HAS_DISPLAY(dev_priv)) return; - WRITE_ONCE(dev_priv->hotplug.poll_enabled, false); - schedule_work(&dev_priv->hotplug.poll_init_work); + WRITE_ONCE(dev_priv->display.hotplug.poll_enabled, false); + schedule_work(&dev_priv->display.hotplug.poll_init_work); } void intel_hpd_init_work(struct drm_i915_private *dev_priv) { - INIT_DELAYED_WORK(&dev_priv->hotplug.hotplug_work, + INIT_DELAYED_WORK(&dev_priv->display.hotplug.hotplug_work, i915_hotplug_work_func); - INIT_WORK(&dev_priv->hotplug.dig_port_work, i915_digport_work_func); - INIT_WORK(&dev_priv->hotplug.poll_init_work, i915_hpd_poll_init_work); - INIT_DELAYED_WORK(&dev_priv->hotplug.reenable_work, + INIT_WORK(&dev_priv->display.hotplug.dig_port_work, i915_digport_work_func); + INIT_WORK(&dev_priv->display.hotplug.poll_init_work, i915_hpd_poll_init_work); + INIT_DELAYED_WORK(&dev_priv->display.hotplug.reenable_work, intel_hpd_irq_storm_reenable_work); } @@ -728,17 +728,17 @@ void intel_hpd_cancel_work(struct drm_i915_private *dev_priv) spin_lock_irq(&dev_priv->irq_lock); - dev_priv->hotplug.long_port_mask = 0; - dev_priv->hotplug.short_port_mask = 0; - dev_priv->hotplug.event_bits = 0; - dev_priv->hotplug.retry_bits = 0; + dev_priv->display.hotplug.long_port_mask = 0; + dev_priv->display.hotplug.short_port_mask = 0; + dev_priv->display.hotplug.event_bits = 0; + dev_priv->display.hotplug.retry_bits = 0; spin_unlock_irq(&dev_priv->irq_lock); - cancel_work_sync(&dev_priv->hotplug.dig_port_work); - cancel_delayed_work_sync(&dev_priv->hotplug.hotplug_work); - cancel_work_sync(&dev_priv->hotplug.poll_init_work); - cancel_delayed_work_sync(&dev_priv->hotplug.reenable_work); + cancel_work_sync(&dev_priv->display.hotplug.dig_port_work); + cancel_delayed_work_sync(&dev_priv->display.hotplug.hotplug_work); + cancel_work_sync(&dev_priv->display.hotplug.poll_init_work); + cancel_delayed_work_sync(&dev_priv->display.hotplug.reenable_work); } bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin) @@ -749,8 +749,8 @@ bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin) return false; spin_lock_irq(&dev_priv->irq_lock); - if (dev_priv->hotplug.stats[pin].state == HPD_ENABLED) { - dev_priv->hotplug.stats[pin].state = HPD_DISABLED; + if (dev_priv->display.hotplug.stats[pin].state == HPD_ENABLED) { + dev_priv->display.hotplug.stats[pin].state = HPD_DISABLED; ret = true; } spin_unlock_irq(&dev_priv->irq_lock); @@ -764,6 +764,6 @@ void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin) return; spin_lock_irq(&dev_priv->irq_lock); - dev_priv->hotplug.stats[pin].state = HPD_ENABLED; + dev_priv->display.hotplug.stats[pin].state = HPD_ENABLED; spin_unlock_irq(&dev_priv->irq_lock); } diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c index 4970bf146c4a8a933987c499e528de9f5f881f18..dca6003ccac80f5da3753039f7b2fe92de0a2ee7 100644 --- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c +++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c @@ -73,8 +73,9 @@ #include "i915_drv.h" #include "intel_de.h" #include "intel_lpe_audio.h" +#include "intel_pci_config.h" -#define HAS_LPE_AUDIO(dev_priv) ((dev_priv)->audio.lpe.platdev != NULL) +#define HAS_LPE_AUDIO(dev_priv) ((dev_priv)->display.audio.lpe.platdev != NULL) static struct platform_device * lpe_audio_platdev_create(struct drm_i915_private *dev_priv) @@ -96,13 +97,13 @@ lpe_audio_platdev_create(struct drm_i915_private *dev_priv) return ERR_PTR(-ENOMEM); } - rsc[0].start = rsc[0].end = dev_priv->audio.lpe.irq; + rsc[0].start = rsc[0].end = dev_priv->display.audio.lpe.irq; rsc[0].flags = IORESOURCE_IRQ; rsc[0].name = "hdmi-lpe-audio-irq"; - rsc[1].start = pci_resource_start(pdev, 0) + + rsc[1].start = pci_resource_start(pdev, GTTMMADR_BAR) + I915_HDMI_LPE_AUDIO_BASE; - rsc[1].end = pci_resource_start(pdev, 0) + + rsc[1].end = pci_resource_start(pdev, GTTMMADR_BAR) + I915_HDMI_LPE_AUDIO_BASE + I915_HDMI_LPE_AUDIO_SIZE - 1; rsc[1].flags = IORESOURCE_MEM; rsc[1].name = "hdmi-lpe-audio-mmio"; @@ -148,7 +149,7 @@ static void lpe_audio_platdev_destroy(struct drm_i915_private *dev_priv) * than us fiddle with its internals. */ - platform_device_unregister(dev_priv->audio.lpe.platdev); + platform_device_unregister(dev_priv->display.audio.lpe.platdev); } static void lpe_audio_irq_unmask(struct irq_data *d) @@ -167,7 +168,7 @@ static struct irq_chip lpe_audio_irqchip = { static int lpe_audio_irq_init(struct drm_i915_private *dev_priv) { - int irq = dev_priv->audio.lpe.irq; + int irq = dev_priv->display.audio.lpe.irq; drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv)); irq_set_chip_and_handler_name(irq, @@ -204,15 +205,15 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv) { int ret; - dev_priv->audio.lpe.irq = irq_alloc_desc(0); - if (dev_priv->audio.lpe.irq < 0) { + dev_priv->display.audio.lpe.irq = irq_alloc_desc(0); + if (dev_priv->display.audio.lpe.irq < 0) { drm_err(&dev_priv->drm, "Failed to allocate IRQ desc: %d\n", - dev_priv->audio.lpe.irq); - ret = dev_priv->audio.lpe.irq; + dev_priv->display.audio.lpe.irq); + ret = dev_priv->display.audio.lpe.irq; goto err; } - drm_dbg(&dev_priv->drm, "irq = %d\n", dev_priv->audio.lpe.irq); + drm_dbg(&dev_priv->drm, "irq = %d\n", dev_priv->display.audio.lpe.irq); ret = lpe_audio_irq_init(dev_priv); @@ -223,10 +224,10 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv) goto err_free_irq; } - dev_priv->audio.lpe.platdev = lpe_audio_platdev_create(dev_priv); + dev_priv->display.audio.lpe.platdev = lpe_audio_platdev_create(dev_priv); - if (IS_ERR(dev_priv->audio.lpe.platdev)) { - ret = PTR_ERR(dev_priv->audio.lpe.platdev); + if (IS_ERR(dev_priv->display.audio.lpe.platdev)) { + ret = PTR_ERR(dev_priv->display.audio.lpe.platdev); drm_err(&dev_priv->drm, "Failed to create lpe audio platform device: %d\n", ret); @@ -241,10 +242,10 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv) return 0; err_free_irq: - irq_free_desc(dev_priv->audio.lpe.irq); + irq_free_desc(dev_priv->display.audio.lpe.irq); err: - dev_priv->audio.lpe.irq = -1; - dev_priv->audio.lpe.platdev = NULL; + dev_priv->display.audio.lpe.irq = -1; + dev_priv->display.audio.lpe.platdev = NULL; return ret; } @@ -262,7 +263,7 @@ void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv) if (!HAS_LPE_AUDIO(dev_priv)) return; - ret = generic_handle_irq(dev_priv->audio.lpe.irq); + ret = generic_handle_irq(dev_priv->display.audio.lpe.irq); if (ret) drm_err_ratelimited(&dev_priv->drm, "error handling LPE audio irq: %d\n", ret); @@ -303,10 +304,10 @@ void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv) lpe_audio_platdev_destroy(dev_priv); - irq_free_desc(dev_priv->audio.lpe.irq); + irq_free_desc(dev_priv->display.audio.lpe.irq); - dev_priv->audio.lpe.irq = -1; - dev_priv->audio.lpe.platdev = NULL; + dev_priv->display.audio.lpe.irq = -1; + dev_priv->display.audio.lpe.platdev = NULL; } /** @@ -333,7 +334,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv, if (!HAS_LPE_AUDIO(dev_priv)) return; - pdata = dev_get_platdata(&dev_priv->audio.lpe.platdev->dev); + pdata = dev_get_platdata(&dev_priv->display.audio.lpe.platdev->dev); ppdata = &pdata->port[port - PORT_B]; spin_lock_irqsave(&pdata->lpe_audio_slock, irqflags); @@ -361,7 +362,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv, } if (pdata->notify_audio_lpe) - pdata->notify_audio_lpe(dev_priv->audio.lpe.platdev, port - PORT_B); + pdata->notify_audio_lpe(dev_priv->display.audio.lpe.platdev, port - PORT_B); spin_unlock_irqrestore(&pdata->lpe_audio_slock, irqflags); } diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c index 730480ac3300ddaa5096decc59afd5a0bd4865de..9aa38e8141b52affdd160fcd9f40cdfa6a3f3f4d 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.c +++ b/drivers/gpu/drm/i915/display/intel_lvds.c @@ -837,12 +837,12 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) /* Skip init on machines we know falsely report LVDS */ if (dmi_check_system(intel_no_lvds)) { - drm_WARN(dev, !dev_priv->vbt.int_lvds_support, + drm_WARN(dev, !dev_priv->display.vbt.int_lvds_support, "Useless DMI match. Internal LVDS support disabled by VBT\n"); return; } - if (!dev_priv->vbt.int_lvds_support) { + if (!dev_priv->display.vbt.int_lvds_support) { drm_dbg_kms(&dev_priv->drm, "Internal LVDS support disabled by VBT\n"); return; diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c index f0e04d3904c6793d0cde35587535303ccd5c03b9..cbfabd58b75a6373401b8d07987ff4ba69648921 100644 --- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c +++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c @@ -23,6 +23,7 @@ #include "intel_modeset_setup.h" #include "intel_pch_display.h" #include "intel_pm.h" +#include "skl_watermark.h" static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, struct drm_modeset_acquire_ctx *ctx) @@ -30,11 +31,11 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, struct intel_encoder *encoder; struct drm_i915_private *i915 = to_i915(crtc->base.dev); struct intel_bw_state *bw_state = - to_intel_bw_state(i915->bw_obj.state); + to_intel_bw_state(i915->display.bw.obj.state); struct intel_cdclk_state *cdclk_state = - to_intel_cdclk_state(i915->cdclk.obj.state); + to_intel_cdclk_state(i915->display.cdclk.obj.state); struct intel_dbuf_state *dbuf_state = - to_intel_dbuf_state(i915->dbuf.obj.state); + to_intel_dbuf_state(i915->display.dbuf.obj.state); struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct intel_plane *plane; @@ -70,7 +71,7 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, drm_WARN_ON(&i915->drm, IS_ERR(temp_crtc_state) || ret); - i915->display->crtc_disable(to_intel_atomic_state(state), crtc); + i915->display.funcs.display->crtc_disable(to_intel_atomic_state(state), crtc); drm_atomic_state_put(state); @@ -415,9 +416,9 @@ static void readout_plane_state(struct drm_i915_private *i915) static void intel_modeset_readout_hw_state(struct drm_i915_private *i915) { struct intel_cdclk_state *cdclk_state = - to_intel_cdclk_state(i915->cdclk.obj.state); + to_intel_cdclk_state(i915->display.cdclk.obj.state); struct intel_dbuf_state *dbuf_state = - to_intel_dbuf_state(i915->dbuf.obj.state); + to_intel_dbuf_state(i915->display.dbuf.obj.state); enum pipe pipe; struct intel_crtc *crtc; struct intel_encoder *encoder; @@ -535,7 +536,7 @@ static void intel_modeset_readout_hw_state(struct drm_i915_private *i915) for_each_intel_crtc(&i915->drm, crtc) { struct intel_bw_state *bw_state = - to_intel_bw_state(i915->bw_obj.state); + to_intel_bw_state(i915->display.bw.obj.state); struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); struct intel_plane *plane; diff --git a/drivers/gpu/drm/i915/display/intel_modeset_verify.c b/drivers/gpu/drm/i915/display/intel_modeset_verify.c index a91586d77cb64d395d4de57ec482c60e7e6b729f..0fdcf2e6d57fedb6ccb4e7545b1f57ce578c05d7 100644 --- a/drivers/gpu/drm/i915/display/intel_modeset_verify.c +++ b/drivers/gpu/drm/i915/display/intel_modeset_verify.c @@ -15,8 +15,8 @@ #include "intel_display_types.h" #include "intel_fdi.h" #include "intel_modeset_verify.h" -#include "intel_pm.h" #include "intel_snps_phy.h" +#include "skl_watermark.h" /* * Cross check the actual hw state with our own modeset state tracking (and its @@ -94,10 +94,10 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv, /* * FDI already provided one idea for the dotclock. - * Yell if the encoder disagrees. + * Yell if the encoder disagrees. Allow for slight + * rounding differences. */ - drm_WARN(&dev_priv->drm, - !intel_fuzzy_clock_check(fdi_dotclock, dotclock), + drm_WARN(&dev_priv->drm, abs(fdi_dotclock - dotclock) > 1, "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n", fdi_dotclock, dotclock); } diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c index 1c0c745c142de628f413f95cc3289e9a2849b18d..caa07ef34f21ef75fb53a0ea6b8d4649161f8038 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.c +++ b/drivers/gpu/drm/i915/display/intel_opregion.c @@ -252,7 +252,7 @@ struct opregion_asle_ext { static int check_swsci_function(struct drm_i915_private *i915, u32 function) { - struct opregion_swsci *swsci = i915->opregion.swsci; + struct opregion_swsci *swsci = i915->display.opregion.swsci; u32 main_function, sub_function; if (!swsci) @@ -265,11 +265,11 @@ static int check_swsci_function(struct drm_i915_private *i915, u32 function) /* Check if we can call the function. See swsci_setup for details. */ if (main_function == SWSCI_SBCB) { - if ((i915->opregion.swsci_sbcb_sub_functions & + if ((i915->display.opregion.swsci_sbcb_sub_functions & (1 << sub_function)) == 0) return -EINVAL; } else if (main_function == SWSCI_GBDA) { - if ((i915->opregion.swsci_gbda_sub_functions & + if ((i915->display.opregion.swsci_gbda_sub_functions & (1 << sub_function)) == 0) return -EINVAL; } @@ -280,7 +280,7 @@ static int check_swsci_function(struct drm_i915_private *i915, u32 function) static int swsci(struct drm_i915_private *dev_priv, u32 function, u32 parm, u32 *parm_out) { - struct opregion_swsci *swsci = dev_priv->opregion.swsci; + struct opregion_swsci *swsci = dev_priv->display.opregion.swsci; struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); u32 scic, dslp; u16 swsci_val; @@ -462,7 +462,7 @@ static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp) { struct intel_connector *connector; struct drm_connector_list_iter conn_iter; - struct opregion_asle *asle = dev_priv->opregion.asle; + struct opregion_asle *asle = dev_priv->display.opregion.asle; struct drm_device *dev = &dev_priv->drm; drm_dbg(&dev_priv->drm, "bclp = 0x%08x\n", bclp); @@ -586,8 +586,8 @@ static void asle_work(struct work_struct *work) struct intel_opregion *opregion = container_of(work, struct intel_opregion, asle_work); struct drm_i915_private *dev_priv = - container_of(opregion, struct drm_i915_private, opregion); - struct opregion_asle *asle = dev_priv->opregion.asle; + container_of(opregion, struct drm_i915_private, display.opregion); + struct opregion_asle *asle = dev_priv->display.opregion.asle; u32 aslc_stat = 0; u32 aslc_req; @@ -635,8 +635,8 @@ static void asle_work(struct work_struct *work) void intel_opregion_asle_intr(struct drm_i915_private *dev_priv) { - if (dev_priv->opregion.asle) - schedule_work(&dev_priv->opregion.asle_work); + if (dev_priv->display.opregion.asle) + schedule_work(&dev_priv->display.opregion.asle_work); } #define ACPI_EV_DISPLAY_SWITCH (1<<0) @@ -692,7 +692,7 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val) static void intel_didl_outputs(struct drm_i915_private *dev_priv) { - struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_opregion *opregion = &dev_priv->display.opregion; struct intel_connector *connector; struct drm_connector_list_iter conn_iter; int i = 0, max_outputs; @@ -731,7 +731,7 @@ static void intel_didl_outputs(struct drm_i915_private *dev_priv) static void intel_setup_cadls(struct drm_i915_private *dev_priv) { - struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_opregion *opregion = &dev_priv->display.opregion; struct intel_connector *connector; struct drm_connector_list_iter conn_iter; int i = 0; @@ -761,7 +761,7 @@ static void intel_setup_cadls(struct drm_i915_private *dev_priv) static void swsci_setup(struct drm_i915_private *dev_priv) { - struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_opregion *opregion = &dev_priv->display.opregion; bool requested_callbacks = false; u32 tmp; @@ -839,7 +839,7 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = { static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv) { - struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_opregion *opregion = &dev_priv->display.opregion; const struct firmware *fw = NULL; const char *name = dev_priv->params.vbt_firmware; int ret; @@ -879,7 +879,7 @@ static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv) int intel_opregion_setup(struct drm_i915_private *dev_priv) { - struct intel_opregion *opregion = &dev_priv->opregion; + struct intel_opregion *opregion = &dev_priv->display.opregion; struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); u32 asls, mboxes; char buf[sizeof(OPREGION_SIGNATURE)]; @@ -1106,7 +1106,7 @@ struct edid *intel_opregion_get_edid(struct intel_connector *intel_connector) { struct drm_connector *connector = &intel_connector->base; struct drm_i915_private *i915 = to_i915(connector->dev); - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; const void *in_edid; const struct edid *edid; struct edid *new_edid; @@ -1141,7 +1141,7 @@ struct edid *intel_opregion_get_edid(struct intel_connector *intel_connector) bool intel_opregion_headless_sku(struct drm_i915_private *i915) { - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; struct opregion_header *header = opregion->header; if (!header || header->over.major < 2 || @@ -1153,7 +1153,7 @@ bool intel_opregion_headless_sku(struct drm_i915_private *i915) void intel_opregion_register(struct drm_i915_private *i915) { - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; if (!opregion->header) return; @@ -1169,7 +1169,7 @@ void intel_opregion_register(struct drm_i915_private *i915) void intel_opregion_resume(struct drm_i915_private *i915) { - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; if (!opregion->header) return; @@ -1200,7 +1200,7 @@ void intel_opregion_resume(struct drm_i915_private *i915) void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state) { - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; if (!opregion->header) return; @@ -1210,7 +1210,7 @@ void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state) if (opregion->asle) opregion->asle->ardy = ASLE_ARDY_NOT_READY; - cancel_work_sync(&i915->opregion.asle_work); + cancel_work_sync(&i915->display.opregion.asle_work); if (opregion->acpi) opregion->acpi->drdy = 0; @@ -1218,7 +1218,7 @@ void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state) void intel_opregion_unregister(struct drm_i915_private *i915) { - struct intel_opregion *opregion = &i915->opregion; + struct intel_opregion *opregion = &i915->display.opregion; intel_opregion_suspend(i915, PCI_D1); diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index 79ed8bd04a072b526d0186f0c2d9968905521161..c12bdca8da9ba60b0ccc55ce30e49ad1e7a43102 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -211,9 +211,9 @@ static void i830_overlay_clock_gating(struct drm_i915_private *dev_priv, /* WA_OVERLAY_CLKGATE:alm */ if (enable) - intel_de_write(dev_priv, DSPCLK_GATE_D, 0); + intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), 0); else - intel_de_write(dev_priv, DSPCLK_GATE_D, + intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), OVRUNIT_CLOCK_GATE_DISABLE); /* WA_DISABLE_L2CACHE_CLOCK_GATING:alm */ @@ -487,7 +487,7 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) void intel_overlay_reset(struct drm_i915_private *dev_priv) { - struct intel_overlay *overlay = dev_priv->overlay; + struct intel_overlay *overlay = dev_priv->display.overlay; if (!overlay) return; @@ -1113,7 +1113,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *new_bo; int ret; - overlay = dev_priv->overlay; + overlay = dev_priv->display.overlay; if (!overlay) { drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n"); return -ENODEV; @@ -1273,7 +1273,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, struct intel_overlay *overlay; int ret; - overlay = dev_priv->overlay; + overlay = dev_priv->display.overlay; if (!overlay) { drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n"); return -ENODEV; @@ -1416,7 +1416,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) update_polyphase_filter(overlay->regs); update_reg_attrs(overlay, overlay->regs); - dev_priv->overlay = overlay; + dev_priv->display.overlay = overlay; drm_info(&dev_priv->drm, "Initialized overlay support.\n"); return; @@ -1428,7 +1428,7 @@ void intel_overlay_cleanup(struct drm_i915_private *dev_priv) { struct intel_overlay *overlay; - overlay = fetch_and_zero(&dev_priv->overlay); + overlay = fetch_and_zero(&dev_priv->display.overlay); if (!overlay) return; @@ -1457,7 +1457,7 @@ struct intel_overlay_error_state { struct intel_overlay_error_state * intel_overlay_capture_error_state(struct drm_i915_private *dev_priv) { - struct intel_overlay *overlay = dev_priv->overlay; + struct intel_overlay *overlay = dev_priv->display.overlay; struct intel_overlay_error_state *error; if (!overlay || !overlay->active) diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index 237a40623dd7b8dab8e9efb53e22f832740c2e6f..a3a3f9fe4342f476e35dda69bc0eefdce1409cfd 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -37,13 +37,14 @@ #include "intel_display_types.h" #include "intel_drrs.h" #include "intel_panel.h" +#include "intel_quirks.h" bool intel_panel_use_ssc(struct drm_i915_private *i915) { if (i915->params.panel_use_ssc >= 0) return i915->params.panel_use_ssc != 0; - return i915->vbt.lvds_use_ssc - && !(i915->quirks & QUIRK_LVDS_SSC_DISABLE); + return i915->display.vbt.lvds_use_ssc && + !intel_has_quirk(i915, QUIRK_LVDS_SSC_DISABLE); } const struct drm_display_mode * @@ -81,15 +82,14 @@ static bool is_alt_drrs_mode(const struct drm_display_mode *mode, mode->clock != preferred_mode->clock; } -static bool is_alt_vrr_mode(const struct drm_display_mode *mode, - const struct drm_display_mode *preferred_mode) +static bool is_alt_fixed_mode(const struct drm_display_mode *mode, + const struct drm_display_mode *preferred_mode) { return drm_mode_match(mode, preferred_mode, DRM_MODE_MATCH_FLAGS | DRM_MODE_MATCH_3D_FLAGS) && mode->hdisplay == preferred_mode->hdisplay && - mode->vdisplay == preferred_mode->vdisplay && - mode->clock != preferred_mode->clock; + mode->vdisplay == preferred_mode->vdisplay; } const struct drm_display_mode * @@ -114,6 +114,21 @@ intel_panel_downclock_mode(struct intel_connector *connector, return best_mode; } +const struct drm_display_mode * +intel_panel_highest_mode(struct intel_connector *connector, + const struct drm_display_mode *adjusted_mode) +{ + const struct drm_display_mode *fixed_mode, *best_mode = adjusted_mode; + + /* pick the fixed_mode that has the highest clock */ + list_for_each_entry(fixed_mode, &connector->panel.fixed_modes, head) { + if (fixed_mode->clock > best_mode->clock) + best_mode = fixed_mode; + } + + return best_mode; +} + int intel_panel_get_modes(struct intel_connector *connector) { const struct drm_display_mode *fixed_mode; @@ -172,19 +187,7 @@ int intel_panel_compute_config(struct intel_connector *connector, return 0; } -static bool is_alt_fixed_mode(const struct drm_display_mode *mode, - const struct drm_display_mode *preferred_mode, - bool has_vrr) -{ - /* is_alt_drrs_mode() is a subset of is_alt_vrr_mode() */ - if (has_vrr) - return is_alt_vrr_mode(mode, preferred_mode); - else - return is_alt_drrs_mode(mode, preferred_mode); -} - -static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connector, - bool has_vrr) +static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); const struct drm_display_mode *preferred_mode = @@ -192,7 +195,7 @@ static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connect struct drm_display_mode *mode, *next; list_for_each_entry_safe(mode, next, &connector->base.probed_modes, head) { - if (!is_alt_fixed_mode(mode, preferred_mode, has_vrr)) + if (!is_alt_fixed_mode(mode, preferred_mode)) continue; drm_dbg_kms(&dev_priv->drm, @@ -255,7 +258,7 @@ void intel_panel_add_edid_fixed_modes(struct intel_connector *connector, { intel_panel_add_edid_preferred_mode(connector); if (intel_panel_preferred_fixed_mode(connector) && (has_drrs || has_vrr)) - intel_panel_add_edid_alt_fixed_modes(connector, has_vrr); + intel_panel_add_edid_alt_fixed_modes(connector); intel_panel_destroy_probed_modes(connector); } diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h index b087c0c3cc6db54610f2241f50c5896ed34e6986..eff3ffd3d08255af6b29660038395fd5437e015e 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.h +++ b/drivers/gpu/drm/i915/display/intel_panel.h @@ -31,6 +31,9 @@ intel_panel_fixed_mode(struct intel_connector *connector, const struct drm_display_mode * intel_panel_downclock_mode(struct intel_connector *connector, const struct drm_display_mode *adjusted_mode); +const struct drm_display_mode * +intel_panel_highest_mode(struct intel_connector *connector, + const struct drm_display_mode *adjusted_mode); int intel_panel_get_modes(struct intel_connector *connector); enum drrs_type intel_panel_drrs_type(struct intel_connector *connector); enum drm_mode_status diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.c b/drivers/gpu/drm/i915/display/intel_pch_refclk.c index 9934c8a9e240f2b0ac13f46a72c78f6d92ce8594..a66097cdc1e0b3181eb6807aa0328f74a60c6d82 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_refclk.c +++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.c @@ -167,6 +167,15 @@ static void lpt_compute_iclkip(struct iclkip_params *p, int clock) } } +int lpt_iclkip(const struct intel_crtc_state *crtc_state) +{ + struct iclkip_params p; + + lpt_compute_iclkip(&p, crtc_state->hw.adjusted_mode.crtc_clock); + + return lpt_iclkip_freq(&p); +} + /* Program iCLKIP clock to the desired frequency */ void lpt_program_iclkip(const struct intel_crtc_state *crtc_state) { @@ -179,6 +188,7 @@ void lpt_program_iclkip(const struct intel_crtc_state *crtc_state) lpt_disable_iclkip(dev_priv); lpt_compute_iclkip(&p, clock); + drm_WARN_ON(&dev_priv->drm, lpt_iclkip_freq(&p) != clock); /* This should not happen with any sane values */ drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(p.divsel) & @@ -514,7 +524,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv) } if (HAS_PCH_IBX(dev_priv)) { - has_ck505 = dev_priv->vbt.display_clock_mode; + has_ck505 = dev_priv->display.vbt.display_clock_mode; can_ssc = has_ck505; } else { has_ck505 = false; @@ -522,7 +532,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv) } /* Check if any DPLLs are using the SSC source */ - for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { + for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) { u32 temp = intel_de_read(dev_priv, PCH_DPLL(i)); if (!(temp & DPLL_VCO_ENABLE)) @@ -654,7 +664,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv) } } - BUG_ON(val != final); + drm_WARN_ON(&dev_priv->drm, val != final); } /* diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.h b/drivers/gpu/drm/i915/display/intel_pch_refclk.h index 12ab2c75a800a1ad6dcb7d5be468b54afec955ad..9bcf56629f24c3e27d1fdbd63c8785916b38f1cd 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_refclk.h +++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.h @@ -14,6 +14,7 @@ struct intel_crtc_state; void lpt_program_iclkip(const struct intel_crtc_state *crtc_state); void lpt_disable_iclkip(struct drm_i915_private *dev_priv); int lpt_get_iclkip(struct drm_i915_private *dev_priv); +int lpt_iclkip(const struct intel_crtc_state *crtc_state); void intel_init_pch_refclk(struct drm_i915_private *dev_priv); void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c index d10f27d0b7b091662c770f522223c107ca167cfc..76be796df255d94f67d9b8dd3070d08d095ff701 100644 --- a/drivers/gpu/drm/i915/display/intel_plane_initial.c +++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c @@ -311,7 +311,7 @@ void intel_crtc_initial_plane_config(struct intel_crtc *crtc) * can even allow for smooth boot transitions if the BIOS * fb is large enough for the active pipe configuration. */ - dev_priv->display->get_initial_plane_config(crtc, &plane_config); + dev_priv->display.funcs.display->get_initial_plane_config(crtc, &plane_config); /* * If the fb is shared between multiple heads, we'll diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c index 1b21a341962f4a149a523286a76d593be35568f1..21944f5bf3a83ccd79e276694d6b4cc9aaba9058 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.c +++ b/drivers/gpu/drm/i915/display/intel_pps.c @@ -12,6 +12,7 @@ #include "intel_dpll.h" #include "intel_lvds.h" #include "intel_pps.h" +#include "intel_quirks.h" static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, enum pipe pipe); @@ -28,7 +29,7 @@ intel_wakeref_t intel_pps_lock(struct intel_dp *intel_dp) * See intel_pps_reset_all() why we need a power domain reference here. */ wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE); - mutex_lock(&dev_priv->pps_mutex); + mutex_lock(&dev_priv->display.pps.mutex); return wakeref; } @@ -38,7 +39,7 @@ intel_wakeref_t intel_pps_unlock(struct intel_dp *intel_dp, { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - mutex_unlock(&dev_priv->pps_mutex); + mutex_unlock(&dev_priv->display.pps.mutex); intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref); return 0; @@ -163,7 +164,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); enum pipe pipe; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); /* We should never land here with regular DP ports */ drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp)); @@ -212,7 +213,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp) struct intel_connector *connector = intel_dp->attached_connector; int backlight_controller = connector->panel.vbt.backlight.controller; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); /* We should never land here with regular DP ports */ drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp)); @@ -282,7 +283,7 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp) struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); enum port port = dig_port->base.port; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); /* try to find a pipe with this port selected */ /* first pick one where the panel is on */ @@ -407,7 +408,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && intel_dp->pps.pps_pipe == INVALID_PIPE) @@ -420,7 +421,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && intel_dp->pps.pps_pipe == INVALID_PIPE) @@ -463,7 +464,7 @@ static void wait_panel_status(struct intel_dp *intel_dp, struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); i915_reg_t pp_stat_reg, pp_ctrl_reg; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); intel_pps_verify_state(intel_dp); @@ -556,7 +557,7 @@ static u32 ilk_get_pp_control(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); u32 control; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); control = intel_de_read(dev_priv, _pp_ctrl_reg(intel_dp)); if (drm_WARN_ON(&dev_priv->drm, !HAS_DDI(dev_priv) && @@ -580,7 +581,7 @@ bool intel_pps_vdd_on_unlocked(struct intel_dp *intel_dp) i915_reg_t pp_stat_reg, pp_ctrl_reg; bool need_to_disable = !intel_dp->pps.want_panel_vdd; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!intel_dp_is_edp(intel_dp)) return false; @@ -657,7 +658,7 @@ static void intel_pps_vdd_off_sync_unlocked(struct intel_dp *intel_dp) u32 pp; i915_reg_t pp_stat_reg, pp_ctrl_reg; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); drm_WARN_ON(&dev_priv->drm, intel_dp->pps.want_panel_vdd); @@ -748,7 +749,7 @@ void intel_pps_vdd_off_unlocked(struct intel_dp *intel_dp, bool sync) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!intel_dp_is_edp(intel_dp)) return; @@ -771,7 +772,7 @@ void intel_pps_on_unlocked(struct intel_dp *intel_dp) u32 pp; i915_reg_t pp_ctrl_reg; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!intel_dp_is_edp(intel_dp)) return; @@ -832,7 +833,7 @@ void intel_pps_off_unlocked(struct intel_dp *intel_dp) u32 pp; i915_reg_t pp_ctrl_reg; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!intel_dp_is_edp(intel_dp)) return; @@ -991,7 +992,7 @@ static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, { struct intel_encoder *encoder; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); for_each_intel_dp(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); @@ -1021,7 +1022,7 @@ void vlv_pps_init(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); drm_WARN_ON(&dev_priv->drm, intel_dp->pps.active_pipe != INVALID_PIPE); @@ -1064,7 +1065,7 @@ static void pps_vdd_init(struct intel_dp *intel_dp) struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!edp_have_panel_vdd(intel_dp)) return; @@ -1176,7 +1177,7 @@ static void pps_init_delays_bios(struct intel_dp *intel_dp, { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); if (!pps_delays_valid(&intel_dp->pps.bios_pps_delays)) intel_pps_readout_hw_state(intel_dp, &intel_dp->pps.bios_pps_delays); @@ -1202,7 +1203,7 @@ static void pps_init_delays_vbt(struct intel_dp *intel_dp, * just fails to power back on. Increasing the delay to 800ms * seems sufficient to avoid this problem. */ - if (dev_priv->quirks & QUIRK_INCREASE_T12_DELAY) { + if (intel_has_quirk(dev_priv, QUIRK_INCREASE_T12_DELAY)) { vbt->t11_t12 = max_t(u16, vbt->t11_t12, 1300 * 10); drm_dbg_kms(&dev_priv->drm, "Increasing T12 panel delay as per the quirk to %d\n", @@ -1223,7 +1224,7 @@ static void pps_init_delays_spec(struct intel_dp *intel_dp, { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); /* Upper limits from eDP 1.3 spec. Note that we use the clunky units of * our hw here, which are all in 100usec. */ @@ -1246,7 +1247,7 @@ static void pps_init_delays(struct intel_dp *intel_dp) struct edp_power_seq cur, vbt, spec, *final = &intel_dp->pps.pps_delays; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); /* already initialized? */ if (pps_delays_valid(final)) @@ -1312,7 +1313,7 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd enum port port = dp_to_dig_port(intel_dp)->base.port; const struct edp_power_seq *seq = &intel_dp->pps.pps_delays; - lockdep_assert_held(&dev_priv->pps_mutex); + lockdep_assert_held(&dev_priv->display.pps.mutex); intel_pps_get_registers(intel_dp, ®s); @@ -1487,11 +1488,11 @@ void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv) void intel_pps_setup(struct drm_i915_private *i915) { if (HAS_PCH_SPLIT(i915) || IS_GEMINILAKE(i915) || IS_BROXTON(i915)) - i915->pps_mmio_base = PCH_PPS_BASE; + i915->display.pps.mmio_base = PCH_PPS_BASE; else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) - i915->pps_mmio_base = VLV_PPS_BASE; + i915->display.pps.mmio_base = VLV_PPS_BASE; else - i915->pps_mmio_base = PPS_BASE; + i915->display.pps.mmio_base = PPS_BASE; } void assert_pps_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index e6a870641cd2532d14337994c71debcae98441d0..d4cce627d7a87f4b51575f1a45994f699130e019 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -116,34 +116,56 @@ static bool psr2_global_enabled(struct intel_dp *intel_dp) } } +static u32 psr_irq_psr_error_bit_get(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_ERROR : + EDP_PSR_ERROR(intel_dp->psr.transcoder); +} + +static u32 psr_irq_post_exit_bit_get(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_POST_EXIT : + EDP_PSR_POST_EXIT(intel_dp->psr.transcoder); +} + +static u32 psr_irq_pre_entry_bit_get(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_PRE_ENTRY : + EDP_PSR_PRE_ENTRY(intel_dp->psr.transcoder); +} + +static u32 psr_irq_mask_get(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + return DISPLAY_VER(dev_priv) >= 12 ? TGL_PSR_MASK : + EDP_PSR_MASK(intel_dp->psr.transcoder); +} + static void psr_irq_control(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - enum transcoder trans_shift; i915_reg_t imr_reg; u32 mask, val; - /* - * gen12+ has registers relative to transcoder and one per transcoder - * using the same bit definition: handle it as TRANSCODER_EDP to force - * 0 shift in bit definition - */ - if (DISPLAY_VER(dev_priv) >= 12) { - trans_shift = 0; + if (DISPLAY_VER(dev_priv) >= 12) imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder); - } else { - trans_shift = intel_dp->psr.transcoder; + else imr_reg = EDP_PSR_IMR; - } - mask = EDP_PSR_ERROR(trans_shift); + mask = psr_irq_psr_error_bit_get(intel_dp); if (intel_dp->psr.debug & I915_PSR_DEBUG_IRQ) - mask |= EDP_PSR_POST_EXIT(trans_shift) | - EDP_PSR_PRE_ENTRY(trans_shift); + mask |= psr_irq_post_exit_bit_get(intel_dp) | + psr_irq_pre_entry_bit_get(intel_dp); - /* Warning: it is masking/setting reserved bits too */ val = intel_de_read(dev_priv, imr_reg); - val &= ~EDP_PSR_TRANS_MASK(trans_shift); + val &= ~psr_irq_mask_get(intel_dp); val |= ~mask; intel_de_write(dev_priv, imr_reg, val); } @@ -191,25 +213,21 @@ void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir) enum transcoder cpu_transcoder = intel_dp->psr.transcoder; struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); ktime_t time_ns = ktime_get(); - enum transcoder trans_shift; i915_reg_t imr_reg; - if (DISPLAY_VER(dev_priv) >= 12) { - trans_shift = 0; + if (DISPLAY_VER(dev_priv) >= 12) imr_reg = TRANS_PSR_IMR(intel_dp->psr.transcoder); - } else { - trans_shift = intel_dp->psr.transcoder; + else imr_reg = EDP_PSR_IMR; - } - if (psr_iir & EDP_PSR_PRE_ENTRY(trans_shift)) { + if (psr_iir & psr_irq_pre_entry_bit_get(intel_dp)) { intel_dp->psr.last_entry_attempt = time_ns; drm_dbg_kms(&dev_priv->drm, "[transcoder %s] PSR entry attempt in 2 vblanks\n", transcoder_name(cpu_transcoder)); } - if (psr_iir & EDP_PSR_POST_EXIT(trans_shift)) { + if (psr_iir & psr_irq_post_exit_bit_get(intel_dp)) { intel_dp->psr.last_exit = time_ns; drm_dbg_kms(&dev_priv->drm, "[transcoder %s] PSR exit completed\n", @@ -226,7 +244,7 @@ void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir) } } - if (psr_iir & EDP_PSR_ERROR(trans_shift)) { + if (psr_iir & psr_irq_psr_error_bit_get(intel_dp)) { u32 val; drm_warn(&dev_priv->drm, "[transcoder %s] PSR aux error\n", @@ -243,7 +261,7 @@ void intel_psr_irq_handler(struct intel_dp *intel_dp, u32 psr_iir) * or unset irq_aux_error. */ val = intel_de_read(dev_priv, imr_reg); - val |= EDP_PSR_ERROR(trans_shift); + val |= psr_irq_psr_error_bit_get(intel_dp); intel_de_write(dev_priv, imr_reg, val); schedule_work(&intel_dp->psr.work); @@ -706,7 +724,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp, if (crtc_state->enable_psr2_sel_fetch) return; - if (!(dev_priv->dmc.allowed_dc_mask & DC_STATE_EN_DC3CO)) + if (!(dev_priv->display.dmc.allowed_dc_mask & DC_STATE_EN_DC3CO)) return; if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state)) @@ -805,13 +823,14 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d hblank_total = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start; hblank_ns = div_u64(1000000ULL * hblank_total, adjusted_mode->crtc_clock); - /* From spec: (72 / number of lanes) * 1000 / symbol clock frequency MHz */ - req_ns = (72 / crtc_state->lane_count) * 1000 / (crtc_state->port_clock / 1000); + /* From spec: ((60 / number of lanes) + 11) * 1000 / symbol clock frequency MHz */ + req_ns = ((60 / crtc_state->lane_count) + 11) * 1000 / (crtc_state->port_clock / 1000); if ((hblank_ns - req_ns) > 100) return true; - if (DISPLAY_VER(dev_priv) < 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b) + /* Not supported <13 / Wa_22012279113:adl-p */ + if (DISPLAY_VER(dev_priv) <= 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b) return false; crtc_state->req_psr2_sdp_prior_scanline = true; @@ -1193,14 +1212,12 @@ static bool psr_interrupt_error_check(struct intel_dp *intel_dp) * first time that PSR HW tries to activate so lets keep PSR disabled * to avoid any rendering problems. */ - if (DISPLAY_VER(dev_priv) >= 12) { + if (DISPLAY_VER(dev_priv) >= 12) val = intel_de_read(dev_priv, TRANS_PSR_IIR(intel_dp->psr.transcoder)); - val &= EDP_PSR_ERROR(0); - } else { + else val = intel_de_read(dev_priv, EDP_PSR_IIR); - val &= EDP_PSR_ERROR(intel_dp->psr.transcoder); - } + val &= psr_irq_psr_error_bit_get(intel_dp); if (val) { intel_dp->psr.sink_not_reliable = true; drm_dbg_kms(&dev_priv->drm, @@ -1721,8 +1738,6 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, new_plane_state, i) { struct drm_rect src, damaged_area = { .x1 = 0, .y1 = -1, .x2 = INT_MAX }; - struct drm_atomic_helper_damage_iter iter; - struct drm_rect clip; if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc) continue; @@ -1767,22 +1782,18 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, continue; } - drm_rect_fp_to_int(&src, &new_plane_state->uapi.src); + src = drm_plane_state_src(&new_plane_state->uapi); + drm_rect_fp_to_int(&src, &src); - drm_atomic_helper_damage_iter_init(&iter, - &old_plane_state->uapi, - &new_plane_state->uapi); - drm_atomic_for_each_plane_damage(&iter, &clip) { - if (drm_rect_intersect(&clip, &src)) - clip_area_update(&damaged_area, &clip, - &crtc_state->pipe_src); - } - - if (damaged_area.y1 == -1) + if (!drm_atomic_helper_damage_merged(&old_plane_state->uapi, + &new_plane_state->uapi, &damaged_area)) continue; damaged_area.y1 += new_plane_state->uapi.dst.y1 - src.y1; damaged_area.y2 += new_plane_state->uapi.dst.y1 - src.y1; + damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1; + damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1; + clip_area_update(&pipe_clip, &damaged_area, &crtc_state->pipe_src); } @@ -1863,7 +1874,9 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *i915 = to_i915(state->base.dev); - const struct intel_crtc_state *crtc_state = + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + const struct intel_crtc_state *new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); struct intel_encoder *encoder; @@ -1871,7 +1884,7 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state, return; for_each_intel_encoder_mask_with_psr(state->base.dev, encoder, - crtc_state->uapi.encoder_mask) { + old_crtc_state->uapi.encoder_mask) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_psr *psr = &intel_dp->psr; bool needs_to_disable = false; @@ -1884,10 +1897,10 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state, * - All planes will go inactive * - Changing between PSR versions */ - needs_to_disable |= intel_crtc_needs_modeset(crtc_state); - needs_to_disable |= !crtc_state->has_psr; - needs_to_disable |= !crtc_state->active_planes; - needs_to_disable |= crtc_state->has_psr2 != psr->psr2_enabled; + needs_to_disable |= intel_crtc_needs_modeset(new_crtc_state); + needs_to_disable |= !new_crtc_state->has_psr; + needs_to_disable |= !new_crtc_state->active_planes; + needs_to_disable |= new_crtc_state->has_psr2 != psr->psr2_enabled; if (psr->enabled && needs_to_disable) intel_psr_disable_locked(intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index c8488f5ebd044ddd945dacbd6435a48a71f095f3..6e48d3bcdfec5acf4c1d582bf6f7b6cf95d46fdd 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -9,12 +9,17 @@ #include "intel_display_types.h" #include "intel_quirks.h" +static void intel_set_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk) +{ + i915->display.quirks.mask |= BIT(quirk); +} + /* * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason */ static void quirk_ssc_force_disable(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_LVDS_SSC_DISABLE; + intel_set_quirk(i915, QUIRK_LVDS_SSC_DISABLE); drm_info(&i915->drm, "applying lvds SSC disable quirk\n"); } @@ -24,14 +29,14 @@ static void quirk_ssc_force_disable(struct drm_i915_private *i915) */ static void quirk_invert_brightness(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_INVERT_BRIGHTNESS; + intel_set_quirk(i915, QUIRK_INVERT_BRIGHTNESS); drm_info(&i915->drm, "applying inverted panel brightness quirk\n"); } /* Some VBT's incorrectly indicate no backlight is present */ static void quirk_backlight_present(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_BACKLIGHT_PRESENT; + intel_set_quirk(i915, QUIRK_BACKLIGHT_PRESENT); drm_info(&i915->drm, "applying backlight present quirk\n"); } @@ -40,7 +45,7 @@ static void quirk_backlight_present(struct drm_i915_private *i915) */ static void quirk_increase_t12_delay(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_INCREASE_T12_DELAY; + intel_set_quirk(i915, QUIRK_INCREASE_T12_DELAY); drm_info(&i915->drm, "Applying T12 delay quirk\n"); } @@ -50,13 +55,13 @@ static void quirk_increase_t12_delay(struct drm_i915_private *i915) */ static void quirk_increase_ddi_disabled_time(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME; + intel_set_quirk(i915, QUIRK_INCREASE_DDI_DISABLED_TIME); drm_info(&i915->drm, "Applying Increase DDI Disabled quirk\n"); } static void quirk_no_pps_backlight_power_hook(struct drm_i915_private *i915) { - i915->quirks |= QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK; + intel_set_quirk(i915, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK); drm_info(&i915->drm, "Applying no pps backlight power quirk\n"); } @@ -191,6 +196,9 @@ static struct intel_quirk intel_quirks[] = { /* ASRock ITX*/ { 0x3185, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, { 0x3184, 0x1849, 0x2212, quirk_increase_ddi_disabled_time }, + /* ECS Liva Q2 */ + { 0x3185, 0x1019, 0xa94d, quirk_increase_ddi_disabled_time }, + { 0x3184, 0x1019, 0xa94d, quirk_increase_ddi_disabled_time }, }; void intel_init_quirks(struct drm_i915_private *i915) @@ -213,3 +221,8 @@ void intel_init_quirks(struct drm_i915_private *i915) intel_dmi_quirks[i].hook(i915); } } + +bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk) +{ + return i915->display.quirks.mask & BIT(quirk); +} diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h index b0fcff142a56f129fb0339076457738fed035eb0..10a4d163149fd1d37abf3b8c5d20335566976537 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.h +++ b/drivers/gpu/drm/i915/display/intel_quirks.h @@ -6,8 +6,20 @@ #ifndef __INTEL_QUIRKS_H__ #define __INTEL_QUIRKS_H__ +#include + struct drm_i915_private; -void intel_init_quirks(struct drm_i915_private *dev_priv); +enum intel_quirk_id { + QUIRK_BACKLIGHT_PRESENT, + QUIRK_INCREASE_DDI_DISABLED_TIME, + QUIRK_INCREASE_T12_DELAY, + QUIRK_INVERT_BRIGHTNESS, + QUIRK_LVDS_SSC_DISABLE, + QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK, +}; + +void intel_init_quirks(struct drm_i915_private *i915); +bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk); #endif /* __INTEL_QUIRKS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 19122bc6d2abaf241b32bc8ce81f57c4de2cbf66..f5b744bef18ffe974a5b41b3ae0fd49002c24f00 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -2016,7 +2016,7 @@ intel_sdvo_get_analog_edid(struct drm_connector *connector) return drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, - dev_priv->vbt.crt_ddc_pin)); + dev_priv->display.vbt.crt_ddc_pin)); } static enum drm_connector_status @@ -2581,9 +2581,9 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv, struct sdvo_device_mapping *mapping; if (sdvo->port == PORT_B) - mapping = &dev_priv->vbt.sdvo_mappings[0]; + mapping = &dev_priv->display.vbt.sdvo_mappings[0]; else - mapping = &dev_priv->vbt.sdvo_mappings[1]; + mapping = &dev_priv->display.vbt.sdvo_mappings[1]; if (mapping->initialized) sdvo->ddc_bus = 1 << ((mapping->ddc_pin & 0xf0) >> 4); @@ -2599,9 +2599,9 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv, u8 pin; if (sdvo->port == PORT_B) - mapping = &dev_priv->vbt.sdvo_mappings[0]; + mapping = &dev_priv->display.vbt.sdvo_mappings[0]; else - mapping = &dev_priv->vbt.sdvo_mappings[1]; + mapping = &dev_priv->display.vbt.sdvo_mappings[1]; if (mapping->initialized && intel_gmbus_is_valid_pin(dev_priv, mapping->i2c_pin)) @@ -2639,11 +2639,11 @@ intel_sdvo_get_slave_addr(struct drm_i915_private *dev_priv, struct sdvo_device_mapping *my_mapping, *other_mapping; if (sdvo->port == PORT_B) { - my_mapping = &dev_priv->vbt.sdvo_mappings[0]; - other_mapping = &dev_priv->vbt.sdvo_mappings[1]; + my_mapping = &dev_priv->display.vbt.sdvo_mappings[0]; + other_mapping = &dev_priv->display.vbt.sdvo_mappings[1]; } else { - my_mapping = &dev_priv->vbt.sdvo_mappings[1]; - other_mapping = &dev_priv->vbt.sdvo_mappings[0]; + my_mapping = &dev_priv->display.vbt.sdvo_mappings[1]; + other_mapping = &dev_priv->display.vbt.sdvo_mappings[0]; } /* If the BIOS described our SDVO device, take advantage of it. */ diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c index 0bdbedc67d7d376513dd7230114b1ddbc7d762f4..937cefd6f78f71cd959bb63b69599d7f2834a06a 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy.c +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c @@ -518,6 +518,1086 @@ static const struct intel_mpllb_state dg2_hdmi_148_5 = { }; /* values in the below table are calculted using the algo */ +static const struct intel_mpllb_state dg2_hdmi_25200 = { + .clock = 25200, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 128) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 41943) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2621), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_27027 = { + .clock = 27027, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 140) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 31876) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 46555), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_28320 = { + .clock = 28320, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 148) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40894) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 30408), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_30240 = { + .clock = 30240, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 160) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 50331) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 42466), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_31500 = { + .clock = 31500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 68) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_36000 = { + .clock = 36000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 82) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_40000 = { + .clock = 40000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 96) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_49500 = { + .clock = 49500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 126) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_50000 = { + .clock = 50000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 128) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_57284 = { + .clock = 57284, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 150) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 42886) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 49701), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_58000 = { + .clock = 58000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 152) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_65000 = { + .clock = 65000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 72) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_71000 = { + .clock = 71000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 80) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_74176 = { + .clock = 74176, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22334) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 43829), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_75000 = { + .clock = 75000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 88) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_78750 = { + .clock = 78750, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 94) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_85500 = { + .clock = 85500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 104) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_88750 = { + .clock = 88750, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 110) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_106500 = { + .clock = 106500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 138) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_108000 = { + .clock = 108000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 140) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_115500 = { + .clock = 115500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 152) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_119000 = { + .clock = 119000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 158) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_135000 = { + .clock = 135000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 76) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_138500 = { + .clock = 138500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 78) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_147160 = { + .clock = 147160, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 84) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 56623) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 6815), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_148352 = { + .clock = 148352, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22334) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 43829), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_154000 = { + .clock = 154000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 13) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 90) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_162000 = { + .clock = 162000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 96) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_209800 = { + .clock = 209800, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 134) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 60293) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 7864), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_262750 = { + .clock = 262750, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 72) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36044) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_268500 = { + .clock = 268500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 74) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 45875) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_296703 = { + .clock = 296703, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 36804), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_241500 = { + .clock = 241500, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 160) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_497750 = { + .clock = 497750, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 166) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36044) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_592000 = { + .clock = 592000, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + +static const struct intel_mpllb_state dg2_hdmi_593407 = { + .clock = 593407, + .ref_control = + REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3), + .mpllb_cp = + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124), + .mpllb_div = + REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 0) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3), + .mpllb_div2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1), + .mpllb_fracn1 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535), + .mpllb_fracn2 = + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22328) | + REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 7549), + .mpllb_sscen = + REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1), +}; + static const struct intel_mpllb_state dg2_hdmi_297 = { .clock = 297000, .ref_control = @@ -584,6 +1664,42 @@ static const struct intel_mpllb_state * const dg2_hdmi_tables[] = { &dg2_hdmi_148_5, &dg2_hdmi_297, &dg2_hdmi_594, + &dg2_hdmi_25200, + &dg2_hdmi_27027, + &dg2_hdmi_28320, + &dg2_hdmi_30240, + &dg2_hdmi_31500, + &dg2_hdmi_36000, + &dg2_hdmi_40000, + &dg2_hdmi_49500, + &dg2_hdmi_50000, + &dg2_hdmi_57284, + &dg2_hdmi_58000, + &dg2_hdmi_65000, + &dg2_hdmi_71000, + &dg2_hdmi_74176, + &dg2_hdmi_75000, + &dg2_hdmi_78750, + &dg2_hdmi_85500, + &dg2_hdmi_88750, + &dg2_hdmi_106500, + &dg2_hdmi_108000, + &dg2_hdmi_115500, + &dg2_hdmi_119000, + &dg2_hdmi_135000, + &dg2_hdmi_138500, + &dg2_hdmi_147160, + &dg2_hdmi_148352, + &dg2_hdmi_154000, + &dg2_hdmi_162000, + &dg2_hdmi_209800, + &dg2_hdmi_241500, + &dg2_hdmi_262750, + &dg2_hdmi_268500, + &dg2_hdmi_296703, + &dg2_hdmi_497750, + &dg2_hdmi_592000, + &dg2_hdmi_593407, NULL, }; diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index 6773840f6cc709cdbc181ccde6e65ad1907a0b2c..e5af955b5600f0fefe0da6b5753e3423916cc888 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -246,7 +246,7 @@ static u32 icl_tc_port_live_status_mask(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); struct intel_uncore *uncore = &i915->uncore; - u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin]; + u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin]; u32 mask = 0; u32 val; @@ -279,7 +279,7 @@ static u32 adl_tc_port_live_status_mask(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port); - u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin]; + u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin]; struct intel_uncore *uncore = &i915->uncore; u32 val, mask = 0; diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c index 9379f3463344b6e14e3da07f16dd0ff48811885c..dcf89d701f0f68feb2968ff12e8aa565f562afb9 100644 --- a/drivers/gpu/drm/i915/display/intel_tv.c +++ b/drivers/gpu/drm/i915/display/intel_tv.c @@ -39,6 +39,7 @@ #include "intel_crtc.h" #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dpll.h" #include "intel_hotplug.h" #include "intel_tv.h" @@ -982,10 +983,10 @@ intel_tv_mode_vdisplay(const struct tv_mode *tv_mode) static void intel_tv_mode_to_mode(struct drm_display_mode *mode, - const struct tv_mode *tv_mode) + const struct tv_mode *tv_mode, + int clock) { - mode->clock = tv_mode->clock / - (tv_mode->oversample >> !tv_mode->progressive); + mode->clock = clock / (tv_mode->oversample >> !tv_mode->progressive); /* * tv_mode horizontal timings: @@ -1143,7 +1144,7 @@ intel_tv_get_config(struct intel_encoder *encoder, xsize = tmp >> 16; ysize = tmp & 0xffff; - intel_tv_mode_to_mode(&mode, &tv_mode); + intel_tv_mode_to_mode(&mode, &tv_mode, pipe_config->port_clock); drm_dbg_kms(&dev_priv->drm, "TV mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(&mode)); @@ -1184,6 +1185,9 @@ intel_tv_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { + struct intel_atomic_state *state = + to_intel_atomic_state(pipe_config->uapi.state); + struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_tv_connector_state *tv_conn_state = to_intel_tv_connector_state(conn_state); @@ -1192,6 +1196,7 @@ intel_tv_compute_config(struct intel_encoder *encoder, &pipe_config->hw.adjusted_mode; int hdisplay = adjusted_mode->crtc_hdisplay; int vdisplay = adjusted_mode->crtc_vdisplay; + int ret; if (!tv_mode) return -EINVAL; @@ -1206,7 +1211,13 @@ intel_tv_compute_config(struct intel_encoder *encoder, pipe_config->port_clock = tv_mode->clock; - intel_tv_mode_to_mode(adjusted_mode, tv_mode); + ret = intel_dpll_crtc_compute_clock(state, crtc); + if (ret) + return ret; + + pipe_config->clock_set = true; + + intel_tv_mode_to_mode(adjusted_mode, tv_mode, pipe_config->port_clock); drm_mode_set_crtcinfo(adjusted_mode, 0); if (intel_tv_source_too_wide(dev_priv, hdisplay) || @@ -1804,7 +1815,7 @@ intel_tv_get_modes(struct drm_connector *connector) * about the actual timings of the mode. We * do ignore the margins though. */ - intel_tv_mode_to_mode(mode, tv_mode); + intel_tv_mode_to_mode(mode, tv_mode, tv_mode->clock); if (count == 0) { drm_dbg_kms(&dev_priv->drm, "TV mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode)); diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h index 509b0a419c20ac9fe43998b4d77493ee7b9761b3..a9f44abfc9fc283da886724d9d2447c4b8ad59f4 100644 --- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h @@ -75,6 +75,20 @@ struct bdb_header { u16 bdb_size; } __packed; +/* + * BDB version number dependencies are documented as: + * + * + + * indicates the field was introduced in version + * and is still valid + * + * - + * indicates the field was introduced in version + * and obsoleted in version +1. + * + * ??? indicates the specific version number is unknown + */ + /* * There are several types of BIOS data blocks (BDBs), each block has * an ID and size in the first 3 bytes (ID in first, size in next 2). @@ -144,12 +158,12 @@ struct bdb_general_features { /* bits 3 */ u8 disable_smooth_vision:1; u8 single_dvi:1; - u8 rotate_180:1; /* 181 */ + u8 rotate_180:1; /* 181+ */ u8 fdi_rx_polarity_inverted:1; - u8 vbios_extended_mode:1; /* 160 */ - u8 copy_ilfp_dtd_to_sdvo_lvds_dtd:1; /* 160 */ - u8 panel_best_fit_timing:1; /* 160 */ - u8 ignore_strap_state:1; /* 160 */ + u8 vbios_extended_mode:1; /* 160+ */ + u8 copy_ilfp_dtd_to_sdvo_lvds_dtd:1; /* 160+ */ + u8 panel_best_fit_timing:1; /* 160+ */ + u8 ignore_strap_state:1; /* 160+ */ /* bits 4 */ u8 legacy_monitor_detect; @@ -164,11 +178,11 @@ struct bdb_general_features { u8 rsvd11:2; /* finish byte */ /* bits 6 */ - u8 tc_hpd_retry_timeout:7; /* 242 */ + u8 tc_hpd_retry_timeout:7; /* 242+ */ u8 rsvd12:1; /* bits 7 */ - u8 afc_startup_config:2;/* 249 */ + u8 afc_startup_config:2; /* 249+ */ u8 rsvd13:6; } __packed; @@ -183,6 +197,15 @@ struct bdb_general_features { #define GPIO_PIN_ADD_DDC_I2C 0x06 /* "ADDCARD DDC/I2C GPIO pins" */ /* Device handle */ +#define DEVICE_HANDLE_CRT 0x0001 +#define DEVICE_HANDLE_EFP1 0x0004 +#define DEVICE_HANDLE_EFP2 0x0040 +#define DEVICE_HANDLE_EFP3 0x0020 +#define DEVICE_HANDLE_EFP4 0x0010 /* 194+ */ +#define DEVICE_HANDLE_EFP5 0x0002 /* 215+ */ +#define DEVICE_HANDLE_EFP6 0x0001 /* 217+ */ +#define DEVICE_HANDLE_EFP7 0x0100 /* 217+ */ +#define DEVICE_HANDLE_EFP8 0x0200 /* 217+ */ #define DEVICE_HANDLE_LFP1 0x0008 #define DEVICE_HANDLE_LFP2 0x0080 @@ -275,27 +298,27 @@ struct bdb_general_features { #define DVO_PORT_DPC 8 #define DVO_PORT_DPD 9 #define DVO_PORT_DPA 10 -#define DVO_PORT_DPE 11 /* 193 */ -#define DVO_PORT_HDMIE 12 /* 193 */ +#define DVO_PORT_DPE 11 /* 193+ */ +#define DVO_PORT_HDMIE 12 /* 193+ */ #define DVO_PORT_DPF 13 /* N/A */ #define DVO_PORT_HDMIF 14 /* N/A */ -#define DVO_PORT_DPG 15 /* 217 */ -#define DVO_PORT_HDMIG 16 /* 217 */ -#define DVO_PORT_DPH 17 /* 217 */ -#define DVO_PORT_HDMIH 18 /* 217 */ -#define DVO_PORT_DPI 19 /* 217 */ -#define DVO_PORT_HDMII 20 /* 217 */ -#define DVO_PORT_MIPIA 21 /* 171 */ -#define DVO_PORT_MIPIB 22 /* 171 */ -#define DVO_PORT_MIPIC 23 /* 171 */ -#define DVO_PORT_MIPID 24 /* 171 */ - -#define HDMI_MAX_DATA_RATE_PLATFORM 0 /* 204 */ -#define HDMI_MAX_DATA_RATE_297 1 /* 204 */ -#define HDMI_MAX_DATA_RATE_165 2 /* 204 */ -#define HDMI_MAX_DATA_RATE_594 3 /* 249 */ -#define HDMI_MAX_DATA_RATE_340 4 /* 249 */ -#define HDMI_MAX_DATA_RATE_300 5 /* 249 */ +#define DVO_PORT_DPG 15 /* 217+ */ +#define DVO_PORT_HDMIG 16 /* 217+ */ +#define DVO_PORT_DPH 17 /* 217+ */ +#define DVO_PORT_HDMIH 18 /* 217+ */ +#define DVO_PORT_DPI 19 /* 217+ */ +#define DVO_PORT_HDMII 20 /* 217+ */ +#define DVO_PORT_MIPIA 21 /* 171+ */ +#define DVO_PORT_MIPIB 22 /* 171+ */ +#define DVO_PORT_MIPIC 23 /* 171+ */ +#define DVO_PORT_MIPID 24 /* 171+ */ + +#define HDMI_MAX_DATA_RATE_PLATFORM 0 /* 204+ */ +#define HDMI_MAX_DATA_RATE_297 1 /* 204+ */ +#define HDMI_MAX_DATA_RATE_165 2 /* 204+ */ +#define HDMI_MAX_DATA_RATE_594 3 /* 249+ */ +#define HDMI_MAX_DATA_RATE_340 4 /* 249+ */ +#define HDMI_MAX_DATA_RATE_300 5 /* 249+ */ #define LEGACY_CHILD_DEVICE_CONFIG_SIZE 33 @@ -362,10 +385,10 @@ enum vbt_gmbus_ddi { * basically any of the fields to ensure the correct interpretation for the BDB * version in question. * - * When we copy the child device configs to dev_priv->vbt.child_dev, we reserve - * space for the full structure below, and initialize the tail not actually - * present in VBT to zeros. Accessing those fields is fine, as long as the - * default zero is taken into account, again according to the BDB version. + * When we copy the child device configs to dev_priv->display.vbt.child_dev, we + * reserve space for the full structure below, and initialize the tail not + * actually present in VBT to zeros. Accessing those fields is fine, as long as + * the default zero is taken into account, again according to the BDB version. * * BDB versions 155 and below are considered legacy, and version 155 seems to be * a baseline for some of the VBT documentation. When adding new fields, please @@ -379,20 +402,30 @@ struct child_device_config { u8 device_id[10]; /* ascii string */ struct { u8 i2c_speed; - u8 dp_onboard_redriver; /* 158 */ - u8 dp_ondock_redriver; /* 158 */ - u8 hdmi_level_shifter_value:5; /* 169 */ - u8 hdmi_max_data_rate:3; /* 204 */ - u16 dtd_buf_ptr; /* 161 */ - u8 edidless_efp:1; /* 161 */ - u8 compression_enable:1; /* 198 */ - u8 compression_method_cps:1; /* 198 */ - u8 ganged_edp:1; /* 202 */ - u8 reserved0:4; - u8 compression_structure_index:4; /* 198 */ - u8 reserved1:4; - u8 slave_port; /* 202 */ - u8 reserved2; + u8 dp_onboard_redriver_preemph:3; /* 158+ */ + u8 dp_onboard_redriver_vswing:3; /* 158+ */ + u8 dp_onboard_redriver_present:1; /* 158+ */ + u8 reserved0:1; + u8 dp_ondock_redriver_preemph:3; /* 158+ */ + u8 dp_ondock_redriver_vswing:3; /* 158+ */ + u8 dp_ondock_redriver_present:1; /* 158+ */ + u8 reserved1:1; + u8 hdmi_level_shifter_value:5; /* 158+ */ + u8 hdmi_max_data_rate:3; /* 204+ */ + u16 dtd_buf_ptr; /* 161+ */ + u8 edidless_efp:1; /* 161+ */ + u8 compression_enable:1; /* 198+ */ + u8 compression_method_cps:1; /* 198+ */ + u8 ganged_edp:1; /* 202+ */ + u8 lttpr_non_transparent:1; /* 235+ */ + u8 disable_compression_for_ext_disp:1; /* 251+ */ + u8 reserved2:2; + u8 compression_structure_index:4; /* 198+ */ + u8 reserved3:4; + u8 hdmi_max_frl_rate:4; /* 237+ */ + u8 hdmi_max_frl_rate_valid:1; /* 237+ */ + u8 reserved4:3; /* 237+ */ + u8 reserved5; } __packed; } __packed; @@ -412,16 +445,16 @@ struct child_device_config { u8 ddc2_pin; } __packed; struct { - u8 efp_routed:1; /* 158 */ - u8 lane_reversal:1; /* 184 */ - u8 lspcon:1; /* 192 */ - u8 iboost:1; /* 196 */ - u8 hpd_invert:1; /* 196 */ - u8 use_vbt_vswing:1; /* 218 */ - u8 flag_reserved:2; - u8 hdmi_support:1; /* 158 */ - u8 dp_support:1; /* 158 */ - u8 tmds_support:1; /* 158 */ + u8 efp_routed:1; /* 158+ */ + u8 lane_reversal:1; /* 184+ */ + u8 lspcon:1; /* 192+ */ + u8 iboost:1; /* 196+ */ + u8 hpd_invert:1; /* 196+ */ + u8 use_vbt_vswing:1; /* 218+ */ + u8 dp_max_lane_count:2; /* 244+ */ + u8 hdmi_support:1; /* 158+ */ + u8 dp_support:1; /* 158+ */ + u8 tmds_support:1; /* 158+ */ u8 support_reserved:5; u8 aux_channel; u8 dongle_detect; @@ -429,7 +462,7 @@ struct child_device_config { } __packed; u8 pipe_cap:2; - u8 sdvo_stall:1; /* 158 */ + u8 sdvo_stall:1; /* 158+ */ u8 hpd_status:2; u8 integrated_encoder:1; u8 capabilities_reserved:2; @@ -437,21 +470,21 @@ struct child_device_config { union { u8 dvo2_wiring; - u8 mipi_bridge_type; /* 171 */ + u8 mipi_bridge_type; /* 171+ */ } __packed; u16 extended_type; u8 dvo_function; - u8 dp_usb_type_c:1; /* 195 */ - u8 tbt:1; /* 209 */ - u8 flags2_reserved:2; /* 195 */ - u8 dp_port_trace_length:4; /* 209 */ - u8 dp_gpio_index; /* 195 */ - u16 dp_gpio_pin_num; /* 195 */ - u8 dp_iboost_level:4; /* 196 */ - u8 hdmi_iboost_level:4; /* 196 */ - u8 dp_max_link_rate:3; /* 216/230 GLK+ */ - u8 dp_max_link_rate_reserved:5; /* 216/230 */ + u8 dp_usb_type_c:1; /* 195+ */ + u8 tbt:1; /* 209+ */ + u8 flags2_reserved:2; /* 195+ */ + u8 dp_port_trace_length:4; /* 209+ */ + u8 dp_gpio_index; /* 195+ */ + u16 dp_gpio_pin_num; /* 195+ */ + u8 dp_iboost_level:4; /* 196+ */ + u8 hdmi_iboost_level:4; /* 196+ */ + u8 dp_max_link_rate:3; /* 216+ */ + u8 dp_max_link_rate_reserved:5; /* 216+ */ } __packed; struct bdb_general_definitions { @@ -459,7 +492,7 @@ struct bdb_general_definitions { u8 crt_ddc_gmbus_pin; /* DPMS bits */ - u8 dpms_acpi:1; + u8 dpms_non_acpi:1; u8 skip_boot_crt_detect:1; u8 dpms_aim:1; u8 rsvd1:5; /* finish byte */ @@ -488,25 +521,25 @@ struct bdb_general_definitions { struct psr_table { /* Feature bits */ - u8 full_link:1; - u8 require_aux_to_wakeup:1; + u8 full_link:1; /* 165+ */ + u8 require_aux_to_wakeup:1; /* 165+ */ u8 feature_bits_rsvd:6; /* Wait times */ - u8 idle_frames:4; - u8 lines_to_wait:3; + u8 idle_frames:4; /* 165+ */ + u8 lines_to_wait:3; /* 165+ */ u8 wait_times_rsvd:1; /* TP wake up time in multiple of 100 */ - u16 tp1_wakeup_time; - u16 tp2_tp3_wakeup_time; + u16 tp1_wakeup_time; /* 165+ */ + u16 tp2_tp3_wakeup_time; /* 165+ */ } __packed; struct bdb_psr { struct psr_table psr_table[16]; /* PSR2 TP2/TP3 wakeup time for 16 panels */ - u32 psr2_tp2_tp3_wakeup_time; + u32 psr2_tp2_tp3_wakeup_time; /* 226+ */ } __packed; /* @@ -519,9 +552,10 @@ struct bdb_psr { #define BDB_DRIVER_FEATURE_INT_SDVO_LVDS 3 struct bdb_driver_features { + /* Driver bits */ u8 boot_dev_algorithm:1; - u8 block_display_switch:1; - u8 allow_display_switch:1; + u8 allow_display_switch_dvd:1; + u8 allow_display_switch_dos:1; u8 hotplug_dvo:1; u8 dual_view_zoom:1; u8 int15h_hook:1; @@ -533,6 +567,7 @@ struct bdb_driver_features { u8 boot_mode_bpp; u8 boot_mode_refresh; + /* Extended Driver Bits 1 */ u16 enable_lfp_primary:1; u16 selective_mode_pruning:1; u16 dual_frequency:1; @@ -548,29 +583,40 @@ struct bdb_driver_features { u16 tv_hotplug:1; u16 hdmi_config:2; - u8 static_display:1; - u8 reserved2:7; + /* Driver Flags 1 */ + u8 static_display:1; /* 163+ */ + u8 embedded_platform:1; /* 163+ */ + u8 display_subsystem_enable:1; /* 163+ */ + u8 reserved0:5; + u16 legacy_crt_max_x; u16 legacy_crt_max_y; u8 legacy_crt_max_refresh; - u8 hdmi_termination; - u8 custom_vbt_version; - /* Driver features data block */ - u16 rmpm_enabled:1; - u16 s2ddt_enabled:1; - u16 dpst_enabled:1; - u16 bltclt_enabled:1; - u16 adb_enabled:1; - u16 drrs_enabled:1; - u16 grs_enabled:1; - u16 gpmt_enabled:1; - u16 tbt_enabled:1; - u16 psr_enabled:1; - u16 ips_enabled:1; - u16 reserved3:1; - u16 dmrrs_enabled:1; - u16 reserved4:2; + /* Extended Driver Bits 2 */ + u8 hdmi_termination:1; + u8 cea861d_hdmi_support:1; + u8 self_refresh_enable:1; + u8 reserved1:5; + + u8 custom_vbt_version; /* 155+ */ + + /* Driver Feature Flags */ + u16 rmpm_enabled:1; /* 165+ */ + u16 s2ddt_enabled:1; /* 165+ */ + u16 dpst_enabled:1; /* 165-227 */ + u16 bltclt_enabled:1; /* 165+ */ + u16 adb_enabled:1; /* 165-227 */ + u16 drrs_enabled:1; /* 165-227 */ + u16 grs_enabled:1; /* 165+ */ + u16 gpmt_enabled:1; /* 165+ */ + u16 tbt_enabled:1; /* 165+ */ + u16 psr_enabled:1; /* 165-227 */ + u16 ips_enabled:1; /* 165+ */ + u16 dpfs_enabled:1; /* 165+ */ + u16 dmrrs_enabled:1; /* 174-227 */ + u16 adt_enabled:1; /* ???-228 */ + u16 hpd_wake:1; /* 201-240 */ u16 pc_feature_valid:1; } __packed; @@ -657,7 +703,7 @@ struct bdb_sdvo_panel_dtds { struct edp_fast_link_params { - u8 rate:4; + u8 rate:4; /* ???-223 */ u8 lanes:4; u8 preemphasis:4; u8 vswing:4; @@ -690,18 +736,18 @@ struct bdb_edp { u32 sdrrs_msa_timing_delay; /* ith bit indicates enabled/disabled for (i+1)th panel */ - u16 edp_s3d_feature; /* 162 */ - u16 edp_t3_optimization; /* 165 */ - u64 edp_vswing_preemph; /* 173 */ - u16 fast_link_training; /* 182 */ - u16 dpcd_600h_write_required; /* 185 */ - struct edp_pwm_delays pwm_delays[16]; /* 186 */ - u16 full_link_params_provided; /* 199 */ - struct edp_full_link_params full_link_params[16]; /* 199 */ - u16 apical_enable; /* 203 */ - struct edp_apical_params apical_params[16]; /* 203 */ - u16 edp_fast_link_training_rate[16]; /* 224 */ - u16 edp_max_port_link_rate[16]; /* 244 */ + u16 edp_s3d_feature; /* 162+ */ + u16 edp_t3_optimization; /* 165+ */ + u64 edp_vswing_preemph; /* 173+ */ + u16 fast_link_training; /* 182+ */ + u16 dpcd_600h_write_required; /* 185+ */ + struct edp_pwm_delays pwm_delays[16]; /* 186+ */ + u16 full_link_params_provided; /* 199+ */ + struct edp_full_link_params full_link_params[16]; /* 199+ */ + u16 apical_enable; /* 203+ */ + struct edp_apical_params apical_params[16]; /* 203+ */ + u16 edp_fast_link_training_rate[16]; /* 224+ */ + u16 edp_max_port_link_rate[16]; /* 244+ */ } __packed; /* @@ -710,14 +756,14 @@ struct bdb_edp { struct bdb_lvds_options { u8 panel_type; - u8 panel_type2; /* 212 */ + u8 panel_type2; /* 212+ */ /* LVDS capabilities, stored in a dword */ u8 pfit_mode:2; u8 pfit_text_mode_enhanced:1; u8 pfit_gfx_mode_enhanced:1; u8 pfit_ratio_auto:1; u8 pixel_dither:1; - u8 lvds_edid:1; + u8 lvds_edid:1; /* ???-240 */ u8 rsvd2:1; u8 rsvd4; /* LVDS Panel channel bits stored here */ @@ -731,11 +777,11 @@ struct bdb_lvds_options { /* LVDS panel type bits stored here */ u32 dps_panel_type_bits; /* LVDS backlight control type bits stored here */ - u32 blt_control_type_bits; + u32 blt_control_type_bits; /* ???-240 */ - u16 lcdvcc_s0_enable; /* 200 */ - u32 rotation; /* 228 */ - u32 position; /* 240 */ + u16 lcdvcc_s0_enable; /* 200+ */ + u32 rotation; /* 228+ */ + u32 position; /* 240+ */ } __packed; /* @@ -756,7 +802,7 @@ struct lvds_lfp_data_ptr { struct bdb_lvds_lfp_data_ptrs { u8 lvds_entries; struct lvds_lfp_data_ptr ptr[16]; - struct lvds_lfp_data_ptr_table panel_name; /* 156-163? */ + struct lvds_lfp_data_ptr_table panel_name; /* (156-163?)+ */ } __packed; /* @@ -808,20 +854,20 @@ struct lvds_lfp_panel_name { } __packed; struct lvds_lfp_black_border { - u8 top; /* 227 */ - u8 bottom; /* 227 */ - u8 left; /* 238 */ - u8 right; /* 238 */ + u8 top; /* 227+ */ + u8 bottom; /* 227+ */ + u8 left; /* 238+ */ + u8 right; /* 238+ */ } __packed; struct bdb_lvds_lfp_data_tail { - struct lvds_lfp_panel_name panel_name[16]; /* 156-163? */ - u16 scaling_enable; /* 187 */ - u8 seamless_drrs_min_refresh_rate[16]; /* 188 */ - u8 pixel_overlap_count[16]; /* 208 */ - struct lvds_lfp_black_border black_border[16]; /* 227 */ - u16 dual_lfp_port_sync_enable; /* 231 */ - u16 gpu_dithering_for_banding_artifacts; /* 245 */ + struct lvds_lfp_panel_name panel_name[16]; /* (156-163?)+ */ + u16 scaling_enable; /* 187+ */ + u8 seamless_drrs_min_refresh_rate[16]; /* 188+ */ + u8 pixel_overlap_count[16]; /* 208+ */ + struct lvds_lfp_black_border black_border[16]; /* 227+ */ + u16 dual_lfp_port_sync_enable; /* 231+ */ + u16 gpu_dithering_for_banding_artifacts; /* 245+ */ } __packed; /* @@ -836,7 +882,7 @@ struct lfp_backlight_data_entry { u8 active_low_pwm:1; u8 obsolete1:5; u16 pwm_freq_hz; - u8 min_brightness; /* Obsolete from 234+ */ + u8 min_brightness; /* ???-233 */ u8 obsolete2; u8 obsolete3; } __packed; @@ -859,7 +905,7 @@ struct lfp_brightness_level { struct bdb_lfp_backlight_data { u8 entry_size; struct lfp_backlight_data_entry data[16]; - u8 level[16]; /* Obsolete from 234+ */ + u8 level[16]; /* ???-233 */ struct lfp_backlight_control_method backlight_control[16]; struct lfp_brightness_level brightness_level[16]; /* 234+ */ struct lfp_brightness_level brightness_min_level[16]; /* 234+ */ @@ -874,8 +920,8 @@ struct lfp_power_features { u8 reserved1:1; u8 power_conservation_pref:3; u8 reserved2:1; - u8 lace_enabled_status:1; - u8 lace_support:1; + u8 lace_enabled_status:1; /* 210+ */ + u8 lace_support:1; /* 210+ */ u8 als_enable:1; } __packed; @@ -895,24 +941,24 @@ struct aggressiveness_profile2_entry { } __packed; struct bdb_lfp_power { - struct lfp_power_features features; + struct lfp_power_features features; /* ???-227 */ struct als_data_entry als[5]; - u8 lace_aggressiveness_profile:3; + u8 lace_aggressiveness_profile:3; /* 210-227 */ u8 reserved1:5; - u16 dpst; - u16 psr; - u16 drrs; - u16 lace_support; - u16 adt; - u16 dmrrs; - u16 adb; - u16 lace_enabled_status; - struct aggressiveness_profile_entry aggressiveness[16]; - u16 hobl; /* 232+ */ - u16 vrr_feature_enabled; /* 233+ */ - u16 elp; /* 247+ */ - u16 opst; /* 247+ */ - struct aggressiveness_profile2_entry aggressiveness2[16]; /* 247+ */ + u16 dpst; /* 228+ */ + u16 psr; /* 228+ */ + u16 drrs; /* 228+ */ + u16 lace_support; /* 228+ */ + u16 adt; /* 228+ */ + u16 dmrrs; /* 228+ */ + u16 adb; /* 228+ */ + u16 lace_enabled_status; /* 228+ */ + struct aggressiveness_profile_entry aggressiveness[16]; /* 228+ */ + u16 hobl; /* 232+ */ + u16 vrr_feature_enabled; /* 233+ */ + u16 elp; /* 247+ */ + u16 opst; /* 247+ */ + struct aggressiveness_profile2_entry aggressiveness2[16]; /* 247+ */ } __packed; /* @@ -922,10 +968,10 @@ struct bdb_lfp_power { #define MAX_MIPI_CONFIGURATIONS 6 struct bdb_mipi_config { - struct mipi_config config[MAX_MIPI_CONFIGURATIONS]; /* 175 */ - struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS]; /* 177 */ - struct edp_pwm_delays pwm_delays[MAX_MIPI_CONFIGURATIONS]; /* 186 */ - u8 pmic_i2c_bus_number[MAX_MIPI_CONFIGURATIONS]; /* 190 */ + struct mipi_config config[MAX_MIPI_CONFIGURATIONS]; /* 175+ */ + struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS]; /* 177+ */ + struct edp_pwm_delays pwm_delays[MAX_MIPI_CONFIGURATIONS]; /* 186+ */ + u8 pmic_i2c_bus_number[MAX_MIPI_CONFIGURATIONS]; /* 190+ */ } __packed; /* diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 43e1bbc1e3035ff989f437133d7b6e3232d941d5..269f9792390de8ce23e5980c4f9a2a0542e9bd19 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -344,7 +344,7 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state) struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - if (!INTEL_INFO(i915)->display.has_dsc) + if (!RUNTIME_INFO(i915)->has_dsc) return false; if (DISPLAY_VER(i915) >= 12) @@ -460,7 +460,6 @@ int intel_dsc_compute_params(struct intel_crtc_state *pipe_config) u8 i = 0; vdsc_cfg->pic_width = pipe_config->hw.adjusted_mode.crtc_hdisplay; - vdsc_cfg->pic_height = pipe_config->hw.adjusted_mode.crtc_vdisplay; vdsc_cfg->slice_width = DIV_ROUND_UP(vdsc_cfg->pic_width, pipe_config->dsc.slice_count); @@ -597,6 +596,8 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state) DSC_VER_MIN_SHIFT | vdsc_cfg->bits_per_component << DSC_BPC_SHIFT | vdsc_cfg->line_buf_depth << DSC_LINE_BUF_DEPTH_SHIFT; + if (vdsc_cfg->dsc_version_minor == 2) + pps_val |= DSC_ALT_ICH_SEL; if (vdsc_cfg->block_pred_enable) pps_val |= DSC_BLOCK_PREDICTION; if (vdsc_cfg->convert_rgb) diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index 04250a0fec3c1f71847f8fd4b3cc80c7e4749c7c..5eac99021875ed88dc8b19fe16d8d062194fe5e2 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -142,11 +142,16 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state, * For XE_LPD+, we use guardband and pipeline override * is deprecated. */ - if (DISPLAY_VER(i915) >= 13) + if (DISPLAY_VER(i915) >= 13) { + /* + * FIXME: Subtract Window2 delay from below value. + * + * Window2 specifies time required to program DSB (Window2) in + * number of scan lines. Assuming 0 for no DSB. + */ crtc_state->vrr.guardband = - crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - - i915->window2_delay; - else + crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay; + } else { /* * FIXME: s/4/framestart_delay/ to get consistent * earliest/latest points for register latching regardless @@ -159,6 +164,7 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state, */ crtc_state->vrr.pipeline_full = min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - 4 - 1); + } crtc_state->mode_flags |= I915_MODE_FLAG_VRR; } diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 4d6a27757065c3c3f7e7756108506e36fbdadacb..7cb7130434089839223054d9d61ad5db2f426454 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -14,11 +14,11 @@ #include "intel_display_types.h" #include "intel_fb.h" #include "intel_fbc.h" -#include "intel_pm.h" #include "intel_psr.h" #include "intel_sprite.h" #include "skl_scaler.h" #include "skl_universal_plane.h" +#include "skl_watermark.h" #include "pxp/intel_pxp.h" static const u32 skl_plane_formats[] = { @@ -1928,7 +1928,7 @@ static enum intel_fbc_id skl_fbc_id_for_pipe(enum pipe pipe) static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv, enum intel_fbc_id fbc_id, enum plane_id plane_id) { - if ((INTEL_INFO(dev_priv)->display.fbc_mask & BIT(fbc_id)) == 0) + if ((RUNTIME_INFO(dev_priv)->fbc_mask & BIT(fbc_id)) == 0) return false; return plane_id == PLANE_PRIMARY; @@ -1940,7 +1940,7 @@ static struct intel_fbc *skl_plane_fbc(struct drm_i915_private *dev_priv, enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(pipe); if (skl_plane_has_fbc(dev_priv, fbc_id, plane_id)) - return dev_priv->fbc[fbc_id]; + return dev_priv->display.fbc[fbc_id]; else return NULL; } diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c new file mode 100644 index 0000000000000000000000000000000000000000..18178b01375e4e5acefd95306d3d5f47b7a6fb91 --- /dev/null +++ b/drivers/gpu/drm/i915/display/skl_watermark.c @@ -0,0 +1,3574 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "intel_atomic.h" +#include "intel_atomic_plane.h" +#include "intel_bw.h" +#include "intel_de.h" +#include "intel_display.h" +#include "intel_display_power.h" +#include "intel_display_types.h" +#include "intel_fb.h" +#include "skl_watermark.h" + +#include "i915_drv.h" +#include "i915_fixed.h" +#include "i915_reg.h" +#include "intel_pcode.h" +#include "intel_pm.h" + +static void skl_sagv_disable(struct drm_i915_private *i915); + +/* Stores plane specific WM parameters */ +struct skl_wm_params { + bool x_tiled, y_tiled; + bool rc_surface; + bool is_planar; + u32 width; + u8 cpp; + u32 plane_pixel_rate; + u32 y_min_scanlines; + u32 plane_bytes_per_line; + uint_fixed_16_16_t plane_blocks_per_line; + uint_fixed_16_16_t y_tile_minimum; + u32 linetime_us; + u32 dbuf_block_size; +}; + +u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915) +{ + u8 enabled_slices = 0; + enum dbuf_slice slice; + + for_each_dbuf_slice(i915, slice) { + if (intel_uncore_read(&i915->uncore, + DBUF_CTL_S(slice)) & DBUF_POWER_STATE) + enabled_slices |= BIT(slice); + } + + return enabled_slices; +} + +/* + * FIXME: We still don't have the proper code detect if we need to apply the WA, + * so assume we'll always need it in order to avoid underruns. + */ +static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915) +{ + return DISPLAY_VER(i915) == 9; +} + +static bool +intel_has_sagv(struct drm_i915_private *i915) +{ + return DISPLAY_VER(i915) >= 9 && !IS_LP(i915) && + i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED; +} + +static u32 +intel_sagv_block_time(struct drm_i915_private *i915) +{ + if (DISPLAY_VER(i915) >= 14) { + u32 val; + + val = intel_uncore_read(&i915->uncore, MTL_LATENCY_SAGV); + + return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val); + } else if (DISPLAY_VER(i915) >= 12) { + u32 val = 0; + int ret; + + ret = snb_pcode_read(&i915->uncore, + GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, + &val, NULL); + if (ret) { + drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n"); + return 0; + } + + return val; + } else if (DISPLAY_VER(i915) == 11) { + return 10; + } else if (DISPLAY_VER(i915) == 9 && !IS_LP(i915)) { + return 30; + } else { + return 0; + } +} + +static void intel_sagv_init(struct drm_i915_private *i915) +{ + if (!intel_has_sagv(i915)) + i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; + + /* + * Probe to see if we have working SAGV control. + * For icl+ this was already determined by intel_bw_init_hw(). + */ + if (DISPLAY_VER(i915) < 11) + skl_sagv_disable(i915); + + drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN); + + i915->display.sagv.block_time_us = intel_sagv_block_time(i915); + + drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n", + str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us); + + /* avoid overflow when adding with wm0 latency/etc. */ + if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX, + "Excessive SAGV block time %u, ignoring\n", + i915->display.sagv.block_time_us)) + i915->display.sagv.block_time_us = 0; + + if (!intel_has_sagv(i915)) + i915->display.sagv.block_time_us = 0; +} + +/* + * SAGV dynamically adjusts the system agent voltage and clock frequencies + * depending on power and performance requirements. The display engine access + * to system memory is blocked during the adjustment time. Because of the + * blocking time, having this enabled can cause full system hangs and/or pipe + * underruns if we don't meet all of the following requirements: + * + * - <= 1 pipe enabled + * - All planes can enable watermarks for latencies >= SAGV engine block time + * - We're not using an interlaced display configuration + */ +static void skl_sagv_enable(struct drm_i915_private *i915) +{ + int ret; + + if (!intel_has_sagv(i915)) + return; + + if (i915->display.sagv.status == I915_SAGV_ENABLED) + return; + + drm_dbg_kms(&i915->drm, "Enabling SAGV\n"); + ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL, + GEN9_SAGV_ENABLE); + + /* We don't need to wait for SAGV when enabling */ + + /* + * Some skl systems, pre-release machines in particular, + * don't actually have SAGV. + */ + if (IS_SKYLAKE(i915) && ret == -ENXIO) { + drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n"); + i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; + return; + } else if (ret < 0) { + drm_err(&i915->drm, "Failed to enable SAGV\n"); + return; + } + + i915->display.sagv.status = I915_SAGV_ENABLED; +} + +static void skl_sagv_disable(struct drm_i915_private *i915) +{ + int ret; + + if (!intel_has_sagv(i915)) + return; + + if (i915->display.sagv.status == I915_SAGV_DISABLED) + return; + + drm_dbg_kms(&i915->drm, "Disabling SAGV\n"); + /* bspec says to keep retrying for at least 1 ms */ + ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL, + GEN9_SAGV_DISABLE, + GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, + 1); + /* + * Some skl systems, pre-release machines in particular, + * don't actually have SAGV. + */ + if (IS_SKYLAKE(i915) && ret == -ENXIO) { + drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n"); + i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED; + return; + } else if (ret < 0) { + drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret); + return; + } + + i915->display.sagv.status = I915_SAGV_DISABLED; +} + +static void skl_sagv_pre_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + + if (!new_bw_state) + return; + + if (!intel_can_enable_sagv(i915, new_bw_state)) + skl_sagv_disable(i915); +} + +static void skl_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + + if (!new_bw_state) + return; + + if (intel_can_enable_sagv(i915, new_bw_state)) + skl_sagv_enable(i915); +} + +static void icl_sagv_pre_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + + if (!new_bw_state) + return; + + old_mask = old_bw_state->qgv_points_mask; + new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + + if (old_mask == new_mask) + return; + + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + + /* + * Restrict required qgv points before updating the configuration. + * According to BSpec we can't mask and unmask qgv points at the same + * time. Also masking should be done before updating the configuration + * and unmasking afterwards. + */ + icl_pcode_restrict_qgv_points(i915, new_mask); +} + +static void icl_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_bw_state *old_bw_state = + intel_atomic_get_old_bw_state(state); + const struct intel_bw_state *new_bw_state = + intel_atomic_get_new_bw_state(state); + u16 old_mask, new_mask; + + if (!new_bw_state) + return; + + old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + new_mask = new_bw_state->qgv_points_mask; + + if (old_mask == new_mask) + return; + + WARN_ON(!new_bw_state->base.changed); + + drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n", + old_mask, new_mask); + + /* + * Allow required qgv points after updating the configuration. + * According to BSpec we can't mask and unmask qgv points at the same + * time. Also masking should be done before updating the configuration + * and unmasking afterwards. + */ + icl_pcode_restrict_qgv_points(i915, new_mask); +} + +void intel_sagv_pre_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + + /* + * Just return if we can't control SAGV or don't have it. + * This is different from situation when we have SAGV but just can't + * afford it due to DBuf limitation - in case if SAGV is completely + * disabled in a BIOS, we are not even allowed to send a PCode request, + * as it will throw an error. So have to check it here. + */ + if (!intel_has_sagv(i915)) + return; + + if (DISPLAY_VER(i915) >= 11) + icl_sagv_pre_plane_update(state); + else + skl_sagv_pre_plane_update(state); +} + +void intel_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + + /* + * Just return if we can't control SAGV or don't have it. + * This is different from situation when we have SAGV but just can't + * afford it due to DBuf limitation - in case if SAGV is completely + * disabled in a BIOS, we are not even allowed to send a PCode request, + * as it will throw an error. So have to check it here. + */ + if (!intel_has_sagv(i915)) + return; + + if (DISPLAY_VER(i915) >= 11) + icl_sagv_post_plane_update(state); + else + skl_sagv_post_plane_update(state); +} + +static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + enum plane_id plane_id; + int max_level = INT_MAX; + + if (!intel_has_sagv(i915)) + return false; + + if (!crtc_state->hw.active) + return true; + + if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE) + return false; + + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + int level; + + /* Skip this plane if it's not enabled */ + if (!wm->wm[0].enable) + continue; + + /* Find the highest enabled wm level for this plane */ + for (level = ilk_wm_max_level(i915); + !wm->wm[level].enable; --level) + { } + + /* Highest common enabled wm level for all planes */ + max_level = min(level, max_level); + } + + /* No enabled planes? */ + if (max_level == INT_MAX) + return true; + + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + /* + * All enabled planes must have enabled a common wm level that + * can tolerate memory latencies higher than sagv_block_time_us + */ + if (wm->wm[0].enable && !wm->wm[max_level].can_sagv) + return false; + } + + return true; +} + +static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + enum plane_id plane_id; + + if (!crtc_state->hw.active) + return true; + + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + if (wm->wm[0].enable && !wm->sagv.wm0.enable) + return false; + } + + return true; +} + +static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + + if (DISPLAY_VER(i915) >= 12) + return tgl_crtc_can_enable_sagv(crtc_state); + else + return skl_crtc_can_enable_sagv(crtc_state); +} + +bool intel_can_enable_sagv(struct drm_i915_private *i915, + const struct intel_bw_state *bw_state) +{ + if (DISPLAY_VER(i915) < 11 && + bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes)) + return false; + + return bw_state->pipe_sagv_reject == 0; +} + +static int intel_compute_sagv_mask(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + int ret; + struct intel_crtc *crtc; + struct intel_crtc_state *new_crtc_state; + struct intel_bw_state *new_bw_state = NULL; + const struct intel_bw_state *old_bw_state = NULL; + int i; + + for_each_new_intel_crtc_in_state(state, crtc, + new_crtc_state, i) { + new_bw_state = intel_atomic_get_bw_state(state); + if (IS_ERR(new_bw_state)) + return PTR_ERR(new_bw_state); + + old_bw_state = intel_atomic_get_old_bw_state(state); + + if (intel_crtc_can_enable_sagv(new_crtc_state)) + new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe); + else + new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe); + } + + if (!new_bw_state) + return 0; + + new_bw_state->active_pipes = + intel_calc_active_pipes(state, old_bw_state->active_pipes); + + if (new_bw_state->active_pipes != old_bw_state->active_pipes) { + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; + } + + if (intel_can_enable_sagv(i915, new_bw_state) != + intel_can_enable_sagv(i915, old_bw_state)) { + ret = intel_atomic_serialize_global_state(&new_bw_state->base); + if (ret) + return ret; + } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) { + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; + } + + for_each_new_intel_crtc_in_state(state, crtc, + new_crtc_state, i) { + struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal; + + /* + * We store use_sagv_wm in the crtc state rather than relying on + * that bw state since we have no convenient way to get at the + * latter from the plane commit hooks (especially in the legacy + * cursor case) + */ + pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) && + DISPLAY_VER(i915) >= 12 && + intel_can_enable_sagv(i915, new_bw_state); + } + + return 0; +} + +static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry, + u16 start, u16 end) +{ + entry->start = start; + entry->end = end; + + return end; +} + +static int intel_dbuf_slice_size(struct drm_i915_private *i915) +{ + return INTEL_INFO(i915)->display.dbuf.size / + hweight8(INTEL_INFO(i915)->display.dbuf.slice_mask); +} + +static void +skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask, + struct skl_ddb_entry *ddb) +{ + int slice_size = intel_dbuf_slice_size(i915); + + if (!slice_mask) { + ddb->start = 0; + ddb->end = 0; + return; + } + + ddb->start = (ffs(slice_mask) - 1) * slice_size; + ddb->end = fls(slice_mask) * slice_size; + + WARN_ON(ddb->start >= ddb->end); + WARN_ON(ddb->end > INTEL_INFO(i915)->display.dbuf.size); +} + +static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask) +{ + struct skl_ddb_entry ddb; + + if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2))) + slice_mask = BIT(DBUF_S1); + else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4))) + slice_mask = BIT(DBUF_S3); + + skl_ddb_entry_for_slices(i915, slice_mask, &ddb); + + return ddb.start; +} + +u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915, + const struct skl_ddb_entry *entry) +{ + int slice_size = intel_dbuf_slice_size(i915); + enum dbuf_slice start_slice, end_slice; + u8 slice_mask = 0; + + if (!skl_ddb_entry_size(entry)) + return 0; + + start_slice = entry->start / slice_size; + end_slice = (entry->end - 1) / slice_size; + + /* + * Per plane DDB entry can in a really worst case be on multiple slices + * but single entry is anyway contigious. + */ + while (start_slice <= end_slice) { + slice_mask |= BIT(start_slice); + start_slice++; + } + + return slice_mask; +} + +static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state) +{ + const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; + int hdisplay, vdisplay; + + if (!crtc_state->hw.active) + return 0; + + /* + * Watermark/ddb requirement highly depends upon width of the + * framebuffer, So instead of allocating DDB equally among pipes + * distribute DDB based on resolution/width of the display. + */ + drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay); + + return hdisplay; +} + +static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state, + enum pipe for_pipe, + unsigned int *weight_start, + unsigned int *weight_end, + unsigned int *weight_total) +{ + struct drm_i915_private *i915 = + to_i915(dbuf_state->base.state->base.dev); + enum pipe pipe; + + *weight_start = 0; + *weight_end = 0; + *weight_total = 0; + + for_each_pipe(i915, pipe) { + int weight = dbuf_state->weight[pipe]; + + /* + * Do not account pipes using other slice sets + * luckily as of current BSpec slice sets do not partially + * intersect(pipes share either same one slice or same slice set + * i.e no partial intersection), so it is enough to check for + * equality for now. + */ + if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe]) + continue; + + *weight_total += weight; + if (pipe < for_pipe) { + *weight_start += weight; + *weight_end += weight; + } else if (pipe == for_pipe) { + *weight_end += weight; + } + } +} + +static int +skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + unsigned int weight_total, weight_start, weight_end; + const struct intel_dbuf_state *old_dbuf_state = + intel_atomic_get_old_dbuf_state(state); + struct intel_dbuf_state *new_dbuf_state = + intel_atomic_get_new_dbuf_state(state); + struct intel_crtc_state *crtc_state; + struct skl_ddb_entry ddb_slices; + enum pipe pipe = crtc->pipe; + unsigned int mbus_offset = 0; + u32 ddb_range_size; + u32 dbuf_slice_mask; + u32 start, end; + int ret; + + if (new_dbuf_state->weight[pipe] == 0) { + skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0); + goto out; + } + + dbuf_slice_mask = new_dbuf_state->slices[pipe]; + + skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices); + mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask); + ddb_range_size = skl_ddb_entry_size(&ddb_slices); + + intel_crtc_dbuf_weights(new_dbuf_state, pipe, + &weight_start, &weight_end, &weight_total); + + start = ddb_range_size * weight_start / weight_total; + end = ddb_range_size * weight_end / weight_total; + + skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], + ddb_slices.start - mbus_offset + start, + ddb_slices.start - mbus_offset + end); + +out: + if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] && + skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], + &new_dbuf_state->ddb[pipe])) + return 0; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + + crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + /* + * Used for checking overlaps, so we need absolute + * offsets instead of MBUS relative offsets. + */ + crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start; + crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end; + + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n", + crtc->base.base.id, crtc->base.name, + old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe], + old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end, + new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end, + old_dbuf_state->active_pipes, new_dbuf_state->active_pipes); + + return 0; +} + +static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, + int width, const struct drm_format_info *format, + u64 modifier, unsigned int rotation, + u32 plane_pixel_rate, struct skl_wm_params *wp, + int color_plane); + +static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, + int level, + unsigned int latency, + const struct skl_wm_params *wp, + const struct skl_wm_level *result_prev, + struct skl_wm_level *result /* out */); + +static unsigned int +skl_cursor_allocation(const struct intel_crtc_state *crtc_state, + int num_active) +{ + struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + int level, max_level = ilk_wm_max_level(i915); + struct skl_wm_level wm = {}; + int ret, min_ddb_alloc = 0; + struct skl_wm_params wp; + + ret = skl_compute_wm_params(crtc_state, 256, + drm_format_info(DRM_FORMAT_ARGB8888), + DRM_FORMAT_MOD_LINEAR, + DRM_MODE_ROTATE_0, + crtc_state->pixel_rate, &wp, 0); + drm_WARN_ON(&i915->drm, ret); + + for (level = 0; level <= max_level; level++) { + unsigned int latency = i915->display.wm.skl_latency[level]; + + skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm); + if (wm.min_ddb_alloc == U16_MAX) + break; + + min_ddb_alloc = wm.min_ddb_alloc; + } + + return max(num_active == 1 ? 32 : 8, min_ddb_alloc); +} + +static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg) +{ + skl_ddb_entry_init(entry, + REG_FIELD_GET(PLANE_BUF_START_MASK, reg), + REG_FIELD_GET(PLANE_BUF_END_MASK, reg)); + if (entry->end) + entry->end++; +} + +static void +skl_ddb_get_hw_plane_state(struct drm_i915_private *i915, + const enum pipe pipe, + const enum plane_id plane_id, + struct skl_ddb_entry *ddb, + struct skl_ddb_entry *ddb_y) +{ + u32 val; + + /* Cursor doesn't support NV12/planar, so no extra calculation needed */ + if (plane_id == PLANE_CURSOR) { + val = intel_uncore_read(&i915->uncore, CUR_BUF_CFG(pipe)); + skl_ddb_entry_init_from_hw(ddb, val); + return; + } + + val = intel_uncore_read(&i915->uncore, PLANE_BUF_CFG(pipe, plane_id)); + skl_ddb_entry_init_from_hw(ddb, val); + + if (DISPLAY_VER(i915) >= 11) + return; + + val = intel_uncore_read(&i915->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id)); + skl_ddb_entry_init_from_hw(ddb_y, val); +} + +static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, + struct skl_ddb_entry *ddb, + struct skl_ddb_entry *ddb_y) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + enum intel_display_power_domain power_domain; + enum pipe pipe = crtc->pipe; + intel_wakeref_t wakeref; + enum plane_id plane_id; + + power_domain = POWER_DOMAIN_PIPE(pipe); + wakeref = intel_display_power_get_if_enabled(i915, power_domain); + if (!wakeref) + return; + + for_each_plane_id_on_crtc(crtc, plane_id) + skl_ddb_get_hw_plane_state(i915, pipe, + plane_id, + &ddb[plane_id], + &ddb_y[plane_id]); + + intel_display_power_put(i915, power_domain, wakeref); +} + +struct dbuf_slice_conf_entry { + u8 active_pipes; + u8 dbuf_mask[I915_MAX_PIPES]; + bool join_mbus; +}; + +/* + * Table taken from Bspec 12716 + * Pipes do have some preferred DBuf slice affinity, + * plus there are some hardcoded requirements on how + * those should be distributed for multipipe scenarios. + * For more DBuf slices algorithm can get even more messy + * and less readable, so decided to use a table almost + * as is from BSpec itself - that way it is at least easier + * to compare, change and check. + */ +static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] = +/* Autogenerated with igt/tools/intel_dbuf_map tool: */ +{ + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + }, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + {} +}; + +/* + * Table taken from Bspec 49255 + * Pipes do have some preferred DBuf slice affinity, + * plus there are some hardcoded requirements on how + * those should be distributed for multipipe scenarios. + * For more DBuf slices algorithm can get even more messy + * and less readable, so decided to use a table almost + * as is from BSpec itself - that way it is at least easier + * to compare, change and check. + */ +static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = +/* Autogenerated with igt/tools/intel_dbuf_map tool: */ +{ + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S1), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_D), + .dbuf_mask = { + [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S1), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S1), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S1), + [PIPE_C] = BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S2), + }, + }, + {} +}; + +static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = { + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_D), + .dbuf_mask = { + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1), + [PIPE_B] = BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3), + [PIPE_D] = BIT(DBUF_S4), + }, + }, + {} +}; + +static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = { + /* + * Keep the join_mbus cases first so check_mbus_joined() + * will prefer them over the !join_mbus cases. + */ + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), + }, + .join_mbus = true, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), + }, + .join_mbus = true, + }, + { + .active_pipes = BIT(PIPE_A), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + .join_mbus = false, + }, + { + .active_pipes = BIT(PIPE_B), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + .join_mbus = false, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_C), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + }, + }, + { + .active_pipes = BIT(PIPE_D), + .dbuf_mask = { + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + { + .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .dbuf_mask = { + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), + [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, + }, + {} + +}; + +static bool check_mbus_joined(u8 active_pipes, + const struct dbuf_slice_conf_entry *dbuf_slices) +{ + int i; + + for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { + if (dbuf_slices[i].active_pipes == active_pipes) + return dbuf_slices[i].join_mbus; + } + return false; +} + +static bool adlp_check_mbus_joined(u8 active_pipes) +{ + return check_mbus_joined(active_pipes, adlp_allowed_dbufs); +} + +static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus, + const struct dbuf_slice_conf_entry *dbuf_slices) +{ + int i; + + for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { + if (dbuf_slices[i].active_pipes == active_pipes && + dbuf_slices[i].join_mbus == join_mbus) + return dbuf_slices[i].dbuf_mask[pipe]; + } + return 0; +} + +/* + * This function finds an entry with same enabled pipe configuration and + * returns correspondent DBuf slice mask as stated in BSpec for particular + * platform. + */ +static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) +{ + /* + * FIXME: For ICL this is still a bit unclear as prev BSpec revision + * required calculating "pipe ratio" in order to determine + * if one or two slices can be used for single pipe configurations + * as additional constraint to the existing table. + * However based on recent info, it should be not "pipe ratio" + * but rather ratio between pixel_rate and cdclk with additional + * constants, so for now we are using only table until this is + * clarified. Also this is the reason why crtc_state param is + * still here - we will need it once those additional constraints + * pop up. + */ + return compute_dbuf_slices(pipe, active_pipes, join_mbus, + icl_allowed_dbufs); +} + +static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) +{ + return compute_dbuf_slices(pipe, active_pipes, join_mbus, + tgl_allowed_dbufs); +} + +static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) +{ + return compute_dbuf_slices(pipe, active_pipes, join_mbus, + adlp_allowed_dbufs); +} + +static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) +{ + return compute_dbuf_slices(pipe, active_pipes, join_mbus, + dg2_allowed_dbufs); +} + +static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + + if (IS_DG2(i915)) + return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus); + else if (DISPLAY_VER(i915) >= 13) + return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus); + else if (DISPLAY_VER(i915) == 12) + return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus); + else if (DISPLAY_VER(i915) == 11) + return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus); + /* + * For anything else just return one slice yet. + * Should be extended for other platforms. + */ + return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0; +} + +static bool +use_minimal_wm0_only(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + + return DISPLAY_VER(i915) >= 13 && + crtc_state->uapi.async_flip && + plane->async_flip; +} + +static u64 +skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + enum plane_id plane_id; + u64 data_rate = 0; + + for_each_plane_id_on_crtc(crtc, plane_id) { + if (plane_id == PLANE_CURSOR) + continue; + + data_rate += crtc_state->rel_data_rate[plane_id]; + + if (DISPLAY_VER(i915) < 11) + data_rate += crtc_state->rel_data_rate_y[plane_id]; + } + + return data_rate; +} + +static const struct skl_wm_level * +skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm, + enum plane_id plane_id, + int level) +{ + const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; + + if (level == 0 && pipe_wm->use_sagv_wm) + return &wm->sagv.wm0; + + return &wm->wm[level]; +} + +static const struct skl_wm_level * +skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm, + enum plane_id plane_id) +{ + const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; + + if (pipe_wm->use_sagv_wm) + return &wm->sagv.trans_wm; + + return &wm->trans_wm; +} + +/* + * We only disable the watermarks for each plane if + * they exceed the ddb allocation of said plane. This + * is done so that we don't end up touching cursor + * watermarks needlessly when some other plane reduces + * our max possible watermark level. + * + * Bspec has this to say about the PLANE_WM enable bit: + * "All the watermarks at this level for all enabled + * planes must be enabled before the level will be used." + * So this is actually safe to do. + */ +static void +skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb) +{ + if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) + memset(wm, 0, sizeof(*wm)); +} + +static void +skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm, + const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb) +{ + if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) || + uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) { + memset(wm, 0, sizeof(*wm)); + memset(uv_wm, 0, sizeof(*uv_wm)); + } +} + +static bool icl_need_wm1_wa(struct drm_i915_private *i915, + enum plane_id plane_id) +{ + /* + * Wa_1408961008:icl, ehl + * Wa_14012656716:tgl, adl + * Underruns with WM1+ disabled + */ + return DISPLAY_VER(i915) == 11 || + (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR); +} + +struct skl_plane_ddb_iter { + u64 data_rate; + u16 start, size; +}; + +static void +skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter, + struct skl_ddb_entry *ddb, + const struct skl_wm_level *wm, + u64 data_rate) +{ + u16 size, extra = 0; + + if (data_rate) { + extra = min_t(u16, iter->size, + DIV64_U64_ROUND_UP(iter->size * data_rate, + iter->data_rate)); + iter->size -= extra; + iter->data_rate -= data_rate; + } + + /* + * Keep ddb entry of all disabled planes explicitly zeroed + * to avoid skl_ddb_add_affected_planes() adding them to + * the state when other planes change their allocations. + */ + size = wm->min_ddb_alloc + extra; + if (size) + iter->start = skl_ddb_entry_init(ddb, iter->start, + iter->start + size); +} + +static int +skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + const struct intel_dbuf_state *dbuf_state = + intel_atomic_get_new_dbuf_state(state); + const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe]; + int num_active = hweight8(dbuf_state->active_pipes); + struct skl_plane_ddb_iter iter; + enum plane_id plane_id; + u16 cursor_size; + u32 blocks; + int level; + + /* Clear the partitioning for disabled planes. */ + memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb)); + memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); + + if (!crtc_state->hw.active) + return 0; + + iter.start = alloc->start; + iter.size = skl_ddb_entry_size(alloc); + if (iter.size == 0) + return 0; + + /* Allocate fixed number of blocks for cursor. */ + cursor_size = skl_cursor_allocation(crtc_state, num_active); + iter.size -= cursor_size; + skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR], + alloc->end - cursor_size, alloc->end); + + iter.data_rate = skl_total_relative_data_rate(crtc_state); + + /* + * Find the highest watermark level for which we can satisfy the block + * requirement of active planes. + */ + for (level = ilk_wm_max_level(i915); level >= 0; level--) { + blocks = 0; + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + if (plane_id == PLANE_CURSOR) { + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + + if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) { + drm_WARN_ON(&i915->drm, + wm->wm[level].min_ddb_alloc != U16_MAX); + blocks = U32_MAX; + break; + } + continue; + } + + blocks += wm->wm[level].min_ddb_alloc; + blocks += wm->uv_wm[level].min_ddb_alloc; + } + + if (blocks <= iter.size) { + iter.size -= blocks; + break; + } + } + + if (level < 0) { + drm_dbg_kms(&i915->drm, + "Requested display configuration exceeds system DDB limitations"); + drm_dbg_kms(&i915->drm, "minimum required %d/%d\n", + blocks, iter.size); + return -EINVAL; + } + + /* avoid the WARN later when we don't allocate any extra DDB */ + if (iter.data_rate == 0) + iter.size = 0; + + /* + * Grant each plane the blocks it requires at the highest achievable + * watermark level, plus an extra share of the leftover blocks + * proportional to its relative data rate. + */ + for_each_plane_id_on_crtc(crtc, plane_id) { + struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + if (plane_id == PLANE_CURSOR) + continue; + + if (DISPLAY_VER(i915) < 11 && + crtc_state->nv12_planes & BIT(plane_id)) { + skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level], + crtc_state->rel_data_rate_y[plane_id]); + skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level], + crtc_state->rel_data_rate[plane_id]); + } else { + skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level], + crtc_state->rel_data_rate[plane_id]); + } + } + drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0); + + /* + * When we calculated watermark values we didn't know how high + * of a level we'd actually be able to hit, so we just marked + * all levels as "enabled." Go back now and disable the ones + * that aren't actually possible. + */ + for (level++; level <= ilk_wm_max_level(i915); level++) { + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + const struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + if (DISPLAY_VER(i915) < 11 && + crtc_state->nv12_planes & BIT(plane_id)) + skl_check_nv12_wm_level(&wm->wm[level], + &wm->uv_wm[level], + ddb_y, ddb); + else + skl_check_wm_level(&wm->wm[level], ddb); + + if (icl_need_wm1_wa(i915, plane_id) && + level == 1 && wm->wm[0].enable) { + wm->wm[level].blocks = wm->wm[0].blocks; + wm->wm[level].lines = wm->wm[0].lines; + wm->wm[level].ignore_lines = wm->wm[0].ignore_lines; + } + } + } + + /* + * Go back and disable the transition and SAGV watermarks + * if it turns out we don't have enough DDB blocks for them. + */ + for_each_plane_id_on_crtc(crtc, plane_id) { + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + const struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + if (DISPLAY_VER(i915) < 11 && + crtc_state->nv12_planes & BIT(plane_id)) { + skl_check_wm_level(&wm->trans_wm, ddb_y); + } else { + WARN_ON(skl_ddb_entry_size(ddb_y)); + + skl_check_wm_level(&wm->trans_wm, ddb); + } + + skl_check_wm_level(&wm->sagv.wm0, ddb); + skl_check_wm_level(&wm->sagv.trans_wm, ddb); + } + + return 0; +} + +/* + * The max latency should be 257 (max the punit can code is 255 and we add 2us + * for the read latency) and cpp should always be <= 8, so that + * should allow pixel_rate up to ~2 GHz which seems sufficient since max + * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. + */ +static uint_fixed_16_16_t +skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate, + u8 cpp, u32 latency, u32 dbuf_block_size) +{ + u32 wm_intermediate_val; + uint_fixed_16_16_t ret; + + if (latency == 0) + return FP_16_16_MAX; + + wm_intermediate_val = latency * pixel_rate * cpp; + ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size); + + if (DISPLAY_VER(i915) >= 10) + ret = add_fixed16_u32(ret, 1); + + return ret; +} + +static uint_fixed_16_16_t +skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency, + uint_fixed_16_16_t plane_blocks_per_line) +{ + u32 wm_intermediate_val; + uint_fixed_16_16_t ret; + + if (latency == 0) + return FP_16_16_MAX; + + wm_intermediate_val = latency * pixel_rate; + wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val, + pipe_htotal * 1000); + ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line); + return ret; +} + +static uint_fixed_16_16_t +intel_get_linetime_us(const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + u32 pixel_rate; + u32 crtc_htotal; + uint_fixed_16_16_t linetime_us; + + if (!crtc_state->hw.active) + return u32_to_fixed16(0); + + pixel_rate = crtc_state->pixel_rate; + + if (drm_WARN_ON(&i915->drm, pixel_rate == 0)) + return u32_to_fixed16(0); + + crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal; + linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); + + return linetime_us; +} + +static int +skl_compute_wm_params(const struct intel_crtc_state *crtc_state, + int width, const struct drm_format_info *format, + u64 modifier, unsigned int rotation, + u32 plane_pixel_rate, struct skl_wm_params *wp, + int color_plane) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + u32 interm_pbpl; + + /* only planar format has two planes */ + if (color_plane == 1 && + !intel_format_info_is_yuv_semiplanar(format, modifier)) { + drm_dbg_kms(&i915->drm, + "Non planar format have single plane\n"); + return -EINVAL; + } + + wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED || + modifier == I915_FORMAT_MOD_4_TILED || + modifier == I915_FORMAT_MOD_Yf_TILED || + modifier == I915_FORMAT_MOD_Y_TILED_CCS || + modifier == I915_FORMAT_MOD_Yf_TILED_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC || + modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS || + modifier == I915_FORMAT_MOD_4_TILED_DG2_MC_CCS || + modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC; + wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED; + wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS || + modifier == I915_FORMAT_MOD_Yf_TILED_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS || + modifier == I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC || + modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS || + modifier == I915_FORMAT_MOD_4_TILED_DG2_MC_CCS || + modifier == I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC; + wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier); + + wp->width = width; + if (color_plane == 1 && wp->is_planar) + wp->width /= 2; + + wp->cpp = format->cpp[color_plane]; + wp->plane_pixel_rate = plane_pixel_rate; + + if (DISPLAY_VER(i915) >= 11 && + modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1) + wp->dbuf_block_size = 256; + else + wp->dbuf_block_size = 512; + + if (drm_rotation_90_or_270(rotation)) { + switch (wp->cpp) { + case 1: + wp->y_min_scanlines = 16; + break; + case 2: + wp->y_min_scanlines = 8; + break; + case 4: + wp->y_min_scanlines = 4; + break; + default: + MISSING_CASE(wp->cpp); + return -EINVAL; + } + } else { + wp->y_min_scanlines = 4; + } + + if (skl_needs_memory_bw_wa(i915)) + wp->y_min_scanlines *= 2; + + wp->plane_bytes_per_line = wp->width * wp->cpp; + if (wp->y_tiled) { + interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line * + wp->y_min_scanlines, + wp->dbuf_block_size); + + if (DISPLAY_VER(i915) >= 10) + interm_pbpl++; + + wp->plane_blocks_per_line = div_fixed16(interm_pbpl, + wp->y_min_scanlines); + } else { + interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, + wp->dbuf_block_size); + + if (!wp->x_tiled || DISPLAY_VER(i915) >= 10) + interm_pbpl++; + + wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); + } + + wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines, + wp->plane_blocks_per_line); + + wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state)); + + return 0; +} + +static int +skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + struct skl_wm_params *wp, int color_plane) +{ + const struct drm_framebuffer *fb = plane_state->hw.fb; + int width; + + /* + * Src coordinates are already rotated by 270 degrees for + * the 90/270 degree plane rotation cases (to match the + * GTT mapping), hence no need to account for rotation here. + */ + width = drm_rect_width(&plane_state->uapi.src) >> 16; + + return skl_compute_wm_params(crtc_state, width, + fb->format, fb->modifier, + plane_state->hw.rotation, + intel_plane_pixel_rate(crtc_state, plane_state), + wp, color_plane); +} + +static bool skl_wm_has_lines(struct drm_i915_private *i915, int level) +{ + if (DISPLAY_VER(i915) >= 10) + return true; + + /* The number of lines are ignored for the level 0 watermark. */ + return level > 0; +} + +static int skl_wm_max_lines(struct drm_i915_private *i915) +{ + if (DISPLAY_VER(i915) >= 13) + return 255; + else + return 31; +} + +static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, + int level, + unsigned int latency, + const struct skl_wm_params *wp, + const struct skl_wm_level *result_prev, + struct skl_wm_level *result /* out */) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + uint_fixed_16_16_t method1, method2; + uint_fixed_16_16_t selected_result; + u32 blocks, lines, min_ddb_alloc = 0; + + if (latency == 0 || + (use_minimal_wm0_only(crtc_state, plane) && level > 0)) { + /* reject it */ + result->min_ddb_alloc = U16_MAX; + return; + } + + /* + * WaIncreaseLatencyIPCEnabled: kbl,cfl + * Display WA #1141: kbl,cfl + */ + if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) && + skl_watermark_ipc_enabled(i915)) + latency += 4; + + if (skl_needs_memory_bw_wa(i915) && wp->x_tiled) + latency += 15; + + method1 = skl_wm_method1(i915, wp->plane_pixel_rate, + wp->cpp, latency, wp->dbuf_block_size); + method2 = skl_wm_method2(wp->plane_pixel_rate, + crtc_state->hw.pipe_mode.crtc_htotal, + latency, + wp->plane_blocks_per_line); + + if (wp->y_tiled) { + selected_result = max_fixed16(method2, wp->y_tile_minimum); + } else { + if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal / + wp->dbuf_block_size < 1) && + (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { + selected_result = method2; + } else if (latency >= wp->linetime_us) { + if (DISPLAY_VER(i915) == 9) + selected_result = min_fixed16(method1, method2); + else + selected_result = method2; + } else { + selected_result = method1; + } + } + + blocks = fixed16_to_u32_round_up(selected_result) + 1; + /* + * Lets have blocks at minimum equivalent to plane_blocks_per_line + * as there will be at minimum one line for lines configuration. This + * is a work around for FIFO underruns observed with resolutions like + * 4k 60 Hz in single channel DRAM configurations. + * + * As per the Bspec 49325, if the ddb allocation can hold at least + * one plane_blocks_per_line, we should have selected method2 in + * the above logic. Assuming that modern versions have enough dbuf + * and method2 guarantees blocks equivalent to at least 1 line, + * select the blocks as plane_blocks_per_line. + * + * TODO: Revisit the logic when we have better understanding on DRAM + * channels' impact on the level 0 memory latency and the relevant + * wm calculations. + */ + if (skl_wm_has_lines(i915, level)) + blocks = max(blocks, + fixed16_to_u32_round_up(wp->plane_blocks_per_line)); + lines = div_round_up_fixed16(selected_result, + wp->plane_blocks_per_line); + + if (DISPLAY_VER(i915) == 9) { + /* Display WA #1125: skl,bxt,kbl */ + if (level == 0 && wp->rc_surface) + blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); + + /* Display WA #1126: skl,bxt,kbl */ + if (level >= 1 && level <= 7) { + if (wp->y_tiled) { + blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); + lines += wp->y_min_scanlines; + } else { + blocks++; + } + + /* + * Make sure result blocks for higher latency levels are + * at least as high as level below the current level. + * Assumption in DDB algorithm optimization for special + * cases. Also covers Display WA #1125 for RC. + */ + if (result_prev->blocks > blocks) + blocks = result_prev->blocks; + } + } + + if (DISPLAY_VER(i915) >= 11) { + if (wp->y_tiled) { + int extra_lines; + + if (lines % wp->y_min_scanlines == 0) + extra_lines = wp->y_min_scanlines; + else + extra_lines = wp->y_min_scanlines * 2 - + lines % wp->y_min_scanlines; + + min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines, + wp->plane_blocks_per_line); + } else { + min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10); + } + } + + if (!skl_wm_has_lines(i915, level)) + lines = 0; + + if (lines > skl_wm_max_lines(i915)) { + /* reject it */ + result->min_ddb_alloc = U16_MAX; + return; + } + + /* + * If lines is valid, assume we can use this watermark level + * for now. We'll come back and disable it after we calculate the + * DDB allocation if it turns out we don't actually have enough + * blocks to satisfy it. + */ + result->blocks = blocks; + result->lines = lines; + /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ + result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1; + result->enable = true; + + if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us) + result->can_sagv = latency >= i915->display.sagv.block_time_us; +} + +static void +skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, + const struct skl_wm_params *wm_params, + struct skl_wm_level *levels) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + int level, max_level = ilk_wm_max_level(i915); + struct skl_wm_level *result_prev = &levels[0]; + + for (level = 0; level <= max_level; level++) { + struct skl_wm_level *result = &levels[level]; + unsigned int latency = i915->display.wm.skl_latency[level]; + + skl_compute_plane_wm(crtc_state, plane, level, latency, + wm_params, result_prev, result); + + result_prev = result; + } +} + +static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state, + struct intel_plane *plane, + const struct skl_wm_params *wm_params, + struct skl_plane_wm *plane_wm) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0; + struct skl_wm_level *levels = plane_wm->wm; + unsigned int latency = 0; + + if (i915->display.sagv.block_time_us) + latency = i915->display.sagv.block_time_us + i915->display.wm.skl_latency[0]; + + skl_compute_plane_wm(crtc_state, plane, 0, latency, + wm_params, &levels[0], + sagv_wm); +} + +static void skl_compute_transition_wm(struct drm_i915_private *i915, + struct skl_wm_level *trans_wm, + const struct skl_wm_level *wm0, + const struct skl_wm_params *wp) +{ + u16 trans_min, trans_amount, trans_y_tile_min; + u16 wm0_blocks, trans_offset, blocks; + + /* Transition WM don't make any sense if ipc is disabled */ + if (!skl_watermark_ipc_enabled(i915)) + return; + + /* + * WaDisableTWM:skl,kbl,cfl,bxt + * Transition WM are not recommended by HW team for GEN9 + */ + if (DISPLAY_VER(i915) == 9) + return; + + if (DISPLAY_VER(i915) >= 11) + trans_min = 4; + else + trans_min = 14; + + /* Display WA #1140: glk,cnl */ + if (DISPLAY_VER(i915) == 10) + trans_amount = 0; + else + trans_amount = 10; /* This is configurable amount */ + + trans_offset = trans_min + trans_amount; + + /* + * The spec asks for Selected Result Blocks for wm0 (the real value), + * not Result Blocks (the integer value). Pay attention to the capital + * letters. The value wm_l0->blocks is actually Result Blocks, but + * since Result Blocks is the ceiling of Selected Result Blocks plus 1, + * and since we later will have to get the ceiling of the sum in the + * transition watermarks calculation, we can just pretend Selected + * Result Blocks is Result Blocks minus 1 and it should work for the + * current platforms. + */ + wm0_blocks = wm0->blocks - 1; + + if (wp->y_tiled) { + trans_y_tile_min = + (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum); + blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset; + } else { + blocks = wm0_blocks + trans_offset; + } + blocks++; + + /* + * Just assume we can enable the transition watermark. After + * computing the DDB we'll come back and disable it if that + * assumption turns out to be false. + */ + trans_wm->blocks = blocks; + trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1); + trans_wm->enable = true; +} + +static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + struct intel_plane *plane, int color_plane) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; + struct skl_wm_params wm_params; + int ret; + + ret = skl_compute_plane_wm_params(crtc_state, plane_state, + &wm_params, color_plane); + if (ret) + return ret; + + skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm); + + skl_compute_transition_wm(i915, &wm->trans_wm, + &wm->wm[0], &wm_params); + + if (DISPLAY_VER(i915) >= 12) { + tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm); + + skl_compute_transition_wm(i915, &wm->sagv.trans_wm, + &wm->sagv.wm0, &wm_params); + } + + return 0; +} + +static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state, + struct intel_plane *plane) +{ + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; + struct skl_wm_params wm_params; + int ret; + + wm->is_planar = true; + + /* uv plane watermarks must also be validated for NV12/Planar */ + ret = skl_compute_plane_wm_params(crtc_state, plane_state, + &wm_params, 1); + if (ret) + return ret; + + skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm); + + return 0; +} + +static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + const struct drm_framebuffer *fb = plane_state->hw.fb; + int ret; + + memset(wm, 0, sizeof(*wm)); + + if (!intel_wm_plane_visible(crtc_state, plane_state)) + return 0; + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane, 0); + if (ret) + return ret; + + if (fb->format->is_yuv && fb->format->num_planes > 1) { + ret = skl_build_plane_wm_uv(crtc_state, plane_state, + plane); + if (ret) + return ret; + } + + return 0; +} + +static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + struct drm_i915_private *i915 = to_i915(plane->base.dev); + enum plane_id plane_id = plane->id; + struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; + int ret; + + /* Watermarks calculated in master */ + if (plane_state->planar_slave) + return 0; + + memset(wm, 0, sizeof(*wm)); + + if (plane_state->planar_linked_plane) { + const struct drm_framebuffer *fb = plane_state->hw.fb; + + drm_WARN_ON(&i915->drm, + !intel_wm_plane_visible(crtc_state, plane_state)); + drm_WARN_ON(&i915->drm, !fb->format->is_yuv || + fb->format->num_planes == 1); + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane_state->planar_linked_plane, 0); + if (ret) + return ret; + + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane, 1); + if (ret) + return ret; + } else if (intel_wm_plane_visible(crtc_state, plane_state)) { + ret = skl_build_plane_wm_single(crtc_state, plane_state, + plane, 0); + if (ret) + return ret; + } + + return 0; +} + +static int skl_build_pipe_wm(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + const struct intel_plane_state *plane_state; + struct intel_plane *plane; + int ret, i; + + for_each_new_intel_plane_in_state(state, plane, plane_state, i) { + /* + * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc + * instead but we don't populate that correctly for NV12 Y + * planes so for now hack this. + */ + if (plane->pipe != crtc->pipe) + continue; + + if (DISPLAY_VER(i915) >= 11) + ret = icl_build_plane_wm(crtc_state, plane_state); + else + ret = skl_build_plane_wm(crtc_state, plane_state); + if (ret) + return ret; + } + + crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw; + + return 0; +} + +static void skl_ddb_entry_write(struct drm_i915_private *i915, + i915_reg_t reg, + const struct skl_ddb_entry *entry) +{ + if (entry->end) + intel_de_write_fw(i915, reg, + PLANE_BUF_END(entry->end - 1) | + PLANE_BUF_START(entry->start)); + else + intel_de_write_fw(i915, reg, 0); +} + +static void skl_write_wm_level(struct drm_i915_private *i915, + i915_reg_t reg, + const struct skl_wm_level *level) +{ + u32 val = 0; + + if (level->enable) + val |= PLANE_WM_EN; + if (level->ignore_lines) + val |= PLANE_WM_IGNORE_LINES; + val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks); + val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines); + + intel_de_write_fw(i915, reg, val); +} + +void skl_write_plane_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + int level, max_level = ilk_wm_max_level(i915); + enum plane_id plane_id = plane->id; + enum pipe pipe = plane->pipe; + const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + const struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + + for (level = 0; level <= max_level; level++) + skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level), + skl_plane_wm_level(pipe_wm, plane_id, level)); + + skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id), + skl_plane_trans_wm(pipe_wm, plane_id)); + + if (HAS_HW_SAGV_WM(i915)) { + const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; + + skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id), + &wm->sagv.wm0); + skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id), + &wm->sagv.trans_wm); + } + + skl_ddb_entry_write(i915, + PLANE_BUF_CFG(pipe, plane_id), ddb); + + if (DISPLAY_VER(i915) < 11) + skl_ddb_entry_write(i915, + PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y); +} + +void skl_write_cursor_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + int level, max_level = ilk_wm_max_level(i915); + enum plane_id plane_id = plane->id; + enum pipe pipe = plane->pipe; + const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; + const struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + + for (level = 0; level <= max_level; level++) + skl_write_wm_level(i915, CUR_WM(pipe, level), + skl_plane_wm_level(pipe_wm, plane_id, level)); + + skl_write_wm_level(i915, CUR_WM_TRANS(pipe), + skl_plane_trans_wm(pipe_wm, plane_id)); + + if (HAS_HW_SAGV_WM(i915)) { + const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; + + skl_write_wm_level(i915, CUR_WM_SAGV(pipe), + &wm->sagv.wm0); + skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe), + &wm->sagv.trans_wm); + } + + skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb); +} + +static bool skl_wm_level_equals(const struct skl_wm_level *l1, + const struct skl_wm_level *l2) +{ + return l1->enable == l2->enable && + l1->ignore_lines == l2->ignore_lines && + l1->lines == l2->lines && + l1->blocks == l2->blocks; +} + +static bool skl_plane_wm_equals(struct drm_i915_private *i915, + const struct skl_plane_wm *wm1, + const struct skl_plane_wm *wm2) +{ + int level, max_level = ilk_wm_max_level(i915); + + for (level = 0; level <= max_level; level++) { + /* + * We don't check uv_wm as the hardware doesn't actually + * use it. It only gets used for calculating the required + * ddb allocation. + */ + if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level])) + return false; + } + + return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) && + skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) && + skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm); +} + +static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, + const struct skl_ddb_entry *b) +{ + return a->start < b->end && b->start < a->end; +} + +static void skl_ddb_entry_union(struct skl_ddb_entry *a, + const struct skl_ddb_entry *b) +{ + if (a->end && b->end) { + a->start = min(a->start, b->start); + a->end = max(a->end, b->end); + } else if (b->end) { + a->start = b->start; + a->end = b->end; + } +} + +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, + const struct skl_ddb_entry *entries, + int num_entries, int ignore_idx) +{ + int i; + + for (i = 0; i < num_entries; i++) { + if (i != ignore_idx && + skl_ddb_entries_overlap(ddb, &entries[i])) + return true; + } + + return false; +} + +static int +skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state, + struct intel_crtc_state *new_crtc_state) +{ + struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state); + struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct intel_plane *plane; + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + struct intel_plane_state *plane_state; + enum plane_id plane_id = plane->id; + + if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id], + &new_crtc_state->wm.skl.plane_ddb[plane_id]) && + skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id], + &new_crtc_state->wm.skl.plane_ddb_y[plane_id])) + continue; + + plane_state = intel_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) + return PTR_ERR(plane_state); + + new_crtc_state->update_planes |= BIT(plane_id); + } + + return 0; +} + +static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state) +{ + struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev); + u8 enabled_slices; + enum pipe pipe; + + /* + * FIXME: For now we always enable slice S1 as per + * the Bspec display initialization sequence. + */ + enabled_slices = BIT(DBUF_S1); + + for_each_pipe(i915, pipe) + enabled_slices |= dbuf_state->slices[pipe]; + + return enabled_slices; +} + +static int +skl_compute_ddb(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_dbuf_state *old_dbuf_state; + struct intel_dbuf_state *new_dbuf_state = NULL; + const struct intel_crtc_state *old_crtc_state; + struct intel_crtc_state *new_crtc_state; + struct intel_crtc *crtc; + int ret, i; + + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + new_dbuf_state = intel_atomic_get_dbuf_state(state); + if (IS_ERR(new_dbuf_state)) + return PTR_ERR(new_dbuf_state); + + old_dbuf_state = intel_atomic_get_old_dbuf_state(state); + break; + } + + if (!new_dbuf_state) + return 0; + + new_dbuf_state->active_pipes = + intel_calc_active_pipes(state, old_dbuf_state->active_pipes); + + if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) { + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + } + + if (HAS_MBUS_JOINING(i915)) + new_dbuf_state->joined_mbus = + adlp_check_mbus_joined(new_dbuf_state->active_pipes); + + for_each_intel_crtc(&i915->drm, crtc) { + enum pipe pipe = crtc->pipe; + + new_dbuf_state->slices[pipe] = + skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes, + new_dbuf_state->joined_mbus); + + if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe]) + continue; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + } + + new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state); + + if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices || + old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { + ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); + if (ret) + return ret; + + if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { + /* TODO: Implement vblank synchronized MBUS joining changes */ + ret = intel_modeset_all_pipes(state); + if (ret) + return ret; + } + + drm_dbg_kms(&i915->drm, + "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n", + old_dbuf_state->enabled_slices, + new_dbuf_state->enabled_slices, + INTEL_INFO(i915)->display.dbuf.slice_mask, + str_yes_no(old_dbuf_state->joined_mbus), + str_yes_no(new_dbuf_state->joined_mbus)); + } + + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + enum pipe pipe = crtc->pipe; + + new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state); + + if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe]) + continue; + + ret = intel_atomic_lock_global_state(&new_dbuf_state->base); + if (ret) + return ret; + } + + for_each_intel_crtc(&i915->drm, crtc) { + ret = skl_crtc_allocate_ddb(state, crtc); + if (ret) + return ret; + } + + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + ret = skl_crtc_allocate_plane_ddb(state, crtc); + if (ret) + return ret; + + ret = skl_ddb_add_affected_planes(old_crtc_state, + new_crtc_state); + if (ret) + return ret; + } + + return 0; +} + +static char enast(bool enable) +{ + return enable ? '*' : ' '; +} + +static void +skl_print_wm_changes(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state; + const struct intel_crtc_state *new_crtc_state; + struct intel_plane *plane; + struct intel_crtc *crtc; + int i; + + if (!drm_debug_enabled(DRM_UT_KMS)) + return; + + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm; + + old_pipe_wm = &old_crtc_state->wm.skl.optimal; + new_pipe_wm = &new_crtc_state->wm.skl.optimal; + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + enum plane_id plane_id = plane->id; + const struct skl_ddb_entry *old, *new; + + old = &old_crtc_state->wm.skl.plane_ddb[plane_id]; + new = &new_crtc_state->wm.skl.plane_ddb[plane_id]; + + if (skl_ddb_entry_equal(old, new)) + continue; + + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n", + plane->base.base.id, plane->base.name, + old->start, old->end, new->start, new->end, + skl_ddb_entry_size(old), skl_ddb_entry_size(new)); + } + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + enum plane_id plane_id = plane->id; + const struct skl_plane_wm *old_wm, *new_wm; + + old_wm = &old_pipe_wm->planes[plane_id]; + new_wm = &new_pipe_wm->planes[plane_id]; + + if (skl_plane_wm_equals(i915, old_wm, new_wm)) + continue; + + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm" + " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n", + plane->base.base.id, plane->base.name, + enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable), + enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable), + enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable), + enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable), + enast(old_wm->trans_wm.enable), + enast(old_wm->sagv.wm0.enable), + enast(old_wm->sagv.trans_wm.enable), + enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable), + enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable), + enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable), + enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable), + enast(new_wm->trans_wm.enable), + enast(new_wm->sagv.wm0.enable), + enast(new_wm->sagv.trans_wm.enable)); + + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d" + " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n", + plane->base.base.id, plane->base.name, + enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines, + enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines, + enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines, + enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines, + enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines, + enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines, + enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines, + enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines, + enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines, + enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines, + enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines, + enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines, + enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines, + enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines, + enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines, + enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines, + enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines, + enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines, + enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines, + enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines, + enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines, + enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines); + + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" + " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", + plane->base.base.id, plane->base.name, + old_wm->wm[0].blocks, old_wm->wm[1].blocks, + old_wm->wm[2].blocks, old_wm->wm[3].blocks, + old_wm->wm[4].blocks, old_wm->wm[5].blocks, + old_wm->wm[6].blocks, old_wm->wm[7].blocks, + old_wm->trans_wm.blocks, + old_wm->sagv.wm0.blocks, + old_wm->sagv.trans_wm.blocks, + new_wm->wm[0].blocks, new_wm->wm[1].blocks, + new_wm->wm[2].blocks, new_wm->wm[3].blocks, + new_wm->wm[4].blocks, new_wm->wm[5].blocks, + new_wm->wm[6].blocks, new_wm->wm[7].blocks, + new_wm->trans_wm.blocks, + new_wm->sagv.wm0.blocks, + new_wm->sagv.trans_wm.blocks); + + drm_dbg_kms(&i915->drm, + "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" + " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", + plane->base.base.id, plane->base.name, + old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc, + old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc, + old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc, + old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc, + old_wm->trans_wm.min_ddb_alloc, + old_wm->sagv.wm0.min_ddb_alloc, + old_wm->sagv.trans_wm.min_ddb_alloc, + new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc, + new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc, + new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc, + new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc, + new_wm->trans_wm.min_ddb_alloc, + new_wm->sagv.wm0.min_ddb_alloc, + new_wm->sagv.trans_wm.min_ddb_alloc); + } + } +} + +static bool skl_plane_selected_wm_equals(struct intel_plane *plane, + const struct skl_pipe_wm *old_pipe_wm, + const struct skl_pipe_wm *new_pipe_wm) +{ + struct drm_i915_private *i915 = to_i915(plane->base.dev); + int level, max_level = ilk_wm_max_level(i915); + + for (level = 0; level <= max_level; level++) { + /* + * We don't check uv_wm as the hardware doesn't actually + * use it. It only gets used for calculating the required + * ddb allocation. + */ + if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level), + skl_plane_wm_level(new_pipe_wm, plane->id, level))) + return false; + } + + if (HAS_HW_SAGV_WM(i915)) { + const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id]; + const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id]; + + if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) || + !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm)) + return false; + } + + return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id), + skl_plane_trans_wm(new_pipe_wm, plane->id)); +} + +/* + * To make sure the cursor watermark registers are always consistent + * with our computed state the following scenario needs special + * treatment: + * + * 1. enable cursor + * 2. move cursor entirely offscreen + * 3. disable cursor + * + * Step 2. does call .disable_plane() but does not zero the watermarks + * (since we consider an offscreen cursor still active for the purposes + * of watermarks). Step 3. would not normally call .disable_plane() + * because the actual plane visibility isn't changing, and we don't + * deallocate the cursor ddb until the pipe gets disabled. So we must + * force step 3. to call .disable_plane() to update the watermark + * registers properly. + * + * Other planes do not suffer from this issues as their watermarks are + * calculated based on the actual plane visibility. The only time this + * can trigger for the other planes is during the initial readout as the + * default value of the watermarks registers is not zero. + */ +static int skl_wm_add_affected_planes(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + struct intel_plane *plane; + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + struct intel_plane_state *plane_state; + enum plane_id plane_id = plane->id; + + /* + * Force a full wm update for every plane on modeset. + * Required because the reset value of the wm registers + * is non-zero, whereas we want all disabled planes to + * have zero watermarks. So if we turn off the relevant + * power well the hardware state will go out of sync + * with the software state. + */ + if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) && + skl_plane_selected_wm_equals(plane, + &old_crtc_state->wm.skl.optimal, + &new_crtc_state->wm.skl.optimal)) + continue; + + plane_state = intel_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) + return PTR_ERR(plane_state); + + new_crtc_state->update_planes |= BIT(plane_id); + } + + return 0; +} + +static int +skl_compute_wm(struct intel_atomic_state *state) +{ + struct intel_crtc *crtc; + struct intel_crtc_state *new_crtc_state; + int ret, i; + + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + ret = skl_build_pipe_wm(state, crtc); + if (ret) + return ret; + } + + ret = skl_compute_ddb(state); + if (ret) + return ret; + + ret = intel_compute_sagv_mask(state); + if (ret) + return ret; + + /* + * skl_compute_ddb() will have adjusted the final watermarks + * based on how much ddb is available. Now we can actually + * check if the final watermarks changed. + */ + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + ret = skl_wm_add_affected_planes(state, crtc); + if (ret) + return ret; + } + + skl_print_wm_changes(state); + + return 0; +} + +static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level) +{ + level->enable = val & PLANE_WM_EN; + level->ignore_lines = val & PLANE_WM_IGNORE_LINES; + level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val); + level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val); +} + +static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, + struct skl_pipe_wm *out) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + enum pipe pipe = crtc->pipe; + int level, max_level; + enum plane_id plane_id; + u32 val; + + max_level = ilk_wm_max_level(i915); + + for_each_plane_id_on_crtc(crtc, plane_id) { + struct skl_plane_wm *wm = &out->planes[plane_id]; + + for (level = 0; level <= max_level; level++) { + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&i915->uncore, PLANE_WM(pipe, plane_id, level)); + else + val = intel_uncore_read(&i915->uncore, CUR_WM(pipe, level)); + + skl_wm_level_from_reg_val(val, &wm->wm[level]); + } + + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&i915->uncore, PLANE_WM_TRANS(pipe, plane_id)); + else + val = intel_uncore_read(&i915->uncore, CUR_WM_TRANS(pipe)); + + skl_wm_level_from_reg_val(val, &wm->trans_wm); + + if (HAS_HW_SAGV_WM(i915)) { + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&i915->uncore, + PLANE_WM_SAGV(pipe, plane_id)); + else + val = intel_uncore_read(&i915->uncore, + CUR_WM_SAGV(pipe)); + + skl_wm_level_from_reg_val(val, &wm->sagv.wm0); + + if (plane_id != PLANE_CURSOR) + val = intel_uncore_read(&i915->uncore, + PLANE_WM_SAGV_TRANS(pipe, plane_id)); + else + val = intel_uncore_read(&i915->uncore, + CUR_WM_SAGV_TRANS(pipe)); + + skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm); + } else if (DISPLAY_VER(i915) >= 12) { + wm->sagv.wm0 = wm->wm[0]; + wm->sagv.trans_wm = wm->trans_wm; + } + } +} + +void skl_wm_get_hw_state(struct drm_i915_private *i915) +{ + struct intel_dbuf_state *dbuf_state = + to_intel_dbuf_state(i915->display.dbuf.obj.state); + struct intel_crtc *crtc; + + if (HAS_MBUS_JOINING(i915)) + dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN; + + for_each_intel_crtc(&i915->drm, crtc) { + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + enum pipe pipe = crtc->pipe; + unsigned int mbus_offset; + enum plane_id plane_id; + u8 slices; + + memset(&crtc_state->wm.skl.optimal, 0, + sizeof(crtc_state->wm.skl.optimal)); + if (crtc_state->hw.active) + skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); + crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal; + + memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe])); + + for_each_plane_id_on_crtc(crtc, plane_id) { + struct skl_ddb_entry *ddb = + &crtc_state->wm.skl.plane_ddb[plane_id]; + struct skl_ddb_entry *ddb_y = + &crtc_state->wm.skl.plane_ddb_y[plane_id]; + + if (!crtc_state->hw.active) + continue; + + skl_ddb_get_hw_plane_state(i915, crtc->pipe, + plane_id, ddb, ddb_y); + + skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb); + skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y); + } + + dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state); + + /* + * Used for checking overlaps, so we need absolute + * offsets instead of MBUS relative offsets. + */ + slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, + dbuf_state->joined_mbus); + mbus_offset = mbus_ddb_offset(i915, slices); + crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start; + crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end; + + /* The slices actually used by the planes on the pipe */ + dbuf_state->slices[pipe] = + skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb); + + drm_dbg_kms(&i915->drm, + "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n", + crtc->base.base.id, crtc->base.name, + dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start, + dbuf_state->ddb[pipe].end, dbuf_state->active_pipes, + str_yes_no(dbuf_state->joined_mbus)); + } + + dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices; +} + +static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915) +{ + const struct intel_dbuf_state *dbuf_state = + to_intel_dbuf_state(i915->display.dbuf.obj.state); + struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; + struct intel_crtc *crtc; + + for_each_intel_crtc(&i915->drm, crtc) { + const struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + + entries[crtc->pipe] = crtc_state->wm.skl.ddb; + } + + for_each_intel_crtc(&i915->drm, crtc) { + const struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + u8 slices; + + slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, + dbuf_state->joined_mbus); + if (dbuf_state->slices[crtc->pipe] & ~slices) + return true; + + if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries, + I915_MAX_PIPES, crtc->pipe)) + return true; + } + + return false; +} + +void skl_wm_sanitize(struct drm_i915_private *i915) +{ + struct intel_crtc *crtc; + + /* + * On TGL/RKL (at least) the BIOS likes to assign the planes + * to the wrong DBUF slices. This will cause an infinite loop + * in skl_commit_modeset_enables() as it can't find a way to + * transition between the old bogus DBUF layout to the new + * proper DBUF layout without DBUF allocation overlaps between + * the planes (which cannot be allowed or else the hardware + * may hang). If we detect a bogus DBUF layout just turn off + * all the planes so that skl_commit_modeset_enables() can + * simply ignore them. + */ + if (!skl_dbuf_is_misconfigured(i915)) + return; + + drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n"); + + for_each_intel_crtc(&i915->drm, crtc) { + struct intel_plane *plane = to_intel_plane(crtc->base.primary); + const struct intel_plane_state *plane_state = + to_intel_plane_state(plane->base.state); + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + + if (plane_state->uapi.visible) + intel_plane_disable_noatomic(crtc, plane); + + drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0); + + memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb)); + } +} + +void intel_wm_state_verify(struct intel_crtc *crtc, + struct intel_crtc_state *new_crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + struct skl_hw_state { + struct skl_ddb_entry ddb[I915_MAX_PLANES]; + struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; + struct skl_pipe_wm wm; + } *hw; + const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal; + int level, max_level = ilk_wm_max_level(i915); + struct intel_plane *plane; + u8 hw_enabled_slices; + + if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active) + return; + + hw = kzalloc(sizeof(*hw), GFP_KERNEL); + if (!hw) + return; + + skl_pipe_wm_get_hw_state(crtc, &hw->wm); + + skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y); + + hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915); + + if (DISPLAY_VER(i915) >= 11 && + hw_enabled_slices != i915->display.dbuf.enabled_slices) + drm_err(&i915->drm, + "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n", + i915->display.dbuf.enabled_slices, + hw_enabled_slices); + + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { + const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry; + const struct skl_wm_level *hw_wm_level, *sw_wm_level; + + /* Watermarks */ + for (level = 0; level <= max_level; level++) { + hw_wm_level = &hw->wm.planes[plane->id].wm[level]; + sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level); + + if (skl_wm_level_equals(hw_wm_level, sw_wm_level)) + continue; + + drm_err(&i915->drm, + "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, level, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + + hw_wm_level = &hw->wm.planes[plane->id].trans_wm; + sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id); + + if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) { + drm_err(&i915->drm, + "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + + hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0; + sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0; + + if (HAS_HW_SAGV_WM(i915) && + !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { + drm_err(&i915->drm, + "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + + hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm; + sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm; + + if (HAS_HW_SAGV_WM(i915) && + !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { + drm_err(&i915->drm, + "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", + plane->base.base.id, plane->base.name, + sw_wm_level->enable, + sw_wm_level->blocks, + sw_wm_level->lines, + hw_wm_level->enable, + hw_wm_level->blocks, + hw_wm_level->lines); + } + + /* DDB */ + hw_ddb_entry = &hw->ddb[PLANE_CURSOR]; + sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR]; + + if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { + drm_err(&i915->drm, + "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n", + plane->base.base.id, plane->base.name, + sw_ddb_entry->start, sw_ddb_entry->end, + hw_ddb_entry->start, hw_ddb_entry->end); + } + } + + kfree(hw); +} + +bool skl_watermark_ipc_enabled(struct drm_i915_private *i915) +{ + return i915->display.wm.ipc_enabled; +} + +void skl_watermark_ipc_update(struct drm_i915_private *i915) +{ + if (!HAS_IPC(i915)) + return; + + intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL2, DISP_IPC_ENABLE, + skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0); +} + +static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915) +{ + /* Display WA #0477 WaDisableIPC: skl */ + if (IS_SKYLAKE(i915)) + return false; + + /* Display WA #1141: SKL:all KBL:all CFL */ + if (IS_KABYLAKE(i915) || + IS_COFFEELAKE(i915) || + IS_COMETLAKE(i915)) + return i915->dram_info.symmetric_memory; + + return true; +} + +void skl_watermark_ipc_init(struct drm_i915_private *i915) +{ + if (!HAS_IPC(i915)) + return; + + i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915); + + skl_watermark_ipc_update(i915); +} + +static void +adjust_wm_latency(struct drm_i915_private *i915, + u16 wm[], int max_level, int read_latency) +{ + bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed; + int i, level; + + /* + * If a level n (n > 1) has a 0us latency, all levels m (m >= n) + * need to be disabled. We make sure to sanitize the values out + * of the punit to satisfy this requirement. + */ + for (level = 1; level <= max_level; level++) { + if (wm[level] == 0) { + for (i = level + 1; i <= max_level; i++) + wm[i] = 0; + + max_level = level - 1; + break; + } + } + + /* + * WaWmMemoryReadLatency + * + * punit doesn't take into account the read latency so we need + * to add proper adjustement to each valid level we retrieve + * from the punit when level 0 response data is 0us. + */ + if (wm[0] == 0) { + for (level = 0; level <= max_level; level++) + wm[level] += read_latency; + } + + /* + * WA Level-0 adjustment for 16GB DIMMs: SKL+ + * If we could not get dimm info enable this WA to prevent from + * any underrun. If not able to get Dimm info assume 16GB dimm + * to avoid any underrun. + */ + if (wm_lv_0_adjust_needed) + wm[0] += 1; +} + +static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) +{ + struct intel_uncore *uncore = &i915->uncore; + int max_level = ilk_wm_max_level(i915); + u32 val; + + val = intel_uncore_read(uncore, MTL_LATENCY_LP0_LP1); + wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); + wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); + + val = intel_uncore_read(uncore, MTL_LATENCY_LP2_LP3); + wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); + wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); + + val = intel_uncore_read(uncore, MTL_LATENCY_LP4_LP5); + wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val); + wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val); + + adjust_wm_latency(i915, wm, max_level, 6); +} + +static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) +{ + int max_level = ilk_wm_max_level(i915); + int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2; + int mult = IS_DG2(i915) ? 2 : 1; + u32 val; + int ret; + + /* read the first set of memory latencies[0:3] */ + val = 0; /* data0 to be programmed to 0 for first set */ + ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL); + if (ret) { + drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret); + return; + } + + wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult; + wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult; + wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult; + wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult; + + /* read the second set of memory latencies[4:7] */ + val = 1; /* data0 to be programmed to 1 for second set */ + ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL); + if (ret) { + drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret); + return; + } + + wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult; + wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult; + wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult; + wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult; + + adjust_wm_latency(i915, wm, max_level, read_latency); +} + +static void skl_setup_wm_latency(struct drm_i915_private *i915) +{ + if (DISPLAY_VER(i915) >= 14) + mtl_read_wm_latency(i915, i915->display.wm.skl_latency); + else + skl_read_wm_latency(i915, i915->display.wm.skl_latency); + + intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency); +} + +static const struct intel_wm_funcs skl_wm_funcs = { + .compute_global_watermarks = skl_compute_wm, +}; + +void skl_wm_init(struct drm_i915_private *i915) +{ + intel_sagv_init(i915); + + skl_setup_wm_latency(i915); + + i915->display.funcs.wm = &skl_wm_funcs; +} + +static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj) +{ + struct intel_dbuf_state *dbuf_state; + + dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL); + if (!dbuf_state) + return NULL; + + return &dbuf_state->base; +} + +static void intel_dbuf_destroy_state(struct intel_global_obj *obj, + struct intel_global_state *state) +{ + kfree(state); +} + +static const struct intel_global_state_funcs intel_dbuf_funcs = { + .atomic_duplicate_state = intel_dbuf_duplicate_state, + .atomic_destroy_state = intel_dbuf_destroy_state, +}; + +struct intel_dbuf_state * +intel_atomic_get_dbuf_state(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + struct intel_global_state *dbuf_state; + + dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj); + if (IS_ERR(dbuf_state)) + return ERR_CAST(dbuf_state); + + return to_intel_dbuf_state(dbuf_state); +} + +int intel_dbuf_init(struct drm_i915_private *i915) +{ + struct intel_dbuf_state *dbuf_state; + + dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL); + if (!dbuf_state) + return -ENOMEM; + + intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj, + &dbuf_state->base, &intel_dbuf_funcs); + + return 0; +} + +/* + * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before + * update the request state of all DBUS slices. + */ +static void update_mbus_pre_enable(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + u32 mbus_ctl, dbuf_min_tracker_val; + enum dbuf_slice slice; + const struct intel_dbuf_state *dbuf_state = + intel_atomic_get_new_dbuf_state(state); + + if (!HAS_MBUS_JOINING(i915)) + return; + + /* + * TODO: Implement vblank synchronized MBUS joining changes. + * Must be properly coordinated with dbuf reprogramming. + */ + if (dbuf_state->joined_mbus) { + mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN | + MBUS_JOIN_PIPE_SELECT_NONE; + dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3); + } else { + mbus_ctl = MBUS_HASHING_MODE_2x2 | + MBUS_JOIN_PIPE_SELECT_NONE; + dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1); + } + + intel_de_rmw(i915, MBUS_CTL, + MBUS_HASHING_MODE_MASK | MBUS_JOIN | + MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl); + + for_each_dbuf_slice(i915, slice) + intel_de_rmw(i915, DBUF_CTL_S(slice), + DBUF_MIN_TRACKER_STATE_SERVICE_MASK, + dbuf_min_tracker_val); +} + +void intel_dbuf_pre_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_dbuf_state *new_dbuf_state = + intel_atomic_get_new_dbuf_state(state); + const struct intel_dbuf_state *old_dbuf_state = + intel_atomic_get_old_dbuf_state(state); + + if (!new_dbuf_state || + (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices && + new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)) + return; + + WARN_ON(!new_dbuf_state->base.changed); + + update_mbus_pre_enable(state); + gen9_dbuf_slices_update(i915, + old_dbuf_state->enabled_slices | + new_dbuf_state->enabled_slices); +} + +void intel_dbuf_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_dbuf_state *new_dbuf_state = + intel_atomic_get_new_dbuf_state(state); + const struct intel_dbuf_state *old_dbuf_state = + intel_atomic_get_old_dbuf_state(state); + + if (!new_dbuf_state || + (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices && + new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)) + return; + + WARN_ON(!new_dbuf_state->base.changed); + + gen9_dbuf_slices_update(i915, + new_dbuf_state->enabled_slices); +} + +static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes) +{ + switch (pipe) { + case PIPE_A: + return !(active_pipes & BIT(PIPE_D)); + case PIPE_D: + return !(active_pipes & BIT(PIPE_A)); + case PIPE_B: + return !(active_pipes & BIT(PIPE_C)); + case PIPE_C: + return !(active_pipes & BIT(PIPE_B)); + default: /* to suppress compiler warning */ + MISSING_CASE(pipe); + break; + } + + return false; +} + +void intel_mbus_dbox_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *i915 = to_i915(state->base.dev); + const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state; + const struct intel_crtc_state *new_crtc_state; + const struct intel_crtc *crtc; + u32 val = 0; + int i; + + if (DISPLAY_VER(i915) < 11) + return; + + new_dbuf_state = intel_atomic_get_new_dbuf_state(state); + old_dbuf_state = intel_atomic_get_old_dbuf_state(state); + if (!new_dbuf_state || + (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus && + new_dbuf_state->active_pipes == old_dbuf_state->active_pipes)) + return; + + if (DISPLAY_VER(i915) >= 14) + val |= MBUS_DBOX_I_CREDIT(2); + + if (DISPLAY_VER(i915) >= 12) { + val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16); + val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1); + val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN; + } + + if (DISPLAY_VER(i915) >= 14) + val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) : + MBUS_DBOX_A_CREDIT(8); + else if (IS_ALDERLAKE_P(i915)) + /* Wa_22010947358:adl-p */ + val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) : + MBUS_DBOX_A_CREDIT(4); + else + val |= MBUS_DBOX_A_CREDIT(2); + + if (DISPLAY_VER(i915) >= 14) { + val |= MBUS_DBOX_B_CREDIT(0xA); + } else if (IS_ALDERLAKE_P(i915)) { + val |= MBUS_DBOX_BW_CREDIT(2); + val |= MBUS_DBOX_B_CREDIT(8); + } else if (DISPLAY_VER(i915) >= 12) { + val |= MBUS_DBOX_BW_CREDIT(2); + val |= MBUS_DBOX_B_CREDIT(12); + } else { + val |= MBUS_DBOX_BW_CREDIT(1); + val |= MBUS_DBOX_B_CREDIT(8); + } + + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + u32 pipe_val = val; + + if (!new_crtc_state->hw.active) + continue; + + if (DISPLAY_VER(i915) >= 14) { + if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe, + new_dbuf_state->active_pipes)) + pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL; + else + pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL; + } + + intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val); + } +} + +static int skl_watermark_ipc_status_show(struct seq_file *m, void *data) +{ + struct drm_i915_private *i915 = m->private; + + seq_printf(m, "Isochronous Priority Control: %s\n", + str_yes_no(skl_watermark_ipc_enabled(i915))); + return 0; +} + +static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file) +{ + struct drm_i915_private *i915 = inode->i_private; + + return single_open(file, skl_watermark_ipc_status_show, i915); +} + +static ssize_t skl_watermark_ipc_status_write(struct file *file, + const char __user *ubuf, + size_t len, loff_t *offp) +{ + struct seq_file *m = file->private_data; + struct drm_i915_private *i915 = m->private; + intel_wakeref_t wakeref; + bool enable; + int ret; + + ret = kstrtobool_from_user(ubuf, len, &enable); + if (ret < 0) + return ret; + + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + if (!skl_watermark_ipc_enabled(i915) && enable) + drm_info(&i915->drm, + "Enabling IPC: WM will be proper only after next commit\n"); + i915->display.wm.ipc_enabled = enable; + skl_watermark_ipc_update(i915); + } + + return len; +} + +static const struct file_operations skl_watermark_ipc_status_fops = { + .owner = THIS_MODULE, + .open = skl_watermark_ipc_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, + .write = skl_watermark_ipc_status_write +}; + +void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915) +{ + struct drm_minor *minor = i915->drm.primary; + + if (!HAS_IPC(i915)) + return; + + debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915, + &skl_watermark_ipc_status_fops); +} diff --git a/drivers/gpu/drm/i915/display/skl_watermark.h b/drivers/gpu/drm/i915/display/skl_watermark.h new file mode 100644 index 0000000000000000000000000000000000000000..7a5a4e67cd73878cc74981208f6ebd13558f349a --- /dev/null +++ b/drivers/gpu/drm/i915/display/skl_watermark.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __SKL_WATERMARK_H__ +#define __SKL_WATERMARK_H__ + +#include + +#include "intel_display.h" +#include "intel_global_state.h" +#include "intel_pm_types.h" + +struct drm_i915_private; +struct intel_atomic_state; +struct intel_bw_state; +struct intel_crtc; +struct intel_crtc_state; +struct intel_plane; + +u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915); + +void intel_sagv_pre_plane_update(struct intel_atomic_state *state); +void intel_sagv_post_plane_update(struct intel_atomic_state *state); +bool intel_can_enable_sagv(struct drm_i915_private *i915, + const struct intel_bw_state *bw_state); + +u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915, + const struct skl_ddb_entry *entry); + +void skl_write_plane_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state); +void skl_write_cursor_wm(struct intel_plane *plane, + const struct intel_crtc_state *crtc_state); + +bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, + const struct skl_ddb_entry *entries, + int num_entries, int ignore_idx); + +void skl_wm_get_hw_state(struct drm_i915_private *i915); +void skl_wm_sanitize(struct drm_i915_private *i915); + +void intel_wm_state_verify(struct intel_crtc *crtc, + struct intel_crtc_state *new_crtc_state); + +void skl_watermark_ipc_init(struct drm_i915_private *i915); +void skl_watermark_ipc_update(struct drm_i915_private *i915); +bool skl_watermark_ipc_enabled(struct drm_i915_private *i915); +void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915); + +void skl_wm_init(struct drm_i915_private *i915); + +struct intel_dbuf_state { + struct intel_global_state base; + + struct skl_ddb_entry ddb[I915_MAX_PIPES]; + unsigned int weight[I915_MAX_PIPES]; + u8 slices[I915_MAX_PIPES]; + u8 enabled_slices; + u8 active_pipes; + bool joined_mbus; +}; + +struct intel_dbuf_state * +intel_atomic_get_dbuf_state(struct intel_atomic_state *state); + +#define to_intel_dbuf_state(x) container_of((x), struct intel_dbuf_state, base) +#define intel_atomic_get_old_dbuf_state(state) \ + to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj)) +#define intel_atomic_get_new_dbuf_state(state) \ + to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj)) + +int intel_dbuf_init(struct drm_i915_private *i915); +void intel_dbuf_pre_plane_update(struct intel_atomic_state *state); +void intel_dbuf_post_plane_update(struct intel_atomic_state *state); +void intel_mbus_dbox_update(struct intel_atomic_state *state); + +#endif /* __SKL_WATERMARK_H__ */ + diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c index b9b1fed998740bb446afa02a29ea5172ede722e9..b3f5ca280ef2683e16806cd0af59bed38289ad10 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi.c @@ -822,9 +822,9 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state, u32 val; /* Disable DPOunit clock gating, can stall pipe */ - val = intel_de_read(dev_priv, DSPCLK_GATE_D); + val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv)); val |= DPOUNIT_CLOCK_GATE_DISABLE; - intel_de_write(dev_priv, DSPCLK_GATE_D, val); + intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val); } if (!IS_GEMINILAKE(dev_priv)) @@ -998,9 +998,9 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state, vlv_dsi_pll_disable(encoder); - val = intel_de_read(dev_priv, DSPCLK_GATE_D); + val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv)); val &= ~DPOUNIT_CLOCK_GATE_DISABLE; - intel_de_write(dev_priv, DSPCLK_GATE_D, val); + intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val); } /* Assert reset */ @@ -1277,13 +1277,12 @@ static void intel_dsi_get_config(struct intel_encoder *encoder, pclk = vlv_dsi_get_pclk(encoder, pipe_config); } - if (intel_dsi->dual_link) - pclk *= 2; + pipe_config->port_clock = pclk; - if (pclk) { - pipe_config->hw.adjusted_mode.crtc_clock = pclk; - pipe_config->port_clock = pclk; - } + /* FIXME definitely not right for burst/cmd mode/pixel overlap */ + pipe_config->hw.adjusted_mode.crtc_clock = pclk; + if (intel_dsi->dual_link) + pipe_config->hw.adjusted_mode.crtc_clock *= 2; } /* return txclkesc cycles in terms of divider and duration in us */ @@ -1872,9 +1871,9 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) return; if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - dev_priv->mipi_mmio_base = BXT_MIPI_BASE; + dev_priv->display.dsi.mmio_base = BXT_MIPI_BASE; else - dev_priv->mipi_mmio_base = VLV_MIPI_BASE; + dev_priv->display.dsi.mmio_base = VLV_MIPI_BASE; intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL); if (!intel_dsi) @@ -1933,8 +1932,11 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv) else intel_dsi->ports = BIT(port); - intel_dsi->dcs_backlight_ports = intel_connector->panel.vbt.dsi.bl_ports; - intel_dsi->dcs_cabc_ports = intel_connector->panel.vbt.dsi.cabc_ports; + if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports)) + intel_connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports; + + if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports)) + intel_connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports; /* Create a DSI host (and a device) for each port. */ for_each_dsi_port(port, intel_dsi->ports) { diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c index 5894b0138343846a55593fb55bb8be9db9108fba..af7402127cd99a99f07f22b41a1396b4c275382a 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c @@ -113,6 +113,61 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv, return 0; } +static int vlv_dsi_pclk(struct intel_encoder *encoder, + struct intel_crtc_state *config) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); + int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); + u32 dsi_clock; + u32 pll_ctl, pll_div; + u32 m = 0, p = 0, n; + int refclk = IS_CHERRYVIEW(dev_priv) ? 100000 : 25000; + int i; + + pll_ctl = config->dsi_pll.ctrl; + pll_div = config->dsi_pll.div; + + /* mask out other bits and extract the P1 divisor */ + pll_ctl &= DSI_PLL_P1_POST_DIV_MASK; + pll_ctl = pll_ctl >> (DSI_PLL_P1_POST_DIV_SHIFT - 2); + + /* N1 divisor */ + n = (pll_div & DSI_PLL_N1_DIV_MASK) >> DSI_PLL_N1_DIV_SHIFT; + n = 1 << n; /* register has log2(N1) */ + + /* mask out the other bits and extract the M1 divisor */ + pll_div &= DSI_PLL_M1_DIV_MASK; + pll_div = pll_div >> DSI_PLL_M1_DIV_SHIFT; + + while (pll_ctl) { + pll_ctl = pll_ctl >> 1; + p++; + } + p--; + + if (!p) { + drm_err(&dev_priv->drm, "wrong P1 divisor\n"); + return 0; + } + + for (i = 0; i < ARRAY_SIZE(lfsr_converts); i++) { + if (lfsr_converts[i] == pll_div) + break; + } + + if (i == ARRAY_SIZE(lfsr_converts)) { + drm_err(&dev_priv->drm, "wrong m_seed programmed\n"); + return 0; + } + + m = i + 62; + + dsi_clock = (m * refclk) / (p * n); + + return DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, bpp); +} + /* * XXX: The muxing and gating is hard coded for now. Need to add support for * sharing PLLs with two DSI outputs. @@ -122,8 +177,7 @@ int vlv_dsi_pll_compute(struct intel_encoder *encoder, { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); - int ret; - u32 dsi_clk; + int pclk, dsi_clk, ret; dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format, intel_dsi->lane_count); @@ -145,6 +199,14 @@ int vlv_dsi_pll_compute(struct intel_encoder *encoder, drm_dbg_kms(&dev_priv->drm, "dsi pll div %08x, ctrl %08x\n", config->dsi_pll.div, config->dsi_pll.ctrl); + pclk = vlv_dsi_pclk(encoder, config); + config->port_clock = pclk; + + /* FIXME definitely not right for burst/cmd mode/pixel overlap */ + config->hw.adjusted_mode.crtc_clock = pclk; + if (intel_dsi->dual_link) + config->hw.adjusted_mode.crtc_clock *= 2; + return 0; } @@ -262,13 +324,7 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); - int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); - u32 dsi_clock, pclk; u32 pll_ctl, pll_div; - u32 m = 0, p = 0, n; - int refclk = IS_CHERRYVIEW(dev_priv) ? 100000 : 25000; - int i; drm_dbg_kms(&dev_priv->drm, "\n"); @@ -280,65 +336,31 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, config->dsi_pll.ctrl = pll_ctl & ~DSI_PLL_LOCK; config->dsi_pll.div = pll_div; - /* mask out other bits and extract the P1 divisor */ - pll_ctl &= DSI_PLL_P1_POST_DIV_MASK; - pll_ctl = pll_ctl >> (DSI_PLL_P1_POST_DIV_SHIFT - 2); - - /* N1 divisor */ - n = (pll_div & DSI_PLL_N1_DIV_MASK) >> DSI_PLL_N1_DIV_SHIFT; - n = 1 << n; /* register has log2(N1) */ - - /* mask out the other bits and extract the M1 divisor */ - pll_div &= DSI_PLL_M1_DIV_MASK; - pll_div = pll_div >> DSI_PLL_M1_DIV_SHIFT; - - while (pll_ctl) { - pll_ctl = pll_ctl >> 1; - p++; - } - p--; - - if (!p) { - drm_err(&dev_priv->drm, "wrong P1 divisor\n"); - return 0; - } - - for (i = 0; i < ARRAY_SIZE(lfsr_converts); i++) { - if (lfsr_converts[i] == pll_div) - break; - } - - if (i == ARRAY_SIZE(lfsr_converts)) { - drm_err(&dev_priv->drm, "wrong m_seed programmed\n"); - return 0; - } - - m = i + 62; + return vlv_dsi_pclk(encoder, config); +} - dsi_clock = (m * refclk) / (p * n); +static int bxt_dsi_pclk(struct intel_encoder *encoder, + const struct intel_crtc_state *config) +{ + struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); + int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); + u32 dsi_ratio, dsi_clk; - pclk = DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, bpp); + dsi_ratio = config->dsi_pll.ctrl & BXT_DSI_PLL_RATIO_MASK; + dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2; - return pclk; + return DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, bpp); } u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, struct intel_crtc_state *config) { - u32 pclk; - u32 dsi_clk; - u32 dsi_ratio; - struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); + u32 pclk; config->dsi_pll.ctrl = intel_de_read(dev_priv, BXT_DSI_PLL_CTL); - dsi_ratio = config->dsi_pll.ctrl & BXT_DSI_PLL_RATIO_MASK; - - dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2; - - pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, bpp); + pclk = bxt_dsi_pclk(encoder, config); drm_dbg(&dev_priv->drm, "Calculated pclk=%u\n", pclk); return pclk; @@ -463,6 +485,7 @@ int bxt_dsi_pll_compute(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); u8 dsi_ratio, dsi_ratio_min, dsi_ratio_max; u32 dsi_clk; + int pclk; dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format, intel_dsi->lane_count); @@ -502,6 +525,14 @@ int bxt_dsi_pll_compute(struct intel_encoder *encoder, if (IS_BROXTON(dev_priv) && dsi_ratio <= 50) config->dsi_pll.ctrl |= BXT_DSI_PLL_PVD_RATIO_1; + pclk = bxt_dsi_pclk(encoder, config); + config->port_clock = pclk; + + /* FIXME definitely not right for burst/cmd mode/pixel overlap */ + config->hw.adjusted_mode.crtc_clock = pclk; + if (intel_dsi->dual_link) + config->hw.adjusted_mode.crtc_clock *= 2; + return 0; } diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h index 356e515153463307550d767b875251032b3f49d3..e065b8f2ee08417546b9eec19469279bfbaf199b 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h +++ b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h @@ -11,6 +11,8 @@ #define VLV_MIPI_BASE VLV_DISPLAY_BASE #define BXT_MIPI_BASE 0x60000 +#define _MIPI_MMIO_BASE(__i915) ((__i915)->display.dsi.mmio_base) + #define _MIPI_PORT(port, a, c) (((port) == PORT_A) ? a : c) /* ports A and C only */ #define _MMIO_MIPI(port, a, c) _MMIO(_MIPI_PORT(port, a, c)) @@ -96,8 +98,8 @@ /* MIPI DSI Controller and D-PHY registers */ -#define _MIPIA_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb000) -#define _MIPIC_DEVICE_READY (dev_priv->mipi_mmio_base + 0xb800) +#define _MIPIA_DEVICE_READY (_MIPI_MMIO_BASE(dev_priv) + 0xb000) +#define _MIPIC_DEVICE_READY (_MIPI_MMIO_BASE(dev_priv) + 0xb800) #define MIPI_DEVICE_READY(port) _MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY) #define BUS_POSSESSION (1 << 3) /* set to give bus to receiver */ #define ULPS_STATE_MASK (3 << 1) @@ -106,11 +108,11 @@ #define ULPS_STATE_NORMAL_OPERATION (0 << 1) #define DEVICE_READY (1 << 0) -#define _MIPIA_INTR_STAT (dev_priv->mipi_mmio_base + 0xb004) -#define _MIPIC_INTR_STAT (dev_priv->mipi_mmio_base + 0xb804) +#define _MIPIA_INTR_STAT (_MIPI_MMIO_BASE(dev_priv) + 0xb004) +#define _MIPIC_INTR_STAT (_MIPI_MMIO_BASE(dev_priv) + 0xb804) #define MIPI_INTR_STAT(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT) -#define _MIPIA_INTR_EN (dev_priv->mipi_mmio_base + 0xb008) -#define _MIPIC_INTR_EN (dev_priv->mipi_mmio_base + 0xb808) +#define _MIPIA_INTR_EN (_MIPI_MMIO_BASE(dev_priv) + 0xb008) +#define _MIPIC_INTR_EN (_MIPI_MMIO_BASE(dev_priv) + 0xb808) #define MIPI_INTR_EN(port) _MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN) #define TEARING_EFFECT (1 << 31) #define SPL_PKT_SENT_INTERRUPT (1 << 30) @@ -145,8 +147,8 @@ #define RXSOT_SYNC_ERROR (1 << 1) #define RXSOT_ERROR (1 << 0) -#define _MIPIA_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb00c) -#define _MIPIC_DSI_FUNC_PRG (dev_priv->mipi_mmio_base + 0xb80c) +#define _MIPIA_DSI_FUNC_PRG (_MIPI_MMIO_BASE(dev_priv) + 0xb00c) +#define _MIPIC_DSI_FUNC_PRG (_MIPI_MMIO_BASE(dev_priv) + 0xb80c) #define MIPI_DSI_FUNC_PRG(port) _MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG) #define CMD_MODE_DATA_WIDTH_MASK (7 << 13) #define CMD_MODE_NOT_SUPPORTED (0 << 13) @@ -168,76 +170,76 @@ #define DATA_LANES_PRG_REG_SHIFT 0 #define DATA_LANES_PRG_REG_MASK (7 << 0) -#define _MIPIA_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb010) -#define _MIPIC_HS_TX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb810) +#define _MIPIA_HS_TX_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb010) +#define _MIPIC_HS_TX_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb810) #define MIPI_HS_TX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT) #define HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK 0xffffff -#define _MIPIA_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb014) -#define _MIPIC_LP_RX_TIMEOUT (dev_priv->mipi_mmio_base + 0xb814) +#define _MIPIA_LP_RX_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb014) +#define _MIPIC_LP_RX_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb814) #define MIPI_LP_RX_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT) #define LOW_POWER_RX_TIMEOUT_COUNTER_MASK 0xffffff -#define _MIPIA_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb018) -#define _MIPIC_TURN_AROUND_TIMEOUT (dev_priv->mipi_mmio_base + 0xb818) +#define _MIPIA_TURN_AROUND_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb018) +#define _MIPIC_TURN_AROUND_TIMEOUT (_MIPI_MMIO_BASE(dev_priv) + 0xb818) #define MIPI_TURN_AROUND_TIMEOUT(port) _MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT) #define TURN_AROUND_TIMEOUT_MASK 0x3f -#define _MIPIA_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb01c) -#define _MIPIC_DEVICE_RESET_TIMER (dev_priv->mipi_mmio_base + 0xb81c) +#define _MIPIA_DEVICE_RESET_TIMER (_MIPI_MMIO_BASE(dev_priv) + 0xb01c) +#define _MIPIC_DEVICE_RESET_TIMER (_MIPI_MMIO_BASE(dev_priv) + 0xb81c) #define MIPI_DEVICE_RESET_TIMER(port) _MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER) #define DEVICE_RESET_TIMER_MASK 0xffff -#define _MIPIA_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb020) -#define _MIPIC_DPI_RESOLUTION (dev_priv->mipi_mmio_base + 0xb820) +#define _MIPIA_DPI_RESOLUTION (_MIPI_MMIO_BASE(dev_priv) + 0xb020) +#define _MIPIC_DPI_RESOLUTION (_MIPI_MMIO_BASE(dev_priv) + 0xb820) #define MIPI_DPI_RESOLUTION(port) _MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION) #define VERTICAL_ADDRESS_SHIFT 16 #define VERTICAL_ADDRESS_MASK (0xffff << 16) #define HORIZONTAL_ADDRESS_SHIFT 0 #define HORIZONTAL_ADDRESS_MASK 0xffff -#define _MIPIA_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb024) -#define _MIPIC_DBI_FIFO_THROTTLE (dev_priv->mipi_mmio_base + 0xb824) +#define _MIPIA_DBI_FIFO_THROTTLE (_MIPI_MMIO_BASE(dev_priv) + 0xb024) +#define _MIPIC_DBI_FIFO_THROTTLE (_MIPI_MMIO_BASE(dev_priv) + 0xb824) #define MIPI_DBI_FIFO_THROTTLE(port) _MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE) #define DBI_FIFO_EMPTY_HALF (0 << 0) #define DBI_FIFO_EMPTY_QUARTER (1 << 0) #define DBI_FIFO_EMPTY_7_LOCATIONS (2 << 0) /* regs below are bits 15:0 */ -#define _MIPIA_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb028) -#define _MIPIC_HSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb828) +#define _MIPIA_HSYNC_PADDING_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb028) +#define _MIPIC_HSYNC_PADDING_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb828) #define MIPI_HSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT) -#define _MIPIA_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb02c) -#define _MIPIC_HBP_COUNT (dev_priv->mipi_mmio_base + 0xb82c) +#define _MIPIA_HBP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb02c) +#define _MIPIC_HBP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb82c) #define MIPI_HBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT) -#define _MIPIA_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb030) -#define _MIPIC_HFP_COUNT (dev_priv->mipi_mmio_base + 0xb830) +#define _MIPIA_HFP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb030) +#define _MIPIC_HFP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb830) #define MIPI_HFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT) -#define _MIPIA_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb034) -#define _MIPIC_HACTIVE_AREA_COUNT (dev_priv->mipi_mmio_base + 0xb834) +#define _MIPIA_HACTIVE_AREA_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb034) +#define _MIPIC_HACTIVE_AREA_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb834) #define MIPI_HACTIVE_AREA_COUNT(port) _MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT) -#define _MIPIA_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb038) -#define _MIPIC_VSYNC_PADDING_COUNT (dev_priv->mipi_mmio_base + 0xb838) +#define _MIPIA_VSYNC_PADDING_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb038) +#define _MIPIC_VSYNC_PADDING_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb838) #define MIPI_VSYNC_PADDING_COUNT(port) _MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT) -#define _MIPIA_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb03c) -#define _MIPIC_VBP_COUNT (dev_priv->mipi_mmio_base + 0xb83c) +#define _MIPIA_VBP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb03c) +#define _MIPIC_VBP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb83c) #define MIPI_VBP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT) -#define _MIPIA_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb040) -#define _MIPIC_VFP_COUNT (dev_priv->mipi_mmio_base + 0xb840) +#define _MIPIA_VFP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb040) +#define _MIPIC_VFP_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb840) #define MIPI_VFP_COUNT(port) _MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT) -#define _MIPIA_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb044) -#define _MIPIC_HIGH_LOW_SWITCH_COUNT (dev_priv->mipi_mmio_base + 0xb844) +#define _MIPIA_HIGH_LOW_SWITCH_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb044) +#define _MIPIC_HIGH_LOW_SWITCH_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb844) #define MIPI_HIGH_LOW_SWITCH_COUNT(port) _MMIO_MIPI(port, _MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT) -#define _MIPIA_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb048) -#define _MIPIC_DPI_CONTROL (dev_priv->mipi_mmio_base + 0xb848) +#define _MIPIA_DPI_CONTROL (_MIPI_MMIO_BASE(dev_priv) + 0xb048) +#define _MIPIC_DPI_CONTROL (_MIPI_MMIO_BASE(dev_priv) + 0xb848) #define MIPI_DPI_CONTROL(port) _MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL) #define DPI_LP_MODE (1 << 6) #define BACKLIGHT_OFF (1 << 5) @@ -247,27 +249,27 @@ #define TURN_ON (1 << 1) #define SHUTDOWN (1 << 0) -#define _MIPIA_DPI_DATA (dev_priv->mipi_mmio_base + 0xb04c) -#define _MIPIC_DPI_DATA (dev_priv->mipi_mmio_base + 0xb84c) +#define _MIPIA_DPI_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb04c) +#define _MIPIC_DPI_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb84c) #define MIPI_DPI_DATA(port) _MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA) #define COMMAND_BYTE_SHIFT 0 #define COMMAND_BYTE_MASK (0x3f << 0) -#define _MIPIA_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb050) -#define _MIPIC_INIT_COUNT (dev_priv->mipi_mmio_base + 0xb850) +#define _MIPIA_INIT_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb050) +#define _MIPIC_INIT_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb850) #define MIPI_INIT_COUNT(port) _MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT) #define MASTER_INIT_TIMER_SHIFT 0 #define MASTER_INIT_TIMER_MASK (0xffff << 0) -#define _MIPIA_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb054) -#define _MIPIC_MAX_RETURN_PKT_SIZE (dev_priv->mipi_mmio_base + 0xb854) +#define _MIPIA_MAX_RETURN_PKT_SIZE (_MIPI_MMIO_BASE(dev_priv) + 0xb054) +#define _MIPIC_MAX_RETURN_PKT_SIZE (_MIPI_MMIO_BASE(dev_priv) + 0xb854) #define MIPI_MAX_RETURN_PKT_SIZE(port) _MMIO_MIPI(port, \ _MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE) #define MAX_RETURN_PKT_SIZE_SHIFT 0 #define MAX_RETURN_PKT_SIZE_MASK (0x3ff << 0) -#define _MIPIA_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb058) -#define _MIPIC_VIDEO_MODE_FORMAT (dev_priv->mipi_mmio_base + 0xb858) +#define _MIPIA_VIDEO_MODE_FORMAT (_MIPI_MMIO_BASE(dev_priv) + 0xb058) +#define _MIPIC_VIDEO_MODE_FORMAT (_MIPI_MMIO_BASE(dev_priv) + 0xb858) #define MIPI_VIDEO_MODE_FORMAT(port) _MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT) #define RANDOM_DPI_DISPLAY_RESOLUTION (1 << 4) #define DISABLE_VIDEO_BTA (1 << 3) @@ -276,8 +278,8 @@ #define VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS (2 << 0) #define VIDEO_MODE_BURST (3 << 0) -#define _MIPIA_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb05c) -#define _MIPIC_EOT_DISABLE (dev_priv->mipi_mmio_base + 0xb85c) +#define _MIPIA_EOT_DISABLE (_MIPI_MMIO_BASE(dev_priv) + 0xb05c) +#define _MIPIC_EOT_DISABLE (_MIPI_MMIO_BASE(dev_priv) + 0xb85c) #define MIPI_EOT_DISABLE(port) _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE) #define BXT_DEFEATURE_DPI_FIFO_CTR (1 << 9) #define BXT_DPHY_DEFEATURE_EN (1 << 8) @@ -290,35 +292,35 @@ #define CLOCKSTOP (1 << 1) #define EOT_DISABLE (1 << 0) -#define _MIPIA_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb060) -#define _MIPIC_LP_BYTECLK (dev_priv->mipi_mmio_base + 0xb860) +#define _MIPIA_LP_BYTECLK (_MIPI_MMIO_BASE(dev_priv) + 0xb060) +#define _MIPIC_LP_BYTECLK (_MIPI_MMIO_BASE(dev_priv) + 0xb860) #define MIPI_LP_BYTECLK(port) _MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK) #define LP_BYTECLK_SHIFT 0 #define LP_BYTECLK_MASK (0xffff << 0) -#define _MIPIA_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb0a4) -#define _MIPIC_TLPX_TIME_COUNT (dev_priv->mipi_mmio_base + 0xb8a4) +#define _MIPIA_TLPX_TIME_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb0a4) +#define _MIPIC_TLPX_TIME_COUNT (_MIPI_MMIO_BASE(dev_priv) + 0xb8a4) #define MIPI_TLPX_TIME_COUNT(port) _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT) -#define _MIPIA_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb098) -#define _MIPIC_CLK_LANE_TIMING (dev_priv->mipi_mmio_base + 0xb898) +#define _MIPIA_CLK_LANE_TIMING (_MIPI_MMIO_BASE(dev_priv) + 0xb098) +#define _MIPIC_CLK_LANE_TIMING (_MIPI_MMIO_BASE(dev_priv) + 0xb898) #define MIPI_CLK_LANE_TIMING(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING) /* bits 31:0 */ -#define _MIPIA_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb064) -#define _MIPIC_LP_GEN_DATA (dev_priv->mipi_mmio_base + 0xb864) +#define _MIPIA_LP_GEN_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb064) +#define _MIPIC_LP_GEN_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb864) #define MIPI_LP_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA) /* bits 31:0 */ -#define _MIPIA_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb068) -#define _MIPIC_HS_GEN_DATA (dev_priv->mipi_mmio_base + 0xb868) +#define _MIPIA_HS_GEN_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb068) +#define _MIPIC_HS_GEN_DATA (_MIPI_MMIO_BASE(dev_priv) + 0xb868) #define MIPI_HS_GEN_DATA(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA) -#define _MIPIA_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb06c) -#define _MIPIC_LP_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb86c) +#define _MIPIA_LP_GEN_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb06c) +#define _MIPIC_LP_GEN_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb86c) #define MIPI_LP_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL) -#define _MIPIA_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb070) -#define _MIPIC_HS_GEN_CTRL (dev_priv->mipi_mmio_base + 0xb870) +#define _MIPIA_HS_GEN_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb070) +#define _MIPIC_HS_GEN_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb870) #define MIPI_HS_GEN_CTRL(port) _MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL) #define LONG_PACKET_WORD_COUNT_SHIFT 8 #define LONG_PACKET_WORD_COUNT_MASK (0xffff << 8) @@ -330,8 +332,8 @@ #define DATA_TYPE_MASK (0x3f << 0) /* data type values, see include/video/mipi_display.h */ -#define _MIPIA_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb074) -#define _MIPIC_GEN_FIFO_STAT (dev_priv->mipi_mmio_base + 0xb874) +#define _MIPIA_GEN_FIFO_STAT (_MIPI_MMIO_BASE(dev_priv) + 0xb074) +#define _MIPIC_GEN_FIFO_STAT (_MIPI_MMIO_BASE(dev_priv) + 0xb874) #define MIPI_GEN_FIFO_STAT(port) _MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT) #define DPI_FIFO_EMPTY (1 << 28) #define DBI_FIFO_EMPTY (1 << 27) @@ -348,15 +350,15 @@ #define HS_DATA_FIFO_HALF_EMPTY (1 << 1) #define HS_DATA_FIFO_FULL (1 << 0) -#define _MIPIA_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb078) -#define _MIPIC_HS_LS_DBI_ENABLE (dev_priv->mipi_mmio_base + 0xb878) +#define _MIPIA_HS_LS_DBI_ENABLE (_MIPI_MMIO_BASE(dev_priv) + 0xb078) +#define _MIPIC_HS_LS_DBI_ENABLE (_MIPI_MMIO_BASE(dev_priv) + 0xb878) #define MIPI_HS_LP_DBI_ENABLE(port) _MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE) #define DBI_HS_LP_MODE_MASK (1 << 0) #define DBI_LP_MODE (1 << 0) #define DBI_HS_MODE (0 << 0) -#define _MIPIA_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb080) -#define _MIPIC_DPHY_PARAM (dev_priv->mipi_mmio_base + 0xb880) +#define _MIPIA_DPHY_PARAM (_MIPI_MMIO_BASE(dev_priv) + 0xb080) +#define _MIPIC_DPHY_PARAM (_MIPI_MMIO_BASE(dev_priv) + 0xb880) #define MIPI_DPHY_PARAM(port) _MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM) #define EXIT_ZERO_COUNT_SHIFT 24 #define EXIT_ZERO_COUNT_MASK (0x3f << 24) @@ -367,34 +369,34 @@ #define PREPARE_COUNT_SHIFT 0 #define PREPARE_COUNT_MASK (0x3f << 0) -#define _MIPIA_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb084) -#define _MIPIC_DBI_BW_CTRL (dev_priv->mipi_mmio_base + 0xb884) +#define _MIPIA_DBI_BW_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb084) +#define _MIPIC_DBI_BW_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb884) #define MIPI_DBI_BW_CTRL(port) _MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL) -#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb088) -#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (dev_priv->mipi_mmio_base + 0xb888) +#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT (_MIPI_MMIO_BASE(dev_priv) + 0xb088) +#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT (_MIPI_MMIO_BASE(dev_priv) + 0xb888) #define MIPI_CLK_LANE_SWITCH_TIME_CNT(port) _MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT) #define LP_HS_SSW_CNT_SHIFT 16 #define LP_HS_SSW_CNT_MASK (0xffff << 16) #define HS_LP_PWR_SW_CNT_SHIFT 0 #define HS_LP_PWR_SW_CNT_MASK (0xffff << 0) -#define _MIPIA_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb08c) -#define _MIPIC_STOP_STATE_STALL (dev_priv->mipi_mmio_base + 0xb88c) +#define _MIPIA_STOP_STATE_STALL (_MIPI_MMIO_BASE(dev_priv) + 0xb08c) +#define _MIPIC_STOP_STATE_STALL (_MIPI_MMIO_BASE(dev_priv) + 0xb88c) #define MIPI_STOP_STATE_STALL(port) _MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL) #define STOP_STATE_STALL_COUNTER_SHIFT 0 #define STOP_STATE_STALL_COUNTER_MASK (0xff << 0) -#define _MIPIA_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb090) -#define _MIPIC_INTR_STAT_REG_1 (dev_priv->mipi_mmio_base + 0xb890) +#define _MIPIA_INTR_STAT_REG_1 (_MIPI_MMIO_BASE(dev_priv) + 0xb090) +#define _MIPIC_INTR_STAT_REG_1 (_MIPI_MMIO_BASE(dev_priv) + 0xb890) #define MIPI_INTR_STAT_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1) -#define _MIPIA_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb094) -#define _MIPIC_INTR_EN_REG_1 (dev_priv->mipi_mmio_base + 0xb894) +#define _MIPIA_INTR_EN_REG_1 (_MIPI_MMIO_BASE(dev_priv) + 0xb094) +#define _MIPIC_INTR_EN_REG_1 (_MIPI_MMIO_BASE(dev_priv) + 0xb894) #define MIPI_INTR_EN_REG_1(port) _MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1) #define RX_CONTENTION_DETECTED (1 << 0) /* XXX: only pipe A ?!? */ -#define MIPIA_DBI_TYPEC_CTRL (dev_priv->mipi_mmio_base + 0xb100) +#define MIPIA_DBI_TYPEC_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb100) #define DBI_TYPEC_ENABLE (1 << 31) #define DBI_TYPEC_WIP (1 << 30) #define DBI_TYPEC_OPTION_SHIFT 28 @@ -407,8 +409,8 @@ /* MIPI adapter registers */ -#define _MIPIA_CTRL (dev_priv->mipi_mmio_base + 0xb104) -#define _MIPIC_CTRL (dev_priv->mipi_mmio_base + 0xb904) +#define _MIPIA_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb104) +#define _MIPIC_CTRL (_MIPI_MMIO_BASE(dev_priv) + 0xb904) #define MIPI_CTRL(port) _MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL) #define ESCAPE_CLOCK_DIVIDER_SHIFT 5 /* A only */ #define ESCAPE_CLOCK_DIVIDER_MASK (3 << 5) @@ -440,21 +442,21 @@ #define GLK_MIPIIO_PORT_POWERED (1 << 1) /* RO */ #define GLK_MIPIIO_ENABLE (1 << 0) -#define _MIPIA_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb108) -#define _MIPIC_DATA_ADDRESS (dev_priv->mipi_mmio_base + 0xb908) +#define _MIPIA_DATA_ADDRESS (_MIPI_MMIO_BASE(dev_priv) + 0xb108) +#define _MIPIC_DATA_ADDRESS (_MIPI_MMIO_BASE(dev_priv) + 0xb908) #define MIPI_DATA_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS) #define DATA_MEM_ADDRESS_SHIFT 5 #define DATA_MEM_ADDRESS_MASK (0x7ffffff << 5) #define DATA_VALID (1 << 0) -#define _MIPIA_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb10c) -#define _MIPIC_DATA_LENGTH (dev_priv->mipi_mmio_base + 0xb90c) +#define _MIPIA_DATA_LENGTH (_MIPI_MMIO_BASE(dev_priv) + 0xb10c) +#define _MIPIC_DATA_LENGTH (_MIPI_MMIO_BASE(dev_priv) + 0xb90c) #define MIPI_DATA_LENGTH(port) _MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH) #define DATA_LENGTH_SHIFT 0 #define DATA_LENGTH_MASK (0xfffff << 0) -#define _MIPIA_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb110) -#define _MIPIC_COMMAND_ADDRESS (dev_priv->mipi_mmio_base + 0xb910) +#define _MIPIA_COMMAND_ADDRESS (_MIPI_MMIO_BASE(dev_priv) + 0xb110) +#define _MIPIC_COMMAND_ADDRESS (_MIPI_MMIO_BASE(dev_priv) + 0xb910) #define MIPI_COMMAND_ADDRESS(port) _MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS) #define COMMAND_MEM_ADDRESS_SHIFT 5 #define COMMAND_MEM_ADDRESS_MASK (0x7ffffff << 5) @@ -462,18 +464,18 @@ #define MEMORY_WRITE_DATA_FROM_PIPE_RENDERING (1 << 1) #define COMMAND_VALID (1 << 0) -#define _MIPIA_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb114) -#define _MIPIC_COMMAND_LENGTH (dev_priv->mipi_mmio_base + 0xb914) +#define _MIPIA_COMMAND_LENGTH (_MIPI_MMIO_BASE(dev_priv) + 0xb114) +#define _MIPIC_COMMAND_LENGTH (_MIPI_MMIO_BASE(dev_priv) + 0xb914) #define MIPI_COMMAND_LENGTH(port) _MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH) #define COMMAND_LENGTH_SHIFT(n) (8 * (n)) /* n: 0...3 */ #define COMMAND_LENGTH_MASK(n) (0xff << (8 * (n))) -#define _MIPIA_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb118) -#define _MIPIC_READ_DATA_RETURN0 (dev_priv->mipi_mmio_base + 0xb918) +#define _MIPIA_READ_DATA_RETURN0 (_MIPI_MMIO_BASE(dev_priv) + 0xb118) +#define _MIPIC_READ_DATA_RETURN0 (_MIPI_MMIO_BASE(dev_priv) + 0xb918) #define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */ -#define _MIPIA_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb138) -#define _MIPIC_READ_DATA_VALID (dev_priv->mipi_mmio_base + 0xb938) +#define _MIPIA_READ_DATA_VALID (_MIPI_MMIO_BASE(dev_priv) + 0xb138) +#define _MIPIC_READ_DATA_VALID (_MIPI_MMIO_BASE(dev_priv) + 0xb938) #define MIPI_READ_DATA_VALID(port) _MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID) #define READ_DATA_VALID(n) (1 << (n)) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index dabdfe09f5e51438c7d9b203194316f48b71b1d9..1e29b1e6d186875b2c132880f2a806e1484dc425 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -1269,6 +1269,10 @@ static void i915_gem_context_release_work(struct work_struct *work) trace_i915_context_free(ctx); GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); + spin_lock(&ctx->i915->gem.contexts.lock); + list_del(&ctx->link); + spin_unlock(&ctx->i915->gem.contexts.lock); + if (ctx->syncobj) drm_syncobj_put(ctx->syncobj); @@ -1383,14 +1387,8 @@ kill_engines(struct i915_gem_engines *engines, bool exit, bool persistent) */ for_each_gem_engine(ce, engines, it) { struct intel_engine_cs *engine; - bool skip = false; - - if (exit) - skip = intel_context_set_exiting(ce); - else if (!persistent) - skip = intel_context_exit_nonpersistent(ce, NULL); - if (skip) + if ((exit || !persistent) && intel_context_revoke(ce)) continue; /* Already marked. */ /* @@ -1521,10 +1519,6 @@ static void context_close(struct i915_gem_context *ctx) ctx->file_priv = ERR_PTR(-EBADF); - spin_lock(&ctx->i915->gem.contexts.lock); - list_del(&ctx->link); - spin_unlock(&ctx->i915->gem.contexts.lock); - client = ctx->client; if (client) { spin_lock(&client->ctx_lock); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index cd75b0ca2555f49dde7b8a80d14e9bd97361700f..845023c14eb36f1b525d943fe632e58da95f82d8 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -2424,7 +2424,7 @@ gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv, /* Check whether the file_priv has already selected one ring. */ if ((int)file_priv->bsd_engine < 0) file_priv->bsd_engine = - get_random_int() % num_vcs_engines(dev_priv); + prandom_u32_max(num_vcs_engines(dev_priv)); return file_priv->bsd_engine; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h index 1b88ea13435c02939c42dcbabfb7507397be750d..5a7a14e85c3fc78bcabb67838d054710998e5bda 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h @@ -12,8 +12,6 @@ struct drm_i915_private; struct drm_i915_gem_object; struct intel_memory_region; -extern const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops; - void __iomem * i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj, unsigned long n, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index 3218981488cc476569b0f562efef6cf121621caf..73d9eda1d6b7a6fdd237da6bff783e67495648cc 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -413,7 +413,7 @@ retry: vma->mmo = mmo; if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) - intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, + intel_wakeref_auto(&to_gt(i915)->userfault_wakeref, msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); if (write) { @@ -550,6 +550,20 @@ out: intel_runtime_pm_put(&i915->runtime_pm, wakeref); } +void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj) +{ + struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); + struct ttm_device *bdev = bo->bdev; + + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + + if (obj->userfault_count) { + /* rpm wakeref provide exclusive access */ + list_del(&obj->userfault_link); + obj->userfault_count = 0; + } +} + void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) { struct i915_mmap_offset *mmo, *mn; @@ -573,6 +587,13 @@ void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) spin_lock(&obj->mmo.lock); } spin_unlock(&obj->mmo.lock); + + if (obj->userfault_count) { + mutex_lock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock); + list_del(&obj->userfault_link); + mutex_unlock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock); + obj->userfault_count = 0; + } } static struct i915_mmap_offset * diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.h b/drivers/gpu/drm/i915/gem/i915_gem_mman.h index efee9e0d2508678f435e2ac36c75a320cb3fdf63..1fa91b3033b35aa44abc3665516e635b2506658b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.h @@ -27,6 +27,7 @@ int i915_gem_dumb_mmap_offset(struct drm_file *file_priv, void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj); void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj); +void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj); void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj); #endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 7cab89618badafa9e48e13bbe4d9bbc7ffc97435..6b8710ba8ded80ddd425ca955988def5f23faedd 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -238,7 +238,7 @@ static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj) { /* Skip serialisation and waking the device if known to be not used. */ - if (obj->userfault_count) + if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev))) i915_gem_object_release_mmap_gtt(obj); if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) { @@ -666,6 +666,41 @@ bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, int i915_gem_object_migrate(struct drm_i915_gem_object *obj, struct i915_gem_ww_ctx *ww, enum intel_region_id id) +{ + return __i915_gem_object_migrate(obj, ww, id, obj->flags); +} + +/** + * __i915_gem_object_migrate - Migrate an object to the desired region id, with + * control of the extra flags + * @obj: The object to migrate. + * @ww: An optional struct i915_gem_ww_ctx. If NULL, the backend may + * not be successful in evicting other objects to make room for this object. + * @id: The region id to migrate to. + * @flags: The object flags. Normally just obj->flags. + * + * Attempt to migrate the object to the desired memory region. The + * object backend must support migration and the object may not be + * pinned, (explicitly pinned pages or pinned vmas). The object must + * be locked. + * On successful completion, the object will have pages pointing to + * memory in the new region, but an async migration task may not have + * completed yet, and to accomplish that, i915_gem_object_wait_migration() + * must be called. + * + * Note: the @ww parameter is not used yet, but included to make sure + * callers put some effort into obtaining a valid ww ctx if one is + * available. + * + * Return: 0 on success. Negative error code on failure. In particular may + * return -ENXIO on lack of region space, -EDEADLK for deadlock avoidance + * if @ww is set, -EINTR or -ERESTARTSYS if signal pending, and + * -EBUSY if the object is pinned. + */ +int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + enum intel_region_id id, + unsigned int flags) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct intel_memory_region *mr; @@ -686,7 +721,7 @@ int i915_gem_object_migrate(struct drm_i915_gem_object *obj, return 0; } - return obj->ops->migrate(obj, mr); + return obj->ops->migrate(obj, mr, flags); } /** diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 7317d4102955f5eb7081e805ec33cbd71e414e71..1723af9b0f6a223c611e2efd2218af105216cc80 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -608,6 +608,10 @@ bool i915_gem_object_migratable(struct drm_i915_gem_object *obj); int i915_gem_object_migrate(struct drm_i915_gem_object *obj, struct i915_gem_ww_ctx *ww, enum intel_region_id id); +int __i915_gem_object_migrate(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + enum intel_region_id id, + unsigned int flags); bool i915_gem_object_can_migrate(struct drm_i915_gem_object *obj, enum intel_region_id id); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index 9f6b14ec189a2e33d62be44958a494724ae7b189..d0d6772e6f36a2e9002b035f91a772a4de773af9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -107,7 +107,8 @@ struct drm_i915_gem_object_ops { * pinning or for as long as the object lock is held. */ int (*migrate)(struct drm_i915_gem_object *obj, - struct intel_memory_region *mr); + struct intel_memory_region *mr, + unsigned int flags); void (*release)(struct drm_i915_gem_object *obj); @@ -298,7 +299,8 @@ struct drm_i915_gem_object { }; /** - * Whether the object is currently in the GGTT mmap. + * Whether the object is currently in the GGTT or any other supported + * fake offset mmap backed by lmem. */ unsigned int userfault_count; struct list_head userfault_link; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 8357dbdcab5cb0e406e2bba97a8f7f528fb85ba1..4df50b049ceaf1c30c4ac958811e713c5ae7f4df 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -20,7 +20,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, unsigned int sg_page_sizes) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - unsigned long supported = INTEL_INFO(i915)->page_sizes; + unsigned long supported = RUNTIME_INFO(i915)->page_sizes; bool shrinkable; int i; @@ -66,7 +66,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, shrinkable = i915_gem_object_is_shrinkable(obj); if (i915_gem_object_is_tiled(obj) && - i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { + i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) { GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_set_tiling_quirk(obj); GEM_BUG_ON(!list_empty(&obj->mm.link)); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c index 00359ec9d58b0535c274e1ab139ea1f023a50e31..3428f735e786c01d9a0dfc214312678dd7fea229 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c @@ -24,7 +24,7 @@ void i915_gem_suspend(struct drm_i915_private *i915) { GEM_TRACE("%s\n", dev_name(i915->drm.dev)); - intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, 0); + intel_wakeref_auto(&to_gt(i915)->userfault_wakeref, 0); flush_workqueue(i915->wq); /* diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 166d0a4b9e8c0c394c386960d731eac182795cdb..acc561c0f0aa2a2a72f98502cfe3ab2f708e4b74 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -18,10 +18,12 @@ #include "gt/intel_region_lmem.h" #include "i915_drv.h" #include "i915_gem_stolen.h" +#include "i915_pci.h" #include "i915_reg.h" #include "i915_utils.h" #include "i915_vgpu.h" #include "intel_mchbar_regs.h" +#include "intel_pci_config.h" /* * The BIOS typically reserves some of the system's memory for the exclusive @@ -428,48 +430,29 @@ static int i915_gem_init_stolen(struct intel_memory_region *mem) reserved_base = stolen_top; reserved_size = 0; - switch (GRAPHICS_VER(i915)) { - case 2: - case 3: - break; - case 4: - if (!IS_G4X(i915)) - break; - fallthrough; - case 5: - g4x_get_stolen_reserved(i915, uncore, + if (GRAPHICS_VER(i915) >= 11) { + icl_get_stolen_reserved(i915, uncore, &reserved_base, &reserved_size); - break; - case 6: - gen6_get_stolen_reserved(i915, uncore, - &reserved_base, &reserved_size); - break; - case 7: - if (IS_VALLEYVIEW(i915)) - vlv_get_stolen_reserved(i915, uncore, - &reserved_base, &reserved_size); - else - gen7_get_stolen_reserved(i915, uncore, - &reserved_base, &reserved_size); - break; - case 8: - case 9: + } else if (GRAPHICS_VER(i915) >= 8) { if (IS_LP(i915)) chv_get_stolen_reserved(i915, uncore, &reserved_base, &reserved_size); else bdw_get_stolen_reserved(i915, uncore, &reserved_base, &reserved_size); - break; - default: - MISSING_CASE(GRAPHICS_VER(i915)); - fallthrough; - case 11: - case 12: - icl_get_stolen_reserved(i915, uncore, - &reserved_base, - &reserved_size); - break; + } else if (GRAPHICS_VER(i915) >= 7) { + if (IS_VALLEYVIEW(i915)) + vlv_get_stolen_reserved(i915, uncore, + &reserved_base, &reserved_size); + else + gen7_get_stolen_reserved(i915, uncore, + &reserved_base, &reserved_size); + } else if (GRAPHICS_VER(i915) >= 6) { + gen6_get_stolen_reserved(i915, uncore, + &reserved_base, &reserved_size); + } else if (GRAPHICS_VER(i915) >= 5 || IS_G4X(i915)) { + g4x_get_stolen_reserved(i915, uncore, + &reserved_base, &reserved_size); } /* @@ -827,10 +810,13 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type, if (WARN_ON_ONCE(instance)) return ERR_PTR(-ENODEV); + if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR)) + return ERR_PTR(-ENXIO); + /* Use DSM base address instead for stolen memory */ dsm_base = intel_uncore_read64(uncore, GEN12_DSMBASE); if (IS_DG1(uncore->i915)) { - lmem_size = pci_resource_len(pdev, 2); + lmem_size = pci_resource_len(pdev, GEN12_LMEM_BAR); if (WARN_ON(lmem_size < dsm_base)) return ERR_PTR(-ENODEV); } else { @@ -842,11 +828,11 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type, } dsm_size = lmem_size - dsm_base; - if (pci_resource_len(pdev, 2) < lmem_size) { + if (pci_resource_len(pdev, GEN12_LMEM_BAR) < lmem_size) { io_start = 0; io_size = 0; } else { - io_start = pci_resource_start(pdev, 2) + dsm_base; + io_start = pci_resource_start(pdev, GEN12_LMEM_BAR) + dsm_base; io_size = dsm_size; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index 85518b28cd721bd4086c616444f3e5ba32b3397f..fd42b89b7162b7e6e8210266ced13ba592da4920 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -278,7 +278,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, */ if (i915_gem_object_has_pages(obj) && obj->mm.madv == I915_MADV_WILLNEED && - i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { + i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) { if (tiling == I915_TILING_NONE) { GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_clear_tiling_quirk(obj); @@ -458,7 +458,7 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, } /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ - if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) + if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN; else args->phys_swizzle_mode = args->swizzle_mode; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index f64a3deb12fce16ae8a4404ba454299c45316cd3..4f861782c3e85a16ce4e56b07908624ea9f91c4f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -509,9 +509,18 @@ static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags) static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo) { struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo); + intel_wakeref_t wakeref = 0; + + if (bo->resource && likely(obj)) { + /* ttm_bo_release() already has dma_resv_lock */ + if (i915_ttm_cpu_maps_iomem(bo->resource)) + wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); - if (likely(obj)) { __i915_gem_object_pages_fini(obj); + + if (wakeref) + intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); + i915_ttm_free_cached_io_rsgt(obj); } } @@ -839,9 +848,10 @@ static int __i915_ttm_migrate(struct drm_i915_gem_object *obj, } static int i915_ttm_migrate(struct drm_i915_gem_object *obj, - struct intel_memory_region *mr) + struct intel_memory_region *mr, + unsigned int flags) { - return __i915_ttm_migrate(obj, mr, obj->flags); + return __i915_ttm_migrate(obj, mr, flags); } static void i915_ttm_put_pages(struct drm_i915_gem_object *obj, @@ -981,6 +991,7 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) struct ttm_buffer_object *bo = area->vm_private_data; struct drm_device *dev = bo->base.dev; struct drm_i915_gem_object *obj; + intel_wakeref_t wakeref = 0; vm_fault_t ret; int idx; @@ -1002,6 +1013,9 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } + if (i915_ttm_cpu_maps_iomem(bo->resource)) + wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); + if (!i915_ttm_resource_mappable(bo->resource)) { int err = -ENODEV; int i; @@ -1023,7 +1037,8 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) if (err) { drm_dbg(dev, "Unable to make resource CPU accessible\n"); dma_resv_unlock(bo->base.resv); - return VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; + goto out_rpm; } } @@ -1034,12 +1049,30 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) } else { ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); } + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - return ret; + goto out_rpm; + + /* ttm_bo_vm_reserve() already has dma_resv_lock */ + if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) { + obj->userfault_count = 1; + mutex_lock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock); + list_add(&obj->userfault_link, &to_gt(to_i915(obj->base.dev))->lmem_userfault_list); + mutex_unlock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock); + } + + if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) + intel_wakeref_auto(&to_gt(to_i915(obj->base.dev))->userfault_wakeref, + msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); i915_ttm_adjust_lru(obj); dma_resv_unlock(bo->base.resv); + +out_rpm: + if (wakeref) + intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref); + return ret; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 8423df021b7138a02f77897e1b9cc98af743e35d..d4398948f01623d7474593eb4fb8de2690136301 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -426,12 +426,11 @@ static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { static int probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) { - const unsigned long end = addr + len; + VMA_ITERATOR(vmi, mm, addr); struct vm_area_struct *vma; - int ret = -EFAULT; mmap_read_lock(mm); - for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { + for_each_vma_range(vmi, vma, addr + len) { /* Check for holes, note that we also update the addr below */ if (vma->vm_start > addr) break; @@ -439,16 +438,13 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) break; - if (vma->vm_end >= end) { - ret = 0; - break; - } - addr = vma->vm_end; } mmap_read_unlock(mm); - return ret; + if (vma) + return -EFAULT; + return 0; } /* diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 72ce2c9f42fd1285aa57a22b722c1f033371676e..c570cf780079a0a9920b090050130f9b51c7f8f8 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -358,7 +358,7 @@ fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) static int igt_check_page_sizes(struct i915_vma *vma) { struct drm_i915_private *i915 = vma->vm->i915; - unsigned int supported = INTEL_INFO(i915)->page_sizes; + unsigned int supported = RUNTIME_INFO(i915)->page_sizes; struct drm_i915_gem_object *obj = vma->obj; int err; @@ -419,7 +419,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) { struct i915_ppgtt *ppgtt = arg; struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; + unsigned int saved_mask = RUNTIME_INFO(i915)->page_sizes; struct drm_i915_gem_object *obj; struct i915_vma *vma; int i, j, single; @@ -438,7 +438,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) combination |= page_sizes[j]; } - mkwrite_device_info(i915)->page_sizes = combination; + RUNTIME_INFO(i915)->page_sizes = combination; for (single = 0; single <= 1; ++single) { obj = fake_huge_pages_object(i915, combination, !!single); @@ -485,7 +485,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) out_put: i915_gem_object_put(obj); out_device: - mkwrite_device_info(i915)->page_sizes = saved_mask; + RUNTIME_INFO(i915)->page_sizes = saved_mask; return err; } @@ -495,7 +495,7 @@ static int igt_mock_memory_region_huge_pages(void *arg) const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS }; struct i915_ppgtt *ppgtt = arg; struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned long supported = INTEL_INFO(i915)->page_sizes; + unsigned long supported = RUNTIME_INFO(i915)->page_sizes; struct intel_memory_region *mem; struct drm_i915_gem_object *obj; struct i915_vma *vma; @@ -573,7 +573,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) { struct i915_ppgtt *ppgtt = arg; struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned long supported = INTEL_INFO(i915)->page_sizes; + unsigned long supported = RUNTIME_INFO(i915)->page_sizes; struct drm_i915_gem_object *obj; int bit; int err; @@ -1390,7 +1390,7 @@ out_put: static int igt_ppgtt_sanity_check(void *arg) { struct drm_i915_private *i915 = arg; - unsigned int supported = INTEL_INFO(i915)->page_sizes; + unsigned int supported = RUNTIME_INFO(i915)->page_sizes; struct { igt_create_fn fn; unsigned int flags; @@ -1764,8 +1764,8 @@ int i915_gem_huge_page_mock_selftests(void) return -ENOMEM; /* Pretend to be a device which supports the 48b PPGTT */ - mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; - mkwrite_device_info(dev_priv)->ppgtt_size = 48; + RUNTIME_INFO(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; + RUNTIME_INFO(dev_priv)->ppgtt_size = 48; ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); if (IS_ERR(ppgtt)) { diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 3cfc621ef363d2e097dc863b4941be578ff337e9..9a6a6b5b722b6015ea5b47c7ccc64ea6851047ef 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -711,7 +711,7 @@ static bool bad_swizzling(struct drm_i915_private *i915) { struct i915_ggtt *ggtt = to_gt(i915)->ggtt; - if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) + if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) return true; if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) || diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 55bf23dc0e542be0c48db8aec59a54cad41a3b84..b73c91aa5450d23677ae139a5bc029297de0f750 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -367,7 +367,7 @@ static int igt_partial_tiling(void *arg) unsigned int pitch; struct tile tile; - if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) + if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) /* * The swizzling pattern is actually unknown as it * varies based on physical address of each page. @@ -464,7 +464,7 @@ static int igt_smoke_tiling(void *arg) * Remember to look at the st_seed if we see a flip-flop in BAT! */ - if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) + if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) return 0; obj = huge_gem_object(i915, diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c index 1bb766c79dcbe7571d2ed6d54de0995943b173b2..5aaacc53fa4ca4b11d6729281f51d3b2b438512e 100644 --- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c @@ -247,6 +247,7 @@ err_scratch1: i915_gem_object_put(vm->scratch[1]); err_scratch0: i915_gem_object_put(vm->scratch[0]); + vm->scratch[0] = NULL; return ret; } @@ -268,9 +269,10 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm) gen6_ppgtt_free_pd(ppgtt); free_scratch(vm); - mutex_destroy(&ppgtt->flush); + if (ppgtt->base.pd) + free_pd(&ppgtt->base.vm, ppgtt->base.pd); - free_pd(&ppgtt->base.vm, ppgtt->base.pd); + mutex_destroy(&ppgtt->flush); } static void pd_vma_bind(struct i915_address_space *vm, @@ -449,19 +451,17 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt) err = gen6_ppgtt_init_scratch(ppgtt); if (err) - goto err_free; + goto err_put; ppgtt->base.pd = gen6_alloc_top_pd(ppgtt); if (IS_ERR(ppgtt->base.pd)) { err = PTR_ERR(ppgtt->base.pd); - goto err_scratch; + goto err_put; } return &ppgtt->base; -err_scratch: - free_scratch(&ppgtt->base.vm); -err_free: - kfree(ppgtt); +err_put: + i915_vm_put(&ppgtt->base.vm); return ERR_PTR(err); } diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c index 98645797962f59799a6863b2ef3e3307df78809d..e49fa6fa6aee1cc22f510c9d27a66f5f35618d6a 100644 --- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c @@ -165,10 +165,12 @@ static u32 preparser_disable(bool state) return MI_ARB_CHECK | 1 << 8 | state; } -u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg) +u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg) { + u32 gsi_offset = gt->uncore->gsi_offset; + *cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; - *cs++ = i915_mmio_reg_offset(inv_reg); + *cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset; *cs++ = AUX_INV; *cs++ = MI_NOOP; @@ -254,7 +256,8 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode) if (!HAS_FLAT_CCS(rq->engine->i915)) { /* hsdes: 1809175790 */ - cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV); + cs = gen12_emit_aux_table_inv(rq->engine->gt, + cs, GEN12_GFX_CCS_AUX_NV); } *cs++ = preparser_disable(false); @@ -313,9 +316,11 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode) if (aux_inv) { /* hsdes: 1809175790 */ if (rq->engine->class == VIDEO_DECODE_CLASS) - cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV); + cs = gen12_emit_aux_table_inv(rq->engine->gt, + cs, GEN12_VD0_AUX_NV); else - cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV); + cs = gen12_emit_aux_table_inv(rq->engine->gt, + cs, GEN12_VE0_AUX_NV); } if (mode & EMIT_INVALIDATE) diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h index 32e3d2b831bb4ee7e3b54bcba871425ecfc28b99..e4d24c811dd61dbbfdea8ced5bf6d272d0c341af 100644 --- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h +++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h @@ -13,6 +13,7 @@ #include "intel_gt_regs.h" #include "intel_gpu_commands.h" +struct intel_gt; struct i915_request; int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode); @@ -45,7 +46,7 @@ u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs); -u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg); +u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg); static inline u32 * __gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index c7bd5d71b03e53aaf85905eb09eb2ba378bfe0a8..2128b7a72a2575dce41f28279a6bb6f18abd7e7b 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -196,7 +196,10 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm) if (intel_vgpu_active(vm->i915)) gen8_ppgtt_notify_vgt(ppgtt, false); - __gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top); + if (ppgtt->pd) + __gen8_ppgtt_cleanup(vm, ppgtt->pd, + gen8_pd_top_count(vm), vm->top); + free_scratch(vm); } @@ -803,8 +806,10 @@ static int gen8_init_scratch(struct i915_address_space *vm) struct drm_i915_gem_object *obj; obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K); - if (IS_ERR(obj)) + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); goto free_scratch; + } ret = map_pt_dma(vm, obj); if (ret) { @@ -823,7 +828,8 @@ static int gen8_init_scratch(struct i915_address_space *vm) free_scratch: while (i--) i915_gem_object_put(vm->scratch[i]); - return -ENOMEM; + vm->scratch[0] = NULL; + return ret; } static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt) @@ -901,6 +907,7 @@ err_pd: struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, unsigned long lmem_pt_obj_flags) { + struct i915_page_directory *pd; struct i915_ppgtt *ppgtt; int err; @@ -946,21 +953,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, ppgtt->vm.alloc_scratch_dma = alloc_pt_dma; } - err = gen8_init_scratch(&ppgtt->vm); - if (err) - goto err_free; - - ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm); - if (IS_ERR(ppgtt->pd)) { - err = PTR_ERR(ppgtt->pd); - goto err_free_scratch; - } - - if (!i915_vm_is_4lvl(&ppgtt->vm)) { - err = gen8_preallocate_top_level_pdp(ppgtt); - if (err) - goto err_free_pd; - } + ppgtt->vm.pte_encode = gen8_pte_encode; ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND; ppgtt->vm.insert_entries = gen8_ppgtt_insert; @@ -971,22 +964,31 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt, ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; ppgtt->vm.clear_range = gen8_ppgtt_clear; ppgtt->vm.foreach = gen8_ppgtt_foreach; + ppgtt->vm.cleanup = gen8_ppgtt_cleanup; - ppgtt->vm.pte_encode = gen8_pte_encode; + err = gen8_init_scratch(&ppgtt->vm); + if (err) + goto err_put; + + pd = gen8_alloc_top_pd(&ppgtt->vm); + if (IS_ERR(pd)) { + err = PTR_ERR(pd); + goto err_put; + } + ppgtt->pd = pd; + + if (!i915_vm_is_4lvl(&ppgtt->vm)) { + err = gen8_preallocate_top_level_pdp(ppgtt); + if (err) + goto err_put; + } if (intel_vgpu_active(gt->i915)) gen8_ppgtt_notify_vgt(ppgtt, true); - ppgtt->vm.cleanup = gen8_ppgtt_cleanup; - return ppgtt; -err_free_pd: - __gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd, - gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top); -err_free_scratch: - free_scratch(&ppgtt->vm); -err_free: - kfree(ppgtt); +err_put: + i915_vm_put(&ppgtt->vm); return ERR_PTR(err); } diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 654a092ed3d69c547704799a877b8d6095343670..e94365b08f1efc133c8b6e88f2ba43156f414fa9 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -614,13 +614,12 @@ bool intel_context_ban(struct intel_context *ce, struct i915_request *rq) return ret; } -bool intel_context_exit_nonpersistent(struct intel_context *ce, - struct i915_request *rq) +bool intel_context_revoke(struct intel_context *ce) { bool ret = intel_context_set_exiting(ce); if (ce->ops->revoke) - ce->ops->revoke(ce, rq, ce->engine->props.preempt_timeout_ms); + ce->ops->revoke(ce, NULL, ce->engine->props.preempt_timeout_ms); return ret; } diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index 8e2d70630c49e82c456bf9499b0da5deb952ceda..be09fb2e883a547d8e132d7f858d10ff7d5e6dd0 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -329,8 +329,7 @@ static inline bool intel_context_set_exiting(struct intel_context *ce) return test_and_set_bit(CONTEXT_EXITING, &ce->flags); } -bool intel_context_exit_nonpersistent(struct intel_context *ce, - struct i915_request *rq); +bool intel_context_revoke(struct intel_context *ce); static inline bool intel_context_force_single_submission(const struct intel_context *ce) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 17e7f20bbb48a8a5db3e55926299768c691d4edb..1f7188129cd1f6a1f15a91f15121d42a333f43ce 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -654,16 +654,83 @@ bool gen11_vdbox_has_sfc(struct intel_gt *gt, */ if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0) return false; - else if (GRAPHICS_VER(i915) == 12) + else if (MEDIA_VER(i915) >= 12) return (physical_vdbox % 2 == 0) || !(BIT(physical_vdbox - 1) & vdbox_mask); - else if (GRAPHICS_VER(i915) == 11) + else if (MEDIA_VER(i915) == 11) return logical_vdbox % 2 == 0; - MISSING_CASE(GRAPHICS_VER(i915)); return false; } +static void engine_mask_apply_media_fuses(struct intel_gt *gt) +{ + struct drm_i915_private *i915 = gt->i915; + unsigned int logical_vdbox = 0; + unsigned int i; + u32 media_fuse, fuse1; + u16 vdbox_mask; + u16 vebox_mask; + + if (MEDIA_VER(gt->i915) < 11) + return; + + /* + * On newer platforms the fusing register is called 'enable' and has + * enable semantics, while on older platforms it is called 'disable' + * and bits have disable semantices. + */ + media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); + if (MEDIA_VER_FULL(i915) < IP_VER(12, 50)) + media_fuse = ~media_fuse; + + vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; + vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> + GEN11_GT_VEBOX_DISABLE_SHIFT; + + if (MEDIA_VER_FULL(i915) >= IP_VER(12, 50)) { + fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1); + gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1); + } else { + gt->info.sfc_mask = ~0; + } + + for (i = 0; i < I915_MAX_VCS; i++) { + if (!HAS_ENGINE(gt, _VCS(i))) { + vdbox_mask &= ~BIT(i); + continue; + } + + if (!(BIT(i) & vdbox_mask)) { + gt->info.engine_mask &= ~BIT(_VCS(i)); + drm_dbg(&i915->drm, "vcs%u fused off\n", i); + continue; + } + + if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask)) + gt->info.vdbox_sfc_access |= BIT(i); + logical_vdbox++; + } + drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n", + vdbox_mask, VDBOX_MASK(gt)); + GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt)); + + for (i = 0; i < I915_MAX_VECS; i++) { + if (!HAS_ENGINE(gt, _VECS(i))) { + vebox_mask &= ~BIT(i); + continue; + } + + if (!(BIT(i) & vebox_mask)) { + gt->info.engine_mask &= ~BIT(_VECS(i)); + drm_dbg(&i915->drm, "vecs%u fused off\n", i); + } + } + drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n", + vebox_mask, VEBOX_MASK(gt)); + GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); +} + static void engine_mask_apply_compute_fuses(struct intel_gt *gt) { struct drm_i915_private *i915 = gt->i915; @@ -672,6 +739,9 @@ static void engine_mask_apply_compute_fuses(struct intel_gt *gt) unsigned long ccs_mask; unsigned int i; + if (GRAPHICS_VER(i915) < 11) + return; + if (hweight32(CCS_MASK(gt)) <= 1) return; @@ -694,6 +764,10 @@ static void engine_mask_apply_copy_fuses(struct intel_gt *gt) unsigned long meml3_mask; unsigned long quad; + if (!(GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60) && + GRAPHICS_VER_FULL(i915) < IP_VER(12, 70))) + return; + meml3_mask = intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3); meml3_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, meml3_mask); @@ -727,75 +801,11 @@ static void engine_mask_apply_copy_fuses(struct intel_gt *gt) */ static intel_engine_mask_t init_engine_mask(struct intel_gt *gt) { - struct drm_i915_private *i915 = gt->i915; struct intel_gt_info *info = >->info; - struct intel_uncore *uncore = gt->uncore; - unsigned int logical_vdbox = 0; - unsigned int i; - u32 media_fuse, fuse1; - u16 vdbox_mask; - u16 vebox_mask; - - info->engine_mask = INTEL_INFO(i915)->platform_engine_mask; - - if (GRAPHICS_VER(i915) < 11) - return info->engine_mask; - /* - * On newer platforms the fusing register is called 'enable' and has - * enable semantics, while on older platforms it is called 'disable' - * and bits have disable semantices. - */ - media_fuse = intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE); - if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50)) - media_fuse = ~media_fuse; - - vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; - vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> - GEN11_GT_VEBOX_DISABLE_SHIFT; - - if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) { - fuse1 = intel_uncore_read(uncore, HSW_PAVP_FUSE1); - gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1); - } else { - gt->info.sfc_mask = ~0; - } - - for (i = 0; i < I915_MAX_VCS; i++) { - if (!HAS_ENGINE(gt, _VCS(i))) { - vdbox_mask &= ~BIT(i); - continue; - } - - if (!(BIT(i) & vdbox_mask)) { - info->engine_mask &= ~BIT(_VCS(i)); - drm_dbg(&i915->drm, "vcs%u fused off\n", i); - continue; - } - - if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask)) - gt->info.vdbox_sfc_access |= BIT(i); - logical_vdbox++; - } - drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n", - vdbox_mask, VDBOX_MASK(gt)); - GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt)); - - for (i = 0; i < I915_MAX_VECS; i++) { - if (!HAS_ENGINE(gt, _VECS(i))) { - vebox_mask &= ~BIT(i); - continue; - } - - if (!(BIT(i) & vebox_mask)) { - info->engine_mask &= ~BIT(_VECS(i)); - drm_dbg(&i915->drm, "vecs%u fused off\n", i); - } - } - drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n", - vebox_mask, VEBOX_MASK(gt)); - GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt)); + GEM_BUG_ON(!info->engine_mask); + engine_mask_apply_media_fuses(gt); engine_mask_apply_compute_fuses(gt); engine_mask_apply_copy_fuses(gt); @@ -1688,9 +1698,9 @@ bool intel_engine_irq_enable(struct intel_engine_cs *engine) return false; /* Caller disables interrupts */ - spin_lock(&engine->gt->irq_lock); + spin_lock(engine->gt->irq_lock); engine->irq_enable(engine); - spin_unlock(&engine->gt->irq_lock); + spin_unlock(engine->gt->irq_lock); return true; } @@ -1701,9 +1711,9 @@ void intel_engine_irq_disable(struct intel_engine_cs *engine) return; /* Caller disables interrupts */ - spin_lock(&engine->gt->irq_lock); + spin_lock(engine->gt->irq_lock); engine->irq_disable(engine); - spin_unlock(&engine->gt->irq_lock); + spin_unlock(engine->gt->irq_lock); } void intel_engines_reset_default_submission(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_regs.h b/drivers/gpu/drm/i915/gt/intel_engine_regs.h index 889f0df3940b8f5c7c5628c9777faba4c9af94a6..fe1a0d5fd4b1ad439cb4f6b0ecc1c4ba424c25a3 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_regs.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_regs.h @@ -110,6 +110,7 @@ #define RING_SBBSTATE(base) _MMIO((base) + 0x118) /* hsw+ */ #define RING_SBBADDR_UDW(base) _MMIO((base) + 0x11c) /* gen8+ */ #define RING_BBADDR(base) _MMIO((base) + 0x140) +#define RING_BB_OFFSET(base) _MMIO((base) + 0x158) #define RING_BBADDR_UDW(base) _MMIO((base) + 0x168) /* gen8+ */ #define CCID(base) _MMIO((base) + 0x180) #define CCID_EN BIT(0) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 633a7e5dba3b4bfd6f5b5c72d7992261096bce40..6b5d4ea22b673bacf320b18aa477de2466ee4171 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -165,6 +165,21 @@ struct intel_engine_execlists { */ struct timer_list preempt; + /** + * @preempt_target: active request at the time of the preemption request + * + * We force a preemption to occur if the pending contexts have not + * been promoted to active upon receipt of the CS ack event within + * the timeout. This timeout maybe chosen based on the target, + * using a very short timeout if the context is no longer schedulable. + * That short timeout may not be applicable to other contexts, so + * if a context switch should happen within before the preemption + * timeout, we may shoot early at an innocent context. To prevent this, + * we record which context was active at the time of the preemption + * request and only reset that context upon the timeout. + */ + const struct i915_request *preempt_target; + /** * @ccid: identifier for contexts submitted to this engine */ diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index 4b909cb88cdfb7b7422bd5cc1629f536f9422032..c718e6dc40b5158c0f4fdd89d27deddd02dbef3d 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -1241,6 +1241,9 @@ static unsigned long active_preempt_timeout(struct intel_engine_cs *engine, if (!rq) return 0; + /* Only allow ourselves to force reset the currently active context */ + engine->execlists.preempt_target = rq; + /* Force a fast reset for terminated contexts (ignoring sysfs!) */ if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq))) return INTEL_CONTEXT_BANNED_PREEMPT_TIMEOUT_MS; @@ -2427,8 +2430,24 @@ static void execlists_submission_tasklet(struct tasklet_struct *t) GEM_BUG_ON(inactive - post > ARRAY_SIZE(post)); if (unlikely(preempt_timeout(engine))) { + const struct i915_request *rq = *engine->execlists.active; + + /* + * If after the preempt-timeout expired, we are still on the + * same active request/context as before we initiated the + * preemption, reset the engine. + * + * However, if we have processed a CS event to switch contexts, + * but not yet processed the CS event for the pending + * preemption, reset the timer allowing the new context to + * gracefully exit. + */ cancel_timer(&engine->execlists.preempt); - engine->execlists.error_interrupt |= ERROR_PREEMPT; + if (rq == engine->execlists.preempt_target) + engine->execlists.error_interrupt |= ERROR_PREEMPT; + else + set_timer_ms(&engine->execlists.preempt, + active_preempt_timeout(engine, rq)); } if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) { diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 15a915bb4088e9e7e5601688791c8c0043e84049..2049a00417afad506b0c92b55bcc8c5a8185b521 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -16,7 +16,9 @@ #include "intel_ggtt_gmch.h" #include "intel_gt.h" #include "intel_gt_regs.h" +#include "intel_pci_config.h" #include "i915_drv.h" +#include "i915_pci.h" #include "i915_scatterlist.h" #include "i915_utils.h" #include "i915_vgpu.h" @@ -869,8 +871,8 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) u32 pte_flags; int ret; - GEM_WARN_ON(pci_resource_len(pdev, 0) != gen6_gttmmadr_size(i915)); - phys_addr = pci_resource_start(pdev, 0) + gen6_gttadr_offset(i915); + GEM_WARN_ON(pci_resource_len(pdev, GTTMMADR_BAR) != gen6_gttmmadr_size(i915)); + phys_addr = pci_resource_start(pdev, GTTMMADR_BAR) + gen6_gttadr_offset(i915); /* * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range @@ -930,7 +932,10 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) u16 snb_gmch_ctl; if (!HAS_LMEM(i915)) { - ggtt->gmadr = pci_resource(pdev, 2); + if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR)) + return -ENXIO; + + ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR); ggtt->mappable_end = resource_size(&ggtt->gmadr); } @@ -1084,7 +1089,10 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt) unsigned int size; u16 snb_gmch_ctl; - ggtt->gmadr = pci_resource(pdev, 2); + if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR)) + return -ENXIO; + + ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR); ggtt->mappable_end = resource_size(&ggtt->gmadr); /* @@ -1267,10 +1275,16 @@ bool i915_ggtt_resume_vm(struct i915_address_space *vm) atomic_read(&vma->flags) & I915_VMA_BIND_MASK; GEM_BUG_ON(!was_bound); - if (!retained_ptes) + if (!retained_ptes) { + /* + * Clear the bound flags of the vma resource to allow + * ptes to be repopulated. + */ + vma->resource->bound_flags = 0; vma->ops->bind_vma(vm, NULL, vma->resource, obj ? obj->cache_level : 0, was_bound); + } if (obj) { /* only used during resume => exclusive access */ write_domain_objs |= fetch_and_zero(&obj->write_domain); obj->read_domains |= I915_GEM_DOMAIN_GTT; diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 6ebda3d65086482fd394979e9afb195c41cf5571..ea775e601686d1e271258df9ddc346f4e1a37051 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -727,7 +727,7 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) * bit17 dependent, and so we need to also prevent the pages * from being moved. */ - i915->quirks |= QUIRK_PIN_SWIZZLED_PAGES; + i915->gem_quirks |= GEM_QUIRK_PIN_SWIZZLED_PAGES; swizzle_x = I915_BIT_6_SWIZZLE_NONE; swizzle_y = I915_BIT_6_SWIZZLE_NONE; } @@ -842,7 +842,6 @@ void intel_ggtt_init_fences(struct i915_ggtt *ggtt) INIT_LIST_HEAD(&ggtt->fence_list); INIT_LIST_HEAD(&ggtt->userfault_list); - intel_wakeref_auto_init(&ggtt->userfault_wakeref, uncore->rpm); detect_bit_6_swizzle(ggtt); diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.c b/drivers/gpu/drm/i915/gt/intel_gsc.c index 0e494028b81d07e007ab46e15740c74048a095f1..7af6db3194ddbf6bd8c77c47bbd6c795b35fab0e 100644 --- a/drivers/gpu/drm/i915/gt/intel_gsc.c +++ b/drivers/gpu/drm/i915/gt/intel_gsc.c @@ -7,6 +7,7 @@ #include #include "i915_drv.h" #include "i915_reg.h" +#include "gem/i915_gem_region.h" #include "gt/intel_gsc.h" #include "gt/intel_gt.h" @@ -36,10 +37,56 @@ static int gsc_irq_init(int irq) return irq_set_chip_data(irq, NULL); } +static int +gsc_ext_om_alloc(struct intel_gsc *gsc, struct intel_gsc_intf *intf, size_t size) +{ + struct intel_gt *gt = gsc_to_gt(gsc); + struct drm_i915_gem_object *obj; + int err; + + obj = i915_gem_object_create_lmem(gt->i915, size, + I915_BO_ALLOC_CONTIGUOUS | + I915_BO_ALLOC_CPU_CLEAR); + if (IS_ERR(obj)) { + drm_err(>->i915->drm, "Failed to allocate gsc memory\n"); + return PTR_ERR(obj); + } + + err = i915_gem_object_pin_pages_unlocked(obj); + if (err) { + drm_err(>->i915->drm, "Failed to pin pages for gsc memory\n"); + goto out_put; + } + + intf->gem_obj = obj; + + return 0; + +out_put: + i915_gem_object_put(obj); + return err; +} + +static void gsc_ext_om_destroy(struct intel_gsc_intf *intf) +{ + struct drm_i915_gem_object *obj = fetch_and_zero(&intf->gem_obj); + + if (!obj) + return; + + if (i915_gem_object_has_pinned_pages(obj)) + i915_gem_object_unpin_pages(obj); + + i915_gem_object_put(obj); +} + struct gsc_def { const char *name; unsigned long bar; size_t bar_size; + bool use_polling; + bool slow_firmware; + size_t lmem_size; }; /* gsc resources and definitions (HECI1 and HECI2) */ @@ -54,11 +101,25 @@ static const struct gsc_def gsc_def_dg1[] = { } }; +static const struct gsc_def gsc_def_xehpsdv[] = { + { + /* HECI1 not enabled on the device. */ + }, + { + .name = "mei-gscfi", + .bar = DG1_GSC_HECI2_BASE, + .bar_size = GSC_BAR_LENGTH, + .use_polling = true, + .slow_firmware = true, + } +}; + static const struct gsc_def gsc_def_dg2[] = { { .name = "mei-gsc", .bar = DG2_GSC_HECI1_BASE, .bar_size = GSC_BAR_LENGTH, + .lmem_size = SZ_4M, }, { .name = "mei-gscfi", @@ -75,26 +136,32 @@ static void gsc_release_dev(struct device *dev) kfree(adev); } -static void gsc_destroy_one(struct intel_gsc_intf *intf) +static void gsc_destroy_one(struct drm_i915_private *i915, + struct intel_gsc *gsc, unsigned int intf_id) { + struct intel_gsc_intf *intf = &gsc->intf[intf_id]; + if (intf->adev) { auxiliary_device_delete(&intf->adev->aux_dev); auxiliary_device_uninit(&intf->adev->aux_dev); intf->adev = NULL; } + if (intf->irq >= 0) irq_free_desc(intf->irq); intf->irq = -1; + + gsc_ext_om_destroy(intf); } -static void gsc_init_one(struct drm_i915_private *i915, - struct intel_gsc_intf *intf, +static void gsc_init_one(struct drm_i915_private *i915, struct intel_gsc *gsc, unsigned int intf_id) { struct pci_dev *pdev = to_pci_dev(i915->drm.dev); struct mei_aux_device *adev; struct auxiliary_device *aux_dev; const struct gsc_def *def; + struct intel_gsc_intf *intf = &gsc->intf[intf_id]; int ret; intf->irq = -1; @@ -105,6 +172,8 @@ static void gsc_init_one(struct drm_i915_private *i915, if (IS_DG1(i915)) { def = &gsc_def_dg1[intf_id]; + } else if (IS_XEHPSDV(i915)) { + def = &gsc_def_xehpsdv[intf_id]; } else if (IS_DG2(i915)) { def = &gsc_def_dg2[intf_id]; } else { @@ -117,10 +186,14 @@ static void gsc_init_one(struct drm_i915_private *i915, return; } + /* skip irq initialization */ + if (def->use_polling) + goto add_device; + intf->irq = irq_alloc_desc(0); if (intf->irq < 0) { drm_err(&i915->drm, "gsc irq error %d\n", intf->irq); - return; + goto fail; } ret = gsc_irq_init(intf->irq); @@ -129,16 +202,31 @@ static void gsc_init_one(struct drm_i915_private *i915, goto fail; } +add_device: adev = kzalloc(sizeof(*adev), GFP_KERNEL); if (!adev) goto fail; + if (def->lmem_size) { + drm_dbg(&i915->drm, "setting up GSC lmem\n"); + + if (gsc_ext_om_alloc(gsc, intf, def->lmem_size)) { + drm_err(&i915->drm, "setting up gsc extended operational memory failed\n"); + kfree(adev); + goto fail; + } + + adev->ext_op_mem.start = i915_gem_object_get_dma_address(intf->gem_obj, 0); + adev->ext_op_mem.end = adev->ext_op_mem.start + def->lmem_size; + } + adev->irq = intf->irq; adev->bar.parent = &pdev->resource[0]; adev->bar.start = def->bar + pdev->resource[0].start; adev->bar.end = adev->bar.start + def->bar_size - 1; adev->bar.flags = IORESOURCE_MEM; adev->bar.desc = IORES_DESC_NONE; + adev->slow_firmware = def->slow_firmware; aux_dev = &adev->aux_dev; aux_dev->name = def->name; @@ -165,7 +253,7 @@ static void gsc_init_one(struct drm_i915_private *i915, return; fail: - gsc_destroy_one(intf); + gsc_destroy_one(i915, gsc, intf->id); } static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id) @@ -182,10 +270,8 @@ static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id) return; } - if (gt->gsc.intf[intf_id].irq < 0) { - drm_err_ratelimited(>->i915->drm, "GSC irq: irq not set"); + if (gt->gsc.intf[intf_id].irq < 0) return; - } ret = generic_handle_irq(gt->gsc.intf[intf_id].irq); if (ret) @@ -208,7 +294,7 @@ void intel_gsc_init(struct intel_gsc *gsc, struct drm_i915_private *i915) return; for (i = 0; i < INTEL_GSC_NUM_INTERFACES; i++) - gsc_init_one(i915, &gsc->intf[i], i); + gsc_init_one(i915, gsc, i); } void intel_gsc_fini(struct intel_gsc *gsc) @@ -220,5 +306,5 @@ void intel_gsc_fini(struct intel_gsc *gsc) return; for (i = 0; i < INTEL_GSC_NUM_INTERFACES; i++) - gsc_destroy_one(&gsc->intf[i]); + gsc_destroy_one(gt->i915, gsc, i); } diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.h b/drivers/gpu/drm/i915/gt/intel_gsc.h index 68582f912b21a9cf62e021ccee89cb9bca100a6b..fcac1775e9c397793680349578f224f17370a20b 100644 --- a/drivers/gpu/drm/i915/gt/intel_gsc.h +++ b/drivers/gpu/drm/i915/gt/intel_gsc.h @@ -20,11 +20,14 @@ struct mei_aux_device; /** * struct intel_gsc - graphics security controller + * + * @gem_obj: scratch memory GSC operations * @intf : gsc interface */ struct intel_gsc { struct intel_gsc_intf { struct mei_aux_device *adev; + struct drm_i915_gem_object *gem_obj; int irq; unsigned int id; } intf[INTEL_GSC_NUM_INTERFACES]; diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index f435e06125aab0b99d79a842b857c6cffdfab068..d0b03a928b9acaaae274907fa73e4431186470e6 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -26,18 +26,22 @@ #include "intel_gt_requests.h" #include "intel_migrate.h" #include "intel_mocs.h" +#include "intel_pci_config.h" #include "intel_pm.h" #include "intel_rc6.h" #include "intel_renderstate.h" #include "intel_rps.h" +#include "intel_sa_media.h" #include "intel_gt_sysfs.h" #include "intel_uncore.h" #include "shmem_utils.h" -static void __intel_gt_init_early(struct intel_gt *gt) +void intel_gt_common_init_early(struct intel_gt *gt) { - spin_lock_init(>->irq_lock); + spin_lock_init(gt->irq_lock); + INIT_LIST_HEAD(>->lmem_userfault_list); + mutex_init(>->lmem_userfault_lock); INIT_LIST_HEAD(>->closed_vma); spin_lock_init(>->closed_lock); @@ -57,14 +61,19 @@ static void __intel_gt_init_early(struct intel_gt *gt) } /* Preliminary initialization of Tile 0 */ -void intel_root_gt_init_early(struct drm_i915_private *i915) +int intel_root_gt_init_early(struct drm_i915_private *i915) { struct intel_gt *gt = to_gt(i915); gt->i915 = i915; gt->uncore = &i915->uncore; + gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL); + if (!gt->irq_lock) + return -ENOMEM; - __intel_gt_init_early(gt); + intel_gt_common_init_early(gt); + + return 0; } static int intel_gt_probe_lmem(struct intel_gt *gt) @@ -780,26 +789,25 @@ static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr) int ret; if (!gt_is_root(gt)) { - struct intel_uncore_mmio_debug *mmio_debug; struct intel_uncore *uncore; + spinlock_t *irq_lock; - uncore = kzalloc(sizeof(*uncore), GFP_KERNEL); + uncore = drmm_kzalloc(>->i915->drm, sizeof(*uncore), GFP_KERNEL); if (!uncore) return -ENOMEM; - mmio_debug = kzalloc(sizeof(*mmio_debug), GFP_KERNEL); - if (!mmio_debug) { - kfree(uncore); + irq_lock = drmm_kzalloc(>->i915->drm, sizeof(*irq_lock), GFP_KERNEL); + if (!irq_lock) return -ENOMEM; - } gt->uncore = uncore; - gt->uncore->debug = mmio_debug; + gt->irq_lock = irq_lock; - __intel_gt_init_early(gt); + intel_gt_common_init_early(gt); } intel_uncore_init_early(gt->uncore, gt); + intel_wakeref_auto_init(>->userfault_wakeref, gt->uncore->rpm); ret = intel_uncore_setup_mmio(gt->uncore, phys_addr); if (ret) @@ -810,27 +818,17 @@ static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr) return 0; } -static void -intel_gt_tile_cleanup(struct intel_gt *gt) -{ - intel_uncore_cleanup_mmio(gt->uncore); - - if (!gt_is_root(gt)) { - kfree(gt->uncore->debug); - kfree(gt->uncore); - kfree(gt); - } -} - int intel_gt_probe_all(struct drm_i915_private *i915) { struct pci_dev *pdev = to_pci_dev(i915->drm.dev); struct intel_gt *gt = &i915->gt0; + const struct intel_gt_definition *gtdef; phys_addr_t phys_addr; unsigned int mmio_bar; + unsigned int i; int ret; - mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0; + mmio_bar = GRAPHICS_VER(i915) == 2 ? GEN2_GTTMMADR_BAR : GTTMMADR_BAR; phys_addr = pci_resource_start(pdev, mmio_bar); /* @@ -838,14 +836,74 @@ int intel_gt_probe_all(struct drm_i915_private *i915) * and it has been already initialized early during probe * in i915_driver_probe() */ + gt->i915 = i915; + gt->name = "Primary GT"; + gt->info.engine_mask = RUNTIME_INFO(i915)->platform_engine_mask; + + drm_dbg(&i915->drm, "Setting up %s\n", gt->name); ret = intel_gt_tile_setup(gt, phys_addr); if (ret) return ret; i915->gt[0] = gt; - /* TODO: add more tiles */ + if (!HAS_EXTRA_GT_LIST(i915)) + return 0; + + for (i = 1, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]; + gtdef->name != NULL; + i++, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]) { + gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL); + if (!gt) { + ret = -ENOMEM; + goto err; + } + + gt->i915 = i915; + gt->name = gtdef->name; + gt->type = gtdef->type; + gt->info.engine_mask = gtdef->engine_mask; + gt->info.id = i; + + drm_dbg(&i915->drm, "Setting up %s\n", gt->name); + if (GEM_WARN_ON(range_overflows_t(resource_size_t, + gtdef->mapping_base, + SZ_16M, + pci_resource_len(pdev, mmio_bar)))) { + ret = -ENODEV; + goto err; + } + + switch (gtdef->type) { + case GT_TILE: + ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base); + break; + + case GT_MEDIA: + ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base, + gtdef->gsi_offset); + break; + + case GT_PRIMARY: + /* Primary GT should not appear in extra GT list */ + default: + MISSING_CASE(gtdef->type); + ret = -ENODEV; + } + + if (ret) + goto err; + + i915->gt[i] = gt; + } + return 0; + +err: + i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret); + intel_gt_release_all(i915); + + return ret; } int intel_gt_tiles_init(struct drm_i915_private *i915) @@ -868,10 +926,8 @@ void intel_gt_release_all(struct drm_i915_private *i915) struct intel_gt *gt; unsigned int id; - for_each_gt(gt, i915, id) { - intel_gt_tile_cleanup(gt); + for_each_gt(gt, i915, id) i915->gt[id] = NULL; - } } void intel_gt_info_print(const struct intel_gt_info *info, diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h index 40b06adf509a28c7a3c44f32e0c8ed03e798ebad..2ee582e287c8dd7d698d695fd96a77f9f67ad3ee 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt.h @@ -44,7 +44,8 @@ static inline struct intel_gt *gsc_to_gt(struct intel_gsc *gsc) return container_of(gsc, struct intel_gt, gsc); } -void intel_root_gt_init_early(struct drm_i915_private *i915); +void intel_gt_common_init_early(struct intel_gt *gt); +int intel_root_gt_init_early(struct drm_i915_private *i915); int intel_gt_assign_ggtt(struct intel_gt *gt); int intel_gt_init_mmio(struct intel_gt *gt); int __must_check intel_gt_init_hw(struct intel_gt *gt); @@ -54,7 +55,6 @@ void intel_gt_driver_register(struct intel_gt *gt); void intel_gt_driver_unregister(struct intel_gt *gt); void intel_gt_driver_remove(struct intel_gt *gt); void intel_gt_driver_release(struct intel_gt *gt); - void intel_gt_driver_late_release_all(struct drm_i915_private *i915); int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c index d5d1b04dbcad2ce714257b7fa792ace03a114a1e..3f656d3dba9a8c8b82128ad47dea3f052abadba9 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c @@ -26,26 +26,6 @@ static u32 read_reference_ts_freq(struct intel_uncore *uncore) return base_freq + frac_freq; } -static u32 gen9_get_crystal_clock_freq(struct intel_uncore *uncore, - u32 rpm_config_reg) -{ - u32 f19_2_mhz = 19200000; - u32 f24_mhz = 24000000; - u32 crystal_clock = - (rpm_config_reg & GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >> - GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT; - - switch (crystal_clock) { - case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ: - return f19_2_mhz; - case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ: - return f24_mhz; - default: - MISSING_CASE(crystal_clock); - return 0; - } -} - static u32 gen11_get_crystal_clock_freq(struct intel_uncore *uncore, u32 rpm_config_reg) { @@ -72,98 +52,106 @@ static u32 gen11_get_crystal_clock_freq(struct intel_uncore *uncore, } } -static u32 read_clock_frequency(struct intel_uncore *uncore) +static u32 gen11_read_clock_frequency(struct intel_uncore *uncore) { - u32 f12_5_mhz = 12500000; - u32 f19_2_mhz = 19200000; - u32 f24_mhz = 24000000; + u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE); + u32 freq = 0; + + /* + * Note that on gen11+, the clock frequency may be reconfigured. + * We do not, and we assume nobody else does. + * + * First figure out the reference frequency. There are 2 ways + * we can compute the frequency, either through the + * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE + * tells us which one we should use. + */ + if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { + freq = read_reference_ts_freq(uncore); + } else { + u32 c0 = intel_uncore_read(uncore, RPM_CONFIG0); + + freq = gen11_get_crystal_clock_freq(uncore, c0); - if (GRAPHICS_VER(uncore->i915) <= 4) { - /* - * PRMs say: - * - * "The value in this register increments once every 16 - * hclks." (through the “Clocking Configuration” - * (“CLKCFG”) MCHBAR register) - */ - return RUNTIME_INFO(uncore->i915)->rawclk_freq * 1000 / 16; - } else if (GRAPHICS_VER(uncore->i915) <= 8) { /* - * PRMs say: - * - * "The PCU TSC counts 10ns increments; this timestamp - * reflects bits 38:3 of the TSC (i.e. 80ns granularity, - * rolling over every 1.5 hours). + * Now figure out how the command stream's timestamp + * register increments from this frequency (it might + * increment only every few clock cycle). */ - return f12_5_mhz; - } else if (GRAPHICS_VER(uncore->i915) <= 9) { - u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE); - u32 freq = 0; - - if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { - freq = read_reference_ts_freq(uncore); - } else { - freq = IS_GEN9_LP(uncore->i915) ? f19_2_mhz : f24_mhz; - - /* - * Now figure out how the command stream's timestamp - * register increments from this frequency (it might - * increment only every few clock cycle). - */ - freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >> - CTC_SHIFT_PARAMETER_SHIFT); - } - - return freq; - } else if (GRAPHICS_VER(uncore->i915) <= 12) { - u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE); - u32 freq = 0; + freq >>= 3 - ((c0 & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >> + GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT); + } + + return freq; +} + +static u32 gen9_read_clock_frequency(struct intel_uncore *uncore) +{ + u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE); + u32 freq = 0; + + if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { + freq = read_reference_ts_freq(uncore); + } else { + freq = IS_GEN9_LP(uncore->i915) ? 19200000 : 24000000; /* - * First figure out the reference frequency. There are 2 ways - * we can compute the frequency, either through the - * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE - * tells us which one we should use. + * Now figure out how the command stream's timestamp + * register increments from this frequency (it might + * increment only every few clock cycle). */ - if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { - freq = read_reference_ts_freq(uncore); - } else { - u32 c0 = intel_uncore_read(uncore, RPM_CONFIG0); - - if (GRAPHICS_VER(uncore->i915) >= 11) - freq = gen11_get_crystal_clock_freq(uncore, c0); - else - freq = gen9_get_crystal_clock_freq(uncore, c0); - - /* - * Now figure out how the command stream's timestamp - * register increments from this frequency (it might - * increment only every few clock cycle). - */ - freq >>= 3 - ((c0 & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >> - GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT); - } - - return freq; + freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >> + CTC_SHIFT_PARAMETER_SHIFT); } - MISSING_CASE("Unknown gen, unable to read command streamer timestamp frequency\n"); - return 0; + return freq; } -void intel_gt_init_clock_frequency(struct intel_gt *gt) +static u32 gen5_read_clock_frequency(struct intel_uncore *uncore) { /* - * Note that on gen11+, the clock frequency may be reconfigured. - * We do not, and we assume nobody else does. + * PRMs say: + * + * "The PCU TSC counts 10ns increments; this timestamp + * reflects bits 38:3 of the TSC (i.e. 80ns granularity, + * rolling over every 1.5 hours). + */ + return 12500000; +} + +static u32 gen2_read_clock_frequency(struct intel_uncore *uncore) +{ + /* + * PRMs say: + * + * "The value in this register increments once every 16 + * hclks." (through the “Clocking Configuration” + * (“CLKCFG”) MCHBAR register) */ + return RUNTIME_INFO(uncore->i915)->rawclk_freq * 1000 / 16; +} + +static u32 read_clock_frequency(struct intel_uncore *uncore) +{ + if (GRAPHICS_VER(uncore->i915) >= 11) + return gen11_read_clock_frequency(uncore); + else if (GRAPHICS_VER(uncore->i915) >= 9) + return gen9_read_clock_frequency(uncore); + else if (GRAPHICS_VER(uncore->i915) >= 5) + return gen5_read_clock_frequency(uncore); + else + return gen2_read_clock_frequency(uncore); +} + +void intel_gt_init_clock_frequency(struct intel_gt *gt) +{ gt->clock_frequency = read_clock_frequency(gt->uncore); - if (gt->clock_frequency) - gt->clock_period_ns = intel_gt_clock_interval_to_ns(gt, 1); /* Icelake appears to use another fixed frequency for CTX_TIMESTAMP */ if (GRAPHICS_VER(gt->i915) == 11) gt->clock_period_ns = NSEC_PER_SEC / 13750000; + else if (gt->clock_frequency) + gt->clock_period_ns = intel_gt_clock_interval_to_ns(gt, 1); GT_TRACE(gt, "Using clock frequency: %dkHz, period: %dns, wrap: %lldms\n", diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index 3a72d4fd0214e41fa60924b45697265b8143b1bd..f26882fdc24c9638832acb4b6155b8c7da468efd 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -29,7 +29,7 @@ gen11_gt_engine_identity(struct intel_gt *gt, u32 timeout_ts; u32 ident; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit)); @@ -59,11 +59,17 @@ static void gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, const u16 iir) { + struct intel_gt *media_gt = gt->i915->media_gt; + if (instance == OTHER_GUC_INSTANCE) return guc_irq_handler(>->uc.guc, iir); + if (instance == OTHER_MEDIA_GUC_INSTANCE && media_gt) + return guc_irq_handler(&media_gt->uc.guc, iir); if (instance == OTHER_GTPM_INSTANCE) return gen11_rps_irq_handler(>->rps, iir); + if (instance == OTHER_MEDIA_GTPM_INSTANCE && media_gt) + return gen11_rps_irq_handler(&media_gt->rps, iir); if (instance == OTHER_KCR_INSTANCE) return intel_pxp_irq_handler(>->pxp, iir); @@ -81,6 +87,18 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, { struct intel_engine_cs *engine; + /* + * Platforms with standalone media have their media engines in another + * GT. + */ + if (MEDIA_VER(gt->i915) >= 13 && + (class == VIDEO_DECODE_CLASS || class == VIDEO_ENHANCEMENT_CLASS)) { + if (!gt->i915->media_gt) + goto err; + + gt = gt->i915->media_gt; + } + if (instance <= MAX_ENGINE_INSTANCE) engine = gt->engine_class[class][instance]; else @@ -89,6 +107,7 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, if (likely(engine)) return intel_engine_cs_irq(engine, iir); +err: WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", class, instance); } @@ -120,7 +139,7 @@ gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank) unsigned long intr_dw; unsigned int bit; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); @@ -138,14 +157,14 @@ void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl) { unsigned int bank; - spin_lock(>->irq_lock); + spin_lock(gt->irq_lock); for (bank = 0; bank < 2; bank++) { if (master_ctl & GEN11_GT_DW_IRQ(bank)) gen11_gt_bank_handler(gt, bank); } - spin_unlock(>->irq_lock); + spin_unlock(gt->irq_lock); } bool gen11_gt_reset_one_iir(struct intel_gt *gt, @@ -154,7 +173,7 @@ bool gen11_gt_reset_one_iir(struct intel_gt *gt, void __iomem * const regs = gt->uncore->regs; u32 dw; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank)); if (dw & BIT(bit)) { @@ -310,9 +329,9 @@ static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir) if (!HAS_L3_DPF(gt->i915)) return; - spin_lock(>->irq_lock); + spin_lock(gt->irq_lock); gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915)); - spin_unlock(>->irq_lock); + spin_unlock(gt->irq_lock); if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1) gt->i915->l3_parity.which_slice |= 1 << 1; @@ -434,7 +453,7 @@ static void gen5_gt_update_irq(struct intel_gt *gt, u32 interrupt_mask, u32 enabled_irq_mask) { - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c index 40bdd4cb629feadcd70506d0de871582b688c7b4..108b9e76c32e874924bc1d6b32ff21dce484eff7 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c @@ -504,8 +504,8 @@ void intel_gt_pm_frequency_dump(struct intel_gt *gt, struct drm_printer *p) drm_puts(p, "no P-state info available\n"); } - drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->cdclk.hw.cdclk); - drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->max_cdclk_freq); + drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk); + drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq); drm_printf(p, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq); intel_runtime_pm_put(uncore->rpm, wakeref); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c index 11060f5a4c8984c8aa98f0f598410d243a4335f6..52f2a28b2058e812bba5263a9fcd724b1f449db2 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c @@ -37,7 +37,7 @@ static void gen6_gt_pm_update_irq(struct intel_gt *gt, WARN_ON(enabled_irq_mask & ~interrupt_mask); - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); new_val = gt->pm_imr; new_val &= ~interrupt_mask; @@ -64,7 +64,7 @@ void gen6_gt_pm_reset_iir(struct intel_gt *gt, u32 reset_mask) struct intel_uncore *uncore = gt->uncore; i915_reg_t reg = GRAPHICS_VER(gt->i915) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); intel_uncore_write(uncore, reg, reset_mask); intel_uncore_write(uncore, reg, reset_mask); @@ -92,7 +92,7 @@ static void write_pm_ier(struct intel_gt *gt) void gen6_gt_pm_enable_irq(struct intel_gt *gt, u32 enable_mask) { - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); gt->pm_ier |= enable_mask; write_pm_ier(gt); @@ -101,7 +101,7 @@ void gen6_gt_pm_enable_irq(struct intel_gt *gt, u32 enable_mask) void gen6_gt_pm_disable_irq(struct intel_gt *gt, u32 disable_mask) { - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); gt->pm_ier &= ~disable_mask; gen6_gt_pm_mask_irq(gt, disable_mask); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h index d414785003ccc42f9cb2dbeb3bbfcd765c7f0e74..2275ee47da95589311015189babcc8a40d0cb40d 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h @@ -1554,6 +1554,8 @@ #define OTHER_GTPM_INSTANCE 1 #define OTHER_KCR_INSTANCE 4 #define OTHER_GSC_INSTANCE 6 +#define OTHER_MEDIA_GUC_INSTANCE 16 +#define OTHER_MEDIA_GTPM_INSTANCE 17 #define GEN11_IIR_REG_SELECTOR(x) _MMIO(0x190070 + ((x) * 4)) @@ -1578,4 +1580,12 @@ #define GEN12_SFC_DONE(n) _MMIO(0x1cc000 + (n) * 0x1000) +/* + * Standalone Media's non-engine GT registers are located at their regular GT + * offsets plus 0x380000. This extra offset is stored inside the intel_uncore + * structure so that the existing code can be used for both GTs without + * modification. + */ +#define MTL_MEDIA_GSI_BASE 0x380000 + #endif /* __INTEL_GT_REGS__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c index e066cc33d9f2c8cfe5292f493d72528c01454cac..180dd6f3ef57148c56d0adb1fcfa398c3162e129 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c @@ -545,8 +545,7 @@ static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_ratl, RATL_MASK); static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_vr_thermalert, VR_THERMALERT_MASK); static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_vr_tdc, VR_TDC_MASK); -static const struct attribute *freq_attrs[] = { - &dev_attr_punit_req_freq_mhz.attr, +static const struct attribute *throttle_reason_attrs[] = { &attr_throttle_reason_status.attr, &attr_throttle_reason_pl1.attr, &attr_throttle_reason_pl2.attr, @@ -791,12 +790,20 @@ void intel_gt_sysfs_pm_init(struct intel_gt *gt, struct kobject *kobj) if (!is_object_gt(kobj)) return; - ret = sysfs_create_files(kobj, freq_attrs); + ret = sysfs_create_file(kobj, &dev_attr_punit_req_freq_mhz.attr); if (ret) drm_warn(>->i915->drm, - "failed to create gt%u throttle sysfs files (%pe)", + "failed to create gt%u punit_req_freq_mhz sysfs (%pe)", gt->info.id, ERR_PTR(ret)); + if (GRAPHICS_VER(gt->i915) >= 11) { + ret = sysfs_create_files(kobj, throttle_reason_attrs); + if (ret) + drm_warn(>->i915->drm, + "failed to create gt%u throttle sysfs files (%pe)", + gt->info.id, ERR_PTR(ret)); + } + if (HAS_MEDIA_RATIO_MODE(gt->i915) && intel_uc_uses_guc_slpc(>->uc)) { ret = sysfs_create_files(kobj, media_perf_power_attrs); if (ret) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 4d56f7d5a3be5e01cec748bf67fc641fdcf4cd43..f19c2de77ff667ecd99cd242ef78d3f18a97c8ba 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -81,8 +81,17 @@ struct gt_defaults { u32 max_freq; }; +enum intel_gt_type { + GT_PRIMARY, + GT_TILE, + GT_MEDIA, +}; + struct intel_gt { struct drm_i915_private *i915; + const char *name; + enum intel_gt_type type; + struct intel_uncore *uncore; struct i915_ggtt *ggtt; @@ -132,6 +141,20 @@ struct intel_gt { struct intel_wakeref wakeref; atomic_t user_wakeref; + /** + * Protects access to lmem usefault list. + * It is required, if we are outside of the runtime suspend path, + * access to @lmem_userfault_list requires always first grabbing the + * runtime pm, to ensure we can't race against runtime suspend. + * Once we have that we also need to grab @lmem_userfault_lock, + * at which point we have exclusive access. + * The runtime suspend path is special since it doesn't really hold any locks, + * but instead has exclusive access by virtue of all other accesses requiring + * holding the runtime pm wakeref. + */ + struct mutex lmem_userfault_lock; + struct list_head lmem_userfault_list; + struct list_head closed_vma; spinlock_t closed_lock; /* guards the list of closed_vma */ @@ -147,6 +170,9 @@ struct intel_gt { */ intel_wakeref_t awake; + /* Manual runtime pm autosuspend delay for user GGTT/lmem mmaps */ + struct intel_wakeref_auto userfault_wakeref; + u32 clock_frequency; u32 clock_period_ns; @@ -154,7 +180,7 @@ struct intel_gt { struct intel_rc6 rc6; struct intel_rps rps; - spinlock_t irq_lock; + spinlock_t *irq_lock; u32 gt_imr; u32 pm_ier; u32 pm_imr; @@ -262,6 +288,14 @@ struct intel_gt { struct kobject *sysfs_defaults; }; +struct intel_gt_definition { + enum intel_gt_type type; + char *name; + u32 mapping_base; + u32 gsi_offset; + intel_engine_mask_t engine_mask; +}; + enum intel_gt_scratch_field { /* 8 bytes */ INTEL_GT_SCRATCH_FIELD_DEFAULT = 0, diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index b67831833c9a3c9f19b9d6be2aaf67e4a17ff937..2eaeba14319e997c34fd8a58c8cd03786d729638 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -405,6 +405,9 @@ void free_scratch(struct i915_address_space *vm) { int i; + if (!vm->scratch[0]) + return; + for (i = 0; i <= vm->top; i++) i915_gem_object_put(vm->scratch[i]); } diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index e639434e97fdb40dcfa2af0b0bb84d2de9ff5617..c0ca53cba9f08f8c0d11cfd126aa822269618508 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -386,9 +386,6 @@ struct i915_ggtt { */ struct list_head userfault_list; - /* Manual runtime pm autosuspend delay for user GGTT mmaps */ - struct intel_wakeref_auto userfault_wakeref; - struct mutex error_mutex; struct drm_mm_node error_capture; struct drm_mm_node uc_fw; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 070cec4ff8a4871f4e61ee2038289972d42aea10..3955292483a6f13b2ce547807c4554208b72c8c1 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -662,6 +662,21 @@ static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) return -1; } +static int lrc_ring_bb_offset(const struct intel_engine_cs *engine) +{ + if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) + return 0x80; + else if (GRAPHICS_VER(engine->i915) >= 12) + return 0x70; + else if (GRAPHICS_VER(engine->i915) >= 9) + return 0x64; + else if (GRAPHICS_VER(engine->i915) >= 8 && + engine->class == RENDER_CLASS) + return 0xc4; + else + return -1; +} + static int lrc_ring_gpr0(const struct intel_engine_cs *engine) { if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) @@ -768,6 +783,7 @@ static void init_common_regs(u32 * const regs, bool inhibit) { u32 ctl; + int loc; ctl = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH); ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); @@ -779,6 +795,10 @@ static void init_common_regs(u32 * const regs, regs[CTX_CONTEXT_CONTROL] = ctl; regs[CTX_TIMESTAMP] = ce->stats.runtime.last; + + loc = lrc_ring_bb_offset(engine); + if (loc != -1) + regs[loc + 1] = 0; } static void init_wa_bb_regs(u32 * const regs, @@ -1278,7 +1298,8 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) /* hsdes: 1809175790 */ if (!HAS_FLAT_CCS(ce->engine->i915)) - cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV); + cs = gen12_emit_aux_table_inv(ce->engine->gt, + cs, GEN12_GFX_CCS_AUX_NV); /* Wa_16014892111 */ if (IS_DG2(ce->engine->i915)) @@ -1304,9 +1325,11 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) /* hsdes: 1809175790 */ if (!HAS_FLAT_CCS(ce->engine->i915)) { if (ce->engine->class == VIDEO_DECODE_CLASS) - cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV); + cs = gen12_emit_aux_table_inv(ce->engine->gt, + cs, GEN12_VD0_AUX_NV); else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS) - cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV); + cs = gen12_emit_aux_table_inv(ce->engine->gt, + cs, GEN12_VE0_AUX_NV); } return cs; diff --git a/drivers/gpu/drm/i915/gt/intel_mocs.c b/drivers/gpu/drm/i915/gt/intel_mocs.c index c6ebe2781076465260a51e944d0e424c06843e9c..152244d7f62a03cd2810a1995629ebae90ec89be 100644 --- a/drivers/gpu/drm/i915/gt/intel_mocs.c +++ b/drivers/gpu/drm/i915/gt/intel_mocs.c @@ -207,6 +207,14 @@ static const struct drm_i915_mocs_entry broxton_mocs_table[] = { MOCS_ENTRY(15, \ LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ L3_3_WB), \ + /* Bypass LLC - Uncached (EHL+) */ \ + MOCS_ENTRY(16, \ + LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ + L3_1_UC), \ + /* Bypass LLC - L3 (Read-Only) (EHL+) */ \ + MOCS_ENTRY(17, \ + LE_1_UC | LE_TC_1_LLC | LE_SCF(1), \ + L3_3_WB), \ /* Self-Snoop - L3 + LLC */ \ MOCS_ENTRY(18, \ LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \ diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c index 6ee8d11270168fe7af5ae557a1ad06bb9d386488..7ecfa672f738d7087e85e3fcbceb34e0b0c50572 100644 --- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c @@ -312,7 +312,7 @@ void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt, ppgtt->vm.gt = gt; ppgtt->vm.i915 = i915; ppgtt->vm.dma = i915->drm.dev; - ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size); + ppgtt->vm.total = BIT_ULL(RUNTIME_INFO(i915)->ppgtt_size); ppgtt->vm.lmem_pt_obj_flags = lmem_pt_obj_flags; dma_resv_init(&ppgtt->vm._resv); diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c index aa6aed8371947b116851244b134e1b4763cd3eb6..f3ad93db0b21fa4da66723bf2cf6e9db426af6de 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c @@ -4,8 +4,10 @@ */ #include "i915_drv.h" +#include "i915_pci.h" #include "i915_reg.h" #include "intel_memory_region.h" +#include "intel_pci_config.h" #include "intel_region_lmem.h" #include "intel_region_ttm.h" #include "gem/i915_gem_lmem.h" @@ -45,7 +47,6 @@ _resize_bar(struct drm_i915_private *i915, int resno, resource_size_t size) drm_info(&i915->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); } -#define LMEM_BAR_NUM 2 static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t lmem_size) { struct pci_dev *pdev = to_pci_dev(i915->drm.dev); @@ -56,15 +57,14 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t u32 pci_cmd; int i; - current_size = roundup_pow_of_two(pci_resource_len(pdev, LMEM_BAR_NUM)); + current_size = roundup_pow_of_two(pci_resource_len(pdev, GEN12_LMEM_BAR)); if (i915->params.lmem_bar_size) { u32 bar_sizes; rebar_size = i915->params.lmem_bar_size * (resource_size_t)SZ_1M; - bar_sizes = pci_rebar_get_possible_sizes(pdev, - LMEM_BAR_NUM); + bar_sizes = pci_rebar_get_possible_sizes(pdev, GEN12_LMEM_BAR); if (rebar_size == current_size) return; @@ -107,7 +107,7 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); - _resize_bar(i915, LMEM_BAR_NUM, rebar_size); + _resize_bar(i915, GEN12_LMEM_BAR, rebar_size); pci_assign_unassigned_bus_resources(pdev->bus); pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); @@ -202,6 +202,9 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt) if (!IS_DGFX(i915)) return ERR_PTR(-ENODEV); + if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR)) + return ERR_PTR(-ENXIO); + if (HAS_FLAT_CCS(i915)) { resource_size_t lmem_range; u64 tile_stolen, flat_ccs_base; @@ -236,8 +239,8 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt) mul_u32_u32(i915->params.lmem_size, SZ_1M)); } - io_start = pci_resource_start(pdev, 2); - io_size = min(pci_resource_len(pdev, 2), lmem_size); + io_start = pci_resource_start(pdev, GEN12_LMEM_BAR); + io_size = min(pci_resource_len(pdev, GEN12_LMEM_BAR), lmem_size); if (!io_size) return ERR_PTR(-EIO); diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 6fadde4ee7bf76abf317c145c74362b797dadd5c..6b86250c31ab56c44dcbd7cee0e2599defe300a8 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -194,9 +194,9 @@ static void rps_enable_interrupts(struct intel_rps *rps) rps_reset_ei(rps); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen6_gt_pm_enable_irq(gt, rps->pm_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); @@ -217,14 +217,14 @@ static void rps_reset_interrupts(struct intel_rps *rps) { struct intel_gt *gt = rps_to_gt(rps); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); if (GRAPHICS_VER(gt->i915) >= 11) gen11_rps_reset_interrupts(rps); else gen6_rps_reset_interrupts(rps); rps->pm_iir = 0; - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static void rps_disable_interrupts(struct intel_rps *rps) @@ -234,9 +234,9 @@ static void rps_disable_interrupts(struct intel_rps *rps) intel_uncore_write(gt->uncore, GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); intel_synchronize_irq(gt->i915); @@ -1797,10 +1797,10 @@ static void rps_work(struct work_struct *work) int new_freq, adj, min, max; u32 pm_iir = 0; - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; client_boost = atomic_read(&rps->num_waiters); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); /* Make sure we didn't queue anything we're not going to process. */ if (!pm_iir && !client_boost) @@ -1873,9 +1873,9 @@ static void rps_work(struct work_struct *work) mutex_unlock(&rps->lock); out: - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen6_gt_pm_unmask_irq(gt, rps->pm_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) @@ -1883,7 +1883,7 @@ void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) struct intel_gt *gt = rps_to_gt(rps); const u32 events = rps->pm_events & pm_iir; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); if (unlikely(!events)) return; @@ -1903,7 +1903,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) events = pm_iir & rps->pm_events; if (events) { - spin_lock(>->irq_lock); + spin_lock(gt->irq_lock); GT_TRACE(gt, "irq events:%x\n", events); @@ -1911,7 +1911,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) rps->pm_iir |= events; schedule_work(&rps->work); - spin_unlock(>->irq_lock); + spin_unlock(gt->irq_lock); } if (GRAPHICS_VER(gt->i915) >= 8) diff --git a/drivers/gpu/drm/i915/gt/intel_sa_media.c b/drivers/gpu/drm/i915/gt/intel_sa_media.c new file mode 100644 index 0000000000000000000000000000000000000000..e8f3d18c12b838d37a1e06be16e7ef2aa06cb0fd --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_sa_media.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include + +#include "i915_drv.h" +#include "gt/intel_gt.h" +#include "gt/intel_sa_media.h" + +int intel_sa_mediagt_setup(struct intel_gt *gt, phys_addr_t phys_addr, + u32 gsi_offset) +{ + struct drm_i915_private *i915 = gt->i915; + struct intel_uncore *uncore; + + uncore = drmm_kzalloc(&i915->drm, sizeof(*uncore), GFP_KERNEL); + if (!uncore) + return -ENOMEM; + + uncore->gsi_offset = gsi_offset; + + gt->irq_lock = to_gt(i915)->irq_lock; + intel_gt_common_init_early(gt); + intel_uncore_init_early(uncore, gt); + + /* + * Standalone media shares the general MMIO space with the primary + * GT. We'll re-use the primary GT's mapping. + */ + uncore->regs = i915->uncore.regs; + if (drm_WARN_ON(&i915->drm, uncore->regs == NULL)) + return -EIO; + + gt->uncore = uncore; + gt->phys_addr = phys_addr; + + /* + * For current platforms we can assume there's only a single + * media GT and cache it for quick lookup. + */ + drm_WARN_ON(&i915->drm, i915->media_gt); + i915->media_gt = gt; + + return 0; +} diff --git a/drivers/gpu/drm/i915/gt/intel_sa_media.h b/drivers/gpu/drm/i915/gt/intel_sa_media.h new file mode 100644 index 0000000000000000000000000000000000000000..3afb310de9323526cb962136d4a6a275ee22e538 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_sa_media.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ +#ifndef __INTEL_SA_MEDIA__ +#define __INTEL_SA_MEDIA__ + +#include + +struct intel_gt; + +int intel_sa_mediagt_setup(struct intel_gt *gt, phys_addr_t phys_addr, + u32 gsi_offset); + +#endif /* __INTEL_SA_MEDIA_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c index c6d3050604c89d4f71a770175379e72b245a31aa..66f21c735d548a5301773cdb12cda9dae319bde2 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu.c @@ -382,7 +382,6 @@ static void cherryview_sseu_info_init(struct intel_gt *gt) static void gen9_sseu_info_init(struct intel_gt *gt) { struct drm_i915_private *i915 = gt->i915; - struct intel_device_info *info = mkwrite_device_info(i915); struct sseu_dev_info *sseu = >->info.sseu; struct intel_uncore *uncore = gt->uncore; u32 fuse2, eu_disable, subslice_mask; @@ -471,10 +470,10 @@ static void gen9_sseu_info_init(struct intel_gt *gt) if (IS_GEN9_LP(i915)) { #define IS_SS_DISABLED(ss) (!(sseu->subslice_mask.hsw[0] & BIT(ss))) - info->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3; + RUNTIME_INFO(i915)->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3; sseu->min_eu_in_pool = 0; - if (info->has_pooled_eu) { + if (HAS_POOLED_EU(i915)) { if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0)) sseu->min_eu_in_pool = 3; else if (IS_SS_DISABLED(1)) diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 1109088fe8f632e2a533791af40d80bfe5374cac..82d3f8058995a9bdd4735f8f8c8229dfe679d13f 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -27,6 +27,9 @@ #define NUM_GPR 16 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ +#define LRI_HEADER MI_INSTR(0x22, 0) +#define LRI_LENGTH_MASK GENMASK(7, 0) + static struct i915_vma *create_scratch(struct intel_gt *gt) { return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); @@ -202,7 +205,7 @@ static int live_lrc_layout(void *arg) continue; } - if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + if ((lri & GENMASK(31, 23)) != LRI_HEADER) { pr_err("%s: Expected LRI command at dword %d, found %08x\n", engine->name, dw, lri); err = -EINVAL; @@ -357,6 +360,11 @@ static int live_lrc_fixed(void *arg) lrc_ring_cmd_buf_cctl(engine), "RING_CMD_BUF_CCTL" }, + { + i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)), + lrc_ring_bb_offset(engine), + "RING_BB_OFFSET" + }, { }, }, *t; u32 *hw; @@ -987,18 +995,40 @@ store_context(struct intel_context *ce, struct i915_vma *scratch) hw = defaults; hw += LRC_STATE_OFFSET / sizeof(*hw); do { - u32 len = hw[dw] & 0x7f; + u32 len = hw[dw] & LRI_LENGTH_MASK; + + /* + * Keep it simple, skip parsing complex commands + * + * At present, there are no more MI_LOAD_REGISTER_IMM + * commands after the first 3D state command. Rather + * than include a table (see i915_cmd_parser.c) of all + * the possible commands and their instruction lengths + * (or mask for variable length instructions), assume + * we have gathered the complete list of registers and + * bail out. + */ + if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) + break; if (hw[dw] == 0) { dw++; continue; } - if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { + /* Assume all other MI commands match LRI length mask */ dw += len + 2; continue; } + if (!len) { + pr_err("%s: invalid LRI found in context image\n", + ce->engine->name); + igt_hexdump(defaults, PAGE_SIZE); + break; + } + dw++; len = (len + 1) / 2; while (len--) { @@ -1150,18 +1180,29 @@ static struct i915_vma *load_context(struct intel_context *ce, u32 poison) hw = defaults; hw += LRC_STATE_OFFSET / sizeof(*hw); do { - u32 len = hw[dw] & 0x7f; + u32 len = hw[dw] & LRI_LENGTH_MASK; + + /* For simplicity, break parsing at the first complex command */ + if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) + break; if (hw[dw] == 0) { dw++; continue; } - if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { dw += len + 2; continue; } + if (!len) { + pr_err("%s: invalid LRI found in context image\n", + ce->engine->name); + igt_hexdump(defaults, PAGE_SIZE); + break; + } + dw++; len = (len + 1) / 2; *cs++ = MI_LOAD_REGISTER_IMM(len); @@ -1292,18 +1333,29 @@ static int compare_isolation(struct intel_engine_cs *engine, hw = defaults; hw += LRC_STATE_OFFSET / sizeof(*hw); do { - u32 len = hw[dw] & 0x7f; + u32 len = hw[dw] & LRI_LENGTH_MASK; + + /* For simplicity, break parsing at the first complex command */ + if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT) + break; if (hw[dw] == 0) { dw++; continue; } - if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) { dw += len + 2; continue; } + if (!len) { + pr_err("%s: invalid LRI found in context image\n", + engine->name); + igt_hexdump(defaults, PAGE_SIZE); + break; + } + dw++; len = (len + 1) / 2; while (len--) { @@ -1343,6 +1395,30 @@ err_A0: return err; } +static struct i915_vma * +create_result_vma(struct i915_address_space *vm, unsigned long sz) +{ + struct i915_vma *vma; + void *ptr; + + vma = create_user_vma(vm, sz); + if (IS_ERR(vma)) + return vma; + + /* Set the results to a known value distinct from the poison */ + ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC); + if (IS_ERR(ptr)) { + i915_vma_put(vma); + return ERR_CAST(ptr); + } + + memset(ptr, POISON_INUSE, vma->size); + i915_gem_object_flush_map(vma->obj); + i915_gem_object_unpin_map(vma->obj); + + return vma; +} + static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) { u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); @@ -1361,13 +1437,13 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) goto err_A; } - ref[0] = create_user_vma(A->vm, SZ_64K); + ref[0] = create_result_vma(A->vm, SZ_64K); if (IS_ERR(ref[0])) { err = PTR_ERR(ref[0]); goto err_B; } - ref[1] = create_user_vma(A->vm, SZ_64K); + ref[1] = create_result_vma(A->vm, SZ_64K); if (IS_ERR(ref[1])) { err = PTR_ERR(ref[1]); goto err_ref0; @@ -1389,13 +1465,13 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) } i915_request_put(rq); - result[0] = create_user_vma(A->vm, SZ_64K); + result[0] = create_result_vma(A->vm, SZ_64K); if (IS_ERR(result[0])) { err = PTR_ERR(result[0]); goto err_ref1; } - result[1] = create_user_vma(A->vm, SZ_64K); + result[1] = create_result_vma(A->vm, SZ_64K); if (IS_ERR(result[1])) { err = PTR_ERR(result[1]); goto err_result0; @@ -1408,18 +1484,17 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) } err = poison_registers(B, poison, sema); - if (err) { - WRITE_ONCE(*sema, -1); - i915_request_put(rq); - goto err_result1; - } - - if (i915_request_wait(rq, 0, HZ / 2) < 0) { - i915_request_put(rq); + if (err == 0 && i915_request_wait(rq, 0, HZ / 2) < 0) { + pr_err("%s(%s): wait for results timed out\n", + __func__, engine->name); err = -ETIME; - goto err_result1; } + + /* Always cancel the semaphore wait, just in case the GPU gets stuck */ + WRITE_ONCE(*sema, -1); i915_request_put(rq); + if (err) + goto err_result1; err = compare_isolation(engine, ref, result, A, poison); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 24451d000a6a615ad848fc96014f93e8d89913d8..bac06e3d6f2cc9cd40360763d46eccbc2d5ef39e 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -82,9 +82,9 @@ static void gen9_reset_guc_interrupts(struct intel_guc *guc) assert_rpm_wakelock_held(>->i915->runtime_pm); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen6_gt_pm_reset_iir(gt, gt->pm_guc_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static void gen9_enable_guc_interrupts(struct intel_guc *guc) @@ -93,11 +93,11 @@ static void gen9_enable_guc_interrupts(struct intel_guc *guc) assert_rpm_wakelock_held(>->i915->runtime_pm); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); WARN_ON_ONCE(intel_uncore_read(gt->uncore, GEN8_GT_IIR(2)) & gt->pm_guc_events); gen6_gt_pm_enable_irq(gt, gt->pm_guc_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static void gen9_disable_guc_interrupts(struct intel_guc *guc) @@ -106,11 +106,11 @@ static void gen9_disable_guc_interrupts(struct intel_guc *guc) assert_rpm_wakelock_held(>->i915->runtime_pm); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen6_gt_pm_disable_irq(gt, gt->pm_guc_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); intel_synchronize_irq(gt->i915); gen9_reset_guc_interrupts(guc); @@ -120,9 +120,9 @@ static void gen11_reset_guc_interrupts(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen11_gt_reset_one_iir(gt, 0, GEN11_GUC); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static void gen11_enable_guc_interrupts(struct intel_guc *guc) @@ -130,25 +130,25 @@ static void gen11_enable_guc_interrupts(struct intel_guc *guc) struct intel_gt *gt = guc_to_gt(guc); u32 events = REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GUC)); intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_ENABLE, events); intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_MASK, ~events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static void gen11_disable_guc_interrupts(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_MASK, ~0); intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_ENABLE, 0); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); intel_synchronize_irq(gt->i915); gen11_reset_guc_interrupts(guc); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c index b071973ac41c13815a546417737334dda9aad3cc..55d3ef93e86f856b4e04f10f249af80240bc71d7 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c @@ -36,24 +36,6 @@ struct guc_log_section { const char *name; }; -static s32 scale_log_param(struct intel_guc_log *log, const struct guc_log_section *section, - s32 param) -{ - /* -1 means default */ - if (param < 0) - return section->default_val; - - /* Check for 32-bit overflow */ - if (param >= SZ_4K) { - drm_err(&guc_to_gt(log_to_guc(log))->i915->drm, "Size too large for GuC %s log: %dMB!", - section->name, param); - return section->default_val; - } - - /* Param units are 1MB */ - return param * SZ_1M; -} - static void _guc_log_init_sizes(struct intel_guc_log *log) { struct intel_guc *guc = log_to_guc(log); @@ -78,15 +60,10 @@ static void _guc_log_init_sizes(struct intel_guc_log *log) "capture", } }; - s32 params[GUC_LOG_SECTIONS_LIMIT] = { - GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE / SZ_1M, - GUC_LOG_DEFAULT_DEBUG_BUFFER_SIZE / SZ_1M, - GUC_LOG_DEFAULT_CAPTURE_BUFFER_SIZE / SZ_1M, - }; int i; for (i = 0; i < GUC_LOG_SECTIONS_LIMIT; i++) - log->sizes[i].bytes = scale_log_param(log, sections + i, params[i]); + log->sizes[i].bytes = sections[i].default_val; /* If debug size > 1MB then bump default crash size to keep the same units */ if (log->sizes[GUC_LOG_SECTIONS_DEBUG].bytes >= SZ_1M && diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 64c4e83153f477ee69a597b9354efbd73208d1eb..1db59eeb34db9e02782b50900d660aff97803ec9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -684,7 +684,7 @@ static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq) * Corner case where requests were sitting in the priority list or a * request resubmitted after the context was banned. */ - if (unlikely(intel_context_is_banned(ce))) { + if (unlikely(!intel_context_is_schedulable(ce))) { i915_request_put(i915_request_mark_eio(rq)); intel_engine_signal_breadcrumbs(ce->engine); return 0; @@ -870,15 +870,15 @@ static int guc_wq_item_append(struct intel_guc *guc, struct i915_request *rq) { struct intel_context *ce = request_to_scheduling_context(rq); - int ret = 0; + int ret; - if (likely(!intel_context_is_banned(ce))) { - ret = __guc_wq_item_append(rq); + if (unlikely(!intel_context_is_schedulable(ce))) + return 0; - if (unlikely(ret == -EBUSY)) { - guc->stalled_request = rq; - guc->submission_stall_reason = STALL_MOVE_LRC_TAIL; - } + ret = __guc_wq_item_append(rq); + if (unlikely(ret == -EBUSY)) { + guc->stalled_request = rq; + guc->submission_stall_reason = STALL_MOVE_LRC_TAIL; } return ret; @@ -897,7 +897,7 @@ static bool multi_lrc_submit(struct i915_request *rq) * submitting all the requests generated in parallel. */ return test_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, &rq->fence.flags) || - intel_context_is_banned(ce); + !intel_context_is_schedulable(ce); } static int guc_dequeue_one_context(struct intel_guc *guc) @@ -966,7 +966,7 @@ register_context: struct intel_context *ce = request_to_scheduling_context(last); if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) && - !intel_context_is_banned(ce))) { + intel_context_is_schedulable(ce))) { ret = try_context_registration(ce, false); if (unlikely(ret == -EPIPE)) { goto deadlk; @@ -1438,7 +1438,12 @@ void intel_guc_busyness_park(struct intel_gt *gt) if (!guc_submission_initialized(guc)) return; - cancel_delayed_work(&guc->timestamp.work); + /* + * There is a race with suspend flow where the worker runs after suspend + * and causes an unclaimed register access warning. Cancel the worker + * synchronously here. + */ + cancel_delayed_work_sync(&guc->timestamp.work); /* * Before parking, we should sample engine busyness stats if we need to. @@ -1532,8 +1537,8 @@ void intel_guc_submission_reset_prepare(struct intel_guc *guc) __reset_guc_busyness_stats(guc); /* Flush IRQ handler */ - spin_lock_irq(&guc_to_gt(guc)->irq_lock); - spin_unlock_irq(&guc_to_gt(guc)->irq_lock); + spin_lock_irq(guc_to_gt(guc)->irq_lock); + spin_unlock_irq(guc_to_gt(guc)->irq_lock); guc_flush_submissions(guc); guc_flush_destroyed_contexts(guc); @@ -1571,7 +1576,7 @@ static void guc_reset_state(struct intel_context *ce, u32 head, bool scrub) { struct intel_engine_cs *engine = __context_to_physical_engine(ce); - if (intel_context_is_banned(ce)) + if (!intel_context_is_schedulable(ce)) return; GEM_BUG_ON(!intel_context_is_pinned(ce)); @@ -4419,12 +4424,12 @@ static void guc_handle_context_reset(struct intel_guc *guc, { trace_intel_context_reset(ce); - if (likely(!intel_context_is_banned(ce))) { + if (likely(intel_context_is_schedulable(ce))) { capture_error_state(guc, ce); guc_context_replay(ce); } else { drm_info(&guc_to_gt(guc)->i915->drm, - "Ignoring context reset notification of banned context 0x%04X on %s", + "Ignoring context reset notification of exiting context 0x%04X on %s", ce->guc_id.id, ce->engine->name); } } diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index abf4e142596d055a44249bf6aba01dd12ed3426e..dbd048b77e193be9772d83701c717ed554461356 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -245,9 +245,9 @@ static int guc_enable_communication(struct intel_guc *guc) intel_guc_enable_interrupts(guc); /* check for CT messages received before we enabled interrupts */ - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); intel_guc_ct_event_handler(&guc->ct); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); drm_dbg(&i915->drm, "GuC communication enabled\n"); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index af425916cdf64cfb7715275aa779e1a10afc7294..b91ad4aede1f79b3b43869537fbd18f3de6d6716 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -72,12 +72,14 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * security fixes, etc. to be enabled. */ #define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_maj, guc_mmp) \ - fw_def(DG2, 0, guc_mmp(dg2, 70, 4, 1)) \ + fw_def(DG2, 0, guc_maj(dg2, 70, 5)) \ + fw_def(ALDERLAKE_P, 0, guc_maj(adlp, 70, 5)) \ fw_def(ALDERLAKE_P, 0, guc_mmp(adlp, 70, 1, 1)) \ fw_def(ALDERLAKE_P, 0, guc_mmp(adlp, 69, 0, 3)) \ + fw_def(ALDERLAKE_S, 0, guc_maj(tgl, 70, 5)) \ fw_def(ALDERLAKE_S, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(ALDERLAKE_S, 0, guc_mmp(tgl, 69, 0, 3)) \ - fw_def(DG1, 0, guc_mmp(dg1, 70, 1, 1)) \ + fw_def(DG1, 0, guc_maj(dg1, 70, 5)) \ fw_def(ROCKETLAKE, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(TIGERLAKE, 0, guc_mmp(tgl, 70, 1, 1)) \ fw_def(JASPERLAKE, 0, guc_mmp(ehl, 70, 1, 1)) \ @@ -92,9 +94,11 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, fw_def(SKYLAKE, 0, guc_mmp(skl, 70, 1, 1)) #define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_raw, huc_mmp) \ + fw_def(ALDERLAKE_P, 0, huc_raw(tgl)) \ fw_def(ALDERLAKE_P, 0, huc_mmp(tgl, 7, 9, 3)) \ + fw_def(ALDERLAKE_S, 0, huc_raw(tgl)) \ fw_def(ALDERLAKE_S, 0, huc_mmp(tgl, 7, 9, 3)) \ - fw_def(DG1, 0, huc_mmp(dg1, 7, 9, 3)) \ + fw_def(DG1, 0, huc_raw(dg1)) \ fw_def(ROCKETLAKE, 0, huc_mmp(tgl, 7, 9, 3)) \ fw_def(TIGERLAKE, 0, huc_mmp(tgl, 7, 9, 3)) \ fw_def(JASPERLAKE, 0, huc_mmp(ehl, 9, 0, 0)) \ @@ -232,6 +236,7 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) u32 fw_count; u8 rev = INTEL_REVID(i915); int i; + bool found; /* * The only difference between the ADL GuC FWs is the HWConfig support. @@ -246,6 +251,7 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) fw_blobs = blobs_all[uc_fw->type].blobs; fw_count = blobs_all[uc_fw->type].count; + found = false; for (i = 0; i < fw_count && p <= fw_blobs[i].p; i++) { const struct uc_fw_blob *blob = &fw_blobs[i].blob; @@ -266,9 +272,15 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) uc_fw->file_wanted.path = blob->path; uc_fw->file_wanted.major_ver = blob->major; uc_fw->file_wanted.minor_ver = blob->minor; + found = true; break; } + if (!found && uc_fw->file_selected.path) { + /* Failed to find a match for the last attempt?! */ + uc_fw->file_selected.path = NULL; + } + /* make sure the list is ordered as expected */ if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) && !verified) { verified = true; @@ -322,7 +334,7 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw) continue; bad: - drm_err(&i915->drm, "\x1B[35;1mInvalid FW blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n", + drm_err(&i915->drm, "Invalid FW blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n", intel_platform_name(fw_blobs[i - 1].p), fw_blobs[i - 1].rev, fw_blobs[i - 1].blob.legacy ? "L" : "v", fw_blobs[i - 1].blob.major, @@ -553,10 +565,14 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) err = firmware_request_nowarn(&fw, uc_fw->file_selected.path, dev); memcpy(&file_ideal, &uc_fw->file_wanted, sizeof(file_ideal)); - if (!err || intel_uc_fw_is_overridden(uc_fw)) - goto done; + + /* Any error is terminal if overriding. Don't bother searching for older versions */ + if (err && intel_uc_fw_is_overridden(uc_fw)) + goto fail; while (err == -ENOENT) { + old_ver = true; + __uc_fw_auto_select(i915, uc_fw); if (!uc_fw->file_selected.path) { /* @@ -576,8 +592,6 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) if (err) goto fail; - old_ver = true; -done: if (uc_fw->loaded_via_gsc) err = check_gsc_manifest(fw, uc_fw); else diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 557f3314291a8215aa1e541fe024d2053ad927c5..076c779f776a63293f5bd880da366f7c17ea285e 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -240,13 +240,13 @@ static void free_resource(struct intel_vgpu *vgpu) } static int alloc_resource(struct intel_vgpu *vgpu, - struct intel_vgpu_creation_params *param) + const struct intel_vgpu_config *conf) { struct intel_gvt *gvt = vgpu->gvt; unsigned long request, avail, max, taken; const char *item; - if (!param->low_gm_sz || !param->high_gm_sz || !param->fence_sz) { + if (!conf->low_mm || !conf->high_mm || !conf->fence) { gvt_vgpu_err("Invalid vGPU creation params\n"); return -EINVAL; } @@ -255,7 +255,7 @@ static int alloc_resource(struct intel_vgpu *vgpu, max = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; taken = gvt->gm.vgpu_allocated_low_gm_size; avail = max - taken; - request = MB_TO_BYTES(param->low_gm_sz); + request = conf->low_mm; if (request > avail) goto no_enough_resource; @@ -266,7 +266,7 @@ static int alloc_resource(struct intel_vgpu *vgpu, max = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; taken = gvt->gm.vgpu_allocated_high_gm_size; avail = max - taken; - request = MB_TO_BYTES(param->high_gm_sz); + request = conf->high_mm; if (request > avail) goto no_enough_resource; @@ -277,16 +277,16 @@ static int alloc_resource(struct intel_vgpu *vgpu, max = gvt_fence_sz(gvt) - HOST_FENCE; taken = gvt->fence.vgpu_allocated_fence_num; avail = max - taken; - request = param->fence_sz; + request = conf->fence; if (request > avail) goto no_enough_resource; vgpu_fence_sz(vgpu) = request; - gvt->gm.vgpu_allocated_low_gm_size += MB_TO_BYTES(param->low_gm_sz); - gvt->gm.vgpu_allocated_high_gm_size += MB_TO_BYTES(param->high_gm_sz); - gvt->fence.vgpu_allocated_fence_num += param->fence_sz; + gvt->gm.vgpu_allocated_low_gm_size += conf->low_mm; + gvt->gm.vgpu_allocated_high_gm_size += conf->high_mm; + gvt->fence.vgpu_allocated_fence_num += conf->fence; return 0; no_enough_resource: @@ -298,7 +298,7 @@ no_enough_resource: } /** - * inte_gvt_free_vgpu_resource - free HW resource owned by a vGPU + * intel_vgpu_free_resource() - free HW resource owned by a vGPU * @vgpu: a vGPU * * This function is used to free the HW resource owned by a vGPU. @@ -328,7 +328,7 @@ void intel_vgpu_reset_resource(struct intel_vgpu *vgpu) } /** - * intel_alloc_vgpu_resource - allocate HW resource for a vGPU + * intel_vgpu_alloc_resource() - allocate HW resource for a vGPU * @vgpu: vGPU * @param: vGPU creation params * @@ -340,11 +340,11 @@ void intel_vgpu_reset_resource(struct intel_vgpu *vgpu) * */ int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu, - struct intel_vgpu_creation_params *param) + const struct intel_vgpu_config *conf) { int ret; - ret = alloc_resource(vgpu, param); + ret = alloc_resource(vgpu, conf); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index dad3a60543354c06e2b9515435fa9039f6fe7e9c..eef3bba8a41bceeb7a425860b973ecb71e8f88b2 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -33,6 +33,7 @@ #include "i915_drv.h" #include "gvt.h" +#include "intel_pci_config.h" enum { INTEL_GVT_PCI_BAR_GTTMMIO = 0, @@ -353,9 +354,9 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4); vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size = - pci_resource_len(pdev, 0); + pci_resource_len(pdev, GTTMMADR_BAR); vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size = - pci_resource_len(pdev, 2); + pci_resource_len(pdev, GTT_APERTURE_BAR); memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4); diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c index a30ba2d7b7ba1ac64b2657d039fea1f5166f750f..1b509c1a1e330d4057fc4eb0de4489bc391aa024 100644 --- a/drivers/gpu/drm/i915/gvt/edid.c +++ b/drivers/gpu/drm/i915/gvt/edid.c @@ -32,9 +32,10 @@ * */ +#include "display/intel_gmbus_regs.h" +#include "gvt.h" #include "i915_drv.h" #include "i915_reg.h" -#include "gvt.h" #define GMBUS1_TOTAL_BYTES_SHIFT 16 #define GMBUS1_TOTAL_BYTES_MASK 0x1ff diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index b4f69364f9a1392ea27dc5f5267075a0fa5e0497..ce0eb03709c3f6fa7ef30fa8a6f2049d0db41159 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -2341,7 +2341,7 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, gvt_vgpu_err("fail to populate guest ggtt entry\n"); /* guest driver may read/write the entry when partial * update the entry in this situation p2m will fail - * settting the shadow entry to point to a scratch page + * setting the shadow entry to point to a scratch page */ ops->set_pfn(&m, gvt->gtt.scratch_mfn); } else diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 705689e6401197c6870cb7b9920d15e13ecddb57..dbf8d7470b2c1d7d2ec0fd49d63b17f7dfa80bbe 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -36,6 +36,7 @@ #include #include #include +#include #include "i915_drv.h" #include "intel_gvt.h" @@ -172,6 +173,7 @@ struct intel_vgpu_submission { #define KVMGT_DEBUGFS_FILENAME "kvmgt_nr_cache_entries" struct intel_vgpu { + struct vfio_device vfio_device; struct intel_gvt *gvt; struct mutex vgpu_lock; int id; @@ -211,7 +213,6 @@ struct intel_vgpu { u32 scan_nonprivbb; - struct vfio_device vfio_device; struct vfio_region *region; int num_regions; struct eventfd_ctx *intx_trigger; @@ -294,15 +295,25 @@ struct intel_gvt_firmware { bool firmware_loaded; }; -#define NR_MAX_INTEL_VGPU_TYPES 20 -struct intel_vgpu_type { - char name[16]; - unsigned int avail_instance; - unsigned int low_gm_size; - unsigned int high_gm_size; +struct intel_vgpu_config { + unsigned int low_mm; + unsigned int high_mm; unsigned int fence; + + /* + * A vGPU with a weight of 8 will get twice as much GPU as a vGPU with + * a weight of 4 on a contended host, different vGPU type has different + * weight set. Legal weights range from 1 to 16. + */ unsigned int weight; - enum intel_vgpu_edid resolution; + enum intel_vgpu_edid edid; + const char *name; +}; + +struct intel_vgpu_type { + struct mdev_type type; + char name[16]; + const struct intel_vgpu_config *conf; }; struct intel_gvt { @@ -326,6 +337,8 @@ struct intel_gvt { struct intel_gvt_workload_scheduler scheduler; struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES]; DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS); + struct mdev_parent parent; + struct mdev_type **mdev_types; struct intel_vgpu_type *types; unsigned int num_types; struct intel_vgpu *idle_vgpu; @@ -436,19 +449,8 @@ int intel_gvt_load_firmware(struct intel_gvt *gvt); /* ring context size i.e. the first 0x50 dwords*/ #define RING_CTX_SIZE 320 -struct intel_vgpu_creation_params { - __u64 low_gm_sz; /* in MB */ - __u64 high_gm_sz; /* in MB */ - __u64 fence_sz; - __u64 resolution; - __s32 primary; - __u64 vgpu_id; - - __u32 weight; -}; - int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu, - struct intel_vgpu_creation_params *param); + const struct intel_vgpu_config *conf); void intel_vgpu_reset_resource(struct intel_vgpu *vgpu); void intel_vgpu_free_resource(struct intel_vgpu *vgpu); void intel_vgpu_write_fence(struct intel_vgpu *vgpu, @@ -494,8 +496,8 @@ void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt); struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt); void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu); -struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, - struct intel_vgpu_type *type); +int intel_gvt_create_vgpu(struct intel_vgpu *vgpu, + const struct intel_vgpu_config *conf); void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu); void intel_gvt_release_vgpu(struct intel_vgpu *vgpu); void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index beea5895e499211c91de24684e3b511c4e7a0765..daac2050d77d0440bca30c83ca2d6399f65fc89c 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -498,7 +498,7 @@ static u32 bdw_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port) switch (wrpll_ctl & WRPLL_REF_MASK) { case WRPLL_REF_PCH_SSC: - refclk = vgpu->gvt->gt->i915->dpll.ref_clks.ssc; + refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.ssc; break; case WRPLL_REF_LCPLL: refclk = 2700000; @@ -529,7 +529,7 @@ out: static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port) { u32 dp_br = 0; - int refclk = vgpu->gvt->gt->i915->dpll.ref_clks.nssc; + int refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.nssc; enum dpio_phy phy = DPIO_PHY0; enum dpio_channel ch = DPIO_CH0; struct dpll clock = {0}; @@ -905,7 +905,7 @@ static int update_fdi_rx_iir_status(struct intel_vgpu *vgpu, else if (FDI_RX_IMR_TO_PIPE(offset) != INVALID_INDEX) index = FDI_RX_IMR_TO_PIPE(offset); else { - gvt_vgpu_err("Unsupport registers %x\n", offset); + gvt_vgpu_err("Unsupported registers %x\n", offset); return -EINVAL; } @@ -3052,7 +3052,7 @@ int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, } /** - * intel_t_default_mmio_write - default MMIO write handler + * intel_vgpu_default_mmio_write() - default MMIO write handler * @vgpu: a vGPU * @offset: access offset * @p_data: write data buffer diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index e3cd5894647777b008629184dd41c24cb822ee4d..7a45e5360caf2df17e8be371dcb9d5e2adecc74c 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -34,7 +34,6 @@ */ #include -#include #include #include #include @@ -43,7 +42,6 @@ #include #include #include -#include #include #include @@ -115,117 +113,18 @@ static void kvmgt_page_track_flush_slot(struct kvm *kvm, struct kvm_memory_slot *slot, struct kvm_page_track_notifier_node *node); -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) +static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf) { - struct intel_vgpu_type *type; - unsigned int num = 0; - struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = &gvt->types[mtype_get_type_group_id(mtype)]; - if (!type) - num = 0; - else - num = type->avail_instance; - - return sprintf(buf, "%u\n", num); -} - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} - -static ssize_t description_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - struct intel_vgpu_type *type; - struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = &gvt->types[mtype_get_type_group_id(mtype)]; - if (!type) - return 0; + struct intel_vgpu_type *type = + container_of(mtype, struct intel_vgpu_type, type); return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" "fence: %d\nresolution: %s\n" "weight: %d\n", - BYTES_TO_MB(type->low_gm_size), - BYTES_TO_MB(type->high_gm_size), - type->fence, vgpu_edid_str(type->resolution), - type->weight); -} - -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - struct intel_vgpu_type *type; - struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = &gvt->types[mtype_get_type_group_id(mtype)]; - if (!type) - return 0; - - return sprintf(buf, "%s\n", type->name); -} - -static MDEV_TYPE_ATTR_RO(available_instances); -static MDEV_TYPE_ATTR_RO(device_api); -static MDEV_TYPE_ATTR_RO(description); -static MDEV_TYPE_ATTR_RO(name); - -static struct attribute *gvt_type_attrs[] = { - &mdev_type_attr_available_instances.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_description.attr, - &mdev_type_attr_name.attr, - NULL, -}; - -static struct attribute_group *gvt_vgpu_type_groups[] = { - [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, -}; - -static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i, j; - struct intel_vgpu_type *type; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - type = &gvt->types[i]; - - group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); - if (!group) - goto unwind; - - group->name = type->name; - group->attrs = gvt_type_attrs; - gvt_vgpu_type_groups[i] = group; - } - - return 0; - -unwind: - for (j = 0; j < i; j++) { - group = gvt_vgpu_type_groups[j]; - kfree(group); - } - - return -ENOMEM; -} - -static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - group = gvt_vgpu_type_groups[i]; - gvt_vgpu_type_groups[i] = NULL; - kfree(group); - } + BYTES_TO_MB(type->conf->low_mm), + BYTES_TO_MB(type->conf->high_mm), + type->conf->fence, vgpu_edid_str(type->conf->edid), + type->conf->weight); } static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn, @@ -1546,7 +1445,28 @@ static const struct attribute_group *intel_vgpu_groups[] = { NULL, }; +static int intel_vgpu_init_dev(struct vfio_device *vfio_dev) +{ + struct mdev_device *mdev = to_mdev_device(vfio_dev->dev); + struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); + struct intel_vgpu_type *type = + container_of(mdev->type, struct intel_vgpu_type, type); + + vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt; + return intel_gvt_create_vgpu(vgpu, type->conf); +} + +static void intel_vgpu_release_dev(struct vfio_device *vfio_dev) +{ + struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev); + + intel_gvt_destroy_vgpu(vgpu); + vfio_free_device(vfio_dev); +} + static const struct vfio_device_ops intel_vgpu_dev_ops = { + .init = intel_vgpu_init_dev, + .release = intel_vgpu_release_dev, .open_device = intel_vgpu_open_device, .close_device = intel_vgpu_close_device, .read = intel_vgpu_read, @@ -1558,35 +1478,28 @@ static const struct vfio_device_ops intel_vgpu_dev_ops = { static int intel_vgpu_probe(struct mdev_device *mdev) { - struct device *pdev = mdev_parent_dev(mdev); - struct intel_gvt *gvt = kdev_to_i915(pdev)->gvt; - struct intel_vgpu_type *type; struct intel_vgpu *vgpu; int ret; - type = &gvt->types[mdev_get_type_group_id(mdev)]; - if (!type) - return -EINVAL; - - vgpu = intel_gvt_create_vgpu(gvt, type); + vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev, + &intel_vgpu_dev_ops); if (IS_ERR(vgpu)) { gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu)); return PTR_ERR(vgpu); } - vfio_init_group_dev(&vgpu->vfio_device, &mdev->dev, - &intel_vgpu_dev_ops); - dev_set_drvdata(&mdev->dev, vgpu); ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device); - if (ret) { - intel_gvt_destroy_vgpu(vgpu); - return ret; - } + if (ret) + goto out_put_vdev; gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n", dev_name(mdev_dev(mdev))); return 0; + +out_put_vdev: + vfio_put_device(&vgpu->vfio_device); + return ret; } static void intel_vgpu_remove(struct mdev_device *mdev) @@ -1595,18 +1508,43 @@ static void intel_vgpu_remove(struct mdev_device *mdev) if (WARN_ON_ONCE(vgpu->attached)) return; - intel_gvt_destroy_vgpu(vgpu); + + vfio_unregister_group_dev(&vgpu->vfio_device); + vfio_put_device(&vgpu->vfio_device); +} + +static unsigned int intel_vgpu_get_available(struct mdev_type *mtype) +{ + struct intel_vgpu_type *type = + container_of(mtype, struct intel_vgpu_type, type); + struct intel_gvt *gvt = kdev_to_i915(mtype->parent->dev)->gvt; + unsigned int low_gm_avail, high_gm_avail, fence_avail; + + mutex_lock(&gvt->lock); + low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE - + gvt->gm.vgpu_allocated_low_gm_size; + high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE - + gvt->gm.vgpu_allocated_high_gm_size; + fence_avail = gvt_fence_sz(gvt) - HOST_FENCE - + gvt->fence.vgpu_allocated_fence_num; + mutex_unlock(&gvt->lock); + + return min3(low_gm_avail / type->conf->low_mm, + high_gm_avail / type->conf->high_mm, + fence_avail / type->conf->fence); } static struct mdev_driver intel_vgpu_mdev_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, .driver = { .name = "intel_vgpu_mdev", .owner = THIS_MODULE, .dev_groups = intel_vgpu_groups, }, - .probe = intel_vgpu_probe, - .remove = intel_vgpu_remove, - .supported_type_groups = gvt_vgpu_type_groups, + .probe = intel_vgpu_probe, + .remove = intel_vgpu_remove, + .get_available = intel_vgpu_get_available, + .show_description = intel_vgpu_show_description, }; int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn) @@ -1904,8 +1842,7 @@ static void intel_gvt_clean_device(struct drm_i915_private *i915) if (drm_WARN_ON(&i915->drm, !gvt)) return; - mdev_unregister_device(i915->drm.dev); - intel_gvt_cleanup_vgpu_type_groups(gvt); + mdev_unregister_parent(&gvt->parent); intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); intel_gvt_clean_vgpu_types(gvt); @@ -2005,19 +1942,15 @@ static int intel_gvt_init_device(struct drm_i915_private *i915) intel_gvt_debugfs_init(gvt); - ret = intel_gvt_init_vgpu_type_groups(gvt); + ret = mdev_register_parent(&gvt->parent, i915->drm.dev, + &intel_vgpu_mdev_driver, + gvt->mdev_types, gvt->num_types); if (ret) goto out_destroy_idle_vgpu; - ret = mdev_register_device(i915->drm.dev, &intel_vgpu_mdev_driver); - if (ret) - goto out_cleanup_vgpu_type_groups; - gvt_dbg_core("gvt device initialization is done\n"); return 0; -out_cleanup_vgpu_type_groups: - intel_gvt_cleanup_vgpu_type_groups(gvt); out_destroy_idle_vgpu: intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); intel_gvt_debugfs_clean(gvt); diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index c85bafe7539ee97bc559f42d7035bd8e744fe31a..1c6e941c96666c76b3692e8d378113f273b3f0e1 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -546,7 +546,7 @@ static void switch_mmio(struct intel_vgpu *pre, } /** - * intel_gvt_switch_render_mmio - switch mmio context of specific engine + * intel_gvt_switch_mmio - switch mmio context of specific engine * @pre: the last vGPU that own the engine * @next: the vGPU to switch to * @engine: the engine diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 46da19b3225d2245eb84b7cd3c19bdfdc0d8b297..56c71474008a3780c27af3016d72e96c76255932 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -73,24 +73,21 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu) drm_WARN_ON(&i915->drm, sizeof(struct vgt_if) != VGT_PVINFO_SIZE); } +/* + * vGPU type name is defined as GVTg_Vx_y which contains the physical GPU + * generation type (e.g V4 as BDW server, V5 as SKL server). + * + * Depening on the physical SKU resource, we might see vGPU types like + * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create different types of + * vGPU on same physical GPU depending on available resource. Each vGPU + * type will have a different number of avail_instance to indicate how + * many vGPU instance can be created for this type. + */ #define VGPU_MAX_WEIGHT 16 #define VGPU_WEIGHT(vgpu_num) \ (VGPU_MAX_WEIGHT / (vgpu_num)) -static const struct { - unsigned int low_mm; - unsigned int high_mm; - unsigned int fence; - - /* A vGPU with a weight of 8 will get twice as much GPU as a vGPU - * with a weight of 4 on a contended host, different vGPU type has - * different weight set. Legal weights range from 1 to 16. - */ - unsigned int weight; - enum intel_vgpu_edid edid; - const char *name; -} vgpu_types[] = { -/* Fixed vGPU type table */ +static const struct intel_vgpu_config intel_vgpu_configs[] = { { MB_TO_BYTES(64), MB_TO_BYTES(384), 4, VGPU_WEIGHT(8), GVT_EDID_1024_768, "8" }, { MB_TO_BYTES(128), MB_TO_BYTES(512), 4, VGPU_WEIGHT(4), GVT_EDID_1920_1200, "4" }, { MB_TO_BYTES(256), MB_TO_BYTES(1024), 4, VGPU_WEIGHT(2), GVT_EDID_1920_1200, "2" }, @@ -106,102 +103,58 @@ static const struct { */ int intel_gvt_init_vgpu_types(struct intel_gvt *gvt) { - unsigned int num_types; - unsigned int i, low_avail, high_avail; - unsigned int min_low; - - /* vGPU type name is defined as GVTg_Vx_y which contains - * physical GPU generation type (e.g V4 as BDW server, V5 as - * SKL server). - * - * Depend on physical SKU resource, might see vGPU types like - * GVTg_V4_8, GVTg_V4_4, GVTg_V4_2, etc. We can create - * different types of vGPU on same physical GPU depending on - * available resource. Each vGPU type will have "avail_instance" - * to indicate how many vGPU instance can be created for this - * type. - * - */ - low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; - high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; - num_types = ARRAY_SIZE(vgpu_types); + unsigned int low_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE; + unsigned int high_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE; + unsigned int num_types = ARRAY_SIZE(intel_vgpu_configs); + unsigned int i; gvt->types = kcalloc(num_types, sizeof(struct intel_vgpu_type), GFP_KERNEL); if (!gvt->types) return -ENOMEM; - min_low = MB_TO_BYTES(32); - for (i = 0; i < num_types; ++i) { - if (low_avail / vgpu_types[i].low_mm == 0) - break; - - gvt->types[i].low_gm_size = vgpu_types[i].low_mm; - gvt->types[i].high_gm_size = vgpu_types[i].high_mm; - gvt->types[i].fence = vgpu_types[i].fence; + gvt->mdev_types = kcalloc(num_types, sizeof(*gvt->mdev_types), + GFP_KERNEL); + if (!gvt->mdev_types) + goto out_free_types; - if (vgpu_types[i].weight < 1 || - vgpu_types[i].weight > VGPU_MAX_WEIGHT) - return -EINVAL; + for (i = 0; i < num_types; ++i) { + const struct intel_vgpu_config *conf = &intel_vgpu_configs[i]; - gvt->types[i].weight = vgpu_types[i].weight; - gvt->types[i].resolution = vgpu_types[i].edid; - gvt->types[i].avail_instance = min(low_avail / vgpu_types[i].low_mm, - high_avail / vgpu_types[i].high_mm); + if (low_avail / conf->low_mm == 0) + break; + if (conf->weight < 1 || conf->weight > VGPU_MAX_WEIGHT) + goto out_free_mdev_types; - if (GRAPHICS_VER(gvt->gt->i915) == 8) - sprintf(gvt->types[i].name, "GVTg_V4_%s", - vgpu_types[i].name); - else if (GRAPHICS_VER(gvt->gt->i915) == 9) - sprintf(gvt->types[i].name, "GVTg_V5_%s", - vgpu_types[i].name); + sprintf(gvt->types[i].name, "GVTg_V%u_%s", + GRAPHICS_VER(gvt->gt->i915) == 8 ? 4 : 5, conf->name); + gvt->types[i].conf = conf; gvt_dbg_core("type[%d]: %s avail %u low %u high %u fence %u weight %u res %s\n", i, gvt->types[i].name, - gvt->types[i].avail_instance, - gvt->types[i].low_gm_size, - gvt->types[i].high_gm_size, gvt->types[i].fence, - gvt->types[i].weight, - vgpu_edid_str(gvt->types[i].resolution)); + min(low_avail / conf->low_mm, + high_avail / conf->high_mm), + conf->low_mm, conf->high_mm, conf->fence, + conf->weight, vgpu_edid_str(conf->edid)); + + gvt->mdev_types[i] = &gvt->types[i].type; + gvt->mdev_types[i]->sysfs_name = gvt->types[i].name; } gvt->num_types = i; return 0; -} -void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt) -{ +out_free_mdev_types: + kfree(gvt->mdev_types); +out_free_types: kfree(gvt->types); + return -EINVAL; } -static void intel_gvt_update_vgpu_types(struct intel_gvt *gvt) +void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt) { - int i; - unsigned int low_gm_avail, high_gm_avail, fence_avail; - unsigned int low_gm_min, high_gm_min, fence_min; - - /* Need to depend on maxium hw resource size but keep on - * static config for now. - */ - low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE - - gvt->gm.vgpu_allocated_low_gm_size; - high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE - - gvt->gm.vgpu_allocated_high_gm_size; - fence_avail = gvt_fence_sz(gvt) - HOST_FENCE - - gvt->fence.vgpu_allocated_fence_num; - - for (i = 0; i < gvt->num_types; i++) { - low_gm_min = low_gm_avail / gvt->types[i].low_gm_size; - high_gm_min = high_gm_avail / gvt->types[i].high_gm_size; - fence_min = fence_avail / gvt->types[i].fence; - gvt->types[i].avail_instance = min(min(low_gm_min, high_gm_min), - fence_min); - - gvt_dbg_core("update type[%d]: %s avail %u low %u high %u fence %u\n", - i, gvt->types[i].name, - gvt->types[i].avail_instance, gvt->types[i].low_gm_size, - gvt->types[i].high_gm_size, gvt->types[i].fence); - } + kfree(gvt->mdev_types); + kfree(gvt->types); } /** @@ -298,12 +251,6 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) intel_vgpu_clean_mmio(vgpu); intel_vgpu_dmabuf_cleanup(vgpu); mutex_unlock(&vgpu->vgpu_lock); - - mutex_lock(&gvt->lock); - intel_gvt_update_vgpu_types(gvt); - mutex_unlock(&gvt->lock); - - vfree(vgpu); } #define IDLE_VGPU_IDR 0 @@ -363,42 +310,38 @@ void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu) vfree(vgpu); } -static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, - struct intel_vgpu_creation_params *param) +int intel_gvt_create_vgpu(struct intel_vgpu *vgpu, + const struct intel_vgpu_config *conf) { + struct intel_gvt *gvt = vgpu->gvt; struct drm_i915_private *dev_priv = gvt->gt->i915; - struct intel_vgpu *vgpu; int ret; - gvt_dbg_core("low %llu MB high %llu MB fence %llu\n", - param->low_gm_sz, param->high_gm_sz, - param->fence_sz); - - vgpu = vzalloc(sizeof(*vgpu)); - if (!vgpu) - return ERR_PTR(-ENOMEM); + gvt_dbg_core("low %u MB high %u MB fence %u\n", + BYTES_TO_MB(conf->low_mm), BYTES_TO_MB(conf->high_mm), + conf->fence); + mutex_lock(&gvt->lock); ret = idr_alloc(&gvt->vgpu_idr, vgpu, IDLE_VGPU_IDR + 1, GVT_MAX_VGPU, GFP_KERNEL); if (ret < 0) - goto out_free_vgpu; + goto out_unlock;; vgpu->id = ret; - vgpu->gvt = gvt; - vgpu->sched_ctl.weight = param->weight; + vgpu->sched_ctl.weight = conf->weight; mutex_init(&vgpu->vgpu_lock); mutex_init(&vgpu->dmabuf_lock); INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); idr_init_base(&vgpu->object_idr, 1); - intel_vgpu_init_cfg_space(vgpu, param->primary); + intel_vgpu_init_cfg_space(vgpu, 1); vgpu->d3_entered = false; ret = intel_vgpu_init_mmio(vgpu); if (ret) goto out_clean_idr; - ret = intel_vgpu_alloc_resource(vgpu, param); + ret = intel_vgpu_alloc_resource(vgpu, conf); if (ret) goto out_clean_vgpu_mmio; @@ -412,7 +355,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, if (ret) goto out_clean_gtt; - ret = intel_vgpu_init_display(vgpu, param->resolution); + ret = intel_vgpu_init_display(vgpu, conf->edid); if (ret) goto out_clean_opregion; @@ -437,7 +380,9 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, if (ret) goto out_clean_sched_policy; - return vgpu; + intel_gvt_update_reg_whitelist(vgpu); + mutex_unlock(&gvt->lock); + return 0; out_clean_sched_policy: intel_vgpu_clean_sched_policy(vgpu); @@ -455,48 +400,9 @@ out_clean_vgpu_mmio: intel_vgpu_clean_mmio(vgpu); out_clean_idr: idr_remove(&gvt->vgpu_idr, vgpu->id); -out_free_vgpu: - vfree(vgpu); - return ERR_PTR(ret); -} - -/** - * intel_gvt_create_vgpu - create a virtual GPU - * @gvt: GVT device - * @type: type of the vGPU to create - * - * This function is called when user wants to create a virtual GPU. - * - * Returns: - * pointer to intel_vgpu, error pointer if failed. - */ -struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, - struct intel_vgpu_type *type) -{ - struct intel_vgpu_creation_params param; - struct intel_vgpu *vgpu; - - param.primary = 1; - param.low_gm_sz = type->low_gm_size; - param.high_gm_sz = type->high_gm_size; - param.fence_sz = type->fence; - param.weight = type->weight; - param.resolution = type->resolution; - - /* XXX current param based on MB */ - param.low_gm_sz = BYTES_TO_MB(param.low_gm_sz); - param.high_gm_sz = BYTES_TO_MB(param.high_gm_sz); - - mutex_lock(&gvt->lock); - vgpu = __intel_gvt_create_vgpu(gvt, ¶m); - if (!IS_ERR(vgpu)) { - /* calculate left instance change for types */ - intel_gvt_update_vgpu_types(gvt); - intel_gvt_update_reg_whitelist(vgpu); - } +out_unlock: mutex_unlock(&gvt->lock); - - return vgpu; + return ret; } /** diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 3a8450058548c6e125f5aca759c63b9c8e8d8e46..ae987e92251ddc30b3f31a644a865702ebe02303 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -66,8 +66,7 @@ static int i915_capabilities(struct seq_file *m, void *data) seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(i915)); - intel_device_info_print_static(INTEL_INFO(i915), &p); - intel_device_info_print_runtime(RUNTIME_INFO(i915), &p); + intel_device_info_print(INTEL_INFO(i915), RUNTIME_INFO(i915), &p); i915_print_iommu_status(i915, &p); intel_gt_info_print(&to_gt(i915)->info, &p); intel_driver_caps_print(&i915->caps, &p); @@ -411,7 +410,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_printf(m, "bit6 swizzle for Y-tiling = %s\n", swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_y)); - if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) + if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) seq_puts(m, "L-shaped memory detected\n"); /* On BDW+, swizzling is not used. See detect_bit_6_swizzle() */ @@ -493,7 +492,7 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused) seq_puts(m, "Runtime power management not supported\n"); seq_printf(m, "Runtime power status: %s\n", - str_enabled_disabled(!dev_priv->power_domains.init_wakeref)); + str_enabled_disabled(!dev_priv->display.power.domains.init_wakeref)); seq_printf(m, "GPU idle: %s\n", str_yes_no(!to_gt(dev_priv)->awake)); seq_printf(m, "IRQs disabled: %s\n", diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c index deb8a8b76965a19bbd8fba1f24bd30e9f198d524..c459eb362c47f7e505c920d6736ee361566f46ed 100644 --- a/drivers/gpu/drm/i915/i915_driver.c +++ b/drivers/gpu/drm/i915/i915_driver.c @@ -61,6 +61,7 @@ #include "display/intel_pps.h" #include "display/intel_sprite.h" #include "display/intel_vga.h" +#include "display/skl_watermark.h" #include "gem/i915_gem_context.h" #include "gem/i915_gem_create.h" @@ -105,6 +106,12 @@ static const char irst_name[] = "INT3392"; static const struct drm_driver i915_drm_driver; +static void i915_release_bridge_dev(struct drm_device *dev, + void *bridge) +{ + pci_dev_put(bridge); +} + static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) { int domain = pci_domain_nr(to_pci_dev(dev_priv->drm.dev)->bus); @@ -115,7 +122,9 @@ static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) drm_err(&dev_priv->drm, "bridge device not found\n"); return -EIO; } - return 0; + + return drmm_add_action_or_reset(&dev_priv->drm, i915_release_bridge_dev, + dev_priv->bridge_dev); } /* Allocate space for the MCH regs if needed, return nonzero on error */ @@ -252,8 +261,8 @@ static int i915_workqueues_init(struct drm_i915_private *dev_priv) if (dev_priv->wq == NULL) goto out_err; - dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); - if (dev_priv->hotplug.dp_wq == NULL) + dev_priv->display.hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0); + if (dev_priv->display.hotplug.dp_wq == NULL) goto out_free_wq; return 0; @@ -268,7 +277,7 @@ out_err: static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv) { - destroy_workqueue(dev_priv->hotplug.dp_wq); + destroy_workqueue(dev_priv->display.hotplug.dp_wq); destroy_workqueue(dev_priv->wq); } @@ -302,8 +311,13 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) static void sanitize_gpu(struct drm_i915_private *i915) { - if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) - __intel_gt_reset(to_gt(i915), ALL_ENGINES); + if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) { + struct intel_gt *gt; + unsigned int i; + + for_each_gt(gt, i915, i) + __intel_gt_reset(gt, ALL_ENGINES); + } } /** @@ -326,19 +340,19 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) intel_device_info_subplatform_init(dev_priv); intel_step_init(dev_priv); - intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug); + intel_uncore_mmio_debug_init_early(dev_priv); spin_lock_init(&dev_priv->irq_lock); spin_lock_init(&dev_priv->gpu_error.lock); - mutex_init(&dev_priv->backlight_lock); + mutex_init(&dev_priv->display.backlight.lock); mutex_init(&dev_priv->sb_lock); cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE); - mutex_init(&dev_priv->audio.mutex); - mutex_init(&dev_priv->wm.wm_mutex); - mutex_init(&dev_priv->pps_mutex); - mutex_init(&dev_priv->hdcp_comp_mutex); + mutex_init(&dev_priv->display.audio.mutex); + mutex_init(&dev_priv->display.wm.wm_mutex); + mutex_init(&dev_priv->display.pps.mutex); + mutex_init(&dev_priv->display.hdcp.comp_mutex); i915_memcpy_init_early(dev_priv); intel_runtime_pm_init_early(&dev_priv->runtime_pm); @@ -357,7 +371,9 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) intel_wopcm_init_early(&dev_priv->wopcm); - intel_root_gt_init_early(dev_priv); + ret = intel_root_gt_init_early(dev_priv); + if (ret < 0) + goto err_rootgt; i915_drm_clients_init(&dev_priv->clients, dev_priv); @@ -382,6 +398,7 @@ err_gem: i915_gem_cleanup_early(dev_priv); intel_gt_driver_late_release_all(dev_priv); i915_drm_clients_fini(&dev_priv->clients); +err_rootgt: intel_region_ttm_device_fini(dev_priv); err_ttm: vlv_suspend_cleanup(dev_priv); @@ -423,7 +440,8 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv) */ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) { - int ret; + struct intel_gt *gt; + int ret, i; if (i915_inject_probe_failure(dev_priv)) return -ENODEV; @@ -432,17 +450,27 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) if (ret < 0) return ret; - ret = intel_uncore_init_mmio(&dev_priv->uncore); - if (ret) - return ret; + for_each_gt(gt, dev_priv, i) { + ret = intel_uncore_init_mmio(gt->uncore); + if (ret) + return ret; + + ret = drmm_add_action_or_reset(&dev_priv->drm, + intel_uncore_fini_mmio, + gt->uncore); + if (ret) + return ret; + } /* Try to make sure MCHBAR is enabled before poking at it */ intel_setup_mchbar(dev_priv); intel_device_info_runtime_init(dev_priv); - ret = intel_gt_init_mmio(to_gt(dev_priv)); - if (ret) - goto err_uncore; + for_each_gt(gt, dev_priv, i) { + ret = intel_gt_init_mmio(gt); + if (ret) + goto err_uncore; + } /* As early as possible, scrub existing GPU state before clobbering */ sanitize_gpu(dev_priv); @@ -451,8 +479,6 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv) err_uncore: intel_teardown_mchbar(dev_priv); - intel_uncore_fini_mmio(&dev_priv->uncore); - pci_dev_put(dev_priv->bridge_dev); return ret; } @@ -464,8 +490,6 @@ err_uncore: static void i915_driver_mmio_release(struct drm_i915_private *dev_priv) { intel_teardown_mchbar(dev_priv); - intel_uncore_fini_mmio(&dev_priv->uncore); - pci_dev_put(dev_priv->bridge_dev); } /** @@ -715,6 +739,8 @@ static void i915_driver_hw_remove(struct drm_i915_private *dev_priv) static void i915_driver_register(struct drm_i915_private *dev_priv) { struct drm_device *dev = &dev_priv->drm; + struct intel_gt *gt; + unsigned int i; i915_gem_driver_register(dev_priv); i915_pmu_register(dev_priv); @@ -734,7 +760,8 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) /* Depends on sysfs having been initialized */ i915_perf_register(dev_priv); - intel_gt_driver_register(to_gt(dev_priv)); + for_each_gt(gt, dev_priv, i) + intel_gt_driver_register(gt); intel_display_driver_register(dev_priv); @@ -753,6 +780,9 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) */ static void i915_driver_unregister(struct drm_i915_private *dev_priv) { + struct intel_gt *gt; + unsigned int i; + i915_switcheroo_unregister(dev_priv); intel_unregister_dsm_handler(); @@ -762,7 +792,8 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) intel_display_driver_unregister(dev_priv); - intel_gt_driver_unregister(to_gt(dev_priv)); + for_each_gt(gt, dev_priv, i) + intel_gt_driver_unregister(gt); i915_perf_unregister(dev_priv); i915_pmu_unregister(dev_priv); @@ -784,6 +815,8 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv) { if (drm_debug_enabled(DRM_UT_DRIVER)) { struct drm_printer p = drm_debug_printer("i915 device info:"); + struct intel_gt *gt; + unsigned int i; drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n", INTEL_DEVID(dev_priv), @@ -793,10 +826,11 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv) INTEL_INFO(dev_priv)->platform), GRAPHICS_VER(dev_priv)); - intel_device_info_print_static(INTEL_INFO(dev_priv), &p); - intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p); + intel_device_info_print(INTEL_INFO(dev_priv), + RUNTIME_INFO(dev_priv), &p); i915_print_iommu_status(dev_priv, &p); - intel_gt_info_print(&to_gt(dev_priv)->info, &p); + for_each_gt(gt, dev_priv, i) + intel_gt_info_print(>->info, &p); } if (IS_ENABLED(CONFIG_DRM_I915_DEBUG)) @@ -814,6 +848,7 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) const struct intel_device_info *match_info = (struct intel_device_info *)ent->driver_data; struct intel_device_info *device_info; + struct intel_runtime_info *runtime; struct drm_i915_private *i915; i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver, @@ -829,7 +864,11 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) /* Setup the write-once "constant" device info */ device_info = mkwrite_device_info(i915); memcpy(device_info, match_info, sizeof(*device_info)); - RUNTIME_INFO(i915)->device_id = pdev->device; + + /* Initialize initial runtime info from static const data and pdev. */ + runtime = RUNTIME_INFO(i915); + memcpy(runtime, &INTEL_INFO(i915)->__runtime, sizeof(*runtime)); + runtime->device_id = pdev->device; return i915; } @@ -948,7 +987,9 @@ out_fini: void i915_driver_remove(struct drm_i915_private *i915) { - disable_rpm_wakeref_asserts(&i915->runtime_pm); + intel_wakeref_t wakeref; + + wakeref = intel_runtime_pm_get(&i915->runtime_pm); i915_driver_unregister(i915); @@ -972,18 +1013,19 @@ void i915_driver_remove(struct drm_i915_private *i915) i915_driver_hw_remove(i915); - enable_rpm_wakeref_asserts(&i915->runtime_pm); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); } static void i915_driver_release(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; + intel_wakeref_t wakeref; if (!dev_priv->do_release) return; - disable_rpm_wakeref_asserts(rpm); + wakeref = intel_runtime_pm_get(rpm); i915_gem_driver_release(dev_priv); @@ -994,7 +1036,8 @@ static void i915_driver_release(struct drm_device *dev) i915_driver_mmio_release(dev_priv); - enable_rpm_wakeref_asserts(rpm); + intel_runtime_pm_put(rpm, wakeref); + intel_runtime_pm_driver_release(rpm); i915_driver_late_release(dev_priv); @@ -1206,13 +1249,15 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret; + struct intel_gt *gt; + int ret, i; disable_rpm_wakeref_asserts(rpm); i915_gem_suspend_late(dev_priv); - intel_uncore_suspend(&dev_priv->uncore); + for_each_gt(gt, dev_priv, i) + intel_uncore_suspend(gt->uncore); intel_power_domains_suspend(dev_priv, get_suspend_mode(dev_priv, hibernation)); @@ -1344,7 +1389,8 @@ static int i915_drm_resume_early(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); - int ret; + struct intel_gt *gt; + int ret, i; /* * We have a resume ordering issue with the snd-hda driver also @@ -1398,9 +1444,10 @@ static int i915_drm_resume_early(struct drm_device *dev) drm_err(&dev_priv->drm, "Resume prepare failed: %d, continuing anyway\n", ret); - intel_uncore_resume_early(&dev_priv->uncore); - - intel_gt_check_and_clear_faults(to_gt(dev_priv)); + for_each_gt(gt, dev_priv, i) { + intel_uncore_resume_early(gt->uncore); + intel_gt_check_and_clear_faults(gt); + } intel_display_power_resume_early(dev_priv); @@ -1580,7 +1627,8 @@ static int intel_runtime_suspend(struct device *kdev) { struct drm_i915_private *dev_priv = kdev_to_i915(kdev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret; + struct intel_gt *gt; + int ret, i; if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) return -ENODEV; @@ -1595,11 +1643,13 @@ static int intel_runtime_suspend(struct device *kdev) */ i915_gem_runtime_suspend(dev_priv); - intel_gt_runtime_suspend(to_gt(dev_priv)); + for_each_gt(gt, dev_priv, i) + intel_gt_runtime_suspend(gt); intel_runtime_pm_disable_interrupts(dev_priv); - intel_uncore_suspend(&dev_priv->uncore); + for_each_gt(gt, dev_priv, i) + intel_uncore_suspend(gt->uncore); intel_display_power_suspend(dev_priv); @@ -1663,7 +1713,8 @@ static int intel_runtime_resume(struct device *kdev) { struct drm_i915_private *dev_priv = kdev_to_i915(kdev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret; + struct intel_gt *gt; + int ret, i; if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) return -ENODEV; @@ -1683,7 +1734,8 @@ static int intel_runtime_resume(struct device *kdev) ret = vlv_resume_prepare(dev_priv, true); - intel_uncore_runtime_resume(&dev_priv->uncore); + for_each_gt(gt, dev_priv, i) + intel_uncore_runtime_resume(gt->uncore); intel_runtime_pm_enable_interrupts(dev_priv); @@ -1691,7 +1743,8 @@ static int intel_runtime_resume(struct device *kdev) * No point of rolling back things in case of an error, as the best * we can do is to hope that things will still work (and disable RPM). */ - intel_gt_runtime_resume(to_gt(dev_priv)); + for_each_gt(gt, dev_priv, i) + intel_gt_runtime_resume(gt); /* * On VLV/CHV display interrupts are part of the display @@ -1703,7 +1756,7 @@ static int intel_runtime_resume(struct device *kdev) intel_hpd_poll_disable(dev_priv); } - intel_enable_ipc(dev_priv); + skl_watermark_ipc_update(dev_priv); enable_rpm_wakeref_asserts(rpm); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 7c0a34a33fec8e232b29c1481140404060ecb2b8..bdc81db76dbd1c89e60ace6eaced7f373949d9e2 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -34,20 +34,10 @@ #include -#include #include -#include "display/intel_cdclk.h" #include "display/intel_display.h" -#include "display/intel_display_power.h" -#include "display/intel_dmc.h" -#include "display/intel_dpll_mgr.h" -#include "display/intel_dsb.h" -#include "display/intel_fbc.h" -#include "display/intel_frontbuffer.h" -#include "display/intel_global_state.h" -#include "display/intel_gmbus.h" -#include "display/intel_opregion.h" +#include "display/intel_display_core.h" #include "gem/i915_gem_context_types.h" #include "gem/i915_gem_lmem.h" @@ -70,80 +60,24 @@ #include "intel_device_info.h" #include "intel_memory_region.h" #include "intel_pch.h" -#include "intel_pm_types.h" #include "intel_runtime_pm.h" #include "intel_step.h" #include "intel_uncore.h" #include "intel_wopcm.h" -struct dpll; struct drm_i915_clock_gating_funcs; struct drm_i915_gem_object; struct drm_i915_private; -struct intel_atomic_state; -struct intel_audio_funcs; -struct intel_cdclk_config; -struct intel_cdclk_funcs; -struct intel_cdclk_state; -struct intel_cdclk_vals; -struct intel_color_funcs; struct intel_connector; -struct intel_crtc; struct intel_dp; -struct intel_dpll_funcs; struct intel_encoder; -struct intel_fbdev; -struct intel_fdi_funcs; -struct intel_gmbus; -struct intel_hotplug_funcs; -struct intel_initial_plane_config; struct intel_limit; -struct intel_overlay; struct intel_overlay_error_state; struct vlv_s0ix_state; /* Threshold == 5 for long IRQs, 50 for short */ #define HPD_STORM_DEFAULT_THRESHOLD 50 -struct i915_hotplug { - struct delayed_work hotplug_work; - - const u32 *hpd, *pch_hpd; - - struct { - unsigned long last_jiffies; - int count; - enum { - HPD_ENABLED = 0, - HPD_DISABLED = 1, - HPD_MARK_DISABLED = 2 - } state; - } stats[HPD_NUM_PINS]; - u32 event_bits; - u32 retry_bits; - struct delayed_work reenable_work; - - u32 long_port_mask; - u32 short_port_mask; - struct work_struct dig_port_work; - - struct work_struct poll_init_work; - bool poll_enabled; - - unsigned int hpd_storm_threshold; - /* Whether or not to count short HPD IRQs in HPD storms */ - u8 hpd_short_storm_enabled; - - /* - * if we get a HPD irq from DP and a HPD irq from non-DP - * the non-DP HPD could block the workqueue on a mode config - * mutex getting, that userspace may have taken. However - * userspace is waiting on the DP workqueue to run which is - * blocked behind the non-DP one. - */ - struct workqueue_struct *dp_wq; -}; - #define I915_GEM_GPU_DOMAINS \ (I915_GEM_DOMAIN_RENDER | \ I915_GEM_DOMAIN_SAMPLER | \ @@ -151,55 +85,9 @@ struct i915_hotplug { I915_GEM_DOMAIN_INSTRUCTION | \ I915_GEM_DOMAIN_VERTEX) -struct sdvo_device_mapping { - u8 initialized; - u8 dvo_port; - u8 slave_addr; - u8 dvo_wiring; - u8 i2c_pin; - u8 ddc_pin; -}; - -/* functions used for watermark calcs for display. */ -struct drm_i915_wm_disp_funcs { - /* update_wm is for legacy wm management */ - void (*update_wm)(struct drm_i915_private *dev_priv); - int (*compute_pipe_wm)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - int (*compute_intermediate_wm)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*initial_watermarks)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*atomic_update_watermarks)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*optimize_watermarks)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - int (*compute_global_watermarks)(struct intel_atomic_state *state); -}; - -struct drm_i915_display_funcs { - /* Returns the active state of the crtc, and if the crtc is active, - * fills out the pipe-config with the hw state. */ - bool (*get_pipe_config)(struct intel_crtc *, - struct intel_crtc_state *); - void (*get_initial_plane_config)(struct intel_crtc *, - struct intel_initial_plane_config *); - void (*crtc_enable)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*crtc_disable)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*commit_modeset_enables)(struct intel_atomic_state *state); -}; - #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */ -#define QUIRK_LVDS_SSC_DISABLE (1<<1) -#define QUIRK_INVERT_BRIGHTNESS (1<<2) -#define QUIRK_BACKLIGHT_PRESENT (1<<3) -#define QUIRK_PIN_SWIZZLED_PAGES (1<<5) -#define QUIRK_INCREASE_T12_DELAY (1<<6) -#define QUIRK_INCREASE_DDI_DISABLED_TIME (1<<7) -#define QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK (1<<8) +#define GEM_QUIRK_PIN_SWIZZLED_PAGES BIT(0) struct i915_suspend_saved_registers { u32 saveDSPARB; @@ -289,51 +177,8 @@ i915_fence_timeout(const struct drm_i915_private *i915) return i915_fence_context_timeout(i915, U64_MAX); } -/* Amount of SAGV/QGV points, BSpec precisely defines this */ -#define I915_NUM_QGV_POINTS 8 - #define HAS_HW_SAGV_WM(i915) (DISPLAY_VER(i915) >= 13 && !IS_DGFX(i915)) -/* Amount of PSF GV points, BSpec precisely defines this */ -#define I915_NUM_PSF_GV_POINTS 3 - -struct intel_vbt_data { - /* bdb version */ - u16 version; - - /* Feature bits */ - unsigned int int_tv_support:1; - unsigned int int_crt_support:1; - unsigned int lvds_use_ssc:1; - unsigned int int_lvds_support:1; - unsigned int display_clock_mode:1; - unsigned int fdi_rx_polarity_inverted:1; - int lvds_ssc_freq; - enum drm_panel_orientation orientation; - - bool override_afc_startup; - u8 override_afc_startup_val; - - int crt_ddc_pin; - - struct list_head display_devices; - struct list_head bdb_blocks; - - struct intel_bios_encoder_data *ports[I915_MAX_PORTS]; /* Non-NULL if port present. */ - struct sdvo_device_mapping sdvo_mappings[2]; -}; - -struct i915_frontbuffer_tracking { - spinlock_t lock; - - /* - * Tracking bits for delayed frontbuffer flushing du to gpu activity or - * scheduled flips. - */ - unsigned busy_bits; - unsigned flip_bits; -}; - struct i915_virtual_gpu { struct mutex lock; /* serialises sending of g2v_notify command pkts */ bool active; @@ -348,32 +193,11 @@ struct i915_selftest_stash { struct ida mock_region_instances; }; -/* intel_audio.c private */ -struct intel_audio_private { - /* Display internal audio functions */ - const struct intel_audio_funcs *funcs; - - /* hda/i915 audio component */ - struct i915_audio_component *component; - bool component_registered; - /* mutex for audio/video sync */ - struct mutex mutex; - int power_refcount; - u32 freq_cntrl; - - /* Used to save the pipe-to-encoder mapping for audio */ - struct intel_encoder *encoder_map[I915_MAX_PIPES]; - - /* necessary resource sharing with HDMI LPE audio driver. */ - struct { - struct platform_device *platdev; - int irq; - } lpe; -}; - struct drm_i915_private { struct drm_device drm; + struct intel_display display; + /* FIXME: Device release actions should all be moved to drmm_ */ bool do_release; @@ -417,27 +241,6 @@ struct drm_i915_private { struct intel_wopcm wopcm; - struct intel_dmc dmc; - - struct intel_gmbus *gmbus[GMBUS_NUM_PINS]; - - /** gmbus_mutex protects against concurrent usage of the single hw gmbus - * controller on different i2c buses. */ - struct mutex gmbus_mutex; - - /** - * Base address of where the gmbus and gpio blocks are located (either - * on PCH or on SoC for platforms without PCH). - */ - u32 gpio_mmio_base; - - /* MMIO base address for MIPI regs */ - u32 mipi_mmio_base; - - u32 pps_mmio_base; - - wait_queue_head_t gmbus_wait_queue; - struct pci_dev *bridge_dev; struct rb_root uabi_engines; @@ -461,48 +264,15 @@ struct drm_i915_private { }; u32 pipestat_irq_mask[I915_MAX_PIPES]; - struct i915_hotplug hotplug; - struct intel_fbc *fbc[I915_MAX_FBCS]; - struct intel_opregion opregion; - struct intel_vbt_data vbt; - bool preserve_bios_swizzle; - /* overlay */ - struct intel_overlay *overlay; - - /* backlight registers and fields in struct intel_panel */ - struct mutex backlight_lock; - - /* protects panel power sequencer state */ - struct mutex pps_mutex; - unsigned int fsb_freq, mem_freq, is_ddr3; unsigned int skl_preferred_vco_freq; - unsigned int max_cdclk_freq; unsigned int max_dotclk_freq; unsigned int hpll_freq; - unsigned int fdi_pll_freq; unsigned int czclk_freq; - struct { - /* The current hardware cdclk configuration */ - struct intel_cdclk_config hw; - - /* cdclk, divider, and ratio table from bspec */ - const struct intel_cdclk_vals *table; - - struct intel_global_obj obj; - } cdclk; - - struct { - /* The current hardware dbuf configuration */ - u8 enabled_slices; - - struct intel_global_obj obj; - } dbuf; - /** * wq - Driver workqueue for GEM. * @@ -512,40 +282,14 @@ struct drm_i915_private { */ struct workqueue_struct *wq; - /* ordered wq for modesets */ - struct workqueue_struct *modeset_wq; - /* unbound hipri wq for page flips/plane updates */ - struct workqueue_struct *flip_wq; - /* pm private clock gating functions */ const struct drm_i915_clock_gating_funcs *clock_gating_funcs; - /* pm display functions */ - const struct drm_i915_wm_disp_funcs *wm_disp; - - /* irq display functions */ - const struct intel_hotplug_funcs *hotplug_funcs; - - /* fdi display functions */ - const struct intel_fdi_funcs *fdi_funcs; - - /* display pll funcs */ - const struct intel_dpll_funcs *dpll_funcs; - - /* Display functions */ - const struct drm_i915_display_funcs *display; - - /* Display internal color functions */ - const struct intel_color_funcs *color_funcs; - - /* Display CDCLK functions */ - const struct intel_cdclk_funcs *cdclk_funcs; - /* PCH chipset type */ enum intel_pch pch_type; unsigned short pch_id; - unsigned long quirks; + unsigned long gem_quirks; struct drm_atomic_state *modeset_restore_state; struct drm_modeset_acquire_ctx reset_ctx; @@ -554,34 +298,8 @@ struct drm_i915_private { /* Kernel Modesetting */ - /** - * dpll and cdclk state is protected by connection_mutex - * dpll.lock serializes intel_{prepare,enable,disable}_shared_dpll. - * Must be global rather than per dpll, because on some platforms plls - * share registers. - */ - struct { - struct mutex lock; - - int num_shared_dpll; - struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; - const struct intel_dpll_mgr *mgr; - - struct { - int nssc; - int ssc; - } ref_clks; - } dpll; - struct list_head global_obj_list; - struct i915_frontbuffer_tracking fb_tracking; - - struct intel_atomic_helper { - struct llist_head free_list; - struct work_struct free_work; - } atomic_helper; - bool mchbar_need_disable; struct intel_l3_parity l3_parity; @@ -600,21 +318,8 @@ struct drm_i915_private { */ u32 edram_size_mb; - struct i915_power_domains power_domains; - struct i915_gpu_error gpu_error; - /* list of fbdev register on this device */ - struct intel_fbdev *fbdev; - struct work_struct fbdev_suspend_work; - - struct drm_property *broadcast_rgb_property; - struct drm_property *force_audio_property; - - u32 fdi_rx_config; - - /* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */ - u32 chv_phy_control; /* * Shadows for CHV DPLL_MD regs to keep the state * checker somewhat working in the presence hardware @@ -627,51 +332,6 @@ struct drm_i915_private { struct i915_suspend_saved_registers regfile; struct vlv_s0ix_state *vlv_s0ix_state; - enum { - I915_SAGV_UNKNOWN = 0, - I915_SAGV_DISABLED, - I915_SAGV_ENABLED, - I915_SAGV_NOT_CONTROLLED - } sagv_status; - - u32 sagv_block_time_us; - - struct { - /* - * Raw watermark latency values: - * in 0.1us units for WM0, - * in 0.5us units for WM1+. - */ - /* primary */ - u16 pri_latency[5]; - /* sprite */ - u16 spr_latency[5]; - /* cursor */ - u16 cur_latency[5]; - /* - * Raw watermark memory latency values - * for SKL for all 8 levels - * in 1us units. - */ - u16 skl_latency[8]; - - /* current hardware state */ - union { - struct ilk_wm_values hw; - struct vlv_wm_values vlv; - struct g4x_wm_values g4x; - }; - - u8 max_level; - - /* - * Should be held around atomic WM register writing; also - * protects * intel_crtc->wm.active and - * crtc_state->wm.need_postvbl_update. - */ - struct mutex wm_mutex; - } wm; - struct dram_info { bool wm_lv_0_adjust_needed; u8 num_channels; @@ -689,18 +349,6 @@ struct drm_i915_private { u8 num_psf_gv_points; } dram_info; - struct intel_bw_info { - /* for each QGV point */ - unsigned int deratedbw[I915_NUM_QGV_POINTS]; - /* for each PSF GV point */ - unsigned int psf_bw[I915_NUM_PSF_GV_POINTS]; - u8 num_qgv_points; - u8 num_psf_gv_points; - u8 num_planes; - } max_bw[6]; - - struct intel_global_obj bw_obj; - struct intel_runtime_pm runtime_pm; struct i915_perf perf; @@ -716,6 +364,9 @@ struct drm_i915_private { struct kobject *sysfs_gt; + /* Quick lookup of media GT (current platforms only have one) */ + struct intel_gt *media_gt; + struct { struct i915_gem_contexts { spinlock_t lock; /* locks list */ @@ -733,9 +384,6 @@ struct drm_i915_private { struct file *mmap_singleton; } gem; - /* Window2 specifies time required to program DSB (Window2) in number of scan lines */ - u8 window2_delay; - u8 pch_ssc_use; /* For i915gm/i945gm vblank irq workaround */ @@ -743,31 +391,16 @@ struct drm_i915_private { bool irq_enabled; - union { - /* perform PHY state sanity checks? */ - bool chv_phy_assert[2]; - - /* - * DG2: Mask of PHYs that were not calibrated by the firmware - * and should not be used. - */ - u8 snps_phy_failed_calibration; - }; - - bool ipc_enabled; - - struct intel_audio_private audio; + /* + * DG2: Mask of PHYs that were not calibrated by the firmware + * and should not be used. + */ + u8 snps_phy_failed_calibration; struct i915_pmu pmu; struct i915_drm_clients clients; - struct i915_hdcp_comp_master *hdcp_master; - bool hdcp_comp_added; - - /* Mutex to protect the above hdcp component related values. */ - struct mutex hdcp_comp_mutex; - /* The TTM device structure. */ struct ttm_device bdev; @@ -826,28 +459,6 @@ static inline struct intel_gt *to_gt(struct drm_i915_private *i915) (engine__) && (engine__)->uabi_class == (class__); \ (engine__) = rb_to_uabi_engine(rb_next(&(engine__)->uabi_node))) -#define I915_GTT_OFFSET_NONE ((u32)-1) - -/* - * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is - * considered to be the frontbuffer for the given plane interface-wise. This - * doesn't mean that the hw necessarily already scans it out, but that any - * rendering (by the cpu or gpu) will land in the frontbuffer eventually. - * - * We have one bit per pipe and per scanout plane type. - */ -#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8 -#define INTEL_FRONTBUFFER(pipe, plane_id) ({ \ - BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32); \ - BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE); \ - BIT((plane_id) + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)); \ -}) -#define INTEL_FRONTBUFFER_OVERLAY(pipe) \ - BIT(INTEL_FRONTBUFFER_BITS_PER_PIPE - 1 + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) -#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ - GENMASK(INTEL_FRONTBUFFER_BITS_PER_PIPE * ((pipe) + 1) - 1, \ - INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)) - #define INTEL_INFO(dev_priv) (&(dev_priv)->__info) #define RUNTIME_INFO(dev_priv) (&(dev_priv)->__runtime) #define DRIVER_CAPS(dev_priv) (&(dev_priv)->caps) @@ -856,19 +467,19 @@ static inline struct intel_gt *to_gt(struct drm_i915_private *i915) #define IP_VER(ver, rel) ((ver) << 8 | (rel)) -#define GRAPHICS_VER(i915) (INTEL_INFO(i915)->graphics.ver) -#define GRAPHICS_VER_FULL(i915) IP_VER(INTEL_INFO(i915)->graphics.ver, \ - INTEL_INFO(i915)->graphics.rel) +#define GRAPHICS_VER(i915) (RUNTIME_INFO(i915)->graphics.ip.ver) +#define GRAPHICS_VER_FULL(i915) IP_VER(RUNTIME_INFO(i915)->graphics.ip.ver, \ + RUNTIME_INFO(i915)->graphics.ip.rel) #define IS_GRAPHICS_VER(i915, from, until) \ (GRAPHICS_VER(i915) >= (from) && GRAPHICS_VER(i915) <= (until)) -#define MEDIA_VER(i915) (INTEL_INFO(i915)->media.ver) -#define MEDIA_VER_FULL(i915) IP_VER(INTEL_INFO(i915)->media.ver, \ - INTEL_INFO(i915)->media.rel) +#define MEDIA_VER(i915) (RUNTIME_INFO(i915)->media.ip.ver) +#define MEDIA_VER_FULL(i915) IP_VER(RUNTIME_INFO(i915)->media.ip.ver, \ + RUNTIME_INFO(i915)->media.ip.rel) #define IS_MEDIA_VER(i915, from, until) \ (MEDIA_VER(i915) >= (from) && MEDIA_VER(i915) <= (until)) -#define DISPLAY_VER(i915) (INTEL_INFO(i915)->display.ver) +#define DISPLAY_VER(i915) (RUNTIME_INFO(i915)->display.ip.ver) #define IS_DISPLAY_VER(i915, from, until) \ (DISPLAY_VER(i915) >= (from) && DISPLAY_VER(i915) <= (until)) @@ -1210,7 +821,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv) -#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type) +#define INTEL_PPGTT(dev_priv) (RUNTIME_INFO(dev_priv)->ppgtt_type) #define HAS_PPGTT(dev_priv) \ (INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE) #define HAS_FULL_PPGTT(dev_priv) \ @@ -1218,7 +829,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \ GEM_BUG_ON((sizes) == 0); \ - ((sizes) & ~INTEL_INFO(dev_priv)->page_sizes) == 0; \ + ((sizes) & ~RUNTIME_INFO(dev_priv)->page_sizes) == 0; \ }) #define HAS_OVERLAY(dev_priv) (INTEL_INFO(dev_priv)->display.has_overlay) @@ -1249,13 +860,15 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define I915_HAS_HOTPLUG(dev_priv) (INTEL_INFO(dev_priv)->display.has_hotplug) #define HAS_FW_BLC(dev_priv) (DISPLAY_VER(dev_priv) > 2) -#define HAS_FBC(dev_priv) (INTEL_INFO(dev_priv)->display.fbc_mask != 0) +#define HAS_FBC(dev_priv) (RUNTIME_INFO(dev_priv)->fbc_mask != 0) #define HAS_CUR_FBC(dev_priv) (!HAS_GMCH(dev_priv) && DISPLAY_VER(dev_priv) >= 7) #define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) #define HAS_DP_MST(dev_priv) (INTEL_INFO(dev_priv)->display.has_dp_mst) -#define HAS_DP20(dev_priv) (IS_DG2(dev_priv)) +#define HAS_DP20(dev_priv) (IS_DG2(dev_priv) || DISPLAY_VER(dev_priv) >= 14) + +#define HAS_DOUBLE_BUFFERED_M_N(dev_priv) (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) #define HAS_CDCLK_CRAWL(dev_priv) (INTEL_INFO(dev_priv)->display.has_cdclk_crawl) #define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) @@ -1264,7 +877,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_PSR_HW_TRACKING(dev_priv) \ (INTEL_INFO(dev_priv)->display.has_psr_hw_tracking) #define HAS_PSR2_SEL_FETCH(dev_priv) (DISPLAY_VER(dev_priv) >= 12) -#define HAS_TRANSCODER(dev_priv, trans) ((INTEL_INFO(dev_priv)->display.cpu_transcoder_mask & BIT(trans)) != 0) +#define HAS_TRANSCODER(dev_priv, trans) ((RUNTIME_INFO(dev_priv)->cpu_transcoder_mask & BIT(trans)) != 0) #define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) #define HAS_RC6p(dev_priv) (INTEL_INFO(dev_priv)->has_rc6p) @@ -1272,7 +885,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_RPS(dev_priv) (INTEL_INFO(dev_priv)->has_rps) -#define HAS_DMC(dev_priv) (INTEL_INFO(dev_priv)->display.has_dmc) +#define HAS_DMC(dev_priv) (RUNTIME_INFO(dev_priv)->has_dmc) #define HAS_HECI_PXP(dev_priv) \ (INTEL_INFO(dev_priv)->has_heci_pxp) @@ -1302,9 +915,11 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_IPC(dev_priv) (INTEL_INFO(dev_priv)->display.has_ipc) -#define HAS_REGION(i915, i) (INTEL_INFO(i915)->memory_regions & (i)) +#define HAS_REGION(i915, i) (RUNTIME_INFO(i915)->memory_regions & (i)) #define HAS_LMEM(i915) HAS_REGION(i915, REGION_LMEM) +#define HAS_EXTRA_GT_LIST(dev_priv) (INTEL_INFO(dev_priv)->extra_gt_list) + /* * Platform has the dedicated compression control state for each lmem surfaces * stored in lmem to support the 3D and media compression formats. @@ -1313,7 +928,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_GT_UC(dev_priv) (INTEL_INFO(dev_priv)->has_gt_uc) -#define HAS_POOLED_EU(dev_priv) (INTEL_INFO(dev_priv)->has_pooled_eu) +#define HAS_POOLED_EU(dev_priv) (RUNTIME_INFO(dev_priv)->has_pooled_eu) #define HAS_GLOBAL_MOCS_REGISTERS(dev_priv) (INTEL_INFO(dev_priv)->has_global_mocs) @@ -1335,9 +950,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define GT_FREQUENCY_MULTIPLIER 50 #define GEN9_FREQ_SCALER 3 -#define INTEL_NUM_PIPES(dev_priv) (hweight8(INTEL_INFO(dev_priv)->display.pipe_mask)) +#define INTEL_NUM_PIPES(dev_priv) (hweight8(RUNTIME_INFO(dev_priv)->pipe_mask)) -#define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->display.pipe_mask != 0) +#define HAS_DISPLAY(dev_priv) (RUNTIME_INFO(dev_priv)->pipe_mask != 0) #define HAS_VRR(i915) (DISPLAY_VER(i915) >= 11) @@ -1355,85 +970,12 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_D12_PLANE_MINIMIZATION(dev_priv) (IS_ROCKETLAKE(dev_priv) || \ IS_ALDERLAKE_S(dev_priv)) -#define HAS_MBUS_JOINING(i915) (IS_ALDERLAKE_P(i915)) +#define HAS_MBUS_JOINING(i915) (IS_ALDERLAKE_P(i915) || DISPLAY_VER(i915) >= 14) #define HAS_3D_PIPELINE(i915) (INTEL_INFO(i915)->has_3d_pipeline) #define HAS_ONE_EU_PER_FUSE_BIT(i915) (INTEL_INFO(i915)->has_one_eu_per_fuse_bit) -/* i915_gem.c */ -void i915_gem_init_early(struct drm_i915_private *dev_priv); -void i915_gem_cleanup_early(struct drm_i915_private *dev_priv); - -static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915) -{ - /* - * A single pass should suffice to release all the freed objects (along - * most call paths) , but be a little more paranoid in that freeing - * the objects does take a little amount of time, during which the rcu - * callbacks could have added new objects into the freed list, and - * armed the work again. - */ - while (atomic_read(&i915->mm.free_count)) { - flush_work(&i915->mm.free_work); - flush_delayed_work(&i915->bdev.wq); - rcu_barrier(); - } -} - -static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915) -{ - /* - * Similar to objects above (see i915_gem_drain_freed-objects), in - * general we have workers that are armed by RCU and then rearm - * themselves in their callbacks. To be paranoid, we need to - * drain the workqueue a second time after waiting for the RCU - * grace period so that we catch work queued via RCU from the first - * pass. As neither drain_workqueue() nor flush_workqueue() report - * a result, we make an assumption that we only don't require more - * than 3 passes to catch all _recursive_ RCU delayed work. - * - */ - int pass = 3; - do { - flush_workqueue(i915->wq); - rcu_barrier(); - i915_gem_drain_freed_objects(i915); - } while (--pass); - drain_workqueue(i915->wq); -} - -struct i915_vma * __must_check -i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, - struct i915_gem_ww_ctx *ww, - const struct i915_gtt_view *view, - u64 size, u64 alignment, u64 flags); - -struct i915_vma * __must_check -i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, - const struct i915_gtt_view *view, - u64 size, u64 alignment, u64 flags); - -int i915_gem_object_unbind(struct drm_i915_gem_object *obj, - unsigned long flags); -#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) -#define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) -#define I915_GEM_OBJECT_UNBIND_TEST BIT(2) -#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3) -#define I915_GEM_OBJECT_UNBIND_ASYNC BIT(4) - -void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); - -int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); - -int __must_check i915_gem_init(struct drm_i915_private *dev_priv); -void i915_gem_driver_register(struct drm_i915_private *i915); -void i915_gem_driver_unregister(struct drm_i915_private *i915); -void i915_gem_driver_remove(struct drm_i915_private *dev_priv); -void i915_gem_driver_release(struct drm_i915_private *dev_priv); - -int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); - /* intel_device_info.c */ static inline struct intel_device_info * mkwrite_device_info(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index a3373699835d76df04fb618bf95ccf9577674bc8..2bdddb61ebd7ae6215eaa26706a2df3258d42171 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -842,6 +842,10 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) &to_gt(i915)->ggtt->userfault_list, userfault_link) __i915_gem_object_release_mmap_gtt(obj); + list_for_each_entry_safe(obj, on, + &to_gt(i915)->lmem_userfault_list, userfault_link) + i915_gem_object_runtime_pm_release_mmap_offset(obj); + /* * The fence will be lost when the device powers down. If any were * in use by hardware (i.e. they are pinned), we should not be powering @@ -1035,7 +1039,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, if (i915_gem_object_has_pages(obj) && i915_gem_object_is_tiled(obj) && - i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { + i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) { if (obj->mm.madv == I915_MADV_WILLNEED) { GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_clear_tiling_quirk(obj); @@ -1085,14 +1089,50 @@ out: return err; } +/* + * A single pass should suffice to release all the freed objects (along most + * call paths), but be a little more paranoid in that freeing the objects does + * take a little amount of time, during which the rcu callbacks could have added + * new objects into the freed list, and armed the work again. + */ +void i915_gem_drain_freed_objects(struct drm_i915_private *i915) +{ + while (atomic_read(&i915->mm.free_count)) { + flush_work(&i915->mm.free_work); + flush_delayed_work(&i915->bdev.wq); + rcu_barrier(); + } +} + +/* + * Similar to objects above (see i915_gem_drain_freed-objects), in general we + * have workers that are armed by RCU and then rearm themselves in their + * callbacks. To be paranoid, we need to drain the workqueue a second time after + * waiting for the RCU grace period so that we catch work queued via RCU from + * the first pass. As neither drain_workqueue() nor flush_workqueue() report a + * result, we make an assumption that we only don't require more than 3 passes + * to catch all _recursive_ RCU delayed work. + */ +void i915_gem_drain_workqueue(struct drm_i915_private *i915) +{ + int i; + + for (i = 0; i < 3; i++) { + flush_workqueue(i915->wq); + rcu_barrier(); + i915_gem_drain_freed_objects(i915); + } + + drain_workqueue(i915->wq); +} + int i915_gem_init(struct drm_i915_private *dev_priv) { int ret; /* We need to fallback to 4K pages if host doesn't support huge gtt. */ if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv)) - mkwrite_device_info(dev_priv)->page_sizes = - I915_GTT_PAGE_SIZE_4K; + RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K; ret = i915_gem_init_userptr(dev_priv); if (ret) @@ -1173,7 +1213,7 @@ void i915_gem_driver_unregister(struct drm_i915_private *i915) void i915_gem_driver_remove(struct drm_i915_private *dev_priv) { - intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref); + intel_wakeref_auto_fini(&to_gt(dev_priv)->userfault_wakeref); i915_gem_suspend_late(dev_priv); intel_gt_driver_remove(to_gt(dev_priv)); @@ -1191,7 +1231,8 @@ void i915_gem_driver_release(struct drm_i915_private *dev_priv) intel_uc_cleanup_firmwares(&to_gt(dev_priv)->uc); - i915_gem_drain_freed_objects(dev_priv); + /* Flush any outstanding work, including i915_gem_context.release_work. */ + i915_gem_drain_workqueue(dev_priv); drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list)); } @@ -1213,7 +1254,7 @@ void i915_gem_init_early(struct drm_i915_private *dev_priv) i915_gem_init__mm(dev_priv); i915_gem_init__contexts(dev_priv); - spin_lock_init(&dev_priv->fb_tracking.lock); + spin_lock_init(&dev_priv->display.fb_tracking.lock); } void i915_gem_cleanup_early(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 68d8d52bd541db8e8f488abdca558a8cd0f8dd8c..a5cdf6662d01d1c6b34e4a5736a0b121803ac067 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -26,12 +26,55 @@ #define __I915_GEM_H__ #include +#include #include #include "i915_utils.h" +struct drm_file; +struct drm_i915_gem_object; struct drm_i915_private; +struct i915_gem_ww_ctx; +struct i915_gtt_view; +struct i915_vma; + +void i915_gem_init_early(struct drm_i915_private *i915); +void i915_gem_cleanup_early(struct drm_i915_private *i915); + +void i915_gem_drain_freed_objects(struct drm_i915_private *i915); +void i915_gem_drain_workqueue(struct drm_i915_private *i915); + +struct i915_vma * __must_check +i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, + struct i915_gem_ww_ctx *ww, + const struct i915_gtt_view *view, + u64 size, u64 alignment, u64 flags); + +struct i915_vma * __must_check +i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, + const struct i915_gtt_view *view, + u64 size, u64 alignment, u64 flags); + +int i915_gem_object_unbind(struct drm_i915_gem_object *obj, + unsigned long flags); +#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) +#define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) +#define I915_GEM_OBJECT_UNBIND_TEST BIT(2) +#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3) +#define I915_GEM_OBJECT_UNBIND_ASYNC BIT(4) + +void i915_gem_runtime_suspend(struct drm_i915_private *i915); + +int __must_check i915_gem_init(struct drm_i915_private *i915); +void i915_gem_driver_register(struct drm_i915_private *i915); +void i915_gem_driver_unregister(struct drm_i915_private *i915); +void i915_gem_driver_remove(struct drm_i915_private *i915); +void i915_gem_driver_release(struct drm_i915_private *i915); + +int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file); + +/* FIXME: All of the below belong somewhere else. */ #ifdef CONFIG_DRM_I915_DEBUG_GEM diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 329ff75b80b97d8cc33b8f2e320d3f7904bb4278..7bd1861ddbdfbd60acf7342d07988cc604df2101 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -137,12 +137,12 @@ static u64 random_offset(u64 start, u64 end, u64 len, u64 align) range = round_down(end - len, align) - round_up(start, align); if (range) { if (sizeof(unsigned long) == sizeof(u64)) { - addr = get_random_long(); + addr = get_random_u64(); } else { - addr = get_random_int(); + addr = get_random_u32(); if (range > U32_MAX) { addr <<= 32; - addr |= get_random_int(); + addr |= get_random_u32(); } } div64_u64_rem(addr, range, &addr); diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 6fd15b39570c108a5c354f3f95e903852e525ab9..342c8ca6414ec6476a08ff086a44ca14eab754de 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -36,7 +36,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, value = to_gt(i915)->ggtt->num_fences; break; case I915_PARAM_HAS_OVERLAY: - value = !!i915->overlay; + value = !!i915->display.overlay; break; case I915_PARAM_HAS_BSD: value = !!intel_engine_lookup_user(i915, diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 16d8b7ba65dce016ad694ad0bb36224002f1cd4e..9ea2fe34e7d307f6a40c979ea5a9860a02f9dbdc 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -646,8 +646,7 @@ static void err_print_capabilities(struct drm_i915_error_state_buf *m, { struct drm_printer p = i915_error_printer(m); - intel_device_info_print_static(&error->device_info, &p); - intel_device_info_print_runtime(&error->runtime_info, &p); + intel_device_info_print(&error->device_info, &error->runtime_info, &p); intel_driver_caps_print(&error->driver_caps, &p); } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 783a6ca41a61732556e02c318f817c2e3d61d309..86a42d9e8041258d14002441bf4f0a0172449244 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -185,7 +185,7 @@ static const u32 hpd_sde_dg1[HPD_NUM_PINS] = { static void intel_hpd_init_pins(struct drm_i915_private *dev_priv) { - struct i915_hotplug *hpd = &dev_priv->hotplug; + struct intel_hotplug *hpd = &dev_priv->display.hotplug; if (HAS_GMCH(dev_priv)) { if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || @@ -595,7 +595,7 @@ void i915_disable_pipestat(struct drm_i915_private *dev_priv, static bool i915_has_asle(struct drm_i915_private *dev_priv) { - if (!dev_priv->opregion.asle) + if (!dev_priv->display.opregion.asle) return false; return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); @@ -1104,9 +1104,9 @@ static void ivb_parity_work(struct work_struct *work) out: drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv)); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); mutex_unlock(&dev_priv->drm.struct_mutex); } @@ -1272,7 +1272,7 @@ static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv, u32 enabled_irqs = 0; for_each_intel_encoder(&dev_priv->drm, encoder) - if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) + if (dev_priv->display.hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED) enabled_irqs |= hpd[encoder->hpd_pin]; return enabled_irqs; @@ -1304,12 +1304,12 @@ static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915, static void gmbus_irq_handler(struct drm_i915_private *dev_priv) { - wake_up_all(&dev_priv->gmbus_wait_queue); + wake_up_all(&dev_priv->display.gmbus.wait_queue); } static void dp_aux_irq_handler(struct drm_i915_private *dev_priv) { - wake_up_all(&dev_priv->gmbus_wait_queue); + wake_up_all(&dev_priv->display.gmbus.wait_queue); } #if defined(CONFIG_DEBUG_FS) @@ -1637,7 +1637,7 @@ static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv, if (hotplug_trigger) { intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, hotplug_trigger, - dev_priv->hotplug.hpd, + dev_priv->display.hotplug.hpd, i9xx_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -1841,7 +1841,7 @@ static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.pch_hpd, + dev_priv->display.hotplug.pch_hpd, pch_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -1986,7 +1986,7 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, ddi_hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.pch_hpd, + dev_priv->display.hotplug.pch_hpd, icp_ddi_port_hotplug_long_detect); } @@ -1998,7 +1998,7 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, tc_hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.pch_hpd, + dev_priv->display.hotplug.pch_hpd, icp_tc_port_hotplug_long_detect); } @@ -2024,7 +2024,7 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.pch_hpd, + dev_priv->display.hotplug.pch_hpd, spt_port_hotplug_long_detect); } @@ -2036,7 +2036,7 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug2_trigger, dig_hotplug_reg, - dev_priv->hotplug.pch_hpd, + dev_priv->display.hotplug.pch_hpd, spt_port_hotplug2_long_detect); } @@ -2057,7 +2057,7 @@ static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv, intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.hpd, + dev_priv->display.hotplug.hpd, ilk_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -2237,7 +2237,7 @@ static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv, intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, dig_hotplug_reg, - dev_priv->hotplug.hpd, + dev_priv->display.hotplug.hpd, bxt_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -2257,7 +2257,7 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc, dig_hotplug_reg, - dev_priv->hotplug.hpd, + dev_priv->display.hotplug.hpd, gen11_port_hotplug_long_detect); } @@ -2269,7 +2269,7 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt, dig_hotplug_reg, - dev_priv->hotplug.hpd, + dev_priv->display.hotplug.hpd, gen11_port_hotplug_long_detect); } @@ -2653,9 +2653,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg) } static u32 -gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl) +gen11_gu_misc_irq_ack(struct drm_i915_private *i915, const u32 master_ctl) { - void __iomem * const regs = gt->uncore->regs; + void __iomem * const regs = i915->uncore.regs; u32 iir; if (!(master_ctl & GEN11_GU_MISC_IRQ)) @@ -2669,10 +2669,10 @@ gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl) } static void -gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir) +gen11_gu_misc_irq_handler(struct drm_i915_private *i915, const u32 iir) { if (iir & GEN11_GU_MISC_GSE) - intel_opregion_asle_intr(gt->i915); + intel_opregion_asle_intr(i915); } static inline u32 gen11_master_intr_disable(void __iomem * const regs) @@ -2736,11 +2736,11 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg) if (master_ctl & GEN11_DISPLAY_IRQ) gen11_display_irq_handler(i915); - gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); + gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl); gen11_master_intr_enable(regs); - gen11_gu_misc_irq_handler(gt, gu_misc_iir); + gen11_gu_misc_irq_handler(i915, gu_misc_iir); pmu_irq_stats(i915, IRQ_HANDLED); @@ -2801,11 +2801,11 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) if (master_ctl & GEN11_DISPLAY_IRQ) gen11_display_irq_handler(i915); - gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); + gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl); dg1_master_intr_enable(regs); - gen11_gu_misc_irq_handler(gt, gu_misc_iir); + gen11_gu_misc_irq_handler(i915, gu_misc_iir); pmu_irq_stats(i915, IRQ_HANDLED); @@ -3313,8 +3313,8 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3383,8 +3383,8 @@ static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); if (INTEL_PCH_TYPE(dev_priv) <= PCH_TGP) intel_uncore_write(&dev_priv->uncore, SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ); @@ -3460,8 +3460,8 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv) u32 hotplug_irqs, enabled_irqs; u32 val; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd); val = intel_uncore_read(&dev_priv->uncore, GEN11_DE_HPD_IMR); val &= ~hotplug_irqs; @@ -3538,8 +3538,8 @@ static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) intel_uncore_write(&dev_priv->uncore, SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ); - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd); ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3578,8 +3578,8 @@ static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd); if (DISPLAY_VER(dev_priv) >= 8) bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); @@ -3636,8 +3636,8 @@ static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); - hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd); + hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd); bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); @@ -4370,8 +4370,8 @@ HPD_FUNCS(ilk); void intel_hpd_irq_setup(struct drm_i915_private *i915) { - if (i915->display_irqs_enabled && i915->hotplug_funcs) - i915->hotplug_funcs->hpd_irq_setup(i915); + if (i915->display_irqs_enabled && i915->display.funcs.hotplug) + i915->display.funcs.hotplug->hpd_irq_setup(i915); } /** @@ -4413,33 +4413,33 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) dev_priv->display_irqs_enabled = false; - dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD; + dev_priv->display.hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD; /* If we have MST support, we want to avoid doing short HPD IRQ storm * detection, as short HPD storms will occur as a natural part of * sideband messaging with MST. * On older platforms however, IRQ storms can occur with both long and * short pulses, as seen on some G4x systems. */ - dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv); + dev_priv->display.hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv); if (HAS_GMCH(dev_priv)) { if (I915_HAS_HOTPLUG(dev_priv)) - dev_priv->hotplug_funcs = &i915_hpd_funcs; + dev_priv->display.funcs.hotplug = &i915_hpd_funcs; } else { if (HAS_PCH_DG2(dev_priv)) - dev_priv->hotplug_funcs = &icp_hpd_funcs; + dev_priv->display.funcs.hotplug = &icp_hpd_funcs; else if (HAS_PCH_DG1(dev_priv)) - dev_priv->hotplug_funcs = &dg1_hpd_funcs; + dev_priv->display.funcs.hotplug = &dg1_hpd_funcs; else if (DISPLAY_VER(dev_priv) >= 11) - dev_priv->hotplug_funcs = &gen11_hpd_funcs; + dev_priv->display.funcs.hotplug = &gen11_hpd_funcs; else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) - dev_priv->hotplug_funcs = &bxt_hpd_funcs; + dev_priv->display.funcs.hotplug = &bxt_hpd_funcs; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) - dev_priv->hotplug_funcs = &icp_hpd_funcs; + dev_priv->display.funcs.hotplug = &icp_hpd_funcs; else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) - dev_priv->hotplug_funcs = &spt_hpd_funcs; + dev_priv->display.funcs.hotplug = &spt_hpd_funcs; else - dev_priv->hotplug_funcs = &ilk_hpd_funcs; + dev_priv->display.funcs.hotplug = &ilk_hpd_funcs; } } diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index 6fc475a5db615c36a0115fef1ac3e5268453ae5c..d1e4d528cb174d68d3c08a7f9d29d38ac4b2ade6 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -29,6 +29,18 @@ #include "i915_params.h" #include "i915_drv.h" +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + #define i915_param_named(name, T, perm, desc) \ module_param_named(name, i915_modparams.name, T, perm); \ MODULE_PARM_DESC(name, desc) diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 59a579ed03bb14da44db9862475b8a5bd687e942..cd4487a1d3be0f5b5b4d9f476f1a89cd04b7f1fb 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -26,16 +26,22 @@ #include #include +#include "gt/intel_gt_regs.h" +#include "gt/intel_sa_media.h" + #include "i915_driver.h" #include "i915_drv.h" #include "i915_pci.h" #include "i915_reg.h" +#include "intel_pci_config.h" #define PLATFORM(x) .platform = (x) #define GEN(x) \ - .graphics.ver = (x), \ - .media.ver = (x), \ - .display.ver = (x) + .__runtime.graphics.ip.ver = (x), \ + .__runtime.media.ip.ver = (x), \ + .__runtime.display.ip.ver = (x) + +#define NO_DISPLAY .__runtime.pipe_mask = 0 #define I845_PIPE_OFFSETS \ .display.pipe_offsets = { \ @@ -159,16 +165,16 @@ /* Keep in gen based order, and chronological order within a gen */ #define GEN_DEFAULT_PAGE_SIZES \ - .page_sizes = I915_GTT_PAGE_SIZE_4K + .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K #define GEN_DEFAULT_REGIONS \ - .memory_regions = REGION_SMEM | REGION_STOLEN_SMEM + .__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_SMEM #define I830_FEATURES \ GEN(2), \ .is_mobile = 1, \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_overlay = 1, \ .display.cursor_needs_physical = 1, \ .display.overlay_needs_physical = 1, \ @@ -177,7 +183,7 @@ .has_3d_pipeline = 1, \ .hws_needs_physical = 1, \ .unfenced_needs_alignment = 1, \ - .platform_engine_mask = BIT(RCS0), \ + .__runtime.platform_engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = false, \ .dma_mask_size = 32, \ @@ -189,8 +195,8 @@ #define I845_FEATURES \ GEN(2), \ - .display.pipe_mask = BIT(PIPE_A), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A), \ + .__runtime.pipe_mask = BIT(PIPE_A), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A), \ .display.has_overlay = 1, \ .display.overlay_needs_physical = 1, \ .display.has_gmch = 1, \ @@ -198,7 +204,7 @@ .gpu_reset_clobbers_display = true, \ .hws_needs_physical = 1, \ .unfenced_needs_alignment = 1, \ - .platform_engine_mask = BIT(RCS0), \ + .__runtime.platform_engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = false, \ .dma_mask_size = 32, \ @@ -221,22 +227,22 @@ static const struct intel_device_info i845g_info = { static const struct intel_device_info i85x_info = { I830_FEATURES, PLATFORM(INTEL_I85X), - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), }; static const struct intel_device_info i865g_info = { I845_FEATURES, PLATFORM(INTEL_I865G), - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), }; #define GEN3_FEATURES \ GEN(3), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_gmch = 1, \ .gpu_reset_clobbers_display = true, \ - .platform_engine_mask = BIT(RCS0), \ + .__runtime.platform_engine_mask = BIT(RCS0), \ .has_3d_pipeline = 1, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -266,7 +272,7 @@ static const struct intel_device_info i915gm_info = { .display.has_overlay = 1, .display.overlay_needs_physical = 1, .display.supports_tv = 1, - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -291,7 +297,7 @@ static const struct intel_device_info i945gm_info = { .display.has_overlay = 1, .display.overlay_needs_physical = 1, .display.supports_tv = 1, - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), .hws_needs_physical = 1, .unfenced_needs_alignment = 1, }; @@ -323,12 +329,12 @@ static const struct intel_device_info pnv_m_info = { #define GEN4_FEATURES \ GEN(4), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ .display.has_gmch = 1, \ .gpu_reset_clobbers_display = true, \ - .platform_engine_mask = BIT(RCS0), \ + .__runtime.platform_engine_mask = BIT(RCS0), \ .has_3d_pipeline = 1, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -351,7 +357,7 @@ static const struct intel_device_info i965gm_info = { GEN4_FEATURES, PLATFORM(INTEL_I965GM), .is_mobile = 1, - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), .display.has_overlay = 1, .display.supports_tv = 1, .hws_needs_physical = 1, @@ -361,7 +367,7 @@ static const struct intel_device_info i965gm_info = { static const struct intel_device_info g45_info = { GEN4_FEATURES, PLATFORM(INTEL_G45), - .platform_engine_mask = BIT(RCS0) | BIT(VCS0), + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0), .gpu_reset_clobbers_display = false, }; @@ -369,18 +375,18 @@ static const struct intel_device_info gm45_info = { GEN4_FEATURES, PLATFORM(INTEL_GM45), .is_mobile = 1, - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), .display.supports_tv = 1, - .platform_engine_mask = BIT(RCS0) | BIT(VCS0), + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0), .gpu_reset_clobbers_display = false, }; #define GEN5_FEATURES \ GEN(5), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ - .platform_engine_mask = BIT(RCS0) | BIT(VCS0), \ + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0), \ .has_3d_pipeline = 1, \ .has_snoop = true, \ .has_coherent_ggtt = true, \ @@ -403,16 +409,16 @@ static const struct intel_device_info ilk_m_info = { PLATFORM(INTEL_IRONLAKE), .is_mobile = 1, .has_rps = true, - .display.fbc_mask = BIT(INTEL_FBC_A), + .__runtime.fbc_mask = BIT(INTEL_FBC_A), }; #define GEN6_FEATURES \ GEN(6), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ - .display.fbc_mask = BIT(INTEL_FBC_A), \ - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ + .__runtime.fbc_mask = BIT(INTEL_FBC_A), \ + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ .has_3d_pipeline = 1, \ .has_coherent_ggtt = true, \ .has_llc = 1, \ @@ -420,8 +426,8 @@ static const struct intel_device_info ilk_m_info = { .has_rc6p = 1, \ .has_rps = true, \ .dma_mask_size = 40, \ - .ppgtt_type = INTEL_PPGTT_ALIASING, \ - .ppgtt_size = 31, \ + .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \ + .__runtime.ppgtt_size = 31, \ I9XX_PIPE_OFFSETS, \ I9XX_CURSOR_OFFSETS, \ ILK_COLORS, \ @@ -460,11 +466,11 @@ static const struct intel_device_info snb_m_gt2_info = { #define GEN7_FEATURES \ GEN(7), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \ .display.has_hotplug = 1, \ - .display.fbc_mask = BIT(INTEL_FBC_A), \ - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ + .__runtime.fbc_mask = BIT(INTEL_FBC_A), \ + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ .has_3d_pipeline = 1, \ .has_coherent_ggtt = true, \ .has_llc = 1, \ @@ -473,8 +479,8 @@ static const struct intel_device_info snb_m_gt2_info = { .has_reset_engine = true, \ .has_rps = true, \ .dma_mask_size = 40, \ - .ppgtt_type = INTEL_PPGTT_ALIASING, \ - .ppgtt_size = 31, \ + .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \ + .__runtime.ppgtt_size = 31, \ IVB_PIPE_OFFSETS, \ IVB_CURSOR_OFFSETS, \ IVB_COLORS, \ @@ -515,9 +521,8 @@ static const struct intel_device_info ivb_m_gt2_info = { static const struct intel_device_info ivb_q_info = { GEN7_FEATURES, PLATFORM(INTEL_IVYBRIDGE), + NO_DISPLAY, .gt = 2, - .display.pipe_mask = 0, /* legal, last one wins */ - .display.cpu_transcoder_mask = 0, .has_l3_dpf = 1, }; @@ -525,8 +530,8 @@ static const struct intel_device_info vlv_info = { PLATFORM(INTEL_VALLEYVIEW), GEN(7), .is_lp = 1, - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), .has_runtime_pm = 1, .has_rc6 = 1, .has_reset_engine = true, @@ -534,11 +539,11 @@ static const struct intel_device_info vlv_info = { .display.has_gmch = 1, .display.has_hotplug = 1, .dma_mask_size = 40, - .ppgtt_type = INTEL_PPGTT_ALIASING, - .ppgtt_size = 31, + .__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, + .__runtime.ppgtt_size = 31, .has_snoop = true, .has_coherent_ggtt = false, - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), .display.mmio_offset = VLV_DISPLAY_BASE, I9XX_PIPE_OFFSETS, I9XX_CURSOR_OFFSETS, @@ -549,8 +554,8 @@ static const struct intel_device_info vlv_info = { #define G75_FEATURES \ GEN7_FEATURES, \ - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP), \ .display.has_ddi = 1, \ .display.has_fpga_dbg = 1, \ @@ -584,8 +589,8 @@ static const struct intel_device_info hsw_gt3_info = { GEN(8), \ .has_logical_ring_contexts = 1, \ .dma_mask_size = 39, \ - .ppgtt_type = INTEL_PPGTT_FULL, \ - .ppgtt_size = 48, \ + .__runtime.ppgtt_type = INTEL_PPGTT_FULL, \ + .__runtime.ppgtt_size = 48, \ .has_64bit_reloc = 1 #define BDW_PLATFORM \ @@ -613,18 +618,18 @@ static const struct intel_device_info bdw_rsvd_info = { static const struct intel_device_info bdw_gt3_info = { BDW_PLATFORM, .gt = 3, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), }; static const struct intel_device_info chv_info = { PLATFORM(INTEL_CHERRYVIEW), GEN(8), - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), .display.has_hotplug = 1, .is_lp = 1, - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), .has_64bit_reloc = 1, .has_runtime_pm = 1, .has_rc6 = 1, @@ -632,8 +637,8 @@ static const struct intel_device_info chv_info = { .has_logical_ring_contexts = 1, .display.has_gmch = 1, .dma_mask_size = 39, - .ppgtt_type = INTEL_PPGTT_FULL, - .ppgtt_size = 32, + .__runtime.ppgtt_type = INTEL_PPGTT_FULL, + .__runtime.ppgtt_size = 32, .has_reset_engine = 1, .has_snoop = true, .has_coherent_ggtt = false, @@ -646,16 +651,16 @@ static const struct intel_device_info chv_info = { }; #define GEN9_DEFAULT_PAGE_SIZES \ - .page_sizes = I915_GTT_PAGE_SIZE_4K | \ - I915_GTT_PAGE_SIZE_64K + .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \ + I915_GTT_PAGE_SIZE_64K #define GEN9_FEATURES \ GEN8_FEATURES, \ GEN(9), \ GEN9_DEFAULT_PAGE_SIZES, \ - .display.has_dmc = 1, \ + .__runtime.has_dmc = 1, \ .has_gt_uc = 1, \ - .display.has_hdcp = 1, \ + .__runtime.has_hdcp = 1, \ .display.has_ipc = 1, \ .display.has_psr = 1, \ .display.has_psr_hw_tracking = 1, \ @@ -678,7 +683,7 @@ static const struct intel_device_info skl_gt2_info = { #define SKL_GT3_PLUS_PLATFORM \ SKL_PLATFORM, \ - .platform_engine_mask = \ + .__runtime.platform_engine_mask = \ BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1) @@ -697,29 +702,29 @@ static const struct intel_device_info skl_gt4_info = { .is_lp = 1, \ .display.dbuf.slice_mask = BIT(DBUF_S1), \ .display.has_hotplug = 1, \ - .platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \ BIT(TRANSCODER_DSI_A) | BIT(TRANSCODER_DSI_C), \ .has_3d_pipeline = 1, \ .has_64bit_reloc = 1, \ .display.has_ddi = 1, \ .display.has_fpga_dbg = 1, \ - .display.fbc_mask = BIT(INTEL_FBC_A), \ - .display.has_hdcp = 1, \ + .__runtime.fbc_mask = BIT(INTEL_FBC_A), \ + .__runtime.has_hdcp = 1, \ .display.has_psr = 1, \ .display.has_psr_hw_tracking = 1, \ .has_runtime_pm = 1, \ - .display.has_dmc = 1, \ + .__runtime.has_dmc = 1, \ .has_rc6 = 1, \ .has_rps = true, \ .display.has_dp_mst = 1, \ .has_logical_ring_contexts = 1, \ .has_gt_uc = 1, \ .dma_mask_size = 39, \ - .ppgtt_type = INTEL_PPGTT_FULL, \ - .ppgtt_size = 48, \ + .__runtime.ppgtt_type = INTEL_PPGTT_FULL, \ + .__runtime.ppgtt_size = 48, \ .has_reset_engine = 1, \ .has_snoop = true, \ .has_coherent_ggtt = false, \ @@ -739,7 +744,7 @@ static const struct intel_device_info bxt_info = { static const struct intel_device_info glk_info = { GEN9_LP_FEATURES, PLATFORM(INTEL_GEMINILAKE), - .display.ver = 10, + .__runtime.display.ip.ver = 10, .display.dbuf.size = 1024 - 4, /* 4 blocks for bypass path allocation */ GLK_COLORS, }; @@ -761,7 +766,7 @@ static const struct intel_device_info kbl_gt2_info = { static const struct intel_device_info kbl_gt3_info = { KBL_PLATFORM, .gt = 3, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), }; @@ -782,7 +787,7 @@ static const struct intel_device_info cfl_gt2_info = { static const struct intel_device_info cfl_gt3_info = { CFL_PLATFORM, .gt = 3, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1), }; @@ -801,15 +806,15 @@ static const struct intel_device_info cml_gt2_info = { }; #define GEN11_DEFAULT_PAGE_SIZES \ - .page_sizes = I915_GTT_PAGE_SIZE_4K | \ - I915_GTT_PAGE_SIZE_64K | \ - I915_GTT_PAGE_SIZE_2M + .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \ + I915_GTT_PAGE_SIZE_64K | \ + I915_GTT_PAGE_SIZE_2M #define GEN11_FEATURES \ GEN9_FEATURES, \ GEN11_DEFAULT_PAGE_SIZES, \ .display.abox_mask = BIT(0), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \ BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \ .display.pipe_offsets = { \ @@ -832,37 +837,37 @@ static const struct intel_device_info cml_gt2_info = { ICL_COLORS, \ .display.dbuf.size = 2048, \ .display.dbuf.slice_mask = BIT(DBUF_S1) | BIT(DBUF_S2), \ - .display.has_dsc = 1, \ + .__runtime.has_dsc = 1, \ .has_coherent_ggtt = false, \ .has_logical_ring_elsq = 1 static const struct intel_device_info icl_info = { GEN11_FEATURES, PLATFORM(INTEL_ICELAKE), - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), }; static const struct intel_device_info ehl_info = { GEN11_FEATURES, PLATFORM(INTEL_ELKHARTLAKE), - .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0), - .ppgtt_size = 36, + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0), + .__runtime.ppgtt_size = 36, }; static const struct intel_device_info jsl_info = { GEN11_FEATURES, PLATFORM(INTEL_JASPERLAKE), - .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0), - .ppgtt_size = 36, + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0), + .__runtime.ppgtt_size = 36, }; #define GEN12_FEATURES \ GEN11_FEATURES, \ GEN(12), \ .display.abox_mask = GENMASK(2, 1), \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \ - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \ + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ BIT(TRANSCODER_C) | BIT(TRANSCODER_D) | \ BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \ .display.pipe_offsets = { \ @@ -890,7 +895,7 @@ static const struct intel_device_info tgl_info = { GEN12_FEATURES, PLATFORM(INTEL_TIGERLAKE), .display.has_modular_fia = 1, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), }; @@ -898,17 +903,17 @@ static const struct intel_device_info rkl_info = { GEN12_FEATURES, PLATFORM(INTEL_ROCKETLAKE), .display.abox_mask = BIT(0), - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), .display.has_hti = 1, .display.has_psr_hw_tracking = 0, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0), }; #define DGFX_FEATURES \ - .memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ + .__runtime.memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \ .has_llc = 0, \ .has_pxp = 0, \ .has_snoop = 1, \ @@ -918,24 +923,24 @@ static const struct intel_device_info rkl_info = { static const struct intel_device_info dg1_info = { GEN12_FEATURES, DGFX_FEATURES, - .graphics.rel = 10, + .__runtime.graphics.ip.rel = 10, PLATFORM(INTEL_DG1), - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), .require_force_probe = 1, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), /* Wa_16011227922 */ - .ppgtt_size = 47, + .__runtime.ppgtt_size = 47, }; static const struct intel_device_info adl_s_info = { GEN12_FEATURES, PLATFORM(INTEL_ALDERLAKE_S), - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), .display.has_hti = 1, .display.has_psr_hw_tracking = 0, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), .dma_mask_size = 39, }; @@ -951,18 +956,18 @@ static const struct intel_device_info adl_s_info = { .display.dbuf.slice_mask = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | \ BIT(DBUF_S4), \ .display.has_ddi = 1, \ - .display.has_dmc = 1, \ + .__runtime.has_dmc = 1, \ .display.has_dp_mst = 1, \ .display.has_dsb = 1, \ - .display.has_dsc = 1, \ - .display.fbc_mask = BIT(INTEL_FBC_A), \ + .__runtime.has_dsc = 1, \ + .__runtime.fbc_mask = BIT(INTEL_FBC_A), \ .display.has_fpga_dbg = 1, \ - .display.has_hdcp = 1, \ + .__runtime.has_hdcp = 1, \ .display.has_hotplug = 1, \ .display.has_ipc = 1, \ .display.has_psr = 1, \ - .display.ver = 13, \ - .display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \ + .__runtime.display.ip.ver = 13, \ + .__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \ .display.pipe_offsets = { \ [TRANSCODER_A] = PIPE_A_OFFSET, \ [TRANSCODER_B] = PIPE_B_OFFSET, \ @@ -985,28 +990,28 @@ static const struct intel_device_info adl_p_info = { GEN12_FEATURES, XE_LPD_FEATURES, PLATFORM(INTEL_ALDERLAKE_P), - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C) | BIT(TRANSCODER_D) | BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), .display.has_cdclk_crawl = 1, .display.has_modular_fia = 1, .display.has_psr_hw_tracking = 0, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), - .ppgtt_size = 48, + .__runtime.ppgtt_size = 48, .dma_mask_size = 39, }; #undef GEN #define XE_HP_PAGE_SIZES \ - .page_sizes = I915_GTT_PAGE_SIZE_4K | \ - I915_GTT_PAGE_SIZE_64K | \ - I915_GTT_PAGE_SIZE_2M + .__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \ + I915_GTT_PAGE_SIZE_64K | \ + I915_GTT_PAGE_SIZE_2M #define XE_HP_FEATURES \ - .graphics.ver = 12, \ - .graphics.rel = 50, \ + .__runtime.graphics.ip.ver = 12, \ + .__runtime.graphics.ip.rel = 50, \ XE_HP_PAGE_SIZES, \ .dma_mask_size = 46, \ .has_3d_pipeline = 1, \ @@ -1022,12 +1027,12 @@ static const struct intel_device_info adl_p_info = { .has_reset_engine = 1, \ .has_rps = 1, \ .has_runtime_pm = 1, \ - .ppgtt_size = 48, \ - .ppgtt_type = INTEL_PPGTT_FULL + .__runtime.ppgtt_size = 48, \ + .__runtime.ppgtt_type = INTEL_PPGTT_FULL #define XE_HPM_FEATURES \ - .media.ver = 12, \ - .media.rel = 50 + .__runtime.media.ip.ver = 12, \ + .__runtime.media.ip.rel = 50 __maybe_unused static const struct intel_device_info xehpsdv_info = { @@ -1035,11 +1040,11 @@ static const struct intel_device_info xehpsdv_info = { XE_HPM_FEATURES, DGFX_FEATURES, PLATFORM(INTEL_XEHPSDV), - .display = { }, + NO_DISPLAY, .has_64k_pages = 1, .needs_compact_pt = 1, .has_media_ratio_mode = 1, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) | BIT(VCS0) | BIT(VCS1) | BIT(VCS2) | BIT(VCS3) | @@ -1052,8 +1057,8 @@ static const struct intel_device_info xehpsdv_info = { XE_HP_FEATURES, \ XE_HPM_FEATURES, \ DGFX_FEATURES, \ - .graphics.rel = 55, \ - .media.rel = 55, \ + .__runtime.graphics.ip.rel = 55, \ + .__runtime.media.ip.rel = 55, \ PLATFORM(INTEL_DG2), \ .has_4tile = 1, \ .has_64k_pages = 1, \ @@ -1061,7 +1066,7 @@ static const struct intel_device_info xehpsdv_info = { .has_heci_pxp = 1, \ .needs_compact_pt = 1, \ .has_media_ratio_mode = 1, \ - .platform_engine_mask = \ + .__runtime.platform_engine_mask = \ BIT(RCS0) | BIT(BCS0) | \ BIT(VECS0) | BIT(VECS1) | \ BIT(VCS0) | BIT(VCS2) | \ @@ -1070,14 +1075,14 @@ static const struct intel_device_info xehpsdv_info = { static const struct intel_device_info dg2_info = { DG2_FEATURES, XE_LPD_FEATURES, - .display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | + .__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C) | BIT(TRANSCODER_D), .require_force_probe = 1, }; static const struct intel_device_info ats_m_info = { DG2_FEATURES, - .display = { 0 }, + NO_DISPLAY, .require_force_probe = 1, .tuning_thread_rr_after_dep = 1, }; @@ -1096,12 +1101,12 @@ static const struct intel_device_info pvc_info = { XE_HPC_FEATURES, XE_HPM_FEATURES, DGFX_FEATURES, - .graphics.rel = 60, - .media.rel = 60, + .__runtime.graphics.ip.rel = 60, + .__runtime.media.ip.rel = 60, PLATFORM(INTEL_PONTEVECCHIO), - .display = { 0 }, + NO_DISPLAY, .has_flat_ccs = 0, - .platform_engine_mask = + .__runtime.platform_engine_mask = BIT(BCS0) | BIT(VCS0) | BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3), @@ -1110,8 +1115,19 @@ static const struct intel_device_info pvc_info = { #define XE_LPDP_FEATURES \ XE_LPD_FEATURES, \ - .display.ver = 14, \ - .display.has_cdclk_crawl = 1 + .__runtime.display.ip.ver = 14, \ + .display.has_cdclk_crawl = 1, \ + .__runtime.fbc_mask = BIT(INTEL_FBC_A) | BIT(INTEL_FBC_B) + +static const struct intel_gt_definition xelpmp_extra_gt[] = { + { + .type = GT_MEDIA, + .name = "Standalone Media GT", + .gsi_offset = MTL_MEDIA_GSI_BASE, + .engine_mask = BIT(VECS0) | BIT(VCS0) | BIT(VCS2), + }, + {} +}; __maybe_unused static const struct intel_device_info mtl_info = { @@ -1121,15 +1137,16 @@ static const struct intel_device_info mtl_info = { * Real graphics IP version will be obtained from hardware GMD_ID * register. Value provided here is just for sanity checking. */ - .graphics.ver = 12, - .graphics.rel = 70, - .media.ver = 13, + .__runtime.graphics.ip.ver = 12, + .__runtime.graphics.ip.rel = 70, + .__runtime.media.ip.ver = 13, PLATFORM(INTEL_METEORLAKE), .display.has_modular_fia = 1, + .extra_gt_list = xelpmp_extra_gt, .has_flat_ccs = 0, .has_snoop = 1, - .memory_regions = REGION_SMEM | REGION_STOLEN_LMEM, - .platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0), + .__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_LMEM, + .__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0), .require_force_probe = 1, }; @@ -1263,6 +1280,27 @@ static bool force_probe(u16 device_id, const char *devices) return ret; } +bool i915_pci_resource_valid(struct pci_dev *pdev, int bar) +{ + if (!pci_resource_flags(pdev, bar)) + return false; + + if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET) + return false; + + if (!pci_resource_len(pdev, bar)) + return false; + + return true; +} + +static bool intel_mmio_bar_valid(struct pci_dev *pdev, struct intel_device_info *intel_info) +{ + int gttmmaddr_bar = intel_info->__runtime.graphics.ip.ver == 2 ? GEN2_GTTMMADR_BAR : GTTMMADR_BAR; + + return i915_pci_resource_valid(pdev, gttmmaddr_bar); +} + static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct intel_device_info *intel_info = @@ -1288,6 +1326,9 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (PCI_FUNC(pdev->devfn)) return -ENODEV; + if (!intel_mmio_bar_valid(pdev, intel_info)) + return -ENXIO; + /* Detect if we need to wait for other drivers early on */ if (intel_modeset_probe_defer(pdev)) return -EPROBE_DEFER; diff --git a/drivers/gpu/drm/i915/i915_pci.h b/drivers/gpu/drm/i915/i915_pci.h index ee048c2381740d390ccb0a35bd20b89002e67bff..8dfe19f9a7751f325d8802fb370a1433e799f013 100644 --- a/drivers/gpu/drm/i915/i915_pci.h +++ b/drivers/gpu/drm/i915/i915_pci.h @@ -6,7 +6,13 @@ #ifndef __I915_PCI_H__ #define __I915_PCI_H__ +#include + +struct pci_dev; + int i915_pci_register_driver(void); void i915_pci_unregister_driver(void); +bool i915_pci_resource_valid(struct pci_dev *pdev, int bar); + #endif /* __I915_PCI_H__ */ diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index f3c23fe9ad9ce703c39355d1c563f9e8b390ba2e..0defbb43ceea8365f772d41681f9490df1b869fb 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -1376,7 +1376,8 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream) { struct i915_perf *perf = stream->perf; - BUG_ON(stream != perf->exclusive_stream); + if (WARN_ON(stream != perf->exclusive_stream)) + return; /* * Unset exclusive_stream first, it will be checked while disabling diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 3168d7007e10177bdb15da551c1f5e08caa458f0..0b287a59dc2f4350b501367b6c01283d6ef6b571 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -1125,8 +1125,12 @@ #define MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN REG_BIT(16) /* tgl+ */ #define MBUS_DBOX_BW_CREDIT_MASK REG_GENMASK(15, 14) #define MBUS_DBOX_BW_CREDIT(x) REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, x) +#define MBUS_DBOX_BW_4CREDITS_MTL REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x2) +#define MBUS_DBOX_BW_8CREDITS_MTL REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x3) #define MBUS_DBOX_B_CREDIT_MASK REG_GENMASK(12, 8) #define MBUS_DBOX_B_CREDIT(x) REG_FIELD_PREP(MBUS_DBOX_B_CREDIT_MASK, x) +#define MBUS_DBOX_I_CREDIT_MASK REG_GENMASK(7, 5) +#define MBUS_DBOX_I_CREDIT(x) REG_FIELD_PREP(MBUS_DBOX_I_CREDIT_MASK, x) #define MBUS_DBOX_A_CREDIT_MASK REG_GENMASK(3, 0) #define MBUS_DBOX_A_CREDIT(x) REG_FIELD_PREP(MBUS_DBOX_A_CREDIT_MASK, x) @@ -1461,69 +1465,6 @@ #define FBC_REND_NUKE REG_BIT(2) #define FBC_REND_CACHE_CLEAN REG_BIT(1) -/* - * GPIO regs - */ -#define GPIO(gpio) _MMIO(dev_priv->gpio_mmio_base + 0x5010 + \ - 4 * (gpio)) - -# define GPIO_CLOCK_DIR_MASK (1 << 0) -# define GPIO_CLOCK_DIR_IN (0 << 1) -# define GPIO_CLOCK_DIR_OUT (1 << 1) -# define GPIO_CLOCK_VAL_MASK (1 << 2) -# define GPIO_CLOCK_VAL_OUT (1 << 3) -# define GPIO_CLOCK_VAL_IN (1 << 4) -# define GPIO_CLOCK_PULLUP_DISABLE (1 << 5) -# define GPIO_DATA_DIR_MASK (1 << 8) -# define GPIO_DATA_DIR_IN (0 << 9) -# define GPIO_DATA_DIR_OUT (1 << 9) -# define GPIO_DATA_VAL_MASK (1 << 10) -# define GPIO_DATA_VAL_OUT (1 << 11) -# define GPIO_DATA_VAL_IN (1 << 12) -# define GPIO_DATA_PULLUP_DISABLE (1 << 13) - -#define GMBUS0 _MMIO(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */ -#define GMBUS_AKSV_SELECT (1 << 11) -#define GMBUS_RATE_100KHZ (0 << 8) -#define GMBUS_RATE_50KHZ (1 << 8) -#define GMBUS_RATE_400KHZ (2 << 8) /* reserved on Pineview */ -#define GMBUS_RATE_1MHZ (3 << 8) /* reserved on Pineview */ -#define GMBUS_HOLD_EXT (1 << 7) /* 300ns hold time, rsvd on Pineview */ -#define GMBUS_BYTE_CNT_OVERRIDE (1 << 6) - -#define GMBUS1 _MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */ -#define GMBUS_SW_CLR_INT (1 << 31) -#define GMBUS_SW_RDY (1 << 30) -#define GMBUS_ENT (1 << 29) /* enable timeout */ -#define GMBUS_CYCLE_NONE (0 << 25) -#define GMBUS_CYCLE_WAIT (1 << 25) -#define GMBUS_CYCLE_INDEX (2 << 25) -#define GMBUS_CYCLE_STOP (4 << 25) -#define GMBUS_BYTE_COUNT_SHIFT 16 -#define GMBUS_BYTE_COUNT_MAX 256U -#define GEN9_GMBUS_BYTE_COUNT_MAX 511U -#define GMBUS_SLAVE_INDEX_SHIFT 8 -#define GMBUS_SLAVE_ADDR_SHIFT 1 -#define GMBUS_SLAVE_READ (1 << 0) -#define GMBUS_SLAVE_WRITE (0 << 0) -#define GMBUS2 _MMIO(dev_priv->gpio_mmio_base + 0x5108) /* status */ -#define GMBUS_INUSE (1 << 15) -#define GMBUS_HW_WAIT_PHASE (1 << 14) -#define GMBUS_STALL_TIMEOUT (1 << 13) -#define GMBUS_INT (1 << 12) -#define GMBUS_HW_RDY (1 << 11) -#define GMBUS_SATOER (1 << 10) -#define GMBUS_ACTIVE (1 << 9) -#define GMBUS3 _MMIO(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */ -#define GMBUS4 _MMIO(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */ -#define GMBUS_SLAVE_TIMEOUT_EN (1 << 4) -#define GMBUS_NAK_EN (1 << 3) -#define GMBUS_IDLE_EN (1 << 2) -#define GMBUS_HW_WAIT_EN (1 << 1) -#define GMBUS_HW_RDY_EN (1 << 0) -#define GMBUS5 _MMIO(dev_priv->gpio_mmio_base + 0x5120) /* byte index */ -#define GMBUS_2BYTE_INDEX_EN (1 << 31) - /* * Clock control & power management */ @@ -1700,7 +1641,7 @@ #define DSTATE_PLL_D3_OFF (1 << 3) #define DSTATE_GFX_CLOCK_GATING (1 << 1) #define DSTATE_DOT_CLOCK_GATING (1 << 0) -#define DSPCLK_GATE_D _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x6200) +#define DSPCLK_GATE_D(__i915) _MMIO(DISPLAY_MMIO_BASE(__i915) + 0x6200) # define DPUNIT_B_CLOCK_GATE_DISABLE (1 << 30) /* 965 */ # define VSUNIT_CLOCK_GATE_DISABLE (1 << 29) /* 965 */ # define VRHUNIT_CLOCK_GATE_DISABLE (1 << 28) /* 965 */ @@ -1857,14 +1798,14 @@ #define GT0_PERF_LIMIT_REASONS _MMIO(0x1381a8) #define GT0_PERF_LIMIT_REASONS_MASK 0xde3 -#define PROCHOT_MASK REG_BIT(1) -#define THERMAL_LIMIT_MASK REG_BIT(2) -#define RATL_MASK REG_BIT(6) -#define VR_THERMALERT_MASK REG_BIT(7) -#define VR_TDC_MASK REG_BIT(8) -#define POWER_LIMIT_4_MASK REG_BIT(9) -#define POWER_LIMIT_1_MASK REG_BIT(11) -#define POWER_LIMIT_2_MASK REG_BIT(12) +#define PROCHOT_MASK REG_BIT(0) +#define THERMAL_LIMIT_MASK REG_BIT(1) +#define RATL_MASK REG_BIT(5) +#define VR_THERMALERT_MASK REG_BIT(6) +#define VR_TDC_MASK REG_BIT(7) +#define POWER_LIMIT_4_MASK REG_BIT(8) +#define POWER_LIMIT_1_MASK REG_BIT(10) +#define POWER_LIMIT_2_MASK REG_BIT(11) #define CHV_CLK_CTL1 _MMIO(0x101100) #define VLV_CLK_CTL2 _MMIO(0x101104) @@ -1916,6 +1857,13 @@ #define CLKGATE_DIS_PSL(pipe) \ _MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_A, _CLKGATE_DIS_PSL_B) +#define _CLKGATE_DIS_PSL_EXT_A 0x4654C +#define _CLKGATE_DIS_PSL_EXT_B 0x46550 +#define PIPEDMC_GATING_DIS REG_BIT(12) + +#define CLKGATE_DIS_PSL_EXT(pipe) \ + _MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_EXT_A, _CLKGATE_DIS_PSL_EXT_B) + /* * Display engine regs */ @@ -2209,10 +2157,18 @@ #define TRANS_PSR_IIR(tran) _MMIO_TRANS2(tran, _PSR_IIR_A) #define _EDP_PSR_TRANS_SHIFT(trans) ((trans) == TRANSCODER_EDP ? \ 0 : ((trans) - TRANSCODER_A + 1) * 8) -#define EDP_PSR_TRANS_MASK(trans) (0x7 << _EDP_PSR_TRANS_SHIFT(trans)) -#define EDP_PSR_ERROR(trans) (0x4 << _EDP_PSR_TRANS_SHIFT(trans)) -#define EDP_PSR_POST_EXIT(trans) (0x2 << _EDP_PSR_TRANS_SHIFT(trans)) -#define EDP_PSR_PRE_ENTRY(trans) (0x1 << _EDP_PSR_TRANS_SHIFT(trans)) +#define TGL_PSR_MASK REG_GENMASK(2, 0) +#define TGL_PSR_ERROR REG_BIT(2) +#define TGL_PSR_POST_EXIT REG_BIT(1) +#define TGL_PSR_PRE_ENTRY REG_BIT(0) +#define EDP_PSR_MASK(trans) (TGL_PSR_MASK << \ + _EDP_PSR_TRANS_SHIFT(trans)) +#define EDP_PSR_ERROR(trans) (TGL_PSR_ERROR << \ + _EDP_PSR_TRANS_SHIFT(trans)) +#define EDP_PSR_POST_EXIT(trans) (TGL_PSR_POST_EXIT << \ + _EDP_PSR_TRANS_SHIFT(trans)) +#define EDP_PSR_PRE_ENTRY(trans) (TGL_PSR_PRE_ENTRY << \ + _EDP_PSR_TRANS_SHIFT(trans)) #define _SRD_AUX_DATA_A 0x60814 #define _SRD_AUX_DATA_EDP 0x6f814 @@ -2822,7 +2778,7 @@ #define VLV_PPS_BASE (VLV_DISPLAY_BASE + PPS_BASE) #define PCH_PPS_BASE 0xC7200 -#define _MMIO_PPS(pps_idx, reg) _MMIO(dev_priv->pps_mmio_base - \ +#define _MMIO_PPS(pps_idx, reg) _MMIO(dev_priv->display.pps.mmio_base - \ PPS_BASE + (reg) + \ (pps_idx) * 0x100) @@ -2918,118 +2874,6 @@ #define PFIT_AUTO_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61238) -#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250) -#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350) -#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \ - _VLV_BLC_PWM_CTL2_B) - -#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254) -#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354) -#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \ - _VLV_BLC_PWM_CTL_B) - -#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260) -#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360) -#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \ - _VLV_BLC_HIST_CTL_B) - -/* Backlight control */ -#define BLC_PWM_CTL2 _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */ -#define BLM_PWM_ENABLE (1 << 31) -#define BLM_COMBINATION_MODE (1 << 30) /* gen4 only */ -#define BLM_PIPE_SELECT (1 << 29) -#define BLM_PIPE_SELECT_IVB (3 << 29) -#define BLM_PIPE_A (0 << 29) -#define BLM_PIPE_B (1 << 29) -#define BLM_PIPE_C (2 << 29) /* ivb + */ -#define BLM_TRANSCODER_A BLM_PIPE_A /* hsw */ -#define BLM_TRANSCODER_B BLM_PIPE_B -#define BLM_TRANSCODER_C BLM_PIPE_C -#define BLM_TRANSCODER_EDP (3 << 29) -#define BLM_PIPE(pipe) ((pipe) << 29) -#define BLM_POLARITY_I965 (1 << 28) /* gen4 only */ -#define BLM_PHASE_IN_INTERUPT_STATUS (1 << 26) -#define BLM_PHASE_IN_ENABLE (1 << 25) -#define BLM_PHASE_IN_INTERUPT_ENABL (1 << 24) -#define BLM_PHASE_IN_TIME_BASE_SHIFT (16) -#define BLM_PHASE_IN_TIME_BASE_MASK (0xff << 16) -#define BLM_PHASE_IN_COUNT_SHIFT (8) -#define BLM_PHASE_IN_COUNT_MASK (0xff << 8) -#define BLM_PHASE_IN_INCR_SHIFT (0) -#define BLM_PHASE_IN_INCR_MASK (0xff << 0) -#define BLC_PWM_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254) -/* - * This is the most significant 15 bits of the number of backlight cycles in a - * complete cycle of the modulated backlight control. - * - * The actual value is this field multiplied by two. - */ -#define BACKLIGHT_MODULATION_FREQ_SHIFT (17) -#define BACKLIGHT_MODULATION_FREQ_MASK (0x7fff << 17) -#define BLM_LEGACY_MODE (1 << 16) /* gen2 only */ -/* - * This is the number of cycles out of the backlight modulation cycle for which - * the backlight is on. - * - * This field must be no greater than the number of cycles in the complete - * backlight modulation cycle. - */ -#define BACKLIGHT_DUTY_CYCLE_SHIFT (0) -#define BACKLIGHT_DUTY_CYCLE_MASK (0xffff) -#define BACKLIGHT_DUTY_CYCLE_MASK_PNV (0xfffe) -#define BLM_POLARITY_PNV (1 << 0) /* pnv only */ - -#define BLC_HIST_CTL _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260) -#define BLM_HISTOGRAM_ENABLE (1 << 31) - -/* New registers for PCH-split platforms. Safe where new bits show up, the - * register layout machtes with gen4 BLC_PWM_CTL[12]. */ -#define BLC_PWM_CPU_CTL2 _MMIO(0x48250) -#define BLC_PWM_CPU_CTL _MMIO(0x48254) - -#define HSW_BLC_PWM2_CTL _MMIO(0x48350) - -/* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is - * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */ -#define BLC_PWM_PCH_CTL1 _MMIO(0xc8250) -#define BLM_PCH_PWM_ENABLE (1 << 31) -#define BLM_PCH_OVERRIDE_ENABLE (1 << 30) -#define BLM_PCH_POLARITY (1 << 29) -#define BLC_PWM_PCH_CTL2 _MMIO(0xc8254) - -#define UTIL_PIN_CTL _MMIO(0x48400) -#define UTIL_PIN_ENABLE (1 << 31) -#define UTIL_PIN_PIPE_MASK (3 << 29) -#define UTIL_PIN_PIPE(x) ((x) << 29) -#define UTIL_PIN_MODE_MASK (0xf << 24) -#define UTIL_PIN_MODE_DATA (0 << 24) -#define UTIL_PIN_MODE_PWM (1 << 24) -#define UTIL_PIN_MODE_VBLANK (4 << 24) -#define UTIL_PIN_MODE_VSYNC (5 << 24) -#define UTIL_PIN_MODE_EYE_LEVEL (8 << 24) -#define UTIL_PIN_OUTPUT_DATA (1 << 23) -#define UTIL_PIN_POLARITY (1 << 22) -#define UTIL_PIN_DIRECTION_INPUT (1 << 19) -#define UTIL_PIN_INPUT_DATA (1 << 16) - -/* BXT backlight register definition. */ -#define _BXT_BLC_PWM_CTL1 0xC8250 -#define BXT_BLC_PWM_ENABLE (1 << 31) -#define BXT_BLC_PWM_POLARITY (1 << 29) -#define _BXT_BLC_PWM_FREQ1 0xC8254 -#define _BXT_BLC_PWM_DUTY1 0xC8258 - -#define _BXT_BLC_PWM_CTL2 0xC8350 -#define _BXT_BLC_PWM_FREQ2 0xC8354 -#define _BXT_BLC_PWM_DUTY2 0xC8358 - -#define BXT_BLC_PWM_CTL(controller) _MMIO_PIPE(controller, \ - _BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2) -#define BXT_BLC_PWM_FREQ(controller) _MMIO_PIPE(controller, \ - _BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2) -#define BXT_BLC_PWM_DUTY(controller) _MMIO_PIPE(controller, \ - _BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2) - #define PCH_GTC_CTL _MMIO(0xe7000) #define PCH_GTC_ENABLE (1 << 31) @@ -3619,6 +3463,34 @@ #define DP_AUX_CH_CTL(aux_ch) _MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL) #define DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */ +#define _XELPDP_USBC1_AUX_CH_CTL 0x16F210 +#define _XELPDP_USBC2_AUX_CH_CTL 0x16F410 +#define _XELPDP_USBC3_AUX_CH_CTL 0x16F610 +#define _XELPDP_USBC4_AUX_CH_CTL 0x16F810 + +#define XELPDP_DP_AUX_CH_CTL(aux_ch) _MMIO(_PICK(aux_ch, \ + _DPA_AUX_CH_CTL, \ + _DPB_AUX_CH_CTL, \ + 0, /* port/aux_ch C is non-existent */ \ + _XELPDP_USBC1_AUX_CH_CTL, \ + _XELPDP_USBC2_AUX_CH_CTL, \ + _XELPDP_USBC3_AUX_CH_CTL, \ + _XELPDP_USBC4_AUX_CH_CTL)) + +#define _XELPDP_USBC1_AUX_CH_DATA1 0x16F214 +#define _XELPDP_USBC2_AUX_CH_DATA1 0x16F414 +#define _XELPDP_USBC3_AUX_CH_DATA1 0x16F614 +#define _XELPDP_USBC4_AUX_CH_DATA1 0x16F814 + +#define XELPDP_DP_AUX_CH_DATA(aux_ch, i) _MMIO(_PICK(aux_ch, \ + _DPA_AUX_CH_DATA1, \ + _DPB_AUX_CH_DATA1, \ + 0, /* port/aux_ch C is non-existent */ \ + _XELPDP_USBC1_AUX_CH_DATA1, \ + _XELPDP_USBC2_AUX_CH_DATA1, \ + _XELPDP_USBC3_AUX_CH_DATA1, \ + _XELPDP_USBC4_AUX_CH_DATA1) + (i) * 4) + #define DP_AUX_CH_CTL_SEND_BUSY (1 << 31) #define DP_AUX_CH_CTL_DONE (1 << 30) #define DP_AUX_CH_CTL_INTERRUPT (1 << 29) @@ -3631,6 +3503,8 @@ #define DP_AUX_CH_CTL_RECEIVE_ERROR (1 << 25) #define DP_AUX_CH_CTL_MESSAGE_SIZE_MASK (0x1f << 20) #define DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT 20 +#define XELPDP_DP_AUX_CH_CTL_POWER_REQUEST REG_BIT(19) +#define XELPDP_DP_AUX_CH_CTL_POWER_STATUS REG_BIT(18) #define DP_AUX_CH_CTL_PRECHARGE_2US_MASK (0xf << 16) #define DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT 16 #define DP_AUX_CH_CTL_AUX_AKSV_SELECT (1 << 15) @@ -5862,6 +5736,13 @@ [TRANSCODER_B] = _CHICKEN_TRANS_B, \ [TRANSCODER_C] = _CHICKEN_TRANS_C, \ [TRANSCODER_D] = _CHICKEN_TRANS_D)) + +#define _MTL_CHICKEN_TRANS_A 0x604e0 +#define _MTL_CHICKEN_TRANS_B 0x614e0 +#define MTL_CHICKEN_TRANS(trans) _MMIO_TRANS((trans), \ + _MTL_CHICKEN_TRANS_A, \ + _MTL_CHICKEN_TRANS_B) + #define HSW_FRAME_START_DELAY_MASK REG_GENMASK(28, 27) #define HSW_FRAME_START_DELAY(x) REG_FIELD_PREP(HSW_FRAME_START_DELAY_MASK, x) #define VSC_DATA_SEL_SOFTWARE_CONTROL REG_BIT(25) /* GLK */ @@ -5926,7 +5807,8 @@ _BW_BUDDY1_PAGE_MASK)) #define HSW_NDE_RSTWRN_OPT _MMIO(0x46408) -#define RESET_PCH_HANDSHAKE_ENABLE (1 << 4) +#define MTL_RESET_PICA_HANDSHAKE_EN REG_BIT(6) +#define RESET_PCH_HANDSHAKE_ENABLE REG_BIT(4) #define GEN8_CHICKEN_DCPR_1 _MMIO(0x46430) #define SKL_SELECT_ALTERNATE_DC_EXIT REG_BIT(30) @@ -6718,10 +6600,10 @@ #define GEN6_DECODE_RC6_VID(vids) (((vids) * 5) + 245) #define BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ 0x18 #define GEN9_PCODE_READ_MEM_LATENCY 0x6 -#define GEN9_MEM_LATENCY_LEVEL_MASK 0xFF -#define GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT 8 -#define GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT 16 -#define GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT 24 +#define GEN9_MEM_LATENCY_LEVEL_3_7_MASK REG_GENMASK(31, 24) +#define GEN9_MEM_LATENCY_LEVEL_2_6_MASK REG_GENMASK(23, 16) +#define GEN9_MEM_LATENCY_LEVEL_1_5_MASK REG_GENMASK(15, 8) +#define GEN9_MEM_LATENCY_LEVEL_0_4_MASK REG_GENMASK(7, 0) #define SKL_PCODE_LOAD_HDCP_KEYS 0x5 #define SKL_PCODE_CDCLK_CONTROL 0x7 #define SKL_CDCLK_PREPARE_FOR_CHANGE 0x3 @@ -6937,265 +6819,6 @@ enum skl_power_gate { #define ICL_AUX_ANAOVRD1_LDO_BYPASS (1 << 7) #define ICL_AUX_ANAOVRD1_ENABLE (1 << 0) -/* HDCP Key Registers */ -#define HDCP_KEY_CONF _MMIO(0x66c00) -#define HDCP_AKSV_SEND_TRIGGER BIT(31) -#define HDCP_CLEAR_KEYS_TRIGGER BIT(30) -#define HDCP_KEY_LOAD_TRIGGER BIT(8) -#define HDCP_KEY_STATUS _MMIO(0x66c04) -#define HDCP_FUSE_IN_PROGRESS BIT(7) -#define HDCP_FUSE_ERROR BIT(6) -#define HDCP_FUSE_DONE BIT(5) -#define HDCP_KEY_LOAD_STATUS BIT(1) -#define HDCP_KEY_LOAD_DONE BIT(0) -#define HDCP_AKSV_LO _MMIO(0x66c10) -#define HDCP_AKSV_HI _MMIO(0x66c14) - -/* HDCP Repeater Registers */ -#define HDCP_REP_CTL _MMIO(0x66d00) -#define HDCP_TRANSA_REP_PRESENT BIT(31) -#define HDCP_TRANSB_REP_PRESENT BIT(30) -#define HDCP_TRANSC_REP_PRESENT BIT(29) -#define HDCP_TRANSD_REP_PRESENT BIT(28) -#define HDCP_DDIB_REP_PRESENT BIT(30) -#define HDCP_DDIA_REP_PRESENT BIT(29) -#define HDCP_DDIC_REP_PRESENT BIT(28) -#define HDCP_DDID_REP_PRESENT BIT(27) -#define HDCP_DDIF_REP_PRESENT BIT(26) -#define HDCP_DDIE_REP_PRESENT BIT(25) -#define HDCP_TRANSA_SHA1_M0 (1 << 20) -#define HDCP_TRANSB_SHA1_M0 (2 << 20) -#define HDCP_TRANSC_SHA1_M0 (3 << 20) -#define HDCP_TRANSD_SHA1_M0 (4 << 20) -#define HDCP_DDIB_SHA1_M0 (1 << 20) -#define HDCP_DDIA_SHA1_M0 (2 << 20) -#define HDCP_DDIC_SHA1_M0 (3 << 20) -#define HDCP_DDID_SHA1_M0 (4 << 20) -#define HDCP_DDIF_SHA1_M0 (5 << 20) -#define HDCP_DDIE_SHA1_M0 (6 << 20) /* Bspec says 5? */ -#define HDCP_SHA1_BUSY BIT(16) -#define HDCP_SHA1_READY BIT(17) -#define HDCP_SHA1_COMPLETE BIT(18) -#define HDCP_SHA1_V_MATCH BIT(19) -#define HDCP_SHA1_TEXT_32 (1 << 1) -#define HDCP_SHA1_COMPLETE_HASH (2 << 1) -#define HDCP_SHA1_TEXT_24 (4 << 1) -#define HDCP_SHA1_TEXT_16 (5 << 1) -#define HDCP_SHA1_TEXT_8 (6 << 1) -#define HDCP_SHA1_TEXT_0 (7 << 1) -#define HDCP_SHA_V_PRIME_H0 _MMIO(0x66d04) -#define HDCP_SHA_V_PRIME_H1 _MMIO(0x66d08) -#define HDCP_SHA_V_PRIME_H2 _MMIO(0x66d0C) -#define HDCP_SHA_V_PRIME_H3 _MMIO(0x66d10) -#define HDCP_SHA_V_PRIME_H4 _MMIO(0x66d14) -#define HDCP_SHA_V_PRIME(h) _MMIO((0x66d04 + (h) * 4)) -#define HDCP_SHA_TEXT _MMIO(0x66d18) - -/* HDCP Auth Registers */ -#define _PORTA_HDCP_AUTHENC 0x66800 -#define _PORTB_HDCP_AUTHENC 0x66500 -#define _PORTC_HDCP_AUTHENC 0x66600 -#define _PORTD_HDCP_AUTHENC 0x66700 -#define _PORTE_HDCP_AUTHENC 0x66A00 -#define _PORTF_HDCP_AUTHENC 0x66900 -#define _PORT_HDCP_AUTHENC(port, x) _MMIO(_PICK(port, \ - _PORTA_HDCP_AUTHENC, \ - _PORTB_HDCP_AUTHENC, \ - _PORTC_HDCP_AUTHENC, \ - _PORTD_HDCP_AUTHENC, \ - _PORTE_HDCP_AUTHENC, \ - _PORTF_HDCP_AUTHENC) + (x)) -#define PORT_HDCP_CONF(port) _PORT_HDCP_AUTHENC(port, 0x0) -#define _TRANSA_HDCP_CONF 0x66400 -#define _TRANSB_HDCP_CONF 0x66500 -#define TRANS_HDCP_CONF(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_CONF, \ - _TRANSB_HDCP_CONF) -#define HDCP_CONF(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_CONF(trans) : \ - PORT_HDCP_CONF(port)) - -#define HDCP_CONF_CAPTURE_AN BIT(0) -#define HDCP_CONF_AUTH_AND_ENC (BIT(1) | BIT(0)) -#define PORT_HDCP_ANINIT(port) _PORT_HDCP_AUTHENC(port, 0x4) -#define _TRANSA_HDCP_ANINIT 0x66404 -#define _TRANSB_HDCP_ANINIT 0x66504 -#define TRANS_HDCP_ANINIT(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP_ANINIT, \ - _TRANSB_HDCP_ANINIT) -#define HDCP_ANINIT(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_ANINIT(trans) : \ - PORT_HDCP_ANINIT(port)) - -#define PORT_HDCP_ANLO(port) _PORT_HDCP_AUTHENC(port, 0x8) -#define _TRANSA_HDCP_ANLO 0x66408 -#define _TRANSB_HDCP_ANLO 0x66508 -#define TRANS_HDCP_ANLO(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANLO, \ - _TRANSB_HDCP_ANLO) -#define HDCP_ANLO(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_ANLO(trans) : \ - PORT_HDCP_ANLO(port)) - -#define PORT_HDCP_ANHI(port) _PORT_HDCP_AUTHENC(port, 0xC) -#define _TRANSA_HDCP_ANHI 0x6640C -#define _TRANSB_HDCP_ANHI 0x6650C -#define TRANS_HDCP_ANHI(trans) _MMIO_TRANS(trans, _TRANSA_HDCP_ANHI, \ - _TRANSB_HDCP_ANHI) -#define HDCP_ANHI(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_ANHI(trans) : \ - PORT_HDCP_ANHI(port)) - -#define PORT_HDCP_BKSVLO(port) _PORT_HDCP_AUTHENC(port, 0x10) -#define _TRANSA_HDCP_BKSVLO 0x66410 -#define _TRANSB_HDCP_BKSVLO 0x66510 -#define TRANS_HDCP_BKSVLO(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP_BKSVLO, \ - _TRANSB_HDCP_BKSVLO) -#define HDCP_BKSVLO(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_BKSVLO(trans) : \ - PORT_HDCP_BKSVLO(port)) - -#define PORT_HDCP_BKSVHI(port) _PORT_HDCP_AUTHENC(port, 0x14) -#define _TRANSA_HDCP_BKSVHI 0x66414 -#define _TRANSB_HDCP_BKSVHI 0x66514 -#define TRANS_HDCP_BKSVHI(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP_BKSVHI, \ - _TRANSB_HDCP_BKSVHI) -#define HDCP_BKSVHI(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_BKSVHI(trans) : \ - PORT_HDCP_BKSVHI(port)) - -#define PORT_HDCP_RPRIME(port) _PORT_HDCP_AUTHENC(port, 0x18) -#define _TRANSA_HDCP_RPRIME 0x66418 -#define _TRANSB_HDCP_RPRIME 0x66518 -#define TRANS_HDCP_RPRIME(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP_RPRIME, \ - _TRANSB_HDCP_RPRIME) -#define HDCP_RPRIME(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_RPRIME(trans) : \ - PORT_HDCP_RPRIME(port)) - -#define PORT_HDCP_STATUS(port) _PORT_HDCP_AUTHENC(port, 0x1C) -#define _TRANSA_HDCP_STATUS 0x6641C -#define _TRANSB_HDCP_STATUS 0x6651C -#define TRANS_HDCP_STATUS(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP_STATUS, \ - _TRANSB_HDCP_STATUS) -#define HDCP_STATUS(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP_STATUS(trans) : \ - PORT_HDCP_STATUS(port)) - -#define HDCP_STATUS_STREAM_A_ENC BIT(31) -#define HDCP_STATUS_STREAM_B_ENC BIT(30) -#define HDCP_STATUS_STREAM_C_ENC BIT(29) -#define HDCP_STATUS_STREAM_D_ENC BIT(28) -#define HDCP_STATUS_AUTH BIT(21) -#define HDCP_STATUS_ENC BIT(20) -#define HDCP_STATUS_RI_MATCH BIT(19) -#define HDCP_STATUS_R0_READY BIT(18) -#define HDCP_STATUS_AN_READY BIT(17) -#define HDCP_STATUS_CIPHER BIT(16) -#define HDCP_STATUS_FRAME_CNT(x) (((x) >> 8) & 0xff) - -/* HDCP2.2 Registers */ -#define _PORTA_HDCP2_BASE 0x66800 -#define _PORTB_HDCP2_BASE 0x66500 -#define _PORTC_HDCP2_BASE 0x66600 -#define _PORTD_HDCP2_BASE 0x66700 -#define _PORTE_HDCP2_BASE 0x66A00 -#define _PORTF_HDCP2_BASE 0x66900 -#define _PORT_HDCP2_BASE(port, x) _MMIO(_PICK((port), \ - _PORTA_HDCP2_BASE, \ - _PORTB_HDCP2_BASE, \ - _PORTC_HDCP2_BASE, \ - _PORTD_HDCP2_BASE, \ - _PORTE_HDCP2_BASE, \ - _PORTF_HDCP2_BASE) + (x)) - -#define PORT_HDCP2_AUTH(port) _PORT_HDCP2_BASE(port, 0x98) -#define _TRANSA_HDCP2_AUTH 0x66498 -#define _TRANSB_HDCP2_AUTH 0x66598 -#define TRANS_HDCP2_AUTH(trans) _MMIO_TRANS(trans, _TRANSA_HDCP2_AUTH, \ - _TRANSB_HDCP2_AUTH) -#define AUTH_LINK_AUTHENTICATED BIT(31) -#define AUTH_LINK_TYPE BIT(30) -#define AUTH_FORCE_CLR_INPUTCTR BIT(19) -#define AUTH_CLR_KEYS BIT(18) -#define HDCP2_AUTH(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP2_AUTH(trans) : \ - PORT_HDCP2_AUTH(port)) - -#define PORT_HDCP2_CTL(port) _PORT_HDCP2_BASE(port, 0xB0) -#define _TRANSA_HDCP2_CTL 0x664B0 -#define _TRANSB_HDCP2_CTL 0x665B0 -#define TRANS_HDCP2_CTL(trans) _MMIO_TRANS(trans, _TRANSA_HDCP2_CTL, \ - _TRANSB_HDCP2_CTL) -#define CTL_LINK_ENCRYPTION_REQ BIT(31) -#define HDCP2_CTL(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP2_CTL(trans) : \ - PORT_HDCP2_CTL(port)) - -#define PORT_HDCP2_STATUS(port) _PORT_HDCP2_BASE(port, 0xB4) -#define _TRANSA_HDCP2_STATUS 0x664B4 -#define _TRANSB_HDCP2_STATUS 0x665B4 -#define TRANS_HDCP2_STATUS(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP2_STATUS, \ - _TRANSB_HDCP2_STATUS) -#define LINK_TYPE_STATUS BIT(22) -#define LINK_AUTH_STATUS BIT(21) -#define LINK_ENCRYPTION_STATUS BIT(20) -#define HDCP2_STATUS(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP2_STATUS(trans) : \ - PORT_HDCP2_STATUS(port)) - -#define _PIPEA_HDCP2_STREAM_STATUS 0x668C0 -#define _PIPEB_HDCP2_STREAM_STATUS 0x665C0 -#define _PIPEC_HDCP2_STREAM_STATUS 0x666C0 -#define _PIPED_HDCP2_STREAM_STATUS 0x667C0 -#define PIPE_HDCP2_STREAM_STATUS(pipe) _MMIO(_PICK((pipe), \ - _PIPEA_HDCP2_STREAM_STATUS, \ - _PIPEB_HDCP2_STREAM_STATUS, \ - _PIPEC_HDCP2_STREAM_STATUS, \ - _PIPED_HDCP2_STREAM_STATUS)) - -#define _TRANSA_HDCP2_STREAM_STATUS 0x664C0 -#define _TRANSB_HDCP2_STREAM_STATUS 0x665C0 -#define TRANS_HDCP2_STREAM_STATUS(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP2_STREAM_STATUS, \ - _TRANSB_HDCP2_STREAM_STATUS) -#define STREAM_ENCRYPTION_STATUS BIT(31) -#define STREAM_TYPE_STATUS BIT(30) -#define HDCP2_STREAM_STATUS(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP2_STREAM_STATUS(trans) : \ - PIPE_HDCP2_STREAM_STATUS(pipe)) - -#define _PORTA_HDCP2_AUTH_STREAM 0x66F00 -#define _PORTB_HDCP2_AUTH_STREAM 0x66F04 -#define PORT_HDCP2_AUTH_STREAM(port) _MMIO_PORT(port, \ - _PORTA_HDCP2_AUTH_STREAM, \ - _PORTB_HDCP2_AUTH_STREAM) -#define _TRANSA_HDCP2_AUTH_STREAM 0x66F00 -#define _TRANSB_HDCP2_AUTH_STREAM 0x66F04 -#define TRANS_HDCP2_AUTH_STREAM(trans) _MMIO_TRANS(trans, \ - _TRANSA_HDCP2_AUTH_STREAM, \ - _TRANSB_HDCP2_AUTH_STREAM) -#define AUTH_STREAM_TYPE BIT(31) -#define HDCP2_AUTH_STREAM(dev_priv, trans, port) \ - (GRAPHICS_VER(dev_priv) >= 12 ? \ - TRANS_HDCP2_AUTH_STREAM(trans) : \ - PORT_HDCP2_AUTH_STREAM(port)) - /* Per-pipe DDI Function Control */ #define _TRANS_DDI_FUNC_CTL_A 0x60400 #define _TRANS_DDI_FUNC_CTL_B 0x61400 @@ -7503,16 +7126,16 @@ enum skl_power_gate { /* CDCLK_CTL */ #define CDCLK_CTL _MMIO(0x46000) -#define CDCLK_FREQ_SEL_MASK (3 << 26) -#define CDCLK_FREQ_450_432 (0 << 26) -#define CDCLK_FREQ_540 (1 << 26) -#define CDCLK_FREQ_337_308 (2 << 26) -#define CDCLK_FREQ_675_617 (3 << 26) -#define BXT_CDCLK_CD2X_DIV_SEL_MASK (3 << 22) -#define BXT_CDCLK_CD2X_DIV_SEL_1 (0 << 22) -#define BXT_CDCLK_CD2X_DIV_SEL_1_5 (1 << 22) -#define BXT_CDCLK_CD2X_DIV_SEL_2 (2 << 22) -#define BXT_CDCLK_CD2X_DIV_SEL_4 (3 << 22) +#define CDCLK_FREQ_SEL_MASK REG_GENMASK(27, 26) +#define CDCLK_FREQ_450_432 REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 0) +#define CDCLK_FREQ_540 REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 1) +#define CDCLK_FREQ_337_308 REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 2) +#define CDCLK_FREQ_675_617 REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 3) +#define BXT_CDCLK_CD2X_DIV_SEL_MASK REG_GENMASK(23, 22) +#define BXT_CDCLK_CD2X_DIV_SEL_1 REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 0) +#define BXT_CDCLK_CD2X_DIV_SEL_1_5 REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 1) +#define BXT_CDCLK_CD2X_DIV_SEL_2 REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 2) +#define BXT_CDCLK_CD2X_DIV_SEL_4 REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 3) #define BXT_CDCLK_CD2X_PIPE(pipe) ((pipe) << 20) #define CDCLK_DIVMUX_CD_OVERRIDE (1 << 19) #define BXT_CDCLK_CD2X_PIPE_NONE BXT_CDCLK_CD2X_PIPE(3) @@ -8367,6 +7990,7 @@ enum skl_power_gate { #define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe) _MMIO_PIPE((pipe) - PIPE_B, \ _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \ _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC) +#define DSC_ALT_ICH_SEL (1 << 20) #define DSC_VBR_ENABLE (1 << 19) #define DSC_422_ENABLE (1 << 18) #define DSC_COLOR_SPACE_CONVERSION (1 << 17) @@ -8717,4 +8341,27 @@ enum skl_power_gate { #define GEN12_CULLBIT2 _MMIO(0x7030) #define GEN12_STATE_ACK_DEBUG _MMIO(0x20BC) +#define MTL_LATENCY_LP0_LP1 _MMIO(0x45780) +#define MTL_LATENCY_LP2_LP3 _MMIO(0x45784) +#define MTL_LATENCY_LP4_LP5 _MMIO(0x45788) +#define MTL_LATENCY_LEVEL_EVEN_MASK REG_GENMASK(12, 0) +#define MTL_LATENCY_LEVEL_ODD_MASK REG_GENMASK(28, 16) + +#define MTL_LATENCY_SAGV _MMIO(0x4578b) +#define MTL_LATENCY_QCLK_SAGV REG_GENMASK(12, 0) + +#define MTL_MEM_SS_INFO_GLOBAL _MMIO(0x45700) +#define MTL_N_OF_ENABLED_QGV_POINTS_MASK REG_GENMASK(11, 8) +#define MTL_N_OF_POPULATED_CH_MASK REG_GENMASK(7, 4) +#define MTL_DDR_TYPE_MASK REG_GENMASK(3, 0) + +#define MTL_MEM_SS_INFO_QGV_POINT_LOW(point) _MMIO(0x45710 + (point) * 2) +#define MTL_TRCD_MASK REG_GENMASK(31, 24) +#define MTL_TRP_MASK REG_GENMASK(23, 16) +#define MTL_DCLK_MASK REG_GENMASK(15, 0) + +#define MTL_MEM_SS_INFO_QGV_POINT_HIGH(point) _MMIO(0x45714 + (point) * 2) +#define MTL_TRAS_MASK REG_GENMASK(16, 8) +#define MTL_TRDPRE_MASK REG_GENMASK(7, 0) + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index ae984c66c48a99b2cf808800c0de0c317e05b2dd..6fc0d1b89690210a762c8ae2106c0fe85b006b80 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -241,8 +241,6 @@ void __i915_sw_fence_init(struct i915_sw_fence *fence, const char *name, struct lock_class_key *key) { - BUG_ON(!fn); - __init_waitqueue_head(&fence->wait, name, key); fence->fn = fn; #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h index a7c603bc1b01c9a218b4a76a29050e69c0f3bf95..619fc5a22f0c7bbfd2ea1da4c47f7dd5165e5ccb 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.h +++ b/drivers/gpu/drm/i915/i915_sw_fence.h @@ -48,11 +48,15 @@ void __i915_sw_fence_init(struct i915_sw_fence *fence, do { \ static struct lock_class_key __key; \ \ + BUILD_BUG_ON((fn) == NULL); \ __i915_sw_fence_init((fence), (fn), #fence, &__key); \ } while (0) #else #define i915_sw_fence_init(fence, fn) \ - __i915_sw_fence_init((fence), (fn), NULL, NULL) +do { \ + BUILD_BUG_ON((fn) == NULL); \ + __i915_sw_fence_init((fence), (fn), NULL, NULL); \ +} while (0) #endif void i915_sw_fence_reinit(struct i915_sw_fence *fence); diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index c10d68cdc3ca5631ab4fdd35e6d30d94954ba711..6c14d13364bf789a30edcf2b162e4557aaf32a2f 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -360,10 +360,6 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) #define KHz(x) (1000 * (x)) #define MHz(x) KHz(1000 * (x)) -#define KBps(x) (1000 * (x)) -#define MBps(x) KBps(1000 * (x)) -#define GBps(x) ((u64)1000 * MBps((x))) - void add_taint_for_CI(struct drm_i915_private *i915, unsigned int taint); static inline void __add_taint_for_CI(unsigned int taint) { diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index d98fbbd589aaa43669187b6e937dbe701aed5228..20575eb77ea74db2a19c9ceeeccc51bea97d9f99 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -88,46 +88,57 @@ const char *intel_platform_name(enum intel_platform platform) return platform_names[platform]; } -void intel_device_info_print_static(const struct intel_device_info *info, - struct drm_printer *p) +void intel_device_info_print(const struct intel_device_info *info, + const struct intel_runtime_info *runtime, + struct drm_printer *p) { - if (info->graphics.rel) - drm_printf(p, "graphics version: %u.%02u\n", info->graphics.ver, - info->graphics.rel); + if (runtime->graphics.ip.rel) + drm_printf(p, "graphics version: %u.%02u\n", + runtime->graphics.ip.ver, + runtime->graphics.ip.rel); else - drm_printf(p, "graphics version: %u\n", info->graphics.ver); + drm_printf(p, "graphics version: %u\n", + runtime->graphics.ip.ver); - if (info->media.rel) - drm_printf(p, "media version: %u.%02u\n", info->media.ver, info->media.rel); + if (runtime->media.ip.rel) + drm_printf(p, "media version: %u.%02u\n", + runtime->media.ip.ver, + runtime->media.ip.rel); else - drm_printf(p, "media version: %u\n", info->media.ver); + drm_printf(p, "media version: %u\n", + runtime->media.ip.ver); - if (info->display.rel) - drm_printf(p, "display version: %u.%02u\n", info->display.ver, info->display.rel); + if (runtime->display.ip.rel) + drm_printf(p, "display version: %u.%02u\n", + runtime->display.ip.ver, + runtime->display.ip.rel); else - drm_printf(p, "display version: %u\n", info->display.ver); + drm_printf(p, "display version: %u\n", + runtime->display.ip.ver); drm_printf(p, "gt: %d\n", info->gt); - drm_printf(p, "memory-regions: %x\n", info->memory_regions); - drm_printf(p, "page-sizes: %x\n", info->page_sizes); + drm_printf(p, "memory-regions: %x\n", runtime->memory_regions); + drm_printf(p, "page-sizes: %x\n", runtime->page_sizes); drm_printf(p, "platform: %s\n", intel_platform_name(info->platform)); - drm_printf(p, "ppgtt-size: %d\n", info->ppgtt_size); - drm_printf(p, "ppgtt-type: %d\n", info->ppgtt_type); + drm_printf(p, "ppgtt-size: %d\n", runtime->ppgtt_size); + drm_printf(p, "ppgtt-type: %d\n", runtime->ppgtt_type); drm_printf(p, "dma_mask_size: %u\n", info->dma_mask_size); #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, str_yes_no(info->name)) DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG); #undef PRINT_FLAG + drm_printf(p, "has_pooled_eu: %s\n", str_yes_no(runtime->has_pooled_eu)); + #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, str_yes_no(info->display.name)) DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG); #undef PRINT_FLAG -} -void intel_device_info_print_runtime(const struct intel_runtime_info *info, - struct drm_printer *p) -{ - drm_printf(p, "rawclk rate: %u kHz\n", info->rawclk_freq); + drm_printf(p, "has_hdcp: %s\n", str_yes_no(runtime->has_hdcp)); + drm_printf(p, "has_dmc: %s\n", str_yes_no(runtime->has_dmc)); + drm_printf(p, "has_dsc: %s\n", str_yes_no(runtime->has_dsc)); + + drm_printf(p, "rawclk rate: %u kHz\n", runtime->rawclk_freq); } #undef INTEL_VGA_DEVICE @@ -364,55 +375,55 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) { drm_info(&dev_priv->drm, "Display fused off, disabling\n"); - info->display.pipe_mask = 0; - info->display.cpu_transcoder_mask = 0; - info->display.fbc_mask = 0; + runtime->pipe_mask = 0; + runtime->cpu_transcoder_mask = 0; + runtime->fbc_mask = 0; } else if (fuse_strap & IVB_PIPE_C_DISABLE) { drm_info(&dev_priv->drm, "PipeC fused off\n"); - info->display.pipe_mask &= ~BIT(PIPE_C); - info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_C); + runtime->pipe_mask &= ~BIT(PIPE_C); + runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } } else if (HAS_DISPLAY(dev_priv) && DISPLAY_VER(dev_priv) >= 9) { u32 dfsm = intel_de_read(dev_priv, SKL_DFSM); if (dfsm & SKL_DFSM_PIPE_A_DISABLE) { - info->display.pipe_mask &= ~BIT(PIPE_A); - info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_A); - info->display.fbc_mask &= ~BIT(INTEL_FBC_A); + runtime->pipe_mask &= ~BIT(PIPE_A); + runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_A); + runtime->fbc_mask &= ~BIT(INTEL_FBC_A); } if (dfsm & SKL_DFSM_PIPE_B_DISABLE) { - info->display.pipe_mask &= ~BIT(PIPE_B); - info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_B); + runtime->pipe_mask &= ~BIT(PIPE_B); + runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_B); } if (dfsm & SKL_DFSM_PIPE_C_DISABLE) { - info->display.pipe_mask &= ~BIT(PIPE_C); - info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_C); + runtime->pipe_mask &= ~BIT(PIPE_C); + runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } if (DISPLAY_VER(dev_priv) >= 12 && (dfsm & TGL_DFSM_PIPE_D_DISABLE)) { - info->display.pipe_mask &= ~BIT(PIPE_D); - info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_D); + runtime->pipe_mask &= ~BIT(PIPE_D); + runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_D); } if (dfsm & SKL_DFSM_DISPLAY_HDCP_DISABLE) - info->display.has_hdcp = 0; + runtime->has_hdcp = 0; if (dfsm & SKL_DFSM_DISPLAY_PM_DISABLE) - info->display.fbc_mask = 0; + runtime->fbc_mask = 0; if (DISPLAY_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) - info->display.has_dmc = 0; + runtime->has_dmc = 0; if (DISPLAY_VER(dev_priv) >= 10 && (dfsm & GLK_DFSM_DISPLAY_DSC_DISABLE)) - info->display.has_dsc = 0; + runtime->has_dsc = 0; } if (GRAPHICS_VER(dev_priv) == 6 && i915_vtd_active(dev_priv)) { drm_info(&dev_priv->drm, "Disabling ppGTT for VT-d support\n"); - info->ppgtt_type = INTEL_PPGTT_NONE; + runtime->ppgtt_type = INTEL_PPGTT_NONE; } runtime->rawclk_freq = intel_read_rawclk(dev_priv); @@ -422,8 +433,14 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) dev_priv->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC); memset(&info->display, 0, sizeof(info->display)); + + runtime->cpu_transcoder_mask = 0; memset(runtime->num_sprites, 0, sizeof(runtime->num_sprites)); memset(runtime->num_scalers, 0, sizeof(runtime->num_scalers)); + runtime->fbc_mask = 0; + runtime->has_hdcp = false; + runtime->has_dmc = false; + runtime->has_dsc = false; } } diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index e681bc6ed8e92d4e75328a646b5ee958c391708a..d638235e1d26de8ed4658e4c3fb6e1ba4b530721 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -37,6 +37,7 @@ struct drm_printer; struct drm_i915_private; +struct intel_gt_definition; /* Keep in gen based order, and chronological order within a gen */ enum intel_platform { @@ -164,7 +165,6 @@ enum intel_ppgtt_type { func(has_media_ratio_mode); \ func(has_mslice_steering); \ func(has_one_eu_per_fuse_bit); \ - func(has_pooled_eu); \ func(has_pxp); \ func(has_rc6); \ func(has_rc6p); \ @@ -180,14 +180,11 @@ enum intel_ppgtt_type { /* Keep in alphabetical order */ \ func(cursor_needs_physical); \ func(has_cdclk_crawl); \ - func(has_dmc); \ func(has_ddi); \ func(has_dp_mst); \ func(has_dsb); \ - func(has_dsc); \ func(has_fpga_dbg); \ func(has_gmch); \ - func(has_hdcp); \ func(has_hotplug); \ func(has_hti); \ func(has_ipc); \ @@ -203,23 +200,67 @@ struct ip_version { u8 rel; }; -struct intel_device_info { - struct ip_version graphics; - struct ip_version media; +struct intel_runtime_info { + struct { + struct ip_version ip; + } graphics; + struct { + struct ip_version ip; + } media; + struct { + struct ip_version ip; + } display; + + /* + * Platform mask is used for optimizing or-ed IS_PLATFORM calls into + * single runtime conditionals, and also to provide groundwork for + * future per platform, or per SKU build optimizations. + * + * Array can be extended when necessary if the corresponding + * BUILD_BUG_ON is hit. + */ + u32 platform_mask[2]; + + u16 device_id; intel_engine_mask_t platform_engine_mask; /* Engines supported by the HW */ - enum intel_platform platform; + u32 rawclk_freq; - unsigned int dma_mask_size; /* available DMA address bits */ + struct intel_step_info step; + + unsigned int page_sizes; /* page sizes supported by the HW */ enum intel_ppgtt_type ppgtt_type; unsigned int ppgtt_size; /* log2, e.g. 31/32/48 bits */ - unsigned int page_sizes; /* page sizes supported by the HW */ - u32 memory_regions; /* regions supported by the HW */ + bool has_pooled_eu; + + /* display */ + struct { + u8 pipe_mask; + u8 cpu_transcoder_mask; + + u8 num_sprites[I915_MAX_PIPES]; + u8 num_scalers[I915_MAX_PIPES]; + + u8 fbc_mask; + + bool has_hdcp; + bool has_dmc; + bool has_dsc; + }; +}; + +struct intel_device_info { + enum intel_platform platform; + + unsigned int dma_mask_size; /* available DMA address bits */ + + const struct intel_gt_definition *extra_gt_list; + u8 gt; /* GT number, 0 if undefined */ #define DEFINE_FLAG(name) u8 name:1 @@ -227,12 +268,6 @@ struct intel_device_info { #undef DEFINE_FLAG struct { - u8 ver; - u8 rel; - - u8 pipe_mask; - u8 cpu_transcoder_mask; - u8 fbc_mask; u8 abox_mask; struct { @@ -259,27 +294,11 @@ struct intel_device_info { u32 gamma_lut_tests; } color; } display; -}; -struct intel_runtime_info { /* - * Platform mask is used for optimizing or-ed IS_PLATFORM calls into - * into single runtime conditionals, and also to provide groundwork - * for future per platform, or per SKU build optimizations. - * - * Array can be extended when necessary if the corresponding - * BUILD_BUG_ON is hit. + * Initial runtime info. Do not access outside of i915_driver_create(). */ - u32 platform_mask[2]; - - u16 device_id; - - u8 num_sprites[I915_MAX_PIPES]; - u8 num_scalers[I915_MAX_PIPES]; - - u32 rawclk_freq; - - struct intel_step_info step; + const struct intel_runtime_info __runtime; }; struct intel_driver_caps { @@ -292,10 +311,9 @@ const char *intel_platform_name(enum intel_platform platform); void intel_device_info_subplatform_init(struct drm_i915_private *dev_priv); void intel_device_info_runtime_init(struct drm_i915_private *dev_priv); -void intel_device_info_print_static(const struct intel_device_info *info, - struct drm_printer *p); -void intel_device_info_print_runtime(const struct intel_runtime_info *info, - struct drm_printer *p); +void intel_device_info_print(const struct intel_device_info *info, + const struct intel_runtime_info *runtime, + struct drm_printer *p); void intel_driver_caps_print(const struct intel_driver_caps *caps, struct drm_printer *p); diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c index 4374471197709afafb823d699fe9e8a9b8356b0c..2403ccd52c74a123edfa28013434f5326c0763fe 100644 --- a/drivers/gpu/drm/i915/intel_dram.c +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -466,6 +466,43 @@ static int gen12_get_dram_info(struct drm_i915_private *i915) return icl_pcode_read_mem_global_info(i915); } +static int xelpdp_get_dram_info(struct drm_i915_private *i915) +{ + u32 val = intel_uncore_read(&i915->uncore, MTL_MEM_SS_INFO_GLOBAL); + struct dram_info *dram_info = &i915->dram_info; + + val = REG_FIELD_GET(MTL_DDR_TYPE_MASK, val); + switch (val) { + case 0: + dram_info->type = INTEL_DRAM_DDR4; + break; + case 1: + dram_info->type = INTEL_DRAM_DDR5; + break; + case 2: + dram_info->type = INTEL_DRAM_LPDDR5; + break; + case 3: + dram_info->type = INTEL_DRAM_LPDDR4; + break; + case 4: + dram_info->type = INTEL_DRAM_DDR3; + break; + case 5: + dram_info->type = INTEL_DRAM_LPDDR3; + break; + default: + MISSING_CASE(val); + return -EINVAL; + } + + dram_info->num_channels = REG_FIELD_GET(MTL_N_OF_POPULATED_CH_MASK, val); + dram_info->num_qgv_points = REG_FIELD_GET(MTL_N_OF_ENABLED_QGV_POINTS_MASK, val); + /* PSF GV points not supported in D14+ */ + + return 0; +} + void intel_dram_detect(struct drm_i915_private *i915) { struct dram_info *dram_info = &i915->dram_info; @@ -480,7 +517,9 @@ void intel_dram_detect(struct drm_i915_private *i915) */ dram_info->wm_lv_0_adjust_needed = !IS_GEN9_LP(i915); - if (GRAPHICS_VER(i915) >= 12) + if (DISPLAY_VER(i915) >= 14) + ret = xelpdp_get_dram_info(i915); + else if (GRAPHICS_VER(i915) >= 12) ret = gen12_get_dram_info(i915); else if (GRAPHICS_VER(i915) >= 11) ret = gen11_get_dram_info(i915); diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c index 157e166672d7bf36b15c6784b6858ae056d632e7..8279dc580a3e507bb9aa4a2333df74470a76514c 100644 --- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c +++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c @@ -4,6 +4,7 @@ */ #include "display/intel_audio_regs.h" +#include "display/intel_backlight_regs.h" #include "display/intel_dmc_regs.h" #include "display/vlv_dsi_pll_regs.h" #include "gt/intel_gt_regs.h" @@ -1076,7 +1077,8 @@ static int iterate_skl_plus_mmio(struct intel_gvt_mmio_table_iter *iter) MMIO_D(GEN8_HDC_CHICKEN1); MMIO_D(GEN9_WM_CHICKEN3); - if (IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) + if (IS_KABYLAKE(dev_priv) || + IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv)) MMIO_D(GAMT_CHKN_BIT_REG); if (!IS_BROXTON(dev_priv)) MMIO_D(GEN9_CTX_PREEMPT_REG); diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c index 0fec25be146a2462401b914ddf75d3a3cb0b748e..ba9843cb1b13d8c963c87f07c527c7f530ad3d8e 100644 --- a/drivers/gpu/drm/i915/intel_pch.c +++ b/drivers/gpu/drm/i915/intel_pch.c @@ -138,6 +138,11 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id) drm_WARN_ON(&dev_priv->drm, !IS_ALDERLAKE_S(dev_priv) && !IS_ALDERLAKE_P(dev_priv)); return PCH_ADP; + case INTEL_PCH_MTP_DEVICE_ID_TYPE: + case INTEL_PCH_MTP2_DEVICE_ID_TYPE: + drm_dbg_kms(&dev_priv->drm, "Found Meteor Lake PCH\n"); + drm_WARN_ON(&dev_priv->drm, !IS_METEORLAKE(dev_priv)); + return PCH_MTP; default: return PCH_NONE; } @@ -166,7 +171,9 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv, * make an educated guess as to which PCH is really there. */ - if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) + if (IS_METEORLAKE(dev_priv)) + id = INTEL_PCH_MTP_DEVICE_ID_TYPE; + else if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv)) id = INTEL_PCH_ADP_DEVICE_ID_TYPE; else if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv)) id = INTEL_PCH_TGP_DEVICE_ID_TYPE; diff --git a/drivers/gpu/drm/i915/intel_pch.h b/drivers/gpu/drm/i915/intel_pch.h index 7c8ce9781d1a21d889a915194c51f43a9a01e68b..32aff5a70d04e069024b3d6e429efcae89a62bb6 100644 --- a/drivers/gpu/drm/i915/intel_pch.h +++ b/drivers/gpu/drm/i915/intel_pch.h @@ -25,6 +25,7 @@ enum intel_pch { PCH_ICP, /* Ice Lake/Jasper Lake PCH */ PCH_TGP, /* Tiger Lake/Mule Creek Canyon PCH */ PCH_ADP, /* Alder Lake PCH */ + PCH_MTP, /* Meteor Lake PCH */ /* Fake PCHs, functionality handled on the same PCI dev */ PCH_DG1 = 1024, @@ -57,12 +58,15 @@ enum intel_pch { #define INTEL_PCH_ADP2_DEVICE_ID_TYPE 0x5180 #define INTEL_PCH_ADP3_DEVICE_ID_TYPE 0x7A00 #define INTEL_PCH_ADP4_DEVICE_ID_TYPE 0x5480 +#define INTEL_PCH_MTP_DEVICE_ID_TYPE 0x7E00 +#define INTEL_PCH_MTP2_DEVICE_ID_TYPE 0xAE00 #define INTEL_PCH_P2X_DEVICE_ID_TYPE 0x7100 #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ #define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) #define INTEL_PCH_ID(dev_priv) ((dev_priv)->pch_id) +#define HAS_PCH_MTP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_MTP) #define HAS_PCH_DG2(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_DG2) #define HAS_PCH_ADP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_ADP) #define HAS_PCH_DG1(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_DG1) diff --git a/drivers/gpu/drm/i915/intel_pci_config.h b/drivers/gpu/drm/i915/intel_pci_config.h index 12cd9d4f23dec4b0cb8706503bf6f3e1cda59656..4977a524ce6f3f458704360a6dbeebe4265ece03 100644 --- a/drivers/gpu/drm/i915/intel_pci_config.h +++ b/drivers/gpu/drm/i915/intel_pci_config.h @@ -6,6 +6,13 @@ #ifndef __INTEL_PCI_CONFIG_H__ #define __INTEL_PCI_CONFIG_H__ +/* PCI BARs */ +#define GTTMMADR_BAR 0 +#define GEN2_GTTMMADR_BAR 1 +#define GFXMEM_BAR 2 +#define GTT_APERTURE_BAR GFXMEM_BAR +#define GEN12_LMEM_BAR GFXMEM_BAR + /* BSM in include/drm/i915_drm.h */ #define MCHBAR_I915 0x44 diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index ef7553b494eac9f7437fdf3cdce9a65f93219d32..8f86f56e7ca4a174ce827249dd4ad64c2c8f9360 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -25,60 +25,22 @@ * */ -#include -#include -#include - -#include -#include -#include - -#include "display/intel_atomic.h" -#include "display/intel_atomic_plane.h" -#include "display/intel_bw.h" #include "display/intel_de.h" #include "display/intel_display_trace.h" -#include "display/intel_display_types.h" -#include "display/intel_fb.h" -#include "display/intel_fbc.h" -#include "display/intel_sprite.h" -#include "display/skl_universal_plane.h" +#include "display/skl_watermark.h" #include "gt/intel_engine_regs.h" #include "gt/intel_gt_regs.h" -#include "gt/intel_llc.h" #include "i915_drv.h" -#include "i915_fixed.h" -#include "i915_irq.h" #include "intel_mchbar_regs.h" -#include "intel_pcode.h" #include "intel_pm.h" #include "vlv_sideband.h" -#include "../../../platform/x86/intel_ips.h" - -static void skl_sagv_disable(struct drm_i915_private *dev_priv); struct drm_i915_clock_gating_funcs { void (*init_clock_gating)(struct drm_i915_private *i915); }; -/* Stores plane specific WM parameters */ -struct skl_wm_params { - bool x_tiled, y_tiled; - bool rc_surface; - bool is_planar; - u32 width; - u8 cpp; - u32 plane_pixel_rate; - u32 y_min_scanlines; - u32 plane_bytes_per_line; - uint_fixed_16_16_t plane_blocks_per_line; - uint_fixed_16_16_t y_tile_minimum; - u32 linetime_us; - u32 dbuf_block_size; -}; - /* used in computing the new watermarks state */ struct intel_wm_config { unsigned int num_pipes_active; @@ -468,13 +430,13 @@ bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { bool ret; - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); ret = _intel_set_memory_cxsr(dev_priv, enable); if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - dev_priv->wm.vlv.cxsr = enable; + dev_priv->display.wm.vlv.cxsr = enable; else if (IS_G4X(dev_priv)) - dev_priv->wm.g4x.cxsr = enable; - mutex_unlock(&dev_priv->wm.wm_mutex); + dev_priv->display.wm.g4x.cxsr = enable; + mutex_unlock(&dev_priv->display.wm.wm_mutex); return ret; } @@ -834,11 +796,11 @@ static bool is_enabling(int old, int new, int threshold) static int intel_wm_num_levels(struct drm_i915_private *dev_priv) { - return dev_priv->wm.max_level + 1; + return dev_priv->display.wm.max_level + 1; } -static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) +bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) { struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); @@ -1093,11 +1055,11 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv, static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv) { /* all latencies in usec */ - dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5; - dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12; - dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35; + dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5; + dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12; + dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35; - dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL; + dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL; } static int g4x_plane_fifo_size(enum plane_id plane_id, int level) @@ -1150,7 +1112,7 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state, struct drm_i915_private *dev_priv = to_i915(plane->base.dev); const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; - unsigned int latency = dev_priv->wm.pri_latency[level] * 10; + unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10; unsigned int pixel_rate, htotal, cpp, width, wm; if (latency == 0) @@ -1324,7 +1286,7 @@ static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - if (level > dev_priv->wm.max_level) + if (level > dev_priv->display.wm.max_level) return false; return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) && @@ -1583,7 +1545,7 @@ static void g4x_merge_wm(struct drm_i915_private *dev_priv, static void g4x_program_watermarks(struct drm_i915_private *dev_priv) { - struct g4x_wm_values *old_wm = &dev_priv->wm.g4x; + struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x; struct g4x_wm_values new_wm = {}; g4x_merge_wm(dev_priv, &new_wm); @@ -1609,10 +1571,10 @@ static void g4x_initial_watermarks(struct intel_atomic_state *state, const struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate; g4x_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } static void g4x_optimize_watermarks(struct intel_atomic_state *state, @@ -1625,10 +1587,10 @@ static void g4x_optimize_watermarks(struct intel_atomic_state *state, if (!crtc_state->wm.need_postvbl_update) return; - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); crtc->wm.active.g4x = crtc_state->wm.g4x.optimal; g4x_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } /* latency must be in 0.1us units. */ @@ -1650,15 +1612,15 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate, static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) { /* all latencies in usec */ - dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; + dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; - dev_priv->wm.max_level = VLV_WM_LEVEL_PM2; + dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2; if (IS_CHERRYVIEW(dev_priv)) { - dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; - dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; + dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12; + dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33; - dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS; + dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS; } } @@ -1672,7 +1634,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, &crtc_state->hw.pipe_mode; unsigned int pixel_rate, htotal, cpp, width, wm; - if (dev_priv->wm.pri_latency[level] == 0) + if (dev_priv->display.wm.pri_latency[level] == 0) return USHRT_MAX; if (!intel_wm_plane_visible(crtc_state, plane_state)) @@ -1693,7 +1655,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state, wm = 63; } else { wm = vlv_wm_method2(pixel_rate, htotal, width, cpp, - dev_priv->wm.pri_latency[level] * 10); + dev_priv->display.wm.pri_latency[level] * 10); } return min_t(unsigned int, wm, USHRT_MAX); @@ -2158,7 +2120,7 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv, struct intel_crtc *crtc; int num_active_pipes = 0; - wm->level = dev_priv->wm.max_level; + wm->level = dev_priv->display.wm.max_level; wm->cxsr = true; for_each_intel_crtc(&dev_priv->drm, crtc) { @@ -2197,7 +2159,7 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv, static void vlv_program_watermarks(struct drm_i915_private *dev_priv) { - struct vlv_wm_values *old_wm = &dev_priv->wm.vlv; + struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv; struct vlv_wm_values new_wm = {}; vlv_merge_wm(dev_priv, &new_wm); @@ -2235,10 +2197,10 @@ static void vlv_initial_watermarks(struct intel_atomic_state *state, const struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate; vlv_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } static void vlv_optimize_watermarks(struct intel_atomic_state *state, @@ -2251,10 +2213,10 @@ static void vlv_optimize_watermarks(struct intel_atomic_state *state, if (!crtc_state->wm.need_postvbl_update) return; - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); crtc->wm.active.vlv = crtc_state->wm.vlv.optimal; vlv_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } static void i965_update_wm(struct drm_i915_private *dev_priv) @@ -2835,9 +2797,9 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, const struct intel_plane_state *curstate, struct intel_wm_level *result) { - u16 pri_latency = dev_priv->wm.pri_latency[level]; - u16 spr_latency = dev_priv->wm.spr_latency[level]; - u16 cur_latency = dev_priv->wm.cur_latency[level]; + u16 pri_latency = dev_priv->display.wm.pri_latency[level]; + u16 spr_latency = dev_priv->display.wm.spr_latency[level]; + u16 cur_latency = dev_priv->display.wm.cur_latency[level]; /* WM1+ latency values stored in 0.5us units */ if (level > 0) { @@ -2861,119 +2823,43 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, result->enable = true; } -static void intel_read_wm_latency(struct drm_i915_private *dev_priv, - u16 wm[]) +static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) { - struct intel_uncore *uncore = &dev_priv->uncore; - - if (DISPLAY_VER(dev_priv) >= 9) { - u32 val; - int ret, i; - int level, max_level = ilk_wm_max_level(dev_priv); - int mult = IS_DG2(dev_priv) ? 2 : 1; - - /* read the first set of memory latencies[0:3] */ - val = 0; /* data0 to be programmed to 0 for first set */ - ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY, - &val, NULL); - - if (ret) { - drm_err(&dev_priv->drm, - "SKL Mailbox read error = %d\n", ret); - return; - } + u64 sskpd; - wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - - /* read the second set of memory latencies[4:7] */ - val = 1; /* data0 to be programmed to 1 for second set */ - ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY, - &val, NULL); - if (ret) { - drm_err(&dev_priv->drm, - "SKL Mailbox read error = %d\n", ret); - return; - } + sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD); - wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; - wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) & - GEN9_MEM_LATENCY_LEVEL_MASK) * mult; + wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd); + if (wm[0] == 0) + wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd); + wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd); + wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd); + wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd); + wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd); +} - /* - * If a level n (n > 1) has a 0us latency, all levels m (m >= n) - * need to be disabled. We make sure to sanitize the values out - * of the punit to satisfy this requirement. - */ - for (level = 1; level <= max_level; level++) { - if (wm[level] == 0) { - for (i = level + 1; i <= max_level; i++) - wm[i] = 0; +static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) +{ + u32 sskpd; - max_level = level - 1; + sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD); - break; - } - } + wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd); + wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd); + wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd); + wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd); +} - /* - * WaWmMemoryReadLatency - * - * punit doesn't take into account the read latency so we need - * to add proper adjustement to each valid level we retrieve - * from the punit when level 0 response data is 0us. - */ - if (wm[0] == 0) { - u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2; +static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[]) +{ + u32 mltr; - for (level = 0; level <= max_level; level++) - wm[level] += adjust; - } + mltr = intel_uncore_read(&i915->uncore, MLTR_ILK); - /* - * WA Level-0 adjustment for 16GB DIMMs: SKL+ - * If we could not get dimm info enable this WA to prevent from - * any underrun. If not able to get Dimm info assume 16GB dimm - * to avoid any underrun. - */ - if (dev_priv->dram_info.wm_lv_0_adjust_needed) - wm[0] += 1; - } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) { - u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD); - - wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd); - if (wm[0] == 0) - wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd); - wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd); - wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd); - wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd); - wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd); - } else if (DISPLAY_VER(dev_priv) >= 6) { - u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD); - - wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd); - wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd); - wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd); - wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd); - } else if (DISPLAY_VER(dev_priv) >= 5) { - u32 mltr = intel_uncore_read(uncore, MLTR_ILK); - - /* ILK primary LP0 latency is 700 ns */ - wm[0] = 7; - wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr); - wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr); - } else { - MISSING_CASE(INTEL_DEVID(dev_priv)); - } + /* ILK primary LP0 latency is 700 ns */ + wm[0] = 7; + wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr); + wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr); } static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv, @@ -3007,9 +2893,8 @@ int ilk_wm_max_level(const struct drm_i915_private *dev_priv) return 2; } -static void intel_print_wm_latency(struct drm_i915_private *dev_priv, - const char *name, - const u16 wm[]) +void intel_print_wm_latency(struct drm_i915_private *dev_priv, + const char *name, const u16 wm[]) { int level, max_level = ilk_wm_max_level(dev_priv); @@ -3061,18 +2946,18 @@ static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) * The BIOS provided WM memory latency values are often * inadequate for high resolution displays. Adjust them. */ - changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12); - changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12); - changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12); + changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12); + changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12); + changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12); if (!changed) return; drm_dbg_kms(&dev_priv->drm, "WM latency values increased to avoid potential underruns\n"); - intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); - intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); - intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); + intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); + intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); + intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); } static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv) @@ -3088,37 +2973,42 @@ static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv) * interrupts only. To play it safe we disable LP3 * watermarks entirely. */ - if (dev_priv->wm.pri_latency[3] == 0 && - dev_priv->wm.spr_latency[3] == 0 && - dev_priv->wm.cur_latency[3] == 0) + if (dev_priv->display.wm.pri_latency[3] == 0 && + dev_priv->display.wm.spr_latency[3] == 0 && + dev_priv->display.wm.cur_latency[3] == 0) return; - dev_priv->wm.pri_latency[3] = 0; - dev_priv->wm.spr_latency[3] = 0; - dev_priv->wm.cur_latency[3] = 0; + dev_priv->display.wm.pri_latency[3] = 0; + dev_priv->display.wm.spr_latency[3] = 0; + dev_priv->display.wm.cur_latency[3] = 0; drm_dbg_kms(&dev_priv->drm, "LP3 watermarks disabled due to potential for lost interrupts\n"); - intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); - intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); - intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); + intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); + intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); + intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); } static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) { - intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency); + if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) + hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); + else if (DISPLAY_VER(dev_priv) >= 6) + snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); + else + ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency); - memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, - sizeof(dev_priv->wm.pri_latency)); - memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency, - sizeof(dev_priv->wm.pri_latency)); + memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency, + sizeof(dev_priv->display.wm.pri_latency)); + memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency, + sizeof(dev_priv->display.wm.pri_latency)); - intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency); - intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency); + intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency); + intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency); - intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency); - intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency); - intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); + intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency); + intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency); + intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency); if (DISPLAY_VER(dev_priv) == 6) { snb_wm_latency_quirk(dev_priv); @@ -3126,12 +3016,6 @@ static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) } } -static void skl_setup_wm_latency(struct drm_i915_private *dev_priv) -{ - intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency); - intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency); -} - static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv, struct intel_pipe_wm *pipe_wm) { @@ -3386,7 +3270,7 @@ static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv, if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) return 2 * level; else - return dev_priv->wm.pri_latency[level]; + return dev_priv->display.wm.pri_latency[level]; } static void ilk_compute_wm_results(struct drm_i915_private *dev_priv, @@ -3538,7 +3422,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv, static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, unsigned int dirty) { - struct ilk_wm_values *previous = &dev_priv->wm.hw; + struct ilk_wm_values *previous = &dev_priv->display.wm.hw; bool changed = false; if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) { @@ -3572,7 +3456,7 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv, static void ilk_write_wm_values(struct drm_i915_private *dev_priv, struct ilk_wm_values *results) { - struct ilk_wm_values *previous = &dev_priv->wm.hw; + struct ilk_wm_values *previous = &dev_priv->display.wm.hw; unsigned int dirty; u32 val; @@ -3634,7 +3518,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv, if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2]) intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]); - dev_priv->wm.hw = *results; + dev_priv->display.wm.hw = *results; } bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv) @@ -3642,3195 +3526,243 @@ bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv) return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL); } -u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv) +static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, + struct intel_wm_config *config) { - u8 enabled_slices = 0; - enum dbuf_slice slice; + struct intel_crtc *crtc; - for_each_dbuf_slice(dev_priv, slice) { - if (intel_uncore_read(&dev_priv->uncore, - DBUF_CTL_S(slice)) & DBUF_POWER_STATE) - enabled_slices |= BIT(slice); - } + /* Compute the currently _active_ config */ + for_each_intel_crtc(&dev_priv->drm, crtc) { + const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; - return enabled_slices; -} + if (!wm->pipe_enabled) + continue; -/* - * FIXME: We still don't have the proper code detect if we need to apply the WA, - * so assume we'll always need it in order to avoid underruns. - */ -static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv) -{ - return DISPLAY_VER(dev_priv) == 9; + config->sprites_enabled |= wm->sprites_enabled; + config->sprites_scaled |= wm->sprites_scaled; + config->num_pipes_active++; + } } -static bool -intel_has_sagv(struct drm_i915_private *dev_priv) +static void ilk_program_watermarks(struct drm_i915_private *dev_priv) { - return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) && - dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED; -} + struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; + struct ilk_wm_maximums max; + struct intel_wm_config config = {}; + struct ilk_wm_values results = {}; + enum intel_ddb_partitioning partitioning; -static u32 -intel_sagv_block_time(struct drm_i915_private *dev_priv) -{ - if (DISPLAY_VER(dev_priv) >= 12) { - u32 val = 0; - int ret; + ilk_compute_wm_config(dev_priv, &config); - ret = snb_pcode_read(&dev_priv->uncore, - GEN12_PCODE_READ_SAGV_BLOCK_TIME_US, - &val, NULL); - if (ret) { - drm_dbg_kms(&dev_priv->drm, "Couldn't read SAGV block time!\n"); - return 0; - } + ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max); + ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2); - return val; - } else if (DISPLAY_VER(dev_priv) == 11) { - return 10; - } else if (DISPLAY_VER(dev_priv) == 9 && !IS_LP(dev_priv)) { - return 30; + /* 5/6 split only in single pipe config on IVB+ */ + if (DISPLAY_VER(dev_priv) >= 7 && + config.num_pipes_active == 1 && config.sprites_enabled) { + ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max); + ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6); + + best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6); } else { - return 0; + best_lp_wm = &lp_wm_1_2; } -} - -static void intel_sagv_init(struct drm_i915_private *i915) -{ - if (!intel_has_sagv(i915)) - i915->sagv_status = I915_SAGV_NOT_CONTROLLED; - /* - * Probe to see if we have working SAGV control. - * For icl+ this was already determined by intel_bw_init_hw(). - */ - if (DISPLAY_VER(i915) < 11) - skl_sagv_disable(i915); - - drm_WARN_ON(&i915->drm, i915->sagv_status == I915_SAGV_UNKNOWN); + partitioning = (best_lp_wm == &lp_wm_1_2) ? + INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; - i915->sagv_block_time_us = intel_sagv_block_time(i915); + ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results); - drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n", - str_yes_no(intel_has_sagv(i915)), i915->sagv_block_time_us); + ilk_write_wm_values(dev_priv, &results); +} - /* avoid overflow when adding with wm0 latency/etc. */ - if (drm_WARN(&i915->drm, i915->sagv_block_time_us > U16_MAX, - "Excessive SAGV block time %u, ignoring\n", - i915->sagv_block_time_us)) - i915->sagv_block_time_us = 0; +static void ilk_initial_watermarks(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); - if (!intel_has_sagv(i915)) - i915->sagv_block_time_us = 0; + mutex_lock(&dev_priv->display.wm.wm_mutex); + crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate; + ilk_program_watermarks(dev_priv); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } -/* - * SAGV dynamically adjusts the system agent voltage and clock frequencies - * depending on power and performance requirements. The display engine access - * to system memory is blocked during the adjustment time. Because of the - * blocking time, having this enabled can cause full system hangs and/or pipe - * underruns if we don't meet all of the following requirements: - * - * - <= 1 pipe enabled - * - All planes can enable watermarks for latencies >= SAGV engine block time - * - We're not using an interlaced display configuration - */ -static void skl_sagv_enable(struct drm_i915_private *dev_priv) +static void ilk_optimize_watermarks(struct intel_atomic_state *state, + struct intel_crtc *crtc) { - int ret; - - if (!intel_has_sagv(dev_priv)) - return; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); - if (dev_priv->sagv_status == I915_SAGV_ENABLED) + if (!crtc_state->wm.need_postvbl_update) return; - drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n"); - ret = snb_pcode_write(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL, - GEN9_SAGV_ENABLE); + mutex_lock(&dev_priv->display.wm.wm_mutex); + crtc->wm.active.ilk = crtc_state->wm.ilk.optimal; + ilk_program_watermarks(dev_priv); + mutex_unlock(&dev_priv->display.wm.wm_mutex); +} - /* We don't need to wait for SAGV when enabling */ +static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) +{ + struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct ilk_wm_values *hw = &dev_priv->display.wm.hw; + struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); + struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; + enum pipe pipe = crtc->pipe; - /* - * Some skl systems, pre-release machines in particular, - * don't actually have SAGV. - */ - if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { - drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n"); - dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; - return; - } else if (ret < 0) { - drm_err(&dev_priv->drm, "Failed to enable SAGV\n"); - return; - } + hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe)); - dev_priv->sagv_status = I915_SAGV_ENABLED; -} + memset(active, 0, sizeof(*active)); -static void skl_sagv_disable(struct drm_i915_private *dev_priv) -{ - int ret; + active->pipe_enabled = crtc->active; - if (!intel_has_sagv(dev_priv)) - return; + if (active->pipe_enabled) { + u32 tmp = hw->wm_pipe[pipe]; - if (dev_priv->sagv_status == I915_SAGV_DISABLED) - return; + /* + * For active pipes LP0 watermark is marked as + * enabled, and LP1+ watermaks as disabled since + * we can't really reverse compute them in case + * multiple pipes are active. + */ + active->wm[0].enable = true; + active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp); + active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp); + active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp); + } else { + int level, max_level = ilk_wm_max_level(dev_priv); - drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n"); - /* bspec says to keep retrying for at least 1 ms */ - ret = skl_pcode_request(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL, - GEN9_SAGV_DISABLE, - GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED, - 1); - /* - * Some skl systems, pre-release machines in particular, - * don't actually have SAGV. - */ - if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) { - drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n"); - dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED; - return; - } else if (ret < 0) { - drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret); - return; + /* + * For inactive pipes, all watermark levels + * should be marked as enabled but zeroed, + * which is what we'd compute them to. + */ + for (level = 0; level <= max_level; level++) + active->wm[level].enable = true; } - dev_priv->sagv_status = I915_SAGV_DISABLED; + crtc->wm.active.ilk = *active; } -static void skl_sagv_pre_plane_update(struct intel_atomic_state *state) +#define _FW_WM(value, plane) \ + (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) +#define _FW_WM_VLV(value, plane) \ + (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) + +static void g4x_read_wm_values(struct drm_i915_private *dev_priv, + struct g4x_wm_values *wm) { - struct drm_i915_private *i915 = to_i915(state->base.dev); - const struct intel_bw_state *new_bw_state = - intel_atomic_get_new_bw_state(state); + u32 tmp; - if (!new_bw_state) - return; + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); + wm->sr.plane = _FW_WM(tmp, SR); + wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); + wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); + wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); + + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); + wm->fbc_en = tmp & DSPFW_FBC_SR_EN; + wm->sr.fbc = _FW_WM(tmp, FBC_SR); + wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); + wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB); + wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); + wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); - if (!intel_can_enable_sagv(i915, new_bw_state)) - skl_sagv_disable(i915); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); + wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; + wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); + wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); + wm->hpll.plane = _FW_WM(tmp, HPLL_SR); } -static void skl_sagv_post_plane_update(struct intel_atomic_state *state) +static void vlv_read_wm_values(struct drm_i915_private *dev_priv, + struct vlv_wm_values *wm) { - struct drm_i915_private *i915 = to_i915(state->base.dev); - const struct intel_bw_state *new_bw_state = - intel_atomic_get_new_bw_state(state); + enum pipe pipe; + u32 tmp; - if (!new_bw_state) - return; + for_each_pipe(dev_priv, pipe) { + tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe)); - if (intel_can_enable_sagv(i915, new_bw_state)) - skl_sagv_enable(i915); -} + wm->ddl[pipe].plane[PLANE_PRIMARY] = + (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].plane[PLANE_CURSOR] = + (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].plane[PLANE_SPRITE0] = + (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + wm->ddl[pipe].plane[PLANE_SPRITE1] = + (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); + } -static void icl_sagv_pre_plane_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_bw_state *old_bw_state = - intel_atomic_get_old_bw_state(state); - const struct intel_bw_state *new_bw_state = - intel_atomic_get_new_bw_state(state); - u16 old_mask, new_mask; + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); + wm->sr.plane = _FW_WM(tmp, SR); + wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); + wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); + wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); - if (!new_bw_state) - return; + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); + wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); + wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); + wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); - old_mask = old_bw_state->qgv_points_mask; - new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); + wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); - if (old_mask == new_mask) - return; + if (IS_CHERRYVIEW(dev_priv)) { + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV); + wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); + wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); - WARN_ON(!new_bw_state->base.changed); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV); + wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); + wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); - drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n", - old_mask, new_mask); + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV); + wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); + wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); - /* - * Restrict required qgv points before updating the configuration. - * According to BSpec we can't mask and unmask qgv points at the same - * time. Also masking should be done before updating the configuration - * and unmasking afterwards. - */ - icl_pcode_restrict_qgv_points(dev_priv, new_mask); + tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); + wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; + wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; + wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; + wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8; + wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; + wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; + wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; + } else { + tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7); + wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); + wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); + + tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); + wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; + wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; + wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; + wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; + wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; + } } -static void icl_sagv_post_plane_update(struct intel_atomic_state *state) +#undef _FW_WM +#undef _FW_WM_VLV + +void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_bw_state *old_bw_state = - intel_atomic_get_old_bw_state(state); - const struct intel_bw_state *new_bw_state = - intel_atomic_get_new_bw_state(state); - u16 old_mask, new_mask; + struct g4x_wm_values *wm = &dev_priv->display.wm.g4x; + struct intel_crtc *crtc; - if (!new_bw_state) - return; + g4x_read_wm_values(dev_priv, wm); - old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask; - new_mask = new_bw_state->qgv_points_mask; - - if (old_mask == new_mask) - return; - - WARN_ON(!new_bw_state->base.changed); - - drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n", - old_mask, new_mask); - - /* - * Allow required qgv points after updating the configuration. - * According to BSpec we can't mask and unmask qgv points at the same - * time. Also masking should be done before updating the configuration - * and unmasking afterwards. - */ - icl_pcode_restrict_qgv_points(dev_priv, new_mask); -} - -void intel_sagv_pre_plane_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - - /* - * Just return if we can't control SAGV or don't have it. - * This is different from situation when we have SAGV but just can't - * afford it due to DBuf limitation - in case if SAGV is completely - * disabled in a BIOS, we are not even allowed to send a PCode request, - * as it will throw an error. So have to check it here. - */ - if (!intel_has_sagv(i915)) - return; - - if (DISPLAY_VER(i915) >= 11) - icl_sagv_pre_plane_update(state); - else - skl_sagv_pre_plane_update(state); -} - -void intel_sagv_post_plane_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - - /* - * Just return if we can't control SAGV or don't have it. - * This is different from situation when we have SAGV but just can't - * afford it due to DBuf limitation - in case if SAGV is completely - * disabled in a BIOS, we are not even allowed to send a PCode request, - * as it will throw an error. So have to check it here. - */ - if (!intel_has_sagv(i915)) - return; - - if (DISPLAY_VER(i915) >= 11) - icl_sagv_post_plane_update(state); - else - skl_sagv_post_plane_update(state); -} - -static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum plane_id plane_id; - int max_level = INT_MAX; - - if (!intel_has_sagv(dev_priv)) - return false; - - if (!crtc_state->hw.active) - return true; - - if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE) - return false; - - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - int level; - - /* Skip this plane if it's not enabled */ - if (!wm->wm[0].enable) - continue; - - /* Find the highest enabled wm level for this plane */ - for (level = ilk_wm_max_level(dev_priv); - !wm->wm[level].enable; --level) - { } - - /* Highest common enabled wm level for all planes */ - max_level = min(level, max_level); - } - - /* No enabled planes? */ - if (max_level == INT_MAX) - return true; - - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - /* - * All enabled planes must have enabled a common wm level that - * can tolerate memory latencies higher than sagv_block_time_us - */ - if (wm->wm[0].enable && !wm->wm[max_level].can_sagv) - return false; - } - - return true; -} - -static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - enum plane_id plane_id; - - if (!crtc_state->hw.active) - return true; - - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - if (wm->wm[0].enable && !wm->sagv.wm0.enable) - return false; - } - - return true; -} - -static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - if (DISPLAY_VER(dev_priv) >= 12) - return tgl_crtc_can_enable_sagv(crtc_state); - else - return skl_crtc_can_enable_sagv(crtc_state); -} - -bool intel_can_enable_sagv(struct drm_i915_private *dev_priv, - const struct intel_bw_state *bw_state) -{ - if (DISPLAY_VER(dev_priv) < 11 && - bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes)) - return false; - - return bw_state->pipe_sagv_reject == 0; -} - -static int intel_compute_sagv_mask(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - int ret; - struct intel_crtc *crtc; - struct intel_crtc_state *new_crtc_state; - struct intel_bw_state *new_bw_state = NULL; - const struct intel_bw_state *old_bw_state = NULL; - int i; - - for_each_new_intel_crtc_in_state(state, crtc, - new_crtc_state, i) { - new_bw_state = intel_atomic_get_bw_state(state); - if (IS_ERR(new_bw_state)) - return PTR_ERR(new_bw_state); - - old_bw_state = intel_atomic_get_old_bw_state(state); - - if (intel_crtc_can_enable_sagv(new_crtc_state)) - new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe); - else - new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe); - } - - if (!new_bw_state) - return 0; - - new_bw_state->active_pipes = - intel_calc_active_pipes(state, old_bw_state->active_pipes); - - if (new_bw_state->active_pipes != old_bw_state->active_pipes) { - ret = intel_atomic_lock_global_state(&new_bw_state->base); - if (ret) - return ret; - } - - if (intel_can_enable_sagv(dev_priv, new_bw_state) != - intel_can_enable_sagv(dev_priv, old_bw_state)) { - ret = intel_atomic_serialize_global_state(&new_bw_state->base); - if (ret) - return ret; - } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) { - ret = intel_atomic_lock_global_state(&new_bw_state->base); - if (ret) - return ret; - } - - for_each_new_intel_crtc_in_state(state, crtc, - new_crtc_state, i) { - struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal; - - /* - * We store use_sagv_wm in the crtc state rather than relying on - * that bw state since we have no convenient way to get at the - * latter from the plane commit hooks (especially in the legacy - * cursor case) - */ - pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) && - DISPLAY_VER(dev_priv) >= 12 && - intel_can_enable_sagv(dev_priv, new_bw_state); - } - - return 0; -} - -static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry, - u16 start, u16 end) -{ - entry->start = start; - entry->end = end; - - return end; -} - -static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv) -{ - return INTEL_INFO(dev_priv)->display.dbuf.size / - hweight8(INTEL_INFO(dev_priv)->display.dbuf.slice_mask); -} - -static void -skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask, - struct skl_ddb_entry *ddb) -{ - int slice_size = intel_dbuf_slice_size(dev_priv); - - if (!slice_mask) { - ddb->start = 0; - ddb->end = 0; - return; - } - - ddb->start = (ffs(slice_mask) - 1) * slice_size; - ddb->end = fls(slice_mask) * slice_size; - - WARN_ON(ddb->start >= ddb->end); - WARN_ON(ddb->end > INTEL_INFO(dev_priv)->display.dbuf.size); -} - -static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask) -{ - struct skl_ddb_entry ddb; - - if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2))) - slice_mask = BIT(DBUF_S1); - else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4))) - slice_mask = BIT(DBUF_S3); - - skl_ddb_entry_for_slices(i915, slice_mask, &ddb); - - return ddb.start; -} - -u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv, - const struct skl_ddb_entry *entry) -{ - int slice_size = intel_dbuf_slice_size(dev_priv); - enum dbuf_slice start_slice, end_slice; - u8 slice_mask = 0; - - if (!skl_ddb_entry_size(entry)) - return 0; - - start_slice = entry->start / slice_size; - end_slice = (entry->end - 1) / slice_size; - - /* - * Per plane DDB entry can in a really worst case be on multiple slices - * but single entry is anyway contigious. - */ - while (start_slice <= end_slice) { - slice_mask |= BIT(start_slice); - start_slice++; - } - - return slice_mask; -} - -static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state) -{ - const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode; - int hdisplay, vdisplay; - - if (!crtc_state->hw.active) - return 0; - - /* - * Watermark/ddb requirement highly depends upon width of the - * framebuffer, So instead of allocating DDB equally among pipes - * distribute DDB based on resolution/width of the display. - */ - drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay); - - return hdisplay; -} - -static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state, - enum pipe for_pipe, - unsigned int *weight_start, - unsigned int *weight_end, - unsigned int *weight_total) -{ - struct drm_i915_private *dev_priv = - to_i915(dbuf_state->base.state->base.dev); - enum pipe pipe; - - *weight_start = 0; - *weight_end = 0; - *weight_total = 0; - - for_each_pipe(dev_priv, pipe) { - int weight = dbuf_state->weight[pipe]; - - /* - * Do not account pipes using other slice sets - * luckily as of current BSpec slice sets do not partially - * intersect(pipes share either same one slice or same slice set - * i.e no partial intersection), so it is enough to check for - * equality for now. - */ - if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe]) - continue; - - *weight_total += weight; - if (pipe < for_pipe) { - *weight_start += weight; - *weight_end += weight; - } else if (pipe == for_pipe) { - *weight_end += weight; - } - } -} - -static int -skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - unsigned int weight_total, weight_start, weight_end; - const struct intel_dbuf_state *old_dbuf_state = - intel_atomic_get_old_dbuf_state(state); - struct intel_dbuf_state *new_dbuf_state = - intel_atomic_get_new_dbuf_state(state); - struct intel_crtc_state *crtc_state; - struct skl_ddb_entry ddb_slices; - enum pipe pipe = crtc->pipe; - unsigned int mbus_offset = 0; - u32 ddb_range_size; - u32 dbuf_slice_mask; - u32 start, end; - int ret; - - if (new_dbuf_state->weight[pipe] == 0) { - skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0); - goto out; - } - - dbuf_slice_mask = new_dbuf_state->slices[pipe]; - - skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices); - mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask); - ddb_range_size = skl_ddb_entry_size(&ddb_slices); - - intel_crtc_dbuf_weights(new_dbuf_state, pipe, - &weight_start, &weight_end, &weight_total); - - start = ddb_range_size * weight_start / weight_total; - end = ddb_range_size * weight_end / weight_total; - - skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], - ddb_slices.start - mbus_offset + start, - ddb_slices.start - mbus_offset + end); - -out: - if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] && - skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe], - &new_dbuf_state->ddb[pipe])) - return 0; - - ret = intel_atomic_lock_global_state(&new_dbuf_state->base); - if (ret) - return ret; - - crtc_state = intel_atomic_get_crtc_state(&state->base, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - - /* - * Used for checking overlaps, so we need absolute - * offsets instead of MBUS relative offsets. - */ - crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start; - crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end; - - drm_dbg_kms(&dev_priv->drm, - "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n", - crtc->base.base.id, crtc->base.name, - old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe], - old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end, - new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end, - old_dbuf_state->active_pipes, new_dbuf_state->active_pipes); - - return 0; -} - -static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, - int width, const struct drm_format_info *format, - u64 modifier, unsigned int rotation, - u32 plane_pixel_rate, struct skl_wm_params *wp, - int color_plane); - -static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, - struct intel_plane *plane, - int level, - unsigned int latency, - const struct skl_wm_params *wp, - const struct skl_wm_level *result_prev, - struct skl_wm_level *result /* out */); - -static unsigned int -skl_cursor_allocation(const struct intel_crtc_state *crtc_state, - int num_active) -{ - struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor); - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - int level, max_level = ilk_wm_max_level(dev_priv); - struct skl_wm_level wm = {}; - int ret, min_ddb_alloc = 0; - struct skl_wm_params wp; - - ret = skl_compute_wm_params(crtc_state, 256, - drm_format_info(DRM_FORMAT_ARGB8888), - DRM_FORMAT_MOD_LINEAR, - DRM_MODE_ROTATE_0, - crtc_state->pixel_rate, &wp, 0); - drm_WARN_ON(&dev_priv->drm, ret); - - for (level = 0; level <= max_level; level++) { - unsigned int latency = dev_priv->wm.skl_latency[level]; - - skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm); - if (wm.min_ddb_alloc == U16_MAX) - break; - - min_ddb_alloc = wm.min_ddb_alloc; - } - - return max(num_active == 1 ? 32 : 8, min_ddb_alloc); -} - -static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg) -{ - skl_ddb_entry_init(entry, - REG_FIELD_GET(PLANE_BUF_START_MASK, reg), - REG_FIELD_GET(PLANE_BUF_END_MASK, reg)); - if (entry->end) - entry->end++; -} - -static void -skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv, - const enum pipe pipe, - const enum plane_id plane_id, - struct skl_ddb_entry *ddb, - struct skl_ddb_entry *ddb_y) -{ - u32 val; - - /* Cursor doesn't support NV12/planar, so no extra calculation needed */ - if (plane_id == PLANE_CURSOR) { - val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe)); - skl_ddb_entry_init_from_hw(ddb, val); - return; - } - - val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id)); - skl_ddb_entry_init_from_hw(ddb, val); - - if (DISPLAY_VER(dev_priv) >= 11) - return; - - val = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id)); - skl_ddb_entry_init_from_hw(ddb_y, val); -} - -static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc, - struct skl_ddb_entry *ddb, - struct skl_ddb_entry *ddb_y) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum intel_display_power_domain power_domain; - enum pipe pipe = crtc->pipe; - intel_wakeref_t wakeref; - enum plane_id plane_id; - - power_domain = POWER_DOMAIN_PIPE(pipe); - wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); - if (!wakeref) - return; - - for_each_plane_id_on_crtc(crtc, plane_id) - skl_ddb_get_hw_plane_state(dev_priv, pipe, - plane_id, - &ddb[plane_id], - &ddb_y[plane_id]); - - intel_display_power_put(dev_priv, power_domain, wakeref); -} - -struct dbuf_slice_conf_entry { - u8 active_pipes; - u8 dbuf_mask[I915_MAX_PIPES]; - bool join_mbus; -}; - -/* - * Table taken from Bspec 12716 - * Pipes do have some preferred DBuf slice affinity, - * plus there are some hardcoded requirements on how - * those should be distributed for multipipe scenarios. - * For more DBuf slices algorithm can get even more messy - * and less readable, so decided to use a table almost - * as is from BSpec itself - that way it is at least easier - * to compare, change and check. - */ -static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] = -/* Autogenerated with igt/tools/intel_dbuf_map tool: */ -{ - { - .active_pipes = BIT(PIPE_A), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - }, - }, - { - .active_pipes = BIT(PIPE_B), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_C), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - {} -}; - -/* - * Table taken from Bspec 49255 - * Pipes do have some preferred DBuf slice affinity, - * plus there are some hardcoded requirements on how - * those should be distributed for multipipe scenarios. - * For more DBuf slices algorithm can get even more messy - * and less readable, so decided to use a table almost - * as is from BSpec itself - that way it is at least easier - * to compare, change and check. - */ -static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = -/* Autogenerated with igt/tools/intel_dbuf_map tool: */ -{ - { - .active_pipes = BIT(PIPE_A), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S1), - }, - }, - { - .active_pipes = BIT(PIPE_C), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_D), - .dbuf_mask = { - [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2), - }, - }, - {} -}; - -static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = { - { - .active_pipes = BIT(PIPE_A), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_C), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_D), - .dbuf_mask = { - [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S3), - [PIPE_D] = BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3), - [PIPE_D] = BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3), - [PIPE_D] = BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3), - [PIPE_D] = BIT(DBUF_S4), - }, - }, - {} -}; - -static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = { - /* - * Keep the join_mbus cases first so check_mbus_joined() - * will prefer them over the !join_mbus cases. - */ - { - .active_pipes = BIT(PIPE_A), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), - }, - .join_mbus = true, - }, - { - .active_pipes = BIT(PIPE_B), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4), - }, - .join_mbus = true, - }, - { - .active_pipes = BIT(PIPE_A), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - .join_mbus = false, - }, - { - .active_pipes = BIT(PIPE_B), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - .join_mbus = false, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_C), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - }, - }, - { - .active_pipes = BIT(PIPE_D), - .dbuf_mask = { - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - { - .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4), - [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2), - }, - }, - {} - -}; - -static bool check_mbus_joined(u8 active_pipes, - const struct dbuf_slice_conf_entry *dbuf_slices) -{ - int i; - - for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { - if (dbuf_slices[i].active_pipes == active_pipes) - return dbuf_slices[i].join_mbus; - } - return false; -} - -static bool adlp_check_mbus_joined(u8 active_pipes) -{ - return check_mbus_joined(active_pipes, adlp_allowed_dbufs); -} - -static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus, - const struct dbuf_slice_conf_entry *dbuf_slices) -{ - int i; - - for (i = 0; dbuf_slices[i].active_pipes != 0; i++) { - if (dbuf_slices[i].active_pipes == active_pipes && - dbuf_slices[i].join_mbus == join_mbus) - return dbuf_slices[i].dbuf_mask[pipe]; - } - return 0; -} - -/* - * This function finds an entry with same enabled pipe configuration and - * returns correspondent DBuf slice mask as stated in BSpec for particular - * platform. - */ -static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) -{ - /* - * FIXME: For ICL this is still a bit unclear as prev BSpec revision - * required calculating "pipe ratio" in order to determine - * if one or two slices can be used for single pipe configurations - * as additional constraint to the existing table. - * However based on recent info, it should be not "pipe ratio" - * but rather ratio between pixel_rate and cdclk with additional - * constants, so for now we are using only table until this is - * clarified. Also this is the reason why crtc_state param is - * still here - we will need it once those additional constraints - * pop up. - */ - return compute_dbuf_slices(pipe, active_pipes, join_mbus, - icl_allowed_dbufs); -} - -static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) -{ - return compute_dbuf_slices(pipe, active_pipes, join_mbus, - tgl_allowed_dbufs); -} - -static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) -{ - return compute_dbuf_slices(pipe, active_pipes, join_mbus, - adlp_allowed_dbufs); -} - -static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus) -{ - return compute_dbuf_slices(pipe, active_pipes, join_mbus, - dg2_allowed_dbufs); -} - -static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - - if (IS_DG2(dev_priv)) - return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus); - else if (IS_ALDERLAKE_P(dev_priv)) - return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus); - else if (DISPLAY_VER(dev_priv) == 12) - return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus); - else if (DISPLAY_VER(dev_priv) == 11) - return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus); - /* - * For anything else just return one slice yet. - * Should be extended for other platforms. - */ - return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0; -} - -static bool -use_minimal_wm0_only(const struct intel_crtc_state *crtc_state, - struct intel_plane *plane) -{ - struct drm_i915_private *i915 = to_i915(plane->base.dev); - - return DISPLAY_VER(i915) >= 13 && - crtc_state->uapi.async_flip && - plane->async_flip; -} - -static u64 -skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *i915 = to_i915(crtc->base.dev); - enum plane_id plane_id; - u64 data_rate = 0; - - for_each_plane_id_on_crtc(crtc, plane_id) { - if (plane_id == PLANE_CURSOR) - continue; - - data_rate += crtc_state->rel_data_rate[plane_id]; - - if (DISPLAY_VER(i915) < 11) - data_rate += crtc_state->rel_data_rate_y[plane_id]; - } - - return data_rate; -} - -static const struct skl_wm_level * -skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm, - enum plane_id plane_id, - int level) -{ - const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; - - if (level == 0 && pipe_wm->use_sagv_wm) - return &wm->sagv.wm0; - - return &wm->wm[level]; -} - -static const struct skl_wm_level * -skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm, - enum plane_id plane_id) -{ - const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; - - if (pipe_wm->use_sagv_wm) - return &wm->sagv.trans_wm; - - return &wm->trans_wm; -} - -/* - * We only disable the watermarks for each plane if - * they exceed the ddb allocation of said plane. This - * is done so that we don't end up touching cursor - * watermarks needlessly when some other plane reduces - * our max possible watermark level. - * - * Bspec has this to say about the PLANE_WM enable bit: - * "All the watermarks at this level for all enabled - * planes must be enabled before the level will be used." - * So this is actually safe to do. - */ -static void -skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb) -{ - if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) - memset(wm, 0, sizeof(*wm)); -} - -static void -skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm, - const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb) -{ - if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) || - uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) { - memset(wm, 0, sizeof(*wm)); - memset(uv_wm, 0, sizeof(*uv_wm)); - } -} - -static bool icl_need_wm1_wa(struct drm_i915_private *i915, - enum plane_id plane_id) -{ - /* - * Wa_1408961008:icl, ehl - * Wa_14012656716:tgl, adl - * Underruns with WM1+ disabled - */ - return DISPLAY_VER(i915) == 11 || - (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR); -} - -struct skl_plane_ddb_iter { - u64 data_rate; - u16 start, size; -}; - -static void -skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter, - struct skl_ddb_entry *ddb, - const struct skl_wm_level *wm, - u64 data_rate) -{ - u16 size, extra = 0; - - if (data_rate) { - extra = min_t(u16, iter->size, - DIV64_U64_ROUND_UP(iter->size * data_rate, - iter->data_rate)); - iter->size -= extra; - iter->data_rate -= data_rate; - } - - /* - * Keep ddb entry of all disabled planes explicitly zeroed - * to avoid skl_ddb_add_affected_planes() adding them to - * the state when other planes change their allocations. - */ - size = wm->min_ddb_alloc + extra; - if (size) - iter->start = skl_ddb_entry_init(ddb, iter->start, - iter->start + size); -} - -static int -skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - const struct intel_dbuf_state *dbuf_state = - intel_atomic_get_new_dbuf_state(state); - const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe]; - int num_active = hweight8(dbuf_state->active_pipes); - struct skl_plane_ddb_iter iter; - enum plane_id plane_id; - u16 cursor_size; - u32 blocks; - int level; - - /* Clear the partitioning for disabled planes. */ - memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb)); - memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); - - if (!crtc_state->hw.active) - return 0; - - iter.start = alloc->start; - iter.size = skl_ddb_entry_size(alloc); - if (iter.size == 0) - return 0; - - /* Allocate fixed number of blocks for cursor. */ - cursor_size = skl_cursor_allocation(crtc_state, num_active); - iter.size -= cursor_size; - skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR], - alloc->end - cursor_size, alloc->end); - - iter.data_rate = skl_total_relative_data_rate(crtc_state); - - /* - * Find the highest watermark level for which we can satisfy the block - * requirement of active planes. - */ - for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) { - blocks = 0; - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - if (plane_id == PLANE_CURSOR) { - const struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - - if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) { - drm_WARN_ON(&dev_priv->drm, - wm->wm[level].min_ddb_alloc != U16_MAX); - blocks = U32_MAX; - break; - } - continue; - } - - blocks += wm->wm[level].min_ddb_alloc; - blocks += wm->uv_wm[level].min_ddb_alloc; - } - - if (blocks <= iter.size) { - iter.size -= blocks; - break; - } - } - - if (level < 0) { - drm_dbg_kms(&dev_priv->drm, - "Requested display configuration exceeds system DDB limitations"); - drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n", - blocks, iter.size); - return -EINVAL; - } - - /* avoid the WARN later when we don't allocate any extra DDB */ - if (iter.data_rate == 0) - iter.size = 0; - - /* - * Grant each plane the blocks it requires at the highest achievable - * watermark level, plus an extra share of the leftover blocks - * proportional to its relative data rate. - */ - for_each_plane_id_on_crtc(crtc, plane_id) { - struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - struct skl_ddb_entry *ddb_y = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - const struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - if (plane_id == PLANE_CURSOR) - continue; - - if (DISPLAY_VER(dev_priv) < 11 && - crtc_state->nv12_planes & BIT(plane_id)) { - skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level], - crtc_state->rel_data_rate_y[plane_id]); - skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level], - crtc_state->rel_data_rate[plane_id]); - } else { - skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level], - crtc_state->rel_data_rate[plane_id]); - } - } - drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0); - - /* - * When we calculated watermark values we didn't know how high - * of a level we'd actually be able to hit, so we just marked - * all levels as "enabled." Go back now and disable the ones - * that aren't actually possible. - */ - for (level++; level <= ilk_wm_max_level(dev_priv); level++) { - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - const struct skl_ddb_entry *ddb_y = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - if (DISPLAY_VER(dev_priv) < 11 && - crtc_state->nv12_planes & BIT(plane_id)) - skl_check_nv12_wm_level(&wm->wm[level], - &wm->uv_wm[level], - ddb_y, ddb); - else - skl_check_wm_level(&wm->wm[level], ddb); - - if (icl_need_wm1_wa(dev_priv, plane_id) && - level == 1 && wm->wm[0].enable) { - wm->wm[level].blocks = wm->wm[0].blocks; - wm->wm[level].lines = wm->wm[0].lines; - wm->wm[level].ignore_lines = wm->wm[0].ignore_lines; - } - } - } - - /* - * Go back and disable the transition and SAGV watermarks - * if it turns out we don't have enough DDB blocks for them. - */ - for_each_plane_id_on_crtc(crtc, plane_id) { - const struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - const struct skl_ddb_entry *ddb_y = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - struct skl_plane_wm *wm = - &crtc_state->wm.skl.optimal.planes[plane_id]; - - if (DISPLAY_VER(dev_priv) < 11 && - crtc_state->nv12_planes & BIT(plane_id)) { - skl_check_wm_level(&wm->trans_wm, ddb_y); - } else { - WARN_ON(skl_ddb_entry_size(ddb_y)); - - skl_check_wm_level(&wm->trans_wm, ddb); - } - - skl_check_wm_level(&wm->sagv.wm0, ddb); - skl_check_wm_level(&wm->sagv.trans_wm, ddb); - } - - return 0; -} - -/* - * The max latency should be 257 (max the punit can code is 255 and we add 2us - * for the read latency) and cpp should always be <= 8, so that - * should allow pixel_rate up to ~2 GHz which seems sufficient since max - * 2xcdclk is 1350 MHz and the pixel rate should never exceed that. -*/ -static uint_fixed_16_16_t -skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate, - u8 cpp, u32 latency, u32 dbuf_block_size) -{ - u32 wm_intermediate_val; - uint_fixed_16_16_t ret; - - if (latency == 0) - return FP_16_16_MAX; - - wm_intermediate_val = latency * pixel_rate * cpp; - ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size); - - if (DISPLAY_VER(dev_priv) >= 10) - ret = add_fixed16_u32(ret, 1); - - return ret; -} - -static uint_fixed_16_16_t -skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency, - uint_fixed_16_16_t plane_blocks_per_line) -{ - u32 wm_intermediate_val; - uint_fixed_16_16_t ret; - - if (latency == 0) - return FP_16_16_MAX; - - wm_intermediate_val = latency * pixel_rate; - wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val, - pipe_htotal * 1000); - ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line); - return ret; -} - -static uint_fixed_16_16_t -intel_get_linetime_us(const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u32 pixel_rate; - u32 crtc_htotal; - uint_fixed_16_16_t linetime_us; - - if (!crtc_state->hw.active) - return u32_to_fixed16(0); - - pixel_rate = crtc_state->pixel_rate; - - if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0)) - return u32_to_fixed16(0); - - crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal; - linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate); - - return linetime_us; -} - -static int -skl_compute_wm_params(const struct intel_crtc_state *crtc_state, - int width, const struct drm_format_info *format, - u64 modifier, unsigned int rotation, - u32 plane_pixel_rate, struct skl_wm_params *wp, - int color_plane) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 interm_pbpl; - - /* only planar format has two planes */ - if (color_plane == 1 && - !intel_format_info_is_yuv_semiplanar(format, modifier)) { - drm_dbg_kms(&dev_priv->drm, - "Non planar format have single plane\n"); - return -EINVAL; - } - - wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED || - modifier == I915_FORMAT_MOD_4_TILED || - modifier == I915_FORMAT_MOD_Yf_TILED || - modifier == I915_FORMAT_MOD_Y_TILED_CCS || - modifier == I915_FORMAT_MOD_Yf_TILED_CCS; - wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED; - wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS || - modifier == I915_FORMAT_MOD_Yf_TILED_CCS; - wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier); - - wp->width = width; - if (color_plane == 1 && wp->is_planar) - wp->width /= 2; - - wp->cpp = format->cpp[color_plane]; - wp->plane_pixel_rate = plane_pixel_rate; - - if (DISPLAY_VER(dev_priv) >= 11 && - modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1) - wp->dbuf_block_size = 256; - else - wp->dbuf_block_size = 512; - - if (drm_rotation_90_or_270(rotation)) { - switch (wp->cpp) { - case 1: - wp->y_min_scanlines = 16; - break; - case 2: - wp->y_min_scanlines = 8; - break; - case 4: - wp->y_min_scanlines = 4; - break; - default: - MISSING_CASE(wp->cpp); - return -EINVAL; - } - } else { - wp->y_min_scanlines = 4; - } - - if (skl_needs_memory_bw_wa(dev_priv)) - wp->y_min_scanlines *= 2; - - wp->plane_bytes_per_line = wp->width * wp->cpp; - if (wp->y_tiled) { - interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line * - wp->y_min_scanlines, - wp->dbuf_block_size); - - if (DISPLAY_VER(dev_priv) >= 10) - interm_pbpl++; - - wp->plane_blocks_per_line = div_fixed16(interm_pbpl, - wp->y_min_scanlines); - } else { - interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, - wp->dbuf_block_size); - - if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10) - interm_pbpl++; - - wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); - } - - wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines, - wp->plane_blocks_per_line); - - wp->linetime_us = fixed16_to_u32_round_up( - intel_get_linetime_us(crtc_state)); - - return 0; -} - -static int -skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - struct skl_wm_params *wp, int color_plane) -{ - const struct drm_framebuffer *fb = plane_state->hw.fb; - int width; - - /* - * Src coordinates are already rotated by 270 degrees for - * the 90/270 degree plane rotation cases (to match the - * GTT mapping), hence no need to account for rotation here. - */ - width = drm_rect_width(&plane_state->uapi.src) >> 16; - - return skl_compute_wm_params(crtc_state, width, - fb->format, fb->modifier, - plane_state->hw.rotation, - intel_plane_pixel_rate(crtc_state, plane_state), - wp, color_plane); -} - -static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level) -{ - if (DISPLAY_VER(dev_priv) >= 10) - return true; - - /* The number of lines are ignored for the level 0 watermark. */ - return level > 0; -} - -static int skl_wm_max_lines(struct drm_i915_private *dev_priv) -{ - if (DISPLAY_VER(dev_priv) >= 13) - return 255; - else - return 31; -} - -static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, - struct intel_plane *plane, - int level, - unsigned int latency, - const struct skl_wm_params *wp, - const struct skl_wm_level *result_prev, - struct skl_wm_level *result /* out */) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - uint_fixed_16_16_t method1, method2; - uint_fixed_16_16_t selected_result; - u32 blocks, lines, min_ddb_alloc = 0; - - if (latency == 0 || - (use_minimal_wm0_only(crtc_state, plane) && level > 0)) { - /* reject it */ - result->min_ddb_alloc = U16_MAX; - return; - } - - /* - * WaIncreaseLatencyIPCEnabled: kbl,cfl - * Display WA #1141: kbl,cfl - */ - if ((IS_KABYLAKE(dev_priv) || - IS_COFFEELAKE(dev_priv) || - IS_COMETLAKE(dev_priv)) && - dev_priv->ipc_enabled) - latency += 4; - - if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled) - latency += 15; - - method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate, - wp->cpp, latency, wp->dbuf_block_size); - method2 = skl_wm_method2(wp->plane_pixel_rate, - crtc_state->hw.pipe_mode.crtc_htotal, - latency, - wp->plane_blocks_per_line); - - if (wp->y_tiled) { - selected_result = max_fixed16(method2, wp->y_tile_minimum); - } else { - if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal / - wp->dbuf_block_size < 1) && - (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) { - selected_result = method2; - } else if (latency >= wp->linetime_us) { - if (DISPLAY_VER(dev_priv) == 9) - selected_result = min_fixed16(method1, method2); - else - selected_result = method2; - } else { - selected_result = method1; - } - } - - blocks = fixed16_to_u32_round_up(selected_result) + 1; - /* - * Lets have blocks at minimum equivalent to plane_blocks_per_line - * as there will be at minimum one line for lines configuration. This - * is a work around for FIFO underruns observed with resolutions like - * 4k 60 Hz in single channel DRAM configurations. - * - * As per the Bspec 49325, if the ddb allocation can hold at least - * one plane_blocks_per_line, we should have selected method2 in - * the above logic. Assuming that modern versions have enough dbuf - * and method2 guarantees blocks equivalent to at least 1 line, - * select the blocks as plane_blocks_per_line. - * - * TODO: Revisit the logic when we have better understanding on DRAM - * channels' impact on the level 0 memory latency and the relevant - * wm calculations. - */ - if (skl_wm_has_lines(dev_priv, level)) - blocks = max(blocks, - fixed16_to_u32_round_up(wp->plane_blocks_per_line)); - lines = div_round_up_fixed16(selected_result, - wp->plane_blocks_per_line); - - if (DISPLAY_VER(dev_priv) == 9) { - /* Display WA #1125: skl,bxt,kbl */ - if (level == 0 && wp->rc_surface) - blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); - - /* Display WA #1126: skl,bxt,kbl */ - if (level >= 1 && level <= 7) { - if (wp->y_tiled) { - blocks += fixed16_to_u32_round_up(wp->y_tile_minimum); - lines += wp->y_min_scanlines; - } else { - blocks++; - } - - /* - * Make sure result blocks for higher latency levels are - * atleast as high as level below the current level. - * Assumption in DDB algorithm optimization for special - * cases. Also covers Display WA #1125 for RC. - */ - if (result_prev->blocks > blocks) - blocks = result_prev->blocks; - } - } - - if (DISPLAY_VER(dev_priv) >= 11) { - if (wp->y_tiled) { - int extra_lines; - - if (lines % wp->y_min_scanlines == 0) - extra_lines = wp->y_min_scanlines; - else - extra_lines = wp->y_min_scanlines * 2 - - lines % wp->y_min_scanlines; - - min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines, - wp->plane_blocks_per_line); - } else { - min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10); - } - } - - if (!skl_wm_has_lines(dev_priv, level)) - lines = 0; - - if (lines > skl_wm_max_lines(dev_priv)) { - /* reject it */ - result->min_ddb_alloc = U16_MAX; - return; - } - - /* - * If lines is valid, assume we can use this watermark level - * for now. We'll come back and disable it after we calculate the - * DDB allocation if it turns out we don't actually have enough - * blocks to satisfy it. - */ - result->blocks = blocks; - result->lines = lines; - /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */ - result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1; - result->enable = true; - - if (DISPLAY_VER(dev_priv) < 12 && dev_priv->sagv_block_time_us) - result->can_sagv = latency >= dev_priv->sagv_block_time_us; -} - -static void -skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, - struct intel_plane *plane, - const struct skl_wm_params *wm_params, - struct skl_wm_level *levels) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - int level, max_level = ilk_wm_max_level(dev_priv); - struct skl_wm_level *result_prev = &levels[0]; - - for (level = 0; level <= max_level; level++) { - struct skl_wm_level *result = &levels[level]; - unsigned int latency = dev_priv->wm.skl_latency[level]; - - skl_compute_plane_wm(crtc_state, plane, level, latency, - wm_params, result_prev, result); - - result_prev = result; - } -} - -static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state, - struct intel_plane *plane, - const struct skl_wm_params *wm_params, - struct skl_plane_wm *plane_wm) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0; - struct skl_wm_level *levels = plane_wm->wm; - unsigned int latency = 0; - - if (dev_priv->sagv_block_time_us) - latency = dev_priv->sagv_block_time_us + dev_priv->wm.skl_latency[0]; - - skl_compute_plane_wm(crtc_state, plane, 0, latency, - wm_params, &levels[0], - sagv_wm); -} - -static void skl_compute_transition_wm(struct drm_i915_private *dev_priv, - struct skl_wm_level *trans_wm, - const struct skl_wm_level *wm0, - const struct skl_wm_params *wp) -{ - u16 trans_min, trans_amount, trans_y_tile_min; - u16 wm0_blocks, trans_offset, blocks; - - /* Transition WM don't make any sense if ipc is disabled */ - if (!dev_priv->ipc_enabled) - return; - - /* - * WaDisableTWM:skl,kbl,cfl,bxt - * Transition WM are not recommended by HW team for GEN9 - */ - if (DISPLAY_VER(dev_priv) == 9) - return; - - if (DISPLAY_VER(dev_priv) >= 11) - trans_min = 4; - else - trans_min = 14; - - /* Display WA #1140: glk,cnl */ - if (DISPLAY_VER(dev_priv) == 10) - trans_amount = 0; - else - trans_amount = 10; /* This is configurable amount */ - - trans_offset = trans_min + trans_amount; - - /* - * The spec asks for Selected Result Blocks for wm0 (the real value), - * not Result Blocks (the integer value). Pay attention to the capital - * letters. The value wm_l0->blocks is actually Result Blocks, but - * since Result Blocks is the ceiling of Selected Result Blocks plus 1, - * and since we later will have to get the ceiling of the sum in the - * transition watermarks calculation, we can just pretend Selected - * Result Blocks is Result Blocks minus 1 and it should work for the - * current platforms. - */ - wm0_blocks = wm0->blocks - 1; - - if (wp->y_tiled) { - trans_y_tile_min = - (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum); - blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset; - } else { - blocks = wm0_blocks + trans_offset; - } - blocks++; - - /* - * Just assume we can enable the transition watermark. After - * computing the DDB we'll come back and disable it if that - * assumption turns out to be false. - */ - trans_wm->blocks = blocks; - trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1); - trans_wm->enable = true; -} - -static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - struct intel_plane *plane, int color_plane) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; - struct skl_wm_params wm_params; - int ret; - - ret = skl_compute_plane_wm_params(crtc_state, plane_state, - &wm_params, color_plane); - if (ret) - return ret; - - skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm); - - skl_compute_transition_wm(dev_priv, &wm->trans_wm, - &wm->wm[0], &wm_params); - - if (DISPLAY_VER(dev_priv) >= 12) { - tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm); - - skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm, - &wm->sagv.wm0, &wm_params); - } - - return 0; -} - -static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state, - struct intel_plane *plane) -{ - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id]; - struct skl_wm_params wm_params; - int ret; - - wm->is_planar = true; - - /* uv plane watermarks must also be validated for NV12/Planar */ - ret = skl_compute_plane_wm_params(crtc_state, plane_state, - &wm_params, 1); - if (ret) - return ret; - - skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm); - - return 0; -} - -static int skl_build_plane_wm(struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); - enum plane_id plane_id = plane->id; - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; - const struct drm_framebuffer *fb = plane_state->hw.fb; - int ret; - - memset(wm, 0, sizeof(*wm)); - - if (!intel_wm_plane_visible(crtc_state, plane_state)) - return 0; - - ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane, 0); - if (ret) - return ret; - - if (fb->format->is_yuv && fb->format->num_planes > 1) { - ret = skl_build_plane_wm_uv(crtc_state, plane_state, - plane); - if (ret) - return ret; - } - - return 0; -} - -static int icl_build_plane_wm(struct intel_crtc_state *crtc_state, - const struct intel_plane_state *plane_state) -{ - struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - enum plane_id plane_id = plane->id; - struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id]; - int ret; - - /* Watermarks calculated in master */ - if (plane_state->planar_slave) - return 0; - - memset(wm, 0, sizeof(*wm)); - - if (plane_state->planar_linked_plane) { - const struct drm_framebuffer *fb = plane_state->hw.fb; - - drm_WARN_ON(&dev_priv->drm, - !intel_wm_plane_visible(crtc_state, plane_state)); - drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv || - fb->format->num_planes == 1); - - ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane_state->planar_linked_plane, 0); - if (ret) - return ret; - - ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane, 1); - if (ret) - return ret; - } else if (intel_wm_plane_visible(crtc_state, plane_state)) { - ret = skl_build_plane_wm_single(crtc_state, plane_state, - plane, 0); - if (ret) - return ret; - } - - return 0; -} - -static int skl_build_pipe_wm(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - const struct intel_plane_state *plane_state; - struct intel_plane *plane; - int ret, i; - - for_each_new_intel_plane_in_state(state, plane, plane_state, i) { - /* - * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc - * instead but we don't populate that correctly for NV12 Y - * planes so for now hack this. - */ - if (plane->pipe != crtc->pipe) - continue; - - if (DISPLAY_VER(dev_priv) >= 11) - ret = icl_build_plane_wm(crtc_state, plane_state); - else - ret = skl_build_plane_wm(crtc_state, plane_state); - if (ret) - return ret; - } - - crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw; - - return 0; -} - -static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, - i915_reg_t reg, - const struct skl_ddb_entry *entry) -{ - if (entry->end) - intel_de_write_fw(dev_priv, reg, - PLANE_BUF_END(entry->end - 1) | - PLANE_BUF_START(entry->start)); - else - intel_de_write_fw(dev_priv, reg, 0); -} - -static void skl_write_wm_level(struct drm_i915_private *dev_priv, - i915_reg_t reg, - const struct skl_wm_level *level) -{ - u32 val = 0; - - if (level->enable) - val |= PLANE_WM_EN; - if (level->ignore_lines) - val |= PLANE_WM_IGNORE_LINES; - val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks); - val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines); - - intel_de_write_fw(dev_priv, reg, val); -} - -void skl_write_plane_wm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - int level, max_level = ilk_wm_max_level(dev_priv); - enum plane_id plane_id = plane->id; - enum pipe pipe = plane->pipe; - const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; - const struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - const struct skl_ddb_entry *ddb_y = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - - for (level = 0; level <= max_level; level++) - skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level), - skl_plane_wm_level(pipe_wm, plane_id, level)); - - skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id), - skl_plane_trans_wm(pipe_wm, plane_id)); - - if (HAS_HW_SAGV_WM(dev_priv)) { - const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; - - skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id), - &wm->sagv.wm0); - skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id), - &wm->sagv.trans_wm); - } - - skl_ddb_entry_write(dev_priv, - PLANE_BUF_CFG(pipe, plane_id), ddb); - - if (DISPLAY_VER(dev_priv) < 11) - skl_ddb_entry_write(dev_priv, - PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y); -} - -void skl_write_cursor_wm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); - int level, max_level = ilk_wm_max_level(dev_priv); - enum plane_id plane_id = plane->id; - enum pipe pipe = plane->pipe; - const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal; - const struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - - for (level = 0; level <= max_level; level++) - skl_write_wm_level(dev_priv, CUR_WM(pipe, level), - skl_plane_wm_level(pipe_wm, plane_id, level)); - - skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), - skl_plane_trans_wm(pipe_wm, plane_id)); - - if (HAS_HW_SAGV_WM(dev_priv)) { - const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id]; - - skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe), - &wm->sagv.wm0); - skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe), - &wm->sagv.trans_wm); - } - - skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb); -} - -static bool skl_wm_level_equals(const struct skl_wm_level *l1, - const struct skl_wm_level *l2) -{ - return l1->enable == l2->enable && - l1->ignore_lines == l2->ignore_lines && - l1->lines == l2->lines && - l1->blocks == l2->blocks; -} - -static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv, - const struct skl_plane_wm *wm1, - const struct skl_plane_wm *wm2) -{ - int level, max_level = ilk_wm_max_level(dev_priv); - - for (level = 0; level <= max_level; level++) { - /* - * We don't check uv_wm as the hardware doesn't actually - * use it. It only gets used for calculating the required - * ddb allocation. - */ - if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level])) - return false; - } - - return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) && - skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) && - skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm); -} - -static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, - const struct skl_ddb_entry *b) -{ - return a->start < b->end && b->start < a->end; -} - -static void skl_ddb_entry_union(struct skl_ddb_entry *a, - const struct skl_ddb_entry *b) -{ - if (a->end && b->end) { - a->start = min(a->start, b->start); - a->end = max(a->end, b->end); - } else if (b->end) { - a->start = b->start; - a->end = b->end; - } -} - -bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, - const struct skl_ddb_entry *entries, - int num_entries, int ignore_idx) -{ - int i; - - for (i = 0; i < num_entries; i++) { - if (i != ignore_idx && - skl_ddb_entries_overlap(ddb, &entries[i])) - return true; - } - - return false; -} - -static int -skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) -{ - struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state); - struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_plane *plane; - - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { - struct intel_plane_state *plane_state; - enum plane_id plane_id = plane->id; - - if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id], - &new_crtc_state->wm.skl.plane_ddb[plane_id]) && - skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id], - &new_crtc_state->wm.skl.plane_ddb_y[plane_id])) - continue; - - plane_state = intel_atomic_get_plane_state(state, plane); - if (IS_ERR(plane_state)) - return PTR_ERR(plane_state); - - new_crtc_state->update_planes |= BIT(plane_id); - } - - return 0; -} - -static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state) -{ - struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev); - u8 enabled_slices; - enum pipe pipe; - - /* - * FIXME: For now we always enable slice S1 as per - * the Bspec display initialization sequence. - */ - enabled_slices = BIT(DBUF_S1); - - for_each_pipe(dev_priv, pipe) - enabled_slices |= dbuf_state->slices[pipe]; - - return enabled_slices; -} - -static int -skl_compute_ddb(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dbuf_state *old_dbuf_state; - struct intel_dbuf_state *new_dbuf_state = NULL; - const struct intel_crtc_state *old_crtc_state; - struct intel_crtc_state *new_crtc_state; - struct intel_crtc *crtc; - int ret, i; - - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - new_dbuf_state = intel_atomic_get_dbuf_state(state); - if (IS_ERR(new_dbuf_state)) - return PTR_ERR(new_dbuf_state); - - old_dbuf_state = intel_atomic_get_old_dbuf_state(state); - break; - } - - if (!new_dbuf_state) - return 0; - - new_dbuf_state->active_pipes = - intel_calc_active_pipes(state, old_dbuf_state->active_pipes); - - if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) { - ret = intel_atomic_lock_global_state(&new_dbuf_state->base); - if (ret) - return ret; - } - - if (HAS_MBUS_JOINING(dev_priv)) - new_dbuf_state->joined_mbus = - adlp_check_mbus_joined(new_dbuf_state->active_pipes); - - for_each_intel_crtc(&dev_priv->drm, crtc) { - enum pipe pipe = crtc->pipe; - - new_dbuf_state->slices[pipe] = - skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes, - new_dbuf_state->joined_mbus); - - if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe]) - continue; - - ret = intel_atomic_lock_global_state(&new_dbuf_state->base); - if (ret) - return ret; - } - - new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state); - - if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices || - old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { - ret = intel_atomic_serialize_global_state(&new_dbuf_state->base); - if (ret) - return ret; - - if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) { - /* TODO: Implement vblank synchronized MBUS joining changes */ - ret = intel_modeset_all_pipes(state); - if (ret) - return ret; - } - - drm_dbg_kms(&dev_priv->drm, - "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n", - old_dbuf_state->enabled_slices, - new_dbuf_state->enabled_slices, - INTEL_INFO(dev_priv)->display.dbuf.slice_mask, - str_yes_no(old_dbuf_state->joined_mbus), - str_yes_no(new_dbuf_state->joined_mbus)); - } - - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - enum pipe pipe = crtc->pipe; - - new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state); - - if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe]) - continue; - - ret = intel_atomic_lock_global_state(&new_dbuf_state->base); - if (ret) - return ret; - } - - for_each_intel_crtc(&dev_priv->drm, crtc) { - ret = skl_crtc_allocate_ddb(state, crtc); - if (ret) - return ret; - } - - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - ret = skl_crtc_allocate_plane_ddb(state, crtc); - if (ret) - return ret; - - ret = skl_ddb_add_affected_planes(old_crtc_state, - new_crtc_state); - if (ret) - return ret; - } - - return 0; -} - -static char enast(bool enable) -{ - return enable ? '*' : ' '; -} - -static void -skl_print_wm_changes(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_crtc_state *old_crtc_state; - const struct intel_crtc_state *new_crtc_state; - struct intel_plane *plane; - struct intel_crtc *crtc; - int i; - - if (!drm_debug_enabled(DRM_UT_KMS)) - return; - - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { - const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm; - - old_pipe_wm = &old_crtc_state->wm.skl.optimal; - new_pipe_wm = &new_crtc_state->wm.skl.optimal; - - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { - enum plane_id plane_id = plane->id; - const struct skl_ddb_entry *old, *new; - - old = &old_crtc_state->wm.skl.plane_ddb[plane_id]; - new = &new_crtc_state->wm.skl.plane_ddb[plane_id]; - - if (skl_ddb_entry_equal(old, new)) - continue; - - drm_dbg_kms(&dev_priv->drm, - "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n", - plane->base.base.id, plane->base.name, - old->start, old->end, new->start, new->end, - skl_ddb_entry_size(old), skl_ddb_entry_size(new)); - } - - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { - enum plane_id plane_id = plane->id; - const struct skl_plane_wm *old_wm, *new_wm; - - old_wm = &old_pipe_wm->planes[plane_id]; - new_wm = &new_pipe_wm->planes[plane_id]; - - if (skl_plane_wm_equals(dev_priv, old_wm, new_wm)) - continue; - - drm_dbg_kms(&dev_priv->drm, - "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm" - " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n", - plane->base.base.id, plane->base.name, - enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable), - enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable), - enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable), - enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable), - enast(old_wm->trans_wm.enable), - enast(old_wm->sagv.wm0.enable), - enast(old_wm->sagv.trans_wm.enable), - enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable), - enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable), - enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable), - enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable), - enast(new_wm->trans_wm.enable), - enast(new_wm->sagv.wm0.enable), - enast(new_wm->sagv.trans_wm.enable)); - - drm_dbg_kms(&dev_priv->drm, - "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d" - " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n", - plane->base.base.id, plane->base.name, - enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines, - enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines, - enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines, - enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines, - enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines, - enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines, - enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines, - enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines, - enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines, - enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines, - enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines, - enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines, - enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines, - enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines, - enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines, - enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines, - enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines, - enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines, - enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines, - enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines, - enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines, - enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines); - - drm_dbg_kms(&dev_priv->drm, - "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" - " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", - plane->base.base.id, plane->base.name, - old_wm->wm[0].blocks, old_wm->wm[1].blocks, - old_wm->wm[2].blocks, old_wm->wm[3].blocks, - old_wm->wm[4].blocks, old_wm->wm[5].blocks, - old_wm->wm[6].blocks, old_wm->wm[7].blocks, - old_wm->trans_wm.blocks, - old_wm->sagv.wm0.blocks, - old_wm->sagv.trans_wm.blocks, - new_wm->wm[0].blocks, new_wm->wm[1].blocks, - new_wm->wm[2].blocks, new_wm->wm[3].blocks, - new_wm->wm[4].blocks, new_wm->wm[5].blocks, - new_wm->wm[6].blocks, new_wm->wm[7].blocks, - new_wm->trans_wm.blocks, - new_wm->sagv.wm0.blocks, - new_wm->sagv.trans_wm.blocks); - - drm_dbg_kms(&dev_priv->drm, - "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d" - " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n", - plane->base.base.id, plane->base.name, - old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc, - old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc, - old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc, - old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc, - old_wm->trans_wm.min_ddb_alloc, - old_wm->sagv.wm0.min_ddb_alloc, - old_wm->sagv.trans_wm.min_ddb_alloc, - new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc, - new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc, - new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc, - new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc, - new_wm->trans_wm.min_ddb_alloc, - new_wm->sagv.wm0.min_ddb_alloc, - new_wm->sagv.trans_wm.min_ddb_alloc); - } - } -} - -static bool skl_plane_selected_wm_equals(struct intel_plane *plane, - const struct skl_pipe_wm *old_pipe_wm, - const struct skl_pipe_wm *new_pipe_wm) -{ - struct drm_i915_private *i915 = to_i915(plane->base.dev); - int level, max_level = ilk_wm_max_level(i915); - - for (level = 0; level <= max_level; level++) { - /* - * We don't check uv_wm as the hardware doesn't actually - * use it. It only gets used for calculating the required - * ddb allocation. - */ - if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level), - skl_plane_wm_level(new_pipe_wm, plane->id, level))) - return false; - } - - if (HAS_HW_SAGV_WM(i915)) { - const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id]; - const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id]; - - if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) || - !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm)) - return false; - } - - return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id), - skl_plane_trans_wm(new_pipe_wm, plane->id)); -} - -/* - * To make sure the cursor watermark registers are always consistent - * with our computed state the following scenario needs special - * treatment: - * - * 1. enable cursor - * 2. move cursor entirely offscreen - * 3. disable cursor - * - * Step 2. does call .disable_plane() but does not zero the watermarks - * (since we consider an offscreen cursor still active for the purposes - * of watermarks). Step 3. would not normally call .disable_plane() - * because the actual plane visibility isn't changing, and we don't - * deallocate the cursor ddb until the pipe gets disabled. So we must - * force step 3. to call .disable_plane() to update the watermark - * registers properly. - * - * Other planes do not suffer from this issues as their watermarks are - * calculated based on the actual plane visibility. The only time this - * can trigger for the other planes is during the initial readout as the - * default value of the watermarks registers is not zero. - */ -static int skl_wm_add_affected_planes(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - const struct intel_crtc_state *old_crtc_state = - intel_atomic_get_old_crtc_state(state, crtc); - struct intel_crtc_state *new_crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - struct intel_plane *plane; - - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { - struct intel_plane_state *plane_state; - enum plane_id plane_id = plane->id; - - /* - * Force a full wm update for every plane on modeset. - * Required because the reset value of the wm registers - * is non-zero, whereas we want all disabled planes to - * have zero watermarks. So if we turn off the relevant - * power well the hardware state will go out of sync - * with the software state. - */ - if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) && - skl_plane_selected_wm_equals(plane, - &old_crtc_state->wm.skl.optimal, - &new_crtc_state->wm.skl.optimal)) - continue; - - plane_state = intel_atomic_get_plane_state(state, plane); - if (IS_ERR(plane_state)) - return PTR_ERR(plane_state); - - new_crtc_state->update_planes |= BIT(plane_id); - } - - return 0; -} - -static int -skl_compute_wm(struct intel_atomic_state *state) -{ - struct intel_crtc *crtc; - struct intel_crtc_state *new_crtc_state; - int ret, i; - - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - ret = skl_build_pipe_wm(state, crtc); - if (ret) - return ret; - } - - ret = skl_compute_ddb(state); - if (ret) - return ret; - - ret = intel_compute_sagv_mask(state); - if (ret) - return ret; - - /* - * skl_compute_ddb() will have adjusted the final watermarks - * based on how much ddb is available. Now we can actually - * check if the final watermarks changed. - */ - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - ret = skl_wm_add_affected_planes(state, crtc); - if (ret) - return ret; - } - - skl_print_wm_changes(state); - - return 0; -} - -static void ilk_compute_wm_config(struct drm_i915_private *dev_priv, - struct intel_wm_config *config) -{ - struct intel_crtc *crtc; - - /* Compute the currently _active_ config */ - for_each_intel_crtc(&dev_priv->drm, crtc) { - const struct intel_pipe_wm *wm = &crtc->wm.active.ilk; - - if (!wm->pipe_enabled) - continue; - - config->sprites_enabled |= wm->sprites_enabled; - config->sprites_scaled |= wm->sprites_scaled; - config->num_pipes_active++; - } -} - -static void ilk_program_watermarks(struct drm_i915_private *dev_priv) -{ - struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm; - struct ilk_wm_maximums max; - struct intel_wm_config config = {}; - struct ilk_wm_values results = {}; - enum intel_ddb_partitioning partitioning; - - ilk_compute_wm_config(dev_priv, &config); - - ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max); - ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2); - - /* 5/6 split only in single pipe config on IVB+ */ - if (DISPLAY_VER(dev_priv) >= 7 && - config.num_pipes_active == 1 && config.sprites_enabled) { - ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max); - ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6); - - best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6); - } else { - best_lp_wm = &lp_wm_1_2; - } - - partitioning = (best_lp_wm == &lp_wm_1_2) ? - INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6; - - ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results); - - ilk_write_wm_values(dev_priv, &results); -} - -static void ilk_initial_watermarks(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - const struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - mutex_lock(&dev_priv->wm.wm_mutex); - crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate; - ilk_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); -} - -static void ilk_optimize_watermarks(struct intel_atomic_state *state, - struct intel_crtc *crtc) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - const struct intel_crtc_state *crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - - if (!crtc_state->wm.need_postvbl_update) - return; - - mutex_lock(&dev_priv->wm.wm_mutex); - crtc->wm.active.ilk = crtc_state->wm.ilk.optimal; - ilk_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); -} - -static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level) -{ - level->enable = val & PLANE_WM_EN; - level->ignore_lines = val & PLANE_WM_IGNORE_LINES; - level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val); - level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val); -} - -static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, - struct skl_pipe_wm *out) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - int level, max_level; - enum plane_id plane_id; - u32 val; - - max_level = ilk_wm_max_level(dev_priv); - - for_each_plane_id_on_crtc(crtc, plane_id) { - struct skl_plane_wm *wm = &out->planes[plane_id]; - - for (level = 0; level <= max_level; level++) { - if (plane_id != PLANE_CURSOR) - val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level)); - else - val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level)); - - skl_wm_level_from_reg_val(val, &wm->wm[level]); - } - - if (plane_id != PLANE_CURSOR) - val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id)); - else - val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe)); - - skl_wm_level_from_reg_val(val, &wm->trans_wm); - - if (HAS_HW_SAGV_WM(dev_priv)) { - if (plane_id != PLANE_CURSOR) - val = intel_uncore_read(&dev_priv->uncore, - PLANE_WM_SAGV(pipe, plane_id)); - else - val = intel_uncore_read(&dev_priv->uncore, - CUR_WM_SAGV(pipe)); - - skl_wm_level_from_reg_val(val, &wm->sagv.wm0); - - if (plane_id != PLANE_CURSOR) - val = intel_uncore_read(&dev_priv->uncore, - PLANE_WM_SAGV_TRANS(pipe, plane_id)); - else - val = intel_uncore_read(&dev_priv->uncore, - CUR_WM_SAGV_TRANS(pipe)); - - skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm); - } else if (DISPLAY_VER(dev_priv) >= 12) { - wm->sagv.wm0 = wm->wm[0]; - wm->sagv.trans_wm = wm->trans_wm; - } - } -} - -void skl_wm_get_hw_state(struct drm_i915_private *dev_priv) -{ - struct intel_dbuf_state *dbuf_state = - to_intel_dbuf_state(dev_priv->dbuf.obj.state); - struct intel_crtc *crtc; - - if (HAS_MBUS_JOINING(dev_priv)) - dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN; - - for_each_intel_crtc(&dev_priv->drm, crtc) { - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - enum pipe pipe = crtc->pipe; - unsigned int mbus_offset; - enum plane_id plane_id; - u8 slices; - - skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal); - crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal; - - memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe])); - - for_each_plane_id_on_crtc(crtc, plane_id) { - struct skl_ddb_entry *ddb = - &crtc_state->wm.skl.plane_ddb[plane_id]; - struct skl_ddb_entry *ddb_y = - &crtc_state->wm.skl.plane_ddb_y[plane_id]; - - skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe, - plane_id, ddb, ddb_y); - - skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb); - skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y); - } - - dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state); - - /* - * Used for checking overlaps, so we need absolute - * offsets instead of MBUS relative offsets. - */ - slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, - dbuf_state->joined_mbus); - mbus_offset = mbus_ddb_offset(dev_priv, slices); - crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start; - crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end; - - /* The slices actually used by the planes on the pipe */ - dbuf_state->slices[pipe] = - skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb); - - drm_dbg_kms(&dev_priv->drm, - "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n", - crtc->base.base.id, crtc->base.name, - dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start, - dbuf_state->ddb[pipe].end, dbuf_state->active_pipes, - str_yes_no(dbuf_state->joined_mbus)); - } - - dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices; -} - -static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915) -{ - const struct intel_dbuf_state *dbuf_state = - to_intel_dbuf_state(i915->dbuf.obj.state); - struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; - struct intel_crtc *crtc; - - for_each_intel_crtc(&i915->drm, crtc) { - const struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - - entries[crtc->pipe] = crtc_state->wm.skl.ddb; - } - - for_each_intel_crtc(&i915->drm, crtc) { - const struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - u8 slices; - - slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes, - dbuf_state->joined_mbus); - if (dbuf_state->slices[crtc->pipe] & ~slices) - return true; - - if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries, - I915_MAX_PIPES, crtc->pipe)) - return true; - } - - return false; -} - -void skl_wm_sanitize(struct drm_i915_private *i915) -{ - struct intel_crtc *crtc; - - /* - * On TGL/RKL (at least) the BIOS likes to assign the planes - * to the wrong DBUF slices. This will cause an infinite loop - * in skl_commit_modeset_enables() as it can't find a way to - * transition between the old bogus DBUF layout to the new - * proper DBUF layout without DBUF allocation overlaps between - * the planes (which cannot be allowed or else the hardware - * may hang). If we detect a bogus DBUF layout just turn off - * all the planes so that skl_commit_modeset_enables() can - * simply ignore them. - */ - if (!skl_dbuf_is_misconfigured(i915)) - return; - - drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n"); - - for_each_intel_crtc(&i915->drm, crtc) { - struct intel_plane *plane = to_intel_plane(crtc->base.primary); - const struct intel_plane_state *plane_state = - to_intel_plane_state(plane->base.state); - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - - if (plane_state->uapi.visible) - intel_plane_disable_noatomic(crtc, plane); - - drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0); - - memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb)); - } -} - -static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc) -{ - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct ilk_wm_values *hw = &dev_priv->wm.hw; - struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state); - struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal; - enum pipe pipe = crtc->pipe; - - hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe)); - - memset(active, 0, sizeof(*active)); - - active->pipe_enabled = crtc->active; - - if (active->pipe_enabled) { - u32 tmp = hw->wm_pipe[pipe]; - - /* - * For active pipes LP0 watermark is marked as - * enabled, and LP1+ watermaks as disabled since - * we can't really reverse compute them in case - * multiple pipes are active. - */ - active->wm[0].enable = true; - active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp); - active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp); - active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp); - } else { - int level, max_level = ilk_wm_max_level(dev_priv); - - /* - * For inactive pipes, all watermark levels - * should be marked as enabled but zeroed, - * which is what we'd compute them to. - */ - for (level = 0; level <= max_level; level++) - active->wm[level].enable = true; - } - - crtc->wm.active.ilk = *active; -} - -#define _FW_WM(value, plane) \ - (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT) -#define _FW_WM_VLV(value, plane) \ - (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT) - -static void g4x_read_wm_values(struct drm_i915_private *dev_priv, - struct g4x_wm_values *wm) -{ - u32 tmp; - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); - wm->sr.plane = _FW_WM(tmp, SR); - wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); - wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB); - wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); - wm->fbc_en = tmp & DSPFW_FBC_SR_EN; - wm->sr.fbc = _FW_WM(tmp, FBC_SR); - wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR); - wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB); - wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); - wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); - wm->hpll_en = tmp & DSPFW_HPLL_SR_EN; - wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); - wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR); - wm->hpll.plane = _FW_WM(tmp, HPLL_SR); -} - -static void vlv_read_wm_values(struct drm_i915_private *dev_priv, - struct vlv_wm_values *wm) -{ - enum pipe pipe; - u32 tmp; - - for_each_pipe(dev_priv, pipe) { - tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe)); - - wm->ddl[pipe].plane[PLANE_PRIMARY] = - (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); - wm->ddl[pipe].plane[PLANE_CURSOR] = - (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); - wm->ddl[pipe].plane[PLANE_SPRITE0] = - (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); - wm->ddl[pipe].plane[PLANE_SPRITE1] = - (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK); - } - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1); - wm->sr.plane = _FW_WM(tmp, SR); - wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB); - wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB); - wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2); - wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB); - wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA); - wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3); - wm->sr.cursor = _FW_WM(tmp, CURSOR_SR); - - if (IS_CHERRYVIEW(dev_priv)) { - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV); - wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); - wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV); - wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF); - wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV); - wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC); - wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); - wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; - wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8; - wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8; - wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8; - wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; - wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; - wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; - } else { - tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7); - wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED); - wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC); - - tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM); - wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9; - wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8; - wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8; - wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8; - wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8; - } -} - -#undef _FW_WM -#undef _FW_WM_VLV - -void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv) -{ - struct g4x_wm_values *wm = &dev_priv->wm.g4x; - struct intel_crtc *crtc; - - g4x_read_wm_values(dev_priv, wm); - - wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; + wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN; for_each_intel_crtc(&dev_priv->drm, crtc) { struct intel_crtc_state *crtc_state = @@ -6919,7 +3851,7 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv) struct intel_plane *plane; struct intel_crtc *crtc; - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); for_each_intel_plane(&dev_priv->drm, plane) { struct intel_crtc *crtc = @@ -6967,12 +3899,12 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv) g4x_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct vlv_wm_values *wm = &dev_priv->wm.vlv; + struct vlv_wm_values *wm = &dev_priv->display.wm.vlv; struct intel_crtc *crtc; u32 val; @@ -7006,7 +3938,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv) drm_dbg_kms(&dev_priv->drm, "Punit not acking DDR DVFS request, " "assuming DDR DVFS is disabled\n"); - dev_priv->wm.max_level = VLV_WM_LEVEL_PM5; + dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5; } else { val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2); if ((val & FORCE_DDR_HIGH_FREQ) == 0) @@ -7075,7 +4007,7 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv) struct intel_plane *plane; struct intel_crtc *crtc; - mutex_lock(&dev_priv->wm.wm_mutex); + mutex_lock(&dev_priv->display.wm.wm_mutex); for_each_intel_plane(&dev_priv->drm, plane) { struct intel_crtc *crtc = @@ -7116,7 +4048,7 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv) vlv_program_watermarks(dev_priv); - mutex_unlock(&dev_priv->wm.wm_mutex); + mutex_unlock(&dev_priv->display.wm.wm_mutex); } /* @@ -7137,7 +4069,7 @@ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) { - struct ilk_wm_values *hw = &dev_priv->wm.hw; + struct ilk_wm_values *hw = &dev_priv->display.wm.hw; struct intel_crtc *crtc; ilk_init_lp_watermarks(dev_priv); @@ -7166,168 +4098,6 @@ void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv) !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS); } -void intel_wm_state_verify(struct intel_crtc *crtc, - struct intel_crtc_state *new_crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct skl_hw_state { - struct skl_ddb_entry ddb[I915_MAX_PLANES]; - struct skl_ddb_entry ddb_y[I915_MAX_PLANES]; - struct skl_pipe_wm wm; - } *hw; - const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal; - int level, max_level = ilk_wm_max_level(dev_priv); - struct intel_plane *plane; - u8 hw_enabled_slices; - - if (DISPLAY_VER(dev_priv) < 9 || !new_crtc_state->hw.active) - return; - - hw = kzalloc(sizeof(*hw), GFP_KERNEL); - if (!hw) - return; - - skl_pipe_wm_get_hw_state(crtc, &hw->wm); - - skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y); - - hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv); - - if (DISPLAY_VER(dev_priv) >= 11 && - hw_enabled_slices != dev_priv->dbuf.enabled_slices) - drm_err(&dev_priv->drm, - "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n", - dev_priv->dbuf.enabled_slices, - hw_enabled_slices); - - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { - const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry; - const struct skl_wm_level *hw_wm_level, *sw_wm_level; - - /* Watermarks */ - for (level = 0; level <= max_level; level++) { - hw_wm_level = &hw->wm.planes[plane->id].wm[level]; - sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level); - - if (skl_wm_level_equals(hw_wm_level, sw_wm_level)) - continue; - - drm_err(&dev_priv->drm, - "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", - plane->base.base.id, plane->base.name, level, - sw_wm_level->enable, - sw_wm_level->blocks, - sw_wm_level->lines, - hw_wm_level->enable, - hw_wm_level->blocks, - hw_wm_level->lines); - } - - hw_wm_level = &hw->wm.planes[plane->id].trans_wm; - sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id); - - if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) { - drm_err(&dev_priv->drm, - "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", - plane->base.base.id, plane->base.name, - sw_wm_level->enable, - sw_wm_level->blocks, - sw_wm_level->lines, - hw_wm_level->enable, - hw_wm_level->blocks, - hw_wm_level->lines); - } - - hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0; - sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0; - - if (HAS_HW_SAGV_WM(dev_priv) && - !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { - drm_err(&dev_priv->drm, - "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", - plane->base.base.id, plane->base.name, - sw_wm_level->enable, - sw_wm_level->blocks, - sw_wm_level->lines, - hw_wm_level->enable, - hw_wm_level->blocks, - hw_wm_level->lines); - } - - hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm; - sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm; - - if (HAS_HW_SAGV_WM(dev_priv) && - !skl_wm_level_equals(hw_wm_level, sw_wm_level)) { - drm_err(&dev_priv->drm, - "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n", - plane->base.base.id, plane->base.name, - sw_wm_level->enable, - sw_wm_level->blocks, - sw_wm_level->lines, - hw_wm_level->enable, - hw_wm_level->blocks, - hw_wm_level->lines); - } - - /* DDB */ - hw_ddb_entry = &hw->ddb[PLANE_CURSOR]; - sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR]; - - if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { - drm_err(&dev_priv->drm, - "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n", - plane->base.base.id, plane->base.name, - sw_ddb_entry->start, sw_ddb_entry->end, - hw_ddb_entry->start, hw_ddb_entry->end); - } - } - - kfree(hw); -} - -void intel_enable_ipc(struct drm_i915_private *dev_priv) -{ - u32 val; - - if (!HAS_IPC(dev_priv)) - return; - - val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2); - - if (dev_priv->ipc_enabled) - val |= DISP_IPC_ENABLE; - else - val &= ~DISP_IPC_ENABLE; - - intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val); -} - -static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv) -{ - /* Display WA #0477 WaDisableIPC: skl */ - if (IS_SKYLAKE(dev_priv)) - return false; - - /* Display WA #1141: SKL:all KBL:all CFL */ - if (IS_KABYLAKE(dev_priv) || - IS_COFFEELAKE(dev_priv) || - IS_COMETLAKE(dev_priv)) - return dev_priv->dram_info.symmetric_memory; - - return true; -} - -void intel_init_ipc(struct drm_i915_private *dev_priv) -{ - if (!HAS_IPC(dev_priv)) - return; - - dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv); - - intel_enable_ipc(dev_priv); -} - static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) { /* @@ -7435,7 +4205,7 @@ static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe)); val |= TRANS_CHICKEN2_TIMING_OVERRIDE; val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED; - if (dev_priv->vbt.fdi_rx_polarity_inverted) + if (dev_priv->display.vbt.fdi_rx_polarity_inverted) val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED; val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER; val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH; @@ -7586,9 +4356,8 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv) { - /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */ - if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) || - IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv)) + /* Wa_1409120013 */ + if (DISPLAY_VER(dev_priv) == 12) intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), DPFC_CHICKEN_COMP_DUMMY_PIXEL); @@ -7965,7 +4734,7 @@ static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) OVCUNIT_CLOCK_GATE_DISABLE; if (IS_GM45(dev_priv)) dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE; - intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate); + intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate); g4x_disable_trickle_feed(dev_priv); } @@ -7976,7 +4745,7 @@ static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv) intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); intel_uncore_write(uncore, RENCLK_GATE_D2, 0); - intel_uncore_write(uncore, DSPCLK_GATE_D, 0); + intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0); intel_uncore_write(uncore, RAMCLK_GATE_D, 0); intel_uncore_write16(uncore, DEUC, 0); intel_uncore_write(uncore, @@ -8168,18 +4937,14 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) } } -static const struct drm_i915_wm_disp_funcs skl_wm_funcs = { - .compute_global_watermarks = skl_compute_wm, -}; - -static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = { +static const struct intel_wm_funcs ilk_wm_funcs = { .compute_pipe_wm = ilk_compute_pipe_wm, .compute_intermediate_wm = ilk_compute_intermediate_wm, .initial_watermarks = ilk_initial_watermarks, .optimize_watermarks = ilk_optimize_watermarks, }; -static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = { +static const struct intel_wm_funcs vlv_wm_funcs = { .compute_pipe_wm = vlv_compute_pipe_wm, .compute_intermediate_wm = vlv_compute_intermediate_wm, .initial_watermarks = vlv_initial_watermarks, @@ -8187,67 +4952,67 @@ static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = { .atomic_update_watermarks = vlv_atomic_update_fifo, }; -static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = { +static const struct intel_wm_funcs g4x_wm_funcs = { .compute_pipe_wm = g4x_compute_pipe_wm, .compute_intermediate_wm = g4x_compute_intermediate_wm, .initial_watermarks = g4x_initial_watermarks, .optimize_watermarks = g4x_optimize_watermarks, }; -static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = { +static const struct intel_wm_funcs pnv_wm_funcs = { .update_wm = pnv_update_wm, }; -static const struct drm_i915_wm_disp_funcs i965_wm_funcs = { +static const struct intel_wm_funcs i965_wm_funcs = { .update_wm = i965_update_wm, }; -static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = { +static const struct intel_wm_funcs i9xx_wm_funcs = { .update_wm = i9xx_update_wm, }; -static const struct drm_i915_wm_disp_funcs i845_wm_funcs = { +static const struct intel_wm_funcs i845_wm_funcs = { .update_wm = i845_update_wm, }; -static const struct drm_i915_wm_disp_funcs nop_funcs = { +static const struct intel_wm_funcs nop_funcs = { }; /* Set up chip specific power management-related functions */ void intel_init_pm(struct drm_i915_private *dev_priv) { + if (DISPLAY_VER(dev_priv) >= 9) { + skl_wm_init(dev_priv); + return; + } + /* For cxsr */ if (IS_PINEVIEW(dev_priv)) pnv_get_mem_freq(dev_priv); else if (GRAPHICS_VER(dev_priv) == 5) ilk_get_mem_freq(dev_priv); - intel_sagv_init(dev_priv); - /* For FIFO watermark updates */ - if (DISPLAY_VER(dev_priv) >= 9) { - skl_setup_wm_latency(dev_priv); - dev_priv->wm_disp = &skl_wm_funcs; - } else if (HAS_PCH_SPLIT(dev_priv)) { + if (HAS_PCH_SPLIT(dev_priv)) { ilk_setup_wm_latency(dev_priv); - if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] && - dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || - (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] && - dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) { - dev_priv->wm_disp = &ilk_wm_funcs; + if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] && + dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) || + (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] && + dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) { + dev_priv->display.funcs.wm = &ilk_wm_funcs; } else { drm_dbg_kms(&dev_priv->drm, "Failed to read display plane latency. " "Disable CxSR\n"); - dev_priv->wm_disp = &nop_funcs; + dev_priv->display.funcs.wm = &nop_funcs; } } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { vlv_setup_wm_latency(dev_priv); - dev_priv->wm_disp = &vlv_wm_funcs; + dev_priv->display.funcs.wm = &vlv_wm_funcs; } else if (IS_G4X(dev_priv)) { g4x_setup_wm_latency(dev_priv); - dev_priv->wm_disp = &g4x_wm_funcs; + dev_priv->display.funcs.wm = &g4x_wm_funcs; } else if (IS_PINEVIEW(dev_priv)) { if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv), dev_priv->is_ddr3, @@ -8261,22 +5026,22 @@ void intel_init_pm(struct drm_i915_private *dev_priv) dev_priv->fsb_freq, dev_priv->mem_freq); /* Disable CxSR and never update its watermark again */ intel_set_memory_cxsr(dev_priv, false); - dev_priv->wm_disp = &nop_funcs; + dev_priv->display.funcs.wm = &nop_funcs; } else - dev_priv->wm_disp = &pnv_wm_funcs; + dev_priv->display.funcs.wm = &pnv_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 4) { - dev_priv->wm_disp = &i965_wm_funcs; + dev_priv->display.funcs.wm = &i965_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 3) { - dev_priv->wm_disp = &i9xx_wm_funcs; + dev_priv->display.funcs.wm = &i9xx_wm_funcs; } else if (DISPLAY_VER(dev_priv) == 2) { if (INTEL_NUM_PIPES(dev_priv) == 1) - dev_priv->wm_disp = &i845_wm_funcs; + dev_priv->display.funcs.wm = &i845_wm_funcs; else - dev_priv->wm_disp = &i9xx_wm_funcs; + dev_priv->display.funcs.wm = &i9xx_wm_funcs; } else { drm_err(&dev_priv->drm, "unexpected fall-through in %s\n", __func__); - dev_priv->wm_disp = &nop_funcs; + dev_priv->display.funcs.wm = &nop_funcs; } } @@ -8285,183 +5050,3 @@ void intel_pm_setup(struct drm_i915_private *dev_priv) dev_priv->runtime_pm.suspended = false; atomic_set(&dev_priv->runtime_pm.wakeref_count, 0); } - -static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj) -{ - struct intel_dbuf_state *dbuf_state; - - dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL); - if (!dbuf_state) - return NULL; - - return &dbuf_state->base; -} - -static void intel_dbuf_destroy_state(struct intel_global_obj *obj, - struct intel_global_state *state) -{ - kfree(state); -} - -static const struct intel_global_state_funcs intel_dbuf_funcs = { - .atomic_duplicate_state = intel_dbuf_duplicate_state, - .atomic_destroy_state = intel_dbuf_destroy_state, -}; - -struct intel_dbuf_state * -intel_atomic_get_dbuf_state(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_global_state *dbuf_state; - - dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj); - if (IS_ERR(dbuf_state)) - return ERR_CAST(dbuf_state); - - return to_intel_dbuf_state(dbuf_state); -} - -int intel_dbuf_init(struct drm_i915_private *dev_priv) -{ - struct intel_dbuf_state *dbuf_state; - - dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL); - if (!dbuf_state) - return -ENOMEM; - - intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj, - &dbuf_state->base, &intel_dbuf_funcs); - - return 0; -} - -/* - * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before - * update the request state of all DBUS slices. - */ -static void update_mbus_pre_enable(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - u32 mbus_ctl, dbuf_min_tracker_val; - enum dbuf_slice slice; - const struct intel_dbuf_state *dbuf_state = - intel_atomic_get_new_dbuf_state(state); - - if (!HAS_MBUS_JOINING(dev_priv)) - return; - - /* - * TODO: Implement vblank synchronized MBUS joining changes. - * Must be properly coordinated with dbuf reprogramming. - */ - if (dbuf_state->joined_mbus) { - mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN | - MBUS_JOIN_PIPE_SELECT_NONE; - dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3); - } else { - mbus_ctl = MBUS_HASHING_MODE_2x2 | - MBUS_JOIN_PIPE_SELECT_NONE; - dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1); - } - - intel_de_rmw(dev_priv, MBUS_CTL, - MBUS_HASHING_MODE_MASK | MBUS_JOIN | - MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl); - - for_each_dbuf_slice(dev_priv, slice) - intel_de_rmw(dev_priv, DBUF_CTL_S(slice), - DBUF_MIN_TRACKER_STATE_SERVICE_MASK, - dbuf_min_tracker_val); -} - -void intel_dbuf_pre_plane_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dbuf_state *new_dbuf_state = - intel_atomic_get_new_dbuf_state(state); - const struct intel_dbuf_state *old_dbuf_state = - intel_atomic_get_old_dbuf_state(state); - - if (!new_dbuf_state || - ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) - && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))) - return; - - WARN_ON(!new_dbuf_state->base.changed); - - update_mbus_pre_enable(state); - gen9_dbuf_slices_update(dev_priv, - old_dbuf_state->enabled_slices | - new_dbuf_state->enabled_slices); -} - -void intel_dbuf_post_plane_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dbuf_state *new_dbuf_state = - intel_atomic_get_new_dbuf_state(state); - const struct intel_dbuf_state *old_dbuf_state = - intel_atomic_get_old_dbuf_state(state); - - if (!new_dbuf_state || - ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices) - && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))) - return; - - WARN_ON(!new_dbuf_state->base.changed); - - gen9_dbuf_slices_update(dev_priv, - new_dbuf_state->enabled_slices); -} - -void intel_mbus_dbox_update(struct intel_atomic_state *state) -{ - struct drm_i915_private *i915 = to_i915(state->base.dev); - const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state; - const struct intel_crtc_state *new_crtc_state; - const struct intel_crtc *crtc; - u32 val = 0; - int i; - - if (DISPLAY_VER(i915) < 11) - return; - - new_dbuf_state = intel_atomic_get_new_dbuf_state(state); - old_dbuf_state = intel_atomic_get_old_dbuf_state(state); - if (!new_dbuf_state || - (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus && - new_dbuf_state->active_pipes == old_dbuf_state->active_pipes)) - return; - - if (DISPLAY_VER(i915) >= 12) { - val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16); - val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1); - val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN; - } - - /* Wa_22010947358:adl-p */ - if (IS_ALDERLAKE_P(i915)) - val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) : - MBUS_DBOX_A_CREDIT(4); - else - val |= MBUS_DBOX_A_CREDIT(2); - - if (IS_ALDERLAKE_P(i915)) { - val |= MBUS_DBOX_BW_CREDIT(2); - val |= MBUS_DBOX_B_CREDIT(8); - } else if (DISPLAY_VER(i915) >= 12) { - val |= MBUS_DBOX_BW_CREDIT(2); - val |= MBUS_DBOX_B_CREDIT(12); - } else { - val |= MBUS_DBOX_BW_CREDIT(1); - val |= MBUS_DBOX_B_CREDIT(8); - } - - for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { - if (!new_crtc_state->hw.active || - !intel_crtc_needs_modeset(new_crtc_state)) - continue; - - intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), val); - } -} diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index 945503ae493eb1975c45efd3b14ad2cbfe56b1f0..c09b872d65c84c36022e3939b6b4876fc3da4d82 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -8,22 +8,9 @@ #include -#include "display/intel_display.h" -#include "display/intel_global_state.h" - -#include "i915_drv.h" - -struct drm_device; struct drm_i915_private; -struct i915_request; -struct intel_atomic_state; -struct intel_bw_state; -struct intel_crtc; struct intel_crtc_state; -struct intel_plane; -struct skl_ddb_entry; -struct skl_pipe_wm; -struct skl_wm_level; +struct intel_plane_state; void intel_init_clock_gating(struct drm_i915_private *dev_priv); void intel_suspend_hw(struct drm_i915_private *dev_priv); @@ -34,56 +21,14 @@ void intel_pm_setup(struct drm_i915_private *dev_priv); void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv); void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv); void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv); -void skl_wm_get_hw_state(struct drm_i915_private *dev_priv); -void intel_wm_state_verify(struct intel_crtc *crtc, - struct intel_crtc_state *new_crtc_state); -u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv); -void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv); -u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv, - const struct skl_ddb_entry *entry); void g4x_wm_sanitize(struct drm_i915_private *dev_priv); void vlv_wm_sanitize(struct drm_i915_private *dev_priv); -void skl_wm_sanitize(struct drm_i915_private *dev_priv); -bool intel_can_enable_sagv(struct drm_i915_private *dev_priv, - const struct intel_bw_state *bw_state); -void intel_sagv_pre_plane_update(struct intel_atomic_state *state); -void intel_sagv_post_plane_update(struct intel_atomic_state *state); -bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, - const struct skl_ddb_entry *entries, - int num_entries, int ignore_idx); -void skl_write_plane_wm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state); -void skl_write_cursor_wm(struct intel_plane *plane, - const struct intel_crtc_state *crtc_state); bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv); -void intel_init_ipc(struct drm_i915_private *dev_priv); -void intel_enable_ipc(struct drm_i915_private *dev_priv); +bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state); +void intel_print_wm_latency(struct drm_i915_private *dev_priv, + const char *name, const u16 wm[]); bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable); -struct intel_dbuf_state { - struct intel_global_state base; - - struct skl_ddb_entry ddb[I915_MAX_PIPES]; - unsigned int weight[I915_MAX_PIPES]; - u8 slices[I915_MAX_PIPES]; - u8 enabled_slices; - u8 active_pipes; - bool joined_mbus; -}; - -struct intel_dbuf_state * -intel_atomic_get_dbuf_state(struct intel_atomic_state *state); - -#define to_intel_dbuf_state(x) container_of((x), struct intel_dbuf_state, base) -#define intel_atomic_get_old_dbuf_state(state) \ - to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->dbuf.obj)) -#define intel_atomic_get_new_dbuf_state(state) \ - to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->dbuf.obj)) - -int intel_dbuf_init(struct drm_i915_private *dev_priv); -void intel_dbuf_pre_plane_update(struct intel_atomic_state *state); -void intel_dbuf_post_plane_update(struct intel_atomic_state *state); -void intel_mbus_dbox_update(struct intel_atomic_state *state); - #endif /* __INTEL_PM_H__ */ diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index c7ef5f2ff2e1593edeb8c2468719aa590ba31af2..5cd423c7b64649f4e19f26d9e78780e70ce85a12 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -21,6 +21,7 @@ * IN THE SOFTWARE. */ +#include #include #include "gt/intel_engine_regs.h" @@ -44,29 +45,47 @@ fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains) } void -intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug) +intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915) { - spin_lock_init(&mmio_debug->lock); - mmio_debug->unclaimed_mmio_check = 1; + spin_lock_init(&i915->mmio_debug.lock); + i915->mmio_debug.unclaimed_mmio_check = 1; + + i915->uncore.debug = &i915->mmio_debug; } -static void mmio_debug_suspend(struct intel_uncore_mmio_debug *mmio_debug) +static void mmio_debug_suspend(struct intel_uncore *uncore) { - lockdep_assert_held(&mmio_debug->lock); + if (!uncore->debug) + return; + + spin_lock(&uncore->debug->lock); /* Save and disable mmio debugging for the user bypass */ - if (!mmio_debug->suspend_count++) { - mmio_debug->saved_mmio_check = mmio_debug->unclaimed_mmio_check; - mmio_debug->unclaimed_mmio_check = 0; + if (!uncore->debug->suspend_count++) { + uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check; + uncore->debug->unclaimed_mmio_check = 0; } + + spin_unlock(&uncore->debug->lock); } -static void mmio_debug_resume(struct intel_uncore_mmio_debug *mmio_debug) +static bool check_for_unclaimed_mmio(struct intel_uncore *uncore); + +static void mmio_debug_resume(struct intel_uncore *uncore) { - lockdep_assert_held(&mmio_debug->lock); + if (!uncore->debug) + return; + + spin_lock(&uncore->debug->lock); + + if (!--uncore->debug->suspend_count) + uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check; - if (!--mmio_debug->suspend_count) - mmio_debug->unclaimed_mmio_check = mmio_debug->saved_mmio_check; + if (check_for_unclaimed_mmio(uncore)) + drm_info(&uncore->i915->drm, + "Invalid mmio detected during user access\n"); + + spin_unlock(&uncore->debug->lock); } static const char * const forcewake_domain_names[] = { @@ -677,9 +696,7 @@ void intel_uncore_forcewake_user_get(struct intel_uncore *uncore) spin_lock_irq(&uncore->lock); if (!uncore->user_forcewake_count++) { intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL); - spin_lock(&uncore->debug->lock); - mmio_debug_suspend(uncore->debug); - spin_unlock(&uncore->debug->lock); + mmio_debug_suspend(uncore); } spin_unlock_irq(&uncore->lock); } @@ -695,14 +712,7 @@ void intel_uncore_forcewake_user_put(struct intel_uncore *uncore) { spin_lock_irq(&uncore->lock); if (!--uncore->user_forcewake_count) { - spin_lock(&uncore->debug->lock); - mmio_debug_resume(uncore->debug); - - if (check_for_unclaimed_mmio(uncore)) - drm_info(&uncore->i915->drm, - "Invalid mmio detected during user access\n"); - spin_unlock(&uncore->debug->lock); - + mmio_debug_resume(uncore); intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL); } spin_unlock_irq(&uncore->lock); @@ -918,6 +928,9 @@ find_fw_domain(struct intel_uncore *uncore, u32 offset) { const struct intel_forcewake_range *entry; + if (IS_GSI_REG(offset)) + offset += uncore->gsi_offset; + entry = BSEARCH(offset, uncore->fw_domains_table, uncore->fw_domains_table_entries, @@ -1133,6 +1146,9 @@ static bool is_shadowed(struct intel_uncore *uncore, u32 offset) if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table)) return false; + if (IS_GSI_REG(offset)) + offset += uncore->gsi_offset; + return BSEARCH(offset, uncore->shadowed_reg_table, uncore->shadowed_reg_table_entries, @@ -1704,7 +1720,7 @@ unclaimed_reg_debug(struct intel_uncore *uncore, const bool read, const bool before) { - if (likely(!uncore->i915->params.mmio_debug)) + if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug) return; /* interrupts are disabled and re-enabled around uncore->lock usage */ @@ -1985,8 +2001,8 @@ static int __fw_domain_init(struct intel_uncore *uncore, d->uncore = uncore; d->wake_count = 0; - d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set); - d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack); + d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset; + d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset; d->id = domain_id; @@ -2070,7 +2086,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore) if (GRAPHICS_VER(i915) >= 11) { /* we'll prune the domains of missing engines later */ - intel_engine_mask_t emask = INTEL_INFO(i915)->platform_engine_mask; + intel_engine_mask_t emask = RUNTIME_INFO(i915)->platform_engine_mask; int i; uncore->fw_get_funcs = &uncore_get_fallback; @@ -2223,6 +2239,11 @@ static int i915_pmic_bus_access_notifier(struct notifier_block *nb, return NOTIFY_OK; } +static void uncore_unmap_mmio(struct drm_device *drm, void *regs) +{ + iounmap(regs); +} + int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr) { struct drm_i915_private *i915 = uncore->i915; @@ -2251,12 +2272,7 @@ int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr) return -EIO; } - return 0; -} - -void intel_uncore_cleanup_mmio(struct intel_uncore *uncore) -{ - iounmap(uncore->regs); + return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio, uncore->regs); } void intel_uncore_init_early(struct intel_uncore *uncore, @@ -2266,7 +2282,6 @@ void intel_uncore_init_early(struct intel_uncore *uncore, uncore->i915 = gt->i915; uncore->gt = gt; uncore->rpm = >->i915->runtime_pm; - uncore->debug = >->i915->mmio_debug; } static void uncore_raw_init(struct intel_uncore *uncore) @@ -2446,8 +2461,11 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, } } -void intel_uncore_fini_mmio(struct intel_uncore *uncore) +/* Called via drm-managed action */ +void intel_uncore_fini_mmio(struct drm_device *dev, void *data) { + struct intel_uncore *uncore = data; + if (intel_uncore_has_forcewake(uncore)) { iosf_mbi_punit_acquire(); iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( @@ -2577,6 +2595,9 @@ bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore) { bool ret; + if (!uncore->debug) + return false; + spin_lock_irq(&uncore->debug->lock); ret = check_for_unclaimed_mmio(uncore); spin_unlock_irq(&uncore->debug->lock); @@ -2589,6 +2610,9 @@ intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore) { bool ret = false; + if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug)) + return false; + spin_lock_irq(&uncore->debug->lock); if (unlikely(uncore->debug->unclaimed_mmio_check <= 0)) diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index b1fa912a65e75db0eaad8e0206b576893adfc409..5022bac80b6797d40cc63d8466cab2c5b92e4d55 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -33,6 +33,7 @@ #include "i915_reg_defs.h" +struct drm_device; struct drm_i915_private; struct intel_runtime_pm; struct intel_uncore; @@ -135,6 +136,16 @@ struct intel_uncore { spinlock_t lock; /** lock is also taken in irq contexts. */ + /* + * Do we need to apply an additional offset to reach the beginning + * of the basic non-engine GT registers (referred to as "GSI" on + * newer platforms, or "GT block" on older platforms)? If so, we'll + * track that here and apply it transparently to registers in the + * appropriate range to maintain compatibility with our existing + * register definitions and GT code. + */ + u32 gsi_offset; + unsigned int flags; #define UNCORE_HAS_FORCEWAKE BIT(0) #define UNCORE_HAS_FPGA_DBG_UNCLAIMED BIT(1) @@ -210,8 +221,7 @@ intel_uncore_has_fifo(const struct intel_uncore *uncore) return uncore->flags & UNCORE_HAS_FIFO; } -void -intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug); +void intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915); void intel_uncore_init_early(struct intel_uncore *uncore, struct intel_gt *gt); int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr); @@ -221,7 +231,7 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore, bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore); bool intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore); void intel_uncore_cleanup_mmio(struct intel_uncore *uncore); -void intel_uncore_fini_mmio(struct intel_uncore *uncore); +void intel_uncore_fini_mmio(struct drm_device *dev, void *data); void intel_uncore_suspend(struct intel_uncore *uncore); void intel_uncore_resume_early(struct intel_uncore *uncore); void intel_uncore_runtime_resume(struct intel_uncore *uncore); @@ -294,19 +304,27 @@ intel_wait_for_register_fw(struct intel_uncore *uncore, 2, timeout_ms, NULL); } +#define IS_GSI_REG(reg) ((reg) < 0x40000) + /* register access functions */ #define __raw_read(x__, s__) \ static inline u##x__ __raw_uncore_read##x__(const struct intel_uncore *uncore, \ i915_reg_t reg) \ { \ - return read##s__(uncore->regs + i915_mmio_reg_offset(reg)); \ + u32 offset = i915_mmio_reg_offset(reg); \ + if (IS_GSI_REG(offset)) \ + offset += uncore->gsi_offset; \ + return read##s__(uncore->regs + offset); \ } #define __raw_write(x__, s__) \ static inline void __raw_uncore_write##x__(const struct intel_uncore *uncore, \ i915_reg_t reg, u##x__ val) \ { \ - write##s__(val, uncore->regs + i915_mmio_reg_offset(reg)); \ + u32 offset = i915_mmio_reg_offset(reg); \ + if (IS_GSI_REG(offset)) \ + offset += uncore->gsi_offset; \ + write##s__(val, uncore->regs + offset); \ } __raw_read(8, b) __raw_read(16, w) @@ -447,6 +465,18 @@ static inline int intel_uncore_write_and_verify(struct intel_uncore *uncore, return (reg_val & mask) != expected_val ? -EINVAL : 0; } +/* + * The raw_reg_{read,write} macros are intended as a micro-optimization for + * interrupt handlers so that the pointer indirection on uncore->regs can + * be computed once (and presumably cached in a register) instead of generating + * extra load instructions for each MMIO access. + * + * Given that these macros are only intended for non-GSI interrupt registers + * (and the goal is to avoid extra instructions generated by the compiler), + * these macros do not account for uncore->gsi_offset. Any caller that needs + * to use these macros on a GSI register is responsible for adding the + * appropriate GSI offset to the 'base' parameter. + */ #define raw_reg_read(base, reg) \ readl(base + i915_mmio_reg_offset(reg)) #define raw_reg_write(base, reg, value) \ diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.c b/drivers/gpu/drm/i915/pxp/intel_pxp.c index 17109c513259a0c890bb40beeac97ffabd489837..69cdaaddc4a9067192e8550a1f4d0c850da7e4e7 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp.c @@ -169,11 +169,11 @@ static void pxp_queue_termination(struct intel_pxp *pxp) * We want to get the same effect as if we received a termination * interrupt, so just pretend that we did. */ - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); intel_pxp_mark_termination_in_progress(pxp); pxp->session_events |= PXP_TERMINATION_REQUEST; queue_work(system_unbound_wq, &pxp->session_work); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } static bool pxp_component_bound(struct intel_pxp *pxp) diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c index e888b5124a07a0d49755fbb35209f87bf1d392e1..4359e8be4101831522a378204bbf20e6763703c5 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c @@ -47,9 +47,9 @@ static int pxp_terminate_set(void *data, u64 val) return -ENODEV; /* simulate a termination interrupt */ - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); intel_pxp_irq_handler(pxp, GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); if (!wait_for_completion_timeout(&pxp->termination, msecs_to_jiffies(100))) diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c index 04745f9144074dc8e270608d75bf813aa2ab015c..c28be430718aebb71983775c3f300ace84abd982 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c @@ -25,7 +25,7 @@ void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir) if (GEM_WARN_ON(!intel_pxp_is_enabled(pxp))) return; - lockdep_assert_held(>->irq_lock); + lockdep_assert_held(gt->irq_lock); if (unlikely(!iir)) return; @@ -55,16 +55,16 @@ static inline void __pxp_set_interrupts(struct intel_gt *gt, u32 interrupts) static inline void pxp_irq_reset(struct intel_gt *gt) { - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); gen11_gt_reset_one_iir(gt, 0, GEN11_KCR); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } void intel_pxp_irq_enable(struct intel_pxp *pxp) { struct intel_gt *gt = pxp_to_gt(pxp); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); if (!pxp->irq_enabled) WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_KCR)); @@ -72,7 +72,7 @@ void intel_pxp_irq_enable(struct intel_pxp *pxp) __pxp_set_interrupts(gt, GEN12_PXP_INTERRUPTS); pxp->irq_enabled = true; - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); } void intel_pxp_irq_disable(struct intel_pxp *pxp) @@ -88,12 +88,12 @@ void intel_pxp_irq_disable(struct intel_pxp *pxp) */ GEM_WARN_ON(intel_pxp_is_active(pxp)); - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); pxp->irq_enabled = false; __pxp_set_interrupts(gt, 0); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); intel_synchronize_irq(gt->i915); pxp_irq_reset(gt); diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c index 92b00b4de240f4801fa460bd25dad1669ee4fe8f..1bb5b5249157f1e939ea64af98adfe3a7cb07c68 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c @@ -144,9 +144,9 @@ void intel_pxp_session_work(struct work_struct *work) intel_wakeref_t wakeref; u32 events = 0; - spin_lock_irq(>->irq_lock); + spin_lock_irq(gt->irq_lock); events = fetch_and_zero(&pxp->session_events); - spin_unlock_irq(>->irq_lock); + spin_unlock_irq(gt->irq_lock); if (!events) return; diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index c4e932368b37e751e83641f5195e7a48037acbe0..39da0fb0d6d26f531f2207ebfc446d2e93f19fb7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -135,7 +135,7 @@ static int __run_selftests(const char *name, int err = 0; while (!i915_selftest.random_seed) - i915_selftest.random_seed = get_random_int(); + i915_selftest.random_seed = get_random_u32(); i915_selftest.timeout_jiffies = i915_selftest.timeout_ms ? diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 9c31a16f8380004c09809df0de7b67463106bb0f..fff11c90f1fa599f444fc16edb9976e2e1271743 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -115,6 +115,7 @@ static struct dev_pm_domain pm_domain = { static void mock_gt_probe(struct drm_i915_private *i915) { i915->gt[0] = &i915->gt0; + i915->gt[0]->name = "Mock GT"; } struct drm_i915_private *mock_gem_device(void) @@ -172,14 +173,14 @@ struct drm_i915_private *mock_gem_device(void) /* Using the global GTT may ask questions about KMS users, so prepare */ drm_mode_config_init(&i915->drm); - mkwrite_device_info(i915)->graphics.ver = -1; + RUNTIME_INFO(i915)->graphics.ip.ver = -1; - mkwrite_device_info(i915)->page_sizes = + RUNTIME_INFO(i915)->page_sizes = I915_GTT_PAGE_SIZE_4K | I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_2M; - mkwrite_device_info(i915)->memory_regions = REGION_SMEM; + RUNTIME_INFO(i915)->memory_regions = REGION_SMEM; intel_memory_regions_hw_probe(i915); spin_lock_init(&i915->gpu_error.lock); @@ -209,7 +210,7 @@ struct drm_i915_private *mock_gem_device(void) mock_init_ggtt(to_gt(i915)); to_gt(i915)->vm = i915_vm_get(&to_gt(i915)->ggtt->vm); - mkwrite_device_info(i915)->platform_engine_mask = BIT(0); + RUNTIME_INFO(i915)->platform_engine_mask = BIT(0); to_gt(i915)->info.engine_mask = BIT(0); to_gt(i915)->engine[RCS0] = mock_engine(i915, "mock", RCS0); diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c index 630a4e301ef6c2cbf4091d873a75a3d1d2d10756..508a6d994e831a750f0c8b5c3f37ef62bb8832da 100644 --- a/drivers/gpu/drm/mediatek/mtk_dpi.c +++ b/drivers/gpu/drm/mediatek/mtk_dpi.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c index 2d72cc5ddaba4fea36ee175d1159ecb39990abf8..6b6d5335c8347bed246d357a6454c35622f5b4e5 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_ddp_comp.c @@ -157,7 +157,7 @@ static void mtk_dither_config(struct device *dev, unsigned int w, { struct mtk_ddp_comp_dev *priv = dev_get_drvdata(dev); - mtk_ddp_write(cmdq_pkt, h << 16 | w, &priv->cmdq_reg, priv->regs, DISP_REG_DITHER_SIZE); + mtk_ddp_write(cmdq_pkt, w << 16 | h, &priv->cmdq_reg, priv->regs, DISP_REG_DITHER_SIZE); mtk_ddp_write(cmdq_pkt, DITHER_RELAY_MODE, &priv->cmdq_reg, priv->regs, DISP_REG_DITHER_CFG); mtk_dither_set_common(priv->regs, &priv->cmdq_reg, bpc, DISP_REG_DITHER_CFG, diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c index 5f02f8d0e4fcfde1ce1dbc867d70f76ea80805a5..91f58db5915f5b9e013e36ab025e9ab5daf91f08 100644 --- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c +++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c @@ -833,11 +833,8 @@ static int mtk_drm_sys_prepare(struct device *dev) { struct mtk_drm_private *private = dev_get_drvdata(dev); struct drm_device *drm = private->drm; - int ret; - - ret = drm_mode_config_helper_suspend(drm); - return ret; + return drm_mode_config_helper_suspend(drm); } static void mtk_drm_sys_complete(struct device *dev) diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c index 9cc406e1eee197a1a150d11e6d8ea7a388fc4493..3b7d13028fb6b7577111b0dfac42a1338cfbfd8b 100644 --- a/drivers/gpu/drm/mediatek/mtk_dsi.c +++ b/drivers/gpu/drm/mediatek/mtk_dsi.c @@ -685,6 +685,16 @@ static void mtk_dsi_poweroff(struct mtk_dsi *dsi) if (--dsi->refcount != 0) return; + /* + * mtk_dsi_stop() and mtk_dsi_start() is asymmetric, since + * mtk_dsi_stop() should be called after mtk_drm_crtc_atomic_disable(), + * which needs irq for vblank, and mtk_dsi_stop() will disable irq. + * mtk_dsi_start() needs to be called in mtk_output_dsi_enable(), + * after dsi is fully set. + */ + mtk_dsi_stop(dsi); + + mtk_dsi_switch_to_cmd_mode(dsi, VM_DONE_INT_FLAG, 500); mtk_dsi_reset_engine(dsi); mtk_dsi_lane0_ulp_mode_enter(dsi); mtk_dsi_clk_ulp_mode_enter(dsi); @@ -735,17 +745,6 @@ static void mtk_output_dsi_disable(struct mtk_dsi *dsi) if (!dsi->enabled) return; - /* - * mtk_dsi_stop() and mtk_dsi_start() is asymmetric, since - * mtk_dsi_stop() should be called after mtk_drm_crtc_atomic_disable(), - * which needs irq for vblank, and mtk_dsi_stop() will disable irq. - * mtk_dsi_start() needs to be called in mtk_output_dsi_enable(), - * after dsi is fully set. - */ - mtk_dsi_stop(dsi); - - mtk_dsi_switch_to_cmd_mode(dsi, VM_DONE_INT_FLAG, 500); - dsi->enabled = false; } @@ -808,10 +807,13 @@ static void mtk_dsi_bridge_atomic_post_disable(struct drm_bridge *bridge, static const struct drm_bridge_funcs mtk_dsi_bridge_funcs = { .attach = mtk_dsi_bridge_attach, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, .atomic_disable = mtk_dsi_bridge_atomic_disable, + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, .atomic_enable = mtk_dsi_bridge_atomic_enable, .atomic_pre_enable = mtk_dsi_bridge_atomic_pre_enable, .atomic_post_disable = mtk_dsi_bridge_atomic_post_disable, + .atomic_reset = drm_atomic_helper_bridge_reset, .mode_set = mtk_dsi_bridge_mode_set, }; diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c index 3196189429bcf895138d2160843b2e5dc2f2bc56..4c80b6896dc3d6cf4f59e0ae0f1a366db8f57bf3 100644 --- a/drivers/gpu/drm/mediatek/mtk_hdmi.c +++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c index dfd6a9f33ddab02167860d681d68d07c12347c71..815dfe30492b6cf80618954728599ac92da25cde 100644 --- a/drivers/gpu/drm/meson/meson_plane.c +++ b/drivers/gpu/drm/meson/meson_plane.c @@ -169,7 +169,7 @@ static void meson_plane_atomic_update(struct drm_plane *plane, /* Enable OSD and BLK0, set max global alpha */ priv->viu.osd1_ctrl_stat = OSD_ENABLE | - (0xFF << OSD_GLOBAL_ALPHA_SHIFT) | + (0x100 << OSD_GLOBAL_ALPHA_SHIFT) | OSD_BLK0_ENABLE; priv->viu.osd1_ctrl_stat2 = readl(priv->io_base + diff --git a/drivers/gpu/drm/meson/meson_viu.c b/drivers/gpu/drm/meson/meson_viu.c index bb7e109534de1e958c69fdfd52feae53c5112a0b..d4b907889a21d199a43f0af7a7a6bcf56ec201c8 100644 --- a/drivers/gpu/drm/meson/meson_viu.c +++ b/drivers/gpu/drm/meson/meson_viu.c @@ -94,7 +94,7 @@ static void meson_viu_set_g12a_osd1_matrix(struct meson_drm *priv, priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF11_12)); writel(((m[9] & 0x1fff) << 16) | (m[10] & 0x1fff), priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF20_21)); - writel((m[11] & 0x1fff) << 16, + writel((m[11] & 0x1fff), priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF22)); writel(((m[18] & 0xfff) << 16) | (m[19] & 0xfff), diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c index 4d38b8e18030c9649c2fd530d626c85c28cfb2d8..ece6cd102dbb1fafddd99c320f6142996bd4484b 100644 --- a/drivers/gpu/drm/mgag200/mgag200_drv.c +++ b/drivers/gpu/drm/mgag200/mgag200_drv.c @@ -263,7 +263,11 @@ mgag200_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) return ret; - drm_fbdev_generic_setup(dev, 0); + /* + * FIXME: A 24-bit color depth does not work with 24 bpp on + * G200ER. Force 32 bpp. + */ + drm_fbdev_generic_setup(dev, 32); return 0; } diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c index 0ab0e1dd8bbb0706dbbc5f524e96ae832a760656..2c8b9899625b06bba642010acef5fb0b1be37865 100644 --- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c @@ -68,7 +68,7 @@ static void a3xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) /* BIT(31) of CACHE_FLUSH_TS triggers CACHE_FLUSH_TS IRQ from GPU */ OUT_PKT3(ring, CP_EVENT_WRITE, 3); - OUT_RING(ring, CACHE_FLUSH_TS | BIT(31)); + OUT_RING(ring, CACHE_FLUSH_TS | CP_EVENT_WRITE_0_IRQ); OUT_RING(ring, rbmemptr(ring, fence)); OUT_RING(ring, submit->seqno); diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c index 0c6b2a6d0b4c95e9ab894fc9e99252c4f4db7cf7..7cb8d9849c07327097ce383c246f923e4190b93e 100644 --- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c @@ -62,7 +62,7 @@ static void a4xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) /* BIT(31) of CACHE_FLUSH_TS triggers CACHE_FLUSH_TS IRQ from GPU */ OUT_PKT3(ring, CP_EVENT_WRITE, 3); - OUT_RING(ring, CACHE_FLUSH_TS | BIT(31)); + OUT_RING(ring, CACHE_FLUSH_TS | CP_EVENT_WRITE_0_IRQ); OUT_RING(ring, rbmemptr(ring, fence)); OUT_RING(ring, submit->seqno); diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h index b03e2c413ab19e39b6379d052fbdf1927c7c1b8a..beea4a7fc1df76042530aa33df37009f45390863 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h @@ -1413,6 +1413,10 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00 #define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00000011 +#define REG_A6XX_RBBM_GBIF_HALT 0x00000016 + +#define REG_A6XX_RBBM_GBIF_HALT_ACK 0x00000017 + #define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD 0x0000001c #define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE 0x00000001 diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 310a317885a129a08d31b03b2ea6479f010f78fc..e033d6a67a20c260abcbd9644dc3dda768a441b1 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -873,9 +873,47 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) (val & 1), 100, 1000); } +#define GBIF_CLIENT_HALT_MASK BIT(0) +#define GBIF_ARB_HALT_MASK BIT(1) + +static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) +{ + struct msm_gpu *gpu = &adreno_gpu->base; + + if (!a6xx_has_gbif(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); + spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & + 0xf) == 0xf); + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); + + return; + } + + /* Halt the gx side of GBIF */ + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1); + spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1); + + /* Halt new client requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); + + /* Halt all AXI requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); + + /* The GBIF halt needs to be explicitly cleared */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); +} + /* Force the GMU off in case it isn't responsive */ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) { + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + /* Flush all the queues */ a6xx_hfi_stop(gmu); @@ -887,6 +925,15 @@ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) /* Make sure there are no outstanding RPMh votes */ a6xx_gmu_rpmh_off(gmu); + + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + + a6xx_bus_clear_pending_transactions(adreno_gpu); + + /* Reset GPU core blocks */ + gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, 1); + udelay(100); } static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) @@ -1014,36 +1061,6 @@ bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) return true; } -#define GBIF_CLIENT_HALT_MASK BIT(0) -#define GBIF_ARB_HALT_MASK BIT(1) - -static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) -{ - struct msm_gpu *gpu = &adreno_gpu->base; - - if (!a6xx_has_gbif(adreno_gpu)) { - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); - spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & - 0xf) == 0xf); - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); - - return; - } - - /* Halt new client requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); - - /* Halt all AXI requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); - - /* The GBIF halt needs to be explicitly cleared */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); -} - /* Gracefully try to shut down the GMU and by extension the GPU */ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) { @@ -1069,7 +1086,11 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) a6xx_bus_clear_pending_transactions(adreno_gpu); /* tell the GMU we want to slumber */ - a6xx_gmu_notify_slumber(gmu); + ret = a6xx_gmu_notify_slumber(gmu); + if (ret) { + a6xx_gmu_force_off(gmu); + return; + } ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 4d501100b9e45e445a17a3a3418298634bd19db9..fdc578016e0bf33dcf6a2b460bec5ec2f94aa9af 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -10,6 +10,7 @@ #include #include +#include #include #define GPU_PAS_ID 13 @@ -146,7 +147,7 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu, */ OUT_PKT7(ring, CP_EVENT_WRITE, 1); - OUT_RING(ring, 0x31); + OUT_RING(ring, CACHE_INVALIDATE); if (!sysprof) { /* @@ -987,6 +988,10 @@ static int hw_init(struct msm_gpu *gpu) /* Make sure the GMU keeps the GPU on while we set it up */ a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + /* Clear GBIF halt in case GX domain was not collapsed */ + if (a6xx_has_gbif(adreno_gpu)) + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0); + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); /* @@ -1261,7 +1266,7 @@ static void a6xx_recover(struct msm_gpu *gpu) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); - int i; + int i, active_submits; adreno_dump_info(gpu); @@ -1272,14 +1277,46 @@ static void a6xx_recover(struct msm_gpu *gpu) if (hang_debug) a6xx_dump(gpu); + /* Halt SQE first */ + gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3); + /* * Turn off keep alive that might have been enabled by the hang * interrupt */ gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); - gpu->funcs->pm_suspend(gpu); - gpu->funcs->pm_resume(gpu); + pm_runtime_dont_use_autosuspend(&gpu->pdev->dev); + + /* active_submit won't change until we make a submission */ + mutex_lock(&gpu->active_lock); + active_submits = gpu->active_submits; + + /* + * Temporarily clear active_submits count to silence a WARN() in the + * runtime suspend cb + */ + gpu->active_submits = 0; + + /* Drop the rpm refcount from active submits */ + if (active_submits) + pm_runtime_put(&gpu->pdev->dev); + + /* And the final one from recover worker */ + pm_runtime_put_sync(&gpu->pdev->dev); + + /* Call into gpucc driver to poll for cx gdsc collapse */ + reset_control_reset(gpu->cx_collapse); + + pm_runtime_use_autosuspend(&gpu->pdev->dev); + + if (active_submits) + pm_runtime_get(&gpu->pdev->dev); + + pm_runtime_get_sync(&gpu->pdev->dev); + + gpu->active_submits = active_submits; + mutex_unlock(&gpu->active_lock); msm_gpu_hw_init(gpu); } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index 781dcd3fb2836b8f6eb2ee27d33473c35f659262..13ce321283ff92eec7f21b2bca50b0b386dc1fcb 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -412,7 +412,6 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, struct dpu_format *format; struct dpu_hw_ctl *ctl = mixer->lm_ctl; - u32 flush_mask; uint32_t stage_idx, lm_idx; int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 }; bool bg_alpha_enable = false; @@ -420,6 +419,8 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, memset(fetch_active, 0, sizeof(fetch_active)); drm_atomic_crtc_for_each_plane(plane, crtc) { + enum dpu_sspp sspp_idx; + state = plane->state; if (!state) continue; @@ -430,14 +431,14 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, pstate = to_dpu_plane_state(state); fb = state->fb; - dpu_plane_get_ctl_flush(plane, ctl, &flush_mask); - set_bit(dpu_plane_pipe(plane), fetch_active); + sspp_idx = dpu_plane_pipe(plane); + set_bit(sspp_idx, fetch_active); DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d\n", crtc->base.id, pstate->stage, plane->base.id, - dpu_plane_pipe(plane) - SSPP_VIG0, + sspp_idx - SSPP_VIG0, state->fb ? state->fb->base.id : -1); format = to_dpu_format(msm_framebuffer_format(pstate->base.fb)); @@ -447,13 +448,13 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, stage_idx = zpos_cnt[pstate->stage]++; stage_cfg->stage[pstate->stage][stage_idx] = - dpu_plane_pipe(plane); + sspp_idx; stage_cfg->multirect_index[pstate->stage][stage_idx] = pstate->multirect_index; trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane), state, pstate, stage_idx, - dpu_plane_pipe(plane) - SSPP_VIG0, + sspp_idx - SSPP_VIG0, format->base.pixel_format, fb ? fb->modifier : 0); @@ -462,7 +463,8 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc, _dpu_crtc_setup_blend_cfg(mixer + lm_idx, pstate, format); - mixer[lm_idx].flush_mask |= flush_mask; + mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl, + sspp_idx); if (bg_alpha_enable && !format->alpha_enable) mixer[lm_idx].mixer_op_mode = 0; @@ -496,7 +498,6 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc) for (i = 0; i < cstate->num_mixers; i++) { mixer[i].mixer_op_mode = 0; - mixer[i].flush_mask = 0; if (mixer[i].lm_ctl->ops.clear_all_blendstages) mixer[i].lm_ctl->ops.clear_all_blendstages( mixer[i].lm_ctl); @@ -513,17 +514,14 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc) lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode); - mixer[i].flush_mask |= ctl->ops.get_bitmask_mixer(ctl, - mixer[i].hw_lm->idx); - /* stage config flush mask */ - ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask); + ctl->ops.update_pending_flush_mixer(ctl, + mixer[i].hw_lm->idx); - DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d, flush mask 0x%x\n", + DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n", mixer[i].hw_lm->idx - LM_0, mixer[i].mixer_op_mode, - ctl->idx - CTL_0, - mixer[i].flush_mask); + ctl->idx - CTL_0); ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx, &stage_cfg); @@ -767,16 +765,9 @@ static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc) dspp->ops.setup_pcc(dspp, &cfg); } - mixer[i].flush_mask |= ctl->ops.get_bitmask_dspp(ctl, - mixer[i].hw_dspp->idx); - /* stage config flush mask */ - ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask); - - DRM_DEBUG_ATOMIC("lm %d, ctl %d, flush mask 0x%x\n", - mixer[i].hw_lm->idx - DSPP_0, - ctl->idx - CTL_0, - mixer[i].flush_mask); + ctl->ops.update_pending_flush_dspp(ctl, + mixer[i].hw_dspp->idx); } } @@ -1235,17 +1226,8 @@ static int dpu_crtc_atomic_check(struct drm_crtc *crtc, } for (i = 1; i < SSPP_MAX; i++) { - if (pipe_staged[i]) { + if (pipe_staged[i]) dpu_plane_clear_multirect(pipe_staged[i]); - - if (is_dpu_plane_virtual(pipe_staged[i]->plane)) { - DPU_ERROR( - "r1 only virt plane:%d not supported\n", - pipe_staged[i]->plane->base.id); - rc = -EINVAL; - goto end; - } - } } z_pos = -1; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h index 9b67645c25741372b839827334db51fdfce10647..539b68b1626a3e2d9dca1dde3ae64572749f5f61 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h @@ -97,7 +97,6 @@ struct dpu_crtc_mixer { struct dpu_hw_ctl *lm_ctl; struct dpu_hw_dspp *hw_dspp; u32 mixer_op_mode; - u32 flush_mask; }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index c682d4e02d1bcb71fd53d299cb65b5a4e187967c..9c6817b5a19439e6125d6d2a418befa1799eca57 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -162,7 +162,7 @@ enum dpu_enc_rc_states { * @vsync_event_work: worker to handle vsync event for autorefresh * @topology: topology of the display * @idle_timeout: idle timeout duration in milliseconds - * @dsc: msm_display_dsc_config pointer, for DSC-enabled encoders + * @dsc: drm_dsc_config pointer, for DSC-enabled encoders */ struct dpu_encoder_virt { struct drm_encoder base; @@ -208,7 +208,7 @@ struct dpu_encoder_virt { bool wide_bus_en; /* DSC configuration */ - struct msm_display_dsc_config *dsc; + struct drm_dsc_config *dsc; }; #define to_dpu_encoder_virt(x) container_of(x, struct dpu_encoder_virt, base) @@ -1791,12 +1791,12 @@ static void dpu_encoder_vsync_event_work_handler(struct kthread_work *work) } static u32 -dpu_encoder_dsc_initial_line_calc(struct msm_display_dsc_config *dsc, +dpu_encoder_dsc_initial_line_calc(struct drm_dsc_config *dsc, u32 enc_ip_width) { int ssm_delay, total_pixels, soft_slice_per_enc; - soft_slice_per_enc = enc_ip_width / dsc->drm->slice_width; + soft_slice_per_enc = enc_ip_width / dsc->slice_width; /* * minimum number of initial line pixels is a sum of: @@ -1808,16 +1808,16 @@ dpu_encoder_dsc_initial_line_calc(struct msm_display_dsc_config *dsc, * 5. 6 additional pixels as the output of the rate buffer is * 48 bits wide */ - ssm_delay = ((dsc->drm->bits_per_component < 10) ? 84 : 92); - total_pixels = ssm_delay * 3 + dsc->drm->initial_xmit_delay + 47; + ssm_delay = ((dsc->bits_per_component < 10) ? 84 : 92); + total_pixels = ssm_delay * 3 + dsc->initial_xmit_delay + 47; if (soft_slice_per_enc > 1) total_pixels += (ssm_delay * 3); - return DIV_ROUND_UP(total_pixels, dsc->drm->slice_width); + return DIV_ROUND_UP(total_pixels, dsc->slice_width); } static void dpu_encoder_dsc_pipe_cfg(struct dpu_hw_dsc *hw_dsc, struct dpu_hw_pingpong *hw_pp, - struct msm_display_dsc_config *dsc, + struct drm_dsc_config *dsc, u32 common_mode, u32 initial_lines) { @@ -1835,7 +1835,7 @@ static void dpu_encoder_dsc_pipe_cfg(struct dpu_hw_dsc *hw_dsc, } static void dpu_encoder_prep_dsc(struct dpu_encoder_virt *dpu_enc, - struct msm_display_dsc_config *dsc) + struct drm_dsc_config *dsc) { /* coding only for 2LM, 2enc, 1 dsc config */ struct dpu_encoder_phys *enc_master = dpu_enc->cur_master; @@ -1858,14 +1858,15 @@ static void dpu_encoder_prep_dsc(struct dpu_encoder_virt *dpu_enc, } } - pic_width = dsc->drm->pic_width; + dsc_common_mode = 0; + pic_width = dsc->pic_width; dsc_common_mode = DSC_MODE_MULTIPLEX | DSC_MODE_SPLIT_PANEL; if (enc_master->intf_mode == INTF_MODE_VIDEO) dsc_common_mode |= DSC_MODE_VIDEO; - this_frame_slices = pic_width / dsc->drm->slice_width; - intf_ip_w = this_frame_slices * dsc->drm->slice_width; + this_frame_slices = pic_width / dsc->slice_width; + intf_ip_w = this_frame_slices * dsc->slice_width; /* * dsc merge case: when using 2 encoders for the same stream, @@ -1980,7 +1981,6 @@ static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc) { struct dpu_hw_mixer_cfg mixer; int i, num_lm; - u32 flush_mask = 0; struct dpu_global_state *global_state; struct dpu_hw_blk *hw_lm[2]; struct dpu_hw_mixer *hw_mixer[2]; @@ -1999,9 +1999,8 @@ static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc) for (i = 0; i < num_lm; i++) { hw_mixer[i] = to_dpu_hw_mixer(hw_lm[i]); - flush_mask = phys_enc->hw_ctl->ops.get_bitmask_mixer(ctl, hw_mixer[i]->idx); - if (phys_enc->hw_ctl->ops.update_pending_flush) - phys_enc->hw_ctl->ops.update_pending_flush(ctl, flush_mask); + if (phys_enc->hw_ctl->ops.update_pending_flush_mixer) + phys_enc->hw_ctl->ops.update_pending_flush_mixer(ctl, hw_mixer[i]->idx); /* clear all blendstages */ if (phys_enc->hw_ctl->ops.setup_blendstage) @@ -2061,6 +2060,12 @@ void dpu_encoder_helper_phys_cleanup(struct dpu_encoder_phys *phys_enc) intf_cfg.stream_sel = 0; /* Don't care value for video mode */ intf_cfg.mode_3d = dpu_encoder_helper_get_3d_blend_mode(phys_enc); + + if (phys_enc->hw_intf) + intf_cfg.intf = phys_enc->hw_intf->idx; + if (phys_enc->hw_wb) + intf_cfg.wb = phys_enc->hw_wb->idx; + if (phys_enc->hw_pp->merge_3d) intf_cfg.merge_3d = phys_enc->hw_pp->merge_3d->idx; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h index d4d1ecd416e392a99676bed96beda13d1d1442d8..9e7236ef34e6d6041497223e3005156023516143 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h @@ -36,7 +36,7 @@ struct msm_display_info { uint32_t h_tile_instance[MAX_H_TILES_PER_DISPLAY]; bool is_cmd_mode; bool is_te_using_watchdog_timer; - struct msm_display_dsc_config *dsc; + struct drm_dsc_config *dsc; }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c index 0239a811d5ecb3121cf13e4c22bca29627ac0779..27f029fdc68298c6f182148a75f2bdaf340ee7af 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c @@ -1333,7 +1333,7 @@ static const struct dpu_vbif_dynamic_ot_cfg msm8998_ot_rdwr_cfg[] = { static const struct dpu_vbif_cfg msm8998_vbif[] = { { - .name = "vbif_0", .id = VBIF_0, + .name = "vbif_rt", .id = VBIF_RT, .base = 0, .len = 0x1040, .default_ot_rd_limit = 32, .default_ot_wr_limit = 32, @@ -1363,7 +1363,7 @@ static const struct dpu_vbif_cfg msm8998_vbif[] = { static const struct dpu_vbif_cfg sdm845_vbif[] = { { - .name = "vbif_0", .id = VBIF_0, + .name = "vbif_rt", .id = VBIF_RT, .base = 0, .len = 0x1040, .features = BIT(DPU_VBIF_QOS_REMAP), .xin_halt_timeout = 0x4000, @@ -1939,11 +1939,6 @@ static const struct dpu_mdss_hw_cfg_handler cfg_handler[] = { const struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev) { int i; - struct dpu_mdss_cfg *dpu_cfg; - - dpu_cfg = kzalloc(sizeof(*dpu_cfg), GFP_KERNEL); - if (!dpu_cfg) - return ERR_PTR(-ENOMEM); for (i = 0; i < ARRAY_SIZE(cfg_handler); i++) { if (cfg_handler[i].hw_rev == hw_rev) diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h index 71fe4c505f5b204053d1362d787fe422e0c39b3a..38aa38ab1568581bcab69ac067e26ea9118146bc 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h @@ -76,7 +76,7 @@ enum { /** * MDP TOP BLOCK features - * @DPU_MDP_PANIC_PER_PIPE Panic configuration needs to be be done per pipe + * @DPU_MDP_PANIC_PER_PIPE Panic configuration needs to be done per pipe * @DPU_MDP_10BIT_SUPPORT, Chipset supports 10 bit pixel formats * @DPU_MDP_BWC, MDSS HW supports Bandwidth compression. * @DPU_MDP_UBWC_1_0, This chipsets supports Universal Bandwidth diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c index e12b7fa48a7b3a0a03798b567d9fe34ea61be2f9..a35ecb6676c888e61cc3f586ecdbedf695e16d5d 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c @@ -150,92 +150,84 @@ static inline void dpu_hw_ctl_trigger_flush(struct dpu_hw_ctl *ctx) DPU_REG_WRITE(&ctx->hw, CTL_FLUSH, ctx->pending_flush_mask); } -static uint32_t dpu_hw_ctl_get_bitmask_sspp(struct dpu_hw_ctl *ctx, +static void dpu_hw_ctl_update_pending_flush_sspp(struct dpu_hw_ctl *ctx, enum dpu_sspp sspp) { - uint32_t flushbits = 0; - switch (sspp) { case SSPP_VIG0: - flushbits = BIT(0); + ctx->pending_flush_mask |= BIT(0); break; case SSPP_VIG1: - flushbits = BIT(1); + ctx->pending_flush_mask |= BIT(1); break; case SSPP_VIG2: - flushbits = BIT(2); + ctx->pending_flush_mask |= BIT(2); break; case SSPP_VIG3: - flushbits = BIT(18); + ctx->pending_flush_mask |= BIT(18); break; case SSPP_RGB0: - flushbits = BIT(3); + ctx->pending_flush_mask |= BIT(3); break; case SSPP_RGB1: - flushbits = BIT(4); + ctx->pending_flush_mask |= BIT(4); break; case SSPP_RGB2: - flushbits = BIT(5); + ctx->pending_flush_mask |= BIT(5); break; case SSPP_RGB3: - flushbits = BIT(19); + ctx->pending_flush_mask |= BIT(19); break; case SSPP_DMA0: - flushbits = BIT(11); + ctx->pending_flush_mask |= BIT(11); break; case SSPP_DMA1: - flushbits = BIT(12); + ctx->pending_flush_mask |= BIT(12); break; case SSPP_DMA2: - flushbits = BIT(24); + ctx->pending_flush_mask |= BIT(24); break; case SSPP_DMA3: - flushbits = BIT(25); + ctx->pending_flush_mask |= BIT(25); break; case SSPP_CURSOR0: - flushbits = BIT(22); + ctx->pending_flush_mask |= BIT(22); break; case SSPP_CURSOR1: - flushbits = BIT(23); + ctx->pending_flush_mask |= BIT(23); break; default: break; } - - return flushbits; } -static uint32_t dpu_hw_ctl_get_bitmask_mixer(struct dpu_hw_ctl *ctx, +static void dpu_hw_ctl_update_pending_flush_mixer(struct dpu_hw_ctl *ctx, enum dpu_lm lm) { - uint32_t flushbits = 0; - switch (lm) { case LM_0: - flushbits = BIT(6); + ctx->pending_flush_mask |= BIT(6); break; case LM_1: - flushbits = BIT(7); + ctx->pending_flush_mask |= BIT(7); break; case LM_2: - flushbits = BIT(8); + ctx->pending_flush_mask |= BIT(8); break; case LM_3: - flushbits = BIT(9); + ctx->pending_flush_mask |= BIT(9); break; case LM_4: - flushbits = BIT(10); + ctx->pending_flush_mask |= BIT(10); break; case LM_5: - flushbits = BIT(20); + ctx->pending_flush_mask |= BIT(20); break; default: - return -EINVAL; + break; } - flushbits |= CTL_FLUSH_MASK_CTL; - - return flushbits; + ctx->pending_flush_mask |= CTL_FLUSH_MASK_CTL; } static void dpu_hw_ctl_update_pending_flush_intf(struct dpu_hw_ctl *ctx, @@ -294,29 +286,25 @@ static void dpu_hw_ctl_update_pending_flush_merge_3d_v1(struct dpu_hw_ctl *ctx, ctx->pending_flush_mask |= BIT(MERGE_3D_IDX); } -static uint32_t dpu_hw_ctl_get_bitmask_dspp(struct dpu_hw_ctl *ctx, +static void dpu_hw_ctl_update_pending_flush_dspp(struct dpu_hw_ctl *ctx, enum dpu_dspp dspp) { - uint32_t flushbits = 0; - switch (dspp) { case DSPP_0: - flushbits = BIT(13); + ctx->pending_flush_mask |= BIT(13); break; case DSPP_1: - flushbits = BIT(14); + ctx->pending_flush_mask |= BIT(14); break; case DSPP_2: - flushbits = BIT(15); + ctx->pending_flush_mask |= BIT(15); break; case DSPP_3: - flushbits = BIT(21); + ctx->pending_flush_mask |= BIT(21); break; default: - return 0; + break; } - - return flushbits; } static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us) @@ -685,9 +673,9 @@ static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops, ops->wait_reset_status = dpu_hw_ctl_wait_reset_status; ops->clear_all_blendstages = dpu_hw_ctl_clear_all_blendstages; ops->setup_blendstage = dpu_hw_ctl_setup_blendstage; - ops->get_bitmask_sspp = dpu_hw_ctl_get_bitmask_sspp; - ops->get_bitmask_mixer = dpu_hw_ctl_get_bitmask_mixer; - ops->get_bitmask_dspp = dpu_hw_ctl_get_bitmask_dspp; + ops->update_pending_flush_sspp = dpu_hw_ctl_update_pending_flush_sspp; + ops->update_pending_flush_mixer = dpu_hw_ctl_update_pending_flush_mixer; + ops->update_pending_flush_dspp = dpu_hw_ctl_update_pending_flush_dspp; if (cap & BIT(DPU_CTL_FETCH_ACTIVE)) ops->set_active_pipes = dpu_hw_ctl_set_fetch_pipe_active; }; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h index 7d9ad6a3f9f6ab3cb0d99ebfecd84753957e78c1..96c012ec8467689527f736cc3454a0e104a2a57c 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h @@ -129,6 +129,32 @@ struct dpu_hw_ctl_ops { void (*update_pending_flush_merge_3d)(struct dpu_hw_ctl *ctx, enum dpu_merge_3d blk); + /** + * OR in the given flushbits to the cached pending_flush_mask + * No effect on hardware + * @ctx : ctl path ctx pointer + * @blk : SSPP block index + */ + void (*update_pending_flush_sspp)(struct dpu_hw_ctl *ctx, + enum dpu_sspp blk); + + /** + * OR in the given flushbits to the cached pending_flush_mask + * No effect on hardware + * @ctx : ctl path ctx pointer + * @blk : LM block index + */ + void (*update_pending_flush_mixer)(struct dpu_hw_ctl *ctx, + enum dpu_lm blk); + + /** + * OR in the given flushbits to the cached pending_flush_mask + * No effect on hardware + * @ctx : ctl path ctx pointer + * @blk : DSPP block index + */ + void (*update_pending_flush_dspp)(struct dpu_hw_ctl *ctx, + enum dpu_dspp blk); /** * Write the value of the pending_flush_mask to hardware * @ctx : ctl path ctx pointer @@ -171,15 +197,6 @@ struct dpu_hw_ctl_ops { */ int (*wait_reset_status)(struct dpu_hw_ctl *ctx); - uint32_t (*get_bitmask_sspp)(struct dpu_hw_ctl *ctx, - enum dpu_sspp blk); - - uint32_t (*get_bitmask_mixer)(struct dpu_hw_ctl *ctx, - enum dpu_lm blk); - - uint32_t (*get_bitmask_dspp)(struct dpu_hw_ctl *ctx, - enum dpu_dspp blk); - /** * Set all blend stages to disabled * @ctx : ctl path ctx pointer diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c index 411689ae638259452f3b525893c41f0c4d20732c..f2ddcfb6f7ee6e10a80d78bca82c59acf2e6334f 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c @@ -37,7 +37,7 @@ static void dpu_hw_dsc_disable(struct dpu_hw_dsc *dsc) } static void dpu_hw_dsc_config(struct dpu_hw_dsc *hw_dsc, - struct msm_display_dsc_config *dsc, + struct drm_dsc_config *dsc, u32 mode, u32 initial_lines) { @@ -52,89 +52,89 @@ static void dpu_hw_dsc_config(struct dpu_hw_dsc *hw_dsc, if (is_cmd_mode) initial_lines += 1; - slice_last_group_size = 3 - (dsc->drm->slice_width % 3); + slice_last_group_size = 3 - (dsc->slice_width % 3); data = (initial_lines << 20); data |= ((slice_last_group_size - 1) << 18); /* bpp is 6.4 format, 4 LSBs bits are for fractional part */ - data |= dsc->drm->bits_per_pixel << 12; - lsb = dsc->drm->bits_per_pixel % 4; - bpp = dsc->drm->bits_per_pixel / 4; + data |= dsc->bits_per_pixel << 12; + lsb = dsc->bits_per_pixel % 4; + bpp = dsc->bits_per_pixel / 4; bpp *= 4; bpp <<= 4; bpp |= lsb; data |= bpp << 8; - data |= (dsc->drm->block_pred_enable << 7); - data |= (dsc->drm->line_buf_depth << 3); - data |= (dsc->drm->simple_422 << 2); - data |= (dsc->drm->convert_rgb << 1); - data |= dsc->drm->bits_per_component; + data |= (dsc->block_pred_enable << 7); + data |= (dsc->line_buf_depth << 3); + data |= (dsc->simple_422 << 2); + data |= (dsc->convert_rgb << 1); + data |= dsc->bits_per_component; DPU_REG_WRITE(c, DSC_ENC, data); - data = dsc->drm->pic_width << 16; - data |= dsc->drm->pic_height; + data = dsc->pic_width << 16; + data |= dsc->pic_height; DPU_REG_WRITE(c, DSC_PICTURE, data); - data = dsc->drm->slice_width << 16; - data |= dsc->drm->slice_height; + data = dsc->slice_width << 16; + data |= dsc->slice_height; DPU_REG_WRITE(c, DSC_SLICE, data); - data = dsc->drm->slice_chunk_size << 16; + data = dsc->slice_chunk_size << 16; DPU_REG_WRITE(c, DSC_CHUNK_SIZE, data); - data = dsc->drm->initial_dec_delay << 16; - data |= dsc->drm->initial_xmit_delay; + data = dsc->initial_dec_delay << 16; + data |= dsc->initial_xmit_delay; DPU_REG_WRITE(c, DSC_DELAY, data); - data = dsc->drm->initial_scale_value; + data = dsc->initial_scale_value; DPU_REG_WRITE(c, DSC_SCALE_INITIAL, data); - data = dsc->drm->scale_decrement_interval; + data = dsc->scale_decrement_interval; DPU_REG_WRITE(c, DSC_SCALE_DEC_INTERVAL, data); - data = dsc->drm->scale_increment_interval; + data = dsc->scale_increment_interval; DPU_REG_WRITE(c, DSC_SCALE_INC_INTERVAL, data); - data = dsc->drm->first_line_bpg_offset; + data = dsc->first_line_bpg_offset; DPU_REG_WRITE(c, DSC_FIRST_LINE_BPG_OFFSET, data); - data = dsc->drm->nfl_bpg_offset << 16; - data |= dsc->drm->slice_bpg_offset; + data = dsc->nfl_bpg_offset << 16; + data |= dsc->slice_bpg_offset; DPU_REG_WRITE(c, DSC_BPG_OFFSET, data); - data = dsc->drm->initial_offset << 16; - data |= dsc->drm->final_offset; + data = dsc->initial_offset << 16; + data |= dsc->final_offset; DPU_REG_WRITE(c, DSC_DSC_OFFSET, data); - det_thresh_flatness = 7 + 2 * (dsc->drm->bits_per_component - 8); + det_thresh_flatness = 7 + 2 * (dsc->bits_per_component - 8); data = det_thresh_flatness << 10; - data |= dsc->drm->flatness_max_qp << 5; - data |= dsc->drm->flatness_min_qp; + data |= dsc->flatness_max_qp << 5; + data |= dsc->flatness_min_qp; DPU_REG_WRITE(c, DSC_FLATNESS, data); - data = dsc->drm->rc_model_size; + data = dsc->rc_model_size; DPU_REG_WRITE(c, DSC_RC_MODEL_SIZE, data); - data = dsc->drm->rc_tgt_offset_low << 18; - data |= dsc->drm->rc_tgt_offset_high << 14; - data |= dsc->drm->rc_quant_incr_limit1 << 9; - data |= dsc->drm->rc_quant_incr_limit0 << 4; - data |= dsc->drm->rc_edge_factor; + data = dsc->rc_tgt_offset_low << 18; + data |= dsc->rc_tgt_offset_high << 14; + data |= dsc->rc_quant_incr_limit1 << 9; + data |= dsc->rc_quant_incr_limit0 << 4; + data |= dsc->rc_edge_factor; DPU_REG_WRITE(c, DSC_RC, data); } static void dpu_hw_dsc_config_thresh(struct dpu_hw_dsc *hw_dsc, - struct msm_display_dsc_config *dsc) + struct drm_dsc_config *dsc) { - struct drm_dsc_rc_range_parameters *rc = dsc->drm->rc_range_params; + struct drm_dsc_rc_range_parameters *rc = dsc->rc_range_params; struct dpu_hw_blk_reg_map *c = &hw_dsc->hw; u32 off; int i; off = DSC_RC_BUF_THRESH; for (i = 0; i < DSC_NUM_BUF_RANGES - 1 ; i++) { - DPU_REG_WRITE(c, off, dsc->drm->rc_buf_thresh[i]); + DPU_REG_WRITE(c, off, dsc->rc_buf_thresh[i]); off += 4; } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h index 45e4118f1fa276ca97dffc20487d33d878efb719..c0b77fe1a69688be591fb9e4aef7624dd1d6fd06 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h @@ -31,7 +31,7 @@ struct dpu_hw_dsc_ops { * @initial_lines: amount of initial lines to be used */ void (*dsc_config)(struct dpu_hw_dsc *hw_dsc, - struct msm_display_dsc_config *dsc, + struct drm_dsc_config *dsc, u32 mode, u32 initial_lines); @@ -41,7 +41,7 @@ struct dpu_hw_dsc_ops { * @dsc: panel dsc parameters */ void (*dsc_config_thresh)(struct dpu_hw_dsc *hw_dsc, - struct msm_display_dsc_config *dsc); + struct drm_dsc_config *dsc); }; struct dpu_hw_dsc { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h index 9f402be55fbf7c349cf1b7cad862ee7b5a127bd7..d3b0ed0a9c6cb811e7871ec45a1f84ab9ac5624a 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h @@ -273,11 +273,9 @@ enum dpu_wd_timer { }; enum dpu_vbif { - VBIF_0, - VBIF_1, + VBIF_RT, + VBIF_NRT, VBIF_MAX, - VBIF_RT = VBIF_0, - VBIF_NRT = VBIF_1 }; /** diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c index 102c21bb4192130c14f63b697627da984405ae97..691c471b08c27db2904d7ed4e78dbbcd7054e8aa 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c @@ -780,8 +780,7 @@ static const struct dpu_sspp_cfg *_sspp_offset(enum dpu_sspp sspp, } struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx, - void __iomem *addr, const struct dpu_mdss_cfg *catalog, - bool is_virtual_pipe) + void __iomem *addr, const struct dpu_mdss_cfg *catalog) { struct dpu_hw_pipe *hw_pipe; const struct dpu_sspp_cfg *cfg; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h index 78b1bc9e004fdbd0333accc149d6806f0c1e93be..0c95b7e64f6c2f0ccf4f01d8a329f27e8c8b5d80 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h @@ -377,11 +377,9 @@ struct dpu_kms; * @idx: Pipe index for which driver object is required * @addr: Mapped register io address of MDP * @catalog : Pointer to mdss catalog data - * @is_virtual_pipe: is this pipe virtual pipe */ struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx, - void __iomem *addr, const struct dpu_mdss_cfg *catalog, - bool is_virtual_pipe); + void __iomem *addr, const struct dpu_mdss_cfg *catalog); /** * dpu_hw_sspp_destroy(): Destroys SSPP driver context diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 008e1420e6e5510b6592261df110bac7480e4495..5e6e2626151e8d174f8ce78facb7cc0e2e0a2361 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -384,12 +384,9 @@ static int dpu_kms_parse_data_bus_icc_path(struct dpu_kms *dpu_kms) struct icc_path *path1; struct drm_device *dev = dpu_kms->dev; struct device *dpu_dev = dev->dev; - struct device *mdss_dev = dpu_dev->parent; - /* Interconnects are a part of MDSS device tree binding, not the - * MDP/DPU device. */ - path0 = of_icc_get(mdss_dev, "mdp0-mem"); - path1 = of_icc_get(mdss_dev, "mdp1-mem"); + path0 = msm_icc_get(dpu_dev, "mdp0-mem"); + path1 = msm_icc_get(dpu_dev, "mdp1-mem"); if (IS_ERR_OR_NULL(path0)) return PTR_ERR_OR_ZERO(path0); @@ -782,7 +779,7 @@ static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms) catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR)); plane = dpu_plane_init(dev, catalog->sspp[i].id, type, - (1UL << max_crtc_count) - 1, 0); + (1UL << max_crtc_count) - 1); if (IS_ERR(plane)) { DPU_ERROR("dpu_plane_init failed\n"); ret = PTR_ERR(plane); @@ -826,12 +823,10 @@ static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms) _dpu_kms_mmu_destroy(dpu_kms); if (dpu_kms->catalog) { - for (i = 0; i < dpu_kms->catalog->vbif_count; i++) { - u32 vbif_idx = dpu_kms->catalog->vbif[i].id; - - if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx]) { - dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]); - dpu_kms->hw_vbif[vbif_idx] = NULL; + for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) { + if (dpu_kms->hw_vbif[i]) { + dpu_hw_vbif_destroy(dpu_kms->hw_vbif[i]); + dpu_kms->hw_vbif[i] = NULL; } } } @@ -902,12 +897,10 @@ static void dpu_kms_mdp_snapshot(struct msm_disp_state *disp_state, struct msm_k int i; struct dpu_kms *dpu_kms; const struct dpu_mdss_cfg *cat; - struct dpu_hw_mdp *top; dpu_kms = to_dpu_kms(kms); cat = dpu_kms->catalog; - top = dpu_kms->hw_mdp; pm_runtime_get_sync(&dpu_kms->pdev->dev); @@ -1113,12 +1106,10 @@ static int dpu_kms_hw_init(struct msm_kms *kms) for (i = 0; i < dpu_kms->catalog->vbif_count; i++) { u32 vbif_idx = dpu_kms->catalog->vbif[i].id; - dpu_kms->hw_vbif[i] = dpu_hw_vbif_init(vbif_idx, + dpu_kms->hw_vbif[vbif_idx] = dpu_hw_vbif_init(vbif_idx, dpu_kms->vbif[vbif_idx], dpu_kms->catalog); - if (IS_ERR_OR_NULL(dpu_kms->hw_vbif[vbif_idx])) { + if (IS_ERR(dpu_kms->hw_vbif[vbif_idx])) { rc = PTR_ERR(dpu_kms->hw_vbif[vbif_idx]); - if (!dpu_kms->hw_vbif[vbif_idx]) - rc = -EINVAL; DPU_ERROR("failed to init vbif %d: %d\n", vbif_idx, rc); dpu_kms->hw_vbif[vbif_idx] = NULL; goto power_error; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c index a617a3d8b1bc7187bf4e599e287d025473c82b48..658005f609f4b3d1e373dd8d3dbaf888ad80e122 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c @@ -91,7 +91,7 @@ enum dpu_plane_qos { /* * struct dpu_plane - local dpu plane structure * @aspace: address space pointer - * @mplane_list: List of multirect planes of the same pipe + * @csc_ptr: Points to dpu_csc_cfg structure to use for current * @catalog: Points to dpu catalog structure * @revalidate: force revalidation of all the plane properties */ @@ -106,8 +106,6 @@ struct dpu_plane { uint32_t color_fill; bool is_error; bool is_rt_pipe; - bool is_virtual; - struct list_head mplane_list; const struct dpu_mdss_cfg *catalog; }; @@ -225,7 +223,7 @@ static void _dpu_plane_calc_clk(struct drm_plane *plane, struct dpu_hw_pipe_cfg static int _dpu_plane_calc_fill_level(struct drm_plane *plane, const struct dpu_format *fmt, u32 src_width) { - struct dpu_plane *pdpu, *tmp; + struct dpu_plane *pdpu; struct dpu_plane_state *pstate; u32 fixed_buff_size; u32 total_fl; @@ -239,19 +237,7 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane, pstate = to_dpu_plane_state(plane->state); fixed_buff_size = pdpu->catalog->caps->pixel_ram_size; - list_for_each_entry(tmp, &pdpu->mplane_list, mplane_list) { - u32 tmp_width; - - if (!tmp->base.state->visible) - continue; - tmp_width = drm_rect_width(&tmp->base.state->src) >> 16; - DPU_DEBUG("plane%d/%d src_width:%d/%d\n", - pdpu->base.base.id, tmp->base.base.id, - src_width, - tmp_width); - src_width = max_t(u32, src_width, - tmp_width); - } + /* FIXME: in multirect case account for the src_width of all the planes */ if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) { if (fmt->chroma_sample == DPU_CHROMA_420) { @@ -854,13 +840,8 @@ int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane) } done: - if (dpu_plane[R0]->is_virtual) { - pstate[R0]->multirect_index = DPU_SSPP_RECT_1; - pstate[R1]->multirect_index = DPU_SSPP_RECT_0; - } else { - pstate[R0]->multirect_index = DPU_SSPP_RECT_0; - pstate[R1]->multirect_index = DPU_SSPP_RECT_1; - } + pstate[R0]->multirect_index = DPU_SSPP_RECT_0; + pstate[R1]->multirect_index = DPU_SSPP_RECT_1; DPU_DEBUG_PLANE(dpu_plane[R0], "R0: %d - %d\n", pstate[R0]->multirect_mode, pstate[R0]->multirect_index); @@ -869,18 +850,6 @@ done: return 0; } -/** - * dpu_plane_get_ctl_flush - get control flush for the given plane - * @plane: Pointer to drm plane structure - * @ctl: Pointer to hardware control driver - * @flush_sspp: Pointer to sspp flush control word - */ -void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl, - u32 *flush_sspp) -{ - *flush_sspp = ctl->ops.get_bitmask_sspp(ctl, dpu_plane_pipe(plane)); -} - static int dpu_plane_prepare_fb(struct drm_plane *plane, struct drm_plane_state *new_state) { @@ -1266,19 +1235,13 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane) static void _dpu_plane_atomic_disable(struct drm_plane *plane) { - struct dpu_plane *pdpu = to_dpu_plane(plane); struct drm_plane_state *state = plane->state; struct dpu_plane_state *pstate = to_dpu_plane_state(state); - trace_dpu_plane_disable(DRMID(plane), is_dpu_plane_virtual(plane), + trace_dpu_plane_disable(DRMID(plane), false, pstate->multirect_mode); pstate->pending = true; - - if (is_dpu_plane_virtual(plane) && - pdpu->pipe_hw && pdpu->pipe_hw->ops.setup_multirect) - pdpu->pipe_hw->ops.setup_multirect(pdpu->pipe_hw, - DPU_SSPP_RECT_SOLO, DPU_SSPP_MULTIRECT_NONE); } static void dpu_plane_atomic_update(struct drm_plane *plane, @@ -1493,22 +1456,16 @@ enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane) return plane ? to_dpu_plane(plane)->pipe : SSPP_NONE; } -bool is_dpu_plane_virtual(struct drm_plane *plane) -{ - return plane ? to_dpu_plane(plane)->is_virtual : false; -} - /* initialize plane */ struct drm_plane *dpu_plane_init(struct drm_device *dev, uint32_t pipe, enum drm_plane_type type, - unsigned long possible_crtcs, u32 master_plane_id) + unsigned long possible_crtcs) { - struct drm_plane *plane = NULL, *master_plane = NULL; + struct drm_plane *plane = NULL; const uint32_t *format_list; struct dpu_plane *pdpu; struct msm_drm_private *priv = dev->dev_private; struct dpu_kms *kms = to_dpu_kms(priv->kms); - int zpos_max = DPU_ZPOS_MAX; uint32_t num_formats; uint32_t supported_rotations; int ret = -EINVAL; @@ -1524,18 +1481,9 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, /* cache local stuff for later */ plane = &pdpu->base; pdpu->pipe = pipe; - pdpu->is_virtual = (master_plane_id != 0); - INIT_LIST_HEAD(&pdpu->mplane_list); - master_plane = drm_plane_find(dev, NULL, master_plane_id); - if (master_plane) { - struct dpu_plane *mpdpu = to_dpu_plane(master_plane); - - list_add_tail(&pdpu->mplane_list, &mpdpu->mplane_list); - } /* initialize underlying h/w driver */ - pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog, - master_plane_id != 0); + pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog); if (IS_ERR(pdpu->pipe_hw)) { DPU_ERROR("[%u]SSPP init failed\n", pipe); ret = PTR_ERR(pdpu->pipe_hw); @@ -1545,14 +1493,8 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, goto clean_sspp; } - if (pdpu->is_virtual) { - format_list = pdpu->pipe_hw->cap->sblk->virt_format_list; - num_formats = pdpu->pipe_hw->cap->sblk->virt_num_formats; - } - else { - format_list = pdpu->pipe_hw->cap->sblk->format_list; - num_formats = pdpu->pipe_hw->cap->sblk->num_formats; - } + format_list = pdpu->pipe_hw->cap->sblk->format_list; + num_formats = pdpu->pipe_hw->cap->sblk->num_formats; ret = drm_universal_plane_init(dev, plane, 0xff, &dpu_plane_funcs, format_list, num_formats, @@ -1562,14 +1504,7 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, pdpu->catalog = kms->catalog; - if (kms->catalog->mixer_count && - kms->catalog->mixer[0].sblk->maxblendstages) { - zpos_max = kms->catalog->mixer[0].sblk->maxblendstages - 1; - if (zpos_max > DPU_STAGE_MAX - DPU_STAGE_0 - 1) - zpos_max = DPU_STAGE_MAX - DPU_STAGE_0 - 1; - } - - ret = drm_plane_create_zpos_property(plane, 0, 0, zpos_max); + ret = drm_plane_create_zpos_property(plane, 0, 0, DPU_ZPOS_MAX); if (ret) DPU_ERROR("failed to install zpos property, rc = %d\n", ret); @@ -1594,15 +1529,14 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev, mutex_init(&pdpu->lock); - DPU_DEBUG("%s created for pipe:%u id:%u virtual:%u\n", plane->name, - pipe, plane->base.id, master_plane_id); + DPU_DEBUG("%s created for pipe:%u id:%u\n", plane->name, + pipe, plane->base.id); return plane; clean_sspp: if (pdpu && pdpu->pipe_hw) dpu_hw_sspp_destroy(pdpu->pipe_hw); clean_plane: - list_del(&pdpu->mplane_list); kfree(pdpu); return ERR_PTR(ret); } diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h index e1463107a6fc9f50118f4eca3e5b9b76ed8d84ab..b7b1b05199c217387971553af48fdecd68e69433 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h @@ -64,23 +64,6 @@ struct dpu_multirect_plane_states { */ enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane); -/** - * is_dpu_plane_virtual - check for virtual plane - * @plane: Pointer to DRM plane object - * returns: true - if the plane is virtual - * false - if the plane is primary - */ -bool is_dpu_plane_virtual(struct drm_plane *plane); - -/** - * dpu_plane_get_ctl_flush - get control flush mask - * @plane: Pointer to DRM plane object - * @ctl: Pointer to control hardware - * @flush_sspp: Pointer to sspp flush control word - */ -void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl, - u32 *flush_sspp); - /** * dpu_plane_flush - final plane operations before commit flush * @plane: Pointer to drm plane structure @@ -99,14 +82,11 @@ void dpu_plane_set_error(struct drm_plane *plane, bool error); * @pipe: dpu hardware pipe identifier * @type: Plane type - PRIMARY/OVERLAY/CURSOR * @possible_crtcs: bitmask of crtc that can be attached to the given pipe - * @master_plane_id: primary plane id of a multirect pipe. 0 value passed for - * a regular plane initialization. A non-zero primary plane - * id will be passed for a virtual pipe initialization. * */ struct drm_plane *dpu_plane_init(struct drm_device *dev, uint32_t pipe, enum drm_plane_type type, - unsigned long possible_crtcs, u32 master_plane_id); + unsigned long possible_crtcs); /** * dpu_plane_validate_multirecti_v2 - validate the multirect planes diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c index 21d20373eb8b325f923430e0af01064b9695646e..1305e250b71eb7f43c81d8a5099a299b702888e6 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c @@ -11,6 +11,26 @@ #include "dpu_hw_vbif.h" #include "dpu_trace.h" +static struct dpu_hw_vbif *dpu_get_vbif(struct dpu_kms *dpu_kms, enum dpu_vbif vbif_idx) +{ + if (vbif_idx < ARRAY_SIZE(dpu_kms->hw_vbif)) + return dpu_kms->hw_vbif[vbif_idx]; + + return NULL; +} + +static const char *dpu_vbif_name(enum dpu_vbif idx) +{ + switch (idx) { + case VBIF_RT: + return "VBIF_RT"; + case VBIF_NRT: + return "VBIF_NRT"; + default: + return "??"; + } +} + /** * _dpu_vbif_wait_for_xin_halt - wait for the xin to halt * @vbif: Pointer to hardware vbif driver @@ -42,12 +62,12 @@ static int _dpu_vbif_wait_for_xin_halt(struct dpu_hw_vbif *vbif, u32 xin_id) if (!status) { rc = -ETIMEDOUT; - DPU_ERROR("VBIF %d client %d not halting. TIMEDOUT.\n", - vbif->idx - VBIF_0, xin_id); + DPU_ERROR("%s client %d not halting. TIMEDOUT.\n", + dpu_vbif_name(vbif->idx), xin_id); } else { rc = 0; - DRM_DEBUG_ATOMIC("VBIF %d client %d is halted\n", - vbif->idx - VBIF_0, xin_id); + DRM_DEBUG_ATOMIC("%s client %d is halted\n", + dpu_vbif_name(vbif->idx), xin_id); } return rc; @@ -87,8 +107,8 @@ static void _dpu_vbif_apply_dynamic_ot_limit(struct dpu_hw_vbif *vbif, } } - DRM_DEBUG_ATOMIC("vbif:%d xin:%d w:%d h:%d fps:%d pps:%llu ot:%u\n", - vbif->idx - VBIF_0, params->xin_id, + DRM_DEBUG_ATOMIC("%s xin:%d w:%d h:%d fps:%d pps:%llu ot:%u\n", + dpu_vbif_name(vbif->idx), params->xin_id, params->width, params->height, params->frame_rate, pps, *ot_lim); } @@ -133,8 +153,8 @@ static u32 _dpu_vbif_get_ot_limit(struct dpu_hw_vbif *vbif, } exit: - DRM_DEBUG_ATOMIC("vbif:%d xin:%d ot_lim:%d\n", - vbif->idx - VBIF_0, params->xin_id, ot_lim); + DRM_DEBUG_ATOMIC("%s xin:%d ot_lim:%d\n", + dpu_vbif_name(vbif->idx), params->xin_id, ot_lim); return ot_lim; } @@ -148,20 +168,15 @@ exit: void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms, struct dpu_vbif_set_ot_params *params) { - struct dpu_hw_vbif *vbif = NULL; + struct dpu_hw_vbif *vbif; struct dpu_hw_mdp *mdp; bool forced_on = false; u32 ot_lim; - int ret, i; + int ret; mdp = dpu_kms->hw_mdp; - for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) { - if (dpu_kms->hw_vbif[i] && - dpu_kms->hw_vbif[i]->idx == params->vbif_idx) - vbif = dpu_kms->hw_vbif[i]; - } - + vbif = dpu_get_vbif(dpu_kms, params->vbif_idx); if (!vbif || !mdp) { DRM_DEBUG_ATOMIC("invalid arguments vbif %d mdp %d\n", vbif != NULL, mdp != NULL); @@ -204,7 +219,7 @@ void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms, void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms, struct dpu_vbif_set_qos_params *params) { - struct dpu_hw_vbif *vbif = NULL; + struct dpu_hw_vbif *vbif; struct dpu_hw_mdp *mdp; bool forced_on = false; const struct dpu_vbif_qos_tbl *qos_tbl; @@ -216,13 +231,7 @@ void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms, } mdp = dpu_kms->hw_mdp; - for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) { - if (dpu_kms->hw_vbif[i] && - dpu_kms->hw_vbif[i]->idx == params->vbif_idx) { - vbif = dpu_kms->hw_vbif[i]; - break; - } - } + vbif = dpu_get_vbif(dpu_kms, params->vbif_idx); if (!vbif || !vbif->cap) { DPU_ERROR("invalid vbif %d\n", params->vbif_idx); @@ -245,8 +254,8 @@ void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms, forced_on = mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, true); for (i = 0; i < qos_tbl->npriority_lvl; i++) { - DRM_DEBUG_ATOMIC("vbif:%d xin:%d lvl:%d/%d\n", - params->vbif_idx, params->xin_id, i, + DRM_DEBUG_ATOMIC("%s xin:%d lvl:%d/%d\n", + dpu_vbif_name(params->vbif_idx), params->xin_id, i, qos_tbl->priority_lvl[i]); vbif->ops.set_qos_remap(vbif, params->xin_id, i, qos_tbl->priority_lvl[i]); @@ -266,8 +275,8 @@ void dpu_vbif_clear_errors(struct dpu_kms *dpu_kms) if (vbif && vbif->ops.clear_errors) { vbif->ops.clear_errors(vbif, &pnd, &src); if (pnd || src) { - DRM_DEBUG_KMS("VBIF %d: pnd 0x%X, src 0x%X\n", - vbif->idx - VBIF_0, pnd, src); + DRM_DEBUG_KMS("%s: pnd 0x%X, src 0x%X\n", + dpu_vbif_name(vbif->idx), pnd, src); } } } diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index d2a48caf9d27ee87c05668ac405bff9bab0d94dd..b0d21838a13439027f1df3c3fd23b21c857eb1e5 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -902,12 +902,9 @@ fail: static int mdp5_setup_interconnect(struct platform_device *pdev) { - /* Interconnects are a part of MDSS device tree binding, not the - * MDP5 device. */ - struct device *mdss_dev = pdev->dev.parent; - struct icc_path *path0 = of_icc_get(mdss_dev, "mdp0-mem"); - struct icc_path *path1 = of_icc_get(mdss_dev, "mdp1-mem"); - struct icc_path *path_rot = of_icc_get(mdss_dev, "rotator-mem"); + struct icc_path *path0 = msm_icc_get(&pdev->dev, "mdp0-mem"); + struct icc_path *path1 = msm_icc_get(&pdev->dev, "mdp1-mem"); + struct icc_path *path_rot = msm_icc_get(&pdev->dev, "rotator-mem"); if (IS_ERR(path0)) return PTR_ERR(path0); diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c index 7257515871a9faa2c9301813a64d3e4e1203016e..676279d0ca8d9e04d7b5b5f1fbedae6f18242843 100644 --- a/drivers/gpu/drm/msm/dp/dp_catalog.c +++ b/drivers/gpu/drm/msm/dp/dp_catalog.c @@ -431,7 +431,7 @@ void dp_catalog_ctrl_config_msa(struct dp_catalog *dp_catalog, if (rate == link_rate_hbr3) pixel_div = 6; - else if (rate == 1620000 || rate == 270000) + else if (rate == 162000 || rate == 270000) pixel_div = 2; else if (rate == link_rate_hbr2) pixel_div = 4; diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index ab6aa13b1639a0776a1a7899e458ac773899fbc0..3854c9f1f7e9020288d350f3b37311a748a1e341 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -1214,7 +1214,7 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, if (ret) return ret; - dp_ctrl_train_pattern_set(ctrl, pattern | DP_RECOVERED_CLOCK_OUT_EN); + dp_ctrl_train_pattern_set(ctrl, pattern); for (tries = 0; tries <= maximum_retries; tries++) { drm_dp_link_train_channel_eq_delay(ctrl->aux, ctrl->panel->dpcd); @@ -1238,8 +1238,6 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl, return -ETIMEDOUT; } -static int dp_ctrl_reinitialize_mainlink(struct dp_ctrl_private *ctrl); - static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, int *training_step) { @@ -1358,25 +1356,7 @@ static int dp_ctrl_enable_mainlink_clocks(struct dp_ctrl_private *ctrl) if (ret) DRM_ERROR("Unable to start link clocks. ret=%d\n", ret); - drm_dbg_dp(ctrl->drm_dev, "link rate=%d pixel_clk=%d\n", - ctrl->link->link_params.rate, ctrl->dp_ctrl.pixel_rate); - - return ret; -} - -static int dp_ctrl_enable_stream_clocks(struct dp_ctrl_private *ctrl) -{ - int ret = 0; - - dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel", - ctrl->dp_ctrl.pixel_rate * 1000); - - ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true); - if (ret) - DRM_ERROR("Unabled to start pixel clocks. ret=%d\n", ret); - - drm_dbg_dp(ctrl->drm_dev, "link rate=%d pixel_clk=%d\n", - ctrl->link->link_params.rate, ctrl->dp_ctrl.pixel_rate); + drm_dbg_dp(ctrl->drm_dev, "link rate=%d\n", ctrl->link->link_params.rate); return ret; } @@ -1520,8 +1500,6 @@ static int dp_ctrl_link_maintenance(struct dp_ctrl_private *ctrl) ctrl->link->phy_params.p_level = 0; ctrl->link->phy_params.v_level = 0; - ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; - ret = dp_ctrl_setup_main_link(ctrl, &training_step); if (ret) goto end; @@ -1535,38 +1513,6 @@ end: return ret; } -static int dp_ctrl_on_stream_phy_test_report(struct dp_ctrl *dp_ctrl); - -static int dp_ctrl_process_phy_test_request(struct dp_ctrl_private *ctrl) -{ - int ret = 0; - - if (!ctrl->link->phy_params.phy_test_pattern_sel) { - drm_dbg_dp(ctrl->drm_dev, - "no test pattern selected by sink\n"); - return ret; - } - - /* - * The global reset will need DP link related clocks to be - * running. Add the global reset just before disabling the - * link clocks and core clocks. - */ - ret = dp_ctrl_off(&ctrl->dp_ctrl); - if (ret) { - DRM_ERROR("failed to disable DP controller\n"); - return ret; - } - - ret = dp_ctrl_on_link(&ctrl->dp_ctrl); - if (!ret) - ret = dp_ctrl_on_stream_phy_test_report(&ctrl->dp_ctrl); - else - DRM_ERROR("failed to enable DP link controller\n"); - - return ret; -} - static bool dp_ctrl_send_phy_test_pattern(struct dp_ctrl_private *ctrl) { bool success = false; @@ -1619,6 +1565,48 @@ static bool dp_ctrl_send_phy_test_pattern(struct dp_ctrl_private *ctrl) return success; } +static int dp_ctrl_process_phy_test_request(struct dp_ctrl_private *ctrl) +{ + int ret; + unsigned long pixel_rate; + + if (!ctrl->link->phy_params.phy_test_pattern_sel) { + drm_dbg_dp(ctrl->drm_dev, + "no test pattern selected by sink\n"); + return 0; + } + + /* + * The global reset will need DP link related clocks to be + * running. Add the global reset just before disabling the + * link clocks and core clocks. + */ + ret = dp_ctrl_off(&ctrl->dp_ctrl); + if (ret) { + DRM_ERROR("failed to disable DP controller\n"); + return ret; + } + + ret = dp_ctrl_on_link(&ctrl->dp_ctrl); + if (ret) { + DRM_ERROR("failed to enable DP link controller\n"); + return ret; + } + + pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; + dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel", pixel_rate * 1000); + + ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true); + if (ret) { + DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); + return ret; + } + + dp_ctrl_send_phy_test_pattern(ctrl); + + return 0; +} + void dp_ctrl_handle_sink_request(struct dp_ctrl *dp_ctrl) { struct dp_ctrl_private *ctrl; @@ -1689,11 +1677,12 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) { int rc = 0; struct dp_ctrl_private *ctrl; - u32 rate = 0; + u32 rate; int link_train_max_retries = 5; u32 const phy_cts_pixel_clk_khz = 148500; u8 link_status[DP_LINK_STATUS_SIZE]; unsigned int training_step; + unsigned long pixel_rate; if (!dp_ctrl) return -EINVAL; @@ -1701,25 +1690,24 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl) ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl); rate = ctrl->panel->link_info.rate; + pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; dp_power_clk_enable(ctrl->power, DP_CORE_PM, true); if (ctrl->link->sink_request & DP_TEST_LINK_PHY_TEST_PATTERN) { drm_dbg_dp(ctrl->drm_dev, "using phy test link parameters\n"); - if (!ctrl->panel->dp_mode.drm_mode.clock) - ctrl->dp_ctrl.pixel_rate = phy_cts_pixel_clk_khz; + if (!pixel_rate) + pixel_rate = phy_cts_pixel_clk_khz; } else { ctrl->link->link_params.rate = rate; ctrl->link->link_params.num_lanes = ctrl->panel->link_info.num_lanes; - ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; } - drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%d\n", + drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%lu\n", ctrl->link->link_params.rate, ctrl->link->link_params.num_lanes, - ctrl->dp_ctrl.pixel_rate); - + pixel_rate); rc = dp_ctrl_enable_mainlink_clocks(ctrl); if (rc) @@ -1816,31 +1804,12 @@ static int dp_ctrl_link_retrain(struct dp_ctrl_private *ctrl) return dp_ctrl_setup_main_link(ctrl, &training_step); } -static int dp_ctrl_on_stream_phy_test_report(struct dp_ctrl *dp_ctrl) -{ - int ret; - struct dp_ctrl_private *ctrl; - - ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl); - - ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; - - ret = dp_ctrl_enable_stream_clocks(ctrl); - if (ret) { - DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); - return ret; - } - - dp_ctrl_send_phy_test_pattern(ctrl); - - return 0; -} - int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train) { int ret = 0; bool mainlink_ready = false; struct dp_ctrl_private *ctrl; + unsigned long pixel_rate; unsigned long pixel_rate_orig; if (!dp_ctrl) @@ -1848,15 +1817,14 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train) ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl); - ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock; + pixel_rate = pixel_rate_orig = ctrl->panel->dp_mode.drm_mode.clock; - pixel_rate_orig = ctrl->dp_ctrl.pixel_rate; if (dp_ctrl->wide_bus_en) - ctrl->dp_ctrl.pixel_rate >>= 1; + pixel_rate >>= 1; - drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%d\n", + drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%lu\n", ctrl->link->link_params.rate, - ctrl->link->link_params.num_lanes, ctrl->dp_ctrl.pixel_rate); + ctrl->link->link_params.num_lanes, pixel_rate); if (!dp_power_clk_status(ctrl->power, DP_CTRL_PM)) { /* link clk is off */ ret = dp_ctrl_enable_mainlink_clocks(ctrl); @@ -1866,9 +1834,11 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train) } } - ret = dp_ctrl_enable_stream_clocks(ctrl); + dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel", pixel_rate * 1000); + + ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true); if (ret) { - DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret); + DRM_ERROR("Unable to start pixel clocks. ret=%d\n", ret); goto end; } diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h index b563e2e3bfe5d225ef23c392365359e05f50888f..9f29734af81ca88dc141bab21d620847659f94e6 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.h +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h @@ -16,7 +16,6 @@ struct dp_ctrl { bool orientation; atomic_t aborted; - u32 pixel_rate; bool wide_bus_en; }; diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c index 36f0af02749fa49bb5e30474fea7272cffb9b992..36bb6191d2f030fb11322dedad1d9ed10b116170 100644 --- a/drivers/gpu/drm/msm/dp/dp_link.c +++ b/drivers/gpu/drm/msm/dp/dp_link.c @@ -786,7 +786,7 @@ static int dp_link_process_link_training_request(struct dp_link_private *link) link->request.test_lane_count); link->dp_link.link_params.num_lanes = link->request.test_lane_count; - link->dp_link.link_params.rate = + link->dp_link.link_params.rate = drm_dp_bw_code_to_link_rate(link->request.test_link_rate); return 0; @@ -965,8 +965,7 @@ static int dp_link_process_link_status_update(struct dp_link_private *link) if (channel_eq_done && clock_recovery_done) return -EINVAL; - - return 0; + return 0; } /** diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c index 1625328fa43024e66fa1f67a90a747e4d5ce5fbb..39bbabb5daf6f2290c39c7974adc2e318f32ffb0 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.c +++ b/drivers/gpu/drm/msm/dsi/dsi.c @@ -6,14 +6,6 @@ #include "dsi.h" #include "dsi_cfg.h" -struct drm_encoder *msm_dsi_get_encoder(struct msm_dsi *msm_dsi) -{ - if (!msm_dsi || !msm_dsi_device_connected(msm_dsi)) - return NULL; - - return msm_dsi->encoder; -} - bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi) { unsigned long host_flags = msm_dsi_host_get_mode_flags(msm_dsi->host); @@ -21,7 +13,7 @@ bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi) return !(host_flags & MIPI_DSI_MODE_VIDEO); } -struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi) +struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi) { return msm_dsi_host_get_dsc_config(msm_dsi->host); } @@ -220,7 +212,6 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev, struct drm_encoder *encoder) { struct msm_drm_private *priv; - struct drm_bridge *ext_bridge; int ret; if (WARN_ON(!encoder) || WARN_ON(!msm_dsi) || WARN_ON(!dev)) @@ -254,26 +245,10 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev, goto fail; } - /* - * check if the dsi encoder output is connected to a panel or an - * external bridge. We create a connector only if we're connected to a - * drm_panel device. When we're connected to an external bridge, we - * assume that the drm_bridge driver will create the connector itself. - */ - ext_bridge = msm_dsi_host_get_bridge(msm_dsi->host); - - if (ext_bridge) - msm_dsi->connector = - msm_dsi_manager_ext_bridge_init(msm_dsi->id); - else - msm_dsi->connector = - msm_dsi_manager_connector_init(msm_dsi->id); - - if (IS_ERR(msm_dsi->connector)) { - ret = PTR_ERR(msm_dsi->connector); + ret = msm_dsi_manager_ext_bridge_init(msm_dsi->id); + if (ret) { DRM_DEV_ERROR(dev->dev, "failed to create dsi connector: %d\n", ret); - msm_dsi->connector = NULL; goto fail; } @@ -287,12 +262,6 @@ fail: msm_dsi->bridge = NULL; } - /* don't destroy connector if we didn't make it */ - if (msm_dsi->connector && !msm_dsi->external_bridge) - msm_dsi->connector->funcs->destroy(msm_dsi->connector); - - msm_dsi->connector = NULL; - return ret; } diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h index 580a1e6358bfca8a90f5d7c05ab2aa42e503c118..2a96b4fe7839fd1b6d18b2473d326d0d89a67a00 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.h +++ b/drivers/gpu/drm/msm/dsi/dsi.h @@ -12,7 +12,6 @@ #include #include #include -#include #include "msm_drv.h" #include "disp/msm_disp_snapshot.h" @@ -30,27 +29,12 @@ enum msm_dsi_phy_usecase { MSM_DSI_PHY_SLAVE, }; -#define DSI_DEV_REGULATOR_MAX 8 #define DSI_BUS_CLK_MAX 4 -/* Regulators for DSI devices */ -struct dsi_reg_entry { - char name[32]; - int enable_load; - int disable_load; -}; - -struct dsi_reg_config { - int num; - struct dsi_reg_entry regs[DSI_DEV_REGULATOR_MAX]; -}; - struct msm_dsi { struct drm_device *dev; struct platform_device *pdev; - /* connector managed by us when we're connected to a drm_panel */ - struct drm_connector *connector; /* internal dsi bridge attached to MDP interface */ struct drm_bridge *bridge; @@ -58,10 +42,8 @@ struct msm_dsi { struct msm_dsi_phy *phy; /* - * panel/external_bridge connected to dsi bridge output, only one of the - * two can be valid at a time + * external_bridge connected to dsi bridge output */ - struct drm_panel *panel; struct drm_bridge *external_bridge; struct device *phy_dev; @@ -76,8 +58,7 @@ struct msm_dsi { /* dsi manager */ struct drm_bridge *msm_dsi_manager_bridge_init(u8 id); void msm_dsi_manager_bridge_destroy(struct drm_bridge *bridge); -struct drm_connector *msm_dsi_manager_connector_init(u8 id); -struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id); +int msm_dsi_manager_ext_bridge_init(u8 id); int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg); bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len); int msm_dsi_manager_register(struct msm_dsi *msm_dsi); @@ -87,11 +68,9 @@ void msm_dsi_manager_tpg_enable(void); /* msm dsi */ static inline bool msm_dsi_device_connected(struct msm_dsi *msm_dsi) { - return msm_dsi->panel || msm_dsi->external_bridge; + return msm_dsi->external_bridge; } -struct drm_encoder *msm_dsi_get_encoder(struct msm_dsi *msm_dsi); - /* dsi host */ struct msm_dsi_host; int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host, @@ -116,9 +95,7 @@ int msm_dsi_host_set_display_mode(struct mipi_dsi_host *host, const struct drm_display_mode *mode); enum drm_mode_status msm_dsi_host_check_dsc(struct mipi_dsi_host *host, const struct drm_display_mode *mode); -struct drm_panel *msm_dsi_host_get_panel(struct mipi_dsi_host *host); unsigned long msm_dsi_host_get_mode_flags(struct mipi_dsi_host *host); -struct drm_bridge *msm_dsi_host_get_bridge(struct mipi_dsi_host *host); int msm_dsi_host_register(struct mipi_dsi_host *host); void msm_dsi_host_unregister(struct mipi_dsi_host *host); void msm_dsi_host_set_phy_mode(struct mipi_dsi_host *host, @@ -154,7 +131,7 @@ int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_bonded_dsi); int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi); void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_host *host); void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host); -struct msm_display_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host); +struct drm_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host); /* dsi phy */ struct msm_dsi_phy; diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c index 2c23324a2296be86ea6691c1b92bf0dfb476db05..7e97c239ed489b71e661350d43ba868ebf68e548 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c @@ -9,16 +9,16 @@ static const char * const dsi_v2_bus_clk_names[] = { "core_mmss", "iface", "bus", }; +static const struct regulator_bulk_data apq8064_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 100000 }, /* 1.2 V */ + { .supply = "avdd", .init_load_uA = 10000 }, /* 3.0 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + static const struct msm_dsi_config apq8064_dsi_cfg = { .io_offset = 0, - .reg_cfg = { - .num = 3, - .regs = { - {"vdda", 100000, 100}, /* 1.2 V */ - {"avdd", 10000, 100}, /* 3.0 V */ - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = apq8064_dsi_regulators, + .num_regulators = ARRAY_SIZE(apq8064_dsi_regulators), .bus_clk_names = dsi_v2_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_v2_bus_clk_names), .io_start = { 0x4700000, 0x5800000 }, @@ -29,16 +29,16 @@ static const char * const dsi_6g_bus_clk_names[] = { "mdp_core", "iface", "bus", "core_mmss", }; +static const struct regulator_bulk_data msm8974_apq8084_regulators[] = { + { .supply = "vdd", .init_load_uA = 150000 }, /* 3.0 V */ + { .supply = "vdda", .init_load_uA = 100000 }, /* 1.2 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 3, - .regs = { - {"vdd", 150000, 100}, /* 3.0 V */ - {"vdda", 100000, 100}, /* 1.2 V */ - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = msm8974_apq8084_regulators, + .num_regulators = ARRAY_SIZE(msm8974_apq8084_regulators), .bus_clk_names = dsi_6g_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names), .io_start = { 0xfd922800, 0xfd922b00 }, @@ -49,15 +49,15 @@ static const char * const dsi_8916_bus_clk_names[] = { "mdp_core", "iface", "bus", }; +static const struct regulator_bulk_data msm8916_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 100000 }, /* 1.2 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + static const struct msm_dsi_config msm8916_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 2, - .regs = { - {"vdda", 100000, 100}, /* 1.2 V */ - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = msm8916_dsi_regulators, + .num_regulators = ARRAY_SIZE(msm8916_dsi_regulators), .bus_clk_names = dsi_8916_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_8916_bus_clk_names), .io_start = { 0x1a98000 }, @@ -68,34 +68,34 @@ static const char * const dsi_8976_bus_clk_names[] = { "mdp_core", "iface", "bus", }; +static const struct regulator_bulk_data msm8976_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 100000 }, /* 1.2 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + static const struct msm_dsi_config msm8976_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 2, - .regs = { - {"vdda", 100000, 100}, /* 1.2 V */ - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = msm8976_dsi_regulators, + .num_regulators = ARRAY_SIZE(msm8976_dsi_regulators), .bus_clk_names = dsi_8976_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_8976_bus_clk_names), .io_start = { 0x1a94000, 0x1a96000 }, .num_dsi = 2, }; +static const struct regulator_bulk_data msm8994_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 100000 }, /* 1.25 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ + { .supply = "vcca", .init_load_uA = 10000 }, /* 1.0 V */ + { .supply = "vdd", .init_load_uA = 100000 }, /* 1.8 V */ + { .supply = "lab_reg", .init_load_uA = -1 }, + { .supply = "ibb_reg", .init_load_uA = -1 }, +}; + static const struct msm_dsi_config msm8994_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 6, - .regs = { - {"vdda", 100000, 100}, /* 1.25 V */ - {"vddio", 100000, 100}, /* 1.8 V */ - {"vcca", 10000, 100}, /* 1.0 V */ - {"vdd", 100000, 100}, /* 1.8 V */ - {"lab_reg", -1, -1}, - {"ibb_reg", -1, -1}, - }, - }, + .regulator_data = msm8994_dsi_regulators, + .num_regulators = ARRAY_SIZE(msm8994_dsi_regulators), .bus_clk_names = dsi_6g_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names), .io_start = { 0xfd998000, 0xfd9a0000 }, @@ -106,16 +106,16 @@ static const char * const dsi_8996_bus_clk_names[] = { "mdp_core", "iface", "bus", "core_mmss", }; +static const struct regulator_bulk_data msm8996_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 18160 }, /* 1.25 V */ + { .supply = "vcca", .init_load_uA = 17000 }, /* 0.925 V */ + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + static const struct msm_dsi_config msm8996_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 2, - .regs = { - {"vdda", 18160, 1 }, /* 1.25 V */ - {"vcca", 17000, 32 }, /* 0.925 V */ - {"vddio", 100000, 100 },/* 1.8 V */ - }, - }, + .regulator_data = msm8996_dsi_regulators, + .num_regulators = ARRAY_SIZE(msm8996_dsi_regulators), .bus_clk_names = dsi_8996_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_8996_bus_clk_names), .io_start = { 0x994000, 0x996000 }, @@ -126,15 +126,15 @@ static const char * const dsi_msm8998_bus_clk_names[] = { "iface", "bus", "core", }; +static const struct regulator_bulk_data msm8998_dsi_regulators[] = { + { .supply = "vdd", .init_load_uA = 367000 }, /* 0.9 V */ + { .supply = "vdda", .init_load_uA = 62800 }, /* 1.2 V */ +}; + static const struct msm_dsi_config msm8998_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 2, - .regs = { - {"vdd", 367000, 16 }, /* 0.9 V */ - {"vdda", 62800, 2 }, /* 1.2 V */ - }, - }, + .regulator_data = msm8998_dsi_regulators, + .num_regulators = ARRAY_SIZE(msm8998_dsi_regulators), .bus_clk_names = dsi_msm8998_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_msm8998_bus_clk_names), .io_start = { 0xc994000, 0xc996000 }, @@ -145,14 +145,14 @@ static const char * const dsi_sdm660_bus_clk_names[] = { "iface", "bus", "core", "core_mmss", }; +static const struct regulator_bulk_data sdm660_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 12560 }, /* 1.2 V */ +}; + static const struct msm_dsi_config sdm660_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 2, - .regs = { - {"vdda", 12560, 4 }, /* 1.2 V */ - }, - }, + .regulator_data = sdm660_dsi_regulators, + .num_regulators = ARRAY_SIZE(sdm660_dsi_regulators), .bus_clk_names = dsi_sdm660_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_sdm660_bus_clk_names), .io_start = { 0xc994000, 0xc996000 }, @@ -167,28 +167,28 @@ static const char * const dsi_sc7180_bus_clk_names[] = { "iface", "bus", }; +static const struct regulator_bulk_data sdm845_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 21800 }, /* 1.2 V */ +}; + static const struct msm_dsi_config sdm845_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 1, - .regs = { - {"vdda", 21800, 4 }, /* 1.2 V */ - }, - }, + .regulator_data = sdm845_dsi_regulators, + .num_regulators = ARRAY_SIZE(sdm845_dsi_regulators), .bus_clk_names = dsi_sdm845_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_sdm845_bus_clk_names), .io_start = { 0xae94000, 0xae96000 }, .num_dsi = 2, }; +static const struct regulator_bulk_data sc7180_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 21800 }, /* 1.2 V */ +}; + static const struct msm_dsi_config sc7180_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 1, - .regs = { - {"vdda", 21800, 4 }, /* 1.2 V */ - }, - }, + .regulator_data = sc7180_dsi_regulators, + .num_regulators = ARRAY_SIZE(sc7180_dsi_regulators), .bus_clk_names = dsi_sc7180_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_sc7180_bus_clk_names), .io_start = { 0xae94000 }, @@ -199,14 +199,14 @@ static const char * const dsi_sc7280_bus_clk_names[] = { "iface", "bus", }; +static const struct regulator_bulk_data sc7280_dsi_regulators[] = { + { .supply = "vdda", .init_load_uA = 8350 }, /* 1.2 V */ +}; + static const struct msm_dsi_config sc7280_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 1, - .regs = { - {"vdda", 8350, 0 }, /* 1.2 V */ - }, - }, + .regulator_data = sc7280_dsi_regulators, + .num_regulators = ARRAY_SIZE(sc7280_dsi_regulators), .bus_clk_names = dsi_sc7280_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_sc7280_bus_clk_names), .io_start = { 0xae94000 }, @@ -217,14 +217,14 @@ static const char * const dsi_qcm2290_bus_clk_names[] = { "iface", "bus", }; +static const struct regulator_bulk_data qcm2290_dsi_cfg_regulators[] = { + { .supply = "vdda", .init_load_uA = 21800 }, /* 1.2 V */ +}; + static const struct msm_dsi_config qcm2290_dsi_cfg = { .io_offset = DSI_6G_REG_SHIFT, - .reg_cfg = { - .num = 1, - .regs = { - {"vdda", 21800, 4 }, /* 1.2 V */ - }, - }, + .regulator_data = qcm2290_dsi_cfg_regulators, + .num_regulators = ARRAY_SIZE(qcm2290_dsi_cfg_regulators), .bus_clk_names = dsi_qcm2290_bus_clk_names, .num_bus_clks = ARRAY_SIZE(dsi_qcm2290_bus_clk_names), .io_start = { 0x5e94000 }, diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h index fe54a999968bd53feac8f8f13a6cadce67ff26ee..8f04e685a74e9ab81abaebfc4341cf44bb5701f2 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h @@ -32,7 +32,8 @@ struct msm_dsi_config { u32 io_offset; - struct dsi_reg_config reg_cfg; + const struct regulator_bulk_data *regulator_data; + int num_regulators; const char * const *bus_clk_names; const int num_bus_clks; const resource_size_t io_start[DSI_MAX]; diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index a34078497af12b1b8e5ac17cb0a0e4b708219e16..7fbf391c024f8e9d69bd0952deba7a584feb7c47 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -33,7 +33,7 @@ #define DSI_RESET_TOGGLE_DELAY_MS 20 -static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc); +static int dsi_populate_dsc_params(struct drm_dsc_config *dsc); static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor) { @@ -108,7 +108,7 @@ struct msm_dsi_host { void __iomem *ctrl_base; phys_addr_t ctrl_size; - struct regulator_bulk_data supplies[DSI_DEV_REGULATOR_MAX]; + struct regulator_bulk_data *supplies; int num_bus_clks; struct clk_bulk_data bus_clks[DSI_BUS_CLK_MAX]; @@ -144,7 +144,6 @@ struct msm_dsi_host { u32 err_work_state; struct work_struct err_work; - struct work_struct hpd_work; struct workqueue_struct *workqueue; /* DSI 6G TX buffer*/ @@ -161,10 +160,9 @@ struct msm_dsi_host { struct regmap *sfpb; struct drm_display_mode *mode; - struct msm_display_dsc_config *dsc; + struct drm_dsc_config *dsc; /* connected device info */ - struct device_node *device_node; unsigned int channel; unsigned int lanes; enum mipi_dsi_pixel_format format; @@ -205,9 +203,6 @@ static inline void dsi_write(struct msm_dsi_host *msm_host, u32 reg, u32 data) msm_writel(data, msm_host->ctrl_base + reg); } -static int dsi_host_regulator_enable(struct msm_dsi_host *msm_host); -static void dsi_host_regulator_disable(struct msm_dsi_host *msm_host); - static const struct msm_dsi_cfg_handler *dsi_get_config( struct msm_dsi_host *msm_host) { @@ -258,76 +253,6 @@ static inline struct msm_dsi_host *to_msm_dsi_host(struct mipi_dsi_host *host) return container_of(host, struct msm_dsi_host, base); } -static void dsi_host_regulator_disable(struct msm_dsi_host *msm_host) -{ - struct regulator_bulk_data *s = msm_host->supplies; - const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs; - int num = msm_host->cfg_hnd->cfg->reg_cfg.num; - int i; - - DBG(""); - for (i = num - 1; i >= 0; i--) - if (regs[i].disable_load >= 0) - regulator_set_load(s[i].consumer, - regs[i].disable_load); - - regulator_bulk_disable(num, s); -} - -static int dsi_host_regulator_enable(struct msm_dsi_host *msm_host) -{ - struct regulator_bulk_data *s = msm_host->supplies; - const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs; - int num = msm_host->cfg_hnd->cfg->reg_cfg.num; - int ret, i; - - DBG(""); - for (i = 0; i < num; i++) { - if (regs[i].enable_load >= 0) { - ret = regulator_set_load(s[i].consumer, - regs[i].enable_load); - if (ret < 0) { - pr_err("regulator %d set op mode failed, %d\n", - i, ret); - goto fail; - } - } - } - - ret = regulator_bulk_enable(num, s); - if (ret < 0) { - pr_err("regulator enable failed, %d\n", ret); - goto fail; - } - - return 0; - -fail: - for (i--; i >= 0; i--) - regulator_set_load(s[i].consumer, regs[i].disable_load); - return ret; -} - -static int dsi_regulator_init(struct msm_dsi_host *msm_host) -{ - struct regulator_bulk_data *s = msm_host->supplies; - const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs; - int num = msm_host->cfg_hnd->cfg->reg_cfg.num; - int i, ret; - - for (i = 0; i < num; i++) - s[i].supply = regs[i].name; - - ret = devm_regulator_bulk_get(&msm_host->pdev->dev, num, s); - if (ret < 0) { - pr_err("%s: failed to init regulator, ret=%d\n", - __func__, ret); - return ret; - } - - return 0; -} - int dsi_clk_init_v2(struct msm_dsi_host *msm_host) { struct platform_device *pdev = msm_host->pdev; @@ -916,7 +841,7 @@ static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable, static void dsi_update_dsc_timing(struct msm_dsi_host *msm_host, bool is_cmd_mode, u32 hdisplay) { - struct msm_display_dsc_config *dsc = msm_host->dsc; + struct drm_dsc_config *dsc = msm_host->dsc; u32 reg, intf_width, reg_ctrl, reg_ctrl2; u32 slice_per_intf, total_bytes_per_intf; u32 pkt_per_line; @@ -927,24 +852,24 @@ static void dsi_update_dsc_timing(struct msm_dsi_host *msm_host, bool is_cmd_mod * compress mode registers */ intf_width = hdisplay; - slice_per_intf = DIV_ROUND_UP(intf_width, dsc->drm->slice_width); + slice_per_intf = DIV_ROUND_UP(intf_width, dsc->slice_width); /* If slice_per_pkt is greater than slice_per_intf * then default to 1. This can happen during partial * update. */ - if (slice_per_intf > dsc->drm->slice_count) - dsc->drm->slice_count = 1; + if (slice_per_intf > dsc->slice_count) + dsc->slice_count = 1; - slice_per_intf = DIV_ROUND_UP(hdisplay, dsc->drm->slice_width); - bytes_in_slice = DIV_ROUND_UP(dsc->drm->slice_width * dsc->drm->bits_per_pixel, 8); + slice_per_intf = DIV_ROUND_UP(hdisplay, dsc->slice_width); + bytes_in_slice = DIV_ROUND_UP(dsc->slice_width * dsc->bits_per_pixel, 8); - dsc->drm->slice_chunk_size = bytes_in_slice; + dsc->slice_chunk_size = bytes_in_slice; total_bytes_per_intf = bytes_in_slice * slice_per_intf; eol_byte_num = total_bytes_per_intf % 3; - pkt_per_line = slice_per_intf / dsc->drm->slice_count; + pkt_per_line = slice_per_intf / dsc->slice_count; if (is_cmd_mode) /* packet data type */ reg = DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE(MIPI_DSI_DCS_LONG_WRITE); @@ -1009,7 +934,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) } if (msm_host->dsc) { - struct msm_display_dsc_config *dsc = msm_host->dsc; + struct drm_dsc_config *dsc = msm_host->dsc; /* update dsc params with timing params */ if (!dsc || !mode->hdisplay || !mode->vdisplay) { @@ -1018,9 +943,9 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) return; } - dsc->drm->pic_width = mode->hdisplay; - dsc->drm->pic_height = mode->vdisplay; - DBG("Mode %dx%d\n", dsc->drm->pic_width, dsc->drm->pic_height); + dsc->pic_width = mode->hdisplay; + dsc->pic_height = mode->vdisplay; + DBG("Mode %dx%d\n", dsc->pic_width, dsc->pic_height); /* we do the calculations for dsc parameters here so that * panel can use these parameters @@ -1500,14 +1425,6 @@ static int dsi_cmds2buf_tx(struct msm_dsi_host *msm_host, return len; } -static void dsi_hpd_worker(struct work_struct *work) -{ - struct msm_dsi_host *msm_host = - container_of(work, struct msm_dsi_host, hpd_work); - - drm_helper_hpd_irq_event(msm_host->dev); -} - static void dsi_err_worker(struct work_struct *work) { struct msm_dsi_host *msm_host = @@ -1686,6 +1603,8 @@ static int dsi_host_attach(struct mipi_dsi_host *host, msm_host->lanes = dsi->lanes; msm_host->format = dsi->format; msm_host->mode_flags = dsi->mode_flags; + if (dsi->dsc) + msm_host->dsc = dsi->dsc; /* Some gpios defined in panel DT need to be controlled by host */ ret = dsi_host_init_panel_gpios(msm_host, &dsi->dev); @@ -1697,8 +1616,6 @@ static int dsi_host_attach(struct mipi_dsi_host *host, return ret; DBG("id=%d", msm_host->id); - if (msm_host->dev) - queue_work(msm_host->workqueue, &msm_host->hpd_work); return 0; } @@ -1710,11 +1627,7 @@ static int dsi_host_detach(struct mipi_dsi_host *host, dsi_dev_detach(msm_host->pdev); - msm_host->device_node = NULL; - DBG("id=%d", msm_host->id); - if (msm_host->dev) - queue_work(msm_host->workqueue, &msm_host->hpd_work); return 0; } @@ -1841,7 +1754,7 @@ static char bpg_offset[DSC_NUM_BUF_RANGES] = { 2, 0, 0, -2, -4, -6, -8, -8, -8, -10, -10, -12, -12, -12, -12 }; -static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc) +static int dsi_populate_dsc_params(struct drm_dsc_config *dsc) { int mux_words_size; int groups_per_line, groups_total; @@ -1854,98 +1767,98 @@ static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc) int final_value, final_scale; int i; - dsc->drm->rc_model_size = 8192; - dsc->drm->first_line_bpg_offset = 12; - dsc->drm->rc_edge_factor = 6; - dsc->drm->rc_tgt_offset_high = 3; - dsc->drm->rc_tgt_offset_low = 3; - dsc->drm->simple_422 = 0; - dsc->drm->convert_rgb = 1; - dsc->drm->vbr_enable = 0; + dsc->rc_model_size = 8192; + dsc->first_line_bpg_offset = 12; + dsc->rc_edge_factor = 6; + dsc->rc_tgt_offset_high = 3; + dsc->rc_tgt_offset_low = 3; + dsc->simple_422 = 0; + dsc->convert_rgb = 1; + dsc->vbr_enable = 0; /* handle only bpp = bpc = 8 */ for (i = 0; i < DSC_NUM_BUF_RANGES - 1 ; i++) - dsc->drm->rc_buf_thresh[i] = dsi_dsc_rc_buf_thresh[i]; + dsc->rc_buf_thresh[i] = dsi_dsc_rc_buf_thresh[i]; for (i = 0; i < DSC_NUM_BUF_RANGES; i++) { - dsc->drm->rc_range_params[i].range_min_qp = min_qp[i]; - dsc->drm->rc_range_params[i].range_max_qp = max_qp[i]; - dsc->drm->rc_range_params[i].range_bpg_offset = bpg_offset[i]; + dsc->rc_range_params[i].range_min_qp = min_qp[i]; + dsc->rc_range_params[i].range_max_qp = max_qp[i]; + dsc->rc_range_params[i].range_bpg_offset = bpg_offset[i]; } - dsc->drm->initial_offset = 6144; /* Not bpp 12 */ - if (dsc->drm->bits_per_pixel != 8) - dsc->drm->initial_offset = 2048; /* bpp = 12 */ + dsc->initial_offset = 6144; /* Not bpp 12 */ + if (dsc->bits_per_pixel != 8) + dsc->initial_offset = 2048; /* bpp = 12 */ mux_words_size = 48; /* bpc == 8/10 */ - if (dsc->drm->bits_per_component == 12) + if (dsc->bits_per_component == 12) mux_words_size = 64; - dsc->drm->initial_xmit_delay = 512; - dsc->drm->initial_scale_value = 32; - dsc->drm->first_line_bpg_offset = 12; - dsc->drm->line_buf_depth = dsc->drm->bits_per_component + 1; + dsc->initial_xmit_delay = 512; + dsc->initial_scale_value = 32; + dsc->first_line_bpg_offset = 12; + dsc->line_buf_depth = dsc->bits_per_component + 1; /* bpc 8 */ - dsc->drm->flatness_min_qp = 3; - dsc->drm->flatness_max_qp = 12; - dsc->drm->rc_quant_incr_limit0 = 11; - dsc->drm->rc_quant_incr_limit1 = 11; - dsc->drm->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC; + dsc->flatness_min_qp = 3; + dsc->flatness_max_qp = 12; + dsc->rc_quant_incr_limit0 = 11; + dsc->rc_quant_incr_limit1 = 11; + dsc->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC; /* FIXME: need to call drm_dsc_compute_rc_parameters() so that rest of * params are calculated */ - groups_per_line = DIV_ROUND_UP(dsc->drm->slice_width, 3); - dsc->drm->slice_chunk_size = dsc->drm->slice_width * dsc->drm->bits_per_pixel / 8; - if ((dsc->drm->slice_width * dsc->drm->bits_per_pixel) % 8) - dsc->drm->slice_chunk_size++; + groups_per_line = DIV_ROUND_UP(dsc->slice_width, 3); + dsc->slice_chunk_size = dsc->slice_width * dsc->bits_per_pixel / 8; + if ((dsc->slice_width * dsc->bits_per_pixel) % 8) + dsc->slice_chunk_size++; /* rbs-min */ - min_rate_buffer_size = dsc->drm->rc_model_size - dsc->drm->initial_offset + - dsc->drm->initial_xmit_delay * dsc->drm->bits_per_pixel + - groups_per_line * dsc->drm->first_line_bpg_offset; + min_rate_buffer_size = dsc->rc_model_size - dsc->initial_offset + + dsc->initial_xmit_delay * dsc->bits_per_pixel + + groups_per_line * dsc->first_line_bpg_offset; - hrd_delay = DIV_ROUND_UP(min_rate_buffer_size, dsc->drm->bits_per_pixel); + hrd_delay = DIV_ROUND_UP(min_rate_buffer_size, dsc->bits_per_pixel); - dsc->drm->initial_dec_delay = hrd_delay - dsc->drm->initial_xmit_delay; + dsc->initial_dec_delay = hrd_delay - dsc->initial_xmit_delay; - dsc->drm->initial_scale_value = 8 * dsc->drm->rc_model_size / - (dsc->drm->rc_model_size - dsc->drm->initial_offset); + dsc->initial_scale_value = 8 * dsc->rc_model_size / + (dsc->rc_model_size - dsc->initial_offset); - slice_bits = 8 * dsc->drm->slice_chunk_size * dsc->drm->slice_height; + slice_bits = 8 * dsc->slice_chunk_size * dsc->slice_height; - groups_total = groups_per_line * dsc->drm->slice_height; + groups_total = groups_per_line * dsc->slice_height; - data = dsc->drm->first_line_bpg_offset * 2048; + data = dsc->first_line_bpg_offset * 2048; - dsc->drm->nfl_bpg_offset = DIV_ROUND_UP(data, (dsc->drm->slice_height - 1)); + dsc->nfl_bpg_offset = DIV_ROUND_UP(data, (dsc->slice_height - 1)); - pre_num_extra_mux_bits = 3 * (mux_words_size + (4 * dsc->drm->bits_per_component + 4) - 2); + pre_num_extra_mux_bits = 3 * (mux_words_size + (4 * dsc->bits_per_component + 4) - 2); num_extra_mux_bits = pre_num_extra_mux_bits - (mux_words_size - ((slice_bits - pre_num_extra_mux_bits) % mux_words_size)); - data = 2048 * (dsc->drm->rc_model_size - dsc->drm->initial_offset + num_extra_mux_bits); - dsc->drm->slice_bpg_offset = DIV_ROUND_UP(data, groups_total); + data = 2048 * (dsc->rc_model_size - dsc->initial_offset + num_extra_mux_bits); + dsc->slice_bpg_offset = DIV_ROUND_UP(data, groups_total); /* bpp * 16 + 0.5 */ - data = dsc->drm->bits_per_pixel * 16; + data = dsc->bits_per_pixel * 16; data *= 2; data++; data /= 2; target_bpp_x16 = data; - data = (dsc->drm->initial_xmit_delay * target_bpp_x16) / 16; - final_value = dsc->drm->rc_model_size - data + num_extra_mux_bits; - dsc->drm->final_offset = final_value; + data = (dsc->initial_xmit_delay * target_bpp_x16) / 16; + final_value = dsc->rc_model_size - data + num_extra_mux_bits; + dsc->final_offset = final_value; - final_scale = 8 * dsc->drm->rc_model_size / (dsc->drm->rc_model_size - final_value); + final_scale = 8 * dsc->rc_model_size / (dsc->rc_model_size - final_value); - data = (final_scale - 9) * (dsc->drm->nfl_bpg_offset + dsc->drm->slice_bpg_offset); - dsc->drm->scale_increment_interval = (2048 * dsc->drm->final_offset) / data; + data = (final_scale - 9) * (dsc->nfl_bpg_offset + dsc->slice_bpg_offset); + dsc->scale_increment_interval = (2048 * dsc->final_offset) / data; - dsc->drm->scale_decrement_interval = groups_per_line / (dsc->drm->initial_scale_value - 8); + dsc->scale_decrement_interval = groups_per_line / (dsc->initial_scale_value - 8); return 0; } @@ -1954,7 +1867,7 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host) { struct device *dev = &msm_host->pdev->dev; struct device_node *np = dev->of_node; - struct device_node *endpoint, *device_node; + struct device_node *endpoint; int ret = 0; /* @@ -1977,16 +1890,6 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host) goto err; } - /* Get panel node from the output port's endpoint data */ - device_node = of_graph_get_remote_node(np, 1, 0); - if (!device_node) { - DRM_DEV_DEBUG(dev, "%s: no valid device\n", __func__); - ret = -ENODEV; - goto err; - } - - msm_host->device_node = device_node; - if (of_property_read_bool(np, "syscon-sfpb")) { msm_host->sfpb = syscon_regmap_lookup_by_phandle(np, "syscon-sfpb"); @@ -1997,8 +1900,6 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host) } } - of_node_put(device_node); - err: of_node_put(endpoint); @@ -2028,6 +1929,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi) { struct msm_dsi_host *msm_host = NULL; struct platform_device *pdev = msm_dsi->pdev; + const struct msm_dsi_config *cfg; int ret; msm_host = devm_kzalloc(&pdev->dev, sizeof(*msm_host), GFP_KERNEL); @@ -2060,6 +1962,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi) pr_err("%s: get config failed\n", __func__); goto fail; } + cfg = msm_host->cfg_hnd->cfg; msm_host->id = dsi_host_get_id(msm_host); if (msm_host->id < 0) { @@ -2069,13 +1972,13 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi) } /* fixup base address by io offset */ - msm_host->ctrl_base += msm_host->cfg_hnd->cfg->io_offset; + msm_host->ctrl_base += cfg->io_offset; - ret = dsi_regulator_init(msm_host); - if (ret) { - pr_err("%s: regulator init failed\n", __func__); + ret = devm_regulator_bulk_get_const(&pdev->dev, cfg->num_regulators, + cfg->regulator_data, + &msm_host->supplies); + if (ret) goto fail; - } ret = dsi_clk_init(msm_host); if (ret) { @@ -2126,7 +2029,6 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi) /* setup workqueue */ msm_host->workqueue = alloc_ordered_workqueue("dsi_drm_work", 0); INIT_WORK(&msm_host->err_work, dsi_err_worker); - INIT_WORK(&msm_host->hpd_work, dsi_hpd_worker); msm_dsi->id = msm_host->id; @@ -2159,23 +2061,9 @@ int msm_dsi_host_modeset_init(struct mipi_dsi_host *host, { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd; - struct drm_panel *panel; int ret; msm_host->dev = dev; - panel = msm_dsi_host_get_panel(&msm_host->base); - - if (!IS_ERR(panel) && panel->dsc) { - struct msm_display_dsc_config *dsc = msm_host->dsc; - - if (!dsc) { - dsc = devm_kzalloc(&msm_host->pdev->dev, sizeof(*dsc), GFP_KERNEL); - if (!dsc) - return -ENOMEM; - dsc->drm = panel->dsc; - msm_host->dsc = dsc; - } - } ret = cfg_hnd->ops->tx_buf_alloc(msm_host, SZ_4K); if (ret) { @@ -2556,7 +2444,8 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host, msm_dsi_sfpb_config(msm_host, true); - ret = dsi_host_regulator_enable(msm_host); + ret = regulator_bulk_enable(msm_host->cfg_hnd->cfg->num_regulators, + msm_host->supplies); if (ret) { pr_err("%s:Failed to enable vregs.ret=%d\n", __func__, ret); @@ -2596,7 +2485,8 @@ fail_disable_clk: cfg_hnd->ops->link_clk_disable(msm_host); pm_runtime_put(&msm_host->pdev->dev); fail_disable_reg: - dsi_host_regulator_disable(msm_host); + regulator_bulk_disable(msm_host->cfg_hnd->cfg->num_regulators, + msm_host->supplies); unlock_ret: mutex_unlock(&msm_host->dev_mutex); return ret; @@ -2623,7 +2513,8 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host) cfg_hnd->ops->link_clk_disable(msm_host); pm_runtime_put(&msm_host->pdev->dev); - dsi_host_regulator_disable(msm_host); + regulator_bulk_disable(msm_host->cfg_hnd->cfg->num_regulators, + msm_host->supplies); msm_dsi_sfpb_config(msm_host, false); @@ -2659,45 +2550,33 @@ enum drm_mode_status msm_dsi_host_check_dsc(struct mipi_dsi_host *host, const struct drm_display_mode *mode) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); - struct msm_display_dsc_config *dsc = msm_host->dsc; + struct drm_dsc_config *dsc = msm_host->dsc; int pic_width = mode->hdisplay; int pic_height = mode->vdisplay; if (!msm_host->dsc) return MODE_OK; - if (pic_width % dsc->drm->slice_width) { + if (pic_width % dsc->slice_width) { pr_err("DSI: pic_width %d has to be multiple of slice %d\n", - pic_width, dsc->drm->slice_width); + pic_width, dsc->slice_width); return MODE_H_ILLEGAL; } - if (pic_height % dsc->drm->slice_height) { + if (pic_height % dsc->slice_height) { pr_err("DSI: pic_height %d has to be multiple of slice %d\n", - pic_height, dsc->drm->slice_height); + pic_height, dsc->slice_height); return MODE_V_ILLEGAL; } return MODE_OK; } -struct drm_panel *msm_dsi_host_get_panel(struct mipi_dsi_host *host) -{ - return of_drm_find_panel(to_msm_dsi_host(host)->device_node); -} - unsigned long msm_dsi_host_get_mode_flags(struct mipi_dsi_host *host) { return to_msm_dsi_host(host)->mode_flags; } -struct drm_bridge *msm_dsi_host_get_bridge(struct mipi_dsi_host *host) -{ - struct msm_dsi_host *msm_host = to_msm_dsi_host(host); - - return of_drm_find_bridge(msm_host->device_node); -} - void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_host *host) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); @@ -2771,7 +2650,7 @@ void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host) DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER_SW_TRIGGER); } -struct msm_display_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host) +struct drm_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host) { struct msm_dsi_host *msm_host = to_msm_dsi_host(host); diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c index cb84d185d73a560dbdd50ca7773de56811ecdb89..3a14173972832a06d956cd48dc2ac253edf19a68 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_manager.c +++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c @@ -141,14 +141,11 @@ static int enable_phy(struct msm_dsi *msm_dsi, struct msm_dsi_phy_shared_timings *shared_timings) { struct msm_dsi_phy_clk_request clk_req; - int ret; bool is_bonded_dsi = IS_BONDED_DSI(); msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req, is_bonded_dsi); - ret = msm_dsi_phy_enable(msm_dsi->phy, &clk_req, shared_timings); - - return ret; + return msm_dsi_phy_enable(msm_dsi->phy, &clk_req, shared_timings); } static int @@ -214,39 +211,26 @@ static void dsi_mgr_phy_disable(int id) } } -struct dsi_connector { - struct drm_connector base; - int id; -}; - struct dsi_bridge { struct drm_bridge base; int id; }; -#define to_dsi_connector(x) container_of(x, struct dsi_connector, base) #define to_dsi_bridge(x) container_of(x, struct dsi_bridge, base) -static inline int dsi_mgr_connector_get_id(struct drm_connector *connector) -{ - struct dsi_connector *dsi_connector = to_dsi_connector(connector); - return dsi_connector->id; -} - static int dsi_mgr_bridge_get_id(struct drm_bridge *bridge) { struct dsi_bridge *dsi_bridge = to_dsi_bridge(bridge); return dsi_bridge->id; } -static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) +static void msm_dsi_manager_set_split_display(u8 id) { - struct msm_drm_private *priv = conn->dev->dev_private; - struct msm_kms *kms = priv->kms; struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct msm_dsi *other_dsi = dsi_mgr_get_other_dsi(id); + struct msm_drm_private *priv = msm_dsi->dev->dev_private; + struct msm_kms *kms = priv->kms; struct msm_dsi *master_dsi, *slave_dsi; - struct drm_panel *panel; if (IS_BONDED_DSI() && !IS_MASTER_DSI_LINK(id)) { master_dsi = other_dsi; @@ -256,89 +240,18 @@ static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id) slave_dsi = other_dsi; } - /* - * There is only 1 panel in the global panel list for bonded DSI mode. - * Therefore slave dsi should get the drm_panel instance from master - * dsi. - */ - panel = msm_dsi_host_get_panel(master_dsi->host); - if (IS_ERR(panel)) { - DRM_ERROR("Could not find panel for %u (%ld)\n", msm_dsi->id, - PTR_ERR(panel)); - return PTR_ERR(panel); - } - - if (!panel || !IS_BONDED_DSI()) - goto out; - - drm_object_attach_property(&conn->base, - conn->dev->mode_config.tile_property, 0); + if (!msm_dsi->external_bridge || !IS_BONDED_DSI()) + return; /* * Set split display info to kms once bonded DSI panel is connected to * both hosts. */ - if (other_dsi && other_dsi->panel && kms->funcs->set_split_display) { + if (other_dsi && other_dsi->external_bridge && kms->funcs->set_split_display) { kms->funcs->set_split_display(kms, master_dsi->encoder, slave_dsi->encoder, msm_dsi_is_cmd_mode(msm_dsi)); } - -out: - msm_dsi->panel = panel; - return 0; -} - -static enum drm_connector_status dsi_mgr_connector_detect( - struct drm_connector *connector, bool force) -{ - int id = dsi_mgr_connector_get_id(connector); - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - - return msm_dsi->panel ? connector_status_connected : - connector_status_disconnected; -} - -static void dsi_mgr_connector_destroy(struct drm_connector *connector) -{ - struct dsi_connector *dsi_connector = to_dsi_connector(connector); - - DBG(""); - - drm_connector_cleanup(connector); - - kfree(dsi_connector); -} - -static int dsi_mgr_connector_get_modes(struct drm_connector *connector) -{ - int id = dsi_mgr_connector_get_id(connector); - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - struct drm_panel *panel = msm_dsi->panel; - int num; - - if (!panel) - return 0; - - /* - * In bonded DSI mode, we have one connector that can be - * attached to the drm_panel. - */ - num = drm_panel_get_modes(panel, connector); - if (!num) - return 0; - - return num; -} - -static struct drm_encoder * -dsi_mgr_connector_best_encoder(struct drm_connector *connector) -{ - int id = dsi_mgr_connector_get_id(connector); - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - - DBG(""); - return msm_dsi_get_encoder(msm_dsi); } static void dsi_mgr_bridge_power_on(struct drm_bridge *bridge) @@ -403,7 +316,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge) struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct msm_dsi *msm_dsi1 = dsi_mgr_get_dsi(DSI_1); struct mipi_dsi_host *host = msm_dsi->host; - struct drm_panel *panel = msm_dsi->panel; bool is_bonded_dsi = IS_BONDED_DSI(); int ret; @@ -418,18 +330,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge) if (!dsi_mgr_power_on_early(bridge)) dsi_mgr_bridge_power_on(bridge); - /* Always call panel functions once, because even for dual panels, - * there is only one drm_panel instance. - */ - if (panel) { - ret = drm_panel_prepare(panel); - if (ret) { - pr_err("%s: prepare panel %d failed, %d\n", __func__, - id, ret); - goto panel_prep_fail; - } - } - ret = msm_dsi_host_enable(host); if (ret) { pr_err("%s: enable host %d failed, %d\n", __func__, id, ret); @@ -449,9 +349,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge) host1_en_fail: msm_dsi_host_disable(host); host_en_fail: - if (panel) - drm_panel_unprepare(panel); -panel_prep_fail: return; } @@ -469,62 +366,12 @@ void msm_dsi_manager_tpg_enable(void) } } -static void dsi_mgr_bridge_enable(struct drm_bridge *bridge) -{ - int id = dsi_mgr_bridge_get_id(bridge); - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - struct drm_panel *panel = msm_dsi->panel; - bool is_bonded_dsi = IS_BONDED_DSI(); - int ret; - - DBG("id=%d", id); - if (!msm_dsi_device_connected(msm_dsi)) - return; - - /* Do nothing with the host if it is slave-DSI in case of bonded DSI */ - if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) - return; - - if (panel) { - ret = drm_panel_enable(panel); - if (ret) { - pr_err("%s: enable panel %d failed, %d\n", __func__, id, - ret); - } - } -} - -static void dsi_mgr_bridge_disable(struct drm_bridge *bridge) -{ - int id = dsi_mgr_bridge_get_id(bridge); - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - struct drm_panel *panel = msm_dsi->panel; - bool is_bonded_dsi = IS_BONDED_DSI(); - int ret; - - DBG("id=%d", id); - if (!msm_dsi_device_connected(msm_dsi)) - return; - - /* Do nothing with the host if it is slave-DSI in case of bonded DSI */ - if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id)) - return; - - if (panel) { - ret = drm_panel_disable(panel); - if (ret) - pr_err("%s: Panel %d OFF failed, %d\n", __func__, id, - ret); - } -} - static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge) { int id = dsi_mgr_bridge_get_id(bridge); struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct msm_dsi *msm_dsi1 = dsi_mgr_get_dsi(DSI_1); struct mipi_dsi_host *host = msm_dsi->host; - struct drm_panel *panel = msm_dsi->panel; bool is_bonded_dsi = IS_BONDED_DSI(); int ret; @@ -551,13 +398,6 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge) pr_err("%s: host1 disable failed, %d\n", __func__, ret); } - if (panel) { - ret = drm_panel_unprepare(panel); - if (ret) - pr_err("%s: Panel %d unprepare failed,%d\n", __func__, - id, ret); - } - msm_dsi_host_disable_irq(host); if (is_bonded_dsi && msm_dsi1) msm_dsi_host_disable_irq(msm_dsi1->host); @@ -614,76 +454,13 @@ static enum drm_mode_status dsi_mgr_bridge_mode_valid(struct drm_bridge *bridge, return msm_dsi_host_check_dsc(host, mode); } -static const struct drm_connector_funcs dsi_mgr_connector_funcs = { - .detect = dsi_mgr_connector_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = dsi_mgr_connector_destroy, - .reset = drm_atomic_helper_connector_reset, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, -}; - -static const struct drm_connector_helper_funcs dsi_mgr_conn_helper_funcs = { - .get_modes = dsi_mgr_connector_get_modes, - .best_encoder = dsi_mgr_connector_best_encoder, -}; - static const struct drm_bridge_funcs dsi_mgr_bridge_funcs = { .pre_enable = dsi_mgr_bridge_pre_enable, - .enable = dsi_mgr_bridge_enable, - .disable = dsi_mgr_bridge_disable, .post_disable = dsi_mgr_bridge_post_disable, .mode_set = dsi_mgr_bridge_mode_set, .mode_valid = dsi_mgr_bridge_mode_valid, }; -/* initialize connector when we're connected to a drm_panel */ -struct drm_connector *msm_dsi_manager_connector_init(u8 id) -{ - struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); - struct drm_connector *connector = NULL; - struct dsi_connector *dsi_connector; - int ret; - - dsi_connector = kzalloc(sizeof(*dsi_connector), GFP_KERNEL); - if (!dsi_connector) - return ERR_PTR(-ENOMEM); - - dsi_connector->id = id; - - connector = &dsi_connector->base; - - ret = drm_connector_init(msm_dsi->dev, connector, - &dsi_mgr_connector_funcs, DRM_MODE_CONNECTOR_DSI); - if (ret) - return ERR_PTR(ret); - - drm_connector_helper_add(connector, &dsi_mgr_conn_helper_funcs); - - /* Enable HPD to let hpd event is handled - * when panel is attached to the host. - */ - connector->polled = DRM_CONNECTOR_POLL_HPD; - - /* Display driver doesn't support interlace now. */ - connector->interlace_allowed = 0; - connector->doublescan_allowed = 0; - - drm_connector_attach_encoder(connector, msm_dsi->encoder); - - ret = msm_dsi_manager_panel_init(connector, id); - if (ret) { - DRM_DEV_ERROR(msm_dsi->dev->dev, "init panel failed %d\n", ret); - goto fail; - } - - return connector; - -fail: - connector->funcs->destroy(connector); - return ERR_PTR(ret); -} - /* initialize bridge */ struct drm_bridge *msm_dsi_manager_bridge_init(u8 id) { @@ -722,18 +499,21 @@ fail: return ERR_PTR(ret); } -struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id) +int msm_dsi_manager_ext_bridge_init(u8 id) { struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id); struct drm_device *dev = msm_dsi->dev; - struct drm_connector *connector; struct drm_encoder *encoder; struct drm_bridge *int_bridge, *ext_bridge; int ret; int_bridge = msm_dsi->bridge; - ext_bridge = msm_dsi->external_bridge = - msm_dsi_host_get_bridge(msm_dsi->host); + ext_bridge = devm_drm_of_get_bridge(&msm_dsi->pdev->dev, + msm_dsi->pdev->dev.of_node, 1, 0); + if (IS_ERR(ext_bridge)) + return PTR_ERR(ext_bridge); + + msm_dsi->external_bridge = ext_bridge; encoder = msm_dsi->encoder; @@ -745,36 +525,32 @@ struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id) ret = drm_bridge_attach(encoder, ext_bridge, int_bridge, DRM_BRIDGE_ATTACH_NO_CONNECTOR); if (ret == -EINVAL) { - struct drm_connector *connector; - struct list_head *connector_list; - - /* link the internal dsi bridge to the external bridge */ - drm_bridge_attach(encoder, ext_bridge, int_bridge, 0); - /* - * we need the drm_connector created by the external bridge - * driver (or someone else) to feed it to our driver's - * priv->connector[] list, mainly for msm_fbdev_init() + * link the internal dsi bridge to the external bridge, + * connector is created by the next bridge. */ - connector_list = &dev->mode_config.connector_list; + ret = drm_bridge_attach(encoder, ext_bridge, int_bridge, 0); + if (ret < 0) + return ret; + } else { + struct drm_connector *connector; - list_for_each_entry(connector, connector_list, head) { - if (drm_connector_has_possible_encoder(connector, encoder)) - return connector; + /* We are in charge of the connector, create one now. */ + connector = drm_bridge_connector_init(dev, encoder); + if (IS_ERR(connector)) { + DRM_ERROR("Unable to create bridge connector\n"); + return PTR_ERR(connector); } - return ERR_PTR(-ENODEV); - } - - connector = drm_bridge_connector_init(dev, encoder); - if (IS_ERR(connector)) { - DRM_ERROR("Unable to create bridge connector\n"); - return ERR_CAST(connector); + ret = drm_connector_attach_encoder(connector, encoder); + if (ret < 0) + return ret; } - drm_connector_attach_encoder(connector, encoder); + /* The pipeline is ready, ping encoders if necessary */ + msm_dsi_manager_set_split_display(id); - return connector; + return 0; } void msm_dsi_manager_bridge_destroy(struct drm_bridge *bridge) diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c index a39de3bdc7fafd1172c012b7c4d3e18686ed3ce7..7fc0975cb869332c653ebea99d038cdc8dce72e6 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c @@ -347,7 +347,7 @@ int msm_dsi_dphy_timing_calc_v3(struct msm_dsi_dphy_timing *timing, } else { timing->shared_timings.clk_pre = linear_inter(tmax, tmin, pcnt2, 0, false); - timing->shared_timings.clk_pre_inc_by_2 = 0; + timing->shared_timings.clk_pre_inc_by_2 = 0; } timing->ta_go = 3; @@ -507,82 +507,6 @@ int msm_dsi_cphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing, return 0; } -static int dsi_phy_regulator_init(struct msm_dsi_phy *phy) -{ - struct regulator_bulk_data *s = phy->supplies; - const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs; - struct device *dev = &phy->pdev->dev; - int num = phy->cfg->reg_cfg.num; - int i, ret; - - for (i = 0; i < num; i++) - s[i].supply = regs[i].name; - - ret = devm_regulator_bulk_get(dev, num, s); - if (ret < 0) { - if (ret != -EPROBE_DEFER) { - DRM_DEV_ERROR(dev, - "%s: failed to init regulator, ret=%d\n", - __func__, ret); - } - - return ret; - } - - return 0; -} - -static void dsi_phy_regulator_disable(struct msm_dsi_phy *phy) -{ - struct regulator_bulk_data *s = phy->supplies; - const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs; - int num = phy->cfg->reg_cfg.num; - int i; - - DBG(""); - for (i = num - 1; i >= 0; i--) - if (regs[i].disable_load >= 0) - regulator_set_load(s[i].consumer, regs[i].disable_load); - - regulator_bulk_disable(num, s); -} - -static int dsi_phy_regulator_enable(struct msm_dsi_phy *phy) -{ - struct regulator_bulk_data *s = phy->supplies; - const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs; - struct device *dev = &phy->pdev->dev; - int num = phy->cfg->reg_cfg.num; - int ret, i; - - DBG(""); - for (i = 0; i < num; i++) { - if (regs[i].enable_load >= 0) { - ret = regulator_set_load(s[i].consumer, - regs[i].enable_load); - if (ret < 0) { - DRM_DEV_ERROR(dev, - "regulator %d set op mode failed, %d\n", - i, ret); - goto fail; - } - } - } - - ret = regulator_bulk_enable(num, s); - if (ret < 0) { - DRM_DEV_ERROR(dev, "regulator enable failed, %d\n", ret); - goto fail; - } - - return 0; - -fail: - for (i--; i >= 0; i--) - regulator_set_load(s[i].consumer, regs[i].disable_load); - return ret; -} - static int dsi_phy_enable_resource(struct msm_dsi_phy *phy) { struct device *dev = &phy->pdev->dev; @@ -697,12 +621,9 @@ static int dsi_phy_driver_probe(struct platform_device *pdev) phy->pdev = pdev; phy->id = dsi_phy_get_id(phy); - if (phy->id < 0) { - ret = phy->id; - DRM_DEV_ERROR(dev, "%s: couldn't identify PHY index, %d\n", - __func__, ret); - goto fail; - } + if (phy->id < 0) + return dev_err_probe(dev, phy->id, + "Couldn't identify PHY index\n"); phy->regulator_ldo_mode = of_property_read_bool(dev->of_node, "qcom,dsi-phy-regulator-ldo-mode"); @@ -710,86 +631,71 @@ static int dsi_phy_driver_probe(struct platform_device *pdev) phy->cphy_mode = (phy_type == PHY_TYPE_CPHY); phy->base = msm_ioremap_size(pdev, "dsi_phy", &phy->base_size); - if (IS_ERR(phy->base)) { - DRM_DEV_ERROR(dev, "%s: failed to map phy base\n", __func__); - ret = -ENOMEM; - goto fail; - } + if (IS_ERR(phy->base)) + return dev_err_probe(dev, PTR_ERR(phy->base), + "Failed to map phy base\n"); phy->pll_base = msm_ioremap_size(pdev, "dsi_pll", &phy->pll_size); - if (IS_ERR(phy->pll_base)) { - DRM_DEV_ERROR(&pdev->dev, "%s: failed to map pll base\n", __func__); - ret = -ENOMEM; - goto fail; - } + if (IS_ERR(phy->pll_base)) + return dev_err_probe(dev, PTR_ERR(phy->pll_base), + "Failed to map pll base\n"); if (phy->cfg->has_phy_lane) { phy->lane_base = msm_ioremap_size(pdev, "dsi_phy_lane", &phy->lane_size); - if (IS_ERR(phy->lane_base)) { - DRM_DEV_ERROR(&pdev->dev, "%s: failed to map phy lane base\n", __func__); - ret = -ENOMEM; - goto fail; - } + if (IS_ERR(phy->lane_base)) + return dev_err_probe(dev, PTR_ERR(phy->lane_base), + "Failed to map phy lane base\n"); } if (phy->cfg->has_phy_regulator) { phy->reg_base = msm_ioremap_size(pdev, "dsi_phy_regulator", &phy->reg_size); - if (IS_ERR(phy->reg_base)) { - DRM_DEV_ERROR(&pdev->dev, "%s: failed to map phy regulator base\n", __func__); - ret = -ENOMEM; - goto fail; - } + if (IS_ERR(phy->reg_base)) + return dev_err_probe(dev, PTR_ERR(phy->reg_base), + "Failed to map phy regulator base\n"); } if (phy->cfg->ops.parse_dt_properties) { ret = phy->cfg->ops.parse_dt_properties(phy); if (ret) - goto fail; + return ret; } - ret = dsi_phy_regulator_init(phy); + ret = devm_regulator_bulk_get_const(dev, phy->cfg->num_regulators, + phy->cfg->regulator_data, + &phy->supplies); if (ret) - goto fail; + return ret; phy->ahb_clk = msm_clk_get(pdev, "iface"); - if (IS_ERR(phy->ahb_clk)) { - DRM_DEV_ERROR(dev, "%s: Unable to get ahb clk\n", __func__); - ret = PTR_ERR(phy->ahb_clk); - goto fail; - } + if (IS_ERR(phy->ahb_clk)) + return dev_err_probe(dev, PTR_ERR(phy->ahb_clk), + "Unable to get ahb clk\n"); /* PLL init will call into clk_register which requires * register access, so we need to enable power and ahb clock. */ ret = dsi_phy_enable_resource(phy); if (ret) - goto fail; + return ret; if (phy->cfg->ops.pll_init) { ret = phy->cfg->ops.pll_init(phy); - if (ret) { - DRM_DEV_INFO(dev, - "%s: pll init failed: %d, need separate pll clk driver\n", - __func__, ret); - goto fail; - } + if (ret) + return dev_err_probe(dev, ret, + "PLL init failed; need separate clk driver\n"); } ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, phy->provided_clocks); - if (ret) { - DRM_DEV_ERROR(dev, "%s: failed to register clk provider: %d\n", __func__, ret); - goto fail; - } + if (ret) + return dev_err_probe(dev, ret, + "Failed to register clk provider\n"); dsi_phy_disable_resource(phy); platform_set_drvdata(pdev, phy); return 0; - -fail: - return ret; } static struct platform_driver dsi_phy_platform_driver = { @@ -829,7 +735,7 @@ int msm_dsi_phy_enable(struct msm_dsi_phy *phy, goto res_en_fail; } - ret = dsi_phy_regulator_enable(phy); + ret = regulator_bulk_enable(phy->cfg->num_regulators, phy->supplies); if (ret) { DRM_DEV_ERROR(dev, "%s: regulator enable failed, %d\n", __func__, ret); @@ -866,7 +772,7 @@ pll_restor_fail: if (phy->cfg->ops.disable) phy->cfg->ops.disable(phy); phy_en_fail: - dsi_phy_regulator_disable(phy); + regulator_bulk_disable(phy->cfg->num_regulators, phy->supplies); reg_en_fail: dsi_phy_disable_resource(phy); res_en_fail: @@ -880,7 +786,7 @@ void msm_dsi_phy_disable(struct msm_dsi_phy *phy) phy->cfg->ops.disable(phy); - dsi_phy_regulator_disable(phy); + regulator_bulk_disable(phy->cfg->num_regulators, phy->supplies); dsi_phy_disable_resource(phy); } diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index dc91b43d5a38517748a299428ac0212db75a6664..60a99c6525b20aa3a179c784dc68426d4ce90557 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -29,7 +29,8 @@ struct msm_dsi_phy_ops { }; struct msm_dsi_phy_cfg { - struct dsi_reg_config reg_cfg; + const struct regulator_bulk_data *regulator_data; + int num_regulators; struct msm_dsi_phy_ops ops; unsigned long min_pll_rate; @@ -98,7 +99,7 @@ struct msm_dsi_phy { int id; struct clk *ahb_clk; - struct regulator_bulk_data supplies[DSI_DEV_REGULATOR_MAX]; + struct regulator_bulk_data *supplies; struct msm_dsi_dphy_timing timing; const struct msm_dsi_phy_cfg *cfg; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c index 08b015ea1b1ecf183f63dd50278d7a2e703484c5..27b592c776a300de9ada408615add978d6345ed1 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c @@ -188,19 +188,19 @@ static void dsi_pll_ssc_commit(struct dsi_pll_10nm *pll, struct dsi_pll_config * pr_debug("SSC is enabled\n"); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_LOW_1, - config->ssc_stepsize & 0xff); + config->ssc_stepsize & 0xff); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_HIGH_1, - config->ssc_stepsize >> 8); + config->ssc_stepsize >> 8); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_LOW_1, - config->ssc_div_per & 0xff); + config->ssc_div_per & 0xff); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_HIGH_1, - config->ssc_div_per >> 8); + config->ssc_div_per >> 8); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_LOW_1, - config->ssc_adj_per & 0xff); + config->ssc_adj_per & 0xff); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_HIGH_1, - config->ssc_adj_per >> 8); + config->ssc_adj_per >> 8); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_CONTROL, - SSC_EN | (config->ssc_center ? SSC_CENTER : 0)); + SSC_EN | (config->ssc_center ? SSC_CENTER : 0)); } } @@ -215,16 +215,19 @@ static void dsi_pll_config_hzindep_reg(struct dsi_pll_10nm *pll) dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FEEDBACK_DIVIDER, 0x4e); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CALIBRATION_SETTINGS, 0x40); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_THREE, - 0xba); - dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE, 0x0c); + 0xba); + dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE, + 0x0c); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_OUTDIV, 0x00); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CORE_OVERRIDE, 0x00); - dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO, 0x08); + dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO, + 0x08); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_PROP_GAIN_RATE_1, 0x08); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_BAND_SET_RATE_1, 0xc0); - dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1, 0xfa); + dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1, + 0xfa); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_FL_INT_GAIN_PFILT_BAND_1, - 0x4c); + 0x4c); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCK_OVERRIDE, 0x80); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PFILT, 0x29); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_IFILT, 0x3f); @@ -236,18 +239,18 @@ static void dsi_pll_commit(struct dsi_pll_10nm *pll, struct dsi_pll_config *conf dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CORE_INPUT_OVERRIDE, 0x12); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_DECIMAL_DIV_START_1, - config->decimal_div_start); + config->decimal_div_start); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_LOW_1, - config->frac_div_start & 0xff); + config->frac_div_start & 0xff); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_MID_1, - (config->frac_div_start & 0xff00) >> 8); + (config->frac_div_start & 0xff00) >> 8); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_HIGH_1, - (config->frac_div_start & 0x30000) >> 16); + (config->frac_div_start & 0x30000) >> 16); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCKDET_RATE_1, 64); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCK_DELAY, 0x06); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CMODE, 0x10); dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CLOCK_INVERTERS, - config->pll_clock_inverters); + config->pll_clock_inverters); } static int dsi_pll_10nm_vco_set_rate(struct clk_hw *hw, unsigned long rate, @@ -306,7 +309,7 @@ static void dsi_pll_disable_pll_bias(struct dsi_pll_10nm *pll) dsi_phy_write(pll->phy->pll_base + REG_DSI_10nm_PHY_PLL_SYSTEM_MUXES, 0); dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0, - data & ~BIT(5)); + data & ~BIT(5)); ndelay(250); } @@ -315,7 +318,7 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_10nm *pll) u32 data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0); dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0, - data | BIT(5)); + data | BIT(5)); dsi_phy_write(pll->phy->pll_base + REG_DSI_10nm_PHY_PLL_SYSTEM_MUXES, 0xc0); ndelay(250); } @@ -326,7 +329,7 @@ static void dsi_pll_disable_global_clk(struct dsi_pll_10nm *pll) data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1); dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1, - data & ~BIT(5)); + data & ~BIT(5)); } static void dsi_pll_enable_global_clk(struct dsi_pll_10nm *pll) @@ -335,7 +338,7 @@ static void dsi_pll_enable_global_clk(struct dsi_pll_10nm *pll) data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1); dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1, - data | BIT(5)); + data | BIT(5)); } static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) @@ -356,7 +359,7 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) /* Start PLL */ dsi_phy_write(pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_PLL_CNTRL, - 0x01); + 0x01); /* * ensure all PLL configurations are written prior to checking @@ -378,10 +381,10 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw) dsi_pll_enable_global_clk(pll_10nm->slave); dsi_phy_write(pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_RBUF_CTRL, - 0x01); + 0x01); if (pll_10nm->slave) dsi_phy_write(pll_10nm->slave->phy->base + - REG_DSI_10nm_PHY_CMN_RBUF_CTRL, 0x01); + REG_DSI_10nm_PHY_CMN_RBUF_CTRL, 0x01); error: return rc; @@ -486,7 +489,7 @@ static void dsi_10nm_pll_save_state(struct msm_dsi_phy *phy) u32 cmn_clk_cfg0, cmn_clk_cfg1; cached->pll_out_div = dsi_phy_read(pll_10nm->phy->pll_base + - REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE); + REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE); cached->pll_out_div &= 0x3; cmn_clk_cfg0 = dsi_phy_read(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG0); @@ -515,7 +518,7 @@ static int dsi_10nm_pll_restore_state(struct msm_dsi_phy *phy) dsi_phy_write(pll_10nm->phy->pll_base + REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE, val); dsi_phy_write(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG0, - cached->bit_clk_div | (cached->pix_clk_div << 4)); + cached->bit_clk_div | (cached->pix_clk_div << 4)); val = dsi_phy_read(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG1); val &= ~0x3; @@ -571,64 +574,59 @@ static int dsi_10nm_set_usecase(struct msm_dsi_phy *phy) */ static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm, struct clk_hw **provided_clocks) { - char clk_name[32], parent[32], vco_name[32]; - char parent2[32], parent3[32], parent4[32]; + char clk_name[32]; struct clk_init_data vco_init = { .parent_data = &(const struct clk_parent_data) { .fw_name = "ref", }, .num_parents = 1, - .name = vco_name, + .name = clk_name, .flags = CLK_IGNORE_UNUSED, .ops = &clk_ops_dsi_pll_10nm_vco, }; struct device *dev = &pll_10nm->phy->pdev->dev; - struct clk_hw *hw; + struct clk_hw *hw, *pll_out_div, *pll_bit, *pll_by_2_bit; + struct clk_hw *pll_post_out_div, *pclk_mux; int ret; DBG("DSI%d", pll_10nm->phy->id); - snprintf(vco_name, 32, "dsi%dvco_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_10nm->phy->id); pll_10nm->clk_hw.init = &vco_init; ret = devm_clk_hw_register(dev, &pll_10nm->clk_hw); if (ret) return ret; - snprintf(clk_name, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%dvco_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_out_div_clk", pll_10nm->phy->id); - hw = devm_clk_hw_register_divider(dev, clk_name, - parent, CLK_SET_RATE_PARENT, - pll_10nm->phy->pll_base + - REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE, - 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_out_div = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + &pll_10nm->clk_hw, CLK_SET_RATE_PARENT, + pll_10nm->phy->pll_base + + REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE, + 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL); + if (IS_ERR(pll_out_div)) { + ret = PTR_ERR(pll_out_div); goto fail; } - snprintf(clk_name, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_bit_clk", pll_10nm->phy->id); /* BIT CLK: DIV_CTRL_3_0 */ - hw = devm_clk_hw_register_divider(dev, clk_name, parent, - CLK_SET_RATE_PARENT, - pll_10nm->phy->base + - REG_DSI_10nm_PHY_CMN_CLK_CFG0, - 0, 4, CLK_DIVIDER_ONE_BASED, - &pll_10nm->postdiv_lock); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_bit = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + pll_out_div, CLK_SET_RATE_PARENT, + pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG0, + 0, 4, CLK_DIVIDER_ONE_BASED, &pll_10nm->postdiv_lock); + if (IS_ERR(pll_bit)) { + ret = PTR_ERR(pll_bit); goto fail; } - snprintf(clk_name, 32, "dsi%d_phy_pll_out_byteclk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_byteclk", pll_10nm->phy->id); /* DSI Byte clock = VCO_CLK / OUT_DIV / BIT_DIV / 8 */ - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - CLK_SET_RATE_PARENT, 1, 8); + hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name, + pll_bit, CLK_SET_RATE_PARENT, 1, 8); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; @@ -636,52 +634,45 @@ static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm, struct clk_hw **prov provided_clocks[DSI_BYTE_PLL_CLK] = hw; - snprintf(clk_name, 32, "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - 0, 1, 2); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_by_2_bit = devm_clk_hw_register_fixed_factor_parent_hw(dev, + clk_name, pll_bit, 0, 1, 2); + if (IS_ERR(pll_by_2_bit)) { + ret = PTR_ERR(pll_by_2_bit); goto fail; } - snprintf(clk_name, 32, "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - 0, 1, 4); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw(dev, + clk_name, pll_out_div, 0, 1, 4); + if (IS_ERR(pll_post_out_div)) { + ret = PTR_ERR(pll_post_out_div); goto fail; } - snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id); - snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id); - snprintf(parent3, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id); - snprintf(parent4, 32, "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id); - - hw = devm_clk_hw_register_mux(dev, clk_name, - ((const char *[]){ - parent, parent2, parent3, parent4 - }), 4, 0, pll_10nm->phy->base + - REG_DSI_10nm_PHY_CMN_CLK_CFG1, - 0, 2, 0, NULL); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pclk_mux", pll_10nm->phy->id); + + pclk_mux = devm_clk_hw_register_mux_parent_hws(dev, clk_name, + ((const struct clk_hw *[]){ + pll_bit, + pll_by_2_bit, + pll_out_div, + pll_post_out_div, + }), 4, 0, pll_10nm->phy->base + + REG_DSI_10nm_PHY_CMN_CLK_CFG1, 0, 2, 0, NULL); + if (IS_ERR(pclk_mux)) { + ret = PTR_ERR(pclk_mux); goto fail; } - snprintf(clk_name, 32, "dsi%d_phy_pll_out_dsiclk", pll_10nm->phy->id); - snprintf(parent, 32, "dsi%d_pclk_mux", pll_10nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_dsiclk", pll_10nm->phy->id); /* PIX CLK DIV : DIV_CTRL_7_4*/ - hw = devm_clk_hw_register_divider(dev, clk_name, parent, - 0, pll_10nm->phy->base + - REG_DSI_10nm_PHY_CMN_CLK_CFG0, - 4, 4, CLK_DIVIDER_ONE_BASED, - &pll_10nm->postdiv_lock); + hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name, pclk_mux, + 0, pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG0, + 4, 4, CLK_DIVIDER_ONE_BASED, &pll_10nm->postdiv_lock); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; @@ -1028,14 +1019,14 @@ static int dsi_10nm_phy_parse_dt(struct msm_dsi_phy *phy) return 0; } +static const struct regulator_bulk_data dsi_phy_10nm_regulators[] = { + { .supply = "vdds", .init_load_uA = 36000 }, +}; + const struct msm_dsi_phy_cfg dsi_phy_10nm_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vdds", 36000, 32}, - }, - }, + .regulator_data = dsi_phy_10nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_10nm_regulators), .ops = { .enable = dsi_10nm_phy_enable, .disable = dsi_10nm_phy_disable, @@ -1052,12 +1043,8 @@ const struct msm_dsi_phy_cfg dsi_phy_10nm_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_10nm_8998_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vdds", 36000, 32}, - }, - }, + .regulator_data = dsi_phy_10nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_10nm_regulators), .ops = { .enable = dsi_10nm_phy_enable, .disable = dsi_10nm_phy_disable, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c index 8199c53567f4e6ee4ffcbdc0aeeefe623608dddc..0f8f4ca464291e52718c27fcc211d59a9214de87 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c @@ -711,7 +711,7 @@ static int dsi_14nm_pll_restore_state(struct msm_dsi_phy *phy) cached_state->vco_rate, 0); if (ret) { DRM_DEV_ERROR(&pll_14nm->phy->pdev->dev, - "restore vco rate failed. ret=%d\n", ret); + "restore vco rate failed. ret=%d\n", ret); return ret; } @@ -764,14 +764,14 @@ static int dsi_14nm_set_usecase(struct msm_dsi_phy *phy) static struct clk_hw *pll_14nm_postdiv_register(struct dsi_pll_14nm *pll_14nm, const char *name, - const char *parent_name, + const struct clk_hw *parent_hw, unsigned long flags, u8 shift) { struct dsi_pll_14nm_postdiv *pll_postdiv; struct device *dev = &pll_14nm->phy->pdev->dev; struct clk_init_data postdiv_init = { - .parent_names = (const char *[]) { parent_name }, + .parent_hws = (const struct clk_hw *[]) { parent_hw }, .num_parents = 1, .name = name, .flags = flags, @@ -800,72 +800,70 @@ static struct clk_hw *pll_14nm_postdiv_register(struct dsi_pll_14nm *pll_14nm, static int pll_14nm_register(struct dsi_pll_14nm *pll_14nm, struct clk_hw **provided_clocks) { - char clk_name[32], parent[32], vco_name[32]; + char clk_name[32]; struct clk_init_data vco_init = { .parent_data = &(const struct clk_parent_data) { .fw_name = "ref", }, .num_parents = 1, - .name = vco_name, + .name = clk_name, .flags = CLK_IGNORE_UNUSED, .ops = &clk_ops_dsi_pll_14nm_vco, }; struct device *dev = &pll_14nm->phy->pdev->dev; - struct clk_hw *hw; + struct clk_hw *hw, *n1_postdiv, *n1_postdivby2; int ret; DBG("DSI%d", pll_14nm->phy->id); - snprintf(vco_name, 32, "dsi%dvco_clk", pll_14nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_14nm->phy->id); pll_14nm->clk_hw.init = &vco_init; ret = devm_clk_hw_register(dev, &pll_14nm->clk_hw); if (ret) return ret; - snprintf(clk_name, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id); - snprintf(parent, 32, "dsi%dvco_clk", pll_14nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dn1_postdiv_clk", pll_14nm->phy->id); /* N1 postdiv, bits 0-3 in REG_DSI_14nm_PHY_CMN_CLK_CFG0 */ - hw = pll_14nm_postdiv_register(pll_14nm, clk_name, parent, - CLK_SET_RATE_PARENT, 0); - if (IS_ERR(hw)) - return PTR_ERR(hw); + n1_postdiv = pll_14nm_postdiv_register(pll_14nm, clk_name, + &pll_14nm->clk_hw, CLK_SET_RATE_PARENT, 0); + if (IS_ERR(n1_postdiv)) + return PTR_ERR(n1_postdiv); - snprintf(clk_name, 32, "dsi%dpllbyte", pll_14nm->phy->id); - snprintf(parent, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_14nm->phy->id); /* DSI Byte clock = VCO_CLK / N1 / 8 */ - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - CLK_SET_RATE_PARENT, 1, 8); + hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name, + n1_postdiv, CLK_SET_RATE_PARENT, 1, 8); if (IS_ERR(hw)) return PTR_ERR(hw); provided_clocks[DSI_BYTE_PLL_CLK] = hw; - snprintf(clk_name, 32, "dsi%dn1_postdivby2_clk", pll_14nm->phy->id); - snprintf(parent, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dn1_postdivby2_clk", pll_14nm->phy->id); /* * Skip the mux for now, force DSICLK_SEL to 1, Add a /2 divider * on the way. Don't let it set parent. */ - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 1, 2); - if (IS_ERR(hw)) - return PTR_ERR(hw); + n1_postdivby2 = devm_clk_hw_register_fixed_factor_parent_hw(dev, + clk_name, n1_postdiv, 0, 1, 2); + if (IS_ERR(n1_postdivby2)) + return PTR_ERR(n1_postdivby2); - snprintf(clk_name, 32, "dsi%dpll", pll_14nm->phy->id); - snprintf(parent, 32, "dsi%dn1_postdivby2_clk", pll_14nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_14nm->phy->id); /* DSI pixel clock = VCO_CLK / N1 / 2 / N2 * This is the output of N2 post-divider, bits 4-7 in * REG_DSI_14nm_PHY_CMN_CLK_CFG0. Don't let it set parent. */ - hw = pll_14nm_postdiv_register(pll_14nm, clk_name, parent, 0, 4); + hw = pll_14nm_postdiv_register(pll_14nm, clk_name, n1_postdivby2, + 0, 4); if (IS_ERR(hw)) return PTR_ERR(hw); - provided_clocks[DSI_PIXEL_PLL_CLK] = hw; + provided_clocks[DSI_PIXEL_PLL_CLK] = hw; return 0; } @@ -952,7 +950,8 @@ static int dsi_14nm_phy_enable(struct msm_dsi_phy *phy, if (msm_dsi_dphy_timing_calc_v2(timing, clk_req)) { DRM_DEV_ERROR(&phy->pdev->dev, - "%s: D-PHY timing calculation failed\n", __func__); + "%s: D-PHY timing calculation failed\n", + __func__); return -EINVAL; } @@ -1005,7 +1004,7 @@ static int dsi_14nm_phy_enable(struct msm_dsi_phy *phy, ret = dsi_14nm_set_usecase(phy); if (ret) { DRM_DEV_ERROR(&phy->pdev->dev, "%s: set pll usecase failed, %d\n", - __func__, ret); + __func__, ret); return ret; } @@ -1024,14 +1023,18 @@ static void dsi_14nm_phy_disable(struct msm_dsi_phy *phy) wmb(); } +static const struct regulator_bulk_data dsi_phy_14nm_17mA_regulators[] = { + { .supply = "vcca", .init_load_uA = 17000 }, +}; + +static const struct regulator_bulk_data dsi_phy_14nm_73p4mA_regulators[] = { + { .supply = "vcca", .init_load_uA = 73400 }, +}; + const struct msm_dsi_phy_cfg dsi_phy_14nm_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vcca", 17000, 32}, - }, - }, + .regulator_data = dsi_phy_14nm_17mA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_14nm_17mA_regulators), .ops = { .enable = dsi_14nm_phy_enable, .disable = dsi_14nm_phy_disable, @@ -1047,12 +1050,8 @@ const struct msm_dsi_phy_cfg dsi_phy_14nm_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_14nm_660_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vcca", 73400, 32}, - }, - }, + .regulator_data = dsi_phy_14nm_73p4mA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_14nm_73p4mA_regulators), .ops = { .enable = dsi_14nm_phy_enable, .disable = dsi_14nm_phy_disable, @@ -1068,12 +1067,8 @@ const struct msm_dsi_phy_cfg dsi_phy_14nm_660_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_14nm_8953_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vcca", 17000, 32}, - }, - }, + .regulator_data = dsi_phy_14nm_17mA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_14nm_17mA_regulators), .ops = { .enable = dsi_14nm_phy_enable, .disable = dsi_14nm_phy_disable, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c index ee7c418a1c2943317dbdb02c3bdc8123722edd4e..c9752b991744586c272cf31efd7076911b979dc5 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c @@ -129,15 +129,15 @@ static void dsi_20nm_phy_disable(struct msm_dsi_phy *phy) dsi_20nm_phy_regulator_ctrl(phy, false); } +static const struct regulator_bulk_data dsi_phy_20nm_regulators[] = { + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ + { .supply = "vcca", .init_load_uA = 10000 }, /* 1.0 V */ +}; + const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs = { .has_phy_regulator = true, - .reg_cfg = { - .num = 2, - .regs = { - {"vddio", 100000, 100}, /* 1.8 V */ - {"vcca", 10000, 100}, /* 1.0 V */ - }, - }, + .regulator_data = dsi_phy_20nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_20nm_regulators), .ops = { .enable = dsi_20nm_phy_enable, .disable = dsi_20nm_phy_disable, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c index 48eab80b548e1349b2c6dcac0d2cf61c3273e2d1..4c1bf55c5f382f552c7b733bb7b5e49e4e0f5f48 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c @@ -104,7 +104,7 @@ static void pll_28nm_software_reset(struct dsi_pll_28nm *pll_28nm) * reset bit off and back on. */ dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_TEST_CFG, - DSI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET, 1); + DSI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET, 1); dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_TEST_CFG, 0x00, 1); } @@ -201,9 +201,9 @@ static int dsi_pll_28nm_clk_set_rate(struct clk_hw *hw, unsigned long rate, dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG1, sdm_cfg1); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG2, - DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0(sdm_cfg2)); + DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0(sdm_cfg2)); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG3, - DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8(sdm_cfg3)); + DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8(sdm_cfg3)); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG4, 0x00); /* Add hardware recommended delay for correct PLL configuration */ @@ -316,12 +316,12 @@ static int _dsi_pll_28nm_vco_prepare_hpm(struct dsi_pll_28nm *pll_28nm) for (i = 0; i < 2; i++) { /* DSI Uniphy lock detect setting */ dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_LKDET_CFG2, - 0x0c, 100); + 0x0c, 100); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_LKDET_CFG2, 0x0d); /* poll for PLL ready status */ - locked = pll_28nm_poll_for_ready(pll_28nm, - max_reads, timeout_us); + locked = pll_28nm_poll_for_ready(pll_28nm, max_reads, + timeout_us); if (locked) break; @@ -508,28 +508,28 @@ static int dsi_28nm_pll_restore_state(struct msm_dsi_phy *phy) } dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_POSTDIV3_CFG, - cached_state->postdiv3); + cached_state->postdiv3); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG, - cached_state->postdiv1); + cached_state->postdiv1); dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_VREG_CFG, - cached_state->byte_mux); + cached_state->byte_mux); return 0; } static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **provided_clocks) { - char clk_name[32], parent1[32], parent2[32], vco_name[32]; + char clk_name[32]; struct clk_init_data vco_init = { .parent_data = &(const struct clk_parent_data) { .fw_name = "ref", .name = "xo", }, .num_parents = 1, - .name = vco_name, + .name = clk_name, .flags = CLK_IGNORE_UNUSED, }; struct device *dev = &pll_28nm->phy->pdev->dev; - struct clk_hw *hw; + struct clk_hw *hw, *analog_postdiv, *indirect_path_div2, *byte_mux; int ret; DBG("%d", pll_28nm->phy->id); @@ -539,55 +539,49 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov else vco_init.ops = &clk_ops_dsi_pll_28nm_vco_hpm; - snprintf(vco_name, 32, "dsi%dvco_clk", pll_28nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_28nm->phy->id); pll_28nm->clk_hw.init = &vco_init; ret = devm_clk_hw_register(dev, &pll_28nm->clk_hw); if (ret) return ret; - snprintf(clk_name, 32, "dsi%danalog_postdiv_clk", pll_28nm->phy->id); - snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id); - hw = devm_clk_hw_register_divider(dev, clk_name, - parent1, CLK_SET_RATE_PARENT, + snprintf(clk_name, sizeof(clk_name), "dsi%danalog_postdiv_clk", pll_28nm->phy->id); + analog_postdiv = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + &pll_28nm->clk_hw, CLK_SET_RATE_PARENT, pll_28nm->phy->pll_base + - REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG, + REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG, 0, 4, 0, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); - - snprintf(clk_name, 32, "dsi%dindirect_path_div2_clk", pll_28nm->phy->id); - snprintf(parent1, 32, "dsi%danalog_postdiv_clk", pll_28nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, - parent1, CLK_SET_RATE_PARENT, - 1, 2); - if (IS_ERR(hw)) - return PTR_ERR(hw); - - snprintf(clk_name, 32, "dsi%dpll", pll_28nm->phy->id); - snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id); - hw = devm_clk_hw_register_divider(dev, clk_name, - parent1, 0, pll_28nm->phy->pll_base + + if (IS_ERR(analog_postdiv)) + return PTR_ERR(analog_postdiv); + + snprintf(clk_name, sizeof(clk_name), "dsi%dindirect_path_div2_clk", pll_28nm->phy->id); + indirect_path_div2 = devm_clk_hw_register_fixed_factor_parent_hw(dev, + clk_name, analog_postdiv, CLK_SET_RATE_PARENT, 1, 2); + if (IS_ERR(indirect_path_div2)) + return PTR_ERR(indirect_path_div2); + + snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_28nm->phy->id); + hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + &pll_28nm->clk_hw, 0, pll_28nm->phy->pll_base + REG_DSI_28nm_PHY_PLL_POSTDIV3_CFG, - 0, 8, 0, NULL); + 0, 8, 0, NULL); if (IS_ERR(hw)) return PTR_ERR(hw); provided_clocks[DSI_PIXEL_PLL_CLK] = hw; - snprintf(clk_name, 32, "dsi%dbyte_mux", pll_28nm->phy->id); - snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id); - snprintf(parent2, 32, "dsi%dindirect_path_div2_clk", pll_28nm->phy->id); - hw = devm_clk_hw_register_mux(dev, clk_name, - ((const char *[]){ - parent1, parent2 + snprintf(clk_name, sizeof(clk_name), "dsi%dbyte_mux", pll_28nm->phy->id); + byte_mux = devm_clk_hw_register_mux_parent_hws(dev, clk_name, + ((const struct clk_hw *[]){ + &pll_28nm->clk_hw, + indirect_path_div2, }), 2, CLK_SET_RATE_PARENT, pll_28nm->phy->pll_base + - REG_DSI_28nm_PHY_PLL_VREG_CFG, 1, 1, 0, NULL); - if (IS_ERR(hw)) - return PTR_ERR(hw); + REG_DSI_28nm_PHY_PLL_VREG_CFG, 1, 1, 0, NULL); + if (IS_ERR(byte_mux)) + return PTR_ERR(byte_mux); - snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->phy->id); - snprintf(parent1, 32, "dsi%dbyte_mux", pll_28nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, - parent1, CLK_SET_RATE_PARENT, 1, 4); + snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_28nm->phy->id); + hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name, + byte_mux, CLK_SET_RATE_PARENT, 1, 4); if (IS_ERR(hw)) return PTR_ERR(hw); provided_clocks[DSI_BYTE_PLL_CLK] = hw; @@ -627,31 +621,31 @@ static void dsi_28nm_dphy_set_timing(struct msm_dsi_phy *phy, void __iomem *base = phy->base; dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_0, - DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero)); + DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_1, - DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail)); + DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_2, - DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare)); + DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare)); if (timing->clk_zero & BIT(8)) dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_3, - DSI_28nm_PHY_TIMING_CTRL_3_CLK_ZERO_8); + DSI_28nm_PHY_TIMING_CTRL_3_CLK_ZERO_8); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_4, - DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit)); + DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_5, - DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero)); + DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_6, - DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare)); + DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_7, - DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail)); + DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_8, - DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst)); + DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_9, - DSI_28nm_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) | - DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure)); + DSI_28nm_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) | + DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_10, - DSI_28nm_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get)); + DSI_28nm_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get)); dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_11, - DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(0)); + DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(0)); } static void dsi_28nm_phy_regulator_enable_dcdc(struct msm_dsi_phy *phy) @@ -713,7 +707,8 @@ static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy, if (msm_dsi_dphy_timing_calc(timing, clk_req)) { DRM_DEV_ERROR(&phy->pdev->dev, - "%s: D-PHY timing calculation failed\n", __func__); + "%s: D-PHY timing calculation failed\n", + __func__); return -EINVAL; } @@ -769,14 +764,14 @@ static void dsi_28nm_phy_disable(struct msm_dsi_phy *phy) wmb(); } +static const struct regulator_bulk_data dsi_phy_28nm_regulators[] = { + { .supply = "vddio", .init_load_uA = 100000 }, +}; + const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs = { .has_phy_regulator = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vddio", 100000, 100}, - }, - }, + .regulator_data = dsi_phy_28nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators), .ops = { .enable = dsi_28nm_phy_enable, .disable = dsi_28nm_phy_disable, @@ -792,12 +787,8 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs = { .has_phy_regulator = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vddio", 100000, 100}, - }, - }, + .regulator_data = dsi_phy_28nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators), .ops = { .enable = dsi_28nm_phy_enable, .disable = dsi_28nm_phy_disable, @@ -813,12 +804,8 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs = { .has_phy_regulator = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = dsi_phy_28nm_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators), .ops = { .enable = dsi_28nm_phy_enable, .disable = dsi_28nm_phy_disable, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c index fc56cdcc9ad6445b22dc4538289c74f20ee7b950..26c08047e20c8c3bafdb1bf5269716fac7d6e1f9 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c @@ -104,29 +104,29 @@ static int dsi_pll_28nm_clk_set_rate(struct clk_hw *hw, unsigned long rate, fb_divider = (temp * VCO_PREF_DIV_RATIO) / val; fb_divider = fb_divider / 2 - 1; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_1, - fb_divider & 0xff); + fb_divider & 0xff); val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2); val |= (fb_divider >> 8) & 0x07; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2, - val); + val); val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3); val |= (VCO_PREF_DIV_RATIO - 1) & 0x3f; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3, - val); + val); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_6, - 0xf); + 0xf); val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8); val |= 0x7 << 4; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8, - val); + val); return 0; } @@ -206,7 +206,7 @@ static int dsi_pll_28nm_vco_prepare(struct clk_hw *hw) /* enable the PLL */ dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_0, - DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE); + DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE); locked = pll_28nm_poll_for_ready(pll_28nm, max_reads, timeout_us); @@ -367,23 +367,23 @@ static int dsi_28nm_pll_restore_state(struct msm_dsi_phy *phy) cached_state->vco_rate, 0); if (ret) { DRM_DEV_ERROR(&pll_28nm->phy->pdev->dev, - "restore vco rate failed. ret=%d\n", ret); + "restore vco rate failed. ret=%d\n", ret); return ret; } dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_10, - cached_state->postdiv3); + cached_state->postdiv3); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9, - cached_state->postdiv2); + cached_state->postdiv2); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8, - cached_state->postdiv1); + cached_state->postdiv1); return 0; } static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **provided_clocks) { - char *clk_name, *parent_name, *vco_name; + char clk_name[32]; struct clk_init_data vco_init = { .parent_data = &(const struct clk_parent_data) { .fw_name = "ref", @@ -404,20 +404,8 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov if (!bytediv) return -ENOMEM; - vco_name = devm_kzalloc(dev, 32, GFP_KERNEL); - if (!vco_name) - return -ENOMEM; - - parent_name = devm_kzalloc(dev, 32, GFP_KERNEL); - if (!parent_name) - return -ENOMEM; - - clk_name = devm_kzalloc(dev, 32, GFP_KERNEL); - if (!clk_name) - return -ENOMEM; - - snprintf(vco_name, 32, "dsi%dvco_clk", pll_28nm->phy->id); - vco_init.name = vco_name; + snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_28nm->phy->id); + vco_init.name = clk_name; pll_28nm->clk_hw.init = &vco_init; @@ -429,13 +417,14 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov bytediv->hw.init = &bytediv_init; bytediv->reg = pll_28nm->phy->pll_base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9; - snprintf(parent_name, 32, "dsi%dvco_clk", pll_28nm->phy->id); - snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->phy->id + 1); + snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_28nm->phy->id + 1); bytediv_init.name = clk_name; bytediv_init.ops = &clk_bytediv_ops; bytediv_init.flags = CLK_SET_RATE_PARENT; - bytediv_init.parent_names = (const char * const *) &parent_name; + bytediv_init.parent_hws = (const struct clk_hw*[]){ + &pll_28nm->clk_hw, + }; bytediv_init.num_parents = 1; /* DIV2 */ @@ -444,12 +433,12 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov return ret; provided_clocks[DSI_BYTE_PLL_CLK] = &bytediv->hw; - snprintf(clk_name, 32, "dsi%dpll", pll_28nm->phy->id + 1); + snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_28nm->phy->id + 1); /* DIV3 */ - hw = devm_clk_hw_register_divider(dev, clk_name, - parent_name, 0, pll_28nm->phy->pll_base + + hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + &pll_28nm->clk_hw, 0, pll_28nm->phy->pll_base + REG_DSI_28nm_8960_PHY_PLL_CTRL_10, - 0, 8, 0, NULL); + 0, 8, 0, NULL); if (IS_ERR(hw)) return PTR_ERR(hw); provided_clocks[DSI_PIXEL_PLL_CLK] = hw; @@ -489,29 +478,29 @@ static void dsi_28nm_dphy_set_timing(struct msm_dsi_phy *phy, void __iomem *base = phy->base; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_0, - DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero)); + DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_1, - DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail)); + DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_2, - DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare)); + DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_3, 0x0); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_4, - DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit)); + DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_5, - DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero)); + DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_6, - DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare)); + DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_7, - DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail)); + DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_8, - DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst)); + DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_9, - DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) | - DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure)); + DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) | + DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_10, - DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get)); + DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get)); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_11, - DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(0)); + DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(0)); } static void dsi_28nm_phy_regulator_init(struct msm_dsi_phy *phy) @@ -523,7 +512,7 @@ static void dsi_28nm_phy_regulator_init(struct msm_dsi_phy *phy) dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2, 1); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3, 0); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4, - 0x100); + 0x100); } static void dsi_28nm_phy_regulator_ctrl(struct msm_dsi_phy *phy) @@ -544,7 +533,7 @@ static void dsi_28nm_phy_calibration(struct msm_dsi_phy *phy) int i = 5000; dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CAL_PWR_CFG, - 0x3); + 0x3); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_2, 0x0); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_1, 0x5a); @@ -577,11 +566,11 @@ static void dsi_28nm_phy_lane_config(struct msm_dsi_phy *phy) dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_1(i), 0x45); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_2(i), 0x00); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_DATAPATH(i), - 0x00); + 0x00); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_0(i), - 0x01); + 0x01); dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_1(i), - 0x66); + 0x66); } dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_CFG_0, 0x40); @@ -602,7 +591,8 @@ static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy, if (msm_dsi_dphy_timing_calc(timing, clk_req)) { DRM_DEV_ERROR(&phy->pdev->dev, - "%s: D-PHY timing calculation failed\n", __func__); + "%s: D-PHY timing calculation failed\n", + __func__); return -EINVAL; } @@ -648,14 +638,14 @@ static void dsi_28nm_phy_disable(struct msm_dsi_phy *phy) wmb(); } +static const struct regulator_bulk_data dsi_phy_28nm_8960_regulators[] = { + { .supply = "vddio", .init_load_uA = 100000 }, /* 1.8 V */ +}; + const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs = { .has_phy_regulator = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vddio", 100000, 100}, /* 1.8 V */ - }, - }, + .regulator_data = dsi_phy_28nm_8960_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_28nm_8960_regulators), .ops = { .enable = dsi_28nm_phy_enable, .disable = dsi_28nm_phy_disable, diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index 66ed1919a1db96d0fbc5f357c378a0301bfa8afe..9e7fa7d88ead227cae470a24e7e117173df428fb 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -176,19 +176,19 @@ static void dsi_pll_ssc_commit(struct dsi_pll_7nm *pll, struct dsi_pll_config *c pr_debug("SSC is enabled\n"); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_LOW_1, - config->ssc_stepsize & 0xff); + config->ssc_stepsize & 0xff); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_HIGH_1, - config->ssc_stepsize >> 8); + config->ssc_stepsize >> 8); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_LOW_1, - config->ssc_div_per & 0xff); + config->ssc_div_per & 0xff); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_HIGH_1, - config->ssc_div_per >> 8); + config->ssc_div_per >> 8); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_ADJPER_LOW_1, - config->ssc_adj_per & 0xff); + config->ssc_adj_per & 0xff); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_ADJPER_HIGH_1, - config->ssc_adj_per >> 8); + config->ssc_adj_per >> 8); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_CONTROL, - SSC_EN | (config->ssc_center ? SSC_CENTER : 0)); + SSC_EN | (config->ssc_center ? SSC_CENTER : 0)); } } @@ -208,7 +208,7 @@ static void dsi_pll_config_hzindep_reg(struct dsi_pll_7nm *pll) } dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE_1, - analog_controls_five_1); + analog_controls_five_1); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_VCO_CONFIG_1, vco_config_1); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE, 0x01); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_TWO, 0x03); @@ -245,17 +245,20 @@ static void dsi_pll_commit(struct dsi_pll_7nm *pll, struct dsi_pll_config *confi void __iomem *base = pll->phy->pll_base; dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CORE_INPUT_OVERRIDE, 0x12); - dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_1, config->decimal_div_start); + dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_1, + config->decimal_div_start); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_LOW_1, - config->frac_div_start & 0xff); + config->frac_div_start & 0xff); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_MID_1, - (config->frac_div_start & 0xff00) >> 8); + (config->frac_div_start & 0xff00) >> 8); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_HIGH_1, - (config->frac_div_start & 0x30000) >> 16); + (config->frac_div_start & 0x30000) >> 16); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCKDET_RATE_1, 0x40); dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCK_DELAY, 0x06); - dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1, pll->phy->cphy_mode ? 0x00 : 0x10); - dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS, config->pll_clock_inverters); + dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1, + pll->phy->cphy_mode ? 0x00 : 0x10); + dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS, + config->pll_clock_inverters); } static int dsi_pll_7nm_vco_set_rate(struct clk_hw *hw, unsigned long rate, @@ -341,7 +344,7 @@ static void dsi_pll_enable_global_clk(struct dsi_pll_7nm *pll) data = dsi_phy_read(pll->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1); dsi_phy_write(pll->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1, - data | BIT(5) | BIT(4)); + data | BIT(5) | BIT(4)); } static void dsi_pll_phy_dig_reset(struct dsi_pll_7nm *pll) @@ -500,7 +503,7 @@ static void dsi_7nm_pll_save_state(struct msm_dsi_phy *phy) u32 cmn_clk_cfg0, cmn_clk_cfg1; cached->pll_out_div = dsi_phy_read(pll_7nm->phy->pll_base + - REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE); + REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE); cached->pll_out_div &= 0x3; cmn_clk_cfg0 = dsi_phy_read(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG0); @@ -529,7 +532,7 @@ static int dsi_7nm_pll_restore_state(struct msm_dsi_phy *phy) dsi_phy_write(pll_7nm->phy->pll_base + REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE, val); dsi_phy_write(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG0, - cached->bit_clk_div | (cached->pix_clk_div << 4)); + cached->bit_clk_div | (cached->pix_clk_div << 4)); val = dsi_phy_read(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG1); val &= ~0x3; @@ -585,65 +588,60 @@ static int dsi_7nm_set_usecase(struct msm_dsi_phy *phy) */ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provided_clocks) { - char clk_name[32], parent[32], vco_name[32]; - char parent2[32]; + char clk_name[32]; struct clk_init_data vco_init = { .parent_data = &(const struct clk_parent_data) { .fw_name = "ref", }, .num_parents = 1, - .name = vco_name, + .name = clk_name, .flags = CLK_IGNORE_UNUSED, .ops = &clk_ops_dsi_pll_7nm_vco, }; struct device *dev = &pll_7nm->phy->pdev->dev; - struct clk_hw *hw; + struct clk_hw *hw, *pll_out_div, *pll_bit, *pll_by_2_bit; + struct clk_hw *pll_post_out_div, *phy_pll_out_dsi_parent; int ret; DBG("DSI%d", pll_7nm->phy->id); - snprintf(vco_name, 32, "dsi%dvco_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_7nm->phy->id); pll_7nm->clk_hw.init = &vco_init; ret = devm_clk_hw_register(dev, &pll_7nm->clk_hw); if (ret) return ret; - snprintf(clk_name, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%dvco_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_out_div_clk", pll_7nm->phy->id); - hw = devm_clk_hw_register_divider(dev, clk_name, - parent, CLK_SET_RATE_PARENT, - pll_7nm->phy->pll_base + - REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE, - 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_out_div = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + &pll_7nm->clk_hw, CLK_SET_RATE_PARENT, + pll_7nm->phy->pll_base + + REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE, + 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL); + if (IS_ERR(pll_out_div)) { + ret = PTR_ERR(pll_out_div); goto fail; } - snprintf(clk_name, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_bit_clk", pll_7nm->phy->id); /* BIT CLK: DIV_CTRL_3_0 */ - hw = devm_clk_hw_register_divider(dev, clk_name, parent, - CLK_SET_RATE_PARENT, - pll_7nm->phy->base + - REG_DSI_7nm_PHY_CMN_CLK_CFG0, - 0, 4, CLK_DIVIDER_ONE_BASED, - &pll_7nm->postdiv_lock); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_bit = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + pll_out_div, CLK_SET_RATE_PARENT, + pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG0, + 0, 4, CLK_DIVIDER_ONE_BASED, &pll_7nm->postdiv_lock); + if (IS_ERR(pll_bit)) { + ret = PTR_ERR(pll_bit); goto fail; } - snprintf(clk_name, 32, "dsi%d_phy_pll_out_byteclk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_byteclk", pll_7nm->phy->id); /* DSI Byte clock = VCO_CLK / OUT_DIV / BIT_DIV / 8 */ - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - CLK_SET_RATE_PARENT, 1, - pll_7nm->phy->cphy_mode ? 7 : 8); + hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name, + pll_bit, CLK_SET_RATE_PARENT, 1, + pll_7nm->phy->cphy_mode ? 7 : 8); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; @@ -651,25 +649,25 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide provided_clocks[DSI_BYTE_PLL_CLK] = hw; - snprintf(clk_name, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id); - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, - 0, 1, 2); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_by_2_bit = devm_clk_hw_register_fixed_factor_parent_hw(dev, + clk_name, pll_bit, 0, 1, 2); + if (IS_ERR(pll_by_2_bit)) { + ret = PTR_ERR(pll_by_2_bit); goto fail; } - snprintf(clk_name, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); if (pll_7nm->phy->cphy_mode) - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 2, 7); + pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw( + dev, clk_name, pll_out_div, 0, 2, 7); else - hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 1, 4); - if (IS_ERR(hw)) { - ret = PTR_ERR(hw); + pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw( + dev, clk_name, pll_out_div, 0, 1, 4); + if (IS_ERR(pll_post_out_div)) { + ret = PTR_ERR(pll_post_out_div); goto fail; } @@ -682,34 +680,32 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide data = dsi_phy_read(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1); dsi_phy_write(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1, data | 3); - snprintf(parent, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id); + phy_pll_out_dsi_parent = pll_post_out_div; } else { - snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); - snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id); - snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id); - - hw = devm_clk_hw_register_mux(dev, clk_name, - ((const char *[]){ - parent, parent2, - }), 2, 0, pll_7nm->phy->base + + snprintf(clk_name, sizeof(clk_name), "dsi%d_pclk_mux", pll_7nm->phy->id); + + hw = devm_clk_hw_register_mux_parent_hws(dev, clk_name, + ((const struct clk_hw *[]){ + pll_bit, + pll_by_2_bit, + }), 2, 0, pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1, - 0, 1, 0, NULL); + 0, 1, 0, NULL); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; } - snprintf(parent, 32, "dsi%d_pclk_mux", pll_7nm->phy->id); + phy_pll_out_dsi_parent = hw; } - snprintf(clk_name, 32, "dsi%d_phy_pll_out_dsiclk", pll_7nm->phy->id); + snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_dsiclk", pll_7nm->phy->id); /* PIX CLK DIV : DIV_CTRL_7_4*/ - hw = devm_clk_hw_register_divider(dev, clk_name, parent, - 0, pll_7nm->phy->base + - REG_DSI_7nm_PHY_CMN_CLK_CFG0, - 4, 4, CLK_DIVIDER_ONE_BASED, - &pll_7nm->postdiv_lock); + hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name, + phy_pll_out_dsi_parent, 0, + pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG0, + 4, 4, CLK_DIVIDER_ONE_BASED, &pll_7nm->postdiv_lock); if (IS_ERR(hw)) { ret = PTR_ERR(hw); goto fail; @@ -841,7 +837,7 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, ret = msm_dsi_dphy_timing_calc_v4(timing, clk_req); if (ret) { DRM_DEV_ERROR(&phy->pdev->dev, - "%s: PHY timing calculation failed\n", __func__); + "%s: PHY timing calculation failed\n", __func__); return -EINVAL; } @@ -960,10 +956,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0, 0x00); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4, timing->hs_exit); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5, - timing->shared_timings.clk_pre); + timing->shared_timings.clk_pre); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6, timing->clk_prepare); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7, - timing->shared_timings.clk_post); + timing->shared_timings.clk_post); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8, timing->hs_rqst); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9, 0x02); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04); @@ -982,9 +978,9 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11, 0x00); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_12, - timing->shared_timings.clk_pre); + timing->shared_timings.clk_pre); dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_13, - timing->shared_timings.clk_post); + timing->shared_timings.clk_post); } /* DSI lane settings */ @@ -1036,14 +1032,18 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) DBG("DSI%d PHY disabled", phy->id); } +static const struct regulator_bulk_data dsi_phy_7nm_36mA_regulators[] = { + { .supply = "vdds", .init_load_uA = 36000 }, +}; + +static const struct regulator_bulk_data dsi_phy_7nm_37750uA_regulators[] = { + { .supply = "vdds", .init_load_uA = 37550 }, +}; + const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vdds", 36000, 32}, - }, - }, + .regulator_data = dsi_phy_7nm_36mA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_7nm_36mA_regulators), .ops = { .enable = dsi_7nm_phy_enable, .disable = dsi_7nm_phy_disable, @@ -1065,12 +1065,8 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vdds", 36000, 32}, - }, - }, + .regulator_data = dsi_phy_7nm_36mA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_7nm_36mA_regulators), .ops = { .enable = dsi_7nm_phy_enable, .disable = dsi_7nm_phy_disable, @@ -1087,12 +1083,8 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs = { const struct msm_dsi_phy_cfg dsi_phy_7nm_7280_cfgs = { .has_phy_lane = true, - .reg_cfg = { - .num = 1, - .regs = { - {"vdds", 37550, 0}, - }, - }, + .regulator_data = dsi_phy_7nm_37750uA_regulators, + .num_regulators = ARRAY_SIZE(dsi_phy_7nm_37750uA_regulators), .ops = { .enable = dsi_7nm_phy_enable, .disable = dsi_7nm_phy_disable, diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c index b06d9d25a1894fe61dce6af3de4ab0406ed30a99..4dd0554166204568bce2b4d7c00ccc466465ddbe 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c @@ -691,15 +691,13 @@ static const struct clk_ops hdmi_8996_pll_ops = { .is_enabled = hdmi_8996_pll_is_enabled, }; -static const char * const hdmi_pll_parents[] = { - "xo", -}; - static const struct clk_init_data pll_init = { .name = "hdmipll", .ops = &hdmi_8996_pll_ops, - .parent_names = hdmi_pll_parents, - .num_parents = ARRAY_SIZE(hdmi_pll_parents), + .parent_data = (const struct clk_parent_data[]){ + { .fw_name = "xo", .name = "xo_board" }, + }, + .num_parents = 1, .flags = CLK_IGNORE_UNUSED, }; @@ -707,8 +705,7 @@ int msm_hdmi_pll_8996_init(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct hdmi_pll_8996 *pll; - struct clk *clk; - int i; + int i, ret; pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL); if (!pll) @@ -735,10 +732,16 @@ int msm_hdmi_pll_8996_init(struct platform_device *pdev) } pll->clk_hw.init = &pll_init; - clk = devm_clk_register(dev, &pll->clk_hw); - if (IS_ERR(clk)) { + ret = devm_clk_hw_register(dev, &pll->clk_hw); + if (ret) { DRM_DEV_ERROR(dev, "failed to register pll clock\n"); - return -EINVAL; + return ret; + } + + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, &pll->clk_hw); + if (ret) { + DRM_DEV_ERROR(dev, "%s: failed to register clk provider: %d\n", __func__, ret); + return ret; } return 0; diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c index 7d2dab260f86741ad8c71b7463de40d0f6b000b2..95f4374ae21c2212c3dfa7c19e53627a71d6da37 100644 --- a/drivers/gpu/drm/msm/msm_debugfs.c +++ b/drivers/gpu/drm/msm/msm_debugfs.c @@ -7,6 +7,7 @@ #ifdef CONFIG_DEBUG_FS #include +#include #include #include @@ -326,6 +327,13 @@ void msm_debugfs_init(struct drm_minor *minor) if (priv->kms && priv->kms->funcs->debugfs_init) priv->kms->funcs->debugfs_init(priv->kms, minor); + +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("fail_gem_alloc", minor->debugfs_root, + &fail_gem_alloc); + fault_create_debugfs_attr("fail_gem_iova", minor->debugfs_root, + &fail_gem_iova); +#endif } #endif diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 226d8d4629d220ba26f639eb61ebd07d9b50cec0..28034c21f6bcd6bb229342e88ab477807b821d27 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -78,6 +79,11 @@ static bool modeset = true; MODULE_PARM_DESC(modeset, "Use kernel modesetting [KMS] (1=on (default), 0=disable)"); module_param(modeset, bool, 0600); +#ifdef CONFIG_FAULT_INJECTION +DECLARE_FAULT_ATTR(fail_gem_alloc); +DECLARE_FAULT_ATTR(fail_gem_iova); +#endif + static irqreturn_t msm_irq(int irq, void *arg) { struct drm_device *dev = arg; @@ -418,14 +424,18 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) INIT_LIST_HEAD(&priv->objects); mutex_init(&priv->obj_lock); - INIT_LIST_HEAD(&priv->inactive_willneed); - INIT_LIST_HEAD(&priv->inactive_dontneed); - INIT_LIST_HEAD(&priv->inactive_unpinned); - mutex_init(&priv->mm_lock); + /* + * Initialize the LRUs: + */ + mutex_init(&priv->lru.lock); + drm_gem_lru_init(&priv->lru.unbacked, &priv->lru.lock); + drm_gem_lru_init(&priv->lru.pinned, &priv->lru.lock); + drm_gem_lru_init(&priv->lru.willneed, &priv->lru.lock); + drm_gem_lru_init(&priv->lru.dontneed, &priv->lru.lock); /* Teach lockdep about lock ordering wrt. shrinker: */ fs_reclaim_acquire(GFP_KERNEL); - might_lock(&priv->mm_lock); + might_lock(&priv->lru.lock); fs_reclaim_release(GFP_KERNEL); drm_mode_config_init(ddev); @@ -469,6 +479,8 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) } } + drm_helper_move_panel_connectors_to_head(ddev); + ddev->mode_config.funcs = &mode_config_funcs; ddev->mode_config.helper_private = &mode_config_helper_funcs; @@ -697,6 +709,9 @@ static int msm_ioctl_gem_new(struct drm_device *dev, void *data, flags |= MSM_BO_WC; } + if (should_fail(&fail_gem_alloc, args->size)) + return -ENOMEM; + return msm_gem_new_handle(dev, file, args->size, args->flags, &args->handle, NULL); } @@ -758,6 +773,9 @@ static int msm_ioctl_gem_info_iova(struct drm_device *dev, if (!priv->gpu) return -EINVAL; + if (should_fail(&fail_gem_iova, obj->size)) + return -ENOMEM; + /* * Don't pin the memory here - just get an address so that userspace can * be productive @@ -779,6 +797,9 @@ static int msm_ioctl_gem_info_set_iova(struct drm_device *dev, if (priv->gpu->aspace == ctx->aspace) return -EOPNOTSUPP; + if (should_fail(&fail_gem_iova, obj->size)) + return -ENOMEM; + return msm_gem_set_iova(obj, ctx->aspace, iova); } @@ -883,13 +904,13 @@ static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id, * retired, so if the fence is not found it means there is nothing * to wait for */ - ret = mutex_lock_interruptible(&queue->lock); + ret = mutex_lock_interruptible(&queue->idr_lock); if (ret) return ret; fence = idr_find(&queue->fence_idr, fence_id); if (fence) fence = dma_fence_get_rcu(fence); - mutex_unlock(&queue->lock); + mutex_unlock(&queue->idr_lock); if (!fence) return 0; diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 7d5fb0fc22ddd8ba99bb0c5366f90469bc547364..b2ea262296a4fb451ab4373530ac7d98edd1044c 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -33,6 +33,13 @@ #include #include +#ifdef CONFIG_FAULT_INJECTION +extern struct fault_attr fail_gem_alloc; +extern struct fault_attr fail_gem_iova; +#else +# define should_fail(attr, size) 0 +#endif + struct msm_kms; struct msm_gpu; struct msm_mmu; @@ -95,11 +102,6 @@ struct msm_drm_thread { struct kthread_worker *worker; }; -/* DSC config */ -struct msm_display_dsc_config { - struct drm_dsc_config *drm; -}; - struct msm_drm_private { struct drm_device *dev; @@ -141,28 +143,60 @@ struct msm_drm_private { struct mutex obj_lock; /** - * LRUs of inactive GEM objects. Every bo is either in one of the - * inactive lists (depending on whether or not it is shrinkable) or - * gpu->active_list (for the gpu it is active on[1]), or transiently - * on a temporary list as the shrinker is running. + * lru: * - * Note that inactive_willneed also contains pinned and vmap'd bos, - * but the number of pinned-but-not-active objects is small (scanout - * buffers, ringbuffer, etc). + * The various LRU's that a GEM object is in at various stages of + * it's lifetime. Objects start out in the unbacked LRU. When + * pinned (for scannout or permanently mapped GPU buffers, like + * ringbuffer, memptr, fw, etc) it moves to the pinned LRU. When + * unpinned, it moves into willneed or dontneed LRU depending on + * madvise state. When backing pages are evicted (willneed) or + * purged (dontneed) it moves back into the unbacked LRU. * - * These lists are protected by mm_lock (which should be acquired - * before per GEM object lock). One should *not* hold mm_lock in - * get_pages()/vmap()/etc paths, as they can trigger the shrinker. - * - * [1] if someone ever added support for the old 2d cores, there could be - * more than one gpu object + * The dontneed LRU is considered by the shrinker for objects + * that are candidate for purging, and the willneed LRU is + * considered for objects that could be evicted. */ - struct list_head inactive_willneed; /* inactive + potentially unpin/evictable */ - struct list_head inactive_dontneed; /* inactive + shrinkable */ - struct list_head inactive_unpinned; /* inactive + purged or unpinned */ - long shrinkable_count; /* write access under mm_lock */ - long evictable_count; /* write access under mm_lock */ - struct mutex mm_lock; + struct { + /** + * unbacked: + * + * The LRU for GEM objects without backing pages allocated. + * This mostly exists so that objects are always is one + * LRU. + */ + struct drm_gem_lru unbacked; + + /** + * pinned: + * + * The LRU for pinned GEM objects + */ + struct drm_gem_lru pinned; + + /** + * willneed: + * + * The LRU for unpinned GEM objects which are in madvise + * WILLNEED state (ie. can be evicted) + */ + struct drm_gem_lru willneed; + + /** + * dontneed: + * + * The LRU for unpinned GEM objects which are in madvise + * DONTNEED state (ie. can be purged) + */ + struct drm_gem_lru dontneed; + + /** + * lock: + * + * Protects manipulation of all of the LRUs. + */ + struct mutex lock; + } lru; struct workqueue_struct *wq; @@ -289,7 +323,7 @@ void msm_dsi_snapshot(struct msm_disp_state *disp_state, struct msm_dsi *msm_dsi bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi); bool msm_dsi_is_bonded_dsi(struct msm_dsi *msm_dsi); bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi); -struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi); +struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi); #else static inline void __init msm_dsi_register(void) { @@ -319,7 +353,7 @@ static inline bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi) return false; } -static inline struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi) +static inline struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi) { return NULL; } @@ -432,6 +466,8 @@ void __iomem *msm_ioremap_size(struct platform_device *pdev, const char *name, phys_addr_t *size); void __iomem *msm_ioremap_quiet(struct platform_device *pdev, const char *name); +struct icc_path *msm_icc_get(struct device *dev, const char *name); + #define msm_writel(data, addr) writel((data), (addr)) #define msm_readl(addr) readl((addr)) diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 8ddbd2e001d46925fa3f07a7c98b3841f386c662..1dee0d18abbb601a6f9e40447dda39b0e4e8bba1 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -19,7 +19,7 @@ #include "msm_gpu.h" #include "msm_mmu.h" -static void update_inactive(struct msm_gem_object *msm_obj); +static void update_lru(struct drm_gem_object *obj); static dma_addr_t physaddr(struct drm_gem_object *obj) { @@ -97,7 +97,7 @@ static struct page **get_pages(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); if (!msm_obj->pages) { struct drm_device *dev = obj->dev; @@ -132,7 +132,7 @@ static struct page **get_pages(struct drm_gem_object *obj) if (msm_obj->flags & MSM_BO_WC) sync_for_device(msm_obj); - update_inactive(msm_obj); + update_lru(obj); } return msm_obj->pages; @@ -174,40 +174,45 @@ static void put_pages(struct drm_gem_object *obj) put_pages_vram(obj); msm_obj->pages = NULL; + update_lru(obj); } } -struct page **msm_gem_get_pages(struct drm_gem_object *obj) +static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); struct page **p; - msm_gem_lock(obj); + msm_gem_assert_locked(obj); if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) { - msm_gem_unlock(obj); return ERR_PTR(-EBUSY); } p = get_pages(obj); - if (!IS_ERR(p)) { - msm_obj->pin_count++; - update_inactive(msm_obj); + to_msm_bo(obj)->pin_count++; + update_lru(obj); } - msm_gem_unlock(obj); return p; } -void msm_gem_put_pages(struct drm_gem_object *obj) +struct page **msm_gem_pin_pages(struct drm_gem_object *obj) { - struct msm_gem_object *msm_obj = to_msm_bo(obj); + struct page **p; msm_gem_lock(obj); - msm_obj->pin_count--; - GEM_WARN_ON(msm_obj->pin_count < 0); - update_inactive(msm_obj); + p = msm_gem_pin_pages_locked(obj); + msm_gem_unlock(obj); + + return p; +} + +void msm_gem_unpin_pages(struct drm_gem_object *obj) +{ + msm_gem_lock(obj); + msm_gem_unpin_locked(obj); msm_gem_unlock(obj); } @@ -273,7 +278,7 @@ static uint64_t mmap_offset(struct drm_gem_object *obj) struct drm_device *dev = obj->dev; int ret; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); /* Make it mmapable */ ret = drm_gem_create_mmap_offset(obj); @@ -302,7 +307,7 @@ static struct msm_gem_vma *add_vma(struct drm_gem_object *obj, struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); vma = kzalloc(sizeof(*vma), GFP_KERNEL); if (!vma) @@ -321,7 +326,7 @@ static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj, struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); list_for_each_entry(vma, &msm_obj->vmas, list) { if (vma->aspace == aspace) @@ -352,7 +357,7 @@ put_iova_spaces(struct drm_gem_object *obj, bool close) struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); list_for_each_entry(vma, &msm_obj->vmas, list) { if (vma->aspace) { @@ -370,7 +375,7 @@ put_iova_vmas(struct drm_gem_object *obj) struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_gem_vma *vma, *tmp; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) { del_vma(vma); @@ -383,7 +388,7 @@ static struct msm_gem_vma *get_vma_locked(struct drm_gem_object *obj, { struct msm_gem_vma *vma; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); vma = lookup_vma(obj, aspace); @@ -423,19 +428,18 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma) if (msm_obj->flags & MSM_BO_CACHED_COHERENT) prot |= IOMMU_CACHE; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) return -EBUSY; - pages = get_pages(obj); + pages = msm_gem_pin_pages_locked(obj); if (IS_ERR(pages)) return PTR_ERR(pages); ret = msm_gem_map_vma(vma->aspace, vma, prot, msm_obj->sgt, obj->size); - - if (!ret) - msm_obj->pin_count++; + if (ret) + msm_gem_unpin_locked(obj); return ret; } @@ -444,12 +448,12 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); msm_obj->pin_count--; GEM_WARN_ON(msm_obj->pin_count < 0); - update_inactive(msm_obj); + update_lru(obj); } struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj, @@ -465,7 +469,7 @@ static int get_and_pin_iova_range_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma; int ret; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); vma = get_vma_locked(obj, aspace, range_start, range_end); if (IS_ERR(vma)) @@ -626,7 +630,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv) struct msm_gem_object *msm_obj = to_msm_bo(obj); int ret = 0; - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); if (obj->import_attach) return ERR_PTR(-ENODEV); @@ -658,7 +662,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv) goto fail; } - update_inactive(msm_obj); + update_lru(obj); } return msm_obj->vaddr; @@ -699,7 +703,7 @@ void msm_gem_put_vaddr_locked(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); GEM_WARN_ON(msm_obj->vmap_count < 1); msm_obj->vmap_count--; @@ -729,8 +733,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv) /* If the obj is inactive, we might need to move it * between inactive lists */ - if (msm_obj->active_count == 0) - update_inactive(msm_obj); + update_lru(obj); msm_gem_unlock(obj); @@ -742,7 +745,7 @@ void msm_gem_purge(struct drm_gem_object *obj) struct drm_device *dev = obj->dev; struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); GEM_WARN_ON(!is_purgeable(msm_obj)); /* Get rid of any iommu mapping(s): */ @@ -757,7 +760,6 @@ void msm_gem_purge(struct drm_gem_object *obj) put_iova_vmas(obj); msm_obj->madv = __MSM_MADV_PURGED; - update_inactive(msm_obj); drm_gem_free_mmap_offset(obj); @@ -780,10 +782,8 @@ void msm_gem_evict(struct drm_gem_object *obj) struct drm_device *dev = obj->dev; struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); GEM_WARN_ON(is_unevictable(msm_obj)); - GEM_WARN_ON(!msm_obj->evictable); - GEM_WARN_ON(msm_obj->active_count); /* Get rid of any iommu mapping(s): */ put_iova_spaces(obj, false); @@ -791,15 +791,13 @@ void msm_gem_evict(struct drm_gem_object *obj) drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); put_pages(obj); - - update_inactive(msm_obj); } void msm_gem_vunmap(struct drm_gem_object *obj) { struct msm_gem_object *msm_obj = to_msm_bo(obj); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(obj); if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj))) return; @@ -808,66 +806,37 @@ void msm_gem_vunmap(struct drm_gem_object *obj) msm_obj->vaddr = NULL; } -void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu) +static void update_lru(struct drm_gem_object *obj) { - struct msm_gem_object *msm_obj = to_msm_bo(obj); struct msm_drm_private *priv = obj->dev->dev_private; - - might_sleep(); - GEM_WARN_ON(!msm_gem_is_locked(obj)); - GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED); - GEM_WARN_ON(msm_obj->dontneed); - - if (msm_obj->active_count++ == 0) { - mutex_lock(&priv->mm_lock); - if (msm_obj->evictable) - mark_unevictable(msm_obj); - list_move_tail(&msm_obj->mm_list, &gpu->active_list); - mutex_unlock(&priv->mm_lock); - } -} - -void msm_gem_active_put(struct drm_gem_object *obj) -{ struct msm_gem_object *msm_obj = to_msm_bo(obj); - might_sleep(); - GEM_WARN_ON(!msm_gem_is_locked(obj)); + msm_gem_assert_locked(&msm_obj->base); - if (--msm_obj->active_count == 0) { - update_inactive(msm_obj); + if (!msm_obj->pages) { + GEM_WARN_ON(msm_obj->pin_count); + GEM_WARN_ON(msm_obj->vmap_count); + + drm_gem_lru_move_tail(&priv->lru.unbacked, obj); + } else if (msm_obj->pin_count || msm_obj->vmap_count) { + drm_gem_lru_move_tail(&priv->lru.pinned, obj); + } else if (msm_obj->madv == MSM_MADV_WILLNEED) { + drm_gem_lru_move_tail(&priv->lru.willneed, obj); + } else { + GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED); + + drm_gem_lru_move_tail(&priv->lru.dontneed, obj); } } -static void update_inactive(struct msm_gem_object *msm_obj) +bool msm_gem_active(struct drm_gem_object *obj) { - struct msm_drm_private *priv = msm_obj->base.dev->dev_private; - - GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); + msm_gem_assert_locked(obj); - if (msm_obj->active_count != 0) - return; - - mutex_lock(&priv->mm_lock); - - if (msm_obj->dontneed) - mark_unpurgeable(msm_obj); - if (msm_obj->evictable) - mark_unevictable(msm_obj); - - list_del(&msm_obj->mm_list); - if ((msm_obj->madv == MSM_MADV_WILLNEED) && msm_obj->sgt) { - list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); - mark_evictable(msm_obj); - } else if (msm_obj->madv == MSM_MADV_DONTNEED) { - list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed); - mark_purgeable(msm_obj); - } else { - GEM_WARN_ON((msm_obj->madv != __MSM_MADV_PURGED) && msm_obj->sgt); - list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); - } + if (to_msm_bo(obj)->pin_count) + return true; - mutex_unlock(&priv->mm_lock); + return !dma_resv_test_signaled(obj->resv, dma_resv_usage_rw(true)); } int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout) @@ -910,7 +879,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, stats->all.count++; stats->all.size += obj->size; - if (is_active(msm_obj)) { + if (msm_gem_active(obj)) { stats->active.count++; stats->active.size += obj->size; } @@ -938,7 +907,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m, } seq_printf(m, "%08x: %c %2d (%2d) %08llx %p", - msm_obj->flags, is_active(msm_obj) ? 'A' : 'I', + msm_obj->flags, msm_gem_active(obj) ? 'A' : 'I', obj->name, kref_read(&obj->refcount), off, msm_obj->vaddr); @@ -1015,15 +984,6 @@ static void msm_gem_free_object(struct drm_gem_object *obj) list_del(&msm_obj->node); mutex_unlock(&priv->obj_lock); - mutex_lock(&priv->mm_lock); - if (msm_obj->dontneed) - mark_unpurgeable(msm_obj); - list_del(&msm_obj->mm_list); - mutex_unlock(&priv->mm_lock); - - /* object should not be on active list: */ - GEM_WARN_ON(is_active(msm_obj)); - put_iova_spaces(obj, true); if (obj->import_attach) { @@ -1183,13 +1143,6 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32 to_msm_bo(obj)->vram_node = &vma->node; - /* Call chain get_pages() -> update_inactive() tries to - * access msm_obj->mm_list, but it is not initialized yet. - * To avoid NULL pointer dereference error, initialize - * mm_list to be empty. - */ - INIT_LIST_HEAD(&msm_obj->mm_list); - msm_gem_lock(obj); pages = get_pages(obj); msm_gem_unlock(obj); @@ -1212,9 +1165,7 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32 mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER); } - mutex_lock(&priv->mm_lock); - list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); - mutex_unlock(&priv->mm_lock); + drm_gem_lru_move_tail(&priv->lru.unbacked, obj); mutex_lock(&priv->obj_lock); list_add_tail(&msm_obj->node, &priv->objects); @@ -1270,9 +1221,7 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev, msm_gem_unlock(obj); - mutex_lock(&priv->mm_lock); - list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned); - mutex_unlock(&priv->mm_lock); + drm_gem_lru_move_tail(&priv->lru.pinned, obj); mutex_lock(&priv->obj_lock); list_add_tail(&msm_obj->node, &priv->objects); diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 432032ad4aeda984822bea1aec2a74f54a657150..c4844cf3a585c2bb7a90fec3b9391431c2827b64 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -93,16 +93,6 @@ struct msm_gem_object { */ uint8_t madv; - /** - * Is object on inactive_dontneed list (ie. counted in priv->shrinkable_count)? - */ - bool dontneed : 1; - - /** - * Is object evictable (ie. counted in priv->evictable_count)? - */ - bool evictable : 1; - /** * count of active vmap'ing */ @@ -114,17 +104,6 @@ struct msm_gem_object { */ struct list_head node; - /** - * An object is either: - * inactive - on priv->inactive_dontneed or priv->inactive_willneed - * (depending on purgeability status) - * active - on one one of the gpu's active_list.. well, at - * least for now we don't have (I don't think) hw sync between - * 2d and 3d one devices which have both, meaning we need to - * block on submit if a bo is already on other ring - */ - struct list_head mm_list; - struct page **pages; struct sg_table *sgt; void *vaddr; @@ -138,7 +117,6 @@ struct msm_gem_object { char name[32]; /* Identifier to print for the debugfs files */ - int active_count; int pin_count; }; #define to_msm_bo(x) container_of(x, struct msm_gem_object, base) @@ -159,8 +137,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova); void msm_gem_unpin_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace); -struct page **msm_gem_get_pages(struct drm_gem_object *obj); -void msm_gem_put_pages(struct drm_gem_object *obj); +struct page **msm_gem_pin_pages(struct drm_gem_object *obj); +void msm_gem_unpin_pages(struct drm_gem_object *obj); int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args); int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, @@ -171,8 +149,7 @@ void *msm_gem_get_vaddr_active(struct drm_gem_object *obj); void msm_gem_put_vaddr_locked(struct drm_gem_object *obj); void msm_gem_put_vaddr(struct drm_gem_object *obj); int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv); -void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu); -void msm_gem_active_put(struct drm_gem_object *obj); +bool msm_gem_active(struct drm_gem_object *obj); int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout); int msm_gem_cpu_fini(struct drm_gem_object *obj); int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file, @@ -208,12 +185,6 @@ msm_gem_lock(struct drm_gem_object *obj) dma_resv_lock(obj->resv, NULL); } -static inline bool __must_check -msm_gem_trylock(struct drm_gem_object *obj) -{ - return dma_resv_trylock(obj->resv); -} - static inline int msm_gem_lock_interruptible(struct drm_gem_object *obj) { @@ -226,8 +197,8 @@ msm_gem_unlock(struct drm_gem_object *obj) dma_resv_unlock(obj->resv); } -static inline bool -msm_gem_is_locked(struct drm_gem_object *obj) +static inline void +msm_gem_assert_locked(struct drm_gem_object *obj) { /* * Destroying the object is a special case.. msm_gem_free_object() @@ -241,13 +212,10 @@ msm_gem_is_locked(struct drm_gem_object *obj) * Unfortunately lockdep is not aware of this detail. So when the * refcount drops to zero, we pretend it is already locked. */ - return dma_resv_is_locked(obj->resv) || (kref_read(&obj->refcount) == 0); -} - -static inline bool is_active(struct msm_gem_object *msm_obj) -{ - GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); - return msm_obj->active_count; + lockdep_assert_once( + (kref_read(&obj->refcount) == 0) || + (lockdep_is_held(&obj->resv->lock.base) != LOCK_STATE_NOT_HELD) + ); } /* imported/exported objects are not purgeable: */ @@ -264,81 +232,15 @@ static inline bool is_purgeable(struct msm_gem_object *msm_obj) static inline bool is_vunmapable(struct msm_gem_object *msm_obj) { - GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base)); + msm_gem_assert_locked(&msm_obj->base); return (msm_obj->vmap_count == 0) && msm_obj->vaddr; } -static inline void mark_purgeable(struct msm_gem_object *msm_obj) -{ - struct msm_drm_private *priv = msm_obj->base.dev->dev_private; - - GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock)); - - if (is_unpurgeable(msm_obj)) - return; - - if (GEM_WARN_ON(msm_obj->dontneed)) - return; - - priv->shrinkable_count += msm_obj->base.size >> PAGE_SHIFT; - msm_obj->dontneed = true; -} - -static inline void mark_unpurgeable(struct msm_gem_object *msm_obj) -{ - struct msm_drm_private *priv = msm_obj->base.dev->dev_private; - - GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock)); - - if (is_unpurgeable(msm_obj)) - return; - - if (GEM_WARN_ON(!msm_obj->dontneed)) - return; - - priv->shrinkable_count -= msm_obj->base.size >> PAGE_SHIFT; - GEM_WARN_ON(priv->shrinkable_count < 0); - msm_obj->dontneed = false; -} - static inline bool is_unevictable(struct msm_gem_object *msm_obj) { return is_unpurgeable(msm_obj) || msm_obj->vaddr; } -static inline void mark_evictable(struct msm_gem_object *msm_obj) -{ - struct msm_drm_private *priv = msm_obj->base.dev->dev_private; - - WARN_ON(!mutex_is_locked(&priv->mm_lock)); - - if (is_unevictable(msm_obj)) - return; - - if (WARN_ON(msm_obj->evictable)) - return; - - priv->evictable_count += msm_obj->base.size >> PAGE_SHIFT; - msm_obj->evictable = true; -} - -static inline void mark_unevictable(struct msm_gem_object *msm_obj) -{ - struct msm_drm_private *priv = msm_obj->base.dev->dev_private; - - WARN_ON(!mutex_is_locked(&priv->mm_lock)); - - if (is_unevictable(msm_obj)) - return; - - if (WARN_ON(!msm_obj->evictable)) - return; - - priv->evictable_count -= msm_obj->base.size >> PAGE_SHIFT; - WARN_ON(priv->evictable_count < 0); - msm_obj->evictable = false; -} - void msm_gem_purge(struct drm_gem_object *obj); void msm_gem_evict(struct drm_gem_object *obj); void msm_gem_vunmap(struct drm_gem_object *obj); @@ -390,9 +292,8 @@ struct msm_gem_submit { /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */ #define BO_VALID 0x8000 /* is current addr in cmdstream correct/valid? */ #define BO_LOCKED 0x4000 /* obj lock is held */ -#define BO_ACTIVE 0x2000 /* active refcnt is held */ -#define BO_OBJ_PINNED 0x1000 /* obj (pages) is pinned and on active list */ -#define BO_VMA_PINNED 0x0800 /* vma (virtual address) is pinned */ +#define BO_OBJ_PINNED 0x2000 /* obj (pages) is pinned and on active list */ +#define BO_VMA_PINNED 0x1000 /* vma (virtual address) is pinned */ uint32_t flags; union { struct msm_gem_object *obj; diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c index dcc8a573bc7625d1d7ef7611372e096fb7f2e118..c1d91863df055b619332b466128d7b3aba9fe6d2 100644 --- a/drivers/gpu/drm/msm/msm_gem_prime.c +++ b/drivers/gpu/drm/msm/msm_gem_prime.c @@ -63,12 +63,12 @@ struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev, int msm_gem_prime_pin(struct drm_gem_object *obj) { if (!obj->import_attach) - msm_gem_get_pages(obj); + msm_gem_pin_pages(obj); return 0; } void msm_gem_prime_unpin(struct drm_gem_object *obj) { if (!obj->import_attach) - msm_gem_put_pages(obj); + msm_gem_unpin_pages(obj); } diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 0317055e32532c62e6efe32199520b15a8320d05..1de14e67f96b02012be0c205faf6fbb65ca0f8f1 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -24,103 +24,77 @@ static bool can_swap(void) return enable_eviction && get_nr_swap_pages() > 0; } +static bool can_block(struct shrink_control *sc) +{ + if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM)) + return false; + return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM); +} + static unsigned long msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); - unsigned count = priv->shrinkable_count; + unsigned count = priv->lru.dontneed.count; if (can_swap()) - count += priv->evictable_count; + count += priv->lru.willneed.count; return count; } static bool -purge(struct msm_gem_object *msm_obj) +purge(struct drm_gem_object *obj) { - if (!is_purgeable(msm_obj)) + if (!is_purgeable(to_msm_bo(obj))) return false; - /* - * This will move the obj out of still_in_list to - * the purged list - */ - msm_gem_purge(&msm_obj->base); + if (msm_gem_active(obj)) + return false; + + msm_gem_purge(obj); return true; } static bool -evict(struct msm_gem_object *msm_obj) +evict(struct drm_gem_object *obj) { - if (is_unevictable(msm_obj)) + if (is_unevictable(to_msm_bo(obj))) + return false; + + if (msm_gem_active(obj)) return false; - msm_gem_evict(&msm_obj->base); + msm_gem_evict(obj); return true; } -static unsigned long -scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list, - bool (*shrink)(struct msm_gem_object *msm_obj)) +static bool +wait_for_idle(struct drm_gem_object *obj) { - unsigned freed = 0; - struct list_head still_in_list; - - INIT_LIST_HEAD(&still_in_list); - - mutex_lock(&priv->mm_lock); - - while (freed < nr_to_scan) { - struct msm_gem_object *msm_obj = list_first_entry_or_null( - list, typeof(*msm_obj), mm_list); - - if (!msm_obj) - break; - - list_move_tail(&msm_obj->mm_list, &still_in_list); - - /* - * If it is in the process of being freed, msm_gem_free_object - * can be blocked on mm_lock waiting to remove it. So just - * skip it. - */ - if (!kref_get_unless_zero(&msm_obj->base.refcount)) - continue; - - /* - * Now that we own a reference, we can drop mm_lock for the - * rest of the loop body, to reduce contention with the - * retire_submit path (which could make more objects purgeable) - */ - - mutex_unlock(&priv->mm_lock); - - /* - * Note that this still needs to be trylock, since we can - * hit shrinker in response to trying to get backing pages - * for this obj (ie. while it's lock is already held) - */ - if (!msm_gem_trylock(&msm_obj->base)) - goto tail; - - if (shrink(msm_obj)) - freed += msm_obj->base.size >> PAGE_SHIFT; + enum dma_resv_usage usage = dma_resv_usage_rw(true); + return dma_resv_wait_timeout(obj->resv, usage, false, 1000) > 0; +} - msm_gem_unlock(&msm_obj->base); +static bool +active_purge(struct drm_gem_object *obj) +{ + if (!wait_for_idle(obj)) + return false; -tail: - drm_gem_object_put(&msm_obj->base); - mutex_lock(&priv->mm_lock); - } + return purge(obj); +} - list_splice_tail(&still_in_list, list); - mutex_unlock(&priv->mm_lock); +static bool +active_evict(struct drm_gem_object *obj) +{ + if (!wait_for_idle(obj)) + return false; - return freed; + return evict(obj); } static unsigned long @@ -128,21 +102,34 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) { struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); - unsigned long freed; - - freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge); - - if (freed > 0) - trace_msm_gem_purge(freed << PAGE_SHIFT); - - if (can_swap() && freed < sc->nr_to_scan) { - int evicted = scan(priv, sc->nr_to_scan - freed, - &priv->inactive_willneed, evict); + struct { + struct drm_gem_lru *lru; + bool (*shrink)(struct drm_gem_object *obj); + bool cond; + unsigned long freed; + } stages[] = { + /* Stages of progressively more aggressive/expensive reclaim: */ + { &priv->lru.dontneed, purge, true }, + { &priv->lru.willneed, evict, can_swap() }, + { &priv->lru.dontneed, active_purge, can_block(sc) }, + { &priv->lru.willneed, active_evict, can_swap() && can_block(sc) }, + }; + long nr = sc->nr_to_scan; + unsigned long freed = 0; - if (evicted > 0) - trace_msm_gem_evict(evicted << PAGE_SHIFT); + for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) { + if (!stages[i].cond) + continue; + stages[i].freed = + drm_gem_lru_scan(stages[i].lru, nr, stages[i].shrink); + nr -= stages[i].freed; + freed += stages[i].freed; + } - freed += evicted; + if (freed) { + trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed, + stages[1].freed, stages[2].freed, + stages[3].freed); } return (freed > 0) ? freed : SHRINK_STOP; @@ -173,12 +160,12 @@ msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan) static const int vmap_shrink_limit = 15; static bool -vmap_shrink(struct msm_gem_object *msm_obj) +vmap_shrink(struct drm_gem_object *obj) { - if (!is_vunmapable(msm_obj)) + if (!is_vunmapable(to_msm_bo(obj))) return false; - msm_gem_vunmap(&msm_obj->base); + msm_gem_vunmap(obj); return true; } @@ -188,17 +175,18 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) { struct msm_drm_private *priv = container_of(nb, struct msm_drm_private, vmap_notifier); - struct list_head *mm_lists[] = { - &priv->inactive_dontneed, - &priv->inactive_willneed, - priv->gpu ? &priv->gpu->active_list : NULL, + struct drm_gem_lru *lrus[] = { + &priv->lru.dontneed, + &priv->lru.willneed, + &priv->lru.pinned, NULL, }; unsigned idx, unmapped = 0; - for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) { - unmapped += scan(priv, vmap_shrink_limit - unmapped, - mm_lists[idx], vmap_shrink); + for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) { + unmapped += drm_gem_lru_scan(lrus[idx], + vmap_shrink_limit - unmapped, + vmap_shrink); } *(unsigned long *)ptr += unmapped; diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index c9e4aeb14f4ac0a9ccbcf35e26d833cf0cbdc234..5599d93ec0d21d281d56418852fff9ae8be77f10 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -26,6 +26,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, struct msm_gpu_submitqueue *queue, uint32_t nr_bos, uint32_t nr_cmds) { + static atomic_t ident = ATOMIC_INIT(0); struct msm_gem_submit *submit; uint64_t sz; int ret; @@ -36,7 +37,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, if (sz > SIZE_MAX) return ERR_PTR(-ENOMEM); - submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); + submit = kzalloc(sz, GFP_KERNEL); if (!submit) return ERR_PTR(-ENOMEM); @@ -52,9 +53,13 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, submit->gpu = gpu; submit->cmd = (void *)&submit->bos[nr_bos]; submit->queue = queue; + submit->pid = get_pid(task_pid(current)); submit->ring = gpu->rb[queue->ring_nr]; submit->fault_dumped = false; + /* Get a unique identifier for the submission for logging purposes */ + submit->ident = atomic_inc_return(&ident) - 1; + INIT_LIST_HEAD(&submit->node); return submit; @@ -67,9 +72,9 @@ void __msm_gem_submit_destroy(struct kref *kref) unsigned i; if (submit->fence_id) { - mutex_lock(&submit->queue->lock); + mutex_lock(&submit->queue->idr_lock); idr_remove(&submit->queue->fence_idr, submit->fence_id); - mutex_unlock(&submit->queue->lock); + mutex_unlock(&submit->queue->idr_lock); } dma_fence_put(submit->user_fence); @@ -238,17 +243,13 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i, if (flags & BO_OBJ_PINNED) msm_gem_unpin_locked(obj); - if (flags & BO_ACTIVE) - msm_gem_active_put(obj); - if (flags & BO_LOCKED) dma_resv_unlock(obj->resv); } static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i) { - unsigned cleanup_flags = BO_VMA_PINNED | BO_OBJ_PINNED | - BO_ACTIVE | BO_LOCKED; + unsigned cleanup_flags = BO_VMA_PINNED | BO_OBJ_PINNED | BO_LOCKED; submit_cleanup_bo(submit, i, cleanup_flags); if (!(submit->bos[i].flags & BO_VALID)) @@ -353,18 +354,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit) submit->valid = true; - /* - * Increment active_count first, so if under memory pressure, we - * don't inadvertently evict a bo needed by the submit in order - * to pin an earlier bo in the same submit. - */ - for (i = 0; i < submit->nr_bos; i++) { - struct drm_gem_object *obj = &submit->bos[i].obj->base; - - msm_gem_active_get(obj, submit->gpu); - submit->bos[i].flags |= BO_ACTIVE; - } - for (i = 0; i < submit->nr_bos; i++) { struct drm_gem_object *obj = &submit->bos[i].obj->base; struct msm_gem_vma *vma; @@ -512,11 +501,11 @@ out: */ static void submit_cleanup(struct msm_gem_submit *submit, bool error) { - unsigned cleanup_flags = BO_LOCKED; + unsigned cleanup_flags = BO_LOCKED | BO_OBJ_PINNED; unsigned i; if (error) - cleanup_flags |= BO_VMA_PINNED | BO_OBJ_PINNED | BO_ACTIVE; + cleanup_flags |= BO_VMA_PINNED; for (i = 0; i < submit->nr_bos; i++) { struct msm_gem_object *msm_obj = submit->bos[i].obj; @@ -533,10 +522,6 @@ void msm_submit_retire(struct msm_gem_submit *submit) for (i = 0; i < submit->nr_bos; i++) { struct drm_gem_object *obj = &submit->bos[i].obj->base; - msm_gem_lock(obj); - /* Note, VMA already fence-unpinned before submit: */ - submit_cleanup_bo(submit, i, BO_OBJ_PINNED | BO_ACTIVE); - msm_gem_unlock(obj); drm_gem_object_put(obj); } } @@ -718,7 +703,6 @@ static void msm_process_post_deps(struct msm_submit_post_dep *post_deps, int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file) { - static atomic_t ident = ATOMIC_INIT(0); struct msm_drm_private *priv = dev->dev_private; struct drm_msm_gem_submit *args = data; struct msm_file_private *ctx = file->driver_priv; @@ -729,10 +713,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct msm_submit_post_dep *post_deps = NULL; struct drm_syncobj **syncobjs_to_reset = NULL; int out_fence_fd = -1; - struct pid *pid = get_pid(task_pid(current)); bool has_ww_ticket = false; unsigned i; - int ret, submitid; + int ret; if (!gpu) return -ENXIO; @@ -764,35 +747,26 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, if (!queue) return -ENOENT; - /* Get a unique identifier for the submission for logging purposes */ - submitid = atomic_inc_return(&ident) - 1; - ring = gpu->rb[queue->ring_nr]; - trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid, - args->nr_bos, args->nr_cmds); - - ret = mutex_lock_interruptible(&queue->lock); - if (ret) - goto out_post_unlock; if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) { out_fence_fd = get_unused_fd_flags(O_CLOEXEC); if (out_fence_fd < 0) { ret = out_fence_fd; - goto out_unlock; + return ret; } } - submit = submit_create(dev, gpu, queue, args->nr_bos, - args->nr_cmds); - if (IS_ERR(submit)) { - ret = PTR_ERR(submit); - submit = NULL; - goto out_unlock; - } + submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds); + if (IS_ERR(submit)) + return PTR_ERR(submit); + + trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident, + args->nr_bos, args->nr_cmds); - submit->pid = pid; - submit->ident = submitid; + ret = mutex_lock_interruptible(&queue->lock); + if (ret) + goto out_post_unlock; if (args->flags & MSM_SUBMIT_SUDO) submit->in_rb = true; @@ -887,6 +861,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, submit->nr_cmds = i; + mutex_lock(&queue->idr_lock); + /* * If using userspace provided seqno fence, validate that the id * is available before arming sched job. Since access to fence_idr @@ -895,6 +871,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, */ if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) && idr_find(&queue->fence_idr, args->fence)) { + mutex_unlock(&queue->idr_lock); ret = -EINVAL; goto out; } @@ -927,6 +904,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, submit->user_fence, 1, INT_MAX, GFP_KERNEL); } + + mutex_unlock(&queue->idr_lock); + if (submit->fence_id < 0) { ret = submit->fence_id; submit->fence_id = 0; @@ -965,9 +945,9 @@ out_unlock: if (ret && (out_fence_fd >= 0)) put_unused_fd(out_fence_fd); mutex_unlock(&queue->lock); +out_post_unlock: if (submit) msm_gem_submit_put(submit); -out_post_unlock: if (!IS_ERR_OR_NULL(post_deps)) { for (i = 0; i < args->nr_out_syncobjs; ++i) { kfree(post_deps[i].chain); diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index c2bfcf3f1f4038cfa3c32d30f714607622bfb280..0098ee8438aae7b5d1c90714135f783d9361ff80 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -16,6 +16,7 @@ #include #include #include +#include #include /* @@ -394,7 +395,6 @@ static void recover_worker(struct kthread_work *work) /* Record the crash state */ pm_runtime_get_sync(&gpu->pdev->dev); msm_gpu_crashstate_capture(gpu, submit, comm, cmd); - pm_runtime_put_sync(&gpu->pdev->dev); kfree(cmd); kfree(comm); @@ -423,9 +423,7 @@ static void recover_worker(struct kthread_work *work) /* retire completed submits, plus the one that hung: */ retire_submits(gpu); - pm_runtime_get_sync(&gpu->pdev->dev); gpu->funcs->recover(gpu); - pm_runtime_put_sync(&gpu->pdev->dev); /* * Replay all remaining submits starting with highest priority @@ -442,6 +440,8 @@ static void recover_worker(struct kthread_work *work) } } + pm_runtime_put(&gpu->pdev->dev); + mutex_unlock(&gpu->lock); msm_gpu_retire(gpu); @@ -664,11 +664,12 @@ static void retire_submit(struct msm_gpu *gpu, struct msm_ringbuffer *ring, mutex_lock(&gpu->active_lock); gpu->active_submits--; WARN_ON(gpu->active_submits < 0); - if (!gpu->active_submits) + if (!gpu->active_submits) { msm_devfreq_idle(gpu); - mutex_unlock(&gpu->active_lock); + pm_runtime_put_autosuspend(&gpu->pdev->dev); + } - pm_runtime_put_autosuspend(&gpu->pdev->dev); + mutex_unlock(&gpu->active_lock); msm_gem_submit_put(submit); } @@ -757,14 +758,17 @@ void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) /* Update devfreq on transition from idle->active: */ mutex_lock(&gpu->active_lock); - if (!gpu->active_submits) + if (!gpu->active_submits) { + pm_runtime_get(&gpu->pdev->dev); msm_devfreq_active(gpu); + } gpu->active_submits++; mutex_unlock(&gpu->active_lock); gpu->funcs->submit(gpu, submit); gpu->cur_ctx_seqno = submit->queue->ctx->seqno; + pm_runtime_put(&gpu->pdev->dev); hangcheck_timer_reset(gpu); } @@ -846,7 +850,6 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, sched_set_fifo_low(gpu->worker->task); - INIT_LIST_HEAD(&gpu->active_list); mutex_init(&gpu->active_lock); mutex_init(&gpu->lock); init_waitqueue_head(&gpu->retire_event); @@ -901,6 +904,9 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, if (IS_ERR(gpu->gpu_cx)) gpu->gpu_cx = NULL; + gpu->cx_collapse = devm_reset_control_get_optional_exclusive(&pdev->dev, + "cx_collapse"); + gpu->pdev = pdev; platform_set_drvdata(pdev, &gpu->adreno_smmu); @@ -974,8 +980,6 @@ void msm_gpu_cleanup(struct msm_gpu *gpu) DBG("%s", gpu->name); - WARN_ON(!list_empty(&gpu->active_list)); - for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) { msm_ringbuffer_destroy(gpu->rb[i]); gpu->rb[i] = NULL; diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index 4d935fedd2acc7593eb4fad1d71f9f031ea3e4ec..ff911e7305ce98f4364af30b73add5a11f42ee6d 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -13,6 +13,7 @@ #include #include #include +#include #include "msm_drv.h" #include "msm_fence.h" @@ -187,12 +188,6 @@ struct msm_gpu { */ int cur_ctx_seqno; - /* - * List of GEM active objects on this gpu. Protected by - * msm_drm_private::mm_lock - */ - struct list_head active_list; - /** * lock: * @@ -277,6 +272,9 @@ struct msm_gpu { bool hw_apriv; struct thermal_cooling_device *cooling; + + /* To poll for cx gdsc collapse during gpu recovery */ + struct reset_control *cx_collapse; }; static inline struct msm_gpu *dev_to_gpu(struct device *dev) @@ -466,7 +464,8 @@ static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio, * @node: node in the context's list of submitqueues * @fence_idr: maps fence-id to dma_fence for userspace visible fence * seqno, protected by submitqueue lock - * @lock: submitqueue lock + * @idr_lock: for serializing access to fence_idr + * @lock: submitqueue lock for serializing submits on a queue * @ref: reference count * @entity: the submit job-queue */ @@ -479,6 +478,7 @@ struct msm_gpu_submitqueue { struct msm_file_private *ctx; struct list_head node; struct idr fence_idr; + struct mutex idr_lock; struct mutex lock; struct kref ref; struct drm_sched_entity *entity; diff --git a/drivers/gpu/drm/msm/msm_gpu_devfreq.c b/drivers/gpu/drm/msm/msm_gpu_devfreq.c index d1f70426f554ed552011d111f5595ce842c06acb..85c443a37e4e82ee53bff95a4c1795ef5c9bfa06 100644 --- a/drivers/gpu/drm/msm/msm_gpu_devfreq.c +++ b/drivers/gpu/drm/msm/msm_gpu_devfreq.c @@ -213,6 +213,8 @@ void msm_devfreq_init(struct msm_gpu *gpu) if (IS_ERR(df->devfreq)) { DRM_DEV_ERROR(&gpu->pdev->dev, "Couldn't initialize GPU devfreq\n"); + dev_pm_qos_remove_request(&df->idle_freq); + dev_pm_qos_remove_request(&df->boost_freq); df->devfreq = NULL; return; } diff --git a/drivers/gpu/drm/msm/msm_gpu_trace.h b/drivers/gpu/drm/msm/msm_gpu_trace.h index ca0b08d7875b7e1db71c67ec1e44c62c87e77150..ac40d857bc4578377b03b4cedd138c87144997e4 100644 --- a/drivers/gpu/drm/msm/msm_gpu_trace.h +++ b/drivers/gpu/drm/msm/msm_gpu_trace.h @@ -115,29 +115,27 @@ TRACE_EVENT(msm_gmu_freq_change, ); -TRACE_EVENT(msm_gem_purge, - TP_PROTO(u32 bytes), - TP_ARGS(bytes), +TRACE_EVENT(msm_gem_shrink, + TP_PROTO(u32 nr_to_scan, u32 purged, u32 evicted, + u32 active_purged, u32 active_evicted), + TP_ARGS(nr_to_scan, purged, evicted, active_purged, active_evicted), TP_STRUCT__entry( - __field(u32, bytes) + __field(u32, nr_to_scan) + __field(u32, purged) + __field(u32, evicted) + __field(u32, active_purged) + __field(u32, active_evicted) ), TP_fast_assign( - __entry->bytes = bytes; + __entry->nr_to_scan = nr_to_scan; + __entry->purged = purged; + __entry->evicted = evicted; + __entry->active_purged = active_purged; + __entry->active_evicted = active_evicted; ), - TP_printk("Purging %u bytes", __entry->bytes) -); - - -TRACE_EVENT(msm_gem_evict, - TP_PROTO(u32 bytes), - TP_ARGS(bytes), - TP_STRUCT__entry( - __field(u32, bytes) - ), - TP_fast_assign( - __entry->bytes = bytes; - ), - TP_printk("Evicting %u bytes", __entry->bytes) + TP_printk("nr_to_scan=%u pg, purged=%u pg, evicted=%u pg, active_purged=%u pg, active_evicted=%u pg", + __entry->nr_to_scan, __entry->purged, __entry->evicted, + __entry->active_purged, __entry->active_evicted) ); diff --git a/drivers/gpu/drm/msm/msm_io_utils.c b/drivers/gpu/drm/msm/msm_io_utils.c index 7b504617833adc849e8fb033614fea395e1c2d63..d02cd29ce8299eacb62cd3bc57e924cfe2ba5c41 100644 --- a/drivers/gpu/drm/msm/msm_io_utils.c +++ b/drivers/gpu/drm/msm/msm_io_utils.c @@ -5,6 +5,8 @@ * Author: Rob Clark */ +#include + #include "msm_drv.h" /* @@ -124,3 +126,23 @@ void msm_hrtimer_work_init(struct msm_hrtimer_work *work, work->worker = worker; kthread_init_work(&work->work, fn); } + +struct icc_path *msm_icc_get(struct device *dev, const char *name) +{ + struct device *mdss_dev = dev->parent; + struct icc_path *path; + + path = of_icc_get(dev, name); + if (path) + return path; + + /* + * If there are no interconnects attached to the corresponding device + * node, of_icc_get() will return NULL. + * + * If the MDP5/DPU device node doesn't have interconnects, lookup the + * path in the parent (MDSS) device. + */ + return of_icc_get(mdss_dev, name); + +} diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c index a54ed354578b53c1ce53dd35831fa4260d434372..5577cea7c0092632c8d46896bdc70e671329c8db 100644 --- a/drivers/gpu/drm/msm/msm_iommu.c +++ b/drivers/gpu/drm/msm/msm_iommu.c @@ -21,6 +21,7 @@ struct msm_iommu_pagetable { struct msm_mmu base; struct msm_mmu *parent; struct io_pgtable_ops *pgtbl_ops; + unsigned long pgsize_bitmap; /* Bitmap of page sizes in use */ phys_addr_t ttbr; u32 asid; }; @@ -29,23 +30,84 @@ static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu) return container_of(mmu, struct msm_iommu_pagetable, base); } +/* based on iommu_pgsize() in iommu.c: */ +static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable, + unsigned long iova, phys_addr_t paddr, + size_t size, size_t *count) +{ + unsigned int pgsize_idx, pgsize_idx_next; + unsigned long pgsizes; + size_t offset, pgsize, pgsize_next; + unsigned long addr_merge = paddr | iova; + + /* Page sizes supported by the hardware and small enough for @size */ + pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0); + + /* Constrain the page sizes further based on the maximum alignment */ + if (likely(addr_merge)) + pgsizes &= GENMASK(__ffs(addr_merge), 0); + + /* Make sure we have at least one suitable page size */ + BUG_ON(!pgsizes); + + /* Pick the biggest page size remaining */ + pgsize_idx = __fls(pgsizes); + pgsize = BIT(pgsize_idx); + if (!count) + return pgsize; + + /* Find the next biggest support page size, if it exists */ + pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0); + if (!pgsizes) + goto out_set_count; + + pgsize_idx_next = __ffs(pgsizes); + pgsize_next = BIT(pgsize_idx_next); + + /* + * There's no point trying a bigger page size unless the virtual + * and physical addresses are similarly offset within the larger page. + */ + if ((iova ^ paddr) & (pgsize_next - 1)) + goto out_set_count; + + /* Calculate the offset to the next page size alignment boundary */ + offset = pgsize_next - (addr_merge & (pgsize_next - 1)); + + /* + * If size is big enough to accommodate the larger page, reduce + * the number of smaller pages. + */ + if (offset + pgsize_next <= size) + size = offset; + +out_set_count: + *count = size >> pgsize_idx; + return pgsize; +} + static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova, size_t size) { struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); struct io_pgtable_ops *ops = pagetable->pgtbl_ops; - size_t unmapped = 0; - /* Unmap the block one page at a time */ while (size) { - unmapped += ops->unmap(ops, iova, 4096, NULL); - iova += 4096; - size -= 4096; + size_t unmapped, pgsize, count; + + pgsize = calc_pgsize(pagetable, iova, iova, size, &count); + + unmapped = ops->unmap_pages(ops, iova, pgsize, count, NULL); + if (!unmapped) + break; + + iova += unmapped; + size -= unmapped; } iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain); - return (unmapped == size) ? 0 : -EINVAL; + return (size == 0) ? 0 : -EINVAL; } static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova, @@ -54,7 +116,6 @@ static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova, struct msm_iommu_pagetable *pagetable = to_pagetable(mmu); struct io_pgtable_ops *ops = pagetable->pgtbl_ops; struct scatterlist *sg; - size_t mapped = 0; u64 addr = iova; unsigned int i; @@ -62,17 +123,26 @@ static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova, size_t size = sg->length; phys_addr_t phys = sg_phys(sg); - /* Map the block one page at a time */ while (size) { - if (ops->map(ops, addr, phys, 4096, prot, GFP_KERNEL)) { - msm_iommu_pagetable_unmap(mmu, iova, mapped); + size_t pgsize, count, mapped = 0; + int ret; + + pgsize = calc_pgsize(pagetable, addr, phys, size, &count); + + ret = ops->map_pages(ops, addr, phys, pgsize, count, + prot, GFP_KERNEL, &mapped); + + /* map_pages could fail after mapping some of the pages, + * so update the counters before error handling. + */ + phys += mapped; + addr += mapped; + size -= mapped; + + if (ret) { + msm_iommu_pagetable_unmap(mmu, iova, addr - iova); return -EINVAL; } - - phys += 4096; - addr += 4096; - size -= 4096; - mapped += 4096; } } @@ -207,6 +277,7 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent) /* Needed later for TLB flush */ pagetable->parent = parent; + pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap; pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr; /* diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c index a92ffde53f0b301453342a453c41e0eeee3bc27f..db2f847c8535fc2c0768a62483db36da6a5e66e1 100644 --- a/drivers/gpu/drm/msm/msm_rd.c +++ b/drivers/gpu/drm/msm/msm_rd.c @@ -196,6 +196,9 @@ static int rd_open(struct inode *inode, struct file *file) file->private_data = rd; rd->open = true; + /* Reset fifo to clear any previously unread data: */ + rd->fifo.head = rd->fifo.tail = 0; + /* the parsing tools need to know gpu-id to know which * register database to load. * diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index 56eecb4a72dc6d3e270316e88b20fcd37932117e..cad4c3525f0be8e7f284bd7012a3b2fade258bc6 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -29,8 +29,6 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job) msm_gem_unlock(obj); } - pm_runtime_get_sync(&gpu->pdev->dev); - /* TODO move submit path over to using a per-ring lock.. */ mutex_lock(&gpu->lock); @@ -38,8 +36,6 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job) mutex_unlock(&gpu->lock); - pm_runtime_put(&gpu->pdev->dev); - return dma_fence_get(submit->hw_fence); } diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c index f486a3cd4e5508995c0edd23474d6a445fe705ae..c6929e205b511e923e122e88e6fa32eabce9ab0a 100644 --- a/drivers/gpu/drm/msm/msm_submitqueue.c +++ b/drivers/gpu/drm/msm/msm_submitqueue.c @@ -200,6 +200,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx, *id = queue->id; idr_init(&queue->fence_idr); + mutex_init(&queue->idr_lock); mutex_init(&queue->lock); list_add_tail(&queue->node, &ctx->submitqueues); diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 35bb0bb3fe61ec8626c8333ccab71d345c7e7426..126b3c6e12f997c8db70bacd9d19e66766deafa9 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -822,6 +822,15 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, if (ret == 0) { ret = nouveau_fence_new(chan, false, &fence); if (ret == 0) { + /* TODO: figure out a better solution here + * + * wait on the fence here explicitly as going through + * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. + * + * Without this the operation can timeout and we'll fallback to a + * software copy, which might take several minutes to finish. + */ + nouveau_fence_wait(fence, false, false); ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c index 16356611b5b9500bc2563b704bb0f8120a5dba2b..5fe209107246f242514dc070be12cf77a5719893 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dmem.c +++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c @@ -139,44 +139,24 @@ static void nouveau_dmem_fence_done(struct nouveau_fence **fence) } } -static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm, - struct vm_fault *vmf, struct migrate_vma *args, - dma_addr_t *dma_addr) +static int nouveau_dmem_copy_one(struct nouveau_drm *drm, struct page *spage, + struct page *dpage, dma_addr_t *dma_addr) { struct device *dev = drm->dev->dev; - struct page *dpage, *spage; - struct nouveau_svmm *svmm; - - spage = migrate_pfn_to_page(args->src[0]); - if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE)) - return 0; - dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); - if (!dpage) - return VM_FAULT_SIGBUS; lock_page(dpage); *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, *dma_addr)) - goto error_free_page; + return -EIO; - svmm = spage->zone_device_data; - mutex_lock(&svmm->mutex); - nouveau_svmm_invalidate(svmm, args->start, args->end); if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr, - NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) - goto error_dma_unmap; - mutex_unlock(&svmm->mutex); + NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage))) { + dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + return -EIO; + } - args->dst[0] = migrate_pfn(page_to_pfn(dpage)); return 0; - -error_dma_unmap: - mutex_unlock(&svmm->mutex); - dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); -error_free_page: - __free_page(dpage); - return VM_FAULT_SIGBUS; } static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) @@ -184,9 +164,11 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) struct nouveau_drm *drm = page_to_drm(vmf->page); struct nouveau_dmem *dmem = drm->dmem; struct nouveau_fence *fence; + struct nouveau_svmm *svmm; + struct page *spage, *dpage; unsigned long src = 0, dst = 0; dma_addr_t dma_addr = 0; - vm_fault_t ret; + vm_fault_t ret = 0; struct migrate_vma args = { .vma = vmf->vma, .start = vmf->address, @@ -207,9 +189,25 @@ static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) if (!args.cpages) return 0; - ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr); - if (ret || dst == 0) + spage = migrate_pfn_to_page(src); + if (!spage || !(src & MIGRATE_PFN_MIGRATE)) + goto done; + + dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address); + if (!dpage) + goto done; + + dst = migrate_pfn(page_to_pfn(dpage)); + + svmm = spage->zone_device_data; + mutex_lock(&svmm->mutex); + nouveau_svmm_invalidate(svmm, args.start, args.end); + ret = nouveau_dmem_copy_one(drm, spage, dpage, &dma_addr); + mutex_unlock(&svmm->mutex); + if (ret) { + ret = VM_FAULT_SIGBUS; goto done; + } nouveau_fence_new(dmem->migrate.chan, false, &fence); migrate_vma_pages(&args); @@ -326,7 +324,7 @@ nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm) return NULL; } - lock_page(page); + zone_device_page_init(page); return page; } @@ -369,6 +367,52 @@ nouveau_dmem_suspend(struct nouveau_drm *drm) mutex_unlock(&drm->dmem->mutex); } +/* + * Evict all pages mapping a chunk. + */ +static void +nouveau_dmem_evict_chunk(struct nouveau_dmem_chunk *chunk) +{ + unsigned long i, npages = range_len(&chunk->pagemap.range) >> PAGE_SHIFT; + unsigned long *src_pfns, *dst_pfns; + dma_addr_t *dma_addrs; + struct nouveau_fence *fence; + + src_pfns = kcalloc(npages, sizeof(*src_pfns), GFP_KERNEL); + dst_pfns = kcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL); + dma_addrs = kcalloc(npages, sizeof(*dma_addrs), GFP_KERNEL); + + migrate_device_range(src_pfns, chunk->pagemap.range.start >> PAGE_SHIFT, + npages); + + for (i = 0; i < npages; i++) { + if (src_pfns[i] & MIGRATE_PFN_MIGRATE) { + struct page *dpage; + + /* + * _GFP_NOFAIL because the GPU is going away and there + * is nothing sensible we can do if we can't copy the + * data back. + */ + dpage = alloc_page(GFP_HIGHUSER | __GFP_NOFAIL); + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); + nouveau_dmem_copy_one(chunk->drm, + migrate_pfn_to_page(src_pfns[i]), dpage, + &dma_addrs[i]); + } + } + + nouveau_fence_new(chunk->drm->dmem->migrate.chan, false, &fence); + migrate_device_pages(src_pfns, dst_pfns, npages); + nouveau_dmem_fence_done(&fence); + migrate_device_finalize(src_pfns, dst_pfns, npages); + kfree(src_pfns); + kfree(dst_pfns); + for (i = 0; i < npages; i++) + dma_unmap_page(chunk->drm->dev->dev, dma_addrs[i], PAGE_SIZE, DMA_BIDIRECTIONAL); + kfree(dma_addrs); +} + void nouveau_dmem_fini(struct nouveau_drm *drm) { @@ -380,8 +424,10 @@ nouveau_dmem_fini(struct nouveau_drm *drm) mutex_lock(&drm->dmem->mutex); list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) { + nouveau_dmem_evict_chunk(chunk); nouveau_bo_unpin(chunk->bo); nouveau_bo_ref(NULL, &chunk->bo); + WARN_ON(chunk->callocated); list_del(&chunk->list); memunmap_pages(&chunk->pagemap); release_mem_region(chunk->pagemap.range.start, diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 561309d447e0278bc2a010ba4b5aa969f43128f0..fd99ec0f4257a6a78308f5d2e0e8c0b192e4efb4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -70,6 +71,18 @@ #include "nouveau_svm.h" #include "nouveau_dmem.h" +DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "DRM_UT_CORE", + "DRM_UT_DRIVER", + "DRM_UT_KMS", + "DRM_UT_PRIME", + "DRM_UT_ATOMIC", + "DRM_UT_VBL", + "DRM_UT_STATE", + "DRM_UT_LEASE", + "DRM_UT_DP", + "DRM_UT_DRMRES"); + MODULE_PARM_DESC(config, "option string to pass to driver core"); static char *nouveau_config; module_param_named(config, nouveau_config, charp, 0400); diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index 4b546b02d3aed476bf62fcfa0e16df932cc2d76e..4b39d1dd9140e00725b107c71b8f83c0ce2bfbf1 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -1285,7 +1285,8 @@ static const struct panel_desc innolux_n116bca_ea1 = { }, .delay = { .hpd_absent = 200, - .prepare_to_enable = 80, + .enable = 80, + .disable = 50, .unprepare = 500, }, }; diff --git a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c index cb5cb27462df16f9467522dc86d96cb3e1dbf047..36a46cb7fe1cd0ee89a4b4a5c7c40c4c71bad4db 100644 --- a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c +++ b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c @@ -288,7 +288,7 @@ static int lcd_olinuxino_probe(struct i2c_client *client, return 0; } -static int lcd_olinuxino_remove(struct i2c_client *client) +static void lcd_olinuxino_remove(struct i2c_client *client) { struct lcd_olinuxino *panel = i2c_get_clientdata(client); @@ -296,8 +296,6 @@ static int lcd_olinuxino_remove(struct i2c_client *client) drm_panel_disable(&panel->panel); drm_panel_unprepare(&panel->panel); - - return 0; } static const struct of_device_id lcd_olinuxino_of_ids[] = { diff --git a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c index a6dc5ab182fa6681cfb17b3041fcef05ec4dfb00..79f852465a846c0d9589a248c8bceb2c752f9591 100644 --- a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c +++ b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c @@ -446,7 +446,7 @@ error: return -ENODEV; } -static int rpi_touchscreen_remove(struct i2c_client *i2c) +static void rpi_touchscreen_remove(struct i2c_client *i2c) { struct rpi_touchscreen *ts = i2c_get_clientdata(i2c); @@ -455,8 +455,6 @@ static int rpi_touchscreen_remove(struct i2c_client *i2c) drm_panel_remove(&ts->base); mipi_dsi_device_unregister(ts->dsi); - - return 0; } static int rpi_touchscreen_dsi_probe(struct mipi_dsi_device *dsi) diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 252fd66011cc40d1ca77405af0ac7425270f07aa..2944228a8e2c4e0666c745f2b032e3ea0f9bc400 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -2252,7 +2252,7 @@ static const struct panel_desc innolux_g121i1_l01 = { .enable = 200, .disable = 20, }, - .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG, + .bus_format = MEDIA_BUS_FMT_RGB666_1X7X3_SPWG, .connector_type = DRM_MODE_CONNECTOR_LVDS, }; diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c index 5110cd9b2425890d3e620cbfd49186b581abd626..fe5f12f16a632cfa17819b86e63c78a29bd9c82a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c +++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c @@ -131,6 +131,17 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev) return PTR_ERR(opp); panfrost_devfreq_profile.initial_freq = cur_freq; + + /* + * Set the recommend OPP this will enable and configure the regulator + * if any and will avoid a switch off by regulator_late_cleanup() + */ + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) { + DRM_DEV_ERROR(dev, "Couldn't set recommended OPP\n"); + return ret; + } + dev_pm_opp_put(opp); /* diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index ac006bed4743bf9d29bf2e5828a6a17e1c151458..8ef25ab305ae7a5d5cf19b4ca71dc0e49d179ddf 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -2056,7 +2056,7 @@ static void ci_clear_vc(struct radeon_device *rdev) static int ci_upload_firmware(struct radeon_device *rdev) { struct ci_power_info *pi = ci_get_pi(rdev); - int i, ret; + int i; for (i = 0; i < rdev->usec_timeout; i++) { if (RREG32_SMC(RCU_UC_EVENTS) & BOOT_SEQ_DONE) @@ -2067,9 +2067,7 @@ static int ci_upload_firmware(struct radeon_device *rdev) ci_stop_smc_clock(rdev); ci_reset_smc(rdev); - ret = ci_load_smc_ucode(rdev, pi->sram_end); - - return ret; + return ci_load_smc_ucode(rdev, pi->sram_end); } diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c index a28d5ceab628be40af343daff088b9f6d4f381b5..6cbe1ab81abad7217681438666e10860c5d5b323 100644 --- a/drivers/gpu/drm/radeon/radeon_drv.c +++ b/drivers/gpu/drm/radeon/radeon_drv.c @@ -512,14 +512,11 @@ long radeon_drm_ioctl(struct file *filp, static long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { unsigned int nr = DRM_IOCTL_NR(cmd); - int ret; if (nr < DRM_COMMAND_BASE) return drm_compat_ioctl(filp, cmd, arg); - ret = radeon_drm_ioctl(filp, cmd, arg); - - return ret; + return radeon_drm_ioctl(filp, cmd, arg); } #endif diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile index e7275b5e7ec8fdf567ac7fe3622b873d31d0e814..6f132325c8b739a1f1aa56efc9d478e82eac13d6 100644 --- a/drivers/gpu/drm/rcar-du/Makefile +++ b/drivers/gpu/drm/rcar-du/Makefile @@ -14,10 +14,3 @@ obj-$(CONFIG_DRM_RCAR_DU) += rcar-du-drm.o obj-$(CONFIG_DRM_RCAR_DW_HDMI) += rcar_dw_hdmi.o obj-$(CONFIG_DRM_RCAR_LVDS) += rcar_lvds.o obj-$(CONFIG_DRM_RCAR_MIPI_DSI) += rcar_mipi_dsi.o - -# 'remote-endpoint' is fixed up at run-time -DTC_FLAGS_rcar_du_of_lvds_r8a7790 += -Wno-graph_endpoint -DTC_FLAGS_rcar_du_of_lvds_r8a7791 += -Wno-graph_endpoint -DTC_FLAGS_rcar_du_of_lvds_r8a7793 += -Wno-graph_endpoint -DTC_FLAGS_rcar_du_of_lvds_r8a7795 += -Wno-graph_endpoint -DTC_FLAGS_rcar_du_of_lvds_r8a7796 += -Wno-graph_endpoint diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c index fd3b94649a01d15f39a1b909871b1194e0f38a33..3619e1ddeb620e460c9f1a9f00fcd71720d9db50 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c @@ -29,6 +29,7 @@ #include "rcar_du_regs.h" #include "rcar_du_vsp.h" #include "rcar_lvds.h" +#include "rcar_mipi_dsi.h" static u32 rcar_du_crtc_read(struct rcar_du_crtc *rcrtc, u32 reg) { @@ -744,7 +745,19 @@ static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc, const struct drm_display_mode *mode = &crtc->state->adjusted_mode; - rcar_lvds_clk_enable(bridge, mode->clock * 1000); + rcar_lvds_pclk_enable(bridge, mode->clock * 1000); + } + + /* + * Similarly to LVDS, on V3U the dot clock is provided by the DSI + * encoder, and we need to enable the DSI clocks before enabling the CRTC. + */ + if ((rcdu->info->dsi_clk_mask & BIT(rcrtc->index)) && + (rstate->outputs & + (BIT(RCAR_DU_OUTPUT_DSI0) | BIT(RCAR_DU_OUTPUT_DSI1)))) { + struct drm_bridge *bridge = rcdu->dsi[rcrtc->index]; + + rcar_mipi_dsi_pclk_enable(bridge, state); } rcar_du_crtc_start(rcrtc); @@ -777,7 +790,20 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc, * Disable the LVDS clock output, see * rcar_du_crtc_atomic_enable(). */ - rcar_lvds_clk_disable(bridge); + rcar_lvds_pclk_disable(bridge); + } + + if ((rcdu->info->dsi_clk_mask & BIT(rcrtc->index)) && + (rstate->outputs & + (BIT(RCAR_DU_OUTPUT_DSI0) | BIT(RCAR_DU_OUTPUT_DSI1)))) { + struct drm_bridge *bridge = rcdu->dsi[rcrtc->index]; + + /* + * Disable the DSI clock output, see + * rcar_du_crtc_atomic_enable(). + */ + + rcar_mipi_dsi_pclk_disable(bridge); } spin_lock_irq(&crtc->dev->event_lock); diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c index 00ac233a115e7ba65675f759c308c471c7c94e51..a2776f1d6f2c298bef1180daed2564d8f5162288 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c @@ -27,7 +27,6 @@ #include "rcar_du_drv.h" #include "rcar_du_kms.h" -#include "rcar_du_regs.h" /* ----------------------------------------------------------------------------- * Device Information @@ -507,7 +506,8 @@ static const struct rcar_du_device_info rcar_du_r8a7799x_info = { static const struct rcar_du_device_info rcar_du_r8a779a0_info = { .gen = 3, .features = RCAR_DU_FEATURE_CRTC_IRQ - | RCAR_DU_FEATURE_VSP1_SOURCE, + | RCAR_DU_FEATURE_VSP1_SOURCE + | RCAR_DU_FEATURE_NO_BLENDING, .channels_mask = BIT(1) | BIT(0), .routes = { /* R8A779A0 has two MIPI DSI outputs. */ diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.h b/drivers/gpu/drm/rcar-du/rcar_du_drv.h index bfad7775d9a19da483b17558a7ad39f9c940ae19..5cfa2bb7ad93dc2453a361934bb48406b1605a50 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_drv.h +++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.h @@ -31,6 +31,7 @@ struct rcar_du_device; #define RCAR_DU_FEATURE_VSP1_SOURCE BIT(2) /* Has inputs from VSP1 */ #define RCAR_DU_FEATURE_INTERLACED BIT(3) /* HW supports interlaced */ #define RCAR_DU_FEATURE_TVM_SYNC BIT(4) /* Has TV switch/sync modes */ +#define RCAR_DU_FEATURE_NO_BLENDING BIT(5) /* PnMR.SPIM does not have ALP nor EOR bits */ #define RCAR_DU_QUIRK_ALIGN_128B BIT(0) /* Align pitches to 128 bytes */ @@ -91,6 +92,7 @@ struct rcar_du_device_info { #define RCAR_DU_MAX_GROUPS DIV_ROUND_UP(RCAR_DU_MAX_CRTCS, 2) #define RCAR_DU_MAX_VSPS 4 #define RCAR_DU_MAX_LVDS 2 +#define RCAR_DU_MAX_DSI 2 struct rcar_du_device { struct device *dev; @@ -107,6 +109,7 @@ struct rcar_du_device { struct platform_device *cmms[RCAR_DU_MAX_CRTCS]; struct rcar_du_vsp vsps[RCAR_DU_MAX_VSPS]; struct drm_bridge *lvds[RCAR_DU_MAX_LVDS]; + struct drm_bridge *dsi[RCAR_DU_MAX_DSI]; struct { struct drm_property *colorkey; diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c index 60d6be78323b7af7766a7457ca6bd7cf40e055c4..b1787be31e92c600164e20d5d98dbbd2db751714 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c @@ -9,18 +9,13 @@ #include #include -#include #include #include -#include -#include -#include #include #include "rcar_du_drv.h" #include "rcar_du_encoder.h" -#include "rcar_du_kms.h" #include "rcar_lvds.h" /* ----------------------------------------------------------------------------- @@ -84,6 +79,10 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu, if (output == RCAR_DU_OUTPUT_LVDS0 || output == RCAR_DU_OUTPUT_LVDS1) rcdu->lvds[output - RCAR_DU_OUTPUT_LVDS0] = bridge; + + if (output == RCAR_DU_OUTPUT_DSI0 || + output == RCAR_DU_OUTPUT_DSI1) + rcdu->dsi[output - RCAR_DU_OUTPUT_DSI0] = bridge; } /* diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c index 21881fb5e84ad73023d83f96fcea5238d7122278..8c2719efda2a20750e9cd433d28a2653b2b89ae4 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c @@ -405,8 +405,8 @@ rcar_du_fb_create(struct drm_device *dev, struct drm_file *file_priv, format = rcar_du_format_info(mode_cmd->pixel_format); if (format == NULL) { - dev_dbg(dev->dev, "unsupported pixel format %08x\n", - mode_cmd->pixel_format); + dev_dbg(dev->dev, "unsupported pixel format %p4cc\n", + &mode_cmd->pixel_format); return ERR_PTR(-EINVAL); } diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c index 9e1f0cbbf642df43743fd37b101ae2e91cbefeeb..d759e019218181ce06d052357a450f33e8d04d9a 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c @@ -506,8 +506,15 @@ static void rcar_du_plane_setup_format_gen3(struct rcar_du_group *rgrp, unsigned int index, const struct rcar_du_plane_state *state) { - rcar_du_plane_write(rgrp, index, PnMR, - PnMR_SPIM_TP_OFF | state->format->pnmr); + struct rcar_du_device *rcdu = rgrp->dev; + u32 pnmr = state->format->pnmr | PnMR_SPIM_TP_OFF; + + if (rcdu->info->features & RCAR_DU_FEATURE_NO_BLENDING) { + /* No blending. ALP and EOR are not supported. */ + pnmr &= ~(PnMR_SPIM_ALP | PnMR_SPIM_EOR); + } + + rcar_du_plane_write(rgrp, index, PnMR, pnmr); rcar_du_plane_write(rgrp, index, PnDDCR4, state->format->edf | PnDDCR4_CODE); @@ -521,7 +528,6 @@ static void rcar_du_plane_setup_format_gen3(struct rcar_du_group *rgrp, * register to 0 to avoid this. */ - /* TODO: Check if alpha-blending should be disabled in PnMR. */ rcar_du_plane_write(rgrp, index, PnALPHAR, 0); } @@ -619,8 +625,8 @@ int __rcar_du_plane_atomic_check(struct drm_plane *plane, *format = rcar_du_format_info(state->fb->format->format); if (*format == NULL) { - dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__, - state->fb->format->format); + dev_dbg(dev->dev, "%s: unsupported format %p4cc\n", __func__, + &state->fb->format->format); return -EINVAL; } diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c index 10b7f1d0877ab1489b115c8d1bd5901dc6439ca6..e465aef41585fcf461ef8a2d3599ccd3ddab80df 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c @@ -152,6 +152,7 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane) .alpha = state->state.alpha >> 8, .zpos = state->state.zpos, }; + u32 fourcc = state->format->fourcc; unsigned int i; cfg.src.left = state->state.src.x1 >> 16; @@ -168,9 +169,27 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane) cfg.mem[i] = sg_dma_address(state->sg_tables[i].sgl) + fb->offsets[i]; - format = rcar_du_format_info(state->format->fourcc); + if (state->state.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE) { + switch (fourcc) { + case DRM_FORMAT_ARGB1555: + fourcc = DRM_FORMAT_XRGB1555; + break; + + case DRM_FORMAT_ARGB4444: + fourcc = DRM_FORMAT_XRGB4444; + break; + + case DRM_FORMAT_ARGB8888: + fourcc = DRM_FORMAT_XRGB8888; + break; + } + } + + format = rcar_du_format_info(fourcc); cfg.pixelformat = format->v4l2; + cfg.premult = state->state.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI; + vsp1_du_atomic_update(plane->vsp->vsp, crtc->vsp_pipe, plane->index, &cfg); } @@ -436,6 +455,11 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np, drm_plane_create_zpos_property(&plane->plane, i, 0, num_planes - 1); + drm_plane_create_blend_mode_property(&plane->plane, + BIT(DRM_MODE_BLEND_PIXEL_NONE) | + BIT(DRM_MODE_BLEND_PREMULTI) | + BIT(DRM_MODE_BLEND_COVERAGE)); + vsp->num_planes++; } diff --git a/drivers/gpu/drm/rcar-du/rcar_du_writeback.c b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c index 25f50a297c11a16c193793c1e6973052e7606c17..8cd37d7b8ae281cbc1fd8cbb243c621174517e23 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_writeback.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c @@ -166,8 +166,8 @@ static int rcar_du_wb_enc_atomic_check(struct drm_encoder *encoder, wb_state->format = rcar_du_format_info(fb->format->format); if (wb_state->format == NULL) { - dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__, - fb->format->format); + dev_dbg(dev->dev, "%s: unsupported format %p4cc\n", __func__, + &fb->format->format); return -EINVAL; } diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.c b/drivers/gpu/drm/rcar-du/rcar_lvds.c index d85aa4bc7f843830769ed5229a80c37db01eb256..81a060c2fe3f0125b5318392f8b1d10c53447864 100644 --- a/drivers/gpu/drm/rcar-du/rcar_lvds.c +++ b/drivers/gpu/drm/rcar-du/rcar_lvds.c @@ -306,7 +306,7 @@ static void rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds, unsigned int freq) * Clock - D3/E3 only */ -int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq) +int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq) { struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge); int ret; @@ -324,9 +324,9 @@ int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq) return 0; } -EXPORT_SYMBOL_GPL(rcar_lvds_clk_enable); +EXPORT_SYMBOL_GPL(rcar_lvds_pclk_enable); -void rcar_lvds_clk_disable(struct drm_bridge *bridge) +void rcar_lvds_pclk_disable(struct drm_bridge *bridge) { struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge); @@ -339,7 +339,7 @@ void rcar_lvds_clk_disable(struct drm_bridge *bridge) clk_disable_unprepare(lvds->clocks.mod); } -EXPORT_SYMBOL_GPL(rcar_lvds_clk_disable); +EXPORT_SYMBOL_GPL(rcar_lvds_pclk_disable); /* ----------------------------------------------------------------------------- * Bridge diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.h b/drivers/gpu/drm/rcar-du/rcar_lvds.h index 3097bf749bec7527d91924457501ef7b91ad1cd6..bee7033b60d60f98bf9ef6dce259ce01951ccd78 100644 --- a/drivers/gpu/drm/rcar-du/rcar_lvds.h +++ b/drivers/gpu/drm/rcar-du/rcar_lvds.h @@ -13,17 +13,17 @@ struct drm_bridge; #if IS_ENABLED(CONFIG_DRM_RCAR_LVDS) -int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq); -void rcar_lvds_clk_disable(struct drm_bridge *bridge); +int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq); +void rcar_lvds_pclk_disable(struct drm_bridge *bridge); bool rcar_lvds_dual_link(struct drm_bridge *bridge); bool rcar_lvds_is_connected(struct drm_bridge *bridge); #else -static inline int rcar_lvds_clk_enable(struct drm_bridge *bridge, - unsigned long freq) +static inline int rcar_lvds_pclk_enable(struct drm_bridge *bridge, + unsigned long freq) { return -ENOSYS; } -static inline void rcar_lvds_clk_disable(struct drm_bridge *bridge) { } +static inline void rcar_lvds_pclk_disable(struct drm_bridge *bridge) { } static inline bool rcar_lvds_dual_link(struct drm_bridge *bridge) { return false; diff --git a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c index 62f7eb84ab01c2f11ffdce0106e88209207ca2eb..a7f2b7f66a176a6525ec4ddf53cc6ba57af75756 100644 --- a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c +++ b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c @@ -25,6 +25,7 @@ #include #include +#include "rcar_mipi_dsi.h" #include "rcar_mipi_dsi_regs.h" struct rcar_mipi_dsi { @@ -414,7 +415,7 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi, /* Enable DOT clock */ vclkset = VCLKSET_CKEN; - rcar_mipi_dsi_set(dsi, VCLKSET, vclkset); + rcar_mipi_dsi_write(dsi, VCLKSET, vclkset); if (dsi_format == 24) vclkset |= VCLKSET_BPP_24; @@ -429,7 +430,7 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi, vclkset |= VCLKSET_COLOR_RGB | VCLKSET_DIV(setup_info.div) | VCLKSET_LANE(dsi->lanes - 1); - rcar_mipi_dsi_set(dsi, VCLKSET, vclkset); + rcar_mipi_dsi_write(dsi, VCLKSET, vclkset); /* After setting VCLKSET register, enable VCLKEN */ rcar_mipi_dsi_set(dsi, VCLKEN, VCLKEN_CKEN); @@ -441,9 +442,21 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi, static void rcar_mipi_dsi_shutdown(struct rcar_mipi_dsi *dsi) { + /* Disable VCLKEN */ + rcar_mipi_dsi_write(dsi, VCLKSET, 0); + + /* Disable DOT clock */ + rcar_mipi_dsi_write(dsi, VCLKSET, 0); + rcar_mipi_dsi_clr(dsi, PHYSETUP, PHYSETUP_RSTZ); rcar_mipi_dsi_clr(dsi, PHYSETUP, PHYSETUP_SHUTDOWNZ); + /* CFGCLK disable */ + rcar_mipi_dsi_clr(dsi, CFGCLKSET, CFGCLKSET_CKEN); + + /* LPCLK disable */ + rcar_mipi_dsi_clr(dsi, LPCLKSET, LPCLKSET_CKEN); + dev_dbg(dsi->dev, "DSI device is shutdown\n"); } @@ -542,6 +555,34 @@ static int rcar_mipi_dsi_start_video(struct rcar_mipi_dsi *dsi) return 0; } +static void rcar_mipi_dsi_stop_video(struct rcar_mipi_dsi *dsi) +{ + u32 status; + int ret; + + /* Disable transmission in video mode. */ + rcar_mipi_dsi_clr(dsi, TXVMCR, TXVMCR_EN_VIDEO); + + ret = read_poll_timeout(rcar_mipi_dsi_read, status, + !(status & TXVMSR_ACT), + 2000, 100000, false, dsi, TXVMSR); + if (ret < 0) { + dev_err(dsi->dev, "Failed to disable video transmission\n"); + return; + } + + /* Assert video FIFO clear. */ + rcar_mipi_dsi_set(dsi, TXVMCR, TXVMCR_VFCLR); + + ret = read_poll_timeout(rcar_mipi_dsi_read, status, + !(status & TXVMSR_VFRDY), + 2000, 100000, false, dsi, TXVMSR); + if (ret < 0) { + dev_err(dsi->dev, "Failed to assert video FIFO clear\n"); + return; + } +} + /* ----------------------------------------------------------------------------- * Bridge */ @@ -558,7 +599,22 @@ static int rcar_mipi_dsi_attach(struct drm_bridge *bridge, static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge, struct drm_bridge_state *old_bridge_state) { - struct drm_atomic_state *state = old_bridge_state->base.state; + struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge); + + rcar_mipi_dsi_start_video(dsi); +} + +static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge, + struct drm_bridge_state *old_bridge_state) +{ + struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge); + + rcar_mipi_dsi_stop_video(dsi); +} + +void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge, + struct drm_atomic_state *state) +{ struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge); const struct drm_display_mode *mode; struct drm_connector *connector; @@ -586,8 +642,6 @@ static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge, if (ret < 0) goto err_dsi_start_hs; - rcar_mipi_dsi_start_video(dsi); - return; err_dsi_start_hs: @@ -595,15 +649,16 @@ err_dsi_start_hs: err_dsi_startup: rcar_mipi_dsi_clk_disable(dsi); } +EXPORT_SYMBOL_GPL(rcar_mipi_dsi_pclk_enable); -static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge, - struct drm_bridge_state *old_bridge_state) +void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge) { struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge); rcar_mipi_dsi_shutdown(dsi); rcar_mipi_dsi_clk_disable(dsi); } +EXPORT_SYMBOL_GPL(rcar_mipi_dsi_pclk_disable); static enum drm_mode_status rcar_mipi_dsi_bridge_mode_valid(struct drm_bridge *bridge, diff --git a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h new file mode 100644 index 0000000000000000000000000000000000000000..528a196e6eddb97f65971b9a9add7fd96dbe237c --- /dev/null +++ b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * R-Car DSI Encoder + * + * Copyright (C) 2022 Renesas Electronics Corporation + * + * Contact: Tomi Valkeinen + */ + +#ifndef __RCAR_MIPI_DSI_H__ +#define __RCAR_MIPI_DSI_H__ + +struct drm_atomic_state; +struct drm_bridge; + +#if IS_ENABLED(CONFIG_DRM_RCAR_MIPI_DSI) +void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge, + struct drm_atomic_state *state); +void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge); +#else +static inline void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge, + struct drm_atomic_state *state) +{ +} + +static inline void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge) +{ +} +#endif /* CONFIG_DRM_RCAR_MIPI_DSI */ + +#endif /* __RCAR_MIPI_DSI_H__ */ diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c index c204e9b95c1f749d9e2de03929b1c6aa1c76ebb1..518ee13b1d6f42b39516bd12a5f637367404d939 100644 --- a/drivers/gpu/drm/rockchip/cdn-dp-core.c +++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c @@ -283,8 +283,9 @@ static int cdn_dp_connector_get_modes(struct drm_connector *connector) return ret; } -static int cdn_dp_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) +static enum drm_mode_status +cdn_dp_connector_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) { struct cdn_dp_device *dp = connector_to_dp(connector); struct drm_display_info *display_info = &dp->connector.display_info; diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 552426d5d3a2aba1a72044b43cabe8f82e227130..aac20be5ac0820cde9dd275ac2d1f9b812c4f01c 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -1438,11 +1438,15 @@ static void rk3568_set_intf_mux(struct vop2_video_port *vp, int id, die &= ~RK3568_SYS_DSP_INFACE_EN_HDMI_MUX; die |= RK3568_SYS_DSP_INFACE_EN_HDMI | FIELD_PREP(RK3568_SYS_DSP_INFACE_EN_HDMI_MUX, vp->id); + dip &= ~RK3568_DSP_IF_POL__HDMI_PIN_POL; + dip |= FIELD_PREP(RK3568_DSP_IF_POL__HDMI_PIN_POL, polflags); break; case ROCKCHIP_VOP2_EP_EDP0: die &= ~RK3568_SYS_DSP_INFACE_EN_EDP_MUX; die |= RK3568_SYS_DSP_INFACE_EN_EDP | FIELD_PREP(RK3568_SYS_DSP_INFACE_EN_EDP_MUX, vp->id); + dip &= ~RK3568_DSP_IF_POL__EDP_PIN_POL; + dip |= FIELD_PREP(RK3568_DSP_IF_POL__EDP_PIN_POL, polflags); break; case ROCKCHIP_VOP2_EP_MIPI0: die &= ~RK3568_SYS_DSP_INFACE_EN_MIPI0_MUX; diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 4cc59bae38dd132986ba2cafe9ae5600b3e570a2..2fab218d7082791453c59409c86043165aacdb1a 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -919,8 +919,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) job = list_first_entry_or_null(&sched->pending_list, struct drm_sched_job, list); - if (job && (!job->s_fence->parent || - dma_fence_is_signaled(job->s_fence->parent))) { + if (job && dma_fence_is_signaled(&job->s_fence->finished)) { /* remove job from pending_list */ list_del_init(&job->list); @@ -930,9 +929,9 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) next = list_first_entry_or_null(&sched->pending_list, typeof(*next), list); - if (next && job->s_fence->parent) { + if (next) { next->s_fence->scheduled.timestamp = - job->s_fence->parent->timestamp; + job->s_fence->finished.timestamp; /* start TO timer for next job */ drm_sched_start_timeout(sched); } diff --git a/drivers/gpu/drm/solomon/ssd130x-i2c.c b/drivers/gpu/drm/solomon/ssd130x-i2c.c index 1e0fcec7be47661ef7853281f29d8bec487db5b9..ddfa0bb5d9c9762cc5543922a82a93ef809b0c03 100644 --- a/drivers/gpu/drm/solomon/ssd130x-i2c.c +++ b/drivers/gpu/drm/solomon/ssd130x-i2c.c @@ -39,13 +39,11 @@ static int ssd130x_i2c_probe(struct i2c_client *client) return 0; } -static int ssd130x_i2c_remove(struct i2c_client *client) +static void ssd130x_i2c_remove(struct i2c_client *client) { struct ssd130x_device *ssd130x = i2c_get_clientdata(client); ssd130x_remove(ssd130x); - - return 0; } static void ssd130x_i2c_shutdown(struct i2c_client *client) diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c index 7a2b2d6bc3fe95942c6364352ac97de6a331a298..62f69589a72d360d1a718b759071649ac7d5b120 100644 --- a/drivers/gpu/drm/tests/drm_buddy_test.c +++ b/drivers/gpu/drm/tests/drm_buddy_test.c @@ -729,7 +729,7 @@ static void drm_test_buddy_alloc_limit(struct kunit *test) static int drm_buddy_init_test(struct kunit *test) { while (!random_seed) - random_seed = get_random_int(); + random_seed = get_random_u32(); return 0; } diff --git a/drivers/gpu/drm/tests/drm_mm_test.c b/drivers/gpu/drm/tests/drm_mm_test.c index 659d1af4dca784714788e9ec72d18d1ab096db3d..c4b66eeae2039f6ce5f754f0006737bf028385bf 100644 --- a/drivers/gpu/drm/tests/drm_mm_test.c +++ b/drivers/gpu/drm/tests/drm_mm_test.c @@ -2212,7 +2212,7 @@ err_nodes: static int drm_mm_init_test(struct kunit *test) { while (!random_seed) - random_seed = get_random_int(); + random_seed = get_random_u32(); return 0; } diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 497ee1fdbad7bf4dfa5ce28f19e30b1e33a07511..fa04e62202c15778832b010db85a5f755c2c3c81 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -239,16 +239,19 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, if (bo->type != ttm_bo_type_sg) fbo->base.base.resv = &fbo->base.base._resv; - if (fbo->base.resource) { - ttm_resource_set_bo(fbo->base.resource, &fbo->base); - bo->resource = NULL; - } - dma_resv_init(&fbo->base.base._resv); fbo->base.base.dev = NULL; ret = dma_resv_trylock(&fbo->base.base._resv); WARN_ON(!ret); + if (fbo->base.resource) { + ttm_resource_set_bo(fbo->base.resource, &fbo->base); + bo->resource = NULL; + ttm_bo_set_bulk_move(&fbo->base, NULL); + } else { + fbo->base.bulk_move = NULL; + } + ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); if (ret) { kfree(fbo); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 6ce92830b5d1f9c90444401f0167f64b92bde816..185a077d59cdd10063d834208845ad033359c353 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -28,7 +28,6 @@ if HID config HID_BATTERY_STRENGTH bool "Battery level reporting for HID devices" - depends on HID select POWER_SUPPLY default n help @@ -38,7 +37,6 @@ config HID_BATTERY_STRENGTH config HIDRAW bool "/dev/hidraw raw HID device support" - depends on HID help Say Y here if you want to support HID devices (from the USB specification standpoint) that aren't strictly user interface @@ -57,7 +55,6 @@ config HIDRAW config UHID tristate "User-space I/O driver support for HID subsystem" - depends on HID default n help Say Y here if you want to provide HID I/O Drivers from user-space. @@ -78,7 +75,6 @@ config UHID config HID_GENERIC tristate "Generic HID driver" - depends on HID default HID help Support for generic devices on the HID bus. This includes most @@ -90,11 +86,9 @@ config HID_GENERIC If unsure, say Y. menu "Special HID drivers" - depends on HID config HID_A4TECH tristate "A4TECH mice" - depends on HID default !EXPERT help Support for some A4TECH mice with two scroll wheels. @@ -113,7 +107,6 @@ config HID_ACCUTOUCH config HID_ACRUX tristate "ACRUX game controller support" - depends on HID help Say Y here if you want to enable support for ACRUX game controllers. @@ -127,7 +120,6 @@ config HID_ACRUX_FF config HID_APPLE tristate "Apple {i,Power,Mac}Books" - depends on HID depends on LEDS_CLASS depends on NEW_LEDS default !EXPERT @@ -167,13 +159,11 @@ config HID_ASUS config HID_AUREAL tristate "Aureal" - depends on HID help Support for Aureal Cy se W-01RN Remote Controller and other Aureal derived remotes. config HID_BELKIN tristate "Belkin Flip KVM and Wireless keyboard" - depends on HID default !EXPERT help Support for Belkin Flip KVM and Wireless keyboard. @@ -202,7 +192,6 @@ config HID_BIGBEN_FF config HID_CHERRY tristate "Cherry Cymotion keyboard" - depends on HID default !EXPERT help Support for Cherry Cymotion keyboard. @@ -227,7 +216,6 @@ config HID_CORSAIR config HID_COUGAR tristate "Cougar devices" - depends on HID help Support for Cougar devices that are not fully compliant with the HID standard. @@ -237,7 +225,6 @@ config HID_COUGAR config HID_MACALLY tristate "Macally devices" - depends on HID help Support for Macally devices that are not fully compliant with the HID standard. @@ -262,7 +249,6 @@ config HID_PRODIKEYS config HID_CMEDIA tristate "CMedia audio chips" - depends on HID help Support for CMedia CM6533 HID audio jack controls and HS100B mute buttons. @@ -288,14 +274,12 @@ config HID_CREATIVE_SB0540 config HID_CYPRESS tristate "Cypress mouse and barcode readers" - depends on HID default !EXPERT help Support for cypress mouse and barcode readers. config HID_DRAGONRISE tristate "DragonRise Inc. game controller" - depends on HID help Say Y here if you have DragonRise Inc. game controllers. These might be branded as: @@ -314,7 +298,6 @@ config DRAGONRISE_FF config HID_EMS_FF tristate "EMS Production Inc. force feedback support" - depends on HID select INPUT_FF_MEMLESS help Say Y here if you want to enable force feedback support for devices by @@ -332,7 +315,6 @@ config HID_ELAN config HID_ELECOM tristate "ELECOM HID devices" - depends on HID help Support for ELECOM devices: - BM084 Bluetooth Mouse @@ -349,7 +331,6 @@ config HID_ELO config HID_EZKEY tristate "Ezkey BTC 8193 keyboard" - depends on HID default !EXPERT help Support for Ezkey BTC 8193 keyboard. @@ -367,19 +348,16 @@ config HID_FT260 config HID_GEMBIRD tristate "Gembird Joypad" - depends on HID help Support for Gembird JPD-DualForce 2. config HID_GFRM tristate "Google Fiber TV Box remote control support" - depends on HID help Support for Google Fiber TV Box remote controls config HID_GLORIOUS tristate "Glorious PC Gaming Race mice" - depends on HID help Support for Glorious PC Gaming Race mice such as the Glorious Model O, O- and D. @@ -424,7 +402,6 @@ config HID_VIVALDI tristate "Vivaldi Keyboard" select HID_VIVALDI_COMMON select INPUT_VIVALDIFMAP - depends on HID help Say Y here if you want to enable support for Vivaldi keyboards. @@ -447,7 +424,6 @@ config HID_GT683R config HID_KEYTOUCH tristate "Keytouch HID devices" - depends on HID help Support for Keytouch HID devices not fully compliant with the specification. Currently supported: @@ -455,7 +431,6 @@ config HID_KEYTOUCH config HID_KYE tristate "KYE/Genius devices" - depends on HID help Support for KYE/Genius devices not fully compliant with HID standard: - Ergo Mouse @@ -471,32 +446,37 @@ config HID_UCLOGIC config HID_WALTOP tristate "Waltop" - depends on HID help Support for Waltop tablets. config HID_VIEWSONIC tristate "ViewSonic/Signotec" - depends on HID help Support for ViewSonic/Signotec PD1011 signature pad. +config HID_VRC2 + tristate "VRC-2 Car Controller" + depends on HID + help + Support for VRC-2 which is a 2-axis controller often used in + car simulators. + + To compile this driver as a module, choose M here: the + module will be called hid-vrc2. + config HID_XIAOMI tristate "Xiaomi" - depends on HID help Adds support for side buttons of Xiaomi Mi Dual Mode Wireless Mouse Silent Edition. config HID_GYRATION tristate "Gyration remote control" - depends on HID help Support for Gyration remote control. config HID_ICADE tristate "ION iCade arcade controller" - depends on HID help Support for the ION iCade arcade controller to work as a joystick. @@ -505,14 +485,12 @@ config HID_ICADE config HID_ITE tristate "ITE devices" - depends on HID default !EXPERT help Support for ITE devices not fully compliant with HID standard. config HID_JABRA tristate "Jabra USB HID Driver" - depends on HID help Support for Jabra USB HID devices. @@ -523,26 +501,22 @@ config HID_JABRA config HID_TWINHAN tristate "Twinhan IR remote control" - depends on HID help Support for Twinhan IR remote control. config HID_KENSINGTON tristate "Kensington Slimblade Trackball" - depends on HID default !EXPERT help Support for Kensington Slimblade Trackball. config HID_LCPOWER tristate "LC-Power" - depends on HID help Support for LC-Power RC1000MCE RF remote control. config HID_LED tristate "Simple RGB LED support" - depends on HID depends on LEDS_CLASS help Support for simple RGB LED devices. Currently supported are: @@ -557,7 +531,6 @@ config HID_LED config HID_LENOVO tristate "Lenovo / Thinkpad devices" - depends on HID select NEW_LEDS select LEDS_CLASS help @@ -675,7 +648,6 @@ config LOGIWHEELS_FF config HID_MAGICMOUSE tristate "Apple Magic Mouse/Trackpad multi-touch support" - depends on HID help Support for the Apple Magic Mouse/Trackpad multi-touch. @@ -684,14 +656,12 @@ config HID_MAGICMOUSE config HID_MALTRON tristate "Maltron L90 keyboard" - depends on HID help Adds support for the volume up, volume down, mute, and play/pause buttons of the Maltron L90 keyboard. config HID_MAYFLASH tristate "Mayflash game controller adapter force feedback" - depends on HID select INPUT_FF_MEMLESS help Say Y here if you have HJZ Mayflash PS3 game controller adapters @@ -707,14 +677,12 @@ config HID_MEGAWORLD_FF config HID_REDRAGON tristate "Redragon keyboards" - depends on HID default !EXPERT help Support for Redragon keyboards that need fix-ups to work properly. config HID_MICROSOFT tristate "Microsoft non-fully HID-compliant devices" - depends on HID default !EXPERT select INPUT_FF_MEMLESS help @@ -722,14 +690,12 @@ config HID_MICROSOFT config HID_MONTEREY tristate "Monterey Genius KB29E keyboard" - depends on HID default !EXPERT help Support for Monterey Genius KB29E. config HID_MULTITOUCH tristate "HID Multitouch panels" - depends on HID help Generic support for HID multitouch panels. @@ -775,7 +741,6 @@ config HID_MULTITOUCH config HID_NINTENDO tristate "Nintendo Joy-Con and Pro Controller support" - depends on HID depends on NEW_LEDS depends on LEDS_CLASS select POWER_SUPPLY @@ -811,7 +776,6 @@ config HID_NTRIG config HID_ORTEK tristate "Ortek PKB-1700/WKB-2000/Skycable wireless keyboard and mouse trackpad" - depends on HID help There are certain devices which have LogicalMaximum wrong in the keyboard usage page of their report descriptor. The most prevailing ones so far @@ -824,7 +788,6 @@ config HID_ORTEK config HID_PANTHERLORD tristate "Pantherlord/GreenAsia game controller" - depends on HID help Say Y here if you have a PantherLord/GreenAsia based game controller or adapter. @@ -850,13 +813,11 @@ config HID_PENMOUNT config HID_PETALYNX tristate "Petalynx Maxter remote control" - depends on HID help Support for Petalynx Maxter remote control. config HID_PICOLCD tristate "PicoLCD (graphic version)" - depends on HID help This provides support for Minibox PicoLCD devices, currently only the graphical ones are supported. @@ -922,7 +883,6 @@ config HID_PICOLCD_CIR config HID_PLANTRONICS tristate "Plantronics USB HID Driver" - depends on HID help Provides HID support for Plantronics USB audio devices. Correctly maps vendor unique volume up/down HID usages to @@ -933,7 +893,6 @@ config HID_PLANTRONICS config HID_PLAYSTATION tristate "PlayStation HID Driver" - depends on HID depends on LEDS_CLASS_MULTICOLOR select CRC32 select POWER_SUPPLY @@ -950,16 +909,23 @@ config PLAYSTATION_FF Say Y here if you would like to enable force feedback support for PlayStation game controllers. +config HID_PXRC + tristate "PhoenixRC HID Flight Controller" + depends on HID + help + Support for PhoenixRC HID Flight Controller, a 8-axis flight controller. + + To compile this driver as a module, choose M here: the + module will be called hid-pxrc. + config HID_RAZER tristate "Razer non-fully HID-compliant devices" - depends on HID help Support for Razer devices that are not fully compliant with the HID standard. config HID_PRIMAX tristate "Primax non-fully HID-compliant devices" - depends on HID help Support for Primax devices that are not fully compliant with the HID standard. @@ -981,7 +947,6 @@ config HID_ROCCAT config HID_SAITEK tristate "Saitek (Mad Catz) non-fully HID-compliant devices" - depends on HID help Support for Saitek devices that are not fully compliant with the HID standard. @@ -999,7 +964,6 @@ config HID_SAMSUNG config HID_SEMITEK tristate "Semitek USB keyboards" - depends on HID help Support for Semitek USB keyboards that are not fully compliant with the HID standard. @@ -1050,13 +1014,11 @@ config SONY_FF config HID_SPEEDLINK tristate "Speedlink VAD Cezanne mouse support" - depends on HID help Support for Speedlink Vicious and Divine Cezanne mouse. config HID_STEAM tristate "Steam Controller support" - depends on HID select POWER_SUPPLY help Say Y here if you have a Steam Controller if you want to use it @@ -1065,19 +1027,16 @@ config HID_STEAM config HID_STEELSERIES tristate "Steelseries SRW-S1 steering wheel support" - depends on HID help Support for Steelseries SRW-S1 steering wheel config HID_SUNPLUS tristate "Sunplus wireless desktop" - depends on HID help Support for Sunplus wireless desktop. config HID_RMI tristate "Synaptics RMI4 device support" - depends on HID select RMI4_CORE select RMI4_F03 select RMI4_F11 @@ -1090,7 +1049,6 @@ config HID_RMI config HID_GREENASIA tristate "GreenAsia (Product ID 0x12) game controller support" - depends on HID help Say Y here if you have a GreenAsia (Product ID 0x12) based game controller or adapter. @@ -1112,7 +1070,6 @@ config HID_HYPERV_MOUSE config HID_SMARTJOYPLUS tristate "SmartJoy PLUS PS2/USB adapter support" - depends on HID help Support for SmartJoy PLUS PS2/USB adapter, Super Dual Box, Super Joy Box 3 Pro, Super Dual Box Pro, and Super Joy Box 5 Pro. @@ -1130,20 +1087,23 @@ config SMARTJOYPLUS_FF config HID_TIVO tristate "TiVo Slide Bluetooth remote control support" - depends on HID help Say Y if you have a TiVo Slide Bluetooth remote control. config HID_TOPSEED tristate "TopSeed Cyberlink, BTC Emprex, Conceptronic remote control support" - depends on HID help Say Y if you have a TopSeed Cyberlink or BTC Emprex or Conceptronic CLLRCMCE remote control. +config HID_TOPRE + tristate "Topre REALFORCE keyboards" + depends on HID + help + Say Y for N-key rollover support on Topre REALFORCE R2 108 key keyboards. + config HID_THINGM tristate "ThingM blink(1) USB RGB LED" - depends on HID depends on LEDS_CLASS select HID_LED help @@ -1170,7 +1130,6 @@ config THRUSTMASTER_FF config HID_UDRAW_PS3 tristate "THQ PS3 uDraw tablet" - depends on HID help Say Y here if you want to use the THQ uDraw gaming tablet for the PS3. @@ -1207,7 +1166,6 @@ config HID_WACOM config HID_WIIMOTE tristate "Nintendo Wii / Wii U peripherals" - depends on HID depends on LEDS_CLASS select POWER_SUPPLY select INPUT_FF_MEMLESS @@ -1232,7 +1190,6 @@ config HID_WIIMOTE config HID_XINMO tristate "Xin-Mo non-fully compliant devices" - depends on HID help Support for Xin-Mo devices that are not fully compliant with the HID standard. Currently only supports the Xin-Mo Dual Arcade. Say Y here @@ -1240,7 +1197,6 @@ config HID_XINMO config HID_ZEROPLUS tristate "Zeroplus based game controller support" - depends on HID help Say Y here if you have a Zeroplus based game controller. @@ -1254,13 +1210,12 @@ config ZEROPLUS_FF config HID_ZYDACRON tristate "Zydacron remote control support" - depends on HID help Support for Zydacron remote control. config HID_SENSOR_HUB tristate "HID Sensors framework support" - depends on HID && HAS_IOMEM + depends on HAS_IOMEM select MFD_CORE default n help @@ -1289,7 +1244,6 @@ config HID_SENSOR_CUSTOM_SENSOR config HID_ALPS tristate "Alps HID device support" - depends on HID help Support for Alps I2C HID touchpads and StickPointer. Say Y here if you have a Alps touchpads over i2c-hid or usbhid @@ -1307,7 +1261,7 @@ config HID_MCP2221 will be called hid-mcp2221.ko. config HID_KUNIT_TEST - bool "KUnit tests for HID" if !KUNIT_ALL_TESTS + tristate "KUnit tests for HID" if !KUNIT_ALL_TESTS depends on KUNIT=y depends on HID_UCLOGIC default KUNIT_ALL_TESTS diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index b0bef80981394e616ee7f99da0565de32b9e0850..e8014c1a2f8b632051f6efccc510f3c98d677650 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -101,6 +101,7 @@ hid-picolcd-$(CONFIG_DEBUG_FS) += hid-picolcd_debugfs.o obj-$(CONFIG_HID_PLANTRONICS) += hid-plantronics.o obj-$(CONFIG_HID_PLAYSTATION) += hid-playstation.o obj-$(CONFIG_HID_PRIMAX) += hid-primax.o +obj-$(CONFIG_HID_PXRC) += hid-pxrc.o obj-$(CONFIG_HID_RAZER) += hid-razer.o obj-$(CONFIG_HID_REDRAGON) += hid-redragon.o obj-$(CONFIG_HID_RETRODE) += hid-retrode.o @@ -123,6 +124,7 @@ obj-$(CONFIG_HID_GREENASIA) += hid-gaff.o obj-$(CONFIG_HID_THRUSTMASTER) += hid-tmff.o hid-thrustmaster.o obj-$(CONFIG_HID_TIVO) += hid-tivo.o obj-$(CONFIG_HID_TOPSEED) += hid-topseed.o +obj-$(CONFIG_HID_TOPRE) += hid-topre.o obj-$(CONFIG_HID_TWINHAN) += hid-twinhan.o obj-$(CONFIG_HID_U2FZERO) += hid-u2fzero.o hid-uclogic-objs := hid-uclogic-core.o \ @@ -136,6 +138,7 @@ obj-$(CONFIG_HID_XINMO) += hid-xinmo.o obj-$(CONFIG_HID_ZEROPLUS) += hid-zpff.o obj-$(CONFIG_HID_ZYDACRON) += hid-zydacron.o obj-$(CONFIG_HID_VIEWSONIC) += hid-viewsonic.o +obj-$(CONFIG_HID_VRC2) += hid-vrc2.o wacom-objs := wacom_wac.o wacom_sys.o obj-$(CONFIG_HID_WACOM) += wacom.o @@ -144,8 +147,10 @@ obj-$(CONFIG_HID_WIIMOTE) += hid-wiimote.o obj-$(CONFIG_HID_SENSOR_HUB) += hid-sensor-hub.o obj-$(CONFIG_HID_SENSOR_CUSTOM_SENSOR) += hid-sensor-custom.o -obj-$(CONFIG_HID_KUNIT_TEST) += hid-uclogic-rdesc.o \ +hid-uclogic-test-objs := hid-uclogic-rdesc.o \ + hid-uclogic-params.o \ hid-uclogic-rdesc-test.o +obj-$(CONFIG_HID_KUNIT_TEST) += hid-uclogic-test.o obj-$(CONFIG_USB_HID) += usbhid/ obj-$(CONFIG_USB_MOUSE) += usbhid/ diff --git a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c index 4b90c86ee5f8f6fda3ec75b433a706a23dafaef4..47774b9ab3de0fc85b6ea922db059fcdb4ebe2c8 100644 --- a/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c +++ b/drivers/hid/amd-sfh-hid/amd_sfh_pcie.c @@ -288,11 +288,29 @@ int amd_sfh_irq_init(struct amd_mp2_dev *privdata) return 0; } +static const struct dmi_system_id dmi_nodevs[] = { + { + /* + * Google Chromebooks use Chrome OS Embedded Controller Sensor + * Hub instead of Sensor Hub Fusion and leaves MP2 + * uninitialized, which disables all functionalities, even + * including the registers necessary for feature detections. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Google"), + }, + }, + { } +}; + static int amd_mp2_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct amd_mp2_dev *privdata; int rc; + if (dmi_first_match(dmi_nodevs)) + return -ENODEV; + privdata = devm_kzalloc(&pdev->dev, sizeof(*privdata), GFP_KERNEL); if (!privdata) return -ENOMEM; diff --git a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c index 70436f9fad2f21e4251a8c689671fee2ff8ce52d..4da2f9f62aba3828d31a5ed8135943178b7e59d1 100644 --- a/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c +++ b/drivers/hid/amd-sfh-hid/sfh1_1/amd_sfh_init.c @@ -110,6 +110,8 @@ static int amd_sfh1_1_hid_client_init(struct amd_mp2_dev *privdata) amd_sfh1_1_set_desc_ops(mp2_ops); cl_data->num_hid_devices = amd_sfh_get_sensor_num(privdata, &cl_data->sensor_idx[0]); + if (cl_data->num_hid_devices == 0) + return -ENODEV; INIT_DELAYED_WORK(&cl_data->work, amd_sfh_work); INIT_DELAYED_WORK(&cl_data->work_buffer, amd_sfh_work_buffer); @@ -286,13 +288,13 @@ int amd_sfh1_1_init(struct amd_mp2_dev *mp2) phy_base <<= 21; if (!devm_request_mem_region(dev, phy_base, 128 * 1024, "amd_sfh")) { - dev_err(dev, "can't reserve mmio registers\n"); + dev_dbg(dev, "can't reserve mmio registers\n"); return -ENOMEM; } mp2->vsbase = devm_ioremap(dev, phy_base, 128 * 1024); if (!mp2->vsbase) { - dev_err(dev, "failed to remap vsbase\n"); + dev_dbg(dev, "failed to remap vsbase\n"); return -ENOMEM; } @@ -301,7 +303,7 @@ int amd_sfh1_1_init(struct amd_mp2_dev *mp2) memcpy_fromio(&binfo, mp2->vsbase, sizeof(struct sfh_base_info)); if (binfo.sbase.fw_info.fw_ver == 0 || binfo.sbase.s_list.sl.sensors == 0) { - dev_err(dev, "failed to get sensors\n"); + dev_dbg(dev, "failed to get sensors\n"); return -EOPNOTSUPP; } dev_dbg(dev, "firmware version 0x%x\n", binfo.sbase.fw_info.fw_ver); diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 08c9a9a60ae47a54045aa834aeb92b431205550b..b59c3dafa6a48de3cebd53e14c2475e01485dd6a 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -1212,6 +1212,13 @@ static __u8 *asus_report_fixup(struct hid_device *hdev, __u8 *rdesc, rdesc = new_rdesc; } + if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD && + *rsize == 331 && rdesc[190] == 0x85 && rdesc[191] == 0x5a && + rdesc[204] == 0x95 && rdesc[205] == 0x05) { + hid_info(hdev, "Fixing up Asus N-KEY keyb report descriptor\n"); + rdesc[205] = 0x01; + } + return rdesc; } diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index b7f5566e338d77dffb4ba48f5c817a09dcb854f0..9c1d31f63f850860ca9c672fbf43b586c13c566f 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -55,7 +55,7 @@ MODULE_PARM_DESC(ignore_special_drivers, "Ignore any special drivers and handle */ struct hid_report *hid_register_report(struct hid_device *device, - unsigned int type, unsigned int id, + enum hid_report_type type, unsigned int id, unsigned int application) { struct hid_report_enum *report_enum = device->report_enum + type; @@ -967,7 +967,7 @@ static const char * const hid_report_names[] = { * parsing. */ struct hid_report *hid_validate_values(struct hid_device *hid, - unsigned int type, unsigned int id, + enum hid_report_type type, unsigned int id, unsigned int field_index, unsigned int report_counts) { @@ -1921,7 +1921,7 @@ static struct hid_report *hid_get_report(struct hid_report_enum *report_enum, * DO NOT USE in hid drivers directly, but through hid_hw_request instead. */ int __hid_request(struct hid_device *hid, struct hid_report *report, - int reqtype) + enum hid_class_request reqtype) { char *buf; int ret; @@ -1954,8 +1954,8 @@ out: } EXPORT_SYMBOL_GPL(__hid_request); -int hid_report_raw_event(struct hid_device *hid, int type, u8 *data, u32 size, - int interrupt) +int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, + int interrupt) { struct hid_report_enum *report_enum = hid->report_enum + type; struct hid_report *report; @@ -2019,7 +2019,8 @@ EXPORT_SYMBOL_GPL(hid_report_raw_event); * * This is data entry for lower layers. */ -int hid_input_report(struct hid_device *hid, int type, u8 *data, u32 size, int interrupt) +int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, + int interrupt) { struct hid_report_enum *report_enum; struct hid_driver *hdrv; @@ -2088,6 +2089,7 @@ const struct hid_device_id *hid_match_id(const struct hid_device *hdev, return NULL; } +EXPORT_SYMBOL_GPL(hid_match_id); static const struct hid_device_id hid_hiddev_list[] = { { HID_USB_DEVICE(USB_VENDOR_ID_MGE, USB_DEVICE_ID_MGE_UPS) }, @@ -2352,7 +2354,7 @@ EXPORT_SYMBOL_GPL(hid_hw_close); * @reqtype: hid request type */ void hid_hw_request(struct hid_device *hdev, - struct hid_report *report, int reqtype) + struct hid_report *report, enum hid_class_request reqtype) { if (hdev->ll_driver->request) return hdev->ll_driver->request(hdev, report, reqtype); @@ -2377,7 +2379,7 @@ EXPORT_SYMBOL_GPL(hid_hw_request); */ int hid_hw_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, - size_t len, unsigned char rtype, int reqtype) + size_t len, enum hid_report_type rtype, enum hid_class_request reqtype) { if (len < 1 || len > HID_MAX_BUFFER_SIZE || !buf) return -EINVAL; @@ -2739,10 +2741,12 @@ int hid_add_device(struct hid_device *hdev) hid_warn(hdev, "bad device descriptor (%d)\n", ret); } + hdev->id = atomic_inc_return(&id); + /* XXX hack, any other cleaner solution after the driver core * is converted to allow more than 20 bytes as the device name? */ dev_set_name(&hdev->dev, "%04X:%04X:%04X.%04X", hdev->bus, - hdev->vendor, hdev->product, atomic_inc_return(&id)); + hdev->vendor, hdev->product, hdev->id); hid_debug_register(hdev, dev_name(&hdev->dev)); ret = device_add(&hdev->dev); diff --git a/drivers/hid/hid-debug.c b/drivers/hid/hid-debug.c index 81e7e404a5fce7bc0cf5d779b657475f8862f1f9..2ca6ab600bc9f24c8b6778470f571fddd92ce826 100644 --- a/drivers/hid/hid-debug.c +++ b/drivers/hid/hid-debug.c @@ -1014,7 +1014,8 @@ static const char *absolutes[ABS_CNT] = { [ABS_HAT3Y] = "Hat 3Y", [ABS_PRESSURE] = "Pressure", [ABS_DISTANCE] = "Distance", [ABS_TILT_X] = "XTilt", [ABS_TILT_Y] = "YTilt", [ABS_TOOL_WIDTH] = "ToolWidth", - [ABS_VOLUME] = "Volume", [ABS_MISC] = "Misc", + [ABS_VOLUME] = "Volume", [ABS_PROFILE] = "Profile", + [ABS_MISC] = "Misc", [ABS_MT_TOUCH_MAJOR] = "MTMajor", [ABS_MT_TOUCH_MINOR] = "MTMinor", [ABS_MT_WIDTH_MAJOR] = "MTMajorW", diff --git a/drivers/hid/hid-google-hammer.c b/drivers/hid/hid-google-hammer.c index ff40f1e55c217bf9d37cd685f9df5033112f526a..7ae5f27df54dd6731b1dcef653a6153323cdbfc2 100644 --- a/drivers/hid/hid-google-hammer.c +++ b/drivers/hid/hid-google-hammer.c @@ -608,9 +608,11 @@ static struct hid_driver hammer_driver = { .probe = hammer_probe, .remove = hammer_remove, .feature_mapping = vivaldi_feature_mapping, - .input_configured = vivaldi_input_configured, .input_mapping = hammer_input_mapping, .event = hammer_event, + .driver = { + .dev_groups = vivaldi_attribute_groups, + }, }; static int __init hammer_init(void) diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index 0fb720a96399a8c6bd04e2037a73feaa5b854c7f..da86565f04d4ec1a79253029144ee949e3dddc65 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -185,6 +185,8 @@ #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021 0x029c #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_FINGERPRINT_2021 0x029a #define USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021 0x029f +#define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 +#define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 #define USB_VENDOR_ID_ASUS 0x0486 #define USB_DEVICE_ID_ASUS_T91MT 0x0185 @@ -414,6 +416,7 @@ #define USB_DEVICE_ID_ASUS_UX550_TOUCHSCREEN 0x2706 #define I2C_DEVICE_ID_SURFACE_GO_TOUCHSCREEN 0x261A #define I2C_DEVICE_ID_SURFACE_GO2_TOUCHSCREEN 0x2A1C +#define I2C_DEVICE_ID_LENOVO_YOGA_C630_TOUCHSCREEN 0x279F #define USB_VENDOR_ID_ELECOM 0x056e #define USB_DEVICE_ID_ELECOM_BM084 0x0061 @@ -1228,6 +1231,9 @@ #define USB_DEVICE_ID_TIVO_SLIDE 0x1201 #define USB_DEVICE_ID_TIVO_SLIDE_PRO 0x1203 +#define USB_VENDOR_ID_TOPRE 0x0853 +#define USB_DEVICE_ID_TOPRE_REALFORCE_R2_108 0x0148 + #define USB_VENDOR_ID_TOPSEED 0x0766 #define USB_DEVICE_ID_TOPSEED_CYBERLINK 0x0204 @@ -1276,10 +1282,12 @@ #define USB_DEVICE_ID_YIYNOVA_TABLET 0x004d #define USB_VENDOR_ID_UGEE 0x28bd +#define USB_DEVICE_ID_UGEE_PARBLO_A610_PRO 0x1903 #define USB_DEVICE_ID_UGEE_XPPEN_TABLET_G540 0x0075 #define USB_DEVICE_ID_UGEE_XPPEN_TABLET_G640 0x0094 #define USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO01 0x0042 #define USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_L 0x0935 +#define USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_S 0x0909 #define USB_DEVICE_ID_UGEE_XPPEN_TABLET_STAR06 0x0078 #define USB_DEVICE_ID_UGEE_TABLET_G5 0x0074 #define USB_DEVICE_ID_UGEE_TABLET_EX07S 0x0071 @@ -1383,6 +1391,7 @@ #define USB_VENDOR_ID_MULTIPLE_1781 0x1781 #define USB_DEVICE_ID_RAPHNET_4NES4SNES_OLD 0x0a9d +#define USB_DEVICE_ID_PHOENIXRC 0x0898 #define USB_VENDOR_ID_DRACAL_RAPHNET 0x289b #define USB_DEVICE_ID_RAPHNET_2NES2SNES 0x0002 diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 48c1c02c69f4e7f20cad6cab23da68d9ee8fabb2..859aeb07542e3a76a7dee9850f09d97bafbf010b 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -383,6 +383,8 @@ static const struct hid_device_id hid_battery_quirks[] = { HID_BATTERY_QUIRK_IGNORE }, { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_SURFACE_GO2_TOUCHSCREEN), HID_BATTERY_QUIRK_IGNORE }, + { HID_I2C_DEVICE(USB_VENDOR_ID_ELAN, I2C_DEVICE_ID_LENOVO_YOGA_C630_TOUCHSCREEN), + HID_BATTERY_QUIRK_IGNORE }, {} }; @@ -1532,7 +1534,10 @@ void hidinput_hid_event(struct hid_device *hid, struct hid_field *field, struct * assume ours */ if (!report->tool) - hid_report_set_tool(report, input, usage->code); + report->tool = usage->code; + + /* drivers may have changed the value behind our back, resend it */ + hid_report_set_tool(report, input, report->tool); } else { hid_report_release_tool(report, input, usage->code); } diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 68f9e9d207f42ba21a3607c56eb2a4fc365ed0e7..71a9c258a20bc73b742cb415984d99a057630573 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -41,6 +41,9 @@ module_param(disable_tap_to_click, bool, 0644); MODULE_PARM_DESC(disable_tap_to_click, "Disable Tap-To-Click mode reporting for touchpads (only on the K400 currently)."); +/* Define a non-zero software ID to identify our own requests */ +#define LINUX_KERNEL_SW_ID 0x01 + #define REPORT_ID_HIDPP_SHORT 0x10 #define REPORT_ID_HIDPP_LONG 0x11 #define REPORT_ID_HIDPP_VERY_LONG 0x12 @@ -71,21 +74,18 @@ MODULE_PARM_DESC(disable_tap_to_click, #define HIDPP_QUIRK_NO_HIDINPUT BIT(23) #define HIDPP_QUIRK_FORCE_OUTPUT_REPORTS BIT(24) #define HIDPP_QUIRK_UNIFYING BIT(25) -#define HIDPP_QUIRK_HI_RES_SCROLL_1P0 BIT(26) -#define HIDPP_QUIRK_HI_RES_SCROLL_X2120 BIT(27) -#define HIDPP_QUIRK_HI_RES_SCROLL_X2121 BIT(28) -#define HIDPP_QUIRK_HIDPP_WHEELS BIT(29) -#define HIDPP_QUIRK_HIDPP_EXTRA_MOUSE_BTNS BIT(30) -#define HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS BIT(31) +#define HIDPP_QUIRK_HIDPP_WHEELS BIT(26) +#define HIDPP_QUIRK_HIDPP_EXTRA_MOUSE_BTNS BIT(27) +#define HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS BIT(28) /* These are just aliases for now */ #define HIDPP_QUIRK_KBD_SCROLL_WHEEL HIDPP_QUIRK_HIDPP_WHEELS #define HIDPP_QUIRK_KBD_ZOOM_WHEEL HIDPP_QUIRK_HIDPP_WHEELS /* Convenience constant to check for any high-res support. */ -#define HIDPP_QUIRK_HI_RES_SCROLL (HIDPP_QUIRK_HI_RES_SCROLL_1P0 | \ - HIDPP_QUIRK_HI_RES_SCROLL_X2120 | \ - HIDPP_QUIRK_HI_RES_SCROLL_X2121) +#define HIDPP_CAPABILITY_HI_RES_SCROLL (HIDPP_CAPABILITY_HIDPP10_FAST_SCROLL | \ + HIDPP_CAPABILITY_HIDPP20_HI_RES_SCROLL | \ + HIDPP_CAPABILITY_HIDPP20_HI_RES_WHEEL) #define HIDPP_QUIRK_DELAYED_INIT HIDPP_QUIRK_NO_HIDINPUT @@ -96,6 +96,9 @@ MODULE_PARM_DESC(disable_tap_to_click, #define HIDPP_CAPABILITY_BATTERY_VOLTAGE BIT(4) #define HIDPP_CAPABILITY_BATTERY_PERCENTAGE BIT(5) #define HIDPP_CAPABILITY_UNIFIED_BATTERY BIT(6) +#define HIDPP_CAPABILITY_HIDPP20_HI_RES_WHEEL BIT(7) +#define HIDPP_CAPABILITY_HIDPP20_HI_RES_SCROLL BIT(8) +#define HIDPP_CAPABILITY_HIDPP10_FAST_SCROLL BIT(9) #define lg_map_key_clear(c) hid_map_usage_clear(hi, usage, bit, max, EV_KEY, (c)) @@ -343,7 +346,7 @@ static int hidpp_send_fap_command_sync(struct hidpp_device *hidpp, else message->report_id = REPORT_ID_HIDPP_LONG; message->fap.feature_index = feat_index; - message->fap.funcindex_clientid = funcindex_clientid; + message->fap.funcindex_clientid = funcindex_clientid | LINUX_KERNEL_SW_ID; memcpy(&message->fap.params, params, param_count); ret = hidpp_send_message_sync(hidpp, message, response); @@ -856,8 +859,8 @@ static int hidpp_unifying_init(struct hidpp_device *hidpp) #define HIDPP_PAGE_ROOT 0x0000 #define HIDPP_PAGE_ROOT_IDX 0x00 -#define CMD_ROOT_GET_FEATURE 0x01 -#define CMD_ROOT_GET_PROTOCOL_VERSION 0x11 +#define CMD_ROOT_GET_FEATURE 0x00 +#define CMD_ROOT_GET_PROTOCOL_VERSION 0x10 static int hidpp_root_get_feature(struct hidpp_device *hidpp, u16 feature, u8 *feature_index, u8 *feature_type) @@ -934,9 +937,9 @@ print_version: #define HIDPP_PAGE_GET_DEVICE_NAME_TYPE 0x0005 -#define CMD_GET_DEVICE_NAME_TYPE_GET_COUNT 0x01 -#define CMD_GET_DEVICE_NAME_TYPE_GET_DEVICE_NAME 0x11 -#define CMD_GET_DEVICE_NAME_TYPE_GET_TYPE 0x21 +#define CMD_GET_DEVICE_NAME_TYPE_GET_COUNT 0x00 +#define CMD_GET_DEVICE_NAME_TYPE_GET_DEVICE_NAME 0x10 +#define CMD_GET_DEVICE_NAME_TYPE_GET_TYPE 0x20 static int hidpp_devicenametype_get_count(struct hidpp_device *hidpp, u8 feature_index, u8 *nameLength) @@ -1966,8 +1969,8 @@ static int hidpp_touchpad_fw_items_set(struct hidpp_device *hidpp, #define HIDPP_PAGE_TOUCHPAD_RAW_XY 0x6100 -#define CMD_TOUCHPAD_GET_RAW_INFO 0x01 -#define CMD_TOUCHPAD_SET_RAW_REPORT_STATE 0x21 +#define CMD_TOUCHPAD_GET_RAW_INFO 0x00 +#define CMD_TOUCHPAD_SET_RAW_REPORT_STATE 0x20 #define EVENT_TOUCHPAD_RAW_XY 0x00 @@ -3415,14 +3418,14 @@ static int hi_res_scroll_enable(struct hidpp_device *hidpp) int ret; u8 multiplier = 1; - if (hidpp->quirks & HIDPP_QUIRK_HI_RES_SCROLL_X2121) { + if (hidpp->capabilities & HIDPP_CAPABILITY_HIDPP20_HI_RES_WHEEL) { ret = hidpp_hrw_set_wheel_mode(hidpp, false, true, false); if (ret == 0) ret = hidpp_hrw_get_wheel_capability(hidpp, &multiplier); - } else if (hidpp->quirks & HIDPP_QUIRK_HI_RES_SCROLL_X2120) { + } else if (hidpp->capabilities & HIDPP_CAPABILITY_HIDPP20_HI_RES_SCROLL) { ret = hidpp_hrs_set_highres_scrolling_mode(hidpp, true, &multiplier); - } else /* if (hidpp->quirks & HIDPP_QUIRK_HI_RES_SCROLL_1P0) */ { + } else /* if (hidpp->capabilities & HIDPP_CAPABILITY_HIDPP10_FAST_SCROLL) */ { ret = hidpp10_enable_scrolling_acceleration(hidpp); multiplier = 8; } @@ -3437,6 +3440,49 @@ static int hi_res_scroll_enable(struct hidpp_device *hidpp) return 0; } +static int hidpp_initialize_hires_scroll(struct hidpp_device *hidpp) +{ + int ret; + unsigned long capabilities; + + capabilities = hidpp->capabilities; + + if (hidpp->protocol_major >= 2) { + u8 feature_index; + u8 feature_type; + + ret = hidpp_root_get_feature(hidpp, HIDPP_PAGE_HIRES_WHEEL, + &feature_index, &feature_type); + if (!ret) { + hidpp->capabilities |= HIDPP_CAPABILITY_HIDPP20_HI_RES_WHEEL; + hid_dbg(hidpp->hid_dev, "Detected HID++ 2.0 hi-res scroll wheel\n"); + return 0; + } + ret = hidpp_root_get_feature(hidpp, HIDPP_PAGE_HI_RESOLUTION_SCROLLING, + &feature_index, &feature_type); + if (!ret) { + hidpp->capabilities |= HIDPP_CAPABILITY_HIDPP20_HI_RES_SCROLL; + hid_dbg(hidpp->hid_dev, "Detected HID++ 2.0 hi-res scrolling\n"); + } + } else { + struct hidpp_report response; + + ret = hidpp_send_rap_command_sync(hidpp, + REPORT_ID_HIDPP_SHORT, + HIDPP_GET_REGISTER, + HIDPP_ENABLE_FAST_SCROLL, + NULL, 0, &response); + if (!ret) { + hidpp->capabilities |= HIDPP_CAPABILITY_HIDPP10_FAST_SCROLL; + hid_dbg(hidpp->hid_dev, "Detected HID++ 1.0 fast scroll\n"); + } + } + + if (hidpp->capabilities == capabilities) + hid_dbg(hidpp->hid_dev, "Did not detect HID++ hi-res scrolling hardware support\n"); + return 0; +} + /* -------------------------------------------------------------------------- */ /* Generic HID++ devices */ /* -------------------------------------------------------------------------- */ @@ -3691,8 +3737,9 @@ static int hidpp_event(struct hid_device *hdev, struct hid_field *field, * cases we must return early (falling back to default behaviour) to * avoid a crash in hidpp_scroll_counter_handle_scroll. */ - if (!(hidpp->quirks & HIDPP_QUIRK_HI_RES_SCROLL) || value == 0 - || hidpp->input == NULL || counter->wheel_multiplier == 0) + if (!(hidpp->capabilities & HIDPP_CAPABILITY_HI_RES_SCROLL) + || value == 0 || hidpp->input == NULL + || counter->wheel_multiplier == 0) return 0; hidpp_scroll_counter_handle_scroll(hidpp->input, counter, value); @@ -3924,6 +3971,7 @@ static void hidpp_connect_event(struct hidpp_device *hidpp) } hidpp_initialize_battery(hidpp); + hidpp_initialize_hires_scroll(hidpp); /* forward current battery state */ if (hidpp->capabilities & HIDPP_CAPABILITY_HIDPP10_BATTERY) { @@ -3943,7 +3991,7 @@ static void hidpp_connect_event(struct hidpp_device *hidpp) if (hidpp->battery.ps) power_supply_changed(hidpp->battery.ps); - if (hidpp->quirks & HIDPP_QUIRK_HI_RES_SCROLL) + if (hidpp->capabilities & HIDPP_CAPABILITY_HI_RES_SCROLL) hi_res_scroll_enable(hidpp); if (!(hidpp->quirks & HIDPP_QUIRK_NO_HIDINPUT) || hidpp->delayed_input) @@ -3959,8 +4007,10 @@ static void hidpp_connect_event(struct hidpp_device *hidpp) hidpp_populate_input(hidpp, input); ret = input_register_device(input); - if (ret) + if (ret) { input_free_device(input); + return; + } hidpp->delayed_input = input; } @@ -4219,6 +4269,21 @@ static void hidpp_remove(struct hid_device *hdev) mutex_destroy(&hidpp->send_mutex); } +static const struct hid_device_id unhandled_hidpp_devices[] = { + /* Logitech Harmony Adapter for PS3, handled in hid-sony */ + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_HARMONY_PS3) }, + /* Handled in hid-generic */ + { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_DINOVO_EDGE_KBD) }, + {} +}; + +static bool hidpp_match(struct hid_device *hdev, + bool ignore_special_driver) +{ + /* Refuse to handle devices handled by other HID drivers */ + return !hid_match_id(hdev, unhandled_hidpp_devices); +} + #define LDJ_DEVICE(product) \ HID_DEVICE(BUS_USB, HID_GROUP_LOGITECH_DJ_DEVICE, \ USB_VENDOR_ID_LOGITECH, (product)) @@ -4239,42 +4304,9 @@ static const struct hid_device_id hidpp_devices[] = { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, USB_DEVICE_ID_LOGITECH_T651), .driver_data = HIDPP_QUIRK_CLASS_WTP }, - { /* Mouse Logitech Anywhere MX */ - LDJ_DEVICE(0x1017), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_1P0 }, - { /* Mouse Logitech Cube */ - LDJ_DEVICE(0x4010), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2120 }, - { /* Mouse Logitech M335 */ - LDJ_DEVICE(0x4050), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech M515 */ - LDJ_DEVICE(0x4007), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2120 }, { /* Mouse logitech M560 */ LDJ_DEVICE(0x402d), - .driver_data = HIDPP_QUIRK_DELAYED_INIT | HIDPP_QUIRK_CLASS_M560 - | HIDPP_QUIRK_HI_RES_SCROLL_X2120 }, - { /* Mouse Logitech M705 (firmware RQM17) */ - LDJ_DEVICE(0x101b), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_1P0 }, - { /* Mouse Logitech M705 (firmware RQM67) */ - LDJ_DEVICE(0x406d), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech M720 */ - LDJ_DEVICE(0x405e), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech MX Anywhere 2 */ - LDJ_DEVICE(0x404a), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0x4072), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0xb013), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0xb018), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0xb01f), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech MX Anywhere 2S */ - LDJ_DEVICE(0x406a), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech MX Master */ - LDJ_DEVICE(0x4041), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0x4060), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { LDJ_DEVICE(0x4071), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech MX Master 2S */ - LDJ_DEVICE(0x4069), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech MX Master 3 */ - LDJ_DEVICE(0x4082), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* Mouse Logitech Performance MX */ - LDJ_DEVICE(0x101a), .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_1P0 }, + .driver_data = HIDPP_QUIRK_DELAYED_INIT | HIDPP_QUIRK_CLASS_M560 }, { /* Keyboard logitech K400 */ LDJ_DEVICE(0x4024), .driver_data = HIDPP_QUIRK_CLASS_K400 }, @@ -4335,18 +4367,9 @@ static const struct hid_device_id hidpp_devices[] = { { /* MX5500 keyboard over Bluetooth */ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb30b), .driver_data = HIDPP_QUIRK_HIDPP_CONSUMER_VENDOR_KEYS }, - { /* M-RCQ142 V470 Cordless Laser Mouse over Bluetooth */ - HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb008) }, - { /* MX Master mouse over Bluetooth */ - HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb012), - .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* MX Ergo trackball over Bluetooth */ - HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb01d) }, - { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb01e), - .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, - { /* MX Master 3 mouse over Bluetooth */ - HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_LOGITECH, 0xb023), - .driver_data = HIDPP_QUIRK_HI_RES_SCROLL_X2121 }, + + { /* And try to enable HID++ for all the Logitech Bluetooth devices */ + HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_ANY, USB_VENDOR_ID_LOGITECH, HID_ANY_ID) }, {} }; @@ -4360,6 +4383,7 @@ static const struct hid_usage_id hidpp_usages[] = { static struct hid_driver hidpp_driver = { .name = "logitech-hidpp-device", .id_table = hidpp_devices, + .match = hidpp_match, .report_fixup = hidpp_report_fixup, .probe = hidpp_probe, .remove = hidpp_remove, diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 2e72922e36f56524cac964ce697e2fef31793d2d..91a4d3fc30e0802a7e85acc94dbab98ef1d4f753 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -1186,7 +1186,7 @@ static void mt_touch_report(struct hid_device *hid, int contact_count = -1; /* sticky fingers release in progress, abort */ - if (test_and_set_bit(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) + if (test_and_set_bit_lock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) return; scantime = *app->scantime; @@ -1267,7 +1267,7 @@ static void mt_touch_report(struct hid_device *hid, del_timer(&td->release_timer); } - clear_bit(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); + clear_bit_unlock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); } static int mt_touch_input_configured(struct hid_device *hdev, @@ -1699,11 +1699,11 @@ static void mt_expired_timeout(struct timer_list *t) * An input report came in just before we release the sticky fingers, * it will take care of the sticky fingers. */ - if (test_and_set_bit(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) + if (test_and_set_bit_lock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags)) return; if (test_bit(MT_IO_FLAGS_PENDING_SLOTS, &td->mt_io_flags)) mt_release_contacts(hdev); - clear_bit(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); + clear_bit_unlock(MT_IO_FLAGS_RUNNING, &td->mt_io_flags); } static int mt_probe(struct hid_device *hdev, const struct hid_device_id *id) diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c index 92ac4f605f134606486b20027b80e097b0d67223..5bfc0c45046085b84f90a0c2dfb339f0ad0a6c2d 100644 --- a/drivers/hid/hid-nintendo.c +++ b/drivers/hid/hid-nintendo.c @@ -760,12 +760,31 @@ static int joycon_read_stick_calibration(struct joycon_ctlr *ctlr, u16 cal_addr, cal_y->max = cal_y->center + y_max_above; cal_y->min = cal_y->center - y_min_below; - return 0; + /* check if calibration values are plausible */ + if (cal_x->min >= cal_x->center || cal_x->center >= cal_x->max || + cal_y->min >= cal_y->center || cal_y->center >= cal_y->max) + ret = -EINVAL; + + return ret; } static const u16 DFLT_STICK_CAL_CEN = 2000; static const u16 DFLT_STICK_CAL_MAX = 3500; static const u16 DFLT_STICK_CAL_MIN = 500; +static void joycon_use_default_calibration(struct hid_device *hdev, + struct joycon_stick_cal *cal_x, + struct joycon_stick_cal *cal_y, + const char *stick, int ret) +{ + hid_warn(hdev, + "Failed to read %s stick cal, using defaults; e=%d\n", + stick, ret); + + cal_x->center = cal_y->center = DFLT_STICK_CAL_CEN; + cal_x->max = cal_y->max = DFLT_STICK_CAL_MAX; + cal_x->min = cal_y->min = DFLT_STICK_CAL_MIN; +} + static int joycon_request_calibration(struct joycon_ctlr *ctlr) { u16 left_stick_addr = JC_CAL_FCT_DATA_LEFT_ADDR; @@ -793,38 +812,24 @@ static int joycon_request_calibration(struct joycon_ctlr *ctlr) &ctlr->left_stick_cal_x, &ctlr->left_stick_cal_y, true); - if (ret) { - hid_warn(ctlr->hdev, - "Failed to read left stick cal, using dflts; e=%d\n", - ret); - - ctlr->left_stick_cal_x.center = DFLT_STICK_CAL_CEN; - ctlr->left_stick_cal_x.max = DFLT_STICK_CAL_MAX; - ctlr->left_stick_cal_x.min = DFLT_STICK_CAL_MIN; - ctlr->left_stick_cal_y.center = DFLT_STICK_CAL_CEN; - ctlr->left_stick_cal_y.max = DFLT_STICK_CAL_MAX; - ctlr->left_stick_cal_y.min = DFLT_STICK_CAL_MIN; - } + if (ret) + joycon_use_default_calibration(ctlr->hdev, + &ctlr->left_stick_cal_x, + &ctlr->left_stick_cal_y, + "left", ret); /* read the right stick calibration data */ ret = joycon_read_stick_calibration(ctlr, right_stick_addr, &ctlr->right_stick_cal_x, &ctlr->right_stick_cal_y, false); - if (ret) { - hid_warn(ctlr->hdev, - "Failed to read right stick cal, using dflts; e=%d\n", - ret); - - ctlr->right_stick_cal_x.center = DFLT_STICK_CAL_CEN; - ctlr->right_stick_cal_x.max = DFLT_STICK_CAL_MAX; - ctlr->right_stick_cal_x.min = DFLT_STICK_CAL_MIN; - ctlr->right_stick_cal_y.center = DFLT_STICK_CAL_CEN; - ctlr->right_stick_cal_y.max = DFLT_STICK_CAL_MAX; - ctlr->right_stick_cal_y.min = DFLT_STICK_CAL_MIN; - } + if (ret) + joycon_use_default_calibration(ctlr->hdev, + &ctlr->right_stick_cal_x, + &ctlr->right_stick_cal_y, + "right", ret); hid_dbg(ctlr->hdev, "calibration:\n" "l_x_c=%d l_x_max=%d l_x_min=%d\n" @@ -1221,6 +1226,7 @@ static void joycon_parse_report(struct joycon_ctlr *ctlr, spin_lock_irqsave(&ctlr->lock, flags); if (IS_ENABLED(CONFIG_NINTENDO_FF) && rep->vibrator_report && + ctlr->ctlr_state != JOYCON_CTLR_STATE_REMOVED && (msecs - ctlr->rumble_msecs) >= JC_RUMBLE_PERIOD_MS && (ctlr->rumble_queue_head != ctlr->rumble_queue_tail || ctlr->rumble_zero_countdown > 0)) { @@ -1545,12 +1551,13 @@ static int joycon_set_rumble(struct joycon_ctlr *ctlr, u16 amp_r, u16 amp_l, ctlr->rumble_queue_head = 0; memcpy(ctlr->rumble_data[ctlr->rumble_queue_head], data, JC_RUMBLE_DATA_SIZE); - spin_unlock_irqrestore(&ctlr->lock, flags); /* don't wait for the periodic send (reduces latency) */ - if (schedule_now) + if (schedule_now && ctlr->ctlr_state != JOYCON_CTLR_STATE_REMOVED) queue_work(ctlr->rumble_queue, &ctlr->rumble_worker); + spin_unlock_irqrestore(&ctlr->lock, flags); + return 0; } @@ -1902,9 +1909,8 @@ static int joycon_leds_create(struct joycon_ctlr *ctlr) /* Set the home LED to 0 as default state */ ret = joycon_home_led_brightness_set(led, 0); if (ret) { - hid_err(hdev, "Failed to set home LED dflt; ret=%d\n", - ret); - return ret; + hid_warn(hdev, "Failed to set home LED default, unregistering home LED"); + devm_led_classdev_unregister(&hdev->dev, led); } } diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index b1b5721b5d8f7a22cc6e3dbf1fb3c32b3c35c6e9..40050eb85c0a5324d52efa743bf3a8d3d8fa0f4a 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -692,15 +692,12 @@ static ssize_t hardware_version_show(struct device *dev, static DEVICE_ATTR_RO(hardware_version); -static struct attribute *ps_device_attributes[] = { +static struct attribute *ps_device_attrs[] = { &dev_attr_firmware_version.attr, &dev_attr_hardware_version.attr, NULL }; - -static const struct attribute_group ps_device_attribute_group = { - .attrs = ps_device_attributes, -}; +ATTRIBUTE_GROUPS(ps_device); static int dualsense_get_calibration_data(struct dualsense *ds) { @@ -1448,12 +1445,6 @@ static int ps_probe(struct hid_device *hdev, const struct hid_device_id *id) } } - ret = devm_device_add_group(&hdev->dev, &ps_device_attribute_group); - if (ret) { - hid_err(hdev, "Failed to register sysfs nodes.\n"); - goto err_close; - } - return ret; err_close: @@ -1487,6 +1478,9 @@ static struct hid_driver ps_driver = { .probe = ps_probe, .remove = ps_remove, .raw_event = ps_raw_event, + .driver = { + .dev_groups = ps_device_groups, + }, }; static int __init ps_init(void) diff --git a/drivers/hid/hid-pxrc.c b/drivers/hid/hid-pxrc.c new file mode 100644 index 0000000000000000000000000000000000000000..b0e517f9cde7b48ab11bf3ebfcd97cca80e967d6 --- /dev/null +++ b/drivers/hid/hid-pxrc.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for PhoenixRC 8-axis flight controller + * + * Copyright (C) 2022 Marcus Folkesson + */ + +#include +#include +#include + +#include "hid-ids.h" + +struct pxrc_priv { + u8 slider; + u8 dial; + bool alternate; +}; + +static __u8 pxrc_rdesc_fixed[] = { + 0x05, 0x01, // Usage Page (Generic Desktop Ctrls) + 0x09, 0x04, // Usage (Joystick) + 0xA1, 0x01, // Collection (Application) + 0x09, 0x01, // Usage (Pointer) + 0xA1, 0x00, // Collection (Physical) + 0x09, 0x30, // Usage (X) + 0x09, 0x36, // Usage (Slider) + 0x09, 0x31, // Usage (Y) + 0x09, 0x32, // Usage (Z) + 0x09, 0x33, // Usage (Rx) + 0x09, 0x34, // Usage (Ry) + 0x09, 0x35, // Usage (Rz) + 0x09, 0x37, // Usage (Dial) + 0x15, 0x00, // Logical Minimum (0) + 0x26, 0xFF, 0x00, // Logical Maximum (255) + 0x35, 0x00, // Physical Minimum (0) + 0x46, 0xFF, 0x00, // Physical Maximum (255) + 0x75, 0x08, // Report Size (8) + 0x95, 0x08, // Report Count (8) + 0x81, 0x02, // Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, // End Collection + 0xC0, // End Collection +}; + +static __u8 *pxrc_report_fixup(struct hid_device *hdev, __u8 *rdesc, + unsigned int *rsize) +{ + hid_info(hdev, "fixing up PXRC report descriptor\n"); + *rsize = sizeof(pxrc_rdesc_fixed); + return pxrc_rdesc_fixed; +} + +static int pxrc_raw_event(struct hid_device *hdev, struct hid_report *report, + u8 *data, int size) +{ + struct pxrc_priv *priv = hid_get_drvdata(hdev); + + if (priv->alternate) + priv->slider = data[7]; + else + priv->dial = data[7]; + + data[1] = priv->slider; + data[7] = priv->dial; + + priv->alternate = !priv->alternate; + return 0; +} + +static int pxrc_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + int ret; + struct pxrc_priv *priv; + + priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + hid_set_drvdata(hdev, priv); + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "parse failed\n"); + return ret; + } + + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); + if (ret) { + hid_err(hdev, "hw start failed\n"); + return ret; + } + + return 0; +} + +static const struct hid_device_id pxrc_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_MULTIPLE_1781, USB_DEVICE_ID_PHOENIXRC) }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(hid, pxrc_devices); + +static struct hid_driver pxrc_driver = { + .name = "hid-pxrc", + .id_table = pxrc_devices, + .report_fixup = pxrc_report_fixup, + .probe = pxrc_probe, + .raw_event = pxrc_raw_event, +}; +module_hid_driver(pxrc_driver); + +MODULE_AUTHOR("Marcus Folkesson "); +MODULE_DESCRIPTION("HID driver for PXRC 8-axis flight controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-quirks.c b/drivers/hid/hid-quirks.c index dc67717d2dabc4f37e98f061f39bdc935f36db4c..70f602c64fd13ccee717af7b3857aa21ec4a5926 100644 --- a/drivers/hid/hid-quirks.c +++ b/drivers/hid/hid-quirks.c @@ -314,6 +314,8 @@ static const struct hid_device_id hid_have_special_driver[] = { { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021) }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_FINGERPRINT_2021) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT) }, + { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY) }, #endif #if IS_ENABLED(CONFIG_HID_APPLEIR) { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_IRCONTROL) }, diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c index 311eee599ce97e819ba9d2b25a15de70696468bc..bb1f423f4ace3da1090517df34cc6a0c8d0d04b4 100644 --- a/drivers/hid/hid-rmi.c +++ b/drivers/hid/hid-rmi.c @@ -237,8 +237,7 @@ static int rmi_hid_read_block(struct rmi_transport_dev *xport, u16 addr, read_input_count = data->readReport[1]; memcpy(buf + bytes_read, &data->readReport[2], - read_input_count < bytes_needed ? - read_input_count : bytes_needed); + min(read_input_count, bytes_needed)); bytes_read += read_input_count; bytes_needed -= read_input_count; @@ -347,8 +346,7 @@ static int rmi_read_data_event(struct hid_device *hdev, u8 *data, int size) return 0; } - memcpy(hdata->readReport, data, size < hdata->input_report_size ? - size : hdata->input_report_size); + memcpy(hdata->readReport, data, min((u32)size, hdata->input_report_size)); set_bit(RMI_READ_DATA_PENDING, &hdata->flags); wake_up(&hdata->wait); diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index 26373b82fe812510a6755a15ef96755e3396231e..6da80e442fdd1065ed5b5e8b871e402f74b099cf 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -257,6 +257,8 @@ int roccat_report_event(int minor, u8 const *data) if (!new_value) return -ENOMEM; + mutex_lock(&device->cbuf_lock); + report = &device->cbuf[device->cbuf_end]; /* passing NULL is safe */ @@ -276,6 +278,8 @@ int roccat_report_event(int minor, u8 const *data) reader->cbuf_start = (reader->cbuf_start + 1) % ROCCAT_CBUF_SIZE; } + mutex_unlock(&device->cbuf_lock); + wake_up_interruptible(&device->wait); return 0; } diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c index 60ec2b29d54de49e0b19c57cc57288efa47af7a3..03691cdcfb8e10d2bcd3523528f5f423928e6d69 100644 --- a/drivers/hid/hid-sony.c +++ b/drivers/hid/hid-sony.c @@ -368,7 +368,7 @@ static const unsigned int buzz_keymap[] = { }; /* The Navigation controller is a partial DS3 and uses the same HID report - * and hence the same keymap indices, however not not all axes/buttons + * and hence the same keymap indices, however not all axes/buttons * are physically present. We use the same axis and button mapping as * the DS3, which uses the Linux gamepad spec. */ diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c index a3b151b29bd711c2f87a6d4b6de8ac43b9219429..8ee43cb225fce10dd1129df6e3a45de5dc84f945 100644 --- a/drivers/hid/hid-steam.c +++ b/drivers/hid/hid-steam.c @@ -134,6 +134,11 @@ static int steam_recv_report(struct steam_device *steam, int ret; r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0]; + if (!r) { + hid_err(steam->hdev, "No HID_FEATURE_REPORT submitted - nothing to read\n"); + return -EINVAL; + } + if (hid_report_len(r) < 64) return -EINVAL; @@ -165,6 +170,11 @@ static int steam_send_report(struct steam_device *steam, int ret; r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0]; + if (!r) { + hid_err(steam->hdev, "No HID_FEATURE_REPORT submitted - nothing to read\n"); + return -EINVAL; + } + if (hid_report_len(r) < 64) return -EINVAL; @@ -246,7 +256,7 @@ static int steam_get_serial(struct steam_device *steam) if (reply[0] != 0xae || reply[1] != 0x15 || reply[2] != 0x01) return -EIO; reply[3 + STEAM_SERIAL_LEN] = 0; - strlcpy(steam->serial_no, reply + 3, sizeof(steam->serial_no)); + strscpy(steam->serial_no, reply + 3, sizeof(steam->serial_no)); return 0; } @@ -514,7 +524,7 @@ static int steam_register(struct steam_device *steam) */ mutex_lock(&steam->mutex); if (steam_get_serial(steam) < 0) - strlcpy(steam->serial_no, "XXXXXXXXXX", + strscpy(steam->serial_no, "XXXXXXXXXX", sizeof(steam->serial_no)); mutex_unlock(&steam->mutex); @@ -689,9 +699,9 @@ static struct hid_device *steam_create_client_hid(struct hid_device *hdev) client_hdev->version = hdev->version; client_hdev->type = hdev->type; client_hdev->country = hdev->country; - strlcpy(client_hdev->name, hdev->name, + strscpy(client_hdev->name, hdev->name, sizeof(client_hdev->name)); - strlcpy(client_hdev->phys, hdev->phys, + strscpy(client_hdev->phys, hdev->phys, sizeof(client_hdev->phys)); /* * Since we use the same device info than the real interface to diff --git a/drivers/hid/hid-thrustmaster.c b/drivers/hid/hid-thrustmaster.c index c3e6d69fdfbd9775da293634a9c8404b73de4517..cf1679b0d4fbb5194e1aa4d63c62db88a09f2b8b 100644 --- a/drivers/hid/hid-thrustmaster.c +++ b/drivers/hid/hid-thrustmaster.c @@ -67,12 +67,13 @@ static const struct tm_wheel_info tm_wheels_infos[] = { {0x0200, 0x0005, "Thrustmaster T300RS (Missing Attachment)"}, {0x0206, 0x0005, "Thrustmaster T300RS"}, {0x0209, 0x0005, "Thrustmaster T300RS (Open Wheel Attachment)"}, + {0x020a, 0x0005, "Thrustmaster T300RS (Sparco R383 Mod)"}, {0x0204, 0x0005, "Thrustmaster T300 Ferrari Alcantara Edition"}, {0x0002, 0x0002, "Thrustmaster T500RS"} //{0x0407, 0x0001, "Thrustmaster TMX"} }; -static const uint8_t tm_wheels_infos_length = 4; +static const uint8_t tm_wheels_infos_length = 7; /* * This structs contains (in little endian) the response data diff --git a/drivers/hid/hid-topre.c b/drivers/hid/hid-topre.c new file mode 100644 index 0000000000000000000000000000000000000000..88a91cdad5f800e951121b6e252fa7780629df22 --- /dev/null +++ b/drivers/hid/hid-topre.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * HID driver for Topre REALFORCE Keyboards + * + * Copyright (c) 2022 Harry Stern + * + * Based on the hid-macally driver + */ + +#include +#include + +#include "hid-ids.h" + +MODULE_AUTHOR("Harry Stern "); +MODULE_DESCRIPTION("REALFORCE R2 Keyboard driver"); +MODULE_LICENSE("GPL"); + +/* + * Fix the REALFORCE R2's non-boot interface's report descriptor to match the + * events it's actually sending. It claims to send array events but is instead + * sending variable events. + */ +static __u8 *topre_report_fixup(struct hid_device *hdev, __u8 *rdesc, + unsigned int *rsize) +{ + if (*rsize >= 119 && rdesc[69] == 0x29 && rdesc[70] == 0xe7 && + rdesc[71] == 0x81 && rdesc[72] == 0x00) { + hid_info(hdev, + "fixing up Topre REALFORCE keyboard report descriptor\n"); + rdesc[72] = 0x02; + } + return rdesc; +} + +static const struct hid_device_id topre_id_table[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_TOPRE, + USB_DEVICE_ID_TOPRE_REALFORCE_R2_108) }, + { } +}; +MODULE_DEVICE_TABLE(hid, topre_id_table); + +static struct hid_driver topre_driver = { + .name = "topre", + .id_table = topre_id_table, + .report_fixup = topre_report_fixup, +}; + +module_hid_driver(topre_driver); diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c index 47a17375c7fce709e9001d0e5fb51e26e3ea4a47..0fbc408c26070e6072ef0f939bf2426a9c1e8180 100644 --- a/drivers/hid/hid-uclogic-core.c +++ b/drivers/hid/hid-uclogic-core.c @@ -153,6 +153,7 @@ static int uclogic_input_configured(struct hid_device *hdev, suffix = "Pad"; break; case HID_DG_PEN: + case HID_DG_DIGITIZER: suffix = "Pen"; break; case HID_CP_CONSUMER_CONTROL: @@ -509,6 +510,8 @@ static const struct hid_device_id uclogic_devices[] = { USB_DEVICE_ID_UGTIZER_TABLET_GP0610) }, { HID_USB_DEVICE(USB_VENDOR_ID_UGTIZER, USB_DEVICE_ID_UGTIZER_TABLET_GT5040) }, + { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, + USB_DEVICE_ID_UGEE_PARBLO_A610_PRO) }, { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_TABLET_G5) }, { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, @@ -523,6 +526,8 @@ static const struct hid_device_id uclogic_devices[] = { USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO01) }, { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_L) }, + { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, + USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_S) }, { HID_USB_DEVICE(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_STAR06) }, { } diff --git a/drivers/hid/hid-uclogic-params-test.c b/drivers/hid/hid-uclogic-params-test.c new file mode 100644 index 0000000000000000000000000000000000000000..57ef5d3e4b74c5f3b3ebf54d0841fe032adbcd0e --- /dev/null +++ b/drivers/hid/hid-uclogic-params-test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/* + * HID driver for UC-Logic devices not fully compliant with HID standard + * + * Copyright (c) 2022 José Expósito + */ + +#include +#include "./hid-uclogic-params.h" +#include "./hid-uclogic-rdesc.h" + +#define MAX_STR_DESC_SIZE 14 + +struct uclogic_parse_ugee_v2_desc_case { + const char *name; + int res; + const __u8 str_desc[MAX_STR_DESC_SIZE]; + size_t str_desc_size; + const s32 desc_params[UCLOGIC_RDESC_PH_ID_NUM]; + enum uclogic_params_frame_type frame_type; +}; + +static struct uclogic_parse_ugee_v2_desc_case uclogic_parse_ugee_v2_desc_cases[] = { + { + .name = "invalid_str_desc", + .res = -EINVAL, + .str_desc = {}, + .str_desc_size = 0, + .desc_params = {}, + .frame_type = UCLOGIC_PARAMS_FRAME_BUTTONS, + }, + { + .name = "resolution_with_value_0", + .res = 0, + .str_desc = { + 0x0E, 0x03, + 0x70, 0xB2, + 0x10, 0x77, + 0x08, + 0x00, + 0xFF, 0x1F, + 0x00, 0x00, + }, + .str_desc_size = 12, + .desc_params = { + [UCLOGIC_RDESC_PEN_PH_ID_X_LM] = 0xB270, + [UCLOGIC_RDESC_PEN_PH_ID_X_PM] = 0, + [UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = 0x7710, + [UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = 0, + [UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = 0x1FFF, + [UCLOGIC_RDESC_FRAME_PH_ID_UM] = 0x08, + }, + .frame_type = UCLOGIC_PARAMS_FRAME_BUTTONS, + }, + /* XP-PEN Deco L str_desc: Frame with 8 buttons */ + { + .name = "frame_type_buttons", + .res = 0, + .str_desc = { + 0x0E, 0x03, + 0x70, 0xB2, + 0x10, 0x77, + 0x08, + 0x00, + 0xFF, 0x1F, + 0xD8, 0x13, + }, + .str_desc_size = 12, + .desc_params = { + [UCLOGIC_RDESC_PEN_PH_ID_X_LM] = 0xB270, + [UCLOGIC_RDESC_PEN_PH_ID_X_PM] = 0x2320, + [UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = 0x7710, + [UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = 0x1770, + [UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = 0x1FFF, + [UCLOGIC_RDESC_FRAME_PH_ID_UM] = 0x08, + }, + .frame_type = UCLOGIC_PARAMS_FRAME_BUTTONS, + }, + /* PARBLO A610 PRO str_desc: Frame with 9 buttons and dial */ + { + .name = "frame_type_dial", + .res = 0, + .str_desc = { + 0x0E, 0x03, + 0x96, 0xC7, + 0xF9, 0x7C, + 0x09, + 0x01, + 0xFF, 0x1F, + 0xD8, 0x13, + }, + .str_desc_size = 12, + .desc_params = { + [UCLOGIC_RDESC_PEN_PH_ID_X_LM] = 0xC796, + [UCLOGIC_RDESC_PEN_PH_ID_X_PM] = 0x2749, + [UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = 0x7CF9, + [UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = 0x1899, + [UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = 0x1FFF, + [UCLOGIC_RDESC_FRAME_PH_ID_UM] = 0x09, + }, + .frame_type = UCLOGIC_PARAMS_FRAME_DIAL, + }, + /* XP-PEN Deco Pro S str_desc: Frame with 8 buttons and mouse */ + { + .name = "frame_type_mouse", + .res = 0, + .str_desc = { + 0x0E, 0x03, + 0xC8, 0xB3, + 0x34, 0x65, + 0x08, + 0x02, + 0xFF, 0x1F, + 0xD8, 0x13, + }, + .str_desc_size = 12, + .desc_params = { + [UCLOGIC_RDESC_PEN_PH_ID_X_LM] = 0xB3C8, + [UCLOGIC_RDESC_PEN_PH_ID_X_PM] = 0x2363, + [UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = 0x6534, + [UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = 0x13EC, + [UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = 0x1FFF, + [UCLOGIC_RDESC_FRAME_PH_ID_UM] = 0x08, + }, + .frame_type = UCLOGIC_PARAMS_FRAME_MOUSE, + }, +}; + +static void uclogic_parse_ugee_v2_desc_case_desc(struct uclogic_parse_ugee_v2_desc_case *t, + char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(uclogic_parse_ugee_v2_desc, uclogic_parse_ugee_v2_desc_cases, + uclogic_parse_ugee_v2_desc_case_desc); + +static void uclogic_parse_ugee_v2_desc_test(struct kunit *test) +{ + int res; + s32 desc_params[UCLOGIC_RDESC_PH_ID_NUM]; + enum uclogic_params_frame_type frame_type; + const struct uclogic_parse_ugee_v2_desc_case *params = test->param_value; + + res = uclogic_params_parse_ugee_v2_desc(params->str_desc, + params->str_desc_size, + desc_params, + ARRAY_SIZE(desc_params), + &frame_type); + KUNIT_ASSERT_EQ(test, res, params->res); + + if (res) + return; + + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_LM], + desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_LM]); + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_PM], + desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_PM]); + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_LM], + desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_LM]); + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_PM], + desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_PM]); + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM], + desc_params[UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM]); + KUNIT_EXPECT_EQ(test, + params->desc_params[UCLOGIC_RDESC_FRAME_PH_ID_UM], + desc_params[UCLOGIC_RDESC_FRAME_PH_ID_UM]); + KUNIT_EXPECT_EQ(test, params->frame_type, frame_type); +} + +static struct kunit_case hid_uclogic_params_test_cases[] = { + KUNIT_CASE_PARAM(uclogic_parse_ugee_v2_desc_test, + uclogic_parse_ugee_v2_desc_gen_params), + {} +}; + +static struct kunit_suite hid_uclogic_params_test_suite = { + .name = "hid_uclogic_params_test", + .test_cases = hid_uclogic_params_test_cases, +}; + +kunit_test_suite(hid_uclogic_params_test_suite); + +MODULE_DESCRIPTION("KUnit tests for the UC-Logic driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("José Expósito "); diff --git a/drivers/hid/hid-uclogic-params.c b/drivers/hid/hid-uclogic-params.c index c11fa239e6a29281c19fe5237583fa6d35cf8bd4..34fa991e6267e3228bd78cdca94c6e97d7f7e349 100644 --- a/drivers/hid/hid-uclogic-params.c +++ b/drivers/hid/hid-uclogic-params.c @@ -1056,6 +1056,161 @@ cleanup: return rc; } +/** + * uclogic_params_parse_ugee_v2_desc - parse the string descriptor containing + * pen and frame parameters returned by UGEE v2 devices. + * + * @str_desc: String descriptor, cannot be NULL. + * @str_desc_size: Size of the string descriptor. + * @desc_params: Output description params list. + * @desc_params_size: Size of the output description params list. + * @frame_type: Output frame type. + * + * Returns: + * Zero, if successful. A negative errno code on error. + */ +static int uclogic_params_parse_ugee_v2_desc(const __u8 *str_desc, + size_t str_desc_size, + s32 *desc_params, + size_t desc_params_size, + enum uclogic_params_frame_type *frame_type) +{ + s32 pen_x_lm, pen_y_lm; + s32 pen_x_pm, pen_y_pm; + s32 pen_pressure_lm; + s32 frame_num_buttons; + s32 resolution; + + /* Minimum descriptor length required, maximum seen so far is 14 */ + const int min_str_desc_size = 12; + + if (!str_desc || str_desc_size < min_str_desc_size) + return -EINVAL; + + if (desc_params_size != UCLOGIC_RDESC_PH_ID_NUM) + return -EINVAL; + + pen_x_lm = get_unaligned_le16(str_desc + 2); + pen_y_lm = get_unaligned_le16(str_desc + 4); + frame_num_buttons = str_desc[6]; + *frame_type = str_desc[7]; + pen_pressure_lm = get_unaligned_le16(str_desc + 8); + + resolution = get_unaligned_le16(str_desc + 10); + if (resolution == 0) { + pen_x_pm = 0; + pen_y_pm = 0; + } else { + pen_x_pm = pen_x_lm * 1000 / resolution; + pen_y_pm = pen_y_lm * 1000 / resolution; + } + + desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_LM] = pen_x_lm; + desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_PM] = pen_x_pm; + desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = pen_y_lm; + desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = pen_y_pm; + desc_params[UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = pen_pressure_lm; + desc_params[UCLOGIC_RDESC_FRAME_PH_ID_UM] = frame_num_buttons; + + return 0; +} + +/** + * uclogic_params_ugee_v2_init_frame_buttons() - initialize a UGEE v2 frame with + * buttons. + * @p: Parameters to fill in, cannot be NULL. + * @desc_params: Device description params list. + * @desc_params_size: Size of the description params list. + * + * Returns: + * Zero, if successful. A negative errno code on error. + */ +static int uclogic_params_ugee_v2_init_frame_buttons(struct uclogic_params *p, + const s32 *desc_params, + size_t desc_params_size) +{ + __u8 *rdesc_frame = NULL; + int rc = 0; + + if (!p || desc_params_size != UCLOGIC_RDESC_PH_ID_NUM) + return -EINVAL; + + rdesc_frame = uclogic_rdesc_template_apply( + uclogic_rdesc_ugee_v2_frame_btn_template_arr, + uclogic_rdesc_ugee_v2_frame_btn_template_size, + desc_params, UCLOGIC_RDESC_PH_ID_NUM); + if (!rdesc_frame) + return -ENOMEM; + + rc = uclogic_params_frame_init_with_desc(&p->frame_list[0], + rdesc_frame, + uclogic_rdesc_ugee_v2_frame_btn_template_size, + UCLOGIC_RDESC_V1_FRAME_ID); + kfree(rdesc_frame); + return rc; +} + +/** + * uclogic_params_ugee_v2_init_frame_dial() - initialize a UGEE v2 frame with a + * bitmap dial. + * @p: Parameters to fill in, cannot be NULL. + * @desc_params: Device description params list. + * @desc_params_size: Size of the description params list. + * + * Returns: + * Zero, if successful. A negative errno code on error. + */ +static int uclogic_params_ugee_v2_init_frame_dial(struct uclogic_params *p, + const s32 *desc_params, + size_t desc_params_size) +{ + __u8 *rdesc_frame = NULL; + int rc = 0; + + if (!p || desc_params_size != UCLOGIC_RDESC_PH_ID_NUM) + return -EINVAL; + + rdesc_frame = uclogic_rdesc_template_apply( + uclogic_rdesc_ugee_v2_frame_dial_template_arr, + uclogic_rdesc_ugee_v2_frame_dial_template_size, + desc_params, UCLOGIC_RDESC_PH_ID_NUM); + if (!rdesc_frame) + return -ENOMEM; + + rc = uclogic_params_frame_init_with_desc(&p->frame_list[0], + rdesc_frame, + uclogic_rdesc_ugee_v2_frame_dial_template_size, + UCLOGIC_RDESC_V1_FRAME_ID); + kfree(rdesc_frame); + if (rc) + return rc; + + p->frame_list[0].bitmap_dial_byte = 7; + return 0; +} + +/** + * uclogic_params_ugee_v2_init_frame_mouse() - initialize a UGEE v2 frame with a + * mouse. + * @p: Parameters to fill in, cannot be NULL. + * + * Returns: + * Zero, if successful. A negative errno code on error. + */ +static int uclogic_params_ugee_v2_init_frame_mouse(struct uclogic_params *p) +{ + int rc = 0; + + if (!p) + return -EINVAL; + + rc = uclogic_params_frame_init_with_desc(&p->frame_list[1], + uclogic_rdesc_ugee_v2_frame_mouse_template_arr, + uclogic_rdesc_ugee_v2_frame_mouse_template_size, + UCLOGIC_RDESC_V1_FRAME_ID); + return rc; +} + /** * uclogic_params_ugee_v2_init() - initialize a UGEE graphics tablets by * discovering their parameters. @@ -1084,9 +1239,8 @@ static int uclogic_params_ugee_v2_init(struct uclogic_params *params, const int str_desc_len = 12; __u8 *str_desc = NULL; __u8 *rdesc_pen = NULL; - __u8 *rdesc_frame = NULL; s32 desc_params[UCLOGIC_RDESC_PH_ID_NUM]; - s32 resolution; + enum uclogic_params_frame_type frame_type; __u8 magic_arr[] = { 0x02, 0xb0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; @@ -1100,6 +1254,15 @@ static int uclogic_params_ugee_v2_init(struct uclogic_params *params, iface = to_usb_interface(hdev->dev.parent); bInterfaceNumber = iface->cur_altsetting->desc.bInterfaceNumber; + + if (bInterfaceNumber == 0) { + rc = uclogic_params_ugee_v2_init_frame_mouse(&p); + if (rc) + goto cleanup; + + goto output; + } + if (bInterfaceNumber != 2) { uclogic_params_init_invalid(&p); goto output; @@ -1128,25 +1291,13 @@ static int uclogic_params_ugee_v2_init(struct uclogic_params *params, goto output; } - desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_LM] = - get_unaligned_le16(str_desc + 2); - desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_LM] = - get_unaligned_le16(str_desc + 4); - desc_params[UCLOGIC_RDESC_FRAME_PH_ID_UM] = str_desc[6]; - desc_params[UCLOGIC_RDESC_PEN_PH_ID_PRESSURE_LM] = - get_unaligned_le16(str_desc + 8); - resolution = get_unaligned_le16(str_desc + 10); - if (resolution == 0) { - desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_PM] = 0; - desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = 0; - } else { - desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_PM] = - desc_params[UCLOGIC_RDESC_PEN_PH_ID_X_LM] * 1000 / - resolution; - desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_PM] = - desc_params[UCLOGIC_RDESC_PEN_PH_ID_Y_LM] * 1000 / - resolution; - } + rc = uclogic_params_parse_ugee_v2_desc(str_desc, str_desc_len, + desc_params, + ARRAY_SIZE(desc_params), + &frame_type); + if (rc) + goto cleanup; + kfree(str_desc); str_desc = NULL; @@ -1167,24 +1318,21 @@ static int uclogic_params_ugee_v2_init(struct uclogic_params *params, p.pen.subreport_list[0].id = UCLOGIC_RDESC_V1_FRAME_ID; /* Initialize the frame interface */ - rdesc_frame = uclogic_rdesc_template_apply( - uclogic_rdesc_ugee_v2_frame_btn_template_arr, - uclogic_rdesc_ugee_v2_frame_btn_template_size, - desc_params, ARRAY_SIZE(desc_params)); - if (!rdesc_frame) { - rc = -ENOMEM; - goto cleanup; + switch (frame_type) { + case UCLOGIC_PARAMS_FRAME_DIAL: + case UCLOGIC_PARAMS_FRAME_MOUSE: + rc = uclogic_params_ugee_v2_init_frame_dial(&p, desc_params, + ARRAY_SIZE(desc_params)); + break; + case UCLOGIC_PARAMS_FRAME_BUTTONS: + default: + rc = uclogic_params_ugee_v2_init_frame_buttons(&p, desc_params, + ARRAY_SIZE(desc_params)); + break; } - rc = uclogic_params_frame_init_with_desc(&p.frame_list[0], - rdesc_frame, - uclogic_rdesc_ugee_v2_frame_btn_template_size, - UCLOGIC_RDESC_V1_FRAME_ID); - kfree(rdesc_frame); - if (rc) { - uclogic_params_init_invalid(&p); - goto output; - } + if (rc) + goto cleanup; output: /* Output parameters */ @@ -1432,8 +1580,12 @@ int uclogic_params_init(struct uclogic_params *params, uclogic_params_init_invalid(&p); } break; + case VID_PID(USB_VENDOR_ID_UGEE, + USB_DEVICE_ID_UGEE_PARBLO_A610_PRO): case VID_PID(USB_VENDOR_ID_UGEE, USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_L): + case VID_PID(USB_VENDOR_ID_UGEE, + USB_DEVICE_ID_UGEE_XPPEN_TABLET_DECO_PRO_S): rc = uclogic_params_ugee_v2_init(&p, hdev); if (rc != 0) goto cleanup; @@ -1517,3 +1669,7 @@ cleanup: uclogic_params_cleanup(&p); return rc; } + +#ifdef CONFIG_HID_KUNIT_TEST +#include "hid-uclogic-params-test.c" +#endif diff --git a/drivers/hid/hid-uclogic-params.h b/drivers/hid/hid-uclogic-params.h index 5bef8daaa60741195fdff24838139266fcf2aef8..a97477c02ff82dc65b0f3ea94fb6e8b007a142be 100644 --- a/drivers/hid/hid-uclogic-params.h +++ b/drivers/hid/hid-uclogic-params.h @@ -29,6 +29,16 @@ enum uclogic_params_pen_inrange { UCLOGIC_PARAMS_PEN_INRANGE_NONE, }; +/* Types of frames */ +enum uclogic_params_frame_type { + /* Frame with buttons */ + UCLOGIC_PARAMS_FRAME_BUTTONS = 0, + /* Frame with buttons and a dial */ + UCLOGIC_PARAMS_FRAME_DIAL, + /* Frame with buttons and a mouse (shaped as a dial + touchpad) */ + UCLOGIC_PARAMS_FRAME_MOUSE, +}; + /* * Pen report's subreport data. */ diff --git a/drivers/hid/hid-uclogic-rdesc-test.c b/drivers/hid/hid-uclogic-rdesc-test.c index ebebffef5f8aecc553d33a6bb151ddec664dc500..3971a0854c3e1d4d997ea83c3e224ab3826f8739 100644 --- a/drivers/hid/hid-uclogic-rdesc-test.c +++ b/drivers/hid/hid-uclogic-rdesc-test.c @@ -97,7 +97,7 @@ static const __u8 template_params_none[] = { static struct uclogic_template_case uclogic_template_cases[] = { { - .name = "Empty template", + .name = "empty_template", .template = template_empty, .template_size = sizeof(template_empty), .param_list = params_pen_all, @@ -105,7 +105,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = template_empty, }, { - .name = "Template smaller than the placeholder", + .name = "template_smaller_than_the_placeholder", .template = template_small, .template_size = sizeof(template_small), .param_list = params_pen_all, @@ -113,7 +113,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = template_small, }, { - .name = "No placeholder", + .name = "no_placeholder", .template = template_no_ph, .template_size = sizeof(template_no_ph), .param_list = params_pen_all, @@ -121,7 +121,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = template_no_ph, }, { - .name = "Pen placeholder at the end, without ID", + .name = "pen_placeholder_at_the_end_without_id", .template = template_pen_ph_end, .template_size = sizeof(template_pen_ph_end), .param_list = params_pen_all, @@ -129,7 +129,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = template_pen_ph_end, }, { - .name = "Frame button placeholder at the end, without ID", + .name = "frame_button_placeholder_at_the_end_without_id", .template = template_btn_ph_end, .template_size = sizeof(template_btn_ph_end), .param_list = params_frame_all, @@ -137,7 +137,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = template_btn_ph_end, }, { - .name = "All params present in the pen template", + .name = "all_params_present_in_the_pen_template", .template = template_pen_all_params, .template_size = sizeof(template_pen_all_params), .param_list = params_pen_all, @@ -145,7 +145,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = expected_pen_all_params, }, { - .name = "All params present in the frame template", + .name = "all_params_present_in_the_frame_template", .template = template_frame_all_params, .template_size = sizeof(template_frame_all_params), .param_list = params_frame_all, @@ -153,7 +153,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = expected_frame_all_params, }, { - .name = "Some params present in the pen template (complete param list)", + .name = "some_params_present_in_the_pen_template_with_complete_param_list", .template = template_pen_some_params, .template_size = sizeof(template_pen_some_params), .param_list = params_pen_all, @@ -161,7 +161,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = expected_pen_some_params, }, { - .name = "Some params present in the pen template (incomplete param list)", + .name = "some_params_present_in_the_pen_template_with_incomplete_param_list", .template = template_pen_some_params, .template_size = sizeof(template_pen_some_params), .param_list = params_pen_some, @@ -169,7 +169,7 @@ static struct uclogic_template_case uclogic_template_cases[] = { .expected = expected_pen_some_params, }, { - .name = "No params present in the template", + .name = "no_params_present_in_the_template", .template = template_params_none, .template_size = sizeof(template_params_none), .param_list = params_pen_some, @@ -208,7 +208,7 @@ static struct kunit_case hid_uclogic_rdesc_test_cases[] = { }; static struct kunit_suite hid_uclogic_rdesc_test_suite = { - .name = "hid-uclogic-rdesc-test", + .name = "hid_uclogic_rdesc_test", .test_cases = hid_uclogic_rdesc_test_cases, }; diff --git a/drivers/hid/hid-uclogic-rdesc.c b/drivers/hid/hid-uclogic-rdesc.c index 3d68e8b0784d0190439a249699aa58cbeb01105d..4bd54c4fb5b0baca541d6b54e98e1052cf680a8a 100644 --- a/drivers/hid/hid-uclogic-rdesc.c +++ b/drivers/hid/hid-uclogic-rdesc.c @@ -961,6 +961,80 @@ const __u8 uclogic_rdesc_ugee_v2_frame_btn_template_arr[] = { const size_t uclogic_rdesc_ugee_v2_frame_btn_template_size = sizeof(uclogic_rdesc_ugee_v2_frame_btn_template_arr); +/* Fixed report descriptor template for UGEE v2 frame reports (dial) */ +const __u8 uclogic_rdesc_ugee_v2_frame_dial_template_arr[] = { + 0x05, 0x01, /* Usage Page (Desktop), */ + 0x09, 0x07, /* Usage (Keypad), */ + 0xA1, 0x01, /* Collection (Application), */ + 0x85, UCLOGIC_RDESC_V1_FRAME_ID, + /* Report ID, */ + 0x05, 0x0D, /* Usage Page (Digitizer), */ + 0x09, 0x39, /* Usage (Tablet Function Keys), */ + 0xA0, /* Collection (Physical), */ + 0x75, 0x01, /* Report Size (1), */ + 0x95, 0x08, /* Report Count (8), */ + 0x81, 0x01, /* Input (Constant), */ + 0x05, 0x09, /* Usage Page (Button), */ + 0x19, 0x01, /* Usage Minimum (01h), */ + UCLOGIC_RDESC_FRAME_PH_BTN, + /* Usage Maximum (PLACEHOLDER), */ + 0x95, 0x0A, /* Report Count (10), */ + 0x14, /* Logical Minimum (0), */ + 0x25, 0x01, /* Logical Maximum (1), */ + 0x81, 0x02, /* Input (Variable), */ + 0x95, 0x06, /* Report Count (6), */ + 0x81, 0x01, /* Input (Constant), */ + 0x75, 0x08, /* Report Size (8), */ + 0x95, 0x03, /* Report Count (3), */ + 0x81, 0x01, /* Input (Constant), */ + 0x05, 0x01, /* Usage Page (Desktop), */ + 0x09, 0x38, /* Usage (Wheel), */ + 0x95, 0x01, /* Report Count (1), */ + 0x15, 0xFF, /* Logical Minimum (-1), */ + 0x25, 0x01, /* Logical Maximum (1), */ + 0x81, 0x06, /* Input (Variable, Relative), */ + 0x95, 0x02, /* Report Count (2), */ + 0x81, 0x01, /* Input (Constant), */ + 0xC0, /* End Collection, */ + 0xC0 /* End Collection */ +}; +const size_t uclogic_rdesc_ugee_v2_frame_dial_template_size = + sizeof(uclogic_rdesc_ugee_v2_frame_dial_template_arr); + +/* Fixed report descriptor template for UGEE v2 frame reports (mouse) */ +const __u8 uclogic_rdesc_ugee_v2_frame_mouse_template_arr[] = { + 0x05, 0x01, /* Usage Page (Desktop), */ + 0x09, 0x02, /* Usage (Mouse), */ + 0xA1, 0x01, /* Collection (Application), */ + 0x85, 0x01, /* Report ID (1), */ + 0x05, 0x01, /* Usage Page (Pointer), */ + 0xA0, /* Collection (Physical), */ + 0x75, 0x01, /* Report Size (1), */ + 0x95, 0x02, /* Report Count (2), */ + 0x05, 0x09, /* Usage Page (Button), */ + 0x19, 0x01, /* Usage Minimum (01h), */ + 0x29, 0x02, /* Usage Maximum (02h), */ + 0x14, /* Logical Minimum (0), */ + 0x25, 0x01, /* Logical Maximum (1), */ + 0x81, 0x02, /* Input (Variable), */ + 0x95, 0x06, /* Report Count (6), */ + 0x81, 0x01, /* Input (Constant), */ + 0x05, 0x01, /* Usage Page (Generic Desktop), */ + 0x09, 0x30, /* Usage (X), */ + 0x09, 0x31, /* Usage (Y), */ + 0x75, 0x10, /* Report Size (16), */ + 0x95, 0x02, /* Report Count (2), */ + 0x16, 0x00, 0x80, /* Logical Minimum (-32768), */ + 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ + 0x81, 0x06, /* Input (Variable, Relative), */ + 0x95, 0x01, /* Report Count (1), */ + 0x81, 0x01, /* Input (Constant), */ + 0xC0, /* End Collection, */ + 0xC0 /* End Collection */ +}; +const size_t uclogic_rdesc_ugee_v2_frame_mouse_template_size = + sizeof(uclogic_rdesc_ugee_v2_frame_mouse_template_arr); + /* Fixed report descriptor for Ugee EX07 frame */ const __u8 uclogic_rdesc_ugee_ex07_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ @@ -1113,7 +1187,7 @@ __u8 *uclogic_rdesc_template_apply(const __u8 *template_ptr, memcmp(p, pen_head, sizeof(pen_head)) == 0 && p[sizeof(pen_head)] < param_num) { v = param_list[p[sizeof(pen_head)]]; - put_unaligned(cpu_to_le32(v), (s32 *)p); + put_unaligned((__force u32)cpu_to_le32(v), (s32 *)p); p += sizeof(pen_head) + 1; } else if (memcmp(p, btn_head, sizeof(btn_head)) == 0 && p[sizeof(btn_head)] < param_num) { diff --git a/drivers/hid/hid-uclogic-rdesc.h b/drivers/hid/hid-uclogic-rdesc.h index 86e64a9ee6bd70235c748f82b016c5afc94e3d48..0502a06564964772b2f4aed6018b0c937513f94f 100644 --- a/drivers/hid/hid-uclogic-rdesc.h +++ b/drivers/hid/hid-uclogic-rdesc.h @@ -169,6 +169,14 @@ extern const size_t uclogic_rdesc_ugee_v2_pen_template_size; extern const __u8 uclogic_rdesc_ugee_v2_frame_btn_template_arr[]; extern const size_t uclogic_rdesc_ugee_v2_frame_btn_template_size; +/* Fixed report descriptor template for UGEE v2 frame reports (dial) */ +extern const __u8 uclogic_rdesc_ugee_v2_frame_dial_template_arr[]; +extern const size_t uclogic_rdesc_ugee_v2_frame_dial_template_size; + +/* Fixed report descriptor template for UGEE v2 frame reports (mouse) */ +extern const __u8 uclogic_rdesc_ugee_v2_frame_mouse_template_arr[]; +extern const size_t uclogic_rdesc_ugee_v2_frame_mouse_template_size; + /* Fixed report descriptor for Ugee EX07 frame */ extern const __u8 uclogic_rdesc_ugee_ex07_frame_arr[]; extern const size_t uclogic_rdesc_ugee_ex07_frame_size; diff --git a/drivers/hid/hid-vivaldi-common.c b/drivers/hid/hid-vivaldi-common.c index 8b3e515d0f06d7ee32627ffd1e8472504d28c6cf..b0af2be948952c645222f04cbec2d70d7a326032 100644 --- a/drivers/hid/hid-vivaldi-common.c +++ b/drivers/hid/hid-vivaldi-common.c @@ -116,25 +116,26 @@ static struct attribute *vivaldi_sysfs_attrs[] = { NULL }; -static const struct attribute_group vivaldi_attribute_group = { - .attrs = vivaldi_sysfs_attrs, -}; - -/** - * vivaldi_input_configured - Complete initialization of device using vivaldi map - * @hdev: HID device to which vivaldi attributes should be attached - * @hidinput: HID input device (unused) - */ -int vivaldi_input_configured(struct hid_device *hdev, - struct hid_input *hidinput) +static umode_t vivaldi_is_visible(struct kobject *kobj, struct attribute *attr, + int n) { + struct hid_device *hdev = to_hid_device(kobj_to_dev(kobj)); struct vivaldi_data *data = hid_get_drvdata(hdev); if (!data->num_function_row_keys) return 0; - - return devm_device_add_group(&hdev->dev, &vivaldi_attribute_group); + return attr->mode; } -EXPORT_SYMBOL_GPL(vivaldi_input_configured); + +static const struct attribute_group vivaldi_attribute_group = { + .attrs = vivaldi_sysfs_attrs, + .is_visible = vivaldi_is_visible, +}; + +const struct attribute_group *vivaldi_attribute_groups[] = { + &vivaldi_attribute_group, + NULL, +}; +EXPORT_SYMBOL_GPL(vivaldi_attribute_groups); MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-vivaldi-common.h b/drivers/hid/hid-vivaldi-common.h index d42e82d77825b7db46b343bae58f0d5fae141f83..ba9adfa08a2d787dba5989c1f2d0f936ba5783ca 100644 --- a/drivers/hid/hid-vivaldi-common.h +++ b/drivers/hid/hid-vivaldi-common.h @@ -4,13 +4,11 @@ struct hid_device; struct hid_field; -struct hid_input; struct hid_usage; void vivaldi_feature_mapping(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage); -int vivaldi_input_configured(struct hid_device *hdev, - struct hid_input *hidinput); +extern const struct attribute_group *vivaldi_attribute_groups[]; #endif /* _HID_VIVALDI_COMMON_H */ diff --git a/drivers/hid/hid-vivaldi.c b/drivers/hid/hid-vivaldi.c index 3a979123e7d33f072be079425bbcfc9a619d92a2..cda5938fb07060be3ba5d38f43710886382dcdd8 100644 --- a/drivers/hid/hid-vivaldi.c +++ b/drivers/hid/hid-vivaldi.c @@ -45,7 +45,9 @@ static struct hid_driver hid_vivaldi = { .id_table = vivaldi_table, .probe = vivaldi_probe, .feature_mapping = vivaldi_feature_mapping, - .input_configured = vivaldi_input_configured, + .driver = { + .dev_groups = vivaldi_attribute_groups, + }, }; module_hid_driver(hid_vivaldi); diff --git a/drivers/hid/hid-vrc2.c b/drivers/hid/hid-vrc2.c new file mode 100644 index 0000000000000000000000000000000000000000..80a2b7ef5e66a00752d480577938549c5fff90ac --- /dev/null +++ b/drivers/hid/hid-vrc2.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * HID driver for VRC-2 2-axis Car controller + * + * Copyright (C) 2022 Marcus Folkesson + */ + +#include +#include +#include + +/* + * VID/PID are probably "borrowed", so keep them locally and + * do not populate hid-ids.h with those. + */ +#define USB_VENDOR_ID_VRC2 (0x07c0) +#define USB_DEVICE_ID_VRC2 (0x1125) + +static __u8 vrc2_rdesc_fixed[] = { + 0x05, 0x01, // Usage Page (Generic Desktop Ctrls) + 0x09, 0x04, // Usage (Joystick) + 0xA1, 0x01, // Collection (Application) + 0x09, 0x01, // Usage (Pointer) + 0xA1, 0x00, // Collection (Physical) + 0x09, 0x30, // Usage (X) + 0x09, 0x31, // Usage (Y) + 0x15, 0x00, // Logical Minimum (0) + 0x26, 0xFF, 0x07, // Logical Maximum (2047) + 0x35, 0x00, // Physical Minimum (0) + 0x46, 0xFF, 0x00, // Physical Maximum (255) + 0x75, 0x10, // Report Size (16) + 0x95, 0x02, // Report Count (2) + 0x81, 0x02, // Input (Data,Var,Abs,No Wrap,Linear,Preferred State,No Null Position) + 0xC0, // End Collection + 0x75, 0x08, // Report Size (8) + 0x95, 0x03, // Report Count (3) + 0x81, 0x03, // Input (Cnst,Var,Abs) + 0xC0, // End Collection +}; + +static __u8 *vrc2_report_fixup(struct hid_device *hdev, __u8 *rdesc, + unsigned int *rsize) +{ + hid_info(hdev, "fixing up VRC-2 report descriptor\n"); + *rsize = sizeof(vrc2_rdesc_fixed); + return vrc2_rdesc_fixed; +} + +static int vrc2_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + int ret; + + /* + * The device gives us 2 separate USB endpoints. + * One of those (the one with report descriptor size of 23) is just bogus so ignore it + */ + if (hdev->dev_rsize == 23) + return -ENODEV; + + ret = hid_parse(hdev); + if (ret) { + hid_err(hdev, "parse failed\n"); + return ret; + } + + ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT); + if (ret) { + hid_err(hdev, "hw start failed\n"); + return ret; + } + + return 0; +} + +static const struct hid_device_id vrc2_devices[] = { + { HID_USB_DEVICE(USB_VENDOR_ID_VRC2, USB_DEVICE_ID_VRC2) }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(hid, vrc2_devices); + +static struct hid_driver vrc2_driver = { + .name = "vrc2", + .id_table = vrc2_devices, + .report_fixup = vrc2_report_fixup, + .probe = vrc2_probe, +}; +module_hid_driver(vrc2_driver); + +MODULE_AUTHOR("Marcus Folkesson "); +MODULE_DESCRIPTION("HID driver for VRC-2 2-axis Car controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hidraw.c b/drivers/hid/hidraw.c index 681614a8302a50db51e0a821961b21430142d0c0..197b1e7bf029e64f4a1874f80ae4b24d2553621c 100644 --- a/drivers/hid/hidraw.c +++ b/drivers/hid/hidraw.c @@ -350,6 +350,8 @@ static int hidraw_release(struct inode * inode, struct file * file) down_write(&minors_rwsem); spin_lock_irqsave(&hidraw_table[minor]->list_lock, flags); + for (int i = list->tail; i < list->head; i++) + kfree(list->buffer[i].value); list_del(&list->node); spin_unlock_irqrestore(&hidraw_table[minor]->list_lock, flags); kfree(list); diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index c078f09a2318a04f528889bec961ef0a94739b20..0667b6022c3b768747862934a8e3631f114907fc 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -1036,7 +1036,7 @@ int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops, snprintf(hid->name, sizeof(hid->name), "%s %04X:%04X", client->name, (u16)hid->vendor, (u16)hid->product); - strlcpy(hid->phys, dev_name(&client->dev), sizeof(hid->phys)); + strscpy(hid->phys, dev_name(&client->dev), sizeof(hid->phys)); ihid->quirks = i2c_hid_lookup_quirk(hid->vendor, hid->product); @@ -1064,7 +1064,7 @@ err_powered: } EXPORT_SYMBOL_GPL(i2c_hid_core_probe); -int i2c_hid_core_remove(struct i2c_client *client) +void i2c_hid_core_remove(struct i2c_client *client) { struct i2c_hid *ihid = i2c_get_clientdata(client); struct hid_device *hid; @@ -1078,8 +1078,6 @@ int i2c_hid_core_remove(struct i2c_client *client) i2c_hid_free_buffers(ihid); i2c_hid_core_power_down(ihid); - - return 0; } EXPORT_SYMBOL_GPL(i2c_hid_core_remove); diff --git a/drivers/hid/i2c-hid/i2c-hid.h b/drivers/hid/i2c-hid/i2c-hid.h index 236cc062d5ef805c3da43455b951861d667d206c..96c75510ad3f14daa7697586bf0585d527017192 100644 --- a/drivers/hid/i2c-hid/i2c-hid.h +++ b/drivers/hid/i2c-hid/i2c-hid.h @@ -33,7 +33,7 @@ struct i2chid_ops { int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops, u16 hid_descriptor_address, u32 quirks); -int i2c_hid_core_remove(struct i2c_client *client); +void i2c_hid_core_remove(struct i2c_client *client); void i2c_hid_core_shutdown(struct i2c_client *client); diff --git a/drivers/hid/intel-ish-hid/ipc/hw-ish.h b/drivers/hid/intel-ish-hid/ipc/hw-ish.h index e600dbf04dfc675568f75a31587eb5ad789f8dfa..fc108f19a64c3e15ab1c48338d820cf7fd470df5 100644 --- a/drivers/hid/intel-ish-hid/ipc/hw-ish.h +++ b/drivers/hid/intel-ish-hid/ipc/hw-ish.h @@ -32,6 +32,7 @@ #define ADL_P_DEVICE_ID 0x51FC #define ADL_N_DEVICE_ID 0x54FC #define RPL_S_DEVICE_ID 0x7A78 +#define MTL_P_DEVICE_ID 0x7E45 #define REVISION_ID_CHT_A0 0x6 #define REVISION_ID_CHT_Ax_SI 0x0 diff --git a/drivers/hid/intel-ish-hid/ipc/pci-ish.c b/drivers/hid/intel-ish-hid/ipc/pci-ish.c index 2c67ec17bec6fd935b40b5540d0994c2f028c8c9..7120b30ac51d063769759bbbf6ad5e1565557d2e 100644 --- a/drivers/hid/intel-ish-hid/ipc/pci-ish.c +++ b/drivers/hid/intel-ish-hid/ipc/pci-ish.c @@ -43,6 +43,7 @@ static const struct pci_device_id ish_pci_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_INTEL, ADL_P_DEVICE_ID)}, {PCI_DEVICE(PCI_VENDOR_ID_INTEL, ADL_N_DEVICE_ID)}, {PCI_DEVICE(PCI_VENDOR_ID_INTEL, RPL_S_DEVICE_ID)}, + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, MTL_P_DEVICE_ID)}, {0, } }; MODULE_DEVICE_TABLE(pci, ish_pci_tbl); diff --git a/drivers/hid/intel-ish-hid/ishtp-hid.h b/drivers/hid/intel-ish-hid/ishtp-hid.h index 6a5cc11aefd8944a705570755b3ce0fd2e668744..35dddc5015b378028ecd57b20c80aa98c2758e0f 100644 --- a/drivers/hid/intel-ish-hid/ishtp-hid.h +++ b/drivers/hid/intel-ish-hid/ishtp-hid.h @@ -105,7 +105,7 @@ struct report_list { * @multi_packet_cnt: Count of fragmented packet count * * This structure is used to store completion flags and per client data like - * like report description, number of HID devices etc. + * report description, number of HID devices etc. */ struct ishtp_cl_data { /* completion flags */ diff --git a/drivers/hid/intel-ish-hid/ishtp/client.c b/drivers/hid/intel-ish-hid/ishtp/client.c index 405e0d5212cc8d35f1a0c9ba098e408fe9a54a13..df0a825694f52f3fd56020a2ead1de5c453dfa14 100644 --- a/drivers/hid/intel-ish-hid/ishtp/client.c +++ b/drivers/hid/intel-ish-hid/ishtp/client.c @@ -626,13 +626,14 @@ static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb) } /** - * ipc_tx_callback() - IPC tx callback function + * ipc_tx_send() - IPC tx send function * @prm: Pointer to client device instance * - * Send message over IPC either first time or on callback on previous message - * completion + * Send message over IPC. Message will be split into fragments + * if message size is bigger than IPC FIFO size, and all + * fragments will be sent one by one. */ -static void ipc_tx_callback(void *prm) +static void ipc_tx_send(void *prm) { struct ishtp_cl *cl = prm; struct ishtp_cl_tx_ring *cl_msg; @@ -677,32 +678,41 @@ static void ipc_tx_callback(void *prm) list); rem = cl_msg->send_buf.size - cl->tx_offs; - ishtp_hdr.host_addr = cl->host_client_id; - ishtp_hdr.fw_addr = cl->fw_client_id; - ishtp_hdr.reserved = 0; - pmsg = cl_msg->send_buf.data + cl->tx_offs; + while (rem > 0) { + ishtp_hdr.host_addr = cl->host_client_id; + ishtp_hdr.fw_addr = cl->fw_client_id; + ishtp_hdr.reserved = 0; + pmsg = cl_msg->send_buf.data + cl->tx_offs; + + if (rem <= dev->mtu) { + /* Last fragment or only one packet */ + ishtp_hdr.length = rem; + ishtp_hdr.msg_complete = 1; + /* Submit to IPC queue with no callback */ + ishtp_write_message(dev, &ishtp_hdr, pmsg); + cl->tx_offs = 0; + cl->sending = 0; - if (rem <= dev->mtu) { - ishtp_hdr.length = rem; - ishtp_hdr.msg_complete = 1; - cl->sending = 0; - list_del_init(&cl_msg->list); /* Must be before write */ - spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); - /* Submit to IPC queue with no callback */ - ishtp_write_message(dev, &ishtp_hdr, pmsg); - spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); - list_add_tail(&cl_msg->list, &cl->tx_free_list.list); - ++cl->tx_ring_free_size; - spin_unlock_irqrestore(&cl->tx_free_list_spinlock, - tx_free_flags); - } else { - /* Send IPC fragment */ - spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); - cl->tx_offs += dev->mtu; - ishtp_hdr.length = dev->mtu; - ishtp_hdr.msg_complete = 0; - ishtp_send_msg(dev, &ishtp_hdr, pmsg, ipc_tx_callback, cl); + break; + } else { + /* Send ipc fragment */ + ishtp_hdr.length = dev->mtu; + ishtp_hdr.msg_complete = 0; + /* All fregments submitted to IPC queue with no callback */ + ishtp_write_message(dev, &ishtp_hdr, pmsg); + cl->tx_offs += dev->mtu; + rem = cl_msg->send_buf.size - cl->tx_offs; + } } + + list_del_init(&cl_msg->list); + spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags); + + spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags); + list_add_tail(&cl_msg->list, &cl->tx_free_list.list); + ++cl->tx_ring_free_size; + spin_unlock_irqrestore(&cl->tx_free_list_spinlock, + tx_free_flags); } /** @@ -720,7 +730,7 @@ static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev, return; cl->tx_offs = 0; - ipc_tx_callback(cl); + ipc_tx_send(cl); ++cl->send_msg_cnt_ipc; } diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c index 4490e2f7252ac7c55023c2d11a0384e5924e0128..be4c731aaa65daf68a7431f0165838c21ce0e9f7 100644 --- a/drivers/hid/usbhid/hid-core.c +++ b/drivers/hid/usbhid/hid-core.c @@ -1381,7 +1381,7 @@ static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id * hid->type = HID_TYPE_USBNONE; if (dev->manufacturer) - strlcpy(hid->name, dev->manufacturer, sizeof(hid->name)); + strscpy(hid->name, dev->manufacturer, sizeof(hid->name)); if (dev->product) { if (dev->manufacturer) diff --git a/drivers/hid/usbhid/usbkbd.c b/drivers/hid/usbhid/usbkbd.c index b4b007c4beb6eb381262b24d8554f6e59decd108..c439ed2f16dbca3bbd229c2a631b41b8af7415fd 100644 --- a/drivers/hid/usbhid/usbkbd.c +++ b/drivers/hid/usbhid/usbkbd.c @@ -294,7 +294,7 @@ static int usb_kbd_probe(struct usb_interface *iface, spin_lock_init(&kbd->leds_lock); if (dev->manufacturer) - strlcpy(kbd->name, dev->manufacturer, sizeof(kbd->name)); + strscpy(kbd->name, dev->manufacturer, sizeof(kbd->name)); if (dev->product) { if (dev->manufacturer) diff --git a/drivers/hid/usbhid/usbmouse.c b/drivers/hid/usbhid/usbmouse.c index fb1d7d1f6999465babfb2227663af968209d1a90..3fd93c2e4f4ad064106764264b7857e1f8f519dc 100644 --- a/drivers/hid/usbhid/usbmouse.c +++ b/drivers/hid/usbhid/usbmouse.c @@ -142,7 +142,7 @@ static int usb_mouse_probe(struct usb_interface *intf, const struct usb_device_i mouse->dev = input_dev; if (dev->manufacturer) - strlcpy(mouse->name, dev->manufacturer, sizeof(mouse->name)); + strscpy(mouse->name, dev->manufacturer, sizeof(mouse->name)); if (dev->product) { if (dev->manufacturer) diff --git a/drivers/hid/wacom.h b/drivers/hid/wacom.h index 3f8b24a57014b954b0c841b7cf19b05c341e6d08..4da50e19808ef439ba6452c8dcfce3821c54e8a2 100644 --- a/drivers/hid/wacom.h +++ b/drivers/hid/wacom.h @@ -1,7 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * drivers/input/tablet/wacom.h - * * USB Wacom tablet support * * Copyright (c) 2000-2004 Vojtech Pavlik @@ -78,10 +76,9 @@ * - integration of the Bluetooth devices */ -/* - */ #ifndef WACOM_H #define WACOM_H + #include #include #include diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 194a2e32759148da8a2afeffb6683365f80e44d0..634263e4556b03b0eeed3ec7df2cd1c9866de58f 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -1,13 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * drivers/input/tablet/wacom_sys.c - * * USB Wacom tablet support - system specific code */ -/* - */ - #include "wacom_wac.h" #include "wacom.h" #include @@ -2226,7 +2221,7 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix) } else if (strstr(product_name, "Wacom") || strstr(product_name, "wacom") || strstr(product_name, "WACOM")) { - strlcpy(name, product_name, sizeof(name)); + strscpy(name, product_name, sizeof(name)); } else { snprintf(name, sizeof(name), "Wacom %s", product_name); } @@ -2244,7 +2239,7 @@ static void wacom_update_name(struct wacom *wacom, const char *suffix) if (name[strlen(name)-1] == ' ') name[strlen(name)-1] = '\0'; } else { - strlcpy(name, features->name, sizeof(name)); + strscpy(name, features->name, sizeof(name)); } snprintf(wacom_wac->name, sizeof(wacom_wac->name), "%s%s", @@ -2509,7 +2504,7 @@ static void wacom_wireless_work(struct work_struct *work) goto fail; } - strlcpy(wacom_wac->name, wacom_wac1->name, + strscpy(wacom_wac->name, wacom_wac1->name, sizeof(wacom_wac->name)); error = wacom_initialize_battery(wacom); if (error) diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index d049239256a268b467c6fe565f3a516d2d609ab1..77486962a773f5a197d151d62ed5e387f1890570 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -1,13 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * drivers/input/tablet/wacom_wac.c - * * USB Wacom tablet support - Wacom specific code */ -/* - */ - #include "wacom_wac.h" #include "wacom.h" #include @@ -713,11 +708,14 @@ static int wacom_intuos_get_tool_type(int tool_id) case 0x802: /* Intuos4/5 13HD/24HD General Pen */ case 0x8e2: /* IntuosHT2 pen */ case 0x022: + case 0x200: /* Pro Pen 3 */ + case 0x04200: /* Pro Pen 3 */ case 0x10842: /* MobileStudio Pro Pro Pen slim */ case 0x14802: /* Intuos4/5 13HD/24HD Classic Pen */ case 0x16802: /* Cintiq 13HD Pro Pen */ case 0x18802: /* DTH2242 Pen */ case 0x10802: /* Intuos4/5 13HD/24HD General Pen */ + case 0x80842: /* Intuos Pro and Cintiq Pro 3D Pen */ tool_type = BTN_TOOL_PEN; break; @@ -4875,6 +4873,10 @@ static const struct wacom_features wacom_features_0x3c6 = static const struct wacom_features wacom_features_0x3c8 = { "Wacom Intuos BT M", 21600, 13500, 4095, 63, INTUOSHT3_BT, WACOM_INTUOS_RES, WACOM_INTUOS_RES, 4 }; +static const struct wacom_features wacom_features_0x3dd = + { "Wacom Intuos Pro S", 31920, 19950, 8191, 63, + INTUOSP2S_BT, WACOM_INTUOS3_RES, WACOM_INTUOS3_RES, 7, + .touch_max = 10 }; static const struct wacom_features wacom_features_HID_ANY_ID = { "Wacom HID", .type = HID_GENERIC, .oVid = HID_ANY_ID, .oPid = HID_ANY_ID }; @@ -5050,6 +5052,7 @@ const struct hid_device_id wacom_ids[] = { { BT_DEVICE_WACOM(0x393) }, { BT_DEVICE_WACOM(0x3c6) }, { BT_DEVICE_WACOM(0x3c8) }, + { BT_DEVICE_WACOM(0x3dd) }, { USB_DEVICE_WACOM(0x4001) }, { USB_DEVICE_WACOM(0x4004) }, { USB_DEVICE_WACOM(0x5000) }, diff --git a/drivers/hid/wacom_wac.h b/drivers/hid/wacom_wac.h index fef1538005b5d582605c85957bca1ad9d40d7430..5ca6c06d143be4921873995c00b5665af539f0bc 100644 --- a/drivers/hid/wacom_wac.h +++ b/drivers/hid/wacom_wac.h @@ -1,7 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * drivers/input/tablet/wacom_wac.h - */ + #ifndef WACOM_WAC_H #define WACOM_WAC_H diff --git a/drivers/hsi/clients/cmt_speech.c b/drivers/hsi/clients/cmt_speech.c index e014ef36d872d3ae6eb2ace2fafcd7210c0fa65e..8069f795c86493ad0f5e1482ad0514b758309218 100644 --- a/drivers/hsi/clients/cmt_speech.c +++ b/drivers/hsi/clients/cmt_speech.c @@ -1089,7 +1089,7 @@ static vm_fault_t cs_char_vma_fault(struct vm_fault *vmf) struct cs_char *csdata = vmf->vma->vm_private_data; struct page *page; - page = virt_to_page(csdata->mmap_base); + page = virt_to_page((void *)csdata->mmap_base); get_page(page); vmf->page = page; diff --git a/drivers/hsi/clients/nokia-modem.c b/drivers/hsi/clients/nokia-modem.c index cd7ebf4c2e2fcefdddba7208673b7ad0f545fa7a..97ba59e606638b95200474e89f59f5cb9bc6341f 100644 --- a/drivers/hsi/clients/nokia-modem.c +++ b/drivers/hsi/clients/nokia-modem.c @@ -13,7 +13,6 @@ #include #include #include -#include #include static unsigned int pm = 1; @@ -75,8 +74,7 @@ static int nokia_modem_gpio_probe(struct device *dev) struct nokia_modem_device *modem = dev_get_drvdata(dev); int gpio_count, gpio_name_count, i, err; - gpio_count = of_gpio_count(np); - + gpio_count = gpiod_count(dev, NULL); if (gpio_count < 0) { dev_err(dev, "missing gpios: %d\n", gpio_count); return gpio_count; diff --git a/drivers/hsi/clients/ssi_protocol.c b/drivers/hsi/clients/ssi_protocol.c index 21f11a5b965b11b97fcebbb384e42a9092927bbd..274ad8443f8c282dea6af460cfa71af763dea766 100644 --- a/drivers/hsi/clients/ssi_protocol.c +++ b/drivers/hsi/clients/ssi_protocol.c @@ -796,7 +796,6 @@ static void ssip_rx_strans(struct hsi_client *cl, u32 cmd) dev_err(&cl->device, "No memory for rx skb\n"); goto out1; } - skb->dev = ssi->netdev; skb_put(skb, len * 4); msg = ssip_alloc_data(ssi, skb, GFP_ATOMIC); if (unlikely(!msg)) { @@ -931,6 +930,7 @@ static int ssip_pn_open(struct net_device *dev) if (err < 0) { dev_err(&cl->device, "Register HSI port event failed (%d)\n", err); + hsi_release_port(cl); return err; } dev_dbg(&cl->device, "Configuring SSI port\n"); diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c index 44a3f5660c1090e3eb64c21a8acdbceb641fecb8..eb982015831858b7725f85c26bce24f9c2f9c98f 100644 --- a/drivers/hsi/controllers/omap_ssi_core.c +++ b/drivers/hsi/controllers/omap_ssi_core.c @@ -524,6 +524,7 @@ static int ssi_probe(struct platform_device *pd) if (!childpdev) { err = -ENODEV; dev_err(&pd->dev, "failed to create ssi controller port\n"); + of_node_put(child); goto out3; } } diff --git a/drivers/hsi/controllers/omap_ssi_port.c b/drivers/hsi/controllers/omap_ssi_port.c index a0cb5be246e1cf2ca47f53c5bea5859128474768..b9495b720f1bd121970deaf4f151a517c27b33d4 100644 --- a/drivers/hsi/controllers/omap_ssi_port.c +++ b/drivers/hsi/controllers/omap_ssi_port.c @@ -230,10 +230,10 @@ static int ssi_start_dma(struct hsi_msg *msg, int lch) if (msg->ttype == HSI_MSG_READ) { err = dma_map_sg(&ssi->device, msg->sgt.sgl, msg->sgt.nents, DMA_FROM_DEVICE); - if (err < 0) { + if (!err) { dev_dbg(&ssi->device, "DMA map SG failed !\n"); pm_runtime_put_autosuspend(omap_port->pdev); - return err; + return -EIO; } csdp = SSI_DST_BURST_4x32_BIT | SSI_DST_MEMORY_PORT | SSI_SRC_SINGLE_ACCESS0 | SSI_SRC_PERIPHERAL_PORT | @@ -247,10 +247,10 @@ static int ssi_start_dma(struct hsi_msg *msg, int lch) } else { err = dma_map_sg(&ssi->device, msg->sgt.sgl, msg->sgt.nents, DMA_TO_DEVICE); - if (err < 0) { + if (!err) { dev_dbg(&ssi->device, "DMA map SG failed !\n"); pm_runtime_put_autosuspend(omap_port->pdev); - return err; + return -EIO; } csdp = SSI_SRC_BURST_4x32_BIT | SSI_SRC_MEMORY_PORT | SSI_DST_SINGLE_ACCESS0 | SSI_DST_PERIPHERAL_PORT | diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index eca7afd366d6de0948636db918058073032c10df..9dc27e5d367a20c9bce2cb4f6253913724bf02e2 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -431,34 +431,29 @@ struct vmbus_channel *relid2channel(u32 relid) void vmbus_on_event(unsigned long data) { struct vmbus_channel *channel = (void *) data; - unsigned long time_limit = jiffies + 2; + void (*callback_fn)(void *context); trace_vmbus_on_event(channel); hv_debug_delay_test(channel, INTERRUPT_DELAY); - do { - void (*callback_fn)(void *); - /* A channel once created is persistent even when - * there is no driver handling the device. An - * unloading driver sets the onchannel_callback to NULL. - */ - callback_fn = READ_ONCE(channel->onchannel_callback); - if (unlikely(callback_fn == NULL)) - return; - - (*callback_fn)(channel->channel_callback_context); + /* A channel once created is persistent even when + * there is no driver handling the device. An + * unloading driver sets the onchannel_callback to NULL. + */ + callback_fn = READ_ONCE(channel->onchannel_callback); + if (unlikely(!callback_fn)) + return; - if (channel->callback_mode != HV_CALL_BATCHED) - return; + (*callback_fn)(channel->channel_callback_context); - if (likely(hv_end_read(&channel->inbound) == 0)) - return; + if (channel->callback_mode != HV_CALL_BATCHED) + return; - hv_begin_read(&channel->inbound); - } while (likely(time_before(jiffies, time_limit))); + if (likely(hv_end_read(&channel->inbound) == 0)) + return; - /* The time limit (2 jiffies) has been reached */ + hv_begin_read(&channel->inbound); tasklet_schedule(&channel->callback_event); } diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index 660036da744953ef93ccd204df470b04e64daaee..922d83eb7ddfacf2a43132a438c57af494f0d1f0 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -129,7 +129,7 @@ static void fcopy_send_data(struct work_struct *dummy) /* * The strings sent from the host are encoded in - * in utf16; convert it to utf8 strings. + * utf16; convert it to utf8 strings. * The host assures us that the utf16 strings will not exceed * the max lengths specified. We will however, reserve room * for the string terminating character - in the utf16s_utf8s() diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 23c680d1a0f5415735f2ba9d99ec47500a7fbe44..8b2e413bf19cc1ef9f55c9cd9d2bd9213ddc2a0e 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include "hyperv_vmbus.h" @@ -45,8 +46,6 @@ struct vmbus_dynid { static struct acpi_device *hv_acpi_dev; -static struct completion probe_event; - static int hyperv_cpuhp_online; static void *hv_panic_page; @@ -1131,7 +1130,8 @@ void vmbus_on_msg_dpc(unsigned long data) return; INIT_WORK(&ctx->work, vmbus_onmessage_work); - memcpy(&ctx->msg, &msg_copy, sizeof(msg->header) + payload_size); + ctx->msg.header = msg_copy.header; + memcpy(&ctx->msg.payload, msg_copy.u.payload, payload_size); /* * The host can generate a rescind message while we @@ -1572,7 +1572,7 @@ err_setup: } /** - * __vmbus_child_driver_register() - Register a vmbus's driver + * __vmbus_driver_register() - Register a vmbus's driver * @hv_driver: Pointer to driver structure you want to register * @owner: owner module of the drv * @mod_name: module name string @@ -2051,7 +2051,7 @@ struct hv_device *vmbus_device_create(const guid_t *type, child_device_obj->channel = channel; guid_copy(&child_device_obj->dev_type, type); guid_copy(&child_device_obj->dev_instance, instance); - child_device_obj->vendor_id = 0x1414; /* MSFT vendor ID */ + child_device_obj->vendor_id = PCI_VENDOR_ID_MICROSOFT; return child_device_obj; } @@ -2262,26 +2262,43 @@ static int vmbus_acpi_remove(struct acpi_device *device) static void vmbus_reserve_fb(void) { - int size; + resource_size_t start = 0, size; + struct pci_dev *pdev; + + if (efi_enabled(EFI_BOOT)) { + /* Gen2 VM: get FB base from EFI framebuffer */ + start = screen_info.lfb_base; + size = max_t(__u32, screen_info.lfb_size, 0x800000); + } else { + /* Gen1 VM: get FB base from PCI */ + pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT, + PCI_DEVICE_ID_HYPERV_VIDEO, NULL); + if (!pdev) + return; + + if (pdev->resource[0].flags & IORESOURCE_MEM) { + start = pci_resource_start(pdev, 0); + size = pci_resource_len(pdev, 0); + } + + /* + * Release the PCI device so hyperv_drm or hyperv_fb driver can + * grab it later. + */ + pci_dev_put(pdev); + } + + if (!start) + return; + /* * Make a claim for the frame buffer in the resource tree under the * first node, which will be the one below 4GB. The length seems to * be underreported, particularly in a Generation 1 VM. So start out * reserving a larger area and make it smaller until it succeeds. */ - - if (screen_info.lfb_base) { - if (efi_enabled(EFI_BOOT)) - size = max_t(__u32, screen_info.lfb_size, 0x800000); - else - size = max_t(__u32, screen_info.lfb_size, 0x4000000); - - for (; !fb_mmio && (size >= 0x100000); size >>= 1) { - fb_mmio = __request_region(hyperv_mmio, - screen_info.lfb_base, size, - fb_mmio_name, 0); - } - } + for (; !fb_mmio && (size >= 0x100000); size >>= 1) + fb_mmio = __request_region(hyperv_mmio, start, size, fb_mmio_name, 0); } /** @@ -2313,7 +2330,7 @@ int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, bool fb_overlap_ok) { struct resource *iter, *shadow; - resource_size_t range_min, range_max, start; + resource_size_t range_min, range_max, start, end; const char *dev_n = dev_name(&device_obj->device); int retval; @@ -2348,6 +2365,14 @@ int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, range_max = iter->end; start = (range_min + align - 1) & ~(align - 1); for (; start + size - 1 <= range_max; start += align) { + end = start + size - 1; + + /* Skip the whole fb_mmio region if not fb_overlap_ok */ + if (!fb_overlap_ok && fb_mmio && + (((start >= fb_mmio->start) && (start <= fb_mmio->end)) || + ((end >= fb_mmio->start) && (end <= fb_mmio->end)))) + continue; + shadow = __request_region(iter, start, size, NULL, IORESOURCE_BUSY); if (!shadow) @@ -2427,7 +2452,8 @@ static int vmbus_acpi_add(struct acpi_device *device) * Some ancestor of the vmbus acpi device (Gen1 or Gen2 * firmware) is the VMOD that has the mmio ranges. Get that. */ - for (ancestor = device->parent; ancestor; ancestor = ancestor->parent) { + for (ancestor = acpi_dev_parent(device); ancestor; + ancestor = acpi_dev_parent(ancestor)) { result = acpi_walk_resources(ancestor->handle, METHOD_NAME__CRS, vmbus_walk_resources, NULL); @@ -2441,7 +2467,6 @@ static int vmbus_acpi_add(struct acpi_device *device) ret_val = 0; acpi_walk_err: - complete(&probe_event); if (ret_val) vmbus_acpi_remove(device); return ret_val; @@ -2620,6 +2645,7 @@ static struct acpi_driver vmbus_acpi_driver = { .remove = vmbus_acpi_remove, }, .drv.pm = &vmbus_bus_pm, + .drv.probe_type = PROBE_FORCE_SYNCHRONOUS, }; static void hv_kexec_handler(void) @@ -2692,7 +2718,7 @@ static struct syscore_ops hv_synic_syscore_ops = { static int __init hv_acpi_init(void) { - int ret, t; + int ret; if (!hv_is_hyperv_initialized()) return -ENODEV; @@ -2700,8 +2726,6 @@ static int __init hv_acpi_init(void) if (hv_root_partition) return 0; - init_completion(&probe_event); - /* * Get ACPI resources first. */ @@ -2710,9 +2734,8 @@ static int __init hv_acpi_init(void) if (ret) return ret; - t = wait_for_completion_timeout(&probe_event, 5*HZ); - if (t == 0) { - ret = -ETIMEDOUT; + if (!hv_acpi_dev) { + ret = -ENODEV; goto cleanup; } diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index e70d9614bec2c74b7206259da8dfc43824a05076..7ac3daaf59ce0618ebc5252d7d9e7a78ed73e341 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -257,14 +257,14 @@ config SENSORS_AHT10 will be called aht10. config SENSORS_AQUACOMPUTER_D5NEXT - tristate "Aquacomputer D5 Next, Octo, Quadro, Farbwerk, and Farbwerk 360" + tristate "Aquacomputer D5 Next, Octo, Quadro, Farbwerk, Farbwerk 360, High Flow Next" depends on USB_HID select CRC16 help If you say yes here you get support for sensors and fans of the Aquacomputer D5 Next watercooling pump, Octo and Quadro fan - controllers, Farbwerk and Farbwerk 360 RGB controllers, where - available. + controllers, Farbwerk and Farbwerk 360 RGB controllers, High Flow + Next sensor, where available. This driver can also be built as a module. If so, the module will be called aquacomputer_d5next. @@ -393,6 +393,7 @@ config SENSORS_ASB100 config SENSORS_ASPEED tristate "ASPEED AST2400/AST2500 PWM and Fan tach driver" + depends on ARCH_ASPEED || COMPILE_TEST depends on THERMAL || THERMAL=n select REGMAP help @@ -1066,6 +1067,18 @@ config SENSORS_MAX31730 This driver can also be built as a module. If so, the module will be called max31730. +config SENSORS_MAX31760 + tristate "MAX31760 fan speed controller" + depends on I2C + select REGMAP_I2C + help + Support for the Analog Devices MAX31760 Precision Fan-Speed + Controller. MAX31760 integrates temperature sensing along with + precision PWM fan control. + + This driver can also be built as a module. If so, the module + will be called max31760. + config SENSORS_MAX6620 tristate "Maxim MAX6620 fan controller" depends on I2C @@ -1745,6 +1758,7 @@ config SENSORS_SIS5595 config SENSORS_SY7636A tristate "Silergy SY7636A" + depends on MFD_SY7636A help If you say yes here you get support for the thermistor readout of the Silergy SY7636A PMIC. @@ -1785,6 +1799,19 @@ config SENSORS_EMC2103 This driver can also be built as a module. If so, the module will be called emc2103. +config SENSORS_EMC2305 + tristate "Microchip EMC2305 and compatible EMC2301/2/3" + depends on I2C + imply THERMAL + help + If you say yes here you get support for the Microchip EMC2305 + fan controller chips. + The Microchip EMC2305 is a fan controller for up to 5 fans. + Fan rotation speeds are reported in RPM. + + This driver can also be built as a module. If so, the module + will be called emc2305. + config SENSORS_EMC6W201 tristate "SMSC EMC6W201" depends on I2C @@ -2341,21 +2368,6 @@ config SENSORS_ASUS_WMI This driver can also be built as a module. If so, the module will be called asus_wmi_sensors. -config SENSORS_ASUS_WMI_EC - tristate "ASUS WMI B550/X570" - depends on ACPI_WMI && SENSORS_ASUS_EC=n - help - If you say yes here you get support for the ACPI embedded controller - hardware monitoring interface found in B550/X570 ASUS motherboards. - This driver will provide readings of fans, voltages and temperatures - through the system firmware. - - This driver is deprecated in favor of the ASUS EC Sensors driver - which provides fully compatible output. - - This driver can also be built as a module. If so, the module - will be called asus_wmi_sensors_ec. - config SENSORS_ASUS_EC tristate "ASUS EC Sensors" depends on X86 diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile index 007e829d1d0d0c5b427b72b57ae628deb4a6bd67..11d076cad8a2db9af6f3797cd46b37841242e3e9 100644 --- a/drivers/hwmon/Makefile +++ b/drivers/hwmon/Makefile @@ -11,7 +11,6 @@ obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o obj-$(CONFIG_SENSORS_ATK0110) += asus_atk0110.o obj-$(CONFIG_SENSORS_ASUS_EC) += asus-ec-sensors.o obj-$(CONFIG_SENSORS_ASUS_WMI) += asus_wmi_sensors.o -obj-$(CONFIG_SENSORS_ASUS_WMI_EC) += asus_wmi_ec_sensors.o # Native drivers # asb100, then w83781d go first, as they can override other drivers' addresses. @@ -70,6 +69,7 @@ obj-$(CONFIG_SENSORS_DS620) += ds620.o obj-$(CONFIG_SENSORS_DS1621) += ds1621.o obj-$(CONFIG_SENSORS_EMC1403) += emc1403.o obj-$(CONFIG_SENSORS_EMC2103) += emc2103.o +obj-$(CONFIG_SENSORS_EMC2305) += emc2305.o obj-$(CONFIG_SENSORS_EMC6W201) += emc6w201.o obj-$(CONFIG_SENSORS_F71805F) += f71805f.o obj-$(CONFIG_SENSORS_F71882FG) += f71882fg.o @@ -140,6 +140,7 @@ obj-$(CONFIG_SENSORS_MAX1668) += max1668.o obj-$(CONFIG_SENSORS_MAX197) += max197.o obj-$(CONFIG_SENSORS_MAX31722) += max31722.o obj-$(CONFIG_SENSORS_MAX31730) += max31730.o +obj-$(CONFIG_SENSORS_MAX31760) += max31760.o obj-$(CONFIG_SENSORS_MAX6620) += max6620.o obj-$(CONFIG_SENSORS_MAX6621) += max6621.o obj-$(CONFIG_SENSORS_MAX6639) += max6639.o diff --git a/drivers/hwmon/abituguru.c b/drivers/hwmon/abituguru.c index 681f0623868f08fb575a2396292d695dd83f035c..a7cae6568155810c3b58b5ade099256763604394 100644 --- a/drivers/hwmon/abituguru.c +++ b/drivers/hwmon/abituguru.c @@ -1504,7 +1504,6 @@ LEAVE_UPDATE: return NULL; } -#ifdef CONFIG_PM_SLEEP static int abituguru_suspend(struct device *dev) { struct abituguru_data *data = dev_get_drvdata(dev); @@ -1526,16 +1525,12 @@ static int abituguru_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(abituguru_pm, abituguru_suspend, abituguru_resume); -#define ABIT_UGURU_PM (&abituguru_pm) -#else -#define ABIT_UGURU_PM NULL -#endif /* CONFIG_PM */ +static DEFINE_SIMPLE_DEV_PM_OPS(abituguru_pm, abituguru_suspend, abituguru_resume); static struct platform_driver abituguru_driver = { .driver = { .name = ABIT_UGURU_NAME, - .pm = ABIT_UGURU_PM, + .pm = pm_sleep_ptr(&abituguru_pm), }, .probe = abituguru_probe, .remove = abituguru_remove, diff --git a/drivers/hwmon/abituguru3.c b/drivers/hwmon/abituguru3.c index 8229ad30c9090d8babf3aabdfa8219b7f682aa0e..afb21f73032d777e02046bc19e04e9a09de83c0e 100644 --- a/drivers/hwmon/abituguru3.c +++ b/drivers/hwmon/abituguru3.c @@ -1127,7 +1127,6 @@ LEAVE_UPDATE: return NULL; } -#ifdef CONFIG_PM_SLEEP static int abituguru3_suspend(struct device *dev) { struct abituguru3_data *data = dev_get_drvdata(dev); @@ -1146,16 +1145,12 @@ static int abituguru3_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(abituguru3_pm, abituguru3_suspend, abituguru3_resume); -#define ABIT_UGURU3_PM (&abituguru3_pm) -#else -#define ABIT_UGURU3_PM NULL -#endif /* CONFIG_PM */ +static DEFINE_SIMPLE_DEV_PM_OPS(abituguru3_pm, abituguru3_suspend, abituguru3_resume); static struct platform_driver abituguru3_driver = { .driver = { .name = ABIT_UGURU3_NAME, - .pm = ABIT_UGURU3_PM + .pm = pm_sleep_ptr(&abituguru3_pm), }, .probe = abituguru3_probe, .remove = abituguru3_remove, diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c index d2545a1be9fc079145ecf39b2ed4f0fd0031d29a..0962c12eba5a007f1e6f001648b420e5710f807c 100644 --- a/drivers/hwmon/acpi_power_meter.c +++ b/drivers/hwmon/acpi_power_meter.c @@ -598,7 +598,7 @@ static int read_domain_devices(struct acpi_power_meter_resource *resource) continue; /* Create a symlink to domain objects */ - obj = acpi_bus_get_acpi_device(element->reference.handle); + obj = acpi_get_acpi_dev(element->reference.handle); resource->domain_devices[i] = obj; if (!obj) continue; @@ -927,8 +927,6 @@ static int acpi_power_meter_remove(struct acpi_device *device) return 0; } -#ifdef CONFIG_PM_SLEEP - static int acpi_power_meter_resume(struct device *dev) { struct acpi_power_meter_resource *resource; @@ -946,9 +944,8 @@ static int acpi_power_meter_resume(struct device *dev) return 0; } -#endif /* CONFIG_PM_SLEEP */ - -static SIMPLE_DEV_PM_OPS(acpi_power_meter_pm, NULL, acpi_power_meter_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(acpi_power_meter_pm, NULL, + acpi_power_meter_resume); static struct acpi_driver acpi_power_meter_driver = { .name = "power_meter", @@ -959,7 +956,7 @@ static struct acpi_driver acpi_power_meter_driver = { .remove = acpi_power_meter_remove, .notify = acpi_power_meter_notify, }, - .drv.pm = &acpi_power_meter_pm, + .drv.pm = pm_sleep_ptr(&acpi_power_meter_pm), }; /* Module init/exit routines */ diff --git a/drivers/hwmon/adc128d818.c b/drivers/hwmon/adc128d818.c index fd938c70293fb2010f2b535814cf38e17cb7d9fc..97b330b6c165c0fa5f6cfe9e906a3804bd13c154 100644 --- a/drivers/hwmon/adc128d818.c +++ b/drivers/hwmon/adc128d818.c @@ -384,7 +384,7 @@ static int adc128_detect(struct i2c_client *client, struct i2c_board_info *info) if (i2c_smbus_read_byte_data(client, ADC128_REG_BUSY_STATUS) & 0xfc) return -ENODEV; - strlcpy(info->type, "adc128d818", I2C_NAME_SIZE); + strscpy(info->type, "adc128d818", I2C_NAME_SIZE); return 0; } @@ -495,14 +495,12 @@ error: return err; } -static int adc128_remove(struct i2c_client *client) +static void adc128_remove(struct i2c_client *client) { struct adc128_data *data = i2c_get_clientdata(client); if (data->regulator) regulator_disable(data->regulator); - - return 0; } static const struct i2c_device_id adc128_id[] = { diff --git a/drivers/hwmon/adm1021.c b/drivers/hwmon/adm1021.c index 91ecfee243bf9e25001d89603e3b29d1a1751316..2dc45e958730e6d0a9557d4dc8ad5682df15deb0 100644 --- a/drivers/hwmon/adm1021.c +++ b/drivers/hwmon/adm1021.c @@ -426,7 +426,7 @@ static int adm1021_detect(struct i2c_client *client, pr_debug("Detected chip %s at adapter %d, address 0x%02x.\n", type_name, i2c_adapter_id(adapter), client->addr); - strlcpy(info->type, type_name, I2C_NAME_SIZE); + strscpy(info->type, type_name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adm1025.c b/drivers/hwmon/adm1025.c index 4352f6a884e8f3eec93dd324336a50c4c021dd84..2984c4f98496fe41a0681319d1e0b73a45592489 100644 --- a/drivers/hwmon/adm1025.c +++ b/drivers/hwmon/adm1025.c @@ -470,7 +470,7 @@ static int adm1025_detect(struct i2c_client *client, else return -ENODEV; - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adm1026.c b/drivers/hwmon/adm1026.c index 69b3ec752944744aa0e7deacad484b9eacaf65a7..1f084f7087435decd1095a834e03bce61945acc9 100644 --- a/drivers/hwmon/adm1026.c +++ b/drivers/hwmon/adm1026.c @@ -1610,7 +1610,7 @@ static int adm1026_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "adm1026", I2C_NAME_SIZE); + strscpy(info->type, "adm1026", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adm1029.c b/drivers/hwmon/adm1029.c index 3e1999413f3252128d3caeecd251ba396fececf6..eaf6e5e04aac13ec46d82915f359c6c9ecf0519c 100644 --- a/drivers/hwmon/adm1029.c +++ b/drivers/hwmon/adm1029.c @@ -329,7 +329,7 @@ static int adm1029_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "adm1029", I2C_NAME_SIZE); + strscpy(info->type, "adm1029", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adm1031.c b/drivers/hwmon/adm1031.c index ac841fa3a369d6edd6afc6c44dad920403202579..b42797bcb5b479ed22fa9089299f4fb7e36ba240 100644 --- a/drivers/hwmon/adm1031.c +++ b/drivers/hwmon/adm1031.c @@ -985,7 +985,7 @@ static int adm1031_detect(struct i2c_client *client, return -ENODEV; name = (id == 0x30) ? "adm1030" : "adm1031"; - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c index 483cd757abd339a402ff6bd3ccf53dabd4b7eda1..40e3558d37096fc9dd879134389c2572004018ae 100644 --- a/drivers/hwmon/adm9240.c +++ b/drivers/hwmon/adm9240.c @@ -501,17 +501,23 @@ static int adm9240_fan_read(struct device *dev, u32 attr, int channel, long *val switch (attr) { case hwmon_fan_input: + mutex_lock(&data->update_lock); err = regmap_read(data->regmap, ADM9240_REG_FAN(channel), ®val); - if (err < 0) + if (err < 0) { + mutex_unlock(&data->update_lock); return err; + } if (regval == 255 && data->fan_div[channel] < 3) { /* adjust fan clock divider on overflow */ err = adm9240_write_fan_div(data, channel, ++data->fan_div[channel]); - if (err) + if (err) { + mutex_unlock(&data->update_lock); return err; + } } *val = FAN_FROM_REG(regval, BIT(data->fan_div[channel])); + mutex_unlock(&data->update_lock); break; case hwmon_fan_div: *val = BIT(data->fan_div[channel]); diff --git a/drivers/hwmon/adt7310.c b/drivers/hwmon/adt7310.c index 1efc0bdcceabdc1d2bffd28872420dfb1b821865..067865f4887a794cfb9199cd6c259e6b3bf2b8bb 100644 --- a/drivers/hwmon/adt7310.c +++ b/drivers/hwmon/adt7310.c @@ -152,7 +152,7 @@ MODULE_DEVICE_TABLE(spi, adt7310_id); static struct spi_driver adt7310_driver = { .driver = { .name = "adt7310", - .pm = ADT7X10_DEV_PM_OPS, + .pm = pm_sleep_ptr(&adt7x10_dev_pm_ops), }, .probe = adt7310_spi_probe, .id_table = adt7310_id, diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c index aede5baca7b9f92ef5ce4f84bb50d0063ca36e09..0cebf6777239eb3e0c090f5b2c01e44032e16b87 100644 --- a/drivers/hwmon/adt7410.c +++ b/drivers/hwmon/adt7410.c @@ -98,7 +98,7 @@ static struct i2c_driver adt7410_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "adt7410", - .pm = ADT7X10_DEV_PM_OPS, + .pm = pm_sleep_ptr(&adt7x10_dev_pm_ops), }, .probe_new = adt7410_i2c_probe, .id_table = adt7410_ids, diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c index fad74aa62b642ddc0bd9d2d723383363f282370b..bf5c5618f8d0d92d949d564c135aaebba888fa40 100644 --- a/drivers/hwmon/adt7411.c +++ b/drivers/hwmon/adt7411.c @@ -590,7 +590,7 @@ static int adt7411_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "adt7411", I2C_NAME_SIZE); + strscpy(info->type, "adt7411", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c index e75bbd87ad0933bb35575d48db18924b76f0e75a..9c0235849d4b6882ff85c1a245a40321853f9642 100644 --- a/drivers/hwmon/adt7462.c +++ b/drivers/hwmon/adt7462.c @@ -1782,7 +1782,7 @@ static int adt7462_detect(struct i2c_client *client, if (revision != ADT7462_REVISION) return -ENODEV; - strlcpy(info->type, "adt7462", I2C_NAME_SIZE); + strscpy(info->type, "adt7462", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c index c67cd037a93fde44ee7e4d9d0fc1e53b642c27a6..927f8df05b7c9318bfe762e8791a380aaf573f3b 100644 --- a/drivers/hwmon/adt7470.c +++ b/drivers/hwmon/adt7470.c @@ -1296,12 +1296,11 @@ static int adt7470_probe(struct i2c_client *client) return 0; } -static int adt7470_remove(struct i2c_client *client) +static void adt7470_remove(struct i2c_client *client) { struct adt7470_data *data = i2c_get_clientdata(client); kthread_stop(data->auto_update); - return 0; } static const struct i2c_device_id adt7470_id[] = { diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c index ac480e6e4818b5763b3a0ed9d2b1fdf21a00e945..51b3d16c32233d78a524a9afca79af57dffce362 100644 --- a/drivers/hwmon/adt7475.c +++ b/drivers/hwmon/adt7475.c @@ -1342,7 +1342,7 @@ static int adt7475_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/adt7x10.c b/drivers/hwmon/adt7x10.c index ce54bffab2ec2c8774f03f4832d414e7440cc83d..da67734edafd756b574d3be9a394849d37f20ffd 100644 --- a/drivers/hwmon/adt7x10.c +++ b/drivers/hwmon/adt7x10.c @@ -397,8 +397,6 @@ int adt7x10_probe(struct device *dev, const char *name, int irq, } EXPORT_SYMBOL_GPL(adt7x10_probe); -#ifdef CONFIG_PM_SLEEP - static int adt7x10_suspend(struct device *dev) { struct adt7x10_data *data = dev_get_drvdata(dev); @@ -414,10 +412,7 @@ static int adt7x10_resume(struct device *dev) return regmap_write(data->regmap, ADT7X10_CONFIG, data->config); } -SIMPLE_DEV_PM_OPS(adt7x10_dev_pm_ops, adt7x10_suspend, adt7x10_resume); -EXPORT_SYMBOL_GPL(adt7x10_dev_pm_ops); - -#endif /* CONFIG_PM_SLEEP */ +EXPORT_SIMPLE_DEV_PM_OPS(adt7x10_dev_pm_ops, adt7x10_suspend, adt7x10_resume); MODULE_AUTHOR("Hartmut Knaack"); MODULE_DESCRIPTION("ADT7410/ADT7420, ADT7310/ADT7320 common code"); diff --git a/drivers/hwmon/adt7x10.h b/drivers/hwmon/adt7x10.h index ba22c32c83552bb16f62ea9ddee9dc91490bdbb6..46caf3e21978756d678580b9f96a52d6700c4804 100644 --- a/drivers/hwmon/adt7x10.h +++ b/drivers/hwmon/adt7x10.h @@ -20,11 +20,6 @@ struct device; int adt7x10_probe(struct device *dev, const char *name, int irq, struct regmap *regmap); -#ifdef CONFIG_PM_SLEEP extern const struct dev_pm_ops adt7x10_dev_pm_ops; -#define ADT7X10_DEV_PM_OPS (&adt7x10_dev_pm_ops) -#else -#define ADT7X10_DEV_PM_OPS NULL -#endif #endif diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c index 0c16face3fd3a686283de57167da5d834e509b63..3bfd12ff4b3ca67aa2cc2883524a9b93b8f26f7d 100644 --- a/drivers/hwmon/amc6821.c +++ b/drivers/hwmon/amc6821.c @@ -809,7 +809,7 @@ static int amc6821_detect( } dev_info(&adapter->dev, "amc6821: chip found at 0x%02x.\n", address); - strlcpy(info->type, "amc6821", I2C_NAME_SIZE); + strscpy(info->type, "amc6821", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c index 66430553cc45106d0ee5e96465cdb42aac089ce6..c51a2678f0eb553f790f35a5afbbd1317f104040 100644 --- a/drivers/hwmon/aquacomputer_d5next.c +++ b/drivers/hwmon/aquacomputer_d5next.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ /* * hwmon driver for Aquacomputer devices (D5 Next, Farbwerk, Farbwerk 360, Octo, - * Quadro) + * Quadro, High Flow Next) * * Aquacomputer devices send HID reports (with ID 0x01) every second to report * sensor values. @@ -26,15 +26,17 @@ #define USB_PRODUCT_ID_D5NEXT 0xf00e #define USB_PRODUCT_ID_FARBWERK360 0xf010 #define USB_PRODUCT_ID_OCTO 0xf011 +#define USB_PRODUCT_ID_HIGHFLOWNEXT 0xf012 -enum kinds { d5next, farbwerk, farbwerk360, octo, quadro }; +enum kinds { d5next, farbwerk, farbwerk360, octo, quadro, highflownext }; static const char *const aqc_device_names[] = { [d5next] = "d5next", [farbwerk] = "farbwerk", [farbwerk360] = "farbwerk360", [octo] = "octo", - [quadro] = "quadro" + [quadro] = "quadro", + [highflownext] = "highflownext" }; #define DRIVER_NAME "aquacomputer_d5next" @@ -71,6 +73,8 @@ static u8 secondary_ctrl_report[] = { #define D5NEXT_COOLANT_TEMP 0x57 #define D5NEXT_NUM_FANS 2 #define D5NEXT_NUM_SENSORS 1 +#define D5NEXT_NUM_VIRTUAL_SENSORS 8 +#define D5NEXT_VIRTUAL_SENSORS_START 0x3f #define D5NEXT_PUMP_OFFSET 0x6c #define D5NEXT_FAN_OFFSET 0x5f #define D5NEXT_5V_VOLTAGE 0x39 @@ -86,14 +90,18 @@ static u16 d5next_ctrl_fan_offsets[] = { 0x97, 0x42 }; #define FARBWERK_SENSOR_START 0x2f /* Register offsets for the Farbwerk 360 RGB controller */ -#define FARBWERK360_NUM_SENSORS 4 -#define FARBWERK360_SENSOR_START 0x32 +#define FARBWERK360_NUM_SENSORS 4 +#define FARBWERK360_SENSOR_START 0x32 +#define FARBWERK360_NUM_VIRTUAL_SENSORS 16 +#define FARBWERK360_VIRTUAL_SENSORS_START 0x3a /* Register offsets for the Octo fan controller */ #define OCTO_POWER_CYCLES 0x18 #define OCTO_NUM_FANS 8 #define OCTO_NUM_SENSORS 4 #define OCTO_SENSOR_START 0x3D +#define OCTO_NUM_VIRTUAL_SENSORS 16 +#define OCTO_VIRTUAL_SENSORS_START 0x45 #define OCTO_CTRL_REPORT_SIZE 0x65F static u8 octo_sensor_fan_offsets[] = { 0x7D, 0x8A, 0x97, 0xA4, 0xB1, 0xBE, 0xCB, 0xD8 }; @@ -105,12 +113,24 @@ static u16 octo_ctrl_fan_offsets[] = { 0x5B, 0xB0, 0x105, 0x15A, 0x1AF, 0x204, 0 #define QUADRO_NUM_FANS 4 #define QUADRO_NUM_SENSORS 4 #define QUADRO_SENSOR_START 0x34 +#define QUADRO_NUM_VIRTUAL_SENSORS 16 +#define QUADRO_VIRTUAL_SENSORS_START 0x3c #define QUADRO_CTRL_REPORT_SIZE 0x3c1 #define QUADRO_FLOW_SENSOR_OFFSET 0x6e static u8 quadro_sensor_fan_offsets[] = { 0x70, 0x7D, 0x8A, 0x97 }; /* Fan speed registers in Quadro control report (from 0-100%) */ -static u16 quadro_ctrl_fan_offsets[] = { 0x36, 0x8b, 0xe0, 0x135 }; +static u16 quadro_ctrl_fan_offsets[] = { 0x37, 0x8c, 0xe1, 0x136 }; + +/* Register offsets for the High Flow Next */ +#define HIGHFLOWNEXT_NUM_SENSORS 2 +#define HIGHFLOWNEXT_SENSOR_START 85 +#define HIGHFLOWNEXT_FLOW 81 +#define HIGHFLOWNEXT_WATER_QUALITY 89 +#define HIGHFLOWNEXT_POWER 91 +#define HIGHFLOWNEXT_CONDUCTIVITY 95 +#define HIGHFLOWNEXT_5V_VOLTAGE 97 +#define HIGHFLOWNEXT_5V_VOLTAGE_USB 99 /* Labels for D5 Next */ static const char *const label_d5next_temp[] = { @@ -147,6 +167,25 @@ static const char *const label_temp_sensors[] = { "Sensor 4" }; +static const char *const label_virtual_temp_sensors[] = { + "Virtual sensor 1", + "Virtual sensor 2", + "Virtual sensor 3", + "Virtual sensor 4", + "Virtual sensor 5", + "Virtual sensor 6", + "Virtual sensor 7", + "Virtual sensor 8", + "Virtual sensor 9", + "Virtual sensor 10", + "Virtual sensor 11", + "Virtual sensor 12", + "Virtual sensor 13", + "Virtual sensor 14", + "Virtual sensor 15", + "Virtual sensor 16", +}; + /* Labels for Octo and Quadro (except speed) */ static const char *const label_fan_speed[] = { "Fan 1 speed", @@ -201,6 +240,27 @@ static const char *const label_quadro_speeds[] = { "Flow speed [dL/h]" }; +/* Labels for High Flow Next */ +static const char *const label_highflownext_temp_sensors[] = { + "Coolant temp", + "External sensor" +}; + +static const char *const label_highflownext_fan_speed[] = { + "Flow [dL/h]", + "Water quality [%]", + "Conductivity [nS/cm]", +}; + +static const char *const label_highflownext_power[] = { + "Dissipated power", +}; + +static const char *const label_highflownext_voltage[] = { + "+5V voltage", + "+5V USB voltage" +}; + struct aqc_data { struct hid_device *hdev; struct device *hwmon_dev; @@ -220,6 +280,8 @@ struct aqc_data { u16 *fan_ctrl_offsets; int num_temp_sensors; int temp_sensor_start_offset; + int num_virtual_temp_sensors; + int virtual_temp_sensor_start_offset; u16 power_cycle_count_offset; u8 flow_sensor_offset; @@ -231,7 +293,7 @@ struct aqc_data { u32 power_cycles; /* Sensor values */ - s32 temp_input[4]; + s32 temp_input[20]; /* Max 4 physical and 16 virtual */ u16 speed_input[8]; u32 power_input[8]; u16 voltage_input[8]; @@ -239,6 +301,7 @@ struct aqc_data { /* Label values */ const char *const *temp_label; + const char *const *virtual_temp_label; const char *const *speed_label; const char *const *power_label; const char *const *voltage_label; @@ -345,7 +408,7 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3 switch (type) { case hwmon_temp: - if (channel < priv->num_temp_sensors) + if (channel < priv->num_temp_sensors + priv->num_virtual_temp_sensors) return 0444; break; case hwmon_pwm: @@ -360,6 +423,11 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3 break; case hwmon_fan: switch (priv->kind) { + case highflownext: + /* Special case to support flow sensor, water quality and conductivity */ + if (channel < 3) + return 0444; + break; case quadro: /* Special case to support flow sensor */ if (channel < priv->num_fans + 1) @@ -372,6 +440,18 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3 } break; case hwmon_power: + switch (priv->kind) { + case highflownext: + /* Special case to support one power sensor */ + if (channel == 0) + return 0444; + break; + default: + if (channel < priv->num_fans) + return 0444; + break; + } + break; case hwmon_curr: if (channel < priv->num_fans) return 0444; @@ -383,6 +463,11 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3 if (channel < priv->num_fans + 2) return 0444; break; + case highflownext: + /* Special case to support two voltage sensors */ + if (channel < 2) + return 0444; + break; default: if (channel < priv->num_fans) return 0444; @@ -447,7 +532,10 @@ static int aqc_read_string(struct device *dev, enum hwmon_sensor_types type, u32 switch (type) { case hwmon_temp: - *str = priv->temp_label[channel]; + if (channel < priv->num_temp_sensors) + *str = priv->temp_label[channel]; + else + *str = priv->virtual_temp_label[channel - priv->num_temp_sensors]; break; case hwmon_fan: *str = priv->speed_label[channel]; @@ -509,6 +597,22 @@ static const struct hwmon_ops aqc_hwmon_ops = { static const struct hwmon_channel_info *aqc_info[] = { HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, HWMON_T_INPUT | HWMON_T_LABEL, @@ -568,7 +672,7 @@ static const struct hwmon_chip_info aqc_chip_info = { static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { - int i, sensor_value; + int i, j, sensor_value; struct aqc_data *priv; if (report->id != STATUS_REPORT_ID) @@ -581,7 +685,7 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 priv->serial_number[1] = get_unaligned_be16(data + SERIAL_SECOND_PART); priv->firmware_version = get_unaligned_be16(data + FIRMWARE_VERSION); - /* Temperature sensor readings */ + /* Physical temperature sensor readings */ for (i = 0; i < priv->num_temp_sensors; i++) { sensor_value = get_unaligned_be16(data + priv->temp_sensor_start_offset + @@ -592,6 +696,18 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 priv->temp_input[i] = sensor_value * 10; } + /* Virtual temperature sensor readings */ + for (j = 0; j < priv->num_virtual_temp_sensors; j++) { + sensor_value = get_unaligned_be16(data + + priv->virtual_temp_sensor_start_offset + + j * AQC_TEMP_SENSOR_SIZE); + if (sensor_value == AQC_TEMP_SENSOR_DISCONNECTED) + priv->temp_input[i] = -ENODATA; + else + priv->temp_input[i] = sensor_value * 10; + i++; + } + /* Fan speed and related readings */ for (i = 0; i < priv->num_fans; i++) { priv->speed_input[i] = @@ -618,6 +734,22 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 case quadro: priv->speed_input[4] = get_unaligned_be16(data + priv->flow_sensor_offset); break; + case highflownext: + /* If external temp sensor is not connected, its power reading is also N/A */ + if (priv->temp_input[1] == -ENODATA) + priv->power_input[0] = -ENODATA; + else + priv->power_input[0] = + get_unaligned_be16(data + HIGHFLOWNEXT_POWER) * 1000000; + + priv->voltage_input[0] = get_unaligned_be16(data + HIGHFLOWNEXT_5V_VOLTAGE) * 10; + priv->voltage_input[1] = + get_unaligned_be16(data + HIGHFLOWNEXT_5V_VOLTAGE_USB) * 10; + + priv->speed_input[0] = get_unaligned_be16(data + HIGHFLOWNEXT_FLOW); + priv->speed_input[1] = get_unaligned_be16(data + HIGHFLOWNEXT_WATER_QUALITY); + priv->speed_input[2] = get_unaligned_be16(data + HIGHFLOWNEXT_CONDUCTIVITY); + break; default: break; } @@ -717,10 +849,13 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) priv->fan_ctrl_offsets = d5next_ctrl_fan_offsets; priv->num_temp_sensors = D5NEXT_NUM_SENSORS; priv->temp_sensor_start_offset = D5NEXT_COOLANT_TEMP; + priv->num_virtual_temp_sensors = D5NEXT_NUM_VIRTUAL_SENSORS; + priv->virtual_temp_sensor_start_offset = D5NEXT_VIRTUAL_SENSORS_START; priv->power_cycle_count_offset = D5NEXT_POWER_CYCLES; priv->buffer_size = D5NEXT_CTRL_REPORT_SIZE; priv->temp_label = label_d5next_temp; + priv->virtual_temp_label = label_virtual_temp_sensors; priv->speed_label = label_d5next_speeds; priv->power_label = label_d5next_power; priv->voltage_label = label_d5next_voltages; @@ -740,7 +875,11 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) priv->num_fans = 0; priv->num_temp_sensors = FARBWERK360_NUM_SENSORS; priv->temp_sensor_start_offset = FARBWERK360_SENSOR_START; + priv->num_virtual_temp_sensors = FARBWERK360_NUM_VIRTUAL_SENSORS; + priv->virtual_temp_sensor_start_offset = FARBWERK360_VIRTUAL_SENSORS_START; + priv->temp_label = label_temp_sensors; + priv->virtual_temp_label = label_virtual_temp_sensors; break; case USB_PRODUCT_ID_OCTO: priv->kind = octo; @@ -750,10 +889,13 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) priv->fan_ctrl_offsets = octo_ctrl_fan_offsets; priv->num_temp_sensors = OCTO_NUM_SENSORS; priv->temp_sensor_start_offset = OCTO_SENSOR_START; + priv->num_virtual_temp_sensors = OCTO_NUM_VIRTUAL_SENSORS; + priv->virtual_temp_sensor_start_offset = OCTO_VIRTUAL_SENSORS_START; priv->power_cycle_count_offset = OCTO_POWER_CYCLES; priv->buffer_size = OCTO_CTRL_REPORT_SIZE; priv->temp_label = label_temp_sensors; + priv->virtual_temp_label = label_virtual_temp_sensors; priv->speed_label = label_fan_speed; priv->power_label = label_fan_power; priv->voltage_label = label_fan_voltage; @@ -767,16 +909,32 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id) priv->fan_ctrl_offsets = quadro_ctrl_fan_offsets; priv->num_temp_sensors = QUADRO_NUM_SENSORS; priv->temp_sensor_start_offset = QUADRO_SENSOR_START; + priv->num_virtual_temp_sensors = QUADRO_NUM_VIRTUAL_SENSORS; + priv->virtual_temp_sensor_start_offset = QUADRO_VIRTUAL_SENSORS_START; priv->power_cycle_count_offset = QUADRO_POWER_CYCLES; priv->buffer_size = QUADRO_CTRL_REPORT_SIZE; priv->flow_sensor_offset = QUADRO_FLOW_SENSOR_OFFSET; priv->temp_label = label_temp_sensors; + priv->virtual_temp_label = label_virtual_temp_sensors; priv->speed_label = label_quadro_speeds; priv->power_label = label_fan_power; priv->voltage_label = label_fan_voltage; priv->current_label = label_fan_current; break; + case USB_PRODUCT_ID_HIGHFLOWNEXT: + priv->kind = highflownext; + + priv->num_fans = 0; + priv->num_temp_sensors = HIGHFLOWNEXT_NUM_SENSORS; + priv->temp_sensor_start_offset = HIGHFLOWNEXT_SENSOR_START; + priv->power_cycle_count_offset = QUADRO_POWER_CYCLES; + + priv->temp_label = label_highflownext_temp_sensors; + priv->speed_label = label_highflownext_fan_speed; + priv->power_label = label_highflownext_power; + priv->voltage_label = label_highflownext_voltage; + break; default: break; } @@ -833,6 +991,7 @@ static const struct hid_device_id aqc_table[] = { { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_FARBWERK360) }, { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_OCTO) }, { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_QUADRO) }, + { HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_HIGHFLOWNEXT) }, { } }; diff --git a/drivers/hwmon/asb100.c b/drivers/hwmon/asb100.c index 8cf0bcb85eb4e31f55f8b5ff527024e15678588e..ce4da836765c1d8447e4970a9bcdc95c67031deb 100644 --- a/drivers/hwmon/asb100.c +++ b/drivers/hwmon/asb100.c @@ -208,7 +208,7 @@ static void asb100_write_value(struct i2c_client *client, u16 reg, u16 val); static int asb100_probe(struct i2c_client *client); static int asb100_detect(struct i2c_client *client, struct i2c_board_info *info); -static int asb100_remove(struct i2c_client *client); +static void asb100_remove(struct i2c_client *client); static struct asb100_data *asb100_update_device(struct device *dev); static void asb100_init_client(struct i2c_client *client); @@ -769,7 +769,7 @@ static int asb100_detect(struct i2c_client *client, if (val1 != 0x31 || val2 != 0x06) return -ENODEV; - strlcpy(info->type, "asb100", I2C_NAME_SIZE); + strscpy(info->type, "asb100", I2C_NAME_SIZE); return 0; } @@ -822,7 +822,7 @@ ERROR3: return err; } -static int asb100_remove(struct i2c_client *client) +static void asb100_remove(struct i2c_client *client) { struct asb100_data *data = i2c_get_clientdata(client); @@ -831,8 +831,6 @@ static int asb100_remove(struct i2c_client *client) i2c_unregister_device(data->lm75[1]); i2c_unregister_device(data->lm75[0]); - - return 0; } /* diff --git a/drivers/hwmon/asc7621.c b/drivers/hwmon/asc7621.c index e835605a7456991a12dd4fff1fbf8ee8e870ff1c..54595454537bd1162ed76ae4f7372d64afe9ef5c 100644 --- a/drivers/hwmon/asc7621.c +++ b/drivers/hwmon/asc7621.c @@ -1153,7 +1153,7 @@ static int asc7621_detect(struct i2c_client *client, if (company == asc7621_chips[chip_index].company_id && verstep == asc7621_chips[chip_index].verstep_id) { - strlcpy(info->type, asc7621_chips[chip_index].name, + strscpy(info->type, asc7621_chips[chip_index].name, I2C_NAME_SIZE); dev_info(&adapter->dev, "Matched %s at 0x%02x\n", @@ -1165,7 +1165,7 @@ static int asc7621_detect(struct i2c_client *client, return -ENODEV; } -static int asc7621_remove(struct i2c_client *client) +static void asc7621_remove(struct i2c_client *client) { struct asc7621_data *data = i2c_get_clientdata(client); int i; @@ -1176,8 +1176,6 @@ static int asc7621_remove(struct i2c_client *client) device_remove_file(&client->dev, &(asc7621_params[i].sda.dev_attr)); } - - return 0; } static const struct i2c_device_id asc7621_id[] = { diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 61a4684fc020e1f5852bf3afc59c87efbd27bc42..81e688975c6a79583e61004f2ff4d47901521f93 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -266,9 +266,7 @@ static const struct ec_sensor_info sensors_family_intel_600[] = { #define SENSOR_SET_WATER_BLOCK \ (SENSOR_TEMP_WATER_BLOCK_IN | SENSOR_TEMP_WATER_BLOCK_OUT) - struct ec_board_info { - const char *board_names[MAX_IDENTICAL_BOARD_VARIATIONS]; unsigned long sensors; /* * Defines which mutex to use for guarding access to the state and the @@ -281,152 +279,194 @@ struct ec_board_info { enum board_family family; }; -static const struct ec_board_info board_info[] = { - { - .board_names = {"PRIME X470-PRO"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | - SENSOR_FAN_CPU_OPT | - SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, - .mutex_path = ACPI_GLOBAL_LOCK_PSEUDO_PATH, - .family = family_amd_400_series, - }, - { - .board_names = {"PRIME X570-PRO"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ProArt X570-CREATOR WIFI"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT | - SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, - }, - { - .board_names = {"Pro WS X570-ACE"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET | - SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG CROSSHAIR VIII DARK HERO"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | - SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | - SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW | - SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = { - "ROG CROSSHAIR VIII FORMULA", - "ROG CROSSHAIR VIII HERO", - "ROG CROSSHAIR VIII HERO (WI-FI)", - }, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | - SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | - SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET | - SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = { - "ROG MAXIMUS XI HERO", - "ROG MAXIMUS XI HERO (WI-FI)", - }, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | - SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | - SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_intel_300_series, - }, - { - .board_names = {"ROG CROSSHAIR VIII IMPACT"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | - SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX B550-E GAMING"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | - SENSOR_FAN_CPU_OPT, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX B550-I GAMING"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | - SENSOR_FAN_VRM_HS | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX X570-E GAMING"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | - SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX X570-E GAMING WIFI II"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_CURR_CPU | - SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX X570-F GAMING"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | - SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX X570-I GAMING"}, - .sensors = SENSOR_TEMP_CHIPSET | SENSOR_TEMP_VRM | - SENSOR_TEMP_T_SENSOR | - SENSOR_FAN_VRM_HS | SENSOR_FAN_CHIPSET | - SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, - .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, - .family = family_amd_500_series, - }, - { - .board_names = {"ROG STRIX Z690-A GAMING WIFI D4"}, - .sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM, - .mutex_path = ASUS_HW_ACCESS_MUTEX_RMTW_ASMX, - .family = family_intel_600_series, - }, - { - .board_names = {"ROG ZENITH II EXTREME"}, - .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR | - SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | - SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET | SENSOR_FAN_VRM_HS | - SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE | - SENSOR_SET_WATER_BLOCK | - SENSOR_TEMP_T_SENSOR_2 | SENSOR_TEMP_SENSOR_EXTRA_1 | - SENSOR_TEMP_SENSOR_EXTRA_2 | SENSOR_TEMP_SENSOR_EXTRA_3, - .mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0, - .family = family_amd_500_series, - }, - {} +static const struct ec_board_info board_info_prime_x470_pro = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | + SENSOR_FAN_CPU_OPT | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, + .mutex_path = ACPI_GLOBAL_LOCK_PSEUDO_PATH, + .family = family_amd_400_series, +}; + +static const struct ec_board_info board_info_prime_x570_pro = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | + SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_pro_art_x570_creator_wifi = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | + SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CPU_OPT | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_pro_ws_x570_ace = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM | + SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_crosshair_viii_dark_hero = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_crosshair_viii_hero = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET | + SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU | + SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_maximus_xi_hero = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_intel_300_series, +}; + +static const struct ec_board_info board_info_crosshair_viii_impact = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | + SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU | + SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_b550_e_gaming = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | + SENSOR_FAN_CPU_OPT, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_b550_i_gaming = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | + SENSOR_FAN_VRM_HS | SENSOR_CURR_CPU | + SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_x570_e_gaming = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM | + SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU | + SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_x570_e_gaming_wifi_ii = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_CURR_CPU | + SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_x570_f_gaming = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | + SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_x570_i_gaming = { + .sensors = SENSOR_TEMP_CHIPSET | SENSOR_TEMP_VRM | + SENSOR_TEMP_T_SENSOR | + SENSOR_FAN_VRM_HS | SENSOR_FAN_CHIPSET | + SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE, + .mutex_path = ASUS_HW_ACCESS_MUTEX_ASMX, + .family = family_amd_500_series, +}; + +static const struct ec_board_info board_info_strix_z690_a_gaming_wifi_d4 = { + .sensors = SENSOR_TEMP_T_SENSOR | SENSOR_TEMP_VRM, + .mutex_path = ASUS_HW_ACCESS_MUTEX_RMTW_ASMX, + .family = family_intel_600_series, +}; + +static const struct ec_board_info board_info_zenith_ii_extreme = { + .sensors = SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR | + SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER | + SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET | SENSOR_FAN_VRM_HS | + SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU | SENSOR_IN_CPU_CORE | + SENSOR_SET_WATER_BLOCK | + SENSOR_TEMP_T_SENSOR_2 | SENSOR_TEMP_SENSOR_EXTRA_1 | + SENSOR_TEMP_SENSOR_EXTRA_2 | SENSOR_TEMP_SENSOR_EXTRA_3, + .mutex_path = ASUS_HW_ACCESS_MUTEX_SB_PCI0_SBRG_SIO1_MUT0, + .family = family_amd_500_series, +}; + +#define DMI_EXACT_MATCH_ASUS_BOARD_NAME(name, board_info) \ + { \ + .matches = { \ + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, \ + "ASUSTeK COMPUTER INC."), \ + DMI_EXACT_MATCH(DMI_BOARD_NAME, name), \ + }, \ + .driver_data = (void *)board_info, \ + } + +static const struct dmi_system_id dmi_table[] = { + DMI_EXACT_MATCH_ASUS_BOARD_NAME("PRIME X470-PRO", + &board_info_prime_x470_pro), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("PRIME X570-PRO", + &board_info_prime_x570_pro), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ProArt X570-CREATOR WIFI", + &board_info_pro_art_x570_creator_wifi), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE", + &board_info_pro_ws_x570_ace), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII DARK HERO", + &board_info_crosshair_viii_dark_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII FORMULA", + &board_info_crosshair_viii_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII HERO", + &board_info_crosshair_viii_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII HERO (WI-FI)", + &board_info_crosshair_viii_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS XI HERO", + &board_info_maximus_xi_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG MAXIMUS XI HERO (WI-FI)", + &board_info_maximus_xi_hero), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII IMPACT", + &board_info_crosshair_viii_impact), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-E GAMING", + &board_info_strix_b550_e_gaming), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-I GAMING", + &board_info_strix_b550_i_gaming), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-E GAMING", + &board_info_strix_x570_e_gaming), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-E GAMING WIFI II", + &board_info_strix_x570_e_gaming_wifi_ii), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-F GAMING", + &board_info_strix_x570_f_gaming), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-I GAMING", + &board_info_strix_x570_i_gaming), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX Z690-A GAMING WIFI D4", + &board_info_strix_z690_a_gaming_wifi_d4), + DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG ZENITH II EXTREME", + &board_info_zenith_ii_extreme), + {}, }; struct ec_sensor { @@ -537,12 +577,12 @@ static int find_ec_sensor_index(const struct ec_sensors_data *ec, return -ENOENT; } -static int __init bank_compare(const void *a, const void *b) +static int bank_compare(const void *a, const void *b) { return *((const s8 *)a) - *((const s8 *)b); } -static void __init setup_sensor_data(struct ec_sensors_data *ec) +static void setup_sensor_data(struct ec_sensors_data *ec) { struct ec_sensor *s = ec->sensors; bool bank_found; @@ -574,7 +614,7 @@ static void __init setup_sensor_data(struct ec_sensors_data *ec) sort(ec->banks, ec->nr_banks, 1, bank_compare, NULL); } -static void __init fill_ec_registers(struct ec_sensors_data *ec) +static void fill_ec_registers(struct ec_sensors_data *ec) { const struct ec_sensor_info *si; unsigned int i, j, register_idx = 0; @@ -589,7 +629,7 @@ static void __init fill_ec_registers(struct ec_sensors_data *ec) } } -static int __init setup_lock_data(struct device *dev) +static int setup_lock_data(struct device *dev) { const char *mutex_path; int status; @@ -812,7 +852,7 @@ static umode_t asus_ec_hwmon_is_visible(const void *drvdata, return find_ec_sensor_index(state, type, channel) >= 0 ? S_IRUGO : 0; } -static int __init +static int asus_ec_hwmon_add_chan_info(struct hwmon_channel_info *asus_ec_hwmon_chan, struct device *dev, int num, enum hwmon_sensor_types type, u32 config) @@ -841,27 +881,15 @@ static struct hwmon_chip_info asus_ec_chip_info = { .ops = &asus_ec_hwmon_ops, }; -static const struct ec_board_info * __init get_board_info(void) +static const struct ec_board_info *get_board_info(void) { - const char *dmi_board_vendor = dmi_get_system_info(DMI_BOARD_VENDOR); - const char *dmi_board_name = dmi_get_system_info(DMI_BOARD_NAME); - const struct ec_board_info *board; - - if (!dmi_board_vendor || !dmi_board_name || - strcasecmp(dmi_board_vendor, "ASUSTeK COMPUTER INC.")) - return NULL; - - for (board = board_info; board->sensors; board++) { - if (match_string(board->board_names, - MAX_IDENTICAL_BOARD_VARIATIONS, - dmi_board_name) >= 0) - return board; - } + const struct dmi_system_id *dmi_entry; - return NULL; + dmi_entry = dmi_first_match(dmi_table); + return dmi_entry ? dmi_entry->driver_data : NULL; } -static int __init asus_ec_probe(struct platform_device *pdev) +static int asus_ec_probe(struct platform_device *pdev) { const struct hwmon_channel_info **ptr_asus_ec_ci; int nr_count[hwmon_max] = { 0 }, nr_types = 0; @@ -970,29 +998,37 @@ static int __init asus_ec_probe(struct platform_device *pdev) return PTR_ERR_OR_ZERO(hwdev); } - -static const struct acpi_device_id acpi_ec_ids[] = { - /* Embedded Controller Device */ - { "PNP0C09", 0 }, - {} -}; +MODULE_DEVICE_TABLE(dmi, dmi_table); static struct platform_driver asus_ec_sensors_platform_driver = { .driver = { .name = "asus-ec-sensors", - .acpi_match_table = acpi_ec_ids, }, + .probe = asus_ec_probe, }; -MODULE_DEVICE_TABLE(acpi, acpi_ec_ids); -/* - * we use module_platform_driver_probe() rather than module_platform_driver() - * because the probe function (and its dependants) are marked with __init, which - * means we can't put it into the .probe member of the platform_driver struct - * above, and we can't mark the asus_ec_sensors_platform_driver object as __init - * because the object is referenced from the module exit code. - */ -module_platform_driver_probe(asus_ec_sensors_platform_driver, asus_ec_probe); +static struct platform_device *asus_ec_sensors_platform_device; + +static int __init asus_ec_init(void) +{ + asus_ec_sensors_platform_device = + platform_create_bundle(&asus_ec_sensors_platform_driver, + asus_ec_probe, NULL, 0, NULL, 0); + + if (IS_ERR(asus_ec_sensors_platform_device)) + return PTR_ERR(asus_ec_sensors_platform_device); + + return 0; +} + +static void __exit asus_ec_exit(void) +{ + platform_device_unregister(asus_ec_sensors_platform_device); + platform_driver_unregister(&asus_ec_sensors_platform_driver); +} + +module_init(asus_ec_init); +module_exit(asus_ec_exit); module_param_named(mutex_path, mutex_path_override, charp, 0); MODULE_PARM_DESC(mutex_path, diff --git a/drivers/hwmon/asus_wmi_ec_sensors.c b/drivers/hwmon/asus_wmi_ec_sensors.c deleted file mode 100644 index a3a2f014dec03d65879d89ec896849dd33fce454..0000000000000000000000000000000000000000 --- a/drivers/hwmon/asus_wmi_ec_sensors.c +++ /dev/null @@ -1,622 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * HWMON driver for ASUS B550/X570 motherboards that publish sensor - * values via the embedded controller registers. - * - * Copyright (C) 2021 Eugene Shalygin - * Copyright (C) 2018-2019 Ed Brindley - * - * EC provides: - * - Chipset temperature - * - CPU temperature - * - Motherboard temperature - * - T_Sensor temperature - * - VRM temperature - * - Water In temperature - * - Water Out temperature - * - CPU Optional Fan RPM - * - Chipset Fan RPM - * - Water Flow Fan RPM - * - CPU current - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define ASUSWMI_MONITORING_GUID "466747A0-70EC-11DE-8A39-0800200C9A66" -#define ASUSWMI_METHODID_BLOCK_READ_EC 0x42524543 /* BREC */ -/* From the ASUS DSDT source */ -#define ASUSWMI_BREC_REGISTERS_MAX 16 -#define ASUSWMI_MAX_BUF_LEN 128 -#define SENSOR_LABEL_LEN 16 - -static u32 hwmon_attributes[hwmon_max] = { - [hwmon_chip] = HWMON_C_REGISTER_TZ, - [hwmon_temp] = HWMON_T_INPUT | HWMON_T_LABEL, - [hwmon_in] = HWMON_I_INPUT | HWMON_I_LABEL, - [hwmon_curr] = HWMON_C_INPUT | HWMON_C_LABEL, - [hwmon_fan] = HWMON_F_INPUT | HWMON_F_LABEL, -}; - -struct asus_wmi_ec_sensor_address { - u8 index; - u8 bank; - u8 size; -}; - -#define MAKE_SENSOR_ADDRESS(size_i, bank_i, index_i) { \ - .size = size_i, \ - .bank = bank_i, \ - .index = index_i, \ -} - -struct ec_sensor_info { - struct asus_wmi_ec_sensor_address addr; - char label[SENSOR_LABEL_LEN]; - enum hwmon_sensor_types type; -}; - -#define EC_SENSOR(sensor_label, sensor_type, size, bank, index) { \ - .addr = MAKE_SENSOR_ADDRESS(size, bank, index), \ - .label = sensor_label, \ - .type = sensor_type, \ -} - -enum known_ec_sensor { - SENSOR_TEMP_CHIPSET, - SENSOR_TEMP_CPU, - SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, - SENSOR_TEMP_VRM, - SENSOR_FAN_CPU_OPT, - SENSOR_FAN_CHIPSET, - SENSOR_FAN_VRM_HS, - SENSOR_FAN_WATER_FLOW, - SENSOR_CURR_CPU, - SENSOR_TEMP_WATER_IN, - SENSOR_TEMP_WATER_OUT, - SENSOR_MAX -}; - -/* All known sensors for ASUS EC controllers */ -static const struct ec_sensor_info known_ec_sensors[] = { - [SENSOR_TEMP_CHIPSET] = EC_SENSOR("Chipset", hwmon_temp, 1, 0x00, 0x3a), - [SENSOR_TEMP_CPU] = EC_SENSOR("CPU", hwmon_temp, 1, 0x00, 0x3b), - [SENSOR_TEMP_MB] = EC_SENSOR("Motherboard", hwmon_temp, 1, 0x00, 0x3c), - [SENSOR_TEMP_T_SENSOR] = EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d), - [SENSOR_TEMP_VRM] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e), - [SENSOR_FAN_CPU_OPT] = EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0), - [SENSOR_FAN_VRM_HS] = EC_SENSOR("VRM HS", hwmon_fan, 2, 0x00, 0xb2), - [SENSOR_FAN_CHIPSET] = EC_SENSOR("Chipset", hwmon_fan, 2, 0x00, 0xb4), - [SENSOR_FAN_WATER_FLOW] = EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbc), - [SENSOR_CURR_CPU] = EC_SENSOR("CPU", hwmon_curr, 1, 0x00, 0xf4), - [SENSOR_TEMP_WATER_IN] = EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00), - [SENSOR_TEMP_WATER_OUT] = EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01), -}; - -struct asus_wmi_data { - const enum known_ec_sensor known_board_sensors[SENSOR_MAX + 1]; -}; - -/* boards with EC support */ -static struct asus_wmi_data sensors_board_PW_X570_P = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_FAN_CHIPSET, - SENSOR_MAX - }, -}; - -static struct asus_wmi_data sensors_board_PW_X570_A = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, SENSOR_TEMP_VRM, - SENSOR_FAN_CHIPSET, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -static struct asus_wmi_data sensors_board_R_C8H = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_TEMP_WATER_IN, SENSOR_TEMP_WATER_OUT, - SENSOR_FAN_CPU_OPT, SENSOR_FAN_CHIPSET, SENSOR_FAN_WATER_FLOW, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -/* Same as Hero but without chipset fan */ -static struct asus_wmi_data sensors_board_R_C8DH = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_TEMP_WATER_IN, SENSOR_TEMP_WATER_OUT, - SENSOR_FAN_CPU_OPT, SENSOR_FAN_WATER_FLOW, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -/* Same as Hero but without water */ -static struct asus_wmi_data sensors_board_R_C8F = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_FAN_CPU_OPT, SENSOR_FAN_CHIPSET, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -static struct asus_wmi_data sensors_board_RS_B550_E_G = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_FAN_CPU_OPT, - SENSOR_MAX - }, -}; - -static struct asus_wmi_data sensors_board_RS_B550_I_G = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_FAN_VRM_HS, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -static struct asus_wmi_data sensors_board_RS_X570_E_G = { - .known_board_sensors = { - SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, - SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM, - SENSOR_FAN_CHIPSET, - SENSOR_CURR_CPU, - SENSOR_MAX - }, -}; - -#define DMI_EXACT_MATCH_ASUS_BOARD_NAME(name, sensors) { \ - .matches = { \ - DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), \ - DMI_EXACT_MATCH(DMI_BOARD_NAME, name), \ - }, \ - .driver_data = sensors, \ -} - -static const struct dmi_system_id asus_wmi_ec_dmi_table[] = { - DMI_EXACT_MATCH_ASUS_BOARD_NAME("PRIME X570-PRO", &sensors_board_PW_X570_P), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE", &sensors_board_PW_X570_A), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII DARK HERO", &sensors_board_R_C8DH), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII FORMULA", &sensors_board_R_C8F), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII HERO", &sensors_board_R_C8H), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-E GAMING", &sensors_board_RS_B550_E_G), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-I GAMING", &sensors_board_RS_B550_I_G), - DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-E GAMING", &sensors_board_RS_X570_E_G), - {} -}; -MODULE_DEVICE_TABLE(dmi, asus_wmi_ec_dmi_table); - -struct ec_sensor { - enum known_ec_sensor info_index; - long cached_value; -}; - -/** - * struct asus_wmi_ec_info - sensor info. - * @sensors: list of sensors. - * @read_arg: UTF-16LE string to pass to BRxx() WMI function. - * @read_buffer: decoded output from WMI result. - * @nr_sensors: number of board EC sensors. - * @nr_registers: number of EC registers to read (sensor might span more than 1 register). - * @last_updated: in jiffies. - */ -struct asus_wmi_ec_info { - struct ec_sensor sensors[SENSOR_MAX]; - char read_arg[(ASUSWMI_BREC_REGISTERS_MAX * 4 + 1) * 2]; - u8 read_buffer[ASUSWMI_BREC_REGISTERS_MAX]; - unsigned int nr_sensors; - unsigned int nr_registers; - unsigned long last_updated; -}; - -struct asus_wmi_sensors { - struct asus_wmi_ec_info ec; - /* lock access to internal cache */ - struct mutex lock; -}; - -static int asus_wmi_ec_fill_board_sensors(struct asus_wmi_ec_info *ec, - const enum known_ec_sensor *bsi) -{ - struct ec_sensor *s = ec->sensors; - int i; - - ec->nr_sensors = 0; - ec->nr_registers = 0; - - for (i = 0; bsi[i] != SENSOR_MAX; i++) { - s[i].info_index = bsi[i]; - ec->nr_sensors++; - ec->nr_registers += known_ec_sensors[bsi[i]].addr.size; - } - - return 0; -} - -/* - * The next four functions convert to or from BRxx string argument format. - * The format of the string is as follows: - * - The string consists of two-byte UTF-16LE characters. - * - The value of the very first byte in the string is equal to the total - * length of the next string in bytes, thus excluding the first two-byte - * character. - * - The rest of the string encodes the pairs of (bank, index) pairs, where - * both values are byte-long (0x00 to 0xFF). - * - Numbers are encoded as UTF-16LE hex values. - */ -static int asus_wmi_ec_decode_reply_buffer(const u8 *in, u32 length, u8 *out) -{ - char buffer[ASUSWMI_MAX_BUF_LEN * 2]; - u32 len = min_t(u32, get_unaligned_le16(in), length - 2); - - utf16s_to_utf8s((wchar_t *)(in + 2), len / 2, UTF16_LITTLE_ENDIAN, buffer, sizeof(buffer)); - - return hex2bin(out, buffer, len / 4); -} - -static void asus_wmi_ec_encode_registers(const u8 *in, u32 len, char *out) -{ - char buffer[ASUSWMI_MAX_BUF_LEN * 2]; - - bin2hex(buffer, in, len); - - utf8s_to_utf16s(buffer, len * 2, UTF16_LITTLE_ENDIAN, (wchar_t *)(out + 2), len * 2); - - put_unaligned_le16(len * 4, out); -} - -static void asus_wmi_ec_make_block_read_query(struct asus_wmi_ec_info *ec) -{ - u8 registers[ASUSWMI_BREC_REGISTERS_MAX * 2]; - const struct ec_sensor_info *si; - int i, j, offset; - - offset = 0; - for (i = 0; i < ec->nr_sensors; i++) { - si = &known_ec_sensors[ec->sensors[i].info_index]; - for (j = 0; j < si->addr.size; j++) { - registers[offset++] = si->addr.bank; - registers[offset++] = si->addr.index + j; - } - } - - asus_wmi_ec_encode_registers(registers, offset, ec->read_arg); -} - -static int asus_wmi_ec_block_read(u32 method_id, char *query, u8 *out) -{ - struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; - struct acpi_buffer input; - union acpi_object *obj; - acpi_status status; - int ret; - - /* The first byte of the BRxx() argument string has to be the string size. */ - input.length = query[0] + 2; - input.pointer = query; - status = wmi_evaluate_method(ASUSWMI_MONITORING_GUID, 0, method_id, &input, &output); - if (ACPI_FAILURE(status)) - return -EIO; - - obj = output.pointer; - if (!obj) - return -EIO; - - if (obj->type != ACPI_TYPE_BUFFER || obj->buffer.length < 2) { - ret = -EIO; - goto out_free_obj; - } - - ret = asus_wmi_ec_decode_reply_buffer(obj->buffer.pointer, obj->buffer.length, out); - -out_free_obj: - ACPI_FREE(obj); - return ret; -} - -static inline long get_sensor_value(const struct ec_sensor_info *si, u8 *data) -{ - switch (si->addr.size) { - case 1: - return *data; - case 2: - return get_unaligned_be16(data); - case 4: - return get_unaligned_be32(data); - default: - return 0; - } -} - -static void asus_wmi_ec_update_ec_sensors(struct asus_wmi_ec_info *ec) -{ - const struct ec_sensor_info *si; - struct ec_sensor *s; - u8 i_sensor; - u8 *data; - - data = ec->read_buffer; - for (i_sensor = 0; i_sensor < ec->nr_sensors; i_sensor++) { - s = &ec->sensors[i_sensor]; - si = &known_ec_sensors[s->info_index]; - s->cached_value = get_sensor_value(si, data); - data += si->addr.size; - } -} - -static long asus_wmi_ec_scale_sensor_value(long value, int data_type) -{ - switch (data_type) { - case hwmon_curr: - case hwmon_temp: - case hwmon_in: - return value * MILLI; - default: - return value; - } -} - -static int asus_wmi_ec_find_sensor_index(const struct asus_wmi_ec_info *ec, - enum hwmon_sensor_types type, int channel) -{ - int i; - - for (i = 0; i < ec->nr_sensors; i++) { - if (known_ec_sensors[ec->sensors[i].info_index].type == type) { - if (channel == 0) - return i; - - channel--; - } - } - return -EINVAL; -} - -static int asus_wmi_ec_get_cached_value_or_update(struct asus_wmi_sensors *sensor_data, - int sensor_index, - long *value) -{ - struct asus_wmi_ec_info *ec = &sensor_data->ec; - int ret = 0; - - mutex_lock(&sensor_data->lock); - - if (time_after(jiffies, ec->last_updated + HZ)) { - ret = asus_wmi_ec_block_read(ASUSWMI_METHODID_BLOCK_READ_EC, - ec->read_arg, ec->read_buffer); - if (ret) - goto unlock; - - asus_wmi_ec_update_ec_sensors(ec); - ec->last_updated = jiffies; - } - - *value = ec->sensors[sensor_index].cached_value; - -unlock: - mutex_unlock(&sensor_data->lock); - - return ret; -} - -/* Now follow the functions that implement the hwmon interface */ - -static int asus_wmi_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type, - u32 attr, int channel, long *val) -{ - struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev); - struct asus_wmi_ec_info *ec = &sensor_data->ec; - int ret, sidx, info_index; - long value = 0; - - sidx = asus_wmi_ec_find_sensor_index(ec, type, channel); - if (sidx < 0) - return sidx; - - ret = asus_wmi_ec_get_cached_value_or_update(sensor_data, sidx, &value); - if (ret) - return ret; - - info_index = ec->sensors[sidx].info_index; - *val = asus_wmi_ec_scale_sensor_value(value, known_ec_sensors[info_index].type); - - return ret; -} - -static int asus_wmi_ec_hwmon_read_string(struct device *dev, - enum hwmon_sensor_types type, u32 attr, - int channel, const char **str) -{ - struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev); - struct asus_wmi_ec_info *ec = &sensor_data->ec; - int sensor_index; - - sensor_index = asus_wmi_ec_find_sensor_index(ec, type, channel); - *str = known_ec_sensors[ec->sensors[sensor_index].info_index].label; - - return 0; -} - -static umode_t asus_wmi_ec_hwmon_is_visible(const void *drvdata, - enum hwmon_sensor_types type, u32 attr, - int channel) -{ - const struct asus_wmi_sensors *sensor_data = drvdata; - const struct asus_wmi_ec_info *ec = &sensor_data->ec; - int index; - - index = asus_wmi_ec_find_sensor_index(ec, type, channel); - - return index < 0 ? 0 : 0444; -} - -static int asus_wmi_hwmon_add_chan_info(struct hwmon_channel_info *asus_wmi_hwmon_chan, - struct device *dev, int num, - enum hwmon_sensor_types type, u32 config) -{ - u32 *cfg; - - cfg = devm_kcalloc(dev, num + 1, sizeof(*cfg), GFP_KERNEL); - if (!cfg) - return -ENOMEM; - - asus_wmi_hwmon_chan->type = type; - asus_wmi_hwmon_chan->config = cfg; - memset32(cfg, config, num); - - return 0; -} - -static const struct hwmon_ops asus_wmi_ec_hwmon_ops = { - .is_visible = asus_wmi_ec_hwmon_is_visible, - .read = asus_wmi_ec_hwmon_read, - .read_string = asus_wmi_ec_hwmon_read_string, -}; - -static struct hwmon_chip_info asus_wmi_ec_chip_info = { - .ops = &asus_wmi_ec_hwmon_ops, -}; - -static int asus_wmi_ec_configure_sensor_setup(struct device *dev, - const enum known_ec_sensor *bsi) -{ - struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev); - struct asus_wmi_ec_info *ec = &sensor_data->ec; - struct hwmon_channel_info *asus_wmi_hwmon_chan; - const struct hwmon_channel_info **asus_wmi_ci; - int nr_count[hwmon_max] = {}, nr_types = 0; - const struct hwmon_chip_info *chip_info; - const struct ec_sensor_info *si; - enum hwmon_sensor_types type; - struct device *hwdev; - int i, ret; - - ret = asus_wmi_ec_fill_board_sensors(ec, bsi); - if (ret) - return ret; - - if (!sensor_data->ec.nr_sensors) - return -ENODEV; - - for (i = 0; i < ec->nr_sensors; i++) { - si = &known_ec_sensors[ec->sensors[i].info_index]; - if (!nr_count[si->type]) - nr_types++; - nr_count[si->type]++; - } - - if (nr_count[hwmon_temp]) { - nr_count[hwmon_chip]++; - nr_types++; - } - - /* - * If we can get values for all the registers in a single query, - * the query will not change from call to call. - */ - asus_wmi_ec_make_block_read_query(ec); - - asus_wmi_hwmon_chan = devm_kcalloc(dev, nr_types, sizeof(*asus_wmi_hwmon_chan), - GFP_KERNEL); - if (!asus_wmi_hwmon_chan) - return -ENOMEM; - - asus_wmi_ci = devm_kcalloc(dev, nr_types + 1, sizeof(*asus_wmi_ci), GFP_KERNEL); - if (!asus_wmi_ci) - return -ENOMEM; - - asus_wmi_ec_chip_info.info = asus_wmi_ci; - chip_info = &asus_wmi_ec_chip_info; - - for (type = 0; type < hwmon_max; type++) { - if (!nr_count[type]) - continue; - - ret = asus_wmi_hwmon_add_chan_info(asus_wmi_hwmon_chan, dev, - nr_count[type], type, - hwmon_attributes[type]); - if (ret) - return ret; - - *asus_wmi_ci++ = asus_wmi_hwmon_chan++; - } - - dev_dbg(dev, "board has %d EC sensors that span %d registers", - ec->nr_sensors, ec->nr_registers); - - hwdev = devm_hwmon_device_register_with_info(dev, "asus_wmi_ec_sensors", - sensor_data, chip_info, NULL); - - return PTR_ERR_OR_ZERO(hwdev); -} - -static int asus_wmi_probe(struct wmi_device *wdev, const void *context) -{ - struct asus_wmi_sensors *sensor_data; - struct asus_wmi_data *board_sensors; - const struct dmi_system_id *dmi_id; - const enum known_ec_sensor *bsi; - struct device *dev = &wdev->dev; - - dmi_id = dmi_first_match(asus_wmi_ec_dmi_table); - if (!dmi_id) - return -ENODEV; - - board_sensors = dmi_id->driver_data; - bsi = board_sensors->known_board_sensors; - - sensor_data = devm_kzalloc(dev, sizeof(*sensor_data), GFP_KERNEL); - if (!sensor_data) - return -ENOMEM; - - mutex_init(&sensor_data->lock); - - dev_set_drvdata(dev, sensor_data); - - return asus_wmi_ec_configure_sensor_setup(dev, bsi); -} - -static const struct wmi_device_id asus_ec_wmi_id_table[] = { - { ASUSWMI_MONITORING_GUID, NULL }, - { } -}; - -static struct wmi_driver asus_sensors_wmi_driver = { - .driver = { - .name = "asus_wmi_ec_sensors", - }, - .id_table = asus_ec_wmi_id_table, - .probe = asus_wmi_probe, -}; -module_wmi_driver(asus_sensors_wmi_driver); - -MODULE_AUTHOR("Ed Brindley "); -MODULE_AUTHOR("Eugene Shalygin "); -MODULE_DESCRIPTION("Asus WMI Sensors Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c index 96c4a5c45291fc7efbf0a8b8acdc94bb7902225e..6724e0dd30880979887dd367b33a59cc603204e9 100644 --- a/drivers/hwmon/axi-fan-control.c +++ b/drivers/hwmon/axi-fan-control.c @@ -394,11 +394,6 @@ static int axi_fan_control_init(struct axi_fan_control_data *ctl, return ret; } -static void axi_fan_control_clk_disable(void *clk) -{ - clk_disable_unprepare(clk); -} - static const struct hwmon_channel_info *axi_fan_control_info[] = { HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT), HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_LABEL), @@ -478,20 +473,12 @@ static int axi_fan_control_probe(struct platform_device *pdev) if (IS_ERR(ctl->base)) return PTR_ERR(ctl->base); - clk = devm_clk_get(&pdev->dev, NULL); + clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(clk)) { dev_err(&pdev->dev, "clk_get failed with %ld\n", PTR_ERR(clk)); return PTR_ERR(clk); } - ret = clk_prepare_enable(clk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(&pdev->dev, axi_fan_control_clk_disable, clk); - if (ret) - return ret; - ctl->clk_rate = clk_get_rate(clk); if (!ctl->clk_rate) return -EINVAL; diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c index 14389fd7afb896a9a4032d640b3418e810880d9e..345d883ab04428d6922dcffc395656de5f0550f4 100644 --- a/drivers/hwmon/corsair-psu.c +++ b/drivers/hwmon/corsair-psu.c @@ -55,6 +55,7 @@ #define SECONDS_PER_DAY (SECONDS_PER_HOUR * 24) #define RAIL_COUNT 3 /* 3v3 + 5v + 12v */ #define TEMP_COUNT 2 +#define OCP_MULTI_RAIL 0x02 #define PSU_CMD_SELECT_RAIL 0x00 /* expects length 2 */ #define PSU_CMD_RAIL_VOLTS_HCRIT 0x40 /* the rest of the commands expect length 3 */ @@ -71,9 +72,10 @@ #define PSU_CMD_RAIL_WATTS 0x96 #define PSU_CMD_VEND_STR 0x99 #define PSU_CMD_PROD_STR 0x9A -#define PSU_CMD_TOTAL_WATTS 0xEE #define PSU_CMD_TOTAL_UPTIME 0xD1 #define PSU_CMD_UPTIME 0xD2 +#define PSU_CMD_OCPMODE 0xD8 +#define PSU_CMD_TOTAL_WATTS 0xEE #define PSU_CMD_INIT 0xFE #define L_IN_VOLTS "v_in" @@ -268,6 +270,7 @@ static int corsairpsu_get_value(struct corsairpsu_data *priv, u8 cmd, u8 rail, l break; case PSU_CMD_TOTAL_UPTIME: case PSU_CMD_UPTIME: + case PSU_CMD_OCPMODE: *val = tmp; break; default: @@ -660,6 +663,29 @@ static int product_show(struct seq_file *seqf, void *unused) } DEFINE_SHOW_ATTRIBUTE(product); +static int ocpmode_show(struct seq_file *seqf, void *unused) +{ + struct corsairpsu_data *priv = seqf->private; + long val; + int ret; + + /* + * The rail mode is switchable on the fly. The RAW interface can be used for this. But it + * will not be included here, because I consider it somewhat dangerous for the health of the + * PSU. The returned value can be a bogus one, if the PSU is in the process of switching and + * getting of the value itself can also fail during this. Because of this every other value + * than OCP_MULTI_RAIL can be considered as "single rail". + */ + ret = corsairpsu_get_value(priv, PSU_CMD_OCPMODE, 0, &val); + if (ret < 0) + seq_puts(seqf, "N/A\n"); + else + seq_printf(seqf, "%s\n", (val == OCP_MULTI_RAIL) ? "multi rail" : "single rail"); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ocpmode); + static void corsairpsu_debugfs_init(struct corsairpsu_data *priv) { char name[32]; @@ -671,6 +697,7 @@ static void corsairpsu_debugfs_init(struct corsairpsu_data *priv) debugfs_create_file("uptime_total", 0444, priv->debugfs, priv, &uptime_total_fops); debugfs_create_file("vendor", 0444, priv->debugfs, priv, &vendor_fops); debugfs_create_file("product", 0444, priv->debugfs, priv, &product_fops); + debugfs_create_file("ocpmode", 0444, priv->debugfs, priv, &ocpmode_fops); } #else @@ -786,13 +813,14 @@ static const struct hid_device_id corsairpsu_idtable[] = { { HID_USB_DEVICE(0x1b1c, 0x1c04) }, /* Corsair HX650i */ { HID_USB_DEVICE(0x1b1c, 0x1c05) }, /* Corsair HX750i */ { HID_USB_DEVICE(0x1b1c, 0x1c06) }, /* Corsair HX850i */ - { HID_USB_DEVICE(0x1b1c, 0x1c07) }, /* Corsair HX1000i */ + { HID_USB_DEVICE(0x1b1c, 0x1c07) }, /* Corsair HX1000i revision 1 */ { HID_USB_DEVICE(0x1b1c, 0x1c08) }, /* Corsair HX1200i */ { HID_USB_DEVICE(0x1b1c, 0x1c09) }, /* Corsair RM550i */ { HID_USB_DEVICE(0x1b1c, 0x1c0a) }, /* Corsair RM650i */ { HID_USB_DEVICE(0x1b1c, 0x1c0b) }, /* Corsair RM750i */ { HID_USB_DEVICE(0x1b1c, 0x1c0c) }, /* Corsair RM850i */ { HID_USB_DEVICE(0x1b1c, 0x1c0d) }, /* Corsair RM1000i */ + { HID_USB_DEVICE(0x1b1c, 0x1c1e) }, /* Corsaur HX1000i revision 2 */ { }, }; MODULE_DEVICE_TABLE(hid, corsairpsu_idtable); diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 7f8d95dd2717642944dd226017d53b1832d96354..1572b54160158f22fbd5b34a9e1bc2ceb3d5a811 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1355,15 +1355,21 @@ static int __init dell_smm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); if (dmi_check_system(i8k_blacklist_fan_support_dmi_table)) { - dev_warn(&pdev->dev, "broken Dell BIOS detected, disallow fan support\n"); - if (!force) + if (!force) { + dev_notice(&pdev->dev, "Disabling fan support due to BIOS bugs\n"); data->disallow_fan_support = true; + } else { + dev_warn(&pdev->dev, "Enabling fan support despite BIOS bugs\n"); + } } if (dmi_check_system(i8k_blacklist_fan_type_dmi_table)) { - dev_warn(&pdev->dev, "broken Dell BIOS detected, disallow fan type call\n"); - if (!force) + if (!force) { + dev_notice(&pdev->dev, "Disabling fan type call due to BIOS bugs\n"); data->disallow_fan_type_call = true; + } else { + dev_warn(&pdev->dev, "Enabling fan type call despite BIOS bugs\n"); + } } strscpy(data->bios_version, i8k_get_dmi_data(DMI_BIOS_VERSION), diff --git a/drivers/hwmon/dme1737.c b/drivers/hwmon/dme1737.c index e3ad4c2d0038c8225ef7c54551995e496390d637..66c48f70fae73b0f3a94c23c4f749711dcb68e83 100644 --- a/drivers/hwmon/dme1737.c +++ b/drivers/hwmon/dme1737.c @@ -2456,7 +2456,7 @@ static int dme1737_i2c_detect(struct i2c_client *client, dev_info(dev, "Found a %s chip at 0x%02x (rev 0x%02x).\n", verstep == SCH5027_VERSTEP ? "SCH5027" : "DME1737", client->addr, verstep); - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } @@ -2508,14 +2508,12 @@ exit_remove: return err; } -static int dme1737_i2c_remove(struct i2c_client *client) +static void dme1737_i2c_remove(struct i2c_client *client) { struct dme1737_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); dme1737_remove_files(&client->dev); - - return 0; } static const struct i2c_device_id dme1737_id[] = { diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index 314838272049940109a9fed32f69aa9e4517795b..61d59189a6d10759fd03d1ab832adb191a84d10f 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -329,22 +329,22 @@ static int emc1403_detect(struct i2c_client *client, id = i2c_smbus_read_byte_data(client, THERMAL_PID_REG); switch (id) { case 0x20: - strlcpy(info->type, "emc1402", I2C_NAME_SIZE); + strscpy(info->type, "emc1402", I2C_NAME_SIZE); break; case 0x21: - strlcpy(info->type, "emc1403", I2C_NAME_SIZE); + strscpy(info->type, "emc1403", I2C_NAME_SIZE); break; case 0x22: - strlcpy(info->type, "emc1422", I2C_NAME_SIZE); + strscpy(info->type, "emc1422", I2C_NAME_SIZE); break; case 0x23: - strlcpy(info->type, "emc1423", I2C_NAME_SIZE); + strscpy(info->type, "emc1423", I2C_NAME_SIZE); break; case 0x25: - strlcpy(info->type, "emc1404", I2C_NAME_SIZE); + strscpy(info->type, "emc1404", I2C_NAME_SIZE); break; case 0x27: - strlcpy(info->type, "emc1424", I2C_NAME_SIZE); + strscpy(info->type, "emc1424", I2C_NAME_SIZE); break; default: return -ENODEV; diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c index e4c95ca9e19fa094c8cf5363fb2cb070d8ea0043..361cf9292456d62e3b6138c17bd5d1dfd84d1e5f 100644 --- a/drivers/hwmon/emc2103.c +++ b/drivers/hwmon/emc2103.c @@ -643,7 +643,7 @@ emc2103_detect(struct i2c_client *new_client, struct i2c_board_info *info) if ((product != 0x24) && (product != 0x26)) return -ENODEV; - strlcpy(info->type, "emc2103", I2C_NAME_SIZE); + strscpy(info->type, "emc2103", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c new file mode 100644 index 0000000000000000000000000000000000000000..aa1f25add0b6b3e175e5c318583b736c9ca0a532 --- /dev/null +++ b/drivers/hwmon/emc2305.c @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Hardware monitoring driver for EMC2305 fan controller + * + * Copyright (C) 2022 Nvidia Technologies Ltd. + */ + +#include +#include +#include +#include +#include +#include + +static const unsigned short +emc2305_normal_i2c[] = { 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d, I2C_CLIENT_END }; + +#define EMC2305_REG_DRIVE_FAIL_STATUS 0x27 +#define EMC2305_REG_DEVICE 0xfd +#define EMC2305_REG_VENDOR 0xfe +#define EMC2305_FAN_MAX 0xff +#define EMC2305_FAN_MIN 0x00 +#define EMC2305_FAN_MAX_STATE 10 +#define EMC2305_DEVICE 0x34 +#define EMC2305_VENDOR 0x5d +#define EMC2305_REG_PRODUCT_ID 0xfd +#define EMC2305_TACH_REGS_UNUSE_BITS 3 +#define EMC2305_TACH_CNT_MULTIPLIER 0x02 +#define EMC2305_TACH_RANGE_MIN 480 + +#define EMC2305_PWM_DUTY2STATE(duty, max_state, pwm_max) \ + DIV_ROUND_CLOSEST((duty) * (max_state), (pwm_max)) +#define EMC2305_PWM_STATE2DUTY(state, max_state, pwm_max) \ + DIV_ROUND_CLOSEST((state) * (pwm_max), (max_state)) + +/* + * Factor by equations [2] and [3] from data sheet; valid for fans where the number of edges + * equal (poles * 2 + 1). + */ +#define EMC2305_RPM_FACTOR 3932160 + +#define EMC2305_REG_FAN_DRIVE(n) (0x30 + 0x10 * (n)) +#define EMC2305_REG_FAN_MIN_DRIVE(n) (0x38 + 0x10 * (n)) +#define EMC2305_REG_FAN_TACH(n) (0x3e + 0x10 * (n)) + +enum emc230x_product_id { + EMC2305 = 0x34, + EMC2303 = 0x35, + EMC2302 = 0x36, + EMC2301 = 0x37, +}; + +static const struct i2c_device_id emc2305_ids[] = { + { "emc2305", 0 }, + { "emc2303", 0 }, + { "emc2302", 0 }, + { "emc2301", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, emc2305_ids); + +/** + * @cdev: cooling device; + * @curr_state: cooling current state; + * @last_hwmon_state: last cooling state updated by hwmon subsystem; + * @last_thermal_state: last cooling state updated by thermal subsystem; + * + * The 'last_hwmon_state' and 'last_thermal_state' fields are provided to support fan low limit + * speed feature. The purpose of this feature is to provides ability to limit fan speed + * according to some system wise considerations, like absence of some replaceable units (PSU or + * line cards), high system ambient temperature, unreliable transceivers temperature sensing or + * some other factors which indirectly impacts system's airflow + * Fan low limit feature is supported through 'hwmon' interface: 'hwmon' 'pwm' attribute is + * used for setting low limit for fan speed in case 'thermal' subsystem is configured in + * kernel. In this case setting fan speed through 'hwmon' will never let the 'thermal' + * subsystem to select a lower duty cycle than the duty cycle selected with the 'pwm' + * attribute. + * From other side, fan speed is to be updated in hardware through 'pwm' only in case the + * requested fan speed is above last speed set by 'thermal' subsystem, otherwise requested fan + * speed will be just stored with no PWM update. + */ +struct emc2305_cdev_data { + struct thermal_cooling_device *cdev; + unsigned int cur_state; + unsigned long last_hwmon_state; + unsigned long last_thermal_state; +}; + +/** + * @client: i2c client; + * @hwmon_dev: hwmon device; + * @max_state: maximum cooling state of the cooling device; + * @pwm_num: number of PWM channels; + * @pwm_separate: separate PWM settings for every channel; + * @pwm_min: array of minimum PWM per channel; + * @cdev_data: array of cooling devices data; + */ +struct emc2305_data { + struct i2c_client *client; + struct device *hwmon_dev; + u8 max_state; + u8 pwm_num; + bool pwm_separate; + u8 pwm_min[EMC2305_PWM_MAX]; + struct emc2305_cdev_data cdev_data[EMC2305_PWM_MAX]; +}; + +static char *emc2305_fan_name[] = { + "emc2305_fan", + "emc2305_fan1", + "emc2305_fan2", + "emc2305_fan3", + "emc2305_fan4", + "emc2305_fan5", +}; + +static void emc2305_unset_tz(struct device *dev); + +static int emc2305_get_max_channel(const struct emc2305_data *data) +{ + return data->pwm_num; +} + +static int emc2305_get_cdev_idx(struct thermal_cooling_device *cdev) +{ + struct emc2305_data *data = cdev->devdata; + size_t len = strlen(cdev->type); + int ret; + + if (len <= 0) + return -EINVAL; + + /* + * Returns index of cooling device 0..4 in case of separate PWM setting. + * Zero index is used in case of one common PWM setting. + * If the mode is not set as pwm_separate, all PWMs are to be bound + * to the common thermal zone and should work at the same speed + * to perform cooling for the same thermal junction. + * Otherwise, return specific channel that will be used in bound + * related PWM to the thermal zone. + */ + if (!data->pwm_separate) + return 0; + + ret = cdev->type[len - 1]; + switch (ret) { + case '1' ... '5': + return ret - '1'; + default: + break; + } + return -EINVAL; +} + +static int emc2305_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + int cdev_idx; + struct emc2305_data *data = cdev->devdata; + + cdev_idx = emc2305_get_cdev_idx(cdev); + if (cdev_idx < 0) + return cdev_idx; + + *state = data->cdev_data[cdev_idx].cur_state; + return 0; +} + +static int emc2305_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct emc2305_data *data = cdev->devdata; + *state = data->max_state; + return 0; +} + +static int emc2305_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) +{ + int cdev_idx, ret; + struct emc2305_data *data = cdev->devdata; + struct i2c_client *client = data->client; + u8 val, i; + + if (state > data->max_state) + return -EINVAL; + + cdev_idx = emc2305_get_cdev_idx(cdev); + if (cdev_idx < 0) + return cdev_idx; + + /* Save thermal state. */ + data->cdev_data[cdev_idx].last_thermal_state = state; + state = max_t(unsigned long, state, data->cdev_data[cdev_idx].last_hwmon_state); + + val = EMC2305_PWM_STATE2DUTY(state, data->max_state, EMC2305_FAN_MAX); + + data->cdev_data[cdev_idx].cur_state = state; + if (data->pwm_separate) { + ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(cdev_idx), val); + if (ret < 0) + return ret; + } else { + /* + * Set the same PWM value in all channels + * if common PWM channel is used. + */ + for (i = 0; i < data->pwm_num; i++) { + ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(i), val); + if (ret < 0) + return ret; + } + } + + return 0; +} + +static const struct thermal_cooling_device_ops emc2305_cooling_ops = { + .get_max_state = emc2305_get_max_state, + .get_cur_state = emc2305_get_cur_state, + .set_cur_state = emc2305_set_cur_state, +}; + +static int emc2305_show_fault(struct device *dev, int channel) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int status_reg; + + status_reg = i2c_smbus_read_byte_data(client, EMC2305_REG_DRIVE_FAIL_STATUS); + if (status_reg < 0) + return status_reg; + + return status_reg & (1 << channel) ? 1 : 0; +} + +static int emc2305_show_fan(struct device *dev, int channel) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int ret; + + ret = i2c_smbus_read_word_swapped(client, EMC2305_REG_FAN_TACH(channel)); + if (ret <= 0) + return ret; + + ret = ret >> EMC2305_TACH_REGS_UNUSE_BITS; + ret = EMC2305_RPM_FACTOR / ret; + if (ret <= EMC2305_TACH_RANGE_MIN) + return 0; + + return ret * EMC2305_TACH_CNT_MULTIPLIER; +} + +static int emc2305_show_pwm(struct device *dev, int channel) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + + return i2c_smbus_read_byte_data(client, EMC2305_REG_FAN_DRIVE(channel)); +} + +static int emc2305_set_pwm(struct device *dev, long val, int channel) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + struct i2c_client *client = data->client; + int ret; + + if (val < data->pwm_min[channel] || val > EMC2305_FAN_MAX) + return -EINVAL; + + ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(channel), val); + if (ret < 0) + return ret; + data->cdev_data[channel].cur_state = EMC2305_PWM_DUTY2STATE(val, data->max_state, + EMC2305_FAN_MAX); + return 0; +} + +static int emc2305_set_single_tz(struct device *dev, int idx) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + long pwm; + int i, cdev_idx, ret; + + cdev_idx = (idx) ? idx - 1 : 0; + pwm = data->pwm_min[cdev_idx]; + + data->cdev_data[cdev_idx].cdev = + thermal_cooling_device_register(emc2305_fan_name[idx], data, + &emc2305_cooling_ops); + + if (IS_ERR(data->cdev_data[cdev_idx].cdev)) { + dev_err(dev, "Failed to register cooling device %s\n", emc2305_fan_name[idx]); + return PTR_ERR(data->cdev_data[cdev_idx].cdev); + } + /* Set minimal PWM speed. */ + if (data->pwm_separate) { + ret = emc2305_set_pwm(dev, pwm, cdev_idx); + if (ret < 0) + return ret; + } else { + for (i = 0; i < data->pwm_num; i++) { + ret = emc2305_set_pwm(dev, pwm, i); + if (ret < 0) + return ret; + } + } + data->cdev_data[cdev_idx].cur_state = + EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state, + EMC2305_FAN_MAX); + data->cdev_data[cdev_idx].last_hwmon_state = + EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state, + EMC2305_FAN_MAX); + return 0; +} + +static int emc2305_set_tz(struct device *dev) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + int i, ret; + + if (!data->pwm_separate) + return emc2305_set_single_tz(dev, 0); + + for (i = 0; i < data->pwm_num; i++) { + ret = emc2305_set_single_tz(dev, i + 1); + if (ret) + goto thermal_cooling_device_register_fail; + } + return 0; + +thermal_cooling_device_register_fail: + emc2305_unset_tz(dev); + return ret; +} + +static void emc2305_unset_tz(struct device *dev) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + int i; + + /* Unregister cooling device. */ + for (i = 0; i < EMC2305_PWM_MAX; i++) + if (data->cdev_data[i].cdev) + thermal_cooling_device_unregister(data->cdev_data[i].cdev); +} + +static umode_t +emc2305_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel) +{ + int max_channel = emc2305_get_max_channel(data); + + /* Don't show channels which are not physically connected. */ + if (channel >= max_channel) + return 0; + switch (type) { + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + return 0444; + case hwmon_fan_fault: + return 0444; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + return 0644; + default: + break; + } + break; + default: + break; + } + + return 0; +}; + +static int +emc2305_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val) +{ + struct emc2305_data *data = dev_get_drvdata(dev); + int cdev_idx; + + switch (type) { + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + /* If thermal is configured - handle PWM limit setting. */ + if (IS_REACHABLE(CONFIG_THERMAL)) { + if (data->pwm_separate) + cdev_idx = channel; + else + cdev_idx = 0; + data->cdev_data[cdev_idx].last_hwmon_state = + EMC2305_PWM_DUTY2STATE(val, data->max_state, + EMC2305_FAN_MAX); + /* + * Update PWM only in case requested state is not less than the + * last thermal state. + */ + if (data->cdev_data[cdev_idx].last_hwmon_state >= + data->cdev_data[cdev_idx].last_thermal_state) + return emc2305_set_cur_state(data->cdev_data[cdev_idx].cdev, + data->cdev_data[cdev_idx].last_hwmon_state); + return 0; + } + return emc2305_set_pwm(dev, val, channel); + default: + break; + } + break; + default: + break; + } + + return -EOPNOTSUPP; +}; + +static int +emc2305_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) +{ + int ret; + + switch (type) { + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + ret = emc2305_show_fan(dev, channel); + if (ret < 0) + return ret; + *val = ret; + return 0; + case hwmon_fan_fault: + ret = emc2305_show_fault(dev, channel); + if (ret < 0) + return ret; + *val = ret; + return 0; + default: + break; + } + break; + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + ret = emc2305_show_pwm(dev, channel); + if (ret < 0) + return ret; + *val = ret; + return 0; + default: + break; + } + break; + default: + break; + } + + return -EOPNOTSUPP; +}; + +static const struct hwmon_ops emc2305_ops = { + .is_visible = emc2305_is_visible, + .read = emc2305_read, + .write = emc2305_write, +}; + +static const struct hwmon_channel_info *emc2305_info[] = { + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_FAULT, + HWMON_F_INPUT | HWMON_F_FAULT, + HWMON_F_INPUT | HWMON_F_FAULT, + HWMON_F_INPUT | HWMON_F_FAULT, + HWMON_F_INPUT | HWMON_F_FAULT), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT, + HWMON_PWM_INPUT), + NULL +}; + +static const struct hwmon_chip_info emc2305_chip_info = { + .ops = &emc2305_ops, + .info = emc2305_info, +}; + +static int emc2305_identify(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct emc2305_data *data = i2c_get_clientdata(client); + int ret; + + ret = i2c_smbus_read_byte_data(client, EMC2305_REG_PRODUCT_ID); + if (ret < 0) + return ret; + + switch (ret) { + case EMC2305: + data->pwm_num = 5; + break; + case EMC2303: + data->pwm_num = 3; + break; + case EMC2302: + data->pwm_num = 2; + break; + case EMC2301: + data->pwm_num = 1; + break; + default: + return -ENODEV; + } + + return 0; +} + +static int emc2305_probe(struct i2c_client *client, const struct i2c_device_id *id) +{ + struct i2c_adapter *adapter = client->adapter; + struct device *dev = &client->dev; + struct emc2305_data *data; + struct emc2305_platform_data *pdata; + int vendor, device; + int ret; + int i; + + if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA)) + return -ENODEV; + + vendor = i2c_smbus_read_byte_data(client, EMC2305_REG_VENDOR); + if (vendor != EMC2305_VENDOR) + return -ENODEV; + + device = i2c_smbus_read_byte_data(client, EMC2305_REG_DEVICE); + if (device != EMC2305_DEVICE) + return -ENODEV; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + i2c_set_clientdata(client, data); + data->client = client; + + ret = emc2305_identify(dev); + if (ret) + return ret; + + pdata = dev_get_platdata(&client->dev); + if (pdata) { + if (!pdata->max_state || pdata->max_state > EMC2305_FAN_MAX_STATE) + return -EINVAL; + data->max_state = pdata->max_state; + /* + * Validate a number of active PWM channels. Note that + * configured number can be less than the actual maximum + * supported by the device. + */ + if (!pdata->pwm_num || pdata->pwm_num > EMC2305_PWM_MAX) + return -EINVAL; + data->pwm_num = pdata->pwm_num; + data->pwm_separate = pdata->pwm_separate; + for (i = 0; i < EMC2305_PWM_MAX; i++) + data->pwm_min[i] = pdata->pwm_min[i]; + } else { + data->max_state = EMC2305_FAN_MAX_STATE; + data->pwm_separate = false; + for (i = 0; i < EMC2305_PWM_MAX; i++) + data->pwm_min[i] = EMC2305_FAN_MIN; + } + + data->hwmon_dev = devm_hwmon_device_register_with_info(dev, "emc2305", data, + &emc2305_chip_info, NULL); + if (IS_ERR(data->hwmon_dev)) + return PTR_ERR(data->hwmon_dev); + + if (IS_REACHABLE(CONFIG_THERMAL)) { + ret = emc2305_set_tz(dev); + if (ret != 0) + return ret; + } + + for (i = 0; i < data->pwm_num; i++) { + ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_MIN_DRIVE(i), + data->pwm_min[i]); + if (ret < 0) + return ret; + } + + return 0; +} + +static void emc2305_remove(struct i2c_client *client) +{ + struct device *dev = &client->dev; + + if (IS_REACHABLE(CONFIG_THERMAL)) + emc2305_unset_tz(dev); +} + +static struct i2c_driver emc2305_driver = { + .class = I2C_CLASS_HWMON, + .driver = { + .name = "emc2305", + }, + .probe = emc2305_probe, + .remove = emc2305_remove, + .id_table = emc2305_ids, + .address_list = emc2305_normal_i2c, +}; + +module_i2c_driver(emc2305_driver); + +MODULE_AUTHOR("Nvidia"); +MODULE_DESCRIPTION("Microchip EMC2305 fan controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c index 29082c8463f4984fe31ef3a77c055c78a3557566..bcd93f0fe982b9e911dd8b1bea1a0bb89653e75b 100644 --- a/drivers/hwmon/emc6w201.c +++ b/drivers/hwmon/emc6w201.c @@ -439,7 +439,7 @@ static int emc6w201_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "emc6w201", I2C_NAME_SIZE); + strscpy(info->type, "emc6w201", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c index 19b6c643059a79a2954b1674ede819738ffc5fa0..70121482a6173ace70888e201676f6d9980355db 100644 --- a/drivers/hwmon/f71882fg.c +++ b/drivers/hwmon/f71882fg.c @@ -237,13 +237,6 @@ static const char f71882fg_nr_temps[] = { static struct platform_device *f71882fg_pdev; -/* Super-I/O Function prototypes */ -static inline int superio_inb(int base, int reg); -static inline int superio_inw(int base, int reg); -static inline int superio_enter(int base); -static inline void superio_select(int base, int ld); -static inline void superio_exit(int base); - struct f71882fg_sio_data { enum chips type; }; @@ -292,968 +285,116 @@ struct f71882fg_data { s8 pwm_auto_point_temp[4][4]; }; -/* Sysfs in */ -static ssize_t show_in(struct device *dev, struct device_attribute *devattr, - char *buf); -static ssize_t show_in_max(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_in_max(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_in_beep(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_in_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_in_alarm(struct device *dev, struct device_attribute - *devattr, char *buf); -/* Sysfs Fan */ -static ssize_t show_fan(struct device *dev, struct device_attribute *devattr, - char *buf); -static ssize_t show_fan_full_speed(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_fan_full_speed(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_fan_beep(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_fan_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_fan_alarm(struct device *dev, struct device_attribute - *devattr, char *buf); -/* Sysfs Temp */ -static ssize_t show_temp(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t show_temp_max(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_temp_max(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_temp_max_hyst(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_temp_max_hyst(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_temp_crit(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_temp_crit(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_temp_crit_hyst(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t show_temp_type(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t show_temp_beep(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t store_temp_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count); -static ssize_t show_temp_alarm(struct device *dev, struct device_attribute - *devattr, char *buf); -static ssize_t show_temp_fault(struct device *dev, struct device_attribute - *devattr, char *buf); -/* PWM and Auto point control */ -static ssize_t show_pwm(struct device *dev, struct device_attribute *devattr, - char *buf); -static ssize_t store_pwm(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count); -static ssize_t show_simple_pwm(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_simple_pwm(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_enable(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_enable(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_interpolate(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_interpolate(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_auto_point_channel(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_auto_point_channel(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_auto_point_pwm(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_auto_point_pwm(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -static ssize_t show_pwm_auto_point_temp(struct device *dev, - struct device_attribute *devattr, char *buf); -static ssize_t store_pwm_auto_point_temp(struct device *dev, - struct device_attribute *devattr, const char *buf, size_t count); -/* Sysfs misc */ -static ssize_t name_show(struct device *dev, struct device_attribute *devattr, - char *buf); +static u8 f71882fg_read8(struct f71882fg_data *data, u8 reg) +{ + u8 val; -static int f71882fg_probe(struct platform_device *pdev); -static int f71882fg_remove(struct platform_device *pdev); + outb(reg, data->addr + ADDR_REG_OFFSET); + val = inb(data->addr + DATA_REG_OFFSET); -static struct platform_driver f71882fg_driver = { - .driver = { - .name = DRVNAME, - }, - .probe = f71882fg_probe, - .remove = f71882fg_remove, -}; + return val; +} -static DEVICE_ATTR_RO(name); +static u16 f71882fg_read16(struct f71882fg_data *data, u8 reg) +{ + u16 val; -/* - * Temp attr for the f71858fg, the f71858fg is special as it has its - * temperature indexes start at 0 (the others start at 1) - */ -static struct sensor_device_attribute_2 f71858fg_temp_attr[] = { - SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 0), - SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 0), - SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 0), - SENSOR_ATTR_2(temp1_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 0), - SENSOR_ATTR_2(temp1_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 0), - SENSOR_ATTR_2(temp1_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 0), - SENSOR_ATTR_2(temp1_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 4), - SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 0), - SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 1), - SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 1), - SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 1), - SENSOR_ATTR_2(temp2_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 1), - SENSOR_ATTR_2(temp2_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 1), - SENSOR_ATTR_2(temp2_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 1), - SENSOR_ATTR_2(temp2_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), - SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), - SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 2), - SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 2), - SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 2), - SENSOR_ATTR_2(temp3_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 2), - SENSOR_ATTR_2(temp3_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 2), - SENSOR_ATTR_2(temp3_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 2), - SENSOR_ATTR_2(temp3_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), - SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), -}; + val = f71882fg_read8(data, reg) << 8; + val |= f71882fg_read8(data, reg + 1); -/* Temp attr for the standard models */ -static struct sensor_device_attribute_2 fxxxx_temp_attr[3][9] = { { - SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 1), - SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 1), - SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 1), - /* - * Should really be temp1_max_alarm, but older versions did not handle - * the max and crit alarms separately and lm_sensors v2 depends on the - * presence of temp#_alarm files. The same goes for temp2/3 _alarm. - */ - SENSOR_ATTR_2(temp1_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 1), - SENSOR_ATTR_2(temp1_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 1), - SENSOR_ATTR_2(temp1_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 1), - SENSOR_ATTR_2(temp1_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), - SENSOR_ATTR_2(temp1_type, S_IRUGO, show_temp_type, NULL, 0, 1), - SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), -}, { - SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 2), - SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 2), - SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 2), - /* Should be temp2_max_alarm, see temp1_alarm note */ - SENSOR_ATTR_2(temp2_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 2), - SENSOR_ATTR_2(temp2_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 2), - SENSOR_ATTR_2(temp2_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 2), - SENSOR_ATTR_2(temp2_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), - SENSOR_ATTR_2(temp2_type, S_IRUGO, show_temp_type, NULL, 0, 2), - SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), -}, { - SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 3), - SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 3), - SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, - store_temp_max_hyst, 0, 3), - /* Should be temp3_max_alarm, see temp1_alarm note */ - SENSOR_ATTR_2(temp3_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 3), - SENSOR_ATTR_2(temp3_crit, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 3), - SENSOR_ATTR_2(temp3_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, - 0, 3), - SENSOR_ATTR_2(temp3_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 7), - SENSOR_ATTR_2(temp3_type, S_IRUGO, show_temp_type, NULL, 0, 3), - SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 3), -} }; + return val; +} -/* Temp attr for models which can beep on temp alarm */ -static struct sensor_device_attribute_2 fxxxx_temp_beep_attr[3][2] = { { - SENSOR_ATTR_2(temp1_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 1), - SENSOR_ATTR_2(temp1_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 5), -}, { - SENSOR_ATTR_2(temp2_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 2), - SENSOR_ATTR_2(temp2_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 6), -}, { - SENSOR_ATTR_2(temp3_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 3), - SENSOR_ATTR_2(temp3_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 7), -} }; +static inline int fan_from_reg(u16 reg) +{ + return reg ? (1500000 / reg) : 0; +} -static struct sensor_device_attribute_2 f81866_temp_beep_attr[3][2] = { { - SENSOR_ATTR_2(temp1_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 0), - SENSOR_ATTR_2(temp1_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 4), -}, { - SENSOR_ATTR_2(temp2_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 1), - SENSOR_ATTR_2(temp2_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 5), -}, { - SENSOR_ATTR_2(temp3_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 2), - SENSOR_ATTR_2(temp3_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, - store_temp_beep, 0, 6), -} }; +static inline u16 fan_to_reg(int fan) +{ + return fan ? (1500000 / fan) : 0; +} -/* - * Temp attr for the f8000 - * Note on the f8000 temp_ovt (crit) is used as max, and temp_high (max) - * is used as hysteresis value to clear alarms - * Also like the f71858fg its temperature indexes start at 0 - */ -static struct sensor_device_attribute_2 f8000_temp_attr[] = { - SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 0), - SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 0), - SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 0), - SENSOR_ATTR_2(temp1_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 4), - SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 0), - SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 1), - SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 1), - SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 1), - SENSOR_ATTR_2(temp2_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), - SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), - SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 2), - SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_crit, - store_temp_crit, 0, 2), - SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, - store_temp_max, 0, 2), - SENSOR_ATTR_2(temp3_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), - SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), -}; +static void f71882fg_write8(struct f71882fg_data *data, u8 reg, u8 val) +{ + outb(reg, data->addr + ADDR_REG_OFFSET); + outb(val, data->addr + DATA_REG_OFFSET); +} -/* in attr for all models */ -static struct sensor_device_attribute_2 fxxxx_in_attr[] = { - SENSOR_ATTR_2(in0_input, S_IRUGO, show_in, NULL, 0, 0), - SENSOR_ATTR_2(in1_input, S_IRUGO, show_in, NULL, 0, 1), - SENSOR_ATTR_2(in2_input, S_IRUGO, show_in, NULL, 0, 2), - SENSOR_ATTR_2(in3_input, S_IRUGO, show_in, NULL, 0, 3), - SENSOR_ATTR_2(in4_input, S_IRUGO, show_in, NULL, 0, 4), - SENSOR_ATTR_2(in5_input, S_IRUGO, show_in, NULL, 0, 5), - SENSOR_ATTR_2(in6_input, S_IRUGO, show_in, NULL, 0, 6), - SENSOR_ATTR_2(in7_input, S_IRUGO, show_in, NULL, 0, 7), - SENSOR_ATTR_2(in8_input, S_IRUGO, show_in, NULL, 0, 8), - SENSOR_ATTR_2(in9_input, S_IRUGO, show_in, NULL, 0, 9), - SENSOR_ATTR_2(in10_input, S_IRUGO, show_in, NULL, 0, 10), -}; +static void f71882fg_write16(struct f71882fg_data *data, u8 reg, u16 val) +{ + f71882fg_write8(data, reg, val >> 8); + f71882fg_write8(data, reg + 1, val & 0xff); +} -/* For models with in1 alarm capability */ -static struct sensor_device_attribute_2 fxxxx_in1_alarm_attr[] = { - SENSOR_ATTR_2(in1_max, S_IRUGO|S_IWUSR, show_in_max, store_in_max, - 0, 1), - SENSOR_ATTR_2(in1_beep, S_IRUGO|S_IWUSR, show_in_beep, store_in_beep, - 0, 1), - SENSOR_ATTR_2(in1_alarm, S_IRUGO, show_in_alarm, NULL, 0, 1), -}; +static u16 f71882fg_read_temp(struct f71882fg_data *data, int nr) +{ + if (data->type == f71858fg) + return f71882fg_read16(data, F71882FG_REG_TEMP(nr)); + else + return f71882fg_read8(data, F71882FG_REG_TEMP(nr)); +} -/* Fan / PWM attr common to all models */ -static struct sensor_device_attribute_2 fxxxx_fan_attr[4][6] = { { - SENSOR_ATTR_2(fan1_input, S_IRUGO, show_fan, NULL, 0, 0), - SENSOR_ATTR_2(fan1_full_speed, S_IRUGO|S_IWUSR, - show_fan_full_speed, - store_fan_full_speed, 0, 0), - SENSOR_ATTR_2(fan1_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 0), - SENSOR_ATTR_2(pwm1, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 0), - SENSOR_ATTR_2(pwm1_enable, S_IRUGO|S_IWUSR, show_pwm_enable, - store_pwm_enable, 0, 0), - SENSOR_ATTR_2(pwm1_interpolate, S_IRUGO|S_IWUSR, - show_pwm_interpolate, store_pwm_interpolate, 0, 0), -}, { - SENSOR_ATTR_2(fan2_input, S_IRUGO, show_fan, NULL, 0, 1), - SENSOR_ATTR_2(fan2_full_speed, S_IRUGO|S_IWUSR, - show_fan_full_speed, - store_fan_full_speed, 0, 1), - SENSOR_ATTR_2(fan2_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 1), - SENSOR_ATTR_2(pwm2, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 1), - SENSOR_ATTR_2(pwm2_enable, S_IRUGO|S_IWUSR, show_pwm_enable, - store_pwm_enable, 0, 1), - SENSOR_ATTR_2(pwm2_interpolate, S_IRUGO|S_IWUSR, - show_pwm_interpolate, store_pwm_interpolate, 0, 1), -}, { - SENSOR_ATTR_2(fan3_input, S_IRUGO, show_fan, NULL, 0, 2), - SENSOR_ATTR_2(fan3_full_speed, S_IRUGO|S_IWUSR, - show_fan_full_speed, - store_fan_full_speed, 0, 2), - SENSOR_ATTR_2(fan3_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 2), - SENSOR_ATTR_2(pwm3, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 2), - SENSOR_ATTR_2(pwm3_enable, S_IRUGO|S_IWUSR, show_pwm_enable, - store_pwm_enable, 0, 2), - SENSOR_ATTR_2(pwm3_interpolate, S_IRUGO|S_IWUSR, - show_pwm_interpolate, store_pwm_interpolate, 0, 2), -}, { - SENSOR_ATTR_2(fan4_input, S_IRUGO, show_fan, NULL, 0, 3), - SENSOR_ATTR_2(fan4_full_speed, S_IRUGO|S_IWUSR, - show_fan_full_speed, - store_fan_full_speed, 0, 3), - SENSOR_ATTR_2(fan4_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 3), - SENSOR_ATTR_2(pwm4, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 3), - SENSOR_ATTR_2(pwm4_enable, S_IRUGO|S_IWUSR, show_pwm_enable, - store_pwm_enable, 0, 3), - SENSOR_ATTR_2(pwm4_interpolate, S_IRUGO|S_IWUSR, - show_pwm_interpolate, store_pwm_interpolate, 0, 3), -} }; +static struct f71882fg_data *f71882fg_update_device(struct device *dev) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int nr_fans = f71882fg_nr_fans[data->type]; + int nr_temps = f71882fg_nr_temps[data->type]; + int nr, reg, point; -/* Attr for the third fan of the f71808a, which only has manual pwm */ -static struct sensor_device_attribute_2 f71808a_fan3_attr[] = { - SENSOR_ATTR_2(fan3_input, S_IRUGO, show_fan, NULL, 0, 2), - SENSOR_ATTR_2(fan3_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 2), - SENSOR_ATTR_2(pwm3, S_IRUGO|S_IWUSR, - show_simple_pwm, store_simple_pwm, 0, 2), -}; + mutex_lock(&data->update_lock); -/* Attr for models which can beep on Fan alarm */ -static struct sensor_device_attribute_2 fxxxx_fan_beep_attr[] = { - SENSOR_ATTR_2(fan1_beep, S_IRUGO|S_IWUSR, show_fan_beep, - store_fan_beep, 0, 0), - SENSOR_ATTR_2(fan2_beep, S_IRUGO|S_IWUSR, show_fan_beep, - store_fan_beep, 0, 1), - SENSOR_ATTR_2(fan3_beep, S_IRUGO|S_IWUSR, show_fan_beep, - store_fan_beep, 0, 2), - SENSOR_ATTR_2(fan4_beep, S_IRUGO|S_IWUSR, show_fan_beep, - store_fan_beep, 0, 3), -}; + /* Update once every 60 seconds */ + if (time_after(jiffies, data->last_limits + 60 * HZ) || + !data->valid) { + if (f71882fg_has_in1_alarm[data->type]) { + if (data->type == f81866a) { + data->in1_max = + f71882fg_read8(data, + F81866_REG_IN1_HIGH); + data->in_beep = + f71882fg_read8(data, + F81866_REG_IN_BEEP); + } else { + data->in1_max = + f71882fg_read8(data, + F71882FG_REG_IN1_HIGH); + data->in_beep = + f71882fg_read8(data, + F71882FG_REG_IN_BEEP); + } + } -/* - * PWM attr for the f71862fg, fewer pwms and fewer zones per pwm than the - * standard models - */ -static struct sensor_device_attribute_2 f71862fg_auto_pwm_attr[3][7] = { { - SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 0), - SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 0), - SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 0), -}, { - SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 1), - SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 1), - SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 1), -}, { - SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 2), - SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 2), - SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 2), -} }; + /* Get High & boundary temps*/ + for (nr = data->temp_start; nr < nr_temps + data->temp_start; + nr++) { + data->temp_ovt[nr] = f71882fg_read8(data, + F71882FG_REG_TEMP_OVT(nr)); + data->temp_high[nr] = f71882fg_read8(data, + F71882FG_REG_TEMP_HIGH(nr)); + } -/* - * PWM attr for the f71808e/f71869, almost identical to the f71862fg, but the - * pwm setting when the temperature is above the pwmX_auto_point1_temp can be - * programmed instead of being hardcoded to 0xff - */ -static struct sensor_device_attribute_2 f71869_auto_pwm_attr[3][8] = { { - SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 0), - SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 0), - SENSOR_ATTR_2(pwm1_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 0), -}, { - SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 1), - SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 1), - SENSOR_ATTR_2(pwm2_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 1), -}, { - SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 2), - SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 2), - SENSOR_ATTR_2(pwm3_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 2), -} }; + if (data->type != f8000) { + data->temp_hyst[0] = f71882fg_read8(data, + F71882FG_REG_TEMP_HYST(0)); + data->temp_hyst[1] = f71882fg_read8(data, + F71882FG_REG_TEMP_HYST(1)); + } + /* All but the f71858fg / f8000 have this register */ + if ((data->type != f71858fg) && (data->type != f8000)) { + reg = f71882fg_read8(data, F71882FG_REG_TEMP_TYPE); + data->temp_type[1] = (reg & 0x02) ? 2 : 4; + data->temp_type[2] = (reg & 0x04) ? 2 : 4; + data->temp_type[3] = (reg & 0x08) ? 2 : 4; + } -/* PWM attr for the standard models */ -static struct sensor_device_attribute_2 fxxxx_auto_pwm_attr[4][14] = { { - SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 0), - SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 0), - SENSOR_ATTR_2(pwm1_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 0), - SENSOR_ATTR_2(pwm1_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 0), - SENSOR_ATTR_2(pwm1_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 0), - SENSOR_ATTR_2(pwm1_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 0), - SENSOR_ATTR_2(pwm1_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 0), - SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 0), - SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 0), - SENSOR_ATTR_2(pwm1_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 0), - SENSOR_ATTR_2(pwm1_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 0), -}, { - SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 1), - SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 1), - SENSOR_ATTR_2(pwm2_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 1), - SENSOR_ATTR_2(pwm2_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 1), - SENSOR_ATTR_2(pwm2_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 1), - SENSOR_ATTR_2(pwm2_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 1), - SENSOR_ATTR_2(pwm2_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 1), - SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 1), - SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 1), - SENSOR_ATTR_2(pwm2_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 1), - SENSOR_ATTR_2(pwm2_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 1), -}, { - SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 2), - SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 2), - SENSOR_ATTR_2(pwm3_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 2), - SENSOR_ATTR_2(pwm3_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 2), - SENSOR_ATTR_2(pwm3_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 2), - SENSOR_ATTR_2(pwm3_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 2), - SENSOR_ATTR_2(pwm3_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 2), - SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 2), - SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 2), - SENSOR_ATTR_2(pwm3_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 2), - SENSOR_ATTR_2(pwm3_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 2), -}, { - SENSOR_ATTR_2(pwm4_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 3), - SENSOR_ATTR_2(pwm4_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 3), - SENSOR_ATTR_2(pwm4_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 3), - SENSOR_ATTR_2(pwm4_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 3), - SENSOR_ATTR_2(pwm4_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 3), - SENSOR_ATTR_2(pwm4_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 3), - SENSOR_ATTR_2(pwm4_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 3), - SENSOR_ATTR_2(pwm4_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 3), - SENSOR_ATTR_2(pwm4_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 3), - SENSOR_ATTR_2(pwm4_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 3), - SENSOR_ATTR_2(pwm4_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 3), - SENSOR_ATTR_2(pwm4_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 3), - SENSOR_ATTR_2(pwm4_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 3), - SENSOR_ATTR_2(pwm4_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 3), -} }; + if (f71882fg_fan_has_beep[data->type]) + data->fan_beep = f71882fg_read8(data, + F71882FG_REG_FAN_BEEP); -/* Fan attr specific to the f8000 (4th fan input can only measure speed) */ -static struct sensor_device_attribute_2 f8000_fan_attr[] = { - SENSOR_ATTR_2(fan4_input, S_IRUGO, show_fan, NULL, 0, 3), -}; - -/* - * PWM attr for the f8000, zones mapped to temp instead of to pwm! - * Also the register block at offset A0 maps to TEMP1 (so our temp2, as the - * F8000 starts counting temps at 0), B0 maps the TEMP2 and C0 maps to TEMP0 - */ -static struct sensor_device_attribute_2 f8000_auto_pwm_attr[3][14] = { { - SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 0), - SENSOR_ATTR_2(temp1_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 2), - SENSOR_ATTR_2(temp1_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 2), - SENSOR_ATTR_2(temp1_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 2), - SENSOR_ATTR_2(temp1_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 2), - SENSOR_ATTR_2(temp1_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 2), - SENSOR_ATTR_2(temp1_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 2), - SENSOR_ATTR_2(temp1_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 2), - SENSOR_ATTR_2(temp1_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 2), - SENSOR_ATTR_2(temp1_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 2), - SENSOR_ATTR_2(temp1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 2), - SENSOR_ATTR_2(temp1_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 2), - SENSOR_ATTR_2(temp1_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 2), - SENSOR_ATTR_2(temp1_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 2), -}, { - SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 1), - SENSOR_ATTR_2(temp2_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 0), - SENSOR_ATTR_2(temp2_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 0), - SENSOR_ATTR_2(temp2_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 0), - SENSOR_ATTR_2(temp2_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 0), - SENSOR_ATTR_2(temp2_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 0), - SENSOR_ATTR_2(temp2_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 0), - SENSOR_ATTR_2(temp2_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 0), - SENSOR_ATTR_2(temp2_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 0), - SENSOR_ATTR_2(temp2_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 0), - SENSOR_ATTR_2(temp2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 0), - SENSOR_ATTR_2(temp2_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 0), - SENSOR_ATTR_2(temp2_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 0), - SENSOR_ATTR_2(temp2_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 0), -}, { - SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_channel, - store_pwm_auto_point_channel, 0, 2), - SENSOR_ATTR_2(temp3_auto_point1_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 0, 1), - SENSOR_ATTR_2(temp3_auto_point2_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 1, 1), - SENSOR_ATTR_2(temp3_auto_point3_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 2, 1), - SENSOR_ATTR_2(temp3_auto_point4_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 3, 1), - SENSOR_ATTR_2(temp3_auto_point5_pwm, S_IRUGO|S_IWUSR, - show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, - 4, 1), - SENSOR_ATTR_2(temp3_auto_point1_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 0, 1), - SENSOR_ATTR_2(temp3_auto_point2_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 1, 1), - SENSOR_ATTR_2(temp3_auto_point3_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 2, 1), - SENSOR_ATTR_2(temp3_auto_point4_temp, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp, store_pwm_auto_point_temp, - 3, 1), - SENSOR_ATTR_2(temp3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, - show_pwm_auto_point_temp_hyst, - store_pwm_auto_point_temp_hyst, - 0, 1), - SENSOR_ATTR_2(temp3_auto_point2_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 1, 1), - SENSOR_ATTR_2(temp3_auto_point3_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 2, 1), - SENSOR_ATTR_2(temp3_auto_point4_temp_hyst, S_IRUGO, - show_pwm_auto_point_temp_hyst, NULL, 3, 1), -} }; - -/* Super I/O functions */ -static inline int superio_inb(int base, int reg) -{ - outb(reg, base); - return inb(base + 1); -} - -static int superio_inw(int base, int reg) -{ - int val; - val = superio_inb(base, reg) << 8; - val |= superio_inb(base, reg + 1); - return val; -} - -static inline int superio_enter(int base) -{ - /* Don't step on other drivers' I/O space by accident */ - if (!request_muxed_region(base, 2, DRVNAME)) { - pr_err("I/O address 0x%04x already in use\n", base); - return -EBUSY; - } - - /* according to the datasheet the key must be send twice! */ - outb(SIO_UNLOCK_KEY, base); - outb(SIO_UNLOCK_KEY, base); - - return 0; -} - -static inline void superio_select(int base, int ld) -{ - outb(SIO_REG_LDSEL, base); - outb(ld, base + 1); -} - -static inline void superio_exit(int base) -{ - outb(SIO_LOCK_KEY, base); - release_region(base, 2); -} - -static inline int fan_from_reg(u16 reg) -{ - return reg ? (1500000 / reg) : 0; -} - -static inline u16 fan_to_reg(int fan) -{ - return fan ? (1500000 / fan) : 0; -} - -static u8 f71882fg_read8(struct f71882fg_data *data, u8 reg) -{ - u8 val; - - outb(reg, data->addr + ADDR_REG_OFFSET); - val = inb(data->addr + DATA_REG_OFFSET); - - return val; -} - -static u16 f71882fg_read16(struct f71882fg_data *data, u8 reg) -{ - u16 val; - - val = f71882fg_read8(data, reg) << 8; - val |= f71882fg_read8(data, reg + 1); - - return val; -} - -static void f71882fg_write8(struct f71882fg_data *data, u8 reg, u8 val) -{ - outb(reg, data->addr + ADDR_REG_OFFSET); - outb(val, data->addr + DATA_REG_OFFSET); -} - -static void f71882fg_write16(struct f71882fg_data *data, u8 reg, u16 val) -{ - f71882fg_write8(data, reg, val >> 8); - f71882fg_write8(data, reg + 1, val & 0xff); -} - -static u16 f71882fg_read_temp(struct f71882fg_data *data, int nr) -{ - if (data->type == f71858fg) - return f71882fg_read16(data, F71882FG_REG_TEMP(nr)); - else - return f71882fg_read8(data, F71882FG_REG_TEMP(nr)); -} - -static struct f71882fg_data *f71882fg_update_device(struct device *dev) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int nr_fans = f71882fg_nr_fans[data->type]; - int nr_temps = f71882fg_nr_temps[data->type]; - int nr, reg, point; - - mutex_lock(&data->update_lock); - - /* Update once every 60 seconds */ - if (time_after(jiffies, data->last_limits + 60 * HZ) || - !data->valid) { - if (f71882fg_has_in1_alarm[data->type]) { - if (data->type == f81866a) { - data->in1_max = - f71882fg_read8(data, - F81866_REG_IN1_HIGH); - data->in_beep = - f71882fg_read8(data, - F81866_REG_IN_BEEP); - } else { - data->in1_max = - f71882fg_read8(data, - F71882FG_REG_IN1_HIGH); - data->in_beep = - f71882fg_read8(data, - F71882FG_REG_IN_BEEP); - } - } - - /* Get High & boundary temps*/ - for (nr = data->temp_start; nr < nr_temps + data->temp_start; - nr++) { - data->temp_ovt[nr] = f71882fg_read8(data, - F71882FG_REG_TEMP_OVT(nr)); - data->temp_high[nr] = f71882fg_read8(data, - F71882FG_REG_TEMP_HIGH(nr)); - } - - if (data->type != f8000) { - data->temp_hyst[0] = f71882fg_read8(data, - F71882FG_REG_TEMP_HYST(0)); - data->temp_hyst[1] = f71882fg_read8(data, - F71882FG_REG_TEMP_HYST(1)); - } - /* All but the f71858fg / f8000 have this register */ - if ((data->type != f71858fg) && (data->type != f8000)) { - reg = f71882fg_read8(data, F71882FG_REG_TEMP_TYPE); - data->temp_type[1] = (reg & 0x02) ? 2 : 4; - data->temp_type[2] = (reg & 0x04) ? 2 : 4; - data->temp_type[3] = (reg & 0x08) ? 2 : 4; - } - - if (f71882fg_fan_has_beep[data->type]) - data->fan_beep = f71882fg_read8(data, - F71882FG_REG_FAN_BEEP); - - if (f71882fg_temp_has_beep[data->type]) - data->temp_beep = f71882fg_read8(data, - F71882FG_REG_TEMP_BEEP); + if (f71882fg_temp_has_beep[data->type]) + data->temp_beep = f71882fg_read8(data, + F71882FG_REG_TEMP_BEEP); data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE); @@ -1369,225 +510,43 @@ static struct f71882fg_data *f71882fg_update_device(struct device *dev) return data; } -/* Sysfs Interface */ -static ssize_t show_fan(struct device *dev, struct device_attribute *devattr, +static ssize_t name_show(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + return sprintf(buf, "%s\n", f71882fg_names[data->type]); +} + +static DEVICE_ATTR_RO(name); + +static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, char *buf) { struct f71882fg_data *data = f71882fg_update_device(dev); int nr = to_sensor_dev_attr_2(devattr)->index; - int speed = fan_from_reg(data->fan[nr]); + int sign, temp; - if (speed == FAN_MIN_DETECT) - speed = 0; + if (data->type == f71858fg) { + /* TEMP_TABLE_SEL 1 or 3 ? */ + if (data->temp_config & 1) { + sign = data->temp[nr] & 0x0001; + temp = (data->temp[nr] >> 5) & 0x7ff; + } else { + sign = data->temp[nr] & 0x8000; + temp = (data->temp[nr] >> 5) & 0x3ff; + } + temp *= 125; + if (sign) + temp -= 128000; + } else { + temp = ((s8)data->temp[nr]) * 1000; + } - return sprintf(buf, "%d\n", speed); + return sprintf(buf, "%d\n", temp); } -static ssize_t show_fan_full_speed(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - int speed = fan_from_reg(data->fan_full_speed[nr]); - return sprintf(buf, "%d\n", speed); -} - -static ssize_t store_fan_full_speed(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - long val; - - err = kstrtol(buf, 10, &val); - if (err) - return err; - - val = clamp_val(val, 23, 1500000); - val = fan_to_reg(val); - - mutex_lock(&data->update_lock); - f71882fg_write16(data, F71882FG_REG_FAN_FULL_SPEED(nr), val); - data->fan_full_speed[nr] = val; - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_fan_beep(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - if (data->fan_beep & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); -} - -static ssize_t store_fan_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - unsigned long val; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - mutex_lock(&data->update_lock); - data->fan_beep = f71882fg_read8(data, F71882FG_REG_FAN_BEEP); - if (val) - data->fan_beep |= 1 << nr; - else - data->fan_beep &= ~(1 << nr); - - f71882fg_write8(data, F71882FG_REG_FAN_BEEP, data->fan_beep); - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_fan_alarm(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - if (data->fan_status & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); -} - -static ssize_t show_in(struct device *dev, struct device_attribute *devattr, - char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - return sprintf(buf, "%d\n", data->in[nr] * 8); -} - -static ssize_t show_in_max(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - - return sprintf(buf, "%d\n", data->in1_max * 8); -} - -static ssize_t store_in_max(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err; - long val; - - err = kstrtol(buf, 10, &val); - if (err) - return err; - - val /= 8; - val = clamp_val(val, 0, 255); - - mutex_lock(&data->update_lock); - if (data->type == f81866a) - f71882fg_write8(data, F81866_REG_IN1_HIGH, val); - else - f71882fg_write8(data, F71882FG_REG_IN1_HIGH, val); - data->in1_max = val; - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_in_beep(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - if (data->in_beep & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); -} - -static ssize_t store_in_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - unsigned long val; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - mutex_lock(&data->update_lock); - if (data->type == f81866a) - data->in_beep = f71882fg_read8(data, F81866_REG_IN_BEEP); - else - data->in_beep = f71882fg_read8(data, F71882FG_REG_IN_BEEP); - - if (val) - data->in_beep |= 1 << nr; - else - data->in_beep &= ~(1 << nr); - - if (data->type == f81866a) - f71882fg_write8(data, F81866_REG_IN_BEEP, data->in_beep); - else - f71882fg_write8(data, F71882FG_REG_IN_BEEP, data->in_beep); - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_in_alarm(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - if (data->in_status & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); -} - -static ssize_t show_temp(struct device *dev, struct device_attribute *devattr, - char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - int sign, temp; - - if (data->type == f71858fg) { - /* TEMP_TABLE_SEL 1 or 3 ? */ - if (data->temp_config & 1) { - sign = data->temp[nr] & 0x0001; - temp = (data->temp[nr] >> 5) & 0x7ff; - } else { - sign = data->temp[nr] & 0x8000; - temp = (data->temp[nr] >> 5) & 0x3ff; - } - temp *= 125; - if (sign) - temp -= 128000; - } else { - temp = ((s8)data->temp[nr]) * 1000; - } - - return sprintf(buf, "%d\n", temp); -} - -static ssize_t show_temp_max(struct device *dev, struct device_attribute - *devattr, char *buf) +static ssize_t show_temp_max(struct device *dev, struct device_attribute + *devattr, char *buf) { struct f71882fg_data *data = f71882fg_update_device(dev); int nr = to_sensor_dev_attr_2(devattr)->index; @@ -1670,6 +629,18 @@ static ssize_t store_temp_max_hyst(struct device *dev, struct device_attribute return ret; } +static ssize_t show_temp_alarm(struct device *dev, struct device_attribute + *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + + if (data->temp_status & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); +} + static ssize_t show_temp_crit(struct device *dev, struct device_attribute *devattr, char *buf) { @@ -1719,161 +690,455 @@ static ssize_t show_temp_crit_hyst(struct device *dev, struct device_attribute return sprintf(buf, "%d\n", temp_crit_hyst); } -static ssize_t show_temp_type(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - return sprintf(buf, "%d\n", data->temp_type[nr]); -} - -static ssize_t show_temp_beep(struct device *dev, struct device_attribute +static ssize_t show_temp_fault(struct device *dev, struct device_attribute *devattr, char *buf) { struct f71882fg_data *data = f71882fg_update_device(dev); int nr = to_sensor_dev_attr_2(devattr)->index; - if (data->temp_beep & (1 << nr)) + if (data->temp_diode_open & (1 << nr)) return sprintf(buf, "1\n"); else return sprintf(buf, "0\n"); } -static ssize_t store_temp_beep(struct device *dev, struct device_attribute - *devattr, const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - unsigned long val; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - mutex_lock(&data->update_lock); - data->temp_beep = f71882fg_read8(data, F71882FG_REG_TEMP_BEEP); - if (val) - data->temp_beep |= 1 << nr; - else - data->temp_beep &= ~(1 << nr); - - f71882fg_write8(data, F71882FG_REG_TEMP_BEEP, data->temp_beep); - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_temp_alarm(struct device *dev, struct device_attribute - *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - if (data->temp_status & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); -} +/* + * Temp attr for the f71858fg, the f71858fg is special as it has its + * temperature indexes start at 0 (the others start at 1) + */ +static struct sensor_device_attribute_2 f71858fg_temp_attr[] = { + SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 0), + SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 0), + SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 0), + SENSOR_ATTR_2(temp1_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 0), + SENSOR_ATTR_2(temp1_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 0), + SENSOR_ATTR_2(temp1_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 0), + SENSOR_ATTR_2(temp1_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 4), + SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 0), + SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 1), + SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 1), + SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 1), + SENSOR_ATTR_2(temp2_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 1), + SENSOR_ATTR_2(temp2_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 1), + SENSOR_ATTR_2(temp2_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 1), + SENSOR_ATTR_2(temp2_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), + SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), + SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 2), + SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 2), + SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 2), + SENSOR_ATTR_2(temp3_max_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 2), + SENSOR_ATTR_2(temp3_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 2), + SENSOR_ATTR_2(temp3_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 2), + SENSOR_ATTR_2(temp3_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), + SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), +}; -static ssize_t show_temp_fault(struct device *dev, struct device_attribute +static ssize_t show_temp_type(struct device *dev, struct device_attribute *devattr, char *buf) { struct f71882fg_data *data = f71882fg_update_device(dev); int nr = to_sensor_dev_attr_2(devattr)->index; - if (data->temp_diode_open & (1 << nr)) - return sprintf(buf, "1\n"); - else - return sprintf(buf, "0\n"); + return sprintf(buf, "%d\n", data->temp_type[nr]); } -static ssize_t show_pwm(struct device *dev, - struct device_attribute *devattr, char *buf) +/* Temp attr for the standard models */ +static struct sensor_device_attribute_2 fxxxx_temp_attr[3][9] = { { + SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 1), + SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 1), + SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 1), + /* + * Should really be temp1_max_alarm, but older versions did not handle + * the max and crit alarms separately and lm_sensors v2 depends on the + * presence of temp#_alarm files. The same goes for temp2/3 _alarm. + */ + SENSOR_ATTR_2(temp1_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 1), + SENSOR_ATTR_2(temp1_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 1), + SENSOR_ATTR_2(temp1_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 1), + SENSOR_ATTR_2(temp1_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), + SENSOR_ATTR_2(temp1_type, S_IRUGO, show_temp_type, NULL, 0, 1), + SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), +}, { + SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 2), + SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 2), + SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 2), + /* Should be temp2_max_alarm, see temp1_alarm note */ + SENSOR_ATTR_2(temp2_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 2), + SENSOR_ATTR_2(temp2_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 2), + SENSOR_ATTR_2(temp2_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 2), + SENSOR_ATTR_2(temp2_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), + SENSOR_ATTR_2(temp2_type, S_IRUGO, show_temp_type, NULL, 0, 2), + SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), +}, { + SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 3), + SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 3), + SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max_hyst, + store_temp_max_hyst, 0, 3), + /* Should be temp3_max_alarm, see temp1_alarm note */ + SENSOR_ATTR_2(temp3_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 3), + SENSOR_ATTR_2(temp3_crit, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 3), + SENSOR_ATTR_2(temp3_crit_hyst, S_IRUGO, show_temp_crit_hyst, NULL, + 0, 3), + SENSOR_ATTR_2(temp3_crit_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 7), + SENSOR_ATTR_2(temp3_type, S_IRUGO, show_temp_type, NULL, 0, 3), + SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 3), +} }; + +static ssize_t show_temp_beep(struct device *dev, struct device_attribute + *devattr, char *buf) { struct f71882fg_data *data = f71882fg_update_device(dev); - int val, nr = to_sensor_dev_attr_2(devattr)->index; - mutex_lock(&data->update_lock); - if (data->pwm_enable & (1 << (2 * nr))) - /* PWM mode */ - val = data->pwm[nr]; - else { - /* RPM mode */ - val = 255 * fan_from_reg(data->fan_target[nr]) - / fan_from_reg(data->fan_full_speed[nr]); - } - mutex_unlock(&data->update_lock); - return sprintf(buf, "%d\n", val); + int nr = to_sensor_dev_attr_2(devattr)->index; + + if (data->temp_beep & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); } -static ssize_t store_pwm(struct device *dev, - struct device_attribute *devattr, const char *buf, - size_t count) +static ssize_t store_temp_beep(struct device *dev, struct device_attribute + *devattr, const char *buf, size_t count) { struct f71882fg_data *data = dev_get_drvdata(dev); int err, nr = to_sensor_dev_attr_2(devattr)->index; - long val; + unsigned long val; - err = kstrtol(buf, 10, &val); + err = kstrtoul(buf, 10, &val); if (err) return err; - val = clamp_val(val, 0, 255); - mutex_lock(&data->update_lock); - data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE); - if ((data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 3) != 2) || - (data->type != f8000 && !((data->pwm_enable >> 2 * nr) & 2))) { - count = -EROFS; - goto leave; - } - if (data->pwm_enable & (1 << (2 * nr))) { - /* PWM mode */ - f71882fg_write8(data, F71882FG_REG_PWM(nr), val); - data->pwm[nr] = val; - } else { - /* RPM mode */ - int target, full_speed; - full_speed = f71882fg_read16(data, - F71882FG_REG_FAN_FULL_SPEED(nr)); - target = fan_to_reg(val * fan_from_reg(full_speed) / 255); - f71882fg_write16(data, F71882FG_REG_FAN_TARGET(nr), target); - data->fan_target[nr] = target; - data->fan_full_speed[nr] = full_speed; - } -leave: + data->temp_beep = f71882fg_read8(data, F71882FG_REG_TEMP_BEEP); + if (val) + data->temp_beep |= 1 << nr; + else + data->temp_beep &= ~(1 << nr); + + f71882fg_write8(data, F71882FG_REG_TEMP_BEEP, data->temp_beep); mutex_unlock(&data->update_lock); return count; } -static ssize_t show_simple_pwm(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct f71882fg_data *data = f71882fg_update_device(dev); - int val, nr = to_sensor_dev_attr_2(devattr)->index; +/* Temp attr for models which can beep on temp alarm */ +static struct sensor_device_attribute_2 fxxxx_temp_beep_attr[3][2] = { { + SENSOR_ATTR_2(temp1_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 1), + SENSOR_ATTR_2(temp1_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 5), +}, { + SENSOR_ATTR_2(temp2_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 2), + SENSOR_ATTR_2(temp2_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 6), +}, { + SENSOR_ATTR_2(temp3_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 3), + SENSOR_ATTR_2(temp3_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 7), +} }; - val = data->pwm[nr]; - return sprintf(buf, "%d\n", val); -} +static struct sensor_device_attribute_2 f81866_temp_beep_attr[3][2] = { { + SENSOR_ATTR_2(temp1_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 0), + SENSOR_ATTR_2(temp1_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 4), +}, { + SENSOR_ATTR_2(temp2_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 1), + SENSOR_ATTR_2(temp2_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 5), +}, { + SENSOR_ATTR_2(temp3_max_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 2), + SENSOR_ATTR_2(temp3_crit_beep, S_IRUGO|S_IWUSR, show_temp_beep, + store_temp_beep, 0, 6), +} }; -static ssize_t store_simple_pwm(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - long val; +/* + * Temp attr for the f8000 + * Note on the f8000 temp_ovt (crit) is used as max, and temp_high (max) + * is used as hysteresis value to clear alarms + * Also like the f71858fg its temperature indexes start at 0 + */ +static struct sensor_device_attribute_2 f8000_temp_attr[] = { + SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 0), + SENSOR_ATTR_2(temp1_max, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 0), + SENSOR_ATTR_2(temp1_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 0), + SENSOR_ATTR_2(temp1_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 4), + SENSOR_ATTR_2(temp1_fault, S_IRUGO, show_temp_fault, NULL, 0, 0), + SENSOR_ATTR_2(temp2_input, S_IRUGO, show_temp, NULL, 0, 1), + SENSOR_ATTR_2(temp2_max, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 1), + SENSOR_ATTR_2(temp2_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 1), + SENSOR_ATTR_2(temp2_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 5), + SENSOR_ATTR_2(temp2_fault, S_IRUGO, show_temp_fault, NULL, 0, 1), + SENSOR_ATTR_2(temp3_input, S_IRUGO, show_temp, NULL, 0, 2), + SENSOR_ATTR_2(temp3_max, S_IRUGO|S_IWUSR, show_temp_crit, + store_temp_crit, 0, 2), + SENSOR_ATTR_2(temp3_max_hyst, S_IRUGO|S_IWUSR, show_temp_max, + store_temp_max, 0, 2), + SENSOR_ATTR_2(temp3_alarm, S_IRUGO, show_temp_alarm, NULL, 0, 6), + SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2), +}; - err = kstrtol(buf, 10, &val); - if (err) - return err; +static ssize_t show_in(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; - val = clamp_val(val, 0, 255); + return sprintf(buf, "%d\n", data->in[nr] * 8); +} - mutex_lock(&data->update_lock); - f71882fg_write8(data, F71882FG_REG_PWM(nr), val); - data->pwm[nr] = val; +/* in attr for all models */ +static struct sensor_device_attribute_2 fxxxx_in_attr[] = { + SENSOR_ATTR_2(in0_input, S_IRUGO, show_in, NULL, 0, 0), + SENSOR_ATTR_2(in1_input, S_IRUGO, show_in, NULL, 0, 1), + SENSOR_ATTR_2(in2_input, S_IRUGO, show_in, NULL, 0, 2), + SENSOR_ATTR_2(in3_input, S_IRUGO, show_in, NULL, 0, 3), + SENSOR_ATTR_2(in4_input, S_IRUGO, show_in, NULL, 0, 4), + SENSOR_ATTR_2(in5_input, S_IRUGO, show_in, NULL, 0, 5), + SENSOR_ATTR_2(in6_input, S_IRUGO, show_in, NULL, 0, 6), + SENSOR_ATTR_2(in7_input, S_IRUGO, show_in, NULL, 0, 7), + SENSOR_ATTR_2(in8_input, S_IRUGO, show_in, NULL, 0, 8), + SENSOR_ATTR_2(in9_input, S_IRUGO, show_in, NULL, 0, 9), + SENSOR_ATTR_2(in10_input, S_IRUGO, show_in, NULL, 0, 10), +}; + +static ssize_t show_in_max(struct device *dev, struct device_attribute + *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + + return sprintf(buf, "%d\n", data->in1_max * 8); +} + +static ssize_t store_in_max(struct device *dev, struct device_attribute + *devattr, const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val /= 8; + val = clamp_val(val, 0, 255); + + mutex_lock(&data->update_lock); + if (data->type == f81866a) + f71882fg_write8(data, F81866_REG_IN1_HIGH, val); + else + f71882fg_write8(data, F71882FG_REG_IN1_HIGH, val); + data->in1_max = val; + mutex_unlock(&data->update_lock); + + return count; +} + +static ssize_t show_in_beep(struct device *dev, struct device_attribute + *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + + if (data->in_beep & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); +} + +static ssize_t store_in_beep(struct device *dev, struct device_attribute + *devattr, const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + unsigned long val; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + mutex_lock(&data->update_lock); + if (data->type == f81866a) + data->in_beep = f71882fg_read8(data, F81866_REG_IN_BEEP); + else + data->in_beep = f71882fg_read8(data, F71882FG_REG_IN_BEEP); + + if (val) + data->in_beep |= 1 << nr; + else + data->in_beep &= ~(1 << nr); + + if (data->type == f81866a) + f71882fg_write8(data, F81866_REG_IN_BEEP, data->in_beep); + else + f71882fg_write8(data, F71882FG_REG_IN_BEEP, data->in_beep); + mutex_unlock(&data->update_lock); + + return count; +} + +static ssize_t show_in_alarm(struct device *dev, struct device_attribute + *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + + if (data->in_status & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); +} + +/* For models with in1 alarm capability */ +static struct sensor_device_attribute_2 fxxxx_in1_alarm_attr[] = { + SENSOR_ATTR_2(in1_max, S_IRUGO|S_IWUSR, show_in_max, store_in_max, + 0, 1), + SENSOR_ATTR_2(in1_beep, S_IRUGO|S_IWUSR, show_in_beep, store_in_beep, + 0, 1), + SENSOR_ATTR_2(in1_alarm, S_IRUGO, show_in_alarm, NULL, 0, 1), +}; + +static ssize_t show_fan(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + int speed = fan_from_reg(data->fan[nr]); + + if (speed == FAN_MIN_DETECT) + speed = 0; + + return sprintf(buf, "%d\n", speed); +} + +static ssize_t show_fan_full_speed(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + int speed = fan_from_reg(data->fan_full_speed[nr]); + return sprintf(buf, "%d\n", speed); +} + +static ssize_t store_fan_full_speed(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val = clamp_val(val, 23, 1500000); + val = fan_to_reg(val); + + mutex_lock(&data->update_lock); + f71882fg_write16(data, F71882FG_REG_FAN_FULL_SPEED(nr), val); + data->fan_full_speed[nr] = val; + mutex_unlock(&data->update_lock); + + return count; +} + +static ssize_t show_fan_alarm(struct device *dev, struct device_attribute + *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + + if (data->fan_status & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); +} + +static ssize_t show_pwm(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int val, nr = to_sensor_dev_attr_2(devattr)->index; + mutex_lock(&data->update_lock); + if (data->pwm_enable & (1 << (2 * nr))) + /* PWM mode */ + val = data->pwm[nr]; + else { + /* RPM mode */ + val = 255 * fan_from_reg(data->fan_target[nr]) + / fan_from_reg(data->fan_full_speed[nr]); + } + mutex_unlock(&data->update_lock); + return sprintf(buf, "%d\n", val); +} + +static ssize_t store_pwm(struct device *dev, + struct device_attribute *devattr, const char *buf, + size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val = clamp_val(val, 0, 255); + + mutex_lock(&data->update_lock); + data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE); + if ((data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 3) != 2) || + (data->type != f8000 && !((data->pwm_enable >> 2 * nr) & 2))) { + count = -EROFS; + goto leave; + } + if (data->pwm_enable & (1 << (2 * nr))) { + /* PWM mode */ + f71882fg_write8(data, F71882FG_REG_PWM(nr), val); + data->pwm[nr] = val; + } else { + /* RPM mode */ + int target, full_speed; + full_speed = f71882fg_read16(data, + F71882FG_REG_FAN_FULL_SPEED(nr)); + target = fan_to_reg(val * fan_from_reg(full_speed) / 255); + f71882fg_write16(data, F71882FG_REG_FAN_TARGET(nr), target); + data->fan_target[nr] = target; + data->fan_full_speed[nr] = full_speed; + } +leave: mutex_unlock(&data->update_lock); return count; @@ -1961,252 +1226,878 @@ leave: return count; } -static ssize_t show_pwm_auto_point_pwm(struct device *dev, - struct device_attribute *devattr, - char *buf) +static ssize_t show_pwm_interpolate(struct device *dev, + struct device_attribute *devattr, char *buf) { int result; struct f71882fg_data *data = f71882fg_update_device(dev); - int pwm = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; + int nr = to_sensor_dev_attr_2(devattr)->index; - mutex_lock(&data->update_lock); - if (data->pwm_enable & (1 << (2 * pwm))) { - /* PWM mode */ - result = data->pwm_auto_point_pwm[pwm][point]; - } else { - /* RPM mode */ - result = 32 * 255 / (32 + data->pwm_auto_point_pwm[pwm][point]); - } - mutex_unlock(&data->update_lock); + result = (data->pwm_auto_point_mapping[nr] >> 4) & 1; return sprintf(buf, "%d\n", result); } -static ssize_t store_pwm_auto_point_pwm(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) +static ssize_t store_pwm_interpolate(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) { struct f71882fg_data *data = dev_get_drvdata(dev); - int err, pwm = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; - long val; + int err, nr = to_sensor_dev_attr_2(devattr)->index; + unsigned long val; - err = kstrtol(buf, 10, &val); + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + mutex_lock(&data->update_lock); + data->pwm_auto_point_mapping[nr] = + f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr)); + if (val) + val = data->pwm_auto_point_mapping[nr] | (1 << 4); + else + val = data->pwm_auto_point_mapping[nr] & (~(1 << 4)); + f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val); + data->pwm_auto_point_mapping[nr] = val; + mutex_unlock(&data->update_lock); + + return count; +} + +/* Fan / PWM attr common to all models */ +static struct sensor_device_attribute_2 fxxxx_fan_attr[4][6] = { { + SENSOR_ATTR_2(fan1_input, S_IRUGO, show_fan, NULL, 0, 0), + SENSOR_ATTR_2(fan1_full_speed, S_IRUGO|S_IWUSR, + show_fan_full_speed, + store_fan_full_speed, 0, 0), + SENSOR_ATTR_2(fan1_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 0), + SENSOR_ATTR_2(pwm1, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 0), + SENSOR_ATTR_2(pwm1_enable, S_IRUGO|S_IWUSR, show_pwm_enable, + store_pwm_enable, 0, 0), + SENSOR_ATTR_2(pwm1_interpolate, S_IRUGO|S_IWUSR, + show_pwm_interpolate, store_pwm_interpolate, 0, 0), +}, { + SENSOR_ATTR_2(fan2_input, S_IRUGO, show_fan, NULL, 0, 1), + SENSOR_ATTR_2(fan2_full_speed, S_IRUGO|S_IWUSR, + show_fan_full_speed, + store_fan_full_speed, 0, 1), + SENSOR_ATTR_2(fan2_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 1), + SENSOR_ATTR_2(pwm2, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 1), + SENSOR_ATTR_2(pwm2_enable, S_IRUGO|S_IWUSR, show_pwm_enable, + store_pwm_enable, 0, 1), + SENSOR_ATTR_2(pwm2_interpolate, S_IRUGO|S_IWUSR, + show_pwm_interpolate, store_pwm_interpolate, 0, 1), +}, { + SENSOR_ATTR_2(fan3_input, S_IRUGO, show_fan, NULL, 0, 2), + SENSOR_ATTR_2(fan3_full_speed, S_IRUGO|S_IWUSR, + show_fan_full_speed, + store_fan_full_speed, 0, 2), + SENSOR_ATTR_2(fan3_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 2), + SENSOR_ATTR_2(pwm3, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 2), + SENSOR_ATTR_2(pwm3_enable, S_IRUGO|S_IWUSR, show_pwm_enable, + store_pwm_enable, 0, 2), + SENSOR_ATTR_2(pwm3_interpolate, S_IRUGO|S_IWUSR, + show_pwm_interpolate, store_pwm_interpolate, 0, 2), +}, { + SENSOR_ATTR_2(fan4_input, S_IRUGO, show_fan, NULL, 0, 3), + SENSOR_ATTR_2(fan4_full_speed, S_IRUGO|S_IWUSR, + show_fan_full_speed, + store_fan_full_speed, 0, 3), + SENSOR_ATTR_2(fan4_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 3), + SENSOR_ATTR_2(pwm4, S_IRUGO|S_IWUSR, show_pwm, store_pwm, 0, 3), + SENSOR_ATTR_2(pwm4_enable, S_IRUGO|S_IWUSR, show_pwm_enable, + store_pwm_enable, 0, 3), + SENSOR_ATTR_2(pwm4_interpolate, S_IRUGO|S_IWUSR, + show_pwm_interpolate, store_pwm_interpolate, 0, 3), +} }; + +static ssize_t show_simple_pwm(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct f71882fg_data *data = f71882fg_update_device(dev); + int val, nr = to_sensor_dev_attr_2(devattr)->index; + + val = data->pwm[nr]; + return sprintf(buf, "%d\n", val); +} + +static ssize_t store_simple_pwm(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + long val; + + err = kstrtol(buf, 10, &val); if (err) return err; val = clamp_val(val, 0, 255); mutex_lock(&data->update_lock); - data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE); - if (data->pwm_enable & (1 << (2 * pwm))) { - /* PWM mode */ - } else { - /* RPM mode */ - if (val < 29) /* Prevent negative numbers */ - val = 255; - else - val = (255 - val) * 32 / val; - } - f71882fg_write8(data, F71882FG_REG_POINT_PWM(pwm, point), val); - data->pwm_auto_point_pwm[pwm][point] = val; + f71882fg_write8(data, F71882FG_REG_PWM(nr), val); + data->pwm[nr] = val; mutex_unlock(&data->update_lock); return count; } -static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev, - struct device_attribute *devattr, - char *buf) +/* Attr for the third fan of the f71808a, which only has manual pwm */ +static struct sensor_device_attribute_2 f71808a_fan3_attr[] = { + SENSOR_ATTR_2(fan3_input, S_IRUGO, show_fan, NULL, 0, 2), + SENSOR_ATTR_2(fan3_alarm, S_IRUGO, show_fan_alarm, NULL, 0, 2), + SENSOR_ATTR_2(pwm3, S_IRUGO|S_IWUSR, + show_simple_pwm, store_simple_pwm, 0, 2), +}; + +static ssize_t show_fan_beep(struct device *dev, struct device_attribute + *devattr, char *buf) { - int result = 0; struct f71882fg_data *data = f71882fg_update_device(dev); int nr = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; + + if (data->fan_beep & (1 << nr)) + return sprintf(buf, "1\n"); + else + return sprintf(buf, "0\n"); +} + +static ssize_t store_fan_beep(struct device *dev, struct device_attribute + *devattr, const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + unsigned long val; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; mutex_lock(&data->update_lock); - if (nr & 1) - result = data->pwm_auto_point_hyst[nr / 2] >> 4; + data->fan_beep = f71882fg_read8(data, F71882FG_REG_FAN_BEEP); + if (val) + data->fan_beep |= 1 << nr; else - result = data->pwm_auto_point_hyst[nr / 2] & 0x0f; - result = 1000 * (data->pwm_auto_point_temp[nr][point] - result); + data->fan_beep &= ~(1 << nr); + + f71882fg_write8(data, F71882FG_REG_FAN_BEEP, data->fan_beep); mutex_unlock(&data->update_lock); + return count; +} + +/* Attr for models which can beep on Fan alarm */ +static struct sensor_device_attribute_2 fxxxx_fan_beep_attr[] = { + SENSOR_ATTR_2(fan1_beep, S_IRUGO|S_IWUSR, show_fan_beep, + store_fan_beep, 0, 0), + SENSOR_ATTR_2(fan2_beep, S_IRUGO|S_IWUSR, show_fan_beep, + store_fan_beep, 0, 1), + SENSOR_ATTR_2(fan3_beep, S_IRUGO|S_IWUSR, show_fan_beep, + store_fan_beep, 0, 2), + SENSOR_ATTR_2(fan4_beep, S_IRUGO|S_IWUSR, show_fan_beep, + store_fan_beep, 0, 3), +}; + +static ssize_t show_pwm_auto_point_channel(struct device *dev, + struct device_attribute *devattr, + char *buf) +{ + int result; + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + + result = 1 << ((data->pwm_auto_point_mapping[nr] & 3) - + data->temp_start); + return sprintf(buf, "%d\n", result); } -static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) +static ssize_t store_pwm_auto_point_channel(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) { struct f71882fg_data *data = dev_get_drvdata(dev); int err, nr = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; - u8 reg; long val; err = kstrtol(buf, 10, &val); if (err) return err; - val /= 1000; - + switch (val) { + case 1: + val = 0; + break; + case 2: + val = 1; + break; + case 4: + val = 2; + break; + default: + return -EINVAL; + } + val += data->temp_start; mutex_lock(&data->update_lock); - data->pwm_auto_point_temp[nr][point] = - f71882fg_read8(data, F71882FG_REG_POINT_TEMP(nr, point)); - val = clamp_val(val, data->pwm_auto_point_temp[nr][point] - 15, - data->pwm_auto_point_temp[nr][point]); - val = data->pwm_auto_point_temp[nr][point] - val; - - reg = f71882fg_read8(data, F71882FG_REG_FAN_HYST(nr / 2)); - if (nr & 1) - reg = (reg & 0x0f) | (val << 4); - else - reg = (reg & 0xf0) | val; - - f71882fg_write8(data, F71882FG_REG_FAN_HYST(nr / 2), reg); - data->pwm_auto_point_hyst[nr / 2] = reg; + data->pwm_auto_point_mapping[nr] = + f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr)); + val = (data->pwm_auto_point_mapping[nr] & 0xfc) | val; + f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val); + data->pwm_auto_point_mapping[nr] = val; mutex_unlock(&data->update_lock); return count; } -static ssize_t show_pwm_interpolate(struct device *dev, - struct device_attribute *devattr, char *buf) +static ssize_t show_pwm_auto_point_pwm(struct device *dev, + struct device_attribute *devattr, + char *buf) { int result; struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; + int pwm = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + + mutex_lock(&data->update_lock); + if (data->pwm_enable & (1 << (2 * pwm))) { + /* PWM mode */ + result = data->pwm_auto_point_pwm[pwm][point]; + } else { + /* RPM mode */ + result = 32 * 255 / (32 + data->pwm_auto_point_pwm[pwm][point]); + } + mutex_unlock(&data->update_lock); + + return sprintf(buf, "%d\n", result); +} + +static ssize_t store_pwm_auto_point_pwm(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, pwm = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val = clamp_val(val, 0, 255); + + mutex_lock(&data->update_lock); + data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE); + if (data->pwm_enable & (1 << (2 * pwm))) { + /* PWM mode */ + } else { + /* RPM mode */ + if (val < 29) /* Prevent negative numbers */ + val = 255; + else + val = (255 - val) * 32 / val; + } + f71882fg_write8(data, F71882FG_REG_POINT_PWM(pwm, point), val); + data->pwm_auto_point_pwm[pwm][point] = val; + mutex_unlock(&data->update_lock); + + return count; +} + +static ssize_t show_pwm_auto_point_temp(struct device *dev, + struct device_attribute *devattr, + char *buf) +{ + int result; + struct f71882fg_data *data = f71882fg_update_device(dev); + int pwm = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + + result = data->pwm_auto_point_temp[pwm][point]; + return sprintf(buf, "%d\n", 1000 * result); +} + +static ssize_t store_pwm_auto_point_temp(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, pwm = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val /= 1000; + + if (data->auto_point_temp_signed) + val = clamp_val(val, -128, 127); + else + val = clamp_val(val, 0, 127); + + mutex_lock(&data->update_lock); + f71882fg_write8(data, F71882FG_REG_POINT_TEMP(pwm, point), val); + data->pwm_auto_point_temp[pwm][point] = val; + mutex_unlock(&data->update_lock); + + return count; +} + +static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev, + struct device_attribute *devattr, + char *buf) +{ + int result = 0; + struct f71882fg_data *data = f71882fg_update_device(dev); + int nr = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + + mutex_lock(&data->update_lock); + if (nr & 1) + result = data->pwm_auto_point_hyst[nr / 2] >> 4; + else + result = data->pwm_auto_point_hyst[nr / 2] & 0x0f; + result = 1000 * (data->pwm_auto_point_temp[nr][point] - result); + mutex_unlock(&data->update_lock); + + return sprintf(buf, "%d\n", result); +} + +static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct f71882fg_data *data = dev_get_drvdata(dev); + int err, nr = to_sensor_dev_attr_2(devattr)->index; + int point = to_sensor_dev_attr_2(devattr)->nr; + u8 reg; + long val; + + err = kstrtol(buf, 10, &val); + if (err) + return err; + + val /= 1000; + + mutex_lock(&data->update_lock); + data->pwm_auto_point_temp[nr][point] = + f71882fg_read8(data, F71882FG_REG_POINT_TEMP(nr, point)); + val = clamp_val(val, data->pwm_auto_point_temp[nr][point] - 15, + data->pwm_auto_point_temp[nr][point]); + val = data->pwm_auto_point_temp[nr][point] - val; + + reg = f71882fg_read8(data, F71882FG_REG_FAN_HYST(nr / 2)); + if (nr & 1) + reg = (reg & 0x0f) | (val << 4); + else + reg = (reg & 0xf0) | val; + + f71882fg_write8(data, F71882FG_REG_FAN_HYST(nr / 2), reg); + data->pwm_auto_point_hyst[nr / 2] = reg; + mutex_unlock(&data->update_lock); + + return count; +} + +/* + * PWM attr for the f71862fg, fewer pwms and fewer zones per pwm than the + * standard models + */ +static struct sensor_device_attribute_2 f71862fg_auto_pwm_attr[3][7] = { { + SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 0), + SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 0), + SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 0), +}, { + SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 1), + SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 1), + SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 1), +}, { + SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 2), + SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 2), + SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 2), +} }; + +/* + * PWM attr for the f71808e/f71869, almost identical to the f71862fg, but the + * pwm setting when the temperature is above the pwmX_auto_point1_temp can be + * programmed instead of being hardcoded to 0xff + */ +static struct sensor_device_attribute_2 f71869_auto_pwm_attr[3][8] = { { + SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 0), + SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 0), + SENSOR_ATTR_2(pwm1_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 0), +}, { + SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 1), + SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 1), + SENSOR_ATTR_2(pwm2_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 1), +}, { + SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 2), + SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 2), + SENSOR_ATTR_2(pwm3_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 2), +} }; + +/* PWM attr for the standard models */ +static struct sensor_device_attribute_2 fxxxx_auto_pwm_attr[4][14] = { { + SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 0), + SENSOR_ATTR_2(pwm1_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 0), + SENSOR_ATTR_2(pwm1_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 0), + SENSOR_ATTR_2(pwm1_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 0), + SENSOR_ATTR_2(pwm1_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 0), + SENSOR_ATTR_2(pwm1_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 0), + SENSOR_ATTR_2(pwm1_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 0), + SENSOR_ATTR_2(pwm1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 0), + SENSOR_ATTR_2(pwm1_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 0), + SENSOR_ATTR_2(pwm1_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 0), + SENSOR_ATTR_2(pwm1_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 0), +}, { + SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 1), + SENSOR_ATTR_2(pwm2_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 1), + SENSOR_ATTR_2(pwm2_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 1), + SENSOR_ATTR_2(pwm2_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 1), + SENSOR_ATTR_2(pwm2_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 1), + SENSOR_ATTR_2(pwm2_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 1), + SENSOR_ATTR_2(pwm2_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 1), + SENSOR_ATTR_2(pwm2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 1), + SENSOR_ATTR_2(pwm2_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 1), + SENSOR_ATTR_2(pwm2_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 1), + SENSOR_ATTR_2(pwm2_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 1), +}, { + SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 2), + SENSOR_ATTR_2(pwm3_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 2), + SENSOR_ATTR_2(pwm3_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 2), + SENSOR_ATTR_2(pwm3_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 2), + SENSOR_ATTR_2(pwm3_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 2), + SENSOR_ATTR_2(pwm3_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 2), + SENSOR_ATTR_2(pwm3_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 2), + SENSOR_ATTR_2(pwm3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 2), + SENSOR_ATTR_2(pwm3_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 2), + SENSOR_ATTR_2(pwm3_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 2), + SENSOR_ATTR_2(pwm3_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 2), +}, { + SENSOR_ATTR_2(pwm4_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 3), + SENSOR_ATTR_2(pwm4_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 3), + SENSOR_ATTR_2(pwm4_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 3), + SENSOR_ATTR_2(pwm4_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 3), + SENSOR_ATTR_2(pwm4_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 3), + SENSOR_ATTR_2(pwm4_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 3), + SENSOR_ATTR_2(pwm4_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 3), + SENSOR_ATTR_2(pwm4_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 3), + SENSOR_ATTR_2(pwm4_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 3), + SENSOR_ATTR_2(pwm4_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 3), + SENSOR_ATTR_2(pwm4_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 3), + SENSOR_ATTR_2(pwm4_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 3), + SENSOR_ATTR_2(pwm4_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 3), + SENSOR_ATTR_2(pwm4_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 3), +} }; - result = (data->pwm_auto_point_mapping[nr] >> 4) & 1; +/* Fan attr specific to the f8000 (4th fan input can only measure speed) */ +static struct sensor_device_attribute_2 f8000_fan_attr[] = { + SENSOR_ATTR_2(fan4_input, S_IRUGO, show_fan, NULL, 0, 3), +}; - return sprintf(buf, "%d\n", result); -} +/* + * PWM attr for the f8000, zones mapped to temp instead of to pwm! + * Also the register block at offset A0 maps to TEMP1 (so our temp2, as the + * F8000 starts counting temps at 0), B0 maps the TEMP2 and C0 maps to TEMP0 + */ +static struct sensor_device_attribute_2 f8000_auto_pwm_attr[3][14] = { { + SENSOR_ATTR_2(pwm1_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 0), + SENSOR_ATTR_2(temp1_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 2), + SENSOR_ATTR_2(temp1_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 2), + SENSOR_ATTR_2(temp1_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 2), + SENSOR_ATTR_2(temp1_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 2), + SENSOR_ATTR_2(temp1_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 2), + SENSOR_ATTR_2(temp1_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 2), + SENSOR_ATTR_2(temp1_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 2), + SENSOR_ATTR_2(temp1_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 2), + SENSOR_ATTR_2(temp1_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 2), + SENSOR_ATTR_2(temp1_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 2), + SENSOR_ATTR_2(temp1_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 2), + SENSOR_ATTR_2(temp1_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 2), + SENSOR_ATTR_2(temp1_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 2), +}, { + SENSOR_ATTR_2(pwm2_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 1), + SENSOR_ATTR_2(temp2_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 0), + SENSOR_ATTR_2(temp2_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 0), + SENSOR_ATTR_2(temp2_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 0), + SENSOR_ATTR_2(temp2_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 0), + SENSOR_ATTR_2(temp2_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 0), + SENSOR_ATTR_2(temp2_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 0), + SENSOR_ATTR_2(temp2_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 0), + SENSOR_ATTR_2(temp2_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 0), + SENSOR_ATTR_2(temp2_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 0), + SENSOR_ATTR_2(temp2_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 0), + SENSOR_ATTR_2(temp2_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 0), + SENSOR_ATTR_2(temp2_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 0), + SENSOR_ATTR_2(temp2_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 0), +}, { + SENSOR_ATTR_2(pwm3_auto_channels_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_channel, + store_pwm_auto_point_channel, 0, 2), + SENSOR_ATTR_2(temp3_auto_point1_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 0, 1), + SENSOR_ATTR_2(temp3_auto_point2_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 1, 1), + SENSOR_ATTR_2(temp3_auto_point3_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 2, 1), + SENSOR_ATTR_2(temp3_auto_point4_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 3, 1), + SENSOR_ATTR_2(temp3_auto_point5_pwm, S_IRUGO|S_IWUSR, + show_pwm_auto_point_pwm, store_pwm_auto_point_pwm, + 4, 1), + SENSOR_ATTR_2(temp3_auto_point1_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 0, 1), + SENSOR_ATTR_2(temp3_auto_point2_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 1, 1), + SENSOR_ATTR_2(temp3_auto_point3_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 2, 1), + SENSOR_ATTR_2(temp3_auto_point4_temp, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp, store_pwm_auto_point_temp, + 3, 1), + SENSOR_ATTR_2(temp3_auto_point1_temp_hyst, S_IRUGO|S_IWUSR, + show_pwm_auto_point_temp_hyst, + store_pwm_auto_point_temp_hyst, + 0, 1), + SENSOR_ATTR_2(temp3_auto_point2_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 1, 1), + SENSOR_ATTR_2(temp3_auto_point3_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 2, 1), + SENSOR_ATTR_2(temp3_auto_point4_temp_hyst, S_IRUGO, + show_pwm_auto_point_temp_hyst, NULL, 3, 1), +} }; -static ssize_t store_pwm_interpolate(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) +/* Super I/O functions */ +static inline int superio_inb(int base, int reg) { - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - unsigned long val; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - mutex_lock(&data->update_lock); - data->pwm_auto_point_mapping[nr] = - f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr)); - if (val) - val = data->pwm_auto_point_mapping[nr] | (1 << 4); - else - val = data->pwm_auto_point_mapping[nr] & (~(1 << 4)); - f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val); - data->pwm_auto_point_mapping[nr] = val; - mutex_unlock(&data->update_lock); - - return count; + outb(reg, base); + return inb(base + 1); } -static ssize_t show_pwm_auto_point_channel(struct device *dev, - struct device_attribute *devattr, - char *buf) +static int superio_inw(int base, int reg) { - int result; - struct f71882fg_data *data = f71882fg_update_device(dev); - int nr = to_sensor_dev_attr_2(devattr)->index; - - result = 1 << ((data->pwm_auto_point_mapping[nr] & 3) - - data->temp_start); - - return sprintf(buf, "%d\n", result); + int val; + val = superio_inb(base, reg) << 8; + val |= superio_inb(base, reg + 1); + return val; } -static ssize_t store_pwm_auto_point_channel(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) +static inline int superio_enter(int base) { - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, nr = to_sensor_dev_attr_2(devattr)->index; - long val; - - err = kstrtol(buf, 10, &val); - if (err) - return err; - - switch (val) { - case 1: - val = 0; - break; - case 2: - val = 1; - break; - case 4: - val = 2; - break; - default: - return -EINVAL; + /* Don't step on other drivers' I/O space by accident */ + if (!request_muxed_region(base, 2, DRVNAME)) { + pr_err("I/O address 0x%04x already in use\n", base); + return -EBUSY; } - val += data->temp_start; - mutex_lock(&data->update_lock); - data->pwm_auto_point_mapping[nr] = - f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr)); - val = (data->pwm_auto_point_mapping[nr] & 0xfc) | val; - f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val); - data->pwm_auto_point_mapping[nr] = val; - mutex_unlock(&data->update_lock); - - return count; -} - -static ssize_t show_pwm_auto_point_temp(struct device *dev, - struct device_attribute *devattr, - char *buf) -{ - int result; - struct f71882fg_data *data = f71882fg_update_device(dev); - int pwm = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; - - result = data->pwm_auto_point_temp[pwm][point]; - return sprintf(buf, "%d\n", 1000 * result); -} - -static ssize_t store_pwm_auto_point_temp(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) -{ - struct f71882fg_data *data = dev_get_drvdata(dev); - int err, pwm = to_sensor_dev_attr_2(devattr)->index; - int point = to_sensor_dev_attr_2(devattr)->nr; - long val; - - err = kstrtol(buf, 10, &val); - if (err) - return err; - - val /= 1000; - - if (data->auto_point_temp_signed) - val = clamp_val(val, -128, 127); - else - val = clamp_val(val, 0, 127); - mutex_lock(&data->update_lock); - f71882fg_write8(data, F71882FG_REG_POINT_TEMP(pwm, point), val); - data->pwm_auto_point_temp[pwm][point] = val; - mutex_unlock(&data->update_lock); + /* according to the datasheet the key must be send twice! */ + outb(SIO_UNLOCK_KEY, base); + outb(SIO_UNLOCK_KEY, base); - return count; + return 0; } -static ssize_t name_show(struct device *dev, struct device_attribute *devattr, - char *buf) +static inline void superio_select(int base, int ld) { - struct f71882fg_data *data = dev_get_drvdata(dev); - return sprintf(buf, "%s\n", f71882fg_names[data->type]); + outb(SIO_REG_LDSEL, base); + outb(ld, base + 1); +} + +static inline void superio_exit(int base) +{ + outb(SIO_LOCK_KEY, base); + release_region(base, 2); } static int f71882fg_create_sysfs_files(struct platform_device *pdev, @@ -2329,6 +2220,119 @@ static int f71882fg_create_fan_sysfs_files( return err; } +static int f71882fg_remove(struct platform_device *pdev) +{ + struct f71882fg_data *data = platform_get_drvdata(pdev); + int nr_fans = f71882fg_nr_fans[data->type]; + int nr_temps = f71882fg_nr_temps[data->type]; + int i; + u8 start_reg = f71882fg_read8(data, F71882FG_REG_START); + + if (data->hwmon_dev) + hwmon_device_unregister(data->hwmon_dev); + + device_remove_file(&pdev->dev, &dev_attr_name); + + if (start_reg & 0x01) { + switch (data->type) { + case f71858fg: + if (data->temp_config & 0x10) + f71882fg_remove_sysfs_files(pdev, + f8000_temp_attr, + ARRAY_SIZE(f8000_temp_attr)); + else + f71882fg_remove_sysfs_files(pdev, + f71858fg_temp_attr, + ARRAY_SIZE(f71858fg_temp_attr)); + break; + case f8000: + f71882fg_remove_sysfs_files(pdev, + f8000_temp_attr, + ARRAY_SIZE(f8000_temp_attr)); + break; + case f81866a: + f71882fg_remove_sysfs_files(pdev, + f71858fg_temp_attr, + ARRAY_SIZE(f71858fg_temp_attr)); + break; + default: + f71882fg_remove_sysfs_files(pdev, + &fxxxx_temp_attr[0][0], + ARRAY_SIZE(fxxxx_temp_attr[0]) * nr_temps); + } + if (f71882fg_temp_has_beep[data->type]) { + if (data->type == f81866a) + f71882fg_remove_sysfs_files(pdev, + &f81866_temp_beep_attr[0][0], + ARRAY_SIZE(f81866_temp_beep_attr[0]) + * nr_temps); + else + f71882fg_remove_sysfs_files(pdev, + &fxxxx_temp_beep_attr[0][0], + ARRAY_SIZE(fxxxx_temp_beep_attr[0]) + * nr_temps); + } + + for (i = 0; i < F71882FG_MAX_INS; i++) { + if (f71882fg_has_in[data->type][i]) { + device_remove_file(&pdev->dev, + &fxxxx_in_attr[i].dev_attr); + } + } + if (f71882fg_has_in1_alarm[data->type]) { + f71882fg_remove_sysfs_files(pdev, + fxxxx_in1_alarm_attr, + ARRAY_SIZE(fxxxx_in1_alarm_attr)); + } + } + + if (start_reg & 0x02) { + f71882fg_remove_sysfs_files(pdev, &fxxxx_fan_attr[0][0], + ARRAY_SIZE(fxxxx_fan_attr[0]) * nr_fans); + + if (f71882fg_fan_has_beep[data->type]) { + f71882fg_remove_sysfs_files(pdev, + fxxxx_fan_beep_attr, nr_fans); + } + + switch (data->type) { + case f71808a: + f71882fg_remove_sysfs_files(pdev, + &fxxxx_auto_pwm_attr[0][0], + ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans); + f71882fg_remove_sysfs_files(pdev, + f71808a_fan3_attr, + ARRAY_SIZE(f71808a_fan3_attr)); + break; + case f71862fg: + f71882fg_remove_sysfs_files(pdev, + &f71862fg_auto_pwm_attr[0][0], + ARRAY_SIZE(f71862fg_auto_pwm_attr[0]) * + nr_fans); + break; + case f71808e: + case f71869: + f71882fg_remove_sysfs_files(pdev, + &f71869_auto_pwm_attr[0][0], + ARRAY_SIZE(f71869_auto_pwm_attr[0]) * nr_fans); + break; + case f8000: + f71882fg_remove_sysfs_files(pdev, + f8000_fan_attr, + ARRAY_SIZE(f8000_fan_attr)); + f71882fg_remove_sysfs_files(pdev, + &f8000_auto_pwm_attr[0][0], + ARRAY_SIZE(f8000_auto_pwm_attr[0]) * nr_fans); + break; + default: + f71882fg_remove_sysfs_files(pdev, + &fxxxx_auto_pwm_attr[0][0], + ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans); + } + } + return 0; +} + static int f71882fg_probe(struct platform_device *pdev) { struct f71882fg_data *data; @@ -2502,119 +2506,6 @@ exit_unregister_sysfs: return err; /* f71882fg_remove() also frees our data */ } -static int f71882fg_remove(struct platform_device *pdev) -{ - struct f71882fg_data *data = platform_get_drvdata(pdev); - int nr_fans = f71882fg_nr_fans[data->type]; - int nr_temps = f71882fg_nr_temps[data->type]; - int i; - u8 start_reg = f71882fg_read8(data, F71882FG_REG_START); - - if (data->hwmon_dev) - hwmon_device_unregister(data->hwmon_dev); - - device_remove_file(&pdev->dev, &dev_attr_name); - - if (start_reg & 0x01) { - switch (data->type) { - case f71858fg: - if (data->temp_config & 0x10) - f71882fg_remove_sysfs_files(pdev, - f8000_temp_attr, - ARRAY_SIZE(f8000_temp_attr)); - else - f71882fg_remove_sysfs_files(pdev, - f71858fg_temp_attr, - ARRAY_SIZE(f71858fg_temp_attr)); - break; - case f8000: - f71882fg_remove_sysfs_files(pdev, - f8000_temp_attr, - ARRAY_SIZE(f8000_temp_attr)); - break; - case f81866a: - f71882fg_remove_sysfs_files(pdev, - f71858fg_temp_attr, - ARRAY_SIZE(f71858fg_temp_attr)); - break; - default: - f71882fg_remove_sysfs_files(pdev, - &fxxxx_temp_attr[0][0], - ARRAY_SIZE(fxxxx_temp_attr[0]) * nr_temps); - } - if (f71882fg_temp_has_beep[data->type]) { - if (data->type == f81866a) - f71882fg_remove_sysfs_files(pdev, - &f81866_temp_beep_attr[0][0], - ARRAY_SIZE(f81866_temp_beep_attr[0]) - * nr_temps); - else - f71882fg_remove_sysfs_files(pdev, - &fxxxx_temp_beep_attr[0][0], - ARRAY_SIZE(fxxxx_temp_beep_attr[0]) - * nr_temps); - } - - for (i = 0; i < F71882FG_MAX_INS; i++) { - if (f71882fg_has_in[data->type][i]) { - device_remove_file(&pdev->dev, - &fxxxx_in_attr[i].dev_attr); - } - } - if (f71882fg_has_in1_alarm[data->type]) { - f71882fg_remove_sysfs_files(pdev, - fxxxx_in1_alarm_attr, - ARRAY_SIZE(fxxxx_in1_alarm_attr)); - } - } - - if (start_reg & 0x02) { - f71882fg_remove_sysfs_files(pdev, &fxxxx_fan_attr[0][0], - ARRAY_SIZE(fxxxx_fan_attr[0]) * nr_fans); - - if (f71882fg_fan_has_beep[data->type]) { - f71882fg_remove_sysfs_files(pdev, - fxxxx_fan_beep_attr, nr_fans); - } - - switch (data->type) { - case f71808a: - f71882fg_remove_sysfs_files(pdev, - &fxxxx_auto_pwm_attr[0][0], - ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans); - f71882fg_remove_sysfs_files(pdev, - f71808a_fan3_attr, - ARRAY_SIZE(f71808a_fan3_attr)); - break; - case f71862fg: - f71882fg_remove_sysfs_files(pdev, - &f71862fg_auto_pwm_attr[0][0], - ARRAY_SIZE(f71862fg_auto_pwm_attr[0]) * - nr_fans); - break; - case f71808e: - case f71869: - f71882fg_remove_sysfs_files(pdev, - &f71869_auto_pwm_attr[0][0], - ARRAY_SIZE(f71869_auto_pwm_attr[0]) * nr_fans); - break; - case f8000: - f71882fg_remove_sysfs_files(pdev, - f8000_fan_attr, - ARRAY_SIZE(f8000_fan_attr)); - f71882fg_remove_sysfs_files(pdev, - &f8000_auto_pwm_attr[0][0], - ARRAY_SIZE(f8000_auto_pwm_attr[0]) * nr_fans); - break; - default: - f71882fg_remove_sysfs_files(pdev, - &fxxxx_auto_pwm_attr[0][0], - ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans); - } - } - return 0; -} - static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data) { u16 devid; @@ -2760,6 +2651,14 @@ exit_device_put: return err; } +static struct platform_driver f71882fg_driver = { + .driver = { + .name = DRVNAME, + }, + .probe = f71882fg_probe, + .remove = f71882fg_remove, +}; + static int __init f71882fg_init(void) { int err; diff --git a/drivers/hwmon/f75375s.c b/drivers/hwmon/f75375s.c index 57c8a473698d20968fc9f104e4be7645aa272d80..64fbb8cf687c078cc9cafc9e7b0cda936eb4e9ed 100644 --- a/drivers/hwmon/f75375s.c +++ b/drivers/hwmon/f75375s.c @@ -114,7 +114,7 @@ struct f75375_data { static int f75375_detect(struct i2c_client *client, struct i2c_board_info *info); static int f75375_probe(struct i2c_client *client); -static int f75375_remove(struct i2c_client *client); +static void f75375_remove(struct i2c_client *client); static const struct i2c_device_id f75375_id[] = { { "f75373", f75373 }, @@ -864,12 +864,11 @@ exit_remove: return err; } -static int f75375_remove(struct i2c_client *client) +static void f75375_remove(struct i2c_client *client) { struct f75375_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); sysfs_remove_group(&client->dev.kobj, &f75375_group); - return 0; } /* Return 0 if detection is successful, -ENODEV otherwise */ @@ -897,7 +896,7 @@ static int f75375_detect(struct i2c_client *client, version = f75375_read8(client, F75375_REG_VERSION); dev_info(&adapter->dev, "found %s version: %02X\n", name, version); - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/fschmd.c b/drivers/hwmon/fschmd.c index c26195e3aad78076c5c471427183de70ab7e953d..0a77d61619288e7d444fe3118ed7a04268babf7d 100644 --- a/drivers/hwmon/fschmd.c +++ b/drivers/hwmon/fschmd.c @@ -217,7 +217,7 @@ static const int FSCHMD_NO_TEMP_SENSORS[7] = { 3, 3, 4, 3, 5, 5, 11 }; static int fschmd_probe(struct i2c_client *client); static int fschmd_detect(struct i2c_client *client, struct i2c_board_info *info); -static int fschmd_remove(struct i2c_client *client); +static void fschmd_remove(struct i2c_client *client); static struct fschmd_data *fschmd_update_device(struct device *dev); /* @@ -1075,7 +1075,7 @@ static int fschmd_detect(struct i2c_client *client, else return -ENODEV; - strlcpy(info->type, fschmd_id[kind].name, I2C_NAME_SIZE); + strscpy(info->type, fschmd_id[kind].name, I2C_NAME_SIZE); return 0; } @@ -1248,7 +1248,7 @@ exit_detach: return err; } -static int fschmd_remove(struct i2c_client *client) +static void fschmd_remove(struct i2c_client *client) { struct fschmd_data *data = i2c_get_clientdata(client); int i; @@ -1291,8 +1291,6 @@ static int fschmd_remove(struct i2c_client *client) mutex_lock(&watchdog_data_mutex); kref_put(&data->kref, fschmd_release_resources); mutex_unlock(&watchdog_data_mutex); - - return 0; } static struct fschmd_data *fschmd_update_device(struct device *dev) diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c index ceffc76a0c515437f70387d7f0eba2378597aae9..f5b8e724a8ca1964bcb3341653e035e15c283163 100644 --- a/drivers/hwmon/ftsteutates.c +++ b/drivers/hwmon/ftsteutates.c @@ -739,17 +739,16 @@ static int fts_detect(struct i2c_client *client, if (val != 0x11) return -ENODEV; - strlcpy(info->type, fts_id[0].name, I2C_NAME_SIZE); + strscpy(info->type, fts_id[0].name, I2C_NAME_SIZE); info->flags = 0; return 0; } -static int fts_remove(struct i2c_client *client) +static void fts_remove(struct i2c_client *client) { struct fts_data *data = dev_get_drvdata(&client->dev); watchdog_unregister_device(&data->wdd); - return 0; } static int fts_probe(struct i2c_client *client) diff --git a/drivers/hwmon/gl518sm.c b/drivers/hwmon/gl518sm.c index dd683b0a648f9a6d8526d1866e037f109f33b6c3..95286c40f55a966e94ad7d64478985cecea84272 100644 --- a/drivers/hwmon/gl518sm.c +++ b/drivers/hwmon/gl518sm.c @@ -586,7 +586,7 @@ static int gl518_detect(struct i2c_client *client, struct i2c_board_info *info) if (rev != 0x00 && rev != 0x80) return -ENODEV; - strlcpy(info->type, "gl518sm", I2C_NAME_SIZE); + strscpy(info->type, "gl518sm", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/gl520sm.c b/drivers/hwmon/gl520sm.c index 096ba97972115477dce57c5f149aa2b1dd007025..394da4ac977c5a8613e24e6be1454928f0815e72 100644 --- a/drivers/hwmon/gl520sm.c +++ b/drivers/hwmon/gl520sm.c @@ -811,7 +811,7 @@ static int gl520_detect(struct i2c_client *client, struct i2c_board_info *info) return -ENODEV; } - strlcpy(info->type, "gl520sm", I2C_NAME_SIZE); + strscpy(info->type, "gl520sm", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c index befe989ca7b946f27bb1f773b316f97dbf76836a..ba408942dbe73bfa1fefa3f1bb1863c4dac8e2db 100644 --- a/drivers/hwmon/gpio-fan.c +++ b/drivers/hwmon/gpio-fan.c @@ -37,9 +37,7 @@ struct gpio_fan_data { int num_speed; struct gpio_fan_speed *speed; int speed_index; -#ifdef CONFIG_PM_SLEEP int resume_speed; -#endif bool pwm_enable; struct gpio_desc *alarm_gpio; struct work_struct alarm_work; @@ -391,6 +389,9 @@ static int gpio_fan_set_cur_state(struct thermal_cooling_device *cdev, if (!fan_data) return -EINVAL; + if (state >= fan_data->num_speed) + return -EINVAL; + set_fan_speed(fan_data, state); return 0; } @@ -554,7 +555,6 @@ static void gpio_fan_shutdown(struct platform_device *pdev) set_fan_speed(fan_data, 0); } -#ifdef CONFIG_PM_SLEEP static int gpio_fan_suspend(struct device *dev) { struct gpio_fan_data *fan_data = dev_get_drvdata(dev); @@ -577,18 +577,14 @@ static int gpio_fan_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(gpio_fan_pm, gpio_fan_suspend, gpio_fan_resume); -#define GPIO_FAN_PM (&gpio_fan_pm) -#else -#define GPIO_FAN_PM NULL -#endif +static DEFINE_SIMPLE_DEV_PM_OPS(gpio_fan_pm, gpio_fan_suspend, gpio_fan_resume); static struct platform_driver gpio_fan_driver = { .probe = gpio_fan_probe, .shutdown = gpio_fan_shutdown, .driver = { .name = "gpio-fan", - .pm = GPIO_FAN_PM, + .pm = pm_sleep_ptr(&gpio_fan_pm), .of_match_table = of_match_ptr(of_gpio_fan_match), }, }; diff --git a/drivers/hwmon/gsc-hwmon.c b/drivers/hwmon/gsc-hwmon.c index d64be48f1ef6c51c2490ee633824440b068f00b3..b60ec95b5edbf248da1d4abfd1abff494180beaa 100644 --- a/drivers/hwmon/gsc-hwmon.c +++ b/drivers/hwmon/gsc-hwmon.c @@ -267,6 +267,7 @@ gsc_hwmon_get_devtree_pdata(struct device *dev) pdata->nchannels = nchannels; /* fan controller base address */ + of_node_get(dev->parent->of_node); fan = of_find_compatible_node(dev->parent->of_node, NULL, "gw,gsc-fan"); if (fan && of_property_read_u32(fan, "reg", &pdata->fan_base)) { of_node_put(fan); diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 2e2cd79d89ebc93607b9bdb7186a8799d3b63493..4218750d5a66ba0425c246f50e5672f40e18a8cd 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -151,9 +151,9 @@ static DEFINE_IDA(hwmon_ida); * between hwmon and thermal_sys modules. */ #ifdef CONFIG_THERMAL_OF -static int hwmon_thermal_get_temp(void *data, int *temp) +static int hwmon_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct hwmon_thermal_data *tdata = data; + struct hwmon_thermal_data *tdata = tz->devdata; struct hwmon_device *hwdev = to_hwmon_device(tdata->dev); int ret; long t; @@ -168,9 +168,9 @@ static int hwmon_thermal_get_temp(void *data, int *temp) return 0; } -static int hwmon_thermal_set_trips(void *data, int low, int high) +static int hwmon_thermal_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct hwmon_thermal_data *tdata = data; + struct hwmon_thermal_data *tdata = tz->devdata; struct hwmon_device *hwdev = to_hwmon_device(tdata->dev); const struct hwmon_chip_info *chip = hwdev->chip; const struct hwmon_channel_info **info = chip->info; @@ -203,7 +203,7 @@ static int hwmon_thermal_set_trips(void *data, int low, int high) return 0; } -static const struct thermal_zone_of_device_ops hwmon_thermal_ops = { +static const struct thermal_zone_device_ops hwmon_thermal_ops = { .get_temp = hwmon_thermal_get_temp, .set_trips = hwmon_thermal_set_trips, }; @@ -227,8 +227,8 @@ static int hwmon_thermal_add_sensor(struct device *dev, int index) tdata->dev = dev; tdata->index = index; - tzd = devm_thermal_zone_of_sensor_register(dev, index, tdata, - &hwmon_thermal_ops); + tzd = devm_thermal_of_zone_register(dev, index, tdata, + &hwmon_thermal_ops); if (IS_ERR(tzd)) { if (PTR_ERR(tzd) != -ENODEV) return PTR_ERR(tzd); diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c index 580a7d125b88667b2c2a3cc26fe29a064339915f..3aa40893fc095abc5a177dc7b3d62cbed9557e2b 100644 --- a/drivers/hwmon/iio_hwmon.c +++ b/drivers/hwmon/iio_hwmon.c @@ -6,11 +6,13 @@ #include #include +#include #include #include #include +#include + #include -#include #include #include #include @@ -149,8 +151,8 @@ static int iio_hwmon_probe(struct platform_device *pdev) st->attr_group.attrs = st->attrs; st->groups[0] = &st->attr_group; - if (dev->of_node) { - sname = devm_kasprintf(dev, GFP_KERNEL, "%pOFn", dev->of_node); + if (dev_fwnode(dev)) { + sname = devm_kasprintf(dev, GFP_KERNEL, "%pfwP", dev_fwnode(dev)); if (!sname) return -ENOMEM; strreplace(sname, '-', '_'); diff --git a/drivers/hwmon/ina209.c b/drivers/hwmon/ina209.c index fc3007c3e85caf20cf3caacff8067e159126c242..9b58655d2de4a9c7ead354770a6f201c194a9fb2 100644 --- a/drivers/hwmon/ina209.c +++ b/drivers/hwmon/ina209.c @@ -568,13 +568,11 @@ out_restore_conf: return ret; } -static int ina209_remove(struct i2c_client *client) +static void ina209_remove(struct i2c_client *client) { struct ina209_data *data = i2c_get_clientdata(client); ina209_restore_conf(client, data); - - return 0; } static const struct i2c_device_id ina209_id[] = { diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c index 58d3828e2ec0c0b0fe3c5a3482f75fa9bcc80e87..2a57f4b60c29d5042dcc2adc7fbcc240204a5c78 100644 --- a/drivers/hwmon/ina3221.c +++ b/drivers/hwmon/ina3221.c @@ -913,7 +913,7 @@ fail: return ret; } -static int ina3221_remove(struct i2c_client *client) +static void ina3221_remove(struct i2c_client *client) { struct ina3221_data *ina = dev_get_drvdata(&client->dev); int i; @@ -926,11 +926,9 @@ static int ina3221_remove(struct i2c_client *client) pm_runtime_put_noidle(ina->pm_dev); mutex_destroy(&ina->lock); - - return 0; } -static int __maybe_unused ina3221_suspend(struct device *dev) +static int ina3221_suspend(struct device *dev) { struct ina3221_data *ina = dev_get_drvdata(dev); int ret; @@ -953,7 +951,7 @@ static int __maybe_unused ina3221_suspend(struct device *dev) return 0; } -static int __maybe_unused ina3221_resume(struct device *dev) +static int ina3221_resume(struct device *dev) { struct ina3221_data *ina = dev_get_drvdata(dev); int ret; @@ -996,11 +994,8 @@ static int __maybe_unused ina3221_resume(struct device *dev) return 0; } -static const struct dev_pm_ops ina3221_pm = { - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(ina3221_suspend, ina3221_resume, NULL) -}; +static DEFINE_RUNTIME_DEV_PM_OPS(ina3221_pm, ina3221_suspend, ina3221_resume, + NULL); static const struct of_device_id ina3221_of_match_table[] = { { .compatible = "ti,ina3221", }, @@ -1020,7 +1015,7 @@ static struct i2c_driver ina3221_i2c_driver = { .driver = { .name = INA3221_DRIVER_NAME, .of_match_table = ina3221_of_match_table, - .pm = &ina3221_pm, + .pm = pm_ptr(&ina3221_pm), }, .id_table = ina3221_ids, }; diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c index 0e543dbe0a6b3ece7c81c690a405432740350cd5..7bd154ba351b90fad179c94cc5b17c58a9e0f052 100644 --- a/drivers/hwmon/it87.c +++ b/drivers/hwmon/it87.c @@ -3179,7 +3179,7 @@ static int it87_probe(struct platform_device *pdev) return PTR_ERR_OR_ZERO(hwmon_dev); } -static void __maybe_unused it87_resume_sio(struct platform_device *pdev) +static void it87_resume_sio(struct platform_device *pdev) { struct it87_data *data = dev_get_drvdata(&pdev->dev); int err; @@ -3211,7 +3211,7 @@ static void __maybe_unused it87_resume_sio(struct platform_device *pdev) superio_exit(data->sioaddr); } -static int __maybe_unused it87_resume(struct device *dev) +static int it87_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct it87_data *data = dev_get_drvdata(dev); @@ -3238,12 +3238,12 @@ static int __maybe_unused it87_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(it87_dev_pm_ops, NULL, it87_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(it87_dev_pm_ops, NULL, it87_resume); static struct platform_driver it87_driver = { .driver = { .name = DRVNAME, - .pm = &it87_dev_pm_ops, + .pm = pm_sleep_ptr(&it87_dev_pm_ops), }, .probe = it87_probe, }; diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c index 07f7f8b5b73d74ddf6d45d60d8ffbd24dea62bf0..30888feaf589bf1f5c63523e13b821f2be0426b4 100644 --- a/drivers/hwmon/jc42.c +++ b/drivers/hwmon/jc42.c @@ -441,7 +441,7 @@ static int jc42_detect(struct i2c_client *client, struct i2c_board_info *info) struct jc42_chips *chip = &jc42_chips[i]; if (manid == chip->manid && (devid & chip->devid_mask) == chip->devid) { - strlcpy(info->type, "jc42", I2C_NAME_SIZE); + strscpy(info->type, "jc42", I2C_NAME_SIZE); return 0; } } @@ -524,7 +524,7 @@ static int jc42_probe(struct i2c_client *client) return PTR_ERR_OR_ZERO(hwmon_dev); } -static int jc42_remove(struct i2c_client *client) +static void jc42_remove(struct i2c_client *client) { struct jc42_data *data = i2c_get_clientdata(client); @@ -537,7 +537,6 @@ static int jc42_remove(struct i2c_client *client) | (data->config & JC42_CFG_HYST_MASK); i2c_smbus_write_word_swapped(client, JC42_REG_CONFIG, config); } - return 0; } #ifdef CONFIG_PM diff --git a/drivers/hwmon/lm63.c b/drivers/hwmon/lm63.c index 339a145afc093c06247818ce0277b217f6b6fe16..9ab2cab4c7106d00d7e329d30bd10d9c5693bb1f 100644 --- a/drivers/hwmon/lm63.c +++ b/drivers/hwmon/lm63.c @@ -996,11 +996,11 @@ static int lm63_detect(struct i2c_client *client, } if (chip_id == 0x41 && address == 0x4c) - strlcpy(info->type, "lm63", I2C_NAME_SIZE); + strscpy(info->type, "lm63", I2C_NAME_SIZE); else if (chip_id == 0x51 && (address == 0x18 || address == 0x4e)) - strlcpy(info->type, "lm64", I2C_NAME_SIZE); + strscpy(info->type, "lm64", I2C_NAME_SIZE); else if (chip_id == 0x49 && address == 0x4c) - strlcpy(info->type, "lm96163", I2C_NAME_SIZE); + strscpy(info->type, "lm96163", I2C_NAME_SIZE); else return -ENODEV; diff --git a/drivers/hwmon/lm73.c b/drivers/hwmon/lm73.c index beb0d61bcd821c9baef68eec568e7a3e6bf74004..1346b3b3f4635a8460ee99be69c9b8b69c5264db 100644 --- a/drivers/hwmon/lm73.c +++ b/drivers/hwmon/lm73.c @@ -257,7 +257,7 @@ static int lm73_detect(struct i2c_client *new_client, if (id < 0 || id != LM73_ID) return -ENODEV; - strlcpy(info->type, "lm73", I2C_NAME_SIZE); + strscpy(info->type, "lm73", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c index 66dc826f7962c69ac912f66a0f5f1f18118cbf4d..bcc3adcb3af1a5574b0f0ac1d2095b5e52cf117d 100644 --- a/drivers/hwmon/lm75.c +++ b/drivers/hwmon/lm75.c @@ -893,7 +893,7 @@ static int lm75_detect(struct i2c_client *new_client, return -ENODEV; } - strlcpy(info->type, is_lm75a ? "lm75a" : "lm75", I2C_NAME_SIZE); + strscpy(info->type, is_lm75a ? "lm75a" : "lm75", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm77.c b/drivers/hwmon/lm77.c index df6af85e170aa838380ad49d9e5caacea0db662c..645cb2191abe433ee25b6f302e766cfc70bfb929 100644 --- a/drivers/hwmon/lm77.c +++ b/drivers/hwmon/lm77.c @@ -302,7 +302,7 @@ static int lm77_detect(struct i2c_client *client, struct i2c_board_info *info) || i2c_smbus_read_word_data(client, 7) != min) return -ENODEV; - strlcpy(info->type, "lm77", I2C_NAME_SIZE); + strscpy(info->type, "lm77", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm78.c b/drivers/hwmon/lm78.c index 5e129cbec1cbbd90948c08ce87891a8ad0525054..694e171cab7f02700d8b741b247e4ec1681ebafc 100644 --- a/drivers/hwmon/lm78.c +++ b/drivers/hwmon/lm78.c @@ -617,7 +617,7 @@ static int lm78_i2c_detect(struct i2c_client *client, if (isa) mutex_unlock(&isa->update_lock); - strlcpy(info->type, client_name, I2C_NAME_SIZE); + strscpy(info->type, client_name, I2C_NAME_SIZE); return 0; diff --git a/drivers/hwmon/lm80.c b/drivers/hwmon/lm80.c index e85e062bbf327dbca2d4e41225cd1515e99b17bd..35db0b97f9128b174ec9ef7c7866fe445e893774 100644 --- a/drivers/hwmon/lm80.c +++ b/drivers/hwmon/lm80.c @@ -586,7 +586,7 @@ static int lm80_detect(struct i2c_client *client, struct i2c_board_info *info) name = "lm80"; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c index 905f5689f9074df702352e6507016b24893deaad..616449f2cc504367fc98a2e12b44e24a3daf660e 100644 --- a/drivers/hwmon/lm83.c +++ b/drivers/hwmon/lm83.c @@ -412,7 +412,7 @@ static int lm83_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c index 88cf2012d34b415911f673b14d3dc97b31f05cb1..8d33c24847553459ccd6e20c6b8bd5c37cfbbb92 100644 --- a/drivers/hwmon/lm85.c +++ b/drivers/hwmon/lm85.c @@ -1539,7 +1539,7 @@ static int lm85_detect(struct i2c_client *client, struct i2c_board_info *info) if (!type_name) return -ENODEV; - strlcpy(info->type, type_name, I2C_NAME_SIZE); + strscpy(info->type, type_name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm87.c b/drivers/hwmon/lm87.c index 1750bc588856c22011012cb07c325d05d4b77e11..818fb619524572ba4f167b312401123f115eb1c7 100644 --- a/drivers/hwmon/lm87.c +++ b/drivers/hwmon/lm87.c @@ -833,7 +833,7 @@ static int lm87_detect(struct i2c_client *client, struct i2c_board_info *info) return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c index 221de01a327aa5c8b99273442ef520754447191f..db595f7d01f8ad5f1c0fd1a1f0fd692ff1f2e644 100644 --- a/drivers/hwmon/lm90.c +++ b/drivers/hwmon/lm90.c @@ -2547,7 +2547,7 @@ static int lm90_detect(struct i2c_client *client, struct i2c_board_info *info) return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } @@ -2956,7 +2956,7 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type, } } -static int __maybe_unused lm90_suspend(struct device *dev) +static int lm90_suspend(struct device *dev) { struct lm90_data *data = dev_get_drvdata(dev); struct i2c_client *client = data->client; @@ -2967,7 +2967,7 @@ static int __maybe_unused lm90_suspend(struct device *dev) return 0; } -static int __maybe_unused lm90_resume(struct device *dev) +static int lm90_resume(struct device *dev) { struct lm90_data *data = dev_get_drvdata(dev); struct i2c_client *client = data->client; @@ -2978,14 +2978,14 @@ static int __maybe_unused lm90_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(lm90_pm_ops, lm90_suspend, lm90_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(lm90_pm_ops, lm90_suspend, lm90_resume); static struct i2c_driver lm90_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "lm90", .of_match_table = of_match_ptr(lm90_of_match), - .pm = &lm90_pm_ops, + .pm = pm_sleep_ptr(&lm90_pm_ops), }, .probe_new = lm90_probe, .alert = lm90_alert, diff --git a/drivers/hwmon/lm92.c b/drivers/hwmon/lm92.c index 5bae6eedcaf1f8c0f426c4e2d4c3bbac3248a331..2ff3044a677d896acc7b3b0c17994562f5d14cb1 100644 --- a/drivers/hwmon/lm92.c +++ b/drivers/hwmon/lm92.c @@ -287,7 +287,7 @@ static int lm92_detect(struct i2c_client *new_client, else return -ENODEV; - strlcpy(info->type, "lm92", I2C_NAME_SIZE); + strscpy(info->type, "lm92", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm93.c b/drivers/hwmon/lm93.c index dc67bf954b2164573b920374ed497703a1a481b8..4cf50d5f4f5949e90145f7818ef6cf8caac34550 100644 --- a/drivers/hwmon/lm93.c +++ b/drivers/hwmon/lm93.c @@ -2575,7 +2575,7 @@ static int lm93_detect(struct i2c_client *client, struct i2c_board_info *info) return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); dev_dbg(&adapter->dev, "loading %s at %d, 0x%02x\n", client->name, i2c_adapter_id(client->adapter), client->addr); diff --git a/drivers/hwmon/lm95234.c b/drivers/hwmon/lm95234.c index ac169a994ae007d3fc1b433af72de5ce9ac0be7e..b4a9d0c223c4a1a1d119935ed76bb5e1489e37ad 100644 --- a/drivers/hwmon/lm95234.c +++ b/drivers/hwmon/lm95234.c @@ -644,7 +644,7 @@ static int lm95234_detect(struct i2c_client *client, if (val & model_mask) return -ENODEV; - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm95241.c b/drivers/hwmon/lm95241.c index 8ea46ff20be5fac7f28fde75da56b412af667560..f1ed777a87353ccd5bea8fac6c2a9ade0410b28e 100644 --- a/drivers/hwmon/lm95241.c +++ b/drivers/hwmon/lm95241.c @@ -389,7 +389,7 @@ static int lm95241_detect(struct i2c_client *new_client, } /* Fill the i2c board info */ - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c index 29388fcf5f74d64617acaeb649dbe42a98e59af9..c433f0af2d3185fefca94430e1f74123eef1e92f 100644 --- a/drivers/hwmon/lm95245.c +++ b/drivers/hwmon/lm95245.c @@ -461,7 +461,7 @@ static int lm95245_detect(struct i2c_client *new_client, return -ENODEV; } - strlcpy(info->type, name, I2C_NAME_SIZE); + strscpy(info->type, name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/ltc2947-core.c b/drivers/hwmon/ltc2947-core.c index 5423466de697af5bf704217f26eaccea058f404d..7404e974762fd7e9cba7cee36e7e1bf2e2e02408 100644 --- a/drivers/hwmon/ltc2947-core.c +++ b/drivers/hwmon/ltc2947-core.c @@ -956,13 +956,6 @@ static struct attribute *ltc2947_attrs[] = { }; ATTRIBUTE_GROUPS(ltc2947); -static void ltc2947_clk_disable(void *data) -{ - struct clk *extclk = data; - - clk_disable_unprepare(extclk); -} - static int ltc2947_setup(struct ltc2947_data *st) { int ret; @@ -989,7 +982,7 @@ static int ltc2947_setup(struct ltc2947_data *st) return ret; /* check external clock presence */ - extclk = devm_clk_get_optional(st->dev, NULL); + extclk = devm_clk_get_optional_enabled(st->dev, NULL); if (IS_ERR(extclk)) return dev_err_probe(st->dev, PTR_ERR(extclk), "Failed to get external clock\n"); @@ -1007,14 +1000,6 @@ static int ltc2947_setup(struct ltc2947_data *st) return -EINVAL; } - ret = clk_prepare_enable(extclk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(st->dev, ltc2947_clk_disable, - extclk); - if (ret) - return ret; /* as in table 1 of the datasheet */ if (rate_hz >= LTC2947_CLK_MIN && rate_hz <= 1000000) pre = 0; @@ -1135,7 +1120,7 @@ int ltc2947_core_probe(struct regmap *map, const char *name) } EXPORT_SYMBOL_GPL(ltc2947_core_probe); -static int __maybe_unused ltc2947_resume(struct device *dev) +static int ltc2947_resume(struct device *dev) { struct ltc2947_data *st = dev_get_drvdata(dev); u32 ctrl = 0; @@ -1164,7 +1149,7 @@ static int __maybe_unused ltc2947_resume(struct device *dev) LTC2947_CONT_MODE_MASK, LTC2947_CONT_MODE(1)); } -static int __maybe_unused ltc2947_suspend(struct device *dev) +static int ltc2947_suspend(struct device *dev) { struct ltc2947_data *st = dev_get_drvdata(dev); @@ -1172,8 +1157,7 @@ static int __maybe_unused ltc2947_suspend(struct device *dev) LTC2947_SHUTDOWN_MASK, 1); } -SIMPLE_DEV_PM_OPS(ltc2947_pm_ops, ltc2947_suspend, ltc2947_resume); -EXPORT_SYMBOL_GPL(ltc2947_pm_ops); +EXPORT_SIMPLE_DEV_PM_OPS(ltc2947_pm_ops, ltc2947_suspend, ltc2947_resume); const struct of_device_id ltc2947_of_match[] = { { .compatible = "adi,ltc2947" }, diff --git a/drivers/hwmon/ltc2947-i2c.c b/drivers/hwmon/ltc2947-i2c.c index ad0dfd3efbf89c51405169be8f9de62478c20fec..96852bc8a96415cbde2d07615e27c62092acd56e 100644 --- a/drivers/hwmon/ltc2947-i2c.c +++ b/drivers/hwmon/ltc2947-i2c.c @@ -36,7 +36,7 @@ static struct i2c_driver ltc2947_driver = { .driver = { .name = "ltc2947", .of_match_table = ltc2947_of_match, - .pm = <c2947_pm_ops, + .pm = pm_sleep_ptr(<c2947_pm_ops), }, .probe_new = ltc2947_probe, .id_table = ltc2947_id, diff --git a/drivers/hwmon/ltc2947-spi.c b/drivers/hwmon/ltc2947-spi.c index c24ca569db1b2c12e19d62ec7bcc243db6982206..a33be110098cd0ed25b5c0bf656291ff2153a27b 100644 --- a/drivers/hwmon/ltc2947-spi.c +++ b/drivers/hwmon/ltc2947-spi.c @@ -38,7 +38,7 @@ static struct spi_driver ltc2947_driver = { .driver = { .name = "ltc2947", .of_match_table = ltc2947_of_match, - .pm = <c2947_pm_ops, + .pm = pm_sleep_ptr(<c2947_pm_ops), }, .probe = ltc2947_probe, .id_table = ltc2947_id, diff --git a/drivers/hwmon/max1619.c b/drivers/hwmon/max1619.c index eae9e68027bcbe7085a1cef803a746c79e8405b2..445c77197f69f12097c521e14396a6a6bc1e8ecb 100644 --- a/drivers/hwmon/max1619.c +++ b/drivers/hwmon/max1619.c @@ -241,7 +241,7 @@ static int max1619_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "max1619", I2C_NAME_SIZE); + strscpy(info->type, "max1619", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/max1668.c b/drivers/hwmon/max1668.c index 78688e6cb87d99be3094df1933c52b675e421706..9f748973d6a371d62aa15ead2b38e4afc911b36e 100644 --- a/drivers/hwmon/max1668.c +++ b/drivers/hwmon/max1668.c @@ -386,7 +386,7 @@ static int max1668_detect(struct i2c_client *client, if (!type_name) return -ENODEV; - strlcpy(info->type, type_name, I2C_NAME_SIZE); + strscpy(info->type, type_name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/max31722.c b/drivers/hwmon/max31722.c index 93e048ee49557c08acc1dd88ef381ade83750685..9a31ef3883969d8bd38b337951ac602181bc80bd 100644 --- a/drivers/hwmon/max31722.c +++ b/drivers/hwmon/max31722.c @@ -113,7 +113,7 @@ static void max31722_remove(struct spi_device *spi) dev_warn(&spi->dev, "Failed to put device in stand-by mode\n"); } -static int __maybe_unused max31722_suspend(struct device *dev) +static int max31722_suspend(struct device *dev) { struct spi_device *spi_device = to_spi_device(dev); struct max31722_data *data = spi_get_drvdata(spi_device); @@ -121,7 +121,7 @@ static int __maybe_unused max31722_suspend(struct device *dev) return max31722_set_mode(data, MAX31722_MODE_STANDBY); } -static int __maybe_unused max31722_resume(struct device *dev) +static int max31722_resume(struct device *dev) { struct spi_device *spi_device = to_spi_device(dev); struct max31722_data *data = spi_get_drvdata(spi_device); @@ -129,7 +129,7 @@ static int __maybe_unused max31722_resume(struct device *dev) return max31722_set_mode(data, MAX31722_MODE_CONTINUOUS); } -static SIMPLE_DEV_PM_OPS(max31722_pm_ops, max31722_suspend, max31722_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(max31722_pm_ops, max31722_suspend, max31722_resume); static const struct spi_device_id max31722_spi_id[] = { {"max31722", 0}, @@ -141,7 +141,7 @@ MODULE_DEVICE_TABLE(spi, max31722_spi_id); static struct spi_driver max31722_driver = { .driver = { .name = "max31722", - .pm = &max31722_pm_ops, + .pm = pm_sleep_ptr(&max31722_pm_ops), }, .probe = max31722_probe, .remove = max31722_remove, diff --git a/drivers/hwmon/max31730.c b/drivers/hwmon/max31730.c index 23598b8b879343e80cd9b098a21fd0420d7dc6c7..746a767c9fc663395a6d6e52472796847f5cefbd 100644 --- a/drivers/hwmon/max31730.c +++ b/drivers/hwmon/max31730.c @@ -399,33 +399,33 @@ static int max31730_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "max31730", I2C_NAME_SIZE); + strscpy(info->type, "max31730", I2C_NAME_SIZE); return 0; } -static int __maybe_unused max31730_suspend(struct device *dev) +static int max31730_suspend(struct device *dev) { struct max31730_data *data = dev_get_drvdata(dev); return max31730_write_config(data, MAX31730_STOP, 0); } -static int __maybe_unused max31730_resume(struct device *dev) +static int max31730_resume(struct device *dev) { struct max31730_data *data = dev_get_drvdata(dev); return max31730_write_config(data, 0, MAX31730_STOP); } -static SIMPLE_DEV_PM_OPS(max31730_pm_ops, max31730_suspend, max31730_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(max31730_pm_ops, max31730_suspend, max31730_resume); static struct i2c_driver max31730_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "max31730", .of_match_table = of_match_ptr(max31730_of_match), - .pm = &max31730_pm_ops, + .pm = pm_sleep_ptr(&max31730_pm_ops), }, .probe_new = max31730_probe, .id_table = max31730_ids, diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c new file mode 100644 index 0000000000000000000000000000000000000000..06d5f39dc33d30cffa16555d050fa5bfd31f5992 --- /dev/null +++ b/drivers/hwmon/max31760.c @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include + +#define REG_CR1 0x00 +#define CR1_HYST BIT(5) +#define CR1_DRV GENMASK(4, 3) +#define CR1_TEMP_SRC GENMASK(1, 0) +#define REG_CR2 0x01 +#define CR2_STBY BIT(7) +#define CR2_ALERTS BIT(6) +#define CR2_DFC BIT(0) +#define REG_CR3 0x02 +#define REG_PWMR 0x50 +#define REG_PWMV 0x51 +#define REG_STATUS 0x5A +#define STATUS_ALARM_CRIT(ch) BIT(2 + 2 * (ch)) +#define STATUS_ALARM_MAX(ch) BIT(3 + 2 * (ch)) +#define STATUS_RDFA BIT(6) + +#define REG_TACH(ch) (0x52 + (ch) * 2) +#define REG_TEMP_INPUT(ch) (0x56 + (ch) * 2) +#define REG_TEMP_MAX(ch) (0x06 + (ch) * 2) +#define REG_TEMP_CRIT(ch) (0x0A + (ch) * 2) + +#define TEMP11_FROM_REG(reg) ((reg) / 32 * 125) +#define TEMP11_TO_REG(val) (DIV_ROUND_CLOSEST(clamp_val((val), -128000, \ + 127875), 125) * 32) + +#define LUT_SIZE 48 + +#define REG_LUT(index) (0x20 + (index)) + +struct max31760_state { + struct regmap *regmap; + + struct lut_attribute { + char name[24]; + struct sensor_device_attribute sda; + } lut[LUT_SIZE]; + + struct attribute *attrs[LUT_SIZE + 2]; + struct attribute_group group; + const struct attribute_group *groups[2]; +}; + +static bool max31760_volatile_reg(struct device *dev, unsigned int reg) +{ + return reg > 0x50; +} + +static const struct regmap_config regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = 0x5B, + .cache_type = REGCACHE_RBTREE, + .volatile_reg = max31760_volatile_reg, +}; + +static const int max31760_pwm_freq[] = {33, 150, 1500, 25000}; + +static int tach_to_rpm(u16 tach) +{ + if (tach == 0) + tach = 1; + + return 60 * 100000 / tach / 2; +} + +static int max31760_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct max31760_state *state = dev_get_drvdata(dev); + unsigned int regval; + unsigned int reg_temp; + s16 temp; + u8 reg[2]; + int ret; + + switch (type) { + case hwmon_temp: + switch (attr) { + case hwmon_temp_fault: + ret = regmap_read(state->regmap, REG_STATUS, ®val); + if (ret) + return ret; + + *val = FIELD_GET(STATUS_RDFA, regval); + + return 0; + case hwmon_temp_max_alarm: + ret = regmap_read(state->regmap, REG_STATUS, ®val); + if (ret) + return ret; + + if (channel) + *val = FIELD_GET(STATUS_ALARM_MAX(1), regval); + else + *val = FIELD_GET(STATUS_ALARM_MAX(0), regval); + + return 0; + case hwmon_temp_crit_alarm: + ret = regmap_read(state->regmap, REG_STATUS, ®val); + if (ret) + return ret; + + if (channel) + *val = FIELD_GET(STATUS_ALARM_CRIT(1), regval); + else + *val = FIELD_GET(STATUS_ALARM_CRIT(0), regval); + + return 0; + case hwmon_temp_input: + reg_temp = REG_TEMP_INPUT(channel); + break; + case hwmon_temp_max: + reg_temp = REG_TEMP_MAX(channel); + break; + case hwmon_temp_crit: + reg_temp = REG_TEMP_CRIT(channel); + break; + default: + return -EOPNOTSUPP; + } + + ret = regmap_bulk_read(state->regmap, reg_temp, reg, 2); + if (ret) + return ret; + + temp = (reg[0] << 8) | reg[1]; + + *val = TEMP11_FROM_REG(temp); + + return 0; + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + ret = regmap_bulk_read(state->regmap, REG_TACH(channel), reg, 2); + if (ret) + return ret; + + *val = tach_to_rpm(reg[0] * 256 + reg[1]); + + return 0; + case hwmon_fan_fault: + ret = regmap_read(state->regmap, REG_STATUS, ®val); + if (ret) + return ret; + + if (channel) + *val = FIELD_GET(BIT(1), regval); + else + *val = FIELD_GET(BIT(0), regval); + + return 0; + case hwmon_fan_enable: + ret = regmap_read(state->regmap, REG_CR3, ®val); + if (ret) + return ret; + + if (channel) + *val = FIELD_GET(BIT(1), regval); + else + *val = FIELD_GET(BIT(0), regval); + + return 0; + default: + return -EOPNOTSUPP; + } + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + ret = regmap_read(state->regmap, REG_PWMV, ®val); + if (ret) + return ret; + + *val = regval; + + return 0; + case hwmon_pwm_freq: + ret = regmap_read(state->regmap, REG_CR1, ®val); + if (ret) + return ret; + + regval = FIELD_GET(CR1_DRV, regval); + if (regval >= ARRAY_SIZE(max31760_pwm_freq)) + return -EINVAL; + + *val = max31760_pwm_freq[regval]; + + return 0; + case hwmon_pwm_enable: + ret = regmap_read(state->regmap, REG_CR2, ®val); + if (ret) + return ret; + + *val = 2 - FIELD_GET(CR2_DFC, regval); + + return 0; + case hwmon_pwm_auto_channels_temp: + ret = regmap_read(state->regmap, REG_CR1, ®val); + if (ret) + return ret; + + switch (FIELD_GET(CR1_TEMP_SRC, regval)) { + case 0: + *val = 2; + break; + case 1: + *val = 1; + break; + case 2: + case 3: + *val = 3; + break; + default: + return -EINVAL; + } + + return 0; + default: + return -EOPNOTSUPP; + } + default: + return -EOPNOTSUPP; + } +} + +static int max31760_write(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long val) +{ + struct max31760_state *state = dev_get_drvdata(dev); + unsigned int pwm_index; + unsigned int reg_temp; + int temp; + u8 reg_val[2]; + + switch (type) { + case hwmon_temp: + switch (attr) { + case hwmon_temp_max: + reg_temp = REG_TEMP_MAX(channel); + break; + case hwmon_temp_crit: + reg_temp = REG_TEMP_CRIT(channel); + break; + default: + return -EOPNOTSUPP; + } + + temp = TEMP11_TO_REG(val); + reg_val[0] = temp >> 8; + reg_val[1] = temp & 0xFF; + + return regmap_bulk_write(state->regmap, reg_temp, reg_val, 2); + case hwmon_fan: + switch (attr) { + case hwmon_fan_enable: + if (val == 0) + return regmap_clear_bits(state->regmap, REG_CR3, BIT(channel)); + + if (val == 1) + return regmap_set_bits(state->regmap, REG_CR3, BIT(channel)); + + return -EINVAL; + default: + return -EOPNOTSUPP; + } + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_input: + if (val < 0 || val > 255) + return -EINVAL; + + return regmap_write(state->regmap, REG_PWMR, val); + case hwmon_pwm_enable: + if (val == 1) + return regmap_set_bits(state->regmap, REG_CR2, CR2_DFC); + + if (val == 2) + return regmap_clear_bits(state->regmap, REG_CR2, CR2_DFC); + + return -EINVAL; + case hwmon_pwm_freq: + pwm_index = find_closest(val, max31760_pwm_freq, + ARRAY_SIZE(max31760_pwm_freq)); + + return regmap_update_bits(state->regmap, + REG_CR1, CR1_DRV, + FIELD_PREP(CR1_DRV, pwm_index)); + case hwmon_pwm_auto_channels_temp: + switch (val) { + case 1: + break; + case 2: + val = 0; + break; + case 3: + val = 2; + break; + default: + return -EINVAL; + } + + return regmap_update_bits(state->regmap, REG_CR1, CR1_TEMP_SRC, val); + default: + return -EOPNOTSUPP; + } + default: + return -EOPNOTSUPP; + } +} + +static const struct hwmon_channel_info *max31760_info[] = { + HWMON_CHANNEL_INFO(chip, + HWMON_C_REGISTER_TZ), + HWMON_CHANNEL_INFO(fan, + HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_ENABLE), + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_FAULT | + HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_LABEL, + HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | + HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_LABEL), + HWMON_CHANNEL_INFO(pwm, + HWMON_PWM_ENABLE | HWMON_PWM_FREQ | HWMON_PWM_INPUT | + HWMON_PWM_AUTO_CHANNELS_TEMP), + NULL +}; + +static umode_t max31760_is_visible(const void *data, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + switch (type) { + case hwmon_temp: + switch (attr) { + case hwmon_temp_input: + case hwmon_temp_max_alarm: + case hwmon_temp_crit_alarm: + case hwmon_temp_fault: + case hwmon_temp_label: + return 0444; + case hwmon_temp_max: + case hwmon_temp_crit: + return 0644; + default: + return 0; + } + case hwmon_fan: + switch (attr) { + case hwmon_fan_input: + case hwmon_fan_fault: + return 0444; + case hwmon_fan_enable: + return 0644; + default: + return 0; + } + case hwmon_pwm: + switch (attr) { + case hwmon_pwm_enable: + case hwmon_pwm_input: + case hwmon_pwm_freq: + case hwmon_pwm_auto_channels_temp: + return 0644; + default: + return 0; + } + default: + return 0; + } +} + +static int max31760_read_string(struct device *dev, + enum hwmon_sensor_types type, + u32 attr, int channel, const char **str) +{ + switch (type) { + case hwmon_temp: + if (attr != hwmon_temp_label) + return -EOPNOTSUPP; + + *str = channel ? "local" : "remote"; + + return 0; + default: + return -EOPNOTSUPP; + } +} + +static const struct hwmon_ops max31760_hwmon_ops = { + .is_visible = max31760_is_visible, + .read = max31760_read, + .write = max31760_write, + .read_string = max31760_read_string +}; + +static const struct hwmon_chip_info max31760_chip_info = { + .ops = &max31760_hwmon_ops, + .info = max31760_info, +}; + +static ssize_t lut_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *sda = to_sensor_dev_attr(devattr); + struct max31760_state *state = dev_get_drvdata(dev); + int ret; + unsigned int regval; + + ret = regmap_read(state->regmap, REG_LUT(sda->index), ®val); + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", regval); +} + +static ssize_t lut_store(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct sensor_device_attribute *sda = to_sensor_dev_attr(devattr); + struct max31760_state *state = dev_get_drvdata(dev); + int ret; + u8 pwm; + + ret = kstrtou8(buf, 10, &pwm); + if (ret) + return ret; + + ret = regmap_write(state->regmap, REG_LUT(sda->index), pwm); + if (ret) + return ret; + + return count; +} + +static ssize_t pwm1_auto_point_temp_hyst_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct max31760_state *state = dev_get_drvdata(dev); + unsigned int regval; + int ret; + + ret = regmap_read(state->regmap, REG_CR1, ®val); + if (ret) + return ret; + + return sysfs_emit(buf, "%d\n", (1 + (int)FIELD_GET(CR1_HYST, regval)) * 2000); +} + +static ssize_t pwm1_auto_point_temp_hyst_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct max31760_state *state = dev_get_drvdata(dev); + unsigned int hyst; + int ret; + + ret = kstrtou32(buf, 10, &hyst); + if (ret) + return ret; + + if (hyst < 3000) + ret = regmap_clear_bits(state->regmap, REG_CR1, CR1_HYST); + else + ret = regmap_set_bits(state->regmap, REG_CR1, CR1_HYST); + + if (ret) + return ret; + + return count; +} + +static DEVICE_ATTR_RW(pwm1_auto_point_temp_hyst); + +static void max31760_create_lut_nodes(struct max31760_state *state) +{ + int i; + struct sensor_device_attribute *sda; + struct lut_attribute *lut; + + for (i = 0; i < LUT_SIZE; ++i) { + lut = &state->lut[i]; + sda = &lut->sda; + + snprintf(lut->name, sizeof(lut->name), + "pwm1_auto_point%d_pwm", i + 1); + + sda->dev_attr.attr.mode = 0644; + sda->index = i; + sda->dev_attr.show = lut_show; + sda->dev_attr.store = lut_store; + sda->dev_attr.attr.name = lut->name; + + sysfs_attr_init(&sda->dev_attr.attr); + + state->attrs[i] = &sda->dev_attr.attr; + } + + state->attrs[i] = &dev_attr_pwm1_auto_point_temp_hyst.attr; + + state->group.attrs = state->attrs; + state->groups[0] = &state->group; +} + +static int max31760_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct max31760_state *state; + struct device *hwmon_dev; + int ret; + + state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + state->regmap = devm_regmap_init_i2c(client, ®map_config); + if (IS_ERR(state->regmap)) + return dev_err_probe(dev, + PTR_ERR(state->regmap), + "regmap initialization failed\n"); + + dev_set_drvdata(dev, state); + + /* Set alert output to comparator mode */ + ret = regmap_set_bits(state->regmap, REG_CR2, CR2_ALERTS); + if (ret) + return dev_err_probe(dev, ret, "cannot write register\n"); + + max31760_create_lut_nodes(state); + + hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, + state, + &max31760_chip_info, + state->groups); + + return PTR_ERR_OR_ZERO(hwmon_dev); +} + +static const struct of_device_id max31760_of_match[] = { + {.compatible = "adi,max31760"}, + { } +}; +MODULE_DEVICE_TABLE(of, max31760_of_match); + +static const struct i2c_device_id max31760_id[] = { + {"max31760"}, + { } +}; +MODULE_DEVICE_TABLE(i2c, max31760_id); + +static int max31760_suspend(struct device *dev) +{ + struct max31760_state *state = dev_get_drvdata(dev); + + return regmap_set_bits(state->regmap, REG_CR2, CR2_STBY); +} + +static int max31760_resume(struct device *dev) +{ + struct max31760_state *state = dev_get_drvdata(dev); + + return regmap_clear_bits(state->regmap, REG_CR2, CR2_STBY); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(max31760_pm_ops, max31760_suspend, + max31760_resume); + +static struct i2c_driver max31760_driver = { + .class = I2C_CLASS_HWMON, + .driver = { + .name = "max31760", + .of_match_table = max31760_of_match, + .pm = pm_ptr(&max31760_pm_ops) + }, + .probe_new = max31760_probe, + .id_table = max31760_id +}; +module_i2c_driver(max31760_driver); + +MODULE_AUTHOR("Ibrahim Tilki "); +MODULE_DESCRIPTION("Analog Devices MAX31760 Fan Speed Controller"); +MODULE_SOFTDEP("pre: regmap_i2c"); +MODULE_VERSION("1.0"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c index 7e9362f6dc29ee9c6ed79bd5c9973cedc43e937c..20bf5ffadefead89ec674bedb141ccf93e5e0c68 100644 --- a/drivers/hwmon/max31790.c +++ b/drivers/hwmon/max31790.c @@ -202,6 +202,9 @@ static int max31790_read_fan(struct device *dev, u32 attr, int channel, } mutex_unlock(&data->update_lock); return 0; + case hwmon_fan_enable: + *val = !!(data->fan_config[channel] & MAX31790_FAN_CFG_TACH_INPUT_EN); + return 0; default: return -EOPNOTSUPP; } @@ -214,7 +217,7 @@ static int max31790_write_fan(struct device *dev, u32 attr, int channel, struct i2c_client *client = data->client; int target_count; int err = 0; - u8 bits; + u8 bits, fan_config; int sr; mutex_lock(&data->update_lock); @@ -243,6 +246,23 @@ static int max31790_write_fan(struct device *dev, u32 attr, int channel, MAX31790_REG_TARGET_COUNT(channel), data->target_count[channel]); break; + case hwmon_fan_enable: + fan_config = data->fan_config[channel]; + if (val == 0) { + fan_config &= ~MAX31790_FAN_CFG_TACH_INPUT_EN; + } else if (val == 1) { + fan_config |= MAX31790_FAN_CFG_TACH_INPUT_EN; + } else { + err = -EINVAL; + break; + } + if (fan_config != data->fan_config[channel]) { + err = i2c_smbus_write_byte_data(client, MAX31790_REG_FAN_CONFIG(channel), + fan_config); + if (!err) + data->fan_config[channel] = fan_config; + } + break; default: err = -EOPNOTSUPP; break; @@ -270,6 +290,10 @@ static umode_t max31790_fan_is_visible(const void *_data, u32 attr, int channel) !(fan_config & MAX31790_FAN_CFG_TACH_INPUT)) return 0644; return 0; + case hwmon_fan_enable: + if (channel < NR_CHANNEL) + return 0644; + return 0; default: return 0; } @@ -423,12 +447,12 @@ static umode_t max31790_is_visible(const void *data, static const struct hwmon_channel_info *max31790_info[] = { HWMON_CHANNEL_INFO(fan, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, - HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, + HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE, HWMON_F_INPUT | HWMON_F_FAULT, HWMON_F_INPUT | HWMON_F_FAULT, HWMON_F_INPUT | HWMON_F_FAULT, diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c index 14bb7726f8d7e18582e4404323851bc43a498a0a..9b895402c80d386523d4a71d18df79a0ce2d8b24 100644 --- a/drivers/hwmon/max6639.c +++ b/drivers/hwmon/max6639.c @@ -514,7 +514,7 @@ static int max6639_detect(struct i2c_client *client, if (dev_id != 0x58 || manu_id != 0x4D) return -ENODEV; - strlcpy(info->type, "max6639", I2C_NAME_SIZE); + strscpy(info->type, "max6639", I2C_NAME_SIZE); return 0; } @@ -571,7 +571,6 @@ static int max6639_probe(struct i2c_client *client) return PTR_ERR_OR_ZERO(hwmon_dev); } -#ifdef CONFIG_PM_SLEEP static int max6639_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); @@ -609,7 +608,6 @@ static int max6639_resume(struct device *dev) return i2c_smbus_write_byte_data(client, MAX6639_REG_GCONFIG, ret & ~MAX6639_GCONFIG_STANDBY); } -#endif /* CONFIG_PM_SLEEP */ static const struct i2c_device_id max6639_id[] = { {"max6639", 0}, @@ -618,13 +616,13 @@ static const struct i2c_device_id max6639_id[] = { MODULE_DEVICE_TABLE(i2c, max6639_id); -static SIMPLE_DEV_PM_OPS(max6639_pm_ops, max6639_suspend, max6639_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(max6639_pm_ops, max6639_suspend, max6639_resume); static struct i2c_driver max6639_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "max6639", - .pm = &max6639_pm_ops, + .pm = pm_sleep_ptr(&max6639_pm_ops), }, .probe_new = max6639_probe, .id_table = max6639_id, diff --git a/drivers/hwmon/max6642.c b/drivers/hwmon/max6642.c index 699d265aae2e718d1fc40b50cad33cb645872124..47ea34ff78f3d1a5a677595e702158dfd585b8cc 100644 --- a/drivers/hwmon/max6642.c +++ b/drivers/hwmon/max6642.c @@ -148,7 +148,7 @@ static int max6642_detect(struct i2c_client *client, if ((reg_status & 0x2b) != 0x00) return -ENODEV; - strlcpy(info->type, "max6642", I2C_NAME_SIZE); + strscpy(info->type, "max6642", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/mr75203.c b/drivers/hwmon/mr75203.c index 26278b0f17a989ac0a6a23bcd5dc8d5143bd46e6..394a4c7e46abcc8c39eba6c71128f03375b5f556 100644 --- a/drivers/hwmon/mr75203.c +++ b/drivers/hwmon/mr75203.c @@ -9,6 +9,7 @@ */ #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include #include /* PVT Common register */ @@ -30,6 +32,8 @@ #define CH_NUM_MSK GENMASK(31, 24) #define CH_NUM_SFT 24 +#define VM_NUM_MAX (VM_NUM_MSK >> VM_NUM_SFT) + /* Macro Common Register */ #define CLK_SYNTH 0x00 #define CLK_SYNTH_LO_SFT 0 @@ -68,8 +72,9 @@ /* VM Individual Macro Register */ #define VM_COM_REG_SIZE 0x200 -#define VM_SDIF_DONE(n) (VM_COM_REG_SIZE + 0x34 + 0x200 * (n)) -#define VM_SDIF_DATA(n) (VM_COM_REG_SIZE + 0x40 + 0x200 * (n)) +#define VM_SDIF_DONE(vm) (VM_COM_REG_SIZE + 0x34 + 0x200 * (vm)) +#define VM_SDIF_DATA(vm, ch) \ + (VM_COM_REG_SIZE + 0x40 + 0x200 * (vm) + 0x4 * (ch)) /* SDA Slave Register */ #define IP_CTRL 0x00 @@ -98,13 +103,67 @@ #define PVT_POLL_DELAY_US 20 #define PVT_POLL_TIMEOUT_US 20000 -#define PVT_H_CONST 100000 -#define PVT_CAL5_CONST 2047 -#define PVT_G_CONST 40000 #define PVT_CONV_BITS 10 #define PVT_N_CONST 90 #define PVT_R_CONST 245805 +#define PVT_TEMP_MIN_mC -40000 +#define PVT_TEMP_MAX_mC 125000 + +/* Temperature coefficients for series 5 */ +#define PVT_SERIES5_H_CONST 200000 +#define PVT_SERIES5_G_CONST 60000 +#define PVT_SERIES5_J_CONST -100 +#define PVT_SERIES5_CAL5_CONST 4094 + +/* Temperature coefficients for series 6 */ +#define PVT_SERIES6_H_CONST 249400 +#define PVT_SERIES6_G_CONST 57400 +#define PVT_SERIES6_J_CONST 0 +#define PVT_SERIES6_CAL5_CONST 4096 + +#define TEMPERATURE_SENSOR_SERIES_5 5 +#define TEMPERATURE_SENSOR_SERIES_6 6 + +#define PRE_SCALER_X1 1 +#define PRE_SCALER_X2 2 + +/** + * struct voltage_device - VM single input parameters. + * @vm_map: Map channel number to VM index. + * @ch_map: Map channel number to channel index. + * @pre_scaler: Pre scaler value (1 or 2) used to normalize the voltage output + * result. + * + * The structure provides mapping between channel-number (0..N-1) to VM-index + * (0..num_vm-1) and channel-index (0..ch_num-1) where N = num_vm * ch_num. + * It also provides normalization factor for the VM equation. + */ +struct voltage_device { + u32 vm_map; + u32 ch_map; + u32 pre_scaler; +}; + +/** + * struct voltage_channels - VM channel count. + * @total: Total number of channels in all VMs. + * @max: Maximum number of channels among all VMs. + * + * The structure provides channel count information across all VMs. + */ +struct voltage_channels { + u32 total; + u8 max; +}; + +struct temp_coeff { + u32 h; + u32 g; + u32 cal5; + s32 j; +}; + struct pvt_device { struct regmap *c_map; struct regmap *t_map; @@ -112,13 +171,74 @@ struct pvt_device { struct regmap *v_map; struct clk *clk; struct reset_control *rst; + struct dentry *dbgfs_dir; + struct voltage_device *vd; + struct voltage_channels vm_channels; + struct temp_coeff ts_coeff; u32 t_num; u32 p_num; u32 v_num; u32 ip_freq; - u8 *vm_idx; }; +static ssize_t pvt_ts_coeff_j_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pvt_device *pvt = file->private_data; + unsigned int len; + char buf[13]; + + len = scnprintf(buf, sizeof(buf), "%d\n", pvt->ts_coeff.j); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static ssize_t pvt_ts_coeff_j_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pvt_device *pvt = file->private_data; + int ret; + + ret = kstrtos32_from_user(user_buf, count, 0, &pvt->ts_coeff.j); + if (ret) + return ret; + + return count; +} + +static const struct file_operations pvt_ts_coeff_j_fops = { + .read = pvt_ts_coeff_j_read, + .write = pvt_ts_coeff_j_write, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static void devm_pvt_ts_dbgfs_remove(void *data) +{ + struct pvt_device *pvt = (struct pvt_device *)data; + + debugfs_remove_recursive(pvt->dbgfs_dir); + pvt->dbgfs_dir = NULL; +} + +static int pvt_ts_dbgfs_create(struct pvt_device *pvt, struct device *dev) +{ + pvt->dbgfs_dir = debugfs_create_dir(dev_name(dev), NULL); + + debugfs_create_u32("ts_coeff_h", 0644, pvt->dbgfs_dir, + &pvt->ts_coeff.h); + debugfs_create_u32("ts_coeff_g", 0644, pvt->dbgfs_dir, + &pvt->ts_coeff.g); + debugfs_create_u32("ts_coeff_cal5", 0644, pvt->dbgfs_dir, + &pvt->ts_coeff.cal5); + debugfs_create_file("ts_coeff_j", 0644, pvt->dbgfs_dir, pvt, + &pvt_ts_coeff_j_fops); + + return devm_add_action_or_reset(dev, devm_pvt_ts_dbgfs_remove, pvt); +} + static umode_t pvt_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel) { @@ -137,13 +257,28 @@ static umode_t pvt_is_visible(const void *data, enum hwmon_sensor_types type, return 0; } +static long pvt_calc_temp(struct pvt_device *pvt, u32 nbs) +{ + /* + * Convert the register value to degrees centigrade temperature: + * T = G + H * (n / cal5 - 0.5) + J * F + */ + struct temp_coeff *ts_coeff = &pvt->ts_coeff; + + s64 tmp = ts_coeff->g + + div_s64(ts_coeff->h * (s64)nbs, ts_coeff->cal5) - + ts_coeff->h / 2 + + div_s64(ts_coeff->j * (s64)pvt->ip_freq, HZ_PER_MHZ); + + return clamp_val(tmp, PVT_TEMP_MIN_mC, PVT_TEMP_MAX_mC); +} + static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val) { struct pvt_device *pvt = dev_get_drvdata(dev); struct regmap *t_map = pvt->t_map; u32 stat, nbs; int ret; - u64 tmp; switch (attr) { case hwmon_temp_input: @@ -155,7 +290,7 @@ static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val) return ret; ret = regmap_read(t_map, SDIF_DATA(channel), &nbs); - if(ret < 0) + if (ret < 0) return ret; nbs &= SAMPLE_DATA_MSK; @@ -164,9 +299,7 @@ static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val) * Convert the register value to * degrees centigrade temperature */ - tmp = nbs * PVT_H_CONST; - do_div(tmp, PVT_CAL5_CONST); - *val = tmp - PVT_G_CONST - pvt->ip_freq; + *val = pvt_calc_temp(pvt, nbs); return 0; default: @@ -178,14 +311,15 @@ static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val) { struct pvt_device *pvt = dev_get_drvdata(dev); struct regmap *v_map = pvt->v_map; - u32 n, stat; - u8 vm_idx; + u32 n, stat, pre_scaler; + u8 vm_idx, ch_idx; int ret; - if (channel >= pvt->v_num) + if (channel >= pvt->vm_channels.total) return -EINVAL; - vm_idx = pvt->vm_idx[channel]; + vm_idx = pvt->vd[channel].vm_map; + ch_idx = pvt->vd[channel].ch_map; switch (attr) { case hwmon_in_input: @@ -196,13 +330,25 @@ static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val) if (ret) return ret; - ret = regmap_read(v_map, VM_SDIF_DATA(vm_idx), &n); - if(ret < 0) + ret = regmap_read(v_map, VM_SDIF_DATA(vm_idx, ch_idx), &n); + if (ret < 0) return ret; n &= SAMPLE_DATA_MSK; - /* Convert the N bitstream count into voltage */ - *val = (PVT_N_CONST * n - PVT_R_CONST) >> PVT_CONV_BITS; + pre_scaler = pvt->vd[channel].pre_scaler; + /* + * Convert the N bitstream count into voltage. + * To support negative voltage calculation for 64bit machines + * n must be cast to long, since n and *val differ both in + * signedness and in size. + * Division is used instead of right shift, because for signed + * numbers, the sign bit is used to fill the vacated bit + * positions, and if the number is negative, 1 is used. + * BIT(x) may not be used instead of (1 << x) because it's + * unsigned. + */ + *val = pre_scaler * (PVT_N_CONST * (long)n - PVT_R_CONST) / + (1 << PVT_CONV_BITS); return 0; default: @@ -277,23 +423,23 @@ static int pvt_init(struct pvt_device *pvt) (key >> 1) << CLK_SYNTH_HI_SFT | (key >> 1) << CLK_SYNTH_HOLD_SFT | CLK_SYNTH_EN; - pvt->ip_freq = sys_freq * 100 / (key + 2); + pvt->ip_freq = clk_get_rate(pvt->clk) / (key + 2); if (t_num) { ret = regmap_write(t_map, SDIF_SMPL_CTRL, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(t_map, SDIF_HALT, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(t_map, CLK_SYNTH, clk_synth); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(t_map, SDIF_DISABLE, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_read_poll_timeout(t_map, SDIF_STAT, @@ -306,7 +452,7 @@ static int pvt_init(struct pvt_device *pvt) val = CFG0_MODE_2 | CFG0_PARALLEL_OUT | CFG0_12_BIT | IP_CFG << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(t_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_read_poll_timeout(t_map, SDIF_STAT, @@ -319,7 +465,7 @@ static int pvt_init(struct pvt_device *pvt) val = POWER_DELAY_CYCLE_256 | IP_TMR << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(t_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_read_poll_timeout(t_map, SDIF_STAT, @@ -333,39 +479,52 @@ static int pvt_init(struct pvt_device *pvt) IP_CTRL << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(t_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; } if (p_num) { ret = regmap_write(p_map, SDIF_HALT, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(p_map, SDIF_DISABLE, BIT(p_num) - 1); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(p_map, CLK_SYNTH, clk_synth); - if(ret < 0) + if (ret < 0) return ret; } if (v_num) { ret = regmap_write(v_map, SDIF_SMPL_CTRL, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(v_map, SDIF_HALT, 0x0); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(v_map, CLK_SYNTH, clk_synth); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_write(v_map, SDIF_DISABLE, 0x0); - if(ret < 0) + if (ret < 0) + return ret; + + ret = regmap_read_poll_timeout(v_map, SDIF_STAT, + val, !(val & SDIF_BUSY), + PVT_POLL_DELAY_US, + PVT_POLL_TIMEOUT_US); + if (ret) + return ret; + + val = (BIT(pvt->vm_channels.max) - 1) | VM_CH_INIT | + IP_POLL << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; + ret = regmap_write(v_map, SDIF_W, val); + if (ret < 0) return ret; ret = regmap_read_poll_timeout(v_map, SDIF_STAT, @@ -379,7 +538,7 @@ static int pvt_init(struct pvt_device *pvt) CFG1_14_BIT | IP_CFG << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(v_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_read_poll_timeout(v_map, SDIF_STAT, @@ -392,7 +551,7 @@ static int pvt_init(struct pvt_device *pvt) val = POWER_DELAY_CYCLE_64 | IP_TMR << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(v_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; ret = regmap_read_poll_timeout(v_map, SDIF_STAT, @@ -406,7 +565,7 @@ static int pvt_init(struct pvt_device *pvt) IP_CTRL << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG; ret = regmap_write(v_map, SDIF_W, val); - if(ret < 0) + if (ret < 0) return ret; } @@ -451,46 +610,163 @@ static int pvt_get_regmap(struct platform_device *pdev, char *reg_name, return 0; } -static void pvt_clk_disable(void *data) +static void pvt_reset_control_assert(void *data) { struct pvt_device *pvt = data; - clk_disable_unprepare(pvt->clk); + reset_control_assert(pvt->rst); } -static int pvt_clk_enable(struct device *dev, struct pvt_device *pvt) +static int pvt_reset_control_deassert(struct device *dev, struct pvt_device *pvt) { int ret; - ret = clk_prepare_enable(pvt->clk); + ret = reset_control_deassert(pvt->rst); if (ret) return ret; - return devm_add_action_or_reset(dev, pvt_clk_disable, pvt); + return devm_add_action_or_reset(dev, pvt_reset_control_assert, pvt); } -static void pvt_reset_control_assert(void *data) +static int pvt_get_active_channel(struct device *dev, struct pvt_device *pvt, + u32 vm_num, u32 ch_num, u8 *vm_idx) { - struct pvt_device *pvt = data; + u8 vm_active_ch[VM_NUM_MAX]; + int ret, i, j, k; - reset_control_assert(pvt->rst); + ret = device_property_read_u8_array(dev, "moortec,vm-active-channels", + vm_active_ch, vm_num); + if (ret) { + /* + * Incase "moortec,vm-active-channels" property is not defined, + * we assume each VM sensor has all of its channels active. + */ + memset(vm_active_ch, ch_num, vm_num); + pvt->vm_channels.max = ch_num; + pvt->vm_channels.total = ch_num * vm_num; + } else { + for (i = 0; i < vm_num; i++) { + if (vm_active_ch[i] > ch_num) { + dev_err(dev, "invalid active channels: %u\n", + vm_active_ch[i]); + return -EINVAL; + } + + pvt->vm_channels.total += vm_active_ch[i]; + + if (vm_active_ch[i] > pvt->vm_channels.max) + pvt->vm_channels.max = vm_active_ch[i]; + } + } + + /* + * Map between the channel-number to VM-index and channel-index. + * Example - 3 VMs, "moortec,vm_active_ch" = <5 2 4>: + * vm_map = [0 0 0 0 0 1 1 2 2 2 2] + * ch_map = [0 1 2 3 4 0 1 0 1 2 3] + */ + pvt->vd = devm_kcalloc(dev, pvt->vm_channels.total, sizeof(*pvt->vd), + GFP_KERNEL); + if (!pvt->vd) + return -ENOMEM; + + k = 0; + for (i = 0; i < vm_num; i++) { + for (j = 0; j < vm_active_ch[i]; j++) { + pvt->vd[k].vm_map = vm_idx[i]; + pvt->vd[k].ch_map = j; + k++; + } + } + + return 0; } -static int pvt_reset_control_deassert(struct device *dev, struct pvt_device *pvt) +static int pvt_get_pre_scaler(struct device *dev, struct pvt_device *pvt) { + u8 *pre_scaler_ch_list; + int i, ret, num_ch; + u32 channel; + + /* Set default pre-scaler value to be 1. */ + for (i = 0; i < pvt->vm_channels.total; i++) + pvt->vd[i].pre_scaler = PRE_SCALER_X1; + + /* Get number of channels configured in "moortec,vm-pre-scaler-x2". */ + num_ch = device_property_count_u8(dev, "moortec,vm-pre-scaler-x2"); + if (num_ch <= 0) + return 0; + + pre_scaler_ch_list = kcalloc(num_ch, sizeof(*pre_scaler_ch_list), + GFP_KERNEL); + if (!pre_scaler_ch_list) + return -ENOMEM; + + /* Get list of all channels that have pre-scaler of 2. */ + ret = device_property_read_u8_array(dev, "moortec,vm-pre-scaler-x2", + pre_scaler_ch_list, num_ch); + if (ret) + goto out; + + for (i = 0; i < num_ch; i++) { + channel = pre_scaler_ch_list[i]; + pvt->vd[channel].pre_scaler = PRE_SCALER_X2; + } + +out: + kfree(pre_scaler_ch_list); + + return ret; +} + +static int pvt_set_temp_coeff(struct device *dev, struct pvt_device *pvt) +{ + struct temp_coeff *ts_coeff = &pvt->ts_coeff; + u32 series; int ret; - ret = reset_control_deassert(pvt->rst); + /* Incase ts-series property is not defined, use default 5. */ + ret = device_property_read_u32(dev, "moortec,ts-series", &series); if (ret) - return ret; + series = TEMPERATURE_SENSOR_SERIES_5; + + switch (series) { + case TEMPERATURE_SENSOR_SERIES_5: + ts_coeff->h = PVT_SERIES5_H_CONST; + ts_coeff->g = PVT_SERIES5_G_CONST; + ts_coeff->j = PVT_SERIES5_J_CONST; + ts_coeff->cal5 = PVT_SERIES5_CAL5_CONST; + break; + case TEMPERATURE_SENSOR_SERIES_6: + ts_coeff->h = PVT_SERIES6_H_CONST; + ts_coeff->g = PVT_SERIES6_G_CONST; + ts_coeff->j = PVT_SERIES6_J_CONST; + ts_coeff->cal5 = PVT_SERIES6_CAL5_CONST; + break; + default: + dev_err(dev, "invalid temperature sensor series (%u)\n", + series); + return -EINVAL; + } - return devm_add_action_or_reset(dev, pvt_reset_control_assert, pvt); + dev_dbg(dev, "temperature sensor series = %u\n", series); + + /* Override ts-coeff-h/g/j/cal5 if they are defined. */ + device_property_read_u32(dev, "moortec,ts-coeff-h", &ts_coeff->h); + device_property_read_u32(dev, "moortec,ts-coeff-g", &ts_coeff->g); + device_property_read_u32(dev, "moortec,ts-coeff-j", &ts_coeff->j); + device_property_read_u32(dev, "moortec,ts-coeff-cal5", &ts_coeff->cal5); + + dev_dbg(dev, "ts-coeff: h = %u, g = %u, j = %d, cal5 = %u\n", + ts_coeff->h, ts_coeff->g, ts_coeff->j, ts_coeff->cal5); + + return 0; } static int mr75203_probe(struct platform_device *pdev) { + u32 ts_num, vm_num, pd_num, ch_num, val, index, i; const struct hwmon_channel_info **pvt_info; - u32 ts_num, vm_num, pd_num, val, index, i; struct device *dev = &pdev->dev; u32 *temp_config, *in_config; struct device *hwmon_dev; @@ -505,32 +781,30 @@ static int mr75203_probe(struct platform_device *pdev) if (ret) return ret; - pvt->clk = devm_clk_get(dev, NULL); + pvt->clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(pvt->clk)) return dev_err_probe(dev, PTR_ERR(pvt->clk), "failed to get clock\n"); - ret = pvt_clk_enable(dev, pvt); - if (ret) { - dev_err(dev, "failed to enable clock\n"); - return ret; - } - - pvt->rst = devm_reset_control_get_exclusive(dev, NULL); + pvt->rst = devm_reset_control_get_optional_exclusive(dev, NULL); if (IS_ERR(pvt->rst)) return dev_err_probe(dev, PTR_ERR(pvt->rst), "failed to get reset control\n"); - ret = pvt_reset_control_deassert(dev, pvt); - if (ret) - return dev_err_probe(dev, ret, "cannot deassert reset control\n"); + if (pvt->rst) { + ret = pvt_reset_control_deassert(dev, pvt); + if (ret) + return dev_err_probe(dev, ret, + "cannot deassert reset control\n"); + } ret = regmap_read(pvt->c_map, PVT_IP_CONFIG, &val); - if(ret < 0) + if (ret < 0) return ret; ts_num = (val & TS_NUM_MSK) >> TS_NUM_SFT; pd_num = (val & PD_NUM_MSK) >> PD_NUM_SFT; vm_num = (val & VM_NUM_MSK) >> VM_NUM_SFT; + ch_num = (val & CH_NUM_MSK) >> CH_NUM_SFT; pvt->t_num = ts_num; pvt->p_num = pd_num; pvt->v_num = vm_num; @@ -553,6 +827,10 @@ static int mr75203_probe(struct platform_device *pdev) if (ret) return ret; + ret = pvt_set_temp_coeff(dev, pvt); + if (ret) + return ret; + temp_config = devm_kcalloc(dev, ts_num + 1, sizeof(*temp_config), GFP_KERNEL); if (!temp_config) @@ -561,6 +839,8 @@ static int mr75203_probe(struct platform_device *pdev) memset32(temp_config, HWMON_T_INPUT, ts_num); pvt_temp.config = temp_config; pvt_info[index++] = &pvt_temp; + + pvt_ts_dbgfs_create(pvt, dev); } if (pd_num) { @@ -570,44 +850,45 @@ static int mr75203_probe(struct platform_device *pdev) } if (vm_num) { - u32 num = vm_num; + u8 vm_idx[VM_NUM_MAX]; ret = pvt_get_regmap(pdev, "vm", pvt); if (ret) return ret; - pvt->vm_idx = devm_kcalloc(dev, vm_num, sizeof(*pvt->vm_idx), - GFP_KERNEL); - if (!pvt->vm_idx) - return -ENOMEM; - - ret = device_property_read_u8_array(dev, "intel,vm-map", - pvt->vm_idx, vm_num); + ret = device_property_read_u8_array(dev, "intel,vm-map", vm_idx, + vm_num); if (ret) { - num = 0; + /* + * Incase intel,vm-map property is not defined, we + * assume incremental channel numbers. + */ + for (i = 0; i < vm_num; i++) + vm_idx[i] = i; } else { for (i = 0; i < vm_num; i++) - if (pvt->vm_idx[i] >= vm_num || - pvt->vm_idx[i] == 0xff) { - num = i; + if (vm_idx[i] >= vm_num || vm_idx[i] == 0xff) { + pvt->v_num = i; + vm_num = i; break; } } - /* - * Incase intel,vm-map property is not defined, we assume - * incremental channel numbers. - */ - for (i = num; i < vm_num; i++) - pvt->vm_idx[i] = i; + ret = pvt_get_active_channel(dev, pvt, vm_num, ch_num, vm_idx); + if (ret) + return ret; + + ret = pvt_get_pre_scaler(dev, pvt); + if (ret) + return ret; - in_config = devm_kcalloc(dev, num + 1, + in_config = devm_kcalloc(dev, pvt->vm_channels.total + 1, sizeof(*in_config), GFP_KERNEL); if (!in_config) return -ENOMEM; - memset32(in_config, HWMON_I_INPUT, num); - in_config[num] = 0; + memset32(in_config, HWMON_I_INPUT, pvt->vm_channels.total); + in_config[pvt->vm_channels.total] = 0; pvt_in.config = in_config; pvt_info[index++] = &pvt_in; diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c index 6a9f420e7d32339e0f0fc578009a4c767a21e637..a872f783e9cc374ee9f40631260eef452ba5c3c9 100644 --- a/drivers/hwmon/nct6683.c +++ b/drivers/hwmon/nct6683.c @@ -412,7 +412,7 @@ nct6683_create_attr_group(struct device *dev, struct sensor_device_attr_u *su; struct attribute_group *group; struct attribute **attrs; - int i, j, count; + int i, count; if (repeat <= 0) return ERR_PTR(-EINVAL); @@ -443,7 +443,7 @@ nct6683_create_attr_group(struct device *dev, for (i = 0; i < repeat; i++) { t = tg->templates; - for (j = 0; *t != NULL; j++) { + while (*t) { snprintf(su->name, sizeof(su->name), (*t)->dev_attr.attr.name, tg->base + i); if ((*t)->s2) { diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c index 41c97cfacfb8cbeec554dd41c5c442bd3a039f54..b347837842139f0f7361bb65dc937d9f5e1057b4 100644 --- a/drivers/hwmon/nct6775-platform.c +++ b/drivers/hwmon/nct6775-platform.c @@ -355,7 +355,7 @@ static void nct6791_enable_io_mapping(struct nct6775_sio_data *sio_data) } } -static int __maybe_unused nct6775_suspend(struct device *dev) +static int nct6775_suspend(struct device *dev) { int err; u16 tmp; @@ -386,7 +386,7 @@ out: return err; } -static int __maybe_unused nct6775_resume(struct device *dev) +static int nct6775_resume(struct device *dev) { struct nct6775_data *data = dev_get_drvdata(dev); struct nct6775_sio_data *sio_data = dev_get_platdata(dev); @@ -467,7 +467,7 @@ abort: return err; } -static SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume); static void nct6775_check_fan_inputs(struct nct6775_data *data, struct nct6775_sio_data *sio_data) @@ -934,7 +934,7 @@ static int nct6775_platform_probe(struct platform_device *pdev) static struct platform_driver nct6775_driver = { .driver = { .name = DRVNAME, - .pm = &nct6775_dev_pm_ops, + .pm = pm_sleep_ptr(&nct6775_dev_pm_ops), }, .probe = nct6775_platform_probe, }; diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c index d1eeef02b6dc8375ee7a3f818f6e6eff37588ed0..a175f8283695e459ff52fd679a0054748275b35f 100644 --- a/drivers/hwmon/nct7802.c +++ b/drivers/hwmon/nct7802.c @@ -1038,7 +1038,7 @@ static int nct7802_detect(struct i2c_client *client, if (reg < 0 || (reg & 0x3f)) return -ENODEV; - strlcpy(info->type, "nct7802", I2C_NAME_SIZE); + strscpy(info->type, "nct7802", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/nct7904.c b/drivers/hwmon/nct7904.c index b1c837fc407af9bdb2175fe496b6aa46cea9eca1..ecc5db0011a3e40d8e2f3024730371e2a8e14d1a 100644 --- a/drivers/hwmon/nct7904.c +++ b/drivers/hwmon/nct7904.c @@ -798,7 +798,7 @@ static int nct7904_detect(struct i2c_client *client, (i2c_smbus_read_byte_data(client, BANK_SEL_REG) & 0xf8) != 0x00) return -ENODEV; - strlcpy(info->type, "nct7904", I2C_NAME_SIZE); + strscpy(info->type, "nct7904", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c index dd892ff5a3e850a169d941df230f49c8485a00ea..533f38b0b4e9bfe1af3a38250d744649fa872ba1 100644 --- a/drivers/hwmon/nzxt-smart2.c +++ b/drivers/hwmon/nzxt-smart2.c @@ -787,6 +787,7 @@ static void nzxt_smart2_hid_remove(struct hid_device *hdev) static const struct hid_device_id nzxt_smart2_hid_id_table[] = { { HID_USB_DEVICE(0x1e71, 0x2006) }, /* NZXT Smart Device V2 */ { HID_USB_DEVICE(0x1e71, 0x200d) }, /* NZXT Smart Device V2 */ + { HID_USB_DEVICE(0x1e71, 0x200f) }, /* NZXT Smart Device V2 */ { HID_USB_DEVICE(0x1e71, 0x2009) }, /* NZXT RGB & Fan Controller */ { HID_USB_DEVICE(0x1e71, 0x200e) }, /* NZXT RGB & Fan Controller */ { HID_USB_DEVICE(0x1e71, 0x2010) }, /* NZXT RGB & Fan Controller */ diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index 45407b12db4b481342f3e86e5082ae0aefe8c0ed..dd690f700d4990e5f11785f8777b936061f9215a 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -1216,8 +1217,16 @@ int occ_setup(struct occ *occ) occ->groups[0] = &occ->group; rc = occ_setup_sysfs(occ); - if (rc) + if (rc) { dev_err(occ->bus_dev, "failed to setup sysfs: %d\n", rc); + return rc; + } + + if (!device_property_read_bool(occ->bus_dev, "ibm,no-poll-on-init")) { + rc = occ_active(occ, true); + if (rc) + occ_shutdown_sysfs(occ); + } return rc; } diff --git a/drivers/hwmon/occ/p8_i2c.c b/drivers/hwmon/occ/p8_i2c.c index b221be1f35f31e3a57382a34d146cd1fc72c2450..9e1744fccb35308e3f76858bff0173fcecb5c483 100644 --- a/drivers/hwmon/occ/p8_i2c.c +++ b/drivers/hwmon/occ/p8_i2c.c @@ -227,13 +227,11 @@ static int p8_i2c_occ_probe(struct i2c_client *client) return occ_setup(occ); } -static int p8_i2c_occ_remove(struct i2c_client *client) +static void p8_i2c_occ_remove(struct i2c_client *client) { struct occ *occ = dev_get_drvdata(&client->dev); occ_shutdown(occ); - - return 0; } static const struct of_device_id p8_i2c_occ_of_match[] = { diff --git a/drivers/hwmon/occ/p9_sbe.c b/drivers/hwmon/occ/p9_sbe.c index c1e0a1d96cd47a3479dedfebe7aba171caf3acd8..96521363b696ec9eb02c6446ecfa07e43f82b103 100644 --- a/drivers/hwmon/occ/p9_sbe.c +++ b/drivers/hwmon/occ/p9_sbe.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -14,6 +15,8 @@ #include "common.h" +#define OCC_CHECKSUM_RETRIES 3 + struct p9_sbe_occ { struct occ occ; bool sbe_error; @@ -80,18 +83,23 @@ done: static int p9_sbe_occ_send_cmd(struct occ *occ, u8 *cmd, size_t len, void *resp, size_t resp_len) { + size_t original_resp_len = resp_len; struct p9_sbe_occ *ctx = to_p9_sbe_occ(occ); - int rc; + int rc, i; - rc = fsi_occ_submit(ctx->sbe, cmd, len, resp, &resp_len); - if (rc < 0) { + for (i = 0; i < OCC_CHECKSUM_RETRIES; ++i) { + rc = fsi_occ_submit(ctx->sbe, cmd, len, resp, &resp_len); + if (rc >= 0) + break; if (resp_len) { if (p9_sbe_occ_save_ffdc(ctx, resp, resp_len)) sysfs_notify(&occ->bus_dev->kobj, NULL, bin_attr_ffdc.attr.name); + return rc; } - - return rc; + if (rc != -EBADE) + return rc; + resp_len = original_resp_len; } switch (((struct occ_response *)resp)->return_status) { @@ -174,9 +182,17 @@ static int p9_sbe_occ_remove(struct platform_device *pdev) return 0; } +static const struct of_device_id p9_sbe_occ_of_match[] = { + { .compatible = "ibm,p9-occ-hwmon" }, + { .compatible = "ibm,p10-occ-hwmon" }, + {} +}; +MODULE_DEVICE_TABLE(of, p9_sbe_occ_of_match); + static struct platform_driver p9_sbe_occ_driver = { .driver = { .name = "occ-hwmon", + .of_match_table = p9_sbe_occ_of_match, }, .probe = p9_sbe_occ_probe, .remove = p9_sbe_occ_remove, diff --git a/drivers/hwmon/pc87360.c b/drivers/hwmon/pc87360.c index 0828436a1f6cfee2f15b4b58aea2654df2934a01..a4adc8bd531ff1db0158a3665ba33b68320ab563 100644 --- a/drivers/hwmon/pc87360.c +++ b/drivers/hwmon/pc87360.c @@ -35,6 +35,18 @@ #include #include +#define DRIVER_NAME "pc87360" + +/* (temp & vin) channel conversion status register flags (pdf sec.11.5.12) */ +#define CHAN_CNVRTD 0x80 /* new data ready */ +#define CHAN_ENA 0x01 /* enabled channel (temp or vin) */ +#define CHAN_ALM_ENA 0x10 /* propagate to alarms-reg ?? (chk val!) */ +#define CHAN_READY (CHAN_ENA|CHAN_CNVRTD) /* sample ready mask */ + +#define TEMP_OTS_OE 0x20 /* OTS Output Enable */ +#define VIN_RW1C_MASK (CHAN_READY|CHAN_ALM_MAX|CHAN_ALM_MIN) /* 0x87 */ +#define TEMP_RW1C_MASK (VIN_RW1C_MASK|TEMP_ALM_CRIT|TEMP_FAULT) /* 0xCF */ + static u8 devid; static struct platform_device *pdev; static unsigned short extra_isa[3]; @@ -211,147 +223,239 @@ struct pc87360_data { }; /* - * Functions declaration + * ldi is the logical device index + * bank is for voltages and temperatures only */ +static int pc87360_read_value(struct pc87360_data *data, u8 ldi, u8 bank, + u8 reg) +{ + int res; -static int pc87360_probe(struct platform_device *pdev); -static int pc87360_remove(struct platform_device *pdev); + mutex_lock(&(data->lock)); + if (bank != NO_BANK) + outb_p(bank, data->address[ldi] + PC87365_REG_BANK); + res = inb_p(data->address[ldi] + reg); + mutex_unlock(&(data->lock)); + + return res; +} -static int pc87360_read_value(struct pc87360_data *data, u8 ldi, u8 bank, - u8 reg); static void pc87360_write_value(struct pc87360_data *data, u8 ldi, u8 bank, - u8 reg, u8 value); -static void pc87360_init_device(struct platform_device *pdev, - int use_thermistors); -static struct pc87360_data *pc87360_update_device(struct device *dev); + u8 reg, u8 value) +{ + mutex_lock(&(data->lock)); + if (bank != NO_BANK) + outb_p(bank, data->address[ldi] + PC87365_REG_BANK); + outb_p(value, data->address[ldi] + reg); + mutex_unlock(&(data->lock)); +} -/* - * Driver data - */ +static void pc87360_autodiv(struct device *dev, int nr) +{ + struct pc87360_data *data = dev_get_drvdata(dev); + u8 old_min = data->fan_min[nr]; -static struct platform_driver pc87360_driver = { - .driver = { - .name = "pc87360", - }, - .probe = pc87360_probe, - .remove = pc87360_remove, -}; + /* Increase clock divider if needed and possible */ + if ((data->fan_status[nr] & 0x04) /* overflow flag */ + || (data->fan[nr] >= 224)) { /* next to overflow */ + if ((data->fan_status[nr] & 0x60) != 0x60) { + data->fan_status[nr] += 0x20; + data->fan_min[nr] >>= 1; + data->fan[nr] >>= 1; + dev_dbg(dev, + "Increasing clock divider to %d for fan %d\n", + FAN_DIV_FROM_REG(data->fan_status[nr]), nr + 1); + } + } else { + /* Decrease clock divider if possible */ + while (!(data->fan_min[nr] & 0x80) /* min "nails" divider */ + && data->fan[nr] < 85 /* bad accuracy */ + && (data->fan_status[nr] & 0x60) != 0x00) { + data->fan_status[nr] -= 0x20; + data->fan_min[nr] <<= 1; + data->fan[nr] <<= 1; + dev_dbg(dev, + "Decreasing clock divider to %d for fan %d\n", + FAN_DIV_FROM_REG(data->fan_status[nr]), + nr + 1); + } + } -/* - * Sysfs stuff - */ + /* Write new fan min if it changed */ + if (old_min != data->fan_min[nr]) { + pc87360_write_value(data, LD_FAN, NO_BANK, + PC87360_REG_FAN_MIN(nr), + data->fan_min[nr]); + } +} -static ssize_t fan_input_show(struct device *dev, - struct device_attribute *devattr, char *buf) +static struct pc87360_data *pc87360_update_device(struct device *dev) { - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan[attr->index], - FAN_DIV_FROM_REG(data->fan_status[attr->index]))); + struct pc87360_data *data = dev_get_drvdata(dev); + u8 i; + + mutex_lock(&data->update_lock); + + if (time_after(jiffies, data->last_updated + HZ * 2) || !data->valid) { + dev_dbg(dev, "Data update\n"); + + /* Fans */ + for (i = 0; i < data->fannr; i++) { + if (FAN_CONFIG_MONITOR(data->fan_conf, i)) { + data->fan_status[i] = + pc87360_read_value(data, LD_FAN, + NO_BANK, PC87360_REG_FAN_STATUS(i)); + data->fan[i] = pc87360_read_value(data, LD_FAN, + NO_BANK, PC87360_REG_FAN(i)); + data->fan_min[i] = pc87360_read_value(data, + LD_FAN, NO_BANK, + PC87360_REG_FAN_MIN(i)); + /* Change clock divider if needed */ + pc87360_autodiv(dev, i); + /* Clear bits and write new divider */ + pc87360_write_value(data, LD_FAN, NO_BANK, + PC87360_REG_FAN_STATUS(i), + data->fan_status[i]); + } + if (FAN_CONFIG_CONTROL(data->fan_conf, i)) + data->pwm[i] = pc87360_read_value(data, LD_FAN, + NO_BANK, PC87360_REG_PWM(i)); + } + + /* Voltages */ + for (i = 0; i < data->innr; i++) { + data->in_status[i] = pc87360_read_value(data, LD_IN, i, + PC87365_REG_IN_STATUS); + /* Clear bits */ + pc87360_write_value(data, LD_IN, i, + PC87365_REG_IN_STATUS, + data->in_status[i]); + if ((data->in_status[i] & CHAN_READY) == CHAN_READY) { + data->in[i] = pc87360_read_value(data, LD_IN, + i, PC87365_REG_IN); + } + if (data->in_status[i] & CHAN_ENA) { + data->in_min[i] = pc87360_read_value(data, + LD_IN, i, + PC87365_REG_IN_MIN); + data->in_max[i] = pc87360_read_value(data, + LD_IN, i, + PC87365_REG_IN_MAX); + if (i >= 11) + data->in_crit[i-11] = + pc87360_read_value(data, LD_IN, + i, PC87365_REG_TEMP_CRIT); + } + } + if (data->innr) { + data->in_alarms = pc87360_read_value(data, LD_IN, + NO_BANK, PC87365_REG_IN_ALARMS1) + | ((pc87360_read_value(data, LD_IN, + NO_BANK, PC87365_REG_IN_ALARMS2) + & 0x07) << 8); + data->vid = (data->vid_conf & 0xE0) ? + pc87360_read_value(data, LD_IN, + NO_BANK, PC87365_REG_VID) : 0x1F; + } + + /* Temperatures */ + for (i = 0; i < data->tempnr; i++) { + data->temp_status[i] = pc87360_read_value(data, + LD_TEMP, i, + PC87365_REG_TEMP_STATUS); + /* Clear bits */ + pc87360_write_value(data, LD_TEMP, i, + PC87365_REG_TEMP_STATUS, + data->temp_status[i]); + if ((data->temp_status[i] & CHAN_READY) == CHAN_READY) { + data->temp[i] = pc87360_read_value(data, + LD_TEMP, i, + PC87365_REG_TEMP); + } + if (data->temp_status[i] & CHAN_ENA) { + data->temp_min[i] = pc87360_read_value(data, + LD_TEMP, i, + PC87365_REG_TEMP_MIN); + data->temp_max[i] = pc87360_read_value(data, + LD_TEMP, i, + PC87365_REG_TEMP_MAX); + data->temp_crit[i] = pc87360_read_value(data, + LD_TEMP, i, + PC87365_REG_TEMP_CRIT); + } + } + if (data->tempnr) { + data->temp_alarms = pc87360_read_value(data, LD_TEMP, + NO_BANK, PC87365_REG_TEMP_ALARMS) + & 0x3F; + } + + data->last_updated = jiffies; + data->valid = true; + } + + mutex_unlock(&data->update_lock); + + return data; } -static ssize_t fan_min_show(struct device *dev, - struct device_attribute *devattr, char *buf) + +static ssize_t in_input_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan_min[attr->index], - FAN_DIV_FROM_REG(data->fan_status[attr->index]))); + return sprintf(buf, "%u\n", IN_FROM_REG(data->in[attr->index], + data->in_vref)); } -static ssize_t fan_div_show(struct device *dev, - struct device_attribute *devattr, char *buf) + +static struct sensor_device_attribute in_input[] = { + SENSOR_ATTR_RO(in0_input, in_input, 0), + SENSOR_ATTR_RO(in1_input, in_input, 1), + SENSOR_ATTR_RO(in2_input, in_input, 2), + SENSOR_ATTR_RO(in3_input, in_input, 3), + SENSOR_ATTR_RO(in4_input, in_input, 4), + SENSOR_ATTR_RO(in5_input, in_input, 5), + SENSOR_ATTR_RO(in6_input, in_input, 6), + SENSOR_ATTR_RO(in7_input, in_input, 7), + SENSOR_ATTR_RO(in8_input, in_input, 8), + SENSOR_ATTR_RO(in9_input, in_input, 9), + SENSOR_ATTR_RO(in10_input, in_input, 10), +}; + +static ssize_t in_status_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", - FAN_DIV_FROM_REG(data->fan_status[attr->index])); + return sprintf(buf, "%u\n", data->in_status[attr->index]); } -static ssize_t fan_status_show(struct device *dev, - struct device_attribute *devattr, char *buf) + +static struct sensor_device_attribute in_status[] = { + SENSOR_ATTR_RO(in0_status, in_status, 0), + SENSOR_ATTR_RO(in1_status, in_status, 1), + SENSOR_ATTR_RO(in2_status, in_status, 2), + SENSOR_ATTR_RO(in3_status, in_status, 3), + SENSOR_ATTR_RO(in4_status, in_status, 4), + SENSOR_ATTR_RO(in5_status, in_status, 5), + SENSOR_ATTR_RO(in6_status, in_status, 6), + SENSOR_ATTR_RO(in7_status, in_status, 7), + SENSOR_ATTR_RO(in8_status, in_status, 8), + SENSOR_ATTR_RO(in9_status, in_status, 9), + SENSOR_ATTR_RO(in10_status, in_status, 10), +}; + +static ssize_t in_min_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", - FAN_STATUS_FROM_REG(data->fan_status[attr->index])); + return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index], + data->in_vref)); } -static ssize_t fan_min_store(struct device *dev, - struct device_attribute *devattr, - const char *buf, size_t count) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = dev_get_drvdata(dev); - long fan_min; - int err; - - err = kstrtol(buf, 10, &fan_min); - if (err) - return err; - - mutex_lock(&data->update_lock); - fan_min = FAN_TO_REG(fan_min, - FAN_DIV_FROM_REG(data->fan_status[attr->index])); - - /* If it wouldn't fit, change clock divisor */ - while (fan_min > 255 - && (data->fan_status[attr->index] & 0x60) != 0x60) { - fan_min >>= 1; - data->fan[attr->index] >>= 1; - data->fan_status[attr->index] += 0x20; - } - data->fan_min[attr->index] = fan_min > 255 ? 255 : fan_min; - pc87360_write_value(data, LD_FAN, NO_BANK, - PC87360_REG_FAN_MIN(attr->index), - data->fan_min[attr->index]); - - /* Write new divider, preserve alarm bits */ - pc87360_write_value(data, LD_FAN, NO_BANK, - PC87360_REG_FAN_STATUS(attr->index), - data->fan_status[attr->index] & 0xF9); - mutex_unlock(&data->update_lock); - - return count; -} - -static struct sensor_device_attribute fan_input[] = { - SENSOR_ATTR_RO(fan1_input, fan_input, 0), - SENSOR_ATTR_RO(fan2_input, fan_input, 1), - SENSOR_ATTR_RO(fan3_input, fan_input, 2), -}; -static struct sensor_device_attribute fan_status[] = { - SENSOR_ATTR_RO(fan1_status, fan_status, 0), - SENSOR_ATTR_RO(fan2_status, fan_status, 1), - SENSOR_ATTR_RO(fan3_status, fan_status, 2), -}; -static struct sensor_device_attribute fan_div[] = { - SENSOR_ATTR_RO(fan1_div, fan_div, 0), - SENSOR_ATTR_RO(fan2_div, fan_div, 1), - SENSOR_ATTR_RO(fan3_div, fan_div, 2), -}; -static struct sensor_device_attribute fan_min[] = { - SENSOR_ATTR_RW(fan1_min, fan_min, 0), - SENSOR_ATTR_RW(fan2_min, fan_min, 1), - SENSOR_ATTR_RW(fan3_min, fan_min, 2), -}; - -#define FAN_UNIT_ATTRS(X) \ -{ &fan_input[X].dev_attr.attr, \ - &fan_status[X].dev_attr.attr, \ - &fan_div[X].dev_attr.attr, \ - &fan_min[X].dev_attr.attr, \ - NULL \ -} - -static ssize_t pwm_show(struct device *dev, struct device_attribute *devattr, - char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", - PWM_FROM_REG(data->pwm[attr->index], - FAN_CONFIG_INVERT(data->fan_conf, - attr->index))); -} -static ssize_t pwm_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) + +static ssize_t in_min_store(struct device *dev, + struct device_attribute *devattr, const char *buf, + size_t count) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = dev_get_drvdata(dev); @@ -363,48 +467,27 @@ static ssize_t pwm_store(struct device *dev, struct device_attribute *devattr, return err; mutex_lock(&data->update_lock); - data->pwm[attr->index] = PWM_TO_REG(val, - FAN_CONFIG_INVERT(data->fan_conf, attr->index)); - pc87360_write_value(data, LD_FAN, NO_BANK, PC87360_REG_PWM(attr->index), - data->pwm[attr->index]); + data->in_min[attr->index] = IN_TO_REG(val, data->in_vref); + pc87360_write_value(data, LD_IN, attr->index, PC87365_REG_IN_MIN, + data->in_min[attr->index]); mutex_unlock(&data->update_lock); return count; } -static struct sensor_device_attribute pwm[] = { - SENSOR_ATTR_RW(pwm1, pwm, 0), - SENSOR_ATTR_RW(pwm2, pwm, 1), - SENSOR_ATTR_RW(pwm3, pwm, 2), -}; - -static struct attribute *pc8736x_fan_attr[][5] = { - FAN_UNIT_ATTRS(0), - FAN_UNIT_ATTRS(1), - FAN_UNIT_ATTRS(2) -}; - -static const struct attribute_group pc8736x_fan_attr_group[] = { - { .attrs = pc8736x_fan_attr[0], }, - { .attrs = pc8736x_fan_attr[1], }, - { .attrs = pc8736x_fan_attr[2], }, +static struct sensor_device_attribute in_min[] = { + SENSOR_ATTR_RW(in0_min, in_min, 0), + SENSOR_ATTR_RW(in1_min, in_min, 1), + SENSOR_ATTR_RW(in2_min, in_min, 2), + SENSOR_ATTR_RW(in3_min, in_min, 3), + SENSOR_ATTR_RW(in4_min, in_min, 4), + SENSOR_ATTR_RW(in5_min, in_min, 5), + SENSOR_ATTR_RW(in6_min, in_min, 6), + SENSOR_ATTR_RW(in7_min, in_min, 7), + SENSOR_ATTR_RW(in8_min, in_min, 8), + SENSOR_ATTR_RW(in9_min, in_min, 9), + SENSOR_ATTR_RW(in10_min, in_min, 10), }; -static ssize_t in_input_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", IN_FROM_REG(data->in[attr->index], - data->in_vref)); -} -static ssize_t in_min_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index], - data->in_vref)); -} static ssize_t in_max_show(struct device *dev, struct device_attribute *devattr, char *buf) { @@ -413,33 +496,7 @@ static ssize_t in_max_show(struct device *dev, return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index], data->in_vref)); } -static ssize_t in_status_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", data->in_status[attr->index]); -} -static ssize_t in_min_store(struct device *dev, - struct device_attribute *devattr, const char *buf, - size_t count) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = dev_get_drvdata(dev); - long val; - int err; - - err = kstrtol(buf, 10, &val); - if (err) - return err; - mutex_lock(&data->update_lock); - data->in_min[attr->index] = IN_TO_REG(val, data->in_vref); - pc87360_write_value(data, LD_IN, attr->index, PC87365_REG_IN_MIN, - data->in_min[attr->index]); - mutex_unlock(&data->update_lock); - return count; -} static ssize_t in_max_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) @@ -462,45 +519,6 @@ static ssize_t in_max_store(struct device *dev, return count; } -static struct sensor_device_attribute in_input[] = { - SENSOR_ATTR_RO(in0_input, in_input, 0), - SENSOR_ATTR_RO(in1_input, in_input, 1), - SENSOR_ATTR_RO(in2_input, in_input, 2), - SENSOR_ATTR_RO(in3_input, in_input, 3), - SENSOR_ATTR_RO(in4_input, in_input, 4), - SENSOR_ATTR_RO(in5_input, in_input, 5), - SENSOR_ATTR_RO(in6_input, in_input, 6), - SENSOR_ATTR_RO(in7_input, in_input, 7), - SENSOR_ATTR_RO(in8_input, in_input, 8), - SENSOR_ATTR_RO(in9_input, in_input, 9), - SENSOR_ATTR_RO(in10_input, in_input, 10), -}; -static struct sensor_device_attribute in_status[] = { - SENSOR_ATTR_RO(in0_status, in_status, 0), - SENSOR_ATTR_RO(in1_status, in_status, 1), - SENSOR_ATTR_RO(in2_status, in_status, 2), - SENSOR_ATTR_RO(in3_status, in_status, 3), - SENSOR_ATTR_RO(in4_status, in_status, 4), - SENSOR_ATTR_RO(in5_status, in_status, 5), - SENSOR_ATTR_RO(in6_status, in_status, 6), - SENSOR_ATTR_RO(in7_status, in_status, 7), - SENSOR_ATTR_RO(in8_status, in_status, 8), - SENSOR_ATTR_RO(in9_status, in_status, 9), - SENSOR_ATTR_RO(in10_status, in_status, 10), -}; -static struct sensor_device_attribute in_min[] = { - SENSOR_ATTR_RW(in0_min, in_min, 0), - SENSOR_ATTR_RW(in1_min, in_min, 1), - SENSOR_ATTR_RW(in2_min, in_min, 2), - SENSOR_ATTR_RW(in3_min, in_min, 3), - SENSOR_ATTR_RW(in4_min, in_min, 4), - SENSOR_ATTR_RW(in5_min, in_min, 5), - SENSOR_ATTR_RW(in6_min, in_min, 6), - SENSOR_ATTR_RW(in7_min, in_min, 7), - SENSOR_ATTR_RW(in8_min, in_min, 8), - SENSOR_ATTR_RW(in9_min, in_min, 9), - SENSOR_ATTR_RW(in10_min, in_min, 10), -}; static struct sensor_device_attribute in_max[] = { SENSOR_ATTR_RW(in0_max, in_max, 0), SENSOR_ATTR_RW(in1_max, in_max, 1), @@ -534,14 +552,6 @@ static ssize_t in_min_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MIN)); } -static ssize_t in_max_alarm_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct pc87360_data *data = pc87360_update_device(dev); - unsigned nr = to_sensor_dev_attr(devattr)->index; - - return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX)); -} static struct sensor_device_attribute in_min_alarm[] = { SENSOR_ATTR_RO(in0_min_alarm, in_min_alarm, 0), @@ -556,6 +566,16 @@ static struct sensor_device_attribute in_min_alarm[] = { SENSOR_ATTR_RO(in9_min_alarm, in_min_alarm, 9), SENSOR_ATTR_RO(in10_min_alarm, in_min_alarm, 10), }; + +static ssize_t in_max_alarm_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct pc87360_data *data = pc87360_update_device(dev); + unsigned nr = to_sensor_dev_attr(devattr)->index; + + return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX)); +} + static struct sensor_device_attribute in_max_alarm[] = { SENSOR_ATTR_RO(in0_max_alarm, in_max_alarm, 0), SENSOR_ATTR_RO(in1_max_alarm, in_max_alarm, 1), @@ -592,6 +612,7 @@ static ssize_t vrm_show(struct device *dev, struct device_attribute *attr, struct pc87360_data *data = dev_get_drvdata(dev); return sprintf(buf, "%u\n", data->vrm); } + static ssize_t vrm_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { @@ -648,37 +669,39 @@ static ssize_t therm_input_show(struct device *dev, return sprintf(buf, "%u\n", IN_FROM_REG(data->in[attr->index], data->in_vref)); } -static ssize_t therm_min_show(struct device *dev, - struct device_attribute *devattr, char *buf) + +/* + * the +11 term below reflects the fact that VLM units 11,12,13 are + * used in the chip to measure voltage across the thermistors + */ +static struct sensor_device_attribute therm_input[] = { + SENSOR_ATTR_RO(temp4_input, therm_input, 0 + 11), + SENSOR_ATTR_RO(temp5_input, therm_input, 1 + 11), + SENSOR_ATTR_RO(temp6_input, therm_input, 2 + 11), +}; + +static ssize_t therm_status_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index], - data->in_vref)); + return sprintf(buf, "%u\n", data->in_status[attr->index]); } -static ssize_t therm_max_show(struct device *dev, + +static struct sensor_device_attribute therm_status[] = { + SENSOR_ATTR_RO(temp4_status, therm_status, 0 + 11), + SENSOR_ATTR_RO(temp5_status, therm_status, 1 + 11), + SENSOR_ATTR_RO(temp6_status, therm_status, 2 + 11), +}; + +static ssize_t therm_min_show(struct device *dev, struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index], - data->in_vref)); -} -static ssize_t therm_crit_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", IN_FROM_REG(data->in_crit[attr->index-11], + return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index], data->in_vref)); } -static ssize_t therm_status_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", data->in_status[attr->index]); -} static ssize_t therm_min_store(struct device *dev, struct device_attribute *devattr, @@ -701,7 +724,22 @@ static ssize_t therm_min_store(struct device *dev, return count; } -static ssize_t therm_max_store(struct device *dev, +static struct sensor_device_attribute therm_min[] = { + SENSOR_ATTR_RW(temp4_min, therm_min, 0 + 11), + SENSOR_ATTR_RW(temp5_min, therm_min, 1 + 11), + SENSOR_ATTR_RW(temp6_min, therm_min, 2 + 11), +}; + +static ssize_t therm_max_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index], + data->in_vref)); +} + +static ssize_t therm_max_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) { @@ -721,6 +759,22 @@ static ssize_t therm_max_store(struct device *dev, mutex_unlock(&data->update_lock); return count; } + +static struct sensor_device_attribute therm_max[] = { + SENSOR_ATTR_RW(temp4_max, therm_max, 0 + 11), + SENSOR_ATTR_RW(temp5_max, therm_max, 1 + 11), + SENSOR_ATTR_RW(temp6_max, therm_max, 2 + 11), +}; + +static ssize_t therm_crit_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", IN_FROM_REG(data->in_crit[attr->index-11], + data->in_vref)); +} + static ssize_t therm_crit_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) @@ -742,30 +796,6 @@ static ssize_t therm_crit_store(struct device *dev, return count; } -/* - * the +11 term below reflects the fact that VLM units 11,12,13 are - * used in the chip to measure voltage across the thermistors - */ -static struct sensor_device_attribute therm_input[] = { - SENSOR_ATTR_RO(temp4_input, therm_input, 0 + 11), - SENSOR_ATTR_RO(temp5_input, therm_input, 1 + 11), - SENSOR_ATTR_RO(temp6_input, therm_input, 2 + 11), -}; -static struct sensor_device_attribute therm_status[] = { - SENSOR_ATTR_RO(temp4_status, therm_status, 0 + 11), - SENSOR_ATTR_RO(temp5_status, therm_status, 1 + 11), - SENSOR_ATTR_RO(temp6_status, therm_status, 2 + 11), -}; -static struct sensor_device_attribute therm_min[] = { - SENSOR_ATTR_RW(temp4_min, therm_min, 0 + 11), - SENSOR_ATTR_RW(temp5_min, therm_min, 1 + 11), - SENSOR_ATTR_RW(temp6_min, therm_min, 2 + 11), -}; -static struct sensor_device_attribute therm_max[] = { - SENSOR_ATTR_RW(temp4_max, therm_max, 0 + 11), - SENSOR_ATTR_RW(temp5_max, therm_max, 1 + 11), - SENSOR_ATTR_RW(temp6_max, therm_max, 2 + 11), -}; static struct sensor_device_attribute therm_crit[] = { SENSOR_ATTR_RW(temp4_crit, therm_crit, 0 + 11), SENSOR_ATTR_RW(temp5_crit, therm_crit, 1 + 11), @@ -776,7 +806,6 @@ static struct sensor_device_attribute therm_crit[] = { * show_therm_min/max_alarm() reads data from the per-channel voltage * status register (sec 11.5.12) */ - static ssize_t therm_min_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -786,6 +815,13 @@ static ssize_t therm_min_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MIN)); } + +static struct sensor_device_attribute therm_min_alarm[] = { + SENSOR_ATTR_RO(temp4_min_alarm, therm_min_alarm, 0 + 11), + SENSOR_ATTR_RO(temp5_min_alarm, therm_min_alarm, 1 + 11), + SENSOR_ATTR_RO(temp6_min_alarm, therm_min_alarm, 2 + 11), +}; + static ssize_t therm_max_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -795,6 +831,13 @@ static ssize_t therm_max_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX)); } + +static struct sensor_device_attribute therm_max_alarm[] = { + SENSOR_ATTR_RO(temp4_max_alarm, therm_max_alarm, 0 + 11), + SENSOR_ATTR_RO(temp5_max_alarm, therm_max_alarm, 1 + 11), + SENSOR_ATTR_RO(temp6_max_alarm, therm_max_alarm, 2 + 11), +}; + static ssize_t therm_crit_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -805,16 +848,6 @@ static ssize_t therm_crit_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->in_status[nr] & TEMP_ALM_CRIT)); } -static struct sensor_device_attribute therm_min_alarm[] = { - SENSOR_ATTR_RO(temp4_min_alarm, therm_min_alarm, 0 + 11), - SENSOR_ATTR_RO(temp5_min_alarm, therm_min_alarm, 1 + 11), - SENSOR_ATTR_RO(temp6_min_alarm, therm_min_alarm, 2 + 11), -}; -static struct sensor_device_attribute therm_max_alarm[] = { - SENSOR_ATTR_RO(temp4_max_alarm, therm_max_alarm, 0 + 11), - SENSOR_ATTR_RO(temp5_max_alarm, therm_max_alarm, 1 + 11), - SENSOR_ATTR_RO(temp6_max_alarm, therm_max_alarm, 2 + 11), -}; static struct sensor_device_attribute therm_crit_alarm[] = { SENSOR_ATTR_RO(temp4_crit_alarm, therm_crit_alarm, 0 + 11), SENSOR_ATTR_RO(temp5_crit_alarm, therm_crit_alarm, 1 + 11), @@ -849,37 +882,32 @@ static ssize_t temp_input_show(struct device *dev, return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[attr->index])); } -static ssize_t temp_min_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_min[attr->index])); -} +static struct sensor_device_attribute temp_input[] = { + SENSOR_ATTR_RO(temp1_input, temp_input, 0), + SENSOR_ATTR_RO(temp2_input, temp_input, 1), + SENSOR_ATTR_RO(temp3_input, temp_input, 2), +}; -static ssize_t temp_max_show(struct device *dev, - struct device_attribute *devattr, char *buf) +static ssize_t temp_status_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_max[attr->index])); + return sprintf(buf, "%d\n", data->temp_status[attr->index]); } -static ssize_t temp_crit_show(struct device *dev, - struct device_attribute *devattr, char *buf) -{ - struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%d\n", - TEMP_FROM_REG(data->temp_crit[attr->index])); -} +static struct sensor_device_attribute temp_status[] = { + SENSOR_ATTR_RO(temp1_status, temp_status, 0), + SENSOR_ATTR_RO(temp2_status, temp_status, 1), + SENSOR_ATTR_RO(temp3_status, temp_status, 2), +}; -static ssize_t temp_status_show(struct device *dev, - struct device_attribute *devattr, char *buf) +static ssize_t temp_min_show(struct device *dev, + struct device_attribute *devattr, char *buf) { struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%d\n", data->temp_status[attr->index]); + return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_min[attr->index])); } static ssize_t temp_min_store(struct device *dev, @@ -903,6 +931,20 @@ static ssize_t temp_min_store(struct device *dev, return count; } +static struct sensor_device_attribute temp_min[] = { + SENSOR_ATTR_RW(temp1_min, temp_min, 0), + SENSOR_ATTR_RW(temp2_min, temp_min, 1), + SENSOR_ATTR_RW(temp3_min, temp_min, 2), +}; + +static ssize_t temp_max_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_max[attr->index])); +} + static ssize_t temp_max_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) @@ -924,6 +966,21 @@ static ssize_t temp_max_store(struct device *dev, return count; } +static struct sensor_device_attribute temp_max[] = { + SENSOR_ATTR_RW(temp1_max, temp_max, 0), + SENSOR_ATTR_RW(temp2_max, temp_max, 1), + SENSOR_ATTR_RW(temp3_max, temp_max, 2), +}; + +static ssize_t temp_crit_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%d\n", + TEMP_FROM_REG(data->temp_crit[attr->index])); +} + static ssize_t temp_crit_store(struct device *dev, struct device_attribute *devattr, const char *buf, size_t count) @@ -945,47 +1002,17 @@ static ssize_t temp_crit_store(struct device *dev, return count; } -static struct sensor_device_attribute temp_input[] = { - SENSOR_ATTR_RO(temp1_input, temp_input, 0), - SENSOR_ATTR_RO(temp2_input, temp_input, 1), - SENSOR_ATTR_RO(temp3_input, temp_input, 2), -}; -static struct sensor_device_attribute temp_status[] = { - SENSOR_ATTR_RO(temp1_status, temp_status, 0), - SENSOR_ATTR_RO(temp2_status, temp_status, 1), - SENSOR_ATTR_RO(temp3_status, temp_status, 2), -}; -static struct sensor_device_attribute temp_min[] = { - SENSOR_ATTR_RW(temp1_min, temp_min, 0), - SENSOR_ATTR_RW(temp2_min, temp_min, 1), - SENSOR_ATTR_RW(temp3_min, temp_min, 2), -}; -static struct sensor_device_attribute temp_max[] = { - SENSOR_ATTR_RW(temp1_max, temp_max, 0), - SENSOR_ATTR_RW(temp2_max, temp_max, 1), - SENSOR_ATTR_RW(temp3_max, temp_max, 2), -}; static struct sensor_device_attribute temp_crit[] = { SENSOR_ATTR_RW(temp1_crit, temp_crit, 0), SENSOR_ATTR_RW(temp2_crit, temp_crit, 1), SENSOR_ATTR_RW(temp3_crit, temp_crit, 2), }; -static ssize_t alarms_temp_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct pc87360_data *data = pc87360_update_device(dev); - return sprintf(buf, "%u\n", data->temp_alarms); -} - -static DEVICE_ATTR_RO(alarms_temp); - /* - * show_temp_min/max_alarm() reads data from the per-channel status + * temp_min/max_alarm_show() reads data from the per-channel status * register (sec 12.3.7), not the temp event status registers (sec * 12.3.2) that show_temp_alarm() reads (via data->temp_alarms) */ - static ssize_t temp_min_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -996,6 +1023,12 @@ static ssize_t temp_min_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->temp_status[nr] & CHAN_ALM_MIN)); } +static struct sensor_device_attribute temp_min_alarm[] = { + SENSOR_ATTR_RO(temp1_min_alarm, temp_min_alarm, 0), + SENSOR_ATTR_RO(temp2_min_alarm, temp_min_alarm, 1), + SENSOR_ATTR_RO(temp3_min_alarm, temp_min_alarm, 2), +}; + static ssize_t temp_max_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -1006,6 +1039,12 @@ static ssize_t temp_max_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->temp_status[nr] & CHAN_ALM_MAX)); } +static struct sensor_device_attribute temp_max_alarm[] = { + SENSOR_ATTR_RO(temp1_max_alarm, temp_max_alarm, 0), + SENSOR_ATTR_RO(temp2_max_alarm, temp_max_alarm, 1), + SENSOR_ATTR_RO(temp3_max_alarm, temp_max_alarm, 2), +}; + static ssize_t temp_crit_alarm_show(struct device *dev, struct device_attribute *devattr, char *buf) @@ -1016,18 +1055,6 @@ static ssize_t temp_crit_alarm_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->temp_status[nr] & TEMP_ALM_CRIT)); } -static struct sensor_device_attribute temp_min_alarm[] = { - SENSOR_ATTR_RO(temp1_min_alarm, temp_min_alarm, 0), - SENSOR_ATTR_RO(temp2_min_alarm, temp_min_alarm, 1), - SENSOR_ATTR_RO(temp3_min_alarm, temp_min_alarm, 2), -}; - -static struct sensor_device_attribute temp_max_alarm[] = { - SENSOR_ATTR_RO(temp1_max_alarm, temp_max_alarm, 0), - SENSOR_ATTR_RO(temp2_max_alarm, temp_max_alarm, 1), - SENSOR_ATTR_RO(temp3_max_alarm, temp_max_alarm, 2), -}; - static struct sensor_device_attribute temp_crit_alarm[] = { SENSOR_ATTR_RO(temp1_crit_alarm, temp_crit_alarm, 0), SENSOR_ATTR_RO(temp2_crit_alarm, temp_crit_alarm, 1), @@ -1043,6 +1070,7 @@ static ssize_t temp_fault_show(struct device *dev, return sprintf(buf, "%u\n", !!(data->temp_status[nr] & TEMP_FAULT)); } + static struct sensor_device_attribute temp_fault[] = { SENSOR_ATTR_RO(temp1_fault, temp_fault, 0), SENSOR_ATTR_RO(temp2_fault, temp_fault, 1), @@ -1074,323 +1102,195 @@ static const struct attribute_group pc8736x_temp_attr_group[] = { { .attrs = pc8736x_temp_attr[2] } }; -static ssize_t name_show(struct device *dev, - struct device_attribute *devattr, char *buf) +static ssize_t alarms_temp_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct pc87360_data *data = dev_get_drvdata(dev); - return sprintf(buf, "%s\n", data->name); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", data->temp_alarms); } -static DEVICE_ATTR_RO(name); +static DEVICE_ATTR_RO(alarms_temp); -/* - * Device detection, registration and update - */ +static ssize_t fan_input_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan[attr->index], + FAN_DIV_FROM_REG(data->fan_status[attr->index]))); +} -static int __init pc87360_find(int sioaddr, u8 *devid, - unsigned short *addresses) +static struct sensor_device_attribute fan_input[] = { + SENSOR_ATTR_RO(fan1_input, fan_input, 0), + SENSOR_ATTR_RO(fan2_input, fan_input, 1), + SENSOR_ATTR_RO(fan3_input, fan_input, 2), +}; + +static ssize_t fan_status_show(struct device *dev, + struct device_attribute *devattr, char *buf) { - u16 val; - int i; - int nrdev; /* logical device count */ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", + FAN_STATUS_FROM_REG(data->fan_status[attr->index])); +} - /* No superio_enter */ +static struct sensor_device_attribute fan_status[] = { + SENSOR_ATTR_RO(fan1_status, fan_status, 0), + SENSOR_ATTR_RO(fan2_status, fan_status, 1), + SENSOR_ATTR_RO(fan3_status, fan_status, 2), +}; - /* Identify device */ - val = force_id ? force_id : superio_inb(sioaddr, DEVID); - switch (val) { - case 0xE1: /* PC87360 */ - case 0xE8: /* PC87363 */ - case 0xE4: /* PC87364 */ - nrdev = 1; - break; - case 0xE5: /* PC87365 */ - case 0xE9: /* PC87366 */ - nrdev = 3; - break; - default: - superio_exit(sioaddr); - return -ENODEV; - } - /* Remember the device id */ - *devid = val; - - for (i = 0; i < nrdev; i++) { - /* select logical device */ - superio_outb(sioaddr, DEV, logdev[i]); - - val = superio_inb(sioaddr, ACT); - if (!(val & 0x01)) { - pr_info("Device 0x%02x not activated\n", logdev[i]); - continue; - } - - val = (superio_inb(sioaddr, BASE) << 8) - | superio_inb(sioaddr, BASE + 1); - if (!val) { - pr_info("Base address not set for device 0x%02x\n", - logdev[i]); - continue; - } - - addresses[i] = val; - - if (i == 0) { /* Fans */ - confreg[0] = superio_inb(sioaddr, 0xF0); - confreg[1] = superio_inb(sioaddr, 0xF1); - - pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 1, - (confreg[0] >> 2) & 1, (confreg[0] >> 3) & 1, - (confreg[0] >> 4) & 1); - pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 2, - (confreg[0] >> 5) & 1, (confreg[0] >> 6) & 1, - (confreg[0] >> 7) & 1); - pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 3, - confreg[1] & 1, (confreg[1] >> 1) & 1, - (confreg[1] >> 2) & 1); - } else if (i == 1) { /* Voltages */ - /* Are we using thermistors? */ - if (*devid == 0xE9) { /* PC87366 */ - /* - * These registers are not logical-device - * specific, just that we won't need them if - * we don't use the VLM device - */ - confreg[2] = superio_inb(sioaddr, 0x2B); - confreg[3] = superio_inb(sioaddr, 0x25); - - if (confreg[2] & 0x40) { - pr_info("Using thermistors for temperature monitoring\n"); - } - if (confreg[3] & 0xE0) { - pr_info("VID inputs routed (mode %u)\n", - confreg[3] >> 5); - } - } - } - } - - superio_exit(sioaddr); - return 0; +static ssize_t fan_div_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", + FAN_DIV_FROM_REG(data->fan_status[attr->index])); } -static void pc87360_remove_files(struct device *dev) -{ - int i; +static struct sensor_device_attribute fan_div[] = { + SENSOR_ATTR_RO(fan1_div, fan_div, 0), + SENSOR_ATTR_RO(fan2_div, fan_div, 1), + SENSOR_ATTR_RO(fan3_div, fan_div, 2), +}; - device_remove_file(dev, &dev_attr_name); - device_remove_file(dev, &dev_attr_alarms_temp); - for (i = 0; i < ARRAY_SIZE(pc8736x_temp_attr_group); i++) - sysfs_remove_group(&dev->kobj, &pc8736x_temp_attr_group[i]); - for (i = 0; i < ARRAY_SIZE(pc8736x_fan_attr_group); i++) { - sysfs_remove_group(&pdev->dev.kobj, &pc8736x_fan_attr_group[i]); - device_remove_file(dev, &pwm[i].dev_attr); - } - sysfs_remove_group(&dev->kobj, &pc8736x_therm_group); - sysfs_remove_group(&dev->kobj, &pc8736x_vin_group); +static ssize_t fan_min_show(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan_min[attr->index], + FAN_DIV_FROM_REG(data->fan_status[attr->index]))); } -static int pc87360_probe(struct platform_device *pdev) +static ssize_t fan_min_store(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) { - int i; - struct pc87360_data *data; - int err = 0; - const char *name; - int use_thermistors = 0; - struct device *dev = &pdev->dev; - - data = devm_kzalloc(dev, sizeof(struct pc87360_data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - switch (devid) { - default: - name = "pc87360"; - data->fannr = 2; - break; - case 0xe8: - name = "pc87363"; - data->fannr = 2; - break; - case 0xe4: - name = "pc87364"; - data->fannr = 3; - break; - case 0xe5: - name = "pc87365"; - data->fannr = extra_isa[0] ? 3 : 0; - data->innr = extra_isa[1] ? 11 : 0; - data->tempnr = extra_isa[2] ? 2 : 0; - break; - case 0xe9: - name = "pc87366"; - data->fannr = extra_isa[0] ? 3 : 0; - data->innr = extra_isa[1] ? 14 : 0; - data->tempnr = extra_isa[2] ? 3 : 0; - break; - } - - data->name = name; - mutex_init(&data->lock); - mutex_init(&data->update_lock); - platform_set_drvdata(pdev, data); - - for (i = 0; i < LDNI_MAX; i++) { - data->address[i] = extra_isa[i]; - if (data->address[i] - && !devm_request_region(dev, extra_isa[i], PC87360_EXTENT, - pc87360_driver.driver.name)) { - dev_err(dev, - "Region 0x%x-0x%x already in use!\n", - extra_isa[i], extra_isa[i]+PC87360_EXTENT-1); - return -EBUSY; - } - } - - /* Retrieve the fans configuration from Super-I/O space */ - if (data->fannr) - data->fan_conf = confreg[0] | (confreg[1] << 8); - - /* - * Use the correct reference voltage - * Unless both the VLM and the TMS logical devices agree to - * use an external Vref, the internal one is used. - */ - if (data->innr) { - i = pc87360_read_value(data, LD_IN, NO_BANK, - PC87365_REG_IN_CONFIG); - if (data->tempnr) { - i &= pc87360_read_value(data, LD_TEMP, NO_BANK, - PC87365_REG_TEMP_CONFIG); - } - data->in_vref = (i&0x02) ? 3025 : 2966; - dev_dbg(dev, "Using %s reference voltage\n", - (i&0x02) ? "external" : "internal"); - - data->vid_conf = confreg[3]; - data->vrm = vid_which_vrm(); - } - - /* Fan clock dividers may be needed before any data is read */ - for (i = 0; i < data->fannr; i++) { - if (FAN_CONFIG_MONITOR(data->fan_conf, i)) - data->fan_status[i] = pc87360_read_value(data, - LD_FAN, NO_BANK, - PC87360_REG_FAN_STATUS(i)); - } - - if (init > 0) { - if (devid == 0xe9 && data->address[1]) /* PC87366 */ - use_thermistors = confreg[2] & 0x40; - - pc87360_init_device(pdev, use_thermistors); - } - - /* Register all-or-nothing sysfs groups */ - - if (data->innr) { - err = sysfs_create_group(&dev->kobj, &pc8736x_vin_group); - if (err) - goto error; - } + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = dev_get_drvdata(dev); + long fan_min; + int err; - if (data->innr == 14) { - err = sysfs_create_group(&dev->kobj, &pc8736x_therm_group); - if (err) - goto error; - } + err = kstrtol(buf, 10, &fan_min); + if (err) + return err; - /* create device attr-files for varying sysfs groups */ + mutex_lock(&data->update_lock); + fan_min = FAN_TO_REG(fan_min, + FAN_DIV_FROM_REG(data->fan_status[attr->index])); - if (data->tempnr) { - for (i = 0; i < data->tempnr; i++) { - err = sysfs_create_group(&dev->kobj, - &pc8736x_temp_attr_group[i]); - if (err) - goto error; - } - err = device_create_file(dev, &dev_attr_alarms_temp); - if (err) - goto error; + /* If it wouldn't fit, change clock divisor */ + while (fan_min > 255 + && (data->fan_status[attr->index] & 0x60) != 0x60) { + fan_min >>= 1; + data->fan[attr->index] >>= 1; + data->fan_status[attr->index] += 0x20; } + data->fan_min[attr->index] = fan_min > 255 ? 255 : fan_min; + pc87360_write_value(data, LD_FAN, NO_BANK, + PC87360_REG_FAN_MIN(attr->index), + data->fan_min[attr->index]); - for (i = 0; i < data->fannr; i++) { - if (FAN_CONFIG_MONITOR(data->fan_conf, i)) { - err = sysfs_create_group(&dev->kobj, - &pc8736x_fan_attr_group[i]); - if (err) - goto error; - } - if (FAN_CONFIG_CONTROL(data->fan_conf, i)) { - err = device_create_file(dev, &pwm[i].dev_attr); - if (err) - goto error; - } - } + /* Write new divider, preserve alarm bits */ + pc87360_write_value(data, LD_FAN, NO_BANK, + PC87360_REG_FAN_STATUS(attr->index), + data->fan_status[attr->index] & 0xF9); + mutex_unlock(&data->update_lock); - err = device_create_file(dev, &dev_attr_name); - if (err) - goto error; + return count; +} - data->hwmon_dev = hwmon_device_register(dev); - if (IS_ERR(data->hwmon_dev)) { - err = PTR_ERR(data->hwmon_dev); - goto error; - } - return 0; +static struct sensor_device_attribute fan_min[] = { + SENSOR_ATTR_RW(fan1_min, fan_min, 0), + SENSOR_ATTR_RW(fan2_min, fan_min, 1), + SENSOR_ATTR_RW(fan3_min, fan_min, 2), +}; -error: - pc87360_remove_files(dev); - return err; +#define FAN_UNIT_ATTRS(X) \ +{ &fan_input[X].dev_attr.attr, \ + &fan_status[X].dev_attr.attr, \ + &fan_div[X].dev_attr.attr, \ + &fan_min[X].dev_attr.attr, \ + NULL \ } -static int pc87360_remove(struct platform_device *pdev) -{ - struct pc87360_data *data = platform_get_drvdata(pdev); +static struct attribute *pc8736x_fan_attr[][5] = { + FAN_UNIT_ATTRS(0), + FAN_UNIT_ATTRS(1), + FAN_UNIT_ATTRS(2) +}; - hwmon_device_unregister(data->hwmon_dev); - pc87360_remove_files(&pdev->dev); +static const struct attribute_group pc8736x_fan_attr_group[] = { + { .attrs = pc8736x_fan_attr[0], }, + { .attrs = pc8736x_fan_attr[1], }, + { .attrs = pc8736x_fan_attr[2], }, +}; - return 0; +static ssize_t pwm_show(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = pc87360_update_device(dev); + return sprintf(buf, "%u\n", + PWM_FROM_REG(data->pwm[attr->index], + FAN_CONFIG_INVERT(data->fan_conf, + attr->index))); } -/* - * ldi is the logical device index - * bank is for voltages and temperatures only - */ -static int pc87360_read_value(struct pc87360_data *data, u8 ldi, u8 bank, - u8 reg) +static ssize_t pwm_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { - int res; + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pc87360_data *data = dev_get_drvdata(dev); + long val; + int err; - mutex_lock(&(data->lock)); - if (bank != NO_BANK) - outb_p(bank, data->address[ldi] + PC87365_REG_BANK); - res = inb_p(data->address[ldi] + reg); - mutex_unlock(&(data->lock)); + err = kstrtol(buf, 10, &val); + if (err) + return err; - return res; + mutex_lock(&data->update_lock); + data->pwm[attr->index] = PWM_TO_REG(val, + FAN_CONFIG_INVERT(data->fan_conf, attr->index)); + pc87360_write_value(data, LD_FAN, NO_BANK, PC87360_REG_PWM(attr->index), + data->pwm[attr->index]); + mutex_unlock(&data->update_lock); + return count; } -static void pc87360_write_value(struct pc87360_data *data, u8 ldi, u8 bank, - u8 reg, u8 value) +static struct sensor_device_attribute pwm[] = { + SENSOR_ATTR_RW(pwm1, pwm, 0), + SENSOR_ATTR_RW(pwm2, pwm, 1), + SENSOR_ATTR_RW(pwm3, pwm, 2), +}; + +static ssize_t name_show(struct device *dev, + struct device_attribute *devattr, char *buf) { - mutex_lock(&(data->lock)); - if (bank != NO_BANK) - outb_p(bank, data->address[ldi] + PC87365_REG_BANK); - outb_p(value, data->address[ldi] + reg); - mutex_unlock(&(data->lock)); + struct pc87360_data *data = dev_get_drvdata(dev); + return sprintf(buf, "%s\n", data->name); } -/* (temp & vin) channel conversion status register flags (pdf sec.11.5.12) */ -#define CHAN_CNVRTD 0x80 /* new data ready */ -#define CHAN_ENA 0x01 /* enabled channel (temp or vin) */ -#define CHAN_ALM_ENA 0x10 /* propagate to alarms-reg ?? (chk val!) */ -#define CHAN_READY (CHAN_ENA|CHAN_CNVRTD) /* sample ready mask */ +static DEVICE_ATTR_RO(name); -#define TEMP_OTS_OE 0x20 /* OTS Output Enable */ -#define VIN_RW1C_MASK (CHAN_READY|CHAN_ALM_MAX|CHAN_ALM_MIN) /* 0x87 */ -#define TEMP_RW1C_MASK (VIN_RW1C_MASK|TEMP_ALM_CRIT|TEMP_FAULT) /* 0xCF */ +static void pc87360_remove_files(struct device *dev) +{ + int i; + + device_remove_file(dev, &dev_attr_name); + device_remove_file(dev, &dev_attr_alarms_temp); + for (i = 0; i < ARRAY_SIZE(pc8736x_temp_attr_group); i++) + sysfs_remove_group(&dev->kobj, &pc8736x_temp_attr_group[i]); + for (i = 0; i < ARRAY_SIZE(pc8736x_fan_attr_group); i++) { + sysfs_remove_group(&pdev->dev.kobj, &pc8736x_fan_attr_group[i]); + device_remove_file(dev, &pwm[i].dev_attr); + } + sysfs_remove_group(&dev->kobj, &pc8736x_therm_group); + sysfs_remove_group(&dev->kobj, &pc8736x_vin_group); +} static void pc87360_init_device(struct platform_device *pdev, int use_thermistors) @@ -1530,155 +1430,272 @@ static void pc87360_init_device(struct platform_device *pdev, pc87360_write_value(data, LD_TEMP, NO_BANK, 0xE, 0x05); } } -} - -static void pc87360_autodiv(struct device *dev, int nr) -{ - struct pc87360_data *data = dev_get_drvdata(dev); - u8 old_min = data->fan_min[nr]; +} + +static int pc87360_probe(struct platform_device *pdev) +{ + int i; + struct pc87360_data *data; + int err = 0; + const char *name; + int use_thermistors = 0; + struct device *dev = &pdev->dev; + + data = devm_kzalloc(dev, sizeof(struct pc87360_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + switch (devid) { + default: + name = "pc87360"; + data->fannr = 2; + break; + case 0xe8: + name = "pc87363"; + data->fannr = 2; + break; + case 0xe4: + name = "pc87364"; + data->fannr = 3; + break; + case 0xe5: + name = "pc87365"; + data->fannr = extra_isa[0] ? 3 : 0; + data->innr = extra_isa[1] ? 11 : 0; + data->tempnr = extra_isa[2] ? 2 : 0; + break; + case 0xe9: + name = "pc87366"; + data->fannr = extra_isa[0] ? 3 : 0; + data->innr = extra_isa[1] ? 14 : 0; + data->tempnr = extra_isa[2] ? 3 : 0; + break; + } + + data->name = name; + mutex_init(&data->lock); + mutex_init(&data->update_lock); + platform_set_drvdata(pdev, data); + + for (i = 0; i < LDNI_MAX; i++) { + data->address[i] = extra_isa[i]; + if (data->address[i] + && !devm_request_region(dev, extra_isa[i], PC87360_EXTENT, + DRIVER_NAME)) { + dev_err(dev, + "Region 0x%x-0x%x already in use!\n", + extra_isa[i], extra_isa[i]+PC87360_EXTENT-1); + return -EBUSY; + } + } + + /* Retrieve the fans configuration from Super-I/O space */ + if (data->fannr) + data->fan_conf = confreg[0] | (confreg[1] << 8); + + /* + * Use the correct reference voltage + * Unless both the VLM and the TMS logical devices agree to + * use an external Vref, the internal one is used. + */ + if (data->innr) { + i = pc87360_read_value(data, LD_IN, NO_BANK, + PC87365_REG_IN_CONFIG); + if (data->tempnr) { + i &= pc87360_read_value(data, LD_TEMP, NO_BANK, + PC87365_REG_TEMP_CONFIG); + } + data->in_vref = (i&0x02) ? 3025 : 2966; + dev_dbg(dev, "Using %s reference voltage\n", + (i&0x02) ? "external" : "internal"); + + data->vid_conf = confreg[3]; + data->vrm = vid_which_vrm(); + } + + /* Fan clock dividers may be needed before any data is read */ + for (i = 0; i < data->fannr; i++) { + if (FAN_CONFIG_MONITOR(data->fan_conf, i)) + data->fan_status[i] = pc87360_read_value(data, + LD_FAN, NO_BANK, + PC87360_REG_FAN_STATUS(i)); + } + + if (init > 0) { + if (devid == 0xe9 && data->address[1]) /* PC87366 */ + use_thermistors = confreg[2] & 0x40; + + pc87360_init_device(pdev, use_thermistors); + } + + /* Register all-or-nothing sysfs groups */ + + if (data->innr) { + err = sysfs_create_group(&dev->kobj, &pc8736x_vin_group); + if (err) + goto error; + } + + if (data->innr == 14) { + err = sysfs_create_group(&dev->kobj, &pc8736x_therm_group); + if (err) + goto error; + } + + /* create device attr-files for varying sysfs groups */ + + if (data->tempnr) { + for (i = 0; i < data->tempnr; i++) { + err = sysfs_create_group(&dev->kobj, + &pc8736x_temp_attr_group[i]); + if (err) + goto error; + } + err = device_create_file(dev, &dev_attr_alarms_temp); + if (err) + goto error; + } - /* Increase clock divider if needed and possible */ - if ((data->fan_status[nr] & 0x04) /* overflow flag */ - || (data->fan[nr] >= 224)) { /* next to overflow */ - if ((data->fan_status[nr] & 0x60) != 0x60) { - data->fan_status[nr] += 0x20; - data->fan_min[nr] >>= 1; - data->fan[nr] >>= 1; - dev_dbg(dev, - "Increasing clock divider to %d for fan %d\n", - FAN_DIV_FROM_REG(data->fan_status[nr]), nr + 1); + for (i = 0; i < data->fannr; i++) { + if (FAN_CONFIG_MONITOR(data->fan_conf, i)) { + err = sysfs_create_group(&dev->kobj, + &pc8736x_fan_attr_group[i]); + if (err) + goto error; } - } else { - /* Decrease clock divider if possible */ - while (!(data->fan_min[nr] & 0x80) /* min "nails" divider */ - && data->fan[nr] < 85 /* bad accuracy */ - && (data->fan_status[nr] & 0x60) != 0x00) { - data->fan_status[nr] -= 0x20; - data->fan_min[nr] <<= 1; - data->fan[nr] <<= 1; - dev_dbg(dev, - "Decreasing clock divider to %d for fan %d\n", - FAN_DIV_FROM_REG(data->fan_status[nr]), - nr + 1); + if (FAN_CONFIG_CONTROL(data->fan_conf, i)) { + err = device_create_file(dev, &pwm[i].dev_attr); + if (err) + goto error; } } - /* Write new fan min if it changed */ - if (old_min != data->fan_min[nr]) { - pc87360_write_value(data, LD_FAN, NO_BANK, - PC87360_REG_FAN_MIN(nr), - data->fan_min[nr]); + err = device_create_file(dev, &dev_attr_name); + if (err) + goto error; + + data->hwmon_dev = hwmon_device_register(dev); + if (IS_ERR(data->hwmon_dev)) { + err = PTR_ERR(data->hwmon_dev); + goto error; } + return 0; + +error: + pc87360_remove_files(dev); + return err; } -static struct pc87360_data *pc87360_update_device(struct device *dev) +static int pc87360_remove(struct platform_device *pdev) { - struct pc87360_data *data = dev_get_drvdata(dev); - u8 i; + struct pc87360_data *data = platform_get_drvdata(pdev); - mutex_lock(&data->update_lock); + hwmon_device_unregister(data->hwmon_dev); + pc87360_remove_files(&pdev->dev); - if (time_after(jiffies, data->last_updated + HZ * 2) || !data->valid) { - dev_dbg(dev, "Data update\n"); + return 0; +} - /* Fans */ - for (i = 0; i < data->fannr; i++) { - if (FAN_CONFIG_MONITOR(data->fan_conf, i)) { - data->fan_status[i] = - pc87360_read_value(data, LD_FAN, - NO_BANK, PC87360_REG_FAN_STATUS(i)); - data->fan[i] = pc87360_read_value(data, LD_FAN, - NO_BANK, PC87360_REG_FAN(i)); - data->fan_min[i] = pc87360_read_value(data, - LD_FAN, NO_BANK, - PC87360_REG_FAN_MIN(i)); - /* Change clock divider if needed */ - pc87360_autodiv(dev, i); - /* Clear bits and write new divider */ - pc87360_write_value(data, LD_FAN, NO_BANK, - PC87360_REG_FAN_STATUS(i), - data->fan_status[i]); - } - if (FAN_CONFIG_CONTROL(data->fan_conf, i)) - data->pwm[i] = pc87360_read_value(data, LD_FAN, - NO_BANK, PC87360_REG_PWM(i)); - } +/* + * Driver data + */ +static struct platform_driver pc87360_driver = { + .driver = { + .name = DRIVER_NAME, + }, + .probe = pc87360_probe, + .remove = pc87360_remove, +}; - /* Voltages */ - for (i = 0; i < data->innr; i++) { - data->in_status[i] = pc87360_read_value(data, LD_IN, i, - PC87365_REG_IN_STATUS); - /* Clear bits */ - pc87360_write_value(data, LD_IN, i, - PC87365_REG_IN_STATUS, - data->in_status[i]); - if ((data->in_status[i] & CHAN_READY) == CHAN_READY) { - data->in[i] = pc87360_read_value(data, LD_IN, - i, PC87365_REG_IN); - } - if (data->in_status[i] & CHAN_ENA) { - data->in_min[i] = pc87360_read_value(data, - LD_IN, i, - PC87365_REG_IN_MIN); - data->in_max[i] = pc87360_read_value(data, - LD_IN, i, - PC87365_REG_IN_MAX); - if (i >= 11) - data->in_crit[i-11] = - pc87360_read_value(data, LD_IN, - i, PC87365_REG_TEMP_CRIT); - } +/* + * Device detection, registration and update + */ + +static int __init pc87360_find(int sioaddr, u8 *devid, + unsigned short *addresses) +{ + u16 val; + int i; + int nrdev; /* logical device count */ + + /* No superio_enter */ + + /* Identify device */ + val = force_id ? force_id : superio_inb(sioaddr, DEVID); + switch (val) { + case 0xE1: /* PC87360 */ + case 0xE8: /* PC87363 */ + case 0xE4: /* PC87364 */ + nrdev = 1; + break; + case 0xE5: /* PC87365 */ + case 0xE9: /* PC87366 */ + nrdev = 3; + break; + default: + superio_exit(sioaddr); + return -ENODEV; + } + /* Remember the device id */ + *devid = val; + + for (i = 0; i < nrdev; i++) { + /* select logical device */ + superio_outb(sioaddr, DEV, logdev[i]); + + val = superio_inb(sioaddr, ACT); + if (!(val & 0x01)) { + pr_info("Device 0x%02x not activated\n", logdev[i]); + continue; } - if (data->innr) { - data->in_alarms = pc87360_read_value(data, LD_IN, - NO_BANK, PC87365_REG_IN_ALARMS1) - | ((pc87360_read_value(data, LD_IN, - NO_BANK, PC87365_REG_IN_ALARMS2) - & 0x07) << 8); - data->vid = (data->vid_conf & 0xE0) ? - pc87360_read_value(data, LD_IN, - NO_BANK, PC87365_REG_VID) : 0x1F; + + val = (superio_inb(sioaddr, BASE) << 8) + | superio_inb(sioaddr, BASE + 1); + if (!val) { + pr_info("Base address not set for device 0x%02x\n", + logdev[i]); + continue; } - /* Temperatures */ - for (i = 0; i < data->tempnr; i++) { - data->temp_status[i] = pc87360_read_value(data, - LD_TEMP, i, - PC87365_REG_TEMP_STATUS); - /* Clear bits */ - pc87360_write_value(data, LD_TEMP, i, - PC87365_REG_TEMP_STATUS, - data->temp_status[i]); - if ((data->temp_status[i] & CHAN_READY) == CHAN_READY) { - data->temp[i] = pc87360_read_value(data, - LD_TEMP, i, - PC87365_REG_TEMP); - } - if (data->temp_status[i] & CHAN_ENA) { - data->temp_min[i] = pc87360_read_value(data, - LD_TEMP, i, - PC87365_REG_TEMP_MIN); - data->temp_max[i] = pc87360_read_value(data, - LD_TEMP, i, - PC87365_REG_TEMP_MAX); - data->temp_crit[i] = pc87360_read_value(data, - LD_TEMP, i, - PC87365_REG_TEMP_CRIT); + addresses[i] = val; + + if (i == 0) { /* Fans */ + confreg[0] = superio_inb(sioaddr, 0xF0); + confreg[1] = superio_inb(sioaddr, 0xF1); + + pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 1, + (confreg[0] >> 2) & 1, (confreg[0] >> 3) & 1, + (confreg[0] >> 4) & 1); + pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 2, + (confreg[0] >> 5) & 1, (confreg[0] >> 6) & 1, + (confreg[0] >> 7) & 1); + pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 3, + confreg[1] & 1, (confreg[1] >> 1) & 1, + (confreg[1] >> 2) & 1); + } else if (i == 1) { /* Voltages */ + /* Are we using thermistors? */ + if (*devid == 0xE9) { /* PC87366 */ + /* + * These registers are not logical-device + * specific, just that we won't need them if + * we don't use the VLM device + */ + confreg[2] = superio_inb(sioaddr, 0x2B); + confreg[3] = superio_inb(sioaddr, 0x25); + + if (confreg[2] & 0x40) { + pr_info("Using thermistors for temperature monitoring\n"); + } + if (confreg[3] & 0xE0) { + pr_info("VID inputs routed (mode %u)\n", + confreg[3] >> 5); + } } } - if (data->tempnr) { - data->temp_alarms = pc87360_read_value(data, LD_TEMP, - NO_BANK, PC87365_REG_TEMP_ALARMS) - & 0x3F; - } - - data->last_updated = jiffies; - data->valid = true; } - mutex_unlock(&data->update_lock); - - return data; + superio_exit(sioaddr); + return 0; } static int __init pc87360_device_add(unsigned short address) @@ -1777,10 +1794,10 @@ static void __exit pc87360_exit(void) platform_driver_unregister(&pc87360_driver); } - MODULE_AUTHOR("Jean Delvare "); MODULE_DESCRIPTION("PC8736x hardware monitor"); MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:" DRIVER_NAME); module_init(pc87360_init); module_exit(pc87360_exit); diff --git a/drivers/hwmon/pcf8591.c b/drivers/hwmon/pcf8591.c index a97a51005c616421072c5fc0dfbbeb1566b73534..af9614e918a452eb6589028950a7e2bd77c9609e 100644 --- a/drivers/hwmon/pcf8591.c +++ b/drivers/hwmon/pcf8591.c @@ -228,14 +228,13 @@ exit_sysfs_remove: return err; } -static int pcf8591_remove(struct i2c_client *client) +static void pcf8591_remove(struct i2c_client *client) { struct pcf8591_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); sysfs_remove_group(&client->dev.kobj, &pcf8591_attr_group_opt); sysfs_remove_group(&client->dev.kobj, &pcf8591_attr_group); - return 0; } /* Called when we have found a new PCF8591. */ diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig index 951e4a9ff2d6b67cddfcb18f3f0785def3df4911..89668af67206084deef93e3c8a60ce9729c4506f 100644 --- a/drivers/hwmon/pmbus/Kconfig +++ b/drivers/hwmon/pmbus/Kconfig @@ -397,6 +397,15 @@ config SENSORS_TPS53679 This driver can also be built as a module. If so, the module will be called tps53679. +config SENSORS_TPS546D24 + tristate "TPS546D24" + help + If you say yes here you get hardware monitoring support for TEXAS + TPS546D24. + + This driver can also be built as a module. If so, the module will + be called tps546d24 + config SENSORS_UCD9000 tristate "TI UCD90120, UCD90124, UCD90160, UCD90320, UCD9090, UCD90910" help diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile index e2fe86f989650b15394f6dcb40368f9af15a5f4d..0002dbe22d528b945ebaf046ef432606e72d956e 100644 --- a/drivers/hwmon/pmbus/Makefile +++ b/drivers/hwmon/pmbus/Makefile @@ -41,6 +41,7 @@ obj-$(CONFIG_SENSORS_Q54SJ108A2) += q54sj108a2.o obj-$(CONFIG_SENSORS_STPDDC60) += stpddc60.o obj-$(CONFIG_SENSORS_TPS40422) += tps40422.o obj-$(CONFIG_SENSORS_TPS53679) += tps53679.o +obj-$(CONFIG_SENSORS_TPS546D24) += tps546d24.o obj-$(CONFIG_SENSORS_UCD9000) += ucd9000.o obj-$(CONFIG_SENSORS_UCD9200) += ucd9200.o obj-$(CONFIG_SENSORS_XDPE122) += xdpe12284.o diff --git a/drivers/hwmon/pmbus/mp2888.c b/drivers/hwmon/pmbus/mp2888.c index 8ecd4adfef40ed7e94989a1c0cc811f689bcf1b9..24e5194706cf66137ed86439b703e6c4e42b3df1 100644 --- a/drivers/hwmon/pmbus/mp2888.c +++ b/drivers/hwmon/pmbus/mp2888.c @@ -34,7 +34,7 @@ struct mp2888_data { int curr_sense_gain; }; -#define to_mp2888_data(x) container_of(x, struct mp2888_data, info) +#define to_mp2888_data(x) container_of(x, struct mp2888_data, info) static int mp2888_read_byte_data(struct i2c_client *client, int page, int reg) { @@ -109,7 +109,7 @@ mp2888_read_phase(struct i2c_client *client, struct mp2888_data *data, int page, * - Kcs is the DrMOS current sense gain of power stage, which is obtained from the * register MP2888_MFR_VR_CONFIG1, bits 13-12 with the following selection of DrMOS * (data->curr_sense_gain): - * 00b - 5µA/A, 01b - 8.5µA/A, 10b - 9.7µA/A, 11b - 10µA/A. + * 00b - 8.5µA/A, 01b - 9.7µA/A, 1b - 10µA/A, 11b - 5µA/A. * - Rcs is the internal phase current sense resistor. This parameter depends on hardware * assembly. By default it is set to 1kΩ. In case of different assembly, user should * scale this parameter by dividing it by Rcs. @@ -118,10 +118,9 @@ mp2888_read_phase(struct i2c_client *client, struct mp2888_data *data, int page, * because sampling of current occurrence of bit weight has a big deviation, especially for * light load. */ - ret = DIV_ROUND_CLOSEST(ret * 100 - 9800, data->curr_sense_gain); - ret = (data->phase_curr_resolution) ? ret * 2 : ret; + ret = DIV_ROUND_CLOSEST(ret * 200 - 19600, data->curr_sense_gain); /* Scale according to total current resolution. */ - ret = (data->total_curr_resolution) ? ret * 8 : ret * 4; + ret = (data->total_curr_resolution) ? ret * 2 : ret; return ret; } @@ -212,7 +211,7 @@ static int mp2888_read_word_data(struct i2c_client *client, int page, int phase, ret = pmbus_read_word_data(client, page, phase, reg); if (ret < 0) return ret; - ret = data->total_curr_resolution ? ret * 2 : ret; + ret = data->total_curr_resolution ? ret : DIV_ROUND_CLOSEST(ret, 2); break; case PMBUS_POUT_OP_WARN_LIMIT: ret = pmbus_read_word_data(client, page, phase, reg); @@ -223,7 +222,7 @@ static int mp2888_read_word_data(struct i2c_client *client, int page, int phase, * set 1. Actual power is reported with 0.5W or 1W respectively resolution. Scaling * is needed to match both. */ - ret = data->total_curr_resolution ? ret * 4 : ret * 2; + ret = data->total_curr_resolution ? ret * 2 : ret; break; /* * The below registers are not implemented by device or implemented not according to the diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index f10bac8860fce5e9823647c876f7cfce2a48fd2d..7ec04934747e12c9327d806de7a7ca70ed8260a9 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -1270,9 +1270,9 @@ struct pmbus_thermal_data { struct pmbus_sensor *sensor; }; -static int pmbus_thermal_get_temp(void *data, int *temp) +static int pmbus_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct pmbus_thermal_data *tdata = data; + struct pmbus_thermal_data *tdata = tz->devdata; struct pmbus_sensor *sensor = tdata->sensor; struct pmbus_data *pmbus_data = tdata->pmbus_data; struct i2c_client *client = to_i2c_client(pmbus_data->dev); @@ -1296,7 +1296,7 @@ static int pmbus_thermal_get_temp(void *data, int *temp) return ret; } -static const struct thermal_zone_of_device_ops pmbus_thermal_ops = { +static const struct thermal_zone_device_ops pmbus_thermal_ops = { .get_temp = pmbus_thermal_get_temp, }; @@ -1314,8 +1314,8 @@ static int pmbus_thermal_add_sensor(struct pmbus_data *pmbus_data, tdata->sensor = sensor; tdata->pmbus_data = pmbus_data; - tzd = devm_thermal_zone_of_sensor_register(dev, index, tdata, - &pmbus_thermal_ops); + tzd = devm_thermal_of_zone_register(dev, index, tdata, + &pmbus_thermal_ops); /* * If CONFIG_THERMAL_OF is disabled, this returns -ENODEV, * so ignore that error but forward any other error. @@ -2861,7 +2861,7 @@ static int pmbus_regulator_get_low_margin(struct i2c_client *client, int page) .data = -1, }; - if (!data->vout_low[page]) { + if (data->vout_low[page] < 0) { if (pmbus_check_word_register(client, page, PMBUS_MFR_VOUT_MIN)) s.data = _pmbus_read_word_data(client, page, 0xff, PMBUS_MFR_VOUT_MIN); @@ -2887,7 +2887,7 @@ static int pmbus_regulator_get_high_margin(struct i2c_client *client, int page) .data = -1, }; - if (!data->vout_high[page]) { + if (data->vout_high[page] < 0) { if (pmbus_check_word_register(client, page, PMBUS_MFR_VOUT_MAX)) s.data = _pmbus_read_word_data(client, page, 0xff, PMBUS_MFR_VOUT_MAX); @@ -3016,11 +3016,10 @@ static int pmbus_regulator_register(struct pmbus_data *data) rdev = devm_regulator_register(dev, &info->reg_desc[i], &config); - if (IS_ERR(rdev)) { - dev_err(dev, "Failed to register %s regulator\n", - info->reg_desc[i].name); - return PTR_ERR(rdev); - } + if (IS_ERR(rdev)) + return dev_err_probe(dev, PTR_ERR(rdev), + "Failed to register %s regulator\n", + info->reg_desc[i].name); } return 0; @@ -3320,6 +3319,7 @@ int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info) struct pmbus_data *data; size_t groups_num = 0; int ret; + int i; char *name; if (!info) @@ -3353,6 +3353,11 @@ int pmbus_do_probe(struct i2c_client *client, struct pmbus_driver_info *info) data->currpage = -1; data->currphase = -1; + for (i = 0; i < ARRAY_SIZE(data->vout_low); i++) { + data->vout_low[i] = -1; + data->vout_high[i] = -1; + } + ret = pmbus_init_common(client, data, info); if (ret < 0) return ret; diff --git a/drivers/hwmon/pmbus/tps546d24.c b/drivers/hwmon/pmbus/tps546d24.c new file mode 100644 index 0000000000000000000000000000000000000000..435f94304ad8b3f3d36263bd2be422794b6ccda3 --- /dev/null +++ b/drivers/hwmon/pmbus/tps546d24.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Hardware monitoring driver for TEXAS TPS546D24 buck converter + */ + +#include +#include +#include +#include +#include +#include +#include "pmbus.h" + +static struct pmbus_driver_info tps546d24_info = { + .pages = 1, + .format[PSC_VOLTAGE_IN] = linear, + .format[PSC_VOLTAGE_OUT] = linear, + .format[PSC_TEMPERATURE] = linear, + .format[PSC_CURRENT_OUT] = linear, + .func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_IIN + | PMBUS_HAVE_IOUT | PMBUS_HAVE_VOUT + | PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_VOUT + | PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP, +}; + +static int tps546d24_probe(struct i2c_client *client) +{ + int reg; + + reg = i2c_smbus_read_byte_data(client, PMBUS_VOUT_MODE); + if (reg < 0) + return reg; + + if (reg & 0x80) { + int err; + + err = i2c_smbus_write_byte_data(client, PMBUS_VOUT_MODE, reg & 0x7f); + if (err < 0) + return err; + } + return pmbus_do_probe(client, &tps546d24_info); +} + +static const struct i2c_device_id tps546d24_id[] = { + {"tps546d24", 0}, + {} +}; +MODULE_DEVICE_TABLE(i2c, tps546d24_id); + +static const struct of_device_id __maybe_unused tps546d24_of_match[] = { + {.compatible = "ti,tps546d24"}, + {} +}; +MODULE_DEVICE_TABLE(of, tps546d24_of_match); + +/* This is the driver that will be inserted */ +static struct i2c_driver tps546d24_driver = { + .driver = { + .name = "tps546d24", + .of_match_table = of_match_ptr(tps546d24_of_match), + }, + .probe_new = tps546d24_probe, + .id_table = tps546d24_id, +}; + +module_i2c_driver(tps546d24_driver); + +MODULE_AUTHOR("Duke Du "); +MODULE_DESCRIPTION("PMBus driver for TI tps546d24"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(PMBUS); diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c index 6c08551d8d140e221cf9596e6200b6195a10a46a..dc3d9a22d9176438f29c3299450025ee16d685c3 100644 --- a/drivers/hwmon/pwm-fan.c +++ b/drivers/hwmon/pwm-fan.c @@ -28,11 +28,23 @@ struct pwm_fan_tach { u8 pulses_per_revolution; }; +enum pwm_fan_enable_mode { + pwm_off_reg_off, + pwm_disable_reg_enable, + pwm_enable_reg_enable, + pwm_disable_reg_disable, +}; + struct pwm_fan_ctx { + struct device *dev; + struct mutex lock; struct pwm_device *pwm; struct pwm_state pwm_state; struct regulator *reg_en; + enum pwm_fan_enable_mode enable_mode; + bool regulator_enabled; + bool enabled; int tach_count; struct pwm_fan_tach *tachs; @@ -82,25 +94,140 @@ static void sample_timer(struct timer_list *t) mod_timer(&ctx->rpm_timer, jiffies + HZ); } -static int __set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm) +static void pwm_fan_enable_mode_2_state(int enable_mode, + struct pwm_state *state, + bool *enable_regulator) +{ + switch (enable_mode) { + case pwm_disable_reg_enable: + /* disable pwm, keep regulator enabled */ + state->enabled = false; + *enable_regulator = true; + break; + case pwm_enable_reg_enable: + /* keep pwm and regulator enabled */ + state->enabled = true; + *enable_regulator = true; + break; + case pwm_off_reg_off: + case pwm_disable_reg_disable: + /* disable pwm and regulator */ + state->enabled = false; + *enable_regulator = false; + } +} + +static int pwm_fan_switch_power(struct pwm_fan_ctx *ctx, bool on) { - unsigned long period; int ret = 0; + + if (!ctx->reg_en) + return ret; + + if (!ctx->regulator_enabled && on) { + ret = regulator_enable(ctx->reg_en); + if (ret == 0) + ctx->regulator_enabled = true; + } else if (ctx->regulator_enabled && !on) { + ret = regulator_disable(ctx->reg_en); + if (ret == 0) + ctx->regulator_enabled = false; + } + return ret; +} + +static int pwm_fan_power_on(struct pwm_fan_ctx *ctx) +{ struct pwm_state *state = &ctx->pwm_state; + int ret; - mutex_lock(&ctx->lock); - if (ctx->pwm_value == pwm) - goto exit_set_pwm_err; + if (ctx->enabled) + return 0; + + ret = pwm_fan_switch_power(ctx, true); + if (ret < 0) { + dev_err(ctx->dev, "failed to enable power supply\n"); + return ret; + } + + state->enabled = true; + ret = pwm_apply_state(ctx->pwm, state); + if (ret) { + dev_err(ctx->dev, "failed to enable PWM\n"); + goto disable_regulator; + } + + ctx->enabled = true; + + return 0; + +disable_regulator: + pwm_fan_switch_power(ctx, false); + return ret; +} - period = state->period; - state->duty_cycle = DIV_ROUND_UP(pwm * (period - 1), MAX_PWM); - state->enabled = pwm ? true : false; +static int pwm_fan_power_off(struct pwm_fan_ctx *ctx) +{ + struct pwm_state *state = &ctx->pwm_state; + bool enable_regulator = false; + int ret; + if (!ctx->enabled) + return 0; + + pwm_fan_enable_mode_2_state(ctx->enable_mode, + state, + &enable_regulator); + + state->enabled = false; + state->duty_cycle = 0; ret = pwm_apply_state(ctx->pwm, state); + if (ret) { + dev_err(ctx->dev, "failed to disable PWM\n"); + return ret; + } + + pwm_fan_switch_power(ctx, enable_regulator); + + ctx->enabled = false; + + return 0; +} + +static int __set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm) +{ + struct pwm_state *state = &ctx->pwm_state; + unsigned long period; + int ret = 0; + + if (pwm > 0) { + if (ctx->enable_mode == pwm_off_reg_off) + /* pwm-fan hard disabled */ + return 0; + + period = state->period; + state->duty_cycle = DIV_ROUND_UP(pwm * (period - 1), MAX_PWM); + ret = pwm_apply_state(ctx->pwm, state); + if (ret) + return ret; + ret = pwm_fan_power_on(ctx); + } else { + ret = pwm_fan_power_off(ctx); + } if (!ret) ctx->pwm_value = pwm; -exit_set_pwm_err: + + return ret; +} + +static int set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm) +{ + int ret; + + mutex_lock(&ctx->lock); + ret = __set_pwm(ctx, pwm); mutex_unlock(&ctx->lock); + return ret; } @@ -115,20 +242,76 @@ static void pwm_fan_update_state(struct pwm_fan_ctx *ctx, unsigned long pwm) ctx->pwm_fan_state = i; } +static int pwm_fan_update_enable(struct pwm_fan_ctx *ctx, long val) +{ + int ret = 0; + int old_val; + + mutex_lock(&ctx->lock); + + if (ctx->enable_mode == val) + goto out; + + old_val = ctx->enable_mode; + ctx->enable_mode = val; + + if (val == 0) { + /* Disable pwm-fan unconditionally */ + ret = __set_pwm(ctx, 0); + if (ret) + ctx->enable_mode = old_val; + pwm_fan_update_state(ctx, 0); + } else { + /* + * Change PWM and/or regulator state if currently disabled + * Nothing to do if currently enabled + */ + if (!ctx->enabled) { + struct pwm_state *state = &ctx->pwm_state; + bool enable_regulator = false; + + state->duty_cycle = 0; + pwm_fan_enable_mode_2_state(val, + state, + &enable_regulator); + + pwm_apply_state(ctx->pwm, state); + pwm_fan_switch_power(ctx, enable_regulator); + pwm_fan_update_state(ctx, 0); + } + } +out: + mutex_unlock(&ctx->lock); + + return ret; +} + static int pwm_fan_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val) { struct pwm_fan_ctx *ctx = dev_get_drvdata(dev); int ret; - if (val < 0 || val > MAX_PWM) - return -EINVAL; + switch (attr) { + case hwmon_pwm_input: + if (val < 0 || val > MAX_PWM) + return -EINVAL; + ret = set_pwm(ctx, val); + if (ret) + return ret; + pwm_fan_update_state(ctx, val); + break; + case hwmon_pwm_enable: + if (val < 0 || val > 3) + ret = -EINVAL; + else + ret = pwm_fan_update_enable(ctx, val); - ret = __set_pwm(ctx, val); - if (ret) return ret; + default: + return -EOPNOTSUPP; + } - pwm_fan_update_state(ctx, val); return 0; } @@ -139,9 +322,15 @@ static int pwm_fan_read(struct device *dev, enum hwmon_sensor_types type, switch (type) { case hwmon_pwm: - *val = ctx->pwm_value; - return 0; - + switch (attr) { + case hwmon_pwm_input: + *val = ctx->pwm_value; + return 0; + case hwmon_pwm_enable: + *val = ctx->enable_mode; + return 0; + } + return -EOPNOTSUPP; case hwmon_fan: *val = ctx->tachs[channel].rpm; return 0; @@ -212,7 +401,7 @@ pwm_fan_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) if (state == ctx->pwm_fan_state) return 0; - ret = __set_pwm(ctx, ctx->pwm_fan_cooling_levels[state]); + ret = set_pwm(ctx, ctx->pwm_fan_cooling_levels[state]); if (ret) { dev_err(&cdev->device, "Cannot set pwm!\n"); return ret; @@ -270,18 +459,14 @@ static int pwm_fan_of_get_cooling_data(struct device *dev, return 0; } -static void pwm_fan_regulator_disable(void *data) -{ - regulator_disable(data); -} - -static void pwm_fan_pwm_disable(void *__ctx) +static void pwm_fan_cleanup(void *__ctx) { struct pwm_fan_ctx *ctx = __ctx; - ctx->pwm_state.enabled = false; - pwm_apply_state(ctx->pwm, &ctx->pwm_state); del_timer_sync(&ctx->rpm_timer); + /* Switch off everything */ + ctx->enable_mode = pwm_disable_reg_disable; + pwm_fan_power_off(ctx); } static int pwm_fan_probe(struct platform_device *pdev) @@ -302,7 +487,8 @@ static int pwm_fan_probe(struct platform_device *pdev) mutex_init(&ctx->lock); - ctx->pwm = devm_of_pwm_get(dev, dev->of_node, NULL); + ctx->dev = &pdev->dev; + ctx->pwm = devm_pwm_get(dev, NULL); if (IS_ERR(ctx->pwm)) return dev_err_probe(dev, PTR_ERR(ctx->pwm), "Could not get PWM\n"); @@ -314,22 +500,12 @@ static int pwm_fan_probe(struct platform_device *pdev) return PTR_ERR(ctx->reg_en); ctx->reg_en = NULL; - } else { - ret = regulator_enable(ctx->reg_en); - if (ret) { - dev_err(dev, "Failed to enable fan supply: %d\n", ret); - return ret; - } - ret = devm_add_action_or_reset(dev, pwm_fan_regulator_disable, - ctx->reg_en); - if (ret) - return ret; } pwm_init_state(ctx->pwm, &ctx->pwm_state); /* - * __set_pwm assumes that MAX_PWM * (period - 1) fits into an unsigned + * set_pwm assumes that MAX_PWM * (period - 1) fits into an unsigned * long. Check this here to prevent the fan running at a too low * frequency. */ @@ -338,14 +514,19 @@ static int pwm_fan_probe(struct platform_device *pdev) return -EINVAL; } - /* Set duty cycle to maximum allowed and enable PWM output */ - ret = __set_pwm(ctx, MAX_PWM); + ctx->enable_mode = pwm_disable_reg_enable; + + /* + * Set duty cycle to maximum allowed and enable PWM output as well as + * the regulator. In case of error nothing is changed + */ + ret = set_pwm(ctx, MAX_PWM); if (ret) { dev_err(dev, "Failed to configure PWM: %d\n", ret); return ret; } timer_setup(&ctx->rpm_timer, sample_timer, 0); - ret = devm_add_action_or_reset(dev, pwm_fan_pwm_disable, ctx); + ret = devm_add_action_or_reset(dev, pwm_fan_cleanup, ctx); if (ret) return ret; @@ -377,7 +558,7 @@ static int pwm_fan_probe(struct platform_device *pdev) if (!channels) return -ENOMEM; - channels[0] = HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT); + channels[0] = HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_ENABLE); for (i = 0; i < ctx->tach_count; i++) { struct pwm_fan_tach *tach = &ctx->tachs[i]; @@ -451,65 +632,28 @@ static int pwm_fan_probe(struct platform_device *pdev) return 0; } -static int pwm_fan_disable(struct device *dev) -{ - struct pwm_fan_ctx *ctx = dev_get_drvdata(dev); - int ret; - - if (ctx->pwm_value) { - /* keep ctx->pwm_state unmodified for pwm_fan_resume() */ - struct pwm_state state = ctx->pwm_state; - - state.duty_cycle = 0; - state.enabled = false; - ret = pwm_apply_state(ctx->pwm, &state); - if (ret < 0) - return ret; - } - - if (ctx->reg_en) { - ret = regulator_disable(ctx->reg_en); - if (ret) { - dev_err(dev, "Failed to disable fan supply: %d\n", ret); - return ret; - } - } - - return 0; -} - static void pwm_fan_shutdown(struct platform_device *pdev) { - pwm_fan_disable(&pdev->dev); + struct pwm_fan_ctx *ctx = platform_get_drvdata(pdev); + + pwm_fan_cleanup(ctx); } -#ifdef CONFIG_PM_SLEEP static int pwm_fan_suspend(struct device *dev) { - return pwm_fan_disable(dev); + struct pwm_fan_ctx *ctx = dev_get_drvdata(dev); + + return pwm_fan_power_off(ctx); } static int pwm_fan_resume(struct device *dev) { struct pwm_fan_ctx *ctx = dev_get_drvdata(dev); - int ret; - - if (ctx->reg_en) { - ret = regulator_enable(ctx->reg_en); - if (ret) { - dev_err(dev, "Failed to enable fan supply: %d\n", ret); - return ret; - } - } - - if (ctx->pwm_value == 0) - return 0; - return pwm_apply_state(ctx->pwm, &ctx->pwm_state); + return set_pwm(ctx, ctx->pwm_value); } -#endif -static SIMPLE_DEV_PM_OPS(pwm_fan_pm, pwm_fan_suspend, pwm_fan_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(pwm_fan_pm, pwm_fan_suspend, pwm_fan_resume); static const struct of_device_id of_pwm_fan_match[] = { { .compatible = "pwm-fan", }, @@ -522,7 +666,7 @@ static struct platform_driver pwm_fan_driver = { .shutdown = pwm_fan_shutdown, .driver = { .name = "pwm-fan", - .pm = &pwm_fan_pm, + .pm = pm_sleep_ptr(&pwm_fan_pm), .of_match_table = of_pwm_fan_match, }, }; diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c index 5187c6dd5a4f0400f990011f10e342ea16e4f73c..4d75385f7d5ecfec0e0f7703f09e258c18a2e67e 100644 --- a/drivers/hwmon/scpi-hwmon.c +++ b/drivers/hwmon/scpi-hwmon.c @@ -62,9 +62,9 @@ static void scpi_scale_reading(u64 *value, struct sensor_data *sensor) } } -static int scpi_read_temp(void *dev, int *temp) +static int scpi_read_temp(struct thermal_zone_device *tz, int *temp) { - struct scpi_thermal_zone *zone = dev; + struct scpi_thermal_zone *zone = tz->devdata; struct scpi_sensors *scpi_sensors = zone->scpi_sensors; struct scpi_ops *scpi_ops = scpi_sensors->scpi_ops; struct sensor_data *sensor = &scpi_sensors->data[zone->sensor_id]; @@ -121,7 +121,7 @@ scpi_show_label(struct device *dev, struct device_attribute *attr, char *buf) return sprintf(buf, "%s\n", sensor->info.name); } -static const struct thermal_zone_of_device_ops scpi_sensor_ops = { +static const struct thermal_zone_device_ops scpi_sensor_ops = { .get_temp = scpi_read_temp, }; @@ -275,10 +275,10 @@ static int scpi_hwmon_probe(struct platform_device *pdev) zone->sensor_id = i; zone->scpi_sensors = scpi_sensors; - z = devm_thermal_zone_of_sensor_register(dev, - sensor->info.sensor_id, - zone, - &scpi_sensor_ops); + z = devm_thermal_of_zone_register(dev, + sensor->info.sensor_id, + zone, + &scpi_sensor_ops); /* * The call to thermal_zone_of_sensor_register returns * an error for sensors that are not associated with diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c index c19df3ade48e3023d9d4052fe94af2674e6701d5..13ac2d8f22c79b77817e1faa2a3f6c1090977caf 100644 --- a/drivers/hwmon/sht4x.c +++ b/drivers/hwmon/sht4x.c @@ -129,7 +129,7 @@ unlock: static ssize_t sht4x_interval_write(struct sht4x_data *data, long val) { - data->update_interval = clamp_val(val, SHT4X_MIN_POLL_INTERVAL, UINT_MAX); + data->update_interval = clamp_val(val, SHT4X_MIN_POLL_INTERVAL, INT_MAX); return 0; } diff --git a/drivers/hwmon/sis5595.c b/drivers/hwmon/sis5595.c index 018cb5a7651fba649257a51d7f0e8ca9df1ba61c..b0b05fd12221689cd77a48887dccc75cc4bceb0f 100644 --- a/drivers/hwmon/sis5595.c +++ b/drivers/hwmon/sis5595.c @@ -37,6 +37,7 @@ * 735 0008 0735 */ +#define DRIVER_NAME "sis5595" #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include @@ -191,21 +192,75 @@ struct sis5595_data { static struct pci_dev *s_bridge; /* pointer to the (only) sis5595 */ -static int sis5595_probe(struct platform_device *pdev); -static int sis5595_remove(struct platform_device *pdev); +/* ISA access must be locked explicitly. */ +static int sis5595_read_value(struct sis5595_data *data, u8 reg) +{ + int res; -static int sis5595_read_value(struct sis5595_data *data, u8 reg); -static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value); -static struct sis5595_data *sis5595_update_device(struct device *dev); -static void sis5595_init_device(struct sis5595_data *data); + mutex_lock(&data->lock); + outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET); + res = inb_p(data->addr + SIS5595_DATA_REG_OFFSET); + mutex_unlock(&data->lock); + return res; +} -static struct platform_driver sis5595_driver = { - .driver = { - .name = "sis5595", - }, - .probe = sis5595_probe, - .remove = sis5595_remove, -}; +static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value) +{ + mutex_lock(&data->lock); + outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET); + outb_p(value, data->addr + SIS5595_DATA_REG_OFFSET); + mutex_unlock(&data->lock); +} + +static struct sis5595_data *sis5595_update_device(struct device *dev) +{ + struct sis5595_data *data = dev_get_drvdata(dev); + int i; + + mutex_lock(&data->update_lock); + + if (time_after(jiffies, data->last_updated + HZ + HZ / 2) + || !data->valid) { + + for (i = 0; i <= data->maxins; i++) { + data->in[i] = + sis5595_read_value(data, SIS5595_REG_IN(i)); + data->in_min[i] = + sis5595_read_value(data, + SIS5595_REG_IN_MIN(i)); + data->in_max[i] = + sis5595_read_value(data, + SIS5595_REG_IN_MAX(i)); + } + for (i = 0; i < 2; i++) { + data->fan[i] = + sis5595_read_value(data, SIS5595_REG_FAN(i)); + data->fan_min[i] = + sis5595_read_value(data, + SIS5595_REG_FAN_MIN(i)); + } + if (data->maxins == 3) { + data->temp = + sis5595_read_value(data, SIS5595_REG_TEMP); + data->temp_over = + sis5595_read_value(data, SIS5595_REG_TEMP_OVER); + data->temp_hyst = + sis5595_read_value(data, SIS5595_REG_TEMP_HYST); + } + i = sis5595_read_value(data, SIS5595_REG_FANDIV); + data->fan_div[0] = (i >> 4) & 0x03; + data->fan_div[1] = i >> 6; + data->alarms = + sis5595_read_value(data, SIS5595_REG_ALARM1) | + (sis5595_read_value(data, SIS5595_REG_ALARM2) << 8); + data->last_updated = jiffies; + data->valid = true; + } + + mutex_unlock(&data->update_lock); + + return data; +} /* 4 Voltages */ static ssize_t in_show(struct device *dev, struct device_attribute *da, @@ -568,6 +623,15 @@ static const struct attribute_group sis5595_group_temp1 = { .attrs = sis5595_attributes_temp1, }; +/* Called when we have found a new SIS5595. */ +static void sis5595_init_device(struct sis5595_data *data) +{ + u8 config = sis5595_read_value(data, SIS5595_REG_CONFIG); + if (!(config & 0x01)) + sis5595_write_value(data, SIS5595_REG_CONFIG, + (config & 0xf7) | 0x01); +} + /* This is called when the module is loaded */ static int sis5595_probe(struct platform_device *pdev) { @@ -580,7 +644,7 @@ static int sis5595_probe(struct platform_device *pdev) /* Reserve the ISA region */ res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!devm_request_region(&pdev->dev, res->start, SIS5595_EXTENT, - sis5595_driver.driver.name)) + DRIVER_NAME)) return -EBUSY; data = devm_kzalloc(&pdev->dev, sizeof(struct sis5595_data), @@ -591,7 +655,7 @@ static int sis5595_probe(struct platform_device *pdev) mutex_init(&data->lock); mutex_init(&data->update_lock); data->addr = res->start; - data->name = "sis5595"; + data->name = DRIVER_NAME; platform_set_drvdata(pdev, data); /* @@ -657,85 +721,6 @@ static int sis5595_remove(struct platform_device *pdev) return 0; } -/* ISA access must be locked explicitly. */ -static int sis5595_read_value(struct sis5595_data *data, u8 reg) -{ - int res; - - mutex_lock(&data->lock); - outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET); - res = inb_p(data->addr + SIS5595_DATA_REG_OFFSET); - mutex_unlock(&data->lock); - return res; -} - -static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value) -{ - mutex_lock(&data->lock); - outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET); - outb_p(value, data->addr + SIS5595_DATA_REG_OFFSET); - mutex_unlock(&data->lock); -} - -/* Called when we have found a new SIS5595. */ -static void sis5595_init_device(struct sis5595_data *data) -{ - u8 config = sis5595_read_value(data, SIS5595_REG_CONFIG); - if (!(config & 0x01)) - sis5595_write_value(data, SIS5595_REG_CONFIG, - (config & 0xf7) | 0x01); -} - -static struct sis5595_data *sis5595_update_device(struct device *dev) -{ - struct sis5595_data *data = dev_get_drvdata(dev); - int i; - - mutex_lock(&data->update_lock); - - if (time_after(jiffies, data->last_updated + HZ + HZ / 2) - || !data->valid) { - - for (i = 0; i <= data->maxins; i++) { - data->in[i] = - sis5595_read_value(data, SIS5595_REG_IN(i)); - data->in_min[i] = - sis5595_read_value(data, - SIS5595_REG_IN_MIN(i)); - data->in_max[i] = - sis5595_read_value(data, - SIS5595_REG_IN_MAX(i)); - } - for (i = 0; i < 2; i++) { - data->fan[i] = - sis5595_read_value(data, SIS5595_REG_FAN(i)); - data->fan_min[i] = - sis5595_read_value(data, - SIS5595_REG_FAN_MIN(i)); - } - if (data->maxins == 3) { - data->temp = - sis5595_read_value(data, SIS5595_REG_TEMP); - data->temp_over = - sis5595_read_value(data, SIS5595_REG_TEMP_OVER); - data->temp_hyst = - sis5595_read_value(data, SIS5595_REG_TEMP_HYST); - } - i = sis5595_read_value(data, SIS5595_REG_FANDIV); - data->fan_div[0] = (i >> 4) & 0x03; - data->fan_div[1] = i >> 6; - data->alarms = - sis5595_read_value(data, SIS5595_REG_ALARM1) | - (sis5595_read_value(data, SIS5595_REG_ALARM2) << 8); - data->last_updated = jiffies; - data->valid = true; - } - - mutex_unlock(&data->update_lock); - - return data; -} - static const struct pci_device_id sis5595_pci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_503) }, { 0, } @@ -764,7 +749,7 @@ static int sis5595_device_add(unsigned short address) struct resource res = { .start = address, .end = address + SIS5595_EXTENT - 1, - .name = "sis5595", + .name = DRIVER_NAME, .flags = IORESOURCE_IO, }; int err; @@ -773,7 +758,7 @@ static int sis5595_device_add(unsigned short address) if (err) goto exit; - pdev = platform_device_alloc("sis5595", address); + pdev = platform_device_alloc(DRIVER_NAME, address); if (!pdev) { err = -ENOMEM; pr_err("Device allocation failed\n"); @@ -800,6 +785,14 @@ exit: return err; } +static struct platform_driver sis5595_driver = { + .driver = { + .name = DRIVER_NAME, + }, + .probe = sis5595_probe, + .remove = sis5595_remove, +}; + static int sis5595_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { @@ -886,7 +879,7 @@ exit: } static struct pci_driver sis5595_pci_driver = { - .name = "sis5595", + .name = DRIVER_NAME, .id_table = sis5595_pci_ids, .probe = sis5595_pci_probe, }; diff --git a/drivers/hwmon/smm665.c b/drivers/hwmon/smm665.c index 8c4ed72e5d684571bdfc78eed2528269088c30ff..c36bdbe423de001223263bcd2c7dab5cdbdf14ad 100644 --- a/drivers/hwmon/smm665.c +++ b/drivers/hwmon/smm665.c @@ -671,12 +671,11 @@ out_unregister: return ret; } -static int smm665_remove(struct i2c_client *client) +static void smm665_remove(struct i2c_client *client) { struct smm665_data *data = i2c_get_clientdata(client); i2c_unregister_device(data->cmdreg); - return 0; } static const struct i2c_device_id smm665_id[] = { diff --git a/drivers/hwmon/smsc47m192.c b/drivers/hwmon/smsc47m192.c index a5db15c087ae1b8d0da37778f90a8cf4b059d97b..70d2152234e210700cbe32b67f767d43dd631b77 100644 --- a/drivers/hwmon/smsc47m192.c +++ b/drivers/hwmon/smsc47m192.c @@ -582,7 +582,7 @@ static int smsc47m192_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "smsc47m192", I2C_NAME_SIZE); + strscpy(info->type, "smsc47m192", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/sparx5-temp.c b/drivers/hwmon/sparx5-temp.c index 98be48e3a22aa0ec374b87ff578779940cda9264..04fd8505e5d6caf15772b3dc1d086ae131e0187d 100644 --- a/drivers/hwmon/sparx5-temp.c +++ b/drivers/hwmon/sparx5-temp.c @@ -26,13 +26,6 @@ struct s5_hwmon { struct clk *clk; }; -static void s5_temp_clk_disable(void *data) -{ - struct clk *clk = data; - - clk_disable_unprepare(clk); -} - static void s5_temp_enable(struct s5_hwmon *hwmon) { u32 val = readl(hwmon->base + TEMP_CFG); @@ -113,7 +106,6 @@ static int s5_temp_probe(struct platform_device *pdev) { struct device *hwmon_dev; struct s5_hwmon *hwmon; - int ret; hwmon = devm_kzalloc(&pdev->dev, sizeof(*hwmon), GFP_KERNEL); if (!hwmon) @@ -123,19 +115,10 @@ static int s5_temp_probe(struct platform_device *pdev) if (IS_ERR(hwmon->base)) return PTR_ERR(hwmon->base); - hwmon->clk = devm_clk_get(&pdev->dev, NULL); + hwmon->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(hwmon->clk)) return PTR_ERR(hwmon->clk); - ret = clk_prepare_enable(hwmon->clk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(&pdev->dev, s5_temp_clk_disable, - hwmon->clk); - if (ret) - return ret; - s5_temp_enable(hwmon); hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev, diff --git a/drivers/hwmon/stts751.c b/drivers/hwmon/stts751.c index 0ed28408aa077c3ed20133d2087e31fd5f28fba8..2f67c6747eadbf4f80a029c77dcf1ef1a57f06e5 100644 --- a/drivers/hwmon/stts751.c +++ b/drivers/hwmon/stts751.c @@ -692,7 +692,7 @@ static int stts751_detect(struct i2c_client *new_client, } dev_dbg(&new_client->dev, "Chip %s detected", name); - strlcpy(info->type, stts751_id[0].name, I2C_NAME_SIZE); + strscpy(info->type, stts751_id[0].name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/thmc50.c b/drivers/hwmon/thmc50.c index 6a804f5036f4774fb28105ca6cf300df69edf9c6..81cdb012993caca0ce4072e9aa7043bad390ea81 100644 --- a/drivers/hwmon/thmc50.c +++ b/drivers/hwmon/thmc50.c @@ -352,7 +352,7 @@ static int thmc50_detect(struct i2c_client *client, pr_debug("thmc50: Detected %s (version %x, revision %x)\n", type_name, (revision >> 4) - 0xc, revision & 0xf); - strlcpy(info->type, type_name, I2C_NAME_SIZE); + strscpy(info->type, type_name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c index e867a0c2e5399ca04a83c2288f30dd7268a607c3..2bf496a62206aed9dcb13e94df15458216b7ab72 100644 --- a/drivers/hwmon/tmp102.c +++ b/drivers/hwmon/tmp102.c @@ -260,7 +260,6 @@ static int tmp102_probe(struct i2c_client *client) return 0; } -#ifdef CONFIG_PM_SLEEP static int tmp102_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); @@ -283,9 +282,8 @@ static int tmp102_resume(struct device *dev) return err; } -#endif /* CONFIG_PM */ -static SIMPLE_DEV_PM_OPS(tmp102_dev_pm_ops, tmp102_suspend, tmp102_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(tmp102_dev_pm_ops, tmp102_suspend, tmp102_resume); static const struct i2c_device_id tmp102_id[] = { { "tmp102", 0 }, @@ -302,7 +300,7 @@ MODULE_DEVICE_TABLE(of, tmp102_of_match); static struct i2c_driver tmp102_driver = { .driver.name = DRIVER_NAME, .driver.of_match_table = of_match_ptr(tmp102_of_match), - .driver.pm = &tmp102_dev_pm_ops, + .driver.pm = pm_sleep_ptr(&tmp102_dev_pm_ops), .probe_new = tmp102_probe, .id_table = tmp102_id, }; diff --git a/drivers/hwmon/tmp103.c b/drivers/hwmon/tmp103.c index 5cab4436aa77d655ea61626524e1c16afae1532d..56d5cbf36a45b4e5a499072d45521e4e2bef9c9d 100644 --- a/drivers/hwmon/tmp103.c +++ b/drivers/hwmon/tmp103.c @@ -178,7 +178,7 @@ static int tmp103_probe(struct i2c_client *client) return PTR_ERR_OR_ZERO(hwmon_dev); } -static int __maybe_unused tmp103_suspend(struct device *dev) +static int tmp103_suspend(struct device *dev) { struct regmap *regmap = dev_get_drvdata(dev); @@ -186,7 +186,7 @@ static int __maybe_unused tmp103_suspend(struct device *dev) TMP103_CONF_SD_MASK, 0); } -static int __maybe_unused tmp103_resume(struct device *dev) +static int tmp103_resume(struct device *dev) { struct regmap *regmap = dev_get_drvdata(dev); @@ -194,7 +194,7 @@ static int __maybe_unused tmp103_resume(struct device *dev) TMP103_CONF_SD_MASK, TMP103_CONF_SD); } -static SIMPLE_DEV_PM_OPS(tmp103_dev_pm_ops, tmp103_suspend, tmp103_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(tmp103_dev_pm_ops, tmp103_suspend, tmp103_resume); static const struct i2c_device_id tmp103_id[] = { { "tmp103", 0 }, @@ -212,7 +212,7 @@ static struct i2c_driver tmp103_driver = { .driver = { .name = "tmp103", .of_match_table = of_match_ptr(tmp103_of_match), - .pm = &tmp103_dev_pm_ops, + .pm = pm_sleep_ptr(&tmp103_dev_pm_ops), }, .probe_new = tmp103_probe, .id_table = tmp103_id, diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c index 5435664c3f6e592596b8fa434efa3fb93831cfb8..acb4ba750b09c7cd4b1ffc28b5ecfc297548a094 100644 --- a/drivers/hwmon/tmp108.c +++ b/drivers/hwmon/tmp108.c @@ -390,7 +390,7 @@ static int tmp108_probe(struct i2c_client *client) return PTR_ERR_OR_ZERO(hwmon_dev); } -static int __maybe_unused tmp108_suspend(struct device *dev) +static int tmp108_suspend(struct device *dev) { struct tmp108 *tmp108 = dev_get_drvdata(dev); @@ -398,7 +398,7 @@ static int __maybe_unused tmp108_suspend(struct device *dev) TMP108_CONF_MODE_MASK, TMP108_MODE_SHUTDOWN); } -static int __maybe_unused tmp108_resume(struct device *dev) +static int tmp108_resume(struct device *dev) { struct tmp108 *tmp108 = dev_get_drvdata(dev); int err; @@ -410,7 +410,7 @@ static int __maybe_unused tmp108_resume(struct device *dev) return err; } -static SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume); static const struct i2c_device_id tmp108_i2c_ids[] = { { "tmp108", 0 }, @@ -429,7 +429,7 @@ MODULE_DEVICE_TABLE(of, tmp108_of_ids); static struct i2c_driver tmp108_driver = { .driver = { .name = DRIVER_NAME, - .pm = &tmp108_dev_pm_ops, + .pm = pm_sleep_ptr(&tmp108_dev_pm_ops), .of_match_table = of_match_ptr(tmp108_of_ids), }, .probe_new = tmp108_probe, diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c index cc0a1c219b1f962bfb16fa227d45e4e83673f305..f358ba67962675352fae9cfa3daa024dac4b8453 100644 --- a/drivers/hwmon/tmp401.c +++ b/drivers/hwmon/tmp401.c @@ -671,7 +671,7 @@ static int tmp401_detect(struct i2c_client *client, if (reg > 15) return -ENODEV; - strlcpy(info->type, tmp401_id[kind].name, I2C_NAME_SIZE); + strscpy(info->type, tmp401_id[kind].name, I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/tmp421.c b/drivers/hwmon/tmp421.c index 1fd8d41d90c84a76c7d6467d3f75c52f85c6941a..45fd7fb5ee01f5c688d5b80d8922ef0353de0e9b 100644 --- a/drivers/hwmon/tmp421.c +++ b/drivers/hwmon/tmp421.c @@ -353,7 +353,7 @@ static int tmp421_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, tmp421_id[kind].name, I2C_NAME_SIZE); + strscpy(info->type, tmp421_id[kind].name, I2C_NAME_SIZE); dev_info(&adapter->dev, "Detected TI %s chip at 0x%02x\n", names[kind], client->addr); diff --git a/drivers/hwmon/tps23861.c b/drivers/hwmon/tps23861.c index 42762e87b0147bf6e908ec0602857ce5126fb864..68c77c493270023881177196b498460a1bafcdc1 100644 --- a/drivers/hwmon/tps23861.c +++ b/drivers/hwmon/tps23861.c @@ -372,29 +372,12 @@ static const struct hwmon_chip_info tps23861_chip_info = { .info = tps23861_info, }; -static char *tps23861_port_operating_mode(struct tps23861_data *data, int port) +static char *port_operating_mode_string(uint8_t mode_reg, unsigned int port) { - unsigned int regval; - int mode; - - regmap_read(data->regmap, OPERATING_MODE, ®val); + unsigned int mode = ~0; - switch (port) { - case 1: - mode = FIELD_GET(OPERATING_MODE_PORT_1_MASK, regval); - break; - case 2: - mode = FIELD_GET(OPERATING_MODE_PORT_2_MASK, regval); - break; - case 3: - mode = FIELD_GET(OPERATING_MODE_PORT_3_MASK, regval); - break; - case 4: - mode = FIELD_GET(OPERATING_MODE_PORT_4_MASK, regval); - break; - default: - mode = -EINVAL; - } + if (port < TPS23861_NUM_PORTS) + mode = (mode_reg >> (2 * port)) & OPERATING_MODE_PORT_1_MASK; switch (mode) { case OPERATING_MODE_OFF: @@ -410,15 +393,9 @@ static char *tps23861_port_operating_mode(struct tps23861_data *data, int port) } } -static char *tps23861_port_detect_status(struct tps23861_data *data, int port) +static char *port_detect_status_string(uint8_t status_reg) { - unsigned int regval; - - regmap_read(data->regmap, - PORT_1_STATUS + (port - 1), - ®val); - - switch (FIELD_GET(PORT_STATUS_DETECT_MASK, regval)) { + switch (FIELD_GET(PORT_STATUS_DETECT_MASK, status_reg)) { case PORT_DETECT_UNKNOWN: return "Unknown device"; case PORT_DETECT_SHORT: @@ -448,15 +425,9 @@ static char *tps23861_port_detect_status(struct tps23861_data *data, int port) } } -static char *tps23861_port_class_status(struct tps23861_data *data, int port) +static char *port_class_status_string(uint8_t status_reg) { - unsigned int regval; - - regmap_read(data->regmap, - PORT_1_STATUS + (port - 1), - ®val); - - switch (FIELD_GET(PORT_STATUS_CLASS_MASK, regval)) { + switch (FIELD_GET(PORT_STATUS_CLASS_MASK, status_reg)) { case PORT_CLASS_UNKNOWN: return "Unknown"; case PORT_CLASS_RESERVED: @@ -479,32 +450,27 @@ static char *tps23861_port_class_status(struct tps23861_data *data, int port) } } -static char *tps23861_port_poe_plus_status(struct tps23861_data *data, int port) +static char *port_poe_plus_status_string(uint8_t poe_plus, unsigned int port) { - unsigned int regval; - - regmap_read(data->regmap, POE_PLUS, ®val); - - if (BIT(port + 3) & regval) - return "Yes"; - else - return "No"; + return (BIT(port + 4) & poe_plus) ? "Yes" : "No"; } static int tps23861_port_resistance(struct tps23861_data *data, int port) { - u16 regval; + unsigned int raw_val; + __le16 regval; regmap_bulk_read(data->regmap, - PORT_1_RESISTANCE_LSB + PORT_N_RESISTANCE_LSB_OFFSET * (port - 1), + PORT_1_RESISTANCE_LSB + PORT_N_RESISTANCE_LSB_OFFSET * port, ®val, 2); - switch (FIELD_GET(PORT_RESISTANCE_RSN_MASK, regval)) { + raw_val = le16_to_cpu(regval); + switch (FIELD_GET(PORT_RESISTANCE_RSN_MASK, raw_val)) { case PORT_RESISTANCE_RSN_OTHER: - return (FIELD_GET(PORT_RESISTANCE_MASK, regval) * RESISTANCE_LSB) / 10000; + return (FIELD_GET(PORT_RESISTANCE_MASK, raw_val) * RESISTANCE_LSB) / 10000; case PORT_RESISTANCE_RSN_LOW: - return (FIELD_GET(PORT_RESISTANCE_MASK, regval) * RESISTANCE_LSB_LOW) / 10000; + return (FIELD_GET(PORT_RESISTANCE_MASK, raw_val) * RESISTANCE_LSB_LOW) / 10000; case PORT_RESISTANCE_RSN_SHORT: case PORT_RESISTANCE_RSN_OPEN: default: @@ -515,14 +481,19 @@ static int tps23861_port_resistance(struct tps23861_data *data, int port) static int tps23861_port_status_show(struct seq_file *s, void *data) { struct tps23861_data *priv = s->private; - int i; - - for (i = 1; i < TPS23861_NUM_PORTS + 1; i++) { - seq_printf(s, "Port: \t\t%d\n", i); - seq_printf(s, "Operating mode: %s\n", tps23861_port_operating_mode(priv, i)); - seq_printf(s, "Detected: \t%s\n", tps23861_port_detect_status(priv, i)); - seq_printf(s, "Class: \t\t%s\n", tps23861_port_class_status(priv, i)); - seq_printf(s, "PoE Plus: \t%s\n", tps23861_port_poe_plus_status(priv, i)); + unsigned int i, mode, poe_plus, status; + + regmap_read(priv->regmap, OPERATING_MODE, &mode); + regmap_read(priv->regmap, POE_PLUS, &poe_plus); + + for (i = 0; i < TPS23861_NUM_PORTS; i++) { + regmap_read(priv->regmap, PORT_1_STATUS + i, &status); + + seq_printf(s, "Port: \t\t%d\n", i + 1); + seq_printf(s, "Operating mode: %s\n", port_operating_mode_string(mode, i)); + seq_printf(s, "Detected: \t%s\n", port_detect_status_string(status)); + seq_printf(s, "Class: \t\t%s\n", port_class_status_string(status)); + seq_printf(s, "PoE Plus: \t%s\n", port_poe_plus_status_string(poe_plus, i)); seq_printf(s, "Resistance: \t%d\n", tps23861_port_resistance(priv, i)); seq_putc(s, '\n'); } @@ -532,9 +503,17 @@ static int tps23861_port_status_show(struct seq_file *s, void *data) DEFINE_SHOW_ATTRIBUTE(tps23861_port_status); -static void tps23861_init_debugfs(struct tps23861_data *data) +static void tps23861_init_debugfs(struct tps23861_data *data, + struct device *hwmon_dev) { - data->debugfs_dir = debugfs_create_dir(data->client->name, NULL); + const char *debugfs_name; + + debugfs_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "%s-%s", + data->client->name, dev_name(hwmon_dev)); + if (!debugfs_name) + return; + + data->debugfs_dir = debugfs_create_dir(debugfs_name, NULL); debugfs_create_file("port_status", 0400, @@ -583,18 +562,16 @@ static int tps23861_probe(struct i2c_client *client) if (IS_ERR(hwmon_dev)) return PTR_ERR(hwmon_dev); - tps23861_init_debugfs(data); + tps23861_init_debugfs(data, hwmon_dev); return 0; } -static int tps23861_remove(struct i2c_client *client) +static void tps23861_remove(struct i2c_client *client) { struct tps23861_data *data = i2c_get_clientdata(client); debugfs_remove_recursive(data->debugfs_dir); - - return 0; } static const struct of_device_id __maybe_unused tps23861_of_match[] = { diff --git a/drivers/hwmon/via686a.c b/drivers/hwmon/via686a.c index 55634110c2f962cc3fe04e81f78b09d8b60a24de..37d7374896f672bba9a2319114d96def5cdc805e 100644 --- a/drivers/hwmon/via686a.c +++ b/drivers/hwmon/via686a.c @@ -34,6 +34,8 @@ #include #include +#define DRIVER_NAME "via686a" + /* * If force_addr is set to anything different from 0, we forcibly enable * the device at the given address. @@ -321,9 +323,6 @@ struct via686a_data { static struct pci_dev *s_bridge; /* pointer to the (only) via686a */ -static int via686a_probe(struct platform_device *pdev); -static int via686a_remove(struct platform_device *pdev); - static inline int via686a_read_value(struct via686a_data *data, u8 reg) { return inb_p(data->addr + reg); @@ -335,8 +334,76 @@ static inline void via686a_write_value(struct via686a_data *data, u8 reg, outb_p(value, data->addr + reg); } -static struct via686a_data *via686a_update_device(struct device *dev); -static void via686a_init_device(struct via686a_data *data); +static void via686a_update_fan_div(struct via686a_data *data) +{ + int reg = via686a_read_value(data, VIA686A_REG_FANDIV); + data->fan_div[0] = (reg >> 4) & 0x03; + data->fan_div[1] = reg >> 6; +} + +static struct via686a_data *via686a_update_device(struct device *dev) +{ + struct via686a_data *data = dev_get_drvdata(dev); + int i; + + mutex_lock(&data->update_lock); + + if (time_after(jiffies, data->last_updated + HZ + HZ / 2) + || !data->valid) { + for (i = 0; i <= 4; i++) { + data->in[i] = + via686a_read_value(data, VIA686A_REG_IN(i)); + data->in_min[i] = via686a_read_value(data, + VIA686A_REG_IN_MIN + (i)); + data->in_max[i] = + via686a_read_value(data, VIA686A_REG_IN_MAX(i)); + } + for (i = 1; i <= 2; i++) { + data->fan[i - 1] = + via686a_read_value(data, VIA686A_REG_FAN(i)); + data->fan_min[i - 1] = via686a_read_value(data, + VIA686A_REG_FAN_MIN(i)); + } + for (i = 0; i <= 2; i++) { + data->temp[i] = via686a_read_value(data, + VIA686A_REG_TEMP[i]) << 2; + data->temp_over[i] = + via686a_read_value(data, + VIA686A_REG_TEMP_OVER[i]); + data->temp_hyst[i] = + via686a_read_value(data, + VIA686A_REG_TEMP_HYST[i]); + } + /* + * add in lower 2 bits + * temp1 uses bits 7-6 of VIA686A_REG_TEMP_LOW1 + * temp2 uses bits 5-4 of VIA686A_REG_TEMP_LOW23 + * temp3 uses bits 7-6 of VIA686A_REG_TEMP_LOW23 + */ + data->temp[0] |= (via686a_read_value(data, + VIA686A_REG_TEMP_LOW1) + & 0xc0) >> 6; + data->temp[1] |= + (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) & + 0x30) >> 4; + data->temp[2] |= + (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) & + 0xc0) >> 6; + + via686a_update_fan_div(data); + data->alarms = + via686a_read_value(data, + VIA686A_REG_ALARM1) | + (via686a_read_value(data, VIA686A_REG_ALARM2) << 8); + data->last_updated = jiffies; + data->valid = true; + } + + mutex_unlock(&data->update_lock); + + return data; +} /* following are the sysfs callback functions */ @@ -654,13 +721,23 @@ static const struct attribute_group via686a_group = { .attrs = via686a_attributes, }; -static struct platform_driver via686a_driver = { - .driver = { - .name = "via686a", - }, - .probe = via686a_probe, - .remove = via686a_remove, -}; +static void via686a_init_device(struct via686a_data *data) +{ + u8 reg; + + /* Start monitoring */ + reg = via686a_read_value(data, VIA686A_REG_CONFIG); + via686a_write_value(data, VIA686A_REG_CONFIG, (reg | 0x01) & 0x7F); + + /* Configure temp interrupt mode for continuous-interrupt operation */ + reg = via686a_read_value(data, VIA686A_REG_TEMP_MODE); + via686a_write_value(data, VIA686A_REG_TEMP_MODE, + (reg & ~VIA686A_TEMP_MODE_MASK) + | VIA686A_TEMP_MODE_CONTINUOUS); + + /* Pre-read fan clock divisor values */ + via686a_update_fan_div(data); +} /* This is called when the module is loaded */ static int via686a_probe(struct platform_device *pdev) @@ -672,7 +749,7 @@ static int via686a_probe(struct platform_device *pdev) /* Reserve the ISA region */ res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!devm_request_region(&pdev->dev, res->start, VIA686A_EXTENT, - via686a_driver.driver.name)) { + DRIVER_NAME)) { dev_err(&pdev->dev, "Region 0x%lx-0x%lx already in use!\n", (unsigned long)res->start, (unsigned long)res->end); return -ENODEV; @@ -685,7 +762,7 @@ static int via686a_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); data->addr = res->start; - data->name = "via686a"; + data->name = DRIVER_NAME; mutex_init(&data->update_lock); /* Initialize the VIA686A chip */ @@ -719,94 +796,13 @@ static int via686a_remove(struct platform_device *pdev) return 0; } -static void via686a_update_fan_div(struct via686a_data *data) -{ - int reg = via686a_read_value(data, VIA686A_REG_FANDIV); - data->fan_div[0] = (reg >> 4) & 0x03; - data->fan_div[1] = reg >> 6; -} - -static void via686a_init_device(struct via686a_data *data) -{ - u8 reg; - - /* Start monitoring */ - reg = via686a_read_value(data, VIA686A_REG_CONFIG); - via686a_write_value(data, VIA686A_REG_CONFIG, (reg | 0x01) & 0x7F); - - /* Configure temp interrupt mode for continuous-interrupt operation */ - reg = via686a_read_value(data, VIA686A_REG_TEMP_MODE); - via686a_write_value(data, VIA686A_REG_TEMP_MODE, - (reg & ~VIA686A_TEMP_MODE_MASK) - | VIA686A_TEMP_MODE_CONTINUOUS); - - /* Pre-read fan clock divisor values */ - via686a_update_fan_div(data); -} - -static struct via686a_data *via686a_update_device(struct device *dev) -{ - struct via686a_data *data = dev_get_drvdata(dev); - int i; - - mutex_lock(&data->update_lock); - - if (time_after(jiffies, data->last_updated + HZ + HZ / 2) - || !data->valid) { - for (i = 0; i <= 4; i++) { - data->in[i] = - via686a_read_value(data, VIA686A_REG_IN(i)); - data->in_min[i] = via686a_read_value(data, - VIA686A_REG_IN_MIN - (i)); - data->in_max[i] = - via686a_read_value(data, VIA686A_REG_IN_MAX(i)); - } - for (i = 1; i <= 2; i++) { - data->fan[i - 1] = - via686a_read_value(data, VIA686A_REG_FAN(i)); - data->fan_min[i - 1] = via686a_read_value(data, - VIA686A_REG_FAN_MIN(i)); - } - for (i = 0; i <= 2; i++) { - data->temp[i] = via686a_read_value(data, - VIA686A_REG_TEMP[i]) << 2; - data->temp_over[i] = - via686a_read_value(data, - VIA686A_REG_TEMP_OVER[i]); - data->temp_hyst[i] = - via686a_read_value(data, - VIA686A_REG_TEMP_HYST[i]); - } - /* - * add in lower 2 bits - * temp1 uses bits 7-6 of VIA686A_REG_TEMP_LOW1 - * temp2 uses bits 5-4 of VIA686A_REG_TEMP_LOW23 - * temp3 uses bits 7-6 of VIA686A_REG_TEMP_LOW23 - */ - data->temp[0] |= (via686a_read_value(data, - VIA686A_REG_TEMP_LOW1) - & 0xc0) >> 6; - data->temp[1] |= - (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) & - 0x30) >> 4; - data->temp[2] |= - (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) & - 0xc0) >> 6; - - via686a_update_fan_div(data); - data->alarms = - via686a_read_value(data, - VIA686A_REG_ALARM1) | - (via686a_read_value(data, VIA686A_REG_ALARM2) << 8); - data->last_updated = jiffies; - data->valid = true; - } - - mutex_unlock(&data->update_lock); - - return data; -} +static struct platform_driver via686a_driver = { + .driver = { + .name = DRIVER_NAME, + }, + .probe = via686a_probe, + .remove = via686a_remove, +}; static const struct pci_device_id via686a_pci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C686_4) }, @@ -819,7 +815,7 @@ static int via686a_device_add(unsigned short address) struct resource res = { .start = address, .end = address + VIA686A_EXTENT - 1, - .name = "via686a", + .name = DRIVER_NAME, .flags = IORESOURCE_IO, }; int err; @@ -828,7 +824,7 @@ static int via686a_device_add(unsigned short address) if (err) goto exit; - pdev = platform_device_alloc("via686a", address); + pdev = platform_device_alloc(DRIVER_NAME, address); if (!pdev) { err = -ENOMEM; pr_err("Device allocation failed\n"); @@ -918,7 +914,7 @@ exit: } static struct pci_driver via686a_pci_driver = { - .name = "via686a", + .name = DRIVER_NAME, .id_table = via686a_pci_ids, .probe = via686a_pci_probe, }; diff --git a/drivers/hwmon/vt8231.c b/drivers/hwmon/vt8231.c index 03275ac8ba72c9cd4baf271a74c0b31ef3ca4aec..3b7f8922b0d5a67ec76a87e5d78e195815e286dd 100644 --- a/drivers/hwmon/vt8231.c +++ b/drivers/hwmon/vt8231.c @@ -38,6 +38,8 @@ static struct platform_device *pdev; #define VT8231_BASE_REG 0x70 #define VT8231_ENABLE_REG 0x74 +#define DRIVER_NAME "vt8231" + /* * The VT8231 registers * @@ -162,10 +164,6 @@ struct vt8231_data { }; static struct pci_dev *s_bridge; -static int vt8231_probe(struct platform_device *pdev); -static int vt8231_remove(struct platform_device *pdev); -static struct vt8231_data *vt8231_update_device(struct device *dev); -static void vt8231_init_device(struct vt8231_data *data); static inline int vt8231_read_value(struct vt8231_data *data, u8 reg) { @@ -178,6 +176,74 @@ static inline void vt8231_write_value(struct vt8231_data *data, u8 reg, outb_p(value, data->addr + reg); } +static struct vt8231_data *vt8231_update_device(struct device *dev) +{ + struct vt8231_data *data = dev_get_drvdata(dev); + int i; + u16 low; + + mutex_lock(&data->update_lock); + + if (time_after(jiffies, data->last_updated + HZ + HZ / 2) + || !data->valid) { + for (i = 0; i < 6; i++) { + if (ISVOLT(i, data->uch_config)) { + data->in[i] = vt8231_read_value(data, + regvolt[i]); + data->in_min[i] = vt8231_read_value(data, + regvoltmin[i]); + data->in_max[i] = vt8231_read_value(data, + regvoltmax[i]); + } + } + for (i = 0; i < 2; i++) { + data->fan[i] = vt8231_read_value(data, + VT8231_REG_FAN(i)); + data->fan_min[i] = vt8231_read_value(data, + VT8231_REG_FAN_MIN(i)); + } + + low = vt8231_read_value(data, VT8231_REG_TEMP_LOW01); + low = (low >> 6) | ((low & 0x30) >> 2) + | (vt8231_read_value(data, VT8231_REG_TEMP_LOW25) << 4); + for (i = 0; i < 6; i++) { + if (ISTEMP(i, data->uch_config)) { + data->temp[i] = (vt8231_read_value(data, + regtemp[i]) << 2) + | ((low >> (2 * i)) & 0x03); + data->temp_max[i] = vt8231_read_value(data, + regtempmax[i]); + data->temp_min[i] = vt8231_read_value(data, + regtempmin[i]); + } + } + + i = vt8231_read_value(data, VT8231_REG_FANDIV); + data->fan_div[0] = (i >> 4) & 0x03; + data->fan_div[1] = i >> 6; + data->alarms = vt8231_read_value(data, VT8231_REG_ALARM1) | + (vt8231_read_value(data, VT8231_REG_ALARM2) << 8); + + /* Set alarm flags correctly */ + if (!data->fan[0] && data->fan_min[0]) + data->alarms |= 0x40; + else if (data->fan[0] && !data->fan_min[0]) + data->alarms &= ~0x40; + + if (!data->fan[1] && data->fan_min[1]) + data->alarms |= 0x80; + else if (data->fan[1] && !data->fan_min[1]) + data->alarms &= ~0x80; + + data->last_updated = jiffies; + data->valid = true; + } + + mutex_unlock(&data->update_lock); + + return data; +} + /* following are the sysfs callback functions */ static ssize_t in_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -751,29 +817,11 @@ static const struct attribute_group vt8231_group = { .attrs = vt8231_attributes, }; -static struct platform_driver vt8231_driver = { - .driver = { - .name = "vt8231", - }, - .probe = vt8231_probe, - .remove = vt8231_remove, -}; - -static const struct pci_device_id vt8231_pci_ids[] = { - { PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8231_4) }, - { 0, } -}; - -MODULE_DEVICE_TABLE(pci, vt8231_pci_ids); - -static int vt8231_pci_probe(struct pci_dev *dev, - const struct pci_device_id *id); - -static struct pci_driver vt8231_pci_driver = { - .name = "vt8231", - .id_table = vt8231_pci_ids, - .probe = vt8231_pci_probe, -}; +static void vt8231_init_device(struct vt8231_data *data) +{ + vt8231_write_value(data, VT8231_REG_TEMP1_CONFIG, 0); + vt8231_write_value(data, VT8231_REG_TEMP2_CONFIG, 0); +} static int vt8231_probe(struct platform_device *pdev) { @@ -784,7 +832,7 @@ static int vt8231_probe(struct platform_device *pdev) /* Reserve the ISA region */ res = platform_get_resource(pdev, IORESOURCE_IO, 0); if (!devm_request_region(&pdev->dev, res->start, VT8231_EXTENT, - vt8231_driver.driver.name)) { + DRIVER_NAME)) { dev_err(&pdev->dev, "Region 0x%lx-0x%lx already in use!\n", (unsigned long)res->start, (unsigned long)res->end); return -ENODEV; @@ -796,7 +844,7 @@ static int vt8231_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); data->addr = res->start; - data->name = "vt8231"; + data->name = DRIVER_NAME; mutex_init(&data->update_lock); vt8231_init_device(data); @@ -863,86 +911,28 @@ static int vt8231_remove(struct platform_device *pdev) return 0; } -static void vt8231_init_device(struct vt8231_data *data) -{ - vt8231_write_value(data, VT8231_REG_TEMP1_CONFIG, 0); - vt8231_write_value(data, VT8231_REG_TEMP2_CONFIG, 0); -} - -static struct vt8231_data *vt8231_update_device(struct device *dev) -{ - struct vt8231_data *data = dev_get_drvdata(dev); - int i; - u16 low; - - mutex_lock(&data->update_lock); - - if (time_after(jiffies, data->last_updated + HZ + HZ / 2) - || !data->valid) { - for (i = 0; i < 6; i++) { - if (ISVOLT(i, data->uch_config)) { - data->in[i] = vt8231_read_value(data, - regvolt[i]); - data->in_min[i] = vt8231_read_value(data, - regvoltmin[i]); - data->in_max[i] = vt8231_read_value(data, - regvoltmax[i]); - } - } - for (i = 0; i < 2; i++) { - data->fan[i] = vt8231_read_value(data, - VT8231_REG_FAN(i)); - data->fan_min[i] = vt8231_read_value(data, - VT8231_REG_FAN_MIN(i)); - } - - low = vt8231_read_value(data, VT8231_REG_TEMP_LOW01); - low = (low >> 6) | ((low & 0x30) >> 2) - | (vt8231_read_value(data, VT8231_REG_TEMP_LOW25) << 4); - for (i = 0; i < 6; i++) { - if (ISTEMP(i, data->uch_config)) { - data->temp[i] = (vt8231_read_value(data, - regtemp[i]) << 2) - | ((low >> (2 * i)) & 0x03); - data->temp_max[i] = vt8231_read_value(data, - regtempmax[i]); - data->temp_min[i] = vt8231_read_value(data, - regtempmin[i]); - } - } - - i = vt8231_read_value(data, VT8231_REG_FANDIV); - data->fan_div[0] = (i >> 4) & 0x03; - data->fan_div[1] = i >> 6; - data->alarms = vt8231_read_value(data, VT8231_REG_ALARM1) | - (vt8231_read_value(data, VT8231_REG_ALARM2) << 8); - - /* Set alarm flags correctly */ - if (!data->fan[0] && data->fan_min[0]) - data->alarms |= 0x40; - else if (data->fan[0] && !data->fan_min[0]) - data->alarms &= ~0x40; - - if (!data->fan[1] && data->fan_min[1]) - data->alarms |= 0x80; - else if (data->fan[1] && !data->fan_min[1]) - data->alarms &= ~0x80; - data->last_updated = jiffies; - data->valid = true; - } +static struct platform_driver vt8231_driver = { + .driver = { + .name = DRIVER_NAME, + }, + .probe = vt8231_probe, + .remove = vt8231_remove, +}; - mutex_unlock(&data->update_lock); +static const struct pci_device_id vt8231_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8231_4) }, + { 0, } +}; - return data; -} +MODULE_DEVICE_TABLE(pci, vt8231_pci_ids); static int vt8231_device_add(unsigned short address) { struct resource res = { .start = address, .end = address + VT8231_EXTENT - 1, - .name = "vt8231", + .name = DRIVER_NAME, .flags = IORESOURCE_IO, }; int err; @@ -951,7 +941,7 @@ static int vt8231_device_add(unsigned short address) if (err) goto exit; - pdev = platform_device_alloc("vt8231", address); + pdev = platform_device_alloc(DRIVER_NAME, address); if (!pdev) { err = -ENOMEM; pr_err("Device allocation failed\n"); @@ -1040,6 +1030,12 @@ exit: return -ENODEV; } +static struct pci_driver vt8231_pci_driver = { + .name = DRIVER_NAME, + .id_table = vt8231_pci_ids, + .probe = vt8231_pci_probe, +}; + static int __init sm_vt8231_init(void) { return pci_register_driver(&vt8231_pci_driver); diff --git a/drivers/hwmon/w83627ehf.c b/drivers/hwmon/w83627ehf.c index af89b32a93a56b9504fab6040b0110f4fa60bb1e..939d4c35e713c1e10e169195e9bfed260d12f10f 100644 --- a/drivers/hwmon/w83627ehf.c +++ b/drivers/hwmon/w83627ehf.c @@ -1944,7 +1944,7 @@ static int __init w83627ehf_probe(struct platform_device *pdev) return PTR_ERR_OR_ZERO(hwmon_dev); } -static int __maybe_unused w83627ehf_suspend(struct device *dev) +static int w83627ehf_suspend(struct device *dev) { struct w83627ehf_data *data = w83627ehf_update_device(dev); @@ -1955,7 +1955,7 @@ static int __maybe_unused w83627ehf_suspend(struct device *dev) return 0; } -static int __maybe_unused w83627ehf_resume(struct device *dev) +static int w83627ehf_resume(struct device *dev) { struct w83627ehf_data *data = dev_get_drvdata(dev); int i; @@ -2010,12 +2010,12 @@ static int __maybe_unused w83627ehf_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(w83627ehf_dev_pm_ops, w83627ehf_suspend, w83627ehf_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(w83627ehf_dev_pm_ops, w83627ehf_suspend, w83627ehf_resume); static struct platform_driver w83627ehf_driver = { .driver = { .name = DRVNAME, - .pm = &w83627ehf_dev_pm_ops, + .pm = pm_sleep_ptr(&w83627ehf_dev_pm_ops), }, }; diff --git a/drivers/hwmon/w83627hf.c b/drivers/hwmon/w83627hf.c index 9be277156ed272fdb2abd8b85da30f7d6368c6a6..b638d672ac458caf541a8865b230ead920b740b0 100644 --- a/drivers/hwmon/w83627hf.c +++ b/drivers/hwmon/w83627hf.c @@ -389,14 +389,184 @@ struct w83627hf_data { #endif }; -static int w83627hf_probe(struct platform_device *pdev); -static int w83627hf_remove(struct platform_device *pdev); +/* Registers 0x50-0x5f are banked */ +static inline void w83627hf_set_bank(struct w83627hf_data *data, u16 reg) +{ + if ((reg & 0x00f0) == 0x50) { + outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET); + outb_p(reg >> 8, data->addr + W83781D_DATA_REG_OFFSET); + } +} + +/* Not strictly necessary, but play it safe for now */ +static inline void w83627hf_reset_bank(struct w83627hf_data *data, u16 reg) +{ + if (reg & 0xff00) { + outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET); + outb_p(0, data->addr + W83781D_DATA_REG_OFFSET); + } +} + +static int w83627hf_read_value(struct w83627hf_data *data, u16 reg) +{ + int res, word_sized; + + mutex_lock(&data->lock); + word_sized = (((reg & 0xff00) == 0x100) + || ((reg & 0xff00) == 0x200)) + && (((reg & 0x00ff) == 0x50) + || ((reg & 0x00ff) == 0x53) + || ((reg & 0x00ff) == 0x55)); + w83627hf_set_bank(data, reg); + outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET); + res = inb_p(data->addr + W83781D_DATA_REG_OFFSET); + if (word_sized) { + outb_p((reg & 0xff) + 1, + data->addr + W83781D_ADDR_REG_OFFSET); + res = + (res << 8) + inb_p(data->addr + + W83781D_DATA_REG_OFFSET); + } + w83627hf_reset_bank(data, reg); + mutex_unlock(&data->lock); + return res; +} + +static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value) +{ + int word_sized; + + mutex_lock(&data->lock); + word_sized = (((reg & 0xff00) == 0x100) + || ((reg & 0xff00) == 0x200)) + && (((reg & 0x00ff) == 0x53) + || ((reg & 0x00ff) == 0x55)); + w83627hf_set_bank(data, reg); + outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET); + if (word_sized) { + outb_p(value >> 8, + data->addr + W83781D_DATA_REG_OFFSET); + outb_p((reg & 0xff) + 1, + data->addr + W83781D_ADDR_REG_OFFSET); + } + outb_p(value & 0xff, + data->addr + W83781D_DATA_REG_OFFSET); + w83627hf_reset_bank(data, reg); + mutex_unlock(&data->lock); + return 0; +} + +static void w83627hf_update_fan_div(struct w83627hf_data *data) +{ + int reg; + + reg = w83627hf_read_value(data, W83781D_REG_VID_FANDIV); + data->fan_div[0] = (reg >> 4) & 0x03; + data->fan_div[1] = (reg >> 6) & 0x03; + if (data->type != w83697hf) { + data->fan_div[2] = (w83627hf_read_value(data, + W83781D_REG_PIN) >> 6) & 0x03; + } + reg = w83627hf_read_value(data, W83781D_REG_VBAT); + data->fan_div[0] |= (reg >> 3) & 0x04; + data->fan_div[1] |= (reg >> 4) & 0x04; + if (data->type != w83697hf) + data->fan_div[2] |= (reg >> 5) & 0x04; +} + +static struct w83627hf_data *w83627hf_update_device(struct device *dev) +{ + struct w83627hf_data *data = dev_get_drvdata(dev); + int i, num_temps = (data->type == w83697hf) ? 2 : 3; + int num_pwms = (data->type == w83697hf) ? 2 : 3; + + mutex_lock(&data->update_lock); + + if (time_after(jiffies, data->last_updated + HZ + HZ / 2) + || !data->valid) { + for (i = 0; i <= 8; i++) { + /* skip missing sensors */ + if (((data->type == w83697hf) && (i == 1)) || + ((data->type != w83627hf && data->type != w83697hf) + && (i == 5 || i == 6))) + continue; + data->in[i] = + w83627hf_read_value(data, W83781D_REG_IN(i)); + data->in_min[i] = + w83627hf_read_value(data, + W83781D_REG_IN_MIN(i)); + data->in_max[i] = + w83627hf_read_value(data, + W83781D_REG_IN_MAX(i)); + } + for (i = 0; i <= 2; i++) { + data->fan[i] = + w83627hf_read_value(data, W83627HF_REG_FAN(i)); + data->fan_min[i] = + w83627hf_read_value(data, + W83627HF_REG_FAN_MIN(i)); + } + for (i = 0; i <= 2; i++) { + u8 tmp = w83627hf_read_value(data, + W836X7HF_REG_PWM(data->type, i)); + /* bits 0-3 are reserved in 627THF */ + if (data->type == w83627thf) + tmp &= 0xf0; + data->pwm[i] = tmp; + if (i == 1 && + (data->type == w83627hf || data->type == w83697hf)) + break; + } + if (data->type == w83627hf) { + u8 tmp = w83627hf_read_value(data, + W83627HF_REG_PWM_FREQ); + data->pwm_freq[0] = tmp & 0x07; + data->pwm_freq[1] = (tmp >> 4) & 0x07; + } else if (data->type != w83627thf) { + for (i = 1; i <= 3; i++) { + data->pwm_freq[i - 1] = + w83627hf_read_value(data, + W83637HF_REG_PWM_FREQ[i - 1]); + if (i == 2 && (data->type == w83697hf)) + break; + } + } + if (data->type != w83627hf) { + for (i = 0; i < num_pwms; i++) { + u8 tmp = w83627hf_read_value(data, + W83627THF_REG_PWM_ENABLE[i]); + data->pwm_enable[i] = + ((tmp >> W83627THF_PWM_ENABLE_SHIFT[i]) + & 0x03) + 1; + } + } + for (i = 0; i < num_temps; i++) { + data->temp[i] = w83627hf_read_value( + data, w83627hf_reg_temp[i]); + data->temp_max[i] = w83627hf_read_value( + data, w83627hf_reg_temp_over[i]); + data->temp_max_hyst[i] = w83627hf_read_value( + data, w83627hf_reg_temp_hyst[i]); + } + + w83627hf_update_fan_div(data); + + data->alarms = + w83627hf_read_value(data, W83781D_REG_ALARM1) | + (w83627hf_read_value(data, W83781D_REG_ALARM2) << 8) | + (w83627hf_read_value(data, W83781D_REG_ALARM3) << 16); + i = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2); + data->beep_mask = (i << 8) | + w83627hf_read_value(data, W83781D_REG_BEEP_INTS1) | + w83627hf_read_value(data, W83781D_REG_BEEP_INTS3) << 16; + data->last_updated = jiffies; + data->valid = true; + } + + mutex_unlock(&data->update_lock); -static int w83627hf_read_value(struct w83627hf_data *data, u16 reg); -static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value); -static void w83627hf_update_fan_div(struct w83627hf_data *data); -static struct w83627hf_data *w83627hf_update_device(struct device *dev); -static void w83627hf_init_device(struct platform_device *pdev); + return data; +} #ifdef CONFIG_PM static int w83627hf_suspend(struct device *dev) @@ -464,99 +634,171 @@ static const struct dev_pm_ops w83627hf_dev_pm_ops = { #define W83627HF_DEV_PM_OPS NULL #endif /* CONFIG_PM */ -static struct platform_driver w83627hf_driver = { - .driver = { - .name = DRVNAME, - .pm = W83627HF_DEV_PM_OPS, - }, - .probe = w83627hf_probe, - .remove = w83627hf_remove, -}; - -static ssize_t -in_input_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in[nr])); -} -static ssize_t -in_min_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_min[nr])); -} -static ssize_t -in_max_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_max[nr])); -} -static ssize_t -in_min_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +static int w83627thf_read_gpio5(struct platform_device *pdev) { - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = dev_get_drvdata(dev); - long val; - int err; - - err = kstrtol(buf, 10, &val); - if (err) - return err; + struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev); + int res = 0xff, sel; - mutex_lock(&data->update_lock); - data->in_min[nr] = IN_TO_REG(val); - w83627hf_write_value(data, W83781D_REG_IN_MIN(nr), data->in_min[nr]); - mutex_unlock(&data->update_lock); - return count; -} -static ssize_t -in_max_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = dev_get_drvdata(dev); - long val; - int err; + if (superio_enter(sio_data)) { + /* + * Some other driver reserved the address space for itself. + * We don't want to fail driver instantiation because of that, + * so display a warning and keep going. + */ + dev_warn(&pdev->dev, + "Can not read VID data: Failed to enable SuperIO access\n"); + return res; + } - err = kstrtol(buf, 10, &val); - if (err) - return err; + superio_select(sio_data, W83627HF_LD_GPIO5); - mutex_lock(&data->update_lock); - data->in_max[nr] = IN_TO_REG(val); - w83627hf_write_value(data, W83781D_REG_IN_MAX(nr), data->in_max[nr]); - mutex_unlock(&data->update_lock); - return count; -} + res = 0xff; -static SENSOR_DEVICE_ATTR_RO(in1_input, in_input, 1); -static SENSOR_DEVICE_ATTR_RW(in1_min, in_min, 1); -static SENSOR_DEVICE_ATTR_RW(in1_max, in_max, 1); -static SENSOR_DEVICE_ATTR_RO(in2_input, in_input, 2); -static SENSOR_DEVICE_ATTR_RW(in2_min, in_min, 2); -static SENSOR_DEVICE_ATTR_RW(in2_max, in_max, 2); -static SENSOR_DEVICE_ATTR_RO(in3_input, in_input, 3); -static SENSOR_DEVICE_ATTR_RW(in3_min, in_min, 3); -static SENSOR_DEVICE_ATTR_RW(in3_max, in_max, 3); -static SENSOR_DEVICE_ATTR_RO(in4_input, in_input, 4); -static SENSOR_DEVICE_ATTR_RW(in4_min, in_min, 4); -static SENSOR_DEVICE_ATTR_RW(in4_max, in_max, 4); -static SENSOR_DEVICE_ATTR_RO(in5_input, in_input, 5); -static SENSOR_DEVICE_ATTR_RW(in5_min, in_min, 5); -static SENSOR_DEVICE_ATTR_RW(in5_max, in_max, 5); -static SENSOR_DEVICE_ATTR_RO(in6_input, in_input, 6); -static SENSOR_DEVICE_ATTR_RW(in6_min, in_min, 6); -static SENSOR_DEVICE_ATTR_RW(in6_max, in_max, 6); -static SENSOR_DEVICE_ATTR_RO(in7_input, in_input, 7); -static SENSOR_DEVICE_ATTR_RW(in7_min, in_min, 7); -static SENSOR_DEVICE_ATTR_RW(in7_max, in_max, 7); -static SENSOR_DEVICE_ATTR_RO(in8_input, in_input, 8); -static SENSOR_DEVICE_ATTR_RW(in8_min, in_min, 8); -static SENSOR_DEVICE_ATTR_RW(in8_max, in_max, 8); + /* Make sure these GPIO pins are enabled */ + if (!(superio_inb(sio_data, W83627THF_GPIO5_EN) & (1<<3))) { + dev_dbg(&pdev->dev, "GPIO5 disabled, no VID function\n"); + goto exit; + } + + /* + * Make sure the pins are configured for input + * There must be at least five (VRM 9), and possibly 6 (VRM 10) + */ + sel = superio_inb(sio_data, W83627THF_GPIO5_IOSR) & 0x3f; + if ((sel & 0x1f) != 0x1f) { + dev_dbg(&pdev->dev, "GPIO5 not configured for VID " + "function\n"); + goto exit; + } + + dev_info(&pdev->dev, "Reading VID from GPIO5\n"); + res = superio_inb(sio_data, W83627THF_GPIO5_DR) & sel; + +exit: + superio_exit(sio_data); + return res; +} + +static int w83687thf_read_vid(struct platform_device *pdev) +{ + struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev); + int res = 0xff; + + if (superio_enter(sio_data)) { + /* + * Some other driver reserved the address space for itself. + * We don't want to fail driver instantiation because of that, + * so display a warning and keep going. + */ + dev_warn(&pdev->dev, + "Can not read VID data: Failed to enable SuperIO access\n"); + return res; + } + + superio_select(sio_data, W83627HF_LD_HWM); + + /* Make sure these GPIO pins are enabled */ + if (!(superio_inb(sio_data, W83687THF_VID_EN) & (1 << 2))) { + dev_dbg(&pdev->dev, "VID disabled, no VID function\n"); + goto exit; + } + + /* Make sure the pins are configured for input */ + if (!(superio_inb(sio_data, W83687THF_VID_CFG) & (1 << 4))) { + dev_dbg(&pdev->dev, "VID configured as output, " + "no VID function\n"); + goto exit; + } + + res = superio_inb(sio_data, W83687THF_VID_DATA) & 0x3f; + +exit: + superio_exit(sio_data); + return res; +} + +static void w83627hf_init_device(struct platform_device *pdev) +{ + struct w83627hf_data *data = platform_get_drvdata(pdev); + int i; + enum chips type = data->type; + u8 tmp; + + /* Minimize conflicts with other winbond i2c-only clients... */ + /* disable i2c subclients... how to disable main i2c client?? */ + /* force i2c address to relatively uncommon address */ + if (type == w83627hf) { + w83627hf_write_value(data, W83781D_REG_I2C_SUBADDR, 0x89); + w83627hf_write_value(data, W83781D_REG_I2C_ADDR, force_i2c); + } + + /* Read VID only once */ + if (type == w83627hf || type == w83637hf) { + int lo = w83627hf_read_value(data, W83781D_REG_VID_FANDIV); + int hi = w83627hf_read_value(data, W83781D_REG_CHIPID); + data->vid = (lo & 0x0f) | ((hi & 0x01) << 4); + } else if (type == w83627thf) { + data->vid = w83627thf_read_gpio5(pdev); + } else if (type == w83687thf) { + data->vid = w83687thf_read_vid(pdev); + } + + /* Read VRM & OVT Config only once */ + if (type == w83627thf || type == w83637hf || type == w83687thf) { + data->vrm_ovt = + w83627hf_read_value(data, W83627THF_REG_VRM_OVT_CFG); + } + + tmp = w83627hf_read_value(data, W83781D_REG_SCFG1); + for (i = 1; i <= 3; i++) { + if (!(tmp & BIT_SCFG1[i - 1])) { + data->sens[i - 1] = 4; + } else { + if (w83627hf_read_value + (data, + W83781D_REG_SCFG2) & BIT_SCFG2[i - 1]) + data->sens[i - 1] = 1; + else + data->sens[i - 1] = 2; + } + if ((type == w83697hf) && (i == 2)) + break; + } + + if(init) { + /* Enable temp2 */ + tmp = w83627hf_read_value(data, W83627HF_REG_TEMP2_CONFIG); + if (tmp & 0x01) { + dev_warn(&pdev->dev, "Enabling temp2, readings " + "might not make sense\n"); + w83627hf_write_value(data, W83627HF_REG_TEMP2_CONFIG, + tmp & 0xfe); + } + + /* Enable temp3 */ + if (type != w83697hf) { + tmp = w83627hf_read_value(data, + W83627HF_REG_TEMP3_CONFIG); + if (tmp & 0x01) { + dev_warn(&pdev->dev, "Enabling temp3, " + "readings might not make sense\n"); + w83627hf_write_value(data, + W83627HF_REG_TEMP3_CONFIG, tmp & 0xfe); + } + } + } + + /* Start monitoring */ + w83627hf_write_value(data, W83781D_REG_CONFIG, + (w83627hf_read_value(data, + W83781D_REG_CONFIG) & 0xf7) + | 0x01); + + /* Enable VBAT monitoring if needed */ + tmp = w83627hf_read_value(data, W83781D_REG_VBAT); + if (!(tmp & 0x01)) + w83627hf_write_value(data, W83781D_REG_VBAT, tmp | 0x01); +} /* use a different set of functions for in0 */ static ssize_t show_in_0(struct w83627hf_data *data, char *buf, u8 reg) @@ -582,6 +824,7 @@ static ssize_t in0_input_show(struct device *dev, struct w83627hf_data *data = w83627hf_update_device(dev); return show_in_0(data, buf, data->in[0]); } +static DEVICE_ATTR_RO(in0_input); static ssize_t in0_min_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -590,13 +833,6 @@ static ssize_t in0_min_show(struct device *dev, struct device_attribute *attr, return show_in_0(data, buf, data->in_min[0]); } -static ssize_t in0_max_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct w83627hf_data *data = w83627hf_update_device(dev); - return show_in_0(data, buf, data->in_max[0]); -} - static ssize_t in0_min_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -627,6 +863,15 @@ static ssize_t in0_min_store(struct device *dev, return count; } +static DEVICE_ATTR_RW(in0_min); + +static ssize_t in0_max_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct w83627hf_data *data = w83627hf_update_device(dev); + return show_in_0(data, buf, data->in_max[0]); +} + static ssize_t in0_max_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -657,97 +902,129 @@ static ssize_t in0_max_store(struct device *dev, return count; } -static DEVICE_ATTR_RO(in0_input); -static DEVICE_ATTR_RW(in0_min); static DEVICE_ATTR_RW(in0_max); static ssize_t -fan_input_show(struct device *dev, struct device_attribute *devattr, - char *buf) +alarm_show(struct device *dev, struct device_attribute *attr, char *buf) { - int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan[nr], - (long)DIV_FROM_REG(data->fan_div[nr]))); + int bitnr = to_sensor_dev_attr(attr)->index; + return sprintf(buf, "%u\n", (data->alarms >> bitnr) & 1); } + +static SENSOR_DEVICE_ATTR_RO(in0_alarm, alarm, 0); +static SENSOR_DEVICE_ATTR_RO(in1_alarm, alarm, 1); +static SENSOR_DEVICE_ATTR_RO(in2_alarm, alarm, 2); +static SENSOR_DEVICE_ATTR_RO(in3_alarm, alarm, 3); +static SENSOR_DEVICE_ATTR_RO(in4_alarm, alarm, 8); +static SENSOR_DEVICE_ATTR_RO(in5_alarm, alarm, 9); +static SENSOR_DEVICE_ATTR_RO(in6_alarm, alarm, 10); +static SENSOR_DEVICE_ATTR_RO(in7_alarm, alarm, 16); +static SENSOR_DEVICE_ATTR_RO(in8_alarm, alarm, 17); +static SENSOR_DEVICE_ATTR_RO(fan1_alarm, alarm, 6); +static SENSOR_DEVICE_ATTR_RO(fan2_alarm, alarm, 7); +static SENSOR_DEVICE_ATTR_RO(fan3_alarm, alarm, 11); +static SENSOR_DEVICE_ATTR_RO(temp1_alarm, alarm, 4); +static SENSOR_DEVICE_ATTR_RO(temp2_alarm, alarm, 5); +static SENSOR_DEVICE_ATTR_RO(temp3_alarm, alarm, 13); + static ssize_t -fan_min_show(struct device *dev, struct device_attribute *devattr, char *buf) +beep_show(struct device *dev, struct device_attribute *attr, char *buf) { - int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan_min[nr], - (long)DIV_FROM_REG(data->fan_div[nr]))); + int bitnr = to_sensor_dev_attr(attr)->index; + return sprintf(buf, "%u\n", (data->beep_mask >> bitnr) & 1); } + static ssize_t -fan_min_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +beep_store(struct device *dev, struct device_attribute *attr, const char *buf, + size_t count) { - int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); - unsigned long val; + int bitnr = to_sensor_dev_attr(attr)->index; + u8 reg; + unsigned long bit; int err; - err = kstrtoul(buf, 10, &val); + err = kstrtoul(buf, 10, &bit); if (err) return err; - mutex_lock(&data->update_lock); - data->fan_min[nr] = FAN_TO_REG(val, DIV_FROM_REG(data->fan_div[nr])); - w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr), - data->fan_min[nr]); - - mutex_unlock(&data->update_lock); - return count; -} - -static SENSOR_DEVICE_ATTR_RO(fan1_input, fan_input, 0); -static SENSOR_DEVICE_ATTR_RW(fan1_min, fan_min, 0); -static SENSOR_DEVICE_ATTR_RO(fan2_input, fan_input, 1); -static SENSOR_DEVICE_ATTR_RW(fan2_min, fan_min, 1); -static SENSOR_DEVICE_ATTR_RO(fan3_input, fan_input, 2); -static SENSOR_DEVICE_ATTR_RW(fan3_min, fan_min, 2); + if (bit & ~1) + return -EINVAL; -static ssize_t -temp_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); + mutex_lock(&data->update_lock); + if (bit) + data->beep_mask |= (1 << bitnr); + else + data->beep_mask &= ~(1 << bitnr); - u16 tmp = data->temp[nr]; - return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) - : (long) TEMP_FROM_REG(tmp)); + if (bitnr < 8) { + reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS1); + if (bit) + reg |= (1 << bitnr); + else + reg &= ~(1 << bitnr); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS1, reg); + } else if (bitnr < 16) { + reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2); + if (bit) + reg |= (1 << (bitnr - 8)); + else + reg &= ~(1 << (bitnr - 8)); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS2, reg); + } else { + reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS3); + if (bit) + reg |= (1 << (bitnr - 16)); + else + reg &= ~(1 << (bitnr - 16)); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS3, reg); + } + mutex_unlock(&data->update_lock); + + return count; } +static SENSOR_DEVICE_ATTR_RW(in0_beep, beep, 0); +static SENSOR_DEVICE_ATTR_RW(in1_beep, beep, 1); +static SENSOR_DEVICE_ATTR_RW(in2_beep, beep, 2); +static SENSOR_DEVICE_ATTR_RW(in3_beep, beep, 3); +static SENSOR_DEVICE_ATTR_RW(in4_beep, beep, 8); +static SENSOR_DEVICE_ATTR_RW(in5_beep, beep, 9); +static SENSOR_DEVICE_ATTR_RW(in6_beep, beep, 10); +static SENSOR_DEVICE_ATTR_RW(in7_beep, beep, 16); +static SENSOR_DEVICE_ATTR_RW(in8_beep, beep, 17); +static SENSOR_DEVICE_ATTR_RW(fan1_beep, beep, 6); +static SENSOR_DEVICE_ATTR_RW(fan2_beep, beep, 7); +static SENSOR_DEVICE_ATTR_RW(fan3_beep, beep, 11); +static SENSOR_DEVICE_ATTR_RW(temp1_beep, beep, 4); +static SENSOR_DEVICE_ATTR_RW(temp2_beep, beep, 5); +static SENSOR_DEVICE_ATTR_RW(temp3_beep, beep, 13); +static SENSOR_DEVICE_ATTR_RW(beep_enable, beep, 15); + static ssize_t -temp_max_show(struct device *dev, struct device_attribute *devattr, char *buf) +in_input_show(struct device *dev, struct device_attribute *devattr, char *buf) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - - u16 tmp = data->temp_max[nr]; - return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) - : (long) TEMP_FROM_REG(tmp)); + return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in[nr])); } static ssize_t -temp_max_hyst_show(struct device *dev, struct device_attribute *devattr, - char *buf) +in_min_show(struct device *dev, struct device_attribute *devattr, char *buf) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - - u16 tmp = data->temp_max_hyst[nr]; - return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) - : (long) TEMP_FROM_REG(tmp)); + return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_min[nr])); } static ssize_t -temp_max_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +in_min_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); - u16 tmp; long val; int err; @@ -755,21 +1032,27 @@ temp_max_store(struct device *dev, struct device_attribute *devattr, if (err) return err; - tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val); mutex_lock(&data->update_lock); - data->temp_max[nr] = tmp; - w83627hf_write_value(data, w83627hf_reg_temp_over[nr], tmp); + data->in_min[nr] = IN_TO_REG(val); + w83627hf_write_value(data, W83781D_REG_IN_MIN(nr), data->in_min[nr]); mutex_unlock(&data->update_lock); return count; } static ssize_t -temp_max_hyst_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +in_max_show(struct device *dev, struct device_attribute *devattr, char *buf) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_max[nr])); +} + +static ssize_t +in_max_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); - u16 tmp; long val; int err; @@ -777,42 +1060,62 @@ temp_max_hyst_store(struct device *dev, struct device_attribute *devattr, if (err) return err; - tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val); mutex_lock(&data->update_lock); - data->temp_max_hyst[nr] = tmp; - w83627hf_write_value(data, w83627hf_reg_temp_hyst[nr], tmp); + data->in_max[nr] = IN_TO_REG(val); + w83627hf_write_value(data, W83781D_REG_IN_MAX(nr), data->in_max[nr]); mutex_unlock(&data->update_lock); return count; } -static SENSOR_DEVICE_ATTR_RO(temp1_input, temp, 0); -static SENSOR_DEVICE_ATTR_RW(temp1_max, temp_max, 0); -static SENSOR_DEVICE_ATTR_RW(temp1_max_hyst, temp_max_hyst, 0); -static SENSOR_DEVICE_ATTR_RO(temp2_input, temp, 1); -static SENSOR_DEVICE_ATTR_RW(temp2_max, temp_max, 1); -static SENSOR_DEVICE_ATTR_RW(temp2_max_hyst, temp_max_hyst, 1); -static SENSOR_DEVICE_ATTR_RO(temp3_input, temp, 2); -static SENSOR_DEVICE_ATTR_RW(temp3_max, temp_max, 2); -static SENSOR_DEVICE_ATTR_RW(temp3_max_hyst, temp_max_hyst, 2); +static SENSOR_DEVICE_ATTR_RO(in1_input, in_input, 1); +static SENSOR_DEVICE_ATTR_RW(in1_min, in_min, 1); +static SENSOR_DEVICE_ATTR_RW(in1_max, in_max, 1); +static SENSOR_DEVICE_ATTR_RO(in2_input, in_input, 2); +static SENSOR_DEVICE_ATTR_RW(in2_min, in_min, 2); +static SENSOR_DEVICE_ATTR_RW(in2_max, in_max, 2); +static SENSOR_DEVICE_ATTR_RO(in3_input, in_input, 3); +static SENSOR_DEVICE_ATTR_RW(in3_min, in_min, 3); +static SENSOR_DEVICE_ATTR_RW(in3_max, in_max, 3); +static SENSOR_DEVICE_ATTR_RO(in4_input, in_input, 4); +static SENSOR_DEVICE_ATTR_RW(in4_min, in_min, 4); +static SENSOR_DEVICE_ATTR_RW(in4_max, in_max, 4); +static SENSOR_DEVICE_ATTR_RO(in5_input, in_input, 5); +static SENSOR_DEVICE_ATTR_RW(in5_min, in_min, 5); +static SENSOR_DEVICE_ATTR_RW(in5_max, in_max, 5); +static SENSOR_DEVICE_ATTR_RO(in6_input, in_input, 6); +static SENSOR_DEVICE_ATTR_RW(in6_min, in_min, 6); +static SENSOR_DEVICE_ATTR_RW(in6_max, in_max, 6); +static SENSOR_DEVICE_ATTR_RO(in7_input, in_input, 7); +static SENSOR_DEVICE_ATTR_RW(in7_min, in_min, 7); +static SENSOR_DEVICE_ATTR_RW(in7_max, in_max, 7); +static SENSOR_DEVICE_ATTR_RO(in8_input, in_input, 8); +static SENSOR_DEVICE_ATTR_RW(in8_min, in_min, 8); +static SENSOR_DEVICE_ATTR_RW(in8_max, in_max, 8); static ssize_t -cpu0_vid_show(struct device *dev, struct device_attribute *attr, char *buf) +fan_input_show(struct device *dev, struct device_attribute *devattr, + char *buf) { + int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long) vid_from_reg(data->vid, data->vrm)); + return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan[nr], + (long)DIV_FROM_REG(data->fan_div[nr]))); } -static DEVICE_ATTR_RO(cpu0_vid); static ssize_t -vrm_show(struct device *dev, struct device_attribute *attr, char *buf) +fan_min_show(struct device *dev, struct device_attribute *devattr, char *buf) { - struct w83627hf_data *data = dev_get_drvdata(dev); - return sprintf(buf, "%ld\n", (long) data->vrm); + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan_min[nr], + (long)DIV_FROM_REG(data->fan_div[nr]))); } + static ssize_t -vrm_store(struct device *dev, struct device_attribute *attr, const char *buf, - size_t count) +fan_min_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { + int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); unsigned long val; int err; @@ -821,58 +1124,45 @@ vrm_store(struct device *dev, struct device_attribute *attr, const char *buf, if (err) return err; - if (val > 255) - return -EINVAL; - data->vrm = val; + mutex_lock(&data->update_lock); + data->fan_min[nr] = FAN_TO_REG(val, DIV_FROM_REG(data->fan_div[nr])); + w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr), + data->fan_min[nr]); + mutex_unlock(&data->update_lock); return count; } -static DEVICE_ATTR_RW(vrm); - -static ssize_t -alarms_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long) data->alarms); -} -static DEVICE_ATTR_RO(alarms); -static ssize_t -alarm_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct w83627hf_data *data = w83627hf_update_device(dev); - int bitnr = to_sensor_dev_attr(attr)->index; - return sprintf(buf, "%u\n", (data->alarms >> bitnr) & 1); -} -static SENSOR_DEVICE_ATTR_RO(in0_alarm, alarm, 0); -static SENSOR_DEVICE_ATTR_RO(in1_alarm, alarm, 1); -static SENSOR_DEVICE_ATTR_RO(in2_alarm, alarm, 2); -static SENSOR_DEVICE_ATTR_RO(in3_alarm, alarm, 3); -static SENSOR_DEVICE_ATTR_RO(in4_alarm, alarm, 8); -static SENSOR_DEVICE_ATTR_RO(in5_alarm, alarm, 9); -static SENSOR_DEVICE_ATTR_RO(in6_alarm, alarm, 10); -static SENSOR_DEVICE_ATTR_RO(in7_alarm, alarm, 16); -static SENSOR_DEVICE_ATTR_RO(in8_alarm, alarm, 17); -static SENSOR_DEVICE_ATTR_RO(fan1_alarm, alarm, 6); -static SENSOR_DEVICE_ATTR_RO(fan2_alarm, alarm, 7); -static SENSOR_DEVICE_ATTR_RO(fan3_alarm, alarm, 11); -static SENSOR_DEVICE_ATTR_RO(temp1_alarm, alarm, 4); -static SENSOR_DEVICE_ATTR_RO(temp2_alarm, alarm, 5); -static SENSOR_DEVICE_ATTR_RO(temp3_alarm, alarm, 13); +static SENSOR_DEVICE_ATTR_RO(fan1_input, fan_input, 0); +static SENSOR_DEVICE_ATTR_RW(fan1_min, fan_min, 0); +static SENSOR_DEVICE_ATTR_RO(fan2_input, fan_input, 1); +static SENSOR_DEVICE_ATTR_RW(fan2_min, fan_min, 1); +static SENSOR_DEVICE_ATTR_RO(fan3_input, fan_input, 2); +static SENSOR_DEVICE_ATTR_RW(fan3_min, fan_min, 2); static ssize_t -beep_mask_show(struct device *dev, struct device_attribute *attr, char *buf) +fan_div_show(struct device *dev, struct device_attribute *devattr, char *buf) { + int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); return sprintf(buf, "%ld\n", - (long)BEEP_MASK_FROM_REG(data->beep_mask)); + (long) DIV_FROM_REG(data->fan_div[nr])); } +/* + * Note: we save and restore the fan minimum here, because its value is + * determined in part by the fan divisor. This follows the principle of + * least surprise; the user doesn't expect the fan minimum to change just + * because the divisor changed. + */ static ssize_t -beep_mask_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +fan_div_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { + int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); + unsigned long min; + u8 reg; unsigned long val; int err; @@ -882,289 +1172,121 @@ beep_mask_store(struct device *dev, struct device_attribute *attr, mutex_lock(&data->update_lock); - /* preserve beep enable */ - data->beep_mask = (data->beep_mask & 0x8000) - | BEEP_MASK_TO_REG(val); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS1, - data->beep_mask & 0xff); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS3, - ((data->beep_mask) >> 16) & 0xff); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS2, - (data->beep_mask >> 8) & 0xff); + /* Save fan_min */ + min = FAN_FROM_REG(data->fan_min[nr], + DIV_FROM_REG(data->fan_div[nr])); + + data->fan_div[nr] = DIV_TO_REG(val); + + reg = (w83627hf_read_value(data, nr==2 ? W83781D_REG_PIN : W83781D_REG_VID_FANDIV) + & (nr==0 ? 0xcf : 0x3f)) + | ((data->fan_div[nr] & 0x03) << (nr==0 ? 4 : 6)); + w83627hf_write_value(data, nr==2 ? W83781D_REG_PIN : W83781D_REG_VID_FANDIV, reg); + + reg = (w83627hf_read_value(data, W83781D_REG_VBAT) + & ~(1 << (5 + nr))) + | ((data->fan_div[nr] & 0x04) << (3 + nr)); + w83627hf_write_value(data, W83781D_REG_VBAT, reg); + + /* Restore fan_min */ + data->fan_min[nr] = FAN_TO_REG(min, DIV_FROM_REG(data->fan_div[nr])); + w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr), data->fan_min[nr]); mutex_unlock(&data->update_lock); return count; } -static DEVICE_ATTR_RW(beep_mask); +static SENSOR_DEVICE_ATTR_RW(fan1_div, fan_div, 0); +static SENSOR_DEVICE_ATTR_RW(fan2_div, fan_div, 1); +static SENSOR_DEVICE_ATTR_RW(fan3_div, fan_div, 2); static ssize_t -beep_show(struct device *dev, struct device_attribute *attr, char *buf) +temp_show(struct device *dev, struct device_attribute *devattr, char *buf) { + int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - int bitnr = to_sensor_dev_attr(attr)->index; - return sprintf(buf, "%u\n", (data->beep_mask >> bitnr) & 1); + + u16 tmp = data->temp[nr]; + return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) + : (long) TEMP_FROM_REG(tmp)); } static ssize_t -beep_store(struct device *dev, struct device_attribute *attr, const char *buf, - size_t count) -{ - struct w83627hf_data *data = dev_get_drvdata(dev); - int bitnr = to_sensor_dev_attr(attr)->index; - u8 reg; - unsigned long bit; - int err; - - err = kstrtoul(buf, 10, &bit); - if (err) - return err; - - if (bit & ~1) - return -EINVAL; - - mutex_lock(&data->update_lock); - if (bit) - data->beep_mask |= (1 << bitnr); - else - data->beep_mask &= ~(1 << bitnr); - - if (bitnr < 8) { - reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS1); - if (bit) - reg |= (1 << bitnr); - else - reg &= ~(1 << bitnr); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS1, reg); - } else if (bitnr < 16) { - reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2); - if (bit) - reg |= (1 << (bitnr - 8)); - else - reg &= ~(1 << (bitnr - 8)); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS2, reg); - } else { - reg = w83627hf_read_value(data, W83781D_REG_BEEP_INTS3); - if (bit) - reg |= (1 << (bitnr - 16)); - else - reg &= ~(1 << (bitnr - 16)); - w83627hf_write_value(data, W83781D_REG_BEEP_INTS3, reg); - } - mutex_unlock(&data->update_lock); - - return count; -} - -static SENSOR_DEVICE_ATTR_RW(in0_beep, beep, 0); -static SENSOR_DEVICE_ATTR_RW(in1_beep, beep, 1); -static SENSOR_DEVICE_ATTR_RW(in2_beep, beep, 2); -static SENSOR_DEVICE_ATTR_RW(in3_beep, beep, 3); -static SENSOR_DEVICE_ATTR_RW(in4_beep, beep, 8); -static SENSOR_DEVICE_ATTR_RW(in5_beep, beep, 9); -static SENSOR_DEVICE_ATTR_RW(in6_beep, beep, 10); -static SENSOR_DEVICE_ATTR_RW(in7_beep, beep, 16); -static SENSOR_DEVICE_ATTR_RW(in8_beep, beep, 17); -static SENSOR_DEVICE_ATTR_RW(fan1_beep, beep, 6); -static SENSOR_DEVICE_ATTR_RW(fan2_beep, beep, 7); -static SENSOR_DEVICE_ATTR_RW(fan3_beep, beep, 11); -static SENSOR_DEVICE_ATTR_RW(temp1_beep, beep, 4); -static SENSOR_DEVICE_ATTR_RW(temp2_beep, beep, 5); -static SENSOR_DEVICE_ATTR_RW(temp3_beep, beep, 13); -static SENSOR_DEVICE_ATTR_RW(beep_enable, beep, 15); - -static ssize_t -fan_div_show(struct device *dev, struct device_attribute *devattr, char *buf) +temp_max_show(struct device *dev, struct device_attribute *devattr, char *buf) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", - (long) DIV_FROM_REG(data->fan_div[nr])); -} -/* - * Note: we save and restore the fan minimum here, because its value is - * determined in part by the fan divisor. This follows the principle of - * least surprise; the user doesn't expect the fan minimum to change just - * because the divisor changed. - */ -static ssize_t -fan_div_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = dev_get_drvdata(dev); - unsigned long min; - u8 reg; - unsigned long val; - int err; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - mutex_lock(&data->update_lock); - - /* Save fan_min */ - min = FAN_FROM_REG(data->fan_min[nr], - DIV_FROM_REG(data->fan_div[nr])); - - data->fan_div[nr] = DIV_TO_REG(val); - - reg = (w83627hf_read_value(data, nr==2 ? W83781D_REG_PIN : W83781D_REG_VID_FANDIV) - & (nr==0 ? 0xcf : 0x3f)) - | ((data->fan_div[nr] & 0x03) << (nr==0 ? 4 : 6)); - w83627hf_write_value(data, nr==2 ? W83781D_REG_PIN : W83781D_REG_VID_FANDIV, reg); - reg = (w83627hf_read_value(data, W83781D_REG_VBAT) - & ~(1 << (5 + nr))) - | ((data->fan_div[nr] & 0x04) << (3 + nr)); - w83627hf_write_value(data, W83781D_REG_VBAT, reg); - - /* Restore fan_min */ - data->fan_min[nr] = FAN_TO_REG(min, DIV_FROM_REG(data->fan_div[nr])); - w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr), data->fan_min[nr]); - - mutex_unlock(&data->update_lock); - return count; -} - -static SENSOR_DEVICE_ATTR_RW(fan1_div, fan_div, 0); -static SENSOR_DEVICE_ATTR_RW(fan2_div, fan_div, 1); -static SENSOR_DEVICE_ATTR_RW(fan3_div, fan_div, 2); - -static ssize_t -pwm_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%ld\n", (long) data->pwm[nr]); + u16 tmp = data->temp_max[nr]; + return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) + : (long) TEMP_FROM_REG(tmp)); } static ssize_t -pwm_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +temp_max_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); - unsigned long val; + u16 tmp; + long val; int err; - err = kstrtoul(buf, 10, &val); + err = kstrtol(buf, 10, &val); if (err) return err; + tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val); mutex_lock(&data->update_lock); - - if (data->type == w83627thf) { - /* bits 0-3 are reserved in 627THF */ - data->pwm[nr] = PWM_TO_REG(val) & 0xf0; - w83627hf_write_value(data, - W836X7HF_REG_PWM(data->type, nr), - data->pwm[nr] | - (w83627hf_read_value(data, - W836X7HF_REG_PWM(data->type, nr)) & 0x0f)); - } else { - data->pwm[nr] = PWM_TO_REG(val); - w83627hf_write_value(data, - W836X7HF_REG_PWM(data->type, nr), - data->pwm[nr]); - } - + data->temp_max[nr] = tmp; + w83627hf_write_value(data, w83627hf_reg_temp_over[nr], tmp); mutex_unlock(&data->update_lock); return count; } -static SENSOR_DEVICE_ATTR_RW(pwm1, pwm, 0); -static SENSOR_DEVICE_ATTR_RW(pwm2, pwm, 1); -static SENSOR_DEVICE_ATTR_RW(pwm3, pwm, 2); - static ssize_t -pwm_enable_show(struct device *dev, struct device_attribute *devattr, - char *buf) +temp_max_hyst_show(struct device *dev, struct device_attribute *devattr, + char *buf) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = w83627hf_update_device(dev); - return sprintf(buf, "%d\n", data->pwm_enable[nr]); -} -static ssize_t -pwm_enable_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = dev_get_drvdata(dev); - u8 reg; - unsigned long val; - int err; - - err = kstrtoul(buf, 10, &val); - if (err) - return err; - - if (!val || val > 3) /* modes 1, 2 and 3 are supported */ - return -EINVAL; - mutex_lock(&data->update_lock); - data->pwm_enable[nr] = val; - reg = w83627hf_read_value(data, W83627THF_REG_PWM_ENABLE[nr]); - reg &= ~(0x03 << W83627THF_PWM_ENABLE_SHIFT[nr]); - reg |= (val - 1) << W83627THF_PWM_ENABLE_SHIFT[nr]; - w83627hf_write_value(data, W83627THF_REG_PWM_ENABLE[nr], reg); - mutex_unlock(&data->update_lock); - return count; -} - -static SENSOR_DEVICE_ATTR_RW(pwm1_enable, pwm_enable, 0); -static SENSOR_DEVICE_ATTR_RW(pwm2_enable, pwm_enable, 1); -static SENSOR_DEVICE_ATTR_RW(pwm3_enable, pwm_enable, 2); - -static ssize_t -pwm_freq_show(struct device *dev, struct device_attribute *devattr, char *buf) -{ - int nr = to_sensor_dev_attr(devattr)->index; - struct w83627hf_data *data = w83627hf_update_device(dev); - if (data->type == w83627hf) - return sprintf(buf, "%ld\n", - pwm_freq_from_reg_627hf(data->pwm_freq[nr])); - else - return sprintf(buf, "%ld\n", - pwm_freq_from_reg(data->pwm_freq[nr])); + u16 tmp = data->temp_max_hyst[nr]; + return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp) + : (long) TEMP_FROM_REG(tmp)); } static ssize_t -pwm_freq_store(struct device *dev, struct device_attribute *devattr, - const char *buf, size_t count) +temp_max_hyst_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) { int nr = to_sensor_dev_attr(devattr)->index; struct w83627hf_data *data = dev_get_drvdata(dev); - static const u8 mask[]={0xF8, 0x8F}; - unsigned long val; + u16 tmp; + long val; int err; - err = kstrtoul(buf, 10, &val); + err = kstrtol(buf, 10, &val); if (err) return err; + tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val); mutex_lock(&data->update_lock); - - if (data->type == w83627hf) { - data->pwm_freq[nr] = pwm_freq_to_reg_627hf(val); - w83627hf_write_value(data, W83627HF_REG_PWM_FREQ, - (data->pwm_freq[nr] << (nr*4)) | - (w83627hf_read_value(data, - W83627HF_REG_PWM_FREQ) & mask[nr])); - } else { - data->pwm_freq[nr] = pwm_freq_to_reg(val); - w83627hf_write_value(data, W83637HF_REG_PWM_FREQ[nr], - data->pwm_freq[nr]); - } - + data->temp_max_hyst[nr] = tmp; + w83627hf_write_value(data, w83627hf_reg_temp_hyst[nr], tmp); mutex_unlock(&data->update_lock); return count; } -static SENSOR_DEVICE_ATTR_RW(pwm1_freq, pwm_freq, 0); -static SENSOR_DEVICE_ATTR_RW(pwm2_freq, pwm_freq, 1); -static SENSOR_DEVICE_ATTR_RW(pwm3_freq, pwm_freq, 2); +static SENSOR_DEVICE_ATTR_RO(temp1_input, temp, 0); +static SENSOR_DEVICE_ATTR_RW(temp1_max, temp_max, 0); +static SENSOR_DEVICE_ATTR_RW(temp1_max_hyst, temp_max_hyst, 0); +static SENSOR_DEVICE_ATTR_RO(temp2_input, temp, 1); +static SENSOR_DEVICE_ATTR_RW(temp2_max, temp_max, 1); +static SENSOR_DEVICE_ATTR_RW(temp2_max_hyst, temp_max_hyst, 1); +static SENSOR_DEVICE_ATTR_RO(temp3_input, temp, 2); +static SENSOR_DEVICE_ATTR_RW(temp3_max, temp_max, 2); +static SENSOR_DEVICE_ATTR_RW(temp3_max_hyst, temp_max_hyst, 2); static ssize_t temp_type_show(struct device *dev, struct device_attribute *devattr, @@ -1236,81 +1358,12 @@ static SENSOR_DEVICE_ATTR_RW(temp2_type, temp_type, 1); static SENSOR_DEVICE_ATTR_RW(temp3_type, temp_type, 2); static ssize_t -name_show(struct device *dev, struct device_attribute *devattr, char *buf) +alarms_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct w83627hf_data *data = dev_get_drvdata(dev); - - return sprintf(buf, "%s\n", data->name); -} -static DEVICE_ATTR_RO(name); - -static int __init w83627hf_find(int sioaddr, unsigned short *addr, - struct w83627hf_sio_data *sio_data) -{ - int err; - u16 val; - - static __initconst char *const names[] = { - "W83627HF", - "W83627THF", - "W83697HF", - "W83637HF", - "W83687THF", - }; - - sio_data->sioaddr = sioaddr; - err = superio_enter(sio_data); - if (err) - return err; - - err = -ENODEV; - val = force_id ? force_id : superio_inb(sio_data, DEVID); - switch (val) { - case W627_DEVID: - sio_data->type = w83627hf; - break; - case W627THF_DEVID: - sio_data->type = w83627thf; - break; - case W697_DEVID: - sio_data->type = w83697hf; - break; - case W637_DEVID: - sio_data->type = w83637hf; - break; - case W687THF_DEVID: - sio_data->type = w83687thf; - break; - case 0xff: /* No device at all */ - goto exit; - default: - pr_debug(DRVNAME ": Unsupported chip (DEVID=0x%02x)\n", val); - goto exit; - } - - superio_select(sio_data, W83627HF_LD_HWM); - val = (superio_inb(sio_data, WINB_BASE_REG) << 8) | - superio_inb(sio_data, WINB_BASE_REG + 1); - *addr = val & WINB_ALIGNMENT; - if (*addr == 0) { - pr_warn("Base address not set, skipping\n"); - goto exit; - } - - val = superio_inb(sio_data, WINB_ACT_REG); - if (!(val & 0x01)) { - pr_warn("Enabling HWM logical device\n"); - superio_outb(sio_data, WINB_ACT_REG, val | 0x01); - } - - err = 0; - pr_info(DRVNAME ": Found %s chip at %#x\n", - names[sio_data->type], *addr); - - exit: - superio_exit(sio_data); - return err; + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", (long) data->alarms); } +static DEVICE_ATTR_RO(alarms); #define VIN_UNIT_ATTRS(_X_) \ &sensor_dev_attr_in##_X_##_input.dev_attr.attr, \ @@ -1334,6 +1387,100 @@ static int __init w83627hf_find(int sioaddr, unsigned short *addr, &sensor_dev_attr_temp##_X_##_alarm.dev_attr.attr, \ &sensor_dev_attr_temp##_X_##_beep.dev_attr.attr +static ssize_t +beep_mask_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", + (long)BEEP_MASK_FROM_REG(data->beep_mask)); +} + +static ssize_t +beep_mask_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct w83627hf_data *data = dev_get_drvdata(dev); + unsigned long val; + int err; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + mutex_lock(&data->update_lock); + + /* preserve beep enable */ + data->beep_mask = (data->beep_mask & 0x8000) + | BEEP_MASK_TO_REG(val); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS1, + data->beep_mask & 0xff); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS3, + ((data->beep_mask) >> 16) & 0xff); + w83627hf_write_value(data, W83781D_REG_BEEP_INTS2, + (data->beep_mask >> 8) & 0xff); + + mutex_unlock(&data->update_lock); + return count; +} + +static DEVICE_ATTR_RW(beep_mask); + +static ssize_t +pwm_show(struct device *dev, struct device_attribute *devattr, char *buf) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", (long) data->pwm[nr]); +} + +static ssize_t +pwm_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = dev_get_drvdata(dev); + unsigned long val; + int err; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + mutex_lock(&data->update_lock); + + if (data->type == w83627thf) { + /* bits 0-3 are reserved in 627THF */ + data->pwm[nr] = PWM_TO_REG(val) & 0xf0; + w83627hf_write_value(data, + W836X7HF_REG_PWM(data->type, nr), + data->pwm[nr] | + (w83627hf_read_value(data, + W836X7HF_REG_PWM(data->type, nr)) & 0x0f)); + } else { + data->pwm[nr] = PWM_TO_REG(val); + w83627hf_write_value(data, + W836X7HF_REG_PWM(data->type, nr), + data->pwm[nr]); + } + + mutex_unlock(&data->update_lock); + return count; +} + +static SENSOR_DEVICE_ATTR_RW(pwm1, pwm, 0); +static SENSOR_DEVICE_ATTR_RW(pwm2, pwm, 1); +static SENSOR_DEVICE_ATTR_RW(pwm3, pwm, 2); + +static ssize_t +name_show(struct device *dev, struct device_attribute *devattr, char *buf) +{ + struct w83627hf_data *data = dev_get_drvdata(dev); + + return sprintf(buf, "%s\n", data->name); +} + +static DEVICE_ATTR_RO(name); + static struct attribute *w83627hf_attributes[] = { &dev_attr_in0_input.attr, &dev_attr_in0_min.attr, @@ -1366,60 +1513,185 @@ static const struct attribute_group w83627hf_group = { .attrs = w83627hf_attributes, }; -static struct attribute *w83627hf_attributes_opt[] = { - VIN_UNIT_ATTRS(1), - VIN_UNIT_ATTRS(5), - VIN_UNIT_ATTRS(6), +static ssize_t +pwm_freq_show(struct device *dev, struct device_attribute *devattr, char *buf) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = w83627hf_update_device(dev); + if (data->type == w83627hf) + return sprintf(buf, "%ld\n", + pwm_freq_from_reg_627hf(data->pwm_freq[nr])); + else + return sprintf(buf, "%ld\n", + pwm_freq_from_reg(data->pwm_freq[nr])); +} - FAN_UNIT_ATTRS(3), - TEMP_UNIT_ATTRS(3), - &sensor_dev_attr_pwm3.dev_attr.attr, +static ssize_t +pwm_freq_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = dev_get_drvdata(dev); + static const u8 mask[]={0xF8, 0x8F}; + unsigned long val; + int err; - &sensor_dev_attr_pwm1_freq.dev_attr.attr, - &sensor_dev_attr_pwm2_freq.dev_attr.attr, - &sensor_dev_attr_pwm3_freq.dev_attr.attr, + err = kstrtoul(buf, 10, &val); + if (err) + return err; - &sensor_dev_attr_pwm1_enable.dev_attr.attr, - &sensor_dev_attr_pwm2_enable.dev_attr.attr, - &sensor_dev_attr_pwm3_enable.dev_attr.attr, + mutex_lock(&data->update_lock); - NULL -}; + if (data->type == w83627hf) { + data->pwm_freq[nr] = pwm_freq_to_reg_627hf(val); + w83627hf_write_value(data, W83627HF_REG_PWM_FREQ, + (data->pwm_freq[nr] << (nr*4)) | + (w83627hf_read_value(data, + W83627HF_REG_PWM_FREQ) & mask[nr])); + } else { + data->pwm_freq[nr] = pwm_freq_to_reg(val); + w83627hf_write_value(data, W83637HF_REG_PWM_FREQ[nr], + data->pwm_freq[nr]); + } -static const struct attribute_group w83627hf_group_opt = { - .attrs = w83627hf_attributes_opt, -}; + mutex_unlock(&data->update_lock); + return count; +} -static int w83627hf_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct w83627hf_sio_data *sio_data = dev_get_platdata(dev); - struct w83627hf_data *data; - struct resource *res; - int err, i; +static SENSOR_DEVICE_ATTR_RW(pwm1_freq, pwm_freq, 0); +static SENSOR_DEVICE_ATTR_RW(pwm2_freq, pwm_freq, 1); +static SENSOR_DEVICE_ATTR_RW(pwm3_freq, pwm_freq, 2); - static const char *names[] = { - "w83627hf", - "w83627thf", - "w83697hf", - "w83637hf", - "w83687thf", - }; +static ssize_t +cpu0_vid_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%ld\n", (long) vid_from_reg(data->vid, data->vrm)); +} - res = platform_get_resource(pdev, IORESOURCE_IO, 0); - if (!devm_request_region(dev, res->start, WINB_REGION_SIZE, DRVNAME)) { - dev_err(dev, "Failed to request region 0x%lx-0x%lx\n", - (unsigned long)res->start, - (unsigned long)(res->start + WINB_REGION_SIZE - 1)); - return -EBUSY; - } +static DEVICE_ATTR_RO(cpu0_vid); - data = devm_kzalloc(dev, sizeof(struct w83627hf_data), GFP_KERNEL); - if (!data) - return -ENOMEM; +static ssize_t +vrm_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct w83627hf_data *data = dev_get_drvdata(dev); + return sprintf(buf, "%ld\n", (long) data->vrm); +} - data->addr = res->start; - data->type = sio_data->type; +static ssize_t +vrm_store(struct device *dev, struct device_attribute *attr, const char *buf, + size_t count) +{ + struct w83627hf_data *data = dev_get_drvdata(dev); + unsigned long val; + int err; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + if (val > 255) + return -EINVAL; + data->vrm = val; + + return count; +} + +static DEVICE_ATTR_RW(vrm); + +static ssize_t +pwm_enable_show(struct device *dev, struct device_attribute *devattr, + char *buf) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = w83627hf_update_device(dev); + return sprintf(buf, "%d\n", data->pwm_enable[nr]); +} + +static ssize_t +pwm_enable_store(struct device *dev, struct device_attribute *devattr, + const char *buf, size_t count) +{ + int nr = to_sensor_dev_attr(devattr)->index; + struct w83627hf_data *data = dev_get_drvdata(dev); + u8 reg; + unsigned long val; + int err; + + err = kstrtoul(buf, 10, &val); + if (err) + return err; + + if (!val || val > 3) /* modes 1, 2 and 3 are supported */ + return -EINVAL; + mutex_lock(&data->update_lock); + data->pwm_enable[nr] = val; + reg = w83627hf_read_value(data, W83627THF_REG_PWM_ENABLE[nr]); + reg &= ~(0x03 << W83627THF_PWM_ENABLE_SHIFT[nr]); + reg |= (val - 1) << W83627THF_PWM_ENABLE_SHIFT[nr]; + w83627hf_write_value(data, W83627THF_REG_PWM_ENABLE[nr], reg); + mutex_unlock(&data->update_lock); + return count; +} + +static SENSOR_DEVICE_ATTR_RW(pwm1_enable, pwm_enable, 0); +static SENSOR_DEVICE_ATTR_RW(pwm2_enable, pwm_enable, 1); +static SENSOR_DEVICE_ATTR_RW(pwm3_enable, pwm_enable, 2); + +static struct attribute *w83627hf_attributes_opt[] = { + VIN_UNIT_ATTRS(1), + VIN_UNIT_ATTRS(5), + VIN_UNIT_ATTRS(6), + + FAN_UNIT_ATTRS(3), + TEMP_UNIT_ATTRS(3), + &sensor_dev_attr_pwm3.dev_attr.attr, + + &sensor_dev_attr_pwm1_freq.dev_attr.attr, + &sensor_dev_attr_pwm2_freq.dev_attr.attr, + &sensor_dev_attr_pwm3_freq.dev_attr.attr, + + &sensor_dev_attr_pwm1_enable.dev_attr.attr, + &sensor_dev_attr_pwm2_enable.dev_attr.attr, + &sensor_dev_attr_pwm3_enable.dev_attr.attr, + + NULL +}; + +static const struct attribute_group w83627hf_group_opt = { + .attrs = w83627hf_attributes_opt, +}; + +static int w83627hf_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct w83627hf_sio_data *sio_data = dev_get_platdata(dev); + struct w83627hf_data *data; + struct resource *res; + int err, i; + + static const char *names[] = { + "w83627hf", + "w83627thf", + "w83697hf", + "w83637hf", + "w83687thf", + }; + + res = platform_get_resource(pdev, IORESOURCE_IO, 0); + if (!devm_request_region(dev, res->start, WINB_REGION_SIZE, DRVNAME)) { + dev_err(dev, "Failed to request region 0x%lx-0x%lx\n", + (unsigned long)res->start, + (unsigned long)(res->start + WINB_REGION_SIZE - 1)); + return -EBUSY; + } + + data = devm_kzalloc(dev, sizeof(struct w83627hf_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->addr = res->start; + data->type = sio_data->type; data->name = names[sio_data->type]; mutex_init(&data->lock); mutex_init(&data->update_lock); @@ -1568,349 +1840,81 @@ static int w83627hf_remove(struct platform_device *pdev) return 0; } -/* Registers 0x50-0x5f are banked */ -static inline void w83627hf_set_bank(struct w83627hf_data *data, u16 reg) -{ - if ((reg & 0x00f0) == 0x50) { - outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET); - outb_p(reg >> 8, data->addr + W83781D_DATA_REG_OFFSET); - } -} - -/* Not strictly necessary, but play it safe for now */ -static inline void w83627hf_reset_bank(struct w83627hf_data *data, u16 reg) -{ - if (reg & 0xff00) { - outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET); - outb_p(0, data->addr + W83781D_DATA_REG_OFFSET); - } -} - -static int w83627hf_read_value(struct w83627hf_data *data, u16 reg) -{ - int res, word_sized; - - mutex_lock(&data->lock); - word_sized = (((reg & 0xff00) == 0x100) - || ((reg & 0xff00) == 0x200)) - && (((reg & 0x00ff) == 0x50) - || ((reg & 0x00ff) == 0x53) - || ((reg & 0x00ff) == 0x55)); - w83627hf_set_bank(data, reg); - outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET); - res = inb_p(data->addr + W83781D_DATA_REG_OFFSET); - if (word_sized) { - outb_p((reg & 0xff) + 1, - data->addr + W83781D_ADDR_REG_OFFSET); - res = - (res << 8) + inb_p(data->addr + - W83781D_DATA_REG_OFFSET); - } - w83627hf_reset_bank(data, reg); - mutex_unlock(&data->lock); - return res; -} +static struct platform_driver w83627hf_driver = { + .driver = { + .name = DRVNAME, + .pm = W83627HF_DEV_PM_OPS, + }, + .probe = w83627hf_probe, + .remove = w83627hf_remove, +}; -static int w83627thf_read_gpio5(struct platform_device *pdev) +static int __init w83627hf_find(int sioaddr, unsigned short *addr, + struct w83627hf_sio_data *sio_data) { - struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev); - int res = 0xff, sel; - - if (superio_enter(sio_data)) { - /* - * Some other driver reserved the address space for itself. - * We don't want to fail driver instantiation because of that, - * so display a warning and keep going. - */ - dev_warn(&pdev->dev, - "Can not read VID data: Failed to enable SuperIO access\n"); - return res; - } + int err; + u16 val; - superio_select(sio_data, W83627HF_LD_GPIO5); + static __initconst char *const names[] = { + "W83627HF", + "W83627THF", + "W83697HF", + "W83637HF", + "W83687THF", + }; - res = 0xff; + sio_data->sioaddr = sioaddr; + err = superio_enter(sio_data); + if (err) + return err; - /* Make sure these GPIO pins are enabled */ - if (!(superio_inb(sio_data, W83627THF_GPIO5_EN) & (1<<3))) { - dev_dbg(&pdev->dev, "GPIO5 disabled, no VID function\n"); + err = -ENODEV; + val = force_id ? force_id : superio_inb(sio_data, DEVID); + switch (val) { + case W627_DEVID: + sio_data->type = w83627hf; + break; + case W627THF_DEVID: + sio_data->type = w83627thf; + break; + case W697_DEVID: + sio_data->type = w83697hf; + break; + case W637_DEVID: + sio_data->type = w83637hf; + break; + case W687THF_DEVID: + sio_data->type = w83687thf; + break; + case 0xff: /* No device at all */ goto exit; - } - - /* - * Make sure the pins are configured for input - * There must be at least five (VRM 9), and possibly 6 (VRM 10) - */ - sel = superio_inb(sio_data, W83627THF_GPIO5_IOSR) & 0x3f; - if ((sel & 0x1f) != 0x1f) { - dev_dbg(&pdev->dev, "GPIO5 not configured for VID " - "function\n"); + default: + pr_debug(DRVNAME ": Unsupported chip (DEVID=0x%02x)\n", val); goto exit; } - dev_info(&pdev->dev, "Reading VID from GPIO5\n"); - res = superio_inb(sio_data, W83627THF_GPIO5_DR) & sel; - -exit: - superio_exit(sio_data); - return res; -} - -static int w83687thf_read_vid(struct platform_device *pdev) -{ - struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev); - int res = 0xff; - - if (superio_enter(sio_data)) { - /* - * Some other driver reserved the address space for itself. - * We don't want to fail driver instantiation because of that, - * so display a warning and keep going. - */ - dev_warn(&pdev->dev, - "Can not read VID data: Failed to enable SuperIO access\n"); - return res; - } - superio_select(sio_data, W83627HF_LD_HWM); - - /* Make sure these GPIO pins are enabled */ - if (!(superio_inb(sio_data, W83687THF_VID_EN) & (1 << 2))) { - dev_dbg(&pdev->dev, "VID disabled, no VID function\n"); + val = (superio_inb(sio_data, WINB_BASE_REG) << 8) | + superio_inb(sio_data, WINB_BASE_REG + 1); + *addr = val & WINB_ALIGNMENT; + if (*addr == 0) { + pr_warn("Base address not set, skipping\n"); goto exit; } - /* Make sure the pins are configured for input */ - if (!(superio_inb(sio_data, W83687THF_VID_CFG) & (1 << 4))) { - dev_dbg(&pdev->dev, "VID configured as output, " - "no VID function\n"); - goto exit; + val = superio_inb(sio_data, WINB_ACT_REG); + if (!(val & 0x01)) { + pr_warn("Enabling HWM logical device\n"); + superio_outb(sio_data, WINB_ACT_REG, val | 0x01); } - res = superio_inb(sio_data, W83687THF_VID_DATA) & 0x3f; + err = 0; + pr_info(DRVNAME ": Found %s chip at %#x\n", + names[sio_data->type], *addr); -exit: + exit: superio_exit(sio_data); - return res; -} - -static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value) -{ - int word_sized; - - mutex_lock(&data->lock); - word_sized = (((reg & 0xff00) == 0x100) - || ((reg & 0xff00) == 0x200)) - && (((reg & 0x00ff) == 0x53) - || ((reg & 0x00ff) == 0x55)); - w83627hf_set_bank(data, reg); - outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET); - if (word_sized) { - outb_p(value >> 8, - data->addr + W83781D_DATA_REG_OFFSET); - outb_p((reg & 0xff) + 1, - data->addr + W83781D_ADDR_REG_OFFSET); - } - outb_p(value & 0xff, - data->addr + W83781D_DATA_REG_OFFSET); - w83627hf_reset_bank(data, reg); - mutex_unlock(&data->lock); - return 0; -} - -static void w83627hf_init_device(struct platform_device *pdev) -{ - struct w83627hf_data *data = platform_get_drvdata(pdev); - int i; - enum chips type = data->type; - u8 tmp; - - /* Minimize conflicts with other winbond i2c-only clients... */ - /* disable i2c subclients... how to disable main i2c client?? */ - /* force i2c address to relatively uncommon address */ - if (type == w83627hf) { - w83627hf_write_value(data, W83781D_REG_I2C_SUBADDR, 0x89); - w83627hf_write_value(data, W83781D_REG_I2C_ADDR, force_i2c); - } - - /* Read VID only once */ - if (type == w83627hf || type == w83637hf) { - int lo = w83627hf_read_value(data, W83781D_REG_VID_FANDIV); - int hi = w83627hf_read_value(data, W83781D_REG_CHIPID); - data->vid = (lo & 0x0f) | ((hi & 0x01) << 4); - } else if (type == w83627thf) { - data->vid = w83627thf_read_gpio5(pdev); - } else if (type == w83687thf) { - data->vid = w83687thf_read_vid(pdev); - } - - /* Read VRM & OVT Config only once */ - if (type == w83627thf || type == w83637hf || type == w83687thf) { - data->vrm_ovt = - w83627hf_read_value(data, W83627THF_REG_VRM_OVT_CFG); - } - - tmp = w83627hf_read_value(data, W83781D_REG_SCFG1); - for (i = 1; i <= 3; i++) { - if (!(tmp & BIT_SCFG1[i - 1])) { - data->sens[i - 1] = 4; - } else { - if (w83627hf_read_value - (data, - W83781D_REG_SCFG2) & BIT_SCFG2[i - 1]) - data->sens[i - 1] = 1; - else - data->sens[i - 1] = 2; - } - if ((type == w83697hf) && (i == 2)) - break; - } - - if(init) { - /* Enable temp2 */ - tmp = w83627hf_read_value(data, W83627HF_REG_TEMP2_CONFIG); - if (tmp & 0x01) { - dev_warn(&pdev->dev, "Enabling temp2, readings " - "might not make sense\n"); - w83627hf_write_value(data, W83627HF_REG_TEMP2_CONFIG, - tmp & 0xfe); - } - - /* Enable temp3 */ - if (type != w83697hf) { - tmp = w83627hf_read_value(data, - W83627HF_REG_TEMP3_CONFIG); - if (tmp & 0x01) { - dev_warn(&pdev->dev, "Enabling temp3, " - "readings might not make sense\n"); - w83627hf_write_value(data, - W83627HF_REG_TEMP3_CONFIG, tmp & 0xfe); - } - } - } - - /* Start monitoring */ - w83627hf_write_value(data, W83781D_REG_CONFIG, - (w83627hf_read_value(data, - W83781D_REG_CONFIG) & 0xf7) - | 0x01); - - /* Enable VBAT monitoring if needed */ - tmp = w83627hf_read_value(data, W83781D_REG_VBAT); - if (!(tmp & 0x01)) - w83627hf_write_value(data, W83781D_REG_VBAT, tmp | 0x01); -} - -static void w83627hf_update_fan_div(struct w83627hf_data *data) -{ - int reg; - - reg = w83627hf_read_value(data, W83781D_REG_VID_FANDIV); - data->fan_div[0] = (reg >> 4) & 0x03; - data->fan_div[1] = (reg >> 6) & 0x03; - if (data->type != w83697hf) { - data->fan_div[2] = (w83627hf_read_value(data, - W83781D_REG_PIN) >> 6) & 0x03; - } - reg = w83627hf_read_value(data, W83781D_REG_VBAT); - data->fan_div[0] |= (reg >> 3) & 0x04; - data->fan_div[1] |= (reg >> 4) & 0x04; - if (data->type != w83697hf) - data->fan_div[2] |= (reg >> 5) & 0x04; -} - -static struct w83627hf_data *w83627hf_update_device(struct device *dev) -{ - struct w83627hf_data *data = dev_get_drvdata(dev); - int i, num_temps = (data->type == w83697hf) ? 2 : 3; - int num_pwms = (data->type == w83697hf) ? 2 : 3; - - mutex_lock(&data->update_lock); - - if (time_after(jiffies, data->last_updated + HZ + HZ / 2) - || !data->valid) { - for (i = 0; i <= 8; i++) { - /* skip missing sensors */ - if (((data->type == w83697hf) && (i == 1)) || - ((data->type != w83627hf && data->type != w83697hf) - && (i == 5 || i == 6))) - continue; - data->in[i] = - w83627hf_read_value(data, W83781D_REG_IN(i)); - data->in_min[i] = - w83627hf_read_value(data, - W83781D_REG_IN_MIN(i)); - data->in_max[i] = - w83627hf_read_value(data, - W83781D_REG_IN_MAX(i)); - } - for (i = 0; i <= 2; i++) { - data->fan[i] = - w83627hf_read_value(data, W83627HF_REG_FAN(i)); - data->fan_min[i] = - w83627hf_read_value(data, - W83627HF_REG_FAN_MIN(i)); - } - for (i = 0; i <= 2; i++) { - u8 tmp = w83627hf_read_value(data, - W836X7HF_REG_PWM(data->type, i)); - /* bits 0-3 are reserved in 627THF */ - if (data->type == w83627thf) - tmp &= 0xf0; - data->pwm[i] = tmp; - if (i == 1 && - (data->type == w83627hf || data->type == w83697hf)) - break; - } - if (data->type == w83627hf) { - u8 tmp = w83627hf_read_value(data, - W83627HF_REG_PWM_FREQ); - data->pwm_freq[0] = tmp & 0x07; - data->pwm_freq[1] = (tmp >> 4) & 0x07; - } else if (data->type != w83627thf) { - for (i = 1; i <= 3; i++) { - data->pwm_freq[i - 1] = - w83627hf_read_value(data, - W83637HF_REG_PWM_FREQ[i - 1]); - if (i == 2 && (data->type == w83697hf)) - break; - } - } - if (data->type != w83627hf) { - for (i = 0; i < num_pwms; i++) { - u8 tmp = w83627hf_read_value(data, - W83627THF_REG_PWM_ENABLE[i]); - data->pwm_enable[i] = - ((tmp >> W83627THF_PWM_ENABLE_SHIFT[i]) - & 0x03) + 1; - } - } - for (i = 0; i < num_temps; i++) { - data->temp[i] = w83627hf_read_value( - data, w83627hf_reg_temp[i]); - data->temp_max[i] = w83627hf_read_value( - data, w83627hf_reg_temp_over[i]); - data->temp_max_hyst[i] = w83627hf_read_value( - data, w83627hf_reg_temp_hyst[i]); - } - - w83627hf_update_fan_div(data); - - data->alarms = - w83627hf_read_value(data, W83781D_REG_ALARM1) | - (w83627hf_read_value(data, W83781D_REG_ALARM2) << 8) | - (w83627hf_read_value(data, W83781D_REG_ALARM3) << 16); - i = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2); - data->beep_mask = (i << 8) | - w83627hf_read_value(data, W83781D_REG_BEEP_INTS1) | - w83627hf_read_value(data, W83781D_REG_BEEP_INTS3) << 16; - data->last_updated = jiffies; - data->valid = true; - } - - mutex_unlock(&data->update_lock); - - return data; + return err; } static int __init w83627hf_device_add(unsigned short address, diff --git a/drivers/hwmon/w83781d.c b/drivers/hwmon/w83781d.c index b3579721265f7529d8823ac74a98396600db3d7d..dacabf25e83ff6e7df9e2b35211f15ecd4f53ead 100644 --- a/drivers/hwmon/w83781d.c +++ b/drivers/hwmon/w83781d.c @@ -1171,7 +1171,7 @@ w83781d_detect(struct i2c_client *client, struct i2c_board_info *info) if (isa) mutex_unlock(&isa->update_lock); - strlcpy(info->type, client_name, I2C_NAME_SIZE); + strscpy(info->type, client_name, I2C_NAME_SIZE); return 0; @@ -1239,7 +1239,7 @@ static int w83781d_probe(struct i2c_client *client) return err; } -static int +static void w83781d_remove(struct i2c_client *client) { struct w83781d_data *data = i2c_get_clientdata(client); @@ -1250,8 +1250,6 @@ w83781d_remove(struct i2c_client *client) i2c_unregister_device(data->lm75[0]); i2c_unregister_device(data->lm75[1]); - - return 0; } static int diff --git a/drivers/hwmon/w83791d.c b/drivers/hwmon/w83791d.c index 80a9a78d7ce98a52a1791d4b95c67578b30af235..eaf6913650231c4f078ccef4f9e7fe4aac3ca324 100644 --- a/drivers/hwmon/w83791d.c +++ b/drivers/hwmon/w83791d.c @@ -315,7 +315,7 @@ struct w83791d_data { static int w83791d_probe(struct i2c_client *client); static int w83791d_detect(struct i2c_client *client, struct i2c_board_info *info); -static int w83791d_remove(struct i2c_client *client); +static void w83791d_remove(struct i2c_client *client); static int w83791d_read(struct i2c_client *client, u8 reg); static int w83791d_write(struct i2c_client *client, u8 reg, u8 value); @@ -1333,7 +1333,7 @@ static int w83791d_detect(struct i2c_client *client, if (val1 != 0x71 || val2 != 0x5c) return -ENODEV; - strlcpy(info->type, "w83791d", I2C_NAME_SIZE); + strscpy(info->type, "w83791d", I2C_NAME_SIZE); return 0; } @@ -1405,14 +1405,12 @@ error4: return err; } -static int w83791d_remove(struct i2c_client *client) +static void w83791d_remove(struct i2c_client *client) { struct w83791d_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); sysfs_remove_group(&client->dev.kobj, &w83791d_group); - - return 0; } static void w83791d_init_client(struct i2c_client *client) diff --git a/drivers/hwmon/w83792d.c b/drivers/hwmon/w83792d.c index 31a1cdc308779ac1288813b4266a6fbb9c22bfd4..6d160eee1446077ea07b47433df477e4afba3985 100644 --- a/drivers/hwmon/w83792d.c +++ b/drivers/hwmon/w83792d.c @@ -286,7 +286,7 @@ struct w83792d_data { static int w83792d_probe(struct i2c_client *client); static int w83792d_detect(struct i2c_client *client, struct i2c_board_info *info); -static int w83792d_remove(struct i2c_client *client); +static void w83792d_remove(struct i2c_client *client); static struct w83792d_data *w83792d_update_device(struct device *dev); #ifdef DEBUG @@ -1346,7 +1346,7 @@ w83792d_detect(struct i2c_client *client, struct i2c_board_info *info) if (val1 != 0x7a || val2 != 0x5c) return -ENODEV; - strlcpy(info->type, "w83792d", I2C_NAME_SIZE); + strscpy(info->type, "w83792d", I2C_NAME_SIZE); return 0; } @@ -1429,7 +1429,7 @@ exit_remove_files: return err; } -static int +static void w83792d_remove(struct i2c_client *client) { struct w83792d_data *data = i2c_get_clientdata(client); @@ -1440,8 +1440,6 @@ w83792d_remove(struct i2c_client *client) for (i = 0; i < ARRAY_SIZE(w83792d_group_fan); i++) sysfs_remove_group(&client->dev.kobj, &w83792d_group_fan[i]); - - return 0; } static void diff --git a/drivers/hwmon/w83793.c b/drivers/hwmon/w83793.c index 0a65d164c8f094847f3042da81a479e5e20bc2c2..a4926d907198c2cbc234531344e8507efedb4246 100644 --- a/drivers/hwmon/w83793.c +++ b/drivers/hwmon/w83793.c @@ -285,7 +285,7 @@ static int w83793_write_value(struct i2c_client *client, u16 reg, u8 value); static int w83793_probe(struct i2c_client *client); static int w83793_detect(struct i2c_client *client, struct i2c_board_info *info); -static int w83793_remove(struct i2c_client *client); +static void w83793_remove(struct i2c_client *client); static void w83793_init_client(struct i2c_client *client); static void w83793_update_nonvolatile(struct device *dev); static struct w83793_data *w83793_update_device(struct device *dev); @@ -1495,7 +1495,7 @@ static struct notifier_block watchdog_notifier = { * Init / remove routines */ -static int w83793_remove(struct i2c_client *client) +static void w83793_remove(struct i2c_client *client) { struct w83793_data *data = i2c_get_clientdata(client); struct device *dev = &client->dev; @@ -1554,8 +1554,6 @@ static int w83793_remove(struct i2c_client *client) mutex_lock(&watchdog_data_mutex); kref_put(&data->kref, w83793_release_resources); mutex_unlock(&watchdog_data_mutex); - - return 0; } static int @@ -1636,7 +1634,7 @@ static int w83793_detect(struct i2c_client *client, if (chip_id != 0x7b) return -ENODEV; - strlcpy(info->type, "w83793", I2C_NAME_SIZE); + strscpy(info->type, "w83793", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwmon/w83795.c b/drivers/hwmon/w83795.c index 45b12c4287dfafdf8b788d8d3a47271d45207f14..84ff5c57e98c28eab2cbeebc37756057d9d6ebd7 100644 --- a/drivers/hwmon/w83795.c +++ b/drivers/hwmon/w83795.c @@ -1967,7 +1967,7 @@ static int w83795_detect(struct i2c_client *client, else chip_name = "w83795g"; - strlcpy(info->type, chip_name, I2C_NAME_SIZE); + strscpy(info->type, chip_name, I2C_NAME_SIZE); dev_info(&adapter->dev, "Found %s rev. %c at 0x%02hx\n", chip_name, 'A' + (device_id & 0xf), address); @@ -2235,14 +2235,12 @@ exit_remove: return err; } -static int w83795_remove(struct i2c_client *client) +static void w83795_remove(struct i2c_client *client) { struct w83795_data *data = i2c_get_clientdata(client); hwmon_device_unregister(data->hwmon_dev); w83795_handle_files(&client->dev, device_remove_file_wrapper); - - return 0; } diff --git a/drivers/hwmon/w83l785ts.c b/drivers/hwmon/w83l785ts.c index a41f989d66e2702ffac5b86315b89991f66e9f3d..f3622de0d96fe224bf5c7376efd9dc31cfb0a4a7 100644 --- a/drivers/hwmon/w83l785ts.c +++ b/drivers/hwmon/w83l785ts.c @@ -65,7 +65,7 @@ static const unsigned short normal_i2c[] = { 0x2e, I2C_CLIENT_END }; static int w83l785ts_probe(struct i2c_client *client); static int w83l785ts_detect(struct i2c_client *client, struct i2c_board_info *info); -static int w83l785ts_remove(struct i2c_client *client); +static void w83l785ts_remove(struct i2c_client *client); static u8 w83l785ts_read_value(struct i2c_client *client, u8 reg, u8 defval); static struct w83l785ts_data *w83l785ts_update_device(struct device *dev); @@ -157,7 +157,7 @@ static int w83l785ts_detect(struct i2c_client *client, return -ENODEV; } - strlcpy(info->type, "w83l785ts", I2C_NAME_SIZE); + strscpy(info->type, "w83l785ts", I2C_NAME_SIZE); return 0; } @@ -203,7 +203,7 @@ exit_remove: return err; } -static int w83l785ts_remove(struct i2c_client *client) +static void w83l785ts_remove(struct i2c_client *client) { struct w83l785ts_data *data = i2c_get_clientdata(client); @@ -212,8 +212,6 @@ static int w83l785ts_remove(struct i2c_client *client) &sensor_dev_attr_temp1_input.dev_attr); device_remove_file(&client->dev, &sensor_dev_attr_temp1_max.dev_attr); - - return 0; } static u8 w83l785ts_read_value(struct i2c_client *client, u8 reg, u8 defval) diff --git a/drivers/hwmon/w83l786ng.c b/drivers/hwmon/w83l786ng.c index 11ba23c1af855cbf87da0ac125e1b3de1b48e576..2c4646fa84260dc3de1c07e6bce3a67b1cb21b3a 100644 --- a/drivers/hwmon/w83l786ng.c +++ b/drivers/hwmon/w83l786ng.c @@ -687,7 +687,7 @@ w83l786ng_detect(struct i2c_client *client, struct i2c_board_info *info) return -ENODEV; } - strlcpy(info->type, "w83l786ng", I2C_NAME_SIZE); + strscpy(info->type, "w83l786ng", I2C_NAME_SIZE); return 0; } diff --git a/drivers/hwtracing/Kconfig b/drivers/hwtracing/Kconfig index 13085835a63672cbe0918cccf9f181358a2c823e..911ee977103c0119f9e9aa0aa373d595d37b0485 100644 --- a/drivers/hwtracing/Kconfig +++ b/drivers/hwtracing/Kconfig @@ -5,4 +5,6 @@ source "drivers/hwtracing/stm/Kconfig" source "drivers/hwtracing/intel_th/Kconfig" +source "drivers/hwtracing/ptt/Kconfig" + endmenu diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig index 514a9b8086e3aae453d29ab9f529a95d5862a64a..45c1eb5dfcb763d6a811945cd1963ec8d24ac4c2 100644 --- a/drivers/hwtracing/coresight/Kconfig +++ b/drivers/hwtracing/coresight/Kconfig @@ -193,10 +193,10 @@ config CORESIGHT_TRBE depends on ARM64 && CORESIGHT_SOURCE_ETM4X help This driver provides support for percpu Trace Buffer Extension (TRBE). - TRBE always needs to be used along with it's corresponding percpu ETE + TRBE always needs to be used along with its corresponding percpu ETE component. ETE generates trace data which is then captured with TRBE. Unlike traditional sink devices, TRBE is a CPU feature accessible via - system registers. But it's explicit dependency with trace unit (ETE) + system registers. But its explicit dependency with trace unit (ETE) requires it to be plugged in as a coresight sink device. To compile this driver as a module, choose M here: the module will be diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index e0740c6dbd54235aa8c2cacf7f21a13facea07fb..bc90a03f478fd0364335ddadd32299ac23da7efe 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -365,26 +365,15 @@ static const struct etr_buf_operations etr_catu_buf_ops = { .get_data = catu_get_data_etr_buf, }; -coresight_simple_reg32(struct catu_drvdata, devid, CORESIGHT_DEVID); -coresight_simple_reg32(struct catu_drvdata, control, CATU_CONTROL); -coresight_simple_reg32(struct catu_drvdata, status, CATU_STATUS); -coresight_simple_reg32(struct catu_drvdata, mode, CATU_MODE); -coresight_simple_reg32(struct catu_drvdata, axictrl, CATU_AXICTRL); -coresight_simple_reg32(struct catu_drvdata, irqen, CATU_IRQEN); -coresight_simple_reg64(struct catu_drvdata, sladdr, - CATU_SLADDRLO, CATU_SLADDRHI); -coresight_simple_reg64(struct catu_drvdata, inaddr, - CATU_INADDRLO, CATU_INADDRHI); - static struct attribute *catu_mgmt_attrs[] = { - &dev_attr_devid.attr, - &dev_attr_control.attr, - &dev_attr_status.attr, - &dev_attr_mode.attr, - &dev_attr_axictrl.attr, - &dev_attr_irqen.attr, - &dev_attr_sladdr.attr, - &dev_attr_inaddr.attr, + coresight_simple_reg32(devid, CORESIGHT_DEVID), + coresight_simple_reg32(control, CATU_CONTROL), + coresight_simple_reg32(status, CATU_STATUS), + coresight_simple_reg32(mode, CATU_MODE), + coresight_simple_reg32(axictrl, CATU_AXICTRL), + coresight_simple_reg32(irqen, CATU_IRQEN), + coresight_simple_reg64(sladdr, CATU_SLADDRLO, CATU_SLADDRHI), + coresight_simple_reg64(inaddr, CATU_INADDRLO, CATU_INADDRHI), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-catu.h b/drivers/hwtracing/coresight/coresight-catu.h index 6160c2d75a56dbd2563bbfbd95af8fd08b4755e7..442e034bbfbaf47a57546091c3877cb44072ac7a 100644 --- a/drivers/hwtracing/coresight/coresight-catu.h +++ b/drivers/hwtracing/coresight/coresight-catu.h @@ -70,24 +70,24 @@ struct catu_drvdata { static inline u32 \ catu_read_##name(struct catu_drvdata *drvdata) \ { \ - return coresight_read_reg_pair(drvdata->base, offset, -1); \ + return csdev_access_relaxed_read32(&drvdata->csdev->access, offset); \ } \ static inline void \ catu_write_##name(struct catu_drvdata *drvdata, u32 val) \ { \ - coresight_write_reg_pair(drvdata->base, val, offset, -1); \ + csdev_access_relaxed_write32(&drvdata->csdev->access, val, offset); \ } #define CATU_REG_PAIR(name, lo_off, hi_off) \ static inline u64 \ catu_read_##name(struct catu_drvdata *drvdata) \ { \ - return coresight_read_reg_pair(drvdata->base, lo_off, hi_off); \ + return csdev_access_relaxed_read_pair(&drvdata->csdev->access, lo_off, hi_off); \ } \ static inline void \ catu_write_##name(struct catu_drvdata *drvdata, u64 val) \ { \ - coresight_write_reg_pair(drvdata->base, val, lo_off, hi_off); \ + csdev_access_relaxed_write_pair(&drvdata->csdev->access, val, lo_off, hi_off); \ } CATU_REG32(control, CATU_CONTROL); diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 1edfec1e9d18e49d34f6dad777b1bd3c677495ba..d5dbc67bacb44aed6e1422f175b3d102562d889e 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -60,6 +60,34 @@ EXPORT_SYMBOL_GPL(coresight_barrier_pkt); static const struct cti_assoc_op *cti_assoc_ops; +ssize_t coresight_simple_show_pair(struct device *_dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); + struct cs_pair_attribute *cs_attr = container_of(attr, struct cs_pair_attribute, attr); + u64 val; + + pm_runtime_get_sync(_dev->parent); + val = csdev_access_relaxed_read_pair(&csdev->access, cs_attr->lo_off, cs_attr->hi_off); + pm_runtime_put_sync(_dev->parent); + return sysfs_emit(buf, "0x%llx\n", val); +} +EXPORT_SYMBOL_GPL(coresight_simple_show_pair); + +ssize_t coresight_simple_show32(struct device *_dev, + struct device_attribute *attr, char *buf) +{ + struct coresight_device *csdev = container_of(_dev, struct coresight_device, dev); + struct cs_off_attribute *cs_attr = container_of(attr, struct cs_off_attribute, attr); + u64 val; + + pm_runtime_get_sync(_dev->parent); + val = csdev_access_relaxed_read32(&csdev->access, cs_attr->off); + pm_runtime_put_sync(_dev->parent); + return sysfs_emit(buf, "0x%llx\n", val); +} +EXPORT_SYMBOL_GPL(coresight_simple_show32); + void coresight_set_cti_ops(const struct cti_assoc_op *cti_op) { cti_assoc_ops = cti_op; diff --git a/drivers/hwtracing/coresight/coresight-cti-sysfs.c b/drivers/hwtracing/coresight/coresight-cti-sysfs.c index 7ff7e7780bbfbcde5b74f7a2f118b0a3914d1267..6d59c815ecf5efb84e0d54f5384994abda20054b 100644 --- a/drivers/hwtracing/coresight/coresight-cti-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-cti-sysfs.c @@ -163,48 +163,82 @@ static struct attribute *coresight_cti_attrs[] = { /* register based attributes */ -/* macro to access RO registers with power check only (no enable check). */ -#define coresight_cti_reg(name, offset) \ -static ssize_t name##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ - u32 val = 0; \ - pm_runtime_get_sync(dev->parent); \ - spin_lock(&drvdata->spinlock); \ - if (drvdata->config.hw_powered) \ - val = readl_relaxed(drvdata->base + offset); \ - spin_unlock(&drvdata->spinlock); \ - pm_runtime_put_sync(dev->parent); \ - return sprintf(buf, "0x%x\n", val); \ -} \ -static DEVICE_ATTR_RO(name) +/* Read registers with power check only (no enable check). */ +static ssize_t coresight_cti_reg_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_off_attribute *cti_attr = container_of(attr, struct cs_off_attribute, attr); + u32 val = 0; -/* coresight management registers */ -coresight_cti_reg(devaff0, CTIDEVAFF0); -coresight_cti_reg(devaff1, CTIDEVAFF1); -coresight_cti_reg(authstatus, CORESIGHT_AUTHSTATUS); -coresight_cti_reg(devarch, CORESIGHT_DEVARCH); -coresight_cti_reg(devid, CORESIGHT_DEVID); -coresight_cti_reg(devtype, CORESIGHT_DEVTYPE); -coresight_cti_reg(pidr0, CORESIGHT_PERIPHIDR0); -coresight_cti_reg(pidr1, CORESIGHT_PERIPHIDR1); -coresight_cti_reg(pidr2, CORESIGHT_PERIPHIDR2); -coresight_cti_reg(pidr3, CORESIGHT_PERIPHIDR3); -coresight_cti_reg(pidr4, CORESIGHT_PERIPHIDR4); + pm_runtime_get_sync(dev->parent); + spin_lock(&drvdata->spinlock); + if (drvdata->config.hw_powered) + val = readl_relaxed(drvdata->base + cti_attr->off); + spin_unlock(&drvdata->spinlock); + pm_runtime_put_sync(dev->parent); + return sysfs_emit(buf, "0x%x\n", val); +} +/* Write registers with power check only (no enable check). */ +static __maybe_unused ssize_t coresight_cti_reg_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t size) +{ + struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct cs_off_attribute *cti_attr = container_of(attr, struct cs_off_attribute, attr); + unsigned long val = 0; + + if (kstrtoul(buf, 0, &val)) + return -EINVAL; + + pm_runtime_get_sync(dev->parent); + spin_lock(&drvdata->spinlock); + if (drvdata->config.hw_powered) + cti_write_single_reg(drvdata, cti_attr->off, val); + spin_unlock(&drvdata->spinlock); + pm_runtime_put_sync(dev->parent); + return size; +} + +#define coresight_cti_reg(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_cti_reg_show, NULL), \ + offset \ + } \ + })[0].attr.attr) + +#define coresight_cti_reg_rw(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0644, coresight_cti_reg_show, \ + coresight_cti_reg_store), \ + offset \ + } \ + })[0].attr.attr) + +#define coresight_cti_reg_wo(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0200, NULL, coresight_cti_reg_store), \ + offset \ + } \ + })[0].attr.attr) + +/* coresight management registers */ static struct attribute *coresight_cti_mgmt_attrs[] = { - &dev_attr_devaff0.attr, - &dev_attr_devaff1.attr, - &dev_attr_authstatus.attr, - &dev_attr_devarch.attr, - &dev_attr_devid.attr, - &dev_attr_devtype.attr, - &dev_attr_pidr0.attr, - &dev_attr_pidr1.attr, - &dev_attr_pidr2.attr, - &dev_attr_pidr3.attr, - &dev_attr_pidr4.attr, + coresight_cti_reg(devaff0, CTIDEVAFF0), + coresight_cti_reg(devaff1, CTIDEVAFF1), + coresight_cti_reg(authstatus, CORESIGHT_AUTHSTATUS), + coresight_cti_reg(devarch, CORESIGHT_DEVARCH), + coresight_cti_reg(devid, CORESIGHT_DEVID), + coresight_cti_reg(devtype, CORESIGHT_DEVTYPE), + coresight_cti_reg(pidr0, CORESIGHT_PERIPHIDR0), + coresight_cti_reg(pidr1, CORESIGHT_PERIPHIDR1), + coresight_cti_reg(pidr2, CORESIGHT_PERIPHIDR2), + coresight_cti_reg(pidr3, CORESIGHT_PERIPHIDR3), + coresight_cti_reg(pidr4, CORESIGHT_PERIPHIDR4), NULL, }; @@ -454,86 +488,11 @@ static ssize_t apppulse_store(struct device *dev, } static DEVICE_ATTR_WO(apppulse); -coresight_cti_reg(triginstatus, CTITRIGINSTATUS); -coresight_cti_reg(trigoutstatus, CTITRIGOUTSTATUS); -coresight_cti_reg(chinstatus, CTICHINSTATUS); -coresight_cti_reg(choutstatus, CTICHOUTSTATUS); - /* * Define CONFIG_CORESIGHT_CTI_INTEGRATION_REGS to enable the access to the * integration control registers. Normally only used to investigate connection * data. */ -#ifdef CONFIG_CORESIGHT_CTI_INTEGRATION_REGS - -/* macro to access RW registers with power check only (no enable check). */ -#define coresight_cti_reg_rw(name, offset) \ -static ssize_t name##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ - u32 val = 0; \ - pm_runtime_get_sync(dev->parent); \ - spin_lock(&drvdata->spinlock); \ - if (drvdata->config.hw_powered) \ - val = readl_relaxed(drvdata->base + offset); \ - spin_unlock(&drvdata->spinlock); \ - pm_runtime_put_sync(dev->parent); \ - return sprintf(buf, "0x%x\n", val); \ -} \ - \ -static ssize_t name##_store(struct device *dev, \ - struct device_attribute *attr, \ - const char *buf, size_t size) \ -{ \ - struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ - unsigned long val = 0; \ - if (kstrtoul(buf, 0, &val)) \ - return -EINVAL; \ - \ - pm_runtime_get_sync(dev->parent); \ - spin_lock(&drvdata->spinlock); \ - if (drvdata->config.hw_powered) \ - cti_write_single_reg(drvdata, offset, val); \ - spin_unlock(&drvdata->spinlock); \ - pm_runtime_put_sync(dev->parent); \ - return size; \ -} \ -static DEVICE_ATTR_RW(name) - -/* macro to access WO registers with power check only (no enable check). */ -#define coresight_cti_reg_wo(name, offset) \ -static ssize_t name##_store(struct device *dev, \ - struct device_attribute *attr, \ - const char *buf, size_t size) \ -{ \ - struct cti_drvdata *drvdata = dev_get_drvdata(dev->parent); \ - unsigned long val = 0; \ - if (kstrtoul(buf, 0, &val)) \ - return -EINVAL; \ - \ - pm_runtime_get_sync(dev->parent); \ - spin_lock(&drvdata->spinlock); \ - if (drvdata->config.hw_powered) \ - cti_write_single_reg(drvdata, offset, val); \ - spin_unlock(&drvdata->spinlock); \ - pm_runtime_put_sync(dev->parent); \ - return size; \ -} \ -static DEVICE_ATTR_WO(name) - -coresight_cti_reg_rw(itchout, ITCHOUT); -coresight_cti_reg_rw(ittrigout, ITTRIGOUT); -coresight_cti_reg_rw(itctrl, CORESIGHT_ITCTRL); -coresight_cti_reg_wo(itchinack, ITCHINACK); -coresight_cti_reg_wo(ittriginack, ITTRIGINACK); -coresight_cti_reg(ittrigin, ITTRIGIN); -coresight_cti_reg(itchin, ITCHIN); -coresight_cti_reg(itchoutack, ITCHOUTACK); -coresight_cti_reg(ittrigoutack, ITTRIGOUTACK); - -#endif /* CORESIGHT_CTI_INTEGRATION_REGS */ - static struct attribute *coresight_cti_regs_attrs[] = { &dev_attr_inout_sel.attr, &dev_attr_inen.attr, @@ -544,20 +503,20 @@ static struct attribute *coresight_cti_regs_attrs[] = { &dev_attr_appset.attr, &dev_attr_appclear.attr, &dev_attr_apppulse.attr, - &dev_attr_triginstatus.attr, - &dev_attr_trigoutstatus.attr, - &dev_attr_chinstatus.attr, - &dev_attr_choutstatus.attr, + coresight_cti_reg(triginstatus, CTITRIGINSTATUS), + coresight_cti_reg(trigoutstatus, CTITRIGOUTSTATUS), + coresight_cti_reg(chinstatus, CTICHINSTATUS), + coresight_cti_reg(choutstatus, CTICHOUTSTATUS), #ifdef CONFIG_CORESIGHT_CTI_INTEGRATION_REGS - &dev_attr_itctrl.attr, - &dev_attr_ittrigin.attr, - &dev_attr_itchin.attr, - &dev_attr_ittrigout.attr, - &dev_attr_itchout.attr, - &dev_attr_itchoutack.attr, - &dev_attr_ittrigoutack.attr, - &dev_attr_ittriginack.attr, - &dev_attr_itchinack.attr, + coresight_cti_reg_rw(itctrl, CORESIGHT_ITCTRL), + coresight_cti_reg(ittrigin, ITTRIGIN), + coresight_cti_reg(itchin, ITCHIN), + coresight_cti_reg_rw(ittrigout, ITTRIGOUT), + coresight_cti_reg_rw(itchout, ITCHOUT), + coresight_cti_reg(itchoutack, ITCHOUTACK), + coresight_cti_reg(ittrigoutack, ITTRIGOUTACK), + coresight_cti_reg_wo(ittriginack, ITTRIGINACK), + coresight_cti_reg_wo(itchinack, ITCHINACK), #endif NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index efa39820aceca2c142e64659b0bdaa6ceeffdd29..8aa6e4f83e42b58eaff64f1f373bb67b03cda2c7 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -655,27 +655,15 @@ static const struct file_operations etb_fops = { .llseek = no_llseek, }; -#define coresight_etb10_reg(name, offset) \ - coresight_simple_reg32(struct etb_drvdata, name, offset) - -coresight_etb10_reg(rdp, ETB_RAM_DEPTH_REG); -coresight_etb10_reg(sts, ETB_STATUS_REG); -coresight_etb10_reg(rrp, ETB_RAM_READ_POINTER); -coresight_etb10_reg(rwp, ETB_RAM_WRITE_POINTER); -coresight_etb10_reg(trg, ETB_TRG); -coresight_etb10_reg(ctl, ETB_CTL_REG); -coresight_etb10_reg(ffsr, ETB_FFSR); -coresight_etb10_reg(ffcr, ETB_FFCR); - static struct attribute *coresight_etb_mgmt_attrs[] = { - &dev_attr_rdp.attr, - &dev_attr_sts.attr, - &dev_attr_rrp.attr, - &dev_attr_rwp.attr, - &dev_attr_trg.attr, - &dev_attr_ctl.attr, - &dev_attr_ffsr.attr, - &dev_attr_ffcr.attr, + coresight_simple_reg32(rdp, ETB_RAM_DEPTH_REG), + coresight_simple_reg32(sts, ETB_STATUS_REG), + coresight_simple_reg32(rrp, ETB_RAM_READ_POINTER), + coresight_simple_reg32(rwp, ETB_RAM_WRITE_POINTER), + coresight_simple_reg32(trg, ETB_TRG), + coresight_simple_reg32(ctl, ETB_CTL_REG), + coresight_simple_reg32(ffsr, ETB_FFSR), + coresight_simple_reg32(ffcr, ETB_FFCR), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c index 68fcbf4ce7a8e3fc25719619abf150fa8ecb6542..fd81eca3ec182d4fabbace79986a69ce8b2480ac 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c @@ -1252,31 +1252,17 @@ static struct attribute *coresight_etm_attrs[] = { NULL, }; -#define coresight_etm3x_reg(name, offset) \ - coresight_simple_reg32(struct etm_drvdata, name, offset) - -coresight_etm3x_reg(etmccr, ETMCCR); -coresight_etm3x_reg(etmccer, ETMCCER); -coresight_etm3x_reg(etmscr, ETMSCR); -coresight_etm3x_reg(etmidr, ETMIDR); -coresight_etm3x_reg(etmcr, ETMCR); -coresight_etm3x_reg(etmtraceidr, ETMTRACEIDR); -coresight_etm3x_reg(etmteevr, ETMTEEVR); -coresight_etm3x_reg(etmtssvr, ETMTSSCR); -coresight_etm3x_reg(etmtecr1, ETMTECR1); -coresight_etm3x_reg(etmtecr2, ETMTECR2); - static struct attribute *coresight_etm_mgmt_attrs[] = { - &dev_attr_etmccr.attr, - &dev_attr_etmccer.attr, - &dev_attr_etmscr.attr, - &dev_attr_etmidr.attr, - &dev_attr_etmcr.attr, - &dev_attr_etmtraceidr.attr, - &dev_attr_etmteevr.attr, - &dev_attr_etmtssvr.attr, - &dev_attr_etmtecr1.attr, - &dev_attr_etmtecr2.attr, + coresight_simple_reg32(etmccr, ETMCCR), + coresight_simple_reg32(etmccer, ETMCCER), + coresight_simple_reg32(etmscr, ETMSCR), + coresight_simple_reg32(etmidr, ETMIDR), + coresight_simple_reg32(etmcr, ETMCR), + coresight_simple_reg32(etmtraceidr, ETMTRACEIDR), + coresight_simple_reg32(etmteevr, ETMTEEVR), + coresight_simple_reg32(etmtssvr, ETMTSSCR), + coresight_simple_reg32(etmtecr1, ETMTECR1), + coresight_simple_reg32(etmtecr2, ETMTECR2), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d39660a3e50c5634df15c68d7a50c8304f483368..80fefaba58eebea9f8252d1884ecee5a84e25798 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -966,7 +966,7 @@ static inline bool cpu_supports_sysreg_trace(void) { u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); - return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0; + return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0; } static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, @@ -1054,7 +1054,7 @@ static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) u64 trfcr; drvdata->trfcr = 0; - if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT)) + if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT)) return; /* diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index 6ea8181816fc83673a8f015e58a9986c25206267..9cac848cffafc736df6052729134ac83cafe681c 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2306,6 +2306,34 @@ static ssize_t cpu_show(struct device *dev, } static DEVICE_ATTR_RO(cpu); +static ssize_t ts_source_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int val; + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + + if (!drvdata->trfcr) { + val = -1; + goto out; + } + + switch (drvdata->trfcr & TRFCR_ELx_TS_MASK) { + case TRFCR_ELx_TS_VIRTUAL: + case TRFCR_ELx_TS_GUEST_PHYSICAL: + case TRFCR_ELx_TS_PHYSICAL: + val = FIELD_GET(TRFCR_ELx_TS_MASK, drvdata->trfcr); + break; + default: + val = -1; + break; + } + +out: + return sysfs_emit(buf, "%d\n", val); +} +static DEVICE_ATTR_RO(ts_source); + static struct attribute *coresight_etmv4_attrs[] = { &dev_attr_nr_pe_cmp.attr, &dev_attr_nr_addr_cmp.attr, @@ -2360,6 +2388,7 @@ static struct attribute *coresight_etmv4_attrs[] = { &dev_attr_vmid_val.attr, &dev_attr_vmid_masks.attr, &dev_attr_cpu.attr, + &dev_attr_ts_source.attr, NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index ff1dd2092ac5bbe9a476637be33099b45922cb36..595ce58620567226c77e0600d5a2164e1e6e07ce 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -39,32 +39,37 @@ #define ETM_MODE_EXCL_KERN BIT(30) #define ETM_MODE_EXCL_USER BIT(31) +struct cs_pair_attribute { + struct device_attribute attr; + u32 lo_off; + u32 hi_off; +}; + +struct cs_off_attribute { + struct device_attribute attr; + u32 off; +}; -typedef u32 (*coresight_read_fn)(const struct device *, u32 offset); -#define __coresight_simple_func(type, func, name, lo_off, hi_off) \ -static ssize_t name##_show(struct device *_dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - type *drvdata = dev_get_drvdata(_dev->parent); \ - coresight_read_fn fn = func; \ - u64 val; \ - pm_runtime_get_sync(_dev->parent); \ - if (fn) \ - val = (u64)fn(_dev->parent, lo_off); \ - else \ - val = coresight_read_reg_pair(drvdata->base, \ - lo_off, hi_off); \ - pm_runtime_put_sync(_dev->parent); \ - return scnprintf(buf, PAGE_SIZE, "0x%llx\n", val); \ -} \ -static DEVICE_ATTR_RO(name) - -#define coresight_simple_func(type, func, name, offset) \ - __coresight_simple_func(type, func, name, offset, -1) -#define coresight_simple_reg32(type, name, offset) \ - __coresight_simple_func(type, NULL, name, offset, -1) -#define coresight_simple_reg64(type, name, lo_off, hi_off) \ - __coresight_simple_func(type, NULL, name, lo_off, hi_off) +extern ssize_t coresight_simple_show32(struct device *_dev, + struct device_attribute *attr, char *buf); +extern ssize_t coresight_simple_show_pair(struct device *_dev, + struct device_attribute *attr, char *buf); + +#define coresight_simple_reg32(name, offset) \ + (&((struct cs_off_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_simple_show32, NULL), \ + offset \ + } \ + })[0].attr.attr) + +#define coresight_simple_reg64(name, lo_off, hi_off) \ + (&((struct cs_pair_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_simple_show_pair, NULL), \ + lo_off, hi_off \ + } \ + })[0].attr.attr) extern const u32 coresight_barrier_pkt[4]; #define CORESIGHT_BARRIER_PKT_SIZE (sizeof(coresight_barrier_pkt)) @@ -127,25 +132,6 @@ static inline void CS_UNLOCK(void __iomem *addr) } while (0); } -static inline u64 -coresight_read_reg_pair(void __iomem *addr, s32 lo_offset, s32 hi_offset) -{ - u64 val; - - val = readl_relaxed(addr + lo_offset); - val |= (hi_offset < 0) ? 0 : - (u64)readl_relaxed(addr + hi_offset) << 32; - return val; -} - -static inline void coresight_write_reg_pair(void __iomem *addr, u64 val, - s32 lo_offset, s32 hi_offset) -{ - writel_relaxed((u32)val, addr + lo_offset); - if (hi_offset >= 0) - writel_relaxed((u32)(val >> 32), addr + hi_offset); -} - void coresight_disable_path(struct list_head *path); int coresight_enable_path(struct list_head *path, u32 mode, void *sink_data); struct coresight_device *coresight_get_sink(struct list_head *path); diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index b86acbc74cf03472c955d4e9e97c5823fe464dd2..4dd50546d7e4376ef1e35d18d3d3e40807621619 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -196,15 +196,9 @@ static const struct coresight_ops replicator_cs_ops = { .link_ops = &replicator_link_ops, }; -#define coresight_replicator_reg(name, offset) \ - coresight_simple_reg32(struct replicator_drvdata, name, offset) - -coresight_replicator_reg(idfilter0, REPLICATOR_IDFILTER0); -coresight_replicator_reg(idfilter1, REPLICATOR_IDFILTER1); - static struct attribute *replicator_mgmt_attrs[] = { - &dev_attr_idfilter0.attr, - &dev_attr_idfilter1.attr, + coresight_simple_reg32(idfilter0, REPLICATOR_IDFILTER0), + coresight_simple_reg32(idfilter1, REPLICATOR_IDFILTER1), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index bb14a3a8a921013e5aee9b85a94f19b05127304e..463f449cfb795806cf0490e2dcb191e19b8c07f7 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -634,22 +634,6 @@ static ssize_t traceid_store(struct device *dev, } static DEVICE_ATTR_RW(traceid); -#define coresight_stm_reg(name, offset) \ - coresight_simple_reg32(struct stm_drvdata, name, offset) - -coresight_stm_reg(tcsr, STMTCSR); -coresight_stm_reg(tsfreqr, STMTSFREQR); -coresight_stm_reg(syncr, STMSYNCR); -coresight_stm_reg(sper, STMSPER); -coresight_stm_reg(spter, STMSPTER); -coresight_stm_reg(privmaskr, STMPRIVMASKR); -coresight_stm_reg(spscr, STMSPSCR); -coresight_stm_reg(spmscr, STMSPMSCR); -coresight_stm_reg(spfeat1r, STMSPFEAT1R); -coresight_stm_reg(spfeat2r, STMSPFEAT2R); -coresight_stm_reg(spfeat3r, STMSPFEAT3R); -coresight_stm_reg(devid, CORESIGHT_DEVID); - static struct attribute *coresight_stm_attrs[] = { &dev_attr_hwevent_enable.attr, &dev_attr_hwevent_select.attr, @@ -660,18 +644,18 @@ static struct attribute *coresight_stm_attrs[] = { }; static struct attribute *coresight_stm_mgmt_attrs[] = { - &dev_attr_tcsr.attr, - &dev_attr_tsfreqr.attr, - &dev_attr_syncr.attr, - &dev_attr_sper.attr, - &dev_attr_spter.attr, - &dev_attr_privmaskr.attr, - &dev_attr_spscr.attr, - &dev_attr_spmscr.attr, - &dev_attr_spfeat1r.attr, - &dev_attr_spfeat2r.attr, - &dev_attr_spfeat3r.attr, - &dev_attr_devid.attr, + coresight_simple_reg32(tcsr, STMTCSR), + coresight_simple_reg32(tsfreqr, STMTSFREQR), + coresight_simple_reg32(syncr, STMSYNCR), + coresight_simple_reg32(sper, STMSPER), + coresight_simple_reg32(spter, STMSPTER), + coresight_simple_reg32(privmaskr, STMPRIVMASKR), + coresight_simple_reg32(spscr, STMSPSCR), + coresight_simple_reg32(spmscr, STMSPMSCR), + coresight_simple_reg32(spfeat1r, STMSPFEAT1R), + coresight_simple_reg32(spfeat2r, STMSPFEAT2R), + coresight_simple_reg32(spfeat3r, STMSPFEAT3R), + coresight_simple_reg32(devid, CORESIGHT_DEVID), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index d0276af82494cf367f695855898ae520d64bea90..07abf28ad7253e1503c41b54bfe61e12414a0f3a 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -251,41 +251,21 @@ static enum tmc_mem_intf_width tmc_get_memwidth(u32 devid) return memwidth; } -#define coresight_tmc_reg(name, offset) \ - coresight_simple_reg32(struct tmc_drvdata, name, offset) -#define coresight_tmc_reg64(name, lo_off, hi_off) \ - coresight_simple_reg64(struct tmc_drvdata, name, lo_off, hi_off) - -coresight_tmc_reg(rsz, TMC_RSZ); -coresight_tmc_reg(sts, TMC_STS); -coresight_tmc_reg(trg, TMC_TRG); -coresight_tmc_reg(ctl, TMC_CTL); -coresight_tmc_reg(ffsr, TMC_FFSR); -coresight_tmc_reg(ffcr, TMC_FFCR); -coresight_tmc_reg(mode, TMC_MODE); -coresight_tmc_reg(pscr, TMC_PSCR); -coresight_tmc_reg(axictl, TMC_AXICTL); -coresight_tmc_reg(authstatus, TMC_AUTHSTATUS); -coresight_tmc_reg(devid, CORESIGHT_DEVID); -coresight_tmc_reg64(rrp, TMC_RRP, TMC_RRPHI); -coresight_tmc_reg64(rwp, TMC_RWP, TMC_RWPHI); -coresight_tmc_reg64(dba, TMC_DBALO, TMC_DBAHI); - static struct attribute *coresight_tmc_mgmt_attrs[] = { - &dev_attr_rsz.attr, - &dev_attr_sts.attr, - &dev_attr_rrp.attr, - &dev_attr_rwp.attr, - &dev_attr_trg.attr, - &dev_attr_ctl.attr, - &dev_attr_ffsr.attr, - &dev_attr_ffcr.attr, - &dev_attr_mode.attr, - &dev_attr_pscr.attr, - &dev_attr_devid.attr, - &dev_attr_dba.attr, - &dev_attr_axictl.attr, - &dev_attr_authstatus.attr, + coresight_simple_reg32(rsz, TMC_RSZ), + coresight_simple_reg32(sts, TMC_STS), + coresight_simple_reg64(rrp, TMC_RRP, TMC_RRPHI), + coresight_simple_reg64(rwp, TMC_RWP, TMC_RWPHI), + coresight_simple_reg32(trg, TMC_TRG), + coresight_simple_reg32(ctl, TMC_CTL), + coresight_simple_reg32(ffsr, TMC_FFSR), + coresight_simple_reg32(ffcr, TMC_FFCR), + coresight_simple_reg32(mode, TMC_MODE), + coresight_simple_reg32(pscr, TMC_PSCR), + coresight_simple_reg32(devid, CORESIGHT_DEVID), + coresight_simple_reg64(dba, TMC_DBALO, TMC_DBAHI), + coresight_simple_reg32(axictl, TMC_AXICTL), + coresight_simple_reg32(authstatus, TMC_AUTHSTATUS), NULL, }; diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index 6bec20a392b3219e6b376c215fe26861b51ada55..66959557cf3989377aa241c3c4e6f2dac2728597 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -282,12 +282,12 @@ ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata, static inline u64 \ tmc_read_##name(struct tmc_drvdata *drvdata) \ { \ - return coresight_read_reg_pair(drvdata->base, lo_off, hi_off); \ + return csdev_access_relaxed_read_pair(&drvdata->csdev->access, lo_off, hi_off); \ } \ static inline void \ tmc_write_##name(struct tmc_drvdata *drvdata, u64 val) \ { \ - coresight_write_reg_pair(drvdata->base, val, lo_off, hi_off); \ + csdev_access_relaxed_write_pair(&drvdata->csdev->access, val, lo_off, hi_off); \ } TMC_REG_PAIR(rrp, TMC_RRP, TMC_RRPHI) diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h index 30e4d7db4f8e12e2142852efa9d1a3c5978adc9c..98ff1b17ad079d333f5285d21074fbef1a683941 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.h +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -20,7 +20,8 @@ static inline bool is_trbe_available(void) { u64 aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); - unsigned int trbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_TRBE_SHIFT); + unsigned int trbe = cpuid_feature_extract_unsigned_field(aa64dfr0, + ID_AA64DFR0_EL1_TraceBuffer_SHIFT); return trbe >= 0b0001; } diff --git a/drivers/hwtracing/ptt/Kconfig b/drivers/hwtracing/ptt/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..6d46a09ffeb91992c6e943fa1a8fe7421dd88a99 --- /dev/null +++ b/drivers/hwtracing/ptt/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HISI_PTT + tristate "HiSilicon PCIe Tune and Trace Device" + depends on ARM64 || (COMPILE_TEST && 64BIT) + depends on PCI && HAS_DMA && HAS_IOMEM && PERF_EVENTS + help + HiSilicon PCIe Tune and Trace device exists as a PCIe RCiEP + device, and it provides support for PCIe traffic tuning and + tracing TLP headers to the memory. + + This driver can also be built as a module. If so, the module + will be called hisi_ptt. diff --git a/drivers/hwtracing/ptt/Makefile b/drivers/hwtracing/ptt/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..908c09a981619c99a5a5a222188d41cf816b8896 --- /dev/null +++ b/drivers/hwtracing/ptt/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_HISI_PTT) += hisi_ptt.o diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..5d5526aa60c4023d1107ff6fe560ebd3df52a0d1 --- /dev/null +++ b/drivers/hwtracing/ptt/hisi_ptt.c @@ -0,0 +1,1046 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for HiSilicon PCIe tune and trace device + * + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hisi_ptt.h" + +/* Dynamic CPU hotplug state used by PTT */ +static enum cpuhp_state hisi_ptt_pmu_online; + +static bool hisi_ptt_wait_tuning_finish(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + return !readl_poll_timeout(hisi_ptt->iobase + HISI_PTT_TUNING_INT_STAT, + val, !(val & HISI_PTT_TUNING_INT_STAT_MASK), + HISI_PTT_WAIT_POLL_INTERVAL_US, + HISI_PTT_WAIT_TUNE_TIMEOUT_US); +} + +static ssize_t hisi_ptt_tune_attr_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + struct dev_ext_attribute *ext_attr; + struct hisi_ptt_tune_desc *desc; + u32 reg; + u16 val; + + ext_attr = container_of(attr, struct dev_ext_attribute, attr); + desc = ext_attr->var; + + mutex_lock(&hisi_ptt->tune_lock); + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); + reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, + desc->event_code); + writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + + /* Write all 1 to indicates it's the read process */ + writel(~0U, hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + + if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { + mutex_unlock(&hisi_ptt->tune_lock); + return -ETIMEDOUT; + } + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + reg &= HISI_PTT_TUNING_DATA_VAL_MASK; + val = FIELD_GET(HISI_PTT_TUNING_DATA_VAL_MASK, reg); + + mutex_unlock(&hisi_ptt->tune_lock); + return sysfs_emit(buf, "%u\n", val); +} + +static ssize_t hisi_ptt_tune_attr_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + struct dev_ext_attribute *ext_attr; + struct hisi_ptt_tune_desc *desc; + u32 reg; + u16 val; + + ext_attr = container_of(attr, struct dev_ext_attribute, attr); + desc = ext_attr->var; + + if (kstrtou16(buf, 10, &val)) + return -EINVAL; + + mutex_lock(&hisi_ptt->tune_lock); + + reg = readl(hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + reg &= ~(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB); + reg |= FIELD_PREP(HISI_PTT_TUNING_CTRL_CODE | HISI_PTT_TUNING_CTRL_SUB, + desc->event_code); + writel(reg, hisi_ptt->iobase + HISI_PTT_TUNING_CTRL); + writel(FIELD_PREP(HISI_PTT_TUNING_DATA_VAL_MASK, val), + hisi_ptt->iobase + HISI_PTT_TUNING_DATA); + + if (!hisi_ptt_wait_tuning_finish(hisi_ptt)) { + mutex_unlock(&hisi_ptt->tune_lock); + return -ETIMEDOUT; + } + + mutex_unlock(&hisi_ptt->tune_lock); + return count; +} + +#define HISI_PTT_TUNE_ATTR(_name, _val, _show, _store) \ + static struct hisi_ptt_tune_desc _name##_desc = { \ + .name = #_name, \ + .event_code = (_val), \ + }; \ + static struct dev_ext_attribute hisi_ptt_##_name##_attr = { \ + .attr = __ATTR(_name, 0600, _show, _store), \ + .var = &_name##_desc, \ + } + +#define HISI_PTT_TUNE_ATTR_COMMON(_name, _val) \ + HISI_PTT_TUNE_ATTR(_name, _val, \ + hisi_ptt_tune_attr_show, \ + hisi_ptt_tune_attr_store) + +/* + * The value of the tuning event are composed of two parts: main event code + * in BIT[0,15] and subevent code in BIT[16,23]. For example, qox_tx_cpl is + * a subevent of 'Tx path QoS control' which for tuning the weight of Tx + * completion TLPs. See hisi_ptt.rst documentation for more information. + */ +#define HISI_PTT_TUNE_QOS_TX_CPL (0x4 | (3 << 16)) +#define HISI_PTT_TUNE_QOS_TX_NP (0x4 | (4 << 16)) +#define HISI_PTT_TUNE_QOS_TX_P (0x4 | (5 << 16)) +#define HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL (0x5 | (6 << 16)) +#define HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL (0x5 | (7 << 16)) + +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_cpl, HISI_PTT_TUNE_QOS_TX_CPL); +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_np, HISI_PTT_TUNE_QOS_TX_NP); +HISI_PTT_TUNE_ATTR_COMMON(qos_tx_p, HISI_PTT_TUNE_QOS_TX_P); +HISI_PTT_TUNE_ATTR_COMMON(rx_alloc_buf_level, HISI_PTT_TUNE_RX_ALLOC_BUF_LEVEL); +HISI_PTT_TUNE_ATTR_COMMON(tx_alloc_buf_level, HISI_PTT_TUNE_TX_ALLOC_BUF_LEVEL); + +static struct attribute *hisi_ptt_tune_attrs[] = { + &hisi_ptt_qos_tx_cpl_attr.attr.attr, + &hisi_ptt_qos_tx_np_attr.attr.attr, + &hisi_ptt_qos_tx_p_attr.attr.attr, + &hisi_ptt_rx_alloc_buf_level_attr.attr.attr, + &hisi_ptt_tx_alloc_buf_level_attr.attr.attr, + NULL, +}; + +static struct attribute_group hisi_ptt_tune_group = { + .name = "tune", + .attrs = hisi_ptt_tune_attrs, +}; + +static u16 hisi_ptt_get_filter_val(u16 devid, bool is_port) +{ + if (is_port) + return BIT(HISI_PCIE_CORE_PORT_ID(devid & 0xff)); + + return devid; +} + +static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_STS, + val, val & HISI_PTT_TRACE_IDLE, + HISI_PTT_WAIT_POLL_INTERVAL_US, + HISI_PTT_WAIT_TRACE_TIMEOUT_US); +} + +static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) +{ + u32 val; + + readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, + val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, + HISI_PTT_RESET_TIMEOUT_US); +} + +static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) +{ + writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + hisi_ptt->trace_ctrl.started = false; +} + +static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + u32 val; + int i; + + /* Check device idle before start trace */ + if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) { + pci_err(hisi_ptt->pdev, "Failed to start trace, the device is still busy\n"); + return -EBUSY; + } + + ctrl->started = true; + + /* Reset the DMA before start tracing */ + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val |= HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + hisi_ptt_wait_dma_reset_done(hisi_ptt); + + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val &= ~HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + /* Reset the index of current buffer */ + hisi_ptt->trace_ctrl.buf_index = 0; + + /* Zero the trace buffers */ + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) + memset(ctrl->trace_buf[i].addr, 0, HISI_PTT_TRACE_BUF_SIZE); + + /* Clear the interrupt status */ + writel(HISI_PTT_TRACE_INT_STAT_MASK, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + writel(0, hisi_ptt->iobase + HISI_PTT_TRACE_INT_MASK); + + /* Set the trace control register */ + val = FIELD_PREP(HISI_PTT_TRACE_CTRL_TYPE_SEL, ctrl->type); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_RXTX_SEL, ctrl->direction); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_DATA_FORMAT, ctrl->format); + val |= FIELD_PREP(HISI_PTT_TRACE_CTRL_TARGET_SEL, hisi_ptt->trace_ctrl.filter); + if (!hisi_ptt->trace_ctrl.is_port) + val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; + + /* Start the Trace */ + val |= HISI_PTT_TRACE_CTRL_EN; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + + return 0; +} + +static int hisi_ptt_update_aux(struct hisi_ptt *hisi_ptt, int index, bool stop) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + struct perf_output_handle *handle = &ctrl->handle; + struct perf_event *event = handle->event; + struct hisi_ptt_pmu_buf *buf; + size_t size; + void *addr; + + buf = perf_get_aux(handle); + if (!buf || !handle->size) + return -EINVAL; + + addr = ctrl->trace_buf[ctrl->buf_index].addr; + + /* + * If we're going to stop, read the size of already traced data from + * HISI_PTT_TRACE_WR_STS. Otherwise we're coming from the interrupt, + * the data size is always HISI_PTT_TRACE_BUF_SIZE. + */ + if (stop) { + u32 reg; + + reg = readl(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS); + size = FIELD_GET(HISI_PTT_TRACE_WR_STS_WRITE, reg); + } else { + size = HISI_PTT_TRACE_BUF_SIZE; + } + + memcpy(buf->base + buf->pos, addr, size); + buf->pos += size; + + /* + * Just commit the traced data if we're going to stop. Otherwise if the + * resident AUX buffer cannot contain the data of next trace buffer, + * apply a new one. + */ + if (stop) { + perf_aux_output_end(handle, buf->pos); + } else if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { + perf_aux_output_end(handle, buf->pos); + + buf = perf_aux_output_begin(handle, event); + if (!buf) + return -EINVAL; + + buf->pos = handle->head % buf->length; + if (buf->length - buf->pos < HISI_PTT_TRACE_BUF_SIZE) { + perf_aux_output_end(handle, 0); + return -EINVAL; + } + } + + return 0; +} + +static irqreturn_t hisi_ptt_isr(int irq, void *context) +{ + struct hisi_ptt *hisi_ptt = context; + u32 status, buf_idx; + + status = readl(hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + if (!(status & HISI_PTT_TRACE_INT_STAT_MASK)) + return IRQ_NONE; + + buf_idx = ffs(status) - 1; + + /* Clear the interrupt status of buffer @buf_idx */ + writel(status, hisi_ptt->iobase + HISI_PTT_TRACE_INT_STAT); + + /* + * Update the AUX buffer and cache the current buffer index, + * as we need to know this and save the data when the trace + * is ended out of the interrupt handler. End the trace + * if the updating fails. + */ + if (hisi_ptt_update_aux(hisi_ptt, buf_idx, false)) + hisi_ptt_trace_end(hisi_ptt); + else + hisi_ptt->trace_ctrl.buf_index = (buf_idx + 1) % HISI_PTT_TRACE_BUF_CNT; + + return IRQ_HANDLED; +} + +static void hisi_ptt_irq_free_vectors(void *pdev) +{ + pci_free_irq_vectors(pdev); +} + +static int hisi_ptt_register_irq(struct hisi_ptt *hisi_ptt) +{ + struct pci_dev *pdev = hisi_ptt->pdev; + int ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + pci_err(pdev, "failed to allocate irq vector, ret = %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_irq_free_vectors, pdev); + if (ret < 0) + return ret; + + ret = devm_request_threaded_irq(&pdev->dev, + pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ), + NULL, hisi_ptt_isr, 0, + DRV_NAME, hisi_ptt); + if (ret) { + pci_err(pdev, "failed to request irq %d, ret = %d\n", + pci_irq_vector(pdev, HISI_PTT_TRACE_DMA_IRQ), ret); + return ret; + } + + return 0; +} + +static int hisi_ptt_init_filters(struct pci_dev *pdev, void *data) +{ + struct hisi_ptt_filter_desc *filter; + struct hisi_ptt *hisi_ptt = data; + + /* + * We won't fail the probe if filter allocation failed here. The filters + * should be partial initialized and users would know which filter fails + * through the log. Other functions of PTT device are still available. + */ + filter = kzalloc(sizeof(*filter), GFP_KERNEL); + if (!filter) { + pci_err(hisi_ptt->pdev, "failed to add filter %s\n", pci_name(pdev)); + return -ENOMEM; + } + + filter->devid = PCI_DEVID(pdev->bus->number, pdev->devfn); + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) { + filter->is_port = true; + list_add_tail(&filter->list, &hisi_ptt->port_filters); + + /* Update the available port mask */ + hisi_ptt->port_mask |= hisi_ptt_get_filter_val(filter->devid, true); + } else { + list_add_tail(&filter->list, &hisi_ptt->req_filters); + } + + return 0; +} + +static void hisi_ptt_release_filters(void *data) +{ + struct hisi_ptt_filter_desc *filter, *tmp; + struct hisi_ptt *hisi_ptt = data; + + list_for_each_entry_safe(filter, tmp, &hisi_ptt->req_filters, list) { + list_del(&filter->list); + kfree(filter); + } + + list_for_each_entry_safe(filter, tmp, &hisi_ptt->port_filters, list) { + list_del(&filter->list); + kfree(filter); + } +} + +static int hisi_ptt_config_trace_buf(struct hisi_ptt *hisi_ptt) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + struct device *dev = &hisi_ptt->pdev->dev; + int i; + + ctrl->trace_buf = devm_kcalloc(dev, HISI_PTT_TRACE_BUF_CNT, + sizeof(*ctrl->trace_buf), GFP_KERNEL); + if (!ctrl->trace_buf) + return -ENOMEM; + + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; ++i) { + ctrl->trace_buf[i].addr = dmam_alloc_coherent(dev, HISI_PTT_TRACE_BUF_SIZE, + &ctrl->trace_buf[i].dma, + GFP_KERNEL); + if (!ctrl->trace_buf[i].addr) + return -ENOMEM; + } + + /* Configure the trace DMA buffer */ + for (i = 0; i < HISI_PTT_TRACE_BUF_CNT; i++) { + writel(lower_32_bits(ctrl->trace_buf[i].dma), + hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_LO_0 + + i * HISI_PTT_TRACE_ADDR_STRIDE); + writel(upper_32_bits(ctrl->trace_buf[i].dma), + hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_BASE_HI_0 + + i * HISI_PTT_TRACE_ADDR_STRIDE); + } + writel(HISI_PTT_TRACE_BUF_SIZE, hisi_ptt->iobase + HISI_PTT_TRACE_ADDR_SIZE); + + return 0; +} + +static int hisi_ptt_init_ctrls(struct hisi_ptt *hisi_ptt) +{ + struct pci_dev *pdev = hisi_ptt->pdev; + struct pci_bus *bus; + int ret; + u32 reg; + + INIT_LIST_HEAD(&hisi_ptt->port_filters); + INIT_LIST_HEAD(&hisi_ptt->req_filters); + + ret = hisi_ptt_config_trace_buf(hisi_ptt); + if (ret) + return ret; + + /* + * The device range register provides the information about the root + * ports which the RCiEP can control and trace. The RCiEP and the root + * ports which it supports are on the same PCIe core, with same domain + * number but maybe different bus number. The device range register + * will tell us which root ports we can support, Bit[31:16] indicates + * the upper BDF numbers of the root port, while Bit[15:0] indicates + * the lower. + */ + reg = readl(hisi_ptt->iobase + HISI_PTT_DEVICE_RANGE); + hisi_ptt->upper_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_UPPER, reg); + hisi_ptt->lower_bdf = FIELD_GET(HISI_PTT_DEVICE_RANGE_LOWER, reg); + + bus = pci_find_bus(pci_domain_nr(pdev->bus), PCI_BUS_NUM(hisi_ptt->upper_bdf)); + if (bus) + pci_walk_bus(bus, hisi_ptt_init_filters, hisi_ptt); + + ret = devm_add_action_or_reset(&pdev->dev, hisi_ptt_release_filters, hisi_ptt); + if (ret) + return ret; + + hisi_ptt->trace_ctrl.on_cpu = -1; + return 0; +} + +static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(dev_get_drvdata(dev)); + const cpumask_t *cpumask = cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)); + + return cpumap_print_to_pagebuf(true, buf, cpumask); +} +static DEVICE_ATTR_RO(cpumask); + +static struct attribute *hisi_ptt_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL +}; + +static const struct attribute_group hisi_ptt_cpumask_attr_group = { + .attrs = hisi_ptt_cpumask_attrs, +}; + +/* + * Bit 19 indicates the filter type, 1 for Root Port filter and 0 for Requester + * filter. Bit[15:0] indicates the filter value, for Root Port filter it's + * a bit mask of desired ports and for Requester filter it's the Requester ID + * of the desired PCIe function. Bit[18:16] is reserved for extension. + * + * See hisi_ptt.rst documentation for detailed information. + */ +PMU_FORMAT_ATTR(filter, "config:0-19"); +PMU_FORMAT_ATTR(direction, "config:20-23"); +PMU_FORMAT_ATTR(type, "config:24-31"); +PMU_FORMAT_ATTR(format, "config:32-35"); + +static struct attribute *hisi_ptt_pmu_format_attrs[] = { + &format_attr_filter.attr, + &format_attr_direction.attr, + &format_attr_type.attr, + &format_attr_format.attr, + NULL +}; + +static struct attribute_group hisi_ptt_pmu_format_group = { + .name = "format", + .attrs = hisi_ptt_pmu_format_attrs, +}; + +static const struct attribute_group *hisi_ptt_pmu_groups[] = { + &hisi_ptt_cpumask_attr_group, + &hisi_ptt_pmu_format_group, + &hisi_ptt_tune_group, + NULL +}; + +static int hisi_ptt_trace_valid_direction(u32 val) +{ + /* + * The direction values have different effects according to the data + * format (specified in the parentheses). TLP set A/B means different + * set of TLP types. See hisi_ptt.rst documentation for more details. + */ + static const u32 hisi_ptt_trace_available_direction[] = { + 0, /* inbound(4DW) or reserved(8DW) */ + 1, /* outbound(4DW) */ + 2, /* {in, out}bound(4DW) or inbound(8DW), TLP set A */ + 3, /* {in, out}bound(4DW) or inbound(8DW), TLP set B */ + }; + int i; + + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_direction); i++) { + if (val == hisi_ptt_trace_available_direction[i]) + return 0; + } + + return -EINVAL; +} + +static int hisi_ptt_trace_valid_type(u32 val) +{ + /* Different types can be set simultaneously */ + static const u32 hisi_ptt_trace_available_type[] = { + 1, /* posted_request */ + 2, /* non-posted_request */ + 4, /* completion */ + }; + int i; + + if (!val) + return -EINVAL; + + /* + * Walk the available list and clear the valid bits of + * the config. If there is any resident bit after the + * walk then the config is invalid. + */ + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_available_type); i++) + val &= ~hisi_ptt_trace_available_type[i]; + + if (val) + return -EINVAL; + + return 0; +} + +static int hisi_ptt_trace_valid_format(u32 val) +{ + static const u32 hisi_ptt_trace_availble_format[] = { + 0, /* 4DW */ + 1, /* 8DW */ + }; + int i; + + for (i = 0; i < ARRAY_SIZE(hisi_ptt_trace_availble_format); i++) { + if (val == hisi_ptt_trace_availble_format[i]) + return 0; + } + + return -EINVAL; +} + +static int hisi_ptt_trace_valid_filter(struct hisi_ptt *hisi_ptt, u64 config) +{ + unsigned long val, port_mask = hisi_ptt->port_mask; + struct hisi_ptt_filter_desc *filter; + + hisi_ptt->trace_ctrl.is_port = FIELD_GET(HISI_PTT_PMU_FILTER_IS_PORT, config); + val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, config); + + /* + * Port filters are defined as bit mask. For port filters, check + * the bits in the @val are within the range of hisi_ptt->port_mask + * and whether it's empty or not, otherwise user has specified + * some unsupported root ports. + * + * For Requester ID filters, walk the available filter list to see + * whether we have one matched. + */ + if (!hisi_ptt->trace_ctrl.is_port) { + list_for_each_entry(filter, &hisi_ptt->req_filters, list) { + if (val == hisi_ptt_get_filter_val(filter->devid, filter->is_port)) + return 0; + } + } else if (bitmap_subset(&val, &port_mask, BITS_PER_LONG)) { + return 0; + } + + return -EINVAL; +} + +static void hisi_ptt_pmu_init_configs(struct hisi_ptt *hisi_ptt, struct perf_event *event) +{ + struct hisi_ptt_trace_ctrl *ctrl = &hisi_ptt->trace_ctrl; + u32 val; + + val = FIELD_GET(HISI_PTT_PMU_FILTER_VAL_MASK, event->attr.config); + hisi_ptt->trace_ctrl.filter = val; + + val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); + ctrl->direction = val; + + val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); + ctrl->type = val; + + val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); + ctrl->format = val; +} + +static int hisi_ptt_pmu_event_init(struct perf_event *event) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + int ret; + u32 val; + + if (event->cpu < 0) { + dev_dbg(event->pmu->dev, "Per-task mode not supported\n"); + return -EOPNOTSUPP; + } + + if (event->attr.type != hisi_ptt->hisi_ptt_pmu.type) + return -ENOENT; + + ret = hisi_ptt_trace_valid_filter(hisi_ptt, event->attr.config); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_DIRECTION_MASK, event->attr.config); + ret = hisi_ptt_trace_valid_direction(val); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_TYPE_MASK, event->attr.config); + ret = hisi_ptt_trace_valid_type(val); + if (ret < 0) + return ret; + + val = FIELD_GET(HISI_PTT_PMU_FORMAT_MASK, event->attr.config); + return hisi_ptt_trace_valid_format(val); +} + +static void *hisi_ptt_pmu_setup_aux(struct perf_event *event, void **pages, + int nr_pages, bool overwrite) +{ + struct hisi_ptt_pmu_buf *buf; + struct page **pagelist; + int i; + + if (overwrite) { + dev_warn(event->pmu->dev, "Overwrite mode is not supported\n"); + return NULL; + } + + /* If the pages size less than buffers, we cannot start trace */ + if (nr_pages < HISI_PTT_TRACE_TOTAL_BUF_SIZE / PAGE_SIZE) + return NULL; + + buf = kzalloc(sizeof(*buf), GFP_KERNEL); + if (!buf) + return NULL; + + pagelist = kcalloc(nr_pages, sizeof(*pagelist), GFP_KERNEL); + if (!pagelist) + goto err; + + for (i = 0; i < nr_pages; i++) + pagelist[i] = virt_to_page(pages[i]); + + buf->base = vmap(pagelist, nr_pages, VM_MAP, PAGE_KERNEL); + if (!buf->base) { + kfree(pagelist); + goto err; + } + + buf->nr_pages = nr_pages; + buf->length = nr_pages * PAGE_SIZE; + buf->pos = 0; + + kfree(pagelist); + return buf; +err: + kfree(buf); + return NULL; +} + +static void hisi_ptt_pmu_free_aux(void *aux) +{ + struct hisi_ptt_pmu_buf *buf = aux; + + vunmap(buf->base); + kfree(buf); +} + +static void hisi_ptt_pmu_start(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct perf_output_handle *handle = &hisi_ptt->trace_ctrl.handle; + struct hw_perf_event *hwc = &event->hw; + struct device *dev = event->pmu->dev; + struct hisi_ptt_pmu_buf *buf; + int cpu = event->cpu; + int ret; + + hwc->state = 0; + + /* Serialize the perf process if user specified several CPUs */ + spin_lock(&hisi_ptt->pmu_lock); + if (hisi_ptt->trace_ctrl.started) { + dev_dbg(dev, "trace has already started\n"); + goto stop; + } + + /* + * Handle the interrupt on the same cpu which starts the trace to avoid + * context mismatch. Otherwise we'll trigger the WARN from the perf + * core in event_function_local(). If CPU passed is offline we'll fail + * here, just log it since we can do nothing here. + */ + ret = irq_set_affinity(pci_irq_vector(hisi_ptt->pdev, HISI_PTT_TRACE_DMA_IRQ), + cpumask_of(cpu)); + if (ret) + dev_warn(dev, "failed to set the affinity of trace interrupt\n"); + + hisi_ptt->trace_ctrl.on_cpu = cpu; + + buf = perf_aux_output_begin(handle, event); + if (!buf) { + dev_dbg(dev, "aux output begin failed\n"); + goto stop; + } + + buf->pos = handle->head % buf->length; + + hisi_ptt_pmu_init_configs(hisi_ptt, event); + + ret = hisi_ptt_trace_start(hisi_ptt); + if (ret) { + dev_dbg(dev, "trace start failed, ret = %d\n", ret); + perf_aux_output_end(handle, 0); + goto stop; + } + + spin_unlock(&hisi_ptt->pmu_lock); + return; +stop: + event->hw.state |= PERF_HES_STOPPED; + spin_unlock(&hisi_ptt->pmu_lock); +} + +static void hisi_ptt_pmu_stop(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct hw_perf_event *hwc = &event->hw; + + if (hwc->state & PERF_HES_STOPPED) + return; + + spin_lock(&hisi_ptt->pmu_lock); + if (hisi_ptt->trace_ctrl.started) { + hisi_ptt_trace_end(hisi_ptt); + + if (!hisi_ptt_wait_trace_hw_idle(hisi_ptt)) + dev_warn(event->pmu->dev, "Device is still busy\n"); + + hisi_ptt_update_aux(hisi_ptt, hisi_ptt->trace_ctrl.buf_index, true); + } + spin_unlock(&hisi_ptt->pmu_lock); + + hwc->state |= PERF_HES_STOPPED; + perf_event_update_userpage(event); + hwc->state |= PERF_HES_UPTODATE; +} + +static int hisi_ptt_pmu_add(struct perf_event *event, int flags) +{ + struct hisi_ptt *hisi_ptt = to_hisi_ptt(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int cpu = event->cpu; + + /* Only allow the cpus on the device's node to add the event */ + if (!cpumask_test_cpu(cpu, cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)))) + return 0; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + if (flags & PERF_EF_START) { + hisi_ptt_pmu_start(event, PERF_EF_RELOAD); + if (hwc->state & PERF_HES_STOPPED) + return -EINVAL; + } + + return 0; +} + +static void hisi_ptt_pmu_del(struct perf_event *event, int flags) +{ + hisi_ptt_pmu_stop(event, PERF_EF_UPDATE); +} + +static void hisi_ptt_remove_cpuhp_instance(void *hotplug_node) +{ + cpuhp_state_remove_instance_nocalls(hisi_ptt_pmu_online, hotplug_node); +} + +static void hisi_ptt_unregister_pmu(void *pmu) +{ + perf_pmu_unregister(pmu); +} + +static int hisi_ptt_register_pmu(struct hisi_ptt *hisi_ptt) +{ + u16 core_id, sicl_id; + char *pmu_name; + u32 reg; + int ret; + + ret = cpuhp_state_add_instance_nocalls(hisi_ptt_pmu_online, + &hisi_ptt->hotplug_node); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&hisi_ptt->pdev->dev, + hisi_ptt_remove_cpuhp_instance, + &hisi_ptt->hotplug_node); + if (ret) + return ret; + + mutex_init(&hisi_ptt->tune_lock); + spin_lock_init(&hisi_ptt->pmu_lock); + + hisi_ptt->hisi_ptt_pmu = (struct pmu) { + .module = THIS_MODULE, + .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, + .task_ctx_nr = perf_sw_context, + .attr_groups = hisi_ptt_pmu_groups, + .event_init = hisi_ptt_pmu_event_init, + .setup_aux = hisi_ptt_pmu_setup_aux, + .free_aux = hisi_ptt_pmu_free_aux, + .start = hisi_ptt_pmu_start, + .stop = hisi_ptt_pmu_stop, + .add = hisi_ptt_pmu_add, + .del = hisi_ptt_pmu_del, + }; + + reg = readl(hisi_ptt->iobase + HISI_PTT_LOCATION); + core_id = FIELD_GET(HISI_PTT_CORE_ID, reg); + sicl_id = FIELD_GET(HISI_PTT_SICL_ID, reg); + + pmu_name = devm_kasprintf(&hisi_ptt->pdev->dev, GFP_KERNEL, "hisi_ptt%u_%u", + sicl_id, core_id); + if (!pmu_name) + return -ENOMEM; + + ret = perf_pmu_register(&hisi_ptt->hisi_ptt_pmu, pmu_name, -1); + if (ret) + return ret; + + return devm_add_action_or_reset(&hisi_ptt->pdev->dev, + hisi_ptt_unregister_pmu, + &hisi_ptt->hisi_ptt_pmu); +} + +/* + * The DMA of PTT trace can only use direct mappings due to some + * hardware restriction. Check whether there is no IOMMU or the + * policy of the IOMMU domain is passthrough, otherwise the trace + * cannot work. + * + * The PTT device is supposed to behind an ARM SMMUv3, which + * should have passthrough the device by a quirk. + */ +static int hisi_ptt_check_iommu_mapping(struct pci_dev *pdev) +{ + struct iommu_domain *iommu_domain; + + iommu_domain = iommu_get_domain_for_dev(&pdev->dev); + if (!iommu_domain || iommu_domain->type == IOMMU_DOMAIN_IDENTITY) + return 0; + + return -EOPNOTSUPP; +} + +static int hisi_ptt_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct hisi_ptt *hisi_ptt; + int ret; + + ret = hisi_ptt_check_iommu_mapping(pdev); + if (ret) { + pci_err(pdev, "requires direct DMA mappings\n"); + return ret; + } + + hisi_ptt = devm_kzalloc(&pdev->dev, sizeof(*hisi_ptt), GFP_KERNEL); + if (!hisi_ptt) + return -ENOMEM; + + hisi_ptt->pdev = pdev; + pci_set_drvdata(pdev, hisi_ptt); + + ret = pcim_enable_device(pdev); + if (ret) { + pci_err(pdev, "failed to enable device, ret = %d\n", ret); + return ret; + } + + ret = pcim_iomap_regions(pdev, BIT(2), DRV_NAME); + if (ret) { + pci_err(pdev, "failed to remap io memory, ret = %d\n", ret); + return ret; + } + + hisi_ptt->iobase = pcim_iomap_table(pdev)[2]; + + ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + pci_err(pdev, "failed to set 64 bit dma mask, ret = %d\n", ret); + return ret; + } + + pci_set_master(pdev); + + ret = hisi_ptt_register_irq(hisi_ptt); + if (ret) + return ret; + + ret = hisi_ptt_init_ctrls(hisi_ptt); + if (ret) { + pci_err(pdev, "failed to init controls, ret = %d\n", ret); + return ret; + } + + ret = hisi_ptt_register_pmu(hisi_ptt); + if (ret) { + pci_err(pdev, "failed to register PMU device, ret = %d", ret); + return ret; + } + + return 0; +} + +static const struct pci_device_id hisi_ptt_id_tbl[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa12e) }, + { } +}; +MODULE_DEVICE_TABLE(pci, hisi_ptt_id_tbl); + +static struct pci_driver hisi_ptt_driver = { + .name = DRV_NAME, + .id_table = hisi_ptt_id_tbl, + .probe = hisi_ptt_probe, +}; + +static int hisi_ptt_cpu_teardown(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_ptt *hisi_ptt; + struct device *dev; + int target, src; + + hisi_ptt = hlist_entry_safe(node, struct hisi_ptt, hotplug_node); + src = hisi_ptt->trace_ctrl.on_cpu; + dev = hisi_ptt->hisi_ptt_pmu.dev; + + if (!hisi_ptt->trace_ctrl.started || src != cpu) + return 0; + + target = cpumask_any_but(cpumask_of_node(dev_to_node(&hisi_ptt->pdev->dev)), cpu); + if (target >= nr_cpu_ids) { + dev_err(dev, "no available cpu for perf context migration\n"); + return 0; + } + + perf_pmu_migrate_context(&hisi_ptt->hisi_ptt_pmu, src, target); + + /* + * Also make sure the interrupt bind to the migrated CPU as well. Warn + * the user on failure here. + */ + if (irq_set_affinity(pci_irq_vector(hisi_ptt->pdev, HISI_PTT_TRACE_DMA_IRQ), + cpumask_of(target))) + dev_warn(dev, "failed to set the affinity of trace interrupt\n"); + + hisi_ptt->trace_ctrl.on_cpu = target; + return 0; +} + +static int __init hisi_ptt_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRV_NAME, NULL, + hisi_ptt_cpu_teardown); + if (ret < 0) + return ret; + hisi_ptt_pmu_online = ret; + + ret = pci_register_driver(&hisi_ptt_driver); + if (ret) + cpuhp_remove_multi_state(hisi_ptt_pmu_online); + + return ret; +} +module_init(hisi_ptt_init); + +static void __exit hisi_ptt_exit(void) +{ + pci_unregister_driver(&hisi_ptt_driver); + cpuhp_remove_multi_state(hisi_ptt_pmu_online); +} +module_exit(hisi_ptt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Yicong Yang "); +MODULE_DESCRIPTION("Driver for HiSilicon PCIe tune and trace device"); diff --git a/drivers/hwtracing/ptt/hisi_ptt.h b/drivers/hwtracing/ptt/hisi_ptt.h new file mode 100644 index 0000000000000000000000000000000000000000..5beb1648c93ab7a1aa1f0c27432f4144f06ee049 --- /dev/null +++ b/drivers/hwtracing/ptt/hisi_ptt.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Driver for HiSilicon PCIe tune and trace device + * + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang + */ + +#ifndef _HISI_PTT_H +#define _HISI_PTT_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "hisi_ptt" + +/* + * The definition of the device registers and register fields. + */ +#define HISI_PTT_TUNING_CTRL 0x0000 +#define HISI_PTT_TUNING_CTRL_CODE GENMASK(15, 0) +#define HISI_PTT_TUNING_CTRL_SUB GENMASK(23, 16) +#define HISI_PTT_TUNING_DATA 0x0004 +#define HISI_PTT_TUNING_DATA_VAL_MASK GENMASK(15, 0) +#define HISI_PTT_TRACE_ADDR_SIZE 0x0800 +#define HISI_PTT_TRACE_ADDR_BASE_LO_0 0x0810 +#define HISI_PTT_TRACE_ADDR_BASE_HI_0 0x0814 +#define HISI_PTT_TRACE_ADDR_STRIDE 0x8 +#define HISI_PTT_TRACE_CTRL 0x0850 +#define HISI_PTT_TRACE_CTRL_EN BIT(0) +#define HISI_PTT_TRACE_CTRL_RST BIT(1) +#define HISI_PTT_TRACE_CTRL_RXTX_SEL GENMASK(3, 2) +#define HISI_PTT_TRACE_CTRL_TYPE_SEL GENMASK(7, 4) +#define HISI_PTT_TRACE_CTRL_DATA_FORMAT BIT(14) +#define HISI_PTT_TRACE_CTRL_FILTER_MODE BIT(15) +#define HISI_PTT_TRACE_CTRL_TARGET_SEL GENMASK(31, 16) +#define HISI_PTT_TRACE_INT_STAT 0x0890 +#define HISI_PTT_TRACE_INT_STAT_MASK GENMASK(3, 0) +#define HISI_PTT_TRACE_INT_MASK 0x0894 +#define HISI_PTT_TUNING_INT_STAT 0x0898 +#define HISI_PTT_TUNING_INT_STAT_MASK BIT(0) +#define HISI_PTT_TRACE_WR_STS 0x08a0 +#define HISI_PTT_TRACE_WR_STS_WRITE GENMASK(27, 0) +#define HISI_PTT_TRACE_WR_STS_BUFFER GENMASK(29, 28) +#define HISI_PTT_TRACE_STS 0x08b0 +#define HISI_PTT_TRACE_IDLE BIT(0) +#define HISI_PTT_DEVICE_RANGE 0x0fe0 +#define HISI_PTT_DEVICE_RANGE_UPPER GENMASK(31, 16) +#define HISI_PTT_DEVICE_RANGE_LOWER GENMASK(15, 0) +#define HISI_PTT_LOCATION 0x0fe8 +#define HISI_PTT_CORE_ID GENMASK(15, 0) +#define HISI_PTT_SICL_ID GENMASK(31, 16) + +/* Parameters of PTT trace DMA part. */ +#define HISI_PTT_TRACE_DMA_IRQ 0 +#define HISI_PTT_TRACE_BUF_CNT 4 +#define HISI_PTT_TRACE_BUF_SIZE SZ_4M +#define HISI_PTT_TRACE_TOTAL_BUF_SIZE (HISI_PTT_TRACE_BUF_SIZE * \ + HISI_PTT_TRACE_BUF_CNT) +/* Wait time for hardware DMA to reset */ +#define HISI_PTT_RESET_TIMEOUT_US 10UL +#define HISI_PTT_RESET_POLL_INTERVAL_US 1UL +/* Poll timeout and interval for waiting hardware work to finish */ +#define HISI_PTT_WAIT_TUNE_TIMEOUT_US 1000000UL +#define HISI_PTT_WAIT_TRACE_TIMEOUT_US 100UL +#define HISI_PTT_WAIT_POLL_INTERVAL_US 10UL + +#define HISI_PCIE_CORE_PORT_ID(devfn) ((PCI_SLOT(devfn) & 0x7) << 1) + +/* Definition of the PMU configs */ +#define HISI_PTT_PMU_FILTER_IS_PORT BIT(19) +#define HISI_PTT_PMU_FILTER_VAL_MASK GENMASK(15, 0) +#define HISI_PTT_PMU_DIRECTION_MASK GENMASK(23, 20) +#define HISI_PTT_PMU_TYPE_MASK GENMASK(31, 24) +#define HISI_PTT_PMU_FORMAT_MASK GENMASK(35, 32) + +/** + * struct hisi_ptt_tune_desc - Describe tune event for PTT tune + * @hisi_ptt: PTT device this tune event belongs to + * @name: name of this event + * @event_code: code of the event + */ +struct hisi_ptt_tune_desc { + struct hisi_ptt *hisi_ptt; + const char *name; + u32 event_code; +}; + +/** + * struct hisi_ptt_dma_buffer - Describe a single trace buffer of PTT trace. + * The detail of the data format is described + * in the documentation of PTT device. + * @dma: DMA address of this buffer visible to the device + * @addr: virtual address of this buffer visible to the cpu + */ +struct hisi_ptt_dma_buffer { + dma_addr_t dma; + void *addr; +}; + +/** + * struct hisi_ptt_trace_ctrl - Control and status of PTT trace + * @trace_buf: array of the trace buffers for holding the trace data. + * the length will be HISI_PTT_TRACE_BUF_CNT. + * @handle: perf output handle of current trace session + * @buf_index: the index of current using trace buffer + * @on_cpu: current tracing cpu + * @started: current trace status, true for started + * @is_port: whether we're tracing root port or not + * @direction: direction of the TLP headers to trace + * @filter: filter value for tracing the TLP headers + * @format: format of the TLP headers to trace + * @type: type of the TLP headers to trace + */ +struct hisi_ptt_trace_ctrl { + struct hisi_ptt_dma_buffer *trace_buf; + struct perf_output_handle handle; + u32 buf_index; + int on_cpu; + bool started; + bool is_port; + u32 direction:2; + u32 filter:16; + u32 format:1; + u32 type:4; +}; + +/** + * struct hisi_ptt_filter_desc - Descriptor of the PTT trace filter + * @list: entry of this descriptor in the filter list + * @is_port: the PCI device of the filter is a Root Port or not + * @devid: the PCI device's devid of the filter + */ +struct hisi_ptt_filter_desc { + struct list_head list; + bool is_port; + u16 devid; +}; + +/** + * struct hisi_ptt_pmu_buf - Descriptor of the AUX buffer of PTT trace + * @length: size of the AUX buffer + * @nr_pages: number of pages of the AUX buffer + * @base: start address of AUX buffer + * @pos: position in the AUX buffer to commit traced data + */ +struct hisi_ptt_pmu_buf { + size_t length; + int nr_pages; + void *base; + long pos; +}; + +/** + * struct hisi_ptt - Per PTT device data + * @trace_ctrl: the control information of PTT trace + * @hotplug_node: node for register cpu hotplug event + * @hisi_ptt_pmu: the pum device of trace + * @iobase: base IO address of the device + * @pdev: pci_dev of this PTT device + * @tune_lock: lock to serialize the tune process + * @pmu_lock: lock to serialize the perf process + * @upper_bdf: the upper BDF range of the PCI devices managed by this PTT device + * @lower_bdf: the lower BDF range of the PCI devices managed by this PTT device + * @port_filters: the filter list of root ports + * @req_filters: the filter list of requester ID + * @port_mask: port mask of the managed root ports + */ +struct hisi_ptt { + struct hisi_ptt_trace_ctrl trace_ctrl; + struct hlist_node hotplug_node; + struct pmu hisi_ptt_pmu; + void __iomem *iobase; + struct pci_dev *pdev; + struct mutex tune_lock; + spinlock_t pmu_lock; + u32 upper_bdf; + u32 lower_bdf; + + /* + * The trace TLP headers can either be filtered by certain + * root port, or by the requester ID. Organize the filters + * by @port_filters and @req_filters here. The mask of all + * the valid ports is also cached for doing sanity check + * of user input. + */ + struct list_head port_filters; + struct list_head req_filters; + u16 port_mask; +}; + +#define to_hisi_ptt(pmu) container_of(pmu, struct hisi_ptt, hisi_ptt_pmu) + +#endif /* _HISI_PTT_H */ diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 7284206b278b9e84dc4be0ab81eb5d47d97eaeb0..264e780ae32e1f394b80a490176d9750dbb9f7f3 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -488,8 +488,8 @@ config I2C_BCM_KONA config I2C_BRCMSTB tristate "BRCM Settop/DSL I2C controller" - depends on ARCH_BCM2835 || ARCH_BCM4908 || ARCH_BCMBCA || \ - ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST + depends on ARCH_BCM2835 || ARCH_BCMBCA || ARCH_BRCMSTB || \ + BMIPS_GENERIC || COMPILE_TEST default y help If you say yes to this option, support will be included for the @@ -1267,6 +1267,16 @@ config I2C_PARPORT This support is also available as a module. If so, the module will be called i2c-parport. +config I2C_PCI1XXXX + tristate "PCI1XXXX I2C Host Adapter" + depends on PCI + help + If you say yes to this option, support will be included for + Microchip PCI1XXXX's I2C interface. + + This driver can also be built as a module. If so, the module will + be called i2c-mchp-pci1xxxx. + config I2C_ROBOTFUZZ_OSIF tristate "RobotFuzz Open Source InterFace USB adapter" depends on USB diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index c5cac15f075cebb48c9566ce1a78982be9252a73..e73cdb1d2b5a85bf0973ee0facc92fe4b76704c4 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -133,6 +133,7 @@ obj-$(CONFIG_I2C_DIOLAN_U2C) += i2c-diolan-u2c.o obj-$(CONFIG_I2C_DLN2) += i2c-dln2.o obj-$(CONFIG_I2C_CP2615) += i2c-cp2615.o obj-$(CONFIG_I2C_PARPORT) += i2c-parport.o +obj-$(CONFIG_I2C_PCI1XXXX) += i2c-mchp-pci1xxxx.o obj-$(CONFIG_I2C_ROBOTFUZZ_OSIF) += i2c-robotfuzz-osif.o obj-$(CONFIG_I2C_TAOS_EVM) += i2c-taos-evm.o obj-$(CONFIG_I2C_TINY_USB) += i2c-tiny-usb.o diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index 84b7e6cbc67b0d08c7f783ad19e4d5e608c20e1f..423fe0c8a471e0b29ae455811a12b379b5f16e2c 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -244,14 +244,18 @@ static const struct i2c_adapter_quirks amd_i2c_dev_quirks = { static int i2c_amd_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; int ret; struct amd_i2c_dev *i2c_dev; - struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); struct amd_mp2_dev *mp2_dev; - const char *uid; + u64 uid; - if (!adev) - return -ENODEV; + ret = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &uid); + if (ret) + return dev_err_probe(dev, ret, "missing UID/bus id!\n"); + if (uid >= 2) + return dev_err_probe(dev, -EINVAL, "incorrect UID/bus id \"%llu\"!\n", uid); + dev_dbg(dev, "bus id is %llu\n", uid); /* The ACPI namespace doesn't contain information about which MP2 PCI * device an AMDI0011 ACPI device is related to, so assume that there's @@ -266,6 +270,7 @@ static int i2c_amd_probe(struct platform_device *pdev) if (!i2c_dev) return -ENOMEM; + i2c_dev->common.bus_id = uid; i2c_dev->common.mp2_dev = mp2_dev; i2c_dev->pdev = pdev; platform_set_drvdata(pdev, i2c_dev); @@ -276,20 +281,6 @@ static int i2c_amd_probe(struct platform_device *pdev) i2c_dev->common.resume = &i2c_amd_resume; #endif - uid = adev->pnp.unique_id; - if (!uid) { - dev_err(&pdev->dev, "missing UID/bus id!\n"); - return -EINVAL; - } else if (strcmp(uid, "0") == 0) { - i2c_dev->common.bus_id = 0; - } else if (strcmp(uid, "1") == 0) { - i2c_dev->common.bus_id = 1; - } else { - dev_err(&pdev->dev, "incorrect UID/bus id \"%s\"!\n", uid); - return -EINVAL; - } - dev_dbg(&pdev->dev, "bus id is %u\n", i2c_dev->common.bus_id); - /* Register the adapter */ amd_mp2_pm_runtime_get(mp2_dev); diff --git a/drivers/i2c/busses/i2c-aspeed.c b/drivers/i2c/busses/i2c-aspeed.c index 185dedfebbac9fda9093a5ce547921fc75812400..c64c381b69b7f8ce923b0ada0e219561b7198954 100644 --- a/drivers/i2c/busses/i2c-aspeed.c +++ b/drivers/i2c/busses/i2c-aspeed.c @@ -244,6 +244,7 @@ static u32 aspeed_i2c_slave_irq(struct aspeed_i2c_bus *bus, u32 irq_status) u32 command, irq_handled = 0; struct i2c_client *slave = bus->slave; u8 value; + int ret; if (!slave) return 0; @@ -311,7 +312,13 @@ static u32 aspeed_i2c_slave_irq(struct aspeed_i2c_bus *bus, u32 irq_status) break; case ASPEED_I2C_SLAVE_WRITE_REQUESTED: bus->slave_state = ASPEED_I2C_SLAVE_WRITE_RECEIVED; - i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); + ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); + /* + * Slave ACK's on this address phase already but as the backend driver + * returns an errno, the bus driver should nack the next incoming byte. + */ + if (ret < 0) + writel(ASPEED_I2CD_M_S_RX_CMD_LAST, bus->base + ASPEED_I2C_CMD_REG); break; case ASPEED_I2C_SLAVE_WRITE_RECEIVED: i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED, &value); diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 33f5588a50c0738a049c52239ff406cfec1fc806..fe0cd205502de94658d455b494070df8d76f139c 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -10,10 +10,12 @@ #include #include #include +#include #include #include #include #include +#include /* Register offsets for the I2C device. */ #define CDNS_I2C_CR_OFFSET 0x00 /* Control Register, RW */ @@ -127,6 +129,8 @@ #define CDNS_I2C_TIMEOUT_MAX 0xFF #define CDNS_I2C_BROKEN_HOLD_BIT BIT(0) +#define CDNS_I2C_POLL_US 100000 +#define CDNS_I2C_TIMEOUT_US 500000 #define cdns_i2c_readreg(offset) readl_relaxed(id->membase + offset) #define cdns_i2c_writereg(val, offset) writel_relaxed(val, id->membase + offset) @@ -204,6 +208,7 @@ struct cdns_i2c { struct notifier_block clk_rate_change_nb; u32 quirks; u32 ctrl_reg; + struct i2c_bus_recovery_info rinfo; #if IS_ENABLED(CONFIG_I2C_SLAVE) u16 ctrl_reg_diva_divb; struct i2c_client *slave; @@ -840,8 +845,14 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, #endif /* Check if the bus is free */ - if (cdns_i2c_readreg(CDNS_I2C_SR_OFFSET) & CDNS_I2C_SR_BA) { + + ret = readl_relaxed_poll_timeout(id->membase + CDNS_I2C_SR_OFFSET, + reg, + !(reg & CDNS_I2C_SR_BA), + CDNS_I2C_POLL_US, CDNS_I2C_TIMEOUT_US); + if (ret) { ret = -EAGAIN; + i2c_recover_bus(adap); goto out; } @@ -1250,6 +1261,12 @@ static int cdns_i2c_probe(struct platform_device *pdev) id->quirks = data->quirks; } + id->rinfo.pinctrl = devm_pinctrl_get(&pdev->dev); + if (IS_ERR(id->rinfo.pinctrl)) { + dev_info(&pdev->dev, "can't get pinctrl, bus recovery not supported\n"); + return PTR_ERR(id->rinfo.pinctrl); + } + id->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &r_mem); if (IS_ERR(id->membase)) return PTR_ERR(id->membase); @@ -1266,6 +1283,7 @@ static int cdns_i2c_probe(struct platform_device *pdev) id->adap.retries = 3; /* Default retry value. */ id->adap.algo_data = id; id->adap.dev.parent = &pdev->dev; + id->adap.bus_recovery_info = &id->rinfo; init_completion(&id->xfer_done); snprintf(id->adap.name, sizeof(id->adap.name), "Cadence I2C at %08lx", (unsigned long)r_mem->start); diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c index 471c47db546bf8ddffc9b069d8a58a1793bbed4a..c836cf8841850ac10d167c7c941f06b438c7c07d 100644 --- a/drivers/i2c/busses/i2c-davinci.c +++ b/drivers/i2c/busses/i2c-davinci.c @@ -823,7 +823,7 @@ static int davinci_i2c_probe(struct platform_device *pdev) r = pm_runtime_resume_and_get(dev->dev); if (r < 0) { dev_err(dev->dev, "failed to runtime_get device: %d\n", r); - return r; + goto err_pm; } i2c_davinci_init(dev); @@ -882,6 +882,7 @@ static int davinci_i2c_probe(struct platform_device *pdev) err_unuse_clocks: pm_runtime_dont_use_autosuspend(dev->dev); pm_runtime_put_sync(dev->dev); +err_pm: pm_runtime_disable(dev->dev); return r; diff --git a/drivers/i2c/busses/i2c-designware-amdpsp.c b/drivers/i2c/busses/i2c-designware-amdpsp.c index b624356c945ffff34058e118f12f32c33d7e1ac9..8f36167bce6244fd5dfc74f7612334bead5fe904 100644 --- a/drivers/i2c/busses/i2c-designware-amdpsp.c +++ b/drivers/i2c/busses/i2c-designware-amdpsp.c @@ -6,6 +6,7 @@ #include #include #include +#include #include @@ -15,6 +16,8 @@ #define PSP_MBOX_OFFSET 0x10570 #define PSP_CMD_TIMEOUT_US (500 * USEC_PER_MSEC) +#define PSP_I2C_RESERVATION_TIME_MS 100 + #define PSP_I2C_REQ_BUS_CMD 0x64 #define PSP_I2C_REQ_RETRY_CNT 400 #define PSP_I2C_REQ_RETRY_DELAY_US (25 * USEC_PER_MSEC) @@ -240,6 +243,41 @@ cleanup: return ret; } +static void release_bus(void) +{ + int status; + + if (!psp_i2c_sem_acquired) + return; + + status = psp_send_i2c_req(PSP_I2C_REQ_RELEASE); + if (status) + return; + + dev_dbg(psp_i2c_dev, "PSP semaphore held for %ums\n", + jiffies_to_msecs(jiffies - psp_i2c_sem_acquired)); + + psp_i2c_sem_acquired = 0; +} + +static void psp_release_i2c_bus_deferred(struct work_struct *work) +{ + mutex_lock(&psp_i2c_access_mutex); + + /* + * If there is any pending transaction, cannot release the bus here. + * psp_release_i2c_bus will take care of this later. + */ + if (psp_i2c_access_count) + goto cleanup; + + release_bus(); + +cleanup: + mutex_unlock(&psp_i2c_access_mutex); +} +static DECLARE_DELAYED_WORK(release_queue, psp_release_i2c_bus_deferred); + static int psp_acquire_i2c_bus(void) { int status; @@ -250,21 +288,23 @@ static int psp_acquire_i2c_bus(void) if (psp_i2c_mbox_fail) goto cleanup; + psp_i2c_access_count++; + /* - * Simply increment usage counter and return if PSP semaphore was - * already taken by kernel. + * No need to request bus arbitration once we are inside semaphore + * reservation period. */ - if (psp_i2c_access_count) { - psp_i2c_access_count++; + if (psp_i2c_sem_acquired) goto cleanup; - } status = psp_send_i2c_req(PSP_I2C_REQ_ACQUIRE); if (status) goto cleanup; psp_i2c_sem_acquired = jiffies; - psp_i2c_access_count++; + + schedule_delayed_work(&release_queue, + msecs_to_jiffies(PSP_I2C_RESERVATION_TIME_MS)); /* * In case of errors with PSP arbitrator psp_i2c_mbox_fail variable is @@ -279,8 +319,6 @@ cleanup: static void psp_release_i2c_bus(void) { - int status; - mutex_lock(&psp_i2c_access_mutex); /* Return early if mailbox was malfunctional */ @@ -295,13 +333,12 @@ static void psp_release_i2c_bus(void) if (psp_i2c_access_count) goto cleanup; - /* Send a release command to PSP */ - status = psp_send_i2c_req(PSP_I2C_REQ_RELEASE); - if (status) - goto cleanup; - - dev_dbg(psp_i2c_dev, "PSP semaphore held for %ums\n", - jiffies_to_msecs(jiffies - psp_i2c_sem_acquired)); + /* + * Send a release command to PSP if the semaphore reservation timeout + * elapsed but x86 still owns the controller. + */ + if (!delayed_work_pending(&release_queue)) + release_bus(); cleanup: mutex_unlock(&psp_i2c_access_mutex); diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index 70b80e7109905dc777bfa01f0dd3b70631e4e83f..4d3a3b464ecd8160b659f053e22f4c6bff80e4d9 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -126,8 +126,9 @@ * status codes */ #define STATUS_IDLE 0x0 -#define STATUS_WRITE_IN_PROGRESS 0x1 -#define STATUS_READ_IN_PROGRESS 0x2 +#define STATUS_ACTIVE 0x1 +#define STATUS_WRITE_IN_PROGRESS 0x2 +#define STATUS_READ_IN_PROGRESS 0x4 /* * operation modes @@ -334,12 +335,14 @@ void i2c_dw_disable_int(struct dw_i2c_dev *dev); static inline void __i2c_dw_enable(struct dw_i2c_dev *dev) { + dev->status |= STATUS_ACTIVE; regmap_write(dev->map, DW_IC_ENABLE, 1); } static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) { regmap_write(dev->map, DW_IC_ENABLE, 0); + dev->status &= ~STATUS_ACTIVE; } void __i2c_dw_disable(struct dw_i2c_dev *dev); diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index 44a94b225ed8277e43d73094e8178e1158b43ce0..dc3c5a15a95b9574e168932a9fa398779acdc1e0 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -716,6 +716,19 @@ static int i2c_dw_irq_handler_master(struct dw_i2c_dev *dev) u32 stat; stat = i2c_dw_read_clear_intrbits(dev); + + if (!(dev->status & STATUS_ACTIVE)) { + /* + * Unexpected interrupt in driver point of view. State + * variables are either unset or stale so acknowledge and + * disable interrupts for suppressing further interrupts if + * interrupt really came from this HW (E.g. firmware has left + * the HW active). + */ + regmap_write(dev->map, DW_IC_INTR_MASK, 0); + return 0; + } + if (stat & DW_IC_INTR_TX_ABRT) { dev->cmd_err |= DW_IC_ERR_TX_ABRT; dev->status = STATUS_IDLE; diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 608e61209455643ceee23d88fbdde1c91d77d75f..e499f96506c521ee1212ae87e2d5103e53b2f71e 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -27,7 +27,6 @@ #include "i2c-ccgx-ucsi.h" #define DRIVER_NAME "i2c-designware-pci" -#define AMD_CLK_RATE_HZ 100000 enum dw_pci_ctl_id_t { medfield, @@ -100,11 +99,6 @@ static u32 mfld_get_clk_rate_khz(struct dw_i2c_dev *dev) return 25000; } -static u32 navi_amd_get_clk_rate_khz(struct dw_i2c_dev *dev) -{ - return AMD_CLK_RATE_HZ; -} - static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) { struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev); @@ -126,15 +120,6 @@ static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) return -ENODEV; } -static int navi_amd_setup(struct pci_dev *pdev, struct dw_pci_controller *c) -{ - struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev); - - dev->flags |= MODEL_AMD_NAVI_GPU; - dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; - return 0; -} - static int mrfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) { /* @@ -159,6 +144,20 @@ static u32 ehl_get_clk_rate_khz(struct dw_i2c_dev *dev) return 100000; } +static u32 navi_amd_get_clk_rate_khz(struct dw_i2c_dev *dev) +{ + return 100000; +} + +static int navi_amd_setup(struct pci_dev *pdev, struct dw_pci_controller *c) +{ + struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev); + + dev->flags |= MODEL_AMD_NAVI_GPU; + dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; + return 0; +} + static struct dw_pci_controller dw_pci_controllers[] = { [medfield] = { .bus_num = -1, @@ -243,6 +242,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, int r; struct dw_pci_controller *controller; struct dw_scl_sda_cfg *cfg; + struct i2c_timings *t; if (id->driver_data >= ARRAY_SIZE(dw_pci_controllers)) return dev_err_probe(&pdev->dev, -EINVAL, @@ -263,7 +263,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, return dev_err_probe(&pdev->dev, r, "I/O memory remapping failed\n"); - dev = devm_kzalloc(&pdev->dev, sizeof(struct dw_i2c_dev), GFP_KERNEL); + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); if (!dev) return -ENOMEM; @@ -272,12 +272,14 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, return r; dev->get_clk_rate_khz = controller->get_clk_rate_khz; - dev->timings.bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; dev->base = pcim_iomap_table(pdev)[0]; dev->dev = &pdev->dev; dev->irq = pci_irq_vector(pdev, 0); dev->flags |= controller->flags; + t = &dev->timings; + i2c_parse_fw_timings(&pdev->dev, t, false); + pci_set_drvdata(pdev, dev); if (controller->setup) { @@ -389,6 +391,7 @@ static const struct pci_device_id i2_designware_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x4bbe), elkhartlake }, { PCI_VDEVICE(INTEL, 0x4bbf), elkhartlake }, { PCI_VDEVICE(INTEL, 0x4bc0), elkhartlake }, + /* AMD NAVI */ { PCI_VDEVICE(ATI, 0x7314), navi_amd }, { PCI_VDEVICE(ATI, 0x73a4), navi_amd }, { PCI_VDEVICE(ATI, 0x73e4), navi_amd }, diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index a176296f4fff1ca49379eb12e1ba1b8aea064327..e06509edc5f39f0708ccf5001eee41d962f15db3 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -1838,6 +1838,7 @@ static struct pci_driver i801_driver = { .shutdown = i801_shutdown, .driver = { .pm = &i801_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, }; diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index b51ab3cad2b168b0e3714ec6695d8d3d73e58533..188f2a36d2fd61bb3ffb943f8d879e8bba6bbe52 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -94,7 +94,8 @@ enum lpi2c_imx_pincfg { struct lpi2c_imx_struct { struct i2c_adapter adapter; - struct clk *clk; + int num_clks; + struct clk_bulk_data *clks; void __iomem *base; __u8 *rx_buf; __u8 *tx_buf; @@ -207,7 +208,7 @@ static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx) lpi2c_imx_set_mode(lpi2c_imx); - clk_rate = clk_get_rate(lpi2c_imx->clk); + clk_rate = clk_get_rate(lpi2c_imx->clks[0].clk); if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST) filt = 0; else @@ -561,11 +562,12 @@ static int lpi2c_imx_probe(struct platform_device *pdev) strscpy(lpi2c_imx->adapter.name, pdev->name, sizeof(lpi2c_imx->adapter.name)); - lpi2c_imx->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(lpi2c_imx->clk)) { - dev_err(&pdev->dev, "can't get I2C peripheral clock\n"); - return PTR_ERR(lpi2c_imx->clk); + ret = devm_clk_bulk_get_all(&pdev->dev, &lpi2c_imx->clks); + if (ret < 0) { + dev_err(&pdev->dev, "can't get I2C peripheral clock, ret=%d\n", ret); + return ret; } + lpi2c_imx->num_clks = ret; ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &lpi2c_imx->bitrate); @@ -582,11 +584,9 @@ static int lpi2c_imx_probe(struct platform_device *pdev) i2c_set_adapdata(&lpi2c_imx->adapter, lpi2c_imx); platform_set_drvdata(pdev, lpi2c_imx); - ret = clk_prepare_enable(lpi2c_imx->clk); - if (ret) { - dev_err(&pdev->dev, "clk enable failed %d\n", ret); + ret = clk_bulk_prepare_enable(lpi2c_imx->num_clks, lpi2c_imx->clks); + if (ret) return ret; - } pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT); pm_runtime_use_autosuspend(&pdev->dev); @@ -633,7 +633,7 @@ static int __maybe_unused lpi2c_runtime_suspend(struct device *dev) { struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev); - clk_disable_unprepare(lpi2c_imx->clk); + clk_bulk_disable_unprepare(lpi2c_imx->num_clks, lpi2c_imx->clks); pinctrl_pm_select_sleep_state(dev); return 0; @@ -645,7 +645,7 @@ static int __maybe_unused lpi2c_runtime_resume(struct device *dev) int ret; pinctrl_pm_select_default_state(dev); - ret = clk_prepare_enable(lpi2c_imx->clk); + ret = clk_bulk_prepare_enable(lpi2c_imx->num_clks, lpi2c_imx->clks); if (ret) { dev_err(dev, "failed to enable I2C clock, ret=%d\n", ret); return ret; diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index e47fa34656717811f53909e7cd64964992f1cdd7..3082183bd66a490e51b7cdcd5c57ed79d5b9a405 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1583,7 +1583,7 @@ static int i2c_imx_remove(struct platform_device *pdev) if (i2c_imx->dma) i2c_imx_dma_free(i2c_imx); - if (ret == 0) { + if (ret >= 0) { /* setup chip registers to defaults */ imx_i2c_write_reg(0, i2c_imx, IMX_I2C_IADR); imx_i2c_write_reg(0, i2c_imx, IMX_I2C_IFDR); diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c index 6078fa0c0d488eaea845dc83b621d4c6d87967c5..fe2349590f75e2c0bb87f70420ba08558644519b 100644 --- a/drivers/i2c/busses/i2c-ismt.c +++ b/drivers/i2c/busses/i2c-ismt.c @@ -937,11 +937,8 @@ ismt_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (err) { - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "dma_set_mask fail\n"); - return -ENODEV; - } + dev_err(&pdev->dev, "dma_set_mask fail\n"); + return -ENODEV; } err = ismt_dev_init(priv); diff --git a/drivers/i2c/busses/i2c-mchp-pci1xxxx.c b/drivers/i2c/busses/i2c-mchp-pci1xxxx.c new file mode 100644 index 0000000000000000000000000000000000000000..09af7592114789a924c8bef1cc095df192367151 --- /dev/null +++ b/drivers/i2c/busses/i2c-mchp-pci1xxxx.c @@ -0,0 +1,1210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip PCI1XXXX I2C adapter driver for PCIe Switch + * which has I2C controller in one of its downstream functions + * + * Copyright (C) 2021 - 2022 Microchip Technology Inc. + * + * Authors: Tharun Kumar P + * Kumaravel Thiagarajan + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SMBUS_MAST_CORE_ADDR_BASE 0x00000 +#define SMBUS_MAST_SYS_REG_ADDR_BASE 0x01000 + +/* SMB register space. */ +#define SMB_CORE_CTRL_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x00) + +#define SMB_CORE_CTRL_ESO BIT(6) +#define SMB_CORE_CTRL_FW_ACK BIT(4) +#define SMB_CORE_CTRL_ACK BIT(0) + +#define SMB_CORE_CMD_REG_OFF3 (SMBUS_MAST_CORE_ADDR_BASE + 0x0F) +#define SMB_CORE_CMD_REG_OFF2 (SMBUS_MAST_CORE_ADDR_BASE + 0x0E) +#define SMB_CORE_CMD_REG_OFF1 (SMBUS_MAST_CORE_ADDR_BASE + 0x0D) + +#define SMB_CORE_CMD_READM BIT(4) +#define SMB_CORE_CMD_STOP BIT(2) +#define SMB_CORE_CMD_START BIT(0) + +#define SMB_CORE_CMD_REG_OFF0 (SMBUS_MAST_CORE_ADDR_BASE + 0x0C) + +#define SMB_CORE_CMD_M_PROCEED BIT(1) +#define SMB_CORE_CMD_M_RUN BIT(0) + +#define SMB_CORE_SR_HOLD_TIME_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x18) + +/* + * SR_HOLD_TIME_XK_TICKS field will indicate the number of ticks of the + * baud clock required to program 'Hold Time' at X KHz. + */ +#define SR_HOLD_TIME_100K_TICKS 133 +#define SR_HOLD_TIME_400K_TICKS 20 +#define SR_HOLD_TIME_1000K_TICKS 11 + +#define SMB_CORE_COMPLETION_REG_OFF3 (SMBUS_MAST_CORE_ADDR_BASE + 0x23) + +#define COMPLETION_MDONE BIT(6) +#define COMPLETION_IDLE BIT(5) +#define COMPLETION_MNAKX BIT(0) + +#define SMB_CORE_IDLE_SCALING_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x24) + +/* + * FAIR_BUS_IDLE_MIN_XK_TICKS field will indicate the number of ticks of + * the baud clock required to program 'fair idle delay' at X KHz. Fair idle + * delay establishes the MCTP T(IDLE_DELAY) period. + */ +#define FAIR_BUS_IDLE_MIN_100K_TICKS 969 +#define FAIR_BUS_IDLE_MIN_400K_TICKS 157 +#define FAIR_BUS_IDLE_MIN_1000K_TICKS 157 + +/* + * FAIR_IDLE_DELAY_XK_TICKS field will indicate the number of ticks of the + * baud clock required to satisfy the fairness protocol at X KHz. + */ +#define FAIR_IDLE_DELAY_100K_TICKS 1000 +#define FAIR_IDLE_DELAY_400K_TICKS 500 +#define FAIR_IDLE_DELAY_1000K_TICKS 500 + +#define SMB_IDLE_SCALING_100K \ + ((FAIR_IDLE_DELAY_100K_TICKS << 16) | FAIR_BUS_IDLE_MIN_100K_TICKS) +#define SMB_IDLE_SCALING_400K \ + ((FAIR_IDLE_DELAY_400K_TICKS << 16) | FAIR_BUS_IDLE_MIN_400K_TICKS) +#define SMB_IDLE_SCALING_1000K \ + ((FAIR_IDLE_DELAY_1000K_TICKS << 16) | FAIR_BUS_IDLE_MIN_1000K_TICKS) + +#define SMB_CORE_CONFIG_REG3 (SMBUS_MAST_CORE_ADDR_BASE + 0x2B) + +#define SMB_CONFIG3_ENMI BIT(6) +#define SMB_CONFIG3_ENIDI BIT(5) + +#define SMB_CORE_CONFIG_REG2 (SMBUS_MAST_CORE_ADDR_BASE + 0x2A) +#define SMB_CORE_CONFIG_REG1 (SMBUS_MAST_CORE_ADDR_BASE + 0x29) + +#define SMB_CONFIG1_ASR BIT(7) +#define SMB_CONFIG1_ENAB BIT(2) +#define SMB_CONFIG1_RESET BIT(1) +#define SMB_CONFIG1_FEN BIT(0) + +#define SMB_CORE_BUS_CLK_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x2C) + +/* + * BUS_CLK_XK_LOW_PERIOD_TICKS field defines the number of I2C Baud Clock + * periods that make up the low phase of the I2C/SMBus bus clock at X KHz. + */ +#define BUS_CLK_100K_LOW_PERIOD_TICKS 156 +#define BUS_CLK_400K_LOW_PERIOD_TICKS 41 +#define BUS_CLK_1000K_LOW_PERIOD_TICKS 15 + +/* + * BUS_CLK_XK_HIGH_PERIOD_TICKS field defines the number of I2C Baud Clock + * periods that make up the high phase of the I2C/SMBus bus clock at X KHz. + */ +#define BUS_CLK_100K_HIGH_PERIOD_TICKS 154 +#define BUS_CLK_400K_HIGH_PERIOD_TICKS 35 +#define BUS_CLK_1000K_HIGH_PERIOD_TICKS 14 + +#define BUS_CLK_100K \ + ((BUS_CLK_100K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_100K_LOW_PERIOD_TICKS) +#define BUS_CLK_400K \ + ((BUS_CLK_400K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_400K_LOW_PERIOD_TICKS) +#define BUS_CLK_1000K \ + ((BUS_CLK_1000K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_1000K_LOW_PERIOD_TICKS) + +#define SMB_CORE_CLK_SYNC_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x3C) + +/* + * CLK_SYNC_XK defines the number of clock cycles to sync up to the external + * clock before comparing the internal and external clocks for clock stretching + * at X KHz. + */ +#define CLK_SYNC_100K 4 +#define CLK_SYNC_400K 4 +#define CLK_SYNC_1000K 4 + +#define SMB_CORE_DATA_TIMING_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x40) + +/* + * + * FIRST_START_HOLD_XK_TICKS will indicate the number of ticks of the baud + * clock required to program 'FIRST_START_HOLD' timer at X KHz. This timer + * determines the SCLK hold time following SDAT driven low during the first + * START bit in a transfer. + */ +#define FIRST_START_HOLD_100K_TICKS 22 +#define FIRST_START_HOLD_400K_TICKS 16 +#define FIRST_START_HOLD_1000K_TICKS 6 + +/* + * STOP_SETUP_XK_TICKS will indicate the number of ticks of the baud clock + * required to program 'STOP_SETUP' timer at X KHz. This timer determines the + * SDAT setup time from the rising edge of SCLK for a STOP condition. + */ +#define STOP_SETUP_100K_TICKS 157 +#define STOP_SETUP_400K_TICKS 20 +#define STOP_SETUP_1000K_TICKS 12 + +/* + * RESTART_SETUP_XK_TICKS will indicate the number of ticks of the baud clock + * required to program 'RESTART_SETUP' timer at X KHz. This timer determines the + * SDAT setup time from the rising edge of SCLK for a repeated START condition. + */ +#define RESTART_SETUP_100K_TICKS 157 +#define RESTART_SETUP_400K_TICKS 20 +#define RESTART_SETUP_1000K_TICKS 12 + +/* + * DATA_HOLD_XK_TICKS will indicate the number of ticks of the baud clock + * required to program 'DATA_HOLD' timer at X KHz. This timer determines the + * SDAT hold time following SCLK driven low. + */ +#define DATA_HOLD_100K_TICKS 2 +#define DATA_HOLD_400K_TICKS 2 +#define DATA_HOLD_1000K_TICKS 2 + +#define DATA_TIMING_100K \ + ((FIRST_START_HOLD_100K_TICKS << 24) | (STOP_SETUP_100K_TICKS << 16) | \ + (RESTART_SETUP_100K_TICKS << 8) | DATA_HOLD_100K_TICKS) +#define DATA_TIMING_400K \ + ((FIRST_START_HOLD_400K_TICKS << 24) | (STOP_SETUP_400K_TICKS << 16) | \ + (RESTART_SETUP_400K_TICKS << 8) | DATA_HOLD_400K_TICKS) +#define DATA_TIMING_1000K \ + ((FIRST_START_HOLD_1000K_TICKS << 24) | (STOP_SETUP_1000K_TICKS << 16) | \ + (RESTART_SETUP_1000K_TICKS << 8) | DATA_HOLD_1000K_TICKS) + +#define SMB_CORE_TO_SCALING_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x44) + +/* + * BUS_IDLE_MIN_XK_TICKS defines Bus Idle Minimum Time. + * Bus Idle Minimum time = BUS_IDLE_MIN[7:0] x Baud_Clock_Period x + * (BUS_IDLE_MIN_XK_TICKS[7] ? 4,1) + */ +#define BUS_IDLE_MIN_100K_TICKS 167UL +#define BUS_IDLE_MIN_400K_TICKS 139UL +#define BUS_IDLE_MIN_1000K_TICKS 133UL + +/* + * CTRL_CUM_TIME_OUT_XK_TICKS defines SMBus Controller Cumulative Time-Out. + * SMBus Controller Cumulative Time-Out duration = + * CTRL_CUM_TIME_OUT_XK_TICKS[7:0] x Baud_Clock_Period x 2048 + */ +#define CTRL_CUM_TIME_OUT_100K_TICKS 159 +#define CTRL_CUM_TIME_OUT_400K_TICKS 159 +#define CTRL_CUM_TIME_OUT_1000K_TICKS 159 + +/* + * TARGET_CUM_TIME_OUT_XK_TICKS defines SMBus Target Cumulative Time-Out duration. + * SMBus Target Cumulative Time-Out duration = TARGET_CUM_TIME_OUT_XK_TICKS[7:0] x + * Baud_Clock_Period x 4096 + */ +#define TARGET_CUM_TIME_OUT_100K_TICKS 199 +#define TARGET_CUM_TIME_OUT_400K_TICKS 199 +#define TARGET_CUM_TIME_OUT_1000K_TICKS 199 + +/* + * CLOCK_HIGH_TIME_OUT_XK defines Clock High time out period. + * Clock High time out period = CLOCK_HIGH_TIME_OUT_XK[7:0] x Baud_Clock_Period x 8 + */ +#define CLOCK_HIGH_TIME_OUT_100K_TICKS 204 +#define CLOCK_HIGH_TIME_OUT_400K_TICKS 204 +#define CLOCK_HIGH_TIME_OUT_1000K_TICKS 204 + +#define TO_SCALING_100K \ + ((BUS_IDLE_MIN_100K_TICKS << 24) | (CTRL_CUM_TIME_OUT_100K_TICKS << 16) | \ + (TARGET_CUM_TIME_OUT_100K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_100K_TICKS) +#define TO_SCALING_400K \ + ((BUS_IDLE_MIN_400K_TICKS << 24) | (CTRL_CUM_TIME_OUT_400K_TICKS << 16) | \ + (TARGET_CUM_TIME_OUT_400K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_400K_TICKS) +#define TO_SCALING_1000K \ + ((BUS_IDLE_MIN_1000K_TICKS << 24) | (CTRL_CUM_TIME_OUT_1000K_TICKS << 16) | \ + (TARGET_CUM_TIME_OUT_1000K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_1000K_TICKS) + +#define I2C_SCL_PAD_CTRL_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x100) +#define I2C_SDA_PAD_CTRL_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x101) + +#define I2C_FOD_EN BIT(4) +#define I2C_PULL_UP_EN BIT(3) +#define I2C_PULL_DOWN_EN BIT(2) +#define I2C_INPUT_EN BIT(1) +#define I2C_OUTPUT_EN BIT(0) + +#define SMBUS_CONTROL_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x200) + +#define CTL_RESET_COUNTERS BIT(3) +#define CTL_TRANSFER_DIR BIT(2) +#define CTL_HOST_FIFO_ENTRY BIT(1) +#define CTL_RUN BIT(0) + +#define I2C_DIRN_WRITE 0 +#define I2C_DIRN_READ 1 + +#define SMBUS_STATUS_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x204) + +#define STA_DMA_TERM BIT(7) +#define STA_DMA_REQ BIT(6) +#define STA_THRESHOLD BIT(2) +#define STA_BUF_FULL BIT(1) +#define STA_BUF_EMPTY BIT(0) + +#define SMBUS_INTR_STAT_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x208) + +#define INTR_STAT_DMA_TERM BIT(7) +#define INTR_STAT_THRESHOLD BIT(2) +#define INTR_STAT_BUF_FULL BIT(1) +#define INTR_STAT_BUF_EMPTY BIT(0) + +#define SMBUS_INTR_MSK_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x20C) + +#define INTR_MSK_DMA_TERM BIT(7) +#define INTR_MSK_THRESHOLD BIT(2) +#define INTR_MSK_BUF_FULL BIT(1) +#define INTR_MSK_BUF_EMPTY BIT(0) + +#define ALL_NW_LAYER_INTERRUPTS \ + (INTR_MSK_DMA_TERM | INTR_MSK_THRESHOLD | INTR_MSK_BUF_FULL | \ + INTR_MSK_BUF_EMPTY) + +#define SMBUS_MCU_COUNTER_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x214) + +#define SMBALERT_MST_PAD_CTRL_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x230) + +#define SMBALERT_MST_PU BIT(0) + +#define SMBUS_GEN_INT_STAT_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x23C) + +#define SMBUS_GEN_INT_MASK_REG_OFF (SMBUS_MAST_CORE_ADDR_BASE + 0x240) + +#define SMBALERT_INTR_MASK BIT(10) +#define I2C_BUF_MSTR_INTR_MASK BIT(9) +#define I2C_INTR_MASK BIT(8) +#define SMBALERT_WAKE_INTR_MASK BIT(2) +#define I2C_BUF_MSTR_WAKE_INTR_MASK BIT(1) +#define I2C_WAKE_INTR_MASK BIT(0) + +#define ALL_HIGH_LAYER_INTR \ + (SMBALERT_INTR_MASK | I2C_BUF_MSTR_INTR_MASK | I2C_INTR_MASK | \ + SMBALERT_WAKE_INTR_MASK | I2C_BUF_MSTR_WAKE_INTR_MASK | \ + I2C_WAKE_INTR_MASK) + +#define SMBUS_RESET_REG (SMBUS_MAST_CORE_ADDR_BASE + 0x248) + +#define PERI_SMBUS_D3_RESET_DIS BIT(16) + +#define SMBUS_MST_BUF (SMBUS_MAST_CORE_ADDR_BASE + 0x280) + +#define SMBUS_BUF_MAX_SIZE 0x80 + +#define I2C_FLAGS_DIRECT_MODE BIT(7) +#define I2C_FLAGS_POLLING_MODE BIT(6) +#define I2C_FLAGS_STOP BIT(5) +#define I2C_FLAGS_SMB_BLK_READ BIT(4) + +#define PCI1XXXX_I2C_TIMEOUT_MS 1000 + +/* General Purpose Register. */ +#define SMB_GPR_REG (SMBUS_MAST_CORE_ADDR_BASE + 0x1000 + 0x0c00 + \ + 0x00) + +/* Lock Register. */ +#define SMB_GPR_LOCK_REG (SMBUS_MAST_CORE_ADDR_BASE + 0x1000 + 0x0000 + \ + 0x00A0) + +#define SMBUS_PERI_LOCK BIT(3) + +struct pci1xxxx_i2c { + struct completion i2c_xfer_done; + bool i2c_xfer_in_progress; + struct i2c_adapter adap; + void __iomem *i2c_base; + u32 freq; + u32 flags; +}; + +static int set_sys_lock(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMB_GPR_LOCK_REG; + u8 data; + + writel(SMBUS_PERI_LOCK, p); + data = readl(p); + if (data != SMBUS_PERI_LOCK) + return -EPERM; + + return 0; +} + +static int release_sys_lock(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMB_GPR_LOCK_REG; + u8 data; + + data = readl(p); + if (data != SMBUS_PERI_LOCK) + return 0; + + writel(0, p); + data = readl(p); + if (data & SMBUS_PERI_LOCK) + return -EPERM; + + return 0; +} + +static void pci1xxxx_ack_high_level_intr(struct pci1xxxx_i2c *i2c, u16 intr_msk) +{ + writew(intr_msk, i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF); +} + +static void pci1xxxx_i2c_configure_smbalert_pin(struct pci1xxxx_i2c *i2c, + bool enable) +{ + void __iomem *p = i2c->i2c_base + SMBALERT_MST_PAD_CTRL_REG_OFF; + u8 regval; + + regval = readb(p); + + if (enable) + regval |= SMBALERT_MST_PU; + else + regval &= ~SMBALERT_MST_PU; + + writeb(regval, p); +} + +static void pci1xxxx_i2c_send_start_stop(struct pci1xxxx_i2c *i2c, bool start) +{ + void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1; + u8 regval; + + regval = readb(p); + + if (start) + regval |= SMB_CORE_CMD_START; + else + regval |= SMB_CORE_CMD_STOP; + + writeb(regval, p); +} + +/* + * When accessing the core control reg, we should not do a read modified write + * as they are write '1' to clear bits. Instead we need to write with the + * specific bits that needs to be set. + */ +static void pci1xxxx_i2c_set_clear_FW_ACK(struct pci1xxxx_i2c *i2c, bool set) +{ + u8 regval; + + if (set) + regval = SMB_CORE_CTRL_FW_ACK | SMB_CORE_CTRL_ESO | SMB_CORE_CTRL_ACK; + else + regval = SMB_CORE_CTRL_ESO | SMB_CORE_CTRL_ACK; + + writeb(regval, i2c->i2c_base + SMB_CORE_CTRL_REG_OFF); +} + +static void pci1xxxx_i2c_buffer_write(struct pci1xxxx_i2c *i2c, u8 slaveaddr, + u8 transferlen, unsigned char *buf) +{ + void __iomem *p = i2c->i2c_base + SMBUS_MST_BUF; + + if (slaveaddr) + writeb(slaveaddr, p++); + + if (buf) + memcpy_toio(p, buf, transferlen); +} + +/* + * When accessing the core control reg, we should not do a read modified write + * as there are write '1' to clear bits. Instead we need to write with the + * specific bits that needs to be set. + */ +static void pci1xxxx_i2c_enable_ESO(struct pci1xxxx_i2c *i2c) +{ + writeb(SMB_CORE_CTRL_ESO, i2c->i2c_base + SMB_CORE_CTRL_REG_OFF); +} + +static void pci1xxxx_i2c_reset_counters(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF; + u8 regval; + + regval = readb(p); + regval |= CTL_RESET_COUNTERS; + writeb(regval, p); +} + +static void pci1xxxx_i2c_set_transfer_dir(struct pci1xxxx_i2c *i2c, u8 direction) +{ + void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF; + u8 regval; + + regval = readb(p); + if (direction == I2C_DIRN_WRITE) + regval &= ~CTL_TRANSFER_DIR; + else + regval |= CTL_TRANSFER_DIR; + + writeb(regval, p); +} + +static void pci1xxxx_i2c_set_mcu_count(struct pci1xxxx_i2c *i2c, u8 count) +{ + writeb(count, i2c->i2c_base + SMBUS_MCU_COUNTER_REG_OFF); +} + +static void pci1xxxx_i2c_set_read_count(struct pci1xxxx_i2c *i2c, u8 readcount) +{ + writeb(readcount, i2c->i2c_base + SMB_CORE_CMD_REG_OFF3); +} + +static void pci1xxxx_i2c_set_write_count(struct pci1xxxx_i2c *i2c, u8 writecount) +{ + writeb(writecount, i2c->i2c_base + SMB_CORE_CMD_REG_OFF2); +} + +static void pci1xxxx_i2c_set_DMA_run(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF; + u8 regval; + + regval = readb(p); + regval |= CTL_RUN; + writeb(regval, p); +} + +static void pci1xxxx_i2c_set_mrun_proceed(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF0; + u8 regval; + + regval = readb(p); + regval |= SMB_CORE_CMD_M_RUN; + regval |= SMB_CORE_CMD_M_PROCEED; + writeb(regval, p); +} + +static void pci1xxxx_i2c_start_DMA(struct pci1xxxx_i2c *i2c) +{ + pci1xxxx_i2c_set_DMA_run(i2c); + pci1xxxx_i2c_set_mrun_proceed(i2c); +} + +static void pci1xxxx_i2c_config_asr(struct pci1xxxx_i2c *i2c, bool enable) +{ + void __iomem *p = i2c->i2c_base + SMB_CORE_CONFIG_REG1; + u8 regval; + + regval = readb(p); + if (enable) + regval |= SMB_CONFIG1_ASR; + else + regval &= ~SMB_CONFIG1_ASR; + writeb(regval, p); +} + +static irqreturn_t pci1xxxx_i2c_isr(int irq, void *dev) +{ + struct pci1xxxx_i2c *i2c = dev; + void __iomem *p1 = i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF; + void __iomem *p2 = i2c->i2c_base + SMBUS_INTR_STAT_REG_OFF; + irqreturn_t intr_handled = IRQ_NONE; + u16 reg1; + u8 reg3; + + /* + * Read the SMBus interrupt status register to see if the + * DMA_TERM interrupt has caused this callback. + */ + reg1 = readw(p1); + + if (reg1 & I2C_BUF_MSTR_INTR_MASK) { + reg3 = readb(p2); + if (reg3 & INTR_STAT_DMA_TERM) { + complete(&i2c->i2c_xfer_done); + intr_handled = IRQ_HANDLED; + writeb(INTR_STAT_DMA_TERM, p2); + } + pci1xxxx_ack_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK); + } + + if (reg1 & SMBALERT_INTR_MASK) { + intr_handled = IRQ_HANDLED; + pci1xxxx_ack_high_level_intr(i2c, SMBALERT_INTR_MASK); + } + + return intr_handled; +} + +static void pci1xxxx_i2c_set_count(struct pci1xxxx_i2c *i2c, u8 mcucount, + u8 writecount, u8 readcount) +{ + pci1xxxx_i2c_set_mcu_count(i2c, mcucount); + pci1xxxx_i2c_set_write_count(i2c, writecount); + pci1xxxx_i2c_set_read_count(i2c, readcount); +} + +static void pci1xxxx_i2c_set_readm(struct pci1xxxx_i2c *i2c, bool enable) +{ + void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1; + u8 regval; + + regval = readb(p); + if (enable) + regval |= SMB_CORE_CMD_READM; + else + regval &= ~SMB_CORE_CMD_READM; + + writeb(regval, p); +} + +static void pci1xxxx_ack_nw_layer_intr(struct pci1xxxx_i2c *i2c, u8 ack_intr_msk) +{ + writeb(ack_intr_msk, i2c->i2c_base + SMBUS_INTR_STAT_REG_OFF); +} + +static void pci1xxxx_config_nw_layer_intr(struct pci1xxxx_i2c *i2c, + u8 intr_msk, bool enable) +{ + void __iomem *p = i2c->i2c_base + SMBUS_INTR_MSK_REG_OFF; + u8 regval; + + regval = readb(p); + if (enable) + regval &= ~intr_msk; + else + regval |= intr_msk; + + writeb(regval, p); +} + +static void pci1xxxx_i2c_config_padctrl(struct pci1xxxx_i2c *i2c, bool enable) +{ + void __iomem *p1 = i2c->i2c_base + I2C_SCL_PAD_CTRL_REG_OFF; + void __iomem *p2 = i2c->i2c_base + I2C_SDA_PAD_CTRL_REG_OFF; + u8 regval; + + regval = readb(p1); + if (enable) + regval |= I2C_INPUT_EN | I2C_OUTPUT_EN; + else + regval &= ~(I2C_INPUT_EN | I2C_OUTPUT_EN); + + writeb(regval, p1); + + regval = readb(p2); + if (enable) + regval |= I2C_INPUT_EN | I2C_OUTPUT_EN; + else + regval &= ~(I2C_INPUT_EN | I2C_OUTPUT_EN); + + writeb(regval, p2); +} + +static void pci1xxxx_i2c_set_mode(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF; + u8 regval; + + regval = readb(p); + if (i2c->flags & I2C_FLAGS_DIRECT_MODE) + regval &= ~CTL_HOST_FIFO_ENTRY; + else + regval |= CTL_HOST_FIFO_ENTRY; + + writeb(regval, p); +} + +static void pci1xxxx_i2c_config_high_level_intr(struct pci1xxxx_i2c *i2c, + u16 intr_msk, bool enable) +{ + void __iomem *p = i2c->i2c_base + SMBUS_GEN_INT_MASK_REG_OFF; + u16 regval; + + regval = readw(p); + if (enable) + regval &= ~intr_msk; + else + regval |= intr_msk; + writew(regval, p); +} + +static void pci1xxxx_i2c_configure_core_reg(struct pci1xxxx_i2c *i2c, bool enable) +{ + void __iomem *p1 = i2c->i2c_base + SMB_CORE_CONFIG_REG1; + void __iomem *p3 = i2c->i2c_base + SMB_CORE_CONFIG_REG3; + u8 reg1; + u8 reg3; + + reg1 = readb(p1); + reg3 = readb(p3); + if (enable) { + reg1 |= SMB_CONFIG1_ENAB | SMB_CONFIG1_FEN; + reg3 |= SMB_CONFIG3_ENMI | SMB_CONFIG3_ENIDI; + } else { + reg1 &= ~(SMB_CONFIG1_ENAB | SMB_CONFIG1_FEN); + reg3 &= ~(SMB_CONFIG3_ENMI | SMB_CONFIG3_ENIDI); + } + + writeb(reg1, p1); + writeb(reg3, p3); +} + +static void pci1xxxx_i2c_set_freq(struct pci1xxxx_i2c *i2c) +{ + void __iomem *bp = i2c->i2c_base; + void __iomem *p_idle_scaling = bp + SMB_CORE_IDLE_SCALING_REG_OFF; + void __iomem *p_data_timing = bp + SMB_CORE_DATA_TIMING_REG_OFF; + void __iomem *p_hold_time = bp + SMB_CORE_SR_HOLD_TIME_REG_OFF; + void __iomem *p_to_scaling = bp + SMB_CORE_TO_SCALING_REG_OFF; + void __iomem *p_clk_sync = bp + SMB_CORE_CLK_SYNC_REG_OFF; + void __iomem *p_clk_reg = bp + SMB_CORE_BUS_CLK_REG_OFF; + + switch (i2c->freq) { + case I2C_MAX_STANDARD_MODE_FREQ: + writeb(SR_HOLD_TIME_100K_TICKS, p_hold_time); + writel(SMB_IDLE_SCALING_100K, p_idle_scaling); + writew(BUS_CLK_100K, p_clk_reg); + writel(CLK_SYNC_100K, p_clk_sync); + writel(DATA_TIMING_100K, p_data_timing); + writel(TO_SCALING_100K, p_to_scaling); + break; + + case I2C_MAX_FAST_MODE_PLUS_FREQ: + writeb(SR_HOLD_TIME_1000K_TICKS, p_hold_time); + writel(SMB_IDLE_SCALING_1000K, p_idle_scaling); + writew(BUS_CLK_1000K, p_clk_reg); + writel(CLK_SYNC_1000K, p_clk_sync); + writel(DATA_TIMING_1000K, p_data_timing); + writel(TO_SCALING_1000K, p_to_scaling); + break; + + case I2C_MAX_FAST_MODE_FREQ: + default: + writeb(SR_HOLD_TIME_400K_TICKS, p_hold_time); + writel(SMB_IDLE_SCALING_400K, p_idle_scaling); + writew(BUS_CLK_400K, p_clk_reg); + writel(CLK_SYNC_400K, p_clk_sync); + writel(DATA_TIMING_400K, p_data_timing); + writel(TO_SCALING_400K, p_to_scaling); + break; + } +} + +static void pci1xxxx_i2c_init(struct pci1xxxx_i2c *i2c) +{ + void __iomem *p2 = i2c->i2c_base + SMBUS_STATUS_REG_OFF; + void __iomem *p1 = i2c->i2c_base + SMB_GPR_REG; + u8 regval; + int ret; + + ret = set_sys_lock(i2c); + if (ret == -EPERM) { + /* + * Configure I2C Fast Mode as default frequency if unable + * to acquire sys lock. + */ + regval = 0; + } else { + regval = readl(p1); + release_sys_lock(i2c); + } + + switch (regval) { + case 0: + i2c->freq = I2C_MAX_FAST_MODE_FREQ; + pci1xxxx_i2c_set_freq(i2c); + break; + case 1: + i2c->freq = I2C_MAX_STANDARD_MODE_FREQ; + pci1xxxx_i2c_set_freq(i2c); + break; + case 2: + i2c->freq = I2C_MAX_FAST_MODE_PLUS_FREQ; + pci1xxxx_i2c_set_freq(i2c); + break; + case 3: + default: + break; + } + + pci1xxxx_i2c_config_padctrl(i2c, true); + i2c->flags |= I2C_FLAGS_DIRECT_MODE; + pci1xxxx_i2c_set_mode(i2c); + + /* + * Added as a precaution since BUF_EMPTY in status register + * also trigered an Interrupt. + */ + writeb(STA_BUF_EMPTY, p2); + + /* Configure core I2c control registers. */ + pci1xxxx_i2c_configure_core_reg(i2c, true); + + /* + * Enable pull-up for the SMB alert pin which is just used for + * wakeup right now. + */ + pci1xxxx_i2c_configure_smbalert_pin(i2c, true); +} + +static void pci1xxxx_i2c_clear_flags(struct pci1xxxx_i2c *i2c) +{ + u8 regval; + + /* Reset the internal buffer counters. */ + pci1xxxx_i2c_reset_counters(i2c); + + /* Clear low level interrupts. */ + regval = COMPLETION_MNAKX | COMPLETION_IDLE | COMPLETION_MDONE; + writeb(regval, i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3); + reinit_completion(&i2c->i2c_xfer_done); + pci1xxxx_ack_nw_layer_intr(i2c, ALL_NW_LAYER_INTERRUPTS); + pci1xxxx_ack_high_level_intr(i2c, ALL_HIGH_LAYER_INTR); +} + +static int pci1xxxx_i2c_read(struct pci1xxxx_i2c *i2c, u8 slaveaddr, + unsigned char *buf, u16 total_len) +{ + void __iomem *p2 = i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3; + void __iomem *p1 = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1; + void __iomem *p3 = i2c->i2c_base + SMBUS_MST_BUF; + unsigned long time_left; + u16 remainingbytes; + u8 transferlen; + int retval = 0; + u8 read_count; + u32 regval; + u16 count; + + /* Enable I2C host controller by setting the ESO bit in the CONTROL REG. */ + pci1xxxx_i2c_enable_ESO(i2c); + pci1xxxx_i2c_clear_flags(i2c); + pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, true); + pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, true); + + /* + * The I2C transfer could be more than 128 bytes. Our Core is + * capable of only sending 128 at a time. + * As far as the I2C read is concerned, initailly send the + * read slave address along with the number of bytes to read in + * ReadCount. After sending the slave address the interrupt + * is generated. On seeing the ACK for the slave address, reverse the + * buffer direction and run the DMA to initiate Read from slave. + */ + for (count = 0; count < total_len; count += transferlen) { + + /* + * Before start of any transaction clear the existing + * START/STOP conditions. + */ + writeb(0, p1); + remainingbytes = total_len - count; + transferlen = min_t(u16, remainingbytes, SMBUS_BUF_MAX_SIZE); + + /* + * Send STOP bit for the last chunk in the transaction. + * For I2C read transaction of more than BUF_SIZE, NACK should + * only be sent for the last read. + * Hence a bit FW_ACK is set for all the read chunks except for + * the last chunk. For the last chunk NACK should be sent and + * FW_ACK is cleared Send STOP only when I2C_FLAGS_STOP bit is + * set in the flags and only for the last transaction. + */ + if ((count + transferlen >= total_len) && + (i2c->flags & I2C_FLAGS_STOP)) { + pci1xxxx_i2c_set_clear_FW_ACK(i2c, false); + pci1xxxx_i2c_send_start_stop(i2c, 0); + } else { + pci1xxxx_i2c_set_clear_FW_ACK(i2c, true); + } + + /* Send START bit for the first transaction. */ + if (count == 0) { + pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_WRITE); + pci1xxxx_i2c_send_start_stop(i2c, 1); + + /* Write I2c buffer with just the slave addr. */ + pci1xxxx_i2c_buffer_write(i2c, slaveaddr, 0, NULL); + + /* Set the count. Readcount is the transfer bytes. */ + pci1xxxx_i2c_set_count(i2c, 1, 1, transferlen); + + /* + * Set the Auto_start_read bit so that the HW itself + * will take care of the read phase. + */ + pci1xxxx_i2c_config_asr(i2c, true); + if (i2c->flags & I2C_FLAGS_SMB_BLK_READ) + pci1xxxx_i2c_set_readm(i2c, true); + } else { + pci1xxxx_i2c_set_count(i2c, 0, 0, transferlen); + pci1xxxx_i2c_config_asr(i2c, false); + pci1xxxx_i2c_clear_flags(i2c); + pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_READ); + } + + /* Start the DMA. */ + pci1xxxx_i2c_start_DMA(i2c); + + /* Wait for the DMA_TERM interrupt. */ + time_left = wait_for_completion_timeout(&i2c->i2c_xfer_done, + msecs_to_jiffies(PCI1XXXX_I2C_TIMEOUT_MS)); + if (time_left == 0) { + /* Reset the I2C core to release the bus lock. */ + pci1xxxx_i2c_init(i2c); + retval = -ETIMEDOUT; + goto cleanup; + } + + /* Read the completion reg to know the reason for DMA_TERM. */ + regval = readb(p2); + + /* Slave did not respond. */ + if (regval & COMPLETION_MNAKX) { + writeb(COMPLETION_MNAKX, p2); + retval = -ETIMEDOUT; + goto cleanup; + } + + if (i2c->flags & I2C_FLAGS_SMB_BLK_READ) { + buf[0] = readb(p3); + read_count = buf[0]; + memcpy_fromio(&buf[1], p3 + 1, read_count); + } else { + memcpy_fromio(&buf[count], p3, transferlen); + } + } + +cleanup: + /* Disable all the interrupts. */ + pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, false); + pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, false); + pci1xxxx_i2c_config_asr(i2c, false); + return retval; +} + +static int pci1xxxx_i2c_write(struct pci1xxxx_i2c *i2c, u8 slaveaddr, + unsigned char *buf, u16 total_len) +{ + void __iomem *p2 = i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3; + void __iomem *p1 = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1; + unsigned long time_left; + u16 remainingbytes; + u8 actualwritelen; + u8 transferlen; + int retval = 0; + u32 regval; + u16 count; + + /* Enable I2C host controller by setting the ESO bit in the CONTROL REG. */ + pci1xxxx_i2c_enable_ESO(i2c); + + /* Set the Buffer direction. */ + pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_WRITE); + pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, true); + pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, true); + + /* + * The i2c transfer could be more than 128 bytes. Our Core is + * capable of only sending 128 at a time. + */ + for (count = 0; count < total_len; count += transferlen) { + /* + * Before start of any transaction clear the existing + * START/STOP conditions. + */ + writeb(0, p1); + pci1xxxx_i2c_clear_flags(i2c); + remainingbytes = total_len - count; + + /* If it is the starting of the transaction send START. */ + if (count == 0) { + pci1xxxx_i2c_send_start_stop(i2c, 1); + + /* -1 for the slave address. */ + transferlen = min_t(u16, SMBUS_BUF_MAX_SIZE - 1, + remainingbytes); + pci1xxxx_i2c_buffer_write(i2c, slaveaddr, + transferlen, &buf[count]); + /* + * The actual number of bytes written on the I2C bus + * is including the slave address. + */ + actualwritelen = transferlen + 1; + } else { + transferlen = min_t(u16, SMBUS_BUF_MAX_SIZE, remainingbytes); + pci1xxxx_i2c_buffer_write(i2c, 0, transferlen, &buf[count]); + actualwritelen = transferlen; + } + + pci1xxxx_i2c_set_count(i2c, actualwritelen, actualwritelen, 0); + + /* + * Send STOP only when I2C_FLAGS_STOP bit is set in the flags and + * only for the last transaction. + */ + if (remainingbytes <= transferlen && + (i2c->flags & I2C_FLAGS_STOP)) + pci1xxxx_i2c_send_start_stop(i2c, 0); + + pci1xxxx_i2c_start_DMA(i2c); + + /* + * Wait for the DMA_TERM interrupt. + */ + time_left = wait_for_completion_timeout(&i2c->i2c_xfer_done, + msecs_to_jiffies(PCI1XXXX_I2C_TIMEOUT_MS)); + if (time_left == 0) { + /* Reset the I2C core to release the bus lock. */ + pci1xxxx_i2c_init(i2c); + retval = -ETIMEDOUT; + goto cleanup; + } + + regval = readb(p2); + if (regval & COMPLETION_MNAKX) { + writeb(COMPLETION_MNAKX, p2); + retval = -ETIMEDOUT; + goto cleanup; + } + } +cleanup: + /* Disable all the interrupts. */ + pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, false); + pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, false); + + return retval; +} + +static int pci1xxxx_i2c_xfer(struct i2c_adapter *adap, + struct i2c_msg *msgs, int num) +{ + struct pci1xxxx_i2c *i2c = i2c_get_adapdata(adap); + u8 slaveaddr; + int retval; + u32 i; + + i2c->i2c_xfer_in_progress = true; + for (i = 0; i < num; i++) { + slaveaddr = i2c_8bit_addr_from_msg(&msgs[i]); + + /* + * Send the STOP bit if the transfer is the final one or + * if the I2C_M_STOP flag is set. + */ + if ((i == num - 1) || (msgs[i].flags & I2C_M_STOP)) + i2c->flags |= I2C_FLAGS_STOP; + else + i2c->flags &= ~I2C_FLAGS_STOP; + + if (msgs[i].flags & I2C_M_RECV_LEN) + i2c->flags |= I2C_FLAGS_SMB_BLK_READ; + else + i2c->flags &= ~I2C_FLAGS_SMB_BLK_READ; + + if (msgs[i].flags & I2C_M_RD) + retval = pci1xxxx_i2c_read(i2c, slaveaddr, + msgs[i].buf, msgs[i].len); + else + retval = pci1xxxx_i2c_write(i2c, slaveaddr, + msgs[i].buf, msgs[i].len); + + if (retval < 0) + break; + } + i2c->i2c_xfer_in_progress = false; + + if (retval < 0) + return retval; + + return num; +} + +/* + * List of supported functions by the driver. + */ +static u32 pci1xxxx_i2c_get_funcs(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_PROTOCOL_MANGLING | + I2C_FUNC_SMBUS_BLOCK_PROC_CALL | + I2C_FUNC_SMBUS_BYTE | + I2C_FUNC_SMBUS_BYTE_DATA | + I2C_FUNC_SMBUS_WORD_DATA | + I2C_FUNC_SMBUS_PROC_CALL | + I2C_FUNC_SMBUS_BLOCK_DATA; +} + +static const struct i2c_algorithm pci1xxxx_i2c_algo = { + .master_xfer = pci1xxxx_i2c_xfer, + .functionality = pci1xxxx_i2c_get_funcs, +}; + +static const struct i2c_adapter_quirks pci1xxxx_i2c_quirks = { + .flags = I2C_AQ_NO_ZERO_LEN, +}; + +static const struct i2c_adapter pci1xxxx_i2c_ops = { + .owner = THIS_MODULE, + .name = "PCI1xxxx I2C Adapter", + .algo = &pci1xxxx_i2c_algo, + .quirks = &pci1xxxx_i2c_quirks, +}; + +static int pci1xxxx_i2c_suspend(struct device *dev) +{ + struct pci1xxxx_i2c *i2c = dev_get_drvdata(dev); + void __iomem *p = i2c->i2c_base + SMBUS_RESET_REG; + struct pci_dev *pdev = to_pci_dev(dev); + u32 regval; + + i2c_mark_adapter_suspended(&i2c->adap); + + /* + * If the system is put into 'suspend' state when the I2C transfer is in + * progress, wait until the transfer completes. + */ + while (i2c->i2c_xfer_in_progress) + msleep(20); + + pci1xxxx_i2c_config_high_level_intr(i2c, SMBALERT_WAKE_INTR_MASK, true); + + /* + * Enable the PERST_DIS bit to mask the PERST from resetting the core + * registers. + */ + regval = readl(p); + regval |= PERI_SMBUS_D3_RESET_DIS; + writel(regval, p); + + /* Enable PCI wake in the PMCSR register. */ + device_set_wakeup_enable(dev, true); + pci_wake_from_d3(pdev, true); + + return 0; +} + +static int pci1xxxx_i2c_resume(struct device *dev) +{ + struct pci1xxxx_i2c *i2c = dev_get_drvdata(dev); + void __iomem *p1 = i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF; + void __iomem *p2 = i2c->i2c_base + SMBUS_RESET_REG; + struct pci_dev *pdev = to_pci_dev(dev); + u32 regval; + + regval = readw(p1); + writew(regval, p1); + pci1xxxx_i2c_config_high_level_intr(i2c, SMBALERT_WAKE_INTR_MASK, false); + regval = readl(p2); + regval &= ~PERI_SMBUS_D3_RESET_DIS; + writel(regval, p2); + i2c_mark_adapter_resumed(&i2c->adap); + pci_wake_from_d3(pdev, false); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(pci1xxxx_i2c_pm_ops, pci1xxxx_i2c_suspend, + pci1xxxx_i2c_resume); + +static void pci1xxxx_i2c_shutdown(struct pci1xxxx_i2c *i2c) +{ + pci1xxxx_i2c_config_padctrl(i2c, false); + pci1xxxx_i2c_configure_core_reg(i2c, false); +} + +static int pci1xxxx_i2c_probe_pci(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + struct pci1xxxx_i2c *i2c; + int ret; + + i2c = devm_kzalloc(dev, sizeof(*i2c), GFP_KERNEL); + if (!i2c) + return -ENOMEM; + + pci_set_drvdata(pdev, i2c); + i2c->i2c_xfer_in_progress = false; + + ret = pcim_enable_device(pdev); + if (ret) + return ret; + + pci_set_master(pdev); + + /* + * We are getting the base address of the SMB core. SMB core uses + * BAR0 and size is 32K. + */ + ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (ret < 0) + return ret; + + i2c->i2c_base = pcim_iomap_table(pdev)[0]; + init_completion(&i2c->i2c_xfer_done); + pci1xxxx_i2c_init(i2c); + + ret = devm_add_action(dev, (void (*)(void *))pci1xxxx_i2c_shutdown, i2c); + if (ret) + return ret; + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + return ret; + + ret = devm_request_irq(dev, pci_irq_vector(pdev, 0), pci1xxxx_i2c_isr, + 0, pci_name(pdev), i2c); + if (ret) + return ret; + + i2c->adap = pci1xxxx_i2c_ops; + i2c->adap.dev.parent = dev; + + snprintf(i2c->adap.name, sizeof(i2c->adap.name), + "MCHP PCI1xxxx i2c adapter at %s", pci_name(pdev)); + + i2c_set_adapdata(&i2c->adap, i2c); + + ret = devm_i2c_add_adapter(dev, &i2c->adap); + if (ret) + return dev_err_probe(dev, ret, "i2c add adapter failed\n"); + + return 0; +} + +static const struct pci_device_id pci1xxxx_i2c_pci_id_table[] = { + { PCI_VDEVICE(EFAR, 0xA003) }, + { PCI_VDEVICE(EFAR, 0xA013) }, + { PCI_VDEVICE(EFAR, 0xA023) }, + { PCI_VDEVICE(EFAR, 0xA033) }, + { PCI_VDEVICE(EFAR, 0xA043) }, + { } +}; +MODULE_DEVICE_TABLE(pci, pci1xxxx_i2c_pci_id_table); + +static struct pci_driver pci1xxxx_i2c_pci_driver = { + .name = "i2c-mchp-pci1xxxx", + .id_table = pci1xxxx_i2c_pci_id_table, + .probe = pci1xxxx_i2c_probe_pci, + .driver = { + .pm = pm_sleep_ptr(&pci1xxxx_i2c_pm_ops), + }, +}; +module_pci_driver(pci1xxxx_i2c_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Tharun Kumar P"); +MODULE_AUTHOR("Kumaravel Thiagarajan "); +MODULE_DESCRIPTION("Microchip Technology Inc. pci1xxxx I2C bus driver"); diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c index 8716032f030a016b90cebd6c50951d638d4b343d..e68e775f187e68662722fb007c7e608c230aa7ef 100644 --- a/drivers/i2c/busses/i2c-mlxbf.c +++ b/drivers/i2c/busses/i2c-mlxbf.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -31,8 +32,6 @@ (MLXBF_I2C_FUNC_SMBUS_DEFAULT | MLXBF_I2C_FUNC_SMBUS_BLOCK | \ I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SLAVE) -#define MLXBF_I2C_SMBUS_MAX 3 - /* Shared resources info in BlueField platforms. */ #define MLXBF_I2C_COALESCE_TYU_ADDR 0x02801300 @@ -47,6 +46,9 @@ #define MLXBF_I2C_COREPLL_YU_ADDR 0x02800c30 #define MLXBF_I2C_COREPLL_YU_SIZE 0x00c +#define MLXBF_I2C_COREPLL_RSH_YU_ADDR 0x13409824 +#define MLXBF_I2C_COREPLL_RSH_YU_SIZE 0x00c + #define MLXBF_I2C_SHARED_RES_MAX 3 /* @@ -63,13 +65,14 @@ */ #define MLXBF_I2C_TYU_PLL_OUT_FREQ (400 * 1000 * 1000) /* Reference clock for Bluefield - 156 MHz. */ -#define MLXBF_I2C_PLL_IN_FREQ (156 * 1000 * 1000) +#define MLXBF_I2C_PLL_IN_FREQ 156250000ULL /* Constant used to determine the PLL frequency. */ -#define MLNXBF_I2C_COREPLL_CONST 16384 +#define MLNXBF_I2C_COREPLL_CONST 16384ULL + +#define MLXBF_I2C_FREQUENCY_1GHZ 1000000000ULL /* PLL registers. */ -#define MLXBF_I2C_CORE_PLL_REG0 0x0 #define MLXBF_I2C_CORE_PLL_REG1 0x4 #define MLXBF_I2C_CORE_PLL_REG2 0x8 @@ -129,14 +132,10 @@ /* Slave busy bit reset. */ #define MLXBF_I2C_CAUSE_S_GW_BUSY_FALL BIT(18) -#define MLXBF_I2C_CAUSE_SLAVE_ARBITER_BITS_MASK GENMASK(20, 0) - /* Cause coalesce registers. */ #define MLXBF_I2C_CAUSE_COALESCE_0 0x00 -#define MLXBF_I2C_CAUSE_COALESCE_1 0x04 -#define MLXBF_I2C_CAUSE_COALESCE_2 0x08 -#define MLXBF_I2C_CAUSE_TYU_SLAVE_BIT MLXBF_I2C_SMBUS_MAX +#define MLXBF_I2C_CAUSE_TYU_SLAVE_BIT 3 #define MLXBF_I2C_CAUSE_YU_SLAVE_BIT 1 /* Functional enable register. */ @@ -163,15 +162,6 @@ #define MLXBF_I2C_GPIO_SMBUS_GW_ASSERT_PINS(num, val) \ ((val) | (0x3 << MLXBF_I2C_GPIO_SMBUS_GW_PINS(num))) -/* SMBus timing parameters. */ -#define MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH 0x00 -#define MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE 0x04 -#define MLXBF_I2C_SMBUS_TIMER_THOLD 0x08 -#define MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP 0x0c -#define MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA 0x10 -#define MLXBF_I2C_SMBUS_THIGH_MAX_TBUF 0x14 -#define MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT 0x18 - /* * Defines SMBus operating frequency and core clock frequency. * According to ADB files, default values are compliant to 100KHz SMBus @@ -181,42 +171,46 @@ #define MLXBF_I2C_COREPLL_FREQ MLXBF_I2C_TYU_PLL_OUT_FREQ /* Core PLL TYU configuration. */ -#define MLXBF_I2C_COREPLL_CORE_F_TYU_MASK GENMASK(12, 0) -#define MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK GENMASK(3, 0) -#define MLXBF_I2C_COREPLL_CORE_R_TYU_MASK GENMASK(5, 0) - -#define MLXBF_I2C_COREPLL_CORE_F_TYU_SHIFT 3 -#define MLXBF_I2C_COREPLL_CORE_OD_TYU_SHIFT 16 -#define MLXBF_I2C_COREPLL_CORE_R_TYU_SHIFT 20 +#define MLXBF_I2C_COREPLL_CORE_F_TYU_MASK GENMASK(15, 3) +#define MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK GENMASK(19, 16) +#define MLXBF_I2C_COREPLL_CORE_R_TYU_MASK GENMASK(25, 20) /* Core PLL YU configuration. */ #define MLXBF_I2C_COREPLL_CORE_F_YU_MASK GENMASK(25, 0) #define MLXBF_I2C_COREPLL_CORE_OD_YU_MASK GENMASK(3, 0) -#define MLXBF_I2C_COREPLL_CORE_R_YU_MASK GENMASK(5, 0) +#define MLXBF_I2C_COREPLL_CORE_R_YU_MASK GENMASK(31, 26) -#define MLXBF_I2C_COREPLL_CORE_F_YU_SHIFT 0 -#define MLXBF_I2C_COREPLL_CORE_OD_YU_SHIFT 1 -#define MLXBF_I2C_COREPLL_CORE_R_YU_SHIFT 26 +/* SMBus timing parameters. */ +#define MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH 0x00 +#define MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE 0x04 +#define MLXBF_I2C_SMBUS_TIMER_THOLD 0x08 +#define MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP 0x0c +#define MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA 0x10 +#define MLXBF_I2C_SMBUS_THIGH_MAX_TBUF 0x14 +#define MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT 0x18 -/* Core PLL frequency. */ -static u64 mlxbf_i2c_corepll_frequency; +#define MLXBF_I2C_SHIFT_0 0 +#define MLXBF_I2C_SHIFT_8 8 +#define MLXBF_I2C_SHIFT_16 16 +#define MLXBF_I2C_SHIFT_24 24 + +#define MLXBF_I2C_MASK_8 GENMASK(7, 0) +#define MLXBF_I2C_MASK_16 GENMASK(15, 0) + +#define MLXBF_I2C_MST_ADDR_OFFSET 0x200 /* SMBus Master GW. */ -#define MLXBF_I2C_SMBUS_MASTER_GW 0x200 +#define MLXBF_I2C_SMBUS_MASTER_GW 0x0 /* Number of bytes received and sent. */ -#define MLXBF_I2C_SMBUS_RS_BYTES 0x300 +#define MLXBF_I2C_YU_SMBUS_RS_BYTES 0x100 +#define MLXBF_I2C_RSH_YU_SMBUS_RS_BYTES 0x10c /* Packet error check (PEC) value. */ -#define MLXBF_I2C_SMBUS_MASTER_PEC 0x304 +#define MLXBF_I2C_SMBUS_MASTER_PEC 0x104 /* Status bits (ACK/NACK/FW Timeout). */ -#define MLXBF_I2C_SMBUS_MASTER_STATUS 0x308 +#define MLXBF_I2C_SMBUS_MASTER_STATUS 0x108 /* SMbus Master Finite State Machine. */ -#define MLXBF_I2C_SMBUS_MASTER_FSM 0x310 - -/* - * When enabled, the master will issue a stop condition in case of - * timeout while waiting for FW response. - */ -#define MLXBF_I2C_SMBUS_EN_FW_TIMEOUT 0x31c +#define MLXBF_I2C_YU_SMBUS_MASTER_FSM 0x110 +#define MLXBF_I2C_RSH_YU_SMBUS_MASTER_FSM 0x100 /* SMBus master GW control bits offset in MLXBF_I2C_SMBUS_MASTER_GW[31:3]. */ #define MLXBF_I2C_MASTER_LOCK_BIT BIT(31) /* Lock bit. */ @@ -236,14 +230,14 @@ static u64 mlxbf_i2c_corepll_frequency; #define MLXBF_I2C_MASTER_ENABLE_READ \ (MLXBF_I2C_MASTER_ENABLE | MLXBF_I2C_MASTER_CTL_READ_BIT) -#define MLXBF_I2C_MASTER_SLV_ADDR_SHIFT 12 /* Slave address shift. */ -#define MLXBF_I2C_MASTER_WRITE_SHIFT 21 /* Control write bytes shift. */ -#define MLXBF_I2C_MASTER_SEND_PEC_SHIFT 20 /* Send PEC byte shift. */ -#define MLXBF_I2C_MASTER_PARSE_EXP_SHIFT 11 /* Parse expected bytes shift. */ -#define MLXBF_I2C_MASTER_READ_SHIFT 4 /* Control read bytes shift. */ +#define MLXBF_I2C_MASTER_WRITE_SHIFT 21 /* Control write bytes */ +#define MLXBF_I2C_MASTER_SEND_PEC_SHIFT 20 /* Send PEC byte when set to 1 */ +#define MLXBF_I2C_MASTER_PARSE_EXP_SHIFT 11 /* Control parse expected bytes */ +#define MLXBF_I2C_MASTER_SLV_ADDR_SHIFT 12 /* Slave address */ +#define MLXBF_I2C_MASTER_READ_SHIFT 4 /* Control read bytes */ /* SMBus master GW Data descriptor. */ -#define MLXBF_I2C_MASTER_DATA_DESC_ADDR 0x280 +#define MLXBF_I2C_MASTER_DATA_DESC_ADDR 0x80 #define MLXBF_I2C_MASTER_DATA_DESC_SIZE 0x80 /* Size in bytes. */ /* Maximum bytes to read/write per SMBus transaction. */ @@ -269,19 +263,21 @@ static u64 mlxbf_i2c_corepll_frequency; #define MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK BIT(31) #define MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK BIT(15) +#define MLXBF_I2C_SLV_ADDR_OFFSET 0x400 + /* SMBus slave GW. */ -#define MLXBF_I2C_SMBUS_SLAVE_GW 0x400 +#define MLXBF_I2C_SMBUS_SLAVE_GW 0x0 /* Number of bytes received and sent from/to master. */ -#define MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES 0x500 +#define MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES 0x100 /* Packet error check (PEC) value. */ -#define MLXBF_I2C_SMBUS_SLAVE_PEC 0x504 +#define MLXBF_I2C_SMBUS_SLAVE_PEC 0x104 /* SMBus slave Finite State Machine (FSM). */ -#define MLXBF_I2C_SMBUS_SLAVE_FSM 0x510 +#define MLXBF_I2C_SMBUS_SLAVE_FSM 0x110 /* * Should be set when all raised causes handled, and cleared by HW on * every new cause. */ -#define MLXBF_I2C_SMBUS_SLAVE_READY 0x52c +#define MLXBF_I2C_SMBUS_SLAVE_READY 0x12c /* SMBus slave GW control bits offset in MLXBF_I2C_SMBUS_SLAVE_GW[31:19]. */ #define MLXBF_I2C_SLAVE_BUSY_BIT BIT(30) /* Busy bit. */ @@ -294,23 +290,74 @@ static u64 mlxbf_i2c_corepll_frequency; #define MLXBF_I2C_SLAVE_SEND_PEC_SHIFT 21 /* Send PEC byte shift. */ /* SMBus slave GW Data descriptor. */ -#define MLXBF_I2C_SLAVE_DATA_DESC_ADDR 0x480 +#define MLXBF_I2C_SLAVE_DATA_DESC_ADDR 0x80 #define MLXBF_I2C_SLAVE_DATA_DESC_SIZE 0x80 /* Size in bytes. */ /* SMbus slave configuration registers. */ -#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG 0x514 +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG 0x114 #define MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT 16 -#define MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT 7 +#define MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT BIT(7) #define MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK GENMASK(6, 0) -#define MLXBF_I2C_SLAVE_ADDR_ENABLED(addr) \ - ((addr) & (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT)) - /* * Timeout is given in microsends. Note also that timeout handling is not * exact. */ #define MLXBF_I2C_SMBUS_TIMEOUT (300 * 1000) /* 300ms */ +#define MLXBF_I2C_SMBUS_LOCK_POLL_TIMEOUT (300 * 1000) /* 300ms */ + +/* Polling frequency in microseconds. */ +#define MLXBF_I2C_POLL_FREQ_IN_USEC 200 + +#define MLXBF_I2C_SMBUS_OP_CNT_1 1 +#define MLXBF_I2C_SMBUS_OP_CNT_2 2 +#define MLXBF_I2C_SMBUS_OP_CNT_3 3 +#define MLXBF_I2C_SMBUS_MAX_OP_CNT MLXBF_I2C_SMBUS_OP_CNT_3 + +/* Helper macro to define an I2C resource parameters. */ +#define MLXBF_I2C_RES_PARAMS(addr, size, str) \ + { \ + .start = (addr), \ + .end = (addr) + (size) - 1, \ + .name = (str) \ + } + +enum { + MLXBF_I2C_TIMING_100KHZ = 100000, + MLXBF_I2C_TIMING_400KHZ = 400000, + MLXBF_I2C_TIMING_1000KHZ = 1000000, +}; + +enum { + MLXBF_I2C_F_READ = BIT(0), + MLXBF_I2C_F_WRITE = BIT(1), + MLXBF_I2C_F_NORESTART = BIT(3), + MLXBF_I2C_F_SMBUS_OPERATION = BIT(4), + MLXBF_I2C_F_SMBUS_BLOCK = BIT(5), + MLXBF_I2C_F_SMBUS_PEC = BIT(6), + MLXBF_I2C_F_SMBUS_PROCESS_CALL = BIT(7), +}; + +/* Mellanox BlueField chip type. */ +enum mlxbf_i2c_chip_type { + MLXBF_I2C_CHIP_TYPE_1, /* Mellanox BlueField-1 chip. */ + MLXBF_I2C_CHIP_TYPE_2, /* Mellanox BlueField-2 chip. */ + MLXBF_I2C_CHIP_TYPE_3 /* Mellanox BlueField-3 chip. */ +}; + +/* List of chip resources that are being accessed by the driver. */ +enum { + MLXBF_I2C_SMBUS_RES, + MLXBF_I2C_MST_CAUSE_RES, + MLXBF_I2C_SLV_CAUSE_RES, + MLXBF_I2C_COALESCE_RES, + MLXBF_I2C_SMBUS_TIMER_RES, + MLXBF_I2C_SMBUS_MST_RES, + MLXBF_I2C_SMBUS_SLV_RES, + MLXBF_I2C_COREPLL_RES, + MLXBF_I2C_GPIO_RES, + MLXBF_I2C_END_RES +}; /* Encapsulates timing parameters. */ struct mlxbf_i2c_timings { @@ -331,27 +378,12 @@ struct mlxbf_i2c_timings { u32 timeout; /* Detect clock low timeout. */ }; -enum { - MLXBF_I2C_F_READ = BIT(0), - MLXBF_I2C_F_WRITE = BIT(1), - MLXBF_I2C_F_NORESTART = BIT(3), - MLXBF_I2C_F_SMBUS_OPERATION = BIT(4), - MLXBF_I2C_F_SMBUS_BLOCK = BIT(5), - MLXBF_I2C_F_SMBUS_PEC = BIT(6), - MLXBF_I2C_F_SMBUS_PROCESS_CALL = BIT(7), -}; - struct mlxbf_i2c_smbus_operation { u32 flags; u32 length; /* Buffer length in bytes. */ u8 *buffer; }; -#define MLXBF_I2C_SMBUS_OP_CNT_1 1 -#define MLXBF_I2C_SMBUS_OP_CNT_2 2 -#define MLXBF_I2C_SMBUS_OP_CNT_3 3 -#define MLXBF_I2C_SMBUS_MAX_OP_CNT MLXBF_I2C_SMBUS_OP_CNT_3 - struct mlxbf_i2c_smbus_request { u8 slave; u8 operation_cnt; @@ -365,24 +397,38 @@ struct mlxbf_i2c_resource { u8 type; }; -/* List of chip resources that are being accessed by the driver. */ -enum { - MLXBF_I2C_SMBUS_RES, - MLXBF_I2C_MST_CAUSE_RES, - MLXBF_I2C_SLV_CAUSE_RES, - MLXBF_I2C_COALESCE_RES, - MLXBF_I2C_COREPLL_RES, - MLXBF_I2C_GPIO_RES, - MLXBF_I2C_END_RES, +struct mlxbf_i2c_chip_info { + enum mlxbf_i2c_chip_type type; + /* Chip shared resources that are being used by the I2C controller. */ + struct mlxbf_i2c_resource *shared_res[MLXBF_I2C_SHARED_RES_MAX]; + + /* Callback to calculate the core PLL frequency. */ + u64 (*calculate_freq)(struct mlxbf_i2c_resource *corepll_res); + + /* Registers' address offset */ + u32 smbus_master_rs_bytes_off; + u32 smbus_master_fsm_off; }; -/* Helper macro to define an I2C resource parameters. */ -#define MLXBF_I2C_RES_PARAMS(addr, size, str) \ - { \ - .start = (addr), \ - .end = (addr) + (size) - 1, \ - .name = (str) \ - } +struct mlxbf_i2c_priv { + const struct mlxbf_i2c_chip_info *chip; + struct i2c_adapter adap; + struct mlxbf_i2c_resource *smbus; + struct mlxbf_i2c_resource *timer; + struct mlxbf_i2c_resource *mst; + struct mlxbf_i2c_resource *slv; + struct mlxbf_i2c_resource *mst_cause; + struct mlxbf_i2c_resource *slv_cause; + struct mlxbf_i2c_resource *coalesce; + u64 frequency; /* Core frequency in Hz. */ + int bus; /* Physical bus identifier. */ + int irq; + struct i2c_client *slave[MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT]; + u32 resource_version; +}; + +/* Core PLL frequency. */ +static u64 mlxbf_i2c_corepll_frequency; static struct resource mlxbf_i2c_coalesce_tyu_params = MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COALESCE_TYU_ADDR, @@ -396,6 +442,10 @@ static struct resource mlxbf_i2c_corepll_yu_params = MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_YU_ADDR, MLXBF_I2C_COREPLL_YU_SIZE, "COREPLL_MEM"); +static struct resource mlxbf_i2c_corepll_rsh_yu_params = + MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_RSH_YU_ADDR, + MLXBF_I2C_COREPLL_RSH_YU_SIZE, + "COREPLL_MEM"); static struct resource mlxbf_i2c_gpio_tyu_params = MLXBF_I2C_RES_PARAMS(MLXBF_I2C_GPIO_TYU_ADDR, MLXBF_I2C_GPIO_TYU_SIZE, @@ -405,34 +455,6 @@ static struct mutex mlxbf_i2c_coalesce_lock; static struct mutex mlxbf_i2c_corepll_lock; static struct mutex mlxbf_i2c_gpio_lock; -/* Mellanox BlueField chip type. */ -enum mlxbf_i2c_chip_type { - MLXBF_I2C_CHIP_TYPE_1, /* Mellanox BlueField-1 chip. */ - MLXBF_I2C_CHIP_TYPE_2, /* Mallanox BlueField-2 chip. */ -}; - -struct mlxbf_i2c_chip_info { - enum mlxbf_i2c_chip_type type; - /* Chip shared resources that are being used by the I2C controller. */ - struct mlxbf_i2c_resource *shared_res[MLXBF_I2C_SHARED_RES_MAX]; - - /* Callback to calculate the core PLL frequency. */ - u64 (*calculate_freq)(struct mlxbf_i2c_resource *corepll_res); -}; - -struct mlxbf_i2c_priv { - const struct mlxbf_i2c_chip_info *chip; - struct i2c_adapter adap; - struct mlxbf_i2c_resource *smbus; - struct mlxbf_i2c_resource *mst_cause; - struct mlxbf_i2c_resource *slv_cause; - struct mlxbf_i2c_resource *coalesce; - u64 frequency; /* Core frequency in Hz. */ - int bus; /* Physical bus identifier. */ - int irq; - struct i2c_client *slave; -}; - static struct mlxbf_i2c_resource mlxbf_i2c_coalesce_res[] = { [MLXBF_I2C_CHIP_TYPE_1] = { .params = &mlxbf_i2c_coalesce_tyu_params, @@ -452,6 +474,11 @@ static struct mlxbf_i2c_resource mlxbf_i2c_corepll_res[] = { .params = &mlxbf_i2c_corepll_yu_params, .lock = &mlxbf_i2c_corepll_lock, .type = MLXBF_I2C_COREPLL_RES, + }, + [MLXBF_I2C_CHIP_TYPE_3] = { + .params = &mlxbf_i2c_corepll_rsh_yu_params, + .lock = &mlxbf_i2c_corepll_lock, + .type = MLXBF_I2C_COREPLL_RES, } }; @@ -468,26 +495,13 @@ static u8 mlxbf_i2c_bus_count; static struct mutex mlxbf_i2c_bus_lock; -/* Polling frequency in microseconds. */ -#define MLXBF_I2C_POLL_FREQ_IN_USEC 200 - -#define MLXBF_I2C_SHIFT_0 0 -#define MLXBF_I2C_SHIFT_8 8 -#define MLXBF_I2C_SHIFT_16 16 -#define MLXBF_I2C_SHIFT_24 24 - -#define MLXBF_I2C_MASK_8 GENMASK(7, 0) -#define MLXBF_I2C_MASK_16 GENMASK(15, 0) - -#define MLXBF_I2C_FREQUENCY_1GHZ 1000000000 - /* * Function to poll a set of bits at a specific address; it checks whether * the bits are equal to zero when eq_zero is set to 'true', and not equal * to zero when eq_zero is set to 'false'. * Note that the timeout is given in microseconds. */ -static u32 mlxbf_smbus_poll(void __iomem *io, u32 addr, u32 mask, +static u32 mlxbf_i2c_poll(void __iomem *io, u32 addr, u32 mask, bool eq_zero, u32 timeout) { u32 bits; @@ -509,18 +523,37 @@ static u32 mlxbf_smbus_poll(void __iomem *io, u32 addr, u32 mask, * a transaction. Accordingly, this function polls the Master FSM stop * bit; it returns false when the bit is asserted, true if not. */ -static bool mlxbf_smbus_master_wait_for_idle(struct mlxbf_i2c_priv *priv) +static bool mlxbf_i2c_smbus_master_wait_for_idle(struct mlxbf_i2c_priv *priv) { u32 mask = MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK; - u32 addr = MLXBF_I2C_SMBUS_MASTER_FSM; + u32 addr = priv->chip->smbus_master_fsm_off; u32 timeout = MLXBF_I2C_SMBUS_TIMEOUT; - if (mlxbf_smbus_poll(priv->smbus->io, addr, mask, true, timeout)) + if (mlxbf_i2c_poll(priv->mst->io, addr, mask, true, timeout)) return true; return false; } +/* + * wait for the lock to be released before acquiring it. + */ +static bool mlxbf_i2c_smbus_master_lock(struct mlxbf_i2c_priv *priv) +{ + if (mlxbf_i2c_poll(priv->mst->io, MLXBF_I2C_SMBUS_MASTER_GW, + MLXBF_I2C_MASTER_LOCK_BIT, true, + MLXBF_I2C_SMBUS_LOCK_POLL_TIMEOUT)) + return true; + + return false; +} + +static void mlxbf_i2c_smbus_master_unlock(struct mlxbf_i2c_priv *priv) +{ + /* Clear the gw to clear the lock */ + writel(0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_GW); +} + static bool mlxbf_i2c_smbus_transaction_success(u32 master_status, u32 cause_status) { @@ -558,7 +591,7 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv) * then read the cause and master status bits to determine if * errors occurred during the transaction. */ - mlxbf_smbus_poll(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_GW, + mlxbf_i2c_poll(priv->mst->io, MLXBF_I2C_SMBUS_MASTER_GW, MLXBF_I2C_MASTER_BUSY_BIT, true, MLXBF_I2C_SMBUS_TIMEOUT); @@ -571,7 +604,7 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv) * Parse both Cause and Master GW bits, then return transaction status. */ - master_status_bits = readl(priv->smbus->io + + master_status_bits = readl(priv->mst->io + MLXBF_I2C_SMBUS_MASTER_STATUS); master_status_bits &= MLXBF_I2C_SMBUS_MASTER_STATUS_MASK; @@ -596,7 +629,8 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv) } static void mlxbf_i2c_smbus_write_data(struct mlxbf_i2c_priv *priv, - const u8 *data, u8 length, u32 addr) + const u8 *data, u8 length, u32 addr, + bool is_master) { u8 offset, aligned_length; u32 data32; @@ -613,12 +647,16 @@ static void mlxbf_i2c_smbus_write_data(struct mlxbf_i2c_priv *priv, */ for (offset = 0; offset < aligned_length; offset += sizeof(u32)) { data32 = *((u32 *)(data + offset)); - iowrite32be(data32, priv->smbus->io + addr + offset); + if (is_master) + iowrite32be(data32, priv->mst->io + addr + offset); + else + iowrite32be(data32, priv->slv->io + addr + offset); } } static void mlxbf_i2c_smbus_read_data(struct mlxbf_i2c_priv *priv, - u8 *data, u8 length, u32 addr) + u8 *data, u8 length, u32 addr, + bool is_master) { u32 data32, mask; u8 byte, offset; @@ -634,14 +672,20 @@ static void mlxbf_i2c_smbus_read_data(struct mlxbf_i2c_priv *priv, */ for (offset = 0; offset < (length & ~mask); offset += sizeof(u32)) { - data32 = ioread32be(priv->smbus->io + addr + offset); + if (is_master) + data32 = ioread32be(priv->mst->io + addr + offset); + else + data32 = ioread32be(priv->slv->io + addr + offset); *((u32 *)(data + offset)) = data32; } if (!(length & mask)) return; - data32 = ioread32be(priv->smbus->io + addr + offset); + if (is_master) + data32 = ioread32be(priv->mst->io + addr + offset); + else + data32 = ioread32be(priv->slv->io + addr + offset); for (byte = 0; byte < (length & mask); byte++) { data[offset + byte] = data32 & GENMASK(7, 0); @@ -667,16 +711,16 @@ static int mlxbf_i2c_smbus_enable(struct mlxbf_i2c_priv *priv, u8 slave, command |= rol32(pec_en, MLXBF_I2C_MASTER_SEND_PEC_SHIFT); /* Clear status bits. */ - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_STATUS); + writel(0x0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_STATUS); /* Set the cause data. */ - writel(~0x0, priv->smbus->io + MLXBF_I2C_CAUSE_OR_CLEAR); + writel(~0x0, priv->mst_cause->io + MLXBF_I2C_CAUSE_OR_CLEAR); /* Zero PEC byte. */ - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_PEC); + writel(0x0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_PEC); /* Zero byte count. */ - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_RS_BYTES); + writel(0x0, priv->mst->io + priv->chip->smbus_master_rs_bytes_off); /* GW activation. */ - writel(command, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_GW); + writel(command, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_GW); /* * Poll master status and check status bits. An ACK is sent when @@ -712,10 +756,19 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv, slave = request->slave & GENMASK(6, 0); addr = slave << 1; - /* First of all, check whether the HW is idle. */ - if (WARN_ON(!mlxbf_smbus_master_wait_for_idle(priv))) + /* + * Try to acquire the smbus gw lock before any reads of the GW register since + * a read sets the lock. + */ + if (WARN_ON(!mlxbf_i2c_smbus_master_lock(priv))) return -EBUSY; + /* Check whether the HW is idle */ + if (WARN_ON(!mlxbf_i2c_smbus_master_wait_for_idle(priv))) { + ret = -EBUSY; + goto out_unlock; + } + /* Set first byte. */ data_desc[data_idx++] = addr; @@ -738,6 +791,11 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv, if (flags & MLXBF_I2C_F_WRITE) { write_en = 1; write_len += operation->length; + if (data_idx + operation->length > + MLXBF_I2C_MASTER_DATA_DESC_SIZE) { + ret = -ENOBUFS; + goto out_unlock; + } memcpy(data_desc + data_idx, operation->buffer, operation->length); data_idx += operation->length; @@ -763,25 +821,25 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv, * must be written to the data registers. */ mlxbf_i2c_smbus_write_data(priv, (const u8 *)data_desc, data_len, - MLXBF_I2C_MASTER_DATA_DESC_ADDR); + MLXBF_I2C_MASTER_DATA_DESC_ADDR, true); if (write_en) { ret = mlxbf_i2c_smbus_enable(priv, slave, write_len, block_en, pec_en, 0); if (ret) - return ret; + goto out_unlock; } if (read_en) { /* Write slave address to Master GW data descriptor. */ mlxbf_i2c_smbus_write_data(priv, (const u8 *)&addr, 1, - MLXBF_I2C_MASTER_DATA_DESC_ADDR); + MLXBF_I2C_MASTER_DATA_DESC_ADDR, true); ret = mlxbf_i2c_smbus_enable(priv, slave, read_len, block_en, pec_en, 1); if (!ret) { /* Get Master GW data descriptor. */ mlxbf_i2c_smbus_read_data(priv, data_desc, read_len + 1, - MLXBF_I2C_MASTER_DATA_DESC_ADDR); + MLXBF_I2C_MASTER_DATA_DESC_ADDR, true); /* Get data from Master GW data descriptor. */ memcpy(read_buf, data_desc, read_len + 1); @@ -793,9 +851,12 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv, * next tag integration. */ writel(MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK, - priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_FSM); + priv->mst->io + priv->chip->smbus_master_fsm_off); } +out_unlock: + mlxbf_i2c_smbus_master_unlock(priv); + return ret; } @@ -1082,7 +1143,7 @@ static void mlxbf_i2c_set_timings(struct mlxbf_i2c_priv *priv, timer |= mlxbf_i2c_set_timer(priv, timings->scl_low, false, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); - writel(timer, priv->smbus->io + + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH); timer = mlxbf_i2c_set_timer(priv, timings->sda_rise, false, @@ -1093,34 +1154,34 @@ static void mlxbf_i2c_set_timings(struct mlxbf_i2c_priv *priv, MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_16); timer |= mlxbf_i2c_set_timer(priv, timings->scl_fall, false, MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_24); - writel(timer, priv->smbus->io + + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE); timer = mlxbf_i2c_set_timer(priv, timings->hold_start, true, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); timer |= mlxbf_i2c_set_timer(priv, timings->hold_data, true, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); - writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_TIMER_THOLD); + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_THOLD); timer = mlxbf_i2c_set_timer(priv, timings->setup_start, true, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); timer |= mlxbf_i2c_set_timer(priv, timings->setup_stop, true, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); - writel(timer, priv->smbus->io + + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP); timer = mlxbf_i2c_set_timer(priv, timings->setup_data, true, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); - writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA); + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA); timer = mlxbf_i2c_set_timer(priv, timings->buf, false, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0); timer |= mlxbf_i2c_set_timer(priv, timings->thigh_max, false, MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16); - writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_THIGH_MAX_TBUF); + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_THIGH_MAX_TBUF); timer = timings->timeout; - writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT); + writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT); } enum mlxbf_i2c_timings_config { @@ -1407,24 +1468,19 @@ static int mlxbf_i2c_init_master(struct platform_device *pdev, return 0; } -static u64 mlxbf_calculate_freq_from_tyu(struct mlxbf_i2c_resource *corepll_res) +static u64 mlxbf_i2c_calculate_freq_from_tyu(struct mlxbf_i2c_resource *corepll_res) { - u64 core_frequency, pad_frequency; + u64 core_frequency; u8 core_od, core_r; u32 corepll_val; u16 core_f; - pad_frequency = MLXBF_I2C_PLL_IN_FREQ; - corepll_val = readl(corepll_res->io + MLXBF_I2C_CORE_PLL_REG1); /* Get Core PLL configuration bits. */ - core_f = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_F_TYU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_F_TYU_MASK; - core_od = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_OD_TYU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK; - core_r = rol32(corepll_val, MLXBF_I2C_COREPLL_CORE_R_TYU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_R_TYU_MASK; + core_f = FIELD_GET(MLXBF_I2C_COREPLL_CORE_F_TYU_MASK, corepll_val); + core_od = FIELD_GET(MLXBF_I2C_COREPLL_CORE_OD_TYU_MASK, corepll_val); + core_r = FIELD_GET(MLXBF_I2C_COREPLL_CORE_R_TYU_MASK, corepll_val); /* * Compute PLL output frequency as follow: @@ -1436,31 +1492,26 @@ static u64 mlxbf_calculate_freq_from_tyu(struct mlxbf_i2c_resource *corepll_res) * Where PLL_OUT_FREQ and PLL_IN_FREQ refer to CoreFrequency * and PadFrequency, respectively. */ - core_frequency = pad_frequency * (++core_f); + core_frequency = MLXBF_I2C_PLL_IN_FREQ * (++core_f); core_frequency /= (++core_r) * (++core_od); return core_frequency; } -static u64 mlxbf_calculate_freq_from_yu(struct mlxbf_i2c_resource *corepll_res) +static u64 mlxbf_i2c_calculate_freq_from_yu(struct mlxbf_i2c_resource *corepll_res) { u32 corepll_reg1_val, corepll_reg2_val; - u64 corepll_frequency, pad_frequency; + u64 corepll_frequency; u8 core_od, core_r; u32 core_f; - pad_frequency = MLXBF_I2C_PLL_IN_FREQ; - corepll_reg1_val = readl(corepll_res->io + MLXBF_I2C_CORE_PLL_REG1); corepll_reg2_val = readl(corepll_res->io + MLXBF_I2C_CORE_PLL_REG2); /* Get Core PLL configuration bits */ - core_f = rol32(corepll_reg1_val, MLXBF_I2C_COREPLL_CORE_F_YU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_F_YU_MASK; - core_r = rol32(corepll_reg1_val, MLXBF_I2C_COREPLL_CORE_R_YU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_R_YU_MASK; - core_od = rol32(corepll_reg2_val, MLXBF_I2C_COREPLL_CORE_OD_YU_SHIFT) & - MLXBF_I2C_COREPLL_CORE_OD_YU_MASK; + core_f = FIELD_GET(MLXBF_I2C_COREPLL_CORE_F_YU_MASK, corepll_reg1_val); + core_r = FIELD_GET(MLXBF_I2C_COREPLL_CORE_R_YU_MASK, corepll_reg1_val); + core_od = FIELD_GET(MLXBF_I2C_COREPLL_CORE_OD_YU_MASK, corepll_reg2_val); /* * Compute PLL output frequency as follow: @@ -1472,7 +1523,7 @@ static u64 mlxbf_calculate_freq_from_yu(struct mlxbf_i2c_resource *corepll_res) * Where PLL_OUT_FREQ and PLL_IN_FREQ refer to CoreFrequency * and PadFrequency, respectively. */ - corepll_frequency = (pad_frequency * core_f) / MLNXBF_I2C_COREPLL_CONST; + corepll_frequency = (MLXBF_I2C_PLL_IN_FREQ * core_f) / MLNXBF_I2C_COREPLL_CONST; corepll_frequency /= (++core_r) * (++core_od); return corepll_frequency; @@ -1523,28 +1574,26 @@ static int mlxbf_i2c_calculate_corepll_freq(struct platform_device *pdev, return 0; } -static int mlxbf_slave_enable(struct mlxbf_i2c_priv *priv, u8 addr) +static int mlxbf_i2c_slave_enable(struct mlxbf_i2c_priv *priv, + struct i2c_client *slave) { - u32 slave_reg, slave_reg_tmp, slave_reg_avail, slave_addr_mask; - u8 reg, reg_cnt, byte, addr_tmp, reg_avail, byte_avail; - bool avail, disabled; - - disabled = false; - avail = false; + u8 reg, reg_cnt, byte, addr_tmp; + u32 slave_reg, slave_reg_tmp; if (!priv) return -EPERM; reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2; - slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK; /* * Read the slave registers. There are 4 * 32-bit slave registers. - * Each slave register can hold up to 4 * 8-bit slave configuration - * (7-bit address, 1 status bit (1 if enabled, 0 if not)). + * Each slave register can hold up to 4 * 8-bit slave configuration: + * 1) A 7-bit address + * 2) And a status bit (1 if enabled, 0 if not). + * Look for the next available slave register slot. */ for (reg = 0; reg < reg_cnt; reg++) { - slave_reg = readl(priv->smbus->io + + slave_reg = readl(priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4); /* * Each register holds 4 slave addresses. So, we have to keep @@ -1556,121 +1605,87 @@ static int mlxbf_slave_enable(struct mlxbf_i2c_priv *priv, u8 addr) addr_tmp = slave_reg_tmp & GENMASK(7, 0); /* - * Mark the first available slave address slot, i.e. its - * enabled bit should be unset. This slot might be used - * later on to register our slave. - */ - if (!avail && !MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp)) { - avail = true; - reg_avail = reg; - byte_avail = byte; - slave_reg_avail = slave_reg; - } - - /* - * Parse slave address bytes and check whether the - * slave address already exists and it's enabled, - * i.e. most significant bit is set. + * If an enable bit is not set in the + * MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG register, then the + * slave address slot associated with that bit is + * free. So set the enable bit and write the + * slave address bits. */ - if ((addr_tmp & slave_addr_mask) == addr) { - if (MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp)) - return 0; - disabled = true; - break; + if (!(addr_tmp & MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT)) { + slave_reg &= ~(MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK << (byte * 8)); + slave_reg |= (slave->addr << (byte * 8)); + slave_reg |= MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT << (byte * 8); + writel(slave_reg, priv->slv->io + + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + + (reg * 0x4)); + + /* + * Set the slave at the corresponding index. + */ + priv->slave[(reg * 4) + byte] = slave; + + return 0; } /* Parse next byte. */ slave_reg_tmp >>= 8; } - - /* Exit the loop if the slave address is found. */ - if (disabled) - break; } - if (!avail && !disabled) - return -EINVAL; /* No room for a new slave address. */ - - if (avail && !disabled) { - reg = reg_avail; - byte = byte_avail; - /* Set the slave address. */ - slave_reg_avail &= ~(slave_addr_mask << (byte * 8)); - slave_reg_avail |= addr << (byte * 8); - slave_reg = slave_reg_avail; - } - - /* Enable the slave address and update the register. */ - slave_reg |= (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT) << (byte * 8); - writel(slave_reg, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + - reg * 0x4); - - return 0; + return -EBUSY; } -static int mlxbf_slave_disable(struct mlxbf_i2c_priv *priv) +static int mlxbf_i2c_slave_disable(struct mlxbf_i2c_priv *priv, u8 addr) { - u32 slave_reg, slave_reg_tmp, slave_addr_mask; - u8 addr, addr_tmp, reg, reg_cnt, slave_byte; - struct i2c_client *client = priv->slave; - bool exist; + u8 addr_tmp, reg, reg_cnt, byte; + u32 slave_reg, slave_reg_tmp; - exist = false; - - addr = client->addr; reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2; - slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK; /* * Read the slave registers. There are 4 * 32-bit slave registers. - * Each slave register can hold up to 4 * 8-bit slave configuration - * (7-bit address, 1 status bit (1 if enabled, 0 if not)). + * Each slave register can hold up to 4 * 8-bit slave configuration: + * 1) A 7-bit address + * 2) And a status bit (1 if enabled, 0 if not). + * Check if addr is present in the registers. */ for (reg = 0; reg < reg_cnt; reg++) { - slave_reg = readl(priv->smbus->io + + slave_reg = readl(priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4); /* Check whether the address slots are empty. */ - if (slave_reg == 0) + if (!slave_reg) continue; /* - * Each register holds 4 slave addresses. So, we have to keep - * the byte order consistent with the value read in order to - * update the register correctly, if needed. + * Check if addr matches any of the 4 slave addresses + * in the register. */ slave_reg_tmp = slave_reg; - slave_byte = 0; - while (slave_reg_tmp != 0) { - addr_tmp = slave_reg_tmp & slave_addr_mask; + for (byte = 0; byte < 4; byte++) { + addr_tmp = slave_reg_tmp & MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK; /* * Parse slave address bytes and check whether the * slave address already exists. */ if (addr_tmp == addr) { - exist = true; - break; + /* Clear the slave address slot. */ + slave_reg &= ~(GENMASK(7, 0) << (byte * 8)); + writel(slave_reg, priv->slv->io + + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + + (reg * 0x4)); + /* Free slave at the corresponding index */ + priv->slave[(reg * 4) + byte] = NULL; + + return 0; } /* Parse next byte. */ slave_reg_tmp >>= 8; - slave_byte += 1; } - - /* Exit the loop if the slave address is found. */ - if (exist) - break; } - if (!exist) - return 0; /* Slave is not registered, nothing to do. */ - - /* Cleanup the slave address slot. */ - slave_reg &= ~(GENMASK(7, 0) << (slave_byte * 8)); - writel(slave_reg, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + - reg * 0x4); - - return 0; + return -ENXIO; } static int mlxbf_i2c_init_coalesce(struct platform_device *pdev, @@ -1760,7 +1775,7 @@ static int mlxbf_i2c_init_slave(struct platform_device *pdev, int ret; /* Reset FSM. */ - writel(0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_FSM); + writel(0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_FSM); /* * Enable slave cause interrupt bits. Drive @@ -1775,7 +1790,7 @@ static int mlxbf_i2c_init_slave(struct platform_device *pdev, writel(int_reg, priv->slv_cause->io + MLXBF_I2C_CAUSE_OR_EVTEN0); /* Finally, set the 'ready' bit to start handling transactions. */ - writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY); + writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY); /* Initialize the cause coalesce resource. */ ret = mlxbf_i2c_init_coalesce(pdev, priv); @@ -1820,84 +1835,93 @@ static bool mlxbf_i2c_has_coalesce(struct mlxbf_i2c_priv *priv, bool *read, return true; } -static bool mlxbf_smbus_slave_wait_for_idle(struct mlxbf_i2c_priv *priv, +static bool mlxbf_i2c_slave_wait_for_idle(struct mlxbf_i2c_priv *priv, u32 timeout) { u32 mask = MLXBF_I2C_CAUSE_S_GW_BUSY_FALL; u32 addr = MLXBF_I2C_CAUSE_ARBITER; - if (mlxbf_smbus_poll(priv->slv_cause->io, addr, mask, false, timeout)) + if (mlxbf_i2c_poll(priv->slv_cause->io, addr, mask, false, timeout)) return true; return false; } -/* Send byte to 'external' smbus master. */ -static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes) +static struct i2c_client *mlxbf_i2c_get_slave_from_addr( + struct mlxbf_i2c_priv *priv, u8 addr) { - u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 }; - u8 write_size, pec_en, addr, byte, value, byte_cnt, desc_size; - struct i2c_client *slave = priv->slave; - u32 control32, data32; - int ret; + int i; - if (!slave) - return -EINVAL; + for (i = 0; i < MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT; i++) { + if (!priv->slave[i]) + continue; + + if (priv->slave[i]->addr == addr) + return priv->slave[i]; + } + + return NULL; +} - addr = 0; - byte = 0; - desc_size = MLXBF_I2C_SLAVE_DATA_DESC_SIZE; +/* + * Send byte to 'external' smbus master. This function is executed when + * an external smbus master wants to read data from the BlueField. + */ +static int mlxbf_i2c_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes) +{ + u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 }; + u8 write_size, pec_en, addr, value, byte_cnt; + struct i2c_client *slave; + u32 control32, data32; + int ret = 0; /* - * Read bytes received from the external master. These bytes should - * be located in the first data descriptor register of the slave GW. - * These bytes are the slave address byte and the internal register - * address, if supplied. + * Read the first byte received from the external master to + * determine the slave address. This byte is located in the + * first data descriptor register of the slave GW. */ - if (recv_bytes > 0) { - data32 = ioread32be(priv->smbus->io + - MLXBF_I2C_SLAVE_DATA_DESC_ADDR); - - /* Parse the received bytes. */ - switch (recv_bytes) { - case 2: - byte = (data32 >> 8) & GENMASK(7, 0); - fallthrough; - case 1: - addr = (data32 & GENMASK(7, 0)) >> 1; - } + data32 = ioread32be(priv->slv->io + + MLXBF_I2C_SLAVE_DATA_DESC_ADDR); + addr = (data32 & GENMASK(7, 0)) >> 1; - /* Check whether it's our slave address. */ - if (slave->addr != addr) - return -EINVAL; + /* + * Check if the slave address received in the data descriptor register + * matches any of the slave addresses registered. If there is a match, + * set the slave. + */ + slave = mlxbf_i2c_get_slave_from_addr(priv, addr); + if (!slave) { + ret = -ENXIO; + goto clear_csr; } /* - * I2C read transactions may start by a WRITE followed by a READ. - * Indeed, most slave devices would expect the internal address - * following the slave address byte. So, write that byte first, - * and then, send the requested data bytes to the master. + * An I2C read can consist of a WRITE bit transaction followed by + * a READ bit transaction. Indeed, slave devices often expect + * the slave address to be followed by the internal address. + * So, write the internal address byte first, and then, send the + * requested data to the master. */ if (recv_bytes > 1) { i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); - value = byte; + value = (data32 >> 8) & GENMASK(7, 0); ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED, &value); i2c_slave_event(slave, I2C_SLAVE_STOP, &value); if (ret < 0) - return ret; + goto clear_csr; } /* - * Now, send data to the master; currently, the driver supports - * READ_BYTE, READ_WORD and BLOCK READ protocols. Note that the - * hardware can send up to 128 bytes per transfer. That is the - * size of its data registers. + * Send data to the master. Currently, the driver supports + * READ_BYTE, READ_WORD and BLOCK READ protocols. The + * hardware can send up to 128 bytes per transfer which is + * the total size of the data registers. */ i2c_slave_event(slave, I2C_SLAVE_READ_REQUESTED, &value); - for (byte_cnt = 0; byte_cnt < desc_size; byte_cnt++) { + for (byte_cnt = 0; byte_cnt < MLXBF_I2C_SLAVE_DATA_DESC_SIZE; byte_cnt++) { data_desc[byte_cnt] = value; i2c_slave_event(slave, I2C_SLAVE_READ_PROCESSED, &value); } @@ -1905,14 +1929,12 @@ static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes) /* Send a stop condition to the backend. */ i2c_slave_event(slave, I2C_SLAVE_STOP, &value); - /* Handle the actual transfer. */ - /* Set the number of bytes to write to master. */ write_size = (byte_cnt - 1) & 0x7f; /* Write data to Slave GW data descriptor. */ mlxbf_i2c_smbus_write_data(priv, data_desc, byte_cnt, - MLXBF_I2C_SLAVE_DATA_DESC_ADDR); + MLXBF_I2C_SLAVE_DATA_DESC_ADDR, false); pec_en = 0; /* Disable PEC since it is not supported. */ @@ -1921,46 +1943,52 @@ static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes) control32 |= rol32(write_size, MLXBF_I2C_SLAVE_WRITE_BYTES_SHIFT); control32 |= rol32(pec_en, MLXBF_I2C_SLAVE_SEND_PEC_SHIFT); - writel(control32, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_GW); + writel(control32, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_GW); /* * Wait until the transfer is completed; the driver will wait * until the GW is idle, a cause will rise on fall of GW busy. */ - mlxbf_smbus_slave_wait_for_idle(priv, MLXBF_I2C_SMBUS_TIMEOUT); + mlxbf_i2c_slave_wait_for_idle(priv, MLXBF_I2C_SMBUS_TIMEOUT); +clear_csr: /* Release the Slave GW. */ - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_PEC); - writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY); + writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); + writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_PEC); + writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY); - return 0; + return ret; } -/* Receive bytes from 'external' smbus master. */ -static int mlxbf_smbus_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes) +/* + * Receive bytes from 'external' smbus master. This function is executed when + * an external smbus master wants to write data to the BlueField. + */ +static int mlxbf_i2c_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes) { u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 }; - struct i2c_client *slave = priv->slave; + struct i2c_client *slave; u8 value, byte, addr; int ret = 0; - if (!slave) - return -EINVAL; - /* Read data from Slave GW data descriptor. */ mlxbf_i2c_smbus_read_data(priv, data_desc, recv_bytes, - MLXBF_I2C_SLAVE_DATA_DESC_ADDR); - - /* Check whether its our slave address. */ + MLXBF_I2C_SLAVE_DATA_DESC_ADDR, false); addr = data_desc[0] >> 1; - if (slave->addr != addr) - return -EINVAL; /* - * Notify the slave backend; another I2C master wants to write data - * to us. This event is sent once the slave address and the write bit - * is detected. + * Check if the slave address received in the data descriptor register + * matches any of the slave addresses registered. + */ + slave = mlxbf_i2c_get_slave_from_addr(priv, addr); + if (!slave) { + ret = -EINVAL; + goto clear_csr; + } + + /* + * Notify the slave backend that an smbus master wants to write data + * to the BlueField. */ i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value); @@ -1973,18 +2001,22 @@ static int mlxbf_smbus_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes) break; } - /* Send a stop condition to the backend. */ + /* + * Send a stop event to the slave backend, to signal + * the end of the write transactions. + */ i2c_slave_event(slave, I2C_SLAVE_STOP, &value); +clear_csr: /* Release the Slave GW. */ - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); - writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_PEC); - writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY); + writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); + writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_PEC); + writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY); return ret; } -static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr) +static irqreturn_t mlxbf_i2c_irq(int irq, void *ptr) { struct mlxbf_i2c_priv *priv = ptr; bool read, write, irq_is_set; @@ -2014,7 +2046,7 @@ static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr) * slave, if the higher 8 bits are sent then the slave expect N bytes * from the master. */ - rw_bytes_reg = readl(priv->smbus->io + + rw_bytes_reg = readl(priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES); recv_bytes = (rw_bytes_reg >> 8) & GENMASK(7, 0); @@ -2032,9 +2064,9 @@ static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr) MLXBF_I2C_SLAVE_DATA_DESC_SIZE : recv_bytes; if (read) - mlxbf_smbus_irq_send(priv, recv_bytes); + mlxbf_i2c_irq_send(priv, recv_bytes); else - mlxbf_smbus_irq_recv(priv, recv_bytes); + mlxbf_i2c_irq_recv(priv, recv_bytes); return IRQ_HANDLED; } @@ -2129,23 +2161,21 @@ static s32 mlxbf_i2c_smbus_xfer(struct i2c_adapter *adap, u16 addr, static int mlxbf_i2c_reg_slave(struct i2c_client *slave) { struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter); + struct device *dev = &slave->dev; int ret; - if (priv->slave) - return -EBUSY; - /* * Do not support ten bit chip address and do not use Packet Error * Checking (PEC). */ - if (slave->flags & (I2C_CLIENT_TEN | I2C_CLIENT_PEC)) + if (slave->flags & (I2C_CLIENT_TEN | I2C_CLIENT_PEC)) { + dev_err(dev, "SMBus PEC and 10 bit address not supported\n"); return -EAFNOSUPPORT; + } - ret = mlxbf_slave_enable(priv, slave->addr); - if (ret < 0) - return ret; - - priv->slave = slave; + ret = mlxbf_i2c_slave_enable(priv, slave); + if (ret) + dev_err(dev, "Surpassed max number of registered slaves allowed\n"); return 0; } @@ -2153,18 +2183,19 @@ static int mlxbf_i2c_reg_slave(struct i2c_client *slave) static int mlxbf_i2c_unreg_slave(struct i2c_client *slave) { struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter); + struct device *dev = &slave->dev; int ret; - WARN_ON(!priv->slave); - - /* Unregister slave, i.e. disable the slave address in hardware. */ - ret = mlxbf_slave_disable(priv); - if (ret < 0) - return ret; - - priv->slave = NULL; + /* + * Unregister slave by: + * 1) Disabling the slave address in hardware + * 2) Freeing priv->slave at the corresponding index + */ + ret = mlxbf_i2c_slave_disable(priv, slave->addr); + if (ret) + dev_err(dev, "Unable to find slave 0x%x\n", slave->addr); - return 0; + return ret; } static u32 mlxbf_i2c_functionality(struct i2c_adapter *adap) @@ -2180,14 +2211,27 @@ static struct mlxbf_i2c_chip_info mlxbf_i2c_chip[] = { [1] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_1], [2] = &mlxbf_i2c_gpio_res[MLXBF_I2C_CHIP_TYPE_1] }, - .calculate_freq = mlxbf_calculate_freq_from_tyu + .calculate_freq = mlxbf_i2c_calculate_freq_from_tyu, + .smbus_master_rs_bytes_off = MLXBF_I2C_YU_SMBUS_RS_BYTES, + .smbus_master_fsm_off = MLXBF_I2C_YU_SMBUS_MASTER_FSM }, [MLXBF_I2C_CHIP_TYPE_2] = { .type = MLXBF_I2C_CHIP_TYPE_2, .shared_res = { [0] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_2] }, - .calculate_freq = mlxbf_calculate_freq_from_yu + .calculate_freq = mlxbf_i2c_calculate_freq_from_yu, + .smbus_master_rs_bytes_off = MLXBF_I2C_YU_SMBUS_RS_BYTES, + .smbus_master_fsm_off = MLXBF_I2C_YU_SMBUS_MASTER_FSM + }, + [MLXBF_I2C_CHIP_TYPE_3] = { + .type = MLXBF_I2C_CHIP_TYPE_3, + .shared_res = { + [0] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_3] + }, + .calculate_freq = mlxbf_i2c_calculate_freq_from_yu, + .smbus_master_rs_bytes_off = MLXBF_I2C_RSH_YU_SMBUS_RS_BYTES, + .smbus_master_fsm_off = MLXBF_I2C_RSH_YU_SMBUS_MASTER_FSM } }; @@ -2203,24 +2247,11 @@ static struct i2c_adapter_quirks mlxbf_i2c_quirks = { .max_write_len = MLXBF_I2C_MASTER_DATA_W_LENGTH, }; -static const struct of_device_id mlxbf_i2c_dt_ids[] = { - { - .compatible = "mellanox,i2c-mlxbf1", - .data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1] - }, - { - .compatible = "mellanox,i2c-mlxbf2", - .data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2] - }, - {}, -}; - -MODULE_DEVICE_TABLE(of, mlxbf_i2c_dt_ids); - #ifdef CONFIG_ACPI static const struct acpi_device_id mlxbf_i2c_acpi_ids[] = { { "MLNXBF03", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1] }, { "MLNXBF23", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2] }, + { "MLNXBF31", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_3] }, {}, }; @@ -2229,35 +2260,27 @@ MODULE_DEVICE_TABLE(acpi, mlxbf_i2c_acpi_ids); static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv) { const struct acpi_device_id *aid; - struct acpi_device *adev; - unsigned long bus_id = 0; - const char *uid; + u64 bus_id; int ret; if (acpi_disabled) return -ENOENT; - adev = ACPI_COMPANION(dev); - if (!adev) - return -ENXIO; - aid = acpi_match_device(mlxbf_i2c_acpi_ids, dev); if (!aid) return -ENODEV; priv->chip = (struct mlxbf_i2c_chip_info *)aid->driver_data; - uid = acpi_device_uid(adev); - if (!uid || !(*uid)) { + ret = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &bus_id); + if (ret) { dev_err(dev, "Cannot retrieve UID\n"); - return -ENODEV; + return ret; } - ret = kstrtoul(uid, 0, &bus_id); - if (!ret) - priv->bus = bus_id; + priv->bus = bus_id; - return ret; + return 0; } #else static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv) @@ -2266,36 +2289,12 @@ static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv) } #endif /* CONFIG_ACPI */ -static int mlxbf_i2c_of_probe(struct device *dev, struct mlxbf_i2c_priv *priv) -{ - const struct of_device_id *oid; - int bus_id = -1; - - if (IS_ENABLED(CONFIG_OF) && dev->of_node) { - oid = of_match_node(mlxbf_i2c_dt_ids, dev->of_node); - if (!oid) - return -ENODEV; - - priv->chip = oid->data; - - bus_id = of_alias_get_id(dev->of_node, "i2c"); - if (bus_id >= 0) - priv->bus = bus_id; - } - - if (bus_id < 0) { - dev_err(dev, "Cannot get bus id"); - return bus_id; - } - - return 0; -} - static int mlxbf_i2c_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct mlxbf_i2c_priv *priv; struct i2c_adapter *adap; + u32 resource_version; int irq, ret; priv = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_priv), GFP_KERNEL); @@ -2303,17 +2302,63 @@ static int mlxbf_i2c_probe(struct platform_device *pdev) return -ENOMEM; ret = mlxbf_i2c_acpi_probe(dev, priv); - if (ret < 0 && ret != -ENOENT && ret != -ENXIO) - ret = mlxbf_i2c_of_probe(dev, priv); - if (ret < 0) return ret; - ret = mlxbf_i2c_init_resource(pdev, &priv->smbus, - MLXBF_I2C_SMBUS_RES); - if (ret < 0) { - dev_err(dev, "Cannot fetch smbus resource info"); - return ret; + /* This property allows the driver to stay backward compatible with older + * ACPI tables. + * Starting BlueField-3 SoC, the "smbus" resource was broken down into 3 + * separate resources "timer", "master" and "slave". + */ + if (device_property_read_u32(dev, "resource_version", &resource_version)) + resource_version = 0; + + priv->resource_version = resource_version; + + if (priv->chip->type < MLXBF_I2C_CHIP_TYPE_3 && resource_version == 0) { + priv->timer = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL); + if (!priv->timer) + return -ENOMEM; + + priv->mst = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL); + if (!priv->mst) + return -ENOMEM; + + priv->slv = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL); + if (!priv->slv) + return -ENOMEM; + + ret = mlxbf_i2c_init_resource(pdev, &priv->smbus, + MLXBF_I2C_SMBUS_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch smbus resource info"); + return ret; + } + + priv->timer->io = priv->smbus->io; + priv->mst->io = priv->smbus->io + MLXBF_I2C_MST_ADDR_OFFSET; + priv->slv->io = priv->smbus->io + MLXBF_I2C_SLV_ADDR_OFFSET; + } else { + ret = mlxbf_i2c_init_resource(pdev, &priv->timer, + MLXBF_I2C_SMBUS_TIMER_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch timer resource info"); + return ret; + } + + ret = mlxbf_i2c_init_resource(pdev, &priv->mst, + MLXBF_I2C_SMBUS_MST_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch master resource info"); + return ret; + } + + ret = mlxbf_i2c_init_resource(pdev, &priv->slv, + MLXBF_I2C_SMBUS_SLV_RES); + if (ret < 0) { + dev_err(dev, "Cannot fetch slave resource info"); + return ret; + } } ret = mlxbf_i2c_init_resource(pdev, &priv->mst_cause, @@ -2372,8 +2417,8 @@ static int mlxbf_i2c_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; - ret = devm_request_irq(dev, irq, mlxbf_smbus_irq, - IRQF_ONESHOT | IRQF_SHARED | IRQF_PROBE_SHARED, + ret = devm_request_irq(dev, irq, mlxbf_i2c_irq, + IRQF_SHARED | IRQF_PROBE_SHARED, dev_name(dev), priv); if (ret < 0) { dev_err(dev, "Cannot get irq %d\n", irq); @@ -2401,8 +2446,19 @@ static int mlxbf_i2c_remove(struct platform_device *pdev) struct device *dev = &pdev->dev; struct resource *params; - params = priv->smbus->params; - devm_release_mem_region(dev, params->start, resource_size(params)); + if (priv->chip->type < MLXBF_I2C_CHIP_TYPE_3 && priv->resource_version == 0) { + params = priv->smbus->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + } else { + params = priv->timer->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + + params = priv->mst->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + + params = priv->slv->params; + devm_release_mem_region(dev, params->start, resource_size(params)); + } params = priv->mst_cause->params; devm_release_mem_region(dev, params->start, resource_size(params)); @@ -2434,7 +2490,6 @@ static struct platform_driver mlxbf_i2c_driver = { .remove = mlxbf_i2c_remove, .driver = { .name = "i2c-mlxbf", - .of_match_table = mlxbf_i2c_dt_ids, #ifdef CONFIG_ACPI .acpi_match_table = ACPI_PTR(mlxbf_i2c_acpi_ids), #endif /* CONFIG_ACPI */ @@ -2467,4 +2522,5 @@ module_exit(mlxbf_i2c_exit); MODULE_DESCRIPTION("Mellanox BlueField I2C bus driver"); MODULE_AUTHOR("Khalil Blaiech "); +MODULE_AUTHOR("Asmaa Mnebhi "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c index ea48e6a9cfca7f34a05c7868b1f18d5d028b0a0f..87739fb4388bacf4ec1fec0e23630010f0ac229a 100644 --- a/drivers/i2c/busses/i2c-qcom-cci.c +++ b/drivers/i2c/busses/i2c-qcom-cci.c @@ -807,6 +807,7 @@ static const struct cci_data cci_v2_data = { }; static const struct of_device_id cci_dt_match[] = { + { .compatible = "qcom,msm8226-cci", .data = &cci_v1_data}, { .compatible = "qcom,msm8916-cci", .data = &cci_v1_data}, { .compatible = "qcom,msm8974-cci", .data = &cci_v1_5_data}, { .compatible = "qcom,msm8996-cci", .data = &cci_v2_data}, diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c index ecba1dfc127887346b9043b1b68857a93eb481c3..849848ccb08022825cd13d14fe3f5adfbc453c17 100644 --- a/drivers/i2c/busses/i2c-riic.c +++ b/drivers/i2c/busses/i2c-riic.c @@ -400,7 +400,6 @@ static int riic_i2c_probe(struct platform_device *pdev) { struct riic_dev *riic; struct i2c_adapter *adap; - struct resource *res; struct i2c_timings i2c_t; struct reset_control *rstc; int i, ret; @@ -409,8 +408,7 @@ static int riic_i2c_probe(struct platform_device *pdev) if (!riic) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - riic->base = devm_ioremap_resource(&pdev->dev, res); + riic->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(riic->base)) return PTR_ERR(riic->base); diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 2e98e7793bbae0fd36c4d635de8bd5da8602613b..d1658ed76562b8817e401a977871da581e4796fc 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1165,6 +1165,11 @@ static const struct rk3x_i2c_soc_data rv1108_soc_data = { .calc_timings = rk3x_i2c_v1_calc_timings, }; +static const struct rk3x_i2c_soc_data rv1126_soc_data = { + .grf_offset = 0x118, + .calc_timings = rk3x_i2c_v1_calc_timings, +}; + static const struct rk3x_i2c_soc_data rk3066_soc_data = { .grf_offset = 0x154, .calc_timings = rk3x_i2c_v0_calc_timings, @@ -1195,6 +1200,10 @@ static const struct of_device_id rk3x_i2c_match[] = { .compatible = "rockchip,rv1108-i2c", .data = &rv1108_soc_data }, + { + .compatible = "rockchip,rv1126-i2c", + .data = &rv1126_soc_data + }, { .compatible = "rockchip,rk3066-i2c", .data = &rk3066_soc_data diff --git a/drivers/i2c/busses/i2c-scmi.c b/drivers/i2c/busses/i2c-scmi.c index 6746aa46d96c861330b1c99ec97d48733674de84..0239e134b90f425ece63c0dcde8326ad75a27af0 100644 --- a/drivers/i2c/busses/i2c-scmi.c +++ b/drivers/i2c/busses/i2c-scmi.c @@ -6,15 +6,13 @@ */ #include +#include #include #include #include #include #include -#define ACPI_SMBUS_HC_CLASS "smbus" -#define ACPI_SMBUS_HC_DEVICE_NAME "cmi" - /* SMBUS HID definition as supported by Microsoft Windows */ #define ACPI_SMBUS_MS_HID "SMB0001" @@ -30,7 +28,7 @@ struct acpi_smbus_cmi { u8 cap_info:1; u8 cap_read:1; u8 cap_write:1; - struct smbus_methods_t *methods; + const struct smbus_methods_t *methods; }; static const struct smbus_methods_t smbus_methods = { @@ -358,29 +356,25 @@ static acpi_status acpi_smbus_cmi_query_methods(acpi_handle handle, u32 level, return AE_OK; } -static int acpi_smbus_cmi_add(struct acpi_device *device) +static int smbus_cmi_probe(struct platform_device *device) { + struct device *dev = &device->dev; struct acpi_smbus_cmi *smbus_cmi; - const struct acpi_device_id *id; int ret; smbus_cmi = kzalloc(sizeof(struct acpi_smbus_cmi), GFP_KERNEL); if (!smbus_cmi) return -ENOMEM; - smbus_cmi->handle = device->handle; - strcpy(acpi_device_name(device), ACPI_SMBUS_HC_DEVICE_NAME); - strcpy(acpi_device_class(device), ACPI_SMBUS_HC_CLASS); - device->driver_data = smbus_cmi; + smbus_cmi->handle = ACPI_HANDLE(dev); + smbus_cmi->methods = device_get_match_data(dev); + + platform_set_drvdata(device, smbus_cmi); + smbus_cmi->cap_info = 0; smbus_cmi->cap_read = 0; smbus_cmi->cap_write = 0; - for (id = acpi_smbus_cmi_ids; id->id[0]; id++) - if (!strcmp(id->id, acpi_device_hid(device))) - smbus_cmi->methods = - (struct smbus_methods_t *) id->driver_data; - acpi_walk_namespace(ACPI_TYPE_METHOD, smbus_cmi->handle, 1, acpi_smbus_cmi_query_methods, NULL, smbus_cmi, NULL); @@ -390,8 +384,7 @@ static int acpi_smbus_cmi_add(struct acpi_device *device) } snprintf(smbus_cmi->adapter.name, sizeof(smbus_cmi->adapter.name), - "SMBus CMI adapter %s", - acpi_device_name(device)); + "SMBus CMI adapter %s", dev_name(dev)); smbus_cmi->adapter.owner = THIS_MODULE; smbus_cmi->adapter.algo = &acpi_smbus_cmi_algorithm; smbus_cmi->adapter.algo_data = smbus_cmi; @@ -408,31 +401,28 @@ static int acpi_smbus_cmi_add(struct acpi_device *device) err: kfree(smbus_cmi); - device->driver_data = NULL; return ret; } -static int acpi_smbus_cmi_remove(struct acpi_device *device) +static int smbus_cmi_remove(struct platform_device *device) { - struct acpi_smbus_cmi *smbus_cmi = acpi_driver_data(device); + struct acpi_smbus_cmi *smbus_cmi = platform_get_drvdata(device); i2c_del_adapter(&smbus_cmi->adapter); kfree(smbus_cmi); - device->driver_data = NULL; return 0; } -static struct acpi_driver acpi_smbus_cmi_driver = { - .name = ACPI_SMBUS_HC_DEVICE_NAME, - .class = ACPI_SMBUS_HC_CLASS, - .ids = acpi_smbus_cmi_ids, - .ops = { - .add = acpi_smbus_cmi_add, - .remove = acpi_smbus_cmi_remove, +static struct platform_driver smbus_cmi_driver = { + .probe = smbus_cmi_probe, + .remove = smbus_cmi_remove, + .driver = { + .name = "smbus_cmi", + .acpi_match_table = acpi_smbus_cmi_ids, }, }; -module_acpi_driver(acpi_smbus_cmi_driver); +module_platform_driver(smbus_cmi_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Crane Cai "); diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 031c78ac42e67f6a7676af04214b3ce5cd0551a7..954022c04cc422dbc03569a9770adc7bee054818 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -443,11 +443,16 @@ static int tegra_i2c_init_dma(struct tegra_i2c_dev *i2c_dev) u32 *dma_buf; int err; - if (!i2c_dev->hw->has_apb_dma || i2c_dev->is_vi) + if (i2c_dev->is_vi) return 0; - if (!IS_ENABLED(CONFIG_TEGRA20_APB_DMA)) { - dev_dbg(i2c_dev->dev, "DMA support not enabled\n"); + if (!i2c_dev->hw->has_apb_dma) { + if (!IS_ENABLED(CONFIG_TEGRA20_APB_DMA)) { + dev_dbg(i2c_dev->dev, "APB DMA support not enabled\n"); + return 0; + } + } else if (!IS_ENABLED(CONFIG_TEGRA186_GPC_DMA)) { + dev_dbg(i2c_dev->dev, "GPC DMA support not enabled\n"); return 0; } diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index 08b561f0709d5d8f48bdc14da706942ee199f914..4dd777cc0c89f8a7675ae2d77b84c9761b6b8301 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -26,7 +26,7 @@ struct gsb_buffer { union { u16 wdata; u8 bdata; - u8 data[0]; + DECLARE_FLEX_ARRAY(u8, data); }; } __packed; @@ -137,6 +137,11 @@ static const struct acpi_device_id i2c_acpi_ignored_device_ids[] = { {} }; +struct i2c_acpi_irq_context { + int irq; + bool wake_capable; +}; + static int i2c_acpi_do_lookup(struct acpi_device *adev, struct i2c_acpi_lookup *lookup) { @@ -168,13 +173,19 @@ static int i2c_acpi_do_lookup(struct acpi_device *adev, return 0; } -static int i2c_acpi_add_resource(struct acpi_resource *ares, void *data) +static int i2c_acpi_add_irq_resource(struct acpi_resource *ares, void *data) { - int *irq = data; + struct i2c_acpi_irq_context *irq_ctx = data; struct resource r; - if (*irq <= 0 && acpi_dev_resource_interrupt(ares, 0, &r)) - *irq = i2c_dev_irq_from_resources(&r, 1); + if (irq_ctx->irq > 0) + return 1; + + if (!acpi_dev_resource_interrupt(ares, 0, &r)) + return 1; + + irq_ctx->irq = i2c_dev_irq_from_resources(&r, 1); + irq_ctx->wake_capable = r.flags & IORESOURCE_IRQ_WAKECAPABLE; return 1; /* No need to add resource to the list */ } @@ -182,31 +193,40 @@ static int i2c_acpi_add_resource(struct acpi_resource *ares, void *data) /** * i2c_acpi_get_irq - get device IRQ number from ACPI * @client: Pointer to the I2C client device + * @wake_capable: Set to true if the IRQ is wake capable * * Find the IRQ number used by a specific client device. * * Return: The IRQ number or an error code. */ -int i2c_acpi_get_irq(struct i2c_client *client) +int i2c_acpi_get_irq(struct i2c_client *client, bool *wake_capable) { struct acpi_device *adev = ACPI_COMPANION(&client->dev); struct list_head resource_list; - int irq = -ENOENT; + struct i2c_acpi_irq_context irq_ctx = { + .irq = -ENOENT, + }; int ret; INIT_LIST_HEAD(&resource_list); ret = acpi_dev_get_resources(adev, &resource_list, - i2c_acpi_add_resource, &irq); + i2c_acpi_add_irq_resource, &irq_ctx); if (ret < 0) return ret; acpi_dev_free_resource_list(&resource_list); - if (irq == -ENOENT) - irq = acpi_dev_gpio_irq_get(adev, 0); + if (irq_ctx.irq == -ENOENT) + irq_ctx.irq = acpi_dev_gpio_irq_wake_get(adev, 0, &irq_ctx.wake_capable); + + if (irq_ctx.irq < 0) + return irq_ctx.irq; + + if (wake_capable) + *wake_capable = irq_ctx.wake_capable; - return irq; + return irq_ctx.irq; } static int i2c_acpi_get_info(struct acpi_device *adev, diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 91007558bcb26012716bc0bbe95f57097cb4fc5d..b4edf10e8fd06a575bb1e8f8f494b9a2839c54e7 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -487,7 +487,11 @@ static int i2c_device_probe(struct device *dev) if (irq == -EINVAL || irq == -ENODATA) irq = of_irq_get(dev->of_node, 0); } else if (ACPI_COMPANION(dev)) { - irq = i2c_acpi_get_irq(client); + bool wake_capable; + + irq = i2c_acpi_get_irq(client, &wake_capable); + if (irq > 0 && wake_capable) + client->flags |= I2C_CLIENT_WAKE; } if (irq == -EPROBE_DEFER) { status = irq; @@ -599,13 +603,9 @@ static void i2c_device_remove(struct device *dev) driver = to_i2c_driver(dev->driver); if (driver->remove) { - int status; - dev_dbg(dev, "remove\n"); - status = driver->remove(client); - if (status) - dev_warn(dev, "remove failed (%pe), will be ignored\n", ERR_PTR(status)); + driver->remove(client); } devres_release_group(&client->dev, client->devres_group_id); diff --git a/drivers/i2c/i2c-core.h b/drivers/i2c/i2c-core.h index 87e2c914f1c57ba8a6d98ceb1a22331cbea8e4c0..1247e6e6e97517afdd045c939cb7eee9bd0fd2bc 100644 --- a/drivers/i2c/i2c-core.h +++ b/drivers/i2c/i2c-core.h @@ -61,11 +61,11 @@ static inline int __i2c_check_suspended(struct i2c_adapter *adap) #ifdef CONFIG_ACPI void i2c_acpi_register_devices(struct i2c_adapter *adap); -int i2c_acpi_get_irq(struct i2c_client *client); +int i2c_acpi_get_irq(struct i2c_client *client, bool *wake_capable); #else /* CONFIG_ACPI */ static inline void i2c_acpi_register_devices(struct i2c_adapter *adap) { } -static inline int i2c_acpi_get_irq(struct i2c_client *client) +static inline int i2c_acpi_get_irq(struct i2c_client *client, bool *wake_capable) { return 0; } diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c index 774507b54b57b7e722ee77fa1e8952fb29b30b1c..313904be5f3bde25990ffb631c248e60aa94bb09 100644 --- a/drivers/i2c/i2c-mux.c +++ b/drivers/i2c/i2c-mux.c @@ -243,9 +243,10 @@ struct i2c_mux_core *i2c_mux_alloc(struct i2c_adapter *parent, int (*deselect)(struct i2c_mux_core *, u32)) { struct i2c_mux_core *muxc; + size_t mux_size; - muxc = devm_kzalloc(dev, struct_size(muxc, adapter, max_adapters) - + sizeof_priv, GFP_KERNEL); + mux_size = struct_size(muxc, adapter, max_adapters); + muxc = devm_kzalloc(dev, size_add(mux_size, sizeof_priv), GFP_KERNEL); if (!muxc) return NULL; if (sizeof_priv) diff --git a/drivers/i2c/i2c-slave-eeprom.c b/drivers/i2c/i2c-slave-eeprom.c index 5c7ae421cacf83da9111f74fff430b435a8d4fa9..4abc2d9198815498ede3775eb8789847ccdcbae1 100644 --- a/drivers/i2c/i2c-slave-eeprom.c +++ b/drivers/i2c/i2c-slave-eeprom.c @@ -181,14 +181,12 @@ static int i2c_slave_eeprom_probe(struct i2c_client *client, const struct i2c_de return 0; }; -static int i2c_slave_eeprom_remove(struct i2c_client *client) +static void i2c_slave_eeprom_remove(struct i2c_client *client) { struct eeprom_data *eeprom = i2c_get_clientdata(client); i2c_slave_unregister(client); sysfs_remove_bin_file(&client->dev.kobj, &eeprom->bin); - - return 0; } static const struct i2c_device_id i2c_slave_eeprom_id[] = { diff --git a/drivers/i2c/i2c-slave-testunit.c b/drivers/i2c/i2c-slave-testunit.c index 56dae08dfd48954e72e61fdc4d468a3de17152bf..75ee7ebdb614feb03436fd532e5f302bbe1199d1 100644 --- a/drivers/i2c/i2c-slave-testunit.c +++ b/drivers/i2c/i2c-slave-testunit.c @@ -153,13 +153,12 @@ static int i2c_slave_testunit_probe(struct i2c_client *client) return i2c_slave_register(client, i2c_slave_testunit_slave_cb); }; -static int i2c_slave_testunit_remove(struct i2c_client *client) +static void i2c_slave_testunit_remove(struct i2c_client *client) { struct testunit_data *tu = i2c_get_clientdata(client); cancel_delayed_work_sync(&tu->worker); i2c_slave_unregister(client); - return 0; } static const struct i2c_device_id i2c_slave_testunit_id[] = { diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c index 8ba9b59a3c40fde52bd0b59564185c5eaeecdac9..07c92c8495a3c895551507448e11661d8a4ee691 100644 --- a/drivers/i2c/i2c-smbus.c +++ b/drivers/i2c/i2c-smbus.c @@ -153,12 +153,11 @@ static int smbalert_probe(struct i2c_client *ara, } /* IRQ and memory resources are managed so they are freed automatically */ -static int smbalert_remove(struct i2c_client *ara) +static void smbalert_remove(struct i2c_client *ara) { struct i2c_smbus_alert *alert = i2c_get_clientdata(ara); cancel_work_sync(&alert->alert); - return 0; } static const struct i2c_device_id smbalert_ids[] = { diff --git a/drivers/i2c/muxes/Kconfig b/drivers/i2c/muxes/Kconfig index 1708b1a82da28f5370d853bd0bf11b619f0dce93..ea838dbae32e53c4ebcb505a70572e714a28ca2f 100644 --- a/drivers/i2c/muxes/Kconfig +++ b/drivers/i2c/muxes/Kconfig @@ -9,7 +9,7 @@ menu "Multiplexer I2C Chip support" config I2C_ARB_GPIO_CHALLENGE tristate "GPIO-based I2C arbitration" depends on GPIOLIB || COMPILE_TEST - depends on OF || COMPILE_TEST + depends on OF help If you say yes to this option, support will be included for an I2C multimaster arbitration scheme using GPIOs and a challenge & @@ -34,7 +34,7 @@ config I2C_MUX_GPIO config I2C_MUX_GPMUX tristate "General Purpose I2C multiplexer" select MULTIPLEXER - depends on OF || COMPILE_TEST + depends on OF help If you say yes to this option, support will be included for a general purpose I2C multiplexer. This driver provides access to @@ -77,7 +77,7 @@ config I2C_MUX_PCA954x config I2C_MUX_PINCTRL tristate "pinctrl-based I2C multiplexer" depends on PINCTRL - depends on OF || COMPILE_TEST + depends on OF help If you say yes to this option, support will be included for an I2C multiplexer that uses the pinctrl subsystem, i.e. pin multiplexing. diff --git a/drivers/i2c/muxes/i2c-mux-ltc4306.c b/drivers/i2c/muxes/i2c-mux-ltc4306.c index 704f1e50f6f46d77234395faf52bab7d12e0bcc9..70835825083f529d6b58fdbeb3f86b82e33ae8d9 100644 --- a/drivers/i2c/muxes/i2c-mux-ltc4306.c +++ b/drivers/i2c/muxes/i2c-mux-ltc4306.c @@ -294,13 +294,11 @@ static int ltc4306_probe(struct i2c_client *client) return 0; } -static int ltc4306_remove(struct i2c_client *client) +static void ltc4306_remove(struct i2c_client *client) { struct i2c_mux_core *muxc = i2c_get_clientdata(client); i2c_mux_del_adapters(muxc); - - return 0; } static struct i2c_driver ltc4306_driver = { diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c index 6daec8d3d3310ba4909e71a4e53e1a86f0c29e3d..ea83de78f52db626f2c273ddf8baa29874943e29 100644 --- a/drivers/i2c/muxes/i2c-mux-pca9541.c +++ b/drivers/i2c/muxes/i2c-mux-pca9541.c @@ -325,12 +325,11 @@ static int pca9541_probe(struct i2c_client *client, return 0; } -static int pca9541_remove(struct i2c_client *client) +static void pca9541_remove(struct i2c_client *client) { struct i2c_mux_core *muxc = i2c_get_clientdata(client); i2c_mux_del_adapters(muxc); - return 0; } static struct i2c_driver pca9541_driver = { diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c index 4ad665757dd815c4a8cdc990efaae29e9d0f81c3..a5f458b635df6c3e45df5c971875b3bd50f9c2e5 100644 --- a/drivers/i2c/muxes/i2c-mux-pca954x.c +++ b/drivers/i2c/muxes/i2c-mux-pca954x.c @@ -521,14 +521,13 @@ fail_cleanup: return ret; } -static int pca954x_remove(struct i2c_client *client) +static void pca954x_remove(struct i2c_client *client) { struct i2c_mux_core *muxc = i2c_get_clientdata(client); device_remove_file(&client->dev, &dev_attr_idle_state); pca954x_cleanup(muxc); - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/i2c/muxes/i2c-mux-pinctrl.c b/drivers/i2c/muxes/i2c-mux-pinctrl.c index f1bb00a11ad628d7196494e88a56337989c99c21..d5ad904756fdf3c5005c66c7b3cffc5bcdc45edc 100644 --- a/drivers/i2c/muxes/i2c-mux-pinctrl.c +++ b/drivers/i2c/muxes/i2c-mux-pinctrl.c @@ -185,7 +185,7 @@ MODULE_DEVICE_TABLE(of, i2c_mux_pinctrl_of_match); static struct platform_driver i2c_mux_pinctrl_driver = { .driver = { .name = "i2c-mux-pinctrl", - .of_match_table = of_match_ptr(i2c_mux_pinctrl_of_match), + .of_match_table = i2c_mux_pinctrl_of_match, }, .probe = i2c_mux_pinctrl_probe, .remove = i2c_mux_pinctrl_remove, diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 7850287dfe7a91effaded28609fc2474d0a5443e..351c81a929a6c914b21d153d7587fe40dc695c0b 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -1379,6 +1379,9 @@ static int i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, i3c_bus_set_addr_slot_status(&master->bus, dev->info.dyn_addr, I3C_ADDR_SLOT_I3C_DEV); + if (old_dyn_addr) + i3c_bus_set_addr_slot_status(&master->bus, old_dyn_addr, + I3C_ADDR_SLOT_FREE); } if (master->ops->reattach_i3c_dev) { @@ -1908,10 +1911,6 @@ int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, i3c_master_free_i3c_dev(olddev); } - ret = i3c_master_reattach_i3c_dev(newdev, old_dyn_addr); - if (ret) - goto err_detach_dev; - /* * Depending on our previous state, the expected dynamic address might * differ: diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 3e101719689ac0cdad8185beb45d404506677e9f..cfeb24d40d3789b8bfbfedd7ba2ad0b7fecec885 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -928,6 +928,51 @@ static struct cpuidle_state adl_l_cstates[] __initdata = { .enter = NULL } }; +static struct cpuidle_state adl_n_cstates[] __initdata = { + { + .name = "C1", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE, + .exit_latency = 1, + .target_residency = 1, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C1E", + .desc = "MWAIT 0x01", + .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE, + .exit_latency = 2, + .target_residency = 4, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C6", + .desc = "MWAIT 0x20", + .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 195, + .target_residency = 585, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C8", + .desc = "MWAIT 0x40", + .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 260, + .target_residency = 1040, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .name = "C10", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 660, + .target_residency = 1980, + .enter = &intel_idle, + .enter_s2idle = intel_idle_s2idle, }, + { + .enter = NULL } +}; + static struct cpuidle_state spr_cstates[] __initdata = { { .name = "C1", @@ -1309,6 +1354,10 @@ static const struct idle_cpu idle_cpu_adl_l __initconst = { .state_table = adl_l_cstates, }; +static const struct idle_cpu idle_cpu_adl_n __initconst = { + .state_table = adl_n_cstates, +}; + static const struct idle_cpu idle_cpu_spr __initconst = { .state_table = spr_cstates, .disable_promotion_to_c1e = true, @@ -1379,6 +1428,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &idle_cpu_adl_n), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr), X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl), X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl), @@ -1507,7 +1557,7 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) state = &drv->states[drv->state_count++]; snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate); - strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); + strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN); state->exit_latency = cx->latency; /* * For C1-type C-states use the same number for both the exit @@ -1816,6 +1866,7 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv) break; case INTEL_FAM6_ALDERLAKE: case INTEL_FAM6_ALDERLAKE_L: + case INTEL_FAM6_ALDERLAKE_N: adl_idle_state_table_update(); break; } diff --git a/drivers/iio/accel/Kconfig b/drivers/iio/accel/Kconfig index 35798712f81185a5d50c4cbf38d2ea48681aec9b..ffac66db7ac92901d74aeec22d43f303c268d8d7 100644 --- a/drivers/iio/accel/Kconfig +++ b/drivers/iio/accel/Kconfig @@ -541,6 +541,19 @@ config MMA9553 To compile this driver as a module, choose M here: the module will be called mma9553. +config MSA311 + tristate "MEMSensing Digital 3-Axis Accelerometer Driver" + depends on I2C + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER + select REGMAP_I2C + help + Say yes here to build support for the MEMSensing MSA311 + accelerometer driver. + + To compile this driver as a module, choose M here: the module will be + called msa311. + config MXC4005 tristate "Memsic MXC4005XC 3-Axis Accelerometer Driver" depends on I2C diff --git a/drivers/iio/accel/Makefile b/drivers/iio/accel/Makefile index 4d87926688385a4662d546e1d69ddb29d49b9703..5e45b5fa5ab50f3288611ce9eb4b5e6ff6ffa8d8 100644 --- a/drivers/iio/accel/Makefile +++ b/drivers/iio/accel/Makefile @@ -58,6 +58,8 @@ obj-$(CONFIG_MMA9551_CORE) += mma9551_core.o obj-$(CONFIG_MMA9551) += mma9551.o obj-$(CONFIG_MMA9553) += mma9553.o +obj-$(CONFIG_MSA311) += msa311.o + obj-$(CONFIG_MXC4005) += mxc4005.o obj-$(CONFIG_MXC6255) += mxc6255.o diff --git a/drivers/iio/accel/adxl313.h b/drivers/iio/accel/adxl313.h index 4415f2fc07e176be66d69e5cd6cea81fec76fc34..72f624af46862d71ec09fd89f47a04ecb407eeef 100644 --- a/drivers/iio/accel/adxl313.h +++ b/drivers/iio/accel/adxl313.h @@ -8,6 +8,8 @@ #ifndef _ADXL313_H_ #define _ADXL313_H_ +#include + /* ADXL313 register definitions */ #define ADXL313_REG_DEVID0 0x00 #define ADXL313_REG_DEVID1 0x01 @@ -26,6 +28,7 @@ #define ADXL313_REG_FIFO_STATUS 0x39 #define ADXL313_DEVID0 0xAD +#define ADXL313_DEVID0_ADXL312_314 0xE5 #define ADXL313_DEVID1 0x1D #define ADXL313_PARTID 0xCB #define ADXL313_SOFT_RESET 0x52 @@ -37,18 +40,46 @@ #define ADXL313_MEASUREMENT_MODE BIT(3) #define ADXL313_RANGE_MSK GENMASK(1, 0) -#define ADXL313_RANGE_4G 3 +#define ADXL313_RANGE_MAX 3 #define ADXL313_FULL_RES BIT(3) #define ADXL313_SPI_3WIRE BIT(6) #define ADXL313_I2C_DISABLE BIT(6) +extern const struct regmap_access_table adxl312_readable_regs_table; extern const struct regmap_access_table adxl313_readable_regs_table; +extern const struct regmap_access_table adxl314_readable_regs_table; +extern const struct regmap_access_table adxl312_writable_regs_table; extern const struct regmap_access_table adxl313_writable_regs_table; +extern const struct regmap_access_table adxl314_writable_regs_table; + +enum adxl313_device_type { + ADXL312, + ADXL313, + ADXL314, +}; + +struct adxl313_data { + struct regmap *regmap; + const struct adxl313_chip_info *chip_info; + struct mutex lock; /* lock to protect transf_buf */ + __le16 transf_buf __aligned(IIO_DMA_MINALIGN); +}; + +struct adxl313_chip_info { + const char *name; + enum adxl313_device_type type; + int scale_factor; + bool variable_range; + bool soft_reset; + int (*check_id)(struct device *dev, struct adxl313_data *data); +}; + +extern const struct adxl313_chip_info adxl31x_chip_info[]; int adxl313_core_probe(struct device *dev, struct regmap *regmap, - const char *name, + const struct adxl313_chip_info *chip_info, int (*setup)(struct device *, struct regmap *)); #endif /* _ADXL313_H_ */ diff --git a/drivers/iio/accel/adxl313_core.c b/drivers/iio/accel/adxl313_core.c index afeef779e1d0829d700373d6d2f6f1fa9c6d6b64..4de0a41bd679674dfff8ffa2940635675fdff2ce 100644 --- a/drivers/iio/accel/adxl313_core.c +++ b/drivers/iio/accel/adxl313_core.c @@ -8,12 +8,18 @@ */ #include -#include #include #include #include "adxl313.h" +static const struct regmap_range adxl312_readable_reg_range[] = { + regmap_reg_range(ADXL313_REG_DEVID0, ADXL313_REG_DEVID0), + regmap_reg_range(ADXL313_REG_OFS_AXIS(0), ADXL313_REG_OFS_AXIS(2)), + regmap_reg_range(ADXL313_REG_THRESH_ACT, ADXL313_REG_ACT_INACT_CTL), + regmap_reg_range(ADXL313_REG_BW_RATE, ADXL313_REG_FIFO_STATUS), +}; + static const struct regmap_range adxl313_readable_reg_range[] = { regmap_reg_range(ADXL313_REG_DEVID0, ADXL313_REG_XID), regmap_reg_range(ADXL313_REG_SOFT_RESET, ADXL313_REG_SOFT_RESET), @@ -22,12 +28,109 @@ static const struct regmap_range adxl313_readable_reg_range[] = { regmap_reg_range(ADXL313_REG_BW_RATE, ADXL313_REG_FIFO_STATUS), }; +const struct regmap_access_table adxl312_readable_regs_table = { + .yes_ranges = adxl312_readable_reg_range, + .n_yes_ranges = ARRAY_SIZE(adxl312_readable_reg_range), +}; +EXPORT_SYMBOL_NS_GPL(adxl312_readable_regs_table, IIO_ADXL313); + const struct regmap_access_table adxl313_readable_regs_table = { .yes_ranges = adxl313_readable_reg_range, .n_yes_ranges = ARRAY_SIZE(adxl313_readable_reg_range), }; EXPORT_SYMBOL_NS_GPL(adxl313_readable_regs_table, IIO_ADXL313); +const struct regmap_access_table adxl314_readable_regs_table = { + .yes_ranges = adxl312_readable_reg_range, + .n_yes_ranges = ARRAY_SIZE(adxl312_readable_reg_range), +}; +EXPORT_SYMBOL_NS_GPL(adxl314_readable_regs_table, IIO_ADXL313); + +static int adxl312_check_id(struct device *dev, + struct adxl313_data *data) +{ + unsigned int regval; + int ret; + + ret = regmap_read(data->regmap, ADXL313_REG_DEVID0, ®val); + if (ret) + return ret; + + if (regval != ADXL313_DEVID0_ADXL312_314) + dev_warn(dev, "Invalid manufacturer ID: %#02x\n", regval); + + return 0; +} + +static int adxl313_check_id(struct device *dev, + struct adxl313_data *data) +{ + unsigned int regval; + int ret; + + ret = regmap_read(data->regmap, ADXL313_REG_DEVID0, ®val); + if (ret) + return ret; + + if (regval != ADXL313_DEVID0) + dev_warn(dev, "Invalid manufacturer ID: 0x%02x\n", regval); + + /* Check DEVID1 and PARTID */ + if (regval == ADXL313_DEVID0) { + ret = regmap_read(data->regmap, ADXL313_REG_DEVID1, ®val); + if (ret) + return ret; + + if (regval != ADXL313_DEVID1) + dev_warn(dev, "Invalid mems ID: 0x%02x\n", regval); + + ret = regmap_read(data->regmap, ADXL313_REG_PARTID, ®val); + if (ret) + return ret; + + if (regval != ADXL313_PARTID) + dev_warn(dev, "Invalid device ID: 0x%02x\n", regval); + } + + return 0; +} + +const struct adxl313_chip_info adxl31x_chip_info[] = { + [ADXL312] = { + .name = "adxl312", + .type = ADXL312, + .scale_factor = 28425072, + .variable_range = true, + .soft_reset = false, + .check_id = &adxl312_check_id, + }, + [ADXL313] = { + .name = "adxl313", + .type = ADXL313, + .scale_factor = 9576806, + .variable_range = true, + .soft_reset = true, + .check_id = &adxl313_check_id, + }, + [ADXL314] = { + .name = "adxl314", + .type = ADXL314, + .scale_factor = 478858719, + .variable_range = false, + .soft_reset = false, + .check_id = &adxl312_check_id, + }, +}; +EXPORT_SYMBOL_NS_GPL(adxl31x_chip_info, IIO_ADXL313); + +static const struct regmap_range adxl312_writable_reg_range[] = { + regmap_reg_range(ADXL313_REG_OFS_AXIS(0), ADXL313_REG_OFS_AXIS(2)), + regmap_reg_range(ADXL313_REG_THRESH_ACT, ADXL313_REG_ACT_INACT_CTL), + regmap_reg_range(ADXL313_REG_BW_RATE, ADXL313_REG_INT_MAP), + regmap_reg_range(ADXL313_REG_DATA_FORMAT, ADXL313_REG_DATA_FORMAT), + regmap_reg_range(ADXL313_REG_FIFO_CTL, ADXL313_REG_FIFO_CTL), +}; + static const struct regmap_range adxl313_writable_reg_range[] = { regmap_reg_range(ADXL313_REG_SOFT_RESET, ADXL313_REG_SOFT_RESET), regmap_reg_range(ADXL313_REG_OFS_AXIS(0), ADXL313_REG_OFS_AXIS(2)), @@ -37,17 +140,23 @@ static const struct regmap_range adxl313_writable_reg_range[] = { regmap_reg_range(ADXL313_REG_FIFO_CTL, ADXL313_REG_FIFO_CTL), }; +const struct regmap_access_table adxl312_writable_regs_table = { + .yes_ranges = adxl312_writable_reg_range, + .n_yes_ranges = ARRAY_SIZE(adxl312_writable_reg_range), +}; +EXPORT_SYMBOL_NS_GPL(adxl312_writable_regs_table, IIO_ADXL313); + const struct regmap_access_table adxl313_writable_regs_table = { .yes_ranges = adxl313_writable_reg_range, .n_yes_ranges = ARRAY_SIZE(adxl313_writable_reg_range), }; EXPORT_SYMBOL_NS_GPL(adxl313_writable_regs_table, IIO_ADXL313); -struct adxl313_data { - struct regmap *regmap; - struct mutex lock; /* lock to protect transf_buf */ - __le16 transf_buf __aligned(IIO_DMA_MINALIGN); +const struct regmap_access_table adxl314_writable_regs_table = { + .yes_ranges = adxl312_writable_reg_range, + .n_yes_ranges = ARRAY_SIZE(adxl312_writable_reg_range), }; +EXPORT_SYMBOL_NS_GPL(adxl314_writable_regs_table, IIO_ADXL313); static const int adxl313_odr_freqs[][2] = { [0] = { 6, 250000 }, @@ -156,12 +265,10 @@ static int adxl313_read_raw(struct iio_dev *indio_dev, *val = sign_extend32(ret, chan->scan_type.realbits - 1); return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: - /* - * Scale for any g range is given in datasheet as - * 1024 LSB/g = 0.0009765625 * 9.80665 = 0.009576806640625 m/s^2 - */ *val = 0; - *val2 = 9576806; + + *val2 = data->chip_info->scale_factor; + return IIO_VAL_INT_PLUS_NANO; case IIO_CHAN_INFO_CALIBBIAS: ret = regmap_read(data->regmap, @@ -170,7 +277,7 @@ static int adxl313_read_raw(struct iio_dev *indio_dev, return ret; /* - * 8-bit resolution at +/- 0.5g, that is 4x accel data scale + * 8-bit resolution at minimum range, that is 4x accel data scale * factor at full resolution */ *val = sign_extend32(regval, 7) * 4; @@ -198,7 +305,7 @@ static int adxl313_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_CALIBBIAS: /* - * 8-bit resolution at +/- 0.5g, that is 4x accel data scale + * 8-bit resolution at minimum range, that is 4x accel data scale * factor at full resolution */ if (clamp_val(val, -128 * 4, 127 * 4) != val) @@ -223,14 +330,18 @@ static const struct iio_info adxl313_info = { static int adxl313_setup(struct device *dev, struct adxl313_data *data, int (*setup)(struct device *, struct regmap *)) { - unsigned int regval; int ret; - /* Ensures the device is in a consistent state after start up */ - ret = regmap_write(data->regmap, ADXL313_REG_SOFT_RESET, - ADXL313_SOFT_RESET); - if (ret) - return ret; + /* + * If sw reset available, ensures the device is in a consistent + * state after start up + */ + if (data->chip_info->soft_reset) { + ret = regmap_write(data->regmap, ADXL313_REG_SOFT_RESET, + ADXL313_SOFT_RESET); + if (ret) + return ret; + } if (setup) { ret = setup(dev, data->regmap); @@ -238,46 +349,25 @@ static int adxl313_setup(struct device *dev, struct adxl313_data *data, return ret; } - ret = regmap_read(data->regmap, ADXL313_REG_DEVID0, ®val); + ret = data->chip_info->check_id(dev, data); if (ret) return ret; - if (regval != ADXL313_DEVID0) { - dev_err(dev, "Invalid manufacturer ID: 0x%02x\n", regval); - return -ENODEV; - } - - ret = regmap_read(data->regmap, ADXL313_REG_DEVID1, ®val); - if (ret) - return ret; - - if (regval != ADXL313_DEVID1) { - dev_err(dev, "Invalid mems ID: 0x%02x\n", regval); - return -ENODEV; - } - - ret = regmap_read(data->regmap, ADXL313_REG_PARTID, ®val); - if (ret) - return ret; + /* Sets the range to maximum, full resolution, if applicable */ + if (data->chip_info->variable_range) { + ret = regmap_update_bits(data->regmap, ADXL313_REG_DATA_FORMAT, + ADXL313_RANGE_MSK, + FIELD_PREP(ADXL313_RANGE_MSK, ADXL313_RANGE_MAX)); + if (ret) + return ret; - if (regval != ADXL313_PARTID) { - dev_err(dev, "Invalid device ID: 0x%02x\n", regval); - return -ENODEV; + /* Enables full resolution */ + ret = regmap_update_bits(data->regmap, ADXL313_REG_DATA_FORMAT, + ADXL313_FULL_RES, ADXL313_FULL_RES); + if (ret) + return ret; } - /* Sets the range to +/- 4g */ - ret = regmap_update_bits(data->regmap, ADXL313_REG_DATA_FORMAT, - ADXL313_RANGE_MSK, - FIELD_PREP(ADXL313_RANGE_MSK, ADXL313_RANGE_4G)); - if (ret) - return ret; - - /* Enables full resolution */ - ret = regmap_update_bits(data->regmap, ADXL313_REG_DATA_FORMAT, - ADXL313_FULL_RES, ADXL313_FULL_RES); - if (ret) - return ret; - /* Enables measurement mode */ return regmap_update_bits(data->regmap, ADXL313_REG_POWER_CTL, ADXL313_POWER_CTL_MSK, @@ -288,7 +378,7 @@ static int adxl313_setup(struct device *dev, struct adxl313_data *data, * adxl313_core_probe() - probe and setup for adxl313 accelerometer * @dev: Driver model representation of the device * @regmap: Register map of the device - * @name: Device name buffer reference + * @chip_info: Structure containing device specific data * @setup: Setup routine to be executed right before the standard device * setup, can also be set to NULL if not required * @@ -296,7 +386,7 @@ static int adxl313_setup(struct device *dev, struct adxl313_data *data, */ int adxl313_core_probe(struct device *dev, struct regmap *regmap, - const char *name, + const struct adxl313_chip_info *chip_info, int (*setup)(struct device *, struct regmap *)) { struct adxl313_data *data; @@ -309,9 +399,11 @@ int adxl313_core_probe(struct device *dev, data = iio_priv(indio_dev); data->regmap = regmap; + data->chip_info = chip_info; + mutex_init(&data->lock); - indio_dev->name = name; + indio_dev->name = chip_info->name; indio_dev->info = &adxl313_info; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = adxl313_channels; diff --git a/drivers/iio/accel/adxl313_i2c.c b/drivers/iio/accel/adxl313_i2c.c index c329765dbf60f00ae0b9835b4efe3f0d426c4b24..99cc7fc294882bcb4426756d9279372625b490e8 100644 --- a/drivers/iio/accel/adxl313_i2c.c +++ b/drivers/iio/accel/adxl313_i2c.c @@ -14,42 +14,72 @@ #include "adxl313.h" -static const struct regmap_config adxl313_i2c_regmap_config = { - .reg_bits = 8, - .val_bits = 8, - .rd_table = &adxl313_readable_regs_table, - .wr_table = &adxl313_writable_regs_table, - .max_register = 0x39, +static const struct regmap_config adxl31x_i2c_regmap_config[] = { + [ADXL312] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl312_readable_regs_table, + .wr_table = &adxl312_writable_regs_table, + .max_register = 0x39, + }, + [ADXL313] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl313_readable_regs_table, + .wr_table = &adxl313_writable_regs_table, + .max_register = 0x39, + }, + [ADXL314] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl314_readable_regs_table, + .wr_table = &adxl314_writable_regs_table, + .max_register = 0x39, + }, }; -static int adxl313_i2c_probe(struct i2c_client *client) -{ - struct regmap *regmap; - - regmap = devm_regmap_init_i2c(client, &adxl313_i2c_regmap_config); - if (IS_ERR(regmap)) { - dev_err(&client->dev, "Error initializing i2c regmap: %ld\n", - PTR_ERR(regmap)); - return PTR_ERR(regmap); - } - - return adxl313_core_probe(&client->dev, regmap, client->name, NULL); -} - static const struct i2c_device_id adxl313_i2c_id[] = { - { "adxl313" }, + { .name = "adxl312", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL312] }, + { .name = "adxl313", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL312] }, + { .name = "adxl314", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL312] }, { } }; MODULE_DEVICE_TABLE(i2c, adxl313_i2c_id); static const struct of_device_id adxl313_of_match[] = { - { .compatible = "adi,adxl313" }, + { .compatible = "adi,adxl312", .data = &adxl31x_chip_info[ADXL312] }, + { .compatible = "adi,adxl313", .data = &adxl31x_chip_info[ADXL313] }, + { .compatible = "adi,adxl314", .data = &adxl31x_chip_info[ADXL314] }, { } }; MODULE_DEVICE_TABLE(of, adxl313_of_match); +static int adxl313_i2c_probe(struct i2c_client *client) +{ + const struct adxl313_chip_info *chip_data; + struct regmap *regmap; + + /* + * Retrieves device specific data as a pointer to a + * adxl313_chip_info structure + */ + chip_data = device_get_match_data(&client->dev); + if (!chip_data) + chip_data = (const struct adxl313_chip_info *)i2c_match_id(adxl313_i2c_id, client)->driver_data; + + regmap = devm_regmap_init_i2c(client, + &adxl31x_i2c_regmap_config[chip_data->type]); + if (IS_ERR(regmap)) { + dev_err(&client->dev, "Error initializing i2c regmap: %ld\n", + PTR_ERR(regmap)); + return PTR_ERR(regmap); + } + + return adxl313_core_probe(&client->dev, regmap, chip_data, NULL); +} + static struct i2c_driver adxl313_i2c_driver = { .driver = { .name = "adxl313_i2c", diff --git a/drivers/iio/accel/adxl313_spi.c b/drivers/iio/accel/adxl313_spi.c index a3c6d553462d8867eb13e6bada77205d434b067a..b7cc15678a2b74d72e8776b1350b695375ad4b5c 100644 --- a/drivers/iio/accel/adxl313_spi.c +++ b/drivers/iio/accel/adxl313_spi.c @@ -11,17 +11,38 @@ #include #include #include +#include #include "adxl313.h" -static const struct regmap_config adxl313_spi_regmap_config = { - .reg_bits = 8, - .val_bits = 8, - .rd_table = &adxl313_readable_regs_table, - .wr_table = &adxl313_writable_regs_table, - .max_register = 0x39, - /* Setting bits 7 and 6 enables multiple-byte read */ - .read_flag_mask = BIT(7) | BIT(6), +static const struct regmap_config adxl31x_spi_regmap_config[] = { + [ADXL312] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl312_readable_regs_table, + .wr_table = &adxl312_writable_regs_table, + .max_register = 0x39, + /* Setting bits 7 and 6 enables multiple-byte read */ + .read_flag_mask = BIT(7) | BIT(6), + }, + [ADXL313] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl313_readable_regs_table, + .wr_table = &adxl313_writable_regs_table, + .max_register = 0x39, + /* Setting bits 7 and 6 enables multiple-byte read */ + .read_flag_mask = BIT(7) | BIT(6), + }, + [ADXL314] = { + .reg_bits = 8, + .val_bits = 8, + .rd_table = &adxl314_readable_regs_table, + .wr_table = &adxl314_writable_regs_table, + .max_register = 0x39, + /* Setting bits 7 and 6 enables multiple-byte read */ + .read_flag_mask = BIT(7) | BIT(6), + }, }; static int adxl313_spi_setup(struct device *dev, struct regmap *regmap) @@ -42,7 +63,7 @@ static int adxl313_spi_setup(struct device *dev, struct regmap *regmap) static int adxl313_spi_probe(struct spi_device *spi) { - const struct spi_device_id *id = spi_get_device_id(spi); + const struct adxl313_chip_info *chip_data; struct regmap *regmap; int ret; @@ -51,26 +72,40 @@ static int adxl313_spi_probe(struct spi_device *spi) if (ret) return ret; - regmap = devm_regmap_init_spi(spi, &adxl313_spi_regmap_config); + /* + * Retrieves device specific data as a pointer to a + * adxl313_chip_info structure + */ + chip_data = device_get_match_data(&spi->dev); + if (!chip_data) + chip_data = (const struct adxl313_chip_info *)spi_get_device_id(spi)->driver_data; + + regmap = devm_regmap_init_spi(spi, + &adxl31x_spi_regmap_config[chip_data->type]); + if (IS_ERR(regmap)) { dev_err(&spi->dev, "Error initializing spi regmap: %ld\n", PTR_ERR(regmap)); return PTR_ERR(regmap); } - return adxl313_core_probe(&spi->dev, regmap, id->name, - &adxl313_spi_setup); + return adxl313_core_probe(&spi->dev, regmap, + chip_data, &adxl313_spi_setup); } static const struct spi_device_id adxl313_spi_id[] = { - { "adxl313" }, + { .name = "adxl312", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL312] }, + { .name = "adxl313", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL313] }, + { .name = "adxl314", .driver_data = (kernel_ulong_t)&adxl31x_chip_info[ADXL314] }, { } }; MODULE_DEVICE_TABLE(spi, adxl313_spi_id); static const struct of_device_id adxl313_of_match[] = { - { .compatible = "adi,adxl313" }, + { .compatible = "adi,adxl312", .data = &adxl31x_chip_info[ADXL312] }, + { .compatible = "adi,adxl313", .data = &adxl31x_chip_info[ADXL313] }, + { .compatible = "adi,adxl314", .data = &adxl31x_chip_info[ADXL314] }, { } }; diff --git a/drivers/iio/accel/adxl345_core.c b/drivers/iio/accel/adxl345_core.c index 370bfec1275a156301b7f7ae172b6281dd5bd2aa..1919e0089c1156937fff00810463839c3e72dd44 100644 --- a/drivers/iio/accel/adxl345_core.c +++ b/drivers/iio/accel/adxl345_core.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,6 @@ #define ADXL345_BW_RATE GENMASK(3, 0) #define ADXL345_BASE_RATE_NANO_HZ 97656250LL -#define NHZ_PER_HZ 1000000000LL #define ADXL345_POWER_CTL_MEASURE BIT(3) #define ADXL345_POWER_CTL_STANDBY 0x00 @@ -139,7 +139,7 @@ static int adxl345_read_raw(struct iio_dev *indio_dev, samp_freq_nhz = ADXL345_BASE_RATE_NANO_HZ << (regval & ADXL345_BW_RATE); - *val = div_s64_rem(samp_freq_nhz, NHZ_PER_HZ, val2); + *val = div_s64_rem(samp_freq_nhz, NANOHZ_PER_HZ, val2); return IIO_VAL_INT_PLUS_NANO; } @@ -164,7 +164,8 @@ static int adxl345_write_raw(struct iio_dev *indio_dev, ADXL345_REG_OFS_AXIS(chan->address), val / 4); case IIO_CHAN_INFO_SAMP_FREQ: - n = div_s64(val * NHZ_PER_HZ + val2, ADXL345_BASE_RATE_NANO_HZ); + n = div_s64(val * NANOHZ_PER_HZ + val2, + ADXL345_BASE_RATE_NANO_HZ); return regmap_update_bits(data->regmap, ADXL345_REG_BW_RATE, ADXL345_BW_RATE, diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c index 9c9e985786670c12fb58213d1c1c1bafb473084f..d03fc3400f94e41b9aee585bb6c7389bb38d3ccd 100644 --- a/drivers/iio/accel/bma180.c +++ b/drivers/iio/accel/bma180.c @@ -1045,7 +1045,7 @@ err_disable_vdd: return ret; } -static int bma180_remove(struct i2c_client *client) +static void bma180_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct bma180_data *data = iio_priv(indio_dev); @@ -1062,8 +1062,6 @@ static int bma180_remove(struct i2c_client *client) mutex_unlock(&data->mutex); regulator_disable(data->vddio_supply); regulator_disable(data->vdd_supply); - - return 0; } static int bma180_suspend(struct device *dev) diff --git a/drivers/iio/accel/bma400.h b/drivers/iio/accel/bma400.h index e8f802a82300d6c09062c3fe3be7e33d87aeb296..36edbaff4f7f140ab67ad2588dcdfa0cc55cafbe 100644 --- a/drivers/iio/accel/bma400.h +++ b/drivers/iio/accel/bma400.h @@ -40,6 +40,7 @@ #define BMA400_INT_STAT1_REG 0x0f #define BMA400_INT_STAT2_REG 0x10 #define BMA400_INT12_MAP_REG 0x23 +#define BMA400_INT_ENG_OVRUN_MSK BIT(4) /* Temperature register */ #define BMA400_TEMP_DATA_REG 0x11 @@ -105,6 +106,19 @@ #define BMA400_INT_GEN2_MSK BIT(3) #define BMA400_GEN_HYST_MSK GENMASK(1, 0) +/* TAP config registers */ +#define BMA400_TAP_CONFIG 0x57 +#define BMA400_TAP_CONFIG1 0x58 +#define BMA400_S_TAP_MSK BIT(2) +#define BMA400_D_TAP_MSK BIT(3) +#define BMA400_INT_S_TAP_MSK BIT(10) +#define BMA400_INT_D_TAP_MSK BIT(11) +#define BMA400_TAP_SEN_MSK GENMASK(2, 0) +#define BMA400_TAP_TICSTH_MSK GENMASK(1, 0) +#define BMA400_TAP_QUIET_MSK GENMASK(3, 2) +#define BMA400_TAP_QUIETDT_MSK GENMASK(5, 4) +#define BMA400_TAP_TIM_LIST_LEN 4 + /* * BMA400_SCALE_MIN macro value represents m/s^2 for 1 LSB before * converting to micro values for +-2g range. diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c index c31bdd9b168e916ba6a5ea33144baa64874f5ed7..ad8fce3e08cd0d9ce0a1fafa63360ee7056c8866 100644 --- a/drivers/iio/accel/bma400_core.c +++ b/drivers/iio/accel/bma400_core.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,27 @@ static int bma400_sample_freqs[14]; static const int bma400_osr_range[] = { 0, 1, 3 }; +static int tap_reset_timeout[BMA400_TAP_TIM_LIST_LEN] = { + 300000, + 400000, + 500000, + 600000 +}; + +static int tap_max2min_time[BMA400_TAP_TIM_LIST_LEN] = { + 30000, + 45000, + 60000, + 90000 +}; + +static int double_tap2_min_delay[BMA400_TAP_TIM_LIST_LEN] = { + 20000, + 40000, + 60000, + 80000 +}; + /* See the ACC_CONFIG0 section of the datasheet */ enum bma400_power_mode { POWER_MODE_SLEEP = 0x00, @@ -88,6 +110,7 @@ struct bma400_data { bool step_event_en; bool activity_event_en; unsigned int generic_event_en; + unsigned int tap_event_en_bitmask; /* Correct time stamp alignment */ struct { __le16 buff[3]; @@ -216,6 +239,115 @@ static const struct iio_event_spec bma400_accel_event[] = { BIT(IIO_EV_INFO_HYSTERESIS) | BIT(IIO_EV_INFO_ENABLE), }, + { + .type = IIO_EV_TYPE_GESTURE, + .dir = IIO_EV_DIR_SINGLETAP, + .mask_shared_by_type = BIT(IIO_EV_INFO_VALUE) | + BIT(IIO_EV_INFO_ENABLE) | + BIT(IIO_EV_INFO_RESET_TIMEOUT), + }, + { + .type = IIO_EV_TYPE_GESTURE, + .dir = IIO_EV_DIR_DOUBLETAP, + .mask_shared_by_type = BIT(IIO_EV_INFO_VALUE) | + BIT(IIO_EV_INFO_ENABLE) | + BIT(IIO_EV_INFO_RESET_TIMEOUT) | + BIT(IIO_EV_INFO_TAP2_MIN_DELAY), + }, +}; + +static int usec_to_tapreg_raw(int usec, const int *time_list) +{ + int index; + + for (index = 0; index < BMA400_TAP_TIM_LIST_LEN; index++) { + if (usec == time_list[index]) + return index; + } + return -EINVAL; +} + +static ssize_t in_accel_gesture_tap_maxtomin_time_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct bma400_data *data = iio_priv(indio_dev); + int ret, reg_val, raw, vals[2]; + + ret = regmap_read(data->regmap, BMA400_TAP_CONFIG1, ®_val); + if (ret) + return ret; + + raw = FIELD_GET(BMA400_TAP_TICSTH_MSK, reg_val); + vals[0] = 0; + vals[1] = tap_max2min_time[raw]; + + return iio_format_value(buf, IIO_VAL_INT_PLUS_MICRO, 2, vals); +} + +static ssize_t in_accel_gesture_tap_maxtomin_time_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct bma400_data *data = iio_priv(indio_dev); + int ret, val_int, val_fract, raw; + + ret = iio_str_to_fixpoint(buf, 100000, &val_int, &val_fract); + if (ret) + return ret; + + raw = usec_to_tapreg_raw(val_fract, tap_max2min_time); + if (raw < 0) + return -EINVAL; + + ret = regmap_update_bits(data->regmap, BMA400_TAP_CONFIG1, + BMA400_TAP_TICSTH_MSK, + FIELD_PREP(BMA400_TAP_TICSTH_MSK, raw)); + if (ret) + return ret; + + return len; +} + +static IIO_DEVICE_ATTR_RW(in_accel_gesture_tap_maxtomin_time, 0); + +/* + * Tap interrupts works with 200 Hz input data rate and the time based tap + * controls are in the terms of data samples so the below calculation is + * used to convert the configuration values into seconds. + * e.g.: + * 60 data samples * 0.005 ms = 0.3 seconds. + * 80 data samples * 0.005 ms = 0.4 seconds. + */ + +/* quiet configuration values in seconds */ +static IIO_CONST_ATTR(in_accel_gesture_tap_reset_timeout_available, + "0.3 0.4 0.5 0.6"); + +/* tics_th configuration values in seconds */ +static IIO_CONST_ATTR(in_accel_gesture_tap_maxtomin_time_available, + "0.03 0.045 0.06 0.09"); + +/* quiet_dt configuration values in seconds */ +static IIO_CONST_ATTR(in_accel_gesture_doubletap_tap2_min_delay_available, + "0.02 0.04 0.06 0.08"); + +/* List of sensitivity values available to configure tap interrupts */ +static IIO_CONST_ATTR(in_accel_gesture_tap_value_available, "0 1 2 3 4 5 6 7"); + +static struct attribute *bma400_event_attributes[] = { + &iio_const_attr_in_accel_gesture_tap_value_available.dev_attr.attr, + &iio_const_attr_in_accel_gesture_tap_reset_timeout_available.dev_attr.attr, + &iio_const_attr_in_accel_gesture_tap_maxtomin_time_available.dev_attr.attr, + &iio_const_attr_in_accel_gesture_doubletap_tap2_min_delay_available.dev_attr.attr, + &iio_dev_attr_in_accel_gesture_tap_maxtomin_time.dev_attr.attr, + NULL +}; + +static const struct attribute_group bma400_event_attribute_group = { + .attrs = bma400_event_attributes, }; #define BMA400_ACC_CHANNEL(_index, _axis) { \ @@ -1012,6 +1144,12 @@ static int bma400_read_event_config(struct iio_dev *indio_dev, case IIO_EV_DIR_FALLING: return FIELD_GET(BMA400_INT_GEN2_MSK, data->generic_event_en); + case IIO_EV_DIR_SINGLETAP: + return FIELD_GET(BMA400_S_TAP_MSK, + data->tap_event_en_bitmask); + case IIO_EV_DIR_DOUBLETAP: + return FIELD_GET(BMA400_D_TAP_MSK, + data->tap_event_en_bitmask); default: return -EINVAL; } @@ -1046,7 +1184,8 @@ static int bma400_activity_event_en(struct bma400_data *data, enum iio_event_direction dir, int state) { - int ret, reg, msk, value, field_value; + int ret, reg, msk, value; + int field_value = 0; switch (dir) { case IIO_EV_DIR_RISING: @@ -1101,6 +1240,80 @@ static int bma400_activity_event_en(struct bma400_data *data, return 0; } +static int bma400_tap_event_en(struct bma400_data *data, + enum iio_event_direction dir, int state) +{ + unsigned int mask, field_value; + int ret; + + /* + * Tap interrupts can be configured only in normal mode. + * See table in section 4.3 "Power modes - performance modes" of + * datasheet v1.2. + */ + if (data->power_mode != POWER_MODE_NORMAL) + return -EINVAL; + + /* + * Tap interrupts are operating with a data rate of 200Hz. + * See section 4.7 "Tap sensing interrupt" in datasheet v1.2. + */ + if (data->sample_freq.hz != 200 && state) { + dev_err(data->dev, "Invalid data rate for tap interrupts.\n"); + return -EINVAL; + } + + ret = regmap_update_bits(data->regmap, BMA400_INT12_MAP_REG, + BMA400_S_TAP_MSK, + FIELD_PREP(BMA400_S_TAP_MSK, state)); + if (ret) + return ret; + + switch (dir) { + case IIO_EV_DIR_SINGLETAP: + mask = BMA400_S_TAP_MSK; + set_mask_bits(&field_value, BMA400_S_TAP_MSK, + FIELD_PREP(BMA400_S_TAP_MSK, state)); + break; + case IIO_EV_DIR_DOUBLETAP: + mask = BMA400_D_TAP_MSK; + set_mask_bits(&field_value, BMA400_D_TAP_MSK, + FIELD_PREP(BMA400_D_TAP_MSK, state)); + break; + default: + return -EINVAL; + } + + ret = regmap_update_bits(data->regmap, BMA400_INT_CONFIG1_REG, mask, + field_value); + if (ret) + return ret; + + set_mask_bits(&data->tap_event_en_bitmask, mask, field_value); + + return 0; +} + +static int bma400_disable_adv_interrupt(struct bma400_data *data) +{ + int ret; + + ret = regmap_write(data->regmap, BMA400_INT_CONFIG0_REG, 0); + if (ret) + return ret; + + ret = regmap_write(data->regmap, BMA400_INT_CONFIG1_REG, 0); + if (ret) + return ret; + + data->tap_event_en_bitmask = 0; + data->generic_event_en = 0; + data->step_event_en = false; + data->activity_event_en = false; + + return 0; +} + static int bma400_write_event_config(struct iio_dev *indio_dev, const struct iio_chan_spec *chan, enum iio_event_type type, @@ -1111,10 +1324,20 @@ static int bma400_write_event_config(struct iio_dev *indio_dev, switch (chan->type) { case IIO_ACCEL: - mutex_lock(&data->mutex); - ret = bma400_activity_event_en(data, dir, state); - mutex_unlock(&data->mutex); - return ret; + switch (type) { + case IIO_EV_TYPE_MAG: + mutex_lock(&data->mutex); + ret = bma400_activity_event_en(data, dir, state); + mutex_unlock(&data->mutex); + return ret; + case IIO_EV_TYPE_GESTURE: + mutex_lock(&data->mutex); + ret = bma400_tap_event_en(data, dir, state); + mutex_unlock(&data->mutex); + return ret; + default: + return -EINVAL; + } case IIO_STEPS: mutex_lock(&data->mutex); ret = bma400_steps_event_enable(data, state); @@ -1157,10 +1380,13 @@ static int bma400_read_event_value(struct iio_dev *indio_dev, int *val, int *val2) { struct bma400_data *data = iio_priv(indio_dev); - int ret, reg; + int ret, reg, reg_val, raw; - switch (chan->type) { - case IIO_ACCEL: + if (chan->type != IIO_ACCEL) + return -EINVAL; + + switch (type) { + case IIO_EV_TYPE_MAG: reg = get_gen_config_reg(dir); if (reg < 0) return -EINVAL; @@ -1196,6 +1422,39 @@ static int bma400_read_event_value(struct iio_dev *indio_dev, default: return -EINVAL; } + case IIO_EV_TYPE_GESTURE: + switch (info) { + case IIO_EV_INFO_VALUE: + ret = regmap_read(data->regmap, BMA400_TAP_CONFIG, + ®_val); + if (ret) + return ret; + + *val = FIELD_GET(BMA400_TAP_SEN_MSK, reg_val); + return IIO_VAL_INT; + case IIO_EV_INFO_RESET_TIMEOUT: + ret = regmap_read(data->regmap, BMA400_TAP_CONFIG1, + ®_val); + if (ret) + return ret; + + raw = FIELD_GET(BMA400_TAP_QUIET_MSK, reg_val); + *val = 0; + *val2 = tap_reset_timeout[raw]; + return IIO_VAL_INT_PLUS_MICRO; + case IIO_EV_INFO_TAP2_MIN_DELAY: + ret = regmap_read(data->regmap, BMA400_TAP_CONFIG1, + ®_val); + if (ret) + return ret; + + raw = FIELD_GET(BMA400_TAP_QUIETDT_MSK, reg_val); + *val = 0; + *val2 = double_tap2_min_delay[raw]; + return IIO_VAL_INT_PLUS_MICRO; + default: + return -EINVAL; + } default: return -EINVAL; } @@ -1209,10 +1468,13 @@ static int bma400_write_event_value(struct iio_dev *indio_dev, int val, int val2) { struct bma400_data *data = iio_priv(indio_dev); - int reg, ret; + int reg, ret, raw; - switch (chan->type) { - case IIO_ACCEL: + if (chan->type != IIO_ACCEL) + return -EINVAL; + + switch (type) { + case IIO_EV_TYPE_MAG: reg = get_gen_config_reg(dir); if (reg < 0) return -EINVAL; @@ -1248,6 +1510,40 @@ static int bma400_write_event_value(struct iio_dev *indio_dev, default: return -EINVAL; } + case IIO_EV_TYPE_GESTURE: + switch (info) { + case IIO_EV_INFO_VALUE: + if (val < 0 || val > 7) + return -EINVAL; + + return regmap_update_bits(data->regmap, + BMA400_TAP_CONFIG, + BMA400_TAP_SEN_MSK, + FIELD_PREP(BMA400_TAP_SEN_MSK, + val)); + case IIO_EV_INFO_RESET_TIMEOUT: + raw = usec_to_tapreg_raw(val2, tap_reset_timeout); + if (raw < 0) + return -EINVAL; + + return regmap_update_bits(data->regmap, + BMA400_TAP_CONFIG1, + BMA400_TAP_QUIET_MSK, + FIELD_PREP(BMA400_TAP_QUIET_MSK, + raw)); + case IIO_EV_INFO_TAP2_MIN_DELAY: + raw = usec_to_tapreg_raw(val2, double_tap2_min_delay); + if (raw < 0) + return -EINVAL; + + return regmap_update_bits(data->regmap, + BMA400_TAP_CONFIG1, + BMA400_TAP_QUIETDT_MSK, + FIELD_PREP(BMA400_TAP_QUIETDT_MSK, + raw)); + default: + return -EINVAL; + } default: return -EINVAL; } @@ -1287,6 +1583,7 @@ static const struct iio_info bma400_info = { .write_event_config = bma400_write_event_config, .write_event_value = bma400_write_event_value, .read_event_value = bma400_read_event_value, + .event_attrs = &bma400_event_attribute_group, }; static const struct iio_trigger_ops bma400_trigger_ops = { @@ -1350,6 +1647,32 @@ static irqreturn_t bma400_interrupt(int irq, void *private) if (ret || !data->status) goto unlock_err; + /* + * Disable all advance interrupts if interrupt engine overrun occurs. + * See section 4.7 "Interrupt engine overrun" in datasheet v1.2. + */ + if (FIELD_GET(BMA400_INT_ENG_OVRUN_MSK, le16_to_cpu(data->status))) { + bma400_disable_adv_interrupt(data); + dev_err(data->dev, "Interrupt engine overrun\n"); + goto unlock_err; + } + + if (FIELD_GET(BMA400_INT_S_TAP_MSK, le16_to_cpu(data->status))) + iio_push_event(indio_dev, + IIO_MOD_EVENT_CODE(IIO_ACCEL, 0, + IIO_MOD_X_OR_Y_OR_Z, + IIO_EV_TYPE_GESTURE, + IIO_EV_DIR_SINGLETAP), + timestamp); + + if (FIELD_GET(BMA400_INT_D_TAP_MSK, le16_to_cpu(data->status))) + iio_push_event(indio_dev, + IIO_MOD_EVENT_CODE(IIO_ACCEL, 0, + IIO_MOD_X_OR_Y_OR_Z, + IIO_EV_TYPE_GESTURE, + IIO_EV_DIR_DOUBLETAP), + timestamp); + if (FIELD_GET(BMA400_INT_GEN1_MSK, le16_to_cpu(data->status))) ev_dir = IIO_EV_DIR_RISING; @@ -1467,5 +1790,6 @@ int bma400_probe(struct device *dev, struct regmap *regmap, int irq, EXPORT_SYMBOL_NS(bma400_probe, IIO_BMA400); MODULE_AUTHOR("Dan Robertson "); +MODULE_AUTHOR("Jagath Jog J "); MODULE_DESCRIPTION("Bosch BMA400 triaxial acceleration sensor core"); MODULE_LICENSE("GPL"); diff --git a/drivers/iio/accel/bmc150-accel-i2c.c b/drivers/iio/accel/bmc150-accel-i2c.c index dff4d7dd101c678f9f0fc4f8399fde07f30fdf14..be8cc598b88e3071c7d09c7b5e2afb8ab45eeef5 100644 --- a/drivers/iio/accel/bmc150-accel-i2c.c +++ b/drivers/iio/accel/bmc150-accel-i2c.c @@ -209,13 +209,11 @@ static int bmc150_accel_probe(struct i2c_client *client, return 0; } -static int bmc150_accel_remove(struct i2c_client *client) +static void bmc150_accel_remove(struct i2c_client *client) { bmc150_acpi_dual_accel_remove(client); bmc150_accel_core_remove(&client->dev); - - return 0; } static const struct acpi_device_id bmc150_accel_acpi_match[] = { diff --git a/drivers/iio/accel/bmi088-accel-core.c b/drivers/iio/accel/bmi088-accel-core.c index bca4cf98bf4dffe114c06c3cd35c85ec8a830a47..84edcc78d7961c2a652a3442e6da0d71fa5937bd 100644 --- a/drivers/iio/accel/bmi088-accel-core.c +++ b/drivers/iio/accel/bmi088-accel-core.c @@ -606,7 +606,7 @@ void bmi088_accel_core_remove(struct device *dev) } EXPORT_SYMBOL_NS_GPL(bmi088_accel_core_remove, IIO_BMI088); -static int __maybe_unused bmi088_accel_runtime_suspend(struct device *dev) +static int bmi088_accel_runtime_suspend(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct bmi088_accel_data *data = iio_priv(indio_dev); @@ -614,7 +614,7 @@ static int __maybe_unused bmi088_accel_runtime_suspend(struct device *dev) return bmi088_accel_power_down(data); } -static int __maybe_unused bmi088_accel_runtime_resume(struct device *dev) +static int bmi088_accel_runtime_resume(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct bmi088_accel_data *data = iio_priv(indio_dev); @@ -622,13 +622,10 @@ static int __maybe_unused bmi088_accel_runtime_resume(struct device *dev) return bmi088_accel_power_up(data); } -const struct dev_pm_ops bmi088_accel_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(bmi088_accel_runtime_suspend, - bmi088_accel_runtime_resume, NULL) -}; -EXPORT_SYMBOL_NS_GPL(bmi088_accel_pm_ops, IIO_BMI088); +EXPORT_NS_GPL_RUNTIME_DEV_PM_OPS(bmi088_accel_pm_ops, + bmi088_accel_runtime_suspend, + bmi088_accel_runtime_resume, NULL, + IIO_BMI088); MODULE_AUTHOR("Niek van Agt "); MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/accel/bmi088-accel-spi.c b/drivers/iio/accel/bmi088-accel-spi.c index 9e2ed3bd5661975f026aa129770c0381c1b08631..ee540edd84126b731f8731eef5ca1501b35d9aee 100644 --- a/drivers/iio/accel/bmi088-accel-spi.c +++ b/drivers/iio/accel/bmi088-accel-spi.c @@ -80,7 +80,7 @@ MODULE_DEVICE_TABLE(spi, bmi088_accel_id); static struct spi_driver bmi088_accel_driver = { .driver = { .name = "bmi088_accel_spi", - .pm = &bmi088_accel_pm_ops, + .pm = pm_ptr(&bmi088_accel_pm_ops), .of_match_table = bmi088_of_match, }, .probe = bmi088_accel_probe, diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c index 748b35c2f0c37300e2fe971618d23680a8c92dc4..adc66b3615c0b4d14de88f24aa57fa4392e6442a 100644 --- a/drivers/iio/accel/kxcjk-1013.c +++ b/drivers/iio/accel/kxcjk-1013.c @@ -1064,7 +1064,7 @@ static int kxcjk1013_write_event_config(struct iio_dev *indio_dev, /* * We will expect the enable and disable to do operation in - * in reverse order. This will happen here anyway as our + * reverse order. This will happen here anyway as our * resume operation uses sync mode runtime pm calls, the * suspend operation will be delayed by autosuspend delay * So the disable operation will still happen in reverse of @@ -1611,7 +1611,7 @@ err_poweroff: return ret; } -static int kxcjk1013_remove(struct i2c_client *client) +static void kxcjk1013_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct kxcjk1013_data *data = iio_priv(indio_dev); @@ -1630,8 +1630,6 @@ static int kxcjk1013_remove(struct i2c_client *client) mutex_lock(&data->mutex); kxcjk1013_set_mode(data, STANDBY); mutex_unlock(&data->mutex); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/iio/accel/kxsd9-i2c.c b/drivers/iio/accel/kxsd9-i2c.c index d57f264bd6c8061b690b49b057c0402c9c142cb4..61346ea8ef194d448001745af422004809104f13 100644 --- a/drivers/iio/accel/kxsd9-i2c.c +++ b/drivers/iio/accel/kxsd9-i2c.c @@ -32,11 +32,9 @@ static int kxsd9_i2c_probe(struct i2c_client *i2c, i2c->name); } -static int kxsd9_i2c_remove(struct i2c_client *client) +static void kxsd9_i2c_remove(struct i2c_client *client) { kxsd9_common_remove(&client->dev); - - return 0; } static const struct of_device_id kxsd9_of_match[] = { diff --git a/drivers/iio/accel/mc3230.c b/drivers/iio/accel/mc3230.c index c15d16e7f1da285ba14831fd6e53bd0bc6bfea38..2462000e05198455eae1f170da516f753637df91 100644 --- a/drivers/iio/accel/mc3230.c +++ b/drivers/iio/accel/mc3230.c @@ -151,15 +151,13 @@ static int mc3230_probe(struct i2c_client *client, return ret; } -static int mc3230_remove(struct i2c_client *client) +static void mc3230_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); iio_device_unregister(indio_dev); mc3230_set_opcon(iio_priv(indio_dev), MC3230_MODE_OPCON_STANDBY); - - return 0; } static int mc3230_suspend(struct device *dev) diff --git a/drivers/iio/accel/mma7455_i2c.c b/drivers/iio/accel/mma7455_i2c.c index a3b84e8a3ea8515912c9b39fbc6b21d6daa85820..c63b321b01cda596391ecab4dfe77cb623dc8549 100644 --- a/drivers/iio/accel/mma7455_i2c.c +++ b/drivers/iio/accel/mma7455_i2c.c @@ -26,11 +26,9 @@ static int mma7455_i2c_probe(struct i2c_client *i2c, return mma7455_core_probe(&i2c->dev, regmap, name); } -static int mma7455_i2c_remove(struct i2c_client *i2c) +static void mma7455_i2c_remove(struct i2c_client *i2c) { mma7455_core_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id mma7455_i2c_ids[] = { diff --git a/drivers/iio/accel/mma7660.c b/drivers/iio/accel/mma7660.c index 794f2f383303c4ea4ea0f49eef52165ae0fc5268..85829990bbadad92a708f54cae3a4733ad7c5ad5 100644 --- a/drivers/iio/accel/mma7660.c +++ b/drivers/iio/accel/mma7660.c @@ -207,7 +207,7 @@ static int mma7660_probe(struct i2c_client *client, return ret; } -static int mma7660_remove(struct i2c_client *client) +static void mma7660_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); int ret; @@ -218,8 +218,6 @@ static int mma7660_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to put device in stand-by mode (%pe), ignoring\n", ERR_PTR(ret)); - - return 0; } static int mma7660_suspend(struct device *dev) diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c index c7d9ca96dbaa2a729ad305a4356ca6bd951e5897..3ba28c2ff68a368306ce6f42a4baec4181cc41b5 100644 --- a/drivers/iio/accel/mma8452.c +++ b/drivers/iio/accel/mma8452.c @@ -1735,7 +1735,7 @@ disable_regulator_vdd: return ret; } -static int mma8452_remove(struct i2c_client *client) +static void mma8452_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mma8452_data *data = iio_priv(indio_dev); @@ -1751,8 +1751,6 @@ static int mma8452_remove(struct i2c_client *client) regulator_disable(data->vddio_reg); regulator_disable(data->vdd_reg); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/iio/accel/mma9551.c b/drivers/iio/accel/mma9551.c index 123cdbbb265c167ec77badbf462a32c92d0b527e..f7a793f4a8e342dcdf111fa62746eee611df583e 100644 --- a/drivers/iio/accel/mma9551.c +++ b/drivers/iio/accel/mma9551.c @@ -509,7 +509,7 @@ out_poweroff: return ret; } -static int mma9551_remove(struct i2c_client *client) +static void mma9551_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mma9551_data *data = iio_priv(indio_dev); @@ -522,8 +522,6 @@ static int mma9551_remove(struct i2c_client *client) mutex_lock(&data->mutex); mma9551_set_device_state(data->client, false); mutex_unlock(&data->mutex); - - return 0; } static int mma9551_runtime_suspend(struct device *dev) diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c index 09df58d4be331e5af50a02a57c392ee49d50d648..2da0e005b13eb73d21224683733c277a0942ad57 100644 --- a/drivers/iio/accel/mma9553.c +++ b/drivers/iio/accel/mma9553.c @@ -1148,7 +1148,7 @@ out_poweroff: return ret; } -static int mma9553_remove(struct i2c_client *client) +static void mma9553_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mma9553_data *data = iio_priv(indio_dev); @@ -1161,8 +1161,6 @@ static int mma9553_remove(struct i2c_client *client) mutex_lock(&data->mutex); mma9551_set_device_state(data->client, false); mutex_unlock(&data->mutex); - - return 0; } static int mma9553_runtime_suspend(struct device *dev) diff --git a/drivers/iio/accel/msa311.c b/drivers/iio/accel/msa311.c new file mode 100644 index 0000000000000000000000000000000000000000..2fded37591717b7090dd996e5a8c85b9bdd7b8fa --- /dev/null +++ b/drivers/iio/accel/msa311.c @@ -0,0 +1,1321 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MEMSensing digital 3-Axis accelerometer + * + * MSA311 is a tri-axial, low-g accelerometer with I2C digital output for + * sensitivity consumer applications. It has dynamic user-selectable full + * scales range of +-2g/+-4g/+-8g/+-16g and allows acceleration measurements + * with output data rates from 1Hz to 1000Hz. + * + * MSA311 is available in an ultra small (2mm x 2mm, height 0.95mm) LGA package + * and is guaranteed to operate over -40C to +85C. + * + * This driver supports following MSA311 features: + * - IIO interface + * - Different power modes: NORMAL, SUSPEND + * - ODR (Output Data Rate) selection + * - Scale selection + * - IIO triggered buffer + * - NEW_DATA interrupt + trigger + * + * Below features to be done: + * - Motion Events: ACTIVE, TAP, ORIENT, FREEFALL + * - Low Power mode + * + * Copyright (c) 2022, SberDevices. All Rights Reserved. + * + * Author: Dmitry Rokosov + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define MSA311_SOFT_RESET_REG 0x00 +#define MSA311_PARTID_REG 0x01 +#define MSA311_ACC_X_REG 0x02 +#define MSA311_ACC_Y_REG 0x04 +#define MSA311_ACC_Z_REG 0x06 +#define MSA311_MOTION_INT_REG 0x09 +#define MSA311_DATA_INT_REG 0x0A +#define MSA311_TAP_ACTIVE_STS_REG 0x0B +#define MSA311_ORIENT_STS_REG 0x0C +#define MSA311_RANGE_REG 0x0F +#define MSA311_ODR_REG 0x10 +#define MSA311_PWR_MODE_REG 0x11 +#define MSA311_SWAP_POLARITY_REG 0x12 +#define MSA311_INT_SET_0_REG 0x16 +#define MSA311_INT_SET_1_REG 0x17 +#define MSA311_INT_MAP_0_REG 0x19 +#define MSA311_INT_MAP_1_REG 0x1A +#define MSA311_INT_CONFIG_REG 0x20 +#define MSA311_INT_LATCH_REG 0x21 +#define MSA311_FREEFALL_DUR_REG 0x22 +#define MSA311_FREEFALL_TH_REG 0x23 +#define MSA311_FREEFALL_HY_REG 0x24 +#define MSA311_ACTIVE_DUR_REG 0x27 +#define MSA311_ACTIVE_TH_REG 0x28 +#define MSA311_TAP_DUR_REG 0x2A +#define MSA311_TAP_TH_REG 0x2B +#define MSA311_ORIENT_HY_REG 0x2C +#define MSA311_Z_BLOCK_REG 0x2D +#define MSA311_OFFSET_X_REG 0x38 +#define MSA311_OFFSET_Y_REG 0x39 +#define MSA311_OFFSET_Z_REG 0x3A + +enum msa311_fields { + /* Soft_Reset */ + F_SOFT_RESET_I2C, F_SOFT_RESET_SPI, + /* Motion_Interrupt */ + F_ORIENT_INT, F_S_TAP_INT, F_D_TAP_INT, F_ACTIVE_INT, F_FREEFALL_INT, + /* Data_Interrupt */ + F_NEW_DATA_INT, + /* Tap_Active_Status */ + F_TAP_SIGN, F_TAP_FIRST_X, F_TAP_FIRST_Y, F_TAP_FIRST_Z, F_ACTV_SIGN, + F_ACTV_FIRST_X, F_ACTV_FIRST_Y, F_ACTV_FIRST_Z, + /* Orientation_Status */ + F_ORIENT_Z, F_ORIENT_X_Y, + /* Range */ + F_FS, + /* ODR */ + F_X_AXIS_DIS, F_Y_AXIS_DIS, F_Z_AXIS_DIS, F_ODR, + /* Power Mode/Bandwidth */ + F_PWR_MODE, F_LOW_POWER_BW, + /* Swap_Polarity */ + F_X_POLARITY, F_Y_POLARITY, F_Z_POLARITY, F_X_Y_SWAP, + /* Int_Set_0 */ + F_ORIENT_INT_EN, F_S_TAP_INT_EN, F_D_TAP_INT_EN, F_ACTIVE_INT_EN_Z, + F_ACTIVE_INT_EN_Y, F_ACTIVE_INT_EN_X, + /* Int_Set_1 */ + F_NEW_DATA_INT_EN, F_FREEFALL_INT_EN, + /* Int_Map_0 */ + F_INT1_ORIENT, F_INT1_S_TAP, F_INT1_D_TAP, F_INT1_ACTIVE, + F_INT1_FREEFALL, + /* Int_Map_1 */ + F_INT1_NEW_DATA, + /* Int_Config */ + F_INT1_OD, F_INT1_LVL, + /* Int_Latch */ + F_RESET_INT, F_LATCH_INT, + /* Freefall_Hy */ + F_FREEFALL_MODE, F_FREEFALL_HY, + /* Active_Dur */ + F_ACTIVE_DUR, + /* Tap_Dur */ + F_TAP_QUIET, F_TAP_SHOCK, F_TAP_DUR, + /* Tap_Th */ + F_TAP_TH, + /* Orient_Hy */ + F_ORIENT_HYST, F_ORIENT_BLOCKING, F_ORIENT_MODE, + /* Z_Block */ + F_Z_BLOCKING, + /* End of register map */ + F_MAX_FIELDS, +}; + +static const struct reg_field msa311_reg_fields[] = { + /* Soft_Reset */ + [F_SOFT_RESET_I2C] = REG_FIELD(MSA311_SOFT_RESET_REG, 2, 2), + [F_SOFT_RESET_SPI] = REG_FIELD(MSA311_SOFT_RESET_REG, 5, 5), + /* Motion_Interrupt */ + [F_ORIENT_INT] = REG_FIELD(MSA311_MOTION_INT_REG, 6, 6), + [F_S_TAP_INT] = REG_FIELD(MSA311_MOTION_INT_REG, 5, 5), + [F_D_TAP_INT] = REG_FIELD(MSA311_MOTION_INT_REG, 4, 4), + [F_ACTIVE_INT] = REG_FIELD(MSA311_MOTION_INT_REG, 2, 2), + [F_FREEFALL_INT] = REG_FIELD(MSA311_MOTION_INT_REG, 0, 0), + /* Data_Interrupt */ + [F_NEW_DATA_INT] = REG_FIELD(MSA311_DATA_INT_REG, 0, 0), + /* Tap_Active_Status */ + [F_TAP_SIGN] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 7, 7), + [F_TAP_FIRST_X] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 6, 6), + [F_TAP_FIRST_Y] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 5, 5), + [F_TAP_FIRST_Z] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 4, 4), + [F_ACTV_SIGN] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 3, 3), + [F_ACTV_FIRST_X] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 2, 2), + [F_ACTV_FIRST_Y] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 1, 1), + [F_ACTV_FIRST_Z] = REG_FIELD(MSA311_TAP_ACTIVE_STS_REG, 0, 0), + /* Orientation_Status */ + [F_ORIENT_Z] = REG_FIELD(MSA311_ORIENT_STS_REG, 6, 6), + [F_ORIENT_X_Y] = REG_FIELD(MSA311_ORIENT_STS_REG, 4, 5), + /* Range */ + [F_FS] = REG_FIELD(MSA311_RANGE_REG, 0, 1), + /* ODR */ + [F_X_AXIS_DIS] = REG_FIELD(MSA311_ODR_REG, 7, 7), + [F_Y_AXIS_DIS] = REG_FIELD(MSA311_ODR_REG, 6, 6), + [F_Z_AXIS_DIS] = REG_FIELD(MSA311_ODR_REG, 5, 5), + [F_ODR] = REG_FIELD(MSA311_ODR_REG, 0, 3), + /* Power Mode/Bandwidth */ + [F_PWR_MODE] = REG_FIELD(MSA311_PWR_MODE_REG, 6, 7), + [F_LOW_POWER_BW] = REG_FIELD(MSA311_PWR_MODE_REG, 1, 4), + /* Swap_Polarity */ + [F_X_POLARITY] = REG_FIELD(MSA311_SWAP_POLARITY_REG, 3, 3), + [F_Y_POLARITY] = REG_FIELD(MSA311_SWAP_POLARITY_REG, 2, 2), + [F_Z_POLARITY] = REG_FIELD(MSA311_SWAP_POLARITY_REG, 1, 1), + [F_X_Y_SWAP] = REG_FIELD(MSA311_SWAP_POLARITY_REG, 0, 0), + /* Int_Set_0 */ + [F_ORIENT_INT_EN] = REG_FIELD(MSA311_INT_SET_0_REG, 6, 6), + [F_S_TAP_INT_EN] = REG_FIELD(MSA311_INT_SET_0_REG, 5, 5), + [F_D_TAP_INT_EN] = REG_FIELD(MSA311_INT_SET_0_REG, 4, 4), + [F_ACTIVE_INT_EN_Z] = REG_FIELD(MSA311_INT_SET_0_REG, 2, 2), + [F_ACTIVE_INT_EN_Y] = REG_FIELD(MSA311_INT_SET_0_REG, 1, 1), + [F_ACTIVE_INT_EN_X] = REG_FIELD(MSA311_INT_SET_0_REG, 0, 0), + /* Int_Set_1 */ + [F_NEW_DATA_INT_EN] = REG_FIELD(MSA311_INT_SET_1_REG, 4, 4), + [F_FREEFALL_INT_EN] = REG_FIELD(MSA311_INT_SET_1_REG, 3, 3), + /* Int_Map_0 */ + [F_INT1_ORIENT] = REG_FIELD(MSA311_INT_MAP_0_REG, 6, 6), + [F_INT1_S_TAP] = REG_FIELD(MSA311_INT_MAP_0_REG, 5, 5), + [F_INT1_D_TAP] = REG_FIELD(MSA311_INT_MAP_0_REG, 4, 4), + [F_INT1_ACTIVE] = REG_FIELD(MSA311_INT_MAP_0_REG, 2, 2), + [F_INT1_FREEFALL] = REG_FIELD(MSA311_INT_MAP_0_REG, 0, 0), + /* Int_Map_1 */ + [F_INT1_NEW_DATA] = REG_FIELD(MSA311_INT_MAP_1_REG, 0, 0), + /* Int_Config */ + [F_INT1_OD] = REG_FIELD(MSA311_INT_CONFIG_REG, 1, 1), + [F_INT1_LVL] = REG_FIELD(MSA311_INT_CONFIG_REG, 0, 0), + /* Int_Latch */ + [F_RESET_INT] = REG_FIELD(MSA311_INT_LATCH_REG, 7, 7), + [F_LATCH_INT] = REG_FIELD(MSA311_INT_LATCH_REG, 0, 3), + /* Freefall_Hy */ + [F_FREEFALL_MODE] = REG_FIELD(MSA311_FREEFALL_HY_REG, 2, 2), + [F_FREEFALL_HY] = REG_FIELD(MSA311_FREEFALL_HY_REG, 0, 1), + /* Active_Dur */ + [F_ACTIVE_DUR] = REG_FIELD(MSA311_ACTIVE_DUR_REG, 0, 1), + /* Tap_Dur */ + [F_TAP_QUIET] = REG_FIELD(MSA311_TAP_DUR_REG, 7, 7), + [F_TAP_SHOCK] = REG_FIELD(MSA311_TAP_DUR_REG, 6, 6), + [F_TAP_DUR] = REG_FIELD(MSA311_TAP_DUR_REG, 0, 2), + /* Tap_Th */ + [F_TAP_TH] = REG_FIELD(MSA311_TAP_TH_REG, 0, 4), + /* Orient_Hy */ + [F_ORIENT_HYST] = REG_FIELD(MSA311_ORIENT_HY_REG, 4, 6), + [F_ORIENT_BLOCKING] = REG_FIELD(MSA311_ORIENT_HY_REG, 2, 3), + [F_ORIENT_MODE] = REG_FIELD(MSA311_ORIENT_HY_REG, 0, 1), + /* Z_Block */ + [F_Z_BLOCKING] = REG_FIELD(MSA311_Z_BLOCK_REG, 0, 3), +}; + +#define MSA311_WHO_AM_I 0x13 + +/* + * Possible Full Scale ranges + * + * Axis data is 12-bit signed value, so + * + * fs0 = (2 + 2) * 9.81 / (2^11) = 0.009580 + * fs1 = (4 + 4) * 9.81 / (2^11) = 0.019160 + * fs2 = (8 + 8) * 9.81 / (2^11) = 0.038320 + * fs3 = (16 + 16) * 9.81 / (2^11) = 0.076641 + */ +enum { + MSA311_FS_2G, + MSA311_FS_4G, + MSA311_FS_8G, + MSA311_FS_16G, +}; + +struct iio_decimal_fract { + int integral; + int microfract; +}; + +static const struct iio_decimal_fract msa311_fs_table[] = { + {0, 9580}, {0, 19160}, {0, 38320}, {0, 76641}, +}; + +/* Possible Output Data Rate values */ +enum { + MSA311_ODR_1_HZ, + MSA311_ODR_1_95_HZ, + MSA311_ODR_3_9_HZ, + MSA311_ODR_7_81_HZ, + MSA311_ODR_15_63_HZ, + MSA311_ODR_31_25_HZ, + MSA311_ODR_62_5_HZ, + MSA311_ODR_125_HZ, + MSA311_ODR_250_HZ, + MSA311_ODR_500_HZ, + MSA311_ODR_1000_HZ, +}; + +static const struct iio_decimal_fract msa311_odr_table[] = { + {1, 0}, {1, 950000}, {3, 900000}, {7, 810000}, {15, 630000}, + {31, 250000}, {62, 500000}, {125, 0}, {250, 0}, {500, 0}, {1000, 0}, +}; + +/* All supported power modes */ +#define MSA311_PWR_MODE_NORMAL 0b00 +#define MSA311_PWR_MODE_LOW 0b01 +#define MSA311_PWR_MODE_UNKNOWN 0b10 +#define MSA311_PWR_MODE_SUSPEND 0b11 +static const char * const msa311_pwr_modes[] = { + [MSA311_PWR_MODE_NORMAL] = "normal", + [MSA311_PWR_MODE_LOW] = "low", + [MSA311_PWR_MODE_UNKNOWN] = "unknown", + [MSA311_PWR_MODE_SUSPEND] = "suspend", +}; + +/* Autosuspend delay */ +#define MSA311_PWR_SLEEP_DELAY_MS 2000 + +/* Possible INT1 types and levels */ +enum { + MSA311_INT1_OD_PUSH_PULL, + MSA311_INT1_OD_OPEN_DRAIN, +}; + +enum { + MSA311_INT1_LVL_LOW, + MSA311_INT1_LVL_HIGH, +}; + +/* Latch INT modes */ +#define MSA311_LATCH_INT_NOT_LATCHED 0b0000 +#define MSA311_LATCH_INT_250MS 0b0001 +#define MSA311_LATCH_INT_500MS 0b0010 +#define MSA311_LATCH_INT_1S 0b0011 +#define MSA311_LATCH_INT_2S 0b0100 +#define MSA311_LATCH_INT_4S 0b0101 +#define MSA311_LATCH_INT_8S 0b0110 +#define MSA311_LATCH_INT_1MS 0b1010 +#define MSA311_LATCH_INT_2MS 0b1011 +#define MSA311_LATCH_INT_25MS 0b1100 +#define MSA311_LATCH_INT_50MS 0b1101 +#define MSA311_LATCH_INT_100MS 0b1110 +#define MSA311_LATCH_INT_LATCHED 0b0111 + +static const struct regmap_range msa311_readonly_registers[] = { + regmap_reg_range(MSA311_PARTID_REG, MSA311_ORIENT_STS_REG), +}; + +static const struct regmap_access_table msa311_writeable_table = { + .no_ranges = msa311_readonly_registers, + .n_no_ranges = ARRAY_SIZE(msa311_readonly_registers), +}; + +static const struct regmap_range msa311_writeonly_registers[] = { + regmap_reg_range(MSA311_SOFT_RESET_REG, MSA311_SOFT_RESET_REG), +}; + +static const struct regmap_access_table msa311_readable_table = { + .no_ranges = msa311_writeonly_registers, + .n_no_ranges = ARRAY_SIZE(msa311_writeonly_registers), +}; + +static const struct regmap_range msa311_volatile_registers[] = { + regmap_reg_range(MSA311_ACC_X_REG, MSA311_ORIENT_STS_REG), +}; + +static const struct regmap_access_table msa311_volatile_table = { + .yes_ranges = msa311_volatile_registers, + .n_yes_ranges = ARRAY_SIZE(msa311_volatile_registers), +}; + +static const struct regmap_config msa311_regmap_config = { + .name = "msa311", + .reg_bits = 8, + .val_bits = 8, + .max_register = MSA311_OFFSET_Z_REG, + .wr_table = &msa311_writeable_table, + .rd_table = &msa311_readable_table, + .volatile_table = &msa311_volatile_table, + .cache_type = REGCACHE_RBTREE, +}; + +#define MSA311_GENMASK(field) ({ \ + typeof(&(msa311_reg_fields)[0]) _field; \ + _field = &msa311_reg_fields[(field)]; \ + GENMASK(_field->msb, _field->lsb); \ +}) + +/** + * struct msa311_priv - MSA311 internal private state + * @regs: Underlying I2C bus adapter used to abstract slave + * register accesses + * @fields: Abstract objects for each registers fields access + * @dev: Device handler associated with appropriate bus client + * @lock: Protects msa311 device state between setup and data access routines + * (power transitions, samp_freq/scale tune, retrieving axes data, etc) + * @chip_name: Chip name in the format "msa311-%02x" % partid + * @new_data_trig: Optional NEW_DATA interrupt driven trigger used + * to notify external consumers a new sample is ready + * @vdd: Optional external voltage regulator for the device power supply + */ +struct msa311_priv { + struct regmap *regs; + struct regmap_field *fields[F_MAX_FIELDS]; + + struct device *dev; + struct mutex lock; + char *chip_name; + + struct iio_trigger *new_data_trig; + struct regulator *vdd; +}; + +enum msa311_si { + MSA311_SI_X, + MSA311_SI_Y, + MSA311_SI_Z, + MSA311_SI_TIMESTAMP, +}; + +#define MSA311_ACCEL_CHANNEL(axis) { \ + .type = IIO_ACCEL, \ + .modified = 1, \ + .channel2 = IIO_MOD_##axis, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | \ + BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SCALE) | \ + BIT(IIO_CHAN_INFO_SAMP_FREQ), \ + .scan_index = MSA311_SI_##axis, \ + .scan_type = { \ + .sign = 's', \ + .realbits = 12, \ + .storagebits = 16, \ + .shift = 4, \ + .endianness = IIO_LE, \ + }, \ + .datasheet_name = "ACC_"#axis, \ +} + +static const struct iio_chan_spec msa311_channels[] = { + MSA311_ACCEL_CHANNEL(X), + MSA311_ACCEL_CHANNEL(Y), + MSA311_ACCEL_CHANNEL(Z), + IIO_CHAN_SOFT_TIMESTAMP(MSA311_SI_TIMESTAMP), +}; + +/** + * msa311_get_odr() - Read Output Data Rate (ODR) value from MSA311 accel + * @msa311: MSA311 internal private state + * @odr: output ODR value + * + * This function should be called under msa311->lock. + * + * Return: 0 on success, -ERRNO in other failures + */ +static int msa311_get_odr(struct msa311_priv *msa311, unsigned int *odr) +{ + int err; + + err = regmap_field_read(msa311->fields[F_ODR], odr); + if (err) + return err; + + /* + * Filter the same 1000Hz ODR register values based on datasheet info. + * ODR can be equal to 1010-1111 for 1000Hz, but function returns 1010 + * all the time. + */ + if (*odr > MSA311_ODR_1000_HZ) + *odr = MSA311_ODR_1000_HZ; + + return 0; +} + +/** + * msa311_set_odr() - Setup Output Data Rate (ODR) value for MSA311 accel + * @msa311: MSA311 internal private state + * @odr: requested ODR value + * + * This function should be called under msa311->lock. Possible ODR values: + * - 1Hz (not available in normal mode) + * - 1.95Hz (not available in normal mode) + * - 3.9Hz + * - 7.81Hz + * - 15.63Hz + * - 31.25Hz + * - 62.5Hz + * - 125Hz + * - 250Hz + * - 500Hz + * - 1000Hz + * + * Return: 0 on success, -EINVAL for bad ODR value in the certain power mode, + * -ERRNO in other failures + */ +static int msa311_set_odr(struct msa311_priv *msa311, unsigned int odr) +{ + struct device *dev = msa311->dev; + unsigned int pwr_mode; + bool good_odr; + int err; + + err = regmap_field_read(msa311->fields[F_PWR_MODE], &pwr_mode); + if (err) + return err; + + /* Filter bad ODR values */ + if (pwr_mode == MSA311_PWR_MODE_NORMAL) + good_odr = (odr > MSA311_ODR_1_95_HZ); + else + good_odr = false; + + if (!good_odr) { + dev_err(dev, + "can't set odr %u.%06uHz, not available in %s mode\n", + msa311_odr_table[odr].integral, + msa311_odr_table[odr].microfract, + msa311_pwr_modes[pwr_mode]); + return -EINVAL; + } + + return regmap_field_write(msa311->fields[F_ODR], odr); +} + +/** + * msa311_wait_for_next_data() - Wait next accel data available after resume + * @msa311: MSA311 internal private state + * + * Return: 0 on success, -EINTR if msleep() was interrupted, + * -ERRNO in other failures + */ +static int msa311_wait_for_next_data(struct msa311_priv *msa311) +{ + static const unsigned int unintr_thresh_ms = 20; + struct device *dev = msa311->dev; + unsigned long freq_uhz; + unsigned long wait_ms; + unsigned int odr; + int err; + + err = msa311_get_odr(msa311, &odr); + if (err) { + dev_err(dev, "can't get actual frequency (%pe)\n", + ERR_PTR(err)); + return err; + } + + /* + * After msa311 resuming is done, we need to wait for data + * to be refreshed by accel logic. + * A certain timeout is calculated based on the current ODR value. + * If requested timeout isn't so long (let's assume 20ms), + * we can wait for next data in uninterruptible sleep. + */ + freq_uhz = msa311_odr_table[odr].integral * MICROHZ_PER_HZ + + msa311_odr_table[odr].microfract; + wait_ms = (MICROHZ_PER_HZ / freq_uhz) * MSEC_PER_SEC; + + if (wait_ms < unintr_thresh_ms) + usleep_range(wait_ms * USEC_PER_MSEC, + unintr_thresh_ms * USEC_PER_MSEC); + else if (msleep_interruptible(wait_ms)) + return -EINTR; + + return 0; +} + +/** + * msa311_set_pwr_mode() - Install certain MSA311 power mode + * @msa311: MSA311 internal private state + * @mode: Power mode can be equal to NORMAL or SUSPEND + * + * This function should be called under msa311->lock. + * + * Return: 0 on success, -ERRNO on failure + */ +static int msa311_set_pwr_mode(struct msa311_priv *msa311, unsigned int mode) +{ + struct device *dev = msa311->dev; + unsigned int prev_mode; + int err; + + if (mode >= ARRAY_SIZE(msa311_pwr_modes)) + return -EINVAL; + + dev_dbg(dev, "transition to %s mode\n", msa311_pwr_modes[mode]); + + err = regmap_field_read(msa311->fields[F_PWR_MODE], &prev_mode); + if (err) + return err; + + err = regmap_field_write(msa311->fields[F_PWR_MODE], mode); + if (err) + return err; + + /* Wait actual data if we wake up */ + if (prev_mode == MSA311_PWR_MODE_SUSPEND && + mode == MSA311_PWR_MODE_NORMAL) + return msa311_wait_for_next_data(msa311); + + return 0; +} + +/** + * msa311_get_axis() - Read MSA311 accel data for certain IIO channel axis spec + * @msa311: MSA311 internal private state + * @chan: IIO channel specification + * @axis: Output accel axis data for requested IIO channel spec + * + * This function should be called under msa311->lock. + * + * Return: 0 on success, -EINVAL for unknown IIO channel specification, + * -ERRNO in other failures + */ +static int msa311_get_axis(struct msa311_priv *msa311, + const struct iio_chan_spec * const chan, + __le16 *axis) +{ + struct device *dev = msa311->dev; + unsigned int axis_reg; + + if (chan->scan_index < MSA311_SI_X || chan->scan_index > MSA311_SI_Z) { + dev_err(dev, "invalid scan_index value [%d]\n", + chan->scan_index); + return -EINVAL; + } + + /* Axes data layout has 2 byte gap for each axis starting from X axis */ + axis_reg = MSA311_ACC_X_REG + (chan->scan_index << 1); + + return regmap_bulk_read(msa311->regs, axis_reg, axis, sizeof(*axis)); +} + +static int msa311_read_raw_data(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + __le16 axis; + int err; + + err = pm_runtime_resume_and_get(dev); + if (err) + return err; + + err = iio_device_claim_direct_mode(indio_dev); + if (err) + return err; + + mutex_lock(&msa311->lock); + err = msa311_get_axis(msa311, chan, &axis); + mutex_unlock(&msa311->lock); + + iio_device_release_direct_mode(indio_dev); + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + if (err) { + dev_err(dev, "can't get axis %s (%pe)\n", + chan->datasheet_name, ERR_PTR(err)); + return err; + } + + /* + * Axis data format is: + * ACC_X = (ACC_X_MSB[7:0] << 4) | ACC_X_LSB[7:4] + */ + *val = sign_extend32(le16_to_cpu(axis) >> chan->scan_type.shift, + chan->scan_type.realbits - 1); + + return IIO_VAL_INT; +} + +static int msa311_read_scale(struct iio_dev *indio_dev, int *val, int *val2) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + unsigned int fs; + int err; + + mutex_lock(&msa311->lock); + err = regmap_field_read(msa311->fields[F_FS], &fs); + mutex_unlock(&msa311->lock); + if (err) { + dev_err(dev, "can't get actual scale (%pe)\n", ERR_PTR(err)); + return err; + } + + *val = msa311_fs_table[fs].integral; + *val2 = msa311_fs_table[fs].microfract; + + return IIO_VAL_INT_PLUS_MICRO; +} + +static int msa311_read_samp_freq(struct iio_dev *indio_dev, + int *val, int *val2) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + unsigned int odr; + int err; + + mutex_lock(&msa311->lock); + err = msa311_get_odr(msa311, &odr); + mutex_unlock(&msa311->lock); + if (err) { + dev_err(dev, "can't get actual frequency (%pe)\n", + ERR_PTR(err)); + return err; + } + + *val = msa311_odr_table[odr].integral; + *val2 = msa311_odr_table[odr].microfract; + + return IIO_VAL_INT_PLUS_MICRO; +} + +static int msa311_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_RAW: + return msa311_read_raw_data(indio_dev, chan, val, val2); + + case IIO_CHAN_INFO_SCALE: + return msa311_read_scale(indio_dev, val, val2); + + case IIO_CHAN_INFO_SAMP_FREQ: + return msa311_read_samp_freq(indio_dev, val, val2); + + default: + return -EINVAL; + } +} + +static int msa311_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, + int *length, long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + *vals = (int *)msa311_odr_table; + *type = IIO_VAL_INT_PLUS_MICRO; + /* ODR value has 2 ints (integer and fractional parts) */ + *length = ARRAY_SIZE(msa311_odr_table) * 2; + return IIO_AVAIL_LIST; + + case IIO_CHAN_INFO_SCALE: + *vals = (int *)msa311_fs_table; + *type = IIO_VAL_INT_PLUS_MICRO; + /* FS value has 2 ints (integer and fractional parts) */ + *length = ARRAY_SIZE(msa311_fs_table) * 2; + return IIO_AVAIL_LIST; + + default: + return -EINVAL; + } +} + +static int msa311_write_scale(struct iio_dev *indio_dev, int val, int val2) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + unsigned int fs; + int err; + + /* We do not have fs >= 1, so skip such values */ + if (val) + return 0; + + err = pm_runtime_resume_and_get(dev); + if (err) + return err; + + err = -EINVAL; + for (fs = 0; fs < ARRAY_SIZE(msa311_fs_table); fs++) + /* Do not check msa311_fs_table[fs].integral, it's always 0 */ + if (val2 == msa311_fs_table[fs].microfract) { + mutex_lock(&msa311->lock); + err = regmap_field_write(msa311->fields[F_FS], fs); + mutex_unlock(&msa311->lock); + break; + } + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + if (err) + dev_err(dev, "can't update scale (%pe)\n", ERR_PTR(err)); + + return err; +} + +static int msa311_write_samp_freq(struct iio_dev *indio_dev, int val, int val2) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + unsigned int odr; + int err; + + err = pm_runtime_resume_and_get(dev); + if (err) + return err; + + /* + * Sampling frequency changing is prohibited when buffer mode is + * enabled, because sometimes MSA311 chip returns outliers during + * frequency values growing up in the read operation moment. + */ + err = iio_device_claim_direct_mode(indio_dev); + if (err) + return err; + + err = -EINVAL; + for (odr = 0; odr < ARRAY_SIZE(msa311_odr_table); odr++) + if (val == msa311_odr_table[odr].integral && + val2 == msa311_odr_table[odr].microfract) { + mutex_lock(&msa311->lock); + err = msa311_set_odr(msa311, odr); + mutex_unlock(&msa311->lock); + break; + } + + iio_device_release_direct_mode(indio_dev); + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + if (err) + dev_err(dev, "can't update frequency (%pe)\n", ERR_PTR(err)); + + return err; +} + +static int msa311_write_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_SCALE: + return msa311_write_scale(indio_dev, val, val2); + + case IIO_CHAN_INFO_SAMP_FREQ: + return msa311_write_samp_freq(indio_dev, val, val2); + + default: + return -EINVAL; + } +} + +static int msa311_debugfs_reg_access(struct iio_dev *indio_dev, + unsigned int reg, unsigned int writeval, + unsigned int *readval) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + int err; + + if (reg > regmap_get_max_register(msa311->regs)) + return -EINVAL; + + err = pm_runtime_resume_and_get(dev); + if (err) + return err; + + mutex_lock(&msa311->lock); + + if (readval) + err = regmap_read(msa311->regs, reg, readval); + else + err = regmap_write(msa311->regs, reg, writeval); + + mutex_unlock(&msa311->lock); + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + if (err) + dev_err(dev, "can't %s register %u from debugfs (%pe)\n", + str_read_write(readval), reg, ERR_PTR(err)); + + return err; +} + +static int msa311_buffer_preenable(struct iio_dev *indio_dev) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + + return pm_runtime_resume_and_get(dev); +} + +static int msa311_buffer_postdisable(struct iio_dev *indio_dev) +{ + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + return 0; +} + +static int msa311_set_new_data_trig_state(struct iio_trigger *trig, bool state) +{ + struct iio_dev *indio_dev = iio_trigger_get_drvdata(trig); + struct msa311_priv *msa311 = iio_priv(indio_dev); + struct device *dev = msa311->dev; + int err; + + mutex_lock(&msa311->lock); + err = regmap_field_write(msa311->fields[F_NEW_DATA_INT_EN], state); + mutex_unlock(&msa311->lock); + if (err) + dev_err(dev, + "can't %s buffer due to new_data_int failure (%pe)\n", + str_enable_disable(state), ERR_PTR(err)); + + return err; +} + +static int msa311_validate_device(struct iio_trigger *trig, + struct iio_dev *indio_dev) +{ + return iio_trigger_get_drvdata(trig) == indio_dev ? 0 : -EINVAL; +} + +static irqreturn_t msa311_buffer_thread(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct msa311_priv *msa311 = iio_priv(pf->indio_dev); + struct iio_dev *indio_dev = pf->indio_dev; + const struct iio_chan_spec *chan; + struct device *dev = msa311->dev; + int bit, err, i = 0; + __le16 axis; + struct { + __le16 channels[MSA311_SI_Z + 1]; + s64 ts __aligned(8); + } buf; + + memset(&buf, 0, sizeof(buf)); + + mutex_lock(&msa311->lock); + + for_each_set_bit(bit, indio_dev->active_scan_mask, + indio_dev->masklength) { + chan = &msa311_channels[bit]; + + err = msa311_get_axis(msa311, chan, &axis); + if (err) { + mutex_unlock(&msa311->lock); + dev_err(dev, "can't get axis %s (%pe)\n", + chan->datasheet_name, ERR_PTR(err)); + goto notify_done; + } + + buf.channels[i++] = axis; + } + + mutex_unlock(&msa311->lock); + + iio_push_to_buffers_with_timestamp(indio_dev, &buf, + iio_get_time_ns(indio_dev)); + +notify_done: + iio_trigger_notify_done(indio_dev->trig); + + return IRQ_HANDLED; +} + +static irqreturn_t msa311_irq_thread(int irq, void *p) +{ + struct msa311_priv *msa311 = iio_priv(p); + unsigned int new_data_int_enabled; + struct device *dev = msa311->dev; + int err; + + mutex_lock(&msa311->lock); + + /* + * We do not check NEW_DATA int status, because based on the + * specification it's cleared automatically after a fixed time. + * So just check that is enabled by driver logic. + */ + err = regmap_field_read(msa311->fields[F_NEW_DATA_INT_EN], + &new_data_int_enabled); + + mutex_unlock(&msa311->lock); + if (err) { + dev_err(dev, "can't read new_data interrupt state (%pe)\n", + ERR_PTR(err)); + return IRQ_NONE; + } + + if (new_data_int_enabled) + iio_trigger_poll_chained(msa311->new_data_trig); + + return IRQ_HANDLED; +} + +static const struct iio_info msa311_info = { + .read_raw = msa311_read_raw, + .read_avail = msa311_read_avail, + .write_raw = msa311_write_raw, + .debugfs_reg_access = msa311_debugfs_reg_access, +}; + +static const struct iio_buffer_setup_ops msa311_buffer_setup_ops = { + .preenable = msa311_buffer_preenable, + .postdisable = msa311_buffer_postdisable, +}; + +static const struct iio_trigger_ops msa311_new_data_trig_ops = { + .set_trigger_state = msa311_set_new_data_trig_state, + .validate_device = msa311_validate_device, +}; + +static int msa311_check_partid(struct msa311_priv *msa311) +{ + struct device *dev = msa311->dev; + unsigned int partid; + int err; + + err = regmap_read(msa311->regs, MSA311_PARTID_REG, &partid); + if (err) + return dev_err_probe(dev, err, "failed to read partid\n"); + + if (partid != MSA311_WHO_AM_I) + dev_warn(dev, "invalid partid (%#x), expected (%#x)\n", + partid, MSA311_WHO_AM_I); + + msa311->chip_name = devm_kasprintf(dev, GFP_KERNEL, + "msa311-%02x", partid); + if (!msa311->chip_name) + return dev_err_probe(dev, -ENOMEM, "can't alloc chip name\n"); + + return 0; +} + +static int msa311_soft_reset(struct msa311_priv *msa311) +{ + struct device *dev = msa311->dev; + int err; + + err = regmap_write(msa311->regs, MSA311_SOFT_RESET_REG, + MSA311_GENMASK(F_SOFT_RESET_I2C) | + MSA311_GENMASK(F_SOFT_RESET_SPI)); + if (err) + return dev_err_probe(dev, err, "can't soft reset all logic\n"); + + return 0; +} + +static int msa311_chip_init(struct msa311_priv *msa311) +{ + struct device *dev = msa311->dev; + const char zero_bulk[2] = { }; + int err; + + err = regmap_write(msa311->regs, MSA311_RANGE_REG, MSA311_FS_16G); + if (err) + return dev_err_probe(dev, err, "failed to setup accel range\n"); + + /* Disable all interrupts by default */ + err = regmap_bulk_write(msa311->regs, MSA311_INT_SET_0_REG, + zero_bulk, sizeof(zero_bulk)); + if (err) + return dev_err_probe(dev, err, + "can't disable set0/set1 interrupts\n"); + + /* Unmap all INT1 interrupts by default */ + err = regmap_bulk_write(msa311->regs, MSA311_INT_MAP_0_REG, + zero_bulk, sizeof(zero_bulk)); + if (err) + return dev_err_probe(dev, err, + "failed to unmap map0/map1 interrupts\n"); + + /* Disable all axes by default */ + err = regmap_update_bits(msa311->regs, MSA311_ODR_REG, + MSA311_GENMASK(F_X_AXIS_DIS) | + MSA311_GENMASK(F_Y_AXIS_DIS) | + MSA311_GENMASK(F_Z_AXIS_DIS), 0); + if (err) + return dev_err_probe(dev, err, "can't enable all axes\n"); + + err = msa311_set_odr(msa311, MSA311_ODR_125_HZ); + if (err) + return dev_err_probe(dev, err, + "failed to set accel frequency\n"); + + return 0; +} + +static int msa311_setup_interrupts(struct msa311_priv *msa311) +{ + struct device *dev = msa311->dev; + struct i2c_client *i2c = to_i2c_client(dev); + struct iio_dev *indio_dev = i2c_get_clientdata(i2c); + struct iio_trigger *trig; + int err; + + /* Keep going without interrupts if no initialized I2C IRQ */ + if (i2c->irq <= 0) + return 0; + + err = devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL, + msa311_irq_thread, IRQF_ONESHOT, + msa311->chip_name, indio_dev); + if (err) + return dev_err_probe(dev, err, "failed to request IRQ\n"); + + trig = devm_iio_trigger_alloc(dev, "%s-new-data", msa311->chip_name); + if (!trig) + return dev_err_probe(dev, -ENOMEM, + "can't allocate newdata trigger\n"); + + msa311->new_data_trig = trig; + msa311->new_data_trig->ops = &msa311_new_data_trig_ops; + iio_trigger_set_drvdata(msa311->new_data_trig, indio_dev); + + err = devm_iio_trigger_register(dev, msa311->new_data_trig); + if (err) + return dev_err_probe(dev, err, + "can't register newdata trigger\n"); + + err = regmap_field_write(msa311->fields[F_INT1_OD], + MSA311_INT1_OD_PUSH_PULL); + if (err) + return dev_err_probe(dev, err, + "can't enable push-pull interrupt\n"); + + err = regmap_field_write(msa311->fields[F_INT1_LVL], + MSA311_INT1_LVL_HIGH); + if (err) + return dev_err_probe(dev, err, + "can't set active interrupt level\n"); + + err = regmap_field_write(msa311->fields[F_LATCH_INT], + MSA311_LATCH_INT_LATCHED); + if (err) + return dev_err_probe(dev, err, + "can't latch interrupt\n"); + + err = regmap_field_write(msa311->fields[F_RESET_INT], 1); + if (err) + return dev_err_probe(dev, err, + "can't reset interrupt\n"); + + err = regmap_field_write(msa311->fields[F_INT1_NEW_DATA], 1); + if (err) + return dev_err_probe(dev, err, + "can't map new data interrupt\n"); + + return 0; +} + +static int msa311_regmap_init(struct msa311_priv *msa311) +{ + struct regmap_field **fields = msa311->fields; + struct device *dev = msa311->dev; + struct i2c_client *i2c = to_i2c_client(dev); + struct regmap *regmap; + int i; + + regmap = devm_regmap_init_i2c(i2c, &msa311_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), + "failed to register i2c regmap\n"); + + msa311->regs = regmap; + + for (i = 0; i < F_MAX_FIELDS; i++) { + fields[i] = devm_regmap_field_alloc(dev, + msa311->regs, + msa311_reg_fields[i]); + if (IS_ERR(msa311->fields[i])) + return dev_err_probe(dev, PTR_ERR(msa311->fields[i]), + "can't alloc field[%d]\n", i); + } + + return 0; +} + +static void msa311_powerdown(void *msa311) +{ + msa311_set_pwr_mode(msa311, MSA311_PWR_MODE_SUSPEND); +} + +static void msa311_vdd_disable(void *vdd) +{ + regulator_disable(vdd); +} + +static int msa311_probe(struct i2c_client *i2c) +{ + struct device *dev = &i2c->dev; + struct msa311_priv *msa311; + struct iio_dev *indio_dev; + int err; + + indio_dev = devm_iio_device_alloc(dev, sizeof(*msa311)); + if (!indio_dev) + return dev_err_probe(dev, -ENOMEM, + "IIO device allocation failed\n"); + + msa311 = iio_priv(indio_dev); + msa311->dev = dev; + i2c_set_clientdata(i2c, indio_dev); + + err = msa311_regmap_init(msa311); + if (err) + return err; + + mutex_init(&msa311->lock); + + msa311->vdd = devm_regulator_get(dev, "vdd"); + if (IS_ERR(msa311->vdd)) + return dev_err_probe(dev, PTR_ERR(msa311->vdd), + "can't get vdd supply\n"); + + err = regulator_enable(msa311->vdd); + if (err) + return dev_err_probe(dev, err, "can't enable vdd supply\n"); + + err = devm_add_action_or_reset(dev, msa311_vdd_disable, msa311->vdd); + if (err) + return dev_err_probe(dev, err, + "can't add vdd disable action\n"); + + err = msa311_check_partid(msa311); + if (err) + return err; + + err = msa311_soft_reset(msa311); + if (err) + return err; + + err = msa311_set_pwr_mode(msa311, MSA311_PWR_MODE_NORMAL); + if (err) + return dev_err_probe(dev, err, "failed to power on device\n"); + + /* + * Register powerdown deferred callback which suspends the chip + * after module unloaded. + * + * MSA311 should be in SUSPEND mode in the two cases: + * 1) When driver is loaded, but we do not have any data or + * configuration requests to it (we are solving it using + * autosuspend feature). + * 2) When driver is unloaded and device is not used (devm action is + * used in this case). + */ + err = devm_add_action_or_reset(dev, msa311_powerdown, msa311); + if (err) + return dev_err_probe(dev, err, "can't add powerdown action\n"); + + err = pm_runtime_set_active(dev); + if (err) + return err; + + err = devm_pm_runtime_enable(dev); + if (err) + return err; + + pm_runtime_get_noresume(dev); + pm_runtime_set_autosuspend_delay(dev, MSA311_PWR_SLEEP_DELAY_MS); + pm_runtime_use_autosuspend(dev); + + err = msa311_chip_init(msa311); + if (err) + return err; + + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = msa311_channels; + indio_dev->num_channels = ARRAY_SIZE(msa311_channels); + indio_dev->name = msa311->chip_name; + indio_dev->info = &msa311_info; + + err = devm_iio_triggered_buffer_setup(dev, indio_dev, + iio_pollfunc_store_time, + msa311_buffer_thread, + &msa311_buffer_setup_ops); + if (err) + return dev_err_probe(dev, err, + "can't setup IIO trigger buffer\n"); + + err = msa311_setup_interrupts(msa311); + if (err) + return err; + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + + err = devm_iio_device_register(dev, indio_dev); + if (err) + return dev_err_probe(dev, err, "IIO device register failed\n"); + + return 0; +} + +static int msa311_runtime_suspend(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct msa311_priv *msa311 = iio_priv(indio_dev); + int err; + + mutex_lock(&msa311->lock); + err = msa311_set_pwr_mode(msa311, MSA311_PWR_MODE_SUSPEND); + mutex_unlock(&msa311->lock); + if (err) + dev_err(dev, "failed to power off device (%pe)\n", + ERR_PTR(err)); + + return err; +} + +static int msa311_runtime_resume(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct msa311_priv *msa311 = iio_priv(indio_dev); + int err; + + mutex_lock(&msa311->lock); + err = msa311_set_pwr_mode(msa311, MSA311_PWR_MODE_NORMAL); + mutex_unlock(&msa311->lock); + if (err) + dev_err(dev, "failed to power on device (%pe)\n", + ERR_PTR(err)); + + return err; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(msa311_pm_ops, msa311_runtime_suspend, + msa311_runtime_resume, NULL); + +static const struct i2c_device_id msa311_i2c_id[] = { + { .name = "msa311" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, msa311_i2c_id); + +static const struct of_device_id msa311_of_match[] = { + { .compatible = "memsensing,msa311" }, + { } +}; +MODULE_DEVICE_TABLE(of, msa311_of_match); + +static struct i2c_driver msa311_driver = { + .driver = { + .name = "msa311", + .of_match_table = msa311_of_match, + .pm = pm_ptr(&msa311_pm_ops), + }, + .probe_new = msa311_probe, + .id_table = msa311_i2c_id, +}; +module_i2c_driver(msa311_driver); + +MODULE_AUTHOR("Dmitry Rokosov "); +MODULE_DESCRIPTION("MEMSensing MSA311 3-axis accelerometer driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/accel/stk8312.c b/drivers/iio/accel/stk8312.c index ceca2891335531e69301535d50ce623317a90d81..7b1d6fb692b371e08e56e90682bf38ce2339eb74 100644 --- a/drivers/iio/accel/stk8312.c +++ b/drivers/iio/accel/stk8312.c @@ -597,7 +597,7 @@ err_power_off: return ret; } -static int stk8312_remove(struct i2c_client *client) +static void stk8312_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct stk8312_data *data = iio_priv(indio_dev); @@ -609,8 +609,6 @@ static int stk8312_remove(struct i2c_client *client) iio_trigger_unregister(data->dready_trig); stk8312_set_mode(data, STK8312_MODE_STANDBY); - - return 0; } static int stk8312_suspend(struct device *dev) diff --git a/drivers/iio/accel/stk8ba50.c b/drivers/iio/accel/stk8ba50.c index 7d59efb41e229324159bad1372d065a82fc16f75..2f5e4ab2a6e7d190eae12da5e4176c2acdf5a211 100644 --- a/drivers/iio/accel/stk8ba50.c +++ b/drivers/iio/accel/stk8ba50.c @@ -490,7 +490,7 @@ err_power_off: return ret; } -static int stk8ba50_remove(struct i2c_client *client) +static void stk8ba50_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct stk8ba50_data *data = iio_priv(indio_dev); @@ -502,8 +502,6 @@ static int stk8ba50_remove(struct i2c_client *client) iio_trigger_unregister(data->dready_trig); stk8ba50_set_power(data, STK8BA50_MODE_SUSPEND); - - return 0; } static int stk8ba50_suspend(struct device *dev) diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 7fe5930891e0a123795c3815a992072c03a4b2f6..791612ca601207b58baf709068741845aee46132 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -653,6 +653,20 @@ config MAX1118 To compile this driver as a module, choose M here: the module will be called max1118. +config MAX11205 + tristate "Maxim max11205 ADC driver" + depends on SPI + select AD_SIGMA_DELTA + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER + + help + Say yes here to build support for Maxim max11205 16-bit, single-channel + ultra-low power delta-sigma ADC. + + To compile this driver as a module, choose M here: the module will be + called max11205. + config MAX1241 tristate "Maxim max1241 ADC driver" depends on SPI_MASTER @@ -718,6 +732,8 @@ config MCP3422 config MCP3911 tristate "Microchip Technology MCP3911 driver" depends on SPI + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER help Say yes here to build support for Microchip Technology's MCP3911 analog to digital converter. @@ -919,6 +935,21 @@ config ROCKCHIP_SARADC To compile this driver as a module, choose M here: the module will be called rockchip_saradc. +config RICHTEK_RTQ6056 + tristate "Richtek RTQ6056 Current and Power Monitor ADC" + depends on I2C + select REGMAP_I2C + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER + help + Say yes here to enable RQT6056 ADC support. + RTQ6056 is a high accuracy current-sense monitor with I2C and SMBus + compatible interface, and the device provides full information for + system by reading out the load current and power. + + This driver can also be built as a module. If so, the module will be + called rtq6056. + config RZG2L_ADC tristate "Renesas RZ/G2L ADC driver" depends on ARCH_RZG2L || COMPILE_TEST @@ -1022,22 +1053,6 @@ config STMPE_ADC Say yes here to build support for ST Microelectronics STMPE built-in ADC block (stmpe811). -config STX104 - tristate "Apex Embedded Systems STX104 driver" - depends on PC104 && X86 - select ISA_BUS_API - select GPIOLIB - help - Say yes here to build support for the Apex Embedded Systems STX104 - integrated analog PC/104 card. - - This driver supports the 16 channels of single-ended (8 channels of - differential) analog inputs, 2 channels of analog output, 4 digital - inputs, and 4 digital outputs provided by the STX104. - - The base port addresses for the devices may be configured via the base - array module parameter. - config SUN4I_GPADC tristate "Support for the Allwinner SoCs GPADC" depends on IIO diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile index 1772a549a3c80a7e642f452def6361deac839cde..46caba7a010c9e1974032dee2dbdd534562bf9ff 100644 --- a/drivers/iio/adc/Makefile +++ b/drivers/iio/adc/Makefile @@ -61,6 +61,7 @@ obj-$(CONFIG_LTC2497) += ltc2497.o ltc2497-core.o obj-$(CONFIG_MAX1027) += max1027.o obj-$(CONFIG_MAX11100) += max11100.o obj-$(CONFIG_MAX1118) += max1118.o +obj-$(CONFIG_MAX11205) += max11205.o obj-$(CONFIG_MAX1241) += max1241.o obj-$(CONFIG_MAX1363) += max1363.o obj-$(CONFIG_MAX9611) += max9611.o @@ -85,10 +86,10 @@ obj-$(CONFIG_QCOM_PM8XXX_XOADC) += qcom-pm8xxx-xoadc.o obj-$(CONFIG_RCAR_GYRO_ADC) += rcar-gyroadc.o obj-$(CONFIG_RN5T618_ADC) += rn5t618-adc.o obj-$(CONFIG_ROCKCHIP_SARADC) += rockchip_saradc.o +obj-$(CONFIG_RICHTEK_RTQ6056) += rtq6056.o obj-$(CONFIG_RZG2L_ADC) += rzg2l_adc.o obj-$(CONFIG_SC27XX_ADC) += sc27xx_adc.o obj-$(CONFIG_SPEAR_ADC) += spear_adc.o -obj-$(CONFIG_STX104) += stx104.o obj-$(CONFIG_SUN4I_GPADC) += sun4i-gpadc-iio.o obj-$(CONFIG_STM32_ADC_CORE) += stm32-adc-core.o obj-$(CONFIG_STM32_ADC) += stm32-adc.o diff --git a/drivers/iio/adc/ab8500-gpadc.c b/drivers/iio/adc/ab8500-gpadc.c index 930ce96e6ff5d8b0688419786d9f2fc3bf4c993c..4fa2126a354b2236db9aca977573bcbc05f8fc67 100644 --- a/drivers/iio/adc/ab8500-gpadc.c +++ b/drivers/iio/adc/ab8500-gpadc.c @@ -925,8 +925,8 @@ static int ab8500_gpadc_read_raw(struct iio_dev *indio_dev, return -EINVAL; } -static int ab8500_gpadc_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int ab8500_gpadc_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { int i; @@ -938,7 +938,7 @@ static int ab8500_gpadc_of_xlate(struct iio_dev *indio_dev, } static const struct iio_info ab8500_gpadc_info = { - .of_xlate = ab8500_gpadc_of_xlate, + .fwnode_xlate = ab8500_gpadc_fwnode_xlate, .read_raw = ab8500_gpadc_read_raw, }; @@ -968,7 +968,7 @@ static int ab8500_gpadc_runtime_resume(struct device *dev) /** * ab8500_gpadc_parse_channel() - process devicetree channel configuration * @dev: pointer to containing device - * @np: device tree node for the channel to configure + * @fwnode: fw node for the channel to configure * @ch: channel info to fill in * @iio_chan: IIO channel specification to fill in * @@ -976,15 +976,15 @@ static int ab8500_gpadc_runtime_resume(struct device *dev) * and define usage for things like AUX GPADC inputs more precisely. */ static int ab8500_gpadc_parse_channel(struct device *dev, - struct device_node *np, + struct fwnode_handle *fwnode, struct ab8500_gpadc_chan_info *ch, struct iio_chan_spec *iio_chan) { - const char *name = np->name; + const char *name = fwnode_get_name(fwnode); u32 chan; int ret; - ret = of_property_read_u32(np, "reg", &chan); + ret = fwnode_property_read_u32(fwnode, "reg", &chan); if (ret) { dev_err(dev, "invalid channel number %s\n", name); return ret; @@ -1021,22 +1021,20 @@ static int ab8500_gpadc_parse_channel(struct device *dev, /** * ab8500_gpadc_parse_channels() - Parse the GPADC channels from DT * @gpadc: the GPADC to configure the channels for - * @np: device tree node containing the channel configurations * @chans: the IIO channels we parsed * @nchans: the number of IIO channels we parsed */ static int ab8500_gpadc_parse_channels(struct ab8500_gpadc *gpadc, - struct device_node *np, struct iio_chan_spec **chans_parsed, unsigned int *nchans_parsed) { - struct device_node *child; + struct fwnode_handle *child; struct ab8500_gpadc_chan_info *ch; struct iio_chan_spec *iio_chans; unsigned int nchans; int i; - nchans = of_get_available_child_count(np); + nchans = device_get_child_node_count(gpadc->dev); if (!nchans) { dev_err(gpadc->dev, "no channel children\n"); return -ENODEV; @@ -1054,7 +1052,7 @@ static int ab8500_gpadc_parse_channels(struct ab8500_gpadc *gpadc, return -ENOMEM; i = 0; - for_each_available_child_of_node(np, child) { + device_for_each_child_node(gpadc->dev, child) { struct iio_chan_spec *iio_chan; int ret; @@ -1064,7 +1062,7 @@ static int ab8500_gpadc_parse_channels(struct ab8500_gpadc *gpadc, ret = ab8500_gpadc_parse_channel(gpadc->dev, child, ch, iio_chan); if (ret) { - of_node_put(child); + fwnode_handle_put(child); return ret; } i++; @@ -1081,7 +1079,6 @@ static int ab8500_gpadc_probe(struct platform_device *pdev) struct ab8500_gpadc *gpadc; struct iio_dev *indio_dev; struct device *dev = &pdev->dev; - struct device_node *np = pdev->dev.of_node; struct iio_chan_spec *iio_chans; unsigned int n_iio_chans; int ret; @@ -1096,7 +1093,7 @@ static int ab8500_gpadc_probe(struct platform_device *pdev) gpadc->dev = dev; gpadc->ab8500 = dev_get_drvdata(dev->parent); - ret = ab8500_gpadc_parse_channels(gpadc, np, &iio_chans, &n_iio_chans); + ret = ab8500_gpadc_parse_channels(gpadc, &iio_chans, &n_iio_chans); if (ret) return ret; diff --git a/drivers/iio/adc/ad7124.c b/drivers/iio/adc/ad7124.c index c5b785d8b2415bd643ceec21dae9a2d7064b981e..4088786e1026d5288d34e42f852292248ede1491 100644 --- a/drivers/iio/adc/ad7124.c +++ b/drivers/iio/adc/ad7124.c @@ -936,11 +936,6 @@ static void ad7124_reg_disable(void *r) regulator_disable(r); } -static void ad7124_clk_disable(void *c) -{ - clk_disable_unprepare(c); -} - static int ad7124_probe(struct spi_device *spi) { const struct ad7124_chip_info *info; @@ -993,18 +988,10 @@ static int ad7124_probe(struct spi_device *spi) return ret; } - st->mclk = devm_clk_get(&spi->dev, "mclk"); + st->mclk = devm_clk_get_enabled(&spi->dev, "mclk"); if (IS_ERR(st->mclk)) return PTR_ERR(st->mclk); - ret = clk_prepare_enable(st->mclk); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, ad7124_clk_disable, st->mclk); - if (ret) - return ret; - ret = ad7124_soft_reset(st); if (ret < 0) return ret; diff --git a/drivers/iio/adc/ad7292.c b/drivers/iio/adc/ad7292.c index 92c68d467c505fcefde94eb9c968176603b815ba..a2f9fda25ff3463a39e180e69fa305b3228dbe5b 100644 --- a/drivers/iio/adc/ad7292.c +++ b/drivers/iio/adc/ad7292.c @@ -287,10 +287,8 @@ static int ad7292_probe(struct spi_device *spi) ret = devm_add_action_or_reset(&spi->dev, ad7292_regulator_disable, st); - if (ret) { - regulator_disable(st->reg); + if (ret) return ret; - } ret = regulator_get_voltage(st->reg); if (ret < 0) diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c index 652db768ef37299c8413faba4508fb4906fe7cc4..70a25949142c0d3076455ddbae5dcacb399161fc 100644 --- a/drivers/iio/adc/ad7768-1.c +++ b/drivers/iio/adc/ad7768-1.c @@ -539,13 +539,6 @@ static void ad7768_regulator_disable(void *data) regulator_disable(st->vref); } -static void ad7768_clk_disable(void *data) -{ - struct ad7768_state *st = data; - - clk_disable_unprepare(st->mclk); -} - static int ad7768_set_channel_label(struct iio_dev *indio_dev, int num_channels) { @@ -600,18 +593,10 @@ static int ad7768_probe(struct spi_device *spi) if (ret) return ret; - st->mclk = devm_clk_get(&spi->dev, "mclk"); + st->mclk = devm_clk_get_enabled(&spi->dev, "mclk"); if (IS_ERR(st->mclk)) return PTR_ERR(st->mclk); - ret = clk_prepare_enable(st->mclk); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, ad7768_clk_disable, st); - if (ret) - return ret; - st->mclk_freq = clk_get_rate(st->mclk); mutex_init(&st->lock); diff --git a/drivers/iio/adc/ad7923.c b/drivers/iio/adc/ad7923.c index edad1f30121dd9c8206b8e6b72e09cb7babfe64a..9d6bf6d0927a4b4f2d8b2f0bb2e5fa338cd2412e 100644 --- a/drivers/iio/adc/ad7923.c +++ b/drivers/iio/adc/ad7923.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -93,6 +94,7 @@ enum ad7923_id { .sign = 'u', \ .realbits = (bits), \ .storagebits = 16, \ + .shift = 12 - (bits), \ .endianness = IIO_BE, \ }, \ } @@ -268,7 +270,8 @@ static int ad7923_read_raw(struct iio_dev *indio_dev, return ret; if (chan->address == EXTRACT(ret, 12, 4)) - *val = EXTRACT(ret, 0, 12); + *val = EXTRACT(ret, chan->scan_type.shift, + chan->scan_type.realbits); else return -EIO; @@ -298,6 +301,7 @@ static void ad7923_regulator_disable(void *data) static int ad7923_probe(struct spi_device *spi) { + u32 ad7923_range = AD7923_RANGE; struct ad7923_state *st; struct iio_dev *indio_dev; const struct ad7923_chip_info *info; @@ -309,8 +313,11 @@ static int ad7923_probe(struct spi_device *spi) st = iio_priv(indio_dev); + if (device_property_read_bool(&spi->dev, "adi,range-double")) + ad7923_range = 0; + st->spi = spi; - st->settings = AD7923_CODING | AD7923_RANGE | + st->settings = AD7923_CODING | ad7923_range | AD7923_PM_MODE_WRITE(AD7923_PM_MODE_OPS); info = &ad7923_chip_info[spi_get_device_id(spi)->driver_data]; diff --git a/drivers/iio/adc/ad799x.c b/drivers/iio/adc/ad799x.c index 262bd7665b336a4c6a803c9534e2d671da7dd976..6dbe9d5e08a2b263d8b01b285dabcfa44878df19 100644 --- a/drivers/iio/adc/ad799x.c +++ b/drivers/iio/adc/ad799x.c @@ -880,7 +880,7 @@ error_disable_reg: return ret; } -static int ad799x_remove(struct i2c_client *client) +static void ad799x_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ad799x_state *st = iio_priv(indio_dev); @@ -892,8 +892,6 @@ static int ad799x_remove(struct i2c_client *client) regulator_disable(st->vref); regulator_disable(st->reg); kfree(st->rx_buf); - - return 0; } static int ad799x_suspend(struct device *dev) diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c index 5a5f33f7bc8f5dd3ea7f84dd9bd590a703cc8045..7534572f74757f5540811e1cdd7aab85cf537558 100644 --- a/drivers/iio/adc/ad9467.c +++ b/drivers/iio/adc/ad9467.c @@ -378,13 +378,6 @@ static int ad9467_preenable_setup(struct adi_axi_adc_conv *conv) return ad9467_outputmode_set(st->spi, st->output_mode); } -static void ad9467_clk_disable(void *data) -{ - struct ad9467_state *st = data; - - clk_disable_unprepare(st->clk); -} - static int ad9467_probe(struct spi_device *spi) { const struct ad9467_chip_info *info; @@ -404,18 +397,10 @@ static int ad9467_probe(struct spi_device *spi) st = adi_axi_adc_conv_priv(conv); st->spi = spi; - st->clk = devm_clk_get(&spi->dev, "adc-clk"); + st->clk = devm_clk_get_enabled(&spi->dev, "adc-clk"); if (IS_ERR(st->clk)) return PTR_ERR(st->clk); - ret = clk_prepare_enable(st->clk); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, ad9467_clk_disable, st); - if (ret) - return ret; - st->pwrdown_gpio = devm_gpiod_get_optional(&spi->dev, "powerdown", GPIOD_OUT_LOW); if (IS_ERR(st->pwrdown_gpio)) diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c index 279430c1d88c3c69223145d5fbd49b61483eb6c8..4294d6539cdb36439691aeb0c30850af3aea96f6 100644 --- a/drivers/iio/adc/at91-sama5d2_adc.c +++ b/drivers/iio/adc/at91-sama5d2_adc.c @@ -16,9 +16,11 @@ #include #include #include -#include +#include #include +#include #include +#include #include #include #include @@ -26,9 +28,13 @@ #include #include #include +#include #include +#include #include +#include + struct at91_adc_reg_layout { /* Control Register */ u16 CR; @@ -73,11 +79,14 @@ struct at91_adc_reg_layout { /* Startup Time */ #define AT91_SAMA5D2_MR_STARTUP(v) ((v) << 16) #define AT91_SAMA5D2_MR_STARTUP_MASK GENMASK(19, 16) +/* Minimum startup time for temperature sensor */ +#define AT91_SAMA5D2_MR_STARTUP_TS_MIN (50) /* Analog Change */ #define AT91_SAMA5D2_MR_ANACH BIT(23) /* Tracking Time */ #define AT91_SAMA5D2_MR_TRACKTIM(v) ((v) << 24) -#define AT91_SAMA5D2_MR_TRACKTIM_MAX 0xff +#define AT91_SAMA5D2_MR_TRACKTIM_TS 6 +#define AT91_SAMA5D2_MR_TRACKTIM_MAX 0xf /* Transfer Time */ #define AT91_SAMA5D2_MR_TRANSFER(v) ((v) << 28) #define AT91_SAMA5D2_MR_TRANSFER_MAX 0x3 @@ -138,11 +147,19 @@ struct at91_adc_reg_layout { /* Extended Mode Register */ u16 EMR; /* Extended Mode Register - Oversampling rate */ -#define AT91_SAMA5D2_EMR_OSR(V) ((V) << 16) -#define AT91_SAMA5D2_EMR_OSR_MASK GENMASK(17, 16) +#define AT91_SAMA5D2_EMR_OSR(V, M) (((V) << 16) & (M)) #define AT91_SAMA5D2_EMR_OSR_1SAMPLES 0 #define AT91_SAMA5D2_EMR_OSR_4SAMPLES 1 #define AT91_SAMA5D2_EMR_OSR_16SAMPLES 2 +#define AT91_SAMA5D2_EMR_OSR_64SAMPLES 3 +#define AT91_SAMA5D2_EMR_OSR_256SAMPLES 4 + +/* Extended Mode Register - TRACKX */ +#define AT91_SAMA5D2_TRACKX_MASK GENMASK(23, 22) +#define AT91_SAMA5D2_TRACKX(x) (((x) << 22) & \ + AT91_SAMA5D2_TRACKX_MASK) +/* TRACKX for temperature sensor. */ +#define AT91_SAMA5D2_TRACKX_TS (1) /* Extended Mode Register - Averaging on single trigger event */ #define AT91_SAMA5D2_EMR_ASTE(V) ((V) << 20) @@ -159,6 +176,8 @@ struct at91_adc_reg_layout { u16 ACR; /* Analog Control Register - Pen detect sensitivity mask */ #define AT91_SAMA5D2_ACR_PENDETSENS_MASK GENMASK(1, 0) +/* Analog Control Register - Source last channel */ +#define AT91_SAMA5D2_ACR_SRCLCH BIT(16) /* Touchscreen Mode Register */ u16 TSMR; @@ -226,6 +245,10 @@ struct at91_adc_reg_layout { u16 WPSR; /* Version Register */ u16 VERSION; +/* Temperature Sensor Mode Register */ + u16 TEMPMR; +/* Temperature Sensor Mode - Temperature sensor on */ +#define AT91_SAMA5D2_TEMPMR_TEMPON BIT(0) }; static const struct at91_adc_reg_layout sama5d2_layout = { @@ -280,6 +303,7 @@ static const struct at91_adc_reg_layout sama7g5_layout = { .EOC_IDR = 0x38, .EOC_IMR = 0x3c, .EOC_ISR = 0x40, + .TEMPMR = 0x44, .OVER = 0x4c, .EMR = 0x50, .CWR = 0x54, @@ -305,11 +329,6 @@ static const struct at91_adc_reg_layout sama7g5_layout = { #define AT91_HWFIFO_MAX_SIZE_STR "128" #define AT91_HWFIFO_MAX_SIZE 128 -/* Possible values for oversampling ratio */ -#define AT91_OSR_1SAMPLES 1 -#define AT91_OSR_4SAMPLES 4 -#define AT91_OSR_16SAMPLES 16 - #define AT91_SAMA5D2_CHAN_SINGLE(index, num, addr) \ { \ .type = IIO_VOLTAGE, \ @@ -325,6 +344,8 @@ static const struct at91_adc_reg_layout sama7g5_layout = { .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ)|\ BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .info_mask_shared_by_all_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .datasheet_name = "CH"#num, \ .indexed = 1, \ } @@ -346,6 +367,8 @@ static const struct at91_adc_reg_layout sama7g5_layout = { .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ)|\ BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .info_mask_shared_by_all_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .datasheet_name = "CH"#num"-CH"#num2, \ .indexed = 1, \ } @@ -365,6 +388,8 @@ static const struct at91_adc_reg_layout sama7g5_layout = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ)|\ BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .info_mask_shared_by_all_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .datasheet_name = name, \ } #define AT91_SAMA5D2_CHAN_PRESSURE(num, name) \ @@ -380,6 +405,23 @@ static const struct at91_adc_reg_layout sama7g5_layout = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \ .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ)|\ BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .info_mask_shared_by_all_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .datasheet_name = name, \ + } + +#define AT91_SAMA5D2_CHAN_TEMP(num, name, addr) \ + { \ + .type = IIO_TEMP, \ + .channel = num, \ + .address = addr, \ + .scan_index = num, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), \ + .info_mask_shared_by_all = \ + BIT(IIO_CHAN_INFO_PROCESSED) | \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .info_mask_shared_by_all_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .datasheet_name = name, \ } @@ -403,6 +445,12 @@ static const struct at91_adc_reg_layout sama7g5_layout = { * @max_index: highest channel index (highest index may be higher * than the total channel number) * @hw_trig_cnt: number of possible hardware triggers + * @osr_mask: oversampling ratio bitmask on EMR register + * @oversampling_avail: available oversampling values + * @oversampling_avail_no: number of available oversampling values + * @chan_realbits: realbits for registered channels + * @temp_chan: temperature channel index + * @temp_sensor: temperature sensor supported */ struct at91_adc_platform { const struct at91_adc_reg_layout *layout; @@ -414,20 +462,58 @@ struct at91_adc_platform { unsigned int max_channels; unsigned int max_index; unsigned int hw_trig_cnt; + unsigned int osr_mask; + unsigned int oversampling_avail[5]; + unsigned int oversampling_avail_no; + unsigned int chan_realbits; + unsigned int temp_chan; + bool temp_sensor; +}; + +/** + * struct at91_adc_temp_sensor_clb - at91-sama5d2 temperature sensor + * calibration data structure + * @p1: P1 calibration temperature + * @p4: P4 calibration voltage + * @p6: P6 calibration voltage + */ +struct at91_adc_temp_sensor_clb { + u32 p1; + u32 p4; + u32 p6; +}; + +/** + * enum at91_adc_ts_clb_idx - calibration indexes in NVMEM buffer + * @AT91_ADC_TS_CLB_IDX_P1: index for P1 + * @AT91_ADC_TS_CLB_IDX_P4: index for P4 + * @AT91_ADC_TS_CLB_IDX_P6: index for P6 + * @AT91_ADC_TS_CLB_IDX_MAX: max index for temperature calibration packet in OTP + */ +enum at91_adc_ts_clb_idx { + AT91_ADC_TS_CLB_IDX_P1 = 2, + AT91_ADC_TS_CLB_IDX_P4 = 5, + AT91_ADC_TS_CLB_IDX_P6 = 7, + AT91_ADC_TS_CLB_IDX_MAX = 19, }; +/* Temperature sensor calibration - Vtemp voltage sensitivity to temperature. */ +#define AT91_ADC_TS_VTEMP_DT (2080U) + /** * struct at91_adc_soc_info - at91-sama5d2 soc information struct * @startup_time: device startup time * @min_sample_rate: minimum sample rate in Hz * @max_sample_rate: maximum sample rate in Hz * @platform: pointer to the platform structure + * @temp_sensor_clb: temperature sensor calibration data structure */ struct at91_adc_soc_info { unsigned startup_time; unsigned min_sample_rate; unsigned max_sample_rate; const struct at91_adc_platform *platform; + struct at91_adc_temp_sensor_clb temp_sensor_clb; }; struct at91_adc_trigger { @@ -475,6 +561,18 @@ struct at91_adc_touch { struct work_struct workq; }; +/** + * struct at91_adc_temp - at91-sama5d2 temperature information structure + * @sample_period_val: sample period value + * @saved_sample_rate: saved sample rate + * @saved_oversampling: saved oversampling + */ +struct at91_adc_temp { + u16 sample_period_val; + u16 saved_sample_rate; + u16 saved_oversampling; +}; + /* * Buffer size requirements: * No channels * bytes_per_channel(2) + timestamp bytes (8) @@ -502,7 +600,9 @@ struct at91_adc_state { wait_queue_head_t wq_data_available; struct at91_adc_dma dma_st; struct at91_adc_touch touch_st; + struct at91_adc_temp temp_st; struct iio_dev *indio_dev; + struct device *dev; /* Ensure naturally aligned timestamp */ u16 buffer[AT91_BUFFER_MAX_HWORDS] __aligned(8); /* @@ -591,6 +691,7 @@ static const struct iio_chan_spec at91_sama7g5_adc_channels[] = { AT91_SAMA5D2_CHAN_DIFF(22, 12, 13, 0x90), AT91_SAMA5D2_CHAN_DIFF(23, 14, 15, 0x98), IIO_CHAN_SOFT_TIMESTAMP(24), + AT91_SAMA5D2_CHAN_TEMP(AT91_SAMA7G5_ADC_TEMP_CHANNEL, "temp", 0xdc), }; static const struct at91_adc_platform sama5d2_platform = { @@ -612,6 +713,10 @@ static const struct at91_adc_platform sama5d2_platform = { .max_index = AT91_SAMA5D2_MAX_CHAN_IDX, #define AT91_SAMA5D2_HW_TRIG_CNT 3 .hw_trig_cnt = AT91_SAMA5D2_HW_TRIG_CNT, + .osr_mask = GENMASK(17, 16), + .oversampling_avail = { 1, 4, 16, }, + .oversampling_avail_no = 3, + .chan_realbits = 14, }; static const struct at91_adc_platform sama7g5_platform = { @@ -619,14 +724,23 @@ static const struct at91_adc_platform sama7g5_platform = { .adc_channels = &at91_sama7g5_adc_channels, #define AT91_SAMA7G5_SINGLE_CHAN_CNT 16 #define AT91_SAMA7G5_DIFF_CHAN_CNT 8 +#define AT91_SAMA7G5_TEMP_CHAN_CNT 1 .nr_channels = AT91_SAMA7G5_SINGLE_CHAN_CNT + - AT91_SAMA7G5_DIFF_CHAN_CNT, + AT91_SAMA7G5_DIFF_CHAN_CNT + + AT91_SAMA7G5_TEMP_CHAN_CNT, #define AT91_SAMA7G5_MAX_CHAN_IDX (AT91_SAMA7G5_SINGLE_CHAN_CNT + \ - AT91_SAMA7G5_DIFF_CHAN_CNT) + AT91_SAMA7G5_DIFF_CHAN_CNT + \ + AT91_SAMA7G5_TEMP_CHAN_CNT) .max_channels = ARRAY_SIZE(at91_sama7g5_adc_channels), .max_index = AT91_SAMA7G5_MAX_CHAN_IDX, #define AT91_SAMA7G5_HW_TRIG_CNT 3 .hw_trig_cnt = AT91_SAMA7G5_HW_TRIG_CNT, + .osr_mask = GENMASK(18, 16), + .oversampling_avail = { 1, 4, 16, 64, 256, }, + .oversampling_avail_no = 5, + .chan_realbits = 16, + .temp_sensor = true, + .temp_chan = AT91_SAMA7G5_ADC_TEMP_CHANNEL, }; static int at91_adc_chan_xlate(struct iio_dev *indio_dev, int chan) @@ -650,8 +764,8 @@ at91_adc_chan_get(struct iio_dev *indio_dev, int chan) return indio_dev->channels + index; } -static inline int at91_adc_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static inline int at91_adc_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { return at91_adc_chan_xlate(indio_dev, iiospec->args[0]); } @@ -725,51 +839,91 @@ static void at91_adc_eoc_ena(struct at91_adc_state *st, unsigned int channel) at91_adc_writel(st, EOC_IER, BIT(channel)); } -static void at91_adc_config_emr(struct at91_adc_state *st) +static int at91_adc_config_emr(struct at91_adc_state *st, + u32 oversampling_ratio, u32 trackx) { /* configure the extended mode register */ - unsigned int emr = at91_adc_readl(st, EMR); - - /* select oversampling per single trigger event */ - emr |= AT91_SAMA5D2_EMR_ASTE(1); + unsigned int emr, osr; + unsigned int osr_mask = st->soc_info.platform->osr_mask; + int i, ret; - /* delete leftover content if it's the case */ - emr &= ~AT91_SAMA5D2_EMR_OSR_MASK; + /* Check against supported oversampling values. */ + for (i = 0; i < st->soc_info.platform->oversampling_avail_no; i++) { + if (oversampling_ratio == st->soc_info.platform->oversampling_avail[i]) + break; + } + if (i == st->soc_info.platform->oversampling_avail_no) + return -EINVAL; /* select oversampling ratio from configuration */ - switch (st->oversampling_ratio) { - case AT91_OSR_1SAMPLES: - emr |= AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_1SAMPLES) & - AT91_SAMA5D2_EMR_OSR_MASK; + switch (oversampling_ratio) { + case 1: + osr = AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_1SAMPLES, + osr_mask); break; - case AT91_OSR_4SAMPLES: - emr |= AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_4SAMPLES) & - AT91_SAMA5D2_EMR_OSR_MASK; + case 4: + osr = AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_4SAMPLES, + osr_mask); break; - case AT91_OSR_16SAMPLES: - emr |= AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_16SAMPLES) & - AT91_SAMA5D2_EMR_OSR_MASK; + case 16: + osr = AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_16SAMPLES, + osr_mask); + break; + case 64: + osr = AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_64SAMPLES, + osr_mask); + break; + case 256: + osr = AT91_SAMA5D2_EMR_OSR(AT91_SAMA5D2_EMR_OSR_256SAMPLES, + osr_mask); break; } + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + + emr = at91_adc_readl(st, EMR); + /* select oversampling per single trigger event */ + emr |= AT91_SAMA5D2_EMR_ASTE(1); + /* delete leftover content if it's the case */ + emr &= ~(osr_mask | AT91_SAMA5D2_TRACKX_MASK); + /* Update osr and trackx. */ + emr |= osr | AT91_SAMA5D2_TRACKX(trackx); at91_adc_writel(st, EMR, emr); + + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + + st->oversampling_ratio = oversampling_ratio; + + return 0; } static int at91_adc_adjust_val_osr(struct at91_adc_state *st, int *val) { - if (st->oversampling_ratio == AT91_OSR_1SAMPLES) { - /* - * in this case we only have 12 bits of real data, but channel - * is registered as 14 bits, so shift left two bits - */ - *val <<= 2; - } else if (st->oversampling_ratio == AT91_OSR_4SAMPLES) { - /* - * in this case we have 13 bits of real data, but channel - * is registered as 14 bits, so left shift one bit - */ - *val <<= 1; - } + int nbits, diff; + + if (st->oversampling_ratio == 1) + nbits = 12; + else if (st->oversampling_ratio == 4) + nbits = 13; + else if (st->oversampling_ratio == 16) + nbits = 14; + else if (st->oversampling_ratio == 64) + nbits = 15; + else if (st->oversampling_ratio == 256) + nbits = 16; + else + /* Should not happen. */ + return -EINVAL; + + /* + * We have nbits of real data and channel is registered as + * st->soc_info.platform->chan_realbits, so shift left diff bits. + */ + diff = st->soc_info.platform->chan_realbits - nbits; + *val <<= diff; return IIO_VAL_INT; } @@ -799,15 +953,22 @@ static void at91_adc_adjust_val_osr_array(struct at91_adc_state *st, void *buf, static int at91_adc_configure_touch(struct at91_adc_state *st, bool state) { u32 clk_khz = st->current_sample_rate / 1000; - int i = 0; + int i = 0, ret; u16 pendbc; u32 tsmr, acr; - if (!state) { + if (state) { + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + } else { /* disabling touch IRQs and setting mode to no touch enabled */ at91_adc_writel(st, IDR, AT91_SAMA5D2_IER_PEN | AT91_SAMA5D2_IER_NOPEN); at91_adc_writel(st, TSMR, 0); + + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); return 0; } /* @@ -948,10 +1109,9 @@ static int at91_adc_read_pressure(struct at91_adc_state *st, int chan, u16 *val) return IIO_VAL_INT; } -static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state) +static void at91_adc_configure_trigger_registers(struct at91_adc_state *st, + bool state) { - struct iio_dev *indio = iio_trigger_get_drvdata(trig); - struct at91_adc_state *st = iio_priv(indio); u32 status = at91_adc_readl(st, TRGR); /* clear TRGMOD */ @@ -962,6 +1122,26 @@ static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state) /* set/unset hw trigger */ at91_adc_writel(st, TRGR, status); +} + +static int at91_adc_configure_trigger(struct iio_trigger *trig, bool state) +{ + struct iio_dev *indio = iio_trigger_get_drvdata(trig); + struct at91_adc_state *st = iio_priv(indio); + int ret; + + if (state) { + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + } + + at91_adc_configure_trigger_registers(st, state); + + if (!state) { + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + } return 0; } @@ -1120,11 +1300,15 @@ static int at91_adc_buffer_prepare(struct iio_dev *indio_dev) if (!(iio_device_get_current_mode(indio_dev) & INDIO_ALL_TRIGGERED_MODES)) return -EINVAL; + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + /* we continue with the triggered buffer */ ret = at91_adc_dma_start(indio_dev); if (ret) { dev_err(&indio_dev->dev, "buffer prepare failed\n"); - return ret; + goto pm_runtime_put; } for_each_set_bit(bit, indio_dev->active_scan_mask, @@ -1135,7 +1319,8 @@ static int at91_adc_buffer_prepare(struct iio_dev *indio_dev) continue; /* these channel types cannot be handled by this trigger */ if (chan->type == IIO_POSITIONRELATIVE || - chan->type == IIO_PRESSURE) + chan->type == IIO_PRESSURE || + chan->type == IIO_TEMP) continue; at91_adc_cor(st, chan); @@ -1146,12 +1331,16 @@ static int at91_adc_buffer_prepare(struct iio_dev *indio_dev) if (at91_adc_buffer_check_use_irq(indio_dev, st)) at91_adc_writel(st, IER, AT91_SAMA5D2_IER_DRDY); - return 0; +pm_runtime_put: + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + return ret; } static int at91_adc_buffer_postdisable(struct iio_dev *indio_dev) { struct at91_adc_state *st = iio_priv(indio_dev); + int ret; u8 bit; /* check if we are disabling triggered buffer or the touchscreen */ @@ -1162,6 +1351,10 @@ static int at91_adc_buffer_postdisable(struct iio_dev *indio_dev) if (!(iio_device_get_current_mode(indio_dev) & INDIO_ALL_TRIGGERED_MODES)) return -EINVAL; + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + /* * For each enable channel we must disable it in hardware. * In the case of DMA, we must read the last converted value @@ -1177,7 +1370,8 @@ static int at91_adc_buffer_postdisable(struct iio_dev *indio_dev) continue; /* these channel types are virtual, no need to do anything */ if (chan->type == IIO_POSITIONRELATIVE || - chan->type == IIO_PRESSURE) + chan->type == IIO_PRESSURE || + chan->type == IIO_TEMP) continue; at91_adc_writel(st, CHDR, BIT(chan->channel)); @@ -1196,6 +1390,9 @@ static int at91_adc_buffer_postdisable(struct iio_dev *indio_dev) if (st->dma_st.dma_chan) dmaengine_terminate_sync(st->dma_st.dma_chan); + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + return 0; } @@ -1224,6 +1421,7 @@ static struct iio_trigger *at91_adc_allocate_trigger(struct iio_dev *indio, return trig; } + static void at91_adc_trigger_handler_nodma(struct iio_dev *indio_dev, struct iio_poll_func *pf) { @@ -1377,25 +1575,35 @@ static unsigned at91_adc_startup_time(unsigned startup_time_min, return i; } -static void at91_adc_setup_samp_freq(struct iio_dev *indio_dev, unsigned freq) +static void at91_adc_setup_samp_freq(struct iio_dev *indio_dev, unsigned freq, + unsigned int startup_time, + unsigned int tracktim) { struct at91_adc_state *st = iio_priv(indio_dev); unsigned f_per, prescal, startup, mr; + int ret; f_per = clk_get_rate(st->per_clk); prescal = (f_per / (2 * freq)) - 1; - startup = at91_adc_startup_time(st->soc_info.startup_time, - freq / 1000); + startup = at91_adc_startup_time(startup_time, freq / 1000); + + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return; mr = at91_adc_readl(st, MR); mr &= ~(AT91_SAMA5D2_MR_STARTUP_MASK | AT91_SAMA5D2_MR_PRESCAL_MASK); mr |= AT91_SAMA5D2_MR_STARTUP(startup); mr |= AT91_SAMA5D2_MR_PRESCAL(prescal); + mr |= AT91_SAMA5D2_MR_TRACKTIM(tracktim); at91_adc_writel(st, MR, mr); - dev_dbg(&indio_dev->dev, "freq: %u, startup: %u, prescal: %u\n", - freq, startup, prescal); + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + + dev_dbg(&indio_dev->dev, "freq: %u, startup: %u, prescal: %u, tracktim=%u\n", + freq, startup, prescal, tracktim); st->current_sample_rate = freq; } @@ -1522,6 +1730,7 @@ static irqreturn_t at91_adc_interrupt(int irq, void *private) return IRQ_HANDLED; } +/* This needs to be called with direct mode claimed and st->lock locked. */ static int at91_adc_read_info_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val) { @@ -1529,50 +1738,46 @@ static int at91_adc_read_info_raw(struct iio_dev *indio_dev, u16 tmp_val; int ret; + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + /* * Keep in mind that we cannot use software trigger or touchscreen * if external trigger is enabled */ if (chan->type == IIO_POSITIONRELATIVE) { - ret = iio_device_claim_direct_mode(indio_dev); - if (ret) - return ret; - mutex_lock(&st->lock); - ret = at91_adc_read_position(st, chan->channel, &tmp_val); *val = tmp_val; - mutex_unlock(&st->lock); - iio_device_release_direct_mode(indio_dev); + if (ret > 0) + ret = at91_adc_adjust_val_osr(st, val); - return at91_adc_adjust_val_osr(st, val); + goto pm_runtime_put; } if (chan->type == IIO_PRESSURE) { - ret = iio_device_claim_direct_mode(indio_dev); - if (ret) - return ret; - mutex_lock(&st->lock); - ret = at91_adc_read_pressure(st, chan->channel, &tmp_val); *val = tmp_val; - mutex_unlock(&st->lock); - iio_device_release_direct_mode(indio_dev); + if (ret > 0) + ret = at91_adc_adjust_val_osr(st, val); - return at91_adc_adjust_val_osr(st, val); + goto pm_runtime_put; } - /* in this case we have a voltage channel */ - - ret = iio_device_claim_direct_mode(indio_dev); - if (ret) - return ret; - mutex_lock(&st->lock); + /* in this case we have a voltage or temperature channel */ st->chan = chan; at91_adc_cor(st, chan); at91_adc_writel(st, CHER, BIT(chan->channel)); + /* + * TEMPMR.TEMPON needs to update after CHER otherwise if none + * of the channels are enabled and TEMPMR.TEMPON = 1 will + * trigger DRDY interruption while preparing for temperature read. + */ + if (chan->type == IIO_TEMP) + at91_adc_writel(st, TEMPMR, AT91_SAMA5D2_TEMPMR_TEMPON); at91_adc_eoc_ena(st, chan->channel); at91_adc_writel(st, CR, AT91_SAMA5D2_CR_START); @@ -1592,14 +1797,125 @@ static int at91_adc_read_info_raw(struct iio_dev *indio_dev, } at91_adc_eoc_dis(st, st->chan->channel); + if (chan->type == IIO_TEMP) + at91_adc_writel(st, TEMPMR, 0U); at91_adc_writel(st, CHDR, BIT(chan->channel)); /* Needed to ACK the DRDY interruption */ at91_adc_readl(st, LCDR); +pm_runtime_put: + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + return ret; +} + +static int at91_adc_read_info_locked(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int *val) +{ + struct at91_adc_state *st = iio_priv(indio_dev); + int ret; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + + mutex_lock(&st->lock); + ret = at91_adc_read_info_raw(indio_dev, chan, val); mutex_unlock(&st->lock); iio_device_release_direct_mode(indio_dev); + + return ret; +} + +static void at91_adc_temp_sensor_configure(struct at91_adc_state *st, + bool start) +{ + u32 sample_rate, oversampling_ratio; + u32 startup_time, tracktim, trackx; + + if (start) { + /* + * Configure the sensor for best accuracy: 10MHz frequency, + * oversampling rate of 256, tracktim=0xf and trackx=1. + */ + sample_rate = 10 * MEGA; + oversampling_ratio = 256; + startup_time = AT91_SAMA5D2_MR_STARTUP_TS_MIN; + tracktim = AT91_SAMA5D2_MR_TRACKTIM_TS; + trackx = AT91_SAMA5D2_TRACKX_TS; + + st->temp_st.saved_sample_rate = st->current_sample_rate; + st->temp_st.saved_oversampling = st->oversampling_ratio; + } else { + /* Go back to previous settings. */ + sample_rate = st->temp_st.saved_sample_rate; + oversampling_ratio = st->temp_st.saved_oversampling; + startup_time = st->soc_info.startup_time; + tracktim = 0; + trackx = 0; + } + + at91_adc_setup_samp_freq(st->indio_dev, sample_rate, startup_time, + tracktim); + at91_adc_config_emr(st, oversampling_ratio, trackx); +} + +static int at91_adc_read_temp(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int *val) +{ + struct at91_adc_state *st = iio_priv(indio_dev); + struct at91_adc_temp_sensor_clb *clb = &st->soc_info.temp_sensor_clb; + u64 div1, div2; + u32 tmp; + int ret, vbg, vtemp; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + mutex_lock(&st->lock); + + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + goto unlock; + + at91_adc_temp_sensor_configure(st, true); + + /* Read VBG. */ + tmp = at91_adc_readl(st, ACR); + tmp |= AT91_SAMA5D2_ACR_SRCLCH; + at91_adc_writel(st, ACR, tmp); + ret = at91_adc_read_info_raw(indio_dev, chan, &vbg); + if (ret < 0) + goto restore_config; + + /* Read VTEMP. */ + tmp &= ~AT91_SAMA5D2_ACR_SRCLCH; + at91_adc_writel(st, ACR, tmp); + ret = at91_adc_read_info_raw(indio_dev, chan, &vtemp); + +restore_config: + /* Revert previous settings. */ + at91_adc_temp_sensor_configure(st, false); + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); +unlock: + mutex_unlock(&st->lock); + iio_device_release_direct_mode(indio_dev); + if (ret < 0) + return ret; + + /* + * Temp[milli] = p1[milli] + (vtemp * clb->p6 - clb->p4 * vbg)/ + * (vbg * AT91_ADC_TS_VTEMP_DT) + */ + div1 = DIV_ROUND_CLOSEST_ULL(((u64)vtemp * clb->p6), vbg); + div1 = DIV_ROUND_CLOSEST_ULL((div1 * 1000), AT91_ADC_TS_VTEMP_DT); + div2 = DIV_ROUND_CLOSEST_ULL((u64)clb->p4, AT91_ADC_TS_VTEMP_DT); + div2 *= 1000; + *val = clb->p1 + (int)div1 - (int)div2; + return ret; } @@ -1611,7 +1927,8 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_RAW: - return at91_adc_read_info_raw(indio_dev, chan, val); + return at91_adc_read_info_locked(indio_dev, chan, val); + case IIO_CHAN_INFO_SCALE: *val = st->vref_uv / 1000; if (chan->differential) @@ -1619,6 +1936,11 @@ static int at91_adc_read_raw(struct iio_dev *indio_dev, *val2 = chan->scan_type.realbits; return IIO_VAL_FRACTIONAL_LOG2; + case IIO_CHAN_INFO_PROCESSED: + if (chan->type != IIO_TEMP) + return -EINVAL; + return at91_adc_read_temp(indio_dev, chan, val); + case IIO_CHAN_INFO_SAMP_FREQ: *val = at91_adc_get_sample_freq(st); return IIO_VAL_INT; @@ -1637,31 +1959,60 @@ static int at91_adc_write_raw(struct iio_dev *indio_dev, int val, int val2, long mask) { struct at91_adc_state *st = iio_priv(indio_dev); + int ret; switch (mask) { case IIO_CHAN_INFO_OVERSAMPLING_RATIO: - if ((val != AT91_OSR_1SAMPLES) && (val != AT91_OSR_4SAMPLES) && - (val != AT91_OSR_16SAMPLES)) - return -EINVAL; /* if no change, optimize out */ if (val == st->oversampling_ratio) return 0; - st->oversampling_ratio = val; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + mutex_lock(&st->lock); /* update ratio */ - at91_adc_config_emr(st); - return 0; + ret = at91_adc_config_emr(st, val, 0); + mutex_unlock(&st->lock); + iio_device_release_direct_mode(indio_dev); + return ret; case IIO_CHAN_INFO_SAMP_FREQ: if (val < st->soc_info.min_sample_rate || val > st->soc_info.max_sample_rate) return -EINVAL; - at91_adc_setup_samp_freq(indio_dev, val); + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + mutex_lock(&st->lock); + at91_adc_setup_samp_freq(indio_dev, val, + st->soc_info.startup_time, 0); + mutex_unlock(&st->lock); + iio_device_release_direct_mode(indio_dev); return 0; default: return -EINVAL; } } +static int at91_adc_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + struct at91_adc_state *st = iio_priv(indio_dev); + + switch (mask) { + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + *vals = (int *)st->soc_info.platform->oversampling_avail; + *type = IIO_VAL_INT; + *length = st->soc_info.platform->oversampling_avail_no; + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + static void at91_adc_dma_init(struct at91_adc_state *st) { struct device *dev = &st->indio_dev->dev; @@ -1817,10 +2168,11 @@ static void at91_adc_hw_init(struct iio_dev *indio_dev) at91_adc_writel(st, MR, AT91_SAMA5D2_MR_TRANSFER(2) | AT91_SAMA5D2_MR_ANACH); - at91_adc_setup_samp_freq(indio_dev, st->soc_info.min_sample_rate); + at91_adc_setup_samp_freq(indio_dev, st->soc_info.min_sample_rate, + st->soc_info.startup_time, 0); /* configure extended mode register */ - at91_adc_config_emr(st); + at91_adc_config_emr(st, st->oversampling_ratio, 0); } static ssize_t at91_adc_get_fifo_state(struct device *dev, @@ -1849,20 +2201,6 @@ static IIO_DEVICE_ATTR(hwfifo_watermark, 0444, static IIO_CONST_ATTR(hwfifo_watermark_min, "2"); static IIO_CONST_ATTR(hwfifo_watermark_max, AT91_HWFIFO_MAX_SIZE_STR); -static IIO_CONST_ATTR(oversampling_ratio_available, - __stringify(AT91_OSR_1SAMPLES) " " - __stringify(AT91_OSR_4SAMPLES) " " - __stringify(AT91_OSR_16SAMPLES)); - -static struct attribute *at91_adc_attributes[] = { - &iio_const_attr_oversampling_ratio_available.dev_attr.attr, - NULL, -}; - -static const struct attribute_group at91_adc_attribute_group = { - .attrs = at91_adc_attributes, -}; - static const struct attribute *at91_adc_fifo_attributes[] = { &iio_const_attr_hwfifo_watermark_min.dev_attr.attr, &iio_const_attr_hwfifo_watermark_max.dev_attr.attr, @@ -1872,11 +2210,11 @@ static const struct attribute *at91_adc_fifo_attributes[] = { }; static const struct iio_info at91_adc_info = { - .attrs = &at91_adc_attribute_group, + .read_avail = &at91_adc_read_avail, .read_raw = &at91_adc_read_raw, .write_raw = &at91_adc_write_raw, .update_scan_mode = &at91_adc_update_scan_mode, - .of_xlate = &at91_adc_of_xlate, + .fwnode_xlate = &at91_adc_fwnode_xlate, .hwfifo_set_watermark = &at91_adc_set_watermark, }; @@ -1918,12 +2256,62 @@ static int at91_adc_buffer_and_trigger_init(struct device *dev, return 0; } +static int at91_adc_temp_sensor_init(struct at91_adc_state *st, + struct device *dev) +{ + struct at91_adc_temp_sensor_clb *clb = &st->soc_info.temp_sensor_clb; + struct nvmem_cell *temp_calib; + u32 *buf; + size_t len; + int ret = 0; + + if (!st->soc_info.platform->temp_sensor) + return 0; + + /* Get the calibration data from NVMEM. */ + temp_calib = devm_nvmem_cell_get(dev, "temperature_calib"); + if (IS_ERR(temp_calib)) { + ret = PTR_ERR(temp_calib); + if (ret != -ENOENT) + dev_err(dev, "Failed to get temperature_calib cell!\n"); + return ret; + } + + buf = nvmem_cell_read(temp_calib, &len); + if (IS_ERR(buf)) { + dev_err(dev, "Failed to read calibration data!\n"); + return PTR_ERR(buf); + } + if (len < AT91_ADC_TS_CLB_IDX_MAX * 4) { + dev_err(dev, "Invalid calibration data!\n"); + ret = -EINVAL; + goto free_buf; + } + + /* Store calibration data for later use. */ + clb->p1 = buf[AT91_ADC_TS_CLB_IDX_P1]; + clb->p4 = buf[AT91_ADC_TS_CLB_IDX_P4]; + clb->p6 = buf[AT91_ADC_TS_CLB_IDX_P6]; + + /* + * We prepare here the conversion to milli and also add constant + * factor (5 degrees Celsius) to p1 here to avoid doing it on + * hotpath. + */ + clb->p1 = clb->p1 * 1000 + 5000; + +free_buf: + kfree(buf); + return ret; +} + static int at91_adc_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct iio_dev *indio_dev; struct at91_adc_state *st; struct resource *res; - int ret, i; + int ret, i, num_channels; u32 edge_type = IRQ_TYPE_NONE; indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*st)); @@ -1933,13 +2321,20 @@ static int at91_adc_probe(struct platform_device *pdev) st = iio_priv(indio_dev); st->indio_dev = indio_dev; - st->soc_info.platform = of_device_get_match_data(&pdev->dev); + st->soc_info.platform = device_get_match_data(dev); + + ret = at91_adc_temp_sensor_init(st, &pdev->dev); + /* Don't register temperature channel if initialization failed. */ + if (ret) + num_channels = st->soc_info.platform->max_channels - 1; + else + num_channels = st->soc_info.platform->max_channels; indio_dev->name = dev_name(&pdev->dev); indio_dev->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE; indio_dev->info = &at91_adc_info; indio_dev->channels = *st->soc_info.platform->adc_channels; - indio_dev->num_channels = st->soc_info.platform->max_channels; + indio_dev->num_channels = num_channels; bitmap_set(&st->touch_st.channels_bitmask, st->soc_info.platform->touch_chan_x, 1); @@ -1948,36 +2343,34 @@ static int at91_adc_probe(struct platform_device *pdev) bitmap_set(&st->touch_st.channels_bitmask, st->soc_info.platform->touch_chan_p, 1); - st->oversampling_ratio = AT91_OSR_1SAMPLES; + st->oversampling_ratio = 1; - ret = of_property_read_u32(pdev->dev.of_node, - "atmel,min-sample-rate-hz", - &st->soc_info.min_sample_rate); + ret = device_property_read_u32(dev, "atmel,min-sample-rate-hz", + &st->soc_info.min_sample_rate); if (ret) { dev_err(&pdev->dev, "invalid or missing value for atmel,min-sample-rate-hz\n"); return ret; } - ret = of_property_read_u32(pdev->dev.of_node, - "atmel,max-sample-rate-hz", - &st->soc_info.max_sample_rate); + ret = device_property_read_u32(dev, "atmel,max-sample-rate-hz", + &st->soc_info.max_sample_rate); if (ret) { dev_err(&pdev->dev, "invalid or missing value for atmel,max-sample-rate-hz\n"); return ret; } - ret = of_property_read_u32(pdev->dev.of_node, "atmel,startup-time-ms", - &st->soc_info.startup_time); + ret = device_property_read_u32(dev, "atmel,startup-time-ms", + &st->soc_info.startup_time); if (ret) { dev_err(&pdev->dev, "invalid or missing value for atmel,startup-time-ms\n"); return ret; } - ret = of_property_read_u32(pdev->dev.of_node, - "atmel,trigger-edge-type", &edge_type); + ret = device_property_read_u32(dev, "atmel,trigger-edge-type", + &edge_type); if (ret) { dev_dbg(&pdev->dev, "atmel,trigger-edge-type not specified, only software trigger available\n"); @@ -2051,13 +2444,19 @@ static int at91_adc_probe(struct platform_device *pdev) if (ret) goto vref_disable; - at91_adc_hw_init(indio_dev); - platform_set_drvdata(pdev, indio_dev); + st->dev = &pdev->dev; + pm_runtime_set_autosuspend_delay(st->dev, 500); + pm_runtime_use_autosuspend(st->dev); + pm_runtime_set_active(st->dev); + pm_runtime_enable(st->dev); + pm_runtime_get_noresume(st->dev); + + at91_adc_hw_init(indio_dev); ret = at91_adc_buffer_and_trigger_init(&pdev->dev, indio_dev); if (ret < 0) - goto per_clk_disable_unprepare; + goto err_pm_disable; if (dma_coerce_mask_and_coherent(&indio_dev->dev, DMA_BIT_MASK(32))) dev_info(&pdev->dev, "cannot set DMA mask to 32-bit\n"); @@ -2073,11 +2472,18 @@ static int at91_adc_probe(struct platform_device *pdev) dev_info(&pdev->dev, "version: %x\n", readl_relaxed(st->base + st->soc_info.platform->layout->VERSION)); + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); + return 0; dma_disable: at91_adc_dma_disable(st); -per_clk_disable_unprepare: +err_pm_disable: + pm_runtime_put_noidle(st->dev); + pm_runtime_disable(st->dev); + pm_runtime_set_suspended(st->dev); + pm_runtime_dont_use_autosuspend(st->dev); clk_disable_unprepare(st->per_clk); vref_disable: regulator_disable(st->vref); @@ -2095,6 +2501,8 @@ static int at91_adc_remove(struct platform_device *pdev) at91_adc_dma_disable(st); + pm_runtime_disable(st->dev); + pm_runtime_set_suspended(st->dev); clk_disable_unprepare(st->per_clk); regulator_disable(st->vref); @@ -2107,6 +2515,14 @@ static int at91_adc_suspend(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct at91_adc_state *st = iio_priv(indio_dev); + int ret; + + ret = pm_runtime_resume_and_get(st->dev); + if (ret < 0) + return ret; + + if (iio_buffer_enabled(indio_dev)) + at91_adc_buffer_postdisable(indio_dev); /* * Do a sofware reset of the ADC before we go to suspend. @@ -2116,6 +2532,8 @@ static int at91_adc_suspend(struct device *dev) */ at91_adc_writel(st, CR, AT91_SAMA5D2_CR_SWRST); + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_noidle(st->dev); clk_disable_unprepare(st->per_clk); regulator_disable(st->vref); regulator_disable(st->reg); @@ -2145,21 +2563,28 @@ static int at91_adc_resume(struct device *dev) if (ret) goto vref_disable_resume; + pm_runtime_get_noresume(st->dev); + at91_adc_hw_init(indio_dev); /* reconfiguring trigger hardware state */ - if (!iio_buffer_enabled(indio_dev)) - return 0; + if (iio_buffer_enabled(indio_dev)) { + ret = at91_adc_buffer_prepare(indio_dev); + if (ret) + goto pm_runtime_put; - /* check if we are enabling triggered buffer or the touchscreen */ - if (at91_adc_current_chan_is_touch(indio_dev)) - return at91_adc_configure_touch(st, true); - else - return at91_adc_configure_trigger(st->trig, true); + at91_adc_configure_trigger_registers(st, true); + } + + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_autosuspend(st->dev); - /* not needed but more explicit */ return 0; +pm_runtime_put: + pm_runtime_mark_last_busy(st->dev); + pm_runtime_put_noidle(st->dev); + clk_disable_unprepare(st->per_clk); vref_disable_resume: regulator_disable(st->vref); reg_disable_resume: @@ -2169,8 +2594,29 @@ resume_failed: return ret; } -static DEFINE_SIMPLE_DEV_PM_OPS(at91_adc_pm_ops, at91_adc_suspend, - at91_adc_resume); +static int at91_adc_runtime_suspend(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct at91_adc_state *st = iio_priv(indio_dev); + + clk_disable(st->per_clk); + + return 0; +} + +static int at91_adc_runtime_resume(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct at91_adc_state *st = iio_priv(indio_dev); + + return clk_enable(st->per_clk); +} + +static const struct dev_pm_ops at91_adc_pm_ops = { + SYSTEM_SLEEP_PM_OPS(at91_adc_suspend, at91_adc_resume) + RUNTIME_PM_OPS(at91_adc_runtime_suspend, at91_adc_runtime_resume, + NULL) +}; static const struct of_device_id at91_adc_dt_match[] = { { @@ -2191,7 +2637,7 @@ static struct platform_driver at91_adc_driver = { .driver = { .name = "at91-sama5d2_adc", .of_match_table = at91_adc_dt_match, - .pm = pm_sleep_ptr(&at91_adc_pm_ops), + .pm = pm_ptr(&at91_adc_pm_ops), }, }; module_platform_driver(at91_adc_driver) diff --git a/drivers/iio/adc/imx8qxp-adc.c b/drivers/iio/adc/imx8qxp-adc.c index e48446784a0a3ff74317e807b501bf26c9b0ec16..36777b827165a0e5be3504d19426eaa4796c87fa 100644 --- a/drivers/iio/adc/imx8qxp-adc.c +++ b/drivers/iio/adc/imx8qxp-adc.c @@ -202,7 +202,7 @@ static int imx8qxp_adc_read_raw(struct iio_dev *indio_dev, struct imx8qxp_adc *adc = iio_priv(indio_dev); struct device *dev = adc->dev; - u32 ctrl, vref_uv; + u32 ctrl; long ret; switch (mask) { @@ -245,8 +245,10 @@ static int imx8qxp_adc_read_raw(struct iio_dev *indio_dev, return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: - vref_uv = regulator_get_voltage(adc->vref); - *val = vref_uv / 1000; + ret = regulator_get_voltage(adc->vref); + if (ret < 0) + return ret; + *val = ret / 1000; *val2 = 12; return IIO_VAL_FRACTIONAL_LOG2; diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c index 240e6c420701cc6b655df07259481d7831d260de..910e7e965fc486415c1f65770bc05e3fa72af678 100644 --- a/drivers/iio/adc/ina2xx-adc.c +++ b/drivers/iio/adc/ina2xx-adc.c @@ -1034,7 +1034,7 @@ static int ina2xx_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int ina2xx_remove(struct i2c_client *client) +static void ina2xx_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ina2xx_chip_info *chip = iio_priv(indio_dev); @@ -1048,8 +1048,6 @@ static int ina2xx_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to power down device (%pe)\n", ERR_PTR(ret)); - - return 0; } static const struct i2c_device_id ina2xx_id[] = { diff --git a/drivers/iio/adc/ingenic-adc.c b/drivers/iio/adc/ingenic-adc.c index bf5c03c34f8442e2fc180c75f8d058b426df4388..a7325dbbb99a0417ed00dba18fc98fe622b0289c 100644 --- a/drivers/iio/adc/ingenic-adc.c +++ b/drivers/iio/adc/ingenic-adc.c @@ -719,12 +719,12 @@ static int ingenic_adc_read_raw(struct iio_dev *iio_dev, } } -static int ingenic_adc_of_xlate(struct iio_dev *iio_dev, - const struct of_phandle_args *iiospec) +static int ingenic_adc_fwnode_xlate(struct iio_dev *iio_dev, + const struct fwnode_reference_args *iiospec) { int i; - if (!iiospec->args_count) + if (!iiospec->nargs) return -EINVAL; for (i = 0; i < iio_dev->num_channels; ++i) @@ -734,16 +734,11 @@ static int ingenic_adc_of_xlate(struct iio_dev *iio_dev, return -EINVAL; } -static void ingenic_adc_clk_cleanup(void *data) -{ - clk_unprepare(data); -} - static const struct iio_info ingenic_adc_info = { .write_raw = ingenic_adc_write_raw, .read_raw = ingenic_adc_read_raw, .read_avail = ingenic_adc_read_avail, - .of_xlate = ingenic_adc_of_xlate, + .fwnode_xlate = ingenic_adc_fwnode_xlate, }; static int ingenic_adc_buffer_enable(struct iio_dev *iio_dev) @@ -858,13 +853,13 @@ static int ingenic_adc_probe(struct platform_device *pdev) if (IS_ERR(adc->base)) return PTR_ERR(adc->base); - adc->clk = devm_clk_get(dev, "adc"); + adc->clk = devm_clk_get_prepared(dev, "adc"); if (IS_ERR(adc->clk)) { dev_err(dev, "Unable to get clock\n"); return PTR_ERR(adc->clk); } - ret = clk_prepare_enable(adc->clk); + ret = clk_enable(adc->clk); if (ret) { dev_err(dev, "Failed to enable clock\n"); return ret; @@ -893,12 +888,6 @@ static int ingenic_adc_probe(struct platform_device *pdev) usleep_range(2000, 3000); /* Must wait at least 2ms. */ clk_disable(adc->clk); - ret = devm_add_action_or_reset(dev, ingenic_adc_clk_cleanup, adc->clk); - if (ret) { - dev_err(dev, "Unable to add action\n"); - return ret; - } - iio_dev->name = "jz-adc"; iio_dev->modes = INDIO_DIRECT_MODE | INDIO_BUFFER_SOFTWARE; iio_dev->setup_ops = &ingenic_buffer_setup_ops; diff --git a/drivers/iio/adc/lpc18xx_adc.c b/drivers/iio/adc/lpc18xx_adc.c index 42e6cd6fa6f722a37e9e3151dc0884a45308f183..450a243d1f7c0111d99364e031e9f95e0797f880 100644 --- a/drivers/iio/adc/lpc18xx_adc.c +++ b/drivers/iio/adc/lpc18xx_adc.c @@ -121,11 +121,6 @@ static void lpc18xx_clear_cr_reg(void *data) writel(0, adc->base + LPC18XX_ADC_CR); } -static void lpc18xx_clk_disable(void *clk) -{ - clk_disable_unprepare(clk); -} - static void lpc18xx_regulator_disable(void *vref) { regulator_disable(vref); @@ -151,7 +146,7 @@ static int lpc18xx_adc_probe(struct platform_device *pdev) if (IS_ERR(adc->base)) return PTR_ERR(adc->base); - adc->clk = devm_clk_get(&pdev->dev, NULL); + adc->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(adc->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(adc->clk), "error getting clock\n"); @@ -177,17 +172,6 @@ static int lpc18xx_adc_probe(struct platform_device *pdev) if (ret) return ret; - ret = clk_prepare_enable(adc->clk); - if (ret) { - dev_err(&pdev->dev, "unable to enable clock\n"); - return ret; - } - - ret = devm_add_action_or_reset(&pdev->dev, lpc18xx_clk_disable, - adc->clk); - if (ret) - return ret; - rate = clk_get_rate(adc->clk); clkdiv = DIV_ROUND_UP(rate, LPC18XX_ADC_CLK_TARGET); diff --git a/drivers/iio/adc/ltc2496.c b/drivers/iio/adc/ltc2496.c index dfb3bb5997e579f1cf1cf32456444d026820eeaf..2593fa4322ebd59d02ff98cb275800b2cb929804 100644 --- a/drivers/iio/adc/ltc2496.c +++ b/drivers/iio/adc/ltc2496.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "ltc2497.h" @@ -74,6 +75,7 @@ static int ltc2496_probe(struct spi_device *spi) spi_set_drvdata(spi, indio_dev); st->spi = spi; st->common_ddata.result_and_measure = ltc2496_result_and_measure; + st->common_ddata.chip_info = device_get_match_data(dev); return ltc2497core_probe(dev, indio_dev); } @@ -85,8 +87,13 @@ static void ltc2496_remove(struct spi_device *spi) ltc2497core_remove(indio_dev); } +static const struct ltc2497_chip_info ltc2496_info = { + .resolution = 16, + .name = NULL, +}; + static const struct of_device_id ltc2496_of_match[] = { - { .compatible = "lltc,ltc2496", }, + { .compatible = "lltc,ltc2496", .data = <c2496_info, }, {}, }; MODULE_DEVICE_TABLE(of, ltc2496_of_match); diff --git a/drivers/iio/adc/ltc2497-core.c b/drivers/iio/adc/ltc2497-core.c index 2a485c8a194037ec82b390d9bebcc739f63d97e0..f52d37af4d1f19d61afd85124ecffbb8ca62c60a 100644 --- a/drivers/iio/adc/ltc2497-core.c +++ b/drivers/iio/adc/ltc2497-core.c @@ -95,7 +95,7 @@ static int ltc2497core_read_raw(struct iio_dev *indio_dev, return ret; *val = ret / 1000; - *val2 = 17; + *val2 = ddata->chip_info->resolution + 1; return IIO_VAL_FRACTIONAL_LOG2; @@ -169,7 +169,15 @@ int ltc2497core_probe(struct device *dev, struct iio_dev *indio_dev) struct ltc2497core_driverdata *ddata = iio_priv(indio_dev); int ret; - indio_dev->name = dev_name(dev); + /* + * Keep using dev_name() for the iio_dev's name on some of the parts, + * since updating it would result in a ABI breakage. + */ + if (ddata->chip_info->name) + indio_dev->name = ddata->chip_info->name; + else + indio_dev->name = dev_name(dev); + indio_dev->info = <c2497core_info; indio_dev->modes = INDIO_DIRECT_MODE; indio_dev->channels = ltc2497core_channel; diff --git a/drivers/iio/adc/ltc2497.c b/drivers/iio/adc/ltc2497.c index f7c786f37ceb141743bf2990dc80e7a0b2c206a3..556f10dfb5023f7da9f2d75247f820ba1774e6c7 100644 --- a/drivers/iio/adc/ltc2497.c +++ b/drivers/iio/adc/ltc2497.c @@ -12,18 +12,31 @@ #include #include #include +#include + +#include #include "ltc2497.h" +enum ltc2497_chip_type { + TYPE_LTC2497, + TYPE_LTC2499, +}; + struct ltc2497_driverdata { /* this must be the first member */ struct ltc2497core_driverdata common_ddata; struct i2c_client *client; + u32 recv_size; + u32 sub_lsb; /* * DMA (thus cache coherency maintenance) may require the * transfer buffers to live in their own cache lines. */ - __be32 buf __aligned(IIO_DMA_MINALIGN); + union { + __be32 d32; + u8 d8[3]; + } data __aligned(IIO_DMA_MINALIGN); }; static int ltc2497_result_and_measure(struct ltc2497core_driverdata *ddata, @@ -34,13 +47,43 @@ static int ltc2497_result_and_measure(struct ltc2497core_driverdata *ddata, int ret; if (val) { - ret = i2c_master_recv(st->client, (char *)&st->buf, 3); + if (st->recv_size == 3) + ret = i2c_master_recv(st->client, (char *)&st->data.d8, + st->recv_size); + else + ret = i2c_master_recv(st->client, (char *)&st->data.d32, + st->recv_size); if (ret < 0) { dev_err(&st->client->dev, "i2c_master_recv failed\n"); return ret; } - *val = (be32_to_cpu(st->buf) >> 14) - (1 << 17); + /* + * The data format is 16/24 bit 2s complement, but with an upper sign bit on the + * resolution + 1 position, which is set for positive values only. Given this + * bit's value, subtracting BIT(resolution + 1) from the ADC's result is + * equivalent to a sign extension. + */ + if (st->recv_size == 3) { + *val = (get_unaligned_be24(st->data.d8) >> st->sub_lsb) + - BIT(ddata->chip_info->resolution + 1); + } else { + *val = (be32_to_cpu(st->data.d32) >> st->sub_lsb) + - BIT(ddata->chip_info->resolution + 1); + } + + /* + * The part started a new conversion at the end of the above i2c + * transfer, so if the address didn't change since the last call + * everything is fine and we can return early. + * If not (which should only happen when some sort of bulk + * conversion is implemented) we have to program the new + * address. Note that this probably fails as the conversion that + * was triggered above is like not complete yet and the two + * operations have to be done in a single transfer. + */ + if (ddata->addr_prev == address) + return 0; } ret = i2c_smbus_write_byte(st->client, @@ -54,9 +97,11 @@ static int ltc2497_result_and_measure(struct ltc2497core_driverdata *ddata, static int ltc2497_probe(struct i2c_client *client, const struct i2c_device_id *id) { + const struct ltc2497_chip_info *chip_info; struct iio_dev *indio_dev; struct ltc2497_driverdata *st; struct device *dev = &client->dev; + u32 resolution; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C | I2C_FUNC_SMBUS_WRITE_BYTE)) @@ -71,26 +116,46 @@ static int ltc2497_probe(struct i2c_client *client, st->client = client; st->common_ddata.result_and_measure = ltc2497_result_and_measure; + chip_info = device_get_match_data(dev); + if (!chip_info) + chip_info = (const struct ltc2497_chip_info *)id->driver_data; + st->common_ddata.chip_info = chip_info; + + resolution = chip_info->resolution; + st->sub_lsb = 31 - (resolution + 1); + st->recv_size = BITS_TO_BYTES(resolution) + 1; + return ltc2497core_probe(dev, indio_dev); } -static int ltc2497_remove(struct i2c_client *client) +static void ltc2497_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); ltc2497core_remove(indio_dev); - - return 0; } +static const struct ltc2497_chip_info ltc2497_info[] = { + [TYPE_LTC2497] = { + .resolution = 16, + .name = NULL, + }, + [TYPE_LTC2499] = { + .resolution = 24, + .name = "ltc2499", + }, +}; + static const struct i2c_device_id ltc2497_id[] = { - { "ltc2497", 0 }, + { "ltc2497", (kernel_ulong_t)<c2497_info[TYPE_LTC2497] }, + { "ltc2499", (kernel_ulong_t)<c2497_info[TYPE_LTC2499] }, { } }; MODULE_DEVICE_TABLE(i2c, ltc2497_id); static const struct of_device_id ltc2497_of_match[] = { - { .compatible = "lltc,ltc2497", }, + { .compatible = "lltc,ltc2497", .data = <c2497_info[TYPE_LTC2497] }, + { .compatible = "lltc,ltc2499", .data = <c2497_info[TYPE_LTC2499] }, {}, }; MODULE_DEVICE_TABLE(of, ltc2497_of_match); diff --git a/drivers/iio/adc/ltc2497.h b/drivers/iio/adc/ltc2497.h index d0b42dd6b8ad9891507fb4d4909546de2cffb261..e023de0d88c46ac4701927e412d1306c035db8a4 100644 --- a/drivers/iio/adc/ltc2497.h +++ b/drivers/iio/adc/ltc2497.h @@ -4,9 +4,15 @@ #define LTC2497_CONFIG_DEFAULT LTC2497_ENABLE #define LTC2497_CONVERSION_TIME_MS 150ULL +struct ltc2497_chip_info { + u32 resolution; + const char *name; +}; + struct ltc2497core_driverdata { struct regulator *ref; ktime_t time_prev; + const struct ltc2497_chip_info *chip_info; u8 addr_prev; int (*result_and_measure)(struct ltc2497core_driverdata *ddata, u8 address, int *val); diff --git a/drivers/iio/adc/max11205.c b/drivers/iio/adc/max11205.c new file mode 100644 index 0000000000000000000000000000000000000000..65fc32971ba53255a3298dac5dffd630c506eca2 --- /dev/null +++ b/drivers/iio/adc/max11205.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Maxim MAX11205 16-Bit Delta-Sigma ADC + * + * Datasheet: https://datasheets.maximintegrated.com/en/ds/MAX1240-max11205.pdf + * Copyright (C) 2022 Analog Devices, Inc. + * Author: Ramona Bolboaca + */ + +#include +#include +#include +#include + +#include +#include + +#define MAX11205_BIT_SCALE 15 +#define MAX11205A_OUT_DATA_RATE 116 +#define MAX11205B_OUT_DATA_RATE 13 + +enum max11205_chip_type { + TYPE_MAX11205A, + TYPE_MAX11205B, +}; + +struct max11205_chip_info { + unsigned int out_data_rate; + const char *name; +}; + +struct max11205_state { + const struct max11205_chip_info *chip_info; + struct regulator *vref; + struct ad_sigma_delta sd; +}; + +static const struct ad_sigma_delta_info max11205_sigma_delta_info = { + .has_registers = false, +}; + +static int max11205_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct max11205_state *st = iio_priv(indio_dev); + int reg_mv; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + return ad_sigma_delta_single_conversion(indio_dev, chan, val); + case IIO_CHAN_INFO_SCALE: + reg_mv = regulator_get_voltage(st->vref); + if (reg_mv < 0) + return reg_mv; + reg_mv /= 1000; + *val = reg_mv; + *val2 = MAX11205_BIT_SCALE; + return IIO_VAL_FRACTIONAL_LOG2; + case IIO_CHAN_INFO_SAMP_FREQ: + *val = st->chip_info->out_data_rate; + return IIO_VAL_INT; + default: + return -EINVAL; + } +} + +static const struct iio_info max11205_iio_info = { + .read_raw = max11205_read_raw, + .validate_trigger = ad_sd_validate_trigger, +}; + +static const struct iio_chan_spec max11205_channels[] = { + { + .type = IIO_VOLTAGE, + .indexed = 1, + .scan_type = { + .sign = 's', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_BE, + }, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_SCALE), + }, +}; + +static const struct max11205_chip_info max11205_chip_info[] = { + [TYPE_MAX11205A] = { + .out_data_rate = MAX11205A_OUT_DATA_RATE, + .name = "max11205a", + }, + [TYPE_MAX11205B] = { + .out_data_rate = MAX11205B_OUT_DATA_RATE, + .name = "max11205b", + }, +}; + +static void max11205_reg_disable(void *reg) +{ + regulator_disable(reg); +} + +static int max11205_probe(struct spi_device *spi) +{ + struct max11205_state *st; + struct iio_dev *indio_dev; + int ret; + + indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st)); + if (!indio_dev) + return -ENOMEM; + + st = iio_priv(indio_dev); + + ad_sd_init(&st->sd, indio_dev, spi, &max11205_sigma_delta_info); + + st->chip_info = device_get_match_data(&spi->dev); + if (!st->chip_info) + st->chip_info = + (const struct max11205_chip_info *)spi_get_device_id(spi)->driver_data; + + indio_dev->name = st->chip_info->name; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = max11205_channels; + indio_dev->num_channels = 1; + indio_dev->info = &max11205_iio_info; + + st->vref = devm_regulator_get(&spi->dev, "vref"); + if (IS_ERR(st->vref)) + return dev_err_probe(&spi->dev, PTR_ERR(st->vref), + "Failed to get vref regulator\n"); + + ret = regulator_enable(st->vref); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&spi->dev, max11205_reg_disable, st->vref); + if (ret) + return ret; + + ret = devm_ad_sd_setup_buffer_and_trigger(&spi->dev, indio_dev); + if (ret) + return ret; + + return devm_iio_device_register(&spi->dev, indio_dev); +} + +static const struct spi_device_id max11205_spi_ids[] = { + { "max11205a", (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205A] }, + { "max11205b", (kernel_ulong_t)&max11205_chip_info[TYPE_MAX11205B] }, + { } +}; +MODULE_DEVICE_TABLE(spi, max11205_spi_ids); + +static const struct of_device_id max11205_dt_ids[] = { + { + .compatible = "maxim,max11205a", + .data = &max11205_chip_info[TYPE_MAX11205A], + }, + { + .compatible = "maxim,max11205b", + .data = &max11205_chip_info[TYPE_MAX11205B], + }, + { } +}; +MODULE_DEVICE_TABLE(of, max11205_dt_ids); + +static struct spi_driver max11205_spi_driver = { + .driver = { + .name = "max11205", + .of_match_table = max11205_dt_ids, + }, + .probe = max11205_probe, + .id_table = max11205_spi_ids, +}; +module_spi_driver(max11205_spi_driver); + +MODULE_AUTHOR("Ramona Bolboaca "); +MODULE_DESCRIPTION("MAX11205 ADC driver"); +MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(IIO_AD_SIGMA_DELTA); diff --git a/drivers/iio/adc/max1363.c b/drivers/iio/adc/max1363.c index eef55ed4814a6b1e5eca15bd9d874ff76ff816f4..a28cf86cdce804672572da2e679914d98f204918 100644 --- a/drivers/iio/adc/max1363.c +++ b/drivers/iio/adc/max1363.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -1595,11 +1594,6 @@ static int max1363_probe(struct i2c_client *client, if (!indio_dev) return -ENOMEM; - ret = devm_iio_map_array_register(&client->dev, indio_dev, - client->dev.platform_data); - if (ret < 0) - return ret; - st = iio_priv(indio_dev); mutex_init(&st->lock); diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c index 1cb4590fe4125a9987d5442a5f31964c50eb358d..b35fd2c9c3c006a5f75413a758174e9e056ed671 100644 --- a/drivers/iio/adc/mcp3911.c +++ b/drivers/iio/adc/mcp3911.c @@ -5,16 +5,25 @@ * Copyright (C) 2018 Marcus Folkesson * Copyright (C) 2018 Kent Gustavsson */ +#include +#include #include #include #include -#include #include #include #include #include #include +#include +#include +#include +#include +#include + +#include + #define MCP3911_REG_CHANNEL0 0x00 #define MCP3911_REG_CHANNEL1 0x03 #define MCP3911_REG_MOD 0x06 @@ -22,6 +31,8 @@ #define MCP3911_REG_GAIN 0x09 #define MCP3911_REG_STATUSCOM 0x0a +#define MCP3911_STATUSCOM_DRHIZ BIT(12) +#define MCP3911_STATUSCOM_READ GENMASK(7, 6) #define MCP3911_STATUSCOM_CH1_24WIDTH BIT(4) #define MCP3911_STATUSCOM_CH0_24WIDTH BIT(3) #define MCP3911_STATUSCOM_EN_OFFCAL BIT(2) @@ -30,6 +41,7 @@ #define MCP3911_REG_CONFIG 0x0c #define MCP3911_CONFIG_CLKEXT BIT(1) #define MCP3911_CONFIG_VREFEXT BIT(2) +#define MCP3911_CONFIG_OSR GENMASK(13, 11) #define MCP3911_REG_OFFCAL_CH0 0x0e #define MCP3911_REG_GAINCAL_CH0 0x11 @@ -40,20 +52,30 @@ #define MCP3911_CHANNEL(x) (MCP3911_REG_CHANNEL0 + x * 3) #define MCP3911_OFFCAL(x) (MCP3911_REG_OFFCAL_CH0 + x * 6) -/* Internal voltage reference in uV */ -#define MCP3911_INT_VREF_UV 1200000 +/* Internal voltage reference in mV */ +#define MCP3911_INT_VREF_MV 1200 #define MCP3911_REG_READ(reg, id) ((((reg) << 1) | ((id) << 5) | (1 << 0)) & 0xff) #define MCP3911_REG_WRITE(reg, id) ((((reg) << 1) | ((id) << 5) | (0 << 0)) & 0xff) #define MCP3911_NUM_CHANNELS 2 +static const int mcp3911_osr_table[] = { 32, 64, 128, 256, 512, 1024, 2048, 4096 }; + struct mcp3911 { struct spi_device *spi; struct mutex lock; struct regulator *vref; struct clk *clki; u32 dev_addr; + struct iio_trigger *trig; + struct { + u32 channels[MCP3911_NUM_CHANNELS]; + s64 ts __aligned(8); + } scan; + + u8 tx_buf __aligned(IIO_DMA_MINALIGN); + u8 rx_buf[MCP3911_NUM_CHANNELS * 3]; }; static int mcp3911_read(struct mcp3911 *adc, u8 reg, u32 *val, u8 len) @@ -98,6 +120,36 @@ static int mcp3911_update(struct mcp3911 *adc, u8 reg, u32 mask, return mcp3911_write(adc, reg, val, len); } +static int mcp3911_write_raw_get_fmt(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_SCALE: + return IIO_VAL_INT_PLUS_NANO; + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + return IIO_VAL_INT; + default: + return IIO_VAL_INT_PLUS_NANO; + } +} + +static int mcp3911_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long info) +{ + switch (info) { + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + *type = IIO_VAL_INT; + *vals = mcp3911_osr_table; + *length = ARRAY_SIZE(mcp3911_osr_table); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + static int mcp3911_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *channel, int *val, int *val2, long mask) @@ -113,6 +165,8 @@ static int mcp3911_read_raw(struct iio_dev *indio_dev, if (ret) goto out; + *val = sign_extend32(*val, 23); + ret = IIO_VAL_INT; break; @@ -124,6 +178,15 @@ static int mcp3911_read_raw(struct iio_dev *indio_dev, ret = IIO_VAL_INT; break; + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + ret = mcp3911_read(adc, MCP3911_REG_CONFIG, val, 2); + if (ret) + goto out; + + *val = FIELD_GET(MCP3911_CONFIG_OSR, *val); + *val = 32 << *val; + ret = IIO_VAL_INT; + break; case IIO_CHAN_INFO_SCALE: if (adc->vref) { @@ -137,11 +200,18 @@ static int mcp3911_read_raw(struct iio_dev *indio_dev, *val = ret / 1000; } else { - *val = MCP3911_INT_VREF_UV; + *val = MCP3911_INT_VREF_MV; } - *val2 = 24; - ret = IIO_VAL_FRACTIONAL_LOG2; + /* + * For 24bit Conversion + * Raw = ((Voltage)/(Vref) * 2^23 * Gain * 1.5 + * Voltage = Raw * (Vref)/(2^23 * Gain * 1.5) + */ + + /* val2 = (2^23 * 1.5) */ + *val2 = 12582912; + ret = IIO_VAL_FRACTIONAL; break; } @@ -176,6 +246,17 @@ static int mcp3911_write_raw(struct iio_dev *indio_dev, MCP3911_STATUSCOM_EN_OFFCAL, MCP3911_STATUSCOM_EN_OFFCAL, 2); break; + + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + for (int i = 0; i < sizeof(mcp3911_osr_table); i++) { + if (val == mcp3911_osr_table[i]) { + val = FIELD_PREP(MCP3911_CONFIG_OSR, i); + ret = mcp3911_update(adc, MCP3911_REG_CONFIG, MCP3911_CONFIG_OSR, + val, 2); + break; + } + } + break; } out: @@ -187,28 +268,90 @@ out: .type = IIO_VOLTAGE, \ .indexed = 1, \ .channel = idx, \ + .scan_index = idx, \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | \ BIT(IIO_CHAN_INFO_OFFSET) | \ BIT(IIO_CHAN_INFO_SCALE), \ + .info_mask_shared_by_type_available = \ + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), \ + .scan_type = { \ + .sign = 's', \ + .realbits = 24, \ + .storagebits = 32, \ + .endianness = IIO_BE, \ + }, \ } static const struct iio_chan_spec mcp3911_channels[] = { MCP3911_CHAN(0), MCP3911_CHAN(1), + IIO_CHAN_SOFT_TIMESTAMP(2), }; +static irqreturn_t mcp3911_trigger_handler(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct iio_dev *indio_dev = pf->indio_dev; + struct mcp3911 *adc = iio_priv(indio_dev); + struct spi_transfer xfer[] = { + { + .tx_buf = &adc->tx_buf, + .len = 1, + }, { + .rx_buf = adc->rx_buf, + .len = sizeof(adc->rx_buf), + }, + }; + int scan_index; + int i = 0; + int ret; + + mutex_lock(&adc->lock); + adc->tx_buf = MCP3911_REG_READ(MCP3911_CHANNEL(0), adc->dev_addr); + ret = spi_sync_transfer(adc->spi, xfer, ARRAY_SIZE(xfer)); + if (ret < 0) { + dev_warn(&adc->spi->dev, + "failed to get conversion data\n"); + goto out; + } + + for_each_set_bit(scan_index, indio_dev->active_scan_mask, indio_dev->masklength) { + const struct iio_chan_spec *scan_chan = &indio_dev->channels[scan_index]; + + adc->scan.channels[i] = get_unaligned_be24(&adc->rx_buf[scan_chan->channel * 3]); + i++; + } + iio_push_to_buffers_with_timestamp(indio_dev, &adc->scan, + iio_get_time_ns(indio_dev)); +out: + mutex_unlock(&adc->lock); + iio_trigger_notify_done(indio_dev->trig); + + return IRQ_HANDLED; +} + static const struct iio_info mcp3911_info = { .read_raw = mcp3911_read_raw, .write_raw = mcp3911_write_raw, + .read_avail = mcp3911_read_avail, + .write_raw_get_fmt = mcp3911_write_raw_get_fmt, }; static int mcp3911_config(struct mcp3911 *adc) { struct device *dev = &adc->spi->dev; - u32 configreg; + u32 regval; int ret; - device_property_read_u32(dev, "device-addr", &adc->dev_addr); + ret = device_property_read_u32(dev, "microchip,device-addr", &adc->dev_addr); + + /* + * Fallback to "device-addr" due to historical mismatch between + * dt-bindings and implementation + */ + if (ret) + device_property_read_u32(dev, "device-addr", &adc->dev_addr); if (adc->dev_addr > 3) { dev_err(&adc->spi->dev, "invalid device address (%i). Must be in range 0-3.\n", @@ -217,31 +360,67 @@ static int mcp3911_config(struct mcp3911 *adc) } dev_dbg(&adc->spi->dev, "use device address %i\n", adc->dev_addr); - ret = mcp3911_read(adc, MCP3911_REG_CONFIG, &configreg, 2); + ret = mcp3911_read(adc, MCP3911_REG_CONFIG, ®val, 2); if (ret) return ret; + regval &= ~MCP3911_CONFIG_VREFEXT; if (adc->vref) { dev_dbg(&adc->spi->dev, "use external voltage reference\n"); - configreg |= MCP3911_CONFIG_VREFEXT; + regval |= FIELD_PREP(MCP3911_CONFIG_VREFEXT, 1); } else { dev_dbg(&adc->spi->dev, "use internal voltage reference (1.2V)\n"); - configreg &= ~MCP3911_CONFIG_VREFEXT; + regval |= FIELD_PREP(MCP3911_CONFIG_VREFEXT, 0); } + regval &= ~MCP3911_CONFIG_CLKEXT; if (adc->clki) { dev_dbg(&adc->spi->dev, "use external clock as clocksource\n"); - configreg |= MCP3911_CONFIG_CLKEXT; + regval |= FIELD_PREP(MCP3911_CONFIG_CLKEXT, 1); } else { dev_dbg(&adc->spi->dev, "use crystal oscillator as clocksource\n"); - configreg &= ~MCP3911_CONFIG_CLKEXT; + regval |= FIELD_PREP(MCP3911_CONFIG_CLKEXT, 0); } - return mcp3911_write(adc, MCP3911_REG_CONFIG, configreg, 2); + ret = mcp3911_write(adc, MCP3911_REG_CONFIG, regval, 2); + if (ret) + return ret; + + ret = mcp3911_read(adc, MCP3911_REG_STATUSCOM, ®val, 2); + if (ret) + return ret; + + /* Address counter incremented, cycle through register types */ + regval &= ~MCP3911_STATUSCOM_READ; + regval |= FIELD_PREP(MCP3911_STATUSCOM_READ, 0x02); + + return mcp3911_write(adc, MCP3911_REG_STATUSCOM, regval, 2); } +static void mcp3911_cleanup_regulator(void *vref) +{ + regulator_disable(vref); +} + +static int mcp3911_set_trigger_state(struct iio_trigger *trig, bool enable) +{ + struct mcp3911 *adc = iio_trigger_get_drvdata(trig); + + if (enable) + enable_irq(adc->spi->irq); + else + disable_irq(adc->spi->irq); + + return 0; +} + +static const struct iio_trigger_ops mcp3911_trigger_ops = { + .validate_device = iio_trigger_validate_own_device, + .set_trigger_state = mcp3911_set_trigger_state, +}; + static int mcp3911_probe(struct spi_device *spi) { struct iio_dev *indio_dev; @@ -270,9 +449,14 @@ static int mcp3911_probe(struct spi_device *spi) ret = regulator_enable(adc->vref); if (ret) return ret; + + ret = devm_add_action_or_reset(&spi->dev, + mcp3911_cleanup_regulator, adc->vref); + if (ret) + return ret; } - adc->clki = devm_clk_get(&adc->spi->dev, NULL); + adc->clki = devm_clk_get_enabled(&adc->spi->dev, NULL); if (IS_ERR(adc->clki)) { if (PTR_ERR(adc->clki) == -ENOENT) { adc->clki = NULL; @@ -280,21 +464,22 @@ static int mcp3911_probe(struct spi_device *spi) dev_err(&adc->spi->dev, "failed to get adc clk (%ld)\n", PTR_ERR(adc->clki)); - ret = PTR_ERR(adc->clki); - goto reg_disable; - } - } else { - ret = clk_prepare_enable(adc->clki); - if (ret < 0) { - dev_err(&adc->spi->dev, - "Failed to enable clki: %d\n", ret); - goto reg_disable; + return PTR_ERR(adc->clki); } } ret = mcp3911_config(adc); if (ret) - goto clk_disable; + return ret; + + if (device_property_read_bool(&adc->spi->dev, "microchip,data-ready-hiz")) + ret = mcp3911_update(adc, MCP3911_REG_STATUSCOM, MCP3911_STATUSCOM_DRHIZ, + 0, 2); + else + ret = mcp3911_update(adc, MCP3911_REG_STATUSCOM, MCP3911_STATUSCOM_DRHIZ, + MCP3911_STATUSCOM_DRHIZ, 2); + if (ret) + return ret; indio_dev->name = spi_get_device_id(spi)->name; indio_dev->modes = INDIO_DIRECT_MODE; @@ -306,31 +491,38 @@ static int mcp3911_probe(struct spi_device *spi) mutex_init(&adc->lock); - ret = iio_device_register(indio_dev); - if (ret) - goto clk_disable; - - return ret; + if (spi->irq > 0) { + adc->trig = devm_iio_trigger_alloc(&spi->dev, "%s-dev%d", + indio_dev->name, + iio_device_id(indio_dev)); + if (!adc->trig) + return PTR_ERR(adc->trig); -clk_disable: - clk_disable_unprepare(adc->clki); -reg_disable: - if (adc->vref) - regulator_disable(adc->vref); - - return ret; -} + adc->trig->ops = &mcp3911_trigger_ops; + iio_trigger_set_drvdata(adc->trig, adc); + ret = devm_iio_trigger_register(&spi->dev, adc->trig); + if (ret) + return ret; -static void mcp3911_remove(struct spi_device *spi) -{ - struct iio_dev *indio_dev = spi_get_drvdata(spi); - struct mcp3911 *adc = iio_priv(indio_dev); + /* + * The device generates interrupts as long as it is powered up. + * Some platforms might not allow the option to power it down so + * don't enable the interrupt to avoid extra load on the system. + */ + ret = devm_request_irq(&spi->dev, spi->irq, + &iio_trigger_generic_data_rdy_poll, IRQF_NO_AUTOEN | IRQF_ONESHOT, + indio_dev->name, adc->trig); + if (ret) + return ret; + } - iio_device_unregister(indio_dev); + ret = devm_iio_triggered_buffer_setup(&spi->dev, indio_dev, + NULL, + mcp3911_trigger_handler, NULL); + if (ret) + return ret; - clk_disable_unprepare(adc->clki); - if (adc->vref) - regulator_disable(adc->vref); + return devm_iio_device_register(&adc->spi->dev, indio_dev); } static const struct of_device_id mcp3911_dt_ids[] = { @@ -351,7 +543,6 @@ static struct spi_driver mcp3911_driver = { .of_match_table = mcp3911_dt_ids, }, .probe = mcp3911_probe, - .remove = mcp3911_remove, .id_table = mcp3911_id, }; module_spi_driver(mcp3911_driver); diff --git a/drivers/iio/adc/mt6360-adc.c b/drivers/iio/adc/mt6360-adc.c index 35260d9e4e4747243c8876c594a1fe2547fdce50..3710473e526fc877f4418770f5c6a98f127ffb0d 100644 --- a/drivers/iio/adc/mt6360-adc.c +++ b/drivers/iio/adc/mt6360-adc.c @@ -353,7 +353,7 @@ static int mt6360_adc_probe(struct platform_device *pdev) return devm_iio_device_register(&pdev->dev, indio_dev); } -static const struct of_device_id __maybe_unused mt6360_adc_of_id[] = { +static const struct of_device_id mt6360_adc_of_id[] = { { .compatible = "mediatek,mt6360-adc", }, {} }; diff --git a/drivers/iio/adc/qcom-pm8xxx-xoadc.c b/drivers/iio/adc/qcom-pm8xxx-xoadc.c index 5e9e5682107524e561ecaa7d4bbdda9f99ec23a0..eb424496ee1d25cc9c7d1e58fb1d165fce184d8e 100644 --- a/drivers/iio/adc/qcom-pm8xxx-xoadc.c +++ b/drivers/iio/adc/qcom-pm8xxx-xoadc.c @@ -14,9 +14,9 @@ #include #include #include -#include -#include +#include #include +#include #include #include #include @@ -694,8 +694,8 @@ static int pm8xxx_read_raw(struct iio_dev *indio_dev, } } -static int pm8xxx_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int pm8xxx_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { struct pm8xxx_xoadc *adc = iio_priv(indio_dev); u8 pre_scale_mux; @@ -706,10 +706,10 @@ static int pm8xxx_of_xlate(struct iio_dev *indio_dev, * First cell is prescaler or premux, second cell is analog * mux. */ - if (iiospec->args_count != 2) { - dev_err(&indio_dev->dev, "wrong number of arguments for %pOFn need 2 got %d\n", - iiospec->np, - iiospec->args_count); + if (iiospec->nargs != 2) { + dev_err(&indio_dev->dev, "wrong number of arguments for %pfwP need 2 got %d\n", + iiospec->fwnode, + iiospec->nargs); return -EINVAL; } pre_scale_mux = (u8)iiospec->args[0]; @@ -727,34 +727,34 @@ static int pm8xxx_of_xlate(struct iio_dev *indio_dev, } static const struct iio_info pm8xxx_xoadc_info = { - .of_xlate = pm8xxx_of_xlate, + .fwnode_xlate = pm8xxx_fwnode_xlate, .read_raw = pm8xxx_read_raw, }; static int pm8xxx_xoadc_parse_channel(struct device *dev, - struct device_node *np, + struct fwnode_handle *fwnode, const struct xoadc_channel *hw_channels, struct iio_chan_spec *iio_chan, struct pm8xxx_chan_info *ch) { - const char *name = np->name; + const char *name = fwnode_get_name(fwnode); const struct xoadc_channel *hwchan; - u32 pre_scale_mux, amux_channel; + u32 pre_scale_mux, amux_channel, reg[2]; u32 rsv, dec; int ret; int chid; - ret = of_property_read_u32_index(np, "reg", 0, &pre_scale_mux); + ret = fwnode_property_read_u32_array(fwnode, "reg", reg, + ARRAY_SIZE(reg)); if (ret) { - dev_err(dev, "invalid pre scale/mux number %s\n", name); - return ret; - } - ret = of_property_read_u32_index(np, "reg", 1, &amux_channel); - if (ret) { - dev_err(dev, "invalid amux channel number %s\n", name); + dev_err(dev, "invalid pre scale/mux or amux channel number %s\n", + name); return ret; } + pre_scale_mux = reg[0]; + amux_channel = reg[1]; + /* Find the right channel setting */ chid = 0; hwchan = &hw_channels[0]; @@ -778,7 +778,7 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev, /* Everyone seems to use default ("type 2") decimation */ ch->decimation = VADC_DEF_DECIMATION; - if (!of_property_read_u32(np, "qcom,ratiometric", &rsv)) { + if (!fwnode_property_read_u32(fwnode, "qcom,ratiometric", &rsv)) { ch->calibration = VADC_CALIB_RATIOMETRIC; if (rsv > XOADC_RSV_MAX) { dev_err(dev, "%s too large RSV value %d\n", name, rsv); @@ -791,7 +791,7 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev, } /* Optional decimation, if omitted we use the default */ - ret = of_property_read_u32(np, "qcom,decimation", &dec); + ret = fwnode_property_read_u32(fwnode, "qcom,decimation", &dec); if (!ret) { ret = qcom_vadc_decimation_from_dt(dec); if (ret < 0) { @@ -820,15 +820,14 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev, return 0; } -static int pm8xxx_xoadc_parse_channels(struct pm8xxx_xoadc *adc, - struct device_node *np) +static int pm8xxx_xoadc_parse_channels(struct pm8xxx_xoadc *adc) { - struct device_node *child; + struct fwnode_handle *child; struct pm8xxx_chan_info *ch; int ret; int i; - adc->nchans = of_get_available_child_count(np); + adc->nchans = device_get_child_node_count(adc->dev); if (!adc->nchans) { dev_err(adc->dev, "no channel children\n"); return -ENODEV; @@ -846,14 +845,14 @@ static int pm8xxx_xoadc_parse_channels(struct pm8xxx_xoadc *adc, return -ENOMEM; i = 0; - for_each_available_child_of_node(np, child) { + device_for_each_child_node(adc->dev, child) { ch = &adc->chans[i]; ret = pm8xxx_xoadc_parse_channel(adc->dev, child, adc->variant->channels, &adc->iio_chans[i], ch); if (ret) { - of_node_put(child); + fwnode_handle_put(child); return ret; } i++; @@ -884,12 +883,11 @@ static int pm8xxx_xoadc_probe(struct platform_device *pdev) const struct xoadc_variant *variant; struct pm8xxx_xoadc *adc; struct iio_dev *indio_dev; - struct device_node *np = pdev->dev.of_node; struct regmap *map; struct device *dev = &pdev->dev; int ret; - variant = of_device_get_match_data(dev); + variant = device_get_match_data(dev); if (!variant) return -ENODEV; @@ -904,7 +902,7 @@ static int pm8xxx_xoadc_probe(struct platform_device *pdev) init_completion(&adc->complete); mutex_init(&adc->lock); - ret = pm8xxx_xoadc_parse_channels(adc, np); + ret = pm8xxx_xoadc_parse_channels(adc); if (ret) return ret; diff --git a/drivers/iio/adc/qcom-spmi-adc5.c b/drivers/iio/adc/qcom-spmi-adc5.c index 87438d1e5c0bb6cc67f3c3be0c18f27c91621ce8..821fee60a765170bfacd189c39fe2d97adc4cbb1 100644 --- a/drivers/iio/adc/qcom-spmi-adc5.c +++ b/drivers/iio/adc/qcom-spmi-adc5.c @@ -14,9 +14,9 @@ #include #include #include -#include -#include +#include #include +#include #include #include @@ -403,8 +403,8 @@ static irqreturn_t adc5_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static int adc5_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int adc5_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { struct adc5_chip *adc = iio_priv(indio_dev); int i; @@ -416,8 +416,8 @@ static int adc5_of_xlate(struct iio_dev *indio_dev, return -EINVAL; } -static int adc7_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int adc7_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { struct adc5_chip *adc = iio_priv(indio_dev); int i, v_channel; @@ -481,12 +481,12 @@ static int adc7_read_raw(struct iio_dev *indio_dev, static const struct iio_info adc5_info = { .read_raw = adc5_read_raw, - .of_xlate = adc5_of_xlate, + .fwnode_xlate = adc5_fwnode_xlate, }; static const struct iio_info adc7_info = { .read_raw = adc7_read_raw, - .of_xlate = adc7_of_xlate, + .fwnode_xlate = adc7_fwnode_xlate, }; struct adc5_channels { @@ -526,6 +526,8 @@ static const struct adc5_channels adc5_chans_pmic[ADC5_MAX_CHANNEL] = { SCALE_HW_CALIB_DEFAULT) [ADC5_VBAT_SNS] = ADC5_CHAN_VOLT("vbat_sns", 1, SCALE_HW_CALIB_DEFAULT) + [ADC5_VCOIN] = ADC5_CHAN_VOLT("vcoin", 1, + SCALE_HW_CALIB_DEFAULT) [ADC5_DIE_TEMP] = ADC5_CHAN_TEMP("die_temp", 0, SCALE_HW_CALIB_PMIC_THERM) [ADC5_USB_IN_I] = ADC5_CHAN_VOLT("usb_in_i_uv", 0, @@ -549,6 +551,12 @@ static const struct adc5_channels adc5_chans_pmic[ADC5_MAX_CHANNEL] = { SCALE_HW_CALIB_THERM_100K_PULLUP) [ADC5_AMUX_THM2] = ADC5_CHAN_TEMP("amux_thm2", 0, SCALE_HW_CALIB_PM5_SMB_TEMP) + [ADC5_GPIO1_100K_PU] = ADC5_CHAN_TEMP("gpio1_100k_pu", 0, + SCALE_HW_CALIB_THERM_100K_PULLUP) + [ADC5_GPIO3_100K_PU] = ADC5_CHAN_TEMP("gpio3_100k_pu", 0, + SCALE_HW_CALIB_THERM_100K_PULLUP) + [ADC5_GPIO4_100K_PU] = ADC5_CHAN_TEMP("gpio4_100k_pu", 0, + SCALE_HW_CALIB_THERM_100K_PULLUP) }; static const struct adc5_channels adc7_chans_pmic[ADC5_MAX_CHANNEL] = { @@ -589,6 +597,8 @@ static const struct adc5_channels adc5_chans_rev2[ADC5_MAX_CHANNEL] = { SCALE_HW_CALIB_DEFAULT) [ADC5_1P25VREF] = ADC5_CHAN_VOLT("vref_1p25", 0, SCALE_HW_CALIB_DEFAULT) + [ADC5_VREF_VADC] = ADC5_CHAN_VOLT("vref_vadc", 0, + SCALE_HW_CALIB_DEFAULT) [ADC5_VPH_PWR] = ADC5_CHAN_VOLT("vph_pwr", 1, SCALE_HW_CALIB_DEFAULT) [ADC5_VBAT_SNS] = ADC5_CHAN_VOLT("vbat_sns", 1, @@ -611,18 +621,18 @@ static const struct adc5_channels adc5_chans_rev2[ADC5_MAX_CHANNEL] = { SCALE_HW_CALIB_THERM_100K_PULLUP) }; -static int adc5_get_dt_channel_data(struct adc5_chip *adc, +static int adc5_get_fw_channel_data(struct adc5_chip *adc, struct adc5_channel_prop *prop, - struct device_node *node, + struct fwnode_handle *fwnode, const struct adc5_data *data) { - const char *name = node->name, *channel_name; + const char *name = fwnode_get_name(fwnode), *channel_name; u32 chan, value, varr[2]; u32 sid = 0; int ret; struct device *dev = adc->dev; - ret = of_property_read_u32(node, "reg", &chan); + ret = fwnode_property_read_u32(fwnode, "reg", &chan); if (ret) { dev_err(dev, "invalid channel number %s\n", name); return ret; @@ -647,15 +657,13 @@ static int adc5_get_dt_channel_data(struct adc5_chip *adc, prop->channel = chan; prop->sid = sid; - channel_name = of_get_property(node, - "label", NULL) ? : node->name; - if (!channel_name) { - dev_err(dev, "Invalid channel name\n"); - return -EINVAL; - } + ret = fwnode_property_read_string(fwnode, "label", &channel_name); + if (ret) + channel_name = name; + prop->datasheet_name = channel_name; - ret = of_property_read_u32(node, "qcom,decimation", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,decimation", &value); if (!ret) { ret = qcom_adc5_decimation_from_dt(value, data->decimation); if (ret < 0) { @@ -668,7 +676,7 @@ static int adc5_get_dt_channel_data(struct adc5_chip *adc, prop->decimation = ADC5_DECIMATION_DEFAULT; } - ret = of_property_read_u32_array(node, "qcom,pre-scaling", varr, 2); + ret = fwnode_property_read_u32_array(fwnode, "qcom,pre-scaling", varr, 2); if (!ret) { ret = qcom_adc5_prescaling_from_dt(varr[0], varr[1]); if (ret < 0) { @@ -682,7 +690,7 @@ static int adc5_get_dt_channel_data(struct adc5_chip *adc, adc->data->adc_chans[prop->channel].prescale_index; } - ret = of_property_read_u32(node, "qcom,hw-settle-time", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,hw-settle-time", &value); if (!ret) { u8 dig_version[2]; @@ -713,7 +721,7 @@ static int adc5_get_dt_channel_data(struct adc5_chip *adc, prop->hw_settle_time = VADC_DEF_HW_SETTLE_TIME; } - ret = of_property_read_u32(node, "qcom,avg-samples", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,avg-samples", &value); if (!ret) { ret = qcom_adc5_avg_samples_from_dt(value); if (ret < 0) { @@ -726,7 +734,7 @@ static int adc5_get_dt_channel_data(struct adc5_chip *adc, prop->avg_samples = VADC_DEF_AVG_SAMPLES; } - if (of_property_read_bool(node, "qcom,ratiometric")) + if (fwnode_property_read_bool(fwnode, "qcom,ratiometric")) prop->cal_method = ADC5_RATIOMETRIC_CAL; else prop->cal_method = ADC5_ABSOLUTE_CAL; @@ -801,16 +809,16 @@ static const struct of_device_id adc5_match_table[] = { }; MODULE_DEVICE_TABLE(of, adc5_match_table); -static int adc5_get_dt_data(struct adc5_chip *adc, struct device_node *node) +static int adc5_get_fw_data(struct adc5_chip *adc) { const struct adc5_channels *adc_chan; struct iio_chan_spec *iio_chan; struct adc5_channel_prop prop, *chan_props; - struct device_node *child; + struct fwnode_handle *child; unsigned int index = 0; int ret; - adc->nchannels = of_get_available_child_count(node); + adc->nchannels = device_get_child_node_count(adc->dev); if (!adc->nchannels) return -EINVAL; @@ -826,14 +834,14 @@ static int adc5_get_dt_data(struct adc5_chip *adc, struct device_node *node) chan_props = adc->chan_props; iio_chan = adc->iio_chans; - adc->data = of_device_get_match_data(adc->dev); + adc->data = device_get_match_data(adc->dev); if (!adc->data) adc->data = &adc5_data_pmic; - for_each_available_child_of_node(node, child) { - ret = adc5_get_dt_channel_data(adc, &prop, child, adc->data); + device_for_each_child_node(adc->dev, child) { + ret = adc5_get_fw_channel_data(adc, &prop, child, adc->data); if (ret) { - of_node_put(child); + fwnode_handle_put(child); return ret; } @@ -858,7 +866,6 @@ static int adc5_get_dt_data(struct adc5_chip *adc, struct device_node *node) static int adc5_probe(struct platform_device *pdev) { - struct device_node *node = pdev->dev.of_node; struct device *dev = &pdev->dev; struct iio_dev *indio_dev; struct adc5_chip *adc; @@ -870,7 +877,7 @@ static int adc5_probe(struct platform_device *pdev) if (!regmap) return -ENODEV; - ret = of_property_read_u32(node, "reg", ®); + ret = device_property_read_u32(dev, "reg", ®); if (ret < 0) return ret; @@ -886,7 +893,7 @@ static int adc5_probe(struct platform_device *pdev) init_completion(&adc->complete); mutex_init(&adc->lock); - ret = adc5_get_dt_data(adc, node); + ret = adc5_get_fw_data(adc); if (ret) { dev_err(dev, "adc get dt data failed\n"); return ret; diff --git a/drivers/iio/adc/qcom-spmi-vadc.c b/drivers/iio/adc/qcom-spmi-vadc.c index 34202ba52469928c9fa13b13dd6cdace7fc6a8cb..bcff0f62b70e05b388dd4b097f55366bbe7092dd 100644 --- a/drivers/iio/adc/qcom-spmi-vadc.c +++ b/drivers/iio/adc/qcom-spmi-vadc.c @@ -13,8 +13,9 @@ #include #include #include -#include +#include #include +#include #include #include #include @@ -481,8 +482,8 @@ static int vadc_read_raw(struct iio_dev *indio_dev, return ret; } -static int vadc_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int vadc_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { struct vadc_priv *vadc = iio_priv(indio_dev); unsigned int i; @@ -496,7 +497,7 @@ static int vadc_of_xlate(struct iio_dev *indio_dev, static const struct iio_info vadc_info = { .read_raw = vadc_read_raw, - .of_xlate = vadc_of_xlate, + .fwnode_xlate = vadc_fwnode_xlate, }; struct vadc_channels { @@ -647,15 +648,15 @@ static const struct vadc_channels vadc_chans[] = { VADC_CHAN_NO_SCALE(LR_MUX3_BUF_PU1_PU2_XO_THERM, 0) }; -static int vadc_get_dt_channel_data(struct device *dev, +static int vadc_get_fw_channel_data(struct device *dev, struct vadc_channel_prop *prop, - struct device_node *node) + struct fwnode_handle *fwnode) { - const char *name = node->name; + const char *name = fwnode_get_name(fwnode); u32 chan, value, varr[2]; int ret; - ret = of_property_read_u32(node, "reg", &chan); + ret = fwnode_property_read_u32(fwnode, "reg", &chan); if (ret) { dev_err(dev, "invalid channel number %s\n", name); return ret; @@ -669,7 +670,7 @@ static int vadc_get_dt_channel_data(struct device *dev, /* the channel has DT description */ prop->channel = chan; - ret = of_property_read_u32(node, "qcom,decimation", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,decimation", &value); if (!ret) { ret = qcom_vadc_decimation_from_dt(value); if (ret < 0) { @@ -682,7 +683,7 @@ static int vadc_get_dt_channel_data(struct device *dev, prop->decimation = VADC_DEF_DECIMATION; } - ret = of_property_read_u32_array(node, "qcom,pre-scaling", varr, 2); + ret = fwnode_property_read_u32_array(fwnode, "qcom,pre-scaling", varr, 2); if (!ret) { ret = vadc_prescaling_from_dt(varr[0], varr[1]); if (ret < 0) { @@ -695,7 +696,7 @@ static int vadc_get_dt_channel_data(struct device *dev, prop->prescale = vadc_chans[prop->channel].prescale_index; } - ret = of_property_read_u32(node, "qcom,hw-settle-time", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,hw-settle-time", &value); if (!ret) { ret = vadc_hw_settle_time_from_dt(value); if (ret < 0) { @@ -708,7 +709,7 @@ static int vadc_get_dt_channel_data(struct device *dev, prop->hw_settle_time = VADC_DEF_HW_SETTLE_TIME; } - ret = of_property_read_u32(node, "qcom,avg-samples", &value); + ret = fwnode_property_read_u32(fwnode, "qcom,avg-samples", &value); if (!ret) { ret = vadc_avg_samples_from_dt(value); if (ret < 0) { @@ -721,7 +722,7 @@ static int vadc_get_dt_channel_data(struct device *dev, prop->avg_samples = VADC_DEF_AVG_SAMPLES; } - if (of_property_read_bool(node, "qcom,ratiometric")) + if (fwnode_property_read_bool(fwnode, "qcom,ratiometric")) prop->calibration = VADC_CALIB_RATIOMETRIC; else prop->calibration = VADC_CALIB_ABSOLUTE; @@ -731,16 +732,16 @@ static int vadc_get_dt_channel_data(struct device *dev, return 0; } -static int vadc_get_dt_data(struct vadc_priv *vadc, struct device_node *node) +static int vadc_get_fw_data(struct vadc_priv *vadc) { const struct vadc_channels *vadc_chan; struct iio_chan_spec *iio_chan; struct vadc_channel_prop prop; - struct device_node *child; + struct fwnode_handle *child; unsigned int index = 0; int ret; - vadc->nchannels = of_get_available_child_count(node); + vadc->nchannels = device_get_child_node_count(vadc->dev); if (!vadc->nchannels) return -EINVAL; @@ -756,10 +757,10 @@ static int vadc_get_dt_data(struct vadc_priv *vadc, struct device_node *node) iio_chan = vadc->iio_chans; - for_each_available_child_of_node(node, child) { - ret = vadc_get_dt_channel_data(vadc->dev, &prop, child); + device_for_each_child_node(vadc->dev, child) { + ret = vadc_get_fw_channel_data(vadc->dev, &prop, child); if (ret) { - of_node_put(child); + fwnode_handle_put(child); return ret; } @@ -848,7 +849,6 @@ static int vadc_check_revision(struct vadc_priv *vadc) static int vadc_probe(struct platform_device *pdev) { - struct device_node *node = pdev->dev.of_node; struct device *dev = &pdev->dev; struct iio_dev *indio_dev; struct vadc_priv *vadc; @@ -860,7 +860,7 @@ static int vadc_probe(struct platform_device *pdev) if (!regmap) return -ENODEV; - ret = of_property_read_u32(node, "reg", ®); + ret = device_property_read_u32(dev, "reg", ®); if (ret < 0) return ret; @@ -880,7 +880,7 @@ static int vadc_probe(struct platform_device *pdev) if (ret) return ret; - ret = vadc_get_dt_data(vadc, node); + ret = vadc_get_fw_data(vadc); if (ret) return ret; diff --git a/drivers/iio/adc/rtq6056.c b/drivers/iio/adc/rtq6056.c new file mode 100644 index 0000000000000000000000000000000000000000..c1b2e8dc9a26687c20ca565abe6a7e6600a1a5f3 --- /dev/null +++ b/drivers/iio/adc/rtq6056.c @@ -0,0 +1,661 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022 Richtek Technology Corp. + * + * ChiYuan Huang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define RTQ6056_REG_CONFIG 0x00 +#define RTQ6056_REG_SHUNTVOLT 0x01 +#define RTQ6056_REG_BUSVOLT 0x02 +#define RTQ6056_REG_POWER 0x03 +#define RTQ6056_REG_CURRENT 0x04 +#define RTQ6056_REG_CALIBRATION 0x05 +#define RTQ6056_REG_MASKENABLE 0x06 +#define RTQ6056_REG_ALERTLIMIT 0x07 +#define RTQ6056_REG_MANUFACTID 0xFE +#define RTQ6056_REG_DIEID 0xFF + +#define RTQ6056_VENDOR_ID 0x1214 +#define RTQ6056_DEFAULT_CONFIG 0x4127 +#define RTQ6056_CONT_ALLON 7 + +enum { + RTQ6056_CH_VSHUNT = 0, + RTQ6056_CH_VBUS, + RTQ6056_CH_POWER, + RTQ6056_CH_CURRENT, + RTQ6056_MAX_CHANNEL +}; + +enum { + F_OPMODE = 0, + F_VSHUNTCT, + F_VBUSCT, + F_AVG, + F_RESET, + F_MAX_FIELDS +}; + +struct rtq6056_priv { + struct device *dev; + struct regmap *regmap; + struct regmap_field *rm_fields[F_MAX_FIELDS]; + u32 shunt_resistor_uohm; + int vshuntct_us; + int vbusct_us; + int avg_sample; +}; + +static const struct reg_field rtq6056_reg_fields[F_MAX_FIELDS] = { + [F_OPMODE] = REG_FIELD(RTQ6056_REG_CONFIG, 0, 2), + [F_VSHUNTCT] = REG_FIELD(RTQ6056_REG_CONFIG, 3, 5), + [F_VBUSCT] = REG_FIELD(RTQ6056_REG_CONFIG, 6, 8), + [F_AVG] = REG_FIELD(RTQ6056_REG_CONFIG, 9, 11), + [F_RESET] = REG_FIELD(RTQ6056_REG_CONFIG, 15, 15), +}; + +static const struct iio_chan_spec rtq6056_channels[RTQ6056_MAX_CHANNEL + 1] = { + { + .type = IIO_VOLTAGE, + .indexed = 1, + .channel = 0, + .address = RTQ6056_REG_SHUNTVOLT, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .scan_index = 0, + .scan_type = { + .sign = 's', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_CPU, + }, + }, + { + .type = IIO_VOLTAGE, + .indexed = 1, + .channel = 1, + .address = RTQ6056_REG_BUSVOLT, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_separate_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .scan_index = 1, + .scan_type = { + .sign = 'u', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_CPU, + }, + }, + { + .type = IIO_POWER, + .indexed = 1, + .channel = 2, + .address = RTQ6056_REG_POWER, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE) | + BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .scan_index = 2, + .scan_type = { + .sign = 'u', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_CPU, + }, + }, + { + .type = IIO_CURRENT, + .indexed = 1, + .channel = 3, + .address = RTQ6056_REG_CURRENT, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all_available = BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .scan_index = 3, + .scan_type = { + .sign = 's', + .realbits = 16, + .storagebits = 16, + .endianness = IIO_CPU, + }, + }, + IIO_CHAN_SOFT_TIMESTAMP(RTQ6056_MAX_CHANNEL), +}; + +static int rtq6056_adc_read_channel(struct rtq6056_priv *priv, + struct iio_chan_spec const *ch, + int *val) +{ + struct device *dev = priv->dev; + unsigned int addr = ch->address; + unsigned int regval; + int ret; + + pm_runtime_get_sync(dev); + ret = regmap_read(priv->regmap, addr, ®val); + pm_runtime_mark_last_busy(dev); + pm_runtime_put(dev); + if (ret) + return ret; + + /* Power and VBUS is unsigned 16-bit, others are signed 16-bit */ + if (addr == RTQ6056_REG_BUSVOLT || addr == RTQ6056_REG_POWER) + *val = regval; + else + *val = sign_extend32(regval, 16); + + return IIO_VAL_INT; +} + +static int rtq6056_adc_read_scale(struct iio_chan_spec const *ch, int *val, + int *val2) +{ + switch (ch->address) { + case RTQ6056_REG_SHUNTVOLT: + /* VSHUNT lsb 2.5uV */ + *val = 2500; + *val2 = 1000000; + return IIO_VAL_FRACTIONAL; + case RTQ6056_REG_BUSVOLT: + /* VBUS lsb 1.25mV */ + *val = 1250; + *val2 = 1000; + return IIO_VAL_FRACTIONAL; + case RTQ6056_REG_POWER: + /* Power lsb 25mW */ + *val = 25; + return IIO_VAL_INT; + default: + return -EINVAL; + } +} + +/* + * Sample frequency for channel VSHUNT and VBUS. The indices correspond + * with the bit value expected by the chip. And it can be found at + * https://www.richtek.com/assets/product_file/RTQ6056/DSQ6056-00.pdf + */ +static const int rtq6056_samp_freq_list[] = { + 7194, 4926, 3717, 1904, 964, 485, 243, 122, +}; + +static int rtq6056_adc_set_samp_freq(struct rtq6056_priv *priv, + struct iio_chan_spec const *ch, int val) +{ + struct regmap_field *rm_field; + unsigned int selector; + int *ct, ret; + + if (val > 7194 || val < 122) + return -EINVAL; + + if (ch->address == RTQ6056_REG_SHUNTVOLT) { + rm_field = priv->rm_fields[F_VSHUNTCT]; + ct = &priv->vshuntct_us; + } else if (ch->address == RTQ6056_REG_BUSVOLT) { + rm_field = priv->rm_fields[F_VBUSCT]; + ct = &priv->vbusct_us; + } else + return -EINVAL; + + selector = find_closest_descending(val, rtq6056_samp_freq_list, + ARRAY_SIZE(rtq6056_samp_freq_list)); + + ret = regmap_field_write(rm_field, selector); + if (ret) + return ret; + + *ct = 1000000 / rtq6056_samp_freq_list[selector]; + + return 0; +} + +/* + * Available averaging rate for rtq6056. The indices correspond with the bit + * value expected by the chip. And it can be found at + * https://www.richtek.com/assets/product_file/RTQ6056/DSQ6056-00.pdf + */ +static const int rtq6056_avg_sample_list[] = { + 1, 4, 16, 64, 128, 256, 512, 1024, +}; + +static int rtq6056_adc_set_average(struct rtq6056_priv *priv, int val) +{ + unsigned int selector; + int ret; + + if (val > 1024 || val < 1) + return -EINVAL; + + selector = find_closest(val, rtq6056_avg_sample_list, + ARRAY_SIZE(rtq6056_avg_sample_list)); + + ret = regmap_field_write(priv->rm_fields[F_AVG], selector); + if (ret) + return ret; + + priv->avg_sample = rtq6056_avg_sample_list[selector]; + + return 0; +} + +static int rtq6056_adc_get_sample_freq(struct rtq6056_priv *priv, + struct iio_chan_spec const *ch, int *val) +{ + int sample_time; + + if (ch->address == RTQ6056_REG_SHUNTVOLT) + sample_time = priv->vshuntct_us; + else if (ch->address == RTQ6056_REG_BUSVOLT) + sample_time = priv->vbusct_us; + else { + sample_time = priv->vshuntct_us + priv->vbusct_us; + sample_time *= priv->avg_sample; + } + + *val = 1000000 / sample_time; + + return IIO_VAL_INT; +} + +static int rtq6056_adc_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int *val, + int *val2, long mask) +{ + struct rtq6056_priv *priv = iio_priv(indio_dev); + + switch (mask) { + case IIO_CHAN_INFO_RAW: + return rtq6056_adc_read_channel(priv, chan, val); + case IIO_CHAN_INFO_SCALE: + return rtq6056_adc_read_scale(chan, val, val2); + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + *val = priv->avg_sample; + return IIO_VAL_INT; + case IIO_CHAN_INFO_SAMP_FREQ: + return rtq6056_adc_get_sample_freq(priv, chan, val); + default: + return -EINVAL; + } +} + +static int rtq6056_adc_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + *vals = rtq6056_samp_freq_list; + *type = IIO_VAL_INT; + *length = ARRAY_SIZE(rtq6056_samp_freq_list); + return IIO_AVAIL_LIST; + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + *vals = rtq6056_avg_sample_list; + *type = IIO_VAL_INT; + *length = ARRAY_SIZE(rtq6056_avg_sample_list); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + +static int rtq6056_adc_write_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int val, + int val2, long mask) +{ + struct rtq6056_priv *priv = iio_priv(indio_dev); + int ret; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + ret = rtq6056_adc_set_samp_freq(priv, chan, val); + break; + case IIO_CHAN_INFO_OVERSAMPLING_RATIO: + ret = rtq6056_adc_set_average(priv, val); + break; + default: + ret = -EINVAL; + break; + } + + iio_device_release_direct_mode(indio_dev); + + return ret; +} + +static const char *rtq6056_channel_labels[RTQ6056_MAX_CHANNEL] = { + [RTQ6056_CH_VSHUNT] = "Vshunt", + [RTQ6056_CH_VBUS] = "Vbus", + [RTQ6056_CH_POWER] = "Power", + [RTQ6056_CH_CURRENT] = "Current", +}; + +static int rtq6056_adc_read_label(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + char *label) +{ + return sysfs_emit(label, "%s\n", rtq6056_channel_labels[chan->channel]); +} + +static int rtq6056_set_shunt_resistor(struct rtq6056_priv *priv, + int resistor_uohm) +{ + unsigned int calib_val; + int ret; + + if (resistor_uohm <= 0) { + dev_err(priv->dev, "Invalid resistor [%d]\n", resistor_uohm); + return -EINVAL; + } + + /* calibration = 5120000 / (Rshunt (uOhm) * current lsb (1mA)) */ + calib_val = 5120000 / resistor_uohm; + ret = regmap_write(priv->regmap, RTQ6056_REG_CALIBRATION, calib_val); + if (ret) + return ret; + + priv->shunt_resistor_uohm = resistor_uohm; + + return 0; +} + +static ssize_t shunt_resistor_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct rtq6056_priv *priv = iio_priv(dev_to_iio_dev(dev)); + int vals[2] = { priv->shunt_resistor_uohm, 1000000 }; + + return iio_format_value(buf, IIO_VAL_FRACTIONAL, 1, vals); +} + +static ssize_t shunt_resistor_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct rtq6056_priv *priv = iio_priv(indio_dev); + int val, val_fract, ret; + + ret = iio_device_claim_direct_mode(indio_dev); + if (ret) + return ret; + + ret = iio_str_to_fixpoint(buf, 100000, &val, &val_fract); + if (ret) + goto out_store; + + ret = rtq6056_set_shunt_resistor(priv, val * 1000000 + val_fract); + +out_store: + iio_device_release_direct_mode(indio_dev); + + return ret ?: len; +} + +static IIO_DEVICE_ATTR_RW(shunt_resistor, 0); + +static struct attribute *rtq6056_attributes[] = { + &iio_dev_attr_shunt_resistor.dev_attr.attr, + NULL +}; + +static const struct attribute_group rtq6056_attribute_group = { + .attrs = rtq6056_attributes, +}; + +static const struct iio_info rtq6056_info = { + .attrs = &rtq6056_attribute_group, + .read_raw = rtq6056_adc_read_raw, + .read_avail = rtq6056_adc_read_avail, + .write_raw = rtq6056_adc_write_raw, + .read_label = rtq6056_adc_read_label, +}; + +static irqreturn_t rtq6056_buffer_trigger_handler(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct iio_dev *indio_dev = pf->indio_dev; + struct rtq6056_priv *priv = iio_priv(indio_dev); + struct device *dev = priv->dev; + struct { + u16 vals[RTQ6056_MAX_CHANNEL]; + s64 timestamp __aligned(8); + } data; + unsigned int raw; + int i = 0, bit, ret; + + memset(&data, 0, sizeof(data)); + + pm_runtime_get_sync(dev); + + for_each_set_bit(bit, indio_dev->active_scan_mask, indio_dev->masklength) { + unsigned int addr = rtq6056_channels[bit].address; + + ret = regmap_read(priv->regmap, addr, &raw); + if (ret) + goto out; + + data.vals[i++] = raw; + } + + iio_push_to_buffers_with_timestamp(indio_dev, &data, iio_get_time_ns(indio_dev)); + +out: + pm_runtime_mark_last_busy(dev); + pm_runtime_put(dev); + + iio_trigger_notify_done(indio_dev->trig); + + return IRQ_HANDLED; +} + +static void rtq6056_enter_shutdown_state(void *dev) +{ + struct rtq6056_priv *priv = dev_get_drvdata(dev); + + /* Enter shutdown state */ + regmap_field_write(priv->rm_fields[F_OPMODE], 0); +} + +static bool rtq6056_is_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case RTQ6056_REG_CONFIG ... RTQ6056_REG_ALERTLIMIT: + case RTQ6056_REG_MANUFACTID ... RTQ6056_REG_DIEID: + return true; + default: + return false; + } +} + +static bool rtq6056_is_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case RTQ6056_REG_CONFIG: + case RTQ6056_REG_CALIBRATION ... RTQ6056_REG_ALERTLIMIT: + return true; + default: + return false; + } +} + +static const struct regmap_config rtq6056_regmap_config = { + .reg_bits = 8, + .val_bits = 16, + .val_format_endian = REGMAP_ENDIAN_BIG, + .max_register = RTQ6056_REG_DIEID, + .readable_reg = rtq6056_is_readable_reg, + .writeable_reg = rtq6056_is_writeable_reg, +}; + +static int rtq6056_probe(struct i2c_client *i2c) +{ + struct iio_dev *indio_dev; + struct rtq6056_priv *priv; + struct device *dev = &i2c->dev; + struct regmap *regmap; + unsigned int vendor_id, shunt_resistor_uohm; + int ret; + + if (!i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_WORD_DATA)) + return -EOPNOTSUPP; + + indio_dev = devm_iio_device_alloc(dev, sizeof(*priv)); + if (!indio_dev) + return -ENOMEM; + + priv = iio_priv(indio_dev); + priv->dev = dev; + priv->vshuntct_us = priv->vbusct_us = 1037; + priv->avg_sample = 1; + i2c_set_clientdata(i2c, priv); + + regmap = devm_regmap_init_i2c(i2c, &rtq6056_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), + "Failed to init regmap\n"); + + priv->regmap = regmap; + + ret = regmap_read(regmap, RTQ6056_REG_MANUFACTID, &vendor_id); + if (ret) + return dev_err_probe(dev, ret, + "Failed to get manufacturer info\n"); + + if (vendor_id != RTQ6056_VENDOR_ID) + return dev_err_probe(dev, -ENODEV, + "Invalid vendor id 0x%04x\n", vendor_id); + + ret = devm_regmap_field_bulk_alloc(dev, regmap, priv->rm_fields, + rtq6056_reg_fields, F_MAX_FIELDS); + if (ret) + return dev_err_probe(dev, ret, "Failed to init regmap field\n"); + + /* + * By default, configure average sample as 1, bus and shunt conversion + * time as 1037 microsecond, and operating mode to all on. + */ + ret = regmap_write(regmap, RTQ6056_REG_CONFIG, RTQ6056_DEFAULT_CONFIG); + if (ret) + return dev_err_probe(dev, ret, + "Failed to enable continuous sensing\n"); + + ret = devm_add_action_or_reset(dev, rtq6056_enter_shutdown_state, dev); + if (ret) + return ret; + + pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC); + pm_runtime_use_autosuspend(dev); + pm_runtime_set_active(dev); + pm_runtime_mark_last_busy(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable pm_runtime\n"); + + /* By default, use 2000 micro-Ohm resistor */ + shunt_resistor_uohm = 2000; + device_property_read_u32(dev, "shunt-resistor-micro-ohms", + &shunt_resistor_uohm); + + ret = rtq6056_set_shunt_resistor(priv, shunt_resistor_uohm); + if (ret) + return dev_err_probe(dev, ret, + "Failed to init shunt resistor\n"); + + indio_dev->name = "rtq6056"; + indio_dev->modes = INDIO_DIRECT_MODE; + indio_dev->channels = rtq6056_channels; + indio_dev->num_channels = ARRAY_SIZE(rtq6056_channels); + indio_dev->info = &rtq6056_info; + + ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL, + rtq6056_buffer_trigger_handler, + NULL); + if (ret) + return dev_err_probe(dev, ret, + "Failed to allocate iio trigger buffer\n"); + + return devm_iio_device_register(dev, indio_dev); +} + +static int rtq6056_runtime_suspend(struct device *dev) +{ + struct rtq6056_priv *priv = dev_get_drvdata(dev); + + /* Configure to shutdown mode */ + return regmap_field_write(priv->rm_fields[F_OPMODE], 0); +} + +static int rtq6056_runtime_resume(struct device *dev) +{ + struct rtq6056_priv *priv = dev_get_drvdata(dev); + int sample_rdy_time_us, ret; + + ret = regmap_field_write(priv->rm_fields[F_OPMODE], RTQ6056_CONT_ALLON); + if (ret) + return ret; + + sample_rdy_time_us = priv->vbusct_us + priv->vshuntct_us; + sample_rdy_time_us *= priv->avg_sample; + + usleep_range(sample_rdy_time_us, sample_rdy_time_us + 100); + + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(rtq6056_pm_ops, rtq6056_runtime_suspend, + rtq6056_runtime_resume, NULL); + +static const struct of_device_id rtq6056_device_match[] = { + { .compatible = "richtek,rtq6056" }, + {} +}; +MODULE_DEVICE_TABLE(of, rtq6056_device_match); + +static struct i2c_driver rtq6056_driver = { + .driver = { + .name = "rtq6056", + .of_match_table = rtq6056_device_match, + .pm = pm_ptr(&rtq6056_pm_ops), + }, + .probe_new = rtq6056_probe, +}; +module_i2c_driver(rtq6056_driver); + +MODULE_AUTHOR("ChiYuan Huang "); +MODULE_DESCRIPTION("Richtek RTQ6056 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iio/adc/stm32-adc-core.c b/drivers/iio/adc/stm32-adc-core.c index 1ce52af3fe8bd25dbc772a79e142e99783bf8ff9..81d5db91c67bf7896be5d3884d1a61bb789c214d 100644 --- a/drivers/iio/adc/stm32-adc-core.c +++ b/drivers/iio/adc/stm32-adc-core.c @@ -9,6 +9,7 @@ * */ +#include #include #include #include @@ -62,6 +63,7 @@ struct stm32_adc_priv; * @regs: common registers for all instances * @clk_sel: clock selection routine * @max_clk_rate_hz: maximum analog clock rate (Hz, from datasheet) + * @ipid: adc identification number * @has_syscfg: SYSCFG capability flags * @num_irqs: number of interrupt lines * @num_adcs: maximum number of ADC instances in the common registers @@ -70,6 +72,7 @@ struct stm32_adc_priv_cfg { const struct stm32_adc_common_regs *regs; int (*clk_sel)(struct platform_device *, struct stm32_adc_priv *); u32 max_clk_rate_hz; + u32 ipid; unsigned int has_syscfg; unsigned int num_irqs; unsigned int num_adcs; @@ -78,6 +81,7 @@ struct stm32_adc_priv_cfg { /** * struct stm32_adc_priv - stm32 ADC core private data * @irq: irq(s) for ADC block + * @nb_adc_max: actual maximum number of instance per ADC block * @domain: irq domain reference * @aclk: clock reference for the analog circuitry * @bclk: bus clock common for all ADCs, depends on part used @@ -95,6 +99,7 @@ struct stm32_adc_priv_cfg { */ struct stm32_adc_priv { int irq[STM32_ADC_MAX_ADCS]; + unsigned int nb_adc_max; struct irq_domain *domain; struct clk *aclk; struct clk *bclk; @@ -354,7 +359,7 @@ static void stm32_adc_irq_handler(struct irq_desc *desc) * before invoking the interrupt handler (e.g. call ISR only for * IRQ-enabled ADCs). */ - for (i = 0; i < priv->cfg->num_adcs; i++) { + for (i = 0; i < priv->nb_adc_max; i++) { if ((status & priv->cfg->regs->eoc_msk[i] && stm32_adc_eoc_enabled(priv, i)) || (status & priv->cfg->regs->ovr_msk[i])) @@ -424,7 +429,7 @@ static void stm32_adc_irq_remove(struct platform_device *pdev, int hwirq; unsigned int i; - for (hwirq = 0; hwirq < STM32_ADC_MAX_ADCS; hwirq++) + for (hwirq = 0; hwirq < priv->nb_adc_max; hwirq++) irq_dispose_mapping(irq_find_mapping(priv->domain, hwirq)); irq_domain_remove(priv->domain); @@ -642,6 +647,49 @@ static int stm32_adc_core_switches_probe(struct device *dev, return 0; } +static int stm32_adc_probe_identification(struct platform_device *pdev, + struct stm32_adc_priv *priv) +{ + struct device_node *np = pdev->dev.of_node; + struct device_node *child; + const char *compat; + int ret, count = 0; + u32 id, val; + + if (!priv->cfg->ipid) + return 0; + + id = FIELD_GET(STM32MP1_IPIDR_MASK, + readl_relaxed(priv->common.base + STM32MP1_ADC_IPDR)); + if (id != priv->cfg->ipid) { + dev_err(&pdev->dev, "Unexpected IP version: 0x%x", id); + return -EINVAL; + } + + for_each_child_of_node(np, child) { + ret = of_property_read_string(child, "compatible", &compat); + if (ret) + continue; + /* Count child nodes with stm32 adc compatible */ + if (strstr(compat, "st,stm32") && strstr(compat, "adc")) + count++; + } + + val = readl_relaxed(priv->common.base + STM32MP1_ADC_HWCFGR0); + priv->nb_adc_max = FIELD_GET(STM32MP1_ADCNUM_MASK, val); + if (count > priv->nb_adc_max) { + dev_err(&pdev->dev, "Unexpected child number: %d", count); + return -EINVAL; + } + + val = readl_relaxed(priv->common.base + STM32MP1_ADC_VERR); + dev_dbg(&pdev->dev, "ADC version: %lu.%lu\n", + FIELD_GET(STM32MP1_MAJREV_MASK, val), + FIELD_GET(STM32MP1_MINREV_MASK, val)); + + return 0; +} + static int stm32_adc_probe(struct platform_device *pdev) { struct stm32_adc_priv *priv; @@ -661,6 +709,7 @@ static int stm32_adc_probe(struct platform_device *pdev) priv->cfg = (const struct stm32_adc_priv_cfg *) of_match_device(dev->driver->of_match_table, dev)->data; + priv->nb_adc_max = priv->cfg->num_adcs; spin_lock_init(&priv->common.lock); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -703,6 +752,10 @@ static int stm32_adc_probe(struct platform_device *pdev) if (ret) goto err_pm_stop; + ret = stm32_adc_probe_identification(pdev, priv); + if (ret < 0) + goto err_hw_stop; + ret = regulator_get_voltage(priv->vref); if (ret < 0) { dev_err(&pdev->dev, "vref get voltage failed, %d\n", ret); @@ -811,8 +864,8 @@ static const struct stm32_adc_priv_cfg stm32mp1_adc_priv_cfg = { .clk_sel = stm32h7_adc_clk_sel, .max_clk_rate_hz = 36000000, .has_syscfg = HAS_VBOOSTER | HAS_ANASWVDD, + .ipid = STM32MP15_IPIDR_NUMBER, .num_irqs = 2, - .num_adcs = 2, }; static const struct of_device_id stm32_adc_of_match[] = { diff --git a/drivers/iio/adc/stm32-adc-core.h b/drivers/iio/adc/stm32-adc-core.h index faedf7a495558f46f2a1aa7021f6c3d21920f3d9..2118ef63843d24b78b640b9ea0444c7d8d77242a 100644 --- a/drivers/iio/adc/stm32-adc-core.h +++ b/drivers/iio/adc/stm32-adc-core.h @@ -24,6 +24,7 @@ * | 0x300 | Master & Slave common regs | * -------------------------------------------------------- */ +/* Maximum ADC instances number per ADC block for all supported SoCs */ #define STM32_ADC_MAX_ADCS 3 #define STM32_ADC_OFFSET 0x100 #define STM32_ADCX_COMN_OFFSET 0x300 @@ -105,6 +106,12 @@ /* STM32MP1 - ADC2 instance option register */ #define STM32MP1_ADC2_OR 0xD0 +/* STM32MP1 - Identification registers */ +#define STM32MP1_ADC_HWCFGR0 0x3F0 +#define STM32MP1_ADC_VERR 0x3F4 +#define STM32MP1_ADC_IPDR 0x3F8 +#define STM32MP1_ADC_SIDR 0x3FC + /* STM32H7 - common registers for all ADC instances */ #define STM32H7_ADC_CSR (STM32_ADCX_COMN_OFFSET + 0x00) #define STM32H7_ADC_CCR (STM32_ADCX_COMN_OFFSET + 0x08) @@ -181,6 +188,30 @@ enum stm32h7_adc_dmngt { /* STM32MP1_ADC2_OR - bit fields */ #define STM32MP1_VDDCOREEN BIT(0) +/* STM32MP1_ADC_HWCFGR0 - bit fields */ +#define STM32MP1_ADCNUM_SHIFT 0 +#define STM32MP1_ADCNUM_MASK GENMASK(3, 0) +#define STM32MP1_MULPIPE_SHIFT 4 +#define STM32MP1_MULPIPE_MASK GENMASK(7, 4) +#define STM32MP1_OPBITS_SHIFT 8 +#define STM32MP1_OPBITS_MASK GENMASK(11, 8) +#define STM32MP1_IDLEVALUE_SHIFT 12 +#define STM32MP1_IDLEVALUE_MASK GENMASK(15, 12) + +/* STM32MP1_ADC_VERR - bit fields */ +#define STM32MP1_MINREV_SHIFT 0 +#define STM32MP1_MINREV_MASK GENMASK(3, 0) +#define STM32MP1_MAJREV_SHIFT 4 +#define STM32MP1_MAJREV_MASK GENMASK(7, 4) + +/* STM32MP1_ADC_IPDR - bit fields */ +#define STM32MP1_IPIDR_MASK GENMASK(31, 0) + +/* STM32MP1_ADC_SIDR - bit fields */ +#define STM32MP1_SIDR_MASK GENMASK(31, 0) + +#define STM32MP15_IPIDR_NUMBER 0x00110005 + /** * struct stm32_adc_common - stm32 ADC driver common data (for all instances) * @base: control registers base cpu addr diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c index 130e8dd6f0c89fb3446e172503b491f45bf70d2f..6256977eb7f7bc1c1e24d29bf3e037cb7c51c60e 100644 --- a/drivers/iio/adc/stm32-adc.c +++ b/drivers/iio/adc/stm32-adc.c @@ -21,11 +21,11 @@ #include #include #include +#include #include #include #include -#include -#include +#include #include "stm32-adc-core.h" @@ -241,6 +241,7 @@ struct stm32_adc_cfg { * @chan_name: channel name array * @num_diff: number of differential channels * @int_ch: internal channel indexes array + * @nsmps: number of channels with optional sample time */ struct stm32_adc { struct stm32_adc_common *common; @@ -267,6 +268,7 @@ struct stm32_adc { char chan_name[STM32_ADC_CH_MAX][STM32_ADC_CH_SZ]; u32 num_diff; int int_ch[STM32_ADC_INT_CH_NB]; + int nsmps; }; struct stm32_adc_diff_channel { @@ -1520,8 +1522,8 @@ static int stm32_adc_update_scan_mode(struct iio_dev *indio_dev, return ret; } -static int stm32_adc_of_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int stm32_adc_fwnode_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { int i; @@ -1575,7 +1577,7 @@ static const struct iio_info stm32_adc_iio_info = { .hwfifo_set_watermark = stm32_adc_set_watermark, .update_scan_mode = stm32_adc_update_scan_mode, .debugfs_reg_access = stm32_adc_debugfs_reg_access, - .of_xlate = stm32_adc_of_xlate, + .fwnode_xlate = stm32_adc_fwnode_xlate, }; static unsigned int stm32_adc_dma_residue(struct stm32_adc *adc) @@ -1772,14 +1774,14 @@ static const struct iio_chan_spec_ext_info stm32_adc_ext_info[] = { {}, }; -static int stm32_adc_of_get_resolution(struct iio_dev *indio_dev) +static int stm32_adc_fw_get_resolution(struct iio_dev *indio_dev) { - struct device_node *node = indio_dev->dev.of_node; + struct device *dev = &indio_dev->dev; struct stm32_adc *adc = iio_priv(indio_dev); unsigned int i; u32 res; - if (of_property_read_u32(node, "assigned-resolution-bits", &res)) + if (device_property_read_u32(dev, "assigned-resolution-bits", &res)) res = adc->cfg->adc_info->resolutions[0]; for (i = 0; i < adc->cfg->adc_info->num_res; i++) @@ -1863,11 +1865,11 @@ static void stm32_adc_chan_init_one(struct iio_dev *indio_dev, static int stm32_adc_get_legacy_chan_count(struct iio_dev *indio_dev, struct stm32_adc *adc) { - struct device_node *node = indio_dev->dev.of_node; + struct device *dev = &indio_dev->dev; const struct stm32_adc_info *adc_info = adc->cfg->adc_info; int num_channels = 0, ret; - ret = of_property_count_u32_elems(node, "st,adc-channels"); + ret = device_property_count_u32(dev, "st,adc-channels"); if (ret > adc_info->max_channels) { dev_err(&indio_dev->dev, "Bad st,adc-channels?\n"); return -EINVAL; @@ -1875,8 +1877,15 @@ static int stm32_adc_get_legacy_chan_count(struct iio_dev *indio_dev, struct stm num_channels += ret; } - ret = of_property_count_elems_of_size(node, "st,adc-diff-channels", - sizeof(struct stm32_adc_diff_channel)); + /* + * each st,adc-diff-channels is a group of 2 u32 so we divide @ret + * to get the *real* number of channels. + */ + ret = device_property_count_u32(dev, "st,adc-diff-channels"); + if (ret < 0) + return ret; + + ret /= (int)(sizeof(struct stm32_adc_diff_channel) / sizeof(u32)); if (ret > adc_info->max_channels) { dev_err(&indio_dev->dev, "Bad st,adc-diff-channels?\n"); return -EINVAL; @@ -1886,8 +1895,8 @@ static int stm32_adc_get_legacy_chan_count(struct iio_dev *indio_dev, struct stm } /* Optional sample time is provided either for each, or all channels */ - ret = of_property_count_u32_elems(node, "st,min-sample-time-nsecs"); - if (ret > 1 && ret != num_channels) { + adc->nsmps = device_property_count_u32(dev, "st,min-sample-time-nsecs"); + if (adc->nsmps > 1 && adc->nsmps != num_channels) { dev_err(&indio_dev->dev, "Invalid st,min-sample-time-nsecs\n"); return -EINVAL; } @@ -1897,21 +1906,20 @@ static int stm32_adc_get_legacy_chan_count(struct iio_dev *indio_dev, struct stm static int stm32_adc_legacy_chan_init(struct iio_dev *indio_dev, struct stm32_adc *adc, - struct iio_chan_spec *channels) + struct iio_chan_spec *channels, + int nchans) { - struct device_node *node = indio_dev->dev.of_node; const struct stm32_adc_info *adc_info = adc->cfg->adc_info; struct stm32_adc_diff_channel diff[STM32_ADC_CH_MAX]; + struct device *dev = &indio_dev->dev; u32 num_diff = adc->num_diff; int size = num_diff * sizeof(*diff) / sizeof(u32); - int scan_index = 0, val, ret, i; - struct property *prop; - const __be32 *cur; - u32 smp = 0; + int scan_index = 0, ret, i, c; + u32 smp = 0, smps[STM32_ADC_CH_MAX], chans[STM32_ADC_CH_MAX]; if (num_diff) { - ret = of_property_read_u32_array(node, "st,adc-diff-channels", - (u32 *)diff, size); + ret = device_property_read_u32_array(dev, "st,adc-diff-channels", + (u32 *)diff, size); if (ret) { dev_err(&indio_dev->dev, "Failed to get diff channels %d\n", ret); return ret; @@ -1932,32 +1940,47 @@ static int stm32_adc_legacy_chan_init(struct iio_dev *indio_dev, } } - of_property_for_each_u32(node, "st,adc-channels", prop, cur, val) { - if (val >= adc_info->max_channels) { - dev_err(&indio_dev->dev, "Invalid channel %d\n", val); + ret = device_property_read_u32_array(dev, "st,adc-channels", chans, + nchans); + if (ret) + return ret; + + for (c = 0; c < nchans; c++) { + if (chans[c] >= adc_info->max_channels) { + dev_err(&indio_dev->dev, "Invalid channel %d\n", + chans[c]); return -EINVAL; } /* Channel can't be configured both as single-ended & diff */ for (i = 0; i < num_diff; i++) { - if (val == diff[i].vinp) { - dev_err(&indio_dev->dev, "channel %d misconfigured\n", val); + if (chans[c] == diff[i].vinp) { + dev_err(&indio_dev->dev, "channel %d misconfigured\n", chans[c]); return -EINVAL; } } - stm32_adc_chan_init_one(indio_dev, &channels[scan_index], val, - 0, scan_index, false); + stm32_adc_chan_init_one(indio_dev, &channels[scan_index], + chans[c], 0, scan_index, false); scan_index++; } + if (adc->nsmps > 0) { + ret = device_property_read_u32_array(dev, "st,min-sample-time-nsecs", + smps, adc->nsmps); + if (ret) + return ret; + } + for (i = 0; i < scan_index; i++) { /* - * Using of_property_read_u32_index(), smp value will only be - * modified if valid u32 value can be decoded. This allows to - * get either no value, 1 shared value for all indexes, or one - * value per channel. + * This check is used with the above logic so that smp value + * will only be modified if valid u32 value can be decoded. This + * allows to get either no value, 1 shared value for all indexes, + * or one value per channel. The point is to have the same + * behavior as 'of_property_read_u32_index()'. */ - of_property_read_u32_index(node, "st,min-sample-time-nsecs", i, &smp); + if (i < adc->nsmps) + smp = smps[i]; /* Prepare sampling time settings */ stm32_adc_smpr_init(adc, channels[i].channel, smp); @@ -2005,22 +2028,21 @@ static int stm32_adc_generic_chan_init(struct iio_dev *indio_dev, struct stm32_adc *adc, struct iio_chan_spec *channels) { - struct device_node *node = indio_dev->dev.of_node; const struct stm32_adc_info *adc_info = adc->cfg->adc_info; - struct device_node *child; + struct fwnode_handle *child; const char *name; int val, scan_index = 0, ret; bool differential; u32 vin[2]; - for_each_available_child_of_node(node, child) { - ret = of_property_read_u32(child, "reg", &val); + device_for_each_child_node(&indio_dev->dev, child) { + ret = fwnode_property_read_u32(child, "reg", &val); if (ret) { dev_err(&indio_dev->dev, "Missing channel index %d\n", ret); goto err; } - ret = of_property_read_string(child, "label", &name); + ret = fwnode_property_read_string(child, "label", &name); /* label is optional */ if (!ret) { if (strlen(name) >= STM32_ADC_CH_SZ) { @@ -2047,7 +2069,7 @@ static int stm32_adc_generic_chan_init(struct iio_dev *indio_dev, } differential = false; - ret = of_property_read_u32_array(child, "diff-channels", vin, 2); + ret = fwnode_property_read_u32_array(child, "diff-channels", vin, 2); /* diff-channels is optional */ if (!ret) { differential = true; @@ -2064,7 +2086,7 @@ static int stm32_adc_generic_chan_init(struct iio_dev *indio_dev, stm32_adc_chan_init_one(indio_dev, &channels[scan_index], val, vin[1], scan_index, differential); - ret = of_property_read_u32(child, "st,min-sample-time-ns", &val); + ret = fwnode_property_read_u32(child, "st,min-sample-time-ns", &val); /* st,min-sample-time-ns is optional */ if (!ret) { stm32_adc_smpr_init(adc, channels[scan_index].channel, val); @@ -2082,14 +2104,13 @@ static int stm32_adc_generic_chan_init(struct iio_dev *indio_dev, return scan_index; err: - of_node_put(child); + fwnode_handle_put(child); return ret; } -static int stm32_adc_chan_of_init(struct iio_dev *indio_dev, bool timestamping) +static int stm32_adc_chan_fw_init(struct iio_dev *indio_dev, bool timestamping) { - struct device_node *node = indio_dev->dev.of_node; struct stm32_adc *adc = iio_priv(indio_dev); const struct stm32_adc_info *adc_info = adc->cfg->adc_info; struct iio_chan_spec *channels; @@ -2099,7 +2120,7 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev, bool timestamping) for (i = 0; i < STM32_ADC_INT_CH_NB; i++) adc->int_ch[i] = STM32_ADC_INT_CH_NONE; - num_channels = of_get_available_child_count(node); + num_channels = device_get_child_node_count(&indio_dev->dev); /* If no channels have been found, fallback to channels legacy properties. */ if (!num_channels) { legacy = true; @@ -2130,7 +2151,8 @@ static int stm32_adc_chan_of_init(struct iio_dev *indio_dev, bool timestamping) return -ENOMEM; if (legacy) - ret = stm32_adc_legacy_chan_init(indio_dev, adc, channels); + ret = stm32_adc_legacy_chan_init(indio_dev, adc, channels, + num_channels); else ret = stm32_adc_generic_chan_init(indio_dev, adc, channels); if (ret < 0) @@ -2212,9 +2234,6 @@ static int stm32_adc_probe(struct platform_device *pdev) bool timestamping = false; int ret; - if (!pdev->dev.of_node) - return -ENODEV; - indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(*adc)); if (!indio_dev) return -ENOMEM; @@ -2223,17 +2242,16 @@ static int stm32_adc_probe(struct platform_device *pdev) adc->common = dev_get_drvdata(pdev->dev.parent); spin_lock_init(&adc->lock); init_completion(&adc->completion); - adc->cfg = (const struct stm32_adc_cfg *) - of_match_device(dev->driver->of_match_table, dev)->data; + adc->cfg = device_get_match_data(dev); indio_dev->name = dev_name(&pdev->dev); - indio_dev->dev.of_node = pdev->dev.of_node; + device_set_node(&indio_dev->dev, dev_fwnode(&pdev->dev)); indio_dev->info = &stm32_adc_iio_info; indio_dev->modes = INDIO_DIRECT_MODE | INDIO_HARDWARE_TRIGGERED; platform_set_drvdata(pdev, indio_dev); - ret = of_property_read_u32(pdev->dev.of_node, "reg", &adc->offset); + ret = device_property_read_u32(dev, "reg", &adc->offset); if (ret != 0) { dev_err(&pdev->dev, "missing reg property\n"); return -EINVAL; @@ -2262,7 +2280,7 @@ static int stm32_adc_probe(struct platform_device *pdev) } } - ret = stm32_adc_of_get_resolution(indio_dev); + ret = stm32_adc_fw_get_resolution(indio_dev); if (ret < 0) return ret; @@ -2279,7 +2297,7 @@ static int stm32_adc_probe(struct platform_device *pdev) timestamping = true; } - ret = stm32_adc_chan_of_init(indio_dev, timestamping); + ret = stm32_adc_chan_fw_init(indio_dev, timestamping); if (ret < 0) goto err_dma_disable; diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c index 2d393a4dfff6a8665ef980768993dc59b0cf2780..a6ade70dedf89fbfc9b97b14a6a4ce5edf78de33 100644 --- a/drivers/iio/adc/sun4i-gpadc-iio.c +++ b/drivers/iio/adc/sun4i-gpadc-iio.c @@ -412,9 +412,9 @@ static int sun4i_gpadc_runtime_resume(struct device *dev) return 0; } -static int sun4i_gpadc_get_temp(void *data, int *temp) +static int sun4i_gpadc_get_temp(struct thermal_zone_device *tz, int *temp) { - struct sun4i_gpadc_iio *info = data; + struct sun4i_gpadc_iio *info = tz->devdata; int val, scale, offset; if (sun4i_gpadc_temp_read(info->indio_dev, &val)) @@ -428,7 +428,7 @@ static int sun4i_gpadc_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops sun4i_ts_tz_ops = { +static const struct thermal_zone_device_ops sun4i_ts_tz_ops = { .get_temp = &sun4i_gpadc_get_temp, }; @@ -637,9 +637,9 @@ static int sun4i_gpadc_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); if (IS_ENABLED(CONFIG_THERMAL_OF)) { - info->tzd = thermal_zone_of_sensor_register(info->sensor_device, - 0, info, - &sun4i_ts_tz_ops); + info->tzd = devm_thermal_of_zone_register(info->sensor_device, + 0, info, + &sun4i_ts_tz_ops); /* * Do not fail driver probing when failing to register in * thermal because no thermal DT node is found. @@ -681,8 +681,6 @@ static int sun4i_gpadc_remove(struct platform_device *pdev) if (!IS_ENABLED(CONFIG_THERMAL_OF)) return 0; - thermal_zone_of_sensor_unregister(info->sensor_device, info->tzd); - if (!info->no_irq) iio_map_array_unregister(indio_dev); diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c index e3dfc155fbe2eaf2b3a7bdfe978fd0d040e42119..8bceba69402603719c8f25e1a3cb1034aaf5f869 100644 --- a/drivers/iio/adc/ti-ads1015.c +++ b/drivers/iio/adc/ti-ads1015.c @@ -1094,7 +1094,7 @@ static int ads1015_probe(struct i2c_client *client, return 0; } -static int ads1015_remove(struct i2c_client *client) +static void ads1015_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ads1015_data *data = iio_priv(indio_dev); @@ -1110,8 +1110,6 @@ static int ads1015_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to power down (%pe)\n", ERR_PTR(ret)); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/iio/adc/ti-ads131e08.c b/drivers/iio/adc/ti-ads131e08.c index 32237cacc9a37ca3d43979aadfe45fdebf7d3b3c..5235a93f28bc67c6eec258f518987dbc8be5623c 100644 --- a/drivers/iio/adc/ti-ads131e08.c +++ b/drivers/iio/adc/ti-ads131e08.c @@ -797,13 +797,6 @@ static void ads131e08_regulator_disable(void *data) regulator_disable(st->vref_reg); } -static void ads131e08_clk_disable(void *data) -{ - struct ads131e08_state *st = data; - - clk_disable_unprepare(st->adc_clk); -} - static int ads131e08_probe(struct spi_device *spi) { const struct ads131e08_info *info; @@ -896,21 +889,11 @@ static int ads131e08_probe(struct spi_device *spi) st->vref_reg = NULL; } - st->adc_clk = devm_clk_get(&spi->dev, "adc-clk"); + st->adc_clk = devm_clk_get_enabled(&spi->dev, "adc-clk"); if (IS_ERR(st->adc_clk)) return dev_err_probe(&spi->dev, PTR_ERR(st->adc_clk), "failed to get the ADC clock\n"); - ret = clk_prepare_enable(st->adc_clk); - if (ret) { - dev_err(&spi->dev, "failed to prepare/enable the ADC clock\n"); - return ret; - } - - ret = devm_add_action_or_reset(&spi->dev, ads131e08_clk_disable, st); - if (ret) - return ret; - adc_clk_hz = clk_get_rate(st->adc_clk); if (!adc_clk_hz) { dev_err(&spi->dev, "failed to get the ADC clock rate\n"); diff --git a/drivers/iio/adc/ti-tsc2046.c b/drivers/iio/adc/ti-tsc2046.c index 0d9436a69cbfb811dc8f11577a0c7bae46f55b8d..1bbb51a6683c9512dff5f2ccaa0aa404d4b67fc0 100644 --- a/drivers/iio/adc/ti-tsc2046.c +++ b/drivers/iio/adc/ti-tsc2046.c @@ -8,7 +8,9 @@ #include #include #include +#include #include +#include #include @@ -139,6 +141,7 @@ enum tsc2046_state { struct tsc2046_adc_priv { struct spi_device *spi; const struct tsc2046_adc_dcfg *dcfg; + struct regulator *vref_reg; struct iio_trigger *trig; struct hrtimer trig_timer; @@ -173,6 +176,7 @@ struct tsc2046_adc_priv { u32 scan_interval_us; u32 time_per_scan_us; u32 time_per_bit_ns; + unsigned int vref_mv; struct tsc2046_adc_ch_cfg ch_cfg[TI_TSC2046_MAX_CHAN]; }; @@ -252,7 +256,9 @@ static u8 tsc2046_adc_get_cmd(struct tsc2046_adc_priv *priv, int ch_idx, case TI_TSC2046_ADDR_AUX: case TI_TSC2046_ADDR_VBAT: case TI_TSC2046_ADDR_TEMP0: - pd |= TI_TSC2046_SER | TI_TSC2046_PD1_VREF_ON; + pd |= TI_TSC2046_SER; + if (!priv->vref_reg) + pd |= TI_TSC2046_PD1_VREF_ON; } return TI_TSC2046_START | FIELD_PREP(TI_TSC2046_ADDR, ch_idx) | pd; @@ -468,7 +474,7 @@ static int tsc2046_adc_read_raw(struct iio_dev *indio_dev, * So, it is better to use external voltage-divider driver * instead, which is calculating complete chain. */ - *val = TI_TSC2046_INT_VREF; + *val = priv->vref_mv; *val2 = chan->scan_type.realbits; return IIO_VAL_FRACTIONAL_LOG2; } @@ -740,6 +746,49 @@ static void tsc2046_adc_parse_fwnode(struct tsc2046_adc_priv *priv) } } +static void tsc2046_adc_regulator_disable(void *data) +{ + struct tsc2046_adc_priv *priv = data; + + regulator_disable(priv->vref_reg); +} + +static int tsc2046_adc_configure_regulator(struct tsc2046_adc_priv *priv) +{ + struct device *dev = &priv->spi->dev; + int ret; + + priv->vref_reg = devm_regulator_get_optional(dev, "vref"); + if (IS_ERR(priv->vref_reg)) { + /* If regulator exists but can't be get, return an error */ + if (PTR_ERR(priv->vref_reg) != -ENODEV) + return PTR_ERR(priv->vref_reg); + priv->vref_reg = NULL; + } + if (!priv->vref_reg) { + /* Use internal reference */ + priv->vref_mv = TI_TSC2046_INT_VREF; + return 0; + } + + ret = regulator_enable(priv->vref_reg); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, tsc2046_adc_regulator_disable, + priv); + if (ret) + return ret; + + ret = regulator_get_voltage(priv->vref_reg); + if (ret < 0) + return ret; + + priv->vref_mv = ret / MILLI; + + return 0; +} + static int tsc2046_adc_probe(struct spi_device *spi) { const struct tsc2046_adc_dcfg *dcfg; @@ -756,6 +805,11 @@ static int tsc2046_adc_probe(struct spi_device *spi) } dcfg = device_get_match_data(dev); + if (!dcfg) { + const struct spi_device_id *id = spi_get_device_id(spi); + + dcfg = (const struct tsc2046_adc_dcfg *)id->driver_data; + } if (!dcfg) return -EINVAL; @@ -781,6 +835,10 @@ static int tsc2046_adc_probe(struct spi_device *spi) indio_dev->num_channels = dcfg->num_channels; indio_dev->info = &tsc2046_adc_info; + ret = tsc2046_adc_configure_regulator(priv); + if (ret) + return ret; + tsc2046_adc_parse_fwnode(priv); ret = tsc2046_adc_setup_spi_msg(priv); @@ -833,11 +891,18 @@ static const struct of_device_id ads7950_of_table[] = { }; MODULE_DEVICE_TABLE(of, ads7950_of_table); +static const struct spi_device_id tsc2046_adc_spi_ids[] = { + { "tsc2046e-adc", (unsigned long)&tsc2046_adc_dcfg_tsc2046e }, + { } +}; +MODULE_DEVICE_TABLE(spi, tsc2046_adc_spi_ids); + static struct spi_driver tsc2046_adc_driver = { .driver = { .name = "tsc2046", .of_match_table = ads7950_of_table, }, + .id_table = tsc2046_adc_spi_ids, .probe = tsc2046_adc_probe, }; module_spi_driver(tsc2046_adc_driver); diff --git a/drivers/iio/adc/xilinx-ams.c b/drivers/iio/adc/xilinx-ams.c index 9cd2713146e5578296ce3f316125c1fa7dd5beb1..5b4bdf3a26bb8a942151ddbdea53dc3ec6a8f5ad 100644 --- a/drivers/iio/adc/xilinx-ams.c +++ b/drivers/iio/adc/xilinx-ams.c @@ -1351,11 +1351,6 @@ static const struct of_device_id ams_of_match_table[] = { }; MODULE_DEVICE_TABLE(of, ams_of_match_table); -static void ams_clk_disable_unprepare(void *data) -{ - clk_disable_unprepare(data); -} - static int ams_probe(struct platform_device *pdev) { struct iio_dev *indio_dev; @@ -1380,18 +1375,10 @@ static int ams_probe(struct platform_device *pdev) if (IS_ERR(ams->base)) return PTR_ERR(ams->base); - ams->clk = devm_clk_get(&pdev->dev, NULL); + ams->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(ams->clk)) return PTR_ERR(ams->clk); - ret = clk_prepare_enable(ams->clk); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&pdev->dev, ams_clk_disable_unprepare, ams->clk); - if (ret < 0) - return ret; - ret = devm_delayed_work_autocancel(&pdev->dev, &ams->ams_unmask_work, ams_unmask_worker); if (ret < 0) diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c index 1b247722ba25dccc5d056c677600da75e73f9a1d..292f2892d223a0573d86ecad75e969deae06cb37 100644 --- a/drivers/iio/adc/xilinx-xadc-core.c +++ b/drivers/iio/adc/xilinx-xadc-core.c @@ -1296,13 +1296,6 @@ static const char * const xadc_type_names[] = { [XADC_TYPE_US] = "xilinx-system-monitor", }; -static void xadc_clk_disable_unprepare(void *data) -{ - struct clk *clk = data; - - clk_disable_unprepare(clk); -} - static void xadc_cancel_delayed_work(void *data) { struct delayed_work *work = data; @@ -1374,19 +1367,10 @@ static int xadc_probe(struct platform_device *pdev) } } - xadc->clk = devm_clk_get(dev, NULL); + xadc->clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(xadc->clk)) return PTR_ERR(xadc->clk); - ret = clk_prepare_enable(xadc->clk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(dev, - xadc_clk_disable_unprepare, xadc->clk); - if (ret) - return ret; - /* * Make sure not to exceed the maximum samplerate since otherwise the * resulting interrupt storm will soft-lock the system. diff --git a/drivers/iio/addac/Kconfig b/drivers/iio/addac/Kconfig index 138492362f20974752a8b6eb1c50c55c07a0415a..fcf6d2269bfc25d9f497a25ef4ccf3ae5018e63a 100644 --- a/drivers/iio/addac/Kconfig +++ b/drivers/iio/addac/Kconfig @@ -17,4 +17,20 @@ config AD74413R To compile this driver as a module, choose M here: the module will be called ad74413r. +config STX104 + tristate "Apex Embedded Systems STX104 driver" + depends on PC104 && X86 + select ISA_BUS_API + select GPIOLIB + help + Say yes here to build support for the Apex Embedded Systems STX104 + integrated analog PC/104 card. + + This driver supports the 16 channels of single-ended (8 channels of + differential) analog inputs, 2 channels of analog output, 4 digital + inputs, and 4 digital outputs provided by the STX104. + + The base port addresses for the devices may be configured via the base + array module parameter. + endmenu diff --git a/drivers/iio/addac/Makefile b/drivers/iio/addac/Makefile index cfd4bbe64ad3b506e3b4ccbf7cea239a540b6a2c..17de20ef0d8e9df868a33076b537e56f1ef2ee0f 100644 --- a/drivers/iio/addac/Makefile +++ b/drivers/iio/addac/Makefile @@ -5,3 +5,4 @@ # When adding new entries keep the list in alphabetical order obj-$(CONFIG_AD74413R) += ad74413r.o +obj-$(CONFIG_STX104) += stx104.o diff --git a/drivers/iio/adc/stx104.c b/drivers/iio/addac/stx104.c similarity index 100% rename from drivers/iio/adc/stx104.c rename to drivers/iio/addac/stx104.c diff --git a/drivers/iio/cdc/Kconfig b/drivers/iio/cdc/Kconfig index 5e3319a3ff48e5a6e7d3d71475e1a2b46efb9f7e..e0a5ce66a9845c37aa5c2a938a1d13d2a9de4de3 100644 --- a/drivers/iio/cdc/Kconfig +++ b/drivers/iio/cdc/Kconfig @@ -14,4 +14,14 @@ config AD7150 To compile this driver as a module, choose M here: the module will be called ad7150. +config AD7746 + tristate "Analog Devices AD7745, AD7746 AD7747 capacitive sensor driver" + depends on I2C + help + Say yes here to build support for Analog Devices capacitive sensors. + (AD7745, AD7746, AD7747) Provides direct access via sysfs. + + To compile this driver as a module, choose M here: the + module will be called ad7746. + endmenu diff --git a/drivers/iio/cdc/Makefile b/drivers/iio/cdc/Makefile index ee490637b032516094622cc98464ab6e07e72003..41db756d80201ffc21bee947cf2ed309bd3fdd15 100644 --- a/drivers/iio/cdc/Makefile +++ b/drivers/iio/cdc/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_AD7150) += ad7150.o +obj-$(CONFIG_AD7746) += ad7746.o diff --git a/drivers/staging/iio/cdc/ad7746.c b/drivers/iio/cdc/ad7746.c similarity index 66% rename from drivers/staging/iio/cdc/ad7746.c rename to drivers/iio/cdc/ad7746.c index 52b8957c19c9a30b7dc52761debf80474e2544ca..b266f532814090c05cca23cebc60d5ba2f29a3ee 100644 --- a/drivers/staging/iio/cdc/ad7746.c +++ b/drivers/iio/cdc/ad7746.c @@ -5,6 +5,7 @@ * Copyright 2011 Analog Devices Inc. */ +#include #include #include #include @@ -15,12 +16,12 @@ #include #include +#include + #include #include -/* - * AD7746 Register Definition - */ +/* AD7746 Register Definition */ #define AD7746_REG_STATUS 0 #define AD7746_REG_CAP_DATA_HIGH 1 @@ -48,11 +49,12 @@ #define AD7746_CAPSETUP_CACHOP BIT(0) /* Voltage/Temperature Setup Register Bit Designations (AD7746_REG_VT_SETUP) */ -#define AD7746_VTSETUP_VTEN (1 << 7) -#define AD7746_VTSETUP_VTMD_INT_TEMP (0 << 5) -#define AD7746_VTSETUP_VTMD_EXT_TEMP (1 << 5) -#define AD7746_VTSETUP_VTMD_VDD_MON (2 << 5) -#define AD7746_VTSETUP_VTMD_EXT_VIN (3 << 5) +#define AD7746_VTSETUP_VTEN BIT(7) +#define AD7746_VTSETUP_VTMD_MASK GENMASK(6, 5) +#define AD7746_VTSETUP_VTMD_INT_TEMP 0 +#define AD7746_VTSETUP_VTMD_EXT_TEMP 1 +#define AD7746_VTSETUP_VTMD_VDD_MON 2 +#define AD7746_VTSETUP_VTMD_EXT_VIN 3 #define AD7746_VTSETUP_EXTREF BIT(4) #define AD7746_VTSETUP_VTSHORT BIT(1) #define AD7746_VTSETUP_VTCHOP BIT(0) @@ -64,23 +66,22 @@ #define AD7746_EXCSETUP_NEXCB BIT(4) #define AD7746_EXCSETUP_EXCA BIT(3) #define AD7746_EXCSETUP_NEXCA BIT(2) -#define AD7746_EXCSETUP_EXCLVL(x) (((x) & 0x3) << 0) +#define AD7746_EXCSETUP_EXCLVL_MASK GENMASK(1, 0) /* Config Register Bit Designations (AD7746_REG_CFG) */ -#define AD7746_CONF_VTFS_SHIFT 6 -#define AD7746_CONF_CAPFS_SHIFT 3 #define AD7746_CONF_VTFS_MASK GENMASK(7, 6) #define AD7746_CONF_CAPFS_MASK GENMASK(5, 3) -#define AD7746_CONF_MODE_IDLE (0 << 0) -#define AD7746_CONF_MODE_CONT_CONV (1 << 0) -#define AD7746_CONF_MODE_SINGLE_CONV (2 << 0) -#define AD7746_CONF_MODE_PWRDN (3 << 0) -#define AD7746_CONF_MODE_OFFS_CAL (5 << 0) -#define AD7746_CONF_MODE_GAIN_CAL (6 << 0) +#define AD7746_CONF_MODE_MASK GENMASK(2, 0) +#define AD7746_CONF_MODE_IDLE 0 +#define AD7746_CONF_MODE_CONT_CONV 1 +#define AD7746_CONF_MODE_SINGLE_CONV 2 +#define AD7746_CONF_MODE_PWRDN 3 +#define AD7746_CONF_MODE_OFFS_CAL 5 +#define AD7746_CONF_MODE_GAIN_CAL 6 /* CAPDAC Register Bit Designations (AD7746_REG_CAPDACx) */ #define AD7746_CAPDAC_DACEN BIT(7) -#define AD7746_CAPDAC_DACP(x) ((x) & 0x7F) +#define AD7746_CAPDAC_DACP_MASK GENMASK(6, 0) struct ad7746_chip_info { struct i2c_client *client; @@ -94,11 +95,6 @@ struct ad7746_chip_info { u8 vt_setup; u8 capdac[2][2]; s8 capdac_set; - - union { - __be32 d32; - u8 d8[4]; - } data ____cacheline_aligned; }; enum ad7746_chan { @@ -112,43 +108,87 @@ enum ad7746_chan { CIN2_DIFF, }; +struct ad7746_chan_info { + u8 addr; + union { + u8 vtmd; + struct { /* CAP SETUP fields */ + unsigned int cin2 : 1; + unsigned int capdiff : 1; + }; + }; +}; + +static const struct ad7746_chan_info ad7746_chan_info[] = { + [VIN] = { + .addr = AD7746_REG_VT_DATA_HIGH, + .vtmd = AD7746_VTSETUP_VTMD_EXT_VIN, + }, + [VIN_VDD] = { + .addr = AD7746_REG_VT_DATA_HIGH, + .vtmd = AD7746_VTSETUP_VTMD_VDD_MON, + }, + [TEMP_INT] = { + .addr = AD7746_REG_VT_DATA_HIGH, + .vtmd = AD7746_VTSETUP_VTMD_INT_TEMP, + }, + [TEMP_EXT] = { + .addr = AD7746_REG_VT_DATA_HIGH, + .vtmd = AD7746_VTSETUP_VTMD_EXT_TEMP, + }, + [CIN1] = { + .addr = AD7746_REG_CAP_DATA_HIGH, + }, + [CIN1_DIFF] = { + .addr = AD7746_REG_CAP_DATA_HIGH, + .capdiff = 1, + }, + [CIN2] = { + .addr = AD7746_REG_CAP_DATA_HIGH, + .cin2 = 1, + }, + [CIN2_DIFF] = { + .addr = AD7746_REG_CAP_DATA_HIGH, + .cin2 = 1, + .capdiff = 1, + }, +}; + static const struct iio_chan_spec ad7746_channels[] = { [VIN] = { .type = IIO_VOLTAGE, .indexed = 1, .channel = 0, - .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), - .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | - BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_VT_DATA_HIGH << 8 | - AD7746_VTSETUP_VTMD_EXT_VIN, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = VIN, }, [VIN_VDD] = { .type = IIO_VOLTAGE, .indexed = 1, .channel = 1, .extend_name = "supply", - .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), - .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | - BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_VT_DATA_HIGH << 8 | - AD7746_VTSETUP_VTMD_VDD_MON, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_SCALE), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = VIN_VDD, }, [TEMP_INT] = { .type = IIO_TEMP, .indexed = 1, .channel = 0, - .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), - .address = AD7746_REG_VT_DATA_HIGH << 8 | - AD7746_VTSETUP_VTMD_INT_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), + .address = TEMP_INT, }, [TEMP_EXT] = { .type = IIO_TEMP, .indexed = 1, .channel = 1, - .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), - .address = AD7746_REG_VT_DATA_HIGH << 8 | - AD7746_VTSETUP_VTMD_EXT_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW), + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), + .address = TEMP_EXT, }, [CIN1] = { .type = IIO_CAPACITANCE, @@ -158,7 +198,8 @@ static const struct iio_chan_spec ad7746_channels[] = { BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) | BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_CAP_DATA_HIGH << 8, + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = CIN1, }, [CIN1_DIFF] = { .type = IIO_CAPACITANCE, @@ -167,11 +208,11 @@ static const struct iio_chan_spec ad7746_channels[] = { .channel = 0, .channel2 = 2, .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | - BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_ZEROPOINT), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) | BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_CAP_DATA_HIGH << 8 | - AD7746_CAPSETUP_CAPDIFF + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = CIN1_DIFF, }, [CIN2] = { .type = IIO_CAPACITANCE, @@ -181,8 +222,8 @@ static const struct iio_chan_spec ad7746_channels[] = { BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) | BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_CAP_DATA_HIGH << 8 | - AD7746_CAPSETUP_CIN2, + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = CIN2, }, [CIN2_DIFF] = { .type = IIO_CAPACITANCE, @@ -191,22 +232,22 @@ static const struct iio_chan_spec ad7746_channels[] = { .channel = 1, .channel2 = 3, .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | - BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_CALIBSCALE) | BIT(IIO_CHAN_INFO_ZEROPOINT), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_CALIBBIAS) | BIT(IIO_CHAN_INFO_SCALE) | BIT(IIO_CHAN_INFO_SAMP_FREQ), - .address = AD7746_REG_CAP_DATA_HIGH << 8 | - AD7746_CAPSETUP_CAPDIFF | AD7746_CAPSETUP_CIN2, + .info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SAMP_FREQ), + .address = CIN2_DIFF, } }; /* Values are Update Rate (Hz), Conversion Time (ms) + 1*/ static const unsigned char ad7746_vt_filter_rate_table[][2] = { - {50, 20 + 1}, {31, 32 + 1}, {16, 62 + 1}, {8, 122 + 1}, + { 50, 20 + 1 }, { 31, 32 + 1 }, { 16, 62 + 1 }, { 8, 122 + 1 }, }; static const unsigned char ad7746_cap_filter_rate_table[][2] = { - {91, 11 + 1}, {84, 12 + 1}, {50, 20 + 1}, {26, 38 + 1}, - {16, 62 + 1}, {13, 77 + 1}, {11, 92 + 1}, {9, 110 + 1}, + { 91, 11 + 1 }, { 84, 12 + 1 }, { 50, 20 + 1 }, { 26, 38 + 1 }, + { 16, 62 + 1 }, { 13, 77 + 1 }, { 11, 92 + 1 }, { 9, 110 + 1 }, }; static int ad7746_set_capdac(struct ad7746_chip_info *chip, int channel) @@ -231,10 +272,13 @@ static int ad7746_select_channel(struct iio_dev *indio_dev, switch (chan->type) { case IIO_CAPACITANCE: - cap_setup = (chan->address & 0xFF) | AD7746_CAPSETUP_CAPEN; + cap_setup = FIELD_PREP(AD7746_CAPSETUP_CIN2, + ad7746_chan_info[chan->address].cin2) | + FIELD_PREP(AD7746_CAPSETUP_CAPDIFF, + ad7746_chan_info[chan->address].capdiff) | + FIELD_PREP(AD7746_CAPSETUP_CAPEN, 1); vt_setup = chip->vt_setup & ~AD7746_VTSETUP_VTEN; - idx = (chip->config & AD7746_CONF_CAPFS_MASK) >> - AD7746_CONF_CAPFS_SHIFT; + idx = FIELD_GET(AD7746_CONF_CAPFS_MASK, chip->config); delay = ad7746_cap_filter_rate_table[idx][1]; ret = ad7746_set_capdac(chip, chan->channel); @@ -246,10 +290,11 @@ static int ad7746_select_channel(struct iio_dev *indio_dev, break; case IIO_VOLTAGE: case IIO_TEMP: - vt_setup = (chan->address & 0xFF) | AD7746_VTSETUP_VTEN; + vt_setup = FIELD_PREP(AD7746_VTSETUP_VTMD_MASK, + ad7746_chan_info[chan->address].vtmd) | + FIELD_PREP(AD7746_VTSETUP_VTEN, 1); cap_setup = chip->cap_setup & ~AD7746_CAPSETUP_CAPEN; - idx = (chip->config & AD7746_CONF_VTFS_MASK) >> - AD7746_CONF_VTFS_SHIFT; + idx = FIELD_GET(AD7746_CONF_VTFS_MASK, chip->config); delay = ad7746_cap_filter_rate_table[idx][1]; break; default: @@ -332,7 +377,8 @@ static ssize_t ad7746_start_offset_calib(struct device *dev, return ret; return ad7746_start_calib(dev, attr, buf, len, - AD7746_CONF_MODE_OFFS_CAL); + FIELD_PREP(AD7746_CONF_MODE_MASK, + AD7746_CONF_MODE_OFFS_CAL)); } static ssize_t ad7746_start_gain_calib(struct device *dev, @@ -347,7 +393,8 @@ static ssize_t ad7746_start_gain_calib(struct device *dev, return ret; return ad7746_start_calib(dev, attr, buf, len, - AD7746_CONF_MODE_GAIN_CAL); + FIELD_PREP(AD7746_CONF_MODE_MASK, + AD7746_CONF_MODE_GAIN_CAL)); } static IIO_DEVICE_ATTR(in_capacitance0_calibbias_calibration, @@ -374,7 +421,7 @@ static int ad7746_store_cap_filter_rate_setup(struct ad7746_chip_info *chip, i = ARRAY_SIZE(ad7746_cap_filter_rate_table) - 1; chip->config &= ~AD7746_CONF_CAPFS_MASK; - chip->config |= i << AD7746_CONF_CAPFS_SHIFT; + chip->config |= FIELD_PREP(AD7746_CONF_CAPFS_MASK, i); return 0; } @@ -392,23 +439,17 @@ static int ad7746_store_vt_filter_rate_setup(struct ad7746_chip_info *chip, i = ARRAY_SIZE(ad7746_vt_filter_rate_table) - 1; chip->config &= ~AD7746_CONF_VTFS_MASK; - chip->config |= i << AD7746_CONF_VTFS_SHIFT; + chip->config |= FIELD_PREP(AD7746_CONF_VTFS_MASK, i); return 0; } -static IIO_CONST_ATTR(in_voltage_sampling_frequency_available, "50 31 16 8"); -static IIO_CONST_ATTR(in_capacitance_sampling_frequency_available, - "91 84 50 26 16 13 11 9"); - static struct attribute *ad7746_attributes[] = { &iio_dev_attr_in_capacitance0_calibbias_calibration.dev_attr.attr, &iio_dev_attr_in_capacitance0_calibscale_calibration.dev_attr.attr, &iio_dev_attr_in_capacitance1_calibscale_calibration.dev_attr.attr, &iio_dev_attr_in_capacitance1_calibbias_calibration.dev_attr.attr, &iio_dev_attr_in_voltage0_calibscale_calibration.dev_attr.attr, - &iio_const_attr_in_voltage_sampling_frequency_available.dev_attr.attr, - &iio_const_attr_in_capacitance_sampling_frequency_available.dev_attr.attr, NULL, }; @@ -425,14 +466,10 @@ static int ad7746_write_raw(struct iio_dev *indio_dev, struct ad7746_chip_info *chip = iio_priv(indio_dev); int ret, reg; - mutex_lock(&chip->lock); - switch (mask) { case IIO_CHAN_INFO_CALIBSCALE: - if (val != 1) { - ret = -EINVAL; - goto out; - } + if (val != 1) + return -EINVAL; val = (val2 * 1024) / 15625; @@ -444,33 +481,32 @@ static int ad7746_write_raw(struct iio_dev *indio_dev, reg = AD7746_REG_VOLT_GAINH; break; default: - ret = -EINVAL; - goto out; + return -EINVAL; } + mutex_lock(&chip->lock); ret = i2c_smbus_write_word_swapped(chip->client, reg, val); + mutex_unlock(&chip->lock); if (ret < 0) - goto out; + return ret; - ret = 0; - break; + return 0; case IIO_CHAN_INFO_CALIBBIAS: - if (val < 0 || val > 0xFFFF) { - ret = -EINVAL; - goto out; - } + if (val < 0 || val > 0xFFFF) + return -EINVAL; + + mutex_lock(&chip->lock); ret = i2c_smbus_write_word_swapped(chip->client, AD7746_REG_CAP_OFFH, val); + mutex_unlock(&chip->lock); if (ret < 0) - goto out; + return ret; - ret = 0; - break; + return 0; case IIO_CHAN_INFO_OFFSET: - if (val < 0 || val > 43008000) { /* 21pF */ - ret = -EINVAL; - goto out; - } + case IIO_CHAN_INFO_ZEROPOINT: + if (val < 0 || val > 43008000) /* 21pF */ + return -EINVAL; /* * CAPDAC Scale = 21pF_typ / 127 @@ -479,42 +515,104 @@ static int ad7746_write_raw(struct iio_dev *indio_dev, */ val /= 338646; - + mutex_lock(&chip->lock); chip->capdac[chan->channel][chan->differential] = val > 0 ? - AD7746_CAPDAC_DACP(val) | AD7746_CAPDAC_DACEN : 0; + FIELD_PREP(AD7746_CAPDAC_DACP_MASK, val) | AD7746_CAPDAC_DACEN : 0; ret = ad7746_set_capdac(chip, chan->channel); - if (ret < 0) - goto out; + if (ret < 0) { + mutex_unlock(&chip->lock); + return ret; + } chip->capdac_set = chan->channel; + mutex_unlock(&chip->lock); - ret = 0; - break; + return 0; case IIO_CHAN_INFO_SAMP_FREQ: - if (val2) { - ret = -EINVAL; - goto out; - } + if (val2) + return -EINVAL; switch (chan->type) { case IIO_CAPACITANCE: + mutex_lock(&chip->lock); ret = ad7746_store_cap_filter_rate_setup(chip, val); - break; + mutex_unlock(&chip->lock); + return ret; case IIO_VOLTAGE: + mutex_lock(&chip->lock); ret = ad7746_store_vt_filter_rate_setup(chip, val); - break; + mutex_unlock(&chip->lock); + return ret; default: - ret = -EINVAL; + return -EINVAL; } + default: + return -EINVAL; + } +} + +static const int ad7746_v_samp_freq[] = { 50, 31, 16, 8, }; +static const int ad7746_cap_samp_freq[] = { 91, 84, 50, 26, 16, 13, 11, 9, }; + +static int ad7746_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, const int **vals, + int *type, int *length, long mask) +{ + if (mask != IIO_CHAN_INFO_SAMP_FREQ) + return -EINVAL; + + switch (chan->type) { + case IIO_VOLTAGE: + *vals = ad7746_v_samp_freq; + *length = ARRAY_SIZE(ad7746_v_samp_freq); + break; + case IIO_CAPACITANCE: + *vals = ad7746_cap_samp_freq; + *length = ARRAY_SIZE(ad7746_cap_samp_freq); break; default: - ret = -EINVAL; + return -EINVAL; } + *type = IIO_VAL_INT; + return IIO_AVAIL_LIST; +} -out: - mutex_unlock(&chip->lock); - return ret; +static int ad7746_read_channel(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val) +{ + struct ad7746_chip_info *chip = iio_priv(indio_dev); + int ret, delay; + u8 data[3]; + u8 regval; + + ret = ad7746_select_channel(indio_dev, chan); + if (ret < 0) + return ret; + delay = ret; + + regval = chip->config | FIELD_PREP(AD7746_CONF_MODE_MASK, + AD7746_CONF_MODE_SINGLE_CONV); + ret = i2c_smbus_write_byte_data(chip->client, AD7746_REG_CFG, regval); + if (ret < 0) + return ret; + + msleep(delay); + /* Now read the actual register */ + ret = i2c_smbus_read_i2c_block_data(chip->client, + ad7746_chan_info[chan->address].addr, + sizeof(data), data); + if (ret < 0) + return ret; + + /* + * Offset applied internally becaue the _offset userspace interface is + * needed for the CAP DACs which apply a controllable offset. + */ + *val = get_unaligned_be24(data) - 0x800000; + + return 0; } static int ad7746_read_raw(struct iio_dev *indio_dev, @@ -523,55 +621,18 @@ static int ad7746_read_raw(struct iio_dev *indio_dev, long mask) { struct ad7746_chip_info *chip = iio_priv(indio_dev); - int ret, delay, idx; - u8 regval, reg; - - mutex_lock(&chip->lock); + int ret, idx; + u8 reg; switch (mask) { case IIO_CHAN_INFO_RAW: - case IIO_CHAN_INFO_PROCESSED: - ret = ad7746_select_channel(indio_dev, chan); - if (ret < 0) - goto out; - delay = ret; - - regval = chip->config | AD7746_CONF_MODE_SINGLE_CONV; - ret = i2c_smbus_write_byte_data(chip->client, AD7746_REG_CFG, - regval); - if (ret < 0) - goto out; - - msleep(delay); - /* Now read the actual register */ - - ret = i2c_smbus_read_i2c_block_data(chip->client, - chan->address >> 8, 3, - &chip->data.d8[1]); - + mutex_lock(&chip->lock); + ret = ad7746_read_channel(indio_dev, chan, val); + mutex_unlock(&chip->lock); if (ret < 0) - goto out; - - *val = (be32_to_cpu(chip->data.d32) & 0xFFFFFF) - 0x800000; - - switch (chan->type) { - case IIO_TEMP: - /* - * temperature in milli degrees Celsius - * T = ((*val / 2048) - 4096) * 1000 - */ - *val = (*val * 125) / 256; - break; - case IIO_VOLTAGE: - if (chan->channel == 1) /* supply_raw*/ - *val = *val * 6; - break; - default: - break; - } + return ret; - ret = IIO_VAL_INT; - break; + return IIO_VAL_INT; case IIO_CHAN_INFO_CALIBSCALE: switch (chan->type) { case IIO_CAPACITANCE: @@ -581,83 +642,78 @@ static int ad7746_read_raw(struct iio_dev *indio_dev, reg = AD7746_REG_VOLT_GAINH; break; default: - ret = -EINVAL; - goto out; + return -EINVAL; } + mutex_lock(&chip->lock); ret = i2c_smbus_read_word_swapped(chip->client, reg); + mutex_unlock(&chip->lock); if (ret < 0) - goto out; + return ret; /* 1 + gain_val / 2^16 */ *val = 1; *val2 = (15625 * ret) / 1024; - ret = IIO_VAL_INT_PLUS_MICRO; - break; + return IIO_VAL_INT_PLUS_MICRO; case IIO_CHAN_INFO_CALIBBIAS: + mutex_lock(&chip->lock); ret = i2c_smbus_read_word_swapped(chip->client, AD7746_REG_CAP_OFFH); + mutex_unlock(&chip->lock); if (ret < 0) - goto out; + return ret; *val = ret; - ret = IIO_VAL_INT; - break; + return IIO_VAL_INT; case IIO_CHAN_INFO_OFFSET: - *val = AD7746_CAPDAC_DACP(chip->capdac[chan->channel] - [chan->differential]) * 338646; + case IIO_CHAN_INFO_ZEROPOINT: + *val = FIELD_GET(AD7746_CAPDAC_DACP_MASK, + chip->capdac[chan->channel][chan->differential]) * 338646; - ret = IIO_VAL_INT; - break; + return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: switch (chan->type) { case IIO_CAPACITANCE: /* 8.192pf / 2^24 */ *val = 0; *val2 = 488; - ret = IIO_VAL_INT_PLUS_NANO; - break; + return IIO_VAL_INT_PLUS_NANO; case IIO_VOLTAGE: /* 1170mV / 2^23 */ *val = 1170; + if (chan->channel == 1) + *val *= 6; *val2 = 23; - ret = IIO_VAL_FRACTIONAL_LOG2; - break; + return IIO_VAL_FRACTIONAL_LOG2; + case IIO_TEMP: + *val = 125; + *val2 = 8; + return IIO_VAL_FRACTIONAL_LOG2; default: - ret = -EINVAL; - break; + return -EINVAL; } - - break; case IIO_CHAN_INFO_SAMP_FREQ: switch (chan->type) { case IIO_CAPACITANCE: - idx = (chip->config & AD7746_CONF_CAPFS_MASK) >> - AD7746_CONF_CAPFS_SHIFT; + idx = FIELD_GET(AD7746_CONF_CAPFS_MASK, chip->config); *val = ad7746_cap_filter_rate_table[idx][0]; - ret = IIO_VAL_INT; - break; + return IIO_VAL_INT; case IIO_VOLTAGE: - idx = (chip->config & AD7746_CONF_VTFS_MASK) >> - AD7746_CONF_VTFS_SHIFT; + idx = FIELD_GET(AD7746_CONF_VTFS_MASK, chip->config); *val = ad7746_vt_filter_rate_table[idx][0]; - ret = IIO_VAL_INT; - break; + return IIO_VAL_INT; default: - ret = -EINVAL; + return -EINVAL; } - break; default: - ret = -EINVAL; + return -EINVAL; } -out: - mutex_unlock(&chip->lock); - return ret; } static const struct iio_info ad7746_info = { .attrs = &ad7746_attribute_group, .read_raw = ad7746_read_raw, + .read_avail = ad7746_read_avail, .write_raw = ad7746_write_raw, }; @@ -674,10 +730,9 @@ static int ad7746_probe(struct i2c_client *client, indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*chip)); if (!indio_dev) return -ENOMEM; + chip = iio_priv(indio_dev); mutex_init(&chip->lock); - /* this is only used for device removal purposes */ - i2c_set_clientdata(client, indio_dev); chip->client = client; chip->capdac_set = -1; @@ -710,24 +765,24 @@ static int ad7746_probe(struct i2c_client *client, if (!ret) { switch (vdd_permille) { case 125: - regval |= AD7746_EXCSETUP_EXCLVL(0); + regval |= FIELD_PREP(AD7746_EXCSETUP_EXCLVL_MASK, 0); break; case 250: - regval |= AD7746_EXCSETUP_EXCLVL(1); + regval |= FIELD_PREP(AD7746_EXCSETUP_EXCLVL_MASK, 1); break; case 375: - regval |= AD7746_EXCSETUP_EXCLVL(2); + regval |= FIELD_PREP(AD7746_EXCSETUP_EXCLVL_MASK, 2); break; case 500: - regval |= AD7746_EXCSETUP_EXCLVL(3); + regval |= FIELD_PREP(AD7746_EXCSETUP_EXCLVL_MASK, 3); break; default: break; } } - ret = i2c_smbus_write_byte_data(chip->client, - AD7746_REG_EXC_SETUP, regval); + ret = i2c_smbus_write_byte_data(chip->client, AD7746_REG_EXC_SETUP, + regval); if (ret < 0) return ret; @@ -740,7 +795,6 @@ static const struct i2c_device_id ad7746_id[] = { { "ad7747", 7747 }, {} }; - MODULE_DEVICE_TABLE(i2c, ad7746_id); static const struct of_device_id ad7746_of_match[] = { @@ -749,7 +803,6 @@ static const struct of_device_id ad7746_of_match[] = { { .compatible = "adi,ad7747" }, { }, }; - MODULE_DEVICE_TABLE(of, ad7746_of_match); static struct i2c_driver ad7746_driver = { diff --git a/drivers/iio/chemical/atlas-sensor.c b/drivers/iio/chemical/atlas-sensor.c index 8378c00fa2ff993ca7fd74388ab76485baaf7f4b..7cac77a931c70a89f911213bd55877df6b852813 100644 --- a/drivers/iio/chemical/atlas-sensor.c +++ b/drivers/iio/chemical/atlas-sensor.c @@ -722,7 +722,7 @@ unregister_trigger: return ret; } -static int atlas_remove(struct i2c_client *client) +static void atlas_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct atlas_data *data = iio_priv(indio_dev); @@ -739,8 +739,6 @@ static int atlas_remove(struct i2c_client *client) if (ret) dev_err(&client->dev, "Failed to power down device (%pe)\n", ERR_PTR(ret)); - - return 0; } static int atlas_runtime_suspend(struct device *dev) diff --git a/drivers/iio/chemical/ccs811.c b/drivers/iio/chemical/ccs811.c index 560183efb36f660fa3e5aaa04e52d9efbecca3d9..ba4045e20303585c8d6c3d7208d3472fb3321b34 100644 --- a/drivers/iio/chemical/ccs811.c +++ b/drivers/iio/chemical/ccs811.c @@ -532,7 +532,7 @@ err_poweroff: return ret; } -static int ccs811_remove(struct i2c_client *client) +static void ccs811_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ccs811_data *data = iio_priv(indio_dev); @@ -548,8 +548,6 @@ static int ccs811_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to power down device (%pe)\n", ERR_PTR(ret)); - - return 0; } static const struct i2c_device_id ccs811_id[] = { diff --git a/drivers/iio/chemical/sgp30.c b/drivers/iio/chemical/sgp30.c index 2343d444604d02b1cc7db85902da86300d4bf248..e2c13c78c7e0578d3bc6bac39e4a14e397768cfb 100644 --- a/drivers/iio/chemical/sgp30.c +++ b/drivers/iio/chemical/sgp30.c @@ -552,15 +552,13 @@ static int sgp_probe(struct i2c_client *client, return 0; } -static int sgp_remove(struct i2c_client *client) +static void sgp_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct sgp_data *data = iio_priv(indio_dev); if (data->iaq_thread) kthread_stop(data->iaq_thread); - - return 0; } static const struct i2c_device_id sgp_id[] = { diff --git a/drivers/iio/common/scmi_sensors/scmi_iio.c b/drivers/iio/common/scmi_sensors/scmi_iio.c index 793d628db55fbbc748c691f7169120d2437a11e1..54ccf19ab2bb1f892893b9dfb83c5240ffbc7c4b 100644 --- a/drivers/iio/common/scmi_sensors/scmi_iio.c +++ b/drivers/iio/common/scmi_sensors/scmi_iio.c @@ -18,6 +18,7 @@ #include #include #include +#include #define SCMI_IIO_NUM_OF_AXIS 3 @@ -130,7 +131,6 @@ static const struct iio_buffer_setup_ops scmi_iio_buffer_ops = { static int scmi_iio_set_odr_val(struct iio_dev *iio_dev, int val, int val2) { struct scmi_iio_priv *sensor = iio_priv(iio_dev); - const unsigned long UHZ_PER_HZ = 1000000UL; u64 sec, mult, uHz, sf; u32 sensor_config; char buf[32]; @@ -145,7 +145,7 @@ static int scmi_iio_set_odr_val(struct iio_dev *iio_dev, int val, int val2) return err; } - uHz = val * UHZ_PER_HZ + val2; + uHz = val * MICROHZ_PER_HZ + val2; /* * The seconds field in the sensor interval in SCMI is 16 bits long @@ -156,10 +156,10 @@ static int scmi_iio_set_odr_val(struct iio_dev *iio_dev, int val, int val2) * count the number of characters */ sf = (u64)uHz * 0xFFFF; - do_div(sf, UHZ_PER_HZ); + do_div(sf, MICROHZ_PER_HZ); mult = scnprintf(buf, sizeof(buf), "%llu", sf) - 1; - sec = int_pow(10, mult) * UHZ_PER_HZ; + sec = int_pow(10, mult) * MICROHZ_PER_HZ; do_div(sec, uHz); if (sec == 0) { dev_err(&iio_dev->dev, diff --git a/drivers/iio/common/st_sensors/st_sensors_core.c b/drivers/iio/common/st_sensors/st_sensors_core.c index 9910ba1da085818f81e97afbd756bb8abb129320..35720c64fea8f0b0be96d87da40ac085696318f1 100644 --- a/drivers/iio/common/st_sensors/st_sensors_core.c +++ b/drivers/iio/common/st_sensors/st_sensors_core.c @@ -354,7 +354,7 @@ void st_sensors_dev_name_probe(struct device *dev, char *name, int len) return; /* The name from the match takes precedence if present */ - strlcpy(name, match, len); + strscpy(name, match, len); } EXPORT_SYMBOL_NS(st_sensors_dev_name_probe, IIO_ST_SENSORS); diff --git a/drivers/iio/dac/ad5380.c b/drivers/iio/dac/ad5380.c index 81775152aac62fc340a6fd4809af50c5bb358b14..a81bfa47a221a8c775f40372e5e73715b8caa5a2 100644 --- a/drivers/iio/dac/ad5380.c +++ b/drivers/iio/dac/ad5380.c @@ -559,11 +559,9 @@ static int ad5380_i2c_probe(struct i2c_client *i2c, return ad5380_probe(&i2c->dev, regmap, id->driver_data, id->name); } -static int ad5380_i2c_remove(struct i2c_client *i2c) +static void ad5380_i2c_remove(struct i2c_client *i2c) { ad5380_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id ad5380_i2c_ids[] = { diff --git a/drivers/iio/dac/ad5446.c b/drivers/iio/dac/ad5446.c index 09e242949cd06c813ffea03a0a35fc61b2b81663..7324065d3782173b05c344cef0e53b0a867e1cc6 100644 --- a/drivers/iio/dac/ad5446.c +++ b/drivers/iio/dac/ad5446.c @@ -575,11 +575,9 @@ static int ad5446_i2c_probe(struct i2c_client *i2c, &ad5446_i2c_chip_info[id->driver_data]); } -static int ad5446_i2c_remove(struct i2c_client *i2c) +static void ad5446_i2c_remove(struct i2c_client *i2c) { ad5446_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id ad5446_i2c_ids[] = { diff --git a/drivers/iio/dac/ad5593r.c b/drivers/iio/dac/ad5593r.c index 34e1319a971262645ed1153be497c6092b5f62b9..8e5e014e0c28b1b207db462de8b6e390657e1784 100644 --- a/drivers/iio/dac/ad5593r.c +++ b/drivers/iio/dac/ad5593r.c @@ -13,6 +13,8 @@ #include #include +#include + #define AD5593R_MODE_CONF (0 << 4) #define AD5593R_MODE_DAC_WRITE (1 << 4) #define AD5593R_MODE_ADC_READBACK (4 << 4) @@ -20,6 +22,24 @@ #define AD5593R_MODE_GPIO_READBACK (6 << 4) #define AD5593R_MODE_REG_READBACK (7 << 4) +static int ad5593r_read_word(struct i2c_client *i2c, u8 reg, u16 *value) +{ + int ret; + u8 buf[2]; + + ret = i2c_smbus_write_byte(i2c, reg); + if (ret < 0) + return ret; + + ret = i2c_master_recv(i2c, buf, sizeof(buf)); + if (ret < 0) + return ret; + + *value = get_unaligned_be16(buf); + + return 0; +} + static int ad5593r_write_dac(struct ad5592r_state *st, unsigned chan, u16 value) { struct i2c_client *i2c = to_i2c_client(st->dev); @@ -38,13 +58,7 @@ static int ad5593r_read_adc(struct ad5592r_state *st, unsigned chan, u16 *value) if (val < 0) return (int) val; - val = i2c_smbus_read_word_swapped(i2c, AD5593R_MODE_ADC_READBACK); - if (val < 0) - return (int) val; - - *value = (u16) val; - - return 0; + return ad5593r_read_word(i2c, AD5593R_MODE_ADC_READBACK, value); } static int ad5593r_reg_write(struct ad5592r_state *st, u8 reg, u16 value) @@ -58,25 +72,19 @@ static int ad5593r_reg_write(struct ad5592r_state *st, u8 reg, u16 value) static int ad5593r_reg_read(struct ad5592r_state *st, u8 reg, u16 *value) { struct i2c_client *i2c = to_i2c_client(st->dev); - s32 val; - val = i2c_smbus_read_word_swapped(i2c, AD5593R_MODE_REG_READBACK | reg); - if (val < 0) - return (int) val; - - *value = (u16) val; - - return 0; + return ad5593r_read_word(i2c, AD5593R_MODE_REG_READBACK | reg, value); } static int ad5593r_gpio_read(struct ad5592r_state *st, u8 *value) { struct i2c_client *i2c = to_i2c_client(st->dev); - s32 val; + u16 val; + int ret; - val = i2c_smbus_read_word_swapped(i2c, AD5593R_MODE_GPIO_READBACK); - if (val < 0) - return (int) val; + ret = ad5593r_read_word(i2c, AD5593R_MODE_GPIO_READBACK, &val); + if (ret) + return ret; *value = (u8) val; @@ -94,14 +102,16 @@ static const struct ad5592r_rw_ops ad5593r_rw_ops = { static int ad5593r_i2c_probe(struct i2c_client *i2c, const struct i2c_device_id *id) { + if (!i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_BYTE | I2C_FUNC_I2C)) + return -EOPNOTSUPP; + return ad5592r_probe(&i2c->dev, id->name, &ad5593r_rw_ops); } -static int ad5593r_i2c_remove(struct i2c_client *i2c) +static void ad5593r_i2c_remove(struct i2c_client *i2c) { ad5592r_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id ad5593r_i2c_ids[] = { diff --git a/drivers/iio/dac/ad5696-i2c.c b/drivers/iio/dac/ad5696-i2c.c index 762503c1901bedd081f1d806e9a2787ee6570df6..aa36cbf0137c1f7ef33d8d9f73af974435514313 100644 --- a/drivers/iio/dac/ad5696-i2c.c +++ b/drivers/iio/dac/ad5696-i2c.c @@ -65,11 +65,9 @@ static int ad5686_i2c_probe(struct i2c_client *i2c, ad5686_i2c_write, ad5686_i2c_read); } -static int ad5686_i2c_remove(struct i2c_client *i2c) +static void ad5686_i2c_remove(struct i2c_client *i2c) { ad5686_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id ad5686_i2c_id[] = { diff --git a/drivers/iio/dac/ds4424.c b/drivers/iio/dac/ds4424.c index 509394690bcc1d0eebd26bbb0f6bb8c9c3598feb..3e17a681174e8a91ff012d860ab3cae043a4bf49 100644 --- a/drivers/iio/dac/ds4424.c +++ b/drivers/iio/dac/ds4424.c @@ -281,15 +281,13 @@ fail: return ret; } -static int ds4424_remove(struct i2c_client *client) +static void ds4424_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ds4424_data *data = iio_priv(indio_dev); iio_device_unregister(indio_dev); regulator_disable(data->vcc_reg); - - return 0; } static const struct i2c_device_id ds4424_id[] = { diff --git a/drivers/iio/dac/m62332.c b/drivers/iio/dac/m62332.c index 22b02f50fe411a722423e994b945f0e6f4c5e878..5a812f87970c02a2c3ed0aa0445fb8a9c90d5d66 100644 --- a/drivers/iio/dac/m62332.c +++ b/drivers/iio/dac/m62332.c @@ -218,7 +218,7 @@ err: return ret; } -static int m62332_remove(struct i2c_client *client) +static void m62332_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -226,8 +226,6 @@ static int m62332_remove(struct i2c_client *client) iio_map_array_unregister(indio_dev); m62332_set_value(indio_dev, 0, 0); m62332_set_value(indio_dev, 0, 1); - - return 0; } static const struct i2c_device_id m62332_id[] = { diff --git a/drivers/iio/dac/mcp4725.c b/drivers/iio/dac/mcp4725.c index bb4b85a7b95b6f1fe38b6f694c61bf85bc876129..446d1a8fe4bef5fb20b3607ffafcc57f9b9656ee 100644 --- a/drivers/iio/dac/mcp4725.c +++ b/drivers/iio/dac/mcp4725.c @@ -486,7 +486,7 @@ err_disable_vdd_reg: return err; } -static int mcp4725_remove(struct i2c_client *client) +static void mcp4725_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mcp4725_data *data = iio_priv(indio_dev); @@ -496,8 +496,6 @@ static int mcp4725_remove(struct i2c_client *client) if (data->vref_reg) regulator_disable(data->vref_reg); regulator_disable(data->vdd_reg); - - return 0; } static const struct i2c_device_id mcp4725_id[] = { diff --git a/drivers/iio/dac/ti-dac5571.c b/drivers/iio/dac/ti-dac5571.c index f91f8a50498953eaa9404ec7da48ed64cff53ceb..3210e3098f9ae277fd69d734fd5e324a22efcfdb 100644 --- a/drivers/iio/dac/ti-dac5571.c +++ b/drivers/iio/dac/ti-dac5571.c @@ -382,15 +382,13 @@ static int dac5571_probe(struct i2c_client *client, return ret; } -static int dac5571_remove(struct i2c_client *i2c) +static void dac5571_remove(struct i2c_client *i2c) { struct iio_dev *indio_dev = i2c_get_clientdata(i2c); struct dac5571_data *data = iio_priv(indio_dev); iio_device_unregister(indio_dev); regulator_disable(data->vref); - - return 0; } static const struct of_device_id dac5571_of_id[] = { diff --git a/drivers/iio/frequency/adf4371.c b/drivers/iio/frequency/adf4371.c index 135c8cedc33dcf945447045ff25d4ea12700bf68..b270884648265ab9e84832bf030651e45b51db47 100644 --- a/drivers/iio/frequency/adf4371.c +++ b/drivers/iio/frequency/adf4371.c @@ -540,13 +540,6 @@ static int adf4371_setup(struct adf4371_state *st) return regmap_bulk_write(st->regmap, ADF4371_REG(0x30), st->buf, 5); } -static void adf4371_clk_disable(void *data) -{ - struct adf4371_state *st = data; - - clk_disable_unprepare(st->clkin); -} - static int adf4371_probe(struct spi_device *spi) { const struct spi_device_id *id = spi_get_device_id(spi); @@ -579,18 +572,10 @@ static int adf4371_probe(struct spi_device *spi) indio_dev->channels = st->chip_info->channels; indio_dev->num_channels = st->chip_info->num_channels; - st->clkin = devm_clk_get(&spi->dev, "clkin"); + st->clkin = devm_clk_get_enabled(&spi->dev, "clkin"); if (IS_ERR(st->clkin)) return PTR_ERR(st->clkin); - ret = clk_prepare_enable(st->clkin); - if (ret < 0) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, adf4371_clk_disable, st); - if (ret) - return ret; - st->clkin_freq = clk_get_rate(st->clkin); ret = adf4371_setup(st); diff --git a/drivers/iio/frequency/admv1014.c b/drivers/iio/frequency/admv1014.c index 865addd10db44dc952ba8c872c85b7774578c8c6..bb5e1feef42bfc09fac9955613c0c154ab1fe44a 100644 --- a/drivers/iio/frequency/admv1014.c +++ b/drivers/iio/frequency/admv1014.c @@ -669,8 +669,7 @@ static int admv1014_init(struct admv1014_state *st) chip_id = FIELD_GET(ADMV1014_CHIP_ID_MSK, chip_id); if (chip_id != ADMV1014_CHIP_ID) { dev_err(&spi->dev, "Invalid Chip ID.\n"); - ret = -EINVAL; - return ret; + return -EINVAL; } ret = __admv1014_spi_update_bits(st, ADMV1014_REG_QUAD, diff --git a/drivers/iio/frequency/adrf6780.c b/drivers/iio/frequency/adrf6780.c index 21878bad0909786a5c16739ff4c84e19b448a7a2..b4defb82f37e3081564c022f15c570c27b43f542 100644 --- a/drivers/iio/frequency/adrf6780.c +++ b/drivers/iio/frequency/adrf6780.c @@ -441,11 +441,6 @@ static void adrf6780_properties_parse(struct adrf6780_state *st) st->vdet_out_en = device_property_read_bool(&spi->dev, "adi,vdet-out-en"); } -static void adrf6780_clk_disable(void *data) -{ - clk_disable_unprepare(data); -} - static void adrf6780_powerdown(void *data) { /* Disable all components in the Enable Register */ @@ -473,20 +468,11 @@ static int adrf6780_probe(struct spi_device *spi) adrf6780_properties_parse(st); - st->clkin = devm_clk_get(&spi->dev, "lo_in"); + st->clkin = devm_clk_get_enabled(&spi->dev, "lo_in"); if (IS_ERR(st->clkin)) return dev_err_probe(&spi->dev, PTR_ERR(st->clkin), "failed to get the LO input clock\n"); - ret = clk_prepare_enable(st->clkin); - if (ret) - return ret; - - ret = devm_add_action_or_reset(&spi->dev, adrf6780_clk_disable, - st->clkin); - if (ret) - return ret; - mutex_init(&st->lock); ret = adrf6780_init(st); diff --git a/drivers/iio/gyro/bmg160_i2c.c b/drivers/iio/gyro/bmg160_i2c.c index b3fa46bd02cb89898dde70fe5600cf696eb0a9b0..908ccc385254a8a7ad2d8403cab69f6f81d5f087 100644 --- a/drivers/iio/gyro/bmg160_i2c.c +++ b/drivers/iio/gyro/bmg160_i2c.c @@ -32,11 +32,9 @@ static int bmg160_i2c_probe(struct i2c_client *client, return bmg160_core_probe(&client->dev, regmap, client->irq, name); } -static int bmg160_i2c_remove(struct i2c_client *client) +static void bmg160_i2c_remove(struct i2c_client *client) { bmg160_core_remove(&client->dev); - - return 0; } static const struct acpi_device_id bmg160_acpi_match[] = { diff --git a/drivers/iio/gyro/fxas21002c_i2c.c b/drivers/iio/gyro/fxas21002c_i2c.c index a7807fd97483528b6cda8b3d2d4c54ad3233cf38..13bb52c594d12cfc2a10a70ccbdf2e8c4b25da4c 100644 --- a/drivers/iio/gyro/fxas21002c_i2c.c +++ b/drivers/iio/gyro/fxas21002c_i2c.c @@ -33,11 +33,9 @@ static int fxas21002c_i2c_probe(struct i2c_client *i2c) return fxas21002c_core_probe(&i2c->dev, regmap, i2c->irq, i2c->name); } -static int fxas21002c_i2c_remove(struct i2c_client *i2c) +static void fxas21002c_i2c_remove(struct i2c_client *i2c) { fxas21002c_core_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id fxas21002c_i2c_id[] = { diff --git a/drivers/iio/gyro/itg3200_core.c b/drivers/iio/gyro/itg3200_core.c index 0491c64e1b3295fabf2929cf0cc9f8138dee2942..4215015845875131d3e535ac14633ad5cc27f2bf 100644 --- a/drivers/iio/gyro/itg3200_core.c +++ b/drivers/iio/gyro/itg3200_core.c @@ -350,7 +350,7 @@ error_unconfigure_buffer: return ret; } -static int itg3200_remove(struct i2c_client *client) +static void itg3200_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -360,8 +360,6 @@ static int itg3200_remove(struct i2c_client *client) itg3200_remove_trigger(indio_dev); itg3200_buffer_unconfigure(indio_dev); - - return 0; } static int itg3200_suspend(struct device *dev) diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c index 78f4a0102986ecd36cf4d9576e3078a7454d2b09..12e3afa9dd11dec7605b70671b026e6f11a42268 100644 --- a/drivers/iio/gyro/mpu3050-i2c.c +++ b/drivers/iio/gyro/mpu3050-i2c.c @@ -78,7 +78,7 @@ static int mpu3050_i2c_probe(struct i2c_client *client, return 0; } -static int mpu3050_i2c_remove(struct i2c_client *client) +static void mpu3050_i2c_remove(struct i2c_client *client) { struct iio_dev *indio_dev = dev_get_drvdata(&client->dev); struct mpu3050 *mpu3050 = iio_priv(indio_dev); @@ -87,8 +87,6 @@ static int mpu3050_i2c_remove(struct i2c_client *client) i2c_mux_del_adapters(mpu3050->i2cmux); mpu3050_common_remove(&client->dev); - - return 0; } /* diff --git a/drivers/iio/health/afe4404.c b/drivers/iio/health/afe4404.c index dd7800159051ae990121f26d550c3a3610030ca8..8fca787b25245ff6712fcebad517cce2356b4bbd 100644 --- a/drivers/iio/health/afe4404.c +++ b/drivers/iio/health/afe4404.c @@ -578,7 +578,7 @@ disable_reg: return ret; } -static int afe4404_remove(struct i2c_client *client) +static void afe4404_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct afe4404_data *afe = iio_priv(indio_dev); @@ -594,8 +594,6 @@ static int afe4404_remove(struct i2c_client *client) ret = regulator_disable(afe->regulator); if (ret) dev_err(afe->dev, "Unable to disable regulator\n"); - - return 0; } static const struct i2c_device_id afe4404_ids[] = { diff --git a/drivers/iio/health/max30100.c b/drivers/iio/health/max30100.c index ad571796522318d2064c21a6cec5cd0644558b36..2cca5e0519f89d5f5a2199014bbe59302a8a96d5 100644 --- a/drivers/iio/health/max30100.c +++ b/drivers/iio/health/max30100.c @@ -471,15 +471,13 @@ static int max30100_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int max30100_remove(struct i2c_client *client) +static void max30100_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct max30100_data *data = iio_priv(indio_dev); iio_device_unregister(indio_dev); max30100_set_powermode(data, false); - - return 0; } static const struct i2c_device_id max30100_id[] = { diff --git a/drivers/iio/health/max30102.c b/drivers/iio/health/max30102.c index abbcef56380786273bed2e9e7a2fb2807aeda95b..437298a29f2d323dcad83ece6438cb857a0d5d58 100644 --- a/drivers/iio/health/max30102.c +++ b/drivers/iio/health/max30102.c @@ -592,15 +592,13 @@ static int max30102_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int max30102_remove(struct i2c_client *client) +static void max30102_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct max30102_data *data = iio_priv(indio_dev); iio_device_unregister(indio_dev); max30102_set_power(data, false); - - return 0; } static const struct i2c_device_id max30102_id[] = { diff --git a/drivers/iio/humidity/hdc2010.c b/drivers/iio/humidity/hdc2010.c index 1381df46187c8cdba064651a55cd57f34ee5aac1..d6858ccb056e4eeaadfb796e83f43ea6c57970cd 100644 --- a/drivers/iio/humidity/hdc2010.c +++ b/drivers/iio/humidity/hdc2010.c @@ -308,7 +308,7 @@ static int hdc2010_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int hdc2010_remove(struct i2c_client *client) +static void hdc2010_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct hdc2010_data *data = iio_priv(indio_dev); @@ -318,8 +318,6 @@ static int hdc2010_remove(struct i2c_client *client) /* Disable Automatic Measurement Mode */ if (hdc2010_update_drdy_config(data, HDC2010_AMM, 0)) dev_warn(&client->dev, "Unable to restore default AMM\n"); - - return 0; } static const struct i2c_device_id hdc2010_id[] = { diff --git a/drivers/iio/imu/Kconfig b/drivers/iio/imu/Kconfig index 001ca2c3ff953db91628e7c33073699edca87245..f1d7d4b5e22243134967418b491fbf2cc5ccda47 100644 --- a/drivers/iio/imu/Kconfig +++ b/drivers/iio/imu/Kconfig @@ -52,6 +52,7 @@ config ADIS16480 ADIS16485, ADIS16488 inertial sensors. source "drivers/iio/imu/bmi160/Kconfig" +source "drivers/iio/imu/bno055/Kconfig" config FXOS8700 tristate diff --git a/drivers/iio/imu/Makefile b/drivers/iio/imu/Makefile index c82748096c7743f6ae93dcfbdc42881bd922216a..6eb612034722298439c2e0721c08c7c347b067be 100644 --- a/drivers/iio/imu/Makefile +++ b/drivers/iio/imu/Makefile @@ -15,6 +15,7 @@ adis_lib-$(CONFIG_IIO_ADIS_LIB_BUFFER) += adis_buffer.o obj-$(CONFIG_IIO_ADIS_LIB) += adis_lib.o obj-y += bmi160/ +obj-y += bno055/ obj-$(CONFIG_FXOS8700) += fxos8700_core.o obj-$(CONFIG_FXOS8700_I2C) += fxos8700_i2c.o diff --git a/drivers/iio/imu/adis16475.c b/drivers/iio/imu/adis16475.c index ff2b0fab840a4c4b8d86618d6ca6ba3645438104..aec55f7e1f26023f210b2b8fe4fa7fbb33b1abc7 100644 --- a/drivers/iio/imu/adis16475.c +++ b/drivers/iio/imu/adis16475.c @@ -1120,11 +1120,6 @@ check_burst32: return IRQ_HANDLED; } -static void adis16475_disable_clk(void *data) -{ - clk_disable_unprepare((struct clk *)data); -} - static int adis16475_config_sync_mode(struct adis16475 *st) { int ret; @@ -1150,19 +1145,11 @@ static int adis16475_config_sync_mode(struct adis16475 *st) /* All the other modes require external input signal */ if (sync->sync_mode != ADIS16475_SYNC_OUTPUT) { - struct clk *clk = devm_clk_get(dev, NULL); + struct clk *clk = devm_clk_get_enabled(dev, NULL); if (IS_ERR(clk)) return PTR_ERR(clk); - ret = clk_prepare_enable(clk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(dev, adis16475_disable_clk, clk); - if (ret) - return ret; - st->clk_freq = clk_get_rate(clk); if (st->clk_freq < sync->min_rate || st->clk_freq > sync->max_rate) { diff --git a/drivers/iio/imu/bno055/Kconfig b/drivers/iio/imu/bno055/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..fa79b1ac4f85b7879f7202d9787eb9253a4a2b90 --- /dev/null +++ b/drivers/iio/imu/bno055/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0 + +config BOSCH_BNO055 + tristate + +config BOSCH_BNO055_SERIAL + tristate "Bosch BNO055 attached via UART" + depends on SERIAL_DEV_BUS + select BOSCH_BNO055 + help + Enable this to support Bosch BNO055 IMUs attached via UART. + + This driver can also be built as a module. If so, the module will be + called bno055_sl. + +config BOSCH_BNO055_I2C + tristate "Bosch BNO055 attached via I2C bus" + depends on I2C + select REGMAP_I2C + select BOSCH_BNO055 + help + Enable this to support Bosch BNO055 IMUs attached via I2C bus. + + This driver can also be built as a module. If so, the module will be + called bno055_i2c. diff --git a/drivers/iio/imu/bno055/Makefile b/drivers/iio/imu/bno055/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..98c624730dae4a99a5eadb6affde0a025eb30057 --- /dev/null +++ b/drivers/iio/imu/bno055/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_BOSCH_BNO055) += bno055.o +obj-$(CONFIG_BOSCH_BNO055_SERIAL) += bno055_ser.o +bno055_ser-y := bno055_ser_core.o +# define_trace.h needs to know how to find our header +CFLAGS_bno055_ser_trace.o := -I$(src) +bno055_ser-$(CONFIG_TRACING) += bno055_ser_trace.o + +obj-$(CONFIG_BOSCH_BNO055_I2C) += bno055_i2c.o diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c new file mode 100644 index 0000000000000000000000000000000000000000..307557a609e3f0b099be87d5f38f53e994b07e45 --- /dev/null +++ b/drivers/iio/imu/bno055/bno055.c @@ -0,0 +1,1685 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * IIO driver for Bosch BNO055 IMU + * + * Copyright (C) 2021-2022 Istituto Italiano di Tecnologia + * Electronic Design Laboratory + * Written by Andrea Merello + * + * Portions of this driver are taken from the BNO055 driver patch + * from Vlad Dogaru which is Copyright (c) 2016, Intel Corporation. + * + * This driver is also based on BMI160 driver, which is: + * Copyright (c) 2016, Intel Corporation. + * Copyright (c) 2019, Martin Kelly. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "bno055.h" + +#define BNO055_FW_UID_FMT "bno055-caldata-%*phN.dat" +#define BNO055_FW_GENERIC_NAME "bno055-caldata.dat" + +/* common registers */ +#define BNO055_PAGESEL_REG 0x7 + +/* page 0 registers */ +#define BNO055_CHIP_ID_REG 0x0 +#define BNO055_CHIP_ID_MAGIC 0xA0 +#define BNO055_SW_REV_LSB_REG 0x4 +#define BNO055_SW_REV_MSB_REG 0x5 +#define BNO055_ACC_DATA_X_LSB_REG 0x8 +#define BNO055_ACC_DATA_Y_LSB_REG 0xA +#define BNO055_ACC_DATA_Z_LSB_REG 0xC +#define BNO055_MAG_DATA_X_LSB_REG 0xE +#define BNO055_MAG_DATA_Y_LSB_REG 0x10 +#define BNO055_MAG_DATA_Z_LSB_REG 0x12 +#define BNO055_GYR_DATA_X_LSB_REG 0x14 +#define BNO055_GYR_DATA_Y_LSB_REG 0x16 +#define BNO055_GYR_DATA_Z_LSB_REG 0x18 +#define BNO055_EUL_DATA_X_LSB_REG 0x1A +#define BNO055_EUL_DATA_Y_LSB_REG 0x1C +#define BNO055_EUL_DATA_Z_LSB_REG 0x1E +#define BNO055_QUAT_DATA_W_LSB_REG 0x20 +#define BNO055_LIA_DATA_X_LSB_REG 0x28 +#define BNO055_LIA_DATA_Y_LSB_REG 0x2A +#define BNO055_LIA_DATA_Z_LSB_REG 0x2C +#define BNO055_GRAVITY_DATA_X_LSB_REG 0x2E +#define BNO055_GRAVITY_DATA_Y_LSB_REG 0x30 +#define BNO055_GRAVITY_DATA_Z_LSB_REG 0x32 +#define BNO055_SCAN_CH_COUNT ((BNO055_GRAVITY_DATA_Z_LSB_REG - BNO055_ACC_DATA_X_LSB_REG) / 2) +#define BNO055_TEMP_REG 0x34 +#define BNO055_CALIB_STAT_REG 0x35 +#define BNO055_CALIB_STAT_MAGN_SHIFT 0 +#define BNO055_CALIB_STAT_ACCEL_SHIFT 2 +#define BNO055_CALIB_STAT_GYRO_SHIFT 4 +#define BNO055_CALIB_STAT_SYS_SHIFT 6 +#define BNO055_SYS_ERR_REG 0x3A +#define BNO055_POWER_MODE_REG 0x3E +#define BNO055_POWER_MODE_NORMAL 0 +#define BNO055_SYS_TRIGGER_REG 0x3F +#define BNO055_SYS_TRIGGER_RST_SYS BIT(5) +#define BNO055_SYS_TRIGGER_CLK_SEL BIT(7) +#define BNO055_OPR_MODE_REG 0x3D +#define BNO055_OPR_MODE_CONFIG 0x0 +#define BNO055_OPR_MODE_AMG 0x7 +#define BNO055_OPR_MODE_FUSION_FMC_OFF 0xB +#define BNO055_OPR_MODE_FUSION 0xC +#define BNO055_UNIT_SEL_REG 0x3B +/* Android orientation mode means: pitch value decreases turning clockwise */ +#define BNO055_UNIT_SEL_ANDROID BIT(7) +#define BNO055_UNIT_SEL_GYR_RPS BIT(1) +#define BNO055_CALDATA_START 0x55 +#define BNO055_CALDATA_END 0x6A +#define BNO055_CALDATA_LEN 22 + +/* + * The difference in address between the register that contains the + * value and the register that contains the offset. This applies for + * accel, gyro and magn channels. + */ +#define BNO055_REG_OFFSET_ADDR 0x4D + +/* page 1 registers */ +#define BNO055_PG1(x) ((x) | 0x80) +#define BNO055_ACC_CONFIG_REG BNO055_PG1(0x8) +#define BNO055_ACC_CONFIG_LPF_MASK GENMASK(4, 2) +#define BNO055_ACC_CONFIG_RANGE_MASK GENMASK(1, 0) +#define BNO055_MAG_CONFIG_REG BNO055_PG1(0x9) +#define BNO055_MAG_CONFIG_HIGHACCURACY 0x18 +#define BNO055_MAG_CONFIG_ODR_MASK GENMASK(2, 0) +#define BNO055_GYR_CONFIG_REG BNO055_PG1(0xA) +#define BNO055_GYR_CONFIG_RANGE_MASK GENMASK(2, 0) +#define BNO055_GYR_CONFIG_LPF_MASK GENMASK(5, 3) +#define BNO055_GYR_AM_SET_REG BNO055_PG1(0x1F) +#define BNO055_UID_LOWER_REG BNO055_PG1(0x50) +#define BNO055_UID_HIGHER_REG BNO055_PG1(0x5F) +#define BNO055_UID_LEN 16 + +struct bno055_sysfs_attr { + int *vals; + int len; + int *fusion_vals; + int *hw_xlate; + int type; +}; + +static int bno055_acc_lpf_vals[] = { + 7, 810000, 15, 630000, 31, 250000, 62, 500000, + 125, 0, 250, 0, 500, 0, 1000, 0, +}; + +static struct bno055_sysfs_attr bno055_acc_lpf = { + .vals = bno055_acc_lpf_vals, + .len = ARRAY_SIZE(bno055_acc_lpf_vals), + .fusion_vals = (int[]){62, 500000}, + .type = IIO_VAL_INT_PLUS_MICRO, +}; + +static int bno055_acc_range_vals[] = { + /* G: 2, 4, 8, 16 */ + 1962, 3924, 7848, 15696 +}; + +static struct bno055_sysfs_attr bno055_acc_range = { + .vals = bno055_acc_range_vals, + .len = ARRAY_SIZE(bno055_acc_range_vals), + .fusion_vals = (int[]){3924}, /* 4G */ + .type = IIO_VAL_INT, +}; + +/* + * Theoretically the IMU should return data in a given (i.e. fixed) unit + * regardless of the range setting. This happens for the accelerometer, but not + * for the gyroscope; the gyroscope range setting affects the scale. + * This is probably due to this[0] bug. + * For this reason we map the internal range setting onto the standard IIO scale + * attribute for gyro. + * Since the bug[0] may be fixed in future, we check for the IMU FW version and + * eventually warn the user. + * Currently we just don't care about "range" attributes for gyro. + * + * [0] https://community.bosch-sensortec.com/t5/MEMS-sensors-forum/BNO055-Wrong-sensitivity-resolution-in-datasheet/td-p/10266 + */ + +/* + * dps = hwval * (dps_range/2^15) + * rps = hwval * (rps_range/2^15) + * = hwval * (dps_range/(2^15 * k)) + * where k is rad-to-deg factor + */ +static int bno055_gyr_scale_vals[] = { + 125, 1877467, 250, 1877467, 500, 1877467, + 1000, 1877467, 2000, 1877467, +}; + +static struct bno055_sysfs_attr bno055_gyr_scale = { + .vals = bno055_gyr_scale_vals, + .len = ARRAY_SIZE(bno055_gyr_scale_vals), + .fusion_vals = (int[]){1, 900}, + .hw_xlate = (int[]){4, 3, 2, 1, 0}, + .type = IIO_VAL_FRACTIONAL, +}; + +static int bno055_gyr_lpf_vals[] = {12, 23, 32, 47, 64, 116, 230, 523}; +static struct bno055_sysfs_attr bno055_gyr_lpf = { + .vals = bno055_gyr_lpf_vals, + .len = ARRAY_SIZE(bno055_gyr_lpf_vals), + .fusion_vals = (int[]){32}, + .hw_xlate = (int[]){5, 4, 7, 3, 6, 2, 1, 0}, + .type = IIO_VAL_INT, +}; + +static int bno055_mag_odr_vals[] = {2, 6, 8, 10, 15, 20, 25, 30}; +static struct bno055_sysfs_attr bno055_mag_odr = { + .vals = bno055_mag_odr_vals, + .len = ARRAY_SIZE(bno055_mag_odr_vals), + .fusion_vals = (int[]){20}, + .type = IIO_VAL_INT, +}; + +struct bno055_priv { + struct regmap *regmap; + struct device *dev; + struct clk *clk; + int operation_mode; + int xfer_burst_break_thr; + struct mutex lock; + u8 uid[BNO055_UID_LEN]; + struct gpio_desc *reset_gpio; + bool sw_reset; + struct { + __le16 chans[BNO055_SCAN_CH_COUNT]; + s64 timestamp __aligned(8); + } buf; + struct dentry *debugfs; +}; + +static bool bno055_regmap_volatile(struct device *dev, unsigned int reg) +{ + /* data and status registers */ + if (reg >= BNO055_ACC_DATA_X_LSB_REG && reg <= BNO055_SYS_ERR_REG) + return true; + + /* when in fusion mode, config is updated by chip */ + if (reg == BNO055_MAG_CONFIG_REG || + reg == BNO055_ACC_CONFIG_REG || + reg == BNO055_GYR_CONFIG_REG) + return true; + + /* calibration data may be updated by the IMU */ + if (reg >= BNO055_CALDATA_START && reg <= BNO055_CALDATA_END) + return true; + + return false; +} + +static bool bno055_regmap_readable(struct device *dev, unsigned int reg) +{ + /* unnamed PG0 reserved areas */ + if ((reg < BNO055_PG1(0) && reg > BNO055_CALDATA_END) || + reg == 0x3C) + return false; + + /* unnamed PG1 reserved areas */ + if (reg > BNO055_PG1(BNO055_UID_HIGHER_REG) || + (reg < BNO055_PG1(BNO055_UID_LOWER_REG) && reg > BNO055_PG1(BNO055_GYR_AM_SET_REG)) || + reg == BNO055_PG1(0xE) || + (reg < BNO055_PG1(BNO055_PAGESEL_REG) && reg >= BNO055_PG1(0x0))) + return false; + return true; +} + +static bool bno055_regmap_writeable(struct device *dev, unsigned int reg) +{ + /* + * Unreadable registers are indeed reserved; there are no WO regs + * (except for a single bit in SYS_TRIGGER register) + */ + if (!bno055_regmap_readable(dev, reg)) + return false; + + /* data and status registers */ + if (reg >= BNO055_ACC_DATA_X_LSB_REG && reg <= BNO055_SYS_ERR_REG) + return false; + + /* ID areas */ + if (reg < BNO055_PAGESEL_REG || + (reg <= BNO055_UID_HIGHER_REG && reg >= BNO055_UID_LOWER_REG)) + return false; + + return true; +} + +static const struct regmap_range_cfg bno055_regmap_ranges[] = { + { + .range_min = 0, + .range_max = 0x7f * 2, + .selector_reg = BNO055_PAGESEL_REG, + .selector_mask = GENMASK(7, 0), + .selector_shift = 0, + .window_start = 0, + .window_len = 0x80, + }, +}; + +const struct regmap_config bno055_regmap_config = { + .name = "bno055", + .reg_bits = 8, + .val_bits = 8, + .ranges = bno055_regmap_ranges, + .num_ranges = 1, + .volatile_reg = bno055_regmap_volatile, + .max_register = 0x80 * 2, + .writeable_reg = bno055_regmap_writeable, + .readable_reg = bno055_regmap_readable, + .cache_type = REGCACHE_RBTREE, +}; +EXPORT_SYMBOL_NS_GPL(bno055_regmap_config, IIO_BNO055); + +/* must be called in configuration mode */ +static int bno055_calibration_load(struct bno055_priv *priv, const u8 *data, int len) +{ + if (len != BNO055_CALDATA_LEN) { + dev_dbg(priv->dev, "Invalid calibration file size %d (expected %d)", + len, BNO055_CALDATA_LEN); + return -EINVAL; + } + + dev_dbg(priv->dev, "loading cal data: %*ph", BNO055_CALDATA_LEN, data); + return regmap_bulk_write(priv->regmap, BNO055_CALDATA_START, + data, BNO055_CALDATA_LEN); +} + +static int bno055_operation_mode_do_set(struct bno055_priv *priv, + int operation_mode) +{ + int ret; + + ret = regmap_write(priv->regmap, BNO055_OPR_MODE_REG, + operation_mode); + if (ret) + return ret; + + /* Following datasheet specifications: sensor takes 7mS up to 19 mS to switch mode */ + msleep(20); + + return 0; +} + +static int bno055_system_reset(struct bno055_priv *priv) +{ + int ret; + + if (priv->reset_gpio) { + gpiod_set_value_cansleep(priv->reset_gpio, 0); + usleep_range(5000, 10000); + gpiod_set_value_cansleep(priv->reset_gpio, 1); + } else if (priv->sw_reset) { + ret = regmap_write(priv->regmap, BNO055_SYS_TRIGGER_REG, + BNO055_SYS_TRIGGER_RST_SYS); + if (ret) + return ret; + } else { + return 0; + } + + regcache_drop_region(priv->regmap, 0x0, 0xff); + usleep_range(650000, 700000); + + return 0; +} + +static int bno055_init(struct bno055_priv *priv, const u8 *caldata, int len) +{ + int ret; + + ret = bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_CONFIG); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, BNO055_POWER_MODE_REG, + BNO055_POWER_MODE_NORMAL); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, BNO055_SYS_TRIGGER_REG, + priv->clk ? BNO055_SYS_TRIGGER_CLK_SEL : 0); + if (ret) + return ret; + + /* use standard SI units */ + ret = regmap_write(priv->regmap, BNO055_UNIT_SEL_REG, + BNO055_UNIT_SEL_ANDROID | BNO055_UNIT_SEL_GYR_RPS); + if (ret) + return ret; + + if (caldata) { + ret = bno055_calibration_load(priv, caldata, len); + if (ret) + dev_warn(priv->dev, "failed to load calibration data with error %d\n", + ret); + } + + return 0; +} + +static ssize_t bno055_operation_mode_set(struct bno055_priv *priv, + int operation_mode) +{ + u8 caldata[BNO055_CALDATA_LEN]; + int ret; + + mutex_lock(&priv->lock); + + ret = bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_CONFIG); + if (ret) + goto exit_unlock; + + if (operation_mode == BNO055_OPR_MODE_FUSION || + operation_mode == BNO055_OPR_MODE_FUSION_FMC_OFF) { + /* for entering fusion mode, reset the chip to clear the algo state */ + ret = regmap_bulk_read(priv->regmap, BNO055_CALDATA_START, caldata, + BNO055_CALDATA_LEN); + if (ret) + goto exit_unlock; + + ret = bno055_system_reset(priv); + if (ret) + goto exit_unlock; + + ret = bno055_init(priv, caldata, BNO055_CALDATA_LEN); + if (ret) + goto exit_unlock; + } + + ret = bno055_operation_mode_do_set(priv, operation_mode); + if (ret) + goto exit_unlock; + + priv->operation_mode = operation_mode; + +exit_unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static void bno055_uninit(void *arg) +{ + struct bno055_priv *priv = arg; + + /* stop the IMU */ + bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_CONFIG); +} + +#define BNO055_CHANNEL(_type, _axis, _index, _address, _sep, _sh, _avail) { \ + .address = _address, \ + .type = _type, \ + .modified = 1, \ + .channel2 = IIO_MOD_##_axis, \ + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | (_sep), \ + .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE) | (_sh), \ + .info_mask_shared_by_type_available = _avail, \ + .scan_index = _index, \ + .scan_type = { \ + .sign = 's', \ + .realbits = 16, \ + .storagebits = 16, \ + .endianness = IIO_LE, \ + .repeat = IIO_MOD_##_axis == IIO_MOD_QUATERNION ? 4 : 0, \ + }, \ +} + +/* scan indexes follow DATA register order */ +enum bno055_scan_axis { + BNO055_SCAN_ACCEL_X, + BNO055_SCAN_ACCEL_Y, + BNO055_SCAN_ACCEL_Z, + BNO055_SCAN_MAGN_X, + BNO055_SCAN_MAGN_Y, + BNO055_SCAN_MAGN_Z, + BNO055_SCAN_GYRO_X, + BNO055_SCAN_GYRO_Y, + BNO055_SCAN_GYRO_Z, + BNO055_SCAN_YAW, + BNO055_SCAN_ROLL, + BNO055_SCAN_PITCH, + BNO055_SCAN_QUATERNION, + BNO055_SCAN_LIA_X, + BNO055_SCAN_LIA_Y, + BNO055_SCAN_LIA_Z, + BNO055_SCAN_GRAVITY_X, + BNO055_SCAN_GRAVITY_Y, + BNO055_SCAN_GRAVITY_Z, + BNO055_SCAN_TIMESTAMP, + _BNO055_SCAN_MAX +}; + +static const struct iio_chan_spec bno055_channels[] = { + /* accelerometer */ + BNO055_CHANNEL(IIO_ACCEL, X, BNO055_SCAN_ACCEL_X, + BNO055_ACC_DATA_X_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY)), + BNO055_CHANNEL(IIO_ACCEL, Y, BNO055_SCAN_ACCEL_Y, + BNO055_ACC_DATA_Y_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY)), + BNO055_CHANNEL(IIO_ACCEL, Z, BNO055_SCAN_ACCEL_Z, + BNO055_ACC_DATA_Z_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY)), + /* gyroscope */ + BNO055_CHANNEL(IIO_ANGL_VEL, X, BNO055_SCAN_GYRO_X, + BNO055_GYR_DATA_X_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY) | + BIT(IIO_CHAN_INFO_SCALE)), + BNO055_CHANNEL(IIO_ANGL_VEL, Y, BNO055_SCAN_GYRO_Y, + BNO055_GYR_DATA_Y_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY) | + BIT(IIO_CHAN_INFO_SCALE)), + BNO055_CHANNEL(IIO_ANGL_VEL, Z, BNO055_SCAN_GYRO_Z, + BNO055_GYR_DATA_Z_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY) | + BIT(IIO_CHAN_INFO_SCALE)), + /* magnetometer */ + BNO055_CHANNEL(IIO_MAGN, X, BNO055_SCAN_MAGN_X, + BNO055_MAG_DATA_X_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_SAMP_FREQ), BIT(IIO_CHAN_INFO_SAMP_FREQ)), + BNO055_CHANNEL(IIO_MAGN, Y, BNO055_SCAN_MAGN_Y, + BNO055_MAG_DATA_Y_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_SAMP_FREQ), BIT(IIO_CHAN_INFO_SAMP_FREQ)), + BNO055_CHANNEL(IIO_MAGN, Z, BNO055_SCAN_MAGN_Z, + BNO055_MAG_DATA_Z_LSB_REG, BIT(IIO_CHAN_INFO_OFFSET), + BIT(IIO_CHAN_INFO_SAMP_FREQ), BIT(IIO_CHAN_INFO_SAMP_FREQ)), + /* euler angle */ + BNO055_CHANNEL(IIO_ROT, YAW, BNO055_SCAN_YAW, + BNO055_EUL_DATA_X_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_ROT, ROLL, BNO055_SCAN_ROLL, + BNO055_EUL_DATA_Y_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_ROT, PITCH, BNO055_SCAN_PITCH, + BNO055_EUL_DATA_Z_LSB_REG, 0, 0, 0), + /* quaternion */ + BNO055_CHANNEL(IIO_ROT, QUATERNION, BNO055_SCAN_QUATERNION, + BNO055_QUAT_DATA_W_LSB_REG, 0, 0, 0), + + /* linear acceleration */ + BNO055_CHANNEL(IIO_ACCEL, LINEAR_X, BNO055_SCAN_LIA_X, + BNO055_LIA_DATA_X_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_ACCEL, LINEAR_Y, BNO055_SCAN_LIA_Y, + BNO055_LIA_DATA_Y_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_ACCEL, LINEAR_Z, BNO055_SCAN_LIA_Z, + BNO055_LIA_DATA_Z_LSB_REG, 0, 0, 0), + + /* gravity vector */ + BNO055_CHANNEL(IIO_GRAVITY, X, BNO055_SCAN_GRAVITY_X, + BNO055_GRAVITY_DATA_X_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_GRAVITY, Y, BNO055_SCAN_GRAVITY_Y, + BNO055_GRAVITY_DATA_Y_LSB_REG, 0, 0, 0), + BNO055_CHANNEL(IIO_GRAVITY, Z, BNO055_SCAN_GRAVITY_Z, + BNO055_GRAVITY_DATA_Z_LSB_REG, 0, 0, 0), + + { + .type = IIO_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED), + .scan_index = -1, + }, + IIO_CHAN_SOFT_TIMESTAMP(BNO055_SCAN_TIMESTAMP), +}; + +static int bno055_get_regmask(struct bno055_priv *priv, int *val, int *val2, + int reg, int mask, struct bno055_sysfs_attr *attr) +{ + const int shift = __ffs(mask); + int hwval, idx; + int ret; + int i; + + ret = regmap_read(priv->regmap, reg, &hwval); + if (ret) + return ret; + + idx = (hwval & mask) >> shift; + if (attr->hw_xlate) + for (i = 0; i < attr->len; i++) + if (attr->hw_xlate[i] == idx) { + idx = i; + break; + } + if (attr->type == IIO_VAL_INT) { + *val = attr->vals[idx]; + } else { /* IIO_VAL_INT_PLUS_MICRO or IIO_VAL_FRACTIONAL */ + *val = attr->vals[idx * 2]; + *val2 = attr->vals[idx * 2 + 1]; + } + + return attr->type; +} + +static int bno055_set_regmask(struct bno055_priv *priv, int val, int val2, + int reg, int mask, struct bno055_sysfs_attr *attr) +{ + const int shift = __ffs(mask); + int best_delta; + int req_val; + int tbl_val; + bool first; + int delta; + int hwval; + int ret; + int len; + int i; + + /* + * The closest value the HW supports is only one in fusion mode, + * and it is autoselected, so don't do anything, just return OK, + * as the closest possible value has been (virtually) selected + */ + if (priv->operation_mode != BNO055_OPR_MODE_AMG) + return 0; + + len = attr->len; + + /* + * We always get a request in INT_PLUS_MICRO, but we + * take care of the micro part only when we really have + * non-integer tables. This prevents 32-bit overflow with + * larger integers contained in integer tables. + */ + req_val = val; + if (attr->type != IIO_VAL_INT) { + len /= 2; + req_val = min(val, 2147) * 1000000 + val2; + } + + first = true; + for (i = 0; i < len; i++) { + switch (attr->type) { + case IIO_VAL_INT: + tbl_val = attr->vals[i]; + break; + case IIO_VAL_INT_PLUS_MICRO: + WARN_ON(attr->vals[i * 2] > 2147); + tbl_val = attr->vals[i * 2] * 1000000 + + attr->vals[i * 2 + 1]; + break; + case IIO_VAL_FRACTIONAL: + WARN_ON(attr->vals[i * 2] > 4294); + tbl_val = attr->vals[i * 2] * 1000000 / + attr->vals[i * 2 + 1]; + break; + default: + return -EINVAL; + } + delta = abs(tbl_val - req_val); + if (delta < best_delta || first) { + best_delta = delta; + hwval = i; + first = false; + } + } + + if (attr->hw_xlate) + hwval = attr->hw_xlate[hwval]; + + ret = bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_CONFIG); + if (ret) + return ret; + + ret = regmap_update_bits(priv->regmap, reg, mask, hwval << shift); + if (ret) + return ret; + + return bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_AMG); +} + +static int bno055_read_simple_chan(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, int *val2, long mask) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + __le16 raw_val; + int ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = regmap_bulk_read(priv->regmap, chan->address, + &raw_val, sizeof(raw_val)); + if (ret < 0) + return ret; + *val = sign_extend32(le16_to_cpu(raw_val), 15); + return IIO_VAL_INT; + case IIO_CHAN_INFO_OFFSET: + if (priv->operation_mode != BNO055_OPR_MODE_AMG) { + *val = 0; + } else { + ret = regmap_bulk_read(priv->regmap, + chan->address + + BNO055_REG_OFFSET_ADDR, + &raw_val, sizeof(raw_val)); + if (ret < 0) + return ret; + /* + * IMU reports sensor offsets; IIO wants correction + * offsets, thus we need the 'minus' here. + */ + *val = -sign_extend32(le16_to_cpu(raw_val), 15); + } + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 1; + switch (chan->type) { + case IIO_GRAVITY: + /* Table 3-35: 1 m/s^2 = 100 LSB */ + case IIO_ACCEL: + /* Table 3-17: 1 m/s^2 = 100 LSB */ + *val2 = 100; + break; + case IIO_MAGN: + /* + * Table 3-19: 1 uT = 16 LSB. But we need + * Gauss: 1G = 0.1 uT. + */ + *val2 = 160; + break; + case IIO_ANGL_VEL: + /* + * Table 3-22: 1 Rps = 900 LSB + * .. but this is not exactly true. See comment at the + * beginning of this file. + */ + if (priv->operation_mode != BNO055_OPR_MODE_AMG) { + *val = bno055_gyr_scale.fusion_vals[0]; + *val2 = bno055_gyr_scale.fusion_vals[1]; + return IIO_VAL_FRACTIONAL; + } + + return bno055_get_regmask(priv, val, val2, + BNO055_GYR_CONFIG_REG, + BNO055_GYR_CONFIG_RANGE_MASK, + &bno055_gyr_scale); + break; + case IIO_ROT: + /* Table 3-28: 1 degree = 16 LSB */ + *val2 = 16; + break; + default: + return -EINVAL; + } + return IIO_VAL_FRACTIONAL; + + case IIO_CHAN_INFO_SAMP_FREQ: + if (chan->type != IIO_MAGN) + return -EINVAL; + + return bno055_get_regmask(priv, val, val2, + BNO055_MAG_CONFIG_REG, + BNO055_MAG_CONFIG_ODR_MASK, + &bno055_mag_odr); + + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + switch (chan->type) { + case IIO_ANGL_VEL: + return bno055_get_regmask(priv, val, val2, + BNO055_GYR_CONFIG_REG, + BNO055_GYR_CONFIG_LPF_MASK, + &bno055_gyr_lpf); + case IIO_ACCEL: + return bno055_get_regmask(priv, val, val2, + BNO055_ACC_CONFIG_REG, + BNO055_ACC_CONFIG_LPF_MASK, + &bno055_acc_lpf); + default: + return -EINVAL; + } + + default: + return -EINVAL; + } +} + +static int bno055_sysfs_attr_avail(struct bno055_priv *priv, struct bno055_sysfs_attr *attr, + const int **vals, int *length) +{ + if (priv->operation_mode != BNO055_OPR_MODE_AMG) { + /* locked when fusion enabled */ + *vals = attr->fusion_vals; + if (attr->type == IIO_VAL_INT) + *length = 1; + else + *length = 2; /* IIO_VAL_INT_PLUS_MICRO or IIO_VAL_FRACTIONAL*/ + } else { + *vals = attr->vals; + *length = attr->len; + } + + return attr->type; +} + +static int bno055_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + + switch (mask) { + case IIO_CHAN_INFO_SCALE: + switch (chan->type) { + case IIO_ANGL_VEL: + *type = bno055_sysfs_attr_avail(priv, &bno055_gyr_scale, + vals, length); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + switch (chan->type) { + case IIO_ANGL_VEL: + *type = bno055_sysfs_attr_avail(priv, &bno055_gyr_lpf, + vals, length); + return IIO_AVAIL_LIST; + case IIO_ACCEL: + *type = bno055_sysfs_attr_avail(priv, &bno055_acc_lpf, + vals, length); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } + + break; + case IIO_CHAN_INFO_SAMP_FREQ: + switch (chan->type) { + case IIO_MAGN: + *type = bno055_sysfs_attr_avail(priv, &bno055_mag_odr, + vals, length); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static int bno055_read_temp_chan(struct iio_dev *indio_dev, int *val) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + unsigned int raw_val; + int ret; + + ret = regmap_read(priv->regmap, BNO055_TEMP_REG, &raw_val); + if (ret < 0) + return ret; + + /* + * Tables 3-36 and 3-37: one byte of priv, signed, 1 LSB = 1C. + * ABI wants milliC. + */ + *val = raw_val * 1000; + + return IIO_VAL_INT; +} + +static int bno055_read_quaternion(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int size, int *vals, int *val_len, + long mask) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + __le16 raw_vals[4]; + int i, ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + if (size < 4) + return -EINVAL; + ret = regmap_bulk_read(priv->regmap, + BNO055_QUAT_DATA_W_LSB_REG, + raw_vals, sizeof(raw_vals)); + if (ret < 0) + return ret; + for (i = 0; i < 4; i++) + vals[i] = sign_extend32(le16_to_cpu(raw_vals[i]), 15); + *val_len = 4; + return IIO_VAL_INT_MULTIPLE; + case IIO_CHAN_INFO_SCALE: + /* Table 3-31: 1 quaternion = 2^14 LSB */ + if (size < 2) + return -EINVAL; + vals[0] = 1; + vals[1] = 14; + return IIO_VAL_FRACTIONAL_LOG2; + default: + return -EINVAL; + } +} + +static bool bno055_is_chan_readable(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + + if (priv->operation_mode != BNO055_OPR_MODE_AMG) + return true; + + switch (chan->type) { + case IIO_GRAVITY: + case IIO_ROT: + return false; + case IIO_ACCEL: + if (chan->channel2 == IIO_MOD_LINEAR_X || + chan->channel2 == IIO_MOD_LINEAR_Y || + chan->channel2 == IIO_MOD_LINEAR_Z) + return false; + return true; + default: + return true; + } +} + +static int _bno055_read_raw_multi(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int size, int *vals, int *val_len, + long mask) +{ + if (!bno055_is_chan_readable(indio_dev, chan)) + return -EBUSY; + + switch (chan->type) { + case IIO_MAGN: + case IIO_ACCEL: + case IIO_ANGL_VEL: + case IIO_GRAVITY: + if (size < 2) + return -EINVAL; + *val_len = 2; + return bno055_read_simple_chan(indio_dev, chan, + &vals[0], &vals[1], + mask); + case IIO_TEMP: + *val_len = 1; + return bno055_read_temp_chan(indio_dev, &vals[0]); + case IIO_ROT: + /* + * Rotation is exposed as either a quaternion or three + * Euler angles. + */ + if (chan->channel2 == IIO_MOD_QUATERNION) + return bno055_read_quaternion(indio_dev, chan, + size, vals, + val_len, mask); + if (size < 2) + return -EINVAL; + *val_len = 2; + return bno055_read_simple_chan(indio_dev, chan, + &vals[0], &vals[1], + mask); + default: + return -EINVAL; + } +} + +static int bno055_read_raw_multi(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int size, int *vals, int *val_len, + long mask) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + int ret; + + mutex_lock(&priv->lock); + ret = _bno055_read_raw_multi(indio_dev, chan, size, + vals, val_len, mask); + mutex_unlock(&priv->lock); + return ret; +} + +static int _bno055_write_raw(struct iio_dev *iio_dev, + struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + struct bno055_priv *priv = iio_priv(iio_dev); + + switch (chan->type) { + case IIO_MAGN: + switch (mask) { + case IIO_CHAN_INFO_SAMP_FREQ: + return bno055_set_regmask(priv, val, val2, + BNO055_MAG_CONFIG_REG, + BNO055_MAG_CONFIG_ODR_MASK, + &bno055_mag_odr); + default: + return -EINVAL; + } + case IIO_ACCEL: + switch (mask) { + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + return bno055_set_regmask(priv, val, val2, + BNO055_ACC_CONFIG_REG, + BNO055_ACC_CONFIG_LPF_MASK, + &bno055_acc_lpf); + + default: + return -EINVAL; + } + case IIO_ANGL_VEL: + switch (mask) { + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + return bno055_set_regmask(priv, val, val2, + BNO055_GYR_CONFIG_REG, + BNO055_GYR_CONFIG_LPF_MASK, + &bno055_gyr_lpf); + case IIO_CHAN_INFO_SCALE: + return bno055_set_regmask(priv, val, val2, + BNO055_GYR_CONFIG_REG, + BNO055_GYR_CONFIG_RANGE_MASK, + &bno055_gyr_scale); + default: + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static int bno055_write_raw(struct iio_dev *iio_dev, + struct iio_chan_spec const *chan, + int val, int val2, long mask) +{ + struct bno055_priv *priv = iio_priv(iio_dev); + int ret; + + mutex_lock(&priv->lock); + ret = _bno055_write_raw(iio_dev, chan, val, val2, mask); + mutex_unlock(&priv->lock); + + return ret; +} + +static ssize_t in_accel_range_raw_available_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + int len = 0; + int i; + + if (priv->operation_mode != BNO055_OPR_MODE_AMG) + return sysfs_emit(buf, "%d\n", bno055_acc_range.fusion_vals[0]); + + for (i = 0; i < bno055_acc_range.len; i++) + len += sysfs_emit_at(buf, len, "%d ", bno055_acc_range.vals[i]); + buf[len - 1] = '\n'; + + return len; +} + +static ssize_t fusion_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + + return sysfs_emit(buf, "%d\n", + priv->operation_mode != BNO055_OPR_MODE_AMG); +} + +static ssize_t fusion_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct bno055_priv *priv = iio_priv(indio_dev); + bool en; + int ret; + + if (indio_dev->active_scan_mask && + !bitmap_empty(indio_dev->active_scan_mask, _BNO055_SCAN_MAX)) + return -EBUSY; + + ret = kstrtobool(buf, &en); + if (ret) + return -EINVAL; + + if (!en) + return bno055_operation_mode_set(priv, BNO055_OPR_MODE_AMG) ?: len; + + /* + * Coming from AMG means the FMC was off, just switch to fusion but + * don't change anything that doesn't belong to us (i.e let FMC stay off). + * Coming from any other fusion mode means we don't need to do anything. + */ + if (priv->operation_mode == BNO055_OPR_MODE_AMG) + return bno055_operation_mode_set(priv, BNO055_OPR_MODE_FUSION_FMC_OFF) ?: len; + + return len; +} + +static ssize_t in_magn_calibration_fast_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + + return sysfs_emit(buf, "%d\n", + priv->operation_mode == BNO055_OPR_MODE_FUSION); +} + +static ssize_t in_magn_calibration_fast_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct iio_dev *indio_dev = dev_to_iio_dev(dev); + struct bno055_priv *priv = iio_priv(indio_dev); + int ret; + + if (indio_dev->active_scan_mask && + !bitmap_empty(indio_dev->active_scan_mask, _BNO055_SCAN_MAX)) + return -EBUSY; + + if (sysfs_streq(buf, "0")) { + if (priv->operation_mode == BNO055_OPR_MODE_FUSION) { + ret = bno055_operation_mode_set(priv, BNO055_OPR_MODE_FUSION_FMC_OFF); + if (ret) + return ret; + } + } else { + if (priv->operation_mode == BNO055_OPR_MODE_AMG) + return -EINVAL; + + if (priv->operation_mode != BNO055_OPR_MODE_FUSION) { + ret = bno055_operation_mode_set(priv, BNO055_OPR_MODE_FUSION); + if (ret) + return ret; + } + } + + return len; +} + +static ssize_t in_accel_range_raw_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + int val; + int ret; + + ret = bno055_get_regmask(priv, &val, NULL, + BNO055_ACC_CONFIG_REG, + BNO055_ACC_CONFIG_RANGE_MASK, + &bno055_acc_range); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", val); +} + +static ssize_t in_accel_range_raw_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + unsigned long val; + int ret; + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + + mutex_lock(&priv->lock); + ret = bno055_set_regmask(priv, val, 0, + BNO055_ACC_CONFIG_REG, + BNO055_ACC_CONFIG_RANGE_MASK, + &bno055_acc_range); + mutex_unlock(&priv->lock); + + return ret ?: len; +} + +static ssize_t bno055_get_calib_status(struct device *dev, char *buf, int which) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + int calib; + int ret; + int val; + + if (priv->operation_mode == BNO055_OPR_MODE_AMG || + (priv->operation_mode == BNO055_OPR_MODE_FUSION_FMC_OFF && + which == BNO055_CALIB_STAT_MAGN_SHIFT)) { + calib = 0; + } else { + mutex_lock(&priv->lock); + ret = regmap_read(priv->regmap, BNO055_CALIB_STAT_REG, &val); + mutex_unlock(&priv->lock); + + if (ret) + return -EIO; + + calib = ((val >> which) & GENMASK(1, 0)) + 1; + } + + return sysfs_emit(buf, "%d\n", calib); +} + +static ssize_t serialnumber_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(dev)); + + return sysfs_emit(buf, "%*ph\n", BNO055_UID_LEN, priv->uid); +} + +static ssize_t calibration_data_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, char *buf, + loff_t pos, size_t count) +{ + struct bno055_priv *priv = iio_priv(dev_to_iio_dev(kobj_to_dev(kobj))); + u8 data[BNO055_CALDATA_LEN]; + int ret; + + /* + * Calibration data is volatile; reading it in chunks will possibly + * results in inconsistent data. We require the user to read the whole + * blob in a single chunk + */ + if (count < BNO055_CALDATA_LEN || pos) + return -EINVAL; + + mutex_lock(&priv->lock); + ret = bno055_operation_mode_do_set(priv, BNO055_OPR_MODE_CONFIG); + if (ret) + goto exit_unlock; + + ret = regmap_bulk_read(priv->regmap, BNO055_CALDATA_START, data, + BNO055_CALDATA_LEN); + if (ret) + goto exit_unlock; + + ret = bno055_operation_mode_do_set(priv, priv->operation_mode); + if (ret) + goto exit_unlock; + + memcpy(buf, data, BNO055_CALDATA_LEN); + + ret = BNO055_CALDATA_LEN; +exit_unlock: + mutex_unlock(&priv->lock); + return ret; +} + +static ssize_t sys_calibration_auto_status_show(struct device *dev, + struct device_attribute *a, + char *buf) +{ + return bno055_get_calib_status(dev, buf, BNO055_CALIB_STAT_SYS_SHIFT); +} + +static ssize_t in_accel_calibration_auto_status_show(struct device *dev, + struct device_attribute *a, + char *buf) +{ + return bno055_get_calib_status(dev, buf, BNO055_CALIB_STAT_ACCEL_SHIFT); +} + +static ssize_t in_gyro_calibration_auto_status_show(struct device *dev, + struct device_attribute *a, + char *buf) +{ + return bno055_get_calib_status(dev, buf, BNO055_CALIB_STAT_GYRO_SHIFT); +} + +static ssize_t in_magn_calibration_auto_status_show(struct device *dev, + struct device_attribute *a, + char *buf) +{ + return bno055_get_calib_status(dev, buf, BNO055_CALIB_STAT_MAGN_SHIFT); +} + +static int bno055_debugfs_reg_access(struct iio_dev *iio_dev, unsigned int reg, + unsigned int writeval, unsigned int *readval) +{ + struct bno055_priv *priv = iio_priv(iio_dev); + + if (readval) + return regmap_read(priv->regmap, reg, readval); + else + return regmap_write(priv->regmap, reg, writeval); +} + +static ssize_t bno055_show_fw_version(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct bno055_priv *priv = file->private_data; + int rev, ver; + char *buf; + int ret; + + ret = regmap_read(priv->regmap, BNO055_SW_REV_LSB_REG, &rev); + if (ret) + return ret; + + ret = regmap_read(priv->regmap, BNO055_SW_REV_MSB_REG, &ver); + if (ret) + return ret; + + buf = kasprintf(GFP_KERNEL, "ver: 0x%x, rev: 0x%x\n", ver, rev); + if (!buf) + return -ENOMEM; + + ret = simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf)); + kfree(buf); + + return ret; +} + +static const struct file_operations bno055_fw_version_ops = { + .open = simple_open, + .read = bno055_show_fw_version, + .llseek = default_llseek, + .owner = THIS_MODULE, +}; + +static void bno055_debugfs_remove(void *_priv) +{ + struct bno055_priv *priv = _priv; + + debugfs_remove(priv->debugfs); + priv->debugfs = NULL; +} + +static void bno055_debugfs_init(struct iio_dev *iio_dev) +{ + struct bno055_priv *priv = iio_priv(iio_dev); + + priv->debugfs = debugfs_create_file("firmware_version", 0400, + iio_get_debugfs_dentry(iio_dev), + priv, &bno055_fw_version_ops); + if (!IS_ERR(priv->debugfs)) + devm_add_action_or_reset(priv->dev, bno055_debugfs_remove, + priv); + if (IS_ERR_OR_NULL(priv->debugfs)) + dev_warn(priv->dev, "failed to setup debugfs"); +} + +static IIO_DEVICE_ATTR_RW(fusion_enable, 0); +static IIO_DEVICE_ATTR_RW(in_magn_calibration_fast_enable, 0); +static IIO_DEVICE_ATTR_RW(in_accel_range_raw, 0); + +static IIO_DEVICE_ATTR_RO(in_accel_range_raw_available, 0); +static IIO_DEVICE_ATTR_RO(sys_calibration_auto_status, 0); +static IIO_DEVICE_ATTR_RO(in_accel_calibration_auto_status, 0); +static IIO_DEVICE_ATTR_RO(in_gyro_calibration_auto_status, 0); +static IIO_DEVICE_ATTR_RO(in_magn_calibration_auto_status, 0); +static IIO_DEVICE_ATTR_RO(serialnumber, 0); + +static struct attribute *bno055_attrs[] = { + &iio_dev_attr_in_accel_range_raw_available.dev_attr.attr, + &iio_dev_attr_in_accel_range_raw.dev_attr.attr, + &iio_dev_attr_fusion_enable.dev_attr.attr, + &iio_dev_attr_in_magn_calibration_fast_enable.dev_attr.attr, + &iio_dev_attr_sys_calibration_auto_status.dev_attr.attr, + &iio_dev_attr_in_accel_calibration_auto_status.dev_attr.attr, + &iio_dev_attr_in_gyro_calibration_auto_status.dev_attr.attr, + &iio_dev_attr_in_magn_calibration_auto_status.dev_attr.attr, + &iio_dev_attr_serialnumber.dev_attr.attr, + NULL +}; + +static BIN_ATTR_RO(calibration_data, BNO055_CALDATA_LEN); + +static struct bin_attribute *bno055_bin_attrs[] = { + &bin_attr_calibration_data, + NULL +}; + +static const struct attribute_group bno055_attrs_group = { + .attrs = bno055_attrs, + .bin_attrs = bno055_bin_attrs, +}; + +static const struct iio_info bno055_info = { + .read_raw_multi = bno055_read_raw_multi, + .read_avail = bno055_read_avail, + .write_raw = bno055_write_raw, + .attrs = &bno055_attrs_group, + .debugfs_reg_access = bno055_debugfs_reg_access, +}; + +/* + * Reads len samples from the HW, stores them in buf starting from buf_idx, + * and applies mask to cull (skip) unneeded samples. + * Updates buf_idx incrementing with the number of stored samples. + * Samples from HW are transferred into buf, then in-place copy on buf is + * performed in order to cull samples that need to be skipped. + * This avoids copies of the first samples until we hit the 1st sample to skip, + * and also avoids having an extra bounce buffer. + * buf must be able to contain len elements in spite of how many samples we are + * going to cull. + */ +static int bno055_scan_xfer(struct bno055_priv *priv, + int start_ch, int len, unsigned long mask, + __le16 *buf, int *buf_idx) +{ + const int base = BNO055_ACC_DATA_X_LSB_REG; + bool quat_in_read = false; + int buf_base = *buf_idx; + __le16 *dst, *src; + int offs_fixup = 0; + int xfer_len = len; + int ret; + int i, n; + + if (!mask) + return 0; + + /* + * All channels are made up 1 16-bit sample, except for quaternion that + * is made up 4 16-bit values. + * For us the quaternion CH is just like 4 regular CHs. + * If our read starts past the quaternion make sure to adjust the + * starting offset; if the quaternion is contained in our scan then make + * sure to adjust the read len. + */ + if (start_ch > BNO055_SCAN_QUATERNION) { + start_ch += 3; + } else if ((start_ch <= BNO055_SCAN_QUATERNION) && + ((start_ch + len) > BNO055_SCAN_QUATERNION)) { + quat_in_read = true; + xfer_len += 3; + } + + ret = regmap_bulk_read(priv->regmap, + base + start_ch * sizeof(__le16), + buf + buf_base, + xfer_len * sizeof(__le16)); + if (ret) + return ret; + + for_each_set_bit(i, &mask, len) { + if (quat_in_read && ((start_ch + i) > BNO055_SCAN_QUATERNION)) + offs_fixup = 3; + + dst = buf + *buf_idx; + src = buf + buf_base + offs_fixup + i; + + n = (start_ch + i == BNO055_SCAN_QUATERNION) ? 4 : 1; + + if (dst != src) + memcpy(dst, src, n * sizeof(__le16)); + + *buf_idx += n; + } + return 0; +} + +static irqreturn_t bno055_trigger_handler(int irq, void *p) +{ + struct iio_poll_func *pf = p; + struct iio_dev *iio_dev = pf->indio_dev; + struct bno055_priv *priv = iio_priv(iio_dev); + int xfer_start, start, end, prev_end; + unsigned long mask; + int quat_extra_len; + bool first = true; + int buf_idx = 0; + bool thr_hit; + int ret; + + mutex_lock(&priv->lock); + + /* + * Walk the bitmap and eventually perform several transfers. + * Bitmap ones-fields that are separated by gaps <= xfer_burst_break_thr + * will be included in same transfer. + * Every time the bitmap contains a gap wider than xfer_burst_break_thr + * then we split the transfer, skipping the gap. + */ + for_each_set_bitrange(start, end, iio_dev->active_scan_mask, + iio_dev->masklength) { + /* + * First transfer will start from the beginning of the first + * ones-field in the bitmap + */ + if (first) { + xfer_start = start; + } else { + /* + * We found the next ones-field; check whether to + * include it in * the current transfer or not (i.e. + * let's perform the current * transfer and prepare for + * another one). + */ + + /* + * In case the zeros-gap contains the quaternion bit, + * then its length is actually 4 words instead of 1 + * (i.e. +3 wrt other channels). + */ + quat_extra_len = ((start > BNO055_SCAN_QUATERNION) && + (prev_end <= BNO055_SCAN_QUATERNION)) ? 3 : 0; + + /* If the gap is wider than xfer_burst_break_thr then.. */ + thr_hit = (start - prev_end + quat_extra_len) > + priv->xfer_burst_break_thr; + + /* + * .. transfer all the data up to the gap. Then set the + * next transfer start index at right after the gap + * (i.e. at the start of this ones-field). + */ + if (thr_hit) { + mask = *iio_dev->active_scan_mask >> xfer_start; + ret = bno055_scan_xfer(priv, xfer_start, + prev_end - xfer_start, + mask, priv->buf.chans, &buf_idx); + if (ret) + goto done; + xfer_start = start; + } + } + first = false; + prev_end = end; + } + + /* + * We finished walking the bitmap; no more gaps to check for. Just + * perform the current transfer. + */ + mask = *iio_dev->active_scan_mask >> xfer_start; + ret = bno055_scan_xfer(priv, xfer_start, + prev_end - xfer_start, + mask, priv->buf.chans, &buf_idx); + + if (!ret) + iio_push_to_buffers_with_timestamp(iio_dev, + &priv->buf, pf->timestamp); +done: + mutex_unlock(&priv->lock); + iio_trigger_notify_done(iio_dev->trig); + return IRQ_HANDLED; +} + +static int bno055_buffer_preenable(struct iio_dev *indio_dev) +{ + struct bno055_priv *priv = iio_priv(indio_dev); + const unsigned long fusion_mask = + BIT(BNO055_SCAN_YAW) | + BIT(BNO055_SCAN_ROLL) | + BIT(BNO055_SCAN_PITCH) | + BIT(BNO055_SCAN_QUATERNION) | + BIT(BNO055_SCAN_LIA_X) | + BIT(BNO055_SCAN_LIA_Y) | + BIT(BNO055_SCAN_LIA_Z) | + BIT(BNO055_SCAN_GRAVITY_X) | + BIT(BNO055_SCAN_GRAVITY_Y) | + BIT(BNO055_SCAN_GRAVITY_Z); + + if (priv->operation_mode == BNO055_OPR_MODE_AMG && + bitmap_intersects(indio_dev->active_scan_mask, &fusion_mask, + _BNO055_SCAN_MAX)) + return -EBUSY; + return 0; +} + +static const struct iio_buffer_setup_ops bno055_buffer_setup_ops = { + .preenable = bno055_buffer_preenable, +}; + +int bno055_probe(struct device *dev, struct regmap *regmap, + int xfer_burst_break_thr, bool sw_reset) +{ + const struct firmware *caldata = NULL; + struct bno055_priv *priv; + struct iio_dev *iio_dev; + char *fw_name_buf; + unsigned int val; + int rev, ver; + int ret; + + iio_dev = devm_iio_device_alloc(dev, sizeof(*priv)); + if (!iio_dev) + return -ENOMEM; + + iio_dev->name = "bno055"; + priv = iio_priv(iio_dev); + mutex_init(&priv->lock); + priv->regmap = regmap; + priv->dev = dev; + priv->xfer_burst_break_thr = xfer_burst_break_thr; + priv->sw_reset = sw_reset; + + priv->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(priv->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(priv->reset_gpio), "Failed to get reset GPIO\n"); + + priv->clk = devm_clk_get_optional_enabled(dev, "clk"); + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), "Failed to get CLK\n"); + + if (priv->reset_gpio) { + usleep_range(5000, 10000); + gpiod_set_value_cansleep(priv->reset_gpio, 1); + usleep_range(650000, 750000); + } else if (!sw_reset) { + dev_warn(dev, "No usable reset method; IMU may be unreliable\n"); + } + + ret = regmap_read(priv->regmap, BNO055_CHIP_ID_REG, &val); + if (ret) + return ret; + + if (val != BNO055_CHIP_ID_MAGIC) + dev_warn(dev, "Unrecognized chip ID 0x%x\n", val); + + /* + * In case we haven't a HW reset pin, we can still reset the chip via + * register write. This is probably nonsense in case we can't even + * communicate with the chip or the chip isn't the one we expect (i.e. + * we don't write to unknown chips), so we perform SW reset only after + * chip magic ID check + */ + if (!priv->reset_gpio) { + ret = bno055_system_reset(priv); + if (ret) + return ret; + } + + ret = regmap_read(priv->regmap, BNO055_SW_REV_LSB_REG, &rev); + if (ret) + return ret; + + ret = regmap_read(priv->regmap, BNO055_SW_REV_MSB_REG, &ver); + if (ret) + return ret; + + /* + * The stock FW version contains a bug (see comment at the beginning of + * this file) that causes the anglvel scale to be changed depending on + * the chip range setting. We workaround this, but we don't know what + * other FW versions might do. + */ + if (ver != 0x3 || rev != 0x11) + dev_warn(dev, "Untested firmware version. Anglvel scale may not work as expected\n"); + + ret = regmap_bulk_read(priv->regmap, BNO055_UID_LOWER_REG, + priv->uid, BNO055_UID_LEN); + if (ret) + return ret; + + /* Sensor calibration data */ + fw_name_buf = kasprintf(GFP_KERNEL, BNO055_FW_UID_FMT, + BNO055_UID_LEN, priv->uid); + if (!fw_name_buf) + return -ENOMEM; + + ret = request_firmware(&caldata, fw_name_buf, dev); + kfree(fw_name_buf); + if (ret) + ret = request_firmware(&caldata, BNO055_FW_GENERIC_NAME, dev); + if (ret) { + dev_notice(dev, "Calibration file load failed. See instruction in kernel Documentation/iio/bno055.rst\n"); + ret = bno055_init(priv, NULL, 0); + } else { + ret = bno055_init(priv, caldata->data, caldata->size); + release_firmware(caldata); + } + if (ret) + return ret; + + priv->operation_mode = BNO055_OPR_MODE_FUSION; + ret = bno055_operation_mode_do_set(priv, priv->operation_mode); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, bno055_uninit, priv); + if (ret) + return ret; + + iio_dev->channels = bno055_channels; + iio_dev->num_channels = ARRAY_SIZE(bno055_channels); + iio_dev->info = &bno055_info; + iio_dev->modes = INDIO_DIRECT_MODE; + + ret = devm_iio_triggered_buffer_setup(dev, iio_dev, + iio_pollfunc_store_time, + bno055_trigger_handler, + &bno055_buffer_setup_ops); + if (ret) + return ret; + + ret = devm_iio_device_register(dev, iio_dev); + if (ret) + return ret; + + bno055_debugfs_init(iio_dev); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(bno055_probe, IIO_BNO055); + +MODULE_AUTHOR("Andrea Merello "); +MODULE_DESCRIPTION("Bosch BNO055 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/imu/bno055/bno055.h b/drivers/iio/imu/bno055/bno055.h new file mode 100644 index 0000000000000000000000000000000000000000..64f9fc95cebc95fa01636e62d5415157f986c24c --- /dev/null +++ b/drivers/iio/imu/bno055/bno055.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef __BNO055_H__ +#define __BNO055_H__ + +#include +#include + +struct device; +int bno055_probe(struct device *dev, struct regmap *regmap, + int xfer_burst_break_thr, bool sw_reset); +extern const struct regmap_config bno055_regmap_config; + +#endif diff --git a/drivers/iio/imu/bno055/bno055_i2c.c b/drivers/iio/imu/bno055/bno055_i2c.c new file mode 100644 index 0000000000000000000000000000000000000000..c1bbc0fe34f9bdd467c2506f46d3957699e6da29 --- /dev/null +++ b/drivers/iio/imu/bno055/bno055_i2c.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for I2C-interfaced Bosch BNO055 IMU. + * + * Copyright (C) 2021-2022 Istituto Italiano di Tecnologia + * Electronic Design Laboratory + * Written by Andrea Merello + */ + +#include +#include +#include +#include + +#include "bno055.h" + +#define BNO055_I2C_XFER_BURST_BREAK_THRESHOLD 3 + +static int bno055_i2c_probe(struct i2c_client *client) +{ + struct regmap *regmap; + + regmap = devm_regmap_init_i2c(client, &bno055_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(&client->dev, PTR_ERR(regmap), + "Unable to init register map"); + + return bno055_probe(&client->dev, regmap, + BNO055_I2C_XFER_BURST_BREAK_THRESHOLD, true); +} + +static const struct i2c_device_id bno055_i2c_id[] = { + {"bno055", 0}, + { } +}; +MODULE_DEVICE_TABLE(i2c, bno055_i2c_id); + +static const struct of_device_id __maybe_unused bno055_i2c_of_match[] = { + { .compatible = "bosch,bno055" }, + { } +}; +MODULE_DEVICE_TABLE(of, bno055_i2c_of_match); + +static struct i2c_driver bno055_driver = { + .driver = { + .name = "bno055-i2c", + .of_match_table = bno055_i2c_of_match, + }, + .probe_new = bno055_i2c_probe, + .id_table = bno055_i2c_id, +}; +module_i2c_driver(bno055_driver); + +MODULE_AUTHOR("Andrea Merello"); +MODULE_DESCRIPTION("Bosch BNO055 I2C interface"); +MODULE_IMPORT_NS(IIO_BNO055); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/imu/bno055/bno055_ser_core.c b/drivers/iio/imu/bno055/bno055_ser_core.c new file mode 100644 index 0000000000000000000000000000000000000000..57728a568471560a18dc94d6c06eb05359a6cd39 --- /dev/null +++ b/drivers/iio/imu/bno055/bno055_ser_core.c @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Serial line interface for Bosh BNO055 IMU (via serdev). + * This file implements serial communication up to the register read/write + * level. + * + * Copyright (C) 2021-2022 Istituto Italiano di Tecnologia + * Electronic Design Laboratory + * Written by Andrea Merello + * + * This driver is based on + * Plantower PMS7003 particulate matter sensor driver + * Which is + * Copyright (c) Tomasz Duszynski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bno055_ser_trace.h" +#include "bno055.h" + +/* + * Register writes cmd have the following format + * +------+------+-----+-----+----- ... ----+ + * | 0xAA | 0xOO | REG | LEN | payload[LEN] | + * +------+------+-----+-----+----- ... ----+ + * + * Register write responses have the following format + * +------+----------+ + * | 0xEE | ERROCODE | + * +------+----------+ + * + * .. except when writing the SYS_RST bit (i.e. triggering a system reset); in + * case the IMU accepts the command, then it resets without responding. We don't + * handle this (yet) here (so we inform the common bno055 code not to perform + * sw resets - bno055 on serial bus basically requires the hw reset pin). + * + * Register read have the following format + * +------+------+-----+-----+ + * | 0xAA | 0xO1 | REG | LEN | + * +------+------+-----+-----+ + * + * Successful register read response have the following format + * +------+-----+----- ... ----+ + * | 0xBB | LEN | payload[LEN] | + * +------+-----+----- ... ----+ + * + * Failed register read response have the following format + * +------+--------+ + * | 0xEE | ERRCODE| (ERRCODE always > 1) + * +------+--------+ + * + * Error codes are + * 01: OK + * 02: read/write FAIL + * 04: invalid address + * 05: write on RO + * 06: wrong start byte + * 07: bus overrun + * 08: len too high + * 09: len too low + * 10: bus RX byte timeout (timeout is 30mS) + * + * + * **WORKAROUND ALERT** + * + * Serial communication seems very fragile: the BNO055 buffer seems to overflow + * very easy; BNO055 seems able to sink few bytes, then it needs a brief pause. + * On the other hand, it is also picky on timeout: if there is a pause > 30mS in + * between two bytes then the transaction fails (IMU internal RX FSM resets). + * + * BNO055 has been seen also failing to process commands in case we send them + * too close each other (or if it is somehow busy?) + * + * In particular I saw these scenarios: + * 1) If we send 2 bytes per time, then the IMU never(?) overflows. + * 2) If we send 4 bytes per time (i.e. the full header), then the IMU could + * overflow, but it seem to sink all 4 bytes, then it returns error. + * 3) If we send more than 4 bytes, the IMU could overflow, and I saw it sending + * error after 4 bytes are sent; we have troubles in synchronizing again, + * because we are still sending data, and the IMU interprets it as the 1st + * byte of a new command. + * + * While we must avoid case 3, we could send 4 bytes per time and eventually + * retry in case of failure; this seemed convenient for reads (which requires + * TXing exactly 4 bytes), however it has been seen that, depending by the IMU + * settings (e.g. LPF), failures became less or more frequent; in certain IMU + * configurations they are very rare, but in certain others we keeps failing + * even after like 30 retries. + * + * So, we just split TXes in [2-bytes + delay] steps, and still keep an eye on + * the IMU response; in case it overflows (which is now unlikely), we retry. + */ + +/* + * Read operation overhead: + * 4 bytes req + 2byte resp hdr. + * 6 bytes = 60 bit (considering 1start + 1stop bits). + * 60/115200 = ~520uS + about 2500mS delay -> ~3mS + * In 3mS we could read back about 34 bytes that means 17 samples, this means + * that in case of scattered reads in which the gap is 17 samples or less it is + * still convenient to go for a burst. + * We have to take into account also IMU response time - IMU seems to be often + * reasonably quick to respond, but sometimes it seems to be in some "critical + * section" in which it delays handling of serial protocol. Because of this we + * round-up to 22, which is the max number of samples, always bursting indeed. + */ +#define BNO055_SER_XFER_BURST_BREAK_THRESHOLD 22 + +struct bno055_ser_priv { + enum { + CMD_NONE, + CMD_READ, + CMD_WRITE, + } expect_response; + int expected_data_len; + u8 *response_buf; + + /** + * enum cmd_status - represent the status of a command sent to the HW. + * @STATUS_CRIT: The command failed: the serial communication failed. + * @STATUS_OK: The command executed successfully. + * @STATUS_FAIL: The command failed: HW responded with an error. + */ + enum { + STATUS_CRIT = -1, + STATUS_OK = 0, + STATUS_FAIL = 1, + } cmd_status; + + /* + * Protects all the above fields, which are accessed in behalf of both + * the serdev RX callback and the regmap side + */ + struct mutex lock; + + /* Only accessed in serdev RX callback context*/ + struct { + enum { + RX_IDLE, + RX_START, + RX_DATA, + } state; + int databuf_count; + int expected_len; + int type; + } rx; + + /* Never accessed in behalf of serdev RX callback context */ + bool cmd_stale; + + struct completion cmd_complete; + struct serdev_device *serdev; +}; + +static int bno055_ser_send_chunk(struct bno055_ser_priv *priv, const u8 *data, int len) +{ + int ret; + + trace_send_chunk(len, data); + ret = serdev_device_write(priv->serdev, data, len, msecs_to_jiffies(25)); + if (ret < 0) + return ret; + + if (ret < len) + return -EIO; + + return 0; +} + +/* + * Send a read or write command. + * 'data' can be NULL (used in read case). 'len' parameter is always valid; in + * case 'data' is non-NULL then it must match 'data' size. + */ +static int bno055_ser_do_send_cmd(struct bno055_ser_priv *priv, + bool read, int addr, int len, const u8 *data) +{ + u8 hdr[] = {0xAA, read, addr, len}; + int chunk_len; + int ret; + + ret = bno055_ser_send_chunk(priv, hdr, 2); + if (ret) + goto fail; + usleep_range(2000, 3000); + ret = bno055_ser_send_chunk(priv, hdr + 2, 2); + if (ret) + goto fail; + + if (read) + return 0; + + while (len) { + chunk_len = min(len, 2); + usleep_range(2000, 3000); + ret = bno055_ser_send_chunk(priv, data, chunk_len); + if (ret) + goto fail; + data += chunk_len; + len -= chunk_len; + } + + return 0; +fail: + /* waiting more than 30mS should clear the BNO055 internal state */ + usleep_range(40000, 50000); + return ret; +} + +static int bno055_ser_send_cmd(struct bno055_ser_priv *priv, + bool read, int addr, int len, const u8 *data) +{ + const int retry_max = 5; + int retry = retry_max; + int ret = 0; + + /* + * In case previous command was interrupted we still need to wait it to + * complete before we can issue new commands + */ + if (priv->cmd_stale) { + ret = wait_for_completion_interruptible_timeout(&priv->cmd_complete, + msecs_to_jiffies(100)); + if (ret == -ERESTARTSYS) + return -ERESTARTSYS; + + priv->cmd_stale = false; + /* if serial protocol broke, bail out */ + if (priv->cmd_status == STATUS_CRIT) + return -EIO; + } + + /* + * Try to convince the IMU to cooperate.. as explained in the comments + * at the top of this file, the IMU could also refuse the command (i.e. + * it is not ready yet); retry in this case. + */ + do { + mutex_lock(&priv->lock); + priv->expect_response = read ? CMD_READ : CMD_WRITE; + reinit_completion(&priv->cmd_complete); + mutex_unlock(&priv->lock); + + if (retry != retry_max) + trace_cmd_retry(read, addr, retry_max - retry); + ret = bno055_ser_do_send_cmd(priv, read, addr, len, data); + if (ret) + continue; + + ret = wait_for_completion_interruptible_timeout(&priv->cmd_complete, + msecs_to_jiffies(100)); + if (ret == -ERESTARTSYS) { + priv->cmd_stale = true; + return -ERESTARTSYS; + } + + if (!ret) + return -ETIMEDOUT; + + if (priv->cmd_status == STATUS_OK) + return 0; + if (priv->cmd_status == STATUS_CRIT) + return -EIO; + + /* loop in case priv->cmd_status == STATUS_FAIL */ + } while (--retry); + + if (ret < 0) + return ret; + if (priv->cmd_status == STATUS_FAIL) + return -EINVAL; + return 0; +} + +static int bno055_ser_write_reg(void *context, const void *_data, size_t count) +{ + const u8 *data = _data; + struct bno055_ser_priv *priv = context; + + if (count < 2) { + dev_err(&priv->serdev->dev, "Invalid write count %zu", count); + return -EINVAL; + } + + trace_write_reg(data[0], data[1]); + return bno055_ser_send_cmd(priv, 0, data[0], count - 1, data + 1); +} + +static int bno055_ser_read_reg(void *context, + const void *_reg, size_t reg_size, + void *val, size_t val_size) +{ + int ret; + int reg_addr; + const u8 *reg = _reg; + struct bno055_ser_priv *priv = context; + + if (val_size > 128) { + dev_err(&priv->serdev->dev, "Invalid read valsize %zu", val_size); + return -EINVAL; + } + + reg_addr = *reg; + trace_read_reg(reg_addr, val_size); + mutex_lock(&priv->lock); + priv->expected_data_len = val_size; + priv->response_buf = val; + mutex_unlock(&priv->lock); + + ret = bno055_ser_send_cmd(priv, 1, reg_addr, val_size, NULL); + + mutex_lock(&priv->lock); + priv->response_buf = NULL; + mutex_unlock(&priv->lock); + + return ret; +} + +/* + * Handler for received data; this is called from the receiver callback whenever + * it got some packet from the serial bus. The status tells us whether the + * packet is valid (i.e. header ok && received payload len consistent wrt the + * header). It's now our responsibility to check whether this is what we + * expected, of whether we got some unexpected, yet valid, packet. + */ +static void bno055_ser_handle_rx(struct bno055_ser_priv *priv, int status) +{ + mutex_lock(&priv->lock); + switch (priv->expect_response) { + case CMD_NONE: + dev_warn(&priv->serdev->dev, "received unexpected, yet valid, data from sensor"); + mutex_unlock(&priv->lock); + return; + + case CMD_READ: + priv->cmd_status = status; + if (status == STATUS_OK && + priv->rx.databuf_count != priv->expected_data_len) { + /* + * If we got here, then the lower layer serial protocol + * seems consistent with itself; if we got an unexpected + * amount of data then signal it as a non critical error + */ + priv->cmd_status = STATUS_FAIL; + dev_warn(&priv->serdev->dev, + "received an unexpected amount of, yet valid, data from sensor"); + } + break; + + case CMD_WRITE: + priv->cmd_status = status; + break; + } + + priv->expect_response = CMD_NONE; + mutex_unlock(&priv->lock); + complete(&priv->cmd_complete); +} + +/* + * Serdev receiver FSM. This tracks the serial communication and parse the + * header. It pushes packets to bno055_ser_handle_rx(), eventually communicating + * failures (i.e. malformed packets). + * Ideally it doesn't know anything about upper layer (i.e. if this is the + * packet we were really expecting), but since we copies the payload into the + * receiver buffer (that is not valid when i.e. we don't expect data), we + * snoop a bit in the upper layer.. + * Also, we assume to RX one pkt per time (i.e. the HW doesn't send anything + * unless we require to AND we don't queue more than one request per time). + */ +static int bno055_ser_receive_buf(struct serdev_device *serdev, + const unsigned char *buf, size_t size) +{ + int status; + struct bno055_ser_priv *priv = serdev_device_get_drvdata(serdev); + int remaining = size; + + if (size == 0) + return 0; + + trace_recv(size, buf); + switch (priv->rx.state) { + case RX_IDLE: + /* + * New packet. + * Check for its 1st byte that identifies the pkt type. + */ + if (buf[0] != 0xEE && buf[0] != 0xBB) { + dev_err(&priv->serdev->dev, + "Invalid packet start %x", buf[0]); + bno055_ser_handle_rx(priv, STATUS_CRIT); + break; + } + priv->rx.type = buf[0]; + priv->rx.state = RX_START; + remaining--; + buf++; + priv->rx.databuf_count = 0; + fallthrough; + + case RX_START: + /* + * Packet RX in progress, we expect either 1-byte len or 1-byte + * status depending by the packet type. + */ + if (remaining == 0) + break; + + if (priv->rx.type == 0xEE) { + if (remaining > 1) { + dev_err(&priv->serdev->dev, "EE pkt. Extra data received"); + status = STATUS_CRIT; + } else { + status = (buf[0] == 1) ? STATUS_OK : STATUS_FAIL; + } + bno055_ser_handle_rx(priv, status); + priv->rx.state = RX_IDLE; + break; + + } else { + /*priv->rx.type == 0xBB */ + priv->rx.state = RX_DATA; + priv->rx.expected_len = buf[0]; + remaining--; + buf++; + } + fallthrough; + + case RX_DATA: + /* Header parsed; now receiving packet data payload */ + if (remaining == 0) + break; + + if (priv->rx.databuf_count + remaining > priv->rx.expected_len) { + /* + * This is an inconsistency in serial protocol, we lost + * sync and we don't know how to handle further data + */ + dev_err(&priv->serdev->dev, "BB pkt. Extra data received"); + bno055_ser_handle_rx(priv, STATUS_CRIT); + priv->rx.state = RX_IDLE; + break; + } + + mutex_lock(&priv->lock); + /* + * NULL e.g. when read cmd is stale or when no read cmd is + * actually pending. + */ + if (priv->response_buf && + /* + * Snoop on the upper layer protocol stuff to make sure not + * to write to an invalid memory. Apart for this, let's the + * upper layer manage any inconsistency wrt expected data + * len (as long as the serial protocol is consistent wrt + * itself (i.e. response header is consistent with received + * response len. + */ + (priv->rx.databuf_count + remaining <= priv->expected_data_len)) + memcpy(priv->response_buf + priv->rx.databuf_count, + buf, remaining); + mutex_unlock(&priv->lock); + + priv->rx.databuf_count += remaining; + + /* + * Reached expected len advertised by the IMU for the current + * packet. Pass it to the upper layer (for us it is just valid). + */ + if (priv->rx.databuf_count == priv->rx.expected_len) { + bno055_ser_handle_rx(priv, STATUS_OK); + priv->rx.state = RX_IDLE; + } + break; + } + + return size; +} + +static const struct serdev_device_ops bno055_ser_serdev_ops = { + .receive_buf = bno055_ser_receive_buf, + .write_wakeup = serdev_device_write_wakeup, +}; + +static struct regmap_bus bno055_ser_regmap_bus = { + .write = bno055_ser_write_reg, + .read = bno055_ser_read_reg, +}; + +static int bno055_ser_probe(struct serdev_device *serdev) +{ + struct bno055_ser_priv *priv; + struct regmap *regmap; + int ret; + + priv = devm_kzalloc(&serdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + serdev_device_set_drvdata(serdev, priv); + priv->serdev = serdev; + mutex_init(&priv->lock); + init_completion(&priv->cmd_complete); + + serdev_device_set_client_ops(serdev, &bno055_ser_serdev_ops); + ret = devm_serdev_device_open(&serdev->dev, serdev); + if (ret) + return ret; + + if (serdev_device_set_baudrate(serdev, 115200) != 115200) { + dev_err(&serdev->dev, "Cannot set required baud rate"); + return -EIO; + } + + ret = serdev_device_set_parity(serdev, SERDEV_PARITY_NONE); + if (ret) { + dev_err(&serdev->dev, "Cannot set required parity setting"); + return ret; + } + serdev_device_set_flow_control(serdev, false); + + regmap = devm_regmap_init(&serdev->dev, &bno055_ser_regmap_bus, + priv, &bno055_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(&serdev->dev, PTR_ERR(regmap), + "Unable to init register map"); + + return bno055_probe(&serdev->dev, regmap, + BNO055_SER_XFER_BURST_BREAK_THRESHOLD, false); +} + +static const struct of_device_id bno055_ser_of_match[] = { + { .compatible = "bosch,bno055" }, + { } +}; +MODULE_DEVICE_TABLE(of, bno055_ser_of_match); + +static struct serdev_device_driver bno055_ser_driver = { + .driver = { + .name = "bno055-ser", + .of_match_table = bno055_ser_of_match, + }, + .probe = bno055_ser_probe, +}; +module_serdev_device_driver(bno055_ser_driver); + +MODULE_AUTHOR("Andrea Merello "); +MODULE_DESCRIPTION("Bosch BNO055 serdev interface"); +MODULE_IMPORT_NS(IIO_BNO055); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/imu/bno055/bno055_ser_trace.c b/drivers/iio/imu/bno055/bno055_ser_trace.c new file mode 100644 index 0000000000000000000000000000000000000000..48397b66daef022aec0e617515f51c578835a275 --- /dev/null +++ b/drivers/iio/imu/bno055/bno055_ser_trace.c @@ -0,0 +1,14 @@ +//SPDX-License-Identifier: GPL-2.0 + +/* + * bno055_ser Trace Support + * Copyright (C) 2022 Istituto Italiano di Tecnologia + * Electronic Design Laboratory + * + * Based on: + * Device core Trace Support + * Copyright (C) 2021, Intel Corporation + */ + +#define CREATE_TRACE_POINTS +#include "bno055_ser_trace.h" diff --git a/drivers/iio/imu/bno055/bno055_ser_trace.h b/drivers/iio/imu/bno055/bno055_ser_trace.h new file mode 100644 index 0000000000000000000000000000000000000000..7d9eae166eec068d0762414a30ea81bc2d8fbb58 --- /dev/null +++ b/drivers/iio/imu/bno055/bno055_ser_trace.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#if !defined(__BNO055_SERDEV_TRACE_H__) || defined(TRACE_HEADER_MULTI_READ) +#define __BNO055_SERDEV_TRACE_H__ + +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM bno055_ser + +TRACE_EVENT(send_chunk, + TP_PROTO(int len, const u8 *data), + TP_ARGS(len, data), + TP_STRUCT__entry( + __field(int, len) + __dynamic_array(u8, chunk, len) + ), + TP_fast_assign( + __entry->len = len; + memcpy(__get_dynamic_array(chunk), + data, __entry->len); + ), + TP_printk("len: %d, data: = %*ph", + __entry->len, __entry->len, __get_dynamic_array(chunk) + ) +); + +TRACE_EVENT(cmd_retry, + TP_PROTO(bool read, int addr, int retry), + TP_ARGS(read, addr, retry), + TP_STRUCT__entry( + __field(bool, read) + __field(int, addr) + __field(int, retry) + ), + TP_fast_assign( + __entry->read = read; + __entry->addr = addr; + __entry->retry = retry; + ), + TP_printk("%s addr 0x%x retry #%d", + __entry->read ? "read" : "write", + __entry->addr, __entry->retry + ) +); + +TRACE_EVENT(write_reg, + TP_PROTO(u8 addr, u8 value), + TP_ARGS(addr, value), + TP_STRUCT__entry( + __field(u8, addr) + __field(u8, value) + ), + TP_fast_assign( + __entry->addr = addr; + __entry->value = value; + ), + TP_printk("reg 0x%x = 0x%x", + __entry->addr, __entry->value + ) +); + +TRACE_EVENT(read_reg, + TP_PROTO(int addr, size_t len), + TP_ARGS(addr, len), + TP_STRUCT__entry( + __field(int, addr) + __field(size_t, len) + ), + TP_fast_assign( + __entry->addr = addr; + __entry->len = len; + ), + TP_printk("reg 0x%x (len %zu)", + __entry->addr, __entry->len + ) +); + +TRACE_EVENT(recv, + TP_PROTO(size_t len, const unsigned char *buf), + TP_ARGS(len, buf), + TP_STRUCT__entry( + __field(size_t, len) + __dynamic_array(unsigned char, buf, len) + ), + TP_fast_assign( + __entry->len = len; + memcpy(__get_dynamic_array(buf), + buf, __entry->len); + ), + TP_printk("len: %zu, data: = %*ph", + __entry->len, (int)__entry->len, __get_dynamic_array(buf) + ) +); + +#endif /* __BNO055_SERDEV_TRACE_H__ || TRACE_HEADER_MULTI_READ */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE bno055_ser_trace + +/* This part must be outside protection */ +#include diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c index 9b4298095d3f0ae39a05bbe6d5ba0eb8e300b3f9..f7bce428d9eb45c2e68add02ee31697241dabfc5 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_acpi.c @@ -65,7 +65,7 @@ static int asus_acpi_get_sensor_info(struct acpi_device *adev, sub_elem = &elem->package.elements[j]; if (sub_elem->type == ACPI_TYPE_STRING) - strlcpy(info->type, sub_elem->string.pointer, + strscpy(info->type, sub_elem->string.pointer, sizeof(info->type)); else if (sub_elem->type == ACPI_TYPE_INTEGER) { if (sub_elem->integer.value != client->addr) { @@ -158,7 +158,7 @@ int inv_mpu_acpi_create_mux_client(struct i2c_client *client) char *name; info.addr = secondary; - strlcpy(info.type, dev_name(&adev->dev), + strscpy(info.type, dev_name(&adev->dev), sizeof(info.type)); name = strchr(info.type, ':'); if (name) diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c index 2aa647704a794e99446c0518470a3b7db7d43c1b..14255a918eb1d797abc38f047d5fabfd8aba65bc 100644 --- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c +++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c @@ -157,7 +157,7 @@ out_del_mux: return result; } -static int inv_mpu_remove(struct i2c_client *client) +static void inv_mpu_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct inv_mpu6050_state *st = iio_priv(indio_dev); @@ -166,8 +166,6 @@ static int inv_mpu_remove(struct i2c_client *client) inv_mpu_acpi_delete_mux_client(client); i2c_mux_del_adapters(st->muxc); } - - return 0; } /* diff --git a/drivers/iio/imu/kmx61.c b/drivers/iio/imu/kmx61.c index ec23b1ee472bd10167f3202fdfe66d01a63b5b99..b10c0dcac0bbec3f4870d38b625bf927cdd02d8a 100644 --- a/drivers/iio/imu/kmx61.c +++ b/drivers/iio/imu/kmx61.c @@ -1418,7 +1418,7 @@ err_chip_uninit: return ret; } -static int kmx61_remove(struct i2c_client *client) +static void kmx61_remove(struct i2c_client *client) { struct kmx61_data *data = i2c_get_clientdata(client); @@ -1439,8 +1439,6 @@ static int kmx61_remove(struct i2c_client *client) mutex_lock(&data->lock); kmx61_set_mode(data, KMX61_ALL_STBY, KMX61_ACC | KMX61_MAG, true); mutex_unlock(&data->lock); - - return 0; } static int kmx61_suspend(struct device *dev) diff --git a/drivers/iio/imu/st_lsm6dsx/Kconfig b/drivers/iio/imu/st_lsm6dsx/Kconfig index fefd0b9391000ade95999df09d92d662479f4539..2ed2b3f40c0b15524756d3ce5c5dc089a347addf 100644 --- a/drivers/iio/imu/st_lsm6dsx/Kconfig +++ b/drivers/iio/imu/st_lsm6dsx/Kconfig @@ -12,7 +12,7 @@ config IIO_ST_LSM6DSX Say yes here to build support for STMicroelectronics LSM6DSx imu sensor. Supported devices: lsm6ds3, lsm6ds3h, lsm6dsl, lsm6dsm, ism330dlc, lsm6dso, lsm6dsox, asm330lhh, asm330lhhx, lsm6dsr, - lsm6ds3tr-c, ism330dhcx, lsm6dsrx, lsm6ds0, lsm6dsop, + lsm6ds3tr-c, ism330dhcx, lsm6dsrx, lsm6ds0, lsm6dsop, lsm6dstx, the accelerometer/gyroscope of lsm9ds1 and lsm6dst. To compile this driver as a module, choose M here: the module diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h index a86dd29a473818a135ae70f8877dcddb85fc4328..6b57d47be69ea9e1ef6dec4384fe97e07c877d64 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h @@ -32,6 +32,7 @@ #define ST_LSM6DST_DEV_NAME "lsm6dst" #define ST_LSM6DSOP_DEV_NAME "lsm6dsop" #define ST_ASM330LHHX_DEV_NAME "asm330lhhx" +#define ST_LSM6DSTX_DEV_NAME "lsm6dstx" enum st_lsm6dsx_hw_id { ST_LSM6DS3_ID, @@ -51,6 +52,7 @@ enum st_lsm6dsx_hw_id { ST_LSM6DST_ID, ST_LSM6DSOP_ID, ST_ASM330LHHX_ID, + ST_LSM6DSTX_ID, ST_LSM6DSX_MAX_ID, }; diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c index c7d3730ab1c50a632cc75d73ac3c5332bdeeb066..e49f2d120ed38bf9558e37e789619f96c5f9d58f 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c @@ -15,7 +15,7 @@ * value of the decimation factor and ODR set for each FIFO data set. * * LSM6DSO/LSM6DSOX/ASM330LHH/ASM330LHHX/LSM6DSR/LSM6DSRX/ISM330DHCX/ - * LSM6DST/LSM6DSOP: + * LSM6DST/LSM6DSOP/LSM6DSTX: * The FIFO buffer can be configured to store data from gyroscope and * accelerometer. Each sample is queued with a tag (1B) indicating data * source (gyroscope, accelerometer, hw timer). diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c index b5e4a4113652c032d086e8827f66f1d3eef8e03f..f8bbb005718edaa17a0dfdf167e20920d7e4695b 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c @@ -26,7 +26,8 @@ * - Gyroscope supported full-scale [dps]: +-125/+-245/+-500/+-1000/+-2000 * - FIFO size: 4KB * - * - LSM6DSO/LSM6DSOX/ASM330LHH/ASM330LHHX/LSM6DSR/ISM330DHCX/LSM6DST/LSM6DSOP: + * - LSM6DSO/LSM6DSOX/ASM330LHH/ASM330LHHX/LSM6DSR/ISM330DHCX/LSM6DST/LSM6DSOP/ + * LSM6DSTX: * - Accelerometer/Gyroscope supported ODR [Hz]: 12.5, 26, 52, 104, 208, 416, * 833 * - Accelerometer supported full-scale [g]: +-2/+-4/+-8/+-16 @@ -791,6 +792,10 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { .hw_id = ST_ASM330LHHX_ID, .name = ST_ASM330LHHX_DEV_NAME, .wai = 0x6b, + }, { + .hw_id = ST_LSM6DSTX_ID, + .name = ST_LSM6DSTX_DEV_NAME, + .wai = 0x6d, }, }, .channels = { diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c index 2ea34c0d3a8cdfe9a8f6a7994a0195056711d29d..307c8c436862aa31de1a93339423131fcba4a231 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_i2c.c @@ -105,6 +105,10 @@ static const struct of_device_id st_lsm6dsx_i2c_of_match[] = { .compatible = "st,asm330lhhx", .data = (void *)ST_ASM330LHHX_ID, }, + { + .compatible = "st,lsm6dstx", + .data = (void *)ST_LSM6DSTX_ID, + }, {}, }; MODULE_DEVICE_TABLE(of, st_lsm6dsx_i2c_of_match); @@ -127,6 +131,7 @@ static const struct i2c_device_id st_lsm6dsx_i2c_id_table[] = { { ST_LSM6DST_DEV_NAME, ST_LSM6DST_ID }, { ST_LSM6DSOP_DEV_NAME, ST_LSM6DSOP_ID }, { ST_ASM330LHHX_DEV_NAME, ST_ASM330LHHX_ID }, + { ST_LSM6DSTX_DEV_NAME, ST_LSM6DSTX_ID }, {}, }; MODULE_DEVICE_TABLE(i2c, st_lsm6dsx_i2c_id_table); diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c index 6a8883f022a8e84dc836b134cad30ea366693ff9..6a4eecf4bb050b8e0a92ce9e392ed7045c224f5c 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_spi.c @@ -105,6 +105,10 @@ static const struct of_device_id st_lsm6dsx_spi_of_match[] = { .compatible = "st,asm330lhhx", .data = (void *)ST_ASM330LHHX_ID, }, + { + .compatible = "st,lsm6dstx", + .data = (void *)ST_LSM6DSTX_ID, + }, {}, }; MODULE_DEVICE_TABLE(of, st_lsm6dsx_spi_of_match); @@ -127,6 +131,7 @@ static const struct spi_device_id st_lsm6dsx_spi_id_table[] = { { ST_LSM6DST_DEV_NAME, ST_LSM6DST_ID }, { ST_LSM6DSOP_DEV_NAME, ST_LSM6DSOP_ID }, { ST_ASM330LHHX_DEV_NAME, ST_ASM330LHHX_ID }, + { ST_LSM6DSTX_DEV_NAME, ST_LSM6DSTX_ID }, {}, }; MODULE_DEVICE_TABLE(spi, st_lsm6dsx_spi_id_table); diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index acc2b6c05d57c2ae8f28f0d1cbeb18ad4276bd7c..228598b82a2f3609527edef61f4f5351c8ce5db7 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -843,8 +843,8 @@ static int iio_verify_update(struct iio_dev *indio_dev, * to verify. */ if (remove_buffer && !insert_buffer && - list_is_singular(&iio_dev_opaque->buffer_list)) - return 0; + list_is_singular(&iio_dev_opaque->buffer_list)) + return 0; modes = indio_dev->modes; @@ -940,6 +940,7 @@ struct iio_demux_table { static void iio_buffer_demux_free(struct iio_buffer *buffer) { struct iio_demux_table *p, *q; + list_for_each_entry_safe(p, q, &buffer->demux_list, l) { list_del(&p->l); kfree(p); diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c index 0f4dbda3b9d3622d9c0677d0e9d8a95c5016c212..151ff39933548c73b3362b89e230ad7bb30c807d 100644 --- a/drivers/iio/industrialio-core.c +++ b/drivers/iio/industrialio-core.c @@ -134,6 +134,12 @@ static const char * const iio_modifier_names[] = { [IIO_MOD_ETHANOL] = "ethanol", [IIO_MOD_H2] = "h2", [IIO_MOD_O2] = "o2", + [IIO_MOD_LINEAR_X] = "linear_x", + [IIO_MOD_LINEAR_Y] = "linear_y", + [IIO_MOD_LINEAR_Z] = "linear_z", + [IIO_MOD_PITCH] = "pitch", + [IIO_MOD_YAW] = "yaw", + [IIO_MOD_ROLL] = "roll", }; /* relies on pairs of these shared then separate */ @@ -168,6 +174,7 @@ static const char * const iio_chan_info_postfix[] = { [IIO_CHAN_INFO_OVERSAMPLING_RATIO] = "oversampling_ratio", [IIO_CHAN_INFO_THERMOCOUPLE_TYPE] = "thermocouple_type", [IIO_CHAN_INFO_CALIBAMBIENT] = "calibambient", + [IIO_CHAN_INFO_ZEROPOINT] = "zeropoint", }; /** * iio_device_id() - query the unique ID for the device @@ -236,6 +243,7 @@ static int iio_sysfs_match_string_with_gaps(const char * const *array, size_t n, struct dentry *iio_get_debugfs_dentry(struct iio_dev *indio_dev) { struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev); + return iio_dev_opaque->debugfs_dentry; } EXPORT_SYMBOL_GPL(iio_get_debugfs_dentry); @@ -447,6 +455,7 @@ static const struct file_operations iio_debugfs_reg_fops = { static void iio_device_unregister_debugfs(struct iio_dev *indio_dev) { struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev); + debugfs_remove_recursive(iio_dev_opaque->debugfs_dentry); } @@ -1021,6 +1030,7 @@ int __iio_device_attr_init(struct device_attribute *dev_attr, int ret = 0; char *name = NULL; char *full_postfix; + sysfs_attr_init(&dev_attr->attr); /* Build up postfix of __postfix */ @@ -1299,8 +1309,7 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev, ret = iio_device_add_info_mask_type_avail(indio_dev, chan, IIO_SEPARATE, - &chan-> - info_mask_separate_available); + &chan->info_mask_separate_available); if (ret < 0) return ret; attrcount += ret; @@ -1314,8 +1323,7 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev, ret = iio_device_add_info_mask_type_avail(indio_dev, chan, IIO_SHARED_BY_TYPE, - &chan-> - info_mask_shared_by_type_available); + &chan->info_mask_shared_by_type_available); if (ret < 0) return ret; attrcount += ret; @@ -1355,6 +1363,7 @@ static int iio_device_add_channel_sysfs(struct iio_dev *indio_dev, if (chan->ext_info) { unsigned int i = 0; + for (ext_info = chan->ext_info; ext_info->name; ext_info++) { ret = __iio_add_chan_devattr(ext_info->name, chan, @@ -1403,6 +1412,7 @@ static ssize_t name_show(struct device *dev, struct device_attribute *attr, char *buf) { struct iio_dev *indio_dev = dev_to_iio_dev(dev); + return sysfs_emit(buf, "%s\n", indio_dev->name); } @@ -1412,6 +1422,7 @@ static ssize_t label_show(struct device *dev, struct device_attribute *attr, char *buf) { struct iio_dev *indio_dev = dev_to_iio_dev(dev); + return sysfs_emit(buf, "%s\n", indio_dev->label); } @@ -1565,7 +1576,7 @@ static int iio_device_register_sysfs(struct iio_dev *indio_dev) ret = -ENOMEM; goto error_clear_attrs; } - /* Copy across original attributes */ + /* Copy across original attributes, and point to original binary attributes */ if (indio_dev->info->attrs) { memcpy(iio_dev_opaque->chan_attr_group.attrs, indio_dev->info->attrs->attrs, @@ -1573,6 +1584,8 @@ static int iio_device_register_sysfs(struct iio_dev *indio_dev) *attrcount_orig); iio_dev_opaque->chan_attr_group.is_visible = indio_dev->info->attrs->is_visible; + iio_dev_opaque->chan_attr_group.bin_attrs = + indio_dev->info->attrs->bin_attrs; } attrn = attrcount_orig; /* Add all elements from the list. */ @@ -1621,6 +1634,8 @@ static void iio_dev_release(struct device *device) iio_device_detach_buffers(indio_dev); + lockdep_unregister_key(&iio_dev_opaque->mlock_key); + ida_free(&iio_ida, iio_dev_opaque->id); kfree(iio_dev_opaque); } @@ -1680,6 +1695,9 @@ struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv) INIT_LIST_HEAD(&iio_dev_opaque->buffer_list); INIT_LIST_HEAD(&iio_dev_opaque->ioctl_handlers); + lockdep_register_key(&iio_dev_opaque->mlock_key); + lockdep_set_class(&indio_dev->mlock, &iio_dev_opaque->mlock_key); + return indio_dev; } EXPORT_SYMBOL(iio_device_alloc); @@ -1777,6 +1795,7 @@ static int iio_chrdev_release(struct inode *inode, struct file *filp) struct iio_dev_opaque *iio_dev_opaque = container_of(inode->i_cdev, struct iio_dev_opaque, chrdev); struct iio_dev *indio_dev = &iio_dev_opaque->indio_dev; + kfree(ib); clear_bit(IIO_BUSY_BIT_POS, &iio_dev_opaque->flags); iio_device_put(indio_dev); diff --git a/drivers/iio/industrialio-event.c b/drivers/iio/industrialio-event.c index b5e059e15b0aeedd72e19e87575350f26ac00ab4..3d78da2531a9a2c7d5992d2fb5e762341ab4309b 100644 --- a/drivers/iio/industrialio-event.c +++ b/drivers/iio/industrialio-event.c @@ -231,12 +231,15 @@ static const char * const iio_ev_type_text[] = { [IIO_EV_TYPE_MAG_ADAPTIVE] = "mag_adaptive", [IIO_EV_TYPE_CHANGE] = "change", [IIO_EV_TYPE_MAG_REFERENCED] = "mag_referenced", + [IIO_EV_TYPE_GESTURE] = "gesture", }; static const char * const iio_ev_dir_text[] = { [IIO_EV_DIR_EITHER] = "either", [IIO_EV_DIR_RISING] = "rising", - [IIO_EV_DIR_FALLING] = "falling" + [IIO_EV_DIR_FALLING] = "falling", + [IIO_EV_DIR_SINGLETAP] = "singletap", + [IIO_EV_DIR_DOUBLETAP] = "doubletap", }; static const char * const iio_ev_info_text[] = { @@ -247,6 +250,8 @@ static const char * const iio_ev_info_text[] = { [IIO_EV_INFO_HIGH_PASS_FILTER_3DB] = "high_pass_filter_3db", [IIO_EV_INFO_LOW_PASS_FILTER_3DB] = "low_pass_filter_3db", [IIO_EV_INFO_TIMEOUT] = "timeout", + [IIO_EV_INFO_RESET_TIMEOUT] = "reset_timeout", + [IIO_EV_INFO_TAP2_MIN_DELAY] = "tap2_min_delay", }; static enum iio_event_direction iio_ev_attr_dir(struct iio_dev_attr *attr) @@ -354,9 +359,10 @@ static int iio_device_add_event(struct iio_dev *indio_dev, enum iio_shared_by shared_by, const unsigned long *mask) { struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev); - ssize_t (*show)(struct device *, struct device_attribute *, char *); - ssize_t (*store)(struct device *, struct device_attribute *, - const char *, size_t); + ssize_t (*show)(struct device *dev, struct device_attribute *attr, + char *buf); + ssize_t (*store)(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len); unsigned int attrcount = 0; unsigned int i; char *postfix; diff --git a/drivers/iio/industrialio-trigger.c b/drivers/iio/industrialio-trigger.c index b78814d869b7f31afbef6800e35a2d084b536377..6885a186fe27ad8bd100524aa7f9ab1aebfa4335 100644 --- a/drivers/iio/industrialio-trigger.c +++ b/drivers/iio/industrialio-trigger.c @@ -50,6 +50,7 @@ static ssize_t name_show(struct device *dev, struct device_attribute *attr, char *buf) { struct iio_trigger *trig = to_iio_trigger(dev); + return sysfs_emit(buf, "%s\n", trig->name); } diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c index df74765d33dcb50172c89f01c81a20746f52bd2c..872fd5c241476eadb4ae4062ac3f59a61a0da970 100644 --- a/drivers/iio/inkern.c +++ b/drivers/iio/inkern.c @@ -5,9 +5,9 @@ */ #include #include +#include #include #include -#include #include #include @@ -45,13 +45,13 @@ int iio_map_array_register(struct iio_dev *indio_dev, struct iio_map *maps) int i = 0, ret = 0; struct iio_map_internal *mapi; - if (maps == NULL) + if (!maps) return 0; mutex_lock(&iio_map_list_lock); - while (maps[i].consumer_dev_name != NULL) { + while (maps[i].consumer_dev_name) { mapi = kzalloc(sizeof(*mapi), GFP_KERNEL); - if (mapi == NULL) { + if (!mapi) { ret = -ENOMEM; goto error_ret; } @@ -69,7 +69,6 @@ error_ret: } EXPORT_SYMBOL_GPL(iio_map_array_register); - /* * Remove all map entries associated with the given iio device */ @@ -117,15 +116,8 @@ static const struct iio_chan_spec return chan; } -#ifdef CONFIG_OF - -static int iio_dev_node_match(struct device *dev, const void *data) -{ - return dev->of_node == data && dev->type == &iio_device_type; -} - /** - * __of_iio_simple_xlate - translate iiospec to the IIO channel index + * __fwnode_iio_simple_xlate - translate iiospec to the IIO channel index * @indio_dev: pointer to the iio_dev structure * @iiospec: IIO specifier as found in the device tree * @@ -134,14 +126,14 @@ static int iio_dev_node_match(struct device *dev, const void *data) * whether IIO index is less than num_channels (that is specified in the * iio_dev). */ -static int __of_iio_simple_xlate(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec) +static int __fwnode_iio_simple_xlate(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec) { - if (!iiospec->args_count) + if (!iiospec->nargs) return 0; if (iiospec->args[0] >= indio_dev->num_channels) { - dev_err(&indio_dev->dev, "invalid channel index %u\n", + dev_err(&indio_dev->dev, "invalid channel index %llu\n", iiospec->args[0]); return -EINVAL; } @@ -149,32 +141,33 @@ static int __of_iio_simple_xlate(struct iio_dev *indio_dev, return iiospec->args[0]; } -static int __of_iio_channel_get(struct iio_channel *channel, - struct device_node *np, int index) +static int __fwnode_iio_channel_get(struct iio_channel *channel, + struct fwnode_handle *fwnode, int index) { + struct fwnode_reference_args iiospec; struct device *idev; struct iio_dev *indio_dev; int err; - struct of_phandle_args iiospec; - err = of_parse_phandle_with_args(np, "io-channels", - "#io-channel-cells", - index, &iiospec); + err = fwnode_property_get_reference_args(fwnode, "io-channels", + "#io-channel-cells", 0, + index, &iiospec); if (err) return err; - idev = bus_find_device(&iio_bus_type, NULL, iiospec.np, - iio_dev_node_match); - of_node_put(iiospec.np); - if (idev == NULL) + idev = bus_find_device_by_fwnode(&iio_bus_type, iiospec.fwnode); + if (!idev) { + fwnode_handle_put(iiospec.fwnode); return -EPROBE_DEFER; + } indio_dev = dev_to_iio_dev(idev); channel->indio_dev = indio_dev; - if (indio_dev->info->of_xlate) - index = indio_dev->info->of_xlate(indio_dev, &iiospec); + if (indio_dev->info->fwnode_xlate) + index = indio_dev->info->fwnode_xlate(indio_dev, &iiospec); else - index = __of_iio_simple_xlate(indio_dev, &iiospec); + index = __fwnode_iio_simple_xlate(indio_dev, &iiospec); + fwnode_handle_put(iiospec.fwnode); if (index < 0) goto err_put; channel->channel = &indio_dev->channels[index]; @@ -186,7 +179,8 @@ err_put: return index; } -static struct iio_channel *of_iio_channel_get(struct device_node *np, int index) +static struct iio_channel *fwnode_iio_channel_get(struct fwnode_handle *fwnode, + int index) { struct iio_channel *channel; int err; @@ -195,10 +189,10 @@ static struct iio_channel *of_iio_channel_get(struct device_node *np, int index) return ERR_PTR(-EINVAL); channel = kzalloc(sizeof(*channel), GFP_KERNEL); - if (channel == NULL) + if (!channel) return ERR_PTR(-ENOMEM); - err = __of_iio_channel_get(channel, np, index); + err = __fwnode_iio_channel_get(channel, fwnode, index); if (err) goto err_free_channel; @@ -209,74 +203,116 @@ err_free_channel: return ERR_PTR(err); } -struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, - const char *name) +static struct iio_channel * +__fwnode_iio_channel_get_by_name(struct fwnode_handle *fwnode, const char *name) { - struct iio_channel *chan = NULL; - - /* Walk up the tree of devices looking for a matching iio channel */ - while (np) { - int index = 0; - + struct iio_channel *chan; + int index = 0; + + /* + * For named iio channels, first look up the name in the + * "io-channel-names" property. If it cannot be found, the + * index will be an error code, and fwnode_iio_channel_get() + * will fail. + */ + if (name) + index = fwnode_property_match_string(fwnode, "io-channel-names", + name); + + chan = fwnode_iio_channel_get(fwnode, index); + if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER) + return chan; + if (name) { + if (index >= 0) { + pr_err("ERROR: could not get IIO channel %pfw:%s(%i)\n", + fwnode, name, index); + /* + * In this case, we found 'name' in 'io-channel-names' + * but somehow we still fail so that we should not proceed + * with any other lookup. Hence, explicitly return -EINVAL + * (maybe not the better error code) so that the caller + * won't do a system lookup. + */ + return ERR_PTR(-EINVAL); + } /* - * For named iio channels, first look up the name in the - * "io-channel-names" property. If it cannot be found, the - * index will be an error code, and of_iio_channel_get() - * will fail. + * If index < 0, then fwnode_property_get_reference_args() fails + * with -EINVAL or -ENOENT (ACPI case) which is expected. We + * should not proceed if we get any other error. */ - if (name) - index = of_property_match_string(np, "io-channel-names", - name); - chan = of_iio_channel_get(np, index); - if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER) - break; - else if (name && index >= 0) { - pr_err("ERROR: could not get IIO channel %pOF:%s(%i)\n", - np, name ? name : "", index); - return NULL; - } - + if (PTR_ERR(chan) != -EINVAL && PTR_ERR(chan) != -ENOENT) + return chan; + } else if (PTR_ERR(chan) != -ENOENT) { /* - * No matching IIO channel found on this node. - * If the parent node has a "io-channel-ranges" property, - * then we can try one of its channels. + * if !name, then we should only proceed the lookup if + * fwnode_property_get_reference_args() returns -ENOENT. */ - np = np->parent; - if (np && !of_get_property(np, "io-channel-ranges", NULL)) - return NULL; + return chan; } - return chan; + /* so we continue the lookup */ + return ERR_PTR(-ENODEV); } -EXPORT_SYMBOL_GPL(of_iio_channel_get_by_name); -static struct iio_channel *of_iio_channel_get_all(struct device *dev) +struct iio_channel *fwnode_iio_channel_get_by_name(struct fwnode_handle *fwnode, + const char *name) { + struct fwnode_handle *parent; + struct iio_channel *chan; + + /* Walk up the tree of devices looking for a matching iio channel */ + chan = __fwnode_iio_channel_get_by_name(fwnode, name); + if (!IS_ERR(chan) || PTR_ERR(chan) != -ENODEV) + return chan; + + /* + * No matching IIO channel found on this node. + * If the parent node has a "io-channel-ranges" property, + * then we can try one of its channels. + */ + fwnode_for_each_parent_node(fwnode, parent) { + if (!fwnode_property_present(parent, "io-channel-ranges")) { + fwnode_handle_put(parent); + return ERR_PTR(-ENODEV); + } + + chan = __fwnode_iio_channel_get_by_name(fwnode, name); + if (!IS_ERR(chan) || PTR_ERR(chan) != -ENODEV) { + fwnode_handle_put(parent); + return chan; + } + } + + return ERR_PTR(-ENODEV); +} +EXPORT_SYMBOL_GPL(fwnode_iio_channel_get_by_name); + +static struct iio_channel *fwnode_iio_channel_get_all(struct device *dev) +{ + struct fwnode_handle *fwnode = dev_fwnode(dev); struct iio_channel *chans; int i, mapind, nummaps = 0; int ret; do { - ret = of_parse_phandle_with_args(dev->of_node, - "io-channels", - "#io-channel-cells", - nummaps, NULL); + ret = fwnode_property_get_reference_args(fwnode, "io-channels", + "#io-channel-cells", 0, + nummaps, NULL); if (ret < 0) break; } while (++nummaps); - if (nummaps == 0) /* no error, return NULL to search map table */ - return NULL; + if (nummaps == 0) + return ERR_PTR(-ENODEV); /* NULL terminated array to save passing size */ chans = kcalloc(nummaps + 1, sizeof(*chans), GFP_KERNEL); - if (chans == NULL) + if (!chans) return ERR_PTR(-ENOMEM); - /* Search for OF matches */ + /* Search for FW matches */ for (mapind = 0; mapind < nummaps; mapind++) { - ret = __of_iio_channel_get(&chans[mapind], dev->of_node, - mapind); + ret = __fwnode_iio_channel_get(&chans[mapind], fwnode, mapind); if (ret) goto error_free_chans; } @@ -289,15 +325,6 @@ error_free_chans: return ERR_PTR(ret); } -#else /* CONFIG_OF */ - -static inline struct iio_channel *of_iio_channel_get_all(struct device *dev) -{ - return NULL; -} - -#endif /* CONFIG_OF */ - static struct iio_channel *iio_channel_get_sys(const char *name, const char *channel_name) { @@ -305,7 +332,7 @@ static struct iio_channel *iio_channel_get_sys(const char *name, struct iio_channel *channel; int err; - if (name == NULL && channel_name == NULL) + if (!(name || channel_name)) return ERR_PTR(-ENODEV); /* first find matching entry the channel map */ @@ -320,11 +347,11 @@ static struct iio_channel *iio_channel_get_sys(const char *name, break; } mutex_unlock(&iio_map_list_lock); - if (c == NULL) + if (!c) return ERR_PTR(-ENODEV); channel = kzalloc(sizeof(*channel), GFP_KERNEL); - if (channel == NULL) { + if (!channel) { err = -ENOMEM; goto error_no_mem; } @@ -336,7 +363,7 @@ static struct iio_channel *iio_channel_get_sys(const char *name, iio_chan_spec_from_name(channel->indio_dev, c->map->adc_channel_label); - if (channel->channel == NULL) { + if (!channel->channel) { err = -EINVAL; goto error_no_chan; } @@ -358,9 +385,9 @@ struct iio_channel *iio_channel_get(struct device *dev, struct iio_channel *channel; if (dev) { - channel = of_iio_channel_get_by_name(dev->of_node, - channel_name); - if (channel != NULL) + channel = fwnode_iio_channel_get_by_name(dev_fwnode(dev), + channel_name); + if (!IS_ERR(channel) || PTR_ERR(channel) != -ENODEV) return channel; } @@ -400,14 +427,14 @@ struct iio_channel *devm_iio_channel_get(struct device *dev, } EXPORT_SYMBOL_GPL(devm_iio_channel_get); -struct iio_channel *devm_of_iio_channel_get_by_name(struct device *dev, - struct device_node *np, - const char *channel_name) +struct iio_channel *devm_fwnode_iio_channel_get_by_name(struct device *dev, + struct fwnode_handle *fwnode, + const char *channel_name) { struct iio_channel *channel; int ret; - channel = of_iio_channel_get_by_name(np, channel_name); + channel = fwnode_iio_channel_get_by_name(fwnode, channel_name); if (IS_ERR(channel)) return channel; @@ -417,7 +444,7 @@ struct iio_channel *devm_of_iio_channel_get_by_name(struct device *dev, return channel; } -EXPORT_SYMBOL_GPL(devm_of_iio_channel_get_by_name); +EXPORT_SYMBOL_GPL(devm_fwnode_iio_channel_get_by_name); struct iio_channel *iio_channel_get_all(struct device *dev) { @@ -428,11 +455,15 @@ struct iio_channel *iio_channel_get_all(struct device *dev) int mapind = 0; int i, ret; - if (dev == NULL) + if (!dev) return ERR_PTR(-EINVAL); - chans = of_iio_channel_get_all(dev); - if (chans) + chans = fwnode_iio_channel_get_all(dev); + /* + * We only want to carry on if the error is -ENODEV. Anything else + * should be reported up the stack. + */ + if (!IS_ERR(chans) || PTR_ERR(chans) != -ENODEV) return chans; name = dev_name(dev); @@ -452,7 +483,7 @@ struct iio_channel *iio_channel_get_all(struct device *dev) /* NULL terminated array to save passing size */ chans = kcalloc(nummaps + 1, sizeof(*chans), GFP_KERNEL); - if (chans == NULL) { + if (!chans) { ret = -ENOMEM; goto error_ret; } @@ -466,7 +497,7 @@ struct iio_channel *iio_channel_get_all(struct device *dev) chans[mapind].channel = iio_chan_spec_from_name(chans[mapind].indio_dev, c->map->adc_channel_label); - if (chans[mapind].channel == NULL) { + if (!chans[mapind].channel) { ret = -EINVAL; goto error_free_chans; } @@ -528,14 +559,14 @@ struct iio_channel *devm_iio_channel_get_all(struct device *dev) EXPORT_SYMBOL_GPL(devm_iio_channel_get_all); static int iio_channel_read(struct iio_channel *chan, int *val, int *val2, - enum iio_chan_info_enum info) + enum iio_chan_info_enum info) { int unused; int vals[INDIO_MAX_RAW_ELEMENTS]; int ret; int val_len = 2; - if (val2 == NULL) + if (!val2) val2 = &unused; if (!iio_channel_has_info(chan->channel, info)) @@ -547,9 +578,10 @@ static int iio_channel_read(struct iio_channel *chan, int *val, int *val2, vals, &val_len, info); *val = vals[0]; *val2 = vals[1]; - } else + } else { ret = chan->indio_dev->info->read_raw(chan->indio_dev, chan->channel, val, val2, info); + } return ret; } @@ -560,7 +592,7 @@ int iio_read_channel_raw(struct iio_channel *chan, int *val) int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -579,7 +611,7 @@ int iio_read_channel_average_raw(struct iio_channel *chan, int *val) int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -593,7 +625,8 @@ err_unlock: EXPORT_SYMBOL_GPL(iio_read_channel_average_raw); static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, - int raw, int *processed, unsigned int scale) + int raw, int *processed, + unsigned int scale) { int scale_type, scale_val, scale_val2; int offset_type, offset_val, offset_val2; @@ -626,7 +659,7 @@ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, } scale_type = iio_channel_read(chan, &scale_val, &scale_val2, - IIO_CHAN_INFO_SCALE); + IIO_CHAN_INFO_SCALE); if (scale_type < 0) { /* * If no channel scaling is available apply consumer scale to @@ -671,19 +704,19 @@ static int iio_convert_raw_to_processed_unlocked(struct iio_channel *chan, } int iio_convert_raw_to_processed(struct iio_channel *chan, int raw, - int *processed, unsigned int scale) + int *processed, unsigned int scale) { struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(chan->indio_dev); int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } ret = iio_convert_raw_to_processed_unlocked(chan, raw, processed, - scale); + scale); err_unlock: mutex_unlock(&iio_dev_opaque->info_exist_lock); @@ -698,7 +731,7 @@ int iio_read_channel_attribute(struct iio_channel *chan, int *val, int *val2, int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -724,7 +757,7 @@ int iio_read_channel_processed_scale(struct iio_channel *chan, int *val, int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -802,7 +835,7 @@ int iio_read_avail_channel_raw(struct iio_channel *chan, int type; ret = iio_read_avail_channel_attribute(chan, vals, &type, length, - IIO_CHAN_INFO_RAW); + IIO_CHAN_INFO_RAW); if (ret >= 0 && type != IIO_VAL_INT) /* raw values are assumed to be IIO_VAL_INT */ @@ -886,7 +919,7 @@ int iio_get_channel_type(struct iio_channel *chan, enum iio_chan_type *type) /* Need to verify underlying driver has not gone away */ mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -913,7 +946,7 @@ int iio_write_channel_attribute(struct iio_channel *chan, int val, int val2, int ret; mutex_lock(&iio_dev_opaque->info_exist_lock); - if (chan->indio_dev->info == NULL) { + if (!chan->indio_dev->info) { ret = -ENODEV; goto err_unlock; } @@ -947,9 +980,8 @@ unsigned int iio_get_channel_ext_info_count(struct iio_channel *chan) } EXPORT_SYMBOL_GPL(iio_get_channel_ext_info_count); -static const struct iio_chan_spec_ext_info *iio_lookup_ext_info( - const struct iio_channel *chan, - const char *attr) +static const struct iio_chan_spec_ext_info * +iio_lookup_ext_info(const struct iio_channel *chan, const char *attr) { const struct iio_chan_spec_ext_info *ext_info; diff --git a/drivers/iio/light/Kconfig b/drivers/iio/light/Kconfig index 8537e88f02e31a8255b89e43d31836c5f7105f3a..7cf6e84901231ddb1e2cc22e3fca4dcaf9bc5944 100644 --- a/drivers/iio/light/Kconfig +++ b/drivers/iio/light/Kconfig @@ -331,6 +331,17 @@ config LTR501 This driver can also be built as a module. If so, the module will be called ltr501. +config LTRF216A + tristate "Liteon LTRF216A Light Sensor" + depends on I2C + select REGMAP_I2C + help + If you say Y or M here, you get support for Liteon LTRF216A + Ambient Light Sensor. + + If built as a dynamically linked module, it will be called + ltrf216a. + config LV0104CS tristate "LV0104CS Ambient Light Sensor" depends on I2C diff --git a/drivers/iio/light/Makefile b/drivers/iio/light/Makefile index d10912faf964c61c79c38ce966eb1d88042aaf09..6f23817fae6fc84d4cb5d3e74683de7b1e6d1879 100644 --- a/drivers/iio/light/Makefile +++ b/drivers/iio/light/Makefile @@ -31,6 +31,7 @@ obj-$(CONFIG_ISL29125) += isl29125.o obj-$(CONFIG_JSA1212) += jsa1212.o obj-$(CONFIG_SENSORS_LM3533) += lm3533-als.o obj-$(CONFIG_LTR501) += ltr501.o +obj-$(CONFIG_LTRF216A) += ltrf216a.o obj-$(CONFIG_LV0104CS) += lv0104cs.o obj-$(CONFIG_MAX44000) += max44000.o obj-$(CONFIG_MAX44009) += max44009.o diff --git a/drivers/iio/light/apds9300.c b/drivers/iio/light/apds9300.c index 0f9d775989979cfc28864431fd67678ca4129cc7..b70f2681bcb33dd11682b31880d8fb05769342c7 100644 --- a/drivers/iio/light/apds9300.c +++ b/drivers/iio/light/apds9300.c @@ -452,7 +452,7 @@ err: return ret; } -static int apds9300_remove(struct i2c_client *client) +static void apds9300_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct apds9300_data *data = iio_priv(indio_dev); @@ -462,8 +462,6 @@ static int apds9300_remove(struct i2c_client *client) /* Ensure that power off and interrupts are disabled */ apds9300_set_intr_state(data, 0); apds9300_set_power_state(data, 0); - - return 0; } static int apds9300_suspend(struct device *dev) diff --git a/drivers/iio/light/apds9960.c b/drivers/iio/light/apds9960.c index 09b831f9f40b656458b5156cf6f2b170746d95c8..b62c139baf414c518a5be684e6f34d6b92322a4a 100644 --- a/drivers/iio/light/apds9960.c +++ b/drivers/iio/light/apds9960.c @@ -1067,7 +1067,7 @@ error_power_down: return ret; } -static int apds9960_remove(struct i2c_client *client) +static void apds9960_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct apds9960_data *data = iio_priv(indio_dev); @@ -1076,8 +1076,6 @@ static int apds9960_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); pm_runtime_set_suspended(&client->dev); apds9960_set_powermode(data, 0); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/iio/light/bh1750.c b/drivers/iio/light/bh1750.c index 471985c220bb5a6768d70ca89a265ebae22a4a11..3e92820bc82007111ff4a87f9da5fb528be183de 100644 --- a/drivers/iio/light/bh1750.c +++ b/drivers/iio/light/bh1750.c @@ -263,7 +263,7 @@ static int bh1750_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int bh1750_remove(struct i2c_client *client) +static void bh1750_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct bh1750_data *data = iio_priv(indio_dev); @@ -273,8 +273,6 @@ static int bh1750_remove(struct i2c_client *client) mutex_lock(&data->lock); i2c_smbus_write_byte(client, BH1750_POWER_DOWN); mutex_unlock(&data->lock); - - return 0; } static int bh1750_suspend(struct device *dev) diff --git a/drivers/iio/light/bh1780.c b/drivers/iio/light/bh1780.c index fc7141390117d5431ab1e45501e9e647efa9d239..90bca392b262009502859799b40b536fd0fcc032 100644 --- a/drivers/iio/light/bh1780.c +++ b/drivers/iio/light/bh1780.c @@ -202,7 +202,7 @@ out_disable_pm: return ret; } -static int bh1780_remove(struct i2c_client *client) +static void bh1780_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct bh1780_data *bh1780 = iio_priv(indio_dev); @@ -216,8 +216,6 @@ static int bh1780_remove(struct i2c_client *client) if (ret < 0) dev_err(&client->dev, "failed to power off (%pe)\n", ERR_PTR(ret)); - - return 0; } static int bh1780_runtime_suspend(struct device *dev) diff --git a/drivers/iio/light/cm32181.c b/drivers/iio/light/cm32181.c index edbe6a3138d0b927cc5091eec57890c488c8fbf0..001055d097509e688f7ce330ff2b3e7a24810269 100644 --- a/drivers/iio/light/cm32181.c +++ b/drivers/iio/light/cm32181.c @@ -505,7 +505,7 @@ static int cm32181_resume(struct device *dev) cm32181->conf_regs[CM32181_REG_ADDR_CMD]); } -DEFINE_SIMPLE_DEV_PM_OPS(cm32181_pm_ops, cm32181_suspend, cm32181_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(cm32181_pm_ops, cm32181_suspend, cm32181_resume); static const struct of_device_id cm32181_of_match[] = { { .compatible = "capella,cm3218" }, diff --git a/drivers/iio/light/cm3232.c b/drivers/iio/light/cm3232.c index 2c80a0535d2c4855da208959bdf030a0489c08e7..5214cd014cf8b747c3232751a29dc3258cd66b19 100644 --- a/drivers/iio/light/cm3232.c +++ b/drivers/iio/light/cm3232.c @@ -357,7 +357,7 @@ static int cm3232_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int cm3232_remove(struct i2c_client *client) +static void cm3232_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -365,8 +365,6 @@ static int cm3232_remove(struct i2c_client *client) CM3232_CMD_ALS_DISABLE); iio_device_unregister(indio_dev); - - return 0; } static const struct i2c_device_id cm3232_id[] = { diff --git a/drivers/iio/light/cm3605.c b/drivers/iio/light/cm3605.c index c721b69d50950e37ddb4fe5796f2ac1d2dc386f1..0b30db77f78bd9ed1144b9a769db3a6fe2417367 100644 --- a/drivers/iio/light/cm3605.c +++ b/drivers/iio/light/cm3605.c @@ -226,8 +226,10 @@ static int cm3605_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (irq < 0) - return dev_err_probe(dev, irq, "failed to get irq\n"); + if (irq < 0) { + ret = dev_err_probe(dev, irq, "failed to get irq\n"); + goto out_disable_aset; + } ret = devm_request_threaded_irq(dev, irq, cm3605_prox_irq, NULL, 0, "cm3605", indio_dev); diff --git a/drivers/iio/light/cm36651.c b/drivers/iio/light/cm36651.c index 89f5e48a6642cce395f3800bbd83f58db39fbf83..6615c98b601c3feed34ce17feada7b79c9977c66 100644 --- a/drivers/iio/light/cm36651.c +++ b/drivers/iio/light/cm36651.c @@ -700,7 +700,7 @@ error_disable_reg: return ret; } -static int cm36651_remove(struct i2c_client *client) +static void cm36651_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct cm36651_data *cm36651 = iio_priv(indio_dev); @@ -710,8 +710,6 @@ static int cm36651_remove(struct i2c_client *client) free_irq(client->irq, indio_dev); i2c_unregister_device(cm36651->ps_client); i2c_unregister_device(cm36651->ara_client); - - return 0; } static const struct i2c_device_id cm36651_id[] = { diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index e2707416f9a80ac6ce7fe5ca6059d00ef1633546..8000fa347344fd82f11b202546cb63aea9b82cde 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -619,7 +619,7 @@ out_disable_vdd: return ret; } -static int gp2ap002_remove(struct i2c_client *client) +static void gp2ap002_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct gp2ap002 *gp2ap002 = iio_priv(indio_dev); @@ -631,8 +631,6 @@ static int gp2ap002_remove(struct i2c_client *client) iio_device_unregister(indio_dev); regulator_disable(gp2ap002->vio); regulator_disable(gp2ap002->vdd); - - return 0; } static int gp2ap002_runtime_suspend(struct device *dev) diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c index b820041159f770899809c46425da1554e8748874..826439299e8b03899425cab94064bfb6a765931e 100644 --- a/drivers/iio/light/gp2ap020a00f.c +++ b/drivers/iio/light/gp2ap020a00f.c @@ -1573,7 +1573,7 @@ error_regulator_disable: return err; } -static int gp2ap020a00f_remove(struct i2c_client *client) +static void gp2ap020a00f_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct gp2ap020a00f_data *data = iio_priv(indio_dev); @@ -1589,8 +1589,6 @@ static int gp2ap020a00f_remove(struct i2c_client *client) free_irq(client->irq, indio_dev); iio_triggered_buffer_cleanup(indio_dev); regulator_disable(data->vled_reg); - - return 0; } static const struct i2c_device_id gp2ap020a00f_id[] = { diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c index ff5996d77818135ec22850c86122548b9b20579c..32d58e18f26da3485c1de7b4fe2f5f05a2c24a4c 100644 --- a/drivers/iio/light/isl29028.c +++ b/drivers/iio/light/isl29028.c @@ -636,7 +636,7 @@ static int isl29028_probe(struct i2c_client *client, return 0; } -static int isl29028_remove(struct i2c_client *client) +static void isl29028_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct isl29028_chip *chip = iio_priv(indio_dev); @@ -647,8 +647,6 @@ static int isl29028_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); isl29028_clear_configure_reg(chip); - - return 0; } static int isl29028_suspend(struct device *dev) diff --git a/drivers/iio/light/isl29125.c b/drivers/iio/light/isl29125.c index eb68a52aab8242eed331810cefc0f5e2b81ec6b7..c199e63cce8289ddda3c35738127ba3720414dca 100644 --- a/drivers/iio/light/isl29125.c +++ b/drivers/iio/light/isl29125.c @@ -300,15 +300,13 @@ static int isl29125_powerdown(struct isl29125_data *data) (data->conf1 & ~ISL29125_MODE_MASK) | ISL29125_MODE_PD); } -static int isl29125_remove(struct i2c_client *client) +static void isl29125_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); iio_device_unregister(indio_dev); iio_triggered_buffer_cleanup(indio_dev); isl29125_powerdown(iio_priv(indio_dev)); - - return 0; } static int isl29125_suspend(struct device *dev) diff --git a/drivers/iio/light/jsa1212.c b/drivers/iio/light/jsa1212.c index 5387c12231cf3a5729c1aaf5a8144c837df3e714..57ce6d75966c63d6ac907c4ff01510e233aa1fab 100644 --- a/drivers/iio/light/jsa1212.c +++ b/drivers/iio/light/jsa1212.c @@ -373,7 +373,7 @@ static int jsa1212_power_off(struct jsa1212_data *data) return ret; } -static int jsa1212_remove(struct i2c_client *client) +static void jsa1212_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct jsa1212_data *data = iio_priv(indio_dev); @@ -381,8 +381,6 @@ static int jsa1212_remove(struct i2c_client *client) iio_device_unregister(indio_dev); jsa1212_power_off(data); - - return 0; } static int jsa1212_suspend(struct device *dev) diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c index 679a1e1086ae839f9a74daf03fb652bc6ef808fb..74a1ccda8b9c4a6d0ee8d059671eb62297d29c4d 100644 --- a/drivers/iio/light/ltr501.c +++ b/drivers/iio/light/ltr501.c @@ -1600,15 +1600,13 @@ powerdown_on_error: return ret; } -static int ltr501_remove(struct i2c_client *client) +static void ltr501_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); iio_device_unregister(indio_dev); iio_triggered_buffer_cleanup(indio_dev); ltr501_powerdown(iio_priv(indio_dev)); - - return 0; } static int ltr501_suspend(struct device *dev) diff --git a/drivers/iio/light/ltrf216a.c b/drivers/iio/light/ltrf216a.c new file mode 100644 index 0000000000000000000000000000000000000000..4b8ef36b691251f826b987784e63fb448d6add4c --- /dev/null +++ b/drivers/iio/light/ltrf216a.c @@ -0,0 +1,550 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * LTRF216A Ambient Light Sensor + * + * Copyright (C) 2022 Collabora, Ltd. + * Author: Shreeya Patel + * + * Copyright (C) 2021 Lite-On Technology Corp (Singapore) + * Author: Shi Zhigang + * + * IIO driver for LTRF216A (7-bit I2C slave address 0x53). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#define LTRF216A_ALS_RESET_MASK BIT(4) +#define LTRF216A_ALS_DATA_STATUS BIT(3) +#define LTRF216A_ALS_ENABLE_MASK BIT(1) +#define LTRF216A_MAIN_CTRL 0x00 +#define LTRF216A_ALS_MEAS_RES 0x04 +#define LTRF216A_ALS_GAIN 0x05 +#define LTRF216A_PART_ID 0x06 +#define LTRF216A_MAIN_STATUS 0x07 +#define LTRF216A_ALS_CLEAR_DATA_0 0x0a +#define LTRF216A_ALS_CLEAR_DATA_1 0x0b +#define LTRF216A_ALS_CLEAR_DATA_2 0x0c +#define LTRF216A_ALS_DATA_0 0x0d +#define LTRF216A_ALS_DATA_1 0x0e +#define LTRF216A_ALS_DATA_2 0x0f +#define LTRF216A_INT_CFG 0x19 +#define LTRF216A_INT_PST 0x1a +#define LTRF216A_ALS_THRES_UP_0 0x21 +#define LTRF216A_ALS_THRES_UP_1 0x22 +#define LTRF216A_ALS_THRES_UP_2 0x23 +#define LTRF216A_ALS_THRES_LOW_0 0x24 +#define LTRF216A_ALS_THRES_LOW_1 0x25 +#define LTRF216A_ALS_THRES_LOW_2 0x26 +#define LTRF216A_ALS_READ_DATA_DELAY_US 20000 + +static const int ltrf216a_int_time_available[][2] = { + { 0, 400000 }, + { 0, 200000 }, + { 0, 100000 }, + { 0, 50000 }, + { 0, 25000 }, +}; + +static const int ltrf216a_int_time_reg[][2] = { + { 400, 0x03 }, + { 200, 0x13 }, + { 100, 0x22 }, + { 50, 0x31 }, + { 25, 0x40 }, +}; + +/* + * Window Factor is needed when the device is under Window glass + * with coated tinted ink. This is to compensate for the light loss + * due to the lower transmission rate of the window glass and helps + * in calculating lux. + */ +#define LTRF216A_WIN_FAC 1 + +struct ltrf216a_data { + struct regmap *regmap; + struct i2c_client *client; + u32 int_time; + u16 int_time_fac; + u8 als_gain_fac; + /* + * Protects regmap accesses and makes sure integration time + * remains constant during the measurement of lux. + */ + struct mutex lock; +}; + +static const struct iio_chan_spec ltrf216a_channels[] = { + { + .type = IIO_LIGHT, + .info_mask_separate = + BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_INT_TIME), + .info_mask_separate_available = + BIT(IIO_CHAN_INFO_INT_TIME), + }, +}; + +static void ltrf216a_reset(struct iio_dev *indio_dev) +{ + struct ltrf216a_data *data = iio_priv(indio_dev); + + /* reset sensor, chip fails to respond to this, so ignore any errors */ + regmap_write(data->regmap, LTRF216A_MAIN_CTRL, LTRF216A_ALS_RESET_MASK); + + /* reset time */ + usleep_range(1000, 2000); +} + +static int ltrf216a_enable(struct iio_dev *indio_dev) +{ + struct ltrf216a_data *data = iio_priv(indio_dev); + struct device *dev = &data->client->dev; + int ret; + + /* enable sensor */ + ret = regmap_set_bits(data->regmap, + LTRF216A_MAIN_CTRL, LTRF216A_ALS_ENABLE_MASK); + if (ret) { + dev_err(dev, "failed to enable sensor: %d\n", ret); + return ret; + } + + /* sleep for one integration cycle after enabling the device */ + msleep(ltrf216a_int_time_reg[0][0]); + + return 0; +} + +static int ltrf216a_disable(struct iio_dev *indio_dev) +{ + struct ltrf216a_data *data = iio_priv(indio_dev); + struct device *dev = &data->client->dev; + int ret; + + ret = regmap_write(data->regmap, LTRF216A_MAIN_CTRL, 0); + if (ret) + dev_err(dev, "failed to disable sensor: %d\n", ret); + + return ret; +} + +static void ltrf216a_cleanup(void *data) +{ + struct iio_dev *indio_dev = data; + + ltrf216a_disable(indio_dev); +} + +static int ltrf216a_set_int_time(struct ltrf216a_data *data, int itime) +{ + struct device *dev = &data->client->dev; + unsigned int i; + u8 reg_val; + int ret; + + for (i = 0; i < ARRAY_SIZE(ltrf216a_int_time_available); i++) { + if (ltrf216a_int_time_available[i][1] == itime) + break; + } + if (i == ARRAY_SIZE(ltrf216a_int_time_available)) + return -EINVAL; + + reg_val = ltrf216a_int_time_reg[i][1]; + + ret = regmap_write(data->regmap, LTRF216A_ALS_MEAS_RES, reg_val); + if (ret) { + dev_err(dev, "failed to set integration time: %d\n", ret); + return ret; + } + + data->int_time_fac = ltrf216a_int_time_reg[i][0]; + data->int_time = itime; + + return 0; +} + +static int ltrf216a_get_int_time(struct ltrf216a_data *data, + int *val, int *val2) +{ + *val = 0; + *val2 = data->int_time; + return IIO_VAL_INT_PLUS_MICRO; +} + +static int ltrf216a_set_power_state(struct ltrf216a_data *data, bool on) +{ + struct device *dev = &data->client->dev; + int ret = 0; + + if (on) { + ret = pm_runtime_resume_and_get(dev); + if (ret) { + dev_err(dev, "failed to resume runtime PM: %d\n", ret); + return ret; + } + } else { + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); + } + + return ret; +} + +static int ltrf216a_read_data(struct ltrf216a_data *data, u8 addr) +{ + struct device *dev = &data->client->dev; + int ret, val; + u8 buf[3]; + + ret = regmap_read_poll_timeout(data->regmap, LTRF216A_MAIN_STATUS, + val, val & LTRF216A_ALS_DATA_STATUS, + LTRF216A_ALS_READ_DATA_DELAY_US, + LTRF216A_ALS_READ_DATA_DELAY_US * 50); + if (ret) { + dev_err(dev, "failed to wait for measurement data: %d\n", ret); + return ret; + } + + ret = regmap_bulk_read(data->regmap, addr, buf, sizeof(buf)); + if (ret) { + dev_err(dev, "failed to read measurement data: %d\n", ret); + return ret; + } + + return get_unaligned_le24(&buf[0]); +} + +static int ltrf216a_get_lux(struct ltrf216a_data *data) +{ + int ret, greendata; + u64 lux, div; + + ret = ltrf216a_set_power_state(data, true); + if (ret) + return ret; + + greendata = ltrf216a_read_data(data, LTRF216A_ALS_DATA_0); + if (greendata < 0) + return greendata; + + ltrf216a_set_power_state(data, false); + + lux = greendata * 45 * LTRF216A_WIN_FAC * 100; + div = data->als_gain_fac * data->int_time_fac * 100; + + return div_u64(lux, div); +} + +static int ltrf216a_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int *val, + int *val2, long mask) +{ + struct ltrf216a_data *data = iio_priv(indio_dev); + int ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = ltrf216a_set_power_state(data, true); + if (ret) + return ret; + mutex_lock(&data->lock); + ret = ltrf216a_read_data(data, LTRF216A_ALS_DATA_0); + mutex_unlock(&data->lock); + ltrf216a_set_power_state(data, false); + if (ret < 0) + return ret; + *val = ret; + return IIO_VAL_INT; + case IIO_CHAN_INFO_PROCESSED: + mutex_lock(&data->lock); + ret = ltrf216a_get_lux(data); + mutex_unlock(&data->lock); + if (ret < 0) + return ret; + *val = ret; + return IIO_VAL_INT; + case IIO_CHAN_INFO_INT_TIME: + mutex_lock(&data->lock); + ret = ltrf216a_get_int_time(data, val, val2); + mutex_unlock(&data->lock); + return ret; + default: + return -EINVAL; + } +} + +static int ltrf216a_write_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, int val, + int val2, long mask) +{ + struct ltrf216a_data *data = iio_priv(indio_dev); + int ret; + + switch (mask) { + case IIO_CHAN_INFO_INT_TIME: + if (val != 0) + return -EINVAL; + mutex_lock(&data->lock); + ret = ltrf216a_set_int_time(data, val2); + mutex_unlock(&data->lock); + return ret; + default: + return -EINVAL; + } +} + +static int ltrf216a_read_available(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_INT_TIME: + *length = ARRAY_SIZE(ltrf216a_int_time_available) * 2; + *vals = (const int *)ltrf216a_int_time_available; + *type = IIO_VAL_INT_PLUS_MICRO; + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + +static const struct iio_info ltrf216a_info = { + .read_raw = ltrf216a_read_raw, + .write_raw = ltrf216a_write_raw, + .read_avail = ltrf216a_read_available, +}; + +static bool ltrf216a_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case LTRF216A_MAIN_CTRL: + case LTRF216A_ALS_MEAS_RES: + case LTRF216A_ALS_GAIN: + case LTRF216A_PART_ID: + case LTRF216A_MAIN_STATUS: + case LTRF216A_ALS_CLEAR_DATA_0: + case LTRF216A_ALS_CLEAR_DATA_1: + case LTRF216A_ALS_CLEAR_DATA_2: + case LTRF216A_ALS_DATA_0: + case LTRF216A_ALS_DATA_1: + case LTRF216A_ALS_DATA_2: + case LTRF216A_INT_CFG: + case LTRF216A_INT_PST: + case LTRF216A_ALS_THRES_UP_0: + case LTRF216A_ALS_THRES_UP_1: + case LTRF216A_ALS_THRES_UP_2: + case LTRF216A_ALS_THRES_LOW_0: + case LTRF216A_ALS_THRES_LOW_1: + case LTRF216A_ALS_THRES_LOW_2: + return true; + default: + return false; + } +} + +static bool ltrf216a_writable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case LTRF216A_MAIN_CTRL: + case LTRF216A_ALS_MEAS_RES: + case LTRF216A_ALS_GAIN: + case LTRF216A_INT_CFG: + case LTRF216A_INT_PST: + case LTRF216A_ALS_THRES_UP_0: + case LTRF216A_ALS_THRES_UP_1: + case LTRF216A_ALS_THRES_UP_2: + case LTRF216A_ALS_THRES_LOW_0: + case LTRF216A_ALS_THRES_LOW_1: + case LTRF216A_ALS_THRES_LOW_2: + return true; + default: + return false; + } +} + +static bool ltrf216a_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case LTRF216A_MAIN_STATUS: + case LTRF216A_ALS_CLEAR_DATA_0: + case LTRF216A_ALS_CLEAR_DATA_1: + case LTRF216A_ALS_CLEAR_DATA_2: + case LTRF216A_ALS_DATA_0: + case LTRF216A_ALS_DATA_1: + case LTRF216A_ALS_DATA_2: + return true; + default: + return false; + } +} + +static bool ltrf216a_precious_reg(struct device *dev, unsigned int reg) +{ + return reg == LTRF216A_MAIN_STATUS; +} + +static const struct regmap_config ltrf216a_regmap_config = { + .name = "ltrf216a", + .reg_bits = 8, + .val_bits = 8, + .cache_type = REGCACHE_RBTREE, + .max_register = LTRF216A_ALS_THRES_LOW_2, + .readable_reg = ltrf216a_readable_reg, + .writeable_reg = ltrf216a_writable_reg, + .volatile_reg = ltrf216a_volatile_reg, + .precious_reg = ltrf216a_precious_reg, + .disable_locking = true, +}; + +static int ltrf216a_probe(struct i2c_client *client) +{ + struct ltrf216a_data *data; + struct iio_dev *indio_dev; + int ret; + + indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); + if (!indio_dev) + return -ENOMEM; + + data = iio_priv(indio_dev); + + data->regmap = devm_regmap_init_i2c(client, <rf216a_regmap_config); + if (IS_ERR(data->regmap)) + return dev_err_probe(&client->dev, PTR_ERR(data->regmap), + "regmap initialization failed\n"); + + i2c_set_clientdata(client, indio_dev); + data->client = client; + + mutex_init(&data->lock); + + indio_dev->info = <rf216a_info; + indio_dev->name = "ltrf216a"; + indio_dev->channels = ltrf216a_channels; + indio_dev->num_channels = ARRAY_SIZE(ltrf216a_channels); + indio_dev->modes = INDIO_DIRECT_MODE; + + ret = pm_runtime_set_active(&client->dev); + if (ret) + return ret; + + /* reset sensor, chip fails to respond to this, so ignore any errors */ + ltrf216a_reset(indio_dev); + + ret = regmap_reinit_cache(data->regmap, <rf216a_regmap_config); + if (ret) + return dev_err_probe(&client->dev, ret, + "failed to reinit regmap cache\n"); + + ret = ltrf216a_enable(indio_dev); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&client->dev, ltrf216a_cleanup, + indio_dev); + if (ret) + return ret; + + ret = devm_pm_runtime_enable(&client->dev); + if (ret) + return dev_err_probe(&client->dev, ret, + "failed to enable runtime PM\n"); + + pm_runtime_set_autosuspend_delay(&client->dev, 1000); + pm_runtime_use_autosuspend(&client->dev); + + data->int_time = 100000; + data->int_time_fac = 100; + data->als_gain_fac = 3; + + return devm_iio_device_register(&client->dev, indio_dev); +} + +static int ltrf216a_runtime_suspend(struct device *dev) +{ + struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev)); + struct ltrf216a_data *data = iio_priv(indio_dev); + int ret; + + ret = ltrf216a_disable(indio_dev); + if (ret) + return ret; + + regcache_cache_only(data->regmap, true); + + return 0; +} + +static int ltrf216a_runtime_resume(struct device *dev) +{ + struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev)); + struct ltrf216a_data *data = iio_priv(indio_dev); + int ret; + + regcache_cache_only(data->regmap, false); + regcache_mark_dirty(data->regmap); + ret = regcache_sync(data->regmap); + if (ret) + goto cache_only; + + ret = ltrf216a_enable(indio_dev); + if (ret) + goto cache_only; + + return 0; + +cache_only: + regcache_cache_only(data->regmap, true); + + return ret; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(ltrf216a_pm_ops, ltrf216a_runtime_suspend, + ltrf216a_runtime_resume, NULL); + +static const struct i2c_device_id ltrf216a_id[] = { + { "ltrf216a" }, + {} +}; +MODULE_DEVICE_TABLE(i2c, ltrf216a_id); + +static const struct of_device_id ltrf216a_of_match[] = { + { .compatible = "liteon,ltrf216a" }, + { .compatible = "ltr,ltrf216a" }, + {} +}; +MODULE_DEVICE_TABLE(of, ltrf216a_of_match); + +static struct i2c_driver ltrf216a_driver = { + .driver = { + .name = "ltrf216a", + .pm = pm_ptr(<rf216a_pm_ops), + .of_match_table = ltrf216a_of_match, + }, + .probe_new = ltrf216a_probe, + .id_table = ltrf216a_id, +}; +module_i2c_driver(ltrf216a_driver); + +MODULE_AUTHOR("Shreeya Patel "); +MODULE_AUTHOR("Shi Zhigang "); +MODULE_DESCRIPTION("LTRF216A ambient light sensor driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c index a326d47afc9b040fb8d166e45b56242ed0429510..a26d1c3f954375d53c4a0eab3dae924dd0a40a44 100644 --- a/drivers/iio/light/opt3001.c +++ b/drivers/iio/light/opt3001.c @@ -794,7 +794,7 @@ static int opt3001_probe(struct i2c_client *client, return 0; } -static int opt3001_remove(struct i2c_client *client) +static void opt3001_remove(struct i2c_client *client) { struct iio_dev *iio = i2c_get_clientdata(client); struct opt3001 *opt = iio_priv(iio); @@ -808,7 +808,7 @@ static int opt3001_remove(struct i2c_client *client) if (ret < 0) { dev_err(opt->dev, "failed to read register %02x\n", OPT3001_CONFIGURATION); - return 0; + return; } reg = ret; @@ -820,8 +820,6 @@ static int opt3001_remove(struct i2c_client *client) dev_err(opt->dev, "failed to write register %02x\n", OPT3001_CONFIGURATION); } - - return 0; } static const struct i2c_device_id opt3001_id[] = { diff --git a/drivers/iio/light/pa12203001.c b/drivers/iio/light/pa12203001.c index 772874e707aec759266d95346158119749d44e53..3cb2de51f4aa8749c9d56832b6ab3834bc200462 100644 --- a/drivers/iio/light/pa12203001.c +++ b/drivers/iio/light/pa12203001.c @@ -394,7 +394,7 @@ out_err: return ret; } -static int pa12203001_remove(struct i2c_client *client) +static void pa12203001_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); int ret; @@ -408,8 +408,6 @@ static int pa12203001_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to power down (%pe)\n", ERR_PTR(ret)); - - return 0; } #if defined(CONFIG_PM_SLEEP) || defined(CONFIG_PM) diff --git a/drivers/iio/light/rpr0521.c b/drivers/iio/light/rpr0521.c index dabdd05f0e2cc8b571814d3bb61cce88c8ef6e0e..d1c16dd76058398bb9b5bf65fb5edc894e37d4d6 100644 --- a/drivers/iio/light/rpr0521.c +++ b/drivers/iio/light/rpr0521.c @@ -1041,7 +1041,7 @@ err_poweroff: return ret; } -static int rpr0521_remove(struct i2c_client *client) +static void rpr0521_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -1051,8 +1051,6 @@ static int rpr0521_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); rpr0521_poweroff(iio_priv(indio_dev)); - - return 0; } static int rpr0521_runtime_suspend(struct device *dev) diff --git a/drivers/iio/light/st_uvis25_core.c b/drivers/iio/light/st_uvis25_core.c index 3d4cc1180b6a77ab3eb453ef91dbe373491671aa..c737d3e193aec8b107060b46a93456cf7ff011e2 100644 --- a/drivers/iio/light/st_uvis25_core.c +++ b/drivers/iio/light/st_uvis25_core.c @@ -325,7 +325,7 @@ int st_uvis25_probe(struct device *dev, int irq, struct regmap *regmap) } EXPORT_SYMBOL_NS(st_uvis25_probe, IIO_UVIS25); -static int __maybe_unused st_uvis25_suspend(struct device *dev) +static int st_uvis25_suspend(struct device *dev) { struct iio_dev *iio_dev = dev_get_drvdata(dev); struct st_uvis25_hw *hw = iio_priv(iio_dev); @@ -334,7 +334,7 @@ static int __maybe_unused st_uvis25_suspend(struct device *dev) ST_UVIS25_REG_ODR_MASK, 0); } -static int __maybe_unused st_uvis25_resume(struct device *dev) +static int st_uvis25_resume(struct device *dev) { struct iio_dev *iio_dev = dev_get_drvdata(dev); struct st_uvis25_hw *hw = iio_priv(iio_dev); @@ -346,10 +346,7 @@ static int __maybe_unused st_uvis25_resume(struct device *dev) return 0; } -const struct dev_pm_ops st_uvis25_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(st_uvis25_suspend, st_uvis25_resume) -}; -EXPORT_SYMBOL_NS(st_uvis25_pm_ops, IIO_UVIS25); +EXPORT_NS_SIMPLE_DEV_PM_OPS(st_uvis25_pm_ops, st_uvis25_suspend, st_uvis25_resume, IIO_UVIS25); MODULE_AUTHOR("Lorenzo Bianconi "); MODULE_DESCRIPTION("STMicroelectronics uvis25 sensor driver"); diff --git a/drivers/iio/light/st_uvis25_i2c.c b/drivers/iio/light/st_uvis25_i2c.c index b06d09af28a3579a3540c229c921193bd6110675..c982b0b255cf858b0aa8aa7c750b84def5cc44c6 100644 --- a/drivers/iio/light/st_uvis25_i2c.c +++ b/drivers/iio/light/st_uvis25_i2c.c @@ -55,7 +55,7 @@ MODULE_DEVICE_TABLE(i2c, st_uvis25_i2c_id_table); static struct i2c_driver st_uvis25_driver = { .driver = { .name = "st_uvis25_i2c", - .pm = &st_uvis25_pm_ops, + .pm = pm_sleep_ptr(&st_uvis25_pm_ops), .of_match_table = st_uvis25_i2c_of_match, }, .probe = st_uvis25_i2c_probe, diff --git a/drivers/iio/light/st_uvis25_spi.c b/drivers/iio/light/st_uvis25_spi.c index 3a4dc6d7180c246da623ce3812c8978cfa20d081..86a232320d7d7107d2fdb5fdad867163a265c7ea 100644 --- a/drivers/iio/light/st_uvis25_spi.c +++ b/drivers/iio/light/st_uvis25_spi.c @@ -55,7 +55,7 @@ MODULE_DEVICE_TABLE(spi, st_uvis25_spi_id_table); static struct spi_driver st_uvis25_driver = { .driver = { .name = "st_uvis25_spi", - .pm = &st_uvis25_pm_ops, + .pm = pm_sleep_ptr(&st_uvis25_pm_ops), .of_match_table = st_uvis25_spi_of_match, }, .probe = st_uvis25_spi_probe, diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c index f7cc7a6c0c8d14c7ae28830c944e1ec20b8aada6..7b8e0da6aabc143296fba096f64bb15321311388 100644 --- a/drivers/iio/light/stk3310.c +++ b/drivers/iio/light/stk3310.c @@ -649,14 +649,12 @@ err_standby: return ret; } -static int stk3310_remove(struct i2c_client *client) +static void stk3310_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); iio_device_unregister(indio_dev); stk3310_set_state(iio_priv(indio_dev), STK3310_STATE_STANDBY); - - return 0; } static int stk3310_suspend(struct device *dev) diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c index 823435f59bb6b7d25723d1954a980b41625b6a18..db17fec634be1e284f3b5e3bb51e456f5a915089 100644 --- a/drivers/iio/light/tcs3472.c +++ b/drivers/iio/light/tcs3472.c @@ -559,7 +559,7 @@ static int tcs3472_powerdown(struct tcs3472_data *data) return ret; } -static int tcs3472_remove(struct i2c_client *client) +static void tcs3472_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -568,8 +568,6 @@ static int tcs3472_remove(struct i2c_client *client) free_irq(client->irq, indio_dev); iio_triggered_buffer_cleanup(indio_dev); tcs3472_powerdown(iio_priv(indio_dev)); - - return 0; } static int tcs3472_suspend(struct device *dev) diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c index bbb577459fb987a3e75a3d3039691d5da116b053..951f35ef3f41de994dd88f5339c7eb862ae3eb93 100644 --- a/drivers/iio/light/tsl2563.c +++ b/drivers/iio/light/tsl2563.c @@ -796,7 +796,7 @@ fail: return err; } -static int tsl2563_remove(struct i2c_client *client) +static void tsl2563_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct tsl2563_chip *chip = iio_priv(indio_dev); @@ -809,8 +809,6 @@ static int tsl2563_remove(struct i2c_client *client) i2c_smbus_write_byte_data(chip->client, TSL2563_CMD | TSL2563_REG_INT, chip->intr); tsl2563_set_power(chip, 0); - - return 0; } static int tsl2563_suspend(struct device *dev) diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index 82662dab87c0f0ef38f5e99ea21030232e429a0a..0a2ca1a8146da4690e570c693ec8a68530d855ba 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -873,7 +873,7 @@ static int tsl2583_probe(struct i2c_client *clientp, return 0; } -static int tsl2583_remove(struct i2c_client *client) +static void tsl2583_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct tsl2583_chip *chip = iio_priv(indio_dev); @@ -884,8 +884,6 @@ static int tsl2583_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); tsl2583_set_power_state(chip, TSL2583_CNTL_PWR_OFF); - - return 0; } static int tsl2583_suspend(struct device *dev) diff --git a/drivers/iio/light/tsl4531.c b/drivers/iio/light/tsl4531.c index 6ae1b27e50b6330e841600827d52836fa2e33a3c..090038fed88919f49026134f74d30b220cd8b5e0 100644 --- a/drivers/iio/light/tsl4531.c +++ b/drivers/iio/light/tsl4531.c @@ -207,12 +207,10 @@ static int tsl4531_powerdown(struct i2c_client *client) TSL4531_MODE_POWERDOWN); } -static int tsl4531_remove(struct i2c_client *client) +static void tsl4531_remove(struct i2c_client *client) { iio_device_unregister(i2c_get_clientdata(client)); tsl4531_powerdown(client); - - return 0; } static int tsl4531_suspend(struct device *dev) diff --git a/drivers/iio/light/us5182d.c b/drivers/iio/light/us5182d.c index 80d2299da561563473ad9df50208b0d4b1792de1..3e652d7f3b0ebf6b370c278dde4477622acf082a 100644 --- a/drivers/iio/light/us5182d.c +++ b/drivers/iio/light/us5182d.c @@ -904,7 +904,7 @@ out_err: } -static int us5182d_remove(struct i2c_client *client) +static void us5182d_remove(struct i2c_client *client) { struct us5182d_data *data = iio_priv(i2c_get_clientdata(client)); int ret; @@ -918,8 +918,6 @@ static int us5182d_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to shut down (%pe)\n", ERR_PTR(ret)); - - return 0; } static int us5182d_suspend(struct device *dev) diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c index 3db4e26731bb5a7de1c03165f17e1f461d9a8041..f6c83ecaad8b584db22266a0b9576fc4f9a2c4f3 100644 --- a/drivers/iio/light/vcnl4000.c +++ b/drivers/iio/light/vcnl4000.c @@ -1111,7 +1111,7 @@ static const struct of_device_id vcnl_4000_of_match[] = { }; MODULE_DEVICE_TABLE(of, vcnl_4000_of_match); -static int vcnl4000_remove(struct i2c_client *client) +static void vcnl4000_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct vcnl4000_data *data = iio_priv(indio_dev); @@ -1126,8 +1126,6 @@ static int vcnl4000_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to power down (%pe)\n", ERR_PTR(ret)); - - return 0; } static int vcnl4000_runtime_suspend(struct device *dev) diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c index 6a196cf2270b1f14ec7dc0a38cec2b110d0a6f34..3ed37f6057fbf429573054a8e801d9bedc6fd75c 100644 --- a/drivers/iio/light/vcnl4035.c +++ b/drivers/iio/light/vcnl4035.c @@ -601,7 +601,7 @@ fail_poweroff: return ret; } -static int vcnl4035_remove(struct i2c_client *client) +static void vcnl4035_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); int ret; @@ -616,8 +616,6 @@ static int vcnl4035_remove(struct i2c_client *client) if (ret) dev_warn(&client->dev, "Failed to put device into standby (%pe)\n", ERR_PTR(ret)); - - return 0; } static int vcnl4035_runtime_suspend(struct device *dev) diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c index 1e55e09a8d1658ae25d267674b11a8c683044f32..cfa4e9e7c8038395a9183c3687600deef9a9ed69 100644 --- a/drivers/iio/light/veml6070.c +++ b/drivers/iio/light/veml6070.c @@ -180,15 +180,13 @@ fail: return ret; } -static int veml6070_remove(struct i2c_client *client) +static void veml6070_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct veml6070_data *data = iio_priv(indio_dev); iio_device_unregister(indio_dev); i2c_unregister_device(data->client2); - - return 0; } static const struct i2c_device_id veml6070_id[] = { diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig index 07eb619bcfe8940681bb53e0eeadefe983ec6b9d..b91fc5e6a26e1d58a90d36f7dbe65596916a9bb2 100644 --- a/drivers/iio/magnetometer/Kconfig +++ b/drivers/iio/magnetometer/Kconfig @@ -216,8 +216,8 @@ config YAMAHA_YAS530 select IIO_TRIGGERED_BUFFER help Say Y here to add support for the Yamaha YAS530 series of - 3-Axis Magnetometers. Right now YAS530, YAS532 and YAS533 are - fully supported. + 3-Axis Magnetometers. YAS530, YAS532, YAS533 and YAS537 are + supported. This driver can also be compiled as a module. To compile this driver as a module, choose M here: the module diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c index c89a91db0690799ffcdae979f0608d5d3837445f..7ec9ab3beb45a97bda232ad56fea599182a4480c 100644 --- a/drivers/iio/magnetometer/ak8974.c +++ b/drivers/iio/magnetometer/ak8974.c @@ -969,7 +969,7 @@ disable_pm: return ret; } -static int ak8974_remove(struct i2c_client *i2c) +static void ak8974_remove(struct i2c_client *i2c) { struct iio_dev *indio_dev = i2c_get_clientdata(i2c); struct ak8974 *ak8974 = iio_priv(indio_dev); @@ -981,8 +981,6 @@ static int ak8974_remove(struct i2c_client *i2c) pm_runtime_disable(&i2c->dev); ak8974_set_power(ak8974, AK8974_PWR_OFF); regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs); - - return 0; } static int ak8974_runtime_suspend(struct device *dev) diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c index 2432e697150c655a15a28fa99f01e93fc887357b..caf03a2a98a5decc3433ca932f5af778bc5d6573 100644 --- a/drivers/iio/magnetometer/ak8975.c +++ b/drivers/iio/magnetometer/ak8975.c @@ -1018,7 +1018,7 @@ power_off: return err; } -static int ak8975_remove(struct i2c_client *client) +static void ak8975_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct ak8975_data *data = iio_priv(indio_dev); @@ -1030,8 +1030,6 @@ static int ak8975_remove(struct i2c_client *client) iio_triggered_buffer_cleanup(indio_dev); ak8975_set_mode(data, POWER_DOWN); ak8975_power_off(data); - - return 0; } static int ak8975_runtime_suspend(struct device *dev) diff --git a/drivers/iio/magnetometer/bmc150_magn_i2c.c b/drivers/iio/magnetometer/bmc150_magn_i2c.c index 65c004411d0fa7dadd144e23f2f658a3cef43ad4..570deaa87836b267ac657ab3a13d0d0c1ea42287 100644 --- a/drivers/iio/magnetometer/bmc150_magn_i2c.c +++ b/drivers/iio/magnetometer/bmc150_magn_i2c.c @@ -34,11 +34,9 @@ static int bmc150_magn_i2c_probe(struct i2c_client *client, return bmc150_magn_probe(&client->dev, regmap, client->irq, name); } -static int bmc150_magn_i2c_remove(struct i2c_client *client) +static void bmc150_magn_i2c_remove(struct i2c_client *client) { bmc150_magn_remove(&client->dev); - - return 0; } static const struct acpi_device_id bmc150_magn_acpi_match[] = { diff --git a/drivers/iio/magnetometer/hmc5843.h b/drivers/iio/magnetometer/hmc5843.h index 9120c8bbf3dd59f5078342a52dda247cddadc893..60fbb5431c8809339e8f89c27e359732b2e1d159 100644 --- a/drivers/iio/magnetometer/hmc5843.h +++ b/drivers/iio/magnetometer/hmc5843.h @@ -52,16 +52,5 @@ int hmc5843_common_probe(struct device *dev, struct regmap *regmap, enum hmc5843_ids id, const char *name); void hmc5843_common_remove(struct device *dev); -int hmc5843_common_suspend(struct device *dev); -int hmc5843_common_resume(struct device *dev); - -#ifdef CONFIG_PM_SLEEP -static __maybe_unused SIMPLE_DEV_PM_OPS(hmc5843_pm_ops, - hmc5843_common_suspend, - hmc5843_common_resume); -#define HMC5843_PM_OPS (&hmc5843_pm_ops) -#else -#define HMC5843_PM_OPS NULL -#endif - +extern const struct dev_pm_ops hmc5843_pm_ops; #endif /* HMC5843_CORE_H */ diff --git a/drivers/iio/magnetometer/hmc5843_core.c b/drivers/iio/magnetometer/hmc5843_core.c index 4a63b2da9df03c093b57e22e2f37b3b52dafacee..c5521d61da2958f1f99879e9ad8693e01a2e7da6 100644 --- a/drivers/iio/magnetometer/hmc5843_core.c +++ b/drivers/iio/magnetometer/hmc5843_core.c @@ -603,19 +603,19 @@ static const struct iio_info hmc5843_info = { static const unsigned long hmc5843_scan_masks[] = {0x7, 0}; -int hmc5843_common_suspend(struct device *dev) +static int hmc5843_common_suspend(struct device *dev) { return hmc5843_set_mode(iio_priv(dev_get_drvdata(dev)), HMC5843_MODE_SLEEP); } -EXPORT_SYMBOL_NS(hmc5843_common_suspend, IIO_HMC5843); -int hmc5843_common_resume(struct device *dev) +static int hmc5843_common_resume(struct device *dev) { return hmc5843_set_mode(iio_priv(dev_get_drvdata(dev)), HMC5843_MODE_CONVERSION_CONTINUOUS); } -EXPORT_SYMBOL_NS(hmc5843_common_resume, IIO_HMC5843); +EXPORT_NS_SIMPLE_DEV_PM_OPS(hmc5843_pm_ops, hmc5843_common_suspend, + hmc5843_common_resume, IIO_HMC5843); int hmc5843_common_probe(struct device *dev, struct regmap *regmap, enum hmc5843_ids id, const char *name) diff --git a/drivers/iio/magnetometer/hmc5843_i2c.c b/drivers/iio/magnetometer/hmc5843_i2c.c index 8d2ff8fc204de9e150ab8a38eaa1914c3c5fdd27..18a13dd512961feb6a8d912ee17780bb238f8a4a 100644 --- a/drivers/iio/magnetometer/hmc5843_i2c.c +++ b/drivers/iio/magnetometer/hmc5843_i2c.c @@ -65,11 +65,9 @@ static int hmc5843_i2c_probe(struct i2c_client *cli, id->driver_data, id->name); } -static int hmc5843_i2c_remove(struct i2c_client *client) +static void hmc5843_i2c_remove(struct i2c_client *client) { hmc5843_common_remove(&client->dev); - - return 0; } static const struct i2c_device_id hmc5843_id[] = { @@ -93,7 +91,7 @@ MODULE_DEVICE_TABLE(of, hmc5843_of_match); static struct i2c_driver hmc5843_driver = { .driver = { .name = "hmc5843", - .pm = HMC5843_PM_OPS, + .pm = pm_sleep_ptr(&hmc5843_pm_ops), .of_match_table = hmc5843_of_match, }, .id_table = hmc5843_id, diff --git a/drivers/iio/magnetometer/hmc5843_spi.c b/drivers/iio/magnetometer/hmc5843_spi.c index 8403f09aba39aa57ae49e4a7d13a4b2d4021d378..c42d2e2a6a6ce1e73d94e6cd7bce95d1c523a2c7 100644 --- a/drivers/iio/magnetometer/hmc5843_spi.c +++ b/drivers/iio/magnetometer/hmc5843_spi.c @@ -86,13 +86,13 @@ static const struct spi_device_id hmc5843_id[] = { MODULE_DEVICE_TABLE(spi, hmc5843_id); static struct spi_driver hmc5843_driver = { - .driver = { - .name = "hmc5843", - .pm = HMC5843_PM_OPS, - }, - .id_table = hmc5843_id, - .probe = hmc5843_spi_probe, - .remove = hmc5843_spi_remove, + .driver = { + .name = "hmc5843", + .pm = pm_sleep_ptr(&hmc5843_pm_ops), + }, + .id_table = hmc5843_id, + .probe = hmc5843_spi_probe, + .remove = hmc5843_spi_remove, }; module_spi_driver(hmc5843_driver); diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c index 226439d0bfb505b4ffc368fbd25bec19cbb7a9d9..b870ad8038626e7eed8fbe88409971544f49450e 100644 --- a/drivers/iio/magnetometer/mag3110.c +++ b/drivers/iio/magnetometer/mag3110.c @@ -559,7 +559,7 @@ disable_regulator_vdd: return ret; } -static int mag3110_remove(struct i2c_client *client) +static void mag3110_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mag3110_data *data = iio_priv(indio_dev); @@ -569,8 +569,6 @@ static int mag3110_remove(struct i2c_client *client) mag3110_standby(iio_priv(indio_dev)); regulator_disable(data->vddio_reg); regulator_disable(data->vdd_reg); - - return 0; } static int mag3110_suspend(struct device *dev) diff --git a/drivers/iio/magnetometer/yamaha-yas530.c b/drivers/iio/magnetometer/yamaha-yas530.c index aeaa4da6923b416bb316f1afaab8f0eec1583c6b..801c760feb4d19756e9ea5716609ec3cd910231d 100644 --- a/drivers/iio/magnetometer/yamaha-yas530.c +++ b/drivers/iio/magnetometer/yamaha-yas530.c @@ -10,13 +10,16 @@ * (YAS534 is a magnetic switch, not handled) * YAS535 MS-6C * YAS536 MS-3W - * YAS537 MS-3T (2015 Samsung Galaxy S6, Note 5, Xiaomi) + * YAS537 MS-3T (2015 Samsung Galaxy S6, Note 5, Galaxy S7) * YAS539 MS-3S (2018 Samsung Galaxy A7 SM-A750FN) * * Code functions found in the MPU3050 YAS530 and YAS532 drivers * named "inv_compass" in the Tegra Android kernel tree. * Copyright (C) 2012 InvenSense Corporation * + * Code functions for YAS537 based on Yamaha Android kernel driver. + * Copyright (c) 2014 Yamaha Corporation + * * Author: Linus Walleij */ #include @@ -29,9 +32,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -40,20 +45,39 @@ #include -/* This register map covers YAS530 and YAS532 but differs in YAS 537 and YAS539 */ +/* Commonly used registers */ #define YAS5XX_DEVICE_ID 0x80 -#define YAS5XX_ACTUATE_INIT_COIL 0x81 -#define YAS5XX_MEASURE 0x82 -#define YAS5XX_CONFIG 0x83 -#define YAS5XX_MEASURE_INTERVAL 0x84 -#define YAS5XX_OFFSET_X 0x85 /* [-31 .. 31] */ -#define YAS5XX_OFFSET_Y1 0x86 /* [-31 .. 31] */ -#define YAS5XX_OFFSET_Y2 0x87 /* [-31 .. 31] */ -#define YAS5XX_TEST1 0x88 -#define YAS5XX_TEST2 0x89 -#define YAS5XX_CAL 0x90 #define YAS5XX_MEASURE_DATA 0xB0 +/* These registers are used by YAS530, YAS532 and YAS533 */ +#define YAS530_ACTUATE_INIT_COIL 0x81 +#define YAS530_MEASURE 0x82 +#define YAS530_CONFIG 0x83 +#define YAS530_MEASURE_INTERVAL 0x84 +#define YAS530_OFFSET_X 0x85 /* [-31 .. 31] */ +#define YAS530_OFFSET_Y1 0x86 /* [-31 .. 31] */ +#define YAS530_OFFSET_Y2 0x87 /* [-31 .. 31] */ +#define YAS530_TEST1 0x88 +#define YAS530_TEST2 0x89 +#define YAS530_CAL 0x90 + +/* Registers used by YAS537 */ +#define YAS537_MEASURE 0x81 /* Originally YAS537_REG_CMDR */ +#define YAS537_CONFIG 0x82 /* Originally YAS537_REG_CONFR */ +#define YAS537_MEASURE_INTERVAL 0x83 /* Originally YAS537_REG_INTRVLR */ +#define YAS537_OFFSET_X 0x84 /* Originally YAS537_REG_OXR */ +#define YAS537_OFFSET_Y1 0x85 /* Originally YAS537_REG_OY1R */ +#define YAS537_OFFSET_Y2 0x86 /* Originally YAS537_REG_OY2R */ +#define YAS537_AVR 0x87 +#define YAS537_HCK 0x88 +#define YAS537_LCK 0x89 +#define YAS537_SRST 0x90 +#define YAS537_ADCCAL 0x91 +#define YAS537_MTC 0x93 +#define YAS537_OC 0x9E +#define YAS537_TRM 0x9F +#define YAS537_CAL 0xC0 + /* Bits in the YAS5xx config register */ #define YAS5XX_CONFIG_INTON BIT(0) /* Interrupt on? */ #define YAS5XX_CONFIG_INTHACT BIT(1) /* Interrupt active high? */ @@ -65,6 +89,7 @@ #define YAS5XX_MEASURE_LDTC BIT(1) #define YAS5XX_MEASURE_FORS BIT(2) #define YAS5XX_MEASURE_DLYMES BIT(4) +#define YAS5XX_MEASURE_CONT BIT(5) /* Bits in the measure data register */ #define YAS5XX_MEASURE_DATA_BUSY BIT(7) @@ -88,33 +113,101 @@ #define YAS532_DATA_BITS 13 #define YAS532_DATA_CENTER BIT(YAS532_DATA_BITS - 1) #define YAS532_DATA_OVERFLOW (BIT(YAS532_DATA_BITS) - 1) -#define YAS532_20DEGREES 390 /* Looks like Kelvin */ -/* These variant IDs are known from code dumps */ #define YAS537_DEVICE_ID 0x07 /* YAS537 (MS-3T) */ -#define YAS539_DEVICE_ID 0x08 /* YAS539 (MS-3S) */ +#define YAS537_VERSION_0 0 /* Version naming unknown */ +#define YAS537_VERSION_1 1 /* Version naming unknown */ +#define YAS537_MAG_AVERAGE_32_MASK GENMASK(6, 4) +#define YAS537_MEASURE_TIME_WORST_US 1500 +#define YAS537_DEFAULT_SENSOR_DELAY_MS 50 +#define YAS537_MAG_RCOIL_TIME_US 65 +#define YAS537_MTC3_MASK_PREP GENMASK(7, 0) +#define YAS537_MTC3_MASK_GET GENMASK(7, 5) +#define YAS537_MTC3_ADD_BIT BIT(4) +#define YAS537_HCK_MASK_PREP GENMASK(4, 0) +#define YAS537_HCK_MASK_GET GENMASK(7, 4) +#define YAS537_LCK_MASK_PREP GENMASK(4, 0) +#define YAS537_LCK_MASK_GET GENMASK(3, 0) +#define YAS537_OC_MASK_GET GENMASK(5, 0) /* Turn off device regulators etc after 5 seconds of inactivity */ #define YAS5XX_AUTOSUSPEND_DELAY_MS 5000 +enum chip_ids { + yas530, + yas532, + yas533, + yas537, +}; + +static const int yas530_volatile_reg[] = { + YAS530_ACTUATE_INIT_COIL, + YAS530_MEASURE, +}; + +static const int yas537_volatile_reg[] = { + YAS537_MEASURE, +}; + struct yas5xx_calibration { /* Linearization calibration x, y1, y2 */ s32 r[3]; u32 f[3]; /* Temperature compensation calibration */ - s32 Cx, Cy1, Cy2; + s16 Cx, Cy1, Cy2; /* Misc calibration coefficients */ - s32 a2, a3, a4, a5, a6, a7, a8, a9, k; + s8 a2, a3, a4, a6, a7, a8; + s16 a5, a9; + u8 k; /* clock divider */ u8 dck; }; +struct yas5xx; + +/** + * struct yas5xx_chip_info - device-specific data and function pointers + * @devid: device ID number + * @product_name: product name of the YAS variant + * @version_names: version letters or namings + * @volatile_reg: device-specific volatile registers + * @volatile_reg_qty: quantity of device-specific volatile registers + * @scaling_val2: scaling value for IIO_CHAN_INFO_SCALE + * @t_ref: number of counts at reference temperature 20 °C + * @min_temp_x10: starting point of temperature counting in 1/10:s degrees Celsius + * @get_measure: function pointer to get a measurement + * @get_calibration_data: function pointer to get calibration data + * @dump_calibration: function pointer to dump calibration for debugging + * @measure_offsets: function pointer to measure the offsets + * @power_on: function pointer to power-on procedure + * + * The "t_ref" value for YAS532/533 is known from the Android driver. + * For YAS530 and YAS537 it was approximately measured. + * + * The temperatures "min_temp_x10" are derived from the temperature resolutions + * given in the data sheets. + */ +struct yas5xx_chip_info { + unsigned int devid; + const char *product_name; + const char *version_names[2]; + const int *volatile_reg; + int volatile_reg_qty; + u32 scaling_val2; + u16 t_ref; + s16 min_temp_x10; + int (*get_measure)(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, s32 *zo); + int (*get_calibration_data)(struct yas5xx *yas5xx); + void (*dump_calibration)(struct yas5xx *yas5xx); + int (*measure_offsets)(struct yas5xx *yas5xx); + int (*power_on)(struct yas5xx *yas5xx); +}; + /** * struct yas5xx - state container for the YAS5xx driver * @dev: parent device pointer - * @devid: device ID number + * @chip_info: device-specific data and function pointers * @version: device version - * @name: device name * @calibration: calibration settings from the OTP storage * @hard_offsets: offsets for each axis measured with initcoil actuated * @orientation: mounting matrix, flipped axis etc @@ -128,11 +221,10 @@ struct yas5xx_calibration { */ struct yas5xx { struct device *dev; - unsigned int devid; + const struct yas5xx_chip_info *chip_info; unsigned int version; - char name[16]; struct yas5xx_calibration calibration; - u8 hard_offsets[3]; + s8 hard_offsets[3]; struct iio_mount_matrix orientation; struct regmap *map; struct regulator_bulk_data regs[2]; @@ -179,23 +271,26 @@ static u16 yas532_extract_axis(u8 *data) } /** - * yas5xx_measure() - Make a measure from the hardware + * yas530_measure() - Make a measure from the hardware * @yas5xx: The device state * @t: the raw temperature measurement * @x: the raw x axis measurement * @y1: the y1 axis measurement * @y2: the y2 axis measurement * @return: 0 on success or error code + * + * Used by YAS530, YAS532 and YAS533. */ -static int yas5xx_measure(struct yas5xx *yas5xx, u16 *t, u16 *x, u16 *y1, u16 *y2) +static int yas530_measure(struct yas5xx *yas5xx, u16 *t, u16 *x, u16 *y1, u16 *y2) { + const struct yas5xx_chip_info *ci = yas5xx->chip_info; unsigned int busy; u8 data[8]; int ret; u16 val; mutex_lock(&yas5xx->lock); - ret = regmap_write(yas5xx->map, YAS5XX_MEASURE, YAS5XX_MEASURE_START); + ret = regmap_write(yas5xx->map, YAS530_MEASURE, YAS5XX_MEASURE_START); if (ret < 0) goto out_unlock; @@ -219,7 +314,7 @@ static int yas5xx_measure(struct yas5xx *yas5xx, u16 *t, u16 *x, u16 *y1, u16 *y mutex_unlock(&yas5xx->lock); - switch (yas5xx->devid) { + switch (ci->devid) { case YAS530_DEVICE_ID: /* * The t value is 9 bits in big endian format @@ -261,8 +356,81 @@ out_unlock: return ret; } -static s32 yas5xx_linearize(struct yas5xx *yas5xx, u16 val, int axis) +/** + * yas537_measure() - Make a measure from the hardware + * @yas5xx: The device state + * @t: the raw temperature measurement + * @x: the raw x axis measurement + * @y1: the y1 axis measurement + * @y2: the y2 axis measurement + * @return: 0 on success or error code + */ +static int yas537_measure(struct yas5xx *yas5xx, u16 *t, u16 *x, u16 *y1, u16 *y2) { + struct yas5xx_calibration *c = &yas5xx->calibration; + unsigned int busy; + u8 data[8]; + u16 xy1y2[3]; + s32 h[3], s[3]; + int i, ret; + + mutex_lock(&yas5xx->lock); + + /* Contrary to YAS530/532, also a "cont" bit is set, meaning unknown */ + ret = regmap_write(yas5xx->map, YAS537_MEASURE, YAS5XX_MEASURE_START | + YAS5XX_MEASURE_CONT); + if (ret < 0) + goto out_unlock; + + /* Use same timeout like YAS530/532 but the bit is in data row 2 */ + ret = regmap_read_poll_timeout(yas5xx->map, YAS5XX_MEASURE_DATA + 2, busy, + !(busy & YAS5XX_MEASURE_DATA_BUSY), + 500, 20000); + if (ret) { + dev_err(yas5xx->dev, "timeout waiting for measurement\n"); + goto out_unlock; + } + + ret = regmap_bulk_read(yas5xx->map, YAS5XX_MEASURE_DATA, + data, sizeof(data)); + if (ret) + goto out_unlock; + + mutex_unlock(&yas5xx->lock); + + *t = get_unaligned_be16(&data[0]); + xy1y2[0] = FIELD_GET(GENMASK(13, 0), get_unaligned_be16(&data[2])); + xy1y2[1] = get_unaligned_be16(&data[4]); + xy1y2[2] = get_unaligned_be16(&data[6]); + + /* The second version of YAS537 needs to include calibration coefficients */ + if (yas5xx->version == YAS537_VERSION_1) { + for (i = 0; i < 3; i++) + s[i] = xy1y2[i] - BIT(13); + h[0] = (c->k * (128 * s[0] + c->a2 * s[1] + c->a3 * s[2])) / BIT(13); + h[1] = (c->k * (c->a4 * s[0] + c->a5 * s[1] + c->a6 * s[2])) / BIT(13); + h[2] = (c->k * (c->a7 * s[0] + c->a8 * s[1] + c->a9 * s[2])) / BIT(13); + for (i = 0; i < 3; i++) { + clamp_val(h[i], -BIT(13), BIT(13) - 1); + xy1y2[i] = h[i] + BIT(13); + } + } + + *x = xy1y2[0]; + *y1 = xy1y2[1]; + *y2 = xy1y2[2]; + + return 0; + +out_unlock: + mutex_unlock(&yas5xx->lock); + return ret; +} + +/* Used by YAS530, YAS532 and YAS533 */ +static s32 yas530_linearize(struct yas5xx *yas5xx, u16 val, int axis) +{ + const struct yas5xx_chip_info *ci = yas5xx->chip_info; struct yas5xx_calibration *c = &yas5xx->calibration; static const s32 yas532ac_coef[] = { YAS532_VERSION_AC_COEF_X, @@ -272,7 +440,7 @@ static s32 yas5xx_linearize(struct yas5xx *yas5xx, u16 val, int axis) s32 coef; /* Select coefficients */ - switch (yas5xx->devid) { + switch (ci->devid) { case YAS530_DEVICE_ID: if (yas5xx->version == YAS530_VERSION_A) coef = YAS530_VERSION_A_COEF; @@ -302,8 +470,24 @@ static s32 yas5xx_linearize(struct yas5xx *yas5xx, u16 val, int axis) (yas5xx->hard_offsets[axis] - c->r[axis]) * coef; } +static s32 yas5xx_calc_temperature(struct yas5xx *yas5xx, u16 t) +{ + const struct yas5xx_chip_info *ci = yas5xx->chip_info; + s32 to; + u16 t_ref; + s16 min_temp_x10; + int ref_temp_x10; + + t_ref = ci->t_ref; + min_temp_x10 = ci->min_temp_x10; + ref_temp_x10 = 200; + + to = (min_temp_x10 + ((ref_temp_x10 - min_temp_x10) * t / t_ref)) * 100; + return to; +} + /** - * yas5xx_get_measure() - Measure a sample of all axis and process + * yas530_get_measure() - Measure a sample of all axis and process * @yas5xx: The device state * @to: Temperature out * @xo: X axis out @@ -311,36 +495,50 @@ static s32 yas5xx_linearize(struct yas5xx *yas5xx, u16 val, int axis) * @zo: Z axis out * @return: 0 on success or error code * - * Returned values are in nanotesla according to some code. + * Used by YAS530, YAS532 and YAS533. */ -static int yas5xx_get_measure(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, s32 *zo) +static int yas530_get_measure(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, s32 *zo) { + const struct yas5xx_chip_info *ci = yas5xx->chip_info; struct yas5xx_calibration *c = &yas5xx->calibration; - u16 t, x, y1, y2; - /* These are "signed x, signed y1 etc */ + u16 t_ref, t_comp, t, x, y1, y2; + /* These are signed x, signed y1 etc */ s32 sx, sy1, sy2, sy, sz; int ret; /* We first get raw data that needs to be translated to [x,y,z] */ - ret = yas5xx_measure(yas5xx, &t, &x, &y1, &y2); + ret = yas530_measure(yas5xx, &t, &x, &y1, &y2); if (ret) return ret; /* Do some linearization if available */ - sx = yas5xx_linearize(yas5xx, x, 0); - sy1 = yas5xx_linearize(yas5xx, y1, 1); - sy2 = yas5xx_linearize(yas5xx, y2, 2); + sx = yas530_linearize(yas5xx, x, 0); + sy1 = yas530_linearize(yas5xx, y1, 1); + sy2 = yas530_linearize(yas5xx, y2, 2); + + /* + * Set the temperature for compensation (unit: counts): + * YAS532/YAS533 version AC uses the temperature deviation as a + * multiplier. YAS530 and YAS532 version AB use solely the t value. + */ + t_ref = ci->t_ref; + if (ci->devid == YAS532_DEVICE_ID && + yas5xx->version == YAS532_VERSION_AC) { + t_comp = t - t_ref; + } else { + t_comp = t; + } /* * Temperature compensation for x, y1, y2 respectively: * - * Cx * t - * x' = x - ------ - * 100 + * Cx * t_comp + * x' = x - ----------- + * 100 */ - sx = sx - (c->Cx * t) / 100; - sy1 = sy1 - (c->Cy1 * t) / 100; - sy2 = sy2 - (c->Cy2 * t) / 100; + sx = sx - (c->Cx * t_comp) / 100; + sy1 = sy1 - (c->Cy1 * t_comp) / 100; + sy2 = sy2 - (c->Cy2 * t_comp) / 100; /* * Break y1 and y2 into y and z, y1 and y2 are apparently encoding @@ -349,11 +547,9 @@ static int yas5xx_get_measure(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, sy = sy1 - sy2; sz = -sy1 - sy2; - /* - * FIXME: convert to Celsius? Just guessing this is given - * as 1/10:s of degrees so multiply by 100 to get millicentigrades. - */ - *to = t * 100; + /* Calculate temperature readout */ + *to = yas5xx_calc_temperature(yas5xx, t); + /* * Calibrate [x,y,z] with some formulas like this: * @@ -376,19 +572,56 @@ static int yas5xx_get_measure(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, return 0; } +/** + * yas537_get_measure() - Measure a sample of all axis and process + * @yas5xx: The device state + * @to: Temperature out + * @xo: X axis out + * @yo: Y axis out + * @zo: Z axis out + * @return: 0 on success or error code + */ +static int yas537_get_measure(struct yas5xx *yas5xx, s32 *to, s32 *xo, s32 *yo, s32 *zo) +{ + u16 t, x, y1, y2; + int ret; + + /* We first get raw data that needs to be translated to [x,y,z] */ + ret = yas537_measure(yas5xx, &t, &x, &y1, &y2); + if (ret) + return ret; + + /* Calculate temperature readout */ + *to = yas5xx_calc_temperature(yas5xx, t); + + /* + * Unfortunately, no linearization or temperature compensation formulas + * are known for YAS537. + */ + + /* Calculate x, y, z from x, y1, y2 */ + *xo = (x - BIT(13)) * 300; + *yo = (y1 - y2) * 1732 / 10; + *zo = (-y1 - y2 + BIT(14)) * 300; + + return 0; +} + static int yas5xx_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) { struct yas5xx *yas5xx = iio_priv(indio_dev); + const struct yas5xx_chip_info *ci = yas5xx->chip_info; s32 t, x, y, z; int ret; switch (mask) { + case IIO_CHAN_INFO_PROCESSED: case IIO_CHAN_INFO_RAW: pm_runtime_get_sync(yas5xx->dev); - ret = yas5xx_get_measure(yas5xx, &t, &x, &y, &z); + ret = ci->get_measure(yas5xx, &t, &x, &y, &z); pm_runtime_mark_last_busy(yas5xx->dev); pm_runtime_put_autosuspend(yas5xx->dev); if (ret) @@ -412,19 +645,8 @@ static int yas5xx_read_raw(struct iio_dev *indio_dev, } return IIO_VAL_INT; case IIO_CHAN_INFO_SCALE: - if (chan->address == 0) { - /* Temperature is unscaled */ - *val = 1; - return IIO_VAL_INT; - } - /* - * The axis values are in nanotesla according to the vendor - * drivers, but is clearly in microtesla according to - * experiments. Since 1 uT = 0.01 Gauss, we need to divide - * by 100000000 (10^8) to get to Gauss from the raw value. - */ *val = 1; - *val2 = 100000000; + *val2 = ci->scaling_val2; return IIO_VAL_FRACTIONAL; default: /* Unknown request */ @@ -435,11 +657,12 @@ static int yas5xx_read_raw(struct iio_dev *indio_dev, static void yas5xx_fill_buffer(struct iio_dev *indio_dev) { struct yas5xx *yas5xx = iio_priv(indio_dev); + const struct yas5xx_chip_info *ci = yas5xx->chip_info; s32 t, x, y, z; int ret; pm_runtime_get_sync(yas5xx->dev); - ret = yas5xx_get_measure(yas5xx, &t, &x, &y, &z); + ret = ci->get_measure(yas5xx, &t, &x, &y, &z); pm_runtime_mark_last_busy(yas5xx->dev); pm_runtime_put_autosuspend(yas5xx->dev); if (ret) { @@ -505,7 +728,7 @@ static const struct iio_chan_spec yas5xx_channels[] = { .address = 0, .scan_index = 0, .scan_type = { - .sign = 'u', + .sign = 's', .realbits = 32, .storagebits = 32, .endianness = IIO_CPU, @@ -525,9 +748,26 @@ static const struct iio_info yas5xx_info = { static bool yas5xx_volatile_reg(struct device *dev, unsigned int reg) { - return reg == YAS5XX_ACTUATE_INIT_COIL || - reg == YAS5XX_MEASURE || - (reg >= YAS5XX_MEASURE_DATA && reg <= YAS5XX_MEASURE_DATA + 8); + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct yas5xx *yas5xx = iio_priv(indio_dev); + const struct yas5xx_chip_info *ci = yas5xx->chip_info; + int reg_qty; + int i; + + if (reg >= YAS5XX_MEASURE_DATA && reg < YAS5XX_MEASURE_DATA + 8) + return true; + + /* + * YAS versions share different registers on the same address, + * need to differentiate. + */ + reg_qty = ci->volatile_reg_qty; + for (i = 0; i < reg_qty; i++) { + if (reg == ci->volatile_reg[i]) + return true; + } + + return false; } /* TODO: enable regmap cache, using mark dirty and sync at runtime resume */ @@ -539,11 +779,13 @@ static const struct regmap_config yas5xx_regmap_config = { }; /** - * yas53x_extract_calibration() - extracts the a2-a9 and k calibration + * yas530_extract_calibration() - extracts the a2-a9 and k calibration * @data: the bitfield to use * @c: the calibration to populate + * + * Used by YAS530, YAS532 and YAS533. */ -static void yas53x_extract_calibration(u8 *data, struct yas5xx_calibration *c) +static void yas530_extract_calibration(u8 *data, struct yas5xx_calibration *c) { u64 val = get_unaligned_be64(data); @@ -581,24 +823,27 @@ static int yas530_get_calibration_data(struct yas5xx *yas5xx) int ret; /* Dummy read, first read is ALWAYS wrong */ - ret = regmap_bulk_read(yas5xx->map, YAS5XX_CAL, data, sizeof(data)); + ret = regmap_bulk_read(yas5xx->map, YAS530_CAL, data, sizeof(data)); if (ret) return ret; /* Actual calibration readout */ - ret = regmap_bulk_read(yas5xx->map, YAS5XX_CAL, data, sizeof(data)); + ret = regmap_bulk_read(yas5xx->map, YAS530_CAL, data, sizeof(data)); if (ret) return ret; - dev_dbg(yas5xx->dev, "calibration data: %*ph\n", 14, data); + dev_dbg(yas5xx->dev, "calibration data: %16ph\n", data); + /* Contribute calibration data to the input pool for kernel entropy */ add_device_randomness(data, sizeof(data)); + + /* Extract version */ yas5xx->version = data[15] & GENMASK(1, 0); /* Extract the calibration from the bitfield */ c->Cx = data[0] * 6 - 768; c->Cy1 = data[1] * 6 - 768; c->Cy2 = data[2] * 6 - 768; - yas53x_extract_calibration(&data[3], c); + yas530_extract_calibration(&data[3], c); /* * Extract linearization: @@ -618,6 +863,7 @@ static int yas530_get_calibration_data(struct yas5xx *yas5xx) c->r[0] = sign_extend32(FIELD_GET(GENMASK(28, 23), val), 5); c->r[1] = sign_extend32(FIELD_GET(GENMASK(20, 15), val), 5); c->r[2] = sign_extend32(FIELD_GET(GENMASK(12, 7), val), 5); + return 0; } @@ -629,22 +875,22 @@ static int yas532_get_calibration_data(struct yas5xx *yas5xx) int ret; /* Dummy read, first read is ALWAYS wrong */ - ret = regmap_bulk_read(yas5xx->map, YAS5XX_CAL, data, sizeof(data)); + ret = regmap_bulk_read(yas5xx->map, YAS530_CAL, data, sizeof(data)); if (ret) return ret; /* Actual calibration readout */ - ret = regmap_bulk_read(yas5xx->map, YAS5XX_CAL, data, sizeof(data)); + ret = regmap_bulk_read(yas5xx->map, YAS530_CAL, data, sizeof(data)); if (ret) return ret; - dev_dbg(yas5xx->dev, "calibration data: %*ph\n", 14, data); + dev_dbg(yas5xx->dev, "calibration data: %14ph\n", data); /* Sanity check, is this all zeroes? */ - if (memchr_inv(data, 0x00, 13) == NULL) { - if (!(data[13] & BIT(7))) - dev_warn(yas5xx->dev, "calibration is blank!\n"); - } + if (!memchr_inv(data, 0x00, 13) && !(data[13] & BIT(7))) + dev_warn(yas5xx->dev, "calibration is blank!\n"); + /* Contribute calibration data to the input pool for kernel entropy */ add_device_randomness(data, sizeof(data)); + /* Only one bit of version info reserved here as far as we know */ yas5xx->version = data[13] & BIT(0); @@ -652,7 +898,8 @@ static int yas532_get_calibration_data(struct yas5xx *yas5xx) c->Cx = data[0] * 10 - 1280; c->Cy1 = data[1] * 10 - 1280; c->Cy2 = data[2] * 10 - 1280; - yas53x_extract_calibration(&data[3], c); + yas530_extract_calibration(&data[3], c); + /* * Extract linearization: * Linearization layout in the 32 bits at byte 10: @@ -675,7 +922,204 @@ static int yas532_get_calibration_data(struct yas5xx *yas5xx) return 0; } -static void yas5xx_dump_calibration(struct yas5xx *yas5xx) +static int yas537_get_calibration_data(struct yas5xx *yas5xx) +{ + struct yas5xx_calibration *c = &yas5xx->calibration; + u8 data[17]; + u32 val1, val2, val3, val4; + int i, ret; + + /* Writing SRST register */ + ret = regmap_write(yas5xx->map, YAS537_SRST, BIT(1)); + if (ret) + return ret; + + /* Calibration readout, YAS537 needs one readout only */ + ret = regmap_bulk_read(yas5xx->map, YAS537_CAL, data, sizeof(data)); + if (ret) + return ret; + dev_dbg(yas5xx->dev, "calibration data: %17ph\n", data); + + /* Sanity check, is this all zeroes? */ + if (!memchr_inv(data, 0x00, 16) && !FIELD_GET(GENMASK(5, 0), data[16])) + dev_warn(yas5xx->dev, "calibration is blank!\n"); + + /* Contribute calibration data to the input pool for kernel entropy */ + add_device_randomness(data, sizeof(data)); + + /* Extract version information */ + yas5xx->version = FIELD_GET(GENMASK(7, 6), data[16]); + + /* There are two versions of YAS537 behaving differently */ + switch (yas5xx->version) { + case YAS537_VERSION_0: + /* + * The first version simply writes data back into registers: + * + * data[0] YAS537_MTC 0x93 + * data[1] 0x94 + * data[2] 0x95 + * data[3] 0x96 + * data[4] 0x97 + * data[5] 0x98 + * data[6] 0x99 + * data[7] 0x9a + * data[8] 0x9b + * data[9] 0x9c + * data[10] 0x9d + * data[11] YAS537_OC 0x9e + * + * data[12] YAS537_OFFSET_X 0x84 + * data[13] YAS537_OFFSET_Y1 0x85 + * data[14] YAS537_OFFSET_Y2 0x86 + * + * data[15] YAS537_HCK 0x88 + * data[16] YAS537_LCK 0x89 + */ + for (i = 0; i < 12; i++) { + ret = regmap_write(yas5xx->map, YAS537_MTC + i, + data[i]); + if (ret) + return ret; + } + for (i = 0; i < 3; i++) { + ret = regmap_write(yas5xx->map, YAS537_OFFSET_X + i, + data[i + 12]); + if (ret) + return ret; + yas5xx->hard_offsets[i] = data[i + 12]; + } + for (i = 0; i < 2; i++) { + ret = regmap_write(yas5xx->map, YAS537_HCK + i, + data[i + 15]); + if (ret) + return ret; + } + break; + case YAS537_VERSION_1: + /* + * The second version writes some data into registers but also + * extracts calibration coefficients. + * + * Registers being written: + * + * data[0] YAS537_MTC 0x93 + * data[1] YAS537_MTC+1 0x94 + * data[2] YAS537_MTC+2 0x95 + * data[3] YAS537_MTC+3 (partially) 0x96 + * + * data[12] YAS537_OFFSET_X 0x84 + * data[13] YAS537_OFFSET_Y1 0x85 + * data[14] YAS537_OFFSET_Y2 0x86 + * + * data[15] YAS537_HCK (partially) 0x88 + * YAS537_LCK (partially) 0x89 + * data[16] YAS537_OC (partially) 0x9e + */ + for (i = 0; i < 3; i++) { + ret = regmap_write(yas5xx->map, YAS537_MTC + i, + data[i]); + if (ret) + return ret; + } + for (i = 0; i < 3; i++) { + ret = regmap_write(yas5xx->map, YAS537_OFFSET_X + i, + data[i + 12]); + if (ret) + return ret; + yas5xx->hard_offsets[i] = data[i + 12]; + } + /* + * Visualization of partially taken data: + * + * data[3] n 7 6 5 4 3 2 1 0 + * YAS537_MTC+3 x x x 1 0 0 0 0 + * + * data[15] n 7 6 5 4 3 2 1 0 + * YAS537_HCK x x x x 0 + * + * data[15] n 7 6 5 4 3 2 1 0 + * YAS537_LCK x x x x 0 + * + * data[16] n 7 6 5 4 3 2 1 0 + * YAS537_OC x x x x x x + */ + ret = regmap_write(yas5xx->map, YAS537_MTC + 3, + FIELD_PREP(YAS537_MTC3_MASK_PREP, + FIELD_GET(YAS537_MTC3_MASK_GET, data[3])) | + YAS537_MTC3_ADD_BIT); + if (ret) + return ret; + ret = regmap_write(yas5xx->map, YAS537_HCK, + FIELD_PREP(YAS537_HCK_MASK_PREP, + FIELD_GET(YAS537_HCK_MASK_GET, data[15]))); + if (ret) + return ret; + ret = regmap_write(yas5xx->map, YAS537_LCK, + FIELD_PREP(YAS537_LCK_MASK_PREP, + FIELD_GET(YAS537_LCK_MASK_GET, data[15]))); + if (ret) + return ret; + ret = regmap_write(yas5xx->map, YAS537_OC, + FIELD_GET(YAS537_OC_MASK_GET, data[16])); + if (ret) + return ret; + /* + * For data extraction, build some blocks. Four 32-bit blocks + * look appropriate. + * + * n 7 6 5 4 3 2 1 0 + * data[0] 0 [ Cx Cx Cx Cx Cx Cx Cx Cx ] bits 31 .. 24 + * data[1] 1 [ Cx C1 C1 C1 C1 C1 C1 C1 ] bits 23 .. 16 + * data[2] 2 [ C1 C1 C2 C2 C2 C2 C2 C2 ] bits 15 .. 8 + * data[3] 3 [ C2 C2 C2 ] bits 7 .. 0 + * + * n 7 6 5 4 3 2 1 0 + * data[3] 0 [ a2 a2 a2 a2 a2 ] bits 31 .. 24 + * data[4] 1 [ a2 a2 a3 a3 a3 a3 a3 a3 ] bits 23 .. 16 + * data[5] 2 [ a3 a4 a4 a4 a4 a4 a4 a4 ] bits 15 .. 8 + * data[6] 3 [ a4 ] bits 7 .. 0 + * + * n 7 6 5 4 3 2 1 0 + * data[6] 0 [ a5 a5 a5 a5 a5 a5 a5 ] bits 31 .. 24 + * data[7] 1 [ a5 a5 a6 a6 a6 a6 a6 a6 ] bits 23 .. 16 + * data[8] 2 [ a6 a7 a7 a7 a7 a7 a7 a7 ] bits 15 .. 8 + * data[9] 3 [ a7 ] bits 7 .. 0 + * + * n 7 6 5 4 3 2 1 0 + * data[9] 0 [ a8 a8 a8 a8 a8 a8 a8 ] bits 31 .. 24 + * data[10] 1 [ a9 a9 a9 a9 a9 a9 a9 a9 ] bits 23 .. 16 + * data[11] 2 [ a9 k k k k k k k ] bits 15 .. 8 + * data[12] 3 [ ] bits 7 .. 0 + */ + val1 = get_unaligned_be32(&data[0]); + val2 = get_unaligned_be32(&data[3]); + val3 = get_unaligned_be32(&data[6]); + val4 = get_unaligned_be32(&data[9]); + /* Extract calibration coefficients and modify */ + c->Cx = FIELD_GET(GENMASK(31, 23), val1) - 256; + c->Cy1 = FIELD_GET(GENMASK(22, 14), val1) - 256; + c->Cy2 = FIELD_GET(GENMASK(13, 5), val1) - 256; + c->a2 = FIELD_GET(GENMASK(28, 22), val2) - 64; + c->a3 = FIELD_GET(GENMASK(21, 15), val2) - 64; + c->a4 = FIELD_GET(GENMASK(14, 7), val2) - 128; + c->a5 = FIELD_GET(GENMASK(30, 22), val3) - 112; + c->a6 = FIELD_GET(GENMASK(21, 15), val3) - 64; + c->a7 = FIELD_GET(GENMASK(14, 7), val3) - 128; + c->a8 = FIELD_GET(GENMASK(30, 24), val4) - 64; + c->a9 = FIELD_GET(GENMASK(23, 15), val4) - 112; + c->k = FIELD_GET(GENMASK(14, 8), val4); + break; + default: + dev_err(yas5xx->dev, "unknown version of YAS537\n"); + return -EINVAL; + } + + return 0; +} + +/* Used by YAS530, YAS532 and YAS533 */ +static void yas530_dump_calibration(struct yas5xx *yas5xx) { struct yas5xx_calibration *c = &yas5xx->calibration; @@ -698,20 +1142,42 @@ static void yas5xx_dump_calibration(struct yas5xx *yas5xx) dev_dbg(yas5xx->dev, "dck = %d\n", c->dck); } -static int yas5xx_set_offsets(struct yas5xx *yas5xx, s8 ox, s8 oy1, s8 oy2) +static void yas537_dump_calibration(struct yas5xx *yas5xx) +{ + struct yas5xx_calibration *c = &yas5xx->calibration; + + if (yas5xx->version == YAS537_VERSION_1) { + dev_dbg(yas5xx->dev, "Cx = %d\n", c->Cx); + dev_dbg(yas5xx->dev, "Cy1 = %d\n", c->Cy1); + dev_dbg(yas5xx->dev, "Cy2 = %d\n", c->Cy2); + dev_dbg(yas5xx->dev, "a2 = %d\n", c->a2); + dev_dbg(yas5xx->dev, "a3 = %d\n", c->a3); + dev_dbg(yas5xx->dev, "a4 = %d\n", c->a4); + dev_dbg(yas5xx->dev, "a5 = %d\n", c->a5); + dev_dbg(yas5xx->dev, "a6 = %d\n", c->a6); + dev_dbg(yas5xx->dev, "a7 = %d\n", c->a7); + dev_dbg(yas5xx->dev, "a8 = %d\n", c->a8); + dev_dbg(yas5xx->dev, "a9 = %d\n", c->a9); + dev_dbg(yas5xx->dev, "k = %d\n", c->k); + } +} + +/* Used by YAS530, YAS532 and YAS533 */ +static int yas530_set_offsets(struct yas5xx *yas5xx, s8 ox, s8 oy1, s8 oy2) { int ret; - ret = regmap_write(yas5xx->map, YAS5XX_OFFSET_X, ox); + ret = regmap_write(yas5xx->map, YAS530_OFFSET_X, ox); if (ret) return ret; - ret = regmap_write(yas5xx->map, YAS5XX_OFFSET_Y1, oy1); + ret = regmap_write(yas5xx->map, YAS530_OFFSET_Y1, oy1); if (ret) return ret; - return regmap_write(yas5xx->map, YAS5XX_OFFSET_Y2, oy2); + return regmap_write(yas5xx->map, YAS530_OFFSET_Y2, oy2); } -static s8 yas5xx_adjust_offset(s8 old, int bit, u16 center, u16 measure) +/* Used by YAS530, YAS532 and YAS533 */ +static s8 yas530_adjust_offset(s8 old, int bit, u16 center, u16 measure) { if (measure > center) return old + BIT(bit); @@ -720,8 +1186,10 @@ static s8 yas5xx_adjust_offset(s8 old, int bit, u16 center, u16 measure) return old; } -static int yas5xx_meaure_offsets(struct yas5xx *yas5xx) +/* Used by YAS530, YAS532 and YAS533 */ +static int yas530_measure_offsets(struct yas5xx *yas5xx) { + const struct yas5xx_chip_info *ci = yas5xx->chip_info; int ret; u16 center; u16 t, x, y1, y2; @@ -729,12 +1197,12 @@ static int yas5xx_meaure_offsets(struct yas5xx *yas5xx) int i; /* Actuate the init coil and measure offsets */ - ret = regmap_write(yas5xx->map, YAS5XX_ACTUATE_INIT_COIL, 0); + ret = regmap_write(yas5xx->map, YAS530_ACTUATE_INIT_COIL, 0); if (ret) return ret; /* When the initcoil is active this should be around the center */ - switch (yas5xx->devid) { + switch (ci->devid) { case YAS530_DEVICE_ID: center = YAS530_DATA_CENTER; break; @@ -763,26 +1231,26 @@ static int yas5xx_meaure_offsets(struct yas5xx *yas5xx) oy2 = 0; for (i = 4; i >= 0; i--) { - ret = yas5xx_set_offsets(yas5xx, ox, oy1, oy2); + ret = yas530_set_offsets(yas5xx, ox, oy1, oy2); if (ret) return ret; - ret = yas5xx_measure(yas5xx, &t, &x, &y1, &y2); + ret = yas530_measure(yas5xx, &t, &x, &y1, &y2); if (ret) return ret; dev_dbg(yas5xx->dev, "measurement %d: x=%d, y1=%d, y2=%d\n", 5-i, x, y1, y2); - ox = yas5xx_adjust_offset(ox, i, center, x); - oy1 = yas5xx_adjust_offset(oy1, i, center, y1); - oy2 = yas5xx_adjust_offset(oy2, i, center, y2); + ox = yas530_adjust_offset(ox, i, center, x); + oy1 = yas530_adjust_offset(oy1, i, center, y1); + oy2 = yas530_adjust_offset(oy2, i, center, y2); } /* Needed for calibration algorithm */ yas5xx->hard_offsets[0] = ox; yas5xx->hard_offsets[1] = oy1; yas5xx->hard_offsets[2] = oy2; - ret = yas5xx_set_offsets(yas5xx, ox, oy1, oy2); + ret = yas530_set_offsets(yas5xx, ox, oy1, oy2); if (ret) return ret; @@ -791,35 +1259,139 @@ static int yas5xx_meaure_offsets(struct yas5xx *yas5xx) return 0; } -static int yas5xx_power_on(struct yas5xx *yas5xx) +/* Used by YAS530, YAS532 and YAS533 */ +static int yas530_power_on(struct yas5xx *yas5xx) { unsigned int val; int ret; /* Zero the test registers */ - ret = regmap_write(yas5xx->map, YAS5XX_TEST1, 0); + ret = regmap_write(yas5xx->map, YAS530_TEST1, 0); if (ret) return ret; - ret = regmap_write(yas5xx->map, YAS5XX_TEST2, 0); + ret = regmap_write(yas5xx->map, YAS530_TEST2, 0); if (ret) return ret; /* Set up for no interrupts, calibrated clock divider */ val = FIELD_PREP(YAS5XX_CONFIG_CCK_MASK, yas5xx->calibration.dck); - ret = regmap_write(yas5xx->map, YAS5XX_CONFIG, val); + ret = regmap_write(yas5xx->map, YAS530_CONFIG, val); if (ret) return ret; /* Measure interval 0 (back-to-back?) */ - return regmap_write(yas5xx->map, YAS5XX_MEASURE_INTERVAL, 0); + return regmap_write(yas5xx->map, YAS530_MEASURE_INTERVAL, 0); } +static int yas537_power_on(struct yas5xx *yas5xx) +{ + __be16 buf; + int ret; + u8 intrvl; + + /* Writing ADCCAL and TRM registers */ + buf = cpu_to_be16(GENMASK(9, 3)); + ret = regmap_bulk_write(yas5xx->map, YAS537_ADCCAL, &buf, sizeof(buf)); + if (ret) + return ret; + ret = regmap_write(yas5xx->map, YAS537_TRM, GENMASK(7, 0)); + if (ret) + return ret; + + /* The interval value is static in regular operation */ + intrvl = (YAS537_DEFAULT_SENSOR_DELAY_MS * MILLI + - YAS537_MEASURE_TIME_WORST_US) / 4100; + ret = regmap_write(yas5xx->map, YAS537_MEASURE_INTERVAL, intrvl); + if (ret) + return ret; + + /* The average value is also static in regular operation */ + ret = regmap_write(yas5xx->map, YAS537_AVR, YAS537_MAG_AVERAGE_32_MASK); + if (ret) + return ret; + + /* Perform the "rcoil" part but skip the "last_after_rcoil" read */ + ret = regmap_write(yas5xx->map, YAS537_CONFIG, BIT(3)); + if (ret) + return ret; + + /* Wait until the coil has ramped up */ + usleep_range(YAS537_MAG_RCOIL_TIME_US, YAS537_MAG_RCOIL_TIME_US + 100); + + return 0; +} + +static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = { + [yas530] = { + .devid = YAS530_DEVICE_ID, + .product_name = "YAS530 MS-3E", + .version_names = { "A", "B" }, + .volatile_reg = yas530_volatile_reg, + .volatile_reg_qty = ARRAY_SIZE(yas530_volatile_reg), + .scaling_val2 = 100000000, /* picotesla to Gauss */ + .t_ref = 182, /* counts */ + .min_temp_x10 = -620, /* 1/10:s degrees Celsius */ + .get_measure = yas530_get_measure, + .get_calibration_data = yas530_get_calibration_data, + .dump_calibration = yas530_dump_calibration, + .measure_offsets = yas530_measure_offsets, + .power_on = yas530_power_on, + }, + [yas532] = { + .devid = YAS532_DEVICE_ID, + .product_name = "YAS532 MS-3R", + .version_names = { "AB", "AC" }, + .volatile_reg = yas530_volatile_reg, + .volatile_reg_qty = ARRAY_SIZE(yas530_volatile_reg), + .scaling_val2 = 100000, /* nanotesla to Gauss */ + .t_ref = 390, /* counts */ + .min_temp_x10 = -500, /* 1/10:s degrees Celsius */ + .get_measure = yas530_get_measure, + .get_calibration_data = yas532_get_calibration_data, + .dump_calibration = yas530_dump_calibration, + .measure_offsets = yas530_measure_offsets, + .power_on = yas530_power_on, + }, + [yas533] = { + .devid = YAS532_DEVICE_ID, + .product_name = "YAS533 MS-3F", + .version_names = { "AB", "AC" }, + .volatile_reg = yas530_volatile_reg, + .volatile_reg_qty = ARRAY_SIZE(yas530_volatile_reg), + .scaling_val2 = 100000, /* nanotesla to Gauss */ + .t_ref = 390, /* counts */ + .min_temp_x10 = -500, /* 1/10:s degrees Celsius */ + .get_measure = yas530_get_measure, + .get_calibration_data = yas532_get_calibration_data, + .dump_calibration = yas530_dump_calibration, + .measure_offsets = yas530_measure_offsets, + .power_on = yas530_power_on, + }, + [yas537] = { + .devid = YAS537_DEVICE_ID, + .product_name = "YAS537 MS-3T", + .version_names = { "v0", "v1" }, /* version naming unknown */ + .volatile_reg = yas537_volatile_reg, + .volatile_reg_qty = ARRAY_SIZE(yas537_volatile_reg), + .scaling_val2 = 100000, /* nanotesla to Gauss */ + .t_ref = 8120, /* counts */ + .min_temp_x10 = -3860, /* 1/10:s degrees Celsius */ + .get_measure = yas537_get_measure, + .get_calibration_data = yas537_get_calibration_data, + .dump_calibration = yas537_dump_calibration, + /* .measure_offets is not needed for yas537 */ + .power_on = yas537_power_on, + }, +}; + static int yas5xx_probe(struct i2c_client *i2c, const struct i2c_device_id *id) { struct iio_dev *indio_dev; struct device *dev = &i2c->dev; struct yas5xx *yas5xx; + const struct yas5xx_chip_info *ci; + int id_check; int ret; indio_dev = devm_iio_device_alloc(dev, sizeof(*yas5xx)); @@ -843,10 +1415,8 @@ static int yas5xx_probe(struct i2c_client *i2c, return dev_err_probe(dev, ret, "cannot get regulators\n"); ret = regulator_bulk_enable(ARRAY_SIZE(yas5xx->regs), yas5xx->regs); - if (ret) { - dev_err(dev, "cannot enable regulators\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "cannot enable regulators\n"); /* See comment in runtime resume callback */ usleep_range(31000, 40000); @@ -854,57 +1424,55 @@ static int yas5xx_probe(struct i2c_client *i2c, /* This will take the device out of reset if need be */ yas5xx->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(yas5xx->reset)) { - ret = dev_err_probe(dev, PTR_ERR(yas5xx->reset), - "failed to get reset line\n"); + ret = dev_err_probe(dev, PTR_ERR(yas5xx->reset), "failed to get reset line\n"); goto reg_off; } yas5xx->map = devm_regmap_init_i2c(i2c, &yas5xx_regmap_config); if (IS_ERR(yas5xx->map)) { - dev_err(dev, "failed to allocate register map\n"); - ret = PTR_ERR(yas5xx->map); + ret = dev_err_probe(dev, PTR_ERR(yas5xx->map), "failed to allocate register map\n"); goto assert_reset; } - ret = regmap_read(yas5xx->map, YAS5XX_DEVICE_ID, &yas5xx->devid); + ci = device_get_match_data(dev); + if (!ci) + ci = (const struct yas5xx_chip_info *)id->driver_data; + yas5xx->chip_info = ci; + + ret = regmap_read(yas5xx->map, YAS5XX_DEVICE_ID, &id_check); if (ret) goto assert_reset; - switch (yas5xx->devid) { - case YAS530_DEVICE_ID: - ret = yas530_get_calibration_data(yas5xx); - if (ret) - goto assert_reset; - dev_info(dev, "detected YAS530 MS-3E %s", - yas5xx->version ? "B" : "A"); - strncpy(yas5xx->name, "yas530", sizeof(yas5xx->name)); - break; - case YAS532_DEVICE_ID: - ret = yas532_get_calibration_data(yas5xx); - if (ret) - goto assert_reset; - dev_info(dev, "detected YAS532/YAS533 MS-3R/F %s", - yas5xx->version ? "AC" : "AB"); - strncpy(yas5xx->name, "yas532", sizeof(yas5xx->name)); - break; - default: - ret = -ENODEV; - dev_err(dev, "unhandled device ID %02x\n", yas5xx->devid); + if (id_check != ci->devid) { + ret = dev_err_probe(dev, -ENODEV, + "device ID %02x doesn't match %s\n", + id_check, id->name); goto assert_reset; } - yas5xx_dump_calibration(yas5xx); - ret = yas5xx_power_on(yas5xx); + ret = ci->get_calibration_data(yas5xx); if (ret) goto assert_reset; - ret = yas5xx_meaure_offsets(yas5xx); + + dev_info(dev, "detected %s %s\n", ci->product_name, + ci->version_names[yas5xx->version]); + + ci->dump_calibration(yas5xx); + + ret = ci->power_on(yas5xx); if (ret) goto assert_reset; + if (ci->measure_offsets) { + ret = ci->measure_offsets(yas5xx); + if (ret) + goto assert_reset; + } + indio_dev->info = &yas5xx_info; indio_dev->available_scan_masks = yas5xx_scan_masks; indio_dev->modes = INDIO_DIRECT_MODE; - indio_dev->name = yas5xx->name; + indio_dev->name = id->name; indio_dev->channels = yas5xx_channels; indio_dev->num_channels = ARRAY_SIZE(yas5xx_channels); @@ -912,13 +1480,13 @@ static int yas5xx_probe(struct i2c_client *i2c, yas5xx_handle_trigger, NULL); if (ret) { - dev_err(dev, "triggered buffer setup failed\n"); + dev_err_probe(dev, ret, "triggered buffer setup failed\n"); goto assert_reset; } ret = iio_device_register(indio_dev); if (ret) { - dev_err(dev, "device register failed\n"); + dev_err_probe(dev, ret, "device register failed\n"); goto cleanup_buffer; } @@ -943,7 +1511,7 @@ reg_off: return ret; } -static int yas5xx_remove(struct i2c_client *i2c) +static void yas5xx_remove(struct i2c_client *i2c) { struct iio_dev *indio_dev = i2c_get_clientdata(i2c); struct yas5xx *yas5xx = iio_priv(indio_dev); @@ -961,8 +1529,6 @@ static int yas5xx_remove(struct i2c_client *i2c) pm_runtime_disable(dev); gpiod_set_value_cansleep(yas5xx->reset, 1); regulator_bulk_disable(ARRAY_SIZE(yas5xx->regs), yas5xx->regs); - - return 0; } static int yas5xx_runtime_suspend(struct device *dev) @@ -980,6 +1546,7 @@ static int yas5xx_runtime_resume(struct device *dev) { struct iio_dev *indio_dev = dev_get_drvdata(dev); struct yas5xx *yas5xx = iio_priv(indio_dev); + const struct yas5xx_chip_info *ci = yas5xx->chip_info; int ret; ret = regulator_bulk_enable(ARRAY_SIZE(yas5xx->regs), yas5xx->regs); @@ -996,7 +1563,7 @@ static int yas5xx_runtime_resume(struct device *dev) usleep_range(31000, 40000); gpiod_set_value_cansleep(yas5xx->reset, 0); - ret = yas5xx_power_on(yas5xx); + ret = ci->power_on(yas5xx); if (ret) { dev_err(dev, "cannot power on\n"); goto out_reset; @@ -1015,17 +1582,19 @@ static DEFINE_RUNTIME_DEV_PM_OPS(yas5xx_dev_pm_ops, yas5xx_runtime_suspend, yas5xx_runtime_resume, NULL); static const struct i2c_device_id yas5xx_id[] = { - {"yas530", }, - {"yas532", }, - {"yas533", }, + {"yas530", (kernel_ulong_t)&yas5xx_chip_info_tbl[yas530] }, + {"yas532", (kernel_ulong_t)&yas5xx_chip_info_tbl[yas532] }, + {"yas533", (kernel_ulong_t)&yas5xx_chip_info_tbl[yas533] }, + {"yas537", (kernel_ulong_t)&yas5xx_chip_info_tbl[yas537] }, {} }; MODULE_DEVICE_TABLE(i2c, yas5xx_id); static const struct of_device_id yas5xx_of_match[] = { - { .compatible = "yamaha,yas530", }, - { .compatible = "yamaha,yas532", }, - { .compatible = "yamaha,yas533", }, + { .compatible = "yamaha,yas530", &yas5xx_chip_info_tbl[yas530] }, + { .compatible = "yamaha,yas532", &yas5xx_chip_info_tbl[yas532] }, + { .compatible = "yamaha,yas533", &yas5xx_chip_info_tbl[yas533] }, + { .compatible = "yamaha,yas537", &yas5xx_chip_info_tbl[yas537] }, {} }; MODULE_DEVICE_TABLE(of, yas5xx_of_match); diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c index fe514f0b5506515476a783486c3ed35e8b4c5f0d..5ec7060d31d901cc88a4aefbc9d34f09e0c93304 100644 --- a/drivers/iio/potentiostat/lmp91000.c +++ b/drivers/iio/potentiostat/lmp91000.c @@ -384,7 +384,7 @@ error_unreg_trigger: return ret; } -static int lmp91000_remove(struct i2c_client *client) +static void lmp91000_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct lmp91000_data *data = iio_priv(indio_dev); @@ -396,8 +396,6 @@ static int lmp91000_remove(struct i2c_client *client) iio_triggered_buffer_cleanup(indio_dev); iio_trigger_unregister(data->trig); - - return 0; } static const struct of_device_id lmp91000_of_match[] = { diff --git a/drivers/iio/pressure/Kconfig b/drivers/iio/pressure/Kconfig index 0ff756cea63ab63d38d5d8d91e537fee00b384c9..c9453389e4f724cdd9ffcf2f8821d2b2aa47299b 100644 --- a/drivers/iio/pressure/Kconfig +++ b/drivers/iio/pressure/Kconfig @@ -17,14 +17,14 @@ config ABP060MG will be called abp060mg. config BMP280 - tristate "Bosch Sensortec BMP180/BMP280 pressure sensor I2C driver" + tristate "Bosch Sensortec BMP180/BMP280/BMP380 pressure sensor I2C driver" depends on (I2C || SPI_MASTER) select REGMAP select BMP280_I2C if (I2C) select BMP280_SPI if (SPI_MASTER) help - Say yes here to build support for Bosch Sensortec BMP180 and BMP280 - pressure and temperature sensors. Also supports the BME280 with + Say yes here to build support for Bosch Sensortec BMP180, BMP280 and + BMP380 pressure and temperature sensors. Also supports the BME280 with an additional humidity sensor channel. To compile this driver as a module, choose M here: the core module diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c index fe7aa81e7cc9e535d13e553db4df33467077efcd..c0aff78489b46db396f294e0e1ae9062aec176bb 100644 --- a/drivers/iio/pressure/bmp280-core.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -9,13 +9,22 @@ * Driver for Bosch Sensortec BMP180 and BMP280 digital pressure sensor. * * Datasheet: - * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BMP180-DS000-121.pdf - * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BMP280-DS001-12.pdf - * https://ae-bst.resource.bosch.com/media/_tech/media/datasheets/BST-BME280_DS001-11.pdf + * https://cdn-shop.adafruit.com/datasheets/BST-BMP180-DS000-09.pdf + * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp280-ds001.pdf + * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bme280-ds002.pdf + * https://www.bosch-sensortec.com/media/boschsensortec/downloads/datasheets/bst-bmp388-ds001.pdf + * + * Notice: + * The link to the bmp180 datasheet points to an outdated version missing these changes: + * - Changed document referral from ANP015 to BST-MPS-AN004-00 on page 26 + * - Updated equation for B3 param on section 3.5 to ((((long)AC1 * 4 + X3) << oss) + 2) / 4 + * - Updated RoHS directive to 2011/65/EU effective 8 June 2011 on page 26 */ #define pr_fmt(fmt) "bmp280: " fmt +#include +#include #include #include #include @@ -30,6 +39,8 @@ #include #include +#include + #include "bmp280.h" /* @@ -74,12 +85,51 @@ struct bmp280_calib { s8 H6; }; +/* See datasheet Section 3.11.1. */ +struct bmp380_calib { + u16 T1; + u16 T2; + s8 T3; + s16 P1; + s16 P2; + s8 P3; + s8 P4; + u16 P5; + u16 P6; + s8 P7; + s8 P8; + s16 P9; + s8 P10; + s8 P11; +}; + static const char *const bmp280_supply_names[] = { "vddd", "vdda" }; #define BMP280_NUM_SUPPLIES ARRAY_SIZE(bmp280_supply_names) +enum bmp380_odr { + BMP380_ODR_200HZ, + BMP380_ODR_100HZ, + BMP380_ODR_50HZ, + BMP380_ODR_25HZ, + BMP380_ODR_12_5HZ, + BMP380_ODR_6_25HZ, + BMP380_ODR_3_125HZ, + BMP380_ODR_1_5625HZ, + BMP380_ODR_0_78HZ, + BMP380_ODR_0_39HZ, + BMP380_ODR_0_2HZ, + BMP380_ODR_0_1HZ, + BMP380_ODR_0_05HZ, + BMP380_ODR_0_02HZ, + BMP380_ODR_0_01HZ, + BMP380_ODR_0_006HZ, + BMP380_ODR_0_003HZ, + BMP380_ODR_0_0015HZ, +}; + struct bmp280_data { struct device *dev; struct mutex lock; @@ -90,6 +140,7 @@ struct bmp280_data { union { struct bmp180_calib bmp180; struct bmp280_calib bmp280; + struct bmp380_calib bmp380; } calib; struct regulator_bulk_data supplies[BMP280_NUM_SUPPLIES]; unsigned int start_up_time; /* in microseconds */ @@ -98,36 +149,99 @@ struct bmp280_data { u8 oversampling_press; u8 oversampling_temp; u8 oversampling_humid; + u8 iir_filter_coeff; + + /* + * BMP380 devices introduce sampling frequency configuration. See + * datasheet sections 3.3.3. and 4.3.19 for more details. + * + * BMx280 devices allowed indirect configuration of sampling frequency + * changing the t_standby duration between measurements, as detailed on + * section 3.6.3 of the datasheet. + */ + int sampling_freq; /* * Carryover value from temperature conversion, used in pressure * calculation. */ s32 t_fine; + + /* + * DMA (thus cache coherency maintenance) may require the + * transfer buffers to live in their own cache lines. + */ + union { + /* Sensor data buffer */ + u8 buf[3]; + /* Calibration data buffers */ + __le16 bmp280_cal_buf[BMP280_CONTIGUOUS_CALIB_REGS / 2]; + __be16 bmp180_cal_buf[BMP180_REG_CALIB_COUNT / 2]; + u8 bmp380_cal_buf[BMP380_CALIB_REG_COUNT]; + /* Miscellaneous, endianess-aware data buffers */ + __le16 le16; + __be16 be16; + } __aligned(IIO_DMA_MINALIGN); }; struct bmp280_chip_info { + unsigned int id_reg; + + const struct iio_chan_spec *channels; + int num_channels; + unsigned int start_up_time; + const int *oversampling_temp_avail; int num_oversampling_temp_avail; + int oversampling_temp_default; const int *oversampling_press_avail; int num_oversampling_press_avail; + int oversampling_press_default; const int *oversampling_humid_avail; int num_oversampling_humid_avail; + int oversampling_humid_default; + + const int *iir_filter_coeffs_avail; + int num_iir_filter_coeffs_avail; + int iir_filter_coeff_default; + + const int (*sampling_freq_avail)[2]; + int num_sampling_freq_avail; + int sampling_freq_default; int (*chip_config)(struct bmp280_data *); int (*read_temp)(struct bmp280_data *, int *); int (*read_press)(struct bmp280_data *, int *, int *); int (*read_humid)(struct bmp280_data *, int *, int *); + int (*read_calib)(struct bmp280_data *); }; /* * These enums are used for indexing into the array of compensation * parameters for BMP280. */ -enum { T1, T2, T3 }; -enum { P1, P2, P3, P4, P5, P6, P7, P8, P9 }; +enum { T1, T2, T3, P1, P2, P3, P4, P5, P6, P7, P8, P9 }; + +enum { + /* Temperature calib indexes */ + BMP380_T1 = 0, + BMP380_T2 = 2, + BMP380_T3 = 4, + /* Pressure calib indexes */ + BMP380_P1 = 5, + BMP380_P2 = 7, + BMP380_P3 = 9, + BMP380_P4 = 10, + BMP380_P5 = 11, + BMP380_P6 = 13, + BMP380_P7 = 15, + BMP380_P8 = 16, + BMP380_P9 = 17, + BMP380_P10 = 19, + BMP380_P11 = 20, +}; static const struct iio_chan_spec bmp280_channels[] = { { @@ -147,56 +261,81 @@ static const struct iio_chan_spec bmp280_channels[] = { }, }; -static int bmp280_read_calib(struct bmp280_data *data, - struct bmp280_calib *calib, - unsigned int chip) +static const struct iio_chan_spec bmp380_channels[] = { + { + .type = IIO_PRESSURE, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + }, + { + .type = IIO_TEMP, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + }, + { + .type = IIO_HUMIDITYRELATIVE, + .info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED) | + BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO), + .info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) | + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + }, +}; + +static int bmp280_read_calib(struct bmp280_data *data) { + struct bmp280_calib *calib = &data->calib.bmp280; int ret; - unsigned int tmp; - __le16 l16; - __be16 b16; - struct device *dev = data->dev; - __le16 t_buf[BMP280_COMP_TEMP_REG_COUNT / 2]; - __le16 p_buf[BMP280_COMP_PRESS_REG_COUNT / 2]; - /* Read temperature calibration values. */ + + /* Read temperature and pressure calibration values. */ ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_TEMP_START, - t_buf, BMP280_COMP_TEMP_REG_COUNT); + data->bmp280_cal_buf, sizeof(data->bmp280_cal_buf)); if (ret < 0) { dev_err(data->dev, - "failed to read temperature calibration parameters\n"); + "failed to read temperature and pressure calibration parameters\n"); return ret; } - /* Toss the temperature calibration data into the entropy pool */ - add_device_randomness(t_buf, sizeof(t_buf)); + /* Toss the temperature and pressure calibration data into the entropy pool */ + add_device_randomness(data->bmp280_cal_buf, sizeof(data->bmp280_cal_buf)); + + /* Parse temperature calibration values. */ + calib->T1 = le16_to_cpu(data->bmp280_cal_buf[T1]); + calib->T2 = le16_to_cpu(data->bmp280_cal_buf[T2]); + calib->T3 = le16_to_cpu(data->bmp280_cal_buf[T3]); + + /* Parse pressure calibration values. */ + calib->P1 = le16_to_cpu(data->bmp280_cal_buf[P1]); + calib->P2 = le16_to_cpu(data->bmp280_cal_buf[P2]); + calib->P3 = le16_to_cpu(data->bmp280_cal_buf[P3]); + calib->P4 = le16_to_cpu(data->bmp280_cal_buf[P4]); + calib->P5 = le16_to_cpu(data->bmp280_cal_buf[P5]); + calib->P6 = le16_to_cpu(data->bmp280_cal_buf[P6]); + calib->P7 = le16_to_cpu(data->bmp280_cal_buf[P7]); + calib->P8 = le16_to_cpu(data->bmp280_cal_buf[P8]); + calib->P9 = le16_to_cpu(data->bmp280_cal_buf[P9]); - calib->T1 = le16_to_cpu(t_buf[T1]); - calib->T2 = le16_to_cpu(t_buf[T2]); - calib->T3 = le16_to_cpu(t_buf[T3]); + return 0; +} - /* Read pressure calibration values. */ - ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_PRESS_START, - p_buf, BMP280_COMP_PRESS_REG_COUNT); - if (ret < 0) { - dev_err(data->dev, - "failed to read pressure calibration parameters\n"); +static int bme280_read_calib(struct bmp280_data *data) +{ + struct bmp280_calib *calib = &data->calib.bmp280; + struct device *dev = data->dev; + unsigned int tmp; + int ret; + + /* Load shared calibration params with bmp280 first */ + ret = bmp280_read_calib(data); + if (ret < 0) { + dev_err(dev, "failed to read common bmp280 calibration parameters\n"); return ret; } - /* Toss the pressure calibration data into the entropy pool */ - add_device_randomness(p_buf, sizeof(p_buf)); - - calib->P1 = le16_to_cpu(p_buf[P1]); - calib->P2 = le16_to_cpu(p_buf[P2]); - calib->P3 = le16_to_cpu(p_buf[P3]); - calib->P4 = le16_to_cpu(p_buf[P4]); - calib->P5 = le16_to_cpu(p_buf[P5]); - calib->P6 = le16_to_cpu(p_buf[P6]); - calib->P7 = le16_to_cpu(p_buf[P7]); - calib->P8 = le16_to_cpu(p_buf[P8]); - calib->P9 = le16_to_cpu(p_buf[P9]); - /* * Read humidity calibration values. * Due to some odd register addressing we cannot just @@ -204,8 +343,6 @@ static int bmp280_read_calib(struct bmp280_data *data, * value separately and sometimes do some bit shifting... * Humidity data is only available on BME280. */ - if (chip != BME280_CHIP_ID) - return 0; ret = regmap_read(data->regmap, BMP280_REG_COMP_H1, &tmp); if (ret < 0) { @@ -214,12 +351,13 @@ static int bmp280_read_calib(struct bmp280_data *data, } calib->H1 = tmp; - ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H2, &l16, 2); + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H2, + &data->le16, sizeof(data->le16)); if (ret < 0) { dev_err(dev, "failed to read H2 comp value\n"); return ret; } - calib->H2 = sign_extend32(le16_to_cpu(l16), 15); + calib->H2 = sign_extend32(le16_to_cpu(data->le16), 15); ret = regmap_read(data->regmap, BMP280_REG_COMP_H3, &tmp); if (ret < 0) { @@ -228,20 +366,22 @@ static int bmp280_read_calib(struct bmp280_data *data, } calib->H3 = tmp; - ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H4, &b16, 2); + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H4, + &data->be16, sizeof(data->be16)); if (ret < 0) { dev_err(dev, "failed to read H4 comp value\n"); return ret; } - calib->H4 = sign_extend32(((be16_to_cpu(b16) >> 4) & 0xff0) | - (be16_to_cpu(b16) & 0xf), 11); + calib->H4 = sign_extend32(((be16_to_cpu(data->be16) >> 4) & 0xff0) | + (be16_to_cpu(data->be16) & 0xf), 11); - ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H5, &l16, 2); + ret = regmap_bulk_read(data->regmap, BMP280_REG_COMP_H5, + &data->le16, sizeof(data->le16)); if (ret < 0) { dev_err(dev, "failed to read H5 comp value\n"); return ret; } - calib->H5 = sign_extend32(((le16_to_cpu(l16) >> 4) & 0xfff), 11); + calib->H5 = sign_extend32(FIELD_GET(BMP280_COMP_H5_MASK, le16_to_cpu(data->le16)), 11); ret = regmap_read(data->regmap, BMP280_REG_COMP_H6, &tmp); if (ret < 0) { @@ -261,8 +401,8 @@ static int bmp280_read_calib(struct bmp280_data *data, static u32 bmp280_compensate_humidity(struct bmp280_data *data, s32 adc_humidity) { - s32 var; struct bmp280_calib *calib = &data->calib.bmp280; + s32 var; var = ((s32)data->t_fine) - (s32)76800; var = ((((adc_humidity << 14) - (calib->H4 << 20) - (calib->H5 * var)) @@ -286,8 +426,8 @@ static u32 bmp280_compensate_humidity(struct bmp280_data *data, static s32 bmp280_compensate_temp(struct bmp280_data *data, s32 adc_temp) { - s32 var1, var2; struct bmp280_calib *calib = &data->calib.bmp280; + s32 var1, var2; var1 = (((adc_temp >> 3) - ((s32)calib->T1 << 1)) * ((s32)calib->T2)) >> 11; @@ -309,8 +449,8 @@ static s32 bmp280_compensate_temp(struct bmp280_data *data, static u32 bmp280_compensate_press(struct bmp280_data *data, s32 adc_press) { - s64 var1, var2, p; struct bmp280_calib *calib = &data->calib.bmp280; + s64 var1, var2, p; var1 = ((s64)data->t_fine) - 128000; var2 = var1 * var1 * (s64)calib->P6; @@ -335,17 +475,17 @@ static u32 bmp280_compensate_press(struct bmp280_data *data, static int bmp280_read_temp(struct bmp280_data *data, int *val) { - int ret; - __be32 tmp = 0; s32 adc_temp, comp_temp; + int ret; - ret = regmap_bulk_read(data->regmap, BMP280_REG_TEMP_MSB, &tmp, 3); + ret = regmap_bulk_read(data->regmap, BMP280_REG_TEMP_MSB, + data->buf, sizeof(data->buf)); if (ret < 0) { dev_err(data->dev, "failed to read temperature\n"); return ret; } - adc_temp = be32_to_cpu(tmp) >> 12; + adc_temp = FIELD_GET(BMP280_MEAS_TRIM_MASK, get_unaligned_be24(data->buf)); if (adc_temp == BMP280_TEMP_SKIPPED) { /* reading was skipped */ dev_err(data->dev, "reading temperature skipped\n"); @@ -368,23 +508,23 @@ static int bmp280_read_temp(struct bmp280_data *data, static int bmp280_read_press(struct bmp280_data *data, int *val, int *val2) { - int ret; - __be32 tmp = 0; - s32 adc_press; u32 comp_press; + s32 adc_press; + int ret; /* Read and compensate temperature so we get a reading of t_fine. */ ret = bmp280_read_temp(data, NULL); if (ret < 0) return ret; - ret = regmap_bulk_read(data->regmap, BMP280_REG_PRESS_MSB, &tmp, 3); + ret = regmap_bulk_read(data->regmap, BMP280_REG_PRESS_MSB, + data->buf, sizeof(data->buf)); if (ret < 0) { dev_err(data->dev, "failed to read pressure\n"); return ret; } - adc_press = be32_to_cpu(tmp) >> 12; + adc_press = FIELD_GET(BMP280_MEAS_TRIM_MASK, get_unaligned_be24(data->buf)); if (adc_press == BMP280_PRESS_SKIPPED) { /* reading was skipped */ dev_err(data->dev, "reading pressure skipped\n"); @@ -400,23 +540,23 @@ static int bmp280_read_press(struct bmp280_data *data, static int bmp280_read_humid(struct bmp280_data *data, int *val, int *val2) { - __be16 tmp; - int ret; - s32 adc_humidity; u32 comp_humidity; + s32 adc_humidity; + int ret; /* Read and compensate temperature so we get a reading of t_fine. */ ret = bmp280_read_temp(data, NULL); if (ret < 0) return ret; - ret = regmap_bulk_read(data->regmap, BMP280_REG_HUMIDITY_MSB, &tmp, 2); + ret = regmap_bulk_read(data->regmap, BMP280_REG_HUMIDITY_MSB, + &data->be16, sizeof(data->be16)); if (ret < 0) { dev_err(data->dev, "failed to read humidity\n"); return ret; } - adc_humidity = be16_to_cpu(tmp); + adc_humidity = be16_to_cpu(data->be16); if (adc_humidity == BMP280_HUMIDITY_SKIPPED) { /* reading was skipped */ dev_err(data->dev, "reading humidity skipped\n"); @@ -433,8 +573,8 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) { - int ret; struct bmp280_data *data = iio_priv(indio_dev); + int ret; pm_runtime_get_sync(data->dev); mutex_lock(&data->lock); @@ -475,6 +615,25 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, break; } break; + case IIO_CHAN_INFO_SAMP_FREQ: + if (!data->chip_info->sampling_freq_avail) { + ret = -EINVAL; + break; + } + + *val = data->chip_info->sampling_freq_avail[data->sampling_freq][0]; + *val2 = data->chip_info->sampling_freq_avail[data->sampling_freq][1]; + ret = IIO_VAL_INT_PLUS_MICRO; + break; + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + if (!data->chip_info->iir_filter_coeffs_avail) { + ret = -EINVAL; + break; + } + + *val = (1 << data->iir_filter_coeff) - 1; + ret = IIO_VAL_INT; + break; default: ret = -EINVAL; break; @@ -490,15 +649,23 @@ static int bmp280_read_raw(struct iio_dev *indio_dev, static int bmp280_write_oversampling_ratio_humid(struct bmp280_data *data, int val) { - int i; const int *avail = data->chip_info->oversampling_humid_avail; const int n = data->chip_info->num_oversampling_humid_avail; + int ret, prev; + int i; for (i = 0; i < n; i++) { if (avail[i] == val) { + prev = data->oversampling_humid; data->oversampling_humid = ilog2(val); - return data->chip_info->chip_config(data); + ret = data->chip_info->chip_config(data); + if (ret) { + data->oversampling_humid = prev; + data->chip_info->chip_config(data); + return ret; + } + return 0; } } return -EINVAL; @@ -507,15 +674,23 @@ static int bmp280_write_oversampling_ratio_humid(struct bmp280_data *data, static int bmp280_write_oversampling_ratio_temp(struct bmp280_data *data, int val) { - int i; const int *avail = data->chip_info->oversampling_temp_avail; const int n = data->chip_info->num_oversampling_temp_avail; + int ret, prev; + int i; for (i = 0; i < n; i++) { if (avail[i] == val) { + prev = data->oversampling_temp; data->oversampling_temp = ilog2(val); - return data->chip_info->chip_config(data); + ret = data->chip_info->chip_config(data); + if (ret) { + data->oversampling_temp = prev; + data->chip_info->chip_config(data); + return ret; + } + return 0; } } return -EINVAL; @@ -524,15 +699,73 @@ static int bmp280_write_oversampling_ratio_temp(struct bmp280_data *data, static int bmp280_write_oversampling_ratio_press(struct bmp280_data *data, int val) { - int i; const int *avail = data->chip_info->oversampling_press_avail; const int n = data->chip_info->num_oversampling_press_avail; + int ret, prev; + int i; for (i = 0; i < n; i++) { if (avail[i] == val) { + prev = data->oversampling_press; data->oversampling_press = ilog2(val); - return data->chip_info->chip_config(data); + ret = data->chip_info->chip_config(data); + if (ret) { + data->oversampling_press = prev; + data->chip_info->chip_config(data); + return ret; + } + return 0; + } + } + return -EINVAL; +} + +static int bmp280_write_sampling_frequency(struct bmp280_data *data, + int val, int val2) +{ + const int (*avail)[2] = data->chip_info->sampling_freq_avail; + const int n = data->chip_info->num_sampling_freq_avail; + int ret, prev; + int i; + + for (i = 0; i < n; i++) { + if (avail[i][0] == val && avail[i][1] == val2) { + prev = data->sampling_freq; + data->sampling_freq = i; + + ret = data->chip_info->chip_config(data); + if (ret) { + data->sampling_freq = prev; + data->chip_info->chip_config(data); + return ret; + } + return 0; + } + } + return -EINVAL; +} + +static int bmp280_write_iir_filter_coeffs(struct bmp280_data *data, int val) +{ + const int *avail = data->chip_info->iir_filter_coeffs_avail; + const int n = data->chip_info->num_iir_filter_coeffs_avail; + int ret, prev; + int i; + + for (i = 0; i < n; i++) { + if (avail[i] - 1 == val) { + prev = data->iir_filter_coeff; + data->iir_filter_coeff = i; + + ret = data->chip_info->chip_config(data); + if (ret) { + data->iir_filter_coeff = prev; + data->chip_info->chip_config(data); + return ret; + + } + return 0; } } return -EINVAL; @@ -542,9 +775,15 @@ static int bmp280_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int val, int val2, long mask) { - int ret = 0; struct bmp280_data *data = iio_priv(indio_dev); + int ret = 0; + /* + * Helper functions to update sensor running configuration. + * If an error happens applying new settings, will try restore + * previous parameters to ensure the sensor is left in a known + * working configuration. + */ switch (mask) { case IIO_CHAN_INFO_OVERSAMPLING_RATIO: pm_runtime_get_sync(data->dev); @@ -567,6 +806,22 @@ static int bmp280_write_raw(struct iio_dev *indio_dev, pm_runtime_mark_last_busy(data->dev); pm_runtime_put_autosuspend(data->dev); break; + case IIO_CHAN_INFO_SAMP_FREQ: + pm_runtime_get_sync(data->dev); + mutex_lock(&data->lock); + ret = bmp280_write_sampling_frequency(data, val, val2); + mutex_unlock(&data->lock); + pm_runtime_mark_last_busy(data->dev); + pm_runtime_put_autosuspend(data->dev); + break; + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + pm_runtime_get_sync(data->dev); + mutex_lock(&data->lock); + ret = bmp280_write_iir_filter_coeffs(data, val); + mutex_unlock(&data->lock); + pm_runtime_mark_last_busy(data->dev); + pm_runtime_put_autosuspend(data->dev); + break; default: return -EINVAL; } @@ -597,6 +852,17 @@ static int bmp280_read_avail(struct iio_dev *indio_dev, } *type = IIO_VAL_INT; return IIO_AVAIL_LIST; + case IIO_CHAN_INFO_SAMP_FREQ: + *vals = (const int *)data->chip_info->sampling_freq_avail; + *type = IIO_VAL_INT_PLUS_MICRO; + /* Values are stored in a 2D matrix */ + *length = data->chip_info->num_sampling_freq_avail; + return IIO_AVAIL_LIST; + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + *vals = data->chip_info->iir_filter_coeffs_avail; + *type = IIO_VAL_INT; + *length = data->chip_info->num_iir_filter_coeffs_avail; + return IIO_AVAIL_LIST; default: return -EINVAL; } @@ -610,9 +876,9 @@ static const struct iio_info bmp280_info = { static int bmp280_chip_config(struct bmp280_data *data) { + u8 osrs = FIELD_PREP(BMP280_OSRS_TEMP_MASK, data->oversampling_temp + 1) | + FIELD_PREP(BMP280_OSRS_PRESS_MASK, data->oversampling_press + 1); int ret; - u8 osrs = BMP280_OSRS_TEMP_X(data->oversampling_temp + 1) | - BMP280_OSRS_PRESS_X(data->oversampling_press + 1); ret = regmap_write_bits(data->regmap, BMP280_REG_CTRL_MEAS, BMP280_OSRS_TEMP_MASK | @@ -640,21 +906,39 @@ static int bmp280_chip_config(struct bmp280_data *data) static const int bmp280_oversampling_avail[] = { 1, 2, 4, 8, 16 }; static const struct bmp280_chip_info bmp280_chip_info = { + .id_reg = BMP280_REG_ID, + .start_up_time = 2000, + .channels = bmp280_channels, + .num_channels = 2, + .oversampling_temp_avail = bmp280_oversampling_avail, .num_oversampling_temp_avail = ARRAY_SIZE(bmp280_oversampling_avail), + /* + * Oversampling config values on BMx280 have one additional setting + * that other generations of the family don't: + * The value 0 means the measurement is bypassed instead of + * oversampling set to x1. + * + * To account for this difference, and preserve the same common + * config logic, this is handled later on chip_config callback + * incrementing one unit the oversampling setting. + */ + .oversampling_temp_default = BMP280_OSRS_TEMP_2X - 1, .oversampling_press_avail = bmp280_oversampling_avail, .num_oversampling_press_avail = ARRAY_SIZE(bmp280_oversampling_avail), + .oversampling_press_default = BMP280_OSRS_PRESS_16X - 1, .chip_config = bmp280_chip_config, .read_temp = bmp280_read_temp, .read_press = bmp280_read_press, + .read_calib = bmp280_read_calib, }; static int bme280_chip_config(struct bmp280_data *data) { + u8 osrs = FIELD_PREP(BMP280_OSRS_HUMIDITY_MASK, data->oversampling_humid + 1); int ret; - u8 osrs = BMP280_OSRS_HUMIDITIY_X(data->oversampling_humid + 1); /* * Oversampling of humidity must be set before oversampling of @@ -670,27 +954,405 @@ static int bme280_chip_config(struct bmp280_data *data) } static const struct bmp280_chip_info bme280_chip_info = { + .id_reg = BMP280_REG_ID, + .start_up_time = 2000, + .channels = bmp280_channels, + .num_channels = 3, + .oversampling_temp_avail = bmp280_oversampling_avail, .num_oversampling_temp_avail = ARRAY_SIZE(bmp280_oversampling_avail), + .oversampling_temp_default = BMP280_OSRS_TEMP_2X - 1, .oversampling_press_avail = bmp280_oversampling_avail, .num_oversampling_press_avail = ARRAY_SIZE(bmp280_oversampling_avail), + .oversampling_press_default = BMP280_OSRS_PRESS_16X - 1, .oversampling_humid_avail = bmp280_oversampling_avail, .num_oversampling_humid_avail = ARRAY_SIZE(bmp280_oversampling_avail), + .oversampling_humid_default = BMP280_OSRS_HUMIDITY_16X - 1, .chip_config = bme280_chip_config, .read_temp = bmp280_read_temp, .read_press = bmp280_read_press, .read_humid = bmp280_read_humid, + .read_calib = bme280_read_calib, }; -static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) +/* + * Helper function to send a command to BMP3XX sensors. + * + * Sensor processes commands written to the CMD register and signals + * execution result through "cmd_rdy" and "cmd_error" flags available on + * STATUS and ERROR registers. + */ +static int bmp380_cmd(struct bmp280_data *data, u8 cmd) +{ + unsigned int reg; + int ret; + + /* Check if device is ready to process a command */ + ret = regmap_read(data->regmap, BMP380_REG_STATUS, ®); + if (ret) { + dev_err(data->dev, "failed to read error register\n"); + return ret; + } + if (!(reg & BMP380_STATUS_CMD_RDY_MASK)) { + dev_err(data->dev, "device is not ready to accept commands\n"); + return -EBUSY; + } + + /* Send command to process */ + ret = regmap_write(data->regmap, BMP380_REG_CMD, cmd); + if (ret) { + dev_err(data->dev, "failed to send command to device\n"); + return ret; + } + /* Wait for 2ms for command to be processed */ + usleep_range(data->start_up_time, data->start_up_time + 100); + /* Check for command processing error */ + ret = regmap_read(data->regmap, BMP380_REG_ERROR, ®); + if (ret) { + dev_err(data->dev, "error reading ERROR reg\n"); + return ret; + } + if (reg & BMP380_ERR_CMD_MASK) { + dev_err(data->dev, "error processing command 0x%X\n", cmd); + return -EINVAL; + } + + return 0; +} + +/* + * Returns temperature in Celsius dregrees, resolution is 0.01º C. Output value of + * "5123" equals 51.2º C. t_fine carries fine temperature as global value. + * + * Taken from datasheet, Section Appendix 9, "Compensation formula" and repo + * https://github.com/BoschSensortec/BMP3-Sensor-API. + */ +static s32 bmp380_compensate_temp(struct bmp280_data *data, u32 adc_temp) +{ + s64 var1, var2, var3, var4, var5, var6, comp_temp; + struct bmp380_calib *calib = &data->calib.bmp380; + + var1 = ((s64) adc_temp) - (((s64) calib->T1) << 8); + var2 = var1 * ((s64) calib->T2); + var3 = var1 * var1; + var4 = var3 * ((s64) calib->T3); + var5 = (var2 << 18) + var4; + var6 = var5 >> 32; + data->t_fine = (s32) var6; + comp_temp = (var6 * 25) >> 14; + + comp_temp = clamp_val(comp_temp, BMP380_MIN_TEMP, BMP380_MAX_TEMP); + return (s32) comp_temp; +} + +/* + * Returns pressure in Pa as an unsigned 32 bit integer in fractional Pascal. + * Output value of "9528709" represents 9528709/100 = 95287.09 Pa = 952.8709 hPa. + * + * Taken from datasheet, Section 9.3. "Pressure compensation" and repository + * https://github.com/BoschSensortec/BMP3-Sensor-API. + */ +static u32 bmp380_compensate_press(struct bmp280_data *data, u32 adc_press) +{ + s64 var1, var2, var3, var4, var5, var6, offset, sensitivity; + struct bmp380_calib *calib = &data->calib.bmp380; + u32 comp_press; + + var1 = (s64)data->t_fine * (s64)data->t_fine; + var2 = var1 >> 6; + var3 = (var2 * ((s64) data->t_fine)) >> 8; + var4 = ((s64)calib->P8 * var3) >> 5; + var5 = ((s64)calib->P7 * var1) << 4; + var6 = ((s64)calib->P6 * (s64)data->t_fine) << 22; + offset = ((s64)calib->P5 << 47) + var4 + var5 + var6; + var2 = ((s64)calib->P4 * var3) >> 5; + var4 = ((s64)calib->P3 * var1) << 2; + var5 = ((s64)calib->P2 - ((s64)1 << 14)) * + ((s64)data->t_fine << 21); + sensitivity = (((s64) calib->P1 - ((s64) 1 << 14)) << 46) + + var2 + var4 + var5; + var1 = (sensitivity >> 24) * (s64)adc_press; + var2 = (s64)calib->P10 * (s64)data->t_fine; + var3 = var2 + ((s64)calib->P9 << 16); + var4 = (var3 * (s64)adc_press) >> 13; + + /* + * Dividing by 10 followed by multiplying by 10 to avoid + * possible overflow caused by (uncomp_data->pressure * partial_data4). + */ + var5 = ((s64)adc_press * div_s64(var4, 10)) >> 9; + var5 *= 10; + var6 = (s64)adc_press * (s64)adc_press; + var2 = ((s64)calib->P11 * var6) >> 16; + var3 = (var2 * (s64)adc_press) >> 7; + var4 = (offset >> 2) + var1 + var5 + var3; + comp_press = ((u64)var4 * 25) >> 40; + + comp_press = clamp_val(comp_press, BMP380_MIN_PRES, BMP380_MAX_PRES); + return comp_press; +} + +static int bmp380_read_temp(struct bmp280_data *data, int *val) +{ + s32 comp_temp; + u32 adc_temp; + int ret; + + ret = regmap_bulk_read(data->regmap, BMP380_REG_TEMP_XLSB, + data->buf, sizeof(data->buf)); + if (ret) { + dev_err(data->dev, "failed to read temperature\n"); + return ret; + } + + adc_temp = get_unaligned_le24(data->buf); + if (adc_temp == BMP380_TEMP_SKIPPED) { + dev_err(data->dev, "reading temperature skipped\n"); + return -EIO; + } + comp_temp = bmp380_compensate_temp(data, adc_temp); + + /* + * Val might be NULL if we're called by the read_press routine, + * who only cares about the carry over t_fine value. + */ + if (val) { + /* IIO reports temperatures in milli Celsius */ + *val = comp_temp * 10; + return IIO_VAL_INT; + } + + return 0; +} + +static int bmp380_read_press(struct bmp280_data *data, int *val, int *val2) +{ + s32 comp_press; + u32 adc_press; + int ret; + + /* Read and compensate for temperature so we get a reading of t_fine */ + ret = bmp380_read_temp(data, NULL); + if (ret) + return ret; + + ret = regmap_bulk_read(data->regmap, BMP380_REG_PRESS_XLSB, + data->buf, sizeof(data->buf)); + if (ret) { + dev_err(data->dev, "failed to read pressure\n"); + return ret; + } + + adc_press = get_unaligned_le24(data->buf); + if (adc_press == BMP380_PRESS_SKIPPED) { + dev_err(data->dev, "reading pressure skipped\n"); + return -EIO; + } + comp_press = bmp380_compensate_press(data, adc_press); + + *val = comp_press; + /* Compensated pressure is in cPa (centipascals) */ + *val2 = 100000; + + return IIO_VAL_FRACTIONAL; +} + +static int bmp380_read_calib(struct bmp280_data *data) +{ + struct bmp380_calib *calib = &data->calib.bmp380; + int ret; + + /* Read temperature and pressure calibration data */ + ret = regmap_bulk_read(data->regmap, BMP380_REG_CALIB_TEMP_START, + data->bmp380_cal_buf, sizeof(data->bmp380_cal_buf)); + if (ret) { + dev_err(data->dev, + "failed to read temperature calibration parameters\n"); + return ret; + } + + /* Toss the temperature calibration data into the entropy pool */ + add_device_randomness(data->bmp380_cal_buf, sizeof(data->bmp380_cal_buf)); + + /* Parse calibration values */ + calib->T1 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_T1]); + calib->T2 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_T2]); + calib->T3 = data->bmp380_cal_buf[BMP380_T3]; + calib->P1 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_P1]); + calib->P2 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_P2]); + calib->P3 = data->bmp380_cal_buf[BMP380_P3]; + calib->P4 = data->bmp380_cal_buf[BMP380_P4]; + calib->P5 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_P5]); + calib->P6 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_P6]); + calib->P7 = data->bmp380_cal_buf[BMP380_P7]; + calib->P8 = data->bmp380_cal_buf[BMP380_P8]; + calib->P9 = get_unaligned_le16(&data->bmp380_cal_buf[BMP380_P9]); + calib->P10 = data->bmp380_cal_buf[BMP380_P10]; + calib->P11 = data->bmp380_cal_buf[BMP380_P11]; + + return 0; +} + +static const int bmp380_odr_table[][2] = { + [BMP380_ODR_200HZ] = {200, 0}, + [BMP380_ODR_100HZ] = {100, 0}, + [BMP380_ODR_50HZ] = {50, 0}, + [BMP380_ODR_25HZ] = {25, 0}, + [BMP380_ODR_12_5HZ] = {12, 500000}, + [BMP380_ODR_6_25HZ] = {6, 250000}, + [BMP380_ODR_3_125HZ] = {3, 125000}, + [BMP380_ODR_1_5625HZ] = {1, 562500}, + [BMP380_ODR_0_78HZ] = {0, 781250}, + [BMP380_ODR_0_39HZ] = {0, 390625}, + [BMP380_ODR_0_2HZ] = {0, 195313}, + [BMP380_ODR_0_1HZ] = {0, 97656}, + [BMP380_ODR_0_05HZ] = {0, 48828}, + [BMP380_ODR_0_02HZ] = {0, 24414}, + [BMP380_ODR_0_01HZ] = {0, 12207}, + [BMP380_ODR_0_006HZ] = {0, 6104}, + [BMP380_ODR_0_003HZ] = {0, 3052}, + [BMP380_ODR_0_0015HZ] = {0, 1526}, +}; + +static int bmp380_chip_config(struct bmp280_data *data) { + bool change = false, aux; + unsigned int tmp; + u8 osrs; int ret; + + /* Configure power control register */ + ret = regmap_update_bits(data->regmap, BMP380_REG_POWER_CONTROL, + BMP380_CTRL_SENSORS_MASK, + BMP380_CTRL_SENSORS_PRESS_EN | + BMP380_CTRL_SENSORS_TEMP_EN); + if (ret) { + dev_err(data->dev, + "failed to write operation control register\n"); + return ret; + } + + /* Configure oversampling */ + osrs = FIELD_PREP(BMP380_OSRS_TEMP_MASK, data->oversampling_temp) | + FIELD_PREP(BMP380_OSRS_PRESS_MASK, data->oversampling_press); + + ret = regmap_update_bits_check(data->regmap, BMP380_REG_OSR, + BMP380_OSRS_TEMP_MASK | + BMP380_OSRS_PRESS_MASK, + osrs, &aux); + if (ret) { + dev_err(data->dev, "failed to write oversampling register\n"); + return ret; + } + change = change || aux; + + /* Configure output data rate */ + ret = regmap_update_bits_check(data->regmap, BMP380_REG_ODR, + BMP380_ODRS_MASK, data->sampling_freq, &aux); + if (ret) { + dev_err(data->dev, "failed to write ODR selection register\n"); + return ret; + } + change = change || aux; + + /* Set filter data */ + ret = regmap_update_bits_check(data->regmap, BMP380_REG_CONFIG, BMP380_FILTER_MASK, + FIELD_PREP(BMP380_FILTER_MASK, data->iir_filter_coeff), + &aux); + if (ret) { + dev_err(data->dev, "failed to write config register\n"); + return ret; + } + change = change || aux; + + if (change) { + /* + * The configurations errors are detected on the fly during a measurement + * cycle. If the sampling frequency is too low, it's faster to reset + * the measurement loop than wait until the next measurement is due. + * + * Resets sensor measurement loop toggling between sleep and normal + * operating modes. + */ + ret = regmap_write_bits(data->regmap, BMP380_REG_POWER_CONTROL, + BMP380_MODE_MASK, + FIELD_PREP(BMP380_MODE_MASK, BMP380_MODE_SLEEP)); + if (ret) { + dev_err(data->dev, "failed to set sleep mode\n"); + return ret; + } + usleep_range(2000, 2500); + ret = regmap_write_bits(data->regmap, BMP380_REG_POWER_CONTROL, + BMP380_MODE_MASK, + FIELD_PREP(BMP380_MODE_MASK, BMP380_MODE_NORMAL)); + if (ret) { + dev_err(data->dev, "failed to set normal mode\n"); + return ret; + } + /* + * Waits for measurement before checking configuration error flag. + * Selected longest measure time indicated in section 3.9.1 + * in the datasheet. + */ + msleep(80); + + /* Check config error flag */ + ret = regmap_read(data->regmap, BMP380_REG_ERROR, &tmp); + if (ret) { + dev_err(data->dev, + "failed to read error register\n"); + return ret; + } + if (tmp & BMP380_ERR_CONF_MASK) { + dev_warn(data->dev, + "sensor flagged configuration as incompatible\n"); + return -EINVAL; + } + } + + return 0; +} + +static const int bmp380_oversampling_avail[] = { 1, 2, 4, 8, 16, 32 }; +static const int bmp380_iir_filter_coeffs_avail[] = { 1, 2, 4, 8, 16, 32, 64, 128}; + +static const struct bmp280_chip_info bmp380_chip_info = { + .id_reg = BMP380_REG_ID, + .start_up_time = 2000, + .channels = bmp380_channels, + .num_channels = 2, + + .oversampling_temp_avail = bmp380_oversampling_avail, + .num_oversampling_temp_avail = ARRAY_SIZE(bmp380_oversampling_avail), + .oversampling_temp_default = ilog2(1), + + .oversampling_press_avail = bmp380_oversampling_avail, + .num_oversampling_press_avail = ARRAY_SIZE(bmp380_oversampling_avail), + .oversampling_press_default = ilog2(4), + + .sampling_freq_avail = bmp380_odr_table, + .num_sampling_freq_avail = ARRAY_SIZE(bmp380_odr_table) * 2, + .sampling_freq_default = BMP380_ODR_50HZ, + + .iir_filter_coeffs_avail = bmp380_iir_filter_coeffs_avail, + .num_iir_filter_coeffs_avail = ARRAY_SIZE(bmp380_iir_filter_coeffs_avail), + .iir_filter_coeff_default = 2, + + .chip_config = bmp380_chip_config, + .read_temp = bmp380_read_temp, + .read_press = bmp380_read_press, + .read_calib = bmp380_read_calib, +}; + +static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) +{ const int conversion_time_max[] = { 4500, 7500, 13500, 25500 }; unsigned int delay_us; unsigned int ctrl; + int ret; if (data->use_eoc) reinit_completion(&data->done); @@ -710,7 +1372,7 @@ static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) if (!ret) dev_err(data->dev, "timeout waiting for completion\n"); } else { - if (ctrl_meas == BMP180_MEAS_TEMP) + if (FIELD_GET(BMP180_MEAS_CTRL_MASK, ctrl_meas) == BMP180_MEAS_TEMP) delay_us = 4500; else delay_us = @@ -732,55 +1394,57 @@ static int bmp180_measure(struct bmp280_data *data, u8 ctrl_meas) static int bmp180_read_adc_temp(struct bmp280_data *data, int *val) { - __be16 tmp; int ret; - ret = bmp180_measure(data, BMP180_MEAS_TEMP); + ret = bmp180_measure(data, + FIELD_PREP(BMP180_MEAS_CTRL_MASK, BMP180_MEAS_TEMP) | + BMP180_MEAS_SCO); if (ret) return ret; - ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, &tmp, 2); + ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, + &data->be16, sizeof(data->be16)); if (ret) return ret; - *val = be16_to_cpu(tmp); + *val = be16_to_cpu(data->be16); return 0; } -static int bmp180_read_calib(struct bmp280_data *data, - struct bmp180_calib *calib) +static int bmp180_read_calib(struct bmp280_data *data) { + struct bmp180_calib *calib = &data->calib.bmp180; int ret; int i; - __be16 buf[BMP180_REG_CALIB_COUNT / 2]; - ret = regmap_bulk_read(data->regmap, BMP180_REG_CALIB_START, buf, - sizeof(buf)); + ret = regmap_bulk_read(data->regmap, BMP180_REG_CALIB_START, + data->bmp180_cal_buf, sizeof(data->bmp180_cal_buf)); if (ret < 0) return ret; /* None of the words has the value 0 or 0xFFFF */ - for (i = 0; i < ARRAY_SIZE(buf); i++) { - if (buf[i] == cpu_to_be16(0) || buf[i] == cpu_to_be16(0xffff)) + for (i = 0; i < ARRAY_SIZE(data->bmp180_cal_buf); i++) { + if (data->bmp180_cal_buf[i] == cpu_to_be16(0) || + data->bmp180_cal_buf[i] == cpu_to_be16(0xffff)) return -EIO; } /* Toss the calibration data into the entropy pool */ - add_device_randomness(buf, sizeof(buf)); - - calib->AC1 = be16_to_cpu(buf[AC1]); - calib->AC2 = be16_to_cpu(buf[AC2]); - calib->AC3 = be16_to_cpu(buf[AC3]); - calib->AC4 = be16_to_cpu(buf[AC4]); - calib->AC5 = be16_to_cpu(buf[AC5]); - calib->AC6 = be16_to_cpu(buf[AC6]); - calib->B1 = be16_to_cpu(buf[B1]); - calib->B2 = be16_to_cpu(buf[B2]); - calib->MB = be16_to_cpu(buf[MB]); - calib->MC = be16_to_cpu(buf[MC]); - calib->MD = be16_to_cpu(buf[MD]); + add_device_randomness(data->bmp180_cal_buf, sizeof(data->bmp180_cal_buf)); + + calib->AC1 = be16_to_cpu(data->bmp180_cal_buf[AC1]); + calib->AC2 = be16_to_cpu(data->bmp180_cal_buf[AC2]); + calib->AC3 = be16_to_cpu(data->bmp180_cal_buf[AC3]); + calib->AC4 = be16_to_cpu(data->bmp180_cal_buf[AC4]); + calib->AC5 = be16_to_cpu(data->bmp180_cal_buf[AC5]); + calib->AC6 = be16_to_cpu(data->bmp180_cal_buf[AC6]); + calib->B1 = be16_to_cpu(data->bmp180_cal_buf[B1]); + calib->B2 = be16_to_cpu(data->bmp180_cal_buf[B2]); + calib->MB = be16_to_cpu(data->bmp180_cal_buf[MB]); + calib->MC = be16_to_cpu(data->bmp180_cal_buf[MC]); + calib->MD = be16_to_cpu(data->bmp180_cal_buf[MD]); return 0; } @@ -793,8 +1457,8 @@ static int bmp180_read_calib(struct bmp280_data *data, */ static s32 bmp180_compensate_temp(struct bmp280_data *data, s32 adc_temp) { - s32 x1, x2; struct bmp180_calib *calib = &data->calib.bmp180; + s32 x1, x2; x1 = ((adc_temp - calib->AC6) * calib->AC5) >> 15; x2 = (calib->MC << 11) / (x1 + calib->MD); @@ -805,8 +1469,8 @@ static s32 bmp180_compensate_temp(struct bmp280_data *data, s32 adc_temp) static int bmp180_read_temp(struct bmp280_data *data, int *val) { - int ret; s32 adc_temp, comp_temp; + int ret; ret = bmp180_read_adc_temp(data, &adc_temp); if (ret) @@ -828,19 +1492,22 @@ static int bmp180_read_temp(struct bmp280_data *data, int *val) static int bmp180_read_adc_press(struct bmp280_data *data, int *val) { - int ret; - __be32 tmp = 0; u8 oss = data->oversampling_press; + int ret; - ret = bmp180_measure(data, BMP180_MEAS_PRESS_X(oss)); + ret = bmp180_measure(data, + FIELD_PREP(BMP180_MEAS_CTRL_MASK, BMP180_MEAS_PRESS) | + FIELD_PREP(BMP180_OSRS_PRESS_MASK, oss) | + BMP180_MEAS_SCO); if (ret) return ret; - ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, &tmp, 3); + ret = regmap_bulk_read(data->regmap, BMP180_REG_OUT_MSB, + data->buf, sizeof(data->buf)); if (ret) return ret; - *val = (be32_to_cpu(tmp) >> 8) >> (8 - oss); + *val = get_unaligned_be24(data->buf) >> (8 - oss); return 0; } @@ -852,11 +1519,11 @@ static int bmp180_read_adc_press(struct bmp280_data *data, int *val) */ static u32 bmp180_compensate_press(struct bmp280_data *data, s32 adc_press) { + struct bmp180_calib *calib = &data->calib.bmp180; + s32 oss = data->oversampling_press; s32 x1, x2, x3, p; s32 b3, b6; u32 b4, b7; - s32 oss = data->oversampling_press; - struct bmp180_calib *calib = &data->calib.bmp180; b6 = data->t_fine - 4000; x1 = (calib->B2 * (b6 * b6 >> 12)) >> 11; @@ -883,9 +1550,9 @@ static u32 bmp180_compensate_press(struct bmp280_data *data, s32 adc_press) static int bmp180_read_press(struct bmp280_data *data, int *val, int *val2) { - int ret; - s32 adc_press; u32 comp_press; + s32 adc_press; + int ret; /* Read and compensate temperature so we get a reading of t_fine. */ ret = bmp180_read_temp(data, NULL); @@ -913,17 +1580,25 @@ static const int bmp180_oversampling_temp_avail[] = { 1 }; static const int bmp180_oversampling_press_avail[] = { 1, 2, 4, 8 }; static const struct bmp280_chip_info bmp180_chip_info = { + .id_reg = BMP280_REG_ID, + .start_up_time = 2000, + .channels = bmp280_channels, + .num_channels = 2, + .oversampling_temp_avail = bmp180_oversampling_temp_avail, .num_oversampling_temp_avail = ARRAY_SIZE(bmp180_oversampling_temp_avail), + .oversampling_temp_default = 0, .oversampling_press_avail = bmp180_oversampling_press_avail, .num_oversampling_press_avail = ARRAY_SIZE(bmp180_oversampling_press_avail), + .oversampling_press_default = BMP180_MEAS_PRESS_8X, .chip_config = bmp180_chip_config, .read_temp = bmp180_read_temp, .read_press = bmp180_read_press, + .read_calib = bmp180_read_calib, }; static irqreturn_t bmp085_eoc_irq(int irq, void *d) @@ -990,11 +1665,12 @@ int bmp280_common_probe(struct device *dev, const char *name, int irq) { - int ret; + const struct bmp280_chip_info *chip_info; struct iio_dev *indio_dev; struct bmp280_data *data; - unsigned int chip_id; struct gpio_desc *gpiod; + unsigned int chip_id; + int ret; indio_dev = devm_iio_device_alloc(dev, sizeof(*data)); if (!indio_dev) @@ -1005,36 +1681,36 @@ int bmp280_common_probe(struct device *dev, data->dev = dev; indio_dev->name = name; - indio_dev->channels = bmp280_channels; indio_dev->info = &bmp280_info; indio_dev->modes = INDIO_DIRECT_MODE; switch (chip) { case BMP180_CHIP_ID: - indio_dev->num_channels = 2; - data->chip_info = &bmp180_chip_info; - data->oversampling_press = ilog2(8); - data->oversampling_temp = ilog2(1); - data->start_up_time = 10000; + chip_info = &bmp180_chip_info; break; case BMP280_CHIP_ID: - indio_dev->num_channels = 2; - data->chip_info = &bmp280_chip_info; - data->oversampling_press = ilog2(16); - data->oversampling_temp = ilog2(2); - data->start_up_time = 2000; + chip_info = &bmp280_chip_info; break; case BME280_CHIP_ID: - indio_dev->num_channels = 3; - data->chip_info = &bme280_chip_info; - data->oversampling_press = ilog2(16); - data->oversampling_humid = ilog2(16); - data->oversampling_temp = ilog2(2); - data->start_up_time = 2000; + chip_info = &bme280_chip_info; + break; + case BMP380_CHIP_ID: + chip_info = &bmp380_chip_info; break; default: return -EINVAL; } + data->chip_info = chip_info; + + /* Apply initial values from chip info structure */ + indio_dev->channels = chip_info->channels; + indio_dev->num_channels = chip_info->num_channels; + data->oversampling_press = chip_info->oversampling_press_default; + data->oversampling_humid = chip_info->oversampling_humid_default; + data->oversampling_temp = chip_info->oversampling_temp_default; + data->iir_filter_coeff = chip_info->iir_filter_coeff_default; + data->sampling_freq = chip_info->sampling_freq_default; + data->start_up_time = chip_info->start_up_time; /* Bring up regulators */ regulator_bulk_set_supply_names(data->supplies, @@ -1071,7 +1747,8 @@ int bmp280_common_probe(struct device *dev, } data->regmap = regmap; - ret = regmap_read(regmap, BMP280_REG_ID, &chip_id); + + ret = regmap_read(regmap, data->chip_info->id_reg, &chip_id); if (ret < 0) return ret; if (chip_id != chip) { @@ -1080,6 +1757,13 @@ int bmp280_common_probe(struct device *dev, return -EINVAL; } + /* BMP3xx requires soft-reset as part of initialization */ + if (chip_id == BMP380_CHIP_ID) { + ret = bmp380_cmd(data, BMP380_CMD_SOFT_RESET); + if (ret < 0) + return ret; + } + ret = data->chip_info->chip_config(data); if (ret < 0) return ret; @@ -1091,21 +1775,11 @@ int bmp280_common_probe(struct device *dev, * non-volatile memory during production". Let's read them out at probe * time once. They will not change. */ - if (chip_id == BMP180_CHIP_ID) { - ret = bmp180_read_calib(data, &data->calib.bmp180); - if (ret < 0) { - dev_err(data->dev, - "failed to read calibration coefficients\n"); - return ret; - } - } else if (chip_id == BMP280_CHIP_ID || chip_id == BME280_CHIP_ID) { - ret = bmp280_read_calib(data, &data->calib.bmp280, chip_id); - if (ret < 0) { - dev_err(data->dev, - "failed to read calibration coefficients\n"); - return ret; - } - } + + ret = data->chip_info->read_calib(data); + if (ret < 0) + return dev_err_probe(data->dev, ret, + "failed to read calibration coefficients\n"); /* * Attempt to grab an optional EOC IRQ - only the BMP085 has this diff --git a/drivers/iio/pressure/bmp280-i2c.c b/drivers/iio/pressure/bmp280-i2c.c index bf4a7a6175371d0a5cf59f131814674cdc3cfb7d..0c27211f3ea076417143a6eb3fea15f0581a929f 100644 --- a/drivers/iio/pressure/bmp280-i2c.c +++ b/drivers/iio/pressure/bmp280-i2c.c @@ -19,6 +19,9 @@ static int bmp280_i2c_probe(struct i2c_client *client, case BME280_CHIP_ID: regmap_config = &bmp280_regmap_config; break; + case BMP380_CHIP_ID: + regmap_config = &bmp380_regmap_config; + break; default: return -EINVAL; } @@ -37,19 +40,21 @@ static int bmp280_i2c_probe(struct i2c_client *client, } static const struct of_device_id bmp280_of_i2c_match[] = { - { .compatible = "bosch,bme280", .data = (void *)BME280_CHIP_ID }, - { .compatible = "bosch,bmp280", .data = (void *)BMP280_CHIP_ID }, - { .compatible = "bosch,bmp180", .data = (void *)BMP180_CHIP_ID }, { .compatible = "bosch,bmp085", .data = (void *)BMP180_CHIP_ID }, + { .compatible = "bosch,bmp180", .data = (void *)BMP180_CHIP_ID }, + { .compatible = "bosch,bmp280", .data = (void *)BMP280_CHIP_ID }, + { .compatible = "bosch,bme280", .data = (void *)BME280_CHIP_ID }, + { .compatible = "bosch,bmp380", .data = (void *)BMP380_CHIP_ID }, { }, }; MODULE_DEVICE_TABLE(of, bmp280_of_i2c_match); static const struct i2c_device_id bmp280_i2c_id[] = { - {"bmp280", BMP280_CHIP_ID }, - {"bmp180", BMP180_CHIP_ID }, {"bmp085", BMP180_CHIP_ID }, + {"bmp180", BMP180_CHIP_ID }, + {"bmp280", BMP280_CHIP_ID }, {"bme280", BME280_CHIP_ID }, + {"bmp380", BMP380_CHIP_ID }, { }, }; MODULE_DEVICE_TABLE(i2c, bmp280_i2c_id); diff --git a/drivers/iio/pressure/bmp280-regmap.c b/drivers/iio/pressure/bmp280-regmap.c index 9696985189840965d576fb8ef8214e4e10d9bf22..c98c67970265480aabea6c93e0d3e770cd2c008a 100644 --- a/drivers/iio/pressure/bmp280-regmap.c +++ b/drivers/iio/pressure/bmp280-regmap.c @@ -72,6 +72,49 @@ static bool bmp280_is_volatile_reg(struct device *dev, unsigned int reg) } } +static bool bmp380_is_writeable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP380_REG_CMD: + case BMP380_REG_CONFIG: + case BMP380_REG_FIFO_CONFIG_1: + case BMP380_REG_FIFO_CONFIG_2: + case BMP380_REG_FIFO_WATERMARK_LSB: + case BMP380_REG_FIFO_WATERMARK_MSB: + case BMP380_REG_POWER_CONTROL: + case BMP380_REG_INT_CONTROL: + case BMP380_REG_IF_CONFIG: + case BMP380_REG_ODR: + case BMP380_REG_OSR: + return true; + default: + return false; + } +} + +static bool bmp380_is_volatile_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case BMP380_REG_TEMP_XLSB: + case BMP380_REG_TEMP_LSB: + case BMP380_REG_TEMP_MSB: + case BMP380_REG_PRESS_XLSB: + case BMP380_REG_PRESS_LSB: + case BMP380_REG_PRESS_MSB: + case BMP380_REG_SENSOR_TIME_XLSB: + case BMP380_REG_SENSOR_TIME_LSB: + case BMP380_REG_SENSOR_TIME_MSB: + case BMP380_REG_INT_STATUS: + case BMP380_REG_FIFO_DATA: + case BMP380_REG_STATUS: + case BMP380_REG_ERROR: + case BMP380_REG_EVENT: + return true; + default: + return false; + } +} + const struct regmap_config bmp280_regmap_config = { .reg_bits = 8, .val_bits = 8, @@ -83,3 +126,15 @@ const struct regmap_config bmp280_regmap_config = { .volatile_reg = bmp280_is_volatile_reg, }; EXPORT_SYMBOL_NS(bmp280_regmap_config, IIO_BMP280); + +const struct regmap_config bmp380_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + + .max_register = BMP380_REG_CMD, + .cache_type = REGCACHE_RBTREE, + + .writeable_reg = bmp380_is_writeable_reg, + .volatile_reg = bmp380_is_volatile_reg, +}; +EXPORT_SYMBOL_NS(bmp380_regmap_config, IIO_BMP280); diff --git a/drivers/iio/pressure/bmp280-spi.c b/drivers/iio/pressure/bmp280-spi.c index 4cfaf3e869b8f5c1e8856cefcf238892dbc01261..011c68e07ebfb7b2c23071fd103f9a0e2f76863c 100644 --- a/drivers/iio/pressure/bmp280-spi.c +++ b/drivers/iio/pressure/bmp280-spi.c @@ -66,6 +66,9 @@ static int bmp280_spi_probe(struct spi_device *spi) case BME280_CHIP_ID: regmap_config = &bmp280_regmap_config; break; + case BMP380_CHIP_ID: + regmap_config = &bmp380_regmap_config; + break; default: return -EINVAL; } @@ -92,6 +95,7 @@ static const struct of_device_id bmp280_of_spi_match[] = { { .compatible = "bosch,bmp181", }, { .compatible = "bosch,bmp280", }, { .compatible = "bosch,bme280", }, + { .compatible = "bosch,bmp380", }, { }, }; MODULE_DEVICE_TABLE(of, bmp280_of_spi_match); @@ -101,6 +105,7 @@ static const struct spi_device_id bmp280_spi_id[] = { { "bmp181", BMP180_CHIP_ID }, { "bmp280", BMP280_CHIP_ID }, { "bme280", BME280_CHIP_ID }, + { "bmp380", BMP380_CHIP_ID }, { } }; MODULE_DEVICE_TABLE(spi, bmp280_spi_id); diff --git a/drivers/iio/pressure/bmp280.h b/drivers/iio/pressure/bmp280.h index 57ba0e85db915ab323c52b79a65c10a8de3841c4..c791325c741638e9af7564a19d3a065b3a9a768b 100644 --- a/drivers/iio/pressure/bmp280.h +++ b/drivers/iio/pressure/bmp280.h @@ -3,6 +3,87 @@ #include #include +/* BMP380 specific registers */ +#define BMP380_REG_CMD 0x7E +#define BMP380_REG_CONFIG 0x1F +#define BMP380_REG_ODR 0x1D +#define BMP380_REG_OSR 0x1C +#define BMP380_REG_POWER_CONTROL 0x1B +#define BMP380_REG_IF_CONFIG 0x1A +#define BMP380_REG_INT_CONTROL 0x19 +#define BMP380_REG_INT_STATUS 0x11 +#define BMP380_REG_EVENT 0x10 +#define BMP380_REG_STATUS 0x03 +#define BMP380_REG_ERROR 0x02 +#define BMP380_REG_ID 0x00 + +#define BMP380_REG_FIFO_CONFIG_1 0x18 +#define BMP380_REG_FIFO_CONFIG_2 0x17 +#define BMP380_REG_FIFO_WATERMARK_MSB 0x16 +#define BMP380_REG_FIFO_WATERMARK_LSB 0x15 +#define BMP380_REG_FIFO_DATA 0x14 +#define BMP380_REG_FIFO_LENGTH_MSB 0x13 +#define BMP380_REG_FIFO_LENGTH_LSB 0x12 + +#define BMP380_REG_SENSOR_TIME_MSB 0x0E +#define BMP380_REG_SENSOR_TIME_LSB 0x0D +#define BMP380_REG_SENSOR_TIME_XLSB 0x0C + +#define BMP380_REG_TEMP_MSB 0x09 +#define BMP380_REG_TEMP_LSB 0x08 +#define BMP380_REG_TEMP_XLSB 0x07 + +#define BMP380_REG_PRESS_MSB 0x06 +#define BMP380_REG_PRESS_LSB 0x05 +#define BMP380_REG_PRESS_XLSB 0x04 + +#define BMP380_REG_CALIB_TEMP_START 0x31 +#define BMP380_CALIB_REG_COUNT 21 + +#define BMP380_FILTER_MASK GENMASK(3, 1) +#define BMP380_FILTER_OFF 0 +#define BMP380_FILTER_1X 1 +#define BMP380_FILTER_3X 2 +#define BMP380_FILTER_7X 3 +#define BMP380_FILTER_15X 4 +#define BMP380_FILTER_31X 5 +#define BMP380_FILTER_63X 6 +#define BMP380_FILTER_127X 7 + +#define BMP380_OSRS_TEMP_MASK GENMASK(5, 3) +#define BMP380_OSRS_PRESS_MASK GENMASK(2, 0) + +#define BMP380_ODRS_MASK GENMASK(4, 0) + +#define BMP380_CTRL_SENSORS_MASK GENMASK(1, 0) +#define BMP380_CTRL_SENSORS_PRESS_EN BIT(0) +#define BMP380_CTRL_SENSORS_TEMP_EN BIT(1) +#define BMP380_MODE_MASK GENMASK(5, 4) +#define BMP380_MODE_SLEEP 0 +#define BMP380_MODE_FORCED 1 +#define BMP380_MODE_NORMAL 3 + +#define BMP380_MIN_TEMP -4000 +#define BMP380_MAX_TEMP 8500 +#define BMP380_MIN_PRES 3000000 +#define BMP380_MAX_PRES 12500000 + +#define BMP380_CMD_NOOP 0x00 +#define BMP380_CMD_EXTMODE_EN_MID 0x34 +#define BMP380_CMD_FIFO_FLUSH 0xB0 +#define BMP380_CMD_SOFT_RESET 0xB6 + +#define BMP380_STATUS_CMD_RDY_MASK BIT(4) +#define BMP380_STATUS_DRDY_PRESS_MASK BIT(5) +#define BMP380_STATUS_DRDY_TEMP_MASK BIT(6) + +#define BMP380_ERR_FATAL_MASK BIT(0) +#define BMP380_ERR_CMD_MASK BIT(1) +#define BMP380_ERR_CONF_MASK BIT(2) + +#define BMP380_TEMP_SKIPPED 0x800000 +#define BMP380_PRESS_SKIPPED 0x800000 + /* BMP280 specific registers */ #define BMP280_REG_HUMIDITY_LSB 0xFE #define BMP280_REG_HUMIDITY_MSB 0xFD @@ -13,6 +94,9 @@ #define BMP280_REG_PRESS_LSB 0xF8 #define BMP280_REG_PRESS_MSB 0xF7 +/* Helper mask to truncate excess 4 bits on pressure and temp readings */ +#define BMP280_MEAS_TRIM_MASK GENMASK(24, 4) + #define BMP280_REG_CONFIG 0xF5 #define BMP280_REG_CTRL_MEAS 0xF4 #define BMP280_REG_STATUS 0xF3 @@ -32,44 +116,46 @@ #define BMP280_REG_COMP_PRESS_START 0x8E #define BMP280_COMP_PRESS_REG_COUNT 18 -#define BMP280_FILTER_MASK (BIT(4) | BIT(3) | BIT(2)) +#define BMP280_COMP_H5_MASK GENMASK(15, 4) + +#define BMP280_CONTIGUOUS_CALIB_REGS (BMP280_COMP_TEMP_REG_COUNT + \ + BMP280_COMP_PRESS_REG_COUNT) + +#define BMP280_FILTER_MASK GENMASK(4, 2) #define BMP280_FILTER_OFF 0 -#define BMP280_FILTER_2X BIT(2) -#define BMP280_FILTER_4X BIT(3) -#define BMP280_FILTER_8X (BIT(3) | BIT(2)) -#define BMP280_FILTER_16X BIT(4) +#define BMP280_FILTER_2X 1 +#define BMP280_FILTER_4X 2 +#define BMP280_FILTER_8X 3 +#define BMP280_FILTER_16X 4 -#define BMP280_OSRS_HUMIDITY_MASK (BIT(2) | BIT(1) | BIT(0)) -#define BMP280_OSRS_HUMIDITIY_X(osrs_h) ((osrs_h) << 0) +#define BMP280_OSRS_HUMIDITY_MASK GENMASK(2, 0) #define BMP280_OSRS_HUMIDITY_SKIP 0 -#define BMP280_OSRS_HUMIDITY_1X BMP280_OSRS_HUMIDITIY_X(1) -#define BMP280_OSRS_HUMIDITY_2X BMP280_OSRS_HUMIDITIY_X(2) -#define BMP280_OSRS_HUMIDITY_4X BMP280_OSRS_HUMIDITIY_X(3) -#define BMP280_OSRS_HUMIDITY_8X BMP280_OSRS_HUMIDITIY_X(4) -#define BMP280_OSRS_HUMIDITY_16X BMP280_OSRS_HUMIDITIY_X(5) +#define BMP280_OSRS_HUMIDITY_1X 1 +#define BMP280_OSRS_HUMIDITY_2X 2 +#define BMP280_OSRS_HUMIDITY_4X 3 +#define BMP280_OSRS_HUMIDITY_8X 4 +#define BMP280_OSRS_HUMIDITY_16X 5 -#define BMP280_OSRS_TEMP_MASK (BIT(7) | BIT(6) | BIT(5)) +#define BMP280_OSRS_TEMP_MASK GENMASK(7, 5) #define BMP280_OSRS_TEMP_SKIP 0 -#define BMP280_OSRS_TEMP_X(osrs_t) ((osrs_t) << 5) -#define BMP280_OSRS_TEMP_1X BMP280_OSRS_TEMP_X(1) -#define BMP280_OSRS_TEMP_2X BMP280_OSRS_TEMP_X(2) -#define BMP280_OSRS_TEMP_4X BMP280_OSRS_TEMP_X(3) -#define BMP280_OSRS_TEMP_8X BMP280_OSRS_TEMP_X(4) -#define BMP280_OSRS_TEMP_16X BMP280_OSRS_TEMP_X(5) - -#define BMP280_OSRS_PRESS_MASK (BIT(4) | BIT(3) | BIT(2)) +#define BMP280_OSRS_TEMP_1X 1 +#define BMP280_OSRS_TEMP_2X 2 +#define BMP280_OSRS_TEMP_4X 3 +#define BMP280_OSRS_TEMP_8X 4 +#define BMP280_OSRS_TEMP_16X 5 + +#define BMP280_OSRS_PRESS_MASK GENMASK(4, 2) #define BMP280_OSRS_PRESS_SKIP 0 -#define BMP280_OSRS_PRESS_X(osrs_p) ((osrs_p) << 2) -#define BMP280_OSRS_PRESS_1X BMP280_OSRS_PRESS_X(1) -#define BMP280_OSRS_PRESS_2X BMP280_OSRS_PRESS_X(2) -#define BMP280_OSRS_PRESS_4X BMP280_OSRS_PRESS_X(3) -#define BMP280_OSRS_PRESS_8X BMP280_OSRS_PRESS_X(4) -#define BMP280_OSRS_PRESS_16X BMP280_OSRS_PRESS_X(5) - -#define BMP280_MODE_MASK (BIT(1) | BIT(0)) +#define BMP280_OSRS_PRESS_1X 1 +#define BMP280_OSRS_PRESS_2X 2 +#define BMP280_OSRS_PRESS_4X 3 +#define BMP280_OSRS_PRESS_8X 4 +#define BMP280_OSRS_PRESS_16X 5 + +#define BMP280_MODE_MASK GENMASK(1, 0) #define BMP280_MODE_SLEEP 0 -#define BMP280_MODE_FORCED BIT(0) -#define BMP280_MODE_NORMAL (BIT(1) | BIT(0)) +#define BMP280_MODE_FORCED 1 +#define BMP280_MODE_NORMAL 3 /* BMP180 specific registers */ #define BMP180_REG_OUT_XLSB 0xF8 @@ -79,19 +165,22 @@ #define BMP180_REG_CALIB_START 0xAA #define BMP180_REG_CALIB_COUNT 22 +#define BMP180_MEAS_CTRL_MASK GENMASK(4, 0) +#define BMP180_MEAS_TEMP 0x0E +#define BMP180_MEAS_PRESS 0x14 #define BMP180_MEAS_SCO BIT(5) -#define BMP180_MEAS_TEMP (0x0E | BMP180_MEAS_SCO) -#define BMP180_MEAS_PRESS_X(oss) ((oss) << 6 | 0x14 | BMP180_MEAS_SCO) -#define BMP180_MEAS_PRESS_1X BMP180_MEAS_PRESS_X(0) -#define BMP180_MEAS_PRESS_2X BMP180_MEAS_PRESS_X(1) -#define BMP180_MEAS_PRESS_4X BMP180_MEAS_PRESS_X(2) -#define BMP180_MEAS_PRESS_8X BMP180_MEAS_PRESS_X(3) +#define BMP180_OSRS_PRESS_MASK GENMASK(7, 6) +#define BMP180_MEAS_PRESS_1X 0 +#define BMP180_MEAS_PRESS_2X 1 +#define BMP180_MEAS_PRESS_4X 2 +#define BMP180_MEAS_PRESS_8X 3 /* BMP180 and BMP280 common registers */ #define BMP280_REG_CTRL_MEAS 0xF4 #define BMP280_REG_RESET 0xE0 #define BMP280_REG_ID 0xD0 +#define BMP380_CHIP_ID 0x50 #define BMP180_CHIP_ID 0x55 #define BMP280_CHIP_ID 0x58 #define BME280_CHIP_ID 0x60 @@ -105,6 +194,7 @@ /* Regmap configurations */ extern const struct regmap_config bmp180_regmap_config; extern const struct regmap_config bmp280_regmap_config; +extern const struct regmap_config bmp380_regmap_config; /* Probe called from different transports */ int bmp280_common_probe(struct device *dev, diff --git a/drivers/iio/pressure/dlhl60d.c b/drivers/iio/pressure/dlhl60d.c index 5f6bb3603a8b1a9d7755aa08cc7ea327b52b76f8..f0b0d198c6d472a5533a26c063775f595123bc10 100644 --- a/drivers/iio/pressure/dlhl60d.c +++ b/drivers/iio/pressure/dlhl60d.c @@ -129,9 +129,8 @@ static int dlh_read_direct(struct dlh_state *st, if (ret) return ret; - *pressure = get_unaligned_be32(&st->rx_buf[1]) >> 8; - *temperature = get_unaligned_be32(&st->rx_buf[3]) & - GENMASK(DLH_NUM_TEMP_BITS - 1, 0); + *pressure = get_unaligned_be24(&st->rx_buf[1]); + *temperature = get_unaligned_be24(&st->rx_buf[4]); return 0; } diff --git a/drivers/iio/pressure/dps310.c b/drivers/iio/pressure/dps310.c index 36fb7ae0d0a9d99f81aaea6cf53f06a31c1ab16a..984a3f511a1ae31d14472edf1fe3edf5f3187c6f 100644 --- a/drivers/iio/pressure/dps310.c +++ b/drivers/iio/pressure/dps310.c @@ -89,6 +89,7 @@ struct dps310_data { s32 c00, c10, c20, c30, c01, c11, c21; s32 pressure_raw; s32 temp_raw; + bool timeout_recovery_failed; }; static const struct iio_chan_spec dps310_channels[] = { @@ -159,6 +160,102 @@ static int dps310_get_coefs(struct dps310_data *data) return 0; } +/* + * Some versions of the chip will read temperatures in the ~60C range when + * it's actually ~20C. This is the manufacturer recommended workaround + * to correct the issue. The registers used below are undocumented. + */ +static int dps310_temp_workaround(struct dps310_data *data) +{ + int rc; + int reg; + + rc = regmap_read(data->regmap, 0x32, ®); + if (rc) + return rc; + + /* + * If bit 1 is set then the device is okay, and the workaround does not + * need to be applied + */ + if (reg & BIT(1)) + return 0; + + rc = regmap_write(data->regmap, 0x0e, 0xA5); + if (rc) + return rc; + + rc = regmap_write(data->regmap, 0x0f, 0x96); + if (rc) + return rc; + + rc = regmap_write(data->regmap, 0x62, 0x02); + if (rc) + return rc; + + rc = regmap_write(data->regmap, 0x0e, 0x00); + if (rc) + return rc; + + return regmap_write(data->regmap, 0x0f, 0x00); +} + +static int dps310_startup(struct dps310_data *data) +{ + int rc; + int ready; + + /* + * Set up pressure sensor in single sample, one measurement per second + * mode + */ + rc = regmap_write(data->regmap, DPS310_PRS_CFG, 0); + if (rc) + return rc; + + /* + * Set up external (MEMS) temperature sensor in single sample, one + * measurement per second mode + */ + rc = regmap_write(data->regmap, DPS310_TMP_CFG, DPS310_TMP_EXT); + if (rc) + return rc; + + /* Temp and pressure shifts are disabled when PRC <= 8 */ + rc = regmap_write_bits(data->regmap, DPS310_CFG_REG, + DPS310_PRS_SHIFT_EN | DPS310_TMP_SHIFT_EN, 0); + if (rc) + return rc; + + /* MEAS_CFG doesn't update correctly unless first written with 0 */ + rc = regmap_write_bits(data->regmap, DPS310_MEAS_CFG, + DPS310_MEAS_CTRL_BITS, 0); + if (rc) + return rc; + + /* Turn on temperature and pressure measurement in the background */ + rc = regmap_write_bits(data->regmap, DPS310_MEAS_CFG, + DPS310_MEAS_CTRL_BITS, DPS310_PRS_EN | + DPS310_TEMP_EN | DPS310_BACKGROUND); + if (rc) + return rc; + + /* + * Calibration coefficients required for reporting temperature. + * They are available 40ms after the device has started + */ + rc = regmap_read_poll_timeout(data->regmap, DPS310_MEAS_CFG, ready, + ready & DPS310_COEF_RDY, 10000, 40000); + if (rc) + return rc; + + rc = dps310_get_coefs(data); + if (rc) + return rc; + + return dps310_temp_workaround(data); +} + static int dps310_get_pres_precision(struct dps310_data *data) { int rc; @@ -297,11 +394,69 @@ static int dps310_get_temp_k(struct dps310_data *data) return scale_factors[ilog2(rc)]; } +static int dps310_reset_wait(struct dps310_data *data) +{ + int rc; + + rc = regmap_write(data->regmap, DPS310_RESET, DPS310_RESET_MAGIC); + if (rc) + return rc; + + /* Wait for device chip access: 2.5ms in specification */ + usleep_range(2500, 12000); + return 0; +} + +static int dps310_reset_reinit(struct dps310_data *data) +{ + int rc; + + rc = dps310_reset_wait(data); + if (rc) + return rc; + + return dps310_startup(data); +} + +static int dps310_ready_status(struct dps310_data *data, int ready_bit, int timeout) +{ + int sleep = DPS310_POLL_SLEEP_US(timeout); + int ready; + + return regmap_read_poll_timeout(data->regmap, DPS310_MEAS_CFG, ready, ready & ready_bit, + sleep, timeout); +} + +static int dps310_ready(struct dps310_data *data, int ready_bit, int timeout) +{ + int rc; + + rc = dps310_ready_status(data, ready_bit, timeout); + if (rc) { + if (rc == -ETIMEDOUT && !data->timeout_recovery_failed) { + /* Reset and reinitialize the chip. */ + if (dps310_reset_reinit(data)) { + data->timeout_recovery_failed = true; + } else { + /* Try again to get sensor ready status. */ + if (dps310_ready_status(data, ready_bit, timeout)) + data->timeout_recovery_failed = true; + else + return 0; + } + } + + return rc; + } + + data->timeout_recovery_failed = false; + return 0; +} + static int dps310_read_pres_raw(struct dps310_data *data) { int rc; int rate; - int ready; int timeout; s32 raw; u8 val[3]; @@ -313,9 +468,7 @@ static int dps310_read_pres_raw(struct dps310_data *data) timeout = DPS310_POLL_TIMEOUT_US(rate); /* Poll for sensor readiness; base the timeout upon the sample rate. */ - rc = regmap_read_poll_timeout(data->regmap, DPS310_MEAS_CFG, ready, - ready & DPS310_PRS_RDY, - DPS310_POLL_SLEEP_US(timeout), timeout); + rc = dps310_ready(data, DPS310_PRS_RDY, timeout); if (rc) goto done; @@ -352,7 +505,6 @@ static int dps310_read_temp_raw(struct dps310_data *data) { int rc; int rate; - int ready; int timeout; if (mutex_lock_interruptible(&data->lock)) @@ -362,10 +514,8 @@ static int dps310_read_temp_raw(struct dps310_data *data) timeout = DPS310_POLL_TIMEOUT_US(rate); /* Poll for sensor readiness; base the timeout upon the sample rate. */ - rc = regmap_read_poll_timeout(data->regmap, DPS310_MEAS_CFG, ready, - ready & DPS310_TMP_RDY, - DPS310_POLL_SLEEP_US(timeout), timeout); - if (rc < 0) + rc = dps310_ready(data, DPS310_TMP_RDY, timeout); + if (rc) goto done; rc = dps310_read_temp_ready(data); @@ -660,7 +810,7 @@ static void dps310_reset(void *action_data) { struct dps310_data *data = action_data; - regmap_write(data->regmap, DPS310_RESET, DPS310_RESET_MAGIC); + dps310_reset_wait(data); } static const struct regmap_config dps310_regmap_config = { @@ -677,52 +827,12 @@ static const struct iio_info dps310_info = { .write_raw = dps310_write_raw, }; -/* - * Some verions of chip will read temperatures in the ~60C range when - * its actually ~20C. This is the manufacturer recommended workaround - * to correct the issue. The registers used below are undocumented. - */ -static int dps310_temp_workaround(struct dps310_data *data) -{ - int rc; - int reg; - - rc = regmap_read(data->regmap, 0x32, ®); - if (rc < 0) - return rc; - - /* - * If bit 1 is set then the device is okay, and the workaround does not - * need to be applied - */ - if (reg & BIT(1)) - return 0; - - rc = regmap_write(data->regmap, 0x0e, 0xA5); - if (rc < 0) - return rc; - - rc = regmap_write(data->regmap, 0x0f, 0x96); - if (rc < 0) - return rc; - - rc = regmap_write(data->regmap, 0x62, 0x02); - if (rc < 0) - return rc; - - rc = regmap_write(data->regmap, 0x0e, 0x00); - if (rc < 0) - return rc; - - return regmap_write(data->regmap, 0x0f, 0x00); -} - static int dps310_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct dps310_data *data; struct iio_dev *iio; - int rc, ready; + int rc; iio = devm_iio_device_alloc(&client->dev, sizeof(*data)); if (!iio) @@ -747,54 +857,8 @@ static int dps310_probe(struct i2c_client *client, if (rc) return rc; - /* - * Set up pressure sensor in single sample, one measurement per second - * mode - */ - rc = regmap_write(data->regmap, DPS310_PRS_CFG, 0); - - /* - * Set up external (MEMS) temperature sensor in single sample, one - * measurement per second mode - */ - rc = regmap_write(data->regmap, DPS310_TMP_CFG, DPS310_TMP_EXT); - if (rc < 0) - return rc; - - /* Temp and pressure shifts are disabled when PRC <= 8 */ - rc = regmap_write_bits(data->regmap, DPS310_CFG_REG, - DPS310_PRS_SHIFT_EN | DPS310_TMP_SHIFT_EN, 0); - if (rc < 0) - return rc; - - /* MEAS_CFG doesn't update correctly unless first written with 0 */ - rc = regmap_write_bits(data->regmap, DPS310_MEAS_CFG, - DPS310_MEAS_CTRL_BITS, 0); - if (rc < 0) - return rc; - - /* Turn on temperature and pressure measurement in the background */ - rc = regmap_write_bits(data->regmap, DPS310_MEAS_CFG, - DPS310_MEAS_CTRL_BITS, DPS310_PRS_EN | - DPS310_TEMP_EN | DPS310_BACKGROUND); - if (rc < 0) - return rc; - - /* - * Calibration coefficients required for reporting temperature. - * They are available 40ms after the device has started - */ - rc = regmap_read_poll_timeout(data->regmap, DPS310_MEAS_CFG, ready, - ready & DPS310_COEF_RDY, 10000, 40000); - if (rc < 0) - return rc; - - rc = dps310_get_coefs(data); - if (rc < 0) - return rc; - - rc = dps310_temp_workaround(data); - if (rc < 0) + rc = dps310_startup(data); + if (rc) return rc; rc = devm_iio_device_register(&client->dev, iio); diff --git a/drivers/iio/pressure/icp10100.c b/drivers/iio/pressure/icp10100.c index af4621eaa6b56d3fae488e5d253b06180f93139c..b62f28585db5b8d66cd1b931a65fcf36f35adf13 100644 --- a/drivers/iio/pressure/icp10100.c +++ b/drivers/iio/pressure/icp10100.c @@ -595,7 +595,7 @@ static int icp10100_probe(struct i2c_client *client, return devm_iio_device_register(&client->dev, indio_dev); } -static int __maybe_unused icp10100_suspend(struct device *dev) +static int icp10100_suspend(struct device *dev) { struct icp10100_state *st = iio_priv(dev_get_drvdata(dev)); int ret; @@ -607,7 +607,7 @@ static int __maybe_unused icp10100_suspend(struct device *dev) return ret; } -static int __maybe_unused icp10100_resume(struct device *dev) +static int icp10100_resume(struct device *dev) { struct icp10100_state *st = iio_priv(dev_get_drvdata(dev)); int ret; @@ -626,8 +626,8 @@ out_unlock: return ret; } -static UNIVERSAL_DEV_PM_OPS(icp10100_pm, icp10100_suspend, icp10100_resume, - NULL); +static DEFINE_RUNTIME_DEV_PM_OPS(icp10100_pm, icp10100_suspend, icp10100_resume, + NULL); static const struct of_device_id icp10100_of_match[] = { { @@ -646,7 +646,7 @@ MODULE_DEVICE_TABLE(i2c, icp10100_id); static struct i2c_driver icp10100_driver = { .driver = { .name = "icp10100", - .pm = &icp10100_pm, + .pm = pm_ptr(&icp10100_pm), .of_match_table = icp10100_of_match, }, .probe = icp10100_probe, diff --git a/drivers/iio/pressure/mpl3115.c b/drivers/iio/pressure/mpl3115.c index d4f89e4babed4d9ac103efce6364171ae8d5f4ac..2f22aba61e4dd39a9afcd1c768fdfb53f7ce02e2 100644 --- a/drivers/iio/pressure/mpl3115.c +++ b/drivers/iio/pressure/mpl3115.c @@ -290,15 +290,13 @@ static int mpl3115_standby(struct mpl3115_data *data) data->ctrl_reg1 & ~MPL3115_CTRL_ACTIVE); } -static int mpl3115_remove(struct i2c_client *client) +static void mpl3115_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); iio_device_unregister(indio_dev); iio_triggered_buffer_cleanup(indio_dev); mpl3115_standby(iio_priv(indio_dev)); - - return 0; } static int mpl3115_suspend(struct device *dev) diff --git a/drivers/iio/pressure/ms5611_i2c.c b/drivers/iio/pressure/ms5611_i2c.c index 3b1de71e0d15cfdd89074ae628d0ac23e0a25b2d..b681a4183909891cda293561ba51b148c9af9b8e 100644 --- a/drivers/iio/pressure/ms5611_i2c.c +++ b/drivers/iio/pressure/ms5611_i2c.c @@ -105,11 +105,9 @@ static int ms5611_i2c_probe(struct i2c_client *client, return ms5611_probe(indio_dev, &client->dev, id->name, id->driver_data); } -static int ms5611_i2c_remove(struct i2c_client *client) +static void ms5611_i2c_remove(struct i2c_client *client) { ms5611_remove(i2c_get_clientdata(client)); - - return 0; } static const struct of_device_id ms5611_i2c_matches[] = { diff --git a/drivers/iio/pressure/st_pressure.h b/drivers/iio/pressure/st_pressure.h index 156e6a72dc5cff7c345707eb5665dde1be94f13a..6e11bea784fa76db73ded4e76b5e557a888c5045 100644 --- a/drivers/iio/pressure/st_pressure.h +++ b/drivers/iio/pressure/st_pressure.h @@ -22,6 +22,7 @@ enum st_press_type { LPS33HW, LPS35HW, LPS22HH, + LPS22DF, ST_PRESS_MAX, }; @@ -32,6 +33,7 @@ enum st_press_type { #define LPS33HW_PRESS_DEV_NAME "lps33hw" #define LPS35HW_PRESS_DEV_NAME "lps35hw" #define LPS22HH_PRESS_DEV_NAME "lps22hh" +#define LPS22DF_PRESS_DEV_NAME "lps22df" /** * struct st_sensors_platform_data - default press platform data diff --git a/drivers/iio/pressure/st_pressure_core.c b/drivers/iio/pressure/st_pressure_core.c index 76913a2028d276ad4e0cb03ab6b94e98c85ee7e0..80176e3083afbbbc70d2dffe1ad7a675064db422 100644 --- a/drivers/iio/pressure/st_pressure_core.c +++ b/drivers/iio/pressure/st_pressure_core.c @@ -552,6 +552,76 @@ static const struct st_sensor_settings st_press_sensors_settings[] = { .multi_read_bit = false, .bootime = 2, }, + { + /* + * CUSTOM VALUES FOR LPS22DF SENSOR + * See LPS22DF datasheet: + * http://www.st.com/resource/en/datasheet/lps22df.pdf + */ + .wai = 0xb4, + .wai_addr = ST_SENSORS_DEFAULT_WAI_ADDRESS, + .sensors_supported = { + [0] = LPS22DF_PRESS_DEV_NAME, + }, + .ch = (struct iio_chan_spec *)st_press_lps22hb_channels, + .num_ch = ARRAY_SIZE(st_press_lps22hb_channels), + .odr = { + .addr = 0x10, + .mask = 0x78, + .odr_avl = { + { .hz = 1, .value = 0x01 }, + { .hz = 4, .value = 0x02 }, + { .hz = 10, .value = 0x03 }, + { .hz = 25, .value = 0x04 }, + { .hz = 50, .value = 0x05 }, + { .hz = 75, .value = 0x06 }, + { .hz = 100, .value = 0x07 }, + { .hz = 200, .value = 0x08 }, + }, + }, + .pw = { + .addr = 0x10, + .mask = 0x78, + .value_off = ST_SENSORS_DEFAULT_POWER_OFF_VALUE, + }, + .fs = { + .fs_avl = { + /* + * Pressure and temperature sensitivity values + * as defined in table 2 of LPS22DF datasheet. + */ + [0] = { + .num = ST_PRESS_FS_AVL_1260MB, + .gain = ST_PRESS_KPASCAL_NANO_SCALE, + .gain2 = ST_PRESS_LPS22HB_LSB_PER_CELSIUS, + }, + }, + }, + .bdu = { + .addr = 0x11, + .mask = BIT(3), + }, + .drdy_irq = { + .int1 = { + .addr = 0x13, + .mask = BIT(5), + .addr_od = 0x12, + .mask_od = BIT(1), + }, + .addr_ihl = 0x12, + .mask_ihl = BIT(3), + .stat_drdy = { + .addr = ST_SENSORS_DEFAULT_STAT_ADDR, + .mask = 0x03, + }, + }, + .sim = { + .addr = 0x0E, + .value = BIT(5), + }, + .multi_read_bit = false, + .bootime = 2, + }, }; static int st_press_write_raw(struct iio_dev *indio_dev, diff --git a/drivers/iio/pressure/st_pressure_i2c.c b/drivers/iio/pressure/st_pressure_i2c.c index 7035777fd9887342249ee76336de4f0fac0a9a5e..58fede8618911a9bd2df50859ed04476749a1951 100644 --- a/drivers/iio/pressure/st_pressure_i2c.c +++ b/drivers/iio/pressure/st_pressure_i2c.c @@ -47,6 +47,10 @@ static const struct of_device_id st_press_of_match[] = { .compatible = "st,lps22hh", .data = LPS22HH_PRESS_DEV_NAME, }, + { + .compatible = "st,lps22df", + .data = LPS22DF_PRESS_DEV_NAME, + }, {}, }; MODULE_DEVICE_TABLE(of, st_press_of_match); @@ -67,6 +71,7 @@ static const struct i2c_device_id st_press_id_table[] = { { LPS33HW_PRESS_DEV_NAME, LPS33HW }, { LPS35HW_PRESS_DEV_NAME, LPS35HW }, { LPS22HH_PRESS_DEV_NAME, LPS22HH }, + { LPS22DF_PRESS_DEV_NAME, LPS22DF }, {}, }; MODULE_DEVICE_TABLE(i2c, st_press_id_table); diff --git a/drivers/iio/pressure/st_pressure_spi.c b/drivers/iio/pressure/st_pressure_spi.c index bfab8e7fb0619acd80e4fcf3ddeadc7c8a3a270b..25cca5ad7c5599df8544eb10edcbe223e484aca0 100644 --- a/drivers/iio/pressure/st_pressure_spi.c +++ b/drivers/iio/pressure/st_pressure_spi.c @@ -51,6 +51,10 @@ static const struct of_device_id st_press_of_match[] = { .compatible = "st,lps22hh", .data = LPS22HH_PRESS_DEV_NAME, }, + { + .compatible = "st,lps22df", + .data = LPS22DF_PRESS_DEV_NAME, + }, {}, }; MODULE_DEVICE_TABLE(of, st_press_of_match); @@ -97,6 +101,7 @@ static const struct spi_device_id st_press_id_table[] = { { LPS33HW_PRESS_DEV_NAME }, { LPS35HW_PRESS_DEV_NAME }, { LPS22HH_PRESS_DEV_NAME }, + { LPS22DF_PRESS_DEV_NAME }, { "lps001wp-press" }, { "lps25h-press", }, { "lps331ap-press" }, diff --git a/drivers/iio/pressure/zpa2326_i2c.c b/drivers/iio/pressure/zpa2326_i2c.c index 0db0860d386ba02ddaf348f46f2fd93f01f88c49..f26dd8cbb387c0e4e2783909aec526fc5445eda8 100644 --- a/drivers/iio/pressure/zpa2326_i2c.c +++ b/drivers/iio/pressure/zpa2326_i2c.c @@ -53,11 +53,9 @@ static int zpa2326_probe_i2c(struct i2c_client *client, zpa2326_i2c_hwid(client), regmap); } -static int zpa2326_remove_i2c(struct i2c_client *client) +static void zpa2326_remove_i2c(struct i2c_client *client) { zpa2326_remove(&client->dev); - - return 0; } static const struct i2c_device_id zpa2326_i2c_ids[] = { diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c index 648ae576d6fa8e4b3cf41f845d5500a4213007c8..791a33d5286c666f636b07ad5c47de5c48567f52 100644 --- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c +++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c @@ -311,7 +311,7 @@ error_unreg_buffer: return ret; } -static int lidar_remove(struct i2c_client *client) +static void lidar_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); @@ -320,8 +320,6 @@ static int lidar_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct i2c_device_id lidar_id[] = { diff --git a/drivers/iio/proximity/srf04.c b/drivers/iio/proximity/srf04.c index 05015351a34a853ea2e1b36e89fc98a184c808f1..faf2f806ce80415b42ad1ad056d905c40e141a1e 100644 --- a/drivers/iio/proximity/srf04.c +++ b/drivers/iio/proximity/srf04.c @@ -359,7 +359,7 @@ static int srf04_remove(struct platform_device *pdev) return 0; } -static int __maybe_unused srf04_pm_runtime_suspend(struct device *dev) +static int srf04_pm_runtime_suspend(struct device *dev) { struct platform_device *pdev = container_of(dev, struct platform_device, dev); @@ -371,7 +371,7 @@ static int __maybe_unused srf04_pm_runtime_suspend(struct device *dev) return 0; } -static int __maybe_unused srf04_pm_runtime_resume(struct device *dev) +static int srf04_pm_runtime_resume(struct device *dev) { struct platform_device *pdev = container_of(dev, struct platform_device, dev); @@ -385,8 +385,8 @@ static int __maybe_unused srf04_pm_runtime_resume(struct device *dev) } static const struct dev_pm_ops srf04_pm_ops = { - SET_RUNTIME_PM_OPS(srf04_pm_runtime_suspend, - srf04_pm_runtime_resume, NULL) + RUNTIME_PM_OPS(srf04_pm_runtime_suspend, + srf04_pm_runtime_resume, NULL) }; static struct platform_driver srf04_driver = { @@ -395,7 +395,7 @@ static struct platform_driver srf04_driver = { .driver = { .name = "srf04-gpio", .of_match_table = of_srf04_match, - .pm = &srf04_pm_ops, + .pm = pm_ptr(&srf04_pm_ops), }, }; diff --git a/drivers/iio/proximity/sx9310.c b/drivers/iio/proximity/sx9310.c index ea7318b508ea008b8f9d9deeb669847ee7625253..0e4747ccd3cf74c6c2cb7ebc64a4d80b730ba1aa 100644 --- a/drivers/iio/proximity/sx9310.c +++ b/drivers/iio/proximity/sx9310.c @@ -965,7 +965,7 @@ static int sx9310_probe(struct i2c_client *client) return sx_common_probe(client, &sx9310_chip_info, &sx9310_regmap_config); } -static int __maybe_unused sx9310_suspend(struct device *dev) +static int sx9310_suspend(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); u8 ctrl0; @@ -991,7 +991,7 @@ out: return ret; } -static int __maybe_unused sx9310_resume(struct device *dev) +static int sx9310_resume(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); int ret; @@ -1013,7 +1013,7 @@ out: return 0; } -static SIMPLE_DEV_PM_OPS(sx9310_pm_ops, sx9310_suspend, sx9310_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(sx9310_pm_ops, sx9310_suspend, sx9310_resume); static const struct acpi_device_id sx9310_acpi_match[] = { { "STH9310", SX9310_WHOAMI_VALUE }, @@ -1041,7 +1041,7 @@ static struct i2c_driver sx9310_driver = { .name = "sx9310", .acpi_match_table = sx9310_acpi_match, .of_match_table = sx9310_of_match, - .pm = &sx9310_pm_ops, + .pm = pm_sleep_ptr(&sx9310_pm_ops), /* * Lots of i2c transfers in probe + over 200 ms waiting in diff --git a/drivers/iio/proximity/sx9324.c b/drivers/iio/proximity/sx9324.c index edb5a2ce4e27542d16e9dcd824ecf9e2f35e106b..977cf17cec52b5809ee26d206570cc36292dc56f 100644 --- a/drivers/iio/proximity/sx9324.c +++ b/drivers/iio/proximity/sx9324.c @@ -1073,7 +1073,7 @@ static int sx9324_probe(struct i2c_client *client) return sx_common_probe(client, &sx9324_chip_info, &sx9324_regmap_config); } -static int __maybe_unused sx9324_suspend(struct device *dev) +static int sx9324_suspend(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); unsigned int regval; @@ -1098,7 +1098,7 @@ out: return ret; } -static int __maybe_unused sx9324_resume(struct device *dev) +static int sx9324_resume(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); int ret; @@ -1114,7 +1114,7 @@ static int __maybe_unused sx9324_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(sx9324_pm_ops, sx9324_suspend, sx9324_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(sx9324_pm_ops, sx9324_suspend, sx9324_resume); static const struct acpi_device_id sx9324_acpi_match[] = { { "STH9324", SX9324_WHOAMI_VALUE }, @@ -1139,7 +1139,7 @@ static struct i2c_driver sx9324_driver = { .name = "sx9324", .acpi_match_table = sx9324_acpi_match, .of_match_table = sx9324_of_match, - .pm = &sx9324_pm_ops, + .pm = pm_sleep_ptr(&sx9324_pm_ops), /* * Lots of i2c transfers in probe + over 200 ms waiting in diff --git a/drivers/iio/proximity/sx9360.c b/drivers/iio/proximity/sx9360.c index d9a12e6be6ca60ef4b9902b1079bd58e96171d83..7fa2213d23bafe01056f807ffe48a06b6095f419 100644 --- a/drivers/iio/proximity/sx9360.c +++ b/drivers/iio/proximity/sx9360.c @@ -819,7 +819,7 @@ static int sx9360_probe(struct i2c_client *client) return sx_common_probe(client, &sx9360_chip_info, &sx9360_regmap_config); } -static int __maybe_unused sx9360_suspend(struct device *dev) +static int sx9360_suspend(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); unsigned int regval; @@ -844,7 +844,7 @@ out: return ret; } -static int __maybe_unused sx9360_resume(struct device *dev) +static int sx9360_resume(struct device *dev) { struct sx_common_data *data = iio_priv(dev_get_drvdata(dev)); int ret; @@ -861,7 +861,7 @@ static int __maybe_unused sx9360_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(sx9360_pm_ops, sx9360_suspend, sx9360_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(sx9360_pm_ops, sx9360_suspend, sx9360_resume); static const struct acpi_device_id sx9360_acpi_match[] = { { "STH9360", SX9360_WHOAMI_VALUE }, @@ -886,7 +886,7 @@ static struct i2c_driver sx9360_driver = { .name = "sx9360", .acpi_match_table = sx9360_acpi_match, .of_match_table = sx9360_of_match, - .pm = &sx9360_pm_ops, + .pm = pm_sleep_ptr(&sx9360_pm_ops), /* * Lots of i2c transfers in probe + over 200 ms waiting in diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c index 42589d6200ad294f2f4e6516f7508e6e82f3e948..d4670864ddc7eb003602bcf8f16ae3a26a108fdb 100644 --- a/drivers/iio/proximity/sx9500.c +++ b/drivers/iio/proximity/sx9500.c @@ -979,7 +979,7 @@ out_trigger_unregister: return ret; } -static int sx9500_remove(struct i2c_client *client) +static void sx9500_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct sx9500_data *data = iio_priv(indio_dev); @@ -989,8 +989,6 @@ static int sx9500_remove(struct i2c_client *client) if (client->irq > 0) iio_trigger_unregister(data->trig); kfree(data->buffer); - - return 0; } static int sx9500_suspend(struct device *dev) diff --git a/drivers/iio/temperature/mlx90614.c b/drivers/iio/temperature/mlx90614.c index c253a53159881aa8a7ba720c7ec1382a5d82ec96..8eb0f962ed25cdd638aa079e5549e3c7f4c13ea7 100644 --- a/drivers/iio/temperature/mlx90614.c +++ b/drivers/iio/temperature/mlx90614.c @@ -79,16 +79,15 @@ struct mlx90614_data { /* Bandwidth values for IIR filtering */ static const int mlx90614_iir_values[] = {77, 31, 20, 15, 723, 153, 110, 86}; -static IIO_CONST_ATTR(in_temp_object_filter_low_pass_3db_frequency_available, - "0.15 0.20 0.31 0.77 0.86 1.10 1.53 7.23"); - -static struct attribute *mlx90614_attributes[] = { - &iio_const_attr_in_temp_object_filter_low_pass_3db_frequency_available.dev_attr.attr, - NULL, -}; - -static const struct attribute_group mlx90614_attr_group = { - .attrs = mlx90614_attributes, +static const int mlx90614_freqs[][2] = { + {0, 150000}, + {0, 200000}, + {0, 310000}, + {0, 770000}, + {0, 860000}, + {1, 100000}, + {1, 530000}, + {7, 230000} }; /* @@ -373,6 +372,22 @@ static int mlx90614_write_raw_get_fmt(struct iio_dev *indio_dev, } } +static int mlx90614_read_avail(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + const int **vals, int *type, int *length, + long mask) +{ + switch (mask) { + case IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY: + *vals = (int *)mlx90614_freqs; + *type = IIO_VAL_INT_PLUS_MICRO; + *length = 2 * ARRAY_SIZE(mlx90614_freqs); + return IIO_AVAIL_LIST; + default: + return -EINVAL; + } +} + static const struct iio_chan_spec mlx90614_channels[] = { { .type = IIO_TEMP, @@ -389,6 +404,8 @@ static const struct iio_chan_spec mlx90614_channels[] = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_CALIBEMISSIVITY) | BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + .info_mask_separate_available = + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_SCALE), }, @@ -401,6 +418,8 @@ static const struct iio_chan_spec mlx90614_channels[] = { .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | BIT(IIO_CHAN_INFO_CALIBEMISSIVITY) | BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), + .info_mask_separate_available = + BIT(IIO_CHAN_INFO_LOW_PASS_FILTER_3DB_FREQUENCY), .info_mask_shared_by_type = BIT(IIO_CHAN_INFO_OFFSET) | BIT(IIO_CHAN_INFO_SCALE), }, @@ -410,7 +429,7 @@ static const struct iio_info mlx90614_info = { .read_raw = mlx90614_read_raw, .write_raw = mlx90614_write_raw, .write_raw_get_fmt = mlx90614_write_raw_get_fmt, - .attrs = &mlx90614_attr_group, + .read_avail = mlx90614_read_avail, }; #ifdef CONFIG_PM @@ -571,7 +590,7 @@ static int mlx90614_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int mlx90614_remove(struct i2c_client *client) +static void mlx90614_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mlx90614_data *data = iio_priv(indio_dev); @@ -584,8 +603,6 @@ static int mlx90614_remove(struct i2c_client *client) mlx90614_sleep(data); pm_runtime_set_suspended(&client->dev); } - - return 0; } static const struct i2c_device_id mlx90614_id[] = { diff --git a/drivers/iio/temperature/mlx90632.c b/drivers/iio/temperature/mlx90632.c index 7ee7ff8047a4309b7ebe2edd44e159963bd2289d..f6dec0e5f0973c4c9414e6d97010b152c5bba54e 100644 --- a/drivers/iio/temperature/mlx90632.c +++ b/drivers/iio/temperature/mlx90632.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -128,6 +129,7 @@ * calculations * @object_ambient_temperature: Ambient temperature at object (might differ of * the ambient temperature of sensor. + * @regulator: Regulator of the device */ struct mlx90632_data { struct i2c_client *client; @@ -136,6 +138,7 @@ struct mlx90632_data { u16 emissivity; u8 mtyp; u32 object_ambient_temperature; + struct regulator *regulator; }; static const struct regmap_range mlx90632_volatile_reg_range[] = { @@ -207,6 +210,15 @@ static s32 mlx90632_pwr_continuous(struct regmap *regmap) MLX90632_PWR_STATUS_CONTINUOUS); } +/** + * mlx90632_reset_delay() - Give the mlx90632 some time to reset properly + * If this is not done, the following I2C command(s) will not be accepted. + */ +static void mlx90632_reset_delay(void) +{ + usleep_range(150, 200); +} + /** * mlx90632_perform_measurement() - Trigger and retrieve current measurement cycle * @data: pointer to mlx90632_data object containing regmap information @@ -248,11 +260,7 @@ static int mlx90632_set_meas_type(struct regmap *regmap, u8 type) if (ret < 0) return ret; - /* - * Give the mlx90632 some time to reset properly before sending a new I2C command - * if this is not done, the following I2C command(s) will not be accepted. - */ - usleep_range(150, 200); + mlx90632_reset_delay(); ret = regmap_write_bits(regmap, MLX90632_REG_CONTROL, (MLX90632_CFG_MTYP_MASK | MLX90632_CFG_PWR_MASK), @@ -841,6 +849,32 @@ static int mlx90632_wakeup(struct mlx90632_data *data) return mlx90632_pwr_continuous(data->regmap); } +static void mlx90632_disable_regulator(void *_data) +{ + struct mlx90632_data *data = _data; + int ret; + + ret = regulator_disable(data->regulator); + if (ret < 0) + dev_err(regmap_get_device(data->regmap), + "Failed to disable power regulator: %d\n", ret); +} + +static int mlx90632_enable_regulator(struct mlx90632_data *data) +{ + int ret; + + ret = regulator_enable(data->regulator); + if (ret < 0) { + dev_err(regmap_get_device(data->regmap), "Failed to enable power regulator!\n"); + return ret; + } + + mlx90632_reset_delay(); + + return ret; +} + static int mlx90632_probe(struct i2c_client *client, const struct i2c_device_id *id) { @@ -876,6 +910,23 @@ static int mlx90632_probe(struct i2c_client *client, indio_dev->channels = mlx90632_channels; indio_dev->num_channels = ARRAY_SIZE(mlx90632_channels); + mlx90632->regulator = devm_regulator_get(&client->dev, "vdd"); + if (IS_ERR(mlx90632->regulator)) + return dev_err_probe(&client->dev, PTR_ERR(mlx90632->regulator), + "failed to get vdd regulator"); + + ret = mlx90632_enable_regulator(mlx90632); + if (ret < 0) + return ret; + + ret = devm_add_action_or_reset(&client->dev, mlx90632_disable_regulator, + mlx90632); + if (ret < 0) { + dev_err(&client->dev, "Failed to setup regulator cleanup action %d\n", + ret); + return ret; + } + ret = mlx90632_wakeup(mlx90632); if (ret < 0) { dev_err(&client->dev, "Wakeup failed: %d\n", ret); @@ -924,7 +975,7 @@ static int mlx90632_probe(struct i2c_client *client, return iio_device_register(indio_dev); } -static int mlx90632_remove(struct i2c_client *client) +static void mlx90632_remove(struct i2c_client *client) { struct iio_dev *indio_dev = i2c_get_clientdata(client); struct mlx90632_data *data = iio_priv(indio_dev); @@ -936,8 +987,6 @@ static int mlx90632_remove(struct i2c_client *client) pm_runtime_put_noidle(&client->dev); mlx90632_sleep(data); - - return 0; } static const struct i2c_device_id mlx90632_id[] = { diff --git a/drivers/iio/test/iio-test-rescale.c b/drivers/iio/test/iio-test-rescale.c index cc782ccff8807da0ede18fc02d9e5e7ea9783047..31ee55a6faed92a1316d20fe34c0c78910042f0c 100644 --- a/drivers/iio/test/iio-test-rescale.c +++ b/drivers/iio/test/iio-test-rescale.c @@ -29,7 +29,7 @@ struct rescale_tc_data { const char *expected_off; }; -const struct rescale_tc_data scale_cases[] = { +static const struct rescale_tc_data scale_cases[] = { /* * Typical use cases */ @@ -477,7 +477,7 @@ const struct rescale_tc_data scale_cases[] = { }, }; -const struct rescale_tc_data offset_cases[] = { +static const struct rescale_tc_data offset_cases[] = { /* * Typical use cases */ diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c index b985e0d9bc05e91b10685f670c766aaee45673d0..1f9938a2c47522b8255239243edd26c7dacf402a 100644 --- a/drivers/infiniband/core/cm.c +++ b/drivers/infiniband/core/cm.c @@ -175,6 +175,7 @@ struct cm_device { struct cm_av { struct cm_port *port; struct rdma_ah_attr ah_attr; + u16 dlid_datapath; u16 pkey_index; u8 timeout; }; @@ -617,7 +618,6 @@ static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, struct rb_node *parent = NULL; struct cm_id_private *cur_cm_id_priv; __be64 service_id = cm_id_priv->id.service_id; - __be64 service_mask = cm_id_priv->id.service_mask; unsigned long flags; spin_lock_irqsave(&cm.lock, flags); @@ -625,9 +625,16 @@ static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, parent = *link; cur_cm_id_priv = rb_entry(parent, struct cm_id_private, service_node); - if ((cur_cm_id_priv->id.service_mask & service_id) == - (service_mask & cur_cm_id_priv->id.service_id) && - (cm_id_priv->id.device == cur_cm_id_priv->id.device)) { + + if (cm_id_priv->id.device < cur_cm_id_priv->id.device) + link = &(*link)->rb_left; + else if (cm_id_priv->id.device > cur_cm_id_priv->id.device) + link = &(*link)->rb_right; + else if (be64_lt(service_id, cur_cm_id_priv->id.service_id)) + link = &(*link)->rb_left; + else if (be64_gt(service_id, cur_cm_id_priv->id.service_id)) + link = &(*link)->rb_right; + else { /* * Sharing an ib_cm_id with different handlers is not * supported @@ -643,17 +650,6 @@ static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv, spin_unlock_irqrestore(&cm.lock, flags); return cur_cm_id_priv; } - - if (cm_id_priv->id.device < cur_cm_id_priv->id.device) - link = &(*link)->rb_left; - else if (cm_id_priv->id.device > cur_cm_id_priv->id.device) - link = &(*link)->rb_right; - else if (be64_lt(service_id, cur_cm_id_priv->id.service_id)) - link = &(*link)->rb_left; - else if (be64_gt(service_id, cur_cm_id_priv->id.service_id)) - link = &(*link)->rb_right; - else - link = &(*link)->rb_right; } cm_id_priv->listen_sharecount++; rb_link_node(&cm_id_priv->service_node, parent, link); @@ -670,12 +666,7 @@ static struct cm_id_private *cm_find_listen(struct ib_device *device, while (node) { cm_id_priv = rb_entry(node, struct cm_id_private, service_node); - if ((cm_id_priv->id.service_mask & service_id) == - cm_id_priv->id.service_id && - (cm_id_priv->id.device == device)) { - refcount_inc(&cm_id_priv->refcount); - return cm_id_priv; - } + if (device < cm_id_priv->id.device) node = node->rb_left; else if (device > cm_id_priv->id.device) @@ -684,8 +675,10 @@ static struct cm_id_private *cm_find_listen(struct ib_device *device, node = node->rb_left; else if (be64_gt(service_id, cm_id_priv->id.service_id)) node = node->rb_right; - else - node = node->rb_right; + else { + refcount_inc(&cm_id_priv->refcount); + return cm_id_priv; + } } return NULL; } @@ -1158,22 +1151,17 @@ void ib_destroy_cm_id(struct ib_cm_id *cm_id) } EXPORT_SYMBOL(ib_destroy_cm_id); -static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id, - __be64 service_mask) +static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id) { - service_mask = service_mask ? service_mask : ~cpu_to_be64(0); - service_id &= service_mask; if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID && (service_id != IB_CM_ASSIGN_SERVICE_ID)) return -EINVAL; - if (service_id == IB_CM_ASSIGN_SERVICE_ID) { + if (service_id == IB_CM_ASSIGN_SERVICE_ID) cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++); - cm_id_priv->id.service_mask = ~cpu_to_be64(0); - } else { + else cm_id_priv->id.service_id = service_id; - cm_id_priv->id.service_mask = service_mask; - } + return 0; } @@ -1185,12 +1173,8 @@ static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id, * and service ID resolution requests. The service ID should be specified * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will * assign a service ID to the caller. - * @service_mask: Mask applied to service ID used to listen across a - * range of service IDs. If set to 0, the service ID is matched - * exactly. This parameter is ignored if %service_id is set to - * IB_CM_ASSIGN_SERVICE_ID. */ -int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask) +int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id) { struct cm_id_private *cm_id_priv = container_of(cm_id, struct cm_id_private, id); @@ -1203,7 +1187,7 @@ int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask) goto out; } - ret = cm_init_listen(cm_id_priv, service_id, service_mask); + ret = cm_init_listen(cm_id_priv, service_id); if (ret) goto out; @@ -1251,7 +1235,7 @@ struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, if (IS_ERR(cm_id_priv)) return ERR_CAST(cm_id_priv); - err = cm_init_listen(cm_id_priv, service_id, 0); + err = cm_init_listen(cm_id_priv, service_id); if (err) { ib_destroy_cm_id(&cm_id_priv->id); return ERR_PTR(err); @@ -1321,6 +1305,7 @@ static void cm_format_req(struct cm_req_msg *req_msg, struct sa_path_rec *pri_path = param->primary_path; struct sa_path_rec *alt_path = param->alternate_path; bool pri_ext = false; + __be16 lid; if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA) pri_ext = opa_is_extended_lid(pri_path->opa.dlid, @@ -1380,9 +1365,16 @@ static void cm_format_req(struct cm_req_msg *req_msg, htons(ntohl(sa_path_get_dlid( pri_path))))); } else { + + if (param->primary_path_inbound) { + lid = param->primary_path_inbound->ib.dlid; + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(lid)); + } else + IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, + be16_to_cpu(IB_LID_PERMISSIVE)); + /* Work-around until there's a way to obtain remote LID info */ - IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg, - be16_to_cpu(IB_LID_PERMISSIVE)); IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg, be16_to_cpu(IB_LID_PERMISSIVE)); } @@ -1522,7 +1514,6 @@ int ib_send_cm_req(struct ib_cm_id *cm_id, } } cm_id->service_id = param->service_id; - cm_id->service_mask = ~cpu_to_be64(0); cm_id_priv->timeout_ms = cm_convert_to_ms( param->primary_path->packet_life_time) * 2 + cm_convert_to_ms( @@ -1538,6 +1529,10 @@ int ib_send_cm_req(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); cm_move_av_from_path(&cm_id_priv->av, &av); + if (param->primary_path_outbound) + cm_id_priv->av.dlid_datapath = + be16_to_cpu(param->primary_path_outbound->ib.dlid); + if (param->alternate_path) cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av); @@ -1632,14 +1627,13 @@ static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num, static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, - struct sa_path_rec *alt_path) + struct sa_path_rec *alt_path, + struct ib_wc *wc) { u32 lid; if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) { - sa_path_set_dlid(primary_path, - IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, - req_msg)); + sa_path_set_dlid(primary_path, wc->slid); sa_path_set_slid(primary_path, IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg)); @@ -1676,7 +1670,8 @@ static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg, static void cm_format_paths_from_req(struct cm_req_msg *req_msg, struct sa_path_rec *primary_path, - struct sa_path_rec *alt_path) + struct sa_path_rec *alt_path, + struct ib_wc *wc) { primary_path->dgid = *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg); @@ -1734,7 +1729,7 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg, if (sa_path_is_roce(alt_path)) alt_path->roce.route_resolved = false; } - cm_format_path_lid_from_req(req_msg, primary_path, alt_path); + cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc); } static u16 cm_get_bth_pkey(struct cm_work *work) @@ -2079,7 +2074,6 @@ static int cm_req_handler(struct cm_work *work) cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg)); cm_id_priv->id.service_id = cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)); - cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_id_priv->tid = req_msg->hdr.tid; cm_id_priv->timeout_ms = cm_convert_to_ms( IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg)); @@ -2148,7 +2142,7 @@ static int cm_req_handler(struct cm_work *work) if (cm_req_has_alt_path(req_msg)) work->path[1].rec_type = work->path[0].rec_type; cm_format_paths_from_req(req_msg, &work->path[0], - &work->path[1]); + &work->path[1], work->mad_recv_wc->wc); if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) sa_path_set_dmac(&work->path[0], cm_id_priv->av.ah_attr.roce.dmac); @@ -2173,6 +2167,10 @@ static int cm_req_handler(struct cm_work *work) NULL, 0); goto rejected; } + if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB) + cm_id_priv->av.dlid_datapath = + IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg); + if (cm_req_has_alt_path(req_msg)) { ret = cm_init_av_by_path(&work->path[1], NULL, &cm_id_priv->alt_av); @@ -3486,7 +3484,6 @@ int ib_send_cm_sidr_req(struct ib_cm_id *cm_id, spin_lock_irqsave(&cm_id_priv->lock, flags); cm_move_av_from_path(&cm_id_priv->av, &av); cm_id->service_id = param->service_id; - cm_id->service_mask = ~cpu_to_be64(0); cm_id_priv->timeout_ms = param->timeout_ms; cm_id_priv->max_cm_retries = param->max_cm_retries; if (cm_id->state != IB_CM_IDLE) { @@ -3561,7 +3558,6 @@ static int cm_sidr_req_handler(struct cm_work *work) cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg)); cm_id_priv->id.service_id = cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)); - cm_id_priv->id.service_mask = ~cpu_to_be64(0); cm_id_priv->tid = sidr_req_msg->hdr.tid; wc = work->mad_recv_wc->wc; @@ -4134,6 +4130,10 @@ static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv, *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU | IB_QP_DEST_QPN | IB_QP_RQ_PSN; qp_attr->ah_attr = cm_id_priv->av.ah_attr; + if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) && + cm_id_priv->av.dlid_datapath && + (cm_id_priv->av.dlid_datapath != 0xffff)) + qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath; qp_attr->path_mtu = cm_id_priv->path_mtu; qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn); qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn); diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 46d06678dfbebf935bccfc5c84456f5dcbd3c6ff..cc2222b85c88174a1b3b9c49beb05d701df78e2a 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -1841,8 +1841,8 @@ cma_ib_id_from_event(struct ib_cm_id *cm_id, } if (!validate_net_dev(*net_dev, - (struct sockaddr *)&req->listen_addr_storage, - (struct sockaddr *)&req->src_addr_storage)) { + (struct sockaddr *)&req->src_addr_storage, + (struct sockaddr *)&req->listen_addr_storage)) { id_priv = ERR_PTR(-EHOSTUNREACH); goto err; } @@ -2026,6 +2026,8 @@ static void _destroy_id(struct rdma_id_private *id_priv, cma_id_put(id_priv->id.context); kfree(id_priv->id.route.path_rec); + kfree(id_priv->id.route.path_rec_inbound); + kfree(id_priv->id.route.path_rec_outbound); put_net(id_priv->id.route.addr.dev_addr.net); kfree(id_priv); @@ -2241,14 +2243,14 @@ cma_ib_new_conn_id(const struct rdma_cm_id *listen_id, goto err; rt = &id->route; - rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; - rt->path_rec = kmalloc_array(rt->num_paths, sizeof(*rt->path_rec), - GFP_KERNEL); + rt->num_pri_alt_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; + rt->path_rec = kmalloc_array(rt->num_pri_alt_paths, + sizeof(*rt->path_rec), GFP_KERNEL); if (!rt->path_rec) goto err; rt->path_rec[0] = *path; - if (rt->num_paths == 2) + if (rt->num_pri_alt_paths == 2) rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; if (net_dev) { @@ -2817,26 +2819,72 @@ int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer) } EXPORT_SYMBOL(rdma_set_min_rnr_timer); +static void route_set_path_rec_inbound(struct cma_work *work, + struct sa_path_rec *path_rec) +{ + struct rdma_route *route = &work->id->id.route; + + if (!route->path_rec_inbound) { + route->path_rec_inbound = + kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL); + if (!route->path_rec_inbound) + return; + } + + *route->path_rec_inbound = *path_rec; +} + +static void route_set_path_rec_outbound(struct cma_work *work, + struct sa_path_rec *path_rec) +{ + struct rdma_route *route = &work->id->id.route; + + if (!route->path_rec_outbound) { + route->path_rec_outbound = + kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL); + if (!route->path_rec_outbound) + return; + } + + *route->path_rec_outbound = *path_rec; +} + static void cma_query_handler(int status, struct sa_path_rec *path_rec, - void *context) + int num_prs, void *context) { struct cma_work *work = context; struct rdma_route *route; + int i; route = &work->id->id.route; - if (!status) { - route->num_paths = 1; - *route->path_rec = *path_rec; - } else { - work->old_state = RDMA_CM_ROUTE_QUERY; - work->new_state = RDMA_CM_ADDR_RESOLVED; - work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; - work->event.status = status; - pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n", - status); + if (status) + goto fail; + + for (i = 0; i < num_prs; i++) { + if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP)) + *route->path_rec = path_rec[i]; + else if (path_rec[i].flags & IB_PATH_INBOUND) + route_set_path_rec_inbound(work, &path_rec[i]); + else if (path_rec[i].flags & IB_PATH_OUTBOUND) + route_set_path_rec_outbound(work, &path_rec[i]); } + if (!route->path_rec) { + status = -EINVAL; + goto fail; + } + + route->num_pri_alt_paths = 1; + queue_work(cma_wq, &work->work); + return; +fail: + work->old_state = RDMA_CM_ROUTE_QUERY; + work->new_state = RDMA_CM_ADDR_RESOLVED; + work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; + work->event.status = status; + pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n", + status); queue_work(cma_wq, &work->work); } @@ -3081,7 +3129,7 @@ int rdma_set_ib_path(struct rdma_cm_id *id, dev_put(ndev); } - id->route.num_paths = 1; + id->route.num_pri_alt_paths = 1; return 0; err_free: @@ -3214,7 +3262,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) goto err1; } - route->num_paths = 1; + route->num_pri_alt_paths = 1; ndev = cma_iboe_set_path_rec_l2_fields(id_priv); if (!ndev) { @@ -3274,7 +3322,7 @@ static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) err2: kfree(route->path_rec); route->path_rec = NULL; - route->num_paths = 0; + route->num_pri_alt_paths = 0; err1: kfree(work); return ret; @@ -3759,7 +3807,7 @@ static int cma_alloc_any_port(enum rdma_ucm_port_space ps, inet_get_local_port_range(net, &low, &high); remaining = (high - low) + 1; - rover = prandom_u32() % remaining + low; + rover = prandom_u32_max(remaining) + low; retry: if (last_used_port != rover) { struct rdma_bind_list *bind_list; @@ -4265,7 +4313,9 @@ static int cma_connect_ib(struct rdma_id_private *id_priv, } req.primary_path = &route->path_rec[0]; - if (route->num_paths == 2) + req.primary_path_inbound = route->path_rec_inbound; + req.primary_path_outbound = route->path_rec_outbound; + if (route->num_pri_alt_paths == 2) req.alternate_path = &route->path_rec[1]; req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr; diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c index de8a2d5d741ceb4e48f81e9d914d74a4f51efe02..7b68b3ea979f79dfd682062fa9f89f797742fdd9 100644 --- a/drivers/infiniband/core/cma_configfs.c +++ b/drivers/infiniband/core/cma_configfs.c @@ -292,7 +292,7 @@ static struct config_group *make_cma_dev(struct config_group *group, goto fail; } - strlcpy(cma_dev_group->name, name, sizeof(cma_dev_group->name)); + strscpy(cma_dev_group->name, name, sizeof(cma_dev_group->name)); config_group_init_type_name(&cma_dev_group->ports_group, "ports", &cma_ports_group_type); diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index d275db195f1a1add9779413708558db5badf2488..ae60c73babcc5a05ae0f3beedf38634eafc6f56c 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -422,7 +422,7 @@ int ib_device_rename(struct ib_device *ibdev, const char *name) return ret; } - strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); + strscpy(ibdev->name, name, IB_DEVICE_NAME_MAX); ret = rename_compat_devs(ibdev); downgrade_write(&devices_rwsem); @@ -1217,7 +1217,7 @@ static int assign_name(struct ib_device *device, const char *name) ret = -ENFILE; goto out; } - strlcpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); + strscpy(device->name, dev_name(&device->dev), IB_DEVICE_NAME_MAX); ret = xa_alloc_cyclic(&devices, &device->index, device, xa_limit_31b, &last_id, GFP_KERNEL); diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index 7063e41eaf26125ec6d0276699d050ca41b11c17..c77d7d2559a11d83251d91a23d6156b71fafd2ef 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -7,8 +7,7 @@ #include #include -static struct sk_buff *rdma_build_skb(struct ib_device *device, - struct net_device *netdev, +static struct sk_buff *rdma_build_skb(struct net_device *netdev, struct rdma_ah_attr *ah_attr, gfp_t flags) { @@ -86,7 +85,7 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device, struct net_device *slave; struct sk_buff *skb; - skb = rdma_build_skb(device, master, ah_attr, flags); + skb = rdma_build_skb(master, ah_attr, flags); if (!skb) return ERR_PTR(-ENOMEM); diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c index 003e504feca2ac56ba890f2b802f317643a401be..0de83d9a4985d89b3ebacafe7ffca655baa54807 100644 --- a/drivers/infiniband/core/sa_query.c +++ b/drivers/infiniband/core/sa_query.c @@ -50,6 +50,7 @@ #include #include #include +#include #include "sa.h" #include "core_priv.h" @@ -104,7 +105,8 @@ struct ib_sa_device { }; struct ib_sa_query { - void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *); + void (*callback)(struct ib_sa_query *sa_query, int status, + int num_prs, struct ib_sa_mad *mad); void (*release)(struct ib_sa_query *); struct ib_sa_client *client; struct ib_sa_port *port; @@ -116,6 +118,12 @@ struct ib_sa_query { u32 seq; /* Local svc request sequence number */ unsigned long timeout; /* Local svc timeout */ u8 path_use; /* How will the pathrecord be used */ + + /* A separate buffer to save pathrecords of a response, as in cases + * like IB/netlink, mulptiple pathrecords are supported, so that + * mad->data is not large enough to hold them + */ + void *resp_pr_data; }; #define IB_SA_ENABLE_LOCAL_SERVICE 0x00000001 @@ -123,7 +131,8 @@ struct ib_sa_query { #define IB_SA_QUERY_OPA 0x00000004 struct ib_sa_path_query { - void (*callback)(int, struct sa_path_rec *, void *); + void (*callback)(int status, struct sa_path_rec *rec, + int num_paths, void *context); void *context; struct ib_sa_query sa_query; struct sa_path_rec *conv_pr; @@ -712,7 +721,7 @@ static void ib_nl_set_path_rec_attrs(struct sk_buff *skb, if ((comp_mask & IB_SA_PATH_REC_REVERSIBLE) && sa_rec->reversible != 0) - query->path_use = LS_RESOLVE_PATH_USE_GMP; + query->path_use = LS_RESOLVE_PATH_USE_ALL; else query->path_use = LS_RESOLVE_PATH_USE_UNIDIRECTIONAL; header->path_use = query->path_use; @@ -865,50 +874,81 @@ static void send_handler(struct ib_mad_agent *agent, static void ib_nl_process_good_resolve_rsp(struct ib_sa_query *query, const struct nlmsghdr *nlh) { + struct ib_path_rec_data *srec, *drec; + struct ib_sa_path_query *path_query; struct ib_mad_send_wc mad_send_wc; - struct ib_sa_mad *mad = NULL; const struct nlattr *head, *curr; - struct ib_path_rec_data *rec; - int len, rem; + struct ib_sa_mad *mad = NULL; + int len, rem, num_prs = 0; u32 mask = 0; int status = -EIO; - if (query->callback) { - head = (const struct nlattr *) nlmsg_data(nlh); - len = nlmsg_len(nlh); - switch (query->path_use) { - case LS_RESOLVE_PATH_USE_UNIDIRECTIONAL: - mask = IB_PATH_PRIMARY | IB_PATH_OUTBOUND; - break; + if (!query->callback) + goto out; - case LS_RESOLVE_PATH_USE_ALL: - case LS_RESOLVE_PATH_USE_GMP: - default: - mask = IB_PATH_PRIMARY | IB_PATH_GMP | - IB_PATH_BIDIRECTIONAL; - break; + path_query = container_of(query, struct ib_sa_path_query, sa_query); + mad = query->mad_buf->mad; + if (!path_query->conv_pr && + (be16_to_cpu(mad->mad_hdr.attr_id) == IB_SA_ATTR_PATH_REC)) { + /* Need a larger buffer for possible multiple PRs */ + query->resp_pr_data = kvcalloc(RDMA_PRIMARY_PATH_MAX_REC_NUM, + sizeof(*drec), GFP_KERNEL); + if (!query->resp_pr_data) { + query->callback(query, -ENOMEM, 0, NULL); + return; } - nla_for_each_attr(curr, head, len, rem) { - if (curr->nla_type == LS_NLA_TYPE_PATH_RECORD) { - rec = nla_data(curr); - /* - * Get the first one. In the future, we may - * need to get up to 6 pathrecords. - */ - if ((rec->flags & mask) == mask) { - mad = query->mad_buf->mad; - mad->mad_hdr.method |= - IB_MGMT_METHOD_RESP; - memcpy(mad->data, rec->path_rec, - sizeof(rec->path_rec)); - status = 0; - break; - } - } + } + + head = (const struct nlattr *) nlmsg_data(nlh); + len = nlmsg_len(nlh); + switch (query->path_use) { + case LS_RESOLVE_PATH_USE_UNIDIRECTIONAL: + mask = IB_PATH_PRIMARY | IB_PATH_OUTBOUND; + break; + + case LS_RESOLVE_PATH_USE_ALL: + mask = IB_PATH_PRIMARY; + break; + + case LS_RESOLVE_PATH_USE_GMP: + default: + mask = IB_PATH_PRIMARY | IB_PATH_GMP | + IB_PATH_BIDIRECTIONAL; + break; + } + + drec = (struct ib_path_rec_data *)query->resp_pr_data; + nla_for_each_attr(curr, head, len, rem) { + if (curr->nla_type != LS_NLA_TYPE_PATH_RECORD) + continue; + + srec = nla_data(curr); + if ((srec->flags & mask) != mask) + continue; + + status = 0; + if (!drec) { + memcpy(mad->data, srec->path_rec, + sizeof(srec->path_rec)); + num_prs = 1; + break; } - query->callback(query, status, mad); + + memcpy(drec, srec, sizeof(*drec)); + drec++; + num_prs++; + if (num_prs >= RDMA_PRIMARY_PATH_MAX_REC_NUM) + break; } + if (!status) + mad->mad_hdr.method |= IB_MGMT_METHOD_RESP; + + query->callback(query, status, num_prs, mad); + kvfree(query->resp_pr_data); + query->resp_pr_data = NULL; + +out: mad_send_wc.send_buf = query->mad_buf; mad_send_wc.status = IB_WC_SUCCESS; send_handler(query->mad_buf->mad_agent, &mad_send_wc); @@ -1411,41 +1451,90 @@ static int opa_pr_query_possible(struct ib_sa_client *client, return PR_IB_SUPPORTED; } +static void ib_sa_pr_callback_single(struct ib_sa_path_query *query, + int status, struct ib_sa_mad *mad) +{ + struct sa_path_rec rec = {}; + + ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), + mad->data, &rec); + rec.rec_type = SA_PATH_REC_TYPE_IB; + sa_path_set_dmac_zero(&rec); + + if (query->conv_pr) { + struct sa_path_rec opa; + + memset(&opa, 0, sizeof(struct sa_path_rec)); + sa_convert_path_ib_to_opa(&opa, &rec); + query->callback(status, &opa, 1, query->context); + } else { + query->callback(status, &rec, 1, query->context); + } +} + +/** + * ib_sa_pr_callback_multiple() - Parse path records then do callback. + * + * In a multiple-PR case the PRs are saved in "query->resp_pr_data" + * (instead of"mad->data") and with "ib_path_rec_data" structure format, + * so that rec->flags can be set to indicate the type of PR. + * This is valid only in IB fabric. + */ +static void ib_sa_pr_callback_multiple(struct ib_sa_path_query *query, + int status, int num_prs, + struct ib_path_rec_data *rec_data) +{ + struct sa_path_rec *rec; + int i; + + rec = kvcalloc(num_prs, sizeof(*rec), GFP_KERNEL); + if (!rec) { + query->callback(-ENOMEM, NULL, 0, query->context); + return; + } + + for (i = 0; i < num_prs; i++) { + ib_unpack(path_rec_table, ARRAY_SIZE(path_rec_table), + rec_data[i].path_rec, rec + i); + rec[i].rec_type = SA_PATH_REC_TYPE_IB; + sa_path_set_dmac_zero(rec + i); + rec[i].flags = rec_data[i].flags; + } + + query->callback(status, rec, num_prs, query->context); + kvfree(rec); +} + static void ib_sa_path_rec_callback(struct ib_sa_query *sa_query, - int status, + int status, int num_prs, struct ib_sa_mad *mad) { struct ib_sa_path_query *query = container_of(sa_query, struct ib_sa_path_query, sa_query); + struct sa_path_rec rec; - if (mad) { - struct sa_path_rec rec; - - if (sa_query->flags & IB_SA_QUERY_OPA) { - ib_unpack(opa_path_rec_table, - ARRAY_SIZE(opa_path_rec_table), - mad->data, &rec); - rec.rec_type = SA_PATH_REC_TYPE_OPA; - query->callback(status, &rec, query->context); - } else { - ib_unpack(path_rec_table, - ARRAY_SIZE(path_rec_table), - mad->data, &rec); - rec.rec_type = SA_PATH_REC_TYPE_IB; - sa_path_set_dmac_zero(&rec); - - if (query->conv_pr) { - struct sa_path_rec opa; + if (!mad || !num_prs) { + query->callback(status, NULL, 0, query->context); + return; + } - memset(&opa, 0, sizeof(struct sa_path_rec)); - sa_convert_path_ib_to_opa(&opa, &rec); - query->callback(status, &opa, query->context); - } else { - query->callback(status, &rec, query->context); - } + if (sa_query->flags & IB_SA_QUERY_OPA) { + if (num_prs != 1) { + query->callback(-EINVAL, NULL, 0, query->context); + return; } - } else - query->callback(status, NULL, query->context); + + ib_unpack(opa_path_rec_table, ARRAY_SIZE(opa_path_rec_table), + mad->data, &rec); + rec.rec_type = SA_PATH_REC_TYPE_OPA; + query->callback(status, &rec, num_prs, query->context); + } else { + if (!sa_query->resp_pr_data) + ib_sa_pr_callback_single(query, status, mad); + else + ib_sa_pr_callback_multiple(query, status, num_prs, + sa_query->resp_pr_data); + } } static void ib_sa_path_rec_release(struct ib_sa_query *sa_query) @@ -1489,7 +1578,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, unsigned long timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct sa_path_rec *resp, - void *context), + int num_paths, void *context), void *context, struct ib_sa_query **sa_query) { @@ -1588,7 +1677,7 @@ err1: EXPORT_SYMBOL(ib_sa_path_rec_get); static void ib_sa_mcmember_rec_callback(struct ib_sa_query *sa_query, - int status, + int status, int num_prs, struct ib_sa_mad *mad) { struct ib_sa_mcmember_query *query = @@ -1680,7 +1769,7 @@ err1: /* Support GuidInfoRecord */ static void ib_sa_guidinfo_rec_callback(struct ib_sa_query *sa_query, - int status, + int status, int num_paths, struct ib_sa_mad *mad) { struct ib_sa_guidinfo_query *query = @@ -1790,7 +1879,7 @@ static void ib_classportinfo_cb(void *context) } static void ib_sa_classport_info_rec_callback(struct ib_sa_query *sa_query, - int status, + int status, int num_prs, struct ib_sa_mad *mad) { unsigned long flags; @@ -1966,13 +2055,13 @@ static void send_handler(struct ib_mad_agent *agent, /* No callback -- already got recv */ break; case IB_WC_RESP_TIMEOUT_ERR: - query->callback(query, -ETIMEDOUT, NULL); + query->callback(query, -ETIMEDOUT, 0, NULL); break; case IB_WC_WR_FLUSH_ERR: - query->callback(query, -EINTR, NULL); + query->callback(query, -EINTR, 0, NULL); break; default: - query->callback(query, -EIO, NULL); + query->callback(query, -EIO, 0, NULL); break; } @@ -2000,10 +2089,10 @@ static void recv_handler(struct ib_mad_agent *mad_agent, if (mad_recv_wc->wc->status == IB_WC_SUCCESS) query->callback(query, mad_recv_wc->recv_buf.mad->mad_hdr.status ? - -EINVAL : 0, + -EINVAL : 0, 1, (struct ib_sa_mad *) mad_recv_wc->recv_buf.mad); else - query->callback(query, -EIO, NULL); + query->callback(query, -EIO, 0, NULL); } ib_free_recv_mad(mad_recv_wc); diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index 9d6ac9dff39a28bf865558420d2bb1e106bc52e6..bf42650f125b11e565f465f1323614c84ac6bfa4 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -754,8 +754,8 @@ static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp, { struct rdma_dev_addr *dev_addr; - resp->num_paths = route->num_paths; - switch (route->num_paths) { + resp->num_paths = route->num_pri_alt_paths; + switch (route->num_pri_alt_paths) { case 0: dev_addr = &route->addr.dev_addr; rdma_addr_get_dgid(dev_addr, @@ -781,8 +781,8 @@ static void ucma_copy_iboe_route(struct rdma_ucm_query_route_resp *resp, struct rdma_route *route) { - resp->num_paths = route->num_paths; - switch (route->num_paths) { + resp->num_paths = route->num_pri_alt_paths; + switch (route->num_pri_alt_paths) { case 0: rdma_ip2gid((struct sockaddr *)&route->addr.dst_addr, (union ib_gid *)&resp->ib_route[0].dgid); @@ -921,7 +921,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx, if (!resp) return -ENOMEM; - resp->num_paths = ctx->cm_id->route.num_paths; + resp->num_paths = ctx->cm_id->route.num_pri_alt_paths; for (i = 0, out_len -= sizeof(*resp); i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data); i++, out_len -= sizeof(struct ib_path_rec_data)) { diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index 186ed8859920c7be2ad646d982284baf62d2d5a0..e9fa22d31c233204ccd8b7decc7353b3629bad3c 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -43,8 +43,6 @@ #include #include -#include -#include #include #include "uverbs.h" @@ -462,7 +460,7 @@ retry: mutex_unlock(&umem_odp->umem_mutex); out_put_mm: - mmput(owning_mm); + mmput_async(owning_mm); out_put_task: if (owning_process) put_task_struct(owning_process); diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index 046376bd68e27df243ffcb535bc0073e37cc4e97..4796f6a8828caa47ba0f1b980cdee3b1ec1fb97f 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -739,6 +739,7 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs) mr->uobject = uobj; atomic_inc(&pd->usecnt); mr->iova = cmd.hca_va; + mr->length = cmd.length; rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); rdma_restrack_set_name(&mr->res, NULL); @@ -861,8 +862,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) mr->pd = new_pd; atomic_inc(&new_pd->usecnt); } - if (cmd.flags & IB_MR_REREG_TRANS) + if (cmd.flags & IB_MR_REREG_TRANS) { mr->iova = cmd.hca_va; + mr->length = cmd.length; + } } memset(&resp, 0, sizeof(resp)); diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c index 990f0724acc6b660ef2dfc2adeeb00b510e191f3..d9799706c58e997c225b1e661307eb7e21afd420 100644 --- a/drivers/infiniband/core/uverbs_ioctl.c +++ b/drivers/infiniband/core/uverbs_ioctl.c @@ -337,6 +337,14 @@ static int uverbs_process_attr(struct bundle_priv *pbundle, break; + case UVERBS_ATTR_TYPE_RAW_FD: + if (uattr->attr_data.reserved || uattr->len != 0 || + uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX) + return -EINVAL; + /* _uverbs_get_const_signed() is the accessor */ + e->ptr_attr.data = uattr->data_s64; + break; + case UVERBS_ATTR_TYPE_IDRS_ARRAY: return uverbs_process_idrs_array(pbundle, attr_uapi, &e->objs_arr_attr, uattr, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index e54b3f1b730e00515050a400c181dd20a79e2483..26b021f43ba40fbe930c95cdbd3fab30e5765eb8 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1038,7 +1038,7 @@ struct ib_srq *ib_create_srq_user(struct ib_pd *pd, ret = pd->device->ops.create_srq(srq, srq_init_attr, udata); if (ret) { rdma_restrack_put(&srq->res); - atomic_dec(&srq->pd->usecnt); + atomic_dec(&pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) @@ -2149,6 +2149,8 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, mr->pd = pd; mr->dm = NULL; atomic_inc(&pd->usecnt); + mr->iova = virt_addr; + mr->length = length; rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR); rdma_restrack_parent_name(&mr->res, &pd->res); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 3d6834d3d4fb2016b8beacd87e08c75d5720ab6d..8c0c80a8d338468e51cc7d844a21c69c8d379494 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -725,7 +725,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) /* ib device init */ ibdev->node_type = RDMA_NODE_IB_CA; - strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", + strscpy(ibdev->node_desc, BNXT_RE_DESC " HCA", strlen(BNXT_RE_DESC) + 5); ibdev->phys_port_cnt = 1; diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 14392c942f4928945a91cdd10a15b8e9704299d1..499a425a33791472365b5b1192d41b98fd2bbebb 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -734,7 +734,7 @@ static int send_connect(struct c4iw_ep *ep) &ep->com.remote_addr; int ret; enum chip_type adapter_type = ep->com.dev->rdev.lldi.adapter_type; - u32 isn = (prandom_u32() & ~7UL) - 1; + u32 isn = (get_random_u32() & ~7UL) - 1; struct net_device *netdev; u64 params; @@ -2469,7 +2469,7 @@ static int accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, } if (!is_t4(adapter_type)) { - u32 isn = (prandom_u32() & ~7UL) - 1; + u32 isn = (get_random_u32() & ~7UL) - 1; skb = get_skb(skb, roundup(sizeof(*rpl5), 16), GFP_KERNEL); rpl5 = __skb_put_zero(skb, roundup(sizeof(*rpl5), 16)); diff --git a/drivers/infiniband/hw/cxgb4/id_table.c b/drivers/infiniband/hw/cxgb4/id_table.c index f64e7e02b129f1a4bced436cd4fc570d51a986c5..280d61466855676dfd17df8b2ee154c01e62810e 100644 --- a/drivers/infiniband/hw/cxgb4/id_table.c +++ b/drivers/infiniband/hw/cxgb4/id_table.c @@ -54,7 +54,7 @@ u32 c4iw_id_alloc(struct c4iw_id_table *alloc) if (obj < alloc->max) { if (alloc->flags & C4IW_ID_TABLE_F_RANDOM) - alloc->last += prandom_u32() % RANDOM_SKIP; + alloc->last += prandom_u32_max(RANDOM_SKIP); else alloc->last = obj + 1; if (alloc->last >= alloc->max) @@ -85,7 +85,7 @@ int c4iw_id_table_alloc(struct c4iw_id_table *alloc, u32 start, u32 num, alloc->start = start; alloc->flags = flags; if (flags & C4IW_ID_TABLE_F_RANDOM) - alloc->last = prandom_u32() % RANDOM_SKIP; + alloc->last = prandom_u32_max(RANDOM_SKIP); else alloc->last = 0; alloc->max = num; diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index 0b0b93b529f375c79c8690f6207bad19d76379ef..d4b9226088bd02ac81c464663e402d3ff1ee2f2c 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -444,7 +444,10 @@ struct efa_admin_create_cq_cmd { /* * 4:0 : cq_entry_size_words - size of CQ entry in * 32-bit words, valid values: 4, 8. - * 7:5 : reserved7 - MBZ + * 5 : set_src_addr - If set, source address will be + * filled on RX completions from unknown senders. + * Requires 8 words CQ entry size. + * 7:6 : reserved7 - MBZ */ u8 cq_caps_2; @@ -980,6 +983,7 @@ struct efa_admin_host_info { #define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5) #define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK BIT(6) #define EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK GENMASK(4, 0) +#define EFA_ADMIN_CREATE_CQ_CMD_SET_SRC_ADDR_MASK BIT(5) /* create_cq_resp */ #define EFA_ADMIN_CREATE_CQ_RESP_DB_VALID_MASK BIT(0) diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index fb405da4e1db9c031f9eb6c36203fd3113bd3d42..8f8885e002ba4b9581be6625f744b6ffd6d0642f 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -168,7 +168,10 @@ int efa_com_create_cq(struct efa_com_dev *edev, EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED, 1); create_cmd.eqn = params->eqn; } - + if (params->set_src_addr) { + EFA_SET(&create_cmd.cq_caps_2, + EFA_ADMIN_CREATE_CQ_CMD_SET_SRC_ADDR, 1); + } efa_com_set_dma_addr(params->dma_addr, &create_cmd.cq_ba.mem_addr_high, &create_cmd.cq_ba.mem_addr_low); diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index c33010bbf9e830515832111bc76a4238b9a4a46c..0898ad5bc340526a0f246ab2008c5b9587571f28 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -75,7 +75,8 @@ struct efa_com_create_cq_params { u16 uarn; u16 eqn; u8 entry_size_in_bytes; - bool interrupt_mode_enabled; + u8 interrupt_mode_enabled : 1; + u8 set_src_addr : 1; }; struct efa_com_create_cq_result { diff --git a/drivers/infiniband/hw/efa/efa_io_defs.h b/drivers/infiniband/hw/efa/efa_io_defs.h new file mode 100644 index 0000000000000000000000000000000000000000..17ba8984b11e9f78e51a5d99f2923679124d564a --- /dev/null +++ b/drivers/infiniband/hw/efa/efa_io_defs.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved. + */ + +#ifndef _EFA_IO_H_ +#define _EFA_IO_H_ + +#define EFA_IO_TX_DESC_NUM_BUFS 2 +#define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1 +#define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32 +#define EFA_IO_TX_DESC_IMM_DATA_SIZE 4 + +enum efa_io_queue_type { + /* send queue (of a QP) */ + EFA_IO_SEND_QUEUE = 1, + /* recv queue (of a QP) */ + EFA_IO_RECV_QUEUE = 2, +}; + +enum efa_io_send_op_type { + /* send message */ + EFA_IO_SEND = 0, + /* RDMA read */ + EFA_IO_RDMA_READ = 1, +}; + +enum efa_io_comp_status { + /* Successful completion */ + EFA_IO_COMP_STATUS_OK = 0, + /* Flushed during QP destroy */ + EFA_IO_COMP_STATUS_FLUSHED = 1, + /* Internal QP error */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2, + /* Bad operation type */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_OP_TYPE = 3, + /* Bad AH */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4, + /* LKEY not registered or does not match IOVA */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5, + /* Message too long */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6, + /* Destination ENI is down or does not run EFA */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7, + /* Connection was reset by remote side */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8, + /* Bad dest QP number (QP does not exist or is in error state) */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9, + /* Destination resource not ready (no WQEs posted on RQ) */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10, + /* Receiver SGL too short */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11, + /* Unexpected status returned by responder */ + EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12, + /* Unresponsive remote - detected locally */ + EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13, +}; + +struct efa_io_tx_meta_desc { + /* Verbs-generated Request ID */ + u16 req_id; + + /* + * control flags + * 3:0 : op_type - operation type: send/rdma/fast mem + * ops/etc + * 4 : has_imm - immediate_data field carries valid + * data. + * 5 : inline_msg - inline mode - inline message data + * follows this descriptor (no buffer descriptors). + * Note that it is different from immediate data + * 6 : meta_extension - Extended metadata. MBZ + * 7 : meta_desc - Indicates metadata descriptor. + * Must be set. + */ + u8 ctrl1; + + /* + * control flags + * 0 : phase + * 1 : reserved25 - MBZ + * 2 : first - Indicates first descriptor in + * transaction. Must be set. + * 3 : last - Indicates last descriptor in + * transaction. Must be set. + * 4 : comp_req - Indicates whether completion should + * be posted, after packet is transmitted. Valid only + * for the first descriptor + * 7:5 : reserved29 - MBZ + */ + u8 ctrl2; + + u16 dest_qp_num; + + /* + * If inline_msg bit is set, length of inline message in bytes, + * otherwise length of SGL (number of buffers). + */ + u16 length; + + /* + * immediate data: if has_imm is set, then this field is included + * within Tx message and reported in remote Rx completion. + */ + u32 immediate_data; + + u16 ah; + + u16 reserved; + + /* Queue key */ + u32 qkey; + + u8 reserved2[12]; +}; + +/* + * Tx queue buffer descriptor, for any transport type. Preceded by metadata + * descriptor. + */ +struct efa_io_tx_buf_desc { + /* length in bytes */ + u32 length; + + /* + * 23:0 : lkey - local memory translation key + * 31:24 : reserved - MBZ + */ + u32 lkey; + + /* Buffer address bits[31:0] */ + u32 buf_addr_lo; + + /* Buffer address bits[63:32] */ + u32 buf_addr_hi; +}; + +struct efa_io_remote_mem_addr { + /* length in bytes */ + u32 length; + + /* remote memory translation key */ + u32 rkey; + + /* Buffer address bits[31:0] */ + u32 buf_addr_lo; + + /* Buffer address bits[63:32] */ + u32 buf_addr_hi; +}; + +struct efa_io_rdma_req { + /* Remote memory address */ + struct efa_io_remote_mem_addr remote_mem; + + /* Local memory address */ + struct efa_io_tx_buf_desc local_mem[1]; +}; + +/* + * Tx WQE, composed of tx meta descriptors followed by either tx buffer + * descriptors or inline data + */ +struct efa_io_tx_wqe { + /* TX meta */ + struct efa_io_tx_meta_desc meta; + + union { + /* Send buffer descriptors */ + struct efa_io_tx_buf_desc sgl[2]; + + u8 inline_data[32]; + + /* RDMA local and remote memory addresses */ + struct efa_io_rdma_req rdma_req; + } data; +}; + +/* + * Rx buffer descriptor; RX WQE is composed of one or more RX buffer + * descriptors. + */ +struct efa_io_rx_desc { + /* Buffer address bits[31:0] */ + u32 buf_addr_lo; + + /* Buffer Pointer[63:32] */ + u32 buf_addr_hi; + + /* Verbs-generated request id. */ + u16 req_id; + + /* Length in bytes. */ + u16 length; + + /* + * LKey and control flags + * 23:0 : lkey + * 29:24 : reserved - MBZ + * 30 : first - Indicates first descriptor in WQE + * 31 : last - Indicates last descriptor in WQE + */ + u32 lkey_ctrl; +}; + +/* Common IO completion descriptor */ +struct efa_io_cdesc_common { + /* + * verbs-generated request ID, as provided in the completed tx or rx + * descriptor. + */ + u16 req_id; + + u8 status; + + /* + * flags + * 0 : phase - Phase bit + * 2:1 : q_type - enum efa_io_queue_type: send/recv + * 3 : has_imm - indicates that immediate data is + * present - for RX completions only + * 7:4 : reserved28 - MBZ + */ + u8 flags; + + /* local QP number */ + u16 qp_num; + + /* Transferred length */ + u16 length; +}; + +/* Tx completion descriptor */ +struct efa_io_tx_cdesc { + /* Common completion info */ + struct efa_io_cdesc_common common; +}; + +/* Rx Completion Descriptor */ +struct efa_io_rx_cdesc { + /* Common completion info */ + struct efa_io_cdesc_common common; + + /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */ + u16 ah; + + u16 src_qp_num; + + /* Immediate data */ + u32 imm; +}; + +/* Extended Rx Completion Descriptor */ +struct efa_io_rx_cdesc_ex { + /* Base RX completion info */ + struct efa_io_rx_cdesc rx_cdesc_base; + + /* + * Valid only in case of unknown AH (0xFFFF) and CQ set_src_addr is + * enabled. + */ + u8 src_addr[16]; +}; + +/* tx_meta_desc */ +#define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0) +#define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4) +#define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5) +#define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6) +#define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7) +#define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0) +#define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2) +#define EFA_IO_TX_META_DESC_LAST_MASK BIT(3) +#define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4) + +/* tx_buf_desc */ +#define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0) + +/* rx_desc */ +#define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0) +#define EFA_IO_RX_DESC_FIRST_MASK BIT(30) +#define EFA_IO_RX_DESC_LAST_MASK BIT(31) + +/* cdesc_common */ +#define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0) +#define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1) +#define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3) + +#endif /* _EFA_IO_H_ */ diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index ecfe70eb5efbebf3cef26f3844ed0dcf8c4b4ebd..31454643f8c54fdc799ce095a46583a48f1c242e 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* - * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include @@ -15,6 +15,7 @@ #include #include "efa.h" +#include "efa_io_defs.h" enum { EFA_MMAP_DMA_PAGE = 0, @@ -242,6 +243,7 @@ int efa_query_device(struct ib_device *ibdev, resp.max_rq_wr = dev_attr->max_rq_depth; resp.max_rdma_size = dev_attr->max_rdma_size; + resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID; if (EFA_DEV_CAP(dev, RDMA_READ)) resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ; @@ -1064,6 +1066,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct efa_ibv_create_cq cmd = {}; struct efa_cq *cq = to_ecq(ibcq); int entries = attr->cqe; + bool set_src_addr; int err; ibdev_dbg(ibdev, "create_cq entries %d\n", entries); @@ -1109,7 +1112,10 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, goto err_out; } - if (!cmd.cq_entry_size) { + set_src_addr = !!(cmd.flags & EFA_CREATE_CQ_WITH_SGID); + if ((cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc_ex)) && + (set_src_addr || + cmd.cq_entry_size != sizeof(struct efa_io_rx_cdesc))) { ibdev_dbg(ibdev, "Invalid entry size [%u]\n", cmd.cq_entry_size); err = -EINVAL; @@ -1138,6 +1144,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, params.dma_addr = cq->dma_addr; params.entry_size_in_bytes = cmd.cq_entry_size; params.num_sub_cqs = cmd.num_sub_cqs; + params.set_src_addr = set_src_addr; if (cmd.flags & EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL) { cq->eq = efa_vec2eq(dev, attr->comp_vector); params.eqn = cq->eq->eeq.eqn; diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h index 2aae635c1c8da815621f6982e33d42b80de8fe69..730783fbc8949217c7e4af3ebfc170411145495b 100644 --- a/drivers/infiniband/hw/erdma/erdma.h +++ b/drivers/infiniband/hw/erdma/erdma.h @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -196,6 +197,7 @@ struct erdma_dev { struct erdma_devattr attrs; /* physical port state (only one port per device) */ enum ib_port_state state; + u32 mtu; /* cmdq and aeq use the same msix vector */ struct erdma_irq comm_irq; @@ -269,7 +271,7 @@ void erdma_finish_cmdq_init(struct erdma_dev *dev); void erdma_cmdq_destroy(struct erdma_dev *dev); void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op); -int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, u64 *req, u32 req_size, +int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size, u64 *resp0, u64 *resp1); void erdma_cmdq_completion_handler(struct erdma_cmdq *cmdq); diff --git a/drivers/infiniband/hw/erdma/erdma_cm.c b/drivers/infiniband/hw/erdma/erdma_cm.c index f13f16479ecadf3f7e5cd9774c787fe507bc4988..74f6348f240ac424109f6de509779a5a0b23569d 100644 --- a/drivers/infiniband/hw/erdma/erdma_cm.c +++ b/drivers/infiniband/hw/erdma/erdma_cm.c @@ -10,15 +10,7 @@ /* Copyright (c) 2008-2019, IBM Corporation */ /* Copyright (c) 2017, Open Grid Computing, Inc. */ -#include -#include -#include -#include #include -#include - -#include -#include #include "erdma.h" #include "erdma_cm.h" diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c index 57da0c670472050e7f7761d91b445779ed0606d0..6ebfa6989b11ebf4ba2f34182e78612af7fa9d21 100644 --- a/drivers/infiniband/hw/erdma/erdma_cmdq.c +++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c @@ -4,13 +4,7 @@ /* Kai Shen */ /* Copyright (c) 2020-2022, Alibaba Group. */ -#include -#include -#include - #include "erdma.h" -#include "erdma_hw.h" -#include "erdma_verbs.h" static void arm_cmdq_cq(struct erdma_cmdq *cmdq) { @@ -441,7 +435,7 @@ void erdma_cmdq_build_reqhdr(u64 *hdr, u32 mod, u32 op) FIELD_PREP(ERDMA_CMD_HDR_OPCODE_MASK, op); } -int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, u64 *req, u32 req_size, +int erdma_post_cmd_wait(struct erdma_cmdq *cmdq, void *req, u32 req_size, u64 *resp0, u64 *resp1) { struct erdma_comp_wait *comp_wait; diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index 751c7f9f0de70ddeeabaee39f06818ac226d49bd..58e0dc5c75d1d26ae30c1536b7576d8fafca251a 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -4,9 +4,6 @@ /* Kai Shen */ /* Copyright (c) 2020-2022, Alibaba Group. */ -#include - -#include "erdma_hw.h" #include "erdma_verbs.h" static void *get_next_valid_cqe(struct erdma_cq *cq) @@ -62,7 +59,6 @@ static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = { [ERDMA_OP_RECV_IMM] = IB_WC_RECV_RDMA_WITH_IMM, [ERDMA_OP_RECV_INV] = IB_WC_RECV, [ERDMA_OP_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, - [ERDMA_OP_INVALIDATE] = IB_WC_LOCAL_INV, [ERDMA_OP_RSP_SEND_IMM] = IB_WC_RECV, [ERDMA_OP_SEND_WITH_INV] = IB_WC_SEND, [ERDMA_OP_REG_MR] = IB_WC_REG_MR, diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index 8f2d094e02279c7f01835b9fc1c7121ce9af372b..ed54130d924b30914bc7bf408beea37a26480f65 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -4,12 +4,6 @@ /* Kai Shen */ /* Copyright (c) 2020-2022, Alibaba Group. */ -#include -#include -#include - -#include "erdma.h" -#include "erdma_hw.h" #include "erdma_verbs.h" #define MAX_POLL_CHUNK_SIZE 16 @@ -229,9 +223,7 @@ static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) req.db_dma_addr_l = lower_32_bits(db_info_dma_addr); req.db_dma_addr_h = upper_32_bits(db_info_dma_addr); - return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, - sizeof(struct erdma_cmdq_create_eq_req), - NULL, NULL); + return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) @@ -281,8 +273,7 @@ static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) req.qtype = ERDMA_EQ_TYPE_CEQ; req.vector_idx = ceqn + 1; - err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); if (err) return; diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h index b210c49c669fcab94f853cff3d80a8b170606225..e788887732e1f5e62185cc244d675ae99cebb1f4 100644 --- a/drivers/infiniband/hw/erdma/erdma_hw.h +++ b/drivers/infiniband/hw/erdma/erdma_hw.h @@ -153,6 +153,7 @@ enum CMDQ_COMMON_OPCODE { CMDQ_OPCODE_CREATE_EQ = 0, CMDQ_OPCODE_DESTROY_EQ = 1, CMDQ_OPCODE_QUERY_FW_INFO = 2, + CMDQ_OPCODE_CONF_MTU = 3, }; /* cmdq-SQE HDR */ @@ -190,6 +191,11 @@ struct erdma_cmdq_destroy_eq_req { u8 qtype; }; +struct erdma_cmdq_config_mtu_req { + u64 hdr; + u32 mtu; +}; + /* create_cq cfg0 */ #define ERDMA_CMD_CREATE_CQ_DEPTH_MASK GENMASK(31, 24) #define ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK GENMASK(23, 20) @@ -450,13 +456,13 @@ enum erdma_opcode { ERDMA_OP_RECV_IMM = 5, ERDMA_OP_RECV_INV = 6, - ERDMA_OP_REQ_ERR = 7, - ERDMA_OP_READ_RESPONSE = 8, + ERDMA_OP_RSVD0 = 7, + ERDMA_OP_RSVD1 = 8, ERDMA_OP_WRITE_WITH_IMM = 9, - ERDMA_OP_RECV_ERR = 10, + ERDMA_OP_RSVD2 = 10, + ERDMA_OP_RSVD3 = 11, - ERDMA_OP_INVALIDATE = 11, ERDMA_OP_RSP_SEND_IMM = 12, ERDMA_OP_SEND_WITH_INV = 13, diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 07e743d248470f0c5c5bdd01a3cfd9defe318c6f..49778bb294ae4386095bddf08149454f70875d07 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -4,21 +4,12 @@ /* Kai Shen */ /* Copyright (c) 2020-2022, Alibaba Group. */ -#include -#include -#include -#include #include -#include -#include #include #include -#include -#include #include "erdma.h" #include "erdma_cm.h" -#include "erdma_hw.h" #include "erdma_verbs.h" MODULE_AUTHOR("Cheng Xu "); @@ -43,10 +34,15 @@ static int erdma_netdev_event(struct notifier_block *nb, unsigned long event, dev->state = IB_PORT_DOWN; erdma_port_event(dev, IB_EVENT_PORT_ERR); break; + case NETDEV_CHANGEMTU: + if (dev->mtu != netdev->mtu) { + erdma_set_mtu(dev, netdev->mtu); + dev->mtu = netdev->mtu; + } + break; case NETDEV_REGISTER: case NETDEV_UNREGISTER: case NETDEV_CHANGEADDR: - case NETDEV_CHANGEMTU: case NETDEV_GOING_DOWN: case NETDEV_CHANGE: default: @@ -104,6 +100,7 @@ static int erdma_device_register(struct erdma_dev *dev) if (ret) return ret; + dev->mtu = dev->netdev->mtu; addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr); ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev); diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index bc3ec22a62c57217874f14d5787f764971997af0..5fe1a339a43543904f528dacbec6d748d70498e3 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -6,15 +6,6 @@ /* Authors: Bernard Metzler */ /* Copyright (c) 2008-2019, IBM Corporation */ -#include -#include -#include -#include - -#include -#include - -#include "erdma.h" #include "erdma_cm.h" #include "erdma_verbs.h" @@ -105,8 +96,7 @@ static int erdma_modify_qp_state_to_rts(struct erdma_qp *qp, req.send_nxt += MPA_DEFAULT_HDR_LEN + qp->attrs.pd_len; req.recv_nxt = tp->rcv_nxt; - return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp, @@ -124,8 +114,7 @@ static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp, req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, attrs->state) | FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp)); - return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs, diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 699bd3f59cd340fa597b37416880d2eede547442..62be98e2b94142f916f25fc9b7b602a59554deaa 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -9,21 +9,14 @@ /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ -#include -#include -#include -#include #include #include #include #include -#include -#include #include #include "erdma.h" #include "erdma_cm.h" -#include "erdma_hw.h" #include "erdma_verbs.h" static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp) @@ -102,7 +95,7 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp) req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; } - err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), &resp0, + err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1); if (!err) qp->attrs.cookie = @@ -151,8 +144,7 @@ static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) } post_cmd: - return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq) @@ -202,8 +194,7 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq) req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; } - return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); } static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) @@ -976,8 +967,7 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); - ret = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); if (ret) return ret; @@ -1002,8 +992,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) CMDQ_OPCODE_DESTROY_CQ); req.cqn = cq->cqn; - err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); if (err) return err; @@ -1040,8 +1029,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) CMDQ_OPCODE_DESTROY_QP); req.qpn = QP_ID(qp); - err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, - NULL); + err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); if (err) return err; @@ -1448,6 +1436,17 @@ err_out_xa: return ret; } +void erdma_set_mtu(struct erdma_dev *dev, u32 mtu) +{ + struct erdma_cmdq_config_mtu_req req; + + erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, + CMDQ_OPCODE_CONF_MTU); + req.mtu = mtu; + + erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); +} + void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) { struct ib_event event; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index c7baddb1f292db11f7d7c93ac47fb172f741d850..ab6380635e9e64b35f6f757df671ed9bb19af13e 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -7,15 +7,7 @@ #ifndef __ERDMA_VERBS_H__ #define __ERDMA_VERBS_H__ -#include - -#include -#include -#include - #include "erdma.h" -#include "erdma_cm.h" -#include "erdma_hw.h" /* RDMA Capability. */ #define ERDMA_MAX_PD (128 * 1024) @@ -338,5 +330,6 @@ struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason); +void erdma_set_mtu(struct erdma_dev *dev, u32 mtu); #endif diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index f1245c94ae2629d329d68e19349360b37bf7f725..ebe970f76232dc8f03e5d1bf3262f4f7034e55eb 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -8753,7 +8753,7 @@ static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, /* * When writing a LCB CSR, out_data contains the full value to - * to be written, while in_data contains the relative LCB + * be written, while in_data contains the relative LCB * address in 7:0. Do the work here, rather than the caller, * of distrubting the write data to where it needs to go: * diff --git a/drivers/infiniband/hw/hfi1/file_ops.c b/drivers/infiniband/hw/hfi1/file_ops.c index 629beff053add38a0fc390523b347786e6eee769..f5f9269fdc16276242e4ddd38dfa4541d8ecc866 100644 --- a/drivers/infiniband/hw/hfi1/file_ops.c +++ b/drivers/infiniband/hw/hfi1/file_ops.c @@ -965,7 +965,7 @@ static int allocate_ctxt(struct hfi1_filedata *fd, struct hfi1_devdata *dd, uctxt->userversion = uinfo->userversion; uctxt->flags = hfi1_cap_mask; /* save current flag state */ init_waitqueue_head(&uctxt->wait); - strlcpy(uctxt->comm, current->comm, sizeof(uctxt->comm)); + strscpy(uctxt->comm, current->comm, sizeof(uctxt->comm)); memcpy(uctxt->uuid, uinfo->uuid, sizeof(uctxt->uuid)); uctxt->jkey = generate_jkey(current_uid()); hfi1_stats.sps_ctxts++; diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index aa15a5cc7cf3a342a912f590c21c4934a3fff40a..1d77514ebbee0dad6c3162c8b86fc45d5763be46 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -1114,7 +1114,7 @@ static void turn_off_spicos(struct hfi1_devdata *dd, int flags) * Reset all of the fabric serdes for this HFI in preparation to take the * link to Polling. * - * To do a reset, we need to write to to the serdes registers. Unfortunately, + * To do a reset, we need to write to the serdes registers. Unfortunately, * the fabric serdes download to the other HFI on the ASIC will have turned * off the firmware validation on this HFI. This means we can't write to the * registers to reset the serdes. Work around this by performing a complete diff --git a/drivers/infiniband/hw/hfi1/ipoib_rx.c b/drivers/infiniband/hw/hfi1/ipoib_rx.c index 3afa7545242c4ce2470e026dc000bada35ef6df7..629691a572efe47c7c1eaa7e52c9effcf85ef0db 100644 --- a/drivers/infiniband/hw/hfi1/ipoib_rx.c +++ b/drivers/infiniband/hw/hfi1/ipoib_rx.c @@ -11,13 +11,10 @@ static void copy_ipoib_buf(struct sk_buff *skb, void *data, int size) { - void *dst_data; - skb_checksum_none_assert(skb); skb->protocol = *((__be16 *)data); - dst_data = skb_put(skb, size); - memcpy(dst_data, data, size); + skb_put_data(skb, data, size); skb->mac_header = HFI1_IPOIB_PSEUDO_LEN; skb_pull(skb, HFI1_IPOIB_ENCAP_LEN); } diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index 2a7abf7a1f7fb9da3c672b6f860dfb42158600e3..18b05ffb415a3050586f08db1d6f020de60766ca 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -850,7 +850,7 @@ void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd) int i; for (i = 0; i < RXE_NUM_TID_FLOWS; i++) { - rcd->flows[i].generation = mask_generation(prandom_u32()); + rcd->flows[i].generation = mask_generation(get_random_u32()); kern_set_hw_flow(rcd, KERN_GENERATION_RESERVED, i); } } diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index 6988f6f21bdebbb0e5393adef624c1ebe2e8e8c6..e6e17984553c050c6796d8804dac54f30d0f8a98 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1447,12 +1447,10 @@ static int shut_down_port(struct rvt_dev_info *rdi, u32 port_num) struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi); struct hfi1_devdata *dd = dd_from_dev(verbs_dev); struct hfi1_pportdata *ppd = &dd->pport[port_num - 1]; - int ret; set_link_down_reason(ppd, OPA_LINKDOWN_REASON_UNKNOWN, 0, OPA_LINKDOWN_REASON_UNKNOWN); - ret = set_link_state(ppd, HLS_DN_DOWNDEF); - return ret; + return set_link_state(ppd, HLS_DN_DOWNDEF); } static int hfi1_get_guid_be(struct rvt_dev_info *rdi, struct rvt_ibport *rvp, @@ -1801,7 +1799,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) ib_set_device_ops(ibdev, &hfi1_dev_ops); - strlcpy(ibdev->node_desc, init_utsname()->nodename, + strscpy(ibdev->node_desc, init_utsname()->nodename, sizeof(ibdev->node_desc)); /* diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h index 38565532d6542e674c4f9302163d3896d7db16c5..7f30f32b34dc2070088609a16d06147b609af975 100644 --- a/drivers/infiniband/hw/hfi1/verbs.h +++ b/drivers/infiniband/hw/hfi1/verbs.h @@ -391,9 +391,6 @@ void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait); int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send); -extern const u32 rc_only_opcode; -extern const u32 uc_only_opcode; - int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_packet *packet); u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index 9f04f25d96317bd4a52a064ad5b604bd99b5ecf9..a7d259238305b4505bc9634679cf20fefe31839b 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -10,6 +10,6 @@ hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o ifdef CONFIG_INFINIBAND_HNS_HIP08 -hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) +hns-roce-hw-v2-objs := hns_roce_hw_v2.o $(hns-roce-objs) obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o endif diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c index 492b122d052199f13c062d08f252b5a9be171080..480c062dd04f1f04c82087c0b6f566263a2e0c7c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_ah.c +++ b/drivers/infiniband/hw/hns/hns_roce_ah.c @@ -41,9 +41,8 @@ static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr) u16 sport; if (!fl) - sport = get_random_u32() % - (IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 - - IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) + + sport = prandom_u32_max(IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 - + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) + IB_ROCE_UDP_ENCAP_VALID_PORT_MIN; else sport = rdma_flow_label_to_udp_sport(fl); diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 8acd599ffac1350dc79f244f1c4412cc33de2efa..736dc2f993b4039a65a952a198ddb5f9c7be20cd 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -454,7 +454,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) hr_cq = xa_load(&hr_dev->cq_table.array, cqn & (hr_dev->caps.num_cqs - 1)); if (!hr_cq) { - dev_warn(hr_dev->dev, "Completion event for bogus CQ 0x%06x\n", + dev_warn(hr_dev->dev, "completion event for bogus CQ 0x%06x\n", cqn); return; } @@ -475,14 +475,14 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) hr_cq = xa_load(&hr_dev->cq_table.array, cqn & (hr_dev->caps.num_cqs - 1)); if (!hr_cq) { - dev_warn(dev, "Async event for bogus CQ 0x%06x\n", cqn); + dev_warn(dev, "async event for bogus CQ 0x%06x\n", cqn); return; } if (event_type != HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID && event_type != HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR && event_type != HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW) { - dev_err(dev, "Unexpected event type 0x%x on CQ 0x%06x\n", + dev_err(dev, "unexpected event type 0x%x on CQ 0x%06x\n", event_type, cqn); return; } diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index f848eedc6a239b9c78ad110ca4490877acb3361e..723e55a7de8d9c8b9d7c57df08c6fcb9d72a2001 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -240,7 +240,6 @@ struct hns_roce_hem_table { /* Single obj size */ unsigned long obj_size; unsigned long table_chunk_size; - int lowmem; struct mutex mutex; struct hns_roce_hem **hem; u64 **bt_l1; @@ -599,7 +598,6 @@ struct hns_roce_qp { struct hns_roce_db rdb; struct hns_roce_db sdb; unsigned long en_flags; - u32 doorbell_qpn; enum ib_sig_type sq_signal_bits; struct hns_roce_wq sq; @@ -726,18 +724,17 @@ struct hns_roce_caps { u32 max_sq_sg; u32 max_sq_inline; u32 max_rq_sg; - u32 max_extend_sg; + u32 rsv0; u32 num_qps; u32 num_pi_qps; u32 reserved_qps; - int num_qpc_timer; u32 num_srqs; u32 max_wqes; u32 max_srq_wrs; u32 max_srq_sges; u32 max_sq_desc_sz; u32 max_rq_desc_sz; - u32 max_srq_desc_sz; + u32 rsv2; int max_qp_init_rdma; int max_qp_dest_rdma; u32 num_cqs; @@ -750,7 +747,7 @@ struct hns_roce_caps { int num_comp_vectors; int num_other_vectors; u32 num_mtpts; - u32 num_mtt_segs; + u32 rsv1; u32 num_srqwqe_segs; u32 num_idx_segs; int reserved_mrws; @@ -849,11 +846,6 @@ struct hns_roce_caps { enum cong_type cong_type; }; -struct hns_roce_dfx_hw { - int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn, - int *buffer); -}; - enum hns_roce_device_state { HNS_ROCE_DEVICE_STATE_INITED, HNS_ROCE_DEVICE_STATE_RST_DOWN, @@ -899,6 +891,9 @@ struct hns_roce_hw { int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); int (*write_srqc)(struct hns_roce_srq *srq, void *mb_buf); + int (*query_cqc)(struct hns_roce_dev *hr_dev, u32 cqn, void *buffer); + int (*query_qpc)(struct hns_roce_dev *hr_dev, u32 qpn, void *buffer); + int (*query_mpt)(struct hns_roce_dev *hr_dev, u32 key, void *buffer); const struct ib_device_ops *hns_roce_dev_ops; const struct ib_device_ops *hns_roce_dev_srq_ops; }; @@ -960,7 +955,6 @@ struct hns_roce_dev { void *priv; struct workqueue_struct *irq_workq; struct work_struct ecc_work; - const struct hns_roce_dfx_hw *dfx; u32 func_num; u32 is_vf; u32 cong_algo_tmpl_id; @@ -1228,8 +1222,12 @@ u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u32 port, int gid_index); void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev); int hns_roce_init(struct hns_roce_dev *hr_dev); void hns_roce_exit(struct hns_roce_dev *hr_dev); -int hns_roce_fill_res_cq_entry(struct sk_buff *msg, - struct ib_cq *ib_cq); +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq); +int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq); +int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp); +int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp); +int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr); +int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr); struct hns_user_mmap_entry * hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address, size_t length, diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index ce1a0d2792a35c2ec689eeb7127863e33dd525f7..aa8a08d1c01456b1c546f27b40a87cd002015f85 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -455,7 +455,7 @@ static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, * alloc bt space chunk for MTT/CQE. */ size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; - flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; + flag = GFP_KERNEL | __GFP_NOWARN; table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, size, flag); if (!table->hem[index->buf]) { @@ -588,8 +588,7 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev, table->hem[i] = hns_roce_alloc_hem(hr_dev, table->table_chunk_size >> PAGE_SHIFT, table->table_chunk_size, - (table->lowmem ? GFP_KERNEL : - GFP_HIGHUSER) | __GFP_NOWARN); + GFP_KERNEL | __GFP_NOWARN); if (!table->hem[i]) { ret = -ENOMEM; goto out; @@ -725,9 +724,6 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, int length; int i, j; - if (!table->lowmem) - return NULL; - mutex_lock(&table->mutex); if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { @@ -783,8 +779,7 @@ out: int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, u32 type, - unsigned long obj_size, unsigned long nobj, - int use_lowmem) + unsigned long obj_size, unsigned long nobj) { unsigned long obj_per_chunk; unsigned long num_hem; @@ -861,7 +856,6 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, table->type = type; table->num_hem = num_hem; table->obj_size = obj_size; - table->lowmem = use_lowmem; mutex_init(&table->mutex); return 0; @@ -932,7 +926,7 @@ void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, if (table->hem[i]) { if (hr_dev->hw->clear_hem(hr_dev, table, i * table->table_chunk_size / table->obj_size, 0)) - dev_err(dev, "Clear HEM base address failed.\n"); + dev_err(dev, "clear HEM base address failed.\n"); hns_roce_free_hem(hr_dev, table->hem[i]); } @@ -986,7 +980,7 @@ struct hns_roce_hem_head { static struct hns_roce_hem_item * hem_list_alloc_item(struct hns_roce_dev *hr_dev, int start, int end, int count, - bool exist_bt, int bt_level) + bool exist_bt) { struct hns_roce_hem_item *hem; @@ -1195,7 +1189,7 @@ static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, start_aligned = (distance / step) * step + r->offset; end = min_t(int, start_aligned + step - 1, max_ofs); cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, - true, level); + true); if (!cur) { ret = -ENOMEM; goto err_exit; @@ -1247,7 +1241,7 @@ alloc_root_hem(struct hns_roce_dev *hr_dev, int unit, int *max_ba_num, /* indicate to last region */ r = ®ions[region_cnt - 1]; hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, - ba_num, true, 0); + ba_num, true); if (!hem) return ERR_PTR(-ENOMEM); @@ -1264,7 +1258,7 @@ static int alloc_fake_root_bt(struct hns_roce_dev *hr_dev, void *cpu_base, struct hns_roce_hem_item *hem; hem = hem_list_alloc_item(hr_dev, r->offset, r->offset + r->count - 1, - r->count, false, 0); + r->count, false); if (!hem) return -ENOMEM; @@ -1421,7 +1415,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, &hem_list->btm_bt); if (ret) { dev_err(hr_dev->dev, - "alloc hem trunk fail ret=%d!\n", ret); + "alloc hem trunk fail ret = %d!\n", ret); goto err_alloc; } } @@ -1430,7 +1424,7 @@ int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, region_cnt); if (ret) - dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); + dev_err(hr_dev->dev, "alloc hem root fail ret = %d!\n", ret); else return 0; @@ -1468,19 +1462,17 @@ void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list) void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, - int offset, int *mtt_cnt, u64 *phy_addr) + int offset, int *mtt_cnt) { struct list_head *head = &hem_list->btm_bt; struct hns_roce_hem_item *hem, *temp_hem; void *cpu_base = NULL; - u64 phy_base = 0; int nr = 0; list_for_each_entry_safe(hem, temp_hem, head, sibling) { if (hem_list_page_is_in_range(hem, offset)) { nr = offset - hem->start; cpu_base = hem->addr + nr * BA_BYTE_LEN; - phy_base = hem->dma_addr + nr * BA_BYTE_LEN; nr = hem->end + 1 - offset; break; } @@ -1489,8 +1481,5 @@ void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, if (mtt_cnt) *mtt_cnt = nr; - if (phy_addr) - *phy_addr = phy_base; - return cpu_base; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index 2d84a6b3f05df0440dccb826a5c2bbd2234ea7c7..7d23d3c51da46b177f44a9c9f06ee9dcb5282116 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -111,8 +111,7 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, dma_addr_t *dma_handle); int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, u32 type, - unsigned long obj_size, unsigned long nobj, - int use_lowmem); + unsigned long obj_size, unsigned long nobj); void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table); void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev); @@ -132,7 +131,7 @@ void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list); void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_hem_list *hem_list, - int offset, int *mtt_cnt, u64 *phy_addr); + int offset, int *mtt_cnt); static inline void hns_roce_hem_first(struct hns_roce_hem *hem, struct hns_roce_hem_iter *iter) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index cbdafaac678a146b12776526a7a4817025a13f4d..1ead35fb031b0a11cf4b19db0b0a2ba289337c7a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -193,8 +193,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, unsigned int *sge_idx, u32 msg_len) { struct ib_device *ibdev = &(to_hr_dev(qp->ibqp.device))->ib_dev; - unsigned int dseg_len = sizeof(struct hns_roce_v2_wqe_data_seg); - unsigned int ext_sge_sz = qp->sq.max_gs * dseg_len; + unsigned int ext_sge_sz = qp->sq.max_gs * HNS_ROCE_SGE_SIZE; unsigned int left_len_in_pg; unsigned int idx = *sge_idx; unsigned int i = 0; @@ -222,7 +221,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, if (len <= left_len_in_pg) { memcpy(dseg, addr, len); - idx += len / dseg_len; + idx += len / HNS_ROCE_SGE_SIZE; i++; if (i >= wr->num_sge) @@ -237,7 +236,7 @@ static int fill_ext_sge_inl_data(struct hns_roce_qp *qp, len -= left_len_in_pg; addr += left_len_in_pg; - idx += left_len_in_pg / dseg_len; + idx += left_len_in_pg / HNS_ROCE_SGE_SIZE; dseg = hns_roce_get_extend_sge(qp, idx & (qp->sge.sge_cnt - 1)); left_len_in_pg = 1 << HNS_HW_PAGE_SHIFT; @@ -381,7 +380,7 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, if (unlikely(ibqp->qp_type != IB_QPT_RC && ibqp->qp_type != IB_QPT_GSI && ibqp->qp_type != IB_QPT_UD)) { - ibdev_err(ibdev, "Not supported QP(0x%x)type!\n", + ibdev_err(ibdev, "not supported QP(0x%x)type!\n", ibqp->qp_type); return -EOPNOTSUPP; } else if (unlikely(hr_qp->state == IB_QPS_RESET || @@ -637,7 +636,7 @@ static inline void update_sq_db(struct hns_roce_dev *hr_dev, } else { struct hns_roce_v2_db sq_db = {}; - hr_reg_write(&sq_db, DB_TAG, qp->doorbell_qpn); + hr_reg_write(&sq_db, DB_TAG, qp->qpn); hr_reg_write(&sq_db, DB_CMD, HNS_ROCE_V2_SQ_DB); hr_reg_write(&sq_db, DB_PI, qp->sq.head); hr_reg_write(&sq_db, DB_SL, qp->sl); @@ -1406,20 +1405,20 @@ static void func_clr_hw_resetting_state(struct hns_roce_dev *hr_dev, hr_dev->dis_db = true; dev_warn(hr_dev->dev, - "Func clear is pending, device in resetting state.\n"); + "func clear is pending, device in resetting state.\n"); end = HNS_ROCE_V2_HW_RST_TIMEOUT; while (end) { if (!ops->get_hw_reset_stat(handle)) { hr_dev->is_reset = true; dev_info(hr_dev->dev, - "Func clear success after reset.\n"); + "func clear success after reset.\n"); return; } msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; } - dev_warn(hr_dev->dev, "Func clear failed.\n"); + dev_warn(hr_dev->dev, "func clear failed.\n"); } static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev, @@ -1431,21 +1430,21 @@ static void func_clr_sw_resetting_state(struct hns_roce_dev *hr_dev, hr_dev->dis_db = true; dev_warn(hr_dev->dev, - "Func clear is pending, device in resetting state.\n"); + "func clear is pending, device in resetting state.\n"); end = HNS_ROCE_V2_HW_RST_TIMEOUT; while (end) { if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) { hr_dev->is_reset = true; dev_info(hr_dev->dev, - "Func clear success after sw reset\n"); + "func clear success after sw reset\n"); return; } msleep(HNS_ROCE_V2_HW_RST_COMPLETION_WAIT); end -= HNS_ROCE_V2_HW_RST_COMPLETION_WAIT; } - dev_warn(hr_dev->dev, "Func clear failed because of unfinished sw reset\n"); + dev_warn(hr_dev->dev, "func clear failed because of unfinished sw reset\n"); } static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, @@ -1458,7 +1457,7 @@ static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, if (ops->ae_dev_reset_cnt(handle) != hr_dev->reset_cnt) { hr_dev->dis_db = true; hr_dev->is_reset = true; - dev_info(hr_dev->dev, "Func clear success after reset.\n"); + dev_info(hr_dev->dev, "func clear success after reset.\n"); return; } @@ -1475,9 +1474,9 @@ static void hns_roce_func_clr_rst_proc(struct hns_roce_dev *hr_dev, int retval, if (retval && !flag) dev_warn(hr_dev->dev, - "Func clear read failed, ret = %d.\n", retval); + "func clear read failed, ret = %d.\n", retval); - dev_warn(hr_dev->dev, "Func clear failed.\n"); + dev_warn(hr_dev->dev, "func clear failed.\n"); } static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id) @@ -1498,7 +1497,7 @@ static void __hns_roce_function_clear(struct hns_roce_dev *hr_dev, int vf_id) ret = hns_roce_cmq_send(hr_dev, &desc, 1); if (ret) { fclr_write_fail_flag = true; - dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", + dev_err(hr_dev->dev, "func clear write failed, ret = %d.\n", ret); goto out; } @@ -1966,7 +1965,6 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->min_cqes = HNS_ROCE_MIN_CQE_NUM; caps->max_cqes = HNS_ROCE_V2_MAX_CQE_NUM; caps->max_sq_sg = HNS_ROCE_V2_MAX_SQ_SGE_NUM; - caps->max_extend_sg = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM; caps->max_rq_sg = HNS_ROCE_V2_MAX_RQ_SGE_NUM; caps->num_uars = HNS_ROCE_V2_UAR_NUM; @@ -1977,14 +1975,13 @@ static void set_default_caps(struct hns_roce_dev *hr_dev) caps->num_mtpts = HNS_ROCE_V2_MAX_MTPT_NUM; caps->num_pds = HNS_ROCE_V2_MAX_PD_NUM; - caps->num_qpc_timer = HNS_ROCE_V2_MAX_QPC_TIMER_NUM; + caps->qpc_timer_bt_num = HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM; caps->cqc_timer_bt_num = HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM; caps->max_qp_init_rdma = HNS_ROCE_V2_MAX_QP_INIT_RDMA; caps->max_qp_dest_rdma = HNS_ROCE_V2_MAX_QP_DEST_RDMA; caps->max_sq_desc_sz = HNS_ROCE_V2_MAX_SQ_DESC_SZ; caps->max_rq_desc_sz = HNS_ROCE_V2_MAX_RQ_DESC_SZ; - caps->max_srq_desc_sz = HNS_ROCE_V2_MAX_SRQ_DESC_SZ; caps->irrl_entry_sz = HNS_ROCE_V2_IRRL_ENTRY_SZ; caps->trrl_entry_sz = HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ; caps->cqc_entry_sz = HNS_ROCE_V2_CQC_ENTRY_SZ; @@ -2185,13 +2182,14 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->num_xrcds = HNS_ROCE_V2_MAX_XRCD_NUM; caps->reserved_xrcds = HNS_ROCE_V2_RSV_XRCD_NUM; - caps->num_mtt_segs = HNS_ROCE_V2_MAX_MTT_SEGS; caps->num_srqwqe_segs = HNS_ROCE_V2_MAX_SRQWQE_SEGS; caps->num_idx_segs = HNS_ROCE_V2_MAX_IDX_SEGS; if (!caps->num_comp_vectors) - caps->num_comp_vectors = min_t(u32, caps->eqc_bt_num - 1, - (u32)priv->handle->rinfo.num_vectors - 2); + caps->num_comp_vectors = + min_t(u32, caps->eqc_bt_num - HNS_ROCE_V2_AEQE_VEC_NUM, + (u32)priv->handle->rinfo.num_vectors - + (HNS_ROCE_V2_AEQE_VEC_NUM + HNS_ROCE_V2_ABNORMAL_VEC_NUM)); if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09) { caps->eqe_hop_num = HNS_ROCE_V3_EQE_HOP_NUM; @@ -2272,15 +2270,12 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev) caps->max_sq_inline = le16_to_cpu(resp_a->max_sq_inline); caps->max_rq_sg = le16_to_cpu(resp_a->max_rq_sg); caps->max_rq_sg = roundup_pow_of_two(caps->max_rq_sg); - caps->max_extend_sg = le32_to_cpu(resp_a->max_extend_sg); - caps->num_qpc_timer = le16_to_cpu(resp_a->num_qpc_timer); caps->max_srq_sges = le16_to_cpu(resp_a->max_srq_sges); caps->max_srq_sges = roundup_pow_of_two(caps->max_srq_sges); caps->num_aeq_vectors = resp_a->num_aeq_vectors; caps->num_other_vectors = resp_a->num_other_vectors; caps->max_sq_desc_sz = resp_a->max_sq_desc_sz; caps->max_rq_desc_sz = resp_a->max_rq_desc_sz; - caps->max_srq_desc_sz = resp_a->max_srq_desc_sz; caps->cqe_sz = resp_a->cqe_sz; caps->mtpt_entry_sz = resp_b->mtpt_entry_sz; @@ -4300,7 +4295,6 @@ static inline int get_pdn(struct ib_pd *ib_pd) static void modify_qp_reset_to_init(struct ib_qp *ibqp, const struct ib_qp_attr *attr, - int attr_mask, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4364,7 +4358,7 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp, } static void modify_qp_init_to_init(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, + const struct ib_qp_attr *attr, struct hns_roce_v2_qp_context *context, struct hns_roce_v2_qp_context *qpc_mask) { @@ -4613,7 +4607,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_DQPN); } - memcpy(&(context->dmac), dmac, sizeof(u32)); + memcpy(&context->dmac, dmac, sizeof(u32)); hr_reg_write(context, QPC_DMAC_H, *((u16 *)(&dmac[4]))); qpc_mask->dmac = 0; hr_reg_clear(qpc_mask, QPC_DMAC_H); @@ -5015,11 +5009,9 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { memset(qpc_mask, 0, hr_dev->caps.qpc_sz); - modify_qp_reset_to_init(ibqp, attr, attr_mask, context, - qpc_mask); + modify_qp_reset_to_init(ibqp, attr, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) { - modify_qp_init_to_init(ibqp, attr, attr_mask, context, - qpc_mask); + modify_qp_init_to_init(ibqp, attr, context, qpc_mask); } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context, qpc_mask); @@ -5040,14 +5032,14 @@ static bool check_qp_timeout_cfg_range(struct hns_roce_dev *hr_dev, u8 *timeout) if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) { if (*timeout > QP_ACK_TIMEOUT_MAX_HIP08) { ibdev_warn(&hr_dev->ib_dev, - "Local ACK timeout shall be 0 to 20.\n"); + "local ACK timeout shall be 0 to 20.\n"); return false; } *timeout += QP_ACK_TIMEOUT_OFFSET; } else if (hr_dev->pci_dev->revision > PCI_REVISION_ID_HIP08) { if (*timeout > QP_ACK_TIMEOUT_MAX) { ibdev_warn(&hr_dev->ib_dev, - "Local ACK timeout shall be 0 to 31.\n"); + "local ACK timeout shall be 0 to 31.\n"); return false; } } @@ -5307,9 +5299,8 @@ static int to_ib_qp_st(enum hns_roce_v2_qp_state state) return (state < ARRAY_SIZE(map)) ? map[state] : -1; } -static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, - struct hns_roce_qp *hr_qp, - struct hns_roce_v2_qp_context *hr_context) +static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, u32 qpn, + void *buffer) { struct hns_roce_cmd_mailbox *mailbox; int ret; @@ -5319,11 +5310,11 @@ static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev, return PTR_ERR(mailbox); ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_QPC, - hr_qp->qpn); + qpn); if (ret) goto out; - memcpy(hr_context, mailbox->buf, hr_dev->caps.qpc_sz); + memcpy(buffer, mailbox->buf, hr_dev->caps.qpc_sz); out: hns_roce_free_cmd_mailbox(hr_dev, mailbox); @@ -5353,7 +5344,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, goto done; } - ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, &context); + ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context); if (ret) { ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret); ret = -EINVAL; @@ -5551,7 +5542,7 @@ static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev, msleep(20); } - ibdev_err(ibdev, "Query SCC clr done flag overtime.\n"); + ibdev_err(ibdev, "query SCC clr done flag overtime.\n"); ret = -ETIMEDOUT; out: @@ -5774,6 +5765,64 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) return ret; } +static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn, + void *buffer) +{ + struct hns_roce_v2_cq_context *context; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + context = mailbox->buf; + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, + HNS_ROCE_CMD_QUERY_CQC, cqn); + if (ret) { + ibdev_err(&hr_dev->ib_dev, + "failed to process cmd when querying CQ, ret = %d.\n", + ret); + goto err_mailbox; + } + + memcpy(buffer, context, sizeof(*context)); + +err_mailbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + +static int hns_roce_v2_query_mpt(struct hns_roce_dev *hr_dev, u32 key, + void *buffer) +{ + struct hns_roce_v2_mpt_entry *context; + struct hns_roce_cmd_mailbox *mailbox; + int ret; + + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + + context = mailbox->buf; + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_MPT, + key_to_hw_index(key)); + if (ret) { + ibdev_err(&hr_dev->ib_dev, + "failed to process cmd when querying MPT, ret = %d.\n", + ret); + goto err_mailbox; + } + + memcpy(buffer, context, sizeof(*context)); + +err_mailbox: + hns_roce_free_cmd_mailbox(hr_dev, mailbox); + + return ret; +} + static void hns_roce_irq_work_handle(struct work_struct *work) { struct hns_roce_work *irq_work = @@ -5782,26 +5831,26 @@ static void hns_roce_irq_work_handle(struct work_struct *work) switch (irq_work->event_type) { case HNS_ROCE_EVENT_TYPE_PATH_MIG: - ibdev_info(ibdev, "Path migrated succeeded.\n"); + ibdev_info(ibdev, "path migrated succeeded.\n"); break; case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED: - ibdev_warn(ibdev, "Path migration failed.\n"); + ibdev_warn(ibdev, "path migration failed.\n"); break; case HNS_ROCE_EVENT_TYPE_COMM_EST: break; case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: - ibdev_warn(ibdev, "Send queue drained.\n"); + ibdev_warn(ibdev, "send queue drained.\n"); break; case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR: - ibdev_err(ibdev, "Local work queue 0x%x catast error, sub_event type is: %d\n", + ibdev_err(ibdev, "local work queue 0x%x catast error, sub_event type is: %d\n", irq_work->queue_num, irq_work->sub_type); break; case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR: - ibdev_err(ibdev, "Invalid request local work queue 0x%x error.\n", + ibdev_err(ibdev, "invalid request local work queue 0x%x error.\n", irq_work->queue_num); break; case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR: - ibdev_err(ibdev, "Local access violation work queue 0x%x error, sub_event type is: %d\n", + ibdev_err(ibdev, "local access violation work queue 0x%x error, sub_event type is: %d\n", irq_work->queue_num, irq_work->sub_type); break; case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH: @@ -5823,7 +5872,7 @@ static void hns_roce_irq_work_handle(struct work_struct *work) ibdev_warn(ibdev, "DB overflow.\n"); break; case HNS_ROCE_EVENT_TYPE_FLR: - ibdev_warn(ibdev, "Function level reset.\n"); + ibdev_warn(ibdev, "function level reset.\n"); break; case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION: ibdev_err(ibdev, "xrc domain violation error.\n"); @@ -5847,12 +5896,12 @@ static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev, if (!irq_work) return; - INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle); + INIT_WORK(&irq_work->work, hns_roce_irq_work_handle); irq_work->hr_dev = hr_dev; irq_work->event_type = eq->event_type; irq_work->sub_type = eq->sub_type; irq_work->queue_num = queue_num; - queue_work(hr_dev->irq_workq, &(irq_work->work)); + queue_work(hr_dev->irq_workq, &irq_work->work); } static void update_eq_db(struct hns_roce_eq *eq) @@ -5942,7 +5991,7 @@ static irqreturn_t hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev, case HNS_ROCE_EVENT_TYPE_FLR: break; default: - dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n", + dev_err(dev, "unhandled event %d on EQ %d at idx %u.\n", event_type, eq->eqn, eq->cons_index); break; } @@ -6012,7 +6061,7 @@ static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr) /* Completion event interrupt */ int_work = hns_roce_v2_ceq_int(hr_dev, eq); else - /* Asychronous event interrupt */ + /* Asynchronous event interrupt */ int_work = hns_roce_v2_aeq_int(hr_dev, eq); return IRQ_RETVAL(int_work); @@ -6333,7 +6382,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL, 0); if (err) - dev_err(hr_dev->dev, "Failed to alloc EQE mtr, err %d\n", err); + dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err); return err; } @@ -6422,7 +6471,7 @@ static int __hns_roce_request_irq(struct hns_roce_dev *hr_dev, int irq_num, 0, hr_dev->irq_names[j - comp_num], &eq_table->eq[j - other_num]); if (ret) { - dev_err(hr_dev->dev, "Request irq error!\n"); + dev_err(hr_dev->dev, "request irq error!\n"); goto err_request_failed; } } @@ -6575,10 +6624,6 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev) kfree(eq_table->eq); } -static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = { - .query_cqc_info = hns_roce_v2_query_cqc_info, -}; - static const struct ib_device_ops hns_roce_v2_dev_ops = { .destroy_qp = hns_roce_v2_destroy_qp, .modify_cq = hns_roce_v2_modify_cq, @@ -6619,6 +6664,9 @@ static const struct hns_roce_hw hns_roce_hw_v2 = { .init_eq = hns_roce_v2_init_eq_table, .cleanup_eq = hns_roce_v2_cleanup_eq_table, .write_srqc = hns_roce_v2_write_srqc, + .query_cqc = hns_roce_v2_query_cqc, + .query_qpc = hns_roce_v2_query_qpc, + .query_mpt = hns_roce_v2_query_mpt, .hns_roce_dev_ops = &hns_roce_v2_dev_ops, .hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops, }; @@ -6650,7 +6698,6 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev, hr_dev->is_vf = id->driver_data; hr_dev->dev = &handle->pdev->dev; hr_dev->hw = &hns_roce_hw_v2; - hr_dev->dfx = &hns_roce_dfx_hw_v2; hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG; hr_dev->odb_offset = hr_dev->sdb_offset; @@ -6846,7 +6893,7 @@ static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle) dev_err(dev, "In reset process RoCE reinit failed %d.\n", ret); } else { handle->rinfo.reset_state = HNS_ROCE_STATE_RST_INITED; - dev_info(dev, "Reset done, RoCE client reinit finished.\n"); + dev_info(dev, "reset done, RoCE client reinit finished.\n"); } return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index f96debac30fe9d888ed11ed20fff785074548001..b11579027e82793da2c2c8e43734b3713d53a793 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -36,17 +36,16 @@ #include #define HNS_ROCE_V2_MAX_QP_NUM 0x1000 -#define HNS_ROCE_V2_MAX_QPC_TIMER_NUM 0x200 #define HNS_ROCE_V2_MAX_WQE_NUM 0x8000 #define HNS_ROCE_V2_MAX_SRQ_WR 0x8000 #define HNS_ROCE_V2_MAX_SRQ_SGE 64 #define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 +#define HNS_ROCE_V2_MAX_QPC_TIMER_BT_NUM 0x100 #define HNS_ROCE_V2_MAX_CQC_TIMER_BT_NUM 0x100 #define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 #define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 #define HNS_ROCE_V2_MAX_RQ_SGE_NUM 64 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 64 -#define HNS_ROCE_V2_MAX_EXTEND_SGE_NUM 0x200000 #define HNS_ROCE_V2_MAX_SQ_INLINE 0x20 #define HNS_ROCE_V3_MAX_SQ_INLINE 0x400 #define HNS_ROCE_V2_MAX_RC_INL_INN_SZ 32 @@ -55,7 +54,6 @@ #define HNS_ROCE_V2_AEQE_VEC_NUM 1 #define HNS_ROCE_V2_ABNORMAL_VEC_NUM 1 #define HNS_ROCE_V2_MAX_MTPT_NUM 0x100000 -#define HNS_ROCE_V2_MAX_MTT_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_SRQWQE_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_IDX_SEGS 0x1000000 #define HNS_ROCE_V2_MAX_PD_NUM 0x1000000 @@ -65,7 +63,6 @@ #define HNS_ROCE_V2_MAX_QP_DEST_RDMA 128 #define HNS_ROCE_V2_MAX_SQ_DESC_SZ 64 #define HNS_ROCE_V2_MAX_RQ_DESC_SZ 16 -#define HNS_ROCE_V2_MAX_SRQ_DESC_SZ 64 #define HNS_ROCE_V2_IRRL_ENTRY_SZ 64 #define HNS_ROCE_V2_EXT_ATOMIC_TRRL_ENTRY_SZ 100 #define HNS_ROCE_V2_CQC_ENTRY_SZ 64 @@ -83,7 +80,7 @@ #define HNS_ROCE_V2_QPC_TIMER_ENTRY_SZ PAGE_SIZE #define HNS_ROCE_V2_CQC_TIMER_ENTRY_SZ PAGE_SIZE -#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED 0xFFFFF000 +#define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED 0xFFFF000 #define HNS_ROCE_V2_MAX_INNER_MTPT_NUM 2 #define HNS_ROCE_INVALID_LKEY 0x0 #define HNS_ROCE_INVALID_SGE_LENGTH 0x80000000 @@ -406,6 +403,7 @@ enum hns_roce_v2_qp_state { struct hns_roce_v2_qp_context_ex { __le32 data[64]; }; + struct hns_roce_v2_qp_context { __le32 byte_4_sqpn_tst; __le32 wqe_sge_ba; @@ -758,7 +756,8 @@ struct hns_roce_v2_mpt_entry { #define MPT_INNER_PA_VLD MPT_FIELD_LOC(71, 71) #define MPT_MW_BIND_QPN MPT_FIELD_LOC(95, 72) #define MPT_BOUND_LKEY MPT_FIELD_LOC(127, 96) -#define MPT_LEN MPT_FIELD_LOC(191, 128) +#define MPT_LEN_L MPT_FIELD_LOC(159, 128) +#define MPT_LEN_H MPT_FIELD_LOC(191, 160) #define MPT_LKEY MPT_FIELD_LOC(223, 192) #define MPT_VA MPT_FIELD_LOC(287, 224) #define MPT_PBL_SIZE MPT_FIELD_LOC(319, 288) @@ -1173,7 +1172,7 @@ struct hns_roce_query_pf_caps_a { __le16 max_sq_sg; __le16 max_sq_inline; __le16 max_rq_sg; - __le32 max_extend_sg; + __le32 rsv0; __le16 num_qpc_timer; __le16 num_cqc_timer; __le16 max_srq_sges; @@ -1181,7 +1180,7 @@ struct hns_roce_query_pf_caps_a { u8 num_other_vectors; u8 max_sq_desc_sz; u8 max_rq_desc_sz; - u8 max_srq_desc_sz; + u8 rsv1; u8 cqe_sz; }; @@ -1462,9 +1461,6 @@ struct hns_roce_sccc_clr_done { __le32 rsv[5]; }; -int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, - int *buffer); - static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2], void __iomem *dest) { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c deleted file mode 100644 index f7a75a7cda7491e93f1aa6134facad61623f52a3..0000000000000000000000000000000000000000 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) -// Copyright (c) 2019 Hisilicon Limited. - -#include "hnae3.h" -#include "hns_roce_device.h" -#include "hns_roce_cmd.h" -#include "hns_roce_hw_v2.h" - -int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn, - int *buffer) -{ - struct hns_roce_v2_cq_context *cq_context; - struct hns_roce_cmd_mailbox *mailbox; - int ret; - - mailbox = hns_roce_alloc_cmd_mailbox(hr_dev); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - cq_context = mailbox->buf; - ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, HNS_ROCE_CMD_QUERY_CQC, - cqn); - if (ret) { - dev_err(hr_dev->dev, "QUERY cqc cmd process error\n"); - goto err_mailbox; - } - - memcpy(buffer, cq_context, sizeof(*cq_context)); - -err_mailbox: - hns_roce_free_cmd_mailbox(hr_dev, mailbox); - - return ret; -} diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index c8af4ebd7cbd35dd1b7d0598385a63b647ab03cf..dcf89689a4c628d908af905131b01f54716caf37 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -97,7 +97,7 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u32 port, netdev = hr_dev->iboe.netdevs[port]; if (!netdev) { - dev_err(dev, "Can't find netdev on port(%u)!\n", port); + dev_err(dev, "can't find netdev on port(%u)!\n", port); return -ENODEV; } @@ -239,7 +239,7 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u32 port_num, net_dev = hr_dev->iboe.netdevs[port]; if (!net_dev) { spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - dev_err(dev, "Find netdev %u failed!\n", port); + dev_err(dev, "find netdev %u failed!\n", port); return -EINVAL; } @@ -515,7 +515,6 @@ static const struct ib_device_ops hns_roce_dev_ops = { .destroy_ah = hns_roce_destroy_ah, .destroy_cq = hns_roce_destroy_cq, .disassociate_ucontext = hns_roce_disassociate_ucontext, - .fill_res_cq_entry = hns_roce_fill_res_cq_entry, .get_dma_mr = hns_roce_get_dma_mr, .get_link_layer = hns_roce_get_link_layer, .get_port_immutable = hns_roce_port_immutable, @@ -566,6 +565,15 @@ static const struct ib_device_ops hns_roce_dev_xrcd_ops = { INIT_RDMA_OBJ_SIZE(ib_xrcd, hns_roce_xrcd, ibxrcd), }; +static const struct ib_device_ops hns_roce_dev_restrack_ops = { + .fill_res_cq_entry = hns_roce_fill_res_cq_entry, + .fill_res_cq_entry_raw = hns_roce_fill_res_cq_entry_raw, + .fill_res_qp_entry = hns_roce_fill_res_qp_entry, + .fill_res_qp_entry_raw = hns_roce_fill_res_qp_entry_raw, + .fill_res_mr_entry = hns_roce_fill_res_mr_entry, + .fill_res_mr_entry_raw = hns_roce_fill_res_mr_entry_raw, +}; + static int hns_roce_register_device(struct hns_roce_dev *hr_dev) { int ret; @@ -605,6 +613,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); ib_set_device_ops(ib_dev, &hns_roce_dev_ops); + ib_set_device_ops(ib_dev, &hns_roce_dev_restrack_ops); for (i = 0; i < hr_dev->caps.num_ports; i++) { if (!hr_dev->iboe.netdevs[i]) continue; @@ -650,17 +659,17 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) ret = hns_roce_init_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table, HEM_TYPE_MTPT, hr_dev->caps.mtpt_entry_sz, - hr_dev->caps.num_mtpts, 1); + hr_dev->caps.num_mtpts); if (ret) { - dev_err(dev, "Failed to init MTPT context memory, aborting.\n"); + dev_err(dev, "failed to init MTPT context memory, aborting.\n"); return ret; } ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qp_table.qp_table, HEM_TYPE_QPC, hr_dev->caps.qpc_sz, - hr_dev->caps.num_qps, 1); + hr_dev->caps.num_qps); if (ret) { - dev_err(dev, "Failed to init QP context memory, aborting.\n"); + dev_err(dev, "failed to init QP context memory, aborting.\n"); goto err_unmap_dmpt; } @@ -668,9 +677,9 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) HEM_TYPE_IRRL, hr_dev->caps.irrl_entry_sz * hr_dev->caps.max_qp_init_rdma, - hr_dev->caps.num_qps, 1); + hr_dev->caps.num_qps); if (ret) { - dev_err(dev, "Failed to init irrl_table memory, aborting.\n"); + dev_err(dev, "failed to init irrl_table memory, aborting.\n"); goto err_unmap_qp; } @@ -680,19 +689,19 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) HEM_TYPE_TRRL, hr_dev->caps.trrl_entry_sz * hr_dev->caps.max_qp_dest_rdma, - hr_dev->caps.num_qps, 1); + hr_dev->caps.num_qps); if (ret) { dev_err(dev, - "Failed to init trrl_table memory, aborting.\n"); + "failed to init trrl_table memory, aborting.\n"); goto err_unmap_irrl; } } ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cq_table.table, HEM_TYPE_CQC, hr_dev->caps.cqc_entry_sz, - hr_dev->caps.num_cqs, 1); + hr_dev->caps.num_cqs); if (ret) { - dev_err(dev, "Failed to init CQ context memory, aborting.\n"); + dev_err(dev, "failed to init CQ context memory, aborting.\n"); goto err_unmap_trrl; } @@ -700,10 +709,10 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) ret = hns_roce_init_hem_table(hr_dev, &hr_dev->srq_table.table, HEM_TYPE_SRQC, hr_dev->caps.srqc_entry_sz, - hr_dev->caps.num_srqs, 1); + hr_dev->caps.num_srqs); if (ret) { dev_err(dev, - "Failed to init SRQ context memory, aborting.\n"); + "failed to init SRQ context memory, aborting.\n"); goto err_unmap_cq; } } @@ -713,10 +722,10 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) &hr_dev->qp_table.sccc_table, HEM_TYPE_SCCC, hr_dev->caps.sccc_sz, - hr_dev->caps.num_qps, 1); + hr_dev->caps.num_qps); if (ret) { dev_err(dev, - "Failed to init SCC context memory, aborting.\n"); + "failed to init SCC context memory, aborting.\n"); goto err_unmap_srq; } } @@ -725,10 +734,10 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) ret = hns_roce_init_hem_table(hr_dev, &hr_dev->qpc_timer_table, HEM_TYPE_QPC_TIMER, hr_dev->caps.qpc_timer_entry_sz, - hr_dev->caps.num_qpc_timer, 1); + hr_dev->caps.qpc_timer_bt_num); if (ret) { dev_err(dev, - "Failed to init QPC timer memory, aborting.\n"); + "failed to init QPC timer memory, aborting.\n"); goto err_unmap_ctx; } } @@ -737,10 +746,10 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) ret = hns_roce_init_hem_table(hr_dev, &hr_dev->cqc_timer_table, HEM_TYPE_CQC_TIMER, hr_dev->caps.cqc_timer_entry_sz, - hr_dev->caps.cqc_timer_bt_num, 1); + hr_dev->caps.cqc_timer_bt_num); if (ret) { dev_err(dev, - "Failed to init CQC timer memory, aborting.\n"); + "failed to init CQC timer memory, aborting.\n"); goto err_unmap_qpc_timer; } } @@ -749,7 +758,7 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev) ret = hns_roce_init_hem_table(hr_dev, &hr_dev->gmv_table, HEM_TYPE_GMV, hr_dev->caps.gmv_entry_sz, - hr_dev->caps.gmv_entry_num, 1); + hr_dev->caps.gmv_entry_num); if (ret) { dev_err(dev, "failed to init gmv table memory, ret = %d\n", @@ -818,13 +827,13 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) ret = hns_roce_uar_alloc(hr_dev, &hr_dev->priv_uar); if (ret) { - dev_err(dev, "Failed to allocate priv_uar.\n"); + dev_err(dev, "failed to allocate priv_uar.\n"); goto err_uar_table_free; } ret = hns_roce_init_qp_table(hr_dev); if (ret) { - dev_err(dev, "Failed to init qp_table.\n"); + dev_err(dev, "failed to init qp_table.\n"); goto err_uar_table_free; } @@ -837,9 +846,8 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev) hns_roce_init_cq_table(hr_dev); - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) { + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) hns_roce_init_srq_table(hr_dev); - } return 0; @@ -902,14 +910,14 @@ int hns_roce_init(struct hns_roce_dev *hr_dev) if (hr_dev->hw->cmq_init) { ret = hr_dev->hw->cmq_init(hr_dev); if (ret) { - dev_err(dev, "Init RoCE Command Queue failed!\n"); + dev_err(dev, "init RoCE Command Queue failed!\n"); return ret; } } ret = hr_dev->hw->hw_profile(hr_dev); if (ret) { - dev_err(dev, "Get RoCE engine profile failed!\n"); + dev_err(dev, "get RoCE engine profile failed!\n"); goto error_failed_cmd_init; } diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 867972c2a894dd47886f01473ae0902b7f4f9440..845ac7d3831f420d26f446ed7e36dca5f6201a03 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -190,7 +190,7 @@ struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc) int ret; mr = kzalloc(sizeof(*mr), GFP_KERNEL); - if (mr == NULL) + if (!mr) return ERR_PTR(-ENOMEM); mr->type = MR_TYPE_DMA; @@ -249,7 +249,6 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err_alloc_pbl; mr->ibmr.rkey = mr->ibmr.lkey = mr->key; - mr->ibmr.length = length; return &mr->ibmr; @@ -586,7 +585,7 @@ static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, while (offset < end && npage < max_count) { count = 0; mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, - offset, &count, NULL); + offset, &count); if (!mtts) return -ENOBUFS; @@ -835,7 +834,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, mtt_count = 0; mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, start_index + total, - &mtt_count, NULL); + &mtt_count); if (!mtts || !mtt_count) goto done; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 48d3616a6d71d134069f8ee345b4287d02f86c2a..f0bd82a18069a69054251d9be85c57d4ba21e683 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -56,7 +56,7 @@ static void flush_work_handle(struct work_struct *work) if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) { ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL); if (ret) - dev_err(dev, "Modify QP to error state failed(%d) during CQE flush\n", + dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n", ret); } @@ -105,7 +105,7 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) xa_unlock(&hr_dev->qp_table_xa); if (!qp) { - dev_warn(dev, "Async event for bogus QP %08x\n", qpn); + dev_warn(dev, "async event for bogus QP %08x\n", qpn); return; } @@ -218,7 +218,6 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) if (hr_qp->ibqp.qp_type == IB_QPT_GSI) { num = 1; - hr_qp->doorbell_qpn = 1; } else { mutex_lock(&qp_table->bank_mutex); bankid = get_least_load_bankid_for_qp(qp_table->bank); @@ -234,8 +233,6 @@ static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) qp_table->bank[bankid].inuse++; mutex_unlock(&qp_table->bank_mutex); - - hr_qp->doorbell_qpn = (u32)num; } hr_qp->qpn = num; @@ -278,7 +275,7 @@ static int hns_roce_qp_store(struct hns_roce_dev *hr_dev, ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL)); if (ret) - dev_err(hr_dev->dev, "Failed to xa store for QPC\n"); + dev_err(hr_dev->dev, "failed to xa store for QPC\n"); else /* add QP to device's QP list for softwc */ add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq, @@ -299,14 +296,14 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) /* Alloc memory for QPC */ ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn); if (ret) { - dev_err(dev, "Failed to get QPC table\n"); + dev_err(dev, "failed to get QPC table\n"); goto err_out; } /* Alloc memory for IRRL */ ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn); if (ret) { - dev_err(dev, "Failed to get IRRL table\n"); + dev_err(dev, "failed to get IRRL table\n"); goto err_put_qp; } @@ -315,7 +312,7 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table, hr_qp->qpn); if (ret) { - dev_err(dev, "Failed to get TRRL table\n"); + dev_err(dev, "failed to get TRRL table\n"); goto err_put_irrl; } } @@ -325,7 +322,7 @@ static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table, hr_qp->qpn); if (ret) { - dev_err(dev, "Failed to get SCC CTX table\n"); + dev_err(dev, "failed to get SCC CTX table\n"); goto err_put_trrl; } } @@ -462,11 +459,8 @@ static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) + hr_qp->rq.rsv_sge); - if (hr_dev->caps.max_rq_sg <= HNS_ROCE_SGE_IN_WQE) - hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz); - else - hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * - hr_qp->rq.max_gs); + hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz * + hr_qp->rq.max_gs); hr_qp->rq.wqe_cnt = cnt; if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE && @@ -1209,7 +1203,7 @@ int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr, ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp); if (ret) - ibdev_err(ibdev, "Create QP type 0x%x failed(%d)\n", + ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n", init_attr->qp_type, ret); return ret; diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c index 24a154d646304de3b2bd48bd416568da2324542f..989a2af2e93829371aff1aa024daae382bcb9f03 100644 --- a/drivers/infiniband/hw/hns/hns_roce_restrack.c +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c @@ -9,91 +9,223 @@ #include "hns_roce_device.h" #include "hns_roce_hw_v2.h" -static int hns_roce_fill_cq(struct sk_buff *msg, - struct hns_roce_v2_cq_context *context) +#define MAX_ENTRY_NUM 256 + +int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq) { - if (rdma_nl_put_driver_u32(msg, "state", - hr_reg_read(context, CQC_ARM_ST))) + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + struct nlattr *table_attr; + + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER); + if (!table_attr) + return -EMSGSIZE; + if (rdma_nl_put_driver_u32(msg, "cq_depth", hr_cq->cq_depth)) goto err; - if (rdma_nl_put_driver_u32(msg, "ceqn", - hr_reg_read(context, CQC_CEQN))) + if (rdma_nl_put_driver_u32(msg, "cons_index", hr_cq->cons_index)) goto err; - if (rdma_nl_put_driver_u32(msg, "cqn", - hr_reg_read(context, CQC_CQN))) + if (rdma_nl_put_driver_u32(msg, "cqe_size", hr_cq->cqe_size)) goto err; - if (rdma_nl_put_driver_u32(msg, "hopnum", - hr_reg_read(context, CQC_CQE_HOP_NUM))) + if (rdma_nl_put_driver_u32(msg, "arm_sn", hr_cq->arm_sn)) goto err; - if (rdma_nl_put_driver_u32(msg, "pi", - hr_reg_read(context, CQC_CQ_PRODUCER_IDX))) + nla_nest_end(msg, table_attr); + + return 0; + +err: + nla_nest_cancel(msg, table_attr); + + return -EMSGSIZE; +} + +int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); + struct hns_roce_v2_cq_context context; + u32 data[MAX_ENTRY_NUM] = {}; + int offset = 0; + int ret; + + if (!hr_dev->hw->query_cqc) + return -EINVAL; + + ret = hr_dev->hw->query_cqc(hr_dev, hr_cq->cqn, &context); + if (ret) + return -EINVAL; + + data[offset++] = hr_reg_read(&context, CQC_CQ_ST); + data[offset++] = hr_reg_read(&context, CQC_SHIFT); + data[offset++] = hr_reg_read(&context, CQC_CQE_SIZE); + data[offset++] = hr_reg_read(&context, CQC_CQE_CNT); + data[offset++] = hr_reg_read(&context, CQC_CQ_PRODUCER_IDX); + data[offset++] = hr_reg_read(&context, CQC_CQ_CONSUMER_IDX); + data[offset++] = hr_reg_read(&context, CQC_DB_RECORD_EN); + data[offset++] = hr_reg_read(&context, CQC_ARM_ST); + data[offset++] = hr_reg_read(&context, CQC_CMD_SN); + data[offset++] = hr_reg_read(&context, CQC_CEQN); + data[offset++] = hr_reg_read(&context, CQC_CQ_MAX_CNT); + data[offset++] = hr_reg_read(&context, CQC_CQ_PERIOD); + data[offset++] = hr_reg_read(&context, CQC_CQE_HOP_NUM); + data[offset++] = hr_reg_read(&context, CQC_CQE_BAR_PG_SZ); + data[offset++] = hr_reg_read(&context, CQC_CQE_BUF_PG_SZ); + + ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data); + + return ret; +} + +int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp) +{ + struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp); + struct nlattr *table_attr; + + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER); + if (!table_attr) + return -EMSGSIZE; + + if (rdma_nl_put_driver_u32_hex(msg, "sq_wqe_cnt", hr_qp->sq.wqe_cnt)) goto err; - if (rdma_nl_put_driver_u32(msg, "ci", - hr_reg_read(context, CQC_CQ_CONSUMER_IDX))) + if (rdma_nl_put_driver_u32_hex(msg, "sq_max_gs", hr_qp->sq.max_gs)) goto err; - if (rdma_nl_put_driver_u32(msg, "coalesce", - hr_reg_read(context, CQC_CQ_MAX_CNT))) + if (rdma_nl_put_driver_u32_hex(msg, "rq_wqe_cnt", hr_qp->rq.wqe_cnt)) goto err; - if (rdma_nl_put_driver_u32(msg, "period", - hr_reg_read(context, CQC_CQ_PERIOD))) + if (rdma_nl_put_driver_u32_hex(msg, "rq_max_gs", hr_qp->rq.max_gs)) goto err; - if (rdma_nl_put_driver_u32(msg, "cnt", - hr_reg_read(context, CQC_CQE_CNT))) + if (rdma_nl_put_driver_u32_hex(msg, "ext_sge_sge_cnt", hr_qp->sge.sge_cnt)) goto err; + nla_nest_end(msg, table_attr); + return 0; err: + nla_nest_cancel(msg, table_attr); + return -EMSGSIZE; } -int hns_roce_fill_res_cq_entry(struct sk_buff *msg, - struct ib_cq *ib_cq) +int hns_roce_fill_res_qp_entry_raw(struct sk_buff *msg, struct ib_qp *ib_qp) { - struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); - struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); - struct hns_roce_v2_cq_context *context; - struct nlattr *table_attr; + struct hns_roce_dev *hr_dev = to_hr_dev(ib_qp->device); + struct hns_roce_qp *hr_qp = to_hr_qp(ib_qp); + struct hns_roce_v2_qp_context context; + u32 data[MAX_ENTRY_NUM] = {}; + int offset = 0; int ret; - if (!hr_dev->dfx->query_cqc_info) + if (!hr_dev->hw->query_qpc) return -EINVAL; - context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL); - if (!context) - return -ENOMEM; - - ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context); + ret = hr_dev->hw->query_qpc(hr_dev, hr_qp->qpn, &context); if (ret) - goto err; + return -EINVAL; + + data[offset++] = hr_reg_read(&context, QPC_QP_ST); + data[offset++] = hr_reg_read(&context, QPC_ERR_TYPE); + data[offset++] = hr_reg_read(&context, QPC_CHECK_FLG); + data[offset++] = hr_reg_read(&context, QPC_SRQ_EN); + data[offset++] = hr_reg_read(&context, QPC_SRQN); + data[offset++] = hr_reg_read(&context, QPC_QKEY_XRCD); + data[offset++] = hr_reg_read(&context, QPC_TX_CQN); + data[offset++] = hr_reg_read(&context, QPC_RX_CQN); + data[offset++] = hr_reg_read(&context, QPC_SQ_PRODUCER_IDX); + data[offset++] = hr_reg_read(&context, QPC_SQ_CONSUMER_IDX); + data[offset++] = hr_reg_read(&context, QPC_RQ_RECORD_EN); + data[offset++] = hr_reg_read(&context, QPC_RQ_PRODUCER_IDX); + data[offset++] = hr_reg_read(&context, QPC_RQ_CONSUMER_IDX); + data[offset++] = hr_reg_read(&context, QPC_SQ_SHIFT); + data[offset++] = hr_reg_read(&context, QPC_RQWS); + data[offset++] = hr_reg_read(&context, QPC_RQ_SHIFT); + data[offset++] = hr_reg_read(&context, QPC_SGE_SHIFT); + data[offset++] = hr_reg_read(&context, QPC_SQ_HOP_NUM); + data[offset++] = hr_reg_read(&context, QPC_RQ_HOP_NUM); + data[offset++] = hr_reg_read(&context, QPC_SGE_HOP_NUM); + data[offset++] = hr_reg_read(&context, QPC_WQE_SGE_BA_PG_SZ); + data[offset++] = hr_reg_read(&context, QPC_WQE_SGE_BUF_PG_SZ); + data[offset++] = hr_reg_read(&context, QPC_RETRY_NUM_INIT); + data[offset++] = hr_reg_read(&context, QPC_RETRY_CNT); + data[offset++] = hr_reg_read(&context, QPC_SQ_CUR_PSN); + data[offset++] = hr_reg_read(&context, QPC_SQ_MAX_PSN); + data[offset++] = hr_reg_read(&context, QPC_SQ_FLUSH_IDX); + data[offset++] = hr_reg_read(&context, QPC_SQ_MAX_IDX); + data[offset++] = hr_reg_read(&context, QPC_SQ_TX_ERR); + data[offset++] = hr_reg_read(&context, QPC_SQ_RX_ERR); + data[offset++] = hr_reg_read(&context, QPC_RQ_RX_ERR); + data[offset++] = hr_reg_read(&context, QPC_RQ_TX_ERR); + data[offset++] = hr_reg_read(&context, QPC_RQ_CQE_IDX); + data[offset++] = hr_reg_read(&context, QPC_RQ_RTY_TX_ERR); + + ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data); + + return ret; +} + +int hns_roce_fill_res_mr_entry(struct sk_buff *msg, struct ib_mr *ib_mr) +{ + struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr); + struct nlattr *table_attr; table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER); - if (!table_attr) { - ret = -EMSGSIZE; + if (!table_attr) + return -EMSGSIZE; + + if (rdma_nl_put_driver_u32_hex(msg, "pbl_hop_num", hr_mr->pbl_hop_num)) goto err; - } - if (hns_roce_fill_cq(msg, context)) { - ret = -EMSGSIZE; - goto err_cancel_table; - } + if (rdma_nl_put_driver_u32_hex(msg, "ba_pg_shift", + hr_mr->pbl_mtr.hem_cfg.ba_pg_shift)) + goto err; + + if (rdma_nl_put_driver_u32_hex(msg, "buf_pg_shift", + hr_mr->pbl_mtr.hem_cfg.buf_pg_shift)) + goto err; nla_nest_end(msg, table_attr); - kfree(context); return 0; -err_cancel_table: - nla_nest_cancel(msg, table_attr); err: - kfree(context); + nla_nest_cancel(msg, table_attr); + + return -EMSGSIZE; +} + +int hns_roce_fill_res_mr_entry_raw(struct sk_buff *msg, struct ib_mr *ib_mr) +{ + struct hns_roce_dev *hr_dev = to_hr_dev(ib_mr->device); + struct hns_roce_mr *hr_mr = to_hr_mr(ib_mr); + struct hns_roce_v2_mpt_entry context; + u32 data[MAX_ENTRY_NUM] = {}; + int offset = 0; + int ret; + + if (!hr_dev->hw->query_mpt) + return -EINVAL; + + ret = hr_dev->hw->query_mpt(hr_dev, hr_mr->key, &context); + if (ret) + return -EINVAL; + + data[offset++] = hr_reg_read(&context, MPT_ST); + data[offset++] = hr_reg_read(&context, MPT_PD); + data[offset++] = hr_reg_read(&context, MPT_LKEY); + data[offset++] = hr_reg_read(&context, MPT_LEN_L); + data[offset++] = hr_reg_read(&context, MPT_LEN_H); + data[offset++] = hr_reg_read(&context, MPT_PBL_SIZE); + data[offset++] = hr_reg_read(&context, MPT_PBL_HOP_NUM); + data[offset++] = hr_reg_read(&context, MPT_PBL_BA_PG_SZ); + data[offset++] = hr_reg_read(&context, MPT_PBL_BUF_PG_SZ); + + ret = nla_put(msg, RDMA_NLDEV_ATTR_RES_RAW, offset * sizeof(u32), data); + return ret; } diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h index e03e03082a5fb0aa3c414f7c62fff33c829e1446..c1906cab5c8ad1802f70bd99ab75ca9f9edcecc2 100644 --- a/drivers/infiniband/hw/irdma/defs.h +++ b/drivers/infiniband/hw/irdma/defs.h @@ -314,6 +314,7 @@ enum irdma_cqp_op_type { #define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d #define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e #define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220 +#define IRDMA_AE_INVALID_REQUEST 0x0223 #define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301 #define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303 #define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304 diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index 4f132c6fb653bc1ca351af9484ae1d5501dd546b..ab246447520bd22d3ad392b722e64ecd82379e86 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -138,59 +138,68 @@ static void irdma_set_flush_fields(struct irdma_sc_qp *qp, qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; switch (info->ae_id) { - case IRDMA_AE_AMP_UNALLOCATED_STAG: case IRDMA_AE_AMP_BOUNDS_VIOLATION: case IRDMA_AE_AMP_INVALID_STAG: - qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; - fallthrough; + case IRDMA_AE_AMP_RIGHTS_VIOLATION: + case IRDMA_AE_AMP_UNALLOCATED_STAG: case IRDMA_AE_AMP_BAD_PD: - case IRDMA_AE_UDA_XMIT_BAD_PD: + case IRDMA_AE_AMP_BAD_QP: + case IRDMA_AE_AMP_BAD_STAG_KEY: + case IRDMA_AE_AMP_BAD_STAG_INDEX: + case IRDMA_AE_AMP_TO_WRAP: + case IRDMA_AE_PRIV_OPERATION_DENIED: qp->flush_code = FLUSH_PROT_ERR; + qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; break; - case IRDMA_AE_AMP_BAD_QP: + case IRDMA_AE_UDA_XMIT_BAD_PD: case IRDMA_AE_WQE_UNEXPECTED_OPCODE: qp->flush_code = FLUSH_LOC_QP_OP_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; + break; + case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: + case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: + case IRDMA_AE_UDA_L4LEN_INVALID: + case IRDMA_AE_DDP_UBE_INVALID_MO: + case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: + qp->flush_code = FLUSH_LOC_LEN_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; - case IRDMA_AE_AMP_BAD_STAG_KEY: - case IRDMA_AE_AMP_BAD_STAG_INDEX: - case IRDMA_AE_AMP_TO_WRAP: - case IRDMA_AE_AMP_RIGHTS_VIOLATION: case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case IRDMA_AE_PRIV_OPERATION_DENIED: - case IRDMA_AE_IB_INVALID_REQUEST: case IRDMA_AE_IB_REMOTE_ACCESS_ERROR: qp->flush_code = FLUSH_REM_ACCESS_ERR; qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; break; case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: - case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: - case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT: - case IRDMA_AE_UDA_L4LEN_INVALID: + case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: case IRDMA_AE_ROCE_RSP_LENGTH_ERROR: - qp->flush_code = FLUSH_LOC_LEN_ERR; + case IRDMA_AE_IB_REMOTE_OP_ERROR: + qp->flush_code = FLUSH_REM_OP_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; case IRDMA_AE_LCE_QP_CATASTROPHIC: qp->flush_code = FLUSH_FATAL_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; - case IRDMA_AE_DDP_UBE_INVALID_MO: case IRDMA_AE_IB_RREQ_AND_Q1_FULL: - case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: qp->flush_code = FLUSH_GENERAL_ERR; break; case IRDMA_AE_LLP_TOO_MANY_RETRIES: qp->flush_code = FLUSH_RETRY_EXC_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS: case IRDMA_AE_AMP_MWBIND_BIND_DISABLED: case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS: qp->flush_code = FLUSH_MW_BIND_ERR; + qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR; break; - case IRDMA_AE_IB_REMOTE_OP_ERROR: - qp->flush_code = FLUSH_REM_OP_ERR; + case IRDMA_AE_IB_INVALID_REQUEST: + qp->flush_code = FLUSH_REM_INV_REQ_ERR; + qp->event_type = IRDMA_QP_EVENT_REQ_ERR; break; default: - qp->flush_code = FLUSH_FATAL_ERR; + qp->flush_code = FLUSH_GENERAL_ERR; + qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC; break; } } diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h index 9e7b8ecb137ab2b21b3f3758a817bf01c9cf711a..517d41a1c2894c1f1244cc8857ccd67d83835c5b 100644 --- a/drivers/infiniband/hw/irdma/type.h +++ b/drivers/infiniband/hw/irdma/type.h @@ -98,6 +98,7 @@ enum irdma_term_mpa_errors { enum irdma_qp_event_type { IRDMA_QP_EVENT_CATASTROPHIC, IRDMA_QP_EVENT_ACCESS_ERR, + IRDMA_QP_EVENT_REQ_ERR, }; enum irdma_hw_stats_index_32b { diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index daeab5daed5bc78e7d3a6abbac164b9cf1b06f88..a6e5d350a94ce8f1e3499fe736bd56cb4abf1fce 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -497,7 +497,8 @@ int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); i = 0; } else { - qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->sg_list, + qp->wqe_ops.iw_set_fragment(wqe, 0, + frag_cnt ? op_info->sg_list : NULL, qp->swqe_polarity); i = 1; } @@ -1005,6 +1006,7 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, int ret_code; bool move_cq_head = true; u8 polarity; + u8 op_type; bool ext_valid; __le64 *ext_cqe; @@ -1187,7 +1189,6 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, do { __le64 *sw_wqe; u64 wqe_qword; - u8 op_type; u32 tail; tail = qp->sq_ring.tail; @@ -1204,6 +1205,8 @@ int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq, break; } } while (1); + if (op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR) + info->minor_err = FLUSH_MW_BIND_ERR; qp->sq_flush_seen = true; if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) qp->sq_flush_complete = true; diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h index ddd0ebbdd7d54c568fccaa6eb77e3b9e5de6b926..2ef61923c92685a5164008eceff7fcb246b664df 100644 --- a/drivers/infiniband/hw/irdma/user.h +++ b/drivers/infiniband/hw/irdma/user.h @@ -103,6 +103,7 @@ enum irdma_flush_opcode { FLUSH_FATAL_ERR, FLUSH_RETRY_EXC_ERR, FLUSH_MW_BIND_ERR, + FLUSH_REM_INV_REQ_ERR, }; enum irdma_cmpl_status { diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index fdf4cc88cb91201d57e520bcb8b223253bebcc51..8dfc9e154d733aba6dbfc47b82276dfdfe710c75 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -590,11 +590,14 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_error = cqp_request->compl_info.error; if (cqp_error) { err_code = -EIO; - if (cqp_request->compl_info.maj_err_code == 0xFFFF && - cqp_request->compl_info.min_err_code == 0x8029) { - if (!rf->reset) { - rf->reset = true; - rf->gen_ops.request_reset(rf); + if (cqp_request->compl_info.maj_err_code == 0xFFFF) { + if (cqp_request->compl_info.min_err_code == 0x8002) + err_code = -EBUSY; + else if (cqp_request->compl_info.min_err_code == 0x8029) { + if (!rf->reset) { + rf->reset = true; + rf->gen_ops.request_reset(rf); + } } } } @@ -2476,6 +2479,9 @@ void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event) case IRDMA_QP_EVENT_ACCESS_ERR: ibevent.event = IB_EVENT_QP_ACCESS_ERR; break; + case IRDMA_QP_EVENT_REQ_ERR: + ibevent.event = IB_EVENT_QP_REQ_ERR; + break; } ibevent.device = iwqp->ibqp.device; ibevent.element.qp = &iwqp->ibqp; @@ -2598,7 +2604,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) spin_unlock_irqrestore(&iwqp->lock, flags2); spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); if (compl_generated) - irdma_comp_handler(iwqp->iwrcq); + irdma_comp_handler(iwqp->iwscq); } else { spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 9b07b8af2997fbf7e9985d45ef0f9a86aaf391fb..a22afbb25bc58d3870e15f09cddbd3dc4880052a 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -39,15 +39,18 @@ static int irdma_query_device(struct ib_device *ibdev, props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; props->max_cq = rf->max_cq - rf->used_cqs; - props->max_cqe = rf->max_cqe; + props->max_cqe = rf->max_cqe - 1; props->max_mr = rf->max_mr - rf->used_mrs; props->max_mw = props->max_mr; props->max_pd = rf->max_pd - rf->used_pds; props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; props->max_qp_rd_atom = hw_attrs->max_hw_ird; props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; - if (rdma_protocol_roce(ibdev, 1)) + if (rdma_protocol_roce(ibdev, 1)) { + props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; props->max_pkeys = IRDMA_PKEY_TBL_SZ; + } + props->max_ah = rf->max_ah; props->max_mcast_grp = rf->max_mcg; props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; @@ -296,13 +299,19 @@ static void irdma_alloc_push_page(struct irdma_qp *iwqp) static int irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { +#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8) +#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd) struct ib_device *ibdev = uctx->device; struct irdma_device *iwdev = to_iwdev(ibdev); - struct irdma_alloc_ucontext_req req; + struct irdma_alloc_ucontext_req req = {}; struct irdma_alloc_ucontext_resp uresp = {}; struct irdma_ucontext *ucontext = to_ucontext(uctx); struct irdma_uk_attrs *uk_attrs; + if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || + udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) + return -EINVAL; + if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; @@ -314,7 +323,7 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx, uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; /* GEN_1 legacy support with libi40iw */ - if (udata->outlen < sizeof(uresp)) { + if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { if (uk_attrs->hw_rev != IRDMA_GEN_1) return -EOPNOTSUPP; @@ -386,6 +395,7 @@ static void irdma_dealloc_ucontext(struct ib_ucontext *context) */ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) { +#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd) struct irdma_pd *iwpd = to_iwpd(pd); struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; @@ -395,6 +405,9 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) u32 pd_id = 0; int err; + if (udata && udata->outlen < IRDMA_ALLOC_PD_MIN_RESP_LEN) + return -EINVAL; + err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, &rf->next_pd); if (err) @@ -811,12 +824,14 @@ static int irdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { +#define IRDMA_CREATE_QP_MIN_REQ_LEN offsetofend(struct irdma_create_qp_req, user_compl_ctx) +#define IRDMA_CREATE_QP_MIN_RESP_LEN offsetofend(struct irdma_create_qp_resp, rsvd) struct ib_pd *ibpd = ibqp->pd; struct irdma_pd *iwpd = to_iwpd(ibpd); struct irdma_device *iwdev = to_iwdev(ibpd->device); struct irdma_pci_f *rf = iwdev->rf; struct irdma_qp *iwqp = to_iwqp(ibqp); - struct irdma_create_qp_req req; + struct irdma_create_qp_req req = {}; struct irdma_create_qp_resp uresp = {}; u32 qp_num = 0; int err_code; @@ -833,6 +848,10 @@ static int irdma_create_qp(struct ib_qp *ibqp, if (err_code) return err_code; + if (udata && (udata->inlen < IRDMA_CREATE_QP_MIN_REQ_LEN || + udata->outlen < IRDMA_CREATE_QP_MIN_RESP_LEN)) + return -EINVAL; + sq_size = init_attr->cap.max_send_wr; rq_size = init_attr->cap.max_recv_wr; @@ -1117,6 +1136,8 @@ static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { +#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) +#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_pd *iwpd = to_iwpd(ibqp->pd); struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; @@ -1135,6 +1156,13 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, roce_info = &iwqp->roce_info; udp_info = &iwqp->udp_info; + if (udata) { + /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ + if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || + (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) + return -EINVAL; + } + if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; @@ -1371,7 +1399,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { spin_unlock_irqrestore(&iwqp->lock, flags); - if (udata) { + if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) return -EINVAL; @@ -1423,7 +1451,7 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, } else { iwqp->ibqp_state = attr->qp_state; } - if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { + if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { struct irdma_ucontext *ucontext; ucontext = rdma_udata_to_drv_context(udata, @@ -1463,6 +1491,8 @@ exit: int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { +#define IRDMA_MODIFY_QP_MIN_REQ_LEN offsetofend(struct irdma_modify_qp_req, rq_flush) +#define IRDMA_MODIFY_QP_MIN_RESP_LEN offsetofend(struct irdma_modify_qp_resp, push_valid) struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; @@ -1477,6 +1507,13 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, int err; unsigned long flags; + if (udata) { + /* udata inlen/outlen can be 0 when supporting legacy libi40iw */ + if ((udata->inlen && udata->inlen < IRDMA_MODIFY_QP_MIN_REQ_LEN) || + (udata->outlen && udata->outlen < IRDMA_MODIFY_QP_MIN_RESP_LEN)) + return -EINVAL; + } + if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) return -EOPNOTSUPP; @@ -1562,7 +1599,7 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { spin_unlock_irqrestore(&iwqp->lock, flags); - if (udata) { + if (udata && udata->inlen) { if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) return -EINVAL; @@ -1659,7 +1696,7 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, } } } - if (attr_mask & IB_QP_STATE && udata && + if (attr_mask & IB_QP_STATE && udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { struct irdma_ucontext *ucontext; @@ -1794,6 +1831,7 @@ static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) static int irdma_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) { +#define IRDMA_RESIZE_CQ_MIN_REQ_LEN offsetofend(struct irdma_resize_cq_req, user_cq_buffer) struct irdma_cq *iwcq = to_iwcq(ibcq); struct irdma_sc_dev *dev = iwcq->sc_cq.dev; struct irdma_cqp_request *cqp_request; @@ -1816,6 +1854,9 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, IRDMA_FEATURE_CQ_RESIZE)) return -EOPNOTSUPP; + if (udata && udata->inlen < IRDMA_RESIZE_CQ_MIN_REQ_LEN) + return -EINVAL; + if (entries > rf->max_cqe) return -EINVAL; @@ -1948,6 +1989,8 @@ static int irdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { +#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf) +#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size) struct ib_device *ibdev = ibcq->device; struct irdma_device *iwdev = to_iwdev(ibdev); struct irdma_pci_f *rf = iwdev->rf; @@ -1966,6 +2009,11 @@ static int irdma_create_cq(struct ib_cq *ibcq, err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); if (err_code) return err_code; + + if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN || + udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN)) + return -EINVAL; + err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, &rf->next_cq); if (err_code) @@ -2743,6 +2791,7 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, u64 virt, int access, struct ib_udata *udata) { +#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_ucontext *ucontext; struct irdma_pble_alloc *palloc; @@ -2760,6 +2809,9 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return ERR_PTR(-EINVAL); + if (udata->inlen < IRDMA_MEM_REG_MIN_REQ_LEN) + return ERR_PTR(-EINVAL); + region = ib_umem_get(pd->device, start, len, access); if (IS_ERR(region)) { @@ -3009,6 +3061,7 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; + int status; if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { if (iwmr->region) { @@ -3039,8 +3092,11 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) cqp_info->post_sq = 1; cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; - irdma_handle_cqp_op(iwdev->rf, cqp_request); + status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); + if (status) + return status; + irdma_free_stag(iwdev, iwmr->stag); done: if (iwpbl->pbl_allocated) @@ -3308,6 +3364,8 @@ static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode return IB_WC_RETRY_EXC_ERR; case FLUSH_MW_BIND_ERR: return IB_WC_MW_BIND_ERR; + case FLUSH_REM_INV_REQ_ERR: + return IB_WC_REM_INV_REQ_ERR; case FLUSH_FATAL_ERR: default: return IB_WC_FATAL_ERR; @@ -4289,12 +4347,16 @@ static int irdma_create_user_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, struct ib_udata *udata) { +#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd) struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); struct irdma_device *iwdev = to_iwdev(ibah->pd->device); struct irdma_create_ah_resp uresp; struct irdma_ah *parent_ah; int err; + if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) + return -EINVAL; + err = irdma_setup_ah(ibah, attr); if (err) return err; diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index d13ecbdd439171f36103b6abded2307bfca5d61e..a37cfac5e23f96c3c1642d3463dddb2895b07231 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -96,7 +96,7 @@ static void __propagate_pkey_ev(struct mlx4_ib_dev *dev, int port_num, __be64 mlx4_ib_gen_node_guid(void) { #define NODE_GUID_HI ((u64) (((u64)IB_OPENIB_OUI) << 40)) - return cpu_to_be64(NODE_GUID_HI | prandom_u32()); + return cpu_to_be64(NODE_GUID_HI | get_random_u32()); } __be64 mlx4_ib_get_new_demux_tid(struct mlx4_ib_demux_ctx *ctx) diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 04a67b48160863e4b4dd83c032ebf2cc1d2f1dcc..a40bf58bcdd3ae0797199f2f5d51d311b0e4931a 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -439,7 +439,6 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, goto err_mr; mr->ibmr.rkey = mr->ibmr.lkey = mr->mmr.key; - mr->ibmr.length = length; mr->ibmr.page_size = 1U << shift; return &mr->ibmr; diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 2a2a9e9afc9dadfae51a602353c381b78f3c26e1..2211a0be16f36c32a6e1132503e97a082cc390ac 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -907,6 +907,7 @@ static bool devx_is_whitelist_cmd(void *in) case MLX5_CMD_OP_QUERY_HCA_CAP: case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT: case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT: + case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: return true; default: return false; @@ -962,6 +963,7 @@ static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev) case MLX5_CMD_OP_QUERY_CONG_PARAMS: case MLX5_CMD_OP_QUERY_CONG_STATISTICS: case MLX5_CMD_OP_QUERY_LAG: + case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS: return true; default: return false; @@ -2158,32 +2160,39 @@ err: static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, struct uverbs_attr_bundle *attrs, - struct devx_umem *obj) + struct devx_umem *obj, u32 access_flags) { u64 addr; size_t size; - u32 access; int err; if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) || uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN)) return -EFAULT; - err = uverbs_get_flags32(&access, attrs, - MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, - IB_ACCESS_LOCAL_WRITE | - IB_ACCESS_REMOTE_WRITE | - IB_ACCESS_REMOTE_READ); + err = ib_check_mr_access(&dev->ib_dev, access_flags); if (err) return err; - err = ib_check_mr_access(&dev->ib_dev, access); - if (err) - return err; + if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) { + struct ib_umem_dmabuf *umem_dmabuf; + int dmabuf_fd; + + err = uverbs_get_raw_fd(&dmabuf_fd, attrs, + MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD); + if (err) + return -EFAULT; - obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access); - if (IS_ERR(obj->umem)) - return PTR_ERR(obj->umem); + umem_dmabuf = ib_umem_dmabuf_get_pinned( + &dev->ib_dev, addr, size, dmabuf_fd, access_flags); + if (IS_ERR(umem_dmabuf)) + return PTR_ERR(umem_dmabuf); + obj->umem = &umem_dmabuf->umem; + } else { + obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags); + if (IS_ERR(obj->umem)) + return PTR_ERR(obj->umem); + } return 0; } @@ -2222,7 +2231,8 @@ static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem, static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev, struct uverbs_attr_bundle *attrs, struct devx_umem *obj, - struct devx_umem_reg_cmd *cmd) + struct devx_umem_reg_cmd *cmd, + int access) { unsigned long pgsz_bitmap; unsigned int page_size; @@ -2271,6 +2281,9 @@ static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev, MLX5_SET(umem, umem, page_offset, ib_umem_dma_offset(obj->umem, page_size)); + if (mlx5_umem_needs_ats(dev, obj->umem, access)) + MLX5_SET(umem, umem, ats, 1); + mlx5_ib_populate_pas(obj->umem, page_size, mtt, (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) | MLX5_IB_MTT_READ); @@ -2288,20 +2301,30 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)( struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); + int access_flags; int err; if (!c->devx_uid) return -EINVAL; + err = uverbs_get_flags32(&access_flags, attrs, + MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, + IB_ACCESS_LOCAL_WRITE | + IB_ACCESS_REMOTE_WRITE | + IB_ACCESS_REMOTE_READ | + IB_ACCESS_RELAXED_ORDERING); + if (err) + return err; + obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL); if (!obj) return -ENOMEM; - err = devx_umem_get(dev, &c->ibucontext, attrs, obj); + err = devx_umem_get(dev, &c->ibucontext, attrs, obj, access_flags); if (err) goto err_obj_free; - err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd); + err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd, access_flags); if (err) goto err_umem_release; @@ -2833,6 +2856,8 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN, UVERBS_ATTR_TYPE(u64), UA_MANDATORY), + UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD, + UA_OPTIONAL), UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, enum ib_access_flags), UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP, diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index 293ed709e5ed5dd2efc8c72fa25137cb21b60ad4..9c8a7b206dcf470473fd6eab945a271d58b35c6d 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -147,6 +147,28 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt, vl_15_dropped); } +static int query_ib_ppcnt(struct mlx5_core_dev *dev, u8 port_num, void *out, + size_t sz) +{ + u32 *in; + int err; + + in = kvzalloc(sz, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + return err; + } + + MLX5_SET(ppcnt_reg, in, local_port, port_num); + + MLX5_SET(ppcnt_reg, in, grp, MLX5_INFINIBAND_PORT_COUNTERS_GROUP); + err = mlx5_core_access_reg(dev, in, sz, out, + sz, MLX5_REG_PPCNT, 0, 0); + + kvfree(in); + return err; +} + static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num, const struct ib_mad *in_mad, struct ib_mad *out_mad) { @@ -166,6 +188,12 @@ static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num, mdev = dev->mdev; mdev_port_num = 1; } + if (MLX5_CAP_GEN(dev->mdev, num_ports) == 1) { + /* set local port to one for Function-Per-Port HCA. */ + mdev = dev->mdev; + mdev_port_num = 1; + } + /* Declaring support of extended counters */ if (in_mad->mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) { struct ib_class_port_info cpi = {}; @@ -202,8 +230,7 @@ static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num, goto done; } - err = mlx5_core_query_ib_ppcnt(mdev, mdev_port_num, - out_cnt, sz); + err = query_ib_ppcnt(mdev, mdev_port_num, out_cnt, sz); if (!err) pma_cnt_assign(pma_cnt, out_cnt); } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index fc94a1b25485d7ba65cada9af61496e1fe89d432..c669ef6e47e730dffa584e162c6fbf18ce6204d2 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include #include @@ -46,7 +46,6 @@ #include #include #include -#include #define UVERBS_MODULE_NAME mlx5_ib #include @@ -1826,6 +1825,9 @@ static int set_ucontext_resp(struct ib_ucontext *uctx, if (MLX5_CAP_GEN(dev->mdev, drain_sigerr)) resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS; + resp->comp_mask |= + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG; + return 0; } @@ -4336,7 +4338,7 @@ static int mlx5r_probe(struct auxiliary_device *adev, dev->mdev = mdev; dev->num_ports = num_ports; - if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_is_roce_init_enabled(mdev)) + if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev)) profile = &raw_eth_profile; else profile = &pf_profile; diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 6191aa833ac2b1ad8796dd618fa9a630eb552440..96ffbbaf0a73d188aca25ccfe567f90bd05f3ad5 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include "mlx5_ib.h" #include @@ -152,6 +151,7 @@ static int post_send_nop(struct mlx5_ib_dev *dev, struct ib_qp *ibqp, u64 wr_id, for (i = 0; i < 8; i++) mlx5_write64(&mmio_wqe[i * 2], bf->bfreg->map + bf->offset + i * 8); + io_stop_wc(); bf->offset ^= bf->buf_size; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 2e2ad3918385832c62b38775b2dcc3c3b9005a80..4a7f7064bd0eb5221f49113cd8e52ed1b18af734 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -708,6 +708,7 @@ struct mlx5_ib_umr_context { }; enum { + MLX5_UMR_STATE_UNINIT, MLX5_UMR_STATE_ACTIVE, MLX5_UMR_STATE_RECOVER, MLX5_UMR_STATE_ERR, @@ -1540,6 +1541,18 @@ int mlx5_ib_test_wc(struct mlx5_ib_dev *dev); static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev) { + /* + * If the driver is in hash mode and the port_select_flow_table_bypass cap + * is supported, it means that the driver no longer needs to assign the port + * affinity by default. If a user wants to set the port affinity explicitly, + * the user has a dedicated API to do that, so there is no need to assign + * the port affinity by default. + */ + if (dev->lag_active && + mlx5_lag_mode_is_hash(dev->mdev) && + MLX5_CAP_PORT_SELECTION(dev->mdev, port_select_flow_table_bypass)) + return 0; + return dev->lag_active || (MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 && MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity)); @@ -1550,4 +1563,40 @@ static inline bool rt_supported(int ts_cap) return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_REAL_TIME || ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME; } + +/* + * PCI Peer to Peer is a trainwreck. If no switch is present then things + * sometimes work, depending on the pci_distance_p2p logic for excluding broken + * root complexes. However if a switch is present in the path, then things get + * really ugly depending on how the switch is setup. This table assumes that the + * root complex is strict and is validating that all req/reps are matches + * perfectly - so any scenario where it sees only half the transaction is a + * failure. + * + * CR/RR/DT ATS RO P2P + * 00X X X OK + * 010 X X fails (request is routed to root but root never sees comp) + * 011 0 X fails (request is routed to root but root never sees comp) + * 011 1 X OK + * 10X X 1 OK + * 101 X 0 fails (completion is routed to root but root didn't see req) + * 110 X 0 SLOW + * 111 0 0 SLOW + * 111 1 0 fails (completion is routed to root but root didn't see req) + * 111 1 1 OK + * + * Unfortunately we cannot reliably know if a switch is present or what the + * CR/RR/DT ACS settings are, as in a VM that is all hidden. Assume that + * CR/RR/DT is 111 if the ATS cap is enabled and follow the last three rows. + * + * For now assume if the umem is a dma_buf then it is P2P. + */ +static inline bool mlx5_umem_needs_ats(struct mlx5_ib_dev *dev, + struct ib_umem *umem, int access_flags) +{ + if (!MLX5_CAP_GEN(dev->mdev, ats) || !umem->is_dmabuf) + return false; + return access_flags & IB_ACCESS_RELAXED_ORDERING; +} + #endif /* MLX5_IB_H */ diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 129d531bd01bc8643302156f9347019e7f428cb7..410cc5fd25239db3a784360720048080ecc7d88b 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -39,9 +39,7 @@ #include #include #include -#include #include -#include #include "dm.h" #include "mlx5_ib.h" #include "umr.h" @@ -937,7 +935,8 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, * cache then synchronously create an uncached one. */ if (!ent || ent->limit == 0 || - !mlx5r_umr_can_reconfig(dev, 0, access_flags)) { + !mlx5r_umr_can_reconfig(dev, 0, access_flags) || + mlx5_umem_needs_ats(dev, umem, access_flags)) { mutex_lock(&dev->slow_path_mutex); mr = reg_create(pd, umem, iova, access_flags, page_size, false); mutex_unlock(&dev->slow_path_mutex); @@ -1018,6 +1017,8 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, MLX5_SET(mkc, mkc, translations_octword_size, get_octo_len(iova, umem->length, mr->page_shift)); MLX5_SET(mkc, mkc, log_page_size, mr->page_shift); + if (mlx5_umem_needs_ats(dev, umem, access_flags)) + MLX5_SET(mkc, mkc, ma_translation_mode, 1); if (populate) { MLX5_SET(create_mkey_in, in, translations_octword_actual_size, get_octo_len(iova, umem->length, mr->page_shift)); @@ -1402,7 +1403,6 @@ static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, upd_flags |= MLX5_IB_UPD_XLT_ACCESS; } - mr->ibmr.length = new_umem->length; mr->ibmr.iova = iova; mr->ibmr.length = new_umem->length; mr->page_shift = order_base_2(page_size); diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index e305bf1dc6c2256008c8f0473961aeba9298d004..bc97958818bb5ace9bad3da86af3a70761e0502f 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -30,7 +30,6 @@ * SOFTWARE. */ -#include #include #include #include @@ -795,7 +794,8 @@ static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key) { if (!mmkey) return false; - if (mmkey->type == MLX5_MKEY_MW) + if (mmkey->type == MLX5_MKEY_MW || + mmkey->type == MLX5_MKEY_INDIRECT_DEVX) return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); return mmkey->key == key; } diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index e00b94d1b1ea1e9c039d06548185a6ae4aa03ce0..d5105b5c9979b582083fed732fff25b0f5cc2462 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -177,6 +177,7 @@ int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev) sema_init(&dev->umrc.sem, MAX_UMR_WR); mutex_init(&dev->umrc.lock); + dev->umrc.state = MLX5_UMR_STATE_ACTIVE; return 0; @@ -191,6 +192,8 @@ destroy_pd: void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev) { + if (dev->umrc.state == MLX5_UMR_STATE_UNINIT) + return; ib_destroy_qp(dev->umrc.qp); ib_free_cq(dev->umrc.cq); ib_dealloc_pd(dev->umrc.pd); diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c index bdf5ed38de22054e34fd4b7fccfe0f7b9e4f075f..f330ce895d884943fc3694d684bce4ec61e144ac 100644 --- a/drivers/infiniband/hw/mthca/mthca_cmd.c +++ b/drivers/infiniband/hw/mthca/mthca_cmd.c @@ -1252,7 +1252,7 @@ static void get_board_id(void *vsd, char *board_id) if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { - strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MTHCA_BOARD_ID_LEN); + strscpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MTHCA_BOARD_ID_LEN); } else { /* * The board ID is a string but the firmware byte diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index 265a581133dcd3a6155ba9f15ee1bcd484e802e1..56f06c68f31ac1b495d29228d19b254ca20476f5 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1363,7 +1363,7 @@ static int ocrdma_mbx_get_ctrl_attribs(struct ocrdma_dev *dev) dev->hba_port_num = (hba_attribs->ptpnum_maxdoms_hbast_cv & OCRDMA_HBA_ATTRB_PTNUM_MASK) >> OCRDMA_HBA_ATTRB_PTNUM_SHIFT; - strlcpy(dev->model_number, + strscpy(dev->model_number, hba_attribs->controller_model_number, sizeof(dev->model_number)); } diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index 6861c6384f18ac083ac805c2aaa809650b9f9747..9d2dd135b78449bd318b47c9b1487e3300bb90ce 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -2124,7 +2124,7 @@ static void qib_7322_handle_hwerrors(struct qib_devdata *dd, char *msg, if (hwerrs & HWE_MASK(PowerOnBISTFailed)) { isfatal = 1; - strlcpy(msg, + strscpy(msg, "[Memory BIST test failed, InfiniPath hardware unusable]", msgl); /* ignore from now on, so disable until driver reloaded */ diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c index cb2a02d671e2beed1fb7c480a8c2c2916b00b323..692b64efad97b1dc0662229a69a72cc6d5d92c04 100644 --- a/drivers/infiniband/hw/qib/qib_pcie.c +++ b/drivers/infiniband/hw/qib/qib_pcie.c @@ -295,7 +295,7 @@ void qib_free_irq(struct qib_devdata *dd) * Setup pcie interrupt stuff again after a reset. I'd like to just call * pci_enable_msi() again for msi, but when I do that, * the MSI enable bit doesn't get set in the command word, and - * we switch to to a different interrupt vector, which is confusing, + * we switch to a different interrupt vector, which is confusing, * so I instead just do it all inline. Perhaps somehow can tie this * into the PCIe hotplug support at some point */ diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index 67a1b4562dc29fa3ce7ce72a7434ef2e39a40e12..67923ced6e2d193426d1ca720d1b5c55161df555 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -95,7 +95,6 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, int ret; int off; int i; - int flags; dma_addr_t pa; unsigned int gup_flags; struct mm_struct *mm; @@ -132,8 +131,6 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable, goto out; } - flags = IOMMU_READ | IOMMU_CACHE; - flags |= (writable) ? IOMMU_WRITE : 0; gup_flags = FOLL_WRITE; gup_flags |= (writable) ? 0 : FOLL_FORCE; cur_base = addr & PAGE_MASK; diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c index 59481ae395054441e2a7112819f4a43863d56067..d61f8de7f21c2b6625b1590dc50df139f6470552 100644 --- a/drivers/infiniband/sw/rdmavt/vt.c +++ b/drivers/infiniband/sw/rdmavt/vt.c @@ -15,7 +15,7 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("RDMA Verbs Transport Library"); -static int rvt_init(void) +static int __init rvt_init(void) { int ret = rvt_driver_cq_init(); @@ -26,7 +26,7 @@ static int rvt_init(void) } module_init(rvt_init); -static void rvt_cleanup(void) +static void __exit rvt_cleanup(void) { rvt_cq_exit(); } diff --git a/drivers/infiniband/sw/rxe/rxe_icrc.c b/drivers/infiniband/sw/rxe/rxe_icrc.c index e03af3012590447756d98bde325b9d4f599d2db1..46bb07c5c4df2532736a724e818c70534c748450 100644 --- a/drivers/infiniband/sw/rxe/rxe_icrc.c +++ b/drivers/infiniband/sw/rxe/rxe_icrc.c @@ -151,18 +151,8 @@ int rxe_icrc_check(struct sk_buff *skb, struct rxe_pkt_info *pkt) payload_size(pkt) + bth_pad(pkt)); icrc = ~icrc; - if (unlikely(icrc != pkt_icrc)) { - if (skb->protocol == htons(ETH_P_IPV6)) - pr_warn_ratelimited("bad ICRC from %pI6c\n", - &ipv6_hdr(skb)->saddr); - else if (skb->protocol == htons(ETH_P_IP)) - pr_warn_ratelimited("bad ICRC from %pI4\n", - &ip_hdr(skb)->saddr); - else - pr_warn_ratelimited("bad ICRC from unknown\n"); - + if (unlikely(icrc != pkt_icrc)) return -EINVAL; - } return 0; } diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 22f6cc31d1d6314df060b2ddaef82ca7d313c509..c2a5c8814a48bbefac18316fe81014a9cdad649f 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -64,10 +64,10 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); /* rxe_mr.c */ u8 rxe_get_next_key(u32 last_key); -void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr); -int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, +void rxe_mr_init_dma(int access, struct rxe_mr *mr); +int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access, struct rxe_mr *mr); -int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr); +int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr); int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length, enum rxe_mr_copy_dir dir); int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 850b80f5ad8bd1fb37a58a34f6c86ee0a866a8be..502e9ada99b307a09291a5c38a6d02d155e2187e 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -32,8 +32,8 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) case IB_MR_TYPE_USER: case IB_MR_TYPE_MEM_REG: - if (iova < mr->iova || length > mr->length || - iova > mr->iova + mr->length - length) + if (iova < mr->ibmr.iova || length > mr->ibmr.length || + iova > mr->ibmr.iova + mr->ibmr.length - length) return -EFAULT; return 0; @@ -103,17 +103,16 @@ err1: return -ENOMEM; } -void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr) +void rxe_mr_init_dma(int access, struct rxe_mr *mr) { rxe_mr_init(access, mr); - mr->ibmr.pd = &pd->ibpd; mr->access = access; mr->state = RXE_MR_STATE_VALID; mr->type = IB_MR_TYPE_DMA; } -int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, +int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access, struct rxe_mr *mr) { struct rxe_map **map; @@ -125,7 +124,7 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, int err; int i; - umem = ib_umem_get(pd->ibpd.device, start, length, access); + umem = ib_umem_get(&rxe->ib_dev, start, length, access); if (IS_ERR(umem)) { pr_warn("%s: Unable to pin memory region err = %d\n", __func__, (int)PTR_ERR(umem)); @@ -175,12 +174,8 @@ int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova, } } - mr->ibmr.pd = &pd->ibpd; mr->umem = umem; mr->access = access; - mr->length = length; - mr->iova = iova; - mr->va = start; mr->offset = ib_umem_offset(umem); mr->state = RXE_MR_STATE_VALID; mr->type = IB_MR_TYPE_USER; @@ -197,7 +192,7 @@ err_out: return err; } -int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr) +int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr) { int err; @@ -208,7 +203,6 @@ int rxe_mr_init_fast(struct rxe_pd *pd, int max_pages, struct rxe_mr *mr) if (err) goto err1; - mr->ibmr.pd = &pd->ibpd; mr->max_buf = max_pages; mr->state = RXE_MR_STATE_FREE; mr->type = IB_MR_TYPE_MEM_REG; @@ -222,7 +216,7 @@ err1: static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out, size_t *offset_out) { - size_t offset = iova - mr->iova + mr->offset; + size_t offset = iova - mr->ibmr.iova + mr->offset; int map_index; int buf_index; u64 length; @@ -605,7 +599,7 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) mr->access = access; mr->lkey = key; mr->rkey = (access & IB_ACCESS_REMOTE) ? key : 0; - mr->iova = wqe->wr.wr.reg.mr->iova; + mr->ibmr.iova = wqe->wr.wr.reg.mr->iova; mr->state = RXE_MR_STATE_VALID; return 0; diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c index 104993801a804a64ff7212d3e7c90fe730075d2e..902b7df7aaedb626ce33075da22e2cd3ebdbc685 100644 --- a/drivers/infiniband/sw/rxe/rxe_mw.c +++ b/drivers/infiniband/sw/rxe/rxe_mw.c @@ -114,15 +114,15 @@ static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, /* C10-75 */ if (mw->access & IB_ZERO_BASED) { - if (unlikely(wqe->wr.wr.mw.length > mr->length)) { + if (unlikely(wqe->wr.wr.mw.length > mr->ibmr.length)) { pr_err_once( "attempt to bind a ZB MW outside of the MR\n"); return -EINVAL; } } else { - if (unlikely((wqe->wr.wr.mw.addr < mr->iova) || + if (unlikely((wqe->wr.wr.mw.addr < mr->ibmr.iova) || ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) > - (mr->iova + mr->length)))) { + (mr->ibmr.iova + mr->ibmr.length)))) { pr_err_once( "attempt to bind a VA MW outside of the MR\n"); return -EINVAL; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index c53f4529f098dccc85156ec0f9958fd33bc9d010..35f327b9d4b8ec28e3435a73139149742f8128c7 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -145,7 +145,6 @@ static int rxe_udp_encap_recv(struct sock *sk, struct sk_buff *skb) goto drop; if (skb_linearize(skb)) { - pr_err("skb_linearize failed\n"); ib_device_put(&rxe->ib_dev); goto drop; } diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 516bf9b95e489f4fc651032239c811e8aeefb7e2..a62bab88415cbff117ee50e8fdd921594278e8b9 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -19,34 +19,34 @@ static int rxe_qp_chk_cap(struct rxe_dev *rxe, struct ib_qp_cap *cap, int has_srq) { if (cap->max_send_wr > rxe->attr.max_qp_wr) { - pr_warn("invalid send wr = %d > %d\n", - cap->max_send_wr, rxe->attr.max_qp_wr); + pr_debug("invalid send wr = %u > %d\n", + cap->max_send_wr, rxe->attr.max_qp_wr); goto err1; } if (cap->max_send_sge > rxe->attr.max_send_sge) { - pr_warn("invalid send sge = %d > %d\n", - cap->max_send_sge, rxe->attr.max_send_sge); + pr_debug("invalid send sge = %u > %d\n", + cap->max_send_sge, rxe->attr.max_send_sge); goto err1; } if (!has_srq) { if (cap->max_recv_wr > rxe->attr.max_qp_wr) { - pr_warn("invalid recv wr = %d > %d\n", - cap->max_recv_wr, rxe->attr.max_qp_wr); + pr_debug("invalid recv wr = %u > %d\n", + cap->max_recv_wr, rxe->attr.max_qp_wr); goto err1; } if (cap->max_recv_sge > rxe->attr.max_recv_sge) { - pr_warn("invalid recv sge = %d > %d\n", - cap->max_recv_sge, rxe->attr.max_recv_sge); + pr_debug("invalid recv sge = %u > %d\n", + cap->max_recv_sge, rxe->attr.max_recv_sge); goto err1; } } if (cap->max_inline_data > rxe->max_inline_data) { - pr_warn("invalid max inline data = %d > %d\n", - cap->max_inline_data, rxe->max_inline_data); + pr_debug("invalid max inline data = %u > %d\n", + cap->max_inline_data, rxe->max_inline_data); goto err1; } @@ -73,7 +73,7 @@ int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init) } if (!init->recv_cq || !init->send_cq) { - pr_warn("missing cq\n"); + pr_debug("missing cq\n"); goto err1; } @@ -82,14 +82,14 @@ int rxe_qp_chk_init(struct rxe_dev *rxe, struct ib_qp_init_attr *init) if (init->qp_type == IB_QPT_GSI) { if (!rdma_is_port_valid(&rxe->ib_dev, port_num)) { - pr_warn("invalid port = %d\n", port_num); + pr_debug("invalid port = %d\n", port_num); goto err1; } port = &rxe->port; if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) { - pr_warn("GSI QP exists for port %d\n", port_num); + pr_debug("GSI QP exists for port %d\n", port_num); goto err1; } } @@ -242,9 +242,9 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp, skb_queue_head_init(&qp->req_pkts); - rxe_init_task(rxe, &qp->req.task, qp, + rxe_init_task(&qp->req.task, qp, rxe_requester, "req"); - rxe_init_task(rxe, &qp->comp.task, qp, + rxe_init_task(&qp->comp.task, qp, rxe_completer, "comp"); qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */ @@ -292,7 +292,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp, skb_queue_head_init(&qp->resp_pkts); - rxe_init_task(rxe, &qp->resp.task, qp, + rxe_init_task(&qp->resp.task, qp, rxe_responder, "resp"); qp->resp.opcode = OPCODE_NONE; @@ -402,7 +402,7 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, attr->qp_state : cur_state; if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask)) { - pr_warn("invalid mask or state for qp\n"); + pr_debug("invalid mask or state for qp\n"); goto err1; } @@ -416,7 +416,7 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, if (mask & IB_QP_PORT) { if (!rdma_is_port_valid(&rxe->ib_dev, attr->port_num)) { - pr_warn("invalid port %d\n", attr->port_num); + pr_debug("invalid port %d\n", attr->port_num); goto err1; } } @@ -431,12 +431,12 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, if (rxe_av_chk_attr(rxe, &attr->alt_ah_attr)) goto err1; if (!rdma_is_port_valid(&rxe->ib_dev, attr->alt_port_num)) { - pr_warn("invalid alt port %d\n", attr->alt_port_num); + pr_debug("invalid alt port %d\n", attr->alt_port_num); goto err1; } if (attr->alt_timeout > 31) { - pr_warn("invalid QP alt timeout %d > 31\n", - attr->alt_timeout); + pr_debug("invalid QP alt timeout %d > 31\n", + attr->alt_timeout); goto err1; } } @@ -457,17 +457,16 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp, if (mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic > rxe->attr.max_qp_rd_atom) { - pr_warn("invalid max_rd_atomic %d > %d\n", - attr->max_rd_atomic, - rxe->attr.max_qp_rd_atom); + pr_debug("invalid max_rd_atomic %d > %d\n", + attr->max_rd_atomic, + rxe->attr.max_qp_rd_atom); goto err1; } } if (mask & IB_QP_TIMEOUT) { if (attr->timeout > 31) { - pr_warn("invalid QP timeout %d > 31\n", - attr->timeout); + pr_debug("invalid QP timeout %d > 31\n", attr->timeout); goto err1; } } @@ -797,7 +796,9 @@ static void rxe_qp_do_cleanup(struct work_struct *work) rxe_cleanup_task(&qp->comp.task); /* flush out any receive wr's or pending requests */ - __rxe_do_task(&qp->req.task); + if (qp->req.task.func) + __rxe_do_task(&qp->req.task); + if (qp->sq.queue) { __rxe_do_task(&qp->comp.task); __rxe_do_task(&qp->req.task); @@ -833,8 +834,10 @@ static void rxe_qp_do_cleanup(struct work_struct *work) free_rd_atomic_resources(qp); - kernel_sock_shutdown(qp->sk, SHUT_RDWR); - sock_release(qp->sk); + if (qp->sk) { + kernel_sock_shutdown(qp->sk, SHUT_RDWR); + sock_release(qp->sk); + } } /* called when the last reference to the qp is dropped */ diff --git a/drivers/infiniband/sw/rxe/rxe_queue.c b/drivers/infiniband/sw/rxe/rxe_queue.c index dbd4971039c0c6deef0dac58a111086b8f1262bc..d6dbf5a0058dccee2f9a5de6f03345ce706971a4 100644 --- a/drivers/infiniband/sw/rxe/rxe_queue.c +++ b/drivers/infiniband/sw/rxe/rxe_queue.c @@ -112,23 +112,25 @@ static int resize_finish(struct rxe_queue *q, struct rxe_queue *new_q, unsigned int num_elem) { enum queue_type type = q->type; + u32 new_prod; u32 prod; u32 cons; if (!queue_empty(q, q->type) && (num_elem < queue_count(q, type))) return -EINVAL; - prod = queue_get_producer(new_q, type); + new_prod = queue_get_producer(new_q, type); + prod = queue_get_producer(q, type); cons = queue_get_consumer(q, type); - while (!queue_empty(q, type)) { - memcpy(queue_addr_from_index(new_q, prod), + while ((prod - cons) & q->index_mask) { + memcpy(queue_addr_from_index(new_q, new_prod), queue_addr_from_index(q, cons), new_q->elem_size); - prod = queue_next_index(new_q, prod); + new_prod = queue_next_index(new_q, new_prod); cons = queue_next_index(q, cons); } - new_q->buf->producer_index = prod; + new_q->buf->producer_index = new_prod; q->buf->consumer_index = cons; /* update private index copies */ diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index f3ad7b6dbd976c9e2e47e25d3fa6e9cb9a6b1782..434a693cd4a5a721f7e7c5ac6836f51cdc9debfd 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -16,47 +16,36 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, unsigned int pkt_type; if (unlikely(!qp->valid)) - goto err1; + return -EINVAL; pkt_type = pkt->opcode & 0xe0; switch (qp_type(qp)) { case IB_QPT_RC: - if (unlikely(pkt_type != IB_OPCODE_RC)) { - pr_warn_ratelimited("bad qp type\n"); - goto err1; - } + if (unlikely(pkt_type != IB_OPCODE_RC)) + return -EINVAL; break; case IB_QPT_UC: - if (unlikely(pkt_type != IB_OPCODE_UC)) { - pr_warn_ratelimited("bad qp type\n"); - goto err1; - } + if (unlikely(pkt_type != IB_OPCODE_UC)) + return -EINVAL; break; case IB_QPT_UD: case IB_QPT_GSI: - if (unlikely(pkt_type != IB_OPCODE_UD)) { - pr_warn_ratelimited("bad qp type\n"); - goto err1; - } + if (unlikely(pkt_type != IB_OPCODE_UD)) + return -EINVAL; break; default: - pr_warn_ratelimited("unsupported qp type\n"); - goto err1; + return -EINVAL; } if (pkt->mask & RXE_REQ_MASK) { if (unlikely(qp->resp.state != QP_STATE_READY)) - goto err1; + return -EINVAL; } else if (unlikely(qp->req.state < QP_STATE_READY || - qp->req.state > QP_STATE_DRAINED)) { - goto err1; - } + qp->req.state > QP_STATE_DRAINED)) + return -EINVAL; return 0; - -err1: - return -EINVAL; } static void set_bad_pkey_cntr(struct rxe_port *port) @@ -84,26 +73,20 @@ static int check_keys(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, pkt->pkey_index = 0; if (!pkey_match(pkey, IB_DEFAULT_PKEY_FULL)) { - pr_warn_ratelimited("bad pkey = 0x%x\n", pkey); set_bad_pkey_cntr(port); - goto err1; + return -EINVAL; } if (qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) { u32 qkey = (qpn == 1) ? GSI_QKEY : qp->attr.qkey; if (unlikely(deth_qkey(pkt) != qkey)) { - pr_warn_ratelimited("bad qkey, got 0x%x expected 0x%x for qpn 0x%x\n", - deth_qkey(pkt), qkey, qpn); set_qkey_viol_cntr(port); - goto err1; + return -EINVAL; } } return 0; - -err1: - return -EINVAL; } static int check_addr(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, @@ -112,13 +95,10 @@ static int check_addr(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct sk_buff *skb = PKT_TO_SKB(pkt); if (qp_type(qp) != IB_QPT_RC && qp_type(qp) != IB_QPT_UC) - goto done; + return 0; - if (unlikely(pkt->port_num != qp->attr.port_num)) { - pr_warn_ratelimited("port %d != qp port %d\n", - pkt->port_num, qp->attr.port_num); - goto err1; - } + if (unlikely(pkt->port_num != qp->attr.port_num)) + return -EINVAL; if (skb->protocol == htons(ETH_P_IP)) { struct in_addr *saddr = @@ -126,19 +106,9 @@ static int check_addr(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct in_addr *daddr = &qp->pri_av.dgid_addr._sockaddr_in.sin_addr; - if (ip_hdr(skb)->daddr != saddr->s_addr) { - pr_warn_ratelimited("dst addr %pI4 != qp source addr %pI4\n", - &ip_hdr(skb)->daddr, - &saddr->s_addr); - goto err1; - } - - if (ip_hdr(skb)->saddr != daddr->s_addr) { - pr_warn_ratelimited("source addr %pI4 != qp dst addr %pI4\n", - &ip_hdr(skb)->saddr, - &daddr->s_addr); - goto err1; - } + if ((ip_hdr(skb)->daddr != saddr->s_addr) || + (ip_hdr(skb)->saddr != daddr->s_addr)) + return -EINVAL; } else if (skb->protocol == htons(ETH_P_IPV6)) { struct in6_addr *saddr = @@ -146,24 +116,12 @@ static int check_addr(struct rxe_dev *rxe, struct rxe_pkt_info *pkt, struct in6_addr *daddr = &qp->pri_av.dgid_addr._sockaddr_in6.sin6_addr; - if (memcmp(&ipv6_hdr(skb)->daddr, saddr, sizeof(*saddr))) { - pr_warn_ratelimited("dst addr %pI6 != qp source addr %pI6\n", - &ipv6_hdr(skb)->daddr, saddr); - goto err1; - } - - if (memcmp(&ipv6_hdr(skb)->saddr, daddr, sizeof(*daddr))) { - pr_warn_ratelimited("source addr %pI6 != qp dst addr %pI6\n", - &ipv6_hdr(skb)->saddr, daddr); - goto err1; - } + if (memcmp(&ipv6_hdr(skb)->daddr, saddr, sizeof(*saddr)) || + memcmp(&ipv6_hdr(skb)->saddr, daddr, sizeof(*daddr))) + return -EINVAL; } -done: return 0; - -err1: - return -EINVAL; } static int hdr_check(struct rxe_pkt_info *pkt) @@ -175,24 +133,18 @@ static int hdr_check(struct rxe_pkt_info *pkt) int index; int err; - if (unlikely(bth_tver(pkt) != BTH_TVER)) { - pr_warn_ratelimited("bad tver\n"); + if (unlikely(bth_tver(pkt) != BTH_TVER)) goto err1; - } - if (unlikely(qpn == 0)) { - pr_warn_once("QP 0 not supported"); + if (unlikely(qpn == 0)) goto err1; - } if (qpn != IB_MULTICAST_QPN) { index = (qpn == 1) ? port->qp_gsi_index : qpn; qp = rxe_pool_get_index(&rxe->qp_pool, index); - if (unlikely(!qp)) { - pr_warn_ratelimited("no qp matches qpn 0x%x\n", qpn); + if (unlikely(!qp)) goto err1; - } err = check_type_state(rxe, pkt, qp); if (unlikely(err)) @@ -206,10 +158,8 @@ static int hdr_check(struct rxe_pkt_info *pkt) if (unlikely(err)) goto err2; } else { - if (unlikely((pkt->mask & RXE_GRH_MASK) == 0)) { - pr_warn_ratelimited("no grh for mcast qpn\n"); + if (unlikely((pkt->mask & RXE_GRH_MASK) == 0)) goto err1; - } } pkt->qp = qp; @@ -364,10 +314,8 @@ void rxe_rcv(struct sk_buff *skb) if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; - if (rxe_chk_dgid(rxe, skb) < 0) { - pr_warn_ratelimited("failed checking dgid\n"); + if (rxe_chk_dgid(rxe, skb) < 0) goto drop; - } pkt->opcode = bth_opcode(pkt); pkt->psn = bth_psn(pkt); diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index b36ec5c4d5e070398444916490863cfb11aa0ac9..ed5a09e86417e63ff33ac2e2d54764869c755fcb 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -809,10 +809,8 @@ static enum resp_states read_reply(struct rxe_qp *qp, if (!skb) return RESPST_ERR_RNR; - err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), - payload, RXE_FROM_MR_OBJ); - if (err) - pr_err("Failed copying memory\n"); + rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), + payload, RXE_FROM_MR_OBJ); if (mr) rxe_put(mr); @@ -823,10 +821,8 @@ static enum resp_states read_reply(struct rxe_qp *qp, } err = rxe_xmit_packet(qp, &ack_pkt, skb); - if (err) { - pr_err("Failed sending RDMA reply.\n"); + if (err) return RESPST_ERR_RNR; - } res->read.va += payload; res->read.resid -= payload; @@ -1028,50 +1024,41 @@ finish: return RESPST_CLEANUP; } -static int send_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) + +static int send_common_ack(struct rxe_qp *qp, u8 syndrome, u32 psn, + int opcode, const char *msg) { - int err = 0; + int err; struct rxe_pkt_info ack_pkt; struct sk_buff *skb; - skb = prepare_ack_packet(qp, &ack_pkt, IB_OPCODE_RC_ACKNOWLEDGE, - 0, psn, syndrome); - if (!skb) { - err = -ENOMEM; - goto err1; - } + skb = prepare_ack_packet(qp, &ack_pkt, opcode, 0, psn, syndrome); + if (!skb) + return -ENOMEM; err = rxe_xmit_packet(qp, &ack_pkt, skb); if (err) - pr_err_ratelimited("Failed sending ack\n"); + pr_err_ratelimited("Failed sending %s\n", msg); -err1: return err; } -static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) +static int send_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) { - int err = 0; - struct rxe_pkt_info ack_pkt; - struct sk_buff *skb; - - skb = prepare_ack_packet(qp, &ack_pkt, IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, - 0, psn, syndrome); - if (!skb) { - err = -ENOMEM; - goto out; - } + return send_common_ack(qp, syndrome, psn, + IB_OPCODE_RC_ACKNOWLEDGE, "ACK"); +} - err = rxe_xmit_packet(qp, &ack_pkt, skb); - if (err) - pr_err_ratelimited("Failed sending atomic ack\n"); +static int send_atomic_ack(struct rxe_qp *qp, u8 syndrome, u32 psn) +{ + int ret = send_common_ack(qp, syndrome, psn, + IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, "ATOMIC ACK"); /* have to clear this since it is used to trigger * long read replies */ qp->resp.res = NULL; -out: - return err; + return ret; } static enum resp_states acknowledge(struct rxe_qp *qp, diff --git a/drivers/infiniband/sw/rxe/rxe_task.c b/drivers/infiniband/sw/rxe/rxe_task.c index 2248cf33d776670e7d5fea009a5e6f689da4f210..ec2b7de1c4972105e1552b2bf3c8190869ce86a8 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.c +++ b/drivers/infiniband/sw/rxe/rxe_task.c @@ -94,10 +94,9 @@ void rxe_do_task(struct tasklet_struct *t) task->ret = ret; } -int rxe_init_task(void *obj, struct rxe_task *task, +int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *), char *name) { - task->obj = obj; task->arg = arg; task->func = func; snprintf(task->name, sizeof(task->name), "%s", name); diff --git a/drivers/infiniband/sw/rxe/rxe_task.h b/drivers/infiniband/sw/rxe/rxe_task.h index 11d183fd333863d74a888dcfed916e14067b3150..7f612a1c68a7ba3c76d02aaa1fd5ee04c384a2bd 100644 --- a/drivers/infiniband/sw/rxe/rxe_task.h +++ b/drivers/infiniband/sw/rxe/rxe_task.h @@ -19,7 +19,6 @@ enum { * called again. */ struct rxe_task { - void *obj; struct tasklet_struct tasklet; int state; spinlock_t state_lock; /* spinlock for task state */ @@ -35,7 +34,7 @@ struct rxe_task { * arg => parameter to pass to fcn * func => function to call until it returns != 0 */ -int rxe_init_task(void *obj, struct rxe_task *task, +int rxe_init_task(struct rxe_task *task, void *arg, int (*func)(void *), char *name); /* cleanup task */ diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index e264cf69bf5580c63f9f9763fdd8de5faa4a6cc3..88825edc7dce19c0ac4a2806586fe68202e7d4aa 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -262,7 +262,6 @@ static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_TO_DRIVER); recv_wqe->wr_id = ibwr->wr_id; - recv_wqe->num_sge = num_sge; memcpy(recv_wqe->dma.sge, ibwr->sg_list, num_sge * sizeof(struct ib_sge)); @@ -526,7 +525,6 @@ static void init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr, const struct ib_send_wr *ibwr) { wr->wr_id = ibwr->wr_id; - wr->num_sge = ibwr->num_sge; wr->opcode = ibwr->opcode; wr->send_flags = ibwr->send_flags; @@ -903,7 +901,9 @@ static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) return ERR_PTR(-ENOMEM); rxe_get(pd); - rxe_mr_init_dma(pd, access, mr); + mr->ibmr.pd = ibpd; + + rxe_mr_init_dma(access, mr); rxe_finalize(mr); return &mr->ibmr; @@ -928,8 +928,9 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, rxe_get(pd); + mr->ibmr.pd = ibpd; - err = rxe_mr_init_user(pd, start, length, iova, access, mr); + err = rxe_mr_init_user(rxe, start, length, iova, access, mr); if (err) goto err3; @@ -938,7 +939,6 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, return &mr->ibmr; err3: - rxe_put(pd); rxe_cleanup(mr); err2: return ERR_PTR(err); @@ -962,8 +962,9 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, } rxe_get(pd); + mr->ibmr.pd = ibpd; - err = rxe_mr_init_fast(pd, max_num_sg, mr); + err = rxe_mr_init_fast(max_num_sg, mr); if (err) goto err2; @@ -972,7 +973,6 @@ static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, return &mr->ibmr; err2: - rxe_put(pd); rxe_cleanup(mr); err1: return ERR_PTR(err); @@ -1007,12 +1007,9 @@ static int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rxe_set_page); - mr->va = ibmr->iova; - mr->iova = ibmr->iova; - mr->length = ibmr->length; mr->page_shift = ilog2(ibmr->page_size); mr->page_mask = ibmr->page_size - 1; - mr->offset = mr->iova & mr->page_mask; + mr->offset = ibmr->iova & mr->page_mask; return n; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 96af3e054f4d4c47b870eb5586446867e8705c3b..5f5cbfcb35695b930168df9f9f96c3ff2122bef9 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -305,9 +305,6 @@ struct rxe_mr { u32 rkey; enum rxe_mr_state state; enum ib_mr_type type; - u64 va; - u64 iova; - size_t length; u32 offset; int access; diff --git a/drivers/infiniband/sw/siw/Kconfig b/drivers/infiniband/sw/siw/Kconfig index 1b5105cbabaeedde957394f8548d1db549ee0a02..81b70a3eeb87821a8cac4393939ae06f5e41e078 100644 --- a/drivers/infiniband/sw/siw/Kconfig +++ b/drivers/infiniband/sw/siw/Kconfig @@ -1,7 +1,10 @@ config RDMA_SIW tristate "Software RDMA over TCP/IP (iWARP) driver" - depends on INET && INFINIBAND && LIBCRC32C + depends on INET && INFINIBAND depends on INFINIBAND_VIRT_DMA + select LIBCRC32C + select CRYPTO + select CRYPTO_CRC32C help This driver implements the iWARP RDMA transport over the Linux TCP/IP network stack. It enables a system with a diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h index df03d84c6868ace3d5fe68c47e62cdd33e8b44d2..2f3a9cda3850f61c4b7c0d134c28a995714333c6 100644 --- a/drivers/infiniband/sw/siw/siw.h +++ b/drivers/infiniband/sw/siw/siw.h @@ -418,6 +418,7 @@ struct siw_qp { struct ib_qp base_qp; struct siw_device *sdev; struct kref ref; + struct completion qp_free; struct list_head devq; int tx_cpu; struct siw_qp_attrs attrs; diff --git a/drivers/infiniband/sw/siw/siw_qp.c b/drivers/infiniband/sw/siw/siw_qp.c index 7e01f2438afc5ed759182cac9a18514b92a7b850..e6f634971228e4993f26ac17a19fb41d8a588ed5 100644 --- a/drivers/infiniband/sw/siw/siw_qp.c +++ b/drivers/infiniband/sw/siw/siw_qp.c @@ -1342,6 +1342,6 @@ void siw_free_qp(struct kref *ref) vfree(qp->orq); siw_put_tx_cpu(qp->tx_cpu); - + complete(&qp->qp_free); atomic_dec(&sdev->num_qp); } diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c index 875ea6f1b04a29034b6b41f8bfedaefc01af28c6..fd721cc19682eb2ca510315abb4e3ab60f2f8116 100644 --- a/drivers/infiniband/sw/siw/siw_qp_rx.c +++ b/drivers/infiniband/sw/siw/siw_qp_rx.c @@ -961,27 +961,28 @@ out: static int siw_get_trailer(struct siw_qp *qp, struct siw_rx_stream *srx) { struct sk_buff *skb = srx->skb; + int avail = min(srx->skb_new, srx->fpdu_part_rem); u8 *tbuf = (u8 *)&srx->trailer.crc - srx->pad; __wsum crc_in, crc_own = 0; siw_dbg_qp(qp, "expected %d, available %d, pad %u\n", srx->fpdu_part_rem, srx->skb_new, srx->pad); - if (srx->skb_new < srx->fpdu_part_rem) - return -EAGAIN; - - skb_copy_bits(skb, srx->skb_offset, tbuf, srx->fpdu_part_rem); + skb_copy_bits(skb, srx->skb_offset, tbuf, avail); - if (srx->mpa_crc_hd && srx->pad) - crypto_shash_update(srx->mpa_crc_hd, tbuf, srx->pad); + srx->skb_new -= avail; + srx->skb_offset += avail; + srx->skb_copied += avail; + srx->fpdu_part_rem -= avail; - srx->skb_new -= srx->fpdu_part_rem; - srx->skb_offset += srx->fpdu_part_rem; - srx->skb_copied += srx->fpdu_part_rem; + if (srx->fpdu_part_rem) + return -EAGAIN; if (!srx->mpa_crc_hd) return 0; + if (srx->pad) + crypto_shash_update(srx->mpa_crc_hd, tbuf, srx->pad); /* * CRC32 is computed, transmitted and received directly in NBO, * so there's never a reason to convert byte order. @@ -1083,10 +1084,9 @@ static int siw_get_hdr(struct siw_rx_stream *srx) * completely received. */ if (iwarp_pktinfo[opcode].hdr_len > sizeof(struct iwarp_ctrl_tagged)) { - bytes = iwarp_pktinfo[opcode].hdr_len - MIN_DDP_HDR; + int hdrlen = iwarp_pktinfo[opcode].hdr_len; - if (srx->skb_new < bytes) - return -EAGAIN; + bytes = min_t(int, hdrlen - MIN_DDP_HDR, srx->skb_new); skb_copy_bits(skb, srx->skb_offset, (char *)c_hdr + srx->fpdu_part_rcvd, bytes); @@ -1096,6 +1096,9 @@ static int siw_get_hdr(struct siw_rx_stream *srx) srx->skb_new -= bytes; srx->skb_offset += bytes; srx->skb_copied += bytes; + + if (srx->fpdu_part_rcvd < hdrlen) + return -EAGAIN; } /* diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index 1f4e60257700ef4dffff91a81f3305569c45ebb1..7d47b521070b1b73d69a8822d45a6e879489e669 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -29,7 +29,7 @@ static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx) dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx); if (paddr) - return virt_to_page(paddr); + return virt_to_page((void *)paddr); return NULL; } @@ -533,13 +533,23 @@ static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s) kunmap_local(kaddr); } } else { - u64 va = sge->laddr + sge_off; + /* + * Cast to an uintptr_t to preserve all 64 bits + * in sge->laddr. + */ + uintptr_t va = (uintptr_t)(sge->laddr + sge_off); - page_array[seg] = virt_to_page(va & PAGE_MASK); + /* + * virt_to_page() takes a (void *) pointer + * so cast to a (void *) meaning it will be 64 + * bits on a 64 bit platform and 32 bits on a + * 32 bit platform. + */ + page_array[seg] = virt_to_page((void *)(va & PAGE_MASK)); if (do_crc) crypto_shash_update( c_tx->mpa_crc_hd, - (void *)(uintptr_t)va, + (void *)va, plen); } diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index 8dedae7ae79e681558712aefab90928df2670e58..3e814cfb298cf8342734fa6917eca138b2ee38b4 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -480,6 +480,8 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, list_add_tail(&qp->devq, &sdev->qp_list); spin_unlock_irqrestore(&sdev->lock, flags); + init_completion(&qp->qp_free); + return 0; err_out_xa: @@ -624,6 +626,7 @@ int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata) qp->scq = qp->rcq = NULL; siw_qp_put(qp); + wait_for_completion(&qp->qp_free); return 0; } diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index fd9d7f2c4d648ba1eb3a4f7d23bf854b5cdc8647..b610d36295bb233e0526b5092eb98d14f7467564 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -465,7 +465,7 @@ static int ipoib_cm_req_handler(struct ib_cm_id *cm_id, goto err_qp; } - psn = prandom_u32() & 0xffffff; + psn = get_random_u32() & 0xffffff; ret = ipoib_cm_modify_rx_qp(dev, cm_id, p->qp, psn); if (ret) goto err_modify; @@ -884,8 +884,8 @@ int ipoib_cm_dev_open(struct net_device *dev) goto err_cm; } - ret = ib_cm_listen(priv->cm.id, cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num), - 0); + ret = ib_cm_listen(priv->cm.id, + cpu_to_be64(IPOIB_CM_IETF_ID | priv->qp->qp_num)); if (ret) { pr_warn("%s: failed to listen on ID 0x%llx\n", priv->ca->name, IPOIB_CM_IETF_ID | priv->qp->qp_num); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c index a09ca21f7dff8efda3542005deae03d7733daa2c..8af99b18d3618fbab0048b4f5d4ac65db6ec20a0 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c @@ -65,10 +65,10 @@ static void ipoib_get_drvinfo(struct net_device *netdev, ib_get_device_fw_str(priv->ca, drvinfo->fw_version); - strlcpy(drvinfo->bus_info, dev_name(priv->ca->dev.parent), + strscpy(drvinfo->bus_info, dev_name(priv->ca->dev.parent), sizeof(drvinfo->bus_info)); - strlcpy(drvinfo->driver, "ib_ipoib", sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, "ib_ipoib", sizeof(drvinfo->driver)); } static int ipoib_get_coalesce(struct net_device *dev, diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index a4904371e2dbfdf6c64fbe7e1da3e6767b7512a5..ac25fc80fb337277d0fee73898b1555424476cc3 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -742,7 +742,7 @@ void ipoib_flush_paths(struct net_device *dev) static void path_rec_completion(int status, struct sa_path_rec *pathrec, - void *path_ptr) + int num_prs, void *path_ptr) { struct ipoib_path *path = path_ptr; struct net_device *dev = path->dev; diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c index 42d557dff19d2eb6e7730f7ed6cc506b3c3f614f..29b3d8fce3f5f376ce7a552ae34764525033851f 100644 --- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c +++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_ethtool.c @@ -124,8 +124,8 @@ static struct vnic_stats vnic_gstrings_stats[] = { static void vnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, opa_vnic_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(netdev->dev.parent), + strscpy(drvinfo->driver, opa_vnic_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(netdev->dev.parent), sizeof(drvinfo->bus_info)); } diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile index 3898509be27080f183be44cfbcafb2fd0053c797..5227e7788e1fc723ac67470ca2aee8a4c9e941d0 100644 --- a/drivers/infiniband/ulp/rtrs/Makefile +++ b/drivers/infiniband/ulp/rtrs/Makefile @@ -1,12 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-or-later +CFLAGS_rtrs-clt-trace.o = -I$(src) + rtrs-client-y := rtrs-clt.o \ rtrs-clt-stats.o \ - rtrs-clt-sysfs.o + rtrs-clt-sysfs.o \ + rtrs-clt-trace.o + +CFLAGS_rtrs-srv-trace.o = -I$(src) rtrs-server-y := rtrs-srv.o \ rtrs-srv-stats.o \ - rtrs-srv-sysfs.o + rtrs-srv-sysfs.o \ + rtrs-srv-trace.o rtrs-core-y := rtrs.o diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c new file mode 100644 index 0000000000000000000000000000000000000000..f14fa1f36ce86a9290cbd31ce89604a13fd95952 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#include "rtrs.h" +#include "rtrs-clt.h" + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "rtrs-clt-trace.h" diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h new file mode 100644 index 0000000000000000000000000000000000000000..7738e26768557db3fff79090d8baa34d0a2e3cb9 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rtrs_clt + +#if !defined(_TRACE_RTRS_CLT_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RTRS_CLT_H + +#include + +struct rtrs_clt_path; +struct rtrs_clt_sess; + +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING); +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING_ERR); +TRACE_DEFINE_ENUM(RTRS_CLT_RECONNECTING); +TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTED); +TRACE_DEFINE_ENUM(RTRS_CLT_CLOSING); +TRACE_DEFINE_ENUM(RTRS_CLT_CLOSED); +TRACE_DEFINE_ENUM(RTRS_CLT_DEAD); + +#define show_rtrs_clt_state(x) \ + __print_symbolic(x, \ + { RTRS_CLT_CONNECTING, "CONNECTING" }, \ + { RTRS_CLT_CONNECTING_ERR, "CONNECTING_ERR" }, \ + { RTRS_CLT_RECONNECTING, "RECONNECTING" }, \ + { RTRS_CLT_CONNECTED, "CONNECTED" }, \ + { RTRS_CLT_CLOSING, "CLOSING" }, \ + { RTRS_CLT_CLOSED, "CLOSED" }, \ + { RTRS_CLT_DEAD, "DEAD" }) + +DECLARE_EVENT_CLASS(rtrs_clt_conn_class, + TP_PROTO(struct rtrs_clt_path *clt_path), + + TP_ARGS(clt_path), + + TP_STRUCT__entry( + __field(int, state) + __field(int, reconnect_attempts) + __field(int, max_reconnect_attempts) + __field(int, fail_cnt) + __field(int, success_cnt) + __array(char, sessname, NAME_MAX) + ), + + TP_fast_assign( + struct rtrs_clt_sess *clt = clt_path->clt; + + __entry->state = clt_path->state; + __entry->reconnect_attempts = clt_path->reconnect_attempts; + __entry->max_reconnect_attempts = clt->max_reconnect_attempts; + __entry->fail_cnt = clt_path->stats->reconnects.fail_cnt; + __entry->success_cnt = clt_path->stats->reconnects.successful_cnt; + memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX); + ), + + TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'", + __entry->sessname, + show_rtrs_clt_state(__entry->state), + __entry->reconnect_attempts, + __entry->max_reconnect_attempts, + __entry->fail_cnt, + __entry->success_cnt + ) +); + +#define DEFINE_CLT_CONN_EVENT(name) \ +DEFINE_EVENT(rtrs_clt_conn_class, rtrs_##name, \ + TP_PROTO(struct rtrs_clt_path *clt_path), \ + TP_ARGS(clt_path)) + +DEFINE_CLT_CONN_EVENT(clt_reconnect_work); +DEFINE_CLT_CONN_EVENT(clt_close_conns); +DEFINE_CLT_CONN_EVENT(rdma_error_recovery); + +#endif /* _TRACE_RTRS_CLT_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE rtrs-clt-trace +#include + diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index baecde41d126e5573257df5ae9b73a8473bdf8a4..8546b8816524cba880626287d1dd7568bf683167 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -16,6 +16,7 @@ #include "rtrs-clt.h" #include "rtrs-log.h" +#include "rtrs-clt-trace.h" #define RTRS_CONNECT_TIMEOUT_MS 30000 /* @@ -53,7 +54,10 @@ static inline bool rtrs_clt_is_connected(const struct rtrs_clt_sess *clt) rcu_read_lock(); list_for_each_entry_rcu(clt_path, &clt->paths_list, s.entry) - connected |= READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED; + if (READ_ONCE(clt_path->state) == RTRS_CLT_CONNECTED) { + connected = true; + break; + } rcu_read_unlock(); return connected; @@ -302,6 +306,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) { struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + trace_rtrs_rdma_error_recovery(clt_path); + if (rtrs_clt_change_state_from_to(clt_path, RTRS_CLT_CONNECTED, RTRS_CLT_RECONNECTING)) { @@ -1004,7 +1010,8 @@ rtrs_clt_get_copy_req(struct rtrs_clt_path *alive_path, static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, struct rtrs_clt_io_req *req, struct rtrs_rbuf *rbuf, bool fr_en, - u32 size, u32 imm, struct ib_send_wr *wr, + u32 count, u32 size, u32 imm, + struct ib_send_wr *wr, struct ib_send_wr *tail) { struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); @@ -1024,12 +1031,12 @@ static int rtrs_post_rdma_write_sg(struct rtrs_clt_con *con, num_sge = 2; ptail = tail; } else { - for_each_sg(req->sglist, sg, req->sg_cnt, i) { + for_each_sg(req->sglist, sg, count, i) { sge[i].addr = sg_dma_address(sg); sge[i].length = sg_dma_len(sg); sge[i].lkey = clt_path->s.dev->ib_pd->local_dma_lkey; } - num_sge = 1 + req->sg_cnt; + num_sge = 1 + count; } sge[i].addr = req->iu->dma_addr; sge[i].length = size; @@ -1142,7 +1149,7 @@ static int rtrs_clt_write_req(struct rtrs_clt_io_req *req) */ rtrs_clt_update_all_stats(req, WRITE); - ret = rtrs_post_rdma_write_sg(req->con, req, rbuf, fr_en, + ret = rtrs_post_rdma_write_sg(req->con, req, rbuf, fr_en, count, req->usr_len + sizeof(*msg), imm, wr, &inv_wr); if (ret) { @@ -1510,8 +1517,7 @@ static void rtrs_clt_err_recovery_work(struct work_struct *work) rtrs_clt_stop_and_destroy_conns(clt_path); queue_delayed_work(rtrs_wq, &clt_path->reconnect_dwork, msecs_to_jiffies(delay_ms + - prandom_u32() % - RTRS_RECONNECT_SEED)); + prandom_u32_max(RTRS_RECONNECT_SEED))); } static struct rtrs_clt_path *alloc_path(struct rtrs_clt_sess *clt, @@ -1942,6 +1948,8 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait) { + trace_rtrs_clt_close_conns(clt_path); + if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL)) queue_work(rtrs_wq, &clt_path->close_work); if (wait) @@ -2212,17 +2220,6 @@ static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_path *clt_path) } } -static inline bool xchg_paths(struct rtrs_clt_path __rcu **rcu_ppcpu_path, - struct rtrs_clt_path *clt_path, - struct rtrs_clt_path *next) -{ - struct rtrs_clt_path **ppcpu_path; - - /* Call cmpxchg() without sparse warnings */ - ppcpu_path = (typeof(ppcpu_path))rcu_ppcpu_path; - return clt_path == cmpxchg(ppcpu_path, clt_path, next); -} - static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path) { struct rtrs_clt_sess *clt = clt_path->clt; @@ -2297,7 +2294,8 @@ static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_path *clt_path) * We race with IO code path, which also changes pointer, * thus we have to be careful not to overwrite it. */ - if (xchg_paths(ppcpu_path, clt_path, next)) + if (try_cmpxchg((struct rtrs_clt_path **)ppcpu_path, &clt_path, + next)) /* * @ppcpu_path was successfully replaced with @next, * that means that someone could also pick up the @@ -2648,6 +2646,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work) reconnect_dwork); clt = clt_path->clt; + trace_rtrs_clt_reconnect_work(clt_path); + if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING) return; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-pri.h b/drivers/infiniband/ulp/rtrs/rtrs-pri.h index ac0df734eba8c9b3414b64026c0fc7aa20be659f..a2420eecaf5a1080c8e9f5f49ba9b70123a5b806 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-pri.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-pri.h @@ -26,11 +26,10 @@ /* * Max IB immediate data size is 2^28 (MAX_IMM_PAYL_BITS) * and the minimum chunk size is 4096 (2^12). - * So the maximum sess_queue_depth is 65536 (2^16) in theory. - * But mempool_create, create_qp and ib_post_send fail with - * "cannot allocate memory" error if sess_queue_depth is too big. + * So the maximum sess_queue_depth is 65535 (2^16 - 1) in theory + * since queue_depth in rtrs_msg_conn_rsp is defined as le16. * Therefore the pratical max value of sess_queue_depth is - * somewhere between 1 and 65534 and it depends on the system. + * somewhere between 1 and 65535 and it depends on the system. */ #define MAX_SESS_QUEUE_DEPTH 65535 diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c new file mode 100644 index 0000000000000000000000000000000000000000..29ca59ceb0ddfae834ac4fede061f20ca88241f1 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#include "rtrs.h" +#include "rtrs-pri.h" +#include "rtrs-srv.h" + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "rtrs-srv-trace.h" diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h new file mode 100644 index 0000000000000000000000000000000000000000..587d3e033081264e3a264bc1998466f53cc36be0 --- /dev/null +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-trace.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * RDMA Network Block Driver + * + * Copyright (c) 2022 1&1 IONOS SE. All rights reserved. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM rtrs_srv + +#if !defined(_TRACE_RTRS_SRV_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_RTRS_SRV_H + +#include + +struct rtrs_srv_op; +struct rtrs_srv_con; +struct rtrs_srv_path; + +TRACE_DEFINE_ENUM(RTRS_SRV_CONNECTING); +TRACE_DEFINE_ENUM(RTRS_SRV_CONNECTED); +TRACE_DEFINE_ENUM(RTRS_SRV_CLOSING); +TRACE_DEFINE_ENUM(RTRS_SRV_CLOSED); + +#define show_rtrs_srv_state(x) \ + __print_symbolic(x, \ + { RTRS_SRV_CONNECTING, "CONNECTING" }, \ + { RTRS_SRV_CONNECTED, "CONNECTED" }, \ + { RTRS_SRV_CLOSING, "CLOSING" }, \ + { RTRS_SRV_CLOSED, "CLOSED" }) + +TRACE_EVENT(send_io_resp_imm, + TP_PROTO(struct rtrs_srv_op *id, + bool need_inval, + bool always_invalidate, + int errno), + + TP_ARGS(id, need_inval, always_invalidate, errno), + + TP_STRUCT__entry( + __field(u8, dir) + __field(bool, need_inval) + __field(bool, always_invalidate) + __field(u32, msg_id) + __field(int, wr_cnt) + __field(u32, signal_interval) + __field(int, state) + __field(int, errno) + __array(char, sessname, NAME_MAX) + ), + + TP_fast_assign( + struct rtrs_srv_con *con = id->con; + struct rtrs_path *s = con->c.path; + struct rtrs_srv_path *srv_path = to_srv_path(s); + + __entry->dir = id->dir; + __entry->state = srv_path->state; + __entry->errno = errno; + __entry->need_inval = need_inval; + __entry->always_invalidate = always_invalidate; + __entry->msg_id = id->msg_id; + __entry->wr_cnt = atomic_read(&con->c.wr_cnt); + __entry->signal_interval = s->signal_interval; + memcpy(__entry->sessname, kobject_name(&srv_path->kobj), NAME_MAX); + ), + + TP_printk("sess='%s' state='%s' dir=%s err='%d' inval='%d' glob-inval='%d' msgid='%u' wrcnt='%d' sig-interval='%u'", + __entry->sessname, + show_rtrs_srv_state(__entry->state), + __print_symbolic(__entry->dir, + { READ, "READ" }, + { WRITE, "WRITE" }), + __entry->errno, + __entry->need_inval, + __entry->always_invalidate, + __entry->msg_id, + __entry->wr_cnt, + __entry->signal_interval + ) +); + +#endif /* _TRACE_RTRS_SRV_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE rtrs-srv-trace +#include + diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 34c03bde50641b5ffcdfb51ec3748925947cd2da..22d7ba05e9fe838b1fab5bde3e43b037d9b9bf0a 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -16,6 +16,7 @@ #include "rtrs-log.h" #include #include +#include "rtrs-srv-trace.h" MODULE_DESCRIPTION("RDMA Transport Server"); MODULE_LICENSE("GPL"); @@ -57,11 +58,6 @@ static inline struct rtrs_srv_con *to_srv_con(struct rtrs_con *c) return container_of(c, struct rtrs_srv_con, c); } -static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s) -{ - return container_of(s, struct rtrs_srv_path, s); -} - static bool rtrs_srv_change_state(struct rtrs_srv_path *srv_path, enum rtrs_srv_state new_state) { @@ -375,6 +371,8 @@ static int send_io_resp_imm(struct rtrs_srv_con *con, struct rtrs_srv_op *id, } } + trace_send_io_resp_imm(id, need_inval, always_invalidate, errno); + if (need_inval && always_invalidate) { wr = &inv_wr; inv_wr.next = &rwr.wr; @@ -595,7 +593,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) struct sg_table *sgt = &srv_mr->sgt; struct scatterlist *s; struct ib_mr *mr; - int nr, chunks; + int nr, nr_sgt, chunks; chunks = chunks_per_mr * mri; if (!always_invalidate) @@ -610,19 +608,19 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) sg_set_page(s, srv->chunks[chunks + i], max_chunk_size, 0); - nr = ib_dma_map_sg(srv_path->s.dev->ib_dev, sgt->sgl, + nr_sgt = ib_dma_map_sg(srv_path->s.dev->ib_dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); - if (nr < sgt->nents) { - err = nr < 0 ? nr : -EINVAL; + if (!nr_sgt) { + err = -EINVAL; goto free_sg; } mr = ib_alloc_mr(srv_path->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, - sgt->nents); + nr_sgt); if (IS_ERR(mr)) { err = PTR_ERR(mr); goto unmap_sg; } - nr = ib_map_mr_sg(mr, sgt->sgl, sgt->nents, + nr = ib_map_mr_sg(mr, sgt->sgl, nr_sgt, NULL, max_chunk_size); if (nr < 0 || nr < sgt->nents) { err = nr < 0 ? nr : -EINVAL; @@ -641,7 +639,7 @@ static int map_cont_bufs(struct rtrs_srv_path *srv_path) } } /* Eventually dma addr for each chunk can be cached */ - for_each_sg(sgt->sgl, s, sgt->orig_nents, i) + for_each_sg(sgt->sgl, s, nr_sgt, i) srv_path->dma_addr[chunks + i] = sg_dma_address(s); ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); @@ -1024,7 +1022,7 @@ static void process_read(struct rtrs_srv_con *con, usr_len = le16_to_cpu(msg->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); - ret = ctx->ops.rdma_ev(srv->priv, id, READ, data, data_len, + ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, data + data_len, usr_len); if (ret) { @@ -1077,7 +1075,7 @@ static void process_write(struct rtrs_srv_con *con, usr_len = le16_to_cpu(req->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); - ret = ctx->ops.rdma_ev(srv->priv, id, WRITE, data, data_len, + ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, data + data_len, usr_len); if (ret) { rtrs_err_rl(s, diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h index 186a63c217dfda652cf7fd1f91550a87d66b8108..2f8a638e36fad48089729f1148215460edfa014a 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.h +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h @@ -91,6 +91,11 @@ struct rtrs_srv_path { struct rtrs_srv_stats *stats; }; +static inline struct rtrs_srv_path *to_srv_path(struct rtrs_path *s) +{ + return container_of(s, struct rtrs_srv_path, s); +} + struct rtrs_srv_sess { struct list_head paths_list; int paths_up; diff --git a/drivers/infiniband/ulp/rtrs/rtrs.c b/drivers/infiniband/ulp/rtrs/rtrs.c index 60fa0b0160f45ec9304f32371a81895427d7d1f6..ed324b47d93ae48017c34a80ab70d0b609adfbc7 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs.c +++ b/drivers/infiniband/ulp/rtrs/rtrs.c @@ -175,7 +175,7 @@ int rtrs_iu_post_rdma_write_imm(struct rtrs_con *con, struct rtrs_iu *iu, * length error */ for (i = 0; i < num_sge; i++) - if (WARN_ON(sge[i].length == 0)) + if (WARN_ONCE(sge[i].length == 0, "sg %d is zero length\n", i)) return -EINVAL; return rtrs_post_send(con->qp, head, &wr.wr, tail); diff --git a/drivers/infiniband/ulp/rtrs/rtrs.h b/drivers/infiniband/ulp/rtrs/rtrs.h index 5e57a7ccc7fbff00ce34ed58f371c0fcb5236d34..b48b53a7c143511b9bf75a140fb6fca6ef5b10d1 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs.h +++ b/drivers/infiniband/ulp/rtrs/rtrs.h @@ -139,7 +139,6 @@ struct rtrs_srv_ops { * @priv: Private data set by rtrs_srv_set_sess_priv() * @id: internal RTRS operation id - * @dir: READ/WRITE * @data: Pointer to (bidirectional) rdma memory area: * - in case of %RTRS_SRV_RDMA_EV_RECV contains * data sent by the client @@ -151,7 +150,7 @@ struct rtrs_srv_ops { * @usrlen: Size of the user message */ int (*rdma_ev)(void *priv, - struct rtrs_srv_op *id, int dir, + struct rtrs_srv_op *id, void *data, size_t datalen, const void *usr, size_t usrlen); /** diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 7720ea270ed8c8f4f2e33c72ed27e2dfacdc6216..1075c2ac8fe209deb703a97f38e5593791cfefe9 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -699,7 +699,7 @@ static void srp_free_ch_ib(struct srp_target_port *target, static void srp_path_rec_completion(int status, struct sa_path_rec *pathrec, - void *ch_ptr) + int num_paths, void *ch_ptr) { struct srp_rdma_ch *ch = ch_ptr; struct srp_target_port *target = ch->target; @@ -1961,7 +1961,8 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) if (scmnd) { req = scsi_cmd_priv(scmnd); scmnd = srp_claim_req(ch, req, NULL, scmnd); - } else { + } + if (!scmnd) { shost_printk(KERN_ERR, target->scsi_host, "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", rsp->tag, ch - target->ch, ch->qp->qp_num); @@ -2788,7 +2789,7 @@ static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, static int srp_abort(struct scsi_cmnd *scmnd) { struct srp_target_port *target = host_to_target(scmnd->device->host); - struct srp_request *req = (struct srp_request *) scmnd->host_scribble; + struct srp_request *req = scsi_cmd_priv(scmnd); u32 tag; u16 ch_idx; struct srp_rdma_ch *ch; @@ -2796,8 +2797,6 @@ static int srp_abort(struct scsi_cmnd *scmnd) shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); - if (!req) - return SUCCESS; tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); ch_idx = blk_mq_unique_tag_to_hwq(tag); if (WARN_ON_ONCE(ch_idx >= target->ch_count)) @@ -2990,7 +2989,7 @@ static ssize_t local_ib_port_show(struct device *dev, { struct srp_target_port *target = host_to_target(class_to_shost(dev)); - return sysfs_emit(buf, "%d\n", target->srp_host->port); + return sysfs_emit(buf, "%u\n", target->srp_host->port); } static DEVICE_ATTR_RO(local_ib_port); @@ -3178,11 +3177,16 @@ static void srp_release_dev(struct device *dev) struct srp_host *host = container_of(dev, struct srp_host, dev); - complete(&host->released); + kfree(host); } +static struct attribute *srp_class_attrs[]; + +ATTRIBUTE_GROUPS(srp_class); + static struct class srp_class = { .name = "infiniband_srp", + .dev_groups = srp_class_groups, .dev_release = srp_release_dev }; @@ -3883,12 +3887,19 @@ static ssize_t port_show(struct device *dev, struct device_attribute *attr, { struct srp_host *host = container_of(dev, struct srp_host, dev); - return sysfs_emit(buf, "%d\n", host->port); + return sysfs_emit(buf, "%u\n", host->port); } static DEVICE_ATTR_RO(port); -static struct srp_host *srp_add_port(struct srp_device *device, u8 port) +static struct attribute *srp_class_attrs[] = { + &dev_attr_add_target.attr, + &dev_attr_ibdev.attr, + &dev_attr_port.attr, + NULL +}; + +static struct srp_host *srp_add_port(struct srp_device *device, u32 port) { struct srp_host *host; @@ -3898,33 +3909,24 @@ static struct srp_host *srp_add_port(struct srp_device *device, u8 port) INIT_LIST_HEAD(&host->target_list); spin_lock_init(&host->target_lock); - init_completion(&host->released); mutex_init(&host->add_target_mutex); host->srp_dev = device; host->port = port; + device_initialize(&host->dev); host->dev.class = &srp_class; host->dev.parent = device->dev->dev.parent; - dev_set_name(&host->dev, "srp-%s-%d", dev_name(&device->dev->dev), - port); - - if (device_register(&host->dev)) - goto free_host; - if (device_create_file(&host->dev, &dev_attr_add_target)) - goto err_class; - if (device_create_file(&host->dev, &dev_attr_ibdev)) - goto err_class; - if (device_create_file(&host->dev, &dev_attr_port)) - goto err_class; + if (dev_set_name(&host->dev, "srp-%s-%u", dev_name(&device->dev->dev), + port)) + goto put_host; + if (device_add(&host->dev)) + goto put_host; return host; -err_class: - device_unregister(&host->dev); - -free_host: - kfree(host); - +put_host: + device_del(&host->dev); + put_device(&host->dev); return NULL; } @@ -3936,7 +3938,7 @@ static void srp_rename_dev(struct ib_device *device, void *client_data) list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { char name[IB_DEVICE_NAME_MAX + 8]; - snprintf(name, sizeof(name), "srp-%s-%d", + snprintf(name, sizeof(name), "srp-%s-%u", dev_name(&device->dev), host->port); device_rename(&host->dev, name); } @@ -3948,7 +3950,7 @@ static int srp_add_one(struct ib_device *device) struct ib_device_attr *attr = &device->attrs; struct srp_host *host; int mr_page_shift; - unsigned int p; + u32 p; u64 max_pages_per_mr; unsigned int flags = 0; @@ -4030,12 +4032,11 @@ static void srp_remove_one(struct ib_device *device, void *client_data) srp_dev = client_data; list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { - device_unregister(&host->dev); /* - * Wait for the sysfs entry to go away, so that no new - * target ports can be created. + * Remove the add_target sysfs entry so that no new target ports + * can be created. */ - wait_for_completion(&host->released); + device_del(&host->dev); /* * Remove all target ports. @@ -4053,7 +4054,7 @@ static void srp_remove_one(struct ib_device *device, void *client_data) */ flush_workqueue(srp_remove_wq); - kfree(host); + put_device(&host->dev); } ib_dealloc_pd(srp_dev->pd); diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h index 55a575e2cacefea5cd6175deca6dd167841b7e99..00b0068fda208b5cec93658c1eaa58d999a58367 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.h +++ b/drivers/infiniband/ulp/srp/ib_srp.h @@ -120,11 +120,10 @@ struct srp_device { */ struct srp_host { struct srp_device *srp_dev; - u8 port; + u32 port; struct device dev; struct list_head target_list; spinlock_t target_lock; - struct completion released; struct list_head list; struct mutex add_target_mutex; }; diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 21cbe30d526fdc0fff39f7b79d65675316c396c7..3c3fae738c3edf80a6f47181ce1baf3a6f542b26 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1421,7 +1421,7 @@ static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch, srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID; srp_rsp->sense_data_len = cpu_to_be32(sense_data_len); - memcpy(srp_rsp + 1, sense_data, sense_data_len); + memcpy(srp_rsp->data, sense_data, sense_data_len); } return sizeof(*srp_rsp) + sense_data_len; @@ -2300,7 +2300,7 @@ static int srpt_cm_req_recv(struct srpt_device *const sdev, goto free_recv_ring; } - strlcpy(ch->sess_name, src_addr, sizeof(ch->sess_name)); + strscpy(ch->sess_name, src_addr, sizeof(ch->sess_name)); snprintf(i_port_id, sizeof(i_port_id), "0x%016llx%016llx", be64_to_cpu(*(__be64 *)nexus->i_port_id), be64_to_cpu(*(__be64 *)(nexus->i_port_id + 8))); @@ -3191,7 +3191,7 @@ static int srpt_add_one(struct ib_device *device) * if this HCA is gone bad and replaced by different HCA */ ret = sdev->cm_id ? - ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0) : + ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid)) : 0; if (ret < 0) { pr_err("ib_cm_listen() failed: %d (cm_id state = %d)\n", ret, diff --git a/drivers/input/ff-core.c b/drivers/input/ff-core.c index fa8d1a4660142fd5427e802bc1da9bcee58b0838..16231fe080b006204f33ca666329d9507e40e584 100644 --- a/drivers/input/ff-core.c +++ b/drivers/input/ff-core.c @@ -6,9 +6,6 @@ * Copyright (c) 2006 Dmitry Torokhov */ -/* - */ - /* #define DEBUG */ #include diff --git a/drivers/input/ff-memless.c b/drivers/input/ff-memless.c index 8229a90069176be5d4eeb646d4899c29f485d343..c321cdabd21418dc7bdc21b071e6e4da8009afb7 100644 --- a/drivers/input/ff-memless.c +++ b/drivers/input/ff-memless.c @@ -6,9 +6,6 @@ * Copyright (c) 2006 Dmitry Torokhov */ -/* - */ - /* #define DEBUG */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/input/gameport/emu10k1-gp.c b/drivers/input/gameport/emu10k1-gp.c index 11bbd1edfdb4aa77c032c959e9452fa8fee2758d..76ce41e58df0c78742c95098e88831750dbc3247 100644 --- a/drivers/input/gameport/emu10k1-gp.c +++ b/drivers/input/gameport/emu10k1-gp.c @@ -7,9 +7,6 @@ * EMU10k1 - SB Live / Audigy - gameport driver for Linux */ -/* - */ - #include #include diff --git a/drivers/input/gameport/lightning.c b/drivers/input/gameport/lightning.c index 87eeb4b5b5b57e1a1c7d206734b77268f4f42fa3..2ce717b25a84fcf717e1c95e1f745b379487429e 100644 --- a/drivers/input/gameport/lightning.c +++ b/drivers/input/gameport/lightning.c @@ -7,9 +7,6 @@ * PDPI Lightning 4 gamecard driver for Linux. */ -/* - */ - #include #include #include diff --git a/drivers/input/gameport/ns558.c b/drivers/input/gameport/ns558.c index 2f80b7f1b7362bea203c2127807b2a9c503eebcd..91a8cd346e9b763d1b730ba2dc83260eb7c1db11 100644 --- a/drivers/input/gameport/ns558.c +++ b/drivers/input/gameport/ns558.c @@ -8,9 +8,6 @@ * NS558 based standard IBM game port driver for Linux */ -/* - */ - #include #include diff --git a/drivers/input/joydev.c b/drivers/input/joydev.c index b45ddb45700280c22d2bd7169d93bbbbe6299bea..5824bca02e5a090ec86a53d8ba2603b4c5ea1277 100644 --- a/drivers/input/joydev.c +++ b/drivers/input/joydev.c @@ -746,7 +746,7 @@ static void joydev_cleanup(struct joydev *joydev) } /* - * These codes are copied from from hid-ids.h, unfortunately there is no common + * These codes are copied from hid-ids.h, unfortunately there is no common * usb_ids/bt_ids.h header. */ #define USB_VENDOR_ID_SONY 0x054c diff --git a/drivers/input/joystick/a3d.c b/drivers/input/joystick/a3d.c index 68475fad177c401aad7e91024e415180c2e1f842..fd1827baf27cd3105653f355e33c06a3a2d343b4 100644 --- a/drivers/input/joystick/a3d.c +++ b/drivers/input/joystick/a3d.c @@ -7,9 +7,6 @@ * FP-Gaming Assassin 3D joystick driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/adc-joystick.c b/drivers/input/joystick/adc-joystick.c index e0cfdc84763f417795641de2d6b6ee5a6d426898..c0deff5d42824ab02e91eda6a77e45ae293627cc 100644 --- a/drivers/input/joystick/adc-joystick.c +++ b/drivers/input/joystick/adc-joystick.c @@ -26,8 +26,23 @@ struct adc_joystick { struct adc_joystick_axis *axes; struct iio_channel *chans; int num_chans; + bool polled; }; +static void adc_joystick_poll(struct input_dev *input) +{ + struct adc_joystick *joy = input_get_drvdata(input); + int i, val, ret; + + for (i = 0; i < joy->num_chans; i++) { + ret = iio_read_channel_raw(&joy->chans[i], &val); + if (ret < 0) + return; + input_report_abs(input, joy->axes[i].code, val); + } + input_sync(input); +} + static int adc_joystick_handle(const void *data, void *private) { struct adc_joystick *joy = private; @@ -179,6 +194,7 @@ static int adc_joystick_probe(struct platform_device *pdev) int error; int bits; int i; + unsigned int poll_interval; joy = devm_kzalloc(dev, sizeof(*joy), GFP_KERNEL); if (!joy) @@ -192,8 +208,25 @@ static int adc_joystick_probe(struct platform_device *pdev) return error; } - /* Count how many channels we got. NULL terminated. */ + error = device_property_read_u32(dev, "poll-interval", &poll_interval); + if (error) { + /* -EINVAL means the property is absent. */ + if (error != -EINVAL) + return error; + } else if (poll_interval == 0) { + dev_err(dev, "Unable to get poll-interval\n"); + return -EINVAL; + } else { + joy->polled = true; + } + + /* + * Count how many channels we got. NULL terminated. + * Do not check the storage size if using polling. + */ for (i = 0; joy->chans[i].indio_dev; i++) { + if (joy->polled) + continue; bits = joy->chans[i].channel->scan_type.storagebits; if (!bits || bits > 16) { dev_err(dev, "Unsupported channel storage size\n"); @@ -215,23 +248,31 @@ static int adc_joystick_probe(struct platform_device *pdev) joy->input = input; input->name = pdev->name; input->id.bustype = BUS_HOST; - input->open = adc_joystick_open; - input->close = adc_joystick_close; error = adc_joystick_set_axes(dev, joy); if (error) return error; - joy->buffer = iio_channel_get_all_cb(dev, adc_joystick_handle, joy); - if (IS_ERR(joy->buffer)) { - dev_err(dev, "Unable to allocate callback buffer\n"); - return PTR_ERR(joy->buffer); - } + if (joy->polled) { + input_setup_polling(input, adc_joystick_poll); + input_set_poll_interval(input, poll_interval); + } else { + input->open = adc_joystick_open; + input->close = adc_joystick_close; + + joy->buffer = iio_channel_get_all_cb(dev, adc_joystick_handle, + joy); + if (IS_ERR(joy->buffer)) { + dev_err(dev, "Unable to allocate callback buffer\n"); + return PTR_ERR(joy->buffer); + } - error = devm_add_action_or_reset(dev, adc_joystick_cleanup, joy->buffer); - if (error) { - dev_err(dev, "Unable to add action\n"); - return error; + error = devm_add_action_or_reset(dev, adc_joystick_cleanup, + joy->buffer); + if (error) { + dev_err(dev, "Unable to add action\n"); + return error; + } } input_set_drvdata(input, joy); diff --git a/drivers/input/joystick/adi.c b/drivers/input/joystick/adi.c index e10d57bf1180c7c2eb06824ac31d71239a14ae88..f1a720be458b7d46b3b7e24840cf2b01feef04dc 100644 --- a/drivers/input/joystick/adi.c +++ b/drivers/input/joystick/adi.c @@ -7,9 +7,6 @@ * Logitech ADI joystick family driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/amijoy.c b/drivers/input/joystick/amijoy.c index 12456a196dc73045623884535487dbc5fa0f02c3..3752dc2a20868c1e4525168c47fb9dc1b8115e62 100644 --- a/drivers/input/joystick/amijoy.c +++ b/drivers/input/joystick/amijoy.c @@ -7,9 +7,6 @@ * Driver for Amiga joysticks for Linux/m68k */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/analog.c b/drivers/input/joystick/analog.c index 3088c5b829f07a6f8050d680a226f10979c2dc6d..0c9e172a98181532d19502a10cf0ba88d2775122 100644 --- a/drivers/input/joystick/analog.c +++ b/drivers/input/joystick/analog.c @@ -7,9 +7,6 @@ * Analog joystick and gamepad driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/as5011.c b/drivers/input/joystick/as5011.c index 34bcd99a46f587fa364de798739dfbcb37e25ce9..2beda29021a32afc344781b17830f18615f93667 100644 --- a/drivers/input/joystick/as5011.c +++ b/drivers/input/joystick/as5011.c @@ -327,7 +327,7 @@ err_free_mem: return error; } -static int as5011_remove(struct i2c_client *client) +static void as5011_remove(struct i2c_client *client) { struct as5011_device *as5011 = i2c_get_clientdata(client); @@ -337,8 +337,6 @@ static int as5011_remove(struct i2c_client *client) input_unregister_device(as5011->input_dev); kfree(as5011); - - return 0; } static const struct i2c_device_id as5011_id[] = { diff --git a/drivers/input/joystick/cobra.c b/drivers/input/joystick/cobra.c index 41e1936a847bd196a5c8da800244fb3ac06c1de0..7ff78c9388bd3845de46ad51729c22d70df929a0 100644 --- a/drivers/input/joystick/cobra.c +++ b/drivers/input/joystick/cobra.c @@ -7,9 +7,6 @@ * Creative Labs Blaster GamePad Cobra driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/db9.c b/drivers/input/joystick/db9.c index 434d265fa2e831b3490d1ddd302081713d88848d..4fba28b1a1e75b24537be9aab2c3b5c1567123fb 100644 --- a/drivers/input/joystick/db9.c +++ b/drivers/input/joystick/db9.c @@ -10,9 +10,6 @@ * Atari, Amstrad, Commodore, Amiga, Sega, etc. joystick driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/gamecon.c b/drivers/input/joystick/gamecon.c index d37645e496ff17838d443699d549e42ec0f5d4c7..41d5dac05448137f0280ad32241b596665149382 100644 --- a/drivers/input/joystick/gamecon.c +++ b/drivers/input/joystick/gamecon.c @@ -11,9 +11,6 @@ * Raphael Assenat */ -/* - */ - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include diff --git a/drivers/input/joystick/gf2k.c b/drivers/input/joystick/gf2k.c index 920feba967f6a8d35515ef42eb51118bc2330863..abefbd1484dfea4480662dde20900f9d1b9909f9 100644 --- a/drivers/input/joystick/gf2k.c +++ b/drivers/input/joystick/gf2k.c @@ -7,9 +7,6 @@ * Genius Flight 2000 joystick driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/grip.c b/drivers/input/joystick/grip.c index fe798bc87950193b0d650ae12c032c1a563d526a..0e86b269a90ea5ff7a63fbece8ede1417caaf581 100644 --- a/drivers/input/joystick/grip.c +++ b/drivers/input/joystick/grip.c @@ -7,9 +7,6 @@ * Gravis/Kensington GrIP protocol joystick and gamepad driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/guillemot.c b/drivers/input/joystick/guillemot.c index 8eeacdb007c1d522146a526592b01292e249346f..205eb6f8b84d2633e40dd0fd961103efac0420cf 100644 --- a/drivers/input/joystick/guillemot.c +++ b/drivers/input/joystick/guillemot.c @@ -7,9 +7,6 @@ * Guillemot Digital Interface Protocol driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/iforce/iforce-main.c b/drivers/input/joystick/iforce/iforce-main.c index b2a68bc9f0b4d6b77a439d35c57e7038c2e3ea8f..b86de1312512bd3a1b41ab795956bdbe317f5974 100644 --- a/drivers/input/joystick/iforce/iforce-main.c +++ b/drivers/input/joystick/iforce/iforce-main.c @@ -50,6 +50,7 @@ static struct iforce_device iforce_device[] = { { 0x046d, 0xc291, "Logitech WingMan Formula Force", btn_wheel, abs_wheel, ff_iforce }, { 0x05ef, 0x020a, "AVB Top Shot Pegasus", btn_joystick_avb, abs_avb_pegasus, ff_iforce }, { 0x05ef, 0x8884, "AVB Mag Turbo Force", btn_wheel, abs_wheel, ff_iforce }, + { 0x05ef, 0x8886, "Boeder Force Feedback Wheel", btn_wheel, abs_wheel, ff_iforce }, { 0x05ef, 0x8888, "AVB Top Shot Force Feedback Racing Wheel", btn_wheel, abs_wheel, ff_iforce }, //? { 0x061c, 0xc0a4, "ACT LABS Force RS", btn_wheel, abs_wheel, ff_iforce }, //? { 0x061c, 0xc084, "ACT LABS Force RS", btn_wheel, abs_wheel, ff_iforce }, diff --git a/drivers/input/joystick/iforce/iforce-serio.c b/drivers/input/joystick/iforce/iforce-serio.c index f95a81b9fac72ce2050cac3c27df2b65166c6f13..2380546d79782d5b5dee309a65818f7299b6bff4 100644 --- a/drivers/input/joystick/iforce/iforce-serio.c +++ b/drivers/input/joystick/iforce/iforce-serio.c @@ -39,7 +39,7 @@ static void iforce_serio_xmit(struct iforce *iforce) again: if (iforce->xmit.head == iforce->xmit.tail) { - clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); + iforce_clear_xmit_and_wake(iforce); spin_unlock_irqrestore(&iforce->xmit_lock, flags); return; } @@ -64,7 +64,7 @@ again: if (test_and_clear_bit(IFORCE_XMIT_AGAIN, iforce->xmit_flags)) goto again; - clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); + iforce_clear_xmit_and_wake(iforce); spin_unlock_irqrestore(&iforce->xmit_lock, flags); } @@ -169,7 +169,7 @@ static irqreturn_t iforce_serio_irq(struct serio *serio, iforce_serio->cmd_response_len = iforce_serio->len; /* Signal that command is done */ - wake_up(&iforce->wait); + wake_up_all(&iforce->wait); } else if (likely(iforce->type)) { iforce_process_packet(iforce, iforce_serio->id, iforce_serio->data_in, diff --git a/drivers/input/joystick/iforce/iforce-usb.c b/drivers/input/joystick/iforce/iforce-usb.c index ea58805c480fa91c9d2dfdfec4d857200986021f..cba92bd590a8d83b4ded836059b5d882fb8751b2 100644 --- a/drivers/input/joystick/iforce/iforce-usb.c +++ b/drivers/input/joystick/iforce/iforce-usb.c @@ -30,7 +30,7 @@ static void __iforce_usb_xmit(struct iforce *iforce) spin_lock_irqsave(&iforce->xmit_lock, flags); if (iforce->xmit.head == iforce->xmit.tail) { - clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); + iforce_clear_xmit_and_wake(iforce); spin_unlock_irqrestore(&iforce->xmit_lock, flags); return; } @@ -58,9 +58,9 @@ static void __iforce_usb_xmit(struct iforce *iforce) XMIT_INC(iforce->xmit.tail, n); if ( (n=usb_submit_urb(iforce_usb->out, GFP_ATOMIC)) ) { - clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); dev_warn(&iforce_usb->intf->dev, "usb_submit_urb failed %d\n", n); + iforce_clear_xmit_and_wake(iforce); } /* The IFORCE_XMIT_RUNNING bit is not cleared here. That's intended. @@ -175,15 +175,15 @@ static void iforce_usb_out(struct urb *urb) struct iforce *iforce = &iforce_usb->iforce; if (urb->status) { - clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); dev_dbg(&iforce_usb->intf->dev, "urb->status %d, exiting\n", urb->status); + iforce_clear_xmit_and_wake(iforce); return; } __iforce_usb_xmit(iforce); - wake_up(&iforce->wait); + wake_up_all(&iforce->wait); } static int iforce_usb_probe(struct usb_interface *intf, diff --git a/drivers/input/joystick/iforce/iforce.h b/drivers/input/joystick/iforce/iforce.h index 6aa761ebbdf7725e639ccd7cb974fb36bcd3ad45..9ccb9107ccbef09651a7980023cbd7f30067e0c0 100644 --- a/drivers/input/joystick/iforce/iforce.h +++ b/drivers/input/joystick/iforce/iforce.h @@ -119,6 +119,12 @@ static inline int iforce_get_id_packet(struct iforce *iforce, u8 id, response_data, response_len); } +static inline void iforce_clear_xmit_and_wake(struct iforce *iforce) +{ + clear_bit(IFORCE_XMIT_RUNNING, iforce->xmit_flags); + wake_up_all(&iforce->wait); +} + /* Public functions */ /* iforce-main.c */ int iforce_init_device(struct device *parent, u16 bustype, diff --git a/drivers/input/joystick/interact.c b/drivers/input/joystick/interact.c index ca22d84e5c842ac4d1a124fe65a7efb2c7bf7270..03a9f0829f7ed092f0413534a9b201ff4afedb66 100644 --- a/drivers/input/joystick/interact.c +++ b/drivers/input/joystick/interact.c @@ -10,9 +10,6 @@ * InterAct digital gamepad/joystick driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/joydump.c b/drivers/input/joystick/joydump.c index 70f63f9550e72cd498b519d6614ebda54899f07d..865652a7821da3b2fb49a664fceb9b209d3b7911 100644 --- a/drivers/input/joystick/joydump.c +++ b/drivers/input/joystick/joydump.c @@ -8,9 +8,6 @@ * out of the joystick port into the syslog ... */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/magellan.c b/drivers/input/joystick/magellan.c index edb8e1982e2601282496ed7ed4cb257e8d9fbe6b..017ef8c6170b75eebe301665e01930a5897f22a2 100644 --- a/drivers/input/joystick/magellan.c +++ b/drivers/input/joystick/magellan.c @@ -7,9 +7,6 @@ * Magellan and Space Mouse 6dof controller driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/sidewinder.c b/drivers/input/joystick/sidewinder.c index 8e9672deb1ebef5a5be7a8ef27647340c13d6ab2..7282301c3ae73a1e5ad9c96ed952d90e3cfaa114 100644 --- a/drivers/input/joystick/sidewinder.c +++ b/drivers/input/joystick/sidewinder.c @@ -7,9 +7,6 @@ * Microsoft SideWinder joystick family driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/spaceball.c b/drivers/input/joystick/spaceball.c index a85a4f33aea8ca11eb294a117ed79f335a5c145c..fa8ec533cd6967a24cd52206beb7483c6952c524 100644 --- a/drivers/input/joystick/spaceball.c +++ b/drivers/input/joystick/spaceball.c @@ -11,9 +11,6 @@ * SpaceTec SpaceBall 2003/3003/4000 FLX driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/spaceorb.c b/drivers/input/joystick/spaceorb.c index 557171483256d4433dea9c9ac124e016ac58345a..dbbc69f17c89a735e1982e49956a307c166647ef 100644 --- a/drivers/input/joystick/spaceorb.c +++ b/drivers/input/joystick/spaceorb.c @@ -10,9 +10,6 @@ * SpaceTec SpaceOrb 360 and Avenger 6dof controller driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/stinger.c b/drivers/input/joystick/stinger.c index c20425f52bd8afd233098816fc85fc474f229a83..530de468cb6171bf77572ea92bf9f58b74536d71 100644 --- a/drivers/input/joystick/stinger.c +++ b/drivers/input/joystick/stinger.c @@ -8,9 +8,6 @@ * Gravis Stinger gamepad driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/tmdc.c b/drivers/input/joystick/tmdc.c index 7416de84b955c41fa74a7de323edfeb8fc4cab3f..93562ecc0ca1c03888e34c2bf0829f7b70b1fbda 100644 --- a/drivers/input/joystick/tmdc.c +++ b/drivers/input/joystick/tmdc.c @@ -10,9 +10,6 @@ * ThrustMaster DirectConnect (BSP) joystick family driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/turbografx.c b/drivers/input/joystick/turbografx.c index dfe7a2cacce2810ea8dcbd418529d08568579198..dfb9c684651f355b34bfd1d11447c96869b55b08 100644 --- a/drivers/input/joystick/turbografx.c +++ b/drivers/input/joystick/turbografx.c @@ -10,9 +10,6 @@ * TurboGraFX parallel port interface driver for Linux. */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/twidjoy.c b/drivers/input/joystick/twidjoy.c index 174c69a188fb136a4a2c42f456fd7103a62b9b0b..9b6792ac27f10c693e7c4cc2dbb06824d398bc0c 100644 --- a/drivers/input/joystick/twidjoy.c +++ b/drivers/input/joystick/twidjoy.c @@ -32,9 +32,6 @@ * Arndt Schoenewald */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/warrior.c b/drivers/input/joystick/warrior.c index 42bdbc28d95d20cf40fbc81107a0667e21669301..f66bddf145c22af99f8481577b28cefaa4ff9841 100644 --- a/drivers/input/joystick/warrior.c +++ b/drivers/input/joystick/warrior.c @@ -7,9 +7,6 @@ * Logitech WingMan Warrior joystick driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 18190b529bca3fdf81b6b9e8aee9e11d0ec7da16..2959d80f7fdb64364413a049c0ccb22df7bb938f 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -61,6 +61,7 @@ * Later changes can be tracked in SCM. */ +#include #include #include #include @@ -80,6 +81,9 @@ #define MAP_TRIGGERS_TO_BUTTONS (1 << 1) #define MAP_STICKS_TO_NULL (1 << 2) #define MAP_SELECT_BUTTON (1 << 3) +#define MAP_PADDLES (1 << 4) +#define MAP_PROFILE_BUTTON (1 << 5) + #define DANCEPAD_MAP_CONFIG (MAP_DPAD_TO_BUTTONS | \ MAP_TRIGGERS_TO_BUTTONS | MAP_STICKS_TO_NULL) @@ -89,6 +93,17 @@ #define XTYPE_XBOXONE 3 #define XTYPE_UNKNOWN 4 +/* Send power-off packet to xpad360w after holding the mode button for this many + * seconds + */ +#define XPAD360W_POWEROFF_TIMEOUT 5 + +#define PKT_XB 0 +#define PKT_XBE1 1 +#define PKT_XBE2_FW_OLD 2 +#define PKT_XBE2_FW_5_EARLY 3 +#define PKT_XBE2_FW_5_11 4 + static bool dpad_to_buttons; module_param(dpad_to_buttons, bool, S_IRUGO); MODULE_PARM_DESC(dpad_to_buttons, "Map D-PAD to buttons rather than axes for unknown pads"); @@ -111,8 +126,11 @@ static const struct xpad_device { char *name; u8 mapping; u8 xtype; + u8 packet_type; } xpad_device[] = { { 0x0079, 0x18d4, "GPD Win 2 X-Box Controller", 0, XTYPE_XBOX360 }, + { 0x03eb, 0xff01, "Wooting One (Legacy)", 0, XTYPE_XBOX360 }, + { 0x03eb, 0xff02, "Wooting Two (Legacy)", 0, XTYPE_XBOX360 }, { 0x044f, 0x0f00, "Thrustmaster Wheel", 0, XTYPE_XBOX }, { 0x044f, 0x0f03, "Thrustmaster Wheel", 0, XTYPE_XBOX }, { 0x044f, 0x0f07, "Thrustmaster, Inc. Controller", 0, XTYPE_XBOX }, @@ -128,9 +146,11 @@ static const struct xpad_device { { 0x045e, 0x0291, "Xbox 360 Wireless Receiver (XBOX)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, { 0x045e, 0x02d1, "Microsoft X-Box One pad", 0, XTYPE_XBOXONE }, { 0x045e, 0x02dd, "Microsoft X-Box One pad (Firmware 2015)", 0, XTYPE_XBOXONE }, - { 0x045e, 0x02e3, "Microsoft X-Box One Elite pad", 0, XTYPE_XBOXONE }, + { 0x045e, 0x02e3, "Microsoft X-Box One Elite pad", MAP_PADDLES, XTYPE_XBOXONE }, + { 0x045e, 0x0b00, "Microsoft X-Box One Elite 2 pad", MAP_PADDLES, XTYPE_XBOXONE }, { 0x045e, 0x02ea, "Microsoft X-Box One S pad", 0, XTYPE_XBOXONE }, { 0x045e, 0x0719, "Xbox 360 Wireless Receiver", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360W }, + { 0x045e, 0x0b0a, "Microsoft X-Box Adaptive Controller", MAP_PROFILE_BUTTON, XTYPE_XBOXONE }, { 0x045e, 0x0b12, "Microsoft Xbox Series S|X Controller", MAP_SELECT_BUTTON, XTYPE_XBOXONE }, { 0x046d, 0xc21d, "Logitech Gamepad F310", 0, XTYPE_XBOX360 }, { 0x046d, 0xc21e, "Logitech Gamepad F510", 0, XTYPE_XBOX360 }, @@ -244,6 +264,7 @@ static const struct xpad_device { { 0x0f0d, 0x0063, "Hori Real Arcade Pro Hayabusa (USA) Xbox One", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x0f0d, 0x0067, "HORIPAD ONE", 0, XTYPE_XBOXONE }, { 0x0f0d, 0x0078, "Hori Real Arcade Pro V Kai Xbox One", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, + { 0x0f0d, 0x00c5, "Hori Fighting Commander ONE", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x0f30, 0x010b, "Philips Recoil", 0, XTYPE_XBOX }, { 0x0f30, 0x0202, "Joytech Advanced Controller", 0, XTYPE_XBOX }, { 0x0f30, 0x8888, "BigBen XBMiniPad Controller", 0, XTYPE_XBOX }, @@ -260,6 +281,7 @@ static const struct xpad_device { { 0x1430, 0x8888, "TX6500+ Dance Pad (first generation)", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX }, { 0x1430, 0xf801, "RedOctane Controller", 0, XTYPE_XBOX360 }, { 0x146b, 0x0601, "BigBen Interactive XBOX 360 Controller", 0, XTYPE_XBOX360 }, + { 0x146b, 0x0604, "Bigben Interactive DAIJA Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x1532, 0x0037, "Razer Sabertooth", 0, XTYPE_XBOX360 }, { 0x1532, 0x0a00, "Razer Atrox Arcade Stick", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOXONE }, { 0x1532, 0x0a03, "Razer Wildcat", 0, XTYPE_XBOXONE }, @@ -325,6 +347,7 @@ static const struct xpad_device { { 0x24c6, 0x5502, "Hori Fighting Stick VX Alt", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x5503, "Hori Fighting Edge", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x5506, "Hori SOULCALIBUR V Stick", 0, XTYPE_XBOX360 }, + { 0x24c6, 0x5510, "Hori Fighting Commander ONE (Xbox 360/PC Mode)", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x550d, "Hori GEM Xbox controller", 0, XTYPE_XBOX360 }, { 0x24c6, 0x550e, "Hori Real Arcade Pro V Kai 360", MAP_TRIGGERS_TO_BUTTONS, XTYPE_XBOX360 }, { 0x24c6, 0x551a, "PowerA FUSION Pro Controller", 0, XTYPE_XBOXONE }, @@ -334,6 +357,14 @@ static const struct xpad_device { { 0x24c6, 0x5b03, "Thrustmaster Ferrari 458 Racing Wheel", 0, XTYPE_XBOX360 }, { 0x24c6, 0x5d04, "Razer Sabertooth", 0, XTYPE_XBOX360 }, { 0x24c6, 0xfafe, "Rock Candy Gamepad for Xbox 360", 0, XTYPE_XBOX360 }, + { 0x2563, 0x058d, "OneXPlayer Gamepad", 0, XTYPE_XBOX360 }, + { 0x2dc8, 0x2000, "8BitDo Pro 2 Wired Controller fox Xbox", 0, XTYPE_XBOXONE }, + { 0x31e3, 0x1100, "Wooting One", 0, XTYPE_XBOX360 }, + { 0x31e3, 0x1200, "Wooting Two", 0, XTYPE_XBOX360 }, + { 0x31e3, 0x1210, "Wooting Lekker", 0, XTYPE_XBOX360 }, + { 0x31e3, 0x1220, "Wooting Two HE", 0, XTYPE_XBOX360 }, + { 0x31e3, 0x1300, "Wooting 60HE (AVR)", 0, XTYPE_XBOX360 }, + { 0x31e3, 0x1310, "Wooting 60HE (ARM)", 0, XTYPE_XBOX360 }, { 0x3285, 0x0607, "Nacon GC-100", 0, XTYPE_XBOX360 }, { 0x3767, 0x0101, "Fanatec Speedster 3 Forceshock Wheel", 0, XTYPE_XBOX }, { 0xffff, 0xffff, "Chinese-made Xbox Controller", 0, XTYPE_XBOX }, @@ -390,6 +421,13 @@ static const signed short xpad_abs_triggers[] = { -1 }; +/* used when the controller has extra paddle buttons */ +static const signed short xpad_btn_paddles[] = { + BTN_TRIGGER_HAPPY5, BTN_TRIGGER_HAPPY6, /* paddle upper right, lower right */ + BTN_TRIGGER_HAPPY7, BTN_TRIGGER_HAPPY8, /* paddle upper left, lower left */ + -1 /* terminating entry */ +}; + /* * Xbox 360 has a vendor-specific class, so we cannot match it with only * USB_INTERFACE_INFO (also specifically refused by USB subsystem), so we @@ -419,6 +457,7 @@ static const signed short xpad_abs_triggers[] = { static const struct usb_device_id xpad_table[] = { { USB_INTERFACE_INFO('X', 'B', 0) }, /* X-Box USB-IF not approved class */ XPAD_XBOX360_VENDOR(0x0079), /* GPD Win 2 Controller */ + XPAD_XBOX360_VENDOR(0x03eb), /* Wooting Keyboards (Legacy) */ XPAD_XBOX360_VENDOR(0x044f), /* Thrustmaster X-Box 360 controllers */ XPAD_XBOX360_VENDOR(0x045e), /* Microsoft X-Box 360 controllers */ XPAD_XBOXONE_VENDOR(0x045e), /* Microsoft X-Box One controllers */ @@ -429,6 +468,7 @@ static const struct usb_device_id xpad_table[] = { { USB_DEVICE(0x0738, 0x4540) }, /* Mad Catz Beat Pad */ XPAD_XBOXONE_VENDOR(0x0738), /* Mad Catz FightStick TE 2 */ XPAD_XBOX360_VENDOR(0x07ff), /* Mad Catz GamePad */ + XPAD_XBOX360_VENDOR(0x0c12), /* Zeroplus X-Box 360 controllers */ XPAD_XBOX360_VENDOR(0x0e6f), /* 0x0e6f X-Box 360 controllers */ XPAD_XBOXONE_VENDOR(0x0e6f), /* 0x0e6f X-Box One controllers */ XPAD_XBOX360_VENDOR(0x0f0d), /* Hori Controllers */ @@ -450,8 +490,12 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOXONE_VENDOR(0x20d6), /* PowerA Controllers */ XPAD_XBOX360_VENDOR(0x24c6), /* PowerA Controllers */ XPAD_XBOXONE_VENDOR(0x24c6), /* PowerA Controllers */ + XPAD_XBOX360_VENDOR(0x2563), /* OneXPlayer Gamepad */ + XPAD_XBOX360_VENDOR(0x260d), /* Dareu H101 */ + XPAD_XBOXONE_VENDOR(0x2dc8), /* 8BitDo Pro 2 Wired Controller for Xbox */ XPAD_XBOXONE_VENDOR(0x2e24), /* Hyperkin Duke X-Box One pad */ XPAD_XBOX360_VENDOR(0x2f24), /* GameSir Controllers */ + XPAD_XBOX360_VENDOR(0x31e3), /* Wooting Keyboards */ XPAD_XBOX360_VENDOR(0x3285), /* Nacon GC-100 */ { } }; @@ -473,13 +517,52 @@ struct xboxone_init_packet { .len = ARRAY_SIZE(_data), \ } +/* + * starting with xbox one, the game input protocol is used + * magic numbers are taken from + * - https://github.com/xpadneo/gip-dissector/blob/main/src/gip-dissector.lua + * - https://github.com/medusalix/xone/blob/master/bus/protocol.c + */ +#define GIP_CMD_ACK 0x01 +#define GIP_CMD_IDENTIFY 0x04 +#define GIP_CMD_POWER 0x05 +#define GIP_CMD_AUTHENTICATE 0x06 +#define GIP_CMD_VIRTUAL_KEY 0x07 +#define GIP_CMD_RUMBLE 0x09 +#define GIP_CMD_LED 0x0a +#define GIP_CMD_FIRMWARE 0x0c +#define GIP_CMD_INPUT 0x20 + +#define GIP_SEQ0 0x00 + +#define GIP_OPT_ACK 0x10 +#define GIP_OPT_INTERNAL 0x20 + +/* + * length of the command payload encoded with + * https://en.wikipedia.org/wiki/LEB128 + * which is a no-op for N < 128 + */ +#define GIP_PL_LEN(N) (N) + +/* + * payload specific defines + */ +#define GIP_PWR_ON 0x00 +#define GIP_LED_ON 0x01 + +#define GIP_MOTOR_R BIT(0) +#define GIP_MOTOR_L BIT(1) +#define GIP_MOTOR_RT BIT(2) +#define GIP_MOTOR_LT BIT(3) +#define GIP_MOTOR_ALL (GIP_MOTOR_R | GIP_MOTOR_L | GIP_MOTOR_RT | GIP_MOTOR_LT) /* * This packet is required for all Xbox One pads with 2015 * or later firmware installed (or present from the factory). */ -static const u8 xboxone_fw2015_init[] = { - 0x05, 0x20, 0x00, 0x01, 0x00 +static const u8 xboxone_power_on[] = { + GIP_CMD_POWER, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(1), GIP_PWR_ON }; /* @@ -489,7 +572,16 @@ static const u8 xboxone_fw2015_init[] = { * Bluetooth mode. */ static const u8 xboxone_s_init[] = { - 0x05, 0x20, 0x00, 0x0f, 0x06 + GIP_CMD_POWER, GIP_OPT_INTERNAL, GIP_SEQ0, 0x0f, 0x06 +}; + +/* + * This packet is required to get additional input data + * from Xbox One Elite Series 2 (0x045e:0x0b00) pads. + * We mostly do this right now to get paddle data + */ +static const u8 extra_input_packet_init[] = { + 0x4d, 0x10, 0x01, 0x02, 0x07, 0x00 }; /* @@ -497,9 +589,9 @@ static const u8 xboxone_s_init[] = { * (0x0e6f:0x0165) to finish initialization and for Hori pads * (0x0f0d:0x0067) to make the analog sticks work. */ -static const u8 xboxone_hori_init[] = { - 0x01, 0x20, 0x00, 0x09, 0x00, 0x04, 0x20, 0x3a, - 0x00, 0x00, 0x00, 0x80, 0x00 +static const u8 xboxone_hori_ack_id[] = { + GIP_CMD_ACK, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(9), + 0x00, GIP_CMD_IDENTIFY, GIP_OPT_INTERNAL, 0x3a, 0x00, 0x00, 0x00, 0x80, 0x00 }; /* @@ -507,8 +599,8 @@ static const u8 xboxone_hori_init[] = { * sending input reports. These pads include: (0x0e6f:0x02ab), * (0x0e6f:0x02a4), (0x0e6f:0x02a6). */ -static const u8 xboxone_pdp_init1[] = { - 0x0a, 0x20, 0x00, 0x03, 0x00, 0x01, 0x14 +static const u8 xboxone_pdp_led_on[] = { + GIP_CMD_LED, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(3), 0x00, GIP_LED_ON, 0x14 }; /* @@ -516,8 +608,8 @@ static const u8 xboxone_pdp_init1[] = { * sending input reports. These pads include: (0x0e6f:0x02ab), * (0x0e6f:0x02a4), (0x0e6f:0x02a6). */ -static const u8 xboxone_pdp_init2[] = { - 0x06, 0x20, 0x00, 0x02, 0x01, 0x00 +static const u8 xboxone_pdp_auth[] = { + GIP_CMD_AUTHENTICATE, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(2), 0x01, 0x00 }; /* @@ -525,8 +617,8 @@ static const u8 xboxone_pdp_init2[] = { * sending input reports. One of those pads is (0x24c6:0x543a). */ static const u8 xboxone_rumblebegin_init[] = { - 0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00, - 0x1D, 0x1D, 0xFF, 0x00, 0x00 + GIP_CMD_RUMBLE, 0x00, GIP_SEQ0, GIP_PL_LEN(9), + 0x00, GIP_MOTOR_ALL, 0x00, 0x00, 0x1D, 0x1D, 0xFF, 0x00, 0x00 }; /* @@ -536,8 +628,8 @@ static const u8 xboxone_rumblebegin_init[] = { * spin up to enough speed to actually vibrate the gamepad. */ static const u8 xboxone_rumbleend_init[] = { - 0x09, 0x00, 0x00, 0x09, 0x00, 0x0F, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00 + GIP_CMD_RUMBLE, 0x00, GIP_SEQ0, GIP_PL_LEN(9), + 0x00, GIP_MOTOR_ALL, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; /* @@ -547,13 +639,14 @@ static const u8 xboxone_rumbleend_init[] = { * packet is going to be sent. */ static const struct xboxone_init_packet xboxone_init_packets[] = { - XBOXONE_INIT_PKT(0x0e6f, 0x0165, xboxone_hori_init), - XBOXONE_INIT_PKT(0x0f0d, 0x0067, xboxone_hori_init), - XBOXONE_INIT_PKT(0x0000, 0x0000, xboxone_fw2015_init), + XBOXONE_INIT_PKT(0x0e6f, 0x0165, xboxone_hori_ack_id), + XBOXONE_INIT_PKT(0x0f0d, 0x0067, xboxone_hori_ack_id), + XBOXONE_INIT_PKT(0x0000, 0x0000, xboxone_power_on), XBOXONE_INIT_PKT(0x045e, 0x02ea, xboxone_s_init), XBOXONE_INIT_PKT(0x045e, 0x0b00, xboxone_s_init), - XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_init1), - XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_init2), + XBOXONE_INIT_PKT(0x045e, 0x0b00, extra_input_packet_init), + XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_led_on), + XBOXONE_INIT_PKT(0x0e6f, 0x0000, xboxone_pdp_auth), XBOXONE_INIT_PKT(0x24c6, 0x541a, xboxone_rumblebegin_init), XBOXONE_INIT_PKT(0x24c6, 0x542a, xboxone_rumblebegin_init), XBOXONE_INIT_PKT(0x24c6, 0x543a, xboxone_rumblebegin_init), @@ -608,14 +701,17 @@ struct usb_xpad { int mapping; /* map d-pad to buttons or to axes */ int xtype; /* type of xbox device */ + int packet_type; /* type of the extended packet */ int pad_nr; /* the order x360 pads were attached */ const char *name; /* name of the device */ struct work_struct work; /* init/remove device from callback */ + time64_t mode_btn_down_ts; }; static int xpad_init_input(struct usb_xpad *xpad); static void xpad_deinit_input(struct usb_xpad *xpad); static void xpadone_ack_mode_report(struct usb_xpad *xpad, u8 seq_num); +static void xpad360w_poweroff_controller(struct usb_xpad *xpad); /* * xpad_process_packet @@ -656,10 +752,10 @@ static void xpad_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *d /* digital pad */ if (xpad->mapping & MAP_DPAD_TO_BUTTONS) { /* dpad as buttons (left, right, up, down) */ - input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & 0x04); - input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & 0x08); - input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & 0x01); - input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & 0x02); + input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & BIT(3)); + input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & BIT(1)); } else { input_report_abs(dev, ABS_HAT0X, !!(data[2] & 0x08) - !!(data[2] & 0x04)); @@ -668,10 +764,10 @@ static void xpad_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *d } /* start/back buttons and stick press left/right */ - input_report_key(dev, BTN_START, data[2] & 0x10); - input_report_key(dev, BTN_SELECT, data[2] & 0x20); - input_report_key(dev, BTN_THUMBL, data[2] & 0x40); - input_report_key(dev, BTN_THUMBR, data[2] & 0x80); + input_report_key(dev, BTN_START, data[2] & BIT(4)); + input_report_key(dev, BTN_SELECT, data[2] & BIT(5)); + input_report_key(dev, BTN_THUMBL, data[2] & BIT(6)); + input_report_key(dev, BTN_THUMBR, data[2] & BIT(7)); /* "analog" buttons A, B, X, Y */ input_report_key(dev, BTN_A, data[4]); @@ -683,6 +779,10 @@ static void xpad_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *d input_report_key(dev, BTN_C, data[8]); input_report_key(dev, BTN_Z, data[9]); + /* Profile button has a value of 0-3, so it is reported as an axis */ + if (xpad->mapping & MAP_PROFILE_BUTTON) + input_report_abs(dev, ABS_PROFILE, data[34]); + input_sync(dev); } @@ -706,10 +806,10 @@ static void xpad360_process_packet(struct usb_xpad *xpad, struct input_dev *dev, /* digital pad */ if (xpad->mapping & MAP_DPAD_TO_BUTTONS) { /* dpad as buttons (left, right, up, down) */ - input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & 0x04); - input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & 0x08); - input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & 0x01); - input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & 0x02); + input_report_key(dev, BTN_TRIGGER_HAPPY1, data[2] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY2, data[2] & BIT(3)); + input_report_key(dev, BTN_TRIGGER_HAPPY3, data[2] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY4, data[2] & BIT(1)); } /* @@ -727,21 +827,21 @@ static void xpad360_process_packet(struct usb_xpad *xpad, struct input_dev *dev, } /* start/back buttons */ - input_report_key(dev, BTN_START, data[2] & 0x10); - input_report_key(dev, BTN_SELECT, data[2] & 0x20); + input_report_key(dev, BTN_START, data[2] & BIT(4)); + input_report_key(dev, BTN_SELECT, data[2] & BIT(5)); /* stick press left/right */ - input_report_key(dev, BTN_THUMBL, data[2] & 0x40); - input_report_key(dev, BTN_THUMBR, data[2] & 0x80); + input_report_key(dev, BTN_THUMBL, data[2] & BIT(6)); + input_report_key(dev, BTN_THUMBR, data[2] & BIT(7)); /* buttons A,B,X,Y,TL,TR and MODE */ - input_report_key(dev, BTN_A, data[3] & 0x10); - input_report_key(dev, BTN_B, data[3] & 0x20); - input_report_key(dev, BTN_X, data[3] & 0x40); - input_report_key(dev, BTN_Y, data[3] & 0x80); - input_report_key(dev, BTN_TL, data[3] & 0x01); - input_report_key(dev, BTN_TR, data[3] & 0x02); - input_report_key(dev, BTN_MODE, data[3] & 0x04); + input_report_key(dev, BTN_A, data[3] & BIT(4)); + input_report_key(dev, BTN_B, data[3] & BIT(5)); + input_report_key(dev, BTN_X, data[3] & BIT(6)); + input_report_key(dev, BTN_Y, data[3] & BIT(7)); + input_report_key(dev, BTN_TL, data[3] & BIT(0)); + input_report_key(dev, BTN_TR, data[3] & BIT(1)); + input_report_key(dev, BTN_MODE, data[3] & BIT(2)); if (!(xpad->mapping & MAP_STICKS_TO_NULL)) { /* left stick */ @@ -767,6 +867,23 @@ static void xpad360_process_packet(struct usb_xpad *xpad, struct input_dev *dev, } input_sync(dev); + + /* XBOX360W controllers can't be turned off without driver assistance */ + if (xpad->xtype == XTYPE_XBOX360W) { + if (xpad->mode_btn_down_ts > 0 && xpad->pad_present && + ((ktime_get_seconds() - xpad->mode_btn_down_ts) >= + XPAD360W_POWEROFF_TIMEOUT)) { + xpad360w_poweroff_controller(xpad); + xpad->mode_btn_down_ts = 0; + return; + } + + /* mode button down/up */ + if (data[3] & BIT(2)) + xpad->mode_btn_down_ts = ktime_get_seconds(); + else + xpad->mode_btn_down_ts = 0; + } } static void xpad_presence_work(struct work_struct *work) @@ -846,87 +963,154 @@ static void xpad360w_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned cha static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char *data) { struct input_dev *dev = xpad->dev; + bool do_sync = false; /* the xbox button has its own special report */ - if (data[0] == 0X07) { + if (data[0] == GIP_CMD_VIRTUAL_KEY) { /* * The Xbox One S controller requires these reports to be * acked otherwise it continues sending them forever and * won't report further mode button events. */ - if (data[1] == 0x30) + if (data[1] == (GIP_OPT_ACK | GIP_OPT_INTERNAL)) xpadone_ack_mode_report(xpad, data[2]); - input_report_key(dev, BTN_MODE, data[4] & 0x01); + input_report_key(dev, BTN_MODE, data[4] & GENMASK(1, 0)); input_sync(dev); - return; - } - /* check invalid packet */ - else if (data[0] != 0X20) - return; - /* menu/view buttons */ - input_report_key(dev, BTN_START, data[4] & 0x04); - input_report_key(dev, BTN_SELECT, data[4] & 0x08); - if (xpad->mapping & MAP_SELECT_BUTTON) - input_report_key(dev, KEY_RECORD, data[22] & 0x01); + do_sync = true; + } else if (data[0] == GIP_CMD_FIRMWARE) { + /* Some packet formats force us to use this separate to poll paddle inputs */ + if (xpad->packet_type == PKT_XBE2_FW_5_11) { + /* Mute paddles if controller is in a custom profile slot + * Checked by looking at the active profile slot to + * verify it's the default slot + */ + if (data[19] != 0) + data[18] = 0; - /* buttons A,B,X,Y */ - input_report_key(dev, BTN_A, data[4] & 0x10); - input_report_key(dev, BTN_B, data[4] & 0x20); - input_report_key(dev, BTN_X, data[4] & 0x40); - input_report_key(dev, BTN_Y, data[4] & 0x80); + /* Elite Series 2 split packet paddle bits */ + input_report_key(dev, BTN_TRIGGER_HAPPY5, data[18] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY6, data[18] & BIT(1)); + input_report_key(dev, BTN_TRIGGER_HAPPY7, data[18] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY8, data[18] & BIT(3)); - /* digital pad */ - if (xpad->mapping & MAP_DPAD_TO_BUTTONS) { - /* dpad as buttons (left, right, up, down) */ - input_report_key(dev, BTN_TRIGGER_HAPPY1, data[5] & 0x04); - input_report_key(dev, BTN_TRIGGER_HAPPY2, data[5] & 0x08); - input_report_key(dev, BTN_TRIGGER_HAPPY3, data[5] & 0x01); - input_report_key(dev, BTN_TRIGGER_HAPPY4, data[5] & 0x02); - } else { - input_report_abs(dev, ABS_HAT0X, - !!(data[5] & 0x08) - !!(data[5] & 0x04)); - input_report_abs(dev, ABS_HAT0Y, - !!(data[5] & 0x02) - !!(data[5] & 0x01)); - } - - /* TL/TR */ - input_report_key(dev, BTN_TL, data[5] & 0x10); - input_report_key(dev, BTN_TR, data[5] & 0x20); + do_sync = true; + } + } else if (data[0] == GIP_CMD_INPUT) { /* The main valid packet type for inputs */ + /* menu/view buttons */ + input_report_key(dev, BTN_START, data[4] & BIT(2)); + input_report_key(dev, BTN_SELECT, data[4] & BIT(3)); + if (xpad->mapping & MAP_SELECT_BUTTON) + input_report_key(dev, KEY_RECORD, data[22] & BIT(0)); + + /* buttons A,B,X,Y */ + input_report_key(dev, BTN_A, data[4] & BIT(4)); + input_report_key(dev, BTN_B, data[4] & BIT(5)); + input_report_key(dev, BTN_X, data[4] & BIT(6)); + input_report_key(dev, BTN_Y, data[4] & BIT(7)); + + /* digital pad */ + if (xpad->mapping & MAP_DPAD_TO_BUTTONS) { + /* dpad as buttons (left, right, up, down) */ + input_report_key(dev, BTN_TRIGGER_HAPPY1, data[5] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY2, data[5] & BIT(3)); + input_report_key(dev, BTN_TRIGGER_HAPPY3, data[5] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY4, data[5] & BIT(1)); + } else { + input_report_abs(dev, ABS_HAT0X, + !!(data[5] & 0x08) - !!(data[5] & 0x04)); + input_report_abs(dev, ABS_HAT0Y, + !!(data[5] & 0x02) - !!(data[5] & 0x01)); + } - /* stick press left/right */ - input_report_key(dev, BTN_THUMBL, data[5] & 0x40); - input_report_key(dev, BTN_THUMBR, data[5] & 0x80); + /* TL/TR */ + input_report_key(dev, BTN_TL, data[5] & BIT(4)); + input_report_key(dev, BTN_TR, data[5] & BIT(5)); + + /* stick press left/right */ + input_report_key(dev, BTN_THUMBL, data[5] & BIT(6)); + input_report_key(dev, BTN_THUMBR, data[5] & BIT(7)); + + if (!(xpad->mapping & MAP_STICKS_TO_NULL)) { + /* left stick */ + input_report_abs(dev, ABS_X, + (__s16) le16_to_cpup((__le16 *)(data + 10))); + input_report_abs(dev, ABS_Y, + ~(__s16) le16_to_cpup((__le16 *)(data + 12))); + + /* right stick */ + input_report_abs(dev, ABS_RX, + (__s16) le16_to_cpup((__le16 *)(data + 14))); + input_report_abs(dev, ABS_RY, + ~(__s16) le16_to_cpup((__le16 *)(data + 16))); + } - if (!(xpad->mapping & MAP_STICKS_TO_NULL)) { - /* left stick */ - input_report_abs(dev, ABS_X, - (__s16) le16_to_cpup((__le16 *)(data + 10))); - input_report_abs(dev, ABS_Y, - ~(__s16) le16_to_cpup((__le16 *)(data + 12))); + /* triggers left/right */ + if (xpad->mapping & MAP_TRIGGERS_TO_BUTTONS) { + input_report_key(dev, BTN_TL2, + (__u16) le16_to_cpup((__le16 *)(data + 6))); + input_report_key(dev, BTN_TR2, + (__u16) le16_to_cpup((__le16 *)(data + 8))); + } else { + input_report_abs(dev, ABS_Z, + (__u16) le16_to_cpup((__le16 *)(data + 6))); + input_report_abs(dev, ABS_RZ, + (__u16) le16_to_cpup((__le16 *)(data + 8))); + } - /* right stick */ - input_report_abs(dev, ABS_RX, - (__s16) le16_to_cpup((__le16 *)(data + 14))); - input_report_abs(dev, ABS_RY, - ~(__s16) le16_to_cpup((__le16 *)(data + 16))); - } + /* paddle handling */ + /* based on SDL's SDL_hidapi_xboxone.c */ + if (xpad->mapping & MAP_PADDLES) { + if (xpad->packet_type == PKT_XBE1) { + /* Mute paddles if controller has a custom mapping applied. + * Checked by comparing the current mapping + * config against the factory mapping config + */ + if (memcmp(&data[4], &data[18], 2) != 0) + data[32] = 0; + + /* OG Elite Series Controller paddle bits */ + input_report_key(dev, BTN_TRIGGER_HAPPY5, data[32] & BIT(1)); + input_report_key(dev, BTN_TRIGGER_HAPPY6, data[32] & BIT(3)); + input_report_key(dev, BTN_TRIGGER_HAPPY7, data[32] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY8, data[32] & BIT(2)); + } else if (xpad->packet_type == PKT_XBE2_FW_OLD) { + /* Mute paddles if controller has a custom mapping applied. + * Checked by comparing the current mapping + * config against the factory mapping config + */ + if (data[19] != 0) + data[18] = 0; + + /* Elite Series 2 4.x firmware paddle bits */ + input_report_key(dev, BTN_TRIGGER_HAPPY5, data[18] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY6, data[18] & BIT(1)); + input_report_key(dev, BTN_TRIGGER_HAPPY7, data[18] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY8, data[18] & BIT(3)); + } else if (xpad->packet_type == PKT_XBE2_FW_5_EARLY) { + /* Mute paddles if controller has a custom mapping applied. + * Checked by comparing the current mapping + * config against the factory mapping config + */ + if (data[23] != 0) + data[22] = 0; + + /* Elite Series 2 5.x firmware paddle bits + * (before the packet was split) + */ + input_report_key(dev, BTN_TRIGGER_HAPPY5, data[22] & BIT(0)); + input_report_key(dev, BTN_TRIGGER_HAPPY6, data[22] & BIT(1)); + input_report_key(dev, BTN_TRIGGER_HAPPY7, data[22] & BIT(2)); + input_report_key(dev, BTN_TRIGGER_HAPPY8, data[22] & BIT(3)); + } + } - /* triggers left/right */ - if (xpad->mapping & MAP_TRIGGERS_TO_BUTTONS) { - input_report_key(dev, BTN_TL2, - (__u16) le16_to_cpup((__le16 *)(data + 6))); - input_report_key(dev, BTN_TR2, - (__u16) le16_to_cpup((__le16 *)(data + 8))); - } else { - input_report_abs(dev, ABS_Z, - (__u16) le16_to_cpup((__le16 *)(data + 6))); - input_report_abs(dev, ABS_RZ, - (__u16) le16_to_cpup((__le16 *)(data + 8))); + do_sync = true; } - input_sync(dev); + if (do_sync) + input_sync(dev); } static void xpad_irq_in(struct urb *urb) @@ -1226,8 +1410,8 @@ static void xpadone_ack_mode_report(struct usb_xpad *xpad, u8 seq_num) struct xpad_output_packet *packet = &xpad->out_packets[XPAD_OUT_CMD_IDX]; static const u8 mode_report_ack[] = { - 0x01, 0x20, 0x00, 0x09, 0x00, 0x07, 0x20, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00 + GIP_CMD_ACK, GIP_OPT_INTERNAL, GIP_SEQ0, GIP_PL_LEN(9), + 0x00, GIP_CMD_VIRTUAL_KEY, GIP_OPT_INTERNAL, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00 }; spin_lock_irqsave(&xpad->odata_lock, flags); @@ -1305,14 +1489,14 @@ static int xpad_play_effect(struct input_dev *dev, void *data, struct ff_effect break; case XTYPE_XBOXONE: - packet->data[0] = 0x09; /* activate rumble */ + packet->data[0] = GIP_CMD_RUMBLE; /* activate rumble */ packet->data[1] = 0x00; packet->data[2] = xpad->odata_serial++; - packet->data[3] = 0x09; + packet->data[3] = GIP_PL_LEN(9); packet->data[4] = 0x00; - packet->data[5] = 0x0F; - packet->data[6] = 0x00; - packet->data[7] = 0x00; + packet->data[5] = GIP_MOTOR_ALL; + packet->data[6] = 0x00; /* left trigger */ + packet->data[7] = 0x00; /* right trigger */ packet->data[8] = strong / 512; /* left actuator */ packet->data[9] = weak / 512; /* right actuator */ packet->data[10] = 0xFF; /* on period */ @@ -1622,6 +1806,9 @@ static void xpad_set_up_abs(struct input_dev *input_dev, signed short abs) case ABS_HAT0Y: /* the d-pad (only if dpad is mapped to axes */ input_set_abs_params(input_dev, abs, -1, 1, 0, 0); break; + case ABS_PROFILE: /* 4 value profile button (such as on XAC) */ + input_set_abs_params(input_dev, abs, 0, 4, 0, 0); + break; default: input_set_abs_params(input_dev, abs, 0, 0, 0, 0); break; @@ -1693,6 +1880,12 @@ static int xpad_init_input(struct usb_xpad *xpad) xpad_btn_pad[i]); } + /* set up paddles if the controller has them */ + if (xpad->mapping & MAP_PADDLES) { + for (i = 0; xpad_btn_paddles[i] >= 0; i++) + input_set_capability(input_dev, EV_KEY, xpad_btn_paddles[i]); + } + /* * This should be a simple else block. However historically * xbox360w has mapped DPAD to buttons while xbox360 did not. This @@ -1714,6 +1907,10 @@ static int xpad_init_input(struct usb_xpad *xpad) xpad_set_up_abs(input_dev, xpad_abs_triggers[i]); } + /* setup profile button as an axis with 4 possible values */ + if (xpad->mapping & MAP_PROFILE_BUTTON) + xpad_set_up_abs(input_dev, ABS_PROFILE); + error = xpad_init_ff(xpad); if (error) goto err_free_input; @@ -1779,6 +1976,7 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id xpad->mapping = xpad_device[i].mapping; xpad->xtype = xpad_device[i].xtype; xpad->name = xpad_device[i].name; + xpad->packet_type = PKT_XB; INIT_WORK(&xpad->work, xpad_presence_work); if (xpad->xtype == XTYPE_UNKNOWN) { @@ -1844,6 +2042,38 @@ static int xpad_probe(struct usb_interface *intf, const struct usb_device_id *id usb_set_intfdata(intf, xpad); + /* Packet type detection */ + if (le16_to_cpu(udev->descriptor.idVendor) == 0x045e) { /* Microsoft controllers */ + if (le16_to_cpu(udev->descriptor.idProduct) == 0x02e3) { + /* The original elite controller always uses the oldest + * type of extended packet + */ + xpad->packet_type = PKT_XBE1; + } else if (le16_to_cpu(udev->descriptor.idProduct) == 0x0b00) { + /* The elite 2 controller has seen multiple packet + * revisions. These are tied to specific firmware + * versions + */ + if (le16_to_cpu(udev->descriptor.bcdDevice) < 0x0500) { + /* This is the format that the Elite 2 used + * prior to the BLE update + */ + xpad->packet_type = PKT_XBE2_FW_OLD; + } else if (le16_to_cpu(udev->descriptor.bcdDevice) < + 0x050b) { + /* This is the format that the Elite 2 used + * prior to the update that split the packet + */ + xpad->packet_type = PKT_XBE2_FW_5_EARLY; + } else { + /* The split packet format that was introduced + * in firmware v5.11 + */ + xpad->packet_type = PKT_XBE2_FW_5_11; + } + } + } + if (xpad->xtype == XTYPE_XBOX360W) { /* * Submit the int URB immediately rather than waiting for open @@ -1972,7 +2202,6 @@ static struct usb_driver xpad_driver = { .disconnect = xpad_disconnect, .suspend = xpad_suspend, .resume = xpad_resume, - .reset_resume = xpad_resume, .id_table = xpad_table, }; diff --git a/drivers/input/joystick/zhenhua.c b/drivers/input/joystick/zhenhua.c index d5531179b01f2fec8b0eecc6b06daf4d8d4846c5..3f2460e2b095597144f42a2890800a879dbceac7 100644 --- a/drivers/input/joystick/zhenhua.c +++ b/drivers/input/joystick/zhenhua.c @@ -28,9 +28,6 @@ * coder :-( */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig index a20ee693b22b5ee7f9dce0fd8e1b9f4348920fa9..00292118b79bc76ae9dd82fbf3a9f0bd6eea602d 100644 --- a/drivers/input/keyboard/Kconfig +++ b/drivers/input/keyboard/Kconfig @@ -40,6 +40,9 @@ config KEYBOARD_ADP5520 config KEYBOARD_ADP5588 tristate "ADP5588/87 I2C QWERTY Keypad and IO Expander" depends on I2C + select GPIOLIB + select GPIOLIB_IRQCHIP + select INPUT_MATRIXKMAP help Say Y here if you want to use a ADP5588/87 attached to your system I2C bus. @@ -186,7 +189,7 @@ config KEYBOARD_QT2160 config KEYBOARD_CLPS711X tristate "CLPS711X Keypad support" - depends on OF_GPIO && (ARCH_CLPS711X || COMPILE_TEST) + depends on ARCH_CLPS711X || COMPILE_TEST select INPUT_MATRIXKMAP help Say Y here to enable the matrix keypad on the Cirrus Logic @@ -524,6 +527,19 @@ config KEYBOARD_OPENCORES To compile this driver as a module, choose M here; the module will be called opencores-kbd. +config KEYBOARD_PINEPHONE + tristate "Pine64 PinePhone Keyboard" + depends on I2C && REGULATOR + select CRC8 + select INPUT_MATRIXKMAP + help + Say Y here to enable support for the keyboard in the Pine64 PinePhone + keyboard case. This driver supports the FLOSS firmware available at + https://megous.com/git/pinephone-keyboard/ + + To compile this driver as a module, choose M here; the + module will be called pinephone-keyboard. + config KEYBOARD_PXA27x tristate "PXA27x/PXA3xx keypad support" depends on PXA27x || PXA3xx || ARCH_MMP diff --git a/drivers/input/keyboard/Makefile b/drivers/input/keyboard/Makefile index 721936e9029002c5fe341b55c6647bd1f4e4efad..5f67196bb2c1ea5a05b752eaa16cb7e1fbb3897c 100644 --- a/drivers/input/keyboard/Makefile +++ b/drivers/input/keyboard/Makefile @@ -52,6 +52,7 @@ obj-$(CONFIG_KEYBOARD_NSPIRE) += nspire-keypad.o obj-$(CONFIG_KEYBOARD_OMAP) += omap-keypad.o obj-$(CONFIG_KEYBOARD_OMAP4) += omap4-keypad.o obj-$(CONFIG_KEYBOARD_OPENCORES) += opencores-kbd.o +obj-$(CONFIG_KEYBOARD_PINEPHONE) += pinephone-keyboard.o obj-$(CONFIG_KEYBOARD_PMIC8XXX) += pmic8xxx-keypad.o obj-$(CONFIG_KEYBOARD_PXA27x) += pxa27x_keypad.o obj-$(CONFIG_KEYBOARD_PXA930_ROTARY) += pxa930_rotary.o diff --git a/drivers/input/keyboard/adp5588-keys.c b/drivers/input/keyboard/adp5588-keys.c index 1a1a05d7cd4205deeb857bbfc6f5c85e6df0bc5b..7cd83c8e7110814a43fbb7a41b443db6416bd2b7 100644 --- a/drivers/input/keyboard/adp5588-keys.c +++ b/drivers/input/keyboard/adp5588-keys.c @@ -8,27 +8,163 @@ * Copyright (C) 2008-2010 Analog Devices Inc. */ +#include #include #include +#include #include #include #include +#include #include #include #include #include +#include +#include #include #include +#include #include #include -#include +#define DEV_ID 0x00 /* Device ID */ +#define CFG 0x01 /* Configuration Register1 */ +#define INT_STAT 0x02 /* Interrupt Status Register */ +#define KEY_LCK_EC_STAT 0x03 /* Key Lock and Event Counter Register */ +#define KEY_EVENTA 0x04 /* Key Event Register A */ +#define KEY_EVENTB 0x05 /* Key Event Register B */ +#define KEY_EVENTC 0x06 /* Key Event Register C */ +#define KEY_EVENTD 0x07 /* Key Event Register D */ +#define KEY_EVENTE 0x08 /* Key Event Register E */ +#define KEY_EVENTF 0x09 /* Key Event Register F */ +#define KEY_EVENTG 0x0A /* Key Event Register G */ +#define KEY_EVENTH 0x0B /* Key Event Register H */ +#define KEY_EVENTI 0x0C /* Key Event Register I */ +#define KEY_EVENTJ 0x0D /* Key Event Register J */ +#define KP_LCK_TMR 0x0E /* Keypad Lock1 to Lock2 Timer */ +#define UNLOCK1 0x0F /* Unlock Key1 */ +#define UNLOCK2 0x10 /* Unlock Key2 */ +#define GPIO_INT_STAT1 0x11 /* GPIO Interrupt Status */ +#define GPIO_INT_STAT2 0x12 /* GPIO Interrupt Status */ +#define GPIO_INT_STAT3 0x13 /* GPIO Interrupt Status */ +#define GPIO_DAT_STAT1 0x14 /* GPIO Data Status, Read twice to clear */ +#define GPIO_DAT_STAT2 0x15 /* GPIO Data Status, Read twice to clear */ +#define GPIO_DAT_STAT3 0x16 /* GPIO Data Status, Read twice to clear */ +#define GPIO_DAT_OUT1 0x17 /* GPIO DATA OUT */ +#define GPIO_DAT_OUT2 0x18 /* GPIO DATA OUT */ +#define GPIO_DAT_OUT3 0x19 /* GPIO DATA OUT */ +#define GPIO_INT_EN1 0x1A /* GPIO Interrupt Enable */ +#define GPIO_INT_EN2 0x1B /* GPIO Interrupt Enable */ +#define GPIO_INT_EN3 0x1C /* GPIO Interrupt Enable */ +#define KP_GPIO1 0x1D /* Keypad or GPIO Selection */ +#define KP_GPIO2 0x1E /* Keypad or GPIO Selection */ +#define KP_GPIO3 0x1F /* Keypad or GPIO Selection */ +#define GPI_EM1 0x20 /* GPI Event Mode 1 */ +#define GPI_EM2 0x21 /* GPI Event Mode 2 */ +#define GPI_EM3 0x22 /* GPI Event Mode 3 */ +#define GPIO_DIR1 0x23 /* GPIO Data Direction */ +#define GPIO_DIR2 0x24 /* GPIO Data Direction */ +#define GPIO_DIR3 0x25 /* GPIO Data Direction */ +#define GPIO_INT_LVL1 0x26 /* GPIO Edge/Level Detect */ +#define GPIO_INT_LVL2 0x27 /* GPIO Edge/Level Detect */ +#define GPIO_INT_LVL3 0x28 /* GPIO Edge/Level Detect */ +#define DEBOUNCE_DIS1 0x29 /* Debounce Disable */ +#define DEBOUNCE_DIS2 0x2A /* Debounce Disable */ +#define DEBOUNCE_DIS3 0x2B /* Debounce Disable */ +#define GPIO_PULL1 0x2C /* GPIO Pull Disable */ +#define GPIO_PULL2 0x2D /* GPIO Pull Disable */ +#define GPIO_PULL3 0x2E /* GPIO Pull Disable */ +#define CMP_CFG_STAT 0x30 /* Comparator Configuration and Status Register */ +#define CMP_CONFG_SENS1 0x31 /* Sensor1 Comparator Configuration Register */ +#define CMP_CONFG_SENS2 0x32 /* L2 Light Sensor Reference Level, Output Falling for Sensor 1 */ +#define CMP1_LVL2_TRIP 0x33 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 1 */ +#define CMP1_LVL2_HYS 0x34 /* L3 Light Sensor Reference Level, Output Falling For Sensor 1 */ +#define CMP1_LVL3_TRIP 0x35 /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 1 */ +#define CMP1_LVL3_HYS 0x36 /* Sensor 2 Comparator Configuration Register */ +#define CMP2_LVL2_TRIP 0x37 /* L2 Light Sensor Reference Level, Output Falling for Sensor 2 */ +#define CMP2_LVL2_HYS 0x38 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 2 */ +#define CMP2_LVL3_TRIP 0x39 /* L3 Light Sensor Reference Level, Output Falling For Sensor 2 */ +#define CMP2_LVL3_HYS 0x3A /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 2 */ +#define CMP1_ADC_DAT_R1 0x3B /* Comparator 1 ADC data Register1 */ +#define CMP1_ADC_DAT_R2 0x3C /* Comparator 1 ADC data Register2 */ +#define CMP2_ADC_DAT_R1 0x3D /* Comparator 2 ADC data Register1 */ +#define CMP2_ADC_DAT_R2 0x3E /* Comparator 2 ADC data Register2 */ + +#define ADP5588_DEVICE_ID_MASK 0xF + + /* Configuration Register1 */ +#define ADP5588_AUTO_INC BIT(7) +#define ADP5588_GPIEM_CFG BIT(6) +#define ADP5588_OVR_FLOW_M BIT(5) +#define ADP5588_INT_CFG BIT(4) +#define ADP5588_OVR_FLOW_IEN BIT(3) +#define ADP5588_K_LCK_IM BIT(2) +#define ADP5588_GPI_IEN BIT(1) +#define ADP5588_KE_IEN BIT(0) + +/* Interrupt Status Register */ +#define ADP5588_CMP2_INT BIT(5) +#define ADP5588_CMP1_INT BIT(4) +#define ADP5588_OVR_FLOW_INT BIT(3) +#define ADP5588_K_LCK_INT BIT(2) +#define ADP5588_GPI_INT BIT(1) +#define ADP5588_KE_INT BIT(0) + +/* Key Lock and Event Counter Register */ +#define ADP5588_K_LCK_EN BIT(6) +#define ADP5588_LCK21 0x30 +#define ADP5588_KEC GENMASK(3, 0) + +#define ADP5588_MAXGPIO 18 +#define ADP5588_BANK(offs) ((offs) >> 3) +#define ADP5588_BIT(offs) (1u << ((offs) & 0x7)) + +/* Put one of these structures in i2c_board_info platform_data */ + +/* + * 128 so it fits matrix-keymap maximum number of keys when the full + * 10cols * 8rows are used. + */ +#define ADP5588_KEYMAPSIZE 128 + +#define GPI_PIN_ROW0 97 +#define GPI_PIN_ROW1 98 +#define GPI_PIN_ROW2 99 +#define GPI_PIN_ROW3 100 +#define GPI_PIN_ROW4 101 +#define GPI_PIN_ROW5 102 +#define GPI_PIN_ROW6 103 +#define GPI_PIN_ROW7 104 +#define GPI_PIN_COL0 105 +#define GPI_PIN_COL1 106 +#define GPI_PIN_COL2 107 +#define GPI_PIN_COL3 108 +#define GPI_PIN_COL4 109 +#define GPI_PIN_COL5 110 +#define GPI_PIN_COL6 111 +#define GPI_PIN_COL7 112 +#define GPI_PIN_COL8 113 +#define GPI_PIN_COL9 114 + +#define GPI_PIN_ROW_BASE GPI_PIN_ROW0 +#define GPI_PIN_ROW_END GPI_PIN_ROW7 +#define GPI_PIN_COL_BASE GPI_PIN_COL0 +#define GPI_PIN_COL_END GPI_PIN_COL9 + +#define GPI_PIN_BASE GPI_PIN_ROW_BASE +#define GPI_PIN_END GPI_PIN_COL_END + +#define ADP5588_ROWS_MAX (GPI_PIN_ROW7 - GPI_PIN_ROW0 + 1) +#define ADP5588_COLS_MAX (GPI_PIN_COL9 - GPI_PIN_COL0 + 1) + +#define ADP5588_GPIMAPSIZE_MAX (GPI_PIN_END - GPI_PIN_BASE + 1) /* Key Event Register xy */ -#define KEY_EV_PRESSED (1 << 7) -#define KEY_EV_MASK (0x7F) +#define KEY_EV_PRESSED BIT(7) +#define KEY_EV_MASK GENMASK(6, 0) -#define KP_SEL(x) (0xFFFF >> (16 - x)) /* 2^x-1 */ +#define KP_SEL(x) (BIT(x) - 1) /* 2^x-1 */ #define KEYP_MAX_EVENT 10 @@ -40,21 +176,27 @@ #define WA_DELAYED_READOUT_REVID(rev) ((rev) < 4) #define WA_DELAYED_READOUT_TIME 25 +#define ADP5588_INVALID_HWIRQ (~0UL) + struct adp5588_kpad { struct i2c_client *client; struct input_dev *input; ktime_t irq_time; unsigned long delay; + u32 row_shift; + u32 rows; + u32 cols; + u32 unlock_keys[2]; + int nkeys_unlock; unsigned short keycode[ADP5588_KEYMAPSIZE]; - const struct adp5588_gpi_map *gpimap; - unsigned short gpimapsize; -#ifdef CONFIG_GPIOLIB unsigned char gpiomap[ADP5588_MAXGPIO]; struct gpio_chip gc; struct mutex gpio_lock; /* Protect cached dir, dat_out */ u8 dat_out[3]; u8 dir[3]; -#endif + u8 int_en[3]; + u8 irq_mask[3]; + u8 pull_dis[3]; }; static int adp5588_read(struct i2c_client *client, u8 reg) @@ -72,8 +214,7 @@ static int adp5588_write(struct i2c_client *client, u8 reg, u8 val) return i2c_smbus_write_byte_data(client, reg, val); } -#ifdef CONFIG_GPIOLIB -static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off) +static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned int off) { struct adp5588_kpad *kpad = gpiochip_get_data(chip); unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]); @@ -93,7 +234,7 @@ static int adp5588_gpio_get_value(struct gpio_chip *chip, unsigned off) } static void adp5588_gpio_set_value(struct gpio_chip *chip, - unsigned off, int val) + unsigned int off, int val) { struct adp5588_kpad *kpad = gpiochip_get_data(chip); unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]); @@ -106,13 +247,47 @@ static void adp5588_gpio_set_value(struct gpio_chip *chip, else kpad->dat_out[bank] &= ~bit; - adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank, - kpad->dat_out[bank]); + adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank, kpad->dat_out[bank]); mutex_unlock(&kpad->gpio_lock); } -static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off) +static int adp5588_gpio_set_config(struct gpio_chip *chip, unsigned int off, + unsigned long config) +{ + struct adp5588_kpad *kpad = gpiochip_get_data(chip); + unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]); + unsigned int bit = ADP5588_BIT(kpad->gpiomap[off]); + bool pull_disable; + int ret; + + switch (pinconf_to_config_param(config)) { + case PIN_CONFIG_BIAS_PULL_UP: + pull_disable = false; + break; + case PIN_CONFIG_BIAS_DISABLE: + pull_disable = true; + break; + default: + return -ENOTSUPP; + } + + mutex_lock(&kpad->gpio_lock); + + if (pull_disable) + kpad->pull_dis[bank] |= bit; + else + kpad->pull_dis[bank] &= bit; + + ret = adp5588_write(kpad->client, GPIO_PULL1 + bank, + kpad->pull_dis[bank]); + + mutex_unlock(&kpad->gpio_lock); + + return ret; +} + +static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned int off) { struct adp5588_kpad *kpad = gpiochip_get_data(chip); unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]); @@ -130,7 +305,7 @@ static int adp5588_gpio_direction_input(struct gpio_chip *chip, unsigned off) } static int adp5588_gpio_direction_output(struct gpio_chip *chip, - unsigned off, int val) + unsigned int off, int val) { struct adp5588_kpad *kpad = gpiochip_get_data(chip); unsigned int bank = ADP5588_BANK(kpad->gpiomap[off]); @@ -147,17 +322,19 @@ static int adp5588_gpio_direction_output(struct gpio_chip *chip, kpad->dat_out[bank] &= ~bit; ret = adp5588_write(kpad->client, GPIO_DAT_OUT1 + bank, - kpad->dat_out[bank]); - ret |= adp5588_write(kpad->client, GPIO_DIR1 + bank, - kpad->dir[bank]); + kpad->dat_out[bank]); + if (ret) + goto out_unlock; + + ret = adp5588_write(kpad->client, GPIO_DIR1 + bank, kpad->dir[bank]); +out_unlock: mutex_unlock(&kpad->gpio_lock); return ret; } -static int adp5588_build_gpiomap(struct adp5588_kpad *kpad, - const struct adp5588_kpad_platform_data *pdata) +static int adp5588_build_gpiomap(struct adp5588_kpad *kpad) { bool pin_used[ADP5588_MAXGPIO]; int n_unused = 0; @@ -165,15 +342,12 @@ static int adp5588_build_gpiomap(struct adp5588_kpad *kpad, memset(pin_used, 0, sizeof(pin_used)); - for (i = 0; i < pdata->rows; i++) + for (i = 0; i < kpad->rows; i++) pin_used[i] = true; - for (i = 0; i < pdata->cols; i++) + for (i = 0; i < kpad->cols; i++) pin_used[i + GPI_PIN_COL_BASE - GPI_PIN_BASE] = true; - for (i = 0; i < kpad->gpimapsize; i++) - pin_used[kpad->gpimap[i].pin - GPI_PIN_BASE] = true; - for (i = 0; i < ADP5588_MAXGPIO; i++) if (!pin_used[i]) kpad->gpiomap[n_unused++] = i; @@ -181,47 +355,101 @@ static int adp5588_build_gpiomap(struct adp5588_kpad *kpad, return n_unused; } -static void adp5588_gpio_do_teardown(void *_kpad) +static void adp5588_irq_bus_lock(struct irq_data *d) { - struct adp5588_kpad *kpad = _kpad; - struct device *dev = &kpad->client->dev; - const struct adp5588_kpad_platform_data *pdata = dev_get_platdata(dev); - const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data; - int error; + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct adp5588_kpad *kpad = gpiochip_get_data(gc); - error = gpio_data->teardown(kpad->client, - kpad->gc.base, kpad->gc.ngpio, - gpio_data->context); - if (error) - dev_warn(&kpad->client->dev, "teardown failed %d\n", error); + mutex_lock(&kpad->gpio_lock); } +static void adp5588_irq_bus_sync_unlock(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct adp5588_kpad *kpad = gpiochip_get_data(gc); + int i; + + for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) { + if (kpad->int_en[i] ^ kpad->irq_mask[i]) { + kpad->int_en[i] = kpad->irq_mask[i]; + adp5588_write(kpad->client, GPI_EM1 + i, kpad->int_en[i]); + } + } + + mutex_unlock(&kpad->gpio_lock); +} + +static void adp5588_irq_mask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct adp5588_kpad *kpad = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long real_irq = kpad->gpiomap[hwirq]; + + kpad->irq_mask[ADP5588_BANK(real_irq)] &= ~ADP5588_BIT(real_irq); + gpiochip_disable_irq(gc, hwirq); +} + +static void adp5588_irq_unmask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct adp5588_kpad *kpad = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long real_irq = kpad->gpiomap[hwirq]; + + gpiochip_enable_irq(gc, hwirq); + kpad->irq_mask[ADP5588_BANK(real_irq)] |= ADP5588_BIT(real_irq); +} + +static int adp5588_irq_set_type(struct irq_data *d, unsigned int type) +{ + if (!(type & IRQ_TYPE_EDGE_BOTH)) + return -EINVAL; + + irq_set_handler_locked(d, handle_edge_irq); + + return 0; +} + +static const struct irq_chip adp5588_irq_chip = { + .name = "adp5588", + .irq_mask = adp5588_irq_mask, + .irq_unmask = adp5588_irq_unmask, + .irq_bus_lock = adp5588_irq_bus_lock, + .irq_bus_sync_unlock = adp5588_irq_bus_sync_unlock, + .irq_set_type = adp5588_irq_set_type, + .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int adp5588_gpio_add(struct adp5588_kpad *kpad) { struct device *dev = &kpad->client->dev; - const struct adp5588_kpad_platform_data *pdata = dev_get_platdata(dev); - const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data; + struct gpio_irq_chip *girq; int i, error; - if (!gpio_data) - return 0; - - kpad->gc.ngpio = adp5588_build_gpiomap(kpad, pdata); + kpad->gc.ngpio = adp5588_build_gpiomap(kpad); if (kpad->gc.ngpio == 0) { dev_info(dev, "No unused gpios left to export\n"); return 0; } + kpad->gc.parent = &kpad->client->dev; kpad->gc.direction_input = adp5588_gpio_direction_input; kpad->gc.direction_output = adp5588_gpio_direction_output; kpad->gc.get = adp5588_gpio_get_value; kpad->gc.set = adp5588_gpio_set_value; + kpad->gc.set_config = adp5588_gpio_set_config; kpad->gc.can_sleep = 1; - kpad->gc.base = gpio_data->gpio_start; + kpad->gc.base = -1; kpad->gc.label = kpad->client->name; kpad->gc.owner = THIS_MODULE; - kpad->gc.names = gpio_data->names; + + girq = &kpad->gc.irq; + gpio_irq_chip_set_chip(girq, &adp5588_irq_chip); + girq->handler = handle_bad_irq; + girq->threaded = true; mutex_init(&kpad->gpio_lock); @@ -235,54 +463,87 @@ static int adp5588_gpio_add(struct adp5588_kpad *kpad) kpad->dat_out[i] = adp5588_read(kpad->client, GPIO_DAT_OUT1 + i); kpad->dir[i] = adp5588_read(kpad->client, GPIO_DIR1 + i); + kpad->pull_dis[i] = adp5588_read(kpad->client, GPIO_PULL1 + i); } - if (gpio_data->setup) { - error = gpio_data->setup(kpad->client, - kpad->gc.base, kpad->gc.ngpio, - gpio_data->context); - if (error) - dev_warn(dev, "setup failed: %d\n", error); - } + return 0; +} - if (gpio_data->teardown) { - error = devm_add_action(dev, adp5588_gpio_do_teardown, kpad); - if (error) - dev_warn(dev, "failed to schedule teardown: %d\n", - error); - } +static unsigned long adp5588_gpiomap_get_hwirq(struct device *dev, + const u8 *map, unsigned int gpio, + unsigned int ngpios) +{ + unsigned int hwirq; - return 0; + for (hwirq = 0; hwirq < ngpios; hwirq++) + if (map[hwirq] == gpio) + return hwirq; + + /* should never happen */ + dev_warn_ratelimited(dev, "could not find the hwirq for gpio(%u)\n", gpio); + + return ADP5588_INVALID_HWIRQ; } -#else -static inline int adp5588_gpio_add(struct adp5588_kpad *kpad) +static void adp5588_gpio_irq_handle(struct adp5588_kpad *kpad, int key_val, + int key_press) { - return 0; + unsigned int irq, gpio = key_val - GPI_PIN_BASE, irq_type; + struct i2c_client *client = kpad->client; + struct irq_data *irqd; + unsigned long hwirq; + + hwirq = adp5588_gpiomap_get_hwirq(&client->dev, kpad->gpiomap, + gpio, kpad->gc.ngpio); + if (hwirq == ADP5588_INVALID_HWIRQ) { + dev_err(&client->dev, "Could not get hwirq for key(%u)\n", key_val); + return; + } + + irq = irq_find_mapping(kpad->gc.irq.domain, hwirq); + if (!irq) + return; + + irqd = irq_get_irq_data(irq); + if (!irqd) { + dev_err(&client->dev, "Could not get irq(%u) data\n", irq); + return; + } + + irq_type = irqd_get_trigger_type(irqd); + + /* + * Default is active low which means key_press is asserted on + * the falling edge. + */ + if ((irq_type & IRQ_TYPE_EDGE_RISING && !key_press) || + (irq_type & IRQ_TYPE_EDGE_FALLING && key_press)) + handle_nested_irq(irq); } -#endif static void adp5588_report_events(struct adp5588_kpad *kpad, int ev_cnt) { - int i, j; + int i; for (i = 0; i < ev_cnt; i++) { - int key = adp5588_read(kpad->client, Key_EVENTA + i); + int key = adp5588_read(kpad->client, KEY_EVENTA + i); int key_val = key & KEY_EV_MASK; + int key_press = key & KEY_EV_PRESSED; if (key_val >= GPI_PIN_BASE && key_val <= GPI_PIN_END) { - for (j = 0; j < kpad->gpimapsize; j++) { - if (key_val == kpad->gpimap[j].pin) { - input_report_switch(kpad->input, - kpad->gpimap[j].sw_evt, - key & KEY_EV_PRESSED); - break; - } - } + /* gpio line used as IRQ source */ + adp5588_gpio_irq_handle(kpad, key_val, key_press); } else { + int row = (key_val - 1) / ADP5588_COLS_MAX; + int col = (key_val - 1) % ADP5588_COLS_MAX; + int code = MATRIX_SCAN_CODE(row, col, kpad->row_shift); + + dev_dbg_ratelimited(&kpad->client->dev, + "report key(%d) r(%d) c(%d) code(%d)\n", + key_val, row, col, kpad->keycode[code]); + input_report_key(kpad->input, - kpad->keycode[key_val - 1], - key & KEY_EV_PRESSED); + kpad->keycode[code], key_press); } } } @@ -335,176 +596,145 @@ static irqreturn_t adp5588_thread_irq(int irq, void *handle) return IRQ_HANDLED; } -static int adp5588_setup(struct i2c_client *client) +static int adp5588_setup(struct adp5588_kpad *kpad) { - const struct adp5588_kpad_platform_data *pdata = - dev_get_platdata(&client->dev); - const struct adp5588_gpio_platform_data *gpio_data = pdata->gpio_data; + struct i2c_client *client = kpad->client; int i, ret; - unsigned char evt_mode1 = 0, evt_mode2 = 0, evt_mode3 = 0; - - ret = adp5588_write(client, KP_GPIO1, KP_SEL(pdata->rows)); - ret |= adp5588_write(client, KP_GPIO2, KP_SEL(pdata->cols) & 0xFF); - ret |= adp5588_write(client, KP_GPIO3, KP_SEL(pdata->cols) >> 8); - if (pdata->en_keylock) { - ret |= adp5588_write(client, UNLOCK1, pdata->unlock_key1); - ret |= adp5588_write(client, UNLOCK2, pdata->unlock_key2); - ret |= adp5588_write(client, KEY_LCK_EC_STAT, ADP5588_K_LCK_EN); - } + ret = adp5588_write(client, KP_GPIO1, KP_SEL(kpad->rows)); + if (ret) + return ret; - for (i = 0; i < KEYP_MAX_EVENT; i++) - ret |= adp5588_read(client, Key_EVENTA); + ret = adp5588_write(client, KP_GPIO2, KP_SEL(kpad->cols) & 0xFF); + if (ret) + return ret; - for (i = 0; i < pdata->gpimapsize; i++) { - unsigned short pin = pdata->gpimap[i].pin; + ret = adp5588_write(client, KP_GPIO3, KP_SEL(kpad->cols) >> 8); + if (ret) + return ret; - if (pin <= GPI_PIN_ROW_END) { - evt_mode1 |= (1 << (pin - GPI_PIN_ROW_BASE)); - } else { - evt_mode2 |= ((1 << (pin - GPI_PIN_COL_BASE)) & 0xFF); - evt_mode3 |= ((1 << (pin - GPI_PIN_COL_BASE)) >> 8); - } + for (i = 0; i < kpad->nkeys_unlock; i++) { + ret = adp5588_write(client, UNLOCK1 + i, kpad->unlock_keys[i]); + if (ret) + return ret; } - if (pdata->gpimapsize) { - ret |= adp5588_write(client, GPI_EM1, evt_mode1); - ret |= adp5588_write(client, GPI_EM2, evt_mode2); - ret |= adp5588_write(client, GPI_EM3, evt_mode3); + if (kpad->nkeys_unlock) { + ret = adp5588_write(client, KEY_LCK_EC_STAT, ADP5588_K_LCK_EN); + if (ret) + return ret; } - if (gpio_data) { - for (i = 0; i <= ADP5588_BANK(ADP5588_MAXGPIO); i++) { - int pull_mask = gpio_data->pullup_dis_mask; - - ret |= adp5588_write(client, GPIO_PULL1 + i, - (pull_mask >> (8 * i)) & 0xFF); - } + for (i = 0; i < KEYP_MAX_EVENT; i++) { + ret = adp5588_read(client, KEY_EVENTA); + if (ret) + return ret; } - ret |= adp5588_write(client, INT_STAT, - ADP5588_CMP2_INT | ADP5588_CMP1_INT | - ADP5588_OVR_FLOW_INT | ADP5588_K_LCK_INT | - ADP5588_GPI_INT | ADP5588_KE_INT); /* Status is W1C */ + ret = adp5588_write(client, INT_STAT, + ADP5588_CMP2_INT | ADP5588_CMP1_INT | + ADP5588_OVR_FLOW_INT | ADP5588_K_LCK_INT | + ADP5588_GPI_INT | ADP5588_KE_INT); /* Status is W1C */ + if (ret) + return ret; + + return adp5588_write(client, CFG, ADP5588_INT_CFG | + ADP5588_OVR_FLOW_IEN | ADP5588_KE_IEN); +} - ret |= adp5588_write(client, CFG, ADP5588_INT_CFG | - ADP5588_OVR_FLOW_IEN | - ADP5588_KE_IEN); +static int adp5588_fw_parse(struct adp5588_kpad *kpad) +{ + struct i2c_client *client = kpad->client; + int ret, i; - if (ret < 0) { - dev_err(&client->dev, "Write Error\n"); + ret = matrix_keypad_parse_properties(&client->dev, &kpad->rows, + &kpad->cols); + if (ret) return ret; + + if (kpad->rows > ADP5588_ROWS_MAX || kpad->cols > ADP5588_COLS_MAX) { + dev_err(&client->dev, "Invalid nr of rows(%u) or cols(%u)\n", + kpad->rows, kpad->cols); + return -EINVAL; } - return 0; -} + ret = matrix_keypad_build_keymap(NULL, NULL, kpad->rows, kpad->cols, + kpad->keycode, kpad->input); + if (ret) + return ret; -static void adp5588_report_switch_state(struct adp5588_kpad *kpad) -{ - int gpi_stat1 = adp5588_read(kpad->client, GPIO_DAT_STAT1); - int gpi_stat2 = adp5588_read(kpad->client, GPIO_DAT_STAT2); - int gpi_stat3 = adp5588_read(kpad->client, GPIO_DAT_STAT3); - int gpi_stat_tmp, pin_loc; - int i; + kpad->row_shift = get_count_order(kpad->cols); - for (i = 0; i < kpad->gpimapsize; i++) { - unsigned short pin = kpad->gpimap[i].pin; + if (device_property_read_bool(&client->dev, "autorepeat")) + __set_bit(EV_REP, kpad->input->evbit); - if (pin <= GPI_PIN_ROW_END) { - gpi_stat_tmp = gpi_stat1; - pin_loc = pin - GPI_PIN_ROW_BASE; - } else if ((pin - GPI_PIN_COL_BASE) < 8) { - gpi_stat_tmp = gpi_stat2; - pin_loc = pin - GPI_PIN_COL_BASE; - } else { - gpi_stat_tmp = gpi_stat3; - pin_loc = pin - GPI_PIN_COL_BASE - 8; - } + kpad->nkeys_unlock = device_property_count_u32(&client->dev, + "adi,unlock-keys"); + if (kpad->nkeys_unlock <= 0) { + /* so that we don't end up enabling key lock */ + kpad->nkeys_unlock = 0; + return 0; + } - if (gpi_stat_tmp < 0) { - dev_err(&kpad->client->dev, - "Can't read GPIO_DAT_STAT switch %d default to OFF\n", - pin); - gpi_stat_tmp = 0; + if (kpad->nkeys_unlock > ARRAY_SIZE(kpad->unlock_keys)) { + dev_err(&client->dev, "number of unlock keys(%d) > (%zu)\n", + kpad->nkeys_unlock, ARRAY_SIZE(kpad->unlock_keys)); + return -EINVAL; + } + + ret = device_property_read_u32_array(&client->dev, "adi,unlock-keys", + kpad->unlock_keys, + kpad->nkeys_unlock); + if (ret) + return ret; + + for (i = 0; i < kpad->nkeys_unlock; i++) { + /* + * Even though it should be possible (as stated in the datasheet) + * to use GPIs (which are part of the keys event) as unlock keys, + * it was not working at all and was leading to overflow events + * at some point. Hence, for now, let's just allow keys which are + * part of keypad matrix to be used and if a reliable way of + * using GPIs is found, this condition can be removed/lightened. + */ + if (kpad->unlock_keys[i] >= kpad->cols * kpad->rows) { + dev_err(&client->dev, "Invalid unlock key(%d)\n", + kpad->unlock_keys[i]); + return -EINVAL; } - input_report_switch(kpad->input, - kpad->gpimap[i].sw_evt, - !(gpi_stat_tmp & (1 << pin_loc))); + /* + * Firmware properties keys start from 0 but on the device they + * start from 1. + */ + kpad->unlock_keys[i] += 1; } - input_sync(kpad->input); + return 0; } +static void adp5588_disable_regulator(void *reg) +{ + regulator_disable(reg); +} static int adp5588_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct adp5588_kpad *kpad; - const struct adp5588_kpad_platform_data *pdata = - dev_get_platdata(&client->dev); struct input_dev *input; + struct gpio_desc *gpio; + struct regulator *vcc; unsigned int revid; - int ret, i; + int ret; int error; if (!i2c_check_functionality(client->adapter, - I2C_FUNC_SMBUS_BYTE_DATA)) { + I2C_FUNC_SMBUS_BYTE_DATA)) { dev_err(&client->dev, "SMBUS Byte Data not Supported\n"); return -EIO; } - if (!pdata) { - dev_err(&client->dev, "no platform data?\n"); - return -EINVAL; - } - - if (!pdata->rows || !pdata->cols || !pdata->keymap) { - dev_err(&client->dev, "no rows, cols or keymap from pdata\n"); - return -EINVAL; - } - - if (pdata->keymapsize != ADP5588_KEYMAPSIZE) { - dev_err(&client->dev, "invalid keymapsize\n"); - return -EINVAL; - } - - if (!pdata->gpimap && pdata->gpimapsize) { - dev_err(&client->dev, "invalid gpimap from pdata\n"); - return -EINVAL; - } - - if (pdata->gpimapsize > ADP5588_GPIMAPSIZE_MAX) { - dev_err(&client->dev, "invalid gpimapsize\n"); - return -EINVAL; - } - - for (i = 0; i < pdata->gpimapsize; i++) { - unsigned short pin = pdata->gpimap[i].pin; - - if (pin < GPI_PIN_BASE || pin > GPI_PIN_END) { - dev_err(&client->dev, "invalid gpi pin data\n"); - return -EINVAL; - } - - if (pin <= GPI_PIN_ROW_END) { - if (pin - GPI_PIN_ROW_BASE + 1 <= pdata->rows) { - dev_err(&client->dev, "invalid gpi row data\n"); - return -EINVAL; - } - } else { - if (pin - GPI_PIN_COL_BASE + 1 <= pdata->cols) { - dev_err(&client->dev, "invalid gpi col data\n"); - return -EINVAL; - } - } - } - - if (!client->irq) { - dev_err(&client->dev, "no IRQ?\n"); - return -EINVAL; - } - kpad = devm_kzalloc(&client->dev, sizeof(*kpad), GFP_KERNEL); if (!kpad) return -ENOMEM; @@ -516,11 +746,38 @@ static int adp5588_probe(struct i2c_client *client, kpad->client = client; kpad->input = input; + error = adp5588_fw_parse(kpad); + if (error) + return error; + + vcc = devm_regulator_get(&client->dev, "vcc"); + if (IS_ERR(vcc)) + return PTR_ERR(vcc); + + error = regulator_enable(vcc); + if (error) + return error; + + error = devm_add_action_or_reset(&client->dev, + adp5588_disable_regulator, vcc); + if (error) + return error; + + gpio = devm_gpiod_get_optional(&client->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(gpio)) + return PTR_ERR(gpio); + + if (gpio) { + fsleep(30); + gpiod_set_value_cansleep(gpio, 0); + fsleep(60); + } + ret = adp5588_read(client, DEV_ID); if (ret < 0) return ret; - revid = (u8) ret & ADP5588_DEVICE_ID_MASK; + revid = ret & ADP5588_DEVICE_ID_MASK; if (WA_DELAYED_READOUT_REVID(revid)) kpad->delay = msecs_to_jiffies(WA_DELAYED_READOUT_TIME); @@ -534,32 +791,6 @@ static int adp5588_probe(struct i2c_client *client, input->id.product = 0x0001; input->id.version = revid; - input->keycodesize = sizeof(kpad->keycode[0]); - input->keycodemax = pdata->keymapsize; - input->keycode = kpad->keycode; - - memcpy(kpad->keycode, pdata->keymap, - pdata->keymapsize * input->keycodesize); - - kpad->gpimap = pdata->gpimap; - kpad->gpimapsize = pdata->gpimapsize; - - /* setup input device */ - __set_bit(EV_KEY, input->evbit); - - if (pdata->repeat) - __set_bit(EV_REP, input->evbit); - - for (i = 0; i < input->keycodemax; i++) - if (kpad->keycode[i] <= KEY_MAX) - __set_bit(kpad->keycode[i], input->keybit); - __clear_bit(KEY_RESERVED, input->keybit); - - if (kpad->gpimapsize) - __set_bit(EV_SW, input->evbit); - for (i = 0; i < kpad->gpimapsize; i++) - __set_bit(kpad->gpimap[i].sw_evt, input->swbit); - error = input_register_device(input); if (error) { dev_err(&client->dev, "unable to register input device: %d\n", @@ -567,6 +798,14 @@ static int adp5588_probe(struct i2c_client *client, return error; } + error = adp5588_setup(kpad); + if (error) + return error; + + error = adp5588_gpio_add(kpad); + if (error) + return error; + error = devm_request_threaded_irq(&client->dev, client->irq, adp5588_hard_irq, adp5588_thread_irq, IRQF_TRIGGER_FALLING | IRQF_ONESHOT, @@ -577,30 +816,18 @@ static int adp5588_probe(struct i2c_client *client, return error; } - error = adp5588_setup(client); - if (error) - return error; - - if (kpad->gpimapsize) - adp5588_report_switch_state(kpad); - - error = adp5588_gpio_add(kpad); - if (error) - return error; - dev_info(&client->dev, "Rev.%d keypad, irq %d\n", revid, client->irq); return 0; } -static int adp5588_remove(struct i2c_client *client) +static void adp5588_remove(struct i2c_client *client) { adp5588_write(client, CFG, 0); /* all resources will be freed by devm */ - return 0; } -static int __maybe_unused adp5588_suspend(struct device *dev) +static int adp5588_suspend(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); @@ -609,7 +836,7 @@ static int __maybe_unused adp5588_suspend(struct device *dev) return 0; } -static int __maybe_unused adp5588_resume(struct device *dev) +static int adp5588_resume(struct device *dev) { struct i2c_client *client = to_i2c_client(dev); @@ -618,7 +845,7 @@ static int __maybe_unused adp5588_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(adp5588_dev_pm_ops, adp5588_suspend, adp5588_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(adp5588_dev_pm_ops, adp5588_suspend, adp5588_resume); static const struct i2c_device_id adp5588_id[] = { { "adp5588-keys", 0 }, @@ -627,10 +854,18 @@ static const struct i2c_device_id adp5588_id[] = { }; MODULE_DEVICE_TABLE(i2c, adp5588_id); +static const struct of_device_id adp5588_of_match[] = { + { .compatible = "adi,adp5588" }, + { .compatible = "adi,adp5587" }, + {} +}; +MODULE_DEVICE_TABLE(of, adp5588_of_match); + static struct i2c_driver adp5588_driver = { .driver = { .name = KBUILD_MODNAME, - .pm = &adp5588_dev_pm_ops, + .of_match_table = adp5588_of_match, + .pm = pm_sleep_ptr(&adp5588_dev_pm_ops), }, .probe = adp5588_probe, .remove = adp5588_remove, diff --git a/drivers/input/keyboard/amikbd.c b/drivers/input/keyboard/amikbd.c index 09551f64d53ff3da40114c3fade074971c33338f..a20a4e186639c05bf11004269a40a12b46263b63 100644 --- a/drivers/input/keyboard/amikbd.c +++ b/drivers/input/keyboard/amikbd.c @@ -10,9 +10,6 @@ * Amiga keyboard driver for Linux/m68k */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/applespi.c b/drivers/input/keyboard/applespi.c index cbc6c0d4670a2b26a9b5f75f018a7904a872b79a..91a9810f698073e023b5f491b9e59f23128c5a38 100644 --- a/drivers/input/keyboard/applespi.c +++ b/drivers/input/keyboard/applespi.c @@ -202,7 +202,7 @@ struct command_protocol_tp_info { }; /** - * struct touchpad_info - touchpad info response. + * struct touchpad_info_protocol - touchpad info response. * message.type = 0x1020, message.length = 0x006e * * @unknown1: unknown @@ -311,7 +311,7 @@ struct message { struct command_protocol_mt_init init_mt_command; struct command_protocol_capsl capsl_command; struct command_protocol_bl bl_command; - u8 data[0]; + DECLARE_FLEX_ARRAY(u8, data); }; }; diff --git a/drivers/input/keyboard/atakbd.c b/drivers/input/keyboard/atakbd.c index 77ed54630601ed8df739a0274acf8681934a66fc..07e17e563f9b6573fecb99f47bf8395bbabc0d64 100644 --- a/drivers/input/keyboard/atakbd.c +++ b/drivers/input/keyboard/atakbd.c @@ -21,9 +21,6 @@ * This driver only deals with handing key events off to the input layer. */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index d4131236d18c5442d000e8bb568e8c9161b49a7c..246958795f606ea72d0a5855d0f21ccc675930be 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -323,11 +323,13 @@ static umode_t atkbd_attr_is_visible(struct kobject *kobj, return attr->mode; } -static struct attribute_group atkbd_attribute_group = { +static const struct attribute_group atkbd_attribute_group = { .attrs = atkbd_attributes, .is_visible = atkbd_attr_is_visible, }; +__ATTRIBUTE_GROUPS(atkbd_attribute); + static const unsigned int xl_table[] = { ATKBD_RET_BAT, ATKBD_RET_ERR, ATKBD_RET_ACK, ATKBD_RET_NAK, ATKBD_RET_HANJA, ATKBD_RET_HANGEUL, @@ -922,8 +924,6 @@ static void atkbd_disconnect(struct serio *serio) { struct atkbd *atkbd = serio_get_drvdata(serio); - sysfs_remove_group(&serio->dev.kobj, &atkbd_attribute_group); - atkbd_disable(atkbd); input_unregister_device(atkbd->dev); @@ -1271,21 +1271,16 @@ static int atkbd_connect(struct serio *serio, struct serio_driver *drv) atkbd_set_keycode_table(atkbd); atkbd_set_device_attrs(atkbd); - err = sysfs_create_group(&serio->dev.kobj, &atkbd_attribute_group); - if (err) - goto fail3; - atkbd_enable(atkbd); if (serio->write) atkbd_activate(atkbd); err = input_register_device(atkbd->dev); if (err) - goto fail4; + goto fail3; return 0; - fail4: sysfs_remove_group(&serio->dev.kobj, &atkbd_attribute_group); fail3: serio_close(serio); fail2: serio_set_drvdata(serio, NULL); fail1: input_free_device(dev); @@ -1378,7 +1373,8 @@ MODULE_DEVICE_TABLE(serio, atkbd_serio_ids); static struct serio_driver atkbd_drv = { .driver = { - .name = "atkbd", + .name = "atkbd", + .dev_groups = atkbd_attribute_groups, }, .description = DRIVER_DESC, .id_table = atkbd_serio_ids, diff --git a/drivers/input/keyboard/clps711x-keypad.c b/drivers/input/keyboard/clps711x-keypad.c index 939c88655fc02a20adf3a07305a066f73d04084d..4c1a3e611edd7490c5401ee4e39cc16f9fb4459b 100644 --- a/drivers/input/keyboard/clps711x-keypad.c +++ b/drivers/input/keyboard/clps711x-keypad.c @@ -6,9 +6,11 @@ */ #include +#include #include -#include +#include #include +#include #include #include #include @@ -86,7 +88,6 @@ static int clps711x_keypad_probe(struct platform_device *pdev) { struct clps711x_keypad_data *priv; struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; struct input_dev *input; u32 poll_interval; int i, err; @@ -95,11 +96,11 @@ static int clps711x_keypad_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; - priv->syscon = syscon_regmap_lookup_by_phandle(np, "syscon"); + priv->syscon = syscon_regmap_lookup_by_phandle(dev->of_node, "syscon"); if (IS_ERR(priv->syscon)) return PTR_ERR(priv->syscon); - priv->row_count = of_gpio_named_count(np, "row-gpios"); + priv->row_count = gpiod_count(dev, "row"); if (priv->row_count < 1) return -EINVAL; @@ -119,7 +120,7 @@ static int clps711x_keypad_probe(struct platform_device *pdev) return PTR_ERR(data->desc); } - err = of_property_read_u32(np, "poll-interval", &poll_interval); + err = device_property_read_u32(dev, "poll-interval", &poll_interval); if (err) return err; @@ -143,7 +144,7 @@ static int clps711x_keypad_probe(struct platform_device *pdev) return err; input_set_capability(input, EV_MSC, MSC_SCAN); - if (of_property_read_bool(np, "autorepeat")) + if (device_property_read_bool(dev, "autorepeat")) __set_bit(EV_REP, input->evbit); /* Set all columns to low */ diff --git a/drivers/input/keyboard/ep93xx_keypad.c b/drivers/input/keyboard/ep93xx_keypad.c index 7a3b0664ab4f47eab4acf4849ca1b94ba33faa74..f5bf7524722a7867d0e8bac07b582188c68d77f1 100644 --- a/drivers/input/keyboard/ep93xx_keypad.c +++ b/drivers/input/keyboard/ep93xx_keypad.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/input/keyboard/imx_keypad.c b/drivers/input/keyboard/imx_keypad.c index ae9303848571612c0c1261cbec8a493668d0810d..e15a93619e827ff93659d8441dd2aeb14a287329 100644 --- a/drivers/input/keyboard/imx_keypad.c +++ b/drivers/input/keyboard/imx_keypad.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/input/keyboard/iqs62x-keys.c b/drivers/input/keyboard/iqs62x-keys.c index 93446b21f98f543ad5ea336bd798bb79d6efa89b..db793a550c258c35f8795b53e87f1589dd46a765 100644 --- a/drivers/input/keyboard/iqs62x-keys.c +++ b/drivers/input/keyboard/iqs62x-keys.c @@ -77,6 +77,7 @@ static int iqs62x_keys_parse_prop(struct platform_device *pdev, if (ret) { dev_err(&pdev->dev, "Failed to read switch code: %d\n", ret); + fwnode_handle_put(child); return ret; } iqs62x_keys->switches[i].code = val; @@ -90,6 +91,8 @@ static int iqs62x_keys_parse_prop(struct platform_device *pdev, iqs62x_keys->switches[i].flag = (i == IQS62X_SW_HALL_N ? IQS62X_EVENT_HALL_N_T : IQS62X_EVENT_HALL_S_T); + + fwnode_handle_put(child); } return 0; diff --git a/drivers/input/keyboard/lkkbd.c b/drivers/input/keyboard/lkkbd.c index e4a1839ca934aa97c5e47017b135d4d3b9a10d5a..047b654b3752c61e4900a59c589a1b1efd3210dd 100644 --- a/drivers/input/keyboard/lkkbd.c +++ b/drivers/input/keyboard/lkkbd.c @@ -46,9 +46,6 @@ * http://www.vt100.net/manx/details?pn=EK-104AA-TM-001;id=21;cp=1 */ -/* - */ - #include #include #include @@ -359,18 +356,18 @@ static void lkkbd_detection_done(struct lkkbd *lk) */ switch (lk->id[4]) { case 1: - strlcpy(lk->name, "DEC LK201 keyboard", sizeof(lk->name)); + strscpy(lk->name, "DEC LK201 keyboard", sizeof(lk->name)); if (lk201_compose_is_alt) lk->keycode[0xb1] = KEY_LEFTALT; break; case 2: - strlcpy(lk->name, "DEC LK401 keyboard", sizeof(lk->name)); + strscpy(lk->name, "DEC LK401 keyboard", sizeof(lk->name)); break; default: - strlcpy(lk->name, "Unknown DEC keyboard", sizeof(lk->name)); + strscpy(lk->name, "Unknown DEC keyboard", sizeof(lk->name)); printk(KERN_ERR "lkkbd: keyboard on %s is unknown, please report to " "Jan-Benedict Glaw \n", lk->phys); @@ -626,7 +623,7 @@ static int lkkbd_connect(struct serio *serio, struct serio_driver *drv) lk->ctrlclick_volume = ctrlclick_volume; memcpy(lk->keycode, lkkbd_keycode, sizeof(lk->keycode)); - strlcpy(lk->name, "DEC LK keyboard", sizeof(lk->name)); + strscpy(lk->name, "DEC LK keyboard", sizeof(lk->name)); snprintf(lk->phys, sizeof(lk->phys), "%s/input0", serio->phys); input_dev->name = lk->name; diff --git a/drivers/input/keyboard/lm8323.c b/drivers/input/keyboard/lm8323.c index 6c38d034ec6e181401221f41117fe64b97dda05b..407dd2ad6302578a280dd91c731801bbda5779c6 100644 --- a/drivers/input/keyboard/lm8323.c +++ b/drivers/input/keyboard/lm8323.c @@ -752,7 +752,7 @@ fail1: return err; } -static int lm8323_remove(struct i2c_client *client) +static void lm8323_remove(struct i2c_client *client) { struct lm8323_chip *lm = i2c_get_clientdata(client); int i; @@ -769,8 +769,6 @@ static int lm8323_remove(struct i2c_client *client) led_classdev_unregister(&lm->pwm[i].cdev); kfree(lm); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/input/keyboard/lm8333.c b/drivers/input/keyboard/lm8333.c index 7c5f8c6bb9578fe97a6b2108cc96f5d77d3bc72a..3052cd6dedacd55df78c102033c575f3e1b0f150 100644 --- a/drivers/input/keyboard/lm8333.c +++ b/drivers/input/keyboard/lm8333.c @@ -4,13 +4,13 @@ * Copyright (C) 2012 Wolfram Sang, Pengutronix */ -#include -#include -#include #include -#include +#include #include #include +#include +#include +#include #define LM8333_FIFO_READ 0x20 #define LM8333_DEBOUNCE 0x22 @@ -200,15 +200,13 @@ static int lm8333_probe(struct i2c_client *client, return err; } -static int lm8333_remove(struct i2c_client *client) +static void lm8333_remove(struct i2c_client *client) { struct lm8333 *lm8333 = i2c_get_clientdata(client); free_irq(client->irq, lm8333); input_unregister_device(lm8333->input); kfree(lm8333); - - return 0; } static const struct i2c_device_id lm8333_id[] = { diff --git a/drivers/input/keyboard/matrix_keypad.c b/drivers/input/keyboard/matrix_keypad.c index 30924b57058f251501f8dec1472bd33f5e88131a..7dd3f3eda834e2b60fbc882cf6a68e7b1007cd3f 100644 --- a/drivers/input/keyboard/matrix_keypad.c +++ b/drivers/input/keyboard/matrix_keypad.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -416,9 +417,9 @@ matrix_keypad_parse_dt(struct device *dev) return ERR_PTR(-ENOMEM); } - pdata->num_row_gpios = nrow = of_gpio_named_count(np, "row-gpios"); - pdata->num_col_gpios = ncol = of_gpio_named_count(np, "col-gpios"); - if (nrow <= 0 || ncol <= 0) { + pdata->num_row_gpios = nrow = gpiod_count(dev, "row"); + pdata->num_col_gpios = ncol = gpiod_count(dev, "col"); + if (nrow < 0 || ncol < 0) { dev_err(dev, "number of keypad rows/columns not specified\n"); return ERR_PTR(-EINVAL); } diff --git a/drivers/input/keyboard/mcs_touchkey.c b/drivers/input/keyboard/mcs_touchkey.c index 8cb0062b98e41c862a898fe5d47decec03ed7f07..ac1637a3389e31c269529a3d928e820cd5c138f9 100644 --- a/drivers/input/keyboard/mcs_touchkey.c +++ b/drivers/input/keyboard/mcs_touchkey.c @@ -194,7 +194,7 @@ err_free_mem: return error; } -static int mcs_touchkey_remove(struct i2c_client *client) +static void mcs_touchkey_remove(struct i2c_client *client) { struct mcs_touchkey_data *data = i2c_get_clientdata(client); @@ -203,8 +203,6 @@ static int mcs_touchkey_remove(struct i2c_client *client) data->poweron(false); input_unregister_device(data->input_dev); kfree(data); - - return 0; } static void mcs_touchkey_shutdown(struct i2c_client *client) diff --git a/drivers/input/keyboard/mt6779-keypad.c b/drivers/input/keyboard/mt6779-keypad.c index bf447bf598fbc826532e99b41153da6542b725f9..19f69d167fbd8360d93864280ff0b088da1cba3e 100644 --- a/drivers/input/keyboard/mt6779-keypad.c +++ b/drivers/input/keyboard/mt6779-keypad.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #define MTK_KPD_DEBOUNCE_MASK GENMASK(13, 0) #define MTK_KPD_DEBOUNCE_MAX_MS 256 #define MTK_KPD_SEL 0x0020 +#define MTK_KPD_SEL_DOUBLE_KP_MODE BIT(0) #define MTK_KPD_SEL_COL GENMASK(15, 10) #define MTK_KPD_SEL_ROW GENMASK(9, 4) #define MTK_KPD_SEL_COLMASK(c) GENMASK((c) + 9, 10) @@ -31,6 +33,8 @@ struct mt6779_keypad { struct clk *clk; u32 n_rows; u32 n_cols; + void (*calc_row_col)(unsigned int key, + unsigned int *row, unsigned int *col); DECLARE_BITMAP(keymap_state, MTK_KPD_NUM_BITS); }; @@ -67,8 +71,7 @@ static irqreturn_t mt6779_keypad_irq_handler(int irq, void *dev_id) continue; key = bit_nr / 32 * 16 + bit_nr % 32; - row = key / 9; - col = key % 9; + keypad->calc_row_col(key, &row, &col); scancode = MATRIX_SCAN_CODE(row, col, row_shift); /* 1: not pressed, 0: pressed */ @@ -94,12 +97,29 @@ static void mt6779_keypad_clk_disable(void *data) clk_disable_unprepare(data); } +static void mt6779_keypad_calc_row_col_single(unsigned int key, + unsigned int *row, + unsigned int *col) +{ + *row = key / 9; + *col = key % 9; +} + +static void mt6779_keypad_calc_row_col_double(unsigned int key, + unsigned int *row, + unsigned int *col) +{ + *row = key / 13; + *col = (key % 13) / 2; +} + static int mt6779_keypad_pdrv_probe(struct platform_device *pdev) { struct mt6779_keypad *keypad; void __iomem *base; int irq; u32 debounce; + u32 keys_per_group; bool wakeup; int error; @@ -148,6 +168,23 @@ static int mt6779_keypad_pdrv_probe(struct platform_device *pdev) return -EINVAL; } + if (device_property_read_u32(&pdev->dev, "mediatek,keys-per-group", + &keys_per_group)) + keys_per_group = 1; + + switch (keys_per_group) { + case 1: + keypad->calc_row_col = mt6779_keypad_calc_row_col_single; + break; + case 2: + keypad->calc_row_col = mt6779_keypad_calc_row_col_double; + break; + default: + dev_err(&pdev->dev, + "Invalid keys-per-group: %d\n", keys_per_group); + return -EINVAL; + } + wakeup = device_property_read_bool(&pdev->dev, "wakeup-source"); dev_dbg(&pdev->dev, "n_row=%d n_col=%d debounce=%d\n", @@ -166,6 +203,11 @@ static int mt6779_keypad_pdrv_probe(struct platform_device *pdev) regmap_write(keypad->regmap, MTK_KPD_DEBOUNCE, (debounce * (1 << 5)) & MTK_KPD_DEBOUNCE_MASK); + if (keys_per_group == 2) + regmap_update_bits(keypad->regmap, MTK_KPD_SEL, + MTK_KPD_SEL_DOUBLE_KP_MODE, + MTK_KPD_SEL_DOUBLE_KP_MODE); + regmap_update_bits(keypad->regmap, MTK_KPD_SEL, MTK_KPD_SEL_ROW, MTK_KPD_SEL_ROWMASK(keypad->n_rows)); regmap_update_bits(keypad->regmap, MTK_KPD_SEL, MTK_KPD_SEL_COL, diff --git a/drivers/input/keyboard/mtk-pmic-keys.c b/drivers/input/keyboard/mtk-pmic-keys.c index 6404081253ea187225e451a7648612eb35bd8d97..9b34da0ec260507b3d2efaef0d9e084b891e5a60 100644 --- a/drivers/input/keyboard/mtk-pmic-keys.c +++ b/drivers/input/keyboard/mtk-pmic-keys.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,10 @@ #define MTK_PMIC_PWRKEY_RST BIT(6) #define MTK_PMIC_HOMEKEY_RST BIT(5) +#define MTK_PMIC_MT6331_RST_DU_MASK GENMASK(13, 12) +#define MTK_PMIC_MT6331_PWRKEY_RST BIT(9) +#define MTK_PMIC_MT6331_HOMEKEY_RST BIT(8) + #define MTK_PMIC_PWRKEY_INDEX 0 #define MTK_PMIC_HOMEKEY_INDEX 1 #define MTK_PMIC_MAX_KEY_COUNT 2 @@ -72,6 +77,19 @@ static const struct mtk_pmic_regs mt6323_regs = { .rst_lprst_mask = MTK_PMIC_RST_DU_MASK, }; +static const struct mtk_pmic_regs mt6331_regs = { + .keys_regs[MTK_PMIC_PWRKEY_INDEX] = + MTK_PMIC_KEYS_REGS(MT6331_TOPSTATUS, 0x2, + MT6331_INT_MISC_CON, 0x4, + MTK_PMIC_MT6331_PWRKEY_RST), + .keys_regs[MTK_PMIC_HOMEKEY_INDEX] = + MTK_PMIC_KEYS_REGS(MT6331_TOPSTATUS, 0x4, + MT6331_INT_MISC_CON, 0x2, + MTK_PMIC_MT6331_HOMEKEY_RST), + .pmic_rst_reg = MT6331_TOP_RST_MISC, + .rst_lprst_mask = MTK_PMIC_MT6331_RST_DU_MASK, +}; + static const struct mtk_pmic_regs mt6358_regs = { .keys_regs[MTK_PMIC_PWRKEY_INDEX] = MTK_PMIC_KEYS_REGS(MT6358_TOPSTATUS, @@ -255,6 +273,9 @@ static const struct of_device_id of_mtk_pmic_keys_match_tbl[] = { }, { .compatible = "mediatek,mt6323-keys", .data = &mt6323_regs, + }, { + .compatible = "mediatek,mt6331-keys", + .data = &mt6331_regs, }, { .compatible = "mediatek,mt6358-keys", .data = &mt6358_regs, diff --git a/drivers/input/keyboard/newtonkbd.c b/drivers/input/keyboard/newtonkbd.c index 9742261b2d1a7dac49939ec97d7b280dfed31c59..df00a119aa9a2481dfc55e263acd2ad6b54b5291 100644 --- a/drivers/input/keyboard/newtonkbd.c +++ b/drivers/input/keyboard/newtonkbd.c @@ -7,9 +7,6 @@ * Newton keyboard driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/pinephone-keyboard.c b/drivers/input/keyboard/pinephone-keyboard.c new file mode 100644 index 0000000000000000000000000000000000000000..5548699b8b389f75d5fe123343dd5661445f41f1 --- /dev/null +++ b/drivers/input/keyboard/pinephone-keyboard.c @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright (C) 2021-2022 Samuel Holland + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "pinephone-keyboard" + +#define PPKB_CRC8_POLYNOMIAL 0x07 + +#define PPKB_DEVICE_ID_HI 0x00 +#define PPKB_DEVICE_ID_HI_VALUE 'K' +#define PPKB_DEVICE_ID_LO 0x01 +#define PPKB_DEVICE_ID_LO_VALUE 'B' +#define PPKB_FW_REVISION 0x02 +#define PPKB_FW_FEATURES 0x03 +#define PPKB_MATRIX_SIZE 0x06 +#define PPKB_SCAN_CRC 0x07 +#define PPKB_SCAN_DATA 0x08 +#define PPKB_SYS_CONFIG 0x20 +#define PPKB_SYS_CONFIG_DISABLE_SCAN BIT(0) +#define PPKB_SYS_SMBUS_COMMAND 0x21 +#define PPKB_SYS_SMBUS_DATA 0x22 +#define PPKB_SYS_COMMAND 0x23 +#define PPKB_SYS_COMMAND_SMBUS_READ 0x91 +#define PPKB_SYS_COMMAND_SMBUS_WRITE 0xa1 + +#define PPKB_ROWS 6 +#define PPKB_COLS 12 + +/* Size of the scan buffer, including the CRC byte at the beginning. */ +#define PPKB_BUF_LEN (1 + PPKB_COLS) + +static const uint32_t ppkb_keymap[] = { + KEY(0, 0, KEY_ESC), + KEY(0, 1, KEY_1), + KEY(0, 2, KEY_2), + KEY(0, 3, KEY_3), + KEY(0, 4, KEY_4), + KEY(0, 5, KEY_5), + KEY(0, 6, KEY_6), + KEY(0, 7, KEY_7), + KEY(0, 8, KEY_8), + KEY(0, 9, KEY_9), + KEY(0, 10, KEY_0), + KEY(0, 11, KEY_BACKSPACE), + + KEY(1, 0, KEY_TAB), + KEY(1, 1, KEY_Q), + KEY(1, 2, KEY_W), + KEY(1, 3, KEY_E), + KEY(1, 4, KEY_R), + KEY(1, 5, KEY_T), + KEY(1, 6, KEY_Y), + KEY(1, 7, KEY_U), + KEY(1, 8, KEY_I), + KEY(1, 9, KEY_O), + KEY(1, 10, KEY_P), + KEY(1, 11, KEY_ENTER), + + KEY(2, 0, KEY_LEFTMETA), + KEY(2, 1, KEY_A), + KEY(2, 2, KEY_S), + KEY(2, 3, KEY_D), + KEY(2, 4, KEY_F), + KEY(2, 5, KEY_G), + KEY(2, 6, KEY_H), + KEY(2, 7, KEY_J), + KEY(2, 8, KEY_K), + KEY(2, 9, KEY_L), + KEY(2, 10, KEY_SEMICOLON), + + KEY(3, 0, KEY_LEFTSHIFT), + KEY(3, 1, KEY_Z), + KEY(3, 2, KEY_X), + KEY(3, 3, KEY_C), + KEY(3, 4, KEY_V), + KEY(3, 5, KEY_B), + KEY(3, 6, KEY_N), + KEY(3, 7, KEY_M), + KEY(3, 8, KEY_COMMA), + KEY(3, 9, KEY_DOT), + KEY(3, 10, KEY_SLASH), + + KEY(4, 1, KEY_LEFTCTRL), + KEY(4, 4, KEY_SPACE), + KEY(4, 6, KEY_APOSTROPHE), + KEY(4, 8, KEY_RIGHTBRACE), + KEY(4, 9, KEY_LEFTBRACE), + + KEY(5, 2, KEY_FN), + KEY(5, 3, KEY_LEFTALT), + KEY(5, 5, KEY_RIGHTALT), + + /* FN layer */ + KEY(PPKB_ROWS + 0, 0, KEY_FN_ESC), + KEY(PPKB_ROWS + 0, 1, KEY_F1), + KEY(PPKB_ROWS + 0, 2, KEY_F2), + KEY(PPKB_ROWS + 0, 3, KEY_F3), + KEY(PPKB_ROWS + 0, 4, KEY_F4), + KEY(PPKB_ROWS + 0, 5, KEY_F5), + KEY(PPKB_ROWS + 0, 6, KEY_F6), + KEY(PPKB_ROWS + 0, 7, KEY_F7), + KEY(PPKB_ROWS + 0, 8, KEY_F8), + KEY(PPKB_ROWS + 0, 9, KEY_F9), + KEY(PPKB_ROWS + 0, 10, KEY_F10), + KEY(PPKB_ROWS + 0, 11, KEY_DELETE), + + KEY(PPKB_ROWS + 1, 10, KEY_PAGEUP), + + KEY(PPKB_ROWS + 2, 0, KEY_SYSRQ), + KEY(PPKB_ROWS + 2, 9, KEY_PAGEDOWN), + KEY(PPKB_ROWS + 2, 10, KEY_INSERT), + + KEY(PPKB_ROWS + 3, 0, KEY_LEFTSHIFT), + KEY(PPKB_ROWS + 3, 8, KEY_HOME), + KEY(PPKB_ROWS + 3, 9, KEY_UP), + KEY(PPKB_ROWS + 3, 10, KEY_END), + + KEY(PPKB_ROWS + 4, 1, KEY_LEFTCTRL), + KEY(PPKB_ROWS + 4, 6, KEY_LEFT), + KEY(PPKB_ROWS + 4, 8, KEY_RIGHT), + KEY(PPKB_ROWS + 4, 9, KEY_DOWN), + + KEY(PPKB_ROWS + 5, 3, KEY_LEFTALT), + KEY(PPKB_ROWS + 5, 5, KEY_RIGHTALT), +}; + +static const struct matrix_keymap_data ppkb_keymap_data = { + .keymap = ppkb_keymap, + .keymap_size = ARRAY_SIZE(ppkb_keymap), +}; + +struct pinephone_keyboard { + struct i2c_adapter adapter; + struct input_dev *input; + u8 buf[2][PPKB_BUF_LEN]; + u8 crc_table[CRC8_TABLE_SIZE]; + u8 fn_state[PPKB_COLS]; + bool buf_swap; + bool fn_pressed; +}; + +static int ppkb_adap_smbus_xfer(struct i2c_adapter *adap, u16 addr, + unsigned short flags, char read_write, + u8 command, int size, + union i2c_smbus_data *data) +{ + struct i2c_client *client = adap->algo_data; + u8 buf[3]; + int ret; + + buf[0] = command; + buf[1] = data->byte; + buf[2] = read_write == I2C_SMBUS_READ ? PPKB_SYS_COMMAND_SMBUS_READ + : PPKB_SYS_COMMAND_SMBUS_WRITE; + + ret = i2c_smbus_write_i2c_block_data(client, PPKB_SYS_SMBUS_COMMAND, + sizeof(buf), buf); + if (ret) + return ret; + + /* Read back the command status until it passes or fails. */ + do { + usleep_range(300, 500); + ret = i2c_smbus_read_byte_data(client, PPKB_SYS_COMMAND); + } while (ret == buf[2]); + if (ret < 0) + return ret; + /* Commands return 0x00 on success and 0xff on failure. */ + if (ret) + return -EIO; + + if (read_write == I2C_SMBUS_READ) { + ret = i2c_smbus_read_byte_data(client, PPKB_SYS_SMBUS_DATA); + if (ret < 0) + return ret; + + data->byte = ret; + } + + return 0; +} + +static u32 ppkg_adap_functionality(struct i2c_adapter *adap) +{ + return I2C_FUNC_SMBUS_BYTE_DATA; +} + +static const struct i2c_algorithm ppkb_adap_algo = { + .smbus_xfer = ppkb_adap_smbus_xfer, + .functionality = ppkg_adap_functionality, +}; + +static void ppkb_update(struct i2c_client *client) +{ + struct pinephone_keyboard *ppkb = i2c_get_clientdata(client); + unsigned short *keymap = ppkb->input->keycode; + int row_shift = get_count_order(PPKB_COLS); + u8 *old_buf = ppkb->buf[!ppkb->buf_swap]; + u8 *new_buf = ppkb->buf[ppkb->buf_swap]; + int col, crc, ret, row; + struct device *dev = &client->dev; + + ret = i2c_smbus_read_i2c_block_data(client, PPKB_SCAN_CRC, + PPKB_BUF_LEN, new_buf); + if (ret != PPKB_BUF_LEN) { + dev_err(dev, "Failed to read scan data: %d\n", ret); + return; + } + + crc = crc8(ppkb->crc_table, &new_buf[1], PPKB_COLS, CRC8_INIT_VALUE); + if (crc != new_buf[0]) { + dev_err(dev, "Bad scan data (%02x != %02x)\n", crc, new_buf[0]); + return; + } + + ppkb->buf_swap = !ppkb->buf_swap; + + for (col = 0; col < PPKB_COLS; ++col) { + u8 old = old_buf[1 + col]; + u8 new = new_buf[1 + col]; + u8 changed = old ^ new; + + if (!changed) + continue; + + for (row = 0; row < PPKB_ROWS; ++row) { + u8 mask = BIT(row); + u8 value = new & mask; + unsigned short code; + bool fn_state; + + if (!(changed & mask)) + continue; + + /* + * Save off the FN key state when the key was pressed, + * and use that to determine the code during a release. + */ + fn_state = value ? ppkb->fn_pressed : ppkb->fn_state[col] & mask; + if (fn_state) + ppkb->fn_state[col] ^= mask; + + /* The FN layer is a second set of rows. */ + code = MATRIX_SCAN_CODE(fn_state ? PPKB_ROWS + row : row, + col, row_shift); + input_event(ppkb->input, EV_MSC, MSC_SCAN, code); + input_report_key(ppkb->input, keymap[code], value); + if (keymap[code] == KEY_FN) + ppkb->fn_pressed = value; + } + } + input_sync(ppkb->input); +} + +static irqreturn_t ppkb_irq_thread(int irq, void *data) +{ + struct i2c_client *client = data; + + ppkb_update(client); + + return IRQ_HANDLED; +} + +static int ppkb_set_scan(struct i2c_client *client, bool enable) +{ + struct device *dev = &client->dev; + int ret, val; + + ret = i2c_smbus_read_byte_data(client, PPKB_SYS_CONFIG); + if (ret < 0) { + dev_err(dev, "Failed to read config: %d\n", ret); + return ret; + } + + if (enable) + val = ret & ~PPKB_SYS_CONFIG_DISABLE_SCAN; + else + val = ret | PPKB_SYS_CONFIG_DISABLE_SCAN; + + ret = i2c_smbus_write_byte_data(client, PPKB_SYS_CONFIG, val); + if (ret) { + dev_err(dev, "Failed to write config: %d\n", ret); + return ret; + } + + return 0; +} + +static int ppkb_open(struct input_dev *input) +{ + struct i2c_client *client = input_get_drvdata(input); + int error; + + error = ppkb_set_scan(client, true); + if (error) + return error; + + return 0; +} + +static void ppkb_close(struct input_dev *input) +{ + struct i2c_client *client = input_get_drvdata(input); + + ppkb_set_scan(client, false); +} + +static void ppkb_regulator_disable(void *regulator) +{ + regulator_disable(regulator); +} + +static int ppkb_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + unsigned int phys_rows, phys_cols; + struct pinephone_keyboard *ppkb; + struct regulator *vbat_supply; + u8 info[PPKB_MATRIX_SIZE + 1]; + struct device_node *i2c_bus; + int ret; + int error; + + vbat_supply = devm_regulator_get(dev, "vbat"); + error = PTR_ERR_OR_ZERO(vbat_supply); + if (error) { + dev_err(dev, "Failed to get VBAT supply: %d\n", error); + return error; + } + + error = regulator_enable(vbat_supply); + if (error) { + dev_err(dev, "Failed to enable VBAT: %d\n", error); + return error; + } + + error = devm_add_action_or_reset(dev, ppkb_regulator_disable, + vbat_supply); + if (error) + return error; + + ret = i2c_smbus_read_i2c_block_data(client, 0, sizeof(info), info); + if (ret != sizeof(info)) { + error = ret < 0 ? ret : -EIO; + dev_err(dev, "Failed to read device ID: %d\n", error); + return error; + } + + if (info[PPKB_DEVICE_ID_HI] != PPKB_DEVICE_ID_HI_VALUE || + info[PPKB_DEVICE_ID_LO] != PPKB_DEVICE_ID_LO_VALUE) { + dev_warn(dev, "Unexpected device ID: %#02x %#02x\n", + info[PPKB_DEVICE_ID_HI], info[PPKB_DEVICE_ID_LO]); + return -ENODEV; + } + + dev_info(dev, "Found firmware version %d.%d features %#x\n", + info[PPKB_FW_REVISION] >> 4, + info[PPKB_FW_REVISION] & 0xf, + info[PPKB_FW_FEATURES]); + + phys_rows = info[PPKB_MATRIX_SIZE] & 0xf; + phys_cols = info[PPKB_MATRIX_SIZE] >> 4; + if (phys_rows != PPKB_ROWS || phys_cols != PPKB_COLS) { + dev_err(dev, "Unexpected keyboard size %ux%u\n", + phys_rows, phys_cols); + return -EINVAL; + } + + /* Disable scan by default to save power. */ + error = ppkb_set_scan(client, false); + if (error) + return error; + + ppkb = devm_kzalloc(dev, sizeof(*ppkb), GFP_KERNEL); + if (!ppkb) + return -ENOMEM; + + i2c_set_clientdata(client, ppkb); + + i2c_bus = of_get_child_by_name(dev->of_node, "i2c"); + if (i2c_bus) { + ppkb->adapter.owner = THIS_MODULE; + ppkb->adapter.algo = &ppkb_adap_algo; + ppkb->adapter.algo_data = client; + ppkb->adapter.dev.parent = dev; + ppkb->adapter.dev.of_node = i2c_bus; + strscpy(ppkb->adapter.name, DRV_NAME, sizeof(ppkb->adapter.name)); + + error = devm_i2c_add_adapter(dev, &ppkb->adapter); + if (error) { + dev_err(dev, "Failed to add I2C adapter: %d\n", error); + return error; + } + } + + crc8_populate_msb(ppkb->crc_table, PPKB_CRC8_POLYNOMIAL); + + ppkb->input = devm_input_allocate_device(dev); + if (!ppkb->input) + return -ENOMEM; + + input_set_drvdata(ppkb->input, client); + + ppkb->input->name = "PinePhone Keyboard"; + ppkb->input->phys = DRV_NAME "/input0"; + ppkb->input->id.bustype = BUS_I2C; + ppkb->input->open = ppkb_open; + ppkb->input->close = ppkb_close; + + input_set_capability(ppkb->input, EV_MSC, MSC_SCAN); + __set_bit(EV_REP, ppkb->input->evbit); + + error = matrix_keypad_build_keymap(&ppkb_keymap_data, NULL, + 2 * PPKB_ROWS, PPKB_COLS, NULL, + ppkb->input); + if (error) { + dev_err(dev, "Failed to build keymap: %d\n", error); + return error; + } + + error = input_register_device(ppkb->input); + if (error) { + dev_err(dev, "Failed to register input: %d\n", error); + return error; + } + + error = devm_request_threaded_irq(dev, client->irq, + NULL, ppkb_irq_thread, + IRQF_ONESHOT, client->name, client); + if (error) { + dev_err(dev, "Failed to request IRQ: %d\n", error); + return error; + } + + return 0; +} + +static const struct of_device_id ppkb_of_match[] = { + { .compatible = "pine64,pinephone-keyboard" }, + { } +}; +MODULE_DEVICE_TABLE(of, ppkb_of_match); + +static struct i2c_driver ppkb_driver = { + .probe_new = ppkb_probe, + .driver = { + .name = DRV_NAME, + .of_match_table = ppkb_of_match, + }, +}; +module_i2c_driver(ppkb_driver); + +MODULE_AUTHOR("Samuel Holland "); +MODULE_DESCRIPTION("Pine64 PinePhone keyboard driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/keyboard/qt1070.c b/drivers/input/keyboard/qt1070.c index 7174e1df1ee3b5cccc7294cd195b4d76fc2dc52b..9fcce18b1d656057a4cc30cc3e30bb63dac28c2f 100644 --- a/drivers/input/keyboard/qt1070.c +++ b/drivers/input/keyboard/qt1070.c @@ -216,7 +216,7 @@ err_free_mem: return err; } -static int qt1070_remove(struct i2c_client *client) +static void qt1070_remove(struct i2c_client *client) { struct qt1070_data *data = i2c_get_clientdata(client); @@ -225,8 +225,6 @@ static int qt1070_remove(struct i2c_client *client) input_unregister_device(data->input); kfree(data); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/input/keyboard/qt2160.c b/drivers/input/keyboard/qt2160.c index 32d4a076eaa3182a0c5a7f2ae1635d63dcb43e33..382b1519218c49fef2bfc7f83a748fa7e2590c73 100644 --- a/drivers/input/keyboard/qt2160.c +++ b/drivers/input/keyboard/qt2160.c @@ -432,7 +432,7 @@ err_free_mem: return error; } -static int qt2160_remove(struct i2c_client *client) +static void qt2160_remove(struct i2c_client *client) { struct qt2160_data *qt2160 = i2c_get_clientdata(client); @@ -446,8 +446,6 @@ static int qt2160_remove(struct i2c_client *client) input_unregister_device(qt2160->input); kfree(qt2160); - - return 0; } static const struct i2c_device_id qt2160_idtable[] = { diff --git a/drivers/input/keyboard/snvs_pwrkey.c b/drivers/input/keyboard/snvs_pwrkey.c index 65286762b02ab918403d91de894e849820010ec1..ad8660be0127c2080421b2a9a364c9855ae27caa 100644 --- a/drivers/input/keyboard/snvs_pwrkey.c +++ b/drivers/input/keyboard/snvs_pwrkey.c @@ -20,7 +20,7 @@ #include #include -#define SNVS_HPVIDR1_REG 0xF8 +#define SNVS_HPVIDR1_REG 0xBF8 #define SNVS_LPSR_REG 0x4C /* LP Status Register */ #define SNVS_LPCR_REG 0x38 /* LP Control Register */ #define SNVS_HPSR_REG 0x14 diff --git a/drivers/input/keyboard/st-keyscan.c b/drivers/input/keyboard/st-keyscan.c index a045d61165acc25e5efcaa7cac2307a64d787c1a..a62bb8fff88c6de18956d92748fa911fc0dfc038 100644 --- a/drivers/input/keyboard/st-keyscan.c +++ b/drivers/input/keyboard/st-keyscan.c @@ -8,12 +8,14 @@ * Based on sh_keysc.c, copyright 2008 Magnus Damm */ -#include -#include -#include #include -#include +#include #include +#include +#include +#include +#include +#include #define ST_KEYSCAN_MAXKEYS 16 diff --git a/drivers/input/keyboard/stowaway.c b/drivers/input/keyboard/stowaway.c index a4977193dd4a5d71e38b60950c01c445ab2b2e9a..56e78493605964d1da0362f7cddc83ad179cc4b3 100644 --- a/drivers/input/keyboard/stowaway.c +++ b/drivers/input/keyboard/stowaway.c @@ -10,9 +10,6 @@ * by Justin Cormack */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/sunkbd.c b/drivers/input/keyboard/sunkbd.c index d450f11b98a703598dfa826cd49f74b888cb0efd..b123a208ef36996f96ea2a40f56ca15e142a185d 100644 --- a/drivers/input/keyboard/sunkbd.c +++ b/drivers/input/keyboard/sunkbd.c @@ -7,9 +7,6 @@ * Sun keyboard driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/keyboard/tc3589x-keypad.c b/drivers/input/keyboard/tc3589x-keypad.c index 89b9575dc75dc1900cc68c59689b0ec1f21f1216..78e55318ccd63b1977cf50b2bc548a3fc5e10881 100644 --- a/drivers/input/keyboard/tc3589x-keypad.c +++ b/drivers/input/keyboard/tc3589x-keypad.c @@ -70,7 +70,7 @@ #define TC3589x_KBD_INT_CLR 0x1 /** - * struct tc35893_keypad_platform_data - platform specific keypad data + * struct tc3589x_keypad_platform_data - platform specific keypad data * @keymap_data: matrix scan code table for keycodes * @krow: mask for available rows, value is 0xFF * @kcol: mask for available columns, value is 0xFF diff --git a/drivers/input/keyboard/tca6416-keypad.c b/drivers/input/keyboard/tca6416-keypad.c index 2a97559100652218889538c2750cd8e06c38f6e4..afcdfbb002ff34d1d3d9f11c2c6ba85bab439d36 100644 --- a/drivers/input/keyboard/tca6416-keypad.c +++ b/drivers/input/keyboard/tca6416-keypad.c @@ -307,7 +307,7 @@ fail1: return error; } -static int tca6416_keypad_remove(struct i2c_client *client) +static void tca6416_keypad_remove(struct i2c_client *client) { struct tca6416_keypad_chip *chip = i2c_get_clientdata(client); @@ -318,8 +318,6 @@ static int tca6416_keypad_remove(struct i2c_client *client) input_unregister_device(chip->input); kfree(chip); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/input/keyboard/xtkbd.c b/drivers/input/keyboard/xtkbd.c index 280796df679a300376e2eeb734e51ea1d292dd4c..c9d7c248172602d29bffe7dc8f65340fda6e85ed 100644 --- a/drivers/input/keyboard/xtkbd.c +++ b/drivers/input/keyboard/xtkbd.c @@ -7,9 +7,6 @@ * XT keyboard driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index a18ab7358d8f39c0fea58157ea86dfa467706606..9f088900f863b7f1f37c738a6172828fae4cf423 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -730,6 +730,24 @@ config INPUT_ADXL34X_SPI To compile this driver as a module, choose M here: the module will be called adxl34x-spi. +config INPUT_IBM_PANEL + tristate "IBM Operation Panel driver" + depends on I2C && I2C_SLAVE + help + Say Y here if you have an IBM Operation Panel connected to your system + over I2C. The panel is typically connected only to a system's service + processor (BMC). + + If unsure, say N. + + The Operation Panel is a controller with some buttons and an LCD + display that allows someone with physical access to the system to + perform various administrative tasks. This driver only supports the part + of the controller that sends commands to the system. + + To compile this driver as a module, choose M here: the module will be + called ibm-panel. + config INPUT_IMS_PCU tristate "IMS Passenger Control Unit driver" depends on USB @@ -891,6 +909,15 @@ config INPUT_SC27XX_VIBRA To compile this driver as a module, choose M here. The module will be called sc27xx_vibra. +config INPUT_RT5120_PWRKEY + tristate "RT5120 PMIC power key support" + depends on MFD_RT5120 || COMPILE_TEST + help + This enables support for RT5120 PMIC power key driver. + + To compile this driver as a module, choose M here. the module will + be called rt5120-pwrkey. + config INPUT_STPMIC1_ONKEY tristate "STPMIC1 PMIC Onkey support" depends on MFD_STPMIC1 diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 28dfc444f0a96b0d9b1b00ed43b4a1dc990fcce4..6abefc41037b532faa35144d3c5c58c393ec5b17 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -41,6 +41,7 @@ obj-$(CONFIG_INPUT_GPIO_DECODER) += gpio_decoder.o obj-$(CONFIG_INPUT_GPIO_VIBRA) += gpio-vibra.o obj-$(CONFIG_INPUT_HISI_POWERKEY) += hisi_powerkey.o obj-$(CONFIG_HP_SDC_RTC) += hp_sdc_rtc.o +obj-$(CONFIG_INPUT_IBM_PANEL) += ibm-panel.o obj-$(CONFIG_INPUT_IMS_PCU) += ims-pcu.o obj-$(CONFIG_INPUT_IQS269A) += iqs269a.o obj-$(CONFIG_INPUT_IQS626A) += iqs626a.o @@ -69,6 +70,7 @@ obj-$(CONFIG_INPUT_RAVE_SP_PWRBUTTON) += rave-sp-pwrbutton.o obj-$(CONFIG_INPUT_RB532_BUTTON) += rb532_button.o obj-$(CONFIG_INPUT_REGULATOR_HAPTIC) += regulator-haptic.o obj-$(CONFIG_INPUT_RETU_PWRBUTTON) += retu-pwrbutton.o +obj-$(CONFIG_INPUT_RT5120_PWRKEY) += rt5120-pwrkey.o obj-$(CONFIG_INPUT_AXP20X_PEK) += axp20x-pek.o obj-$(CONFIG_INPUT_GPIO_ROTARY_ENCODER) += rotary_encoder.o obj-$(CONFIG_INPUT_RK805_PWRKEY) += rk805-pwrkey.o diff --git a/drivers/input/misc/adxl34x-i2c.c b/drivers/input/misc/adxl34x-i2c.c index a3b5f88d2bd1603f0e7bac1164ccce40d953c1fd..5be636aaa94fc2a0af7c52177d7809a19f9ab55f 100644 --- a/drivers/input/misc/adxl34x-i2c.c +++ b/drivers/input/misc/adxl34x-i2c.c @@ -99,13 +99,11 @@ static int adxl34x_i2c_probe(struct i2c_client *client, return 0; } -static int adxl34x_i2c_remove(struct i2c_client *client) +static void adxl34x_i2c_remove(struct i2c_client *client) { struct adxl34x *ac = i2c_get_clientdata(client); adxl34x_remove(ac); - - return 0; } static int __maybe_unused adxl34x_i2c_suspend(struct device *dev) diff --git a/drivers/input/misc/bma150.c b/drivers/input/misc/bma150.c index a9d984da95f38f89160338574e73e7c33e03131c..84fe394da7a64f78f4fb767aedd64a0a061b192d 100644 --- a/drivers/input/misc/bma150.c +++ b/drivers/input/misc/bma150.c @@ -513,11 +513,9 @@ static int bma150_probe(struct i2c_client *client, return 0; } -static int bma150_remove(struct i2c_client *client) +static void bma150_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); - - return 0; } static int __maybe_unused bma150_suspend(struct device *dev) diff --git a/drivers/input/misc/cma3000_d0x_i2c.c b/drivers/input/misc/cma3000_d0x_i2c.c index 03fb49127c3ac822116de4aab81126f104da0894..3b23210c46b7dc1e60f7ff0fb33fccc2ea3addf9 100644 --- a/drivers/input/misc/cma3000_d0x_i2c.c +++ b/drivers/input/misc/cma3000_d0x_i2c.c @@ -58,13 +58,11 @@ static int cma3000_i2c_probe(struct i2c_client *client, return 0; } -static int cma3000_i2c_remove(struct i2c_client *client) +static void cma3000_i2c_remove(struct i2c_client *client) { struct cma3000_accl_data *data = i2c_get_clientdata(client); cma3000_exit(data); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/input/misc/ibm-panel.c b/drivers/input/misc/ibm-panel.c new file mode 100644 index 0000000000000000000000000000000000000000..a8fba00547190205b8970fe2a17451c28bce6fdc --- /dev/null +++ b/drivers/input/misc/ibm-panel.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) IBM Corporation 2020 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEVICE_NAME "ibm-panel" +#define PANEL_KEYCODES_COUNT 3 + +struct ibm_panel { + u8 idx; + u8 command[11]; + u32 keycodes[PANEL_KEYCODES_COUNT]; + spinlock_t lock; /* protects writes to idx and command */ + struct input_dev *input; +}; + +static u8 ibm_panel_calculate_checksum(struct ibm_panel *panel) +{ + u8 chksum; + u16 sum = 0; + unsigned int i; + + for (i = 0; i < sizeof(panel->command) - 1; ++i) { + sum += panel->command[i]; + if (sum & 0xff00) { + sum &= 0xff; + sum++; + } + } + + chksum = sum & 0xff; + chksum = ~chksum; + chksum++; + + return chksum; +} + +static void ibm_panel_process_command(struct ibm_panel *panel) +{ + u8 button; + u8 chksum; + + if (panel->command[0] != 0xff && panel->command[1] != 0xf0) { + dev_dbg(&panel->input->dev, "command invalid: %02x %02x\n", + panel->command[0], panel->command[1]); + return; + } + + chksum = ibm_panel_calculate_checksum(panel); + if (chksum != panel->command[sizeof(panel->command) - 1]) { + dev_dbg(&panel->input->dev, + "command failed checksum: %u != %u\n", chksum, + panel->command[sizeof(panel->command) - 1]); + return; + } + + button = panel->command[2] & 0xf; + if (button < PANEL_KEYCODES_COUNT) { + input_report_key(panel->input, panel->keycodes[button], + !(panel->command[2] & 0x80)); + input_sync(panel->input); + } else { + dev_dbg(&panel->input->dev, "unknown button %u\n", + button); + } +} + +static int ibm_panel_i2c_slave_cb(struct i2c_client *client, + enum i2c_slave_event event, u8 *val) +{ + unsigned long flags; + struct ibm_panel *panel = i2c_get_clientdata(client); + + dev_dbg(&panel->input->dev, "event: %u data: %02x\n", event, *val); + + spin_lock_irqsave(&panel->lock, flags); + + switch (event) { + case I2C_SLAVE_STOP: + if (panel->idx == sizeof(panel->command)) + ibm_panel_process_command(panel); + else + dev_dbg(&panel->input->dev, + "command incorrect size %u\n", panel->idx); + fallthrough; + case I2C_SLAVE_WRITE_REQUESTED: + panel->idx = 0; + break; + case I2C_SLAVE_WRITE_RECEIVED: + if (panel->idx < sizeof(panel->command)) + panel->command[panel->idx++] = *val; + else + /* + * The command is too long and therefore invalid, so set the index + * to it's largest possible value. When a STOP is finally received, + * the command will be rejected upon processing. + */ + panel->idx = U8_MAX; + break; + case I2C_SLAVE_READ_REQUESTED: + case I2C_SLAVE_READ_PROCESSED: + *val = 0xff; + break; + default: + break; + } + + spin_unlock_irqrestore(&panel->lock, flags); + + return 0; +} + +static int ibm_panel_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct ibm_panel *panel; + int i; + int error; + + panel = devm_kzalloc(&client->dev, sizeof(*panel), GFP_KERNEL); + if (!panel) + return -ENOMEM; + + spin_lock_init(&panel->lock); + + panel->input = devm_input_allocate_device(&client->dev); + if (!panel->input) + return -ENOMEM; + + panel->input->name = client->name; + panel->input->id.bustype = BUS_I2C; + + error = device_property_read_u32_array(&client->dev, + "linux,keycodes", + panel->keycodes, + PANEL_KEYCODES_COUNT); + if (error) { + /* + * Use gamepad buttons as defaults for compatibility with + * existing applications. + */ + panel->keycodes[0] = BTN_NORTH; + panel->keycodes[1] = BTN_SOUTH; + panel->keycodes[2] = BTN_SELECT; + } + + for (i = 0; i < PANEL_KEYCODES_COUNT; ++i) + input_set_capability(panel->input, EV_KEY, panel->keycodes[i]); + + error = input_register_device(panel->input); + if (error) { + dev_err(&client->dev, + "Failed to register input device: %d\n", error); + return error; + } + + i2c_set_clientdata(client, panel); + error = i2c_slave_register(client, ibm_panel_i2c_slave_cb); + if (error) { + dev_err(&client->dev, + "Failed to register as i2c slave: %d\n", error); + return error; + } + + return 0; +} + +static void ibm_panel_remove(struct i2c_client *client) +{ + i2c_slave_unregister(client); +} + +static const struct of_device_id ibm_panel_match[] = { + { .compatible = "ibm,op-panel" }, + { } +}; +MODULE_DEVICE_TABLE(of, ibm_panel_match); + +static struct i2c_driver ibm_panel_driver = { + .driver = { + .name = DEVICE_NAME, + .of_match_table = ibm_panel_match, + }, + .probe = ibm_panel_probe, + .remove = ibm_panel_remove, +}; +module_i2c_driver(ibm_panel_driver); + +MODULE_AUTHOR("Eddie James "); +MODULE_DESCRIPTION("IBM Operation Panel Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c index 6f38aa23a1ff6a029d1d6e40b202c2990c81f7c4..b2f1292e27ef7d5e9d6c0160c05479bd8f144e1d 100644 --- a/drivers/input/misc/ims-pcu.c +++ b/drivers/input/misc/ims-pcu.c @@ -744,7 +744,7 @@ static int ims_pcu_switch_to_bootloader(struct ims_pcu *pcu) error = ims_pcu_execute_command(pcu, JUMP_TO_BTLDR, NULL, 0); if (error) { dev_err(pcu->dev, - "Failure when sending JUMP TO BOOLTLOADER command, error: %d\n", + "Failure when sending JUMP TO BOOTLOADER command, error: %d\n", error); return error; } diff --git a/drivers/input/misc/iqs7222.c b/drivers/input/misc/iqs7222.c index b2e8097a2e6d97b18efd059f7884c50e22adbf9c..ddb863bf63eec22682058868e98be75938c075bb 100644 --- a/drivers/input/misc/iqs7222.c +++ b/drivers/input/misc/iqs7222.c @@ -1077,7 +1077,7 @@ static int iqs7222_hard_reset(struct iqs7222_private *iqs7222) static int iqs7222_force_comms(struct iqs7222_private *iqs7222) { - u8 msg_buf[] = { 0xFF, 0x00, }; + u8 msg_buf[] = { 0xFF, }; int ret; /* @@ -1771,11 +1771,9 @@ static int iqs7222_parse_chan(struct iqs7222_private *iqs7222, int chan_index) if (!chan_node) return 0; - if (dev_desc->allow_offset) { - sys_setup[dev_desc->allow_offset] |= BIT(chan_index); - if (fwnode_property_present(chan_node, "azoteq,ulp-allow")) - sys_setup[dev_desc->allow_offset] &= ~BIT(chan_index); - } + if (dev_desc->allow_offset && + fwnode_property_present(chan_node, "azoteq,ulp-allow")) + sys_setup[dev_desc->allow_offset] &= ~BIT(chan_index); chan_setup[0] |= IQS7222_CHAN_SETUP_0_CHAN_EN; @@ -2206,6 +2204,9 @@ static int iqs7222_parse_all(struct iqs7222_private *iqs7222) u16 *sys_setup = iqs7222->sys_setup; int error, i; + if (dev_desc->allow_offset) + sys_setup[dev_desc->allow_offset] = U16_MAX; + if (dev_desc->event_offset) sys_setup[dev_desc->event_offset] = IQS7222_EVENT_MASK_ATI; @@ -2326,6 +2327,9 @@ static int iqs7222_report(struct iqs7222_private *iqs7222) int k = 2 + j * (num_chan > 16 ? 2 : 1); u16 state = le16_to_cpu(status[k + i / 16]); + if (!iqs7222->kp_type[i][j]) + continue; + input_event(iqs7222->keypad, iqs7222->kp_type[i][j], iqs7222->kp_code[i][j], diff --git a/drivers/input/misc/keyspan_remote.c b/drivers/input/misc/keyspan_remote.c index 4650f4a949890f577af46898ab7967210b5a9486..bee4b137649144a7fb1a9d0bed9d5d584c971227 100644 --- a/drivers/input/misc/keyspan_remote.c +++ b/drivers/input/misc/keyspan_remote.c @@ -485,7 +485,7 @@ static int keyspan_probe(struct usb_interface *interface, const struct usb_devic } if (udev->manufacturer) - strlcpy(remote->name, udev->manufacturer, sizeof(remote->name)); + strscpy(remote->name, udev->manufacturer, sizeof(remote->name)); if (udev->product) { if (udev->manufacturer) diff --git a/drivers/input/misc/pcf8574_keypad.c b/drivers/input/misc/pcf8574_keypad.c index abc42316552276d6447dd71e5fda51544fac2ed5..cfd6640e4f82a55da669b8f2f5cf02c130e10f1f 100644 --- a/drivers/input/misc/pcf8574_keypad.c +++ b/drivers/input/misc/pcf8574_keypad.c @@ -157,7 +157,7 @@ static int pcf8574_kp_probe(struct i2c_client *client, const struct i2c_device_i return ret; } -static int pcf8574_kp_remove(struct i2c_client *client) +static void pcf8574_kp_remove(struct i2c_client *client) { struct kp_data *lp = i2c_get_clientdata(client); @@ -165,8 +165,6 @@ static int pcf8574_kp_remove(struct i2c_client *client) input_unregister_device(lp->idev); kfree(lp); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/input/misc/rk805-pwrkey.c b/drivers/input/misc/rk805-pwrkey.c index 3fb64dbda1a2189117a8ef6605b9602195feff57..76873aa005b419b0c1644b9eee7f886c389cabd7 100644 --- a/drivers/input/misc/rk805-pwrkey.c +++ b/drivers/input/misc/rk805-pwrkey.c @@ -98,6 +98,7 @@ static struct platform_driver rk805_pwrkey_driver = { }; module_platform_driver(rk805_pwrkey_driver); +MODULE_ALIAS("platform:rk805-pwrkey"); MODULE_AUTHOR("Joseph Chen "); MODULE_DESCRIPTION("RK805 PMIC Power Key driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/input/misc/rt5120-pwrkey.c b/drivers/input/misc/rt5120-pwrkey.c new file mode 100644 index 0000000000000000000000000000000000000000..8a8c1aeeed050ed0c5b8980d0fa8c5fff9091752 --- /dev/null +++ b/drivers/input/misc/rt5120-pwrkey.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * Author: ChiYuan Huang + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define RT5120_REG_INTSTAT 0x1E +#define RT5120_PWRKEYSTAT_MASK BIT(7) + +struct rt5120_priv { + struct regmap *regmap; + struct input_dev *input; +}; + +static irqreturn_t rt5120_pwrkey_handler(int irq, void *devid) +{ + struct rt5120_priv *priv = devid; + unsigned int stat; + int error; + + error = regmap_read(priv->regmap, RT5120_REG_INTSTAT, &stat); + if (error) + return IRQ_NONE; + + input_report_key(priv->input, KEY_POWER, + !(stat & RT5120_PWRKEYSTAT_MASK)); + input_sync(priv->input); + + return IRQ_HANDLED; +} + +static int rt5120_pwrkey_probe(struct platform_device *pdev) +{ + struct rt5120_priv *priv; + struct device *dev = &pdev->dev; + int press_irq, release_irq; + int error; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->regmap = dev_get_regmap(dev->parent, NULL); + if (!priv->regmap) { + dev_err(dev, "Failed to init regmap\n"); + return -ENODEV; + } + + press_irq = platform_get_irq_byname(pdev, "pwrkey-press"); + if (press_irq < 0) + return press_irq; + + release_irq = platform_get_irq_byname(pdev, "pwrkey-release"); + if (release_irq < 0) + return release_irq; + + /* Make input device be device resource managed */ + priv->input = devm_input_allocate_device(dev); + if (!priv->input) + return -ENOMEM; + + priv->input->name = "rt5120_pwrkey"; + priv->input->phys = "rt5120_pwrkey/input0"; + priv->input->id.bustype = BUS_I2C; + input_set_capability(priv->input, EV_KEY, KEY_POWER); + + error = input_register_device(priv->input); + if (error) { + dev_err(dev, "Failed to register input device: %d\n", error); + return error; + } + + error = devm_request_threaded_irq(dev, press_irq, + NULL, rt5120_pwrkey_handler, + 0, "pwrkey-press", priv); + if (error) { + dev_err(dev, + "Failed to register pwrkey press irq: %d\n", error); + return error; + } + + error = devm_request_threaded_irq(dev, release_irq, + NULL, rt5120_pwrkey_handler, + 0, "pwrkey-release", priv); + if (error) { + dev_err(dev, + "Failed to register pwrkey release irq: %d\n", error); + return error; + } + + return 0; +} + +static const struct of_device_id r5120_pwrkey_match_table[] = { + { .compatible = "richtek,rt5120-pwrkey" }, + {} +}; +MODULE_DEVICE_TABLE(of, r5120_pwrkey_match_table); + +static struct platform_driver rt5120_pwrkey_driver = { + .driver = { + .name = "rt5120-pwrkey", + .of_match_table = r5120_pwrkey_match_table, + }, + .probe = rt5120_pwrkey_probe, +}; +module_platform_driver(rt5120_pwrkey_driver); + +MODULE_AUTHOR("ChiYuan Huang "); +MODULE_DESCRIPTION("Richtek RT5120 power key driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/input/misc/twl4030-pwrbutton.c b/drivers/input/misc/twl4030-pwrbutton.c index b307cca170222671948996e6585306c7ec72e71e..e3ee0638ffbaf09e22fc461375afe83ff248114a 100644 --- a/drivers/input/misc/twl4030-pwrbutton.c +++ b/drivers/input/misc/twl4030-pwrbutton.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include diff --git a/drivers/input/misc/twl4030-vibra.c b/drivers/input/misc/twl4030-vibra.c index e0ff616fb857e9b8aa8399f82edb6c2e191c7b57..5619996da86fc3737dffacb27e27e2d198055f1d 100644 --- a/drivers/input/misc/twl4030-vibra.c +++ b/drivers/input/misc/twl4030-vibra.c @@ -163,14 +163,10 @@ static int __maybe_unused twl4030_vibra_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(twl4030_vibra_pm_ops, twl4030_vibra_suspend, twl4030_vibra_resume); -static bool twl4030_vibra_check_coexist(struct twl4030_vibra_data *pdata, - struct device_node *parent) +static bool twl4030_vibra_check_coexist(struct device_node *parent) { struct device_node *node; - if (pdata && pdata->coexist) - return true; - node = of_get_child_by_name(parent, "codec"); if (node) { of_node_put(node); @@ -182,13 +178,12 @@ static bool twl4030_vibra_check_coexist(struct twl4030_vibra_data *pdata, static int twl4030_vibra_probe(struct platform_device *pdev) { - struct twl4030_vibra_data *pdata = dev_get_platdata(&pdev->dev); struct device_node *twl4030_core_node = pdev->dev.parent->of_node; struct vibra_info *info; int ret; - if (!pdata && !twl4030_core_node) { - dev_dbg(&pdev->dev, "platform_data not available\n"); + if (!twl4030_core_node) { + dev_dbg(&pdev->dev, "twl4030 OF node is missing\n"); return -EINVAL; } @@ -197,7 +192,7 @@ static int twl4030_vibra_probe(struct platform_device *pdev) return -ENOMEM; info->dev = &pdev->dev; - info->coexist = twl4030_vibra_check_coexist(pdata, twl4030_core_node); + info->coexist = twl4030_vibra_check_coexist(twl4030_core_node); INIT_WORK(&info->play_work, vibra_play_work); info->input_dev = devm_input_allocate_device(&pdev->dev); diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c index e1758d5ffe421831f64a198daa5db33e920c3033..d4eb59b55bf1fdf69eba7e249e8b65a222553af5 100644 --- a/drivers/input/mouse/elan_i2c_core.c +++ b/drivers/input/mouse/elan_i2c_core.c @@ -1311,12 +1311,6 @@ static int elan_probe(struct i2c_client *client, return error; } - error = devm_device_add_groups(dev, elan_sysfs_groups); - if (error) { - dev_err(dev, "failed to create sysfs attributes: %d\n", error); - return error; - } - error = input_register_device(data->input); if (error) { dev_err(dev, "failed to register input device: %d\n", error); @@ -1442,6 +1436,7 @@ static struct i2c_driver elan_driver = { .acpi_match_table = ACPI_PTR(elan_acpi_id), .of_match_table = of_match_ptr(elan_of_match), .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .dev_groups = elan_sysfs_groups, }, .probe = elan_probe, .id_table = elan_id, diff --git a/drivers/input/mouse/hgpk.c b/drivers/input/mouse/hgpk.c index 4dc441309aacf54463648ca494bbab21355902b5..3c8310da0b05376c46e1a6fc424c47087fa1f3e5 100644 --- a/drivers/input/mouse/hgpk.c +++ b/drivers/input/mouse/hgpk.c @@ -884,7 +884,7 @@ static ssize_t hgpk_trigger_recal(struct psmouse *psmouse, void *data, /* * We queue work instead of doing recalibration right here - * to avoid adding locking to to hgpk_force_recalibrate() + * to avoid adding locking to hgpk_force_recalibrate() * since workqueue provides serialization. */ psmouse_queue_work(psmouse, &priv->recalib_wq, 0); @@ -1057,7 +1057,7 @@ void hgpk_module_init(void) strlen(hgpk_mode_name)); if (hgpk_default_mode == HGPK_MODE_INVALID) { hgpk_default_mode = HGPK_MODE_MOUSE; - strlcpy(hgpk_mode_name, hgpk_mode_names[HGPK_MODE_MOUSE], + strscpy(hgpk_mode_name, hgpk_mode_names[HGPK_MODE_MOUSE], sizeof(hgpk_mode_name)); } } diff --git a/drivers/input/mouse/inport.c b/drivers/input/mouse/inport.c index df5d1160478c447fda2536438f179531d8037768..401d8bff8e842337979af24f4565290923db5b5d 100644 --- a/drivers/input/mouse/inport.c +++ b/drivers/input/mouse/inport.c @@ -13,9 +13,6 @@ * Inport (ATI XL and Microsoft) busmouse driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/mouse/logibm.c b/drivers/input/mouse/logibm.c index bd647f9f505a8e920464ab483a2fec7aa7145367..0aab63dbc30a324f1a14b18073f25cf45bfd2c64 100644 --- a/drivers/input/mouse/logibm.c +++ b/drivers/input/mouse/logibm.c @@ -14,9 +14,6 @@ * Logitech Bus Mouse Driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/mouse/pc110pad.c b/drivers/input/mouse/pc110pad.c index f75574766b85b01355251645b471d3fc573cbbb2..efa58049f746eebb18eec60ec99d7cb296ec05e1 100644 --- a/drivers/input/mouse/pc110pad.c +++ b/drivers/input/mouse/pc110pad.c @@ -10,9 +10,6 @@ * IBM PC110 touchpad driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/mouse/psmouse-base.c b/drivers/input/mouse/psmouse-base.c index 0b4a3039f312f0265071be4a32b2c5b8501af12f..c9a7e87b273ed2efdad2adbb09ae913989ab228d 100644 --- a/drivers/input/mouse/psmouse-base.c +++ b/drivers/input/mouse/psmouse-base.c @@ -94,7 +94,7 @@ PSMOUSE_DEFINE_ATTR(resync_time, S_IWUSR | S_IRUGO, (void *) offsetof(struct psmouse, resync_time), psmouse_show_int_attr, psmouse_set_int_attr); -static struct attribute *psmouse_attributes[] = { +static struct attribute *psmouse_dev_attrs[] = { &psmouse_attr_protocol.dattr.attr, &psmouse_attr_rate.dattr.attr, &psmouse_attr_resolution.dattr.attr, @@ -103,9 +103,7 @@ static struct attribute *psmouse_attributes[] = { NULL }; -static const struct attribute_group psmouse_attribute_group = { - .attrs = psmouse_attributes, -}; +ATTRIBUTE_GROUPS(psmouse_dev); /* * psmouse_mutex protects all operations changing state of mouse @@ -1481,8 +1479,6 @@ static void psmouse_disconnect(struct serio *serio) struct psmouse *psmouse = serio_get_drvdata(serio); struct psmouse *parent = NULL; - sysfs_remove_group(&serio->dev.kobj, &psmouse_attribute_group); - mutex_lock(&psmouse_mutex); psmouse_set_state(psmouse, PSMOUSE_CMD_MODE); @@ -1647,10 +1643,6 @@ static int psmouse_connect(struct serio *serio, struct serio_driver *drv) if (parent && parent->pt_activate) parent->pt_activate(parent); - error = sysfs_create_group(&serio->dev.kobj, &psmouse_attribute_group); - if (error) - goto err_pt_deactivate; - /* * PS/2 devices having SMBus companions should stay disabled * on PS/2 side, in order to have SMBus part operable. @@ -1666,13 +1658,6 @@ static int psmouse_connect(struct serio *serio, struct serio_driver *drv) mutex_unlock(&psmouse_mutex); return retval; - err_pt_deactivate: - if (parent && parent->pt_deactivate) - parent->pt_deactivate(parent); - if (input_dev) { - input_unregister_device(input_dev); - input_dev = NULL; /* so we don't try to free it below */ - } err_protocol_disconnect: if (psmouse->disconnect) psmouse->disconnect(psmouse); @@ -1791,7 +1776,8 @@ MODULE_DEVICE_TABLE(serio, psmouse_serio_ids); static struct serio_driver psmouse_drv = { .driver = { - .name = "psmouse", + .name = "psmouse", + .dev_groups = psmouse_dev_groups, }, .description = DRIVER_DESC, .id_table = psmouse_serio_ids, diff --git a/drivers/input/mouse/sermouse.c b/drivers/input/mouse/sermouse.c index caa79c177c5594dba0df76fd42257cfab1d68d9e..993f90333380814c4570e8665d0ca8c6c7569fa0 100644 --- a/drivers/input/mouse/sermouse.c +++ b/drivers/input/mouse/sermouse.c @@ -7,9 +7,6 @@ * Serial mouse driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index 434d48ae4b12e8a0fc27658d26ff55e18812497f..fa021af8506e4f8c3309026ca474a1c1d20a7824 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -182,11 +182,11 @@ static const char * const smbus_pnp_ids[] = { "LEN0099", /* X1 Extreme Gen 1 / P1 Gen 1 */ "LEN009b", /* T580 */ "LEN0402", /* X1 Extreme Gen 2 / P1 Gen 2 */ + "LEN040f", /* P1 Gen 3 */ "LEN200f", /* T450s */ "LEN2044", /* L470 */ "LEN2054", /* E480 */ "LEN2055", /* E580 */ - "LEN2064", /* T14 Gen 1 AMD / P14s Gen 1 AMD */ "LEN2068", /* T14 Gen 1 */ "SYN3052", /* HP EliteBook 840 G4 */ "SYN3221", /* HP 15-ay000 */ @@ -715,8 +715,8 @@ static void synaptics_pt_create(struct psmouse *psmouse) } serio->id.type = SERIO_PS_PSTHRU; - strlcpy(serio->name, "Synaptics pass-through", sizeof(serio->name)); - strlcpy(serio->phys, "synaptics-pt/serio0", sizeof(serio->phys)); + strscpy(serio->name, "Synaptics pass-through", sizeof(serio->name)); + strscpy(serio->phys, "synaptics-pt/serio0", sizeof(serio->phys)); serio->write = synaptics_pt_write; serio->start = synaptics_pt_start; serio->stop = synaptics_pt_stop; diff --git a/drivers/input/mouse/synaptics_i2c.c b/drivers/input/mouse/synaptics_i2c.c index fa304648d611cb016e1426f7fd54405469583794..987ee67a104531ce8cf4d0649f88320f0badcba3 100644 --- a/drivers/input/mouse/synaptics_i2c.c +++ b/drivers/input/mouse/synaptics_i2c.c @@ -587,7 +587,7 @@ err_mem_free: return ret; } -static int synaptics_i2c_remove(struct i2c_client *client) +static void synaptics_i2c_remove(struct i2c_client *client) { struct synaptics_i2c *touch = i2c_get_clientdata(client); @@ -596,8 +596,6 @@ static int synaptics_i2c_remove(struct i2c_client *client) input_unregister_device(touch->input); kfree(touch); - - return 0; } static int __maybe_unused synaptics_i2c_suspend(struct device *dev) diff --git a/drivers/input/mouse/synaptics_usb.c b/drivers/input/mouse/synaptics_usb.c index b5ff27e32a0c8a8282f426c13beb527231b77c61..75e45f3ae675cf9a384384a288de5e20a4cf44ad 100644 --- a/drivers/input/mouse/synaptics_usb.c +++ b/drivers/input/mouse/synaptics_usb.c @@ -354,7 +354,7 @@ static int synusb_probe(struct usb_interface *intf, synusb->urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; if (udev->manufacturer) - strlcpy(synusb->name, udev->manufacturer, + strscpy(synusb->name, udev->manufacturer, sizeof(synusb->name)); if (udev->product) { diff --git a/drivers/input/mouse/vsxxxaa.c b/drivers/input/mouse/vsxxxaa.c index bd415f4b574e53a08128b071fafc79d97301a622..8af8e4a15f95d6597a6433deb0e5e63ff5ab0508 100644 --- a/drivers/input/mouse/vsxxxaa.c +++ b/drivers/input/mouse/vsxxxaa.c @@ -12,9 +12,6 @@ * Later on, I had access to the device's documentation (referenced below). */ -/* - */ - /* * Building an adaptor to DE9 / DB25 RS232 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -138,12 +135,12 @@ static void vsxxxaa_detection_done(struct vsxxxaa *mouse) { switch (mouse->type) { case 0x02: - strlcpy(mouse->name, "DEC VSXXX-AA/-GA mouse", + strscpy(mouse->name, "DEC VSXXX-AA/-GA mouse", sizeof(mouse->name)); break; case 0x04: - strlcpy(mouse->name, "DEC VSXXX-AB digitizer", + strscpy(mouse->name, "DEC VSXXX-AB digitizer", sizeof(mouse->name)); break; diff --git a/drivers/input/rmi4/rmi_f03.c b/drivers/input/rmi4/rmi_f03.c index c194b1664b108926b2197e651c0ad4b62001d91e..1e11ea30d7bdb81c26e5cc44944a1852688f7f79 100644 --- a/drivers/input/rmi4/rmi_f03.c +++ b/drivers/input/rmi4/rmi_f03.c @@ -181,7 +181,7 @@ static int rmi_f03_register_pt(struct f03_data *f03) serio->close = rmi_f03_pt_close; serio->port_data = f03; - strlcpy(serio->name, "RMI4 PS/2 pass-through", sizeof(serio->name)); + strscpy(serio->name, "RMI4 PS/2 pass-through", sizeof(serio->name)); snprintf(serio->phys, sizeof(serio->phys), "%s/serio0", dev_name(&f03->fn->dev)); serio->dev.parent = &f03->fn->dev; diff --git a/drivers/input/rmi4/rmi_f34.c b/drivers/input/rmi4/rmi_f34.c index e5dca9868f87f3de65447c645ba66cc9638bc52d..0d9a5756e3f5934ba8df6e4b0518687c74bc3a16 100644 --- a/drivers/input/rmi4/rmi_f34.c +++ b/drivers/input/rmi4/rmi_f34.c @@ -114,13 +114,13 @@ static irqreturn_t rmi_f34_attention(int irq, void *ctx) complete(&f34->v5.cmd_done); } else { ret = rmi_read_block(f34->fn->rmi_dev, - f34->fn->fd.data_base_addr + - f34->v7.off.flash_status, - &status, sizeof(status)); - rmi_dbg(RMI_DEBUG_FN, &fn->dev, "%s: status: %#02x, ret: %d\n", + f34->fn->fd.data_base_addr + + V7_COMMAND_OFFSET, + &status, sizeof(status)); + rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev, "%s: cmd: %#02x, ret: %d\n", __func__, status, ret); - if (!ret && !(status & 0x1f)) + if (!ret && status == CMD_V7_IDLE) complete(&f34->v7.cmd_done); } @@ -321,13 +321,13 @@ static ssize_t rmi_driver_bootloader_id_show(struct device *dev, f34 = dev_get_drvdata(&fn->dev); if (f34->bl_version == 5) - return scnprintf(buf, PAGE_SIZE, "%c%c\n", - f34->bootloader_id[0], - f34->bootloader_id[1]); + return sysfs_emit(buf, "%c%c\n", + f34->bootloader_id[0], + f34->bootloader_id[1]); else - return scnprintf(buf, PAGE_SIZE, "V%d.%d\n", - f34->bootloader_id[1], - f34->bootloader_id[0]); + return sysfs_emit(buf, "V%d.%d\n", + f34->bootloader_id[1], + f34->bootloader_id[0]); } return 0; @@ -346,7 +346,7 @@ static ssize_t rmi_driver_configuration_id_show(struct device *dev, if (fn) { f34 = dev_get_drvdata(&fn->dev); - return scnprintf(buf, PAGE_SIZE, "%s\n", f34->configuration_id); + return sysfs_emit(buf, "%s\n", f34->configuration_id); } return 0; @@ -370,7 +370,7 @@ static int rmi_firmware_update(struct rmi_driver_data *data, f34 = dev_get_drvdata(&data->f34_container->dev); - if (f34->bl_version == 7) { + if (f34->bl_version >= 7) { if (data->pdt_props & HAS_BSR) { dev_err(dev, "%s: LTS not supported\n", __func__); return -ENODEV; @@ -382,7 +382,7 @@ static int rmi_firmware_update(struct rmi_driver_data *data, } /* Enter flash mode */ - if (f34->bl_version == 7) + if (f34->bl_version >= 7) ret = rmi_f34v7_start_reflash(f34, fw); else ret = rmi_f34_enable_flash(f34); @@ -413,7 +413,7 @@ static int rmi_firmware_update(struct rmi_driver_data *data, f34 = dev_get_drvdata(&data->f34_container->dev); /* Perform firmware update */ - if (f34->bl_version == 7) + if (f34->bl_version >= 7) ret = rmi_f34v7_do_reflash(f34, fw); else ret = rmi_f34_update_firmware(f34, fw); @@ -499,7 +499,7 @@ static ssize_t rmi_driver_update_fw_status_show(struct device *dev, if (data->f34_container) update_status = rmi_f34_status(data->f34_container); - return scnprintf(buf, PAGE_SIZE, "%d\n", update_status); + return sysfs_emit(buf, "%d\n", update_status); } static DEVICE_ATTR(update_fw_status, 0444, diff --git a/drivers/input/rmi4/rmi_f34.h b/drivers/input/rmi4/rmi_f34.h index 99faa8c2269df2a0ba1c2e8b3f5444a4716100a0..cfa3039804fd2822095c3feb6a245bb86b643c89 100644 --- a/drivers/input/rmi4/rmi_f34.h +++ b/drivers/input/rmi4/rmi_f34.h @@ -222,20 +222,6 @@ struct image_metadata { struct physical_address phyaddr; }; -struct register_offset { - u8 properties; - u8 properties_2; - u8 block_size; - u8 block_count; - u8 gc_block_count; - u8 flash_status; - u8 partition_id; - u8 block_number; - u8 transfer_length; - u8 flash_cmd; - u8 payload; -}; - struct rmi_f34_firmware { __le32 checksum; u8 pad1[3]; @@ -262,7 +248,6 @@ struct f34v5_data { struct f34v7_data { bool has_display_cfg; bool has_guest_code; - bool force_update; bool in_bl_mode; u8 *read_config_buf; size_t read_config_buf_size; @@ -276,9 +261,7 @@ struct f34v7_data { u16 payload_length; u8 partitions; u16 partition_table_bytes; - bool new_partition_table; - struct register_offset off; struct block_count blkcount; struct physical_address phyaddr; struct image_metadata img; diff --git a/drivers/input/rmi4/rmi_f34v7.c b/drivers/input/rmi4/rmi_f34v7.c index 8d7ec9d89b185be069e81d9d90ad472fb8547966..886557b01ebabf7fcb1a2e8e29050ea043d87074 100644 --- a/drivers/input/rmi4/rmi_f34v7.c +++ b/drivers/input/rmi4/rmi_f34v7.c @@ -25,7 +25,7 @@ static int rmi_f34v7_read_flash_status(struct f34_data *f34) int ret; ret = rmi_read_block(f34->fn->rmi_dev, - f34->fn->fd.data_base_addr + f34->v7.off.flash_status, + f34->fn->fd.data_base_addr + V7_FLASH_STATUS_OFFSET, &status, sizeof(status)); if (ret < 0) { @@ -43,7 +43,7 @@ static int rmi_f34v7_read_flash_status(struct f34_data *f34) } ret = rmi_read_block(f34->fn->rmi_dev, - f34->fn->fd.data_base_addr + f34->v7.off.flash_cmd, + f34->fn->fd.data_base_addr + V7_COMMAND_OFFSET, &command, sizeof(command)); if (ret < 0) { @@ -72,6 +72,24 @@ static int rmi_f34v7_wait_for_idle(struct f34_data *f34, int timeout_ms) return 0; } +static int rmi_f34v7_check_command_status(struct f34_data *f34, int timeout_ms) +{ + int ret; + + ret = rmi_f34v7_wait_for_idle(f34, timeout_ms); + if (ret < 0) + return ret; + + ret = rmi_f34v7_read_flash_status(f34); + if (ret < 0) + return ret; + + if (f34->v7.flash_status != 0x00) + return -EIO; + + return 0; +} + static int rmi_f34v7_write_command_single_transaction(struct f34_data *f34, u8 cmd) { @@ -122,7 +140,7 @@ static int rmi_f34v7_write_command_single_transaction(struct f34_data *f34, data_1_5.payload[1] = f34->bootloader_id[1]; ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.partition_id, + base + V7_PARTITION_ID_OFFSET, &data_1_5, sizeof(data_1_5)); if (ret < 0) { dev_err(&f34->fn->dev, @@ -195,7 +213,7 @@ static int rmi_f34v7_write_command(struct f34_data *f34, u8 cmd) __func__, command); ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.flash_cmd, + base + V7_COMMAND_OFFSET, &command, sizeof(command)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write flash command\n", @@ -262,7 +280,7 @@ static int rmi_f34v7_write_partition_id(struct f34_data *f34, u8 cmd) } ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.partition_id, + base + V7_PARTITION_ID_OFFSET, &partition, sizeof(partition)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write partition ID\n", @@ -290,7 +308,7 @@ static int rmi_f34v7_read_partition_table(struct f34_data *f34) return ret; ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.block_number, + base + V7_BLOCK_NUMBER_OFFSET, &block_number, sizeof(block_number)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write block number\n", @@ -301,7 +319,7 @@ static int rmi_f34v7_read_partition_table(struct f34_data *f34) put_unaligned_le16(f34->v7.flash_config_length, &length); ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.transfer_length, + base + V7_TRANSFER_LENGTH_OFFSET, &length, sizeof(length)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write transfer length\n", @@ -318,6 +336,10 @@ static int rmi_f34v7_read_partition_table(struct f34_data *f34) return ret; } + /* + * rmi_f34v7_check_command_status() can't be used here, as this + * function is called before IRQs are available + */ timeout = msecs_to_jiffies(F34_WRITE_WAIT_MS); while (time_before(jiffies, timeout)) { usleep_range(5000, 6000); @@ -330,7 +352,7 @@ static int rmi_f34v7_read_partition_table(struct f34_data *f34) } ret = rmi_read_block(f34->fn->rmi_dev, - base + f34->v7.off.payload, + base + V7_PAYLOAD_OFFSET, f34->v7.read_config_buf, f34->v7.partition_table_bytes); if (ret < 0) { @@ -504,13 +526,6 @@ static int rmi_f34v7_read_queries(struct f34_data *f34) rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev, "%s: f34->v7.block_size = %d\n", __func__, f34->v7.block_size); - f34->v7.off.flash_status = V7_FLASH_STATUS_OFFSET; - f34->v7.off.partition_id = V7_PARTITION_ID_OFFSET; - f34->v7.off.block_number = V7_BLOCK_NUMBER_OFFSET; - f34->v7.off.transfer_length = V7_TRANSFER_LENGTH_OFFSET; - f34->v7.off.flash_cmd = V7_COMMAND_OFFSET; - f34->v7.off.payload = V7_PAYLOAD_OFFSET; - f34->v7.has_display_cfg = query_1_7.partition_support[1] & HAS_DISP_CFG; f34->v7.has_guest_code = query_1_7.partition_support[1] & HAS_GUEST_CODE; @@ -571,68 +586,6 @@ static int rmi_f34v7_read_queries(struct f34_data *f34) return 0; } -static int rmi_f34v7_check_ui_firmware_size(struct f34_data *f34) -{ - u16 block_count; - - block_count = f34->v7.img.ui_firmware.size / f34->v7.block_size; - f34->update_size += block_count; - - if (block_count != f34->v7.blkcount.ui_firmware) { - dev_err(&f34->fn->dev, - "UI firmware size mismatch: %d != %d\n", - block_count, f34->v7.blkcount.ui_firmware); - return -EINVAL; - } - - return 0; -} - -static int rmi_f34v7_check_ui_config_size(struct f34_data *f34) -{ - u16 block_count; - - block_count = f34->v7.img.ui_config.size / f34->v7.block_size; - f34->update_size += block_count; - - if (block_count != f34->v7.blkcount.ui_config) { - dev_err(&f34->fn->dev, "UI config size mismatch\n"); - return -EINVAL; - } - - return 0; -} - -static int rmi_f34v7_check_dp_config_size(struct f34_data *f34) -{ - u16 block_count; - - block_count = f34->v7.img.dp_config.size / f34->v7.block_size; - f34->update_size += block_count; - - if (block_count != f34->v7.blkcount.dp_config) { - dev_err(&f34->fn->dev, "Display config size mismatch\n"); - return -EINVAL; - } - - return 0; -} - -static int rmi_f34v7_check_guest_code_size(struct f34_data *f34) -{ - u16 block_count; - - block_count = f34->v7.img.guest_code.size / f34->v7.block_size; - f34->update_size += block_count; - - if (block_count != f34->v7.blkcount.guest_code) { - dev_err(&f34->fn->dev, "Guest code size mismatch\n"); - return -EINVAL; - } - - return 0; -} - static int rmi_f34v7_check_bl_config_size(struct f34_data *f34) { u16 block_count; @@ -648,58 +601,6 @@ static int rmi_f34v7_check_bl_config_size(struct f34_data *f34) return 0; } -static int rmi_f34v7_erase_config(struct f34_data *f34) -{ - int ret; - - dev_info(&f34->fn->dev, "Erasing config...\n"); - - init_completion(&f34->v7.cmd_done); - - switch (f34->v7.config_area) { - case v7_UI_CONFIG_AREA: - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_UI_CONFIG); - if (ret < 0) - return ret; - break; - case v7_DP_CONFIG_AREA: - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_DISP_CONFIG); - if (ret < 0) - return ret; - break; - case v7_BL_CONFIG_AREA: - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_BL_CONFIG); - if (ret < 0) - return ret; - break; - } - - ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS); - if (ret < 0) - return ret; - - return 0; -} - -static int rmi_f34v7_erase_guest_code(struct f34_data *f34) -{ - int ret; - - dev_info(&f34->fn->dev, "Erasing guest code...\n"); - - init_completion(&f34->v7.cmd_done); - - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_GUEST_CODE); - if (ret < 0) - return ret; - - ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS); - if (ret < 0) - return ret; - - return 0; -} - static int rmi_f34v7_erase_all(struct f34_data *f34) { int ret; @@ -708,32 +609,14 @@ static int rmi_f34v7_erase_all(struct f34_data *f34) init_completion(&f34->v7.cmd_done); - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_UI_FIRMWARE); - if (ret < 0) - return ret; - - ret = rmi_f34v7_wait_for_idle(f34, F34_ERASE_WAIT_MS); + ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_ALL); if (ret < 0) return ret; - f34->v7.config_area = v7_UI_CONFIG_AREA; - ret = rmi_f34v7_erase_config(f34); + ret = rmi_f34v7_check_command_status(f34, F34_ERASE_WAIT_MS); if (ret < 0) return ret; - if (f34->v7.has_display_cfg) { - f34->v7.config_area = v7_DP_CONFIG_AREA; - ret = rmi_f34v7_erase_config(f34); - if (ret < 0) - return ret; - } - - if (f34->v7.new_partition_table && f34->v7.has_guest_code) { - ret = rmi_f34v7_erase_guest_code(f34); - if (ret < 0) - return ret; - } - return 0; } @@ -756,7 +639,7 @@ static int rmi_f34v7_read_blocks(struct f34_data *f34, return ret; ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.block_number, + base + V7_BLOCK_NUMBER_OFFSET, &block_number, sizeof(block_number)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write block number\n", @@ -772,7 +655,7 @@ static int rmi_f34v7_read_blocks(struct f34_data *f34, put_unaligned_le16(transfer, &length); ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.transfer_length, + base + V7_TRANSFER_LENGTH_OFFSET, &length, sizeof(length)); if (ret < 0) { dev_err(&f34->fn->dev, @@ -787,12 +670,12 @@ static int rmi_f34v7_read_blocks(struct f34_data *f34, if (ret < 0) return ret; - ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS); + ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS); if (ret < 0) return ret; ret = rmi_read_block(f34->fn->rmi_dev, - base + f34->v7.off.payload, + base + V7_PAYLOAD_OFFSET, &f34->v7.read_config_buf[index], transfer * f34->v7.block_size); if (ret < 0) { @@ -828,7 +711,7 @@ static int rmi_f34v7_write_f34v7_blocks(struct f34_data *f34, return ret; ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.block_number, + base + V7_BLOCK_NUMBER_OFFSET, &block_number, sizeof(block_number)); if (ret < 0) { dev_err(&f34->fn->dev, "%s: Failed to write block number\n", @@ -848,7 +731,7 @@ static int rmi_f34v7_write_f34v7_blocks(struct f34_data *f34, init_completion(&f34->v7.cmd_done); ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.transfer_length, + base + V7_TRANSFER_LENGTH_OFFSET, &length, sizeof(length)); if (ret < 0) { dev_err(&f34->fn->dev, @@ -862,7 +745,7 @@ static int rmi_f34v7_write_f34v7_blocks(struct f34_data *f34, return ret; ret = rmi_write_block(f34->fn->rmi_dev, - base + f34->v7.off.payload, + base + V7_PAYLOAD_OFFSET, block_ptr, transfer * f34->v7.block_size); if (ret < 0) { dev_err(&f34->fn->dev, @@ -871,7 +754,7 @@ static int rmi_f34v7_write_f34v7_blocks(struct f34_data *f34, return ret; } - ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS); + ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS); if (ret < 0) return ret; @@ -937,17 +820,6 @@ static int rmi_f34v7_write_flash_config(struct f34_data *f34) init_completion(&f34->v7.cmd_done); - ret = rmi_f34v7_write_command(f34, v7_CMD_ERASE_FLASH_CONFIG); - if (ret < 0) - return ret; - - rmi_dbg(RMI_DEBUG_FN, &f34->fn->dev, - "%s: Erase flash config command written\n", __func__); - - ret = rmi_f34v7_wait_for_idle(f34, F34_WRITE_WAIT_MS); - if (ret < 0) - return ret; - ret = rmi_f34v7_write_config(f34); if (ret < 0) return ret; @@ -977,10 +849,6 @@ static int rmi_f34v7_write_partition_table(struct f34_data *f34) if (ret < 0) return ret; - ret = rmi_f34v7_erase_config(f34); - if (ret < 0) - return ret; - ret = rmi_f34v7_write_flash_config(f34); if (ret < 0) return ret; @@ -1007,33 +875,6 @@ static int rmi_f34v7_write_firmware(struct f34_data *f34) blk_count, v7_CMD_WRITE_FW); } -static void rmi_f34v7_compare_partition_tables(struct f34_data *f34) -{ - if (f34->v7.phyaddr.ui_firmware != f34->v7.img.phyaddr.ui_firmware) { - f34->v7.new_partition_table = true; - return; - } - - if (f34->v7.phyaddr.ui_config != f34->v7.img.phyaddr.ui_config) { - f34->v7.new_partition_table = true; - return; - } - - if (f34->v7.has_display_cfg && - f34->v7.phyaddr.dp_config != f34->v7.img.phyaddr.dp_config) { - f34->v7.new_partition_table = true; - return; - } - - if (f34->v7.has_guest_code && - f34->v7.phyaddr.guest_code != f34->v7.img.phyaddr.guest_code) { - f34->v7.new_partition_table = true; - return; - } - - f34->v7.new_partition_table = false; -} - static void rmi_f34v7_parse_img_header_10_bl_container(struct f34_data *f34, const void *image) { @@ -1180,8 +1021,6 @@ static int rmi_f34v7_parse_image_info(struct f34_data *f34) rmi_f34v7_parse_partition_table(f34, f34->v7.img.fl_config.data, &f34->v7.img.blkcount, &f34->v7.img.phyaddr); - rmi_f34v7_compare_partition_tables(f34); - return 0; } @@ -1200,53 +1039,35 @@ int rmi_f34v7_do_reflash(struct f34_data *f34, const struct firmware *fw) ret = rmi_f34v7_parse_image_info(f34); if (ret < 0) - goto fail; - - if (!f34->v7.new_partition_table) { - ret = rmi_f34v7_check_ui_firmware_size(f34); - if (ret < 0) - goto fail; - - ret = rmi_f34v7_check_ui_config_size(f34); - if (ret < 0) - goto fail; - - if (f34->v7.has_display_cfg && - f34->v7.img.contains_display_cfg) { - ret = rmi_f34v7_check_dp_config_size(f34); - if (ret < 0) - goto fail; - } + return ret; - if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) { - ret = rmi_f34v7_check_guest_code_size(f34); - if (ret < 0) - goto fail; - } - } else { - ret = rmi_f34v7_check_bl_config_size(f34); - if (ret < 0) - goto fail; - } + ret = rmi_f34v7_check_bl_config_size(f34); + if (ret < 0) + return ret; ret = rmi_f34v7_erase_all(f34); if (ret < 0) - goto fail; + return ret; - if (f34->v7.new_partition_table) { - ret = rmi_f34v7_write_partition_table(f34); - if (ret < 0) - goto fail; - dev_info(&f34->fn->dev, "%s: Partition table programmed\n", - __func__); - } + ret = rmi_f34v7_write_partition_table(f34); + if (ret < 0) + return ret; + dev_info(&f34->fn->dev, "%s: Partition table programmed\n", __func__); + + /* + * Reset to reload partition table - as the previous firmware has been + * erased, we remain in bootloader mode. + */ + ret = rmi_scan_pdt(f34->fn->rmi_dev, NULL, rmi_initial_reset); + if (ret < 0) + dev_warn(&f34->fn->dev, "RMI reset failed!\n"); dev_info(&f34->fn->dev, "Writing firmware (%d bytes)...\n", f34->v7.img.ui_firmware.size); ret = rmi_f34v7_write_firmware(f34); if (ret < 0) - goto fail; + return ret; dev_info(&f34->fn->dev, "Writing config (%d bytes)...\n", f34->v7.img.ui_config.size); @@ -1254,28 +1075,25 @@ int rmi_f34v7_do_reflash(struct f34_data *f34, const struct firmware *fw) f34->v7.config_area = v7_UI_CONFIG_AREA; ret = rmi_f34v7_write_ui_config(f34); if (ret < 0) - goto fail; + return ret; if (f34->v7.has_display_cfg && f34->v7.img.contains_display_cfg) { dev_info(&f34->fn->dev, "Writing display config...\n"); ret = rmi_f34v7_write_dp_config(f34); if (ret < 0) - goto fail; + return ret; } - if (f34->v7.new_partition_table) { - if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) { - dev_info(&f34->fn->dev, "Writing guest code...\n"); + if (f34->v7.has_guest_code && f34->v7.img.contains_guest_code) { + dev_info(&f34->fn->dev, "Writing guest code...\n"); - ret = rmi_f34v7_write_guest_code(f34); - if (ret < 0) - goto fail; - } + ret = rmi_f34v7_write_guest_code(f34); + if (ret < 0) + return ret; } -fail: - return ret; + return 0; } static int rmi_f34v7_enter_flash_prog(struct f34_data *f34) @@ -1288,8 +1106,11 @@ static int rmi_f34v7_enter_flash_prog(struct f34_data *f34) if (ret < 0) return ret; - if (f34->v7.in_bl_mode) + if (f34->v7.in_bl_mode) { + dev_info(&f34->fn->dev, "%s: Device in bootloader mode\n", + __func__); return 0; + } init_completion(&f34->v7.cmd_done); @@ -1297,7 +1118,7 @@ static int rmi_f34v7_enter_flash_prog(struct f34_data *f34) if (ret < 0) return ret; - ret = rmi_f34v7_wait_for_idle(f34, F34_ENABLE_WAIT_MS); + ret = rmi_f34v7_check_command_status(f34, F34_ENABLE_WAIT_MS); if (ret < 0) return ret; @@ -1308,39 +1129,16 @@ int rmi_f34v7_start_reflash(struct f34_data *f34, const struct firmware *fw) { int ret = 0; - f34->fn->rmi_dev->driver->set_irq_bits(f34->fn->rmi_dev, f34->fn->irq_mask); - f34->v7.config_area = v7_UI_CONFIG_AREA; f34->v7.image = fw->data; ret = rmi_f34v7_parse_image_info(f34); if (ret < 0) - goto exit; - - if (!f34->v7.force_update && f34->v7.new_partition_table) { - dev_err(&f34->fn->dev, "%s: Partition table mismatch\n", - __func__); - ret = -EINVAL; - goto exit; - } + return ret; dev_info(&f34->fn->dev, "Firmware image OK\n"); - ret = rmi_f34v7_read_flash_status(f34); - if (ret < 0) - goto exit; - - if (f34->v7.in_bl_mode) { - dev_info(&f34->fn->dev, "%s: Device in bootloader mode\n", - __func__); - } - - rmi_f34v7_enter_flash_prog(f34); - - return 0; - -exit: - return ret; + return rmi_f34v7_enter_flash_prog(f34); } int rmi_f34v7_probe(struct f34_data *f34) @@ -1384,6 +1182,5 @@ int rmi_f34v7_probe(struct f34_data *f34) if (ret < 0) return ret; - f34->v7.force_update = true; return 0; } diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c index c5ce907535ef998ba5616e5beee0c11fdf8a87de..5c3da910b5b2ce1201e5e05626786f9dc750e4f0 100644 --- a/drivers/input/rmi4/rmi_f54.c +++ b/drivers/input/rmi4/rmi_f54.c @@ -390,8 +390,8 @@ static int rmi_f54_vidioc_querycap(struct file *file, void *priv, { struct f54_data *f54 = video_drvdata(file); - strlcpy(cap->driver, F54_NAME, sizeof(cap->driver)); - strlcpy(cap->card, SYNAPTICS_INPUT_DEVICE_NAME, sizeof(cap->card)); + strscpy(cap->driver, F54_NAME, sizeof(cap->driver)); + strscpy(cap->card, SYNAPTICS_INPUT_DEVICE_NAME, sizeof(cap->card)); snprintf(cap->bus_info, sizeof(cap->bus_info), "rmi4:%s", dev_name(&f54->fn->dev)); @@ -410,7 +410,7 @@ static int rmi_f54_vidioc_enum_input(struct file *file, void *priv, i->type = V4L2_INPUT_TYPE_TOUCH; - strlcpy(i->name, rmi_f54_report_type_names[reptype], sizeof(i->name)); + strscpy(i->name, rmi_f54_report_type_names[reptype], sizeof(i->name)); return 0; } @@ -696,7 +696,7 @@ static int rmi_f54_probe(struct rmi_function *fn) rmi_f54_set_input(f54, 0); /* register video device */ - strlcpy(f54->v4l2.name, F54_NAME, sizeof(f54->v4l2.name)); + strscpy(f54->v4l2.name, F54_NAME, sizeof(f54->v4l2.name)); ret = v4l2_device_register(&fn->dev, &f54->v4l2); if (ret) { dev_err(&fn->dev, "Unable to register video dev.\n"); diff --git a/drivers/input/rmi4/rmi_smbus.c b/drivers/input/rmi4/rmi_smbus.c index 2407ea43de59b7b5f386d0361c3cc49495df62d3..c130468541b7d8ec9394ae35435d2799fab08209 100644 --- a/drivers/input/rmi4/rmi_smbus.c +++ b/drivers/input/rmi4/rmi_smbus.c @@ -338,13 +338,11 @@ static int rmi_smb_probe(struct i2c_client *client, return 0; } -static int rmi_smb_remove(struct i2c_client *client) +static void rmi_smb_remove(struct i2c_client *client) { struct rmi_smb_xport *rmi_smb = i2c_get_clientdata(client); rmi_unregister_transport_device(&rmi_smb->xport); - - return 0; } static int __maybe_unused rmi_smb_suspend(struct device *dev) diff --git a/drivers/input/serio/altera_ps2.c b/drivers/input/serio/altera_ps2.c index 379e9240c2b334798bba36ccc8d1bd3253c06159..3a92304f64fb3f6394f8af0ceedd210c7bddadfb 100644 --- a/drivers/input/serio/altera_ps2.c +++ b/drivers/input/serio/altera_ps2.c @@ -110,8 +110,8 @@ static int altera_ps2_probe(struct platform_device *pdev) serio->write = altera_ps2_write; serio->open = altera_ps2_open; serio->close = altera_ps2_close; - strlcpy(serio->name, dev_name(&pdev->dev), sizeof(serio->name)); - strlcpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys)); + strscpy(serio->name, dev_name(&pdev->dev), sizeof(serio->name)); + strscpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys)); serio->port_data = ps2if; serio->dev.parent = &pdev->dev; ps2if->io = serio; diff --git a/drivers/input/serio/ambakmi.c b/drivers/input/serio/ambakmi.c index 4408245b61d2c6707bf136fc0d82a7e11795916a..c391700fc4ae2c694ff7aa3cf579b005f3b88004 100644 --- a/drivers/input/serio/ambakmi.c +++ b/drivers/input/serio/ambakmi.c @@ -126,8 +126,8 @@ static int amba_kmi_probe(struct amba_device *dev, io->write = amba_kmi_write; io->open = amba_kmi_open; io->close = amba_kmi_close; - strlcpy(io->name, dev_name(&dev->dev), sizeof(io->name)); - strlcpy(io->phys, dev_name(&dev->dev), sizeof(io->phys)); + strscpy(io->name, dev_name(&dev->dev), sizeof(io->name)); + strscpy(io->phys, dev_name(&dev->dev), sizeof(io->phys)); io->port_data = kmi; io->dev.parent = &dev->dev; diff --git a/drivers/input/serio/ams_delta_serio.c b/drivers/input/serio/ams_delta_serio.c index 1c0be299f17904b2cc23ebd18b7068b0854b2c9a..ec93cb4573c3e1a88ba95bdfe81b5c95ea6845c6 100644 --- a/drivers/input/serio/ams_delta_serio.c +++ b/drivers/input/serio/ams_delta_serio.c @@ -159,8 +159,8 @@ static int ams_delta_serio_init(struct platform_device *pdev) serio->id.type = SERIO_8042; serio->open = ams_delta_serio_open; serio->close = ams_delta_serio_close; - strlcpy(serio->name, "AMS DELTA keyboard adapter", sizeof(serio->name)); - strlcpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys)); + strscpy(serio->name, "AMS DELTA keyboard adapter", sizeof(serio->name)); + strscpy(serio->phys, dev_name(&pdev->dev), sizeof(serio->phys)); serio->dev.parent = &pdev->dev; serio->port_data = priv; diff --git a/drivers/input/serio/apbps2.c b/drivers/input/serio/apbps2.c index 974d7bfae0a03a6581b82bed4b55b67a7298962e..9c9ce097f8bf187aa37b12226d9a5c5de2869d14 100644 --- a/drivers/input/serio/apbps2.c +++ b/drivers/input/serio/apbps2.c @@ -176,7 +176,7 @@ static int apbps2_of_probe(struct platform_device *ofdev) priv->io->close = apbps2_close; priv->io->write = apbps2_write; priv->io->port_data = priv; - strlcpy(priv->io->name, "APBPS2 PS/2", sizeof(priv->io->name)); + strscpy(priv->io->name, "APBPS2 PS/2", sizeof(priv->io->name)); snprintf(priv->io->phys, sizeof(priv->io->phys), "apbps2_%d", apbps2_idx++); diff --git a/drivers/input/serio/ct82c710.c b/drivers/input/serio/ct82c710.c index d45009d654bf7e48b258dab28b0e9d02fc2a78ec..3da751f4a6bf675ee286554a35edcbe730c2ec91 100644 --- a/drivers/input/serio/ct82c710.c +++ b/drivers/input/serio/ct82c710.c @@ -7,9 +7,6 @@ * 82C710 C&T mouse port chip driver for Linux */ -/* - */ - #include #include #include @@ -170,7 +167,7 @@ static int ct82c710_probe(struct platform_device *dev) ct82c710_port->open = ct82c710_open; ct82c710_port->close = ct82c710_close; ct82c710_port->write = ct82c710_write; - strlcpy(ct82c710_port->name, "C&T 82c710 mouse port", + strscpy(ct82c710_port->name, "C&T 82c710 mouse port", sizeof(ct82c710_port->name)); snprintf(ct82c710_port->phys, sizeof(ct82c710_port->phys), "isa%16llx/serio0", (unsigned long long)CT82C710_DATA); diff --git a/drivers/input/serio/gscps2.c b/drivers/input/serio/gscps2.c index da2c67cb864225a3b917da1992a7a54a07c784e7..633c7de49d671547e16e257667110e71a1829a67 100644 --- a/drivers/input/serio/gscps2.c +++ b/drivers/input/serio/gscps2.c @@ -361,7 +361,7 @@ static int __init gscps2_probe(struct parisc_device *dev) snprintf(serio->name, sizeof(serio->name), "gsc-ps2-%s", (ps2port->id == GSC_ID_KEYBOARD) ? "keyboard" : "mouse"); - strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); + strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); serio->id.type = SERIO_8042; serio->write = gscps2_write; serio->open = gscps2_open; diff --git a/drivers/input/serio/hyperv-keyboard.c b/drivers/input/serio/hyperv-keyboard.c index 1a7b72a9016d7418818e61f8ebad50a8a718a52a..d62aefb2e24515501fb8a5ac1de751ed671d58f9 100644 --- a/drivers/input/serio/hyperv-keyboard.c +++ b/drivers/input/serio/hyperv-keyboard.c @@ -334,9 +334,9 @@ static int hv_kbd_probe(struct hv_device *hv_dev, hv_serio->dev.parent = &hv_dev->device; hv_serio->id.type = SERIO_8042_XL; hv_serio->port_data = kbd_dev; - strlcpy(hv_serio->name, dev_name(&hv_dev->device), + strscpy(hv_serio->name, dev_name(&hv_dev->device), sizeof(hv_serio->name)); - strlcpy(hv_serio->phys, dev_name(&hv_dev->device), + strscpy(hv_serio->phys, dev_name(&hv_dev->device), sizeof(hv_serio->phys)); hv_serio->start = hv_kbd_start; diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-acpipnpio.h similarity index 99% rename from drivers/input/serio/i8042-x86ia64io.h rename to drivers/input/serio/i8042-acpipnpio.h index 4fbec7bbeccaaa1e7cde926d6bbef3c233550443..0778dc03cd9e08b8bb3e24c0d1b4f651a29adf46 100644 --- a/drivers/input/serio/i8042-x86ia64io.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -1,7 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _I8042_X86IA64IO_H -#define _I8042_X86IA64IO_H +#ifndef _I8042_ACPIPNPIO_H +#define _I8042_ACPIPNPIO_H +#include #ifdef CONFIG_X86 #include @@ -1300,7 +1301,7 @@ static char i8042_pnp_aux_name[32]; static void i8042_pnp_id_to_string(struct pnp_id *id, char *dst, int dst_size) { - strlcpy(dst, "PNP:", dst_size); + strscpy(dst, "PNP:", dst_size); while (id) { strlcat(dst, " ", dst_size); @@ -1320,7 +1321,7 @@ static int i8042_pnp_kbd_probe(struct pnp_dev *dev, const struct pnp_device_id * if (pnp_irq_valid(dev,0)) i8042_pnp_kbd_irq = pnp_irq(dev, 0); - strlcpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name)); + strscpy(i8042_pnp_kbd_name, did->id, sizeof(i8042_pnp_kbd_name)); if (strlen(pnp_dev_name(dev))) { strlcat(i8042_pnp_kbd_name, ":", sizeof(i8042_pnp_kbd_name)); strlcat(i8042_pnp_kbd_name, pnp_dev_name(dev), sizeof(i8042_pnp_kbd_name)); @@ -1347,7 +1348,7 @@ static int i8042_pnp_aux_probe(struct pnp_dev *dev, const struct pnp_device_id * if (pnp_irq_valid(dev, 0)) i8042_pnp_aux_irq = pnp_irq(dev, 0); - strlcpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name)); + strscpy(i8042_pnp_aux_name, did->id, sizeof(i8042_pnp_aux_name)); if (strlen(pnp_dev_name(dev))) { strlcat(i8042_pnp_aux_name, ":", sizeof(i8042_pnp_aux_name)); strlcat(i8042_pnp_aux_name, pnp_dev_name(dev), sizeof(i8042_pnp_aux_name)); @@ -1453,9 +1454,14 @@ static int __init i8042_pnp_init(void) return -ENODEV; #else pr_info("PNP: No PS/2 controller found.\n"); +#if defined(__loongarch__) + if (acpi_disabled == 0) + return -ENODEV; +#else if (x86_platform.legacy.i8042 != X86_LEGACY_I8042_EXPECTED_PRESENT) return -ENODEV; +#endif pr_info("Probing ports directly.\n"); return 0; #endif @@ -1665,4 +1671,4 @@ static inline void i8042_platform_exit(void) i8042_pnp_exit(); } -#endif /* _I8042_X86IA64IO_H */ +#endif /* _I8042_ACPIPNPIO_H */ diff --git a/drivers/input/serio/i8042-sparcio.h b/drivers/input/serio/i8042-sparcio.h index fce76812843bb47a0083fdd9a5c196c3b3bc32a1..c712c1fe060533045d5f10bf80f5f6bc00636632 100644 --- a/drivers/input/serio/i8042-sparcio.h +++ b/drivers/input/serio/i8042-sparcio.h @@ -3,6 +3,7 @@ #define _I8042_SPARCIO_H #include +#include #include #include @@ -103,12 +104,25 @@ static struct platform_driver sparc_i8042_driver = { .remove = sparc_i8042_remove, }; -static int __init i8042_platform_init(void) +static bool i8042_is_mr_coffee(void) { - struct device_node *root = of_find_node_by_path("/"); - const char *name = of_get_property(root, "name", NULL); + struct device_node *root; + const char *name; + bool is_mr_coffee; + + root = of_find_node_by_path("/"); + + name = of_get_property(root, "name", NULL); + is_mr_coffee = name && !strcmp(name, "SUNW,JavaStation-1"); - if (name && !strcmp(name, "SUNW,JavaStation-1")) { + of_node_put(root); + + return is_mr_coffee; +} + +static int __init i8042_platform_init(void) +{ + if (i8042_is_mr_coffee()) { /* Hardcoded values for MrCoffee. */ i8042_kbd_irq = i8042_aux_irq = 13 | 0x20; kbd_iobase = ioremap(0x71300060, 8); @@ -136,10 +150,7 @@ static int __init i8042_platform_init(void) static inline void i8042_platform_exit(void) { - struct device_node *root = of_find_node_by_path("/"); - const char *name = of_get_property(root, "name", NULL); - - if (!name || strcmp(name, "SUNW,JavaStation-1")) + if (!i8042_is_mr_coffee()) platform_driver_unregister(&sparc_i8042_driver); } diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c index 3fc0a89cc785cb6b76bcaf49be8b573fb1682c8d..f9486495baefaa22c1db1d9cf859c75b2ac97384 100644 --- a/drivers/input/serio/i8042.c +++ b/drivers/input/serio/i8042.c @@ -1341,9 +1341,9 @@ static int i8042_create_kbd_port(void) serio->ps2_cmd_mutex = &i8042_mutex; serio->port_data = port; serio->dev.parent = &i8042_platform_device->dev; - strlcpy(serio->name, "i8042 KBD port", sizeof(serio->name)); - strlcpy(serio->phys, I8042_KBD_PHYS_DESC, sizeof(serio->phys)); - strlcpy(serio->firmware_id, i8042_kbd_firmware_id, + strscpy(serio->name, "i8042 KBD port", sizeof(serio->name)); + strscpy(serio->phys, I8042_KBD_PHYS_DESC, sizeof(serio->phys)); + strscpy(serio->firmware_id, i8042_kbd_firmware_id, sizeof(serio->firmware_id)); set_primary_fwnode(&serio->dev, i8042_kbd_fwnode); @@ -1371,15 +1371,15 @@ static int i8042_create_aux_port(int idx) serio->port_data = port; serio->dev.parent = &i8042_platform_device->dev; if (idx < 0) { - strlcpy(serio->name, "i8042 AUX port", sizeof(serio->name)); - strlcpy(serio->phys, I8042_AUX_PHYS_DESC, sizeof(serio->phys)); - strlcpy(serio->firmware_id, i8042_aux_firmware_id, + strscpy(serio->name, "i8042 AUX port", sizeof(serio->name)); + strscpy(serio->phys, I8042_AUX_PHYS_DESC, sizeof(serio->phys)); + strscpy(serio->firmware_id, i8042_aux_firmware_id, sizeof(serio->firmware_id)); serio->close = i8042_port_close; } else { snprintf(serio->name, sizeof(serio->name), "i8042 AUX%d port", idx); snprintf(serio->phys, sizeof(serio->phys), I8042_MUX_PHYS_DESC, idx + 1); - strlcpy(serio->firmware_id, i8042_aux_firmware_id, + strscpy(serio->firmware_id, i8042_aux_firmware_id, sizeof(serio->firmware_id)); } diff --git a/drivers/input/serio/i8042.h b/drivers/input/serio/i8042.h index 55381783dc82ddc4ada059d6ce6c7219e46c240d..adb5173372d3ee6d83d2164f0c6bd434cba9975f 100644 --- a/drivers/input/serio/i8042.h +++ b/drivers/input/serio/i8042.h @@ -19,8 +19,8 @@ #include "i8042-snirm.h" #elif defined(CONFIG_SPARC) #include "i8042-sparcio.h" -#elif defined(CONFIG_X86) || defined(CONFIG_IA64) -#include "i8042-x86ia64io.h" +#elif defined(CONFIG_X86) || defined(CONFIG_IA64) || defined(CONFIG_LOONGARCH) +#include "i8042-acpipnpio.h" #else #include "i8042-io.h" #endif diff --git a/drivers/input/serio/libps2.c b/drivers/input/serio/libps2.c index 250e213cc80c68bed22abaa9cbd8ddfbecc8af22..3e19344eda93ce915bf81c88f685cf8837614e92 100644 --- a/drivers/input/serio/libps2.c +++ b/drivers/input/serio/libps2.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -294,9 +295,11 @@ int __ps2_command(struct ps2dev *ps2dev, u8 *param, unsigned int command) serio_pause_rx(ps2dev->serio); - if (param) + if (param) { for (i = 0; i < receive; i++) param[i] = ps2dev->cmdbuf[(receive - 1) - i]; + kmsan_unpoison_memory(param, receive); + } if (ps2dev->cmdcnt && (command != PS2_CMD_RESET_BAT || ps2dev->cmdcnt != 1)) { diff --git a/drivers/input/serio/olpc_apsp.c b/drivers/input/serio/olpc_apsp.c index 59de8d9b6710e3400405a839d707e1859d241f98..04d2db982fb808c7b454d161187fd17cff4d6f7e 100644 --- a/drivers/input/serio/olpc_apsp.c +++ b/drivers/input/serio/olpc_apsp.c @@ -199,8 +199,8 @@ static int olpc_apsp_probe(struct platform_device *pdev) kb_serio->close = olpc_apsp_close; kb_serio->port_data = priv; kb_serio->dev.parent = &pdev->dev; - strlcpy(kb_serio->name, "sp keyboard", sizeof(kb_serio->name)); - strlcpy(kb_serio->phys, "sp/serio0", sizeof(kb_serio->phys)); + strscpy(kb_serio->name, "sp keyboard", sizeof(kb_serio->name)); + strscpy(kb_serio->phys, "sp/serio0", sizeof(kb_serio->phys)); priv->kbio = kb_serio; serio_register_port(kb_serio); @@ -216,8 +216,8 @@ static int olpc_apsp_probe(struct platform_device *pdev) pad_serio->close = olpc_apsp_close; pad_serio->port_data = priv; pad_serio->dev.parent = &pdev->dev; - strlcpy(pad_serio->name, "sp touchpad", sizeof(pad_serio->name)); - strlcpy(pad_serio->phys, "sp/serio1", sizeof(pad_serio->phys)); + strscpy(pad_serio->name, "sp touchpad", sizeof(pad_serio->name)); + strscpy(pad_serio->phys, "sp/serio1", sizeof(pad_serio->phys)); priv->padio = pad_serio; serio_register_port(pad_serio); diff --git a/drivers/input/serio/parkbd.c b/drivers/input/serio/parkbd.c index 51b68501896c5782bac9dab312c47c690b6dce5d..0d54895428f5d554cf3983065f68469b959bb4ed 100644 --- a/drivers/input/serio/parkbd.c +++ b/drivers/input/serio/parkbd.c @@ -169,7 +169,7 @@ static struct serio *parkbd_allocate_serio(void) if (serio) { serio->id.type = parkbd_mode; serio->write = parkbd_write; - strlcpy(serio->name, "PARKBD AT/XT keyboard adapter", sizeof(serio->name)); + strscpy(serio->name, "PARKBD AT/XT keyboard adapter", sizeof(serio->name)); snprintf(serio->phys, sizeof(serio->phys), "%s/serio0", parkbd_dev->port->name); } diff --git a/drivers/input/serio/pcips2.c b/drivers/input/serio/pcips2.c index bedf75de0a2cec8a0c7ad4243fbfb0aac3407904..05878750f2c2d22eaeb59843fd13b109dafe3b15 100644 --- a/drivers/input/serio/pcips2.c +++ b/drivers/input/serio/pcips2.c @@ -149,8 +149,8 @@ static int pcips2_probe(struct pci_dev *dev, const struct pci_device_id *id) serio->write = pcips2_write; serio->open = pcips2_open; serio->close = pcips2_close; - strlcpy(serio->name, pci_name(dev), sizeof(serio->name)); - strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); + strscpy(serio->name, pci_name(dev), sizeof(serio->name)); + strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); serio->port_data = ps2if; serio->dev.parent = &dev->dev; ps2if->io = serio; diff --git a/drivers/input/serio/ps2-gpio.c b/drivers/input/serio/ps2-gpio.c index 9b02dd5dd2b99895c95db0739f56493aa1cfb397..bc1dc484389b40871476964d409652f660be4888 100644 --- a/drivers/input/serio/ps2-gpio.c +++ b/drivers/input/serio/ps2-gpio.c @@ -449,8 +449,8 @@ static int ps2_gpio_probe(struct platform_device *pdev) serio->write = drvdata->write_enable ? ps2_gpio_write : NULL; serio->port_data = drvdata; serio->dev.parent = dev; - strlcpy(serio->name, dev_name(dev), sizeof(serio->name)); - strlcpy(serio->phys, dev_name(dev), sizeof(serio->phys)); + strscpy(serio->name, dev_name(dev), sizeof(serio->name)); + strscpy(serio->phys, dev_name(dev), sizeof(serio->phys)); drvdata->serio = serio; drvdata->dev = dev; diff --git a/drivers/input/serio/ps2mult.c b/drivers/input/serio/ps2mult.c index 0071dd5ebcc27cca975a9c37ee48309041f598f2..902e81826fbfeab1094604172fe177858daec30e 100644 --- a/drivers/input/serio/ps2mult.c +++ b/drivers/input/serio/ps2mult.c @@ -131,7 +131,7 @@ static int ps2mult_create_port(struct ps2mult *psm, int i) if (!serio) return -ENOMEM; - strlcpy(serio->name, "TQC PS/2 Multiplexer", sizeof(serio->name)); + strscpy(serio->name, "TQC PS/2 Multiplexer", sizeof(serio->name)); snprintf(serio->phys, sizeof(serio->phys), "%s/port%d", mx_serio->phys, i); serio->id.type = SERIO_8042; diff --git a/drivers/input/serio/q40kbd.c b/drivers/input/serio/q40kbd.c index bd248398556a8c5dceaa1787c02e81dfaf7ec82d..ba04058fc3cbde89260357a6ede06323f9af0ad7 100644 --- a/drivers/input/serio/q40kbd.c +++ b/drivers/input/serio/q40kbd.c @@ -10,9 +10,6 @@ * Q40 PS/2 keyboard controller driver for Linux/m68k */ -/* - */ - #include #include #include @@ -126,8 +123,8 @@ static int q40kbd_probe(struct platform_device *pdev) port->close = q40kbd_close; port->port_data = q40kbd; port->dev.parent = &pdev->dev; - strlcpy(port->name, "Q40 Kbd Port", sizeof(port->name)); - strlcpy(port->phys, "Q40", sizeof(port->phys)); + strscpy(port->name, "Q40 Kbd Port", sizeof(port->name)); + strscpy(port->phys, "Q40", sizeof(port->phys)); q40kbd_stop(); diff --git a/drivers/input/serio/rpckbd.c b/drivers/input/serio/rpckbd.c index 37fe6a5711ea0c48b49af979bbb98c544ff4a793..ce420eb1f51bec22d377e6043318a3c816e48263 100644 --- a/drivers/input/serio/rpckbd.c +++ b/drivers/input/serio/rpckbd.c @@ -8,9 +8,6 @@ * Acorn RiscPC PS/2 keyboard controller driver for Linux/ARM */ -/* - */ - #include #include #include @@ -128,8 +125,8 @@ static int rpckbd_probe(struct platform_device *dev) serio->close = rpckbd_close; serio->dev.parent = &dev->dev; serio->port_data = rpckbd; - strlcpy(serio->name, "RiscPC PS/2 kbd port", sizeof(serio->name)); - strlcpy(serio->phys, "rpckbd/serio0", sizeof(serio->phys)); + strscpy(serio->name, "RiscPC PS/2 kbd port", sizeof(serio->name)); + strscpy(serio->phys, "rpckbd/serio0", sizeof(serio->phys)); platform_set_drvdata(dev, serio); serio_register_port(serio); diff --git a/drivers/input/serio/sa1111ps2.c b/drivers/input/serio/sa1111ps2.c index 68fac4801e2e25a031a75e934286214e39fa7771..2724c3aa512cebdbe766633c7639d270ccbb1be3 100644 --- a/drivers/input/serio/sa1111ps2.c +++ b/drivers/input/serio/sa1111ps2.c @@ -267,8 +267,8 @@ static int ps2_probe(struct sa1111_dev *dev) serio->write = ps2_write; serio->open = ps2_open; serio->close = ps2_close; - strlcpy(serio->name, dev_name(&dev->dev), sizeof(serio->name)); - strlcpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); + strscpy(serio->name, dev_name(&dev->dev), sizeof(serio->name)); + strscpy(serio->phys, dev_name(&dev->dev), sizeof(serio->phys)); serio->port_data = ps2if; serio->dev.parent = &dev->dev; ps2if->io = serio; diff --git a/drivers/input/serio/serio.c b/drivers/input/serio/serio.c index ec117be3d8d8349baf0a6ef5913b7e01949e354a..15ce3202322f04e67558dce4ef93ec3fb7e5367f 100644 --- a/drivers/input/serio/serio.c +++ b/drivers/input/serio/serio.c @@ -7,9 +7,6 @@ * Copyright (c) 2003 Daniele Bellucci */ -/* - */ - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include diff --git a/drivers/input/serio/serport.c b/drivers/input/serio/serport.c index 669a728095b8bba77e1a24e41f740e3d4081e18e..7f7ef0e3a7494a39f0885a0e865633b664d9bf69 100644 --- a/drivers/input/serio/serport.c +++ b/drivers/input/serio/serport.c @@ -171,7 +171,7 @@ static ssize_t serport_ldisc_read(struct tty_struct * tty, struct file * file, if (!serio) return -ENOMEM; - strlcpy(serio->name, "Serial port", sizeof(serio->name)); + strscpy(serio->name, "Serial port", sizeof(serio->name)); snprintf(serio->phys, sizeof(serio->phys), "%s/serio0", tty_name(tty)); serio->id = serport->id; serio->id.type = SERIO_RS232; diff --git a/drivers/input/serio/sun4i-ps2.c b/drivers/input/serio/sun4i-ps2.c index f15ed3dcdb9b2074f29a300497008377cf435d78..eb262640192e989e5d1966dc7f155443cfe62248 100644 --- a/drivers/input/serio/sun4i-ps2.c +++ b/drivers/input/serio/sun4i-ps2.c @@ -256,8 +256,8 @@ static int sun4i_ps2_probe(struct platform_device *pdev) serio->close = sun4i_ps2_close; serio->port_data = drvdata; serio->dev.parent = dev; - strlcpy(serio->name, dev_name(dev), sizeof(serio->name)); - strlcpy(serio->phys, dev_name(dev), sizeof(serio->phys)); + strscpy(serio->name, dev_name(dev), sizeof(serio->name)); + strscpy(serio->phys, dev_name(dev), sizeof(serio->phys)); /* shutoff interrupt */ writel(0, drvdata->reg_base + PS2_REG_GCTL); diff --git a/drivers/input/tablet/acecad.c b/drivers/input/tablet/acecad.c index 56c7e471ac32eda41815070d9fdfc4a5f3c78c6b..b20e5a1afbccafa01be89142271f40af4f23e0ad 100644 --- a/drivers/input/tablet/acecad.c +++ b/drivers/input/tablet/acecad.c @@ -9,9 +9,6 @@ * v3.2 - Added sysfs support */ -/* - */ - #include #include #include @@ -155,7 +152,7 @@ static int usb_acecad_probe(struct usb_interface *intf, const struct usb_device_ acecad->input = input_dev; if (dev->manufacturer) - strlcpy(acecad->name, dev->manufacturer, sizeof(acecad->name)); + strscpy(acecad->name, dev->manufacturer, sizeof(acecad->name)); if (dev->product) { if (dev->manufacturer) diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c index 24ec4844a5c3e67fa078211879143b44bb4340c0..baabc51547b83d6f4c9d6d559e07d6d1e2aaa56f 100644 --- a/drivers/input/tablet/aiptek.c +++ b/drivers/input/tablet/aiptek.c @@ -1617,7 +1617,7 @@ static ssize_t show_firmwareCode(struct device *dev, struct device_attribute *at static DEVICE_ATTR(firmware_code, S_IRUGO, show_firmwareCode, NULL); -static struct attribute *aiptek_attributes[] = { +static struct attribute *aiptek_dev_attrs[] = { &dev_attr_size.attr, &dev_attr_pointer_mode.attr, &dev_attr_coordinate_mode.attr, @@ -1641,9 +1641,7 @@ static struct attribute *aiptek_attributes[] = { NULL }; -static const struct attribute_group aiptek_attribute_group = { - .attrs = aiptek_attributes, -}; +ATTRIBUTE_GROUPS(aiptek_dev); /*********************************************************************** * This routine is called when a tablet has been identified. It basically @@ -1842,26 +1840,16 @@ aiptek_probe(struct usb_interface *intf, const struct usb_device_id *id) */ usb_set_intfdata(intf, aiptek); - /* Set up the sysfs files - */ - err = sysfs_create_group(&intf->dev.kobj, &aiptek_attribute_group); - if (err) { - dev_warn(&intf->dev, "cannot create sysfs group err: %d\n", - err); - goto fail3; - } - /* Register the tablet as an Input Device */ err = input_register_device(aiptek->inputdev); if (err) { dev_warn(&intf->dev, "input_register_device returned err: %d\n", err); - goto fail4; + goto fail3; } return 0; - fail4: sysfs_remove_group(&intf->dev.kobj, &aiptek_attribute_group); fail3: usb_free_urb(aiptek->urb); fail2: usb_free_coherent(usbdev, AIPTEK_PACKET_LENGTH, aiptek->data, aiptek->data_dma); @@ -1886,7 +1874,6 @@ static void aiptek_disconnect(struct usb_interface *intf) */ usb_kill_urb(aiptek->urb); input_unregister_device(aiptek->inputdev); - sysfs_remove_group(&intf->dev.kobj, &aiptek_attribute_group); usb_free_urb(aiptek->urb); usb_free_coherent(interface_to_usbdev(intf), AIPTEK_PACKET_LENGTH, @@ -1900,6 +1887,7 @@ static struct usb_driver aiptek_driver = { .probe = aiptek_probe, .disconnect = aiptek_disconnect, .id_table = aiptek_ids, + .dev_groups = aiptek_dev_groups, }; module_usb_driver(aiptek_driver); diff --git a/drivers/input/tablet/hanwang.c b/drivers/input/tablet/hanwang.c index 6d58443bb3e9863bd40e147b5e8b207fb1b2be86..9bc631518b92ddcc9cd989637c094033e0189702 100644 --- a/drivers/input/tablet/hanwang.c +++ b/drivers/input/tablet/hanwang.c @@ -5,9 +5,6 @@ * Copyright (c) 2010 Xing Wei */ -/* - */ - #include #include #include @@ -356,7 +353,7 @@ static int hanwang_probe(struct usb_interface *intf, const struct usb_device_id usb_make_path(dev, hanwang->phys, sizeof(hanwang->phys)); strlcat(hanwang->phys, "/input0", sizeof(hanwang->phys)); - strlcpy(hanwang->name, hanwang->features->name, sizeof(hanwang->name)); + strscpy(hanwang->name, hanwang->features->name, sizeof(hanwang->name)); input_dev->name = hanwang->name; input_dev->phys = hanwang->phys; usb_to_input_id(dev, &input_dev->id); diff --git a/drivers/input/tablet/pegasus_notetaker.c b/drivers/input/tablet/pegasus_notetaker.c index c608ac505d1ba77c86a21de5b4fc8be29ade2d21..d836d3dcc6a2491905535f0c8fcab35d5ae6b541 100644 --- a/drivers/input/tablet/pegasus_notetaker.c +++ b/drivers/input/tablet/pegasus_notetaker.c @@ -319,7 +319,7 @@ static int pegasus_probe(struct usb_interface *intf, pegasus->irq->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; if (dev->manufacturer) - strlcpy(pegasus->name, dev->manufacturer, + strscpy(pegasus->name, dev->manufacturer, sizeof(pegasus->name)); if (dev->product) { diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig index 2d70c945b20a972848d21601b519a83731a6629a..dc90a3ea51eed1098e6514e7699dfb8a44ef7a96 100644 --- a/drivers/input/touchscreen/Kconfig +++ b/drivers/input/touchscreen/Kconfig @@ -1335,7 +1335,7 @@ config TOUCHSCREEN_ZFORCE config TOUCHSCREEN_COLIBRI_VF50 tristate "Toradex Colibri on board touchscreen driver" - depends on IIO && VF610_ADC + depends on IIO depends on GPIOLIB || COMPILE_TEST help Say Y here if you have a Colibri VF50 and plan to use diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c index eb66cd2689b7c64487910cacc75a97f231e804ef..ccecd1441f0bec1c59a4d1d8567a286df8b6de77 100644 --- a/drivers/input/touchscreen/atmel_mxt_ts.c +++ b/drivers/input/touchscreen/atmel_mxt_ts.c @@ -2497,8 +2497,8 @@ static int mxt_vidioc_querycap(struct file *file, void *priv, { struct mxt_data *data = video_drvdata(file); - strlcpy(cap->driver, "atmel_mxt_ts", sizeof(cap->driver)); - strlcpy(cap->card, "atmel_mxt_ts touch", sizeof(cap->card)); + strscpy(cap->driver, "atmel_mxt_ts", sizeof(cap->driver)); + strscpy(cap->card, "atmel_mxt_ts touch", sizeof(cap->card)); snprintf(cap->bus_info, sizeof(cap->bus_info), "I2C:%s", dev_name(&data->client->dev)); return 0; @@ -2514,11 +2514,11 @@ static int mxt_vidioc_enum_input(struct file *file, void *priv, switch (i->index) { case MXT_V4L_INPUT_REFS: - strlcpy(i->name, "Mutual Capacitance References", + strscpy(i->name, "Mutual Capacitance References", sizeof(i->name)); break; case MXT_V4L_INPUT_DELTAS: - strlcpy(i->name, "Mutual Capacitance Deltas", sizeof(i->name)); + strscpy(i->name, "Mutual Capacitance Deltas", sizeof(i->name)); break; } @@ -3284,7 +3284,7 @@ err_disable_regulators: return error; } -static int mxt_remove(struct i2c_client *client) +static void mxt_remove(struct i2c_client *client) { struct mxt_data *data = i2c_get_clientdata(client); @@ -3294,8 +3294,6 @@ static int mxt_remove(struct i2c_client *client) mxt_free_object_table(data); regulator_bulk_disable(ARRAY_SIZE(data->regulators), data->regulators); - - return 0; } static int __maybe_unused mxt_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/auo-pixcir-ts.c b/drivers/input/touchscreen/auo-pixcir-ts.c index c33e63ca61425bf2524bd1dc107a74c8d6da3f80..2deae5a6823a21c7febaac4d07fa03eb46e1350a 100644 --- a/drivers/input/touchscreen/auo-pixcir-ts.c +++ b/drivers/input/touchscreen/auo-pixcir-ts.c @@ -10,6 +10,7 @@ * Copyright (c) 2008 QUALCOMM USA, INC. */ +#include #include #include #include @@ -19,10 +20,9 @@ #include #include #include -#include -#include +#include #include -#include +#include /* * Coordinate calculation: @@ -69,6 +69,16 @@ #define AUO_PIXCIR_INT_RELEASE (1 << 4) #define AUO_PIXCIR_INT_ENABLE (1 << 3) #define AUO_PIXCIR_INT_POL_HIGH (1 << 2) + +/* + * Interrupt modes: + * periodical: interrupt is asserted periodicaly + * compare coordinates: interrupt is asserted when coordinates change + * indicate touch: interrupt is asserted during touch + */ +#define AUO_PIXCIR_INT_PERIODICAL 0x00 +#define AUO_PIXCIR_INT_COMP_COORD 0x01 +#define AUO_PIXCIR_INT_TOUCH_IND 0x02 #define AUO_PIXCIR_INT_MODE_MASK 0x03 /* @@ -103,10 +113,14 @@ struct auo_pixcir_ts { struct i2c_client *client; struct input_dev *input; - const struct auo_pixcir_ts_platdata *pdata; + struct gpio_desc *gpio_int; + struct gpio_desc *gpio_rst; char phys[32]; - /* special handling for touch_indicate interupt mode */ + unsigned int x_max; + unsigned int y_max; + + /* special handling for touch_indicate interrupt mode */ bool touch_ind_mode; wait_queue_head_t wait; @@ -125,7 +139,6 @@ static int auo_pixcir_collect_data(struct auo_pixcir_ts *ts, struct auo_point_t *point) { struct i2c_client *client = ts->client; - const struct auo_pixcir_ts_platdata *pdata = ts->pdata; uint8_t raw_coord[8]; uint8_t raw_area[4]; int i, ret; @@ -152,8 +165,8 @@ static int auo_pixcir_collect_data(struct auo_pixcir_ts *ts, point[i].coord_y = raw_coord[4 * i + 3] << 8 | raw_coord[4 * i + 2]; - if (point[i].coord_x > pdata->x_max || - point[i].coord_y > pdata->y_max) { + if (point[i].coord_x > ts->x_max || + point[i].coord_y > ts->y_max) { dev_warn(&client->dev, "coordinates (%d,%d) invalid\n", point[i].coord_x, point[i].coord_y); point[i].coord_x = point[i].coord_y = 0; @@ -171,7 +184,6 @@ static int auo_pixcir_collect_data(struct auo_pixcir_ts *ts, static irqreturn_t auo_pixcir_interrupt(int irq, void *dev_id) { struct auo_pixcir_ts *ts = dev_id; - const struct auo_pixcir_ts_platdata *pdata = ts->pdata; struct auo_point_t point[AUO_PIXCIR_REPORT_POINTS]; int i; int ret; @@ -182,7 +194,7 @@ static irqreturn_t auo_pixcir_interrupt(int irq, void *dev_id) /* check for up event in touch touch_ind_mode */ if (ts->touch_ind_mode) { - if (gpio_get_value(pdata->gpio_int) == 0) { + if (gpiod_get_value_cansleep(ts->gpio_int) == 0) { input_mt_sync(ts->input); input_report_key(ts->input, BTN_TOUCH, 0); input_sync(ts->input); @@ -278,11 +290,9 @@ static int auo_pixcir_power_mode(struct auo_pixcir_ts *ts, int mode) return 0; } -static int auo_pixcir_int_config(struct auo_pixcir_ts *ts, - int int_setting) +static int auo_pixcir_int_config(struct auo_pixcir_ts *ts, int int_setting) { struct i2c_client *client = ts->client; - const struct auo_pixcir_ts_platdata *pdata = ts->pdata; int ret; ret = i2c_smbus_read_byte_data(client, AUO_PIXCIR_REG_INT_SETTING); @@ -304,7 +314,7 @@ static int auo_pixcir_int_config(struct auo_pixcir_ts *ts, return ret; } - ts->touch_ind_mode = pdata->int_setting == AUO_PIXCIR_INT_TOUCH_IND; + ts->touch_ind_mode = int_setting == AUO_PIXCIR_INT_TOUCH_IND; return 0; } @@ -465,78 +475,22 @@ unlock: static SIMPLE_DEV_PM_OPS(auo_pixcir_pm_ops, auo_pixcir_suspend, auo_pixcir_resume); -#ifdef CONFIG_OF -static struct auo_pixcir_ts_platdata *auo_pixcir_parse_dt(struct device *dev) -{ - struct auo_pixcir_ts_platdata *pdata; - struct device_node *np = dev->of_node; - - if (!np) - return ERR_PTR(-ENOENT); - - pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); - if (!pdata) - return ERR_PTR(-ENOMEM); - - pdata->gpio_int = of_get_gpio(np, 0); - if (!gpio_is_valid(pdata->gpio_int)) { - dev_err(dev, "failed to get interrupt gpio\n"); - return ERR_PTR(-EINVAL); - } - - pdata->gpio_rst = of_get_gpio(np, 1); - if (!gpio_is_valid(pdata->gpio_rst)) { - dev_err(dev, "failed to get reset gpio\n"); - return ERR_PTR(-EINVAL); - } - - if (of_property_read_u32(np, "x-size", &pdata->x_max)) { - dev_err(dev, "failed to get x-size property\n"); - return ERR_PTR(-EINVAL); - } - - if (of_property_read_u32(np, "y-size", &pdata->y_max)) { - dev_err(dev, "failed to get y-size property\n"); - return ERR_PTR(-EINVAL); - } - - /* default to asserting the interrupt when the screen is touched */ - pdata->int_setting = AUO_PIXCIR_INT_TOUCH_IND; - - return pdata; -} -#else -static struct auo_pixcir_ts_platdata *auo_pixcir_parse_dt(struct device *dev) -{ - return ERR_PTR(-EINVAL); -} -#endif - static void auo_pixcir_reset(void *data) { struct auo_pixcir_ts *ts = data; - gpio_set_value(ts->pdata->gpio_rst, 0); + gpiod_set_value_cansleep(ts->gpio_rst, 1); } static int auo_pixcir_probe(struct i2c_client *client, const struct i2c_device_id *id) { - const struct auo_pixcir_ts_platdata *pdata; struct auo_pixcir_ts *ts; struct input_dev *input_dev; int version; int error; - pdata = dev_get_platdata(&client->dev); - if (!pdata) { - pdata = auo_pixcir_parse_dt(&client->dev); - if (IS_ERR(pdata)) - return PTR_ERR(pdata); - } - - ts = devm_kzalloc(&client->dev, - sizeof(struct auo_pixcir_ts), GFP_KERNEL); + ts = devm_kzalloc(&client->dev, sizeof(*ts), GFP_KERNEL); if (!ts) return -ENOMEM; @@ -546,7 +500,6 @@ static int auo_pixcir_probe(struct i2c_client *client, return -ENOMEM; } - ts->pdata = pdata; ts->client = client; ts->input = input_dev; ts->touch_ind_mode = 0; @@ -556,6 +509,16 @@ static int auo_pixcir_probe(struct i2c_client *client, snprintf(ts->phys, sizeof(ts->phys), "%s/input0", dev_name(&client->dev)); + if (device_property_read_u32(&client->dev, "x-size", &ts->x_max)) { + dev_err(&client->dev, "failed to get x-size property\n"); + return -EINVAL; + } + + if (device_property_read_u32(&client->dev, "y-size", &ts->y_max)) { + dev_err(&client->dev, "failed to get y-size property\n"); + return -EINVAL; + } + input_dev->name = "AUO-Pixcir touchscreen"; input_dev->phys = ts->phys; input_dev->id.bustype = BUS_I2C; @@ -569,39 +532,42 @@ static int auo_pixcir_probe(struct i2c_client *client, __set_bit(BTN_TOUCH, input_dev->keybit); /* For single touch */ - input_set_abs_params(input_dev, ABS_X, 0, pdata->x_max, 0, 0); - input_set_abs_params(input_dev, ABS_Y, 0, pdata->y_max, 0, 0); + input_set_abs_params(input_dev, ABS_X, 0, ts->x_max, 0, 0); + input_set_abs_params(input_dev, ABS_Y, 0, ts->y_max, 0, 0); /* For multi touch */ - input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0, - pdata->x_max, 0, 0); - input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0, - pdata->y_max, 0, 0); - input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, 0, - AUO_PIXCIR_MAX_AREA, 0, 0); - input_set_abs_params(input_dev, ABS_MT_TOUCH_MINOR, 0, - AUO_PIXCIR_MAX_AREA, 0, 0); + input_set_abs_params(input_dev, ABS_MT_POSITION_X, 0, ts->x_max, 0, 0); + input_set_abs_params(input_dev, ABS_MT_POSITION_Y, 0, ts->y_max, 0, 0); + input_set_abs_params(input_dev, ABS_MT_TOUCH_MAJOR, + 0, AUO_PIXCIR_MAX_AREA, 0, 0); + input_set_abs_params(input_dev, ABS_MT_TOUCH_MINOR, + 0, AUO_PIXCIR_MAX_AREA, 0, 0); input_set_abs_params(input_dev, ABS_MT_ORIENTATION, 0, 1, 0, 0); input_set_drvdata(ts->input, ts); - error = devm_gpio_request_one(&client->dev, pdata->gpio_int, - GPIOF_DIR_IN, "auo_pixcir_ts_int"); + ts->gpio_int = devm_gpiod_get_index(&client->dev, NULL, 0, GPIOD_IN); + error = PTR_ERR_OR_ZERO(ts->gpio_int); if (error) { - dev_err(&client->dev, "request of gpio %d failed, %d\n", - pdata->gpio_int, error); + dev_err(&client->dev, + "request of int gpio failed: %d\n", error); return error; } - error = devm_gpio_request_one(&client->dev, pdata->gpio_rst, - GPIOF_DIR_OUT | GPIOF_INIT_HIGH, - "auo_pixcir_ts_rst"); + gpiod_set_consumer_name(ts->gpio_int, "auo_pixcir_ts_int"); + + /* Take the chip out of reset */ + ts->gpio_rst = devm_gpiod_get_index(&client->dev, NULL, 1, + GPIOD_OUT_LOW); + error = PTR_ERR_OR_ZERO(ts->gpio_rst); if (error) { - dev_err(&client->dev, "request of gpio %d failed, %d\n", - pdata->gpio_rst, error); + dev_err(&client->dev, + "request of reset gpio failed: %d\n", error); return error; } + gpiod_set_consumer_name(ts->gpio_rst, "auo_pixcir_ts_rst"); + error = devm_add_action_or_reset(&client->dev, auo_pixcir_reset, ts); if (error) { dev_err(&client->dev, "failed to register reset action, %d\n", @@ -619,13 +585,14 @@ static int auo_pixcir_probe(struct i2c_client *client, dev_info(&client->dev, "firmware version 0x%X\n", version); - error = auo_pixcir_int_config(ts, pdata->int_setting); + /* default to asserting the interrupt when the screen is touched */ + error = auo_pixcir_int_config(ts, AUO_PIXCIR_INT_TOUCH_IND); if (error) return error; error = devm_request_threaded_irq(&client->dev, client->irq, NULL, auo_pixcir_interrupt, - IRQF_TRIGGER_RISING | IRQF_ONESHOT, + IRQF_ONESHOT, input_dev->name, ts); if (error) { dev_err(&client->dev, "irq %d requested failed, %d\n", diff --git a/drivers/input/touchscreen/bu21013_ts.c b/drivers/input/touchscreen/bu21013_ts.c index 2f1f0d7607f81e496b7defaa61ca89aa0e703819..34f422e246ef4e52b92b41900d15ef57f8a43937 100644 --- a/drivers/input/touchscreen/bu21013_ts.c +++ b/drivers/input/touchscreen/bu21013_ts.c @@ -552,15 +552,13 @@ static int bu21013_probe(struct i2c_client *client, return 0; } -static int bu21013_remove(struct i2c_client *client) +static void bu21013_remove(struct i2c_client *client) { struct bu21013_ts *ts = i2c_get_clientdata(client); /* Make sure IRQ will exit quickly even if there is contact */ ts->touch_stopped = true; /* The resources will be freed by devm */ - - return 0; } static int __maybe_unused bu21013_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/chipone_icn8505.c b/drivers/input/touchscreen/chipone_icn8505.c index f9ca5502ac8c533d5eb02b92e6d5ede3620f338e..c421f4be27001523d84d66fd157a6a3737265b2a 100644 --- a/drivers/input/touchscreen/chipone_icn8505.c +++ b/drivers/input/touchscreen/chipone_icn8505.c @@ -364,32 +364,20 @@ static irqreturn_t icn8505_irq(int irq, void *dev_id) static int icn8505_probe_acpi(struct icn8505_data *icn8505, struct device *dev) { - struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; - const char *subsys = "unknown"; - struct acpi_device *adev; - union acpi_object *obj; - acpi_status status; - - adev = ACPI_COMPANION(dev); - if (!adev) - return -ENODEV; + const char *subsys; + int error; - status = acpi_evaluate_object(adev->handle, "_SUB", NULL, &buffer); - if (ACPI_SUCCESS(status)) { - obj = buffer.pointer; - if (obj->type == ACPI_TYPE_STRING) - subsys = obj->string.pointer; - else - dev_warn(dev, "Warning ACPI _SUB did not return a string\n"); - } else { - dev_warn(dev, "Warning ACPI _SUB failed: %#x\n", status); - buffer.pointer = NULL; - } + subsys = acpi_get_subsystem_id(ACPI_HANDLE(dev)); + error = PTR_ERR_OR_ZERO(subsys); + if (error == -ENODATA) + subsys = "unknown"; + else if (error) + return error; snprintf(icn8505->firmware_name, sizeof(icn8505->firmware_name), "chipone/icn8505-%s.fw", subsys); - kfree(buffer.pointer); + kfree_const(subsys); return 0; } diff --git a/drivers/input/touchscreen/cyttsp4_i2c.c b/drivers/input/touchscreen/cyttsp4_i2c.c index c65ccb2f4716e28fb4f3edb79810da5f45abfa75..28ae7c15397a605916d6a825ffca164e7f55152b 100644 --- a/drivers/input/touchscreen/cyttsp4_i2c.c +++ b/drivers/input/touchscreen/cyttsp4_i2c.c @@ -43,13 +43,11 @@ static int cyttsp4_i2c_probe(struct i2c_client *client, return PTR_ERR_OR_ZERO(ts); } -static int cyttsp4_i2c_remove(struct i2c_client *client) +static void cyttsp4_i2c_remove(struct i2c_client *client) { struct cyttsp4 *ts = i2c_get_clientdata(client); cyttsp4_remove(ts); - - return 0; } static const struct i2c_device_id cyttsp4_i2c_id[] = { diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c index 82beddb2876160f3f48a7b5a900abf0788865e7e..9ac1378610bc13d8bcbec7efdfe2deac90bd5ac4 100644 --- a/drivers/input/touchscreen/edt-ft5x06.c +++ b/drivers/input/touchscreen/edt-ft5x06.c @@ -912,8 +912,8 @@ static int edt_ft5x06_ts_identify(struct i2c_client *client, p = strchr(rdbuf, '*'); if (p) *p++ = '\0'; - strlcpy(model_name, rdbuf + 1, EDT_NAME_LEN); - strlcpy(fw_version, p ? p : "", EDT_NAME_LEN); + strscpy(model_name, rdbuf + 1, EDT_NAME_LEN); + strscpy(fw_version, p ? p : "", EDT_NAME_LEN); } else if (!strncasecmp(rdbuf, "EP0", 3)) { tsdata->version = EDT_M12; @@ -926,8 +926,8 @@ static int edt_ft5x06_ts_identify(struct i2c_client *client, p = strchr(rdbuf, '*'); if (p) *p++ = '\0'; - strlcpy(model_name, rdbuf, EDT_NAME_LEN); - strlcpy(fw_version, p ? p : "", EDT_NAME_LEN); + strscpy(model_name, rdbuf, EDT_NAME_LEN); + strscpy(fw_version, p ? p : "", EDT_NAME_LEN); } else { /* If it is not an EDT M06/M12 touchscreen, then the model * detection is a bit hairy. The different ft5x06 @@ -945,7 +945,7 @@ static int edt_ft5x06_ts_identify(struct i2c_client *client, if (error) return error; - strlcpy(fw_version, rdbuf, 2); + strscpy(fw_version, rdbuf, 2); error = edt_ft5x06_ts_readwrite(client, 1, "\xA8", 1, rdbuf); @@ -981,7 +981,7 @@ static int edt_ft5x06_ts_identify(struct i2c_client *client, 1, rdbuf); if (error) return error; - strlcpy(fw_version, rdbuf, 1); + strscpy(fw_version, rdbuf, 1); snprintf(model_name, EDT_NAME_LEN, "EVERVISION-FT5726NEi"); break; @@ -1346,13 +1346,11 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client, return 0; } -static int edt_ft5x06_ts_remove(struct i2c_client *client) +static void edt_ft5x06_ts_remove(struct i2c_client *client) { struct edt_ft5x06_ts_data *tsdata = i2c_get_clientdata(client); edt_ft5x06_ts_teardown_debugfs(tsdata); - - return 0; } static int __maybe_unused edt_ft5x06_ts_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index d016505fc081f31aaf6684dbca06c264dcc12d22..a33cc7950cf5b1c4e1fd8f7930180c9770b3db8e 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -95,6 +95,7 @@ static const struct goodix_chip_data gt9x_chip_data = { static const struct goodix_chip_id goodix_chip_ids[] = { { .id = "1151", .data = >1x_chip_data }, + { .id = "1158", .data = >1x_chip_data }, { .id = "5663", .data = >1x_chip_data }, { .id = "5688", .data = >1x_chip_data }, { .id = "917S", .data = >1x_chip_data }, @@ -1382,14 +1383,12 @@ reset: return 0; } -static int goodix_ts_remove(struct i2c_client *client) +static void goodix_ts_remove(struct i2c_client *client) { struct goodix_ts_data *ts = i2c_get_clientdata(client); if (ts->load_cfg_from_disk) wait_for_completion(&ts->firmware_loading_complete); - - return 0; } static int __maybe_unused goodix_suspend(struct device *dev) @@ -1508,6 +1507,7 @@ MODULE_DEVICE_TABLE(acpi, goodix_acpi_match); #ifdef CONFIG_OF static const struct of_device_id goodix_of_match[] = { { .compatible = "goodix,gt1151" }, + { .compatible = "goodix,gt1158" }, { .compatible = "goodix,gt5663" }, { .compatible = "goodix,gt5688" }, { .compatible = "goodix,gt911" }, diff --git a/drivers/input/touchscreen/gunze.c b/drivers/input/touchscreen/gunze.c index e07e8e0fe8ea97d2ee4dd3fd03af1e02a035ae23..5a5f9da73fa180b60e3c1452f29803166eca137c 100644 --- a/drivers/input/touchscreen/gunze.c +++ b/drivers/input/touchscreen/gunze.c @@ -7,9 +7,6 @@ * Gunze AHL-51S touchscreen driver for Linux */ -/* - */ - #include #include #include diff --git a/drivers/input/touchscreen/melfas_mip4.c b/drivers/input/touchscreen/melfas_mip4.c index 2745bf1aee3815f15e5e5c575bd74415ab30b1b6..83f4be05e27b66f3fecb28a7acbb8a44c8241085 100644 --- a/drivers/input/touchscreen/melfas_mip4.c +++ b/drivers/input/touchscreen/melfas_mip4.c @@ -1453,7 +1453,7 @@ static int mip4_probe(struct i2c_client *client, const struct i2c_device_id *id) "ce", GPIOD_OUT_LOW); if (IS_ERR(ts->gpio_ce)) { error = PTR_ERR(ts->gpio_ce); - if (error != EPROBE_DEFER) + if (error != -EPROBE_DEFER) dev_err(&client->dev, "Failed to get gpio: %d\n", error); return error; diff --git a/drivers/input/touchscreen/migor_ts.c b/drivers/input/touchscreen/migor_ts.c index 42d3fd7e04d7c1a7f29fb0689fe5b597ef070692..79cd660d879ed279f02db6179f1610060c399dfe 100644 --- a/drivers/input/touchscreen/migor_ts.c +++ b/drivers/input/touchscreen/migor_ts.c @@ -176,7 +176,7 @@ static int migor_ts_probe(struct i2c_client *client, return error; } -static int migor_ts_remove(struct i2c_client *client) +static void migor_ts_remove(struct i2c_client *client) { struct migor_ts_priv *priv = i2c_get_clientdata(client); @@ -185,8 +185,6 @@ static int migor_ts_remove(struct i2c_client *client) kfree(priv); dev_set_drvdata(&client->dev, NULL); - - return 0; } static int __maybe_unused migor_ts_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/s6sy761.c b/drivers/input/touchscreen/s6sy761.c index 85a1f465c097ea10621f7e2860b0cb777f84bf89..1a7d00289b4ce1bc721034ba133671eada9ece1c 100644 --- a/drivers/input/touchscreen/s6sy761.c +++ b/drivers/input/touchscreen/s6sy761.c @@ -475,11 +475,9 @@ static int s6sy761_probe(struct i2c_client *client, return 0; } -static int s6sy761_remove(struct i2c_client *client) +static void s6sy761_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); - - return 0; } static int __maybe_unused s6sy761_runtime_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c index c175d44c52f37011d8d29d7a664240733e129302..d5bd170808fb10303d7792ad49c4dcee26dbfcad 100644 --- a/drivers/input/touchscreen/stmfts.c +++ b/drivers/input/touchscreen/stmfts.c @@ -738,11 +738,9 @@ static int stmfts_probe(struct i2c_client *client, return 0; } -static int stmfts_remove(struct i2c_client *client) +static void stmfts_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); - - return 0; } static int __maybe_unused stmfts_runtime_suspend(struct device *dev) diff --git a/drivers/input/touchscreen/sun4i-ts.c b/drivers/input/touchscreen/sun4i-ts.c index 742a7e96c1b57d73a565eab8eda5511e9a78e18a..73eb8f80be6ef369143e1755c67574bcae3db81f 100644 --- a/drivers/input/touchscreen/sun4i-ts.c +++ b/drivers/input/touchscreen/sun4i-ts.c @@ -192,12 +192,12 @@ static int sun4i_get_temp(const struct sun4i_ts_data *ts, int *temp) return 0; } -static int sun4i_get_tz_temp(void *data, int *temp) +static int sun4i_get_tz_temp(struct thermal_zone_device *tz, int *temp) { - return sun4i_get_temp(data, temp); + return sun4i_get_temp(tz->devdata, temp); } -static const struct thermal_zone_of_device_ops sun4i_ts_tz_ops = { +static const struct thermal_zone_device_ops sun4i_ts_tz_ops = { .get_temp = sun4i_get_tz_temp, }; @@ -356,8 +356,8 @@ static int sun4i_ts_probe(struct platform_device *pdev) if (IS_ERR(hwmon)) return PTR_ERR(hwmon); - thermal = devm_thermal_zone_of_sensor_register(ts->dev, 0, ts, - &sun4i_ts_tz_ops); + thermal = devm_thermal_of_zone_register(ts->dev, 0, ts, + &sun4i_ts_tz_ops); if (IS_ERR(thermal)) return PTR_ERR(thermal); diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c index 12f2562b0141b318dbc3a5f363a5399cb3972593..8ddb3f7d307aa484907d175559a1b90d6af8771c 100644 --- a/drivers/input/touchscreen/sur40.c +++ b/drivers/input/touchscreen/sur40.c @@ -939,8 +939,8 @@ static int sur40_vidioc_querycap(struct file *file, void *priv, { struct sur40_state *sur40 = video_drvdata(file); - strlcpy(cap->driver, DRIVER_SHORT, sizeof(cap->driver)); - strlcpy(cap->card, DRIVER_LONG, sizeof(cap->card)); + strscpy(cap->driver, DRIVER_SHORT, sizeof(cap->driver)); + strscpy(cap->card, DRIVER_LONG, sizeof(cap->card)); usb_make_path(sur40->usbdev, cap->bus_info, sizeof(cap->bus_info)); return 0; } @@ -952,7 +952,7 @@ static int sur40_vidioc_enum_input(struct file *file, void *priv, return -EINVAL; i->type = V4L2_INPUT_TYPE_TOUCH; i->std = V4L2_STD_UNKNOWN; - strlcpy(i->name, "In-Cell Sensor", sizeof(i->name)); + strscpy(i->name, "In-Cell Sensor", sizeof(i->name)); i->capabilities = 0; return 0; } diff --git a/drivers/input/touchscreen/tsc2004.c b/drivers/input/touchscreen/tsc2004.c index 9fdd870c4c0bd8b4017f81aeae4770f93f2f54dc..a9565353ee985fb8e4610f2330c78f7510427484 100644 --- a/drivers/input/touchscreen/tsc2004.c +++ b/drivers/input/touchscreen/tsc2004.c @@ -43,11 +43,9 @@ static int tsc2004_probe(struct i2c_client *i2c, tsc2004_cmd); } -static int tsc2004_remove(struct i2c_client *i2c) +static void tsc2004_remove(struct i2c_client *i2c) { tsc200x_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id tsc2004_idtable[] = { diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c index 3dda6eaabdab82721c844c8a18ec1f6cf6fb8fc8..d6d04b9f04fc1cb272c67f3968556d1e8024cd08 100644 --- a/drivers/input/touchscreen/usbtouchscreen.c +++ b/drivers/input/touchscreen/usbtouchscreen.c @@ -1708,7 +1708,7 @@ static int usbtouch_probe(struct usb_interface *intf, usbtouch->input = input_dev; if (udev->manufacturer) - strlcpy(usbtouch->name, udev->manufacturer, sizeof(usbtouch->name)); + strscpy(usbtouch->name, udev->manufacturer, sizeof(usbtouch->name)); if (udev->product) { if (udev->manufacturer) diff --git a/drivers/input/touchscreen/wacom_w8001.c b/drivers/input/touchscreen/wacom_w8001.c index 691285ace2289f83f0f270ba091e58bc8ef718d9..928c5ee3ac36c2b5764df5d255f1fc18274c5982 100644 --- a/drivers/input/touchscreen/wacom_w8001.c +++ b/drivers/input/touchscreen/wacom_w8001.c @@ -625,7 +625,7 @@ static int w8001_connect(struct serio *serio, struct serio_driver *drv) /* For backwards-compatibility we compose the basename based on * capabilities and then just append the tool type */ - strlcpy(basename, "Wacom Serial", sizeof(basename)); + strscpy(basename, "Wacom Serial", sizeof(basename)); err_pen = w8001_setup_pen(w8001, basename, sizeof(basename)); err_touch = w8001_setup_touch(w8001, basename, sizeof(basename)); @@ -635,7 +635,7 @@ static int w8001_connect(struct serio *serio, struct serio_driver *drv) } if (!err_pen) { - strlcpy(w8001->pen_name, basename, sizeof(w8001->pen_name)); + strscpy(w8001->pen_name, basename, sizeof(w8001->pen_name)); strlcat(w8001->pen_name, " Pen", sizeof(w8001->pen_name)); input_dev_pen->name = w8001->pen_name; @@ -651,7 +651,7 @@ static int w8001_connect(struct serio *serio, struct serio_driver *drv) } if (!err_touch) { - strlcpy(w8001->touch_name, basename, sizeof(w8001->touch_name)); + strscpy(w8001->touch_name, basename, sizeof(w8001->touch_name)); strlcat(w8001->touch_name, " Finger", sizeof(w8001->touch_name)); input_dev_touch->name = w8001->touch_name; diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index 808f6e7a80482796b23f05899147c788bf91a814..25debded65a8fbcecff98f6511ecd35899f8312b 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -1057,29 +1057,25 @@ EXPORT_SYMBOL_GPL(icc_provider_add); /** * icc_provider_del() - delete previously added interconnect provider * @provider: the interconnect provider that will be removed from topology - * - * Return: 0 on success, or an error code otherwise */ -int icc_provider_del(struct icc_provider *provider) +void icc_provider_del(struct icc_provider *provider) { mutex_lock(&icc_lock); if (provider->users) { pr_warn("interconnect provider still has %d users\n", provider->users); mutex_unlock(&icc_lock); - return -EBUSY; + return; } if (!list_empty(&provider->nodes)) { pr_warn("interconnect provider still has nodes\n"); mutex_unlock(&icc_lock); - return -EBUSY; + return; } list_del(&provider->provider_list); mutex_unlock(&icc_lock); - - return 0; } EXPORT_SYMBOL_GPL(icc_provider_del); diff --git a/drivers/interconnect/imx/imx.c b/drivers/interconnect/imx/imx.c index 48ffd59953bf9e58d9be36c383f80e95de72c37d..823d9be9771a1c70a3d3db7d3b844a68c868854c 100644 --- a/drivers/interconnect/imx/imx.c +++ b/drivers/interconnect/imx/imx.c @@ -324,13 +324,13 @@ provider_del: } EXPORT_SYMBOL_GPL(imx_icc_register); -int imx_icc_unregister(struct platform_device *pdev) +void imx_icc_unregister(struct platform_device *pdev) { struct imx_icc_provider *imx_provider = platform_get_drvdata(pdev); imx_icc_unregister_nodes(&imx_provider->provider); - return icc_provider_del(&imx_provider->provider); + icc_provider_del(&imx_provider->provider); } EXPORT_SYMBOL_GPL(imx_icc_unregister); diff --git a/drivers/interconnect/imx/imx.h b/drivers/interconnect/imx/imx.h index e0a2ee173ecd92bdd94c06c3d2b83b4957c186eb..895907cdcb3bce22b9480e25b0084f6a8ebf579f 100644 --- a/drivers/interconnect/imx/imx.h +++ b/drivers/interconnect/imx/imx.h @@ -103,6 +103,6 @@ int imx_icc_register(struct platform_device *pdev, struct imx_icc_node_desc *nodes, int nodes_count, struct imx_icc_noc_setting *noc_settings); -int imx_icc_unregister(struct platform_device *pdev); +void imx_icc_unregister(struct platform_device *pdev); #endif /* __DRIVERS_INTERCONNECT_IMX_H */ diff --git a/drivers/interconnect/imx/imx8mm.c b/drivers/interconnect/imx/imx8mm.c index ae797412db96093da578308c225e5ab9d4c14d9a..b43325364aa31adbeddee934b8c6a3d480a54b55 100644 --- a/drivers/interconnect/imx/imx8mm.c +++ b/drivers/interconnect/imx/imx8mm.c @@ -88,7 +88,9 @@ static int imx8mm_icc_probe(struct platform_device *pdev) static int imx8mm_icc_remove(struct platform_device *pdev) { - return imx_icc_unregister(pdev); + imx_icc_unregister(pdev); + + return 0; } static struct platform_driver imx8mm_icc_driver = { diff --git a/drivers/interconnect/imx/imx8mn.c b/drivers/interconnect/imx/imx8mn.c index 1ce94c5bdd8ce9947ecd7ff13ea4e67c8f3f5364..8ce6d8e4bf5e9455de9addd06b942e39dcab7bd4 100644 --- a/drivers/interconnect/imx/imx8mn.c +++ b/drivers/interconnect/imx/imx8mn.c @@ -77,7 +77,9 @@ static int imx8mn_icc_probe(struct platform_device *pdev) static int imx8mn_icc_remove(struct platform_device *pdev) { - return imx_icc_unregister(pdev); + imx_icc_unregister(pdev); + + return 0; } static struct platform_driver imx8mn_icc_driver = { diff --git a/drivers/interconnect/imx/imx8mp.c b/drivers/interconnect/imx/imx8mp.c index 5f1c83ed157bc1365214a30a4b18760304ecdbf5..8bfaf173f1da6278b62672f4abe6607230e78c0f 100644 --- a/drivers/interconnect/imx/imx8mp.c +++ b/drivers/interconnect/imx/imx8mp.c @@ -242,7 +242,9 @@ static int imx8mp_icc_probe(struct platform_device *pdev) static int imx8mp_icc_remove(struct platform_device *pdev) { - return imx_icc_unregister(pdev); + imx_icc_unregister(pdev); + + return 0; } static struct platform_driver imx8mp_icc_driver = { diff --git a/drivers/interconnect/imx/imx8mq.c b/drivers/interconnect/imx/imx8mq.c index 7f00a0511c6e291033826fc8ac6238fc4d6b01d5..b6fb71305c99b5549b4a44fb4bb59c7ce2034325 100644 --- a/drivers/interconnect/imx/imx8mq.c +++ b/drivers/interconnect/imx/imx8mq.c @@ -87,7 +87,9 @@ static int imx8mq_icc_probe(struct platform_device *pdev) static int imx8mq_icc_remove(struct platform_device *pdev) { - return imx_icc_unregister(pdev); + imx_icc_unregister(pdev); + + return 0; } static struct platform_driver imx8mq_icc_driver = { diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig index 25d5b4baf6f65e8fd21e5f6d09f4516c945b7289..1a1c941635a20f9a264bb296d1969609f5ab25b7 100644 --- a/drivers/interconnect/qcom/Kconfig +++ b/drivers/interconnect/qcom/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config INTERCONNECT_QCOM - bool "Qualcomm Network-on-Chip interconnect drivers" + tristate "Qualcomm Network-on-Chip interconnect drivers" depends on ARCH_QCOM help Support for Qualcomm's Network-on-Chip interconnect hardware. diff --git a/drivers/interconnect/qcom/icc-common.c b/drivers/interconnect/qcom/icc-common.c index 0822ce207b5d5e06dc880723e8b18f87954d5b77..f27f4fdc453170fc60bb4cedea2d0004b7dad9f7 100644 --- a/drivers/interconnect/qcom/icc-common.c +++ b/drivers/interconnect/qcom/icc-common.c @@ -5,6 +5,7 @@ #include #include +#include #include "icc-common.h" @@ -32,3 +33,5 @@ struct icc_node_data *qcom_icc_xlate_extended(struct of_phandle_args *spec, void return ndata; } EXPORT_SYMBOL_GPL(qcom_icc_xlate_extended); + +MODULE_LICENSE("GPL"); diff --git a/drivers/interconnect/qcom/icc-rpm.c b/drivers/interconnect/qcom/icc-rpm.c index 7f6a70e0256aad274ab94983c175be3b052cd692..39e43b9575998d1dc07ce881a66affb9120b4e5a 100644 --- a/drivers/interconnect/qcom/icc-rpm.c +++ b/drivers/interconnect/qcom/icc-rpm.c @@ -563,6 +563,8 @@ int qnoc_remove(struct platform_device *pdev) icc_nodes_remove(&qp->provider); clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + icc_provider_del(&qp->provider); + + return 0; } EXPORT_SYMBOL(qnoc_remove); diff --git a/drivers/interconnect/qcom/icc-rpmh.c b/drivers/interconnect/qcom/icc-rpmh.c index 114bb8f645738a27ec7800183c668da727181a96..fd17291c61eb96b5b9480669c1f1f090b41c2825 100644 --- a/drivers/interconnect/qcom/icc-rpmh.c +++ b/drivers/interconnect/qcom/icc-rpmh.c @@ -251,7 +251,9 @@ int qcom_icc_rpmh_remove(struct platform_device *pdev) struct qcom_icc_provider *qp = platform_get_drvdata(pdev); icc_nodes_remove(&qp->provider); - return icc_provider_del(&qp->provider); + icc_provider_del(&qp->provider); + + return 0; } EXPORT_SYMBOL_GPL(qcom_icc_rpmh_remove); diff --git a/drivers/interconnect/qcom/msm8974.c b/drivers/interconnect/qcom/msm8974.c index 6fa0ad90fc3da20dd779857c1b065ce88420babd..5ea192f1141dc42f4cee48fb6c28cc51bcd65430 100644 --- a/drivers/interconnect/qcom/msm8974.c +++ b/drivers/interconnect/qcom/msm8974.c @@ -749,7 +749,9 @@ static int msm8974_icc_remove(struct platform_device *pdev) icc_nodes_remove(&qp->provider); clk_bulk_disable_unprepare(qp->num_clks, qp->bus_clks); - return icc_provider_del(&qp->provider); + icc_provider_del(&qp->provider); + + return 0; } static const struct of_device_id msm8974_noc_of_match[] = { diff --git a/drivers/interconnect/qcom/osm-l3.c b/drivers/interconnect/qcom/osm-l3.c index 4198656f4e59c724f7c38dccd2fb88922540f18f..ddbdf0943f94e7181deb7af0a41d36690a8d9439 100644 --- a/drivers/interconnect/qcom/osm-l3.c +++ b/drivers/interconnect/qcom/osm-l3.c @@ -217,7 +217,9 @@ static int qcom_osm_l3_remove(struct platform_device *pdev) struct qcom_osm_l3_icc_provider *qp = platform_get_drvdata(pdev); icc_nodes_remove(&qp->provider); - return icc_provider_del(&qp->provider); + icc_provider_del(&qp->provider); + + return 0; } static int qcom_osm_l3_probe(struct platform_device *pdev) diff --git a/drivers/interconnect/qcom/sm8450.c b/drivers/interconnect/qcom/sm8450.c index e821fd0b2f66eb07c15958db13991fa4fcbf0472..e3a12e3d6e0619275670d3d41a69680f8c61d667 100644 --- a/drivers/interconnect/qcom/sm8450.c +++ b/drivers/interconnect/qcom/sm8450.c @@ -1933,7 +1933,9 @@ static int qnoc_remove(struct platform_device *pdev) struct qcom_icc_provider *qp = platform_get_drvdata(pdev); icc_nodes_remove(&qp->provider); - return icc_provider_del(&qp->provider); + icc_provider_del(&qp->provider); + + return 0; } static const struct of_device_id qnoc_of_match[] = { diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 5c5cb5bee8b6262a3eb8ed11a1a25376ab42e515..dc5f7a156ff5ec733c748448af161fef159eeb55 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -67,6 +67,17 @@ config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST If unsure, say N here. +config IOMMU_IO_PGTABLE_DART + bool "Apple DART Formats" + select IOMMU_IO_PGTABLE + depends on ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64) + help + Enable support for the Apple DART pagetable formats. These include + the t8020 and t6000/t8110 DART formats used in Apple M1/M2 family + SoCs. + + If unsure, say N here. + endmenu config IOMMU_DEBUGFS @@ -137,7 +148,7 @@ config OF_IOMMU # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA - bool + def_bool ARM64 || IA64 || X86 select DMA_OPS select IOMMU_API select IOMMU_IOVA @@ -294,7 +305,7 @@ config APPLE_DART tristate "Apple DART IOMMU Support" depends on ARCH_APPLE || (COMPILE_TEST && !GENERIC_ATOMIC64) select IOMMU_API - select IOMMU_IO_PGTABLE_LPAE + select IOMMU_IO_PGTABLE_DART default ARCH_APPLE help Support for Apple DART (Device Address Resolution Table) IOMMUs @@ -476,7 +487,6 @@ config VIRTIO_IOMMU depends on VIRTIO depends on (ARM64 || X86) select IOMMU_API - select IOMMU_DMA select INTERVAL_TREE select ACPI_VIOT if ACPI help diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 44475a9b3eeaf93e96847bdfbe301f60851b9ee3..cc9f381013c35fd1fcc44a9f280dad8cdf48730b 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o +obj-$(CONFIG_IOMMU_IO_PGTABLE_DART) += io-pgtable-dart.o obj-$(CONFIG_IOASID) += ioasid.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index a3cbafb603f50f0f454d327e3af641ee90168d74..9b5fc3356bf2d8ac1ca9e3a8398200ae8ccd2940 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -9,7 +9,6 @@ config AMD_IOMMU select PCI_PASID select IOMMU_API select IOMMU_IOVA - select IOMMU_DMA select IOMMU_IO_PGTABLE depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help diff --git a/drivers/iommu/amd/Makefile b/drivers/iommu/amd/Makefile index a935f8f4b9744d7d1fb8405e35672d3025686520..773d8aa002837eebce6d25c3cfea5e9b28bb85c3 100644 --- a/drivers/iommu/amd/Makefile +++ b/drivers/iommu/amd/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o +obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o obj-$(CONFIG_AMD_IOMMU_V2) += iommu_v2.o diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 84e5bb1bf01b67d96a41a9a641ddc5ad70f34b27..c160a332ce339ec4f4c3f7b979a68cf6ade62eef 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -18,7 +18,6 @@ extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu); extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); -extern int amd_iommu_init_api(void); extern void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid); #ifdef CONFIG_AMD_IOMMU_DEBUGFS diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 5b1019dab328569b7e50740715756c257b76e68a..1d0a70c85333aa6831c05fc797e5b6ea15cfe111 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -94,6 +94,7 @@ #define FEATURE_HE (1ULL<<8) #define FEATURE_PC (1ULL<<9) #define FEATURE_GAM_VAPIC (1ULL<<21) +#define FEATURE_GIOSUP (1ULL<<48) #define FEATURE_EPHSUP (1ULL<<50) #define FEATURE_SNP (1ULL<<63) @@ -276,6 +277,8 @@ * 512GB Pages are not supported due to a hardware bug */ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) +/* 4K, 2MB, 1G page sizes are supported */ +#define AMD_IOMMU_PGSIZES_V2 (PAGE_SIZE | (1ULL << 21) | (1ULL << 30)) /* Bit value definition for dte irq remapping fields*/ #define DTE_IRQ_PHYS_ADDR_MASK (((1ULL << 45)-1) << 6) @@ -376,6 +379,7 @@ #define DTE_FLAG_IW (1ULL << 62) #define DTE_FLAG_IOTLB (1ULL << 32) +#define DTE_FLAG_GIOV (1ULL << 54) #define DTE_FLAG_GV (1ULL << 55) #define DTE_FLAG_MASK (0x3ffULL << 32) #define DTE_GLX_SHIFT (56) @@ -434,6 +438,7 @@ #define PD_PASSTHROUGH_MASK (1UL << 2) /* domain has no page translation */ #define PD_IOMMUV2_MASK (1UL << 3) /* domain has gcr3 table */ +#define PD_GIOV_MASK (1UL << 4) /* domain enable GIOV support */ extern bool amd_iommu_dump; #define DUMP_printk(format, arg...) \ @@ -456,6 +461,8 @@ struct irq_remap_table { /* Interrupt remapping feature used? */ extern bool amd_iommu_irq_remap; +extern const struct iommu_ops amd_iommu_ops; + /* IVRS indicates that pre-boot remapping was enabled */ extern bool amdr_ivrs_remap_support; @@ -526,7 +533,8 @@ struct amd_io_pgtable { struct io_pgtable iop; int mode; u64 *root; - atomic64_t pt_root; /* pgtable root and pgtable mode */ + atomic64_t pt_root; /* pgtable root and pgtable mode */ + u64 *pgd; /* v2 pgtable pgd pointer */ }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index fdc642362c1401dd21394fb14db9cd1b68404bb3..1a2d425bf5687fffb44e419502ae7b8025687278 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -95,8 +95,6 @@ * out of it. */ -extern const struct iommu_ops amd_iommu_ops; - /* * structure describing one IOMMU in the ACPI table. Typically followed by one * or more ivhd_entrys. @@ -2068,6 +2066,17 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) init_iommu_perf_ctr(iommu); + if (amd_iommu_pgtable == AMD_IOMMU_V2) { + if (!iommu_feature(iommu, FEATURE_GIOSUP) || + !iommu_feature(iommu, FEATURE_GT)) { + pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n"); + amd_iommu_pgtable = AMD_IOMMU_V1; + } else if (iommu_default_passthrough()) { + pr_warn("V2 page table doesn't support passthrough mode. Fallback to v1.\n"); + amd_iommu_pgtable = AMD_IOMMU_V1; + } + } + if (is_rd890_iommu(iommu->dev)) { int i, j; @@ -2146,6 +2155,8 @@ static void print_iommu_info(void) if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) pr_info("X2APIC enabled\n"); } + if (amd_iommu_pgtable == AMD_IOMMU_V2) + pr_info("V2 page table enabled\n"); } static int __init amd_iommu_init_pci(void) @@ -2168,20 +2179,13 @@ static int __init amd_iommu_init_pci(void) /* * Order is important here to make sure any unity map requirements are * fulfilled. The unity mappings are created and written to the device - * table during the amd_iommu_init_api() call. + * table during the iommu_init_pci() call. * * After that we call init_device_table_dma() to make sure any * uninitialized DTE will block DMA, and in the end we flush the caches * of all IOMMUs to make sure the changes to the device table are * active. */ - ret = amd_iommu_init_api(); - if (ret) { - pr_err("IOMMU: Failed to initialize IOMMU-API interface (error=%d)!\n", - ret); - goto out; - } - for_each_pci_segment(pci_seg) init_device_table_dma(pci_seg); @@ -3366,17 +3370,30 @@ static int __init parse_amd_iommu_intr(char *str) static int __init parse_amd_iommu_options(char *str) { - for (; *str; ++str) { + if (!str) + return -EINVAL; + + while (*str) { if (strncmp(str, "fullflush", 9) == 0) { pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n"); iommu_set_dma_strict(); - } - if (strncmp(str, "force_enable", 12) == 0) + } else if (strncmp(str, "force_enable", 12) == 0) { amd_iommu_force_enable = true; - if (strncmp(str, "off", 3) == 0) + } else if (strncmp(str, "off", 3) == 0) { amd_iommu_disabled = true; - if (strncmp(str, "force_isolation", 15) == 0) + } else if (strncmp(str, "force_isolation", 15) == 0) { amd_iommu_force_isolation = true; + } else if (strncmp(str, "pgtbl_v1", 8) == 0) { + amd_iommu_pgtable = AMD_IOMMU_V1; + } else if (strncmp(str, "pgtbl_v2", 8) == 0) { + amd_iommu_pgtable = AMD_IOMMU_V2; + } else { + pr_notice("Unknown option - '%s'\n", str); + } + + str += strcspn(str, ","); + while (*str == ',') + str++; } return 1; diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 7d4b61e5db47b4a457ebb6662fe40de29c61c43f..ace0e9b8b91327d3e65c89339c6df614394b8615 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -360,8 +360,9 @@ static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist) * supporting all features of AMD IOMMU page tables like level skipping * and full 64 bit address spaces. */ -static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int prot, gfp_t gfp) +static int iommu_v1_map_pages(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *mapped) { struct protection_domain *dom = io_pgtable_ops_to_domain(ops); LIST_HEAD(freelist); @@ -369,39 +370,47 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, u64 __pte, *pte; int ret, i, count; - BUG_ON(!IS_ALIGNED(iova, size)); - BUG_ON(!IS_ALIGNED(paddr, size)); + BUG_ON(!IS_ALIGNED(iova, pgsize)); + BUG_ON(!IS_ALIGNED(paddr, pgsize)); ret = -EINVAL; if (!(prot & IOMMU_PROT_MASK)) goto out; - count = PAGE_SIZE_PTE_COUNT(size); - pte = alloc_pte(dom, iova, size, NULL, gfp, &updated); + while (pgcount > 0) { + count = PAGE_SIZE_PTE_COUNT(pgsize); + pte = alloc_pte(dom, iova, pgsize, NULL, gfp, &updated); - ret = -ENOMEM; - if (!pte) - goto out; + ret = -ENOMEM; + if (!pte) + goto out; - for (i = 0; i < count; ++i) - free_clear_pte(&pte[i], pte[i], &freelist); + for (i = 0; i < count; ++i) + free_clear_pte(&pte[i], pte[i], &freelist); - if (!list_empty(&freelist)) - updated = true; + if (!list_empty(&freelist)) + updated = true; - if (count > 1) { - __pte = PAGE_SIZE_PTE(__sme_set(paddr), size); - __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC; - } else - __pte = __sme_set(paddr) | IOMMU_PTE_PR | IOMMU_PTE_FC; + if (count > 1) { + __pte = PAGE_SIZE_PTE(__sme_set(paddr), pgsize); + __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_PR | IOMMU_PTE_FC; + } else + __pte = __sme_set(paddr) | IOMMU_PTE_PR | IOMMU_PTE_FC; - if (prot & IOMMU_PROT_IR) - __pte |= IOMMU_PTE_IR; - if (prot & IOMMU_PROT_IW) - __pte |= IOMMU_PTE_IW; + if (prot & IOMMU_PROT_IR) + __pte |= IOMMU_PTE_IR; + if (prot & IOMMU_PROT_IW) + __pte |= IOMMU_PTE_IW; - for (i = 0; i < count; ++i) - pte[i] = __pte; + for (i = 0; i < count; ++i) + pte[i] = __pte; + + iova += pgsize; + paddr += pgsize; + pgcount--; + if (mapped) + *mapped += pgsize; + } ret = 0; @@ -426,17 +435,18 @@ out: return ret; } -static unsigned long iommu_v1_unmap_page(struct io_pgtable_ops *ops, - unsigned long iova, - size_t size, - struct iommu_iotlb_gather *gather) +static unsigned long iommu_v1_unmap_pages(struct io_pgtable_ops *ops, + unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) { struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); unsigned long long unmapped; unsigned long unmap_size; u64 *pte; + size_t size = pgcount << __ffs(pgsize); - BUG_ON(!is_power_of_2(size)); + BUG_ON(!is_power_of_2(pgsize)); unmapped = 0; @@ -448,14 +458,14 @@ static unsigned long iommu_v1_unmap_page(struct io_pgtable_ops *ops, count = PAGE_SIZE_PTE_COUNT(unmap_size); for (i = 0; i < count; i++) pte[i] = 0ULL; + } else { + return unmapped; } iova = (iova & ~(unmap_size - 1)) + unmap_size; unmapped += unmap_size; } - BUG_ON(unmapped && !is_power_of_2(unmapped)); - return unmapped; } @@ -514,8 +524,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v1_flush_ops; - pgtable->iop.ops.map = iommu_v1_map_page; - pgtable->iop.ops.unmap = iommu_v1_unmap_page; + pgtable->iop.ops.map_pages = iommu_v1_map_pages; + pgtable->iop.ops.unmap_pages = iommu_v1_unmap_pages; pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys; return &pgtable->iop; diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c new file mode 100644 index 0000000000000000000000000000000000000000..8638ddf6fb3b2978dd01efb2e0672023233e188a --- /dev/null +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CPU-agnostic AMD IO page table v2 allocator. + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. + * Author: Suravee Suthikulpanit + * Author: Vasant Hegde + */ + +#define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) + +#include +#include +#include + +#include + +#include "amd_iommu_types.h" +#include "amd_iommu.h" + +#define IOMMU_PAGE_PRESENT BIT_ULL(0) /* Is present */ +#define IOMMU_PAGE_RW BIT_ULL(1) /* Writeable */ +#define IOMMU_PAGE_USER BIT_ULL(2) /* Userspace addressable */ +#define IOMMU_PAGE_PWT BIT_ULL(3) /* Page write through */ +#define IOMMU_PAGE_PCD BIT_ULL(4) /* Page cache disabled */ +#define IOMMU_PAGE_ACCESS BIT_ULL(5) /* Was accessed (updated by IOMMU) */ +#define IOMMU_PAGE_DIRTY BIT_ULL(6) /* Was written to (updated by IOMMU) */ +#define IOMMU_PAGE_PSE BIT_ULL(7) /* Page Size Extensions */ +#define IOMMU_PAGE_NX BIT_ULL(63) /* No execute */ + +#define MAX_PTRS_PER_PAGE 512 + +#define IOMMU_PAGE_SIZE_2M BIT_ULL(21) +#define IOMMU_PAGE_SIZE_1G BIT_ULL(30) + + +static inline int get_pgtable_level(void) +{ + /* 5 level page table is not supported */ + return PAGE_MODE_4_LEVEL; +} + +static inline bool is_large_pte(u64 pte) +{ + return (pte & IOMMU_PAGE_PSE); +} + +static inline void *alloc_pgtable_page(void) +{ + return (void *)get_zeroed_page(GFP_KERNEL); +} + +static inline u64 set_pgtable_attr(u64 *page) +{ + u64 prot; + + prot = IOMMU_PAGE_PRESENT | IOMMU_PAGE_RW | IOMMU_PAGE_USER; + prot |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; + + return (iommu_virt_to_phys(page) | prot); +} + +static inline void *get_pgtable_pte(u64 pte) +{ + return iommu_phys_to_virt(pte & PM_ADDR_MASK); +} + +static u64 set_pte_attr(u64 paddr, u64 pg_size, int prot) +{ + u64 pte; + + pte = __sme_set(paddr & PM_ADDR_MASK); + pte |= IOMMU_PAGE_PRESENT | IOMMU_PAGE_USER; + pte |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; + + if (prot & IOMMU_PROT_IW) + pte |= IOMMU_PAGE_RW; + + /* Large page */ + if (pg_size == IOMMU_PAGE_SIZE_1G || pg_size == IOMMU_PAGE_SIZE_2M) + pte |= IOMMU_PAGE_PSE; + + return pte; +} + +static inline u64 get_alloc_page_size(u64 size) +{ + if (size >= IOMMU_PAGE_SIZE_1G) + return IOMMU_PAGE_SIZE_1G; + + if (size >= IOMMU_PAGE_SIZE_2M) + return IOMMU_PAGE_SIZE_2M; + + return PAGE_SIZE; +} + +static inline int page_size_to_level(u64 pg_size) +{ + if (pg_size == IOMMU_PAGE_SIZE_1G) + return PAGE_MODE_3_LEVEL; + if (pg_size == IOMMU_PAGE_SIZE_2M) + return PAGE_MODE_2_LEVEL; + + return PAGE_MODE_1_LEVEL; +} + +static inline void free_pgtable_page(u64 *pt) +{ + free_page((unsigned long)pt); +} + +static void free_pgtable(u64 *pt, int level) +{ + u64 *p; + int i; + + for (i = 0; i < MAX_PTRS_PER_PAGE; i++) { + /* PTE present? */ + if (!IOMMU_PTE_PRESENT(pt[i])) + continue; + + if (is_large_pte(pt[i])) + continue; + + /* + * Free the next level. No need to look at l1 tables here since + * they can only contain leaf PTEs; just free them directly. + */ + p = get_pgtable_pte(pt[i]); + if (level > 2) + free_pgtable(p, level - 1); + else + free_pgtable_page(p); + } + + free_pgtable_page(pt); +} + +/* Allocate page table */ +static u64 *v2_alloc_pte(u64 *pgd, unsigned long iova, + unsigned long pg_size, bool *updated) +{ + u64 *pte, *page; + int level, end_level; + + level = get_pgtable_level() - 1; + end_level = page_size_to_level(pg_size); + pte = &pgd[PM_LEVEL_INDEX(level, iova)]; + iova = PAGE_SIZE_ALIGN(iova, PAGE_SIZE); + + while (level >= end_level) { + u64 __pte, __npte; + + __pte = *pte; + + if (IOMMU_PTE_PRESENT(__pte) && is_large_pte(__pte)) { + /* Unmap large pte */ + cmpxchg64(pte, *pte, 0ULL); + *updated = true; + continue; + } + + if (!IOMMU_PTE_PRESENT(__pte)) { + page = alloc_pgtable_page(); + if (!page) + return NULL; + + __npte = set_pgtable_attr(page); + /* pte could have been changed somewhere. */ + if (cmpxchg64(pte, __pte, __npte) != __pte) + free_pgtable_page(page); + else if (IOMMU_PTE_PRESENT(__pte)) + *updated = true; + + continue; + } + + level -= 1; + pte = get_pgtable_pte(__pte); + pte = &pte[PM_LEVEL_INDEX(level, iova)]; + } + + /* Tear down existing pte entries */ + if (IOMMU_PTE_PRESENT(*pte)) { + u64 *__pte; + + *updated = true; + __pte = get_pgtable_pte(*pte); + cmpxchg64(pte, *pte, 0ULL); + if (pg_size == IOMMU_PAGE_SIZE_1G) + free_pgtable(__pte, end_level - 1); + else if (pg_size == IOMMU_PAGE_SIZE_2M) + free_pgtable_page(__pte); + } + + return pte; +} + +/* + * This function checks if there is a PTE for a given dma address. + * If there is one, it returns the pointer to it. + */ +static u64 *fetch_pte(struct amd_io_pgtable *pgtable, + unsigned long iova, unsigned long *page_size) +{ + u64 *pte; + int level; + + level = get_pgtable_level() - 1; + pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)]; + /* Default page size is 4K */ + *page_size = PAGE_SIZE; + + while (level) { + /* Not present */ + if (!IOMMU_PTE_PRESENT(*pte)) + return NULL; + + /* Walk to the next level */ + pte = get_pgtable_pte(*pte); + pte = &pte[PM_LEVEL_INDEX(level - 1, iova)]; + + /* Large page */ + if (is_large_pte(*pte)) { + if (level == PAGE_MODE_3_LEVEL) + *page_size = IOMMU_PAGE_SIZE_1G; + else if (level == PAGE_MODE_2_LEVEL) + *page_size = IOMMU_PAGE_SIZE_2M; + else + return NULL; /* Wrongly set PSE bit in PTE */ + + break; + } + + level -= 1; + } + + return pte; +} + +static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int prot, gfp_t gfp, size_t *mapped) +{ + struct protection_domain *pdom = io_pgtable_ops_to_domain(ops); + struct io_pgtable_cfg *cfg = &pdom->iop.iop.cfg; + u64 *pte; + unsigned long map_size; + unsigned long mapped_size = 0; + unsigned long o_iova = iova; + size_t size = pgcount << __ffs(pgsize); + int count = 0; + int ret = 0; + bool updated = false; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize) || !pgcount) + return -EINVAL; + + if (!(prot & IOMMU_PROT_MASK)) + return -EINVAL; + + while (mapped_size < size) { + map_size = get_alloc_page_size(pgsize); + pte = v2_alloc_pte(pdom->iop.pgd, iova, map_size, &updated); + if (!pte) { + ret = -EINVAL; + goto out; + } + + *pte = set_pte_attr(paddr, map_size, prot); + + count++; + iova += map_size; + paddr += map_size; + mapped_size += map_size; + } + +out: + if (updated) { + if (count > 1) + amd_iommu_flush_tlb(&pdom->domain, 0); + else + amd_iommu_flush_page(&pdom->domain, 0, o_iova); + } + + if (mapped) + *mapped += mapped_size; + + return ret; +} + +static unsigned long iommu_v2_unmap_pages(struct io_pgtable_ops *ops, + unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &pgtable->iop.cfg; + unsigned long unmap_size; + unsigned long unmapped = 0; + size_t size = pgcount << __ffs(pgsize); + u64 *pte; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + while (unmapped < size) { + pte = fetch_pte(pgtable, iova, &unmap_size); + if (!pte) + return unmapped; + + *pte = 0ULL; + + iova = (iova & ~(unmap_size - 1)) + unmap_size; + unmapped += unmap_size; + } + + return unmapped; +} + +static phys_addr_t iommu_v2_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) +{ + struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); + unsigned long offset_mask, pte_pgsize; + u64 *pte, __pte; + + pte = fetch_pte(pgtable, iova, &pte_pgsize); + if (!pte || !IOMMU_PTE_PRESENT(*pte)) + return 0; + + offset_mask = pte_pgsize - 1; + __pte = __sme_clr(*pte & PM_ADDR_MASK); + + return (__pte & ~offset_mask) | (iova & offset_mask); +} + +/* + * ---------------------------------------------------- + */ +static void v2_tlb_flush_all(void *cookie) +{ +} + +static void v2_tlb_flush_walk(unsigned long iova, size_t size, + size_t granule, void *cookie) +{ +} + +static void v2_tlb_add_page(struct iommu_iotlb_gather *gather, + unsigned long iova, size_t granule, + void *cookie) +{ +} + +static const struct iommu_flush_ops v2_flush_ops = { + .tlb_flush_all = v2_tlb_flush_all, + .tlb_flush_walk = v2_tlb_flush_walk, + .tlb_add_page = v2_tlb_add_page, +}; + +static void v2_free_pgtable(struct io_pgtable *iop) +{ + struct protection_domain *pdom; + struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); + + pdom = container_of(pgtable, struct protection_domain, iop); + if (!(pdom->flags & PD_IOMMUV2_MASK)) + return; + + /* + * Make changes visible to IOMMUs. No need to clear gcr3 entry + * as gcr3 table is already freed. + */ + amd_iommu_domain_update(pdom); + + /* Free page table */ + free_pgtable(pgtable->pgd, get_pgtable_level()); +} + +static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); + struct protection_domain *pdom = (struct protection_domain *)cookie; + int ret; + + pgtable->pgd = alloc_pgtable_page(); + if (!pgtable->pgd) + return NULL; + + ret = amd_iommu_domain_set_gcr3(&pdom->domain, 0, iommu_virt_to_phys(pgtable->pgd)); + if (ret) + goto err_free_pgd; + + pgtable->iop.ops.map_pages = iommu_v2_map_pages; + pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages; + pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; + + cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, + cfg->ias = IOMMU_IN_ADDR_BIT_SIZE, + cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, + cfg->tlb = &v2_flush_ops; + + return &pgtable->iop; + +err_free_pgd: + free_pgtable_page(pgtable->pgd); + + return NULL; +} + +struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { + .alloc = v2_alloc_pgtable, + .free = v2_free_pgtable, +}; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 65b8e4fd82177872a81b706b538014f1be75a8b7..65856e401949473b27e4399f0a32e70d1c5614ba 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include #include #include @@ -20,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -42,6 +39,7 @@ #include #include "amd_iommu.h" +#include "../dma-iommu.h" #include "../irq_remapping.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -66,10 +64,6 @@ LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); LIST_HEAD(acpihid_map); -/* - * Domain for untranslated devices - only allocated - * if iommu=pt passed on kernel cmd line. - */ const struct iommu_ops amd_iommu_ops; static ATOMIC_NOTIFIER_HEAD(ppr_notifier); @@ -85,6 +79,7 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void detach_device(struct device *dev); +static int domain_enable_v2(struct protection_domain *domain, int pasids); /**************************************************************************** * @@ -939,7 +934,8 @@ static void build_completion_wait(struct iommu_cmd *cmd, memset(cmd, 0, sizeof(*cmd)); cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK; cmd->data[1] = upper_32_bits(paddr); - cmd->data[2] = data; + cmd->data[2] = lower_32_bits(data); + cmd->data[3] = upper_32_bits(data); CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); } @@ -1596,6 +1592,9 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C; flags |= tmp; + + if (domain->flags & PD_GIOV_MASK) + pte_root |= DTE_FLAG_GIOV; } flags &= ~DEV_DOMID_MASK; @@ -1649,6 +1648,10 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; + /* Override supported page sizes */ + if (domain->flags & PD_GIOV_MASK) + domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; + /* Update device table */ set_dte_entry(iommu, dev_data->devid, domain, ats, dev_data->iommu_v2); @@ -1693,7 +1696,7 @@ static void pdev_iommuv2_disable(struct pci_dev *pdev) pci_disable_pasid(pdev); } -static int pdev_iommuv2_enable(struct pci_dev *pdev) +static int pdev_pri_ats_enable(struct pci_dev *pdev) { int ret; @@ -1756,11 +1759,19 @@ static int attach_device(struct device *dev, struct iommu_domain *def_domain = iommu_get_dma_domain(dev); ret = -EINVAL; - if (def_domain->type != IOMMU_DOMAIN_IDENTITY) + + /* + * In case of using AMD_IOMMU_V1 page table mode and the device + * is enabling for PPR/ATS support (using v2 table), + * we need to make sure that the domain type is identity map. + */ + if ((amd_iommu_pgtable == AMD_IOMMU_V1) && + def_domain->type != IOMMU_DOMAIN_IDENTITY) { goto out; + } if (dev_data->iommu_v2) { - if (pdev_iommuv2_enable(pdev) != 0) + if (pdev_pri_ats_enable(pdev) != 0) goto out; dev_data->ats.enabled = true; @@ -1851,6 +1862,10 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) if (!iommu) return ERR_PTR(-ENODEV); + /* Not registered yet? */ + if (!iommu->iommu.ops) + return ERR_PTR(-ENODEV); + if (dev_iommu_priv_get(dev)) return &iommu->iommu; @@ -1937,25 +1952,6 @@ void amd_iommu_domain_update(struct protection_domain *domain) amd_iommu_domain_flush_complete(domain); } -int __init amd_iommu_init_api(void) -{ - int err; - - err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); - if (err) - return err; -#ifdef CONFIG_ARM_AMBA - err = bus_set_iommu(&amba_bustype, &amd_iommu_ops); - if (err) - return err; -#endif - err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops); - if (err) - return err; - - return 0; -} - /***************************************************************************** * * The following functions belong to the exported interface of AMD IOMMU @@ -1988,12 +1984,12 @@ static void protection_domain_free(struct protection_domain *domain) if (!domain) return; - if (domain->id) - domain_id_free(domain->id); - if (domain->iop.pgtbl_cfg.tlb) free_io_pgtable_ops(&domain->iop.iop.ops); + if (domain->id) + domain_id_free(domain->id); + kfree(domain); } @@ -2011,8 +2007,10 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) if (mode != PAGE_MODE_NONE) { pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pt_root) + if (!pt_root) { + domain_id_free(domain->id); return -ENOMEM; + } } amd_iommu_domain_set_pgtable(domain, pt_root, mode); @@ -2020,6 +2018,24 @@ static int protection_domain_init_v1(struct protection_domain *domain, int mode) return 0; } +static int protection_domain_init_v2(struct protection_domain *domain) +{ + spin_lock_init(&domain->lock); + domain->id = domain_id_alloc(); + if (!domain->id) + return -ENOMEM; + INIT_LIST_HEAD(&domain->dev_list); + + domain->flags |= PD_GIOV_MASK; + + if (domain_enable_v2(domain, 1)) { + domain_id_free(domain->id); + return -ENOMEM; + } + + return 0; +} + static struct protection_domain *protection_domain_alloc(unsigned int type) { struct io_pgtable_ops *pgtbl_ops; @@ -2047,6 +2063,9 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) case AMD_IOMMU_V1: ret = protection_domain_init_v1(domain, mode); break; + case AMD_IOMMU_V2: + ret = protection_domain_init_v2(domain); + break; default: ret = -EINVAL; } @@ -2055,8 +2074,10 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) goto out_err; pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain); - if (!pgtbl_ops) + if (!pgtbl_ops) { + domain_id_free(domain->id); goto out_err; + } return domain; out_err: @@ -2174,13 +2195,13 @@ static void amd_iommu_iotlb_sync_map(struct iommu_domain *dom, struct protection_domain *domain = to_pdomain(dom); struct io_pgtable_ops *ops = &domain->iop.iop.ops; - if (ops->map) + if (ops->map_pages) domain_flush_np_cache(domain, iova, size); } -static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, - phys_addr_t paddr, size_t page_size, int iommu_prot, - gfp_t gfp) +static int amd_iommu_map_pages(struct iommu_domain *dom, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) { struct protection_domain *domain = to_pdomain(dom); struct io_pgtable_ops *ops = &domain->iop.iop.ops; @@ -2196,8 +2217,10 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova, if (iommu_prot & IOMMU_WRITE) prot |= IOMMU_PROT_IW; - if (ops->map) - ret = ops->map(ops, iova, paddr, page_size, prot, gfp); + if (ops->map_pages) { + ret = ops->map_pages(ops, iova, paddr, pgsize, + pgcount, prot, gfp, mapped); + } return ret; } @@ -2223,9 +2246,9 @@ static void amd_iommu_iotlb_gather_add_page(struct iommu_domain *domain, iommu_iotlb_gather_add_range(gather, iova, size); } -static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, - size_t page_size, - struct iommu_iotlb_gather *gather) +static size_t amd_iommu_unmap_pages(struct iommu_domain *dom, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) { struct protection_domain *domain = to_pdomain(dom); struct io_pgtable_ops *ops = &domain->iop.iop.ops; @@ -2235,9 +2258,10 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, (domain->iop.mode == PAGE_MODE_NONE)) return 0; - r = (ops->unmap) ? ops->unmap(ops, iova, page_size, gather) : 0; + r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0; - amd_iommu_iotlb_gather_add_page(dom, gather, iova, page_size); + if (r) + amd_iommu_iotlb_gather_add_page(dom, gather, iova, r); return r; } @@ -2251,7 +2275,7 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, return ops->iova_to_phys(ops, iova); } -static bool amd_iommu_capable(enum iommu_cap cap) +static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: @@ -2399,8 +2423,8 @@ const struct iommu_ops amd_iommu_ops = { .default_domain_ops = &(const struct iommu_domain_ops) { .attach_dev = amd_iommu_attach_device, .detach_dev = amd_iommu_detach_device, - .map = amd_iommu_map, - .unmap = amd_iommu_unmap, + .map_pages = amd_iommu_map_pages, + .unmap_pages = amd_iommu_unmap_pages, .iotlb_sync_map = amd_iommu_iotlb_sync_map, .iova_to_phys = amd_iommu_iova_to_phys, .flush_iotlb_all = amd_iommu_flush_iotlb_all, @@ -2447,11 +2471,10 @@ void amd_iommu_domain_direct_map(struct iommu_domain *dom) } EXPORT_SYMBOL(amd_iommu_domain_direct_map); -int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) +/* Note: This function expects iommu_domain->lock to be held prior calling the function. */ +static int domain_enable_v2(struct protection_domain *domain, int pasids) { - struct protection_domain *domain = to_pdomain(dom); - unsigned long flags; - int levels, ret; + int levels; /* Number of GCR3 table levels required */ for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9) @@ -2460,7 +2483,25 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) if (levels > amd_iommu_max_glx_val) return -EINVAL; - spin_lock_irqsave(&domain->lock, flags); + domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC); + if (domain->gcr3_tbl == NULL) + return -ENOMEM; + + domain->glx = levels; + domain->flags |= PD_IOMMUV2_MASK; + + amd_iommu_domain_update(domain); + + return 0; +} + +int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) +{ + struct protection_domain *pdom = to_pdomain(dom); + unsigned long flags; + int ret; + + spin_lock_irqsave(&pdom->lock, flags); /* * Save us all sanity checks whether devices already in the @@ -2468,24 +2509,14 @@ int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids) * devices attached when it is switched into IOMMUv2 mode. */ ret = -EBUSY; - if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK) + if (pdom->dev_cnt > 0 || pdom->flags & PD_IOMMUV2_MASK) goto out; - ret = -ENOMEM; - domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC); - if (domain->gcr3_tbl == NULL) - goto out; - - domain->glx = levels; - domain->flags |= PD_IOMMUV2_MASK; - - amd_iommu_domain_update(domain); - - ret = 0; + if (!pdom->gcr3_tbl) + ret = domain_enable_v2(pdom, pasids); out: - spin_unlock_irqrestore(&domain->lock, flags); - + spin_unlock_irqrestore(&pdom->lock, flags); return ret; } EXPORT_SYMBOL(amd_iommu_domain_enable_v2); diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index 696d5555be5794bb0ef4329e3608753fee70385f..6a1f02c62dffccd32cdd435f96ba322a7da42a44 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -777,6 +777,8 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) if (dev_state->domain == NULL) goto out_free_states; + /* See iommu_is_default_domain() */ + dev_state->domain->type = IOMMU_DOMAIN_IDENTITY; amd_iommu_domain_direct_map(dev_state->domain); ret = amd_iommu_domain_enable_v2(dev_state->domain, pasids); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1b17257592626bb6a2926e2d05a71f8e5a2c5413..4526575b999e7186bd5ef0fa379f83993420ac1b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -33,6 +32,8 @@ #include #include +#include "dma-iommu.h" + #define DART_MAX_STREAMS 16 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 2 @@ -81,10 +82,16 @@ #define DART_TTBR_VALID BIT(31) #define DART_TTBR_SHIFT 12 +struct apple_dart_hw { + u32 oas; + enum io_pgtable_fmt fmt; +}; + /* * Private structure associated with each DART device. * * @dev: device struct + * @hw: SoC-specific hardware data * @regs: mapped MMIO region * @irq: interrupt number, can be shared with other DARTs * @clks: clocks associated with this DART @@ -98,6 +105,7 @@ */ struct apple_dart { struct device *dev; + const struct apple_dart_hw *hw; void __iomem *regs; @@ -421,13 +429,13 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, .ias = 32, - .oas = 36, + .oas = dart->hw->oas, .coherent_walk = 1, .iommu_dev = dart->dev, }; dart_domain->pgtbl_ops = - alloc_io_pgtable_ops(APPLE_DART, &pgtbl_cfg, domain); + alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain); if (!dart_domain->pgtbl_ops) { ret = -ENOMEM; goto done; @@ -820,27 +828,6 @@ static irqreturn_t apple_dart_irq(int irq, void *dev) return IRQ_HANDLED; } -static int apple_dart_set_bus_ops(const struct iommu_ops *ops) -{ - int ret; - - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, ops); - if (ret) - return ret; - } -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - ret = bus_set_iommu(&pci_bus_type, ops); - if (ret) { - bus_set_iommu(&platform_bus_type, NULL); - return ret; - } - } -#endif - return 0; -} - static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -854,6 +841,7 @@ static int apple_dart_probe(struct platform_device *pdev) return -ENOMEM; dart->dev = dev; + dart->hw = of_device_get_match_data(dev); spin_lock_init(&dart->lock); dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); @@ -895,14 +883,10 @@ static int apple_dart_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dart); - ret = apple_dart_set_bus_ops(&apple_dart_iommu_ops); - if (ret) - goto err_free_irq; - ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", dev_name(&pdev->dev)); if (ret) - goto err_remove_bus_ops; + goto err_free_irq; ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); if (ret) @@ -916,8 +900,6 @@ static int apple_dart_probe(struct platform_device *pdev) err_sysfs_remove: iommu_device_sysfs_remove(&dart->iommu); -err_remove_bus_ops: - apple_dart_set_bus_ops(NULL); err_free_irq: free_irq(dart->irq, dart); err_clk_disable: @@ -932,7 +914,6 @@ static int apple_dart_remove(struct platform_device *pdev) apple_dart_hw_reset(dart); free_irq(dart->irq, dart); - apple_dart_set_bus_ops(NULL); iommu_device_unregister(&dart->iommu); iommu_device_sysfs_remove(&dart->iommu); @@ -942,8 +923,18 @@ static int apple_dart_remove(struct platform_device *pdev) return 0; } +static const struct apple_dart_hw apple_dart_hw_t8103 = { + .oas = 36, + .fmt = APPLE_DART, +}; +static const struct apple_dart_hw apple_dart_hw_t6000 = { + .oas = 42, + .fmt = APPLE_DART2, +}; + static const struct of_device_id apple_dart_of_match[] = { - { .compatible = "apple,t8103-dart", .data = NULL }, + { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, + { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, {}, }; MODULE_DEVICE_TABLE(of, apple_dart_of_match); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 1ef7bbb4acf30d64dc48b3bcc8131a6aec027f67..5968a568aae2afea4d8358c8ecfbb7c69dfaa296 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -150,7 +150,7 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) } reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); - par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT); + par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); cd->ttbr = virt_to_phys(mm->pgd); @@ -425,13 +425,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) * addresses larger than what we support. */ reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); - fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT); + fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); oas = id_aa64mmfr0_parange_to_phys_shift(fld); if (smmu->oas < oas) return false; /* We can support bigger ASIDs than the CPU, but not smaller */ - fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_ASID_SHIFT); + fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); asid_bits = fld ? 16 : 8; if (smmu->asid_bits < asid_bits) return false; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d32b02336411dc647bfd6f79de45dc80cbe9ebb6..ba47c73f5b8c8af09272bb8480f5592a68498010 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -28,9 +27,8 @@ #include #include -#include - #include "arm-smmu-v3.h" +#include "../../dma-iommu.h" #include "../../iommu-sva-lib.h" static bool disable_bypass = true; @@ -1992,11 +1990,14 @@ static const struct iommu_flush_ops arm_smmu_flush_ops = { }; /* IOMMU API */ -static bool arm_smmu_capable(enum iommu_cap cap) +static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) { + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - return true; + /* Assume that a coherent TCU implies coherent TBUs */ + return master->smmu->features & ARM_SMMU_FEAT_COHERENCY; case IOMMU_CAP_NOEXEC: return true; default: @@ -2817,6 +2818,26 @@ static int arm_smmu_dev_disable_feature(struct device *dev, } } +/* + * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the + * PCIe link and save the data to memory by DMA. The hardware is restricted to + * use identity mapping only. + */ +#define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \ + (pdev)->device == 0xa12e) + +static int arm_smmu_def_domain_type(struct device *dev) +{ + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + if (IS_HISI_PTT_DEVICE(pdev)) + return IOMMU_DOMAIN_IDENTITY; + } + + return 0; +} + static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_alloc = arm_smmu_domain_alloc, @@ -2831,6 +2852,7 @@ static struct iommu_ops arm_smmu_ops = { .sva_unbind = arm_smmu_sva_unbind, .sva_get_pasid = arm_smmu_sva_get_pasid, .page_response = arm_smmu_page_response, + .def_domain_type = arm_smmu_def_domain_type, .pgsize_bitmap = -1UL, /* Restricted during device attach */ .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -3673,43 +3695,6 @@ static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) return SZ_128K; } -static int arm_smmu_set_bus_ops(struct iommu_ops *ops) -{ - int err; - -#ifdef CONFIG_PCI - if (pci_bus_type.iommu_ops != ops) { - err = bus_set_iommu(&pci_bus_type, ops); - if (err) - return err; - } -#endif -#ifdef CONFIG_ARM_AMBA - if (amba_bustype.iommu_ops != ops) { - err = bus_set_iommu(&amba_bustype, ops); - if (err) - goto err_reset_pci_ops; - } -#endif - if (platform_bus_type.iommu_ops != ops) { - err = bus_set_iommu(&platform_bus_type, ops); - if (err) - goto err_reset_amba_ops; - } - - return 0; - -err_reset_amba_ops: -#ifdef CONFIG_ARM_AMBA - bus_set_iommu(&amba_bustype, NULL); -#endif -err_reset_pci_ops: __maybe_unused; -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif - return err; -} - static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start, resource_size_t size) { @@ -3848,27 +3833,17 @@ static int arm_smmu_device_probe(struct platform_device *pdev) ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_sysfs_remove; + iommu_device_sysfs_remove(&smmu->iommu); + return ret; } - ret = arm_smmu_set_bus_ops(&arm_smmu_ops); - if (ret) - goto err_unregister_device; - return 0; - -err_unregister_device: - iommu_device_unregister(&smmu->iommu); -err_sysfs_remove: - iommu_device_sysfs_remove(&smmu->iommu); - return ret; } static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - arm_smmu_set_bus_ops(NULL); iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_disable(smmu); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index dfa82df00342b9dc574836e042ffd56b9ccb57c5..6c1114a4d6cc135a432816b37da060aee93a60bf 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -37,10 +36,10 @@ #include #include -#include #include #include "arm-smmu.h" +#include "../../dma-iommu.h" /* * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU @@ -93,8 +92,6 @@ static struct platform_driver arm_smmu_driver; static struct iommu_ops arm_smmu_ops; #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS -static int arm_smmu_bus_init(struct iommu_ops *ops); - static struct device_node *dev_get_dev_node(struct device *dev) { if (dev_is_pci(dev)) { @@ -180,20 +177,6 @@ static int arm_smmu_register_legacy_master(struct device *dev, kfree(sids); return err; } - -/* - * With the legacy DT binding in play, we have no guarantees about - * probe order, but then we're also not doing default domains, so we can - * delay setting bus ops until we're sure every possible SMMU is ready, - * and that way ensure that no probe_device() calls get missed. - */ -static int arm_smmu_legacy_bus_init(void) -{ - if (using_legacy_binding) - return arm_smmu_bus_init(&arm_smmu_ops); - return 0; -} -device_initcall_sync(arm_smmu_legacy_bus_init); #else static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) @@ -1330,15 +1313,14 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, return ops->iova_to_phys(ops, iova); } -static bool arm_smmu_capable(enum iommu_cap cap) +static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) { + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); + switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - /* - * Return true here as the SMMU can always send out coherent - * requests. - */ - return true; + /* Assume that a coherent TCU implies coherent TBUs */ + return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; case IOMMU_CAP_NOEXEC: return true; default: @@ -2016,52 +1998,6 @@ static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, return 0; } -static int arm_smmu_bus_init(struct iommu_ops *ops) -{ - int err; - - /* Oh, for a proper bus abstraction */ - if (!iommu_present(&platform_bus_type)) { - err = bus_set_iommu(&platform_bus_type, ops); - if (err) - return err; - } -#ifdef CONFIG_ARM_AMBA - if (!iommu_present(&amba_bustype)) { - err = bus_set_iommu(&amba_bustype, ops); - if (err) - goto err_reset_platform_ops; - } -#endif -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - err = bus_set_iommu(&pci_bus_type, ops); - if (err) - goto err_reset_amba_ops; - } -#endif -#ifdef CONFIG_FSL_MC_BUS - if (!iommu_present(&fsl_mc_bus_type)) { - err = bus_set_iommu(&fsl_mc_bus_type, ops); - if (err) - goto err_reset_pci_ops; - } -#endif - return 0; - -err_reset_pci_ops: __maybe_unused; -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif -err_reset_amba_ops: __maybe_unused; -#ifdef CONFIG_ARM_AMBA - bus_set_iommu(&amba_bustype, NULL); -#endif -err_reset_platform_ops: __maybe_unused; - bus_set_iommu(&platform_bus_type, NULL); - return err; -} - static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) { struct list_head rmr_list; @@ -2226,7 +2162,8 @@ static int arm_smmu_device_probe(struct platform_device *pdev) err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (err) { dev_err(dev, "Failed to register iommu\n"); - goto err_sysfs_remove; + iommu_device_sysfs_remove(&smmu->iommu); + return err; } platform_set_drvdata(pdev, smmu); @@ -2248,24 +2185,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) pm_runtime_enable(dev); } - /* - * For ACPI and generic DT bindings, an SMMU will be probed before - * any device which might need it, so we want the bus ops in place - * ready to handle default domain setup as soon as any SMMU exists. - */ - if (!using_legacy_binding) { - err = arm_smmu_bus_init(&arm_smmu_ops); - if (err) - goto err_unregister_device; - } - return 0; - -err_unregister_device: - iommu_device_unregister(&smmu->iommu); -err_sysfs_remove: - iommu_device_sysfs_remove(&smmu->iommu); - return err; } static int arm_smmu_device_remove(struct platform_device *pdev) @@ -2278,7 +2198,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev) if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_notice(&pdev->dev, "disabling translation\n"); - arm_smmu_bus_init(NULL); iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 17235116d3bbec1c928faa4c1973e68ce1a43ffd..3869c3ecda8cd1193cfdbeec0ac2d235ed3d04de 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -493,7 +493,7 @@ static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain, return ret; } -static bool qcom_iommu_capable(enum iommu_cap cap) +static bool qcom_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: @@ -837,8 +837,6 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) goto err_pm_disable; } - bus_set_iommu(&platform_bus_type, &qcom_iommu_ops); - if (qcom_iommu->local_base) { pm_runtime_get_sync(dev); writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS); @@ -856,8 +854,6 @@ static int qcom_iommu_device_remove(struct platform_device *pdev) { struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev); - bus_set_iommu(&platform_bus_type, NULL); - pm_runtime_force_suspend(&pdev->dev); platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&qcom_iommu->iommu); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 17dd683b2fceaa8c667773d47886801e35411767..9297b741f5e80e2408e864fc3f779410d6b04d49 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -30,6 +29,8 @@ #include #include +#include "dma-iommu.h" + struct iommu_dma_msi_page { struct list_head list; dma_addr_t iova; @@ -1633,6 +1634,13 @@ out_free_page: return NULL; } +/** + * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain + * @desc: MSI descriptor, will store the MSI page + * @msi_addr: MSI target address to be mapped + * + * Return: 0 on success or negative error code if the mapping failed. + */ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) { struct device *dev = msi_desc_to_dev(desc); @@ -1661,8 +1669,12 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) return 0; } -void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg) +/** + * iommu_dma_compose_msi_msg() - Apply translation to an MSI message + * @desc: MSI descriptor prepared by iommu_dma_prepare_msi() + * @msg: MSI message containing target physical address + */ +void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) { struct device *dev = msi_desc_to_dev(desc); const struct iommu_domain *domain = iommu_get_domain_for_dev(dev); diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h new file mode 100644 index 0000000000000000000000000000000000000000..9427900092922c3f3faa02e5bdeb67b63708d9b6 --- /dev/null +++ b/drivers/iommu/dma-iommu.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014-2015 ARM Ltd. + */ +#ifndef __DMA_IOMMU_H +#define __DMA_IOMMU_H + +#include + +#ifdef CONFIG_IOMMU_DMA + +int iommu_get_dma_cookie(struct iommu_domain *domain); +void iommu_put_dma_cookie(struct iommu_domain *domain); + +int iommu_dma_init_fq(struct iommu_domain *domain); + +void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); + +extern bool iommu_dma_forcedac; + +#else /* CONFIG_IOMMU_DMA */ + +static inline int iommu_dma_init_fq(struct iommu_domain *domain) +{ + return -EINVAL; +} + +static inline int iommu_get_dma_cookie(struct iommu_domain *domain) +{ + return -ENODEV; +} + +static inline void iommu_put_dma_cookie(struct iommu_domain *domain) +{ +} + +static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) +{ +} + +#endif /* CONFIG_IOMMU_DMA */ +#endif /* __DMA_IOMMU_H */ diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8e18984a0c4ff13d211da0ebdf26c467e58f9161..45fd4850bacbdbc53d58faa0f87da6ed50563236 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1446,16 +1446,7 @@ static int __init exynos_iommu_init(void) goto err_zero_lv2; } - ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); - if (ret) { - pr_err("%s: Failed to register exynos-iommu driver.\n", - __func__); - goto err_set_iommu; - } - return 0; -err_set_iommu: - kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); err_zero_lv2: platform_driver_unregister(&exynos_sysmmu_driver); err_reg_driver: diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 011f9ab7f74394243b3d728b9e703c4422c772cd..fa20f4b03e12db026b41e8017f7859d4b9e227bd 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -178,7 +178,7 @@ static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, return iova; } -static bool fsl_pamu_capable(enum iommu_cap cap) +static bool fsl_pamu_capable(struct device *dev, enum iommu_cap cap) { return cap == IOMMU_CAP_CACHE_COHERENCY; } @@ -476,11 +476,7 @@ int __init pamu_domain_init(void) if (ret) { iommu_device_sysfs_remove(&pamu_iommu); pr_err("Can't register iommu device\n"); - return ret; } - bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); - bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); - return ret; } diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 39a06d245f12e6a422d2bab42dd28dbc60423bac..b7dff5092fd21c06bd73e293548d3af95c7072f6 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -19,8 +19,9 @@ config INTEL_IOMMU select DMAR_TABLE select SWIOTLB select IOASID - select IOMMU_DMA select PCI_ATS + select PCI_PRI + select PCI_PASID help DMA remapping (DMAR) devices support enables independent address translations for Direct Memory Access (DMA) from devices. @@ -48,10 +49,7 @@ config INTEL_IOMMU_DEBUGFS config INTEL_IOMMU_SVM bool "Support for Shared Virtual Memory with Intel IOMMU" depends on X86_64 - select PCI_PASID - select PCI_PRI select MMU_NOTIFIER - select IOASID select IOMMU_SVA help Shared Virtual Memory (SVM) provides a facility for devices diff --git a/drivers/iommu/intel/cap_audit.c b/drivers/iommu/intel/cap_audit.c index 3ee68393122ffe3b40123c90af7fe562a35ab58e..806986696841405af1c235c686722b801d919a6f 100644 --- a/drivers/iommu/intel/cap_audit.c +++ b/drivers/iommu/intel/cap_audit.c @@ -37,7 +37,7 @@ static inline void check_dmar_capabilities(struct intel_iommu *a, MINIMAL_FEATURE_IOMMU(b, ecap, ECAP_MHMV_MASK); MINIMAL_FEATURE_IOMMU(b, ecap, ECAP_IRO_MASK); - CHECK_FEATURE_MISMATCH(a, b, cap, 5lp_support, CAP_FL5LP_MASK); + CHECK_FEATURE_MISMATCH(a, b, cap, fl5lp_support, CAP_FL5LP_MASK); CHECK_FEATURE_MISMATCH(a, b, cap, fl1gp_support, CAP_FL1GP_MASK); CHECK_FEATURE_MISMATCH(a, b, cap, read_drain, CAP_RD_MASK); CHECK_FEATURE_MISMATCH(a, b, cap, write_drain, CAP_WD_MASK); @@ -84,7 +84,7 @@ static int cap_audit_hotplug(struct intel_iommu *iommu, enum cap_audit_type type goto out; } - CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, 5lp_support, CAP_FL5LP_MASK); + CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, fl5lp_support, CAP_FL5LP_MASK); CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, fl1gp_support, CAP_FL1GP_MASK); CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, read_drain, CAP_RD_MASK); CHECK_FEATURE_MISMATCH_HOTPLUG(iommu, cap, write_drain, CAP_WD_MASK); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 7cca030a508e1e22603a369335d6033b385f1b90..a8b36c3fddf1a5c5a4a058412f63bf5429199cbc 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -26,6 +25,7 @@ #include #include "iommu.h" +#include "../dma-iommu.h" #include "../irq_remapping.h" #include "../iommu-sva-lib.h" #include "pasid.h" @@ -163,38 +163,6 @@ static phys_addr_t root_entry_uctp(struct root_entry *re) return re->hi & VTD_PAGE_MASK; } -static inline void context_clear_pasid_enable(struct context_entry *context) -{ - context->lo &= ~(1ULL << 11); -} - -static inline bool context_pasid_enabled(struct context_entry *context) -{ - return !!(context->lo & (1ULL << 11)); -} - -static inline void context_set_copied(struct context_entry *context) -{ - context->hi |= (1ull << 3); -} - -static inline bool context_copied(struct context_entry *context) -{ - return !!(context->hi & (1ULL << 3)); -} - -static inline bool __context_present(struct context_entry *context) -{ - return (context->lo & 1); -} - -bool context_present(struct context_entry *context) -{ - return context_pasid_enabled(context) ? - __context_present(context) : - __context_present(context) && !context_copied(context); -} - static inline void context_set_present(struct context_entry *context) { context->lo |= 1; @@ -231,6 +199,11 @@ static inline void context_set_domain_id(struct context_entry *context, context->hi |= (value & ((1 << 16) - 1)) << 8; } +static inline void context_set_pasid(struct context_entry *context) +{ + context->lo |= CONTEXT_PASIDE; +} + static inline int context_domain_id(struct context_entry *c) { return((c->hi >> 8) & 0xffff); @@ -242,6 +215,26 @@ static inline void context_clear_entry(struct context_entry *context) context->hi = 0; } +static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) +{ + if (!iommu->copied_tables) + return false; + + return test_bit(((long)bus << 8) | devfn, iommu->copied_tables); +} + +static inline void +set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) +{ + set_bit(((long)bus << 8) | devfn, iommu->copied_tables); +} + +static inline void +clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn) +{ + clear_bit(((long)bus << 8) | devfn, iommu->copied_tables); +} + /* * This domain is a statically identity mapping domain. * 1. This domain creats a static 1:1 mapping to all usable memory. @@ -402,14 +395,36 @@ static inline int domain_pfn_supported(struct dmar_domain *domain, return !(addr_width < BITS_PER_LONG && pfn >> addr_width); } +/* + * Calculate the Supported Adjusted Guest Address Widths of an IOMMU. + * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of + * the returned SAGAW. + */ +static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu) +{ + unsigned long fl_sagaw, sl_sagaw; + + fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0); + sl_sagaw = cap_sagaw(iommu->cap); + + /* Second level only. */ + if (!sm_supported(iommu) || !ecap_flts(iommu->ecap)) + return sl_sagaw; + + /* First level only. */ + if (!ecap_slts(iommu->ecap)) + return fl_sagaw; + + return fl_sagaw & sl_sagaw; +} + static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw) { unsigned long sagaw; int agaw; - sagaw = cap_sagaw(iommu->cap); - for (agaw = width_to_agaw(max_gaw); - agaw >= 0; agaw--) { + sagaw = __iommu_calculate_sagaw(iommu); + for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) { if (test_bit(agaw, &sagaw)) break; } @@ -505,8 +520,9 @@ static int domain_update_device_node(struct dmar_domain *domain) { struct device_domain_info *info; int nid = NUMA_NO_NODE; + unsigned long flags; - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(info, &domain->devices, link) { /* * There could possibly be multiple device numa nodes as devices @@ -518,7 +534,7 @@ static int domain_update_device_node(struct dmar_domain *domain) if (nid != NUMA_NO_NODE) break; } - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); return nid; } @@ -578,6 +594,13 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, struct context_entry *context; u64 *entry; + /* + * Except that the caller requested to allocate a new entry, + * returning a copied context entry makes no sense. + */ + if (!alloc && context_copied(iommu, bus, devfn)) + return NULL; + entry = &root->lo; if (sm_supported(iommu)) { if (devfn >= 0x80) { @@ -795,32 +818,11 @@ static void free_context_table(struct intel_iommu *iommu) } #ifdef CONFIG_DMAR_DEBUG -static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn, u8 bus, u8 devfn) +static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn, + u8 bus, u8 devfn, struct dma_pte *parent, int level) { - struct device_domain_info *info; - struct dma_pte *parent, *pte; - struct dmar_domain *domain; - struct pci_dev *pdev; - int offset, level; - - pdev = pci_get_domain_bus_and_slot(iommu->segment, bus, devfn); - if (!pdev) - return; - - info = dev_iommu_priv_get(&pdev->dev); - if (!info || !info->domain) { - pr_info("device [%02x:%02x.%d] not probed\n", - bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); - return; - } - - domain = info->domain; - level = agaw_to_level(domain->agaw); - parent = domain->pgd; - if (!parent) { - pr_info("no page table setup\n"); - return; - } + struct dma_pte *pte; + int offset; while (1) { offset = pfn_level_offset(pfn, level); @@ -847,9 +849,10 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, struct pasid_entry *entries, *pte; struct context_entry *ctx_entry; struct root_entry *rt_entry; + int i, dir_index, index, level; u8 devfn = source_id & 0xff; u8 bus = source_id >> 8; - int i, dir_index, index; + struct dma_pte *pgtable; pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr); @@ -877,8 +880,11 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, ctx_entry->hi, ctx_entry->lo); /* legacy mode does not require PASID entries */ - if (!sm_supported(iommu)) + if (!sm_supported(iommu)) { + level = agaw_to_level(ctx_entry->hi & 7); + pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK); goto pgtable_walk; + } /* get the pointer to pasid directory entry */ dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK); @@ -905,8 +911,16 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, for (i = 0; i < ARRAY_SIZE(pte->val); i++) pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]); + if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) { + level = pte->val[2] & BIT_ULL(2) ? 5 : 4; + pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK); + } else { + level = agaw_to_level((pte->val[0] >> 2) & 0x7); + pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK); + } + pgtable_walk: - pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn); + pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level); } #endif @@ -1225,6 +1239,13 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) raw_spin_unlock_irqrestore(&iommu->register_lock, flag); + /* + * Hardware invalidates all DMA remapping hardware translation + * caches as part of SRTP flow. + */ + if (cap_esrtps(iommu->cap)) + return; + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); if (sm_supported(iommu)) qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0); @@ -1341,23 +1362,21 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, } static struct device_domain_info * -iommu_support_dev_iotlb(struct dmar_domain *domain, struct intel_iommu *iommu, - u8 bus, u8 devfn) +domain_lookup_dev_info(struct dmar_domain *domain, + struct intel_iommu *iommu, u8 bus, u8 devfn) { struct device_domain_info *info; + unsigned long flags; - if (!iommu->qi) - return NULL; - - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(info, &domain->devices, link) { if (info->iommu == iommu && info->bus == bus && info->devfn == devfn) { - spin_unlock(&domain->lock); - return info->ats_supported ? info : NULL; + spin_unlock_irqrestore(&domain->lock, flags); + return info; } } - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); return NULL; } @@ -1366,8 +1385,9 @@ static void domain_update_iotlb(struct dmar_domain *domain) { struct device_domain_info *info; bool has_iotlb_device = false; + unsigned long flags; - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(info, &domain->devices, link) { if (info->ats_enabled) { has_iotlb_device = true; @@ -1375,10 +1395,10 @@ static void domain_update_iotlb(struct dmar_domain *domain) } } domain->has_iotlb_device = has_iotlb_device; - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); } -static void iommu_enable_dev_iotlb(struct device_domain_info *info) +static void iommu_enable_pci_caps(struct device_domain_info *info) { struct pci_dev *pdev; @@ -1401,7 +1421,6 @@ static void iommu_enable_dev_iotlb(struct device_domain_info *info) info->pfsid = pci_dev_id(pf_pdev); } -#ifdef CONFIG_INTEL_IOMMU_SVM /* The PCIe spec, in its wisdom, declares that the behaviour of the device if you enable PASID support after ATS support is undefined. So always enable PASID support on devices which @@ -1414,7 +1433,7 @@ static void iommu_enable_dev_iotlb(struct device_domain_info *info) (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, PRQ_DEPTH)) info->pri_enabled = 1; -#endif + if (info->ats_supported && pci_ats_page_aligned(pdev) && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) { info->ats_enabled = 1; @@ -1437,16 +1456,16 @@ static void iommu_disable_dev_iotlb(struct device_domain_info *info) info->ats_enabled = 0; domain_update_iotlb(info->domain); } -#ifdef CONFIG_INTEL_IOMMU_SVM + if (info->pri_enabled) { pci_disable_pri(pdev); info->pri_enabled = 0; } + if (info->pasid_enabled) { pci_disable_pasid(pdev); info->pasid_enabled = 0; } -#endif } static void __iommu_flush_dev_iotlb(struct device_domain_info *info, @@ -1467,14 +1486,15 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, u64 addr, unsigned mask) { struct device_domain_info *info; + unsigned long flags; if (!domain->has_iotlb_device) return; - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_for_each_entry(info, &domain->devices, link) __iommu_flush_dev_iotlb(info, addr, mask); - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); } static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, @@ -1688,6 +1708,11 @@ static void free_dmar_iommu(struct intel_iommu *iommu) iommu->domain_ids = NULL; } + if (iommu->copied_tables) { + bitmap_free(iommu->copied_tables); + iommu->copied_tables = NULL; + } + /* free context mapping */ free_context_table(iommu); @@ -1890,7 +1915,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, u8 bus, u8 devfn) { struct device_domain_info *info = - iommu_support_dev_iotlb(domain, iommu, bus, devfn); + domain_lookup_dev_info(domain, iommu, bus, devfn); u16 did = domain_id_iommu(domain, iommu); int translation = CONTEXT_TT_MULTI_LEVEL; struct context_entry *context; @@ -1913,7 +1938,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, goto out_unlock; ret = 0; - if (context_present(context)) + if (context_present(context) && !context_copied(iommu, bus, devfn)) goto out_unlock; /* @@ -1925,7 +1950,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, * in-flight DMA will exist, and we don't need to worry anymore * hereafter. */ - if (context_copied(context)) { + if (context_copied(iommu, bus, devfn)) { u16 did_old = context_domain_id(context); if (did_old < cap_ndoms(iommu->cap)) { @@ -1936,6 +1961,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain, iommu->flush.flush_iotlb(iommu, did_old, 0, 0, DMA_TLB_DSI_FLUSH); } + + clear_context_copied(iommu, bus, devfn); } context_clear_entry(context); @@ -1961,6 +1988,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain, context_set_sm_dte(context); if (info && info->pri_supported) context_set_sm_pre(context); + if (info && info->pasid_supported) + context_set_pasid(context); } else { struct dma_pte *pgd = domain->pgd; int agaw; @@ -2018,7 +2047,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, } else { iommu_flush_write_buffer(iommu); } - iommu_enable_dev_iotlb(info); + iommu_enable_pci_caps(info); ret = 0; @@ -2429,6 +2458,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu; + unsigned long flags; u8 bus, devfn; int ret; @@ -2440,9 +2470,9 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) if (ret) return ret; info->domain = domain; - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_add(&info->link, &domain->devices); - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); /* PASID table is mandatory for a PCI device in scalable mode. */ if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) { @@ -2684,32 +2714,14 @@ static int copy_context_table(struct intel_iommu *iommu, /* Now copy the context entry */ memcpy(&ce, old_ce + idx, sizeof(ce)); - if (!__context_present(&ce)) + if (!context_present(&ce)) continue; did = context_domain_id(&ce); if (did >= 0 && did < cap_ndoms(iommu->cap)) set_bit(did, iommu->domain_ids); - /* - * We need a marker for copied context entries. This - * marker needs to work for the old format as well as - * for extended context entries. - * - * Bit 67 of the context entry is used. In the old - * format this bit is available to software, in the - * extended format it is the PGE bit, but PGE is ignored - * by HW if PASIDs are disabled (and thus still - * available). - * - * So disable PASIDs first and then mark the entry - * copied. This means that we don't copy PASID - * translations from the old kernel, but this is fine as - * faults there are not fatal. - */ - context_clear_pasid_enable(&ce); - context_set_copied(&ce); - + set_context_copied(iommu, bus, devfn); new_ce[idx] = ce; } @@ -2735,8 +2747,8 @@ static int copy_translation_tables(struct intel_iommu *iommu) bool new_ext, ext; rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); - ext = !!(rtaddr_reg & DMA_RTADDR_RTT); - new_ext = !!ecap_ecs(iommu->ecap); + ext = !!(rtaddr_reg & DMA_RTADDR_SMT); + new_ext = !!sm_supported(iommu); /* * The RTT bit can only be changed when translation is disabled, @@ -2747,6 +2759,10 @@ static int copy_translation_tables(struct intel_iommu *iommu) if (new_ext != ext) return -EINVAL; + iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL); + if (!iommu->copied_tables) + return -ENOMEM; + old_rt_phys = rtaddr_reg & VTD_PAGE_MASK; if (!old_rt_phys) return -EINVAL; @@ -3890,7 +3906,6 @@ static int __init probe_acpi_namespace_devices(void) continue; } - pn->dev->bus->iommu_ops = &intel_iommu_ops; ret = iommu_probe_device(pn->dev); if (ret) break; @@ -4023,7 +4038,6 @@ int __init intel_iommu_init(void) } up_read(&dmar_global_lock); - bus_set_iommu(&pci_bus_type, &intel_iommu_ops); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); @@ -4080,6 +4094,7 @@ static void dmar_remove_one_dev_info(struct device *dev) struct device_domain_info *info = dev_iommu_priv_get(dev); struct dmar_domain *domain = info->domain; struct intel_iommu *iommu = info->iommu; + unsigned long flags; if (!dev_is_real_dma_subdevice(info->dev)) { if (dev_is_pci(info->dev) && sm_supported(iommu)) @@ -4091,9 +4106,9 @@ static void dmar_remove_one_dev_info(struct device *dev) intel_pasid_free_table(info->dev); } - spin_lock(&domain->lock); + spin_lock_irqsave(&domain->lock, flags); list_del(&info->link); - spin_unlock(&domain->lock); + spin_unlock_irqrestore(&domain->lock, flags); domain_detach_iommu(domain, iommu); info->domain = NULL; @@ -4412,24 +4427,25 @@ static void domain_set_force_snooping(struct dmar_domain *domain) static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) { struct dmar_domain *dmar_domain = to_dmar_domain(domain); + unsigned long flags; if (dmar_domain->force_snooping) return true; - spin_lock(&dmar_domain->lock); + spin_lock_irqsave(&dmar_domain->lock, flags); if (!domain_support_force_snooping(dmar_domain)) { - spin_unlock(&dmar_domain->lock); + spin_unlock_irqrestore(&dmar_domain->lock, flags); return false; } domain_set_force_snooping(dmar_domain); dmar_domain->force_snooping = true; - spin_unlock(&dmar_domain->lock); + spin_unlock_irqrestore(&dmar_domain->lock, flags); return true; } -static bool intel_iommu_capable(enum iommu_cap cap) +static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) return true; @@ -4449,7 +4465,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) u8 bus, devfn; iommu = device_to_iommu(dev, &bus, &devfn); - if (!iommu) + if (!iommu || !iommu->iommu.ops) return ERR_PTR(-ENODEV); info = kzalloc(sizeof(*info), GFP_KERNEL); @@ -4566,52 +4582,6 @@ static void intel_iommu_get_resv_regions(struct device *device, list_add_tail(®->list, head); } -int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev) -{ - struct device_domain_info *info = dev_iommu_priv_get(dev); - struct context_entry *context; - struct dmar_domain *domain; - u64 ctx_lo; - int ret; - - domain = info->domain; - if (!domain) - return -EINVAL; - - spin_lock(&iommu->lock); - ret = -EINVAL; - if (!info->pasid_supported) - goto out; - - context = iommu_context_addr(iommu, info->bus, info->devfn, 0); - if (WARN_ON(!context)) - goto out; - - ctx_lo = context[0].lo; - - if (!(ctx_lo & CONTEXT_PASIDE)) { - ctx_lo |= CONTEXT_PASIDE; - context[0].lo = ctx_lo; - wmb(); - iommu->flush.flush_context(iommu, - domain_id_iommu(domain, iommu), - PCI_DEVID(info->bus, info->devfn), - DMA_CCMD_MASK_NOBIT, - DMA_CCMD_DEVICE_INVL); - } - - /* Enable PASID support in the device, if it wasn't already */ - if (!info->pasid_enabled) - iommu_enable_dev_iotlb(info); - - ret = 0; - - out: - spin_unlock(&iommu->lock); - - return ret; -} - static struct iommu_group *intel_iommu_device_group(struct device *dev) { if (dev_is_pci(dev)) @@ -4635,9 +4605,6 @@ static int intel_iommu_enable_sva(struct device *dev) if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE)) return -ENODEV; - if (intel_iommu_enable_pasid(iommu, dev)) - return -ENODEV; - if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled) return -EINVAL; diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index fae45bbb0c7f7a354f6e8a0a71a16e7dda9d063b..92023dff9513ab1fe7f71176a1c7326d735e608b 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -146,7 +146,9 @@ /* * Decoding Capability Register */ -#define cap_5lp_support(c) (((c) >> 60) & 1) +#define cap_esrtps(c) (((c) >> 63) & 1) +#define cap_esirtps(c) (((c) >> 62) & 1) +#define cap_fl5lp_support(c) (((c) >> 60) & 1) #define cap_pi_support(c) (((c) >> 59) & 1) #define cap_fl1gp_support(c) (((c) >> 56) & 1) #define cap_read_drain(c) (((c) >> 55) & 1) @@ -197,7 +199,6 @@ #define ecap_dis(e) (((e) >> 27) & 0x1) #define ecap_nest(e) (((e) >> 26) & 0x1) #define ecap_mts(e) (((e) >> 25) & 0x1) -#define ecap_ecs(e) (((e) >> 24) & 0x1) #define ecap_iotlb_offset(e) ((((e) >> 8) & 0x3ff) * 16) #define ecap_max_iotlb_offset(e) (ecap_iotlb_offset(e) + 16) #define ecap_coherent(e) ((e) & 0x1) @@ -265,7 +266,6 @@ #define DMA_GSTS_CFIS (((u32)1) << 23) /* DMA_RTADDR_REG */ -#define DMA_RTADDR_RTT (((u64)1) << 11) #define DMA_RTADDR_SMT (((u64)1) << 10) /* CCMD_REG */ @@ -579,6 +579,7 @@ struct intel_iommu { #ifdef CONFIG_INTEL_IOMMU unsigned long *domain_ids; /* bitmap of domains */ + unsigned long *copied_tables; /* bitmap of copied tables */ spinlock_t lock; /* protect context, domain ids */ struct root_entry *root_entry; /* virtual address */ @@ -587,6 +588,7 @@ struct intel_iommu { #ifdef CONFIG_INTEL_IOMMU_SVM struct page_req_dsc *prq; unsigned char prq_name[16]; /* Name for PRQ interrupt */ + unsigned long prq_seq_number; struct completion prq_complete; struct ioasid_allocator_ops pasid_allocator; /* Custom allocator for PASIDs */ #endif @@ -701,6 +703,11 @@ static inline int nr_pte_to_next_page(struct dma_pte *pte) (struct dma_pte *)ALIGN((unsigned long)pte, VTD_PAGE_SIZE) - pte; } +static inline bool context_present(struct context_entry *context) +{ + return (context->lo & 1); +} + extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); extern int dmar_enable_qi(struct intel_iommu *iommu); @@ -737,7 +744,6 @@ extern int dmar_ir_support(void); void *alloc_pgtable_page(int node); void free_pgtable_page(void *vaddr); void iommu_flush_write_buffer(struct intel_iommu *iommu); -int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev); struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); #ifdef CONFIG_INTEL_IOMMU_SVM @@ -757,7 +763,6 @@ struct intel_svm_dev { struct device *dev; struct intel_iommu *iommu; struct iommu_sva sva; - unsigned long prq_seq_number; u32 pasid; int users; u16 did; @@ -784,7 +789,6 @@ static inline void intel_iommu_debugfs_init(void) {} #endif /* CONFIG_INTEL_IOMMU_DEBUGFS */ extern const struct attribute_group *intel_iommu_groups[]; -bool context_present(struct context_entry *context); struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, u8 devfn, int alloc); diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 2e9683e970f8e50b6f91011803cd40b3be6af01c..5962bb5027d06af10f93c56c54cdb31b541b0178 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -494,7 +494,8 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) * Global invalidation of interrupt entry cache to make sure the * hardware uses the new irq remapping table. */ - qi_global_iec(iommu); + if (!cap_esirtps(iommu->cap)) + qi_global_iec(iommu); } static void iommu_enable_irq_remapping(struct intel_iommu *iommu) @@ -680,7 +681,8 @@ static void iommu_disable_irq_remapping(struct intel_iommu *iommu) * global invalidation of interrupt entry cache before disabling * interrupt-remapping. */ - qi_global_iec(iommu); + if (!cap_esirtps(iommu->cap)) + qi_global_iec(iommu); raw_spin_lock_irqsave(&iommu->register_lock, flags); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index c5e7e8b020a5727f5168d7837d2847e99a6d8b37..c30ddac40ee5fc2d5ce79cc1353854959bf5a998 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -392,16 +392,6 @@ pasid_set_flpm(struct pasid_entry *pe, u64 value) pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2); } -/* - * Setup the Extended Access Flag Enable (EAFE) field (Bit 135) - * of a scalable mode PASID entry. - */ -static inline void -pasid_set_eafe(struct pasid_entry *pe) -{ - pasid_set_bits(&pe->val[2], 1 << 7, 1 << 7); -} - static void pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid) @@ -529,7 +519,7 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, } } - if ((flags & PASID_FLAG_FL5LP) && !cap_5lp_support(iommu->cap)) { + if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) { pr_err("No 5-level paging support for first-level on %s\n", iommu->name); return -EINVAL; diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 8bcfb93dda566763da2540ac0092d6741675cd82..7d08eb034f2d2e2bfb72d2a1f69b48fcc0a46a11 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -48,23 +48,6 @@ static void *pasid_private_find(ioasid_t pasid) return xa_load(&pasid_private_array, pasid); } -static struct intel_svm_dev * -svm_lookup_device_by_sid(struct intel_svm *svm, u16 sid) -{ - struct intel_svm_dev *sdev = NULL, *t; - - rcu_read_lock(); - list_for_each_entry_rcu(t, &svm->devs, list) { - if (t->sid == sid) { - sdev = t; - break; - } - } - rcu_read_unlock(); - - return sdev; -} - static struct intel_svm_dev * svm_lookup_device_by_dev(struct intel_svm *svm, struct device *dev) { @@ -181,7 +164,7 @@ void intel_svm_check(struct intel_iommu *iommu) } if (cpu_feature_enabled(X86_FEATURE_LA57) && - !cap_5lp_support(iommu->cap)) { + !cap_fl5lp_support(iommu->cap)) { pr_err("%s SVM disabled, incompatible paging mode\n", iommu->name); return; @@ -706,11 +689,10 @@ static void handle_bad_prq_event(struct intel_iommu *iommu, static irqreturn_t prq_event_thread(int irq, void *d) { - struct intel_svm_dev *sdev = NULL; struct intel_iommu *iommu = d; - struct intel_svm *svm = NULL; struct page_req_dsc *req; int head, tail, handled; + struct pci_dev *pdev; u64 address; /* @@ -730,8 +712,6 @@ static irqreturn_t prq_event_thread(int irq, void *d) pr_err("IOMMU: %s: Page request without PASID\n", iommu->name); bad_req: - svm = NULL; - sdev = NULL; handle_bad_prq_event(iommu, req, QI_RESP_INVALID); goto prq_advance; } @@ -758,34 +738,19 @@ bad_req: if (unlikely(req->lpig && !req->rd_req && !req->wr_req)) goto prq_advance; - if (!svm || svm->pasid != req->pasid) { - /* - * It can't go away, because the driver is not permitted - * to unbind the mm while any page faults are outstanding. - */ - svm = pasid_private_find(req->pasid); - if (IS_ERR_OR_NULL(svm) || (svm->flags & SVM_FLAG_SUPERVISOR_MODE)) - goto bad_req; - } - - if (!sdev || sdev->sid != req->rid) { - sdev = svm_lookup_device_by_sid(svm, req->rid); - if (!sdev) - goto bad_req; - } - - sdev->prq_seq_number++; - + pdev = pci_get_domain_bus_and_slot(iommu->segment, + PCI_BUS_NUM(req->rid), + req->rid & 0xff); /* * If prq is to be handled outside iommu driver via receiver of * the fault notifiers, we skip the page response here. */ - if (intel_svm_prq_report(iommu, sdev->dev, req)) + if (!pdev || intel_svm_prq_report(iommu, &pdev->dev, req)) handle_bad_prq_event(iommu, req, QI_RESP_INVALID); - trace_prq_report(iommu, sdev->dev, req->qw_0, req->qw_1, + trace_prq_report(iommu, &pdev->dev, req->qw_0, req->qw_1, req->priv_data[0], req->priv_data[1], - sdev->prq_seq_number); + iommu->prq_seq_number++); prq_advance: head = (head + sizeof(*req)) & PRQ_RING_MASK; } @@ -881,8 +846,6 @@ int intel_svm_page_response(struct device *dev, struct iommu_page_response *msg) { struct iommu_fault_page_request *prm; - struct intel_svm_dev *sdev = NULL; - struct intel_svm *svm = NULL; struct intel_iommu *iommu; bool private_present; bool pasid_present; @@ -901,8 +864,6 @@ int intel_svm_page_response(struct device *dev, if (!msg || !evt) return -EINVAL; - mutex_lock(&pasid_mutex); - prm = &evt->fault.prm; sid = PCI_DEVID(bus, devfn); pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; @@ -919,12 +880,6 @@ int intel_svm_page_response(struct device *dev, goto out; } - ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); - if (ret || !sdev) { - ret = -ENODEV; - goto out; - } - /* * Per VT-d spec. v3.0 ch7.7, system software must respond * with page group response if private data is present (PDP) @@ -954,6 +909,5 @@ int intel_svm_page_response(struct device *dev, qi_submit_sync(iommu, &desc, 1, 0); } out: - mutex_unlock(&pasid_mutex); return ret; } diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 94ff319ae8acc5fc306a70e99f8ca5f7bf4e13ab..0ba817e863465b8babf33d025c3309d85e329c87 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -130,9 +130,6 @@ #define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL -#define APPLE_DART_PTE_PROT_NO_WRITE (1<<7) -#define APPLE_DART_PTE_PROT_NO_READ (1<<8) - /* IOPTE accessors */ #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) @@ -200,8 +197,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, void *pages; VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE, - gfp | __GFP_ZERO, order); + p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); if (!p) return NULL; @@ -406,15 +402,6 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, { arm_lpae_iopte pte; - if (data->iop.fmt == APPLE_DART) { - pte = 0; - if (!(prot & IOMMU_WRITE)) - pte |= APPLE_DART_PTE_PROT_NO_WRITE; - if (!(prot & IOMMU_READ)) - pte |= APPLE_DART_PTE_PROT_NO_READ; - return pte; - } - if (data->iop.fmt == ARM_64_LPAE_S1 || data->iop.fmt == ARM_32_LPAE_S1) { pte = ARM_LPAE_PTE_nG; @@ -1107,52 +1094,6 @@ out_free_data: return NULL; } -static struct io_pgtable * -apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) -{ - struct arm_lpae_io_pgtable *data; - int i; - - if (cfg->oas > 36) - return NULL; - - data = arm_lpae_alloc_pgtable(cfg); - if (!data) - return NULL; - - /* - * The table format itself always uses two levels, but the total VA - * space is mapped by four separate tables, making the MMIO registers - * an effective "level 1". For simplicity, though, we treat this - * equivalently to LPAE stage 2 concatenation at level 2, with the - * additional TTBRs each just pointing at consecutive pages. - */ - if (data->start_level < 1) - goto out_free_data; - if (data->start_level == 1 && data->pgd_bits > 2) - goto out_free_data; - if (data->start_level > 1) - data->pgd_bits = 0; - data->start_level = 2; - cfg->apple_dart_cfg.n_ttbrs = 1 << data->pgd_bits; - data->pgd_bits += data->bits_per_level; - - data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, - cfg); - if (!data->pgd) - goto out_free_data; - - for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) - cfg->apple_dart_cfg.ttbr[i] = - virt_to_phys(data->pgd + i * ARM_LPAE_GRANULE(data)); - - return &data->iop; - -out_free_data: - kfree(data); - return NULL; -} - struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { .alloc = arm_64_lpae_alloc_pgtable_s1, .free = arm_lpae_free_pgtable, @@ -1178,11 +1119,6 @@ struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = { .free = arm_lpae_free_pgtable, }; -struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns = { - .alloc = apple_dart_alloc_pgtable, - .free = arm_lpae_free_pgtable, -}; - #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST static struct io_pgtable_cfg *cfg_cookie __initdata; @@ -1343,12 +1279,17 @@ static int __init arm_lpae_do_selftests(void) }; int i, j, pass = 0, fail = 0; + struct device dev; struct io_pgtable_cfg cfg = { .tlb = &dummy_tlb_ops, .oas = 48, .coherent_walk = true, + .iommu_dev = &dev, }; + /* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */ + set_dev_node(&dev, NUMA_NO_NODE); + for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { for (j = 0; j < ARRAY_SIZE(ias); ++j) { cfg.pgsize_bitmap = pgsize[i]; diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c new file mode 100644 index 0000000000000000000000000000000000000000..74b1ef2b96bee119854ce825aac08ec6add76fbe --- /dev/null +++ b/drivers/iommu/io-pgtable-dart.c @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple DART page table allocator. + * + * Copyright (C) 2022 The Asahi Linux Contributors + * + * Based on io-pgtable-arm. + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon + */ + +#define pr_fmt(fmt) "dart io-pgtable: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define DART1_MAX_ADDR_BITS 36 + +#define DART_MAX_TABLES 4 +#define DART_LEVELS 2 + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct dart_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +#define DART_GRANULE(d) \ + (sizeof(dart_iopte) << (d)->bits_per_level) +#define DART_PTES_PER_TABLE(d) \ + (DART_GRANULE(d) >> ilog2(sizeof(dart_iopte))) + +#define APPLE_DART_PTE_SUBPAGE_START GENMASK_ULL(63, 52) +#define APPLE_DART_PTE_SUBPAGE_END GENMASK_ULL(51, 40) + +#define APPLE_DART1_PADDR_MASK GENMASK_ULL(35, 12) +#define APPLE_DART2_PADDR_MASK GENMASK_ULL(37, 10) +#define APPLE_DART2_PADDR_SHIFT (4) + +/* Apple DART1 protection bits */ +#define APPLE_DART1_PTE_PROT_NO_READ BIT(8) +#define APPLE_DART1_PTE_PROT_NO_WRITE BIT(7) +#define APPLE_DART1_PTE_PROT_SP_DIS BIT(1) + +/* Apple DART2 protection bits */ +#define APPLE_DART2_PTE_PROT_NO_READ BIT(3) +#define APPLE_DART2_PTE_PROT_NO_WRITE BIT(2) +#define APPLE_DART2_PTE_PROT_NO_CACHE BIT(1) + +/* marks PTE as valid */ +#define APPLE_DART_PTE_VALID BIT(0) + +/* IOPTE accessors */ +#define iopte_deref(pte, d) __va(iopte_to_paddr(pte, d)) + +struct dart_io_pgtable { + struct io_pgtable iop; + + int tbl_bits; + int bits_per_level; + + void *pgd[DART_MAX_TABLES]; +}; + +typedef u64 dart_iopte; + + +static dart_iopte paddr_to_iopte(phys_addr_t paddr, + struct dart_io_pgtable *data) +{ + dart_iopte pte; + + if (data->iop.fmt == APPLE_DART) + return paddr & APPLE_DART1_PADDR_MASK; + + /* format is APPLE_DART2 */ + pte = paddr >> APPLE_DART2_PADDR_SHIFT; + pte &= APPLE_DART2_PADDR_MASK; + + return pte; +} + +static phys_addr_t iopte_to_paddr(dart_iopte pte, + struct dart_io_pgtable *data) +{ + u64 paddr; + + if (data->iop.fmt == APPLE_DART) + return pte & APPLE_DART1_PADDR_MASK; + + /* format is APPLE_DART2 */ + paddr = pte & APPLE_DART2_PADDR_MASK; + paddr <<= APPLE_DART2_PADDR_SHIFT; + + return paddr; +} + +static void *__dart_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg) +{ + int order = get_order(size); + struct page *p; + + VM_BUG_ON((gfp & __GFP_HIGHMEM)); + p = alloc_pages(gfp | __GFP_ZERO, order); + if (!p) + return NULL; + + return page_address(p); +} + +static int dart_init_pte(struct dart_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + dart_iopte prot, int num_entries, + dart_iopte *ptep) +{ + int i; + dart_iopte pte = prot; + size_t sz = data->iop.cfg.pgsize_bitmap; + + for (i = 0; i < num_entries; i++) + if (ptep[i] & APPLE_DART_PTE_VALID) { + /* We require an unmap first */ + WARN_ON(ptep[i] & APPLE_DART_PTE_VALID); + return -EEXIST; + } + + /* subpage protection: always allow access to the entire page */ + pte |= FIELD_PREP(APPLE_DART_PTE_SUBPAGE_START, 0); + pte |= FIELD_PREP(APPLE_DART_PTE_SUBPAGE_END, 0xfff); + + pte |= APPLE_DART1_PTE_PROT_SP_DIS; + pte |= APPLE_DART_PTE_VALID; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); + + return 0; +} + +static dart_iopte dart_install_table(dart_iopte *table, + dart_iopte *ptep, + dart_iopte curr, + struct dart_io_pgtable *data) +{ + dart_iopte old, new; + + new = paddr_to_iopte(__pa(table), data) | APPLE_DART_PTE_VALID; + + /* + * Ensure the table itself is visible before its PTE can be. + * Whilst we could get away with cmpxchg64_release below, this + * doesn't have any ordering semantics when !CONFIG_SMP. + */ + dma_wmb(); + + old = cmpxchg64_relaxed(ptep, curr, new); + + return old; +} + +static int dart_get_table(struct dart_io_pgtable *data, unsigned long iova) +{ + return (iova >> (3 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) & + ((1 << data->tbl_bits) - 1); +} + +static int dart_get_l1_index(struct dart_io_pgtable *data, unsigned long iova) +{ + + return (iova >> (2 * data->bits_per_level + ilog2(sizeof(dart_iopte)))) & + ((1 << data->bits_per_level) - 1); +} + +static int dart_get_l2_index(struct dart_io_pgtable *data, unsigned long iova) +{ + + return (iova >> (data->bits_per_level + ilog2(sizeof(dart_iopte)))) & + ((1 << data->bits_per_level) - 1); +} + +static dart_iopte *dart_get_l2(struct dart_io_pgtable *data, unsigned long iova) +{ + dart_iopte pte, *ptep; + int tbl = dart_get_table(data, iova); + + ptep = data->pgd[tbl]; + if (!ptep) + return NULL; + + ptep += dart_get_l1_index(data, iova); + pte = READ_ONCE(*ptep); + + /* Valid entry? */ + if (!pte) + return NULL; + + /* Deref to get level 2 table */ + return iopte_deref(pte, data); +} + +static dart_iopte dart_prot_to_pte(struct dart_io_pgtable *data, + int prot) +{ + dart_iopte pte = 0; + + if (data->iop.fmt == APPLE_DART) { + if (!(prot & IOMMU_WRITE)) + pte |= APPLE_DART1_PTE_PROT_NO_WRITE; + if (!(prot & IOMMU_READ)) + pte |= APPLE_DART1_PTE_PROT_NO_READ; + } + if (data->iop.fmt == APPLE_DART2) { + if (!(prot & IOMMU_WRITE)) + pte |= APPLE_DART2_PTE_PROT_NO_WRITE; + if (!(prot & IOMMU_READ)) + pte |= APPLE_DART2_PTE_PROT_NO_READ; + if (!(prot & IOMMU_CACHE)) + pte |= APPLE_DART2_PTE_PROT_NO_CACHE; + } + + return pte; +} + +static int dart_map_pages(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) +{ + struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + size_t tblsz = DART_GRANULE(data); + int ret = 0, tbl, num_entries, max_entries, map_idx_start; + dart_iopte pte, *cptep, *ptep; + dart_iopte prot; + + if (WARN_ON(pgsize != cfg->pgsize_bitmap)) + return -EINVAL; + + if (WARN_ON(paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + tbl = dart_get_table(data, iova); + + ptep = data->pgd[tbl]; + ptep += dart_get_l1_index(data, iova); + pte = READ_ONCE(*ptep); + + /* no L2 table present */ + if (!pte) { + cptep = __dart_alloc_pages(tblsz, gfp, cfg); + if (!cptep) + return -ENOMEM; + + pte = dart_install_table(cptep, ptep, 0, data); + if (pte) + free_pages((unsigned long)cptep, get_order(tblsz)); + + /* L2 table is present (now) */ + pte = READ_ONCE(*ptep); + } + + ptep = iopte_deref(pte, data); + + /* install a leaf entries into L2 table */ + prot = dart_prot_to_pte(data, iommu_prot); + map_idx_start = dart_get_l2_index(data, iova); + max_entries = DART_PTES_PER_TABLE(data) - map_idx_start; + num_entries = min_t(int, pgcount, max_entries); + ptep += map_idx_start; + ret = dart_init_pte(data, iova, paddr, prot, num_entries, ptep); + if (!ret && mapped) + *mapped += num_entries * pgsize; + + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return ret; +} + +static size_t dart_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + int i = 0, num_entries, max_entries, unmap_idx_start; + dart_iopte pte, *ptep; + + if (WARN_ON(pgsize != cfg->pgsize_bitmap || !pgcount)) + return 0; + + ptep = dart_get_l2(data, iova); + + /* Valid L2 IOPTE pointer? */ + if (WARN_ON(!ptep)) + return 0; + + unmap_idx_start = dart_get_l2_index(data, iova); + ptep += unmap_idx_start; + + max_entries = DART_PTES_PER_TABLE(data) - unmap_idx_start; + num_entries = min_t(int, pgcount, max_entries); + + while (i < num_entries) { + pte = READ_ONCE(*ptep); + if (WARN_ON(!pte)) + break; + + /* clear pte */ + *ptep = 0; + + if (!iommu_iotlb_gather_queued(gather)) + io_pgtable_tlb_add_page(&data->iop, gather, + iova + i * pgsize, pgsize); + + ptep++; + i++; + } + + return i * pgsize; +} + +static phys_addr_t dart_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct dart_io_pgtable *data = io_pgtable_ops_to_data(ops); + dart_iopte pte, *ptep; + + ptep = dart_get_l2(data, iova); + + /* Valid L2 IOPTE pointer? */ + if (!ptep) + return 0; + + ptep += dart_get_l2_index(data, iova); + + pte = READ_ONCE(*ptep); + /* Found translation */ + if (pte) { + iova &= (data->iop.cfg.pgsize_bitmap - 1); + return iopte_to_paddr(pte, data) | iova; + } + + /* Ran out of page tables to walk */ + return 0; +} + +static struct dart_io_pgtable * +dart_alloc_pgtable(struct io_pgtable_cfg *cfg) +{ + struct dart_io_pgtable *data; + int tbl_bits, bits_per_level, va_bits, pg_shift; + + pg_shift = __ffs(cfg->pgsize_bitmap); + bits_per_level = pg_shift - ilog2(sizeof(dart_iopte)); + + va_bits = cfg->ias - pg_shift; + + tbl_bits = max_t(int, 0, va_bits - (bits_per_level * DART_LEVELS)); + if ((1 << tbl_bits) > DART_MAX_TABLES) + return NULL; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->tbl_bits = tbl_bits; + data->bits_per_level = bits_per_level; + + data->iop.ops = (struct io_pgtable_ops) { + .map_pages = dart_map_pages, + .unmap_pages = dart_unmap_pages, + .iova_to_phys = dart_iova_to_phys, + }; + + return data; +} + +static struct io_pgtable * +apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct dart_io_pgtable *data; + int i; + + if (!cfg->coherent_walk) + return NULL; + + if (cfg->oas != 36 && cfg->oas != 42) + return NULL; + + if (cfg->ias > cfg->oas) + return NULL; + + if (!(cfg->pgsize_bitmap == SZ_4K || cfg->pgsize_bitmap == SZ_16K)) + return NULL; + + data = dart_alloc_pgtable(cfg); + if (!data) + return NULL; + + cfg->apple_dart_cfg.n_ttbrs = 1 << data->tbl_bits; + + for (i = 0; i < cfg->apple_dart_cfg.n_ttbrs; ++i) { + data->pgd[i] = __dart_alloc_pages(DART_GRANULE(data), GFP_KERNEL, + cfg); + if (!data->pgd[i]) + goto out_free_data; + cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); + } + + return &data->iop; + +out_free_data: + while (--i >= 0) + free_pages((unsigned long)data->pgd[i], + get_order(DART_GRANULE(data))); + kfree(data); + return NULL; +} + +static void apple_dart_free_pgtable(struct io_pgtable *iop) +{ + struct dart_io_pgtable *data = io_pgtable_to_data(iop); + dart_iopte *ptep, *end; + int i; + + for (i = 0; i < (1 << data->tbl_bits) && data->pgd[i]; ++i) { + ptep = data->pgd[i]; + end = (void *)ptep + DART_GRANULE(data); + + while (ptep != end) { + dart_iopte pte = *ptep++; + + if (pte) { + unsigned long page = + (unsigned long)iopte_deref(pte, data); + + free_pages(page, get_order(DART_GRANULE(data))); + } + } + free_pages((unsigned long)data->pgd[i], + get_order(DART_GRANULE(data))); + } + + kfree(data); +} + +struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns = { + .alloc = apple_dart_alloc_pgtable, + .free = apple_dart_free_pgtable, +}; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index f4bfcef98297f4158f32ace205bbd50430d63421..b843fcd365d286668273667401b331a27fc04d23 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -20,13 +20,17 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, [ARM_MALI_LPAE] = &io_pgtable_arm_mali_lpae_init_fns, +#endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_DART [APPLE_DART] = &io_pgtable_apple_dart_init_fns, + [APPLE_DART2] = &io_pgtable_apple_dart_init_fns, #endif #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, #endif #ifdef CONFIG_AMD_IOMMU [AMD_IOMMU_V1] = &io_pgtable_amd_iommu_v1_init_fns, + [AMD_IOMMU_V2] = &io_pgtable_amd_iommu_v2_init_fns, #endif }; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 780fb70715770d710f9a806d443dd886222ce986..4893c2429ca560774aaba0e76587f7ff9f99cc60 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -6,8 +6,8 @@ #define pr_fmt(fmt) "iommu: " fmt +#include #include -#include #include #include #include @@ -16,17 +16,21 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include +#include "dma-iommu.h" + static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); @@ -75,6 +79,8 @@ static const char * const iommu_group_resv_type_string[] = { #define IOMMU_CMD_LINE_DMA_API BIT(0) #define IOMMU_CMD_LINE_STRICT BIT(1) +static int iommu_bus_notifier(struct notifier_block *nb, + unsigned long action, void *data); static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -103,6 +109,22 @@ struct iommu_group_attribute iommu_group_attr_##_name = \ static LIST_HEAD(iommu_device_list); static DEFINE_SPINLOCK(iommu_device_lock); +static struct bus_type * const iommu_buses[] = { + &platform_bus_type, +#ifdef CONFIG_PCI + &pci_bus_type, +#endif +#ifdef CONFIG_ARM_AMBA + &amba_bustype, +#endif +#ifdef CONFIG_FSL_MC_BUS + &fsl_mc_bus_type, +#endif +#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS + &host1x_context_device_bus_type, +#endif +}; + /* * Use a function instead of an array here because the domain-type is a * bit-field, so an array would waste memory. @@ -126,6 +148,8 @@ static const char *iommu_domain_type_str(unsigned int t) static int __init iommu_subsys_init(void) { + struct notifier_block *nb; + if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) iommu_set_default_passthrough(false); @@ -152,10 +176,27 @@ static int __init iommu_subsys_init(void) (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ? "(set via kernel command line)" : ""); + nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL); + if (!nb) + return -ENOMEM; + + for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) { + nb[i].notifier_call = iommu_bus_notifier; + bus_register_notifier(iommu_buses[i], &nb[i]); + } + return 0; } subsys_initcall(iommu_subsys_init); +static int remove_iommu_group(struct device *dev, void *data) +{ + if (dev->iommu && dev->iommu->iommu_dev == data) + iommu_release_device(dev); + + return 0; +} + /** * iommu_device_register() - Register an IOMMU hardware instance * @iommu: IOMMU handle for the instance @@ -167,23 +208,42 @@ subsys_initcall(iommu_subsys_init); int iommu_device_register(struct iommu_device *iommu, const struct iommu_ops *ops, struct device *hwdev) { + int err = 0; + /* We need to be able to take module references appropriately */ if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner)) return -EINVAL; + /* + * Temporarily enforce global restriction to a single driver. This was + * already the de-facto behaviour, since any possible combination of + * existing drivers would compete for at least the PCI or platform bus. + */ + if (iommu_buses[0]->iommu_ops && iommu_buses[0]->iommu_ops != ops) + return -EBUSY; iommu->ops = ops; if (hwdev) - iommu->fwnode = hwdev->fwnode; + iommu->fwnode = dev_fwnode(hwdev); spin_lock(&iommu_device_lock); list_add_tail(&iommu->list, &iommu_device_list); spin_unlock(&iommu_device_lock); - return 0; + + for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++) { + iommu_buses[i]->iommu_ops = ops; + err = bus_iommu_probe(iommu_buses[i]); + } + if (err) + iommu_device_unregister(iommu); + return err; } EXPORT_SYMBOL_GPL(iommu_device_register); void iommu_device_unregister(struct iommu_device *iommu) { + for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) + bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group); + spin_lock(&iommu_device_lock); list_del(&iommu->list); spin_unlock(&iommu_device_lock); @@ -654,7 +714,6 @@ struct iommu_group *iommu_group_alloc(void) ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, NULL, "%d", group->id); if (ret) { - ida_free(&iommu_group_ida, group->id); kobject_put(&group->kobj); return ERR_PTR(ret); } @@ -1612,13 +1671,6 @@ static int probe_iommu_group(struct device *dev, void *data) return ret; } -static int remove_iommu_group(struct device *dev, void *data) -{ - iommu_release_device(dev); - - return 0; -} - static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -1775,75 +1827,6 @@ int bus_iommu_probe(struct bus_type *bus) return ret; } -static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) -{ - struct notifier_block *nb; - int err; - - nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); - if (!nb) - return -ENOMEM; - - nb->notifier_call = iommu_bus_notifier; - - err = bus_register_notifier(bus, nb); - if (err) - goto out_free; - - err = bus_iommu_probe(bus); - if (err) - goto out_err; - - - return 0; - -out_err: - /* Clean up */ - bus_for_each_dev(bus, NULL, NULL, remove_iommu_group); - bus_unregister_notifier(bus, nb); - -out_free: - kfree(nb); - - return err; -} - -/** - * bus_set_iommu - set iommu-callbacks for the bus - * @bus: bus. - * @ops: the callbacks provided by the iommu-driver - * - * This function is called by an iommu driver to set the iommu methods - * used for a particular bus. Drivers for devices on that bus can use - * the iommu-api after these ops are registered. - * This special function is needed because IOMMUs are usually devices on - * the bus itself, so the iommu drivers are not initialized when the bus - * is set up. With this function the iommu-driver can set the iommu-ops - * afterwards. - */ -int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) -{ - int err; - - if (ops == NULL) { - bus->iommu_ops = NULL; - return 0; - } - - if (bus->iommu_ops != NULL) - return -EBUSY; - - bus->iommu_ops = ops; - - /* Do IOMMU specific setup for this bus-type */ - err = iommu_bus_init(bus, ops); - if (err) - bus->iommu_ops = NULL; - - return err; -} -EXPORT_SYMBOL_GPL(bus_set_iommu); - bool iommu_present(struct bus_type *bus) { return bus->iommu_ops != NULL; @@ -1869,19 +1852,10 @@ bool device_iommu_capable(struct device *dev, enum iommu_cap cap) if (!ops->capable) return false; - return ops->capable(cap); + return ops->capable(dev, cap); } EXPORT_SYMBOL_GPL(device_iommu_capable); -bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) -{ - if (!bus->iommu_ops || !bus->iommu_ops->capable) - return false; - - return bus->iommu_ops->capable(cap); -} -EXPORT_SYMBOL_GPL(iommu_capable); - /** * iommu_set_fault_handler() - set a fault handler for an iommu domain * @domain: iommu domain @@ -3076,6 +3050,24 @@ out: return ret; } +static bool iommu_is_default_domain(struct iommu_group *group) +{ + if (group->domain == group->default_domain) + return true; + + /* + * If the default domain was set to identity and it is still an identity + * domain then we consider this a pass. This happens because of + * amd_iommu_init_device() replacing the default idenytity domain with an + * identity domain that has a different configuration for AMDGPU. + */ + if (group->default_domain && + group->default_domain->type == IOMMU_DOMAIN_IDENTITY && + group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY) + return true; + return false; +} + /** * iommu_device_use_default_domain() - Device driver wants to handle device * DMA through the kernel DMA API. @@ -3094,8 +3086,7 @@ int iommu_device_use_default_domain(struct device *dev) mutex_lock(&group->mutex); if (group->owner_cnt) { - if (group->domain != group->default_domain || - group->owner) { + if (group->owner || !iommu_is_default_domain(group)) { ret = -EBUSY; goto unlock_out; } diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 47d1983dfa2a4e05c3456e198d44f0523357ba64..a44ad92fc5eb70e18ca09c9e16329d268804ffe0 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -661,9 +661,6 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) unsigned long flags; int i; - if (!mag) - return; - spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); for (i = 0 ; i < mag->size; ++i) { @@ -683,12 +680,12 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) static bool iova_magazine_full(struct iova_magazine *mag) { - return (mag && mag->size == IOVA_MAG_SIZE); + return mag->size == IOVA_MAG_SIZE; } static bool iova_magazine_empty(struct iova_magazine *mag) { - return (!mag || mag->size == 0); + return mag->size == 0; } static unsigned long iova_magazine_pop(struct iova_magazine *mag, @@ -697,8 +694,6 @@ static unsigned long iova_magazine_pop(struct iova_magazine *mag, int i; unsigned long pfn; - BUG_ON(iova_magazine_empty(mag)); - /* Only fall back to the rbtree if we have no suitable pfns at all */ for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--) if (i == 0) @@ -713,8 +708,6 @@ static unsigned long iova_magazine_pop(struct iova_magazine *mag, static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) { - BUG_ON(iova_magazine_full(mag)); - mag->pfns[mag->size++] = pfn; } @@ -882,7 +875,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, { unsigned int log_size = order_base_2(size); - if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE || !iovad->rcaches) + if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) return 0; return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 1d42084d02767ece7221f2455e8137d322f1745e..3b30c0752274fd43e13ac19908499a6fa1a65e0f 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1090,11 +1090,6 @@ static int ipmmu_probe(struct platform_device *pdev) ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev); if (ret) return ret; - -#if defined(CONFIG_IOMMU_DMA) - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &ipmmu_ops); -#endif } /* @@ -1168,32 +1163,4 @@ static struct platform_driver ipmmu_driver = { .probe = ipmmu_probe, .remove = ipmmu_remove, }; - -static int __init ipmmu_init(void) -{ - struct device_node *np; - static bool setup_done; - int ret; - - if (setup_done) - return 0; - - np = of_find_matching_node(NULL, ipmmu_of_ids); - if (!np) - return 0; - - of_node_put(np); - - ret = platform_driver_register(&ipmmu_driver); - if (ret < 0) - return ret; - -#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &ipmmu_ops); -#endif - - setup_done = true; - return 0; -} -subsys_initcall(ipmmu_init); +builtin_platform_driver(ipmmu_driver); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 6a24aa804ea31280ba8a9e454d79e4abdf4075e1..16179a9a728300934f23f3c919bb9c200b16a728 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -792,8 +792,6 @@ static int msm_iommu_probe(struct platform_device *pdev) goto fail; } - bus_set_iommu(&platform_bus_type, &msm_iommu_ops); - pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7e363b1f24dfe078a4c4c1ecea4def3f782de82b..5a4e00e4bbbc7cff4312aeec5a4bd99658648b07 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -138,6 +138,7 @@ #define PM_CLK_AO BIT(15) #define IFA_IOMMU_PCIE_SUPPORT BIT(16) #define PGTABLE_PA_35_EN BIT(17) +#define TF_PORT_TO_ADDR_MT8173 BIT(18) #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \ ((((pdata)->flags) & (mask)) == (_x)) @@ -157,6 +158,7 @@ enum mtk_iommu_plat { M4U_MT2712, M4U_MT6779, + M4U_MT6795, M4U_MT8167, M4U_MT8173, M4U_MT8183, @@ -955,7 +957,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int ban * Global control settings are in bank0. May re-init these global registers * since no sure if there is bank0 consumers. */ - if (data->plat_data->m4u_plat == M4U_MT8173) { + if (MTK_IOMMU_HAS_FLAG(data->plat_data, TF_PORT_TO_ADDR_MT8173)) { regval = F_MMU_PREFETCH_RT_REPLACE_MOD | F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173; } else { @@ -1243,30 +1245,13 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->hw_list = &data->hw_list_head; } - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); - if (ret) - goto out_list_del; - } - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); if (ret) - goto out_bus_set_null; - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && - MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - ret = bus_set_iommu(&pci_bus_type, &mtk_iommu_ops); - if (ret) /* PCIe fail don't affect platform_bus. */ - goto out_list_del; - } -#endif + goto out_list_del; } return ret; -out_bus_set_null: - bus_set_iommu(&platform_bus_type, NULL); out_list_del: list_del(&data->list); iommu_device_unregister(&data->iommu); @@ -1294,11 +1279,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, &pdev->dev); component_master_del(&pdev->dev, &mtk_iommu_com_ops); - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && - MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif } pm_runtime_disable(&pdev->dev); for (i = 0; i < data->plat_data->banks_num; i++) { @@ -1413,6 +1393,19 @@ static const struct mtk_iommu_plat_data mt6779_data = { .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}}, }; +static const struct mtk_iommu_plat_data mt6795_data = { + .m4u_plat = M4U_MT6795, + .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | + HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | + TF_PORT_TO_ADDR_MT8173, + .inv_sel_reg = REG_MMU_INV_SEL_GEN1, + .banks_num = 1, + .banks_enable = {true}, + .iova_region = single_domain, + .iova_region_nr = ARRAY_SIZE(single_domain), + .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */ +}; + static const struct mtk_iommu_plat_data mt8167_data = { .m4u_plat = M4U_MT8167, .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, @@ -1427,7 +1420,8 @@ static const struct mtk_iommu_plat_data mt8167_data = { static const struct mtk_iommu_plat_data mt8173_data = { .m4u_plat = M4U_MT8173, .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI | - HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM, + HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM | + TF_PORT_TO_ADDR_MT8173, .inv_sel_reg = REG_MMU_INV_SEL_GEN1, .banks_num = 1, .banks_enable = {true}, @@ -1524,6 +1518,7 @@ static const struct mtk_iommu_plat_data mt8195_data_vpp = { static const struct of_device_id mtk_iommu_of_ids[] = { { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data}, { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data}, + { .compatible = "mediatek,mt6795-m4u", .data = &mt6795_data}, { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data}, { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data}, { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data}, diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 128c7a3f17783227c151e90fe407b062614ac65a..6e0e65831eb70d9f3ffbd285d464ef356562c2b7 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -691,19 +691,11 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) if (ret) goto out_sysfs_remove; - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_v1_ops); - if (ret) - goto out_dev_unreg; - } - ret = component_master_add_with_match(dev, &mtk_iommu_v1_com_ops, match); if (ret) - goto out_bus_set_null; + goto out_dev_unreg; return ret; -out_bus_set_null: - bus_set_iommu(&platform_bus_type, NULL); out_dev_unreg: iommu_device_unregister(&data->iommu); out_sysfs_remove: @@ -718,9 +710,6 @@ static int mtk_iommu_v1_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); - if (iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, NULL); - clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops); diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 41f4eb005219268818f57765ce25e9164b645669..5696314ae69e7d23b5ef7f71382bf95c5bda0bd7 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -40,7 +40,7 @@ static int of_iommu_xlate(struct device *dev, * a proper probe-ordering dependency mechanism in future. */ if (!ops) - return -ENODEV; + return driver_deferred_probe_check_state(dev); if (!try_module_get(ops->owner)) return -ENODEV; diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index a99afb5d9011c5375905d54520e9851048bfbc5a..259f65291d909696ca91bbc70f50791a097969f0 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -32,12 +32,12 @@ static inline bool is_omap_iommu_detached(struct omap_iommu *obj) ssize_t bytes; \ const char *str = "%20s: %08x\n"; \ const int maxcol = 32; \ - bytes = snprintf(p, maxcol, str, __stringify(name), \ + if (len < maxcol) \ + goto out; \ + bytes = scnprintf(p, maxcol, str, __stringify(name), \ iommu_read_reg(obj, MMU_##name)); \ p += bytes; \ len -= bytes; \ - if (len < maxcol) \ - goto out; \ } while (0) static ssize_t diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index d9cf2820c02eae1ad3b1c0db47a36074671c7f4c..07ee2600113c20969b8f5b24ae4dfc7fc81be3a3 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1776,14 +1776,8 @@ static int __init omap_iommu_init(void) goto fail_driver; } - ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops); - if (ret) - goto fail_bus; - return 0; -fail_bus: - platform_driver_unregister(&omap_iommu_driver); fail_driver: kmem_cache_destroy(iopte_cachep); return ret; diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index ab57c4b8fade271bc141f30b0a70584e73eb001f..a3fc59b814ab5e7434ad055aa543951e7f11f153 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1300,8 +1300,6 @@ static int rk_iommu_probe(struct platform_device *pdev) if (!dma_dev) dma_dev = &pdev->dev; - bus_set_iommu(&platform_bus_type, &rk_iommu_ops); - pm_runtime_enable(dev); for (i = 0; i < iommu->num_irq; i++) { diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index c898bcbbce118f1509802ee74dadbd87025c52a1..3c071782f6f16d7c3c0f2ef34b6d8088927694a2 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -39,7 +39,7 @@ static struct s390_domain *to_s390_domain(struct iommu_domain *dom) return container_of(dom, struct s390_domain, domain); } -static bool s390_iommu_capable(enum iommu_cap cap) +static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: @@ -185,7 +185,12 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, static struct iommu_device *s390_iommu_probe_device(struct device *dev) { - struct zpci_dev *zdev = to_zpci_dev(dev); + struct zpci_dev *zdev; + + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + zdev = to_zpci_dev(dev); return &zdev->iommu_dev; } @@ -385,9 +390,3 @@ static const struct iommu_ops s390_iommu_ops = { .free = s390_domain_free, } }; - -static int __init s390_iommu_init(void) -{ - return bus_set_iommu(&pci_bus_type, &s390_iommu_ops); -} -subsys_initcall(s390_iommu_init); diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 511959c8a14d94fd88a3e6a66a9eafbc8c4a5519..fadd2c907222b9230b49b2ec54f6dbcfdf5914e5 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -496,9 +496,6 @@ static int sprd_iommu_probe(struct platform_device *pdev) if (ret) goto remove_sysfs; - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &sprd_iommu_ops); - ret = sprd_iommu_clk_enable(sdev); if (ret) goto unregister_iommu; @@ -534,8 +531,6 @@ static int sprd_iommu_remove(struct platform_device *pdev) iommu_group_put(sdev->group); sdev->group = NULL; - bus_set_iommu(&platform_bus_type, NULL); - platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&sdev->iommu); iommu_device_unregister(&sdev->iommu); diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index a84c63518773a760422b59eca4b7bce7494c1567..cd9b74ee24def1b0dd5f7db12f95af359a364053 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -965,8 +965,6 @@ static int sun50i_iommu_probe(struct platform_device *pdev) if (ret < 0) goto err_unregister; - bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops); - return 0; err_unregister: diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 2a8de975fe63bbc9212221cf6bb8994fb282053c..5b1af40221ec1490767ca990a69d44398a67c52f 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -1083,8 +1083,8 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, /* * This is a bit of a hack. Ideally we'd want to simply return this - * value. However the IOMMU registration process will attempt to add - * all devices to the IOMMU when bus_set_iommu() is called. In order + * value. However iommu_device_register() will attempt to add + * all devices to the IOMMU before we get that far. In order * not to rely on global variables to track the IOMMU instance, we * set it here so that it can be looked up from the .probe_device() * callback via the IOMMU device's .drvdata field. @@ -1138,32 +1138,15 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, return ERR_PTR(err); err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev); - if (err) - goto remove_sysfs; - - err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); - if (err < 0) - goto unregister; - -#ifdef CONFIG_PCI - err = bus_set_iommu(&pci_bus_type, &tegra_smmu_ops); - if (err < 0) - goto unset_platform_bus; -#endif + if (err) { + iommu_device_sysfs_remove(&smmu->iommu); + return ERR_PTR(err); + } if (IS_ENABLED(CONFIG_DEBUG_FS)) tegra_smmu_debugfs_init(smmu); return smmu; - -unset_platform_bus: __maybe_unused; - bus_set_iommu(&platform_bus_type, NULL); -unregister: - iommu_device_unregister(&smmu->iommu); -remove_sysfs: - iommu_device_sysfs_remove(&smmu->iommu); - - return ERR_PTR(err); } void tegra_smmu_remove(struct tegra_smmu *smmu) diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 08eeafc9529fa0580da66a4e93fd8fb2b81fc52e..b7c22802f57c08a08d9a2d3cc491db86267ddea7 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -7,9 +7,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include -#include #include #include #include @@ -17,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -25,6 +22,8 @@ #include +#include "dma-iommu.h" + #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 @@ -925,7 +924,7 @@ static struct virtio_driver virtio_iommu_drv; static int viommu_match_node(struct device *dev, const void *data) { - return dev->parent->fwnode == data; + return device_match_fwnode(dev->parent, data); } static struct viommu_dev *viommu_get_by_fwnode(struct fwnode_handle *fwnode) @@ -1006,7 +1005,18 @@ static int viommu_of_xlate(struct device *dev, struct of_phandle_args *args) return iommu_fwspec_add_ids(dev, args->args, 1); } +static bool viommu_capable(struct device *dev, enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return true; + default: + return false; + } +} + static struct iommu_ops viommu_ops = { + .capable = viommu_capable, .domain_alloc = viommu_domain_alloc, .probe_device = viommu_probe_device, .probe_finalize = viommu_probe_finalize, @@ -1145,26 +1155,6 @@ static int viommu_probe(struct virtio_device *vdev) iommu_device_register(&viommu->iommu, &viommu_ops, parent_dev); -#ifdef CONFIG_PCI - if (pci_bus_type.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&pci_bus_type, &viommu_ops); - if (ret) - goto err_unregister; - } -#endif -#ifdef CONFIG_ARM_AMBA - if (amba_bustype.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&amba_bustype, &viommu_ops); - if (ret) - goto err_unregister; - } -#endif - if (platform_bus_type.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&platform_bus_type, &viommu_ops); - if (ret) - goto err_unregister; - } - vdev->priv = viommu; dev_info(dev, "input address: %u bits\n", @@ -1173,9 +1163,6 @@ static int viommu_probe(struct virtio_device *vdev) return 0; -err_unregister: - iommu_device_sysfs_remove(&viommu->iommu); - iommu_device_unregister(&viommu->iommu); err_free_vqs: vdev->config->del_vqs(vdev); diff --git a/drivers/ipack/devices/ipoctal.c b/drivers/ipack/devices/ipoctal.c index 20d2b9ec12270f06cd987d0429673e7505069b9c..fc00274070b6b2888ef03b0522c93a69a8ca8d29 100644 --- a/drivers/ipack/devices/ipoctal.c +++ b/drivers/ipack/devices/ipoctal.c @@ -497,7 +497,7 @@ static unsigned int ipoctal_chars_in_buffer(struct tty_struct *tty) } static void ipoctal_set_termios(struct tty_struct *tty, - struct ktermios *old_termios) + const struct ktermios *old_termios) { unsigned int cflag; unsigned char mr1 = 0; diff --git a/drivers/ipack/ipack.c b/drivers/ipack/ipack.c index b1c3198355e7667f77cc7eba87c779eea81a535a..74d449858a61b29d8023a5ba78a4e386d7e7ef75 100644 --- a/drivers/ipack/ipack.c +++ b/drivers/ipack/ipack.c @@ -429,8 +429,11 @@ int ipack_device_init(struct ipack_device *dev) dev->dev.bus = &ipack_bus_type; dev->dev.release = ipack_device_release; dev->dev.parent = dev->bus->parent; - dev_set_name(&dev->dev, + ret = dev_set_name(&dev->dev, "ipack-dev.%u.%u", dev->bus->bus_nr, dev->slot); + if (ret) + return ret; + device_initialize(&dev->dev); if (dev->bus->ops->set_clockrate(dev, 8)) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 66b9fa408bf2422fd083acddbaf705e6ddce6b9a..7ef9f5e696d317415c30b258f637db95c1fcaf81 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -3,7 +3,7 @@ menu "IRQ chip support" config IRQCHIP def_bool y - depends on OF_IRQ + depends on (OF_IRQ || ACPI_GENERIC_GSI) config ARM_GIC bool @@ -481,6 +481,21 @@ config IMX_INTMUX help Support for the i.MX INTMUX interrupt multiplexer. +config IMX_MU_MSI + tristate "i.MX MU used as MSI controller" + depends on OF && HAS_IOMEM + depends on ARCH_MXC || COMPILE_TEST + default m if ARCH_MXC + select IRQ_DOMAIN + select IRQ_DOMAIN_HIERARCHY + select GENERIC_MSI_IRQ_DOMAIN + help + Provide a driver for the i.MX Messaging Unit block used as a + CPU-to-CPU MSI controller. This requires a specially crafted DT + to make use of this driver. + + If unsure, say N + config LS1X_IRQ bool "Loongson-1 Interrupt Controller" depends on MACH_LOONGSON32 @@ -561,6 +576,11 @@ config IRQ_LOONGARCH_CPU select GENERIC_IRQ_CHIP select IRQ_DOMAIN select GENERIC_IRQ_EFFECTIVE_AFF_MASK + select LOONGSON_LIOINTC + select LOONGSON_EIOINTC + select LOONGSON_PCH_PIC + select LOONGSON_PCH_MSI + select LOONGSON_PCH_LPC help Support for the LoongArch CPU Interrupt Controller. For details of irq chip hierarchy on LoongArch platforms please read the document @@ -623,8 +643,9 @@ config LOONGSON_PCH_MSI config LOONGSON_PCH_LPC bool "Loongson PCH LPC Controller" + depends on LOONGARCH depends on MACH_LOONGSON64 - default (MACH_LOONGSON64 && LOONGARCH) + default MACH_LOONGSON64 select IRQ_DOMAIN_HIERARCHY help Support for the Loongson PCH LPC Controller. diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index b6acbca2248bc24291a5f2719a024a818f45aec2..87b49a10962c7bfdf6dd9fc3d9429e5da5c0dbeb 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_RISCV_INTC) += irq-riscv-intc.o obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o +obj-$(CONFIG_IMX_MU_MSI) += irq-imx-mu-msi.o obj-$(CONFIG_MADERA_IRQ) += irq-madera.o obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index b249d4df899e7b4e886da3598fbc1f233c9d43d9..6e1ac330d7a600a3f37d2430fb05d5d18b3a0667 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -13,7 +13,7 @@ #define pr_fmt(fmt) "GICv2m: " fmt #include -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5ff09de6c48fcbec5a9b954fa284c02617204de4..973ede0197e36f066f1904280bc710cc89c84879 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -11,9 +11,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -1574,13 +1574,15 @@ static int its_select_cpu(struct irq_data *d, const struct cpumask *aff_mask) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); - cpumask_var_t tmpmask; + static DEFINE_RAW_SPINLOCK(tmpmask_lock); + static struct cpumask __tmpmask; + struct cpumask *tmpmask; + unsigned long flags; int cpu, node; - - if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC)) - return -ENOMEM; - node = its_dev->its->numa_node; + tmpmask = &__tmpmask; + + raw_spin_lock_irqsave(&tmpmask_lock, flags); if (!irqd_affinity_is_managed(d)) { /* First try the NUMA node */ @@ -1634,7 +1636,7 @@ static int its_select_cpu(struct irq_data *d, cpu = cpumask_pick_least_loaded(d, tmpmask); } out: - free_cpumask_var(tmpmask); + raw_spin_unlock_irqrestore(&tmpmask_lock, flags); pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); return cpu; diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index a2163d32f17d296c200e99bb61b992dd2eb3e811..e1efdec9e9acfb2e69a9b0015e4c156a8cf753f9 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -6,7 +6,7 @@ #define pr_fmt(fmt) "GICv3: " fmt -#include +#include #include #include #include diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 262658fd5f9e59f9d3ee6395c1af39be4399d9d8..34d58567b78d195ca312efc154590efcbc67bc0f 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -978,7 +978,7 @@ static int __gic_update_rdist_properties(struct redist_region *region, u64 typer = gic_read_typer(ptr + GICR_TYPER); u32 ctlr = readl_relaxed(ptr + GICR_CTLR); - /* Boot-time cleanip */ + /* Boot-time cleanup */ if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) { u64 val; diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 4ea71b28f9f5f21ec0c8979bd91f07bf215c75cf..a6277dea4c7a73ff051a2cf4063ba2ef104de3ae 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -94,7 +94,7 @@ bool gic_cpuif_has_vsgi(void) { unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); - fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT); + fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); return fld >= 0x3; } diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c new file mode 100644 index 0000000000000000000000000000000000000000..229039eda1b1ff089204fc33764d84de317abdbc --- /dev/null +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Freescale MU used as MSI controller + * + * Copyright (c) 2018 Pengutronix, Oleksij Rempel + * Copyright 2022 NXP + * Frank Li + * Peng Fan + * + * Based on drivers/mailbox/imx-mailbox.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IMX_MU_CHANS 4 + +enum imx_mu_xcr { + IMX_MU_GIER, + IMX_MU_GCR, + IMX_MU_TCR, + IMX_MU_RCR, + IMX_MU_xCR_MAX, +}; + +enum imx_mu_xsr { + IMX_MU_SR, + IMX_MU_GSR, + IMX_MU_TSR, + IMX_MU_RSR, + IMX_MU_xSR_MAX +}; + +enum imx_mu_type { + IMX_MU_V2 = BIT(1), +}; + +/* Receive Interrupt Enable */ +#define IMX_MU_xCR_RIEn(data, x) ((data->cfg->type) & IMX_MU_V2 ? BIT(x) : BIT(24 + (3 - (x)))) +#define IMX_MU_xSR_RFn(data, x) ((data->cfg->type) & IMX_MU_V2 ? BIT(x) : BIT(24 + (3 - (x)))) + +struct imx_mu_dcfg { + enum imx_mu_type type; + u32 xTR; /* Transmit Register0 */ + u32 xRR; /* Receive Register0 */ + u32 xSR[IMX_MU_xSR_MAX]; /* Status Registers */ + u32 xCR[IMX_MU_xCR_MAX]; /* Control Registers */ +}; + +struct imx_mu_msi { + raw_spinlock_t lock; + struct irq_domain *msi_domain; + void __iomem *regs; + phys_addr_t msiir_addr; + const struct imx_mu_dcfg *cfg; + unsigned long used; + struct clk *clk; +}; + +static void imx_mu_write(struct imx_mu_msi *msi_data, u32 val, u32 offs) +{ + iowrite32(val, msi_data->regs + offs); +} + +static u32 imx_mu_read(struct imx_mu_msi *msi_data, u32 offs) +{ + return ioread32(msi_data->regs + offs); +} + +static u32 imx_mu_xcr_rmw(struct imx_mu_msi *msi_data, enum imx_mu_xcr type, u32 set, u32 clr) +{ + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&msi_data->lock, flags); + val = imx_mu_read(msi_data, msi_data->cfg->xCR[type]); + val &= ~clr; + val |= set; + imx_mu_write(msi_data, val, msi_data->cfg->xCR[type]); + raw_spin_unlock_irqrestore(&msi_data->lock, flags); + + return val; +} + +static void imx_mu_msi_parent_mask_irq(struct irq_data *data) +{ + struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data); + + imx_mu_xcr_rmw(msi_data, IMX_MU_RCR, 0, IMX_MU_xCR_RIEn(msi_data, data->hwirq)); +} + +static void imx_mu_msi_parent_unmask_irq(struct irq_data *data) +{ + struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data); + + imx_mu_xcr_rmw(msi_data, IMX_MU_RCR, IMX_MU_xCR_RIEn(msi_data, data->hwirq), 0); +} + +static void imx_mu_msi_parent_ack_irq(struct irq_data *data) +{ + struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data); + + imx_mu_read(msi_data, msi_data->cfg->xRR + data->hwirq * 4); +} + +static struct irq_chip imx_mu_msi_irq_chip = { + .name = "MU-MSI", + .irq_ack = irq_chip_ack_parent, +}; + +static struct msi_domain_ops imx_mu_msi_irq_ops = { +}; + +static struct msi_domain_info imx_mu_msi_domain_info = { + .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), + .ops = &imx_mu_msi_irq_ops, + .chip = &imx_mu_msi_irq_chip, +}; + +static void imx_mu_msi_parent_compose_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(data); + u64 addr = msi_data->msiir_addr + 4 * data->hwirq; + + msg->address_hi = upper_32_bits(addr); + msg->address_lo = lower_32_bits(addr); + msg->data = data->hwirq; +} + +static int imx_mu_msi_parent_set_affinity(struct irq_data *irq_data, + const struct cpumask *mask, bool force) +{ + return -EINVAL; +} + +static struct irq_chip imx_mu_msi_parent_chip = { + .name = "MU", + .irq_mask = imx_mu_msi_parent_mask_irq, + .irq_unmask = imx_mu_msi_parent_unmask_irq, + .irq_ack = imx_mu_msi_parent_ack_irq, + .irq_compose_msi_msg = imx_mu_msi_parent_compose_msg, + .irq_set_affinity = imx_mu_msi_parent_set_affinity, +}; + +static int imx_mu_msi_domain_irq_alloc(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs, + void *args) +{ + struct imx_mu_msi *msi_data = domain->host_data; + unsigned long flags; + int pos, err = 0; + + WARN_ON(nr_irqs != 1); + + raw_spin_lock_irqsave(&msi_data->lock, flags); + pos = find_first_zero_bit(&msi_data->used, IMX_MU_CHANS); + if (pos < IMX_MU_CHANS) + __set_bit(pos, &msi_data->used); + else + err = -ENOSPC; + raw_spin_unlock_irqrestore(&msi_data->lock, flags); + + if (err) + return err; + + irq_domain_set_info(domain, virq, pos, + &imx_mu_msi_parent_chip, msi_data, + handle_edge_irq, NULL, NULL); + return 0; +} + +static void imx_mu_msi_domain_irq_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct imx_mu_msi *msi_data = irq_data_get_irq_chip_data(d); + unsigned long flags; + + raw_spin_lock_irqsave(&msi_data->lock, flags); + __clear_bit(d->hwirq, &msi_data->used); + raw_spin_unlock_irqrestore(&msi_data->lock, flags); +} + +static const struct irq_domain_ops imx_mu_msi_domain_ops = { + .alloc = imx_mu_msi_domain_irq_alloc, + .free = imx_mu_msi_domain_irq_free, +}; + +static void imx_mu_msi_irq_handler(struct irq_desc *desc) +{ + struct imx_mu_msi *msi_data = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + u32 status; + int i; + + status = imx_mu_read(msi_data, msi_data->cfg->xSR[IMX_MU_RSR]); + + chained_irq_enter(chip, desc); + for (i = 0; i < IMX_MU_CHANS; i++) { + if (status & IMX_MU_xSR_RFn(msi_data, i)) + generic_handle_domain_irq(msi_data->msi_domain, i); + } + chained_irq_exit(chip, desc); +} + +static int imx_mu_msi_domains_init(struct imx_mu_msi *msi_data, struct device *dev) +{ + struct fwnode_handle *fwnodes = dev_fwnode(dev); + struct irq_domain *parent; + + /* Initialize MSI domain parent */ + parent = irq_domain_create_linear(fwnodes, + IMX_MU_CHANS, + &imx_mu_msi_domain_ops, + msi_data); + if (!parent) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); + + msi_data->msi_domain = platform_msi_create_irq_domain(fwnodes, + &imx_mu_msi_domain_info, + parent); + + if (!msi_data->msi_domain) { + dev_err(dev, "failed to create MSI domain\n"); + irq_domain_remove(parent); + return -ENOMEM; + } + + irq_domain_set_pm_device(msi_data->msi_domain, dev); + + return 0; +} + +/* Register offset of different version MU IP */ +static const struct imx_mu_dcfg imx_mu_cfg_imx6sx = { + .type = 0, + .xTR = 0x0, + .xRR = 0x10, + .xSR = { + [IMX_MU_SR] = 0x20, + [IMX_MU_GSR] = 0x20, + [IMX_MU_TSR] = 0x20, + [IMX_MU_RSR] = 0x20, + }, + .xCR = { + [IMX_MU_GIER] = 0x24, + [IMX_MU_GCR] = 0x24, + [IMX_MU_TCR] = 0x24, + [IMX_MU_RCR] = 0x24, + }, +}; + +static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = { + .type = 0, + .xTR = 0x20, + .xRR = 0x40, + .xSR = { + [IMX_MU_SR] = 0x60, + [IMX_MU_GSR] = 0x60, + [IMX_MU_TSR] = 0x60, + [IMX_MU_RSR] = 0x60, + }, + .xCR = { + [IMX_MU_GIER] = 0x64, + [IMX_MU_GCR] = 0x64, + [IMX_MU_TCR] = 0x64, + [IMX_MU_RCR] = 0x64, + }, +}; + +static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { + .type = IMX_MU_V2, + .xTR = 0x200, + .xRR = 0x280, + .xSR = { + [IMX_MU_SR] = 0xC, + [IMX_MU_GSR] = 0x118, + [IMX_MU_TSR] = 0x124, + [IMX_MU_RSR] = 0x12C, + }, + .xCR = { + [IMX_MU_GIER] = 0x110, + [IMX_MU_GCR] = 0x114, + [IMX_MU_TCR] = 0x120, + [IMX_MU_RCR] = 0x128 + }, +}; + +static int __init imx_mu_of_init(struct device_node *dn, + struct device_node *parent, + const struct imx_mu_dcfg *cfg) +{ + struct platform_device *pdev = of_find_device_by_node(dn); + struct device_link *pd_link_a; + struct device_link *pd_link_b; + struct imx_mu_msi *msi_data; + struct resource *res; + struct device *pd_a; + struct device *pd_b; + struct device *dev; + int ret; + int irq; + + dev = &pdev->dev; + + msi_data = devm_kzalloc(&pdev->dev, sizeof(*msi_data), GFP_KERNEL); + if (!msi_data) + return -ENOMEM; + + msi_data->cfg = cfg; + + msi_data->regs = devm_platform_ioremap_resource_byname(pdev, "processor-a-side"); + if (IS_ERR(msi_data->regs)) { + dev_err(&pdev->dev, "failed to initialize 'regs'\n"); + return PTR_ERR(msi_data->regs); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "processor-b-side"); + if (!res) + return -EIO; + + msi_data->msiir_addr = res->start + msi_data->cfg->xTR; + + irq = platform_get_irq(pdev, 0); + if (irq <= 0) + return -ENODEV; + + platform_set_drvdata(pdev, msi_data); + + msi_data->clk = devm_clk_get(dev, NULL); + if (IS_ERR(msi_data->clk)) + return PTR_ERR(msi_data->clk); + + pd_a = dev_pm_domain_attach_by_name(dev, "processor-a-side"); + if (IS_ERR(pd_a)) + return PTR_ERR(pd_a); + + pd_b = dev_pm_domain_attach_by_name(dev, "processor-b-side"); + if (IS_ERR(pd_b)) + return PTR_ERR(pd_b); + + pd_link_a = device_link_add(dev, pd_a, + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + + if (!pd_link_a) { + dev_err(dev, "Failed to add device_link to mu a.\n"); + goto err_pd_a; + } + + pd_link_b = device_link_add(dev, pd_b, + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + + + if (!pd_link_b) { + dev_err(dev, "Failed to add device_link to mu a.\n"); + goto err_pd_b; + } + + ret = imx_mu_msi_domains_init(msi_data, dev); + if (ret) + goto err_dm_init; + + pm_runtime_enable(dev); + + irq_set_chained_handler_and_data(irq, + imx_mu_msi_irq_handler, + msi_data); + + return 0; + +err_dm_init: + device_link_remove(dev, pd_b); +err_pd_b: + device_link_remove(dev, pd_a); +err_pd_a: + return -EINVAL; +} + +static int __maybe_unused imx_mu_runtime_suspend(struct device *dev) +{ + struct imx_mu_msi *priv = dev_get_drvdata(dev); + + clk_disable_unprepare(priv->clk); + + return 0; +} + +static int __maybe_unused imx_mu_runtime_resume(struct device *dev) +{ + struct imx_mu_msi *priv = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) + dev_err(dev, "failed to enable clock\n"); + + return ret; +} + +static const struct dev_pm_ops imx_mu_pm_ops = { + SET_RUNTIME_PM_OPS(imx_mu_runtime_suspend, + imx_mu_runtime_resume, NULL) +}; + +static int __init imx_mu_imx7ulp_of_init(struct device_node *dn, + struct device_node *parent) +{ + return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx7ulp); +} + +static int __init imx_mu_imx6sx_of_init(struct device_node *dn, + struct device_node *parent) +{ + return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx6sx); +} + +static int __init imx_mu_imx8ulp_of_init(struct device_node *dn, + struct device_node *parent) +{ + return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx8ulp); +} + +IRQCHIP_PLATFORM_DRIVER_BEGIN(imx_mu_msi) +IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_of_init) +IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_of_init) +IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_of_init) +IRQCHIP_PLATFORM_DRIVER_END(imx_mu_msi, .pm = &imx_mu_pm_ops) + + +MODULE_AUTHOR("Frank Li "); +MODULE_DESCRIPTION("Freescale MU MSI controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-ls-extirq.c b/drivers/irqchip/irq-ls-extirq.c index 853b3972dbe7897c2c9f13d1437ab026a3faf0a3..d8d48b1f7c29d47afa700278d2cf6dded1b7051c 100644 --- a/drivers/irqchip/irq-ls-extirq.c +++ b/drivers/irqchip/irq-ls-extirq.c @@ -6,8 +6,7 @@ #include #include #include -#include -#include +#include #include #include @@ -16,13 +15,41 @@ #define LS1021A_SCFGREVCR 0x200 struct ls_extirq_data { - struct regmap *syscon; - u32 intpcr; + void __iomem *intpcr; + raw_spinlock_t lock; + bool big_endian; bool is_ls1021a_or_ls1043a; u32 nirq; struct irq_fwspec map[MAXIRQ]; }; +static void ls_extirq_intpcr_rmw(struct ls_extirq_data *priv, u32 mask, + u32 value) +{ + u32 intpcr; + + /* + * Serialize concurrent calls to ls_extirq_set_type() from multiple + * IRQ descriptors, making sure the read-modify-write is atomic. + */ + raw_spin_lock(&priv->lock); + + if (priv->big_endian) + intpcr = ioread32be(priv->intpcr); + else + intpcr = ioread32(priv->intpcr); + + intpcr &= ~mask; + intpcr |= value; + + if (priv->big_endian) + iowrite32be(intpcr, priv->intpcr); + else + iowrite32(intpcr, priv->intpcr); + + raw_spin_unlock(&priv->lock); +} + static int ls_extirq_set_type(struct irq_data *data, unsigned int type) { @@ -51,7 +78,8 @@ ls_extirq_set_type(struct irq_data *data, unsigned int type) default: return -EINVAL; } - regmap_update_bits(priv->syscon, priv->intpcr, mask, value); + + ls_extirq_intpcr_rmw(priv, mask, value); return irq_chip_set_type_parent(data, type); } @@ -143,7 +171,6 @@ ls_extirq_parse_map(struct ls_extirq_data *priv, struct device_node *node) static int __init ls_extirq_of_init(struct device_node *node, struct device_node *parent) { - struct irq_domain *domain, *parent_domain; struct ls_extirq_data *priv; int ret; @@ -151,40 +178,52 @@ ls_extirq_of_init(struct device_node *node, struct device_node *parent) parent_domain = irq_find_host(parent); if (!parent_domain) { pr_err("Cannot find parent domain\n"); - return -ENODEV; + ret = -ENODEV; + goto err_irq_find_host; } priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->syscon = syscon_node_to_regmap(node->parent); - if (IS_ERR(priv->syscon)) { - ret = PTR_ERR(priv->syscon); - pr_err("Failed to lookup parent regmap\n"); - goto out; + if (!priv) { + ret = -ENOMEM; + goto err_alloc_priv; } - ret = of_property_read_u32(node, "reg", &priv->intpcr); - if (ret) { - pr_err("Missing INTPCR offset value\n"); - goto out; + + /* + * All extirq OF nodes are under a scfg/syscon node with + * the 'ranges' property + */ + priv->intpcr = of_iomap(node, 0); + if (!priv->intpcr) { + pr_err("Cannot ioremap OF node %pOF\n", node); + ret = -ENOMEM; + goto err_iomap; } ret = ls_extirq_parse_map(priv, node); if (ret) - goto out; + goto err_parse_map; + priv->big_endian = of_device_is_big_endian(parent); priv->is_ls1021a_or_ls1043a = of_device_is_compatible(node, "fsl,ls1021a-extirq") || of_device_is_compatible(node, "fsl,ls1043a-extirq"); + raw_spin_lock_init(&priv->lock); domain = irq_domain_add_hierarchy(parent_domain, 0, priv->nirq, node, &extirq_domain_ops, priv); - if (!domain) + if (!domain) { ret = -ENOMEM; + goto err_add_hierarchy; + } -out: - if (ret) - kfree(priv); + return 0; + +err_add_hierarchy: +err_parse_map: + iounmap(priv->intpcr); +err_iomap: + kfree(priv); +err_alloc_priv: +err_irq_find_host: return ret; } diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c index b4927e425f7bc6b5fb63cd560c0e4722c6cb795c..527c90e0920e413452f86f1dc8ed2122353c8d5d 100644 --- a/drivers/irqchip/irq-ls-scfg-msi.c +++ b/drivers/irqchip/irq-ls-scfg-msi.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,7 +19,6 @@ #include #include #include -#include #define MSI_IRQS_PER_MSIR 32 #define MSI_MSIR_OFFSET 4 diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 56bf502d9c6739d60af8c5ae7b0c667bf10de4ba..2a349082af81d7685cce1dd0ee086b1e85c25e3e 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -21,11 +21,33 @@ #define RTL_ICTL_IRR2 0x10 #define RTL_ICTL_IRR3 0x14 +#define RTL_ICTL_NUM_INPUTS 32 + #define REG(x) (realtek_ictl_base + x) static DEFINE_RAW_SPINLOCK(irq_lock); static void __iomem *realtek_ictl_base; +/* + * IRR0-IRR3 store 4 bits per interrupt, but Realtek uses inverted numbering, + * placing IRQ 31 in the first four bits. A routing value of '0' means the + * interrupt is left disconnected. Routing values {1..15} connect to output + * lines {0..14}. + */ +#define IRR_OFFSET(idx) (4 * (3 - (idx * 4) / 32)) +#define IRR_SHIFT(idx) ((idx * 4) % 32) + +static void write_irr(void __iomem *irr0, int idx, u32 value) +{ + unsigned int offset = IRR_OFFSET(idx); + unsigned int shift = IRR_SHIFT(idx); + u32 irr; + + irr = readl(irr0 + offset) & ~(0xf << shift); + irr |= (value & 0xf) << shift; + writel(irr, irr0 + offset); +} + static void realtek_ictl_unmask_irq(struct irq_data *i) { unsigned long flags; @@ -62,8 +84,14 @@ static struct irq_chip realtek_ictl_irq = { static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { + unsigned long flags; + irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq); + raw_spin_lock_irqsave(&irq_lock, flags); + write_irr(REG(RTL_ICTL_IRR0), hw, 1); + raw_spin_unlock_irqrestore(&irq_lock, flags); + return 0; } @@ -95,90 +123,50 @@ out: chained_irq_exit(chip, desc); } -/* - * SoC interrupts are cascaded to MIPS CPU interrupts according to the - * interrupt-map in the device tree. Each SoC interrupt gets 4 bits for - * the CPU interrupt in an Interrupt Routing Register. Max 32 SoC interrupts - * thus go into 4 IRRs. A routing value of '0' means the interrupt is left - * disconnected. Routing values {1..15} connect to output lines {0..14}. - */ -static int __init map_interrupts(struct device_node *node, struct irq_domain *domain) -{ - struct device_node *cpu_ictl; - const __be32 *imap; - u32 imaplen, soc_int, cpu_int, tmp, regs[4]; - int ret, i, irr_regs[] = { - RTL_ICTL_IRR3, - RTL_ICTL_IRR2, - RTL_ICTL_IRR1, - RTL_ICTL_IRR0, - }; - u8 mips_irqs_set; - - ret = of_property_read_u32(node, "#address-cells", &tmp); - if (ret || tmp) - return -EINVAL; - - imap = of_get_property(node, "interrupt-map", &imaplen); - if (!imap || imaplen % 3) - return -EINVAL; - - mips_irqs_set = 0; - memset(regs, 0, sizeof(regs)); - for (i = 0; i < imaplen; i += 3 * sizeof(u32)) { - soc_int = be32_to_cpup(imap); - if (soc_int > 31) - return -EINVAL; - - cpu_ictl = of_find_node_by_phandle(be32_to_cpup(imap + 1)); - if (!cpu_ictl) - return -EINVAL; - ret = of_property_read_u32(cpu_ictl, "#interrupt-cells", &tmp); - of_node_put(cpu_ictl); - if (ret || tmp != 1) - return -EINVAL; - - cpu_int = be32_to_cpup(imap + 2); - if (cpu_int > 7 || cpu_int < 2) - return -EINVAL; - - if (!(mips_irqs_set & BIT(cpu_int))) { - irq_set_chained_handler_and_data(cpu_int, realtek_irq_dispatch, - domain); - mips_irqs_set |= BIT(cpu_int); - } - - /* Use routing values (1..6) for CPU interrupts (2..7) */ - regs[(soc_int * 4) / 32] |= (cpu_int - 1) << (soc_int * 4) % 32; - imap += 3; - } - - for (i = 0; i < 4; i++) - writel(regs[i], REG(irr_regs[i])); - - return 0; -} - static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) { + struct of_phandle_args oirq; struct irq_domain *domain; - int ret; + unsigned int soc_irq; + int parent_irq; realtek_ictl_base = of_iomap(node, 0); if (!realtek_ictl_base) return -ENXIO; - /* Disable all cascaded interrupts */ + /* Disable all cascaded interrupts and clear routing */ writel(0, REG(RTL_ICTL_GIMR)); + for (soc_irq = 0; soc_irq < RTL_ICTL_NUM_INPUTS; soc_irq++) + write_irr(REG(RTL_ICTL_IRR0), soc_irq, 0); + + if (WARN_ON(!of_irq_count(node))) { + /* + * If DT contains no parent interrupts, assume MIPS CPU IRQ 2 + * (HW0) is connected to the first output. This is the case for + * all known hardware anyway. "interrupt-map" is deprecated, so + * don't bother trying to parse that. + */ + oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller"); + oirq.args_count = 1; + oirq.args[0] = 2; + + parent_irq = irq_create_of_mapping(&oirq); + + of_node_put(oirq.np); + } else { + parent_irq = of_irq_get(node, 0); + } - domain = irq_domain_add_simple(node, 32, 0, - &irq_domain_ops, NULL); + if (parent_irq < 0) + return parent_irq; + else if (!parent_irq) + return -ENODEV; - ret = map_interrupts(node, domain); - if (ret) { - pr_err("invalid interrupt map\n"); - return ret; - } + domain = irq_domain_add_linear(node, RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); + if (!domain) + return -ENOMEM; + + irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain); return 0; } diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c index a73763d475f04e277f4794fe7ccc048d8e6313bc..6a3f7498ea8eadacc08807b09d0b1bdbc97cc1d0 100644 --- a/drivers/irqchip/irq-stm32-exti.c +++ b/drivers/irqchip/irq-stm32-exti.c @@ -716,7 +716,7 @@ static int stm32_exti_h_domain_alloc(struct irq_domain *dm, irq_domain_set_hwirq_and_chip(dm, virq, hwirq, chip, chip_data); - if (!host_data->drv_data || !host_data->drv_data->desc_irqs) + if (!host_data->drv_data->desc_irqs) return -EINVAL; desc_irq = host_data->drv_data->desc_irqs[hwirq]; diff --git a/drivers/isdn/capi/kcapi.c b/drivers/isdn/capi/kcapi.c index e69c4bf557bf8c2647ddbee2bfd62978c131238d..ae24848af2332456289a7fe261eef53f674262c1 100644 --- a/drivers/isdn/capi/kcapi.c +++ b/drivers/isdn/capi/kcapi.c @@ -798,7 +798,7 @@ u16 capi20_get_serial(u32 contr, u8 serial[CAPI_SERIAL_LEN]) u16 ret; if (contr == 0) { - strlcpy(serial, driver_serial, CAPI_SERIAL_LEN); + strscpy(serial, driver_serial, CAPI_SERIAL_LEN); return CAPI_NOERROR; } @@ -806,7 +806,7 @@ u16 capi20_get_serial(u32 contr, u8 serial[CAPI_SERIAL_LEN]) ctr = get_capi_ctr_by_nr(contr); if (ctr && ctr->state == CAPI_CTR_RUNNING) { - strlcpy(serial, ctr->serial, CAPI_SERIAL_LEN); + strscpy(serial, ctr->serial, CAPI_SERIAL_LEN); ret = CAPI_NOERROR; } else ret = CAPI_REGNOTINSTALLED; diff --git a/drivers/isdn/hardware/mISDN/hfcpci.c b/drivers/isdn/hardware/mISDN/hfcpci.c index af17459c1a5c02d847d801be97f72605c1870e51..e964a8dd8512aea84f15e75361ae96b3d32c7d70 100644 --- a/drivers/isdn/hardware/mISDN/hfcpci.c +++ b/drivers/isdn/hardware/mISDN/hfcpci.c @@ -2345,8 +2345,7 @@ HFC_init(void) static void __exit HFC_cleanup(void) { - if (timer_pending(&hfc_tl)) - del_timer_sync(&hfc_tl); + del_timer_sync(&hfc_tl); pci_unregister_driver(&hfc_driver); } diff --git a/drivers/isdn/mISDN/l1oip.h b/drivers/isdn/mISDN/l1oip.h index 7ea10db20e3a6567d91b4d6d8b63daa99eadaa98..48133d022812071543fb4e0fee48af6968619f2d 100644 --- a/drivers/isdn/mISDN/l1oip.h +++ b/drivers/isdn/mISDN/l1oip.h @@ -59,6 +59,7 @@ struct l1oip { int bundle; /* bundle channels in one frm */ int codec; /* codec to use for transmis. */ int limit; /* limit number of bchannels */ + bool shutdown; /* if card is released */ /* timer */ struct timer_list keep_tl; diff --git a/drivers/isdn/mISDN/l1oip_core.c b/drivers/isdn/mISDN/l1oip_core.c index 2c40412466e6fb39fb64c155b6337e382c1cedfa..a77195e378b7ba425c3838e8e98db03c27acfef0 100644 --- a/drivers/isdn/mISDN/l1oip_core.c +++ b/drivers/isdn/mISDN/l1oip_core.c @@ -275,7 +275,7 @@ l1oip_socket_send(struct l1oip *hc, u8 localcodec, u8 channel, u32 chanmask, p = frame; /* restart timer */ - if (time_before(hc->keep_tl.expires, jiffies + 5 * HZ)) + if (time_before(hc->keep_tl.expires, jiffies + 5 * HZ) && !hc->shutdown) mod_timer(&hc->keep_tl, jiffies + L1OIP_KEEPALIVE * HZ); else hc->keep_tl.expires = jiffies + L1OIP_KEEPALIVE * HZ; @@ -601,7 +601,9 @@ multiframe: goto multiframe; /* restart timer */ - if (time_before(hc->timeout_tl.expires, jiffies + 5 * HZ) || !hc->timeout_on) { + if ((time_before(hc->timeout_tl.expires, jiffies + 5 * HZ) || + !hc->timeout_on) && + !hc->shutdown) { hc->timeout_on = 1; mod_timer(&hc->timeout_tl, jiffies + L1OIP_TIMEOUT * HZ); } else /* only adjust timer */ @@ -1232,11 +1234,10 @@ release_card(struct l1oip *hc) { int ch; - if (timer_pending(&hc->keep_tl)) - del_timer(&hc->keep_tl); + hc->shutdown = true; - if (timer_pending(&hc->timeout_tl)) - del_timer(&hc->timeout_tl); + del_timer_sync(&hc->keep_tl); + del_timer_sync(&hc->timeout_tl); cancel_work_sync(&hc->workq); diff --git a/drivers/leds/flash/leds-as3645a.c b/drivers/leds/flash/leds-as3645a.c index aa3f82be0a9c9abe088b3a04e362dcd06a4801c9..bb2249771acb29612e987903021a39e7a46868e2 100644 --- a/drivers/leds/flash/leds-as3645a.c +++ b/drivers/leds/flash/leds-as3645a.c @@ -724,7 +724,7 @@ out_put_nodes: return rval; } -static int as3645a_remove(struct i2c_client *client) +static void as3645a_remove(struct i2c_client *client) { struct as3645a *flash = i2c_get_clientdata(client); @@ -740,8 +740,6 @@ static int as3645a_remove(struct i2c_client *client) fwnode_handle_put(flash->flash_node); fwnode_handle_put(flash->indicator_node); - - return 0; } static const struct i2c_device_id as3645a_id_table[] = { diff --git a/drivers/leds/flash/leds-lm3601x.c b/drivers/leds/flash/leds-lm3601x.c index d0e1d4814042e52c5896df61c923e197181c63e3..78730e066a733e35b7f63b05ca4582f98f463c1c 100644 --- a/drivers/leds/flash/leds-lm3601x.c +++ b/drivers/leds/flash/leds-lm3601x.c @@ -440,15 +440,16 @@ static int lm3601x_probe(struct i2c_client *client) return lm3601x_register_leds(led, fwnode); } -static int lm3601x_remove(struct i2c_client *client) +static void lm3601x_remove(struct i2c_client *client) { struct lm3601x_led *led = i2c_get_clientdata(client); + int ret; - mutex_destroy(&led->lock); - - return regmap_update_bits(led->regmap, LM3601X_ENABLE_REG, - LM3601X_ENABLE_MASK, - LM3601X_MODE_STANDBY); + ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG, + LM3601X_ENABLE_MASK, LM3601X_MODE_STANDBY); + if (ret) + dev_warn(&client->dev, + "Failed to put into standby (%pe)\n", ERR_PTR(ret)); } static const struct i2c_device_id lm3601x_id[] = { diff --git a/drivers/leds/flash/leds-rt4505.c b/drivers/leds/flash/leds-rt4505.c index ee129ab7255de75bb17eef54141f835a5fdc46c2..e404fe8b0314f979d0d74b46144c9add5c5f5828 100644 --- a/drivers/leds/flash/leds-rt4505.c +++ b/drivers/leds/flash/leds-rt4505.c @@ -393,12 +393,11 @@ static int rt4505_probe(struct i2c_client *client) return 0; } -static int rt4505_remove(struct i2c_client *client) +static void rt4505_remove(struct i2c_client *client) { struct rt4505_priv *priv = i2c_get_clientdata(client); v4l2_flash_release(priv->v4l2_flash); - return 0; } static void rt4505_shutdown(struct i2c_client *client) diff --git a/drivers/leds/leds-an30259a.c b/drivers/leds/leds-an30259a.c index a0df1fb28774d7e772652fe462feb499d672e2f1..e072ee5409f7e0bc1c52caeb263a411dadc3ba4c 100644 --- a/drivers/leds/leds-an30259a.c +++ b/drivers/leds/leds-an30259a.c @@ -334,13 +334,11 @@ exit: return err; } -static int an30259a_remove(struct i2c_client *client) +static void an30259a_remove(struct i2c_client *client) { struct an30259a *chip = i2c_get_clientdata(client); mutex_destroy(&chip->mutex); - - return 0; } static const struct of_device_id an30259a_match_table[] = { diff --git a/drivers/leds/leds-aw2013.c b/drivers/leds/leds-aw2013.c index 80d937454aeef3fc953dd3dd344f28a5b3a6a95e..0b52fc9097c6e1abcb3459ffb7405bd83c419c3c 100644 --- a/drivers/leds/leds-aw2013.c +++ b/drivers/leds/leds-aw2013.c @@ -401,15 +401,13 @@ error: return ret; } -static int aw2013_remove(struct i2c_client *client) +static void aw2013_remove(struct i2c_client *client) { struct aw2013 *chip = i2c_get_clientdata(client); aw2013_chip_disable(chip); mutex_destroy(&chip->mutex); - - return 0; } static const struct of_device_id aw2013_match_table[] = { diff --git a/drivers/leds/leds-bd2802.c b/drivers/leds/leds-bd2802.c index 8bbaef5a2986fb014654774a77c53719cf1d01e0..2b6678f6bd565381408bd25864b5ea9a73d35da4 100644 --- a/drivers/leds/leds-bd2802.c +++ b/drivers/leds/leds-bd2802.c @@ -722,7 +722,7 @@ failed_unregister_dev_file: return ret; } -static int bd2802_remove(struct i2c_client *client) +static void bd2802_remove(struct i2c_client *client) { struct bd2802_led *led = i2c_get_clientdata(client); int i; @@ -733,8 +733,6 @@ static int bd2802_remove(struct i2c_client *client) bd2802_disable_adv_conf(led); for (i = 0; i < ARRAY_SIZE(bd2802_attributes); i++) device_remove_file(&led->client->dev, bd2802_attributes[i]); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c index bd7d0d5cf3b626a62fb2ca9b500d5b8f57f602a7..3fb6a2fdaefa67bd2818cd29bfdb8f1cb797b7ec 100644 --- a/drivers/leds/leds-blinkm.c +++ b/drivers/leds/leds-blinkm.c @@ -677,7 +677,7 @@ exit: return err; } -static int blinkm_remove(struct i2c_client *client) +static void blinkm_remove(struct i2c_client *client) { struct blinkm_data *data = i2c_get_clientdata(client); int ret = 0; @@ -716,7 +716,6 @@ static int blinkm_remove(struct i2c_client *client) dev_err(&client->dev, "Failure in blinkm_remove ignored. Continuing.\n"); sysfs_remove_group(&client->dev.kobj, &blinkm_group); - return 0; } static const struct i2c_device_id blinkm_id[] = { diff --git a/drivers/leds/leds-is31fl32xx.c b/drivers/leds/leds-is31fl32xx.c index fc63fce38c1965a0e0bf7d35bb3ad3405cb4dfee..0d219c1ac3b54bf64cfd43a0f7cd162d4ff78d22 100644 --- a/drivers/leds/leds-is31fl32xx.c +++ b/drivers/leds/leds-is31fl32xx.c @@ -457,7 +457,7 @@ static int is31fl32xx_probe(struct i2c_client *client, return 0; } -static int is31fl32xx_remove(struct i2c_client *client) +static void is31fl32xx_remove(struct i2c_client *client) { struct is31fl32xx_priv *priv = i2c_get_clientdata(client); int ret; @@ -466,8 +466,6 @@ static int is31fl32xx_remove(struct i2c_client *client) if (ret) dev_err(&client->dev, "Failed to reset registers on removal (%pe)\n", ERR_PTR(ret)); - - return 0; } /* diff --git a/drivers/leds/leds-lm3530.c b/drivers/leds/leds-lm3530.c index e72393534b7218ae1a560aeba54a507745d24486..ba906c253c7f439e9a685e107ab6710078ff48da 100644 --- a/drivers/leds/leds-lm3530.c +++ b/drivers/leds/leds-lm3530.c @@ -470,13 +470,12 @@ static int lm3530_probe(struct i2c_client *client, return 0; } -static int lm3530_remove(struct i2c_client *client) +static void lm3530_remove(struct i2c_client *client) { struct lm3530_data *drvdata = i2c_get_clientdata(client); lm3530_led_disable(drvdata); led_classdev_unregister(&drvdata->led_dev); - return 0; } static const struct i2c_device_id lm3530_id[] = { diff --git a/drivers/leds/leds-lm3532.c b/drivers/leds/leds-lm3532.c index beb53040e09e4801e18f7d8da8a9cbeca7a0ae26..db64d44bcbbfb47deb097c6400551257c0ea326f 100644 --- a/drivers/leds/leds-lm3532.c +++ b/drivers/leds/leds-lm3532.c @@ -704,7 +704,7 @@ static int lm3532_probe(struct i2c_client *client, return ret; } -static int lm3532_remove(struct i2c_client *client) +static void lm3532_remove(struct i2c_client *client) { struct lm3532_data *drvdata = i2c_get_clientdata(client); @@ -712,8 +712,6 @@ static int lm3532_remove(struct i2c_client *client) if (drvdata->enable_gpio) gpiod_direction_output(drvdata->enable_gpio, 0); - - return 0; } static const struct of_device_id of_lm3532_leds_match[] = { diff --git a/drivers/leds/leds-lm355x.c b/drivers/leds/leds-lm355x.c index 2d3e11845ba51696062236e9b4b0edb7128905ee..daa35927b301937d4a881d4460973c2af1266a4c 100644 --- a/drivers/leds/leds-lm355x.c +++ b/drivers/leds/leds-lm355x.c @@ -491,7 +491,7 @@ err_out: return err; } -static int lm355x_remove(struct i2c_client *client) +static void lm355x_remove(struct i2c_client *client) { struct lm355x_chip_data *chip = i2c_get_clientdata(client); struct lm355x_reg_data *preg = chip->regs; @@ -501,8 +501,6 @@ static int lm355x_remove(struct i2c_client *client) led_classdev_unregister(&chip->cdev_torch); led_classdev_unregister(&chip->cdev_flash); dev_info(&client->dev, "%s is removed\n", lm355x_name[chip->type]); - - return 0; } static const struct i2c_device_id lm355x_id[] = { diff --git a/drivers/leds/leds-lm3642.c b/drivers/leds/leds-lm3642.c index 435309154e6b5f80cb44bd2f6b302bcdcd722f59..428a5d92815045d6d7c9f1acb1c566e6ccf3232b 100644 --- a/drivers/leds/leds-lm3642.c +++ b/drivers/leds/leds-lm3642.c @@ -380,7 +380,7 @@ err_out: return err; } -static int lm3642_remove(struct i2c_client *client) +static void lm3642_remove(struct i2c_client *client) { struct lm3642_chip_data *chip = i2c_get_clientdata(client); @@ -388,7 +388,6 @@ static int lm3642_remove(struct i2c_client *client) led_classdev_unregister(&chip->cdev_torch); led_classdev_unregister(&chip->cdev_flash); regmap_write(chip->regmap, REG_ENABLE, 0); - return 0; } static const struct i2c_device_id lm3642_id[] = { diff --git a/drivers/leds/leds-lm3692x.c b/drivers/leds/leds-lm3692x.c index 87cd24ce3f955d05fb31f404b18e9f3d8e4aa95e..54b4662bff41ebe5f92f410fefd2937164b57088 100644 --- a/drivers/leds/leds-lm3692x.c +++ b/drivers/leds/leds-lm3692x.c @@ -491,14 +491,12 @@ static int lm3692x_probe(struct i2c_client *client, return 0; } -static int lm3692x_remove(struct i2c_client *client) +static void lm3692x_remove(struct i2c_client *client) { struct lm3692x_led *led = i2c_get_clientdata(client); lm3692x_leds_disable(led); mutex_destroy(&led->lock); - - return 0; } static const struct i2c_device_id lm3692x_id[] = { diff --git a/drivers/leds/leds-lm3697.c b/drivers/leds/leds-lm3697.c index a8c9322558cc51c912821d9407581aeee33d9e8f..71231a60eebce8cf208fdcc3dbe2853f211b93a1 100644 --- a/drivers/leds/leds-lm3697.c +++ b/drivers/leds/leds-lm3697.c @@ -337,7 +337,7 @@ static int lm3697_probe(struct i2c_client *client, return lm3697_init(led); } -static int lm3697_remove(struct i2c_client *client) +static void lm3697_remove(struct i2c_client *client) { struct lm3697 *led = i2c_get_clientdata(client); struct device *dev = &led->client->dev; @@ -345,10 +345,8 @@ static int lm3697_remove(struct i2c_client *client) ret = regmap_update_bits(led->regmap, LM3697_CTRL_ENABLE, LM3697_CTRL_A_B_EN, 0); - if (ret) { + if (ret) dev_err(dev, "Failed to disable the device\n"); - return ret; - } if (led->enable_gpio) gpiod_direction_output(led->enable_gpio, 0); @@ -360,8 +358,6 @@ static int lm3697_remove(struct i2c_client *client) } mutex_destroy(&led->lock); - - return 0; } static const struct i2c_device_id lm3697_id[] = { diff --git a/drivers/leds/leds-lp3944.c b/drivers/leds/leds-lp3944.c index 437c711b2a27e8a1375ef5bc009781c0a1ebc516..673ad8c04f415be6c08423497269bdc138d199d0 100644 --- a/drivers/leds/leds-lp3944.c +++ b/drivers/leds/leds-lp3944.c @@ -397,7 +397,7 @@ static int lp3944_probe(struct i2c_client *client, return 0; } -static int lp3944_remove(struct i2c_client *client) +static void lp3944_remove(struct i2c_client *client) { struct lp3944_platform_data *pdata = dev_get_platdata(&client->dev); struct lp3944_data *data = i2c_get_clientdata(client); @@ -414,8 +414,6 @@ static int lp3944_remove(struct i2c_client *client) default: break; } - - return 0; } /* lp3944 i2c driver struct */ diff --git a/drivers/leds/leds-lp3952.c b/drivers/leds/leds-lp3952.c index 6ee9131fbf25c4069e8a35266adb76807c9afb65..bf0ad1b5ce24953ed56415574a0530c8f0bb8e41 100644 --- a/drivers/leds/leds-lp3952.c +++ b/drivers/leds/leds-lp3952.c @@ -255,15 +255,13 @@ static int lp3952_probe(struct i2c_client *client, return 0; } -static int lp3952_remove(struct i2c_client *client) +static void lp3952_remove(struct i2c_client *client) { struct lp3952_led_array *priv; priv = i2c_get_clientdata(client); lp3952_on_off(priv, LP3952_LED_ALL, false); gpiod_set_value(priv->enable_gpio, 0); - - return 0; } static const struct i2c_device_id lp3952_id[] = { diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c index e129dcc656b8f35544c6d2ac4e13c28a38c10bd8..28d6b39fa72dbcb04857d63d51c36e4ef53d5c03 100644 --- a/drivers/leds/leds-lp50xx.c +++ b/drivers/leds/leds-lp50xx.c @@ -563,7 +563,7 @@ static int lp50xx_probe(struct i2c_client *client) return lp50xx_probe_dt(led); } -static int lp50xx_remove(struct i2c_client *client) +static void lp50xx_remove(struct i2c_client *client) { struct lp50xx *led = i2c_get_clientdata(client); int ret; @@ -579,8 +579,6 @@ static int lp50xx_remove(struct i2c_client *client) } mutex_destroy(&led->lock); - - return 0; } static const struct i2c_device_id lp50xx_id[] = { diff --git a/drivers/leds/leds-lp5521.c b/drivers/leds/leds-lp5521.c index a9e7507c998cd797b7717e3a60061eef270cbfbe..7ff20c2605041bb8ec79c69288235f89abb115e6 100644 --- a/drivers/leds/leds-lp5521.c +++ b/drivers/leds/leds-lp5521.c @@ -579,7 +579,7 @@ err_init: return ret; } -static int lp5521_remove(struct i2c_client *client) +static void lp5521_remove(struct i2c_client *client) { struct lp55xx_led *led = i2c_get_clientdata(client); struct lp55xx_chip *chip = led->chip; @@ -587,8 +587,6 @@ static int lp5521_remove(struct i2c_client *client) lp5521_stop_all_engines(chip); lp55xx_unregister_sysfs(chip); lp55xx_deinit_device(chip); - - return 0; } static const struct i2c_device_id lp5521_id[] = { diff --git a/drivers/leds/leds-lp5523.c b/drivers/leds/leds-lp5523.c index b1590cb4a188783b0595d9c713357ac8cf12fccb..369d40b0b65b85d64faeb82c43a6772b1d374662 100644 --- a/drivers/leds/leds-lp5523.c +++ b/drivers/leds/leds-lp5523.c @@ -947,7 +947,7 @@ err_init: return ret; } -static int lp5523_remove(struct i2c_client *client) +static void lp5523_remove(struct i2c_client *client) { struct lp55xx_led *led = i2c_get_clientdata(client); struct lp55xx_chip *chip = led->chip; @@ -955,8 +955,6 @@ static int lp5523_remove(struct i2c_client *client) lp5523_stop_all_engines(chip); lp55xx_unregister_sysfs(chip); lp55xx_deinit_device(chip); - - return 0; } static const struct i2c_device_id lp5523_id[] = { diff --git a/drivers/leds/leds-lp5562.c b/drivers/leds/leds-lp5562.c index 31c14016d2896e7ea3597ff75818447c44e6724f..0e490085ff35860749586a6a4409cea9f07559ef 100644 --- a/drivers/leds/leds-lp5562.c +++ b/drivers/leds/leds-lp5562.c @@ -573,7 +573,7 @@ err_init: return ret; } -static int lp5562_remove(struct i2c_client *client) +static void lp5562_remove(struct i2c_client *client) { struct lp55xx_led *led = i2c_get_clientdata(client); struct lp55xx_chip *chip = led->chip; @@ -582,8 +582,6 @@ static int lp5562_remove(struct i2c_client *client) lp55xx_unregister_sysfs(chip); lp55xx_deinit_device(chip); - - return 0; } static const struct i2c_device_id lp5562_id[] = { diff --git a/drivers/leds/leds-lp8501.c b/drivers/leds/leds-lp8501.c index 2d2fda2ab10461e56ed77c254e1dc62d71cb03c1..ae11a02c0ab23b53425d99b5efd57e5f4466aa8e 100644 --- a/drivers/leds/leds-lp8501.c +++ b/drivers/leds/leds-lp8501.c @@ -362,7 +362,7 @@ err_init: return ret; } -static int lp8501_remove(struct i2c_client *client) +static void lp8501_remove(struct i2c_client *client) { struct lp55xx_led *led = i2c_get_clientdata(client); struct lp55xx_chip *chip = led->chip; @@ -370,8 +370,6 @@ static int lp8501_remove(struct i2c_client *client) lp8501_stop_engine(chip); lp55xx_unregister_sysfs(chip); lp55xx_deinit_device(chip); - - return 0; } static const struct i2c_device_id lp8501_id[] = { diff --git a/drivers/leds/leds-lp8860.c b/drivers/leds/leds-lp8860.c index 3c693d5e3b44cdb5c0afd68cdc90e33f8dd8e7c2..e2b36d3187eb63f0a7b89373d95ae06135d15667 100644 --- a/drivers/leds/leds-lp8860.c +++ b/drivers/leds/leds-lp8860.c @@ -445,7 +445,7 @@ static int lp8860_probe(struct i2c_client *client, return 0; } -static int lp8860_remove(struct i2c_client *client) +static void lp8860_remove(struct i2c_client *client) { struct lp8860_led *led = i2c_get_clientdata(client); int ret; @@ -461,8 +461,6 @@ static int lp8860_remove(struct i2c_client *client) } mutex_destroy(&led->lock); - - return 0; } static const struct i2c_device_id lp8860_id[] = { diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c index f72b5d1be3a6979b46130a8e0bec0cb71c124f13..df83d97cb47967f02f89bc8e2c68fdd75b154581 100644 --- a/drivers/leds/leds-pca9532.c +++ b/drivers/leds/leds-pca9532.c @@ -52,7 +52,7 @@ struct pca9532_data { static int pca9532_probe(struct i2c_client *client, const struct i2c_device_id *id); -static int pca9532_remove(struct i2c_client *client); +static void pca9532_remove(struct i2c_client *client); enum { pca9530, @@ -546,13 +546,11 @@ static int pca9532_probe(struct i2c_client *client, return pca9532_configure(client, data, pca9532_pdata); } -static int pca9532_remove(struct i2c_client *client) +static void pca9532_remove(struct i2c_client *client) { struct pca9532_data *data = i2c_get_clientdata(client); pca9532_destroy_devices(data, data->chip_info->num_leds); - - return 0; } module_i2c_driver(pca9532_driver); diff --git a/drivers/leds/leds-pca963x.c b/drivers/leds/leds-pca963x.c index 00aecd67e34834f3b2b719e318349d05d86ec06a..a7e052c1db5315cdbf0343b3f4082ba11fdeb06a 100644 --- a/drivers/leds/leds-pca963x.c +++ b/drivers/leds/leds-pca963x.c @@ -101,6 +101,7 @@ struct pca963x_led { struct pca963x *chip; struct led_classdev led_cdev; int led_num; /* 0 .. 15 potentially */ + bool blinking; u8 gdc; u8 gfrq; }; @@ -129,12 +130,21 @@ static int pca963x_brightness(struct pca963x_led *led, switch (brightness) { case LED_FULL: - val = (ledout & ~mask) | (PCA963X_LED_ON << shift); + if (led->blinking) { + val = (ledout & ~mask) | (PCA963X_LED_GRP_PWM << shift); + ret = i2c_smbus_write_byte_data(client, + PCA963X_PWM_BASE + + led->led_num, + LED_FULL); + } else { + val = (ledout & ~mask) | (PCA963X_LED_ON << shift); + } ret = i2c_smbus_write_byte_data(client, ledout_addr, val); break; case LED_OFF: val = ledout & ~mask; ret = i2c_smbus_write_byte_data(client, ledout_addr, val); + led->blinking = false; break; default: ret = i2c_smbus_write_byte_data(client, @@ -144,7 +154,11 @@ static int pca963x_brightness(struct pca963x_led *led, if (ret < 0) return ret; - val = (ledout & ~mask) | (PCA963X_LED_PWM << shift); + if (led->blinking) + val = (ledout & ~mask) | (PCA963X_LED_GRP_PWM << shift); + else + val = (ledout & ~mask) | (PCA963X_LED_PWM << shift); + ret = i2c_smbus_write_byte_data(client, ledout_addr, val); break; } @@ -181,6 +195,7 @@ static void pca963x_blink(struct pca963x_led *led) } mutex_unlock(&led->chip->mutex); + led->blinking = true; } static int pca963x_power_state(struct pca963x_led *led) @@ -275,6 +290,8 @@ static int pca963x_blink_set(struct led_classdev *led_cdev, led->gfrq = gfrq; pca963x_blink(led); + led->led_cdev.brightness = LED_FULL; + pca963x_led_set(led_cdev, LED_FULL); *delay_on = time_on; *delay_off = time_off; @@ -337,6 +354,7 @@ static int pca963x_register_leds(struct i2c_client *client, led->led_cdev.brightness_set_blocking = pca963x_led_set; if (hw_blink) led->led_cdev.blink_set = pca963x_blink_set; + led->blinking = false; init_data.fwnode = child; /* for backwards compatibility */ diff --git a/drivers/leds/leds-tca6507.c b/drivers/leds/leds-tca6507.c index 1473ced8664c67dd40cd9f2e0a851abb5b221eb2..161bef65c6b7b88f19b5f2340a0a34cca336c4c7 100644 --- a/drivers/leds/leds-tca6507.c +++ b/drivers/leds/leds-tca6507.c @@ -790,7 +790,7 @@ exit: return err; } -static int tca6507_remove(struct i2c_client *client) +static void tca6507_remove(struct i2c_client *client) { int i; struct tca6507_chip *tca = i2c_get_clientdata(client); @@ -802,8 +802,6 @@ static int tca6507_remove(struct i2c_client *client) } tca6507_remove_gpio(tca); cancel_work_sync(&tca->work); - - return 0; } static struct i2c_driver tca6507_driver = { diff --git a/drivers/leds/leds-turris-omnia.c b/drivers/leds/leds-turris-omnia.c index eac6f4a573b2fc4803231d9b9b39945c0600cec8..c7c9851c894a9ddd83b7aa022431c6ea8dac8c64 100644 --- a/drivers/leds/leds-turris-omnia.c +++ b/drivers/leds/leds-turris-omnia.c @@ -242,7 +242,7 @@ static int omnia_leds_probe(struct i2c_client *client, return 0; } -static int omnia_leds_remove(struct i2c_client *client) +static void omnia_leds_remove(struct i2c_client *client) { u8 buf[5]; @@ -258,8 +258,6 @@ static int omnia_leds_remove(struct i2c_client *client) buf[4] = 255; i2c_master_send(client, buf, 5); - - return 0; } static const struct of_device_id of_omnia_leds_match[] = { diff --git a/drivers/leds/simple/simatic-ipc-leds-gpio.c b/drivers/leds/simple/simatic-ipc-leds-gpio.c index 4c9e663a90ba1374645a0b0678a34487230047b1..b9eeb8702df0c345b5df53c1cb06948859528db3 100644 --- a/drivers/leds/simple/simatic-ipc-leds-gpio.c +++ b/drivers/leds/simple/simatic-ipc-leds-gpio.c @@ -13,28 +13,45 @@ #include #include #include +#include -static struct gpiod_lookup_table simatic_ipc_led_gpio_table = { +static struct gpiod_lookup_table *simatic_ipc_led_gpio_table; + +static struct gpiod_lookup_table simatic_ipc_led_gpio_table_127e = { .dev_id = "leds-gpio", .table = { - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 0, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 52, NULL, 1, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 53, NULL, 2, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 57, NULL, 3, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 58, NULL, 4, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 60, NULL, 5, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 0, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 56, NULL, 6, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 59, NULL, 7, GPIO_ACTIVE_HIGH), }, }; +static struct gpiod_lookup_table simatic_ipc_led_gpio_table_227g = { + .dev_id = "leds-gpio", + .table = { + GPIO_LOOKUP_IDX("gpio-f7188x-2", 0, NULL, 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-2", 1, NULL, 1, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-2", 2, NULL, 2, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-2", 3, NULL, 3, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-2", 4, NULL, 4, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-2", 5, NULL, 5, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("gpio-f7188x-3", 6, NULL, 6, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP_IDX("gpio-f7188x-3", 7, NULL, 7, GPIO_ACTIVE_HIGH), + } +}; + static const struct gpio_led simatic_ipc_gpio_leds[] = { - { .name = "green:" LED_FUNCTION_STATUS "-3" }, { .name = "red:" LED_FUNCTION_STATUS "-1" }, { .name = "green:" LED_FUNCTION_STATUS "-1" }, { .name = "red:" LED_FUNCTION_STATUS "-2" }, { .name = "green:" LED_FUNCTION_STATUS "-2" }, { .name = "red:" LED_FUNCTION_STATUS "-3" }, + { .name = "green:" LED_FUNCTION_STATUS "-3" }, }; static const struct gpio_led_platform_data simatic_ipc_gpio_leds_pdata = { @@ -46,7 +63,7 @@ static struct platform_device *simatic_leds_pdev; static int simatic_ipc_leds_gpio_remove(struct platform_device *pdev) { - gpiod_remove_lookup_table(&simatic_ipc_led_gpio_table); + gpiod_remove_lookup_table(simatic_ipc_led_gpio_table); platform_device_unregister(simatic_leds_pdev); return 0; @@ -54,10 +71,25 @@ static int simatic_ipc_leds_gpio_remove(struct platform_device *pdev) static int simatic_ipc_leds_gpio_probe(struct platform_device *pdev) { + const struct simatic_ipc_platform *plat = pdev->dev.platform_data; struct gpio_desc *gpiod; int err; - gpiod_add_lookup_table(&simatic_ipc_led_gpio_table); + switch (plat->devmode) { + case SIMATIC_IPC_DEVICE_127E: + simatic_ipc_led_gpio_table = &simatic_ipc_led_gpio_table_127e; + break; + case SIMATIC_IPC_DEVICE_227G: + if (!IS_ENABLED(CONFIG_GPIO_F7188X)) + return -ENODEV; + request_module("gpio-f7188x"); + simatic_ipc_led_gpio_table = &simatic_ipc_led_gpio_table_227g; + break; + default: + return -ENODEV; + } + + gpiod_add_lookup_table(simatic_ipc_led_gpio_table); simatic_leds_pdev = platform_device_register_resndata(NULL, "leds-gpio", PLATFORM_DEVID_NONE, NULL, 0, &simatic_ipc_gpio_leds_pdata, diff --git a/drivers/macintosh/ams/ams-i2c.c b/drivers/macintosh/ams/ams-i2c.c index d2f0cde6f9c742325417b4abf1dbabedea424afe..3ded340699fb5c4d6a12d857938c3212ff58a74a 100644 --- a/drivers/macintosh/ams/ams-i2c.c +++ b/drivers/macintosh/ams/ams-i2c.c @@ -58,7 +58,7 @@ enum ams_i2c_cmd { static int ams_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id); -static int ams_i2c_remove(struct i2c_client *client); +static void ams_i2c_remove(struct i2c_client *client); static const struct i2c_device_id ams_id[] = { { "MAC,accelerometer_1", 0 }, @@ -230,7 +230,7 @@ static int ams_i2c_probe(struct i2c_client *client, return 0; } -static int ams_i2c_remove(struct i2c_client *client) +static void ams_i2c_remove(struct i2c_client *client) { if (ams_info.has_device) { ams_sensor_detach(); @@ -245,8 +245,6 @@ static int ams_i2c_remove(struct i2c_client *client) ams_info.has_device = 0; } - - return 0; } static void ams_i2c_exit(void) diff --git a/drivers/macintosh/therm_adt746x.c b/drivers/macintosh/therm_adt746x.c index e604cbc91763d20329875014f6acca3fdc56e066..b004ea2a110255d6f356b8ef2a8f2f0900dcf631 100644 --- a/drivers/macintosh/therm_adt746x.c +++ b/drivers/macintosh/therm_adt746x.c @@ -563,7 +563,7 @@ static int probe_thermostat(struct i2c_client *client, return 0; } -static int remove_thermostat(struct i2c_client *client) +static void remove_thermostat(struct i2c_client *client) { struct thermostat *th = i2c_get_clientdata(client); int i; @@ -585,8 +585,6 @@ static int remove_thermostat(struct i2c_client *client) write_both_fan_speed(th, -1); kfree(th); - - return 0; } static const struct i2c_device_id therm_adt746x_id[] = { diff --git a/drivers/macintosh/therm_windtunnel.c b/drivers/macintosh/therm_windtunnel.c index 9226b74fa08f63601727e127e8a54c13f0c86f94..b8228ca404544741fb79c7db1b97146144f79d49 100644 --- a/drivers/macintosh/therm_windtunnel.c +++ b/drivers/macintosh/therm_windtunnel.c @@ -317,24 +317,26 @@ static void do_attach(struct i2c_adapter *adapter) if (x.running || strncmp(adapter->name, "uni-n", 5)) return; + of_node_get(adapter->dev.of_node); np = of_find_compatible_node(adapter->dev.of_node, NULL, "MAC,ds1775"); if (np) { of_node_put(np); } else { - strlcpy(info.type, "MAC,ds1775", I2C_NAME_SIZE); + strscpy(info.type, "MAC,ds1775", I2C_NAME_SIZE); i2c_new_scanned_device(adapter, &info, scan_ds1775, NULL); } + of_node_get(adapter->dev.of_node); np = of_find_compatible_node(adapter->dev.of_node, NULL, "MAC,adm1030"); if (np) { of_node_put(np); } else { - strlcpy(info.type, "MAC,adm1030", I2C_NAME_SIZE); + strscpy(info.type, "MAC,adm1030", I2C_NAME_SIZE); i2c_new_scanned_device(adapter, &info, scan_adm1030, NULL); } } -static int +static void do_remove(struct i2c_client *client) { if (x.running) { @@ -348,8 +350,6 @@ do_remove(struct i2c_client *client) x.fan = NULL; else printk(KERN_ERR "g4fan: bad client\n"); - - return 0; } static int diff --git a/drivers/macintosh/windfarm_ad7417_sensor.c b/drivers/macintosh/windfarm_ad7417_sensor.c index 6ad6441abcbcc8e8ffbb06928da1fefe7fda1704..c5c54a4ce91f244a7af828738640062caf63206a 100644 --- a/drivers/macintosh/windfarm_ad7417_sensor.c +++ b/drivers/macintosh/windfarm_ad7417_sensor.c @@ -289,7 +289,7 @@ static int wf_ad7417_probe(struct i2c_client *client, return 0; } -static int wf_ad7417_remove(struct i2c_client *client) +static void wf_ad7417_remove(struct i2c_client *client) { struct wf_ad7417_priv *pv = dev_get_drvdata(&client->dev); int i; @@ -302,8 +302,6 @@ static int wf_ad7417_remove(struct i2c_client *client) wf_unregister_sensor(&pv->sensors[i]); kref_put(&pv->ref, wf_ad7417_release); - - return 0; } static const struct i2c_device_id wf_ad7417_id[] = { diff --git a/drivers/macintosh/windfarm_fcu_controls.c b/drivers/macintosh/windfarm_fcu_controls.c index 82e7b2005ae78c9bbdaf1ece10df4bef33306339..c5b1ca5bcd7325eda3f77367dfd03294e00f0f75 100644 --- a/drivers/macintosh/windfarm_fcu_controls.c +++ b/drivers/macintosh/windfarm_fcu_controls.c @@ -560,7 +560,7 @@ static int wf_fcu_probe(struct i2c_client *client, return 0; } -static int wf_fcu_remove(struct i2c_client *client) +static void wf_fcu_remove(struct i2c_client *client) { struct wf_fcu_priv *pv = dev_get_drvdata(&client->dev); struct wf_fcu_fan *fan; @@ -571,7 +571,6 @@ static int wf_fcu_remove(struct i2c_client *client) wf_unregister_control(&fan->ctrl); } kref_put(&pv->ref, wf_fcu_release); - return 0; } static const struct i2c_device_id wf_fcu_id[] = { diff --git a/drivers/macintosh/windfarm_lm75_sensor.c b/drivers/macintosh/windfarm_lm75_sensor.c index eb7e7f0bd21905898bb62cc7f3d1b189ff63b02c..204661c8e918f8b2a62ae52f2cc3c9cb6ad7a851 100644 --- a/drivers/macintosh/windfarm_lm75_sensor.c +++ b/drivers/macintosh/windfarm_lm75_sensor.c @@ -147,7 +147,7 @@ static int wf_lm75_probe(struct i2c_client *client, return rc; } -static int wf_lm75_remove(struct i2c_client *client) +static void wf_lm75_remove(struct i2c_client *client) { struct wf_lm75_sensor *lm = i2c_get_clientdata(client); @@ -156,8 +156,6 @@ static int wf_lm75_remove(struct i2c_client *client) /* release sensor */ wf_unregister_sensor(&lm->sens); - - return 0; } static const struct i2c_device_id wf_lm75_id[] = { diff --git a/drivers/macintosh/windfarm_lm87_sensor.c b/drivers/macintosh/windfarm_lm87_sensor.c index 807efdde86bc5f03ebd1b46f073d51223acf08f3..40d25463346ed2bc0b495721bd3aac7ae4cdab7b 100644 --- a/drivers/macintosh/windfarm_lm87_sensor.c +++ b/drivers/macintosh/windfarm_lm87_sensor.c @@ -145,7 +145,7 @@ static int wf_lm87_probe(struct i2c_client *client, return rc; } -static int wf_lm87_remove(struct i2c_client *client) +static void wf_lm87_remove(struct i2c_client *client) { struct wf_lm87_sensor *lm = i2c_get_clientdata(client); @@ -154,8 +154,6 @@ static int wf_lm87_remove(struct i2c_client *client) /* release sensor */ wf_unregister_sensor(&lm->sens); - - return 0; } static const struct i2c_device_id wf_lm87_id[] = { diff --git a/drivers/macintosh/windfarm_max6690_sensor.c b/drivers/macintosh/windfarm_max6690_sensor.c index 55ee417fb878981843967e05ab15be987357b5f6..c0d404ebc7922e92fd0782a6b6b610f6ff3acc84 100644 --- a/drivers/macintosh/windfarm_max6690_sensor.c +++ b/drivers/macintosh/windfarm_max6690_sensor.c @@ -104,14 +104,12 @@ static int wf_max6690_probe(struct i2c_client *client, return rc; } -static int wf_max6690_remove(struct i2c_client *client) +static void wf_max6690_remove(struct i2c_client *client) { struct wf_6690_sensor *max = i2c_get_clientdata(client); max->i2c = NULL; wf_unregister_sensor(&max->sens); - - return 0; } static const struct i2c_device_id wf_max6690_id[] = { diff --git a/drivers/macintosh/windfarm_smu_sat.c b/drivers/macintosh/windfarm_smu_sat.c index 5ade627eaa788d231be2d6b2606c0ed3e4538584..be5d4593db93e00982279250a94cf936aede28cc 100644 --- a/drivers/macintosh/windfarm_smu_sat.c +++ b/drivers/macintosh/windfarm_smu_sat.c @@ -316,7 +316,7 @@ static int wf_sat_probe(struct i2c_client *client, return 0; } -static int wf_sat_remove(struct i2c_client *client) +static void wf_sat_remove(struct i2c_client *client) { struct wf_sat *sat = i2c_get_clientdata(client); struct wf_sat_sensor *sens; @@ -330,8 +330,6 @@ static int wf_sat_remove(struct i2c_client *client) } sat->i2c = NULL; kref_put(&sat->ref, wf_sat_release); - - return 0; } static const struct i2c_device_id wf_sat_id[] = { diff --git a/drivers/mailbox/apple-mailbox.c b/drivers/mailbox/apple-mailbox.c index 496c4951ccb13735535ee1eb8493c55c8848c569..2a3e8d8ff8b5e6cf0085495889272ed2f995ffe6 100644 --- a/drivers/mailbox/apple-mailbox.c +++ b/drivers/mailbox/apple-mailbox.c @@ -17,6 +17,7 @@ */ #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) @@ -100,6 +102,7 @@ struct apple_mbox { struct device *dev; struct mbox_controller controller; + spinlock_t rx_lock; }; static const struct of_device_id apple_mbox_of_match[]; @@ -112,6 +115,14 @@ static bool apple_mbox_hw_can_send(struct apple_mbox *apple_mbox) return !(mbox_ctrl & apple_mbox->hw->control_full); } +static bool apple_mbox_hw_send_empty(struct apple_mbox *apple_mbox) +{ + u32 mbox_ctrl = + readl_relaxed(apple_mbox->regs + apple_mbox->hw->a2i_control); + + return mbox_ctrl & apple_mbox->hw->control_empty; +} + static int apple_mbox_hw_send(struct apple_mbox *apple_mbox, struct apple_mbox_msg *msg) { @@ -195,13 +206,15 @@ static irqreturn_t apple_mbox_send_empty_irq(int irq, void *data) return IRQ_HANDLED; } -static irqreturn_t apple_mbox_recv_irq(int irq, void *data) +static int apple_mbox_poll(struct apple_mbox *apple_mbox) { - struct apple_mbox *apple_mbox = data; struct apple_mbox_msg msg; + int ret = 0; - while (apple_mbox_hw_recv(apple_mbox, &msg) == 0) + while (apple_mbox_hw_recv(apple_mbox, &msg) == 0) { mbox_chan_received_data(&apple_mbox->chan, (void *)&msg); + ret++; + } /* * The interrupt will keep firing even if there are no more messages @@ -216,9 +229,50 @@ static irqreturn_t apple_mbox_recv_irq(int irq, void *data) apple_mbox->regs + apple_mbox->hw->irq_ack); } + return ret; +} + +static irqreturn_t apple_mbox_recv_irq(int irq, void *data) +{ + struct apple_mbox *apple_mbox = data; + + spin_lock(&apple_mbox->rx_lock); + apple_mbox_poll(apple_mbox); + spin_unlock(&apple_mbox->rx_lock); + return IRQ_HANDLED; } +static bool apple_mbox_chan_peek_data(struct mbox_chan *chan) +{ + struct apple_mbox *apple_mbox = chan->con_priv; + unsigned long flags; + int ret; + + spin_lock_irqsave(&apple_mbox->rx_lock, flags); + ret = apple_mbox_poll(apple_mbox); + spin_unlock_irqrestore(&apple_mbox->rx_lock, flags); + + return ret > 0; +} + +static int apple_mbox_chan_flush(struct mbox_chan *chan, unsigned long timeout) +{ + struct apple_mbox *apple_mbox = chan->con_priv; + unsigned long deadline = jiffies + msecs_to_jiffies(timeout); + + while (time_before(jiffies, deadline)) { + if (apple_mbox_hw_send_empty(apple_mbox)) { + mbox_chan_txdone(&apple_mbox->chan, 0); + return 0; + } + + udelay(1); + } + + return -ETIME; +} + static int apple_mbox_chan_startup(struct mbox_chan *chan) { struct apple_mbox *apple_mbox = chan->con_priv; @@ -250,6 +304,8 @@ static void apple_mbox_chan_shutdown(struct mbox_chan *chan) static const struct mbox_chan_ops apple_mbox_ops = { .send_data = apple_mbox_chan_send_data, + .peek_data = apple_mbox_chan_peek_data, + .flush = apple_mbox_chan_flush, .startup = apple_mbox_chan_startup, .shutdown = apple_mbox_chan_shutdown, }; @@ -304,6 +360,7 @@ static int apple_mbox_probe(struct platform_device *pdev) mbox->controller.txdone_irq = true; mbox->controller.of_xlate = apple_mbox_of_xlate; mbox->chan.con_priv = mbox; + spin_lock_init(&mbox->rx_lock); irqname = devm_kasprintf(dev, GFP_KERNEL, "%s-recv", dev_name(dev)); if (!irqname) diff --git a/drivers/mailbox/bcm-flexrm-mailbox.c b/drivers/mailbox/bcm-flexrm-mailbox.c index fda16f76401e0732f7dd2b2aa4be18c6022f949e..bf6e86b0ed09cd1cc9c5a543d94ca5b965f500c8 100644 --- a/drivers/mailbox/bcm-flexrm-mailbox.c +++ b/drivers/mailbox/bcm-flexrm-mailbox.c @@ -622,15 +622,15 @@ static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg) rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src), DMA_TO_DEVICE); - if (rc < 0) - return rc; + if (!rc) + return -EIO; rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), DMA_FROM_DEVICE); - if (rc < 0) { + if (!rc) { dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), DMA_TO_DEVICE); - return rc; + return -EIO; } return 0; diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 02922073c9efd5ac9f487f5e8bf67fdf49f7f44c..20f2ec880ad690e304df234d5b792369b96bb272 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -904,7 +904,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx7ulp = { .xTR = 0x20, .xRR = 0x40, .xSR = {0x60, 0x60, 0x60, 0x60}, - .xCR = {0x64, 0x64, 0x64, 0x64}, + .xCR = {0x64, 0x64, 0x64, 0x64, 0x64}, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { @@ -927,7 +927,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp_s4 = { .xTR = 0x200, .xRR = 0x280, .xSR = {0xC, 0x118, 0x124, 0x12C}, - .xCR = {0x110, 0x114, 0x120, 0x128}, + .xCR = {0x8, 0x110, 0x114, 0x120, 0x128}, }; static const struct imx_mu_dcfg imx_mu_cfg_imx93_s4 = { @@ -938,7 +938,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx93_s4 = { .xTR = 0x200, .xRR = 0x280, .xSR = {0xC, 0x118, 0x124, 0x12C}, - .xCR = {0x110, 0x114, 0x120, 0x128}, + .xCR = {0x8, 0x110, 0x114, 0x120, 0x128}, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8_scu = { @@ -949,7 +949,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8_scu = { .xTR = 0x0, .xRR = 0x10, .xSR = {0x20, 0x20, 0x20, 0x20}, - .xCR = {0x24, 0x24, 0x24, 0x24}, + .xCR = {0x24, 0x24, 0x24, 0x24, 0x24}, }; static const struct imx_mu_dcfg imx_mu_cfg_imx8_seco = { @@ -960,7 +960,7 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8_seco = { .xTR = 0x0, .xRR = 0x10, .xSR = {0x20, 0x20, 0x20, 0x20}, - .xCR = {0x24, 0x24, 0x24, 0x24}, + .xCR = {0x24, 0x24, 0x24, 0x24, 0x24}, }; static const struct of_device_id imx_mu_dt_ids[] = { diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c index 4e34854d12389825f4f926fa617914c70ecb534b..cfacb3f320a6438cba4717c135317f492cc455a8 100644 --- a/drivers/mailbox/mailbox-mpfs.c +++ b/drivers/mailbox/mailbox-mpfs.c @@ -62,6 +62,7 @@ struct mpfs_mbox { struct mbox_controller controller; struct device *dev; int irq; + void __iomem *ctrl_base; void __iomem *mbox_base; void __iomem *int_reg; struct mbox_chan chans[1]; @@ -73,7 +74,7 @@ static bool mpfs_mbox_busy(struct mpfs_mbox *mbox) { u32 status; - status = readl_relaxed(mbox->mbox_base + SERVICES_SR_OFFSET); + status = readl_relaxed(mbox->ctrl_base + SERVICES_SR_OFFSET); return status & SCB_STATUS_BUSY_MASK; } @@ -99,29 +100,27 @@ static int mpfs_mbox_send_data(struct mbox_chan *chan, void *data) for (index = 0; index < (msg->cmd_data_size / 4); index++) writel_relaxed(word_buf[index], - mbox->mbox_base + MAILBOX_REG_OFFSET + index * 0x4); + mbox->mbox_base + msg->mbox_offset + index * 0x4); if (extra_bits) { u8 i; u8 byte_off = ALIGN_DOWN(msg->cmd_data_size, 4); u8 *byte_buf = msg->cmd_data + byte_off; - val = readl_relaxed(mbox->mbox_base + - MAILBOX_REG_OFFSET + index * 0x4); + val = readl_relaxed(mbox->mbox_base + msg->mbox_offset + index * 0x4); for (i = 0u; i < extra_bits; i++) { val &= ~(0xffu << (i * 8u)); val |= (byte_buf[i] << (i * 8u)); } - writel_relaxed(val, - mbox->mbox_base + MAILBOX_REG_OFFSET + index * 0x4); + writel_relaxed(val, mbox->mbox_base + msg->mbox_offset + index * 0x4); } } opt_sel = ((msg->mbox_offset << 7u) | (msg->cmd_opcode & 0x7fu)); tx_trigger = (opt_sel << SCB_CTRL_POS) & SCB_CTRL_MASK; tx_trigger |= SCB_CTRL_REQ_MASK | SCB_STATUS_NOTIFY_MASK; - writel_relaxed(tx_trigger, mbox->mbox_base + SERVICES_CR_OFFSET); + writel_relaxed(tx_trigger, mbox->ctrl_base + SERVICES_CR_OFFSET); return 0; } @@ -141,7 +140,7 @@ static void mpfs_mbox_rx_data(struct mbox_chan *chan) if (!mpfs_mbox_busy(mbox)) { for (i = 0; i < num_words; i++) { response->resp_msg[i] = - readl_relaxed(mbox->mbox_base + MAILBOX_REG_OFFSET + readl_relaxed(mbox->mbox_base + mbox->resp_offset + i * 0x4); } } @@ -200,14 +199,18 @@ static int mpfs_mbox_probe(struct platform_device *pdev) if (!mbox) return -ENOMEM; - mbox->mbox_base = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); - if (IS_ERR(mbox->mbox_base)) - return PTR_ERR(mbox->mbox_base); + mbox->ctrl_base = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); + if (IS_ERR(mbox->ctrl_base)) + return PTR_ERR(mbox->ctrl_base); mbox->int_reg = devm_platform_get_and_ioremap_resource(pdev, 1, ®s); if (IS_ERR(mbox->int_reg)) return PTR_ERR(mbox->int_reg); + mbox->mbox_base = devm_platform_get_and_ioremap_resource(pdev, 2, ®s); + if (IS_ERR(mbox->mbox_base)) // account for the old dt-binding w/ 2 regs + mbox->mbox_base = mbox->ctrl_base + MAILBOX_REG_OFFSET; + mbox->irq = platform_get_irq(pdev, 0); if (mbox->irq < 0) return mbox->irq; diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ebfa33a40fcebf09c2c2edf5ac588d4832992261..3c2bc0ca454cf1005193c45b85c28331c68b7f0a 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -676,7 +676,7 @@ static int pcc_mbox_probe(struct platform_device *pdev) if (pcct_entry->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE && !pcc_mbox_ctrl->txdone_irq) { - pr_err("Plaform Interrupt flag must be set to 1"); + pr_err("Platform Interrupt flag must be set to 1"); rc = -EINVAL; goto err; } diff --git a/drivers/mailbox/qcom-apcs-ipc-mailbox.c b/drivers/mailbox/qcom-apcs-ipc-mailbox.c index 80a54d81412e314e203523cb49ed169168ce48a6..f1f0e87a79e6667a763b4a430efcdd5f4e50d6b1 100644 --- a/drivers/mailbox/qcom-apcs-ipc-mailbox.c +++ b/drivers/mailbox/qcom-apcs-ipc-mailbox.c @@ -142,7 +142,7 @@ static int qcom_apcs_ipc_remove(struct platform_device *pdev) /* .data is the offset of the ipc register within the global block */ static const struct of_device_id qcom_apcs_ipc_of_match[] = { { .compatible = "qcom,ipq6018-apcs-apps-global", .data = &ipq6018_apcs_data }, - { .compatible = "qcom,ipq8074-apcs-apps-global", .data = &msm8994_apcs_data }, + { .compatible = "qcom,ipq8074-apcs-apps-global", .data = &ipq6018_apcs_data }, { .compatible = "qcom,msm8916-apcs-kpss-global", .data = &msm8916_apcs_data }, { .compatible = "qcom,msm8939-apcs-kpss-global", .data = &msm8916_apcs_data }, { .compatible = "qcom,msm8953-apcs-kpss-global", .data = &msm8994_apcs_data }, diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index 31d58b7d55fed27bfbb6488623c026b7cff688d3..7e27acf6c0ccaa8e6323262ee20cd4e5b00deb67 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -308,7 +308,8 @@ static int qcom_ipcc_probe(struct platform_device *pdev) goto err_mbox; ret = devm_request_irq(&pdev->dev, ipcc->irq, qcom_ipcc_irq_fn, - IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, name, ipcc); + IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND | + IRQF_NO_THREAD, name, ipcc); if (ret < 0) { dev_err(&pdev->dev, "Failed to register the irq: %d\n", ret); goto err_req_irq; diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 2acda9cea0f900c23e37b87a011d54e382cab0c2..aebb7ef10e631a6c6fd5f9f4d11987eca86f99d0 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -107,7 +107,7 @@ * * BTREE NODES: * - * Our unit of allocation is a bucket, and we we can't arbitrarily allocate and + * Our unit of allocation is a bucket, and we can't arbitrarily allocate and * free smaller than a bucket - so, that's how big our btree nodes are. * * (If buckets are really big we'll only use part of the bucket for a btree node diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c index 94d38e8a59b323aa576cc1b0f6ed6ef95ebef49d..2bba4d6aaaa28cdc5f6726b7715585f210accd36 100644 --- a/drivers/md/bcache/bset.c +++ b/drivers/md/bcache/bset.c @@ -1264,7 +1264,7 @@ static void __btree_sort(struct btree_keys *b, struct btree_iter *iter, * * Don't worry event 'out' is allocated from mempool, it can * still be swapped here. Because state->pool is a page mempool - * creaated by by mempool_init_page_pool(), which allocates + * created by mempool_init_page_pool(), which allocates * pages by alloc_pages() indeed. */ diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index f2c5a7e06fa9366667736e6952bf6438f72cc4f1..3427555b0ccae4d3f4a0992b8d741eced97e45c7 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -401,7 +401,7 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) } if (bypass_torture_test(dc)) { - if ((get_random_int() & 3) == 3) + if (prandom_u32_max(4) == 3) goto skip; else goto rescale; diff --git a/drivers/md/bcache/stats.h b/drivers/md/bcache/stats.h index ca4f435f7216a7231cf9e1dd4ee8cae56ea93841..bd3afc856d53452bd96113b623ba6e1706aca153 100644 --- a/drivers/md/bcache/stats.h +++ b/drivers/md/bcache/stats.h @@ -54,7 +54,6 @@ void bch_cache_accounting_destroy(struct cache_accounting *acc); void bch_mark_cache_accounting(struct cache_set *c, struct bcache_device *d, bool hit, bool bypass); -void bch_mark_cache_readahead(struct cache_set *c, struct bcache_device *d); void bch_mark_cache_miss_collision(struct cache_set *c, struct bcache_device *d); void bch_mark_sectors_bypassed(struct cache_set *c, diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 3f0ff3aab6f231581696a10b4729e501a2f3b121..0285b676e9834a8b273dd437df26b9cc199e73eb 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -157,6 +157,53 @@ static void __update_writeback_rate(struct cached_dev *dc) dc->writeback_rate_target = target; } +static bool idle_counter_exceeded(struct cache_set *c) +{ + int counter, dev_nr; + + /* + * If c->idle_counter is overflow (idel for really long time), + * reset as 0 and not set maximum rate this time for code + * simplicity. + */ + counter = atomic_inc_return(&c->idle_counter); + if (counter <= 0) { + atomic_set(&c->idle_counter, 0); + return false; + } + + dev_nr = atomic_read(&c->attached_dev_nr); + if (dev_nr == 0) + return false; + + /* + * c->idle_counter is increased by writeback thread of all + * attached backing devices, in order to represent a rough + * time period, counter should be divided by dev_nr. + * Otherwise the idle time cannot be larger with more backing + * device attached. + * The following calculation equals to checking + * (counter / dev_nr) < (dev_nr * 6) + */ + if (counter < (dev_nr * dev_nr * 6)) + return false; + + return true; +} + +/* + * Idle_counter is increased every time when update_writeback_rate() is + * called. If all backing devices attached to the same cache set have + * identical dc->writeback_rate_update_seconds values, it is about 6 + * rounds of update_writeback_rate() on each backing device before + * c->at_max_writeback_rate is set to 1, and then max wrteback rate set + * to each dc->writeback_rate.rate. + * In order to avoid extra locking cost for counting exact dirty cached + * devices number, c->attached_dev_nr is used to calculate the idle + * throushold. It might be bigger if not all cached device are in write- + * back mode, but it still works well with limited extra rounds of + * update_writeback_rate(). + */ static bool set_at_max_writeback_rate(struct cache_set *c, struct cached_dev *dc) { @@ -167,21 +214,8 @@ static bool set_at_max_writeback_rate(struct cache_set *c, /* Don't set max writeback rate if gc is running */ if (!c->gc_mark_valid) return false; - /* - * Idle_counter is increased everytime when update_writeback_rate() is - * called. If all backing devices attached to the same cache set have - * identical dc->writeback_rate_update_seconds values, it is about 6 - * rounds of update_writeback_rate() on each backing device before - * c->at_max_writeback_rate is set to 1, and then max wrteback rate set - * to each dc->writeback_rate.rate. - * In order to avoid extra locking cost for counting exact dirty cached - * devices number, c->attached_dev_nr is used to calculate the idle - * throushold. It might be bigger if not all cached device are in write- - * back mode, but it still works well with limited extra rounds of - * update_writeback_rate(). - */ - if (atomic_inc_return(&c->idle_counter) < - atomic_read(&c->attached_dev_nr) * 6) + + if (!idle_counter_exceeded(c)) return false; if (atomic_read(&c->at_max_writeback_rate) != 1) @@ -195,13 +229,10 @@ static bool set_at_max_writeback_rate(struct cache_set *c, dc->writeback_rate_change = 0; /* - * Check c->idle_counter and c->at_max_writeback_rate agagain in case - * new I/O arrives during before set_at_max_writeback_rate() returns. - * Then the writeback rate is set to 1, and its new value should be - * decided via __update_writeback_rate(). + * In case new I/O arrives during before + * set_at_max_writeback_rate() returns. */ - if ((atomic_read(&c->idle_counter) < - atomic_read(&c->attached_dev_nr) * 6) || + if (!idle_counter_exceeded(c) || !atomic_read(&c->at_max_writeback_rate)) return false; @@ -801,10 +832,9 @@ static int bch_writeback_thread(void *arg) } } - if (dc->writeback_write_wq) { - flush_workqueue(dc->writeback_write_wq); + if (dc->writeback_write_wq) destroy_workqueue(dc->writeback_write_wq); - } + cached_dev_put(dc); wait_for_kthread_stop(); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 4f49bbcce4f1a34f2db215ac05c83c12a816e555..3001b10a3fbfba7a11eab783ab63b825f3f9cbd9 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -292,11 +292,13 @@ static void dm_kill_unmapped_request(struct request *rq, blk_status_t error) dm_complete_request(rq, error); } -static void end_clone_request(struct request *clone, blk_status_t error) +static enum rq_end_io_ret end_clone_request(struct request *clone, + blk_status_t error) { struct dm_rq_target_io *tio = clone->end_io_data; dm_complete_request(tio->orig, error); + return RQ_END_IO_NONE; } static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 332f96b5825292d0c6158d84cf096f489d560a14..d8034ff0cb241e5d0f4e801d2b55ca945fff6dd4 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -1856,9 +1856,7 @@ static bool dm_table_supports_write_zeroes(struct dm_table *t) static int device_not_nowait_capable(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { - struct request_queue *q = bdev_get_queue(dev->bdev); - - return !blk_queue_nowait(q); + return !bdev_nowait(dev->bdev); } static bool dm_table_supports_nowait(struct dm_table *t) diff --git a/drivers/md/dm-verity-loadpin.c b/drivers/md/dm-verity-loadpin.c index 387ec43aef72814f1c700519581e97042e9b8a82..4f78cc55c2514e6329d9780939d320e53faa743d 100644 --- a/drivers/md/dm-verity-loadpin.c +++ b/drivers/md/dm-verity-loadpin.c @@ -14,6 +14,7 @@ LIST_HEAD(dm_verity_loadpin_trusted_root_digests); static bool is_trusted_verity_target(struct dm_target *ti) { + int verity_mode; u8 *root_digest; unsigned int digest_size; struct dm_verity_loadpin_trusted_root_digest *trd; @@ -22,6 +23,13 @@ static bool is_trusted_verity_target(struct dm_target *ti) if (!dm_is_verity_target(ti)) return false; + verity_mode = dm_verity_get_mode(ti); + + if ((verity_mode != DM_VERITY_MODE_EIO) && + (verity_mode != DM_VERITY_MODE_RESTART) && + (verity_mode != DM_VERITY_MODE_PANIC)) + return false; + if (dm_verity_get_root_digest(ti, &root_digest, &digest_size)) return false; diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 94b6cb599db4f1a1a12eb3f79739ec1a178c1b8d..8a00cc42e4985997659bc6a7de3c362090770391 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -1446,6 +1446,22 @@ bool dm_is_verity_target(struct dm_target *ti) return ti->type->module == THIS_MODULE; } +/* + * Get the verity mode (error behavior) of a verity target. + * + * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity + * target. + */ +int dm_verity_get_mode(struct dm_target *ti) +{ + struct dm_verity *v = ti->private; + + if (!dm_is_verity_target(ti)) + return -EINVAL; + + return v->mode; +} + /* * Get the root digest of a verity target. * diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h index 45455de1b4bc594bfd1af67faf137a391b9502cc..98f306ec6a33ddbb28d80a39290a328f0393f9b3 100644 --- a/drivers/md/dm-verity.h +++ b/drivers/md/dm-verity.h @@ -134,6 +134,7 @@ extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, sector_t block, u8 *digest, bool *is_zero); extern bool dm_is_verity_target(struct dm_target *ti); +extern int dm_verity_get_mode(struct dm_target *ti); extern int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size); diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 742b2349fea34abd448cd0425b0a028177b10daf..10e0c5381d01b719a621d3ee42e70dd739309586 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -876,8 +876,8 @@ static int join(struct mddev *mddev, int nodes) memset(str, 0, 64); sprintf(str, "%pU", mddev->uuid); ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name, - DLM_LSFL_FS, LVB_SIZE, - &md_ls_ops, mddev, &ops_rv, &cinfo->lockspace); + 0, LVB_SIZE, &md_ls_ops, mddev, + &ops_rv, &cinfo->lockspace); if (ret) goto err; wait_for_completion(&cinfo->completion); diff --git a/drivers/md/md.c b/drivers/md/md.c index afaf36b2f6ab8b32d53edc24e12ba17fa7a16884..a467b492d4ad34a7efcf1454c18c6cd48dff6d51 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5620,6 +5620,7 @@ struct mddev *md_alloc(dev_t dev, char *name) * removed (mddev_delayed_delete). */ flush_workqueue(md_misc_wq); + flush_workqueue(md_rdev_misc_wq); mutex_lock(&disks_mutex); mddev = mddev_alloc(dev); @@ -5844,7 +5845,7 @@ int md_run(struct mddev *mddev) } } sysfs_notify_dirent_safe(rdev->sysfs_state); - nowait = nowait && blk_queue_nowait(bdev_get_queue(rdev->bdev)); + nowait = nowait && bdev_nowait(rdev->bdev); } if (!bioset_initialized(&mddev->bio_set)) { @@ -6238,11 +6239,11 @@ static void mddev_detach(struct mddev *mddev) static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; + md_bitmap_destroy(mddev); mddev_detach(mddev); /* Ensure ->event_work is done */ if (mddev->event_work.func) flush_workqueue(md_misc_wq); - md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); @@ -6260,6 +6261,7 @@ void md_stop(struct mddev *mddev) /* stop the array and free an attached data structures. * This is called from dm-raid */ + __md_stop_writes(mddev); __md_stop(mddev); bioset_exit(&mddev->bio_set); bioset_exit(&mddev->sync_set); @@ -6980,7 +6982,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) * If the new disk does not support REQ_NOWAIT, * disable on the whole MD. */ - if (!blk_queue_nowait(bdev_get_queue(rdev->bdev))) { + if (!bdev_nowait(rdev->bdev)) { pr_info("%s: Disabling nowait because %pg does not support nowait\n", mdname(mddev), rdev->bdev); blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, mddev->queue); @@ -8154,7 +8156,6 @@ static void *md_seq_start(struct seq_file *seq, loff_t *pos) list_for_each(tmp,&all_mddevs) if (!l--) { mddev = list_entry(tmp, struct mddev, all_mddevs); - mddev_get(mddev); if (!mddev_get(mddev)) continue; spin_unlock(&all_mddevs_lock); diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 78addfe4a0c925c51c58b4f6d20cfa4080e1924a..857c49399c28ed48db0a9edee3dc732f53e69f11 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -47,7 +47,7 @@ static void dump_zones(struct mddev *mddev) int len = 0; for (k = 0; k < conf->strip_zone[j].nb_dev; k++) - len += snprintf(line+len, 200-len, "%s%pg", k?"/":"", + len += scnprintf(line+len, 200-len, "%s%pg", k?"/":"", conf->devlist[j * raid_disks + k]->bdev); pr_debug("md: zone%d=[%s]\n", j, line); diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 9117fcdee1be126b14235eaeac9984179f4b1578..3aa8b6e11d585a8ea30ef537358ca289d8c3e37c 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -79,6 +79,21 @@ static void end_reshape(struct r10conf *conf); #include "raid1-10.c" +#define NULL_CMD +#define cmd_before(conf, cmd) \ + do { \ + write_sequnlock_irq(&(conf)->resync_lock); \ + cmd; \ + } while (0) +#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock) + +#define wait_event_barrier_cmd(conf, cond, cmd) \ + wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \ + cmd_after(conf)) + +#define wait_event_barrier(conf, cond) \ + wait_event_barrier_cmd(conf, cond, NULL_CMD) + /* * for resync bio, r10bio pointer can be retrieved from the per-bio * 'struct resync_pages'. @@ -274,6 +289,12 @@ static void put_buf(struct r10bio *r10_bio) lower_barrier(conf); } +static void wake_up_barrier(struct r10conf *conf) +{ + if (wq_has_sleeper(&conf->wait_barrier)) + wake_up(&conf->wait_barrier); +} + static void reschedule_retry(struct r10bio *r10_bio) { unsigned long flags; @@ -930,78 +951,101 @@ static void flush_pending_writes(struct r10conf *conf) static void raise_barrier(struct r10conf *conf, int force) { + write_seqlock_irq(&conf->resync_lock); BUG_ON(force && !conf->barrier); - spin_lock_irq(&conf->resync_lock); /* Wait until no block IO is waiting (unless 'force') */ - wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, - conf->resync_lock); + wait_event_barrier(conf, force || !conf->nr_waiting); /* block any new IO from starting */ - conf->barrier++; + WRITE_ONCE(conf->barrier, conf->barrier + 1); /* Now wait for all pending IO to complete */ - wait_event_lock_irq(conf->wait_barrier, - !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH, - conf->resync_lock); + wait_event_barrier(conf, !atomic_read(&conf->nr_pending) && + conf->barrier < RESYNC_DEPTH); - spin_unlock_irq(&conf->resync_lock); + write_sequnlock_irq(&conf->resync_lock); } static void lower_barrier(struct r10conf *conf) { unsigned long flags; - spin_lock_irqsave(&conf->resync_lock, flags); - conf->barrier--; - spin_unlock_irqrestore(&conf->resync_lock, flags); + + write_seqlock_irqsave(&conf->resync_lock, flags); + WRITE_ONCE(conf->barrier, conf->barrier - 1); + write_sequnlock_irqrestore(&conf->resync_lock, flags); wake_up(&conf->wait_barrier); } +static bool stop_waiting_barrier(struct r10conf *conf) +{ + struct bio_list *bio_list = current->bio_list; + + /* barrier is dropped */ + if (!conf->barrier) + return true; + + /* + * If there are already pending requests (preventing the barrier from + * rising completely), and the pre-process bio queue isn't empty, then + * don't wait, as we need to empty that queue to get the nr_pending + * count down. + */ + if (atomic_read(&conf->nr_pending) && bio_list && + (!bio_list_empty(&bio_list[0]) || !bio_list_empty(&bio_list[1]))) + return true; + + /* move on if recovery thread is blocked by us */ + if (conf->mddev->thread->tsk == current && + test_bit(MD_RECOVERY_RUNNING, &conf->mddev->recovery) && + conf->nr_queued > 0) + return true; + + return false; +} + +static bool wait_barrier_nolock(struct r10conf *conf) +{ + unsigned int seq = read_seqbegin(&conf->resync_lock); + + if (READ_ONCE(conf->barrier)) + return false; + + atomic_inc(&conf->nr_pending); + if (!read_seqretry(&conf->resync_lock, seq)) + return true; + + if (atomic_dec_and_test(&conf->nr_pending)) + wake_up_barrier(conf); + + return false; +} + static bool wait_barrier(struct r10conf *conf, bool nowait) { bool ret = true; - spin_lock_irq(&conf->resync_lock); + if (wait_barrier_nolock(conf)) + return true; + + write_seqlock_irq(&conf->resync_lock); if (conf->barrier) { - struct bio_list *bio_list = current->bio_list; - conf->nr_waiting++; - /* Wait for the barrier to drop. - * However if there are already pending - * requests (preventing the barrier from - * rising completely), and the - * pre-process bio queue isn't empty, - * then don't wait, as we need to empty - * that queue to get the nr_pending - * count down. - */ /* Return false when nowait flag is set */ if (nowait) { ret = false; } else { + conf->nr_waiting++; raid10_log(conf->mddev, "wait barrier"); - wait_event_lock_irq(conf->wait_barrier, - !conf->barrier || - (atomic_read(&conf->nr_pending) && - bio_list && - (!bio_list_empty(&bio_list[0]) || - !bio_list_empty(&bio_list[1]))) || - /* move on if recovery thread is - * blocked by us - */ - (conf->mddev->thread->tsk == current && - test_bit(MD_RECOVERY_RUNNING, - &conf->mddev->recovery) && - conf->nr_queued > 0), - conf->resync_lock); + wait_event_barrier(conf, stop_waiting_barrier(conf)); + conf->nr_waiting--; } - conf->nr_waiting--; if (!conf->nr_waiting) wake_up(&conf->wait_barrier); } /* Only increment nr_pending when we wait */ if (ret) atomic_inc(&conf->nr_pending); - spin_unlock_irq(&conf->resync_lock); + write_sequnlock_irq(&conf->resync_lock); return ret; } @@ -1009,7 +1053,7 @@ static void allow_barrier(struct r10conf *conf) { if ((atomic_dec_and_test(&conf->nr_pending)) || (conf->array_freeze_pending)) - wake_up(&conf->wait_barrier); + wake_up_barrier(conf); } static void freeze_array(struct r10conf *conf, int extra) @@ -1026,27 +1070,24 @@ static void freeze_array(struct r10conf *conf, int extra) * must match the number of pending IOs (nr_pending) before * we continue. */ - spin_lock_irq(&conf->resync_lock); + write_seqlock_irq(&conf->resync_lock); conf->array_freeze_pending++; - conf->barrier++; + WRITE_ONCE(conf->barrier, conf->barrier + 1); conf->nr_waiting++; - wait_event_lock_irq_cmd(conf->wait_barrier, - atomic_read(&conf->nr_pending) == conf->nr_queued+extra, - conf->resync_lock, - flush_pending_writes(conf)); - + wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) == + conf->nr_queued + extra, flush_pending_writes(conf)); conf->array_freeze_pending--; - spin_unlock_irq(&conf->resync_lock); + write_sequnlock_irq(&conf->resync_lock); } static void unfreeze_array(struct r10conf *conf) { /* reverse the effect of the freeze */ - spin_lock_irq(&conf->resync_lock); - conf->barrier--; + write_seqlock_irq(&conf->resync_lock); + WRITE_ONCE(conf->barrier, conf->barrier - 1); conf->nr_waiting--; wake_up(&conf->wait_barrier); - spin_unlock_irq(&conf->resync_lock); + write_sequnlock_irq(&conf->resync_lock); } static sector_t choose_data_offset(struct r10bio *r10_bio, @@ -1885,7 +1926,7 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) __make_request(mddev, bio, sectors); /* In case raid10d snuck in to freeze_array */ - wake_up(&conf->wait_barrier); + wake_up_barrier(conf); return true; } @@ -1980,7 +2021,7 @@ static int enough(struct r10conf *conf, int ignore) * Otherwise, it must be degraded: * - recovery is interrupted. * - &mddev->degraded is bumped. - + * * @rdev is marked as &Faulty excluding case when array is failed and * &mddev->fail_last_dev is off. */ @@ -2639,18 +2680,18 @@ static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) } static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, - int sectors, struct page *page, int rw) + int sectors, struct page *page, enum req_op op) { sector_t first_bad; int bad_sectors; if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) - && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) + && (op == REQ_OP_READ || test_bit(WriteErrorSeen, &rdev->flags))) return -1; - if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) + if (sync_page_io(rdev, sector, sectors << 9, page, op, false)) /* success */ return 1; - if (rw == WRITE) { + if (op == REQ_OP_WRITE) { set_bit(WriteErrorSeen, &rdev->flags); if (!test_and_set_bit(WantReplacement, &rdev->flags)) set_bit(MD_RECOVERY_NEEDED, @@ -2780,7 +2821,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 if (r10_sync_page_io(rdev, r10_bio->devs[sl].addr + sect, - s, conf->tmppage, WRITE) + s, conf->tmppage, REQ_OP_WRITE) == 0) { /* Well, this device is dead */ pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %pg)\n", @@ -2814,8 +2855,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10 switch (r10_sync_page_io(rdev, r10_bio->devs[sl].addr + sect, - s, conf->tmppage, - READ)) { + s, conf->tmppage, REQ_OP_READ)) { case 0: /* Well, this device is dead */ pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %pg)\n", @@ -4033,7 +4073,7 @@ static struct r10conf *setup_conf(struct mddev *mddev) INIT_LIST_HEAD(&conf->retry_list); INIT_LIST_HEAD(&conf->bio_end_io_list); - spin_lock_init(&conf->resync_lock); + seqlock_init(&conf->resync_lock); init_waitqueue_head(&conf->wait_barrier); atomic_set(&conf->nr_pending, 0); @@ -4352,7 +4392,7 @@ static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) rdev->new_raid_disk = rdev->raid_disk * 2; rdev->sectors = size; } - conf->barrier = 1; + WRITE_ONCE(conf->barrier, 1); } return conf; diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h index 5c0804d8bb1fe7e37b1e467ba05582ccfb625f17..8c072ce0bc542038837ed5640afe475db18b51ba 100644 --- a/drivers/md/raid10.h +++ b/drivers/md/raid10.h @@ -76,7 +76,7 @@ struct r10conf { /* queue pending writes and submit them on unplug */ struct bio_list pending_bio_list; - spinlock_t resync_lock; + seqlock_t resync_lock; atomic_t nr_pending; int nr_waiting; int nr_queued; diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index f4e1cc1ece43de71cac64281346f0f9386859dd4..832d8566e165650d43bd271bb436bc19ecbb8171 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -125,7 +125,7 @@ struct r5l_log { * reclaimed. if it's 0, reclaim spaces * used by io_units which are in * IO_UNIT_STRIPE_END state (eg, reclaim - * dones't wait for specific io_unit + * doesn't wait for specific io_unit * switching to IO_UNIT_STRIPE_END * state) */ wait_queue_head_t iounit_wait; @@ -1327,9 +1327,9 @@ static void r5l_write_super_and_discard_space(struct r5l_log *log, * superblock is updated to new log tail. Updating superblock (either * directly call md_update_sb() or depend on md thread) must hold * reconfig mutex. On the other hand, raid5_quiesce is called with - * reconfig_mutex hold. The first step of raid5_quiesce() is waitting - * for all IO finish, hence waitting for reclaim thread, while reclaim - * thread is calling this function and waitting for reconfig mutex. So + * reconfig_mutex hold. The first step of raid5_quiesce() is waiting + * for all IO finish, hence waiting for reclaim thread, while reclaim + * thread is calling this function and waiting for reconfig mutex. So * there is a deadlock. We workaround this issue with a trylock. * FIXME: we could miss discard if we can't take reconfig mutex */ @@ -1923,7 +1923,8 @@ r5c_recovery_alloc_stripe( { struct stripe_head *sh; - sh = raid5_get_active_stripe(conf, stripe_sect, 0, noblock, 0); + sh = raid5_get_active_stripe(conf, NULL, stripe_sect, + noblock ? R5_GAS_NOBLOCK : 0); if (!sh) return NULL; /* no more stripe available */ @@ -2993,7 +2994,7 @@ static int r5l_load_log(struct r5l_log *log) } create: if (create_super) { - log->last_cp_seq = prandom_u32(); + log->last_cp_seq = get_random_u32(); cp = 0; r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq); /* diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 31a0cbf63384d8a2527cdc88ad2ff99314321592..7b820b81d8c2b0d359292d46697a0a86f07da33a 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -36,6 +36,7 @@ */ #include +#include #include #include #include @@ -789,87 +790,80 @@ struct stripe_request_ctx { */ static bool is_inactive_blocked(struct r5conf *conf, int hash) { - int active = atomic_read(&conf->active_stripes); - if (list_empty(conf->inactive_list + hash)) return false; if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) return true; - return active < (conf->max_nr_stripes * 3 / 4); + return (atomic_read(&conf->active_stripes) < + (conf->max_nr_stripes * 3 / 4)); } -static struct stripe_head *__raid5_get_active_stripe(struct r5conf *conf, +struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, struct stripe_request_ctx *ctx, sector_t sector, - bool previous, bool noblock, bool noquiesce) + unsigned int flags) { struct stripe_head *sh; int hash = stripe_hash_locks_hash(conf, sector); + int previous = !!(flags & R5_GAS_PREVIOUS); pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); spin_lock_irq(conf->hash_locks + hash); -retry: - if (!noquiesce && conf->quiesce) { - /* - * Must release the reference to batch_last before waiting, - * on quiesce, otherwise the batch_last will hold a reference - * to a stripe and raid5_quiesce() will deadlock waiting for - * active_stripes to go to zero. - */ - if (ctx && ctx->batch_last) { - raid5_release_stripe(ctx->batch_last); - ctx->batch_last = NULL; - } - - wait_event_lock_irq(conf->wait_for_quiescent, !conf->quiesce, - *(conf->hash_locks + hash)); - } + for (;;) { + if (!(flags & R5_GAS_NOQUIESCE) && conf->quiesce) { + /* + * Must release the reference to batch_last before + * waiting, on quiesce, otherwise the batch_last will + * hold a reference to a stripe and raid5_quiesce() + * will deadlock waiting for active_stripes to go to + * zero. + */ + if (ctx && ctx->batch_last) { + raid5_release_stripe(ctx->batch_last); + ctx->batch_last = NULL; + } - sh = find_get_stripe(conf, sector, conf->generation - previous, hash); - if (sh) - goto out; + wait_event_lock_irq(conf->wait_for_quiescent, + !conf->quiesce, + *(conf->hash_locks + hash)); + } - if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) - goto wait_for_stripe; + sh = find_get_stripe(conf, sector, conf->generation - previous, + hash); + if (sh) + break; - sh = get_free_stripe(conf, hash); - if (sh) { - r5c_check_stripe_cache_usage(conf); - init_stripe(sh, sector, previous); - atomic_inc(&sh->count); - goto out; - } + if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { + sh = get_free_stripe(conf, hash); + if (sh) { + r5c_check_stripe_cache_usage(conf); + init_stripe(sh, sector, previous); + atomic_inc(&sh->count); + break; + } - if (!test_bit(R5_DID_ALLOC, &conf->cache_state)) - set_bit(R5_ALLOC_MORE, &conf->cache_state); + if (!test_bit(R5_DID_ALLOC, &conf->cache_state)) + set_bit(R5_ALLOC_MORE, &conf->cache_state); + } -wait_for_stripe: - if (noblock) - goto out; + if (flags & R5_GAS_NOBLOCK) + break; - set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state); - r5l_wake_reclaim(conf->log, 0); - wait_event_lock_irq(conf->wait_for_stripe, - is_inactive_blocked(conf, hash), - *(conf->hash_locks + hash)); - clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state); - goto retry; + set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state); + r5l_wake_reclaim(conf->log, 0); + wait_event_lock_irq(conf->wait_for_stripe, + is_inactive_blocked(conf, hash), + *(conf->hash_locks + hash)); + clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state); + } -out: spin_unlock_irq(conf->hash_locks + hash); return sh; } -struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, - sector_t sector, bool previous, bool noblock, bool noquiesce) -{ - return __raid5_get_active_stripe(conf, NULL, sector, previous, noblock, - noquiesce); -} - static bool is_full_stripe_write(struct stripe_head *sh) { BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); @@ -4047,7 +4041,7 @@ static void handle_stripe_fill(struct stripe_head *sh, * back cache (prexor with orig_page, and then xor with * page) in the read path */ - if (s->injournal && s->failed) { + if (s->to_read && s->injournal && s->failed) { if (test_bit(STRIPE_R5C_CACHING, &sh->state)) r5c_make_stripe_write_out(sh); goto out; @@ -4636,7 +4630,8 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) sector_t bn = raid5_compute_blocknr(sh, i, 1); sector_t s = raid5_compute_sector(conf, bn, 0, &dd_idx, NULL); - sh2 = raid5_get_active_stripe(conf, s, 0, 1, 1); + sh2 = raid5_get_active_stripe(conf, NULL, s, + R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE); if (sh2 == NULL) /* so far only the early blocks of this stripe * have been requested. When later blocks @@ -5273,7 +5268,9 @@ static void handle_stripe(struct stripe_head *sh) /* Finish reconstruct operations initiated by the expansion process */ if (sh->reconstruct_state == reconstruct_state_result) { struct stripe_head *sh_src - = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1); + = raid5_get_active_stripe(conf, NULL, sh->sector, + R5_GAS_PREVIOUS | R5_GAS_NOBLOCK | + R5_GAS_NOQUIESCE); if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { /* sh cannot be written until sh_src has been read. * so arrange for sh to be delayed a little @@ -5542,7 +5539,6 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) if (is_badblock(rdev, sector, bio_sectors(raid_bio), &first_bad, &bad_sectors)) { - bio_put(raid_bio); rdev_dec_pending(rdev, mddev); return 0; } @@ -5823,7 +5819,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) DEFINE_WAIT(w); int d; again: - sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0); + sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0); prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); @@ -5978,7 +5974,7 @@ static enum stripe_result make_stripe_request(struct mddev *mddev, enum stripe_result ret; struct stripe_head *sh; sector_t new_sector; - int previous = 0; + int previous = 0, flags = 0; int seq, dd_idx; seq = read_seqcount_begin(&conf->gen_lock); @@ -6012,8 +6008,11 @@ static enum stripe_result make_stripe_request(struct mddev *mddev, pr_debug("raid456: %s, sector %llu logical %llu\n", __func__, new_sector, logical_sector); - sh = __raid5_get_active_stripe(conf, ctx, new_sector, previous, - (bi->bi_opf & REQ_RAHEAD), 0); + if (previous) + flags |= R5_GAS_PREVIOUS; + if (bi->bi_opf & REQ_RAHEAD) + flags |= R5_GAS_NOBLOCK; + sh = raid5_get_active_stripe(conf, ctx, new_sector, flags); if (unlikely(!sh)) { /* cannot get stripe, just give-up */ bi->bi_status = BLK_STS_IOERR; @@ -6362,7 +6361,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) { int j; int skipped_disk = 0; - sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1); + sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i, + R5_GAS_NOQUIESCE); set_bit(STRIPE_EXPANDING, &sh->state); atomic_inc(&conf->reshape_stripes); /* If any of this stripe is beyond the end of the old @@ -6411,7 +6411,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (last_sector >= mddev->dev_sectors) last_sector = mddev->dev_sectors - 1; while (first_sector <= last_sector) { - sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1); + sh = raid5_get_active_stripe(conf, NULL, first_sector, + R5_GAS_PREVIOUS | R5_GAS_NOQUIESCE); set_bit(STRIPE_EXPAND_SOURCE, &sh->state); set_bit(STRIPE_HANDLE, &sh->state); raid5_release_stripe(sh); @@ -6531,9 +6532,10 @@ static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_n md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false); - sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0); + sh = raid5_get_active_stripe(conf, NULL, sector_nr, + R5_GAS_NOBLOCK); if (sh == NULL) { - sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0); + sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0); /* make sure we don't swamp the stripe cache if someone else * is trying to get access */ @@ -6596,8 +6598,8 @@ static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio, /* already done this stripe */ continue; - sh = raid5_get_active_stripe(conf, sector, 0, 1, 1); - + sh = raid5_get_active_stripe(conf, NULL, sector, + R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE); if (!sh) { /* failed to get a stripe - must wait */ conf->retry_read_aligned = raid_bio; @@ -6781,7 +6783,18 @@ static void raid5d(struct md_thread *thread) spin_unlock_irq(&conf->device_lock); md_check_recovery(mddev); spin_lock_irq(&conf->device_lock); + + /* + * Waiting on MD_SB_CHANGE_PENDING below may deadlock + * seeing md_check_recovery() is needed to clear + * the flag when using mdmon. + */ + continue; } + + wait_event_lock_irq(mddev->sb_wait, + !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags), + conf->device_lock); } pr_debug("%d stripes handled\n", handled); diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index a5082bed83c863f0e2bbd939d8bc9d3aad9b1ccc..e873938a6125d3a834df5e95bc12bd3940ee5e03 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h @@ -803,16 +803,24 @@ raid5_get_dev_page(struct stripe_head *sh, int disk_idx) } #endif -extern void md_raid5_kick_device(struct r5conf *conf); -extern int raid5_set_cache_size(struct mddev *mddev, int size); -extern sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous); -extern void raid5_release_stripe(struct stripe_head *sh); -extern sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, - int previous, int *dd_idx, - struct stripe_head *sh); -extern struct stripe_head * -raid5_get_active_stripe(struct r5conf *conf, sector_t sector, - bool previous, bool noblock, bool noquiesce); -extern int raid5_calc_degraded(struct r5conf *conf); -extern int r5c_journal_mode_set(struct mddev *mddev, int journal_mode); +void md_raid5_kick_device(struct r5conf *conf); +int raid5_set_cache_size(struct mddev *mddev, int size); +sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous); +void raid5_release_stripe(struct stripe_head *sh); +sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, + int previous, int *dd_idx, struct stripe_head *sh); + +struct stripe_request_ctx; +/* get stripe from previous generation (when reshaping) */ +#define R5_GAS_PREVIOUS (1 << 0) +/* do not block waiting for a free stripe */ +#define R5_GAS_NOBLOCK (1 << 1) +/* do not block waiting for quiesce to be released */ +#define R5_GAS_NOQUIESCE (1 << 2) +struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, + struct stripe_request_ctx *ctx, sector_t sector, + unsigned int flags); + +int raid5_calc_degraded(struct r5conf *conf); +int r5c_journal_mode_set(struct mddev *mddev, int journal_mode); #endif diff --git a/drivers/media/cec/i2c/ch7322.c b/drivers/media/cec/i2c/ch7322.c index 0814338c43e4bc63bc2c2d17b4cf998f7ff3c32d..34fad7123704fb06ecc321a8a113e79311b32666 100644 --- a/drivers/media/cec/i2c/ch7322.c +++ b/drivers/media/cec/i2c/ch7322.c @@ -565,7 +565,7 @@ err_mutex: return ret; } -static int ch7322_remove(struct i2c_client *client) +static void ch7322_remove(struct i2c_client *client) { struct ch7322 *ch7322 = i2c_get_clientdata(client); @@ -578,8 +578,6 @@ static int ch7322_remove(struct i2c_client *client) mutex_destroy(&ch7322->mutex); dev_info(&client->dev, "device unregistered\n"); - - return 0; } static const struct of_device_id ch7322_of_match[] = { diff --git a/drivers/media/cec/platform/sti/stih-cec.c b/drivers/media/cec/platform/sti/stih-cec.c index abf8e8bcbb34cff6c799478137095c24866e9920..4edbdd09535dd45c0b4f5bb3d1d04f6e638606d0 100644 --- a/drivers/media/cec/platform/sti/stih-cec.c +++ b/drivers/media/cec/platform/sti/stih-cec.c @@ -256,8 +256,8 @@ static void stih_rx_done(struct stih_cec *cec, u32 status) if (!msg.len) return; - if (msg.len > 16) - msg.len = 16; + if (msg.len > CEC_MAX_MSG_SIZE) + msg.len = CEC_MAX_MSG_SIZE; for (i = 0; i < msg.len; i++) msg.msg[i] = readl(cec->regs + CEC_RX_DATA_BASE + i); diff --git a/drivers/media/common/Kconfig b/drivers/media/common/Kconfig index a2ae7127005427e7b5d47c0c90d4b69482818359..852b7d92fbdd93c4a99ec88415f9d8a2ca64e517 100644 --- a/drivers/media/common/Kconfig +++ b/drivers/media/common/Kconfig @@ -22,7 +22,6 @@ config VIDEO_TVEEPROM depends on I2C source "drivers/media/common/b2c2/Kconfig" -source "drivers/media/common/saa7146/Kconfig" source "drivers/media/common/siano/Kconfig" source "drivers/media/common/v4l2-tpg/Kconfig" source "drivers/media/common/videobuf2/Kconfig" diff --git a/drivers/media/common/Makefile b/drivers/media/common/Makefile index ad0b1e95fb124554eb4a9099c89bb656e6b6d6d0..d78a0df154783a8b8e9de40d0fb248543d3d7628 100644 --- a/drivers/media/common/Makefile +++ b/drivers/media/common/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += b2c2/ saa7146/ siano/ v4l2-tpg/ videobuf2/ +obj-y += b2c2/ siano/ v4l2-tpg/ videobuf2/ # Please keep it alphabetically sorted by Kconfig name # (e. g. LC_ALL=C sort Makefile) diff --git a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c index 9b7bcdce6e44e64a129ad62980fd6e7a8a2f1042..303d02b1d71c9e67d34a8710a54f98c5d2576e9f 100644 --- a/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c +++ b/drivers/media/common/v4l2-tpg/v4l2-tpg-core.c @@ -870,7 +870,7 @@ static void precalculate_color(struct tpg_data *tpg, int k) g = tpg_colors[col].g; b = tpg_colors[col].b; } else if (tpg->pattern == TPG_PAT_NOISE) { - r = g = b = prandom_u32_max(256); + r = g = b = get_random_u8(); } else if (k == TPG_COLOR_RANDOM) { r = g = b = tpg->qual_offset + prandom_u32_max(196); } else if (k >= TPG_COLOR_RAMP) { diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c index b203c1e26353eecbb4676de8afa2d8995091ca04..ab9697f3b5f1915d0d379b149b9b2f86d877ccc8 100644 --- a/drivers/media/common/videobuf2/videobuf2-core.c +++ b/drivers/media/common/videobuf2/videobuf2-core.c @@ -398,7 +398,7 @@ static void init_buffer_cache_hints(struct vb2_queue *q, struct vb2_buffer *vb) } /* - * __vb2_queue_alloc() - allocate videobuf buffer structures and (for MMAP type) + * __vb2_queue_alloc() - allocate vb2 buffer structures and (for MMAP type) * video buffer memory for all buffers/planes on the queue and initializes the * queue * @@ -417,7 +417,7 @@ static int __vb2_queue_alloc(struct vb2_queue *q, enum vb2_memory memory, VB2_MAX_FRAME - q->num_buffers); for (buffer = 0; buffer < num_buffers; ++buffer) { - /* Allocate videobuf buffer structures */ + /* Allocate vb2 buffer structures */ vb = kzalloc(q->buf_struct_size, GFP_KERNEL); if (!vb) { dprintk(q, 1, "memory alloc for buffer struct failed\n"); @@ -599,7 +599,7 @@ static int __vb2_queue_free(struct vb2_queue *q, unsigned int buffers) } #endif - /* Free videobuf buffers */ + /* Free vb2 buffers */ for (buffer = q->num_buffers - buffers; buffer < q->num_buffers; ++buffer) { kfree(q->bufs[buffer]); @@ -1949,7 +1949,7 @@ int vb2_core_dqbuf(struct vb2_queue *q, unsigned int *pindex, void *pb, if (pb) call_void_bufop(q, fill_user_buffer, vb, pb); - /* Remove from videobuf queue */ + /* Remove from vb2 queue */ list_del(&vb->queued_entry); q->queued_count--; @@ -1978,7 +1978,7 @@ EXPORT_SYMBOL_GPL(vb2_core_dqbuf); * __vb2_queue_cancel() - cancel and stop (pause) streaming * * Removes all queued buffers from driver's queue and all buffers queued by - * userspace from videobuf's queue. Returns to state after reqbufs. + * userspace from vb2's queue. Returns to state after reqbufs. */ static void __vb2_queue_cancel(struct vb2_queue *q) { @@ -2016,7 +2016,7 @@ static void __vb2_queue_cancel(struct vb2_queue *q) q->uses_qbuf = 0; /* - * Remove all buffers from videobuf's list... + * Remove all buffers from vb2's list... */ INIT_LIST_HEAD(&q->queued_list); /* @@ -2139,7 +2139,7 @@ int vb2_core_streamoff(struct vb2_queue *q, unsigned int type) /* * Cancel will pause streaming and remove all buffers from the driver - * and videobuf, effectively returning control over them to userspace. + * and vb2, effectively returning control over them to userspace. * * Note that we do this even if q->streaming == 0: if you prepare or * queue buffers, and then call streamoff without ever having called diff --git a/drivers/media/common/videobuf2/videobuf2-dvb.c b/drivers/media/common/videobuf2/videobuf2-dvb.c index 9d571c9d31e9c56c76da09f233c7412ce191266c..8c15bcd07eef9a12d738c72513f64b6a56a43563 100644 --- a/drivers/media/common/videobuf2/videobuf2-dvb.c +++ b/drivers/media/common/videobuf2/videobuf2-dvb.c @@ -3,8 +3,8 @@ * * some helper function for simple DVB cards which simply DMA the * complete transport stream and let the computer sort everything else - * (i.e. we are using the software demux, ...). Also uses the - * video-buf to manage DMA buffers. + * (i.e. we are using the software demux, ...). Also uses vb2 + * to manage DMA buffers. * * (c) 2004 Gerd Knorr [SUSE Labs] */ diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c index f26cb8586bd4e15c1534696aa26f9b2f8caf7669..1f5d235a844171b822ae87e0682801b8082e289c 100644 --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c @@ -268,7 +268,7 @@ static int vb2_fill_vb2_v4l2_buffer(struct vb2_buffer *vb, struct v4l2_buffer *b /* * Single-planar buffers do not use planes array, * so fill in relevant v4l2_buffer struct fields instead. - * In videobuf we use our internal V4l2_planes struct for + * In vb2 we use our internal V4l2_planes struct for * single-planar buffers as well, for simplicity. * * If bytesused == 0 for the output buffer, then fall back @@ -625,19 +625,6 @@ static const struct vb2_buf_ops v4l2_buf_ops = { .copy_timestamp = __copy_timestamp, }; -int vb2_find_timestamp(const struct vb2_queue *q, u64 timestamp, - unsigned int start_idx) -{ - unsigned int i; - - for (i = start_idx; i < q->num_buffers; i++) - if (q->bufs[i]->copied_timestamp && - q->bufs[i]->timestamp == timestamp) - return i; - return -1; -} -EXPORT_SYMBOL_GPL(vb2_find_timestamp); - struct vb2_buffer *vb2_find_buffer(struct vb2_queue *q, u64 timestamp) { unsigned int i; @@ -652,7 +639,7 @@ EXPORT_SYMBOL_GPL(vb2_find_buffer); /* * vb2_querybuf() - query video buffer information - * @q: videobuf queue + * @q: vb2 queue * @b: buffer struct passed from userspace to vidioc_querybuf handler * in driver * diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c index a1bd6d9c9223cdf9426b049da35127712c6b33cf..909df82fed3329f38fb73af0c3a3ac5c702566f2 100644 --- a/drivers/media/dvb-core/dvb_vb2.c +++ b/drivers/media/dvb-core/dvb_vb2.c @@ -354,6 +354,12 @@ int dvb_vb2_reqbufs(struct dvb_vb2_ctx *ctx, struct dmx_requestbuffers *req) int dvb_vb2_querybuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b) { + struct vb2_queue *q = &ctx->vb_q; + + if (b->index >= q->num_buffers) { + dprintk(1, "[%s] buffer index out of range\n", ctx->name); + return -EINVAL; + } vb2_core_querybuf(&ctx->vb_q, b->index, b); dprintk(3, "[%s] index=%d\n", ctx->name, b->index); return 0; @@ -378,8 +384,13 @@ int dvb_vb2_expbuf(struct dvb_vb2_ctx *ctx, struct dmx_exportbuffer *exp) int dvb_vb2_qbuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b) { + struct vb2_queue *q = &ctx->vb_q; int ret; + if (b->index >= q->num_buffers) { + dprintk(1, "[%s] buffer index out of range\n", ctx->name); + return -EINVAL; + } ret = vb2_core_qbuf(&ctx->vb_q, b->index, b, NULL); if (ret) { dprintk(1, "[%s] index=%d errno=%d\n", ctx->name, diff --git a/drivers/media/dvb-frontends/a8293.c b/drivers/media/dvb-frontends/a8293.c index 57f52c004a2336e3b677198005ffa726d02f0fa0..ba38783b2b4fa6ec683edf60a5f4e306dd7d983a 100644 --- a/drivers/media/dvb-frontends/a8293.c +++ b/drivers/media/dvb-frontends/a8293.c @@ -98,14 +98,13 @@ err: return ret; } -static int a8293_remove(struct i2c_client *client) +static void a8293_remove(struct i2c_client *client) { struct a8293_dev *dev = i2c_get_clientdata(client); dev_dbg(&client->dev, "\n"); kfree(dev); - return 0; } static const struct i2c_device_id a8293_id_table[] = { diff --git a/drivers/media/dvb-frontends/af9013.c b/drivers/media/dvb-frontends/af9013.c index 7d7c341b2bd84067bd9d8b0c4911540e4fde7055..d85929582c3f934bca010e43231a250b7e17d808 100644 --- a/drivers/media/dvb-frontends/af9013.c +++ b/drivers/media/dvb-frontends/af9013.c @@ -1540,7 +1540,7 @@ err: return ret; } -static int af9013_remove(struct i2c_client *client) +static void af9013_remove(struct i2c_client *client) { struct af9013_state *state = i2c_get_clientdata(client); @@ -1551,8 +1551,6 @@ static int af9013_remove(struct i2c_client *client) regmap_exit(state->regmap); kfree(state); - - return 0; } static const struct i2c_device_id af9013_id_table[] = { diff --git a/drivers/media/dvb-frontends/af9033.c b/drivers/media/dvb-frontends/af9033.c index 785c49b3d3072bb4737c82af72ebe17af1bee67b..808da7a9ffe761fc6b867223aee958912ec9ce68 100644 --- a/drivers/media/dvb-frontends/af9033.c +++ b/drivers/media/dvb-frontends/af9033.c @@ -1163,7 +1163,7 @@ err: return ret; } -static int af9033_remove(struct i2c_client *client) +static void af9033_remove(struct i2c_client *client) { struct af9033_dev *dev = i2c_get_clientdata(client); @@ -1171,8 +1171,6 @@ static int af9033_remove(struct i2c_client *client) regmap_exit(dev->regmap); kfree(dev); - - return 0; } static const struct i2c_device_id af9033_id_table[] = { diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c index 8cdca051e51bf3b32324de5ca5dbc3ee1189ebb0..e4f99bd468cb04f230ac4f350023306edda48973 100644 --- a/drivers/media/dvb-frontends/au8522_decoder.c +++ b/drivers/media/dvb-frontends/au8522_decoder.c @@ -758,13 +758,12 @@ static int au8522_probe(struct i2c_client *client, return 0; } -static int au8522_remove(struct i2c_client *client) +static void au8522_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); au8522_release_state(to_state(sd)); - return 0; } static const struct i2c_device_id au8522_id[] = { diff --git a/drivers/media/dvb-frontends/cxd2099.c b/drivers/media/dvb-frontends/cxd2099.c index 1c8207ab898869d559add8ca09d2fc9002f2f1f4..fbc666fa04ec5c6896d85cbcd9765cfe1d3fe4ad 100644 --- a/drivers/media/dvb-frontends/cxd2099.c +++ b/drivers/media/dvb-frontends/cxd2099.c @@ -664,14 +664,12 @@ err: return ret; } -static int cxd2099_remove(struct i2c_client *client) +static void cxd2099_remove(struct i2c_client *client) { struct cxd *ci = i2c_get_clientdata(client); regmap_exit(ci->regmap); kfree(ci); - - return 0; } static const struct i2c_device_id cxd2099_id[] = { diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c index b1618339eec0e68c440ebc1f910df202b7a54db6..5d98222f9df0919a01e52d2d1bea008626f6ff56 100644 --- a/drivers/media/dvb-frontends/cxd2820r_core.c +++ b/drivers/media/dvb-frontends/cxd2820r_core.c @@ -705,7 +705,7 @@ err: return ret; } -static int cxd2820r_remove(struct i2c_client *client) +static void cxd2820r_remove(struct i2c_client *client) { struct cxd2820r_priv *priv = i2c_get_clientdata(client); @@ -721,8 +721,6 @@ static int cxd2820r_remove(struct i2c_client *client) regmap_exit(priv->regmap[0]); kfree(priv); - - return 0; } static const struct i2c_device_id cxd2820r_id_table[] = { diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c index d67f2dd997d06188bba43fb266829ffd298e8a82..fe19d127abb3f35727ac816564b766d532adde91 100644 --- a/drivers/media/dvb-frontends/dib8000.c +++ b/drivers/media/dvb-frontends/dib8000.c @@ -3212,7 +3212,7 @@ static int dib8000_tune(struct dvb_frontend *fe) case CT_DEMOD_STEP_6: /* (36) if there is an input (diversity) */ if ((state->fe[1] != NULL) && (state->output_mode != OUTMODE_DIVERSITY)) { - /* if there is a diversity fe in input and this fe is has not already failed : wait here until this this fe has succedeed or failed */ + /* if there is a diversity fe in input and this fe is has not already failed : wait here until this fe has succeeded or failed */ if (dib8000_get_status(state->fe[1]) <= FE_STATUS_STD_SUCCESS) /* Something is locked on the input fe */ *tune_state = CT_DEMOD_STEP_8; /* go for mpeg */ else if (dib8000_get_status(state->fe[1]) >= FE_STATUS_TUNE_TIME_TOO_SHORT) { /* fe in input failed also, break the current one */ diff --git a/drivers/media/dvb-frontends/drxk_hard.c b/drivers/media/dvb-frontends/drxk_hard.c index 9430295a8175c750188bdab14dda41ba53173da6..47d83e0a470c72f0496c8cee0c53c2b824fd4e74 100644 --- a/drivers/media/dvb-frontends/drxk_hard.c +++ b/drivers/media/dvb-frontends/drxk_hard.c @@ -3516,7 +3516,7 @@ static int set_dvbt_standard(struct drxk_state *state, status = write16(state, IQM_AF_CLP_LEN__A, 0); if (status < 0) goto error; - /* window size for for sense pre-SAW detection */ + /* window size for sense pre-SAW detection */ status = write16(state, IQM_AF_SNS_LEN__A, 0); if (status < 0) goto error; diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c index d45b4ddc8f91294ce68b626286f678555e9e0b1f..baf2a378e565f172f8ea32e07185fe2c22bb748d 100644 --- a/drivers/media/dvb-frontends/dvb-pll.c +++ b/drivers/media/dvb-frontends/dvb-pll.c @@ -899,14 +899,13 @@ dvb_pll_probe(struct i2c_client *client, const struct i2c_device_id *id) return 0; } -static int dvb_pll_remove(struct i2c_client *client) +static void dvb_pll_remove(struct i2c_client *client) { struct dvb_frontend *fe = i2c_get_clientdata(client); struct dvb_pll_priv *priv = fe->tuner_priv; ida_simple_remove(&pll_ida, priv->nr); dvb_pll_release(fe); - return 0; } diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c index 136b76cb48077686a9dbd5c45096e8ee89667e40..424311afb2bfa24f55ad99d45fd14f52cacdf9ef 100644 --- a/drivers/media/dvb-frontends/lgdt3306a.c +++ b/drivers/media/dvb-frontends/lgdt3306a.c @@ -2226,7 +2226,7 @@ fail: return ret; } -static int lgdt3306a_remove(struct i2c_client *client) +static void lgdt3306a_remove(struct i2c_client *client) { struct lgdt3306a_state *state = i2c_get_clientdata(client); @@ -2237,8 +2237,6 @@ static int lgdt3306a_remove(struct i2c_client *client) kfree(state->cfg); kfree(state); - - return 0; } static const struct i2c_device_id lgdt3306a_id_table[] = { diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c index da3a8c5e18d8ed9204c5e3cb812642359cd92871..ea9ae22fd201646c5df55dfd9241a72a16dfbd16 100644 --- a/drivers/media/dvb-frontends/lgdt330x.c +++ b/drivers/media/dvb-frontends/lgdt330x.c @@ -974,15 +974,13 @@ static const struct dvb_frontend_ops lgdt3303_ops = { .release = lgdt330x_release, }; -static int lgdt330x_remove(struct i2c_client *client) +static void lgdt330x_remove(struct i2c_client *client) { struct lgdt330x_state *state = i2c_get_clientdata(client); dev_dbg(&client->dev, "\n"); kfree(state); - - return 0; } static const struct i2c_device_id lgdt330x_id_table[] = { diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c index bce0f42f3d192d152fbdc4d5be2ea0af76dccebf..4e844b2ef59711d702ed2a574a6698c2fc3c8361 100644 --- a/drivers/media/dvb-frontends/m88ds3103.c +++ b/drivers/media/dvb-frontends/m88ds3103.c @@ -1914,7 +1914,7 @@ err: return ret; } -static int m88ds3103_remove(struct i2c_client *client) +static void m88ds3103_remove(struct i2c_client *client) { struct m88ds3103_dev *dev = i2c_get_clientdata(client); @@ -1926,7 +1926,6 @@ static int m88ds3103_remove(struct i2c_client *client) i2c_mux_del_adapters(dev->muxc); kfree(dev); - return 0; } static const struct i2c_device_id m88ds3103_id_table[] = { diff --git a/drivers/media/dvb-frontends/mn88443x.c b/drivers/media/dvb-frontends/mn88443x.c index fff212c0bf3b5a4b36fc6b15b84018f27f1dfac8..452571b380b720bd3508ea900584dd799fd84a81 100644 --- a/drivers/media/dvb-frontends/mn88443x.c +++ b/drivers/media/dvb-frontends/mn88443x.c @@ -762,15 +762,13 @@ err_i2c_t: return ret; } -static int mn88443x_remove(struct i2c_client *client) +static void mn88443x_remove(struct i2c_client *client) { struct mn88443x_priv *chip = i2c_get_clientdata(client); mn88443x_cmn_power_off(chip); i2c_unregister_device(chip->client_t); - - return 0; } static const struct mn88443x_spec mn88443x_spec_pri = { diff --git a/drivers/media/dvb-frontends/mn88472.c b/drivers/media/dvb-frontends/mn88472.c index 73922fc8f39c22bc4c27ab4ae1b9cf4c065db626..2b01cc678f7efed4916460b5bf62f67cd803ce68 100644 --- a/drivers/media/dvb-frontends/mn88472.c +++ b/drivers/media/dvb-frontends/mn88472.c @@ -691,7 +691,7 @@ err: return ret; } -static int mn88472_remove(struct i2c_client *client) +static void mn88472_remove(struct i2c_client *client) { struct mn88472_dev *dev = i2c_get_clientdata(client); @@ -706,8 +706,6 @@ static int mn88472_remove(struct i2c_client *client) regmap_exit(dev->regmap[0]); kfree(dev); - - return 0; } static const struct i2c_device_id mn88472_id_table[] = { diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c index 4838969ef735c4b240eb9e3cd54b659bebcbbd88..f0ecf5910c022a1cd22e02ce1300f3f15bc6d83a 100644 --- a/drivers/media/dvb-frontends/mn88473.c +++ b/drivers/media/dvb-frontends/mn88473.c @@ -726,7 +726,7 @@ err: return ret; } -static int mn88473_remove(struct i2c_client *client) +static void mn88473_remove(struct i2c_client *client) { struct mn88473_dev *dev = i2c_get_clientdata(client); @@ -741,8 +741,6 @@ static int mn88473_remove(struct i2c_client *client) regmap_exit(dev->regmap[0]); kfree(dev); - - return 0; } static const struct i2c_device_id mn88473_id_table[] = { diff --git a/drivers/media/dvb-frontends/mxl692.c b/drivers/media/dvb-frontends/mxl692.c index dd7954e8f5533f9d05839c3719b8d8067b552b02..129630cbffffe24601d5cd2d837ddbea383a0b5d 100644 --- a/drivers/media/dvb-frontends/mxl692.c +++ b/drivers/media/dvb-frontends/mxl692.c @@ -1337,15 +1337,13 @@ err: return -ENODEV; } -static int mxl692_remove(struct i2c_client *client) +static void mxl692_remove(struct i2c_client *client) { struct mxl692_dev *dev = i2c_get_clientdata(client); dev->fe.demodulator_priv = NULL; i2c_set_clientdata(client, NULL); kfree(dev); - - return 0; } static const struct i2c_device_id mxl692_id_table[] = { diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c index e6b8367c8cce4ec416d24a439325349798667321..e0fbf41316ae7ca1df4c0deede5a60e894ac5907 100644 --- a/drivers/media/dvb-frontends/rtl2830.c +++ b/drivers/media/dvb-frontends/rtl2830.c @@ -865,7 +865,7 @@ err: return ret; } -static int rtl2830_remove(struct i2c_client *client) +static void rtl2830_remove(struct i2c_client *client) { struct rtl2830_dev *dev = i2c_get_clientdata(client); @@ -874,8 +874,6 @@ static int rtl2830_remove(struct i2c_client *client) i2c_mux_del_adapters(dev->muxc); regmap_exit(dev->regmap); kfree(dev); - - return 0; } static const struct i2c_device_id rtl2830_id_table[] = { diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c index dcbeb9f5e12a3bc8530765de81c081702284f062..4fa884eda5d505de195b78bf75c9f0712495d198 100644 --- a/drivers/media/dvb-frontends/rtl2832.c +++ b/drivers/media/dvb-frontends/rtl2832.c @@ -1110,7 +1110,7 @@ err: return ret; } -static int rtl2832_remove(struct i2c_client *client) +static void rtl2832_remove(struct i2c_client *client) { struct rtl2832_dev *dev = i2c_get_clientdata(client); @@ -1123,8 +1123,6 @@ static int rtl2832_remove(struct i2c_client *client) regmap_exit(dev->regmap); kfree(dev); - - return 0; } static const struct i2c_device_id rtl2832_id_table[] = { diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c index 6a4f2997d6f52c89e2ebdff3ce55086eb2a99d59..05f71d1697267ee940c3c5ee5802e611dbd7f633 100644 --- a/drivers/media/dvb-frontends/rtl2832_sdr.c +++ b/drivers/media/dvb-frontends/rtl2832_sdr.c @@ -245,7 +245,7 @@ static void rtl2832_sdr_urb_complete(struct urb *urb) if (unlikely(fbuf == NULL)) { dev->vb_full++; dev_notice_ratelimited(&pdev->dev, - "videobuf is full, %d packets dropped\n", + "video buffer is full, %d packets dropped\n", dev->vb_full); goto skip; } diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c index ebee230afb7b818f069992ecde309e0034491a5e..86b0d59169ddf743b7f00bbdcba2e8717fb6429c 100644 --- a/drivers/media/dvb-frontends/si2165.c +++ b/drivers/media/dvb-frontends/si2165.c @@ -1274,14 +1274,13 @@ error: return ret; } -static int si2165_remove(struct i2c_client *client) +static void si2165_remove(struct i2c_client *client) { struct si2165_state *state = i2c_get_clientdata(client); dev_dbg(&client->dev, "\n"); kfree(state); - return 0; } static const struct i2c_device_id si2165_id_table[] = { diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c index 196e028a66171786eccabb81e34ae3df0eaa08cc..8157df4570d195a375776c46fad877fe3f481dbf 100644 --- a/drivers/media/dvb-frontends/si2168.c +++ b/drivers/media/dvb-frontends/si2168.c @@ -774,7 +774,7 @@ err: return ret; } -static int si2168_remove(struct i2c_client *client) +static void si2168_remove(struct i2c_client *client) { struct si2168_dev *dev = i2c_get_clientdata(client); @@ -786,8 +786,6 @@ static int si2168_remove(struct i2c_client *client) dev->fe.demodulator_priv = NULL; kfree(dev); - - return 0; } static const struct i2c_device_id si2168_id_table[] = { diff --git a/drivers/media/dvb-frontends/sp2.c b/drivers/media/dvb-frontends/sp2.c index 992f22167fbea4ff8e73971659167fea3bb54d1f..27e7037e130ead1d75b944e07dc006f993d31da6 100644 --- a/drivers/media/dvb-frontends/sp2.c +++ b/drivers/media/dvb-frontends/sp2.c @@ -398,14 +398,13 @@ err: return ret; } -static int sp2_remove(struct i2c_client *client) +static void sp2_remove(struct i2c_client *client) { struct sp2 *s = i2c_get_clientdata(client); dev_dbg(&client->dev, "\n"); sp2_exit(client); kfree(s); - return 0; } static const struct i2c_device_id sp2_id[] = { diff --git a/drivers/media/dvb-frontends/stv090x.c b/drivers/media/dvb-frontends/stv090x.c index 90d24131d335f78aa5da365b05aebdeb0a11d67e..0a600c1d7d1b1fd0fed55f8595be6705f445f2a1 100644 --- a/drivers/media/dvb-frontends/stv090x.c +++ b/drivers/media/dvb-frontends/stv090x.c @@ -5032,12 +5032,11 @@ error: return ret; } -static int stv090x_remove(struct i2c_client *client) +static void stv090x_remove(struct i2c_client *client) { struct stv090x_state *state = i2c_get_clientdata(client); stv090x_release(&state->frontend); - return 0; } struct dvb_frontend *stv090x_attach(struct stv090x_config *config, diff --git a/drivers/media/dvb-frontends/stv6110x.c b/drivers/media/dvb-frontends/stv6110x.c index 5012d0231652237ac1de26938e1a32794e142b8c..fbc4dbd62151d0c135fd1418cfdeabaa472d7feb 100644 --- a/drivers/media/dvb-frontends/stv6110x.c +++ b/drivers/media/dvb-frontends/stv6110x.c @@ -436,12 +436,11 @@ static int stv6110x_probe(struct i2c_client *client, return 0; } -static int stv6110x_remove(struct i2c_client *client) +static void stv6110x_remove(struct i2c_client *client) { struct stv6110x_state *stv6110x = i2c_get_clientdata(client); stv6110x_release(stv6110x->frontend); - return 0; } const struct stv6110x_devctl *stv6110x_attach(struct dvb_frontend *fe, diff --git a/drivers/media/dvb-frontends/tc90522.c b/drivers/media/dvb-frontends/tc90522.c index e83836b29715ce29f48796c15439a137d3b15fb3..c22d2a2b2a459e18b33f4f3f4fb769f83b2db25c 100644 --- a/drivers/media/dvb-frontends/tc90522.c +++ b/drivers/media/dvb-frontends/tc90522.c @@ -819,14 +819,13 @@ free_state: return ret; } -static int tc90522_remove(struct i2c_client *client) +static void tc90522_remove(struct i2c_client *client) { struct tc90522_state *state; state = cfg_to_state(i2c_get_clientdata(client)); i2c_del_adapter(&state->tuner_i2c); kfree(state); - return 0; } diff --git a/drivers/media/dvb-frontends/tda1002x.h b/drivers/media/dvb-frontends/tda1002x.h index 60a0952c1bca4d6b7764063372adae9ee2fd3c41..00491bea99758b638add281129416748ca3236d7 100644 --- a/drivers/media/dvb-frontends/tda1002x.h +++ b/drivers/media/dvb-frontends/tda1002x.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* TDA10021/TDA10023 - Single Chip Cable Channel Receiver driver module - used on the the Siemens DVB-C cards + used on the Siemens DVB-C cards Copyright (C) 1999 Convergence Integrated Media GmbH Copyright (C) 2004 Markus Schulz diff --git a/drivers/media/dvb-frontends/tda10048.c b/drivers/media/dvb-frontends/tda10048.c index d1d206ebdedd79e5f161f3d1811f63f1ac02212c..0b3f6999515e358e49bc04b647e4df397e954cd5 100644 --- a/drivers/media/dvb-frontends/tda10048.c +++ b/drivers/media/dvb-frontends/tda10048.c @@ -1118,7 +1118,7 @@ struct dvb_frontend *tda10048_attach(const struct tda10048_config *config, state->pll_pfactor = 0; } - /* Establish any defaults the the user didn't pass */ + /* Establish any defaults the user didn't pass */ tda10048_establish_defaults(&state->frontend); /* Set the xtal and freq defaults */ diff --git a/drivers/media/dvb-frontends/tda10071.c b/drivers/media/dvb-frontends/tda10071.c index 685c0ac71819e917d9c010c11313dbd67bcc12f5..d1098ef20a8bdcf6b691e66054b76303687d2a8c 100644 --- a/drivers/media/dvb-frontends/tda10071.c +++ b/drivers/media/dvb-frontends/tda10071.c @@ -1221,14 +1221,13 @@ err: return ret; } -static int tda10071_remove(struct i2c_client *client) +static void tda10071_remove(struct i2c_client *client) { struct tda10071_dev *dev = i2c_get_clientdata(client); dev_dbg(&client->dev, "\n"); kfree(dev); - return 0; } static const struct i2c_device_id tda10071_id_table[] = { diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c index 3e383912bcfd8f3025792bc8c1f59e6a3871e85c..02338256b974f79d9f90a728e28d51d92a5ca2f5 100644 --- a/drivers/media/dvb-frontends/ts2020.c +++ b/drivers/media/dvb-frontends/ts2020.c @@ -696,7 +696,7 @@ err: return ret; } -static int ts2020_remove(struct i2c_client *client) +static void ts2020_remove(struct i2c_client *client) { struct ts2020_priv *dev = i2c_get_clientdata(client); @@ -708,7 +708,6 @@ static int ts2020_remove(struct i2c_client *client) regmap_exit(dev->regmap); kfree(dev); - return 0; } static const struct i2c_device_id ts2020_id_table[] = { diff --git a/drivers/media/i2c/ad5820.c b/drivers/media/i2c/ad5820.c index 2958a46944614eade6e1c7e5325cbe9fb9abaa07..516de278cc493d374c96a505de8ba03b29a835ad 100644 --- a/drivers/media/i2c/ad5820.c +++ b/drivers/media/i2c/ad5820.c @@ -342,7 +342,7 @@ cleanup: return ret; } -static int ad5820_remove(struct i2c_client *client) +static void ad5820_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct ad5820_device *coil = to_ad5820_device(subdev); @@ -351,7 +351,6 @@ static int ad5820_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&coil->ctrls); media_entity_cleanup(&coil->subdev.entity); mutex_destroy(&coil->power_lock); - return 0; } static const struct i2c_device_id ad5820_id_table[] = { diff --git a/drivers/media/i2c/ad9389b.c b/drivers/media/i2c/ad9389b.c index 8679a44e6413bb8f7bb67e628584a8fd308e28fb..4a255a492918bf8a8ad2c3a5755cf0152bd2a856 100644 --- a/drivers/media/i2c/ad9389b.c +++ b/drivers/media/i2c/ad9389b.c @@ -1174,7 +1174,7 @@ err_hdl: /* ----------------------------------------------------------------------- */ -static int ad9389b_remove(struct i2c_client *client) +static void ad9389b_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ad9389b_state *state = get_ad9389b_state(sd); @@ -1192,7 +1192,6 @@ static int ad9389b_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/adp1653.c b/drivers/media/i2c/adp1653.c index 522a0b10e415d70570527f8779e0b603fa75cf5a..1f353157df07683c19e5af3fafaec8b79be57aae 100644 --- a/drivers/media/i2c/adp1653.c +++ b/drivers/media/i2c/adp1653.c @@ -510,7 +510,7 @@ free_and_quit: return ret; } -static int adp1653_remove(struct i2c_client *client) +static void adp1653_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct adp1653_flash *flash = to_adp1653_flash(subdev); @@ -518,8 +518,6 @@ static int adp1653_remove(struct i2c_client *client) v4l2_device_unregister_subdev(&flash->subdev); v4l2_ctrl_handler_free(&flash->ctrls); media_entity_cleanup(&flash->subdev.entity); - - return 0; } static const struct i2c_device_id adp1653_id_table[] = { diff --git a/drivers/media/i2c/adv7170.c b/drivers/media/i2c/adv7170.c index 714e31f993e1c0a07f7563019599b03d2338bb52..61a2f87d3c62cb25e04188f82c670c4169413067 100644 --- a/drivers/media/i2c/adv7170.c +++ b/drivers/media/i2c/adv7170.c @@ -368,12 +368,11 @@ static int adv7170_probe(struct i2c_client *client, return 0; } -static int adv7170_remove(struct i2c_client *client) +static void adv7170_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/adv7175.c b/drivers/media/i2c/adv7175.c index 1813f67f0fe1d3d498718896ecf67892300a03a7..b58689728243fc189f57eb900ec64dd5171f83be 100644 --- a/drivers/media/i2c/adv7175.c +++ b/drivers/media/i2c/adv7175.c @@ -423,12 +423,11 @@ static int adv7175_probe(struct i2c_client *client, return 0; } -static int adv7175_remove(struct i2c_client *client) +static void adv7175_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 5fde5243722d743271c9c994c718b6d5e271f79a..216fe396973f26bdbe57133819413d29cccca7ab 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -1514,7 +1514,7 @@ err_unregister_csi_client: return ret; } -static int adv7180_remove(struct i2c_client *client) +static void adv7180_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv7180_state *state = to_state(sd); @@ -1534,8 +1534,6 @@ static int adv7180_remove(struct i2c_client *client) adv7180_set_power_pin(state, false); mutex_destroy(&state->mutex); - - return 0; } static const struct i2c_device_id adv7180_id[] = { diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c index ba746a19fd3953bcb933afc593d97d4cdb7c7225..313c706e83354823c4c3c95017c2b7ebae278838 100644 --- a/drivers/media/i2c/adv7183.c +++ b/drivers/media/i2c/adv7183.c @@ -613,13 +613,12 @@ static int adv7183_probe(struct i2c_client *client, return 0; } -static int adv7183_remove(struct i2c_client *client) +static void adv7183_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } static const struct i2c_device_id adv7183_id[] = { diff --git a/drivers/media/i2c/adv7343.c b/drivers/media/i2c/adv7343.c index 63e94dfcb5d33ba2551e28d789b649134585f9b7..7e84869d2434946f82451b57646fc21a2df08bf6 100644 --- a/drivers/media/i2c/adv7343.c +++ b/drivers/media/i2c/adv7343.c @@ -492,15 +492,13 @@ done: return err; } -static int adv7343_remove(struct i2c_client *client) +static void adv7343_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv7343_state *state = to_state(sd); v4l2_async_unregister_subdev(&state->sd); v4l2_ctrl_handler_free(&state->hdl); - - return 0; } static const struct i2c_device_id adv7343_id[] = { diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c index b6234c8231c9ae89089b669b532c53359ee79e99..fb5fefa83b18631f9667f8e2507bb1e4a9d16ed6 100644 --- a/drivers/media/i2c/adv7393.c +++ b/drivers/media/i2c/adv7393.c @@ -437,15 +437,13 @@ static int adv7393_probe(struct i2c_client *client, return err; } -static int adv7393_remove(struct i2c_client *client) +static void adv7393_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv7393_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - - return 0; } static const struct i2c_device_id adv7393_id[] = { diff --git a/drivers/media/i2c/adv748x/adv748x-core.c b/drivers/media/i2c/adv748x/adv748x-core.c index 4e54148147b9adecb6a552dc002a08526ce27c60..4498d78a235701e176f57a274170bc88d29ab298 100644 --- a/drivers/media/i2c/adv748x/adv748x-core.c +++ b/drivers/media/i2c/adv748x/adv748x-core.c @@ -815,7 +815,7 @@ err_free_mutex: return ret; } -static int adv748x_remove(struct i2c_client *client) +static void adv748x_remove(struct i2c_client *client) { struct adv748x_state *state = i2c_get_clientdata(client); @@ -828,8 +828,6 @@ static int adv748x_remove(struct i2c_client *client) adv748x_unregister_clients(state); adv748x_dt_cleanup(state); mutex_destroy(&state->mutex); - - return 0; } static const struct of_device_id adv748x_of_table[] = { diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c index 202e0cd83f908feb4a60b629a38a9f41c1281b01..0d5ce69f12e7386d6d4d693dda83215f2f8d65fa 100644 --- a/drivers/media/i2c/adv7511-v4l2.c +++ b/drivers/media/i2c/adv7511-v4l2.c @@ -943,8 +943,8 @@ static int adv7511_isr(struct v4l2_subdev *sd, u32 status, bool *handled) v4l2_dbg(1, debug, sd, "%s: cec msg len %d\n", __func__, msg.len); - if (msg.len > 16) - msg.len = 16; + if (msg.len > CEC_MAX_MSG_SIZE) + msg.len = CEC_MAX_MSG_SIZE; if (msg.len) { u8 i; @@ -1923,7 +1923,7 @@ err_hdl: /* ----------------------------------------------------------------------- */ -static int adv7511_remove(struct i2c_client *client) +static void adv7511_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv7511_state *state = get_adv7511_state(sd); @@ -1943,7 +1943,6 @@ static int adv7511_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index 497419a5cfddd4375f52bd6f9af5d06bc4db66ec..bda0c547ce44525decb5868166bd01097276537d 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -2047,8 +2047,8 @@ static void adv76xx_cec_isr(struct v4l2_subdev *sd, bool *handled) struct cec_msg msg; msg.len = cec_read(sd, 0x25) & 0x1f; - if (msg.len > 16) - msg.len = 16; + if (msg.len > CEC_MAX_MSG_SIZE) + msg.len = CEC_MAX_MSG_SIZE; if (msg.len) { u8 i; @@ -3660,7 +3660,7 @@ err_hdl: /* ----------------------------------------------------------------------- */ -static int adv76xx_remove(struct i2c_client *client) +static void adv76xx_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv76xx_state *state = to_state(sd); @@ -3677,7 +3677,6 @@ static int adv76xx_remove(struct i2c_client *client) media_entity_cleanup(&sd->entity); adv76xx_unregister_clients(to_state(sd)); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c index 22caa070273b43b87e0e78e9fc08e340c81afef2..7731cc1887e671e243120ca068e6909726d21104 100644 --- a/drivers/media/i2c/adv7842.c +++ b/drivers/media/i2c/adv7842.c @@ -2215,8 +2215,8 @@ static void adv7842_cec_isr(struct v4l2_subdev *sd, bool *handled) struct cec_msg msg; msg.len = cec_read(sd, 0x25) & 0x1f; - if (msg.len > 16) - msg.len = 16; + if (msg.len > CEC_MAX_MSG_SIZE) + msg.len = CEC_MAX_MSG_SIZE; if (msg.len) { u8 i; @@ -3593,7 +3593,7 @@ err_hdl: /* ----------------------------------------------------------------------- */ -static int adv7842_remove(struct i2c_client *client) +static void adv7842_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct adv7842_state *state = to_state(sd); @@ -3604,7 +3604,6 @@ static int adv7842_remove(struct i2c_client *client) media_entity_cleanup(&sd->entity); adv7842_unregister_clients(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/ak7375.c b/drivers/media/i2c/ak7375.c index 40b1a4aa846cae965149070dc2069fba5b729c63..1af9f698eecf89fc581112c2c87a9a8782a7df61 100644 --- a/drivers/media/i2c/ak7375.c +++ b/drivers/media/i2c/ak7375.c @@ -169,7 +169,7 @@ err_cleanup: return ret; } -static int ak7375_remove(struct i2c_client *client) +static void ak7375_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ak7375_device *ak7375_dev = sd_to_ak7375_vcm(sd); @@ -177,8 +177,6 @@ static int ak7375_remove(struct i2c_client *client) ak7375_subdev_cleanup(ak7375_dev); pm_runtime_disable(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } /* diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c index dc569d5a4d9d915e17d3481586e81dfb84d5e90f..0370ad6b6811e10484e98404d669b719cc92d2d2 100644 --- a/drivers/media/i2c/ak881x.c +++ b/drivers/media/i2c/ak881x.c @@ -297,13 +297,11 @@ static int ak881x_probe(struct i2c_client *client, return 0; } -static int ak881x_remove(struct i2c_client *client) +static void ak881x_remove(struct i2c_client *client) { struct ak881x *ak881x = to_ak881x(client); v4l2_device_unregister_subdev(&ak881x->subdev); - - return 0; } static const struct i2c_device_id ak881x_id[] = { diff --git a/drivers/media/i2c/ar0521.c b/drivers/media/i2c/ar0521.c index c7bdfc69b9be83883b175e28aeb1c106c838060c..c6ab531532beb53273ed2b5ede6e8e65261ff495 100644 --- a/drivers/media/i2c/ar0521.c +++ b/drivers/media/i2c/ar0521.c @@ -1018,7 +1018,7 @@ entity_cleanup: return ret; } -static int ar0521_remove(struct i2c_client *client) +static void ar0521_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ar0521_dev *sensor = to_ar0521_dev(sd); @@ -1031,7 +1031,6 @@ static int ar0521_remove(struct i2c_client *client) ar0521_power_off(&client->dev); pm_runtime_set_suspended(&client->dev); mutex_destroy(&sensor->lock); - return 0; } static const struct dev_pm_ops ar0521_pm_ops = { diff --git a/drivers/media/i2c/bt819.c b/drivers/media/i2c/bt819.c index 73bc50c919d78649b96c08ad185fde27a9972187..4d9bb6eb7d6527f9751c2e81e7f38da835d1d665 100644 --- a/drivers/media/i2c/bt819.c +++ b/drivers/media/i2c/bt819.c @@ -446,14 +446,13 @@ static int bt819_probe(struct i2c_client *client, return 0; } -static int bt819_remove(struct i2c_client *client) +static void bt819_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct bt819 *decoder = to_bt819(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&decoder->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/bt856.c b/drivers/media/i2c/bt856.c index c134fda270a177e5ffafbc2506c64dc9304d2b78..70443ef1ac46ae66e4977d2b34b3b2760252045b 100644 --- a/drivers/media/i2c/bt856.c +++ b/drivers/media/i2c/bt856.c @@ -223,12 +223,11 @@ static int bt856_probe(struct i2c_client *client, return 0; } -static int bt856_remove(struct i2c_client *client) +static void bt856_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id bt856_id[] = { diff --git a/drivers/media/i2c/bt866.c b/drivers/media/i2c/bt866.c index 1a8df9f18ffb6cd738cb8ef063a0b57d28dd1c9d..c2508cbafd02fbf77f3c9f96adc46cabf26948c5 100644 --- a/drivers/media/i2c/bt866.c +++ b/drivers/media/i2c/bt866.c @@ -190,12 +190,11 @@ static int bt866_probe(struct i2c_client *client, return 0; } -static int bt866_remove(struct i2c_client *client) +static void bt866_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id bt866_id[] = { diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index 7609add2aff497ab7be774e37ebb6ece43f7431d..4a14d7e5d9f25174c9a142d5c6eafe7ddf09c167 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -3665,7 +3665,7 @@ out_power_off: return rval; } -static int ccs_remove(struct i2c_client *client) +static void ccs_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -3687,8 +3687,6 @@ static int ccs_remove(struct i2c_client *client) kfree(sensor->ccs_limits); kvfree(sensor->sdata.backing); kvfree(sensor->mdata.backing); - - return 0; } static const struct ccs_device smia_device = { diff --git a/drivers/media/i2c/cs3308.c b/drivers/media/i2c/cs3308.c index ebe55e261bff22227c58278a16b7d3647066c4e6..d901a59883a93359cacd16b060b16f4b34cdcfb7 100644 --- a/drivers/media/i2c/cs3308.c +++ b/drivers/media/i2c/cs3308.c @@ -99,13 +99,12 @@ static int cs3308_probe(struct i2c_client *client, /* ----------------------------------------------------------------------- */ -static int cs3308_remove(struct i2c_client *client) +static void cs3308_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); kfree(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/cs5345.c b/drivers/media/i2c/cs5345.c index f6dd5edf77dd9ff1cfdcb7f428c7736274a80535..591b1e7b24eeefa7eaf468568041c4568f1d283f 100644 --- a/drivers/media/i2c/cs5345.c +++ b/drivers/media/i2c/cs5345.c @@ -178,14 +178,13 @@ static int cs5345_probe(struct i2c_client *client, /* ----------------------------------------------------------------------- */ -static int cs5345_remove(struct i2c_client *client) +static void cs5345_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct cs5345_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/cs53l32a.c b/drivers/media/i2c/cs53l32a.c index 9a411106cfb3a8a00b0bfabe86c9e7347186251c..9461589aea30077becc31f45517a91ea96a92871 100644 --- a/drivers/media/i2c/cs53l32a.c +++ b/drivers/media/i2c/cs53l32a.c @@ -190,14 +190,13 @@ static int cs53l32a_probe(struct i2c_client *client, return 0; } -static int cs53l32a_remove(struct i2c_client *client) +static void cs53l32a_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct cs53l32a_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } static const struct i2c_device_id cs53l32a_id[] = { diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c index dc31944c7d5b144b6cfdcc637999d7eb7ad04b15..f1a978af82ef288d8171900b719b09165eab13c8 100644 --- a/drivers/media/i2c/cx25840/cx25840-core.c +++ b/drivers/media/i2c/cx25840/cx25840-core.c @@ -6026,7 +6026,7 @@ static int cx25840_probe(struct i2c_client *client, return 0; } -static int cx25840_remove(struct i2c_client *client) +static void cx25840_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct cx25840_state *state = to_state(sd); @@ -6034,7 +6034,6 @@ static int cx25840_remove(struct i2c_client *client) cx25840_ir_remove(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } static const struct i2c_device_id cx25840_id[] = { diff --git a/drivers/media/i2c/cx25840/cx25840-ir.c b/drivers/media/i2c/cx25840/cx25840-ir.c index 9d7d1d149f1a0e197ca06adacf95d82188dfe92f..8cef9656c61224861f1b0027be5e67fdec18cacd 100644 --- a/drivers/media/i2c/cx25840/cx25840-ir.c +++ b/drivers/media/i2c/cx25840/cx25840-ir.c @@ -196,7 +196,7 @@ static u32 clock_divider_to_resolution(u16 divider) { /* * Resolution is the duration of 1 tick of the readable portion of - * of the pulse width counter as read from the FIFO. The two lsb's are + * the pulse width counter as read from the FIFO. The two lsb's are * not readable, hence the << 2. This function returns ns. */ return DIV_ROUND_CLOSEST((1 << 2) * ((u32) divider + 1) * 1000, diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c index 206d74338b9c9d2ab236da6ef138b1507e8b675a..af59687383aa57ae3ffe08dfdc1d3c8952189aa1 100644 --- a/drivers/media/i2c/dw9714.c +++ b/drivers/media/i2c/dw9714.c @@ -190,7 +190,7 @@ err_cleanup: return rval; } -static int dw9714_remove(struct i2c_client *client) +static void dw9714_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct dw9714_device *dw9714_dev = sd_to_dw9714_vcm(sd); @@ -206,8 +206,6 @@ static int dw9714_remove(struct i2c_client *client) } pm_runtime_set_suspended(&client->dev); dw9714_subdev_cleanup(dw9714_dev); - - return 0; } /* diff --git a/drivers/media/i2c/dw9768.c b/drivers/media/i2c/dw9768.c index c086580efac78a850b8f5d8e853d8209f2fb4877..0f47ef015a1d309dfe84b313f772d805863593f1 100644 --- a/drivers/media/i2c/dw9768.c +++ b/drivers/media/i2c/dw9768.c @@ -499,7 +499,7 @@ err_free_handler: return ret; } -static int dw9768_remove(struct i2c_client *client) +static void dw9768_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct dw9768 *dw9768 = sd_to_dw9768(sd); @@ -511,8 +511,6 @@ static int dw9768_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) dw9768_runtime_suspend(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct of_device_id dw9768_of_table[] = { diff --git a/drivers/media/i2c/dw9807-vcm.c b/drivers/media/i2c/dw9807-vcm.c index 01c372925a806565aae28ce2e9740074a671230f..3599720db7e9959cb9cecb65c931b96fa85a62a7 100644 --- a/drivers/media/i2c/dw9807-vcm.c +++ b/drivers/media/i2c/dw9807-vcm.c @@ -216,7 +216,7 @@ err_cleanup: return rval; } -static int dw9807_remove(struct i2c_client *client) +static void dw9807_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct dw9807_device *dw9807_dev = sd_to_dw9807_vcm(sd); @@ -224,8 +224,6 @@ static int dw9807_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); dw9807_subdev_cleanup(dw9807_dev); - - return 0; } /* diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c index 873d614339bbb9f0cdd0b88d40e465762bd9adf4..ff9bb9fc97dd851e459446d65f7ca5db7dcfa272 100644 --- a/drivers/media/i2c/et8ek8/et8ek8_driver.c +++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c @@ -1460,7 +1460,7 @@ err_mutex: return ret; } -static int __exit et8ek8_remove(struct i2c_client *client) +static void __exit et8ek8_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct et8ek8_sensor *sensor = to_et8ek8_sensor(subdev); @@ -1477,8 +1477,6 @@ static int __exit et8ek8_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&sensor->subdev); media_entity_cleanup(&sensor->subdev.entity); mutex_destroy(&sensor->power_lock); - - return 0; } static const struct of_device_id et8ek8_of_table[] = { diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c index 055d1aa8410e54126d6a9809ccbece85a15415c9..e422ac7609b5ce9c6153d8cf8feb95aee89d513b 100644 --- a/drivers/media/i2c/hi556.c +++ b/drivers/media/i2c/hi556.c @@ -1101,7 +1101,7 @@ check_hwcfg_error: return ret; } -static int hi556_remove(struct i2c_client *client) +static void hi556_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct hi556 *hi556 = to_hi556(sd); @@ -1111,8 +1111,6 @@ static int hi556_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&hi556->mutex); - - return 0; } static int hi556_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/hi846.c b/drivers/media/i2c/hi846.c index ad35c3ff3611589c19ddf0441770fcac4fa14f59..c5b69823f257e836e2cb2e09eff15dc242df8b71 100644 --- a/drivers/media/i2c/hi846.c +++ b/drivers/media/i2c/hi846.c @@ -2143,7 +2143,7 @@ err_mutex: return ret; } -static int hi846_remove(struct i2c_client *client) +static void hi846_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct hi846 *hi846 = to_hi846(sd); @@ -2158,8 +2158,6 @@ static int hi846_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&hi846->mutex); - - return 0; } static const struct dev_pm_ops hi846_pm_ops = { diff --git a/drivers/media/i2c/hi847.c b/drivers/media/i2c/hi847.c index 7e85349e1852a48c0c05c61f4e5f26eff29aa3a4..5a82b15a9513e1e264e2ecb9d13949fa0680a699 100644 --- a/drivers/media/i2c/hi847.c +++ b/drivers/media/i2c/hi847.c @@ -2903,7 +2903,7 @@ check_hwcfg_error: return ret; } -static int hi847_remove(struct i2c_client *client) +static void hi847_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct hi847 *hi847 = to_hi847(sd); @@ -2913,8 +2913,6 @@ static int hi847_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&hi847->mutex); - - return 0; } static int hi847_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/imx208.c b/drivers/media/i2c/imx208.c index b9516b2f1c15e49a41f53db86d5dd6129ca21e79..a0e17bb9d4ca4724669b701376a315399d6d7f85 100644 --- a/drivers/media/i2c/imx208.c +++ b/drivers/media/i2c/imx208.c @@ -1061,7 +1061,7 @@ error_probe: return ret; } -static int imx208_remove(struct i2c_client *client) +static void imx208_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx208 *imx208 = to_imx208(sd); @@ -1075,8 +1075,6 @@ static int imx208_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx208->imx208_mx); - - return 0; } static const struct dev_pm_ops imx208_pm_ops = { diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c index 83c1737abeecec29d86cf86d98d4577acb624dbe..710c9fb515fd09ffb14a403539de082b43060cb4 100644 --- a/drivers/media/i2c/imx214.c +++ b/drivers/media/i2c/imx214.c @@ -1080,7 +1080,7 @@ free_ctrl: return ret; } -static int imx214_remove(struct i2c_client *client) +static void imx214_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx214 *imx214 = to_imx214(sd); @@ -1093,8 +1093,6 @@ static int imx214_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx214->mutex); - - return 0; } static const struct of_device_id imx214_of_match[] = { diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index e10af3f74b38f55a1e791a1f43140b9a2c43eca2..77bd79a5954edfa137e3aa5cf49c40dbbbbef355 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -1562,7 +1562,7 @@ error_power_off: return ret; } -static int imx219_remove(struct i2c_client *client) +static void imx219_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx219 *imx219 = to_imx219(sd); @@ -1575,8 +1575,6 @@ static int imx219_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) imx219_power_off(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct of_device_id imx219_dt_ids[] = { diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c index c249507aa2dbcdcf5c38cf6305911e5537097556..eab5fc1ee2f7403de4a9fb62e9146f7594618191 100644 --- a/drivers/media/i2c/imx258.c +++ b/drivers/media/i2c/imx258.c @@ -1338,7 +1338,7 @@ error_identify: return ret; } -static int imx258_remove(struct i2c_client *client) +static void imx258_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx258 *imx258 = to_imx258(sd); @@ -1351,8 +1351,6 @@ static int imx258_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) imx258_power_off(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct dev_pm_ops imx258_pm_ops = { diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c index 7de1f2948e53124e10dc388e6e38601da7696457..a00761b1e18c2b79a2214f2298aafbfa87791101 100644 --- a/drivers/media/i2c/imx274.c +++ b/drivers/media/i2c/imx274.c @@ -2142,7 +2142,7 @@ err_regmap: return ret; } -static int imx274_remove(struct i2c_client *client) +static void imx274_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct stimx274 *imx274 = to_imx274(sd); @@ -2157,7 +2157,6 @@ static int imx274_remove(struct i2c_client *client) media_entity_cleanup(&sd->entity); mutex_destroy(&imx274->lock); - return 0; } static const struct dev_pm_ops imx274_pm_ops = { diff --git a/drivers/media/i2c/imx290.c b/drivers/media/i2c/imx290.c index 99f2a50d39a4b739ed4d11e5170028cfd3bf0de0..1ce64dcdf7f02f6a21f8f7ba38a41edfd1a1b7f3 100644 --- a/drivers/media/i2c/imx290.c +++ b/drivers/media/i2c/imx290.c @@ -1119,7 +1119,7 @@ free_err: return ret; } -static int imx290_remove(struct i2c_client *client) +static void imx290_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx290 *imx290 = to_imx290(sd); @@ -1134,8 +1134,6 @@ static int imx290_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(imx290->dev)) imx290_power_off(imx290->dev); pm_runtime_set_suspended(imx290->dev); - - return 0; } static const struct of_device_id imx290_of_match[] = { diff --git a/drivers/media/i2c/imx319.c b/drivers/media/i2c/imx319.c index a2b5a34de76b8e835c8321fd454965f3373537b6..245a18fb40ad0d0e7321c170cd3bb67f76c95854 100644 --- a/drivers/media/i2c/imx319.c +++ b/drivers/media/i2c/imx319.c @@ -2523,7 +2523,7 @@ error_probe: return ret; } -static int imx319_remove(struct i2c_client *client) +static void imx319_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx319 *imx319 = to_imx319(sd); @@ -2536,8 +2536,6 @@ static int imx319_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx319->mutex); - - return 0; } static const struct dev_pm_ops imx319_pm_ops = { diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c index 062125501788adbce169a4f9f43dc9bcd57da087..7b0a9086447d1fff0d4b80b31d118b5d38904542 100644 --- a/drivers/media/i2c/imx334.c +++ b/drivers/media/i2c/imx334.c @@ -1089,7 +1089,7 @@ error_mutex_destroy: * * Return: 0 if successful, error code otherwise. */ -static int imx334_remove(struct i2c_client *client) +static void imx334_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx334 *imx334 = to_imx334(sd); @@ -1102,8 +1102,6 @@ static int imx334_remove(struct i2c_client *client) pm_runtime_suspended(&client->dev); mutex_destroy(&imx334->mutex); - - return 0; } static const struct dev_pm_ops imx334_pm_ops = { diff --git a/drivers/media/i2c/imx335.c b/drivers/media/i2c/imx335.c index 410d6b86feb52c7d3b93399ae38667f7ebc4260b..078ede2b7a00fdb9886c2742257258ca9f0419d1 100644 --- a/drivers/media/i2c/imx335.c +++ b/drivers/media/i2c/imx335.c @@ -1083,7 +1083,7 @@ error_mutex_destroy: * * Return: 0 if successful, error code otherwise. */ -static int imx335_remove(struct i2c_client *client) +static void imx335_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx335 *imx335 = to_imx335(sd); @@ -1098,8 +1098,6 @@ static int imx335_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx335->mutex); - - return 0; } static const struct dev_pm_ops imx335_pm_ops = { diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c index 3922b9305978ec31a0f850eb7b300ec08aa2c994..b46178681c0564f51310eaddc366f276ab270b2f 100644 --- a/drivers/media/i2c/imx355.c +++ b/drivers/media/i2c/imx355.c @@ -1810,7 +1810,7 @@ error_probe: return ret; } -static int imx355_remove(struct i2c_client *client) +static void imx355_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx355 *imx355 = to_imx355(sd); @@ -1823,8 +1823,6 @@ static int imx355_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx355->mutex); - - return 0; } static const struct dev_pm_ops imx355_pm_ops = { diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c index a1394d6c14320505281a410d4731e212af9e77fa..7f6d29e0e7c404fbebd597ad653e4bdb43960945 100644 --- a/drivers/media/i2c/imx412.c +++ b/drivers/media/i2c/imx412.c @@ -1257,7 +1257,7 @@ error_mutex_destroy: * * Return: 0 if successful, error code otherwise. */ -static int imx412_remove(struct i2c_client *client) +static void imx412_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct imx412 *imx412 = to_imx412(sd); @@ -1272,8 +1272,6 @@ static int imx412_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&imx412->mutex); - - return 0; } static const struct dev_pm_ops imx412_pm_ops = { diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c index 56674173524fdb25775f812761e140253d723fb5..ee6bbbb977f74dfc563f0129513105a7996dec45 100644 --- a/drivers/media/i2c/ir-kbd-i2c.c +++ b/drivers/media/i2c/ir-kbd-i2c.c @@ -915,7 +915,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id) return err; } -static int ir_remove(struct i2c_client *client) +static void ir_remove(struct i2c_client *client) { struct IR_i2c *ir = i2c_get_clientdata(client); @@ -924,8 +924,6 @@ static int ir_remove(struct i2c_client *client) i2c_unregister_device(ir->tx_c); rc_unregister_device(ir->rc); - - return 0; } static const struct i2c_device_id ir_kbd_id[] = { diff --git a/drivers/media/i2c/isl7998x.c b/drivers/media/i2c/isl7998x.c index dc3068549dfa8aeefb72733ad5b664cdd5a3d9a5..246d8d182a8e52ba3ff5b2a4d9e2cb62147a502c 100644 --- a/drivers/media/i2c/isl7998x.c +++ b/drivers/media/i2c/isl7998x.c @@ -1544,7 +1544,7 @@ err_entity_cleanup: return ret; } -static int isl7998x_remove(struct i2c_client *client) +static void isl7998x_remove(struct i2c_client *client) { struct isl7998x *isl7998x = i2c_to_isl7998x(client); @@ -1552,8 +1552,6 @@ static int isl7998x_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&isl7998x->subdev); isl7998x_remove_controls(isl7998x); media_entity_cleanup(&isl7998x->subdev.entity); - - return 0; } static const struct of_device_id isl7998x_of_match[] = { diff --git a/drivers/media/i2c/ks0127.c b/drivers/media/i2c/ks0127.c index c077f53b9c3091f7d49a91ad57375a866803949c..215d9a43b0b93263a0d97d5a9a3af9202bda3992 100644 --- a/drivers/media/i2c/ks0127.c +++ b/drivers/media/i2c/ks0127.c @@ -675,14 +675,13 @@ static int ks0127_probe(struct i2c_client *client, const struct i2c_device_id *i return 0; } -static int ks0127_remove(struct i2c_client *client) +static void ks0127_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); ks0127_write(sd, KS_OFMTA, 0x20); /* tristate */ ks0127_write(sd, KS_CMDA, 0x2c | 0x80); /* power down */ - return 0; } static const struct i2c_device_id ks0127_id[] = { diff --git a/drivers/media/i2c/lm3560.c b/drivers/media/i2c/lm3560.c index 9e34ccce4fc3c248947e788e967e7658ac11dbda..edad3138cb075fa249ccadcbd772a0f53ee5e2d4 100644 --- a/drivers/media/i2c/lm3560.c +++ b/drivers/media/i2c/lm3560.c @@ -443,7 +443,7 @@ static int lm3560_probe(struct i2c_client *client, return 0; } -static int lm3560_remove(struct i2c_client *client) +static void lm3560_remove(struct i2c_client *client) { struct lm3560_flash *flash = i2c_get_clientdata(client); unsigned int i; @@ -453,8 +453,6 @@ static int lm3560_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&flash->ctrls_led[i]); media_entity_cleanup(&flash->subdev_led[i].entity); } - - return 0; } static const struct i2c_device_id lm3560_id_table[] = { diff --git a/drivers/media/i2c/lm3646.c b/drivers/media/i2c/lm3646.c index c76ccf67a9094e411d8b5dc54e65e0bc67f17839..0aaa963917d8cd5937d400d7c0ce5d981a38c15d 100644 --- a/drivers/media/i2c/lm3646.c +++ b/drivers/media/i2c/lm3646.c @@ -377,15 +377,13 @@ static int lm3646_probe(struct i2c_client *client, return 0; } -static int lm3646_remove(struct i2c_client *client) +static void lm3646_remove(struct i2c_client *client) { struct lm3646_flash *flash = i2c_get_clientdata(client); v4l2_device_unregister_subdev(&flash->subdev_led); v4l2_ctrl_handler_free(&flash->ctrls_led); media_entity_cleanup(&flash->subdev_led.entity); - - return 0; } static const struct i2c_device_id lm3646_id_table[] = { diff --git a/drivers/media/i2c/m52790.c b/drivers/media/i2c/m52790.c index 0a1efc1417bc3da05f221606154b72b8c7b825d5..2ab91b993c33f9c32f7ed8d9d098179a6bce0307 100644 --- a/drivers/media/i2c/m52790.c +++ b/drivers/media/i2c/m52790.c @@ -154,12 +154,11 @@ static int m52790_probe(struct i2c_client *client, return 0; } -static int m52790_remove(struct i2c_client *client) +static void m52790_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/m5mols/m5mols_core.c b/drivers/media/i2c/m5mols/m5mols_core.c index c19590389bfe622d087e33837d67879b6881f5fa..2201d2a26353aa68c9157812fa4a286581c9ef6b 100644 --- a/drivers/media/i2c/m5mols/m5mols_core.c +++ b/drivers/media/i2c/m5mols/m5mols_core.c @@ -1020,15 +1020,13 @@ error: return ret; } -static int m5mols_remove(struct i2c_client *client) +static void m5mols_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); media_entity_cleanup(&sd->entity); - - return 0; } static const struct i2c_device_id m5mols_id[] = { diff --git a/drivers/media/i2c/max2175.c b/drivers/media/i2c/max2175.c index 0eea200124d283bd7b2df3ea28e089074c7f83f9..1019020f3a37b12ecfbb597a29d1623aec704788 100644 --- a/drivers/media/i2c/max2175.c +++ b/drivers/media/i2c/max2175.c @@ -1403,15 +1403,13 @@ err_reg: return ret; } -static int max2175_remove(struct i2c_client *client) +static void max2175_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct max2175 *ctx = max2175_from_sd(sd); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); v4l2_async_unregister_subdev(sd); - - return 0; } static const struct i2c_device_id max2175_id[] = { diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c index 3684faa72253bba710b840c670722f2d7ee56ad7..9c083cf142319e4f882d20834297cdc178cd3340 100644 --- a/drivers/media/i2c/max9286.c +++ b/drivers/media/i2c/max9286.c @@ -1378,7 +1378,7 @@ err_powerdown: return ret; } -static int max9286_remove(struct i2c_client *client) +static void max9286_remove(struct i2c_client *client) { struct max9286_priv *priv = sd_to_max9286(i2c_get_clientdata(client)); @@ -1391,8 +1391,6 @@ static int max9286_remove(struct i2c_client *client) gpiod_set_value_cansleep(priv->gpiod_pwdn, 0); max9286_cleanup_dt(priv); - - return 0; } static const struct of_device_id max9286_dt_ids[] = { diff --git a/drivers/media/i2c/ml86v7667.c b/drivers/media/i2c/ml86v7667.c index 48cc0b0922f4f5d6d0803803bfcb01debce514a0..49ec59b0ca43669dbc79066d0bb06fb317ae767c 100644 --- a/drivers/media/i2c/ml86v7667.c +++ b/drivers/media/i2c/ml86v7667.c @@ -415,15 +415,13 @@ cleanup: return ret; } -static int ml86v7667_remove(struct i2c_client *client) +static void ml86v7667_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ml86v7667_priv *priv = to_ml86v7667(sd); v4l2_ctrl_handler_free(&priv->hdl); v4l2_device_unregister_subdev(&priv->sd); - - return 0; } static const struct i2c_device_id ml86v7667_id[] = { diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c index 39530d43590e3cc54e3212e167c21eb76c8c283a..4ce7a15a988441a70fd0f2dea90e24de2aac10f6 100644 --- a/drivers/media/i2c/msp3400-driver.c +++ b/drivers/media/i2c/msp3400-driver.c @@ -859,7 +859,7 @@ static int msp_probe(struct i2c_client *client, const struct i2c_device_id *id) return 0; } -static int msp_remove(struct i2c_client *client) +static void msp_remove(struct i2c_client *client) { struct msp_state *state = to_state(i2c_get_clientdata(client)); @@ -872,7 +872,6 @@ static int msp_remove(struct i2c_client *client) msp_reset(client); v4l2_ctrl_handler_free(&state->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/mt9m001.c b/drivers/media/i2c/mt9m001.c index ad13b0c890c08f2d71236d80739eeeb56c4d76c9..ebf9cf1e1bce1bfb400e78cb5238201cf1640194 100644 --- a/drivers/media/i2c/mt9m001.c +++ b/drivers/media/i2c/mt9m001.c @@ -833,7 +833,7 @@ error_hdl_free: return ret; } -static int mt9m001_remove(struct i2c_client *client) +static void mt9m001_remove(struct i2c_client *client) { struct mt9m001 *mt9m001 = to_mt9m001(client); @@ -853,8 +853,6 @@ static int mt9m001_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&mt9m001->hdl); mutex_destroy(&mt9m001->mutex); - - return 0; } static const struct i2c_device_id mt9m001_id[] = { diff --git a/drivers/media/i2c/mt9m032.c b/drivers/media/i2c/mt9m032.c index ba0c0ea91c954160327c8f7a40e5cdd87d06531d..76b8c9c08c820930490d6b5f05c6b62183c96f25 100644 --- a/drivers/media/i2c/mt9m032.c +++ b/drivers/media/i2c/mt9m032.c @@ -858,7 +858,7 @@ error_sensor: return ret; } -static int mt9m032_remove(struct i2c_client *client) +static void mt9m032_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct mt9m032 *sensor = to_mt9m032(subdev); @@ -867,7 +867,6 @@ static int mt9m032_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&sensor->ctrls); media_entity_cleanup(&subdev->entity); mutex_destroy(&sensor->lock); - return 0; } static const struct i2c_device_id mt9m032_id_table[] = { diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c index afc86efa9e3ebd739b7fdcaab3bc36e0f5095eba..f5fe272d12055d870e4229e3e36243e12ff94ffd 100644 --- a/drivers/media/i2c/mt9m111.c +++ b/drivers/media/i2c/mt9m111.c @@ -1359,15 +1359,13 @@ out_hdlfree: return ret; } -static int mt9m111_remove(struct i2c_client *client) +static void mt9m111_remove(struct i2c_client *client) { struct mt9m111 *mt9m111 = to_mt9m111(client); v4l2_async_unregister_subdev(&mt9m111->subdev); media_entity_cleanup(&mt9m111->subdev.entity); v4l2_ctrl_handler_free(&mt9m111->hdl); - - return 0; } static const struct of_device_id mt9m111_of_match[] = { { .compatible = "micron,mt9m111", }, diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c index 1fd4dc6e4726fe967b57d313e78bc119c3e52988..45f7b5e52bc3929cf6232af2d4fcbf3ec516de3b 100644 --- a/drivers/media/i2c/mt9p031.c +++ b/drivers/media/i2c/mt9p031.c @@ -1209,7 +1209,7 @@ done: return ret; } -static int mt9p031_remove(struct i2c_client *client) +static void mt9p031_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -1218,8 +1218,6 @@ static int mt9p031_remove(struct i2c_client *client) v4l2_async_unregister_subdev(subdev); media_entity_cleanup(&subdev->entity); mutex_destroy(&mt9p031->power_lock); - - return 0; } static const struct i2c_device_id mt9p031_id[] = { diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c index b651ee4a26e87c732f013619e0da9605d6225a62..d5abe4a7ef078cb183ac9bcd2a5924216201abac 100644 --- a/drivers/media/i2c/mt9t001.c +++ b/drivers/media/i2c/mt9t001.c @@ -961,7 +961,7 @@ done: return ret; } -static int mt9t001_remove(struct i2c_client *client) +static void mt9t001_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct mt9t001 *mt9t001 = to_mt9t001(subdev); @@ -969,7 +969,6 @@ static int mt9t001_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&mt9t001->ctrls); v4l2_device_unregister_subdev(subdev); media_entity_cleanup(&subdev->entity); - return 0; } static const struct i2c_device_id mt9t001_id[] = { diff --git a/drivers/media/i2c/mt9t112.c b/drivers/media/i2c/mt9t112.c index 8d2e3caa9b2869869f4eda6d459d206cad75a32e..ad564095d0cf1e82e1876b17aa5eae134e263d6d 100644 --- a/drivers/media/i2c/mt9t112.c +++ b/drivers/media/i2c/mt9t112.c @@ -1102,14 +1102,12 @@ static int mt9t112_probe(struct i2c_client *client, return v4l2_async_register_subdev(&priv->subdev); } -static int mt9t112_remove(struct i2c_client *client) +static void mt9t112_remove(struct i2c_client *client) { struct mt9t112_priv *priv = to_mt9t112(client); clk_disable_unprepare(priv->clk); v4l2_async_unregister_subdev(&priv->subdev); - - return 0; } static const struct i2c_device_id mt9t112_id[] = { diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c index 7699e64e112717d4f3fd391cd476b1801d6005a3..9952ce06ebb22eddfbcd8c757909143d5cf40d66 100644 --- a/drivers/media/i2c/mt9v011.c +++ b/drivers/media/i2c/mt9v011.c @@ -561,7 +561,7 @@ static int mt9v011_probe(struct i2c_client *c, return 0; } -static int mt9v011_remove(struct i2c_client *c) +static void mt9v011_remove(struct i2c_client *c) { struct v4l2_subdev *sd = i2c_get_clientdata(c); struct mt9v011 *core = to_mt9v011(sd); @@ -572,8 +572,6 @@ static int mt9v011_remove(struct i2c_client *c) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&core->ctrls); - - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c index 4cfdd3dfbd42acfdcd4c63721f609e309298385e..bc4388ccc2a8df02fa2fd2b1fd73ef5e5726c5cf 100644 --- a/drivers/media/i2c/mt9v032.c +++ b/drivers/media/i2c/mt9v032.c @@ -1192,7 +1192,7 @@ err: return ret; } -static int mt9v032_remove(struct i2c_client *client) +static void mt9v032_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -1200,8 +1200,6 @@ static int mt9v032_remove(struct i2c_client *client) v4l2_async_unregister_subdev(subdev); v4l2_ctrl_handler_free(&mt9v032->ctrls); media_entity_cleanup(&subdev->entity); - - return 0; } static const struct mt9v032_model_data mt9v032_model_data[] = { diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c index 2dc4a0f24ce860e16b7f2dfaebd089c247a84e40..fe18e5258d7afb8b2ed5c8e231b3b57fa46233e6 100644 --- a/drivers/media/i2c/mt9v111.c +++ b/drivers/media/i2c/mt9v111.c @@ -1238,7 +1238,7 @@ error_free_ctrls: return ret; } -static int mt9v111_remove(struct i2c_client *client) +static void mt9v111_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct mt9v111_dev *mt9v111 = sd_to_mt9v111(sd); @@ -1253,8 +1253,6 @@ static int mt9v111_remove(struct i2c_client *client) mutex_destroy(&mt9v111->pwr_mutex); mutex_destroy(&mt9v111->stream_mutex); - - return 0; } static const struct of_device_id mt9v111_of_match[] = { diff --git a/drivers/media/i2c/noon010pc30.c b/drivers/media/i2c/noon010pc30.c index bc5187f4636525e163aa531aedeec629519a1e7a..ecaf5e9057f1fd9da3243298d6276a6d9437c791 100644 --- a/drivers/media/i2c/noon010pc30.c +++ b/drivers/media/i2c/noon010pc30.c @@ -789,7 +789,7 @@ np_err: return ret; } -static int noon010_remove(struct i2c_client *client) +static void noon010_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct noon010_info *info = to_noon010(sd); @@ -797,8 +797,6 @@ static int noon010_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&info->hdl); media_entity_cleanup(&sd->entity); - - return 0; } static const struct i2c_device_id noon010_id[] = { diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c index 87179fc04e009a592be1e212784d00c7746aac1a..35663c10fcd9f8ebfd0071b921e49347bcb8f1f9 100644 --- a/drivers/media/i2c/og01a1b.c +++ b/drivers/media/i2c/og01a1b.c @@ -1015,7 +1015,7 @@ check_hwcfg_error: return ret; } -static int og01a1b_remove(struct i2c_client *client) +static void og01a1b_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct og01a1b *og01a1b = to_og01a1b(sd); @@ -1025,8 +1025,6 @@ static int og01a1b_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&og01a1b->mutex); - - return 0; } static int og01a1b_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c index 0f08c05333ea1ab88666b25a381e5d736ef528f0..2c1eb724d8e517044fa2d0711c1b5fd17165ecd5 100644 --- a/drivers/media/i2c/ov02a10.c +++ b/drivers/media/i2c/ov02a10.c @@ -975,7 +975,7 @@ err_destroy_mutex: return ret; } -static int ov02a10_remove(struct i2c_client *client) +static void ov02a10_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov02a10 *ov02a10 = to_ov02a10(sd); @@ -988,8 +988,6 @@ static int ov02a10_remove(struct i2c_client *client) ov02a10_power_off(&client->dev); pm_runtime_set_suspended(&client->dev); mutex_destroy(&ov02a10->mutex); - - return 0; } static const struct of_device_id ov02a10_of_match[] = { diff --git a/drivers/media/i2c/ov08d10.c b/drivers/media/i2c/ov08d10.c index e5ef6466a3ec42a5b56f9a0a0fcac48ae617fa3d..c1703596c3dc8500c615c6a2e8d4e7570c4d03a1 100644 --- a/drivers/media/i2c/ov08d10.c +++ b/drivers/media/i2c/ov08d10.c @@ -1415,7 +1415,7 @@ check_hwcfg_error: return ret; } -static int ov08d10_remove(struct i2c_client *client) +static void ov08d10_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov08d10 *ov08d10 = to_ov08d10(sd); @@ -1425,8 +1425,6 @@ static int ov08d10_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&ov08d10->mutex); - - return 0; } static int ov08d10_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c index d5fe67c763f774b53bcff4f1bd16ef5498bdc6e5..e618b613e0780ebd247d3c02d9e66cd7d404f4c9 100644 --- a/drivers/media/i2c/ov13858.c +++ b/drivers/media/i2c/ov13858.c @@ -1769,7 +1769,7 @@ error_handler_free: return ret; } -static int ov13858_remove(struct i2c_client *client) +static void ov13858_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov13858 *ov13858 = to_ov13858(sd); @@ -1779,8 +1779,6 @@ static int ov13858_remove(struct i2c_client *client) ov13858_free_controls(ov13858); pm_runtime_disable(&client->dev); - - return 0; } static const struct i2c_device_id ov13858_id_table[] = { diff --git a/drivers/media/i2c/ov13b10.c b/drivers/media/i2c/ov13b10.c index 7caeae641051f131470324e6249e6930aa84968f..549e5d93e568e65c4961f3a6dfa4c2a210716582 100644 --- a/drivers/media/i2c/ov13b10.c +++ b/drivers/media/i2c/ov13b10.c @@ -1447,7 +1447,7 @@ error_handler_free: return ret; } -static int ov13b10_remove(struct i2c_client *client) +static void ov13b10_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov13b10 *ov13b = to_ov13b10(sd); @@ -1457,8 +1457,6 @@ static int ov13b10_remove(struct i2c_client *client) ov13b10_free_controls(ov13b); pm_runtime_disable(&client->dev); - - return 0; } static const struct dev_pm_ops ov13b10_pm_ops = { diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c index 4b75da55b26082a9a5c57db1274c5252db83716c..29ed0ef8c03392e960f7c5060375297e49676763 100644 --- a/drivers/media/i2c/ov2640.c +++ b/drivers/media/i2c/ov2640.c @@ -1271,7 +1271,7 @@ err_clk: return ret; } -static int ov2640_remove(struct i2c_client *client) +static void ov2640_remove(struct i2c_client *client) { struct ov2640_priv *priv = to_ov2640(client); @@ -1281,7 +1281,6 @@ static int ov2640_remove(struct i2c_client *client) media_entity_cleanup(&priv->subdev.entity); v4l2_device_unregister_subdev(&priv->subdev); clk_disable_unprepare(priv->clk); - return 0; } static const struct i2c_device_id ov2640_id[] = { diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c index 13ded5b2aa6634d7c403b08c024d48bdb3637e48..42fc64ada08c0582bd0da392502194190a4ace2e 100644 --- a/drivers/media/i2c/ov2659.c +++ b/drivers/media/i2c/ov2659.c @@ -1544,7 +1544,7 @@ error: return ret; } -static int ov2659_remove(struct i2c_client *client) +static void ov2659_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2659 *ov2659 = to_ov2659(sd); @@ -1558,8 +1558,6 @@ static int ov2659_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) ov2659_power_off(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct dev_pm_ops ov2659_pm_ops = { diff --git a/drivers/media/i2c/ov2680.c b/drivers/media/i2c/ov2680.c index 906c711f6821bcd192f928b4e2a235945556d3f0..de66d3395a4dd97b7c687d024b96766225fe3f22 100644 --- a/drivers/media/i2c/ov2680.c +++ b/drivers/media/i2c/ov2680.c @@ -1097,7 +1097,7 @@ lock_destroy: return ret; } -static int ov2680_remove(struct i2c_client *client) +static void ov2680_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2680_dev *sensor = to_ov2680_dev(sd); @@ -1106,8 +1106,6 @@ static int ov2680_remove(struct i2c_client *client) mutex_destroy(&sensor->lock); media_entity_cleanup(&sensor->sd.entity); v4l2_ctrl_handler_free(&sensor->ctrls.handler); - - return 0; } static int __maybe_unused ov2680_suspend(struct device *dev) diff --git a/drivers/media/i2c/ov2685.c b/drivers/media/i2c/ov2685.c index b6e010ea3249ba327ebe0cd56b8059fb9a5630f1..a3b524f15d89afcd00e20d27b3cd1cb64d64ccf0 100644 --- a/drivers/media/i2c/ov2685.c +++ b/drivers/media/i2c/ov2685.c @@ -798,7 +798,7 @@ err_destroy_mutex: return ret; } -static int ov2685_remove(struct i2c_client *client) +static void ov2685_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2685 *ov2685 = to_ov2685(sd); @@ -814,8 +814,6 @@ static int ov2685_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) __ov2685_power_off(ov2685); pm_runtime_set_suspended(&client->dev); - - return 0; } #if IS_ENABLED(CONFIG_OF) diff --git a/drivers/media/i2c/ov2740.c b/drivers/media/i2c/ov2740.c index d5f0eabf20c6a589a856df3380504c38bad9ea1d..5d74ad479214688f89e1695fe8e492430d4c0135 100644 --- a/drivers/media/i2c/ov2740.c +++ b/drivers/media/i2c/ov2740.c @@ -1053,7 +1053,7 @@ check_hwcfg_error: return ret; } -static int ov2740_remove(struct i2c_client *client) +static void ov2740_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2740 *ov2740 = to_ov2740(sd); @@ -1063,8 +1063,6 @@ static int ov2740_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&ov2740->mutex); - - return 0; } static int ov2740_nvmem_read(void *priv, unsigned int off, void *val, diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c index 502f0b62e950542f071f7b634ae40942e0ea912e..1852e1cfc7df0ab30cb275a8ece34960aa41bb6f 100644 --- a/drivers/media/i2c/ov5640.c +++ b/drivers/media/i2c/ov5640.c @@ -3906,7 +3906,7 @@ entity_cleanup: return ret; } -static int ov5640_remove(struct i2c_client *client) +static void ov5640_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5640_dev *sensor = to_ov5640_dev(sd); @@ -3915,8 +3915,6 @@ static int ov5640_remove(struct i2c_client *client) media_entity_cleanup(&sensor->sd.entity); v4l2_ctrl_handler_free(&sensor->ctrls.handler); mutex_destroy(&sensor->lock); - - return 0; } static const struct i2c_device_id ov5640_id[] = { diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c index 562c62f192c458e4faada80b2a2288473b80a670..81e4e87e182188f3019f0fe638498eaa1e4b1a5b 100644 --- a/drivers/media/i2c/ov5645.c +++ b/drivers/media/i2c/ov5645.c @@ -1256,7 +1256,7 @@ free_ctrl: return ret; } -static int ov5645_remove(struct i2c_client *client) +static void ov5645_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5645 *ov5645 = to_ov5645(sd); @@ -1265,8 +1265,6 @@ static int ov5645_remove(struct i2c_client *client) media_entity_cleanup(&ov5645->sd.entity); v4l2_ctrl_handler_free(&ov5645->ctrls); mutex_destroy(&ov5645->power_lock); - - return 0; } static const struct i2c_device_id ov5645_id[] = { diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index d346d18ce629efe271f15e5a29afef305868379a..847a7bbb69c540c6b16e7a7565d626725533418f 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -1448,7 +1448,7 @@ mutex_destroy: return ret; } -static int ov5647_remove(struct i2c_client *client) +static void ov5647_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5647 *sensor = to_sensor(sd); @@ -1459,8 +1459,6 @@ static int ov5647_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); pm_runtime_disable(&client->dev); mutex_destroy(&sensor->lock); - - return 0; } static const struct dev_pm_ops ov5647_pm_ops = { diff --git a/drivers/media/i2c/ov5648.c b/drivers/media/i2c/ov5648.c index dfcd33e9ee136c98d8373b35ed06e0a389802f42..84604ea7bdf9ec3b37f6b6cc78698bb67cde8be3 100644 --- a/drivers/media/i2c/ov5648.c +++ b/drivers/media/i2c/ov5648.c @@ -2587,7 +2587,7 @@ error_endpoint: return ret; } -static int ov5648_remove(struct i2c_client *client) +static void ov5648_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct ov5648_sensor *sensor = ov5648_subdev_sensor(subdev); @@ -2597,8 +2597,6 @@ static int ov5648_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&sensor->ctrls.handler); mutex_destroy(&sensor->mutex); media_entity_cleanup(&subdev->entity); - - return 0; } static const struct dev_pm_ops ov5648_pm_ops = { diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c index 02f75c18e48058534de2331cd1d9462e156293ed..bc9fc3bc90c208a834acca16644f537a5a5d634b 100644 --- a/drivers/media/i2c/ov5670.c +++ b/drivers/media/i2c/ov5670.c @@ -2557,7 +2557,7 @@ error_print: return ret; } -static int ov5670_remove(struct i2c_client *client) +static void ov5670_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5670 *ov5670 = to_ov5670(sd); @@ -2568,8 +2568,6 @@ static int ov5670_remove(struct i2c_client *client) mutex_destroy(&ov5670->mutex); pm_runtime_disable(&client->dev); - - return 0; } static const struct dev_pm_ops ov5670_pm_ops = { diff --git a/drivers/media/i2c/ov5675.c b/drivers/media/i2c/ov5675.c index 82ba9f56baec83f7cb61f15ca51d322b4bd62f51..94dc8cb7a7c00bf9505470442d2cbaa54f14dfbe 100644 --- a/drivers/media/i2c/ov5675.c +++ b/drivers/media/i2c/ov5675.c @@ -1175,7 +1175,7 @@ check_hwcfg_error: return ret; } -static int ov5675_remove(struct i2c_client *client) +static void ov5675_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5675 *ov5675 = to_ov5675(sd); @@ -1185,8 +1185,6 @@ static int ov5675_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&ov5675->mutex); - - return 0; } static int ov5675_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/ov5693.c b/drivers/media/i2c/ov5693.c index 82a9b2de7735e2cdbe496444dacc997fba7ad4cc..a97ec132ba3a70765125411a5b58a54fddd90df0 100644 --- a/drivers/media/i2c/ov5693.c +++ b/drivers/media/i2c/ov5693.c @@ -1501,7 +1501,7 @@ err_ctrl_handler_free: return ret; } -static int ov5693_remove(struct i2c_client *client) +static void ov5693_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5693_device *ov5693 = to_ov5693_sensor(sd); @@ -1519,8 +1519,6 @@ static int ov5693_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) ov5693_sensor_powerdown(ov5693); pm_runtime_set_suspended(&client->dev); - - return 0; } static const struct dev_pm_ops ov5693_pm_ops = { diff --git a/drivers/media/i2c/ov5695.c b/drivers/media/i2c/ov5695.c index 9103097838856caf9d2234983129053ba07532bb..61906fc54e370d32d3f01852e2b0a8a68a1ddde1 100644 --- a/drivers/media/i2c/ov5695.c +++ b/drivers/media/i2c/ov5695.c @@ -1361,7 +1361,7 @@ err_destroy_mutex: return ret; } -static int ov5695_remove(struct i2c_client *client) +static void ov5695_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5695 *ov5695 = to_ov5695(sd); @@ -1377,8 +1377,6 @@ static int ov5695_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(&client->dev)) __ov5695_power_off(ov5695); pm_runtime_set_suspended(&client->dev); - - return 0; } #if IS_ENABLED(CONFIG_OF) diff --git a/drivers/media/i2c/ov6650.c b/drivers/media/i2c/ov6650.c index 6458e96d90916a6dce138c30c2018cd77b2f3d65..18f041e985b7b967ad77aefd65c961fb4e6de1d3 100644 --- a/drivers/media/i2c/ov6650.c +++ b/drivers/media/i2c/ov6650.c @@ -1096,13 +1096,12 @@ ectlhdlfree: return ret; } -static int ov6650_remove(struct i2c_client *client) +static void ov6650_remove(struct i2c_client *client) { struct ov6650 *priv = to_ov6650(client); v4l2_async_unregister_subdev(&priv->subdev); v4l2_ctrl_handler_free(&priv->hdl); - return 0; } static const struct i2c_device_id ov6650_id[] = { diff --git a/drivers/media/i2c/ov7251.c b/drivers/media/i2c/ov7251.c index 1bd797c7926b212bf2a725358afb29676e2eb650..88e98743528531c3302c766448f6dd678a03605e 100644 --- a/drivers/media/i2c/ov7251.c +++ b/drivers/media/i2c/ov7251.c @@ -1767,7 +1767,7 @@ destroy_mutex: return ret; } -static int ov7251_remove(struct i2c_client *client) +static void ov7251_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov7251 *ov7251 = to_ov7251(sd); @@ -1781,8 +1781,6 @@ static int ov7251_remove(struct i2c_client *client) if (!pm_runtime_status_suspended(ov7251->dev)) ov7251_set_power_off(ov7251->dev); pm_runtime_set_suspended(ov7251->dev); - - return 0; } static const struct dev_pm_ops ov7251_pm_ops = { diff --git a/drivers/media/i2c/ov7640.c b/drivers/media/i2c/ov7640.c index 977cd2d8ad337f93f7e2c271eafba50a7c9971e3..5e2d67f0f9f20a8912c13e8538ae00665fc14c08 100644 --- a/drivers/media/i2c/ov7640.c +++ b/drivers/media/i2c/ov7640.c @@ -70,13 +70,11 @@ static int ov7640_probe(struct i2c_client *client, } -static int ov7640_remove(struct i2c_client *client) +static void ov7640_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - - return 0; } static const struct i2c_device_id ov7640_id[] = { diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c index 1be2c0e5bdc1580d5a4d2e6a48e37f85bfa77c51..4b9b156b53c7a57968c30ed2e0287b9bcc2ba62f 100644 --- a/drivers/media/i2c/ov7670.c +++ b/drivers/media/i2c/ov7670.c @@ -2009,7 +2009,7 @@ power_off: return ret; } -static int ov7670_remove(struct i2c_client *client) +static void ov7670_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov7670_info *info = to_state(sd); @@ -2017,7 +2017,6 @@ static int ov7670_remove(struct i2c_client *client) v4l2_async_unregister_subdev(sd); v4l2_ctrl_handler_free(&info->hdl); media_entity_cleanup(&info->sd.entity); - return 0; } static const struct i2c_device_id ov7670_id[] = { diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c index 78602a2f70b0f891f6e81a0350409d23c1a3f921..4189e3fc3d535ef5f19a2cf5c81ff698bd956ed4 100644 --- a/drivers/media/i2c/ov772x.c +++ b/drivers/media/i2c/ov772x.c @@ -1521,7 +1521,7 @@ error_mutex_destroy: return ret; } -static int ov772x_remove(struct i2c_client *client) +static void ov772x_remove(struct i2c_client *client) { struct ov772x_priv *priv = to_ov772x(i2c_get_clientdata(client)); @@ -1532,8 +1532,6 @@ static int ov772x_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&priv->subdev); v4l2_ctrl_handler_free(&priv->hdl); mutex_destroy(&priv->lock); - - return 0; } static const struct i2c_device_id ov772x_id[] = { diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c index 2539cfee85c82772addb369d7e92a3de403b6325..c9fd9b0bc54a66dd19a75823ce77c5c14d19c959 100644 --- a/drivers/media/i2c/ov7740.c +++ b/drivers/media/i2c/ov7740.c @@ -1153,7 +1153,7 @@ error_detect: return ret; } -static int ov7740_remove(struct i2c_client *client) +static void ov7740_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); @@ -1170,7 +1170,6 @@ static int ov7740_remove(struct i2c_client *client) pm_runtime_put_noidle(&client->dev); ov7740_set_power(ov7740, 0); - return 0; } static int __maybe_unused ov7740_runtime_suspend(struct device *dev) diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c index a9728afc81d4ed3f6b495dc69e0d94abed92e76c..efa18d026ac33e743f3146afe140ab512296a1f4 100644 --- a/drivers/media/i2c/ov8856.c +++ b/drivers/media/i2c/ov8856.c @@ -2440,7 +2440,7 @@ check_hwcfg_error: return ret; } -static int ov8856_remove(struct i2c_client *client) +static void ov8856_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov8856 *ov8856 = to_ov8856(sd); @@ -2452,8 +2452,6 @@ static int ov8856_remove(struct i2c_client *client) mutex_destroy(&ov8856->mutex); __ov8856_power_off(ov8856); - - return 0; } static int ov8856_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c index b8f4f0d3e33d7ff6026276c6ad4e5b2c6d35b138..a233c34b168e77b7b05a2d6a06bf7a46f591e5f9 100644 --- a/drivers/media/i2c/ov8865.c +++ b/drivers/media/i2c/ov8865.c @@ -3119,7 +3119,7 @@ error_endpoint: return ret; } -static int ov8865_remove(struct i2c_client *client) +static void ov8865_remove(struct i2c_client *client) { struct v4l2_subdev *subdev = i2c_get_clientdata(client); struct ov8865_sensor *sensor = ov8865_subdev_sensor(subdev); @@ -3131,8 +3131,6 @@ static int ov8865_remove(struct i2c_client *client) media_entity_cleanup(&subdev->entity); v4l2_fwnode_endpoint_free(&sensor->endpoint); - - return 0; } static const struct dev_pm_ops ov8865_pm_ops = { diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c index 2e0b315801e56605bf68682007df59778d513e4e..df144a2f6eda34a959c74284f8140d508e9d8fa9 100644 --- a/drivers/media/i2c/ov9282.c +++ b/drivers/media/i2c/ov9282.c @@ -1091,7 +1091,7 @@ error_mutex_destroy: * * Return: 0 if successful, error code otherwise. */ -static int ov9282_remove(struct i2c_client *client) +static void ov9282_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov9282 *ov9282 = to_ov9282(sd); @@ -1106,8 +1106,6 @@ static int ov9282_remove(struct i2c_client *client) pm_runtime_set_suspended(&client->dev); mutex_destroy(&ov9282->mutex); - - return 0; } static const struct dev_pm_ops ov9282_pm_ops = { diff --git a/drivers/media/i2c/ov9640.c b/drivers/media/i2c/ov9640.c index 9f44ed52d16438930bc5b403b47b0290e0953771..8b80be33c5f47af2c3b6586d4360208c97065bfd 100644 --- a/drivers/media/i2c/ov9640.c +++ b/drivers/media/i2c/ov9640.c @@ -744,15 +744,13 @@ ectrlinit: return ret; } -static int ov9640_remove(struct i2c_client *client) +static void ov9640_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov9640_priv *priv = to_ov9640_sensor(sd); v4l2_async_unregister_subdev(&priv->subdev); v4l2_ctrl_handler_free(&priv->hdl); - - return 0; } static const struct i2c_device_id ov9640_id[] = { diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c index c313e11a9754ef98a84b5c7676af5ff98a61d490..4d458993e6d672f28f6e1cd09d56dd4546bd4084 100644 --- a/drivers/media/i2c/ov9650.c +++ b/drivers/media/i2c/ov9650.c @@ -1584,7 +1584,7 @@ err_mutex: return ret; } -static int ov965x_remove(struct i2c_client *client) +static void ov965x_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov965x *ov965x = to_ov965x(sd); @@ -1593,8 +1593,6 @@ static int ov965x_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); media_entity_cleanup(&sd->entity); mutex_destroy(&ov965x->lock); - - return 0; } static const struct i2c_device_id ov965x_id[] = { diff --git a/drivers/media/i2c/ov9734.c b/drivers/media/i2c/ov9734.c index df538ceb71c36c089c521a007283bec388131a2e..8b0a158cb297282d18ed1167188e54c49afaa9c5 100644 --- a/drivers/media/i2c/ov9734.c +++ b/drivers/media/i2c/ov9734.c @@ -930,7 +930,7 @@ check_hwcfg_error: return ret; } -static int ov9734_remove(struct i2c_client *client) +static void ov9734_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov9734 *ov9734 = to_ov9734(sd); @@ -940,8 +940,6 @@ static int ov9734_remove(struct i2c_client *client) v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(&client->dev); mutex_destroy(&ov9734->mutex); - - return 0; } static int ov9734_probe(struct i2c_client *client) diff --git a/drivers/media/i2c/rdacm20.c b/drivers/media/i2c/rdacm20.c index 2615ad154f498c3c766ecc1917172bc3eab475b9..a2263fa825b5905fc8cc8d23173e8d786acef893 100644 --- a/drivers/media/i2c/rdacm20.c +++ b/drivers/media/i2c/rdacm20.c @@ -646,7 +646,7 @@ error: return ret; } -static int rdacm20_remove(struct i2c_client *client) +static void rdacm20_remove(struct i2c_client *client) { struct rdacm20_device *dev = i2c_to_rdacm20(client); @@ -655,8 +655,6 @@ static int rdacm20_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&dev->ctrls); media_entity_cleanup(&dev->sd.entity); i2c_unregister_device(dev->sensor); - - return 0; } static void rdacm20_shutdown(struct i2c_client *client) diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index ef31cf5f23cac4bab0c4a7c2f315f63df4e5ebd3..9ccc56c30d3b0acde7a12cdcd9f6a4a60de8882c 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -614,7 +614,7 @@ error: return ret; } -static int rdacm21_remove(struct i2c_client *client) +static void rdacm21_remove(struct i2c_client *client) { struct rdacm21_device *dev = sd_to_rdacm21(i2c_get_clientdata(client)); @@ -622,8 +622,6 @@ static int rdacm21_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&dev->ctrls); i2c_unregister_device(dev->isp); fwnode_handle_put(dev->sd.fwnode); - - return 0; } static const struct of_device_id rdacm21_of_ids[] = { diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c index 2e4018c269124a4af6c4b77f0853482c2f88cc0d..1c3502f34cd37f2ab88cafe4d6760557d5355794 100644 --- a/drivers/media/i2c/rj54n1cb0c.c +++ b/drivers/media/i2c/rj54n1cb0c.c @@ -1398,7 +1398,7 @@ err_free_ctrl: return ret; } -static int rj54n1_remove(struct i2c_client *client) +static void rj54n1_remove(struct i2c_client *client) { struct rj54n1 *rj54n1 = to_rj54n1(client); @@ -1410,8 +1410,6 @@ static int rj54n1_remove(struct i2c_client *client) clk_put(rj54n1->clk); v4l2_ctrl_handler_free(&rj54n1->hdl); v4l2_async_unregister_subdev(&rj54n1->subdev); - - return 0; } static const struct i2c_device_id rj54n1_id[] = { diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c index e2b88c5e4f9838af0630b396dd40743e57e0d55a..d96ba58ce1e5350bf56342e7330354f7917fbe72 100644 --- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c +++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c @@ -1770,7 +1770,7 @@ out_err: return ret; } -static int s5c73m3_remove(struct i2c_client *client) +static void s5c73m3_remove(struct i2c_client *client) { struct v4l2_subdev *oif_sd = i2c_get_clientdata(client); struct s5c73m3 *state = oif_sd_to_s5c73m3(oif_sd); @@ -1785,8 +1785,6 @@ static int s5c73m3_remove(struct i2c_client *client) media_entity_cleanup(&sensor_sd->entity); s5c73m3_unregister_spi_driver(state); - - return 0; } static const struct i2c_device_id s5c73m3_id[] = { diff --git a/drivers/media/i2c/s5k4ecgx.c b/drivers/media/i2c/s5k4ecgx.c index af9a305242cd0b4ce53fd60780a549e344fd6aec..3dddcd9dd351c3af7b240ef7c086e35039489446 100644 --- a/drivers/media/i2c/s5k4ecgx.c +++ b/drivers/media/i2c/s5k4ecgx.c @@ -996,7 +996,7 @@ out_err1: return ret; } -static int s5k4ecgx_remove(struct i2c_client *client) +static void s5k4ecgx_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct s5k4ecgx *priv = to_s5k4ecgx(sd); @@ -1006,8 +1006,6 @@ static int s5k4ecgx_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&priv->handler); media_entity_cleanup(&sd->entity); - - return 0; } static const struct i2c_device_id s5k4ecgx_id[] = { diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c index 6a5dceb699a8804a04f656f22f8770e8815347cc..5c2253ab3b6f9866d006031890e532323eebb68b 100644 --- a/drivers/media/i2c/s5k5baf.c +++ b/drivers/media/i2c/s5k5baf.c @@ -2018,7 +2018,7 @@ err_me: return ret; } -static int s5k5baf_remove(struct i2c_client *c) +static void s5k5baf_remove(struct i2c_client *c) { struct v4l2_subdev *sd = i2c_get_clientdata(c); struct s5k5baf *state = to_s5k5baf(sd); @@ -2030,8 +2030,6 @@ static int s5k5baf_remove(struct i2c_client *c) sd = &state->cis_sd; v4l2_device_unregister_subdev(sd); media_entity_cleanup(&sd->entity); - - return 0; } static const struct i2c_device_id s5k5baf_id[] = { diff --git a/drivers/media/i2c/s5k6a3.c b/drivers/media/i2c/s5k6a3.c index f6ecf6f92bb230dd5f6fd8a19338c2d312a6c1f8..a4efd6d10b43d09907df896c0299edbd111945c7 100644 --- a/drivers/media/i2c/s5k6a3.c +++ b/drivers/media/i2c/s5k6a3.c @@ -354,14 +354,13 @@ static int s5k6a3_probe(struct i2c_client *client) return ret; } -static int s5k6a3_remove(struct i2c_client *client) +static void s5k6a3_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); pm_runtime_disable(&client->dev); v4l2_async_unregister_subdev(sd); media_entity_cleanup(&sd->entity); - return 0; } static const struct i2c_device_id s5k6a3_ids[] = { diff --git a/drivers/media/i2c/s5k6aa.c b/drivers/media/i2c/s5k6aa.c index 105a4b7d8354ba53cd732aba7a9c4341561cebef..059211788a65078f70895d2f9a57935a1855be59 100644 --- a/drivers/media/i2c/s5k6aa.c +++ b/drivers/media/i2c/s5k6aa.c @@ -1621,15 +1621,13 @@ out_err: return ret; } -static int s5k6aa_remove(struct i2c_client *client) +static void s5k6aa_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); media_entity_cleanup(&sd->entity); - - return 0; } static const struct i2c_device_id s5k6aa_id[] = { diff --git a/drivers/media/i2c/saa6588.c b/drivers/media/i2c/saa6588.c index d1e0716bdfffde5e26fb61993491edd6059eed34..d6a51beabd02a8892f25334ec32258fb4f9fad8d 100644 --- a/drivers/media/i2c/saa6588.c +++ b/drivers/media/i2c/saa6588.c @@ -484,7 +484,7 @@ static int saa6588_probe(struct i2c_client *client, return 0; } -static int saa6588_remove(struct i2c_client *client) +static void saa6588_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct saa6588 *s = to_saa6588(sd); @@ -492,8 +492,6 @@ static int saa6588_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); cancel_delayed_work_sync(&s->work); - - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c index a7f043cad149b0bf07f1cd26f280efa8db80a15e..5928cc6f45956e2256fb8f5794b381f61e7a3acf 100644 --- a/drivers/media/i2c/saa6752hs.c +++ b/drivers/media/i2c/saa6752hs.c @@ -764,13 +764,12 @@ static int saa6752hs_probe(struct i2c_client *client, return 0; } -static int saa6752hs_remove(struct i2c_client *client) +static void saa6752hs_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&to_state(sd)->hdl); - return 0; } static const struct i2c_device_id saa6752hs_id[] = { diff --git a/drivers/media/i2c/saa7110.c b/drivers/media/i2c/saa7110.c index 0c7a9ce0a693e2e0447d5c334673c09eb3e82636..5067525d8b1118d977f98719cd8bca83991c9442 100644 --- a/drivers/media/i2c/saa7110.c +++ b/drivers/media/i2c/saa7110.c @@ -428,14 +428,13 @@ static int saa7110_probe(struct i2c_client *client, return 0; } -static int saa7110_remove(struct i2c_client *client) +static void saa7110_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct saa7110 *decoder = to_saa7110(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&decoder->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c index 15ff80e6301eed668a15a4dc6004779a8057625b..86e70a980218e0f82caa70239d92b5a028a33499 100644 --- a/drivers/media/i2c/saa7115.c +++ b/drivers/media/i2c/saa7115.c @@ -1927,13 +1927,12 @@ static int saa711x_probe(struct i2c_client *client, /* ----------------------------------------------------------------------- */ -static int saa711x_remove(struct i2c_client *client) +static void saa711x_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } static const struct i2c_device_id saa711x_id[] = { diff --git a/drivers/media/i2c/saa7127.c b/drivers/media/i2c/saa7127.c index 891192f6412a9820a8a1f4976fe6be98a4a35a65..78c9388c2ea116382d74ec99dcff97cfeec11239 100644 --- a/drivers/media/i2c/saa7127.c +++ b/drivers/media/i2c/saa7127.c @@ -785,14 +785,13 @@ static int saa7127_probe(struct i2c_client *client, /* ----------------------------------------------------------------------- */ -static int saa7127_remove(struct i2c_client *client) +static void saa7127_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); /* Turn off TV output */ saa7127_set_video_enable(sd, 0); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c index adf905360171376e4cc70ddc2056fca07275a41b..4f3d1b432a4ecac9b4736ae2d3071717d2c499bc 100644 --- a/drivers/media/i2c/saa717x.c +++ b/drivers/media/i2c/saa717x.c @@ -1324,13 +1324,12 @@ static int saa717x_probe(struct i2c_client *client, return 0; } -static int saa717x_remove(struct i2c_client *client) +static void saa717x_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/saa7185.c b/drivers/media/i2c/saa7185.c index 7a04422df8c8fc6268ad66d5b86481075b5c4169..266462325d30efe72a29c853b0206faf36a3a20d 100644 --- a/drivers/media/i2c/saa7185.c +++ b/drivers/media/i2c/saa7185.c @@ -322,7 +322,7 @@ static int saa7185_probe(struct i2c_client *client, return 0; } -static int saa7185_remove(struct i2c_client *client) +static void saa7185_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct saa7185 *encoder = to_saa7185(sd); @@ -330,7 +330,6 @@ static int saa7185_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); /* SW: output off is active */ saa7185_write(sd, 0x61, (encoder->reg[0x61]) | 0x40); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/sony-btf-mpx.c b/drivers/media/i2c/sony-btf-mpx.c index ad239280c42eeed31264c231c31b9b7a7d9bf6f2..927a9ec41463d30c4c77a40de4f9bb4b61a92cfc 100644 --- a/drivers/media/i2c/sony-btf-mpx.c +++ b/drivers/media/i2c/sony-btf-mpx.c @@ -357,13 +357,11 @@ static int sony_btf_mpx_probe(struct i2c_client *client, return 0; } -static int sony_btf_mpx_remove(struct i2c_client *client) +static void sony_btf_mpx_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/sr030pc30.c b/drivers/media/i2c/sr030pc30.c index 19c0252df2f1e3a2e8b5670c23686f515141c69e..ff18693beb5c2128b3fd7e0cf61d97f85f85e16c 100644 --- a/drivers/media/i2c/sr030pc30.c +++ b/drivers/media/i2c/sr030pc30.c @@ -732,13 +732,12 @@ static int sr030pc30_probe(struct i2c_client *client, return 0; } -static int sr030pc30_remove(struct i2c_client *client) +static void sr030pc30_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } static const struct i2c_device_id sr030pc30_id[] = { diff --git a/drivers/media/i2c/st-mipid02.c b/drivers/media/i2c/st-mipid02.c index 16cc547976dd5f6bcd294a13a8b8054b3ed10c99..31b89aff0e86a0621ec9fba146f18ed35c147abc 100644 --- a/drivers/media/i2c/st-mipid02.c +++ b/drivers/media/i2c/st-mipid02.c @@ -1067,7 +1067,7 @@ mutex_cleanup: return ret; } -static int mipid02_remove(struct i2c_client *client) +static void mipid02_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct mipid02_dev *bridge = to_mipid02_dev(sd); @@ -1078,8 +1078,6 @@ static int mipid02_remove(struct i2c_client *client) mipid02_set_power_off(bridge); media_entity_cleanup(&bridge->sd.entity); mutex_destroy(&bridge->lock); - - return 0; } static const struct of_device_id mipid02_dt_ids[] = { diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c index e18b8947ad7e5f2d26320b75bf8c68fe0d33066a..200841c1f5cf08a6e501b2af46c88e52420e27de 100644 --- a/drivers/media/i2c/tc358743.c +++ b/drivers/media/i2c/tc358743.c @@ -964,6 +964,8 @@ static void tc358743_cec_handler(struct v4l2_subdev *sd, u16 intstatus, v = i2c_rd32(sd, CECRCTR); msg.len = v & 0x1f; + if (msg.len > CEC_MAX_MSG_SIZE) + msg.len = CEC_MAX_MSG_SIZE; for (i = 0; i < msg.len; i++) { v = i2c_rd32(sd, CECRBUF1 + i * 4); msg.msg[i] = v & 0xff; @@ -2169,7 +2171,7 @@ err_hdl: return err; } -static int tc358743_remove(struct i2c_client *client) +static void tc358743_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tc358743_state *state = to_state(sd); @@ -2185,8 +2187,6 @@ static int tc358743_remove(struct i2c_client *client) mutex_destroy(&state->confctl_mutex); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(&state->hdl); - - return 0; } static const struct i2c_device_id tc358743_id[] = { diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c index f66ac14cffad5971d7dd9d05fd2ced2af9acfebb..83931826cf6f532bbbd2a5cc3e438b64e70e6a8b 100644 --- a/drivers/media/i2c/tda1997x.c +++ b/drivers/media/i2c/tda1997x.c @@ -2805,7 +2805,7 @@ err_free_state: return ret; } -static int tda1997x_remove(struct i2c_client *client) +static void tda1997x_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tda1997x_state *state = to_state(sd); @@ -2827,8 +2827,6 @@ static int tda1997x_remove(struct i2c_client *client) mutex_destroy(&state->lock); kfree(state); - - return 0; } static struct i2c_driver tda1997x_i2c_driver = { diff --git a/drivers/media/i2c/tda7432.c b/drivers/media/i2c/tda7432.c index cbdc9be0a5973924f264585a084166d970266371..11e918311b131f62f95dec221716452b6199e7b6 100644 --- a/drivers/media/i2c/tda7432.c +++ b/drivers/media/i2c/tda7432.c @@ -390,7 +390,7 @@ static int tda7432_probe(struct i2c_client *client, return 0; } -static int tda7432_remove(struct i2c_client *client) +static void tda7432_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tda7432 *t = to_state(sd); @@ -398,7 +398,6 @@ static int tda7432_remove(struct i2c_client *client) tda7432_set(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&t->hdl); - return 0; } static const struct i2c_device_id tda7432_id[] = { diff --git a/drivers/media/i2c/tda9840.c b/drivers/media/i2c/tda9840.c index 8c6dfe746b206c5890b043826926273249c53e14..aaa74944fc7cd335bf8c551cbbe6519c5d178446 100644 --- a/drivers/media/i2c/tda9840.c +++ b/drivers/media/i2c/tda9840.c @@ -175,12 +175,11 @@ static int tda9840_probe(struct i2c_client *client, return 0; } -static int tda9840_remove(struct i2c_client *client) +static void tda9840_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id tda9840_id[] = { diff --git a/drivers/media/i2c/tea6415c.c b/drivers/media/i2c/tea6415c.c index 67378dbcc74b986e69b81ff32c765901698f957f..50e74314f315dd1f972fb46f5298c5aaa05326fc 100644 --- a/drivers/media/i2c/tea6415c.c +++ b/drivers/media/i2c/tea6415c.c @@ -134,12 +134,11 @@ static int tea6415c_probe(struct i2c_client *client, return 0; } -static int tea6415c_remove(struct i2c_client *client) +static void tea6415c_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id tea6415c_id[] = { diff --git a/drivers/media/i2c/tea6420.c b/drivers/media/i2c/tea6420.c index 712141b261ed63faf9466d619370477ea31212f9..246f2b10ccc74cf36664d1da762285d27714ce9c 100644 --- a/drivers/media/i2c/tea6420.c +++ b/drivers/media/i2c/tea6420.c @@ -116,12 +116,11 @@ static int tea6420_probe(struct i2c_client *client, return 0; } -static int tea6420_remove(struct i2c_client *client) +static void tea6420_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id tea6420_id[] = { diff --git a/drivers/media/i2c/ths7303.c b/drivers/media/i2c/ths7303.c index 8206bf7a5a8fa4b3e6950465ae9b83dc85afe928..2a0f9a3d1a66bb55fee2deceae2eed79bf4ad54d 100644 --- a/drivers/media/i2c/ths7303.c +++ b/drivers/media/i2c/ths7303.c @@ -358,13 +358,11 @@ static int ths7303_probe(struct i2c_client *client, return 0; } -static int ths7303_remove(struct i2c_client *client) +static void ths7303_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - - return 0; } static const struct i2c_device_id ths7303_id[] = { diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c index c52fe84cba1bd040ea7b86ab0fc0dfa7e8a39645..081ef5a4b950b9206be7fd20ac166384feb27d39 100644 --- a/drivers/media/i2c/ths8200.c +++ b/drivers/media/i2c/ths8200.c @@ -468,7 +468,7 @@ static int ths8200_probe(struct i2c_client *client) return 0; } -static int ths8200_remove(struct i2c_client *client) +static void ths8200_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ths8200_state *decoder = to_state(sd); @@ -478,8 +478,6 @@ static int ths8200_remove(struct i2c_client *client) ths8200_s_power(sd, false); v4l2_async_unregister_subdev(&decoder->sd); - - return 0; } static const struct i2c_device_id ths8200_id[] = { diff --git a/drivers/media/i2c/tlv320aic23b.c b/drivers/media/i2c/tlv320aic23b.c index e4c21990fea99dcf7e166f04a4b1601666bf045c..937fa1dbaecba259a3a2e679454a46f59b8e2cca 100644 --- a/drivers/media/i2c/tlv320aic23b.c +++ b/drivers/media/i2c/tlv320aic23b.c @@ -177,14 +177,13 @@ static int tlv320aic23b_probe(struct i2c_client *client, return 0; } -static int tlv320aic23b_remove(struct i2c_client *client) +static void tlv320aic23b_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tlv320aic23b_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/tvaudio.c b/drivers/media/i2c/tvaudio.c index e6796e94dadfbe098f6859e6c4c7bf7584c5c497..9f1ed078b6616e0b85a23385fd2f0b97d72e64fc 100644 --- a/drivers/media/i2c/tvaudio.c +++ b/drivers/media/i2c/tvaudio.c @@ -2065,7 +2065,7 @@ static int tvaudio_probe(struct i2c_client *client, const struct i2c_device_id * return 0; } -static int tvaudio_remove(struct i2c_client *client) +static void tvaudio_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct CHIPSTATE *chip = to_state(sd); @@ -2079,7 +2079,6 @@ static int tvaudio_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&chip->hdl); - return 0; } /* This driver supports many devices and the idea is to let the driver diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c index cee60f94503676ab21217726ea99dc5c880bb1c3..a746d96875f90c8141a0eb8b6d7980fa6e3746ad 100644 --- a/drivers/media/i2c/tvp514x.c +++ b/drivers/media/i2c/tvp514x.c @@ -1121,7 +1121,7 @@ done: * Unregister decoder as an i2c client device and V4L2 * device. Complement of tvp514x_probe(). */ -static int tvp514x_remove(struct i2c_client *client) +static void tvp514x_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tvp514x_decoder *decoder = to_decoder(sd); @@ -1129,7 +1129,6 @@ static int tvp514x_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&decoder->sd); media_entity_cleanup(&decoder->sd.entity); v4l2_ctrl_handler_free(&decoder->hdl); - return 0; } /* TVP5146 Init/Power on Sequence */ static const struct tvp514x_reg tvp5146_init_reg_seq[] = { diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c index 93a980c4e8990403c35345d7e155910cf3752b64..859f1cb2fa7446b2df255409b37ded90ff8bbabe 100644 --- a/drivers/media/i2c/tvp5150.c +++ b/drivers/media/i2c/tvp5150.c @@ -2230,7 +2230,7 @@ err: return res; } -static int tvp5150_remove(struct i2c_client *c) +static void tvp5150_remove(struct i2c_client *c) { struct v4l2_subdev *sd = i2c_get_clientdata(c); struct tvp5150 *decoder = to_tvp5150(sd); @@ -2250,8 +2250,6 @@ static int tvp5150_remove(struct i2c_client *c) v4l2_ctrl_handler_free(&decoder->hdl); pm_runtime_disable(&c->dev); pm_runtime_set_suspended(&c->dev); - - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c index 2de18833b07b4bd22944d386969b566ab64297af..4ccd218f5584e37cbca1ce740f470464d40dec7d 100644 --- a/drivers/media/i2c/tvp7002.c +++ b/drivers/media/i2c/tvp7002.c @@ -1044,7 +1044,7 @@ error: * Reset the TVP7002 device * Returns zero. */ -static int tvp7002_remove(struct i2c_client *c) +static void tvp7002_remove(struct i2c_client *c) { struct v4l2_subdev *sd = i2c_get_clientdata(c); struct tvp7002 *device = to_tvp7002(sd); @@ -1056,7 +1056,6 @@ static int tvp7002_remove(struct i2c_client *c) media_entity_cleanup(&device->sd.entity); #endif v4l2_ctrl_handler_free(&device->hdl); - return 0; } /* I2C Device ID table */ diff --git a/drivers/media/i2c/tw2804.c b/drivers/media/i2c/tw2804.c index cd05f1ff504d124fec703bd75cd5d245ea5f00b6..c7c8dfe8a8a84d4fafc502bca1bff78dbc033966 100644 --- a/drivers/media/i2c/tw2804.c +++ b/drivers/media/i2c/tw2804.c @@ -405,14 +405,13 @@ static int tw2804_probe(struct i2c_client *client, return 0; } -static int tw2804_remove(struct i2c_client *client) +static void tw2804_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct tw2804 *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } static const struct i2c_device_id tw2804_id[] = { diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c index f8e3ab4909d8bc711ba23e284370f7a35cbddffc..d7eef7986b75d039bf347874304568f102d2c6f5 100644 --- a/drivers/media/i2c/tw9903.c +++ b/drivers/media/i2c/tw9903.c @@ -235,13 +235,12 @@ static int tw9903_probe(struct i2c_client *client, return 0; } -static int tw9903_remove(struct i2c_client *client) +static void tw9903_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&to_state(sd)->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c index c528eb01fed0e389c64a3d704eb5306f25b40b7a..549ad8f72f12b7e3aa9371017953bf59010e6fba 100644 --- a/drivers/media/i2c/tw9906.c +++ b/drivers/media/i2c/tw9906.c @@ -203,13 +203,12 @@ static int tw9906_probe(struct i2c_client *client, return 0; } -static int tw9906_remove(struct i2c_client *client) +static void tw9906_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&to_state(sd)->hdl); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/tw9910.c b/drivers/media/i2c/tw9910.c index 09f5b398692849239162d08c05f9255c034e787f..853b5acead32bbaf851f109fa24b557abcfd52d4 100644 --- a/drivers/media/i2c/tw9910.c +++ b/drivers/media/i2c/tw9910.c @@ -993,7 +993,7 @@ error_clk_put: return ret; } -static int tw9910_remove(struct i2c_client *client) +static void tw9910_remove(struct i2c_client *client) { struct tw9910_priv *priv = to_tw9910(client); @@ -1001,8 +1001,6 @@ static int tw9910_remove(struct i2c_client *client) gpiod_put(priv->pdn_gpio); clk_put(priv->clk); v4l2_async_unregister_subdev(&priv->subdev); - - return 0; } static const struct i2c_device_id tw9910_id[] = { diff --git a/drivers/media/i2c/uda1342.c b/drivers/media/i2c/uda1342.c index b0a9c6d7163fe21197b96aaab92c4bf688022b4f..d0659c4392f283a79d9301ffc474456d7e71d0c0 100644 --- a/drivers/media/i2c/uda1342.c +++ b/drivers/media/i2c/uda1342.c @@ -72,12 +72,11 @@ static int uda1342_probe(struct i2c_client *client, return 0; } -static int uda1342_remove(struct i2c_client *client) +static void uda1342_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } static const struct i2c_device_id uda1342_id[] = { diff --git a/drivers/media/i2c/upd64031a.c b/drivers/media/i2c/upd64031a.c index ef35c65747853700fc709ffab60b7976494289eb..4de26ed2ba003dcf0a36d760382785e0721f60a8 100644 --- a/drivers/media/i2c/upd64031a.c +++ b/drivers/media/i2c/upd64031a.c @@ -210,12 +210,11 @@ static int upd64031a_probe(struct i2c_client *client, return 0; } -static int upd64031a_remove(struct i2c_client *client) +static void upd64031a_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/upd64083.c b/drivers/media/i2c/upd64083.c index d6a1698caa2a0e74491ecf35bfafe01910f0cb0c..2bfd5443d40683fa16ef8753908575e8530771d9 100644 --- a/drivers/media/i2c/upd64083.c +++ b/drivers/media/i2c/upd64083.c @@ -181,12 +181,11 @@ static int upd64083_probe(struct i2c_client *client, return 0; } -static int upd64083_remove(struct i2c_client *client) +static void upd64083_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c index e08e3579c0a1533eea5197e7afc28fea8ed8a257..f15ef2d13059012efee90cc612ee6f7e5a6ed046 100644 --- a/drivers/media/i2c/video-i2c.c +++ b/drivers/media/i2c/video-i2c.c @@ -895,7 +895,7 @@ error_free_device: return ret; } -static int video_i2c_remove(struct i2c_client *client) +static void video_i2c_remove(struct i2c_client *client) { struct video_i2c_data *data = i2c_get_clientdata(client); @@ -908,8 +908,6 @@ static int video_i2c_remove(struct i2c_client *client) data->chip->set_power(data, false); video_unregister_device(&data->vdev); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/media/i2c/vp27smpx.c b/drivers/media/i2c/vp27smpx.c index 492af8749fca78a7502a5393d00af031f8982ff3..c832edad5fa7844174959f3a5c898bb0ed0973a6 100644 --- a/drivers/media/i2c/vp27smpx.c +++ b/drivers/media/i2c/vp27smpx.c @@ -163,12 +163,11 @@ static int vp27smpx_probe(struct i2c_client *client, return 0; } -static int vp27smpx_remove(struct i2c_client *client) +static void vp27smpx_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); - return 0; } /* ----------------------------------------------------------------------- */ diff --git a/drivers/media/i2c/vpx3220.c b/drivers/media/i2c/vpx3220.c index 8be03fe5928cfb1f8e3b0f2f00e37b3b24def1b6..b481ec196b880976affd6dca2dcb6c1a3227b7cd 100644 --- a/drivers/media/i2c/vpx3220.c +++ b/drivers/media/i2c/vpx3220.c @@ -526,15 +526,13 @@ static int vpx3220_probe(struct i2c_client *client, return 0; } -static int vpx3220_remove(struct i2c_client *client) +static void vpx3220_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct vpx3220 *decoder = to_vpx3220(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&decoder->hdl); - - return 0; } static const struct i2c_device_id vpx3220_id[] = { diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c index 29003dec6f2dad893718ee7dcf01b0750e3e1d0a..d496bb45f2015a1825083f225f854f10578cdfb0 100644 --- a/drivers/media/i2c/vs6624.c +++ b/drivers/media/i2c/vs6624.c @@ -824,13 +824,12 @@ static int vs6624_probe(struct i2c_client *client, return ret; } -static int vs6624_remove(struct i2c_client *client) +static void vs6624_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - return 0; } static const struct i2c_device_id vs6624_id[] = { diff --git a/drivers/media/i2c/wm8739.c b/drivers/media/i2c/wm8739.c index ed533834db5474f2d244f00aa595fd27c346bd42..180b3534752130b79e11cf9d51016264ed9a3f4e 100644 --- a/drivers/media/i2c/wm8739.c +++ b/drivers/media/i2c/wm8739.c @@ -234,14 +234,13 @@ static int wm8739_probe(struct i2c_client *client, return 0; } -static int wm8739_remove(struct i2c_client *client) +static void wm8739_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct wm8739_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } static const struct i2c_device_id wm8739_id[] = { diff --git a/drivers/media/i2c/wm8775.c b/drivers/media/i2c/wm8775.c index d4c83c39892a1ccc23ce7b55128e11506e3b145b..8ff97867d3cdd9917228b44278e3bd38bdb16567 100644 --- a/drivers/media/i2c/wm8775.c +++ b/drivers/media/i2c/wm8775.c @@ -280,14 +280,13 @@ static int wm8775_probe(struct i2c_client *client, return 0; } -static int wm8775_remove(struct i2c_client *client) +static void wm8775_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct wm8775_state *state = to_state(sd); v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); - return 0; } static const struct i2c_device_id wm8775_id[] = { diff --git a/drivers/media/pci/Kconfig b/drivers/media/pci/Kconfig index 1224d908713aa0d83eb2a83ef5e500b2866721c8..dff0b450f38758c91d3aa1ce7d68afb9f66de3f2 100644 --- a/drivers/media/pci/Kconfig +++ b/drivers/media/pci/Kconfig @@ -13,12 +13,12 @@ if MEDIA_PCI_SUPPORT if MEDIA_CAMERA_SUPPORT comment "Media capture support" -source "drivers/media/pci/meye/Kconfig" source "drivers/media/pci/solo6x10/Kconfig" source "drivers/media/pci/sta2x11/Kconfig" source "drivers/media/pci/tw5864/Kconfig" source "drivers/media/pci/tw68/Kconfig" source "drivers/media/pci/tw686x/Kconfig" +source "drivers/media/pci/zoran/Kconfig" endif @@ -27,7 +27,6 @@ if MEDIA_ANALOG_TV_SUPPORT source "drivers/media/pci/dt3155/Kconfig" source "drivers/media/pci/ivtv/Kconfig" -source "drivers/media/pci/saa7146/Kconfig" endif @@ -58,7 +57,6 @@ source "drivers/media/pci/pluto2/Kconfig" source "drivers/media/pci/pt1/Kconfig" source "drivers/media/pci/pt3/Kconfig" source "drivers/media/pci/smipcie/Kconfig" -source "drivers/media/pci/ttpci/Kconfig" endif diff --git a/drivers/media/pci/Makefile b/drivers/media/pci/Makefile index 551169a3e434fe9656c973c83cb3de206d58935a..8f887a8a7f179e4f5d712bacff6b35190cd0b401 100644 --- a/drivers/media/pci/Makefile +++ b/drivers/media/pci/Makefile @@ -5,8 +5,7 @@ # Please keep it alphabetically sorted by directory # (e. g. LC_ALL=C sort Makefile) -obj-y += ttpci/ \ - b2c2/ \ +obj-y += b2c2/ \ pluto2/ \ dm1105/ \ pt1/ \ @@ -14,7 +13,6 @@ obj-y += ttpci/ \ mantis/ \ ngene/ \ ddbridge/ \ - saa7146/ \ smipcie/ \ netup_unidvb/ \ intel/ @@ -32,10 +30,10 @@ obj-$(CONFIG_VIDEO_CX25821) += cx25821/ obj-$(CONFIG_VIDEO_CX88) += cx88/ obj-$(CONFIG_VIDEO_DT3155) += dt3155/ obj-$(CONFIG_VIDEO_IVTV) += ivtv/ -obj-$(CONFIG_VIDEO_MEYE) += meye/ obj-$(CONFIG_VIDEO_SAA7134) += saa7134/ obj-$(CONFIG_VIDEO_SAA7164) += saa7164/ obj-$(CONFIG_VIDEO_SOLO6X10) += solo6x10/ obj-$(CONFIG_VIDEO_TW5864) += tw5864/ obj-$(CONFIG_VIDEO_TW686X) += tw686x/ obj-$(CONFIG_VIDEO_TW68) += tw68/ +obj-$(CONFIG_VIDEO_ZORAN) += zoran/ diff --git a/drivers/media/pci/cx18/cx18-av-audio.c b/drivers/media/pci/cx18/cx18-av-audio.c index 833baa9344480113a93f491782f0b0db88d69e9d..78e05df9a7bacf4046ade51ba993e29f4cd50e57 100644 --- a/drivers/media/pci/cx18/cx18-av-audio.c +++ b/drivers/media/pci/cx18/cx18-av-audio.c @@ -50,7 +50,7 @@ static int set_audclk_freq(struct cx18 *cx, u32 freq) * * Many thanks to Jeff Campbell and Mike Bradley for their extensive * investigation, experimentation, testing, and suggested solutions of - * of audio/video sync problems with SVideo and CVBS captures. + * audio/video sync problems with SVideo and CVBS captures. */ if (state->aud_input > CX18_AV_AUDIO_SERIAL2) { diff --git a/drivers/media/pci/cx18/cx18-firmware.c b/drivers/media/pci/cx18/cx18-firmware.c index fdac310d74775de52542db0771c318ce88a045c9..1b038b2802bf5c50df91d8e1c5c97ecbc1153ec8 100644 --- a/drivers/media/pci/cx18/cx18-firmware.c +++ b/drivers/media/pci/cx18/cx18-firmware.c @@ -248,7 +248,7 @@ void cx18_init_power(struct cx18 *cx, int lowpwr) * * Many thanks to Jeff Campbell and Mike Bradley for their extensive * investigation, experimentation, testing, and suggested solutions of - * of audio/video sync problems with SVideo and CVBS captures. + * audio/video sync problems with SVideo and CVBS captures. */ /* the fast clock is at 200/245 MHz */ diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c index a07b18f2034e9c8748384019f3357559429713e5..9232a966bcabb2f8b7ec1b472cd59c5ee67a4364 100644 --- a/drivers/media/pci/cx23885/cx23885-core.c +++ b/drivers/media/pci/cx23885/cx23885-core.c @@ -2086,6 +2086,9 @@ static struct { /* 0x1419 is the PCI ID for the IOMMU found on 15h (Models 10h-1fh) family */ { PCI_VENDOR_ID_AMD, 0x1419 }, + /* 0x1631 is the PCI ID for the IOMMU found on Renoir/Cezanne + */ + { PCI_VENDOR_ID_AMD, 0x1631 }, /* 0x5a23 is the PCI ID for the IOMMU found on RD890S/RD990 */ { PCI_VENDOR_ID_ATI, 0x5a23 }, diff --git a/drivers/media/pci/cx23885/cx23888-ir.c b/drivers/media/pci/cx23885/cx23888-ir.c index ddfd2eb37484d624924f94e7ab6a325373844bd1..222d04421468e998dd02fdcf0784bce9fb7f2794 100644 --- a/drivers/media/pci/cx23885/cx23888-ir.c +++ b/drivers/media/pci/cx23885/cx23888-ir.c @@ -235,7 +235,7 @@ static u32 clock_divider_to_resolution(u16 divider) { /* * Resolution is the duration of 1 tick of the readable portion of - * of the pulse width counter as read from the FIFO. The two lsb's are + * the pulse width counter as read from the FIFO. The two lsb's are * not readable, hence the << 2. This function returns ns. */ return DIV_ROUND_CLOSEST((1 << 2) * ((u32) divider + 1) * 1000, diff --git a/drivers/media/pci/cx88/cx88-dsp.c b/drivers/media/pci/cx88/cx88-dsp.c index f1e1fc1cb4bd7b9760f488c6386e7042bc55a880..e378f3b215c737ce313694a29251ffb30eb48da2 100644 --- a/drivers/media/pci/cx88/cx88-dsp.c +++ b/drivers/media/pci/cx88/cx88-dsp.c @@ -24,7 +24,7 @@ /* * We calculate the baseband frequencies of the carrier and the pilot tones - * based on the the sampling rate of the audio rds fifo. + * based on the sampling rate of the audio rds fifo. */ #define FREQ_A2_CARRIER baseband_freq(54687.5, 2689.36, 0.0) diff --git a/drivers/media/pci/cx88/cx88-vbi.c b/drivers/media/pci/cx88/cx88-vbi.c index a075788c64d45d12dd6173abd10eea570e6b2004..469aeaa725ad9259122f407edf71006f5e324b4d 100644 --- a/drivers/media/pci/cx88/cx88-vbi.c +++ b/drivers/media/pci/cx88/cx88-vbi.c @@ -144,11 +144,10 @@ static int buffer_prepare(struct vb2_buffer *vb) return -EINVAL; vb2_set_plane_payload(vb, 0, size); - cx88_risc_buffer(dev->pci, &buf->risc, sgt->sgl, - 0, VBI_LINE_LENGTH * lines, - VBI_LINE_LENGTH, 0, - lines); - return 0; + return cx88_risc_buffer(dev->pci, &buf->risc, sgt->sgl, + 0, VBI_LINE_LENGTH * lines, + VBI_LINE_LENGTH, 0, + lines); } static void buffer_finish(struct vb2_buffer *vb) diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c index d3729be8925294e1b611509b5e884b770ec04cf1..b509c2a03852bde725973cfed347b5f73445d4d7 100644 --- a/drivers/media/pci/cx88/cx88-video.c +++ b/drivers/media/pci/cx88/cx88-video.c @@ -431,6 +431,7 @@ static int queue_setup(struct vb2_queue *q, static int buffer_prepare(struct vb2_buffer *vb) { + int ret; struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); struct cx8800_dev *dev = vb->vb2_queue->drv_priv; struct cx88_core *core = dev->core; @@ -445,35 +446,35 @@ static int buffer_prepare(struct vb2_buffer *vb) switch (core->field) { case V4L2_FIELD_TOP: - cx88_risc_buffer(dev->pci, &buf->risc, - sgt->sgl, 0, UNSET, - buf->bpl, 0, core->height); + ret = cx88_risc_buffer(dev->pci, &buf->risc, + sgt->sgl, 0, UNSET, + buf->bpl, 0, core->height); break; case V4L2_FIELD_BOTTOM: - cx88_risc_buffer(dev->pci, &buf->risc, - sgt->sgl, UNSET, 0, - buf->bpl, 0, core->height); + ret = cx88_risc_buffer(dev->pci, &buf->risc, + sgt->sgl, UNSET, 0, + buf->bpl, 0, core->height); break; case V4L2_FIELD_SEQ_TB: - cx88_risc_buffer(dev->pci, &buf->risc, - sgt->sgl, - 0, buf->bpl * (core->height >> 1), - buf->bpl, 0, - core->height >> 1); + ret = cx88_risc_buffer(dev->pci, &buf->risc, + sgt->sgl, + 0, buf->bpl * (core->height >> 1), + buf->bpl, 0, + core->height >> 1); break; case V4L2_FIELD_SEQ_BT: - cx88_risc_buffer(dev->pci, &buf->risc, - sgt->sgl, - buf->bpl * (core->height >> 1), 0, - buf->bpl, 0, - core->height >> 1); + ret = cx88_risc_buffer(dev->pci, &buf->risc, + sgt->sgl, + buf->bpl * (core->height >> 1), 0, + buf->bpl, 0, + core->height >> 1); break; case V4L2_FIELD_INTERLACED: default: - cx88_risc_buffer(dev->pci, &buf->risc, - sgt->sgl, 0, buf->bpl, - buf->bpl, buf->bpl, - core->height >> 1); + ret = cx88_risc_buffer(dev->pci, &buf->risc, + sgt->sgl, 0, buf->bpl, + buf->bpl, buf->bpl, + core->height >> 1); break; } dprintk(2, @@ -481,7 +482,7 @@ static int buffer_prepare(struct vb2_buffer *vb) buf, buf->vb.vb2_buf.index, __func__, core->width, core->height, dev->fmt->depth, dev->fmt->fourcc, (unsigned long)buf->risc.dma); - return 0; + return ret; } static void buffer_finish(struct vb2_buffer *vb) diff --git a/drivers/media/pci/ivtv/ivtv-yuv.c b/drivers/media/pci/ivtv/ivtv-yuv.c index e79e8a5a744adbbd10e81eae114741054cea65a4..4ba10c34a16a4283d4a9b1c9f82893d3e6dffaf7 100644 --- a/drivers/media/pci/ivtv/ivtv-yuv.c +++ b/drivers/media/pci/ivtv/ivtv-yuv.c @@ -538,7 +538,7 @@ static void ivtv_yuv_handle_vertical(struct ivtv *itv, struct yuv_frame_info *f) reg_2964 = (reg_2964 << 16) + reg_2964 + (reg_2964 * 46 / 94); /* Okay, we've wasted time working out the correct value, - but if we use it, it fouls the the window alignment. + but if we use it, it fouls the window alignment. Fudge it to what we want... */ reg_2964 = 0x00010001 + ((reg_2964 & 0x0000FFFF) - (reg_2964 >> 16)); reg_2968 = 0x00010001 + ((reg_2968 & 0x0000FFFF) - (reg_2968 >> 16)); diff --git a/drivers/media/pci/ngene/ngene.h b/drivers/media/pci/ngene/ngene.h index 3d296f1998a1a36455fdddc74d3f945cc1f14056..d1d7da84cd9daa5035e13081466c222abd2a1b1b 100644 --- a/drivers/media/pci/ngene/ngene.h +++ b/drivers/media/pci/ngene/ngene.h @@ -596,43 +596,6 @@ struct mychip { int capture_source[MIXER_ADDR_LAST + 1][2]; }; -#ifdef NGENE_V4L -struct ngene_overlay { - int tvnorm; - struct v4l2_rect w; - enum v4l2_field field; - struct v4l2_clip *clips; - int nclips; - int setup_ok; -}; - -struct ngene_tvnorm { - int v4l2_id; - char *name; - u16 swidth, sheight; /* scaled standard width, height */ - int tuner_norm; - int soundstd; -}; - -struct ngene_vopen { - struct ngene_channel *ch; - enum v4l2_priority prio; - int width; - int height; - int depth; - struct videobuf_queue vbuf_q; - struct videobuf_queue vbi; - int fourcc; - int picxcount; - int resources; - enum v4l2_buf_type type; - const struct ngene_format *fmt; - - const struct ngene_format *ovfmt; - struct ngene_overlay ov; -}; -#endif - struct ngene_channel { struct device device; struct i2c_adapter i2c_adapter; @@ -709,18 +672,6 @@ struct ngene_channel { int tvnorm_num; int tvnorm; -#ifdef NGENE_V4L - int videousers; - struct v4l2_prio_state prio; - struct ngene_vopen init; - int resources; - struct v4l2_framebuffer fbuf; - struct ngene_buffer *screen; /* overlay */ - struct list_head capture; /* video capture queue */ - spinlock_t s_lock; - struct semaphore reslock; -#endif - int running; int tsin_offset; @@ -863,35 +814,6 @@ struct ngene_info { int (*switch_ctrl)(struct ngene_channel *, int, int); }; -#ifdef NGENE_V4L -struct ngene_format { - char *name; - int fourcc; /* video4linux 2 */ - int btformat; /* BT848_COLOR_FMT_* */ - int format; - int btswap; /* BT848_COLOR_CTL_* */ - int depth; /* bit/pixel */ - int flags; - int hshift, vshift; /* for planar modes */ - int palette; -}; - -#define RESOURCE_OVERLAY 1 -#define RESOURCE_VIDEO 2 -#define RESOURCE_VBI 4 - -struct ngene_buffer { - /* common v4l buffer stuff -- must be first */ - struct videobuf_buffer vb; - - /* ngene specific */ - const struct ngene_format *fmt; - int tvnorm; - int btformat; - int btswap; -}; -#endif - /* Provided by ngene-core.c */ int ngene_probe(struct pci_dev *pci_dev, const struct pci_device_id *id); diff --git a/drivers/media/pci/pt3/pt3.c b/drivers/media/pci/pt3/pt3.c index 0d51bdf01f4327d81183f9e8445a689364e9047c..f6deac85962e4add5b4a3990314dc6ef3e2b59d9 100644 --- a/drivers/media/pci/pt3/pt3.c +++ b/drivers/media/pci/pt3/pt3.c @@ -445,8 +445,8 @@ static int pt3_fetch_thread(void *data) pt3_proc_dma(adap); delay = ktime_set(0, PT3_FETCH_DELAY * NSEC_PER_MSEC); - set_current_state(TASK_UNINTERRUPTIBLE); - freezable_schedule_hrtimeout_range(&delay, + set_current_state(TASK_UNINTERRUPTIBLE|TASK_FREEZABLE); + schedule_hrtimeout_range(&delay, PT3_FETCH_DELAY_DELTA * NSEC_PER_MSEC, HRTIMER_MODE_REL); } diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c index 7973ae42873a6c5728c0cd92f2c960834b34b563..d5f32e3ff54416199e076575162a502b300c464a 100644 --- a/drivers/media/pci/saa7164/saa7164-core.c +++ b/drivers/media/pci/saa7164/saa7164-core.c @@ -626,7 +626,7 @@ static irqreturn_t saa7164_irq(int irq, void *dev_id) portf = &dev->ports[SAA7164_PORT_VBI2]; /* Check that the hardware is accessible. If the status bytes are - * 0xFF then the device is not accessible, the the IRQ belongs + * 0xFF then the device is not accessible, the IRQ belongs * to another driver. * 4 x u32 interrupt registers. */ diff --git a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c index 80d20e2a2099b4fc81552b9d15a29034ada191e2..0adf3d80f248ed3511509e5a571f1226cac62866 100644 --- a/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c +++ b/drivers/media/pci/solo6x10/solo6x10-v4l2-enc.c @@ -1020,7 +1020,7 @@ static int solo_g_parm(struct file *file, void *priv, cp->timeperframe.numerator = solo_enc->interval; cp->timeperframe.denominator = solo_enc->solo_dev->fps; cp->capturemode = 0; - /* XXX: Shouldn't we be able to get/set this from videobuf? */ + /* XXX: Shouldn't we be able to get/set this from vb2? */ cp->readbuffers = 2; return 0; diff --git a/drivers/staging/media/zoran/Kconfig b/drivers/media/pci/zoran/Kconfig similarity index 100% rename from drivers/staging/media/zoran/Kconfig rename to drivers/media/pci/zoran/Kconfig diff --git a/drivers/staging/media/zoran/Makefile b/drivers/media/pci/zoran/Makefile similarity index 100% rename from drivers/staging/media/zoran/Makefile rename to drivers/media/pci/zoran/Makefile diff --git a/drivers/staging/media/zoran/videocodec.c b/drivers/media/pci/zoran/videocodec.c similarity index 97% rename from drivers/staging/media/zoran/videocodec.c rename to drivers/media/pci/zoran/videocodec.c index a0c8bde5ec118bedf37c4dbaecd0c01068f3ff0a..8efc5e06b0f7ed50ca9f03fc630b2417b6f04326 100644 --- a/drivers/staging/media/zoran/videocodec.c +++ b/drivers/media/pci/zoran/videocodec.c @@ -92,9 +92,8 @@ struct videocodec *videocodec_attach(struct videocodec_master *master) h->attached += 1; return codec; - } else { - kfree(codec); } + kfree(codec); } h = h->next; } @@ -255,8 +254,8 @@ int videocodec_debugfs_show(struct seq_file *m) struct codec_list *h = codeclist_top; struct attached_list *a; - seq_printf(m, "lave or attached aster name type flags magic "); - seq_printf(m, "(connected as)\n"); + seq_puts(m, "lave or attached aster name type flags magic "); + seq_puts(m, "(connected as)\n"); while (h) { seq_printf(m, "S %32s %04x %08lx %08lx (TEMPLATE)\n", diff --git a/drivers/staging/media/zoran/videocodec.h b/drivers/media/pci/zoran/videocodec.h similarity index 65% rename from drivers/staging/media/zoran/videocodec.h rename to drivers/media/pci/zoran/videocodec.h index 5e6057edd33902b3c22dc2302933df8f51d8871b..6b69f69667f9e9114fcffd0b377975705383149e 100644 --- a/drivers/staging/media/zoran/videocodec.h +++ b/drivers/media/pci/zoran/videocodec.h @@ -12,109 +12,109 @@ /* general description */ /* =================== */ -/* Should ease the (re-)usage of drivers supporting cards with (different) - video codecs. The codecs register to this module their functionality, - and the processors (masters) can attach to them if they fit. - - The codecs are typically have a "strong" binding to their master - so I - don't think it makes sense to have a full blown interfacing as with e.g. - i2c. If you have an other opinion, let's discuss & implement it :-))) - - Usage: - - The slave has just to setup the videocodec structure and use two functions: - videocodec_register(codecdata); - videocodec_unregister(codecdata); - The best is just calling them at module (de-)initialisation. - - The master sets up the structure videocodec_master and calls: - codecdata=videocodec_attach(master_codecdata); - videocodec_detach(codecdata); - - The slave is called during attach/detach via functions setup previously - during register. At that time, the master_data pointer is set up - and the slave can access any io registers of the master device (in the case - the slave is bound to it). Otherwise it doesn't need this functions and - therfor they may not be initialized. - - The other functions are just for convenience, as they are for sure used by - most/all of the codecs. The last ones may be omitted, too. - - See the structure declaration below for more information and which data has - to be set up for the master and the slave. - - ---------------------------------------------------------------------------- - The master should have "knowledge" of the slave and vice versa. So the data - structures sent to/from slave via set_data/get_data set_image/get_image are - device dependent and vary between MJPEG/MPEG/WAVELET/... devices. (!!!!) - ---------------------------------------------------------------------------- -*/ +/* + * Should ease the (re-)usage of drivers supporting cards with (different) + * video codecs. The codecs register to this module their functionality, + * and the processors (masters) can attach to them if they fit. + * + * The codecs are typically have a "strong" binding to their master - so I + * don't think it makes sense to have a full blown interfacing as with e.g. + * i2c. If you have an other opinion, let's discuss & implement it :-))) + * + * Usage: + * + * The slave has just to setup the videocodec structure and use two functions: + * videocodec_register(codecdata); + * videocodec_unregister(codecdata); + * The best is just calling them at module (de-)initialisation. + * + * The master sets up the structure videocodec_master and calls: + * codecdata=videocodec_attach(master_codecdata); + * videocodec_detach(codecdata); + * + * The slave is called during attach/detach via functions setup previously + * during register. At that time, the master_data pointer is set up + * and the slave can access any io registers of the master device (in the case + * the slave is bound to it). Otherwise it doesn't need this functions and + * therefor they may not be initialized. + * + * The other functions are just for convenience, as they are for sure used by + * most/all of the codecs. The last ones may be omitted, too. + * + * See the structure declaration below for more information and which data has + * to be set up for the master and the slave. + * + * ---------------------------------------------------------------------------- + * The master should have "knowledge" of the slave and vice versa. So the data + * structures sent to/from slave via set_data/get_data set_image/get_image are + * device dependent and vary between MJPEG/MPEG/WAVELET/... devices. (!!!!) + * ---------------------------------------------------------------------------- + */ /* ========================================== */ /* description of the videocodec_io structure */ /* ========================================== */ /* - ==== master setup ==== - name -> name of the device structure for reference and debugging - master_data -> data ref. for the master (e.g. the zr36055,57,67) - readreg -> ref. to read-fn from register (setup by master, used by slave) - writereg -> ref. to write-fn to register (setup by master, used by slave) - this two functions do the lowlevel I/O job - - ==== slave functionality setup ==== - slave_data -> data ref. for the slave (e.g. the zr36050,60) - check -> fn-ref. checks availability of an device, returns -EIO on failure or - the type on success - this makes espcecially sense if a driver module supports more than - one codec which may be quite similar to access, nevertheless it - is good for a first functionality check - - -- main functions you always need for compression/decompression -- - - set_mode -> this fn-ref. resets the entire codec, and sets up the mode - with the last defined norm/size (or device default if not - available) - it returns 0 if the mode is possible - set_size -> this fn-ref. sets the norm and image size for - compression/decompression (returns 0 on success) - the norm param is defined in videodev2.h (V4L2_STD_*) - - additional setup may be available, too - but the codec should work with - some default values even without this - - set_data -> sets device-specific data (tables, quality etc.) - get_data -> query device-specific data (tables, quality etc.) - - if the device delivers interrupts, they may be setup/handled here - setup_interrupt -> codec irq setup (not needed for 36050/60) - handle_interrupt -> codec irq handling (not needed for 36050/60) - - if the device delivers pictures, they may be handled here - put_image -> puts image data to the codec (not needed for 36050/60) - get_image -> gets image data from the codec (not needed for 36050/60) - the calls include frame numbers and flags (even/odd/...) - if needed and a flag which allows blocking until its ready -*/ + * ==== master setup ==== + * name -> name of the device structure for reference and debugging + * master_data -> data ref. for the master (e.g. the zr36055,57,67) + * readreg -> ref. to read-fn from register (setup by master, used by slave) + * writereg -> ref. to write-fn to register (setup by master, used by slave) + * this two functions do the lowlevel I/O job + * + * ==== slave functionality setup ==== + * slave_data -> data ref. for the slave (e.g. the zr36050,60) + * check -> fn-ref. checks availability of an device, returns -EIO on failure or + * the type on success + * this makes espcecially sense if a driver module supports more than + * one codec which may be quite similar to access, nevertheless it + * is good for a first functionality check + * + * -- main functions you always need for compression/decompression -- + * + * set_mode -> this fn-ref. resets the entire codec, and sets up the mode + * with the last defined norm/size (or device default if not + * available) - it returns 0 if the mode is possible + * set_size -> this fn-ref. sets the norm and image size for + * compression/decompression (returns 0 on success) + * the norm param is defined in videodev2.h (V4L2_STD_*) + * + * additional setup may be available, too - but the codec should work with + * some default values even without this + * + * set_data -> sets device-specific data (tables, quality etc.) + * get_data -> query device-specific data (tables, quality etc.) + * + * if the device delivers interrupts, they may be setup/handled here + * setup_interrupt -> codec irq setup (not needed for 36050/60) + * handle_interrupt -> codec irq handling (not needed for 36050/60) + + * if the device delivers pictures, they may be handled here + * put_image -> puts image data to the codec (not needed for 36050/60) + * get_image -> gets image data from the codec (not needed for 36050/60) + * the calls include frame numbers and flags (even/odd/...) + * if needed and a flag which allows blocking until its ready + */ /* ============== */ /* user interface */ /* ============== */ /* - Currently there is only a information display planned, as the layer - is not visible for the user space at all. - - Information is available via procfs. The current entry is "/proc/videocodecs" - but it makes sense to "hide" it in the /proc/video tree of v4l(2) --TODO--. - -A example for such an output is: - -lave or attached aster name type flags magic (connected as) -S zr36050 0002 0000d001 00000000 (TEMPLATE) -M zr36055[0] 0001 0000c001 00000000 (zr36050[0]) -M zr36055[1] 0001 0000c001 00000000 (zr36050[1]) - -*/ + * Currently there is only a information display planned, as the layer + * is not visible for the user space at all. + * + * Information is available via procfs. The current entry is "/proc/videocodecs" + * but it makes sense to "hide" it in the /proc/video tree of v4l(2) --TODO--. + * + * A example for such an output is: + * + * lave or attached aster name type flags magic (connected as) + * S zr36050 0002 0000d001 00000000 (TEMPLATE) + * M zr36055[0] 0001 0000c001 00000000 (zr36050[0]) + * M zr36055[1] 0001 0000c001 00000000 (zr36050[1]) + */ /* =============================================== */ /* special defines for the videocodec_io structure */ @@ -293,15 +293,15 @@ struct videocodec_master { // * master structure needs to be kmalloc'ed before calling attach // and free'd after calling detach // * returns pointer on success, NULL on failure -extern struct videocodec *videocodec_attach(struct videocodec_master *); +struct videocodec *videocodec_attach(struct videocodec_master *master); // * 0 on success, <0 (errno) on failure -extern int videocodec_detach(struct videocodec *); +int videocodec_detach(struct videocodec *codec); /* register and unregister commands for the slaves */ // * 0 on success, <0 (errno) on failure -extern int videocodec_register(const struct videocodec *); +int videocodec_register(const struct videocodec *codec); // * 0 on success, <0 (errno) on failure -extern int videocodec_unregister(const struct videocodec *); +int videocodec_unregister(const struct videocodec *codec); /* the other calls are directly done via the videocodec structure! */ diff --git a/drivers/staging/media/zoran/zoran.h b/drivers/media/pci/zoran/zoran.h similarity index 90% rename from drivers/staging/media/zoran/zoran.h rename to drivers/media/pci/zoran/zoran.h index 05227e5298f6088f43c8cc40c5880e577b0936c1..56340553b2823977e052963f0ce27d841aeae32b 100644 --- a/drivers/staging/media/zoran/zoran.h +++ b/drivers/media/pci/zoran/zoran.h @@ -140,11 +140,16 @@ struct zoran_v4l_settings { /* jpg-capture/-playback settings */ struct zoran_jpg_settings { - int decimation; /* this bit is used to set everything to default */ - int hor_dcm, ver_dcm, tmp_dcm; /* capture decimation settings (tmp_dcm=1 means both fields) */ - int field_per_buff, odd_even; /* field-settings (odd_even=1 (+tmp_dcm=1) means top-field-first) */ - int img_x, img_y, img_width, img_height; /* crop settings (subframe capture) */ - struct v4l2_jpegcompression jpg_comp; /* JPEG-specific capture settings */ + /* this bit is used to set everything to default */ + int decimation; + /* capture decimation settings (tmp_dcm=1 means both fields) */ + int hor_dcm, ver_dcm, tmp_dcm; + /* field-settings (odd_even=1 (+tmp_dcm=1) means top-field-first) */ + int field_per_buff, odd_even; + /* crop settings (subframe capture) */ + int img_x, img_y, img_width, img_height; + /* JPEG-specific capture settings */ + struct v4l2_jpegcompression jpg_comp; }; struct zoran; @@ -248,7 +253,8 @@ struct zoran { unsigned long vbseq; /* zr36057's code buffer table */ - __le32 *stat_com; /* stat_com[i] is indexed by dma_head/tail & BUZ_MASK_STAT_COM */ + /* stat_com[i] is indexed by dma_head/tail & BUZ_MASK_STAT_COM */ + __le32 *stat_com; /* Additional stuff for testing */ unsigned int ghost_int; @@ -292,14 +298,16 @@ static inline struct zoran *to_zoran(struct v4l2_device *v4l2_dev) return container_of(v4l2_dev, struct zoran, v4l2_dev); } -/* There was something called _ALPHA_BUZ that used the PCI address instead of - * the kernel iomapped address for btread/btwrite. */ +/* + * There was something called _ALPHA_BUZ that used the PCI address instead of + * the kernel iomapped address for btread/btwrite. + */ #define btwrite(dat, adr) writel((dat), zr->zr36057_mem + (adr)) #define btread(adr) readl(zr->zr36057_mem + (adr)) -#define btand(dat, adr) btwrite((dat) & btread(adr), adr) -#define btor(dat, adr) btwrite((dat) | btread(adr), adr) -#define btaor(dat, mask, adr) btwrite((dat) | ((mask) & btread(adr)), adr) +#define btand(dat, adr) btwrite((dat) & btread(adr), (adr)) +#define btor(dat, adr) btwrite((dat) | btread(adr), (adr)) +#define btaor(dat, mask, adr) btwrite((dat) | ((mask) & btread(adr)), (adr)) #endif diff --git a/drivers/staging/media/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c similarity index 97% rename from drivers/staging/media/zoran/zoran_card.c rename to drivers/media/pci/zoran/zoran_card.c index 26f978a1cc729400b45e85b055675fe64d151c61..3975fc1b2ee3138ef84d69b7ff5b00de3e9c596e 100644 --- a/drivers/staging/media/zoran/zoran_card.c +++ b/drivers/media/pci/zoran/zoran_card.c @@ -172,8 +172,6 @@ void zr36016_write(struct videocodec *codec, u16 reg, u32 val) static void dc10_init(struct zoran *zr) { - pci_dbg(zr->pci_dev, "%s\n", __func__); - /* Pixel clock selection */ GPIO(zr, 4, 0); GPIO(zr, 5, 1); @@ -183,13 +181,10 @@ static void dc10_init(struct zoran *zr) static void dc10plus_init(struct zoran *zr) { - pci_dbg(zr->pci_dev, "%s\n", __func__); } static void buz_init(struct zoran *zr) { - pci_dbg(zr->pci_dev, "%s\n", __func__); - /* some stuff from Iomega */ pci_write_config_dword(zr->pci_dev, 0xfc, 0x90680f15); pci_write_config_dword(zr->pci_dev, 0x0c, 0x00012020); @@ -198,8 +193,6 @@ static void buz_init(struct zoran *zr) static void lml33_init(struct zoran *zr) { - pci_dbg(zr->pci_dev, "%s\n", __func__); - GPIO(zr, 2, 1); // Set Composite input/output } @@ -334,10 +327,6 @@ static void videocodec_exit(struct zoran *zr) codec_exit(zr, zr->card.video_vfe); } -// struct tvnorm { -// u16 wt, wa, h_start, h_sync_start, ht, ha, v_start; -// }; - static const struct tvnorm f50sqpixel = { 944, 768, 83, 880, 625, 576, 16 }; static const struct tvnorm f60sqpixel = { 780, 640, 51, 716, 525, 480, 12 }; static const struct tvnorm f50ccir601 = { 864, 720, 75, 804, 625, 576, 18 }; @@ -619,7 +608,10 @@ static struct card_info zoran_cards[NUM_CARDS] = { }, { .type = AVS6EYES, .name = "6-Eyes", -/* AverMedia chose not to brand the 6-Eyes. Thus it can't be autodetected, and requires card=x. */ + /* + * AverMedia chose not to brand the 6-Eyes. Thus it can't be + * autodetected, and requires card=x. + */ .i2c_decoder = "ks0127", .addrs_decoder = ks0127_addrs, .i2c_encoder = "bt866", @@ -764,7 +756,9 @@ int zoran_check_jpg_settings(struct zoran *zr, case 4: if (zr->card.type == DC10_NEW) { - pci_dbg(zr->pci_dev, "%s - HDec by 4 is not supported on the DC10\n", __func__); + pci_dbg(zr->pci_dev, + "%s - HDec by 4 is not supported on the DC10\n", + __func__); err0++; break; } @@ -882,12 +876,7 @@ static int zoran_init_video_device(struct zoran *zr, struct video_device *video_ video_dev->device_caps = V4L2_CAP_STREAMING | dir; strscpy(video_dev->name, ZR_DEVNAME(zr), sizeof(video_dev->name)); - /* - * It's not a mem2mem device, but you can both capture and output from one and the same - * device. This should really be split up into two device nodes, but that's a job for - * another day. - */ - video_dev->vfl_dir = VFL_DIR_M2M; + video_dev->vfl_dir = VFL_DIR_RX; zoran_queue_init(zr, &zr->vq, V4L2_BUF_TYPE_VIDEO_CAPTURE); err = video_register_device(video_dev, VFL_TYPE_VIDEO, video_nr[zr->id]); @@ -1019,7 +1008,9 @@ static int zr36057_init(struct zoran *zr) zr->timing = zr->card.tvn[ZR_NORM_SECAM]; } if (!zr->timing) { - pci_warn(zr->pci_dev, "%s - default TV standard not supported by hardware. PAL will be used.\n", __func__); + pci_warn(zr->pci_dev, + "%s - default TV standard not supported by hardware. PAL will be used.\n", + __func__); zr->norm = V4L2_STD_PAL; zr->timing = zr->card.tvn[ZR_NORM_PAL]; } @@ -1038,9 +1029,9 @@ static int zr36057_init(struct zoran *zr) zr->stat_com = dma_alloc_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), &zr->p_sc, GFP_KERNEL); - if (!zr->stat_com) { + if (!zr->stat_com) return -ENOMEM; - } + for (j = 0; j < BUZ_NUM_STAT_COM; j++) zr->stat_com[j] = cpu_to_le32(1); /* mark as unavailable to zr36057 */ @@ -1066,9 +1057,11 @@ static int zr36057_init(struct zoran *zr) return 0; exit_statcomb: - dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb); + dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, + zr->stat_comb, zr->p_scb); exit_statcom: - dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), zr->stat_com, zr->p_sc); + dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), + zr->stat_com, zr->p_sc); return err; } @@ -1099,8 +1092,10 @@ static void zoran_remove(struct pci_dev *pdev) btwrite(0, ZR36057_SPGPPCR); pci_free_irq(zr->pci_dev, 0, zr); /* unmap and free memory */ - dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), zr->stat_com, zr->p_sc); - dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, zr->stat_comb, zr->p_scb); + dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32), + zr->stat_com, zr->p_sc); + dma_free_coherent(&zr->pci_dev->dev, BUZ_NUM_STAT_COM * sizeof(u32) * 2, + zr->stat_comb, zr->p_scb); pci_release_regions(pdev); pci_disable_device(zr->pci_dev); zoran_exit_video_devices(zr); @@ -1299,7 +1294,8 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_err(pdev, "Unknown card, try specifying card=X module parameter\n"); goto zr_unreg; } - pci_info(zr->pci_dev, "%s() - card %s detected\n", __func__, zoran_cards[card_num].name); + pci_info(zr->pci_dev, "%s() - card %s detected\n", __func__, + zoran_cards[card_num].name); } else { card_num = card[nr]; if (card_num >= NUM_CARDS || card_num < 0) { @@ -1324,7 +1320,8 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) goto zr_unreg; - zr->zr36057_mem = devm_ioremap(&pdev->dev, pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); + zr->zr36057_mem = devm_ioremap(&pdev->dev, pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); if (!zr->zr36057_mem) { pci_err(pdev, "%s() - ioremap failed\n", __func__); goto zr_pci_release; @@ -1348,7 +1345,8 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &latency); need_latency = zr->revision > 1 ? 32 : 48; if (latency != need_latency) { - pci_info(zr->pci_dev, "Changing PCI latency from %d to %d\n", latency, need_latency); + pci_info(zr->pci_dev, "Changing PCI latency from %d to %d\n", + latency, need_latency); pci_write_config_byte(zr->pci_dev, PCI_LATENCY_TIMER, need_latency); } diff --git a/drivers/staging/media/zoran/zoran_card.h b/drivers/media/pci/zoran/zoran_card.h similarity index 72% rename from drivers/staging/media/zoran/zoran_card.h rename to drivers/media/pci/zoran/zoran_card.h index 8e0d634cb30f88af2a668a67fe68d8a510de4095..518cb426b4462c19dbe7c0e8b23ad433cf2f0285 100644 --- a/drivers/staging/media/zoran/zoran_card.h +++ b/drivers/media/pci/zoran/zoran_card.h @@ -19,11 +19,10 @@ extern int zr36067_debug; extern const struct video_device zoran_template; -extern int zoran_check_jpg_settings(struct zoran *zr, - struct zoran_jpg_settings *settings, - int try); -extern void zoran_open_init_params(struct zoran *zr); -extern void zoran_vdev_release(struct video_device *vdev); +int zoran_check_jpg_settings(struct zoran *zr, + struct zoran_jpg_settings *settings, int try); +void zoran_open_init_params(struct zoran *zr); +void zoran_vdev_release(struct video_device *vdev); void zr36016_write(struct videocodec *codec, u16 reg, u32 val); diff --git a/drivers/staging/media/zoran/zoran_device.c b/drivers/media/pci/zoran/zoran_device.c similarity index 96% rename from drivers/staging/media/zoran/zoran_device.c rename to drivers/media/pci/zoran/zoran_device.c index 2470889a58fa014343f5aa7477a4b86244a15a6d..31f049b55529cd32205e9b45a8075376cdaedc32 100644 --- a/drivers/staging/media/zoran/zoran_device.c +++ b/drivers/media/pci/zoran/zoran_device.c @@ -50,7 +50,6 @@ static bool lml33dpath; /* default = 0 module_param(lml33dpath, bool, 0644); MODULE_PARM_DESC(lml33dpath, "Use digital path capture mode (on LML33 cards)"); -int zr_set_buf(struct zoran *zr); /* * initialize video front end */ @@ -108,7 +107,6 @@ int post_office_wait(struct zoran *zr) { u32 por; -// while (((por = btread(ZR36057_POR)) & (ZR36057_POR_PO_PEN | ZR36057_POR_PO_TIME)) == ZR36057_POR_PO_PEN) { while ((por = btread(ZR36057_POR)) & ZR36057_POR_PO_PEN) { /* wait for something to happen */ /* TODO add timeout */ @@ -155,10 +153,12 @@ void jpeg_codec_sleep(struct zoran *zr, int sleep) { GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_SLEEP], !sleep); if (!sleep) { - pci_dbg(zr->pci_dev, "%s() - wake GPIO=0x%08x\n", __func__, btread(ZR36057_GPPGCR1)); - udelay(500); + pci_dbg(zr->pci_dev, "%s() - wake GPIO=0x%08x\n", + __func__, btread(ZR36057_GPPGCR1)); + usleep_range(500, 1000); } else { - pci_dbg(zr->pci_dev, "%s() - sleep GPIO=0x%08x\n", __func__, btread(ZR36057_GPPGCR1)); + pci_dbg(zr->pci_dev, "%s() - sleep GPIO=0x%08x\n", + __func__, btread(ZR36057_GPPGCR1)); udelay(2); } } @@ -284,7 +284,8 @@ static void zr36057_set_vfe(struct zoran *zr, int video_width, int video_height, vcrop1 = (tvn->ha / 2 - he) / 2; vcrop2 = tvn->ha / 2 - he - vcrop1; v_start = tvn->v_start; - v_end = v_start + tvn->ha / 2; // - 1; FIXME SnapShot times out with -1 in 768*576 on the DC10 - LP + // FIXME SnapShot times out with -1 in 768*576 on the DC10 - LP + v_end = v_start + tvn->ha / 2; // - 1; v_start += vcrop1; v_end -= vcrop2; reg = ((v_start & ZR36057_VFEVCR_VMASK) << ZR36057_VFEVCR_V_START) @@ -298,10 +299,12 @@ static void zr36057_set_vfe(struct zoran *zr, int video_width, int video_height, reg |= (hor_dcm << ZR36057_VFESPFR_HOR_DCM); reg |= (ver_dcm << ZR36057_VFESPFR_VER_DCM); reg |= (disp_mode << ZR36057_VFESPFR_DISP_MODE); - /* RJ: I don't know, why the following has to be the opposite + /* + * RJ: I don't know, why the following has to be the opposite * of the corresponding ZR36060 setting, but only this way - * we get the correct colors when uncompressing to the screen */ - //reg |= ZR36057_VFESPFR_VCLK_POL; /**/ + * we get the correct colors when uncompressing to the screen + */ + //reg |= ZR36057_VFESPFR_VCLK_POL; /* RJ: Don't know if that is needed for NTSC also */ if (!(zr->norm & V4L2_STD_NTSC)) reg |= ZR36057_VFESPFR_EXT_FL; // NEEDED!!!!!!! Wolfgang @@ -342,7 +345,7 @@ void zr36057_set_memgrab(struct zoran *zr, int mode) * will be stuck at 1 until capturing is turned back on. */ if (btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SNAP_SHOT) - pci_warn(zr->pci_dev, "zr36057_set_memgrab(1) with SnapShot on!?\n"); + pci_warn(zr->pci_dev, "%s(1) with SnapShot on!?\n", __func__); /* switch on VSync interrupts */ btwrite(IRQ_MASK, ZR36057_ISR); // Clear Interrupts @@ -595,11 +598,9 @@ void jpeg_start(struct zoran *zr) /* enable the Go generation */ btor(ZR36057_JMC_GO_EN, ZR36057_JMC); - udelay(30); + usleep_range(30, 100); set_frame(zr, 1); // /FRAME - - pci_dbg(zr->pci_dev, "jpeg_start\n"); } void zr36057_enable_jpg(struct zoran *zr, enum zoran_codec_mode mode) @@ -803,8 +804,10 @@ static void zoran_reap_stat_com(struct zoran *zr) unsigned int size = 0; u32 fcnt; - /* In motion decompress we don't have a hardware frame counter, - * we just count the interrupts here */ + /* + * In motion decompress we don't have a hardware frame counter, + * we just count the interrupts here + */ if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) zr->jpg_seq_num++; @@ -938,9 +941,9 @@ void zoran_init_hardware(struct zoran *zr) void zr36057_restart(struct zoran *zr) { btwrite(0, ZR36057_SPGPPCR); - udelay(1000); + usleep_range(1000, 2000); btor(ZR36057_SPGPPCR_SOFT_RESET, ZR36057_SPGPPCR); - udelay(1000); + usleep_range(1000, 2000); /* assert P_Reset */ btwrite(0, ZR36057_JPC); diff --git a/drivers/media/pci/zoran/zoran_device.h b/drivers/media/pci/zoran/zoran_device.h new file mode 100644 index 0000000000000000000000000000000000000000..34fd5cc914eb61d483326b2f829982ad7380d3df --- /dev/null +++ b/drivers/media/pci/zoran/zoran_device.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Zoran zr36057/zr36067 PCI controller driver, for the + * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux + * Media Labs LML33/LML33R10. + * + * This part handles card-specific data and detection + * + * Copyright (C) 2000 Serguei Miridonov + */ + +#ifndef __ZORAN_DEVICE_H__ +#define __ZORAN_DEVICE_H__ + +/* general purpose I/O */ +void GPIO(struct zoran *zr, int bit, unsigned int value); + +/* codec (or actually: guest bus) access */ +int post_office_wait(struct zoran *zr); +int post_office_write(struct zoran *zr, unsigned int guest, unsigned int reg, + unsigned int value); +int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg); + +void jpeg_codec_sleep(struct zoran *zr, int sleep); +int jpeg_codec_reset(struct zoran *zr); + +/* zr360x7 access to raw capture */ +void zr36057_overlay(struct zoran *zr, int on); +void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count); +void zr36057_set_memgrab(struct zoran *zr, int mode); +int wait_grab_pending(struct zoran *zr); + +/* interrupts */ +void print_interrupts(struct zoran *zr); +void clear_interrupt_counters(struct zoran *zr); +irqreturn_t zoran_irq(int irq, void *dev_id); + +/* JPEG codec access */ +void jpeg_start(struct zoran *zr); +void zr36057_enable_jpg(struct zoran *zr, enum zoran_codec_mode mode); +void zoran_feed_stat_com(struct zoran *zr); + +/* general */ +void zoran_set_pci_master(struct zoran *zr, int set_master); +void zoran_init_hardware(struct zoran *zr); +void zr36057_restart(struct zoran *zr); + +extern const struct zoran_format zoran_formats[]; + +extern int v4l_bufsize; +extern int jpg_bufsize; +extern int pass_through; + +/* i2c */ +#define decoder_call(zr, o, f, args...) \ + v4l2_subdev_call((zr)->decoder, o, f, ##args) +#define encoder_call(zr, o, f, args...) \ + v4l2_subdev_call((zr)->encoder, o, f, ##args) + +#endif /* __ZORAN_DEVICE_H__ */ diff --git a/drivers/staging/media/zoran/zoran_driver.c b/drivers/media/pci/zoran/zoran_driver.c similarity index 94% rename from drivers/staging/media/zoran/zoran_driver.c rename to drivers/media/pci/zoran/zoran_driver.c index 4304b7e21709b3c610aaa2fb23054283db8e51fa..fa672cc8bc678eb429db3d58e4a078c75d5cfde1 100644 --- a/drivers/staging/media/zoran/zoran_driver.c +++ b/drivers/media/pci/zoran/zoran_driver.c @@ -203,7 +203,6 @@ static int zoran_v4l_set_format(struct zoran *zr, int width, int height, static int zoran_set_norm(struct zoran *zr, v4l2_std_id norm) { - if (!(norm & zr->card.norms)) { pci_dbg(zr->pci_dev, "%s - unsupported norm %llx\n", __func__, norm); return -EINVAL; @@ -287,17 +286,6 @@ static int zoran_enum_fmt_vid_cap(struct file *file, void *__fh, return zoran_enum_fmt(zr, f, ZORAN_FORMAT_CAPTURE); } -#if 0 -/* TODO: output does not work yet */ -static int zoran_enum_fmt_vid_out(struct file *file, void *__fh, - struct v4l2_fmtdesc *f) -{ - struct zoran *zr = video_drvdata(file); - - return zoran_enum_fmt(zr, f, ZORAN_FORMAT_PLAYBACK); -} -#endif - static int zoran_g_fmt_vid_out(struct file *file, void *__fh, struct v4l2_format *fmt) { @@ -430,8 +418,10 @@ static int zoran_try_fmt_vid_cap(struct file *file, void *__fh, fmt->fmt.pix.field = V4L2_FIELD_TOP; bpp = DIV_ROUND_UP(zoran_formats[i].depth, 8); - v4l_bound_align_image(&fmt->fmt.pix.width, BUZ_MIN_WIDTH, BUZ_MAX_WIDTH, bpp == 2 ? 1 : 2, - &fmt->fmt.pix.height, BUZ_MIN_HEIGHT, BUZ_MAX_HEIGHT, 0, 0); + v4l_bound_align_image(&fmt->fmt.pix.width, BUZ_MIN_WIDTH, BUZ_MAX_WIDTH, + bpp == 2 ? 1 : 2, + &fmt->fmt.pix.height, BUZ_MIN_HEIGHT, BUZ_MAX_HEIGHT, + 0, 0); fmt->fmt.pix.bytesperline = fmt->fmt.pix.width * bpp; fmt->fmt.pix.sizeimage = fmt->fmt.pix.bytesperline * fmt->fmt.pix.height; return 0; @@ -627,38 +617,6 @@ static int zoran_s_input(struct file *file, void *__fh, unsigned int input) return res; } -#if 0 -/* TODO: output does not work yet */ -static int zoran_enum_output(struct file *file, void *__fh, - struct v4l2_output *outp) -{ - if (outp->index != 0) - return -EINVAL; - - outp->index = 0; - outp->type = V4L2_OUTPUT_TYPE_ANALOGVGAOVERLAY; - outp->std = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM; - outp->capabilities = V4L2_OUT_CAP_STD; - strscpy(outp->name, "Autodetect", sizeof(outp->name)); - - return 0; -} -static int zoran_g_output(struct file *file, void *__fh, unsigned int *output) -{ - *output = 0; - - return 0; -} - -static int zoran_s_output(struct file *file, void *__fh, unsigned int output) -{ - if (output != 0) - return -EINVAL; - - return 0; -} -#endif - /* cropping (sub-frame capture) */ static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selection *sel) { @@ -746,9 +704,6 @@ static const struct v4l2_ioctl_ops zoran_ioctl_ops = { .vidioc_enum_input = zoran_enum_input, .vidioc_g_input = zoran_g_input, .vidioc_s_input = zoran_s_input, -/* .vidioc_enum_output = zoran_enum_output, - .vidioc_g_output = zoran_g_output, - .vidioc_s_output = zoran_s_output,*/ .vidioc_g_std = zoran_g_std, .vidioc_s_std = zoran_s_std, .vidioc_create_bufs = vb2_ioctl_create_bufs, @@ -760,13 +715,9 @@ static const struct v4l2_ioctl_ops zoran_ioctl_ops = { .vidioc_streamon = vb2_ioctl_streamon, .vidioc_streamoff = vb2_ioctl_streamoff, .vidioc_enum_fmt_vid_cap = zoran_enum_fmt_vid_cap, -/* .vidioc_enum_fmt_vid_out = zoran_enum_fmt_vid_out,*/ .vidioc_g_fmt_vid_cap = zoran_g_fmt_vid_cap, -/* .vidioc_g_fmt_vid_out = zoran_g_fmt_vid_out,*/ .vidioc_s_fmt_vid_cap = zoran_s_fmt_vid_cap, -/* .vidioc_s_fmt_vid_out = zoran_s_fmt_vid_out,*/ .vidioc_try_fmt_vid_cap = zoran_try_fmt_vid_cap, -/* .vidioc_try_fmt_vid_out = zoran_try_fmt_vid_out,*/ .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, .vidioc_unsubscribe_event = v4l2_event_unsubscribe, }; @@ -1013,7 +964,7 @@ int zoran_queue_init(struct zoran *zr, struct vb2_queue *vq, int dir) vq->dev = &zr->pci_dev->dev; vq->type = dir; - vq->io_modes = VB2_DMABUF | VB2_MMAP | VB2_READ | VB2_WRITE; + vq->io_modes = VB2_DMABUF | VB2_MMAP; vq->drv_priv = zr; vq->buf_struct_size = sizeof(struct zr_buffer); vq->ops = &zr_video_qops; diff --git a/drivers/staging/media/zoran/zr36016.c b/drivers/media/pci/zoran/zr36016.c similarity index 73% rename from drivers/staging/media/zoran/zr36016.c rename to drivers/media/pci/zoran/zr36016.c index 0e0532537a3e01fe12d1a9e1dd6d42aea70beca4..4b328ad6083fdce76d0bc96a3079d350781d0049 100644 --- a/drivers/staging/media/zoran/zr36016.c +++ b/drivers/media/pci/zoran/zr36016.c @@ -15,18 +15,19 @@ /* codec io API */ #include "videocodec.h" -/* it doesn't make sense to have more than 20 or so, - just to prevent some unwanted loops */ +/* + * it doesn't make sense to have more than 20 or so, + * just to prevent some unwanted loops + */ #define MAX_CODECS 20 /* amount of chips attached via this driver */ static int zr36016_codecs; -/* ========================================================================= - Local hardware I/O functions: - - read/write via codec layer (registers are located in the master device) - ========================================================================= */ +/* + * Local hardware I/O functions: read/write via codec layer + * (registers are located in the master device) + */ /* read and write functions */ static u8 zr36016_read(struct zr36016 *ptr, u16 reg) @@ -58,9 +59,12 @@ static void zr36016_write(struct zr36016 *ptr, u16 reg, u8 value) zrdev_err(zr, "%s: invalid I/O setup, nothing written!\n", ptr->name); } -/* indirect read and write functions */ -/* the 016 supports auto-addr-increment, but - * writing it all time cost not much and is safer... */ +/* + * indirect read and write functions + * + * the 016 supports auto-addr-increment, but + * writing it all time cost not much and is safer... + */ static u8 zr36016_readi(struct zr36016 *ptr, u16 reg) { u8 value = 0; @@ -68,8 +72,8 @@ static u8 zr36016_readi(struct zr36016 *ptr, u16 reg) /* just in case something is wrong... */ if ((ptr->codec->master_data->writereg) && (ptr->codec->master_data->readreg)) { - ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); // ADDR - value = (ptr->codec->master_data->readreg(ptr->codec, ZR016_IDATA)) & 0xFF; // DATA + ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); + value = (ptr->codec->master_data->readreg(ptr->codec, ZR016_IDATA)) & 0xFF; } else { zrdev_err(zr, "%s: invalid I/O setup, nothing read (i)!\n", ptr->name); } @@ -88,18 +92,14 @@ static void zr36016_writei(struct zr36016 *ptr, u16 reg, u8 value) /* just in case something is wrong... */ if (ptr->codec->master_data->writereg) { - ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); // ADDR - ptr->codec->master_data->writereg(ptr->codec, ZR016_IDATA, value & 0x0FF); // DATA + ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F); + ptr->codec->master_data->writereg(ptr->codec, ZR016_IDATA, value & 0x0FF); } else { zrdev_err(zr, "%s: invalid I/O setup, nothing written (i)!\n", ptr->name); } } -/* ========================================================================= - Local helper function: - - version read - ========================================================================= */ +/* Local helper function: version read */ /* version kept in datastructure */ static u8 zr36016_read_version(struct zr36016 *ptr) @@ -108,11 +108,10 @@ static u8 zr36016_read_version(struct zr36016 *ptr) return ptr->version; } -/* ========================================================================= - Local helper function: - - basic test of "connectivity", writes/reads to/from PAX-Lo register - ========================================================================= */ +/* + * Local helper function: basic test of "connectivity", writes/reads + * to/from PAX-Lo register + */ static int zr36016_basic_test(struct zr36016 *ptr) { @@ -150,36 +149,7 @@ static int zr36016_basic_test(struct zr36016 *ptr) return 0; /* looks good! */ } -/* ========================================================================= - Local helper function: - - simple loop for pushing the init datasets - NO USE -- - ========================================================================= */ - -#if 0 -static int zr36016_pushit(struct zr36016 *ptr, - u16 startreg, - u16 len, - const char *data) -{ - struct zoran *zr = videocodec_to_zoran(ptr->codec); - int i = 0; - - zrdev_dbg(zr, "%s: write data block to 0x%04x (len=%d)\n", - ptr->name, startreg, len); - while (i < len) { - zr36016_writei(ptr, startreg++, data[i++]); - } - - return i; -} -#endif - -/* ========================================================================= - Basic datasets & init: - - //TODO// - ========================================================================= */ +/* Basic datasets & init */ static void zr36016_init(struct zr36016 *ptr) { @@ -213,14 +183,16 @@ static void zr36016_init(struct zr36016 *ptr) zr36016_write(ptr, ZR016_GOSTOP, 1); } -/* ========================================================================= - CODEC API FUNCTIONS - - this functions are accessed by the master via the API structure - ========================================================================= */ +/* + * CODEC API FUNCTIONS + * + * These functions are accessed by the master via the API structure + */ -/* set compression/expansion mode and launches codec - - this should be the last call from the master before starting processing */ +/* + * set compression/expansion mode and launches codec - + * this should be the last call from the master before starting processing + */ static int zr36016_set_mode(struct videocodec *codec, int mode) { struct zr36016 *ptr = (struct zr36016 *)codec->data; @@ -249,22 +221,28 @@ static int zr36016_set_video(struct videocodec *codec, const struct tvnorm *norm cap->x, cap->y, cap->width, cap->height, cap->decimation); - /* if () return -EINVAL; + /* + * if () return -EINVAL; * trust the master driver that it knows what it does - so - * we allow invalid startx/y for now ... */ + * we allow invalid startx/y for now ... + */ ptr->width = cap->width; ptr->height = cap->height; - /* (Ronald) This is ugly. zoran_device.c, line 387 + /* + * (Ronald) This is ugly. zoran_device.c, line 387 * already mentions what happens if h_start is even * (blue faces, etc., cr/cb inversed). There's probably * some good reason why h_start is 0 instead of 1, so I'm * leaving it to this for now, but really... This can be - * done a lot simpler */ + * done a lot simpler + */ ptr->xoff = (norm->h_start ? norm->h_start : 1) + cap->x; - /* Something to note here (I don't understand it), setting + /* + * Something to note here (I don't understand it), setting * v_start too high will cause the codec to 'not work'. I * really don't get it. values of 16 (v_start) already break - * it here. Just '0' seems to work. More testing needed! */ + * it here. Just '0' seems to work. More testing needed! + */ ptr->yoff = norm->v_start + cap->y; /* (Ronald) dzjeeh, can't this thing do hor_decimation = 4? */ ptr->xdec = ((cap->decimation & 0xff) == 1) ? 0 : 1; @@ -319,11 +297,11 @@ static int zr36016_control(struct videocodec *codec, int type, int size, void *d return size; } -/* ========================================================================= - Exit and unregister function: - - Deinitializes Zoran's JPEG processor - ========================================================================= */ +/* + * Exit and unregister function: + * + * Deinitializes Zoran's JPEG processor + */ static int zr36016_unset(struct videocodec *codec) { @@ -344,14 +322,14 @@ static int zr36016_unset(struct videocodec *codec) return -EFAULT; } -/* ========================================================================= - Setup and registry function: - - Initializes Zoran's JPEG processor - - Also sets pixel size, average code size, mode (compr./decompr.) - (the given size is determined by the processor with the video interface) - ========================================================================= */ +/* + * Setup and registry function: + * + * Initializes Zoran's JPEG processor + * + * Also sets pixel size, average code size, mode (compr./decompr.) + * (the given size is determined by the processor with the video interface) + */ static int zr36016_setup(struct videocodec *codec) { @@ -410,9 +388,7 @@ static const struct videocodec zr36016_codec = { /* others are not used */ }; -/* ========================================================================= - HOOK IN DRIVER AS KERNEL MODULE - ========================================================================= */ +/* HOOK IN DRIVER AS KERNEL MODULE */ int zr36016_init_module(void) { diff --git a/drivers/staging/media/zoran/zr36016.h b/drivers/media/pci/zoran/zr36016.h similarity index 100% rename from drivers/staging/media/zoran/zr36016.h rename to drivers/media/pci/zoran/zr36016.h diff --git a/drivers/staging/media/zoran/zr36050.c b/drivers/media/pci/zoran/zr36050.c similarity index 82% rename from drivers/staging/media/zoran/zr36050.c rename to drivers/media/pci/zoran/zr36050.c index 6a7ef28d996ccb9661cd19627ea0ff76ada70759..b07d7e5c1b4a33840630672b37347ec5c8b1dd5b 100644 --- a/drivers/staging/media/zoran/zr36050.c +++ b/drivers/media/pci/zoran/zr36050.c @@ -22,18 +22,20 @@ /* codec io API */ #include "videocodec.h" -/* it doesn't make sense to have more than 20 or so, - just to prevent some unwanted loops */ +/* + * it doesn't make sense to have more than 20 or so, + * just to prevent some unwanted loops + */ #define MAX_CODECS 20 /* amount of chips attached via this driver */ static int zr36050_codecs; -/* ========================================================================= - Local hardware I/O functions: - - read/write via codec layer (registers are located in the master device) - ========================================================================= */ +/* + * Local hardware I/O functions: + * + * read/write via codec layer (registers are located in the master device) + */ /* read and write functions */ static u8 zr36050_read(struct zr36050 *ptr, u16 reg) @@ -66,12 +68,6 @@ static void zr36050_write(struct zr36050 *ptr, u16 reg, u8 value) ptr->name); } -/* ========================================================================= - Local helper function: - - status read - ========================================================================= */ - /* status is kept in datastructure */ static u8 zr36050_read_status1(struct zr36050 *ptr) { @@ -81,12 +77,6 @@ static u8 zr36050_read_status1(struct zr36050 *ptr) return ptr->status1; } -/* ========================================================================= - Local helper function: - - scale factor read - ========================================================================= */ - /* scale factor is kept in datastructure */ static u16 zr36050_read_scalefactor(struct zr36050 *ptr) { @@ -98,11 +88,11 @@ static u16 zr36050_read_scalefactor(struct zr36050 *ptr) return ptr->scalefact; } -/* ========================================================================= - Local helper function: - - wait if codec is ready to proceed (end of processing) or time is over - ========================================================================= */ +/* + * Local helper function: + * + * wait if codec is ready to proceed (end of processing) or time is over + */ static void zr36050_wait_end(struct zr36050 *ptr) { @@ -120,11 +110,10 @@ static void zr36050_wait_end(struct zr36050 *ptr) } } -/* ========================================================================= - Local helper function: - - basic test of "connectivity", writes/reads to/from memory the SOF marker - ========================================================================= */ +/* + * Local helper function: basic test of "connectivity", writes/reads + * to/from memory the SOF marker + */ static int zr36050_basic_test(struct zr36050 *ptr) { @@ -160,11 +149,7 @@ static int zr36050_basic_test(struct zr36050 *ptr) return 0; /* looks good! */ } -/* ========================================================================= - Local helper function: - - simple loop for pushing the init datasets - ========================================================================= */ +/* Local helper function: simple loop for pushing the init datasets */ static int zr36050_pushit(struct zr36050 *ptr, u16 startreg, u16 len, const char *data) { @@ -179,16 +164,16 @@ static int zr36050_pushit(struct zr36050 *ptr, u16 startreg, u16 len, const char return i; } -/* ========================================================================= - Basic datasets: - - jpeg baseline setup data (you find it on lots places in internet, or just - extract it from any regular .jpg image...) - - Could be variable, but until it's not needed it they are just fixed to save - memory. Otherwise expand zr36050 structure with arrays, push the values to - it and initialize from there, as e.g. the linux zr36057/60 driver does it. - ========================================================================= */ +/* + * Basic datasets: + * + * jpeg baseline setup data (you find it on lots places in internet, or just + * extract it from any regular .jpg image...) + * + * Could be variable, but until it's not needed it they are just fixed to save + * memory. Otherwise expand zr36050 structure with arrays, push the values to + * it and initialize from there, as e.g. the linux zr36057/60 driver does it. + */ static const char zr36050_dqt[0x86] = { 0xff, 0xdb, //Marker: DQT @@ -281,18 +266,19 @@ static const char zr36050_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's AC static const char zr36050_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 }; static const char zr36050_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 }; -/* ========================================================================= - Local helper functions: - - calculation and setup of parameter-dependent JPEG baseline segments - (needed for compression only) - ========================================================================= */ +/* + * Local helper functions: + * + * calculation and setup of parameter-dependent JPEG baseline segments + * (needed for compression only) + */ /* ------------------------------------------------------------------------- */ -/* SOF (start of frame) segment depends on width, height and sampling ratio - of each color component */ - +/* + * SOF (start of frame) segment depends on width, height and sampling ratio + * of each color component + */ static int zr36050_set_sof(struct zr36050 *ptr) { struct zoran *zr = videocodec_to_zoran(ptr->codec); @@ -313,7 +299,8 @@ static int zr36050_set_sof(struct zr36050 *ptr) sof_data[9] = NO_OF_COMPONENTS; for (i = 0; i < NO_OF_COMPONENTS; i++) { sof_data[10 + (i * 3)] = i; // index identifier - sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) | (ptr->v_samp_ratio[i]); // sampling ratios + sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) | + (ptr->v_samp_ratio[i]); // sampling ratios sof_data[12 + (i * 3)] = zr36050_tq[i]; // Q table selection } return zr36050_pushit(ptr, ZR050_SOF_IDX, @@ -322,8 +309,10 @@ static int zr36050_set_sof(struct zr36050 *ptr) /* ------------------------------------------------------------------------- */ -/* SOS (start of scan) segment depends on the used scan components - of each color component */ +/* + * SOS (start of scan) segment depends on the used scan components + * of each color component + */ static int zr36050_set_sos(struct zr36050 *ptr) { @@ -368,14 +357,14 @@ static int zr36050_set_dri(struct zr36050 *ptr) return zr36050_pushit(ptr, ZR050_DRI_IDX, 6, dri_data); } -/* ========================================================================= - Setup function: - - Setup compression/decompression of Zoran's JPEG processor - ( see also zoran 36050 manual ) - - ... sorry for the spaghetti code ... - ========================================================================= */ +/* + * Setup function: + * + * Setup compression/decompression of Zoran's JPEG processor + * ( see also zoran 36050 manual ) + * + * ... sorry for the spaghetti code ... + */ static void zr36050_init(struct zr36050 *ptr) { int sum = 0; @@ -411,8 +400,10 @@ static void zr36050_init(struct zr36050 *ptr) sum += zr36050_set_sos(ptr); sum += zr36050_set_dri(ptr); - /* setup the fixed jpeg tables - maybe variable, though - - * (see table init section above) */ + /* + * setup the fixed jpeg tables - maybe variable, though - + * (see table init section above) + */ zrdev_dbg(zr, "%s: write DQT, DHT, APP\n", ptr->name); sum += zr36050_pushit(ptr, ZR050_DQT_IDX, sizeof(zr36050_dqt), zr36050_dqt); @@ -522,14 +513,16 @@ static void zr36050_init(struct zr36050 *ptr) zr36050_read(ptr, 0); } -/* ========================================================================= - CODEC API FUNCTIONS - - this functions are accessed by the master via the API structure - ========================================================================= */ +/* + * CODEC API FUNCTIONS + * + * this functions are accessed by the master via the API structure + */ -/* set compression/expansion mode and launches codec - - this should be the last call from the master before starting processing */ +/* + * set compression/expansion mode and launches codec - + * this should be the last call from the master before starting processing + */ static int zr36050_set_mode(struct videocodec *codec, int mode) { struct zr36050 *ptr = (struct zr36050 *)codec->data; @@ -558,9 +551,10 @@ static int zr36050_set_video(struct videocodec *codec, const struct tvnorm *norm ptr->name, norm->h_start, norm->v_start, cap->x, cap->y, cap->width, cap->height, cap->decimation, cap->quality); - /* if () return -EINVAL; + /* * trust the master driver that it knows what it does - so - * we allow invalid startx/y and norm for now ... */ + * we allow invalid startx/y and norm for now ... + */ ptr->width = cap->width / (cap->decimation & 0xff); ptr->height = cap->height / ((cap->decimation >> 8) & 0xff); @@ -579,8 +573,10 @@ static int zr36050_set_video(struct videocodec *codec, const struct tvnorm *norm ptr->real_code_vol = size >> 3; /* in bytes */ - /* Set max_block_vol here (previously in zr36050_init, moved - * here for consistency with zr36060 code */ + /* + * Set max_block_vol here (previously in zr36050_init, moved + * here for consistency with zr36060 code + */ zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol); return 0; @@ -637,8 +633,6 @@ static int zr36050_control(struct videocodec *codec, int type, int size, void *d if (size != sizeof(int)) return -EFAULT; ptr->total_code_vol = *ival; - /* (Kieran Morrissey) - * code copied from zr36060.c to ensure proper bitrate */ ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3; break; @@ -701,11 +695,7 @@ static int zr36050_control(struct videocodec *codec, int type, int size, void *d return size; } -/* ========================================================================= - Exit and unregister function: - - Deinitializes Zoran's JPEG processor - ========================================================================= */ +/* Exit and unregister function: Deinitializes Zoran's JPEG processor */ static int zr36050_unset(struct videocodec *codec) { @@ -727,14 +717,14 @@ static int zr36050_unset(struct videocodec *codec) return -EFAULT; } -/* ========================================================================= - Setup and registry function: - - Initializes Zoran's JPEG processor - - Also sets pixel size, average code size, mode (compr./decompr.) - (the given size is determined by the processor with the video interface) - ========================================================================= */ +/* + * Setup and registry function: + * + * Initializes Zoran's JPEG processor + * + * Also sets pixel size, average code size, mode (compr./decompr.) + * (the given size is determined by the processor with the video interface) + */ static int zr36050_setup(struct videocodec *codec) { @@ -771,8 +761,8 @@ static int zr36050_setup(struct videocodec *codec) memcpy(ptr->h_samp_ratio, zr36050_decimation_h, 8); memcpy(ptr->v_samp_ratio, zr36050_decimation_v, 8); - ptr->bitrate_ctrl = 0; /* 0 or 1 - fixed file size flag - * (what is the difference?) */ + /* 0 or 1 - fixed file size flag (what is the difference?) */ + ptr->bitrate_ctrl = 0; ptr->mode = CODEC_DO_COMPRESSION; ptr->width = 384; ptr->height = 288; @@ -809,9 +799,7 @@ static const struct videocodec zr36050_codec = { // others are not used }; -/* ========================================================================= - HOOK IN DRIVER AS KERNEL MODULE - ========================================================================= */ +/* HOOK IN DRIVER AS KERNEL MODULE */ int zr36050_init_module(void) { diff --git a/drivers/staging/media/zoran/zr36050.h b/drivers/media/pci/zoran/zr36050.h similarity index 100% rename from drivers/staging/media/zoran/zr36050.h rename to drivers/media/pci/zoran/zr36050.h diff --git a/drivers/staging/media/zoran/zr36057.h b/drivers/media/pci/zoran/zr36057.h similarity index 50% rename from drivers/staging/media/zoran/zr36057.h rename to drivers/media/pci/zoran/zr36057.h index a2a75fd9f535469593529e3c52f09631f4e6102f..45d8afc62b37e38719173d9ec33c6bcb327c3a20 100644 --- a/drivers/staging/media/zoran/zr36057.h +++ b/drivers/media/pci/zoran/zr36057.h @@ -11,117 +11,117 @@ /* Zoran ZR36057 registers */ #define ZR36057_VFEHCR 0x000 /* Video Front End, Horizontal Configuration Register */ -#define ZR36057_VFEHCR_HS_POL BIT(30) -#define ZR36057_VFEHCR_H_START 10 +#define ZR36057_VFEHCR_HS_POL BIT(30) +#define ZR36057_VFEHCR_H_START 10 #define ZR36057_VFEHCR_H_END 0 #define ZR36057_VFEHCR_HMASK 0x3ff #define ZR36057_VFEVCR 0x004 /* Video Front End, Vertical Configuration Register */ -#define ZR36057_VFEVCR_VS_POL BIT(30) -#define ZR36057_VFEVCR_V_START 10 +#define ZR36057_VFEVCR_VS_POL BIT(30) +#define ZR36057_VFEVCR_V_START 10 #define ZR36057_VFEVCR_V_END 0 #define ZR36057_VFEVCR_VMASK 0x3ff #define ZR36057_VFESPFR 0x008 /* Video Front End, Scaler and Pixel Format Register */ -#define ZR36057_VFESPFR_EXT_FL BIT(26) -#define ZR36057_VFESPFR_TOP_FIELD BIT(25) -#define ZR36057_VFESPFR_VCLK_POL BIT(24) -#define ZR36057_VFESPFR_H_FILTER 21 -#define ZR36057_VFESPFR_HOR_DCM 14 -#define ZR36057_VFESPFR_VER_DCM 8 -#define ZR36057_VFESPFR_DISP_MODE 6 +#define ZR36057_VFESPFR_EXT_FL BIT(26) +#define ZR36057_VFESPFR_TOP_FIELD BIT(25) +#define ZR36057_VFESPFR_VCLK_POL BIT(24) +#define ZR36057_VFESPFR_H_FILTER 21 +#define ZR36057_VFESPFR_HOR_DCM 14 +#define ZR36057_VFESPFR_VER_DCM 8 +#define ZR36057_VFESPFR_DISP_MODE 6 #define ZR36057_VFESPFR_YUV422 (0 << 3) #define ZR36057_VFESPFR_RGB888 (1 << 3) #define ZR36057_VFESPFR_RGB565 (2 << 3) #define ZR36057_VFESPFR_RGB555 (3 << 3) -#define ZR36057_VFESPFR_ERR_DIF (1 << 2) -#define ZR36057_VFESPFR_PACK24 (1 << 1) -#define ZR36057_VFESPFR_LITTLE_ENDIAN (1 << 0) +#define ZR36057_VFESPFR_ERR_DIF BIT(2) +#define ZR36057_VFESPFR_PACK24 BIT(1) +#define ZR36057_VFESPFR_LITTLE_ENDIAN BIT(0) #define ZR36057_VDTR 0x00c /* Video Display "Top" Register */ #define ZR36057_VDBR 0x010 /* Video Display "Bottom" Register */ #define ZR36057_VSSFGR 0x014 /* Video Stride, Status, and Frame Grab Register */ -#define ZR36057_VSSFGR_DISP_STRIDE 16 -#define ZR36057_VSSFGR_VID_OVF BIT(8) -#define ZR36057_VSSFGR_SNAP_SHOT BIT(1) -#define ZR36057_VSSFGR_FRAME_GRAB BIT(0) +#define ZR36057_VSSFGR_DISP_STRIDE 16 +#define ZR36057_VSSFGR_VID_OVF BIT(8) +#define ZR36057_VSSFGR_SNAP_SHOT BIT(1) +#define ZR36057_VSSFGR_FRAME_GRAB BIT(0) #define ZR36057_VDCR 0x018 /* Video Display Configuration Register */ -#define ZR36057_VDCR_VID_EN BIT(31) -#define ZR36057_VDCR_MIN_PIX 24 -#define ZR36057_VDCR_TRITON BIT(24) -#define ZR36057_VDCR_VID_WIN_HT 12 -#define ZR36057_VDCR_VID_WIN_WID 0 +#define ZR36057_VDCR_VID_EN BIT(31) +#define ZR36057_VDCR_MIN_PIX 24 +#define ZR36057_VDCR_TRITON BIT(24) +#define ZR36057_VDCR_VID_WIN_HT 12 +#define ZR36057_VDCR_VID_WIN_WID 0 #define ZR36057_MMTR 0x01c /* Masking Map "Top" Register */ #define ZR36057_MMBR 0x020 /* Masking Map "Bottom" Register */ #define ZR36057_OCR 0x024 /* Overlay Control Register */ -#define ZR36057_OCR_OVL_ENABLE BIT(15) -#define ZR36057_OCR_MASK_STRIDE 0 +#define ZR36057_OCR_OVL_ENABLE BIT(15) +#define ZR36057_OCR_MASK_STRIDE 0 #define ZR36057_SPGPPCR 0x028 /* System, PCI, and General Purpose Pins Control Register */ -#define ZR36057_SPGPPCR_SOFT_RESET BIT(24) +#define ZR36057_SPGPPCR_SOFT_RESET BIT(24) #define ZR36057_GPPGCR1 0x02c /* General Purpose Pins and GuestBus Control Register (1) */ #define ZR36057_MCSAR 0x030 /* MPEG Code Source Address Register */ #define ZR36057_MCTCR 0x034 /* MPEG Code Transfer Control Register */ -#define ZR36057_MCTCR_COD_TIME BIT(30) -#define ZR36057_MCTCR_C_EMPTY BIT(29) -#define ZR36057_MCTCR_C_FLUSH BIT(28) +#define ZR36057_MCTCR_COD_TIME BIT(30) +#define ZR36057_MCTCR_C_EMPTY BIT(29) +#define ZR36057_MCTCR_C_FLUSH BIT(28) #define ZR36057_MCTCR_COD_GUEST_ID 20 #define ZR36057_MCTCR_COD_GUEST_REG 16 #define ZR36057_MCMPR 0x038 /* MPEG Code Memory Pointer Register */ #define ZR36057_ISR 0x03c /* Interrupt Status Register */ -#define ZR36057_ISR_GIRQ1 BIT(30) -#define ZR36057_ISR_GIRQ0 BIT(29) -#define ZR36057_ISR_COD_REP_IRQ BIT(28) -#define ZR36057_ISR_JPEG_REP_IRQ BIT(27) +#define ZR36057_ISR_GIRQ1 BIT(30) +#define ZR36057_ISR_GIRQ0 BIT(29) +#define ZR36057_ISR_COD_REP_IRQ BIT(28) +#define ZR36057_ISR_JPEG_REP_IRQ BIT(27) #define ZR36057_ICR 0x040 /* Interrupt Control Register */ -#define ZR36057_ICR_GIRQ1 BIT(30) -#define ZR36057_ICR_GIRQ0 BIT(29) -#define ZR36057_ICR_COD_REP_IRQ BIT(28) -#define ZR36057_ICR_JPEG_REP_IRQ BIT(27) -#define ZR36057_ICR_INT_PIN_EN BIT(24) +#define ZR36057_ICR_GIRQ1 BIT(30) +#define ZR36057_ICR_GIRQ0 BIT(29) +#define ZR36057_ICR_COD_REP_IRQ BIT(28) +#define ZR36057_ICR_JPEG_REP_IRQ BIT(27) +#define ZR36057_ICR_INT_PIN_EN BIT(24) #define ZR36057_I2CBR 0x044 /* I2C Bus Register */ -#define ZR36057_I2CBR_SDA BIT(1) -#define ZR36057_I2CBR_SCL BIT(0) +#define ZR36057_I2CBR_SDA BIT(1) +#define ZR36057_I2CBR_SCL BIT(0) #define ZR36057_JMC 0x100 /* JPEG Mode and Control */ -#define ZR36057_JMC_JPG BIT(31) -#define ZR36057_JMC_JPG_EXP_MODE (0 << 29) -#define ZR36057_JMC_JPG_CMP_MODE BIT(29) -#define ZR36057_JMC_MJPG_EXP_MODE (2 << 29) -#define ZR36057_JMC_MJPG_CMP_MODE (3 << 29) -#define ZR36057_JMC_RTBUSY_FB BIT(6) -#define ZR36057_JMC_GO_EN BIT(5) -#define ZR36057_JMC_SYNC_MSTR BIT(4) -#define ZR36057_JMC_FLD_PER_BUFF BIT(3) -#define ZR36057_JMC_VFIFO_FB BIT(2) -#define ZR36057_JMC_CFIFO_FB BIT(1) -#define ZR36057_JMC_STLL_LIT_ENDIAN BIT(0) +#define ZR36057_JMC_JPG BIT(31) +#define ZR36057_JMC_JPG_EXP_MODE (0 << 29) +#define ZR36057_JMC_JPG_CMP_MODE BIT(29) +#define ZR36057_JMC_MJPG_EXP_MODE (2 << 29) +#define ZR36057_JMC_MJPG_CMP_MODE (3 << 29) +#define ZR36057_JMC_RTBUSY_FB BIT(6) +#define ZR36057_JMC_GO_EN BIT(5) +#define ZR36057_JMC_SYNC_MSTR BIT(4) +#define ZR36057_JMC_FLD_PER_BUFF BIT(3) +#define ZR36057_JMC_VFIFO_FB BIT(2) +#define ZR36057_JMC_CFIFO_FB BIT(1) +#define ZR36057_JMC_STLL_LIT_ENDIAN BIT(0) #define ZR36057_JPC 0x104 /* JPEG Process Control */ -#define ZR36057_JPC_P_RESET BIT(7) -#define ZR36057_JPC_COD_TRNS_EN BIT(5) -#define ZR36057_JPC_ACTIVE BIT(0) +#define ZR36057_JPC_P_RESET BIT(7) +#define ZR36057_JPC_COD_TRNS_EN BIT(5) +#define ZR36057_JPC_ACTIVE BIT(0) #define ZR36057_VSP 0x108 /* Vertical Sync Parameters */ -#define ZR36057_VSP_VSYNC_SIZE 16 -#define ZR36057_VSP_FRM_TOT 0 +#define ZR36057_VSP_VSYNC_SIZE 16 +#define ZR36057_VSP_FRM_TOT 0 #define ZR36057_HSP 0x10c /* Horizontal Sync Parameters */ -#define ZR36057_HSP_HSYNC_START 16 -#define ZR36057_HSP_LINE_TOT 0 +#define ZR36057_HSP_HSYNC_START 16 +#define ZR36057_HSP_LINE_TOT 0 #define ZR36057_FHAP 0x110 /* Field Horizontal Active Portion */ #define ZR36057_FHAP_NAX 16 @@ -132,22 +132,22 @@ #define ZR36057_FVAP_PAY 0 #define ZR36057_FPP 0x118 /* Field Process Parameters */ -#define ZR36057_FPP_ODD_EVEN BIT(0) +#define ZR36057_FPP_ODD_EVEN BIT(0) #define ZR36057_JCBA 0x11c /* JPEG Code Base Address */ #define ZR36057_JCFT 0x120 /* JPEG Code FIFO Threshold */ #define ZR36057_JCGI 0x124 /* JPEG Codec Guest ID */ -#define ZR36057_JCGI_JPE_GUEST_ID 4 -#define ZR36057_JCGI_JPE_GUEST_REG 0 +#define ZR36057_JCGI_JPE_GUEST_ID 4 +#define ZR36057_JCGI_JPE_GUEST_REG 0 #define ZR36057_GCR2 0x12c /* GuestBus Control Register (2) */ #define ZR36057_POR 0x200 /* Post Office Register */ -#define ZR36057_POR_PO_PEN BIT(25) -#define ZR36057_POR_PO_TIME BIT(24) -#define ZR36057_POR_PO_DIR BIT(23) +#define ZR36057_POR_PO_PEN BIT(25) +#define ZR36057_POR_PO_TIME BIT(24) +#define ZR36057_POR_PO_DIR BIT(23) #define ZR36057_STR 0x300 /* "Still" Transfer Register */ diff --git a/drivers/staging/media/zoran/zr36060.c b/drivers/media/pci/zoran/zr36060.c similarity index 99% rename from drivers/staging/media/zoran/zr36060.c rename to drivers/media/pci/zoran/zr36060.c index 7798016f1f96271e8e80a2c289b1d23bfd81e76f..75fd167603dcb7053b91d56ea0a2c7f65d3b20ba 100644 --- a/drivers/staging/media/zoran/zr36060.c +++ b/drivers/media/pci/zoran/zr36060.c @@ -243,7 +243,10 @@ static const char zr36060_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 }; //table idx's AC static const char zr36060_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 }; static const char zr36060_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 }; -/* SOF (start of frame) segment depends on width, height and sampling ratio of each color component */ +/* + * SOF (start of frame) segment depends on width, height and sampling ratio + * of each color component + */ static int zr36060_set_sof(struct zr36060 *ptr) { struct zoran *zr = videocodec_to_zoran(ptr->codec); @@ -555,8 +558,6 @@ static int zr36060_set_video(struct videocodec *codec, const struct tvnorm *norm reg = 6 - 1; /* VsyncSize */ zr36060_write(ptr, ZR060_SGR_VSYNC, reg); - //reg = 30 - 1; /* HsyncSize */ -///*CP*/ reg = (zr->params.norm == 1 ? 57 : 68); reg = 68; zr36060_write(ptr, ZR060_SGR_HSYNC, reg); diff --git a/drivers/staging/media/zoran/zr36060.h b/drivers/media/pci/zoran/zr36060.h similarity index 72% rename from drivers/staging/media/zoran/zr36060.h rename to drivers/media/pci/zoran/zr36060.h index fbf5429534ac48f07c85cdf5a2aac01105d7a8c8..75c88677a4bded04ad816ad1fa66fdc6e164535f 100644 --- a/drivers/staging/media/zoran/zr36060.h +++ b/drivers/media/pci/zoran/zr36060.h @@ -124,78 +124,78 @@ struct zr36060 { /* ZR36060 LOAD register bits */ -#define ZR060_LOAD_LOAD BIT(7) -#define ZR060_LOAD_SYNC_RST BIT(0) +#define ZR060_LOAD_LOAD BIT(7) +#define ZR060_LOAD_SYNC_RST BIT(0) /* ZR36060 Code FIFO Status register bits */ -#define ZR060_CFSR_BUSY BIT(7) -#define ZR060_CFSR_C_BUSY BIT(2) +#define ZR060_CFSR_BUSY BIT(7) +#define ZR060_CFSR_C_BUSY BIT(2) #define ZR060_CFSR_CFIFO (3 << 0) /* ZR36060 Code Interface register */ -#define ZR060_CIR_CODE16 BIT(7) -#define ZR060_CIR_ENDIAN BIT(6) -#define ZR060_CIR_CFIS BIT(2) -#define ZR060_CIR_CODE_MSTR BIT(0) +#define ZR060_CIR_CODE16 BIT(7) +#define ZR060_CIR_ENDIAN BIT(6) +#define ZR060_CIR_CFIS BIT(2) +#define ZR060_CIR_CODE_MSTR BIT(0) /* ZR36060 Codec Mode register */ -#define ZR060_CMR_COMP BIT(7) -#define ZR060_CMR_ATP BIT(6) -#define ZR060_CMR_PASS2 BIT(5) -#define ZR060_CMR_TLM BIT(4) -#define ZR060_CMR_BRB BIT(2) -#define ZR060_CMR_FSF BIT(1) +#define ZR060_CMR_COMP BIT(7) +#define ZR060_CMR_ATP BIT(6) +#define ZR060_CMR_PASS2 BIT(5) +#define ZR060_CMR_TLM BIT(4) +#define ZR060_CMR_BRB BIT(2) +#define ZR060_CMR_FSF BIT(1) /* ZR36060 Markers Enable register */ -#define ZR060_MER_APP BIT(7) -#define ZR060_MER_COM BIT(6) -#define ZR060_MER_DRI BIT(5) -#define ZR060_MER_DQT BIT(4) -#define ZR060_MER_DHT BIT(3) +#define ZR060_MER_APP BIT(7) +#define ZR060_MER_COM BIT(6) +#define ZR060_MER_DRI BIT(5) +#define ZR060_MER_DQT BIT(4) +#define ZR060_MER_DHT BIT(3) /* ZR36060 Interrupt Mask register */ -#define ZR060_IMR_EOAV BIT(3) -#define ZR060_IMR_EOI BIT(2) -#define ZR060_IMR_END BIT(1) -#define ZR060_IMR_DATA_ERR BIT(0) +#define ZR060_IMR_EOAV BIT(3) +#define ZR060_IMR_EOI BIT(2) +#define ZR060_IMR_END BIT(1) +#define ZR060_IMR_DATA_ERR BIT(0) /* ZR36060 Interrupt Status register */ #define ZR060_ISR_PRO_CNT (3 << 6) -#define ZR060_ISR_EOAV BIT(3) -#define ZR060_ISR_EOI BIT(2) -#define ZR060_ISR_END BIT(1) -#define ZR060_ISR_DATA_ERR BIT(0) +#define ZR060_ISR_EOAV BIT(3) +#define ZR060_ISR_EOI BIT(2) +#define ZR060_ISR_END BIT(1) +#define ZR060_ISR_DATA_ERR BIT(0) /* ZR36060 Video Control register */ -#define ZR060_VCR_VIDEO8 BIT(7) -#define ZR060_VCR_RANGE BIT(6) -#define ZR060_VCR_FI_DET BIT(3) -#define ZR060_VCR_FI_VEDGE BIT(2) -#define ZR060_VCR_FI_EXT BIT(1) -#define ZR060_VCR_SYNC_MSTR BIT(0) +#define ZR060_VCR_VIDEO8 BIT(7) +#define ZR060_VCR_RANGE BIT(6) +#define ZR060_VCR_FI_DET BIT(3) +#define ZR060_VCR_FI_VEDGE BIT(2) +#define ZR060_VCR_FI_EXT BIT(1) +#define ZR060_VCR_SYNC_MSTR BIT(0) /* ZR36060 Video Polarity register */ -#define ZR060_VPR_VCLK_POL BIT(7) -#define ZR060_VPR_P_VAL_POL BIT(6) -#define ZR060_VPR_POE_POL BIT(5) -#define ZR060_VPR_S_IMG_POL BIT(4) -#define ZR060_VPR_BL_POL BIT(3) -#define ZR060_VPR_FI_POL BIT(2) -#define ZR060_VPR_HS_POL BIT(1) -#define ZR060_VPR_VS_POL BIT(0) +#define ZR060_VPR_VCLK_POL BIT(7) +#define ZR060_VPR_P_VAL_POL BIT(6) +#define ZR060_VPR_POE_POL BIT(5) +#define ZR060_VPR_S_IMG_POL BIT(4) +#define ZR060_VPR_BL_POL BIT(3) +#define ZR060_VPR_FI_POL BIT(2) +#define ZR060_VPR_HS_POL BIT(1) +#define ZR060_VPR_VS_POL BIT(0) /* ZR36060 Scaling register */ -#define ZR060_SR_V_SCALE BIT(2) -#define ZR060_SR_H_SCALE2 BIT(0) +#define ZR060_SR_V_SCALE BIT(2) +#define ZR060_SR_H_SCALE2 BIT(0) #define ZR060_SR_H_SCALE4 (2 << 0) int zr36060_init_module(void); diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index f1056ceaf5a8c57cac1d23c0e9e5595f80e9b16c..a9334263fa9baea627bbea19664886b3ee60e53b 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -81,6 +81,7 @@ source "drivers/media/platform/samsung/Kconfig" source "drivers/media/platform/st/Kconfig" source "drivers/media/platform/sunxi/Kconfig" source "drivers/media/platform/ti/Kconfig" +source "drivers/media/platform/verisilicon/Kconfig" source "drivers/media/platform/via/Kconfig" source "drivers/media/platform/xilinx/Kconfig" diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index a881e97bae950c2430bf0914d84460e9e2578b0c..a91f42024273d7d6044fd0599c5a5f08c7e5113e 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -24,6 +24,7 @@ obj-y += samsung/ obj-y += st/ obj-y += sunxi/ obj-y += ti/ +obj-y += verisilicon/ obj-y += via/ obj-y += xilinx/ diff --git a/drivers/media/platform/amlogic/meson-ge2d/ge2d.c b/drivers/media/platform/amlogic/meson-ge2d/ge2d.c index 5e7b319f300dff2ec8042b9ae318315301e3f6ec..142d421a8d769f8e86369a5f1518396ad3095099 100644 --- a/drivers/media/platform/amlogic/meson-ge2d/ge2d.c +++ b/drivers/media/platform/amlogic/meson-ge2d/ge2d.c @@ -1030,7 +1030,6 @@ static int ge2d_remove(struct platform_device *pdev) video_unregister_device(ge2d->vfd); v4l2_m2m_release(ge2d->m2m_dev); - video_device_release(ge2d->vfd); v4l2_device_unregister(&ge2d->v4l2_dev); clk_disable_unprepare(ge2d->clk); diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c index 9e64041cc1c1d95b95f8213bd3bc5ffe54375411..feb75dc204de83963c779104805ec9cfdf0476b5 100644 --- a/drivers/media/platform/amphion/vdec.c +++ b/drivers/media/platform/amphion/vdec.c @@ -808,14 +808,6 @@ static void vdec_init_fmt(struct vpu_inst *inst) inst->cap_format.field = V4L2_FIELD_NONE; else inst->cap_format.field = V4L2_FIELD_SEQ_TB; - if (vdec->codec_info.color_primaries == V4L2_COLORSPACE_DEFAULT) - vdec->codec_info.color_primaries = V4L2_COLORSPACE_REC709; - if (vdec->codec_info.transfer_chars == V4L2_XFER_FUNC_DEFAULT) - vdec->codec_info.transfer_chars = V4L2_XFER_FUNC_709; - if (vdec->codec_info.matrix_coeffs == V4L2_YCBCR_ENC_DEFAULT) - vdec->codec_info.matrix_coeffs = V4L2_YCBCR_ENC_709; - if (vdec->codec_info.full_range == V4L2_QUANTIZATION_DEFAULT) - vdec->codec_info.full_range = V4L2_QUANTIZATION_LIM_RANGE; } static void vdec_init_crop(struct vpu_inst *inst) @@ -1555,6 +1547,14 @@ static int vdec_get_debug_info(struct vpu_inst *inst, char *str, u32 size, u32 i vdec->codec_info.frame_rate.numerator, vdec->codec_info.frame_rate.denominator); break; + case 9: + num = scnprintf(str, size, "colorspace: %d, %d, %d, %d (%d)\n", + vdec->codec_info.color_primaries, + vdec->codec_info.transfer_chars, + vdec->codec_info.matrix_coeffs, + vdec->codec_info.full_range, + vdec->codec_info.vui_present); + break; default: break; } diff --git a/drivers/media/platform/amphion/venc.c b/drivers/media/platform/amphion/venc.c index 461524dd1e441423f1ffa952a54fbadccc119b96..37212f087fdd983a224378a49da37394fdb17b2c 100644 --- a/drivers/media/platform/amphion/venc.c +++ b/drivers/media/platform/amphion/venc.c @@ -644,7 +644,7 @@ static int venc_ctrl_init(struct vpu_inst *inst) BITRATE_DEFAULT_PEAK); v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops, - V4L2_CID_MPEG_VIDEO_GOP_SIZE, 0, (1 << 16) - 1, 1, 30); + V4L2_CID_MPEG_VIDEO_GOP_SIZE, 1, 8000, 1, 30); v4l2_ctrl_new_std(&inst->ctrl_handler, &venc_ctrl_ops, V4L2_CID_MPEG_VIDEO_B_FRAMES, 0, 4, 1, 0); diff --git a/drivers/media/platform/amphion/vpu.h b/drivers/media/platform/amphion/vpu.h index f914de6ed81e9941ed1649b16760510b065f265c..beac0309ca8d9bf4f744e6bc3a9446eab762b97d 100644 --- a/drivers/media/platform/amphion/vpu.h +++ b/drivers/media/platform/amphion/vpu.h @@ -119,7 +119,6 @@ struct vpu_mbox { enum vpu_core_state { VPU_CORE_DEINIT = 0, VPU_CORE_ACTIVE, - VPU_CORE_SNAPSHOT, VPU_CORE_HANG }; diff --git a/drivers/media/platform/amphion/vpu_core.c b/drivers/media/platform/amphion/vpu_core.c index 73faa50d28653b8afc4b94538725c8c84f271dd7..f9ec1753f7c86f365047f9d6913907ddb1533d07 100644 --- a/drivers/media/platform/amphion/vpu_core.c +++ b/drivers/media/platform/amphion/vpu_core.c @@ -89,7 +89,7 @@ static int vpu_core_boot_done(struct vpu_core *core) core->supported_instance_count = min(core->supported_instance_count, count); } core->fw_version = fw_version; - core->state = VPU_CORE_ACTIVE; + vpu_core_set_state(core, VPU_CORE_ACTIVE); return 0; } @@ -172,10 +172,26 @@ int vpu_alloc_dma(struct vpu_core *core, struct vpu_buffer *buf) return __vpu_alloc_dma(core->dev, buf); } -static void vpu_core_check_hang(struct vpu_core *core) +void vpu_core_set_state(struct vpu_core *core, enum vpu_core_state state) { - if (core->hang_mask) - core->state = VPU_CORE_HANG; + if (state != core->state) + vpu_trace(core->dev, "vpu core state change from %d to %d\n", core->state, state); + core->state = state; + if (core->state == VPU_CORE_DEINIT) + core->hang_mask = 0; +} + +static void vpu_core_update_state(struct vpu_core *core) +{ + if (!vpu_iface_get_power_state(core)) { + if (core->request_count) + vpu_core_set_state(core, VPU_CORE_HANG); + else + vpu_core_set_state(core, VPU_CORE_DEINIT); + + } else if (core->state == VPU_CORE_ACTIVE && core->hang_mask) { + vpu_core_set_state(core, VPU_CORE_HANG); + } } static struct vpu_core *vpu_core_find_proper_by_type(struct vpu_dev *vpu, u32 type) @@ -188,11 +204,13 @@ static struct vpu_core *vpu_core_find_proper_by_type(struct vpu_dev *vpu, u32 ty dev_dbg(c->dev, "instance_mask = 0x%lx, state = %d\n", c->instance_mask, c->state); if (c->type != type) continue; + mutex_lock(&c->lock); + vpu_core_update_state(c); + mutex_unlock(&c->lock); if (c->state == VPU_CORE_DEINIT) { core = c; break; } - vpu_core_check_hang(c); if (c->state != VPU_CORE_ACTIVE) continue; if (c->request_count < request_count) { @@ -409,6 +427,12 @@ int vpu_inst_register(struct vpu_inst *inst) } mutex_lock(&core->lock); + if (core->state != VPU_CORE_ACTIVE) { + dev_err(core->dev, "vpu core is not active, state = %d\n", core->state); + ret = -EINVAL; + goto exit; + } + if (inst->id >= 0 && inst->id < core->supported_instance_count) goto exit; @@ -450,7 +474,7 @@ int vpu_inst_unregister(struct vpu_inst *inst) vpu_core_release_instance(core, inst->id); inst->id = VPU_INST_NULL_ID; } - vpu_core_check_hang(core); + vpu_core_update_state(core); if (core->state == VPU_CORE_HANG && !core->instance_mask) { int err; @@ -459,7 +483,7 @@ int vpu_inst_unregister(struct vpu_inst *inst) err = vpu_core_sw_reset(core); mutex_lock(&core->lock); if (!err) { - core->state = VPU_CORE_ACTIVE; + vpu_core_set_state(core, VPU_CORE_ACTIVE); core->hang_mask = 0; } } @@ -609,7 +633,7 @@ static int vpu_core_probe(struct platform_device *pdev) mutex_init(&core->cmd_lock); init_completion(&core->cmp); init_waitqueue_head(&core->ack_wq); - core->state = VPU_CORE_DEINIT; + vpu_core_set_state(core, VPU_CORE_DEINIT); core->res = of_device_get_match_data(dev); if (!core->res) @@ -758,33 +782,18 @@ static int __maybe_unused vpu_core_resume(struct device *dev) mutex_lock(&core->lock); pm_runtime_resume_and_get(dev); vpu_core_get_vpu(core); - if (core->state != VPU_CORE_SNAPSHOT) - goto exit; - if (!vpu_iface_get_power_state(core)) { - if (!list_empty(&core->instances)) { + if (core->request_count) { + if (!vpu_iface_get_power_state(core)) ret = vpu_core_boot(core, false); - if (ret) { - dev_err(core->dev, "%s boot fail\n", __func__); - core->state = VPU_CORE_DEINIT; - goto exit; - } - } else { - core->state = VPU_CORE_DEINIT; - } - } else { - if (!list_empty(&core->instances)) { + else ret = vpu_core_sw_reset(core); - if (ret) { - dev_err(core->dev, "%s sw_reset fail\n", __func__); - core->state = VPU_CORE_HANG; - goto exit; - } + if (ret) { + dev_err(core->dev, "resume fail\n"); + vpu_core_set_state(core, VPU_CORE_HANG); } - core->state = VPU_CORE_ACTIVE; } - -exit: + vpu_core_update_state(core); pm_runtime_put_sync(dev); mutex_unlock(&core->lock); @@ -798,18 +807,11 @@ static int __maybe_unused vpu_core_suspend(struct device *dev) int ret = 0; mutex_lock(&core->lock); - if (core->state == VPU_CORE_ACTIVE) { - if (!list_empty(&core->instances)) { - ret = vpu_core_snapshot(core); - if (ret) { - mutex_unlock(&core->lock); - return ret; - } - } - - core->state = VPU_CORE_SNAPSHOT; - } + if (core->request_count) + ret = vpu_core_snapshot(core); mutex_unlock(&core->lock); + if (ret) + return ret; vpu_core_cancel_work(core); diff --git a/drivers/media/platform/amphion/vpu_core.h b/drivers/media/platform/amphion/vpu_core.h index 00a662997da4fdc903e501595314d91863902fc1..65b562642603a5cdc0f3ba13b817037578408a1d 100644 --- a/drivers/media/platform/amphion/vpu_core.h +++ b/drivers/media/platform/amphion/vpu_core.h @@ -11,5 +11,6 @@ u32 csr_readl(struct vpu_core *core, u32 reg); int vpu_alloc_dma(struct vpu_core *core, struct vpu_buffer *buf); void vpu_free_dma(struct vpu_buffer *buf); struct vpu_inst *vpu_core_find_instance(struct vpu_core *core, u32 index); +void vpu_core_set_state(struct vpu_core *core, enum vpu_core_state state); #endif diff --git a/drivers/media/platform/amphion/vpu_dbg.c b/drivers/media/platform/amphion/vpu_dbg.c index f72c8a506b2205e840f7814e35ccd6bd0f18d5eb..260f1c4b8f8dc5b7f5db48bdc17e820df375df60 100644 --- a/drivers/media/platform/amphion/vpu_dbg.c +++ b/drivers/media/platform/amphion/vpu_dbg.c @@ -15,6 +15,7 @@ #include #include "vpu.h" #include "vpu_defs.h" +#include "vpu_core.h" #include "vpu_helpers.h" #include "vpu_cmds.h" #include "vpu_rpc.h" @@ -233,6 +234,10 @@ static int vpu_dbg_core(struct seq_file *s, void *data) if (seq_write(s, str, num)) return 0; + num = scnprintf(str, sizeof(str), "power %s\n", + vpu_iface_get_power_state(core) ? "on" : "off"); + if (seq_write(s, str, num)) + return 0; num = scnprintf(str, sizeof(str), "state = %d\n", core->state); if (seq_write(s, str, num)) return 0; @@ -346,10 +351,10 @@ static ssize_t vpu_dbg_core_write(struct file *file, pm_runtime_resume_and_get(core->dev); mutex_lock(&core->lock); - if (core->state != VPU_CORE_DEINIT && !core->instance_mask) { + if (vpu_iface_get_power_state(core) && !core->request_count) { dev_info(core->dev, "reset\n"); if (!vpu_core_sw_reset(core)) { - core->state = VPU_CORE_ACTIVE; + vpu_core_set_state(core, VPU_CORE_ACTIVE); core->hang_mask = 0; } } diff --git a/drivers/media/platform/amphion/vpu_malone.c b/drivers/media/platform/amphion/vpu_malone.c index f4a488bf988014c1b888a26e7e435d8000680e87..51e0702f9ae17a7308621783384d8816f51f34c7 100644 --- a/drivers/media/platform/amphion/vpu_malone.c +++ b/drivers/media/platform/amphion/vpu_malone.c @@ -1293,7 +1293,7 @@ static int vpu_malone_insert_scode_vc1_g_pic(struct malone_scode_t *scode) vbuf = to_vb2_v4l2_buffer(scode->vb); data = vb2_plane_vaddr(scode->vb, 0); - if (vbuf->sequence == 0 || vpu_vb_is_codecconfig(vbuf)) + if (scode->inst->total_input_count == 0 || vpu_vb_is_codecconfig(vbuf)) return 0; if (MALONE_VC1_CONTAIN_NAL(*data)) return 0; diff --git a/drivers/media/platform/intel/pxa_camera.c b/drivers/media/platform/intel/pxa_camera.c index 35145e3348f0e62a78a73a49378822b125aac4c6..54270d6b6f50fe9128069ff2729f4d7d68795d0a 100644 --- a/drivers/media/platform/intel/pxa_camera.c +++ b/drivers/media/platform/intel/pxa_camera.c @@ -854,7 +854,7 @@ fail: return -ENOMEM; } -static void pxa_videobuf_set_actdma(struct pxa_camera_dev *pcdev, +static void pxa_video_buf_set_actdma(struct pxa_camera_dev *pcdev, struct pxa_buffer *buf) { buf->active_dma = DMA_Y; @@ -973,7 +973,7 @@ static void pxa_camera_wakeup(struct pxa_camera_dev *pcdev, * stopped. This means the tailed buffer would never be transferred by DMA. * This function restarts the capture for this corner case, where : * - DADR() == DADDR_STOP - * - a videobuffer is queued on the pcdev->capture list + * - a video buffer is queued on the pcdev->capture list * * Please check the "DMA hot chaining timeslice issue" in * Documentation/driver-api/media/drivers/pxa_camera.rst @@ -1163,7 +1163,7 @@ static void pxa_camera_eof(struct tasklet_struct *t) pcdev->active = list_first_entry(&pcdev->capture, struct pxa_buffer, queue); buf = pcdev->active; - pxa_videobuf_set_actdma(pcdev, buf); + pxa_video_buf_set_actdma(pcdev, buf); pxa_dma_start_channels(pcdev); } @@ -1416,7 +1416,7 @@ static int pxac_vb2_prepare(struct vb2_buffer *vb) * the actual buffer is yours */ buf->inwork = 0; - pxa_videobuf_set_actdma(pcdev, buf); + pxa_video_buf_set_actdma(pcdev, buf); return ret; } diff --git a/drivers/media/platform/marvell/mcam-core.h b/drivers/media/platform/marvell/mcam-core.h index f324d808d737601bf659933acfc0cf5858737052..51e66db45af69caf57685e9ccd340ca2659f9e5a 100644 --- a/drivers/media/platform/marvell/mcam-core.h +++ b/drivers/media/platform/marvell/mcam-core.h @@ -32,7 +32,7 @@ #if !defined(MCAM_MODE_VMALLOC) && !defined(MCAM_MODE_DMA_CONTIG) && \ !defined(MCAM_MODE_DMA_SG) -#error One of the videobuf buffer modes must be selected in the config +#error One of the vb2 buffer modes must be selected in the config #endif diff --git a/drivers/media/platform/mediatek/Kconfig b/drivers/media/platform/mediatek/Kconfig index af47d9888552472cc4bd11f04880355c27cf29a5..84104e2cd02447790ae5c29953a2e82ca4fdd0a7 100644 --- a/drivers/media/platform/mediatek/Kconfig +++ b/drivers/media/platform/mediatek/Kconfig @@ -6,3 +6,4 @@ source "drivers/media/platform/mediatek/jpeg/Kconfig" source "drivers/media/platform/mediatek/mdp/Kconfig" source "drivers/media/platform/mediatek/vcodec/Kconfig" source "drivers/media/platform/mediatek/vpu/Kconfig" +source "drivers/media/platform/mediatek/mdp3/Kconfig" diff --git a/drivers/media/platform/mediatek/Makefile b/drivers/media/platform/mediatek/Makefile index d3850a13f128b220b8edfc6e935d99defa2c3b91..38e6ba917fe5cdd932aa6c88221c9a7aa5a7705a 100644 --- a/drivers/media/platform/mediatek/Makefile +++ b/drivers/media/platform/mediatek/Makefile @@ -3,3 +3,4 @@ obj-y += jpeg/ obj-y += mdp/ obj-y += vcodec/ obj-y += vpu/ +obj-y += mdp3/ diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c index 87685a62a5c237511d1d32b377b3ced4043c4fa9..3071b61946c3bff985224dd05f7ae211e8b744a3 100644 --- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c +++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c @@ -1414,7 +1414,6 @@ static int mtk_jpeg_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); video_unregister_device(jpeg->vdev); - video_device_release(jpeg->vdev); v4l2_m2m_release(jpeg->m2m_dev); v4l2_device_unregister(&jpeg->v4l2_dev); diff --git a/drivers/media/platform/mediatek/mdp3/Kconfig b/drivers/media/platform/mediatek/mdp3/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..50ae07b75b5f25eb917139f806fed1302a5041e4 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VIDEO_MEDIATEK_MDP3 + tristate "MediaTek MDP v3 driver" + depends on MTK_IOMMU || COMPILE_TEST + depends on VIDEO_DEV + depends on ARCH_MEDIATEK || COMPILE_TEST + depends on HAS_DMA + depends on REMOTEPROC + select VIDEOBUF2_DMA_CONTIG + select V4L2_MEM2MEM_DEV + select MTK_MMSYS + select VIDEO_MEDIATEK_VPU + select MTK_CMDQ + select MTK_SCP + default n + help + It is a v4l2 driver and present in MediaTek MT8183 SoC. + The driver supports scaling and color space conversion. + + To compile this driver as a module, choose M here: the + module will be called mtk-mdp3. diff --git a/drivers/media/platform/mediatek/mdp3/Makefile b/drivers/media/platform/mediatek/mdp3/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..63e6c87e480b4cc57e5a9850a9f51079e1252c78 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +mtk-mdp3-y += mtk-mdp3-core.o mtk-mdp3-vpu.o mtk-mdp3-regs.o +mtk-mdp3-y += mtk-mdp3-m2m.o +mtk-mdp3-y += mtk-mdp3-comp.o mtk-mdp3-cmdq.o + +obj-$(CONFIG_VIDEO_MEDIATEK_MDP3) += mtk-mdp3.o diff --git a/drivers/media/platform/mediatek/mdp3/mdp_reg_ccorr.h b/drivers/media/platform/mediatek/mdp3/mdp_reg_ccorr.h new file mode 100644 index 0000000000000000000000000000000000000000..3b2c6531c1943a19b692c48cba3cbd114a94ce02 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mdp_reg_ccorr.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MDP_REG_CCORR_H__ +#define __MDP_REG_CCORR_H__ + +#define MDP_CCORR_EN 0x000 +#define MDP_CCORR_CFG 0x020 +#define MDP_CCORR_SIZE 0x030 + +/* MASK */ +#define MDP_CCORR_EN_MASK 0x00000001 +#define MDP_CCORR_CFG_MASK 0x70001317 +#define MDP_CCORR_SIZE_MASK 0x1fff1fff + +#endif // __MDP_REG_CCORR_H__ diff --git a/drivers/media/platform/mediatek/mdp3/mdp_reg_rdma.h b/drivers/media/platform/mediatek/mdp3/mdp_reg_rdma.h new file mode 100644 index 0000000000000000000000000000000000000000..be4065e252d3d19a9fe1cacfc262fc27df276026 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mdp_reg_rdma.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MDP_REG_RDMA_H__ +#define __MDP_REG_RDMA_H__ + +#define MDP_RDMA_EN 0x000 +#define MDP_RDMA_RESET 0x008 +#define MDP_RDMA_CON 0x020 +#define MDP_RDMA_GMCIF_CON 0x028 +#define MDP_RDMA_SRC_CON 0x030 +#define MDP_RDMA_MF_BKGD_SIZE_IN_BYTE 0x060 +#define MDP_RDMA_MF_BKGD_SIZE_IN_PXL 0x068 +#define MDP_RDMA_MF_SRC_SIZE 0x070 +#define MDP_RDMA_MF_CLIP_SIZE 0x078 +#define MDP_RDMA_MF_OFFSET_1 0x080 +#define MDP_RDMA_SF_BKGD_SIZE_IN_BYTE 0x090 +#define MDP_RDMA_SRC_END_0 0x100 +#define MDP_RDMA_SRC_END_1 0x108 +#define MDP_RDMA_SRC_END_2 0x110 +#define MDP_RDMA_SRC_OFFSET_0 0x118 +#define MDP_RDMA_SRC_OFFSET_1 0x120 +#define MDP_RDMA_SRC_OFFSET_2 0x128 +#define MDP_RDMA_SRC_OFFSET_0_P 0x148 +#define MDP_RDMA_TRANSFORM_0 0x200 +#define MDP_RDMA_RESV_DUMMY_0 0x2a0 +#define MDP_RDMA_MON_STA_1 0x408 +#define MDP_RDMA_SRC_BASE_0 0xf00 +#define MDP_RDMA_SRC_BASE_1 0xf08 +#define MDP_RDMA_SRC_BASE_2 0xf10 +#define MDP_RDMA_UFO_DEC_LENGTH_BASE_Y 0xf20 +#define MDP_RDMA_UFO_DEC_LENGTH_BASE_C 0xf28 + +/* MASK */ +#define MDP_RDMA_EN_MASK 0x00000001 +#define MDP_RDMA_RESET_MASK 0x00000001 +#define MDP_RDMA_CON_MASK 0x00001110 +#define MDP_RDMA_GMCIF_CON_MASK 0xfffb3771 +#define MDP_RDMA_SRC_CON_MASK 0xf3ffffff +#define MDP_RDMA_MF_BKGD_SIZE_IN_BYTE_MASK 0x001fffff +#define MDP_RDMA_MF_BKGD_SIZE_IN_PXL_MASK 0x001fffff +#define MDP_RDMA_MF_SRC_SIZE_MASK 0x1fff1fff +#define MDP_RDMA_MF_CLIP_SIZE_MASK 0x1fff1fff +#define MDP_RDMA_MF_OFFSET_1_MASK 0x003f001f +#define MDP_RDMA_SF_BKGD_SIZE_IN_BYTE_MASK 0x001fffff +#define MDP_RDMA_SRC_END_0_MASK 0xffffffff +#define MDP_RDMA_SRC_END_1_MASK 0xffffffff +#define MDP_RDMA_SRC_END_2_MASK 0xffffffff +#define MDP_RDMA_SRC_OFFSET_0_MASK 0xffffffff +#define MDP_RDMA_SRC_OFFSET_1_MASK 0xffffffff +#define MDP_RDMA_SRC_OFFSET_2_MASK 0xffffffff +#define MDP_RDMA_SRC_OFFSET_0_P_MASK 0xffffffff +#define MDP_RDMA_TRANSFORM_0_MASK 0xff110777 +#define MDP_RDMA_RESV_DUMMY_0_MASK 0xffffffff +#define MDP_RDMA_MON_STA_1_MASK 0xffffffff +#define MDP_RDMA_SRC_BASE_0_MASK 0xffffffff +#define MDP_RDMA_SRC_BASE_1_MASK 0xffffffff +#define MDP_RDMA_SRC_BASE_2_MASK 0xffffffff +#define MDP_RDMA_UFO_DEC_LENGTH_BASE_Y_MASK 0xffffffff +#define MDP_RDMA_UFO_DEC_LENGTH_BASE_C_MASK 0xffffffff + +#endif // __MDP_REG_RDMA_H__ diff --git a/drivers/media/platform/mediatek/mdp3/mdp_reg_rsz.h b/drivers/media/platform/mediatek/mdp3/mdp_reg_rsz.h new file mode 100644 index 0000000000000000000000000000000000000000..484f6d60641ff5953df5eacbbc6f22d1adfbd614 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mdp_reg_rsz.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MDP_REG_RSZ_H__ +#define __MDP_REG_RSZ_H__ + +#define PRZ_ENABLE 0x000 +#define PRZ_CONTROL_1 0x004 +#define PRZ_CONTROL_2 0x008 +#define PRZ_INPUT_IMAGE 0x010 +#define PRZ_OUTPUT_IMAGE 0x014 +#define PRZ_HORIZONTAL_COEFF_STEP 0x018 +#define PRZ_VERTICAL_COEFF_STEP 0x01c +#define PRZ_LUMA_HORIZONTAL_INTEGER_OFFSET 0x020 +#define PRZ_LUMA_HORIZONTAL_SUBPIXEL_OFFSET 0x024 +#define PRZ_LUMA_VERTICAL_INTEGER_OFFSET 0x028 +#define PRZ_LUMA_VERTICAL_SUBPIXEL_OFFSET 0x02c +#define PRZ_CHROMA_HORIZONTAL_INTEGER_OFFSET 0x030 +#define PRZ_CHROMA_HORIZONTAL_SUBPIXEL_OFFSET 0x034 + +/* MASK */ +#define PRZ_ENABLE_MASK 0x00010001 +#define PRZ_CONTROL_1_MASK 0xfffffff3 +#define PRZ_CONTROL_2_MASK 0x0ffffaff +#define PRZ_INPUT_IMAGE_MASK 0xffffffff +#define PRZ_OUTPUT_IMAGE_MASK 0xffffffff +#define PRZ_HORIZONTAL_COEFF_STEP_MASK 0x007fffff +#define PRZ_VERTICAL_COEFF_STEP_MASK 0x007fffff +#define PRZ_LUMA_HORIZONTAL_INTEGER_OFFSET_MASK 0x0000ffff +#define PRZ_LUMA_HORIZONTAL_SUBPIXEL_OFFSET_MASK 0x001fffff +#define PRZ_LUMA_VERTICAL_INTEGER_OFFSET_MASK 0x0000ffff +#define PRZ_LUMA_VERTICAL_SUBPIXEL_OFFSET_MASK 0x001fffff +#define PRZ_CHROMA_HORIZONTAL_INTEGER_OFFSET_MASK 0x0000ffff +#define PRZ_CHROMA_HORIZONTAL_SUBPIXEL_OFFSET_MASK 0x001fffff + +#endif // __MDP_REG_RSZ_H__ diff --git a/drivers/media/platform/mediatek/mdp3/mdp_reg_wdma.h b/drivers/media/platform/mediatek/mdp3/mdp_reg_wdma.h new file mode 100644 index 0000000000000000000000000000000000000000..0280e91c09e432701e5ac9a514eb36c785b02043 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mdp_reg_wdma.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MDP_REG_WDMA_H__ +#define __MDP_REG_WDMA_H__ + +#define WDMA_EN 0x008 +#define WDMA_RST 0x00c +#define WDMA_CFG 0x014 +#define WDMA_SRC_SIZE 0x018 +#define WDMA_CLIP_SIZE 0x01c +#define WDMA_CLIP_COORD 0x020 +#define WDMA_DST_W_IN_BYTE 0x028 +#define WDMA_ALPHA 0x02c +#define WDMA_BUF_CON2 0x03c +#define WDMA_DST_UV_PITCH 0x078 +#define WDMA_DST_ADDR_OFFSET 0x080 +#define WDMA_DST_U_ADDR_OFFSET 0x084 +#define WDMA_DST_V_ADDR_OFFSET 0x088 +#define WDMA_FLOW_CTRL_DBG 0x0a0 +#define WDMA_DST_ADDR 0xf00 +#define WDMA_DST_U_ADDR 0xf04 +#define WDMA_DST_V_ADDR 0xf08 + +/* MASK */ +#define WDMA_EN_MASK 0x00000001 +#define WDMA_RST_MASK 0x00000001 +#define WDMA_CFG_MASK 0xff03bff0 +#define WDMA_SRC_SIZE_MASK 0x3fff3fff +#define WDMA_CLIP_SIZE_MASK 0x3fff3fff +#define WDMA_CLIP_COORD_MASK 0x3fff3fff +#define WDMA_DST_W_IN_BYTE_MASK 0x0000ffff +#define WDMA_ALPHA_MASK 0x800000ff +#define WDMA_BUF_CON2_MASK 0xffffffff +#define WDMA_DST_UV_PITCH_MASK 0x0000ffff +#define WDMA_DST_ADDR_OFFSET_MASK 0x0fffffff +#define WDMA_DST_U_ADDR_OFFSET_MASK 0x0fffffff +#define WDMA_DST_V_ADDR_OFFSET_MASK 0x0fffffff +#define WDMA_FLOW_CTRL_DBG_MASK 0x0000f3ff +#define WDMA_DST_ADDR_MASK 0xffffffff +#define WDMA_DST_U_ADDR_MASK 0xffffffff +#define WDMA_DST_V_ADDR_MASK 0xffffffff + +#endif // __MDP_REG_WDMA_H__ diff --git a/drivers/media/platform/mediatek/mdp3/mdp_reg_wrot.h b/drivers/media/platform/mediatek/mdp3/mdp_reg_wrot.h new file mode 100644 index 0000000000000000000000000000000000000000..6d3ff0e2b6720d3761a449cec3577fcc2c5cb629 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mdp_reg_wrot.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MDP_REG_WROT_H__ +#define __MDP_REG_WROT_H__ + +#define VIDO_CTRL 0x000 +#define VIDO_MAIN_BUF_SIZE 0x008 +#define VIDO_SOFT_RST 0x010 +#define VIDO_SOFT_RST_STAT 0x014 +#define VIDO_CROP_OFST 0x020 +#define VIDO_TAR_SIZE 0x024 +#define VIDO_OFST_ADDR 0x02c +#define VIDO_STRIDE 0x030 +#define VIDO_OFST_ADDR_C 0x038 +#define VIDO_STRIDE_C 0x03c +#define VIDO_DITHER 0x054 +#define VIDO_STRIDE_V 0x06c +#define VIDO_OFST_ADDR_V 0x068 +#define VIDO_RSV_1 0x070 +#define VIDO_IN_SIZE 0x078 +#define VIDO_ROT_EN 0x07c +#define VIDO_FIFO_TEST 0x080 +#define VIDO_MAT_CTRL 0x084 +#define VIDO_BASE_ADDR 0xf00 +#define VIDO_BASE_ADDR_C 0xf04 +#define VIDO_BASE_ADDR_V 0xf08 + +/* MASK */ +#define VIDO_CTRL_MASK 0xf530711f +#define VIDO_MAIN_BUF_SIZE_MASK 0x1fff7f77 +#define VIDO_SOFT_RST_MASK 0x00000001 +#define VIDO_SOFT_RST_STAT_MASK 0x00000001 +#define VIDO_TAR_SIZE_MASK 0x1fff1fff +#define VIDO_CROP_OFST_MASK 0x1fff1fff +#define VIDO_OFST_ADDR_MASK 0x0fffffff +#define VIDO_STRIDE_MASK 0x0000ffff +#define VIDO_OFST_ADDR_C_MASK 0x0fffffff +#define VIDO_STRIDE_C_MASK 0x0000ffff +#define VIDO_DITHER_MASK 0xff000001 +#define VIDO_STRIDE_V_MASK 0x0000ffff +#define VIDO_OFST_ADDR_V_MASK 0x0fffffff +#define VIDO_RSV_1_MASK 0xffffffff +#define VIDO_IN_SIZE_MASK 0x1fff1fff +#define VIDO_ROT_EN_MASK 0x00000001 +#define VIDO_FIFO_TEST_MASK 0x00000fff +#define VIDO_MAT_CTRL_MASK 0x000000f3 +#define VIDO_BASE_ADDR_MASK 0xffffffff +#define VIDO_BASE_ADDR_C_MASK 0xffffffff +#define VIDO_BASE_ADDR_V_MASK 0xffffffff + +#endif // __MDP_REG_WROT_H__ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h b/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h new file mode 100644 index 0000000000000000000000000000000000000000..3e66ebaee2da290397df947f4bb00c13ec964092 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-img-ipi.h @@ -0,0 +1,290 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Holmes Chiou + * Ping-Hsun Wu + */ + +#ifndef __MTK_IMG_IPI_H__ +#define __MTK_IMG_IPI_H__ + +#include + +/* + * ISP-MDP generic input information + * MD5 of the target SCP blob: + * 6da52bdcf4bf76a0983b313e1d4745d6 + */ + +#define IMG_MAX_HW_INPUTS 3 + +#define IMG_MAX_HW_OUTPUTS 4 + +#define IMG_MAX_PLANES 3 + +#define IMG_IPI_INIT 1 +#define IMG_IPI_DEINIT 2 +#define IMG_IPI_FRAME 3 +#define IMG_IPI_DEBUG 4 + +struct img_timeval { + u32 tv_sec; + u32 tv_usec; +} __packed; + +struct img_addr { + u64 va; /* Used for Linux OS access */ + u32 pa; /* Used for CM4 access */ + u32 iova; /* Used for IOMMU HW access */ +} __packed; + +struct tuning_addr { + u64 present; + u32 pa; /* Used for CM4 access */ + u32 iova; /* Used for IOMMU HW access */ +} __packed; + +struct img_sw_addr { + u64 va; /* Used for APMCU access */ + u32 pa; /* Used for CM4 access */ +} __packed; + +struct img_plane_format { + u32 size; + u16 stride; +} __packed; + +struct img_pix_format { + u16 width; + u16 height; + u32 colorformat; /* enum mdp_color */ + u16 ycbcr_prof; /* enum mdp_ycbcr_profile */ + struct img_plane_format plane_fmt[IMG_MAX_PLANES]; +} __packed; + +struct img_image_buffer { + struct img_pix_format format; + u32 iova[IMG_MAX_PLANES]; + /* enum mdp_buffer_usage, FD or advanced ISP usages */ + u32 usage; +} __packed; + +#define IMG_SUBPIXEL_SHIFT 20 + +struct img_crop { + s16 left; + s16 top; + u16 width; + u16 height; + u32 left_subpix; + u32 top_subpix; + u32 width_subpix; + u32 height_subpix; +} __packed; + +#define IMG_CTRL_FLAG_HFLIP BIT(0) +#define IMG_CTRL_FLAG_DITHER BIT(1) +#define IMG_CTRL_FLAG_SHARPNESS BIT(4) +#define IMG_CTRL_FLAG_HDR BIT(5) +#define IMG_CTRL_FLAG_DRE BIT(6) + +struct img_input { + struct img_image_buffer buffer; + u16 flags; /* HDR, DRE, dither */ +} __packed; + +struct img_output { + struct img_image_buffer buffer; + struct img_crop crop; + s16 rotation; + u16 flags; /* H-flip, sharpness, dither */ +} __packed; + +struct img_ipi_frameparam { + u32 index; + u32 frame_no; + struct img_timeval timestamp; + u8 type; /* enum mdp_stream_type */ + u8 state; + u8 num_inputs; + u8 num_outputs; + u64 drv_data; + struct img_input inputs[IMG_MAX_HW_INPUTS]; + struct img_output outputs[IMG_MAX_HW_OUTPUTS]; + struct tuning_addr tuning_data; + struct img_addr subfrm_data; + struct img_sw_addr config_data; + struct img_sw_addr self_data; +} __packed; + +struct img_sw_buffer { + u64 handle; /* Used for APMCU access */ + u32 scp_addr; /* Used for CM4 access */ +} __packed; + +struct img_ipi_param { + u8 usage; + struct img_sw_buffer frm_param; +} __packed; + +struct img_frameparam { + struct list_head list_entry; + struct img_ipi_frameparam frameparam; +}; + +/* ISP-MDP generic output information */ + +struct img_comp_frame { + u32 output_disable:1; + u32 bypass:1; + u16 in_width; + u16 in_height; + u16 out_width; + u16 out_height; + struct img_crop crop; + u16 in_total_width; + u16 out_total_width; +} __packed; + +struct img_region { + s16 left; + s16 right; + s16 top; + s16 bottom; +} __packed; + +struct img_offset { + s16 left; + s16 top; + u32 left_subpix; + u32 top_subpix; +} __packed; + +struct img_comp_subfrm { + u32 tile_disable:1; + struct img_region in; + struct img_region out; + struct img_offset luma; + struct img_offset chroma; + s16 out_vertical; /* Output vertical index */ + s16 out_horizontal; /* Output horizontal index */ +} __packed; + +#define IMG_MAX_SUBFRAMES 14 + +struct mdp_rdma_subfrm { + u32 offset[IMG_MAX_PLANES]; + u32 offset_0_p; + u32 src; + u32 clip; + u32 clip_ofst; +} __packed; + +struct mdp_rdma_data { + u32 src_ctrl; + u32 control; + u32 iova[IMG_MAX_PLANES]; + u32 iova_end[IMG_MAX_PLANES]; + u32 mf_bkgd; + u32 mf_bkgd_in_pxl; + u32 sf_bkgd; + u32 ufo_dec_y; + u32 ufo_dec_c; + u32 transform; + struct mdp_rdma_subfrm subfrms[IMG_MAX_SUBFRAMES]; +} __packed; + +struct mdp_rsz_subfrm { + u32 control2; + u32 src; + u32 clip; +} __packed; + +struct mdp_rsz_data { + u32 coeff_step_x; + u32 coeff_step_y; + u32 control1; + u32 control2; + struct mdp_rsz_subfrm subfrms[IMG_MAX_SUBFRAMES]; +} __packed; + +struct mdp_wrot_subfrm { + u32 offset[IMG_MAX_PLANES]; + u32 src; + u32 clip; + u32 clip_ofst; + u32 main_buf; +} __packed; + +struct mdp_wrot_data { + u32 iova[IMG_MAX_PLANES]; + u32 control; + u32 stride[IMG_MAX_PLANES]; + u32 mat_ctrl; + u32 fifo_test; + u32 filter; + struct mdp_wrot_subfrm subfrms[IMG_MAX_SUBFRAMES]; +} __packed; + +struct mdp_wdma_subfrm { + u32 offset[IMG_MAX_PLANES]; + u32 src; + u32 clip; + u32 clip_ofst; +} __packed; + +struct mdp_wdma_data { + u32 wdma_cfg; + u32 iova[IMG_MAX_PLANES]; + u32 w_in_byte; + u32 uv_stride; + struct mdp_wdma_subfrm subfrms[IMG_MAX_SUBFRAMES]; +} __packed; + +struct isp_data { + u64 dl_flags; /* 1 << (enum mdp_comp_type) */ + u32 smxi_iova[4]; + u32 cq_idx; + u32 cq_iova; + u32 tpipe_iova[IMG_MAX_SUBFRAMES]; +} __packed; + +struct img_compparam { + u16 type; /* enum mdp_comp_type */ + u16 id; /* enum mtk_mdp_comp_id */ + u32 input; + u32 outputs[IMG_MAX_HW_OUTPUTS]; + u32 num_outputs; + struct img_comp_frame frame; + struct img_comp_subfrm subfrms[IMG_MAX_SUBFRAMES]; + u32 num_subfrms; + union { + struct mdp_rdma_data rdma; + struct mdp_rsz_data rsz; + struct mdp_wrot_data wrot; + struct mdp_wdma_data wdma; + struct isp_data isp; + }; +} __packed; + +#define IMG_MAX_COMPONENTS 20 + +struct img_mux { + u32 reg; + u32 value; + u32 subsys_id; +}; + +struct img_mmsys_ctrl { + struct img_mux sets[IMG_MAX_COMPONENTS * 2]; + u32 num_sets; +}; + +struct img_config { + struct img_compparam components[IMG_MAX_COMPONENTS]; + u32 num_components; + struct img_mmsys_ctrl ctrls[IMG_MAX_SUBFRAMES]; + u32 num_subfrms; +} __packed; + +#endif /* __MTK_IMG_IPI_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c new file mode 100644 index 0000000000000000000000000000000000000000..29f6c1cd3de792a707d273393165072fcb35972b --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.c @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include "mtk-mdp3-cmdq.h" +#include "mtk-mdp3-comp.h" +#include "mtk-mdp3-core.h" +#include "mtk-mdp3-m2m.h" + +#define MDP_PATH_MAX_COMPS IMG_MAX_COMPONENTS + +struct mdp_path { + struct mdp_dev *mdp_dev; + struct mdp_comp_ctx comps[MDP_PATH_MAX_COMPS]; + u32 num_comps; + const struct img_config *config; + const struct img_ipi_frameparam *param; + const struct v4l2_rect *composes[IMG_MAX_HW_OUTPUTS]; + struct v4l2_rect bounds[IMG_MAX_HW_OUTPUTS]; +}; + +#define has_op(ctx, op) \ + ((ctx)->comp->ops && (ctx)->comp->ops->op) + #define call_op(ctx, op, ...) \ + (has_op(ctx, op) ? (ctx)->comp->ops->op(ctx, ##__VA_ARGS__) : 0) + +static bool is_output_disabled(const struct img_compparam *param, u32 count) +{ + return (count < param->num_subfrms) ? + (param->frame.output_disable || + param->subfrms[count].tile_disable) : + true; +} + +static int mdp_path_subfrm_require(const struct mdp_path *path, + struct mdp_cmdq_cmd *cmd, + s32 *mutex_id, u32 count) +{ + const struct img_config *config = path->config; + const struct mdp_comp_ctx *ctx; + const struct mtk_mdp_driver_data *data = path->mdp_dev->mdp_data; + struct device *dev = &path->mdp_dev->pdev->dev; + struct mtk_mutex **mutex = path->mdp_dev->mdp_mutex; + int id, index; + + /* Decide which mutex to use based on the current pipeline */ + switch (path->comps[0].comp->id) { + case MDP_COMP_RDMA0: + *mutex_id = MDP_PIPE_RDMA0; + break; + case MDP_COMP_ISP_IMGI: + *mutex_id = MDP_PIPE_IMGI; + break; + case MDP_COMP_WPEI: + *mutex_id = MDP_PIPE_WPEI; + break; + case MDP_COMP_WPEI2: + *mutex_id = MDP_PIPE_WPEI2; + break; + default: + dev_err(dev, "Unknown pipeline and no mutex is assigned"); + return -EINVAL; + } + + /* Set mutex mod */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + if (is_output_disabled(ctx->param, count)) + continue; + id = ctx->comp->id; + mtk_mutex_write_mod(mutex[*mutex_id], + data->mdp_mutex_table_idx[id], false); + } + + mtk_mutex_write_sof(mutex[*mutex_id], + MUTEX_SOF_IDX_SINGLE_MODE); + + return 0; +} + +static int mdp_path_subfrm_run(const struct mdp_path *path, + struct mdp_cmdq_cmd *cmd, + s32 *mutex_id, u32 count) +{ + const struct img_config *config = path->config; + const struct mdp_comp_ctx *ctx; + struct device *dev = &path->mdp_dev->pdev->dev; + struct mtk_mutex **mutex = path->mdp_dev->mdp_mutex; + int index; + s32 event; + + if (-1 == *mutex_id) { + dev_err(dev, "Incorrect mutex id"); + return -EINVAL; + } + + /* Wait WROT SRAM shared to DISP RDMA */ + /* Clear SOF event for each engine */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + if (is_output_disabled(ctx->param, count)) + continue; + event = ctx->comp->gce_event[MDP_GCE_EVENT_SOF]; + if (event != MDP_GCE_NO_EVENT) + MM_REG_CLEAR(cmd, event); + } + + /* Enable the mutex */ + mtk_mutex_enable_by_cmdq(mutex[*mutex_id], (void *)&cmd->pkt); + + /* Wait SOF events and clear mutex modules (optional) */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + if (is_output_disabled(ctx->param, count)) + continue; + event = ctx->comp->gce_event[MDP_GCE_EVENT_SOF]; + if (event != MDP_GCE_NO_EVENT) + MM_REG_WAIT(cmd, event); + } + + return 0; +} + +static int mdp_path_ctx_init(struct mdp_dev *mdp, struct mdp_path *path) +{ + const struct img_config *config = path->config; + int index, ret; + + if (config->num_components < 1) + return -EINVAL; + + for (index = 0; index < config->num_components; index++) { + ret = mdp_comp_ctx_config(mdp, &path->comps[index], + &config->components[index], + path->param); + if (ret) + return ret; + } + + return 0; +} + +static int mdp_path_config_subfrm(struct mdp_cmdq_cmd *cmd, + struct mdp_path *path, u32 count) +{ + const struct img_config *config = path->config; + const struct img_mmsys_ctrl *ctrl = &config->ctrls[count]; + const struct img_mux *set; + struct mdp_comp_ctx *ctx; + s32 mutex_id; + int index, ret; + + /* Acquire components */ + ret = mdp_path_subfrm_require(path, cmd, &mutex_id, count); + if (ret) + return ret; + /* Enable mux settings */ + for (index = 0; index < ctrl->num_sets; index++) { + set = &ctrl->sets[index]; + cmdq_pkt_write_mask(&cmd->pkt, set->subsys_id, set->reg, + set->value, 0xFFFFFFFF); + } + /* Config sub-frame information */ + for (index = (config->num_components - 1); index >= 0; index--) { + ctx = &path->comps[index]; + if (is_output_disabled(ctx->param, count)) + continue; + ret = call_op(ctx, config_subfrm, cmd, count); + if (ret) + return ret; + } + /* Run components */ + ret = mdp_path_subfrm_run(path, cmd, &mutex_id, count); + if (ret) + return ret; + /* Wait components done */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + if (is_output_disabled(ctx->param, count)) + continue; + ret = call_op(ctx, wait_comp_event, cmd); + if (ret) + return ret; + } + /* Advance to the next sub-frame */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + ret = call_op(ctx, advance_subfrm, cmd, count); + if (ret) + return ret; + } + /* Disable mux settings */ + for (index = 0; index < ctrl->num_sets; index++) { + set = &ctrl->sets[index]; + cmdq_pkt_write_mask(&cmd->pkt, set->subsys_id, set->reg, + 0, 0xFFFFFFFF); + } + + return 0; +} + +static int mdp_path_config(struct mdp_dev *mdp, struct mdp_cmdq_cmd *cmd, + struct mdp_path *path) +{ + const struct img_config *config = path->config; + struct mdp_comp_ctx *ctx; + int index, count, ret; + + /* Config path frame */ + /* Reset components */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + ret = call_op(ctx, init_comp, cmd); + if (ret) + return ret; + } + /* Config frame mode */ + for (index = 0; index < config->num_components; index++) { + const struct v4l2_rect *compose = + path->composes[ctx->param->outputs[0]]; + + ctx = &path->comps[index]; + ret = call_op(ctx, config_frame, cmd, compose); + if (ret) + return ret; + } + + /* Config path sub-frames */ + for (count = 0; count < config->num_subfrms; count++) { + ret = mdp_path_config_subfrm(cmd, path, count); + if (ret) + return ret; + } + /* Post processing information */ + for (index = 0; index < config->num_components; index++) { + ctx = &path->comps[index]; + ret = call_op(ctx, post_process, cmd); + if (ret) + return ret; + } + return 0; +} + +static int mdp_cmdq_pkt_create(struct cmdq_client *client, struct cmdq_pkt *pkt, + size_t size) +{ + struct device *dev; + dma_addr_t dma_addr; + + pkt->va_base = kzalloc(size, GFP_KERNEL); + if (!pkt->va_base) { + kfree(pkt); + return -ENOMEM; + } + pkt->buf_size = size; + pkt->cl = (void *)client; + + dev = client->chan->mbox->dev; + dma_addr = dma_map_single(dev, pkt->va_base, pkt->buf_size, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_addr)) { + dev_err(dev, "dma map failed, size=%u\n", (u32)(u64)size); + kfree(pkt->va_base); + return -ENOMEM; + } + + pkt->pa_base = dma_addr; + + return 0; +} + +static void mdp_cmdq_pkt_destroy(struct cmdq_pkt *pkt) +{ + struct cmdq_client *client = (struct cmdq_client *)pkt->cl; + + dma_unmap_single(client->chan->mbox->dev, pkt->pa_base, pkt->buf_size, + DMA_TO_DEVICE); + kfree(pkt->va_base); + pkt->va_base = NULL; +} + +static void mdp_auto_release_work(struct work_struct *work) +{ + struct mdp_cmdq_cmd *cmd; + struct mdp_dev *mdp; + + cmd = container_of(work, struct mdp_cmdq_cmd, auto_release_work); + mdp = cmd->mdp; + + mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]); + mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps, + cmd->num_comps); + + atomic_dec(&mdp->job_count); + wake_up(&mdp->callback_wq); + + mdp_cmdq_pkt_destroy(&cmd->pkt); + kfree(cmd->comps); + cmd->comps = NULL; + kfree(cmd); + cmd = NULL; +} + +static void mdp_handle_cmdq_callback(struct mbox_client *cl, void *mssg) +{ + struct mdp_cmdq_cmd *cmd; + struct cmdq_cb_data *data; + struct mdp_dev *mdp; + struct device *dev; + + if (!mssg) { + pr_info("%s:no callback data\n", __func__); + return; + } + + data = (struct cmdq_cb_data *)mssg; + cmd = container_of(data->pkt, struct mdp_cmdq_cmd, pkt); + mdp = cmd->mdp; + dev = &mdp->pdev->dev; + + if (cmd->mdp_ctx) + mdp_m2m_job_finish(cmd->mdp_ctx); + + if (cmd->user_cmdq_cb) { + struct cmdq_cb_data user_cb_data; + + user_cb_data.sta = data->sta; + user_cb_data.pkt = data->pkt; + cmd->user_cmdq_cb(user_cb_data); + } + + INIT_WORK(&cmd->auto_release_work, mdp_auto_release_work); + if (!queue_work(mdp->clock_wq, &cmd->auto_release_work)) { + dev_err(dev, "%s:queue_work fail!\n", __func__); + mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]); + mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps, + cmd->num_comps); + + atomic_dec(&mdp->job_count); + wake_up(&mdp->callback_wq); + + mdp_cmdq_pkt_destroy(&cmd->pkt); + kfree(cmd->comps); + cmd->comps = NULL; + kfree(cmd); + cmd = NULL; + } +} + +int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param) +{ + struct mdp_path *path = NULL; + struct mdp_cmdq_cmd *cmd = NULL; + struct mdp_comp *comps = NULL; + struct device *dev = &mdp->pdev->dev; + int i, ret; + + atomic_inc(&mdp->job_count); + if (atomic_read(&mdp->suspended)) { + atomic_dec(&mdp->job_count); + return -ECANCELED; + } + + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) { + ret = -ENOMEM; + goto err_cmdq_data; + } + + if (mdp_cmdq_pkt_create(mdp->cmdq_clt, &cmd->pkt, SZ_16K)) { + ret = -ENOMEM; + goto err_cmdq_data; + } + + comps = kcalloc(param->config->num_components, sizeof(*comps), + GFP_KERNEL); + if (!comps) { + ret = -ENOMEM; + goto err_cmdq_data; + } + + path = kzalloc(sizeof(*path), GFP_KERNEL); + if (!path) { + ret = -ENOMEM; + goto err_cmdq_data; + } + + path->mdp_dev = mdp; + path->config = param->config; + path->param = param->param; + for (i = 0; i < param->param->num_outputs; i++) { + path->bounds[i].left = 0; + path->bounds[i].top = 0; + path->bounds[i].width = + param->param->outputs[i].buffer.format.width; + path->bounds[i].height = + param->param->outputs[i].buffer.format.height; + path->composes[i] = param->composes[i] ? + param->composes[i] : &path->bounds[i]; + } + + ret = mdp_path_ctx_init(mdp, path); + if (ret) { + dev_err(dev, "mdp_path_ctx_init error\n"); + goto err_cmdq_data; + } + + mtk_mutex_prepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]); + + ret = mdp_path_config(mdp, cmd, path); + if (ret) { + dev_err(dev, "mdp_path_config error\n"); + goto err_cmdq_data; + } + cmdq_pkt_finalize(&cmd->pkt); + + for (i = 0; i < param->config->num_components; i++) + memcpy(&comps[i], path->comps[i].comp, + sizeof(struct mdp_comp)); + + mdp->cmdq_clt->client.rx_callback = mdp_handle_cmdq_callback; + cmd->mdp = mdp; + cmd->user_cmdq_cb = param->cmdq_cb; + cmd->user_cb_data = param->cb_data; + cmd->comps = comps; + cmd->num_comps = param->config->num_components; + cmd->mdp_ctx = param->mdp_ctx; + + ret = mdp_comp_clocks_on(&mdp->pdev->dev, cmd->comps, cmd->num_comps); + if (ret) { + dev_err(dev, "comp %d failed to enable clock!\n", ret); + goto err_clock_off; + } + + dma_sync_single_for_device(mdp->cmdq_clt->chan->mbox->dev, + cmd->pkt.pa_base, cmd->pkt.cmd_buf_size, + DMA_TO_DEVICE); + ret = mbox_send_message(mdp->cmdq_clt->chan, &cmd->pkt); + if (ret < 0) { + dev_err(dev, "mbox send message fail %d!\n", ret); + goto err_clock_off; + } + mbox_client_txdone(mdp->cmdq_clt->chan, 0); + + kfree(path); + return 0; + +err_clock_off: + mtk_mutex_unprepare(mdp->mdp_mutex[MDP_PIPE_RDMA0]); + mdp_comp_clocks_off(&mdp->pdev->dev, cmd->comps, + cmd->num_comps); +err_cmdq_data: + kfree(path); + atomic_dec(&mdp->job_count); + wake_up(&mdp->callback_wq); + if (cmd->pkt.buf_size > 0) + mdp_cmdq_pkt_destroy(&cmd->pkt); + kfree(comps); + kfree(cmd); + return ret; +} +EXPORT_SYMBOL_GPL(mdp_cmdq_send); diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.h new file mode 100644 index 0000000000000000000000000000000000000000..43475b862ddb919d5f7afb937a2da351bd663858 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-cmdq.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_CMDQ_H__ +#define __MTK_MDP3_CMDQ_H__ + +#include +#include +#include +#include "mtk-img-ipi.h" + +struct platform_device *mdp_get_plat_device(struct platform_device *pdev); + +struct mdp_cmdq_param { + struct img_config *config; + struct img_ipi_frameparam *param; + const struct v4l2_rect *composes[IMG_MAX_HW_OUTPUTS]; + + void (*cmdq_cb)(struct cmdq_cb_data data); + void *cb_data; + void *mdp_ctx; +}; + +struct mdp_cmdq_cmd { + struct work_struct auto_release_work; + struct cmdq_pkt pkt; + s32 *event; + struct mdp_dev *mdp; + void (*user_cmdq_cb)(struct cmdq_cb_data data); + void *user_cb_data; + struct mdp_comp *comps; + void *mdp_ctx; + u8 num_comps; +}; + +struct mdp_dev; + +int mdp_cmdq_send(struct mdp_dev *mdp, struct mdp_cmdq_param *param); + +#endif /* __MTK_MDP3_CMDQ_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c new file mode 100644 index 0000000000000000000000000000000000000000..e62abf3587bffafb940e0c742b5e277610bf4497 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.c @@ -0,0 +1,1033 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include +#include +#include "mtk-mdp3-comp.h" +#include "mtk-mdp3-core.h" +#include "mtk-mdp3-regs.h" + +#include "mdp_reg_rdma.h" +#include "mdp_reg_ccorr.h" +#include "mdp_reg_rsz.h" +#include "mdp_reg_wrot.h" +#include "mdp_reg_wdma.h" + +static u32 mdp_comp_alias_id[MDP_COMP_TYPE_COUNT]; + +static inline const struct mdp_platform_config * +__get_plat_cfg(const struct mdp_comp_ctx *ctx) +{ + if (!ctx) + return NULL; + + return ctx->comp->mdp_dev->mdp_data->mdp_cfg; +} + +static s64 get_comp_flag(const struct mdp_comp_ctx *ctx) +{ + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + + if (mdp_cfg && mdp_cfg->rdma_rsz1_sram_sharing) + if (ctx->comp->id == MDP_COMP_RDMA0) + return BIT(MDP_COMP_RDMA0) | BIT(MDP_COMP_RSZ1); + + return BIT(ctx->comp->id); +} + +static int init_rdma(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if (mdp_cfg && mdp_cfg->rdma_support_10bit) { + struct mdp_comp *prz1 = ctx->comp->mdp_dev->comp[MDP_COMP_RSZ1]; + + /* Disable RSZ1 */ + if (ctx->comp->id == MDP_COMP_RDMA0 && prz1) + MM_REG_WRITE(cmd, subsys_id, prz1->reg_base, PRZ_ENABLE, + 0x0, BIT(0)); + } + + /* Reset RDMA */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_RESET, BIT(0), BIT(0)); + MM_REG_POLL(cmd, subsys_id, base, MDP_RDMA_MON_STA_1, BIT(8), BIT(8)); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_RESET, 0x0, BIT(0)); + return 0; +} + +static int config_rdma_frame(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, + const struct v4l2_rect *compose) +{ + const struct mdp_rdma_data *rdma = &ctx->param->rdma; + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + u32 colorformat = ctx->input->buffer.format.colorformat; + bool block10bit = MDP_COLOR_IS_10BIT_PACKED(colorformat); + bool en_ufo = MDP_COLOR_IS_UFP(colorformat); + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if (mdp_cfg && mdp_cfg->rdma_support_10bit) { + if (block10bit) + MM_REG_WRITE(cmd, subsys_id, base, + MDP_RDMA_RESV_DUMMY_0, 0x7, 0x7); + else + MM_REG_WRITE(cmd, subsys_id, base, + MDP_RDMA_RESV_DUMMY_0, 0x0, 0x7); + } + + /* Setup smi control */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_GMCIF_CON, + (7 << 4) + //burst type to 8 + (1 << 16), //enable pre-ultra + 0x00030071); + + /* Setup source frame info */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_CON, rdma->src_ctrl, + 0x03C8FE0F); + + if (mdp_cfg) + if (mdp_cfg->rdma_support_10bit && en_ufo) { + /* Setup source buffer base */ + MM_REG_WRITE(cmd, subsys_id, + base, MDP_RDMA_UFO_DEC_LENGTH_BASE_Y, + rdma->ufo_dec_y, 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, + base, MDP_RDMA_UFO_DEC_LENGTH_BASE_C, + rdma->ufo_dec_c, 0xFFFFFFFF); + /* Set 10bit source frame pitch */ + if (block10bit) + MM_REG_WRITE(cmd, subsys_id, + base, MDP_RDMA_MF_BKGD_SIZE_IN_PXL, + rdma->mf_bkgd_in_pxl, 0x001FFFFF); + } + + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_CON, rdma->control, + 0x1110); + /* Setup source buffer base */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_0, rdma->iova[0], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_1, rdma->iova[1], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_BASE_2, rdma->iova[2], + 0xFFFFFFFF); + /* Setup source buffer end */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_0, + rdma->iova_end[0], 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_1, + rdma->iova_end[1], 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_END_2, + rdma->iova_end[2], 0xFFFFFFFF); + /* Setup source frame pitch */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_BKGD_SIZE_IN_BYTE, + rdma->mf_bkgd, 0x001FFFFF); + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SF_BKGD_SIZE_IN_BYTE, + rdma->sf_bkgd, 0x001FFFFF); + /* Setup color transform */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_TRANSFORM_0, + rdma->transform, 0x0F110000); + + return 0; +} + +static int config_rdma_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct mdp_rdma_subfrm *subfrm = &ctx->param->rdma.subfrms[index]; + const struct img_comp_subfrm *csf = &ctx->param->subfrms[index]; + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + u32 colorformat = ctx->input->buffer.format.colorformat; + bool block10bit = MDP_COLOR_IS_10BIT_PACKED(colorformat); + bool en_ufo = MDP_COLOR_IS_UFP(colorformat); + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Enable RDMA */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_EN, BIT(0), BIT(0)); + + /* Set Y pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_0, + subfrm->offset[0], 0xFFFFFFFF); + + /* Set 10bit UFO mode */ + if (mdp_cfg) + if (mdp_cfg->rdma_support_10bit && block10bit && en_ufo) + MM_REG_WRITE(cmd, subsys_id, base, + MDP_RDMA_SRC_OFFSET_0_P, + subfrm->offset_0_p, 0xFFFFFFFF); + + /* Set U pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_1, + subfrm->offset[1], 0xFFFFFFFF); + /* Set V pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_SRC_OFFSET_2, + subfrm->offset[2], 0xFFFFFFFF); + /* Set source size */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_SRC_SIZE, subfrm->src, + 0x1FFF1FFF); + /* Set target size */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_CLIP_SIZE, + subfrm->clip, 0x1FFF1FFF); + /* Set crop offset */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_MF_OFFSET_1, + subfrm->clip_ofst, 0x003F001F); + + if (mdp_cfg && mdp_cfg->rdma_upsample_repeat_only) + if ((csf->in.right - csf->in.left + 1) > 320) + MM_REG_WRITE(cmd, subsys_id, base, + MDP_RDMA_RESV_DUMMY_0, BIT(2), BIT(2)); + + return 0; +} + +static int wait_rdma_event(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + struct device *dev = &ctx->comp->mdp_dev->pdev->dev; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if (ctx->comp->alias_id == 0) + MM_REG_WAIT(cmd, ctx->comp->gce_event[MDP_GCE_EVENT_EOF]); + else + dev_err(dev, "Do not support RDMA1_DONE event\n"); + + /* Disable RDMA */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_RDMA_EN, 0x0, BIT(0)); + return 0; +} + +static const struct mdp_comp_ops rdma_ops = { + .get_comp_flag = get_comp_flag, + .init_comp = init_rdma, + .config_frame = config_rdma_frame, + .config_subfrm = config_rdma_subfrm, + .wait_comp_event = wait_rdma_event, +}; + +static int init_rsz(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Reset RSZ */ + MM_REG_WRITE(cmd, subsys_id, base, PRZ_ENABLE, 0x10000, BIT(16)); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_ENABLE, 0x0, BIT(16)); + /* Enable RSZ */ + MM_REG_WRITE(cmd, subsys_id, base, PRZ_ENABLE, BIT(0), BIT(0)); + return 0; +} + +static int config_rsz_frame(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, + const struct v4l2_rect *compose) +{ + const struct mdp_rsz_data *rsz = &ctx->param->rsz; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if (ctx->param->frame.bypass) { + /* Disable RSZ */ + MM_REG_WRITE(cmd, subsys_id, base, PRZ_ENABLE, 0x0, BIT(0)); + return 0; + } + + MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, rsz->control1, + 0x03FFFDF3); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, rsz->control2, + 0x0FFFC290); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_HORIZONTAL_COEFF_STEP, + rsz->coeff_step_x, 0x007FFFFF); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_VERTICAL_COEFF_STEP, + rsz->coeff_step_y, 0x007FFFFF); + return 0; +} + +static int config_rsz_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct mdp_rsz_subfrm *subfrm = &ctx->param->rsz.subfrms[index]; + const struct img_comp_subfrm *csf = &ctx->param->subfrms[index]; + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_2, subfrm->control2, + 0x00003800); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_INPUT_IMAGE, subfrm->src, + 0xFFFFFFFF); + + if (mdp_cfg && mdp_cfg->rsz_disable_dcm_small_sample) + if ((csf->in.right - csf->in.left + 1) <= 16) + MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, + BIT(27), BIT(27)); + + MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_HORIZONTAL_INTEGER_OFFSET, + csf->luma.left, 0xFFFF); + MM_REG_WRITE(cmd, subsys_id, + base, PRZ_LUMA_HORIZONTAL_SUBPIXEL_OFFSET, + csf->luma.left_subpix, 0x1FFFFF); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_VERTICAL_INTEGER_OFFSET, + csf->luma.top, 0xFFFF); + MM_REG_WRITE(cmd, subsys_id, base, PRZ_LUMA_VERTICAL_SUBPIXEL_OFFSET, + csf->luma.top_subpix, 0x1FFFFF); + MM_REG_WRITE(cmd, subsys_id, + base, PRZ_CHROMA_HORIZONTAL_INTEGER_OFFSET, + csf->chroma.left, 0xFFFF); + MM_REG_WRITE(cmd, subsys_id, + base, PRZ_CHROMA_HORIZONTAL_SUBPIXEL_OFFSET, + csf->chroma.left_subpix, 0x1FFFFF); + + MM_REG_WRITE(cmd, subsys_id, base, PRZ_OUTPUT_IMAGE, subfrm->clip, + 0xFFFFFFFF); + + return 0; +} + +static int advance_rsz_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + + if (mdp_cfg && mdp_cfg->rsz_disable_dcm_small_sample) { + const struct img_comp_subfrm *csf = &ctx->param->subfrms[index]; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if ((csf->in.right - csf->in.left + 1) <= 16) + MM_REG_WRITE(cmd, subsys_id, base, PRZ_CONTROL_1, 0x0, + BIT(27)); + } + + return 0; +} + +static const struct mdp_comp_ops rsz_ops = { + .get_comp_flag = get_comp_flag, + .init_comp = init_rsz, + .config_frame = config_rsz_frame, + .config_subfrm = config_rsz_subfrm, + .advance_subfrm = advance_rsz_subfrm, +}; + +static int init_wrot(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Reset WROT */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_SOFT_RST, BIT(0), BIT(0)); + MM_REG_POLL(cmd, subsys_id, base, VIDO_SOFT_RST_STAT, BIT(0), BIT(0)); + MM_REG_WRITE(cmd, subsys_id, base, VIDO_SOFT_RST, 0x0, BIT(0)); + MM_REG_POLL(cmd, subsys_id, base, VIDO_SOFT_RST_STAT, 0x0, BIT(0)); + return 0; +} + +static int config_wrot_frame(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, + const struct v4l2_rect *compose) +{ + const struct mdp_wrot_data *wrot = &ctx->param->wrot; + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Write frame base address */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR, wrot->iova[0], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_C, wrot->iova[1], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, VIDO_BASE_ADDR_V, wrot->iova[2], + 0xFFFFFFFF); + /* Write frame related registers */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_CTRL, wrot->control, + 0xF131510F); + /* Write frame Y pitch */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE, wrot->stride[0], + 0x0000FFFF); + /* Write frame UV pitch */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_C, wrot->stride[1], + 0xFFFF); + MM_REG_WRITE(cmd, subsys_id, base, VIDO_STRIDE_V, wrot->stride[2], + 0xFFFF); + /* Write matrix control */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAT_CTRL, wrot->mat_ctrl, 0xF3); + + /* Set the fixed ALPHA as 0xFF */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_DITHER, 0xFF000000, + 0xFF000000); + /* Set VIDO_EOL_SEL */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_RSV_1, BIT(31), BIT(31)); + /* Set VIDO_FIFO_TEST */ + if (wrot->fifo_test != 0) + MM_REG_WRITE(cmd, subsys_id, base, VIDO_FIFO_TEST, + wrot->fifo_test, 0xFFF); + /* Filter enable */ + if (mdp_cfg && mdp_cfg->wrot_filter_constraint) + MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAIN_BUF_SIZE, + wrot->filter, 0x77); + + return 0; +} + +static int config_wrot_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct mdp_wrot_subfrm *subfrm = &ctx->param->wrot.subfrms[index]; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Write Y pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR, + subfrm->offset[0], 0x0FFFFFFF); + /* Write U pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR_C, + subfrm->offset[1], 0x0FFFFFFF); + /* Write V pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_OFST_ADDR_V, + subfrm->offset[2], 0x0FFFFFFF); + /* Write source size */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_IN_SIZE, subfrm->src, + 0x1FFF1FFF); + /* Write target size */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_TAR_SIZE, subfrm->clip, + 0x1FFF1FFF); + MM_REG_WRITE(cmd, subsys_id, base, VIDO_CROP_OFST, subfrm->clip_ofst, + 0x1FFF1FFF); + + MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAIN_BUF_SIZE, + subfrm->main_buf, 0x1FFF7F00); + + /* Enable WROT */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_ROT_EN, BIT(0), BIT(0)); + + return 0; +} + +static int wait_wrot_event(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + const struct mdp_platform_config *mdp_cfg = __get_plat_cfg(ctx); + struct device *dev = &ctx->comp->mdp_dev->pdev->dev; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + if (ctx->comp->alias_id == 0) + MM_REG_WAIT(cmd, ctx->comp->gce_event[MDP_GCE_EVENT_EOF]); + else + dev_err(dev, "Do not support WROT1_DONE event\n"); + + if (mdp_cfg && mdp_cfg->wrot_filter_constraint) + MM_REG_WRITE(cmd, subsys_id, base, VIDO_MAIN_BUF_SIZE, 0x0, + 0x77); + + /* Disable WROT */ + MM_REG_WRITE(cmd, subsys_id, base, VIDO_ROT_EN, 0x0, BIT(0)); + + return 0; +} + +static const struct mdp_comp_ops wrot_ops = { + .get_comp_flag = get_comp_flag, + .init_comp = init_wrot, + .config_frame = config_wrot_frame, + .config_subfrm = config_wrot_subfrm, + .wait_comp_event = wait_wrot_event, +}; + +static int init_wdma(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Reset WDMA */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_RST, BIT(0), BIT(0)); + MM_REG_POLL(cmd, subsys_id, base, WDMA_FLOW_CTRL_DBG, BIT(0), BIT(0)); + MM_REG_WRITE(cmd, subsys_id, base, WDMA_RST, 0x0, BIT(0)); + return 0; +} + +static int config_wdma_frame(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, + const struct v4l2_rect *compose) +{ + const struct mdp_wdma_data *wdma = &ctx->param->wdma; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + MM_REG_WRITE(cmd, subsys_id, base, WDMA_BUF_CON2, 0x10101050, + 0xFFFFFFFF); + + /* Setup frame information */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_CFG, wdma->wdma_cfg, + 0x0F01B8F0); + /* Setup frame base address */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_ADDR, wdma->iova[0], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_U_ADDR, wdma->iova[1], + 0xFFFFFFFF); + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_V_ADDR, wdma->iova[2], + 0xFFFFFFFF); + /* Setup Y pitch */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_W_IN_BYTE, + wdma->w_in_byte, 0x0000FFFF); + /* Setup UV pitch */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_UV_PITCH, + wdma->uv_stride, 0x0000FFFF); + /* Set the fixed ALPHA as 0xFF */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_ALPHA, 0x800000FF, + 0x800000FF); + + return 0; +} + +static int config_wdma_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct mdp_wdma_subfrm *subfrm = &ctx->param->wdma.subfrms[index]; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* Write Y pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_ADDR_OFFSET, + subfrm->offset[0], 0x0FFFFFFF); + /* Write U pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_U_ADDR_OFFSET, + subfrm->offset[1], 0x0FFFFFFF); + /* Write V pixel offset */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_DST_V_ADDR_OFFSET, + subfrm->offset[2], 0x0FFFFFFF); + /* Write source size */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_SRC_SIZE, subfrm->src, + 0x3FFF3FFF); + /* Write target size */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_SIZE, subfrm->clip, + 0x3FFF3FFF); + /* Write clip offset */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_CLIP_COORD, subfrm->clip_ofst, + 0x3FFF3FFF); + + /* Enable WDMA */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_EN, BIT(0), BIT(0)); + + return 0; +} + +static int wait_wdma_event(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + MM_REG_WAIT(cmd, ctx->comp->gce_event[MDP_GCE_EVENT_EOF]); + /* Disable WDMA */ + MM_REG_WRITE(cmd, subsys_id, base, WDMA_EN, 0x0, BIT(0)); + return 0; +} + +static const struct mdp_comp_ops wdma_ops = { + .get_comp_flag = get_comp_flag, + .init_comp = init_wdma, + .config_frame = config_wdma_frame, + .config_subfrm = config_wdma_subfrm, + .wait_comp_event = wait_wdma_event, +}; + +static int init_ccorr(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd) +{ + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + + /* CCORR enable */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_CCORR_EN, BIT(0), BIT(0)); + /* Relay mode */ + MM_REG_WRITE(cmd, subsys_id, base, MDP_CCORR_CFG, BIT(0), BIT(0)); + return 0; +} + +static int config_ccorr_subfrm(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index) +{ + const struct img_comp_subfrm *csf = &ctx->param->subfrms[index]; + phys_addr_t base = ctx->comp->reg_base; + u8 subsys_id = ctx->comp->subsys_id; + u32 hsize, vsize; + + hsize = csf->in.right - csf->in.left + 1; + vsize = csf->in.bottom - csf->in.top + 1; + MM_REG_WRITE(cmd, subsys_id, base, MDP_CCORR_SIZE, + (hsize << 16) + (vsize << 0), 0x1FFF1FFF); + return 0; +} + +static const struct mdp_comp_ops ccorr_ops = { + .get_comp_flag = get_comp_flag, + .init_comp = init_ccorr, + .config_subfrm = config_ccorr_subfrm, +}; + +static const struct mdp_comp_ops *mdp_comp_ops[MDP_COMP_TYPE_COUNT] = { + [MDP_COMP_TYPE_RDMA] = &rdma_ops, + [MDP_COMP_TYPE_RSZ] = &rsz_ops, + [MDP_COMP_TYPE_WROT] = &wrot_ops, + [MDP_COMP_TYPE_WDMA] = &wdma_ops, + [MDP_COMP_TYPE_CCORR] = &ccorr_ops, +}; + +struct mdp_comp_match { + enum mdp_comp_type type; + u32 alias_id; +}; + +static const struct mdp_comp_match mdp_comp_matches[MDP_MAX_COMP_COUNT] = { + [MDP_COMP_WPEI] = { MDP_COMP_TYPE_WPEI, 0 }, + [MDP_COMP_WPEO] = { MDP_COMP_TYPE_EXTO, 2 }, + [MDP_COMP_WPEI2] = { MDP_COMP_TYPE_WPEI, 1 }, + [MDP_COMP_WPEO2] = { MDP_COMP_TYPE_EXTO, 3 }, + [MDP_COMP_ISP_IMGI] = { MDP_COMP_TYPE_IMGI, 0 }, + [MDP_COMP_ISP_IMGO] = { MDP_COMP_TYPE_EXTO, 0 }, + [MDP_COMP_ISP_IMG2O] = { MDP_COMP_TYPE_EXTO, 1 }, + + [MDP_COMP_CAMIN] = { MDP_COMP_TYPE_DL_PATH, 0 }, + [MDP_COMP_CAMIN2] = { MDP_COMP_TYPE_DL_PATH, 1 }, + [MDP_COMP_RDMA0] = { MDP_COMP_TYPE_RDMA, 0 }, + [MDP_COMP_CCORR0] = { MDP_COMP_TYPE_CCORR, 0 }, + [MDP_COMP_RSZ0] = { MDP_COMP_TYPE_RSZ, 0 }, + [MDP_COMP_RSZ1] = { MDP_COMP_TYPE_RSZ, 1 }, + [MDP_COMP_PATH0_SOUT] = { MDP_COMP_TYPE_PATH, 0 }, + [MDP_COMP_PATH1_SOUT] = { MDP_COMP_TYPE_PATH, 1 }, + [MDP_COMP_WROT0] = { MDP_COMP_TYPE_WROT, 0 }, + [MDP_COMP_WDMA] = { MDP_COMP_TYPE_WDMA, 0 }, +}; + +static const struct of_device_id mdp_comp_dt_ids[] = { + { + .compatible = "mediatek,mt8183-mdp3-rdma", + .data = (void *)MDP_COMP_TYPE_RDMA, + }, { + .compatible = "mediatek,mt8183-mdp3-ccorr", + .data = (void *)MDP_COMP_TYPE_CCORR, + }, { + .compatible = "mediatek,mt8183-mdp3-rsz", + .data = (void *)MDP_COMP_TYPE_RSZ, + }, { + .compatible = "mediatek,mt8183-mdp3-wrot", + .data = (void *)MDP_COMP_TYPE_WROT, + }, { + .compatible = "mediatek,mt8183-mdp3-wdma", + .data = (void *)MDP_COMP_TYPE_WDMA, + }, + {} +}; + +static const struct of_device_id mdp_sub_comp_dt_ids[] = { + { + .compatible = "mediatek,mt8183-mdp3-wdma", + .data = (void *)MDP_COMP_TYPE_PATH, + }, { + .compatible = "mediatek,mt8183-mdp3-wrot", + .data = (void *)MDP_COMP_TYPE_PATH, + }, + {} +}; + +/* Used to describe the item order in MDP property */ +struct mdp_comp_info { + u32 clk_num; + u32 clk_ofst; + u32 dts_reg_ofst; +}; + +static const struct mdp_comp_info mdp_comp_dt_info[MDP_MAX_COMP_COUNT] = { + [MDP_COMP_RDMA0] = {2, 0, 0}, + [MDP_COMP_RSZ0] = {1, 0, 0}, + [MDP_COMP_WROT0] = {1, 0, 0}, + [MDP_COMP_WDMA] = {1, 0, 0}, + [MDP_COMP_CCORR0] = {1, 0, 0}, +}; + +static inline bool is_dma_capable(const enum mdp_comp_type type) +{ + return (type == MDP_COMP_TYPE_RDMA || + type == MDP_COMP_TYPE_WROT || + type == MDP_COMP_TYPE_WDMA); +} + +static inline bool is_bypass_gce_event(const enum mdp_comp_type type) +{ + /* + * Subcomponent PATH is only used for the direction of data flow and + * dose not need to wait for GCE event. + */ + return (type == MDP_COMP_TYPE_PATH); +} + +static int mdp_comp_get_id(enum mdp_comp_type type, int alias_id) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mdp_comp_matches); i++) + if (mdp_comp_matches[i].type == type && + mdp_comp_matches[i].alias_id == alias_id) + return i; + return -ENODEV; +} + +int mdp_comp_clock_on(struct device *dev, struct mdp_comp *comp) +{ + int i, ret; + + if (comp->comp_dev) { + ret = pm_runtime_get_sync(comp->comp_dev); + if (ret < 0) { + dev_err(dev, + "Failed to get power, err %d. type:%d id:%d\n", + ret, comp->type, comp->id); + return ret; + } + } + + for (i = 0; i < ARRAY_SIZE(comp->clks); i++) { + if (IS_ERR_OR_NULL(comp->clks[i])) + continue; + ret = clk_prepare_enable(comp->clks[i]); + if (ret) { + dev_err(dev, + "Failed to enable clk %d. type:%d id:%d\n", + i, comp->type, comp->id); + return ret; + } + } + + return 0; +} + +void mdp_comp_clock_off(struct device *dev, struct mdp_comp *comp) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(comp->clks); i++) { + if (IS_ERR_OR_NULL(comp->clks[i])) + continue; + clk_disable_unprepare(comp->clks[i]); + } + + if (comp->comp_dev) + pm_runtime_put(comp->comp_dev); +} + +int mdp_comp_clocks_on(struct device *dev, struct mdp_comp *comps, int num) +{ + int i; + + for (i = 0; i < num; i++) + if (mdp_comp_clock_on(dev, &comps[i]) != 0) + return ++i; + + return 0; +} + +void mdp_comp_clocks_off(struct device *dev, struct mdp_comp *comps, int num) +{ + int i; + + for (i = 0; i < num; i++) + mdp_comp_clock_off(dev, &comps[i]); +} + +static int mdp_get_subsys_id(struct device *dev, struct device_node *node, + struct mdp_comp *comp) +{ + struct platform_device *comp_pdev; + struct cmdq_client_reg cmdq_reg; + int ret = 0; + int index = 0; + + if (!dev || !node || !comp) + return -EINVAL; + + comp_pdev = of_find_device_by_node(node); + + if (!comp_pdev) { + dev_err(dev, "get comp_pdev fail! comp id=%d type=%d\n", + comp->id, comp->type); + return -ENODEV; + } + + index = mdp_comp_dt_info[comp->id].dts_reg_ofst; + ret = cmdq_dev_get_client_reg(&comp_pdev->dev, &cmdq_reg, index); + if (ret != 0) { + dev_err(&comp_pdev->dev, "cmdq_dev_get_subsys fail!\n"); + return -EINVAL; + } + + comp->subsys_id = cmdq_reg.subsys; + dev_dbg(&comp_pdev->dev, "subsys id=%d\n", cmdq_reg.subsys); + + return 0; +} + +static void __mdp_comp_init(struct mdp_dev *mdp, struct device_node *node, + struct mdp_comp *comp) +{ + struct resource res; + phys_addr_t base; + int index = mdp_comp_dt_info[comp->id].dts_reg_ofst; + + if (of_address_to_resource(node, index, &res) < 0) + base = 0L; + else + base = res.start; + + comp->mdp_dev = mdp; + comp->regs = of_iomap(node, 0); + comp->reg_base = base; +} + +static int mdp_comp_init(struct mdp_dev *mdp, struct device_node *node, + struct mdp_comp *comp, enum mtk_mdp_comp_id id) +{ + struct device *dev = &mdp->pdev->dev; + int clk_num; + int clk_ofst; + int i; + s32 event; + + if (id < 0 || id >= MDP_MAX_COMP_COUNT) { + dev_err(dev, "Invalid component id %d\n", id); + return -EINVAL; + } + + comp->id = id; + comp->type = mdp_comp_matches[id].type; + comp->alias_id = mdp_comp_matches[id].alias_id; + comp->ops = mdp_comp_ops[comp->type]; + __mdp_comp_init(mdp, node, comp); + + clk_num = mdp_comp_dt_info[id].clk_num; + clk_ofst = mdp_comp_dt_info[id].clk_ofst; + + for (i = 0; i < clk_num; i++) { + comp->clks[i] = of_clk_get(node, i + clk_ofst); + if (IS_ERR(comp->clks[i])) + break; + } + + mdp_get_subsys_id(dev, node, comp); + + /* Set GCE SOF event */ + if (is_bypass_gce_event(comp->type) || + of_property_read_u32_index(node, "mediatek,gce-events", + MDP_GCE_EVENT_SOF, &event)) + event = MDP_GCE_NO_EVENT; + + comp->gce_event[MDP_GCE_EVENT_SOF] = event; + + /* Set GCE EOF event */ + if (is_dma_capable(comp->type)) { + if (of_property_read_u32_index(node, "mediatek,gce-events", + MDP_GCE_EVENT_EOF, &event)) { + dev_err(dev, "Component id %d has no EOF\n", id); + return -EINVAL; + } + } else { + event = MDP_GCE_NO_EVENT; + } + + comp->gce_event[MDP_GCE_EVENT_EOF] = event; + + return 0; +} + +static void mdp_comp_deinit(struct mdp_comp *comp) +{ + if (!comp) + return; + + if (comp->regs) + iounmap(comp->regs); +} + +static struct mdp_comp *mdp_comp_create(struct mdp_dev *mdp, + struct device_node *node, + enum mtk_mdp_comp_id id) +{ + struct device *dev = &mdp->pdev->dev; + struct mdp_comp *comp; + int ret; + + if (mdp->comp[id]) + return ERR_PTR(-EEXIST); + + comp = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL); + if (!comp) + return ERR_PTR(-ENOMEM); + + ret = mdp_comp_init(mdp, node, comp, id); + if (ret) { + kfree(comp); + return ERR_PTR(ret); + } + mdp->comp[id] = comp; + mdp->comp[id]->mdp_dev = mdp; + + dev_dbg(dev, "%s type:%d alias:%d id:%d base:%#x regs:%p\n", + dev->of_node->name, comp->type, comp->alias_id, id, + (u32)comp->reg_base, comp->regs); + return comp; +} + +static int mdp_comp_sub_create(struct mdp_dev *mdp) +{ + struct device *dev = &mdp->pdev->dev; + struct device_node *node, *parent; + + parent = dev->of_node->parent; + + for_each_child_of_node(parent, node) { + const struct of_device_id *of_id; + enum mdp_comp_type type; + int id, alias_id; + struct mdp_comp *comp; + + of_id = of_match_node(mdp_sub_comp_dt_ids, node); + if (!of_id) + continue; + if (!of_device_is_available(node)) { + dev_dbg(dev, "Skipping disabled sub comp. %pOF\n", + node); + continue; + } + + type = (enum mdp_comp_type)(uintptr_t)of_id->data; + alias_id = mdp_comp_alias_id[type]; + id = mdp_comp_get_id(type, alias_id); + if (id < 0) { + dev_err(dev, + "Fail to get sub comp. id: type %d alias %d\n", + type, alias_id); + return -EINVAL; + } + mdp_comp_alias_id[type]++; + + comp = mdp_comp_create(mdp, node, id); + if (IS_ERR(comp)) + return PTR_ERR(comp); + } + + return 0; +} + +void mdp_comp_destroy(struct mdp_dev *mdp) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(mdp->comp); i++) { + if (mdp->comp[i]) { + pm_runtime_disable(mdp->comp[i]->comp_dev); + mdp_comp_deinit(mdp->comp[i]); + kfree(mdp->comp[i]); + mdp->comp[i] = NULL; + } + } +} + +int mdp_comp_config(struct mdp_dev *mdp) +{ + struct device *dev = &mdp->pdev->dev; + struct device_node *node, *parent; + struct platform_device *pdev; + int ret; + + memset(mdp_comp_alias_id, 0, sizeof(mdp_comp_alias_id)); + + parent = dev->of_node->parent; + /* Iterate over sibling MDP function blocks */ + for_each_child_of_node(parent, node) { + const struct of_device_id *of_id; + enum mdp_comp_type type; + int id, alias_id; + struct mdp_comp *comp; + + of_id = of_match_node(mdp_comp_dt_ids, node); + if (!of_id) + continue; + + if (!of_device_is_available(node)) { + dev_dbg(dev, "Skipping disabled component %pOF\n", + node); + continue; + } + + type = (enum mdp_comp_type)(uintptr_t)of_id->data; + alias_id = mdp_comp_alias_id[type]; + id = mdp_comp_get_id(type, alias_id); + if (id < 0) { + dev_err(dev, + "Fail to get component id: type %d alias %d\n", + type, alias_id); + continue; + } + mdp_comp_alias_id[type]++; + + comp = mdp_comp_create(mdp, node, id); + if (IS_ERR(comp)) { + ret = PTR_ERR(comp); + goto err_init_comps; + } + + /* Only DMA capable components need the pm control */ + comp->comp_dev = NULL; + if (!is_dma_capable(comp->type)) + continue; + + pdev = of_find_device_by_node(node); + if (!pdev) { + dev_warn(dev, "can't find platform device of node:%s\n", + node->name); + return -ENODEV; + } + + comp->comp_dev = &pdev->dev; + pm_runtime_enable(comp->comp_dev); + } + + ret = mdp_comp_sub_create(mdp); + if (ret) + goto err_init_comps; + + return 0; + +err_init_comps: + mdp_comp_destroy(mdp); + return ret; +} + +int mdp_comp_ctx_config(struct mdp_dev *mdp, struct mdp_comp_ctx *ctx, + const struct img_compparam *param, + const struct img_ipi_frameparam *frame) +{ + struct device *dev = &mdp->pdev->dev; + int i; + + if (param->type < 0 || param->type >= MDP_MAX_COMP_COUNT) { + dev_err(dev, "Invalid component id %d", param->type); + return -EINVAL; + } + + ctx->comp = mdp->comp[param->type]; + if (!ctx->comp) { + dev_err(dev, "Uninit component id %d", param->type); + return -EINVAL; + } + + ctx->param = param; + ctx->input = &frame->inputs[param->input]; + for (i = 0; i < param->num_outputs; i++) + ctx->outputs[i] = &frame->outputs[param->outputs[i]]; + return 0; +} diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h new file mode 100644 index 0000000000000000000000000000000000000000..dc48f55ac4f74aaf442297a60b1bbdab75c20014 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-comp.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_COMP_H__ +#define __MTK_MDP3_COMP_H__ + +#include "mtk-mdp3-cmdq.h" + +#define MM_REG_WRITE_MASK(cmd, id, base, ofst, val, mask, ...) \ + cmdq_pkt_write_mask(&((cmd)->pkt), id, \ + (base) + (ofst), (val), (mask), ##__VA_ARGS__) + +#define MM_REG_WRITE(cmd, id, base, ofst, val, mask, ...) \ +do { \ + typeof(mask) (m) = (mask); \ + MM_REG_WRITE_MASK(cmd, id, base, ofst, val, \ + (((m) & (ofst##_MASK)) == (ofst##_MASK)) ? \ + (0xffffffff) : (m), ##__VA_ARGS__); \ +} while (0) + +#define MM_REG_WAIT(cmd, evt) \ +do { \ + typeof(cmd) (c) = (cmd); \ + typeof(evt) (e) = (evt); \ + cmdq_pkt_wfe(&((c)->pkt), (e), true); \ +} while (0) + +#define MM_REG_WAIT_NO_CLEAR(cmd, evt) \ +do { \ + typeof(cmd) (c) = (cmd); \ + typeof(evt) (e) = (evt); \ + cmdq_pkt_wfe(&((c)->pkt), (e), false); \ +} while (0) + +#define MM_REG_CLEAR(cmd, evt) \ +do { \ + typeof(cmd) (c) = (cmd); \ + typeof(evt) (e) = (evt); \ + cmdq_pkt_clear_event(&((c)->pkt), (e)); \ +} while (0) + +#define MM_REG_SET_EVENT(cmd, evt) \ +do { \ + typeof(cmd) (c) = (cmd); \ + typeof(evt) (e) = (evt); \ + cmdq_pkt_set_event(&((c)->pkt), (e)); \ +} while (0) + +#define MM_REG_POLL_MASK(cmd, id, base, ofst, val, _mask, ...) \ +do { \ + typeof(_mask) (_m) = (_mask); \ + cmdq_pkt_poll_mask(&((cmd)->pkt), id, \ + (base) + (ofst), (val), (_m), ##__VA_ARGS__); \ +} while (0) + +#define MM_REG_POLL(cmd, id, base, ofst, val, mask, ...) \ +do { \ + typeof(mask) (m) = (mask); \ + MM_REG_POLL_MASK((cmd), id, base, ofst, val, \ + (((m) & (ofst##_MASK)) == (ofst##_MASK)) ? \ + (0xffffffff) : (m), ##__VA_ARGS__); \ +} while (0) + +enum mtk_mdp_comp_id { + MDP_COMP_NONE = -1, /* Invalid engine */ + + /* ISP */ + MDP_COMP_WPEI = 0, + MDP_COMP_WPEO, /* 1 */ + MDP_COMP_WPEI2, /* 2 */ + MDP_COMP_WPEO2, /* 3 */ + MDP_COMP_ISP_IMGI, /* 4 */ + MDP_COMP_ISP_IMGO, /* 5 */ + MDP_COMP_ISP_IMG2O, /* 6 */ + + /* IPU */ + MDP_COMP_IPUI, /* 7 */ + MDP_COMP_IPUO, /* 8 */ + + /* MDP */ + MDP_COMP_CAMIN, /* 9 */ + MDP_COMP_CAMIN2, /* 10 */ + MDP_COMP_RDMA0, /* 11 */ + MDP_COMP_AAL0, /* 12 */ + MDP_COMP_CCORR0, /* 13 */ + MDP_COMP_RSZ0, /* 14 */ + MDP_COMP_RSZ1, /* 15 */ + MDP_COMP_TDSHP0, /* 16 */ + MDP_COMP_COLOR0, /* 17 */ + MDP_COMP_PATH0_SOUT, /* 18 */ + MDP_COMP_PATH1_SOUT, /* 19 */ + MDP_COMP_WROT0, /* 20 */ + MDP_COMP_WDMA, /* 21 */ + + /* Dummy Engine */ + MDP_COMP_RDMA1, /* 22 */ + MDP_COMP_RSZ2, /* 23 */ + MDP_COMP_TDSHP1, /* 24 */ + MDP_COMP_WROT1, /* 25 */ + + MDP_MAX_COMP_COUNT /* ALWAYS keep at the end */ +}; + +enum mdp_comp_type { + MDP_COMP_TYPE_INVALID = 0, + + MDP_COMP_TYPE_RDMA, + MDP_COMP_TYPE_RSZ, + MDP_COMP_TYPE_WROT, + MDP_COMP_TYPE_WDMA, + MDP_COMP_TYPE_PATH, + + MDP_COMP_TYPE_TDSHP, + MDP_COMP_TYPE_COLOR, + MDP_COMP_TYPE_DRE, + MDP_COMP_TYPE_CCORR, + MDP_COMP_TYPE_HDR, + + MDP_COMP_TYPE_IMGI, + MDP_COMP_TYPE_WPEI, + MDP_COMP_TYPE_EXTO, /* External path */ + MDP_COMP_TYPE_DL_PATH, /* Direct-link path */ + + MDP_COMP_TYPE_COUNT /* ALWAYS keep at the end */ +}; + +#define MDP_GCE_NO_EVENT (-1) +enum { + MDP_GCE_EVENT_SOF = 0, + MDP_GCE_EVENT_EOF = 1, + MDP_GCE_EVENT_MAX, +}; + +struct mdp_comp_ops; + +struct mdp_comp { + struct mdp_dev *mdp_dev; + void __iomem *regs; + phys_addr_t reg_base; + u8 subsys_id; + struct clk *clks[6]; + struct device *comp_dev; + enum mdp_comp_type type; + enum mtk_mdp_comp_id id; + u32 alias_id; + s32 gce_event[MDP_GCE_EVENT_MAX]; + const struct mdp_comp_ops *ops; +}; + +struct mdp_comp_ctx { + struct mdp_comp *comp; + const struct img_compparam *param; + const struct img_input *input; + const struct img_output *outputs[IMG_MAX_HW_OUTPUTS]; +}; + +struct mdp_comp_ops { + s64 (*get_comp_flag)(const struct mdp_comp_ctx *ctx); + int (*init_comp)(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd); + int (*config_frame)(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd, + const struct v4l2_rect *compose); + int (*config_subfrm)(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index); + int (*wait_comp_event)(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd); + int (*advance_subfrm)(struct mdp_comp_ctx *ctx, + struct mdp_cmdq_cmd *cmd, u32 index); + int (*post_process)(struct mdp_comp_ctx *ctx, struct mdp_cmdq_cmd *cmd); +}; + +struct mdp_dev; + +int mdp_comp_config(struct mdp_dev *mdp); +void mdp_comp_destroy(struct mdp_dev *mdp); +int mdp_comp_clock_on(struct device *dev, struct mdp_comp *comp); +void mdp_comp_clock_off(struct device *dev, struct mdp_comp *comp); +int mdp_comp_clocks_on(struct device *dev, struct mdp_comp *comps, int num); +void mdp_comp_clocks_off(struct device *dev, struct mdp_comp *comps, int num); +int mdp_comp_ctx_config(struct mdp_dev *mdp, struct mdp_comp_ctx *ctx, + const struct img_compparam *param, + const struct img_ipi_frameparam *frame); + +#endif /* __MTK_MDP3_COMP_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c new file mode 100644 index 0000000000000000000000000000000000000000..cde59579b7aebaa8704e00db9646cf454ac9076f --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "mtk-mdp3-core.h" +#include "mtk-mdp3-m2m.h" + +static const struct mdp_platform_config mt8183_plat_cfg = { + .rdma_support_10bit = true, + .rdma_rsz1_sram_sharing = true, + .rdma_upsample_repeat_only = true, + .rsz_disable_dcm_small_sample = false, + .wrot_filter_constraint = false, +}; + +static const struct of_device_id mt8183_mdp_probe_infra[MDP_INFRA_MAX] = { + [MDP_INFRA_MMSYS] = { .compatible = "mediatek,mt8183-mmsys" }, + [MDP_INFRA_MUTEX] = { .compatible = "mediatek,mt8183-disp-mutex" }, + [MDP_INFRA_SCP] = { .compatible = "mediatek,mt8183-scp" } +}; + +static const u32 mt8183_mutex_idx[MDP_MAX_COMP_COUNT] = { + [MDP_COMP_RDMA0] = MUTEX_MOD_IDX_MDP_RDMA0, + [MDP_COMP_RSZ0] = MUTEX_MOD_IDX_MDP_RSZ0, + [MDP_COMP_RSZ1] = MUTEX_MOD_IDX_MDP_RSZ1, + [MDP_COMP_TDSHP0] = MUTEX_MOD_IDX_MDP_TDSHP0, + [MDP_COMP_WROT0] = MUTEX_MOD_IDX_MDP_WROT0, + [MDP_COMP_WDMA] = MUTEX_MOD_IDX_MDP_WDMA, + [MDP_COMP_AAL0] = MUTEX_MOD_IDX_MDP_AAL0, + [MDP_COMP_CCORR0] = MUTEX_MOD_IDX_MDP_CCORR0, +}; + +static const struct mtk_mdp_driver_data mt8183_mdp_driver_data = { + .mdp_probe_infra = mt8183_mdp_probe_infra, + .mdp_cfg = &mt8183_plat_cfg, + .mdp_mutex_table_idx = mt8183_mutex_idx, +}; + +static const struct of_device_id mdp_of_ids[] = { + { .compatible = "mediatek,mt8183-mdp3-rdma", + .data = &mt8183_mdp_driver_data, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, mdp_of_ids); + +static struct platform_device *__get_pdev_by_id(struct platform_device *pdev, + enum mdp_infra_id id) +{ + struct device_node *node; + struct platform_device *mdp_pdev = NULL; + const struct mtk_mdp_driver_data *mdp_data; + const char *compat; + + if (!pdev) + return NULL; + + if (id < MDP_INFRA_MMSYS || id >= MDP_INFRA_MAX) { + dev_err(&pdev->dev, "Illegal infra id %d\n", id); + return NULL; + } + + mdp_data = of_device_get_match_data(&pdev->dev); + if (!mdp_data) { + dev_err(&pdev->dev, "have no driver data to find node\n"); + return NULL; + } + compat = mdp_data->mdp_probe_infra[id].compatible; + + node = of_find_compatible_node(NULL, NULL, compat); + if (WARN_ON(!node)) { + dev_err(&pdev->dev, "find node from id %d failed\n", id); + return NULL; + } + + mdp_pdev = of_find_device_by_node(node); + of_node_put(node); + if (WARN_ON(!mdp_pdev)) { + dev_err(&pdev->dev, "find pdev from id %d failed\n", id); + return NULL; + } + + return mdp_pdev; +} + +struct platform_device *mdp_get_plat_device(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *mdp_node; + struct platform_device *mdp_pdev; + + mdp_node = of_parse_phandle(dev->of_node, MDP_PHANDLE_NAME, 0); + if (!mdp_node) { + dev_err(dev, "can't get node %s\n", MDP_PHANDLE_NAME); + return NULL; + } + + mdp_pdev = of_find_device_by_node(mdp_node); + of_node_put(mdp_node); + + return mdp_pdev; +} +EXPORT_SYMBOL_GPL(mdp_get_plat_device); + +int mdp_vpu_get_locked(struct mdp_dev *mdp) +{ + int ret = 0; + + if (mdp->vpu_count++ == 0) { + ret = rproc_boot(mdp->rproc_handle); + if (ret) { + dev_err(&mdp->pdev->dev, + "vpu_load_firmware failed %d\n", ret); + goto err_load_vpu; + } + ret = mdp_vpu_register(mdp); + if (ret) { + dev_err(&mdp->pdev->dev, + "mdp_vpu register failed %d\n", ret); + goto err_reg_vpu; + } + ret = mdp_vpu_dev_init(&mdp->vpu, mdp->scp, &mdp->vpu_lock); + if (ret) { + dev_err(&mdp->pdev->dev, + "mdp_vpu device init failed %d\n", ret); + goto err_init_vpu; + } + } + return 0; + +err_init_vpu: + mdp_vpu_unregister(mdp); +err_reg_vpu: +err_load_vpu: + mdp->vpu_count--; + return ret; +} + +void mdp_vpu_put_locked(struct mdp_dev *mdp) +{ + if (--mdp->vpu_count == 0) { + mdp_vpu_dev_deinit(&mdp->vpu); + mdp_vpu_unregister(mdp); + } +} + +void mdp_video_device_release(struct video_device *vdev) +{ + struct mdp_dev *mdp = (struct mdp_dev *)video_get_drvdata(vdev); + int i; + + scp_put(mdp->scp); + + destroy_workqueue(mdp->job_wq); + destroy_workqueue(mdp->clock_wq); + + pm_runtime_disable(&mdp->pdev->dev); + + vb2_dma_contig_clear_max_seg_size(&mdp->pdev->dev); + + mdp_comp_destroy(mdp); + for (i = 0; i < MDP_PIPE_MAX; i++) + mtk_mutex_put(mdp->mdp_mutex[i]); + + mdp_vpu_shared_mem_free(&mdp->vpu); + v4l2_m2m_release(mdp->m2m_dev); + kfree(mdp); +} + +static int mdp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mdp_dev *mdp; + struct platform_device *mm_pdev; + int ret, i; + + mdp = kzalloc(sizeof(*mdp), GFP_KERNEL); + if (!mdp) { + ret = -ENOMEM; + goto err_return; + } + + mdp->pdev = pdev; + mdp->mdp_data = of_device_get_match_data(&pdev->dev); + + mm_pdev = __get_pdev_by_id(pdev, MDP_INFRA_MMSYS); + if (!mm_pdev) { + ret = -ENODEV; + goto err_return; + } + mdp->mdp_mmsys = &mm_pdev->dev; + + mm_pdev = __get_pdev_by_id(pdev, MDP_INFRA_MUTEX); + if (WARN_ON(!mm_pdev)) { + ret = -ENODEV; + goto err_return; + } + for (i = 0; i < MDP_PIPE_MAX; i++) { + mdp->mdp_mutex[i] = mtk_mutex_get(&mm_pdev->dev); + if (!mdp->mdp_mutex[i]) { + ret = -ENODEV; + goto err_return; + } + } + + ret = mdp_comp_config(mdp); + if (ret) { + dev_err(dev, "Failed to config mdp components\n"); + goto err_return; + } + + mdp->job_wq = alloc_workqueue(MDP_MODULE_NAME, WQ_FREEZABLE, 0); + if (!mdp->job_wq) { + dev_err(dev, "Unable to create job workqueue\n"); + ret = -ENOMEM; + goto err_deinit_comp; + } + + mdp->clock_wq = alloc_workqueue(MDP_MODULE_NAME "-clock", WQ_FREEZABLE, + 0); + if (!mdp->clock_wq) { + dev_err(dev, "Unable to create clock workqueue\n"); + ret = -ENOMEM; + goto err_destroy_job_wq; + } + + mm_pdev = __get_pdev_by_id(pdev, MDP_INFRA_SCP); + if (WARN_ON(!mm_pdev)) { + dev_err(&pdev->dev, "Could not get scp device\n"); + ret = -ENODEV; + goto err_destroy_clock_wq; + } + mdp->scp = platform_get_drvdata(mm_pdev); + mdp->rproc_handle = scp_get_rproc(mdp->scp); + dev_dbg(&pdev->dev, "MDP rproc_handle: %pK", mdp->rproc_handle); + + mutex_init(&mdp->vpu_lock); + mutex_init(&mdp->m2m_lock); + + mdp->cmdq_clt = cmdq_mbox_create(dev, 0); + if (IS_ERR(mdp->cmdq_clt)) { + ret = PTR_ERR(mdp->cmdq_clt); + goto err_put_scp; + } + + init_waitqueue_head(&mdp->callback_wq); + ida_init(&mdp->mdp_ida); + platform_set_drvdata(pdev, mdp); + + vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); + + ret = v4l2_device_register(dev, &mdp->v4l2_dev); + if (ret) { + dev_err(dev, "Failed to register v4l2 device\n"); + ret = -EINVAL; + goto err_mbox_destroy; + } + + ret = mdp_m2m_device_register(mdp); + if (ret) { + v4l2_err(&mdp->v4l2_dev, "Failed to register m2m device\n"); + goto err_unregister_device; + } + + dev_dbg(dev, "mdp-%d registered successfully\n", pdev->id); + return 0; + +err_unregister_device: + v4l2_device_unregister(&mdp->v4l2_dev); +err_mbox_destroy: + cmdq_mbox_destroy(mdp->cmdq_clt); +err_put_scp: + scp_put(mdp->scp); +err_destroy_clock_wq: + destroy_workqueue(mdp->clock_wq); +err_destroy_job_wq: + destroy_workqueue(mdp->job_wq); +err_deinit_comp: + mdp_comp_destroy(mdp); +err_return: + for (i = 0; i < MDP_PIPE_MAX; i++) + mtk_mutex_put(mdp->mdp_mutex[i]); + kfree(mdp); + dev_dbg(dev, "Errno %d\n", ret); + return ret; +} + +static int mdp_remove(struct platform_device *pdev) +{ + struct mdp_dev *mdp = platform_get_drvdata(pdev); + + v4l2_device_unregister(&mdp->v4l2_dev); + + dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name); + return 0; +} + +static int __maybe_unused mdp_suspend(struct device *dev) +{ + struct mdp_dev *mdp = dev_get_drvdata(dev); + int ret; + + atomic_set(&mdp->suspended, 1); + + if (atomic_read(&mdp->job_count)) { + ret = wait_event_timeout(mdp->callback_wq, + !atomic_read(&mdp->job_count), + 2 * HZ); + if (ret == 0) { + dev_err(dev, + "%s:flushed cmdq task incomplete, count=%d\n", + __func__, atomic_read(&mdp->job_count)); + return -EBUSY; + } + } + + return 0; +} + +static int __maybe_unused mdp_resume(struct device *dev) +{ + struct mdp_dev *mdp = dev_get_drvdata(dev); + + atomic_set(&mdp->suspended, 0); + + return 0; +} + +static const struct dev_pm_ops mdp_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mdp_suspend, mdp_resume) +}; + +static struct platform_driver mdp_driver = { + .probe = mdp_probe, + .remove = mdp_remove, + .driver = { + .name = MDP_MODULE_NAME, + .pm = &mdp_pm_ops, + .of_match_table = of_match_ptr(mdp_of_ids), + }, +}; + +module_platform_driver(mdp_driver); + +MODULE_AUTHOR("Ping-Hsun Wu "); +MODULE_DESCRIPTION("MediaTek image processor 3 driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h new file mode 100644 index 0000000000000000000000000000000000000000..2ef5fbc4f25a88e62fa963109eb84653642c92af --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_CORE_H__ +#define __MTK_MDP3_CORE_H__ + +#include +#include +#include +#include +#include "mtk-mdp3-comp.h" +#include "mtk-mdp3-vpu.h" + +#define MDP_MODULE_NAME "mtk-mdp3" +#define MDP_DEVICE_NAME "MediaTek MDP3" +#define MDP_PHANDLE_NAME "mediatek,mdp3" + +enum mdp_infra_id { + MDP_INFRA_MMSYS, + MDP_INFRA_MUTEX, + MDP_INFRA_SCP, + MDP_INFRA_MAX +}; + +enum mdp_buffer_usage { + MDP_BUFFER_USAGE_HW_READ, + MDP_BUFFER_USAGE_MDP, + MDP_BUFFER_USAGE_MDP2, + MDP_BUFFER_USAGE_ISP, + MDP_BUFFER_USAGE_WPE, +}; + +struct mdp_platform_config { + bool rdma_support_10bit; + bool rdma_rsz1_sram_sharing; + bool rdma_upsample_repeat_only; + bool rsz_disable_dcm_small_sample; + bool wrot_filter_constraint; +}; + +/* indicate which mutex is used by each pipepline */ +enum mdp_pipe_id { + MDP_PIPE_RDMA0, + MDP_PIPE_IMGI, + MDP_PIPE_WPEI, + MDP_PIPE_WPEI2, + MDP_PIPE_MAX +}; + +struct mtk_mdp_driver_data { + const struct of_device_id *mdp_probe_infra; + const struct mdp_platform_config *mdp_cfg; + const u32 *mdp_mutex_table_idx; +}; + +struct mdp_dev { + struct platform_device *pdev; + struct device *mdp_mmsys; + struct mtk_mutex *mdp_mutex[MDP_PIPE_MAX]; + struct mdp_comp *comp[MDP_MAX_COMP_COUNT]; + const struct mtk_mdp_driver_data *mdp_data; + + struct workqueue_struct *job_wq; + struct workqueue_struct *clock_wq; + struct mdp_vpu_dev vpu; + struct mtk_scp *scp; + struct rproc *rproc_handle; + /* synchronization protect for accessing vpu working buffer info */ + struct mutex vpu_lock; + s32 vpu_count; + u32 id_count; + struct ida mdp_ida; + struct cmdq_client *cmdq_clt; + wait_queue_head_t callback_wq; + + struct v4l2_device v4l2_dev; + struct video_device *m2m_vdev; + struct v4l2_m2m_dev *m2m_dev; + /* synchronization protect for m2m device operation */ + struct mutex m2m_lock; + atomic_t suspended; + atomic_t job_count; +}; + +int mdp_vpu_get_locked(struct mdp_dev *mdp); +void mdp_vpu_put_locked(struct mdp_dev *mdp); +int mdp_vpu_register(struct mdp_dev *mdp); +void mdp_vpu_unregister(struct mdp_dev *mdp); +void mdp_video_device_release(struct video_device *vdev); + +#endif /* __MTK_MDP3_CORE_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c new file mode 100644 index 0000000000000000000000000000000000000000..5f74ea3b7a524800f5f28877fdf02e67388bb3b5 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.c @@ -0,0 +1,724 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include +#include +#include "mtk-mdp3-m2m.h" + +static inline struct mdp_m2m_ctx *fh_to_ctx(struct v4l2_fh *fh) +{ + return container_of(fh, struct mdp_m2m_ctx, fh); +} + +static inline struct mdp_m2m_ctx *ctrl_to_ctx(struct v4l2_ctrl *ctrl) +{ + return container_of(ctrl->handler, struct mdp_m2m_ctx, ctrl_handler); +} + +static inline struct mdp_frame *ctx_get_frame(struct mdp_m2m_ctx *ctx, + enum v4l2_buf_type type) +{ + if (V4L2_TYPE_IS_OUTPUT(type)) + return &ctx->curr_param.output; + else + return &ctx->curr_param.captures[0]; +} + +static inline void mdp_m2m_ctx_set_state(struct mdp_m2m_ctx *ctx, u32 state) +{ + atomic_or(state, &ctx->curr_param.state); +} + +static inline bool mdp_m2m_ctx_is_state_set(struct mdp_m2m_ctx *ctx, u32 mask) +{ + return ((atomic_read(&ctx->curr_param.state) & mask) == mask); +} + +static void mdp_m2m_process_done(void *priv, int vb_state) +{ + struct mdp_m2m_ctx *ctx = priv; + struct vb2_v4l2_buffer *src_vbuf, *dst_vbuf; + + src_vbuf = (struct vb2_v4l2_buffer *) + v4l2_m2m_src_buf_remove(ctx->m2m_ctx); + dst_vbuf = (struct vb2_v4l2_buffer *) + v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); + ctx->curr_param.frame_no = ctx->frame_count[MDP_M2M_SRC]; + src_vbuf->sequence = ctx->frame_count[MDP_M2M_SRC]++; + dst_vbuf->sequence = ctx->frame_count[MDP_M2M_DST]++; + v4l2_m2m_buf_copy_metadata(src_vbuf, dst_vbuf, true); + + v4l2_m2m_buf_done(src_vbuf, vb_state); + v4l2_m2m_buf_done(dst_vbuf, vb_state); + v4l2_m2m_job_finish(ctx->mdp_dev->m2m_dev, ctx->m2m_ctx); +} + +static void mdp_m2m_device_run(void *priv) +{ + struct mdp_m2m_ctx *ctx = priv; + struct mdp_frame *frame; + struct vb2_v4l2_buffer *src_vb, *dst_vb; + struct img_ipi_frameparam param = {}; + struct mdp_cmdq_param task = {}; + enum vb2_buffer_state vb_state = VB2_BUF_STATE_ERROR; + int ret; + + if (mdp_m2m_ctx_is_state_set(ctx, MDP_M2M_CTX_ERROR)) { + dev_err(&ctx->mdp_dev->pdev->dev, + "mdp_m2m_ctx is in error state\n"); + goto worker_end; + } + + param.frame_no = ctx->curr_param.frame_no; + param.type = ctx->curr_param.type; + param.num_inputs = 1; + param.num_outputs = 1; + + frame = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + src_vb = v4l2_m2m_next_src_buf(ctx->m2m_ctx); + mdp_set_src_config(¶m.inputs[0], frame, &src_vb->vb2_buf); + + frame = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + dst_vb = v4l2_m2m_next_dst_buf(ctx->m2m_ctx); + mdp_set_dst_config(¶m.outputs[0], frame, &dst_vb->vb2_buf); + + ret = mdp_vpu_process(&ctx->vpu, ¶m); + if (ret) { + dev_err(&ctx->mdp_dev->pdev->dev, + "VPU MDP process failed: %d\n", ret); + goto worker_end; + } + + task.config = ctx->vpu.config; + task.param = ¶m; + task.composes[0] = &frame->compose; + task.cmdq_cb = NULL; + task.cb_data = NULL; + task.mdp_ctx = ctx; + + ret = mdp_cmdq_send(ctx->mdp_dev, &task); + if (ret) { + dev_err(&ctx->mdp_dev->pdev->dev, + "CMDQ sendtask failed: %d\n", ret); + goto worker_end; + } + + return; + +worker_end: + mdp_m2m_process_done(ctx, vb_state); +} + +static int mdp_m2m_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct mdp_m2m_ctx *ctx = vb2_get_drv_priv(q); + struct mdp_frame *capture; + struct vb2_queue *vq; + int ret; + bool out_streaming, cap_streaming; + + if (V4L2_TYPE_IS_OUTPUT(q->type)) + ctx->frame_count[MDP_M2M_SRC] = 0; + + if (V4L2_TYPE_IS_CAPTURE(q->type)) + ctx->frame_count[MDP_M2M_DST] = 0; + + capture = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + vq = v4l2_m2m_get_src_vq(ctx->m2m_ctx); + out_streaming = vb2_is_streaming(vq); + vq = v4l2_m2m_get_dst_vq(ctx->m2m_ctx); + cap_streaming = vb2_is_streaming(vq); + + /* Check to see if scaling ratio is within supported range */ + if ((V4L2_TYPE_IS_OUTPUT(q->type) && cap_streaming) || + (V4L2_TYPE_IS_CAPTURE(q->type) && out_streaming)) { + ret = mdp_check_scaling_ratio(&capture->crop.c, + &capture->compose, + capture->rotation, + ctx->curr_param.limit); + if (ret) { + dev_err(&ctx->mdp_dev->pdev->dev, + "Out of scaling range\n"); + return ret; + } + } + + if (!mdp_m2m_ctx_is_state_set(ctx, MDP_VPU_INIT)) { + ret = mdp_vpu_get_locked(ctx->mdp_dev); + if (ret) + return ret; + + ret = mdp_vpu_ctx_init(&ctx->vpu, &ctx->mdp_dev->vpu, + MDP_DEV_M2M); + if (ret) { + dev_err(&ctx->mdp_dev->pdev->dev, + "VPU init failed %d\n", ret); + return -EINVAL; + } + mdp_m2m_ctx_set_state(ctx, MDP_VPU_INIT); + } + + return 0; +} + +static struct vb2_v4l2_buffer *mdp_m2m_buf_remove(struct mdp_m2m_ctx *ctx, + unsigned int type) +{ + if (V4L2_TYPE_IS_OUTPUT(type)) + return (struct vb2_v4l2_buffer *) + v4l2_m2m_src_buf_remove(ctx->m2m_ctx); + else + return (struct vb2_v4l2_buffer *) + v4l2_m2m_dst_buf_remove(ctx->m2m_ctx); +} + +static void mdp_m2m_stop_streaming(struct vb2_queue *q) +{ + struct mdp_m2m_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_v4l2_buffer *vb; + + vb = mdp_m2m_buf_remove(ctx, q->type); + while (vb) { + v4l2_m2m_buf_done(vb, VB2_BUF_STATE_ERROR); + vb = mdp_m2m_buf_remove(ctx, q->type); + } +} + +static int mdp_m2m_queue_setup(struct vb2_queue *q, + unsigned int *num_buffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct mdp_m2m_ctx *ctx = vb2_get_drv_priv(q); + struct v4l2_pix_format_mplane *pix_mp; + u32 i; + + pix_mp = &ctx_get_frame(ctx, q->type)->format.fmt.pix_mp; + + /* from VIDIOC_CREATE_BUFS */ + if (*num_planes) { + if (*num_planes != pix_mp->num_planes) + return -EINVAL; + for (i = 0; i < pix_mp->num_planes; ++i) + if (sizes[i] < pix_mp->plane_fmt[i].sizeimage) + return -EINVAL; + } else {/* from VIDIOC_REQBUFS */ + *num_planes = pix_mp->num_planes; + for (i = 0; i < pix_mp->num_planes; ++i) + sizes[i] = pix_mp->plane_fmt[i].sizeimage; + } + + return 0; +} + +static int mdp_m2m_buf_prepare(struct vb2_buffer *vb) +{ + struct mdp_m2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct v4l2_pix_format_mplane *pix_mp; + struct vb2_v4l2_buffer *v4l2_buf = to_vb2_v4l2_buffer(vb); + u32 i; + + v4l2_buf->field = V4L2_FIELD_NONE; + + if (V4L2_TYPE_IS_CAPTURE(vb->type)) { + pix_mp = &ctx_get_frame(ctx, vb->type)->format.fmt.pix_mp; + for (i = 0; i < pix_mp->num_planes; ++i) { + vb2_set_plane_payload(vb, i, + pix_mp->plane_fmt[i].sizeimage); + } + } + return 0; +} + +static int mdp_m2m_buf_out_validate(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *v4l2_buf = to_vb2_v4l2_buffer(vb); + + v4l2_buf->field = V4L2_FIELD_NONE; + + return 0; +} + +static void mdp_m2m_buf_queue(struct vb2_buffer *vb) +{ + struct mdp_m2m_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct vb2_v4l2_buffer *v4l2_buf = to_vb2_v4l2_buffer(vb); + + v4l2_buf->field = V4L2_FIELD_NONE; + + v4l2_m2m_buf_queue(ctx->m2m_ctx, to_vb2_v4l2_buffer(vb)); +} + +static const struct vb2_ops mdp_m2m_qops = { + .queue_setup = mdp_m2m_queue_setup, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, + .buf_prepare = mdp_m2m_buf_prepare, + .start_streaming = mdp_m2m_start_streaming, + .stop_streaming = mdp_m2m_stop_streaming, + .buf_queue = mdp_m2m_buf_queue, + .buf_out_validate = mdp_m2m_buf_out_validate, +}; + +static int mdp_m2m_querycap(struct file *file, void *fh, + struct v4l2_capability *cap) +{ + strscpy(cap->driver, MDP_MODULE_NAME, sizeof(cap->driver)); + strscpy(cap->card, MDP_DEVICE_NAME, sizeof(cap->card)); + + return 0; +} + +static int mdp_m2m_enum_fmt_mplane(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return mdp_enum_fmt_mplane(f); +} + +static int mdp_m2m_g_fmt_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(fh); + struct mdp_frame *frame; + struct v4l2_pix_format_mplane *pix_mp; + + frame = ctx_get_frame(ctx, f->type); + *f = frame->format; + pix_mp = &f->fmt.pix_mp; + pix_mp->colorspace = ctx->curr_param.colorspace; + pix_mp->xfer_func = ctx->curr_param.xfer_func; + pix_mp->ycbcr_enc = ctx->curr_param.ycbcr_enc; + pix_mp->quantization = ctx->curr_param.quant; + + return 0; +} + +static int mdp_m2m_s_fmt_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(fh); + struct mdp_frame *frame = ctx_get_frame(ctx, f->type); + struct mdp_frame *capture; + const struct mdp_format *fmt; + struct vb2_queue *vq; + + fmt = mdp_try_fmt_mplane(f, &ctx->curr_param, ctx->id); + if (!fmt) + return -EINVAL; + + vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type); + if (vb2_is_busy(vq)) + return -EBUSY; + + frame->format = *f; + frame->mdp_fmt = fmt; + frame->ycbcr_prof = mdp_map_ycbcr_prof_mplane(f, fmt->mdp_color); + frame->usage = V4L2_TYPE_IS_OUTPUT(f->type) ? + MDP_BUFFER_USAGE_HW_READ : MDP_BUFFER_USAGE_MDP; + + capture = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + if (V4L2_TYPE_IS_OUTPUT(f->type)) { + capture->crop.c.left = 0; + capture->crop.c.top = 0; + capture->crop.c.width = f->fmt.pix_mp.width; + capture->crop.c.height = f->fmt.pix_mp.height; + ctx->curr_param.colorspace = f->fmt.pix_mp.colorspace; + ctx->curr_param.ycbcr_enc = f->fmt.pix_mp.ycbcr_enc; + ctx->curr_param.quant = f->fmt.pix_mp.quantization; + ctx->curr_param.xfer_func = f->fmt.pix_mp.xfer_func; + } else { + capture->compose.left = 0; + capture->compose.top = 0; + capture->compose.width = f->fmt.pix_mp.width; + capture->compose.height = f->fmt.pix_mp.height; + } + + return 0; +} + +static int mdp_m2m_try_fmt_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(fh); + + if (!mdp_try_fmt_mplane(f, &ctx->curr_param, ctx->id)) + return -EINVAL; + + return 0; +} + +static int mdp_m2m_g_selection(struct file *file, void *fh, + struct v4l2_selection *s) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(fh); + struct mdp_frame *frame; + bool valid = false; + + if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) + valid = mdp_target_is_crop(s->target); + else if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) + valid = mdp_target_is_compose(s->target); + + if (!valid) + return -EINVAL; + + switch (s->target) { + case V4L2_SEL_TGT_CROP: + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + frame = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + s->r = frame->crop.c; + return 0; + case V4L2_SEL_TGT_COMPOSE: + if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) + return -EINVAL; + frame = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + s->r = frame->compose; + return 0; + case V4L2_SEL_TGT_CROP_DEFAULT: + case V4L2_SEL_TGT_CROP_BOUNDS: + if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + frame = ctx_get_frame(ctx, s->type); + s->r.left = 0; + s->r.top = 0; + s->r.width = frame->format.fmt.pix_mp.width; + s->r.height = frame->format.fmt.pix_mp.height; + return 0; + case V4L2_SEL_TGT_COMPOSE_DEFAULT: + case V4L2_SEL_TGT_COMPOSE_BOUNDS: + if (s->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) + return -EINVAL; + frame = ctx_get_frame(ctx, s->type); + s->r.left = 0; + s->r.top = 0; + s->r.width = frame->format.fmt.pix_mp.width; + s->r.height = frame->format.fmt.pix_mp.height; + return 0; + } + return -EINVAL; +} + +static int mdp_m2m_s_selection(struct file *file, void *fh, + struct v4l2_selection *s) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(fh); + struct mdp_frame *frame = ctx_get_frame(ctx, s->type); + struct mdp_frame *capture; + struct v4l2_rect r; + struct device *dev = &ctx->mdp_dev->pdev->dev; + bool valid = false; + int ret; + + if (s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) + valid = (s->target == V4L2_SEL_TGT_CROP); + else if (s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) + valid = (s->target == V4L2_SEL_TGT_COMPOSE); + + if (!valid) { + dev_dbg(dev, "[%s:%d] invalid type:%u target:%u", __func__, + ctx->id, s->type, s->target); + return -EINVAL; + } + + ret = mdp_try_crop(ctx, &r, s, frame); + if (ret) + return ret; + capture = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + + if (mdp_target_is_crop(s->target)) + capture->crop.c = r; + else + capture->compose = r; + + s->r = r; + + return 0; +} + +static const struct v4l2_ioctl_ops mdp_m2m_ioctl_ops = { + .vidioc_querycap = mdp_m2m_querycap, + .vidioc_enum_fmt_vid_cap = mdp_m2m_enum_fmt_mplane, + .vidioc_enum_fmt_vid_out = mdp_m2m_enum_fmt_mplane, + .vidioc_g_fmt_vid_cap_mplane = mdp_m2m_g_fmt_mplane, + .vidioc_g_fmt_vid_out_mplane = mdp_m2m_g_fmt_mplane, + .vidioc_s_fmt_vid_cap_mplane = mdp_m2m_s_fmt_mplane, + .vidioc_s_fmt_vid_out_mplane = mdp_m2m_s_fmt_mplane, + .vidioc_try_fmt_vid_cap_mplane = mdp_m2m_try_fmt_mplane, + .vidioc_try_fmt_vid_out_mplane = mdp_m2m_try_fmt_mplane, + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + .vidioc_g_selection = mdp_m2m_g_selection, + .vidioc_s_selection = mdp_m2m_s_selection, + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +static int mdp_m2m_queue_init(void *priv, + struct vb2_queue *src_vq, + struct vb2_queue *dst_vq) +{ + struct mdp_m2m_ctx *ctx = priv; + int ret; + + src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + src_vq->io_modes = VB2_MMAP | VB2_DMABUF; + src_vq->ops = &mdp_m2m_qops; + src_vq->mem_ops = &vb2_dma_contig_memops; + src_vq->drv_priv = ctx; + src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src_vq->dev = &ctx->mdp_dev->pdev->dev; + src_vq->lock = &ctx->ctx_lock; + + ret = vb2_queue_init(src_vq); + if (ret) + return ret; + + dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + dst_vq->io_modes = VB2_MMAP | VB2_DMABUF; + dst_vq->ops = &mdp_m2m_qops; + dst_vq->mem_ops = &vb2_dma_contig_memops; + dst_vq->drv_priv = ctx; + dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst_vq->dev = &ctx->mdp_dev->pdev->dev; + dst_vq->lock = &ctx->ctx_lock; + + return vb2_queue_init(dst_vq); +} + +static int mdp_m2m_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct mdp_m2m_ctx *ctx = ctrl_to_ctx(ctrl); + struct mdp_frame *capture; + + capture = ctx_get_frame(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + switch (ctrl->id) { + case V4L2_CID_HFLIP: + capture->hflip = ctrl->val; + break; + case V4L2_CID_VFLIP: + capture->vflip = ctrl->val; + break; + case V4L2_CID_ROTATE: + capture->rotation = ctrl->val; + break; + } + + return 0; +} + +static const struct v4l2_ctrl_ops mdp_m2m_ctrl_ops = { + .s_ctrl = mdp_m2m_s_ctrl, +}; + +static int mdp_m2m_ctrls_create(struct mdp_m2m_ctx *ctx) +{ + v4l2_ctrl_handler_init(&ctx->ctrl_handler, MDP_MAX_CTRLS); + ctx->ctrls.hflip = v4l2_ctrl_new_std(&ctx->ctrl_handler, + &mdp_m2m_ctrl_ops, V4L2_CID_HFLIP, + 0, 1, 1, 0); + ctx->ctrls.vflip = v4l2_ctrl_new_std(&ctx->ctrl_handler, + &mdp_m2m_ctrl_ops, V4L2_CID_VFLIP, + 0, 1, 1, 0); + ctx->ctrls.rotate = v4l2_ctrl_new_std(&ctx->ctrl_handler, + &mdp_m2m_ctrl_ops, + V4L2_CID_ROTATE, 0, 270, 90, 0); + + if (ctx->ctrl_handler.error) { + int err = ctx->ctrl_handler.error; + + v4l2_ctrl_handler_free(&ctx->ctrl_handler); + dev_err(&ctx->mdp_dev->pdev->dev, + "Failed to register controls\n"); + return err; + } + return 0; +} + +static int mdp_m2m_open(struct file *file) +{ + struct video_device *vdev = video_devdata(file); + struct mdp_dev *mdp = video_get_drvdata(vdev); + struct mdp_m2m_ctx *ctx; + struct device *dev = &mdp->pdev->dev; + int ret; + struct v4l2_format default_format = {}; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + if (mutex_lock_interruptible(&mdp->m2m_lock)) { + ret = -ERESTARTSYS; + goto err_free_ctx; + } + + ctx->id = ida_alloc(&mdp->mdp_ida, GFP_KERNEL); + ctx->mdp_dev = mdp; + + v4l2_fh_init(&ctx->fh, vdev); + file->private_data = &ctx->fh; + ret = mdp_m2m_ctrls_create(ctx); + if (ret) + goto err_exit_fh; + + /* Use separate control handler per file handle */ + ctx->fh.ctrl_handler = &ctx->ctrl_handler; + v4l2_fh_add(&ctx->fh); + + mutex_init(&ctx->ctx_lock); + ctx->m2m_ctx = v4l2_m2m_ctx_init(mdp->m2m_dev, ctx, mdp_m2m_queue_init); + if (IS_ERR(ctx->m2m_ctx)) { + dev_err(dev, "Failed to initialize m2m context\n"); + ret = PTR_ERR(ctx->m2m_ctx); + goto err_release_handler; + } + ctx->fh.m2m_ctx = ctx->m2m_ctx; + + ctx->curr_param.ctx = ctx; + ret = mdp_frameparam_init(&ctx->curr_param); + if (ret) { + dev_err(dev, "Failed to initialize mdp parameter\n"); + goto err_release_m2m_ctx; + } + + mutex_unlock(&mdp->m2m_lock); + + /* Default format */ + default_format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + default_format.fmt.pix_mp.width = 32; + default_format.fmt.pix_mp.height = 32; + default_format.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_YUV420M; + mdp_m2m_s_fmt_mplane(file, &ctx->fh, &default_format); + default_format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + mdp_m2m_s_fmt_mplane(file, &ctx->fh, &default_format); + + dev_dbg(dev, "%s:[%d]", __func__, ctx->id); + + return 0; + +err_release_m2m_ctx: + v4l2_m2m_ctx_release(ctx->m2m_ctx); +err_release_handler: + v4l2_ctrl_handler_free(&ctx->ctrl_handler); + v4l2_fh_del(&ctx->fh); +err_exit_fh: + v4l2_fh_exit(&ctx->fh); + mutex_unlock(&mdp->m2m_lock); +err_free_ctx: + kfree(ctx); + + return ret; +} + +static int mdp_m2m_release(struct file *file) +{ + struct mdp_m2m_ctx *ctx = fh_to_ctx(file->private_data); + struct mdp_dev *mdp = video_drvdata(file); + struct device *dev = &mdp->pdev->dev; + + mutex_lock(&mdp->m2m_lock); + v4l2_m2m_ctx_release(ctx->m2m_ctx); + if (mdp_m2m_ctx_is_state_set(ctx, MDP_VPU_INIT)) { + mdp_vpu_ctx_deinit(&ctx->vpu); + mdp_vpu_put_locked(mdp); + } + + v4l2_ctrl_handler_free(&ctx->ctrl_handler); + v4l2_fh_del(&ctx->fh); + v4l2_fh_exit(&ctx->fh); + ida_free(&mdp->mdp_ida, ctx->id); + mutex_unlock(&mdp->m2m_lock); + + dev_dbg(dev, "%s:[%d]", __func__, ctx->id); + kfree(ctx); + + return 0; +} + +static const struct v4l2_file_operations mdp_m2m_fops = { + .owner = THIS_MODULE, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, + .open = mdp_m2m_open, + .release = mdp_m2m_release, +}; + +static const struct v4l2_m2m_ops mdp_m2m_ops = { + .device_run = mdp_m2m_device_run, +}; + +int mdp_m2m_device_register(struct mdp_dev *mdp) +{ + struct device *dev = &mdp->pdev->dev; + int ret = 0; + + mdp->m2m_vdev = video_device_alloc(); + if (!mdp->m2m_vdev) { + dev_err(dev, "Failed to allocate video device\n"); + ret = -ENOMEM; + goto err_video_alloc; + } + mdp->m2m_vdev->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | + V4L2_CAP_STREAMING; + mdp->m2m_vdev->fops = &mdp_m2m_fops; + mdp->m2m_vdev->ioctl_ops = &mdp_m2m_ioctl_ops; + mdp->m2m_vdev->release = mdp_video_device_release; + mdp->m2m_vdev->lock = &mdp->m2m_lock; + mdp->m2m_vdev->vfl_dir = VFL_DIR_M2M; + mdp->m2m_vdev->v4l2_dev = &mdp->v4l2_dev; + snprintf(mdp->m2m_vdev->name, sizeof(mdp->m2m_vdev->name), "%s:m2m", + MDP_MODULE_NAME); + video_set_drvdata(mdp->m2m_vdev, mdp); + + mdp->m2m_dev = v4l2_m2m_init(&mdp_m2m_ops); + if (IS_ERR(mdp->m2m_dev)) { + dev_err(dev, "Failed to initialize v4l2-m2m device\n"); + ret = PTR_ERR(mdp->m2m_dev); + goto err_m2m_init; + } + + ret = video_register_device(mdp->m2m_vdev, VFL_TYPE_VIDEO, -1); + if (ret) { + dev_err(dev, "Failed to register video device\n"); + goto err_video_register; + } + + v4l2_info(&mdp->v4l2_dev, "Driver registered as /dev/video%d", + mdp->m2m_vdev->num); + return 0; + +err_video_register: + v4l2_m2m_release(mdp->m2m_dev); +err_m2m_init: + video_device_release(mdp->m2m_vdev); +err_video_alloc: + + return ret; +} + +void mdp_m2m_device_unregister(struct mdp_dev *mdp) +{ + video_unregister_device(mdp->m2m_vdev); +} + +void mdp_m2m_job_finish(struct mdp_m2m_ctx *ctx) +{ + enum vb2_buffer_state vb_state = VB2_BUF_STATE_DONE; + + mdp_m2m_process_done(ctx, vb_state); +} diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h new file mode 100644 index 0000000000000000000000000000000000000000..61ddbaf1bf139008b4930ed0cbf72a6b45346f94 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-m2m.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_M2M_H__ +#define __MTK_MDP3_M2M_H__ + +#include +#include "mtk-mdp3-core.h" +#include "mtk-mdp3-vpu.h" +#include "mtk-mdp3-regs.h" + +#define MDP_MAX_CTRLS 10 + +enum { + MDP_M2M_SRC = 0, + MDP_M2M_DST = 1, + MDP_M2M_MAX, +}; + +struct mdp_m2m_ctrls { + struct v4l2_ctrl *hflip; + struct v4l2_ctrl *vflip; + struct v4l2_ctrl *rotate; +}; + +struct mdp_m2m_ctx { + u32 id; + struct mdp_dev *mdp_dev; + struct v4l2_fh fh; + struct v4l2_ctrl_handler ctrl_handler; + struct mdp_m2m_ctrls ctrls; + struct v4l2_m2m_ctx *m2m_ctx; + struct mdp_vpu_ctx vpu; + u32 frame_count[MDP_M2M_MAX]; + + struct mdp_frameparam curr_param; + /* synchronization protect for mdp m2m context */ + struct mutex ctx_lock; +}; + +int mdp_m2m_device_register(struct mdp_dev *mdp); +void mdp_m2m_device_unregister(struct mdp_dev *mdp); +void mdp_m2m_job_finish(struct mdp_m2m_ctx *ctx); + +#endif /* __MTK_MDP3_M2M_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c new file mode 100644 index 0000000000000000000000000000000000000000..4e84a37ecdfc1a7ce0bbc3ad8d227d8654b819c3 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.c @@ -0,0 +1,735 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include +#include "mtk-mdp3-core.h" +#include "mtk-mdp3-regs.h" +#include "mtk-mdp3-m2m.h" + +/* + * All 10-bit related formats are not added in the basic format list, + * please add the corresponding format settings before use. + */ +static const struct mdp_format mdp_formats[] = { + { + .pixelformat = V4L2_PIX_FMT_GREY, + .mdp_color = MDP_COLOR_GREY, + .depth = { 8 }, + .row_depth = { 8 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_RGB565X, + .mdp_color = MDP_COLOR_BGR565, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_RGB565, + .mdp_color = MDP_COLOR_RGB565, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_RGB24, + .mdp_color = MDP_COLOR_RGB888, + .depth = { 24 }, + .row_depth = { 24 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_BGR24, + .mdp_color = MDP_COLOR_BGR888, + .depth = { 24 }, + .row_depth = { 24 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_ABGR32, + .mdp_color = MDP_COLOR_BGRA8888, + .depth = { 32 }, + .row_depth = { 32 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_ARGB32, + .mdp_color = MDP_COLOR_ARGB8888, + .depth = { 32 }, + .row_depth = { 32 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_UYVY, + .mdp_color = MDP_COLOR_UYVY, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_VYUY, + .mdp_color = MDP_COLOR_VYUY, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_YUYV, + .mdp_color = MDP_COLOR_YUYV, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_YVYU, + .mdp_color = MDP_COLOR_YVYU, + .depth = { 16 }, + .row_depth = { 16 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_YUV420, + .mdp_color = MDP_COLOR_I420, + .depth = { 12 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_YVU420, + .mdp_color = MDP_COLOR_YV12, + .depth = { 12 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_NV12, + .mdp_color = MDP_COLOR_NV12, + .depth = { 12 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_NV21, + .mdp_color = MDP_COLOR_NV21, + .depth = { 12 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_NV16, + .mdp_color = MDP_COLOR_NV16, + .depth = { 16 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_NV61, + .mdp_color = MDP_COLOR_NV61, + .depth = { 16 }, + .row_depth = { 8 }, + .num_planes = 1, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_NV24, + .mdp_color = MDP_COLOR_NV24, + .depth = { 24 }, + .row_depth = { 8 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_NV42, + .mdp_color = MDP_COLOR_NV42, + .depth = { 24 }, + .row_depth = { 8 }, + .num_planes = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_MT21C, + .mdp_color = MDP_COLOR_420_BLK_UFO, + .depth = { 8, 4 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 4, + .halign = 5, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_MM21, + .mdp_color = MDP_COLOR_420_BLK, + .depth = { 8, 4 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 4, + .halign = 5, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_NV12M, + .mdp_color = MDP_COLOR_NV12, + .depth = { 8, 4 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_NV21M, + .mdp_color = MDP_COLOR_NV21, + .depth = { 8, 4 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_NV16M, + .mdp_color = MDP_COLOR_NV16, + .depth = { 8, 8 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_NV61M, + .mdp_color = MDP_COLOR_NV61, + .depth = { 8, 8 }, + .row_depth = { 8, 8 }, + .num_planes = 2, + .walign = 1, + .flags = MDP_FMT_FLAG_OUTPUT, + }, { + .pixelformat = V4L2_PIX_FMT_YUV420M, + .mdp_color = MDP_COLOR_I420, + .depth = { 8, 2, 2 }, + .row_depth = { 8, 4, 4 }, + .num_planes = 3, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + }, { + .pixelformat = V4L2_PIX_FMT_YVU420M, + .mdp_color = MDP_COLOR_YV12, + .depth = { 8, 2, 2 }, + .row_depth = { 8, 4, 4 }, + .num_planes = 3, + .walign = 1, + .halign = 1, + .flags = MDP_FMT_FLAG_OUTPUT | MDP_FMT_FLAG_CAPTURE, + } +}; + +static const struct mdp_limit mdp_def_limit = { + .out_limit = { + .wmin = 16, + .hmin = 16, + .wmax = 8176, + .hmax = 8176, + }, + .cap_limit = { + .wmin = 2, + .hmin = 2, + .wmax = 8176, + .hmax = 8176, + }, + .h_scale_up_max = 32, + .v_scale_up_max = 32, + .h_scale_down_max = 20, + .v_scale_down_max = 128, +}; + +static const struct mdp_format *mdp_find_fmt(u32 pixelformat, u32 type) +{ + u32 i, flag; + + flag = V4L2_TYPE_IS_OUTPUT(type) ? MDP_FMT_FLAG_OUTPUT : + MDP_FMT_FLAG_CAPTURE; + for (i = 0; i < ARRAY_SIZE(mdp_formats); ++i) { + if (!(mdp_formats[i].flags & flag)) + continue; + if (mdp_formats[i].pixelformat == pixelformat) + return &mdp_formats[i]; + } + return NULL; +} + +static const struct mdp_format *mdp_find_fmt_by_index(u32 index, u32 type) +{ + u32 i, flag, num = 0; + + flag = V4L2_TYPE_IS_OUTPUT(type) ? MDP_FMT_FLAG_OUTPUT : + MDP_FMT_FLAG_CAPTURE; + for (i = 0; i < ARRAY_SIZE(mdp_formats); ++i) { + if (!(mdp_formats[i].flags & flag)) + continue; + if (index == num) + return &mdp_formats[i]; + num++; + } + return NULL; +} + +enum mdp_ycbcr_profile mdp_map_ycbcr_prof_mplane(struct v4l2_format *f, + u32 mdp_color) +{ + struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; + + if (MDP_COLOR_IS_RGB(mdp_color)) + return MDP_YCBCR_PROFILE_FULL_BT601; + + switch (pix_mp->colorspace) { + case V4L2_COLORSPACE_JPEG: + return MDP_YCBCR_PROFILE_JPEG; + case V4L2_COLORSPACE_REC709: + case V4L2_COLORSPACE_DCI_P3: + if (pix_mp->quantization == V4L2_QUANTIZATION_FULL_RANGE) + return MDP_YCBCR_PROFILE_FULL_BT709; + return MDP_YCBCR_PROFILE_BT709; + case V4L2_COLORSPACE_BT2020: + if (pix_mp->quantization == V4L2_QUANTIZATION_FULL_RANGE) + return MDP_YCBCR_PROFILE_FULL_BT2020; + return MDP_YCBCR_PROFILE_BT2020; + default: + if (pix_mp->quantization == V4L2_QUANTIZATION_FULL_RANGE) + return MDP_YCBCR_PROFILE_FULL_BT601; + return MDP_YCBCR_PROFILE_BT601; + } +} + +static void mdp_bound_align_image(u32 *w, u32 *h, + struct v4l2_frmsize_stepwise *s, + unsigned int salign) +{ + unsigned int org_w, org_h; + + org_w = *w; + org_h = *h; + v4l_bound_align_image(w, s->min_width, s->max_width, s->step_width, + h, s->min_height, s->max_height, s->step_height, + salign); + + s->min_width = org_w; + s->min_height = org_h; + v4l2_apply_frmsize_constraints(w, h, s); +} + +static int mdp_clamp_align(s32 *x, int min, int max, unsigned int align) +{ + unsigned int mask; + + if (min < 0 || max < 0) + return -ERANGE; + + /* Bits that must be zero to be aligned */ + mask = ~((1 << align) - 1); + + min = 0 ? 0 : ((min + ~mask) & mask); + max = max & mask; + if ((unsigned int)min > (unsigned int)max) + return -ERANGE; + + /* Clamp to aligned min and max */ + *x = clamp(*x, min, max); + + /* Round to nearest aligned value */ + if (align) + *x = (*x + (1 << (align - 1))) & mask; + return 0; +} + +int mdp_enum_fmt_mplane(struct v4l2_fmtdesc *f) +{ + const struct mdp_format *fmt; + + fmt = mdp_find_fmt_by_index(f->index, f->type); + if (!fmt) + return -EINVAL; + + f->pixelformat = fmt->pixelformat; + return 0; +} + +const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f, + struct mdp_frameparam *param, + u32 ctx_id) +{ + struct device *dev = ¶m->ctx->mdp_dev->pdev->dev; + struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; + const struct mdp_format *fmt; + const struct mdp_pix_limit *pix_limit; + struct v4l2_frmsize_stepwise s; + u32 org_w, org_h; + unsigned int i; + + fmt = mdp_find_fmt(pix_mp->pixelformat, f->type); + if (!fmt) { + fmt = mdp_find_fmt_by_index(0, f->type); + if (!fmt) { + dev_dbg(dev, "%d: pixelformat %c%c%c%c invalid", ctx_id, + (pix_mp->pixelformat & 0xff), + (pix_mp->pixelformat >> 8) & 0xff, + (pix_mp->pixelformat >> 16) & 0xff, + (pix_mp->pixelformat >> 24) & 0xff); + return NULL; + } + } + + pix_mp->field = V4L2_FIELD_NONE; + pix_mp->flags = 0; + pix_mp->pixelformat = fmt->pixelformat; + if (V4L2_TYPE_IS_CAPTURE(f->type)) { + pix_mp->colorspace = param->colorspace; + pix_mp->xfer_func = param->xfer_func; + pix_mp->ycbcr_enc = param->ycbcr_enc; + pix_mp->quantization = param->quant; + } + + pix_limit = V4L2_TYPE_IS_OUTPUT(f->type) ? ¶m->limit->out_limit : + ¶m->limit->cap_limit; + s.min_width = pix_limit->wmin; + s.max_width = pix_limit->wmax; + s.step_width = fmt->walign; + s.min_height = pix_limit->hmin; + s.max_height = pix_limit->hmax; + s.step_height = fmt->halign; + org_w = pix_mp->width; + org_h = pix_mp->height; + + mdp_bound_align_image(&pix_mp->width, &pix_mp->height, &s, fmt->salign); + if (org_w != pix_mp->width || org_h != pix_mp->height) + dev_dbg(dev, "%d: size change: %ux%u to %ux%u", ctx_id, + org_w, org_h, pix_mp->width, pix_mp->height); + + if (pix_mp->num_planes && pix_mp->num_planes != fmt->num_planes) + dev_dbg(dev, "%d num of planes change: %u to %u", ctx_id, + pix_mp->num_planes, fmt->num_planes); + pix_mp->num_planes = fmt->num_planes; + + for (i = 0; i < pix_mp->num_planes; ++i) { + u32 min_bpl = (pix_mp->width * fmt->row_depth[i]) >> 3; + u32 max_bpl = (pix_limit->wmax * fmt->row_depth[i]) >> 3; + u32 bpl = pix_mp->plane_fmt[i].bytesperline; + u32 min_si, max_si; + u32 si = pix_mp->plane_fmt[i].sizeimage; + + bpl = clamp(bpl, min_bpl, max_bpl); + pix_mp->plane_fmt[i].bytesperline = bpl; + + min_si = (bpl * pix_mp->height * fmt->depth[i]) / + fmt->row_depth[i]; + max_si = (bpl * s.max_height * fmt->depth[i]) / + fmt->row_depth[i]; + + si = clamp(si, min_si, max_si); + pix_mp->plane_fmt[i].sizeimage = si; + + dev_dbg(dev, "%d: p%u, bpl:%u [%u, %u], sizeimage:%u [%u, %u]", + ctx_id, i, bpl, min_bpl, max_bpl, si, min_si, max_si); + } + + return fmt; +} + +static int mdp_clamp_start(s32 *x, int min, int max, unsigned int align, + u32 flags) +{ + if (flags & V4L2_SEL_FLAG_GE) + max = *x; + if (flags & V4L2_SEL_FLAG_LE) + min = *x; + return mdp_clamp_align(x, min, max, align); +} + +static int mdp_clamp_end(s32 *x, int min, int max, unsigned int align, + u32 flags) +{ + if (flags & V4L2_SEL_FLAG_GE) + min = *x; + if (flags & V4L2_SEL_FLAG_LE) + max = *x; + return mdp_clamp_align(x, min, max, align); +} + +int mdp_try_crop(struct mdp_m2m_ctx *ctx, struct v4l2_rect *r, + const struct v4l2_selection *s, struct mdp_frame *frame) +{ + struct device *dev = &ctx->mdp_dev->pdev->dev; + s32 left, top, right, bottom; + u32 framew, frameh, walign, halign; + int ret; + + dev_dbg(dev, "%d target:%d, set:(%d,%d) %ux%u", ctx->id, + s->target, s->r.left, s->r.top, s->r.width, s->r.height); + + left = s->r.left; + top = s->r.top; + right = s->r.left + s->r.width; + bottom = s->r.top + s->r.height; + framew = frame->format.fmt.pix_mp.width; + frameh = frame->format.fmt.pix_mp.height; + + if (mdp_target_is_crop(s->target)) { + walign = 1; + halign = 1; + } else { + walign = frame->mdp_fmt->walign; + halign = frame->mdp_fmt->halign; + } + + dev_dbg(dev, "%d align:%u,%u, bound:%ux%u", ctx->id, + walign, halign, framew, frameh); + + ret = mdp_clamp_start(&left, 0, right, walign, s->flags); + if (ret) + return ret; + ret = mdp_clamp_start(&top, 0, bottom, halign, s->flags); + if (ret) + return ret; + ret = mdp_clamp_end(&right, left, framew, walign, s->flags); + if (ret) + return ret; + ret = mdp_clamp_end(&bottom, top, frameh, halign, s->flags); + if (ret) + return ret; + + r->left = left; + r->top = top; + r->width = right - left; + r->height = bottom - top; + + dev_dbg(dev, "%d crop:(%d,%d) %ux%u", ctx->id, + r->left, r->top, r->width, r->height); + return 0; +} + +int mdp_check_scaling_ratio(const struct v4l2_rect *crop, + const struct v4l2_rect *compose, s32 rotation, + const struct mdp_limit *limit) +{ + u32 crop_w, crop_h, comp_w, comp_h; + + crop_w = crop->width; + crop_h = crop->height; + if (90 == rotation || 270 == rotation) { + comp_w = compose->height; + comp_h = compose->width; + } else { + comp_w = compose->width; + comp_h = compose->height; + } + + if ((crop_w / comp_w) > limit->h_scale_down_max || + (crop_h / comp_h) > limit->v_scale_down_max || + (comp_w / crop_w) > limit->h_scale_up_max || + (comp_h / crop_h) > limit->v_scale_up_max) + return -ERANGE; + return 0; +} + +/* Stride that is accepted by MDP HW */ +static u32 mdp_fmt_get_stride(const struct mdp_format *fmt, + u32 bytesperline, unsigned int plane) +{ + enum mdp_color c = fmt->mdp_color; + u32 stride; + + stride = (bytesperline * MDP_COLOR_BITS_PER_PIXEL(c)) + / fmt->row_depth[0]; + if (plane == 0) + return stride; + if (plane < MDP_COLOR_GET_PLANE_COUNT(c)) { + if (MDP_COLOR_IS_BLOCK_MODE(c)) + stride = stride / 2; + return stride; + } + return 0; +} + +/* Stride that is accepted by MDP HW of format with contiguous planes */ +static u32 mdp_fmt_get_stride_contig(const struct mdp_format *fmt, + u32 pix_stride, unsigned int plane) +{ + enum mdp_color c = fmt->mdp_color; + u32 stride = pix_stride; + + if (plane == 0) + return stride; + if (plane < MDP_COLOR_GET_PLANE_COUNT(c)) { + stride = stride >> MDP_COLOR_GET_H_SUBSAMPLE(c); + if (MDP_COLOR_IS_UV_COPLANE(c) && !MDP_COLOR_IS_BLOCK_MODE(c)) + stride = stride * 2; + return stride; + } + return 0; +} + +/* Plane size that is accepted by MDP HW */ +static u32 mdp_fmt_get_plane_size(const struct mdp_format *fmt, + u32 stride, u32 height, unsigned int plane) +{ + enum mdp_color c = fmt->mdp_color; + u32 bytesperline; + + bytesperline = (stride * fmt->row_depth[0]) + / MDP_COLOR_BITS_PER_PIXEL(c); + if (plane == 0) + return bytesperline * height; + if (plane < MDP_COLOR_GET_PLANE_COUNT(c)) { + height = height >> MDP_COLOR_GET_V_SUBSAMPLE(c); + if (MDP_COLOR_IS_BLOCK_MODE(c)) + bytesperline = bytesperline * 2; + return bytesperline * height; + } + return 0; +} + +static void mdp_prepare_buffer(struct img_image_buffer *b, + struct mdp_frame *frame, struct vb2_buffer *vb) +{ + struct v4l2_pix_format_mplane *pix_mp = &frame->format.fmt.pix_mp; + unsigned int i; + + b->format.colorformat = frame->mdp_fmt->mdp_color; + b->format.ycbcr_prof = frame->ycbcr_prof; + for (i = 0; i < pix_mp->num_planes; ++i) { + u32 stride = mdp_fmt_get_stride(frame->mdp_fmt, + pix_mp->plane_fmt[i].bytesperline, i); + + b->format.plane_fmt[i].stride = stride; + b->format.plane_fmt[i].size = + mdp_fmt_get_plane_size(frame->mdp_fmt, stride, + pix_mp->height, i); + b->iova[i] = vb2_dma_contig_plane_dma_addr(vb, i); + } + for (; i < MDP_COLOR_GET_PLANE_COUNT(b->format.colorformat); ++i) { + u32 stride = mdp_fmt_get_stride_contig(frame->mdp_fmt, + b->format.plane_fmt[0].stride, i); + + b->format.plane_fmt[i].stride = stride; + b->format.plane_fmt[i].size = + mdp_fmt_get_plane_size(frame->mdp_fmt, stride, + pix_mp->height, i); + b->iova[i] = b->iova[i - 1] + b->format.plane_fmt[i - 1].size; + } + b->usage = frame->usage; +} + +void mdp_set_src_config(struct img_input *in, + struct mdp_frame *frame, struct vb2_buffer *vb) +{ + in->buffer.format.width = frame->format.fmt.pix_mp.width; + in->buffer.format.height = frame->format.fmt.pix_mp.height; + mdp_prepare_buffer(&in->buffer, frame, vb); +} + +static u32 mdp_to_fixed(u32 *r, struct v4l2_fract *f) +{ + u32 q; + + if (f->denominator == 0) { + *r = 0; + return 0; + } + + q = f->numerator / f->denominator; + *r = div_u64(((u64)f->numerator - q * f->denominator) << + IMG_SUBPIXEL_SHIFT, f->denominator); + return q; +} + +static void mdp_set_src_crop(struct img_crop *c, struct mdp_crop *crop) +{ + c->left = crop->c.left + + mdp_to_fixed(&c->left_subpix, &crop->left_subpix); + c->top = crop->c.top + + mdp_to_fixed(&c->top_subpix, &crop->top_subpix); + c->width = crop->c.width + + mdp_to_fixed(&c->width_subpix, &crop->width_subpix); + c->height = crop->c.height + + mdp_to_fixed(&c->height_subpix, &crop->height_subpix); +} + +static void mdp_set_orientation(struct img_output *out, + s32 rotation, bool hflip, bool vflip) +{ + u8 flip = 0; + + if (hflip) + flip ^= 1; + if (vflip) { + /* + * A vertical flip is equivalent to + * a 180-degree rotation with a horizontal flip + */ + rotation += 180; + flip ^= 1; + } + + out->rotation = rotation % 360; + if (flip != 0) + out->flags |= IMG_CTRL_FLAG_HFLIP; + else + out->flags &= ~IMG_CTRL_FLAG_HFLIP; +} + +void mdp_set_dst_config(struct img_output *out, + struct mdp_frame *frame, struct vb2_buffer *vb) +{ + out->buffer.format.width = frame->compose.width; + out->buffer.format.height = frame->compose.height; + mdp_prepare_buffer(&out->buffer, frame, vb); + mdp_set_src_crop(&out->crop, &frame->crop); + mdp_set_orientation(out, frame->rotation, frame->hflip, frame->vflip); +} + +int mdp_frameparam_init(struct mdp_frameparam *param) +{ + struct mdp_frame *frame; + + if (!param) + return -EINVAL; + + INIT_LIST_HEAD(¶m->list); + param->limit = &mdp_def_limit; + param->type = MDP_STREAM_TYPE_BITBLT; + + frame = ¶m->output; + frame->format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + frame->mdp_fmt = mdp_try_fmt_mplane(&frame->format, param, 0); + frame->ycbcr_prof = + mdp_map_ycbcr_prof_mplane(&frame->format, + frame->mdp_fmt->mdp_color); + frame->usage = MDP_BUFFER_USAGE_HW_READ; + + param->num_captures = 1; + frame = ¶m->captures[0]; + frame->format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + frame->mdp_fmt = mdp_try_fmt_mplane(&frame->format, param, 0); + frame->ycbcr_prof = + mdp_map_ycbcr_prof_mplane(&frame->format, + frame->mdp_fmt->mdp_color); + frame->usage = MDP_BUFFER_USAGE_MDP; + frame->crop.c.width = param->output.format.fmt.pix_mp.width; + frame->crop.c.height = param->output.format.fmt.pix_mp.height; + frame->compose.width = frame->format.fmt.pix_mp.width; + frame->compose.height = frame->format.fmt.pix_mp.height; + + return 0; +} diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h new file mode 100644 index 0000000000000000000000000000000000000000..f995e536d45fd4837118134b8fd4041823a89984 --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-regs.h @@ -0,0 +1,373 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_REGS_H__ +#define __MTK_MDP3_REGS_H__ + +#include +#include +#include "mtk-img-ipi.h" + +/* + * MDP native color code + * Plane count: 1, 2, 3 + * H-subsample: 0, 1, 2 + * V-subsample: 0, 1 + * Color group: 0-RGB, 1-YUV, 2-raw + */ +#define MDP_COLOR(PACKED, LOOSE, VIDEO, PLANE, HF, VF, BITS, GROUP, SWAP, ID)\ + (((PACKED) << 27) | ((LOOSE) << 26) | ((VIDEO) << 23) |\ + ((PLANE) << 21) | ((HF) << 19) | ((VF) << 18) | ((BITS) << 8) |\ + ((GROUP) << 6) | ((SWAP) << 5) | ((ID) << 0)) + +#define MDP_COLOR_IS_10BIT_PACKED(c) ((0x08000000 & (c)) >> 27) +#define MDP_COLOR_IS_10BIT_LOOSE(c) (((0x0c000000 & (c)) >> 26) == 1) +#define MDP_COLOR_IS_10BIT_TILE(c) (((0x0c000000 & (c)) >> 26) == 3) +#define MDP_COLOR_IS_UFP(c) ((0x02000000 & (c)) >> 25) +#define MDP_COLOR_IS_INTERLACED(c) ((0x01000000 & (c)) >> 24) +#define MDP_COLOR_IS_BLOCK_MODE(c) ((0x00800000 & (c)) >> 23) +#define MDP_COLOR_GET_PLANE_COUNT(c) ((0x00600000 & (c)) >> 21) +#define MDP_COLOR_GET_H_SUBSAMPLE(c) ((0x00180000 & (c)) >> 19) +#define MDP_COLOR_GET_V_SUBSAMPLE(c) ((0x00040000 & (c)) >> 18) +#define MDP_COLOR_BITS_PER_PIXEL(c) ((0x0003ff00 & (c)) >> 8) +#define MDP_COLOR_GET_GROUP(c) ((0x000000c0 & (c)) >> 6) +#define MDP_COLOR_IS_SWAPPED(c) ((0x00000020 & (c)) >> 5) +#define MDP_COLOR_GET_UNIQUE_ID(c) ((0x0000001f & (c)) >> 0) +#define MDP_COLOR_GET_HW_FORMAT(c) ((0x0000001f & (c)) >> 0) + +#define MDP_COLOR_IS_RGB(c) (MDP_COLOR_GET_GROUP(c) == 0) +#define MDP_COLOR_IS_YUV(c) (MDP_COLOR_GET_GROUP(c) == 1) + +enum mdp_color { + MDP_COLOR_UNKNOWN = 0, + + //MDP_COLOR_FULLG8, + MDP_COLOR_FULLG8_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 8, 2, 0, 21), + MDP_COLOR_FULLG8_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 8, 2, 0, 21), + MDP_COLOR_FULLG8_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 8, 2, 0, 21), + MDP_COLOR_FULLG8_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 8, 2, 0, 21), + MDP_COLOR_FULLG8 = MDP_COLOR_FULLG8_BGGR, + + //MDP_COLOR_FULLG10, + MDP_COLOR_FULLG10_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2, 0, 21), + MDP_COLOR_FULLG10_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 10, 2, 0, 21), + MDP_COLOR_FULLG10_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 10, 2, 0, 21), + MDP_COLOR_FULLG10_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 10, 2, 0, 21), + MDP_COLOR_FULLG10 = MDP_COLOR_FULLG10_BGGR, + + //MDP_COLOR_FULLG12, + MDP_COLOR_FULLG12_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2, 0, 21), + MDP_COLOR_FULLG12_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 12, 2, 0, 21), + MDP_COLOR_FULLG12_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 12, 2, 0, 21), + MDP_COLOR_FULLG12_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 12, 2, 0, 21), + MDP_COLOR_FULLG12 = MDP_COLOR_FULLG12_BGGR, + + //MDP_COLOR_FULLG14, + MDP_COLOR_FULLG14_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2, 0, 21), + MDP_COLOR_FULLG14_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 14, 2, 0, 21), + MDP_COLOR_FULLG14_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 14, 2, 0, 21), + MDP_COLOR_FULLG14_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 14, 2, 0, 21), + MDP_COLOR_FULLG14 = MDP_COLOR_FULLG14_BGGR, + + MDP_COLOR_UFO10 = MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2, 0, 24), + + //MDP_COLOR_BAYER8, + MDP_COLOR_BAYER8_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 8, 2, 0, 20), + MDP_COLOR_BAYER8_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 8, 2, 0, 20), + MDP_COLOR_BAYER8_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 8, 2, 0, 20), + MDP_COLOR_BAYER8_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 8, 2, 0, 20), + MDP_COLOR_BAYER8 = MDP_COLOR_BAYER8_BGGR, + + //MDP_COLOR_BAYER10, + MDP_COLOR_BAYER10_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2, 0, 20), + MDP_COLOR_BAYER10_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 10, 2, 0, 20), + MDP_COLOR_BAYER10_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 10, 2, 0, 20), + MDP_COLOR_BAYER10_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 10, 2, 0, 20), + MDP_COLOR_BAYER10 = MDP_COLOR_BAYER10_BGGR, + + //MDP_COLOR_BAYER12, + MDP_COLOR_BAYER12_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2, 0, 20), + MDP_COLOR_BAYER12_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 12, 2, 0, 20), + MDP_COLOR_BAYER12_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 12, 2, 0, 20), + MDP_COLOR_BAYER12_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 12, 2, 0, 20), + MDP_COLOR_BAYER12 = MDP_COLOR_BAYER12_BGGR, + + //MDP_COLOR_BAYER14, + MDP_COLOR_BAYER14_RGGB = MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2, 0, 20), + MDP_COLOR_BAYER14_GRBG = MDP_COLOR(0, 0, 0, 1, 0, 1, 14, 2, 0, 20), + MDP_COLOR_BAYER14_GBRG = MDP_COLOR(0, 0, 0, 1, 1, 0, 14, 2, 0, 20), + MDP_COLOR_BAYER14_BGGR = MDP_COLOR(0, 0, 0, 1, 1, 1, 14, 2, 0, 20), + MDP_COLOR_BAYER14 = MDP_COLOR_BAYER14_BGGR, + + MDP_COLOR_RGB48 = MDP_COLOR(0, 0, 0, 1, 0, 0, 48, 0, 0, 23), + /* For bayer+mono raw-16 */ + MDP_COLOR_RGB565_RAW = MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 2, 0, 0), + + MDP_COLOR_BAYER8_UNPAK = MDP_COLOR(0, 0, 0, 1, 0, 0, 8, 2, 0, 22), + MDP_COLOR_BAYER10_UNPAK = MDP_COLOR(0, 0, 0, 1, 0, 0, 10, 2, 0, 22), + MDP_COLOR_BAYER12_UNPAK = MDP_COLOR(0, 0, 0, 1, 0, 0, 12, 2, 0, 22), + MDP_COLOR_BAYER14_UNPAK = MDP_COLOR(0, 0, 0, 1, 0, 0, 14, 2, 0, 22), + + /* Unified formats */ + MDP_COLOR_GREY = MDP_COLOR(0, 0, 0, 1, 0, 0, 8, 1, 0, 7), + + MDP_COLOR_RGB565 = MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 0, 0, 0), + MDP_COLOR_BGR565 = MDP_COLOR(0, 0, 0, 1, 0, 0, 16, 0, 1, 0), + MDP_COLOR_RGB888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 0, 1, 1), + MDP_COLOR_BGR888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 0, 0, 1), + MDP_COLOR_RGBA8888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0, 1, 2), + MDP_COLOR_BGRA8888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0, 0, 2), + MDP_COLOR_ARGB8888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0, 1, 3), + MDP_COLOR_ABGR8888 = MDP_COLOR(0, 0, 0, 1, 0, 0, 32, 0, 0, 3), + + MDP_COLOR_UYVY = MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1, 0, 4), + MDP_COLOR_VYUY = MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1, 1, 4), + MDP_COLOR_YUYV = MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1, 0, 5), + MDP_COLOR_YVYU = MDP_COLOR(0, 0, 0, 1, 1, 0, 16, 1, 1, 5), + + MDP_COLOR_I420 = MDP_COLOR(0, 0, 0, 3, 1, 1, 8, 1, 0, 8), + MDP_COLOR_YV12 = MDP_COLOR(0, 0, 0, 3, 1, 1, 8, 1, 1, 8), + MDP_COLOR_I422 = MDP_COLOR(0, 0, 0, 3, 1, 0, 8, 1, 0, 9), + MDP_COLOR_YV16 = MDP_COLOR(0, 0, 0, 3, 1, 0, 8, 1, 1, 9), + MDP_COLOR_I444 = MDP_COLOR(0, 0, 0, 3, 0, 0, 8, 1, 0, 10), + MDP_COLOR_YV24 = MDP_COLOR(0, 0, 0, 3, 0, 0, 8, 1, 1, 10), + + MDP_COLOR_NV12 = MDP_COLOR(0, 0, 0, 2, 1, 1, 8, 1, 0, 12), + MDP_COLOR_NV21 = MDP_COLOR(0, 0, 0, 2, 1, 1, 8, 1, 1, 12), + MDP_COLOR_NV16 = MDP_COLOR(0, 0, 0, 2, 1, 0, 8, 1, 0, 13), + MDP_COLOR_NV61 = MDP_COLOR(0, 0, 0, 2, 1, 0, 8, 1, 1, 13), + MDP_COLOR_NV24 = MDP_COLOR(0, 0, 0, 2, 0, 0, 8, 1, 0, 14), + MDP_COLOR_NV42 = MDP_COLOR(0, 0, 0, 2, 0, 0, 8, 1, 1, 14), + + /* MediaTek proprietary formats */ + /* UFO encoded block mode */ + MDP_COLOR_420_BLK_UFO = MDP_COLOR(0, 0, 5, 2, 1, 1, 256, 1, 0, 12), + /* Block mode */ + MDP_COLOR_420_BLK = MDP_COLOR(0, 0, 1, 2, 1, 1, 256, 1, 0, 12), + /* Block mode + field mode */ + MDP_COLOR_420_BLKI = MDP_COLOR(0, 0, 3, 2, 1, 1, 256, 1, 0, 12), + /* Block mode */ + MDP_COLOR_422_BLK = MDP_COLOR(0, 0, 1, 1, 1, 0, 512, 1, 0, 4), + + MDP_COLOR_IYU2 = MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 1, 0, 25), + MDP_COLOR_YUV444 = MDP_COLOR(0, 0, 0, 1, 0, 0, 24, 1, 0, 30), + + /* Packed 10-bit formats */ + MDP_COLOR_RGBA1010102 = MDP_COLOR(1, 0, 0, 1, 0, 0, 32, 0, 1, 2), + MDP_COLOR_BGRA1010102 = MDP_COLOR(1, 0, 0, 1, 0, 0, 32, 0, 0, 2), + /* Packed 10-bit UYVY */ + MDP_COLOR_UYVY_10P = MDP_COLOR(1, 0, 0, 1, 1, 0, 20, 1, 0, 4), + /* Packed 10-bit NV21 */ + MDP_COLOR_NV21_10P = MDP_COLOR(1, 0, 0, 2, 1, 1, 10, 1, 1, 12), + /* 10-bit block mode */ + MDP_COLOR_420_BLK_10_H = MDP_COLOR(1, 0, 1, 2, 1, 1, 320, 1, 0, 12), + /* 10-bit HEVC tile mode */ + MDP_COLOR_420_BLK_10_V = MDP_COLOR(1, 1, 1, 2, 1, 1, 320, 1, 0, 12), + /* UFO encoded 10-bit block mode */ + MDP_COLOR_420_BLK_U10_H = MDP_COLOR(1, 0, 5, 2, 1, 1, 320, 1, 0, 12), + /* UFO encoded 10-bit HEVC tile mode */ + MDP_COLOR_420_BLK_U10_V = MDP_COLOR(1, 1, 5, 2, 1, 1, 320, 1, 0, 12), + + /* Loose 10-bit formats */ + MDP_COLOR_UYVY_10L = MDP_COLOR(0, 1, 0, 1, 1, 0, 20, 1, 0, 4), + MDP_COLOR_VYUY_10L = MDP_COLOR(0, 1, 0, 1, 1, 0, 20, 1, 1, 4), + MDP_COLOR_YUYV_10L = MDP_COLOR(0, 1, 0, 1, 1, 0, 20, 1, 0, 5), + MDP_COLOR_YVYU_10L = MDP_COLOR(0, 1, 0, 1, 1, 0, 20, 1, 1, 5), + MDP_COLOR_NV12_10L = MDP_COLOR(0, 1, 0, 2, 1, 1, 10, 1, 0, 12), + MDP_COLOR_NV21_10L = MDP_COLOR(0, 1, 0, 2, 1, 1, 10, 1, 1, 12), + MDP_COLOR_NV16_10L = MDP_COLOR(0, 1, 0, 2, 1, 0, 10, 1, 0, 13), + MDP_COLOR_NV61_10L = MDP_COLOR(0, 1, 0, 2, 1, 0, 10, 1, 1, 13), + MDP_COLOR_YV12_10L = MDP_COLOR(0, 1, 0, 3, 1, 1, 10, 1, 1, 8), + MDP_COLOR_I420_10L = MDP_COLOR(0, 1, 0, 3, 1, 1, 10, 1, 0, 8), +}; + +static inline bool MDP_COLOR_IS_UV_COPLANE(enum mdp_color c) +{ + return (MDP_COLOR_GET_PLANE_COUNT(c) == 2 && MDP_COLOR_IS_YUV(c)); +} + +/* Minimum Y stride that is accepted by MDP HW */ +static inline u32 mdp_color_get_min_y_stride(enum mdp_color c, u32 width) +{ + return ((MDP_COLOR_BITS_PER_PIXEL(c) * width) + 4) >> 3; +} + +/* Minimum UV stride that is accepted by MDP HW */ +static inline u32 mdp_color_get_min_uv_stride(enum mdp_color c, u32 width) +{ + u32 min_stride; + + if (MDP_COLOR_GET_PLANE_COUNT(c) == 1) + return 0; + min_stride = mdp_color_get_min_y_stride(c, width) + >> MDP_COLOR_GET_H_SUBSAMPLE(c); + if (MDP_COLOR_IS_UV_COPLANE(c) && !MDP_COLOR_IS_BLOCK_MODE(c)) + min_stride = min_stride * 2; + return min_stride; +} + +/* Minimum Y plane size that is necessary in buffer */ +static inline u32 mdp_color_get_min_y_size(enum mdp_color c, + u32 width, u32 height) +{ + if (MDP_COLOR_IS_BLOCK_MODE(c)) + return ((MDP_COLOR_BITS_PER_PIXEL(c) * width) >> 8) * height; + return mdp_color_get_min_y_stride(c, width) * height; +} + +/* Minimum UV plane size that is necessary in buffer */ +static inline u32 mdp_color_get_min_uv_size(enum mdp_color c, + u32 width, u32 height) +{ + height = height >> MDP_COLOR_GET_V_SUBSAMPLE(c); + if (MDP_COLOR_IS_BLOCK_MODE(c) && (MDP_COLOR_GET_PLANE_COUNT(c) > 1)) + return ((MDP_COLOR_BITS_PER_PIXEL(c) * width) >> 8) * height; + return mdp_color_get_min_uv_stride(c, width) * height; +} + +/* Combine colorspace, xfer_func, ycbcr_encoding, and quantization */ +enum mdp_ycbcr_profile { + /* V4L2_YCBCR_ENC_601 and V4L2_QUANTIZATION_LIM_RANGE */ + MDP_YCBCR_PROFILE_BT601, + /* V4L2_YCBCR_ENC_709 and V4L2_QUANTIZATION_LIM_RANGE */ + MDP_YCBCR_PROFILE_BT709, + /* V4L2_YCBCR_ENC_601 and V4L2_QUANTIZATION_FULL_RANGE */ + MDP_YCBCR_PROFILE_JPEG, + MDP_YCBCR_PROFILE_FULL_BT601 = MDP_YCBCR_PROFILE_JPEG, + + /* Colorspaces not support for capture */ + /* V4L2_YCBCR_ENC_BT2020 and V4L2_QUANTIZATION_LIM_RANGE */ + MDP_YCBCR_PROFILE_BT2020, + /* V4L2_YCBCR_ENC_709 and V4L2_QUANTIZATION_FULL_RANGE */ + MDP_YCBCR_PROFILE_FULL_BT709, + /* V4L2_YCBCR_ENC_BT2020 and V4L2_QUANTIZATION_FULL_RANGE */ + MDP_YCBCR_PROFILE_FULL_BT2020, +}; + +#define MDP_FMT_FLAG_OUTPUT BIT(0) +#define MDP_FMT_FLAG_CAPTURE BIT(1) + +struct mdp_format { + u32 pixelformat; + u32 mdp_color; + u8 depth[VIDEO_MAX_PLANES]; + u8 row_depth[VIDEO_MAX_PLANES]; + u8 num_planes; + u8 walign; + u8 halign; + u8 salign; + u32 flags; +}; + +struct mdp_pix_limit { + u32 wmin; + u32 hmin; + u32 wmax; + u32 hmax; +}; + +struct mdp_limit { + struct mdp_pix_limit out_limit; + struct mdp_pix_limit cap_limit; + u32 h_scale_up_max; + u32 v_scale_up_max; + u32 h_scale_down_max; + u32 v_scale_down_max; +}; + +enum mdp_stream_type { + MDP_STREAM_TYPE_UNKNOWN, + MDP_STREAM_TYPE_BITBLT, + MDP_STREAM_TYPE_GPU_BITBLT, + MDP_STREAM_TYPE_DUAL_BITBLT, + MDP_STREAM_TYPE_2ND_BITBLT, + MDP_STREAM_TYPE_ISP_IC, + MDP_STREAM_TYPE_ISP_VR, + MDP_STREAM_TYPE_ISP_ZSD, + MDP_STREAM_TYPE_ISP_IP, + MDP_STREAM_TYPE_ISP_VSS, + MDP_STREAM_TYPE_ISP_ZSD_SLOW, + MDP_STREAM_TYPE_WPE, + MDP_STREAM_TYPE_WPE2, +}; + +struct mdp_crop { + struct v4l2_rect c; + struct v4l2_fract left_subpix; + struct v4l2_fract top_subpix; + struct v4l2_fract width_subpix; + struct v4l2_fract height_subpix; +}; + +struct mdp_frame { + struct v4l2_format format; + const struct mdp_format *mdp_fmt; + u32 ycbcr_prof; /* enum mdp_ycbcr_profile */ + u32 usage; /* enum mdp_buffer_usage */ + struct mdp_crop crop; + struct v4l2_rect compose; + s32 rotation; + u32 hflip:1; + u32 vflip:1; + u32 hdr:1; + u32 dre:1; + u32 sharpness:1; + u32 dither:1; +}; + +static inline bool mdp_target_is_crop(u32 target) +{ + return (target == V4L2_SEL_TGT_CROP) || + (target == V4L2_SEL_TGT_CROP_DEFAULT) || + (target == V4L2_SEL_TGT_CROP_BOUNDS); +} + +static inline bool mdp_target_is_compose(u32 target) +{ + return (target == V4L2_SEL_TGT_COMPOSE) || + (target == V4L2_SEL_TGT_COMPOSE_DEFAULT) || + (target == V4L2_SEL_TGT_COMPOSE_BOUNDS); +} + +#define MDP_MAX_CAPTURES IMG_MAX_HW_OUTPUTS + +#define MDP_VPU_INIT BIT(0) +#define MDP_M2M_CTX_ERROR BIT(1) + +struct mdp_frameparam { + struct list_head list; + struct mdp_m2m_ctx *ctx; + atomic_t state; + const struct mdp_limit *limit; + u32 type; /* enum mdp_stream_type */ + u32 frame_no; + struct mdp_frame output; + struct mdp_frame captures[MDP_MAX_CAPTURES]; + u32 num_captures; + enum v4l2_colorspace colorspace; + enum v4l2_ycbcr_encoding ycbcr_enc; + enum v4l2_xfer_func xfer_func; + enum v4l2_quantization quant; +}; + +int mdp_enum_fmt_mplane(struct v4l2_fmtdesc *f); +const struct mdp_format *mdp_try_fmt_mplane(struct v4l2_format *f, + struct mdp_frameparam *param, + u32 ctx_id); +enum mdp_ycbcr_profile mdp_map_ycbcr_prof_mplane(struct v4l2_format *f, + u32 mdp_color); +int mdp_try_crop(struct mdp_m2m_ctx *ctx, struct v4l2_rect *r, + const struct v4l2_selection *s, struct mdp_frame *frame); +int mdp_check_scaling_ratio(const struct v4l2_rect *crop, + const struct v4l2_rect *compose, s32 rotation, + const struct mdp_limit *limit); +void mdp_set_src_config(struct img_input *in, + struct mdp_frame *frame, struct vb2_buffer *vb); +void mdp_set_dst_config(struct img_output *out, + struct mdp_frame *frame, struct vb2_buffer *vb); +int mdp_frameparam_init(struct mdp_frameparam *param); + +#endif /* __MTK_MDP3_REGS_H__ */ diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c new file mode 100644 index 0000000000000000000000000000000000000000..9f5844385c8fc1229bf55ac028ef963c72e0ad4a --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.c @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#include +#include +#include "mtk-mdp3-vpu.h" +#include "mtk-mdp3-core.h" + +#define MDP_VPU_MESSAGE_TIMEOUT 500U +#define vpu_alloc_size 0x600000 + +static inline struct mdp_dev *vpu_to_mdp(struct mdp_vpu_dev *vpu) +{ + return container_of(vpu, struct mdp_dev, vpu); +} + +static int mdp_vpu_shared_mem_alloc(struct mdp_vpu_dev *vpu) +{ + if (vpu->work && vpu->work_addr) + return 0; + + vpu->work = dma_alloc_coherent(scp_get_device(vpu->scp), vpu_alloc_size, + &vpu->work_addr, GFP_KERNEL); + + if (!vpu->work) + return -ENOMEM; + else + return 0; +} + +void mdp_vpu_shared_mem_free(struct mdp_vpu_dev *vpu) +{ + if (vpu->work && vpu->work_addr) + dma_free_coherent(scp_get_device(vpu->scp), vpu_alloc_size, + vpu->work, vpu->work_addr); +} + +static void mdp_vpu_ipi_handle_init_ack(void *data, unsigned int len, + void *priv) +{ + struct mdp_ipi_init_msg *msg = (struct mdp_ipi_init_msg *)data; + struct mdp_vpu_dev *vpu = + (struct mdp_vpu_dev *)(unsigned long)msg->drv_data; + + if (!vpu->work_size) + vpu->work_size = msg->work_size; + + vpu->status = msg->status; + complete(&vpu->ipi_acked); +} + +static void mdp_vpu_ipi_handle_deinit_ack(void *data, unsigned int len, + void *priv) +{ + struct mdp_ipi_deinit_msg *msg = (struct mdp_ipi_deinit_msg *)data; + struct mdp_vpu_dev *vpu = + (struct mdp_vpu_dev *)(unsigned long)msg->drv_data; + + vpu->status = msg->status; + complete(&vpu->ipi_acked); +} + +static void mdp_vpu_ipi_handle_frame_ack(void *data, unsigned int len, + void *priv) +{ + struct img_sw_addr *addr = (struct img_sw_addr *)data; + struct img_ipi_frameparam *param = + (struct img_ipi_frameparam *)(unsigned long)addr->va; + struct mdp_vpu_ctx *ctx = + (struct mdp_vpu_ctx *)(unsigned long)param->drv_data; + + if (param->state) { + struct mdp_dev *mdp = vpu_to_mdp(ctx->vpu_dev); + + dev_err(&mdp->pdev->dev, "VPU MDP failure:%d\n", param->state); + } + ctx->vpu_dev->status = param->state; + complete(&ctx->vpu_dev->ipi_acked); +} + +int mdp_vpu_register(struct mdp_dev *mdp) +{ + int err; + struct mtk_scp *scp = mdp->scp; + struct device *dev = &mdp->pdev->dev; + + err = scp_ipi_register(scp, SCP_IPI_MDP_INIT, + mdp_vpu_ipi_handle_init_ack, NULL); + if (err) { + dev_err(dev, "scp_ipi_register failed %d\n", err); + goto err_ipi_init; + } + err = scp_ipi_register(scp, SCP_IPI_MDP_DEINIT, + mdp_vpu_ipi_handle_deinit_ack, NULL); + if (err) { + dev_err(dev, "scp_ipi_register failed %d\n", err); + goto err_ipi_deinit; + } + err = scp_ipi_register(scp, SCP_IPI_MDP_FRAME, + mdp_vpu_ipi_handle_frame_ack, NULL); + if (err) { + dev_err(dev, "scp_ipi_register failed %d\n", err); + goto err_ipi_frame; + } + return 0; + +err_ipi_frame: + scp_ipi_unregister(scp, SCP_IPI_MDP_DEINIT); +err_ipi_deinit: + scp_ipi_unregister(scp, SCP_IPI_MDP_INIT); +err_ipi_init: + + return err; +} + +void mdp_vpu_unregister(struct mdp_dev *mdp) +{ + scp_ipi_unregister(mdp->scp, SCP_IPI_MDP_INIT); + scp_ipi_unregister(mdp->scp, SCP_IPI_MDP_DEINIT); + scp_ipi_unregister(mdp->scp, SCP_IPI_MDP_FRAME); +} + +static int mdp_vpu_sendmsg(struct mdp_vpu_dev *vpu, enum scp_ipi_id id, + void *buf, unsigned int len) +{ + struct mdp_dev *mdp = vpu_to_mdp(vpu); + unsigned int t = MDP_VPU_MESSAGE_TIMEOUT; + int ret; + + if (!vpu->scp) { + dev_dbg(&mdp->pdev->dev, "vpu scp is NULL"); + return -EINVAL; + } + ret = scp_ipi_send(vpu->scp, id, buf, len, 2000); + + if (ret) { + dev_err(&mdp->pdev->dev, "scp_ipi_send failed %d\n", ret); + return -EPERM; + } + ret = wait_for_completion_timeout(&vpu->ipi_acked, + msecs_to_jiffies(t)); + if (!ret) + ret = -ETIME; + else if (vpu->status) + ret = -EINVAL; + else + ret = 0; + return ret; +} + +int mdp_vpu_dev_init(struct mdp_vpu_dev *vpu, struct mtk_scp *scp, + struct mutex *lock) +{ + struct mdp_ipi_init_msg msg = { + .drv_data = (unsigned long)vpu, + }; + size_t mem_size; + phys_addr_t pool; + const size_t pool_size = sizeof(struct mdp_config_pool); + struct mdp_dev *mdp = vpu_to_mdp(vpu); + int err; + + init_completion(&vpu->ipi_acked); + vpu->scp = scp; + vpu->lock = lock; + vpu->work_size = 0; + err = mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_INIT, &msg, sizeof(msg)); + if (err) + goto err_work_size; + /* vpu work_size was set in mdp_vpu_ipi_handle_init_ack */ + + mem_size = vpu_alloc_size; + if (mdp_vpu_shared_mem_alloc(vpu)) { + dev_err(&mdp->pdev->dev, "VPU memory alloc fail!"); + goto err_mem_alloc; + } + + pool = ALIGN((uintptr_t)vpu->work + vpu->work_size, 8); + if (pool + pool_size - (uintptr_t)vpu->work > mem_size) { + dev_err(&mdp->pdev->dev, + "VPU memory insufficient: %zx + %zx > %zx", + vpu->work_size, pool_size, mem_size); + err = -ENOMEM; + goto err_mem_size; + } + + dev_dbg(&mdp->pdev->dev, + "VPU work:%pK pa:%pad sz:%zx pool:%pa sz:%zx (mem sz:%zx)", + vpu->work, &vpu->work_addr, vpu->work_size, + &pool, pool_size, mem_size); + vpu->pool = (struct mdp_config_pool *)(uintptr_t)pool; + msg.work_addr = vpu->work_addr; + msg.work_size = vpu->work_size; + err = mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_INIT, &msg, sizeof(msg)); + if (err) + goto err_work_size; + + memset(vpu->pool, 0, sizeof(*vpu->pool)); + return 0; + +err_work_size: + switch (vpu->status) { + case -MDP_IPI_EBUSY: + err = -EBUSY; + break; + case -MDP_IPI_ENOMEM: + err = -ENOSPC; /* -ENOMEM */ + break; + } + return err; +err_mem_size: +err_mem_alloc: + return err; +} + +int mdp_vpu_dev_deinit(struct mdp_vpu_dev *vpu) +{ + struct mdp_ipi_deinit_msg msg = { + .drv_data = (unsigned long)vpu, + .work_addr = vpu->work_addr, + }; + + return mdp_vpu_sendmsg(vpu, SCP_IPI_MDP_DEINIT, &msg, sizeof(msg)); +} + +static struct img_config *mdp_config_get(struct mdp_vpu_dev *vpu, + enum mdp_config_id id, uint32_t *addr) +{ + struct img_config *config; + + if (id < 0 || id >= MDP_CONFIG_POOL_SIZE) + return ERR_PTR(-EINVAL); + + mutex_lock(vpu->lock); + vpu->pool->cfg_count[id]++; + config = &vpu->pool->configs[id]; + *addr = vpu->work_addr + ((uintptr_t)config - (uintptr_t)vpu->work); + mutex_unlock(vpu->lock); + + return config; +} + +static int mdp_config_put(struct mdp_vpu_dev *vpu, + enum mdp_config_id id, + const struct img_config *config) +{ + int err = 0; + + if (id < 0 || id >= MDP_CONFIG_POOL_SIZE) + return -EINVAL; + if (vpu->lock) + mutex_lock(vpu->lock); + if (!vpu->pool->cfg_count[id] || config != &vpu->pool->configs[id]) + err = -EINVAL; + else + vpu->pool->cfg_count[id]--; + if (vpu->lock) + mutex_unlock(vpu->lock); + return err; +} + +int mdp_vpu_ctx_init(struct mdp_vpu_ctx *ctx, struct mdp_vpu_dev *vpu, + enum mdp_config_id id) +{ + ctx->config = mdp_config_get(vpu, id, &ctx->inst_addr); + if (IS_ERR(ctx->config)) { + int err = PTR_ERR(ctx->config); + + ctx->config = NULL; + return err; + } + ctx->config_id = id; + ctx->vpu_dev = vpu; + return 0; +} + +int mdp_vpu_ctx_deinit(struct mdp_vpu_ctx *ctx) +{ + int err = mdp_config_put(ctx->vpu_dev, ctx->config_id, ctx->config); + + ctx->config_id = 0; + ctx->config = NULL; + ctx->inst_addr = 0; + return err; +} + +int mdp_vpu_process(struct mdp_vpu_ctx *ctx, struct img_ipi_frameparam *param) +{ + struct mdp_vpu_dev *vpu = ctx->vpu_dev; + struct mdp_dev *mdp = vpu_to_mdp(vpu); + struct img_sw_addr addr; + + if (!ctx->vpu_dev->work || !ctx->vpu_dev->work_addr) { + if (mdp_vpu_shared_mem_alloc(vpu)) { + dev_err(&mdp->pdev->dev, "VPU memory alloc fail!"); + return -ENOMEM; + } + } + memset((void *)ctx->vpu_dev->work, 0, ctx->vpu_dev->work_size); + memset(ctx->config, 0, sizeof(*ctx->config)); + param->config_data.va = (unsigned long)ctx->config; + param->config_data.pa = ctx->inst_addr; + param->drv_data = (unsigned long)ctx; + + memcpy((void *)ctx->vpu_dev->work, param, sizeof(*param)); + addr.pa = ctx->vpu_dev->work_addr; + addr.va = (uintptr_t)ctx->vpu_dev->work; + return mdp_vpu_sendmsg(ctx->vpu_dev, SCP_IPI_MDP_FRAME, + &addr, sizeof(addr)); +} diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h new file mode 100644 index 0000000000000000000000000000000000000000..244b3a32d689ec4034301f8b70165781e8ef773e --- /dev/null +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-vpu.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 MediaTek Inc. + * Author: Ping-Hsun Wu + */ + +#ifndef __MTK_MDP3_VPU_H__ +#define __MTK_MDP3_VPU_H__ + +#include +#include "mtk-img-ipi.h" + +enum mdp_ipi_result { + MDP_IPI_SUCCESS = 0, + MDP_IPI_ENOMEM = 12, + MDP_IPI_EBUSY = 16, + MDP_IPI_EINVAL = 22, + MDP_IPI_EMINST = 24, + MDP_IPI_ERANGE = 34, + MDP_IPI_NR_ERRNO, + + MDP_IPI_EOTHER = MDP_IPI_NR_ERRNO, + MDP_IPI_PATH_CANT_MERGE, + MDP_IPI_OP_FAIL, +}; + +struct mdp_ipi_init_msg { + u32 status; + u64 drv_data; + u32 work_addr; /* [in] working buffer address */ + u32 work_size; /* [in] working buffer size */ +} __packed; + +struct mdp_ipi_deinit_msg { + u32 status; + u64 drv_data; + u32 work_addr; +} __packed; + +enum mdp_config_id { + MDP_DEV_M2M = 0, + MDP_CONFIG_POOL_SIZE /* ALWAYS keep at the end */ +}; + +struct mdp_config_pool { + u64 cfg_count[MDP_CONFIG_POOL_SIZE]; + struct img_config configs[MDP_CONFIG_POOL_SIZE]; +}; + +struct mdp_vpu_dev { + /* synchronization protect for accessing vpu working buffer info */ + struct mutex *lock; + struct mtk_scp *scp; + struct completion ipi_acked; + void *work; + dma_addr_t work_addr; + size_t work_size; + struct mdp_config_pool *pool; + u32 status; +}; + +struct mdp_vpu_ctx { + struct mdp_vpu_dev *vpu_dev; + u32 config_id; + struct img_config *config; + u32 inst_addr; +}; + +void mdp_vpu_shared_mem_free(struct mdp_vpu_dev *vpu); +int mdp_vpu_dev_init(struct mdp_vpu_dev *vpu, struct mtk_scp *scp, + struct mutex *lock /* for sync */); +int mdp_vpu_dev_deinit(struct mdp_vpu_dev *vpu); +int mdp_vpu_ctx_init(struct mdp_vpu_ctx *ctx, struct mdp_vpu_dev *vpu, + enum mdp_config_id id); +int mdp_vpu_ctx_deinit(struct mdp_vpu_ctx *ctx); +int mdp_vpu_process(struct mdp_vpu_ctx *vpu, struct img_ipi_frameparam *param); + +#endif /* __MTK_MDP3_VPU_H__ */ diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c index 7d194a476713341a4287621c989f7b5a1468b866..641f533c417fd3764d18ec1328b1945b5e4e271b 100644 --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec.c @@ -227,6 +227,8 @@ static int mtk_vcodec_dec_get_chip_name(void *priv) return 8195; else if (of_device_is_compatible(dev->of_node, "mediatek,mt8186-vcodec-dec")) return 8186; + else if (of_device_is_compatible(dev->of_node, "mediatek,mt8188-vcodec-dec")) + return 8188; else return 8173; } diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c index e0b6ae9d6caaa2562cfab2c3e9de8cda2ac678a2..174a6eec2f549afd80f7eb7c97d690e2b3f38308 100644 --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_dec_drv.c @@ -478,6 +478,10 @@ static const struct of_device_id mtk_vcodec_match[] = { .compatible = "mediatek,mt8195-vcodec-dec", .data = &mtk_lat_sig_core_pdata, }, + { + .compatible = "mediatek,mt8188-vcodec-dec", + .data = &mtk_lat_sig_core_pdata, + }, {}, }; diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h index ef4584a46417e755da95357169ff7c92f74c57d5..9acab54fd65017cc48d840593a8ad416be83e085 100644 --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_drv.h @@ -278,6 +278,7 @@ struct vdec_pic_info { * @hw_id: hardware index used to identify different hardware. * * @msg_queue: msg queue used to store lat buffer information. + * @q_mutex: vb2_queue mutex. */ struct mtk_vcodec_ctx { enum mtk_instance_type type; @@ -324,6 +325,8 @@ struct mtk_vcodec_ctx { int hw_id; struct vdec_msg_queue msg_queue; + + struct mutex q_mutex; }; /* @@ -401,6 +404,7 @@ struct mtk_vcodec_dec_pdata { * @output_formats: array of supported output formats * @num_output_formats: number of entries in output_formats * @core_id: stand for h264 or vp8 encode index + * @uses_34bit: whether the encoder uses 34-bit iova */ struct mtk_vcodec_enc_pdata { bool uses_ext; @@ -411,9 +415,11 @@ struct mtk_vcodec_enc_pdata { const struct mtk_video_fmt *output_formats; size_t num_output_formats; int core_id; + bool uses_34bit; }; #define MTK_ENC_CTX_IS_EXT(ctx) ((ctx)->dev->venc_pdata->uses_ext) +#define MTK_ENC_IOVA_IS_34BIT(ctx) ((ctx)->dev->venc_pdata->uses_34bit) /** * struct mtk_vcodec_dev - driver data diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c index 25e81686359752e349940edf06aad92f3d588d2e..d810a78dde51d95f5428804d782ca52e23ae074f 100644 --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc.c @@ -225,6 +225,8 @@ static int mtk_vcodec_enc_get_chip_name(void *priv) return 8192; else if (of_device_is_compatible(dev->of_node, "mediatek,mt8195-vcodec-enc")) return 8195; + else if (of_device_is_compatible(dev->of_node, "mediatek,mt8188-vcodec-enc")) + return 8188; else return 8173; } @@ -503,13 +505,13 @@ static int vidioc_venc_s_fmt_out(struct file *file, void *priv, f->fmt.pix.pixelformat = fmt->fourcc; } - ret = vidioc_try_fmt_out(ctx, f, fmt); + q_data->visible_width = f->fmt.pix_mp.width; + q_data->visible_height = f->fmt.pix_mp.height; + q_data->fmt = fmt; + ret = vidioc_try_fmt_out(ctx, f, q_data->fmt); if (ret) return ret; - q_data->fmt = fmt; - q_data->visible_width = f->fmt.pix_mp.width; - q_data->visible_height = f->fmt.pix_mp.height; q_data->coded_width = f->fmt.pix_mp.width; q_data->coded_height = f->fmt.pix_mp.height; @@ -1300,7 +1302,7 @@ void mtk_vcodec_enc_set_default_params(struct mtk_vcodec_ctx *ctx) { struct mtk_q_data *q_data; - ctx->m2m_ctx->q_lock = &ctx->dev->dev_mutex; + ctx->m2m_ctx->q_lock = &ctx->q_mutex; ctx->fh.m2m_ctx = ctx->m2m_ctx; ctx->fh.ctrl_handler = &ctx->ctrl_hdl; INIT_WORK(&ctx->encode_work, mtk_venc_worker); @@ -1403,7 +1405,8 @@ int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx) V4L2_MPEG_VIDEO_VP8_PROFILE_0, 0, V4L2_MPEG_VIDEO_VP8_PROFILE_0); v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_BITRATE_MODE, V4L2_MPEG_VIDEO_BITRATE_MODE_CBR, - 0, V4L2_MPEG_VIDEO_BITRATE_MODE_CBR); + ~(1 << V4L2_MPEG_VIDEO_BITRATE_MODE_CBR), + V4L2_MPEG_VIDEO_BITRATE_MODE_CBR); if (handler->error) { @@ -1435,7 +1438,7 @@ int mtk_vcodec_enc_queue_init(void *priv, struct vb2_queue *src_vq, src_vq->ops = &mtk_venc_vb2_ops; src_vq->mem_ops = &vb2_dma_contig_memops; src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; - src_vq->lock = &ctx->dev->dev_mutex; + src_vq->lock = &ctx->q_mutex; src_vq->dev = &ctx->dev->plat_dev->dev; ret = vb2_queue_init(src_vq); @@ -1449,7 +1452,7 @@ int mtk_vcodec_enc_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->ops = &mtk_venc_vb2_ops; dst_vq->mem_ops = &vb2_dma_contig_memops; dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; - dst_vq->lock = &ctx->dev->dev_mutex; + dst_vq->lock = &ctx->q_mutex; dst_vq->dev = &ctx->dev->plat_dev->dev; return vb2_queue_init(dst_vq); diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c index 95e8c29ccc651158ec3cf0610a1f52efc34b326d..9095186d549533a1c1b33e162c9641129b2a4c02 100644 --- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c +++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c @@ -130,6 +130,7 @@ static int fops_vcodec_open(struct file *file) INIT_LIST_HEAD(&ctx->list); ctx->dev = dev; init_waitqueue_head(&ctx->queue[0]); + mutex_init(&ctx->q_mutex); ctx->type = MTK_INST_ENCODER; ret = mtk_vcodec_enc_ctrls_setup(ctx); @@ -228,7 +229,6 @@ static int mtk_vcodec_probe(struct platform_device *pdev) { struct mtk_vcodec_dev *dev; struct video_device *vfd_enc; - struct resource *res; phandle rproc_phandle; enum mtk_vcodec_fw_type fw_type; int ret; @@ -272,14 +272,12 @@ static int mtk_vcodec_probe(struct platform_device *pdev) goto err_res; } - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (res == NULL) { - dev_err(&pdev->dev, "failed to get irq resource"); - ret = -ENOENT; + dev->enc_irq = platform_get_irq(pdev, 0); + if (dev->enc_irq < 0) { + ret = dev->enc_irq; goto err_res; } - dev->enc_irq = platform_get_irq(pdev, 0); irq_set_status_flags(dev->enc_irq, IRQ_NOAUTOEN); ret = devm_request_irq(&pdev->dev, dev->enc_irq, mtk_vcodec_enc_irq_handler, @@ -406,6 +404,18 @@ static const struct mtk_vcodec_enc_pdata mt8183_pdata = { .core_id = VENC_SYS, }; +static const struct mtk_vcodec_enc_pdata mt8188_pdata = { + .uses_ext = true, + .capture_formats = mtk_video_formats_capture_h264, + .num_capture_formats = ARRAY_SIZE(mtk_video_formats_capture_h264), + .output_formats = mtk_video_formats_output, + .num_output_formats = ARRAY_SIZE(mtk_video_formats_output), + .min_bitrate = 64, + .max_bitrate = 50000000, + .core_id = VENC_SYS, + .uses_34bit = true, +}; + static const struct mtk_vcodec_enc_pdata mt8192_pdata = { .uses_ext = true, .capture_formats = mtk_video_formats_capture_h264, @@ -434,6 +444,7 @@ static const struct of_device_id mtk_vcodec_enc_match[] = { {.compatible = "mediatek,mt8173-vcodec-enc-vp8", .data = &mt8173_vp8_pdata}, {.compatible = "mediatek,mt8183-vcodec-enc", .data = &mt8183_pdata}, + {.compatible = "mediatek,mt8188-vcodec-enc", .data = &mt8188_pdata}, {.compatible = "mediatek,mt8192-vcodec-enc", .data = &mt8192_pdata}, {.compatible = "mediatek,mt8195-vcodec-enc", .data = &mt8195_pdata}, {}, diff --git a/drivers/media/platform/mediatek/vcodec/venc/venc_h264_if.c b/drivers/media/platform/mediatek/vcodec/venc/venc_h264_if.c index 4d9b8798dffe18330b96f58455eae31157d2c45c..13c4f860fa69cf154b03c8f060ab1fdef8e0bce3 100644 --- a/drivers/media/platform/mediatek/vcodec/venc/venc_h264_if.c +++ b/drivers/media/platform/mediatek/vcodec/venc/venc_h264_if.c @@ -127,6 +127,72 @@ struct venc_h264_vsi { struct venc_h264_vpu_buf work_bufs[VENC_H264_VPU_WORK_BUF_MAX]; }; +/** + * struct venc_h264_vpu_config_ext - Structure for h264 encoder configuration + * AP-W/R : AP is writer/reader on this item + * VPU-W/R: VPU is write/reader on this item + * @input_fourcc: input fourcc + * @bitrate: target bitrate (in bps) + * @pic_w: picture width. Picture size is visible stream resolution, in pixels, + * to be used for display purposes; must be smaller or equal to buffer + * size. + * @pic_h: picture height + * @buf_w: buffer width. Buffer size is stream resolution in pixels aligned to + * hardware requirements. + * @buf_h: buffer height + * @gop_size: group of picture size (idr frame) + * @intra_period: intra frame period + * @framerate: frame rate in fps + * @profile: as specified in standard + * @level: as specified in standard + * @wfd: WFD mode 1:on, 0:off + * @max_qp: max quant parameter + * @min_qp: min quant parameter + * @reserved: reserved configs + */ +struct venc_h264_vpu_config_ext { + u32 input_fourcc; + u32 bitrate; + u32 pic_w; + u32 pic_h; + u32 buf_w; + u32 buf_h; + u32 gop_size; + u32 intra_period; + u32 framerate; + u32 profile; + u32 level; + u32 wfd; + u32 max_qp; + u32 min_qp; + u32 reserved[8]; +}; + +/** + * struct venc_h264_vpu_buf_34 - Structure for 34-bit buffer information + * AP-W/R : AP is writer/reader on this item + * VPU-W/R: VPU is write/reader on this item + * @iova: 34-bit IO virtual address + * @vpua: VPU side memory addr which is used by RC_CODE + * @size: buffer size (in bytes) + */ +struct venc_h264_vpu_buf_34 { + u64 iova; + u32 vpua; + u32 size; +}; + +/** + * struct venc_h264_vsi_34 - Structure for VPU driver control and info share + * Used for 34-bit iova sharing + * @config: h264 encoder configuration + * @work_bufs: working buffer information in VPU side + */ +struct venc_h264_vsi_34 { + struct venc_h264_vpu_config_ext config; + struct venc_h264_vpu_buf_34 work_bufs[VENC_H264_VPU_WORK_BUF_MAX]; +}; + /* * struct venc_h264_inst - h264 encoder AP driver instance * @hw_base: h264 encoder hardware register base @@ -140,6 +206,8 @@ struct venc_h264_vsi { * @vpu_inst: VPU instance to exchange information between AP and VPU * @vsi: driver structure allocated by VPU side and shared to AP side for * control and info share + * @vsi_34: driver structure allocated by VPU side and shared to AP side for + * control and info share, used for 34-bit iova sharing. * @ctx: context for v4l2 layer integration */ struct venc_h264_inst { @@ -152,6 +220,7 @@ struct venc_h264_inst { unsigned int prepend_hdr; struct venc_vpu_inst vpu_inst; struct venc_h264_vsi *vsi; + struct venc_h264_vsi_34 *vsi_34; struct mtk_vcodec_ctx *ctx; }; @@ -244,14 +313,21 @@ static void h264_enc_free_work_buf(struct venc_h264_inst *inst) mtk_vcodec_debug_leave(inst); } -static int h264_enc_alloc_work_buf(struct venc_h264_inst *inst) +static int h264_enc_alloc_work_buf(struct venc_h264_inst *inst, bool is_34bit) { + struct venc_h264_vpu_buf *wb = NULL; + struct venc_h264_vpu_buf_34 *wb_34 = NULL; int i; + u32 vpua, wb_size; int ret = 0; - struct venc_h264_vpu_buf *wb = inst->vsi->work_bufs; mtk_vcodec_debug_enter(inst); + if (is_34bit) + wb_34 = inst->vsi_34->work_bufs; + else + wb = inst->vsi->work_bufs; + for (i = 0; i < VENC_H264_VPU_WORK_BUF_MAX; i++) { /* * This 'wb' structure is set by VPU side and shared to AP for @@ -269,13 +345,22 @@ static int h264_enc_alloc_work_buf(struct venc_h264_inst *inst) * address and do some memcpy access to move to bitstream buffer * assigned by v4l2 layer. */ - inst->work_bufs[i].size = wb[i].size; + if (is_34bit) { + inst->work_bufs[i].size = wb_34[i].size; + vpua = wb_34[i].vpua; + wb_size = wb_34[i].size; + } else { + inst->work_bufs[i].size = wb[i].size; + vpua = wb[i].vpua; + wb_size = wb[i].size; + } + if (i == VENC_H264_VPU_WORK_BUF_SKIP_FRAME) { struct mtk_vcodec_fw *handler; handler = inst->vpu_inst.ctx->dev->fw_handler; inst->work_bufs[i].va = - mtk_vcodec_fw_map_dm_addr(handler, wb[i].vpua); + mtk_vcodec_fw_map_dm_addr(handler, vpua); inst->work_bufs[i].dma_addr = 0; } else { ret = mtk_vcodec_mem_alloc(inst->ctx, @@ -297,12 +382,14 @@ static int h264_enc_alloc_work_buf(struct venc_h264_inst *inst) handler = inst->vpu_inst.ctx->dev->fw_handler; tmp_va = mtk_vcodec_fw_map_dm_addr(handler, - wb[i].vpua); - memcpy(inst->work_bufs[i].va, tmp_va, - wb[i].size); + vpua); + memcpy(inst->work_bufs[i].va, tmp_va, wb_size); } } - wb[i].iova = inst->work_bufs[i].dma_addr; + if (is_34bit) + wb_34[i].iova = inst->work_bufs[i].dma_addr; + else + wb[i].iova = inst->work_bufs[i].dma_addr; mtk_vcodec_debug(inst, "work_buf[%d] va=0x%p iova=%pad size=%zu", @@ -342,22 +429,22 @@ static unsigned int h264_enc_wait_venc_done(struct venc_h264_inst *inst) return irq_status; } -static int h264_frame_type(struct venc_h264_inst *inst) +static int h264_frame_type(unsigned int frm_cnt, unsigned int gop_size, + unsigned int intra_period) { - if ((inst->vsi->config.gop_size != 0 && - (inst->frm_cnt % inst->vsi->config.gop_size) == 0) || - (inst->frm_cnt == 0 && inst->vsi->config.gop_size == 0)) { + if ((gop_size != 0 && (frm_cnt % gop_size) == 0) || + (frm_cnt == 0 && gop_size == 0)) { /* IDR frame */ return VENC_H264_IDR_FRM; - } else if ((inst->vsi->config.intra_period != 0 && - (inst->frm_cnt % inst->vsi->config.intra_period) == 0) || - (inst->frm_cnt == 0 && inst->vsi->config.intra_period == 0)) { + } else if ((intra_period != 0 && (frm_cnt % intra_period) == 0) || + (frm_cnt == 0 && intra_period == 0)) { /* I frame */ return VENC_H264_I_FRM; } else { return VENC_H264_P_FRM; /* Note: B frames are not supported */ } } + static int h264_encode_sps(struct venc_h264_inst *inst, struct mtk_vcodec_mem *bs_buf, unsigned int *bs_size) @@ -438,18 +525,32 @@ static int h264_encode_frame(struct venc_h264_inst *inst, unsigned int *bs_size) { int ret = 0; + unsigned int gop_size; + unsigned int intra_period; unsigned int irq_status; struct venc_frame_info frame_info; + struct mtk_vcodec_ctx *ctx = inst->ctx; mtk_vcodec_debug_enter(inst); mtk_vcodec_debug(inst, "frm_cnt = %d\n ", inst->frm_cnt); + + if (MTK_ENC_IOVA_IS_34BIT(ctx)) { + gop_size = inst->vsi_34->config.gop_size; + intra_period = inst->vsi_34->config.intra_period; + } else { + gop_size = inst->vsi->config.gop_size; + intra_period = inst->vsi->config.intra_period; + } frame_info.frm_count = inst->frm_cnt; frame_info.skip_frm_count = inst->skip_frm_cnt; - frame_info.frm_type = h264_frame_type(inst); + frame_info.frm_type = h264_frame_type(inst->frm_cnt, gop_size, + intra_period); mtk_vcodec_debug(inst, "frm_count = %d,skip_frm_count =%d,frm_type=%d.\n", frame_info.frm_count, frame_info.skip_frm_count, frame_info.frm_type); - ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_FRAME, frm_buf, bs_buf, &frame_info); + + ret = vpu_enc_encode(&inst->vpu_inst, H264_BS_MODE_FRAME, + frm_buf, bs_buf, &frame_info); if (ret) return ret; @@ -517,7 +618,10 @@ static int h264_enc_init(struct mtk_vcodec_ctx *ctx) ret = vpu_enc_init(&inst->vpu_inst); - inst->vsi = (struct venc_h264_vsi *)inst->vpu_inst.vsi; + if (MTK_ENC_IOVA_IS_34BIT(ctx)) + inst->vsi_34 = (struct venc_h264_vsi_34 *)inst->vpu_inst.vsi; + else + inst->vsi = (struct venc_h264_vsi *)inst->vpu_inst.vsi; mtk_vcodec_debug_leave(inst); @@ -624,31 +728,61 @@ encode_err: return ret; } +static void h264_enc_set_vsi_configs(struct venc_h264_inst *inst, + struct venc_enc_param *enc_prm) +{ + inst->vsi->config.input_fourcc = enc_prm->input_yuv_fmt; + inst->vsi->config.bitrate = enc_prm->bitrate; + inst->vsi->config.pic_w = enc_prm->width; + inst->vsi->config.pic_h = enc_prm->height; + inst->vsi->config.buf_w = enc_prm->buf_width; + inst->vsi->config.buf_h = enc_prm->buf_height; + inst->vsi->config.gop_size = enc_prm->gop_size; + inst->vsi->config.framerate = enc_prm->frm_rate; + inst->vsi->config.intra_period = enc_prm->intra_period; + inst->vsi->config.profile = + h264_get_profile(inst, enc_prm->h264_profile); + inst->vsi->config.level = + h264_get_level(inst, enc_prm->h264_level); + inst->vsi->config.wfd = 0; +} + +static void h264_enc_set_vsi_34_configs(struct venc_h264_inst *inst, + struct venc_enc_param *enc_prm) +{ + inst->vsi_34->config.input_fourcc = enc_prm->input_yuv_fmt; + inst->vsi_34->config.bitrate = enc_prm->bitrate; + inst->vsi_34->config.pic_w = enc_prm->width; + inst->vsi_34->config.pic_h = enc_prm->height; + inst->vsi_34->config.buf_w = enc_prm->buf_width; + inst->vsi_34->config.buf_h = enc_prm->buf_height; + inst->vsi_34->config.gop_size = enc_prm->gop_size; + inst->vsi_34->config.framerate = enc_prm->frm_rate; + inst->vsi_34->config.intra_period = enc_prm->intra_period; + inst->vsi_34->config.profile = + h264_get_profile(inst, enc_prm->h264_profile); + inst->vsi_34->config.level = + h264_get_level(inst, enc_prm->h264_level); + inst->vsi_34->config.wfd = 0; +} + static int h264_enc_set_param(void *handle, enum venc_set_param_type type, struct venc_enc_param *enc_prm) { int ret = 0; struct venc_h264_inst *inst = (struct venc_h264_inst *)handle; + struct mtk_vcodec_ctx *ctx = inst->ctx; + const bool is_34bit = MTK_ENC_IOVA_IS_34BIT(ctx); mtk_vcodec_debug(inst, "->type=%d", type); switch (type) { case VENC_SET_PARAM_ENC: - inst->vsi->config.input_fourcc = enc_prm->input_yuv_fmt; - inst->vsi->config.bitrate = enc_prm->bitrate; - inst->vsi->config.pic_w = enc_prm->width; - inst->vsi->config.pic_h = enc_prm->height; - inst->vsi->config.buf_w = enc_prm->buf_width; - inst->vsi->config.buf_h = enc_prm->buf_height; - inst->vsi->config.gop_size = enc_prm->gop_size; - inst->vsi->config.framerate = enc_prm->frm_rate; - inst->vsi->config.intra_period = enc_prm->intra_period; - inst->vsi->config.profile = - h264_get_profile(inst, enc_prm->h264_profile); - inst->vsi->config.level = - h264_get_level(inst, enc_prm->h264_level); - inst->vsi->config.wfd = 0; + if (is_34bit) + h264_enc_set_vsi_34_configs(inst, enc_prm); + else + h264_enc_set_vsi_configs(inst, enc_prm); ret = vpu_enc_set_param(&inst->vpu_inst, type, enc_prm); if (ret) break; @@ -656,7 +790,7 @@ static int h264_enc_set_param(void *handle, h264_enc_free_work_buf(inst); inst->work_buf_allocated = false; } - ret = h264_enc_alloc_work_buf(inst); + ret = h264_enc_alloc_work_buf(inst, is_34bit); if (ret) break; inst->work_buf_allocated = true; diff --git a/drivers/media/platform/mediatek/vcodec/venc_ipi_msg.h b/drivers/media/platform/mediatek/vcodec/venc_ipi_msg.h index 587a2cf15b765cea354e7615b50c5a5d3831282e..bb16d96a7f571e2689f184b4dab334d7c31b8854 100644 --- a/drivers/media/platform/mediatek/vcodec/venc_ipi_msg.h +++ b/drivers/media/platform/mediatek/vcodec/venc_ipi_msg.h @@ -100,6 +100,30 @@ struct venc_ap_ipi_msg_enc_ext { uint32_t data[32]; }; +/** + * struct venc_ap_ipi_msg_enc_ext_34 - AP to SCP extended enc cmd structure + * @msg_id: message id (AP_IPIMSG_XXX_ENC_ENCODE) + * @vpu_inst_addr: VPU encoder instance addr + * @bs_mode: bitstream mode for h264 + * @reserved: for struct padding + * @input_addr: input frame buffer 34 bit address + * @bs_addr: output bitstream buffer 34 bit address + * @bs_size: bitstream buffer size + * @data_item: number of items in the data array + * @data: data array to store the set parameters + */ +struct venc_ap_ipi_msg_enc_ext_34 { + u32 msg_id; + u32 vpu_inst_addr; + u32 bs_mode; + u32 reserved; + u64 input_addr[3]; + u64 bs_addr; + u32 bs_size; + u32 data_item; + u32 data[32]; +}; + /** * struct venc_ap_ipi_msg_deinit - AP to VPU deinit cmd structure * @msg_id: message id (AP_IPIMSG_XXX_ENC_DEINIT) diff --git a/drivers/media/platform/mediatek/vcodec/venc_vpu_if.c b/drivers/media/platform/mediatek/vcodec/venc_vpu_if.c index d3570c4c177d7ba315f684f628626b37f80b2e74..09e7eaa25aabe2df169f6dc51b0ae65c10dcf91d 100644 --- a/drivers/media/platform/mediatek/vcodec/venc_vpu_if.c +++ b/drivers/media/platform/mediatek/vcodec/venc_vpu_if.c @@ -222,10 +222,11 @@ int vpu_enc_set_param(struct venc_vpu_inst *vpu, return 0; } -int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode, - struct venc_frm_buf *frm_buf, - struct mtk_vcodec_mem *bs_buf, - struct venc_frame_info *frame_info) +static int vpu_enc_encode_32bits(struct venc_vpu_inst *vpu, + unsigned int bs_mode, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_frame_info *frame_info) { const bool is_ext = MTK_ENC_CTX_IS_EXT(vpu->ctx); size_t msg_size = is_ext ? @@ -267,6 +268,73 @@ int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode, return -EINVAL; } + return 0; +} + +static int vpu_enc_encode_34bits(struct venc_vpu_inst *vpu, + unsigned int bs_mode, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_frame_info *frame_info) +{ + struct venc_ap_ipi_msg_enc_ext_34 out; + size_t msg_size = sizeof(struct venc_ap_ipi_msg_enc_ext_34); + + mtk_vcodec_debug(vpu, "bs_mode %d ->", bs_mode); + + memset(&out, 0, sizeof(out)); + out.msg_id = AP_IPIMSG_ENC_ENCODE; + out.vpu_inst_addr = vpu->inst_addr; + out.bs_mode = bs_mode; + + if (frm_buf) { + if ((frm_buf->fb_addr[0].dma_addr % 16 == 0) && + (frm_buf->fb_addr[1].dma_addr % 16 == 0) && + (frm_buf->fb_addr[2].dma_addr % 16 == 0)) { + out.input_addr[0] = frm_buf->fb_addr[0].dma_addr; + out.input_addr[1] = frm_buf->fb_addr[1].dma_addr; + out.input_addr[2] = frm_buf->fb_addr[2].dma_addr; + } else { + mtk_vcodec_err(vpu, "dma_addr not align to 16"); + return -EINVAL; + } + } + if (bs_buf) { + out.bs_addr = bs_buf->dma_addr; + out.bs_size = bs_buf->size; + } + if (frame_info) { + out.data_item = 3; + out.data[0] = frame_info->frm_count; + out.data[1] = frame_info->skip_frm_count; + out.data[2] = frame_info->frm_type; + } + if (vpu_enc_send_msg(vpu, &out, msg_size)) { + mtk_vcodec_err(vpu, "AP_IPIMSG_ENC_ENCODE %d fail", + bs_mode); + return -EINVAL; + } + + return 0; +} + +int vpu_enc_encode(struct venc_vpu_inst *vpu, unsigned int bs_mode, + struct venc_frm_buf *frm_buf, + struct mtk_vcodec_mem *bs_buf, + struct venc_frame_info *frame_info) +{ + int ret; + + if (MTK_ENC_IOVA_IS_34BIT(vpu->ctx)) + ret = vpu_enc_encode_34bits(vpu, bs_mode, + frm_buf, bs_buf, frame_info); + else + ret = vpu_enc_encode_32bits(vpu, bs_mode, + frm_buf, bs_buf, frame_info); + + if (ret) + return ret; + mtk_vcodec_debug(vpu, "bs_mode %d state %d size %d key_frm %d <-", bs_mode, vpu->state, vpu->bs_size, vpu->is_key_frm); diff --git a/drivers/media/platform/nxp/Kconfig b/drivers/media/platform/nxp/Kconfig index 1ac0a6e9111119db96e5b21794e2c709c3c013b2..5917634889b564c149f05f683a3e90ec01dd4c27 100644 --- a/drivers/media/platform/nxp/Kconfig +++ b/drivers/media/platform/nxp/Kconfig @@ -15,18 +15,6 @@ config VIDEO_IMX_MIPI_CSIS Video4Linux2 sub-device driver for the MIPI CSI-2 CSIS receiver v3.3/v3.6.3 found on some i.MX7 and i.MX8 SoCs. -config VIDEO_VIU - tristate "NXP VIU Video Driver" - depends on V4L_PLATFORM_DRIVERS - depends on VIDEO_DEV && (PPC_MPC512x || COMPILE_TEST) && I2C - select VIDEOBUF_DMA_CONTIG - help - Support for Freescale VIU video driver. This device captures - video data, or overlays video on DIU frame buffer. - - Say Y here if you want to enable VIU device on MPC5121e Rev2+. - In doubt, say N. - # mem2mem drivers config VIDEO_IMX_PXP @@ -51,4 +39,5 @@ config VIDEO_MX2_EMMAPRP memory to memory. Operations include resizing and format conversion. +source "drivers/media/platform/nxp/dw100/Kconfig" source "drivers/media/platform/nxp/imx-jpeg/Kconfig" diff --git a/drivers/media/platform/nxp/Makefile b/drivers/media/platform/nxp/Makefile index efc38c6578ce5b59a7344a4270117dac65a0f520..81ab304ef31c2997fd9df34f5db8798c5b2a96d9 100644 --- a/drivers/media/platform/nxp/Makefile +++ b/drivers/media/platform/nxp/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-y += dw100/ obj-y += imx-jpeg/ obj-$(CONFIG_VIDEO_IMX_MIPI_CSIS) += imx-mipi-csis.o obj-$(CONFIG_VIDEO_IMX_PXP) += imx-pxp.o obj-$(CONFIG_VIDEO_MX2_EMMAPRP) += mx2_emmaprp.o -obj-$(CONFIG_VIDEO_VIU) += fsl-viu.o diff --git a/drivers/media/platform/nxp/dw100/Kconfig b/drivers/media/platform/nxp/dw100/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..cd4531bb3110221ff606922a4c4e7b26019058f9 --- /dev/null +++ b/drivers/media/platform/nxp/dw100/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_DW100 + tristate "NXP i.MX DW100 dewarper" + depends on V4L_MEM2MEM_DRIVERS + depends on VIDEO_DEV + depends on ARCH_MXC || COMPILE_TEST + select MEDIA_CONTROLLER + select V4L2_MEM2MEM_DEV + select VIDEOBUF2_DMA_CONTIG + help + DW100 is a memory-to-memory engine performing geometrical + transformation on source images through a programmable dewarping map. + + To compile this driver as a module, choose M here: the module + will be called dw100. diff --git a/drivers/media/platform/nxp/dw100/Makefile b/drivers/media/platform/nxp/dw100/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..49db80589e9a64fcad46b4868aae0c8c158dae3e --- /dev/null +++ b/drivers/media/platform/nxp/dw100/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0+ + +obj-$(CONFIG_VIDEO_DW100) += dw100.o diff --git a/drivers/media/platform/nxp/dw100/dw100.c b/drivers/media/platform/nxp/dw100/dw100.c new file mode 100644 index 0000000000000000000000000000000000000000..b3b057798ab67f50f1b9c77651cdd264f944003e --- /dev/null +++ b/drivers/media/platform/nxp/dw100/dw100.c @@ -0,0 +1,1707 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * DW100 Hardware dewarper + * + * Copyright 2022 NXP + * Author: Xavier Roumegue (xavier.roumegue@oss.nxp.com) + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "dw100_regs.h" + +#define DRV_NAME "dw100" + +#define DW100_MIN_W 176u +#define DW100_MIN_H 144u +#define DW100_MAX_W 4096u +#define DW100_MAX_H 3072u +#define DW100_ALIGN_W 3 +#define DW100_ALIGN_H 3 + +#define DW100_BLOCK_SIZE 16 + +#define DW100_DEF_W 640u +#define DW100_DEF_H 480u +#define DW100_DEF_LUT_W (DIV_ROUND_UP(DW100_DEF_W, DW100_BLOCK_SIZE) + 1) +#define DW100_DEF_LUT_H (DIV_ROUND_UP(DW100_DEF_H, DW100_BLOCK_SIZE) + 1) + +/* + * 16 controls have been reserved for this driver for future extension, but + * let's limit the related driver allocation to the effective number of controls + * in use. + */ +#define DW100_MAX_CTRLS 1 +#define DW100_CTRL_DEWARPING_MAP 0 + +enum { + DW100_QUEUE_SRC = 0, + DW100_QUEUE_DST = 1, +}; + +enum { + DW100_FMT_CAPTURE = BIT(0), + DW100_FMT_OUTPUT = BIT(1), +}; + +struct dw100_device { + struct platform_device *pdev; + struct v4l2_m2m_dev *m2m_dev; + struct v4l2_device v4l2_dev; + struct video_device vfd; + struct media_device mdev; + /* Video device lock */ + struct mutex vfd_mutex; + void __iomem *mmio; + struct clk_bulk_data *clks; + int num_clks; + struct dentry *debugfs_root; +}; + +struct dw100_q_data { + struct v4l2_pix_format_mplane pix_fmt; + unsigned int sequence; + const struct dw100_fmt *fmt; + struct v4l2_rect crop; +}; + +struct dw100_ctx { + struct v4l2_fh fh; + struct dw100_device *dw_dev; + struct v4l2_ctrl_handler hdl; + struct v4l2_ctrl *ctrls[DW100_MAX_CTRLS]; + /* per context m2m queue lock */ + struct mutex vq_mutex; + + /* Look Up Table for pixel remapping */ + unsigned int *map; + dma_addr_t map_dma; + size_t map_size; + unsigned int map_width; + unsigned int map_height; + bool user_map_is_set; + + /* Source and destination queue data */ + struct dw100_q_data q_data[2]; +}; + +static const struct v4l2_frmsize_stepwise dw100_frmsize_stepwise = { + .min_width = DW100_MIN_W, + .min_height = DW100_MIN_H, + .max_width = DW100_MAX_W, + .max_height = DW100_MAX_H, + .step_width = 1UL << DW100_ALIGN_W, + .step_height = 1UL << DW100_ALIGN_H, +}; + +static const struct dw100_fmt { + u32 fourcc; + u32 types; + u32 reg_format; + bool reg_swap_uv; +} formats[] = { + { + .fourcc = V4L2_PIX_FMT_NV16, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_SP, + .reg_swap_uv = false, + }, { + .fourcc = V4L2_PIX_FMT_NV16M, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_SP, + .reg_swap_uv = false, + }, { + .fourcc = V4L2_PIX_FMT_NV61, + .types = DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_SP, + .reg_swap_uv = true, + }, { + .fourcc = V4L2_PIX_FMT_NV61M, + .types = DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_SP, + .reg_swap_uv = true, + }, { + .fourcc = V4L2_PIX_FMT_YUYV, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED, + .reg_swap_uv = false, + }, { + .fourcc = V4L2_PIX_FMT_UYVY, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED, + .reg_swap_uv = true, + }, { + .fourcc = V4L2_PIX_FMT_NV12, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV420_SP, + .reg_swap_uv = false, + }, { + .fourcc = V4L2_PIX_FMT_NV12M, + .types = DW100_FMT_OUTPUT | DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV420_SP, + .reg_swap_uv = false, + }, { + .fourcc = V4L2_PIX_FMT_NV21, + .types = DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV420_SP, + .reg_swap_uv = true, + }, { + .fourcc = V4L2_PIX_FMT_NV21M, + .types = DW100_FMT_CAPTURE, + .reg_format = DW100_DEWARP_CTRL_FORMAT_YUV420_SP, + .reg_swap_uv = true, + }, +}; + +static inline int to_dw100_fmt_type(enum v4l2_buf_type type) +{ + if (V4L2_TYPE_IS_OUTPUT(type)) + return DW100_FMT_OUTPUT; + else + return DW100_FMT_CAPTURE; +} + +static const struct dw100_fmt *dw100_find_pixel_format(u32 pixel_format, + int fmt_type) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(formats); i++) { + const struct dw100_fmt *fmt = &formats[i]; + + if (fmt->fourcc == pixel_format && fmt->types & fmt_type) + return fmt; + } + + return NULL; +} + +static const struct dw100_fmt *dw100_find_format(struct v4l2_format *f) +{ + return dw100_find_pixel_format(f->fmt.pix_mp.pixelformat, + to_dw100_fmt_type(f->type)); +} + +static inline u32 dw100_read(struct dw100_device *dw_dev, u32 reg) +{ + return readl(dw_dev->mmio + reg); +} + +static inline void dw100_write(struct dw100_device *dw_dev, u32 reg, u32 val) +{ + writel(val, dw_dev->mmio + reg); +} + +static inline int dw100_dump_regs(struct seq_file *m) +{ + struct dw100_device *dw_dev = m->private; +#define __DECLARE_REG(x) { #x, x } + unsigned int i; + static const struct reg_desc { + const char * const name; + unsigned int addr; + } dw100_regs[] = { + __DECLARE_REG(DW100_DEWARP_ID), + __DECLARE_REG(DW100_DEWARP_CTRL), + __DECLARE_REG(DW100_MAP_LUT_ADDR), + __DECLARE_REG(DW100_MAP_LUT_SIZE), + __DECLARE_REG(DW100_MAP_LUT_ADDR2), + __DECLARE_REG(DW100_MAP_LUT_SIZE2), + __DECLARE_REG(DW100_SRC_IMG_Y_BASE), + __DECLARE_REG(DW100_SRC_IMG_UV_BASE), + __DECLARE_REG(DW100_SRC_IMG_SIZE), + __DECLARE_REG(DW100_SRC_IMG_STRIDE), + __DECLARE_REG(DW100_DST_IMG_Y_BASE), + __DECLARE_REG(DW100_DST_IMG_UV_BASE), + __DECLARE_REG(DW100_DST_IMG_SIZE), + __DECLARE_REG(DW100_DST_IMG_STRIDE), + __DECLARE_REG(DW100_DST_IMG_Y_SIZE1), + __DECLARE_REG(DW100_DST_IMG_UV_SIZE1), + __DECLARE_REG(DW100_SRC_IMG_Y_BASE2), + __DECLARE_REG(DW100_SRC_IMG_UV_BASE2), + __DECLARE_REG(DW100_SRC_IMG_SIZE2), + __DECLARE_REG(DW100_SRC_IMG_STRIDE2), + __DECLARE_REG(DW100_DST_IMG_Y_BASE2), + __DECLARE_REG(DW100_DST_IMG_UV_BASE2), + __DECLARE_REG(DW100_DST_IMG_SIZE2), + __DECLARE_REG(DW100_DST_IMG_STRIDE2), + __DECLARE_REG(DW100_DST_IMG_Y_SIZE2), + __DECLARE_REG(DW100_DST_IMG_UV_SIZE2), + __DECLARE_REG(DW100_SWAP_CONTROL), + __DECLARE_REG(DW100_VERTICAL_SPLIT_LINE), + __DECLARE_REG(DW100_HORIZON_SPLIT_LINE), + __DECLARE_REG(DW100_SCALE_FACTOR), + __DECLARE_REG(DW100_ROI_START), + __DECLARE_REG(DW100_BOUNDARY_PIXEL), + __DECLARE_REG(DW100_INTERRUPT_STATUS), + __DECLARE_REG(DW100_BUS_CTRL), + __DECLARE_REG(DW100_BUS_CTRL1), + __DECLARE_REG(DW100_BUS_TIME_OUT_CYCLE), + }; + + for (i = 0; i < ARRAY_SIZE(dw100_regs); i++) + seq_printf(m, "%s: %#x\n", dw100_regs[i].name, + dw100_read(dw_dev, dw100_regs[i].addr)); + + return 0; +} + +static inline struct dw100_ctx *dw100_file2ctx(struct file *file) +{ + return container_of(file->private_data, struct dw100_ctx, fh); +} + +static struct dw100_q_data *dw100_get_q_data(struct dw100_ctx *ctx, + enum v4l2_buf_type type) +{ + if (type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return &ctx->q_data[DW100_QUEUE_SRC]; + else + return &ctx->q_data[DW100_QUEUE_DST]; +} + +static u32 dw100_get_n_vertices_from_length(u32 length) +{ + return DIV_ROUND_UP(length, DW100_BLOCK_SIZE) + 1; +} + +static u16 dw100_map_convert_to_uq12_4(u32 a) +{ + return (u16)((a & 0xfff) << 4); +} + +static u32 dw100_map_format_coordinates(u16 xq, u16 yq) +{ + return (u32)((yq << 16) | xq); +} + +static u32 *dw100_get_user_map(struct dw100_ctx *ctx) +{ + struct v4l2_ctrl *ctrl = ctx->ctrls[DW100_CTRL_DEWARPING_MAP]; + + return ctrl->p_cur.p_u32; +} + +/* + * Create the dewarp map used by the hardware from the V4L2 control values which + * have been initialized with an identity map or set by the application. + */ +static int dw100_create_mapping(struct dw100_ctx *ctx) +{ + u32 *user_map; + + if (ctx->map) + dma_free_coherent(&ctx->dw_dev->pdev->dev, ctx->map_size, + ctx->map, ctx->map_dma); + + ctx->map = dma_alloc_coherent(&ctx->dw_dev->pdev->dev, ctx->map_size, + &ctx->map_dma, GFP_KERNEL); + + if (!ctx->map) + return -ENOMEM; + + user_map = dw100_get_user_map(ctx); + memcpy(ctx->map, user_map, ctx->map_size); + + dev_dbg(&ctx->dw_dev->pdev->dev, + "%ux%u %s mapping created (d:%pad-c:%p) for stream %ux%u->%ux%u\n", + ctx->map_width, ctx->map_height, + ctx->user_map_is_set ? "user" : "identity", + &ctx->map_dma, ctx->map, + ctx->q_data[DW100_QUEUE_SRC].pix_fmt.width, + ctx->q_data[DW100_QUEUE_DST].pix_fmt.height, + ctx->q_data[DW100_QUEUE_SRC].pix_fmt.width, + ctx->q_data[DW100_QUEUE_DST].pix_fmt.height); + + return 0; +} + +static void dw100_destroy_mapping(struct dw100_ctx *ctx) +{ + if (ctx->map) { + dma_free_coherent(&ctx->dw_dev->pdev->dev, ctx->map_size, + ctx->map, ctx->map_dma); + ctx->map = NULL; + } +} + +static int dw100_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct dw100_ctx *ctx = + container_of(ctrl->handler, struct dw100_ctx, hdl); + + switch (ctrl->id) { + case V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP: + ctx->user_map_is_set = true; + break; + } + + return 0; +} + +static const struct v4l2_ctrl_ops dw100_ctrl_ops = { + .s_ctrl = dw100_s_ctrl, +}; + +/* + * Initialize the dewarping map with an identity mapping. + * + * A 16 pixels cell size grid is mapped on the destination image. + * The last cells width/height might be lesser than 16 if the destination image + * width/height is not divisible by 16. This dewarping grid map specifies the + * source image pixel location (x, y) on each grid intersection point. + * Bilinear interpolation is used to compute inner cell points locations. + * + * The coordinates are saved in UQ12.4 fixed point format. + */ +static void dw100_ctrl_dewarping_map_init(const struct v4l2_ctrl *ctrl, + u32 from_idx, u32 elems, + union v4l2_ctrl_ptr ptr) +{ + struct dw100_ctx *ctx = + container_of(ctrl->handler, struct dw100_ctx, hdl); + + u32 sw, sh, mw, mh, idx; + u16 qx, qy, qdx, qdy, qsh, qsw; + u32 *map = ctrl->p_cur.p_u32; + + sw = ctx->q_data[DW100_QUEUE_SRC].pix_fmt.width; + sh = ctx->q_data[DW100_QUEUE_SRC].pix_fmt.height; + + mw = ctrl->dims[0]; + mh = ctrl->dims[1]; + + qsw = dw100_map_convert_to_uq12_4(sw); + qsh = dw100_map_convert_to_uq12_4(sh); + qdx = qsw / (mw - 1); + qdy = qsh / (mh - 1); + + ctx->map_width = mw; + ctx->map_height = mh; + ctx->map_size = mh * mw * sizeof(u32); + + for (idx = from_idx; idx < elems; idx++) { + qy = min_t(u32, (idx / mw) * qdy, qsh); + qx = min_t(u32, (idx % mw) * qdx, qsw); + map[idx] = dw100_map_format_coordinates(qx, qy); + } + + ctx->user_map_is_set = false; +} + +static const struct v4l2_ctrl_type_ops dw100_ctrl_type_ops = { + .init = dw100_ctrl_dewarping_map_init, + .validate = v4l2_ctrl_type_op_validate, + .log = v4l2_ctrl_type_op_log, + .equal = v4l2_ctrl_type_op_equal, +}; + +static const struct v4l2_ctrl_config controls[] = { + [DW100_CTRL_DEWARPING_MAP] = { + .ops = &dw100_ctrl_ops, + .type_ops = &dw100_ctrl_type_ops, + .id = V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP, + .name = "Dewarping Vertex Map", + .type = V4L2_CTRL_TYPE_U32, + .min = 0x00000000, + .max = 0xffffffff, + .step = 1, + .def = 0, + .dims = { DW100_DEF_LUT_W, DW100_DEF_LUT_H }, + }, +}; + +static int dw100_queue_setup(struct vb2_queue *vq, + unsigned int *nbuffers, unsigned int *nplanes, + unsigned int sizes[], struct device *alloc_devs[]) +{ + struct dw100_ctx *ctx = vb2_get_drv_priv(vq); + const struct v4l2_pix_format_mplane *format; + unsigned int i; + + format = &dw100_get_q_data(ctx, vq->type)->pix_fmt; + + if (*nplanes) { + if (*nplanes != format->num_planes) + return -EINVAL; + + for (i = 0; i < *nplanes; ++i) { + if (sizes[i] < format->plane_fmt[i].sizeimage) + return -EINVAL; + } + + return 0; + } + + *nplanes = format->num_planes; + + for (i = 0; i < format->num_planes; ++i) + sizes[i] = format->plane_fmt[i].sizeimage; + + return 0; +} + +static int dw100_buf_prepare(struct vb2_buffer *vb) +{ + unsigned int i; + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct dw100_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + struct dw100_device *dw_dev = ctx->dw_dev; + const struct v4l2_pix_format_mplane *pix_fmt = + &dw100_get_q_data(ctx, vb->vb2_queue->type)->pix_fmt; + + if (V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type)) { + if (vbuf->field != V4L2_FIELD_NONE) { + dev_dbg(&dw_dev->pdev->dev, "%x field isn't supported\n", + vbuf->field); + return -EINVAL; + } + } + + for (i = 0; i < pix_fmt->num_planes; i++) { + unsigned long size = pix_fmt->plane_fmt[i].sizeimage; + + if (vb2_plane_size(vb, i) < size) { + dev_dbg(&dw_dev->pdev->dev, + "User buffer too small (%lu < %lu)\n", + vb2_plane_size(vb, i), size); + return -EINVAL; + } + + vb2_set_plane_payload(vb, i, size); + } + + return 0; +} + +static void dw100_buf_queue(struct vb2_buffer *vb) +{ + struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb); + struct dw100_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue); + + v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf); +} + +static void dw100_return_all_buffers(struct vb2_queue *q, + enum vb2_buffer_state state) +{ + struct dw100_ctx *ctx = vb2_get_drv_priv(q); + struct vb2_v4l2_buffer *vbuf; + + for (;;) { + if (V4L2_TYPE_IS_OUTPUT(q->type)) + vbuf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx); + else + vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx); + if (!vbuf) + return; + v4l2_m2m_buf_done(vbuf, state); + } +} + +static int dw100_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct dw100_ctx *ctx = vb2_get_drv_priv(q); + struct dw100_q_data *q_data = dw100_get_q_data(ctx, q->type); + int ret; + + q_data->sequence = 0; + + ret = dw100_create_mapping(ctx); + if (ret) + goto err; + + ret = pm_runtime_resume_and_get(&ctx->dw_dev->pdev->dev); + if (ret) { + dw100_destroy_mapping(ctx); + goto err; + } + + return 0; +err: + dw100_return_all_buffers(q, VB2_BUF_STATE_QUEUED); + return ret; +} + +static void dw100_stop_streaming(struct vb2_queue *q) +{ + struct dw100_ctx *ctx = vb2_get_drv_priv(q); + + dw100_return_all_buffers(q, VB2_BUF_STATE_ERROR); + + pm_runtime_put_sync(&ctx->dw_dev->pdev->dev); + + dw100_destroy_mapping(ctx); +} + +static const struct vb2_ops dw100_qops = { + .queue_setup = dw100_queue_setup, + .buf_prepare = dw100_buf_prepare, + .buf_queue = dw100_buf_queue, + .start_streaming = dw100_start_streaming, + .stop_streaming = dw100_stop_streaming, + .wait_prepare = vb2_ops_wait_prepare, + .wait_finish = vb2_ops_wait_finish, +}; + +static int dw100_m2m_queue_init(void *priv, struct vb2_queue *src_vq, + struct vb2_queue *dst_vq) +{ + struct dw100_ctx *ctx = priv; + int ret; + + src_vq->type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; + src_vq->io_modes = VB2_MMAP | VB2_DMABUF; + src_vq->drv_priv = ctx; + src_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + src_vq->ops = &dw100_qops; + src_vq->mem_ops = &vb2_dma_contig_memops; + src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + src_vq->lock = &ctx->vq_mutex; + src_vq->dev = ctx->dw_dev->v4l2_dev.dev; + + ret = vb2_queue_init(src_vq); + if (ret) + return ret; + + dst_vq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + dst_vq->io_modes = VB2_MMAP | VB2_DMABUF; + dst_vq->drv_priv = ctx; + dst_vq->buf_struct_size = sizeof(struct v4l2_m2m_buffer); + dst_vq->ops = &dw100_qops; + dst_vq->mem_ops = &vb2_dma_contig_memops; + dst_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY; + dst_vq->lock = &ctx->vq_mutex; + dst_vq->dev = ctx->dw_dev->v4l2_dev.dev; + + return vb2_queue_init(dst_vq); +} + +static int dw100_open(struct file *file) +{ + struct dw100_device *dw_dev = video_drvdata(file); + struct dw100_ctx *ctx; + struct v4l2_ctrl_handler *hdl; + struct v4l2_pix_format_mplane *pix_fmt; + int ret, i; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + mutex_init(&ctx->vq_mutex); + v4l2_fh_init(&ctx->fh, video_devdata(file)); + file->private_data = &ctx->fh; + ctx->dw_dev = dw_dev; + + ctx->q_data[DW100_QUEUE_SRC].fmt = &formats[0]; + + pix_fmt = &ctx->q_data[DW100_QUEUE_SRC].pix_fmt; + pix_fmt->field = V4L2_FIELD_NONE; + pix_fmt->colorspace = V4L2_COLORSPACE_REC709; + pix_fmt->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix_fmt->colorspace); + pix_fmt->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix_fmt->colorspace); + pix_fmt->quantization = + V4L2_MAP_QUANTIZATION_DEFAULT(false, pix_fmt->colorspace, + pix_fmt->ycbcr_enc); + + v4l2_fill_pixfmt_mp(pix_fmt, formats[0].fourcc, DW100_DEF_W, DW100_DEF_H); + + ctx->q_data[DW100_QUEUE_SRC].crop.top = 0; + ctx->q_data[DW100_QUEUE_SRC].crop.left = 0; + ctx->q_data[DW100_QUEUE_SRC].crop.width = DW100_DEF_W; + ctx->q_data[DW100_QUEUE_SRC].crop.height = DW100_DEF_H; + + ctx->q_data[DW100_QUEUE_DST] = ctx->q_data[DW100_QUEUE_SRC]; + + hdl = &ctx->hdl; + v4l2_ctrl_handler_init(hdl, ARRAY_SIZE(controls)); + for (i = 0; i < ARRAY_SIZE(controls); i++) { + ctx->ctrls[i] = v4l2_ctrl_new_custom(hdl, &controls[i], NULL); + if (hdl->error) { + dev_err(&ctx->dw_dev->pdev->dev, + "Adding control (%d) failed\n", i); + ret = hdl->error; + goto err; + } + } + ctx->fh.ctrl_handler = hdl; + + ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dw_dev->m2m_dev, + ctx, &dw100_m2m_queue_init); + + if (IS_ERR(ctx->fh.m2m_ctx)) { + ret = PTR_ERR(ctx->fh.m2m_ctx); + goto err; + } + + v4l2_fh_add(&ctx->fh); + + return 0; + +err: + v4l2_ctrl_handler_free(hdl); + v4l2_fh_exit(&ctx->fh); + mutex_destroy(&ctx->vq_mutex); + kfree(ctx); + + return ret; +} + +static int dw100_release(struct file *file) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + + v4l2_fh_del(&ctx->fh); + v4l2_fh_exit(&ctx->fh); + v4l2_ctrl_handler_free(&ctx->hdl); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + mutex_destroy(&ctx->vq_mutex); + kfree(ctx); + + return 0; +} + +static const struct v4l2_file_operations dw100_fops = { + .owner = THIS_MODULE, + .open = dw100_open, + .release = dw100_release, + .poll = v4l2_m2m_fop_poll, + .unlocked_ioctl = video_ioctl2, + .mmap = v4l2_m2m_fop_mmap, +}; + +static int dw100_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + strscpy(cap->driver, DRV_NAME, sizeof(cap->driver)); + strscpy(cap->card, "DW100 dewarper", sizeof(cap->card)); + + return 0; +} + +static int dw100_enum_fmt_vid(struct file *file, void *priv, + struct v4l2_fmtdesc *f) +{ + int i, num = 0; + + for (i = 0; i < ARRAY_SIZE(formats); i++) { + if (formats[i].types & to_dw100_fmt_type(f->type)) { + if (num == f->index) { + f->pixelformat = formats[i].fourcc; + return 0; + } + ++num; + } + } + + return -EINVAL; +} + +static int dw100_enum_framesizes(struct file *file, void *priv, + struct v4l2_frmsizeenum *fsize) +{ + const struct dw100_fmt *fmt; + + if (fsize->index) + return -EINVAL; + + fmt = dw100_find_pixel_format(fsize->pixel_format, + DW100_FMT_OUTPUT | DW100_FMT_CAPTURE); + if (!fmt) + return -EINVAL; + + fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE; + fsize->stepwise = dw100_frmsize_stepwise; + + return 0; +} + +static int dw100_g_fmt_vid(struct file *file, void *priv, struct v4l2_format *f) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + struct vb2_queue *vq; + struct dw100_q_data *q_data; + + vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + if (!vq) + return -EINVAL; + + q_data = dw100_get_q_data(ctx, f->type); + + f->fmt.pix_mp = q_data->pix_fmt; + + return 0; +} + +static int dw100_try_fmt(struct file *file, struct v4l2_format *f) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + struct v4l2_pix_format_mplane *pix = &f->fmt.pix_mp; + const struct dw100_fmt *fmt; + + fmt = dw100_find_format(f); + if (!fmt) { + fmt = &formats[0]; + pix->pixelformat = fmt->fourcc; + } + + v4l2_apply_frmsize_constraints(&pix->width, &pix->height, + &dw100_frmsize_stepwise); + + v4l2_fill_pixfmt_mp(pix, fmt->fourcc, pix->width, pix->height); + + pix->field = V4L2_FIELD_NONE; + + if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { + if (pix->colorspace == V4L2_COLORSPACE_DEFAULT) + pix->colorspace = V4L2_COLORSPACE_REC709; + if (pix->xfer_func == V4L2_XFER_FUNC_DEFAULT) + pix->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(pix->colorspace); + if (pix->ycbcr_enc == V4L2_YCBCR_ENC_DEFAULT) + pix->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(pix->colorspace); + if (pix->quantization == V4L2_QUANTIZATION_DEFAULT) + pix->quantization = + V4L2_MAP_QUANTIZATION_DEFAULT(false, + pix->colorspace, + pix->ycbcr_enc); + } else { + /* + * The DW100 can't perform colorspace conversion, the colorspace + * on the capture queue must be identical to the output queue. + */ + const struct dw100_q_data *q_data = + dw100_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + + pix->colorspace = q_data->pix_fmt.colorspace; + pix->xfer_func = q_data->pix_fmt.xfer_func; + pix->ycbcr_enc = q_data->pix_fmt.ycbcr_enc; + pix->quantization = q_data->pix_fmt.quantization; + } + + return 0; +} + +static int dw100_s_fmt(struct dw100_ctx *ctx, struct v4l2_format *f) +{ + struct dw100_q_data *q_data; + struct vb2_queue *vq; + + vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type); + if (!vq) + return -EINVAL; + + q_data = dw100_get_q_data(ctx, f->type); + if (!q_data) + return -EINVAL; + + if (vb2_is_busy(vq)) { + dev_dbg(&ctx->dw_dev->pdev->dev, "%s queue busy\n", __func__); + return -EBUSY; + } + + q_data->fmt = dw100_find_format(f); + q_data->pix_fmt = f->fmt.pix_mp; + q_data->crop.top = 0; + q_data->crop.left = 0; + q_data->crop.width = f->fmt.pix_mp.width; + q_data->crop.height = f->fmt.pix_mp.height; + + /* Propagate buffers encoding */ + + if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { + struct dw100_q_data *dst_q_data = + dw100_get_q_data(ctx, + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE); + + dst_q_data->pix_fmt.colorspace = q_data->pix_fmt.colorspace; + dst_q_data->pix_fmt.ycbcr_enc = q_data->pix_fmt.ycbcr_enc; + dst_q_data->pix_fmt.quantization = q_data->pix_fmt.quantization; + dst_q_data->pix_fmt.xfer_func = q_data->pix_fmt.xfer_func; + } + + dev_dbg(&ctx->dw_dev->pdev->dev, + "Setting format for type %u, wxh: %ux%u, fmt: %p4cc\n", + f->type, q_data->pix_fmt.width, q_data->pix_fmt.height, + &q_data->pix_fmt.pixelformat); + + if (f->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { + int ret; + u32 dims[V4L2_CTRL_MAX_DIMS] = {}; + struct v4l2_ctrl *ctrl = ctx->ctrls[DW100_CTRL_DEWARPING_MAP]; + + dims[0] = dw100_get_n_vertices_from_length(q_data->pix_fmt.width); + dims[1] = dw100_get_n_vertices_from_length(q_data->pix_fmt.height); + + ret = v4l2_ctrl_modify_dimensions(ctrl, dims); + + if (ret) { + dev_err(&ctx->dw_dev->pdev->dev, + "Modifying LUT dimensions failed with error %d\n", + ret); + return ret; + } + } + + return 0; +} + +static int dw100_try_fmt_vid_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) + return -EINVAL; + + return dw100_try_fmt(file, f); +} + +static int dw100_s_fmt_vid_cap(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + int ret; + + ret = dw100_try_fmt_vid_cap(file, priv, f); + if (ret) + return ret; + + ret = dw100_s_fmt(ctx, f); + if (ret) + return ret; + + return 0; +} + +static int dw100_try_fmt_vid_out(struct file *file, void *priv, + struct v4l2_format *f) +{ + if (f->type != V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) + return -EINVAL; + + return dw100_try_fmt(file, f); +} + +static int dw100_s_fmt_vid_out(struct file *file, void *priv, + struct v4l2_format *f) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + int ret; + + ret = dw100_try_fmt_vid_out(file, priv, f); + if (ret) + return ret; + + ret = dw100_s_fmt(ctx, f); + if (ret) + return ret; + + return 0; +} + +static int dw100_g_selection(struct file *file, void *fh, + struct v4l2_selection *sel) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + struct dw100_q_data *src_q_data; + + if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + + src_q_data = dw100_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + + switch (sel->target) { + case V4L2_SEL_TGT_CROP_DEFAULT: + case V4L2_SEL_TGT_CROP_BOUNDS: + sel->r.top = 0; + sel->r.left = 0; + sel->r.width = src_q_data->pix_fmt.width; + sel->r.height = src_q_data->pix_fmt.height; + break; + case V4L2_SEL_TGT_CROP: + sel->r.top = src_q_data->crop.top; + sel->r.left = src_q_data->crop.left; + sel->r.width = src_q_data->crop.width; + sel->r.height = src_q_data->crop.height; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int dw100_s_selection(struct file *file, void *fh, + struct v4l2_selection *sel) +{ + struct dw100_ctx *ctx = dw100_file2ctx(file); + struct dw100_q_data *src_q_data; + u32 qscalex, qscaley, qscale; + int x, y, w, h; + unsigned int wframe, hframe; + + if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT) + return -EINVAL; + + src_q_data = dw100_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE); + + dev_dbg(&ctx->dw_dev->pdev->dev, + ">>> Buffer Type: %u Target: %u Rect: %ux%u@%d.%d\n", + sel->type, sel->target, + sel->r.width, sel->r.height, sel->r.left, sel->r.top); + + switch (sel->target) { + case V4L2_SEL_TGT_CROP: + wframe = src_q_data->pix_fmt.width; + hframe = src_q_data->pix_fmt.height; + + sel->r.top = clamp_t(int, sel->r.top, 0, hframe - DW100_MIN_H); + sel->r.left = clamp_t(int, sel->r.left, 0, wframe - DW100_MIN_W); + sel->r.height = + clamp(sel->r.height, DW100_MIN_H, hframe - sel->r.top); + sel->r.width = + clamp(sel->r.width, DW100_MIN_W, wframe - sel->r.left); + + /* UQ16.16 for float operations */ + qscalex = (sel->r.width << 16) / wframe; + qscaley = (sel->r.height << 16) / hframe; + y = sel->r.top; + x = sel->r.left; + if (qscalex == qscaley) { + qscale = qscalex; + } else { + switch (sel->flags) { + case 0: + qscale = (qscalex + qscaley) / 2; + break; + case V4L2_SEL_FLAG_GE: + qscale = max(qscaley, qscalex); + break; + case V4L2_SEL_FLAG_LE: + qscale = min(qscaley, qscalex); + break; + case V4L2_SEL_FLAG_LE | V4L2_SEL_FLAG_GE: + return -ERANGE; + default: + return -EINVAL; + } + } + + w = (u32)((((u64)wframe << 16) * qscale) >> 32); + h = (u32)((((u64)hframe << 16) * qscale) >> 32); + x = x + (sel->r.width - w) / 2; + y = y + (sel->r.height - h) / 2; + x = min(wframe - w, (unsigned int)max(0, x)); + y = min(hframe - h, (unsigned int)max(0, y)); + + sel->r.top = y; + sel->r.left = x; + sel->r.width = w; + sel->r.height = h; + + src_q_data->crop.top = sel->r.top; + src_q_data->crop.left = sel->r.left; + src_q_data->crop.width = sel->r.width; + src_q_data->crop.height = sel->r.height; + break; + + default: + return -EINVAL; + } + + dev_dbg(&ctx->dw_dev->pdev->dev, + "<<< Buffer Type: %u Target: %u Rect: %ux%u@%d.%d\n", + sel->type, sel->target, + sel->r.width, sel->r.height, sel->r.left, sel->r.top); + + return 0; +} + +static const struct v4l2_ioctl_ops dw100_ioctl_ops = { + .vidioc_querycap = dw100_querycap, + + .vidioc_enum_fmt_vid_cap = dw100_enum_fmt_vid, + .vidioc_enum_framesizes = dw100_enum_framesizes, + .vidioc_g_fmt_vid_cap_mplane = dw100_g_fmt_vid, + .vidioc_try_fmt_vid_cap_mplane = dw100_try_fmt_vid_cap, + .vidioc_s_fmt_vid_cap_mplane = dw100_s_fmt_vid_cap, + + .vidioc_enum_fmt_vid_out = dw100_enum_fmt_vid, + .vidioc_g_fmt_vid_out_mplane = dw100_g_fmt_vid, + .vidioc_try_fmt_vid_out_mplane = dw100_try_fmt_vid_out, + .vidioc_s_fmt_vid_out_mplane = dw100_s_fmt_vid_out, + + .vidioc_g_selection = dw100_g_selection, + .vidioc_s_selection = dw100_s_selection, + .vidioc_reqbufs = v4l2_m2m_ioctl_reqbufs, + .vidioc_querybuf = v4l2_m2m_ioctl_querybuf, + .vidioc_qbuf = v4l2_m2m_ioctl_qbuf, + .vidioc_dqbuf = v4l2_m2m_ioctl_dqbuf, + .vidioc_prepare_buf = v4l2_m2m_ioctl_prepare_buf, + .vidioc_create_bufs = v4l2_m2m_ioctl_create_bufs, + .vidioc_expbuf = v4l2_m2m_ioctl_expbuf, + + .vidioc_streamon = v4l2_m2m_ioctl_streamon, + .vidioc_streamoff = v4l2_m2m_ioctl_streamoff, + + .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, + .vidioc_unsubscribe_event = v4l2_event_unsubscribe, +}; + +static void dw100_job_finish(struct dw100_device *dw_dev, bool with_error) +{ + struct dw100_ctx *curr_ctx; + struct vb2_v4l2_buffer *src_vb, *dst_vb; + enum vb2_buffer_state buf_state; + + curr_ctx = v4l2_m2m_get_curr_priv(dw_dev->m2m_dev); + + if (!curr_ctx) { + dev_err(&dw_dev->pdev->dev, + "Instance released before the end of transaction\n"); + return; + } + + src_vb = v4l2_m2m_src_buf_remove(curr_ctx->fh.m2m_ctx); + dst_vb = v4l2_m2m_dst_buf_remove(curr_ctx->fh.m2m_ctx); + + if (likely(!with_error)) + buf_state = VB2_BUF_STATE_DONE; + else + buf_state = VB2_BUF_STATE_ERROR; + + v4l2_m2m_buf_done(src_vb, buf_state); + v4l2_m2m_buf_done(dst_vb, buf_state); + + dev_dbg(&dw_dev->pdev->dev, "Finishing transaction with%s error(s)\n", + with_error ? "" : "out"); + + v4l2_m2m_job_finish(dw_dev->m2m_dev, curr_ctx->fh.m2m_ctx); +} + +static void dw100_hw_reset(struct dw100_device *dw_dev) +{ + u32 val; + + val = dw100_read(dw_dev, DW100_DEWARP_CTRL); + val |= DW100_DEWARP_CTRL_ENABLE; + val |= DW100_DEWARP_CTRL_SOFT_RESET; + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); + val &= ~DW100_DEWARP_CTRL_SOFT_RESET; + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); +} + +static void _dw100_hw_set_master_bus_enable(struct dw100_device *dw_dev, + unsigned int enable) +{ + u32 val; + + dev_dbg(&dw_dev->pdev->dev, "%sable master bus\n", + enable ? "En" : "Dis"); + + val = dw100_read(dw_dev, DW100_BUS_CTRL); + + if (enable) + val |= DW100_BUS_CTRL_AXI_MASTER_ENABLE; + else + val &= ~DW100_BUS_CTRL_AXI_MASTER_ENABLE; + + dw100_write(dw_dev, DW100_BUS_CTRL, val); +} + +static void dw100_hw_master_bus_enable(struct dw100_device *dw_dev) +{ + _dw100_hw_set_master_bus_enable(dw_dev, 1); +} + +static void dw100_hw_master_bus_disable(struct dw100_device *dw_dev) +{ + _dw100_hw_set_master_bus_enable(dw_dev, 0); +} + +static void dw100_hw_dewarp_start(struct dw100_device *dw_dev) +{ + u32 val; + + val = dw100_read(dw_dev, DW100_DEWARP_CTRL); + + dev_dbg(&dw_dev->pdev->dev, "Starting Hardware CTRL:0x%08x\n", val); + dw100_write(dw_dev, DW100_DEWARP_CTRL, val | DW100_DEWARP_CTRL_START); + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); +} + +static void dw100_hw_init_ctrl(struct dw100_device *dw_dev) +{ + u32 val; + /* + * Input format YUV422_SP + * Output format YUV422_SP + * No hardware handshake (SW) + * No automatic double src buffering (Single) + * No automatic double dst buffering (Single) + * No Black Line + * Prefetch image pixel traversal + */ + + val = DW100_DEWARP_CTRL_ENABLE + /* Valid only for auto prefetch mode*/ + | DW100_DEWARP_CTRL_PREFETCH_THRESHOLD(32); + + /* + * Calculation mode required to support any scaling factor, + * but x4 slower than traversal mode. + * + * DW100_DEWARP_CTRL_PREFETCH_MODE_TRAVERSAL + * DW100_DEWARP_CTRL_PREFETCH_MODE_CALCULATION + * DW100_DEWARP_CTRL_PREFETCH_MODE_AUTO + * + * TODO: Find heuristics requiring calculation mode + */ + val |= DW100_DEWARP_CTRL_PREFETCH_MODE_CALCULATION; + + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); +} + +static void dw100_hw_set_pixel_boundary(struct dw100_device *dw_dev) +{ + u32 val; + + val = DW100_BOUNDARY_PIXEL_V(128) + | DW100_BOUNDARY_PIXEL_U(128) + | DW100_BOUNDARY_PIXEL_Y(0); + + dw100_write(dw_dev, DW100_BOUNDARY_PIXEL, val); +} + +static void dw100_hw_set_scale(struct dw100_device *dw_dev, u8 scale) +{ + dev_dbg(&dw_dev->pdev->dev, "Setting scale factor to %u\n", scale); + + dw100_write(dw_dev, DW100_SCALE_FACTOR, scale); +} + +static void dw100_hw_set_roi(struct dw100_device *dw_dev, u32 x, u32 y) +{ + u32 val; + + dev_dbg(&dw_dev->pdev->dev, "Setting ROI region to %u.%u\n", x, y); + + val = DW100_ROI_START_X(x) | DW100_ROI_START_Y(y); + + dw100_write(dw_dev, DW100_ROI_START, val); +} + +static void dw100_hw_set_src_crop(struct dw100_device *dw_dev, + const struct dw100_q_data *src_q_data, + const struct dw100_q_data *dst_q_data) +{ + const struct v4l2_rect *rect = &src_q_data->crop; + u32 src_scale, qscale, left_scale, top_scale; + + /* HW Scale is UQ1.7 encoded */ + src_scale = (rect->width << 7) / src_q_data->pix_fmt.width; + dw100_hw_set_scale(dw_dev, src_scale); + + qscale = (dst_q_data->pix_fmt.width << 7) / src_q_data->pix_fmt.width; + + left_scale = ((rect->left << 7) * qscale) >> 14; + top_scale = ((rect->top << 7) * qscale) >> 14; + + dw100_hw_set_roi(dw_dev, left_scale, top_scale); +} + +static void dw100_hw_set_source(struct dw100_device *dw_dev, + const struct dw100_q_data *q_data, + struct vb2_buffer *buffer) +{ + u32 width, height, stride, fourcc, val; + const struct dw100_fmt *fmt = q_data->fmt; + dma_addr_t addr_y = vb2_dma_contig_plane_dma_addr(buffer, 0); + dma_addr_t addr_uv; + + width = q_data->pix_fmt.width; + height = q_data->pix_fmt.height; + stride = q_data->pix_fmt.plane_fmt[0].bytesperline; + fourcc = q_data->fmt->fourcc; + + if (q_data->pix_fmt.num_planes == 2) + addr_uv = vb2_dma_contig_plane_dma_addr(buffer, 1); + else + addr_uv = addr_y + (stride * height); + + dev_dbg(&dw_dev->pdev->dev, + "Set HW source registers for %ux%u - stride %u, pixfmt: %p4cc, dma:%pad\n", + width, height, stride, &fourcc, &addr_y); + + /* Pixel Format */ + val = dw100_read(dw_dev, DW100_DEWARP_CTRL); + + val &= ~DW100_DEWARP_CTRL_INPUT_FORMAT_MASK; + val |= DW100_DEWARP_CTRL_INPUT_FORMAT(fmt->reg_format); + + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); + + /* Swap */ + val = dw100_read(dw_dev, DW100_SWAP_CONTROL); + + val &= ~DW100_SWAP_CONTROL_SRC_MASK; + /* + * Data swapping is performed only on Y plane for source image. + */ + if (fmt->reg_swap_uv && + fmt->reg_format == DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED) + val |= DW100_SWAP_CONTROL_SRC(DW100_SWAP_CONTROL_Y + (DW100_SWAP_CONTROL_BYTE)); + + dw100_write(dw_dev, DW100_SWAP_CONTROL, val); + + /* Image resolution */ + dw100_write(dw_dev, DW100_SRC_IMG_SIZE, + DW100_IMG_SIZE_WIDTH(width) | DW100_IMG_SIZE_HEIGHT(height)); + + dw100_write(dw_dev, DW100_SRC_IMG_STRIDE, stride); + + /* Buffers */ + dw100_write(dw_dev, DW100_SRC_IMG_Y_BASE, DW100_IMG_Y_BASE(addr_y)); + dw100_write(dw_dev, DW100_SRC_IMG_UV_BASE, DW100_IMG_UV_BASE(addr_uv)); +} + +static void dw100_hw_set_destination(struct dw100_device *dw_dev, + const struct dw100_q_data *q_data, + const struct dw100_fmt *ifmt, + struct vb2_buffer *buffer) +{ + u32 width, height, stride, fourcc, val, size_y, size_uv; + const struct dw100_fmt *fmt = q_data->fmt; + dma_addr_t addr_y, addr_uv; + + width = q_data->pix_fmt.width; + height = q_data->pix_fmt.height; + stride = q_data->pix_fmt.plane_fmt[0].bytesperline; + fourcc = fmt->fourcc; + + addr_y = vb2_dma_contig_plane_dma_addr(buffer, 0); + size_y = q_data->pix_fmt.plane_fmt[0].sizeimage; + + if (q_data->pix_fmt.num_planes == 2) { + addr_uv = vb2_dma_contig_plane_dma_addr(buffer, 1); + size_uv = q_data->pix_fmt.plane_fmt[1].sizeimage; + } else { + addr_uv = addr_y + ALIGN(stride * height, 16); + size_uv = size_y; + if (fmt->reg_format == DW100_DEWARP_CTRL_FORMAT_YUV420_SP) + size_uv /= 2; + } + + dev_dbg(&dw_dev->pdev->dev, + "Set HW source registers for %ux%u - stride %u, pixfmt: %p4cc, dma:%pad\n", + width, height, stride, &fourcc, &addr_y); + + /* Pixel Format */ + val = dw100_read(dw_dev, DW100_DEWARP_CTRL); + + val &= ~DW100_DEWARP_CTRL_OUTPUT_FORMAT_MASK; + val |= DW100_DEWARP_CTRL_OUTPUT_FORMAT(fmt->reg_format); + + dw100_write(dw_dev, DW100_DEWARP_CTRL, val); + + /* Swap */ + val = dw100_read(dw_dev, DW100_SWAP_CONTROL); + + val &= ~DW100_SWAP_CONTROL_DST_MASK; + + /* + * Avoid to swap twice + */ + if (fmt->reg_swap_uv ^ + (ifmt->reg_swap_uv && ifmt->reg_format != + DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED)) { + if (fmt->reg_format == DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED) + val |= DW100_SWAP_CONTROL_DST(DW100_SWAP_CONTROL_Y + (DW100_SWAP_CONTROL_BYTE)); + else + val |= DW100_SWAP_CONTROL_DST(DW100_SWAP_CONTROL_UV + (DW100_SWAP_CONTROL_BYTE)); + } + + dw100_write(dw_dev, DW100_SWAP_CONTROL, val); + + /* Image resolution */ + dw100_write(dw_dev, DW100_DST_IMG_SIZE, + DW100_IMG_SIZE_WIDTH(width) | DW100_IMG_SIZE_HEIGHT(height)); + dw100_write(dw_dev, DW100_DST_IMG_STRIDE, stride); + dw100_write(dw_dev, DW100_DST_IMG_Y_BASE, DW100_IMG_Y_BASE(addr_y)); + dw100_write(dw_dev, DW100_DST_IMG_UV_BASE, DW100_IMG_UV_BASE(addr_uv)); + dw100_write(dw_dev, DW100_DST_IMG_Y_SIZE1, DW100_DST_IMG_Y_SIZE(size_y)); + dw100_write(dw_dev, DW100_DST_IMG_UV_SIZE1, + DW100_DST_IMG_UV_SIZE(size_uv)); +} + +static void dw100_hw_set_mapping(struct dw100_device *dw_dev, dma_addr_t addr, + u32 width, u32 height) +{ + dev_dbg(&dw_dev->pdev->dev, + "Set HW mapping registers for %ux%u addr:%pad", + width, height, &addr); + + dw100_write(dw_dev, DW100_MAP_LUT_ADDR, DW100_MAP_LUT_ADDR_ADDR(addr)); + dw100_write(dw_dev, DW100_MAP_LUT_SIZE, DW100_MAP_LUT_SIZE_WIDTH(width) + | DW100_MAP_LUT_SIZE_HEIGHT(height)); +} + +static void dw100_hw_clear_irq(struct dw100_device *dw_dev, unsigned int irq) +{ + dw100_write(dw_dev, DW100_INTERRUPT_STATUS, + DW100_INTERRUPT_STATUS_INT_CLEAR(irq)); +} + +static void dw100_hw_enable_irq(struct dw100_device *dw_dev) +{ + dw100_write(dw_dev, DW100_INTERRUPT_STATUS, + DW100_INTERRUPT_STATUS_INT_ENABLE_MASK); +} + +static void dw100_hw_disable_irq(struct dw100_device *dw_dev) +{ + dw100_write(dw_dev, DW100_INTERRUPT_STATUS, 0); +} + +static u32 dw_hw_get_pending_irqs(struct dw100_device *dw_dev) +{ + u32 val; + + val = dw100_read(dw_dev, DW100_INTERRUPT_STATUS); + + return DW100_INTERRUPT_STATUS_INT_STATUS(val); +} + +static irqreturn_t dw100_irq_handler(int irq, void *dev_id) +{ + struct dw100_device *dw_dev = dev_id; + u32 pending_irqs, err_irqs, frame_done_irq; + bool with_error = true; + + pending_irqs = dw_hw_get_pending_irqs(dw_dev); + frame_done_irq = pending_irqs & DW100_INTERRUPT_STATUS_INT_FRAME_DONE; + err_irqs = DW100_INTERRUPT_STATUS_INT_ERR_STATUS(pending_irqs); + + if (frame_done_irq) { + dev_dbg(&dw_dev->pdev->dev, "Frame done interrupt\n"); + with_error = false; + err_irqs &= ~DW100_INTERRUPT_STATUS_INT_ERR_STATUS + (DW100_INTERRUPT_STATUS_INT_ERR_FRAME_DONE); + } + + if (err_irqs) + dev_err(&dw_dev->pdev->dev, "Interrupt error: %#x\n", err_irqs); + + dw100_hw_disable_irq(dw_dev); + dw100_hw_master_bus_disable(dw_dev); + dw100_hw_clear_irq(dw_dev, pending_irqs | + DW100_INTERRUPT_STATUS_INT_ERR_TIME_OUT); + + dw100_job_finish(dw_dev, with_error); + + return IRQ_HANDLED; +} + +static void dw100_start(struct dw100_ctx *ctx, struct vb2_v4l2_buffer *in_vb, + struct vb2_v4l2_buffer *out_vb) +{ + struct dw100_device *dw_dev = ctx->dw_dev; + + out_vb->sequence = + dw100_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)->sequence++; + in_vb->sequence = + dw100_get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)->sequence++; + + dev_dbg(&ctx->dw_dev->pdev->dev, + "Starting queues %p->%p, sequence %u->%u\n", + v4l2_m2m_get_vq(ctx->fh.m2m_ctx, + V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE), + v4l2_m2m_get_vq(ctx->fh.m2m_ctx, + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE), + in_vb->sequence, out_vb->sequence); + + v4l2_m2m_buf_copy_metadata(in_vb, out_vb, true); + + /* Now, let's deal with hardware ... */ + dw100_hw_master_bus_disable(dw_dev); + dw100_hw_init_ctrl(dw_dev); + dw100_hw_set_pixel_boundary(dw_dev); + dw100_hw_set_src_crop(dw_dev, &ctx->q_data[DW100_QUEUE_SRC], + &ctx->q_data[DW100_QUEUE_DST]); + dw100_hw_set_source(dw_dev, &ctx->q_data[DW100_QUEUE_SRC], + &in_vb->vb2_buf); + dw100_hw_set_destination(dw_dev, &ctx->q_data[DW100_QUEUE_DST], + ctx->q_data[DW100_QUEUE_SRC].fmt, + &out_vb->vb2_buf); + dw100_hw_set_mapping(dw_dev, ctx->map_dma, + ctx->map_width, ctx->map_height); + dw100_hw_enable_irq(dw_dev); + dw100_hw_dewarp_start(dw_dev); + + /* Enable Bus */ + dw100_hw_master_bus_enable(dw_dev); +} + +static void dw100_device_run(void *priv) +{ + struct dw100_ctx *ctx = priv; + struct vb2_v4l2_buffer *src_buf, *dst_buf; + + src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); + dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); + + dw100_start(ctx, src_buf, dst_buf); +} + +static const struct v4l2_m2m_ops dw100_m2m_ops = { + .device_run = dw100_device_run, +}; + +static struct video_device *dw100_init_video_device(struct dw100_device *dw_dev) +{ + struct video_device *vfd = &dw_dev->vfd; + + vfd->vfl_dir = VFL_DIR_M2M; + vfd->fops = &dw100_fops; + vfd->device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING; + vfd->ioctl_ops = &dw100_ioctl_ops; + vfd->minor = -1; + vfd->release = video_device_release_empty; + vfd->v4l2_dev = &dw_dev->v4l2_dev; + vfd->lock = &dw_dev->vfd_mutex; + + strscpy(vfd->name, DRV_NAME, sizeof(vfd->name)); + mutex_init(vfd->lock); + video_set_drvdata(vfd, dw_dev); + + return vfd; +} + +static int dw100_dump_regs_show(struct seq_file *m, void *private) +{ + struct dw100_device *dw_dev = m->private; + int ret; + + ret = pm_runtime_resume_and_get(&dw_dev->pdev->dev); + if (ret < 0) + return ret; + + ret = dw100_dump_regs(m); + + pm_runtime_put_sync(&dw_dev->pdev->dev); + + return ret; +} +DEFINE_SHOW_ATTRIBUTE(dw100_dump_regs); + +static void dw100_debugfs_init(struct dw100_device *dw_dev) +{ + dw_dev->debugfs_root = + debugfs_create_dir(dev_name(&dw_dev->pdev->dev), NULL); + + debugfs_create_file("dump_regs", 0600, dw_dev->debugfs_root, dw_dev, + &dw100_dump_regs_fops); +} + +static void dw100_debugfs_exit(struct dw100_device *dw_dev) +{ + debugfs_remove_recursive(dw_dev->debugfs_root); +} + +static int dw100_probe(struct platform_device *pdev) +{ + struct dw100_device *dw_dev; + struct video_device *vfd; + struct resource *res; + int ret, irq; + + dw_dev = devm_kzalloc(&pdev->dev, sizeof(*dw_dev), GFP_KERNEL); + if (!dw_dev) + return -ENOMEM; + dw_dev->pdev = pdev; + + ret = devm_clk_bulk_get_all(&pdev->dev, &dw_dev->clks); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to get clocks: %d\n", ret); + return ret; + } + dw_dev->num_clks = ret; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dw_dev->mmio = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(dw_dev->mmio)) + return PTR_ERR(dw_dev->mmio); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + platform_set_drvdata(pdev, dw_dev); + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to resume the device: %d\n", ret); + goto err_pm; + } + + pm_runtime_put_sync(&pdev->dev); + + ret = devm_request_irq(&pdev->dev, irq, dw100_irq_handler, IRQF_ONESHOT, + dev_name(&pdev->dev), dw_dev); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to request irq: %d\n", ret); + return ret; + } + + ret = v4l2_device_register(&pdev->dev, &dw_dev->v4l2_dev); + if (ret) + goto err_pm; + + vfd = dw100_init_video_device(dw_dev); + + dw_dev->m2m_dev = v4l2_m2m_init(&dw100_m2m_ops); + if (IS_ERR(dw_dev->m2m_dev)) { + dev_err(&pdev->dev, "Failed to init mem2mem device\n"); + ret = PTR_ERR(dw_dev->m2m_dev); + goto err_v4l2; + } + + dw_dev->mdev.dev = &pdev->dev; + strscpy(dw_dev->mdev.model, "dw100", sizeof(dw_dev->mdev.model)); + media_device_init(&dw_dev->mdev); + dw_dev->v4l2_dev.mdev = &dw_dev->mdev; + + ret = video_register_device(vfd, VFL_TYPE_VIDEO, -1); + if (ret) { + dev_err(&pdev->dev, "Failed to register video device\n"); + goto err_m2m; + } + + ret = v4l2_m2m_register_media_controller(dw_dev->m2m_dev, vfd, + MEDIA_ENT_F_PROC_VIDEO_SCALER); + if (ret) { + dev_err(&pdev->dev, "Failed to init mem2mem media controller\n"); + goto error_v4l2; + } + + ret = media_device_register(&dw_dev->mdev); + if (ret) { + dev_err(&pdev->dev, "Failed to register mem2mem media device\n"); + goto error_m2m_mc; + } + + dw100_debugfs_init(dw_dev); + + dev_info(&pdev->dev, + "dw100 v4l2 m2m registered as /dev/video%u\n", vfd->num); + + return 0; + +error_m2m_mc: + v4l2_m2m_unregister_media_controller(dw_dev->m2m_dev); +error_v4l2: + video_unregister_device(vfd); +err_m2m: + media_device_cleanup(&dw_dev->mdev); + v4l2_m2m_release(dw_dev->m2m_dev); +err_v4l2: + v4l2_device_unregister(&dw_dev->v4l2_dev); +err_pm: + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static int dw100_remove(struct platform_device *pdev) +{ + struct dw100_device *dw_dev = platform_get_drvdata(pdev); + + dw100_debugfs_exit(dw_dev); + + pm_runtime_disable(&pdev->dev); + + media_device_unregister(&dw_dev->mdev); + v4l2_m2m_unregister_media_controller(dw_dev->m2m_dev); + media_device_cleanup(&dw_dev->mdev); + + video_unregister_device(&dw_dev->vfd); + mutex_destroy(dw_dev->vfd.lock); + v4l2_m2m_release(dw_dev->m2m_dev); + v4l2_device_unregister(&dw_dev->v4l2_dev); + + return 0; +} + +static int __maybe_unused dw100_runtime_suspend(struct device *dev) +{ + struct dw100_device *dw_dev = dev_get_drvdata(dev); + + clk_bulk_disable_unprepare(dw_dev->num_clks, dw_dev->clks); + + return 0; +} + +static int __maybe_unused dw100_runtime_resume(struct device *dev) +{ + int ret; + struct dw100_device *dw_dev = dev_get_drvdata(dev); + + ret = clk_bulk_prepare_enable(dw_dev->num_clks, dw_dev->clks); + + if (ret) + return ret; + + dw100_hw_reset(dw_dev); + + return 0; +} + +static const struct dev_pm_ops dw100_pm = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(dw100_runtime_suspend, + dw100_runtime_resume, NULL) +}; + +static const struct of_device_id dw100_dt_ids[] = { + { .compatible = "nxp,imx8mp-dw100", .data = NULL }, + { }, +}; +MODULE_DEVICE_TABLE(of, dw100_dt_ids); + +static struct platform_driver dw100_driver = { + .probe = dw100_probe, + .remove = dw100_remove, + .driver = { + .name = DRV_NAME, + .pm = &dw100_pm, + .of_match_table = dw100_dt_ids, + }, +}; + +module_platform_driver(dw100_driver); + +MODULE_DESCRIPTION("DW100 Hardware dewarper"); +MODULE_AUTHOR("Xavier Roumegue "); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/nxp/dw100/dw100_regs.h b/drivers/media/platform/nxp/dw100/dw100_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..e85dfeff90568b735a0b54e441c3345e28c474dc --- /dev/null +++ b/drivers/media/platform/nxp/dw100/dw100_regs.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * DW100 Hardware dewarper + * + * Copyright 2022 NXP + * Author: Xavier Roumegue (xavier.roumegue@oss.nxp.com) + */ + +#ifndef _DW100_REGS_H_ +#define _DW100_REGS_H_ + +/* AHB register offset */ +#define DW100_DEWARP_ID 0x00 +#define DW100_DEWARP_CTRL 0x04 +#define DW100_DEWARP_CTRL_ENABLE BIT(0) +#define DW100_DEWARP_CTRL_START BIT(1) +#define DW100_DEWARP_CTRL_SOFT_RESET BIT(2) +#define DW100_DEWARP_CTRL_FORMAT_YUV422_SP 0UL +#define DW100_DEWARP_CTRL_FORMAT_YUV422_PACKED 1UL +#define DW100_DEWARP_CTRL_FORMAT_YUV420_SP 2UL +#define DW100_DEWARP_CTRL_INPUT_FORMAT_MASK GENMASK(5, 4) +#define DW100_DEWARP_CTRL_INPUT_FORMAT(x) ((x) << 4) +#define DW100_DEWARP_CTRL_OUTPUT_FORMAT(x) ((x) << 6) +#define DW100_DEWARP_CTRL_OUTPUT_FORMAT_MASK GENMASK(7, 6) +#define DW100_DEWARP_CTRL_SRC_AUTO_SHADOW BIT(8) +#define DW100_DEWARP_CTRL_HW_HANDSHAKE BIT(9) +#define DW100_DEWARP_CTRL_DST_AUTO_SHADOW BIT(10) +#define DW100_DEWARP_CTRL_SPLIT_LINE BIT(11) +#define DW100_DEWARP_CTRL_PREFETCH_MODE_MASK GENMASK(17, 16) +#define DW100_DEWARP_CTRL_PREFETCH_MODE_TRAVERSAL (0UL << 16) +#define DW100_DEWARP_CTRL_PREFETCH_MODE_CALCULATION (1UL << 16) +#define DW100_DEWARP_CTRL_PREFETCH_MODE_AUTO (2UL << 16) +#define DW100_DEWARP_CTRL_PREFETCH_THRESHOLD_MASK GENMASK(24, 18) +#define DW100_DEWARP_CTRL_PREFETCH_THRESHOLD(x) ((x) << 18) + +#define DW100_MAP_LUT_ADDR 0x08 +#define DW100_MAP_LUT_ADDR_ADDR(addr) (((addr) >> 4) & GENMASK(29, 0)) +#define DW100_MAP_LUT_SIZE 0x0c +#define DW100_MAP_LUT_SIZE_WIDTH(w) (((w) & GENMASK(10, 0)) << 0) +#define DW100_MAP_LUT_SIZE_HEIGHT(h) (((h) & GENMASK(10, 0)) << 16) +#define DW100_SRC_IMG_Y_BASE 0x10 +#define DW100_IMG_Y_BASE(base) (((base) >> 4) & GENMASK(29, 0)) +#define DW100_SRC_IMG_UV_BASE 0x14 +#define DW100_IMG_UV_BASE(base) (((base) >> 4) & GENMASK(29, 0)) +#define DW100_SRC_IMG_SIZE 0x18 +#define DW100_IMG_SIZE_WIDTH(w) (((w) & GENMASK(12, 0)) << 0) +#define DW100_IMG_SIZE_HEIGHT(h) (((h) & GENMASK(12, 0)) << 16) + +#define DW100_SRC_IMG_STRIDE 0x1c +#define DW100_MAP_LUT_ADDR2 0x20 +#define DW100_MAP_LUT_SIZE2 0x24 +#define DW100_SRC_IMG_Y_BASE2 0x28 +#define DW100_SRC_IMG_UV_BASE2 0x2c +#define DW100_SRC_IMG_SIZE2 0x30 +#define DW100_SRC_IMG_STRIDE2 0x34 +#define DW100_DST_IMG_Y_BASE 0x38 +#define DW100_DST_IMG_UV_BASE 0x3c +#define DW100_DST_IMG_SIZE 0x40 +#define DW100_DST_IMG_STRIDE 0x44 +#define DW100_DST_IMG_Y_BASE2 0x48 +#define DW100_DST_IMG_UV_BASE2 0x4c +#define DW100_DST_IMG_SIZE2 0x50 +#define DW100_DST_IMG_STRIDE2 0x54 +#define DW100_SWAP_CONTROL 0x58 +#define DW100_SWAP_CONTROL_BYTE BIT(0) +#define DW100_SWAP_CONTROL_SHORT BIT(1) +#define DW100_SWAP_CONTROL_WORD BIT(2) +#define DW100_SWAP_CONTROL_LONG BIT(3) +#define DW100_SWAP_CONTROL_Y(x) (((x) & GENMASK(3, 0)) << 0) +#define DW100_SWAP_CONTROL_UV(x) (((x) & GENMASK(3, 0)) << 4) +#define DW100_SWAP_CONTROL_SRC(x) (((x) & GENMASK(7, 0)) << 0) +#define DW100_SWAP_CONTROL_DST(x) (((x) & GENMASK(7, 0)) << 8) +#define DW100_SWAP_CONTROL_SRC2(x) (((x) & GENMASK(7, 0)) << 16) +#define DW100_SWAP_CONTROL_DST2(x) (((x) & GENMASK(7, 0)) << 24) +#define DW100_SWAP_CONTROL_SRC_MASK GENMASK(7, 0) +#define DW100_SWAP_CONTROL_DST_MASK GENMASK(15, 8) +#define DW100_SWAP_CONTROL_SRC2_MASK GENMASK(23, 16) +#define DW100_SWAP_CONTROL_DST2_MASK GENMASK(31, 24) +#define DW100_VERTICAL_SPLIT_LINE 0x5c +#define DW100_HORIZON_SPLIT_LINE 0x60 +#define DW100_SCALE_FACTOR 0x64 +#define DW100_ROI_START 0x68 +#define DW100_ROI_START_X(x) (((x) & GENMASK(12, 0)) << 0) +#define DW100_ROI_START_Y(y) (((y) & GENMASK(12, 0)) << 16) +#define DW100_BOUNDARY_PIXEL 0x6c +#define DW100_BOUNDARY_PIXEL_V(v) (((v) & GENMASK(7, 0)) << 0) +#define DW100_BOUNDARY_PIXEL_U(u) (((u) & GENMASK(7, 0)) << 8) +#define DW100_BOUNDARY_PIXEL_Y(y) (((y) & GENMASK(7, 0)) << 16) + +#define DW100_INTERRUPT_STATUS 0x70 +#define DW100_INTERRUPT_STATUS_INT_FRAME_DONE BIT(0) +#define DW100_INTERRUPT_STATUS_INT_ERR_TIME_OUT BIT(1) +#define DW100_INTERRUPT_STATUS_INT_ERR_AXI_RESP BIT(2) +#define DW100_INTERRUPT_STATUS_INT_ERR_X BIT(3) +#define DW100_INTERRUPT_STATUS_INT_ERR_MB_FETCH BIT(4) +#define DW100_INTERRUPT_STATUS_INT_ERR_FRAME2 BIT(5) +#define DW100_INTERRUPT_STATUS_INT_ERR_FRAME3 BIT(6) +#define DW100_INTERRUPT_STATUS_INT_ERR_FRAME_DONE BIT(7) +#define DW100_INTERRUPT_STATUS_INT_ERR_STATUS(x) (((x) >> 1) & 0x7f) +#define DW100_INTERRUPT_STATUS_INT_STATUS(x) ((x) & 0xff) + +#define DW100_INTERRUPT_STATUS_INT_ENABLE_MASK GENMASK(15, 8) +#define DW100_INTERRUPT_STATUS_INT_ENABLE(x) (((x) & GENMASK(7, 0)) << 8) +#define DW100_INTERRUPT_STATUS_FRAME_BUSY BIT(16) +#define DW100_INTERRUPT_STATUS_INT_CLEAR(x) (((x) & GENMASK(7, 0)) << 24) +#define DW100_BUS_CTRL 0x74 +#define DW100_BUS_CTRL_AXI_MASTER_ENABLE BIT(31) +#define DW100_BUS_CTRL1 0x78 +#define DW100_BUS_TIME_OUT_CYCLE 0x7c +#define DW100_DST_IMG_Y_SIZE1 0x80 +#define DW100_DST_IMG_Y_SIZE(sz) (((sz) >> 4) & GENMASK(29, 0)) +#define DW100_DST_IMG_UV_SIZE(sz) (((sz) >> 4) & GENMASK(29, 0)) +#define DW100_DST_IMG_UV_SIZE1 0x84 +#define DW100_DST_IMG_Y_SIZE2 0x88 +#define DW100_DST_IMG_UV_SIZE2 0x8c + +#endif /* _DW100_REGS_H_ */ diff --git a/drivers/media/platform/renesas/vsp1/vsp1.h b/drivers/media/platform/renesas/vsp1/vsp1.h index 37cf33c7e6ca16f38e11646c9cb57c734487cbcd..2f6f0c6ae55514d312e48232a5f8c8673f69ba13 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1.h +++ b/drivers/media/platform/renesas/vsp1/vsp1.h @@ -22,6 +22,7 @@ struct clk; struct device; struct rcar_fcp_device; +struct reset_control; struct vsp1_drm; struct vsp1_entity; @@ -54,6 +55,7 @@ struct vsp1_uif; #define VSP1_HAS_HGT BIT(8) #define VSP1_HAS_BRS BIT(9) #define VSP1_HAS_EXT_DL BIT(10) +#define VSP1_HAS_NON_ZERO_LBA BIT(11) struct vsp1_device_info { u32 version; @@ -66,6 +68,7 @@ struct vsp1_device_info { unsigned int uif_count; unsigned int wpf_count; unsigned int num_bru_inputs; + u8 soc; bool uapi; }; @@ -79,6 +82,7 @@ struct vsp1_device { void __iomem *mmio; struct rcar_fcp_device *fcp; struct device *bus_master; + struct reset_control *rstc; struct vsp1_brx *brs; struct vsp1_brx *bru; diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drm.c b/drivers/media/platform/renesas/vsp1/vsp1_drm.c index 0c2507dc03d6cbe1340a83137d2e874cbec09306..c6f25200982c8525eb91bf907797379572691402 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_drm.c +++ b/drivers/media/platform/renesas/vsp1/vsp1_drm.c @@ -856,6 +856,8 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index, rpf->mem.addr[1] = cfg->mem[1]; rpf->mem.addr[2] = cfg->mem[2]; + rpf->format.flags = cfg->premult ? V4L2_PIX_FMT_FLAG_PREMUL_ALPHA : 0; + vsp1->drm->inputs[rpf_index].crop = cfg->src; vsp1->drm->inputs[rpf_index].compose = cfg->dst; vsp1->drm->inputs[rpf_index].zpos = cfg->zpos; diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drv.c b/drivers/media/platform/renesas/vsp1/vsp1_drv.c index 1f73c48eb738183df4325193f5b829a3c07b8ed5..c260d318d2988dbfaf4e1ca51cea1c9f5645cdb4 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_drv.c +++ b/drivers/media/platform/renesas/vsp1/vsp1_drv.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -622,6 +623,7 @@ static int __maybe_unused vsp1_pm_runtime_suspend(struct device *dev) struct vsp1_device *vsp1 = dev_get_drvdata(dev); rcar_fcp_disable(vsp1->fcp); + reset_control_assert(vsp1->rstc); return 0; } @@ -631,13 +633,31 @@ static int __maybe_unused vsp1_pm_runtime_resume(struct device *dev) struct vsp1_device *vsp1 = dev_get_drvdata(dev); int ret; + ret = reset_control_deassert(vsp1->rstc); + if (ret < 0) + return ret; + if (vsp1->info) { + /* + * On R-Car Gen2 and RZ/G1, vsp1 register access after deassert + * can cause lock-up. It is a special case and needs some delay + * to avoid this lock-up. + */ + if (vsp1->info->gen == 2) + udelay(1); + ret = vsp1_device_init(vsp1); if (ret < 0) - return ret; + goto done; } - return rcar_fcp_enable(vsp1->fcp); + ret = rcar_fcp_enable(vsp1->fcp); + +done: + if (ret < 0) + reset_control_assert(vsp1->rstc); + + return ret; } static const struct dev_pm_ops vsp1_pm_ops = { @@ -768,6 +788,7 @@ static const struct vsp1_device_info vsp1_device_infos[] = { }, { .version = VI6_IP_VERSION_MODEL_VSPD_V3, .model = "VSP2-D", + .soc = VI6_IP_VERSION_SOC_V3H, .gen = 3, .features = VSP1_HAS_BRS | VSP1_HAS_BRU, .lif_count = 1, @@ -775,6 +796,17 @@ static const struct vsp1_device_info vsp1_device_infos[] = { .uif_count = 1, .wpf_count = 1, .num_bru_inputs = 5, + }, { + .version = VI6_IP_VERSION_MODEL_VSPD_V3, + .model = "VSP2-D", + .soc = VI6_IP_VERSION_SOC_V3M, + .gen = 3, + .features = VSP1_HAS_BRS | VSP1_HAS_BRU | VSP1_HAS_NON_ZERO_LBA, + .lif_count = 1, + .rpf_count = 5, + .uif_count = 1, + .wpf_count = 1, + .num_bru_inputs = 5, }, { .version = VI6_IP_VERSION_MODEL_VSPDL_GEN3, .model = "VSP2-DL", @@ -798,11 +830,55 @@ static const struct vsp1_device_info vsp1_device_infos[] = { }, }; +static const struct vsp1_device_info rzg2l_vsp2_device_info = { + .version = VI6_IP_VERSION_MODEL_VSPD_RZG2L, + .model = "VSP2-D", + .soc = VI6_IP_VERSION_SOC_RZG2L, + .gen = 3, + .features = VSP1_HAS_BRS | VSP1_HAS_WPF_VFLIP | VSP1_HAS_EXT_DL + | VSP1_HAS_NON_ZERO_LBA, + .lif_count = 1, + .rpf_count = 2, + .wpf_count = 1, +}; + +static const struct vsp1_device_info *vsp1_lookup_info(struct vsp1_device *vsp1) +{ + const struct vsp1_device_info *info; + unsigned int i; + u32 model; + u32 soc; + + /* + * Try the info stored in match data first for devices that don't have + * a version register. + */ + info = of_device_get_match_data(vsp1->dev); + if (info) { + vsp1->version = VI6_IP_VERSION_VSP_SW | info->version | info->soc; + return info; + } + + vsp1->version = vsp1_read(vsp1, VI6_IP_VERSION); + model = vsp1->version & VI6_IP_VERSION_MODEL_MASK; + soc = vsp1->version & VI6_IP_VERSION_SOC_MASK; + + for (i = 0; i < ARRAY_SIZE(vsp1_device_infos); ++i) { + info = &vsp1_device_infos[i]; + + if (model == info->version && (!info->soc || soc == info->soc)) + return info; + } + + dev_err(vsp1->dev, "unsupported IP version 0x%08x\n", vsp1->version); + + return NULL; +} + static int vsp1_probe(struct platform_device *pdev) { struct vsp1_device *vsp1; struct device_node *fcp_node; - unsigned int i; int ret; int irq; @@ -825,6 +901,11 @@ static int vsp1_probe(struct platform_device *pdev) if (irq < 0) return irq; + vsp1->rstc = devm_reset_control_get_shared(&pdev->dev, NULL); + if (IS_ERR(vsp1->rstc)) + return dev_err_probe(&pdev->dev, PTR_ERR(vsp1->rstc), + "failed to get reset control\n"); + /* FCP (optional). */ fcp_node = of_parse_phandle(pdev->dev.of_node, "renesas,fcp", 0); if (fcp_node) { @@ -853,19 +934,8 @@ static int vsp1_probe(struct platform_device *pdev) if (ret < 0) goto done; - vsp1->version = vsp1_read(vsp1, VI6_IP_VERSION); - - for (i = 0; i < ARRAY_SIZE(vsp1_device_infos); ++i) { - if ((vsp1->version & VI6_IP_VERSION_MODEL_MASK) == - vsp1_device_infos[i].version) { - vsp1->info = &vsp1_device_infos[i]; - break; - } - } - + vsp1->info = vsp1_lookup_info(vsp1); if (!vsp1->info) { - dev_err(&pdev->dev, "unsupported IP version 0x%08x\n", - vsp1->version); vsp1_device_put(vsp1); ret = -ENXIO; goto done; @@ -922,6 +992,7 @@ static int vsp1_remove(struct platform_device *pdev) static const struct of_device_id vsp1_of_match[] = { { .compatible = "renesas,vsp1" }, { .compatible = "renesas,vsp2" }, + { .compatible = "renesas,r9a07g044-vsp2", .data = &rzg2l_vsp2_device_info }, { }, }; MODULE_DEVICE_TABLE(of, vsp1_of_match); diff --git a/drivers/media/platform/renesas/vsp1/vsp1_lif.c b/drivers/media/platform/renesas/vsp1/vsp1_lif.c index 6a6857ac932707e93c048e68991de7502c8c35a7..186a5730e1e32fc1dcb049a5c2c94700029ab2e1 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_lif.c +++ b/drivers/media/platform/renesas/vsp1/vsp1_lif.c @@ -107,6 +107,7 @@ static void lif_configure_stream(struct vsp1_entity *entity, case VI6_IP_VERSION_MODEL_VSPDL_GEN3: case VI6_IP_VERSION_MODEL_VSPD_V3: + case VI6_IP_VERSION_MODEL_VSPD_RZG2L: hbth = 0; obth = 1500; lbth = 0; @@ -130,13 +131,12 @@ static void lif_configure_stream(struct vsp1_entity *entity, VI6_LIF_CTRL_REQSEL | VI6_LIF_CTRL_LIF_EN); /* - * On R-Car V3M the LIF0 buffer attribute register has to be set to a - * non-default value to guarantee proper operation (otherwise artifacts - * may appear on the output). The value required by the manual is not - * explained but is likely a buffer size or threshold. + * On R-Car V3M and RZ/G2L the LIF0 buffer attribute register has to be + * set to a non-default value to guarantee proper operation (otherwise + * artifacts may appear on the output). The value required by the + * manual is not explained but is likely a buffer size or threshold. */ - if ((entity->vsp1->version & VI6_IP_VERSION_MASK) == - (VI6_IP_VERSION_MODEL_VSPD_V3 | VI6_IP_VERSION_SOC_V3M)) + if (vsp1_feature(entity->vsp1, VSP1_HAS_NON_ZERO_LBA)) vsp1_lif_write(lif, dlb, VI6_LIF_LBA, VI6_LIF_LBA_LBA0 | (1536 << VI6_LIF_LBA_LBA1_SHIFT)); diff --git a/drivers/media/platform/renesas/vsp1/vsp1_regs.h b/drivers/media/platform/renesas/vsp1/vsp1_regs.h index fae7286eb01eb8266775cf738b4e7bd2befce005..8928f4c6bb557b5a3204864acb8c0b54ebcad548 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_regs.h +++ b/drivers/media/platform/renesas/vsp1/vsp1_regs.h @@ -767,6 +767,8 @@ #define VI6_IP_VERSION_MODEL_VSPDL_GEN3 (0x19 << 8) #define VI6_IP_VERSION_MODEL_VSPBS_GEN3 (0x1a << 8) #define VI6_IP_VERSION_MODEL_VSPD_V3U (0x1c << 8) +/* RZ/G2L SoCs have no version register, So use 0x80 as the model version */ +#define VI6_IP_VERSION_MODEL_VSPD_RZG2L (0x80 << 8) #define VI6_IP_VERSION_SOC_MASK (0xff << 0) #define VI6_IP_VERSION_SOC_H2 (0x01 << 0) @@ -780,6 +782,10 @@ #define VI6_IP_VERSION_SOC_M3N (0x04 << 0) #define VI6_IP_VERSION_SOC_E3 (0x04 << 0) #define VI6_IP_VERSION_SOC_V3U (0x05 << 0) +/* RZ/G2L SoCs have no version register, So use 0x80 for SoC Identification */ +#define VI6_IP_VERSION_SOC_RZG2L (0x80 << 0) + +#define VI6_IP_VERSION_VSP_SW (0xfffe << 16) /* SW VSP version */ /* ----------------------------------------------------------------------------- * RPF CLUT Registers diff --git a/drivers/media/platform/renesas/vsp1/vsp1_video.c b/drivers/media/platform/renesas/vsp1/vsp1_video.c index e8e0ee5f2277059c3b677449d6f21c00c2d35da4..df1606b49d77aecd1f811982854369821653790c 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_video.c +++ b/drivers/media/platform/renesas/vsp1/vsp1_video.c @@ -305,7 +305,7 @@ static int vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe) * @video: the video node * * This function completes the current buffer by filling its sequence number, - * time stamp and payload size, and hands it back to the videobuf core. + * time stamp and payload size, and hands it back to the vb2 core. * * Return the next queued buffer or NULL if the queue is empty. */ diff --git a/drivers/media/platform/rockchip/rga/rga.c b/drivers/media/platform/rockchip/rga/rga.c index 2f8df74ad0fdec71e5372349007a31a9fa369df0..61b25fcf826e92a38a56dfde1449ed6677e033dc 100644 --- a/drivers/media/platform/rockchip/rga/rga.c +++ b/drivers/media/platform/rockchip/rga/rga.c @@ -816,7 +816,7 @@ static int rga_probe(struct platform_device *pdev) ret = rga_parse_dt(rga); if (ret) - dev_err(&pdev->dev, "Unable to parse OF data\n"); + return dev_err_probe(&pdev->dev, ret, "Unable to parse OF data\n"); pm_runtime_enable(rga->dev); diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-core.h b/drivers/media/platform/samsung/exynos4-is/fimc-core.h index 7a058f3e629831f78b3bc53986570803f2921b9a..2b0760add092937e135dedfc14e2813368de8baf 100644 --- a/drivers/media/platform/samsung/exynos4-is/fimc-core.h +++ b/drivers/media/platform/samsung/exynos4-is/fimc-core.h @@ -215,7 +215,7 @@ struct fimc_addr { /** * struct fimc_vid_buffer - the driver's video buffer - * @vb: v4l videobuf buffer + * @vb: v4l vb2 buffer * @list: linked list structure for buffer queue * @addr: precalculated DMA address set * @index: buffer index for the output DMA engine diff --git a/drivers/media/platform/samsung/exynos4-is/fimc-is.c b/drivers/media/platform/samsung/exynos4-is/fimc-is.c index e3072d69c49fa797acf358ff8c77e9912568847b..a7704ff069d6c4c7fb2c5936b9f903da2f50d40d 100644 --- a/drivers/media/platform/samsung/exynos4-is/fimc-is.c +++ b/drivers/media/platform/samsung/exynos4-is/fimc-is.c @@ -213,6 +213,7 @@ static int fimc_is_register_subdevs(struct fimc_is *is) if (ret < 0 || index >= FIMC_IS_SENSORS_NUM) { of_node_put(child); + of_node_put(i2c_bus); return ret; } index++; diff --git a/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c b/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c index 761341934925e7a1becdaf51622e3ee12bdaa84e..fca5c6405eec3bc684cf3b570adf197710375962 100644 --- a/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c +++ b/drivers/media/platform/samsung/s5p-mfc/s5p_mfc.c @@ -323,7 +323,7 @@ static void s5p_mfc_handle_frame_new(struct s5p_mfc_ctx *ctx, unsigned int err) } ctx->sequence++; /* The MFC returns address of the buffer, now we have to - * check which videobuf does it correspond to */ + * check which vb2_buffer does it correspond to */ list_for_each_entry(dst_buf, &ctx->dst_queue, list) { u32 addr = (u32)vb2_dma_contig_plane_dma_addr(&dst_buf->b->vb2_buf, 0); @@ -1399,6 +1399,7 @@ static int s5p_mfc_probe(struct platform_device *pdev) /* Deinit MFC if probe had failed */ err_enc_reg: video_unregister_device(dev->vfd_dec); + dev->vfd_dec = NULL; err_dec_reg: video_device_release(dev->vfd_enc); err_enc_alloc: @@ -1444,8 +1445,6 @@ static int s5p_mfc_remove(struct platform_device *pdev) video_unregister_device(dev->vfd_enc); video_unregister_device(dev->vfd_dec); - video_device_release(dev->vfd_enc); - video_device_release(dev->vfd_dec); v4l2_device_unregister(&dev->v4l2_dev); s5p_mfc_unconfigure_dma_memory(dev); diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c index 1d46e113d01d5095c1910e77e5c78c5f14295b9a..74d64a20ba5bfe2714e0ee717cfa7e65a028d960 100644 --- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c +++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c @@ -177,7 +177,7 @@ static int sun6i_video_start_streaming(struct vb2_queue *vq, unsigned int count) /* * CSI will lookup the next dma buffer for next frame before the - * the current frame done IRQ triggered. This is not documented + * current frame done IRQ triggered. This is not documented * but reported by Ondřej Jirman. * The BSP code has workaround for this too. It skip to mark the * first buffer as frame done for VB2 and pass the second buffer diff --git a/drivers/media/platform/ti/am437x/am437x-vpfe.h b/drivers/media/platform/ti/am437x/am437x-vpfe.h index 05ee37db02732045f91a524757fd1c2f3af7744d..f8b4e917b91a36ccbd158a59ba7f3dd2c20f0828 100644 --- a/drivers/media/platform/ti/am437x/am437x-vpfe.h +++ b/drivers/media/platform/ti/am437x/am437x-vpfe.h @@ -267,7 +267,7 @@ struct vpfe_device { * is different from the image window */ struct v4l2_rect crop; - /* Buffer queue used in video-buf */ + /* Buffer queue used in vb2 */ struct vb2_queue buffer_queue; /* Queue of filled frames */ struct list_head dma_queue; diff --git a/drivers/media/platform/ti/cal/cal-camerarx.c b/drivers/media/platform/ti/cal/cal-camerarx.c index e136d70b40487c4d251a8e80ab627469bf617bac..16ae52879a79bf5d53e28e1062e1409875cb4ad7 100644 --- a/drivers/media/platform/ti/cal/cal-camerarx.c +++ b/drivers/media/platform/ti/cal/cal-camerarx.c @@ -622,12 +622,12 @@ static inline struct cal_camerarx *to_cal_camerarx(struct v4l2_subdev *sd) static struct v4l2_mbus_framefmt * cal_camerarx_get_pad_format(struct cal_camerarx *phy, - struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_state *state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&phy->subdev, sd_state, pad); + return v4l2_subdev_get_try_format(&phy->subdev, state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &phy->formats[pad]; default: @@ -653,7 +653,7 @@ static int cal_camerarx_sd_s_stream(struct v4l2_subdev *sd, int enable) } static int cal_camerarx_sd_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_state *state, struct v4l2_subdev_mbus_code_enum *code) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -670,7 +670,7 @@ static int cal_camerarx_sd_enum_mbus_code(struct v4l2_subdev *sd, goto out; } - fmt = cal_camerarx_get_pad_format(phy, sd_state, + fmt = cal_camerarx_get_pad_format(phy, state, CAL_CAMERARX_PAD_SINK, code->which); code->code = fmt->code; @@ -690,7 +690,7 @@ out: } static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_state *state, struct v4l2_subdev_frame_size_enum *fse) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -706,7 +706,7 @@ static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd, if (cal_rx_pad_is_source(fse->pad)) { struct v4l2_mbus_framefmt *fmt; - fmt = cal_camerarx_get_pad_format(phy, sd_state, + fmt = cal_camerarx_get_pad_format(phy, state, CAL_CAMERARX_PAD_SINK, fse->which); if (fse->code != fmt->code) { @@ -738,7 +738,7 @@ out: } static int cal_camerarx_sd_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -746,7 +746,7 @@ static int cal_camerarx_sd_get_fmt(struct v4l2_subdev *sd, mutex_lock(&phy->mutex); - fmt = cal_camerarx_get_pad_format(phy, sd_state, format->pad, + fmt = cal_camerarx_get_pad_format(phy, state, format->pad, format->which); format->format = *fmt; @@ -756,7 +756,7 @@ static int cal_camerarx_sd_get_fmt(struct v4l2_subdev *sd, } static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -766,7 +766,7 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, /* No transcoding, source and sink formats must match. */ if (cal_rx_pad_is_source(format->pad)) - return cal_camerarx_sd_get_fmt(sd, sd_state, format); + return cal_camerarx_sd_get_fmt(sd, state, format); /* * Default to the first format if the requested media bus code isn't @@ -792,12 +792,12 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, mutex_lock(&phy->mutex); - fmt = cal_camerarx_get_pad_format(phy, sd_state, + fmt = cal_camerarx_get_pad_format(phy, state, CAL_CAMERARX_PAD_SINK, format->which); *fmt = format->format; - fmt = cal_camerarx_get_pad_format(phy, sd_state, + fmt = cal_camerarx_get_pad_format(phy, state, CAL_CAMERARX_PAD_FIRST_SOURCE, format->which); *fmt = format->format; @@ -808,10 +808,10 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, } static int cal_camerarx_sd_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_state *sd_state) + struct v4l2_subdev_state *state) { struct v4l2_subdev_format format = { - .which = sd_state ? V4L2_SUBDEV_FORMAT_TRY + .which = state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE, .pad = CAL_CAMERARX_PAD_SINK, .format = { @@ -826,7 +826,7 @@ static int cal_camerarx_sd_init_cfg(struct v4l2_subdev *sd, }, }; - return cal_camerarx_sd_set_fmt(sd, sd_state, &format); + return cal_camerarx_sd_set_fmt(sd, state, &format); } static const struct v4l2_subdev_video_ops cal_camerarx_video_ops = { @@ -871,6 +871,7 @@ struct cal_camerarx *cal_camerarx_create(struct cal_dev *cal, phy->cal = cal; phy->instance = instance; + spin_lock_init(&phy->vc_lock); mutex_init(&phy->mutex); phy->res = platform_get_resource_byname(pdev, IORESOURCE_MEM, diff --git a/drivers/media/platform/ti/cal/cal-video.c b/drivers/media/platform/ti/cal/cal-video.c index 776da0cfcdbe284de54cc9be204d06ee8c489b28..21e3d0aabf706d7e108ed31c12db81564d001431 100644 --- a/drivers/media/platform/ti/cal/cal-video.c +++ b/drivers/media/platform/ti/cal/cal-video.c @@ -191,7 +191,7 @@ static int cal_legacy_try_fmt_vid_cap(struct file *file, void *priv, struct cal_ctx *ctx = video_drvdata(file); const struct cal_format_info *fmtinfo; struct v4l2_subdev_frame_size_enum fse; - int ret, found; + int found; fmtinfo = find_format_by_pix(ctx, f->fmt.pix.pixelformat); if (!fmtinfo) { @@ -206,12 +206,13 @@ static int cal_legacy_try_fmt_vid_cap(struct file *file, void *priv, f->fmt.pix.field = ctx->v_fmt.fmt.pix.field; /* check for/find a valid width/height */ - ret = 0; found = false; fse.pad = 0; fse.code = fmtinfo->code; fse.which = V4L2_SUBDEV_FORMAT_ACTIVE; for (fse.index = 0; ; fse.index++) { + int ret; + ret = v4l2_subdev_call(ctx->phy->source, pad, enum_frame_size, NULL, &fse); if (ret) diff --git a/drivers/media/platform/ti/cal/cal.c b/drivers/media/platform/ti/cal/cal.c index 425b4f4b7ed7e1d40a148be6acd97af8c4e57700..56b61c0583cf8b1af3c0b8b54d8659ad66ed9713 100644 --- a/drivers/media/platform/ti/cal/cal.c +++ b/drivers/media/platform/ti/cal/cal.c @@ -543,7 +543,22 @@ void cal_ctx_unprepare(struct cal_ctx *ctx) void cal_ctx_start(struct cal_ctx *ctx) { - ctx->sequence = 0; + struct cal_camerarx *phy = ctx->phy; + + /* + * Reset the frame number & sequence number, but only if the + * virtual channel is not already in use. + */ + + spin_lock(&phy->vc_lock); + + if (phy->vc_enable_count[ctx->vc]++ == 0) { + phy->vc_frame_number[ctx->vc] = 0; + phy->vc_sequence[ctx->vc] = 0; + } + + spin_unlock(&phy->vc_lock); + ctx->dma.state = CAL_DMA_RUNNING; /* Configure the CSI-2, pixel processing and write DMA contexts. */ @@ -563,8 +578,15 @@ void cal_ctx_start(struct cal_ctx *ctx) void cal_ctx_stop(struct cal_ctx *ctx) { + struct cal_camerarx *phy = ctx->phy; long timeout; + WARN_ON(phy->vc_enable_count[ctx->vc] == 0); + + spin_lock(&phy->vc_lock); + phy->vc_enable_count[ctx->vc]--; + spin_unlock(&phy->vc_lock); + /* * Request DMA stop and wait until it completes. If completion times * out, forcefully disable the DMA. @@ -601,6 +623,34 @@ void cal_ctx_stop(struct cal_ctx *ctx) * ------------------------------------------------------------------ */ +/* + * Track a sequence number for each virtual channel, which is shared by + * all contexts using the same virtual channel. This is done using the + * CSI-2 frame number as a base. + */ +static void cal_update_seq_number(struct cal_ctx *ctx) +{ + struct cal_dev *cal = ctx->cal; + struct cal_camerarx *phy = ctx->phy; + u16 prev_frame_num, frame_num; + u8 vc = ctx->vc; + + frame_num = + cal_read(cal, CAL_CSI2_STATUS(phy->instance, ctx->csi2_ctx)) & + 0xffff; + + if (phy->vc_frame_number[vc] != frame_num) { + prev_frame_num = phy->vc_frame_number[vc]; + + if (prev_frame_num >= frame_num) + phy->vc_sequence[vc] += 1; + else + phy->vc_sequence[vc] += frame_num - prev_frame_num; + + phy->vc_frame_number[vc] = frame_num; + } +} + static inline void cal_irq_wdma_start(struct cal_ctx *ctx) { spin_lock(&ctx->dma.lock); @@ -631,6 +681,8 @@ static inline void cal_irq_wdma_start(struct cal_ctx *ctx) } spin_unlock(&ctx->dma.lock); + + cal_update_seq_number(ctx); } static inline void cal_irq_wdma_end(struct cal_ctx *ctx) @@ -657,27 +709,62 @@ static inline void cal_irq_wdma_end(struct cal_ctx *ctx) if (buf) { buf->vb.vb2_buf.timestamp = ktime_get_ns(); buf->vb.field = ctx->v_fmt.fmt.pix.field; - buf->vb.sequence = ctx->sequence++; + buf->vb.sequence = ctx->phy->vc_sequence[ctx->vc]; + vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE); } } +static void cal_irq_handle_wdma(struct cal_ctx *ctx, bool start, bool end) +{ + /* + * CAL HW interrupts are inherently racy. If we get both start and end + * interrupts, we don't know what has happened: did the DMA for a single + * frame start and end, or did one frame end and a new frame start? + * + * Usually for normal pixel frames we get the interrupts separately. If + * we do get both, we have to guess. The assumption in the code below is + * that the active vertical area is larger than the blanking vertical + * area, and thus it is more likely that we get the end of the old frame + * and the start of a new frame. + * + * However, for embedded data, which is only a few lines high, we always + * get both interrupts. Here the assumption is that we get both for the + * same frame. + */ + if (ctx->v_fmt.fmt.pix.height < 10) { + if (start) + cal_irq_wdma_start(ctx); + + if (end) + cal_irq_wdma_end(ctx); + } else { + if (end) + cal_irq_wdma_end(ctx); + + if (start) + cal_irq_wdma_start(ctx); + } +} + static irqreturn_t cal_irq(int irq_cal, void *data) { struct cal_dev *cal = data; - u32 status; - - status = cal_read(cal, CAL_HL_IRQSTATUS(0)); - if (status) { - unsigned int i; + u32 status[3]; + unsigned int i; - cal_write(cal, CAL_HL_IRQSTATUS(0), status); + for (i = 0; i < 3; ++i) { + status[i] = cal_read(cal, CAL_HL_IRQSTATUS(i)); + if (status[i]) + cal_write(cal, CAL_HL_IRQSTATUS(i), status[i]); + } - if (status & CAL_HL_IRQ_OCPO_ERR_MASK) + if (status[0]) { + if (status[0] & CAL_HL_IRQ_OCPO_ERR_MASK) dev_err_ratelimited(cal->dev, "OCPO ERROR\n"); for (i = 0; i < cal->data->num_csi2_phy; ++i) { - if (status & CAL_HL_IRQ_CIO_MASK(i)) { + if (status[0] & CAL_HL_IRQ_CIO_MASK(i)) { u32 cio_stat = cal_read(cal, CAL_CSI2_COMPLEXIO_IRQSTATUS(i)); @@ -688,7 +775,7 @@ static irqreturn_t cal_irq(int irq_cal, void *data) cio_stat); } - if (status & CAL_HL_IRQ_VC_MASK(i)) { + if (status[0] & CAL_HL_IRQ_VC_MASK(i)) { u32 vc_stat = cal_read(cal, CAL_CSI2_VC_IRQSTATUS(i)); dev_err_ratelimited(cal->dev, @@ -700,32 +787,12 @@ static irqreturn_t cal_irq(int irq_cal, void *data) } } - /* Check which DMA just finished */ - status = cal_read(cal, CAL_HL_IRQSTATUS(1)); - if (status) { - unsigned int i; - - /* Clear Interrupt status */ - cal_write(cal, CAL_HL_IRQSTATUS(1), status); - - for (i = 0; i < cal->num_contexts; ++i) { - if (status & CAL_HL_IRQ_WDMA_END_MASK(i)) - cal_irq_wdma_end(cal->ctx[i]); - } - } - - /* Check which DMA just started */ - status = cal_read(cal, CAL_HL_IRQSTATUS(2)); - if (status) { - unsigned int i; - - /* Clear Interrupt status */ - cal_write(cal, CAL_HL_IRQSTATUS(2), status); + for (i = 0; i < cal->num_contexts; ++i) { + bool end = !!(status[1] & CAL_HL_IRQ_WDMA_END_MASK(i)); + bool start = !!(status[2] & CAL_HL_IRQ_WDMA_START_MASK(i)); - for (i = 0; i < cal->num_contexts; ++i) { - if (status & CAL_HL_IRQ_WDMA_START_MASK(i)) - cal_irq_wdma_start(cal->ctx[i]); - } + if (start || end) + cal_irq_handle_wdma(cal->ctx[i], start, end); } return IRQ_HANDLED; diff --git a/drivers/media/platform/ti/cal/cal.h b/drivers/media/platform/ti/cal/cal.h index 61409ddced98b0d3c85ee67b36cd261189a2d4aa..80f2c9c73c719cba4af0440c92b3ff4622cb99f7 100644 --- a/drivers/media/platform/ti/cal/cal.h +++ b/drivers/media/platform/ti/cal/cal.h @@ -180,6 +180,12 @@ struct cal_camerarx { struct media_pad pads[CAL_CAMERARX_NUM_PADS]; struct v4l2_mbus_framefmt formats[CAL_CAMERARX_NUM_PADS]; + /* protects the vc_* fields below */ + spinlock_t vc_lock; + u8 vc_enable_count[4]; + u16 vc_frame_number[4]; + u32 vc_sequence[4]; + /* * Lock for camerarx ops. Protects: * - formats @@ -242,7 +248,6 @@ struct cal_ctx { const struct cal_format_info **active_fmt; unsigned int num_active_fmt; - unsigned int sequence; struct vb2_queue vb_vidq; u8 dma_ctx; u8 cport; diff --git a/drivers/media/platform/ti/davinci/Kconfig b/drivers/media/platform/ti/davinci/Kconfig index c61e697aeb12e68fb51d34c68f8d572c0fbbb1e9..96d4bed7fe9ee1da8473fae6093de20dd1dd0f87 100644 --- a/drivers/media/platform/ti/davinci/Kconfig +++ b/drivers/media/platform/ti/davinci/Kconfig @@ -32,55 +32,6 @@ config VIDEO_DAVINCI_VPIF_CAPTURE To compile this driver as a module, choose M here. There will be two modules called vpif.ko and vpif_capture.ko -config VIDEO_DM6446_CCDC - tristate "TI DM6446 CCDC video capture driver" - depends on V4L_PLATFORM_DRIVERS - depends on VIDEO_DEV - depends on ARCH_DAVINCI || COMPILE_TEST - depends on I2C - select VIDEOBUF_DMA_CONTIG - help - Enables DaVinci CCD hw module. DaVinci CCDC hw interfaces - with decoder modules such as TVP5146 over BT656 or - sensor module such as MT9T001 over a raw interface. This - module configures the interface and CCDC/ISIF to do - video frame capture from slave decoders. - - To compile this driver as a module, choose M here. There will - be three modules called vpfe_capture.ko, vpss.ko and dm644x_ccdc.ko - -config VIDEO_DM355_CCDC - tristate "TI DM355 CCDC video capture driver" - depends on V4L_PLATFORM_DRIVERS - depends on VIDEO_DEV - depends on ARCH_DAVINCI || COMPILE_TEST - depends on I2C - select VIDEOBUF_DMA_CONTIG - help - Enables DM355 CCD hw module. DM355 CCDC hw interfaces - with decoder modules such as TVP5146 over BT656 or - sensor module such as MT9T001 over a raw interface. This - module configures the interface and CCDC/ISIF to do - video frame capture from a slave decoders - - To compile this driver as a module, choose M here. There will - be three modules called vpfe_capture.ko, vpss.ko and dm355_ccdc.ko - -config VIDEO_DM365_ISIF - tristate "TI DM365 ISIF video capture driver" - depends on V4L_PLATFORM_DRIVERS - depends on VIDEO_DEV - depends on ARCH_DAVINCI || COMPILE_TEST - depends on I2C - select VIDEOBUF_DMA_CONTIG - help - Enables ISIF hw module. This is the hardware module for - configuring ISIF in VPFE to capture Raw Bayer RGB data from - a image sensor or YUV data from a YUV source. - - To compile this driver as a module, choose M here. There will - be three modules called vpfe_capture.ko, vpss.ko and isif.ko - config VIDEO_DAVINCI_VPBE_DISPLAY tristate "TI DaVinci VPBE V4L2-Display driver" depends on V4L_PLATFORM_DRIVERS diff --git a/drivers/media/platform/ti/davinci/Makefile b/drivers/media/platform/ti/davinci/Makefile index 05c45bf371aaeb1612a9a5e0e99f3421bfd24249..b20a91653162663a32f516722bfa85465b2090dd 100644 --- a/drivers/media/platform/ti/davinci/Makefile +++ b/drivers/media/platform/ti/davinci/Makefile @@ -8,9 +8,5 @@ obj-$(CONFIG_VIDEO_DAVINCI_VPIF_DISPLAY) += vpif.o vpif_display.o #VPIF Capture driver obj-$(CONFIG_VIDEO_DAVINCI_VPIF_CAPTURE) += vpif.o vpif_capture.o -# Capture: DM6446 and DM355 -obj-$(CONFIG_VIDEO_DM6446_CCDC) += vpfe_capture.o vpss.o dm644x_ccdc.o -obj-$(CONFIG_VIDEO_DM355_CCDC) += vpfe_capture.o vpss.o dm355_ccdc.o -obj-$(CONFIG_VIDEO_DM365_ISIF) += vpfe_capture.o vpss.o isif.o obj-$(CONFIG_VIDEO_DAVINCI_VPBE_DISPLAY) += vpss.o vpbe.o vpbe_osd.o \ vpbe_venc.o vpbe_display.o diff --git a/drivers/media/platform/ti/davinci/vpbe.c b/drivers/media/platform/ti/davinci/vpbe.c index 5f0aeb744e81bb8710a5f5f3749174ee7d3a332e..509ecc84624e0358e93d0467da6f909ff4c118c6 100644 --- a/drivers/media/platform/ti/davinci/vpbe.c +++ b/drivers/media/platform/ti/davinci/vpbe.c @@ -280,7 +280,7 @@ static int vpbe_set_default_output(struct vpbe_device *vpbe_dev) * vpbe_get_output - Get output * @vpbe_dev: vpbe device ptr * - * return current vpbe output to the the index + * return current vpbe output to the index */ static unsigned int vpbe_get_output(struct vpbe_device *vpbe_dev) { diff --git a/drivers/media/platform/ti/davinci/vpif.h b/drivers/media/platform/ti/davinci/vpif.h index 651943e3e3756917bd2184c1260433ef14c41d79..52ecc25622162df127b65cd385f426529b936737 100644 --- a/drivers/media/platform/ti/davinci/vpif.h +++ b/drivers/media/platform/ti/davinci/vpif.h @@ -322,10 +322,10 @@ static inline void channel1_intr_enable(int enable) } /* inline function to set buffer addresses in case of Y/C non mux mode */ -static inline void ch0_set_videobuf_addr_yc_nmux(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch0_set_video_buf_addr_yc_nmux(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH0_TOP_STRT_ADD_LUMA); regw(btm_strt_luma, VPIF_CH0_BTM_STRT_ADD_LUMA); @@ -334,10 +334,10 @@ static inline void ch0_set_videobuf_addr_yc_nmux(unsigned long top_strt_luma, } /* inline function to set buffer addresses in VPIF registers for video data */ -static inline void ch0_set_videobuf_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch0_set_video_buf_addr(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH0_TOP_STRT_ADD_LUMA); regw(btm_strt_luma, VPIF_CH0_BTM_STRT_ADD_LUMA); @@ -345,10 +345,10 @@ static inline void ch0_set_videobuf_addr(unsigned long top_strt_luma, regw(btm_strt_chroma, VPIF_CH0_BTM_STRT_ADD_CHROMA); } -static inline void ch1_set_videobuf_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch1_set_video_buf_addr(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH1_TOP_STRT_ADD_LUMA); @@ -538,10 +538,10 @@ static inline void channel3_clipping_enable(int enable) } /* inline function to set buffer addresses in case of Y/C non mux mode */ -static inline void ch2_set_videobuf_addr_yc_nmux(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch2_set_video_buf_addr_yc_nmux(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH2_TOP_STRT_ADD_LUMA); regw(btm_strt_luma, VPIF_CH2_BTM_STRT_ADD_LUMA); @@ -550,10 +550,10 @@ static inline void ch2_set_videobuf_addr_yc_nmux(unsigned long top_strt_luma, } /* inline function to set buffer addresses in VPIF registers for video data */ -static inline void ch2_set_videobuf_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch2_set_video_buf_addr(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH2_TOP_STRT_ADD_LUMA); regw(btm_strt_luma, VPIF_CH2_BTM_STRT_ADD_LUMA); @@ -561,10 +561,10 @@ static inline void ch2_set_videobuf_addr(unsigned long top_strt_luma, regw(btm_strt_chroma, VPIF_CH2_BTM_STRT_ADD_CHROMA); } -static inline void ch3_set_videobuf_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) +static inline void ch3_set_video_buf_addr(unsigned long top_strt_luma, + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH3_TOP_STRT_ADD_LUMA); regw(btm_strt_luma, VPIF_CH3_BTM_STRT_ADD_LUMA); @@ -574,18 +574,18 @@ static inline void ch3_set_videobuf_addr(unsigned long top_strt_luma, /* inline function to set buffer addresses in VPIF registers for vbi data */ static inline void ch2_set_vbi_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH2_TOP_STRT_ADD_VANC); regw(btm_strt_luma, VPIF_CH2_BTM_STRT_ADD_VANC); } static inline void ch3_set_vbi_addr(unsigned long top_strt_luma, - unsigned long btm_strt_luma, - unsigned long top_strt_chroma, - unsigned long btm_strt_chroma) + unsigned long btm_strt_luma, + unsigned long top_strt_chroma, + unsigned long btm_strt_chroma) { regw(top_strt_luma, VPIF_CH3_TOP_STRT_ADD_VANC); regw(btm_strt_luma, VPIF_CH3_BTM_STRT_ADD_VANC); diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c index b91eec899eb5c29512b7289a84211d386c6c004d..580723333fccb5516e7e78093cc9190df7b7ad34 100644 --- a/drivers/media/platform/ti/davinci/vpif_capture.c +++ b/drivers/media/platform/ti/davinci/vpif_capture.c @@ -632,11 +632,11 @@ static void vpif_config_addr(struct channel_obj *ch, int muxmode) common = &(ch->common[VPIF_VIDEO_INDEX]); if (VPIF_CHANNEL1_VIDEO == ch->channel_id) - common->set_addr = ch1_set_videobuf_addr; + common->set_addr = ch1_set_video_buf_addr; else if (2 == muxmode) - common->set_addr = ch0_set_videobuf_addr_yc_nmux; + common->set_addr = ch0_set_video_buf_addr_yc_nmux; else - common->set_addr = ch0_set_videobuf_addr; + common->set_addr = ch0_set_video_buf_addr; } /** diff --git a/drivers/media/platform/ti/davinci/vpif_capture.h b/drivers/media/platform/ti/davinci/vpif_capture.h index d5951f61df473b7a9aecf798ef0214d55531ac0f..6191056500cf6ed6920be95c29e0672eef2df879 100644 --- a/drivers/media/platform/ti/davinci/vpif_capture.h +++ b/drivers/media/platform/ti/davinci/vpif_capture.h @@ -50,7 +50,7 @@ struct common_obj { struct vpif_cap_buffer *next_frm; /* Used to store pixel format */ struct v4l2_format fmt; - /* Buffer queue used in video-buf */ + /* Buffer queue used in vb2 */ struct vb2_queue buffer_queue; /* Queue of filled frames */ struct list_head dma_queue; diff --git a/drivers/media/platform/ti/davinci/vpif_display.c b/drivers/media/platform/ti/davinci/vpif_display.c index 5d524acc995d09fde64492849badf632bf9d0ff7..b2df81603f623210b5cc7db403e169973ee47a1c 100644 --- a/drivers/media/platform/ti/davinci/vpif_display.c +++ b/drivers/media/platform/ti/davinci/vpif_display.c @@ -563,12 +563,12 @@ static void vpif_config_addr(struct channel_obj *ch, int muxmode) struct common_obj *common = &ch->common[VPIF_VIDEO_INDEX]; if (VPIF_CHANNEL3_VIDEO == ch->channel_id) { - common->set_addr = ch3_set_videobuf_addr; + common->set_addr = ch3_set_video_buf_addr; } else { if (2 == muxmode) - common->set_addr = ch2_set_videobuf_addr_yc_nmux; + common->set_addr = ch2_set_video_buf_addr_yc_nmux; else - common->set_addr = ch2_set_videobuf_addr; + common->set_addr = ch2_set_video_buf_addr; } } diff --git a/drivers/media/platform/ti/davinci/vpif_display.h b/drivers/media/platform/ti/davinci/vpif_display.h index f27474e0fc36459e25c2dd10d611953f97b58f0a..dae20053dd732259590e5e41242d627cb64ddf83 100644 --- a/drivers/media/platform/ti/davinci/vpif_display.h +++ b/drivers/media/platform/ti/davinci/vpif_display.h @@ -64,11 +64,11 @@ struct common_obj { struct vpif_disp_buffer *next_frm; /* Pointer pointing to next * vb2_buffer */ struct v4l2_format fmt; /* Used to store the format */ - struct vb2_queue buffer_queue; /* Buffer queue used in - * video-buf */ + struct vb2_queue buffer_queue; /* Buffer queue used in vb2 */ struct list_head dma_queue; /* Queue of filled frames */ - spinlock_t irqlock; /* Used in video-buf */ + spinlock_t irqlock; /* Used for video buffer + * handling */ /* channel specific parameters */ struct mutex lock; /* lock used to access this diff --git a/drivers/media/platform/ti/omap/omap_voutlib.c b/drivers/media/platform/ti/omap/omap_voutlib.c index fdea2309ee373eaaa0ec309e384ded6563284685..0ac46458e41ccc831082f04e1bd79ad1544ff2e5 100644 --- a/drivers/media/platform/ti/omap/omap_voutlib.c +++ b/drivers/media/platform/ti/omap/omap_voutlib.c @@ -107,7 +107,7 @@ EXPORT_SYMBOL_GPL(omap_vout_try_window); /* Given a new render window in new_win, adjust the window to the * nearest supported configuration. The image cropping window in crop * will also be adjusted if necessary. Preference is given to keeping the - * the window as close to the requested configuration as possible. If + * window as close to the requested configuration as possible. If * successful, new_win, vout->win, and crop are updated. * Returns zero if successful, or -EINVAL if the requested preview window is * impossible and cannot reasonably be adjusted. diff --git a/drivers/media/platform/ti/omap3isp/isp.c b/drivers/media/platform/ti/omap3isp/isp.c index d251736eb4202d2c4c7bd499a15f99cca538302c..a6052df9bb19e33fa9d80e504c01fd440b617f59 100644 --- a/drivers/media/platform/ti/omap3isp/isp.c +++ b/drivers/media/platform/ti/omap3isp/isp.c @@ -1528,7 +1528,7 @@ void omap3isp_print_status(struct isp_device *isp) * To solve this problem power management support is split into prepare/complete * and suspend/resume operations. The pipelines are stopped in prepare() and the * ISP clocks get disabled in suspend(). Similarly, the clocks are re-enabled in - * resume(), and the the pipelines are restarted in complete(). + * resume(), and the pipelines are restarted in complete(). * * TODO: PM dependencies between the ISP and sensors are not modelled explicitly * yet. diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c index d7059180e80ee66b9bdced93ad6d7b4550a92c99..cc9a97d5d5051dddce82981e2fe1fc1bc4d7776e 100644 --- a/drivers/media/platform/ti/omap3isp/ispvideo.c +++ b/drivers/media/platform/ti/omap3isp/ispvideo.c @@ -1071,7 +1071,7 @@ static int isp_video_check_external_subdevs(struct isp_video *video, * processing might be possible but requires more testing. * * Stream start must be delayed until buffers are available at both the input - * and output. The pipeline must be started in the videobuf queue callback with + * and output. The pipeline must be started in the vb2 queue callback with * the buffers queue spinlock held. The modules subdev set stream operation must * not sleep. */ diff --git a/drivers/staging/media/hantro/Kconfig b/drivers/media/platform/verisilicon/Kconfig similarity index 91% rename from drivers/staging/media/hantro/Kconfig rename to drivers/media/platform/verisilicon/Kconfig index 0172a6822ec22876e6538072cdc18f562b25d4f4..e65b836b9d7888cdcf77423974223ceda492bb45 100644 --- a/drivers/staging/media/hantro/Kconfig +++ b/drivers/media/platform/verisilicon/Kconfig @@ -1,7 +1,11 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: GPL-2.0-only + +comment "Verisilicon media platform drivers" + config VIDEO_HANTRO tristate "Hantro VPU driver" depends on ARCH_MXC || ARCH_ROCKCHIP || ARCH_AT91 || ARCH_SUNXI || COMPILE_TEST + depends on V4L_MEM2MEM_DRIVERS depends on VIDEO_DEV select MEDIA_CONTROLLER select MEDIA_CONTROLLER_REQUEST_API diff --git a/drivers/staging/media/hantro/Makefile b/drivers/media/platform/verisilicon/Makefile similarity index 100% rename from drivers/staging/media/hantro/Makefile rename to drivers/media/platform/verisilicon/Makefile diff --git a/drivers/staging/media/hantro/hantro.h b/drivers/media/platform/verisilicon/hantro.h similarity index 100% rename from drivers/staging/media/hantro/hantro.h rename to drivers/media/platform/verisilicon/hantro.h diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c similarity index 100% rename from drivers/staging/media/hantro/hantro_drv.c rename to drivers/media/platform/verisilicon/hantro_drv.c diff --git a/drivers/staging/media/hantro/hantro_g1.c b/drivers/media/platform/verisilicon/hantro_g1.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g1.c rename to drivers/media/platform/verisilicon/hantro_g1.c diff --git a/drivers/staging/media/hantro/hantro_g1_h264_dec.c b/drivers/media/platform/verisilicon/hantro_g1_h264_dec.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g1_h264_dec.c rename to drivers/media/platform/verisilicon/hantro_g1_h264_dec.c diff --git a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c b/drivers/media/platform/verisilicon/hantro_g1_mpeg2_dec.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c rename to drivers/media/platform/verisilicon/hantro_g1_mpeg2_dec.c diff --git a/drivers/staging/media/hantro/hantro_g1_regs.h b/drivers/media/platform/verisilicon/hantro_g1_regs.h similarity index 100% rename from drivers/staging/media/hantro/hantro_g1_regs.h rename to drivers/media/platform/verisilicon/hantro_g1_regs.h diff --git a/drivers/staging/media/hantro/hantro_g1_vp8_dec.c b/drivers/media/platform/verisilicon/hantro_g1_vp8_dec.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g1_vp8_dec.c rename to drivers/media/platform/verisilicon/hantro_g1_vp8_dec.c diff --git a/drivers/staging/media/hantro/hantro_g2.c b/drivers/media/platform/verisilicon/hantro_g2.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g2.c rename to drivers/media/platform/verisilicon/hantro_g2.c diff --git a/drivers/staging/media/hantro/hantro_g2_hevc_dec.c b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g2_hevc_dec.c rename to drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c diff --git a/drivers/staging/media/hantro/hantro_g2_regs.h b/drivers/media/platform/verisilicon/hantro_g2_regs.h similarity index 100% rename from drivers/staging/media/hantro/hantro_g2_regs.h rename to drivers/media/platform/verisilicon/hantro_g2_regs.h diff --git a/drivers/staging/media/hantro/hantro_g2_vp9_dec.c b/drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c similarity index 100% rename from drivers/staging/media/hantro/hantro_g2_vp9_dec.c rename to drivers/media/platform/verisilicon/hantro_g2_vp9_dec.c diff --git a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c b/drivers/media/platform/verisilicon/hantro_h1_jpeg_enc.c similarity index 100% rename from drivers/staging/media/hantro/hantro_h1_jpeg_enc.c rename to drivers/media/platform/verisilicon/hantro_h1_jpeg_enc.c diff --git a/drivers/staging/media/hantro/hantro_h1_regs.h b/drivers/media/platform/verisilicon/hantro_h1_regs.h similarity index 100% rename from drivers/staging/media/hantro/hantro_h1_regs.h rename to drivers/media/platform/verisilicon/hantro_h1_regs.h diff --git a/drivers/staging/media/hantro/hantro_h264.c b/drivers/media/platform/verisilicon/hantro_h264.c similarity index 100% rename from drivers/staging/media/hantro/hantro_h264.c rename to drivers/media/platform/verisilicon/hantro_h264.c diff --git a/drivers/staging/media/hantro/hantro_hevc.c b/drivers/media/platform/verisilicon/hantro_hevc.c similarity index 100% rename from drivers/staging/media/hantro/hantro_hevc.c rename to drivers/media/platform/verisilicon/hantro_hevc.c diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/media/platform/verisilicon/hantro_hw.h similarity index 100% rename from drivers/staging/media/hantro/hantro_hw.h rename to drivers/media/platform/verisilicon/hantro_hw.h diff --git a/drivers/staging/media/hantro/hantro_jpeg.c b/drivers/media/platform/verisilicon/hantro_jpeg.c similarity index 100% rename from drivers/staging/media/hantro/hantro_jpeg.c rename to drivers/media/platform/verisilicon/hantro_jpeg.c diff --git a/drivers/staging/media/hantro/hantro_jpeg.h b/drivers/media/platform/verisilicon/hantro_jpeg.h similarity index 100% rename from drivers/staging/media/hantro/hantro_jpeg.h rename to drivers/media/platform/verisilicon/hantro_jpeg.h diff --git a/drivers/staging/media/hantro/hantro_mpeg2.c b/drivers/media/platform/verisilicon/hantro_mpeg2.c similarity index 100% rename from drivers/staging/media/hantro/hantro_mpeg2.c rename to drivers/media/platform/verisilicon/hantro_mpeg2.c diff --git a/drivers/staging/media/hantro/hantro_postproc.c b/drivers/media/platform/verisilicon/hantro_postproc.c similarity index 100% rename from drivers/staging/media/hantro/hantro_postproc.c rename to drivers/media/platform/verisilicon/hantro_postproc.c diff --git a/drivers/staging/media/hantro/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c similarity index 100% rename from drivers/staging/media/hantro/hantro_v4l2.c rename to drivers/media/platform/verisilicon/hantro_v4l2.c diff --git a/drivers/staging/media/hantro/hantro_v4l2.h b/drivers/media/platform/verisilicon/hantro_v4l2.h similarity index 100% rename from drivers/staging/media/hantro/hantro_v4l2.h rename to drivers/media/platform/verisilicon/hantro_v4l2.h diff --git a/drivers/staging/media/hantro/hantro_vp8.c b/drivers/media/platform/verisilicon/hantro_vp8.c similarity index 100% rename from drivers/staging/media/hantro/hantro_vp8.c rename to drivers/media/platform/verisilicon/hantro_vp8.c diff --git a/drivers/staging/media/hantro/hantro_vp9.c b/drivers/media/platform/verisilicon/hantro_vp9.c similarity index 100% rename from drivers/staging/media/hantro/hantro_vp9.c rename to drivers/media/platform/verisilicon/hantro_vp9.c diff --git a/drivers/staging/media/hantro/hantro_vp9.h b/drivers/media/platform/verisilicon/hantro_vp9.h similarity index 100% rename from drivers/staging/media/hantro/hantro_vp9.h rename to drivers/media/platform/verisilicon/hantro_vp9.h diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/media/platform/verisilicon/imx8m_vpu_hw.c similarity index 100% rename from drivers/staging/media/hantro/imx8m_vpu_hw.c rename to drivers/media/platform/verisilicon/imx8m_vpu_hw.c diff --git a/drivers/staging/media/hantro/rockchip_vpu2_hw_h264_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu2_hw_h264_dec.c similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu2_hw_h264_dec.c rename to drivers/media/platform/verisilicon/rockchip_vpu2_hw_h264_dec.c diff --git a/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c b/drivers/media/platform/verisilicon/rockchip_vpu2_hw_jpeg_enc.c similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c rename to drivers/media/platform/verisilicon/rockchip_vpu2_hw_jpeg_enc.c diff --git a/drivers/staging/media/hantro/rockchip_vpu2_hw_mpeg2_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu2_hw_mpeg2_dec.c similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu2_hw_mpeg2_dec.c rename to drivers/media/platform/verisilicon/rockchip_vpu2_hw_mpeg2_dec.c diff --git a/drivers/staging/media/hantro/rockchip_vpu2_hw_vp8_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu2_hw_vp8_dec.c similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu2_hw_vp8_dec.c rename to drivers/media/platform/verisilicon/rockchip_vpu2_hw_vp8_dec.c diff --git a/drivers/staging/media/hantro/rockchip_vpu2_regs.h b/drivers/media/platform/verisilicon/rockchip_vpu2_regs.h similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu2_regs.h rename to drivers/media/platform/verisilicon/rockchip_vpu2_regs.h diff --git a/drivers/staging/media/hantro/rockchip_vpu_hw.c b/drivers/media/platform/verisilicon/rockchip_vpu_hw.c similarity index 100% rename from drivers/staging/media/hantro/rockchip_vpu_hw.c rename to drivers/media/platform/verisilicon/rockchip_vpu_hw.c diff --git a/drivers/staging/media/hantro/sama5d4_vdec_hw.c b/drivers/media/platform/verisilicon/sama5d4_vdec_hw.c similarity index 100% rename from drivers/staging/media/hantro/sama5d4_vdec_hw.c rename to drivers/media/platform/verisilicon/sama5d4_vdec_hw.c diff --git a/drivers/staging/media/hantro/sunxi_vpu_hw.c b/drivers/media/platform/verisilicon/sunxi_vpu_hw.c similarity index 100% rename from drivers/staging/media/hantro/sunxi_vpu_hw.c rename to drivers/media/platform/verisilicon/sunxi_vpu_hw.c diff --git a/drivers/media/platform/xilinx/xilinx-csi2rxss.c b/drivers/media/platform/xilinx/xilinx-csi2rxss.c index cf8e892c47f0e86723c5518924a9ed0b50e9b0c0..29b53febc2e7a01e3a312ce16a4ec33ce4e899c1 100644 --- a/drivers/media/platform/xilinx/xilinx-csi2rxss.c +++ b/drivers/media/platform/xilinx/xilinx-csi2rxss.c @@ -188,6 +188,7 @@ static const u32 xcsi2dt_mbus_lut[][2] = { { MIPI_CSI2_DT_RAW12, MEDIA_BUS_FMT_SBGGR12_1X12 }, { MIPI_CSI2_DT_RAW12, MEDIA_BUS_FMT_SGBRG12_1X12 }, { MIPI_CSI2_DT_RAW12, MEDIA_BUS_FMT_SGRBG12_1X12 }, + { MIPI_CSI2_DT_RAW12, MEDIA_BUS_FMT_Y12_1X12 }, { MIPI_CSI2_DT_RAW16, MEDIA_BUS_FMT_SRGGB16_1X16 }, { MIPI_CSI2_DT_RAW16, MEDIA_BUS_FMT_SBGGR16_1X16 }, { MIPI_CSI2_DT_RAW16, MEDIA_BUS_FMT_SGBRG16_1X16 }, diff --git a/drivers/media/platform/xilinx/xilinx-vip.c b/drivers/media/platform/xilinx/xilinx-vip.c index a0073122798fe8cd189ca6667243954eae22d0a1..5b214bf7f93a474ea12e7aac4cf1e7d904861679 100644 --- a/drivers/media/platform/xilinx/xilinx-vip.c +++ b/drivers/media/platform/xilinx/xilinx-vip.c @@ -40,6 +40,8 @@ static const struct xvip_video_format xvip_video_formats[] = { 1, V4L2_PIX_FMT_SGBRG8 }, { XVIP_VF_MONO_SENSOR, 8, "bggr", MEDIA_BUS_FMT_SBGGR8_1X8, 1, V4L2_PIX_FMT_SBGGR8 }, + { XVIP_VF_MONO_SENSOR, 12, "mono", MEDIA_BUS_FMT_Y12_1X12, + 2, V4L2_PIX_FMT_Y12 }, }; /** diff --git a/drivers/media/platform/xilinx/xilinx-vipp.c b/drivers/media/platform/xilinx/xilinx-vipp.c index f34f8b077e03cb4fb72ac9d9ece71cbd73b336c3..0a16c218a50a7cba9477bdcbbdb395fec1152b93 100644 --- a/drivers/media/platform/xilinx/xilinx-vipp.c +++ b/drivers/media/platform/xilinx/xilinx-vipp.c @@ -471,7 +471,7 @@ static int xvip_graph_dma_init(struct xvip_composite_device *xdev) { struct device_node *ports; struct device_node *port; - int ret; + int ret = 0; ports = of_get_child_by_name(xdev->dev->of_node, "ports"); if (ports == NULL) { @@ -481,13 +481,14 @@ static int xvip_graph_dma_init(struct xvip_composite_device *xdev) for_each_child_of_node(ports, port) { ret = xvip_graph_dma_init_one(xdev, port); - if (ret < 0) { + if (ret) { of_node_put(port); - return ret; + break; } } - return 0; + of_node_put(ports); + return ret; } static void xvip_graph_cleanup(struct xvip_composite_device *xdev) diff --git a/drivers/media/radio/radio-tea5764.c b/drivers/media/radio/radio-tea5764.c index 877a24e5c577d678fa844b1eb9e48255fea7ea26..abda40e816121f32e6776eb541e75290016084b9 100644 --- a/drivers/media/radio/radio-tea5764.c +++ b/drivers/media/radio/radio-tea5764.c @@ -487,7 +487,7 @@ errfr: return ret; } -static int tea5764_i2c_remove(struct i2c_client *client) +static void tea5764_i2c_remove(struct i2c_client *client) { struct tea5764_device *radio = i2c_get_clientdata(client); @@ -499,7 +499,6 @@ static int tea5764_i2c_remove(struct i2c_client *client) v4l2_device_unregister(&radio->v4l2_dev); kfree(radio); } - return 0; } /* I2C subsystem interface */ diff --git a/drivers/media/radio/saa7706h.c b/drivers/media/radio/saa7706h.c index adb66f869dd28e98a04e0d5652701a9080299ccf..f9e990a9c3ef09335d20cfe47d9d155d877b3363 100644 --- a/drivers/media/radio/saa7706h.c +++ b/drivers/media/radio/saa7706h.c @@ -384,7 +384,7 @@ err: return err; } -static int saa7706h_remove(struct i2c_client *client) +static void saa7706h_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct saa7706h_state *state = to_state(sd); @@ -393,7 +393,6 @@ static int saa7706h_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(&state->hdl); kfree(to_state(sd)); - return 0; } static const struct i2c_device_id saa7706h_id[] = { diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c index 59b3d77e282dbf2da62b2ea42610f8e463b95679..a6ad926c2b4e87e53d7589d3612e90459e1ce5b0 100644 --- a/drivers/media/radio/si470x/radio-si470x-i2c.c +++ b/drivers/media/radio/si470x/radio-si470x-i2c.c @@ -461,7 +461,7 @@ err_initial: /* * si470x_i2c_remove - remove the device */ -static int si470x_i2c_remove(struct i2c_client *client) +static void si470x_i2c_remove(struct i2c_client *client) { struct si470x_device *radio = i2c_get_clientdata(client); @@ -472,7 +472,6 @@ static int si470x_i2c_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&radio->hdl); v4l2_device_unregister(&radio->v4l2_dev); - return 0; } diff --git a/drivers/media/radio/si4713/si4713.c b/drivers/media/radio/si4713/si4713.c index adbf43ff6a21ed1c00a2f3fb5112f7d2e0d24202..2aec642133a1e6d721dc12f6f7a04467eadc89b7 100644 --- a/drivers/media/radio/si4713/si4713.c +++ b/drivers/media/radio/si4713/si4713.c @@ -1623,7 +1623,7 @@ exit: } /* si4713_remove - remove the device */ -static int si4713_remove(struct i2c_client *client) +static void si4713_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct si4713_device *sdev = to_si4713_device(sd); @@ -1635,8 +1635,6 @@ static int si4713_remove(struct i2c_client *client) v4l2_device_unregister_subdev(sd); v4l2_ctrl_handler_free(sd->ctrl_handler); - - return 0; } /* si4713_i2c_driver - i2c driver interface */ diff --git a/drivers/media/radio/tef6862.c b/drivers/media/radio/tef6862.c index d8810492db4f58b34f129a4aad470d3d7e7e18dc..7b0870a9785b0de5758d4f28822fa37985ec5b70 100644 --- a/drivers/media/radio/tef6862.c +++ b/drivers/media/radio/tef6862.c @@ -165,13 +165,12 @@ static int tef6862_probe(struct i2c_client *client, return 0; } -static int tef6862_remove(struct i2c_client *client) +static void tef6862_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); v4l2_device_unregister_subdev(sd); kfree(to_state(sd)); - return 0; } static const struct i2c_device_id tef6862_id[] = { diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c index 0834d5f866fd8870f3ba11d265db8e95849130df..39d2b03e26317d7e25b7995734115359a766fd9a 100644 --- a/drivers/media/rc/mceusb.c +++ b/drivers/media/rc/mceusb.c @@ -1416,42 +1416,37 @@ static void mceusb_gen1_init(struct mceusb_dev *ir) { int ret; struct device *dev = ir->dev; - char *data; - - data = kzalloc(USB_CTRL_MSG_SZ, GFP_KERNEL); - if (!data) { - dev_err(dev, "%s: memory allocation failed!", __func__); - return; - } + char data[USB_CTRL_MSG_SZ]; /* * This is a strange one. Windows issues a set address to the device * on the receive control pipe and expect a certain value pair back */ - ret = usb_control_msg(ir->usbdev, usb_rcvctrlpipe(ir->usbdev, 0), - USB_REQ_SET_ADDRESS, USB_TYPE_VENDOR, 0, 0, - data, USB_CTRL_MSG_SZ, 3000); + ret = usb_control_msg_recv(ir->usbdev, 0, USB_REQ_SET_ADDRESS, + USB_DIR_IN | USB_TYPE_VENDOR, + 0, 0, data, USB_CTRL_MSG_SZ, 3000, + GFP_KERNEL); dev_dbg(dev, "set address - ret = %d", ret); dev_dbg(dev, "set address - data[0] = %d, data[1] = %d", data[0], data[1]); /* set feature: bit rate 38400 bps */ - ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0), - USB_REQ_SET_FEATURE, USB_TYPE_VENDOR, - 0xc04e, 0x0000, NULL, 0, 3000); + ret = usb_control_msg_send(ir->usbdev, 0, + USB_REQ_SET_FEATURE, USB_TYPE_VENDOR, + 0xc04e, 0x0000, NULL, 0, 3000, GFP_KERNEL); dev_dbg(dev, "set feature - ret = %d", ret); /* bRequest 4: set char length to 8 bits */ - ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0), - 4, USB_TYPE_VENDOR, - 0x0808, 0x0000, NULL, 0, 3000); + ret = usb_control_msg_send(ir->usbdev, 0, + 4, USB_TYPE_VENDOR, + 0x0808, 0x0000, NULL, 0, 3000, GFP_KERNEL); dev_dbg(dev, "set char length - retB = %d", ret); /* bRequest 2: set handshaking to use DTR/DSR */ - ret = usb_control_msg(ir->usbdev, usb_sndctrlpipe(ir->usbdev, 0), - 2, USB_TYPE_VENDOR, - 0x0000, 0x0100, NULL, 0, 3000); + ret = usb_control_msg_send(ir->usbdev, 0, + 2, USB_TYPE_VENDOR, + 0x0000, 0x0100, NULL, 0, 3000, GFP_KERNEL); dev_dbg(dev, "set handshake - retC = %d", ret); /* device resume */ @@ -1459,8 +1454,6 @@ static void mceusb_gen1_init(struct mceusb_dev *ir) /* get hw/sw revision? */ mce_command_out(ir, GET_REVISION, sizeof(GET_REVISION)); - - kfree(data); } static void mceusb_gen2_init(struct mceusb_dev *ir) diff --git a/drivers/media/test-drivers/vidtv/vidtv_demod.c b/drivers/media/test-drivers/vidtv/vidtv_demod.c index b7823d97b30d5e1f3d2afcd7bff5a7c8bc3c3149..e7959ab1add8d47ab7cde613a5300a8d17ad04e2 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_demod.c +++ b/drivers/media/test-drivers/vidtv/vidtv_demod.c @@ -438,13 +438,11 @@ static int vidtv_demod_i2c_probe(struct i2c_client *client, return 0; } -static int vidtv_demod_i2c_remove(struct i2c_client *client) +static void vidtv_demod_i2c_remove(struct i2c_client *client) { struct vidtv_demod_state *state = i2c_get_clientdata(client); kfree(state); - - return 0; } static struct i2c_driver vidtv_demod_i2c_driver = { diff --git a/drivers/media/test-drivers/vidtv/vidtv_tuner.c b/drivers/media/test-drivers/vidtv/vidtv_tuner.c index 14b6bc902ee13e2a54f88fcc41a76ec4922291de..aabc97ed736b81462667428d78f6f8dbe7bf18c6 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_tuner.c +++ b/drivers/media/test-drivers/vidtv/vidtv_tuner.c @@ -414,13 +414,11 @@ static int vidtv_tuner_i2c_probe(struct i2c_client *client, return 0; } -static int vidtv_tuner_i2c_remove(struct i2c_client *client) +static void vidtv_tuner_i2c_remove(struct i2c_client *client) { struct vidtv_tuner_dev *tuner_dev = i2c_get_clientdata(client); kfree(tuner_dev); - - return 0; } static struct i2c_driver vidtv_tuner_i2c_driver = { diff --git a/drivers/media/test-drivers/vim2m.c b/drivers/media/test-drivers/vim2m.c index 47575490e74af8c692326fedc4a8d548b5b26a83..7964426bf2f77674aee8601b4c136dc918b79a02 100644 --- a/drivers/media/test-drivers/vim2m.c +++ b/drivers/media/test-drivers/vim2m.c @@ -2,7 +2,7 @@ /* * A virtual v4l2-mem2mem example device. * - * This is a virtual device driver for testing mem-to-mem videobuf framework. + * This is a virtual device driver for testing mem-to-mem vb2 framework. * It simulates a device that uses memory buffers for both source and * destination, processes the data and issues an "irq" (simulated by a delayed * workqueue). diff --git a/drivers/media/test-drivers/vivid/vivid-core.h b/drivers/media/test-drivers/vivid/vivid-core.h index 176b72cb143b6fc83bbdca7a580241529a7986c5..bfcfb3515901397b62d26cca5e57092586c9784f 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.h +++ b/drivers/media/test-drivers/vivid/vivid-core.h @@ -35,7 +35,9 @@ #define MAX_HEIGHT 2160 /* The minimum image width/height */ #define MIN_WIDTH 16 -#define MIN_HEIGHT 16 +#define MIN_HEIGHT MIN_WIDTH +/* Pixel Array control divider */ +#define PIXEL_ARRAY_DIV MIN_WIDTH /* The data_offset of plane 0 for the multiplanar formats */ #define PLANE0_DATA_OFFSET 128 @@ -227,6 +229,7 @@ struct vivid_dev { struct v4l2_ctrl *bitmask; struct v4l2_ctrl *int_menu; struct v4l2_ctrl *ro_int32; + struct v4l2_ctrl *pixel_array; struct v4l2_ctrl *test_pattern; struct v4l2_ctrl *colorspace; struct v4l2_ctrl *rgb_range_cap; diff --git a/drivers/media/test-drivers/vivid/vivid-ctrls.c b/drivers/media/test-drivers/vivid/vivid-ctrls.c index a78d676575bcae619d9ad098609180efee77dbea..92b1a75984706669b2d084855403d199db2b27b9 100644 --- a/drivers/media/test-drivers/vivid/vivid-ctrls.c +++ b/drivers/media/test-drivers/vivid/vivid-ctrls.c @@ -35,6 +35,7 @@ #define VIVID_CID_AREA (VIVID_CID_CUSTOM_BASE + 11) #define VIVID_CID_RO_INTEGER (VIVID_CID_CUSTOM_BASE + 12) #define VIVID_CID_U32_DYN_ARRAY (VIVID_CID_CUSTOM_BASE + 13) +#define VIVID_CID_U8_PIXEL_ARRAY (VIVID_CID_CUSTOM_BASE + 14) #define VIVID_CID_VIVID_BASE (0x00f00000 | 0xf000) #define VIVID_CID_VIVID_CLASS (0x00f00000 | 1) @@ -228,6 +229,18 @@ static const struct v4l2_ctrl_config vivid_ctrl_u8_4d_array = { .dims = { 2, 3, 4, 5 }, }; +static const struct v4l2_ctrl_config vivid_ctrl_u8_pixel_array = { + .ops = &vivid_user_gen_ctrl_ops, + .id = VIVID_CID_U8_PIXEL_ARRAY, + .name = "U8 Pixel Array", + .type = V4L2_CTRL_TYPE_U8, + .def = 0x80, + .min = 0x00, + .max = 0xff, + .step = 1, + .dims = { 640 / PIXEL_ARRAY_DIV, 360 / PIXEL_ARRAY_DIV }, +}; + static const char * const vivid_ctrl_menu_strings[] = { "Menu Item 0 (Skipped)", "Menu Item 1", @@ -1642,6 +1655,7 @@ int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap, v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u32_dyn_array, NULL); v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u16_matrix, NULL); v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u8_4d_array, NULL); + dev->pixel_array = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u8_pixel_array, NULL); if (dev->has_vid_cap) { /* Image Processing Controls */ diff --git a/drivers/media/test-drivers/vivid/vivid-radio-rx.c b/drivers/media/test-drivers/vivid/vivid-radio-rx.c index 232cab508f48b0e9a02934dca6196933b73657a5..8bd09589fb153875daaca7f914a2b75711723637 100644 --- a/drivers/media/test-drivers/vivid/vivid-radio-rx.c +++ b/drivers/media/test-drivers/vivid/vivid-radio-rx.c @@ -104,8 +104,8 @@ retry: break; case 2: rds.block |= V4L2_RDS_BLOCK_ERROR; - rds.lsb = prandom_u32_max(256); - rds.msb = prandom_u32_max(256); + rds.lsb = get_random_u8(); + rds.msb = get_random_u8(); break; case 3: /* Skip block altogether */ if (i) diff --git a/drivers/media/test-drivers/vivid/vivid-touch-cap.c b/drivers/media/test-drivers/vivid/vivid-touch-cap.c index 64e3e4cb30c20ce893c84933d39dafddd2018df0..6cc32eb54f9d0d6ec212837d3e72fa3344994c20 100644 --- a/drivers/media/test-drivers/vivid/vivid-touch-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-touch-cap.c @@ -210,7 +210,7 @@ static void vivid_fill_buff_noise(__s16 *tch_buf, int size) /* Fill 10% of the values within range -3 and 3, zero the others */ for (i = 0; i < size; i++) { - unsigned int rand = get_random_int(); + unsigned int rand = get_random_u32(); if (rand % 10) tch_buf[i] = 0; @@ -221,7 +221,7 @@ static void vivid_fill_buff_noise(__s16 *tch_buf, int size) static inline int get_random_pressure(void) { - return get_random_int() % VIVID_PRESSURE_LIMIT; + return prandom_u32_max(VIVID_PRESSURE_LIMIT); } static void vivid_tch_buf_set(struct v4l2_pix_format *f, @@ -272,7 +272,7 @@ void vivid_fillbuff_tch(struct vivid_dev *dev, struct vivid_buffer *buf) return; if (test_pat_idx == 0) - dev->tch_pat_random = get_random_int(); + dev->tch_pat_random = get_random_u32(); rand = dev->tch_pat_random; switch (test_pattern) { diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index b9caa4b26209ef1951d1ef5024e016c89eeb07b9..86b158eeb2d8133ae36bd574145424ae50e35f03 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -381,6 +381,7 @@ static enum tpg_pixel_aspect vivid_get_pixel_aspect(const struct vivid_dev *dev) void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) { struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt; + u32 dims[V4L2_CTRL_MAX_DIMS] = {}; unsigned size; u64 pixelclock; @@ -459,6 +460,9 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) tpg_s_video_aspect(&dev->tpg, vivid_get_video_aspect(dev)); tpg_s_pixel_aspect(&dev->tpg, vivid_get_pixel_aspect(dev)); tpg_update_mv_step(&dev->tpg); + dims[0] = roundup(dev->src_rect.width, PIXEL_ARRAY_DIV); + dims[1] = roundup(dev->src_rect.height, PIXEL_ARRAY_DIV); + v4l2_ctrl_modify_dimensions(dev->pixel_array, dims); } /* Map the field to something that is valid for the current input */ diff --git a/drivers/media/tuners/e4000.c b/drivers/media/tuners/e4000.c index a3a8d051dc6c2347ce150ea45cfb368d65d121b1..61ae884ea59a23828b51ac86e9eb4602eba3d4a3 100644 --- a/drivers/media/tuners/e4000.c +++ b/drivers/media/tuners/e4000.c @@ -706,7 +706,7 @@ err: return ret; } -static int e4000_remove(struct i2c_client *client) +static void e4000_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct e4000_dev *dev = container_of(sd, struct e4000_dev, sd); @@ -717,8 +717,6 @@ static int e4000_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&dev->hdl); #endif kfree(dev); - - return 0; } static const struct i2c_device_id e4000_id_table[] = { diff --git a/drivers/media/tuners/fc2580.c b/drivers/media/tuners/fc2580.c index 1b5961bdf2d5f1e52de6300235e100011b988768..f30932e1a0f38a3fad3bb091daf218929bb527d1 100644 --- a/drivers/media/tuners/fc2580.c +++ b/drivers/media/tuners/fc2580.c @@ -588,7 +588,7 @@ err: return ret; } -static int fc2580_remove(struct i2c_client *client) +static void fc2580_remove(struct i2c_client *client) { struct fc2580_dev *dev = i2c_get_clientdata(client); @@ -598,7 +598,6 @@ static int fc2580_remove(struct i2c_client *client) v4l2_ctrl_handler_free(&dev->hdl); #endif kfree(dev); - return 0; } static const struct i2c_device_id fc2580_id_table[] = { diff --git a/drivers/media/tuners/m88rs6000t.c b/drivers/media/tuners/m88rs6000t.c index 8647c50b66e50d85813443d9ec230c9d19fe38fc..e32e3e9daa15c8632524af53c761165cb7aaea94 100644 --- a/drivers/media/tuners/m88rs6000t.c +++ b/drivers/media/tuners/m88rs6000t.c @@ -697,7 +697,7 @@ err: return ret; } -static int m88rs6000t_remove(struct i2c_client *client) +static void m88rs6000t_remove(struct i2c_client *client) { struct m88rs6000t_dev *dev = i2c_get_clientdata(client); struct dvb_frontend *fe = dev->cfg.fe; @@ -707,8 +707,6 @@ static int m88rs6000t_remove(struct i2c_client *client) memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops)); fe->tuner_priv = NULL; kfree(dev); - - return 0; } static const struct i2c_device_id m88rs6000t_id[] = { diff --git a/drivers/media/tuners/mt2060.c b/drivers/media/tuners/mt2060.c index 204e6186bf7156196507850b9b466df515c374af..322c806228a5a4ed371147c64ed7bf14a1e078ca 100644 --- a/drivers/media/tuners/mt2060.c +++ b/drivers/media/tuners/mt2060.c @@ -509,11 +509,9 @@ err: return ret; } -static int mt2060_remove(struct i2c_client *client) +static void mt2060_remove(struct i2c_client *client) { dev_dbg(&client->dev, "\n"); - - return 0; } static const struct i2c_device_id mt2060_id_table[] = { diff --git a/drivers/media/tuners/mxl301rf.c b/drivers/media/tuners/mxl301rf.c index c628435a1b067ee035b294d7f110cd58b77a8c9f..6422056185a9aa67d893c8345ac8cfc48a661eac 100644 --- a/drivers/media/tuners/mxl301rf.c +++ b/drivers/media/tuners/mxl301rf.c @@ -307,14 +307,13 @@ static int mxl301rf_probe(struct i2c_client *client, return 0; } -static int mxl301rf_remove(struct i2c_client *client) +static void mxl301rf_remove(struct i2c_client *client) { struct mxl301rf_state *state; state = cfg_to_state(i2c_get_clientdata(client)); state->cfg.fe->tuner_priv = NULL; kfree(state); - return 0; } diff --git a/drivers/media/tuners/qm1d1b0004.c b/drivers/media/tuners/qm1d1b0004.c index 008ad870c00f4bcf8331e03f12aaf3c1c7d7938c..9cba0893207cc6db862f9d99e22ed4da4da75f01 100644 --- a/drivers/media/tuners/qm1d1b0004.c +++ b/drivers/media/tuners/qm1d1b0004.c @@ -232,14 +232,13 @@ err_mem: return ret; } -static int qm1d1b0004_remove(struct i2c_client *client) +static void qm1d1b0004_remove(struct i2c_client *client) { struct dvb_frontend *fe; fe = i2c_get_clientdata(client); kfree(fe->tuner_priv); fe->tuner_priv = NULL; - return 0; } diff --git a/drivers/media/tuners/qm1d1c0042.c b/drivers/media/tuners/qm1d1c0042.c index 53aa2558f71e18b448adcd24f95e40993e793c2c..2d60bf501fb5c06a44ffe707bb4c87e54c78c360 100644 --- a/drivers/media/tuners/qm1d1c0042.c +++ b/drivers/media/tuners/qm1d1c0042.c @@ -424,14 +424,13 @@ static int qm1d1c0042_probe(struct i2c_client *client, return 0; } -static int qm1d1c0042_remove(struct i2c_client *client) +static void qm1d1c0042_remove(struct i2c_client *client) { struct qm1d1c0042_state *state; state = cfg_to_state(i2c_get_clientdata(client)); state->cfg.fe->tuner_priv = NULL; kfree(state); - return 0; } diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c index 0de587b412d4e7deaa715eeca94ab17cd273f6dd..476b32c04c203886d360c4242164a6b85fd6d94e 100644 --- a/drivers/media/tuners/si2157.c +++ b/drivers/media/tuners/si2157.c @@ -951,7 +951,7 @@ err: return ret; } -static int si2157_remove(struct i2c_client *client) +static void si2157_remove(struct i2c_client *client) { struct si2157_dev *dev = i2c_get_clientdata(client); struct dvb_frontend *fe = dev->fe; @@ -969,8 +969,6 @@ static int si2157_remove(struct i2c_client *client) memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops)); fe->tuner_priv = NULL; kfree(dev); - - return 0; } /* diff --git a/drivers/media/tuners/tda18212.c b/drivers/media/tuners/tda18212.c index bf48f1cd83d23f559c8002179f2eba1c909251f6..eb97711c9c685882c0bcc62d504fa04631f28cce 100644 --- a/drivers/media/tuners/tda18212.c +++ b/drivers/media/tuners/tda18212.c @@ -242,7 +242,7 @@ err: return ret; } -static int tda18212_remove(struct i2c_client *client) +static void tda18212_remove(struct i2c_client *client) { struct tda18212_dev *dev = i2c_get_clientdata(client); struct dvb_frontend *fe = dev->cfg.fe; @@ -252,8 +252,6 @@ static int tda18212_remove(struct i2c_client *client) memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops)); fe->tuner_priv = NULL; kfree(dev); - - return 0; } static const struct i2c_device_id tda18212_id[] = { diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c index 8a5781b966ee81fb5c2105cea650916eb7de3731..e404a5afad4c9716201d4cc0b5c150644fd8c28c 100644 --- a/drivers/media/tuners/tda18250.c +++ b/drivers/media/tuners/tda18250.c @@ -856,7 +856,7 @@ err: return ret; } -static int tda18250_remove(struct i2c_client *client) +static void tda18250_remove(struct i2c_client *client) { struct tda18250_dev *dev = i2c_get_clientdata(client); struct dvb_frontend *fe = dev->fe; @@ -866,8 +866,6 @@ static int tda18250_remove(struct i2c_client *client) memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops)); fe->tuner_priv = NULL; kfree(dev); - - return 0; } static const struct i2c_device_id tda18250_id_table[] = { diff --git a/drivers/media/tuners/tua9001.c b/drivers/media/tuners/tua9001.c index af7d5ea1f77ee810f2acf9010ca15c04132cab2a..d141d000b8195322c3d36f9d28c67def17996de9 100644 --- a/drivers/media/tuners/tua9001.c +++ b/drivers/media/tuners/tua9001.c @@ -227,7 +227,7 @@ err: return ret; } -static int tua9001_remove(struct i2c_client *client) +static void tua9001_remove(struct i2c_client *client) { struct tua9001_dev *dev = i2c_get_clientdata(client); struct dvb_frontend *fe = dev->fe; @@ -243,7 +243,6 @@ static int tua9001_remove(struct i2c_client *client) dev_err(&client->dev, "Tuner disable failed (%pe)\n", ERR_PTR(ret)); } kfree(dev); - return 0; } static const struct i2c_device_id tua9001_id_table[] = { diff --git a/drivers/media/usb/Kconfig b/drivers/media/usb/Kconfig index af88e076638815791216a1aba9a901a25f9f090c..813171d25ac59a7201ecc486e49d189edbf9f555 100644 --- a/drivers/media/usb/Kconfig +++ b/drivers/media/usb/Kconfig @@ -13,13 +13,11 @@ if MEDIA_USB_SUPPORT if MEDIA_CAMERA_SUPPORT comment "Webcam devices" -source "drivers/media/usb/cpia2/Kconfig" source "drivers/media/usb/gspca/Kconfig" source "drivers/media/usb/pwc/Kconfig" source "drivers/media/usb/s2255/Kconfig" source "drivers/media/usb/usbtv/Kconfig" source "drivers/media/usb/uvc/Kconfig" -source "drivers/media/usb/zr364xx/Kconfig" endif @@ -38,7 +36,6 @@ if (MEDIA_ANALOG_TV_SUPPORT || MEDIA_DIGITAL_TV_SUPPORT) source "drivers/media/usb/au0828/Kconfig" source "drivers/media/usb/cx231xx/Kconfig" -source "drivers/media/usb/tm6000/Kconfig" endif diff --git a/drivers/media/usb/Makefile b/drivers/media/usb/Makefile index 25fa2015b1794553d84bf8331a536dab3f7ef740..6d171beea20d3973db1a46df5e33938e340f2026 100644 --- a/drivers/media/usb/Makefile +++ b/drivers/media/usb/Makefile @@ -12,7 +12,6 @@ obj-y += s2255/ obj-y += siano/ obj-y += ttusb-budget/ obj-y += ttusb-dec/ -obj-y += zr364xx/ # Please keep it alphabetically sorted by Kconfig name # (e. g. LC_ALL=C sort Makefile) @@ -24,12 +23,10 @@ obj-$(CONFIG_USB_MSI2500) += msi2500/ obj-$(CONFIG_USB_PWC) += pwc/ obj-$(CONFIG_USB_VIDEO_CLASS) += uvc/ obj-$(CONFIG_VIDEO_AU0828) += au0828/ -obj-$(CONFIG_VIDEO_CPIA2) += cpia2/ obj-$(CONFIG_VIDEO_CX231XX) += cx231xx/ obj-$(CONFIG_VIDEO_EM28XX) += em28xx/ obj-$(CONFIG_VIDEO_GO7007) += go7007/ obj-$(CONFIG_VIDEO_HDPVR) += hdpvr/ obj-$(CONFIG_VIDEO_PVRUSB2) += pvrusb2/ obj-$(CONFIG_VIDEO_STK1160) += stk1160/ -obj-$(CONFIG_VIDEO_TM6000) += tm6000/ obj-$(CONFIG_VIDEO_USBTV) += usbtv/ diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c index 240a7cc56777d74384ba5f354b3c78891abb4f07..462eb84235063a772f4b90343dd4ba74b9ab2eb8 100644 --- a/drivers/media/usb/airspy/airspy.c +++ b/drivers/media/usb/airspy/airspy.c @@ -294,7 +294,7 @@ static void airspy_urb_complete(struct urb *urb) if (unlikely(fbuf == NULL)) { s->vb_full++; dev_notice_ratelimited(s->dev, - "videobuf is full, %d packets dropped\n", + "video buffer is full, %d packets dropped\n", s->vb_full); goto skip; } @@ -1070,6 +1070,10 @@ static int airspy_probe(struct usb_interface *intf, ret); goto err_free_controls; } + + /* Free buf if success*/ + kfree(buf); + dev_info(s->dev, "Registered as %s\n", video_device_node_name(&s->vdev)); dev_notice(s->dev, "SDR API is still slightly experimental and functionality changes may follow\n"); diff --git a/drivers/media/usb/au0828/au0828-video.c b/drivers/media/usb/au0828/au0828-video.c index c0f118563c7d13b44f8cedc963c0b83cbcbf4a37..eb303e94ccebf3a05103fe501a489465df6e3376 100644 --- a/drivers/media/usb/au0828/au0828-video.c +++ b/drivers/media/usb/au0828/au0828-video.c @@ -384,7 +384,7 @@ static void au0828_copy_video(struct au0828_dev *dev, } /* - * video-buf generic routine to get the next available buffer + * generic routine to get the next available buffer */ static inline void get_next_buf(struct au0828_dmaqueue *dma_q, struct au0828_buffer **buf) @@ -459,7 +459,7 @@ static void au0828_copy_vbi(struct au0828_dev *dev, /* - * video-buf generic routine to get the next available VBI buffer + * generic routine to get the next available VBI buffer */ static inline void vbi_get_next_buf(struct au0828_dmaqueue *dma_q, struct au0828_buffer **buf) diff --git a/drivers/media/usb/b2c2/flexcop-usb.c b/drivers/media/usb/b2c2/flexcop-usb.c index 7835bb0f32fc30f9f9f56f9c4e9b1c1bc780daa0..790787f0eba840c77095943ec53ebd0f4428cf6d 100644 --- a/drivers/media/usb/b2c2/flexcop-usb.c +++ b/drivers/media/usb/b2c2/flexcop-usb.c @@ -425,12 +425,14 @@ static void flexcop_usb_transfer_exit(struct flexcop_usb *fc_usb) static int flexcop_usb_transfer_init(struct flexcop_usb *fc_usb) { - u16 frame_size = le16_to_cpu( - fc_usb->uintf->cur_altsetting->endpoint[0].desc.wMaxPacketSize); - int bufsize = B2C2_USB_NUM_ISO_URB * B2C2_USB_FRAMES_PER_ISO * - frame_size, i, j, ret; + struct usb_host_interface *alt = fc_usb->uintf->cur_altsetting; + u16 frame_size; + int bufsize, i, j, ret; int buffer_offset = 0; + frame_size = usb_endpoint_maxp(&alt->endpoint[0].desc); + bufsize = B2C2_USB_NUM_ISO_URB * B2C2_USB_FRAMES_PER_ISO * frame_size; + deb_ts("creating %d iso-urbs with %d frames each of %d bytes size = %d.\n", B2C2_USB_NUM_ISO_URB, B2C2_USB_FRAMES_PER_ISO, frame_size, bufsize); @@ -501,17 +503,21 @@ urb_error: static int flexcop_usb_init(struct flexcop_usb *fc_usb) { - /* use the alternate setting with the larges buffer */ - int ret = usb_set_interface(fc_usb->udev, 0, 1); + struct usb_host_interface *alt; + int ret; + /* use the alternate setting with the largest buffer */ + ret = usb_set_interface(fc_usb->udev, 0, 1); if (ret) { err("set interface failed."); return ret; } - if (fc_usb->uintf->cur_altsetting->desc.bNumEndpoints < 1) + alt = fc_usb->uintf->cur_altsetting; + + if (alt->desc.bNumEndpoints < 1) return -ENODEV; - if (!usb_endpoint_is_isoc_in(&fc_usb->uintf->cur_altsetting->endpoint[1].desc)) + if (!usb_endpoint_is_isoc_in(&alt->endpoint[0].desc)) return -ENODEV; switch (fc_usb->udev->speed) { diff --git a/drivers/media/usb/cx231xx/cx231xx-vbi.c b/drivers/media/usb/cx231xx/cx231xx-vbi.c index fdc8b7f7b0c10655f73e546feeb7f4fc2cc393d3..33431d9f54c2cf49363d0c528257a69055f08571 100644 --- a/drivers/media/usb/cx231xx/cx231xx-vbi.c +++ b/drivers/media/usb/cx231xx/cx231xx-vbi.c @@ -558,7 +558,7 @@ u32 cx231xx_copy_vbi_line(struct cx231xx *dev, struct cx231xx_dmaqueue *dma_q, } /* - * video-buf generic routine to get the next available buffer + * generic routine to get the next available buffer */ static inline void get_next_vbi_buf(struct cx231xx_dmaqueue *dma_q, struct cx231xx_buffer **buf) diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c index 425e470b0fd354aa7ea3888557f342b7df0639b5..e23b8ccd79d46a289d590cdcce01d9298501ca71 100644 --- a/drivers/media/usb/cx231xx/cx231xx-video.c +++ b/drivers/media/usb/cx231xx/cx231xx-video.c @@ -220,7 +220,7 @@ static inline void print_err_status(struct cx231xx *dev, int packet, int status) } /* - * video-buf generic routine to get the next available buffer + * generic routine to get the next available buffer */ static inline void get_next_buf(struct cx231xx_dmaqueue *dma_q, struct cx231xx_buffer **buf) diff --git a/drivers/media/usb/dvb-usb/technisat-usb2.c b/drivers/media/usb/dvb-usb/technisat-usb2.c index 9c77911fcad424fdd1ca111a99e0333d787dd31d..df90c6c5f3b96c8161bb22b856d39d234d5a5178 100644 --- a/drivers/media/usb/dvb-usb/technisat-usb2.c +++ b/drivers/media/usb/dvb-usb/technisat-usb2.c @@ -786,7 +786,7 @@ static void technisat_usb2_disconnect(struct usb_interface *intf) { struct dvb_usb_device *dev = usb_get_intfdata(intf); - /* work and stuff was only created when the device is is hot-state */ + /* work and stuff was only created when the device is hot-state */ if (dev != NULL) { struct technisat_usb2_state *state = dev->priv; if (state != NULL) diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c index 8181c0e6a25bbdd25cf70fd2e605559ccb868292..25e0620deff1db35d8f2cded1bc23b1194489ee9 100644 --- a/drivers/media/usb/em28xx/em28xx-video.c +++ b/drivers/media/usb/em28xx/em28xx-video.c @@ -440,7 +440,7 @@ static inline void finish_buffer(struct em28xx *dev, } /* - * Copy picture data from USB buffer to videobuf buffer + * Copy picture data from USB buffer to video buffer */ static void em28xx_copy_video(struct em28xx *dev, struct em28xx_buffer *buf, @@ -521,7 +521,7 @@ static void em28xx_copy_video(struct em28xx *dev, } /* - * Copy VBI data from USB buffer to videobuf buffer + * Copy VBI data from USB buffer to video buffer */ static void em28xx_copy_vbi(struct em28xx *dev, struct em28xx_buffer *buf, diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c index 1fa6f10ee157bb839d1c298a70c5027e1032950c..2f45188bf9d474829ee1d44ece09fa109e6ad612 100644 --- a/drivers/media/usb/go7007/s2250-board.c +++ b/drivers/media/usb/go7007/s2250-board.c @@ -601,7 +601,7 @@ fail: return err; } -static int s2250_remove(struct i2c_client *client) +static void s2250_remove(struct i2c_client *client) { struct s2250 *state = to_state(i2c_get_clientdata(client)); @@ -609,7 +609,6 @@ static int s2250_remove(struct i2c_client *client) v4l2_device_unregister_subdev(&state->sd); v4l2_ctrl_handler_free(&state->hdl); kfree(state); - return 0; } static const struct i2c_device_id s2250_id[] = { diff --git a/drivers/media/usb/gspca/finepix.c b/drivers/media/usb/gspca/finepix.c index 66c8e5122a0a93884637a460a534ad086199fdcf..bc6133b525e3b4a85c3c627d953ed73461ecc200 100644 --- a/drivers/media/usb/gspca/finepix.c +++ b/drivers/media/usb/gspca/finepix.c @@ -129,7 +129,7 @@ again: * for, then it's the end of the * frame. Sometimes the jpeg is not complete, * but there's nothing we can do. We also end - * here if the the jpeg ends right at the end + * here if the jpeg ends right at the end * of the frame. */ gspca_frame_add(gspca_dev, LAST_PACKET, data, len); diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c index 71de6b4c4e4ce75f2c95cbf3fd15bcea1b3b5247..5a1f2698efb7b10ac8cbe6813eb38fccab0003d2 100644 --- a/drivers/media/usb/msi2500/msi2500.c +++ b/drivers/media/usb/msi2500/msi2500.c @@ -411,7 +411,7 @@ static void msi2500_isoc_handler(struct urb *urb) if (unlikely(fbuf == NULL)) { dev->vb_full++; dev_dbg_ratelimited(dev->dev, - "videobuf is full, %d packets dropped\n", + "video buffer is full, %d packets dropped\n", dev->vb_full); continue; } diff --git a/drivers/media/usb/pvrusb2/pvrusb2-dvb.c b/drivers/media/usb/pvrusb2/pvrusb2-dvb.c index 6954584526a326f43572332a30a6b37d9066c0c8..26811efe0fb58b32dbfe05c63cb1380e3704aa5c 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-dvb.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-dvb.c @@ -80,7 +80,7 @@ static int pvr2_dvb_feed_func(struct pvr2_dvb_adapter *adap) static int pvr2_dvb_feed_thread(void *data) { int stat = pvr2_dvb_feed_func(data); - /* from videobuf-dvb.c: */ + while (!kthread_should_stop()) { set_current_state(TASK_INTERRUPTIBLE); schedule(); diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c index 8c208db9600b46aeaee44d0994e9e8c08d6b9b35..c95a2229f4fa9f44dfe580520f482ec5684d8f7d 100644 --- a/drivers/media/usb/uvc/uvc_ctrl.c +++ b/drivers/media/usb/uvc/uvc_ctrl.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "uvcvideo.h" @@ -985,36 +986,56 @@ static s32 __uvc_ctrl_get_value(struct uvc_control_mapping *mapping, return value; } -static int __uvc_ctrl_get(struct uvc_video_chain *chain, - struct uvc_control *ctrl, struct uvc_control_mapping *mapping, - s32 *value) +static int __uvc_ctrl_load_cur(struct uvc_video_chain *chain, + struct uvc_control *ctrl) { + u8 *data; int ret; - if ((ctrl->info.flags & UVC_CTRL_FLAG_GET_CUR) == 0) - return -EACCES; + if (ctrl->loaded) + return 0; - if (!ctrl->loaded) { - if (ctrl->entity->get_cur) { - ret = ctrl->entity->get_cur(chain->dev, - ctrl->entity, - ctrl->info.selector, - uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), - ctrl->info.size); - } else { - ret = uvc_query_ctrl(chain->dev, UVC_GET_CUR, - ctrl->entity->id, - chain->dev->intfnum, - ctrl->info.selector, - uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), - ctrl->info.size); - } - if (ret < 0) - return ret; + data = uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT); + if ((ctrl->info.flags & UVC_CTRL_FLAG_GET_CUR) == 0) { + memset(data, 0, ctrl->info.size); ctrl->loaded = 1; + + return 0; } + if (ctrl->entity->get_cur) + ret = ctrl->entity->get_cur(chain->dev, ctrl->entity, + ctrl->info.selector, data, + ctrl->info.size); + else + ret = uvc_query_ctrl(chain->dev, UVC_GET_CUR, + ctrl->entity->id, chain->dev->intfnum, + ctrl->info.selector, data, + ctrl->info.size); + + if (ret < 0) + return ret; + + ctrl->loaded = 1; + + return ret; +} + +static int __uvc_ctrl_get(struct uvc_video_chain *chain, + struct uvc_control *ctrl, + struct uvc_control_mapping *mapping, + s32 *value) +{ + int ret; + + if ((ctrl->info.flags & UVC_CTRL_FLAG_GET_CUR) == 0) + return -EACCES; + + ret = __uvc_ctrl_load_cur(chain, ctrl); + if (ret < 0) + return ret; + *value = __uvc_ctrl_get_value(mapping, uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT)); @@ -1810,21 +1831,10 @@ int uvc_ctrl_set(struct uvc_fh *handle, * needs to be loaded from the device to perform the read-modify-write * operation. */ - if (!ctrl->loaded && (ctrl->info.size * 8) != mapping->size) { - if ((ctrl->info.flags & UVC_CTRL_FLAG_GET_CUR) == 0) { - memset(uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), - 0, ctrl->info.size); - } else { - ret = uvc_query_ctrl(chain->dev, UVC_GET_CUR, - ctrl->entity->id, chain->dev->intfnum, - ctrl->info.selector, - uvc_ctrl_data(ctrl, UVC_CTRL_DATA_CURRENT), - ctrl->info.size); - if (ret < 0) - return ret; - } - - ctrl->loaded = 1; + if ((ctrl->info.size * 8) != mapping->size) { + ret = __uvc_ctrl_load_cur(chain, ctrl); + if (ret < 0) + return ret; } /* Backup the current value in case we need to rollback later. */ @@ -2411,10 +2421,9 @@ static void uvc_ctrl_prune_entity(struct uvc_device *dev, static void uvc_ctrl_init_ctrl(struct uvc_video_chain *chain, struct uvc_control *ctrl) { - const struct uvc_control_info *info = uvc_ctrls; - const struct uvc_control_info *iend = info + ARRAY_SIZE(uvc_ctrls); - const struct uvc_control_mapping *mapping; - const struct uvc_control_mapping *mend; + const struct uvc_control_mapping *mappings; + unsigned int num_mappings; + unsigned int i; /* * XU controls initialization requires querying the device for control @@ -2425,7 +2434,9 @@ static void uvc_ctrl_init_ctrl(struct uvc_video_chain *chain, if (UVC_ENTITY_TYPE(ctrl->entity) == UVC_VC_EXTENSION_UNIT) return; - for (; info < iend; ++info) { + for (i = 0; i < ARRAY_SIZE(uvc_ctrls); ++i) { + const struct uvc_control_info *info = &uvc_ctrls[i]; + if (uvc_entity_match_guid(ctrl->entity, info->entity) && ctrl->index == info->index) { uvc_ctrl_add_info(chain->dev, ctrl, info); @@ -2452,9 +2463,11 @@ static void uvc_ctrl_init_ctrl(struct uvc_video_chain *chain, */ if (chain->dev->info->mappings) { bool custom = false; - unsigned int i; - for (i = 0; (mapping = chain->dev->info->mappings[i]); ++i) { + for (i = 0; chain->dev->info->mappings[i]; ++i) { + const struct uvc_control_mapping *mapping = + chain->dev->info->mappings[i]; + if (uvc_entity_match_guid(ctrl->entity, mapping->entity) && ctrl->info.selector == mapping->selector) { __uvc_ctrl_add_mapping(chain, ctrl, mapping); @@ -2467,10 +2480,9 @@ static void uvc_ctrl_init_ctrl(struct uvc_video_chain *chain, } /* Process common mappings next. */ - mapping = uvc_ctrl_mappings; - mend = mapping + ARRAY_SIZE(uvc_ctrl_mappings); + for (i = 0; i < ARRAY_SIZE(uvc_ctrl_mappings); ++i) { + const struct uvc_control_mapping *mapping = &uvc_ctrl_mappings[i]; - for (; mapping < mend; ++mapping) { if (uvc_entity_match_guid(ctrl->entity, mapping->entity) && ctrl->info.selector == mapping->selector) __uvc_ctrl_add_mapping(chain, ctrl, mapping); @@ -2478,14 +2490,16 @@ static void uvc_ctrl_init_ctrl(struct uvc_video_chain *chain, /* Finally process version-specific mappings. */ if (chain->dev->uvc_version < 0x0150) { - mapping = uvc_ctrl_mappings_uvc11; - mend = mapping + ARRAY_SIZE(uvc_ctrl_mappings_uvc11); + mappings = uvc_ctrl_mappings_uvc11; + num_mappings = ARRAY_SIZE(uvc_ctrl_mappings_uvc11); } else { - mapping = uvc_ctrl_mappings_uvc15; - mend = mapping + ARRAY_SIZE(uvc_ctrl_mappings_uvc15); + mappings = uvc_ctrl_mappings_uvc15; + num_mappings = ARRAY_SIZE(uvc_ctrl_mappings_uvc15); } - for (; mapping < mend; ++mapping) { + for (i = 0; i < num_mappings; ++i) { + const struct uvc_control_mapping *mapping = &mappings[i]; + if (uvc_entity_match_guid(ctrl->entity, mapping->entity) && ctrl->info.selector == mapping->selector) __uvc_ctrl_add_mapping(chain, ctrl, mapping); diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c index 9c05776f11d1f09ce7d0e9706c15e62cc7835faa..215fb483efb00b89523f2eb9c3b233cff72923ec 100644 --- a/drivers/media/usb/uvc/uvc_driver.c +++ b/drivers/media/usb/uvc/uvc_driver.c @@ -20,6 +20,7 @@ #include #include +#include #include "uvcvideo.h" @@ -34,198 +35,6 @@ static unsigned int uvc_quirks_param = -1; unsigned int uvc_dbg_param; unsigned int uvc_timeout_param = UVC_CTRL_STREAMING_TIMEOUT; -/* ------------------------------------------------------------------------ - * Video formats - */ - -static struct uvc_format_desc uvc_fmts[] = { - { - .name = "YUV 4:2:2 (YUYV)", - .guid = UVC_GUID_FORMAT_YUY2, - .fcc = V4L2_PIX_FMT_YUYV, - }, - { - .name = "YUV 4:2:2 (YUYV)", - .guid = UVC_GUID_FORMAT_YUY2_ISIGHT, - .fcc = V4L2_PIX_FMT_YUYV, - }, - { - .name = "YUV 4:2:0 (NV12)", - .guid = UVC_GUID_FORMAT_NV12, - .fcc = V4L2_PIX_FMT_NV12, - }, - { - .name = "MJPEG", - .guid = UVC_GUID_FORMAT_MJPEG, - .fcc = V4L2_PIX_FMT_MJPEG, - }, - { - .name = "YVU 4:2:0 (YV12)", - .guid = UVC_GUID_FORMAT_YV12, - .fcc = V4L2_PIX_FMT_YVU420, - }, - { - .name = "YUV 4:2:0 (I420)", - .guid = UVC_GUID_FORMAT_I420, - .fcc = V4L2_PIX_FMT_YUV420, - }, - { - .name = "YUV 4:2:0 (M420)", - .guid = UVC_GUID_FORMAT_M420, - .fcc = V4L2_PIX_FMT_M420, - }, - { - .name = "YUV 4:2:2 (UYVY)", - .guid = UVC_GUID_FORMAT_UYVY, - .fcc = V4L2_PIX_FMT_UYVY, - }, - { - .name = "Greyscale 8-bit (Y800)", - .guid = UVC_GUID_FORMAT_Y800, - .fcc = V4L2_PIX_FMT_GREY, - }, - { - .name = "Greyscale 8-bit (Y8 )", - .guid = UVC_GUID_FORMAT_Y8, - .fcc = V4L2_PIX_FMT_GREY, - }, - { - .name = "Greyscale 8-bit (D3DFMT_L8)", - .guid = UVC_GUID_FORMAT_D3DFMT_L8, - .fcc = V4L2_PIX_FMT_GREY, - }, - { - .name = "IR 8-bit (L8_IR)", - .guid = UVC_GUID_FORMAT_KSMEDIA_L8_IR, - .fcc = V4L2_PIX_FMT_GREY, - }, - { - .name = "Greyscale 10-bit (Y10 )", - .guid = UVC_GUID_FORMAT_Y10, - .fcc = V4L2_PIX_FMT_Y10, - }, - { - .name = "Greyscale 12-bit (Y12 )", - .guid = UVC_GUID_FORMAT_Y12, - .fcc = V4L2_PIX_FMT_Y12, - }, - { - .name = "Greyscale 16-bit (Y16 )", - .guid = UVC_GUID_FORMAT_Y16, - .fcc = V4L2_PIX_FMT_Y16, - }, - { - .name = "BGGR Bayer (BY8 )", - .guid = UVC_GUID_FORMAT_BY8, - .fcc = V4L2_PIX_FMT_SBGGR8, - }, - { - .name = "BGGR Bayer (BA81)", - .guid = UVC_GUID_FORMAT_BA81, - .fcc = V4L2_PIX_FMT_SBGGR8, - }, - { - .name = "GBRG Bayer (GBRG)", - .guid = UVC_GUID_FORMAT_GBRG, - .fcc = V4L2_PIX_FMT_SGBRG8, - }, - { - .name = "GRBG Bayer (GRBG)", - .guid = UVC_GUID_FORMAT_GRBG, - .fcc = V4L2_PIX_FMT_SGRBG8, - }, - { - .name = "RGGB Bayer (RGGB)", - .guid = UVC_GUID_FORMAT_RGGB, - .fcc = V4L2_PIX_FMT_SRGGB8, - }, - { - .name = "RGB565", - .guid = UVC_GUID_FORMAT_RGBP, - .fcc = V4L2_PIX_FMT_RGB565, - }, - { - .name = "BGR 8:8:8 (BGR3)", - .guid = UVC_GUID_FORMAT_BGR3, - .fcc = V4L2_PIX_FMT_BGR24, - }, - { - .name = "H.264", - .guid = UVC_GUID_FORMAT_H264, - .fcc = V4L2_PIX_FMT_H264, - }, - { - .name = "H.265", - .guid = UVC_GUID_FORMAT_H265, - .fcc = V4L2_PIX_FMT_HEVC, - }, - { - .name = "Greyscale 8 L/R (Y8I)", - .guid = UVC_GUID_FORMAT_Y8I, - .fcc = V4L2_PIX_FMT_Y8I, - }, - { - .name = "Greyscale 12 L/R (Y12I)", - .guid = UVC_GUID_FORMAT_Y12I, - .fcc = V4L2_PIX_FMT_Y12I, - }, - { - .name = "Depth data 16-bit (Z16)", - .guid = UVC_GUID_FORMAT_Z16, - .fcc = V4L2_PIX_FMT_Z16, - }, - { - .name = "Bayer 10-bit (SRGGB10P)", - .guid = UVC_GUID_FORMAT_RW10, - .fcc = V4L2_PIX_FMT_SRGGB10P, - }, - { - .name = "Bayer 16-bit (SBGGR16)", - .guid = UVC_GUID_FORMAT_BG16, - .fcc = V4L2_PIX_FMT_SBGGR16, - }, - { - .name = "Bayer 16-bit (SGBRG16)", - .guid = UVC_GUID_FORMAT_GB16, - .fcc = V4L2_PIX_FMT_SGBRG16, - }, - { - .name = "Bayer 16-bit (SRGGB16)", - .guid = UVC_GUID_FORMAT_RG16, - .fcc = V4L2_PIX_FMT_SRGGB16, - }, - { - .name = "Bayer 16-bit (SGRBG16)", - .guid = UVC_GUID_FORMAT_GR16, - .fcc = V4L2_PIX_FMT_SGRBG16, - }, - { - .name = "Depth data 16-bit (Z16)", - .guid = UVC_GUID_FORMAT_INVZ, - .fcc = V4L2_PIX_FMT_Z16, - }, - { - .name = "Greyscale 10-bit (Y10 )", - .guid = UVC_GUID_FORMAT_INVI, - .fcc = V4L2_PIX_FMT_Y10, - }, - { - .name = "IR:Depth 26-bit (INZI)", - .guid = UVC_GUID_FORMAT_INZI, - .fcc = V4L2_PIX_FMT_INZI, - }, - { - .name = "4-bit Depth Confidence (Packed)", - .guid = UVC_GUID_FORMAT_CNF4, - .fcc = V4L2_PIX_FMT_CNF4, - }, - { - .name = "HEVC", - .guid = UVC_GUID_FORMAT_HEVC, - .fcc = V4L2_PIX_FMT_HEVC, - }, -}; - /* ------------------------------------------------------------------------ * Utility functions */ @@ -245,19 +54,6 @@ struct usb_host_endpoint *uvc_find_endpoint(struct usb_host_interface *alts, return NULL; } -static struct uvc_format_desc *uvc_format_by_guid(const u8 guid[16]) -{ - unsigned int len = ARRAY_SIZE(uvc_fmts); - unsigned int i; - - for (i = 0; i < len; ++i) { - if (memcmp(guid, uvc_fmts[i].guid, 16) == 0) - return &uvc_fmts[i]; - } - - return NULL; -} - static enum v4l2_colorspace uvc_colorspace(const u8 primaries) { static const enum v4l2_colorspace colorprimaries[] = { @@ -329,90 +125,6 @@ static enum v4l2_ycbcr_encoding uvc_ycbcr_enc(const u8 matrix_coefficients) return V4L2_YCBCR_ENC_DEFAULT; /* Reserved */ } -/* - * Simplify a fraction using a simple continued fraction decomposition. The - * idea here is to convert fractions such as 333333/10000000 to 1/30 using - * 32 bit arithmetic only. The algorithm is not perfect and relies upon two - * arbitrary parameters to remove non-significative terms from the simple - * continued fraction decomposition. Using 8 and 333 for n_terms and threshold - * respectively seems to give nice results. - */ -void uvc_simplify_fraction(u32 *numerator, u32 *denominator, - unsigned int n_terms, unsigned int threshold) -{ - u32 *an; - u32 x, y, r; - unsigned int i, n; - - an = kmalloc_array(n_terms, sizeof(*an), GFP_KERNEL); - if (an == NULL) - return; - - /* - * Convert the fraction to a simple continued fraction. See - * https://en.wikipedia.org/wiki/Continued_fraction - * Stop if the current term is bigger than or equal to the given - * threshold. - */ - x = *numerator; - y = *denominator; - - for (n = 0; n < n_terms && y != 0; ++n) { - an[n] = x / y; - if (an[n] >= threshold) { - if (n < 2) - n++; - break; - } - - r = x - an[n] * y; - x = y; - y = r; - } - - /* Expand the simple continued fraction back to an integer fraction. */ - x = 0; - y = 1; - - for (i = n; i > 0; --i) { - r = y; - y = an[i-1] * y + x; - x = r; - } - - *numerator = y; - *denominator = x; - kfree(an); -} - -/* - * Convert a fraction to a frame interval in 100ns multiples. The idea here is - * to compute numerator / denominator * 10000000 using 32 bit fixed point - * arithmetic only. - */ -u32 uvc_fraction_to_interval(u32 numerator, u32 denominator) -{ - u32 multiplier; - - /* Saturate the result if the operation would overflow. */ - if (denominator == 0 || - numerator/denominator >= ((u32)-1)/10000000) - return (u32)-1; - - /* - * Divide both the denominator and the multiplier by two until - * numerator * multiplier doesn't overflow. If anyone knows a better - * algorithm please let me know. - */ - multiplier = 10000000; - while (numerator > ((u32)-1)/multiplier) { - multiplier /= 2; - denominator /= 2; - } - - return denominator ? numerator * multiplier / denominator : 0; -} - /* ------------------------------------------------------------------------ * Terminal and unit management */ @@ -1553,10 +1265,6 @@ static int uvc_gpio_parse(struct uvc_device *dev) if (IS_ERR_OR_NULL(gpio_privacy)) return PTR_ERR_OR_ZERO(gpio_privacy); - unit = uvc_alloc_entity(UVC_EXT_GPIO_UNIT, UVC_EXT_GPIO_UNIT_ID, 0, 1); - if (!unit) - return -ENOMEM; - irq = gpiod_to_irq(gpio_privacy); if (irq < 0) { if (irq != EPROBE_DEFER) @@ -1565,6 +1273,10 @@ static int uvc_gpio_parse(struct uvc_device *dev) return irq; } + unit = uvc_alloc_entity(UVC_EXT_GPIO_UNIT, UVC_EXT_GPIO_UNIT_ID, 0, 1); + if (!unit) + return -ENOMEM; + unit->gpio.gpio_privacy = gpio_privacy; unit->gpio.irq = irq; unit->gpio.bControlSize = 1; @@ -2740,7 +2452,7 @@ static const struct usb_device_id uvc_ids[] = { .idProduct = 0x4034, .bInterfaceClass = USB_CLASS_VIDEO, .bInterfaceSubClass = 1, - .bInterfaceProtocol = 0, + .bInterfaceProtocol = UVC_PC_PROTOCOL_15, .driver_info = (kernel_ulong_t)&uvc_ctrl_power_line_limited }, /* LogiLink Wireless Webcam */ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE @@ -3264,6 +2976,15 @@ static const struct usb_device_id uvc_ids[] = { .bInterfaceSubClass = 1, .bInterfaceProtocol = 0, .driver_info = UVC_INFO_QUIRK(UVC_QUIRK_FORCE_BPP) }, + /* Sonix Technology USB 2.0 Camera */ + { .match_flags = USB_DEVICE_ID_MATCH_DEVICE + | USB_DEVICE_ID_MATCH_INT_INFO, + .idVendor = 0x3277, + .idProduct = 0x0072, + .bInterfaceClass = USB_CLASS_VIDEO, + .bInterfaceSubClass = 1, + .bInterfaceProtocol = 0, + .driver_info = (kernel_ulong_t)&uvc_ctrl_power_line_limited }, /* Acer EasyCamera */ { .match_flags = USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_INT_INFO, diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c index 4cc3fa6b8c9812050b7a5ad3a34b82e7ac0e0529..f4d4c33b6dfbd7ef86f2bef050eeab6ff2615ebf 100644 --- a/drivers/media/usb/uvc/uvc_v4l2.c +++ b/drivers/media/usb/uvc/uvc_v4l2.c @@ -386,7 +386,7 @@ static int uvc_v4l2_get_streamparm(struct uvc_streaming *stream, mutex_unlock(&stream->mutex); denominator = 10000000; - uvc_simplify_fraction(&numerator, &denominator, 8, 333); + v4l2_simplify_fraction(&numerator, &denominator, 8, 333); memset(parm, 0, sizeof(*parm)); parm->type = stream->type; @@ -427,7 +427,7 @@ static int uvc_v4l2_set_streamparm(struct uvc_streaming *stream, else timeperframe = parm->parm.output.timeperframe; - interval = uvc_fraction_to_interval(timeperframe.numerator, + interval = v4l2_fraction_to_interval(timeperframe.numerator, timeperframe.denominator); uvc_dbg(stream->dev, FORMAT, "Setting frame interval to %u/%u (%u)\n", timeperframe.numerator, timeperframe.denominator, interval); @@ -481,7 +481,7 @@ static int uvc_v4l2_set_streamparm(struct uvc_streaming *stream, /* Return the actual frame period. */ timeperframe.numerator = probe.dwFrameInterval; timeperframe.denominator = 10000000; - uvc_simplify_fraction(&timeperframe.numerator, + v4l2_simplify_fraction(&timeperframe.numerator, &timeperframe.denominator, 8, 333); if (parm->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) { @@ -1275,7 +1275,7 @@ static int uvc_ioctl_enum_frameintervals(struct file *file, void *fh, fival->discrete.numerator = frame->dwFrameInterval[index]; fival->discrete.denominator = 10000000; - uvc_simplify_fraction(&fival->discrete.numerator, + v4l2_simplify_fraction(&fival->discrete.numerator, &fival->discrete.denominator, 8, 333); } else { fival->type = V4L2_FRMIVAL_TYPE_STEPWISE; @@ -1285,11 +1285,11 @@ static int uvc_ioctl_enum_frameintervals(struct file *file, void *fh, fival->stepwise.max.denominator = 10000000; fival->stepwise.step.numerator = frame->dwFrameInterval[2]; fival->stepwise.step.denominator = 10000000; - uvc_simplify_fraction(&fival->stepwise.min.numerator, + v4l2_simplify_fraction(&fival->stepwise.min.numerator, &fival->stepwise.min.denominator, 8, 333); - uvc_simplify_fraction(&fival->stepwise.max.numerator, + v4l2_simplify_fraction(&fival->stepwise.max.numerator, &fival->stepwise.max.denominator, 8, 333); - uvc_simplify_fraction(&fival->stepwise.step.numerator, + v4l2_simplify_fraction(&fival->stepwise.step.numerator, &fival->stepwise.step.denominator, 8, 333); } diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index 170a008f4006370b5c32da767f1f57b9aee7dac4..d2eb9066e4dccda874a4067d043631e84a320242 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -1095,7 +1095,7 @@ static int uvc_video_decode_start(struct uvc_streaming *stream, /* * Synchronize to the input stream by waiting for the FID bit to be - * toggled when the the buffer state is not UVC_BUF_STATE_ACTIVE. + * toggled when the buffer state is not UVC_BUF_STATE_ACTIVE. * stream->last_fid is initialized to -1, so the first isochronous * frame will always be in sync. * diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h index 24c911aeebce569277cb2bc3ec06449fafdf425e..df93db259312e7877a1874c8eaba62daf4d21176 100644 --- a/drivers/media/usb/uvc/uvcvideo.h +++ b/drivers/media/usb/uvc/uvcvideo.h @@ -41,144 +41,6 @@ #define UVC_EXT_GPIO_UNIT 0x7ffe #define UVC_EXT_GPIO_UNIT_ID 0x100 -/* ------------------------------------------------------------------------ - * GUIDs - */ -#define UVC_GUID_UVC_CAMERA \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} -#define UVC_GUID_UVC_OUTPUT \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02} -#define UVC_GUID_UVC_MEDIA_TRANSPORT_INPUT \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03} -#define UVC_GUID_UVC_PROCESSING \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01} -#define UVC_GUID_UVC_SELECTOR \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02} -#define UVC_GUID_EXT_GPIO_CONTROLLER \ - {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03} - -#define UVC_GUID_FORMAT_MJPEG \ - { 'M', 'J', 'P', 'G', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_YUY2 \ - { 'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_YUY2_ISIGHT \ - { 'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0x00, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_NV12 \ - { 'N', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_YV12 \ - { 'Y', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_I420 \ - { 'I', '4', '2', '0', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_UYVY \ - { 'U', 'Y', 'V', 'Y', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y800 \ - { 'Y', '8', '0', '0', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y8 \ - { 'Y', '8', ' ', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y10 \ - { 'Y', '1', '0', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y12 \ - { 'Y', '1', '2', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y16 \ - { 'Y', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_BY8 \ - { 'B', 'Y', '8', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_BA81 \ - { 'B', 'A', '8', '1', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_GBRG \ - { 'G', 'B', 'R', 'G', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_GRBG \ - { 'G', 'R', 'B', 'G', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_RGGB \ - { 'R', 'G', 'G', 'B', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_BG16 \ - { 'B', 'G', '1', '6', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_GB16 \ - { 'G', 'B', '1', '6', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_RG16 \ - { 'R', 'G', '1', '6', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_GR16 \ - { 'G', 'R', '1', '6', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_RGBP \ - { 'R', 'G', 'B', 'P', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_BGR3 \ - { 0x7d, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, \ - 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70} -#define UVC_GUID_FORMAT_M420 \ - { 'M', '4', '2', '0', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} - -#define UVC_GUID_FORMAT_H264 \ - { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_H265 \ - { 'H', '2', '6', '5', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y8I \ - { 'Y', '8', 'I', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Y12I \ - { 'Y', '1', '2', 'I', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_Z16 \ - { 'Z', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_RW10 \ - { 'R', 'W', '1', '0', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_INVZ \ - { 'I', 'N', 'V', 'Z', 0x90, 0x2d, 0x58, 0x4a, \ - 0x92, 0x0b, 0x77, 0x3f, 0x1f, 0x2c, 0x55, 0x6b} -#define UVC_GUID_FORMAT_INZI \ - { 'I', 'N', 'Z', 'I', 0x66, 0x1a, 0x42, 0xa2, \ - 0x90, 0x65, 0xd0, 0x18, 0x14, 0xa8, 0xef, 0x8a} -#define UVC_GUID_FORMAT_INVI \ - { 'I', 'N', 'V', 'I', 0xdb, 0x57, 0x49, 0x5e, \ - 0x8e, 0x3f, 0xf4, 0x79, 0x53, 0x2b, 0x94, 0x6f} -#define UVC_GUID_FORMAT_CNF4 \ - { 'C', ' ', ' ', ' ', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} - -#define UVC_GUID_FORMAT_D3DFMT_L8 \ - {0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} -#define UVC_GUID_FORMAT_KSMEDIA_L8_IR \ - {0x32, 0x00, 0x00, 0x00, 0x02, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} - -#define UVC_GUID_FORMAT_HEVC \ - { 'H', 'E', 'V', 'C', 0x00, 0x00, 0x10, 0x00, \ - 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} - - /* ------------------------------------------------------------------------ * Driver specific constants. */ @@ -283,12 +145,6 @@ struct uvc_control { struct uvc_fh *handle; /* File handle that last changed the control. */ }; -struct uvc_format_desc { - char *name; - u8 guid[16]; - u32 fcc; -}; - /* * The term 'entity' refers to both UVC units and UVC terminals. * @@ -911,9 +767,6 @@ int uvc_xu_ctrl_query(struct uvc_video_chain *chain, struct uvc_xu_control_query *xqry); /* Utility functions */ -void uvc_simplify_fraction(u32 *numerator, u32 *denominator, - unsigned int n_terms, unsigned int threshold); -u32 uvc_fraction_to_interval(u32 numerator, u32 denominator); struct usb_host_endpoint *uvc_find_endpoint(struct usb_host_interface *alts, u8 epaddr); u16 uvc_endpoint_max_bpi(struct usb_device *dev, struct usb_host_endpoint *ep); diff --git a/drivers/media/v4l2-core/tuner-core.c b/drivers/media/v4l2-core/tuner-core.c index 2d47c10de0624e0c41f118cecd0bc0347a3d644a..33162dc1daf697f6b6d1ec4da99050b1ed11a532 100644 --- a/drivers/media/v4l2-core/tuner-core.c +++ b/drivers/media/v4l2-core/tuner-core.c @@ -779,7 +779,7 @@ register_client: * @client: i2c_client descriptor */ -static int tuner_remove(struct i2c_client *client) +static void tuner_remove(struct i2c_client *client) { struct tuner *t = to_tuner(i2c_get_clientdata(client)); @@ -789,7 +789,6 @@ static int tuner_remove(struct i2c_client *client) list_del(&t->list); kfree(t); - return 0; } /* diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c index e0fbe6ba4b6c4936cb75bfe4ab380a0a26725aba..40f56e044640d75ae9d41475c39b6b7c8cf0e9ae 100644 --- a/drivers/media/v4l2-core/v4l2-common.c +++ b/drivers/media/v4l2-core/v4l2-common.c @@ -484,3 +484,89 @@ s64 v4l2_get_link_freq(struct v4l2_ctrl_handler *handler, unsigned int mul, return freq > 0 ? freq : -EINVAL; } EXPORT_SYMBOL_GPL(v4l2_get_link_freq); + +/* + * Simplify a fraction using a simple continued fraction decomposition. The + * idea here is to convert fractions such as 333333/10000000 to 1/30 using + * 32 bit arithmetic only. The algorithm is not perfect and relies upon two + * arbitrary parameters to remove non-significative terms from the simple + * continued fraction decomposition. Using 8 and 333 for n_terms and threshold + * respectively seems to give nice results. + */ +void v4l2_simplify_fraction(u32 *numerator, u32 *denominator, + unsigned int n_terms, unsigned int threshold) +{ + u32 *an; + u32 x, y, r; + unsigned int i, n; + + an = kmalloc_array(n_terms, sizeof(*an), GFP_KERNEL); + if (an == NULL) + return; + + /* + * Convert the fraction to a simple continued fraction. See + * https://en.wikipedia.org/wiki/Continued_fraction + * Stop if the current term is bigger than or equal to the given + * threshold. + */ + x = *numerator; + y = *denominator; + + for (n = 0; n < n_terms && y != 0; ++n) { + an[n] = x / y; + if (an[n] >= threshold) { + if (n < 2) + n++; + break; + } + + r = x - an[n] * y; + x = y; + y = r; + } + + /* Expand the simple continued fraction back to an integer fraction. */ + x = 0; + y = 1; + + for (i = n; i > 0; --i) { + r = y; + y = an[i-1] * y + x; + x = r; + } + + *numerator = y; + *denominator = x; + kfree(an); +} +EXPORT_SYMBOL_GPL(v4l2_simplify_fraction); + +/* + * Convert a fraction to a frame interval in 100ns multiples. The idea here is + * to compute numerator / denominator * 10000000 using 32 bit fixed point + * arithmetic only. + */ +u32 v4l2_fraction_to_interval(u32 numerator, u32 denominator) +{ + u32 multiplier; + + /* Saturate the result if the operation would overflow. */ + if (denominator == 0 || + numerator/denominator >= ((u32)-1)/10000000) + return (u32)-1; + + /* + * Divide both the denominator and the multiplier by two until + * numerator * multiplier doesn't overflow. If anyone knows a better + * algorithm please let me know. + */ + multiplier = 10000000; + while (numerator > ((u32)-1)/multiplier) { + multiplier /= 2; + denominator /= 2; + } + + return denominator ? numerator * multiplier / denominator : 0; +} +EXPORT_SYMBOL_GPL(v4l2_fraction_to_interval); diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c index 0f3d6b5667b07e6a08a332cafa18333295e365fd..55c26e7d370e9292fa5f81f06dad0654c66f5b42 100644 --- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c +++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c @@ -1040,6 +1040,8 @@ int v4l2_compat_get_array_args(struct file *file, void *mbuf, { int err = 0; + memset(mbuf, 0, array_size); + switch (cmd) { case VIDIOC_G_FMT32: case VIDIOC_S_FMT32: diff --git a/drivers/media/v4l2-core/v4l2-ctrls-api.c b/drivers/media/v4l2-core/v4l2-ctrls-api.c index 50d012ba3c02e791f408260f0cf92370bf68ee26..a8c354ad3d234c2e430a2001b98198246e037723 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-api.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-api.c @@ -89,10 +89,7 @@ static int req_to_user(struct v4l2_ext_control *c, /* Helper function: copy the initial control value back to the caller */ static int def_to_user(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) { - int idx; - - for (idx = 0; idx < ctrl->elems; idx++) - ctrl->type_ops->init(ctrl, idx, ctrl->p_new); + ctrl->type_ops->init(ctrl, 0, ctrl->elems, ctrl->p_new); return ptr_to_user(c, ctrl, ctrl->p_new); } @@ -105,8 +102,8 @@ static int user_to_new(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) ctrl->is_new = 0; if (ctrl->is_dyn_array && - c->size > ctrl->p_dyn_alloc_elems * ctrl->elem_size) { - void *old = ctrl->p_dyn; + c->size > ctrl->p_array_alloc_elems * ctrl->elem_size) { + void *old = ctrl->p_array; void *tmp = kvzalloc(2 * c->size, GFP_KERNEL); if (!tmp) @@ -115,14 +112,13 @@ static int user_to_new(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) memcpy(tmp + c->size, ctrl->p_cur.p, ctrl->elems * ctrl->elem_size); ctrl->p_new.p = tmp; ctrl->p_cur.p = tmp + c->size; - ctrl->p_dyn = tmp; - ctrl->p_dyn_alloc_elems = c->size / ctrl->elem_size; + ctrl->p_array = tmp; + ctrl->p_array_alloc_elems = c->size / ctrl->elem_size; kvfree(old); } if (ctrl->is_ptr && !ctrl->is_string) { unsigned int elems = c->size / ctrl->elem_size; - unsigned int idx; if (copy_from_user(ctrl->p_new.p, c->ptr, c->size)) return -EFAULT; @@ -130,8 +126,7 @@ static int user_to_new(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) if (ctrl->is_dyn_array) ctrl->new_elems = elems; else if (ctrl->is_array) - for (idx = elems; idx < ctrl->elems; idx++) - ctrl->type_ops->init(ctrl, idx, ctrl->p_new); + ctrl->type_ops->init(ctrl, elems, ctrl->elems, ctrl->p_new); return 0; } @@ -467,7 +462,7 @@ int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, if (is_default) ret = def_to_user(cs->controls + idx, ref->ctrl); - else if (is_request && ref->p_req_dyn_enomem) + else if (is_request && ref->p_req_array_enomem) ret = -ENOMEM; else if (is_request && ref->p_req_valid) ret = req_to_user(cs->controls + idx, ref); @@ -499,12 +494,7 @@ EXPORT_SYMBOL(v4l2_g_ext_ctrls); /* Validate a new control */ static int validate_new(const struct v4l2_ctrl *ctrl, union v4l2_ctrl_ptr p_new) { - unsigned int idx; - int err = 0; - - for (idx = 0; !err && idx < ctrl->new_elems; idx++) - err = ctrl->type_ops->validate(ctrl, idx, p_new); - return err; + return ctrl->type_ops->validate(ctrl, ctrl->new_elems, p_new); } /* Validate controls. */ @@ -989,6 +979,42 @@ int __v4l2_ctrl_modify_range(struct v4l2_ctrl *ctrl, } EXPORT_SYMBOL(__v4l2_ctrl_modify_range); +int __v4l2_ctrl_modify_dimensions(struct v4l2_ctrl *ctrl, + u32 dims[V4L2_CTRL_MAX_DIMS]) +{ + unsigned int elems = 1; + unsigned int i; + void *p_array; + + lockdep_assert_held(ctrl->handler->lock); + + if (!ctrl->is_array || ctrl->is_dyn_array) + return -EINVAL; + + for (i = 0; i < ctrl->nr_of_dims; i++) + elems *= dims[i]; + if (elems == 0) + return -EINVAL; + p_array = kvzalloc(2 * elems * ctrl->elem_size, GFP_KERNEL); + if (!p_array) + return -ENOMEM; + kvfree(ctrl->p_array); + ctrl->p_array_alloc_elems = elems; + ctrl->elems = elems; + ctrl->new_elems = elems; + ctrl->p_array = p_array; + ctrl->p_new.p = p_array; + ctrl->p_cur.p = p_array + elems * ctrl->elem_size; + for (i = 0; i < ctrl->nr_of_dims; i++) + ctrl->dims[i] = dims[i]; + ctrl->type_ops->init(ctrl, 0, elems, ctrl->p_cur); + cur_to_new(ctrl); + send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_VALUE | + V4L2_EVENT_CTRL_CH_DIMENSIONS); + return 0; +} +EXPORT_SYMBOL(__v4l2_ctrl_modify_dimensions); + /* Implement VIDIOC_QUERY_EXT_CTRL */ int v4l2_query_ext_ctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_query_ext_ctrl *qc) { diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c index 1f85828d6694dc9ac0a99b2aec0613f2b2064dbc..01f00093f259132d7063623f512f55334d96dd05 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-core.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -65,33 +65,29 @@ void send_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 changes) v4l2_event_queue_fh(sev->fh, &ev); } -static bool std_equal(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr1, - union v4l2_ctrl_ptr ptr2) +bool v4l2_ctrl_type_op_equal(const struct v4l2_ctrl *ctrl, u32 elems, + union v4l2_ctrl_ptr ptr1, union v4l2_ctrl_ptr ptr2) { + unsigned int i; + switch (ctrl->type) { case V4L2_CTRL_TYPE_BUTTON: return false; case V4L2_CTRL_TYPE_STRING: - idx *= ctrl->elem_size; - /* strings are always 0-terminated */ - return !strcmp(ptr1.p_char + idx, ptr2.p_char + idx); - case V4L2_CTRL_TYPE_INTEGER64: - return ptr1.p_s64[idx] == ptr2.p_s64[idx]; - case V4L2_CTRL_TYPE_U8: - return ptr1.p_u8[idx] == ptr2.p_u8[idx]; - case V4L2_CTRL_TYPE_U16: - return ptr1.p_u16[idx] == ptr2.p_u16[idx]; - case V4L2_CTRL_TYPE_U32: - return ptr1.p_u32[idx] == ptr2.p_u32[idx]; + for (i = 0; i < elems; i++) { + unsigned int idx = i * ctrl->elem_size; + + /* strings are always 0-terminated */ + if (strcmp(ptr1.p_char + idx, ptr2.p_char + idx)) + return false; + } + return true; default: - if (ctrl->is_int) - return ptr1.p_s32[idx] == ptr2.p_s32[idx]; - idx *= ctrl->elem_size; - return !memcmp(ptr1.p_const + idx, ptr2.p_const + idx, - ctrl->elem_size); + return !memcmp(ptr1.p_const, ptr2.p_const, + elems * ctrl->elem_size); } } +EXPORT_SYMBOL(v4l2_ctrl_type_op_equal); /* Default intra MPEG-2 quantisation coefficients, from the specification. */ static const u8 mpeg2_intra_quant_matrix[64] = { @@ -181,45 +177,76 @@ static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, } } -static void std_init(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) +void v4l2_ctrl_type_op_init(const struct v4l2_ctrl *ctrl, u32 from_idx, + u32 tot_elems, union v4l2_ctrl_ptr ptr) { + unsigned int i; + u32 elems = tot_elems - from_idx; + + if (from_idx >= tot_elems) + return; + switch (ctrl->type) { case V4L2_CTRL_TYPE_STRING: - idx *= ctrl->elem_size; - memset(ptr.p_char + idx, ' ', ctrl->minimum); - ptr.p_char[idx + ctrl->minimum] = '\0'; + for (i = from_idx; i < tot_elems; i++) { + unsigned int offset = i * ctrl->elem_size; + + memset(ptr.p_char + offset, ' ', ctrl->minimum); + ptr.p_char[offset + ctrl->minimum] = '\0'; + } break; case V4L2_CTRL_TYPE_INTEGER64: - ptr.p_s64[idx] = ctrl->default_value; + if (ctrl->default_value) { + for (i = from_idx; i < tot_elems; i++) + ptr.p_s64[i] = ctrl->default_value; + } else { + memset(ptr.p_s64 + from_idx, 0, elems * sizeof(s64)); + } break; case V4L2_CTRL_TYPE_INTEGER: case V4L2_CTRL_TYPE_INTEGER_MENU: case V4L2_CTRL_TYPE_MENU: case V4L2_CTRL_TYPE_BITMASK: case V4L2_CTRL_TYPE_BOOLEAN: - ptr.p_s32[idx] = ctrl->default_value; + if (ctrl->default_value) { + for (i = from_idx; i < tot_elems; i++) + ptr.p_s32[i] = ctrl->default_value; + } else { + memset(ptr.p_s32 + from_idx, 0, elems * sizeof(s32)); + } break; case V4L2_CTRL_TYPE_BUTTON: case V4L2_CTRL_TYPE_CTRL_CLASS: - ptr.p_s32[idx] = 0; + memset(ptr.p_s32 + from_idx, 0, elems * sizeof(s32)); break; case V4L2_CTRL_TYPE_U8: - ptr.p_u8[idx] = ctrl->default_value; + memset(ptr.p_u8 + from_idx, ctrl->default_value, elems); break; case V4L2_CTRL_TYPE_U16: - ptr.p_u16[idx] = ctrl->default_value; + if (ctrl->default_value) { + for (i = from_idx; i < tot_elems; i++) + ptr.p_u16[i] = ctrl->default_value; + } else { + memset(ptr.p_u16 + from_idx, 0, elems * sizeof(u16)); + } break; case V4L2_CTRL_TYPE_U32: - ptr.p_u32[idx] = ctrl->default_value; + if (ctrl->default_value) { + for (i = from_idx; i < tot_elems; i++) + ptr.p_u32[i] = ctrl->default_value; + } else { + memset(ptr.p_u32 + from_idx, 0, elems * sizeof(u32)); + } break; default: - std_init_compound(ctrl, idx, ptr); + for (i = from_idx; i < tot_elems; i++) + std_init_compound(ctrl, i, ptr); break; } } +EXPORT_SYMBOL(v4l2_ctrl_type_op_init); -static void std_log(const struct v4l2_ctrl *ctrl) +void v4l2_ctrl_type_op_log(const struct v4l2_ctrl *ctrl) { union v4l2_ctrl_ptr ptr = ctrl->p_cur; @@ -327,6 +354,7 @@ static void std_log(const struct v4l2_ctrl *ctrl) break; } } +EXPORT_SYMBOL(v4l2_ctrl_type_op_log); /* * Round towards the closest legal value. Be careful when we are @@ -520,7 +548,8 @@ validate_vp9_frame(struct v4l2_ctrl_vp9_frame *frame) /* * Compound controls validation requires setting unused fields/flags to zero - * in order to properly detect unchanged controls with std_equal's memcmp. + * in order to properly detect unchanged controls with v4l2_ctrl_type_op_equal's + * memcmp. */ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, union v4l2_ctrl_ptr ptr) @@ -895,8 +924,8 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, return 0; } -static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) +static int std_validate_elem(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr) { size_t len; u64 offset; @@ -966,11 +995,43 @@ static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, } } +int v4l2_ctrl_type_op_validate(const struct v4l2_ctrl *ctrl, u32 elems, + union v4l2_ctrl_ptr ptr) +{ + unsigned int i; + int ret = 0; + + switch ((u32)ctrl->type) { + case V4L2_CTRL_TYPE_U8: + if (ctrl->maximum == 0xff && ctrl->minimum == 0 && ctrl->step == 1) + return 0; + break; + case V4L2_CTRL_TYPE_U16: + if (ctrl->maximum == 0xffff && ctrl->minimum == 0 && ctrl->step == 1) + return 0; + break; + case V4L2_CTRL_TYPE_U32: + if (ctrl->maximum == 0xffffffff && ctrl->minimum == 0 && ctrl->step == 1) + return 0; + break; + + case V4L2_CTRL_TYPE_BUTTON: + case V4L2_CTRL_TYPE_CTRL_CLASS: + memset(ptr.p_s32, 0, elems * sizeof(s32)); + return 0; + } + + for (i = 0; !ret && i < elems; i++) + ret = std_validate_elem(ctrl, i, ptr); + return ret; +} +EXPORT_SYMBOL(v4l2_ctrl_type_op_validate); + static const struct v4l2_ctrl_type_ops std_type_ops = { - .equal = std_equal, - .init = std_init, - .log = std_log, - .validate = std_validate, + .equal = v4l2_ctrl_type_op_equal, + .init = v4l2_ctrl_type_op_init, + .log = v4l2_ctrl_type_op_log, + .validate = v4l2_ctrl_type_op_validate, }; void v4l2_ctrl_notify(struct v4l2_ctrl *ctrl, v4l2_ctrl_notify_fnc notify, void *priv) @@ -1048,23 +1109,26 @@ void cur_to_new(struct v4l2_ctrl *ctrl) ptr_to_ptr(ctrl, ctrl->p_cur, ctrl->p_new, ctrl->new_elems); } -static bool req_alloc_dyn_array(struct v4l2_ctrl_ref *ref, u32 elems) +static bool req_alloc_array(struct v4l2_ctrl_ref *ref, u32 elems) { void *tmp; - if (elems < ref->p_req_dyn_alloc_elems) + if (elems == ref->p_req_array_alloc_elems) + return true; + if (ref->ctrl->is_dyn_array && + elems < ref->p_req_array_alloc_elems) return true; tmp = kvmalloc(elems * ref->ctrl->elem_size, GFP_KERNEL); if (!tmp) { - ref->p_req_dyn_enomem = true; + ref->p_req_array_enomem = true; return false; } - ref->p_req_dyn_enomem = false; + ref->p_req_array_enomem = false; kvfree(ref->p_req.p); ref->p_req.p = tmp; - ref->p_req_dyn_alloc_elems = elems; + ref->p_req_array_alloc_elems = elems; return true; } @@ -1077,7 +1141,7 @@ void new_to_req(struct v4l2_ctrl_ref *ref) return; ctrl = ref->ctrl; - if (ctrl->is_dyn_array && !req_alloc_dyn_array(ref, ctrl->new_elems)) + if (ctrl->is_array && !req_alloc_array(ref, ctrl->new_elems)) return; ref->p_req_elems = ctrl->new_elems; @@ -1094,7 +1158,7 @@ void cur_to_req(struct v4l2_ctrl_ref *ref) return; ctrl = ref->ctrl; - if (ctrl->is_dyn_array && !req_alloc_dyn_array(ref, ctrl->elems)) + if (ctrl->is_array && !req_alloc_array(ref, ctrl->elems)) return; ref->p_req_elems = ctrl->elems; @@ -1123,26 +1187,30 @@ int req_to_new(struct v4l2_ctrl_ref *ref) return 0; } - /* Not a dynamic array, so just copy the request value */ - if (!ctrl->is_dyn_array) { + /* Not an array, so just copy the request value */ + if (!ctrl->is_array) { ptr_to_ptr(ctrl, ref->p_req, ctrl->p_new, ctrl->new_elems); return 0; } /* Sanity check, should never happen */ - if (WARN_ON(!ref->p_req_dyn_alloc_elems)) + if (WARN_ON(!ref->p_req_array_alloc_elems)) + return -ENOMEM; + + if (!ctrl->is_dyn_array && + ref->p_req_elems != ctrl->p_array_alloc_elems) return -ENOMEM; /* * Check if the number of elements in the request is more than the - * elements in ctrl->p_dyn. If so, attempt to realloc ctrl->p_dyn. - * Note that p_dyn is allocated with twice the number of elements + * elements in ctrl->p_array. If so, attempt to realloc ctrl->p_array. + * Note that p_array is allocated with twice the number of elements * in the dynamic array since it has to store both the current and * new value of such a control. */ - if (ref->p_req_elems > ctrl->p_dyn_alloc_elems) { + if (ref->p_req_elems > ctrl->p_array_alloc_elems) { unsigned int sz = ref->p_req_elems * ctrl->elem_size; - void *old = ctrl->p_dyn; + void *old = ctrl->p_array; void *tmp = kvzalloc(2 * sz, GFP_KERNEL); if (!tmp) @@ -1151,8 +1219,8 @@ int req_to_new(struct v4l2_ctrl_ref *ref) memcpy(tmp + sz, ctrl->p_cur.p, ctrl->elems * ctrl->elem_size); ctrl->p_new.p = tmp; ctrl->p_cur.p = tmp + sz; - ctrl->p_dyn = tmp; - ctrl->p_dyn_alloc_elems = ref->p_req_elems; + ctrl->p_array = tmp; + ctrl->p_array_alloc_elems = ref->p_req_elems; kvfree(old); } @@ -1243,7 +1311,7 @@ void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) /* Free all nodes */ list_for_each_entry_safe(ref, next_ref, &hdl->ctrl_refs, node) { list_del(&ref->node); - if (ref->p_req_dyn_alloc_elems) + if (ref->p_req_array_alloc_elems) kvfree(ref->p_req.p); kfree(ref); } @@ -1252,7 +1320,7 @@ void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) list_del(&ctrl->node); list_for_each_entry_safe(sev, next_sev, &ctrl->ev_subs, node) list_del(&sev->node); - kvfree(ctrl->p_dyn); + kvfree(ctrl->p_array); kvfree(ctrl); } kvfree(hdl->buckets); @@ -1368,7 +1436,7 @@ int handler_new_ref(struct v4l2_ctrl_handler *hdl, if (hdl->error) return hdl->error; - if (allocate_req && !ctrl->is_dyn_array) + if (allocate_req && !ctrl->is_array) size_extra_req = ctrl->elems * ctrl->elem_size; new_ref = kzalloc(sizeof(*new_ref) + size_extra_req, GFP_KERNEL); if (!new_ref) @@ -1442,7 +1510,6 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, unsigned elems = 1; bool is_array; unsigned tot_ctrl_size; - unsigned idx; void *data; int err; @@ -1584,11 +1651,10 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; else if (type == V4L2_CTRL_TYPE_CTRL_CLASS) flags |= V4L2_CTRL_FLAG_READ_ONLY; - else if (!(flags & V4L2_CTRL_FLAG_DYNAMIC_ARRAY) && + else if (!is_array && (type == V4L2_CTRL_TYPE_INTEGER64 || type == V4L2_CTRL_TYPE_STRING || - type >= V4L2_CTRL_COMPOUND_TYPES || - is_array)) + type >= V4L2_CTRL_COMPOUND_TYPES)) sz_extra += 2 * tot_ctrl_size; if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) @@ -1632,14 +1698,14 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, ctrl->cur.val = ctrl->val = def; data = &ctrl[1]; - if (ctrl->is_dyn_array) { - ctrl->p_dyn_alloc_elems = elems; - ctrl->p_dyn = kvzalloc(2 * elems * elem_size, GFP_KERNEL); - if (!ctrl->p_dyn) { + if (ctrl->is_array) { + ctrl->p_array_alloc_elems = elems; + ctrl->p_array = kvzalloc(2 * elems * elem_size, GFP_KERNEL); + if (!ctrl->p_array) { kvfree(ctrl); return NULL; } - data = ctrl->p_dyn; + data = ctrl->p_array; } if (!ctrl->is_int) { @@ -1651,20 +1717,18 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, } if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) { - if (ctrl->is_dyn_array) + if (ctrl->is_array) ctrl->p_def.p = &ctrl[1]; else ctrl->p_def.p = ctrl->p_cur.p + tot_ctrl_size; memcpy(ctrl->p_def.p, p_def.p_const, elem_size); } - for (idx = 0; idx < elems; idx++) { - ctrl->type_ops->init(ctrl, idx, ctrl->p_cur); - ctrl->type_ops->init(ctrl, idx, ctrl->p_new); - } + ctrl->type_ops->init(ctrl, 0, elems, ctrl->p_cur); + cur_to_new(ctrl); if (handler_new_ref(hdl, ctrl, NULL, false, false)) { - kvfree(ctrl->p_dyn); + kvfree(ctrl->p_array); kvfree(ctrl); return NULL; } @@ -1978,7 +2042,6 @@ void update_from_auto_cluster(struct v4l2_ctrl *master) static int cluster_changed(struct v4l2_ctrl *master) { bool changed = false; - unsigned int idx; int i; for (i = 0; i < master->ncontrols; i++) { @@ -2004,10 +2067,9 @@ static int cluster_changed(struct v4l2_ctrl *master) if (ctrl->elems != ctrl->new_elems) ctrl_changed = true; - - for (idx = 0; !ctrl_changed && idx < ctrl->elems; idx++) - ctrl_changed = !ctrl->type_ops->equal(ctrl, idx, - ctrl->p_cur, ctrl->p_new); + if (!ctrl_changed) + ctrl_changed = !ctrl->type_ops->equal(ctrl, + ctrl->elems, ctrl->p_cur, ctrl->p_new); ctrl->has_changed = ctrl_changed; changed |= ctrl->has_changed; } diff --git a/drivers/media/v4l2-core/v4l2-flash-led-class.c b/drivers/media/v4l2-core/v4l2-flash-led-class.c index e70e128ccc9c578f6e414a46313ca160937a8045..355595a0fefac72c2f6941a30fa430d37dbdccfe 100644 --- a/drivers/media/v4l2-core/v4l2-flash-led-class.c +++ b/drivers/media/v4l2-core/v4l2-flash-led-class.c @@ -94,7 +94,7 @@ static int v4l2_flash_set_led_brightness(struct v4l2_flash *v4l2_flash, * brightness <-> intensity conversion, it also must have defined * related v4l2 control step == 1. In such a case a backward conversion * from led brightness to v4l2 intensity is required to find out the - * the aligned intensity value. + * aligned intensity value. */ if (has_flash_op(v4l2_flash, led_brightness_to_intensity)) ctrl->val = call_flash_op(v4l2_flash, diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index c314025d977e76ff87fd7f3a0a9cc7b13b1422fc..fddba75d90745811d6936090af252881ed248c73 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -1045,7 +1045,7 @@ static void v4l_sanitize_format(struct v4l2_format *fmt) /* * The v4l2_pix_format structure has been extended with fields that were * not previously required to be set to zero by applications. The priv - * field, when set to a magic value, indicates the the extended fields + * field, when set to a magic value, indicates that the extended fields * are valid. Otherwise they will contain undefined values. To simplify * the API towards drivers zero the extended fields and set the priv * field to the magic value when the extended pixel format structure @@ -2872,9 +2872,9 @@ static const struct v4l2_ioctl_info v4l2_ioctls[] = { IOCTL_INFO(VIDIOC_S_PRIORITY, v4l_s_priority, v4l_print_u32, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_SLICED_VBI_CAP, v4l_g_sliced_vbi_cap, v4l_print_sliced_vbi_cap, INFO_FL_CLEAR(v4l2_sliced_vbi_cap, type)), IOCTL_INFO(VIDIOC_LOG_STATUS, v4l_log_status, v4l_print_newline, 0), - IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL), - IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL), - IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL), + IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), + IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), + IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_ENUM_FRAMESIZES, v4l_stub_enum_framesizes, v4l_print_frmsizeenum, INFO_FL_CLEAR(v4l2_frmsizeenum, pixel_format)), IOCTL_INFO(VIDIOC_ENUM_FRAMEINTERVALS, v4l_stub_enum_frameintervals, v4l_print_frmivalenum, INFO_FL_CLEAR(v4l2_frmivalenum, height)), IOCTL_INFO(VIDIOC_G_ENC_INDEX, v4l_stub_g_enc_index, v4l_print_enc_idx, 0), @@ -3367,8 +3367,7 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, array_buf = kvmalloc(array_size, GFP_KERNEL); err = -ENOMEM; if (array_buf == NULL) - goto out_array_args; - err = -EFAULT; + goto out; if (in_compat_syscall()) err = v4l2_compat_get_array_args(file, array_buf, user_ptr, array_size, @@ -3377,7 +3376,7 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, err = copy_from_user(array_buf, user_ptr, array_size) ? -EFAULT : 0; if (err) - goto out_array_args; + goto out; *kernel_ptr = array_buf; } @@ -3395,6 +3394,13 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, trace_v4l2_qbuf(video_devdata(file)->minor, parg); } + /* + * Some ioctls can return an error, but still have valid + * results that must be returned. + */ + if (err < 0 && !always_copy) + goto out; + if (has_array_args) { *kernel_ptr = (void __force *)user_ptr; if (in_compat_syscall()) { @@ -3409,16 +3415,8 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, } else if (copy_to_user(user_ptr, array_buf, array_size)) { err = -EFAULT; } - goto out_array_args; } - /* - * Some ioctls can return an error, but still have valid - * results that must be returned. - */ - if (err < 0 && !always_copy) - goto out; -out_array_args: if (video_put_user((void __user *)arg, parg, cmd, orig_cmd)) err = -EFAULT; out: diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem.c index 837e1855f94bfeba9a8c706b646866b0d114abd0..be7fde1ed3eaac17eeea7f668da4150664cd532d 100644 --- a/drivers/media/v4l2-core/v4l2-mem2mem.c +++ b/drivers/media/v4l2-core/v4l2-mem2mem.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Memory-to-memory device framework for Video for Linux 2 and videobuf. + * Memory-to-memory device framework for Video for Linux 2 and vb2. * - * Helper functions for devices that use videobuf buffers for both their + * Helper functions for devices that use vb2 buffers for both their * source and destination. * * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd. @@ -21,7 +21,7 @@ #include #include -MODULE_DESCRIPTION("Mem to mem device framework for videobuf"); +MODULE_DESCRIPTION("Mem to mem device framework for vb2"); MODULE_AUTHOR("Pawel Osciak, "); MODULE_LICENSE("GPL"); diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index ac1a411648d8656fad73bcb3caeea7632025119c..fac290e48e0b86b84aa4d1a20e8e6eaec018b7fc 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -66,6 +66,15 @@ config BRCMSTB_DPFE for the DRAM's temperature. Slower refresh rate means cooler RAM, higher refresh rate means hotter RAM. +config BRCMSTB_MEMC + tristate "Broadcom STB MEMC driver" + default ARCH_BRCMSTB + depends on ARCH_BRCMSTB || COMPILE_TEST + help + This driver provides a way to configure the Broadcom STB memory + controller and specifically control the Self Refresh Power Down + (SRPD) inactivity timeout. + config BT1_L2_CTL bool "Baikal-T1 CM2 L2-RAM Cache Control Block" depends on MIPS_BAIKAL_T1 || COMPILE_TEST diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index bc7663ed1c250350eb64721edcba2bccd42f7624..e148f636c08273a0ea51335c8412025b41fc2f7d 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_ARM_PL172_MPMC) += pl172.o obj-$(CONFIG_ATMEL_SDRAMC) += atmel-sdramc.o obj-$(CONFIG_ATMEL_EBI) += atmel-ebi.o obj-$(CONFIG_BRCMSTB_DPFE) += brcmstb_dpfe.o +obj-$(CONFIG_BRCMSTB_MEMC) += brcmstb_memc.o obj-$(CONFIG_BT1_L2_CTL) += bt1-l2-ctl.o obj-$(CONFIG_TI_AEMIF) += ti-aemif.o obj-$(CONFIG_TI_EMIF) += emif.o diff --git a/drivers/memory/brcmstb_memc.c b/drivers/memory/brcmstb_memc.c new file mode 100644 index 0000000000000000000000000000000000000000..233a53f5bce1f651a1d152a16041421a5782686d --- /dev/null +++ b/drivers/memory/brcmstb_memc.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DDR Self-Refresh Power Down (SRPD) support for Broadcom STB SoCs + * + */ + +#include +#include +#include +#include +#include +#include + +#define REG_MEMC_CNTRLR_CONFIG 0x00 +#define CNTRLR_CONFIG_LPDDR4_SHIFT 5 +#define CNTRLR_CONFIG_MASK 0xf +#define REG_MEMC_SRPD_CFG_21 0x20 +#define REG_MEMC_SRPD_CFG_20 0x34 +#define REG_MEMC_SRPD_CFG_1x 0x3c +#define INACT_COUNT_SHIFT 0 +#define INACT_COUNT_MASK 0xffff +#define SRPD_EN_SHIFT 16 + +struct brcmstb_memc_data { + u32 srpd_offset; +}; + +struct brcmstb_memc { + struct device *dev; + void __iomem *ddr_ctrl; + unsigned int timeout_cycles; + u32 frequency; + u32 srpd_offset; +}; + +static int brcmstb_memc_uses_lpddr4(struct brcmstb_memc *memc) +{ + void __iomem *config = memc->ddr_ctrl + REG_MEMC_CNTRLR_CONFIG; + u32 reg; + + reg = readl_relaxed(config) & CNTRLR_CONFIG_MASK; + + return reg == CNTRLR_CONFIG_LPDDR4_SHIFT; +} + +static int brcmstb_memc_srpd_config(struct brcmstb_memc *memc, + unsigned int cycles) +{ + void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset; + u32 val; + + /* Max timeout supported in HW */ + if (cycles > INACT_COUNT_MASK) + return -EINVAL; + + memc->timeout_cycles = cycles; + + val = (cycles << INACT_COUNT_SHIFT) & INACT_COUNT_MASK; + if (cycles) + val |= BIT(SRPD_EN_SHIFT); + + writel_relaxed(val, cfg); + /* Ensure the write is committed to the controller */ + (void)readl_relaxed(cfg); + + return 0; +} + +static ssize_t frequency_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", memc->frequency); +} + +static ssize_t srpd_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", memc->timeout_cycles); +} + +static ssize_t srpd_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + unsigned int val; + int ret; + + /* + * Cannot change the inactivity timeout on LPDDR4 chips because the + * dynamic tuning process will also get affected by the inactivity + * timeout, thus making it non functional. + */ + if (brcmstb_memc_uses_lpddr4(memc)) + return -EOPNOTSUPP; + + ret = kstrtouint(buf, 10, &val); + if (ret < 0) + return ret; + + ret = brcmstb_memc_srpd_config(memc, val); + if (ret) + return ret; + + return count; +} + +static DEVICE_ATTR_RO(frequency); +static DEVICE_ATTR_RW(srpd); + +static struct attribute *dev_attrs[] = { + &dev_attr_frequency.attr, + &dev_attr_srpd.attr, + NULL, +}; + +static struct attribute_group dev_attr_group = { + .attrs = dev_attrs, +}; + +static const struct of_device_id brcmstb_memc_of_match[]; + +static int brcmstb_memc_probe(struct platform_device *pdev) +{ + const struct brcmstb_memc_data *memc_data; + const struct of_device_id *of_id; + struct device *dev = &pdev->dev; + struct brcmstb_memc *memc; + int ret; + + memc = devm_kzalloc(dev, sizeof(*memc), GFP_KERNEL); + if (!memc) + return -ENOMEM; + + dev_set_drvdata(dev, memc); + + of_id = of_match_device(brcmstb_memc_of_match, dev); + memc_data = of_id->data; + memc->srpd_offset = memc_data->srpd_offset; + + memc->ddr_ctrl = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(memc->ddr_ctrl)) + return PTR_ERR(memc->ddr_ctrl); + + of_property_read_u32(pdev->dev.of_node, "clock-frequency", + &memc->frequency); + + ret = sysfs_create_group(&dev->kobj, &dev_attr_group); + if (ret) + return ret; + + return 0; +} + +static int brcmstb_memc_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + sysfs_remove_group(&dev->kobj, &dev_attr_group); + + return 0; +} + +enum brcmstb_memc_hwtype { + BRCMSTB_MEMC_V21, + BRCMSTB_MEMC_V20, + BRCMSTB_MEMC_V1X, +}; + +static const struct brcmstb_memc_data brcmstb_memc_versions[] = { + { .srpd_offset = REG_MEMC_SRPD_CFG_21 }, + { .srpd_offset = REG_MEMC_SRPD_CFG_20 }, + { .srpd_offset = REG_MEMC_SRPD_CFG_1x }, +}; + +static const struct of_device_id brcmstb_memc_of_match[] = { + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V20] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.2", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.3", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.5", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.6", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.7", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.2.8", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.3.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-b.3.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.0", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.1", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.2", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.3", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + { + .compatible = "brcm,brcmstb-memc-ddr-rev-c.1.4", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21] + }, + /* default to the original offset */ + { + .compatible = "brcm,brcmstb-memc-ddr", + .data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X] + }, + {} +}; + +static int brcmstb_memc_suspend(struct device *dev) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset; + u32 val; + + if (memc->timeout_cycles == 0) + return 0; + + /* + * Disable SRPD prior to suspending the system since that can + * cause issues with other memory clients managed by the ARM + * trusted firmware to access memory. + */ + val = readl_relaxed(cfg); + val &= ~BIT(SRPD_EN_SHIFT); + writel_relaxed(val, cfg); + /* Ensure the write is committed to the controller */ + (void)readl_relaxed(cfg); + + return 0; +} + +static int brcmstb_memc_resume(struct device *dev) +{ + struct brcmstb_memc *memc = dev_get_drvdata(dev); + + if (memc->timeout_cycles == 0) + return 0; + + return brcmstb_memc_srpd_config(memc, memc->timeout_cycles); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(brcmstb_memc_pm_ops, brcmstb_memc_suspend, + brcmstb_memc_resume); + +static struct platform_driver brcmstb_memc_driver = { + .probe = brcmstb_memc_probe, + .remove = brcmstb_memc_remove, + .driver = { + .name = "brcmstb_memc", + .of_match_table = brcmstb_memc_of_match, + .pm = pm_ptr(&brcmstb_memc_pm_ops), + }, +}; +module_platform_driver(brcmstb_memc_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Broadcom"); +MODULE_DESCRIPTION("DDR SRPD driver for Broadcom STB chips"); diff --git a/drivers/memory/dfl-emif.c b/drivers/memory/dfl-emif.c index 3f719816771d4fbeb651736fba028f883e234fa4..da06cd30a01617389a211387c233f1e27ce3e854 100644 --- a/drivers/memory/dfl-emif.c +++ b/drivers/memory/dfl-emif.c @@ -24,11 +24,24 @@ #define EMIF_STAT_CLEAR_BUSY_SFT 16 #define EMIF_CTRL 0x10 #define EMIF_CTRL_CLEAR_EN_SFT 0 -#define EMIF_CTRL_CLEAR_EN_MSK GENMASK_ULL(3, 0) +#define EMIF_CTRL_CLEAR_EN_MSK GENMASK_ULL(7, 0) #define EMIF_POLL_INVL 10000 /* us */ #define EMIF_POLL_TIMEOUT 5000000 /* us */ +/* + * The Capability Register replaces the Control Register (at the same + * offset) for EMIF feature revisions > 0. The bitmask that indicates + * the presence of memory channels exists in both the Capability Register + * and Control Register definitions. These can be thought of as a C union. + * The Capability Register definitions are used to check for the existence + * of a memory channel, and the Control Register definitions are used for + * managing the memory-clear functionality in revision 0. + */ +#define EMIF_CAPABILITY_BASE 0x10 +#define EMIF_CAPABILITY_CHN_MSK_V0 GENMASK_ULL(3, 0) +#define EMIF_CAPABILITY_CHN_MSK GENMASK_ULL(7, 0) + struct dfl_emif { struct device *dev; void __iomem *base; @@ -106,16 +119,30 @@ emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 0); emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 1); emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 2); emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 3); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 4); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 5); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 6); +emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 7); emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 0); emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 1); emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 2); emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 3); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 4); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 5); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 6); +emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 7); + emif_clear_attr(0); emif_clear_attr(1); emif_clear_attr(2); emif_clear_attr(3); +emif_clear_attr(4); +emif_clear_attr(5); +emif_clear_attr(6); +emif_clear_attr(7); + static struct attribute *dfl_emif_attrs[] = { &emif_attr_inf0_init_done.attr.attr, @@ -134,6 +161,22 @@ static struct attribute *dfl_emif_attrs[] = { &emif_attr_inf3_cal_fail.attr.attr, &emif_attr_inf3_clear.attr.attr, + &emif_attr_inf4_init_done.attr.attr, + &emif_attr_inf4_cal_fail.attr.attr, + &emif_attr_inf4_clear.attr.attr, + + &emif_attr_inf5_init_done.attr.attr, + &emif_attr_inf5_cal_fail.attr.attr, + &emif_attr_inf5_clear.attr.attr, + + &emif_attr_inf6_init_done.attr.attr, + &emif_attr_inf6_cal_fail.attr.attr, + &emif_attr_inf6_clear.attr.attr, + + &emif_attr_inf7_init_done.attr.attr, + &emif_attr_inf7_cal_fail.attr.attr, + &emif_attr_inf7_clear.attr.attr, + NULL, }; @@ -143,15 +186,24 @@ static umode_t dfl_emif_visible(struct kobject *kobj, struct dfl_emif *de = dev_get_drvdata(kobj_to_dev(kobj)); struct emif_attr *eattr = container_of(attr, struct emif_attr, attr.attr); + struct dfl_device *ddev = to_dfl_dev(de->dev); u64 val; /* - * This device supports upto 4 memory interfaces, but not all + * This device supports up to 8 memory interfaces, but not all * interfaces are used on different platforms. The read out value of - * CLEAN_EN field (which is a bitmap) could tell how many interfaces - * are available. + * CAPABILITY_CHN_MSK field (which is a bitmap) indicates which + * interfaces are available. */ - val = FIELD_GET(EMIF_CTRL_CLEAR_EN_MSK, readq(de->base + EMIF_CTRL)); + if (ddev->revision > 0 && strstr(attr->name, "_clear")) + return 0; + + if (ddev->revision == 0) + val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK_V0, + readq(de->base + EMIF_CAPABILITY_BASE)); + else + val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK, + readq(de->base + EMIF_CAPABILITY_BASE)); return (val & BIT_ULL(eattr->index)) ? attr->mode : 0; } diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c index d7cb7ead2ac7fc04f52af23b14f3fcb76cdd42e9..5a9754442bc75fa3f883b65c3461a1ca0d8c5f29 100644 --- a/drivers/memory/mtk-smi.c +++ b/drivers/memory/mtk-smi.c @@ -3,6 +3,7 @@ * Copyright (c) 2015-2016 MediaTek Inc. * Author: Yong Wu */ +#include #include #include #include @@ -14,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +91,7 @@ #define MTK_SMI_FLAG_THRT_UPDATE BIT(0) #define MTK_SMI_FLAG_SW_FLAG BIT(1) #define MTK_SMI_FLAG_SLEEP_CTL BIT(2) +#define MTK_SMI_FLAG_CFG_PORT_SEC_CTL BIT(3) #define MTK_SMI_CAPS(flags, _x) (!!((flags) & (_x))) struct mtk_smi_reg_pair { @@ -127,7 +130,7 @@ struct mtk_smi_common_plat { struct mtk_smi_larb_gen { int port_in_larb[MTK_LARB_NR_MAX + 1]; - void (*config_port)(struct device *dev); + int (*config_port)(struct device *dev); unsigned int larb_direct_to_common_mask; unsigned int flags_general; const u8 (*ostd)[SMI_LARB_PORT_NR_MAX]; @@ -185,7 +188,7 @@ static const struct component_ops mtk_smi_larb_component_ops = { .unbind = mtk_smi_larb_unbind, }; -static void mtk_smi_larb_config_port_gen1(struct device *dev) +static int mtk_smi_larb_config_port_gen1(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen; @@ -214,31 +217,35 @@ static void mtk_smi_larb_config_port_gen1(struct device *dev) common->smi_ao_base + REG_SMI_SECUR_CON_ADDR(m4u_port_id)); } + return 0; } -static void mtk_smi_larb_config_port_mt8167(struct device *dev) +static int mtk_smi_larb_config_port_mt8167(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); writel(*larb->mmu, larb->base + MT8167_SMI_LARB_MMU_EN); + return 0; } -static void mtk_smi_larb_config_port_mt8173(struct device *dev) +static int mtk_smi_larb_config_port_mt8173(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); writel(*larb->mmu, larb->base + MT8173_SMI_LARB_MMU_EN); + return 0; } -static void mtk_smi_larb_config_port_gen2_general(struct device *dev) +static int mtk_smi_larb_config_port_gen2_general(struct device *dev) { struct mtk_smi_larb *larb = dev_get_drvdata(dev); u32 reg, flags_general = larb->larb_gen->flags_general; const u8 *larbostd = larb->larb_gen->ostd ? larb->larb_gen->ostd[larb->larbid] : NULL; + struct arm_smccc_res res; int i; if (BIT(larb->larbid) & larb->larb_gen->larb_direct_to_common_mask) - return; + return 0; if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_THRT_UPDATE)) { reg = readl_relaxed(larb->base + SMI_LARB_CMD_THRT_CON); @@ -253,14 +260,78 @@ static void mtk_smi_larb_config_port_gen2_general(struct device *dev) for (i = 0; i < SMI_LARB_PORT_NR_MAX && larbostd && !!larbostd[i]; i++) writel_relaxed(larbostd[i], larb->base + SMI_LARB_OSTDL_PORTx(i)); + /* + * When mmu_en bits are in security world, the bank_sel still is in the + * LARB_NONSEC_CON below. And the mmu_en bits of LARB_NONSEC_CON have no + * effect in this case. + */ + if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_CFG_PORT_SEC_CTL)) { + arm_smccc_smc(MTK_SIP_KERNEL_IOMMU_CONTROL, IOMMU_ATF_CMD_CONFIG_SMI_LARB, + larb->larbid, *larb->mmu, 0, 0, 0, 0, &res); + if (res.a0 != 0) { + dev_err(dev, "Enable iommu fail, ret %ld\n", res.a0); + return -EINVAL; + } + } + for_each_set_bit(i, (unsigned long *)larb->mmu, 32) { reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i)); reg |= F_MMU_EN; reg |= BANK_SEL(larb->bank[i]); writel(reg, larb->base + SMI_LARB_NONSEC_CON(i)); } + return 0; } +static const u8 mtk_smi_larb_mt8188_ostd[][SMI_LARB_PORT_NR_MAX] = { + [0] = {0x02, 0x18, 0x22, 0x22, 0x01, 0x02, 0x0a,}, + [1] = {0x12, 0x02, 0x14, 0x14, 0x01, 0x18, 0x0a,}, + [2] = {0x12, 0x12, 0x12, 0x12, 0x0a,}, + [3] = {0x12, 0x12, 0x12, 0x12, 0x28, 0x28, 0x0a,}, + [4] = {0x06, 0x01, 0x17, 0x06, 0x0a, 0x07, 0x07,}, + [5] = {0x02, 0x01, 0x04, 0x02, 0x06, 0x01, 0x06, 0x0a,}, + [6] = {0x06, 0x01, 0x06, 0x0a,}, + [7] = {0x0c, 0x0c, 0x12,}, + [8] = {0x0c, 0x01, 0x0a, 0x05, 0x02, 0x03, 0x01, 0x01, 0x14, 0x14, + 0x0a, 0x14, 0x1e, 0x01, 0x0c, 0x0a, 0x05, 0x02, 0x02, 0x05, + 0x03, 0x01, 0x1e, 0x01, 0x05,}, + [9] = {0x1e, 0x01, 0x0a, 0x0a, 0x01, 0x01, 0x03, 0x1e, 0x1e, 0x10, + 0x07, 0x01, 0x0a, 0x06, 0x03, 0x03, 0x0e, 0x01, 0x04, 0x28,}, + [10] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [11] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [12] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c, + 0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14, + 0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,}, + [13] = {0x07, 0x02, 0x04, 0x02, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x07, 0x02, 0x04, 0x02, 0x05, 0x05,}, + [14] = {0x02, 0x02, 0x0c, 0x0c, 0x0c, 0x0c, 0x01, 0x01, 0x02, 0x02, + 0x02, 0x02, 0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x01, 0x01,}, + [15] = {0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x01, 0x01, 0x0c, 0x0c, + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x01, 0x02, + 0x0c, 0x01, 0x01,}, + [16] = {0x28, 0x28, 0x03, 0x01, 0x01, 0x03, 0x14, 0x14, 0x0a, 0x0d, + 0x03, 0x05, 0x0e, 0x01, 0x01, 0x05, 0x06, 0x0d, 0x01,}, + [17] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a, + 0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,}, + [18] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a, + 0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,}, + [19] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, + [20] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,}, + [21] = {0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x01, + 0x01, 0x01, 0x04, 0x0a, 0x06, 0x01, 0x01, 0x01, 0x0a, 0x06, + 0x01, 0x01, 0x05, 0x03, 0x03, 0x04, 0x01,}, + [22] = {0x28, 0x19, 0x0c, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, + 0x01,}, + [23] = {0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x18, 0x01, 0x01,}, + [24] = {0x12, 0x06, 0x12, 0x06,}, + [25] = {0x01}, +}; + static const u8 mtk_smi_larb_mt8195_ostd[][SMI_LARB_PORT_NR_MAX] = { [0] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb0 */ [1] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb1 */ @@ -347,6 +418,13 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt8186 = { .flags_general = MTK_SMI_FLAG_SLEEP_CTL, }; +static const struct mtk_smi_larb_gen mtk_smi_larb_mt8188 = { + .config_port = mtk_smi_larb_config_port_gen2_general, + .flags_general = MTK_SMI_FLAG_THRT_UPDATE | MTK_SMI_FLAG_SW_FLAG | + MTK_SMI_FLAG_SLEEP_CTL | MTK_SMI_FLAG_CFG_PORT_SEC_CTL, + .ostd = mtk_smi_larb_mt8188_ostd, +}; + static const struct mtk_smi_larb_gen mtk_smi_larb_mt8192 = { .config_port = mtk_smi_larb_config_port_gen2_general, }; @@ -367,6 +445,7 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = { {.compatible = "mediatek,mt8173-smi-larb", .data = &mtk_smi_larb_mt8173}, {.compatible = "mediatek,mt8183-smi-larb", .data = &mtk_smi_larb_mt8183}, {.compatible = "mediatek,mt8186-smi-larb", .data = &mtk_smi_larb_mt8186}, + {.compatible = "mediatek,mt8188-smi-larb", .data = &mtk_smi_larb_mt8188}, {.compatible = "mediatek,mt8192-smi-larb", .data = &mtk_smi_larb_mt8192}, {.compatible = "mediatek,mt8195-smi-larb", .data = &mtk_smi_larb_mt8195}, {} @@ -511,9 +590,7 @@ static int __maybe_unused mtk_smi_larb_resume(struct device *dev) mtk_smi_larb_sleep_ctrl_disable(larb); /* Configure the basic setting for this larb */ - larb_gen->config_port(dev); - - return 0; + return larb_gen->config_port(dev); } static int __maybe_unused mtk_smi_larb_suspend(struct device *dev) @@ -597,6 +674,18 @@ static const struct mtk_smi_common_plat mtk_smi_common_mt8186 = { .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(4) | F_MMU1_LARB(7), }; +static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vdo = { + .type = MTK_SMI_GEN2, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(5) | F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + +static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vpp = { + .type = MTK_SMI_GEN2, + .bus_sel = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(7), + .init = mtk_smi_common_mt8195_init, +}; + static const struct mtk_smi_common_plat mtk_smi_common_mt8192 = { .type = MTK_SMI_GEN2, .has_gals = true, @@ -633,6 +722,8 @@ static const struct of_device_id mtk_smi_common_of_ids[] = { {.compatible = "mediatek,mt8173-smi-common", .data = &mtk_smi_common_gen2}, {.compatible = "mediatek,mt8183-smi-common", .data = &mtk_smi_common_mt8183}, {.compatible = "mediatek,mt8186-smi-common", .data = &mtk_smi_common_mt8186}, + {.compatible = "mediatek,mt8188-smi-common-vdo", .data = &mtk_smi_common_mt8188_vdo}, + {.compatible = "mediatek,mt8188-smi-common-vpp", .data = &mtk_smi_common_mt8188_vpp}, {.compatible = "mediatek,mt8192-smi-common", .data = &mtk_smi_common_mt8192}, {.compatible = "mediatek,mt8195-smi-common-vdo", .data = &mtk_smi_common_mt8195_vdo}, {.compatible = "mediatek,mt8195-smi-common-vpp", .data = &mtk_smi_common_mt8195_vpp}, diff --git a/drivers/memory/of_memory.c b/drivers/memory/of_memory.c index dbdf87bc0b78efd89f6df696149d899df9bccf21..fcd20d85d38574a7d98b42a48eb84b10b032199e 100644 --- a/drivers/memory/of_memory.c +++ b/drivers/memory/of_memory.c @@ -134,6 +134,7 @@ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr, for_each_child_of_node(np_ddr, np_tim) { if (of_device_is_compatible(np_tim, tim_compat)) { if (of_do_get_timings(np_tim, &timings[i])) { + of_node_put(np_tim); devm_kfree(dev, timings); goto default_timings; } @@ -284,6 +285,7 @@ const struct lpddr3_timings if (of_device_is_compatible(np_tim, tim_compat)) { if (of_lpddr3_do_get_timings(np_tim, &timings[i])) { devm_kfree(dev, timings); + of_node_put(np_tim); goto default_timings; } i++; diff --git a/drivers/memory/pl353-smc.c b/drivers/memory/pl353-smc.c index f84b98278745c827cde481573b3493d846c93f82..d39ee7d06665bdb8485f505ff8fe19f922e52484 100644 --- a/drivers/memory/pl353-smc.c +++ b/drivers/memory/pl353-smc.c @@ -122,6 +122,7 @@ static int pl353_smc_probe(struct amba_device *adev, const struct amba_id *id) } of_platform_device_create(child, NULL, &adev->dev); + of_node_put(child); return 0; diff --git a/drivers/message/fusion/mptctl.c b/drivers/message/fusion/mptctl.c index f9ee957072c39ad1e385c4f1e72c1f674e822415..52c7020c9d191207e2df1154c5a9cb4e955d5078 100644 --- a/drivers/message/fusion/mptctl.c +++ b/drivers/message/fusion/mptctl.c @@ -620,7 +620,6 @@ __mptctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { mpt_ioctl_header __user *uhdr = (void __user *) arg; mpt_ioctl_header khdr; - int iocnum; unsigned iocnumX; int nonblock = (file->f_flags & O_NONBLOCK); int ret; @@ -634,12 +633,11 @@ __mptctl_ioctl(struct file *file, unsigned int cmd, unsigned long arg) } ret = -ENXIO; /* (-6) No such device or address */ - /* Verify intended MPT adapter - set iocnum and the adapter + /* Verify intended MPT adapter - set iocnumX and the adapter * pointer (iocp) */ iocnumX = khdr.iocnum & 0xFF; - if (((iocnum = mpt_verify_adapter(iocnumX, &iocp)) < 0) || - (iocp == NULL)) + if ((mpt_verify_adapter(iocnumX, &iocp) < 0) || (iocp == NULL)) return -ENODEV; if (!iocp->active) { diff --git a/drivers/mfd/88pm800.c b/drivers/mfd/88pm800.c index eaf9845633b4d60a1346579b5687ca16fac5224e..a30e47b7432705b842b98d0a6dc998b57ecf12de 100644 --- a/drivers/mfd/88pm800.c +++ b/drivers/mfd/88pm800.c @@ -583,7 +583,7 @@ out_init: return ret; } -static int pm800_remove(struct i2c_client *client) +static void pm800_remove(struct i2c_client *client) { struct pm80x_chip *chip = i2c_get_clientdata(client); @@ -592,8 +592,6 @@ static int pm800_remove(struct i2c_client *client) pm800_pages_exit(chip); pm80x_deinit(); - - return 0; } static struct i2c_driver pm800_driver = { diff --git a/drivers/mfd/88pm805.c b/drivers/mfd/88pm805.c index ada6c513302b7c8d3f312cfd03401518e1929923..10d3637840c8cc152fba4411a0b5879459f89f7e 100644 --- a/drivers/mfd/88pm805.c +++ b/drivers/mfd/88pm805.c @@ -239,7 +239,7 @@ out_init: return ret; } -static int pm805_remove(struct i2c_client *client) +static void pm805_remove(struct i2c_client *client) { struct pm80x_chip *chip = i2c_get_clientdata(client); @@ -247,8 +247,6 @@ static int pm805_remove(struct i2c_client *client) device_irq_exit_805(chip); pm80x_deinit(); - - return 0; } static struct i2c_driver pm805_driver = { diff --git a/drivers/mfd/88pm860x-core.c b/drivers/mfd/88pm860x-core.c index b1e829ea909ba33300b5c0b27b17007e1688d7b2..5dc86dd66202f4becb82dac4c5af22a779cc53e6 100644 --- a/drivers/mfd/88pm860x-core.c +++ b/drivers/mfd/88pm860x-core.c @@ -1201,7 +1201,7 @@ static int pm860x_probe(struct i2c_client *client) return 0; } -static int pm860x_remove(struct i2c_client *client) +static void pm860x_remove(struct i2c_client *client) { struct pm860x_chip *chip = i2c_get_clientdata(client); @@ -1210,7 +1210,6 @@ static int pm860x_remove(struct i2c_client *client) regmap_exit(chip->regmap_companion); i2c_unregister_device(chip->companion); } - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index abb58ab1a1a4f8871e428f05fb6430788190577e..8b93856de432aeede26b670ec190ea45260569b0 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -589,8 +589,8 @@ config LPC_SCH config INTEL_SOC_PMIC bool "Support for Crystal Cove PMIC" - depends on ACPI && HAS_IOMEM && I2C=y && GPIOLIB && COMMON_CLK - depends on X86 || COMPILE_TEST + depends on HAS_IOMEM && I2C=y && GPIOLIB && COMMON_CLK + depends on (X86 && ACPI) || COMPILE_TEST depends on I2C_DESIGNWARE_PLATFORM=y select MFD_CORE select REGMAP_I2C @@ -938,6 +938,22 @@ config MFD_MT6360 PMIC part includes 2-channel BUCKs and 2-channel LDOs LDO part includes 4-channel LDOs +config MFD_MT6370 + tristate "MediaTek MT6370 SubPMIC" + select MFD_CORE + select REGMAP_I2C + select REGMAP_IRQ + depends on I2C + help + Say Y here to enable MT6370 SubPMIC functional support. + It consists of a single cell battery charger with ADC monitoring, RGB + LEDs, dual channel flashlight, WLED backlight driver, display bias + voltage supply, one general purpose LDO, and the USB Type-C & PD + controller complies with the latest USB Type-C and PD standards. + + This driver can also be built as a module. If so, the module + will be called "mt6370". + config MFD_MT6397 tristate "MediaTek MT6397 PMIC Support" select MFD_CORE @@ -963,6 +979,27 @@ config MFD_MENF21BMC This driver can also be built as a module. If so the module will be called menf21bmc. +config MFD_OCELOT + tristate "Microsemi Ocelot External Control Support" + depends on SPI_MASTER + select MFD_CORE + select REGMAP_SPI + help + Ocelot is a family of networking chips that support multiple ethernet + and fibre interfaces. In addition to networking, they contain several + other functions, including pinctrl, MDIO, and communication with + external chips. While some chips have an internal processor capable of + running an OS, others don't. All chips can be controlled externally + through different interfaces, including SPI, I2C, and PCIe. + + Say yes here to add support for Ocelot chips (VSC7511, VSC7512, + VSC7513, VSC7514) controlled externally. + + To compile this driver as a module, choose M here: the module will be + called ocelot-soc. + + If unsure, say N. + config EZX_PCAP bool "Motorola EZXPCAP Support" depends on SPI_MASTER @@ -1096,6 +1133,16 @@ config MFD_SPMI_PMIC Say M here if you want to include support for the SPMI PMIC series as a module. The module will be called "qcom-spmi-pmic". +config MFD_SY7636A + tristate "Silergy SY7636A voltage regulator" + depends on I2C + select MFD_SIMPLE_MFD_I2C + help + Enable support for Silergy SY7636A voltage regulator. + + To enable support for building sub-devices as modules, + choose M here. + config MFD_RDC321X tristate "RDC R-321x southbridge" select MFD_CORE @@ -1128,6 +1175,18 @@ config MFD_RT5033 sub-devices like charger, fuel gauge, flash LED, current source, LDO and Buck. +config MFD_RT5120 + tristate "Richtek RT5120 Power Management IC" + depends on I2C + select MFD_CORE + select REGMAP_I2C + select REGMAP_IRQ + help + The enables support for Richtek RT5120 PMIC. It includes four high + efficiency buck converters and one LDO voltage regulator. The device + is targeted at providing the CPU voltage, memory, I/O and peripheral + power rails in home entertainment devices. + config MFD_RC5T583 bool "Ricoh RC5T583 Power Management system device" depends on I2C=y @@ -1203,7 +1262,7 @@ config MFD_SI476X_CORE module will be called si476x-core. config MFD_SIMPLE_MFD_I2C - tristate "Simple Multi-Functional Device support (I2C)" + tristate depends on I2C select MFD_CORE select REGMAP_I2C diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile index 858cacf659d6584d535e93eddcca7699c40a8783..7ed3ef4a698cf2299404b58cee9c09e44c154b21 100644 --- a/drivers/mfd/Makefile +++ b/drivers/mfd/Makefile @@ -120,6 +120,9 @@ obj-$(CONFIG_MFD_MC13XXX_I2C) += mc13xxx-i2c.o obj-$(CONFIG_MFD_CORE) += mfd-core.o +ocelot-soc-objs := ocelot-core.o ocelot-spi.o +obj-$(CONFIG_MFD_OCELOT) += ocelot-soc.o + obj-$(CONFIG_EZX_PCAP) += ezx-pcap.o obj-$(CONFIG_MFD_CPCAP) += motorola-cpcap.o @@ -172,6 +175,11 @@ obj-$(CONFIG_MFD_MAX8998) += max8998.o max8998-irq.o obj-$(CONFIG_MFD_MP2629) += mp2629.o +obj-$(CONFIG_MFD_MT6360) += mt6360-core.o +obj-$(CONFIG_MFD_MT6370) += mt6370.o +mt6397-objs := mt6397-core.o mt6397-irq.o mt6358-irq.o +obj-$(CONFIG_MFD_MT6397) += mt6397.o + pcf50633-objs := pcf50633-core.o pcf50633-irq.o obj-$(CONFIG_MFD_PCF50633) += pcf50633.o obj-$(CONFIG_PCF50633_ADC) += pcf50633-adc.o @@ -234,16 +242,13 @@ obj-$(CONFIG_MFD_HI655X_PMIC) += hi655x-pmic.o obj-$(CONFIG_MFD_DLN2) += dln2.o obj-$(CONFIG_MFD_RT4831) += rt4831.o obj-$(CONFIG_MFD_RT5033) += rt5033.o +obj-$(CONFIG_MFD_RT5120) += rt5120.o obj-$(CONFIG_MFD_SKY81452) += sky81452.o -intel-soc-pmic-objs := intel_soc_pmic_core.o intel_soc_pmic_crc.o -obj-$(CONFIG_INTEL_SOC_PMIC) += intel-soc-pmic.o +obj-$(CONFIG_INTEL_SOC_PMIC) += intel_soc_pmic_crc.o obj-$(CONFIG_INTEL_SOC_PMIC_BXTWC) += intel_soc_pmic_bxtwc.o obj-$(CONFIG_INTEL_SOC_PMIC_CHTWC) += intel_soc_pmic_chtwc.o obj-$(CONFIG_INTEL_SOC_PMIC_CHTDC_TI) += intel_soc_pmic_chtdc_ti.o -obj-$(CONFIG_MFD_MT6360) += mt6360-core.o -mt6397-objs := mt6397-core.o mt6397-irq.o mt6358-irq.o -obj-$(CONFIG_MFD_MT6397) += mt6397.o obj-$(CONFIG_INTEL_SOC_PMIC_MRFLD) += intel_soc_pmic_mrfld.o obj-$(CONFIG_MFD_ALTERA_A10SR) += altera-a10sr.o diff --git a/drivers/mfd/acer-ec-a500.c b/drivers/mfd/acer-ec-a500.c index 80c2fdd14fc492b3c87c29240133dfb0c39fb42b..7fd8b998807586c72f1b76f4119a5c8193f6d61e 100644 --- a/drivers/mfd/acer-ec-a500.c +++ b/drivers/mfd/acer-ec-a500.c @@ -169,7 +169,7 @@ static int a500_ec_probe(struct i2c_client *client) return 0; } -static int a500_ec_remove(struct i2c_client *client) +static void a500_ec_remove(struct i2c_client *client) { if (of_device_is_system_power_controller(client->dev.of_node)) { if (pm_power_off == a500_ec_poweroff) @@ -177,8 +177,6 @@ static int a500_ec_remove(struct i2c_client *client) unregister_restart_handler(&a500_ec_restart_handler); } - - return 0; } static const struct of_device_id a500_ec_match[] = { diff --git a/drivers/mfd/arizona-i2c.c b/drivers/mfd/arizona-i2c.c index 6d83e6b9a692d9c19d6a0d6f6bb42cc5612df76b..bfc7cf56ff2c7401b1b4d6ebee50ac08fdfc0915 100644 --- a/drivers/mfd/arizona-i2c.c +++ b/drivers/mfd/arizona-i2c.c @@ -84,13 +84,11 @@ static int arizona_i2c_probe(struct i2c_client *i2c, return arizona_dev_init(arizona); } -static int arizona_i2c_remove(struct i2c_client *i2c) +static void arizona_i2c_remove(struct i2c_client *i2c) { struct arizona *arizona = dev_get_drvdata(&i2c->dev); arizona_dev_exit(arizona); - - return 0; } static const struct i2c_device_id arizona_i2c_id[] = { diff --git a/drivers/mfd/axp20x-i2c.c b/drivers/mfd/axp20x-i2c.c index 00ab48018d8d2e4dcf934ded988f89c960c92e7d..8fd6727dc30a1ee5ca428d77b2a0f081a5044f77 100644 --- a/drivers/mfd/axp20x-i2c.c +++ b/drivers/mfd/axp20x-i2c.c @@ -50,13 +50,11 @@ static int axp20x_i2c_probe(struct i2c_client *i2c, return axp20x_device_probe(axp20x); } -static int axp20x_i2c_remove(struct i2c_client *i2c) +static void axp20x_i2c_remove(struct i2c_client *i2c) { struct axp20x_dev *axp20x = i2c_get_clientdata(i2c); axp20x_device_remove(axp20x); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/mfd/da903x.c b/drivers/mfd/da903x.c index a818fbb559888657bc5fc4cd5b948683d5104cf8..3f8f6ad3a98c41f3a643439cdbc57d5500348454 100644 --- a/drivers/mfd/da903x.c +++ b/drivers/mfd/da903x.c @@ -532,12 +532,11 @@ static int da903x_probe(struct i2c_client *client, return da903x_add_subdevs(chip, pdata); } -static int da903x_remove(struct i2c_client *client) +static void da903x_remove(struct i2c_client *client) { struct da903x_chip *chip = i2c_get_clientdata(client); da903x_remove_subdevs(chip); - return 0; } static struct i2c_driver da903x_driver = { diff --git a/drivers/mfd/da9052-i2c.c b/drivers/mfd/da9052-i2c.c index 8de93db35f3a31be9d017e9fd19f740f3743835e..5a74696c8704ff7ea0c3bcd974e8816c4215bf93 100644 --- a/drivers/mfd/da9052-i2c.c +++ b/drivers/mfd/da9052-i2c.c @@ -168,12 +168,11 @@ static int da9052_i2c_probe(struct i2c_client *client, return da9052_device_init(da9052, id->driver_data); } -static int da9052_i2c_remove(struct i2c_client *client) +static void da9052_i2c_remove(struct i2c_client *client) { struct da9052 *da9052 = i2c_get_clientdata(client); da9052_device_exit(da9052); - return 0; } static struct i2c_driver da9052_i2c_driver = { diff --git a/drivers/mfd/da9055-i2c.c b/drivers/mfd/da9055-i2c.c index bc60433b68db7d128d3019addda77fd6d2eb376b..276c7d1c509e0e2439f86a8ceecb39855d3a63cb 100644 --- a/drivers/mfd/da9055-i2c.c +++ b/drivers/mfd/da9055-i2c.c @@ -41,13 +41,11 @@ static int da9055_i2c_probe(struct i2c_client *i2c, return da9055_device_init(da9055); } -static int da9055_i2c_remove(struct i2c_client *i2c) +static void da9055_i2c_remove(struct i2c_client *i2c) { struct da9055 *da9055 = i2c_get_clientdata(i2c); da9055_device_exit(da9055); - - return 0; } /* diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c index 2774b2cbaea6d8e9b8166cf08231544e8339f941..a26e473507c7110614af8ec1e459b5d73848bcc8 100644 --- a/drivers/mfd/da9062-core.c +++ b/drivers/mfd/da9062-core.c @@ -453,6 +453,7 @@ static const struct regmap_range da9061_aa_writeable_ranges[] = { regmap_reg_range(DA9062AA_VBUCK1_B, DA9062AA_VBUCK4_B), regmap_reg_range(DA9062AA_VBUCK3_B, DA9062AA_VBUCK3_B), regmap_reg_range(DA9062AA_VLDO1_B, DA9062AA_VLDO4_B), + regmap_reg_range(DA9062AA_CONFIG_J, DA9062AA_CONFIG_J), regmap_reg_range(DA9062AA_GP_ID_0, DA9062AA_GP_ID_19), }; @@ -723,14 +724,12 @@ static int da9062_i2c_probe(struct i2c_client *i2c, return ret; } -static int da9062_i2c_remove(struct i2c_client *i2c) +static void da9062_i2c_remove(struct i2c_client *i2c) { struct da9062 *chip = i2c_get_clientdata(i2c); mfd_remove_devices(chip->dev); regmap_del_irq_chip(i2c->irq, chip->regmap_irq); - - return 0; } static const struct i2c_device_id da9062_i2c_id[] = { diff --git a/drivers/mfd/da9150-core.c b/drivers/mfd/da9150-core.c index 58009c8cb87058f54f9ed9f1898737a9f48a2270..6ae56e46d24e6658448fa9fdc60c67581c2d11c7 100644 --- a/drivers/mfd/da9150-core.c +++ b/drivers/mfd/da9150-core.c @@ -471,15 +471,13 @@ regmap_irq_fail: return ret; } -static int da9150_remove(struct i2c_client *client) +static void da9150_remove(struct i2c_client *client) { struct da9150 *da9150 = i2c_get_clientdata(client); regmap_del_irq_chip(da9150->irq, da9150->regmap_irq_data); mfd_remove_devices(da9150->dev); i2c_unregister_device(da9150->core_qif); - - return 0; } static void da9150_shutdown(struct i2c_client *client) diff --git a/drivers/mfd/dm355evm_msp.c b/drivers/mfd/dm355evm_msp.c index 54fb6cbd2aa06ade3bc2f96d0aa01ea8994016e0..759c596906801cc2453be4ff498b4039d0a98d78 100644 --- a/drivers/mfd/dm355evm_msp.c +++ b/drivers/mfd/dm355evm_msp.c @@ -375,11 +375,10 @@ static void dm355evm_power_off(void) dm355evm_command(MSP_COMMAND_POWEROFF); } -static int dm355evm_msp_remove(struct i2c_client *client) +static void dm355evm_msp_remove(struct i2c_client *client) { pm_power_off = NULL; msp430 = NULL; - return 0; } static int diff --git a/drivers/mfd/ene-kb3930.c b/drivers/mfd/ene-kb3930.c index 1b73318d1f1fbb1f3e3e366bcc3058513406d4db..3eff98e26bea9a33c6c2299291f91e4f84b73fb7 100644 --- a/drivers/mfd/ene-kb3930.c +++ b/drivers/mfd/ene-kb3930.c @@ -177,7 +177,7 @@ static int kb3930_probe(struct i2c_client *client) return 0; } -static int kb3930_remove(struct i2c_client *client) +static void kb3930_remove(struct i2c_client *client) { struct kb3930 *ddata = i2c_get_clientdata(client); @@ -187,8 +187,6 @@ static int kb3930_remove(struct i2c_client *client) unregister_restart_handler(&kb3930_restart_nb); } kb3930_power_off = NULL; - - return 0; } static const struct of_device_id kb3930_dt_ids[] = { diff --git a/drivers/mfd/fsl-imx25-tsadc.c b/drivers/mfd/fsl-imx25-tsadc.c index 37e5e02a1d059024ceebf29ed607d362396536e1..823595bcc9b7c6de645068309ec28a618a4c6465 100644 --- a/drivers/mfd/fsl-imx25-tsadc.c +++ b/drivers/mfd/fsl-imx25-tsadc.c @@ -69,7 +69,7 @@ static int mx25_tsadc_setup_irq(struct platform_device *pdev, int irq; irq = platform_get_irq(pdev, 0); - if (irq <= 0) + if (irq < 0) return irq; tsadc->domain = irq_domain_add_simple(np, 2, 0, &mx25_tsadc_domain_ops, @@ -84,6 +84,19 @@ static int mx25_tsadc_setup_irq(struct platform_device *pdev, return 0; } +static int mx25_tsadc_unset_irq(struct platform_device *pdev) +{ + struct mx25_tsadc *tsadc = platform_get_drvdata(pdev); + int irq = platform_get_irq(pdev, 0); + + if (irq >= 0) { + irq_set_chained_handler_and_data(irq, NULL, NULL); + irq_domain_remove(tsadc->domain); + } + + return 0; +} + static void mx25_tsadc_setup_clk(struct platform_device *pdev, struct mx25_tsadc *tsadc) { @@ -171,18 +184,21 @@ static int mx25_tsadc_probe(struct platform_device *pdev) platform_set_drvdata(pdev, tsadc); - return devm_of_platform_populate(dev); + ret = devm_of_platform_populate(dev); + if (ret) + goto err_irq; + + return 0; + +err_irq: + mx25_tsadc_unset_irq(pdev); + + return ret; } static int mx25_tsadc_remove(struct platform_device *pdev) { - struct mx25_tsadc *tsadc = platform_get_drvdata(pdev); - int irq = platform_get_irq(pdev, 0); - - if (irq) { - irq_set_chained_handler_and_data(irq, NULL, NULL); - irq_domain_remove(tsadc->domain); - } + mx25_tsadc_unset_irq(pdev); return 0; } diff --git a/drivers/mfd/gateworks-gsc.c b/drivers/mfd/gateworks-gsc.c index d87876747b91351ab823976861ad3dc8e115a528..9d7d870c44a8cad8264133d391d2fcd7bd211f0a 100644 --- a/drivers/mfd/gateworks-gsc.c +++ b/drivers/mfd/gateworks-gsc.c @@ -255,11 +255,9 @@ static int gsc_probe(struct i2c_client *client) return 0; } -static int gsc_remove(struct i2c_client *client) +static void gsc_remove(struct i2c_client *client) { sysfs_remove_group(&client->dev.kobj, &attr_group); - - return 0; } static struct i2c_driver gsc_driver = { diff --git a/drivers/mfd/htc-i2cpld.c b/drivers/mfd/htc-i2cpld.c index 417b0355d904d43be8c2a32047f8f21659a3157d..b45b1346ab5442f438124268acbb8eaaef96cc4f 100644 --- a/drivers/mfd/htc-i2cpld.c +++ b/drivers/mfd/htc-i2cpld.c @@ -20,7 +20,9 @@ #include #include #include -#include +#include +#include +#include #include struct htcpld_chip { @@ -58,8 +60,8 @@ struct htcpld_data { uint irq_start; int nirqs; uint chained_irq; - unsigned int int_reset_gpio_hi; - unsigned int int_reset_gpio_lo; + struct gpio_desc *int_reset_gpio_hi; + struct gpio_desc *int_reset_gpio_lo; /* htcpld info */ struct htcpld_chip *chip; @@ -196,9 +198,9 @@ static irqreturn_t htcpld_handler(int irq, void *dev) * be asserted. */ if (htcpld->int_reset_gpio_hi) - gpio_set_value(htcpld->int_reset_gpio_hi, 1); + gpiod_set_value(htcpld->int_reset_gpio_hi, 1); if (htcpld->int_reset_gpio_lo) - gpio_set_value(htcpld->int_reset_gpio_lo, 0); + gpiod_set_value(htcpld->int_reset_gpio_lo, 0); return IRQ_HANDLED; } @@ -352,7 +354,7 @@ static int htcpld_register_chip_i2c( memset(&info, 0, sizeof(struct i2c_board_info)); info.addr = plat_chip_data->addr; - strlcpy(info.type, "htcpld-chip", I2C_NAME_SIZE); + strscpy(info.type, "htcpld-chip", I2C_NAME_SIZE); info.platform_data = chip; /* Add the I2C device. This calls the probe() function. */ @@ -562,34 +564,28 @@ static int htcpld_core_probe(struct platform_device *pdev) return ret; /* Request the GPIO(s) for the int reset and set them up */ - if (pdata->int_reset_gpio_hi) { - ret = gpio_request(pdata->int_reset_gpio_hi, "htcpld-core"); - if (ret) { - /* - * If it failed, that sucks, but we can probably - * continue on without it. - */ - dev_warn(dev, "Unable to request int_reset_gpio_hi -- interrupts may not work\n"); - htcpld->int_reset_gpio_hi = 0; - } else { - htcpld->int_reset_gpio_hi = pdata->int_reset_gpio_hi; - gpio_set_value(htcpld->int_reset_gpio_hi, 1); - } + htcpld->int_reset_gpio_hi = gpiochip_request_own_desc(&htcpld->chip[2].chip_out, + 7, "htcpld-core", GPIO_ACTIVE_HIGH, + GPIOD_OUT_HIGH); + if (IS_ERR(htcpld->int_reset_gpio_hi)) { + /* + * If it failed, that sucks, but we can probably + * continue on without it. + */ + htcpld->int_reset_gpio_hi = NULL; + dev_warn(dev, "Unable to request int_reset_gpio_hi -- interrupts may not work\n"); } - if (pdata->int_reset_gpio_lo) { - ret = gpio_request(pdata->int_reset_gpio_lo, "htcpld-core"); - if (ret) { - /* - * If it failed, that sucks, but we can probably - * continue on without it. - */ - dev_warn(dev, "Unable to request int_reset_gpio_lo -- interrupts may not work\n"); - htcpld->int_reset_gpio_lo = 0; - } else { - htcpld->int_reset_gpio_lo = pdata->int_reset_gpio_lo; - gpio_set_value(htcpld->int_reset_gpio_lo, 0); - } + htcpld->int_reset_gpio_lo = gpiochip_request_own_desc(&htcpld->chip[2].chip_out, + 0, "htcpld-core", GPIO_ACTIVE_HIGH, + GPIOD_OUT_LOW); + if (IS_ERR(htcpld->int_reset_gpio_lo)) { + /* + * If it failed, that sucks, but we can probably + * continue on without it. + */ + htcpld->int_reset_gpio_lo = NULL; + dev_warn(dev, "Unable to request int_reset_gpio_lo -- interrupts may not work\n"); } dev_info(dev, "Initialized successfully\n"); diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c index bb08b7a73fe11f374847d2d7c52d0c636ebe475a..dde31c50a6320ac05cb05f0faacf1583b3e26912 100644 --- a/drivers/mfd/intel-lpss-pci.c +++ b/drivers/mfd/intel-lpss-pci.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "intel-lpss.h" @@ -73,8 +74,18 @@ static void intel_lpss_pci_remove(struct pci_dev *pdev) static INTEL_LPSS_PM_OPS(intel_lpss_pci_pm_ops); +static const struct property_entry spt_spi_properties[] = { + PROPERTY_ENTRY_U32("intel,spi-pxa2xx-type", LPSS_SPT_SSP), + { } +}; + +static const struct software_node spt_spi_node = { + .properties = spt_spi_properties, +}; + static const struct intel_lpss_platform_info spt_info = { .clk_rate = 120000000, + .swnode = &spt_spi_node, }; static const struct property_entry spt_i2c_properties[] = { @@ -108,8 +119,18 @@ static const struct intel_lpss_platform_info spt_uart_info = { .swnode = &uart_node, }; +static const struct property_entry bxt_spi_properties[] = { + PROPERTY_ENTRY_U32("intel,spi-pxa2xx-type", LPSS_BXT_SSP), + { } +}; + +static const struct software_node bxt_spi_node = { + .properties = bxt_spi_properties, +}; + static const struct intel_lpss_platform_info bxt_info = { .clk_rate = 100000000, + .swnode = &bxt_spi_node, }; static const struct intel_lpss_platform_info bxt_uart_info = { @@ -166,6 +187,20 @@ static const struct intel_lpss_platform_info glk_i2c_info = { .swnode = &glk_i2c_node, }; +static const struct property_entry cnl_spi_properties[] = { + PROPERTY_ENTRY_U32("intel,spi-pxa2xx-type", LPSS_CNL_SSP), + { } +}; + +static const struct software_node cnl_spi_node = { + .properties = cnl_spi_properties, +}; + +static const struct intel_lpss_platform_info cnl_info = { + .clk_rate = 120000000, + .swnode = &cnl_spi_node, +}; + static const struct intel_lpss_platform_info cnl_i2c_info = { .clk_rate = 216000000, .swnode = &spt_i2c_node, @@ -176,12 +211,26 @@ static const struct intel_lpss_platform_info ehl_i2c_info = { .swnode = &bxt_i2c_node, }; +static const struct property_entry tgl_spi_properties[] = { + PROPERTY_ENTRY_U32("intel,spi-pxa2xx-type", LPSS_CNL_SSP), + { } +}; + +static const struct software_node tgl_spi_node = { + .properties = tgl_spi_properties, +}; + +static const struct intel_lpss_platform_info tgl_info = { + .clk_rate = 100000000, + .swnode = &tgl_spi_node, +}; + static const struct pci_device_id intel_lpss_pci_ids[] = { /* CML-LP */ { PCI_VDEVICE(INTEL, 0x02a8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x02a9), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0x02aa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0x02ab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x02aa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x02ab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0x02c5), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x02c6), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x02c7), (kernel_ulong_t)&spt_uart_info }, @@ -189,18 +238,18 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x02e9), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x02ea), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x02eb), (kernel_ulong_t)&cnl_i2c_info }, - { PCI_VDEVICE(INTEL, 0x02fb), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x02fb), (kernel_ulong_t)&cnl_info }, /* CML-H */ { PCI_VDEVICE(INTEL, 0x06a8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x06a9), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0x06aa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0x06ab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x06aa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x06ab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0x06c7), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x06e8), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x06e9), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x06ea), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x06eb), (kernel_ulong_t)&cnl_i2c_info }, - { PCI_VDEVICE(INTEL, 0x06fb), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x06fb), (kernel_ulong_t)&cnl_info }, /* BXT A-Step */ { PCI_VDEVICE(INTEL, 0x0aac), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x0aae), (kernel_ulong_t)&bxt_i2c_info }, @@ -255,8 +304,8 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { /* ICL-LP */ { PCI_VDEVICE(INTEL, 0x34a8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x34a9), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0x34aa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0x34ab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x34aa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x34ab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0x34c5), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x34c6), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x34c7), (kernel_ulong_t)&spt_uart_info }, @@ -264,15 +313,15 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x34e9), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x34ea), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x34eb), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x34fb), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x34fb), (kernel_ulong_t)&cnl_info }, /* ICL-N */ { PCI_VDEVICE(INTEL, 0x38a8), (kernel_ulong_t)&spt_uart_info }, /* TGL-H */ { PCI_VDEVICE(INTEL, 0x43a7), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x43a8), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x43a9), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x43aa), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x43ab), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x43aa), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x43ab), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x43ad), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x43ae), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x43d8), (kernel_ulong_t)&bxt_i2c_info }, @@ -281,8 +330,8 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x43e9), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x43ea), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x43eb), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x43fb), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x43fd), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x43fb), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x43fd), (kernel_ulong_t)&tgl_info }, /* EHL */ { PCI_VDEVICE(INTEL, 0x4b28), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x4b29), (kernel_ulong_t)&bxt_uart_info }, @@ -301,8 +350,8 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { /* JSL */ { PCI_VDEVICE(INTEL, 0x4da8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x4da9), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0x4daa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0x4dab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x4daa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x4dab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0x4dc5), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4dc6), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4dc7), (kernel_ulong_t)&spt_uart_info }, @@ -310,12 +359,12 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x4de9), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4dea), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x4deb), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x4dfb), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x4dfb), (kernel_ulong_t)&cnl_info }, /* ADL-P */ { PCI_VDEVICE(INTEL, 0x51a8), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x51a9), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x51aa), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x51ab), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x51aa), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x51ab), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x51c5), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x51c6), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x51c7), (kernel_ulong_t)&bxt_uart_info }, @@ -325,12 +374,12 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x51e9), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x51ea), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x51eb), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x51fb), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x51fb), (kernel_ulong_t)&tgl_info }, /* ADL-M */ { PCI_VDEVICE(INTEL, 0x54a8), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x54a9), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x54aa), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x54ab), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x54aa), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x54ab), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x54c5), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x54c6), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x54c7), (kernel_ulong_t)&bxt_uart_info }, @@ -338,7 +387,7 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x54e9), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x54ea), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x54eb), (kernel_ulong_t)&bxt_i2c_info }, - { PCI_VDEVICE(INTEL, 0x54fb), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x54fb), (kernel_ulong_t)&tgl_info }, /* APL */ { PCI_VDEVICE(INTEL, 0x5aac), (kernel_ulong_t)&apl_i2c_info }, { PCI_VDEVICE(INTEL, 0x5aae), (kernel_ulong_t)&apl_i2c_info }, @@ -358,39 +407,39 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { /* RPL-S */ { PCI_VDEVICE(INTEL, 0x7a28), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7a29), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x7a2a), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7a2b), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7a2a), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7a2b), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x7a4c), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a4d), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a4e), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a4f), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a5c), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x7a79), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7a7b), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7a79), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7a7b), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x7a7c), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a7d), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7a7e), (kernel_ulong_t)&bxt_uart_info }, /* ADL-S */ { PCI_VDEVICE(INTEL, 0x7aa8), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7aa9), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x7aaa), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7aab), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7aaa), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7aab), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x7acc), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7acd), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7ace), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7acf), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7adc), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x7af9), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7afb), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7af9), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7afb), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x7afc), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7afd), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7afe), (kernel_ulong_t)&bxt_uart_info }, /* MTL-P */ { PCI_VDEVICE(INTEL, 0x7e25), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0x7e26), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0x7e27), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7e30), (kernel_ulong_t)&bxt_info }, - { PCI_VDEVICE(INTEL, 0x7e46), (kernel_ulong_t)&bxt_info }, + { PCI_VDEVICE(INTEL, 0x7e27), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7e30), (kernel_ulong_t)&tgl_info }, + { PCI_VDEVICE(INTEL, 0x7e46), (kernel_ulong_t)&tgl_info }, { PCI_VDEVICE(INTEL, 0x7e50), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7e51), (kernel_ulong_t)&bxt_i2c_info }, { PCI_VDEVICE(INTEL, 0x7e52), (kernel_ulong_t)&bxt_uart_info }, @@ -424,8 +473,8 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { /* CNL-LP */ { PCI_VDEVICE(INTEL, 0x9da8), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0x9da9), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0x9daa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0x9dab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x9daa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0x9dab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0x9dc5), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x9dc6), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x9dc7), (kernel_ulong_t)&spt_uart_info }, @@ -433,12 +482,12 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x9de9), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x9dea), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0x9deb), (kernel_ulong_t)&cnl_i2c_info }, - { PCI_VDEVICE(INTEL, 0x9dfb), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0x9dfb), (kernel_ulong_t)&cnl_info }, /* TGL-LP */ { PCI_VDEVICE(INTEL, 0xa0a8), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0xa0a9), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0xa0aa), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0xa0ab), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa0aa), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xa0ab), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa0c5), (kernel_ulong_t)&spt_i2c_info }, { PCI_VDEVICE(INTEL, 0xa0c6), (kernel_ulong_t)&spt_i2c_info }, { PCI_VDEVICE(INTEL, 0xa0c7), (kernel_ulong_t)&bxt_uart_info }, @@ -448,15 +497,15 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0xa0db), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0xa0dc), (kernel_ulong_t)&bxt_uart_info }, { PCI_VDEVICE(INTEL, 0xa0dd), (kernel_ulong_t)&bxt_uart_info }, - { PCI_VDEVICE(INTEL, 0xa0de), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0xa0df), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa0de), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xa0df), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa0e8), (kernel_ulong_t)&spt_i2c_info }, { PCI_VDEVICE(INTEL, 0xa0e9), (kernel_ulong_t)&spt_i2c_info }, { PCI_VDEVICE(INTEL, 0xa0ea), (kernel_ulong_t)&spt_i2c_info }, { PCI_VDEVICE(INTEL, 0xa0eb), (kernel_ulong_t)&spt_i2c_info }, - { PCI_VDEVICE(INTEL, 0xa0fb), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0xa0fd), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0xa0fe), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa0fb), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xa0fd), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xa0fe), (kernel_ulong_t)&cnl_info }, /* SPT-H */ { PCI_VDEVICE(INTEL, 0xa127), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0xa128), (kernel_ulong_t)&spt_uart_info }, @@ -479,14 +528,14 @@ static const struct pci_device_id intel_lpss_pci_ids[] = { /* CNL-H */ { PCI_VDEVICE(INTEL, 0xa328), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0xa329), (kernel_ulong_t)&spt_uart_info }, - { PCI_VDEVICE(INTEL, 0xa32a), (kernel_ulong_t)&spt_info }, - { PCI_VDEVICE(INTEL, 0xa32b), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa32a), (kernel_ulong_t)&cnl_info }, + { PCI_VDEVICE(INTEL, 0xa32b), (kernel_ulong_t)&cnl_info }, { PCI_VDEVICE(INTEL, 0xa347), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0xa368), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0xa369), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0xa36a), (kernel_ulong_t)&cnl_i2c_info }, { PCI_VDEVICE(INTEL, 0xa36b), (kernel_ulong_t)&cnl_i2c_info }, - { PCI_VDEVICE(INTEL, 0xa37b), (kernel_ulong_t)&spt_info }, + { PCI_VDEVICE(INTEL, 0xa37b), (kernel_ulong_t)&cnl_info }, /* CML-V */ { PCI_VDEVICE(INTEL, 0xa3a7), (kernel_ulong_t)&spt_uart_info }, { PCI_VDEVICE(INTEL, 0xa3a8), (kernel_ulong_t)&spt_uart_info }, diff --git a/drivers/mfd/intel-m10-bmc.c b/drivers/mfd/intel-m10-bmc.c index f4d0d72573c883c759cfa55f775a3e37c2134401..7e3319e5b22ffda0a42b3f61c690aab9a7197981 100644 --- a/drivers/mfd/intel-m10-bmc.c +++ b/drivers/mfd/intel-m10-bmc.c @@ -21,6 +21,7 @@ enum m10bmc_type { static struct mfd_cell m10bmc_d5005_subdevs[] = { { .name = "d5005bmc-hwmon" }, + { .name = "d5005bmc-sec-update" } }; static struct mfd_cell m10bmc_pacn3000_subdevs[] = { diff --git a/drivers/mfd/intel_soc_pmic_chtdc_ti.c b/drivers/mfd/intel_soc_pmic_chtdc_ti.c index 1c7577b881ff9fc10a08c54326c385436c4669bd..282b8fd08009bebac591edca0ed7b8fab6bf00d5 100644 --- a/drivers/mfd/intel_soc_pmic_chtdc_ti.c +++ b/drivers/mfd/intel_soc_pmic_chtdc_ti.c @@ -140,7 +140,7 @@ static void chtdc_ti_shutdown(struct i2c_client *i2c) disable_irq(pmic->irq); } -static int __maybe_unused chtdc_ti_suspend(struct device *dev) +static int chtdc_ti_suspend(struct device *dev) { struct intel_soc_pmic *pmic = dev_get_drvdata(dev); @@ -149,7 +149,7 @@ static int __maybe_unused chtdc_ti_suspend(struct device *dev) return 0; } -static int __maybe_unused chtdc_ti_resume(struct device *dev) +static int chtdc_ti_resume(struct device *dev) { struct intel_soc_pmic *pmic = dev_get_drvdata(dev); @@ -158,7 +158,7 @@ static int __maybe_unused chtdc_ti_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(chtdc_ti_pm_ops, chtdc_ti_suspend, chtdc_ti_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(chtdc_ti_pm_ops, chtdc_ti_suspend, chtdc_ti_resume); static const struct acpi_device_id chtdc_ti_acpi_ids[] = { { "INT33F5" }, @@ -169,7 +169,7 @@ MODULE_DEVICE_TABLE(acpi, chtdc_ti_acpi_ids); static struct i2c_driver chtdc_ti_i2c_driver = { .driver = { .name = "intel_soc_pmic_chtdc_ti", - .pm = &chtdc_ti_pm_ops, + .pm = pm_sleep_ptr(&chtdc_ti_pm_ops), .acpi_match_table = chtdc_ti_acpi_ids, }, .probe_new = chtdc_ti_probe, diff --git a/drivers/mfd/intel_soc_pmic_core.c b/drivers/mfd/intel_soc_pmic_core.c deleted file mode 100644 index 5e8c94e008ed1a9982f9d7109073e078f1f88714..0000000000000000000000000000000000000000 --- a/drivers/mfd/intel_soc_pmic_core.c +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Intel SoC PMIC MFD Driver - * - * Copyright (C) 2013, 2014 Intel Corporation. All rights reserved. - * - * Author: Yang, Bin - * Author: Zhu, Lejun - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "intel_soc_pmic_core.h" - -/* PWM consumed by the Intel GFX */ -static struct pwm_lookup crc_pwm_lookup[] = { - PWM_LOOKUP("crystal_cove_pwm", 0, "0000:00:02.0", "pwm_pmic_backlight", 0, PWM_POLARITY_NORMAL), -}; - -static int intel_soc_pmic_i2c_probe(struct i2c_client *i2c, - const struct i2c_device_id *i2c_id) -{ - struct device *dev = &i2c->dev; - struct intel_soc_pmic_config *config; - struct intel_soc_pmic *pmic; - int ret; - - if (soc_intel_is_byt()) - config = &intel_soc_pmic_config_byt_crc; - else - config = &intel_soc_pmic_config_cht_crc; - - pmic = devm_kzalloc(dev, sizeof(*pmic), GFP_KERNEL); - if (!pmic) - return -ENOMEM; - - dev_set_drvdata(dev, pmic); - - pmic->regmap = devm_regmap_init_i2c(i2c, config->regmap_config); - if (IS_ERR(pmic->regmap)) - return PTR_ERR(pmic->regmap); - - pmic->irq = i2c->irq; - - ret = regmap_add_irq_chip(pmic->regmap, pmic->irq, - config->irq_flags | IRQF_ONESHOT, - 0, config->irq_chip, - &pmic->irq_chip_data); - if (ret) - return ret; - - ret = enable_irq_wake(pmic->irq); - if (ret) - dev_warn(dev, "Can't enable IRQ as wake source: %d\n", ret); - - /* Add lookup table for crc-pwm */ - pwm_add_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup)); - - /* To distuingish this domain from the GPIO/charger's irqchip domains */ - irq_domain_update_bus_token(regmap_irq_get_domain(pmic->irq_chip_data), - DOMAIN_BUS_NEXUS); - - ret = mfd_add_devices(dev, -1, config->cell_dev, - config->n_cell_devs, NULL, 0, - regmap_irq_get_domain(pmic->irq_chip_data)); - if (ret) - goto err_del_irq_chip; - - return 0; - -err_del_irq_chip: - regmap_del_irq_chip(pmic->irq, pmic->irq_chip_data); - return ret; -} - -static int intel_soc_pmic_i2c_remove(struct i2c_client *i2c) -{ - struct intel_soc_pmic *pmic = dev_get_drvdata(&i2c->dev); - - regmap_del_irq_chip(pmic->irq, pmic->irq_chip_data); - - /* remove crc-pwm lookup table */ - pwm_remove_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup)); - - mfd_remove_devices(&i2c->dev); - - return 0; -} - -static void intel_soc_pmic_shutdown(struct i2c_client *i2c) -{ - struct intel_soc_pmic *pmic = dev_get_drvdata(&i2c->dev); - - disable_irq(pmic->irq); - - return; -} - -#if defined(CONFIG_PM_SLEEP) -static int intel_soc_pmic_suspend(struct device *dev) -{ - struct intel_soc_pmic *pmic = dev_get_drvdata(dev); - - disable_irq(pmic->irq); - - return 0; -} - -static int intel_soc_pmic_resume(struct device *dev) -{ - struct intel_soc_pmic *pmic = dev_get_drvdata(dev); - - enable_irq(pmic->irq); - - return 0; -} -#endif - -static SIMPLE_DEV_PM_OPS(intel_soc_pmic_pm_ops, intel_soc_pmic_suspend, - intel_soc_pmic_resume); - -static const struct i2c_device_id intel_soc_pmic_i2c_id[] = { - { } -}; -MODULE_DEVICE_TABLE(i2c, intel_soc_pmic_i2c_id); - -#if defined(CONFIG_ACPI) -static const struct acpi_device_id intel_soc_pmic_acpi_match[] = { - { "INT33FD" }, - { }, -}; -MODULE_DEVICE_TABLE(acpi, intel_soc_pmic_acpi_match); -#endif - -static struct i2c_driver intel_soc_pmic_i2c_driver = { - .driver = { - .name = "intel_soc_pmic_i2c", - .pm = &intel_soc_pmic_pm_ops, - .acpi_match_table = ACPI_PTR(intel_soc_pmic_acpi_match), - }, - .probe = intel_soc_pmic_i2c_probe, - .remove = intel_soc_pmic_i2c_remove, - .id_table = intel_soc_pmic_i2c_id, - .shutdown = intel_soc_pmic_shutdown, -}; - -module_i2c_driver(intel_soc_pmic_i2c_driver); - -MODULE_DESCRIPTION("I2C driver for Intel SoC PMIC"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Yang, Bin "); -MODULE_AUTHOR("Zhu, Lejun "); diff --git a/drivers/mfd/intel_soc_pmic_core.h b/drivers/mfd/intel_soc_pmic_core.h deleted file mode 100644 index d490685845eb7de2036902230abf497622818528..0000000000000000000000000000000000000000 --- a/drivers/mfd/intel_soc_pmic_core.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Intel SoC PMIC MFD Driver - * - * Copyright (C) 2012-2014 Intel Corporation. All rights reserved. - * - * Author: Yang, Bin - * Author: Zhu, Lejun - */ - -#ifndef __INTEL_SOC_PMIC_CORE_H__ -#define __INTEL_SOC_PMIC_CORE_H__ - -struct intel_soc_pmic_config { - unsigned long irq_flags; - struct mfd_cell *cell_dev; - int n_cell_devs; - const struct regmap_config *regmap_config; - const struct regmap_irq_chip *irq_chip; -}; - -extern struct intel_soc_pmic_config intel_soc_pmic_config_byt_crc; -extern struct intel_soc_pmic_config intel_soc_pmic_config_cht_crc; - -#endif /* __INTEL_SOC_PMIC_CORE_H__ */ diff --git a/drivers/mfd/intel_soc_pmic_crc.c b/drivers/mfd/intel_soc_pmic_crc.c index 5bb0367bd974f1a70cc0e51ee8dde95f683bbc86..b1548a933dc3ee841e7dcb96c4c0363e8dedbd3c 100644 --- a/drivers/mfd/intel_soc_pmic_crc.c +++ b/drivers/mfd/intel_soc_pmic_crc.c @@ -2,18 +2,21 @@ /* * Device access for Crystal Cove PMIC * - * Copyright (C) 2013, 2014 Intel Corporation. All rights reserved. + * Copyright (C) 2012-2014, 2022 Intel Corporation. All rights reserved. * * Author: Yang, Bin * Author: Zhu, Lejun */ +#include #include -#include +#include +#include #include #include - -#include "intel_soc_pmic_core.h" +#include +#include +#include #define CRYSTAL_COVE_MAX_REGISTER 0xC6 @@ -132,7 +135,20 @@ static const struct regmap_irq_chip crystal_cove_irq_chip = { .mask_base = CRYSTAL_COVE_REG_MIRQLVL1, }; -struct intel_soc_pmic_config intel_soc_pmic_config_byt_crc = { +/* PWM consumed by the Intel GFX */ +static struct pwm_lookup crc_pwm_lookup[] = { + PWM_LOOKUP("crystal_cove_pwm", 0, "0000:00:02.0", "pwm_pmic_backlight", 0, PWM_POLARITY_NORMAL), +}; + +struct crystal_cove_config { + unsigned long irq_flags; + struct mfd_cell *cell_dev; + int n_cell_devs; + const struct regmap_config *regmap_config; + const struct regmap_irq_chip *irq_chip; +}; + +static const struct crystal_cove_config crystal_cove_config_byt_crc = { .irq_flags = IRQF_TRIGGER_RISING, .cell_dev = crystal_cove_byt_dev, .n_cell_devs = ARRAY_SIZE(crystal_cove_byt_dev), @@ -140,10 +156,121 @@ struct intel_soc_pmic_config intel_soc_pmic_config_byt_crc = { .irq_chip = &crystal_cove_irq_chip, }; -struct intel_soc_pmic_config intel_soc_pmic_config_cht_crc = { +static const struct crystal_cove_config crystal_cove_config_cht_crc = { .irq_flags = IRQF_TRIGGER_RISING, .cell_dev = crystal_cove_cht_dev, .n_cell_devs = ARRAY_SIZE(crystal_cove_cht_dev), .regmap_config = &crystal_cove_regmap_config, .irq_chip = &crystal_cove_irq_chip, }; + +static int crystal_cove_i2c_probe(struct i2c_client *i2c) +{ + const struct crystal_cove_config *config; + struct device *dev = &i2c->dev; + struct intel_soc_pmic *pmic; + int ret; + + if (soc_intel_is_byt()) + config = &crystal_cove_config_byt_crc; + else + config = &crystal_cove_config_cht_crc; + + pmic = devm_kzalloc(dev, sizeof(*pmic), GFP_KERNEL); + if (!pmic) + return -ENOMEM; + + i2c_set_clientdata(i2c, pmic); + + pmic->regmap = devm_regmap_init_i2c(i2c, config->regmap_config); + if (IS_ERR(pmic->regmap)) + return PTR_ERR(pmic->regmap); + + pmic->irq = i2c->irq; + + ret = devm_regmap_add_irq_chip(dev, pmic->regmap, pmic->irq, + config->irq_flags | IRQF_ONESHOT, + 0, config->irq_chip, &pmic->irq_chip_data); + if (ret) + return ret; + + ret = enable_irq_wake(pmic->irq); + if (ret) + dev_warn(dev, "Can't enable IRQ as wake source: %d\n", ret); + + /* Add lookup table for crc-pwm */ + pwm_add_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup)); + + /* To distuingish this domain from the GPIO/charger's irqchip domains */ + irq_domain_update_bus_token(regmap_irq_get_domain(pmic->irq_chip_data), + DOMAIN_BUS_NEXUS); + + ret = mfd_add_devices(dev, PLATFORM_DEVID_NONE, config->cell_dev, + config->n_cell_devs, NULL, 0, + regmap_irq_get_domain(pmic->irq_chip_data)); + if (ret) + pwm_remove_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup)); + + return ret; +} + +static void crystal_cove_i2c_remove(struct i2c_client *i2c) +{ + /* remove crc-pwm lookup table */ + pwm_remove_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup)); + + mfd_remove_devices(&i2c->dev); +} + +static void crystal_cove_shutdown(struct i2c_client *i2c) +{ + struct intel_soc_pmic *pmic = i2c_get_clientdata(i2c); + + disable_irq(pmic->irq); + + return; +} + +static int crystal_cove_suspend(struct device *dev) +{ + struct intel_soc_pmic *pmic = dev_get_drvdata(dev); + + disable_irq(pmic->irq); + + return 0; +} + +static int crystal_cove_resume(struct device *dev) +{ + struct intel_soc_pmic *pmic = dev_get_drvdata(dev); + + enable_irq(pmic->irq); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(crystal_cove_pm_ops, crystal_cove_suspend, crystal_cove_resume); + +static const struct acpi_device_id crystal_cove_acpi_match[] = { + { "INT33FD" }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, crystal_cove_acpi_match); + +static struct i2c_driver crystal_cove_i2c_driver = { + .driver = { + .name = "crystal_cove_i2c", + .pm = pm_sleep_ptr(&crystal_cove_pm_ops), + .acpi_match_table = crystal_cove_acpi_match, + }, + .probe_new = crystal_cove_i2c_probe, + .remove = crystal_cove_i2c_remove, + .shutdown = crystal_cove_shutdown, +}; + +module_i2c_driver(crystal_cove_i2c_driver); + +MODULE_DESCRIPTION("I2C driver for Intel SoC PMIC"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Yang, Bin "); +MODULE_AUTHOR("Zhu, Lejun "); diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c index 575ab67e243dd6b37a4232d5a77349605973fe8e..1895fce25b060b1845506133fb6e49cb4432b64e 100644 --- a/drivers/mfd/iqs62x.c +++ b/drivers/mfd/iqs62x.c @@ -1008,13 +1008,11 @@ static int iqs62x_probe(struct i2c_client *client) return ret; } -static int iqs62x_remove(struct i2c_client *client) +static void iqs62x_remove(struct i2c_client *client) { struct iqs62x_core *iqs62x = i2c_get_clientdata(client); wait_for_completion(&iqs62x->fw_done); - - return 0; } static int __maybe_unused iqs62x_suspend(struct device *dev) diff --git a/drivers/mfd/lm3533-core.c b/drivers/mfd/lm3533-core.c index 5690768f3e63bf1b3e3e1a49f7f994db0061c016..be32ffc5af384fdeb6d9c7774c579e217d71418c 100644 --- a/drivers/mfd/lm3533-core.c +++ b/drivers/mfd/lm3533-core.c @@ -607,15 +607,13 @@ static int lm3533_i2c_probe(struct i2c_client *i2c, return lm3533_device_init(lm3533); } -static int lm3533_i2c_remove(struct i2c_client *i2c) +static void lm3533_i2c_remove(struct i2c_client *i2c) { struct lm3533 *lm3533 = i2c_get_clientdata(i2c); dev_dbg(&i2c->dev, "%s\n", __func__); lm3533_device_exit(lm3533); - - return 0; } static const struct i2c_device_id lm3533_i2c_ids[] = { diff --git a/drivers/mfd/lp8788-irq.c b/drivers/mfd/lp8788-irq.c index 348439a3fbbd45f56b9abb53124ab2b64c0030f2..39006297f3d2703ac852f2ffef33abf1cb13132a 100644 --- a/drivers/mfd/lp8788-irq.c +++ b/drivers/mfd/lp8788-irq.c @@ -175,6 +175,7 @@ int lp8788_irq_init(struct lp8788 *lp, int irq) IRQF_TRIGGER_FALLING | IRQF_ONESHOT, "lp8788-irq", irqd); if (ret) { + irq_domain_remove(lp->irqdm); dev_err(lp->dev, "failed to create a thread for IRQ_N\n"); return ret; } @@ -188,4 +189,6 @@ void lp8788_irq_exit(struct lp8788 *lp) { if (lp->irq) free_irq(lp->irq, lp->irqdm); + if (lp->irqdm) + irq_domain_remove(lp->irqdm); } diff --git a/drivers/mfd/lp8788.c b/drivers/mfd/lp8788.c index c223d2c6a363593df39c77ea7d84a95d7794f84d..724a5712b36baa1ffda07a8e7a5d0e302f3f20df 100644 --- a/drivers/mfd/lp8788.c +++ b/drivers/mfd/lp8788.c @@ -195,17 +195,24 @@ static int lp8788_probe(struct i2c_client *cl, const struct i2c_device_id *id) if (ret) return ret; - return mfd_add_devices(lp->dev, -1, lp8788_devs, - ARRAY_SIZE(lp8788_devs), NULL, 0, NULL); + ret = mfd_add_devices(lp->dev, -1, lp8788_devs, + ARRAY_SIZE(lp8788_devs), NULL, 0, NULL); + if (ret) + goto err_exit_irq; + + return 0; + +err_exit_irq: + lp8788_irq_exit(lp); + return ret; } -static int lp8788_remove(struct i2c_client *cl) +static void lp8788_remove(struct i2c_client *cl) { struct lp8788 *lp = i2c_get_clientdata(cl); mfd_remove_devices(lp->dev); lp8788_irq_exit(lp); - return 0; } static const struct i2c_device_id lp8788_ids[] = { diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c index 650951f89f1c9e9cb4df513cc5f351e6a65c9757..7b1c597b6879fb38d6c9a3a34e2d6baa9ade66d1 100644 --- a/drivers/mfd/lpc_ich.c +++ b/drivers/mfd/lpc_ich.c @@ -959,7 +959,7 @@ static int lpc_ich_finalize_wdt_cell(struct pci_dev *dev) info = &lpc_chipset_info[priv->chipset]; pdata->version = info->iTCO_version; - strlcpy(pdata->name, info->name, sizeof(pdata->name)); + strscpy(pdata->name, info->name, sizeof(pdata->name)); cell->platform_data = pdata; cell->pdata_size = sizeof(*pdata); diff --git a/drivers/mfd/madera-i2c.c b/drivers/mfd/madera-i2c.c index 7df5b9ba585546477fe696b7ac44f97e915cf808..915d2f95bad313f2d80c28b57cc81345d93c33d7 100644 --- a/drivers/mfd/madera-i2c.c +++ b/drivers/mfd/madera-i2c.c @@ -112,13 +112,11 @@ static int madera_i2c_probe(struct i2c_client *i2c, return madera_dev_init(madera); } -static int madera_i2c_remove(struct i2c_client *i2c) +static void madera_i2c_remove(struct i2c_client *i2c) { struct madera *madera = dev_get_drvdata(&i2c->dev); madera_dev_exit(madera); - - return 0; } static const struct i2c_device_id madera_i2c_id[] = { diff --git a/drivers/mfd/max14577.c b/drivers/mfd/max14577.c index 6c487fa14e9c805ddd325d155a4f9efd472508f0..d44ad6f337425a9223315465e73287a0d65a5f91 100644 --- a/drivers/mfd/max14577.c +++ b/drivers/mfd/max14577.c @@ -463,7 +463,7 @@ err_max77836: return ret; } -static int max14577_i2c_remove(struct i2c_client *i2c) +static void max14577_i2c_remove(struct i2c_client *i2c) { struct max14577 *max14577 = i2c_get_clientdata(i2c); @@ -471,8 +471,6 @@ static int max14577_i2c_remove(struct i2c_client *i2c) regmap_del_irq_chip(max14577->irq, max14577->irq_data); if (max14577->dev_type == MAXIM_DEVICE_TYPE_MAX77836) max77836_remove(max14577); - - return 0; } static const struct i2c_device_id max14577_i2c_id[] = { diff --git a/drivers/mfd/max77693.c b/drivers/mfd/max77693.c index 4e6244e1755939d3bde8fe9a76b01cd90f341918..7088cb6f917457914c150c8186d5e7ecfbefb8f7 100644 --- a/drivers/mfd/max77693.c +++ b/drivers/mfd/max77693.c @@ -294,7 +294,7 @@ err_i2c_haptic: return ret; } -static int max77693_i2c_remove(struct i2c_client *i2c) +static void max77693_i2c_remove(struct i2c_client *i2c) { struct max77693_dev *max77693 = i2c_get_clientdata(i2c); @@ -307,8 +307,6 @@ static int max77693_i2c_remove(struct i2c_client *i2c) i2c_unregister_device(max77693->i2c_muic); i2c_unregister_device(max77693->i2c_haptic); - - return 0; } static const struct i2c_device_id max77693_i2c_id[] = { diff --git a/drivers/mfd/max8907.c b/drivers/mfd/max8907.c index 41f566e6a0965fd655f1f9d9f47d6fa9426741d0..c340080971cef8fb4e34e4cfaa47ecc38741c805 100644 --- a/drivers/mfd/max8907.c +++ b/drivers/mfd/max8907.c @@ -282,7 +282,7 @@ err_alloc_drvdata: return ret; } -static int max8907_i2c_remove(struct i2c_client *i2c) +static void max8907_i2c_remove(struct i2c_client *i2c) { struct max8907 *max8907 = i2c_get_clientdata(i2c); @@ -293,8 +293,6 @@ static int max8907_i2c_remove(struct i2c_client *i2c) regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_chg); i2c_unregister_device(max8907->i2c_rtc); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/mfd/max8925-i2c.c b/drivers/mfd/max8925-i2c.c index 114e905bef25ae1c46a2c4aede9fa4848fc6e22f..04101da42bd314454756115481a1755ba1020f2c 100644 --- a/drivers/mfd/max8925-i2c.c +++ b/drivers/mfd/max8925-i2c.c @@ -198,14 +198,13 @@ static int max8925_probe(struct i2c_client *client, return 0; } -static int max8925_remove(struct i2c_client *client) +static void max8925_remove(struct i2c_client *client) { struct max8925_chip *chip = i2c_get_clientdata(client); max8925_device_exit(chip); i2c_unregister_device(chip->adc); i2c_unregister_device(chip->rtc); - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/mfd/mc13xxx-i2c.c b/drivers/mfd/mc13xxx-i2c.c index fb937f66277e59ab5bae6249fc92b867868b9877..eb94f3004cf336437dbc3842984d90193cba79d0 100644 --- a/drivers/mfd/mc13xxx-i2c.c +++ b/drivers/mfd/mc13xxx-i2c.c @@ -85,10 +85,9 @@ static int mc13xxx_i2c_probe(struct i2c_client *client, return mc13xxx_common_init(&client->dev); } -static int mc13xxx_i2c_remove(struct i2c_client *client) +static void mc13xxx_i2c_remove(struct i2c_client *client) { mc13xxx_common_exit(&client->dev); - return 0; } static struct i2c_driver mc13xxx_i2c_driver = { diff --git a/drivers/mfd/menelaus.c b/drivers/mfd/menelaus.c index 07e0ca2e467cc2861f04d6ddeeabc02ca5e5cd7e..eb08f69001f91d2cb302676337698ee091cfef7c 100644 --- a/drivers/mfd/menelaus.c +++ b/drivers/mfd/menelaus.c @@ -1222,14 +1222,13 @@ fail: return err; } -static int menelaus_remove(struct i2c_client *client) +static void menelaus_remove(struct i2c_client *client) { struct menelaus_chip *menelaus = i2c_get_clientdata(client); free_irq(client->irq, menelaus); flush_work(&menelaus->work); the_menelaus = NULL; - return 0; } static const struct i2c_device_id menelaus_id[] = { diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 8b058200d5adaf5fd718c4641e4583867913fee0..16d1861e96823c1402672457e029c516c348e040 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -105,7 +105,7 @@ static void mfd_acpi_add_device(const struct mfd_cell *cell, .ids = ids, }; - strlcpy(ids[0].id, match->pnpid, sizeof(ids[0].id)); + strscpy(ids[0].id, match->pnpid, sizeof(ids[0].id)); acpi_dev_for_each_child(parent, match_device_ids, &wd); adev = wd.adev; } else { @@ -368,6 +368,7 @@ static int mfd_remove_devices_fn(struct device *dev, void *data) { struct platform_device *pdev; const struct mfd_cell *cell; + struct mfd_of_node_entry *of_entry, *tmp; int *level = data; if (dev->type != &mfd_dev_type) @@ -382,6 +383,12 @@ static int mfd_remove_devices_fn(struct device *dev, void *data) if (cell->swnode) device_remove_software_node(&pdev->dev); + list_for_each_entry_safe(of_entry, tmp, &mfd_of_node_list, list) + if (of_entry->dev == &pdev->dev) { + list_del(&of_entry->list); + kfree(of_entry); + } + regulator_bulk_unregister_supply_alias(dev, cell->parent_supplies, cell->num_parent_supplies); diff --git a/drivers/mfd/mt6370.c b/drivers/mfd/mt6370.c new file mode 100644 index 0000000000000000000000000000000000000000..cf19cce2fdc0bc16684bb958c7b96a598d07f066 --- /dev/null +++ b/drivers/mfd/mt6370.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * + * Author: ChiYuan Huang + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mt6370.h" + +#define MT6370_REG_DEV_INFO 0x100 +#define MT6370_REG_CHG_IRQ1 0x1C0 +#define MT6370_REG_CHG_MASK1 0x1E0 +#define MT6370_REG_MAXADDR 0x1FF + +#define MT6370_VENID_MASK GENMASK(7, 4) + +#define MT6370_NUM_IRQREGS 16 +#define MT6370_USBC_I2CADDR 0x4E +#define MT6370_MAX_ADDRLEN 2 + +#define MT6370_VENID_RT5081 0x8 +#define MT6370_VENID_RT5081A 0xA +#define MT6370_VENID_MT6370 0xE +#define MT6370_VENID_MT6371 0xF +#define MT6370_VENID_MT6372P 0x9 +#define MT6370_VENID_MT6372CP 0xB + +static const struct regmap_irq mt6370_irqs[] = { + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHGON, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_TREG, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_AICR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_MIVR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_PWR_RDY, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FL_CHG_VINOVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_VSYSUV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_VSYSOV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_VBATOV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_VINOVPCHG, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TS_BAT_COLD, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TS_BAT_COOL, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TS_BAT_WARM, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TS_BAT_HOT, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TS_STATC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_FAULT, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_STATC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_TMR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_BATABS, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_ADPBAD, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_RVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_TSHUTDOWN, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_IINMEAS, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_ICCMEAS, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHGDET_DONE, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_WDTMR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_SSFINISH, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_RECHG, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_TERM, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHG_IEOC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_ADC_DONE, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_PUMPX_DONE, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_BST_BATUV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_BST_MIDOV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_BST_OLP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_ATTACH, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DETACH, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_HVDCP_STPDONE, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_HVDCP_VBUSDET_DONE, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_HVDCP_DET, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_CHGDET, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DCDT, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHG_VGOK, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHG_WDTMR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHG_UC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHG_OC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DIRCHG_OV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OVPCTRL_SWON, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OVPCTRL_UVP_D, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OVPCTRL_UVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OVPCTRL_OVP_D, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OVPCTRL_OVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED_STRBPIN, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED_TORPIN, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED_TX, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED_LVF, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED2_SHORT, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED1_SHORT, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED2_STRB, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED1_STRB, 8), + REGMAP_IRQ_REG_LINE(mT6370_IRQ_FLED2_STRB_TO, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED1_STRB_TO, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED2_TOR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_FLED1_TOR, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_OTP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_VDDA_OVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_VDDA_UV, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_LDO_OC, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_BLED_OCP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_BLED_OVP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DSV_VNEG_OCP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DSV_VPOS_OCP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DSV_BST_OCP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DSV_VNEG_SCP, 8), + REGMAP_IRQ_REG_LINE(MT6370_IRQ_DSV_VPOS_SCP, 8), +}; + +static const struct regmap_irq_chip mt6370_irq_chip = { + .name = "mt6370-irqs", + .status_base = MT6370_REG_CHG_IRQ1, + .mask_base = MT6370_REG_CHG_MASK1, + .num_regs = MT6370_NUM_IRQREGS, + .irqs = mt6370_irqs, + .num_irqs = ARRAY_SIZE(mt6370_irqs), +}; + +static const struct resource mt6370_regulator_irqs[] = { + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_DSV_VPOS_SCP, "db_vpos_scp"), + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_DSV_VNEG_SCP, "db_vneg_scp"), + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_DSV_BST_OCP, "db_vbst_ocp"), + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_DSV_VPOS_OCP, "db_vpos_ocp"), + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_DSV_VNEG_OCP, "db_vneg_ocp"), + DEFINE_RES_IRQ_NAMED(MT6370_IRQ_LDO_OC, "ldo_oc"), +}; + +static const struct mfd_cell mt6370_devices[] = { + MFD_CELL_OF("mt6370-adc", + NULL, NULL, 0, 0, "mediatek,mt6370-adc"), + MFD_CELL_OF("mt6370-charger", + NULL, NULL, 0, 0, "mediatek,mt6370-charger"), + MFD_CELL_OF("mt6370-flashlight", + NULL, NULL, 0, 0, "mediatek,mt6370-flashlight"), + MFD_CELL_OF("mt6370-indicator", + NULL, NULL, 0, 0, "mediatek,mt6370-indicator"), + MFD_CELL_OF("mt6370-tcpc", + NULL, NULL, 0, 0, "mediatek,mt6370-tcpc"), + MFD_CELL_RES("mt6370-regulator", mt6370_regulator_irqs), +}; + +static const struct mfd_cell mt6370_exclusive_devices[] = { + MFD_CELL_OF("mt6370-backlight", + NULL, NULL, 0, 0, "mediatek,mt6370-backlight"), +}; + +static const struct mfd_cell mt6372_exclusive_devices[] = { + MFD_CELL_OF("mt6370-backlight", + NULL, NULL, 0, 0, "mediatek,mt6372-backlight"), +}; + +static int mt6370_check_vendor_info(struct device *dev, struct regmap *rmap, + int *vid) +{ + unsigned int devinfo; + int ret; + + ret = regmap_read(rmap, MT6370_REG_DEV_INFO, &devinfo); + if (ret) + return ret; + + *vid = FIELD_GET(MT6370_VENID_MASK, devinfo); + switch (*vid) { + case MT6370_VENID_RT5081: + case MT6370_VENID_RT5081A: + case MT6370_VENID_MT6370: + case MT6370_VENID_MT6371: + case MT6370_VENID_MT6372P: + case MT6370_VENID_MT6372CP: + return 0; + default: + dev_err(dev, "Unknown Vendor ID 0x%02x\n", devinfo); + return -ENODEV; + } +} + +static int mt6370_regmap_read(void *context, const void *reg_buf, + size_t reg_size, void *val_buf, size_t val_size) +{ + struct mt6370_info *info = context; + const u8 *u8_buf = reg_buf; + u8 bank_idx, bank_addr; + int ret; + + bank_idx = u8_buf[0]; + bank_addr = u8_buf[1]; + + ret = i2c_smbus_read_i2c_block_data(info->i2c[bank_idx], bank_addr, + val_size, val_buf); + if (ret < 0) + return ret; + + if (ret != val_size) + return -EIO; + + return 0; +} + +static int mt6370_regmap_write(void *context, const void *data, size_t count) +{ + struct mt6370_info *info = context; + const u8 *u8_buf = data; + u8 bank_idx, bank_addr; + int len = count - MT6370_MAX_ADDRLEN; + + bank_idx = u8_buf[0]; + bank_addr = u8_buf[1]; + + return i2c_smbus_write_i2c_block_data(info->i2c[bank_idx], bank_addr, + len, data + MT6370_MAX_ADDRLEN); +} + +static const struct regmap_bus mt6370_regmap_bus = { + .read = mt6370_regmap_read, + .write = mt6370_regmap_write, +}; + +static const struct regmap_config mt6370_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .reg_format_endian = REGMAP_ENDIAN_BIG, + .max_register = MT6370_REG_MAXADDR, +}; + +static int mt6370_probe(struct i2c_client *i2c) +{ + struct mt6370_info *info; + struct i2c_client *usbc_i2c; + struct regmap *regmap; + struct device *dev = &i2c->dev; + int ret, vid; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + usbc_i2c = devm_i2c_new_dummy_device(dev, i2c->adapter, + MT6370_USBC_I2CADDR); + if (IS_ERR(usbc_i2c)) + return dev_err_probe(dev, PTR_ERR(usbc_i2c), + "Failed to register USBC I2C client\n"); + + /* Assign I2C client for PMU and TypeC */ + info->i2c[MT6370_PMU_I2C] = i2c; + info->i2c[MT6370_USBC_I2C] = usbc_i2c; + + regmap = devm_regmap_init(dev, &mt6370_regmap_bus, + info, &mt6370_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), + "Failed to init regmap\n"); + + ret = mt6370_check_vendor_info(dev, regmap, &vid); + if (ret) + return dev_err_probe(dev, ret, "Failed to check vendor info\n"); + + ret = devm_regmap_add_irq_chip(dev, regmap, i2c->irq, + IRQF_ONESHOT, -1, &mt6370_irq_chip, + &info->irq_data); + if (ret) + return dev_err_probe(dev, ret, "Failed to add irq chip\n"); + + switch (vid) { + case MT6370_VENID_MT6372P: + case MT6370_VENID_MT6372CP: + ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, + mt6372_exclusive_devices, + ARRAY_SIZE(mt6372_exclusive_devices), + NULL, 0, + regmap_irq_get_domain(info->irq_data)); + break; + default: + ret = devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, + mt6370_exclusive_devices, + ARRAY_SIZE(mt6370_exclusive_devices), + NULL, 0, + regmap_irq_get_domain(info->irq_data)); + break; + } + + if (ret) + return dev_err_probe(dev, ret, "Failed to add the exclusive devices\n"); + + return devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, + mt6370_devices, ARRAY_SIZE(mt6370_devices), + NULL, 0, + regmap_irq_get_domain(info->irq_data)); +} + +static const struct of_device_id mt6370_match_table[] = { + { .compatible = "mediatek,mt6370" }, + {} +}; +MODULE_DEVICE_TABLE(of, mt6370_match_table); + +static struct i2c_driver mt6370_driver = { + .driver = { + .name = "mt6370", + .of_match_table = mt6370_match_table, + }, + .probe_new = mt6370_probe, +}; +module_i2c_driver(mt6370_driver); + +MODULE_AUTHOR("ChiYuan Huang "); +MODULE_DESCRIPTION("MediaTek MT6370 SubPMIC Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mfd/mt6370.h b/drivers/mfd/mt6370.h new file mode 100644 index 0000000000000000000000000000000000000000..094e59e4af4e9669ce70c44cc84eaf7cfcf83b7e --- /dev/null +++ b/drivers/mfd/mt6370.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2022 Richtek Technology Corp. + * + * Author: ChiYuan Huang + */ + +#ifndef __MFD_MT6370_H__ +#define __MFD_MT6370_H__ + +/* IRQ definitions */ +#define MT6370_IRQ_DIRCHGON 0 +#define MT6370_IRQ_CHG_TREG 4 +#define MT6370_IRQ_CHG_AICR 5 +#define MT6370_IRQ_CHG_MIVR 6 +#define MT6370_IRQ_PWR_RDY 7 +#define MT6370_IRQ_FL_CHG_VINOVP 11 +#define MT6370_IRQ_CHG_VSYSUV 12 +#define MT6370_IRQ_CHG_VSYSOV 13 +#define MT6370_IRQ_CHG_VBATOV 14 +#define MT6370_IRQ_CHG_VINOVPCHG 15 +#define MT6370_IRQ_TS_BAT_COLD 20 +#define MT6370_IRQ_TS_BAT_COOL 21 +#define MT6370_IRQ_TS_BAT_WARM 22 +#define MT6370_IRQ_TS_BAT_HOT 23 +#define MT6370_IRQ_TS_STATC 24 +#define MT6370_IRQ_CHG_FAULT 25 +#define MT6370_IRQ_CHG_STATC 26 +#define MT6370_IRQ_CHG_TMR 27 +#define MT6370_IRQ_CHG_BATABS 28 +#define MT6370_IRQ_CHG_ADPBAD 29 +#define MT6370_IRQ_CHG_RVP 30 +#define MT6370_IRQ_TSHUTDOWN 31 +#define MT6370_IRQ_CHG_IINMEAS 32 +#define MT6370_IRQ_CHG_ICCMEAS 33 +#define MT6370_IRQ_CHGDET_DONE 34 +#define MT6370_IRQ_WDTMR 35 +#define MT6370_IRQ_SSFINISH 36 +#define MT6370_IRQ_CHG_RECHG 37 +#define MT6370_IRQ_CHG_TERM 38 +#define MT6370_IRQ_CHG_IEOC 39 +#define MT6370_IRQ_ADC_DONE 40 +#define MT6370_IRQ_PUMPX_DONE 41 +#define MT6370_IRQ_BST_BATUV 45 +#define MT6370_IRQ_BST_MIDOV 46 +#define MT6370_IRQ_BST_OLP 47 +#define MT6370_IRQ_ATTACH 48 +#define MT6370_IRQ_DETACH 49 +#define MT6370_IRQ_HVDCP_STPDONE 51 +#define MT6370_IRQ_HVDCP_VBUSDET_DONE 52 +#define MT6370_IRQ_HVDCP_DET 53 +#define MT6370_IRQ_CHGDET 54 +#define MT6370_IRQ_DCDT 55 +#define MT6370_IRQ_DIRCHG_VGOK 59 +#define MT6370_IRQ_DIRCHG_WDTMR 60 +#define MT6370_IRQ_DIRCHG_UC 61 +#define MT6370_IRQ_DIRCHG_OC 62 +#define MT6370_IRQ_DIRCHG_OV 63 +#define MT6370_IRQ_OVPCTRL_SWON 67 +#define MT6370_IRQ_OVPCTRL_UVP_D 68 +#define MT6370_IRQ_OVPCTRL_UVP 69 +#define MT6370_IRQ_OVPCTRL_OVP_D 70 +#define MT6370_IRQ_OVPCTRL_OVP 71 +#define MT6370_IRQ_FLED_STRBPIN 72 +#define MT6370_IRQ_FLED_TORPIN 73 +#define MT6370_IRQ_FLED_TX 74 +#define MT6370_IRQ_FLED_LVF 75 +#define MT6370_IRQ_FLED2_SHORT 78 +#define MT6370_IRQ_FLED1_SHORT 79 +#define MT6370_IRQ_FLED2_STRB 80 +#define MT6370_IRQ_FLED1_STRB 81 +#define mT6370_IRQ_FLED2_STRB_TO 82 +#define MT6370_IRQ_FLED1_STRB_TO 83 +#define MT6370_IRQ_FLED2_TOR 84 +#define MT6370_IRQ_FLED1_TOR 85 +#define MT6370_IRQ_OTP 93 +#define MT6370_IRQ_VDDA_OVP 94 +#define MT6370_IRQ_VDDA_UV 95 +#define MT6370_IRQ_LDO_OC 103 +#define MT6370_IRQ_BLED_OCP 118 +#define MT6370_IRQ_BLED_OVP 119 +#define MT6370_IRQ_DSV_VNEG_OCP 123 +#define MT6370_IRQ_DSV_VPOS_OCP 124 +#define MT6370_IRQ_DSV_BST_OCP 125 +#define MT6370_IRQ_DSV_VNEG_SCP 126 +#define MT6370_IRQ_DSV_VPOS_SCP 127 + +enum { + MT6370_USBC_I2C = 0, + MT6370_PMU_I2C, + MT6370_MAX_I2C +}; + +struct mt6370_info { + struct i2c_client *i2c[MT6370_MAX_I2C]; + struct regmap_irq_chip_data *irq_data; +}; + +#endif /* __MFD_MT6375_H__ */ diff --git a/drivers/mfd/ntxec.c b/drivers/mfd/ntxec.c index b711e73eedcb065e24bc32a2df4e73ab13a09444..e16a7a82a929d8124efe60dcf1a7d31060aaf4c5 100644 --- a/drivers/mfd/ntxec.c +++ b/drivers/mfd/ntxec.c @@ -239,15 +239,13 @@ static int ntxec_probe(struct i2c_client *client) return res; } -static int ntxec_remove(struct i2c_client *client) +static void ntxec_remove(struct i2c_client *client) { if (client == poweroff_restart_client) { poweroff_restart_client = NULL; pm_power_off = NULL; unregister_restart_handler(&ntxec_restart_handler); } - - return 0; } static const struct of_device_id of_ntxec_match_table[] = { diff --git a/drivers/mfd/ocelot-core.c b/drivers/mfd/ocelot-core.c new file mode 100644 index 0000000000000000000000000000000000000000..1816d52c65c5121d8d8c378c46d6b60c6d2dcc2f --- /dev/null +++ b/drivers/mfd/ocelot-core.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Core driver for the Ocelot chip family. + * + * The VSC7511, 7512, 7513, and 7514 can be controlled internally via an + * on-chip MIPS processor, or externally via SPI, I2C, PCIe. This core driver is + * intended to be the bus-agnostic glue between, for example, the SPI bus and + * the child devices. + * + * Copyright 2021-2022 Innovative Advantage Inc. + * + * Author: Colin Foster + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocelot.h" + +#define REG_GCB_SOFT_RST 0x0008 + +#define BIT_SOFT_CHIP_RST BIT(0) + +#define VSC7512_MIIM0_RES_START 0x7107009c +#define VSC7512_MIIM1_RES_START 0x710700c0 +#define VSC7512_MIIM_RES_SIZE 0x024 + +#define VSC7512_PHY_RES_START 0x710700f0 +#define VSC7512_PHY_RES_SIZE 0x004 + +#define VSC7512_GPIO_RES_START 0x71070034 +#define VSC7512_GPIO_RES_SIZE 0x06c + +#define VSC7512_SIO_CTRL_RES_START 0x710700f8 +#define VSC7512_SIO_CTRL_RES_SIZE 0x100 + +#define VSC7512_GCB_RST_SLEEP_US 100 +#define VSC7512_GCB_RST_TIMEOUT_US 100000 + +static int ocelot_gcb_chip_rst_status(struct ocelot_ddata *ddata) +{ + int val, err; + + err = regmap_read(ddata->gcb_regmap, REG_GCB_SOFT_RST, &val); + if (err) + return err; + + return val; +} + +int ocelot_chip_reset(struct device *dev) +{ + struct ocelot_ddata *ddata = dev_get_drvdata(dev); + int ret, val; + + /* + * Reset the entire chip here to put it into a completely known state. + * Other drivers may want to reset their own subsystems. The register + * self-clears, so one write is all that is needed and wait for it to + * clear. + */ + ret = regmap_write(ddata->gcb_regmap, REG_GCB_SOFT_RST, BIT_SOFT_CHIP_RST); + if (ret) + return ret; + + return readx_poll_timeout(ocelot_gcb_chip_rst_status, ddata, val, !val, + VSC7512_GCB_RST_SLEEP_US, VSC7512_GCB_RST_TIMEOUT_US); +} +EXPORT_SYMBOL_NS(ocelot_chip_reset, MFD_OCELOT); + +static const struct resource vsc7512_miim0_resources[] = { + DEFINE_RES_REG_NAMED(VSC7512_MIIM0_RES_START, VSC7512_MIIM_RES_SIZE, "gcb_miim0"), + DEFINE_RES_REG_NAMED(VSC7512_PHY_RES_START, VSC7512_PHY_RES_SIZE, "gcb_phy"), +}; + +static const struct resource vsc7512_miim1_resources[] = { + DEFINE_RES_REG_NAMED(VSC7512_MIIM1_RES_START, VSC7512_MIIM_RES_SIZE, "gcb_miim1"), +}; + +static const struct resource vsc7512_pinctrl_resources[] = { + DEFINE_RES_REG_NAMED(VSC7512_GPIO_RES_START, VSC7512_GPIO_RES_SIZE, "gcb_gpio"), +}; + +static const struct resource vsc7512_sgpio_resources[] = { + DEFINE_RES_REG_NAMED(VSC7512_SIO_CTRL_RES_START, VSC7512_SIO_CTRL_RES_SIZE, "gcb_sio"), +}; + +static const struct mfd_cell vsc7512_devs[] = { + { + .name = "ocelot-pinctrl", + .of_compatible = "mscc,ocelot-pinctrl", + .num_resources = ARRAY_SIZE(vsc7512_pinctrl_resources), + .resources = vsc7512_pinctrl_resources, + }, { + .name = "ocelot-sgpio", + .of_compatible = "mscc,ocelot-sgpio", + .num_resources = ARRAY_SIZE(vsc7512_sgpio_resources), + .resources = vsc7512_sgpio_resources, + }, { + .name = "ocelot-miim0", + .of_compatible = "mscc,ocelot-miim", + .of_reg = VSC7512_MIIM0_RES_START, + .use_of_reg = true, + .num_resources = ARRAY_SIZE(vsc7512_miim0_resources), + .resources = vsc7512_miim0_resources, + }, { + .name = "ocelot-miim1", + .of_compatible = "mscc,ocelot-miim", + .of_reg = VSC7512_MIIM1_RES_START, + .use_of_reg = true, + .num_resources = ARRAY_SIZE(vsc7512_miim1_resources), + .resources = vsc7512_miim1_resources, + }, +}; + +static void ocelot_core_try_add_regmap(struct device *dev, + const struct resource *res) +{ + if (dev_get_regmap(dev, res->name)) + return; + + ocelot_spi_init_regmap(dev, res); +} + +static void ocelot_core_try_add_regmaps(struct device *dev, + const struct mfd_cell *cell) +{ + int i; + + for (i = 0; i < cell->num_resources; i++) + ocelot_core_try_add_regmap(dev, &cell->resources[i]); +} + +int ocelot_core_init(struct device *dev) +{ + int i, ndevs; + + ndevs = ARRAY_SIZE(vsc7512_devs); + + for (i = 0; i < ndevs; i++) + ocelot_core_try_add_regmaps(dev, &vsc7512_devs[i]); + + return devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, vsc7512_devs, ndevs, NULL, 0, NULL); +} +EXPORT_SYMBOL_NS(ocelot_core_init, MFD_OCELOT); + +MODULE_DESCRIPTION("Externally Controlled Ocelot Chip Driver"); +MODULE_AUTHOR("Colin Foster "); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(MFD_OCELOT_SPI); diff --git a/drivers/mfd/ocelot-spi.c b/drivers/mfd/ocelot-spi.c new file mode 100644 index 0000000000000000000000000000000000000000..2ecd271de2fb960a8604a160813a8bacb16b9071 --- /dev/null +++ b/drivers/mfd/ocelot-spi.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * SPI core driver for the Ocelot chip family. + * + * This driver will handle everything necessary to allow for communication over + * SPI to the VSC7511, VSC7512, VSC7513 and VSC7514 chips. The main functions + * are to prepare the chip's SPI interface for a specific bus speed, and a host + * processor's endianness. This will create and distribute regmaps for any + * children. + * + * Copyright 2021-2022 Innovative Advantage Inc. + * + * Author: Colin Foster + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ocelot.h" + +#define REG_DEV_CPUORG_IF_CTRL 0x0000 +#define REG_DEV_CPUORG_IF_CFGSTAT 0x0004 + +#define CFGSTAT_IF_NUM_VCORE (0 << 24) +#define CFGSTAT_IF_NUM_VRAP (1 << 24) +#define CFGSTAT_IF_NUM_SI (2 << 24) +#define CFGSTAT_IF_NUM_MIIM (3 << 24) + +#define VSC7512_DEVCPU_ORG_RES_START 0x71000000 +#define VSC7512_DEVCPU_ORG_RES_SIZE 0x38 + +#define VSC7512_CHIP_REGS_RES_START 0x71070000 +#define VSC7512_CHIP_REGS_RES_SIZE 0x14 + +static const struct resource vsc7512_dev_cpuorg_resource = + DEFINE_RES_REG_NAMED(VSC7512_DEVCPU_ORG_RES_START, + VSC7512_DEVCPU_ORG_RES_SIZE, + "devcpu_org"); + +static const struct resource vsc7512_gcb_resource = + DEFINE_RES_REG_NAMED(VSC7512_CHIP_REGS_RES_START, + VSC7512_CHIP_REGS_RES_SIZE, + "devcpu_gcb_chip_regs"); + +static int ocelot_spi_initialize(struct device *dev) +{ + struct ocelot_ddata *ddata = dev_get_drvdata(dev); + u32 val, check; + int err; + + val = OCELOT_SPI_BYTE_ORDER; + + /* + * The SPI address must be big-endian, but we want the payload to match + * our CPU. These are two bits (0 and 1) but they're repeated such that + * the write from any configuration will be valid. The four + * configurations are: + * + * 0b00: little-endian, MSB first + * | 111111 | 22221111 | 33222222 | + * | 76543210 | 54321098 | 32109876 | 10987654 | + * + * 0b01: big-endian, MSB first + * | 33222222 | 22221111 | 111111 | | + * | 10987654 | 32109876 | 54321098 | 76543210 | + * + * 0b10: little-endian, LSB first + * | 111111 | 11112222 | 22222233 | + * | 01234567 | 89012345 | 67890123 | 45678901 | + * + * 0b11: big-endian, LSB first + * | 22222233 | 11112222 | 111111 | | + * | 45678901 | 67890123 | 89012345 | 01234567 | + */ + err = regmap_write(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CTRL, val); + if (err) + return err; + + /* + * Apply the number of padding bytes between a read request and the data + * payload. Some registers have access times of up to 1us, so if the + * first payload bit is shifted out too quickly, the read will fail. + */ + val = ddata->spi_padding_bytes; + err = regmap_write(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CFGSTAT, val); + if (err) + return err; + + /* + * After we write the interface configuration, read it back here. This + * will verify several different things. The first is that the number of + * padding bytes actually got written correctly. These are found in bits + * 0:3. + * + * The second is that bit 16 is cleared. Bit 16 is IF_CFGSTAT:IF_STAT, + * and will be set if the register access is too fast. This would be in + * the condition that the number of padding bytes is insufficient for + * the SPI bus frequency. + * + * The last check is for bits 31:24, which define the interface by which + * the registers are being accessed. Since we're accessing them via the + * serial interface, it must return IF_NUM_SI. + */ + check = val | CFGSTAT_IF_NUM_SI; + + err = regmap_read(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CFGSTAT, &val); + if (err) + return err; + + if (check != val) + return -ENODEV; + + return 0; +} + +static const struct regmap_config ocelot_spi_regmap_config = { + .reg_bits = 24, + .reg_stride = 4, + .reg_downshift = 2, + .val_bits = 32, + + .write_flag_mask = 0x80, + + .use_single_write = true, + .can_multi_write = false, + + .reg_format_endian = REGMAP_ENDIAN_BIG, + .val_format_endian = REGMAP_ENDIAN_NATIVE, +}; + +static int ocelot_spi_regmap_bus_read(void *context, const void *reg, size_t reg_size, + void *val, size_t val_size) +{ + struct spi_transfer xfers[3] = {0}; + struct device *dev = context; + struct ocelot_ddata *ddata; + struct spi_device *spi; + struct spi_message msg; + unsigned int index = 0; + + ddata = dev_get_drvdata(dev); + spi = to_spi_device(dev); + + xfers[index].tx_buf = reg; + xfers[index].len = reg_size; + index++; + + if (ddata->spi_padding_bytes) { + xfers[index].len = ddata->spi_padding_bytes; + xfers[index].tx_buf = ddata->dummy_buf; + xfers[index].dummy_data = 1; + index++; + } + + xfers[index].rx_buf = val; + xfers[index].len = val_size; + index++; + + spi_message_init_with_transfers(&msg, xfers, index); + + return spi_sync(spi, &msg); +} + +static int ocelot_spi_regmap_bus_write(void *context, const void *data, size_t count) +{ + struct device *dev = context; + struct spi_device *spi = to_spi_device(dev); + + return spi_write(spi, data, count); +} + +static const struct regmap_bus ocelot_spi_regmap_bus = { + .write = ocelot_spi_regmap_bus_write, + .read = ocelot_spi_regmap_bus_read, +}; + +struct regmap *ocelot_spi_init_regmap(struct device *dev, const struct resource *res) +{ + struct regmap_config regmap_config; + + memcpy(®map_config, &ocelot_spi_regmap_config, sizeof(regmap_config)); + + regmap_config.name = res->name; + regmap_config.max_register = resource_size(res) - 1; + regmap_config.reg_base = res->start; + + return devm_regmap_init(dev, &ocelot_spi_regmap_bus, dev, ®map_config); +} +EXPORT_SYMBOL_NS(ocelot_spi_init_regmap, MFD_OCELOT_SPI); + +static int ocelot_spi_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct ocelot_ddata *ddata; + struct regmap *r; + int err; + + ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) + return -ENOMEM; + + spi_set_drvdata(spi, ddata); + + if (spi->max_speed_hz <= 500000) { + ddata->spi_padding_bytes = 0; + } else { + /* + * Calculation taken from the manual for IF_CFGSTAT:IF_CFG. + * Register access time is 1us, so we need to configure and send + * out enough padding bytes between the read request and data + * transmission that lasts at least 1 microsecond. + */ + ddata->spi_padding_bytes = 1 + (spi->max_speed_hz / HZ_PER_MHZ + 2) / 8; + + ddata->dummy_buf = devm_kzalloc(dev, ddata->spi_padding_bytes, GFP_KERNEL); + if (!ddata->dummy_buf) + return -ENOMEM; + } + + spi->bits_per_word = 8; + + err = spi_setup(spi); + if (err) + return dev_err_probe(&spi->dev, err, "Error performing SPI setup\n"); + + r = ocelot_spi_init_regmap(dev, &vsc7512_dev_cpuorg_resource); + if (IS_ERR(r)) + return PTR_ERR(r); + + ddata->cpuorg_regmap = r; + + r = ocelot_spi_init_regmap(dev, &vsc7512_gcb_resource); + if (IS_ERR(r)) + return PTR_ERR(r); + + ddata->gcb_regmap = r; + + /* + * The chip must be set up for SPI before it gets initialized and reset. + * This must be done before calling init, and after a chip reset is + * performed. + */ + err = ocelot_spi_initialize(dev); + if (err) + return dev_err_probe(dev, err, "Error initializing SPI bus\n"); + + err = ocelot_chip_reset(dev); + if (err) + return dev_err_probe(dev, err, "Error resetting device\n"); + + /* + * A chip reset will clear the SPI configuration, so it needs to be done + * again before we can access any registers. + */ + err = ocelot_spi_initialize(dev); + if (err) + return dev_err_probe(dev, err, "Error initializing SPI bus after reset\n"); + + err = ocelot_core_init(dev); + if (err) + return dev_err_probe(dev, err, "Error initializing Ocelot core\n"); + + return 0; +} + +static const struct spi_device_id ocelot_spi_ids[] = { + { "vsc7512", 0 }, + { } +}; +MODULE_DEVICE_TABLE(spi, ocelot_spi_ids); + +static const struct of_device_id ocelot_spi_of_match[] = { + { .compatible = "mscc,vsc7512" }, + { } +}; +MODULE_DEVICE_TABLE(of, ocelot_spi_of_match); + +static struct spi_driver ocelot_spi_driver = { + .driver = { + .name = "ocelot-soc", + .of_match_table = ocelot_spi_of_match, + }, + .id_table = ocelot_spi_ids, + .probe = ocelot_spi_probe, +}; +module_spi_driver(ocelot_spi_driver); + +MODULE_DESCRIPTION("SPI Controlled Ocelot Chip Driver"); +MODULE_AUTHOR("Colin Foster "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_IMPORT_NS(MFD_OCELOT); diff --git a/drivers/mfd/ocelot.h b/drivers/mfd/ocelot.h new file mode 100644 index 0000000000000000000000000000000000000000..b8bc2f1486e2480d1bfe47563c85013975cee69d --- /dev/null +++ b/drivers/mfd/ocelot.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* Copyright 2021, 2022 Innovative Advantage Inc. */ + +#ifndef _MFD_OCELOT_H +#define _MFD_OCELOT_H + +#include + +struct device; +struct regmap; +struct resource; + +/** + * struct ocelot_ddata - Private data for an external Ocelot chip + * @gcb_regmap: General Configuration Block regmap. Used for + * operations like chip reset. + * @cpuorg_regmap: CPU Device Origin Block regmap. Used for operations + * like SPI bus configuration. + * @spi_padding_bytes: Number of padding bytes that must be thrown out before + * read data gets returned. This is calculated during + * initialization based on bus speed. + * @dummy_buf: Zero-filled buffer of spi_padding_bytes size. The dummy + * bytes that will be sent out between the address and + * data of a SPI read operation. + */ +struct ocelot_ddata { + struct regmap *gcb_regmap; + struct regmap *cpuorg_regmap; + int spi_padding_bytes; + void *dummy_buf; +}; + +int ocelot_chip_reset(struct device *dev); +int ocelot_core_init(struct device *dev); + +/* SPI-specific routines that won't be necessary for other interfaces */ +struct regmap *ocelot_spi_init_regmap(struct device *dev, + const struct resource *res); + +#define OCELOT_SPI_BYTE_ORDER_LE 0x00000000 +#define OCELOT_SPI_BYTE_ORDER_BE 0x81818181 + +#ifdef __LITTLE_ENDIAN +#define OCELOT_SPI_BYTE_ORDER OCELOT_SPI_BYTE_ORDER_LE +#else +#define OCELOT_SPI_BYTE_ORDER OCELOT_SPI_BYTE_ORDER_BE +#endif + +#endif diff --git a/drivers/mfd/palmas.c b/drivers/mfd/palmas.c index f5b3fa973b132e97c5cfeb5dfa94d6d23f569f9c..8b7429bd2e3eb14f1994fe272256ee0b81646476 100644 --- a/drivers/mfd/palmas.c +++ b/drivers/mfd/palmas.c @@ -700,7 +700,7 @@ err_i2c: return ret; } -static int palmas_i2c_remove(struct i2c_client *i2c) +static void palmas_i2c_remove(struct i2c_client *i2c) { struct palmas *palmas = i2c_get_clientdata(i2c); int i; @@ -716,8 +716,6 @@ static int palmas_i2c_remove(struct i2c_client *i2c) pm_power_off = NULL; palmas_dev = NULL; } - - return 0; } static const struct i2c_device_id palmas_i2c_id[] = { diff --git a/drivers/mfd/pcf50633-core.c b/drivers/mfd/pcf50633-core.c index e9c565cf0f54f29b1eb704298301bc8d6b311135..4ccc2c3e7681d78bbb505c72e3ecdd85d9b6fb72 100644 --- a/drivers/mfd/pcf50633-core.c +++ b/drivers/mfd/pcf50633-core.c @@ -273,7 +273,7 @@ err2: return ret; } -static int pcf50633_remove(struct i2c_client *client) +static void pcf50633_remove(struct i2c_client *client) { struct pcf50633 *pcf = i2c_get_clientdata(client); int i; @@ -289,8 +289,6 @@ static int pcf50633_remove(struct i2c_client *client) for (i = 0; i < PCF50633_NUM_REGULATORS; i++) platform_device_unregister(pcf->regulator_pdev[i]); - - return 0; } static const struct i2c_device_id pcf50633_id_table[] = { diff --git a/drivers/mfd/qcom-spmi-pmic.c b/drivers/mfd/qcom-spmi-pmic.c index 00003a868d287097cb36c70ffb8262de7b480883..7e2cd79d17ebf850ab3bf0072275450bb795f95b 100644 --- a/drivers/mfd/qcom-spmi-pmic.c +++ b/drivers/mfd/qcom-spmi-pmic.c @@ -60,6 +60,7 @@ static const struct of_device_id pmic_spmi_id_table[] = { { .compatible = "qcom,pmi8994", .data = N_USIDS(2) }, { .compatible = "qcom,pmi8998", .data = N_USIDS(2) }, { .compatible = "qcom,pmk8002", .data = N_USIDS(2) }, + { .compatible = "qcom,pmp8074", .data = N_USIDS(2) }, { .compatible = "qcom,smb2351", .data = N_USIDS(2) }, { .compatible = "qcom,spmi-pmic", .data = N_USIDS(1) }, { } diff --git a/drivers/mfd/retu-mfd.c b/drivers/mfd/retu-mfd.c index c748fd29a22045fde5c983912f46680a1be2cf55..3b5acf7ca39cbca4dd02ed7cbe5c4271958692e9 100644 --- a/drivers/mfd/retu-mfd.c +++ b/drivers/mfd/retu-mfd.c @@ -287,7 +287,7 @@ static int retu_probe(struct i2c_client *i2c, const struct i2c_device_id *id) return 0; } -static int retu_remove(struct i2c_client *i2c) +static void retu_remove(struct i2c_client *i2c) { struct retu_dev *rdev = i2c_get_clientdata(i2c); @@ -297,8 +297,6 @@ static int retu_remove(struct i2c_client *i2c) } mfd_remove_devices(rdev->dev); regmap_del_irq_chip(i2c->irq, rdev->irq_data); - - return 0; } static const struct i2c_device_id retu_id[] = { diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c index 4142b638e5fa8491107e639c444b7342aa0d27ef..e00da7c7e3b11c83530c211f62271c4251a8d80b 100644 --- a/drivers/mfd/rk808.c +++ b/drivers/mfd/rk808.c @@ -67,6 +67,10 @@ static bool rk817_is_volatile_reg(struct device *dev, unsigned int reg) case RK817_SECONDS_REG ... RK817_WEEKS_REG: case RK817_RTC_STATUS_REG: case RK817_CODEC_DTOP_LPT_SRST: + case RK817_GAS_GAUGE_ADC_CONFIG0 ... RK817_GAS_GAUGE_CUR_ADC_K0: + case RK817_PMIC_CHRG_STS: + case RK817_PMIC_CHRG_OUT: + case RK817_PMIC_CHRG_IN: case RK817_INT_STS_REG0: case RK817_INT_STS_REG1: case RK817_INT_STS_REG2: @@ -74,7 +78,7 @@ static bool rk817_is_volatile_reg(struct device *dev, unsigned int reg) return true; } - return true; + return false; } static const struct regmap_config rk818_regmap_config = { @@ -127,6 +131,11 @@ static const struct resource rk817_pwrkey_resources[] = { DEFINE_RES_IRQ(RK817_IRQ_PWRON_FALL), }; +static const struct resource rk817_charger_resources[] = { + DEFINE_RES_IRQ(RK817_IRQ_PLUG_IN), + DEFINE_RES_IRQ(RK817_IRQ_PLUG_OUT), +}; + static const struct mfd_cell rk805s[] = { { .name = "rk808-clkout", }, { .name = "rk808-regulator", }, @@ -166,6 +175,11 @@ static const struct mfd_cell rk817s[] = { .resources = &rk817_rtc_resources[0], }, { .name = "rk817-codec",}, + { + .name = "rk817-charger", + .num_resources = ARRAY_SIZE(rk817_charger_resources), + .resources = &rk817_charger_resources[0], + }, }; static const struct mfd_cell rk818s[] = { @@ -778,7 +792,7 @@ err_irq: return ret; } -static int rk808_remove(struct i2c_client *client) +static void rk808_remove(struct i2c_client *client) { struct rk808 *rk808 = i2c_get_clientdata(client); @@ -792,8 +806,6 @@ static int rk808_remove(struct i2c_client *client) pm_power_off = NULL; unregister_restart_handler(&rk808_restart_handler); - - return 0; } static int __maybe_unused rk8xx_suspend(struct device *dev) diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c index 384acb4594272bb7be3b0c947fb65aa40d3b53a2..eb8005b4e58d3aa0737fc3fd5a48b65ff5f979cf 100644 --- a/drivers/mfd/rn5t618.c +++ b/drivers/mfd/rn5t618.c @@ -241,7 +241,7 @@ static int rn5t618_i2c_probe(struct i2c_client *i2c) return rn5t618_irq_init(priv); } -static int rn5t618_i2c_remove(struct i2c_client *i2c) +static void rn5t618_i2c_remove(struct i2c_client *i2c) { if (i2c == rn5t618_pm_power_off) { rn5t618_pm_power_off = NULL; @@ -249,8 +249,6 @@ static int rn5t618_i2c_remove(struct i2c_client *i2c) } unregister_restart_handler(&rn5t618_restart_handler); - - return 0; } static int __maybe_unused rn5t618_i2c_suspend(struct device *dev) diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c index dc001c9791c16e656042325429b0827802ff1fcf..f716ab8039a06eeb6c1e9fd0009ebab2b72f7273 100644 --- a/drivers/mfd/rsmu_i2c.c +++ b/drivers/mfd/rsmu_i2c.c @@ -146,13 +146,11 @@ static int rsmu_i2c_probe(struct i2c_client *client, return rsmu_core_init(rsmu); } -static int rsmu_i2c_remove(struct i2c_client *client) +static void rsmu_i2c_remove(struct i2c_client *client) { struct rsmu_ddata *rsmu = i2c_get_clientdata(client); rsmu_core_exit(rsmu); - - return 0; } static const struct i2c_device_id rsmu_i2c_id[] = { diff --git a/drivers/mfd/rt4831.c b/drivers/mfd/rt4831.c index fb3bd788a3eb2a8f0d93a111f8305a5ace6a2667..c6d34dc2b5206d9e2235ea04b1877cc6291dbc6e 100644 --- a/drivers/mfd/rt4831.c +++ b/drivers/mfd/rt4831.c @@ -87,7 +87,7 @@ static int rt4831_probe(struct i2c_client *client) ARRAY_SIZE(rt4831_subdevs), NULL, 0, NULL); } -static int rt4831_remove(struct i2c_client *client) +static void rt4831_remove(struct i2c_client *client) { struct regmap *regmap = dev_get_regmap(&client->dev, NULL); int ret; @@ -96,8 +96,6 @@ static int rt4831_remove(struct i2c_client *client) ret = regmap_update_bits(regmap, RT4831_REG_ENABLE, RT4831_RESET_MASK, RT4831_RESET_MASK); if (ret) dev_warn(&client->dev, "Failed to disable outputs (%pe)\n", ERR_PTR(ret)); - - return 0; } static const struct of_device_id __maybe_unused rt4831_of_match[] = { diff --git a/drivers/mfd/rt5120.c b/drivers/mfd/rt5120.c new file mode 100644 index 0000000000000000000000000000000000000000..8046e383bc92b521e00ab109822c105080a724fe --- /dev/null +++ b/drivers/mfd/rt5120.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * Author: ChiYuan Huang + */ + +#include +#include +#include +#include +#include +#include + +#define RT5120_REG_INTENABLE 0x1D +#define RT5120_REG_INTSTAT 0x1E +#define RT5120_REG_FZCMODE 0x44 + +#define RT5120_INT_HOTDIE 0 +#define RT5120_INT_PWRKEY_REL 5 +#define RT5120_INT_PWRKEY_PRESS 6 + +static const struct regmap_range rt5120_rd_yes_ranges[] = { + regmap_reg_range(0x03, 0x13), + regmap_reg_range(0x1c, 0x20), + regmap_reg_range(0x44, 0x44), +}; + +static const struct regmap_range rt5120_wr_yes_ranges[] = { + regmap_reg_range(0x06, 0x13), + regmap_reg_range(0x1c, 0x20), + regmap_reg_range(0x44, 0x44), +}; + +static const struct regmap_access_table rt5120_rd_table = { + .yes_ranges = rt5120_rd_yes_ranges, + .n_yes_ranges = ARRAY_SIZE(rt5120_rd_yes_ranges), +}; + +static const struct regmap_access_table rt5120_wr_table = { + .yes_ranges = rt5120_wr_yes_ranges, + .n_yes_ranges = ARRAY_SIZE(rt5120_wr_yes_ranges), +}; + +static const struct regmap_config rt5120_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = RT5120_REG_FZCMODE, + + .wr_table = &rt5120_wr_table, + .rd_table = &rt5120_rd_table, +}; + +static const struct regmap_irq rt5120_irqs[] = { + REGMAP_IRQ_REG_LINE(RT5120_INT_HOTDIE, 8), + REGMAP_IRQ_REG_LINE(RT5120_INT_PWRKEY_REL, 8), + REGMAP_IRQ_REG_LINE(RT5120_INT_PWRKEY_PRESS, 8), +}; + +static const struct regmap_irq_chip rt5120_irq_chip = { + .name = "rt5120-pmic", + .status_base = RT5120_REG_INTSTAT, + .mask_base = RT5120_REG_INTENABLE, + .ack_base = RT5120_REG_INTSTAT, + .mask_invert = true, + .use_ack = true, + .num_regs = 1, + .irqs = rt5120_irqs, + .num_irqs = ARRAY_SIZE(rt5120_irqs), +}; + +static const struct resource rt5120_regulator_resources[] = { + DEFINE_RES_IRQ(RT5120_INT_HOTDIE), +}; + +static const struct resource rt5120_pwrkey_resources[] = { + DEFINE_RES_IRQ_NAMED(RT5120_INT_PWRKEY_PRESS, "pwrkey-press"), + DEFINE_RES_IRQ_NAMED(RT5120_INT_PWRKEY_REL, "pwrkey-release"), +}; + +static const struct mfd_cell rt5120_devs[] = { + MFD_CELL_RES("rt5120-regulator", rt5120_regulator_resources), + MFD_CELL_OF("rt5120-pwrkey", rt5120_pwrkey_resources, NULL, 0, 0, "richtek,rt5120-pwrkey"), +}; + +static int rt5120_probe(struct i2c_client *i2c) +{ + struct device *dev = &i2c->dev; + struct regmap *regmap; + struct regmap_irq_chip_data *irq_data; + int ret; + + regmap = devm_regmap_init_i2c(i2c, &rt5120_regmap_config); + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), + "Failed to init regmap\n"); + + ret = devm_regmap_add_irq_chip(dev, regmap, i2c->irq, IRQF_ONESHOT, 0, + &rt5120_irq_chip, &irq_data); + if (ret) + return dev_err_probe(dev, ret, "Failed to add IRQ chip\n"); + + return devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, rt5120_devs, + ARRAY_SIZE(rt5120_devs), NULL, 0, + regmap_irq_get_domain(irq_data)); +} + +static const struct of_device_id rt5120_device_match_table[] = { + { .compatible = "richtek,rt5120" }, + {} +}; +MODULE_DEVICE_TABLE(of, rt5120_device_match_table); + +static struct i2c_driver rt5120_driver = { + .driver = { + .name = "rt5120", + .of_match_table = rt5120_device_match_table, + }, + .probe_new = rt5120_probe, +}; +module_i2c_driver(rt5120_driver); + +MODULE_AUTHOR("ChiYuan Huang "); +MODULE_DESCRIPTION("Richtek RT5120 I2C driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mfd/si476x-i2c.c b/drivers/mfd/si476x-i2c.c index a2635c2d9d1a9f9dab88935bb8425b64ecb07720..8166949b725cc6f6d789bbc713540974eb1ccd3a 100644 --- a/drivers/mfd/si476x-i2c.c +++ b/drivers/mfd/si476x-i2c.c @@ -835,7 +835,7 @@ free_gpio: return rval; } -static int si476x_core_remove(struct i2c_client *client) +static void si476x_core_remove(struct i2c_client *client) { struct si476x_core *core = i2c_get_clientdata(client); @@ -851,8 +851,6 @@ static int si476x_core_remove(struct i2c_client *client) if (gpio_is_valid(core->gpio_reset)) gpio_free(core->gpio_reset); - - return 0; } diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c index bc0a2c38653e51c80dd0714c01456ac00cf946a3..3ac4508a6742ac8e489e6c4502000834140d2e75 100644 --- a/drivers/mfd/sm501.c +++ b/drivers/mfd/sm501.c @@ -1720,7 +1720,12 @@ static struct platform_driver sm501_plat_driver = { static int __init sm501_base_init(void) { - platform_driver_register(&sm501_plat_driver); + int ret; + + ret = platform_driver_register(&sm501_plat_driver); + if (ret < 0) + return ret; + return pci_register_driver(&sm501_pci_driver); } diff --git a/drivers/mfd/stmfx.c b/drivers/mfd/stmfx.c index 122f96094410fb3e57c555c84eed2f28e7634178..5dd7d96884596f75e90877a1c4f768c8cf42ae30 100644 --- a/drivers/mfd/stmfx.c +++ b/drivers/mfd/stmfx.c @@ -467,13 +467,11 @@ err_chip_exit: return ret; } -static int stmfx_remove(struct i2c_client *client) +static void stmfx_remove(struct i2c_client *client) { stmfx_irq_exit(client); stmfx_chip_exit(client); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/mfd/stmpe-i2c.c b/drivers/mfd/stmpe-i2c.c index d3eedf3d607eaf89882f57af972e14c32c059112..4d55494a97c4b0bf68f535a657fafd5e47552332 100644 --- a/drivers/mfd/stmpe-i2c.c +++ b/drivers/mfd/stmpe-i2c.c @@ -91,13 +91,11 @@ stmpe_i2c_probe(struct i2c_client *i2c, const struct i2c_device_id *id) return stmpe_probe(&i2c_ci, partnum); } -static int stmpe_i2c_remove(struct i2c_client *i2c) +static void stmpe_i2c_remove(struct i2c_client *i2c) { struct stmpe *stmpe = dev_get_drvdata(&i2c->dev); stmpe_remove(stmpe); - - return 0; } static const struct i2c_device_id stmpe_i2c_id[] = { diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c index aeb9ea55f97d98c1b33e879414da4b81d79e1752..0c4f74197d3e04c6befbef757b66d81db9ff72c9 100644 --- a/drivers/mfd/stmpe.c +++ b/drivers/mfd/stmpe.c @@ -8,14 +8,13 @@ */ #include -#include +#include #include #include #include #include #include #include -#include #include #include #include @@ -30,17 +29,12 @@ * @irq_trigger: IRQ trigger to use for the interrupt to the host * @autosleep: bool to enable/disable stmpe autosleep * @autosleep_timeout: inactivity timeout in milliseconds for autosleep - * @irq_over_gpio: true if gpio is used to get irq - * @irq_gpio: gpio number over which irq will be requested (significant only if - * irq_over_gpio is true) */ struct stmpe_platform_data { int id; unsigned int blocks; unsigned int irq_trigger; bool autosleep; - bool irq_over_gpio; - int irq_gpio; int autosleep_timeout; }; @@ -1349,32 +1343,22 @@ static void stmpe_of_probe(struct stmpe_platform_data *pdata, if (pdata->id < 0) pdata->id = -1; - pdata->irq_gpio = of_get_named_gpio_flags(np, "irq-gpio", 0, - &pdata->irq_trigger); - if (gpio_is_valid(pdata->irq_gpio)) - pdata->irq_over_gpio = 1; - else - pdata->irq_trigger = IRQF_TRIGGER_NONE; - of_property_read_u32(np, "st,autosleep-timeout", &pdata->autosleep_timeout); pdata->autosleep = (pdata->autosleep_timeout) ? true : false; for_each_available_child_of_node(np, child) { - if (of_node_name_eq(child, "stmpe_gpio")) { + if (of_device_is_compatible(child, stmpe_gpio_cell.of_compatible)) pdata->blocks |= STMPE_BLOCK_GPIO; - } else if (of_node_name_eq(child, "stmpe_keypad")) { + else if (of_device_is_compatible(child, stmpe_keypad_cell.of_compatible)) pdata->blocks |= STMPE_BLOCK_KEYPAD; - } else if (of_node_name_eq(child, "stmpe_touchscreen")) { + else if (of_device_is_compatible(child, stmpe_ts_cell.of_compatible)) pdata->blocks |= STMPE_BLOCK_TOUCHSCREEN; - } else if (of_node_name_eq(child, "stmpe_adc")) { + else if (of_device_is_compatible(child, stmpe_adc_cell.of_compatible)) pdata->blocks |= STMPE_BLOCK_ADC; - } else if (of_node_name_eq(child, "stmpe_pwm")) { + else if (of_device_is_compatible(child, stmpe_pwm_cell.of_compatible)) pdata->blocks |= STMPE_BLOCK_PWM; - } else if (of_node_name_eq(child, "stmpe_rotator")) { - pdata->blocks |= STMPE_BLOCK_ROTATOR; - } } } @@ -1384,6 +1368,7 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) struct stmpe_platform_data *pdata; struct device_node *np = ci->dev->of_node; struct stmpe *stmpe; + struct gpio_desc *irq_gpio; int ret; u32 val; @@ -1437,18 +1422,20 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) if (ci->init) ci->init(stmpe); - if (pdata->irq_over_gpio) { - ret = devm_gpio_request_one(ci->dev, pdata->irq_gpio, - GPIOF_DIR_IN, "stmpe"); - if (ret) { - dev_err(stmpe->dev, "failed to request IRQ GPIO: %d\n", - ret); - return ret; - } + irq_gpio = devm_gpiod_get_optional(ci->dev, "irq", GPIOD_ASIS); + ret = PTR_ERR_OR_ZERO(irq_gpio); + if (ret) { + dev_err(stmpe->dev, "failed to request IRQ GPIO: %d\n", ret); + return ret; + } - stmpe->irq = gpio_to_irq(pdata->irq_gpio); + if (irq_gpio) { + stmpe->irq = gpiod_to_irq(irq_gpio); + pdata->irq_trigger = gpiod_is_active_low(irq_gpio) ? + IRQF_TRIGGER_LOW : IRQF_TRIGGER_HIGH; } else { stmpe->irq = ci->irq; + pdata->irq_trigger = IRQF_TRIGGER_NONE; } if (stmpe->irq < 0) { diff --git a/drivers/mfd/syscon.c b/drivers/mfd/syscon.c index bdb2ce7ff03b9d086b6ec0c95d0055cb4eb68635..9489e80e905a0f269f50ebb21e6a4add04567cc4 100644 --- a/drivers/mfd/syscon.c +++ b/drivers/mfd/syscon.c @@ -66,14 +66,6 @@ static struct syscon *of_syscon_register(struct device_node *np, bool check_clk) goto err_map; } - /* Parse the device's DT node for an endianness specification */ - if (of_property_read_bool(np, "big-endian")) - syscon_config.val_format_endian = REGMAP_ENDIAN_BIG; - else if (of_property_read_bool(np, "little-endian")) - syscon_config.val_format_endian = REGMAP_ENDIAN_LITTLE; - else if (of_property_read_bool(np, "native-endian")) - syscon_config.val_format_endian = REGMAP_ENDIAN_NATIVE; - /* * search for reg-io-width property in DT. If it is not provided, * default to 4 bytes. regmap_init_mmio will return an error if values diff --git a/drivers/mfd/tc3589x.c b/drivers/mfd/tc3589x.c index 13583cdb93b6fdbdc221d02ead09cb0b0f339501..d5d0ec117acb270ebb9946d7223a1404ba1c6e65 100644 --- a/drivers/mfd/tc3589x.c +++ b/drivers/mfd/tc3589x.c @@ -429,13 +429,11 @@ static int tc3589x_probe(struct i2c_client *i2c, return 0; } -static int tc3589x_remove(struct i2c_client *client) +static void tc3589x_remove(struct i2c_client *client) { struct tc3589x *tc3589x = i2c_get_clientdata(client); mfd_remove_devices(tc3589x->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/mfd/tps6105x.c b/drivers/mfd/tps6105x.c index c906324d293e236ac55a28fcad6383982f426175..b360568ea6750321fbed2dd550c28af89b8af266 100644 --- a/drivers/mfd/tps6105x.c +++ b/drivers/mfd/tps6105x.c @@ -179,7 +179,7 @@ static int tps6105x_probe(struct i2c_client *client, return ret; } -static int tps6105x_remove(struct i2c_client *client) +static void tps6105x_remove(struct i2c_client *client) { struct tps6105x *tps6105x = i2c_get_clientdata(client); @@ -189,8 +189,6 @@ static int tps6105x_remove(struct i2c_client *client) regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0, TPS6105X_REG0_MODE_MASK, TPS6105X_MODE_SHUTDOWN << TPS6105X_REG0_MODE_SHIFT); - - return 0; } static const struct i2c_device_id tps6105x_id[] = { diff --git a/drivers/mfd/tps65010.c b/drivers/mfd/tps65010.c index 7e7dbee58ca90feb10b330d31add4e61d1784e4c..c2afa2e69f42f8ab36ba3bd43245fd5feca91e92 100644 --- a/drivers/mfd/tps65010.c +++ b/drivers/mfd/tps65010.c @@ -501,7 +501,7 @@ static int tps65010_gpio_get(struct gpio_chip *chip, unsigned offset) static struct tps65010 *the_tps; -static int tps65010_remove(struct i2c_client *client) +static void tps65010_remove(struct i2c_client *client) { struct tps65010 *tps = i2c_get_clientdata(client); struct tps65010_board *board = dev_get_platdata(&client->dev); @@ -517,7 +517,6 @@ static int tps65010_remove(struct i2c_client *client) cancel_delayed_work_sync(&tps->work); debugfs_remove(tps->file); the_tps = NULL; - return 0; } static int tps65010_probe(struct i2c_client *client, diff --git a/drivers/mfd/tps65086.c b/drivers/mfd/tps65086.c index cbae9777a24e848df6a0a09e9e1819b52b8af4ab..81a7360a87bbcc3f23baba370d287e07f777de82 100644 --- a/drivers/mfd/tps65086.c +++ b/drivers/mfd/tps65086.c @@ -111,14 +111,12 @@ static int tps65086_probe(struct i2c_client *client, return ret; } -static int tps65086_remove(struct i2c_client *client) +static void tps65086_remove(struct i2c_client *client) { struct tps65086 *tps = i2c_get_clientdata(client); if (tps->irq > 0) regmap_del_irq_chip(tps->irq, tps->irq_data); - - return 0; } static const struct i2c_device_id tps65086_id_table[] = { diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c index 8e8da204a02e4c505e4ed16c7ce83bf74761c858..eebd60601b01ddc4b00f861e98813c9686998d2d 100644 --- a/drivers/mfd/tps65217.c +++ b/drivers/mfd/tps65217.c @@ -374,7 +374,7 @@ static int tps65217_probe(struct i2c_client *client) return 0; } -static int tps65217_remove(struct i2c_client *client) +static void tps65217_remove(struct i2c_client *client) { struct tps65217 *tps = i2c_get_clientdata(client); unsigned int virq; @@ -388,8 +388,6 @@ static int tps65217_remove(struct i2c_client *client) irq_domain_remove(tps->irq_domain); tps->irq_domain = NULL; - - return 0; } static const struct i2c_device_id tps65217_id_table[] = { diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c index c9303d3d660272a27dd5b928613e4d1cdff28370..fb340da64bbc42f331dc941a74b6adec88c43c5c 100644 --- a/drivers/mfd/tps6586x.c +++ b/drivers/mfd/tps6586x.c @@ -579,7 +579,7 @@ err_mfd_add: return ret; } -static int tps6586x_i2c_remove(struct i2c_client *client) +static void tps6586x_i2c_remove(struct i2c_client *client) { struct tps6586x *tps6586x = i2c_get_clientdata(client); @@ -587,7 +587,6 @@ static int tps6586x_i2c_remove(struct i2c_client *client) mfd_remove_devices(tps6586x->dev); if (client->irq) free_irq(client->irq, tps6586x); - return 0; } static int __maybe_unused tps6586x_i2c_suspend(struct device *dev) diff --git a/drivers/mfd/tps65912-i2c.c b/drivers/mfd/tps65912-i2c.c index afb7f7d97dc05d5e1f716c084fb3c2255fbce45c..7e2b19efe8679de96d3fffd576ea11335040ffc2 100644 --- a/drivers/mfd/tps65912-i2c.c +++ b/drivers/mfd/tps65912-i2c.c @@ -43,13 +43,11 @@ static int tps65912_i2c_probe(struct i2c_client *client, return tps65912_device_init(tps); } -static int tps65912_i2c_remove(struct i2c_client *client) +static void tps65912_i2c_remove(struct i2c_client *client) { struct tps65912 *tps = i2c_get_clientdata(client); tps65912_device_exit(tps); - - return 0; } static const struct i2c_device_id tps65912_i2c_id_table[] = { diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c index 2cb9326f3e61b2afded551d7bd634af47d2cebe9..f6b4b9d94bbd3c194e92fdd4719749dc3b771744 100644 --- a/drivers/mfd/twl-core.c +++ b/drivers/mfd/twl-core.c @@ -727,7 +727,7 @@ static void clocks_init(struct device *dev) /*----------------------------------------------------------------------*/ -static int twl_remove(struct i2c_client *client) +static void twl_remove(struct i2c_client *client) { unsigned i, num_slaves; @@ -745,7 +745,6 @@ static int twl_remove(struct i2c_client *client) twl->client = NULL; } twl_priv->ready = false; - return 0; } static struct of_dev_auxdata twl_auxdata_lookup[] = { @@ -883,7 +882,7 @@ twl_probe(struct i2c_client *client, const struct i2c_device_id *id) * SR_I2C_SCL_CTRL_PU(bit 4)=0 and SR_I2C_SDA_CTRL_PU(bit 6)=0. * * Also, always enable SmartReflex bit as that's needed for omaps to - * to do anything over I2C4 for voltage scaling even if SmartReflex + * do anything over I2C4 for voltage scaling even if SmartReflex * is disabled. Without the SmartReflex bit omap sys_clkreq idle * signal will never trigger for retention idle. */ diff --git a/drivers/mfd/twl4030-irq.c b/drivers/mfd/twl4030-irq.c index 4f576f0160a94ee7f34b70863b361dee33357907..87496c1cb8bca617db30382cf06429482ceeeb7d 100644 --- a/drivers/mfd/twl4030-irq.c +++ b/drivers/mfd/twl4030-irq.c @@ -14,6 +14,7 @@ * by syed khasim */ +#include #include #include #include diff --git a/drivers/mfd/twl6040.c b/drivers/mfd/twl6040.c index b9c6d94b400206b8ae34830cfa9741ee95364d0c..f429b8f00db61c11ea9689138ddb61a2db7febbc 100644 --- a/drivers/mfd/twl6040.c +++ b/drivers/mfd/twl6040.c @@ -808,7 +808,7 @@ gpio_err: return ret; } -static int twl6040_remove(struct i2c_client *client) +static void twl6040_remove(struct i2c_client *client) { struct twl6040 *twl6040 = i2c_get_clientdata(client); @@ -820,8 +820,6 @@ static int twl6040_remove(struct i2c_client *client) mfd_remove_devices(&client->dev); regulator_bulk_disable(TWL6040_NUM_SUPPLIES, twl6040->supplies); - - return 0; } static const struct i2c_device_id twl6040_i2c_id[] = { diff --git a/drivers/mfd/wm8994-core.c b/drivers/mfd/wm8994-core.c index 7b1d270722baa7443a8d37d0705515ebc5bd27c0..7e88f5b0abe6a064365bc677877349482409f32f 100644 --- a/drivers/mfd/wm8994-core.c +++ b/drivers/mfd/wm8994-core.c @@ -657,13 +657,11 @@ static int wm8994_i2c_probe(struct i2c_client *i2c, return wm8994_device_init(wm8994, i2c->irq); } -static int wm8994_i2c_remove(struct i2c_client *i2c) +static void wm8994_i2c_remove(struct i2c_client *i2c) { struct wm8994 *wm8994 = i2c_get_clientdata(i2c); wm8994_device_exit(wm8994); - - return 0; } static const struct i2c_device_id wm8994_i2c_id[] = { diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 94e9fb4cdd7611de6cc24c02b7c3849c14407092..358ad56f65245e2094960dad0e6ee6f52f0110b1 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -513,4 +513,5 @@ source "drivers/misc/cardreader/Kconfig" source "drivers/misc/habanalabs/Kconfig" source "drivers/misc/uacce/Kconfig" source "drivers/misc/pvpanic/Kconfig" +source "drivers/misc/mchp_pci1xxxx/Kconfig" endmenu diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 2be8542616dda506b9ca674a1c4f7879a5ce275c..ac9b3e757ba1dfbbd44d5410ab31fd335fba90e9 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -60,4 +60,5 @@ obj-$(CONFIG_XILINX_SDFEC) += xilinx_sdfec.o obj-$(CONFIG_HISI_HIKEY_USB) += hisi_hikey_usb.o obj-$(CONFIG_HI6421V600_IRQ) += hi6421v600-irq.o obj-$(CONFIG_OPEN_DICE) += open-dice.o -obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o \ No newline at end of file +obj-$(CONFIG_GP_PCI1XXXX) += mchp_pci1xxxx/ +obj-$(CONFIG_VCPU_STALL_DETECTOR) += vcpu_stall_detector.o diff --git a/drivers/misc/ad525x_dpot-i2c.c b/drivers/misc/ad525x_dpot-i2c.c index 0ee0c6d808c3b8dd5591411883664ca70db4ba76..28ffb4377d98992600c905088b6da5ffbc88b494 100644 --- a/drivers/misc/ad525x_dpot-i2c.c +++ b/drivers/misc/ad525x_dpot-i2c.c @@ -67,10 +67,9 @@ static int ad_dpot_i2c_probe(struct i2c_client *client, return ad_dpot_probe(&client->dev, &bdata, id->driver_data, id->name); } -static int ad_dpot_i2c_remove(struct i2c_client *client) +static void ad_dpot_i2c_remove(struct i2c_client *client) { ad_dpot_remove(&client->dev); - return 0; } static const struct i2c_device_id ad_dpot_id[] = { diff --git a/drivers/misc/altera-stapl/altera.c b/drivers/misc/altera-stapl/altera.c index 075f3a36d51232f2750d75f940043ca50bad2bf6..a58b7cb81d98148d4f99713719f70e8bc374bba5 100644 --- a/drivers/misc/altera-stapl/altera.c +++ b/drivers/misc/altera-stapl/altera.c @@ -1014,7 +1014,7 @@ exit_done: * ...argument 0 is string ID */ count = strlen(msg_buff); - strlcpy(&msg_buff[count], + strscpy(&msg_buff[count], &p[str_table + args[0]], ALTERA_MESSAGE_LENGTH - count); break; @@ -2146,7 +2146,7 @@ static int altera_get_note(u8 *p, s32 program_size, s32 *offset, &p[note_table + (8 * i) + 4])]; if (value != NULL) - strlcpy(value, value_ptr, vallen); + strscpy(value, value_ptr, vallen); } } @@ -2162,13 +2162,13 @@ static int altera_get_note(u8 *p, s32 program_size, s32 *offset, status = 0; if (key != NULL) - strlcpy(key, &p[note_strings + + strscpy(key, &p[note_strings + get_unaligned_be32( &p[note_table + (8 * i)])], keylen); if (value != NULL) - strlcpy(value, &p[note_strings + + strscpy(value, &p[note_strings + get_unaligned_be32( &p[note_table + (8 * i) + 4])], vallen); diff --git a/drivers/misc/apds9802als.c b/drivers/misc/apds9802als.c index 6fff44b952bd1ddaba49f0ce2893e8ae7916352d..a32431f4b370abcb8292e36deffc8655a5c5cf8e 100644 --- a/drivers/misc/apds9802als.c +++ b/drivers/misc/apds9802als.c @@ -242,7 +242,7 @@ als_error1: return res; } -static int apds9802als_remove(struct i2c_client *client) +static void apds9802als_remove(struct i2c_client *client) { struct als_data *data = i2c_get_clientdata(client); @@ -256,7 +256,6 @@ static int apds9802als_remove(struct i2c_client *client) pm_runtime_put_noidle(&client->dev); kfree(data); - return 0; } #ifdef CONFIG_PM diff --git a/drivers/misc/apds990x.c b/drivers/misc/apds990x.c index 45f5b997a0e10916d8bda7e2e4012601d87b03ca..e2100cc42ce863dc68cb50383d027b204b452679 100644 --- a/drivers/misc/apds990x.c +++ b/drivers/misc/apds990x.c @@ -1185,7 +1185,7 @@ fail1: return err; } -static int apds990x_remove(struct i2c_client *client) +static void apds990x_remove(struct i2c_client *client) { struct apds990x_chip *chip = i2c_get_clientdata(client); @@ -1205,7 +1205,6 @@ static int apds990x_remove(struct i2c_client *client) regulator_bulk_free(ARRAY_SIZE(chip->regs), chip->regs); kfree(chip); - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/misc/bcm-vk/bcm_vk_dev.c b/drivers/misc/bcm-vk/bcm_vk_dev.c index a16b99bdaa137a1d36a443bf1bbf83e598fa7c18..d4a96137728db69529151ba27ef75159e7d2eb3e 100644 --- a/drivers/misc/bcm-vk/bcm_vk_dev.c +++ b/drivers/misc/bcm-vk/bcm_vk_dev.c @@ -1339,7 +1339,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, vk); irq = pci_alloc_irq_vectors(pdev, - 1, + VK_MSIX_IRQ_MIN_REQ, VK_MSIX_IRQ_MAX, PCI_IRQ_MSI | PCI_IRQ_MSIX); @@ -1401,7 +1401,7 @@ static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) bcm_vk_tty_set_irq_enabled(vk, i); } - id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); + id = ida_alloc(&bcm_vk_ida, GFP_KERNEL); if (id < 0) { err = id; dev_err(dev, "unable to get id\n"); @@ -1500,7 +1500,7 @@ err_kfree_name: misc_device->name = NULL; err_ida_remove: - ida_simple_remove(&bcm_vk_ida, id); + ida_free(&bcm_vk_ida, id); err_irq: for (i = 0; i < vk->num_irqs; i++) @@ -1573,7 +1573,7 @@ static void bcm_vk_remove(struct pci_dev *pdev) if (misc_device->name) { misc_deregister(misc_device); kfree(misc_device->name); - ida_simple_remove(&bcm_vk_ida, vk->devid); + ida_free(&bcm_vk_ida, vk->devid); } for (i = 0; i < vk->num_irqs; i++) devm_free_irq(&pdev->dev, pci_irq_vector(pdev, i), vk); diff --git a/drivers/misc/bh1770glc.c b/drivers/misc/bh1770glc.c index 0581bb9cef2eabd68c02a87d8396784717ee5aa3..d0dfa674414c34e131095fa021055ed73eab5996 100644 --- a/drivers/misc/bh1770glc.c +++ b/drivers/misc/bh1770glc.c @@ -1280,7 +1280,7 @@ fail0: return err; } -static int bh1770_remove(struct i2c_client *client) +static void bh1770_remove(struct i2c_client *client) { struct bh1770_chip *chip = i2c_get_clientdata(client); @@ -1299,8 +1299,6 @@ static int bh1770_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); pm_runtime_set_suspended(&client->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index 60c829113299bd771580030b1d616cf272d1daf3..2c64f55cf01f8db08cc5a88a7da47bf5a32d3b53 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -280,22 +280,6 @@ void cxl_handle_fault(struct work_struct *fault_work) mmput(mm); } -static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) -{ - struct mm_struct *mm; - - mm = get_mem_context(ctx); - if (mm == NULL) { - pr_devel("cxl_prefault_one unable to get mm %i\n", - pid_nr(ctx->pid)); - return; - } - - cxl_fault_segment(ctx, mm, ea); - - mmput(mm); -} - static u64 next_segment(u64 ea, u64 vsid) { if (vsid & SLB_VSID_B_1T) @@ -306,23 +290,16 @@ static u64 next_segment(u64 ea, u64 vsid) return ea + 1; } -static void cxl_prefault_vma(struct cxl_context *ctx) +static void cxl_prefault_vma(struct cxl_context *ctx, struct mm_struct *mm) { u64 ea, last_esid = 0; struct copro_slb slb; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; int rc; - struct mm_struct *mm; - - mm = get_mem_context(ctx); - if (mm == NULL) { - pr_devel("cxl_prefault_vm unable to get mm %i\n", - pid_nr(ctx->pid)); - return; - } mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { for (ea = vma->vm_start; ea < vma->vm_end; ea = next_segment(ea, slb.vsid)) { rc = copro_calculate_slb(mm, ea, &slb); @@ -337,20 +314,28 @@ static void cxl_prefault_vma(struct cxl_context *ctx) } } mmap_read_unlock(mm); - - mmput(mm); } void cxl_prefault(struct cxl_context *ctx, u64 wed) { + struct mm_struct *mm = get_mem_context(ctx); + + if (mm == NULL) { + pr_devel("cxl_prefault unable to get mm %i\n", + pid_nr(ctx->pid)); + return; + } + switch (ctx->afu->prefault_mode) { case CXL_PREFAULT_WED: - cxl_prefault_one(ctx, wed); + cxl_fault_segment(ctx, mm, wed); break; case CXL_PREFAULT_ALL: - cxl_prefault_vma(ctx); + cxl_prefault_vma(ctx, mm); break; default: break; } + + mmput(mm); } diff --git a/drivers/misc/ds1682.c b/drivers/misc/ds1682.c index 42f316c2d71953b2dab0d909f0eb6db3ad2f86e2..0698ddc5f4d54597883c92c1765f1f4c9eb204d6 100644 --- a/drivers/misc/ds1682.c +++ b/drivers/misc/ds1682.c @@ -228,11 +228,10 @@ static int ds1682_probe(struct i2c_client *client, return rc; } -static int ds1682_remove(struct i2c_client *client) +static void ds1682_remove(struct i2c_client *client) { sysfs_remove_bin_file(&client->dev.kobj, &ds1682_eeprom_attr); sysfs_remove_group(&client->dev.kobj, &ds1682_group); - return 0; } static const struct i2c_device_id ds1682_id[] = { diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c index 633e1cf08d6e25f4b8765a9b9db8b703d1e51973..938c4f41b98c7a2eaacb88ab5fc287c99df07f56 100644 --- a/drivers/misc/eeprom/at24.c +++ b/drivers/misc/eeprom/at24.c @@ -791,7 +791,7 @@ static int at24_probe(struct i2c_client *client) return 0; } -static int at24_remove(struct i2c_client *client) +static void at24_remove(struct i2c_client *client) { struct at24_data *at24 = i2c_get_clientdata(client); @@ -801,8 +801,6 @@ static int at24_remove(struct i2c_client *client) regulator_disable(at24->vcc_reg); pm_runtime_set_suspended(&client->dev); } - - return 0; } static int __maybe_unused at24_suspend(struct device *dev) diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c index 9fbfe784d71015aeaee46adaa57c2ecfaf96e077..c8c6deb7ed8943dbcb7e97c559e75c0985088f91 100644 --- a/drivers/misc/eeprom/ee1004.c +++ b/drivers/misc/eeprom/ee1004.c @@ -219,14 +219,12 @@ static int ee1004_probe(struct i2c_client *client) return err; } -static int ee1004_remove(struct i2c_client *client) +static void ee1004_remove(struct i2c_client *client) { /* Remove page select clients if this is the last device */ mutex_lock(&ee1004_bus_lock); ee1004_cleanup(EE1004_NUM_PAGES); mutex_unlock(&ee1004_bus_lock); - - return 0; } /*-------------------------------------------------------------------------*/ diff --git a/drivers/misc/eeprom/eeprom.c b/drivers/misc/eeprom/eeprom.c index 34fa385dfd4b087c25ae9c670e61701797f57876..8a841a75d893175146ed3c996d3003cb1fc17be4 100644 --- a/drivers/misc/eeprom/eeprom.c +++ b/drivers/misc/eeprom/eeprom.c @@ -136,7 +136,7 @@ static int eeprom_detect(struct i2c_client *client, struct i2c_board_info *info) && !i2c_check_functionality(adapter, I2C_FUNC_SMBUS_READ_I2C_BLOCK)) return -ENODEV; - strlcpy(info->type, "eeprom", I2C_NAME_SIZE); + strscpy(info->type, "eeprom", I2C_NAME_SIZE); return 0; } @@ -183,11 +183,9 @@ static int eeprom_probe(struct i2c_client *client, return sysfs_create_bin_file(&client->dev.kobj, &eeprom_attr); } -static int eeprom_remove(struct i2c_client *client) +static void eeprom_remove(struct i2c_client *client) { sysfs_remove_bin_file(&client->dev.kobj, &eeprom_attr); - - return 0; } static const struct i2c_device_id eeprom_id[] = { diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c index 9aec3338e37d7c238140508312601d563b4a2109..bb3ed352b95f99c8babd6571e3dc8dc384e7ce18 100644 --- a/drivers/misc/eeprom/idt_89hpesx.c +++ b/drivers/misc/eeprom/idt_89hpesx.c @@ -1075,7 +1075,7 @@ static const struct i2c_device_id *idt_ee_match_id(struct fwnode_handle *fwnode) return NULL; p = strchr(compatible, ','); - strlcpy(devname, p ? p + 1 : compatible, sizeof(devname)); + strscpy(devname, p ? p + 1 : compatible, sizeof(devname)); /* Search through the device name */ while (id->name[0]) { if (strcmp(devname, id->name) == 0) @@ -1405,7 +1405,7 @@ err_free_pdev: /* * idt_remove() - IDT 89HPESx driver remove() callback method */ -static int idt_remove(struct i2c_client *client) +static void idt_remove(struct i2c_client *client) { struct idt_89hpesx_dev *pdev = i2c_get_clientdata(client); @@ -1417,8 +1417,6 @@ static int idt_remove(struct i2c_client *client) /* Discard driver data structure */ idt_free_pdev(pdev); - - return 0; } /* diff --git a/drivers/misc/eeprom/max6875.c b/drivers/misc/eeprom/max6875.c index 9da81f6d4a1c92e97150fc8290c1dd888d0c9ad1..6bd4f4339af4afc8a064a13f3d13d312b3dfa9b9 100644 --- a/drivers/misc/eeprom/max6875.c +++ b/drivers/misc/eeprom/max6875.c @@ -173,7 +173,7 @@ exit_kfree: return err; } -static int max6875_remove(struct i2c_client *client) +static void max6875_remove(struct i2c_client *client) { struct max6875_data *data = i2c_get_clientdata(client); @@ -181,8 +181,6 @@ static int max6875_remove(struct i2c_client *client) sysfs_remove_bin_file(&client->dev.kobj, &user_eeprom_attr); kfree(data); - - return 0; } static const struct i2c_device_id max6875_id[] = { diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 6fcfb2e9f7a7b606d02e1832e7821bc3201e9218..1ad5808655254b6e495d333929acdac4e32e9fab 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -25,7 +25,7 @@ #define SDSP_DOMAIN_ID (2) #define CDSP_DOMAIN_ID (3) #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/ -#define FASTRPC_MAX_SESSIONS 13 /*12 compute, 1 cpz*/ +#define FASTRPC_MAX_SESSIONS 14 #define FASTRPC_MAX_VMIDS 16 #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 @@ -1515,7 +1515,7 @@ static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1]; args[1].length = dsp_attr_buf_len; args[1].fd = -1; - fl->pd = 1; + fl->pd = USER_PD; return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE, FASTRPC_SCALARS(0, 1, 1), args); @@ -1943,7 +1943,12 @@ static int fastrpc_cb_probe(struct platform_device *pdev) of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions); spin_lock_irqsave(&cctx->lock, flags); - sess = &cctx->session[cctx->sesscount]; + if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) { + dev_err(&pdev->dev, "too many sessions\n"); + spin_unlock_irqrestore(&cctx->lock, flags); + return -ENOSPC; + } + sess = &cctx->session[cctx->sesscount++]; sess->used = false; sess->valid = true; sess->dev = dev; @@ -1956,13 +1961,12 @@ static int fastrpc_cb_probe(struct platform_device *pdev) struct fastrpc_session_ctx *dup_sess; for (i = 1; i < sessions; i++) { - if (cctx->sesscount++ >= FASTRPC_MAX_SESSIONS) + if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) break; - dup_sess = &cctx->session[cctx->sesscount]; + dup_sess = &cctx->session[cctx->sesscount++]; memcpy(dup_sess, sess, sizeof(*dup_sess)); } } - cctx->sesscount++; spin_unlock_irqrestore(&cctx->lock, flags); rc = dma_set_mask(dev, DMA_BIT_MASK(32)); if (rc) { diff --git a/drivers/misc/habanalabs/Kconfig b/drivers/misc/habanalabs/Kconfig index 861c81006c6d0261ea51510f2f875c890229b3a0..bd01d0d940c018ec6e769b1d67364d37e005f62c 100644 --- a/drivers/misc/habanalabs/Kconfig +++ b/drivers/misc/habanalabs/Kconfig @@ -10,6 +10,7 @@ config HABANA_AI select HWMON select DMA_SHARED_BUFFER select CRC32 + select FW_LOADER help Enables PCIe card driver for Habana's AI Processors (AIP) that are designed to accelerate Deep Learning inference and training workloads. diff --git a/drivers/misc/habanalabs/Makefile b/drivers/misc/habanalabs/Makefile index b35d7000c86b672290e6ded4c5dbc0a82d725988..a48a9e0969ed2da61bbf20be752f80e57a0d7ace 100644 --- a/drivers/misc/habanalabs/Makefile +++ b/drivers/misc/habanalabs/Makefile @@ -8,13 +8,13 @@ obj-$(CONFIG_HABANA_AI) := habanalabs.o include $(src)/common/Makefile habanalabs-y += $(HL_COMMON_FILES) -include $(src)/goya/Makefile -habanalabs-y += $(HL_GOYA_FILES) +include $(src)/gaudi2/Makefile +habanalabs-y += $(HL_GAUDI2_FILES) include $(src)/gaudi/Makefile habanalabs-y += $(HL_GAUDI_FILES) -include $(src)/gaudi2/Makefile -habanalabs-y += $(HL_GAUDI2_FILES) +include $(src)/goya/Makefile +habanalabs-y += $(HL_GOYA_FILES) habanalabs-$(CONFIG_DEBUG_FS) += common/debugfs.o diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c index b027f66f8bd4755f026752e35b54443ab181466a..2b332991ac6a75f9a809eb55375ed579e6accb99 100644 --- a/drivers/misc/habanalabs/common/command_buffer.c +++ b/drivers/misc/habanalabs/common/command_buffer.c @@ -12,20 +12,18 @@ #include #include +#define CB_VA_POOL_SIZE (4UL * SZ_1G) + static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) { struct hl_device *hdev = ctx->hdev; struct asic_fixed_properties *prop = &hdev->asic_prop; - struct hl_vm_va_block *va_block, *tmp; - dma_addr_t bus_addr; - u64 virt_addr; u32 page_size = prop->pmmu.page_size; - s32 offset; int rc; if (!hdev->supports_cb_mapping) { dev_err_ratelimited(hdev->dev, - "Cannot map CB because no VA range is allocated for CB mapping\n"); + "Mapping a CB to the device's MMU is not supported\n"); return -EINVAL; } @@ -35,106 +33,45 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb) return -EINVAL; } - INIT_LIST_HEAD(&cb->va_block_list); - - for (bus_addr = cb->bus_address; - bus_addr < cb->bus_address + cb->size; - bus_addr += page_size) { - - virt_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, page_size); - if (!virt_addr) { - dev_err(hdev->dev, - "Failed to allocate device virtual address for CB\n"); - rc = -ENOMEM; - goto err_va_pool_free; - } + if (cb->is_mmu_mapped) + return 0; - va_block = kzalloc(sizeof(*va_block), GFP_KERNEL); - if (!va_block) { - rc = -ENOMEM; - gen_pool_free(ctx->cb_va_pool, virt_addr, page_size); - goto err_va_pool_free; - } + cb->roundup_size = roundup(cb->size, page_size); - va_block->start = virt_addr; - va_block->end = virt_addr + page_size - 1; - va_block->size = page_size; - list_add_tail(&va_block->node, &cb->va_block_list); + cb->virtual_addr = (u64) gen_pool_alloc(ctx->cb_va_pool, cb->roundup_size); + if (!cb->virtual_addr) { + dev_err(hdev->dev, "Failed to allocate device virtual address for CB\n"); + return -ENOMEM; } - mutex_lock(&ctx->mmu_lock); - - bus_addr = cb->bus_address; - offset = 0; - list_for_each_entry(va_block, &cb->va_block_list, node) { - rc = hl_mmu_map_page(ctx, va_block->start, bus_addr, - va_block->size, list_is_last(&va_block->node, - &cb->va_block_list)); - if (rc) { - dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", - va_block->start); - goto err_va_umap; - } - - bus_addr += va_block->size; - offset += va_block->size; + mutex_lock(&hdev->mmu_lock); + rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size); + if (rc) { + dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr); + goto err_va_umap; } - rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV); - - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); cb->is_mmu_mapped = true; - return rc; err_va_umap: - list_for_each_entry(va_block, &cb->va_block_list, node) { - if (offset <= 0) - break; - hl_mmu_unmap_page(ctx, va_block->start, va_block->size, - offset <= va_block->size); - offset -= va_block->size; - } - - rc = hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); - - mutex_unlock(&ctx->mmu_lock); - -err_va_pool_free: - list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) { - gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size); - list_del(&va_block->node); - kfree(va_block); - } - + mutex_unlock(&hdev->mmu_lock); + gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); return rc; } static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb) { struct hl_device *hdev = ctx->hdev; - struct hl_vm_va_block *va_block, *tmp; - - mutex_lock(&ctx->mmu_lock); - - list_for_each_entry(va_block, &cb->va_block_list, node) - if (hl_mmu_unmap_page(ctx, va_block->start, va_block->size, - list_is_last(&va_block->node, - &cb->va_block_list))) - dev_warn_ratelimited(hdev->dev, - "Failed to unmap CB's va 0x%llx\n", - va_block->start); + mutex_lock(&hdev->mmu_lock); + hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size); hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); + mutex_unlock(&hdev->mmu_lock); - mutex_unlock(&ctx->mmu_lock); - - list_for_each_entry_safe(va_block, tmp, &cb->va_block_list, node) { - gen_pool_free(ctx->cb_va_pool, va_block->start, va_block->size); - list_del(&va_block->node); - kfree(va_block); - } + gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size); } static void cb_fini(struct hl_device *hdev, struct hl_cb *cb) @@ -376,7 +313,6 @@ int hl_cb_destroy(struct hl_mem_mgr *mmg, u64 cb_handle) static int hl_cb_info(struct hl_mem_mgr *mmg, u64 handle, u32 flags, u32 *usage_cnt, u64 *device_va) { - struct hl_vm_va_block *va_block; struct hl_cb *cb; int rc = 0; @@ -388,9 +324,8 @@ static int hl_cb_info(struct hl_mem_mgr *mmg, } if (flags & HL_CB_FLAGS_GET_DEVICE_VA) { - va_block = list_first_entry(&cb->va_block_list, struct hl_vm_va_block, node); - if (va_block) { - *device_va = va_block->start; + if (cb->is_mmu_mapped) { + *device_va = cb->virtual_addr; } else { dev_err(mmg->dev, "CB is not mapped to the device's MMU\n"); rc = -EINVAL; @@ -566,16 +501,23 @@ int hl_cb_va_pool_init(struct hl_ctx *ctx) return -ENOMEM; } - rc = gen_pool_add(ctx->cb_va_pool, prop->cb_va_start_addr, - prop->cb_va_end_addr - prop->cb_va_start_addr, -1); + ctx->cb_va_pool_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, + CB_VA_POOL_SIZE, HL_MMU_VA_ALIGNMENT_NOT_NEEDED); + if (!ctx->cb_va_pool_base) { + rc = -ENOMEM; + goto err_pool_destroy; + } + rc = gen_pool_add(ctx->cb_va_pool, ctx->cb_va_pool_base, CB_VA_POOL_SIZE, -1); if (rc) { dev_err(hdev->dev, "Failed to add memory to VA gen pool for CB mapping\n"); - goto err_pool_destroy; + goto err_unreserve_va_block; } return 0; +err_unreserve_va_block: + hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); err_pool_destroy: gen_pool_destroy(ctx->cb_va_pool); @@ -590,4 +532,5 @@ void hl_cb_va_pool_fini(struct hl_ctx *ctx) return; gen_pool_destroy(ctx->cb_va_pool); + hl_unreserve_va_block(hdev, ctx, ctx->cb_va_pool_base, CB_VA_POOL_SIZE); } diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 90a4574cbe2d9fab705c3ef67e98ebe2344357b3..fa05770865c65930ef913029920cb62b227f5420 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -12,7 +12,9 @@ #include #define HL_CS_FLAGS_TYPE_MASK (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \ - HL_CS_FLAGS_COLLECTIVE_WAIT) + HL_CS_FLAGS_COLLECTIVE_WAIT | HL_CS_FLAGS_RESERVE_SIGNALS_ONLY | \ + HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY | HL_CS_FLAGS_ENGINE_CORE_COMMAND) + #define MAX_TS_ITER_NUM 10 @@ -824,10 +826,10 @@ static void cs_timedout(struct work_struct *work) } /* Save only the first CS timeout parameters */ - rc = atomic_cmpxchg(&hdev->last_error.cs_timeout.write_enable, 1, 0); + rc = atomic_cmpxchg(&hdev->captured_err_info.cs_timeout.write_enable, 1, 0); if (rc) { - hdev->last_error.cs_timeout.timestamp = ktime_get(); - hdev->last_error.cs_timeout.seq = cs->sequence; + hdev->captured_err_info.cs_timeout.timestamp = ktime_get(); + hdev->captured_err_info.cs_timeout.seq = cs->sequence; event_mask = device_reset ? (HL_NOTIFIER_EVENT_CS_TIMEOUT | HL_NOTIFIER_EVENT_DEVICE_RESET) : HL_NOTIFIER_EVENT_CS_TIMEOUT; @@ -1242,6 +1244,8 @@ static enum hl_cs_type hl_cs_get_cs_type(u32 cs_type_flags) return CS_RESERVE_SIGNALS; else if (cs_type_flags & HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY) return CS_UNRESERVE_SIGNALS; + else if (cs_type_flags & HL_CS_FLAGS_ENGINE_CORE_COMMAND) + return CS_TYPE_ENGINE_CORE; else return CS_TYPE_DEFAULT; } @@ -1253,6 +1257,7 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) u32 cs_type_flags, num_chunks; enum hl_device_status status; enum hl_cs_type cs_type; + bool is_sync_stream; if (!hl_device_operational(hdev, &status)) { return -EBUSY; @@ -1276,9 +1281,10 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) cs_type = hl_cs_get_cs_type(cs_type_flags); num_chunks = args->in.num_chunks_execute; - if (unlikely((cs_type == CS_TYPE_SIGNAL || cs_type == CS_TYPE_WAIT || - cs_type == CS_TYPE_COLLECTIVE_WAIT) && - !hdev->supports_sync_stream)) { + is_sync_stream = (cs_type == CS_TYPE_SIGNAL || cs_type == CS_TYPE_WAIT || + cs_type == CS_TYPE_COLLECTIVE_WAIT); + + if (unlikely(is_sync_stream && !hdev->supports_sync_stream)) { dev_err(hdev->dev, "Sync stream CS is not supported\n"); return -EINVAL; } @@ -1288,7 +1294,7 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) dev_err(hdev->dev, "Got execute CS with 0 chunks, context %d\n", ctx->asid); return -EINVAL; } - } else if (num_chunks != 1) { + } else if (is_sync_stream && num_chunks != 1) { dev_err(hdev->dev, "Sync stream CS mandates one chunk only, context %d\n", ctx->asid); @@ -1584,13 +1590,14 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; bool need_soft_reset = false; - int rc = 0, do_ctx_switch; + int rc = 0, do_ctx_switch = 0; void __user *chunks; u32 num_chunks, tmp; u16 sob_count; int ret; - do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0); + if (hdev->supports_ctx_switch) + do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0); if (do_ctx_switch || (args->in.cs_flags & HL_CS_FLAGS_FORCE_RESTORE)) { mutex_lock(&hpriv->restore_phase_mutex); @@ -1661,9 +1668,10 @@ wait_again: } } - ctx->thread_ctx_switch_wait_token = 1; + if (hdev->supports_ctx_switch) + ctx->thread_ctx_switch_wait_token = 1; - } else if (!ctx->thread_ctx_switch_wait_token) { + } else if (hdev->supports_ctx_switch && !ctx->thread_ctx_switch_wait_token) { rc = hl_poll_timeout_memory(hdev, &ctx->thread_ctx_switch_wait_token, tmp, (tmp == 1), 100, jiffies_to_usecs(hdev->timeout_jiffies), false); @@ -2351,6 +2359,41 @@ out: return rc; } +static int cs_ioctl_engine_cores(struct hl_fpriv *hpriv, u64 engine_cores, + u32 num_engine_cores, u32 core_command) +{ + int rc; + struct hl_device *hdev = hpriv->hdev; + void __user *engine_cores_arr; + u32 *cores; + + if (!num_engine_cores || num_engine_cores > hdev->asic_prop.num_engine_cores) { + dev_err(hdev->dev, "Number of engine cores %d is invalid\n", num_engine_cores); + return -EINVAL; + } + + if (core_command != HL_ENGINE_CORE_RUN && core_command != HL_ENGINE_CORE_HALT) { + dev_err(hdev->dev, "Engine core command is invalid\n"); + return -EINVAL; + } + + engine_cores_arr = (void __user *) (uintptr_t) engine_cores; + cores = kmalloc_array(num_engine_cores, sizeof(u32), GFP_KERNEL); + if (!cores) + return -ENOMEM; + + if (copy_from_user(cores, engine_cores_arr, num_engine_cores * sizeof(u32))) { + dev_err(hdev->dev, "Failed to copy core-ids array from user\n"); + kfree(cores); + return -EFAULT; + } + + rc = hdev->asic_funcs->set_engine_cores(hdev, cores, num_engine_cores, core_command); + kfree(cores); + + return rc; +} + int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) { union hl_cs_args *args = data; @@ -2403,6 +2446,10 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) rc = cs_ioctl_unreserve_signals(hpriv, args->in.encaps_sig_handle_id); break; + case CS_TYPE_ENGINE_CORE: + rc = cs_ioctl_engine_cores(hpriv, args->in.engine_cores, + args->in.num_engine_cores, args->in.core_command); + break; default: rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq, args->in.cs_flags, @@ -2524,7 +2571,7 @@ static int hl_cs_poll_fences(struct multi_cs_data *mcs_data, struct multi_cs_com ktime_t max_ktime, first_cs_time; enum hl_cs_wait_status status; - memset(fence_ptr, 0, arr_len * sizeof(*fence_ptr)); + memset(fence_ptr, 0, arr_len * sizeof(struct hl_fence *)); /* get all fences under the same lock */ rc = hl_ctx_get_fences(mcs_data->ctx, seq_arr, fence_ptr, arr_len); @@ -2826,7 +2873,7 @@ static int hl_multi_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) } /* allocate array for the fences */ - fence_arr = kmalloc_array(seq_arr_len, sizeof(*fence_arr), GFP_KERNEL); + fence_arr = kmalloc_array(seq_arr_len, sizeof(struct hl_fence *), GFP_KERNEL); if (!fence_arr) { rc = -ENOMEM; goto free_seq_arr; diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index 64439f33a19b1a5ac94a4ecdb14288c6db8314d7..48d3ec8b5c8257bbb8ddd7085c5354f4a5e65cd0 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -291,14 +291,16 @@ static int vm_show(struct seq_file *s, void *data) if (ctx->asid != HL_KERNEL_ASID_ID && !list_empty(&ctx->hw_block_mem_list)) { seq_puts(s, "\nhw_block mappings:\n\n"); - seq_puts(s, " virtual address size HW block id\n"); - seq_puts(s, "-------------------------------------------\n"); + seq_puts(s, + " virtual address block size mapped size HW block id\n"); + seq_puts(s, + "---------------------------------------------------------------\n"); mutex_lock(&ctx->hw_block_list_lock); - list_for_each_entry(lnode, &ctx->hw_block_mem_list, - node) { + list_for_each_entry(lnode, &ctx->hw_block_mem_list, node) { seq_printf(s, - " 0x%-14lx %-6u %-9u\n", - lnode->vaddr, lnode->size, lnode->id); + " 0x%-14lx %-6u %-6u %-9u\n", + lnode->vaddr, lnode->block_size, lnode->mapped_size, + lnode->id); } mutex_unlock(&ctx->hw_block_list_lock); } @@ -591,6 +593,7 @@ static int engines_show(struct seq_file *s, void *data) struct hl_debugfs_entry *entry = s->private; struct hl_dbg_device_entry *dev_entry = entry->dev_entry; struct hl_device *hdev = dev_entry->hdev; + struct engines_data eng_data; if (hdev->reset_info.in_reset) { dev_warn_ratelimited(hdev->dev, @@ -598,7 +601,25 @@ static int engines_show(struct seq_file *s, void *data) return 0; } - hdev->asic_funcs->is_device_idle(hdev, NULL, 0, s); + eng_data.actual_size = 0; + eng_data.allocated_buf_size = HL_ENGINES_DATA_MAX_SIZE; + eng_data.buf = vmalloc(eng_data.allocated_buf_size); + if (!eng_data.buf) + return -ENOMEM; + + hdev->asic_funcs->is_device_idle(hdev, NULL, 0, &eng_data); + + if (eng_data.actual_size > eng_data.allocated_buf_size) { + dev_err(hdev->dev, + "Engines data size (%d Bytes) is bigger than allocated size (%u Bytes)\n", + eng_data.actual_size, eng_data.allocated_buf_size); + vfree(eng_data.buf); + return -ENOMEM; + } + + seq_write(s, eng_data.buf, eng_data.actual_size); + + vfree(eng_data.buf); return 0; } diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index b30aeb1c657f1149e58b68dbfaea55c793d8df75..233d8b46c831fcc3e1ef8c6b28d13a5b41868cb6 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -13,6 +13,8 @@ #include #include +#include + #define HL_RESET_DELAY_USEC 10000 /* 10ms */ enum dma_alloc_type { @@ -26,8 +28,9 @@ enum dma_alloc_type { /* * hl_set_dram_bar- sets the bar to allow later access to address * - * @hdev: pointer to habanalabs device structure + * @hdev: pointer to habanalabs device structure. * @addr: the address the caller wants to access. + * @region: the PCI region. * * @return: the old BAR base address on success, U64_MAX for failure. * The caller should set it back to the old address after use. @@ -37,58 +40,64 @@ enum dma_alloc_type { * This function can be called also if the bar doesn't need to be set, * in that case it just won't change the base. */ -static uint64_t hl_set_dram_bar(struct hl_device *hdev, u64 addr) +static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region) { struct asic_fixed_properties *prop = &hdev->asic_prop; - u64 bar_base_addr; + u64 bar_base_addr, old_base; - bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull); + if (is_power_of_2(prop->dram_pci_bar_size)) + bar_base_addr = addr & ~(prop->dram_pci_bar_size - 0x1ull); + else + bar_base_addr = DIV_ROUND_DOWN_ULL(addr, prop->dram_pci_bar_size) * + prop->dram_pci_bar_size; - return hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); -} + old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr); + /* in case of success we need to update the new BAR base */ + if (old_base != U64_MAX) + region->region_base = bar_base_addr; + + return old_base; +} static int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val, enum debugfs_access_type acc_type, enum pci_region region_type) { struct pci_mem_region *region = &hdev->pci_mem_region[region_type]; + void __iomem *acc_addr; u64 old_base = 0, rc; if (region_type == PCI_REGION_DRAM) { - old_base = hl_set_dram_bar(hdev, addr); + old_base = hl_set_dram_bar(hdev, addr, region); if (old_base == U64_MAX) return -EIO; } + acc_addr = hdev->pcie_bar[region->bar_id] + addr - region->region_base + + region->offset_in_bar; switch (acc_type) { case DEBUGFS_READ8: - *val = readb(hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + *val = readb(acc_addr); break; case DEBUGFS_WRITE8: - writeb(*val, hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + writeb(*val, acc_addr); break; case DEBUGFS_READ32: - *val = readl(hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + *val = readl(acc_addr); break; case DEBUGFS_WRITE32: - writel(*val, hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + writel(*val, acc_addr); break; case DEBUGFS_READ64: - *val = readq(hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + *val = readq(acc_addr); break; case DEBUGFS_WRITE64: - writeq(*val, hdev->pcie_bar[region->bar_id] + - addr - region->region_base + region->offset_in_bar); + writeq(*val, acc_addr); break; } if (region_type == PCI_REGION_DRAM) { - rc = hl_set_dram_bar(hdev, old_base); + rc = hl_set_dram_bar(hdev, old_base, region); if (rc == U64_MAX) return -EIO; } @@ -97,9 +106,10 @@ static int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val } static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, - gfp_t flag, enum dma_alloc_type alloc_type) + gfp_t flag, enum dma_alloc_type alloc_type, + const char *caller) { - void *ptr; + void *ptr = NULL; switch (alloc_type) { case DMA_ALLOC_COHERENT: @@ -113,11 +123,16 @@ static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t break; } + if (trace_habanalabs_dma_alloc_enabled() && !ZERO_OR_NULL_PTR(ptr)) + trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size, + caller); + return ptr; } static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, enum dma_alloc_type alloc_type) + dma_addr_t dma_handle, enum dma_alloc_type alloc_type, + const char *caller) { switch (alloc_type) { case DMA_ALLOC_COHERENT: @@ -130,39 +145,44 @@ static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *c hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle); break; } + + trace_habanalabs_dma_free(hdev->dev, (u64) (uintptr_t) cpu_addr, dma_handle, size, caller); } -void *hl_asic_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, - gfp_t flag) +void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, + gfp_t flag, const char *caller) { - return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT); + return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller); } -void hl_asic_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, - dma_addr_t dma_handle) +void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, const char *caller) { - hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT); + hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller); } -void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle) +void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size, + dma_addr_t *dma_handle, const char *caller) { - return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE); + return hl_dma_alloc_common(hdev, size, dma_handle, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); } -void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr) +void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr, + const char *caller) { - hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE); + hl_asic_dma_free_common(hdev, size, vaddr, 0, DMA_ALLOC_CPU_ACCESSIBLE, caller); } -void *hl_asic_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, - dma_addr_t *dma_handle) +void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, + dma_addr_t *dma_handle, const char *caller) { - return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL); + return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller); } -void hl_asic_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr) +void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, + const char *caller) { - hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL); + hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller); } int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir) @@ -267,6 +287,30 @@ int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type, return 0; } +void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...) +{ + va_list args; + int str_size; + + va_start(args, fmt); + /* Calculate formatted string length. Assuming each string is null terminated, hence + * increment result by 1 + */ + str_size = vsnprintf(NULL, 0, fmt, args) + 1; + va_end(args); + + if ((e->actual_size + str_size) < e->allocated_buf_size) { + va_start(args, fmt); + vsnprintf(e->buf + e->actual_size, str_size, fmt, args); + va_end(args); + } + + /* Need to update the size even when not updating destination buffer to get the exact size + * of all input strings + */ + e->actual_size += str_size; +} + enum hl_device_status hl_device_status(struct hl_device *hdev) { enum hl_device_status status; @@ -322,6 +366,8 @@ static void hpriv_release(struct kref *ref) hdev = hpriv->hdev; + hdev->asic_funcs->send_device_activity(hdev, false); + put_pid(hpriv->taskpid); hl_debugfs_remove_file(hpriv); @@ -673,7 +719,7 @@ static int device_early_init(struct hl_device *hdev) if (hdev->asic_prop.completion_queues_count) { hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count, - sizeof(*hdev->cq_wq), + sizeof(struct workqueue_struct *), GFP_KERNEL); if (!hdev->cq_wq) { rc = -ENOMEM; @@ -1091,7 +1137,9 @@ int hl_device_resume(struct hl_device *hdev) /* 'in_reset' was set to true during suspend, now we must clear it in order * for hard reset to be performed */ + spin_lock(&hdev->reset_info.lock); hdev->reset_info.in_reset = 0; + spin_unlock(&hdev->reset_info.lock); rc = hl_device_reset(hdev, HL_DRV_RESET_HARD); if (rc) { @@ -1518,6 +1566,13 @@ kill_processes: */ hdev->disabled = false; + /* F/W security enabled indication might be updated after hard-reset */ + if (hard_reset) { + rc = hl_fw_read_preboot_status(hdev); + if (rc) + goto out_err; + } + rc = hdev->asic_funcs->hw_init(hdev); if (rc) { dev_err(hdev->dev, "failed to initialize the H/W after reset\n"); @@ -1556,7 +1611,7 @@ kill_processes: if (!hdev->asic_prop.fw_security_enabled) hl_fw_set_max_power(hdev); } else { - rc = hdev->asic_funcs->non_hard_reset_late_init(hdev); + rc = hdev->asic_funcs->compute_reset_late_init(hdev); if (rc) { if (reset_upon_device_release) dev_err(hdev->dev, @@ -1704,7 +1759,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) char *name; bool add_cdev_sysfs_on_err = false; - name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2); + hdev->cdev_idx = hdev->id / 2; + + name = kasprintf(GFP_KERNEL, "hl%d", hdev->cdev_idx); if (!name) { rc = -ENOMEM; goto out_disabled; @@ -1719,7 +1776,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) if (rc) goto out_disabled; - name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2); + name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->cdev_idx); if (!name) { rc = -ENOMEM; goto free_dev; @@ -1806,7 +1863,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass) } hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs, - sizeof(*hdev->shadow_cs_queue), GFP_KERNEL); + sizeof(struct hl_cs *), GFP_KERNEL); if (!hdev->shadow_cs_queue) { rc = -ENOMEM; goto cq_fini; @@ -1997,10 +2054,10 @@ out_disabled: if (hdev->pdev) dev_err(&hdev->pdev->dev, "Failed to initialize hl%d. Device is NOT usable !\n", - hdev->id / 2); + hdev->cdev_idx); else pr_err("Failed to initialize hl%d. Device is NOT usable !\n", - hdev->id / 2); + hdev->cdev_idx); return rc; } diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 608ca67527a572e26f7cd196dc1d27e489291c99..2de6a9bd564de351e27c3db867537eeb4473275f 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -15,14 +15,6 @@ #define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */ -struct fw_binning_conf { - u64 tpc_binning; - u32 dec_binning; - u32 hbm_binning; - u32 edma_binning; - u32 mme_redundancy; -}; - static char *extract_fw_ver_from_str(const char *fw_str) { char *str, *fw_ver, *whitespace; @@ -260,7 +252,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, struct cpucp_packet *pkt; dma_addr_t pkt_dma_addr; struct hl_bd *sent_bd; - u32 tmp, expected_ack_val, pi; + u32 tmp, expected_ack_val, pi, opcode; int rc; pkt = hl_cpu_accessible_dma_pool_alloc(hdev, len, &pkt_dma_addr); @@ -327,8 +319,35 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT; if (rc) { - dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", - rc, (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT); + opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT; + + if (!prop->supports_advanced_cpucp_rc) { + dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode); + goto scrub_descriptor; + } + + switch (rc) { + case cpucp_packet_invalid: + dev_err(hdev->dev, + "CPU packet %d is not supported by F/W\n", opcode); + break; + case cpucp_packet_fault: + dev_err(hdev->dev, + "F/W failed processing CPU packet %d\n", opcode); + break; + case cpucp_packet_invalid_pkt: + dev_dbg(hdev->dev, + "CPU packet %d is not supported by F/W\n", opcode); + break; + case cpucp_packet_invalid_params: + dev_err(hdev->dev, + "F/W reports invalid parameters for CPU packet %d\n", opcode); + break; + + default: + dev_err(hdev->dev, + "Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode); + } /* propagate the return code from the f/w to the callers who want to check it */ if (result) @@ -340,6 +359,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg, *result = le64_to_cpu(pkt->result); } +scrub_descriptor: /* Scrub previous buffer descriptor 'ctl' field which contains the * previous PI value written during packet submission. * We must do this or else F/W can read an old value upon queue wraparound. @@ -462,6 +482,21 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, size); } +int hl_fw_send_device_activity(struct hl_device *hdev, bool open) +{ + struct cpucp_packet pkt; + int rc; + + memset(&pkt, 0, sizeof(pkt)); + pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET << CPUCP_PKT_CTL_OPCODE_SHIFT); + pkt.value = cpu_to_le64(open); + rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL); + if (rc) + dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open); + + return rc; +} + int hl_fw_send_heartbeat(struct hl_device *hdev) { struct cpucp_packet hb_pkt; @@ -581,6 +616,15 @@ static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val); /* All warnings should go here in order not to reach the unknown error validation */ + if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) { + dev_warn(hdev->dev, + "Device boot warning - EEPROM failure detected, default settings applied\n"); + /* This is a warning so we don't want it to disable the + * device + */ + err_val &= ~CPU_BOOT_ERR0_EEPROM_FAIL; + } + if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED) { dev_warn(hdev->dev, "Device boot warning - Skipped DRAM initialization\n"); @@ -1476,6 +1520,8 @@ static void hl_fw_preboot_update_state(struct hl_device *hdev) */ prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN); + prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN); + dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n", cpu_boot_dev_sts0); @@ -1514,7 +1560,7 @@ int hl_fw_read_preboot_status(struct hl_device *hdev) hdev->asic_funcs->init_firmware_preload_params(hdev); /* - * In order to determine boot method (static VS dymanic) we need to + * In order to determine boot method (static VS dynamic) we need to * read the boot caps register */ rc = hl_fw_read_preboot_caps(hdev); @@ -1781,7 +1827,7 @@ int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev, * * @return the CRC32 result * - * NOTE: kernel's CRC32 differ's from standard CRC32 calculation. + * NOTE: kernel's CRC32 differs from standard CRC32 calculation. * in order to be aligned we need to flip the bits of both the input * initial CRC and kernel's CRC32 result. * in addition both sides use initial CRC of 0, @@ -1798,7 +1844,7 @@ static u32 hl_fw_compat_crc32(u8 *data, size_t size) * * @hdev: pointer to the habanalabs device structure * @addr: device address of memory transfer - * @size: memory transter size + * @size: memory transfer size * @region: PCI memory region * * @return 0 on success, otherwise non-zero error code @@ -1854,50 +1900,36 @@ static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev, u64 addr; int rc; - if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) { - dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n", + if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) + dev_warn(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n", fw_desc->header.magic); - return -EIO; - } - if (fw_desc->header.version != HL_COMMS_DESC_VER) { - dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n", + if (fw_desc->header.version != HL_COMMS_DESC_VER) + dev_warn(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n", fw_desc->header.version); - return -EIO; - } /* - * calc CRC32 of data without header. + * Calc CRC32 of data without header. use the size of the descriptor + * reported by firmware, without calculating it ourself, to allow adding + * more fields to the lkd_fw_comms_desc structure. * note that no alignment/stride address issues here as all structures - * are 64 bit padded + * are 64 bit padded. */ - data_size = sizeof(struct lkd_fw_comms_desc) - - sizeof(struct comms_desc_header); data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header); - - if (le16_to_cpu(fw_desc->header.size) != data_size) { - dev_err(hdev->dev, - "Invalid descriptor size 0x%x, expected size 0x%zx\n", - le16_to_cpu(fw_desc->header.size), data_size); - return -EIO; - } + data_size = le16_to_cpu(fw_desc->header.size); data_crc32 = hl_fw_compat_crc32(data_ptr, data_size); - if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) { - dev_err(hdev->dev, - "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n", - data_crc32, fw_desc->header.crc32); + dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n", + data_crc32, fw_desc->header.crc32); return -EIO; } /* find memory region to which to copy the image */ addr = le64_to_cpu(fw_desc->img_addr); region_id = hl_get_pci_memory_region(hdev, addr); - if ((region_id != PCI_REGION_SRAM) && - ((region_id != PCI_REGION_DRAM))) { - dev_err(hdev->dev, - "Invalid region to copy FW image address=%llx\n", addr); + if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) { + dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr); return -EIO; } @@ -1914,8 +1946,7 @@ static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev, fw_loader->dynamic_loader.fw_image_size, region); if (rc) { - dev_err(hdev->dev, - "invalid mem transfer request for FW image\n"); + dev_err(hdev->dev, "invalid mem transfer request for FW image\n"); return rc; } @@ -2422,18 +2453,6 @@ static int hl_fw_dynamic_send_msg(struct hl_device *hdev, msg.reset_cause = *(__u8 *) data; break; - case HL_COMMS_BINNING_CONF_TYPE: - { - struct fw_binning_conf *binning_conf = (struct fw_binning_conf *) data; - - msg.tpc_binning_conf = cpu_to_le64(binning_conf->tpc_binning); - msg.dec_binning_conf = cpu_to_le32(binning_conf->dec_binning); - msg.hbm_binning_conf = cpu_to_le32(binning_conf->hbm_binning); - msg.edma_binning_conf = cpu_to_le32(binning_conf->edma_binning); - msg.mme_redundancy_conf = cpu_to_le32(binning_conf->mme_redundancy); - break; - } - default: dev_err(hdev->dev, "Send COMMS message - invalid message type %u\n", @@ -2503,13 +2522,6 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, */ dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs; - /* if no preboot loaded indication- wait for preboot */ - if (!(hdev->fw_loader.fw_comp_loaded & FW_TYPE_PREBOOT_CPU)) { - rc = hl_fw_wait_preboot_ready(hdev); - if (rc) - return -EIO; - } - rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE, 0, true, fw_loader->cpu_timeout); @@ -2547,7 +2559,7 @@ static int hl_fw_dynamic_init_cpu(struct hl_device *hdev, /* * when testing FW load (without Linux) on PLDM we don't want to * wait until boot fit is active as it may take several hours. - * instead, we load the bootfit and let it do all initializations in + * instead, we load the bootfit and let it do all initialization in * the background. */ if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX)) @@ -2961,3 +2973,49 @@ void hl_fw_set_max_power(struct hl_device *hdev) if (rc) dev_err(hdev->dev, "Failed to set max power, error %d\n", rc); } + +static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size, + u32 nonce, u32 timeout) +{ + struct cpucp_packet pkt = {}; + dma_addr_t req_dma_addr; + void *req_cpu_addr; + int rc; + + req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr); + if (!data) { + dev_err(hdev->dev, + "Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id); + return -ENOMEM; + } + + memset(data, 0, size); + + pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT); + pkt.addr = cpu_to_le64(req_dma_addr); + pkt.data_max_size = cpu_to_le32(size); + pkt.nonce = cpu_to_le32(nonce); + + rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), + timeout, NULL); + if (rc) { + dev_err(hdev->dev, + "Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc); + goto out; + } + + memcpy(data, req_cpu_addr, size); + +out: + hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr); + + return rc; +} + +int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, + u32 nonce) +{ + return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info, + sizeof(struct cpucp_sec_attest_info), nonce, + HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC); +} diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index d59bba9e55c9d9bfc1039b601f38c13558ca69b9..58c95b13be69a4706ab9f75144ca6be7d55fa55d 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -66,6 +66,7 @@ struct hl_fpriv; #define HL_CPUCP_INFO_TIMEOUT_USEC 10000000 /* 10s */ #define HL_CPUCP_EEPROM_TIMEOUT_USEC 10000000 /* 10s */ #define HL_CPUCP_MON_DUMP_TIMEOUT_USEC 10000000 /* 10s */ +#define HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC 10000000 /* 10s */ #define HL_FW_STATUS_POLL_INTERVAL_USEC 10000 /* 10ms */ #define HL_FW_COMMS_STATUS_PLDM_POLL_INTERVAL_USEC 1000000 /* 1s */ @@ -94,7 +95,7 @@ struct hl_fpriv; #define MMU_HASH_TABLE_BITS 7 /* 1 << 7 buckets */ /** - * enum hl_mmu_page_table_locaion - mmu page table location + * enum hl_mmu_page_table_location - mmu page table location * @MMU_DR_PGT: page-table is located on device DRAM. * @MMU_HR_PGT: page-table is located on host memory. * @MMU_NUM_PGT_LOCATIONS: number of page-table locations currently supported. @@ -143,6 +144,25 @@ enum hl_mmu_enablement { #define HL_MAX_DCORES 8 +/* DMA alloc/free wrappers */ +#define hl_asic_dma_alloc_coherent(hdev, size, dma_handle, flags) \ + hl_asic_dma_alloc_coherent_caller(hdev, size, dma_handle, flags, __func__) + +#define hl_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle) \ + hl_cpu_accessible_dma_pool_alloc_caller(hdev, size, dma_handle, __func__) + +#define hl_asic_dma_pool_zalloc(hdev, size, mem_flags, dma_handle) \ + hl_asic_dma_pool_zalloc_caller(hdev, size, mem_flags, dma_handle, __func__) + +#define hl_asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle) \ + hl_asic_dma_free_coherent_caller(hdev, size, cpu_addr, dma_handle, __func__) + +#define hl_cpu_accessible_dma_pool_free(hdev, size, vaddr) \ + hl_cpu_accessible_dma_pool_free_caller(hdev, size, vaddr, __func__) + +#define hl_asic_dma_pool_free(hdev, vaddr, dma_addr) \ + hl_asic_dma_pool_free_caller(hdev, vaddr, dma_addr, __func__) + /* * Reset Flags * @@ -208,6 +228,7 @@ enum hl_protection_levels { * struct iterate_module_ctx - HW module iterator * @fn: function to apply to each HW module instance * @data: optional internal data to the function iterator + * @rc: return code for optional use of iterator/iterator-caller */ struct iterate_module_ctx { /* @@ -217,10 +238,12 @@ struct iterate_module_ctx { * @inst: HW module instance within the block * @offset: current HW module instance offset from the 1-st HW module instance * in the 1-st block - * @data: function specific data + * @ctx: the iterator context. */ - void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset, void *data); + void (*fn)(struct hl_device *hdev, int block, int inst, u32 offset, + struct iterate_module_ctx *ctx); void *data; + int rc; }; struct hl_block_glbl_sec { @@ -342,7 +365,8 @@ enum hl_cs_type { CS_TYPE_WAIT, CS_TYPE_COLLECTIVE_WAIT, CS_RESERVE_SIGNALS, - CS_UNRESERVE_SIGNALS + CS_UNRESERVE_SIGNALS, + CS_TYPE_ENGINE_CORE }; /* @@ -544,10 +568,6 @@ struct hl_hints_range { * @tpc_binning_mask: which TPCs are binned. 0 means usable and 1 means binned. * @dram_enabled_mask: which DRAMs are enabled. * @dram_binning_mask: which DRAMs are binned. 0 means usable, 1 means binned. - * @cb_va_start_addr: virtual start address of command buffers which are mapped - * to the device's MMU. - * @cb_va_end_addr: virtual end address of command buffers which are mapped to - * the device's MMU. * @dram_hints_align_mask: dram va hint addresses alignment mask which is used * for hints validity check. * @cfg_base_address: config space base address. @@ -614,6 +634,7 @@ struct hl_hints_range { * which the property supports_user_set_page_size is true * (i.e. the DRAM supports multiple page sizes), otherwise * it will shall be equal to dram_page_size. + * @num_engine_cores: number of engine cpu cores * @collective_first_sob: first sync object available for collective use * @collective_first_mon: first monitor available for collective use * @sync_stream_first_sob: first sync object available for sync stream use @@ -658,6 +679,7 @@ struct hl_hints_range { * @set_max_power_on_device_init: true if need to set max power in F/W on device init. * @supports_user_set_page_size: true if user can set the allocation page size. * @dma_mask: the dma mask to be set for this device + * @supports_advanced_cpucp_rc: true if new cpucp opcodes are supported. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -689,8 +711,6 @@ struct asic_fixed_properties { u64 tpc_binning_mask; u64 dram_enabled_mask; u64 dram_binning_mask; - u64 cb_va_start_addr; - u64 cb_va_end_addr; u64 dram_hints_align_mask; u64 cfg_base_address; u64 mmu_cache_mng_addr; @@ -734,6 +754,7 @@ struct asic_fixed_properties { u32 faulty_dram_cluster_map; u32 xbar_edge_enabled_mask; u32 device_mem_alloc_default_page_size; + u32 num_engine_cores; u16 collective_first_sob; u16 collective_first_mon; u16 sync_stream_first_sob; @@ -766,6 +787,7 @@ struct asic_fixed_properties { u8 set_max_power_on_device_init; u8 supports_user_set_page_size; u8 dma_mask; + u8 supports_advanced_cpucp_rc; }; /** @@ -797,7 +819,7 @@ struct hl_fence { * @lock: spinlock to protect fence. * @hdev: habanalabs device structure. * @hw_sob: the H/W SOB used in this signal/wait CS. - * @encaps_sig_hdl: encaps signals hanlder. + * @encaps_sig_hdl: encaps signals handler. * @cs_seq: command submission sequence number. * @type: type of the CS - signal/wait. * @sob_val: the SOB value that is used in this signal/wait CS. @@ -898,14 +920,14 @@ struct hl_mmap_mem_buf { * @buf: back pointer to the parent mappable memory buffer * @debugfs_list: node in debugfs list of command buffers. * @pool_list: node in pool list of command buffers. - * @va_block_list: list of virtual addresses blocks of the CB if it is mapped to - * the device's MMU. * @kernel_address: Holds the CB's kernel virtual address. + * @virtual_addr: Holds the CB's virtual address. * @bus_address: Holds the CB's DMA address. * @size: holds the CB's size. + * @roundup_size: holds the cb size after roundup to page size. * @cs_cnt: holds number of CS that this CB participates in. * @is_pool: true if CB was acquired from the pool, false otherwise. - * @is_internal: internaly allocated + * @is_internal: internally allocated * @is_mmu_mapped: true if the CB is mapped to the device's MMU. */ struct hl_cb { @@ -914,10 +936,11 @@ struct hl_cb { struct hl_mmap_mem_buf *buf; struct list_head debugfs_list; struct list_head pool_list; - struct list_head va_block_list; void *kernel_address; + u64 virtual_addr; dma_addr_t bus_address; u32 size; + u32 roundup_size; atomic_t cs_cnt; u8 is_pool; u8 is_internal; @@ -1113,7 +1136,7 @@ struct timestamp_reg_info { * @fence: hl fence object for interrupt completion * @cq_target_value: CQ target value * @cq_kernel_addr: CQ kernel address, to be used in the cq interrupt - * handler for taget value comparison + * handler for target value comparison */ struct hl_user_pending_interrupt { struct timestamp_reg_info ts_reg_info; @@ -1371,6 +1394,18 @@ struct fw_load_mgr { struct hl_cs; +/** + * struct engines_data - asic engines data + * @buf: buffer for engines data in ascii + * @actual_size: actual size of data that was written by the driver to the allocated buffer + * @allocated_buf_size: total size of allocated buffer + */ +struct engines_data { + char *buf; + int actual_size; + u32 allocated_buf_size; +}; + /** * struct hl_asic_funcs - ASIC specific functions that are can be called from * common code. @@ -1434,11 +1469,9 @@ struct hl_cs; * @send_heartbeat: send is-alive packet to CPU-CP and verify response. * @debug_coresight: perform certain actions on Coresight for debugging. * @is_device_idle: return true if device is idle, false otherwise. - * @non_hard_reset_late_init: perform certain actions needed after a reset which is not hard-reset + * @compute_reset_late_init: perform certain actions needed after a compute reset * @hw_queues_lock: acquire H/W queues lock. * @hw_queues_unlock: release H/W queues lock. - * @kdma_lock: acquire H/W queues lock. Relevant from GRECO ASIC - * @kdma_unlock: release H/W queues lock. Relevant from GRECO ASIC * @get_pci_id: retrieve PCI ID. * @get_eeprom_data: retrieve EEPROM data from F/W. * @get_monitor_dump: retrieve monitor registers dump from F/W. @@ -1498,6 +1531,8 @@ struct hl_cs; * @check_if_razwi_happened: check if there was a razwi due to RR violation. * @access_dev_mem: access device memory * @set_dram_bar_base: set the base of the DRAM BAR + * @set_engine_cores: set a config command to enigne cores + * @send_device_activity: indication to FW about device availability */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1570,13 +1605,11 @@ struct hl_asic_funcs { int (*mmu_prefetch_cache_range)(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size); int (*send_heartbeat)(struct hl_device *hdev); int (*debug_coresight)(struct hl_device *hdev, struct hl_ctx *ctx, void *data); - bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, - u8 mask_len, struct seq_file *s); - int (*non_hard_reset_late_init)(struct hl_device *hdev); + bool (*is_device_idle)(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, + struct engines_data *e); + int (*compute_reset_late_init)(struct hl_device *hdev); void (*hw_queues_lock)(struct hl_device *hdev); void (*hw_queues_unlock)(struct hl_device *hdev); - void (*kdma_lock)(struct hl_device *hdev, int dcore_id); - void (*kdma_unlock)(struct hl_device *hdev, int dcore_id); u32 (*get_pci_id)(struct hl_device *hdev); int (*get_eeprom_data)(struct hl_device *hdev, void *data, size_t max_size); int (*get_monitor_dump)(struct hl_device *hdev, void *data); @@ -1634,6 +1667,9 @@ struct hl_asic_funcs { int (*access_dev_mem)(struct hl_device *hdev, enum pci_region region_type, u64 addr, u64 *val, enum debugfs_access_type acc_type); u64 (*set_dram_bar_base)(struct hl_device *hdev, u64 addr); + int (*set_engine_cores)(struct hl_device *hdev, u32 *core_ids, + u32 num_cores, u32 core_command); + int (*send_device_activity)(struct hl_device *hdev, bool open); }; @@ -1727,10 +1763,10 @@ struct hl_cs_outcome { /** * struct hl_cs_outcome_store - represents a limited store of completed CS outcomes - * @outcome_map: index of completed CS searcheable by sequence number + * @outcome_map: index of completed CS searchable by sequence number * @used_list: list of outcome objects currently in use * @free_list: list of outcome objects currently not in use - * @nodes_pool: a static pool of preallocated outcome objects + * @nodes_pool: a static pool of pre-allocated outcome objects * @db_lock: any operation on the store must take this lock */ struct hl_cs_outcome_store { @@ -1754,12 +1790,10 @@ struct hl_cs_outcome_store { * @refcount: reference counter for the context. Context is released only when * this hits 0l. It is incremented on CS and CS_WAIT. * @cs_pending: array of hl fence objects representing pending CS. - * @outcome_store: storage data structure used to remember ouitcomes of completed + * @outcome_store: storage data structure used to remember outcomes of completed * command submissions for a long time after CS id wraparound. * @va_range: holds available virtual addresses for host and dram mappings. * @mem_hash_lock: protects the mem_hash. - * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the - * MMU hash or walking the PGT requires talking this lock. * @hw_block_list_lock: protects the HW block memory list. * @debugfs_list: node in debugfs list of contexts. * @hw_block_mem_list: list of HW block virtual mapped addresses. @@ -1767,6 +1801,7 @@ struct hl_cs_outcome_store { * @cb_va_pool: device VA pool for command buffers which are mapped to the * device's MMU. * @sig_mgr: encaps signals handle manager. + * @cb_va_pool_base: the base address for the device VA pool * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed * to user so user could inquire about CS. It is used as * index to cs_pending array. @@ -1795,13 +1830,13 @@ struct hl_ctx { struct hl_cs_outcome_store outcome_store; struct hl_va_range *va_range[HL_VA_RANGE_TYPE_MAX]; struct mutex mem_hash_lock; - struct mutex mmu_lock; struct mutex hw_block_list_lock; struct list_head debugfs_list; struct list_head hw_block_mem_list; struct hl_cs_counters_atomic cs_counters; struct gen_pool *cb_va_pool; struct hl_encaps_signals_mgr sig_mgr; + u64 cb_va_pool_base; u64 cs_sequence; u64 *dram_default_hops; spinlock_t cs_lock; @@ -1823,7 +1858,6 @@ struct hl_ctx_mgr { }; - /* * COMMAND SUBMISSIONS */ @@ -1889,7 +1923,7 @@ struct hl_userptr { * @tdr_active: true if TDR was activated for this CS (to prevent * double TDR activation). * @aborted: true if CS was aborted due to some device error. - * @timestamp: true if a timestmap must be captured upon completion. + * @timestamp: true if a timestamp must be captured upon completion. * @staged_last: true if this is the last staged CS and needs completion. * @staged_first: true if this is the first staged CS and we need to receive * timeout for this CS. @@ -2047,14 +2081,16 @@ struct hl_vm_hash_node { * @node: node to hang on the list in context object. * @ctx: the context this node belongs to. * @vaddr: virtual address of the HW block. - * @size: size of the block. + * @block_size: size of the block. + * @mapped_size: size of the block which is mapped. May change if partial un-mappings are done. * @id: HW block id (handle). */ struct hl_vm_hw_block_list_node { struct list_head node; struct hl_ctx *ctx; unsigned long vaddr; - u32 size; + u32 block_size; + u32 mapped_size; u32 id; }; @@ -2214,7 +2250,7 @@ struct hl_info_list { /** * struct hl_debugfs_entry - debugfs dentry wrapper. - * @info_ent: dentry realted ops. + * @info_ent: dentry related ops. * @dev_entry: ASIC specific debugfs manager. */ struct hl_debugfs_entry { @@ -2492,7 +2528,7 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); break; \ (val) = __elbi_read; \ } else {\ - (val) = RREG32((u32)addr); \ + (val) = RREG32((u32)(addr)); \ } \ if (cond) \ break; \ @@ -2503,7 +2539,7 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val); break; \ (val) = __elbi_read; \ } else {\ - (val) = RREG32((u32)addr); \ + (val) = RREG32((u32)(addr)); \ } \ break; \ } \ @@ -2919,7 +2955,7 @@ struct razwi_info { * struct undefined_opcode_info - info about last undefined opcode error * @timestamp: timestamp of the undefined opcode error * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ - * entiers. In case all streams array entries are + * entries. In case all streams array entries are * filled with values, it means the execution was in Lower-CP. * @cq_addr: the address of the current handled command buffer * @cq_size: the size of the current handled command buffer @@ -2946,12 +2982,12 @@ struct undefined_opcode_info { }; /** - * struct last_error_session_info - info about last session errors occurred. - * @cs_timeout: CS timeout error last information. - * @razwi: razwi last information. + * struct hl_error_info - holds information collected during an error. + * @cs_timeout: CS timeout error information. + * @razwi: razwi information. * @undef_opcode: undefined opcode information */ -struct last_error_session_info { +struct hl_error_info { struct cs_timeout_info cs_timeout; struct razwi_info razwi; struct undefined_opcode_info undef_opcode; @@ -2960,7 +2996,7 @@ struct last_error_session_info { /** * struct hl_reset_info - holds current device reset information. * @lock: lock to protect critical reset flows. - * @compute_reset_cnt: number of compte resets since the driver was loaded. + * @compute_reset_cnt: number of compute resets since the driver was loaded. * @hard_reset_cnt: number of hard resets since the driver was loaded. * @hard_reset_schedule_flags: hard reset is scheduled to after current compute reset, * here we hold the hard reset flags. @@ -2971,7 +3007,7 @@ struct last_error_session_info { * @hard_reset_pending: is there a hard reset work pending. * @curr_reset_cause: saves an enumerated reset cause when a hard reset is * triggered, and cleared after it is shared with preboot. - * @prev_reset_trigger: saves the previous trigger which caused a reset, overidden + * @prev_reset_trigger: saves the previous trigger which caused a reset, overridden * with a new value on next reset * @reset_trigger_repeated: set if device reset is triggered more than once with * same cause. @@ -3041,6 +3077,12 @@ struct hl_reset_info { * @asid_mutex: protects asid_bitmap. * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue. * @debug_lock: protects critical section of setting debug mode for device + * @mmu_lock: protects the MMU page tables and invalidation h/w. Although the + * page tables are per context, the invalidation h/w is per MMU. + * Therefore, we can't allow multiple contexts (we only have two, + * user and kernel) to access the invalidation h/w at the same time. + * In addition, any change to the PGT, modifying the MMU hash or + * walking the PGT requires talking this lock. * @asic_prop: ASIC specific immutable properties. * @asic_funcs: ASIC specific functions. * @asic_specific: ASIC specific information to use only from ASIC files. @@ -3049,7 +3091,7 @@ struct hl_reset_info { * @hl_chip_info: ASIC's sensors information. * @device_status_description: device status description. * @hl_debugfs: device's debugfs manager. - * @cb_pool: list of preallocated CBs. + * @cb_pool: list of pre allocated CBs. * @cb_pool_lock: protects the CB pool. * @internal_cb_pool_virt_addr: internal command buffer pool virtual address. * @internal_cb_pool_dma_addr: internal command buffer pool dma address. @@ -3070,7 +3112,7 @@ struct hl_reset_info { * @state_dump_specs: constants and dictionaries needed to dump system state. * @multi_cs_completion: array of multi-CS completion. * @clk_throttling: holds information about current/previous clock throttling events - * @last_error: holds information about last session in which CS timeout or razwi error occurred. + * @captured_err_info: holds information about errors. * @reset_info: holds current device reset information. * @stream_master_qid_arr: pointer to array with QIDs of master streams. * @fw_major_version: major version of current loaded preboot. @@ -3111,7 +3153,8 @@ struct hl_reset_info { * @edma_binning: contains mask of edma engines that is received from the f/w which * indicates which edma engines are binned-out * @id: device minor. - * @id_control: minor of the control device + * @id_control: minor of the control device. + * @cdev_idx: char device index. Used for setting its name. * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit * addresses. * @is_in_dram_scrub: true if dram scrub operation is on going. @@ -3165,6 +3208,7 @@ struct hl_reset_info { * Used only for testing. * @heartbeat: Controls if we want to enable the heartbeat mechanism vs. the f/w, which verifies * that the f/w is always alive. Used only for testing. + * @supports_ctx_switch: true if a ctx switch is required upon first submission. */ struct hl_device { struct pci_dev *pdev; @@ -3204,6 +3248,7 @@ struct hl_device { struct mutex asid_mutex; struct mutex send_cpu_message_lock; struct mutex debug_lock; + struct mutex mmu_lock; struct asic_fixed_properties asic_prop; const struct hl_asic_funcs *asic_funcs; void *asic_specific; @@ -3242,7 +3287,7 @@ struct hl_device { struct multi_cs_completion multi_cs_completion[ MULTI_CS_MAX_USER_CTX]; struct hl_clk_throttle clk_throttling; - struct last_error_session_info last_error; + struct hl_error_info captured_err_info; struct hl_reset_info reset_info; @@ -3271,6 +3316,7 @@ struct hl_device { u32 edma_binning; u16 id; u16 id_control; + u16 cdev_idx; u16 cpu_pci_msb_addr; u8 is_in_dram_scrub; u8 disabled; @@ -3300,6 +3346,7 @@ struct hl_device { u8 compute_ctx_in_release; u8 supports_mmu_prefetch; u8 reset_upon_device_release; + u8 supports_ctx_switch; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -3426,15 +3473,18 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, } uint64_t hl_set_dram_bar_default(struct hl_device *hdev, u64 addr); -void *hl_asic_dma_alloc_coherent(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, - gfp_t flag); -void hl_asic_dma_free_coherent(struct hl_device *hdev, size_t size, void *cpu_addr, - dma_addr_t dma_handle); -void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle); -void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); -void *hl_asic_dma_pool_zalloc(struct hl_device *hdev, size_t size, gfp_t mem_flags, - dma_addr_t *dma_handle); -void hl_asic_dma_pool_free(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr); +void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle, + gfp_t flag, const char *caller); +void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, const char *caller); +void *hl_cpu_accessible_dma_pool_alloc_caller(struct hl_device *hdev, size_t size, + dma_addr_t *dma_handle, const char *caller); +void hl_cpu_accessible_dma_pool_free_caller(struct hl_device *hdev, size_t size, void *vaddr, + const char *caller); +void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags, + dma_addr_t *dma_handle, const char *caller); +void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr, + const char *caller); int hl_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); void hl_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt, enum dma_data_direction dir); @@ -3513,6 +3563,7 @@ void hl_sysfs_fini(struct hl_device *hdev); int hl_hwmon_init(struct hl_device *hdev); void hl_hwmon_fini(struct hl_device *hdev); +void hl_hwmon_release_resources(struct hl_device *hdev); int hl_cb_create(struct hl_device *hdev, struct hl_mem_mgr *mmg, struct hl_ctx *ctx, u32 cb_size, bool internal_cb, @@ -3557,7 +3608,7 @@ void hl_hw_block_mem_init(struct hl_ctx *ctx); void hl_hw_block_mem_fini(struct hl_ctx *ctx); u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, - enum hl_va_range_type type, u32 size, u32 alignment); + enum hl_va_range_type type, u64 size, u32 alignment); int hl_unreserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, u64 start_addr, u64 size); int hl_pin_host_memory(struct hl_device *hdev, u64 addr, u64 size, @@ -3674,6 +3725,7 @@ int hl_fw_dram_replaced_row_get(struct hl_device *hdev, struct cpucp_hbm_row_info *info); int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num); int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid); +int hl_fw_send_device_activity(struct hl_device *hdev, bool open); int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3], bool is_wc[3]); int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data); @@ -3697,6 +3749,8 @@ int hl_get_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long *va void hl_set_pwm_info(struct hl_device *hdev, int sensor_index, u32 attr, long value); long hl_fw_get_max_power(struct hl_device *hdev); void hl_fw_set_max_power(struct hl_device *hdev); +int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, + u32 nonce); int hl_set_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long value); int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); int hl_set_power(struct hl_device *hdev, int sensor_index, u32 attr, long value); @@ -3743,6 +3797,7 @@ struct hl_mmap_mem_buf * hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg, struct hl_mmap_mem_buf_behavior *behavior, gfp_t gfp, void *args); +__printf(2, 3) void hl_engine_data_sprintf(struct engines_data *e, const char *fmt, ...); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index f733ead605e7978a3543500268eea5dccf247eee..112632afe7d53806be61dd058fa0aba86647e7ba 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -14,6 +14,9 @@ #include #include +#define CREATE_TRACE_POINTS +#include + #define HL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team" #define HL_DRIVER_DESC "Driver for HabanaLabs's AI Accelerators" @@ -27,7 +30,10 @@ static struct class *hl_class; static DEFINE_IDR(hl_devs_idr); static DEFINE_MUTEX(hl_devs_idr_lock); -static int timeout_locked = 30; +#define HL_DEFAULT_TIMEOUT_LOCKED 30 /* 30 seconds */ +#define GAUDI_DEFAULT_TIMEOUT_LOCKED 600 /* 10 minutes */ + +static int timeout_locked = HL_DEFAULT_TIMEOUT_LOCKED; static int reset_on_lockup = 1; static int memory_scrub; static ulong boot_error_status_mask = ULONG_MAX; @@ -55,14 +61,12 @@ MODULE_PARM_DESC(boot_error_status_mask, #define PCI_IDS_GAUDI_SEC 0x1010 #define PCI_IDS_GAUDI2 0x1020 -#define PCI_IDS_GAUDI2_SEC 0x1030 static const struct pci_device_id ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), }, { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), }, { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), }, { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI2), }, - { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI2_SEC), }, { 0, } }; MODULE_DEVICE_TABLE(pci, ids); @@ -92,9 +96,6 @@ static enum hl_asic_type get_asic_type(u16 device) case PCI_IDS_GAUDI2: asic_type = ASIC_GAUDI2; break; - case PCI_IDS_GAUDI2_SEC: - asic_type = ASIC_GAUDI2_SEC; - break; default: asic_type = ASIC_INVALID; break; @@ -107,7 +108,6 @@ static bool is_asic_secured(enum hl_asic_type asic_type) { switch (asic_type) { case ASIC_GAUDI_SEC: - case ASIC_GAUDI2_SEC: return true; default: return false; @@ -161,7 +161,7 @@ int hl_device_open(struct inode *inode, struct file *filp) mutex_lock(&hdev->fpriv_list_lock); if (!hl_device_operational(hdev, &status)) { - dev_err_ratelimited(hdev->dev, + dev_dbg_ratelimited(hdev->dev, "Can't open %s because it is %s\n", dev_name(hdev->dev), hdev->status[status]); @@ -207,11 +207,13 @@ int hl_device_open(struct inode *inode, struct file *filp) list_add(&hpriv->dev_node, &hdev->fpriv_list); mutex_unlock(&hdev->fpriv_list_lock); + hdev->asic_funcs->send_device_activity(hdev, true); + hl_debugfs_add_file(hpriv); - atomic_set(&hdev->last_error.cs_timeout.write_enable, 1); - atomic_set(&hdev->last_error.razwi.write_enable, 1); - hdev->last_error.undef_opcode.write_enable = true; + atomic_set(&hdev->captured_err_info.cs_timeout.write_enable, 1); + atomic_set(&hdev->captured_err_info.razwi.write_enable, 1); + hdev->captured_err_info.undef_opcode.write_enable = true; hdev->open_counter++; hdev->last_successful_open_jif = jiffies; @@ -269,7 +271,7 @@ int hl_device_open_ctrl(struct inode *inode, struct file *filp) mutex_lock(&hdev->fpriv_ctrl_list_lock); if (!hl_device_operational(hdev, NULL)) { - dev_err_ratelimited(hdev->dev_ctrl, + dev_dbg_ratelimited(hdev->dev_ctrl, "Can't open %s because it is disabled or in reset\n", dev_name(hdev->dev_ctrl)); rc = -EPERM; @@ -314,12 +316,22 @@ static void copy_kernel_module_params_to_device(struct hl_device *hdev) hdev->boot_error_status_mask = boot_error_status_mask; } -static void fixup_device_params_per_asic(struct hl_device *hdev) +static void fixup_device_params_per_asic(struct hl_device *hdev, int timeout) { switch (hdev->asic_type) { - case ASIC_GOYA: case ASIC_GAUDI: case ASIC_GAUDI_SEC: + /* If user didn't request a different timeout than the default one, we have + * a different default timeout for Gaudi + */ + if (timeout == HL_DEFAULT_TIMEOUT_LOCKED) + hdev->timeout_jiffies = msecs_to_jiffies(GAUDI_DEFAULT_TIMEOUT_LOCKED * + MSEC_PER_SEC); + + hdev->reset_upon_device_release = 0; + break; + + case ASIC_GOYA: hdev->reset_upon_device_release = 0; break; @@ -339,7 +351,7 @@ static int fixup_device_params(struct hl_device *hdev) hdev->fw_comms_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC; if (tmp_timeout) - hdev->timeout_jiffies = msecs_to_jiffies(tmp_timeout * 1000); + hdev->timeout_jiffies = msecs_to_jiffies(tmp_timeout * MSEC_PER_SEC); else hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT; @@ -360,7 +372,7 @@ static int fixup_device_params(struct hl_device *hdev) if (!hdev->cpu_queues_enable) hdev->heartbeat = 0; - fixup_device_params_per_asic(hdev); + fixup_device_params_per_asic(hdev, tmp_timeout); return 0; } diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 6a30bd98ab5e771d171dba642b69b3d6130bec72..43afe40966e50a3c7700c451b8fe122997e1d192 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -14,6 +14,7 @@ #include #include #include +#include static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = { [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr), @@ -103,6 +104,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) hw_ip.edma_enabled_mask = prop->edma_enabled_mask; hw_ip.server_type = prop->server_type; + hw_ip.security_enabled = prop->fw_security_enabled; return copy_to_user(out, &hw_ip, min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0; @@ -591,8 +593,8 @@ static int cs_timeout_info(struct hl_fpriv *hpriv, struct hl_info_args *args) if ((!max_size) || (!out)) return -EINVAL; - info.seq = hdev->last_error.cs_timeout.seq; - info.timestamp = ktime_to_ns(hdev->last_error.cs_timeout.timestamp); + info.seq = hdev->captured_err_info.cs_timeout.seq; + info.timestamp = ktime_to_ns(hdev->captured_err_info.cs_timeout.timestamp); return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; } @@ -607,12 +609,12 @@ static int razwi_info(struct hl_fpriv *hpriv, struct hl_info_args *args) if ((!max_size) || (!out)) return -EINVAL; - info.timestamp = ktime_to_ns(hdev->last_error.razwi.timestamp); - info.addr = hdev->last_error.razwi.addr; - info.engine_id_1 = hdev->last_error.razwi.engine_id_1; - info.engine_id_2 = hdev->last_error.razwi.engine_id_2; - info.no_engine_id = hdev->last_error.razwi.non_engine_initiator; - info.error_type = hdev->last_error.razwi.type; + info.timestamp = ktime_to_ns(hdev->captured_err_info.razwi.timestamp); + info.addr = hdev->captured_err_info.razwi.addr; + info.engine_id_1 = hdev->captured_err_info.razwi.engine_id_1; + info.engine_id_2 = hdev->captured_err_info.razwi.engine_id_2; + info.no_engine_id = hdev->captured_err_info.razwi.non_engine_initiator; + info.error_type = hdev->captured_err_info.razwi.type; return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; } @@ -627,13 +629,13 @@ static int undefined_opcode_info(struct hl_fpriv *hpriv, struct hl_info_args *ar if ((!max_size) || (!out)) return -EINVAL; - info.timestamp = ktime_to_ns(hdev->last_error.undef_opcode.timestamp); - info.engine_id = hdev->last_error.undef_opcode.engine_id; - info.cq_addr = hdev->last_error.undef_opcode.cq_addr; - info.cq_size = hdev->last_error.undef_opcode.cq_size; - info.stream_id = hdev->last_error.undef_opcode.stream_id; - info.cb_addr_streams_len = hdev->last_error.undef_opcode.cb_addr_streams_len; - memcpy(info.cb_addr_streams, hdev->last_error.undef_opcode.cb_addr_streams, + info.timestamp = ktime_to_ns(hdev->captured_err_info.undef_opcode.timestamp); + info.engine_id = hdev->captured_err_info.undef_opcode.engine_id; + info.cq_addr = hdev->captured_err_info.undef_opcode.cq_addr; + info.cq_size = hdev->captured_err_info.undef_opcode.cq_size; + info.stream_id = hdev->captured_err_info.undef_opcode.stream_id; + info.cb_addr_streams_len = hdev->captured_err_info.undef_opcode.cb_addr_streams_len; + memcpy(info.cb_addr_streams, hdev->captured_err_info.undef_opcode.cb_addr_streams, sizeof(info.cb_addr_streams)); return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; @@ -660,6 +662,55 @@ static int dev_mem_alloc_page_sizes_info(struct hl_fpriv *hpriv, struct hl_info_ return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; } +static int sec_attest_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + struct cpucp_sec_attest_info *sec_attest_info; + struct hl_info_sec_attest *info; + u32 max_size = args->return_size; + int rc; + + if ((!max_size) || (!out)) + return -EINVAL; + + sec_attest_info = kmalloc(sizeof(*sec_attest_info), GFP_KERNEL); + if (!sec_attest_info) + return -ENOMEM; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + rc = -ENOMEM; + goto free_sec_attest_info; + } + + rc = hl_fw_get_sec_attest_info(hpriv->hdev, sec_attest_info, args->sec_attest_nonce); + if (rc) + goto free_info; + + info->nonce = le32_to_cpu(sec_attest_info->nonce); + info->pcr_quote_len = le16_to_cpu(sec_attest_info->pcr_quote_len); + info->pub_data_len = le16_to_cpu(sec_attest_info->pub_data_len); + info->certificate_len = le16_to_cpu(sec_attest_info->certificate_len); + info->pcr_num_reg = sec_attest_info->pcr_num_reg; + info->pcr_reg_len = sec_attest_info->pcr_reg_len; + info->quote_sig_len = sec_attest_info->quote_sig_len; + memcpy(&info->pcr_data, &sec_attest_info->pcr_data, sizeof(info->pcr_data)); + memcpy(&info->pcr_quote, &sec_attest_info->pcr_quote, sizeof(info->pcr_quote)); + memcpy(&info->public_data, &sec_attest_info->public_data, sizeof(info->public_data)); + memcpy(&info->certificate, &sec_attest_info->certificate, sizeof(info->certificate)); + memcpy(&info->quote_sig, &sec_attest_info->quote_sig, sizeof(info->quote_sig)); + + rc = copy_to_user(out, info, + min_t(size_t, max_size, sizeof(*info))) ? -EFAULT : 0; + +free_info: + kfree(info); +free_sec_attest_info: + kfree(sec_attest_info); + + return rc; +} + static int eventfd_register(struct hl_fpriv *hpriv, struct hl_info_args *args) { int rc; @@ -697,6 +748,42 @@ static int eventfd_unregister(struct hl_fpriv *hpriv, struct hl_info_args *args) return 0; } +static int engine_status_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + u32 status_buf_size = args->return_size; + struct hl_device *hdev = hpriv->hdev; + struct engines_data eng_data; + int rc; + + if ((status_buf_size < SZ_1K) || (status_buf_size > HL_ENGINES_DATA_MAX_SIZE) || (!out)) + return -EINVAL; + + eng_data.actual_size = 0; + eng_data.allocated_buf_size = status_buf_size; + eng_data.buf = vmalloc(status_buf_size); + if (!eng_data.buf) + return -ENOMEM; + + hdev->asic_funcs->is_device_idle(hdev, NULL, 0, &eng_data); + + if (eng_data.actual_size > eng_data.allocated_buf_size) { + dev_err(hdev->dev, + "Engines data size (%d Bytes) is bigger than allocated size (%u Bytes)\n", + eng_data.actual_size, status_buf_size); + vfree(eng_data.buf); + return -ENOMEM; + } + + args->user_buffer_actual_size = eng_data.actual_size; + rc = copy_to_user(out, eng_data.buf, min_t(size_t, status_buf_size, eng_data.actual_size)) ? + -EFAULT : 0; + + vfree(eng_data.buf); + + return rc; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -806,12 +893,18 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_DRAM_PENDING_ROWS: return dram_pending_rows_info(hpriv, args); + case HL_INFO_SECURED_ATTESTATION: + return sec_attest_info(hpriv, args); + case HL_INFO_REGISTER_EVENTFD: return eventfd_register(hpriv, args); case HL_INFO_UNREGISTER_EVENTFD: return eventfd_unregister(hpriv, args); + case HL_INFO_ENGINE_STATUS: + return engine_status_info(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -EINVAL; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index 3f15ab9d827ff113a7efc18abae62ed75f4ddba5..d0087c0ec48c9f5758a2eb8614ae31efb27ce9fa 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -826,9 +826,7 @@ static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q, q->kernel_address = p; - q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, - sizeof(*q->shadow_queue), - GFP_KERNEL); + q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, sizeof(struct hl_cs_job *), GFP_KERNEL); if (!q->shadow_queue) { dev_err(hdev->dev, "Failed to allocate shadow queue for H/W queue %d\n", diff --git a/drivers/misc/habanalabs/common/hwmon.c b/drivers/misc/habanalabs/common/hwmon.c index 57f5d2c483305e30de2392cd4d68226df5d947c8..55eb0203817f1761aab75d595a83e6078059105b 100644 --- a/drivers/misc/habanalabs/common/hwmon.c +++ b/drivers/misc/habanalabs/common/hwmon.c @@ -194,7 +194,8 @@ int hl_build_hwmon_channel_info(struct hl_device *hdev, struct cpucp_sensor *sen curr_arr[sensors_by_type_next_index[type]++] = flags; } - channels_info = kcalloc(num_active_sensor_types + 1, sizeof(*channels_info), GFP_KERNEL); + channels_info = kcalloc(num_active_sensor_types + 1, sizeof(struct hwmon_channel_info *), + GFP_KERNEL); if (!channels_info) { rc = -ENOMEM; goto channels_info_array_err; @@ -910,3 +911,24 @@ void hl_hwmon_fini(struct hl_device *hdev) hwmon_device_unregister(hdev->hwmon_dev); } + +void hl_hwmon_release_resources(struct hl_device *hdev) +{ + const struct hwmon_channel_info **channel_info_arr; + int i = 0; + + if (!hdev->hl_chip_info->info) + return; + + channel_info_arr = hdev->hl_chip_info->info; + + while (channel_info_arr[i]) { + kfree(channel_info_arr[i]->config); + kfree(channel_info_arr[i]); + i++; + } + + kfree(channel_info_arr); + + hdev->hl_chip_info->info = NULL; +} diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 61bc1bfe984a2535714e44f84c02e9d5755d7586..ef28f3b37b93216bf9a7b50f296f21fcdff62261 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -457,7 +457,7 @@ static void merge_va_blocks_locked(struct hl_device *hdev, prev = list_prev_entry(va_block, node); if (&prev->node != va_list && prev->end + 1 == va_block->start) { prev->end = va_block->end; - prev->size = prev->end - prev->start; + prev->size = prev->end - prev->start + 1; list_del(&va_block->node); kfree(va_block); va_block = prev; @@ -466,7 +466,7 @@ static void merge_va_blocks_locked(struct hl_device *hdev, next = list_next_entry(va_block, node); if (&next->node != va_list && va_block->end + 1 == next->start) { next->start = va_block->start; - next->size = next->end - next->start; + next->size = next->end - next->start + 1; list_del(&va_block->node); kfree(va_block); } @@ -755,7 +755,7 @@ out: * - Return the start address of the virtual block. */ u64 hl_reserve_va_block(struct hl_device *hdev, struct hl_ctx *ctx, - enum hl_va_range_type type, u32 size, u32 alignment) + enum hl_va_range_type type, u64 size, u32 alignment) { return get_va_block(hdev, ctx->va_range[type], size, 0, max(alignment, ctx->va_range[type]->page_size), @@ -1210,18 +1210,18 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device goto va_block_err; } - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack); if (rc) { dev_err(hdev->dev, "mapping page pack failed for handle %u\n", handle); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); goto map_err; } rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV, ctx->asid, ret_vaddr, phys_pg_pack->total_size); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); if (rc) goto map_err; @@ -1362,7 +1362,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, else vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack); @@ -1375,7 +1375,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, rc = hl_mmu_invalidate_cache_range(hdev, true, *vm_type, ctx->asid, vaddr, phys_pg_pack->total_size); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); /* * If the context is closing we don't need to check for the MMU cache @@ -1418,18 +1418,23 @@ vm_type_err: return rc; } -static int map_block(struct hl_device *hdev, u64 address, u64 *handle, - u32 *size) +static int map_block(struct hl_device *hdev, u64 address, u64 *handle, u32 *size) { - u32 block_id = 0; + u32 block_id; int rc; + *handle = 0; + if (size) + *size = 0; + rc = hdev->asic_funcs->get_hw_block_id(hdev, address, size, &block_id); + if (rc) + return rc; *handle = block_id | HL_MMAP_TYPE_BLOCK; *handle <<= PAGE_SHIFT; - return rc; + return 0; } static void hw_block_vm_close(struct vm_area_struct *vma) @@ -1437,6 +1442,13 @@ static void hw_block_vm_close(struct vm_area_struct *vma) struct hl_vm_hw_block_list_node *lnode = (struct hl_vm_hw_block_list_node *) vma->vm_private_data; struct hl_ctx *ctx = lnode->ctx; + long new_mmap_size; + + new_mmap_size = lnode->mapped_size - (vma->vm_end - vma->vm_start); + if (new_mmap_size > 0) { + lnode->mapped_size = new_mmap_size; + return; + } mutex_lock(&ctx->hw_block_list_lock); list_del(&lnode->node); @@ -1487,23 +1499,23 @@ int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) if (!lnode) return -ENOMEM; - vma->vm_ops = &hw_block_vm_ops; - vma->vm_private_data = lnode; - - hl_ctx_get(ctx); - rc = hdev->asic_funcs->hw_block_mmap(hdev, vma, block_id, block_size); if (rc) { - hl_ctx_put(ctx); kfree(lnode); return rc; } + hl_ctx_get(ctx); + lnode->ctx = ctx; lnode->vaddr = vma->vm_start; - lnode->size = block_size; + lnode->block_size = block_size; + lnode->mapped_size = lnode->block_size; lnode->id = block_id; + vma->vm_private_data = lnode; + vma->vm_ops = &hw_block_vm_ops; + mutex_lock(&ctx->hw_block_list_lock); list_add_tail(&lnode->node, &ctx->hw_block_mem_list); mutex_unlock(&ctx->hw_block_list_lock); @@ -2296,8 +2308,7 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size, return -EFAULT; } - userptr->pages = kvmalloc_array(npages, sizeof(*userptr->pages), - GFP_KERNEL); + userptr->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL); if (!userptr->pages) return -ENOMEM; @@ -2759,13 +2770,13 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx) unmap_device_va(ctx, &args, true); } - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); /* invalidate the cache once after the unmapping loop */ hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); hl_mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); INIT_LIST_HEAD(&free_list); diff --git a/drivers/misc/habanalabs/common/memory_mgr.c b/drivers/misc/habanalabs/common/memory_mgr.c index 56df962d2f3c6bc0573147b7370abda49e365b14..1936d653699ed0e0df3d725b45ffc5de335b083d 100644 --- a/drivers/misc/habanalabs/common/memory_mgr.c +++ b/drivers/misc/habanalabs/common/memory_mgr.c @@ -11,7 +11,7 @@ * hl_mmap_mem_buf_get - increase the buffer refcount and return a pointer to * the buffer descriptor. * - * @mmg: parent unifed memory manager + * @mmg: parent unified memory manager * @handle: requested buffer handle * * Find the buffer in the store and return a pointer to its descriptor. @@ -104,7 +104,7 @@ int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf) * hl_mmap_mem_buf_put_handle - decrease the reference to the buffer with the * given handle. * - * @mmg: parent unifed memory manager + * @mmg: parent unified memory manager * @handle: requested buffer handle * * Decrease the reference to the buffer, and release it if it was the last one. @@ -137,7 +137,7 @@ int hl_mmap_mem_buf_put_handle(struct hl_mem_mgr *mmg, u64 handle) /** * hl_mmap_mem_buf_alloc - allocate a new mappable buffer * - * @mmg: parent unifed memory manager + * @mmg: parent unified memory manager * @behavior: behavior object describing this buffer polymorphic behavior * @gfp: gfp flags to use for the memory allocations * @args: additional args passed to behavior->alloc @@ -222,7 +222,7 @@ static const struct vm_operations_struct hl_mmap_mem_buf_vm_ops = { /** * hl_mem_mgr_mmap - map the given buffer to the user * - * @mmg: unifed memory manager + * @mmg: unified memory manager * @vma: the vma object for which mmap was closed. * @args: additional args passed to behavior->mmap * @@ -322,7 +322,7 @@ void hl_mem_mgr_init(struct device *dev, struct hl_mem_mgr *mmg) /** * hl_mem_mgr_fini - release unified memory manager * - * @mmg: parent unifed memory manager + * @mmg: parent unified memory manager * * Release the unified memory manager. Shall be called from an interrupt context. */ diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c index 60740de47b34ef66e215664fd4725436a03861e3..cf8946266615c16973952fa90e26dcfa9e7f30b1 100644 --- a/drivers/misc/habanalabs/common/mmu/mmu.c +++ b/drivers/misc/habanalabs/common/mmu/mmu.c @@ -9,6 +9,8 @@ #include "../habanalabs.h" +#include + /** * hl_mmu_get_funcs() - get MMU functions structure * @hdev: habanalabs device structure. @@ -45,6 +47,8 @@ int hl_mmu_init(struct hl_device *hdev) if (!hdev->mmu_enable) return 0; + mutex_init(&hdev->mmu_lock); + if (hdev->mmu_func[MMU_DR_PGT].init != NULL) { rc = hdev->mmu_func[MMU_DR_PGT].init(hdev); if (rc) @@ -86,6 +90,8 @@ void hl_mmu_fini(struct hl_device *hdev) if (hdev->mmu_func[MMU_HR_PGT].fini != NULL) hdev->mmu_func[MMU_HR_PGT].fini(hdev); + + mutex_destroy(&hdev->mmu_lock); } /** @@ -104,8 +110,6 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx) if (!hdev->mmu_enable) return 0; - mutex_init(&ctx->mmu_lock); - if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) { rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx); if (rc) @@ -149,8 +153,6 @@ void hl_mmu_ctx_fini(struct hl_ctx *ctx) if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL) hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx); - - mutex_destroy(&ctx->mmu_lock); } /* @@ -259,6 +261,9 @@ int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flu if (flush_pte) mmu_funcs->flush(ctx); + if (trace_habanalabs_mmu_unmap_enabled() && !rc) + trace_habanalabs_mmu_unmap(hdev->dev, virt_addr, 0, page_size, flush_pte); + return rc; } @@ -344,6 +349,8 @@ int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_s if (flush_pte) mmu_funcs->flush(ctx); + trace_habanalabs_mmu_map(hdev->dev, virt_addr, phys_addr, page_size, flush_pte); + return 0; err: @@ -403,6 +410,8 @@ int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, dev_err(hdev->dev, "Map failed for va 0x%llx to pa 0x%llx\n", curr_va, curr_pa); + /* last mapping failed so don't try to unmap it - reduce off by page_size */ + off -= page_size; goto unmap; } } @@ -600,9 +609,9 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr); - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); if (rc) return rc; @@ -692,16 +701,16 @@ static void hl_mmu_prefetch_work_function(struct work_struct *work) { struct hl_prefetch_work *pfw = container_of(work, struct hl_prefetch_work, pf_work); struct hl_ctx *ctx = pfw->ctx; + struct hl_device *hdev = ctx->hdev; - if (!hl_device_operational(ctx->hdev, NULL)) + if (!hl_device_operational(hdev, NULL)) goto put_ctx; - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); - ctx->hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, - pfw->va, pfw->size); + hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); put_ctx: /* diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c index 6c5271f01160c78062c0acb9ec817dc8be8af6e9..36e9814139d172c8faa35c099445d8c454d70c4b 100644 --- a/drivers/misc/habanalabs/common/sysfs.c +++ b/drivers/misc/habanalabs/common/sysfs.c @@ -375,6 +375,14 @@ out: return max_size; } +static ssize_t security_enabled_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hl_device *hdev = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", hdev->asic_prop.fw_security_enabled); +} + static DEVICE_ATTR_RO(armcp_kernel_ver); static DEVICE_ATTR_RO(armcp_ver); static DEVICE_ATTR_RO(cpld_ver); @@ -393,6 +401,7 @@ static DEVICE_ATTR_RO(status); static DEVICE_ATTR_RO(thermal_ver); static DEVICE_ATTR_RO(uboot_ver); static DEVICE_ATTR_RO(fw_os_ver); +static DEVICE_ATTR_RO(security_enabled); static struct bin_attribute bin_attr_eeprom = { .attr = {.name = "eeprom", .mode = (0444)}, @@ -417,6 +426,7 @@ static struct attribute *hl_dev_attrs[] = { &dev_attr_thermal_ver.attr, &dev_attr_uboot_ver.attr, &dev_attr_fw_os_ver.attr, + &dev_attr_security_enabled.attr, NULL, }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index cb2988e2c7a89e5a5638a27d7b7ad7d8facbf9b5..92560414e84333ded0f7564eb8e1c2f137ed5f42 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -899,12 +899,13 @@ static int gaudi_early_fini(struct hl_device *hdev) */ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) { - struct asic_fixed_properties *prop = &hdev->asic_prop; u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel; + struct asic_fixed_properties *prop = &hdev->asic_prop; u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq; int rc; - if (hdev->asic_prop.fw_security_enabled) { + if ((hdev->fw_components & FW_TYPE_LINUX) && + (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PLL_INFO_EN)) { struct gaudi_device *gaudi = hdev->asic_specific; if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) @@ -939,9 +940,7 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) else freq = pll_clk / (div_fctr + 1); } else { - dev_warn(hdev->dev, - "Received invalid div select value: %d", - div_sel); + dev_warn(hdev->dev, "Received invalid div select value: %#x", div_sel); freq = 0; } } @@ -985,9 +984,10 @@ static int _gaudi_init_tpc_mem(struct hl_device *hdev, init_tpc_mem_pkt->ctl = cpu_to_le32(ctl); init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr); - dst_addr = (prop->sram_user_base_address & - GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >> - GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT; + + /* TPC_CMD is configured with I$ prefetch enabled, so address should be aligned to 8KB */ + dst_addr = FIELD_PREP(GAUDI_PKT_LIN_DMA_DST_ADDR_MASK, + round_up(prop->sram_user_base_address, SZ_8K)); init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr); job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true); @@ -1683,23 +1683,7 @@ disable_pci_access: static void gaudi_late_fini(struct hl_device *hdev) { - const struct hwmon_channel_info **channel_info_arr; - int i = 0; - - if (!hdev->hl_chip_info->info) - return; - - channel_info_arr = hdev->hl_chip_info->info; - - while (channel_info_arr[i]) { - kfree(channel_info_arr[i]->config); - kfree(channel_info_arr[i]); - i++; - } - - kfree(channel_info_arr); - - hdev->hl_chip_info->info = NULL; + hl_hwmon_release_resources(hdev); } static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev) @@ -4723,7 +4707,7 @@ static int gaudi_scrub_device_mem(struct hl_device *hdev) addr = prop->sram_user_base_address; size = hdev->pldm ? 0x10000 : prop->sram_size - SRAM_USER_BASE_OFFSET; - dev_dbg(hdev->dev, "Scrubing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n", + dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n", addr, addr + size, val); rc = gaudi_memset_device_memory(hdev, addr, size, val); if (rc) { @@ -6911,9 +6895,9 @@ static void gaudi_handle_sw_config_stream_data(struct hl_device *hdev, u32 strea stream, cq_ptr, size); if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) { - hdev->last_error.undef_opcode.cq_addr = cq_ptr; - hdev->last_error.undef_opcode.cq_size = size; - hdev->last_error.undef_opcode.stream_id = stream; + hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr; + hdev->captured_err_info.undef_opcode.cq_size = size; + hdev->captured_err_info.undef_opcode.stream_id = stream; } } @@ -6979,7 +6963,7 @@ static void gaudi_handle_last_pqes_on_err(struct hl_device *hdev, u32 qid_base, } if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) { - struct undefined_opcode_info *undef_opcode = &hdev->last_error.undef_opcode; + struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; u32 arr_idx = undef_opcode->cb_addr_streams_len; if (arr_idx == 0) { @@ -7063,11 +7047,11 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev, } /* check for undefined opcode */ if (glbl_sts_val & TPC0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK && - hdev->last_error.undef_opcode.write_enable) { - memset(&hdev->last_error.undef_opcode, 0, - sizeof(hdev->last_error.undef_opcode)); + hdev->captured_err_info.undef_opcode.write_enable) { + memset(&hdev->captured_err_info.undef_opcode, 0, + sizeof(hdev->captured_err_info.undef_opcode)); - hdev->last_error.undef_opcode.write_enable = false; + hdev->captured_err_info.undef_opcode.write_enable = false; *event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE; } @@ -7233,12 +7217,6 @@ static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *e switch (event_type) { case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: - /* In TPC QM event, notify on TPC assertion. While there isn't - * a specific event for assertion yet, the FW generates QM event. - * The SW upper layer will inspect an internal mapped area to indicate - * if the event is a tpc assertion or tpc QM. - */ - *event_mask |= HL_NOTIFIER_EVENT_TPC_ASSERT; index = event_type - GAUDI_EVENT_TPC0_QM; qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS; qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET; @@ -7349,18 +7327,19 @@ static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type, gaudi_print_and_get_mmu_error_info(hdev, &razwi_addr, &razwi_type); /* In case it's the first razwi, save its parameters*/ - rc = atomic_cmpxchg(&hdev->last_error.razwi.write_enable, 1, 0); + rc = atomic_cmpxchg(&hdev->captured_err_info.razwi.write_enable, 1, 0); if (rc) { - hdev->last_error.razwi.timestamp = ktime_get(); - hdev->last_error.razwi.addr = razwi_addr; - hdev->last_error.razwi.engine_id_1 = engine_id_1; - hdev->last_error.razwi.engine_id_2 = engine_id_2; + hdev->captured_err_info.razwi.timestamp = ktime_get(); + hdev->captured_err_info.razwi.addr = razwi_addr; + hdev->captured_err_info.razwi.engine_id_1 = engine_id_1; + hdev->captured_err_info.razwi.engine_id_2 = engine_id_2; /* * If first engine id holds non valid value the razwi initiator * does not have engine id */ - hdev->last_error.razwi.non_engine_initiator = (engine_id_1 == U16_MAX); - hdev->last_error.razwi.type = razwi_type; + hdev->captured_err_info.razwi.non_engine_initiator = + (engine_id_1 == U16_MAX); + hdev->captured_err_info.razwi.type = razwi_type; } } @@ -7427,7 +7406,7 @@ static void gaudi_print_nic_axi_irq_info(struct hl_device *hdev, u16 event_type, event_type, desc); } -static int gaudi_non_hard_reset_late_init(struct hl_device *hdev) +static int gaudi_compute_reset_late_init(struct hl_device *hdev) { /* GAUDI doesn't support any reset except hard-reset */ return -EPERM; @@ -7702,6 +7681,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR: gaudi_print_irq_info(hdev, event_type, true); gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; goto reset_device; @@ -7711,6 +7691,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17: gaudi_print_irq_info(hdev, event_type, false); fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; goto reset_device; case GAUDI_EVENT_HBM0_SPI_0: @@ -7722,6 +7703,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr gaudi_hbm_event_to_dev(event_type), &eq_entry->hbm_ecc_data); fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; goto reset_device; case GAUDI_EVENT_HBM0_SPI_1: @@ -7733,6 +7715,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr gaudi_hbm_event_to_dev(event_type), &eq_entry->hbm_ecc_data); hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI_EVENT_TPC0_DEC: @@ -7743,10 +7726,17 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr case GAUDI_EVENT_TPC5_DEC: case GAUDI_EVENT_TPC6_DEC: case GAUDI_EVENT_TPC7_DEC: + /* In TPC DEC event, notify on TPC assertion. While there isn't + * a specific event for assertion yet, the FW generates TPC DEC event. + * The SW upper layer will inspect an internal mapped area to indicate + * if the event is a TPC Assertion or a "real" TPC DEC. + */ + event_mask |= HL_NOTIFIER_EVENT_TPC_ASSERT; gaudi_print_irq_info(hdev, event_type, true); reset_required = gaudi_tpc_read_interrupts(hdev, tpc_dec_event_to_tpc_id(event_type), "AXI_SLV_DEC_Error"); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; if (reset_required) { dev_err(hdev->dev, "reset required due to %s\n", gaudi_irq_map_table[event_type].name); @@ -7755,6 +7745,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr goto reset_device; } else { hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; } break; @@ -7770,6 +7761,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr reset_required = gaudi_tpc_read_interrupts(hdev, tpc_krn_event_to_tpc_id(event_type), "KRN_ERR"); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; if (reset_required) { dev_err(hdev->dev, "reset required due to %s\n", gaudi_irq_map_table[event_type].name); @@ -7778,6 +7770,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr goto reset_device; } else { hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; } break; @@ -7806,9 +7799,25 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr gaudi_print_irq_info(hdev, event_type, true); gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI_EVENT_PCIE_DEC: + case GAUDI_EVENT_CPU_AXI_SPLITTER: + case GAUDI_EVENT_PSOC_AXI_DEC: + case GAUDI_EVENT_PSOC_PRSTN_FALL: + gaudi_print_irq_info(hdev, event_type, true); + hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; + break; + + case GAUDI_EVENT_MMU_PAGE_FAULT: + case GAUDI_EVENT_MMU_WR_PERM: + gaudi_print_irq_info(hdev, event_type, true); + hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; + break; + case GAUDI_EVENT_MME0_WBC_RSP: case GAUDI_EVENT_MME0_SBAB0_RSP: case GAUDI_EVENT_MME1_WBC_RSP: @@ -7817,11 +7826,6 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr case GAUDI_EVENT_MME2_SBAB0_RSP: case GAUDI_EVENT_MME3_WBC_RSP: case GAUDI_EVENT_MME3_SBAB0_RSP: - case GAUDI_EVENT_CPU_AXI_SPLITTER: - case GAUDI_EVENT_PSOC_AXI_DEC: - case GAUDI_EVENT_PSOC_PRSTN_FALL: - case GAUDI_EVENT_MMU_PAGE_FAULT: - case GAUDI_EVENT_MMU_WR_PERM: case GAUDI_EVENT_RAZWI_OR_ADC: case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: @@ -7841,10 +7845,12 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr gaudi_print_irq_info(hdev, event_type, true); gaudi_handle_qman_err(hdev, event_type, &event_mask); hl_fw_unmask_irq(hdev, event_type); + event_mask |= (HL_NOTIFIER_EVENT_USER_ENGINE_ERR | HL_NOTIFIER_EVENT_DEVICE_RESET); break; case GAUDI_EVENT_RAZWI_OR_ADC_SW: gaudi_print_irq_info(hdev, event_type, true); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; goto reset_device; case GAUDI_EVENT_TPC0_BMON_SPMU: @@ -7858,11 +7864,13 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr case GAUDI_EVENT_DMA_BM_CH0 ... GAUDI_EVENT_DMA_BM_CH7: gaudi_print_irq_info(hdev, event_type, false); hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI_EVENT_NIC_SEI_0 ... GAUDI_EVENT_NIC_SEI_4: gaudi_print_nic_axi_irq_info(hdev, event_type, &data); hl_fw_unmask_irq(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI_EVENT_DMA_IF_SEI_0 ... GAUDI_EVENT_DMA_IF_SEI_3: @@ -7870,6 +7878,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr gaudi_print_sm_sei_info(hdev, event_type, &eq_entry->sm_sei_data); rc = hl_state_dump(hdev); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; if (rc) dev_err(hdev->dev, "Error during system state dump %d\n", rc); @@ -7880,6 +7889,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr break; case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E: + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; gaudi_print_clk_change_info(hdev, event_type); hl_fw_unmask_irq(hdev, event_type); break; @@ -7889,20 +7899,24 @@ static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entr dev_err(hdev->dev, "Received high temp H/W interrupt %d (cause %d)\n", event_type, cause); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI_EVENT_DEV_RESET_REQ: gaudi_print_irq_info(hdev, event_type, false); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; goto reset_device; case GAUDI_EVENT_PKT_QUEUE_OUT_SYNC: gaudi_print_irq_info(hdev, event_type, false); gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; goto reset_device; case GAUDI_EVENT_FW_ALIVE_S: gaudi_print_irq_info(hdev, event_type, false); gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; goto reset_device; default: @@ -8066,8 +8080,8 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) return 0; } -static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, - u8 mask_len, struct seq_file *s) +static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, + struct engines_data *e) { struct gaudi_device *gaudi = hdev->asic_specific; const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n"; @@ -8079,8 +8093,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u64 offset; int i, dma_id, port; - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nDMA is_idle QM_GLBL_STS0 QM_CGM_STS DMA_CORE_STS0\n" "--- ------- ------------ ---------- -------------\n"); @@ -8097,14 +8111,14 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GAUDI_ENGINE_ID_DMA_0 + dma_id, mask); - if (s) - seq_printf(s, fmt, dma_id, + if (e) + hl_engine_data_sprintf(e, fmt, dma_id, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts, dma_core_sts0); } - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nTPC is_idle QM_GLBL_STS0 QM_CGM_STS CFG_STATUS\n" "--- ------- ------------ ---------- ----------\n"); @@ -8119,14 +8133,14 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GAUDI_ENGINE_ID_TPC_0 + i, mask); - if (s) - seq_printf(s, fmt, i, + if (e) + hl_engine_data_sprintf(e, fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); } - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nMME is_idle QM_GLBL_STS0 QM_CGM_STS ARCH_STATUS\n" "--- ------- ------------ ---------- -----------\n"); @@ -8147,20 +8161,21 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GAUDI_ENGINE_ID_MME_0 + i, mask); - if (s) { + if (e) { if (!is_slave) - seq_printf(s, fmt, i, + hl_engine_data_sprintf(e, fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts, mme_arch_sts); else - seq_printf(s, mme_slave_fmt, i, + hl_engine_data_sprintf(e, mme_slave_fmt, i, is_eng_idle ? "Y" : "N", "-", "-", mme_arch_sts); } } - if (s) - seq_puts(s, "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" + if (e) + hl_engine_data_sprintf(e, + "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" "--- ------- ------------ ----------\n"); for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) { @@ -8174,8 +8189,8 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); - if (s) - seq_printf(s, nic_fmt, port, + if (e) + hl_engine_data_sprintf(e, nic_fmt, port, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts); } @@ -8189,15 +8204,15 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask); - if (s) - seq_printf(s, nic_fmt, port, + if (e) + hl_engine_data_sprintf(e, nic_fmt, port, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts); } } - if (s) - seq_puts(s, "\n"); + if (e) + hl_engine_data_sprintf(e, "\n"); return is_idle; } @@ -8392,13 +8407,13 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev, goto destroy_internal_cb_pool; } - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, HOST_SPACE_INTERNAL_CB_SZ); hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); if (rc) goto unreserve_internal_cb_pool; @@ -8425,13 +8440,13 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) return; - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); gen_pool_destroy(hdev->internal_cb_pool); @@ -9148,6 +9163,11 @@ static void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group dev_vrm_attr_grp->attrs = gaudi_vrm_dev_attrs; } +static int gaudi_send_device_activity(struct hl_device *hdev, bool open) +{ + return 0; +} + static const struct hl_asic_funcs gaudi_funcs = { .early_init = gaudi_early_init, .early_fini = gaudi_early_fini, @@ -9192,11 +9212,9 @@ static const struct hl_asic_funcs gaudi_funcs = { .send_heartbeat = gaudi_send_heartbeat, .debug_coresight = gaudi_debug_coresight, .is_device_idle = gaudi_is_device_idle, - .non_hard_reset_late_init = gaudi_non_hard_reset_late_init, + .compute_reset_late_init = gaudi_compute_reset_late_init, .hw_queues_lock = gaudi_hw_queues_lock, .hw_queues_unlock = gaudi_hw_queues_unlock, - .kdma_lock = NULL, - .kdma_unlock = NULL, .get_pci_id = gaudi_get_pci_id, .get_eeprom_data = gaudi_get_eeprom_data, .get_monitor_dump = gaudi_get_monitor_dump, @@ -9242,6 +9260,7 @@ static const struct hl_asic_funcs gaudi_funcs = { .mmu_get_real_page_size = hl_mmu_get_real_page_size, .access_dev_mem = hl_access_dev_mem, .set_dram_bar_base = gaudi_set_hbm_bar_base, + .send_device_activity = gaudi_send_device_activity, }; /** diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2.c b/drivers/misc/habanalabs/gaudi2/gaudi2.c index 98336a1a84b0cce042956e0e168c5bd04e1f390c..65e6cae6100a469062175d7785f6df6713b69af9 100644 --- a/drivers/misc/habanalabs/gaudi2/gaudi2.c +++ b/drivers/misc/habanalabs/gaudi2/gaudi2.c @@ -21,7 +21,7 @@ #define GAUDI2_DMA_POOL_BLK_SIZE SZ_256 /* 256 bytes */ -#define GAUDI2_RESET_TIMEOUT_MSEC 500 /* 500ms */ +#define GAUDI2_RESET_TIMEOUT_MSEC 2000 /* 2000ms */ #define GAUDI2_RESET_POLL_TIMEOUT_USEC 50000 /* 50ms */ #define GAUDI2_PLDM_HRESET_TIMEOUT_MSEC 25000 /* 25s */ #define GAUDI2_PLDM_SRESET_TIMEOUT_MSEC 25000 /* 25s */ @@ -117,6 +117,12 @@ #define MMU_RANGE_INV_ASID_EN_SHIFT 1 #define MMU_RANGE_INV_ASID_SHIFT 2 +/* The last SPI_SEI cause bit, "burst_fifo_full", is expected to be triggered in PMMU because it has + * a 2 entries FIFO, and hence it is not enabled for it. + */ +#define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0) +#define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0) + #define GAUDI2_MAX_STRING_LEN 64 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \ @@ -610,7 +616,7 @@ static const char * const guadi2_mme_error_cause[GAUDI2_NUM_OF_MME_ERR_CAUSE] = "qman_axi_err", "wap sei (wbc axi err)", "arc sei", - "mme_cfg_unalign_addr", + "cfg access error", "qm_sw_err", "sbte_dbg_intr_0", "sbte_dbg_intr_1", @@ -1525,17 +1531,57 @@ static const u32 rtr_coordinates_to_rtr_id[NUM_OF_RTR_PER_DCORE * NUM_OF_DCORES] RTR_ID_X_Y(17, 11) }; +enum rtr_id { + DCORE0_RTR0, + DCORE0_RTR1, + DCORE0_RTR2, + DCORE0_RTR3, + DCORE0_RTR4, + DCORE0_RTR5, + DCORE0_RTR6, + DCORE0_RTR7, + DCORE1_RTR0, + DCORE1_RTR1, + DCORE1_RTR2, + DCORE1_RTR3, + DCORE1_RTR4, + DCORE1_RTR5, + DCORE1_RTR6, + DCORE1_RTR7, + DCORE2_RTR0, + DCORE2_RTR1, + DCORE2_RTR2, + DCORE2_RTR3, + DCORE2_RTR4, + DCORE2_RTR5, + DCORE2_RTR6, + DCORE2_RTR7, + DCORE3_RTR0, + DCORE3_RTR1, + DCORE3_RTR2, + DCORE3_RTR3, + DCORE3_RTR4, + DCORE3_RTR5, + DCORE3_RTR6, + DCORE3_RTR7, +}; + static const u32 gaudi2_tpc_initiator_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = { - 1, 1, 2, 2, 3, 3, 14, 14, 13, 13, 12, 12, 19, 19, 18, 18, 17, - 17, 28, 28, 29, 29, 30, 30, 0 + DCORE0_RTR1, DCORE0_RTR1, DCORE0_RTR2, DCORE0_RTR2, DCORE0_RTR3, DCORE0_RTR3, + DCORE1_RTR6, DCORE1_RTR6, DCORE1_RTR5, DCORE1_RTR5, DCORE1_RTR4, DCORE1_RTR4, + DCORE2_RTR3, DCORE2_RTR3, DCORE2_RTR2, DCORE2_RTR2, DCORE2_RTR1, DCORE2_RTR1, + DCORE3_RTR4, DCORE3_RTR4, DCORE3_RTR5, DCORE3_RTR5, DCORE3_RTR6, DCORE3_RTR6, + DCORE0_RTR0 }; static const u32 gaudi2_dec_initiator_rtr_id[NUMBER_OF_DEC] = { - 0, 0, 15, 15, 16, 16, 31, 31, 0, 0 + DCORE0_RTR0, DCORE0_RTR0, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, DCORE2_RTR0, + DCORE3_RTR7, DCORE3_RTR7, DCORE0_RTR0, DCORE0_RTR0 }; static const u32 gaudi2_nic_initiator_rtr_id[NIC_NUMBER_OF_MACROS] = { - 15, 15, 15, 15, 15, 16, 16, 16, 16, 31, 31, 31 + DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE1_RTR7, DCORE2_RTR0, + DCORE2_RTR0, DCORE2_RTR0, DCORE2_RTR0, DCORE3_RTR7, DCORE3_RTR7, DCORE3_RTR7 }; struct sft_info { @@ -1548,11 +1594,11 @@ static const struct sft_info gaudi2_edma_initiator_sft_id[NUM_OF_EDMA_PER_DCORE }; static const u32 gaudi2_pdma_initiator_rtr_id[NUM_OF_PDMA] = { - 0, 0 + DCORE0_RTR0, DCORE0_RTR0 }; static const u32 gaudi2_rot_initiator_rtr_id[NUM_OF_ROT] = { - 16, 31 + DCORE2_RTR0, DCORE3_RTR7 }; struct mme_initiators_rtr_id { @@ -1663,7 +1709,7 @@ struct gaudi2_cache_invld_params { }; struct gaudi2_tpc_idle_data { - struct seq_file *s; + struct engines_data *e; unsigned long *mask; bool *is_idle; const char *tpc_fmt; @@ -1706,6 +1752,9 @@ void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) int dcore, inst, tpc_seq; u32 offset; + /* init the return code */ + ctx->rc = 0; + for (dcore = 0; dcore < NUM_OF_DCORES; dcore++) { for (inst = 0; inst < NUM_OF_TPC_PER_DCORE; inst++) { tpc_seq = dcore * NUM_OF_TPC_PER_DCORE + inst; @@ -1715,7 +1764,12 @@ void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) offset = (DCORE_OFFSET * dcore) + (DCORE_TPC_OFFSET * inst); - ctx->fn(hdev, dcore, inst, offset, ctx->data); + ctx->fn(hdev, dcore, inst, offset, ctx); + if (ctx->rc) { + dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", + dcore, inst); + return; + } } } @@ -1724,7 +1778,9 @@ void gaudi2_iterate_tpcs(struct hl_device *hdev, struct iterate_module_ctx *ctx) /* special check for PCI TPC (DCORE0_TPC6) */ offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); - ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx->data); + ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); + if (ctx->rc) + dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); } static bool gaudi2_host_phys_addr_valid(u64 addr) @@ -1973,6 +2029,7 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev) prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; } + prop->num_engine_cores = CPU_ID_MAX; prop->cfg_size = CFG_SIZE; prop->max_asid = MAX_ASID; prop->num_of_events = GAUDI2_EVENT_SIZE; @@ -2005,9 +2062,6 @@ static int gaudi2_set_fixed_properties(struct hl_device *hdev) prop->server_type = HL_SERVER_TYPE_UNKNOWN; - prop->cb_va_start_addr = VA_HOST_SPACE_USER_MAPPED_CB_START; - prop->cb_va_end_addr = VA_HOST_SPACE_USER_MAPPED_CB_END; - prop->max_dec = NUMBER_OF_DEC; prop->clk_pll_index = HL_GAUDI2_MME_PLL; @@ -2477,7 +2531,6 @@ static int gaudi2_early_init(struct hl_device *hdev) struct asic_fixed_properties *prop = &hdev->asic_prop; struct pci_dev *pdev = hdev->pdev; resource_size_t pci_bar_size; - u32 fw_boot_status; int rc; rc = gaudi2_set_fixed_properties(hdev); @@ -2505,22 +2558,14 @@ static int gaudi2_early_init(struct hl_device *hdev) prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); - /* If FW security is enabled at this point it means no access to ELBI */ - if (hdev->asic_prop.fw_security_enabled) { - hdev->asic_prop.iatu_done_by_fw = true; - goto pci_init; - } - - rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0, &fw_boot_status); - if (rc) - goto free_queue_props; - - /* Check whether FW is configuring iATU */ - if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) && - (fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN)) + /* + * Only in pldm driver config iATU + */ + if (hdev->pldm) + hdev->asic_prop.iatu_done_by_fw = false; + else hdev->asic_prop.iatu_done_by_fw = true; -pci_init: rc = hl_pci_init(hdev); if (rc) goto free_queue_props; @@ -2676,6 +2721,8 @@ static int gaudi2_late_init(struct hl_device *hdev) struct gaudi2_device *gaudi2 = hdev->asic_specific; int rc; + hdev->asic_prop.supports_advanced_cpucp_rc = true; + rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, gaudi2->virt_msix_db_dma_addr); if (rc) { @@ -2703,23 +2750,7 @@ disable_pci_access: static void gaudi2_late_fini(struct hl_device *hdev) { - const struct hwmon_channel_info **channel_info_arr; - int i = 0; - - if (!hdev->hl_chip_info->info) - return; - - channel_info_arr = hdev->hl_chip_info->info; - - while (channel_info_arr[i]) { - kfree(channel_info_arr[i]->config); - kfree(channel_info_arr[i]); - i++; - } - - kfree(channel_info_arr); - - hdev->hl_chip_info->info = NULL; + hl_hwmon_release_resources(hdev); } static void gaudi2_user_mapped_dec_init(struct gaudi2_device *gaudi2, u32 start_idx) @@ -2917,7 +2948,7 @@ static void gaudi2_user_interrupt_setup(struct hl_device *hdev) static inline int gaudi2_get_non_zero_random_int(void) { - int rand = get_random_int(); + int rand = get_random_u32(); return rand ? rand : 1; } @@ -2994,7 +3025,6 @@ static int gaudi2_sw_init(struct hl_device *hdev) } spin_lock_init(&gaudi2->hw_queues_lock); - spin_lock_init(&gaudi2->kdma_lock); gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, &gaudi2->scratchpad_bus_address, @@ -3551,7 +3581,7 @@ static int gaudi2_enable_msix(struct hl_device *hdev) rc = gaudi2_dec_enable_msix(hdev); if (rc) { dev_err(hdev->dev, "Failed to enable decoder IRQ"); - goto free_completion_irq; + goto free_event_irq; } for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; @@ -3582,6 +3612,10 @@ free_user_irq: gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); +free_event_irq: + irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); + free_irq(irq, cq); + free_completion_irq: irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); free_irq(irq, cq); @@ -3745,14 +3779,16 @@ static void gaudi2_stop_dec(struct hl_device *hdev) gaudi2_stop_pcie_dec(hdev); } -static void gaudi2_halt_arc(struct hl_device *hdev, u32 cpu_id) +static void gaudi2_set_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) { u32 reg_base, reg_val; reg_base = gaudi2_arc_blocks_bases[cpu_id]; + if (run_mode == HL_ENGINE_CORE_RUN) + reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); + else + reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); - /* Halt ARC */ - reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); WREG32(reg_base + ARC_HALT_REQ_OFFSET, reg_val); } @@ -3762,8 +3798,35 @@ static void gaudi2_halt_arcs(struct hl_device *hdev) for (arc_id = CPU_ID_SCHED_ARC0; arc_id < CPU_ID_MAX; arc_id++) { if (gaudi2_is_arc_enabled(hdev, arc_id)) - gaudi2_halt_arc(hdev, arc_id); + gaudi2_set_arc_running_mode(hdev, arc_id, HL_ENGINE_CORE_HALT); + } +} + +static int gaudi2_verify_arc_running_mode(struct hl_device *hdev, u32 cpu_id, u32 run_mode) +{ + int rc; + u32 reg_base, val, ack_mask, timeout_usec = 100000; + + if (hdev->pldm) + timeout_usec *= 100; + + reg_base = gaudi2_arc_blocks_bases[cpu_id]; + if (run_mode == HL_ENGINE_CORE_RUN) + ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_RUN_ACK_MASK; + else + ack_mask = ARC_FARM_ARC0_AUX_RUN_HALT_ACK_HALT_ACK_MASK; + + rc = hl_poll_timeout(hdev, reg_base + ARC_HALT_ACK_OFFSET, + val, ((val & ack_mask) == ack_mask), + 1000, timeout_usec); + + if (!rc) { + /* Clear */ + val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 0); + WREG32(reg_base + ARC_HALT_REQ_OFFSET, val); } + + return rc; } static void gaudi2_reset_arcs(struct hl_device *hdev) @@ -3790,8 +3853,39 @@ static void gaudi2_nic_qmans_manual_flush(struct hl_device *hdev) queue_id = GAUDI2_QUEUE_ID_NIC_0_0; - for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) + for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++, queue_id += NUM_OF_PQ_PER_QMAN) { + if (!(hdev->nic_ports_mask & BIT(i))) + continue; + gaudi2_qman_manual_flush_common(hdev, queue_id); + } +} + +static int gaudi2_set_engine_cores(struct hl_device *hdev, u32 *core_ids, + u32 num_cores, u32 core_command) +{ + int i, rc; + + + for (i = 0 ; i < num_cores ; i++) { + if (gaudi2_is_arc_enabled(hdev, core_ids[i])) + gaudi2_set_arc_running_mode(hdev, core_ids[i], core_command); + } + + for (i = 0 ; i < num_cores ; i++) { + if (gaudi2_is_arc_enabled(hdev, core_ids[i])) { + rc = gaudi2_verify_arc_running_mode(hdev, core_ids[i], core_command); + + if (rc) { + dev_err(hdev->dev, "failed to %s arc: %d\n", + (core_command == HL_ENGINE_CORE_HALT) ? + "HALT" : "RUN", core_ids[i]); + return -1; + } + } + } + + return 0; } static void gaudi2_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset) @@ -4124,11 +4218,15 @@ static void gaudi2_init_qman_common(struct hl_device *hdev, u32 reg_base, WREG32(reg_base + QM_GLBL_CFG2_OFFSET, 0); /* Enable the QMAN channel. - * PDMA1 QMAN configuration is different, as we do not allow user to - * access CP2/3, it is reserved for the ARC usage. + * PDMA QMAN configuration is different, as we do not allow user to + * access some of the CPs. + * PDMA0: CP2/3 are reserved for the ARC usage. + * PDMA1: CP1/2/3 are reserved for the ARC usage. */ if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_1_0]) WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA1_QMAN_ENABLE); + else if (reg_base == gaudi2_qm_blocks_bases[GAUDI2_QUEUE_ID_PDMA_0_0]) + WREG32(reg_base + QM_GLBL_CFG0_OFFSET, PDMA0_QMAN_ENABLE); else WREG32(reg_base + QM_GLBL_CFG0_OFFSET, QMAN_ENABLE); } @@ -4501,10 +4599,10 @@ struct gaudi2_tpc_init_cfg_data { }; static void gaudi2_init_tpc_config(struct hl_device *hdev, int dcore, int inst, - u32 offset, void *data) + u32 offset, struct iterate_module_ctx *ctx) { struct gaudi2_device *gaudi2 = hdev->asic_specific; - struct gaudi2_tpc_init_cfg_data *cfg_data = data; + struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; u32 queue_id_base; u8 seq; @@ -4956,8 +5054,7 @@ static int gaudi2_mmu_update_hop0_addr(struct hl_device *hdev, u32 stlb_base) return 0; } -static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, - u32 stlb_base) +static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, u32 stlb_base) { u32 status, timeout_usec; int rc; @@ -4985,7 +5082,6 @@ static int gaudi2_mmu_init_common(struct hl_device *hdev, u32 mmu_base, return rc; WREG32(mmu_base + MMU_BYPASS_OFFSET, 0); - WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, 0xF); rc = hl_poll_timeout( hdev, @@ -5042,6 +5138,8 @@ static int gaudi2_pci_mmu_init(struct hl_device *hdev) DCORE0_HMMU0_MMU_STATIC_MULTI_PAGE_SIZE_CFG_8_BITS_HOP_MODE_EN_MASK); } + WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_PMMU_SPI_SEI_ENABLE_MASK); + rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); if (rc) return rc; @@ -5092,6 +5190,8 @@ static int gaudi2_dcore_hmmu_init(struct hl_device *hdev, int dcore_id, RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, STLB_HOP_CONFIGURATION_ONLY_LARGE_PAGE_MASK); + WREG32(mmu_base + MMU_SPI_SEI_MASK_OFFSET, GAUDI2_HMMU_SPI_SEI_ENABLE_MASK); + rc = gaudi2_mmu_init_common(hdev, mmu_base, stlb_base); if (rc) return rc; @@ -5339,7 +5439,10 @@ static void gaudi2_execute_soft_reset(struct hl_device *hdev, u32 reset_sleep_ms if (!driver_performs_reset) { /* set SP to indicate reset request sent to FW */ - WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); + if (dyn_regs->cpu_rst_status) + WREG32(le32_to_cpu(dyn_regs->cpu_rst_status), CPU_RST_STATUS_NA); + else + WREG32(mmCPU_RST_STATUS_TO_HOST, CPU_RST_STATUS_NA); WREG32(le32_to_cpu(dyn_regs->gic_host_soft_rst_irq), gaudi2_irq_map_table[GAUDI2_EVENT_CPU_SOFT_RESET].cpu_id); @@ -5527,10 +5630,11 @@ static bool gaudi2_is_queue_enabled(struct hl_device *hdev, u32 hw_queue_id) u64 hw_test_cap_bit = 0; switch (hw_queue_id) { - case GAUDI2_QUEUE_ID_PDMA_0_0 ... GAUDI2_QUEUE_ID_PDMA_1_1: + case GAUDI2_QUEUE_ID_PDMA_0_0: + case GAUDI2_QUEUE_ID_PDMA_0_1: + case GAUDI2_QUEUE_ID_PDMA_1_0: hw_cap_mask = HW_CAP_PDMA_MASK; break; - case GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0...GAUDI2_QUEUE_ID_DCORE0_EDMA_1_3: hw_test_cap_bit = HW_CAP_EDMA_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); @@ -6129,7 +6233,7 @@ done: return ret_val; } -static int gaudi2_non_hard_reset_late_init(struct hl_device *hdev) +static int gaudi2_compute_reset_late_init(struct hl_device *hdev) { struct gaudi2_device *gaudi2 = hdev->asic_specific; size_t irq_arr_size; @@ -6147,9 +6251,9 @@ static int gaudi2_non_hard_reset_late_init(struct hl_device *hdev) } static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int inst, u32 offset, - void *data) + struct iterate_module_ctx *ctx) { - struct gaudi2_tpc_idle_data *idle_data = (struct gaudi2_tpc_idle_data *)data; + struct gaudi2_tpc_idle_data *idle_data = ctx->data; u32 tpc_cfg_sts, qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts; bool is_eng_idle; int engine_idx; @@ -6172,14 +6276,15 @@ static void gaudi2_is_tpc_engine_idle(struct hl_device *hdev, int dcore, int ins if (idle_data->mask && !is_eng_idle) set_bit(engine_idx, idle_data->mask); - if (idle_data->s) - seq_printf(idle_data->s, idle_data->tpc_fmt, dcore, inst, + if (idle_data->e) + hl_engine_data_sprintf(idle_data->e, + idle_data->tpc_fmt, dcore, inst, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts); } -static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, - u8 mask_len, struct seq_file *s) +static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, + struct engines_data *e) { u32 qm_glbl_sts0, qm_glbl_sts1, qm_cgm_sts, dma_core_idle_ind_mask, mme_arch_sts, dec_swreg15, dec_enabled_bit; @@ -6197,7 +6302,7 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, struct gaudi2_tpc_idle_data tpc_idle_data = { .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", - .s = s, + .e = e, .mask = mask, .is_idle = &is_idle, }; @@ -6209,8 +6314,8 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, int engine_idx, i, j; /* EDMA, Two engines per Dcore */ - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nCORE EDMA is_idle QM_GLBL_STS0 DMA_CORE_IDLE_IND_MASK\n" "---- ---- ------- ------------ ----------------------\n"); @@ -6239,19 +6344,19 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, edma_fmt, i, j, - is_eng_idle ? "Y" : "N", - qm_glbl_sts0, - dma_core_idle_ind_mask); + if (e) + hl_engine_data_sprintf(e, edma_fmt, i, j, + is_eng_idle ? "Y" : "N", + qm_glbl_sts0, + dma_core_idle_ind_mask); } } /* PDMA, Two engines in Full chip */ - if (s) - seq_puts(s, - "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_IDLE_IND_MASK\n" - "---- ------- ------------ ----------------------\n"); + if (e) + hl_engine_data_sprintf(e, + "\nPDMA is_idle QM_GLBL_STS0 DMA_CORE_IDLE_IND_MASK\n" + "---- ------- ------------ ----------------------\n"); for (i = 0 ; i < NUM_OF_PDMA ; i++) { engine_idx = GAUDI2_ENGINE_ID_PDMA_0 + i; @@ -6269,16 +6374,16 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, pdma_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, - dma_core_idle_ind_mask); + if (e) + hl_engine_data_sprintf(e, pdma_fmt, i, is_eng_idle ? "Y" : "N", + qm_glbl_sts0, dma_core_idle_ind_mask); } /* NIC, twelve macros in Full chip */ - if (s && hdev->nic_ports_mask) - seq_puts(s, - "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" - "--- ------- ------------ ----------\n"); + if (e && hdev->nic_ports_mask) + hl_engine_data_sprintf(e, + "\nNIC is_idle QM_GLBL_STS0 QM_CGM_STS\n" + "--- ------- ------------ ----------\n"); for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { if (!(i & 1)) @@ -6302,15 +6407,15 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, nic_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, - qm_cgm_sts); + if (e) + hl_engine_data_sprintf(e, nic_fmt, i, is_eng_idle ? "Y" : "N", + qm_glbl_sts0, qm_cgm_sts); } - if (s) - seq_puts(s, - "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" - "--- ---- ------- ------------ ---------------\n"); + if (e) + hl_engine_data_sprintf(e, + "\nMME Stub is_idle QM_GLBL_STS0 MME_ARCH_STATUS\n" + "--- ---- ------- ------------ ---------------\n"); /* MME, one per Dcore */ for (i = 0 ; i < NUM_OF_DCORES ; i++) { engine_idx = GAUDI2_DCORE0_ENGINE_ID_MME + i * GAUDI2_ENGINE_ID_DCORE_OFFSET; @@ -6327,8 +6432,8 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, is_eng_idle &= IS_MME_IDLE(mme_arch_sts); is_idle &= is_eng_idle; - if (s) - seq_printf(s, mme_fmt, i, "N", + if (e) + hl_engine_data_sprintf(e, mme_fmt, i, "N", is_eng_idle ? "Y" : "N", qm_glbl_sts0, mme_arch_sts); @@ -6340,16 +6445,16 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, /* * TPC */ - if (s && prop->tpc_enabled_mask) - seq_puts(s, + if (e && prop->tpc_enabled_mask) + hl_engine_data_sprintf(e, "\nCORE TPC is_idle QM_GLBL_STS0 QM_CGM_STS DMA_CORE_IDLE_IND_MASK\n" "---- --- -------- ------------ ---------- ----------------------\n"); gaudi2_iterate_tpcs(hdev, &tpc_iter); /* Decoders, two each Dcore and two shared PCIe decoders */ - if (s && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) - seq_puts(s, + if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) + hl_engine_data_sprintf(e, "\nCORE DEC is_idle VSI_CMD_SWREG15\n" "---- --- ------- ---------------\n"); @@ -6370,13 +6475,14 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, dec_fmt, i, j, is_eng_idle ? "Y" : "N", dec_swreg15); + if (e) + hl_engine_data_sprintf(e, dec_fmt, i, j, + is_eng_idle ? "Y" : "N", dec_swreg15); } } - if (s && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) - seq_puts(s, + if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) + hl_engine_data_sprintf(e, "\nPCIe DEC is_idle VSI_CMD_SWREG15\n" "-------- ------- ---------------\n"); @@ -6395,12 +6501,13 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, pcie_dec_fmt, i, is_eng_idle ? "Y" : "N", dec_swreg15); + if (e) + hl_engine_data_sprintf(e, pcie_dec_fmt, i, + is_eng_idle ? "Y" : "N", dec_swreg15); } - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nCORE ROT is_idle QM_GLBL_STS0 QM_CGM_STS DMA_CORE_STS0\n" "---- ---- ------- ------------ ---------- -------------\n"); @@ -6419,8 +6526,8 @@ static bool gaudi2_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(engine_idx, mask); - if (s) - seq_printf(s, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", + if (e) + hl_engine_data_sprintf(e, rot_fmt, i, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, qm_cgm_sts, "-"); } @@ -6443,22 +6550,6 @@ static void gaudi2_hw_queues_unlock(struct hl_device *hdev) spin_unlock(&gaudi2->hw_queues_lock); } -static void gaudi2_kdma_lock(struct hl_device *hdev, int dcore_id) - __acquires(&gaudi2->kdma_lock) -{ - struct gaudi2_device *gaudi2 = hdev->asic_specific; - - spin_lock(&gaudi2->kdma_lock); -} - -static void gaudi2_kdma_unlock(struct hl_device *hdev, int dcore_id) - __releases(&gaudi2->kdma_lock) -{ - struct gaudi2_device *gaudi2 = hdev->asic_specific; - - spin_unlock(&gaudi2->kdma_lock); -} - static u32 gaudi2_get_pci_id(struct hl_device *hdev) { return hdev->pdev->device; @@ -6725,9 +6816,9 @@ static int gaudi2_mmu_shared_prepare(struct hl_device *hdev, u32 asid) } static void gaudi2_tpc_mmu_prepare(struct hl_device *hdev, int dcore, int inst, u32 offset, - void *data) + struct iterate_module_ctx *ctx) { - struct gaudi2_tpc_mmu_data *mmu_data = (struct gaudi2_tpc_mmu_data *)data; + struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_MMU_BP + offset, 0); WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); @@ -7020,10 +7111,6 @@ static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, razwi_lo = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_lo_reg); razwi_xy = le32_to_cpu(razwi_info->hbw.rr_aw_razwi_id_reg); } - - dev_err_ratelimited(hdev->dev, - "%s-RAZWI SHARED RR HBW WR error, captured address HI 0x%x LO 0x%x, Initiator coordinates 0x%x\n", - name, razwi_hi, razwi_lo, razwi_xy); } else { if (read_razwi_regs) { razwi_hi = RREG32(rtr_mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HI); @@ -7034,11 +7121,11 @@ static void gaudi2_razwi_rr_hbw_shared_printf_info(struct hl_device *hdev, razwi_lo = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_lo_reg); razwi_xy = le32_to_cpu(razwi_info->hbw.rr_ar_razwi_id_reg); } - - dev_err_ratelimited(hdev->dev, - "%s-RAZWI SHARED RR HBW AR error, captured address HI 0x%x LO 0x%x, Initiator coordinates 0x%x\n", - name, razwi_hi, razwi_lo, razwi_xy); } + + dev_err_ratelimited(hdev->dev, + "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", + name, is_write ? "WR" : "RD", (u64)razwi_hi << 32 | razwi_lo, razwi_xy); } static void gaudi2_razwi_rr_lbw_shared_printf_info(struct hl_device *hdev, @@ -7296,7 +7383,79 @@ static void gaudi2_check_if_razwi_happened(struct hl_device *hdev) gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_ROT, mod_idx, 0, NULL); } -static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, +static const char *gaudi2_get_initiators_name(u32 rtr_id) +{ + switch (rtr_id) { + case DCORE0_RTR0: + return "DEC0/1/8/9, TPC24, PDMA0/1, PMMU, PCIE_IF, EDMA0/2, HMMU0/2/4/6, CPU"; + case DCORE0_RTR1: + return "TPC0/1"; + case DCORE0_RTR2: + return "TPC2/3"; + case DCORE0_RTR3: + return "TPC4/5"; + case DCORE0_RTR4: + return "MME0_SBTE0/1"; + case DCORE0_RTR5: + return "MME0_WAP0/SBTE2"; + case DCORE0_RTR6: + return "MME0_CTRL_WR/SBTE3"; + case DCORE0_RTR7: + return "MME0_WAP1/CTRL_RD/SBTE4"; + case DCORE1_RTR0: + return "MME1_WAP1/CTRL_RD/SBTE4"; + case DCORE1_RTR1: + return "MME1_CTRL_WR/SBTE3"; + case DCORE1_RTR2: + return "MME1_WAP0/SBTE2"; + case DCORE1_RTR3: + return "MME1_SBTE0/1"; + case DCORE1_RTR4: + return "TPC10/11"; + case DCORE1_RTR5: + return "TPC8/9"; + case DCORE1_RTR6: + return "TPC6/7"; + case DCORE1_RTR7: + return "DEC2/3, NIC0/1/2/3/4, ARC_FARM, KDMA, EDMA1/3, HMMU1/3/5/7"; + case DCORE2_RTR0: + return "DEC4/5, NIC5/6/7/8, EDMA4/6, HMMU8/10/12/14, ROT0"; + case DCORE2_RTR1: + return "TPC16/17"; + case DCORE2_RTR2: + return "TPC14/15"; + case DCORE2_RTR3: + return "TPC12/13"; + case DCORE2_RTR4: + return "MME2_SBTE0/1"; + case DCORE2_RTR5: + return "MME2_WAP0/SBTE2"; + case DCORE2_RTR6: + return "MME2_CTRL_WR/SBTE3"; + case DCORE2_RTR7: + return "MME2_WAP1/CTRL_RD/SBTE4"; + case DCORE3_RTR0: + return "MME3_WAP1/CTRL_RD/SBTE4"; + case DCORE3_RTR1: + return "MME3_CTRL_WR/SBTE3"; + case DCORE3_RTR2: + return "MME3_WAP0/SBTE2"; + case DCORE3_RTR3: + return "MME3_SBTE0/1"; + case DCORE3_RTR4: + return "TPC18/19"; + case DCORE3_RTR5: + return "TPC20/21"; + case DCORE3_RTR6: + return "TPC22/23"; + case DCORE3_RTR7: + return "DEC6/7, NIC9/10/11, EDMA5/7, HMMU9/11/13/15, ROT1, PSOC"; + default: + return "N/A"; + } +} + +static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, u32 rtr_id, u64 rtr_ctrl_base_addr, bool is_write) { u32 razwi_hi, razwi_lo; @@ -7305,50 +7464,47 @@ static void gaudi2_razwi_unmapped_addr_hbw_printf_info(struct hl_device *hdev, razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_HI); razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_ADDR_LO); - dev_err_ratelimited(hdev->dev, - "RAZWI PSOC unmapped HBW WR error, ctr_base 0x%llx, captured address HI 0x%x, LO 0x%x\n", - rtr_ctrl_base_addr, razwi_hi, razwi_lo); - /* Clear set indication */ WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AW_SET, 0x1); } else { razwi_hi = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_HI); - razwi_lo = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_ADDR_LO); - dev_err_ratelimited(hdev->dev, - "RAZWI PSOC unmapped HBW AR error, ctr_base 0x%llx, captured address HI 0x%x, LO 0x%x\n", - rtr_ctrl_base_addr, razwi_hi, razwi_lo); - /* Clear set indication */ WREG32(rtr_ctrl_base_addr + DEC_RAZWI_HBW_AR_SET, 0x1); } + + dev_err_ratelimited(hdev->dev, + "RAZWI PSOC unmapped HBW %s error, rtr id %u, address %#llx\n", + is_write ? "WR" : "RD", rtr_id, (u64)razwi_hi << 32 | razwi_lo); + + dev_err_ratelimited(hdev->dev, + "Initiators: %s\n", gaudi2_get_initiators_name(rtr_id)); } -static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev, - u64 rtr_ctrl_base_addr, bool is_write) +static void gaudi2_razwi_unmapped_addr_lbw_printf_info(struct hl_device *hdev, u32 rtr_id, + u64 rtr_ctrl_base_addr, bool is_write) { u32 razwi_addr; if (is_write) { razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_ADDR); - dev_err_ratelimited(hdev->dev, - "RAZWI PSOC unmapped LBW WR error, ctr_base 0x%llx, captured address 0x%x\n", - rtr_ctrl_base_addr, razwi_addr); - /* Clear set indication */ WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AW_SET, 0x1); } else { razwi_addr = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_ADDR); - dev_err_ratelimited(hdev->dev, - "RAZWI PSOC unmapped LBW AR error, ctr_base 0x%llx, captured address 0x%x\n", - rtr_ctrl_base_addr, razwi_addr); - /* Clear set indication */ WREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET, 0x1); } + + dev_err_ratelimited(hdev->dev, + "RAZWI PSOC unmapped LBW %s error, rtr id %u, address %#x\n", + is_write ? "WR" : "RD", rtr_id, razwi_addr); + + dev_err_ratelimited(hdev->dev, + "Initiators: %s\n", gaudi2_get_initiators_name(rtr_id)); } /* PSOC RAZWI interrupt occurs only when trying to access a bad address */ @@ -7366,21 +7522,16 @@ static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev) } razwi_mask_info = RREG32(mmPSOC_GLOBAL_CONF_RAZWI_MASK_INFO); - - xy = (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK) - >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_SHIFT; + xy = FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_L_MASK, razwi_mask_info); dev_err_ratelimited(hdev->dev, - "PSOC RAZWI interrupt: Mask %d, WAS_AR %d, WAS_AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", - (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK) - >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_SHIFT, - (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK) - >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_SHIFT, - (razwi_mask_info & PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK) - >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_SHIFT, xy, - (razwi_mask_info & - PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK) - >> PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_SHIFT); + "PSOC RAZWI interrupt: Mask %d, AR %d, AW %d, AXUSER_L 0x%x AXUSER_H 0x%x\n", + FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_MASK_MASK, razwi_mask_info), + FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AR_MASK, razwi_mask_info), + FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_WAS_AW_MASK, razwi_mask_info), + xy, + FIELD_GET(PSOC_GLOBAL_CONF_RAZWI_MASK_INFO_AXUSER_H_MASK, razwi_mask_info)); + if (xy == 0) { dev_err_ratelimited(hdev->dev, "PSOC RAZWI interrupt: received event from 0 rtr coordinates\n"); @@ -7410,16 +7561,20 @@ static void gaudi2_ack_psoc_razwi_event_handler(struct hl_device *hdev) lbw_ar_set = RREG32(rtr_ctrl_base_addr + DEC_RAZWI_LBW_AR_SET); if (hbw_aw_set) - gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_ctrl_base_addr, true); + gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id, + rtr_ctrl_base_addr, true); if (hbw_ar_set) - gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_ctrl_base_addr, false); + gaudi2_razwi_unmapped_addr_hbw_printf_info(hdev, rtr_id, + rtr_ctrl_base_addr, false); if (lbw_aw_set) - gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_ctrl_base_addr, true); + gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id, + rtr_ctrl_base_addr, true); if (lbw_ar_set) - gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_ctrl_base_addr, false); + gaudi2_razwi_unmapped_addr_lbw_printf_info(hdev, rtr_id, + rtr_ctrl_base_addr, false); clear: /* Clear Interrupts only on pldm or if f/w doesn't handle interrupts */ @@ -7811,14 +7966,58 @@ static void gaudi2_handle_dma_core_event(struct hl_device *hdev, u64 intr_cause_ gaudi2_dma_core_interrupts_cause[i]); } +static void gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(struct hl_device *hdev) +{ + u32 mstr_if_base_addr = mmPCIE_MSTR_RR_MSTR_IF_RR_SHRD_HBW_BASE, razwi_happened_addr; + + razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AW_RAZWI_HAPPENED; + if (RREG32(razwi_happened_addr)) { + gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", true, + NULL); + WREG32(razwi_happened_addr, 0x1); + } + + razwi_happened_addr = mstr_if_base_addr + RR_SHRD_HBW_AR_RAZWI_HAPPENED; + if (RREG32(razwi_happened_addr)) { + gaudi2_razwi_rr_hbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", true, + NULL); + WREG32(razwi_happened_addr, 0x1); + } + + razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AW_RAZWI_HAPPENED; + if (RREG32(razwi_happened_addr)) { + gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, true, "PCIE", true, + NULL); + WREG32(razwi_happened_addr, 0x1); + } + + razwi_happened_addr = mstr_if_base_addr + RR_SHRD_LBW_AR_RAZWI_HAPPENED; + if (RREG32(razwi_happened_addr)) { + gaudi2_razwi_rr_lbw_shared_printf_info(hdev, mstr_if_base_addr, false, "PCIE", true, + NULL); + WREG32(razwi_happened_addr, 0x1); + } +} + static void gaudi2_print_pcie_addr_dec_info(struct hl_device *hdev, u64 intr_cause_data) { int i; - for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE; i++) - if (intr_cause_data & BIT_ULL(i)) - dev_err_ratelimited(hdev->dev, "PCIE ADDR DEC Error: %s\n", - gaudi2_pcie_addr_dec_error_cause[i]); + for (i = 0 ; i < GAUDI2_NUM_OF_PCIE_ADDR_DEC_ERR_CAUSE ; i++) { + if (!(intr_cause_data & BIT_ULL(i))) + continue; + + dev_err_ratelimited(hdev->dev, "PCIE ADDR DEC Error: %s\n", + gaudi2_pcie_addr_dec_error_cause[i]); + + switch (intr_cause_data & BIT_ULL(i)) { + case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK: + break; + case PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK: + gaudi2_print_pcie_mstr_rr_mstr_if_razwi_info(hdev); + break; + } + } } static void gaudi2_handle_pif_fatal(struct hl_device *hdev, u64 intr_cause_data) @@ -8158,10 +8357,17 @@ static bool gaudi2_handle_hbm_mc_sei_err(struct hl_device *hdev, u16 event_type, return true; } - dev_err_ratelimited(hdev->dev, - "System Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Critical(%u). Error cause: %s\n", - hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, - sei_data->hdr.is_critical, hbm_mc_sei_cause[cause_idx]); + if (sei_data->hdr.is_critical) + dev_err(hdev->dev, + "System Critical Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s\n", + hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, + hbm_mc_sei_cause[cause_idx]); + + else + dev_err_ratelimited(hdev->dev, + "System Non-Critical Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s\n", + hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, + hbm_mc_sei_cause[cause_idx]); /* Print error-specific info */ switch (cause_idx) { @@ -8371,6 +8577,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent struct gaudi2_device *gaudi2 = hdev->asic_specific; bool reset_required = false, skip_reset = false; int index, sbte_index; + u64 event_mask = 0; u16 event_type; ctl = le32_to_cpu(eq_entry->hdr.ctl); @@ -8392,6 +8599,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent fallthrough; case GAUDI2_EVENT_ROTATOR0_SERR ... GAUDI2_EVENT_ROTATOR1_DERR: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); break; @@ -8401,21 +8609,25 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent fallthrough; case GAUDI2_EVENT_NIC0_QM0 ... GAUDI2_EVENT_NIC11_QM1: gaudi2_handle_qman_err(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_ARC_AXI_ERROR_RESPONSE_0: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; gaudi2_handle_arc_farm_sei_err(hdev); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_AXI_ERR_RSP: gaudi2_handle_cpu_sei_err(hdev); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP: case GAUDI2_EVENT_PDMA_CH1_AXI_ERR_RSP: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; gaudi2_handle_qm_sei_err(hdev, event_type, &eq_entry->razwi_info); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE: @@ -8423,6 +8635,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; gaudi2_handle_rot_err(hdev, index, &eq_entry->razwi_with_intr_cause); gaudi2_handle_qm_sei_err(hdev, event_type, NULL); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_TPC0_AXI_ERR_RSP ... GAUDI2_EVENT_TPC24_AXI_ERR_RSP: @@ -8430,11 +8643,13 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent gaudi2_tpc_ack_interrupts(hdev, index, "AXI_ERR_RSP", &eq_entry->razwi_with_intr_cause); gaudi2_handle_qm_sei_err(hdev, event_type, NULL); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE ... GAUDI2_EVENT_DEC9_AXI_ERR_RSPONSE: index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; gaudi2_handle_dec_err(hdev, index, "AXI_ERR_RESPONSE", &eq_entry->razwi_info); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_TPC0_KERNEL_ERR: @@ -8465,6 +8680,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); gaudi2_tpc_ack_interrupts(hdev, index, "KRN_ERR", &eq_entry->razwi_with_intr_cause); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_DEC0_SPI: @@ -8480,6 +8696,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent index = (event_type - GAUDI2_EVENT_DEC0_SPI) / (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); gaudi2_handle_dec_err(hdev, index, "SPI", &eq_entry->razwi_info); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE: @@ -8492,6 +8709,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent gaudi2_handle_mme_err(hdev, index, "CTRL_AXI_ERROR_RESPONSE", &eq_entry->razwi_info); gaudi2_handle_qm_sei_err(hdev, event_type, NULL); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_MME0_QMAN_SW_ERROR: @@ -8502,6 +8720,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - GAUDI2_EVENT_MME0_QMAN_SW_ERROR); gaudi2_handle_mme_err(hdev, index, "QMAN_SW_ERROR", &eq_entry->razwi_info); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID: @@ -8512,22 +8731,27 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID); gaudi2_handle_mme_wap_err(hdev, index, &eq_entry->razwi_info); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_KDMA_CH0_AXI_ERR_RSP: case GAUDI2_EVENT_KDMA0_CORE: gaudi2_handle_kdma_core_event(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_HDMA2_CORE ... GAUDI2_EVENT_PDMA1_CORE: gaudi2_handle_dma_core_event(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_PCIE_ADDR_DEC_ERR: gaudi2_print_pcie_addr_dec_info(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_HMMU0_PAGE_FAULT_OR_WR_PERM ... GAUDI2_EVENT_HMMU12_SECURITY_ERROR: @@ -8536,25 +8760,30 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent case GAUDI2_EVENT_PMMU_AXI_ERR_RSP_0: gaudi2_handle_mmu_spi_sei_err(hdev, event_type); reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_HIF0_FATAL ... GAUDI2_EVENT_HIF12_FATAL: gaudi2_handle_hif_fatal(hdev, event_type, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PMMU_FATAL_0: gaudi2_handle_pif_fatal(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PSOC63_RAZWI_OR_PID_MIN_MAX_INTERRUPT: gaudi2_ack_psoc_razwi_event_handler(hdev); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE ... GAUDI2_EVENT_HBM5_MC1_SEI_NON_SEVERE: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; reset_required = true; @@ -8563,25 +8792,31 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent case GAUDI2_EVENT_HBM_CATTRIP_0 ... GAUDI2_EVENT_HBM_CATTRIP_5: gaudi2_handle_hbm_cattrip(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_HBM0_MC0_SPI ... GAUDI2_EVENT_HBM5_MC1_SPI: gaudi2_handle_hbm_mc_spi(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PCIE_DRAIN_COMPLETE: gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PSOC59_RPM_ERROR_OR_DRAIN: gaudi2_handle_psoc_drain(hdev, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_AXI_ECC: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_L2_RAM_ECC: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME0_SBTE4_AXI_ERR_RSP: case GAUDI2_EVENT_MME1_SBTE0_AXI_ERR_RSP ... GAUDI2_EVENT_MME1_SBTE4_AXI_ERR_RSP: @@ -8595,17 +8830,24 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent GAUDI2_EVENT_MME0_SBTE0_AXI_ERR_RSP); gaudi2_handle_mme_sbte_err(hdev, index, sbte_index, le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_VM0_ALARM_A ... GAUDI2_EVENT_VM3_ALARM_B: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PSOC_AXI_ERR_RSP: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; + break; case GAUDI2_EVENT_PSOC_PRSTN_FALL: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PCIE_APB_TIMEOUT: reset_flags |= HL_DRV_RESET_FW_FATAL_ERR; + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PCIE_FATAL_ERR: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_TPC0_BMON_SPMU: case GAUDI2_EVENT_TPC1_BMON_SPMU: @@ -8657,6 +8899,7 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent case GAUDI2_EVENT_DEC8_BMON_SPMU: case GAUDI2_EVENT_DEC9_BMON_SPMU: case GAUDI2_EVENT_ROTATOR0_BMON_SPMU ... GAUDI2_EVENT_SM3_BMON_SPMU: + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_CPU_FIX_POWER_ENV_S: @@ -8664,43 +8907,53 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_S: case GAUDI2_EVENT_CPU_FIX_THERMAL_ENV_E: gaudi2_print_clk_change_info(hdev, event_type); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_CPU_PKT_QUEUE_OUT_SYNC: gaudi2_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_PCIE_FLR_REQUESTED: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; /* Do nothing- FW will handle it */ break; case GAUDI2_EVENT_PCIE_P2P_MSIX: gaudi2_handle_pcie_p2p_msix(hdev); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE ... GAUDI2_EVENT_SM3_AXI_ERROR_RESPONSE: index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; skip_reset = !gaudi2_handle_sm_err(hdev, index); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; case GAUDI2_EVENT_PSOC_MME_PLL_LOCK_ERR ... GAUDI2_EVENT_DCORE2_HBM_PLL_LOCK_ERR: + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_CAUSE: dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", le64_to_cpu(eq_entry->data[0])); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_CPLD_SHUTDOWN_EVENT: dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", le64_to_cpu(eq_entry->data[0])); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_CPU_PKT_SANITY_FAILED: gaudi2_print_cpu_pkt_failure_info(hdev, &eq_entry->pkt_sync_err); + event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR; break; case GAUDI2_EVENT_ARC_DCCM_FULL: hl_arc_event_handle(hdev, &eq_entry->arc_data); + event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR; break; default: @@ -8716,15 +8969,22 @@ static void gaudi2_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_ent if (!gaudi2_irq_map_table[event_type].msg) hl_fw_unmask_irq(hdev, event_type); + if (event_mask) + hl_notifier_event_send_all(hdev, event_mask); + return; reset_device: if (hdev->hard_reset_on_fw_events) { hl_device_reset(hdev, reset_flags); + event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET; } else { if (!gaudi2_irq_map_table[event_type].msg) hl_fw_unmask_irq(hdev, event_type); } + + if (event_mask) + hl_notifier_event_send_all(hdev, event_mask); } static int gaudi2_memset_device_memory(struct hl_device *hdev, u64 addr, u64 size, u64 val) @@ -9090,19 +9350,17 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v } /* Create mapping on asic side */ - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M); hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV, ctx->asid, reserved_va_base, SZ_2M); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); if (rc) { dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); goto unreserve_va; } - hdev->asic_funcs->kdma_lock(hdev, 0); - /* Enable MMU on KDMA */ gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); @@ -9130,13 +9388,11 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID); - hdev->asic_funcs->kdma_unlock(hdev, 0); - - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M); hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR, ctx->asid, reserved_va_base, SZ_2M); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); unreserve_va: hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M); free_data_buffer: @@ -9189,11 +9445,11 @@ static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *c goto destroy_internal_cb_pool; } - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, HOST_SPACE_INTERNAL_CB_SZ); hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); if (rc) goto unreserve_internal_cb_pool; @@ -9218,11 +9474,11 @@ static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx * if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) return; - mutex_lock(&ctx->mmu_lock); + mutex_lock(&hdev->mmu_lock); hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR); - mutex_unlock(&ctx->mmu_lock); + mutex_unlock(&hdev->mmu_lock); gen_pool_destroy(hdev->internal_cb_pool); @@ -9336,7 +9592,7 @@ static u32 gaudi2_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) static u32 gaudi2_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, u32 size, bool eb) { - struct hl_cb *cb = (struct hl_cb *) data; + struct hl_cb *cb = data; struct packet_msg_short *pkt; u32 value, ctl, pkt_size = sizeof(*pkt); @@ -9429,7 +9685,7 @@ static u32 gaudi2_add_fence_pkt(struct packet_fence *pkt) static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_properties *prop) { - struct hl_cb *cb = (struct hl_cb *) prop->data; + struct hl_cb *cb = prop->data; void *buf = (void *) (uintptr_t) (cb->kernel_address); u64 monitor_base, fence_addr = 0; @@ -9481,7 +9737,7 @@ static u32 gaudi2_gen_wait_cb(struct hl_device *hdev, struct hl_gen_wait_propert static void gaudi2_reset_sob(struct hl_device *hdev, void *data) { - struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data; + struct hl_hw_sob *hw_sob = data; dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); @@ -9724,7 +9980,7 @@ static int gaudi2_get_mmu_base(struct hl_device *hdev, u64 mmu_id, u32 *mmu_base static void gaudi2_ack_mmu_error(struct hl_device *hdev, u64 mmu_id) { - bool is_pmmu = (mmu_id == HW_CAP_PMMU ? true : false); + bool is_pmmu = (mmu_id == HW_CAP_PMMU); struct gaudi2_device *gaudi2 = hdev->asic_specific; u32 mmu_base; @@ -9881,6 +10137,17 @@ static int gaudi2_get_monitor_dump(struct hl_device *hdev, void *data) return -EOPNOTSUPP; } +int gaudi2_send_device_activity(struct hl_device *hdev, bool open) +{ + struct gaudi2_device *gaudi2 = hdev->asic_specific; + + if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) || hdev->fw_major_version < 37) + return 0; + + /* TODO: add check for FW version using minor ver once it's known */ + return hl_fw_send_device_activity(hdev, open); +} + static const struct hl_asic_funcs gaudi2_funcs = { .early_init = gaudi2_early_init, .early_fini = gaudi2_early_fini, @@ -9927,11 +10194,9 @@ static const struct hl_asic_funcs gaudi2_funcs = { .send_heartbeat = gaudi2_send_heartbeat, .debug_coresight = gaudi2_debug_coresight, .is_device_idle = gaudi2_is_device_idle, - .non_hard_reset_late_init = gaudi2_non_hard_reset_late_init, + .compute_reset_late_init = gaudi2_compute_reset_late_init, .hw_queues_lock = gaudi2_hw_queues_lock, .hw_queues_unlock = gaudi2_hw_queues_unlock, - .kdma_lock = gaudi2_kdma_lock, - .kdma_unlock = gaudi2_kdma_unlock, .get_pci_id = gaudi2_get_pci_id, .get_eeprom_data = gaudi2_get_eeprom_data, .get_monitor_dump = gaudi2_get_monitor_dump, @@ -9978,6 +10243,8 @@ static const struct hl_asic_funcs gaudi2_funcs = { .mmu_get_real_page_size = gaudi2_mmu_get_real_page_size, .access_dev_mem = hl_access_dev_mem, .set_dram_bar_base = gaudi2_set_hbm_bar_base, + .set_engine_cores = gaudi2_set_engine_cores, + .send_device_activity = gaudi2_send_device_activity, }; void gaudi2_set_asic_funcs(struct hl_device *hdev) diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2P.h b/drivers/misc/habanalabs/gaudi2/gaudi2P.h index e4bc4009f05bf650d235479fec6f684d65cfac0c..a99c348bbf3907e953557c495a7eb8f0b0287802 100644 --- a/drivers/misc/habanalabs/gaudi2/gaudi2P.h +++ b/drivers/misc/habanalabs/gaudi2/gaudi2P.h @@ -15,7 +15,6 @@ #include "../include/gaudi2/gaudi2_packets.h" #include "../include/gaudi2/gaudi2_fw_if.h" #include "../include/gaudi2/gaudi2_async_events.h" -#include "../include/gaudi2/gaudi2_async_virt_events.h" #define GAUDI2_LINUX_FW_FILE "habanalabs/gaudi2/gaudi2-fit.itb" #define GAUDI2_BOOT_FIT_FILE "habanalabs/gaudi2/gaudi2-boot-fit.itb" @@ -140,9 +139,6 @@ #define VA_HOST_SPACE_HPAGE_START 0xFFF0800000000000ull #define VA_HOST_SPACE_HPAGE_END 0xFFF1000000000000ull /* 140TB */ -#define VA_HOST_SPACE_USER_MAPPED_CB_START 0xFFF1000000000000ull -#define VA_HOST_SPACE_USER_MAPPED_CB_END 0xFFF1000100000000ull /* 4GB */ - /* 140TB */ #define VA_HOST_SPACE_PAGE_SIZE (VA_HOST_SPACE_PAGE_END - VA_HOST_SPACE_PAGE_START) @@ -458,7 +454,6 @@ struct dup_block_ctx { * the user can map. * @lfsr_rand_seeds: array of MME ACC random seeds to set. * @hw_queues_lock: protects the H/W queues from concurrent access. - * @kdma_lock: protects the KDMA engine from concurrent access. * @scratchpad_kernel_address: general purpose PAGE_SIZE contiguous memory, * this memory region should be write-only. * currently used for HBW QMAN writes which is @@ -510,9 +505,6 @@ struct dup_block_ctx { * @flush_db_fifo: flag to force flush DB FIFO after a write. * @hbm_cfg: HBM subsystem settings * @hw_queues_lock_mutex: used by simulator instead of hw_queues_lock. - * @kdma_lock_mutex: used by simulator instead of kdma_lock. - * @use_deprecated_event_mappings: use old event mappings which are about to be - * deprecated */ struct gaudi2_device { int (*cpucp_info_get)(struct hl_device *hdev); @@ -521,7 +513,6 @@ struct gaudi2_device { int lfsr_rand_seeds[MME_NUM_OF_LFSR_SEEDS]; spinlock_t hw_queues_lock; - spinlock_t kdma_lock; void *scratchpad_kernel_address; dma_addr_t scratchpad_bus_address; @@ -562,5 +553,6 @@ void gaudi2_pb_print_security_errors(struct hl_device *hdev, u32 block_addr, u32 u32 offended_addr); int gaudi2_init_security(struct hl_device *hdev); void gaudi2_ack_protection_bits_errors(struct hl_device *hdev); +int gaudi2_send_device_activity(struct hl_device *hdev, bool open); #endif /* GAUDI2P_H_ */ diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2_masks.h b/drivers/misc/habanalabs/gaudi2/gaudi2_masks.h index eed16d642a5a43b051abcdcec4abd4fad6ee569b..e9ac87828221c82be2f1669aef9c361cbd22bcbc 100644 --- a/drivers/misc/habanalabs/gaudi2/gaudi2_masks.h +++ b/drivers/misc/habanalabs/gaudi2/gaudi2_masks.h @@ -51,12 +51,18 @@ (0x1F << PDMA0_QM_GLBL_CFG0_CP_EN_SHIFT) | \ (0x1 << PDMA0_QM_GLBL_CFG0_ARC_CQF_EN_SHIFT)) -#define PDMA1_QMAN_ENABLE \ +#define PDMA0_QMAN_ENABLE \ ((0x3 << PDMA0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \ (0x1F << PDMA0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \ (0x1F << PDMA0_QM_GLBL_CFG0_CP_EN_SHIFT) | \ (0x1 << PDMA0_QM_GLBL_CFG0_ARC_CQF_EN_SHIFT)) +#define PDMA1_QMAN_ENABLE \ + ((0x1 << PDMA0_QM_GLBL_CFG0_PQF_EN_SHIFT) | \ + (0x1F << PDMA0_QM_GLBL_CFG0_CQF_EN_SHIFT) | \ + (0x1F << PDMA0_QM_GLBL_CFG0_CP_EN_SHIFT) | \ + (0x1 << PDMA0_QM_GLBL_CFG0_ARC_CQF_EN_SHIFT)) + /* QM_IDLE_MASK is valid for all engines QM idle check */ #define QM_IDLE_MASK (DCORE0_EDMA0_QM_GLBL_STS0_PQF_IDLE_MASK | \ DCORE0_EDMA0_QM_GLBL_STS0_CQF_IDLE_MASK | \ @@ -138,4 +144,17 @@ #define DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_SHIFT 15 #define DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK 0x8000 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_ERR_INTR_SHIFT 0 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_ERR_INTR_MASK 0x1 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_SHIFT 1 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK 0x2 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_SHIFT 2 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK 0x4 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_ERR_INTR_MASK_SHIFT 3 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_ERR_INTR_MASK_MASK 0x8 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK_SHIFT 4 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_AXI_LBW_ERR_INTR_MASK_MASK 0x10 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK_SHIFT 5 +#define PCIE_WRAP_PCIE_IC_SEI_INTR_IND_BAD_ACCESS_INTR_MASK_MASK 0x20 + #endif /* GAUDI2_MASKS_H_ */ diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2_security.c b/drivers/misc/habanalabs/gaudi2/gaudi2_security.c index 89a06ff5ba3410d883372f8926607e145b3db61a..c6906fb142290c7d89b38c970c58a6505b87761d 100644 --- a/drivers/misc/habanalabs/gaudi2/gaudi2_security.c +++ b/drivers/misc/habanalabs/gaudi2/gaudi2_security.c @@ -2559,6 +2559,10 @@ static const u32 gaudi2_pb_pcie[] = { mmPCIE_WRAP_BASE, }; +static const u32 gaudi2_pb_pcie_unsecured_regs[] = { + mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0, +}; + static const u32 gaudi2_pb_thermal_sensor0[] = { mmDCORE0_XFT_BASE, mmDCORE0_TSTDVS_BASE, @@ -2583,9 +2587,9 @@ struct gaudi2_tpc_pb_data { }; static void gaudi2_config_tpcs_glbl_sec(struct hl_device *hdev, int dcore, int inst, u32 offset, - void *data) + struct iterate_module_ctx *ctx) { - struct gaudi2_tpc_pb_data *pb_data = (struct gaudi2_tpc_pb_data *)data; + struct gaudi2_tpc_pb_data *pb_data = ctx->data; hl_config_glbl_sec(hdev, gaudi2_pb_dcr0_tpc0, pb_data->glbl_sec, offset, pb_data->block_array_size); @@ -2660,15 +2664,14 @@ static int gaudi2_init_pb_tpc(struct hl_device *hdev) struct gaudi2_tpc_arc_pb_data { u32 unsecured_regs_arr_size; u32 arc_regs_arr_size; - int rc; }; static void gaudi2_config_tpcs_pb_ranges(struct hl_device *hdev, int dcore, int inst, u32 offset, - void *data) + struct iterate_module_ctx *ctx) { - struct gaudi2_tpc_arc_pb_data *pb_data = (struct gaudi2_tpc_arc_pb_data *)data; + struct gaudi2_tpc_arc_pb_data *pb_data = ctx->data; - pb_data->rc |= hl_init_pb_ranges(hdev, HL_PB_SHARED, HL_PB_NA, 1, + ctx->rc = hl_init_pb_ranges(hdev, HL_PB_SHARED, HL_PB_NA, 1, offset, gaudi2_pb_dcr0_tpc0_arc, pb_data->arc_regs_arr_size, gaudi2_pb_dcr0_tpc0_arc_unsecured_regs, @@ -2683,12 +2686,12 @@ static int gaudi2_init_pb_tpc_arc(struct hl_device *hdev) tpc_arc_pb_data.arc_regs_arr_size = ARRAY_SIZE(gaudi2_pb_dcr0_tpc0_arc); tpc_arc_pb_data.unsecured_regs_arr_size = ARRAY_SIZE(gaudi2_pb_dcr0_tpc0_arc_unsecured_regs); - tpc_arc_pb_data.rc = 0; + tpc_iter.fn = &gaudi2_config_tpcs_pb_ranges; tpc_iter.data = &tpc_arc_pb_data; gaudi2_iterate_tpcs(hdev, &tpc_iter); - return tpc_arc_pb_data.rc; + return tpc_iter.rc; } static int gaudi2_init_pb_sm_objs(struct hl_device *hdev) @@ -3419,7 +3422,8 @@ static int gaudi2_init_protection_bits(struct hl_device *hdev) rc |= hl_init_pb(hdev, HL_PB_SHARED, HL_PB_NA, HL_PB_SINGLE_INSTANCE, HL_PB_NA, gaudi2_pb_pcie, ARRAY_SIZE(gaudi2_pb_pcie), - NULL, HL_PB_NA); + gaudi2_pb_pcie_unsecured_regs, + ARRAY_SIZE(gaudi2_pb_pcie_unsecured_regs)); /* Thermal Sensor. * Skip when security is enabled in F/W, because the blocks are protected by privileged RR. @@ -3547,9 +3551,9 @@ struct gaudi2_ack_pb_tpc_data { }; static void gaudi2_ack_pb_tpc_config(struct hl_device *hdev, int dcore, int inst, u32 offset, - void *data) + struct iterate_module_ctx *ctx) { - struct gaudi2_ack_pb_tpc_data *pb_data = (struct gaudi2_ack_pb_tpc_data *)data; + struct gaudi2_ack_pb_tpc_data *pb_data = ctx->data; hl_ack_pb_single_dcore(hdev, offset, HL_PB_SINGLE_INSTANCE, HL_PB_NA, gaudi2_pb_dcr0_tpc0, pb_data->tpc_regs_array_size); diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index db4487c335823fee5c574c8616d5d97909a3ff68..5ef9e3ca97a69acb1dc704d7d7e4173d531cd5c4 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -916,26 +916,11 @@ int goya_late_init(struct hl_device *hdev) */ void goya_late_fini(struct hl_device *hdev) { - const struct hwmon_channel_info **channel_info_arr; struct goya_device *goya = hdev->asic_specific; - int i = 0; cancel_delayed_work_sync(&goya->goya_work->work_freq); - if (!hdev->hl_chip_info->info) - return; - - channel_info_arr = hdev->hl_chip_info->info; - - while (channel_info_arr[i]) { - kfree(channel_info_arr[i]->config); - kfree(channel_info_arr[i]); - i++; - } - - kfree(channel_info_arr); - - hdev->hl_chip_info->info = NULL; + hl_hwmon_release_resources(hdev); } static void goya_set_pci_memory_regions(struct hl_device *hdev) @@ -1040,6 +1025,7 @@ static int goya_sw_init(struct hl_device *hdev) hdev->asic_prop.supports_compute_reset = true; hdev->asic_prop.allow_inference_soft_reset = true; hdev->supports_wait_for_multi_cs = false; + hdev->supports_ctx_switch = true; hdev->asic_funcs->set_pci_memory_regions(hdev); @@ -4559,7 +4545,7 @@ static int goya_unmask_irq_arr(struct hl_device *hdev, u32 *irq_arr, return rc; } -static int goya_non_hard_reset_late_init(struct hl_device *hdev) +static int goya_compute_reset_late_init(struct hl_device *hdev) { /* * Unmask all IRQs since some could have been received @@ -5137,8 +5123,8 @@ int goya_cpucp_info_get(struct hl_device *hdev) return 0; } -static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, - u8 mask_len, struct seq_file *s) +static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len, + struct engines_data *e) { const char *fmt = "%-5d%-9s%#-14x%#-16x%#x\n"; const char *dma_fmt = "%-5d%-9s%#-14x%#x\n"; @@ -5149,9 +5135,9 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u64 offset; int i; - if (s) - seq_puts(s, "\nDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0\n" - "--- ------- ------------ -------------\n"); + if (e) + hl_engine_data_sprintf(e, "\nDMA is_idle QM_GLBL_STS0 DMA_CORE_STS0\n" + "--- ------- ------------ -------------\n"); offset = mmDMA_QM_1_GLBL_STS0 - mmDMA_QM_0_GLBL_STS0; @@ -5164,13 +5150,13 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GOYA_ENGINE_ID_DMA_0 + i, mask); - if (s) - seq_printf(s, dma_fmt, i, is_eng_idle ? "Y" : "N", + if (e) + hl_engine_data_sprintf(e, dma_fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, dma_core_sts0); } - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nTPC is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 CFG_STATUS\n" "--- ------- ------------ -------------- ----------\n"); @@ -5187,13 +5173,13 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GOYA_ENGINE_ID_TPC_0 + i, mask); - if (s) - seq_printf(s, fmt, i, is_eng_idle ? "Y" : "N", + if (e) + hl_engine_data_sprintf(e, fmt, i, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, tpc_cfg_sts); } - if (s) - seq_puts(s, + if (e) + hl_engine_data_sprintf(e, "\nMME is_idle QM_GLBL_STS0 CMDQ_GLBL_STS0 ARCH_STATUS\n" "--- ------- ------------ -------------- -----------\n"); @@ -5207,10 +5193,10 @@ static bool goya_is_device_idle(struct hl_device *hdev, u64 *mask_arr, if (mask && !is_eng_idle) set_bit(GOYA_ENGINE_ID_MME_0, mask); - if (s) { - seq_printf(s, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, + if (e) { + hl_engine_data_sprintf(e, fmt, 0, is_eng_idle ? "Y" : "N", qm_glbl_sts0, cmdq_glbl_sts0, mme_arch_sts); - seq_puts(s, "\n"); + hl_engine_data_sprintf(e, "\n"); } return is_idle; @@ -5434,6 +5420,11 @@ static int goya_scrub_device_dram(struct hl_device *hdev, u64 val) return -EOPNOTSUPP; } +static int goya_send_device_activity(struct hl_device *hdev, bool open) +{ + return 0; +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5478,11 +5469,9 @@ static const struct hl_asic_funcs goya_funcs = { .send_heartbeat = goya_send_heartbeat, .debug_coresight = goya_debug_coresight, .is_device_idle = goya_is_device_idle, - .non_hard_reset_late_init = goya_non_hard_reset_late_init, + .compute_reset_late_init = goya_compute_reset_late_init, .hw_queues_lock = goya_hw_queues_lock, .hw_queues_unlock = goya_hw_queues_unlock, - .kdma_lock = NULL, - .kdma_unlock = NULL, .get_pci_id = goya_get_pci_id, .get_eeprom_data = goya_get_eeprom_data, .get_monitor_dump = goya_get_monitor_dump, @@ -5528,6 +5517,7 @@ static const struct hl_asic_funcs goya_funcs = { .mmu_get_real_page_size = hl_mmu_get_real_page_size, .access_dev_mem = hl_access_dev_mem, .set_dram_bar_base = goya_set_ddr_bar_base, + .send_device_activity = goya_send_device_activity, }; /* diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index abf40e1c4965a67fea63584ea5f539c93acee6d9..baa5aa43b6f490acfffb6458e8061e39bc300106 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -629,6 +629,12 @@ enum pq_init_status { * CPUCP_PACKET_ENGINE_CORE_ASID_SET - * Packet to perform engine core ASID configuration * + * CPUCP_PACKET_SEC_ATTEST_GET - + * Get the attestaion data that is collected during various stages of the + * boot sequence. the attestation data is also hashed with some unique + * number (nonce) provided by the host to prevent replay attacks. + * public key and certificate also provided as part of the FW response. + * * CPUCP_PACKET_MONITOR_DUMP_GET - * Get monitors registers dump from the CpuCP kernel. * The CPU will put the registers dump in the a buffer allocated by the driver @@ -636,6 +642,10 @@ enum pq_init_status { * passes the max size it allows the CpuCP to write to the structure, to prevent * data corruption in case of mismatched driver/FW versions. * Relevant only to Gaudi. + * + * CPUCP_PACKET_ACTIVE_STATUS_SET - + * LKD sends FW indication whether device is free or in use, this indication is reported + * also to the BMC. */ enum cpucp_packet_id { @@ -687,10 +697,17 @@ enum cpucp_packet_id { CPUCP_PACKET_RESERVED, /* not used */ CPUCP_PACKET_ENGINE_CORE_ASID_SET, /* internal */ CPUCP_PACKET_RESERVED2, /* not used */ + CPUCP_PACKET_SEC_ATTEST_GET, /* internal */ CPUCP_PACKET_RESERVED3, /* not used */ CPUCP_PACKET_RESERVED4, /* not used */ - CPUCP_PACKET_RESERVED5, /* not used */ CPUCP_PACKET_MONITOR_DUMP_GET, /* debugfs */ + CPUCP_PACKET_RESERVED5, /* not used */ + CPUCP_PACKET_RESERVED6, /* not used */ + CPUCP_PACKET_RESERVED7, /* not used */ + CPUCP_PACKET_RESERVED8, /* not used */ + CPUCP_PACKET_RESERVED9, /* not used */ + CPUCP_PACKET_ACTIVE_STATUS_SET, /* internal */ + CPUCP_PACKET_ID_MAX /* must be last */ }; #define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5 @@ -783,6 +800,9 @@ struct cpucp_packet { * result cannot be used to hold general purpose data. */ __le32 status_mask; + + /* random, used once number, for security packets */ + __le32 nonce; }; /* For NIC requests */ @@ -813,10 +833,25 @@ enum cpucp_led_index { CPUCP_LED2_INDEX }; +/* + * enum cpucp_packet_rc - Error return code + * @cpucp_packet_success -> in case of success. + * @cpucp_packet_invalid -> this is to support Goya and Gaudi platform. + * @cpucp_packet_fault -> in case of processing error like failing to + * get device binding or semaphore etc. + * @cpucp_packet_invalid_pkt -> when cpucp packet is un-supported. This is + * supported Greco onwards. + * @cpucp_packet_invalid_params -> when checking parameter like length of buffer + * or attribute value etc. Supported Greco onwards. + * @cpucp_packet_rc_max -> It indicates size of enum so should be at last. + */ enum cpucp_packet_rc { cpucp_packet_success, cpucp_packet_invalid, - cpucp_packet_fault + cpucp_packet_fault, + cpucp_packet_invalid_pkt, + cpucp_packet_invalid_params, + cpucp_packet_rc_max }; /* @@ -1193,6 +1228,70 @@ enum cpu_reset_status { CPU_RST_STATUS_SOFT_RST_DONE = 1, }; +#define SEC_PCR_DATA_BUF_SZ 256 +#define SEC_PCR_QUOTE_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */ +#define SEC_SIGNATURE_BUF_SZ 255 /* (256 - 1) 1 byte used for size */ +#define SEC_PUB_DATA_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */ +#define SEC_CERTIFICATE_BUF_SZ 2046 /* (2048 - 2) 2 bytes used for size */ + +/* + * struct cpucp_sec_attest_info - attestation report of the boot + * @pcr_data: raw values of the PCR registers + * @pcr_num_reg: number of PCR registers in the pcr_data array + * @pcr_reg_len: length of each PCR register in the pcr_data array (bytes) + * @nonce: number only used once. random number provided by host. this also + * passed to the quote command as a qualifying data. + * @pcr_quote_len: length of the attestation quote data (bytes) + * @pcr_quote: attestation report data structure + * @quote_sig_len: length of the attestation report signature (bytes) + * @quote_sig: signature structure of the attestation report + * @pub_data_len: length of the public data (bytes) + * @public_data: public key for the signed attestation + * (outPublic + name + qualifiedName) + * @certificate_len: length of the certificate (bytes) + * @certificate: certificate for the attestation signing key + */ +struct cpucp_sec_attest_info { + __u8 pcr_data[SEC_PCR_DATA_BUF_SZ]; + __u8 pcr_num_reg; + __u8 pcr_reg_len; + __le16 pad0; + __le32 nonce; + __le16 pcr_quote_len; + __u8 pcr_quote[SEC_PCR_QUOTE_BUF_SZ]; + __u8 quote_sig_len; + __u8 quote_sig[SEC_SIGNATURE_BUF_SZ]; + __le16 pub_data_len; + __u8 public_data[SEC_PUB_DATA_BUF_SZ]; + __le16 certificate_len; + __u8 certificate[SEC_CERTIFICATE_BUF_SZ]; +}; + +/* + * struct cpucp_dev_info_signed - device information signed by a secured device + * @info: device information structure as defined above + * @nonce: number only used once. random number provided by host. this number is + * hashed and signed along with the device information. + * @info_sig_len: length of the attestation signature (bytes) + * @info_sig: signature of the info + nonce data. + * @pub_data_len: length of the public data (bytes) + * @public_data: public key info signed info data + * (outPublic + name + qualifiedName) + * @certificate_len: length of the certificate (bytes) + * @certificate: certificate for the signing key + */ +struct cpucp_dev_info_signed { + struct cpucp_info info; /* assumed to be 64bit aligned */ + __le32 nonce; + __le32 pad0; + __u8 info_sig_len; + __u8 info_sig[SEC_SIGNATURE_BUF_SZ]; + __le16 pub_data_len; + __u8 public_data[SEC_PUB_DATA_BUF_SZ]; + __le16 certificate_len; + __u8 certificate[SEC_CERTIFICATE_BUF_SZ]; +}; + /* * struct dcore_monitor_regs_data - DCORE monitor regs data. * the structure follows sync manager block layout. relevant only to Gaudi. diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index a3594119bc5169295f37ce00e44cda35061a1f6a..e0ea51cc74756c050343864b384cefd8a7436b6c 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -34,6 +34,7 @@ enum cpu_boot_err { CPU_BOOT_ERR_BINNING_FAIL = 19, CPU_BOOT_ERR_TPM_FAIL = 20, CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL = 21, + CPU_BOOT_ERR_EEPROM_FAIL = 22, CPU_BOOT_ERR_ENABLED = 31, CPU_BOOT_ERR_SCND_EN = 63, CPU_BOOT_ERR_LAST = 64 /* we have 2 registers of 32 bits */ @@ -115,6 +116,9 @@ enum cpu_boot_err { * CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL Failed to set threshold for tmperature * sensor. * + * CPU_BOOT_ERR_EEPROM_FAIL Failed reading EEPROM data. Defaults + * are used. + * * CPU_BOOT_ERR0_ENABLED Error registers enabled. * This is a main indication that the * running FW populates the error @@ -139,6 +143,7 @@ enum cpu_boot_err { #define CPU_BOOT_ERR0_BINNING_FAIL (1 << CPU_BOOT_ERR_BINNING_FAIL) #define CPU_BOOT_ERR0_TPM_FAIL (1 << CPU_BOOT_ERR_TPM_FAIL) #define CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL (1 << CPU_BOOT_ERR_TMP_THRESH_INIT_FAIL) +#define CPU_BOOT_ERR0_EEPROM_FAIL (1 << CPU_BOOT_ERR_EEPROM_FAIL) #define CPU_BOOT_ERR0_ENABLED (1 << CPU_BOOT_ERR_ENABLED) #define CPU_BOOT_ERR1_ENABLED (1 << CPU_BOOT_ERR_ENABLED) @@ -426,7 +431,9 @@ struct cpu_dyn_regs { __le32 gic_host_ints_irq; __le32 gic_host_soft_rst_irq; __le32 gic_rot_qm_irq_ctrl; - __le32 reserved1[22]; /* reserve for future use */ + __le32 cpu_rst_status; + __le32 eng_arc_irq_ctrl; + __le32 reserved1[20]; /* reserve for future use */ }; /* TODO: remove the desc magic after the code is updated to use message */ @@ -465,6 +472,26 @@ enum comms_msg_type { HL_COMMS_BINNING_CONF_TYPE = 3, }; +/* + * Binning information shared between LKD and FW + * @tpc_mask - TPC binning information + * @dec_mask - Decoder binning information + * @hbm_mask - HBM binning information + * @edma_mask - EDMA binning information + * @mme_mask_l - MME binning information lower 32 + * @mme_mask_h - MME binning information upper 32 + * @reserved - reserved field for 64 bit alignment + */ +struct lkd_fw_binning_info { + __le64 tpc_mask; + __le32 dec_mask; + __le32 hbm_mask; + __le32 edma_mask; + __le32 mme_mask_l; + __le32 mme_mask_h; + __le32 reserved; +}; + /* TODO: remove this struct after the code is updated to use message */ /* this is the comms descriptor header - meta data */ struct comms_desc_header { @@ -525,13 +552,7 @@ struct lkd_fw_comms_msg { struct { __u8 fw_cfg_skip; /* 1 - skip, 0 - don't skip */ }; - struct { - __le64 tpc_binning_conf; - __le32 dec_binning_conf; - __le32 hbm_binning_conf; - __le32 edma_binning_conf; - __le32 mme_redundancy_conf; /* use MME_REDUNDANT_COLUMN */ - }; + struct lkd_fw_binning_info binning_info; }; }; diff --git a/drivers/misc/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h b/drivers/misc/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h index d0e2c68a639fbd8a48f3ce77ca98de9f456afaed..6aa1b14124629e6fa27980e779fbfdc79f2494d8 100644 --- a/drivers/misc/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h +++ b/drivers/misc/habanalabs/include/gaudi2/asic_reg/gaudi2_regs.h @@ -132,6 +132,7 @@ #include "dcore0_mme_ctrl_lo_arch_tensor_a_regs.h" #include "dcore0_mme_ctrl_lo_arch_tensor_b_regs.h" #include "dcore0_mme_ctrl_lo_arch_tensor_cout_regs.h" +#include "pcie_wrap_special_regs.h" #include "pdma0_qm_masks.h" #include "pdma0_core_masks.h" @@ -239,6 +240,7 @@ #define SFT_IF_RTR_OFFSET (mmSFT0_HBW_RTR_IF1_RTR_H3_BASE - mmSFT0_HBW_RTR_IF0_RTR_H3_BASE) #define ARC_HALT_REQ_OFFSET (mmARC_FARM_ARC0_AUX_RUN_HALT_REQ - mmARC_FARM_ARC0_AUX_BASE) +#define ARC_HALT_ACK_OFFSET (mmARC_FARM_ARC0_AUX_RUN_HALT_ACK - mmARC_FARM_ARC0_AUX_BASE) #define ARC_REGION_CFG_OFFSET(region) \ (mmARC_FARM_ARC0_AUX_ARC_REGION_CFG_0 + (region * 4) - mmARC_FARM_ARC0_AUX_BASE) diff --git a/drivers/misc/habanalabs/include/gaudi2/asic_reg/pcie_wrap_special_regs.h b/drivers/misc/habanalabs/include/gaudi2/asic_reg/pcie_wrap_special_regs.h new file mode 100644 index 0000000000000000000000000000000000000000..46558e7a7f634eba3e363180c638b91a2cd93c79 --- /dev/null +++ b/drivers/misc/habanalabs/include/gaudi2/asic_reg/pcie_wrap_special_regs.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2016-2020 HabanaLabs, Ltd. + * All Rights Reserved. + * + */ + +/************************************ + ** This is an auto-generated file ** + ** DO NOT EDIT BELOW ** + ************************************/ + +#ifndef ASIC_REG_PCIE_WRAP_SPECIAL_REGS_H_ +#define ASIC_REG_PCIE_WRAP_SPECIAL_REGS_H_ + +/* + ***************************************** + * PCIE_WRAP_SPECIAL + * (Prototype: SPECIAL_REGS) + ***************************************** + */ + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_0 0x4C01E80 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_1 0x4C01E84 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_2 0x4C01E88 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_3 0x4C01E8C + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_4 0x4C01E90 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_5 0x4C01E94 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_6 0x4C01E98 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_7 0x4C01E9C + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_8 0x4C01EA0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_9 0x4C01EA4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_10 0x4C01EA8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_11 0x4C01EAC + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_12 0x4C01EB0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_13 0x4C01EB4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_14 0x4C01EB8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_15 0x4C01EBC + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_16 0x4C01EC0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_17 0x4C01EC4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_18 0x4C01EC8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_19 0x4C01ECC + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_20 0x4C01ED0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_21 0x4C01ED4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_22 0x4C01ED8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_23 0x4C01EDC + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_24 0x4C01EE0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_25 0x4C01EE4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_26 0x4C01EE8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_27 0x4C01EEC + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_28 0x4C01EF0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_29 0x4C01EF4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_30 0x4C01EF8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_PRIV_31 0x4C01EFC + +#define mmPCIE_WRAP_SPECIAL_MEM_GW_DATA 0x4C01F00 + +#define mmPCIE_WRAP_SPECIAL_MEM_GW_REQ 0x4C01F04 + +#define mmPCIE_WRAP_SPECIAL_MEM_NUMOF 0x4C01F0C + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_SEL 0x4C01F10 + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_CTL 0x4C01F14 + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_ERR_MASK 0x4C01F18 + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_GLBL_ERR_MASK 0x4C01F1C + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_ERR_STS 0x4C01F20 + +#define mmPCIE_WRAP_SPECIAL_MEM_ECC_ERR_ADDR 0x4C01F24 + +#define mmPCIE_WRAP_SPECIAL_MEM_RM 0x4C01F28 + +#define mmPCIE_WRAP_SPECIAL_GLBL_ERR_MASK 0x4C01F40 + +#define mmPCIE_WRAP_SPECIAL_GLBL_ERR_ADDR 0x4C01F44 + +#define mmPCIE_WRAP_SPECIAL_GLBL_ERR_CAUSE 0x4C01F48 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0 0x4C01F60 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SPARE_1 0x4C01F64 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SPARE_2 0x4C01F68 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SPARE_3 0x4C01F6C + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_0 0x4C01F80 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_1 0x4C01F84 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_2 0x4C01F88 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_3 0x4C01F8C + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_4 0x4C01F90 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_5 0x4C01F94 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_6 0x4C01F98 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_7 0x4C01F9C + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_8 0x4C01FA0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_9 0x4C01FA4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_10 0x4C01FA8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_11 0x4C01FAC + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_12 0x4C01FB0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_13 0x4C01FB4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_14 0x4C01FB8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_15 0x4C01FBC + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_16 0x4C01FC0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_17 0x4C01FC4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_18 0x4C01FC8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_19 0x4C01FCC + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_20 0x4C01FD0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_21 0x4C01FD4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_22 0x4C01FD8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_23 0x4C01FDC + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_24 0x4C01FE0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_25 0x4C01FE4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_26 0x4C01FE8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_27 0x4C01FEC + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_28 0x4C01FF0 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_29 0x4C01FF4 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_30 0x4C01FF8 + +#define mmPCIE_WRAP_SPECIAL_GLBL_SEC_31 0x4C01FFC + +#endif /* ASIC_REG_PCIE_WRAP_SPECIAL_REGS_H_ */ diff --git a/drivers/misc/habanalabs/include/gaudi2/gaudi2_async_virt_events.h b/drivers/misc/habanalabs/include/gaudi2/gaudi2_async_virt_events.h deleted file mode 100644 index 6d6ed7838a6419d7a8039d4ff4eeb5756a29c930..0000000000000000000000000000000000000000 --- a/drivers/misc/habanalabs/include/gaudi2/gaudi2_async_virt_events.h +++ /dev/null @@ -1,57 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * Copyright 2022 HabanaLabs, Ltd. - * All Rights Reserved. - * - */ - -#ifndef __GAUDI2_ASYNC_VIRT_EVENTS_H_ -#define __GAUDI2_ASYNC_VIRT_EVENTS_H_ - -enum gaudi2_async_virt_event_id { - GAUDI2_EVENT_NIC3_QM1_OLD = 1206, - GAUDI2_EVENT_NIC4_QM0_OLD = 1207, - GAUDI2_EVENT_NIC4_QM1_OLD = 1208, - GAUDI2_EVENT_NIC5_QM0_OLD = 1209, - GAUDI2_EVENT_NIC5_QM1_OLD = 1210, - GAUDI2_EVENT_NIC6_QM0_OLD = 1211, - GAUDI2_EVENT_NIC6_QM1_OLD = 1212, - GAUDI2_EVENT_NIC7_QM0_OLD = 1213, - GAUDI2_EVENT_NIC7_QM1_OLD = 1214, - GAUDI2_EVENT_NIC8_QM0_OLD = 1215, - GAUDI2_EVENT_NIC8_QM1_OLD = 1216, - GAUDI2_EVENT_NIC9_QM0_OLD = 1217, - GAUDI2_EVENT_NIC9_QM1_OLD = 1218, - GAUDI2_EVENT_NIC10_QM0_OLD = 1219, - GAUDI2_EVENT_NIC10_QM1_OLD = 1220, - GAUDI2_EVENT_NIC11_QM0_OLD = 1221, - GAUDI2_EVENT_NIC11_QM1_OLD = 1222, - GAUDI2_EVENT_CPU_PKT_SANITY_FAILED_OLD = 1223, - GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG0_OLD = 1224, - GAUDI2_EVENT_CPU0_STATUS_NIC0_ENG1_OLD = 1225, - GAUDI2_EVENT_CPU1_STATUS_NIC1_ENG0_OLD = 1226, - GAUDI2_EVENT_CPU1_STATUS_NIC1_ENG1_OLD = 1227, - GAUDI2_EVENT_CPU2_STATUS_NIC2_ENG0_OLD = 1228, - GAUDI2_EVENT_CPU2_STATUS_NIC2_ENG1_OLD = 1229, - GAUDI2_EVENT_CPU3_STATUS_NIC3_ENG0_OLD = 1230, - GAUDI2_EVENT_CPU3_STATUS_NIC3_ENG1_OLD = 1231, - GAUDI2_EVENT_CPU4_STATUS_NIC4_ENG0_OLD = 1232, - GAUDI2_EVENT_CPU4_STATUS_NIC4_ENG1_OLD = 1233, - GAUDI2_EVENT_CPU5_STATUS_NIC5_ENG0_OLD = 1234, - GAUDI2_EVENT_CPU5_STATUS_NIC5_ENG1_OLD = 1235, - GAUDI2_EVENT_CPU6_STATUS_NIC6_ENG0_OLD = 1236, - GAUDI2_EVENT_CPU6_STATUS_NIC6_ENG1_OLD = 1237, - GAUDI2_EVENT_CPU7_STATUS_NIC7_ENG0_OLD = 1238, - GAUDI2_EVENT_CPU7_STATUS_NIC7_ENG1_OLD = 1239, - GAUDI2_EVENT_CPU8_STATUS_NIC8_ENG0_OLD = 1240, - GAUDI2_EVENT_CPU8_STATUS_NIC8_ENG1_OLD = 1241, - GAUDI2_EVENT_CPU9_STATUS_NIC9_ENG0_OLD = 1242, - GAUDI2_EVENT_CPU9_STATUS_NIC9_ENG1_OLD = 1243, - GAUDI2_EVENT_CPU10_STATUS_NIC10_ENG0_OLD = 1244, - GAUDI2_EVENT_CPU10_STATUS_NIC10_ENG1_OLD = 1245, - GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG0_OLD = 1246, - GAUDI2_EVENT_CPU11_STATUS_NIC11_ENG1_OLD = 1247, - GAUDI2_EVENT_ARC_DCCM_FULL_OLD = 1248, -}; - -#endif /* __GAUDI2_ASYNC_VIRT_EVENTS_H_ */ diff --git a/drivers/misc/hmc6352.c b/drivers/misc/hmc6352.c index 572a2ff10f0085189aeb30cc1863b476e0b621ed..42b9adef28a3c092cb9587aa120d57d23179578e 100644 --- a/drivers/misc/hmc6352.c +++ b/drivers/misc/hmc6352.c @@ -116,10 +116,9 @@ static int hmc6352_probe(struct i2c_client *client, return 0; } -static int hmc6352_remove(struct i2c_client *client) +static void hmc6352_remove(struct i2c_client *client) { sysfs_remove_group(&client->dev.kobj, &m_compass_gr); - return 0; } static const struct i2c_device_id hmc6352_id[] = { diff --git a/drivers/misc/ibmvmc.c b/drivers/misc/ibmvmc.c index c0fe3295c3303fd00ee7b26d54a2bfd6ba80b849..cbaf6d35e854e7699ed37c3f433da9afdd55a8f7 100644 --- a/drivers/misc/ibmvmc.c +++ b/drivers/misc/ibmvmc.c @@ -1039,6 +1039,7 @@ static unsigned int ibmvmc_poll(struct file *file, poll_table *wait) static ssize_t ibmvmc_write(struct file *file, const char *buffer, size_t count, loff_t *ppos) { + struct inode *inode; struct ibmvmc_buffer *vmc_buffer; struct ibmvmc_file_session *session; struct crq_server_adapter *adapter; @@ -1122,8 +1123,9 @@ static ssize_t ibmvmc_write(struct file *file, const char *buffer, if (p == buffer) goto out; - file->f_path.dentry->d_inode->i_mtime = current_time(file_inode(file)); - mark_inode_dirty(file->f_path.dentry->d_inode); + inode = file_inode(file); + inode->i_mtime = current_time(inode); + mark_inode_dirty(inode); dev_dbg(adapter->dev, "write: file = 0x%lx, count = 0x%lx\n", (unsigned long)file, (unsigned long)count); diff --git a/drivers/misc/ics932s401.c b/drivers/misc/ics932s401.c index 0f9ea75b0b18976d0807e05c95c342639e2bf55d..1cb71df966a4ca4444127fd3e74a124b188bc8aa 100644 --- a/drivers/misc/ics932s401.c +++ b/drivers/misc/ics932s401.c @@ -93,7 +93,7 @@ static int ics932s401_probe(struct i2c_client *client, const struct i2c_device_id *id); static int ics932s401_detect(struct i2c_client *client, struct i2c_board_info *info); -static int ics932s401_remove(struct i2c_client *client); +static void ics932s401_remove(struct i2c_client *client); static const struct i2c_device_id ics932s401_id[] = { { "ics932s401", 0 }, @@ -424,7 +424,7 @@ static int ics932s401_detect(struct i2c_client *client, if (revision != ICS932S401_REV) dev_info(&adapter->dev, "Unknown revision %d\n", revision); - strlcpy(info->type, "ics932s401", I2C_NAME_SIZE); + strscpy(info->type, "ics932s401", I2C_NAME_SIZE); return 0; } @@ -460,13 +460,12 @@ exit: return err; } -static int ics932s401_remove(struct i2c_client *client) +static void ics932s401_remove(struct i2c_client *client) { struct ics932s401_data *data = i2c_get_clientdata(client); sysfs_remove_group(&client->dev.kobj, &data->attrs); kfree(data); - return 0; } module_i2c_driver(ics932s401_driver); diff --git a/drivers/misc/isl29003.c b/drivers/misc/isl29003.c index 703d20e83ebd7dd3af3c701a426f300f4b247bca..8ab61be79c762939a11bc5991325fcb786aa1aac 100644 --- a/drivers/misc/isl29003.c +++ b/drivers/misc/isl29003.c @@ -410,12 +410,11 @@ exit_kfree: return err; } -static int isl29003_remove(struct i2c_client *client) +static void isl29003_remove(struct i2c_client *client) { sysfs_remove_group(&client->dev.kobj, &isl29003_attr_group); isl29003_set_power_state(client, 0); kfree(i2c_get_clientdata(client)); - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/misc/isl29020.c b/drivers/misc/isl29020.c index fc5ff2805b944ac5a1771be334b7a5a0607f2363..c6f2a94f501acc58380ed51fd57bf90df8b1ffa2 100644 --- a/drivers/misc/isl29020.c +++ b/drivers/misc/isl29020.c @@ -171,11 +171,10 @@ static int isl29020_probe(struct i2c_client *client, return res; } -static int isl29020_remove(struct i2c_client *client) +static void isl29020_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); sysfs_remove_group(&client->dev.kobj, &m_als_gr); - return 0; } static const struct i2c_device_id isl29020_id[] = { diff --git a/drivers/misc/lis3lv02d/lis3lv02d_i2c.c b/drivers/misc/lis3lv02d/lis3lv02d_i2c.c index 52555d2e824bd9cbe033c17bf54530d1a4e1d45a..d7daa01fe7ca957b40ad7911b52cc1d370ef2dbd 100644 --- a/drivers/misc/lis3lv02d/lis3lv02d_i2c.c +++ b/drivers/misc/lis3lv02d/lis3lv02d_i2c.c @@ -177,7 +177,7 @@ fail: return ret; } -static int lis3lv02d_i2c_remove(struct i2c_client *client) +static void lis3lv02d_i2c_remove(struct i2c_client *client) { struct lis3lv02d *lis3 = i2c_get_clientdata(client); struct lis3lv02d_platform_data *pdata = client->dev.platform_data; @@ -190,7 +190,6 @@ static int lis3lv02d_i2c_remove(struct i2c_client *client) regulator_bulk_free(ARRAY_SIZE(lis3->regulators), lis3_dev.regulators); - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c index 71483cb1e422aa583ddf0cf10e811519f2a9c44e..5245cf6013c95560af1434bc04dd312cffa71075 100644 --- a/drivers/misc/lkdtm/cfi.c +++ b/drivers/misc/lkdtm/cfi.c @@ -20,6 +20,13 @@ static noinline int lkdtm_increment_int(int *counter) return *counter; } + +/* Don't allow the compiler to inline the calls. */ +static noinline void lkdtm_indirect_call(void (*func)(int *)) +{ + func(&called_count); +} + /* * This tries to call an indirect function with a mismatched prototype. */ @@ -29,15 +36,11 @@ static void lkdtm_CFI_FORWARD_PROTO(void) * Matches lkdtm_increment_void()'s prototype, but not * lkdtm_increment_int()'s prototype. */ - void (*func)(int *); - pr_info("Calling matched prototype ...\n"); - func = lkdtm_increment_void; - func(&called_count); + lkdtm_indirect_call(lkdtm_increment_void); pr_info("Calling mismatched prototype ...\n"); - func = (void *)lkdtm_increment_int; - func(&called_count); + lkdtm_indirect_call((void *)lkdtm_increment_int); pr_err("FAIL: survived mismatched prototype function call!\n"); pr_expected_config(CONFIG_CFI_CLANG); diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c index 080293fa3c521a71ff334b6be612daf53052962b..0159276656780d560cd9f9dffb9ef9f152950641 100644 --- a/drivers/misc/lkdtm/fortify.c +++ b/drivers/misc/lkdtm/fortify.c @@ -10,28 +10,31 @@ static volatile int fortify_scratch_space; -static void lkdtm_FORTIFIED_OBJECT(void) +static void lkdtm_FORTIFY_STR_OBJECT(void) { struct target { char a[10]; - } target[2] = {}; + int foo; + } target[3] = {}; /* * Using volatile prevents the compiler from determining the value of * 'size' at compile time. Without that, we would get a compile error * rather than a runtime error. */ - volatile int size = 11; + volatile int size = 20; + + pr_info("trying to strcmp() past the end of a struct\n"); - pr_info("trying to read past the end of a struct\n"); + strncpy(target[0].a, target[1].a, size); /* Store result to global to prevent the code from being eliminated */ - fortify_scratch_space = memcmp(&target[0], &target[1], size); + fortify_scratch_space = target[0].a[3]; - pr_err("FAIL: fortify did not block an object overread!\n"); + pr_err("FAIL: fortify did not block a strncpy() object write overflow!\n"); pr_expected_config(CONFIG_FORTIFY_SOURCE); } -static void lkdtm_FORTIFIED_SUBOBJECT(void) +static void lkdtm_FORTIFY_STR_MEMBER(void) { struct target { char a[10]; @@ -44,7 +47,7 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void) strscpy(src, "over ten bytes", size); size = strlen(src) + 1; - pr_info("trying to strncpy past the end of a member of a struct\n"); + pr_info("trying to strncpy() past the end of a struct member...\n"); /* * strncpy(target.a, src, 20); will hit a compile error because the @@ -56,7 +59,72 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void) /* Store result to global to prevent the code from being eliminated */ fortify_scratch_space = target.a[3]; - pr_err("FAIL: fortify did not block an sub-object overrun!\n"); + pr_err("FAIL: fortify did not block a strncpy() struct member write overflow!\n"); + pr_expected_config(CONFIG_FORTIFY_SOURCE); + + kfree(src); +} + +static void lkdtm_FORTIFY_MEM_OBJECT(void) +{ + int before[10]; + struct target { + char a[10]; + int foo; + } target = {}; + int after[10]; + /* + * Using volatile prevents the compiler from determining the value of + * 'size' at compile time. Without that, we would get a compile error + * rather than a runtime error. + */ + volatile int size = 20; + + memset(before, 0, sizeof(before)); + memset(after, 0, sizeof(after)); + fortify_scratch_space = before[5]; + fortify_scratch_space = after[5]; + + pr_info("trying to memcpy() past the end of a struct\n"); + + pr_info("0: %zu\n", __builtin_object_size(&target, 0)); + pr_info("1: %zu\n", __builtin_object_size(&target, 1)); + pr_info("s: %d\n", size); + memcpy(&target, &before, size); + + /* Store result to global to prevent the code from being eliminated */ + fortify_scratch_space = target.a[3]; + + pr_err("FAIL: fortify did not block a memcpy() object write overflow!\n"); + pr_expected_config(CONFIG_FORTIFY_SOURCE); +} + +static void lkdtm_FORTIFY_MEM_MEMBER(void) +{ + struct target { + char a[10]; + char b[10]; + } target; + volatile int size = 20; + char *src; + + src = kmalloc(size, GFP_KERNEL); + strscpy(src, "over ten bytes", size); + size = strlen(src) + 1; + + pr_info("trying to memcpy() past the end of a struct member...\n"); + + /* + * strncpy(target.a, src, 20); will hit a compile error because the + * compiler knows at build time that target.a < 20 bytes. Use a + * volatile to force a runtime error. + */ + memcpy(target.a, src, size); + + /* Store result to global to prevent the code from being eliminated */ + fortify_scratch_space = target.a[3]; + + pr_err("FAIL: fortify did not block a memcpy() struct member write overflow!\n"); pr_expected_config(CONFIG_FORTIFY_SOURCE); kfree(src); @@ -67,7 +135,7 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void) * strscpy and generate a panic because there is a write overflow (i.e. src * length is greater than dst length). */ -static void lkdtm_FORTIFIED_STRSCPY(void) +static void lkdtm_FORTIFY_STRSCPY(void) { char *src; char dst[5]; @@ -136,9 +204,11 @@ static void lkdtm_FORTIFIED_STRSCPY(void) } static struct crashtype crashtypes[] = { - CRASHTYPE(FORTIFIED_OBJECT), - CRASHTYPE(FORTIFIED_SUBOBJECT), - CRASHTYPE(FORTIFIED_STRSCPY), + CRASHTYPE(FORTIFY_STR_OBJECT), + CRASHTYPE(FORTIFY_STR_MEMBER), + CRASHTYPE(FORTIFY_MEM_OBJECT), + CRASHTYPE(FORTIFY_MEM_MEMBER), + CRASHTYPE(FORTIFY_STRSCPY), }; struct crashtype_category fortify_crashtypes = { diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c index 6215ec995cd3cf6d7a6278186ed02151fc7716fc..67db57249a3476ed2bcf93fab0ab4d4776a80a6a 100644 --- a/drivers/misc/lkdtm/usercopy.c +++ b/drivers/misc/lkdtm/usercopy.c @@ -330,7 +330,7 @@ static void lkdtm_USERCOPY_KERNEL(void) pr_info("attempting bad copy_to_user from kernel text: %px\n", vm_mmap); - if (copy_to_user((void __user *)user_addr, function_nocfi(vm_mmap), + if (copy_to_user((void __user *)user_addr, vm_mmap, unconst + PAGE_SIZE)) { pr_warn("copy_to_user failed, but lacked Oops\n"); goto free_user; diff --git a/drivers/misc/mchp_pci1xxxx/Kconfig b/drivers/misc/mchp_pci1xxxx/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..4abb47de7219de5ac6277bc9fb9eeb2bc6782858 --- /dev/null +++ b/drivers/misc/mchp_pci1xxxx/Kconfig @@ -0,0 +1,13 @@ +config GP_PCI1XXXX + tristate "Microchip PCI1XXXX PCIe to GPIO Expander + OTP/EEPROM manager" + depends on PCI + depends on GPIOLIB + select GPIOLIB_IRQCHIP + select AUXILIARY_BUS + help + PCI1XXXX is a PCIe GEN 3 switch with one of the endpoints having + multiple functions and one of the functions is a GPIO controller + which also has registers to interface with the OTP and EEPROM. + Select yes, no or module here to include or exclude the driver + for the GPIO function. + diff --git a/drivers/misc/mchp_pci1xxxx/Makefile b/drivers/misc/mchp_pci1xxxx/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fc4615cfe28bef80be494e7829a969e8afb534cb --- /dev/null +++ b/drivers/misc/mchp_pci1xxxx/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_GP_PCI1XXXX) := mchp_pci1xxxx_gp.o mchp_pci1xxxx_gpio.o diff --git a/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.c b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.c new file mode 100644 index 0000000000000000000000000000000000000000..32af2b14ff3448500f97a948fe45cd2006c0634a --- /dev/null +++ b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2022 Microchip Technology Inc. + +#include +#include +#include +#include +#include +#include +#include +#include +#include "mchp_pci1xxxx_gp.h" + +struct aux_bus_device { + struct auxiliary_device_wrapper *aux_device_wrapper[2]; +}; + +static DEFINE_IDA(gp_client_ida); +static const char aux_dev_otp_e2p_name[15] = "gp_otp_e2p"; +static const char aux_dev_gpio_name[15] = "gp_gpio"; + +static void gp_auxiliary_device_release(struct device *dev) +{ + struct auxiliary_device_wrapper *aux_device_wrapper = + (struct auxiliary_device_wrapper *)container_of(dev, + struct auxiliary_device_wrapper, aux_dev.dev); + + ida_free(&gp_client_ida, aux_device_wrapper->aux_dev.id); + kfree(aux_device_wrapper); +} + +static int gp_aux_bus_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct aux_bus_device *aux_bus; + int retval; + + retval = pcim_enable_device(pdev); + if (retval) + return retval; + + aux_bus = devm_kzalloc(&pdev->dev, sizeof(*aux_bus), GFP_KERNEL); + if (!aux_bus) + return -ENOMEM; + + aux_bus->aux_device_wrapper[0] = kzalloc(sizeof(*aux_bus->aux_device_wrapper[0]), + GFP_KERNEL); + if (!aux_bus->aux_device_wrapper[0]) + return -ENOMEM; + + retval = ida_alloc(&gp_client_ida, GFP_KERNEL); + if (retval < 0) + goto err_ida_alloc_0; + + aux_bus->aux_device_wrapper[0]->aux_dev.name = aux_dev_otp_e2p_name; + aux_bus->aux_device_wrapper[0]->aux_dev.dev.parent = &pdev->dev; + aux_bus->aux_device_wrapper[0]->aux_dev.dev.release = gp_auxiliary_device_release; + aux_bus->aux_device_wrapper[0]->aux_dev.id = retval; + + aux_bus->aux_device_wrapper[0]->gp_aux_data.region_start = pci_resource_start(pdev, 0); + aux_bus->aux_device_wrapper[0]->gp_aux_data.region_length = pci_resource_end(pdev, 0); + + retval = auxiliary_device_init(&aux_bus->aux_device_wrapper[0]->aux_dev); + if (retval < 0) + goto err_aux_dev_init_0; + + retval = auxiliary_device_add(&aux_bus->aux_device_wrapper[0]->aux_dev); + if (retval) + goto err_aux_dev_add_0; + + aux_bus->aux_device_wrapper[1] = kzalloc(sizeof(*aux_bus->aux_device_wrapper[1]), + GFP_KERNEL); + if (!aux_bus->aux_device_wrapper[1]) + return -ENOMEM; + + retval = ida_alloc(&gp_client_ida, GFP_KERNEL); + if (retval < 0) + goto err_ida_alloc_1; + + aux_bus->aux_device_wrapper[1]->aux_dev.name = aux_dev_gpio_name; + aux_bus->aux_device_wrapper[1]->aux_dev.dev.parent = &pdev->dev; + aux_bus->aux_device_wrapper[1]->aux_dev.dev.release = gp_auxiliary_device_release; + aux_bus->aux_device_wrapper[1]->aux_dev.id = retval; + + aux_bus->aux_device_wrapper[1]->gp_aux_data.region_start = pci_resource_start(pdev, 0); + aux_bus->aux_device_wrapper[1]->gp_aux_data.region_length = pci_resource_end(pdev, 0); + + retval = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + + if (retval < 0) + goto err_aux_dev_init_1; + + retval = pci_irq_vector(pdev, 0); + if (retval < 0) + goto err_aux_dev_init_1; + + pdev->irq = retval; + aux_bus->aux_device_wrapper[1]->gp_aux_data.irq_num = pdev->irq; + + retval = auxiliary_device_init(&aux_bus->aux_device_wrapper[1]->aux_dev); + if (retval < 0) + goto err_aux_dev_init_1; + + retval = auxiliary_device_add(&aux_bus->aux_device_wrapper[1]->aux_dev); + if (retval) + goto err_aux_dev_add_1; + + pci_set_drvdata(pdev, aux_bus); + pci_set_master(pdev); + + return 0; + +err_aux_dev_add_1: + auxiliary_device_uninit(&aux_bus->aux_device_wrapper[1]->aux_dev); + +err_aux_dev_init_1: + ida_free(&gp_client_ida, aux_bus->aux_device_wrapper[1]->aux_dev.id); + +err_ida_alloc_1: + kfree(aux_bus->aux_device_wrapper[1]); + +err_aux_dev_add_0: + auxiliary_device_uninit(&aux_bus->aux_device_wrapper[0]->aux_dev); + +err_aux_dev_init_0: + ida_free(&gp_client_ida, aux_bus->aux_device_wrapper[0]->aux_dev.id); + +err_ida_alloc_0: + kfree(aux_bus->aux_device_wrapper[0]); + + return retval; +} + +static void gp_aux_bus_remove(struct pci_dev *pdev) +{ + struct aux_bus_device *aux_bus = pci_get_drvdata(pdev); + + auxiliary_device_delete(&aux_bus->aux_device_wrapper[0]->aux_dev); + auxiliary_device_uninit(&aux_bus->aux_device_wrapper[0]->aux_dev); + auxiliary_device_delete(&aux_bus->aux_device_wrapper[1]->aux_dev); + auxiliary_device_uninit(&aux_bus->aux_device_wrapper[1]->aux_dev); +} + +static const struct pci_device_id pci1xxxx_tbl[] = { + { PCI_DEVICE(0x1055, 0xA005) }, + { PCI_DEVICE(0x1055, 0xA015) }, + { PCI_DEVICE(0x1055, 0xA025) }, + { PCI_DEVICE(0x1055, 0xA035) }, + { PCI_DEVICE(0x1055, 0xA045) }, + { PCI_DEVICE(0x1055, 0xA055) }, + {0,} +}; +MODULE_DEVICE_TABLE(pci, pci1xxxx_tbl); + +static struct pci_driver pci1xxxx_gp_driver = { + .name = "PCI1xxxxGP", + .id_table = pci1xxxx_tbl, + .probe = gp_aux_bus_probe, + .remove = gp_aux_bus_remove, +}; + +module_pci_driver(pci1xxxx_gp_driver); + +MODULE_DESCRIPTION("Microchip Technology Inc. PCI1xxxx GP expander"); +MODULE_AUTHOR("Kumaravel Thiagarajan "); +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.h b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.h new file mode 100644 index 0000000000000000000000000000000000000000..37eec73b20d784112296df9c1db9014ec8ee335e --- /dev/null +++ b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gp.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2022 Microchip Technology Inc. */ + +#ifndef _GPIO_PCI1XXXX_H +#define _GPIO_PCI1XXXX_H + +#include +#include +#include +#include +#include + +/* Perform operations like variable length write, read and write with read back for OTP / EEPROM + * Perform bit mode write in OTP + */ + +struct gp_aux_data_type { + int irq_num; + resource_size_t region_start; + resource_size_t region_length; +}; + +struct auxiliary_device_wrapper { + struct auxiliary_device aux_dev; + struct gp_aux_data_type gp_aux_data; +}; + +#endif diff --git a/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c new file mode 100644 index 0000000000000000000000000000000000000000..3389803cb281a7d598ab8dcb16e64373ba3da394 --- /dev/null +++ b/drivers/misc/mchp_pci1xxxx/mchp_pci1xxxx_gpio.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2022 Microchip Technology Inc. +// pci1xxxx gpio driver + +#include +#include +#include +#include +#include +#include +#include + +#include "mchp_pci1xxxx_gp.h" + +#define PCI1XXXX_NR_PINS 93 +#define PERI_GEN_RESET 0 +#define OUT_EN_OFFSET(x) ((((x) / 32) * 4) + 0x400) +#define INP_EN_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x10) +#define OUT_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x20) +#define INP_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x30) +#define PULLUP_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x40) +#define PULLDOWN_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x50) +#define OPENDRAIN_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x60) +#define WAKEMASK_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x70) +#define MODE_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x80) +#define INTR_LO_TO_HI_EDGE_CONFIG(x) ((((x) / 32) * 4) + 0x400 + 0x90) +#define INTR_HI_TO_LO_EDGE_CONFIG(x) ((((x) / 32) * 4) + 0x400 + 0xA0) +#define INTR_LEVEL_CONFIG_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0xB0) +#define INTR_LEVEL_MASK_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0xC0) +#define INTR_STAT_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0xD0) +#define DEBOUNCE_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0xE0) +#define PIO_GLOBAL_CONFIG_OFFSET (0x400 + 0xF0) +#define PIO_PCI_CTRL_REG_OFFSET (0x400 + 0xF4) +#define INTR_MASK_OFFSET(x) ((((x) / 32) * 4) + 0x400 + 0x100) +#define INTR_STATUS_OFFSET(x) (((x) * 4) + 0x400 + 0xD0) + +struct pci1xxxx_gpio { + struct auxiliary_device *aux_dev; + void __iomem *reg_base; + struct gpio_chip gpio; + spinlock_t lock; + int irq_base; +}; + +static int pci1xxxx_gpio_get_direction(struct gpio_chip *gpio, unsigned int nr) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + u32 data; + int ret = -EINVAL; + + data = readl(priv->reg_base + INP_EN_OFFSET(nr)); + if (data & BIT(nr % 32)) { + ret = 1; + } else { + data = readl(priv->reg_base + OUT_EN_OFFSET(nr)); + if (data & BIT(nr % 32)) + ret = 0; + } + + return ret; +} + +static inline void pci1xxx_assign_bit(void __iomem *base_addr, unsigned int reg_offset, + unsigned int bitpos, bool set) +{ + u32 data; + + data = readl(base_addr + reg_offset); + if (set) + data |= BIT(bitpos); + else + data &= ~BIT(bitpos); + writel(data, base_addr + reg_offset); +} + +static int pci1xxxx_gpio_direction_input(struct gpio_chip *gpio, unsigned int nr) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, INP_EN_OFFSET(nr), (nr % 32), true); + pci1xxx_assign_bit(priv->reg_base, OUT_EN_OFFSET(nr), (nr % 32), false); + spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int pci1xxxx_gpio_get(struct gpio_chip *gpio, unsigned int nr) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + + return (readl(priv->reg_base + INP_OFFSET(nr)) >> (nr % 32)) & 1; +} + +static int pci1xxxx_gpio_direction_output(struct gpio_chip *gpio, + unsigned int nr, int val) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + unsigned long flags; + u32 data; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, INP_EN_OFFSET(nr), (nr % 32), false); + pci1xxx_assign_bit(priv->reg_base, OUT_EN_OFFSET(nr), (nr % 32), true); + data = readl(priv->reg_base + OUT_OFFSET(nr)); + if (val) + data |= (1 << (nr % 32)); + else + data &= ~(1 << (nr % 32)); + writel(data, priv->reg_base + OUT_OFFSET(nr)); + spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static void pci1xxxx_gpio_set(struct gpio_chip *gpio, + unsigned int nr, int val) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, OUT_OFFSET(nr), (nr % 32), val); + spin_unlock_irqrestore(&priv->lock, flags); +} + +static int pci1xxxx_gpio_set_config(struct gpio_chip *gpio, unsigned int offset, + unsigned long config) +{ + struct pci1xxxx_gpio *priv = gpiochip_get_data(gpio); + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&priv->lock, flags); + switch (pinconf_to_config_param(config)) { + case PIN_CONFIG_BIAS_PULL_UP: + pci1xxx_assign_bit(priv->reg_base, PULLUP_OFFSET(offset), (offset % 32), true); + break; + case PIN_CONFIG_BIAS_PULL_DOWN: + pci1xxx_assign_bit(priv->reg_base, PULLDOWN_OFFSET(offset), (offset % 32), true); + break; + case PIN_CONFIG_BIAS_DISABLE: + pci1xxx_assign_bit(priv->reg_base, PULLUP_OFFSET(offset), (offset % 32), false); + pci1xxx_assign_bit(priv->reg_base, PULLDOWN_OFFSET(offset), (offset % 32), false); + break; + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + pci1xxx_assign_bit(priv->reg_base, OPENDRAIN_OFFSET(offset), (offset % 32), true); + break; + default: + ret = -EOPNOTSUPP; + break; + } + spin_unlock_irqrestore(&priv->lock, flags); + + return ret; +} + +static void pci1xxxx_gpio_irq_ack(struct irq_data *data) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(data); + struct pci1xxxx_gpio *priv = gpiochip_get_data(chip); + unsigned int gpio = irqd_to_hwirq(data); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, INTR_STAT_OFFSET(gpio), (gpio % 32), true); + spin_unlock_irqrestore(&priv->lock, flags); +} + +static void pci1xxxx_gpio_irq_set_mask(struct irq_data *data, bool set) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(data); + struct pci1xxxx_gpio *priv = gpiochip_get_data(chip); + unsigned int gpio = irqd_to_hwirq(data); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, INTR_MASK_OFFSET(gpio), (gpio % 32), set); + spin_unlock_irqrestore(&priv->lock, flags); +} + +static void pci1xxxx_gpio_irq_mask(struct irq_data *data) +{ + pci1xxxx_gpio_irq_set_mask(data, true); +} + +static void pci1xxxx_gpio_irq_unmask(struct irq_data *data) +{ + pci1xxxx_gpio_irq_set_mask(data, false); +} + +static int pci1xxxx_gpio_set_type(struct irq_data *data, unsigned int trigger_type) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(data); + struct pci1xxxx_gpio *priv = gpiochip_get_data(chip); + unsigned int gpio = irqd_to_hwirq(data); + unsigned int bitpos = gpio % 32; + + if (trigger_type & IRQ_TYPE_EDGE_FALLING) { + pci1xxx_assign_bit(priv->reg_base, INTR_HI_TO_LO_EDGE_CONFIG(gpio), + bitpos, false); + pci1xxx_assign_bit(priv->reg_base, MODE_OFFSET(gpio), + bitpos, false); + irq_set_handler_locked(data, handle_edge_irq); + } else { + pci1xxx_assign_bit(priv->reg_base, INTR_HI_TO_LO_EDGE_CONFIG(gpio), + bitpos, true); + } + + if (trigger_type & IRQ_TYPE_EDGE_RISING) { + pci1xxx_assign_bit(priv->reg_base, INTR_LO_TO_HI_EDGE_CONFIG(gpio), + bitpos, false); + pci1xxx_assign_bit(priv->reg_base, MODE_OFFSET(gpio), bitpos, + false); + irq_set_handler_locked(data, handle_edge_irq); + } else { + pci1xxx_assign_bit(priv->reg_base, INTR_LO_TO_HI_EDGE_CONFIG(gpio), + bitpos, true); + } + + if (trigger_type & IRQ_TYPE_LEVEL_LOW) { + pci1xxx_assign_bit(priv->reg_base, INTR_LEVEL_CONFIG_OFFSET(gpio), + bitpos, true); + pci1xxx_assign_bit(priv->reg_base, INTR_LEVEL_MASK_OFFSET(gpio), + bitpos, false); + pci1xxx_assign_bit(priv->reg_base, MODE_OFFSET(gpio), bitpos, + true); + irq_set_handler_locked(data, handle_edge_irq); + } + + if (trigger_type & IRQ_TYPE_LEVEL_HIGH) { + pci1xxx_assign_bit(priv->reg_base, INTR_LEVEL_CONFIG_OFFSET(gpio), + bitpos, false); + pci1xxx_assign_bit(priv->reg_base, INTR_LEVEL_MASK_OFFSET(gpio), + bitpos, false); + pci1xxx_assign_bit(priv->reg_base, MODE_OFFSET(gpio), bitpos, + true); + irq_set_handler_locked(data, handle_edge_irq); + } + + if ((!(trigger_type & IRQ_TYPE_LEVEL_LOW)) && (!(trigger_type & IRQ_TYPE_LEVEL_HIGH))) + pci1xxx_assign_bit(priv->reg_base, INTR_LEVEL_MASK_OFFSET(gpio), bitpos, true); + + return true; +} + +static irqreturn_t pci1xxxx_gpio_irq_handler(int irq, void *dev_id) +{ + struct pci1xxxx_gpio *priv = dev_id; + struct gpio_chip *gc = &priv->gpio; + unsigned long int_status = 0; + unsigned long flags; + u8 pincount; + int bit; + u8 gpiobank; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, 16, true); + spin_unlock_irqrestore(&priv->lock, flags); + for (gpiobank = 0; gpiobank < 3; gpiobank++) { + spin_lock_irqsave(&priv->lock, flags); + int_status = readl(priv->reg_base + INTR_STATUS_OFFSET(gpiobank)); + spin_unlock_irqrestore(&priv->lock, flags); + if (gpiobank == 2) + pincount = 29; + else + pincount = 32; + for_each_set_bit(bit, &int_status, pincount) { + unsigned int irq; + + spin_lock_irqsave(&priv->lock, flags); + writel(BIT(bit), priv->reg_base + INTR_STATUS_OFFSET(gpiobank)); + spin_unlock_irqrestore(&priv->lock, flags); + irq = irq_find_mapping(gc->irq.domain, (bit + (gpiobank * 32))); + generic_handle_irq(irq); + } + } + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, 16, false); + spin_unlock_irqrestore(&priv->lock, flags); + + return IRQ_HANDLED; +} + +static struct irq_chip pci1xxxx_gpio_irqchip = { + .name = "pci1xxxx_gpio", + .irq_ack = pci1xxxx_gpio_irq_ack, + .irq_mask = pci1xxxx_gpio_irq_mask, + .irq_unmask = pci1xxxx_gpio_irq_unmask, + .irq_set_type = pci1xxxx_gpio_set_type, +}; + +static int pci1xxxx_gpio_suspend(struct device *dev) +{ + struct pci1xxxx_gpio *priv = dev_get_drvdata(dev); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, + 16, true); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, + 17, false); + pci1xxx_assign_bit(priv->reg_base, PERI_GEN_RESET, 16, true); + spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int pci1xxxx_gpio_resume(struct device *dev) +{ + struct pci1xxxx_gpio *priv = dev_get_drvdata(dev); + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, + 17, true); + pci1xxx_assign_bit(priv->reg_base, PIO_GLOBAL_CONFIG_OFFSET, + 16, false); + pci1xxx_assign_bit(priv->reg_base, PERI_GEN_RESET, 16, false); + spin_unlock_irqrestore(&priv->lock, flags); + + return 0; +} + +static int pci1xxxx_gpio_setup(struct pci1xxxx_gpio *priv, int irq) +{ + struct gpio_chip *gchip = &priv->gpio; + struct gpio_irq_chip *girq; + int retval; + + gchip->label = dev_name(&priv->aux_dev->dev); + gchip->parent = &priv->aux_dev->dev; + gchip->owner = THIS_MODULE; + gchip->direction_input = pci1xxxx_gpio_direction_input; + gchip->direction_output = pci1xxxx_gpio_direction_output; + gchip->get_direction = pci1xxxx_gpio_get_direction; + gchip->get = pci1xxxx_gpio_get; + gchip->set = pci1xxxx_gpio_set; + gchip->set_config = pci1xxxx_gpio_set_config; + gchip->dbg_show = NULL; + gchip->base = -1; + gchip->ngpio = PCI1XXXX_NR_PINS; + gchip->can_sleep = false; + + retval = devm_request_threaded_irq(&priv->aux_dev->dev, irq, + NULL, pci1xxxx_gpio_irq_handler, + IRQF_ONESHOT, "PCI1xxxxGPIO", priv); + + if (retval) + return retval; + + girq = &priv->gpio.irq; + girq->chip = &pci1xxxx_gpio_irqchip; + girq->parent_handler = NULL; + girq->num_parents = 0; + girq->parents = NULL; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_bad_irq; + + return 0; +} + +static int pci1xxxx_gpio_probe(struct auxiliary_device *aux_dev, + const struct auxiliary_device_id *id) + +{ + struct auxiliary_device_wrapper *aux_dev_wrapper; + struct gp_aux_data_type *pdata; + struct pci1xxxx_gpio *priv; + int retval; + + aux_dev_wrapper = (struct auxiliary_device_wrapper *) + container_of(aux_dev, struct auxiliary_device_wrapper, aux_dev); + + pdata = &aux_dev_wrapper->gp_aux_data; + + if (!pdata) + return -EINVAL; + + priv = devm_kzalloc(&aux_dev->dev, sizeof(struct pci1xxxx_gpio), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + spin_lock_init(&priv->lock); + priv->aux_dev = aux_dev; + + if (!devm_request_mem_region(&aux_dev->dev, pdata->region_start, 0x800, aux_dev->name)) + return -EBUSY; + + priv->reg_base = devm_ioremap(&aux_dev->dev, pdata->region_start, 0x800); + if (!priv->reg_base) + return -ENOMEM; + + writel(0x0264, (priv->reg_base + 0x400 + 0xF0)); + + retval = pci1xxxx_gpio_setup(priv, pdata->irq_num); + + if (retval < 0) + return retval; + + dev_set_drvdata(&aux_dev->dev, priv); + + return devm_gpiochip_add_data(&aux_dev->dev, &priv->gpio, priv); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(pci1xxxx_gpio_pm_ops, pci1xxxx_gpio_suspend, pci1xxxx_gpio_resume); + +static const struct auxiliary_device_id pci1xxxx_gpio_auxiliary_id_table[] = { + {.name = "mchp_pci1xxxx_gp.gp_gpio"}, + {} +}; +MODULE_DEVICE_TABLE(auxiliary, pci1xxxx_gpio_auxiliary_id_table); + +static struct auxiliary_driver pci1xxxx_gpio_driver = { + .driver = { + .name = "PCI1xxxxGPIO", + .pm = &pci1xxxx_gpio_pm_ops, + }, + .probe = pci1xxxx_gpio_probe, + .id_table = pci1xxxx_gpio_auxiliary_id_table +}; +module_auxiliary_driver(pci1xxxx_gpio_driver); + +MODULE_DESCRIPTION("Microchip Technology Inc. PCI1xxxx GPIO controller"); +MODULE_AUTHOR("Kumaravel Thiagarajan "); +MODULE_LICENSE("GPL"); diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index 59506ba6fc48f3eb3bb65f030bdc2426bf0066e9..71fbf0bc845324ca311862659ebabba244822c42 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2013-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2013-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -15,6 +15,7 @@ #include "mei_dev.h" #include "client.h" +#include "mkhi.h" #define MEI_UUID_NFC_INFO UUID_LE(0xd2de1625, 0x382d, 0x417d, \ 0x48, 0xa4, 0xef, 0xab, 0xba, 0x8a, 0x12, 0x06) @@ -89,20 +90,6 @@ struct mei_os_ver { u8 reserved2; } __packed; -#define MKHI_FEATURE_PTT 0x10 - -struct mkhi_rule_id { - __le16 rule_type; - u8 feature_id; - u8 reserved; -} __packed; - -struct mkhi_fwcaps { - struct mkhi_rule_id id; - u8 len; - u8 data[]; -} __packed; - struct mkhi_fw_ver_block { u16 minor; u8 major; @@ -115,22 +102,6 @@ struct mkhi_fw_ver { struct mkhi_fw_ver_block ver[MEI_MAX_FW_VER_BLOCKS]; } __packed; -#define MKHI_FWCAPS_GROUP_ID 0x3 -#define MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD 6 -#define MKHI_GEN_GROUP_ID 0xFF -#define MKHI_GEN_GET_FW_VERSION_CMD 0x2 -struct mkhi_msg_hdr { - u8 group_id; - u8 command; - u8 reserved; - u8 result; -} __packed; - -struct mkhi_msg { - struct mkhi_msg_hdr hdr; - u8 data[]; -} __packed; - #define MKHI_OSVER_BUF_LEN (sizeof(struct mkhi_msg_hdr) + \ sizeof(struct mkhi_fwcaps) + \ sizeof(struct mei_os_ver)) @@ -164,7 +135,6 @@ static int mei_osver(struct mei_cl_device *cldev) sizeof(struct mkhi_fw_ver)) #define MKHI_FWVER_LEN(__num) (sizeof(struct mkhi_msg_hdr) + \ sizeof(struct mkhi_fw_ver_block) * (__num)) -#define MKHI_RCV_TIMEOUT 500 /* receive timeout in msec */ static int mei_fwver(struct mei_cl_device *cldev) { char buf[MKHI_FWVER_BUF_LEN]; @@ -187,7 +157,7 @@ static int mei_fwver(struct mei_cl_device *cldev) ret = 0; bytes_recv = __mei_cl_recv(cldev->cl, buf, sizeof(buf), NULL, 0, - MKHI_RCV_TIMEOUT); + cldev->bus->timeouts.mkhi_recv); if (bytes_recv < 0 || (size_t)bytes_recv < MKHI_FWVER_LEN(1)) { /* * Should be at least one version block, @@ -218,6 +188,19 @@ static int mei_fwver(struct mei_cl_device *cldev) return ret; } +static int mei_gfx_memory_ready(struct mei_cl_device *cldev) +{ + struct mkhi_gfx_mem_ready req = {0}; + unsigned int mode = MEI_CL_IO_TX_INTERNAL; + + req.hdr.group_id = MKHI_GROUP_ID_GFX; + req.hdr.command = MKHI_GFX_MEMORY_READY_CMD_REQ; + req.flags = MKHI_GFX_MEM_READY_PXP_ALLOWED; + + dev_dbg(&cldev->dev, "Sending memory ready command\n"); + return __mei_cl_send(cldev->cl, (u8 *)&req, sizeof(req), 0, mode); +} + static void mei_mkhi_fix(struct mei_cl_device *cldev) { int ret; @@ -264,6 +247,39 @@ static void mei_gsc_mkhi_ver(struct mei_cl_device *cldev) dev_err(&cldev->dev, "FW version command failed %d\n", ret); mei_cldev_disable(cldev); } + +static void mei_gsc_mkhi_fix_ver(struct mei_cl_device *cldev) +{ + int ret; + + /* No need to enable the client if nothing is needed from it */ + if (!cldev->bus->fw_f_fw_ver_supported && + cldev->bus->pxp_mode != MEI_DEV_PXP_INIT) + return; + + ret = mei_cldev_enable(cldev); + if (ret) + return; + + if (cldev->bus->pxp_mode == MEI_DEV_PXP_INIT) { + ret = mei_gfx_memory_ready(cldev); + if (ret < 0) + dev_err(&cldev->dev, "memory ready command failed %d\n", ret); + else + dev_dbg(&cldev->dev, "memory ready command sent\n"); + /* we go to reset after that */ + cldev->bus->pxp_mode = MEI_DEV_PXP_SETUP; + goto out; + } + + ret = mei_fwver(cldev); + if (ret < 0) + dev_err(&cldev->dev, "FW version command failed %d\n", + ret); +out: + mei_cldev_disable(cldev); +} + /** * mei_wd - wd client on the bus, change protocol version * as the API has changed. @@ -471,7 +487,7 @@ static void mei_nfc(struct mei_cl_device *cldev) } dev_dbg(bus->dev, "nfc radio %s\n", radio_name); - strlcpy(cldev->name, radio_name, sizeof(cldev->name)); + strscpy(cldev->name, radio_name, sizeof(cldev->name)); disconnect: mutex_lock(&bus->device_lock); @@ -503,6 +519,26 @@ static void vt_support(struct mei_cl_device *cldev) cldev->do_match = 1; } +/** + * pxp_is_ready - enable bus client if pxp is ready + * + * @cldev: me clients device + */ +static void pxp_is_ready(struct mei_cl_device *cldev) +{ + struct mei_device *bus = cldev->bus; + + switch (bus->pxp_mode) { + case MEI_DEV_PXP_READY: + case MEI_DEV_PXP_DEFAULT: + cldev->do_match = 1; + break; + default: + cldev->do_match = 0; + break; + } +} + #define MEI_FIXUP(_uuid, _hook) { _uuid, _hook } static struct mei_fixup { @@ -516,10 +552,10 @@ static struct mei_fixup { MEI_FIXUP(MEI_UUID_WD, mei_wd), MEI_FIXUP(MEI_UUID_MKHIF_FIX, mei_mkhi_fix), MEI_FIXUP(MEI_UUID_IGSC_MKHI, mei_gsc_mkhi_ver), - MEI_FIXUP(MEI_UUID_IGSC_MKHI_FIX, mei_gsc_mkhi_ver), + MEI_FIXUP(MEI_UUID_IGSC_MKHI_FIX, mei_gsc_mkhi_fix_ver), MEI_FIXUP(MEI_UUID_HDCP, whitelist), MEI_FIXUP(MEI_UUID_ANY, vt_support), - MEI_FIXUP(MEI_UUID_PAVP, whitelist), + MEI_FIXUP(MEI_UUID_PAVP, pxp_is_ready), }; /** diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 31264ab2eb1354e23de59bb1ef36b92e87f280ca..0b2fbe1335a7747dc195db0da91dec35b7af1a20 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -870,7 +870,7 @@ static int mei_cl_send_disconnect(struct mei_cl *cl, struct mei_cl_cb *cb) } list_move_tail(&cb->list, &dev->ctrl_rd_list); - cl->timer_count = MEI_CONNECT_TIMEOUT; + cl->timer_count = dev->timeouts.connect; mei_schedule_stall_timer(dev); return 0; @@ -945,7 +945,7 @@ static int __mei_cl_disconnect(struct mei_cl *cl) wait_event_timeout(cl->wait, cl->state == MEI_FILE_DISCONNECT_REPLY || cl->state == MEI_FILE_DISCONNECTED, - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); rets = cl->status; @@ -1065,7 +1065,7 @@ static int mei_cl_send_connect(struct mei_cl *cl, struct mei_cl_cb *cb) } list_move_tail(&cb->list, &dev->ctrl_rd_list); - cl->timer_count = MEI_CONNECT_TIMEOUT; + cl->timer_count = dev->timeouts.connect; mei_schedule_stall_timer(dev); return 0; } @@ -1164,7 +1164,7 @@ int mei_cl_connect(struct mei_cl *cl, struct mei_me_client *me_cl, cl->state == MEI_FILE_DISCONNECTED || cl->state == MEI_FILE_DISCONNECT_REQUIRED || cl->state == MEI_FILE_DISCONNECT_REPLY), - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); if (!mei_cl_is_connected(cl)) { @@ -1562,7 +1562,7 @@ int mei_cl_notify_request(struct mei_cl *cl, cl->notify_en == request || cl->status || !mei_cl_is_connected(cl), - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); if (cl->notify_en != request && !cl->status) @@ -2336,7 +2336,7 @@ int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp, mutex_unlock(&dev->device_lock); wait_event_timeout(cl->wait, cl->dma_mapped || cl->status, - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); if (!cl->dma_mapped && !cl->status) @@ -2415,7 +2415,7 @@ int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp) mutex_unlock(&dev->device_lock); wait_event_timeout(cl->wait, !cl->dma_mapped || cl->status, - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); if (cl->dma_mapped && !cl->status) diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c index 1ce61e9e24fcae35f6b54165fd5f536c4b1f971d..3b098d4c8e3de63e94b328d8ac13a63be00bc576 100644 --- a/drivers/misc/mei/debugfs.c +++ b/drivers/misc/mei/debugfs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2012-2016, Intel Corporation. All rights reserved + * Copyright (c) 2012-2022, Intel Corporation. All rights reserved * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -86,6 +86,20 @@ out: } DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_active); +static const char *mei_dev_pxp_mode_str(enum mei_dev_pxp_mode state) +{ +#define MEI_PXP_MODE(state) case MEI_DEV_PXP_##state: return #state + switch (state) { + MEI_PXP_MODE(DEFAULT); + MEI_PXP_MODE(INIT); + MEI_PXP_MODE(SETUP); + MEI_PXP_MODE(READY); + default: + return "unknown"; + } +#undef MEI_PXP_MODE +} + static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused) { struct mei_device *dev = m->private; @@ -112,6 +126,9 @@ static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused) seq_printf(m, "pg: %s, %s\n", mei_pg_is_enabled(dev) ? "ENABLED" : "DISABLED", mei_pg_state_str(mei_pg_state(dev))); + + seq_printf(m, "pxp: %s\n", mei_dev_pxp_mode_str(dev->pxp_mode)); + return 0; } DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_devstate); diff --git a/drivers/misc/mei/gsc-me.c b/drivers/misc/mei/gsc-me.c index c8145e9b62b61bed46aacb78732f6ea7ef68af25..e63cabd0818dc9fdd00312fad270a4a435fa2de2 100644 --- a/drivers/misc/mei/gsc-me.c +++ b/drivers/misc/mei/gsc-me.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "mei_dev.h" #include "hw-me.h" @@ -31,6 +32,17 @@ static int mei_gsc_read_hfs(const struct mei_device *dev, int where, u32 *val) return 0; } +static void mei_gsc_set_ext_op_mem(const struct mei_me_hw *hw, struct resource *mem) +{ + u32 low = lower_32_bits(mem->start); + u32 hi = upper_32_bits(mem->start); + u32 limit = (resource_size(mem) / SZ_4K) | GSC_EXT_OP_MEM_VALID; + + iowrite32(low, hw->mem_addr + H_GSC_EXT_OP_MEM_BASE_ADDR_LO_REG); + iowrite32(hi, hw->mem_addr + H_GSC_EXT_OP_MEM_BASE_ADDR_HI_REG); + iowrite32(limit, hw->mem_addr + H_GSC_EXT_OP_MEM_LIMIT_REG); +} + static int mei_gsc_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *aux_dev_id) { @@ -47,7 +59,7 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev, device = &aux_dev->dev; - dev = mei_me_dev_init(device, cfg); + dev = mei_me_dev_init(device, cfg, adev->slow_firmware); if (!dev) { ret = -ENOMEM; goto err; @@ -56,7 +68,6 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev, hw = to_me_hw(dev); hw->mem_addr = devm_ioremap_resource(device, &adev->bar); if (IS_ERR(hw->mem_addr)) { - dev_err(device, "mmio not mapped\n"); ret = PTR_ERR(hw->mem_addr); goto err; } @@ -66,13 +77,33 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev, dev_set_drvdata(device, dev); - ret = devm_request_threaded_irq(device, hw->irq, - mei_me_irq_quick_handler, - mei_me_irq_thread_handler, - IRQF_ONESHOT, KBUILD_MODNAME, dev); - if (ret) { - dev_err(device, "irq register failed %d\n", ret); - goto err; + if (adev->ext_op_mem.start) { + mei_gsc_set_ext_op_mem(hw, &adev->ext_op_mem); + dev->pxp_mode = MEI_DEV_PXP_INIT; + } + + /* use polling */ + if (mei_me_hw_use_polling(hw)) { + mei_disable_interrupts(dev); + mei_clear_interrupts(dev); + init_waitqueue_head(&hw->wait_active); + hw->is_active = true; /* start in active mode for initialization */ + hw->polling_thread = kthread_run(mei_me_polling_thread, dev, + "kmegscirqd/%s", dev_name(device)); + if (IS_ERR(hw->polling_thread)) { + ret = PTR_ERR(hw->polling_thread); + dev_err(device, "unable to create kernel thread: %d\n", ret); + goto err; + } + } else { + ret = devm_request_threaded_irq(device, hw->irq, + mei_me_irq_quick_handler, + mei_me_irq_thread_handler, + IRQF_ONESHOT, KBUILD_MODNAME, dev); + if (ret) { + dev_err(device, "irq register failed %d\n", ret); + goto err; + } } pm_runtime_get_noresume(device); @@ -98,7 +129,8 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev, register_err: mei_stop(dev); - devm_free_irq(device, hw->irq, dev); + if (!mei_me_hw_use_polling(hw)) + devm_free_irq(device, hw->irq, dev); err: dev_err(device, "probe failed: %d\n", ret); @@ -119,12 +151,17 @@ static void mei_gsc_remove(struct auxiliary_device *aux_dev) mei_stop(dev); + hw = to_me_hw(dev); + if (mei_me_hw_use_polling(hw)) + kthread_stop(hw->polling_thread); + mei_deregister(dev); pm_runtime_disable(&aux_dev->dev); mei_disable_interrupts(dev); - devm_free_irq(&aux_dev->dev, hw->irq, dev); + if (!mei_me_hw_use_polling(hw)) + devm_free_irq(&aux_dev->dev, hw->irq, dev); } static int __maybe_unused mei_gsc_pm_suspend(struct device *device) @@ -144,11 +181,22 @@ static int __maybe_unused mei_gsc_pm_suspend(struct device *device) static int __maybe_unused mei_gsc_pm_resume(struct device *device) { struct mei_device *dev = dev_get_drvdata(device); + struct auxiliary_device *aux_dev; + struct mei_aux_device *adev; int err; + struct mei_me_hw *hw; if (!dev) return -ENODEV; + hw = to_me_hw(dev); + aux_dev = to_auxiliary_dev(device); + adev = auxiliary_dev_to_mei_aux_dev(aux_dev); + if (adev->ext_op_mem.start) { + mei_gsc_set_ext_op_mem(hw, &adev->ext_op_mem); + dev->pxp_mode = MEI_DEV_PXP_INIT; + } + err = mei_restart(dev); if (err) return err; @@ -185,6 +233,9 @@ static int __maybe_unused mei_gsc_pm_runtime_suspend(struct device *device) if (mei_write_is_idle(dev)) { hw = to_me_hw(dev); hw->pg_state = MEI_PG_ON; + + if (mei_me_hw_use_polling(hw)) + hw->is_active = false; ret = 0; } else { ret = -EAGAIN; @@ -209,6 +260,11 @@ static int __maybe_unused mei_gsc_pm_runtime_resume(struct device *device) hw = to_me_hw(dev); hw->pg_state = MEI_PG_OFF; + if (mei_me_hw_use_polling(hw)) { + hw->is_active = true; + wake_up(&hw->wait_active); + } + mutex_unlock(&dev->device_lock); irq_ret = mei_me_irq_thread_handler(1, dev); diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c index cf2b8261da1444470a310f80c4cdc9e9cd4afeb5..de712cbf5d07253ad0fd3ee1177f077e229fe096 100644 --- a/drivers/misc/mei/hbm.c +++ b/drivers/misc/mei/hbm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ #include @@ -232,7 +232,7 @@ int mei_hbm_start_wait(struct mei_device *dev) mutex_unlock(&dev->device_lock); ret = wait_event_timeout(dev->wait_hbm_start, dev->hbm_state != MEI_HBM_STARTING, - mei_secs_to_jiffies(MEI_HBM_TIMEOUT)); + dev->timeouts.hbm); mutex_lock(&dev->device_lock); if (ret == 0 && (dev->hbm_state <= MEI_HBM_STARTING)) { @@ -275,7 +275,7 @@ int mei_hbm_start_req(struct mei_device *dev) } dev->hbm_state = MEI_HBM_STARTING; - dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT; + dev->init_clients_timer = dev->timeouts.client_init; mei_schedule_stall_timer(dev); return 0; } @@ -316,7 +316,7 @@ static int mei_hbm_dma_setup_req(struct mei_device *dev) } dev->hbm_state = MEI_HBM_DR_SETUP; - dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT; + dev->init_clients_timer = dev->timeouts.client_init; mei_schedule_stall_timer(dev); return 0; } @@ -351,7 +351,7 @@ static int mei_hbm_capabilities_req(struct mei_device *dev) } dev->hbm_state = MEI_HBM_CAP_SETUP; - dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT; + dev->init_clients_timer = dev->timeouts.client_init; mei_schedule_stall_timer(dev); return 0; } @@ -385,7 +385,7 @@ static int mei_hbm_enum_clients_req(struct mei_device *dev) return ret; } dev->hbm_state = MEI_HBM_ENUM_CLIENTS; - dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT; + dev->init_clients_timer = dev->timeouts.client_init; mei_schedule_stall_timer(dev); return 0; } @@ -751,7 +751,7 @@ static int mei_hbm_prop_req(struct mei_device *dev, unsigned long start_idx) return ret; } - dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT; + dev->init_clients_timer = dev->timeouts.client_init; mei_schedule_stall_timer(dev); return 0; diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index 15e8e2b322b1a398678555f0a07091c7a1fe84b3..99966cd3e7d892ca0f86666614bc7f664e978251 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (c) 2003-2019, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ #ifndef _MEI_HW_MEI_REGS_H_ @@ -127,6 +127,8 @@ # define PCI_CFG_HFS_3_FW_SKU_SPS 0x00000060 #define PCI_CFG_HFS_4 0x64 #define PCI_CFG_HFS_5 0x68 +# define GSC_CFG_HFS_5_BOOT_TYPE_MSK 0x00000003 +# define GSC_CFG_HFS_5_BOOT_TYPE_PXP 3 #define PCI_CFG_HFS_6 0x6C /* MEI registers */ @@ -143,6 +145,11 @@ /* H_D0I3C - D0I3 Control */ #define H_D0I3C 0x800 +#define H_GSC_EXT_OP_MEM_BASE_ADDR_LO_REG 0x100 +#define H_GSC_EXT_OP_MEM_BASE_ADDR_HI_REG 0x104 +#define H_GSC_EXT_OP_MEM_LIMIT_REG 0x108 +#define GSC_EXT_OP_MEM_VALID BIT(31) + /* register bits of H_CSR (Host Control Status register) */ /* Host Circular Buffer Depth - maximum number of 32-bit entries in CB */ #define H_CBD 0xFF000000 diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c index 3a95fe7d4e3306e7c1d8ad079596156ab18b1fe1..9e2f781c6ed527b779ac5945d1c3c36a188caf65 100644 --- a/drivers/misc/mei/hw-me.c +++ b/drivers/misc/mei/hw-me.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -10,6 +10,7 @@ #include #include #include +#include #include "mei_dev.h" #include "hbm.h" @@ -327,9 +328,12 @@ static void mei_me_intr_clear(struct mei_device *dev) */ static void mei_me_intr_enable(struct mei_device *dev) { - u32 hcsr = mei_hcsr_read(dev); + u32 hcsr; + + if (mei_me_hw_use_polling(to_me_hw(dev))) + return; - hcsr |= H_CSR_IE_MASK; + hcsr = mei_hcsr_read(dev) | H_CSR_IE_MASK; mei_hcsr_set(dev, hcsr); } @@ -354,6 +358,9 @@ static void mei_me_synchronize_irq(struct mei_device *dev) { struct mei_me_hw *hw = to_me_hw(dev); + if (mei_me_hw_use_polling(hw)) + return; + synchronize_irq(hw->irq); } @@ -380,7 +387,10 @@ static void mei_me_host_set_ready(struct mei_device *dev) { u32 hcsr = mei_hcsr_read(dev); - hcsr |= H_CSR_IE_MASK | H_IG | H_RDY; + if (!mei_me_hw_use_polling(to_me_hw(dev))) + hcsr |= H_CSR_IE_MASK; + + hcsr |= H_IG | H_RDY; mei_hcsr_set(dev, hcsr); } @@ -423,6 +433,29 @@ static bool mei_me_hw_is_resetting(struct mei_device *dev) return (mecsr & ME_RST_HRA) == ME_RST_HRA; } +/** + * mei_gsc_pxp_check - check for gsc firmware entering pxp mode + * + * @dev: the device structure + */ +static void mei_gsc_pxp_check(struct mei_device *dev) +{ + struct mei_me_hw *hw = to_me_hw(dev); + u32 fwsts5 = 0; + + if (dev->pxp_mode == MEI_DEV_PXP_DEFAULT) + return; + + hw->read_fws(dev, PCI_CFG_HFS_5, &fwsts5); + trace_mei_pci_cfg_read(dev->dev, "PCI_CFG_HFS_5", PCI_CFG_HFS_5, fwsts5); + if ((fwsts5 & GSC_CFG_HFS_5_BOOT_TYPE_MSK) == GSC_CFG_HFS_5_BOOT_TYPE_PXP) { + dev_dbg(dev->dev, "pxp mode is ready 0x%08x\n", fwsts5); + dev->pxp_mode = MEI_DEV_PXP_READY; + } else { + dev_dbg(dev->dev, "pxp mode is not ready 0x%08x\n", fwsts5); + } +} + /** * mei_me_hw_ready_wait - wait until the me(hw) has turned ready * or timeout is reached @@ -435,13 +468,15 @@ static int mei_me_hw_ready_wait(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_hw_ready, dev->recvd_hw_ready, - mei_secs_to_jiffies(MEI_HW_READY_TIMEOUT)); + dev->timeouts.hw_ready); mutex_lock(&dev->device_lock); if (!dev->recvd_hw_ready) { dev_err(dev->dev, "wait hw ready failed\n"); return -ETIME; } + mei_gsc_pxp_check(dev); + mei_me_hw_reset_release(dev); dev->recvd_hw_ready = false; return 0; @@ -697,7 +732,6 @@ static void mei_me_pg_unset(struct mei_device *dev) static int mei_me_pg_legacy_enter_sync(struct mei_device *dev) { struct mei_me_hw *hw = to_me_hw(dev); - unsigned long timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT); int ret; dev->pg_event = MEI_PG_EVENT_WAIT; @@ -708,7 +742,8 @@ static int mei_me_pg_legacy_enter_sync(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_RECEIVED, timeout); + dev->pg_event == MEI_PG_EVENT_RECEIVED, + dev->timeouts.pgi); mutex_lock(&dev->device_lock); if (dev->pg_event == MEI_PG_EVENT_RECEIVED) { @@ -734,7 +769,6 @@ static int mei_me_pg_legacy_enter_sync(struct mei_device *dev) static int mei_me_pg_legacy_exit_sync(struct mei_device *dev) { struct mei_me_hw *hw = to_me_hw(dev); - unsigned long timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT); int ret; if (dev->pg_event == MEI_PG_EVENT_RECEIVED) @@ -746,7 +780,8 @@ static int mei_me_pg_legacy_exit_sync(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_RECEIVED, timeout); + dev->pg_event == MEI_PG_EVENT_RECEIVED, + dev->timeouts.pgi); mutex_lock(&dev->device_lock); reply: @@ -762,7 +797,8 @@ reply: mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, timeout); + dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, + dev->timeouts.pgi); mutex_lock(&dev->device_lock); if (dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED) @@ -877,8 +913,6 @@ static u32 mei_me_d0i3_unset(struct mei_device *dev) static int mei_me_d0i3_enter_sync(struct mei_device *dev) { struct mei_me_hw *hw = to_me_hw(dev); - unsigned long d0i3_timeout = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT); - unsigned long pgi_timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT); int ret; u32 reg; @@ -900,7 +934,8 @@ static int mei_me_d0i3_enter_sync(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_RECEIVED, pgi_timeout); + dev->pg_event == MEI_PG_EVENT_RECEIVED, + dev->timeouts.pgi); mutex_lock(&dev->device_lock); if (dev->pg_event != MEI_PG_EVENT_RECEIVED) { @@ -920,7 +955,8 @@ static int mei_me_d0i3_enter_sync(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, d0i3_timeout); + dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, + dev->timeouts.d0i3); mutex_lock(&dev->device_lock); if (dev->pg_event != MEI_PG_EVENT_INTR_RECEIVED) { @@ -980,7 +1016,6 @@ on: static int mei_me_d0i3_exit_sync(struct mei_device *dev) { struct mei_me_hw *hw = to_me_hw(dev); - unsigned long timeout = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT); int ret; u32 reg; @@ -1003,7 +1038,8 @@ static int mei_me_d0i3_exit_sync(struct mei_device *dev) mutex_unlock(&dev->device_lock); wait_event_timeout(dev->wait_pg, - dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, timeout); + dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, + dev->timeouts.d0i3); mutex_lock(&dev->device_lock); if (dev->pg_event != MEI_PG_EVENT_INTR_RECEIVED) { @@ -1176,7 +1212,7 @@ static int mei_me_hw_reset(struct mei_device *dev, bool intr_enable) hcsr |= H_RST | H_IG | H_CSR_IS_MASK; - if (!intr_enable) + if (!intr_enable || mei_me_hw_use_polling(to_me_hw(dev))) hcsr &= ~H_CSR_IE_MASK; dev->recvd_hw_ready = false; @@ -1259,7 +1295,8 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id) /* check if ME wants a reset */ if (!mei_hw_is_ready(dev) && dev->dev_state != MEI_DEV_RESETTING) { - dev_warn(dev->dev, "FW not ready: resetting.\n"); + dev_warn(dev->dev, "FW not ready: resetting: dev_state = %d pxp = %d\n", + dev->dev_state, dev->pxp_mode); if (dev->dev_state == MEI_DEV_POWERING_DOWN || dev->dev_state == MEI_DEV_POWER_DOWN) mei_cl_all_disconnect(dev); @@ -1331,6 +1368,66 @@ end: } EXPORT_SYMBOL_GPL(mei_me_irq_thread_handler); +#define MEI_POLLING_TIMEOUT_ACTIVE 100 +#define MEI_POLLING_TIMEOUT_IDLE 500 + +/** + * mei_me_polling_thread - interrupt register polling thread + * + * The thread monitors the interrupt source register and calls + * mei_me_irq_thread_handler() to handle the firmware + * input. + * + * The function polls in MEI_POLLING_TIMEOUT_ACTIVE timeout + * in case there was an event, in idle case the polling + * time increases yet again by MEI_POLLING_TIMEOUT_ACTIVE + * up to MEI_POLLING_TIMEOUT_IDLE. + * + * @_dev: mei device + * + * Return: always 0 + */ +int mei_me_polling_thread(void *_dev) +{ + struct mei_device *dev = _dev; + irqreturn_t irq_ret; + long polling_timeout = MEI_POLLING_TIMEOUT_ACTIVE; + + dev_dbg(dev->dev, "kernel thread is running\n"); + while (!kthread_should_stop()) { + struct mei_me_hw *hw = to_me_hw(dev); + u32 hcsr; + + wait_event_timeout(hw->wait_active, + hw->is_active || kthread_should_stop(), + msecs_to_jiffies(MEI_POLLING_TIMEOUT_IDLE)); + + if (kthread_should_stop()) + break; + + hcsr = mei_hcsr_read(dev); + if (me_intr_src(hcsr)) { + polling_timeout = MEI_POLLING_TIMEOUT_ACTIVE; + irq_ret = mei_me_irq_thread_handler(1, dev); + if (irq_ret != IRQ_HANDLED) + dev_err(dev->dev, "irq_ret %d\n", irq_ret); + } else { + /* + * Increase timeout by MEI_POLLING_TIMEOUT_ACTIVE + * up to MEI_POLLING_TIMEOUT_IDLE + */ + polling_timeout = clamp_val(polling_timeout + MEI_POLLING_TIMEOUT_ACTIVE, + MEI_POLLING_TIMEOUT_ACTIVE, + MEI_POLLING_TIMEOUT_IDLE); + } + + schedule_timeout_interruptible(msecs_to_jiffies(polling_timeout)); + } + + return 0; +} +EXPORT_SYMBOL_GPL(mei_me_polling_thread); + static const struct mei_hw_ops mei_me_hw_ops = { .trc_status = mei_me_trc_status, @@ -1636,11 +1733,12 @@ EXPORT_SYMBOL_GPL(mei_me_get_cfg); * * @parent: device associated with physical device (pci/platform) * @cfg: per device generation config + * @slow_fw: configure longer timeouts as FW is slow * * Return: The mei_device pointer on success, NULL on failure. */ struct mei_device *mei_me_dev_init(struct device *parent, - const struct mei_cfg *cfg) + const struct mei_cfg *cfg, bool slow_fw) { struct mei_device *dev; struct mei_me_hw *hw; @@ -1655,7 +1753,7 @@ struct mei_device *mei_me_dev_init(struct device *parent, for (i = 0; i < DMA_DSCR_NUM; i++) dev->dr_dscr[i].size = cfg->dma_size[i]; - mei_device_init(dev, parent, &mei_me_hw_ops); + mei_device_init(dev, parent, slow_fw, &mei_me_hw_ops); hw->cfg = cfg; dev->fw_f_fw_ver_supported = cfg->fw_ver_supported; diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h index a071c645e9054560e6c776afa60e31e304660761..95cf830b7c7b63e1cf555048ca929a7bc78a7d39 100644 --- a/drivers/misc/mei/hw-me.h +++ b/drivers/misc/mei/hw-me.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2012-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2012-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -51,6 +51,9 @@ struct mei_cfg { * @d0i3_supported: di03 support * @hbuf_depth: depth of hardware host/write buffer in slots * @read_fws: read FW status register handler + * @polling_thread: interrupt polling thread + * @wait_active: the polling thread activity wait queue + * @is_active: the device is active */ struct mei_me_hw { const struct mei_cfg *cfg; @@ -60,10 +63,19 @@ struct mei_me_hw { bool d0i3_supported; u8 hbuf_depth; int (*read_fws)(const struct mei_device *dev, int where, u32 *val); + /* polling */ + struct task_struct *polling_thread; + wait_queue_head_t wait_active; + bool is_active; }; #define to_me_hw(dev) (struct mei_me_hw *)((dev)->hw) +static inline bool mei_me_hw_use_polling(const struct mei_me_hw *hw) +{ + return hw->irq < 0; +} + /** * enum mei_cfg_idx - indices to platform specific configurations. * @@ -120,12 +132,13 @@ enum mei_cfg_idx { const struct mei_cfg *mei_me_get_cfg(kernel_ulong_t idx); struct mei_device *mei_me_dev_init(struct device *parent, - const struct mei_cfg *cfg); + const struct mei_cfg *cfg, bool slow_fw); int mei_me_pg_enter_sync(struct mei_device *dev); int mei_me_pg_exit_sync(struct mei_device *dev); irqreturn_t mei_me_irq_quick_handler(int irq, void *dev_id); irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id); +int mei_me_polling_thread(void *_dev); #endif /* _MEI_INTERFACE_H_ */ diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c index 00652c137cc74e79fac6108f2d75035034d8bc28..5d0f68b95c29177d335062836bc42bc3254281df 100644 --- a/drivers/misc/mei/hw-txe.c +++ b/drivers/misc/mei/hw-txe.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2013-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2013-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -176,7 +176,7 @@ static bool mei_txe_aliveness_set(struct mei_device *dev, u32 req) * @dev: the device structure * * Extract HICR_HOST_ALIVENESS_RESP_ACK bit from - * from HICR_HOST_ALIVENESS_REQ register value + * HICR_HOST_ALIVENESS_REQ register value * * Return: SICR_HOST_ALIVENESS_REQ_REQUESTED bit value */ @@ -1201,7 +1201,7 @@ struct mei_device *mei_txe_dev_init(struct pci_dev *pdev) if (!dev) return NULL; - mei_device_init(dev, &pdev->dev, &mei_txe_hw_ops); + mei_device_init(dev, &pdev->dev, false, &mei_txe_hw_ops); hw = to_txe_hw(dev); diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index b46077b1711401b224252fd8e5f60d59b8d8e090..e7e020dba6b1d54075f0d09436295e3abbe21203 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -16,11 +16,16 @@ #define MEI_CONNECT_TIMEOUT 3 /* HPS: at least 2 seconds */ #define MEI_CL_CONNECT_TIMEOUT 15 /* HPS: Client Connect Timeout */ +#define MEI_CL_CONNECT_TIMEOUT_SLOW 30 /* HPS: Client Connect Timeout, slow FW */ #define MEI_CLIENTS_INIT_TIMEOUT 15 /* HPS: Clients Enumeration Timeout */ #define MEI_PGI_TIMEOUT 1 /* PG Isolation time response 1 sec */ #define MEI_D0I3_TIMEOUT 5 /* D0i3 set/unset max response time */ #define MEI_HBM_TIMEOUT 1 /* 1 second */ +#define MEI_HBM_TIMEOUT_SLOW 5 /* 5 second, slow FW */ + +#define MKHI_RCV_TIMEOUT 500 /* receive timeout in msec */ +#define MKHI_RCV_TIMEOUT_SLOW 10000 /* receive timeout in msec, slow FW */ /* * FW page size for DMA allocations diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c index eb052005ca86d89dccac83f00dd223ee1a4fc5b0..bac8852aad510b9fb645900189f82b610be2f612 100644 --- a/drivers/misc/mei/init.c +++ b/drivers/misc/mei/init.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2012-2019, Intel Corporation. All rights reserved. + * Copyright (c) 2012-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -218,16 +218,6 @@ int mei_start(struct mei_device *dev) goto err; } - if (!mei_host_is_ready(dev)) { - dev_err(dev->dev, "host is not ready.\n"); - goto err; - } - - if (!mei_hw_is_ready(dev)) { - dev_err(dev->dev, "ME is not ready.\n"); - goto err; - } - if (!mei_hbm_version_is_supported(dev)) { dev_dbg(dev->dev, "MEI start failed.\n"); goto err; @@ -320,6 +310,8 @@ void mei_stop(struct mei_device *dev) mei_clear_interrupts(dev); mei_synchronize_irq(dev); + /* to catch HW-initiated reset */ + mei_cancel_work(dev); mutex_lock(&dev->device_lock); @@ -357,14 +349,16 @@ bool mei_write_is_idle(struct mei_device *dev) EXPORT_SYMBOL_GPL(mei_write_is_idle); /** - * mei_device_init -- initialize mei_device structure + * mei_device_init - initialize mei_device structure * * @dev: the mei device * @device: the device structure + * @slow_fw: configure longer timeouts as FW is slow * @hw_ops: hw operations */ void mei_device_init(struct mei_device *dev, struct device *device, + bool slow_fw, const struct mei_hw_ops *hw_ops) { /* setup our list array */ @@ -393,6 +387,8 @@ void mei_device_init(struct mei_device *dev, bitmap_zero(dev->host_clients_map, MEI_CLIENTS_MAX); dev->open_handle_count = 0; + dev->pxp_mode = MEI_DEV_PXP_DEFAULT; + /* * Reserving the first client ID * 0: Reserved for MEI Bus Message communications @@ -402,6 +398,21 @@ void mei_device_init(struct mei_device *dev, dev->pg_event = MEI_PG_EVENT_IDLE; dev->ops = hw_ops; dev->dev = device; + + dev->timeouts.hw_ready = mei_secs_to_jiffies(MEI_HW_READY_TIMEOUT); + dev->timeouts.connect = MEI_CONNECT_TIMEOUT; + dev->timeouts.client_init = MEI_CLIENTS_INIT_TIMEOUT; + dev->timeouts.pgi = mei_secs_to_jiffies(MEI_PGI_TIMEOUT); + dev->timeouts.d0i3 = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT); + if (slow_fw) { + dev->timeouts.cl_connect = mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT_SLOW); + dev->timeouts.hbm = mei_secs_to_jiffies(MEI_HBM_TIMEOUT_SLOW); + dev->timeouts.mkhi_recv = msecs_to_jiffies(MKHI_RCV_TIMEOUT_SLOW); + } else { + dev->timeouts.cl_connect = mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT); + dev->timeouts.hbm = mei_secs_to_jiffies(MEI_HBM_TIMEOUT); + dev->timeouts.mkhi_recv = msecs_to_jiffies(MKHI_RCV_TIMEOUT); + } } EXPORT_SYMBOL_GPL(mei_device_init); diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c index 786f7c8f7f6195c5eccf2478990a885634a93a1b..930887e7e38d6810a7f3ef858948cc907b8fd868 100644 --- a/drivers/misc/mei/main.c +++ b/drivers/misc/mei/main.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -571,7 +571,7 @@ static int mei_ioctl_connect_vtag(struct file *file, cl->state == MEI_FILE_DISCONNECTED || cl->state == MEI_FILE_DISCONNECT_REQUIRED || cl->state == MEI_FILE_DISCONNECT_REPLY), - mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT)); + dev->timeouts.cl_connect); mutex_lock(&dev->device_lock); } diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 694f866f87efc468207ff7d3bf5889fa760bc544..6bb3e1ba9ded4979ea2eb710ec330718e2ccc36f 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (c) 2003-2019, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -62,6 +62,21 @@ enum mei_dev_state { MEI_DEV_POWER_UP }; +/** + * enum mei_dev_pxp_mode - MEI PXP mode state + * + * @MEI_DEV_PXP_DEFAULT: PCH based device, no initailization required + * @MEI_DEV_PXP_INIT: device requires initialization, send setup message to firmware + * @MEI_DEV_PXP_SETUP: device is in setup stage, waiting for firmware repsonse + * @MEI_DEV_PXP_READY: device initialized + */ +enum mei_dev_pxp_mode { + MEI_DEV_PXP_DEFAULT = 0, + MEI_DEV_PXP_INIT = 1, + MEI_DEV_PXP_SETUP = 2, + MEI_DEV_PXP_READY = 3, +}; + const char *mei_dev_state_str(int state); enum mei_file_transaction_states { @@ -415,6 +430,17 @@ struct mei_fw_version { #define MEI_MAX_FW_VER_BLOCKS 3 +struct mei_dev_timeouts { + unsigned long hw_ready; /* Timeout on ready message, in jiffies */ + int connect; /* HPS: at least 2 seconds, in seconds */ + unsigned long cl_connect; /* HPS: Client Connect Timeout, in jiffies */ + int client_init; /* HPS: Clients Enumeration Timeout, in seconds */ + unsigned long pgi; /* PG Isolation time response, in jiffies */ + unsigned int d0i3; /* D0i3 set/unset max response time, in jiffies */ + unsigned long hbm; /* HBM operation timeout, in jiffies */ + unsigned long mkhi_recv; /* receive timeout, in jiffies */ +}; + /** * struct mei_device - MEI private device struct * @@ -443,6 +469,7 @@ struct mei_fw_version { * @reset_count : number of consecutive resets * @dev_state : device state * @hbm_state : state of host bus message protocol + * @pxp_mode : PXP device mode * @init_clients_timer : HBM init handshake timeout * * @pg_event : power gating event @@ -480,6 +507,8 @@ struct mei_fw_version { * @allow_fixed_address: allow user space to connect a fixed client * @override_fixed_address: force allow fixed address behavior * + * @timeouts: actual timeout values + * * @reset_work : work item for the device reset * @bus_rescan_work : work item for the bus rescan * @@ -524,6 +553,7 @@ struct mei_device { unsigned long reset_count; enum mei_dev_state dev_state; enum mei_hbm_state hbm_state; + enum mei_dev_pxp_mode pxp_mode; u16 init_clients_timer; /* @@ -568,6 +598,8 @@ struct mei_device { bool allow_fixed_address; bool override_fixed_address; + struct mei_dev_timeouts timeouts; + struct work_struct reset_work; struct work_struct bus_rescan_work; @@ -632,6 +664,7 @@ static inline u32 mei_slots2data(int slots) */ void mei_device_init(struct mei_device *dev, struct device *device, + bool slow_fw, const struct mei_hw_ops *hw_ops); int mei_reset(struct mei_device *dev); int mei_start(struct mei_device *dev); diff --git a/drivers/misc/mei/mkhi.h b/drivers/misc/mei/mkhi.h new file mode 100644 index 0000000000000000000000000000000000000000..1473ea48966623ae2ea714afdd1f29dff4650842 --- /dev/null +++ b/drivers/misc/mei/mkhi.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. + * Intel Management Engine Interface (Intel MEI) Linux driver + */ + +#ifndef _MEI_MKHI_H_ +#define _MEI_MKHI_H_ + +#include + +#define MKHI_FEATURE_PTT 0x10 + +#define MKHI_FWCAPS_GROUP_ID 0x3 +#define MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD 6 +#define MKHI_GEN_GROUP_ID 0xFF +#define MKHI_GEN_GET_FW_VERSION_CMD 0x2 + +#define MKHI_GROUP_ID_GFX 0x30 +#define MKHI_GFX_RESET_WARN_CMD_REQ 0x0 +#define MKHI_GFX_MEMORY_READY_CMD_REQ 0x1 + +/* Allow transition to PXP mode without approval */ +#define MKHI_GFX_MEM_READY_PXP_ALLOWED 0x1 + +struct mkhi_rule_id { + __le16 rule_type; + u8 feature_id; + u8 reserved; +} __packed; + +struct mkhi_fwcaps { + struct mkhi_rule_id id; + u8 len; + u8 data[]; +} __packed; + +struct mkhi_msg_hdr { + u8 group_id; + u8 command; + u8 reserved; + u8 result; +} __packed; + +struct mkhi_msg { + struct mkhi_msg_hdr hdr; + u8 data[]; +} __packed; + +struct mkhi_gfx_mem_ready { + struct mkhi_msg_hdr hdr; + u32 flags; +} __packed; + +#endif /* _MEI_MKHI_H_ */ diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 5435604327a71e9077d242c30386af505905f9e3..704cd0caa172caec9bbd459dbe15ae7db879364c 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2003-2020, Intel Corporation. All rights reserved. + * Copyright (c) 2003-2022, Intel Corporation. All rights reserved. * Intel Management Engine Interface (Intel MEI) Linux driver */ @@ -203,7 +203,7 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* allocates and initializes the mei dev structure */ - dev = mei_me_dev_init(&pdev->dev, cfg); + dev = mei_me_dev_init(&pdev->dev, cfg, false); if (!dev) { err = -ENOMEM; goto end; diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index 6777c419a8da2a3ac3d24bd11da7da28ae7e9d3e..d46dba2df5a10af576e1156516643e4423c66136 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -257,6 +257,8 @@ static long afu_ioctl(struct file *file, unsigned int cmd, if (IS_ERR(ev_ctx)) return PTR_ERR(ev_ctx); rc = ocxl_irq_set_handler(ctx, irq_id, irq_handler, irq_free, ev_ctx); + if (rc) + eventfd_ctx_put(ev_ctx); break; case OCXL_IOCTL_GET_METADATA: diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 8f786a225dcf88fb0ce46185b331976c96226ce7..11530b4ec3892194d6b3180edec5e26e5dd86af2 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -332,6 +332,22 @@ static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test, return false; } +static int pci_endpoint_test_validate_xfer_params(struct device *dev, + struct pci_endpoint_test_xfer_param *param, size_t alignment) +{ + if (!param->size) { + dev_dbg(dev, "Data size is zero\n"); + return -EINVAL; + } + + if (param->size > SIZE_MAX - alignment) { + dev_dbg(dev, "Maximum transfer data size exceeded\n"); + return -EINVAL; + } + + return 0; +} + static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, unsigned long arg) { @@ -363,9 +379,11 @@ static bool pci_endpoint_test_copy(struct pci_endpoint_test *test, return false; } + err = pci_endpoint_test_validate_xfer_params(dev, ¶m, alignment); + if (err) + return false; + size = param.size; - if (size > SIZE_MAX - alignment) - goto err; use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); if (use_dma) @@ -497,9 +515,11 @@ static bool pci_endpoint_test_write(struct pci_endpoint_test *test, return false; } + err = pci_endpoint_test_validate_xfer_params(dev, ¶m, alignment); + if (err) + return false; + size = param.size; - if (size > SIZE_MAX - alignment) - goto err; use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); if (use_dma) @@ -595,9 +615,11 @@ static bool pci_endpoint_test_read(struct pci_endpoint_test *test, return false; } + err = pci_endpoint_test_validate_xfer_params(dev, ¶m, alignment); + if (err) + return false; + size = param.size; - if (size > SIZE_MAX - alignment) - goto err; use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA); if (use_dma) diff --git a/drivers/misc/sgi-xp/xp.h b/drivers/misc/sgi-xp/xp.h index 9f9af77f8d2e13107241fe4bc6bcffd263cae014..f1336f43d3bd13eeeb64820c92c8523c7fe6f890 100644 --- a/drivers/misc/sgi-xp/xp.h +++ b/drivers/misc/sgi-xp/xp.h @@ -334,10 +334,6 @@ extern int (*xp_cpu_to_nasid) (int); extern enum xp_retval (*xp_expand_memprotect) (unsigned long, unsigned long); extern enum xp_retval (*xp_restrict_memprotect) (unsigned long, unsigned long); -extern u64 xp_nofault_PIOR_target; -extern int xp_nofault_PIOR(void *); -extern int xp_error_PIOR(void); - extern struct device *xp; extern enum xp_retval xp_init_uv(void); extern void xp_exit_uv(void); diff --git a/drivers/misc/tsl2550.c b/drivers/misc/tsl2550.c index 6d71865c80427ed535690b8fa436ee4afe5c4c00..1652fb9b3856ce2a425dc9c1dc7780f6efde9f6a 100644 --- a/drivers/misc/tsl2550.c +++ b/drivers/misc/tsl2550.c @@ -389,7 +389,7 @@ exit: return err; } -static int tsl2550_remove(struct i2c_client *client) +static void tsl2550_remove(struct i2c_client *client) { sysfs_remove_group(&client->dev.kobj, &tsl2550_attr_group); @@ -397,8 +397,6 @@ static int tsl2550_remove(struct i2c_client *client) tsl2550_set_power_state(client, 0); kfree(i2c_get_clientdata(client)); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index 8f2de1893245ae91140629e3923b5bd26d09ae8d..e71068f7759b34b6ac650df1272ff724879691cc 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -324,7 +324,7 @@ static void *qp_alloc_queue(u64 size, u32 flags) /* * Copies from a given buffer or iovector to a VMCI Queue. Uses - * kmap()/kunmap() to dynamically map/unmap required portions of the queue + * kmap_local_page() to dynamically map required portions of the queue * by traversing the offset -> page translation structure for the queue. * Assumes that offset + size does not wrap around in the queue. */ @@ -345,7 +345,7 @@ static int qp_memcpy_to_queue_iter(struct vmci_queue *queue, size_t to_copy; if (kernel_if->host) - va = kmap(kernel_if->u.h.page[page_index]); + va = kmap_local_page(kernel_if->u.h.page[page_index]); else va = kernel_if->u.g.vas[page_index + 1]; /* Skip header. */ @@ -359,12 +359,12 @@ static int qp_memcpy_to_queue_iter(struct vmci_queue *queue, if (!copy_from_iter_full((u8 *)va + page_offset, to_copy, from)) { if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_local(va); return VMCI_ERROR_INVALID_ARGS; } bytes_copied += to_copy; if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_local(va); } return VMCI_SUCCESS; @@ -372,7 +372,7 @@ static int qp_memcpy_to_queue_iter(struct vmci_queue *queue, /* * Copies to a given buffer or iovector from a VMCI Queue. Uses - * kmap()/kunmap() to dynamically map/unmap required portions of the queue + * kmap_local_page() to dynamically map required portions of the queue * by traversing the offset -> page translation structure for the queue. * Assumes that offset + size does not wrap around in the queue. */ @@ -393,7 +393,7 @@ static int qp_memcpy_from_queue_iter(struct iov_iter *to, int err; if (kernel_if->host) - va = kmap(kernel_if->u.h.page[page_index]); + va = kmap_local_page(kernel_if->u.h.page[page_index]); else va = kernel_if->u.g.vas[page_index + 1]; /* Skip header. */ @@ -407,12 +407,12 @@ static int qp_memcpy_from_queue_iter(struct iov_iter *to, err = copy_to_iter((u8 *)va + page_offset, to_copy, to); if (err != to_copy) { if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_local(va); return VMCI_ERROR_INVALID_ARGS; } bytes_copied += to_copy; if (kernel_if->host) - kunmap(kernel_if->u.h.page[page_index]); + kunmap_local(va); } return VMCI_SUCCESS; diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c index d6e3c650bd112bf5aaf3f2e075641a5d0c98e06f..cb9506f9cbd07077756625a9621580a6ec6269b9 100644 --- a/drivers/misc/xilinx_sdfec.c +++ b/drivers/misc/xilinx_sdfec.c @@ -636,7 +636,7 @@ static int xsdfec_table_write(struct xsdfec_dev *xsdfec, u32 offset, } for (i = 0; i < nr_pages; i++) { - addr = kmap(pages[i]); + addr = kmap_local_page(pages[i]); do { xsdfec_regwrite(xsdfec, base_addr + ((offset + reg) * @@ -645,6 +645,7 @@ static int xsdfec_table_write(struct xsdfec_dev *xsdfec, u32 offset, reg++; } while ((reg < len) && ((reg * XSDFEC_REG_WIDTH_JUMP) % PAGE_SIZE)); + kunmap_local(addr); unpin_user_page(pages[i]); } return 0; diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index ce89611a136e919e66d3c9440cf49feed898c2b0..54cd009aee50e9870db70eac04f8490e650bc6e0 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -1140,8 +1140,12 @@ static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req) { struct mmc_blk_data *md = mq->blkdata; struct mmc_card *card = md->queue.card; + unsigned int arg = card->erase_arg; - mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, card->erase_arg); + if (mmc_card_broken_sd_discard(card)) + arg = SD_ERASE_ARG; + + mmc_blk_issue_erase_rq(mq, req, MMC_BLK_DISCARD, arg); } static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq, diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h index 99045e138ba48bad3390d70024dbccf015104878..cfdd1ff40b865e4a4a8b4be002170106f4fc3e57 100644 --- a/drivers/mmc/core/card.h +++ b/drivers/mmc/core/card.h @@ -73,6 +73,7 @@ struct mmc_fixup { #define EXT_CSD_REV_ANY (-1u) #define CID_MANFID_SANDISK 0x2 +#define CID_MANFID_SANDISK_SD 0x3 #define CID_MANFID_ATP 0x9 #define CID_MANFID_TOSHIBA 0x11 #define CID_MANFID_MICRON 0x13 @@ -258,4 +259,9 @@ static inline int mmc_card_broken_hpi(const struct mmc_card *c) return c->quirks & MMC_QUIRK_BROKEN_HPI; } +static inline int mmc_card_broken_sd_discard(const struct mmc_card *c) +{ + return c->quirks & MMC_QUIRK_BROKEN_SD_DISCARD; +} + #endif diff --git a/drivers/mmc/core/core.c b/drivers/mmc/core/core.c index ef53a25788248ed6d57472f7a9cf26ee979f254e..95fa8fb1d45f2b5475805147b0bcf40b601abbfd 100644 --- a/drivers/mmc/core/core.c +++ b/drivers/mmc/core/core.c @@ -97,8 +97,8 @@ static void mmc_should_fail_request(struct mmc_host *host, !should_fail(&host->fail_mmc_request, data->blksz * data->blocks)) return; - data->error = data_errors[prandom_u32() % ARRAY_SIZE(data_errors)]; - data->bytes_xfered = (prandom_u32() % (data->bytes_xfered >> 9)) << 9; + data->error = data_errors[prandom_u32_max(ARRAY_SIZE(data_errors))]; + data->bytes_xfered = prandom_u32_max(data->bytes_xfered >> 9) << 9; } #else /* CONFIG_FAIL_MMC_REQUEST */ diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c index 0fd91f749b3aec6840ea32c985aacd0cfd5de52f..b89dca1f15e9c6da4203e74c18cd1a97b4d2dd67 100644 --- a/drivers/mmc/core/host.c +++ b/drivers/mmc/core/host.c @@ -565,7 +565,7 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev) spin_lock_init(&host->lock); init_waitqueue_head(&host->wq); INIT_DELAYED_WORK(&host->detect, mmc_rescan); - INIT_DELAYED_WORK(&host->sdio_irq_work, sdio_irq_work); + INIT_WORK(&host->sdio_irq_work, sdio_irq_work); timer_setup(&host->retune_timer, mmc_retune_timer, 0); /* diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h index be4393988086828e807f5ab85898ce25a666d45d..29b9497936df99d73e293babf7117d4ca53da2ce 100644 --- a/drivers/mmc/core/quirks.h +++ b/drivers/mmc/core/quirks.h @@ -100,6 +100,12 @@ static const struct mmc_fixup __maybe_unused mmc_blk_fixups[] = { MMC_FIXUP("V10016", CID_MANFID_KINGSTON, CID_OEMID_ANY, add_quirk_mmc, MMC_QUIRK_TRIM_BROKEN), + /* + * Some SD cards reports discard support while they don't + */ + MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_SD, 0x5344, add_quirk_sd, + MMC_QUIRK_BROKEN_SD_DISCARD), + END_FIXUP }; diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c index cee4c0b59f43d62e697667cd350ebce1ff06561c..3662bf5320ce56d1bc4b9c8664a32c83f91db292 100644 --- a/drivers/mmc/core/sd.c +++ b/drivers/mmc/core/sd.c @@ -870,7 +870,8 @@ try_again: * the CCS bit is set as well. We deliberately deviate from the spec in * regards to this, which allows UHS-I to be supported for SDSC cards. */ - if (!mmc_host_is_spi(host) && rocr && (*rocr & SD_ROCR_S18A)) { + if (!mmc_host_is_spi(host) && (ocr & SD_OCR_S18R) && + rocr && (*rocr & SD_ROCR_S18A)) { err = mmc_set_uhs_voltage(host, pocr); if (err == -EAGAIN) { retries--; @@ -949,15 +950,16 @@ int mmc_sd_setup_card(struct mmc_host *host, struct mmc_card *card, /* Erase init depends on CSD and SSR */ mmc_init_erase(card); - - /* - * Fetch switch information from card. - */ - err = mmc_read_switch(card); - if (err) - return err; } + /* + * Fetch switch information from card. Note, sd3_bus_mode can change if + * voltage switch outcome changes, so do this always. + */ + err = mmc_read_switch(card); + if (err) + return err; + /* * For SPI, enable CRC as appropriate. * This CRC enable is located AFTER the reading of the @@ -1480,26 +1482,15 @@ retry: if (!v18_fixup_failed && !mmc_host_is_spi(host) && mmc_host_uhs(host) && mmc_sd_card_using_v18(card) && host->ios.signal_voltage != MMC_SIGNAL_VOLTAGE_180) { - /* - * Re-read switch information in case it has changed since - * oldcard was initialized. - */ - if (oldcard) { - err = mmc_read_switch(card); - if (err) - goto free_card; - } - if (mmc_sd_card_using_v18(card)) { - if (mmc_host_set_uhs_voltage(host) || - mmc_sd_init_uhs_card(card)) { - v18_fixup_failed = true; - mmc_power_cycle(host, ocr); - if (!oldcard) - mmc_remove_card(card); - goto retry; - } - goto done; + if (mmc_host_set_uhs_voltage(host) || + mmc_sd_init_uhs_card(card)) { + v18_fixup_failed = true; + mmc_power_cycle(host, ocr); + if (!oldcard) + mmc_remove_card(card); + goto retry; } + goto cont; } /* Initialization sequence for UHS-I cards */ @@ -1534,7 +1525,7 @@ retry: mmc_set_bus_width(host, MMC_BUS_WIDTH_4); } } - +cont: if (!oldcard) { /* Read/parse the extension registers. */ err = sd_read_ext_regs(card); @@ -1566,7 +1557,7 @@ retry: err = -EINVAL; goto free_card; } -done: + host->card = card; return 0; diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c index 0b682a31cd3e8ad9199fea87670413dcfb30ef9c..f64b9ac76a5cdb66fc6ba7c7b1374b2734b17154 100644 --- a/drivers/mmc/core/sdio.c +++ b/drivers/mmc/core/sdio.c @@ -1043,7 +1043,7 @@ static int mmc_sdio_suspend(struct mmc_host *host) /* Prevent processing of SDIO IRQs in suspended state. */ mmc_card_set_suspended(host->card); - cancel_delayed_work_sync(&host->sdio_irq_work); + cancel_work_sync(&host->sdio_irq_work); mmc_claim_host(host); @@ -1103,7 +1103,7 @@ static int mmc_sdio_resume(struct mmc_host *host) if (!(host->caps2 & MMC_CAP2_SDIO_IRQ_NOTHREAD)) wake_up_process(host->sdio_irq_thread); else if (host->caps & MMC_CAP_SDIO_IRQ) - queue_delayed_work(system_wq, &host->sdio_irq_work, 0); + schedule_work(&host->sdio_irq_work); } out: diff --git a/drivers/mmc/core/sdio_irq.c b/drivers/mmc/core/sdio_irq.c index 4b1f7c966ec8449e7e2f5493b7ebc18ec111ac74..2b24bdf38296078cccb02b0d44f8a327d1888370 100644 --- a/drivers/mmc/core/sdio_irq.c +++ b/drivers/mmc/core/sdio_irq.c @@ -124,7 +124,7 @@ static void sdio_run_irqs(struct mmc_host *host) void sdio_irq_work(struct work_struct *work) { struct mmc_host *host = - container_of(work, struct mmc_host, sdio_irq_work.work); + container_of(work, struct mmc_host, sdio_irq_work); sdio_run_irqs(host); } @@ -132,7 +132,7 @@ void sdio_irq_work(struct work_struct *work) void sdio_signal_irq(struct mmc_host *host) { host->sdio_irq_pending = true; - queue_delayed_work(system_wq, &host->sdio_irq_work, 0); + schedule_work(&host->sdio_irq_work); } EXPORT_SYMBOL_GPL(sdio_signal_irq); diff --git a/drivers/mmc/core/sdio_uart.c b/drivers/mmc/core/sdio_uart.c index 414aa82abc39bf3f01112d16c5dad996ddc992ed..ae7ef2e038be127bf0f82a9ca8aa85c120b6fa95 100644 --- a/drivers/mmc/core/sdio_uart.c +++ b/drivers/mmc/core/sdio_uart.c @@ -246,7 +246,7 @@ static inline void sdio_uart_update_mctrl(struct sdio_uart_port *port, static void sdio_uart_change_speed(struct sdio_uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned char cval, fcr = 0; unsigned int baud, quot; @@ -859,7 +859,7 @@ static void sdio_uart_unthrottle(struct tty_struct *tty) } static void sdio_uart_set_termios(struct tty_struct *tty, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct sdio_uart_port *port = tty->driver_data; unsigned int cflag = tty->termios.c_cflag; diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index 10c563999d3dab719e1c58f47399ed99909ba5f7..f324daadaf7019c2182fdff7f989f18d9fe124eb 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -157,6 +157,7 @@ config MMC_SDHCI_OF_ARASAN config MMC_SDHCI_OF_ASPEED tristate "SDHCI OF support for the ASPEED SDHCI controller" + depends on ARCH_ASPEED || COMPILE_TEST depends on MMC_SDHCI_PLTFM depends on OF && OF_ADDRESS select MMC_SDHCI_IO_ACCESSORS @@ -171,6 +172,7 @@ config MMC_SDHCI_OF_ASPEED config MMC_SDHCI_OF_ASPEED_TEST bool "Tests for the ASPEED SDHCI driver" if !KUNIT_ALL_TESTS depends on MMC_SDHCI_OF_ASPEED && KUNIT + depends on (MMC_SDHCI_OF_ASPEED=m || KUNIT=y) default KUNIT_ALL_TESTS help Enable KUnit tests for the ASPEED SDHCI driver. Select this diff --git a/drivers/mmc/host/au1xmmc.c b/drivers/mmc/host/au1xmmc.c index a9a0837153d87c67f6d1f564fb7160263fc6036a..c88b039dc9fbd07fb4ad2358121680c5dee648cc 100644 --- a/drivers/mmc/host/au1xmmc.c +++ b/drivers/mmc/host/au1xmmc.c @@ -1097,8 +1097,9 @@ out5: if (host->platdata && host->platdata->cd_setup && !(mmc->caps & MMC_CAP_NEEDS_POLL)) host->platdata->cd_setup(mmc, 0); -out_clk: + clk_disable_unprepare(host->clk); +out_clk: clk_put(host->clk); out_irq: free_irq(host->irq, host); diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c index 581614196a8413317c651e9d405a891e2c102add..c78bbc22e0d1eaaa05df6ca0bb38e07191f9d0c1 100644 --- a/drivers/mmc/host/dw_mmc.c +++ b/drivers/mmc/host/dw_mmc.c @@ -1858,7 +1858,7 @@ static void dw_mci_start_fault_timer(struct dw_mci *host) * Try to inject the error at random points during the data transfer. */ hrtimer_start(&host->fault_timer, - ms_to_ktime(prandom_u32() % 25), + ms_to_ktime(prandom_u32_max(25)), HRTIMER_MODE_REL); } diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c index b1d563b2ed1b01a10f2da7608a1102e95d240fd1..dc2db9c185ea0d4fe9ea7c48825b755378803e13 100644 --- a/drivers/mmc/host/jz4740_mmc.c +++ b/drivers/mmc/host/jz4740_mmc.c @@ -298,7 +298,7 @@ static int jz4740_mmc_prepare_dma_data(struct jz4740_mmc_host *host, { struct dma_chan *chan = jz4740_mmc_get_dma_chan(host, data); enum dma_data_direction dir = mmc_get_dma_dir(data); - int sg_count; + unsigned int sg_count; if (data->host_cookie == COOKIE_PREMAPPED) return data->sg_count; @@ -308,7 +308,7 @@ static int jz4740_mmc_prepare_dma_data(struct jz4740_mmc_host *host, data->sg_len, dir); - if (sg_count <= 0) { + if (!sg_count) { dev_err(mmc_dev(host->mmc), "Failed to map scatterlist for DMA operation\n"); return -EINVAL; diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c index fc462995cf94a40201d6dfab17bfeb03b9efcd21..df05e60bed9a23a497768bc836bd072dc26fb6be 100644 --- a/drivers/mmc/host/meson-gx-mmc.c +++ b/drivers/mmc/host/meson-gx-mmc.c @@ -41,14 +41,17 @@ #define CLK_V2_TX_DELAY_MASK GENMASK(19, 16) #define CLK_V2_RX_DELAY_MASK GENMASK(23, 20) #define CLK_V2_ALWAYS_ON BIT(24) +#define CLK_V2_IRQ_SDIO_SLEEP BIT(25) #define CLK_V3_TX_DELAY_MASK GENMASK(21, 16) #define CLK_V3_RX_DELAY_MASK GENMASK(27, 22) #define CLK_V3_ALWAYS_ON BIT(28) +#define CLK_V3_IRQ_SDIO_SLEEP BIT(29) #define CLK_TX_DELAY_MASK(h) (h->data->tx_delay_mask) #define CLK_RX_DELAY_MASK(h) (h->data->rx_delay_mask) #define CLK_ALWAYS_ON(h) (h->data->always_on) +#define CLK_IRQ_SDIO_SLEEP(h) (h->data->irq_sdio_sleep) #define SD_EMMC_DELAY 0x4 #define SD_EMMC_ADJUST 0x8 @@ -101,8 +104,7 @@ #define IRQ_RESP_STATUS BIT(14) #define IRQ_SDIO BIT(15) #define IRQ_EN_MASK \ - (IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN | IRQ_RESP_STATUS |\ - IRQ_SDIO) + (IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN) #define SD_EMMC_CMD_CFG 0x50 #define SD_EMMC_CMD_ARG 0x54 @@ -136,6 +138,7 @@ struct meson_mmc_data { unsigned int rx_delay_mask; unsigned int always_on; unsigned int adjust; + unsigned int irq_sdio_sleep; }; struct sd_emmc_desc { @@ -175,6 +178,7 @@ struct meson_host { bool vqmmc_enabled; bool needs_pre_post_req; + spinlock_t lock; }; #define CMD_CFG_LENGTH_MASK GENMASK(8, 0) @@ -431,6 +435,7 @@ static int meson_mmc_clk_init(struct meson_host *host) clk_reg |= FIELD_PREP(CLK_CORE_PHASE_MASK, CLK_PHASE_180); clk_reg |= FIELD_PREP(CLK_TX_PHASE_MASK, CLK_PHASE_0); clk_reg |= FIELD_PREP(CLK_RX_PHASE_MASK, CLK_PHASE_0); + clk_reg |= CLK_IRQ_SDIO_SLEEP(host); writel(clk_reg, host->regs + SD_EMMC_CLOCK); /* get the mux parents */ @@ -929,33 +934,54 @@ static void meson_mmc_read_resp(struct mmc_host *mmc, struct mmc_command *cmd) } } +static void __meson_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable) +{ + struct meson_host *host = mmc_priv(mmc); + u32 reg_irqen = IRQ_EN_MASK; + + if (enable) + reg_irqen |= IRQ_SDIO; + writel(reg_irqen, host->regs + SD_EMMC_IRQ_EN); +} + static irqreturn_t meson_mmc_irq(int irq, void *dev_id) { struct meson_host *host = dev_id; struct mmc_command *cmd; - struct mmc_data *data; - u32 irq_en, status, raw_status; + u32 status, raw_status; irqreturn_t ret = IRQ_NONE; - irq_en = readl(host->regs + SD_EMMC_IRQ_EN); raw_status = readl(host->regs + SD_EMMC_STATUS); - status = raw_status & irq_en; + status = raw_status & (IRQ_EN_MASK | IRQ_SDIO); if (!status) { dev_dbg(host->dev, - "Unexpected IRQ! irq_en 0x%08x - status 0x%08x\n", - irq_en, raw_status); + "Unexpected IRQ! irq_en 0x%08lx - status 0x%08x\n", + IRQ_EN_MASK | IRQ_SDIO, raw_status); return IRQ_NONE; } - if (WARN_ON(!host) || WARN_ON(!host->cmd)) + if (WARN_ON(!host)) return IRQ_NONE; /* ack all raised interrupts */ writel(status, host->regs + SD_EMMC_STATUS); cmd = host->cmd; - data = cmd->data; + + if (status & IRQ_SDIO) { + spin_lock(&host->lock); + __meson_mmc_enable_sdio_irq(host->mmc, 0); + sdio_signal_irq(host->mmc); + spin_unlock(&host->lock); + status &= ~IRQ_SDIO; + if (!status) + return IRQ_HANDLED; + } + + if (WARN_ON(!cmd)) + return IRQ_NONE; + cmd->error = 0; if (status & IRQ_CRC_ERR) { dev_dbg(host->dev, "CRC Error - status 0x%08x\n", status); @@ -973,12 +999,9 @@ static irqreturn_t meson_mmc_irq(int irq, void *dev_id) meson_mmc_read_resp(host->mmc, cmd); - if (status & IRQ_SDIO) { - dev_dbg(host->dev, "IRQ: SDIO TODO.\n"); - ret = IRQ_HANDLED; - } - if (status & (IRQ_END_OF_CHAIN | IRQ_RESP_STATUS)) { + struct mmc_data *data = cmd->data; + if (data && !cmd->error) data->bytes_xfered = data->blksz * data->blocks; if (meson_mmc_bounce_buf_read(data) || @@ -1121,6 +1144,21 @@ static int meson_mmc_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios) return -EINVAL; } +static void meson_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable) +{ + struct meson_host *host = mmc_priv(mmc); + unsigned long flags; + + spin_lock_irqsave(&host->lock, flags); + __meson_mmc_enable_sdio_irq(mmc, enable); + spin_unlock_irqrestore(&host->lock, flags); +} + +static void meson_mmc_ack_sdio_irq(struct mmc_host *mmc) +{ + meson_mmc_enable_sdio_irq(mmc, 1); +} + static const struct mmc_host_ops meson_mmc_ops = { .request = meson_mmc_request, .set_ios = meson_mmc_set_ios, @@ -1130,6 +1168,8 @@ static const struct mmc_host_ops meson_mmc_ops = { .execute_tuning = meson_mmc_resampling_tuning, .card_busy = meson_mmc_card_busy, .start_signal_voltage_switch = meson_mmc_voltage_switch, + .enable_sdio_irq = meson_mmc_enable_sdio_irq, + .ack_sdio_irq = meson_mmc_ack_sdio_irq, }; static int meson_mmc_probe(struct platform_device *pdev) @@ -1226,10 +1266,8 @@ static int meson_mmc_probe(struct platform_device *pdev) /* clear, ack and enable interrupts */ writel(0, host->regs + SD_EMMC_IRQ_EN); - writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN, - host->regs + SD_EMMC_STATUS); - writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN, - host->regs + SD_EMMC_IRQ_EN); + writel(IRQ_EN_MASK, host->regs + SD_EMMC_STATUS); + writel(IRQ_EN_MASK, host->regs + SD_EMMC_IRQ_EN); ret = request_threaded_irq(host->irq, meson_mmc_irq, meson_mmc_irq_thread, IRQF_ONESHOT, @@ -1237,7 +1275,13 @@ static int meson_mmc_probe(struct platform_device *pdev) if (ret) goto err_init_clk; + spin_lock_init(&host->lock); + mmc->caps |= MMC_CAP_CMD23; + + if (mmc->caps & MMC_CAP_SDIO_IRQ) + mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; + if (host->dram_access_quirk) { /* Limit segments to 1 due to low available sram memory */ mmc->max_segs = 1; @@ -1328,6 +1372,7 @@ static const struct meson_mmc_data meson_gx_data = { .rx_delay_mask = CLK_V2_RX_DELAY_MASK, .always_on = CLK_V2_ALWAYS_ON, .adjust = SD_EMMC_ADJUST, + .irq_sdio_sleep = CLK_V2_IRQ_SDIO_SLEEP, }; static const struct meson_mmc_data meson_axg_data = { @@ -1335,6 +1380,7 @@ static const struct meson_mmc_data meson_axg_data = { .rx_delay_mask = CLK_V3_RX_DELAY_MASK, .always_on = CLK_V3_ALWAYS_ON, .adjust = SD_EMMC_V3_ADJUST, + .irq_sdio_sleep = CLK_V3_IRQ_SDIO_SLEEP, }; static const struct of_device_id meson_mmc_of_match[] = { diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c index e92e63cb5641cf30beb9d076d8cff3609373cd9f..da85c2f2acb83f3072091f416f0a8c2277ed0c92 100644 --- a/drivers/mmc/host/meson-mx-sdhc-mmc.c +++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c @@ -381,14 +381,14 @@ static void meson_mx_sdhc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) static int meson_mx_sdhc_map_dma(struct mmc_host *mmc, struct mmc_request *mrq) { struct mmc_data *data = mrq->data; - int dma_len; + unsigned int dma_len; if (!data) return 0; dma_len = dma_map_sg(mmc_dev(mmc), data->sg, data->sg_len, mmc_get_dma_dir(data)); - if (dma_len <= 0) { + if (!dma_len) { dev_err(mmc_dev(mmc), "dma_map_sg failed\n"); return -ENOMEM; } diff --git a/drivers/mmc/host/mmc_hsq.c b/drivers/mmc/host/mmc_hsq.c index a5e05ed0fda3ebca5369ee35544beda9b9dafdc9..9d35453e7371b8e0d644b905a8bd3f50475ca5b5 100644 --- a/drivers/mmc/host/mmc_hsq.c +++ b/drivers/mmc/host/mmc_hsq.c @@ -34,7 +34,7 @@ static void mmc_hsq_pump_requests(struct mmc_hsq *hsq) spin_lock_irqsave(&hsq->lock, flags); /* Make sure we are not already running a request now */ - if (hsq->mrq) { + if (hsq->mrq || hsq->recovery_halt) { spin_unlock_irqrestore(&hsq->lock, flags); return; } diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c index b6eb75f4bbfc6d1be06dc9556d7e1cf02ac88bb6..dfc3ffd5b1f8c2cde263848dcf09b533a9a443c2 100644 --- a/drivers/mmc/host/moxart-mmc.c +++ b/drivers/mmc/host/moxart-mmc.c @@ -111,8 +111,8 @@ #define CLK_DIV_MASK 0x7f /* REG_BUS_WIDTH */ -#define BUS_WIDTH_8 BIT(2) -#define BUS_WIDTH_4 BIT(1) +#define BUS_WIDTH_4_SUPPORT BIT(3) +#define BUS_WIDTH_4 BIT(2) #define BUS_WIDTH_1 BIT(0) #define MMC_VDD_360 23 @@ -524,9 +524,6 @@ static void moxart_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) case MMC_BUS_WIDTH_4: writel(BUS_WIDTH_4, host->base + REG_BUS_WIDTH); break; - case MMC_BUS_WIDTH_8: - writel(BUS_WIDTH_8, host->base + REG_BUS_WIDTH); - break; default: writel(BUS_WIDTH_1, host->base + REG_BUS_WIDTH); break; @@ -651,16 +648,8 @@ static int moxart_probe(struct platform_device *pdev) dmaengine_slave_config(host->dma_chan_rx, &cfg); } - switch ((readl(host->base + REG_BUS_WIDTH) >> 3) & 3) { - case 1: + if (readl(host->base + REG_BUS_WIDTH) & BUS_WIDTH_4_SUPPORT) mmc->caps |= MMC_CAP_4_BIT_DATA; - break; - case 2: - mmc->caps |= MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA; - break; - default: - break; - } writel(0, host->base + REG_INTERRUPT_MASK); diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c index 69d78604d1fc3f2d09ab507965ee9825de0a0789..df941438aef57610062521bb6ba9c0eb2ed354d7 100644 --- a/drivers/mmc/host/mtk-sd.c +++ b/drivers/mmc/host/mtk-sd.c @@ -474,33 +474,20 @@ struct msdc_host { struct cqhci_host *cq_host; }; -static const struct mtk_mmc_compatible mt8135_compat = { - .clk_div_bits = 8, +static const struct mtk_mmc_compatible mt2701_compat = { + .clk_div_bits = 12, .recheck_sdio_irq = true, .hs400_tune = false, - .pad_tune_reg = MSDC_PAD_TUNE, - .async_fifo = false, - .data_tune = false, - .busy_check = false, - .stop_clk_fix = false, - .enhance_rx = false, - .support_64g = false, -}; - -static const struct mtk_mmc_compatible mt8173_compat = { - .clk_div_bits = 8, - .recheck_sdio_irq = true, - .hs400_tune = true, - .pad_tune_reg = MSDC_PAD_TUNE, - .async_fifo = false, - .data_tune = false, + .pad_tune_reg = MSDC_PAD_TUNE0, + .async_fifo = true, + .data_tune = true, .busy_check = false, .stop_clk_fix = false, .enhance_rx = false, .support_64g = false, }; -static const struct mtk_mmc_compatible mt8183_compat = { +static const struct mtk_mmc_compatible mt2712_compat = { .clk_div_bits = 12, .recheck_sdio_irq = false, .hs400_tune = false, @@ -513,30 +500,43 @@ static const struct mtk_mmc_compatible mt8183_compat = { .support_64g = true, }; -static const struct mtk_mmc_compatible mt2701_compat = { +static const struct mtk_mmc_compatible mt6779_compat = { .clk_div_bits = 12, - .recheck_sdio_irq = true, + .recheck_sdio_irq = false, .hs400_tune = false, .pad_tune_reg = MSDC_PAD_TUNE0, .async_fifo = true, .data_tune = true, + .busy_check = true, + .stop_clk_fix = true, + .enhance_rx = true, + .support_64g = true, +}; + +static const struct mtk_mmc_compatible mt6795_compat = { + .clk_div_bits = 8, + .recheck_sdio_irq = false, + .hs400_tune = true, + .pad_tune_reg = MSDC_PAD_TUNE, + .async_fifo = false, + .data_tune = false, .busy_check = false, .stop_clk_fix = false, .enhance_rx = false, .support_64g = false, }; -static const struct mtk_mmc_compatible mt2712_compat = { - .clk_div_bits = 12, - .recheck_sdio_irq = false, +static const struct mtk_mmc_compatible mt7620_compat = { + .clk_div_bits = 8, + .recheck_sdio_irq = true, .hs400_tune = false, - .pad_tune_reg = MSDC_PAD_TUNE0, - .async_fifo = true, - .data_tune = true, - .busy_check = true, - .stop_clk_fix = true, - .enhance_rx = true, - .support_64g = true, + .pad_tune_reg = MSDC_PAD_TUNE, + .async_fifo = false, + .data_tune = false, + .busy_check = false, + .stop_clk_fix = false, + .enhance_rx = false, + .use_internal_cd = true, }; static const struct mtk_mmc_compatible mt7622_compat = { @@ -552,31 +552,33 @@ static const struct mtk_mmc_compatible mt7622_compat = { .support_64g = false, }; -static const struct mtk_mmc_compatible mt8516_compat = { - .clk_div_bits = 12, +static const struct mtk_mmc_compatible mt8135_compat = { + .clk_div_bits = 8, .recheck_sdio_irq = true, .hs400_tune = false, - .pad_tune_reg = MSDC_PAD_TUNE0, - .async_fifo = true, - .data_tune = true, - .busy_check = true, - .stop_clk_fix = true, + .pad_tune_reg = MSDC_PAD_TUNE, + .async_fifo = false, + .data_tune = false, + .busy_check = false, + .stop_clk_fix = false, + .enhance_rx = false, + .support_64g = false, }; -static const struct mtk_mmc_compatible mt7620_compat = { +static const struct mtk_mmc_compatible mt8173_compat = { .clk_div_bits = 8, .recheck_sdio_irq = true, - .hs400_tune = false, + .hs400_tune = true, .pad_tune_reg = MSDC_PAD_TUNE, .async_fifo = false, .data_tune = false, .busy_check = false, .stop_clk_fix = false, .enhance_rx = false, - .use_internal_cd = true, + .support_64g = false, }; -static const struct mtk_mmc_compatible mt6779_compat = { +static const struct mtk_mmc_compatible mt8183_compat = { .clk_div_bits = 12, .recheck_sdio_irq = false, .hs400_tune = false, @@ -589,16 +591,29 @@ static const struct mtk_mmc_compatible mt6779_compat = { .support_64g = true, }; +static const struct mtk_mmc_compatible mt8516_compat = { + .clk_div_bits = 12, + .recheck_sdio_irq = true, + .hs400_tune = false, + .pad_tune_reg = MSDC_PAD_TUNE0, + .async_fifo = true, + .data_tune = true, + .busy_check = true, + .stop_clk_fix = true, +}; + static const struct of_device_id msdc_of_ids[] = { - { .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat}, - { .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat}, - { .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat}, { .compatible = "mediatek,mt2701-mmc", .data = &mt2701_compat}, { .compatible = "mediatek,mt2712-mmc", .data = &mt2712_compat}, + { .compatible = "mediatek,mt6779-mmc", .data = &mt6779_compat}, + { .compatible = "mediatek,mt6795-mmc", .data = &mt6795_compat}, + { .compatible = "mediatek,mt7620-mmc", .data = &mt7620_compat}, { .compatible = "mediatek,mt7622-mmc", .data = &mt7622_compat}, + { .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat}, + { .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat}, + { .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat}, { .compatible = "mediatek,mt8516-mmc", .data = &mt8516_compat}, - { .compatible = "mediatek,mt7620-mmc", .data = &mt7620_compat}, - { .compatible = "mediatek,mt6779-mmc", .data = &mt6779_compat}, + {} }; MODULE_DEVICE_TABLE(of, msdc_of_ids); diff --git a/drivers/mmc/host/renesas_sdhi_core.c b/drivers/mmc/host/renesas_sdhi_core.c index 6edbf5c161ab94958b103d87b469c580fa97a261..b970699743e0a168cf452596ca77604e8c5a5d3e 100644 --- a/drivers/mmc/host/renesas_sdhi_core.c +++ b/drivers/mmc/host/renesas_sdhi_core.c @@ -128,6 +128,7 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, struct clk *ref_clk = priv->clk; unsigned int freq, diff, best_freq = 0, diff_min = ~0; unsigned int new_clock, clkh_shift = 0; + unsigned int new_upper_limit; int i; /* @@ -153,13 +154,20 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, * greater than, new_clock. As we can divide by 1 << i for * any i in [0, 9] we want the input clock to be as close as * possible, but no greater than, new_clock << i. + * + * Add an upper limit of 1/1024 rate higher to the clock rate to fix + * clk rate jumping to lower rate due to rounding error (eg: RZ/G2L has + * 3 clk sources 533.333333 MHz, 400 MHz and 266.666666 MHz. The request + * for 533.333333 MHz will selects a slower 400 MHz due to rounding + * error (533333333 Hz / 4 * 4 = 533333332 Hz < 533333333 Hz)). */ for (i = min(9, ilog2(UINT_MAX / new_clock)); i >= 0; i--) { freq = clk_round_rate(ref_clk, new_clock << i); - if (freq > (new_clock << i)) { + new_upper_limit = (new_clock << i) + ((new_clock << i) >> 10); + if (freq > new_upper_limit) { /* Too fast; look for a slightly slower option */ freq = clk_round_rate(ref_clk, (new_clock << i) / 4 * 3); - if (freq > (new_clock << i)) + if (freq > new_upper_limit) continue; } @@ -181,6 +189,7 @@ static unsigned int renesas_sdhi_clk_update(struct tmio_mmc_host *host, static void renesas_sdhi_set_clock(struct tmio_mmc_host *host, unsigned int new_clock) { + unsigned int clk_margin; u32 clk = 0, clock; sd_ctrl_write16(host, CTL_SD_CARD_CLK_CTL, ~CLK_CTL_SCLKEN & @@ -194,7 +203,13 @@ static void renesas_sdhi_set_clock(struct tmio_mmc_host *host, host->mmc->actual_clock = renesas_sdhi_clk_update(host, new_clock); clock = host->mmc->actual_clock / 512; - for (clk = 0x80000080; new_clock >= (clock << 1); clk >>= 1) + /* + * Add a margin of 1/1024 rate higher to the clock rate in order + * to avoid clk variable setting a value of 0 due to the margin + * provided for actual_clock in renesas_sdhi_clk_update(). + */ + clk_margin = new_clock >> 10; + for (clk = 0x80000080; new_clock + clk_margin >= (clock << 1); clk >>= 1) clock <<= 1; /* 1/1 clock is option */ diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c index 5fe4528e296e68719a196786dedc91cf25559615..5798aee066531c31077ed80cc96bad300b026164 100644 --- a/drivers/mmc/host/rtsx_usb_sdmmc.c +++ b/drivers/mmc/host/rtsx_usb_sdmmc.c @@ -1042,7 +1042,6 @@ static int sd_set_timing(struct rtsx_usb_sdmmc *host, unsigned char timing, bool *ddr_mode) { struct rtsx_ucr *ucr = host->ucr; - int err; *ddr_mode = false; @@ -1097,9 +1096,7 @@ static int sd_set_timing(struct rtsx_usb_sdmmc *host, break; } - err = rtsx_usb_send_cmd(ucr, MODE_C, 100); - - return err; + return rtsx_usb_send_cmd(ucr, MODE_C, 100); } static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index dc2991422a87331e4a4142551139d1754bca9591..3a091a387ecbceb38eae9935b6a1dc151600d45a 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -2441,6 +2441,7 @@ static const struct of_device_id sdhci_msm_dt_match[] = { */ {.compatible = "qcom,sdhci-msm-v4", .data = &sdhci_msm_mci_var}, {.compatible = "qcom,sdhci-msm-v5", .data = &sdhci_msm_v5_var}, + {.compatible = "qcom,sdm670-sdhci", .data = &sdm845_sdhci_var}, {.compatible = "qcom,sdm845-sdhci", .data = &sdm845_sdhci_var}, {.compatible = "qcom,sc7180-sdhci", .data = &sdm845_sdhci_var}, {}, diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 622b7de96c7f679428a7f9d0107d8f81a52c8431..169b84761041f44e0fe4d8dbaeaec6ad89026761 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -297,6 +297,27 @@ static const struct sdhci_pci_fixes sdhci_ricoh_mmc = { SDHCI_QUIRK_MISSING_CAPS }; +static void ene_714_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) +{ + struct sdhci_host *host = mmc_priv(mmc); + + sdhci_set_ios(mmc, ios); + + /* + * Some (ENE) controllers misbehave on some ios operations, + * signalling timeout and CRC errors even on CMD0. Resetting + * it on each ios seems to solve the problem. + */ + if (!(host->flags & SDHCI_DEVICE_DEAD)) + sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); +} + +static int ene_714_probe_slot(struct sdhci_pci_slot *slot) +{ + slot->host->mmc_host_ops.set_ios = ene_714_set_ios; + return 0; +} + static const struct sdhci_pci_fixes sdhci_ene_712 = { .quirks = SDHCI_QUIRK_SINGLE_POWER_WRITE | SDHCI_QUIRK_BROKEN_DMA, @@ -304,8 +325,8 @@ static const struct sdhci_pci_fixes sdhci_ene_712 = { static const struct sdhci_pci_fixes sdhci_ene_714 = { .quirks = SDHCI_QUIRK_SINGLE_POWER_WRITE | - SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS | SDHCI_QUIRK_BROKEN_DMA, + .probe_slot = ene_714_probe_slot, }; static const struct sdhci_pci_fixes sdhci_cafe = { diff --git a/drivers/mmc/host/sdhci-pci-o2micro.c b/drivers/mmc/host/sdhci-pci-o2micro.c index 0d4d343dbb77d19774d08f6eba2d798e68948eea..ad457cd9cbaabc8c7b6f719f00c6372f812d46b6 100644 --- a/drivers/mmc/host/sdhci-pci-o2micro.c +++ b/drivers/mmc/host/sdhci-pci-o2micro.c @@ -317,11 +317,12 @@ static int sdhci_o2_execute_tuning(struct mmc_host *mmc, u32 opcode) u32 reg_val; /* - * This handler only implements the eMMC tuning that is specific to + * This handler implements the hardware tuning that is specific to * this controller. Fall back to the standard method for other TIMING. */ if ((host->timing != MMC_TIMING_MMC_HS200) && - (host->timing != MMC_TIMING_UHS_SDR104)) + (host->timing != MMC_TIMING_UHS_SDR104) && + (host->timing != MMC_TIMING_UHS_SDR50)) return sdhci_execute_tuning(mmc, opcode); if (WARN_ON((opcode != MMC_SEND_TUNING_BLOCK_HS200) && @@ -631,6 +632,8 @@ static int sdhci_pci_o2_probe_slot(struct sdhci_pci_slot *slot) if (reg & 0x1) host->quirks |= SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12; + host->quirks2 |= SDHCI_QUIRK2_BROKEN_DDR50; + sdhci_pci_o2_enable_msi(chip, host); if (chip->pdev->device == PCI_DEVICE_ID_O2_SEABIRD0) { diff --git a/drivers/mmc/host/sdhci-sprd.c b/drivers/mmc/host/sdhci-sprd.c index f33e9349e4e62eddd6d6c3370cfdcd4a017eeee6..b92a408f138ddf66868513cb173105d1988e7de1 100644 --- a/drivers/mmc/host/sdhci-sprd.c +++ b/drivers/mmc/host/sdhci-sprd.c @@ -205,14 +205,14 @@ static inline u32 sdhci_sprd_calc_div(u32 base_clk, u32 clk) if ((base_clk / div) > (clk * 2)) div++; - if (div > SDHCI_SPRD_CLK_MAX_DIV) - div = SDHCI_SPRD_CLK_MAX_DIV; - if (div % 2) div = (div + 1) / 2; else div = div / 2; + if (div > SDHCI_SPRD_CLK_MAX_DIV) + div = SDHCI_SPRD_CLK_MAX_DIV; + return div; } @@ -309,7 +309,7 @@ static unsigned int sdhci_sprd_get_max_clock(struct sdhci_host *host) static unsigned int sdhci_sprd_get_min_clock(struct sdhci_host *host) { - return 400000; + return 100000; } static void sdhci_sprd_set_uhs_signaling(struct sdhci_host *host, diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c index 2d2d8260c6814d666879cb0b95eb3ec0760d11a6..413925bce0ca8c6587b28afcaee44f5ef332eb3f 100644 --- a/drivers/mmc/host/sdhci-tegra.c +++ b/drivers/mmc/host/sdhci-tegra.c @@ -773,7 +773,7 @@ static void tegra_sdhci_set_clock(struct sdhci_host *host, unsigned int clock) dev_err(dev, "failed to set clk rate to %luHz: %d\n", host_clk, err); - tegra_host->curr_clk_rate = host_clk; + tegra_host->curr_clk_rate = clk_get_rate(pltfm_host->clk); if (tegra_host->ddr_signaling) host->max_clk = host_clk; else diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 7689ffec5ad19bc5d0512ef7690cb46e7f48543e..fef03de85b99f3b7dd2b5a2ab46d7ecdbbcd9d7a 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -233,28 +233,62 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) } EXPORT_SYMBOL_GPL(sdhci_reset); -static void sdhci_do_reset(struct sdhci_host *host, u8 mask) +static bool sdhci_do_reset(struct sdhci_host *host, u8 mask) { if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) { struct mmc_host *mmc = host->mmc; if (!mmc->ops->get_cd(mmc)) - return; + return false; } host->ops->reset(host, mask); - if (mask & SDHCI_RESET_ALL) { + return true; +} + +static void sdhci_reset_for_all(struct sdhci_host *host) +{ + if (sdhci_do_reset(host, SDHCI_RESET_ALL)) { if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) { if (host->ops->enable_dma) host->ops->enable_dma(host); } - /* Resetting the controller clears many */ host->preset_enabled = false; } } +enum sdhci_reset_reason { + SDHCI_RESET_FOR_INIT, + SDHCI_RESET_FOR_REQUEST_ERROR, + SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY, + SDHCI_RESET_FOR_TUNING_ABORT, + SDHCI_RESET_FOR_CARD_REMOVED, + SDHCI_RESET_FOR_CQE_RECOVERY, +}; + +static void sdhci_reset_for_reason(struct sdhci_host *host, enum sdhci_reset_reason reason) +{ + switch (reason) { + case SDHCI_RESET_FOR_INIT: + sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + break; + case SDHCI_RESET_FOR_REQUEST_ERROR: + case SDHCI_RESET_FOR_TUNING_ABORT: + case SDHCI_RESET_FOR_CARD_REMOVED: + case SDHCI_RESET_FOR_CQE_RECOVERY: + sdhci_do_reset(host, SDHCI_RESET_CMD); + sdhci_do_reset(host, SDHCI_RESET_DATA); + break; + case SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY: + sdhci_do_reset(host, SDHCI_RESET_DATA); + break; + } +} + +#define sdhci_reset_for(h, r) sdhci_reset_for_reason((h), SDHCI_RESET_FOR_##r) + static void sdhci_set_default_irqs(struct sdhci_host *host) { host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT | @@ -323,9 +357,9 @@ static void sdhci_init(struct sdhci_host *host, int soft) unsigned long flags; if (soft) - sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + sdhci_reset_for(host, INIT); else - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); if (host->v4_mode) sdhci_do_enable_v4_mode(host); @@ -1538,8 +1572,9 @@ static void __sdhci_finish_data(struct sdhci_host *host, bool sw_data_timeout) */ if (data->error) { if (!host->cmd || host->cmd == data_cmd) - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, REQUEST_ERROR); + else + sdhci_reset_for(host, REQUEST_ERROR_DATA_ONLY); } if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) == @@ -2403,14 +2438,6 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) host->ops->set_clock(host, host->clock); } else sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); - - /* - * Some (ENE) controllers go apeshit on some ios operation, - * signalling timeout and CRC errors even on CMD0. Resetting - * it on each ios seems to solve the problem. - */ - if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS) - sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); } EXPORT_SYMBOL_GPL(sdhci_set_ios); @@ -2718,8 +2745,7 @@ void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode) { sdhci_reset_tuning(host); - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, TUNING_ABORT); sdhci_end_tuning(host); @@ -2987,8 +3013,7 @@ static void sdhci_card_event(struct mmc_host *mmc) pr_err("%s: Resetting controller.\n", mmc_hostname(mmc)); - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, CARD_REMOVED); sdhci_error_out_mrqs(host, -ENOMEDIUM); } @@ -3059,12 +3084,7 @@ static bool sdhci_request_done(struct sdhci_host *host) /* This is to force an update */ host->ops->set_clock(host, host->clock); - /* - * Spec says we should do both at the same time, but Ricoh - * controllers do not like that. - */ - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, REQUEST_ERROR); host->pending_reset = false; } @@ -3905,10 +3925,8 @@ void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery) host->cqe_on = false; - if (recovery) { - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); - } + if (recovery) + sdhci_reset_for(host, CQE_RECOVERY); pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n", mmc_hostname(mmc), host->ier, @@ -3928,7 +3946,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) { *cmd_error = -EILSEQ; - if (!mmc_op_tuning(host->cmd->opcode)) + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) sdhci_err_stats_inc(host, CMD_CRC); } else if (intmask & SDHCI_INT_TIMEOUT) { *cmd_error = -ETIMEDOUT; @@ -3938,7 +3956,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) { *data_error = -EILSEQ; - if (!mmc_op_tuning(host->cmd->opcode)) + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) sdhci_err_stats_inc(host, DAT_CRC); } else if (intmask & SDHCI_INT_DATA_TIMEOUT) { *data_error = -ETIMEDOUT; @@ -4066,7 +4084,7 @@ void __sdhci_read_caps(struct sdhci_host *host, const u16 *ver, if (debug_quirks2) host->quirks2 = debug_quirks2; - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); if (host->v4_mode) sdhci_do_enable_v4_mode(host); @@ -4807,7 +4825,7 @@ int __sdhci_add_host(struct sdhci_host *host) unled: sdhci_led_unregister(host); unirq: - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); sdhci_writel(host, 0, SDHCI_INT_ENABLE); sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); free_irq(host->irq, host); @@ -4865,7 +4883,7 @@ void sdhci_remove_host(struct sdhci_host *host, int dead) sdhci_led_unregister(host); if (!dead) - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); sdhci_writel(host, 0, SDHCI_INT_ENABLE); sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index 95a08f09df30eaa46fdb3081b987d16020302c81..d750c464bd1e59291fe4c85f7f3d189f3aff1587 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -379,8 +379,6 @@ struct sdhci_host { #define SDHCI_QUIRK_NO_CARD_NO_RESET (1<<2) /* Controller doesn't like clearing the power reg before a change */ #define SDHCI_QUIRK_SINGLE_POWER_WRITE (1<<3) -/* Controller has flaky internal state so reset it on each ios change */ -#define SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS (1<<4) /* Controller has an unusable DMA engine */ #define SDHCI_QUIRK_BROKEN_DMA (1<<5) /* Controller has an unusable ADMA engine */ diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c index e7ced1496a0731afba7edeaa7f4508aba50c4877..8f1023480e12ccb95ec1c81d5f44af756e3ff0b8 100644 --- a/drivers/mmc/host/sdhci_am654.c +++ b/drivers/mmc/host/sdhci_am654.c @@ -554,7 +554,6 @@ static const struct cqhci_host_ops sdhci_am654_cqhci_ops = { static int sdhci_am654_cqe_add_host(struct sdhci_host *host) { struct cqhci_host *cq_host; - int ret; cq_host = devm_kzalloc(mmc_dev(host->mmc), sizeof(struct cqhci_host), GFP_KERNEL); @@ -568,9 +567,7 @@ static int sdhci_am654_cqe_add_host(struct sdhci_host *host) host->mmc->caps2 |= MMC_CAP2_CQE; - ret = cqhci_init(cq_host, host->mmc, 1); - - return ret; + return cqhci_init(cq_host, host->mmc, 1); } static int sdhci_am654_get_otap_delay(struct sdhci_host *host, diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c index 163ac9df8cca0da734bca989d5f2317257bae551..9b5c503e3a3fcb05c0725cd4aaaa46d9d4ae9cee 100644 --- a/drivers/mmc/host/wmt-sdmmc.c +++ b/drivers/mmc/host/wmt-sdmmc.c @@ -846,7 +846,7 @@ static int wmt_mci_probe(struct platform_device *pdev) if (IS_ERR(priv->clk_sdmmc)) { dev_err(&pdev->dev, "Error getting clock\n"); ret = PTR_ERR(priv->clk_sdmmc); - goto fail5; + goto fail5_and_a_half; } ret = clk_prepare_enable(priv->clk_sdmmc); @@ -863,6 +863,9 @@ static int wmt_mci_probe(struct platform_device *pdev) return 0; fail6: clk_put(priv->clk_sdmmc); +fail5_and_a_half: + dma_free_coherent(&pdev->dev, mmc->max_blk_count * 16, + priv->dma_desc_buffer, priv->dma_desc_device_addr); fail5: free_irq(dma_irq, priv); fail4: diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c index 40d7211485da24af0f59834e2d8add3967315cf7..4cd37ec45762b6f92add0557923ff445a92f1f01 100644 --- a/drivers/mtd/devices/block2mtd.c +++ b/drivers/mtd/devices/block2mtd.c @@ -461,7 +461,7 @@ static int block2mtd_setup(const char *val, const struct kernel_param *kp) the device (even kmalloc() fails). Deter that work to block2mtd_setup2(). */ - strlcpy(block2mtd_paramline, val, sizeof(block2mtd_paramline)); + strscpy(block2mtd_paramline, val, sizeof(block2mtd_paramline)); return 0; #endif diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c index 5b0ae5ddad74554799cd4f2ede229b3f3f6eb7e8..a7714e3de887f317f1c618d1a1f4900c9b13ff50 100644 --- a/drivers/mtd/devices/docg3.c +++ b/drivers/mtd/devices/docg3.c @@ -300,7 +300,7 @@ static void doc_write_data_area(struct docg3 *docg3, const void *buf, int len) } /** - * doc_set_data_mode - Sets the flash to normal or reliable data mode + * doc_set_reliable_mode - Sets the flash to normal or reliable data mode * @docg3: the device * * The reliable data mode is a bit slower than the fast mode, but less errors @@ -442,7 +442,7 @@ static void doc_setup_writeaddr_sector(struct docg3 *docg3, int sector, int ofs) } /** - * doc_seek - Set both flash planes to the specified block, page for reading + * doc_read_seek - Set both flash planes to the specified block, page for reading * @docg3: the device * @block0: the first plane block index * @block1: the second plane block index @@ -871,6 +871,7 @@ static int doc_read_oob(struct mtd_info *mtd, loff_t from, u8 *buf = ops->datbuf; size_t len, ooblen, nbdata, nboob; u8 hwecc[DOC_ECC_BCH_SIZE], eccconf1; + struct mtd_ecc_stats old_stats; int max_bitflips = 0; if (buf) @@ -895,6 +896,7 @@ static int doc_read_oob(struct mtd_info *mtd, loff_t from, ret = 0; skip = from % DOC_LAYOUT_PAGE_SIZE; mutex_lock(&docg3->cascade->lock); + old_stats = mtd->ecc_stats; while (ret >= 0 && (len > 0 || ooblen > 0)) { calc_block_sector(from - skip, &block0, &block1, &page, &ofs, docg3->reliable); @@ -966,6 +968,12 @@ static int doc_read_oob(struct mtd_info *mtd, loff_t from, } out: + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } mutex_unlock(&docg3->cascade->lock); return ret; err_in_read: @@ -1951,7 +1959,7 @@ static int docg3_suspend(struct platform_device *pdev, pm_message_t state) } /** - * doc_probe - Probe the IO space for a DiskOnChip G3 chip + * docg3_probe - Probe the IO space for a DiskOnChip G3 chip * @pdev: platform device * * Probes for a G3 chip at the specified IO space in the platform data @@ -1974,9 +1982,14 @@ static int __init docg3_probe(struct platform_device *pdev) dev_err(dev, "No I/O memory resource defined\n"); return ret; } - base = devm_ioremap(dev, ress->start, DOC_IOSPACE_SIZE); ret = -ENOMEM; + base = devm_ioremap(dev, ress->start, DOC_IOSPACE_SIZE); + if (!base) { + dev_err(dev, "devm_ioremap dev failed\n"); + return ret; + } + cascade = devm_kcalloc(dev, DOC_MAX_NBFLOORS, sizeof(*cascade), GFP_KERNEL); if (!cascade) diff --git a/drivers/mtd/ftl.c b/drivers/mtd/ftl.c index f655d290527052f8c069fcec4a134152dbe4e71f..8c22064ead38705f39d6ba028d0242a0a9855c85 100644 --- a/drivers/mtd/ftl.c +++ b/drivers/mtd/ftl.c @@ -941,7 +941,7 @@ static int ftl_write(partition_t *part, caddr_t buffer, static int ftl_getgeo(struct mtd_blktrans_dev *dev, struct hd_geometry *geo) { - partition_t *part = (void *)dev; + partition_t *part = container_of(dev, struct partition_t, mbd); u_long sect; /* Sort of arbitrary: round size down to 4KiB boundary */ @@ -969,7 +969,7 @@ static int ftl_writesect(struct mtd_blktrans_dev *dev, static int ftl_discardsect(struct mtd_blktrans_dev *dev, unsigned long sector, unsigned nr_sects) { - partition_t *part = (void *)dev; + partition_t *part = container_of(dev, struct partition_t, mbd); uint32_t bsize = 1 << part->header.EraseUnitSize; pr_debug("FTL erase sector %ld for %d sectors\n", diff --git a/drivers/mtd/inftlcore.c b/drivers/mtd/inftlcore.c index 6b48397c750c2246b7e4b1efd19f60a09245b8a3..58ca1c21ebe671b724620d689a025c81c828637d 100644 --- a/drivers/mtd/inftlcore.c +++ b/drivers/mtd/inftlcore.c @@ -136,7 +136,7 @@ static void inftl_remove_dev(struct mtd_blktrans_dev *dev) int inftl_read_oob(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; @@ -156,7 +156,7 @@ int inftl_read_oob(struct mtd_info *mtd, loff_t offs, size_t len, int inftl_write_oob(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; @@ -176,7 +176,7 @@ int inftl_write_oob(struct mtd_info *mtd, loff_t offs, size_t len, static int inftl_write(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf, uint8_t *oob) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; diff --git a/drivers/mtd/maps/physmap-core.c b/drivers/mtd/maps/physmap-core.c index 85eca6a192e62da9996df2ebb9fa954f27a59916..c73854da513633a04f66fb76bde8e3b46ae14221 100644 --- a/drivers/mtd/maps/physmap-core.c +++ b/drivers/mtd/maps/physmap-core.c @@ -300,6 +300,9 @@ static const char *of_select_probe_type(struct platform_device *dev) const char *probe_type; match = of_match_device(of_flash_match, &dev->dev); + if (!match) + return NULL; + probe_type = match->data; if (probe_type) return probe_type; diff --git a/drivers/mtd/maps/pismo.c b/drivers/mtd/maps/pismo.c index 946ba80f97588dc77b0ee9dfba852a6ed76e1429..5fcefcd0bacac165c481bdddd2eb6be1391a4518 100644 --- a/drivers/mtd/maps/pismo.c +++ b/drivers/mtd/maps/pismo.c @@ -195,7 +195,7 @@ static void pismo_add_one(struct pismo_data *pismo, int i, } } -static int pismo_remove(struct i2c_client *client) +static void pismo_remove(struct i2c_client *client) { struct pismo_data *pismo = i2c_get_clientdata(client); int i; @@ -204,8 +204,6 @@ static int pismo_remove(struct i2c_client *client) platform_device_unregister(pismo->dev[i]); kfree(pismo); - - return 0; } static int pismo_probe(struct i2c_client *client, diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c index 05860288a7afd73027a516a3ee81554a5fb909e0..01f1c6792df9c790622d2b039ee0fab992e6d0da 100644 --- a/drivers/mtd/mtdchar.c +++ b/drivers/mtd/mtdchar.c @@ -688,6 +688,137 @@ static int mtdchar_write_ioctl(struct mtd_info *mtd, return ret; } +static int mtdchar_read_ioctl(struct mtd_info *mtd, + struct mtd_read_req __user *argp) +{ + struct mtd_info *master = mtd_get_master(mtd); + struct mtd_read_req req; + void __user *usr_data, *usr_oob; + uint8_t *datbuf = NULL, *oobbuf = NULL; + size_t datbuf_len, oobbuf_len; + size_t orig_len, orig_ooblen; + int ret = 0; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + orig_len = req.len; + orig_ooblen = req.ooblen; + + usr_data = (void __user *)(uintptr_t)req.usr_data; + usr_oob = (void __user *)(uintptr_t)req.usr_oob; + + if (!master->_read_oob) + return -EOPNOTSUPP; + + if (!usr_data) + req.len = 0; + + if (!usr_oob) + req.ooblen = 0; + + req.ecc_stats.uncorrectable_errors = 0; + req.ecc_stats.corrected_bitflips = 0; + req.ecc_stats.max_bitflips = 0; + + req.len &= 0xffffffff; + req.ooblen &= 0xffffffff; + + if (req.start + req.len > mtd->size) { + ret = -EINVAL; + goto out; + } + + datbuf_len = min_t(size_t, req.len, mtd->erasesize); + if (datbuf_len > 0) { + datbuf = kvmalloc(datbuf_len, GFP_KERNEL); + if (!datbuf) { + ret = -ENOMEM; + goto out; + } + } + + oobbuf_len = min_t(size_t, req.ooblen, mtd->erasesize); + if (oobbuf_len > 0) { + oobbuf = kvmalloc(oobbuf_len, GFP_KERNEL); + if (!oobbuf) { + ret = -ENOMEM; + goto out; + } + } + + while (req.len > 0 || (!usr_data && req.ooblen > 0)) { + struct mtd_req_stats stats; + struct mtd_oob_ops ops = { + .mode = req.mode, + .len = min_t(size_t, req.len, datbuf_len), + .ooblen = min_t(size_t, req.ooblen, oobbuf_len), + .datbuf = datbuf, + .oobbuf = oobbuf, + .stats = &stats, + }; + + /* + * Shorten non-page-aligned, eraseblock-sized reads so that the + * read ends on an eraseblock boundary. This is necessary in + * order to prevent OOB data for some pages from being + * duplicated in the output of non-page-aligned reads requiring + * multiple mtd_read_oob() calls to be completed. + */ + if (ops.len == mtd->erasesize) + ops.len -= mtd_mod_by_ws(req.start + ops.len, mtd); + + ret = mtd_read_oob(mtd, (loff_t)req.start, &ops); + + req.ecc_stats.uncorrectable_errors += + stats.uncorrectable_errors; + req.ecc_stats.corrected_bitflips += stats.corrected_bitflips; + req.ecc_stats.max_bitflips = + max(req.ecc_stats.max_bitflips, stats.max_bitflips); + + if (ret && !mtd_is_bitflip_or_eccerr(ret)) + break; + + if (copy_to_user(usr_data, ops.datbuf, ops.retlen) || + copy_to_user(usr_oob, ops.oobbuf, ops.oobretlen)) { + ret = -EFAULT; + break; + } + + req.start += ops.retlen; + req.len -= ops.retlen; + usr_data += ops.retlen; + + req.ooblen -= ops.oobretlen; + usr_oob += ops.oobretlen; + } + + /* + * As multiple iterations of the above loop (and therefore multiple + * mtd_read_oob() calls) may be necessary to complete the read request, + * adjust the final return code to ensure it accounts for all detected + * ECC errors. + */ + if (!ret || mtd_is_bitflip(ret)) { + if (req.ecc_stats.uncorrectable_errors > 0) + ret = -EBADMSG; + else if (req.ecc_stats.corrected_bitflips > 0) + ret = -EUCLEAN; + } + +out: + req.len = orig_len - req.len; + req.ooblen = orig_ooblen - req.ooblen; + + if (copy_to_user(argp, &req, sizeof(req))) + ret = -EFAULT; + + kvfree(datbuf); + kvfree(oobbuf); + + return ret; +} + static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) { struct mtd_file_info *mfi = file->private_data; @@ -710,6 +841,7 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) case MEMGETINFO: case MEMREADOOB: case MEMREADOOB64: + case MEMREAD: case MEMISLOCKED: case MEMGETOOBSEL: case MEMGETBADBLOCK: @@ -884,6 +1016,13 @@ static int mtdchar_ioctl(struct file *file, u_int cmd, u_long arg) break; } + case MEMREAD: + { + ret = mtdchar_read_ioctl(mtd, + (struct mtd_read_req __user *)arg); + break; + } + case MEMLOCK: { struct erase_info_user einfo; diff --git a/drivers/mtd/mtdconcat.c b/drivers/mtd/mtdconcat.c index f685a581df481a12837d0f3f055801213a5429d5..193428de6a4bdfc324bd7c985ec7ebda28a658e6 100644 --- a/drivers/mtd/mtdconcat.c +++ b/drivers/mtd/mtdconcat.c @@ -836,7 +836,7 @@ struct mtd_info *mtd_concat_create(struct mtd_info *subdev[], /* subdevices to c /* * walk the map of the new device once more and fill in - * in erase region info: + * erase region info: */ curr_erasesize = subdev[0]->erasesize; begin = position = 0; diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index a9b8be9f40dc2706ec4c980cca7e2691f3939be5..18aa54460d36ce0f62075a0d08bed3c63956e385 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -1217,6 +1217,34 @@ int __get_mtd_device(struct mtd_info *mtd) } EXPORT_SYMBOL_GPL(__get_mtd_device); +/** + * of_get_mtd_device_by_node - obtain an MTD device associated with a given node + * + * @np: device tree node + */ +struct mtd_info *of_get_mtd_device_by_node(struct device_node *np) +{ + struct mtd_info *mtd = NULL; + struct mtd_info *tmp; + int err; + + mutex_lock(&mtd_table_mutex); + + err = -EPROBE_DEFER; + mtd_for_each_device(tmp) { + if (mtd_get_of_node(tmp) == np) { + mtd = tmp; + err = __get_mtd_device(mtd); + break; + } + } + + mutex_unlock(&mtd_table_mutex); + + return err ? ERR_PTR(err) : mtd; +} +EXPORT_SYMBOL_GPL(of_get_mtd_device_by_node); + /** * get_mtd_device_nm - obtain a validated handle for an MTD device by * device name @@ -1624,6 +1652,9 @@ int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) if (!master->_read_oob && (!master->_read || ops->oobbuf)) return -EOPNOTSUPP; + if (ops->stats) + memset(ops->stats, 0, sizeof(*ops->stats)); + if (mtd->flags & MTD_SLC_ON_MLC_EMULATION) ret_code = mtd_io_emulated_slc(mtd, from, true, ops); else @@ -1641,6 +1672,8 @@ int mtd_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) return ret_code; if (mtd->ecc_strength == 0) return 0; /* device lacks ecc */ + if (ops->stats) + ops->stats->max_bitflips = ret_code; return ret_code >= mtd->bitflip_threshold ? -EUCLEAN : 0; } EXPORT_SYMBOL_GPL(mtd_read_oob); diff --git a/drivers/mtd/mtdpstore.c b/drivers/mtd/mtdpstore.c index e13d42c0acb0f5407bdc57de95eecda8cdd0461f..7ac8ac90130685c9fdd7b4d166dee51dbe3d695e 100644 --- a/drivers/mtd/mtdpstore.c +++ b/drivers/mtd/mtdpstore.c @@ -401,7 +401,7 @@ static void mtdpstore_notify_add(struct mtd_info *mtd) /* * kmsg_size must be aligned to 4096 Bytes, which is limited by * psblk. The default value of kmsg_size is 64KB. If kmsg_size - * is larger than erasesize, some errors will occur since mtdpsotre + * is larger than erasesize, some errors will occur since mtdpstore * is designed on it. */ if (mtd->erasesize < info->kmsg_size) { diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c index dc7f1532a37f7860146841003913b4b7e19323a2..680366616da240039ecfb4cc59c83fcc4b1bf615 100644 --- a/drivers/mtd/mtdswap.c +++ b/drivers/mtd/mtdswap.c @@ -323,7 +323,7 @@ static int mtdswap_read_markers(struct mtdswap_dev *d, struct swap_eb *eb) struct mtdswap_oobdata *data, *data2; int ret; loff_t offset; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; offset = mtdswap_eb_offset(d, eb); @@ -370,7 +370,7 @@ static int mtdswap_write_marker(struct mtdswap_dev *d, struct swap_eb *eb, struct mtdswap_oobdata n; int ret; loff_t offset; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; ops.ooboffs = 0; ops.oobbuf = (uint8_t *)&n; @@ -878,7 +878,7 @@ static unsigned int mtdswap_eblk_passes(struct mtdswap_dev *d, loff_t base, pos; unsigned int *p1 = (unsigned int *)d->page_buf; unsigned char *p2 = (unsigned char *)d->oob_buf; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int ret; ops.mode = MTD_OPS_AUTO_OOB; diff --git a/drivers/mtd/nand/bbt.c b/drivers/mtd/nand/bbt.c index 64af6898131d656401a42135855c22e183df4109..db4f93a903e4876976acb07f0bdf5b2da95438c5 100644 --- a/drivers/mtd/nand/bbt.c +++ b/drivers/mtd/nand/bbt.c @@ -24,11 +24,8 @@ int nanddev_bbt_init(struct nand_device *nand) { unsigned int bits_per_block = fls(NAND_BBT_BLOCK_NUM_STATUS); unsigned int nblocks = nanddev_neraseblocks(nand); - unsigned int nwords = DIV_ROUND_UP(nblocks * bits_per_block, - BITS_PER_LONG); - nand->bbt.cache = kcalloc(nwords, sizeof(*nand->bbt.cache), - GFP_KERNEL); + nand->bbt.cache = bitmap_zalloc(nblocks * bits_per_block, GFP_KERNEL); if (!nand->bbt.cache) return -ENOMEM; @@ -44,7 +41,7 @@ EXPORT_SYMBOL_GPL(nanddev_bbt_init); */ void nanddev_bbt_cleanup(struct nand_device *nand) { - kfree(nand->bbt.cache); + bitmap_free(nand->bbt.cache); } EXPORT_SYMBOL_GPL(nanddev_bbt_cleanup); diff --git a/drivers/mtd/nand/onenand/onenand_base.c b/drivers/mtd/nand/onenand/onenand_base.c index 958bac54b19078554a06c08019ddcd7abe02c41a..f66385faf631cde6dd3fea7c707d40369e1ade3e 100644 --- a/drivers/mtd/nand/onenand/onenand_base.c +++ b/drivers/mtd/nand/onenand/onenand_base.c @@ -1440,6 +1440,7 @@ static int onenand_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) { struct onenand_chip *this = mtd->priv; + struct mtd_ecc_stats old_stats; int ret; switch (ops->mode) { @@ -1453,12 +1454,23 @@ static int onenand_read_oob(struct mtd_info *mtd, loff_t from, } onenand_get_device(mtd, FL_READING); + + old_stats = mtd->ecc_stats; + if (ops->datbuf) ret = ONENAND_IS_4KB_PAGE(this) ? onenand_mlc_read_ops_nolock(mtd, from, ops) : onenand_read_ops_nolock(mtd, from, ops); else ret = onenand_read_oob_nolock(mtd, from, ops); + + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } + onenand_release_device(mtd); return ret; @@ -2935,7 +2947,7 @@ static int do_otp_write(struct mtd_info *mtd, loff_t to, size_t len, struct onenand_chip *this = mtd->priv; unsigned char *pbuf = buf; int ret; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; /* Force buffer page aligned */ if (len < mtd->writesize) { @@ -2977,7 +2989,7 @@ static int do_otp_lock(struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf) { struct onenand_chip *this = mtd->priv; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int ret; if (FLEXONENAND(this)) { diff --git a/drivers/mtd/nand/onenand/onenand_bbt.c b/drivers/mtd/nand/onenand/onenand_bbt.c index b17315f8e1d47a388b0e7ae1cceea35d951ea8a1..d7fe35bc45cba50649650f2f166a4d20a07eb189 100644 --- a/drivers/mtd/nand/onenand/onenand_bbt.c +++ b/drivers/mtd/nand/onenand/onenand_bbt.c @@ -61,7 +61,7 @@ static int create_bbt(struct mtd_info *mtd, uint8_t *buf, struct nand_bbt_descr int startblock; loff_t from; size_t readlen; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int rgn; printk(KERN_INFO "Scanning device for bad blocks\n"); diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig index 8b6d7a515445e6dc828a8e01a481e6d4a92018ce..4cd40af362de28ee112705cd7e6d8a2c04cd1176 100644 --- a/drivers/mtd/nand/raw/Kconfig +++ b/drivers/mtd/nand/raw/Kconfig @@ -200,27 +200,7 @@ config MTD_NAND_TMIO Support for NAND flash connected to a Toshiba Mobile IO Controller in some PDAs, including the Sharp SL6000x. -config MTD_NAND_BRCMNAND - tristate "Broadcom STB NAND controller" - depends on ARM || ARM64 || MIPS || COMPILE_TEST - depends on HAS_IOMEM - help - Enables the Broadcom NAND controller driver. The controller was - originally designed for Set-Top Box but is used on various BCM7xxx, - BCM3xxx, BCM63xxx, iProc/Cygnus and more. - -if MTD_NAND_BRCMNAND - -config MTD_NAND_BRCMNAND_BCMA - tristate "Broadcom BCMA NAND controller" - depends on BCMA_NFLASH - depends on BCMA - help - Enables the BRCMNAND controller over BCMA on BCM47186/BCM5358 SoCs. - The glue driver will take care of performing the low-level I/O - operations to interface the BRCMNAND controller over the BCMA bus. - -endif # MTD_NAND_BRCMNAND +source "drivers/mtd/nand/raw/brcmnand/Kconfig" config MTD_NAND_BCM47XXNFLASH tristate "BCM4706 BCMA NAND controller" @@ -410,7 +390,7 @@ config MTD_NAND_STM32_FMC2 config MTD_NAND_MESON tristate "Support for NAND controller on Amlogic's Meson SoCs" - depends on ARCH_MESON || COMPILE_TEST + depends on COMMON_CLK && (ARCH_MESON || COMPILE_TEST) select MFD_SYSCON help Enables support for NAND controller on Amlogic's Meson SoCs. diff --git a/drivers/mtd/nand/raw/arasan-nand-controller.c b/drivers/mtd/nand/raw/arasan-nand-controller.c index 296fb16c8dc3cd3af8900d3eb11f9a0192840308..ec7e6eeac55f9f1c36c099c541cf9c956f70f402 100644 --- a/drivers/mtd/nand/raw/arasan-nand-controller.c +++ b/drivers/mtd/nand/raw/arasan-nand-controller.c @@ -915,7 +915,7 @@ static int anfc_check_op(struct nand_chip *chip, if (instr->ctx.data.len > ANFC_MAX_CHUNK_SIZE) return -ENOTSUPP; - if (anfc_pkt_len_config(instr->ctx.data.len, 0, 0)) + if (anfc_pkt_len_config(instr->ctx.data.len, NULL, NULL)) return -ENOTSUPP; break; diff --git a/drivers/mtd/nand/raw/atmel/nand-controller.c b/drivers/mtd/nand/raw/atmel/nand-controller.c index c9ac3baf68c0258c20d5ecbd31d43dff6dda5cfd..41c6bd6e2d72ca39df9e0b859bcbb0334c946597 100644 --- a/drivers/mtd/nand/raw/atmel/nand-controller.c +++ b/drivers/mtd/nand/raw/atmel/nand-controller.c @@ -405,6 +405,7 @@ static int atmel_nand_dma_transfer(struct atmel_nand_controller *nc, dma_async_issue_pending(nc->dmac); wait_for_completion(&finished); + dma_unmap_single(nc->dev, buf_dma, len, dir); return 0; diff --git a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c index 8bb17c5a66c3eab035196aeb48e2faa92535c689..6487dfc642588863d1aa5930df52cbb7c95eba37 100644 --- a/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c +++ b/drivers/mtd/nand/raw/bcm47xxnflash/ops_bcm4706.c @@ -14,7 +14,7 @@ #include /* Broadcom uses 1'000'000 but it seems to be too many. Tests on WNDR4500 has - * shown ~1000 retries as maxiumum. */ + * shown ~1000 retries as maximum. */ #define NFLASH_READY_RETRIES 10000 #define NFLASH_SECTOR_SIZE 512 diff --git a/drivers/mtd/nand/raw/brcmnand/Kconfig b/drivers/mtd/nand/raw/brcmnand/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..4bc51bf60aca51351fbb9d9e7b2fdb1eff537ea7 --- /dev/null +++ b/drivers/mtd/nand/raw/brcmnand/Kconfig @@ -0,0 +1,49 @@ +config MTD_NAND_BRCMNAND + tristate "Broadcom STB NAND controller" + depends on ARM || ARM64 || MIPS || COMPILE_TEST + depends on HAS_IOMEM + help + Enables the Broadcom NAND controller driver. The controller was + originally designed for Set-Top Box but is used on various BCM7xxx, + BCM3xxx, BCM63xxx, iProc/Cygnus and more. + +if MTD_NAND_BRCMNAND + +config MTD_NAND_BRCMNAND_BCM63XX + tristate "Broadcom BCM63xx NAND controller glue" + default BCM63XX + help + Enables the BRCMNAND glue driver to register the NAND controller + on Broadcom BCM63xx MIPS-based DSL platforms. + +config MTD_NAND_BRCMNAND_BCMA + tristate "Broadcom BCMA NAND controller" + depends on BCMA_NFLASH + depends on BCMA + help + Enables the BRCMNAND controller over BCMA on BCM47186/BCM5358 SoCs. + The glue driver will take care of performing the low-level I/O + operations to interface the BRCMNAND controller over the BCMA bus. + +config MTD_NAND_BRCMNAND_BCMBCA + tristate "Broadcom BCMBCA NAND controller glue" + default ARCH_BCMBCA + help + Enables the BRCMNAND glue driver to register the NAND controller + on Broadcom BCA platforms. + +config MTD_NAND_BRCMNAND_BRCMSTB + tristate "Broadcom STB Nand controller glue" + default ARCH_BRCMSTB + help + Enables the BRCMNAND glue driver to register the NAND controller + on Broadcom STB platforms. + +config MTD_NAND_BRCMNAND_IPROC + tristate "Broadcom iProc NAND controller glue" + default ARCH_BCM_IPROC + help + Enables the BRCMNAND controller glue driver to register the NAND + controller on Broadcom iProc platforms. + +endif # MTD_NAND_BRCMNAND diff --git a/drivers/mtd/nand/raw/brcmnand/Makefile b/drivers/mtd/nand/raw/brcmnand/Makefile index 16dc7254200edcf64e5a0fa1464c38beac3bc400..9907e3ec4bb2d8bc275e247354d39dd143ab8e84 100644 --- a/drivers/mtd/nand/raw/brcmnand/Makefile +++ b/drivers/mtd/nand/raw/brcmnand/Makefile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: GPL-2.0 # link order matters; don't link the more generic brcmstb_nand.o before the # more specific iproc_nand.o, for instance -obj-$(CONFIG_MTD_NAND_BRCMNAND) += iproc_nand.o -obj-$(CONFIG_MTD_NAND_BRCMNAND) += bcm63138_nand.o -obj-$(CONFIG_MTD_NAND_BRCMNAND) += bcm6368_nand.o -obj-$(CONFIG_MTD_NAND_BRCMNAND) += brcmstb_nand.o +obj-$(CONFIG_MTD_NAND_BRCMNAND_IPROC) += iproc_nand.o +obj-$(CONFIG_MTD_NAND_BRCMNAND_BCMBCA) += bcm63138_nand.o +obj-$(CONFIG_MTD_NAND_BRCMNAND_BCM63XX) += bcm6368_nand.o +obj-$(CONFIG_MTD_NAND_BRCMNAND_BRCMSTB) += brcmstb_nand.o obj-$(CONFIG_MTD_NAND_BRCMNAND) += brcmnand.o obj-$(CONFIG_MTD_NAND_BRCMNAND_BCMA) += bcma_nand.o diff --git a/drivers/mtd/nand/raw/cadence-nand-controller.c b/drivers/mtd/nand/raw/cadence-nand-controller.c index 0d72672f8b64d5648557891405f37d7d923ed7d8..9dac3ca69d576e085c7b07479796ea5013d8c23f 100644 --- a/drivers/mtd/nand/raw/cadence-nand-controller.c +++ b/drivers/mtd/nand/raw/cadence-nand-controller.c @@ -1979,7 +1979,6 @@ static int cadence_nand_force_byte_access(struct nand_chip *chip, bool force_8bit) { struct cdns_nand_ctrl *cdns_ctrl = to_cdns_nand_ctrl(chip->controller); - int status; /* * Callers of this function do not verify if the NAND is using a 16-bit @@ -1990,9 +1989,7 @@ static int cadence_nand_force_byte_access(struct nand_chip *chip, if (!(chip->options & NAND_BUSWIDTH_16)) return 0; - status = cadence_nand_set_access_width16(cdns_ctrl, !force_8bit); - - return status; + return cadence_nand_set_access_width16(cdns_ctrl, !force_8bit); } static int cadence_nand_cmd_opcode(struct nand_chip *chip, diff --git a/drivers/mtd/nand/raw/cafe_nand.c b/drivers/mtd/nand/raw/cafe_nand.c index af119e3763523f4be9976388575523f3f4e1d409..66385c4fb994b9c68af9602897c29a439a5a4d7e 100644 --- a/drivers/mtd/nand/raw/cafe_nand.c +++ b/drivers/mtd/nand/raw/cafe_nand.c @@ -358,7 +358,7 @@ static int cafe_nand_read_oob(struct nand_chip *chip, int page) return nand_read_oob_op(chip, page, 0, chip->oob_poi, mtd->oobsize); } /** - * cafe_nand_read_page_syndrome - [REPLACEABLE] hardware ecc syndrome based page read + * cafe_nand_read_page - [REPLACEABLE] hardware ecc syndrome based page read * @chip: nand chip info structure * @buf: buffer to store read data * @oob_required: caller expects OOB data read to chip->oob_poi diff --git a/drivers/mtd/nand/raw/fsl_elbc_nand.c b/drivers/mtd/nand/raw/fsl_elbc_nand.c index aab93b9e6052d63196516fbe94546485178b8fbc..a18d121396aa5ab2fc80220c5197b949a3d7dd0b 100644 --- a/drivers/mtd/nand/raw/fsl_elbc_nand.c +++ b/drivers/mtd/nand/raw/fsl_elbc_nand.c @@ -726,36 +726,40 @@ static int fsl_elbc_attach_chip(struct nand_chip *chip) struct fsl_lbc_regs __iomem *lbc = ctrl->regs; unsigned int al; - switch (chip->ecc.engine_type) { /* * if ECC was not chosen in DT, decide whether to use HW or SW ECC from * CS Base Register */ - case NAND_ECC_ENGINE_TYPE_NONE: + if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_INVALID) { /* If CS Base Register selects full hardware ECC then use it */ if ((in_be32(&lbc->bank[priv->bank].br) & BR_DECC) == BR_DECC_CHK_GEN) { - chip->ecc.read_page = fsl_elbc_read_page; - chip->ecc.write_page = fsl_elbc_write_page; - chip->ecc.write_subpage = fsl_elbc_write_subpage; - chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST; - mtd_set_ooblayout(mtd, &fsl_elbc_ooblayout_ops); - chip->ecc.size = 512; - chip->ecc.bytes = 3; - chip->ecc.strength = 1; } else { /* otherwise fall back to default software ECC */ chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_SOFT; chip->ecc.algo = NAND_ECC_ALGO_HAMMING; } + } + + switch (chip->ecc.engine_type) { + /* if HW ECC was chosen, setup ecc and oob layout */ + case NAND_ECC_ENGINE_TYPE_ON_HOST: + chip->ecc.read_page = fsl_elbc_read_page; + chip->ecc.write_page = fsl_elbc_write_page; + chip->ecc.write_subpage = fsl_elbc_write_subpage; + mtd_set_ooblayout(mtd, &fsl_elbc_ooblayout_ops); + chip->ecc.size = 512; + chip->ecc.bytes = 3; + chip->ecc.strength = 1; break; - /* if SW ECC was chosen in DT, we do not need to set anything here */ + /* if none or SW ECC was chosen, we do not need to set anything here */ + case NAND_ECC_ENGINE_TYPE_NONE: case NAND_ECC_ENGINE_TYPE_SOFT: + case NAND_ECC_ENGINE_TYPE_ON_DIE: break; - /* should we also implement *_ECC_ENGINE_CONTROLLER to do as above? */ default: return -EINVAL; } diff --git a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c index 93da23682d862c3ea2c0407ee332cc165fd54304..01ccbde748f3062aa8de99e804f610bb2ada4b18 100644 --- a/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c +++ b/drivers/mtd/nand/raw/gpmi-nand/gpmi-nand.c @@ -1361,7 +1361,7 @@ error_alloc: /* * Handles block mark swapping. * It can be called in swapping the block mark, or swapping it back, - * because the the operations are the same. + * because the operations are the same. */ static void block_mark_swapping(struct gpmi_nand_data *this, void *payload, void *auxiliary) diff --git a/drivers/mtd/nand/raw/intel-nand-controller.c b/drivers/mtd/nand/raw/intel-nand-controller.c index e91b879b32bdb78ef28c11301149299d88924607..d4a0987e93ace7f0be0b75d249215821f9042a66 100644 --- a/drivers/mtd/nand/raw/intel-nand-controller.c +++ b/drivers/mtd/nand/raw/intel-nand-controller.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -99,15 +100,12 @@ #define HSNAND_ECC_OFFSET 0x008 -#define NAND_DATA_IFACE_CHECK_ONLY -1 - #define MAX_CS 2 #define USEC_PER_SEC 1000000L struct ebu_nand_cs { void __iomem *chipaddr; - dma_addr_t nand_pa; u32 addr_sel; }; @@ -120,7 +118,6 @@ struct ebu_nand_controller { struct dma_chan *dma_tx; struct dma_chan *dma_rx; struct completion dma_access_complete; - unsigned long clk_rate; struct clk *clk; u32 nd_para0; u8 cs_num; @@ -580,6 +577,7 @@ static int ebu_nand_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct ebu_nand_controller *ebu_host; + struct device_node *chip_np; struct nand_chip *nand; struct mtd_info *mtd; struct resource *res; @@ -594,17 +592,20 @@ static int ebu_nand_probe(struct platform_device *pdev) ebu_host->dev = dev; nand_controller_init(&ebu_host->controller); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ebunand"); - ebu_host->ebu = devm_ioremap_resource(&pdev->dev, res); + ebu_host->ebu = devm_platform_ioremap_resource_byname(pdev, "ebunand"); if (IS_ERR(ebu_host->ebu)) return PTR_ERR(ebu_host->ebu); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "hsnand"); - ebu_host->hsnand = devm_ioremap_resource(&pdev->dev, res); + ebu_host->hsnand = devm_platform_ioremap_resource_byname(pdev, "hsnand"); if (IS_ERR(ebu_host->hsnand)) return PTR_ERR(ebu_host->hsnand); - ret = device_property_read_u32(dev, "reg", &cs); + chip_np = of_get_next_child(dev->of_node, NULL); + if (!chip_np) + return dev_err_probe(dev, -EINVAL, + "Could not find child node for the NAND chip\n"); + + ret = of_property_read_u32(chip_np, "reg", &cs); if (ret) { dev_err(dev, "failed to get chip select: %d\n", ret); return ret; @@ -617,11 +618,10 @@ static int ebu_nand_probe(struct platform_device *pdev) ebu_host->cs_num = cs; resname = devm_kasprintf(dev, GFP_KERNEL, "nand_cs%d", cs); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, resname); - ebu_host->cs[cs].chipaddr = devm_ioremap_resource(dev, res); + ebu_host->cs[cs].chipaddr = devm_platform_ioremap_resource_byname(pdev, + resname); if (IS_ERR(ebu_host->cs[cs].chipaddr)) return PTR_ERR(ebu_host->cs[cs].chipaddr); - ebu_host->cs[cs].nand_pa = res->start; ebu_host->clk = devm_clk_get(dev, NULL); if (IS_ERR(ebu_host->clk)) @@ -633,7 +633,6 @@ static int ebu_nand_probe(struct platform_device *pdev) dev_err(dev, "failed to enable clock: %d\n", ret); return ret; } - ebu_host->clk_rate = clk_get_rate(ebu_host->clk); ebu_host->dma_tx = dma_request_chan(dev, "tx"); if (IS_ERR(ebu_host->dma_tx)) { @@ -660,7 +659,7 @@ static int ebu_nand_probe(struct platform_device *pdev) writel(ebu_host->cs[cs].addr_sel | EBU_ADDR_MASK(5) | EBU_ADDR_SEL_REGEN, ebu_host->ebu + EBU_ADDR_SEL(cs)); - nand_set_flash_node(&ebu_host->chip, dev->of_node); + nand_set_flash_node(&ebu_host->chip, chip_np); mtd = nand_to_mtd(&ebu_host->chip); if (!mtd->name) { @@ -716,7 +715,6 @@ static int ebu_nand_remove(struct platform_device *pdev) } static const struct of_device_id ebu_nand_match[] = { - { .compatible = "intel,nand-controller" }, { .compatible = "intel,lgm-ebunand" }, {} }; diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index 2455a581fd70cfa21e88db4a5e22c1ffe86e0ceb..d9f2f1d0b5efccbcbf79c51b7e8a9554aaaa8814 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -865,13 +865,19 @@ static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc, marvell_nfc_enable_dma(nfc); /* Prepare the DMA transfer */ sg_init_one(&sg, nfc->dma_buf, dma_len); - dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction); + ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction); + if (!ret) { + dev_err(nfc->dev, "Could not map DMA S/G list\n"); + return -ENXIO; + } + tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1, direction == DMA_FROM_DEVICE ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT); if (!tx) { dev_err(nfc->dev, "Could not prepare DMA S/G list\n"); + dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction); return -ENXIO; } diff --git a/drivers/mtd/nand/raw/meson_nand.c b/drivers/mtd/nand/raw/meson_nand.c index 829b76b303aae0bdbd7c435de15d16f62b55a974..5ee01231ac4cdc5dbb156851a2fdb6c13a402e07 100644 --- a/drivers/mtd/nand/raw/meson_nand.c +++ b/drivers/mtd/nand/raw/meson_nand.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,9 @@ #define NFC_RB_IRQ_EN BIT(21) +#define CLK_DIV_SHIFT 0 +#define CLK_DIV_WIDTH 6 + #define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \ ( \ (cmd_dir) | \ @@ -151,15 +155,15 @@ struct meson_nfc { struct nand_controller controller; struct clk *core_clk; struct clk *device_clk; - struct clk *phase_tx; - struct clk *phase_rx; + struct clk *nand_clk; + struct clk_divider nand_divider; unsigned long clk_rate; u32 bus_timing; struct device *dev; void __iomem *reg_base; - struct regmap *reg_clk; + void __iomem *reg_clk; struct completion completion; struct list_head chips; const struct meson_nfc_data *data; @@ -235,7 +239,7 @@ static void meson_nfc_select_chip(struct nand_chip *nand, int chip) nfc->timing.tbers_max = meson_chip->tbers_max; if (nfc->clk_rate != meson_chip->clk_rate) { - ret = clk_set_rate(nfc->device_clk, meson_chip->clk_rate); + ret = clk_set_rate(nfc->nand_clk, meson_chip->clk_rate); if (ret) { dev_err(nfc->dev, "failed to set clock rate\n"); return; @@ -454,7 +458,7 @@ static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips, if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) { mtd->ecc_stats.corrected += ECC_ERR_CNT(*info); *bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info)); - *correct_bitmap |= 1 >> i; + *correct_bitmap |= BIT_ULL(i); continue; } if ((nand->options & NAND_NEED_SCRAMBLING) && @@ -800,7 +804,7 @@ static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf, u8 *data = buf + i * ecc->size; u8 *oob = nand->oob_poi + i * (ecc->bytes + 2); - if (correct_bitmap & (1 << i)) + if (correct_bitmap & BIT_ULL(i)) continue; ret = nand_check_erased_ecc_chunk(data, ecc->size, oob, ecc->bytes + 2, @@ -987,6 +991,8 @@ static const struct mtd_ooblayout_ops meson_ooblayout_ops = { static int meson_nfc_clk_init(struct meson_nfc *nfc) { + struct clk_parent_data nfc_divider_parent_data[1]; + struct clk_init_data init = {0}; int ret; /* request core clock */ @@ -1002,21 +1008,28 @@ static int meson_nfc_clk_init(struct meson_nfc *nfc) return PTR_ERR(nfc->device_clk); } - nfc->phase_tx = devm_clk_get(nfc->dev, "tx"); - if (IS_ERR(nfc->phase_tx)) { - dev_err(nfc->dev, "failed to get TX clk\n"); - return PTR_ERR(nfc->phase_tx); - } - - nfc->phase_rx = devm_clk_get(nfc->dev, "rx"); - if (IS_ERR(nfc->phase_rx)) { - dev_err(nfc->dev, "failed to get RX clk\n"); - return PTR_ERR(nfc->phase_rx); - } + init.name = devm_kasprintf(nfc->dev, + GFP_KERNEL, "%s#div", + dev_name(nfc->dev)); + init.ops = &clk_divider_ops; + nfc_divider_parent_data[0].fw_name = "device"; + init.parent_data = nfc_divider_parent_data; + init.num_parents = 1; + nfc->nand_divider.reg = nfc->reg_clk; + nfc->nand_divider.shift = CLK_DIV_SHIFT; + nfc->nand_divider.width = CLK_DIV_WIDTH; + nfc->nand_divider.hw.init = &init; + nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED | + CLK_DIVIDER_ROUND_CLOSEST | + CLK_DIVIDER_ALLOW_ZERO; + + nfc->nand_clk = devm_clk_register(nfc->dev, &nfc->nand_divider.hw); + if (IS_ERR(nfc->nand_clk)) + return PTR_ERR(nfc->nand_clk); /* init SD_EMMC_CLOCK to sane defaults w/min clock rate */ - regmap_update_bits(nfc->reg_clk, - 0, CLK_SELECT_NAND, CLK_SELECT_NAND); + writel(CLK_SELECT_NAND | readl(nfc->reg_clk), + nfc->reg_clk); ret = clk_prepare_enable(nfc->core_clk); if (ret) { @@ -1030,29 +1043,21 @@ static int meson_nfc_clk_init(struct meson_nfc *nfc) goto err_device_clk; } - ret = clk_prepare_enable(nfc->phase_tx); + ret = clk_prepare_enable(nfc->nand_clk); if (ret) { - dev_err(nfc->dev, "failed to enable TX clock\n"); - goto err_phase_tx; + dev_err(nfc->dev, "pre enable NFC divider fail\n"); + goto err_nand_clk; } - ret = clk_prepare_enable(nfc->phase_rx); - if (ret) { - dev_err(nfc->dev, "failed to enable RX clock\n"); - goto err_phase_rx; - } - - ret = clk_set_rate(nfc->device_clk, 24000000); + ret = clk_set_rate(nfc->nand_clk, 24000000); if (ret) - goto err_disable_rx; + goto err_disable_clk; return 0; -err_disable_rx: - clk_disable_unprepare(nfc->phase_rx); -err_phase_rx: - clk_disable_unprepare(nfc->phase_tx); -err_phase_tx: +err_disable_clk: + clk_disable_unprepare(nfc->nand_clk); +err_nand_clk: clk_disable_unprepare(nfc->device_clk); err_device_clk: clk_disable_unprepare(nfc->core_clk); @@ -1061,8 +1066,7 @@ err_device_clk: static void meson_nfc_disable_clk(struct meson_nfc *nfc) { - clk_disable_unprepare(nfc->phase_rx); - clk_disable_unprepare(nfc->phase_tx); + clk_disable_unprepare(nfc->nand_clk); clk_disable_unprepare(nfc->device_clk); clk_disable_unprepare(nfc->core_clk); } @@ -1368,7 +1372,6 @@ static int meson_nfc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct meson_nfc *nfc; - struct resource *res; int ret, irq; nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL); @@ -1385,18 +1388,13 @@ static int meson_nfc_probe(struct platform_device *pdev) nfc->dev = dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - nfc->reg_base = devm_ioremap_resource(dev, res); + nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, "nfc"); if (IS_ERR(nfc->reg_base)) return PTR_ERR(nfc->reg_base); - nfc->reg_clk = - syscon_regmap_lookup_by_phandle(dev->of_node, - "amlogic,mmc-syscon"); - if (IS_ERR(nfc->reg_clk)) { - dev_err(dev, "Failed to lookup clock base\n"); + nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, "emmc"); + if (IS_ERR(nfc->reg_clk)) return PTR_ERR(nfc->reg_clk); - } irq = platform_get_irq(pdev, 0); if (irq < 0) diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 6b67b7dfe7ce69601534d80e50fc86434d8e2c43..33f2c98a030ebb700139e2e9fc67eef339703255 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -335,8 +335,6 @@ static int nand_isbad_bbm(struct nand_chip *chip, loff_t ofs) * @chip: NAND chip structure * * Lock the device and its controller for exclusive access - * - * Return: -EBUSY if the chip has been suspended, 0 otherwise */ static void nand_get_device(struct nand_chip *chip) { @@ -3818,6 +3816,7 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, struct mtd_oob_ops *ops) { struct nand_chip *chip = mtd_to_nand(mtd); + struct mtd_ecc_stats old_stats; int ret; ops->retlen = 0; @@ -3829,11 +3828,20 @@ static int nand_read_oob(struct mtd_info *mtd, loff_t from, nand_get_device(chip); + old_stats = mtd->ecc_stats; + if (!ops->datbuf) ret = nand_do_read_oob(chip, from, ops); else ret = nand_do_read_ops(chip, from, ops); + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } + nand_release_device(chip); return ret; } @@ -5331,11 +5339,10 @@ static int of_get_nand_secure_regions(struct nand_chip *chip) int rawnand_dt_parse_gpio_cs(struct device *dev, struct gpio_desc ***cs_array, unsigned int *ncs_array) { - struct device_node *np = dev->of_node; struct gpio_desc **descs; int ndescs, i; - ndescs = of_gpio_named_count(np, "cs-gpios"); + ndescs = gpiod_count(dev, "cs"); if (ndescs < 0) { dev_dbg(dev, "No valid cs-gpios property\n"); return 0; diff --git a/drivers/mtd/nand/raw/nand_bbt.c b/drivers/mtd/nand/raw/nand_bbt.c index a3723da2e0a019951ccb6624113a6c8b0199df32..e4664fa6fd9ef2b199f07a27c05f5299b6c5a2ec 100644 --- a/drivers/mtd/nand/raw/nand_bbt.c +++ b/drivers/mtd/nand/raw/nand_bbt.c @@ -313,7 +313,7 @@ static int scan_read_oob(struct nand_chip *this, uint8_t *buf, loff_t offs, size_t len) { struct mtd_info *mtd = nand_to_mtd(this); - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res, ret = 0; ops.mode = MTD_OPS_PLACE_OOB; @@ -354,7 +354,7 @@ static int scan_write_bbt(struct nand_chip *this, loff_t offs, size_t len, uint8_t *buf, uint8_t *oob) { struct mtd_info *mtd = nand_to_mtd(this); - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; ops.mode = MTD_OPS_PLACE_OOB; ops.ooboffs = 0; @@ -416,7 +416,7 @@ static int scan_block_fast(struct nand_chip *this, struct nand_bbt_descr *bd, { struct mtd_info *mtd = nand_to_mtd(this); - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int ret, page_offset; ops.ooblen = mtd->oobsize; @@ -756,7 +756,7 @@ static int write_bbt(struct nand_chip *this, uint8_t *buf, uint8_t rcode = td->reserved_block_code; size_t retlen, len = 0; loff_t to; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; ops.ooblen = mtd->oobsize; ops.ooboffs = 0; diff --git a/drivers/mtd/nand/raw/nandsim.c b/drivers/mtd/nand/raw/nandsim.c index 24beade95c7fb1fa32444911e8c592824a887c9e..672719023241b7e2f1ed0c3b36252334f9ee10d2 100644 --- a/drivers/mtd/nand/raw/nandsim.c +++ b/drivers/mtd/nand/raw/nandsim.c @@ -1393,7 +1393,7 @@ static int ns_do_read_error(struct nandsim *ns, int num) unsigned int page_no = ns->regs.row; if (ns_read_error(page_no)) { - prandom_bytes(ns->buf.byte, num); + get_random_bytes(ns->buf.byte, num); NS_WARN("simulating read error in page %u\n", page_no); return 1; } @@ -1402,12 +1402,12 @@ static int ns_do_read_error(struct nandsim *ns, int num) static void ns_do_bit_flips(struct nandsim *ns, int num) { - if (bitflips && prandom_u32() < (1 << 22)) { + if (bitflips && get_random_u16() < (1 << 6)) { int flips = 1; if (bitflips > 1) - flips = (prandom_u32() % (int) bitflips) + 1; + flips = prandom_u32_max(bitflips) + 1; while (flips--) { - int pos = prandom_u32() % (num * 8); + int pos = prandom_u32_max(num * 8); ns->buf.byte[pos / 8] ^= (1 << (pos % 8)); NS_WARN("read_page: flipping bit %d in page %d " "reading from %d ecc: corrected=%u failed=%u\n", diff --git a/drivers/mtd/nand/raw/orion_nand.c b/drivers/mtd/nand/raw/orion_nand.c index 2c87c7d892058046e0aa55a50433b43840ecd7f4..1bfecf502216056b304315e7c4034db9cdfb4eea 100644 --- a/drivers/mtd/nand/raw/orion_nand.c +++ b/drivers/mtd/nand/raw/orion_nand.c @@ -170,18 +170,11 @@ static int __init orion_nand_probe(struct platform_device *pdev) platform_set_drvdata(pdev, info); - /* Not all platforms can gate the clock, so it is not - an error if the clock does not exists. */ - info->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(info->clk)) { - ret = PTR_ERR(info->clk); - if (ret == -ENOENT) { - info->clk = NULL; - } else { - dev_err(&pdev->dev, "failed to get clock!\n"); - return ret; - } - } + /* Not all platforms can gate the clock, so it is optional. */ + info->clk = devm_clk_get_optional(&pdev->dev, NULL); + if (IS_ERR(info->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(info->clk), + "failed to get clock!\n"); ret = clk_prepare_enable(info->clk); if (ret) { diff --git a/drivers/mtd/nand/raw/sm_common.c b/drivers/mtd/nand/raw/sm_common.c index b2b42dd1a2de629bb52465b64bb352d7a4107a66..24f52a30fb1304c9dbe4625a544b541a2db1eaae 100644 --- a/drivers/mtd/nand/raw/sm_common.c +++ b/drivers/mtd/nand/raw/sm_common.c @@ -99,7 +99,7 @@ static const struct mtd_ooblayout_ops oob_sm_small_ops = { static int sm_block_markbad(struct nand_chip *chip, loff_t ofs) { struct mtd_info *mtd = nand_to_mtd(chip); - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; struct sm_oob oob; int ret; diff --git a/drivers/mtd/nand/raw/stm32_fmc2_nand.c b/drivers/mtd/nand/raw/stm32_fmc2_nand.c index 87c1c7dd97eb4dde8e23d9127b9561bf0392be7b..5d627048c420de5d2516b2135736bb2aec65780b 100644 --- a/drivers/mtd/nand/raw/stm32_fmc2_nand.c +++ b/drivers/mtd/nand/raw/stm32_fmc2_nand.c @@ -862,8 +862,8 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, ret = dma_map_sg(nfc->dev, nfc->dma_data_sg.sgl, eccsteps, dma_data_dir); - if (ret < 0) - return ret; + if (!ret) + return -EIO; desc_data = dmaengine_prep_slave_sg(dma_ch, nfc->dma_data_sg.sgl, eccsteps, dma_transfer_dir, @@ -893,8 +893,10 @@ static int stm32_fmc2_nfc_xfer(struct nand_chip *chip, const u8 *buf, ret = dma_map_sg(nfc->dev, nfc->dma_ecc_sg.sgl, eccsteps, dma_data_dir); - if (ret < 0) + if (!ret) { + ret = -EIO; goto err_unmap_data; + } desc_ecc = dmaengine_prep_slave_sg(nfc->dma_ecc_ch, nfc->dma_ecc_sg.sgl, @@ -1799,9 +1801,8 @@ static int stm32_fmc2_nfc_parse_child(struct stm32_fmc2_nfc *nfc, nand->cs_used[i] = cs; } - nand->wp_gpio = devm_gpiod_get_from_of_node(nfc->dev, dn, - "wp-gpios", 0, - GPIOD_OUT_HIGH, "wp"); + nand->wp_gpio = devm_fwnode_gpiod_get(nfc->dev, of_fwnode_handle(dn), + "wp", GPIOD_OUT_HIGH, "wp"); if (IS_ERR(nand->wp_gpio)) { ret = PTR_ERR(nand->wp_gpio); if (ret != -ENOENT) diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 9d73910a7ae82744c57876e2db24d24a3462e79f..dacd9c0e8b202cf4cdf926f2fb70c2c17453e3b8 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -635,6 +635,7 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, { struct spinand_device *spinand = mtd_to_spinand(mtd); struct nand_device *nand = mtd_to_nanddev(mtd); + struct mtd_ecc_stats old_stats; unsigned int max_bitflips = 0; struct nand_io_iter iter; bool disable_ecc = false; @@ -646,6 +647,8 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, mutex_lock(&spinand->lock); + old_stats = mtd->ecc_stats; + nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) { if (disable_ecc) iter.req.mode = MTD_OPS_RAW; @@ -668,6 +671,13 @@ static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, ops->oobretlen += iter.req.ooblen; } + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } + mutex_unlock(&spinand->lock); if (ecc_failed && !ret) diff --git a/drivers/mtd/nftlcore.c b/drivers/mtd/nftlcore.c index 913db0dd6a8debde938a10e7b11af3421fa8ccef..64d319e959b239bbab101cf71918080cd733f13f 100644 --- a/drivers/mtd/nftlcore.c +++ b/drivers/mtd/nftlcore.c @@ -124,7 +124,7 @@ int nftl_read_oob(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf) { loff_t mask = mtd->writesize - 1; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; @@ -145,7 +145,7 @@ int nftl_write_oob(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf) { loff_t mask = mtd->writesize - 1; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; @@ -168,7 +168,7 @@ static int nftl_write(struct mtd_info *mtd, loff_t offs, size_t len, size_t *retlen, uint8_t *buf, uint8_t *oob) { loff_t mask = mtd->writesize - 1; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int res; ops.mode = MTD_OPS_PLACE_OOB; diff --git a/drivers/mtd/parsers/Kconfig b/drivers/mtd/parsers/Kconfig index b43df73927a02ca23dc4f40271bcf2ed9e6e967d..aaa06050c9bc94596f890607a36699aaa9788524 100644 --- a/drivers/mtd/parsers/Kconfig +++ b/drivers/mtd/parsers/Kconfig @@ -20,6 +20,16 @@ config MTD_BCM63XX_PARTS This provides partition parsing for BCM63xx devices with CFE bootloaders. +config MTD_BRCM_U_BOOT + tristate "Broadcom's U-Boot partition parser" + depends on ARCH_BCM4908 || COMPILE_TEST + help + Broadcom uses a custom way of storing U-Boot environment variables. + They are placed inside U-Boot partition itself at unspecified offset. + It's possible to locate them by looking for a custom header with a + magic value. This driver does that and creates subpartitions for + each found environment variables block. + config MTD_CMDLINE_PARTS tristate "Command line partition table parsing" depends on MTD @@ -69,8 +79,8 @@ config MTD_OF_PARTS config MTD_OF_PARTS_BCM4908 bool "BCM4908 partitioning support" - depends on MTD_OF_PARTS && (ARCH_BCM4908 || COMPILE_TEST) - default ARCH_BCM4908 + depends on MTD_OF_PARTS && (ARCH_BCMBCA || COMPILE_TEST) + default ARCH_BCMBCA help This provides partitions parser for BCM4908 family devices that can have multiple "firmware" partitions. It takes care of @@ -78,7 +88,7 @@ config MTD_OF_PARTS_BCM4908 config MTD_OF_PARTS_LINKSYS_NS bool "Linksys Northstar partitioning support" - depends on MTD_OF_PARTS && (ARCH_BCM_5301X || ARCH_BCM4908 || COMPILE_TEST) + depends on MTD_OF_PARTS && (ARCH_BCM_5301X || ARCH_BCMBCA || COMPILE_TEST) default ARCH_BCM_5301X help This provides partitions parser for Linksys devices based on Broadcom diff --git a/drivers/mtd/parsers/Makefile b/drivers/mtd/parsers/Makefile index 2fcf0ab9e7da7ceb3cea8c379d73df381381402c..23fa4de4016f1a6f323ff09470de6e6c0cf8b861 100644 --- a/drivers/mtd/parsers/Makefile +++ b/drivers/mtd/parsers/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_MTD_AR7_PARTS) += ar7part.o obj-$(CONFIG_MTD_BCM47XX_PARTS) += bcm47xxpart.o obj-$(CONFIG_MTD_BCM63XX_PARTS) += bcm63xxpart.o +obj-$(CONFIG_MTD_BRCM_U_BOOT) += brcm_u-boot.o obj-$(CONFIG_MTD_CMDLINE_PARTS) += cmdlinepart.o obj-$(CONFIG_MTD_OF_PARTS) += ofpart.o ofpart-y += ofpart_core.o diff --git a/drivers/mtd/parsers/brcm_u-boot.c b/drivers/mtd/parsers/brcm_u-boot.c new file mode 100644 index 0000000000000000000000000000000000000000..7c338dc7b8f39fc9e0dbb92a33ce8b02d9b614ec --- /dev/null +++ b/drivers/mtd/parsers/brcm_u-boot.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2022 Rafał Miłecki + */ + +#include +#include +#include +#include +#include + +#define BRCM_U_BOOT_MAX_OFFSET 0x200000 +#define BRCM_U_BOOT_STEP 0x1000 + +#define BRCM_U_BOOT_MAX_PARTS 2 + +#define BRCM_U_BOOT_MAGIC 0x75456e76 /* uEnv */ + +struct brcm_u_boot_header { + __le32 magic; + __le32 length; +} __packed; + +static const char *names[BRCM_U_BOOT_MAX_PARTS] = { + "u-boot-env", + "u-boot-env-backup", +}; + +static int brcm_u_boot_parse(struct mtd_info *mtd, + const struct mtd_partition **pparts, + struct mtd_part_parser_data *data) +{ + struct brcm_u_boot_header header; + struct mtd_partition *parts; + size_t bytes_read; + size_t offset; + int err; + int i = 0; + + parts = kcalloc(BRCM_U_BOOT_MAX_PARTS, sizeof(*parts), GFP_KERNEL); + if (!parts) + return -ENOMEM; + + for (offset = 0; + offset < min_t(size_t, mtd->size, BRCM_U_BOOT_MAX_OFFSET); + offset += BRCM_U_BOOT_STEP) { + err = mtd_read(mtd, offset, sizeof(header), &bytes_read, (uint8_t *)&header); + if (err && !mtd_is_bitflip(err)) { + pr_err("Failed to read from %s at 0x%zx: %d\n", mtd->name, offset, err); + continue; + } + + if (le32_to_cpu(header.magic) != BRCM_U_BOOT_MAGIC) + continue; + + parts[i].name = names[i]; + parts[i].offset = offset; + parts[i].size = sizeof(header) + le32_to_cpu(header.length); + i++; + pr_info("offset:0x%zx magic:0x%08x BINGO\n", offset, header.magic); + + if (i == BRCM_U_BOOT_MAX_PARTS) + break; + } + + *pparts = parts; + + return i; +}; + +static const struct of_device_id brcm_u_boot_of_match_table[] = { + { .compatible = "brcm,u-boot" }, + {}, +}; +MODULE_DEVICE_TABLE(of, brcm_u_boot_of_match_table); + +static struct mtd_part_parser brcm_u_boot_mtd_parser = { + .parse_fn = brcm_u_boot_parse, + .name = "brcm_u-boot", + .of_match_table = brcm_u_boot_of_match_table, +}; +module_mtd_part_parser(brcm_u_boot_mtd_parser); + +MODULE_LICENSE("GPL"); diff --git a/drivers/mtd/parsers/cmdlinepart.c b/drivers/mtd/parsers/cmdlinepart.c index 0ddff1a4b51fbf9ab5a6201d8a107bf556a44984..b34856def8169117b8376d889f4cc4ed7b57c7fa 100644 --- a/drivers/mtd/parsers/cmdlinepart.c +++ b/drivers/mtd/parsers/cmdlinepart.c @@ -193,7 +193,7 @@ static struct mtd_partition * newpart(char *s, parts[this_part].mask_flags = mask_flags; parts[this_part].add_flags = add_flags; if (name) - strlcpy(extra_mem, name, name_len + 1); + strscpy(extra_mem, name, name_len + 1); else sprintf(extra_mem, "Partition_%03d", this_part); parts[this_part].name = extra_mem; @@ -298,7 +298,7 @@ static int mtdpart_setup_real(char *s) this_mtd->parts = parts; this_mtd->num_parts = num_parts; this_mtd->mtd_id = (char*)(this_mtd + 1); - strlcpy(this_mtd->mtd_id, mtd_id, mtd_id_len + 1); + strscpy(this_mtd->mtd_id, mtd_id, mtd_id_len + 1); /* link into chain */ this_mtd->next = partitions; diff --git a/drivers/mtd/sm_ftl.c b/drivers/mtd/sm_ftl.c index 7f955fade83836a352df03c1c78449a7cc9dc68f..4cfec3b7b4461d585381a5037ac8c81e807bba10 100644 --- a/drivers/mtd/sm_ftl.c +++ b/drivers/mtd/sm_ftl.c @@ -239,7 +239,7 @@ static int sm_read_sector(struct sm_ftl *ftl, uint8_t *buffer, struct sm_oob *oob) { struct mtd_info *mtd = ftl->trans->mtd; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; struct sm_oob tmp_oob; int ret = -EIO; int try = 0; @@ -323,7 +323,7 @@ static int sm_write_sector(struct sm_ftl *ftl, int zone, int block, int boffset, uint8_t *buffer, struct sm_oob *oob) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; struct mtd_info *mtd = ftl->trans->mtd; int ret; diff --git a/drivers/mtd/ssfdc.c b/drivers/mtd/ssfdc.c index 1d05c121904ce157fef314f0c3daea3b9597b5c5..04da685c36be497dff88e0c6029adfcaf1e7c728 100644 --- a/drivers/mtd/ssfdc.c +++ b/drivers/mtd/ssfdc.c @@ -163,7 +163,7 @@ static int read_physical_sector(struct mtd_info *mtd, uint8_t *sect_buf, /* Read redundancy area (wrapper to MTD_READ_OOB */ static int read_raw_oob(struct mtd_info *mtd, loff_t offs, uint8_t *buf) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int ret; ops.mode = MTD_OPS_RAW; diff --git a/drivers/mtd/tests/mtd_nandecctest.c b/drivers/mtd/tests/mtd_nandecctest.c index c4f271314f5264acf7f6c0bfa1475f00619e0074..440988562cfdcb0340fee9fed2cfa471463cdeba 100644 --- a/drivers/mtd/tests/mtd_nandecctest.c +++ b/drivers/mtd/tests/mtd_nandecctest.c @@ -47,7 +47,7 @@ struct nand_ecc_test { static void single_bit_error_data(void *error_data, void *correct_data, size_t size) { - unsigned int offset = prandom_u32() % (size * BITS_PER_BYTE); + unsigned int offset = prandom_u32_max(size * BITS_PER_BYTE); memcpy(error_data, correct_data, size); __change_bit_le(offset, error_data); @@ -58,9 +58,9 @@ static void double_bit_error_data(void *error_data, void *correct_data, { unsigned int offset[2]; - offset[0] = prandom_u32() % (size * BITS_PER_BYTE); + offset[0] = prandom_u32_max(size * BITS_PER_BYTE); do { - offset[1] = prandom_u32() % (size * BITS_PER_BYTE); + offset[1] = prandom_u32_max(size * BITS_PER_BYTE); } while (offset[0] == offset[1]); memcpy(error_data, correct_data, size); @@ -71,7 +71,7 @@ static void double_bit_error_data(void *error_data, void *correct_data, static unsigned int random_ecc_bit(size_t size) { - unsigned int offset = prandom_u32() % (3 * BITS_PER_BYTE); + unsigned int offset = prandom_u32_max(3 * BITS_PER_BYTE); if (size == 256) { /* @@ -79,7 +79,7 @@ static unsigned int random_ecc_bit(size_t size) * and 17th bit) in ECC code for 256 byte data block */ while (offset == 16 || offset == 17) - offset = prandom_u32() % (3 * BITS_PER_BYTE); + offset = prandom_u32_max(3 * BITS_PER_BYTE); } return offset; @@ -266,7 +266,7 @@ static int nand_ecc_test_run(const size_t size) goto error; } - prandom_bytes(correct_data, size); + get_random_bytes(correct_data, size); ecc_sw_hamming_calculate(correct_data, size, correct_ecc, sm_order); for (i = 0; i < ARRAY_SIZE(nand_ecc_test); i++) { nand_ecc_test[i].prepare(error_data, error_ecc, diff --git a/drivers/mtd/tests/nandbiterrs.c b/drivers/mtd/tests/nandbiterrs.c index 08084c018a59a8863de361e1f9631a8c8b36df0a..98d7508f95b18d23b576eeb6ed1d17896de9eb60 100644 --- a/drivers/mtd/tests/nandbiterrs.c +++ b/drivers/mtd/tests/nandbiterrs.c @@ -99,7 +99,7 @@ static int write_page(int log) static int rewrite_page(int log) { int err = 0; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; if (log) pr_info("rewrite page\n"); diff --git a/drivers/mtd/tests/oobtest.c b/drivers/mtd/tests/oobtest.c index 532997e10e299ad3449e06dbea37642915877bfa..13fed398937e4034082cd0742b935b3d098714f5 100644 --- a/drivers/mtd/tests/oobtest.c +++ b/drivers/mtd/tests/oobtest.c @@ -56,7 +56,7 @@ static void do_vary_offset(void) static int write_eraseblock(int ebnum) { int i; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int err = 0; loff_t addr = (loff_t)ebnum * mtd->erasesize; @@ -165,7 +165,7 @@ static size_t memffshow(loff_t addr, loff_t offset, const void *cs, static int verify_eraseblock(int ebnum) { int i; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int err = 0; loff_t addr = (loff_t)ebnum * mtd->erasesize; size_t bitflips; @@ -260,7 +260,7 @@ static int verify_eraseblock(int ebnum) static int verify_eraseblock_in_one_go(int ebnum) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int err = 0; loff_t addr = (loff_t)ebnum * mtd->erasesize; size_t len = mtd->oobavail * pgcnt; @@ -338,7 +338,7 @@ static int __init mtd_oobtest_init(void) int err = 0; unsigned int i; uint64_t tmp; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; loff_t addr = 0, addr0; printk(KERN_INFO "\n"); diff --git a/drivers/mtd/tests/readtest.c b/drivers/mtd/tests/readtest.c index e70d588083a32057d808a91ce22dce7e1f06369b..99670ef91f2b891d99e3797cb0bd2bea2fcf21b5 100644 --- a/drivers/mtd/tests/readtest.c +++ b/drivers/mtd/tests/readtest.c @@ -47,7 +47,7 @@ static int read_eraseblock_by_page(int ebnum) err = ret; } if (mtd->oobsize) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; ops.mode = MTD_OPS_PLACE_OOB; ops.len = 0; diff --git a/drivers/mtd/tests/speedtest.c b/drivers/mtd/tests/speedtest.c index c9ec7086bfa1deba599e9db274cfcf01d9e27cfa..075bce32caa51beda6754c1616def1246ed7fb94 100644 --- a/drivers/mtd/tests/speedtest.c +++ b/drivers/mtd/tests/speedtest.c @@ -223,7 +223,7 @@ static int __init mtd_speedtest_init(void) if (!iobuf) goto out; - prandom_bytes(iobuf, mtd->erasesize); + get_random_bytes(iobuf, mtd->erasesize); bbt = kzalloc(ebcnt, GFP_KERNEL); if (!bbt) diff --git a/drivers/mtd/tests/stresstest.c b/drivers/mtd/tests/stresstest.c index cb29c8c1b37033f118333121ab13d7abc9eea68c..75b6ddc5dc4dae2202e33b65a79e2a4ac5c0958c 100644 --- a/drivers/mtd/tests/stresstest.c +++ b/drivers/mtd/tests/stresstest.c @@ -45,9 +45,8 @@ static int rand_eb(void) unsigned int eb; again: - eb = prandom_u32(); /* Read or write up 2 eraseblocks at a time - hence 'ebcnt - 1' */ - eb %= (ebcnt - 1); + eb = prandom_u32_max(ebcnt - 1); if (bbt[eb]) goto again; return eb; @@ -55,20 +54,12 @@ again: static int rand_offs(void) { - unsigned int offs; - - offs = prandom_u32(); - offs %= bufsize; - return offs; + return prandom_u32_max(bufsize); } static int rand_len(int offs) { - unsigned int len; - - len = prandom_u32(); - len %= (bufsize - offs); - return len; + return prandom_u32_max(bufsize - offs); } static int do_read(void) @@ -127,7 +118,7 @@ static int do_write(void) static int do_operation(void) { - if (prandom_u32() & 1) + if (prandom_u32_max(2)) return do_read(); else return do_write(); @@ -192,7 +183,7 @@ static int __init mtd_stresstest_init(void) goto out; for (i = 0; i < ebcnt; i++) offsets[i] = mtd->erasesize; - prandom_bytes(writebuf, bufsize); + get_random_bytes(writebuf, bufsize); bbt = kzalloc(ebcnt, GFP_KERNEL); if (!bbt) diff --git a/drivers/mtd/ubi/block.c b/drivers/mtd/ubi/block.c index 4cf67a2a0d04b3064ef82f03b0f45a46972df0b7..75eaecc8639f00b32ad925167a41410792ad3a89 100644 --- a/drivers/mtd/ubi/block.c +++ b/drivers/mtd/ubi/block.c @@ -409,7 +409,7 @@ int ubiblock_create(struct ubi_volume_info *vi) ret = blk_mq_alloc_tag_set(&dev->tag_set); if (ret) { dev_err(disk_to_dev(dev->gd), "blk_mq_alloc_tag_set failed"); - goto out_free_dev;; + goto out_free_dev; } @@ -441,7 +441,7 @@ int ubiblock_create(struct ubi_volume_info *vi) /* * Create one workqueue per volume (per registered block device). - * Rembember workqueues are cheap, they're not threads. + * Remember workqueues are cheap, they're not threads. */ dev->wq = alloc_workqueue("%s", 0, 0, gd->disk_name); if (!dev->wq) { diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index a32050fecabf308e113524427e41fc55ef30178c..a901f8edfa41d4c1228df067eee00af1fe3f3fbc 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -807,6 +807,7 @@ static int autoresize(struct ubi_device *ubi, int vol_id) * @ubi_num: number to assign to the new UBI device * @vid_hdr_offset: VID header offset * @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs + * @disable_fm: whether disable fastmap * * This function attaches MTD device @mtd_dev to UBI and assign @ubi_num number * to the newly created UBI device, unless @ubi_num is %UBI_DEV_NUM_AUTO, in @@ -814,11 +815,15 @@ static int autoresize(struct ubi_device *ubi, int vol_id) * automatically. Returns the new UBI device number in case of success and a * negative error code in case of failure. * + * If @disable_fm is true, ubi doesn't create new fastmap even the module param + * 'fm_autoconvert' is set, and existed old fastmap will be destroyed after + * doing full scanning. + * * Note, the invocations of this function has to be serialized by the * @ubi_devices_mutex. */ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, - int vid_hdr_offset, int max_beb_per1024) + int vid_hdr_offset, int max_beb_per1024, bool disable_fm) { struct ubi_device *ubi; int i, err; @@ -921,7 +926,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, UBI_FM_MIN_POOL_SIZE); ubi->fm_wl_pool.max_size = ubi->fm_pool.max_size / 2; - ubi->fm_disabled = !fm_autoconvert; + ubi->fm_disabled = (!fm_autoconvert || disable_fm) ? 1 : 0; if (fm_debug) ubi_enable_dbg_chk_fastmap(ubi); @@ -962,7 +967,7 @@ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, if (!ubi->fm_buf) goto out_free; #endif - err = ubi_attach(ubi, 0); + err = ubi_attach(ubi, disable_fm ? 1 : 0); if (err) { ubi_err(ubi, "failed to attach mtd%d, error %d", mtd->index, err); @@ -1242,7 +1247,8 @@ static int __init ubi_init(void) mutex_lock(&ubi_devices_mutex); err = ubi_attach_mtd_dev(mtd, p->ubi_num, - p->vid_hdr_offs, p->max_beb_per1024); + p->vid_hdr_offs, p->max_beb_per1024, + false); mutex_unlock(&ubi_devices_mutex); if (err < 0) { pr_err("UBI error: cannot attach mtd%d\n", diff --git a/drivers/mtd/ubi/cdev.c b/drivers/mtd/ubi/cdev.c index cc9a28cf9d82725ae21405b6bf8123c30d653b89..f43430b9c1e65c326e3c870a7743d32d5181ccd8 100644 --- a/drivers/mtd/ubi/cdev.c +++ b/drivers/mtd/ubi/cdev.c @@ -672,7 +672,7 @@ static int verify_rsvol_req(const struct ubi_device *ubi, * @req: volumes re-name request * * This is a helper function for the volume re-name IOCTL which validates the - * the request, opens the volume and calls corresponding volumes management + * request, opens the volume and calls corresponding volumes management * function. Returns zero in case of success and a negative error code in case * of failure. */ @@ -1041,7 +1041,7 @@ static long ctrl_cdev_ioctl(struct file *file, unsigned int cmd, */ mutex_lock(&ubi_devices_mutex); err = ubi_attach_mtd_dev(mtd, req.ubi_num, req.vid_hdr_offset, - req.max_beb_per1024); + req.max_beb_per1024, !!req.disable_fm); mutex_unlock(&ubi_devices_mutex); if (err < 0) put_mtd_device(mtd); diff --git a/drivers/mtd/ubi/debug.c b/drivers/mtd/ubi/debug.c index 31d427ee191a3805624025389fc55f7e10b6a7fb..908d0e0885574bd7a68e82d4b49372bb908ba7f7 100644 --- a/drivers/mtd/ubi/debug.c +++ b/drivers/mtd/ubi/debug.c @@ -590,7 +590,7 @@ int ubi_dbg_power_cut(struct ubi_device *ubi, int caller) if (ubi->dbg.power_cut_max > ubi->dbg.power_cut_min) { range = ubi->dbg.power_cut_max - ubi->dbg.power_cut_min; - ubi->dbg.power_cut_counter += prandom_u32() % range; + ubi->dbg.power_cut_counter += prandom_u32_max(range); } return 0; } diff --git a/drivers/mtd/ubi/debug.h b/drivers/mtd/ubi/debug.h index 118248a5d7d487f005d7b35613e7bb3b88a5918e..dc8d8f83657a0f782c369816074c75f22891f27b 100644 --- a/drivers/mtd/ubi/debug.h +++ b/drivers/mtd/ubi/debug.h @@ -73,7 +73,7 @@ static inline int ubi_dbg_is_bgt_disabled(const struct ubi_device *ubi) static inline int ubi_dbg_is_bitflip(const struct ubi_device *ubi) { if (ubi->dbg.emulate_bitflips) - return !(prandom_u32() % 200); + return !prandom_u32_max(200); return 0; } @@ -87,7 +87,7 @@ static inline int ubi_dbg_is_bitflip(const struct ubi_device *ubi) static inline int ubi_dbg_is_write_failure(const struct ubi_device *ubi) { if (ubi->dbg.emulate_io_failures) - return !(prandom_u32() % 500); + return !prandom_u32_max(500); return 0; } @@ -101,7 +101,7 @@ static inline int ubi_dbg_is_write_failure(const struct ubi_device *ubi) static inline int ubi_dbg_is_erase_failure(const struct ubi_device *ubi) { if (ubi->dbg.emulate_io_failures) - return !(prandom_u32() % 400); + return !prandom_u32_max(400); return 0; } diff --git a/drivers/mtd/ubi/eba.c b/drivers/mtd/ubi/eba.c index ccc5979642b780e439b887b82f98f36e9c04f0a8..09c408c45a62186a67cc1efe4bfcd66b258dc49d 100644 --- a/drivers/mtd/ubi/eba.c +++ b/drivers/mtd/ubi/eba.c @@ -377,7 +377,7 @@ static int leb_write_lock(struct ubi_device *ubi, int vol_id, int lnum) * * This function locks a logical eraseblock for writing if there is no * contention and does nothing if there is contention. Returns %0 in case of - * success, %1 in case of contention, and and a negative error code in case of + * success, %1 in case of contention, and a negative error code in case of * failure. */ static int leb_write_trylock(struct ubi_device *ubi, int vol_id, int lnum) diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c index 6e95c4b1473e61e85c86068b4547ce765777626c..ca2d9efe62c3c75063b41ecacb6cbbd6f107146d 100644 --- a/drivers/mtd/ubi/fastmap.c +++ b/drivers/mtd/ubi/fastmap.c @@ -20,8 +20,7 @@ static inline unsigned long *init_seen(struct ubi_device *ubi) if (!ubi_dbg_chk_fastmap(ubi)) return NULL; - ret = kcalloc(BITS_TO_LONGS(ubi->peb_count), sizeof(unsigned long), - GFP_KERNEL); + ret = bitmap_zalloc(ubi->peb_count, GFP_KERNEL); if (!ret) return ERR_PTR(-ENOMEM); @@ -34,7 +33,7 @@ static inline unsigned long *init_seen(struct ubi_device *ubi) */ static inline void free_seen(unsigned long *seen) { - kfree(seen); + bitmap_free(seen); } /** @@ -1108,8 +1107,7 @@ int ubi_fastmap_init_checkmap(struct ubi_volume *vol, int leb_count) if (!ubi->fast_attach) return 0; - vol->checkmap = kcalloc(BITS_TO_LONGS(leb_count), sizeof(unsigned long), - GFP_KERNEL); + vol->checkmap = bitmap_zalloc(leb_count, GFP_KERNEL); if (!vol->checkmap) return -ENOMEM; @@ -1118,7 +1116,7 @@ int ubi_fastmap_init_checkmap(struct ubi_volume *vol, int leb_count) void ubi_fastmap_destroy_checkmap(struct ubi_volume *vol) { - kfree(vol->checkmap); + bitmap_free(vol->checkmap); } /** diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c index 8a7306cc1947148ddba9b9bd3be3023d7f45cd3d..01b644861253347f7a0bed1a7cc05f0f867f7177 100644 --- a/drivers/mtd/ubi/io.c +++ b/drivers/mtd/ubi/io.c @@ -1147,7 +1147,7 @@ fail: * @ubi: UBI device description object * @pnum: the physical eraseblock number to check * - * This function returns zero if the erase counter header is all right and and + * This function returns zero if the erase counter header is all right and * a negative error code if not or if an error occurred. */ static int self_check_peb_ec_hdr(const struct ubi_device *ubi, int pnum) diff --git a/drivers/mtd/ubi/ubi-media.h b/drivers/mtd/ubi/ubi-media.h index 386db0598e954c30051fae9181b469681a7880ab..2c9cd3b6434f4060f68ba10d0fc71a4723bec11d 100644 --- a/drivers/mtd/ubi/ubi-media.h +++ b/drivers/mtd/ubi/ubi-media.h @@ -131,7 +131,7 @@ enum { * is changed radically. This field is duplicated in the volume identifier * header. * - * The @vid_hdr_offset and @data_offset fields contain the offset of the the + * The @vid_hdr_offset and @data_offset fields contain the offset of the * volume identifier header and user data, relative to the beginning of the * physical eraseblock. These values have to be the same for all physical * eraseblocks. diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h index 078112e23dfd504bcc8c53fad383ea4c2ff944e5..c8f1bd4fa10080c775042ecd9fd3514f366fc82d 100644 --- a/drivers/mtd/ubi/ubi.h +++ b/drivers/mtd/ubi/ubi.h @@ -86,7 +86,7 @@ void ubi_err(const struct ubi_device *ubi, const char *fmt, ...); * Error codes returned by the I/O sub-system. * * UBI_IO_FF: the read region of flash contains only 0xFFs - * UBI_IO_FF_BITFLIPS: the same as %UBI_IO_FF, but also also there was a data + * UBI_IO_FF_BITFLIPS: the same as %UBI_IO_FF, but also there was a data * integrity error reported by the MTD driver * (uncorrectable ECC error in case of NAND) * UBI_IO_BAD_HDR: the EC or VID header is corrupted (bad magic or CRC) @@ -281,7 +281,7 @@ struct ubi_eba_leb_desc { /** * struct ubi_volume - UBI volume description data structure. - * @dev: device object to make use of the the Linux device model + * @dev: device object to make use of the Linux device model * @cdev: character device object to create character device * @ubi: reference to the UBI device description object * @vol_id: volume ID @@ -439,7 +439,7 @@ struct ubi_debug_info { /** * struct ubi_device - UBI device description structure - * @dev: UBI device object to use the the Linux device model + * @dev: UBI device object to use the Linux device model * @cdev: character device object to create character device * @ubi_num: UBI device number * @ubi_name: UBI device name @@ -937,7 +937,8 @@ int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum, /* build.c */ int ubi_attach_mtd_dev(struct mtd_info *mtd, int ubi_num, - int vid_hdr_offset, int max_beb_per1024); + int vid_hdr_offset, int max_beb_per1024, + bool disable_fm); int ubi_detach_mtd_dev(int ubi_num, int anyway); struct ubi_device *ubi_get_device(int ubi_num); void ubi_put_device(struct ubi_device *ubi); diff --git a/drivers/mtd/ubi/vmt.c b/drivers/mtd/ubi/vmt.c index 6ea95ade4ca6b0e8dad9b3057eab1b465768f297..8fcc0bdf06358d8f3c210e56ee43e97c9ee39821 100644 --- a/drivers/mtd/ubi/vmt.c +++ b/drivers/mtd/ubi/vmt.c @@ -623,7 +623,7 @@ void ubi_free_volume(struct ubi_device *ubi, struct ubi_volume *vol) * @ubi: UBI device description object * @vol_id: volume ID * - * Returns zero if volume is all right and a a negative error code if not. + * Returns zero if volume is all right and a negative error code if not. */ static int self_check_volume(struct ubi_device *ubi, int vol_id) { @@ -776,7 +776,7 @@ fail: * self_check_volumes - check information about all volumes. * @ubi: UBI device description object * - * Returns zero if volumes are all right and a a negative error code if not. + * Returns zero if volumes are all right and a negative error code if not. */ static int self_check_volumes(struct ubi_device *ubi) { diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index 55bae06cf4083f4edc247208d3a78fa8090c5495..68eb0f21b3fe2150d5a89542daf31679eed7c331 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -376,7 +376,7 @@ static struct ubi_wl_entry *find_mean_wl_entry(struct ubi_device *ubi, * refill_wl_user_pool(). * @ubi: UBI device description object * - * This function returns a a wear leveling entry in case of success and + * This function returns a wear leveling entry in case of success and * NULL in case of failure. */ static struct ubi_wl_entry *wl_get_wle(struct ubi_device *ubi) @@ -429,7 +429,7 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) /** * sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @e: the the physical eraseblock to erase + * @e: the physical eraseblock to erase * @torture: if the physical eraseblock has to be tortured * * This function returns zero in case of success and a negative error code in @@ -1016,7 +1016,7 @@ static int ensure_wear_leveling(struct ubi_device *ubi, int nested) /* * If the ubi->scrub tree is not empty, scrubbing is needed, and the - * the WL worker has to be scheduled anyway. + * WL worker has to be scheduled anyway. */ if (!ubi->scrub.rb_node) { #ifdef CONFIG_MTD_UBI_FASTMAP @@ -1464,7 +1464,7 @@ static bool scrub_possible(struct ubi_device *ubi, struct ubi_wl_entry *e) * ubi_bitflip_check - Check an eraseblock for bitflips and scrub it if needed. * @ubi: UBI device description object * @pnum: the physical eraseblock to schedule - * @force: dont't read the block, assume bitflips happened and take action. + * @force: don't read the block, assume bitflips happened and take action. * * This function reads the given eraseblock and checks if bitflips occured. * In case of bitflips, the eraseblock is scheduled for scrubbing. diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 94c889802566a1bb38c63f59666e210929e6bdd1..9e63b8c43f3e2e562fc0dbef2bbf4e6f49f32220 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -76,6 +76,7 @@ config WIREGUARD tristate "WireGuard secure network tunnel" depends on NET && INET depends on IPV6 || !IPV6 + depends on !KMSAN # KMSAN doesn't support the crypto configs below select NET_UDP_TUNNEL select DST_CACHE select CRYPTO @@ -85,8 +86,6 @@ config WIREGUARD select CRYPTO_POLY1305_X86_64 if X86 && 64BIT select CRYPTO_BLAKE2S_X86 if X86 && 64BIT select CRYPTO_CURVE25519_X86 if X86 && 64BIT - select ARM_CRYPTO if ARM - select ARM64_CRYPTO if ARM64 select CRYPTO_CHACHA20_NEON if ARM || (ARM64 && KERNEL_MODE_NEON) select CRYPTO_POLY1305_NEON if ARM64 && KERNEL_MODE_NEON select CRYPTO_POLY1305_ARM if ARM @@ -500,6 +499,8 @@ config NET_SB1000 source "drivers/net/phy/Kconfig" +source "drivers/net/pse-pd/Kconfig" + source "drivers/net/can/Kconfig" source "drivers/net/mctp/Kconfig" diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 3f1192d3c52d3c1eb4d9e9eebcc3096e384c447a..6ce076462dbfd26829472daa13096eea4136d266 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_NET) += loopback.o obj-$(CONFIG_NETDEV_LEGACY_INIT) += Space.o obj-$(CONFIG_NETCONSOLE) += netconsole.o obj-y += phy/ +obj-y += pse-pd/ obj-y += mdio/ obj-y += pcs/ obj-$(CONFIG_RIONET) += rionet.o diff --git a/drivers/net/Space.c b/drivers/net/Space.c index f475eef14390cb7badfbc714b25b826cda13b002..83214e2e70abc89d5fc9c32b6008eae53503d16f 100644 --- a/drivers/net/Space.c +++ b/drivers/net/Space.c @@ -68,7 +68,7 @@ static int netdev_boot_setup_add(char *name, struct ifmap *map) for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { memset(s[i].name, 0, sizeof(s[i].name)); - strlcpy(s[i].name, name, IFNAMSIZ); + strscpy(s[i].name, name, IFNAMSIZ); memcpy(&s[i].map, map, sizeof(s[i].map)); break; } diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 9a247eb7679c2117fb47fc14dc3a048b2c15f497..2d20be6ffb7e59a9d699da5ba6485334ddb88586 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -2894,8 +2894,7 @@ static void amt_event_work(struct work_struct *work) amt_event_send_request(amt); break; default: - if (skb) - kfree_skb(skb); + kfree_skb(skb); break; } } @@ -3033,8 +3032,7 @@ static int amt_dev_stop(struct net_device *dev) cancel_work_sync(&amt->event_wq); for (i = 0; i < AMT_MAX_EVENTS; i++) { skb = amt->events[i].skb; - if (skb) - kfree_skb(skb); + kfree_skb(skb); amt->events[i].event = AMT_EVENT_NONE; amt->events[i].skb = NULL; } diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c index d7fb33c078e81a1c8242d5af793b328a19b79bd2..e58a1e0cadd2e1fc8b6278fe8d0700da4b90afc6 100644 --- a/drivers/net/bonding/bond_3ad.c +++ b/drivers/net/bonding/bond_3ad.c @@ -84,11 +84,13 @@ enum ad_link_speed_type { static const u8 null_mac_addr[ETH_ALEN + 2] __long_aligned = { 0, 0, 0, 0, 0, 0 }; -static u16 ad_ticks_per_sec; + +static const u16 ad_ticks_per_sec = 1000 / AD_TIMER_INTERVAL; static const int ad_delta_in_ticks = (AD_TIMER_INTERVAL * HZ) / 1000; -static const u8 lacpdu_mcast_addr[ETH_ALEN + 2] __long_aligned = - MULTICAST_LACPDU_ADDR; +const u8 lacpdu_mcast_addr[ETH_ALEN + 2] __long_aligned = { + 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 +}; /* ================= main 802.3ad protocol functions ================== */ static int ad_lacpdu_send(struct port *port); @@ -2001,36 +2003,24 @@ void bond_3ad_initiate_agg_selection(struct bonding *bond, int timeout) /** * bond_3ad_initialize - initialize a bond's 802.3ad parameters and structures * @bond: bonding struct to work on - * @tick_resolution: tick duration (millisecond resolution) * * Can be called only after the mac address of the bond is set. */ -void bond_3ad_initialize(struct bonding *bond, u16 tick_resolution) +void bond_3ad_initialize(struct bonding *bond) { - /* check that the bond is not initialized yet */ - if (!MAC_ADDRESS_EQUAL(&(BOND_AD_INFO(bond).system.sys_mac_addr), - bond->dev->dev_addr)) { - - BOND_AD_INFO(bond).aggregator_identifier = 0; - - BOND_AD_INFO(bond).system.sys_priority = - bond->params.ad_actor_sys_prio; - if (is_zero_ether_addr(bond->params.ad_actor_system)) - BOND_AD_INFO(bond).system.sys_mac_addr = - *((struct mac_addr *)bond->dev->dev_addr); - else - BOND_AD_INFO(bond).system.sys_mac_addr = - *((struct mac_addr *)bond->params.ad_actor_system); - - /* initialize how many times this module is called in one - * second (should be about every 100ms) - */ - ad_ticks_per_sec = tick_resolution; + BOND_AD_INFO(bond).aggregator_identifier = 0; + BOND_AD_INFO(bond).system.sys_priority = + bond->params.ad_actor_sys_prio; + if (is_zero_ether_addr(bond->params.ad_actor_system)) + BOND_AD_INFO(bond).system.sys_mac_addr = + *((struct mac_addr *)bond->dev->dev_addr); + else + BOND_AD_INFO(bond).system.sys_mac_addr = + *((struct mac_addr *)bond->params.ad_actor_system); - bond_3ad_initiate_agg_selection(bond, - AD_AGGREGATOR_SELECTION_TIMER * - ad_ticks_per_sec); - } + bond_3ad_initiate_agg_selection(bond, + AD_AGGREGATOR_SELECTION_TIMER * + ad_ticks_per_sec); } /** diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 50e60843020ce2840e1019d5822ccee44c32f80f..e84c49bf4d0c37ff2649bd2baaf737bd84303814 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -865,12 +865,8 @@ static void bond_hw_addr_flush(struct net_device *bond_dev, dev_uc_unsync(slave_dev, bond_dev); dev_mc_unsync(slave_dev, bond_dev); - if (BOND_MODE(bond) == BOND_MODE_8023AD) { - /* del lacpdu mc addr from mc list */ - u8 lacpdu_multicast[ETH_ALEN] = MULTICAST_LACPDU_ADDR; - - dev_mc_del(slave_dev, lacpdu_multicast); - } + if (BOND_MODE(bond) == BOND_MODE_8023AD) + dev_mc_del(slave_dev, lacpdu_mcast_addr); } /*--------------------------- Active slave change ---------------------------*/ @@ -890,7 +886,8 @@ static void bond_hw_addr_swap(struct bonding *bond, struct slave *new_active, if (bond->dev->flags & IFF_ALLMULTI) dev_set_allmulti(old_active->dev, -1); - bond_hw_addr_flush(bond->dev, old_active->dev); + if (bond->dev->flags & IFF_UP) + bond_hw_addr_flush(bond->dev, old_active->dev); } if (new_active) { @@ -901,10 +898,12 @@ static void bond_hw_addr_swap(struct bonding *bond, struct slave *new_active, if (bond->dev->flags & IFF_ALLMULTI) dev_set_allmulti(new_active->dev, 1); - netif_addr_lock_bh(bond->dev); - dev_uc_sync(new_active->dev, bond->dev); - dev_mc_sync(new_active->dev, bond->dev); - netif_addr_unlock_bh(bond->dev); + if (bond->dev->flags & IFF_UP) { + netif_addr_lock_bh(bond->dev); + dev_uc_sync(new_active->dev, bond->dev); + dev_mc_sync(new_active->dev, bond->dev); + netif_addr_unlock_bh(bond->dev); + } } } @@ -2081,7 +2080,7 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, /* Initialize AD with the number of times that the AD timer is called in 1 second * can be called only after the mac address of the bond is set */ - bond_3ad_initialize(bond, 1000/AD_TIMER_INTERVAL); + bond_3ad_initialize(bond); } else { SLAVE_AD_INFO(new_slave)->id = SLAVE_AD_INFO(prev_slave)->id + 1; @@ -2166,16 +2165,14 @@ int bond_enslave(struct net_device *bond_dev, struct net_device *slave_dev, } } - netif_addr_lock_bh(bond_dev); - dev_mc_sync_multiple(slave_dev, bond_dev); - dev_uc_sync_multiple(slave_dev, bond_dev); - netif_addr_unlock_bh(bond_dev); - - if (BOND_MODE(bond) == BOND_MODE_8023AD) { - /* add lacpdu mc addr to mc list */ - u8 lacpdu_multicast[ETH_ALEN] = MULTICAST_LACPDU_ADDR; + if (bond_dev->flags & IFF_UP) { + netif_addr_lock_bh(bond_dev); + dev_mc_sync_multiple(slave_dev, bond_dev); + dev_uc_sync_multiple(slave_dev, bond_dev); + netif_addr_unlock_bh(bond_dev); - dev_mc_add(slave_dev, lacpdu_multicast); + if (BOND_MODE(bond) == BOND_MODE_8023AD) + dev_mc_add(slave_dev, lacpdu_mcast_addr); } } @@ -2447,7 +2444,8 @@ static int __bond_release_one(struct net_device *bond_dev, if (old_flags & IFF_ALLMULTI) dev_set_allmulti(slave_dev, -1); - bond_hw_addr_flush(bond_dev, slave_dev); + if (old_flags & IFF_UP) + bond_hw_addr_flush(bond_dev, slave_dev); } slave_disable_netpoll(slave); @@ -3167,6 +3165,9 @@ static void bond_ns_send_all(struct bonding *bond, struct slave *slave) found: if (!ipv6_dev_get_saddr(dev_net(dst->dev), dst->dev, &targets[i], 0, &saddr)) bond_ns_send(slave, &targets[i], &saddr, tags); + else + bond_ns_send(slave, &targets[i], &in6addr_any, tags); + dst_release(dst); kfree(tags); } @@ -3198,12 +3199,19 @@ static bool bond_has_this_ip6(struct bonding *bond, struct in6_addr *addr) return ret; } -static void bond_validate_ns(struct bonding *bond, struct slave *slave, +static void bond_validate_na(struct bonding *bond, struct slave *slave, struct in6_addr *saddr, struct in6_addr *daddr) { int i; - if (ipv6_addr_any(saddr) || !bond_has_this_ip6(bond, daddr)) { + /* Ignore NAs that: + * 1. Source address is unspecified address. + * 2. Dest address is neither all-nodes multicast address nor + * exist on bond interface. + */ + if (ipv6_addr_any(saddr) || + (!ipv6_addr_equal(daddr, &in6addr_linklocal_allnodes) && + !bond_has_this_ip6(bond, daddr))) { slave_dbg(bond->dev, slave->dev, "%s: sip %pI6c tip %pI6c not found\n", __func__, saddr, daddr); return; @@ -3246,14 +3254,14 @@ static int bond_na_rcv(const struct sk_buff *skb, struct bonding *bond, * see bond_arp_rcv(). */ if (bond_is_active_slave(slave)) - bond_validate_ns(bond, slave, saddr, daddr); + bond_validate_na(bond, slave, saddr, daddr); else if (curr_active_slave && time_after(slave_last_rx(bond, curr_active_slave), curr_active_slave->last_link_up)) - bond_validate_ns(bond, slave, saddr, daddr); + bond_validate_na(bond, slave, saddr, daddr); else if (curr_arp_slave && bond_time_in_interval(bond, slave_last_tx(curr_arp_slave), 1)) - bond_validate_ns(bond, slave, saddr, daddr); + bond_validate_na(bond, slave, saddr, daddr); out: return RX_HANDLER_ANOTHER; @@ -4174,6 +4182,12 @@ static int bond_open(struct net_device *bond_dev) struct list_head *iter; struct slave *slave; + if (BOND_MODE(bond) == BOND_MODE_ROUNDROBIN && !bond->rr_tx_counter) { + bond->rr_tx_counter = alloc_percpu(u32); + if (!bond->rr_tx_counter) + return -ENOMEM; + } + /* reset slave->backup and slave->inactive */ if (bond_has_slaves(bond)) { bond_for_each_slave(bond, slave, iter) { @@ -4211,6 +4225,9 @@ static int bond_open(struct net_device *bond_dev) /* register to receive LACPDUs */ bond->recv_probe = bond_3ad_lacpdu_recv; bond_3ad_initiate_agg_selection(bond, 1); + + bond_for_each_slave(bond, slave, iter) + dev_mc_add(slave->dev, lacpdu_mcast_addr); } if (bond_mode_can_use_xmit_hash(bond)) @@ -4222,6 +4239,7 @@ static int bond_open(struct net_device *bond_dev) static int bond_close(struct net_device *bond_dev) { struct bonding *bond = netdev_priv(bond_dev); + struct slave *slave; bond_work_cancel_all(bond); bond->send_peer_notif = 0; @@ -4229,6 +4247,19 @@ static int bond_close(struct net_device *bond_dev) bond_alb_deinitialize(bond); bond->recv_probe = NULL; + if (bond_uses_primary(bond)) { + rcu_read_lock(); + slave = rcu_dereference(bond->curr_active_slave); + if (slave) + bond_hw_addr_flush(bond_dev, slave->dev); + rcu_read_unlock(); + } else { + struct list_head *iter; + + bond_for_each_slave(bond, slave, iter) + bond_hw_addr_flush(bond_dev, slave->dev); + } + return 0; } @@ -4775,7 +4806,7 @@ static u32 bond_rr_gen_slave_id(struct bonding *bond) switch (packets_per_slave) { case 0: - slave_id = prandom_u32(); + slave_id = get_random_u32(); break; case 1: slave_id = this_cpu_inc_return(*bond->rr_tx_counter); @@ -5619,7 +5650,7 @@ static int bond_ethtool_get_link_ksettings(struct net_device *bond_dev, static void bond_ethtool_get_drvinfo(struct net_device *bond_dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d", BOND_ABI_VERSION); } @@ -6218,15 +6249,6 @@ static int bond_init(struct net_device *bond_dev) if (!bond->wq) return -ENOMEM; - if (BOND_MODE(bond) == BOND_MODE_ROUNDROBIN) { - bond->rr_tx_counter = alloc_percpu(u32); - if (!bond->rr_tx_counter) { - destroy_workqueue(bond->wq); - bond->wq = NULL; - return -ENOMEM; - } - } - spin_lock_init(&bond->stats_lock); netdev_lockdep_set_classes(bond_dev); diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c index 9b5a5df23d2184aef8c9fc49c04f8e77da6e9e85..8996bd0a194ab6ea2c5aaa1591f956201fcd07dd 100644 --- a/drivers/net/bonding/bond_sysfs.c +++ b/drivers/net/bonding/bond_sysfs.c @@ -47,10 +47,10 @@ static ssize_t bonding_show_bonds(struct class *cls, /* not enough space for another interface name */ if ((PAGE_SIZE - res) > 10) res = PAGE_SIZE - 10; - res += sprintf(buf + res, "++more++ "); + res += sysfs_emit_at(buf, res, "++more++ "); break; } - res += sprintf(buf + res, "%s ", bond->dev->name); + res += sysfs_emit_at(buf, res, "%s ", bond->dev->name); } if (res) buf[res-1] = '\n'; /* eat the leftover space */ @@ -178,10 +178,10 @@ static ssize_t bonding_show_slaves(struct device *d, /* not enough space for another interface name */ if ((PAGE_SIZE - res) > 10) res = PAGE_SIZE - 10; - res += sprintf(buf + res, "++more++ "); + res += sysfs_emit_at(buf, res, "++more++ "); break; } - res += sprintf(buf + res, "%s ", slave->dev->name); + res += sysfs_emit_at(buf, res, "%s ", slave->dev->name); } rtnl_unlock(); @@ -203,7 +203,7 @@ static ssize_t bonding_show_mode(struct device *d, val = bond_opt_get_val(BOND_OPT_MODE, BOND_MODE(bond)); - return sprintf(buf, "%s %d\n", val->string, BOND_MODE(bond)); + return sysfs_emit(buf, "%s %d\n", val->string, BOND_MODE(bond)); } static DEVICE_ATTR(mode, 0644, bonding_show_mode, bonding_sysfs_store_option); @@ -217,7 +217,7 @@ static ssize_t bonding_show_xmit_hash(struct device *d, val = bond_opt_get_val(BOND_OPT_XMIT_HASH, bond->params.xmit_policy); - return sprintf(buf, "%s %d\n", val->string, bond->params.xmit_policy); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.xmit_policy); } static DEVICE_ATTR(xmit_hash_policy, 0644, bonding_show_xmit_hash, bonding_sysfs_store_option); @@ -233,7 +233,7 @@ static ssize_t bonding_show_arp_validate(struct device *d, val = bond_opt_get_val(BOND_OPT_ARP_VALIDATE, bond->params.arp_validate); - return sprintf(buf, "%s %d\n", val->string, bond->params.arp_validate); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.arp_validate); } static DEVICE_ATTR(arp_validate, 0644, bonding_show_arp_validate, bonding_sysfs_store_option); @@ -248,7 +248,7 @@ static ssize_t bonding_show_arp_all_targets(struct device *d, val = bond_opt_get_val(BOND_OPT_ARP_ALL_TARGETS, bond->params.arp_all_targets); - return sprintf(buf, "%s %d\n", + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.arp_all_targets); } static DEVICE_ATTR(arp_all_targets, 0644, @@ -265,7 +265,7 @@ static ssize_t bonding_show_fail_over_mac(struct device *d, val = bond_opt_get_val(BOND_OPT_FAIL_OVER_MAC, bond->params.fail_over_mac); - return sprintf(buf, "%s %d\n", val->string, bond->params.fail_over_mac); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.fail_over_mac); } static DEVICE_ATTR(fail_over_mac, 0644, bonding_show_fail_over_mac, bonding_sysfs_store_option); @@ -277,7 +277,7 @@ static ssize_t bonding_show_arp_interval(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.arp_interval); + return sysfs_emit(buf, "%d\n", bond->params.arp_interval); } static DEVICE_ATTR(arp_interval, 0644, bonding_show_arp_interval, bonding_sysfs_store_option); @@ -292,8 +292,8 @@ static ssize_t bonding_show_arp_targets(struct device *d, for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) { if (bond->params.arp_targets[i]) - res += sprintf(buf + res, "%pI4 ", - &bond->params.arp_targets[i]); + res += sysfs_emit_at(buf, res, "%pI4 ", + &bond->params.arp_targets[i]); } if (res) buf[res-1] = '\n'; /* eat the leftover space */ @@ -310,7 +310,7 @@ static ssize_t bonding_show_missed_max(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%u\n", bond->params.missed_max); + return sysfs_emit(buf, "%u\n", bond->params.missed_max); } static DEVICE_ATTR(arp_missed_max, 0644, bonding_show_missed_max, bonding_sysfs_store_option); @@ -322,7 +322,7 @@ static ssize_t bonding_show_downdelay(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.downdelay * bond->params.miimon); + return sysfs_emit(buf, "%d\n", bond->params.downdelay * bond->params.miimon); } static DEVICE_ATTR(downdelay, 0644, bonding_show_downdelay, bonding_sysfs_store_option); @@ -333,7 +333,7 @@ static ssize_t bonding_show_updelay(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.updelay * bond->params.miimon); + return sysfs_emit(buf, "%d\n", bond->params.updelay * bond->params.miimon); } static DEVICE_ATTR(updelay, 0644, @@ -345,8 +345,8 @@ static ssize_t bonding_show_peer_notif_delay(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", - bond->params.peer_notif_delay * bond->params.miimon); + return sysfs_emit(buf, "%d\n", + bond->params.peer_notif_delay * bond->params.miimon); } static DEVICE_ATTR(peer_notif_delay, 0644, bonding_show_peer_notif_delay, bonding_sysfs_store_option); @@ -361,7 +361,7 @@ static ssize_t bonding_show_lacp_active(struct device *d, val = bond_opt_get_val(BOND_OPT_LACP_ACTIVE, bond->params.lacp_active); - return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_active); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_active); } static DEVICE_ATTR(lacp_active, 0644, bonding_show_lacp_active, bonding_sysfs_store_option); @@ -375,7 +375,7 @@ static ssize_t bonding_show_lacp_rate(struct device *d, val = bond_opt_get_val(BOND_OPT_LACP_RATE, bond->params.lacp_fast); - return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_fast); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_fast); } static DEVICE_ATTR(lacp_rate, 0644, bonding_show_lacp_rate, bonding_sysfs_store_option); @@ -386,7 +386,7 @@ static ssize_t bonding_show_min_links(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%u\n", bond->params.min_links); + return sysfs_emit(buf, "%u\n", bond->params.min_links); } static DEVICE_ATTR(min_links, 0644, bonding_show_min_links, bonding_sysfs_store_option); @@ -400,7 +400,7 @@ static ssize_t bonding_show_ad_select(struct device *d, val = bond_opt_get_val(BOND_OPT_AD_SELECT, bond->params.ad_select); - return sprintf(buf, "%s %d\n", val->string, bond->params.ad_select); + return sysfs_emit(buf, "%s %d\n", val->string, bond->params.ad_select); } static DEVICE_ATTR(ad_select, 0644, bonding_show_ad_select, bonding_sysfs_store_option); @@ -412,7 +412,7 @@ static ssize_t bonding_show_num_peer_notif(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.num_peer_notif); + return sysfs_emit(buf, "%d\n", bond->params.num_peer_notif); } static DEVICE_ATTR(num_grat_arp, 0644, bonding_show_num_peer_notif, bonding_sysfs_store_option); @@ -426,7 +426,7 @@ static ssize_t bonding_show_miimon(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.miimon); + return sysfs_emit(buf, "%d\n", bond->params.miimon); } static DEVICE_ATTR(miimon, 0644, bonding_show_miimon, bonding_sysfs_store_option); @@ -443,7 +443,7 @@ static ssize_t bonding_show_primary(struct device *d, rcu_read_lock(); primary = rcu_dereference(bond->primary_slave); if (primary) - count = sprintf(buf, "%s\n", primary->dev->name); + count = sysfs_emit(buf, "%s\n", primary->dev->name); rcu_read_unlock(); return count; @@ -462,8 +462,8 @@ static ssize_t bonding_show_primary_reselect(struct device *d, val = bond_opt_get_val(BOND_OPT_PRIMARY_RESELECT, bond->params.primary_reselect); - return sprintf(buf, "%s %d\n", - val->string, bond->params.primary_reselect); + return sysfs_emit(buf, "%s %d\n", + val->string, bond->params.primary_reselect); } static DEVICE_ATTR(primary_reselect, 0644, bonding_show_primary_reselect, bonding_sysfs_store_option); @@ -475,7 +475,7 @@ static ssize_t bonding_show_carrier(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.use_carrier); + return sysfs_emit(buf, "%d\n", bond->params.use_carrier); } static DEVICE_ATTR(use_carrier, 0644, bonding_show_carrier, bonding_sysfs_store_option); @@ -493,7 +493,7 @@ static ssize_t bonding_show_active_slave(struct device *d, rcu_read_lock(); slave_dev = bond_option_active_slave_get_rcu(bond); if (slave_dev) - count = sprintf(buf, "%s\n", slave_dev->name); + count = sysfs_emit(buf, "%s\n", slave_dev->name); rcu_read_unlock(); return count; @@ -509,7 +509,7 @@ static ssize_t bonding_show_mii_status(struct device *d, struct bonding *bond = to_bond(d); bool active = netif_carrier_ok(bond->dev); - return sprintf(buf, "%s\n", active ? "up" : "down"); + return sysfs_emit(buf, "%s\n", active ? "up" : "down"); } static DEVICE_ATTR(mii_status, 0444, bonding_show_mii_status, NULL); @@ -524,9 +524,9 @@ static ssize_t bonding_show_ad_aggregator(struct device *d, if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct ad_info ad_info; - count = sprintf(buf, "%d\n", - bond_3ad_get_active_agg_info(bond, &ad_info) - ? 0 : ad_info.aggregator_id); + count = sysfs_emit(buf, "%d\n", + bond_3ad_get_active_agg_info(bond, &ad_info) + ? 0 : ad_info.aggregator_id); } return count; @@ -545,9 +545,9 @@ static ssize_t bonding_show_ad_num_ports(struct device *d, if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct ad_info ad_info; - count = sprintf(buf, "%d\n", - bond_3ad_get_active_agg_info(bond, &ad_info) - ? 0 : ad_info.ports); + count = sysfs_emit(buf, "%d\n", + bond_3ad_get_active_agg_info(bond, &ad_info) + ? 0 : ad_info.ports); } return count; @@ -566,9 +566,9 @@ static ssize_t bonding_show_ad_actor_key(struct device *d, if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) { struct ad_info ad_info; - count = sprintf(buf, "%d\n", - bond_3ad_get_active_agg_info(bond, &ad_info) - ? 0 : ad_info.actor_key); + count = sysfs_emit(buf, "%d\n", + bond_3ad_get_active_agg_info(bond, &ad_info) + ? 0 : ad_info.actor_key); } return count; @@ -587,9 +587,9 @@ static ssize_t bonding_show_ad_partner_key(struct device *d, if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) { struct ad_info ad_info; - count = sprintf(buf, "%d\n", - bond_3ad_get_active_agg_info(bond, &ad_info) - ? 0 : ad_info.partner_key); + count = sysfs_emit(buf, "%d\n", + bond_3ad_get_active_agg_info(bond, &ad_info) + ? 0 : ad_info.partner_key); } return count; @@ -609,7 +609,7 @@ static ssize_t bonding_show_ad_partner_mac(struct device *d, struct ad_info ad_info; if (!bond_3ad_get_active_agg_info(bond, &ad_info)) - count = sprintf(buf, "%pM\n", ad_info.partner_system); + count = sysfs_emit(buf, "%pM\n", ad_info.partner_system); } return count; @@ -634,11 +634,11 @@ static ssize_t bonding_show_queue_id(struct device *d, /* not enough space for another interface_name:queue_id pair */ if ((PAGE_SIZE - res) > 10) res = PAGE_SIZE - 10; - res += sprintf(buf + res, "++more++ "); + res += sysfs_emit_at(buf, res, "++more++ "); break; } - res += sprintf(buf + res, "%s:%d ", - slave->dev->name, slave->queue_id); + res += sysfs_emit_at(buf, res, "%s:%d ", + slave->dev->name, slave->queue_id); } if (res) buf[res-1] = '\n'; /* eat the leftover space */ @@ -658,7 +658,7 @@ static ssize_t bonding_show_slaves_active(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.all_slaves_active); + return sysfs_emit(buf, "%d\n", bond->params.all_slaves_active); } static DEVICE_ATTR(all_slaves_active, 0644, bonding_show_slaves_active, bonding_sysfs_store_option); @@ -670,7 +670,7 @@ static ssize_t bonding_show_resend_igmp(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.resend_igmp); + return sysfs_emit(buf, "%d\n", bond->params.resend_igmp); } static DEVICE_ATTR(resend_igmp, 0644, bonding_show_resend_igmp, bonding_sysfs_store_option); @@ -682,7 +682,7 @@ static ssize_t bonding_show_lp_interval(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.lp_interval); + return sysfs_emit(buf, "%d\n", bond->params.lp_interval); } static DEVICE_ATTR(lp_interval, 0644, bonding_show_lp_interval, bonding_sysfs_store_option); @@ -693,7 +693,7 @@ static ssize_t bonding_show_tlb_dynamic_lb(struct device *d, { struct bonding *bond = to_bond(d); - return sprintf(buf, "%d\n", bond->params.tlb_dynamic_lb); + return sysfs_emit(buf, "%d\n", bond->params.tlb_dynamic_lb); } static DEVICE_ATTR(tlb_dynamic_lb, 0644, bonding_show_tlb_dynamic_lb, bonding_sysfs_store_option); @@ -705,7 +705,7 @@ static ssize_t bonding_show_packets_per_slave(struct device *d, struct bonding *bond = to_bond(d); unsigned int packets_per_slave = bond->params.packets_per_slave; - return sprintf(buf, "%u\n", packets_per_slave); + return sysfs_emit(buf, "%u\n", packets_per_slave); } static DEVICE_ATTR(packets_per_slave, 0644, bonding_show_packets_per_slave, bonding_sysfs_store_option); @@ -717,7 +717,7 @@ static ssize_t bonding_show_ad_actor_sys_prio(struct device *d, struct bonding *bond = to_bond(d); if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) - return sprintf(buf, "%hu\n", bond->params.ad_actor_sys_prio); + return sysfs_emit(buf, "%hu\n", bond->params.ad_actor_sys_prio); return 0; } @@ -731,7 +731,7 @@ static ssize_t bonding_show_ad_actor_system(struct device *d, struct bonding *bond = to_bond(d); if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) - return sprintf(buf, "%pM\n", bond->params.ad_actor_system); + return sysfs_emit(buf, "%pM\n", bond->params.ad_actor_system); return 0; } @@ -746,7 +746,7 @@ static ssize_t bonding_show_ad_user_port_key(struct device *d, struct bonding *bond = to_bond(d); if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) - return sprintf(buf, "%hu\n", bond->params.ad_user_port_key); + return sysfs_emit(buf, "%hu\n", bond->params.ad_user_port_key); return 0; } diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c index 69b0a3751dff4ba0d54d07fbf27f6910ef2bcc86..313866f2c0e49ac96299ffea307b1613955713ec 100644 --- a/drivers/net/bonding/bond_sysfs_slave.c +++ b/drivers/net/bonding/bond_sysfs_slave.c @@ -22,30 +22,30 @@ static ssize_t state_show(struct slave *slave, char *buf) { switch (bond_slave_state(slave)) { case BOND_STATE_ACTIVE: - return sprintf(buf, "active\n"); + return sysfs_emit(buf, "active\n"); case BOND_STATE_BACKUP: - return sprintf(buf, "backup\n"); + return sysfs_emit(buf, "backup\n"); default: - return sprintf(buf, "UNKNOWN\n"); + return sysfs_emit(buf, "UNKNOWN\n"); } } static SLAVE_ATTR_RO(state); static ssize_t mii_status_show(struct slave *slave, char *buf) { - return sprintf(buf, "%s\n", bond_slave_link_status(slave->link)); + return sysfs_emit(buf, "%s\n", bond_slave_link_status(slave->link)); } static SLAVE_ATTR_RO(mii_status); static ssize_t link_failure_count_show(struct slave *slave, char *buf) { - return sprintf(buf, "%d\n", slave->link_failure_count); + return sysfs_emit(buf, "%d\n", slave->link_failure_count); } static SLAVE_ATTR_RO(link_failure_count); static ssize_t perm_hwaddr_show(struct slave *slave, char *buf) { - return sprintf(buf, "%*phC\n", + return sysfs_emit(buf, "%*phC\n", slave->dev->addr_len, slave->perm_hwaddr); } @@ -53,7 +53,7 @@ static SLAVE_ATTR_RO(perm_hwaddr); static ssize_t queue_id_show(struct slave *slave, char *buf) { - return sprintf(buf, "%d\n", slave->queue_id); + return sysfs_emit(buf, "%d\n", slave->queue_id); } static SLAVE_ATTR_RO(queue_id); @@ -64,11 +64,11 @@ static ssize_t ad_aggregator_id_show(struct slave *slave, char *buf) if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { agg = SLAVE_AD_INFO(slave)->port.aggregator; if (agg) - return sprintf(buf, "%d\n", - agg->aggregator_identifier); + return sysfs_emit(buf, "%d\n", + agg->aggregator_identifier); } - return sprintf(buf, "N/A\n"); + return sysfs_emit(buf, "N/A\n"); } static SLAVE_ATTR_RO(ad_aggregator_id); @@ -79,11 +79,11 @@ static ssize_t ad_actor_oper_port_state_show(struct slave *slave, char *buf) if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { ad_port = &SLAVE_AD_INFO(slave)->port; if (ad_port->aggregator) - return sprintf(buf, "%u\n", + return sysfs_emit(buf, "%u\n", ad_port->actor_oper_port_state); } - return sprintf(buf, "N/A\n"); + return sysfs_emit(buf, "N/A\n"); } static SLAVE_ATTR_RO(ad_actor_oper_port_state); @@ -94,11 +94,11 @@ static ssize_t ad_partner_oper_port_state_show(struct slave *slave, char *buf) if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { ad_port = &SLAVE_AD_INFO(slave)->port; if (ad_port->aggregator) - return sprintf(buf, "%u\n", + return sysfs_emit(buf, "%u\n", ad_port->partner_oper.port_state); } - return sprintf(buf, "N/A\n"); + return sysfs_emit(buf, "N/A\n"); } static SLAVE_ATTR_RO(ad_partner_oper_port_state); diff --git a/drivers/net/can/c_can/c_can.h b/drivers/net/can/c_can/c_can.h index f23a03300a81e365d0999db9a291b73c13808b86..029cd8194ed54e7707ee8cabb9fe6ac91435451c 100644 --- a/drivers/net/can/c_can/c_can.h +++ b/drivers/net/can/c_can/c_can.h @@ -235,9 +235,22 @@ static inline u8 c_can_get_tx_tail(const struct c_can_tx_ring *ring) return ring->tail & (ring->obj_num - 1); } -static inline u8 c_can_get_tx_free(const struct c_can_tx_ring *ring) +static inline u8 c_can_get_tx_free(const struct c_can_priv *priv, + const struct c_can_tx_ring *ring) { - return ring->obj_num - (ring->head - ring->tail); + u8 head = c_can_get_tx_head(ring); + u8 tail = c_can_get_tx_tail(ring); + + if (priv->type == BOSCH_D_CAN) + return ring->obj_num - (ring->head - ring->tail); + + /* This is not a FIFO. C/D_CAN sends out the buffers + * prioritized. The lowest buffer number wins. + */ + if (head < tail) + return 0; + + return ring->obj_num - head; } #endif /* C_CAN_H */ diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c index dc8132862f333aff1b034a11853bcaf506654b8f..d6605dbb7737bb7227301e8760d00a86fd78c3a7 100644 --- a/drivers/net/can/c_can/c_can_main.c +++ b/drivers/net/can/c_can/c_can_main.c @@ -429,7 +429,7 @@ static void c_can_setup_receive_object(struct net_device *dev, int iface, static bool c_can_tx_busy(const struct c_can_priv *priv, const struct c_can_tx_ring *tx_ring) { - if (c_can_get_tx_free(tx_ring) > 0) + if (c_can_get_tx_free(priv, tx_ring) > 0) return false; netif_stop_queue(priv->dev); @@ -437,7 +437,7 @@ static bool c_can_tx_busy(const struct c_can_priv *priv, /* Memory barrier before checking tx_free (head and tail) */ smp_mb(); - if (c_can_get_tx_free(tx_ring) == 0) { + if (c_can_get_tx_free(priv, tx_ring) == 0) { netdev_dbg(priv->dev, "Stopping tx-queue (tx_head=0x%08x, tx_tail=0x%08x, len=%d).\n", tx_ring->head, tx_ring->tail, @@ -465,7 +465,7 @@ static netdev_tx_t c_can_start_xmit(struct sk_buff *skb, idx = c_can_get_tx_head(tx_ring); tx_ring->head++; - if (c_can_get_tx_free(tx_ring) == 0) + if (c_can_get_tx_free(priv, tx_ring) == 0) netif_stop_queue(dev); if (idx < c_can_get_tx_tail(tx_ring)) @@ -748,7 +748,7 @@ static void c_can_do_tx(struct net_device *dev) return; tx_ring->tail += pkts; - if (c_can_get_tx_free(tx_ring)) { + if (c_can_get_tx_free(priv, tx_ring)) { /* Make sure that anybody stopping the queue after * this sees the new tx_ring->tail. */ @@ -760,8 +760,7 @@ static void c_can_do_tx(struct net_device *dev) stats->tx_packets += pkts; tail = c_can_get_tx_tail(tx_ring); - - if (tail == 0) { + if (priv->type == BOSCH_D_CAN && tail == 0) { u8 head = c_can_get_tx_head(tx_ring); /* Start transmission for all cached messages */ diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c index 3c18d028bd8ced64bdc42c5cab6825fb39d76caa..b8da15ea6ad9a1b050b71cb52ebfe00c2890fbd6 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_base.c +++ b/drivers/net/can/ctucanfd/ctucanfd_base.c @@ -657,7 +657,6 @@ static void ctucan_read_rx_frame(struct ctucan_priv *priv, struct canfd_frame *c cf->can_id = (idw >> 18) & CAN_SFF_MASK; /* BRS, ESI, RTR Flags */ - cf->flags = 0; if (FIELD_GET(REG_FRAME_FORMAT_W_FDF, ffw)) { if (FIELD_GET(REG_FRAME_FORMAT_W_BRS, ffw)) cf->flags |= CANFD_BRS; @@ -1425,7 +1424,7 @@ int ctucan_probe_common(struct device *dev, void __iomem *addr, int irq, unsigne priv->can.clock.freq = can_clk_rate; - netif_napi_add(ndev, &priv->napi, ctucan_rx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, ctucan_rx_poll); ret = register_candev(ndev); if (ret) { diff --git a/drivers/net/can/ctucanfd/ctucanfd_platform.c b/drivers/net/can/ctucanfd/ctucanfd_platform.c index 89d54c2151e190fe407b1060310bbccde987bfbb..f83684f006ea133a2c747b06ae62e52831a1a36c 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_platform.c +++ b/drivers/net/can/ctucanfd/ctucanfd_platform.c @@ -58,7 +58,6 @@ static int ctucan_platform_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); addr = devm_ioremap_resource(dev, res); if (IS_ERR(addr)) { - dev_err(dev, "Cannot remap address.\n"); ret = PTR_ERR(addr); goto err; } diff --git a/drivers/net/can/dev/rx-offload.c b/drivers/net/can/dev/rx-offload.c index a32a01c172d4fae7e05c19ea99b9fad4002ef659..81ebf0562c89d24ec4008d0d69e0256bcc099cb3 100644 --- a/drivers/net/can/dev/rx-offload.c +++ b/drivers/net/can/dev/rx-offload.c @@ -247,7 +247,7 @@ unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload, struct net_device *dev = offload->dev; struct net_device_stats *stats = &dev->stats; struct sk_buff *skb; - u8 len; + unsigned int len; int err; skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr); @@ -329,7 +329,7 @@ static int can_rx_offload_init_queue(struct net_device *dev, { offload->dev = dev; - /* Limit queue len to 4x the weight (rounted to next power of two) */ + /* Limit queue len to 4x the weight (rounded to next power of two) */ offload->skb_queue_len_max = 2 << fls(weight); offload->skb_queue_len_max *= 4; skb_queue_head_init(&offload->skb_queue); diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 07e0feac862925776eb73898b288985c6959814d..791a51e2f5d6414678076f2c204d4b4e903c00b5 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -91,8 +91,8 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, EXPORT_SYMBOL_GPL(can_put_echo_skb); struct sk_buff * -__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, - unsigned int *frame_len_ptr) +__can_get_echo_skb(struct net_device *dev, unsigned int idx, + unsigned int *len_ptr, unsigned int *frame_len_ptr) { struct can_priv *priv = netdev_priv(dev); @@ -108,16 +108,12 @@ __can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr, */ struct sk_buff *skb = priv->echo_skb[idx]; struct can_skb_priv *can_skb_priv = can_skb_prv(skb); - struct canfd_frame *cf = (struct canfd_frame *)skb->data; if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) skb_tstamp_tx(skb, skb_hwtstamps(skb)); /* get the real payload length for netdev statistics */ - if (cf->can_id & CAN_RTR_FLAG) - *len_ptr = 0; - else - *len_ptr = cf->len; + *len_ptr = can_skb_get_data_len(skb); if (frame_len_ptr) *frame_len_ptr = can_skb_priv->frame_len; @@ -147,7 +143,7 @@ unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx, unsigned int *frame_len_ptr) { struct sk_buff *skb; - u8 len; + unsigned int len; skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr); if (!skb) @@ -191,6 +187,20 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx, } EXPORT_SYMBOL_GPL(can_free_echo_skb); +/* fill common values for CAN sk_buffs */ +static void init_can_skb_reserve(struct sk_buff *skb) +{ + skb->pkt_type = PACKET_BROADCAST; + skb->ip_summed = CHECKSUM_UNNECESSARY; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + + can_skb_reserve(skb); + can_skb_prv(skb)->skbcnt = 0; +} + struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) { struct sk_buff *skb; @@ -204,16 +214,8 @@ struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) } skb->protocol = htons(ETH_P_CAN); - skb->pkt_type = PACKET_BROADCAST; - skb->ip_summed = CHECKSUM_UNNECESSARY; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - - can_skb_reserve(skb); + init_can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; *cf = skb_put_zero(skb, sizeof(struct can_frame)); @@ -235,23 +237,51 @@ struct sk_buff *alloc_canfd_skb(struct net_device *dev, } skb->protocol = htons(ETH_P_CANFD); - skb->pkt_type = PACKET_BROADCAST; - skb->ip_summed = CHECKSUM_UNNECESSARY; - - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - - can_skb_reserve(skb); + init_can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; *cfd = skb_put_zero(skb, sizeof(struct canfd_frame)); + /* set CAN FD flag by default */ + (*cfd)->flags = CANFD_FDF; + return skb; } EXPORT_SYMBOL_GPL(alloc_canfd_skb); +struct sk_buff *alloc_canxl_skb(struct net_device *dev, + struct canxl_frame **cxl, + unsigned int data_len) +{ + struct sk_buff *skb; + + if (data_len < CANXL_MIN_DLEN || data_len > CANXL_MAX_DLEN) + goto out_error; + + skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) + + CANXL_HDR_SIZE + data_len); + if (unlikely(!skb)) + goto out_error; + + skb->protocol = htons(ETH_P_CANXL); + init_can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; + + *cxl = skb_put_zero(skb, CANXL_HDR_SIZE + data_len); + + /* set CAN XL flag and length information by default */ + (*cxl)->flags = CANXL_XLF; + (*cxl)->len = data_len; + + return skb; + +out_error: + *cxl = NULL; + + return NULL; +} +EXPORT_SYMBOL_GPL(alloc_canxl_skb); + struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf) { struct sk_buff *skb; @@ -291,6 +321,14 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) skb_reset_mac_header(skb); skb_reset_network_header(skb); skb_reset_transport_header(skb); + + /* set CANFD_FDF flag for CAN FD frames */ + if (can_is_canfd_skb(skb)) { + struct canfd_frame *cfd; + + cfd = (struct canfd_frame *)skb->data; + cfd->flags |= CANFD_FDF; + } } return true; @@ -299,18 +337,25 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) /* Drop a given socketbuffer if it does not contain a valid CAN frame. */ bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb) { - const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct can_priv *priv = netdev_priv(dev); - if (skb->protocol == htons(ETH_P_CAN)) { - if (unlikely(skb->len != CAN_MTU || - cfd->len > CAN_MAX_DLEN)) + switch (ntohs(skb->protocol)) { + case ETH_P_CAN: + if (!can_is_can_skb(skb)) goto inval_skb; - } else if (skb->protocol == htons(ETH_P_CANFD)) { - if (unlikely(skb->len != CANFD_MTU || - cfd->len > CANFD_MAX_DLEN)) + break; + + case ETH_P_CANFD: + if (!can_is_canfd_skb(skb)) goto inval_skb; - } else { + break; + + case ETH_P_CANXL: + if (!can_is_canxl_skb(skb)) + goto inval_skb; + break; + + default: goto inval_skb; } diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c index f857968efed7d3179d192f48ba3430723d480fb5..5ee38e586fd8026d88257a8a24005e85b5ef285f 100644 --- a/drivers/net/can/flexcan/flexcan-core.c +++ b/drivers/net/can/flexcan/flexcan-core.c @@ -295,45 +295,45 @@ static_assert(sizeof(struct flexcan_regs) == 0x4 * 18 + 0xfb8); static const struct flexcan_devtype_data fsl_mcf5441x_devtype_data = { .quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_NR_IRQ_3 | FLEXCAN_QUIRK_NR_MB_16 | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_FIFO, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_FIFO, }; static const struct flexcan_devtype_data fsl_p1010_devtype_data = { .quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_FIFO, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_FIFO, }; static const struct flexcan_devtype_data fsl_imx25_devtype_data = { .quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE | FLEXCAN_QUIRK_BROKEN_PERR_STATE | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_FIFO, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_FIFO, }; static const struct flexcan_devtype_data fsl_imx28_devtype_data = { .quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_FIFO, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_FIFO, }; static const struct flexcan_devtype_data fsl_imx6q_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static const struct flexcan_devtype_data fsl_imx8qm_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static struct flexcan_devtype_data fsl_imx8mp_devtype_data = { @@ -341,23 +341,23 @@ static struct flexcan_devtype_data fsl_imx8mp_devtype_data = { FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR | FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static const struct flexcan_devtype_data fsl_vf610_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SUPPORT_ECC | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static const struct flexcan_devtype_data fsl_ls1021a_r2_devtype_data = { .quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_USE_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static const struct flexcan_devtype_data fsl_lx2160a_r1_devtype_data = { @@ -365,8 +365,8 @@ static const struct flexcan_devtype_data fsl_lx2160a_r1_devtype_data = { FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR, + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR, }; static const struct can_bittiming_const flexcan_bittiming_const = { @@ -941,11 +941,6 @@ static struct sk_buff *flexcan_mailbox_read(struct can_rx_offload *offload, u32 reg_ctrl, reg_id, reg_iflag1; int i; - if (unlikely(drop)) { - skb = ERR_PTR(-ENOBUFS); - goto mark_as_read; - } - mb = flexcan_get_mb(priv, n); if (priv->devtype_data.quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) { @@ -974,6 +969,11 @@ static struct sk_buff *flexcan_mailbox_read(struct can_rx_offload *offload, reg_ctrl = priv->read(&mb->can_ctrl); } + if (unlikely(drop)) { + skb = ERR_PTR(-ENOBUFS); + goto mark_as_read; + } + if (reg_ctrl & FLEXCAN_MB_CNT_EDL) skb = alloc_canfd_skb(offload->dev, &cfd); else @@ -2085,20 +2085,20 @@ static int flexcan_probe(struct platform_device *pdev) if ((devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_FD) && !((devtype_data->quirks & (FLEXCAN_QUIRK_USE_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR | - FLEXCAN_QUIRK_SUPPPORT_RX_FIFO)) == + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR | + FLEXCAN_QUIRK_SUPPORT_RX_FIFO)) == (FLEXCAN_QUIRK_USE_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR))) { + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR))) { dev_err(&pdev->dev, "CAN-FD mode doesn't work in RX-FIFO mode!\n"); return -EINVAL; } if ((devtype_data->quirks & - (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)) == - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR) { + (FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR)) == + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR) { dev_err(&pdev->dev, "Quirks (0x%08x) inconsistent: RX_MAILBOX_RX supported but not RX_MAILBOX\n", devtype_data->quirks); @@ -2177,8 +2177,7 @@ static int flexcan_probe(struct platform_device *pdev) err = flexcan_setup_stop_mode(pdev); if (err < 0) { - if (err != -EPROBE_DEFER) - dev_err(&pdev->dev, "setup stop mode failed\n"); + dev_err_probe(&pdev->dev, err, "setup stop mode failed\n"); goto failed_setup_stop_mode; } diff --git a/drivers/net/can/flexcan/flexcan.h b/drivers/net/can/flexcan/flexcan.h index 8621a8ea1deaf322c54a7cf20fc411e45ba45694..025c3417031f42b6b3865081b7407d3e4bc02af5 100644 --- a/drivers/net/can/flexcan/flexcan.h +++ b/drivers/net/can/flexcan/flexcan.h @@ -63,11 +63,11 @@ /* Setup 16 mailboxes */ #define FLEXCAN_QUIRK_NR_MB_16 BIT(13) /* Device supports RX via mailboxes */ -#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX BIT(14) +#define FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX BIT(14) /* Device supports RTR reception via mailboxes */ -#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR BIT(15) +#define FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR BIT(15) /* Device supports RX via FIFO */ -#define FLEXCAN_QUIRK_SUPPPORT_RX_FIFO BIT(16) +#define FLEXCAN_QUIRK_SUPPORT_RX_FIFO BIT(16) struct flexcan_devtype_data { u32 quirks; /* quirks needed for different IP cores */ @@ -121,7 +121,7 @@ flexcan_supports_rx_mailbox(const struct flexcan_priv *priv) { const u32 quirks = priv->devtype_data.quirks; - return quirks & FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX; + return quirks & FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX; } static inline bool @@ -129,10 +129,10 @@ flexcan_supports_rx_mailbox_rtr(const struct flexcan_priv *priv) { const u32 quirks = priv->devtype_data.quirks; - return (quirks & (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)) == - (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX | - FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR); + return (quirks & (FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR)) == + (FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX | + FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR); } static inline bool @@ -140,7 +140,7 @@ flexcan_supports_rx_fifo(const struct flexcan_priv *priv) { const u32 quirks = priv->devtype_data.quirks; - return quirks & FLEXCAN_QUIRK_SUPPPORT_RX_FIFO; + return quirks & FLEXCAN_QUIRK_SUPPORT_RX_FIFO; } static inline bool @@ -149,7 +149,7 @@ flexcan_active_rx_rtr(const struct flexcan_priv *priv) const u32 quirks = priv->devtype_data.quirks; if (quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) { - if (quirks & FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR) + if (quirks & FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR) return true; } else { /* RX-FIFO is always RTR capable */ diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index ad7a89b95da7128ac563ae3154b652248e85ddb6..8d42b7e6661f2d37bd1f9414236bf4e0dc6d0ffa 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -973,7 +973,7 @@ static int ifi_canfd_plat_probe(struct platform_device *pdev) priv->ndev = ndev; priv->base = addr; - netif_napi_add(ndev, &priv->napi, ifi_canfd_poll, 64); + netif_napi_add(ndev, &priv->napi, ifi_canfd_poll); priv->can.state = CAN_STATE_STOPPED; diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index ed54c0b3c7d462de94156ab5f9c49cee4c367476..4e9680c8eb3447ae18d3fa07530ed6f8bad47802 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -329,12 +329,9 @@ MODULE_DEVICE_TABLE(pci, kvaser_pciefd_id_table); static int kvaser_pciefd_spi_wait_loop(struct kvaser_pciefd *pcie, int msk) { u32 res; - int ret; - - ret = readl_poll_timeout(pcie->reg_base + KVASER_PCIEFD_SPI_STATUS_REG, - res, res & msk, 0, 10); - return ret; + return readl_poll_timeout(pcie->reg_base + KVASER_PCIEFD_SPI_STATUS_REG, + res, res & msk, 0, 10); } static int kvaser_pciefd_spi_cmd(struct kvaser_pciefd *pcie, const u8 *tx, diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index 4709c012b1dc90c2ccf58d0c3b3d826911b7deb4..dcb582563d5e43c82b8421bdae2f127159ab2576 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -1467,8 +1467,7 @@ static int m_can_dev_setup(struct m_can_classdev *cdev) } if (!cdev->is_peripheral) - netif_napi_add(dev, &cdev->napi, - m_can_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &cdev->napi, m_can_poll); /* Shared properties of all M_CAN versions */ cdev->version = m_can_version; diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 27085b796e752b8f4b0faef2a718684712cfdce8..567620d215f83b09150eea01f53d25d7b3670c25 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1880,10 +1880,9 @@ static int rcar_canfd_probe(struct platform_device *pdev) /* Global controller context */ gpriv = devm_kzalloc(&pdev->dev, sizeof(*gpriv), GFP_KERNEL); - if (!gpriv) { - err = -ENOMEM; - goto fail_dev; - } + if (!gpriv) + return -ENOMEM; + gpriv->pdev = pdev; gpriv->channels_mask = channels_mask; gpriv->fdmode = fdmode; @@ -1904,12 +1903,9 @@ static int rcar_canfd_probe(struct platform_device *pdev) /* Peripheral clock */ gpriv->clkp = devm_clk_get(&pdev->dev, "fck"); - if (IS_ERR(gpriv->clkp)) { - err = PTR_ERR(gpriv->clkp); - dev_err(&pdev->dev, "cannot get peripheral clock, error %d\n", - err); - goto fail_dev; - } + if (IS_ERR(gpriv->clkp)) + return dev_err_probe(&pdev->dev, PTR_ERR(gpriv->clkp), + "cannot get peripheral clock\n"); /* fCAN clock: Pick External clock. If not available fallback to * CANFD clock @@ -1917,12 +1913,10 @@ static int rcar_canfd_probe(struct platform_device *pdev) gpriv->can_clk = devm_clk_get(&pdev->dev, "can_clk"); if (IS_ERR(gpriv->can_clk) || (clk_get_rate(gpriv->can_clk) == 0)) { gpriv->can_clk = devm_clk_get(&pdev->dev, "canfd"); - if (IS_ERR(gpriv->can_clk)) { - err = PTR_ERR(gpriv->can_clk); - dev_err(&pdev->dev, - "cannot get canfd clock, error %d\n", err); - goto fail_dev; - } + if (IS_ERR(gpriv->can_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(gpriv->can_clk), + "cannot get canfd clock\n"); + gpriv->fcan = RCANFD_CANFDCLK; } else { diff --git a/drivers/net/can/sja1000/peak_pcmcia.c b/drivers/net/can/sja1000/peak_pcmcia.c index 131a084c35351e921ed0d835e33a67f7373974d3..ebd5941c3f53038f884dde3ec3139454b1379185 100644 --- a/drivers/net/can/sja1000/peak_pcmcia.c +++ b/drivers/net/can/sja1000/peak_pcmcia.c @@ -478,7 +478,7 @@ static void pcan_free_channels(struct pcan_pccard *card) if (!netdev) continue; - strlcpy(name, netdev->name, IFNAMSIZ); + strscpy(name, netdev->name, IFNAMSIZ); unregister_sja1000dev(netdev); diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index 98dfd5f295a7128045161b50c7e18bb1b3b13ef4..1bb1129b04503364b403160d8612bfc4ad18ad15 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -661,8 +661,6 @@ static const struct ethtool_ops sja1000_ethtool_ops = { int register_sja1000dev(struct net_device *dev) { - int ret; - if (!sja1000_probe_chip(dev)) return -ENODEV; @@ -673,9 +671,7 @@ int register_sja1000dev(struct net_device *dev) set_reset_mode(dev); chipset_init(dev); - ret = register_candev(dev); - - return ret; + return register_candev(dev); } EXPORT_SYMBOL_GPL(register_sja1000dev); diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c index 81bc741905fd080d8e0fee8bf5f7966fb3d3c8df..6779d5357069f6df903160b19008fabc849333db 100644 --- a/drivers/net/can/sja1000/sja1000_platform.c +++ b/drivers/net/can/sja1000/sja1000_platform.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -103,6 +104,11 @@ static void sp_technologic_init(struct sja1000_priv *priv, struct device_node *o spin_lock_init(&tp->io_lock); } +static void sp_rzn1_init(struct sja1000_priv *priv, struct device_node *of) +{ + priv->flags = SJA1000_QUIRK_NO_CDR_REG; +} + static void sp_populate(struct sja1000_priv *priv, struct sja1000_platform_data *pdata, unsigned long resource_mem_flags) @@ -153,11 +159,13 @@ static void sp_populate_of(struct sja1000_priv *priv, struct device_node *of) priv->write_reg = sp_write_reg8; } - err = of_property_read_u32(of, "nxp,external-clock-frequency", &prop); - if (!err) - priv->can.clock.freq = prop / 2; - else - priv->can.clock.freq = SP_CAN_CLOCK; /* default */ + if (!priv->can.clock.freq) { + err = of_property_read_u32(of, "nxp,external-clock-frequency", &prop); + if (!err) + priv->can.clock.freq = prop / 2; + else + priv->can.clock.freq = SP_CAN_CLOCK; /* default */ + } err = of_property_read_u32(of, "nxp,tx-output-mode", &prop); if (!err) @@ -192,8 +200,13 @@ static struct sja1000_of_data technologic_data = { .init = sp_technologic_init, }; +static struct sja1000_of_data renesas_data = { + .init = sp_rzn1_init, +}; + static const struct of_device_id sp_of_table[] = { { .compatible = "nxp,sja1000", .data = NULL, }, + { .compatible = "renesas,rzn1-sja1000", .data = &renesas_data, }, { .compatible = "technologic,sja1000", .data = &technologic_data, }, { /* sentinel */ }, }; @@ -210,6 +223,7 @@ static int sp_probe(struct platform_device *pdev) struct device_node *of = pdev->dev.of_node; const struct sja1000_of_data *of_data = NULL; size_t priv_sz = 0; + struct clk *clk; pdata = dev_get_platdata(&pdev->dev); if (!pdata && !of) { @@ -234,6 +248,11 @@ static int sp_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; + + clk = devm_clk_get_optional_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "CAN clk operation failed"); } else { res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); if (!res_irq) @@ -262,6 +281,15 @@ static int sp_probe(struct platform_device *pdev) priv->reg_base = addr; if (of) { + if (clk) { + priv->can.clock.freq = clk_get_rate(clk) / 2; + if (!priv->can.clock.freq) { + err = -EINVAL; + dev_err(&pdev->dev, "Zero CAN clk rate"); + goto exit_free; + } + } + sp_populate_of(priv, of); if (of_data && of_data->init) diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.h b/drivers/net/can/usb/etas_es58x/es58x_core.h index d769bdf740b7de9f371633a23a50939cff819e1c..640fe0a1df63665cfba8faf4bd80ba8474ceb479 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.h +++ b/drivers/net/can/usb/etas_es58x/es58x_core.h @@ -222,7 +222,7 @@ union es58x_urb_cmd { u8 cmd_type; u8 cmd_id; } __packed; - u8 raw_cmd[0]; + DECLARE_FLEX_ARRAY(u8, raw_cmd); }; /** diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index baf749c8cda3f5fb68be38625aa75b0d02fb6b81..f0065d40eb2410933a47492fd5c0defe91672947 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -10,20 +10,24 @@ */ #include +#include #include #include #include #include #include +#include +#include #include +#include #include #include #include /* Device specific constants */ -#define USB_GSUSB_1_VENDOR_ID 0x1d50 -#define USB_GSUSB_1_PRODUCT_ID 0x606f +#define USB_GS_USB_1_VENDOR_ID 0x1d50 +#define USB_GS_USB_1_PRODUCT_ID 0x606f #define USB_CANDLELIGHT_VENDOR_ID 0x1209 #define USB_CANDLELIGHT_PRODUCT_ID 0x2323 @@ -34,8 +38,16 @@ #define USB_ABE_CANDEBUGGER_FD_VENDOR_ID 0x16d0 #define USB_ABE_CANDEBUGGER_FD_PRODUCT_ID 0x10b8 -#define GSUSB_ENDPOINT_IN 1 -#define GSUSB_ENDPOINT_OUT 2 +#define GS_USB_ENDPOINT_IN 1 +#define GS_USB_ENDPOINT_OUT 2 + +/* Timestamp 32 bit timer runs at 1 MHz (1 µs tick). Worker accounts + * for timer overflow (will be after ~71 minutes) + */ +#define GS_USB_TIMESTAMP_TIMER_HZ (1 * HZ_PER_MHZ) +#define GS_USB_TIMESTAMP_WORK_DELAY_SEC 1800 +static_assert(GS_USB_TIMESTAMP_WORK_DELAY_SEC < + CYCLECOUNTER_MASK(32) / GS_USB_TIMESTAMP_TIMER_HZ / 2); /* Device specific constants */ enum gs_usb_breq { @@ -52,6 +64,8 @@ enum gs_usb_breq { GS_USB_BREQ_SET_USER_ID, GS_USB_BREQ_DATA_BITTIMING, GS_USB_BREQ_BT_CONST_EXT, + GS_USB_BREQ_SET_TERMINATION, + GS_USB_BREQ_GET_TERMINATION, }; enum gs_can_mode { @@ -75,6 +89,14 @@ enum gs_can_identify_mode { GS_CAN_IDENTIFY_ON }; +enum gs_can_termination_state { + GS_CAN_TERMINATION_STATE_OFF = 0, + GS_CAN_TERMINATION_STATE_ON +}; + +#define GS_USB_TERMINATION_DISABLED CAN_TERMINATION_DISABLED +#define GS_USB_TERMINATION_ENABLED 120 + /* data types passed between host and device */ /* The firmware on the original USB2CAN by Geschwister Schneider @@ -111,6 +133,7 @@ struct gs_device_config { #define GS_CAN_MODE_FD BIT(8) /* GS_CAN_FEATURE_REQ_USB_QUIRK_LPC546XX BIT(9) */ /* GS_CAN_FEATURE_BT_CONST_EXT BIT(10) */ +/* GS_CAN_FEATURE_TERMINATION BIT(11) */ struct gs_device_mode { __le32 mode; @@ -135,6 +158,10 @@ struct gs_identify_mode { __le32 mode; } __packed; +struct gs_device_termination_state { + __le32 state; +} __packed; + #define GS_CAN_FEATURE_LISTEN_ONLY BIT(0) #define GS_CAN_FEATURE_LOOP_BACK BIT(1) #define GS_CAN_FEATURE_TRIPLE_SAMPLE BIT(2) @@ -146,7 +173,8 @@ struct gs_identify_mode { #define GS_CAN_FEATURE_FD BIT(8) #define GS_CAN_FEATURE_REQ_USB_QUIRK_LPC546XX BIT(9) #define GS_CAN_FEATURE_BT_CONST_EXT BIT(10) -#define GS_CAN_FEATURE_MASK GENMASK(10, 0) +#define GS_CAN_FEATURE_TERMINATION BIT(11) +#define GS_CAN_FEATURE_MASK GENMASK(11, 0) /* internal quirks - keep in GS_CAN_FEATURE space for now */ @@ -199,6 +227,11 @@ struct classic_can { u8 data[8]; } __packed; +struct classic_can_ts { + u8 data[8]; + __le32 timestamp_us; +} __packed; + struct classic_can_quirk { u8 data[8]; u8 quirk; @@ -208,6 +241,11 @@ struct canfd { u8 data[64]; } __packed; +struct canfd_ts { + u8 data[64]; + __le32 timestamp_us; +} __packed; + struct canfd_quirk { u8 data[64]; u8 quirk; @@ -224,8 +262,10 @@ struct gs_host_frame { union { DECLARE_FLEX_ARRAY(struct classic_can, classic_can); + DECLARE_FLEX_ARRAY(struct classic_can_ts, classic_can_ts); DECLARE_FLEX_ARRAY(struct classic_can_quirk, classic_can_quirk); DECLARE_FLEX_ARRAY(struct canfd, canfd); + DECLARE_FLEX_ARRAY(struct canfd_ts, canfd_ts); DECLARE_FLEX_ARRAY(struct canfd_quirk, canfd_quirk); }; } __packed; @@ -259,6 +299,12 @@ struct gs_can { struct can_bittiming_const bt_const, data_bt_const; unsigned int channel; /* channel number */ + /* time counter for hardware timestamps */ + struct cyclecounter cc; + struct timecounter tc; + spinlock_t tc_lock; /* spinlock to guard access tc->cycle_last */ + struct delayed_work timestamp; + u32 feature; unsigned int hf_size_tx; @@ -268,8 +314,6 @@ struct gs_can { struct usb_anchor tx_submitted; atomic_t active_tx_urbs; - void *rxbuf[GS_MAX_RX_URBS]; - dma_addr_t rxbuf_dma[GS_MAX_RX_URBS]; }; /* usb interface struct */ @@ -328,27 +372,109 @@ static struct gs_tx_context *gs_get_tx_context(struct gs_can *dev, return NULL; } -static int gs_cmd_reset(struct gs_can *gsdev) +static int gs_cmd_reset(struct gs_can *dev) +{ + struct gs_device_mode dm = { + .mode = GS_CAN_MODE_RESET, + }; + + return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_MODE, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, &dm, sizeof(dm), 1000, + GFP_KERNEL); +} + +static inline int gs_usb_get_timestamp(const struct gs_can *dev, + u32 *timestamp_p) { - struct gs_device_mode *dm; - struct usb_interface *intf = gsdev->iface; + __le32 timestamp; int rc; - dm = kzalloc(sizeof(*dm), GFP_KERNEL); - if (!dm) - return -ENOMEM; + rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_TIMESTAMP, + USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, + ×tamp, sizeof(timestamp), + USB_CTRL_GET_TIMEOUT, + GFP_KERNEL); + if (rc) + return rc; - dm->mode = GS_CAN_MODE_RESET; + *timestamp_p = le32_to_cpu(timestamp); - rc = usb_control_msg(interface_to_usbdev(intf), - usb_sndctrlpipe(interface_to_usbdev(intf), 0), - GS_USB_BREQ_MODE, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - gsdev->channel, 0, dm, sizeof(*dm), 1000); + return 0; +} - kfree(dm); +static u64 gs_usb_timestamp_read(const struct cyclecounter *cc) __must_hold(&dev->tc_lock) +{ + struct gs_can *dev = container_of(cc, struct gs_can, cc); + u32 timestamp = 0; + int err; + + lockdep_assert_held(&dev->tc_lock); + + /* drop lock for synchronous USB transfer */ + spin_unlock_bh(&dev->tc_lock); + err = gs_usb_get_timestamp(dev, ×tamp); + spin_lock_bh(&dev->tc_lock); + if (err) + netdev_err(dev->netdev, + "Error %d while reading timestamp. HW timestamps may be inaccurate.", + err); + + return timestamp; +} - return rc; +static void gs_usb_timestamp_work(struct work_struct *work) +{ + struct delayed_work *delayed_work = to_delayed_work(work); + struct gs_can *dev; + + dev = container_of(delayed_work, struct gs_can, timestamp); + spin_lock_bh(&dev->tc_lock); + timecounter_read(&dev->tc); + spin_unlock_bh(&dev->tc_lock); + + schedule_delayed_work(&dev->timestamp, + GS_USB_TIMESTAMP_WORK_DELAY_SEC * HZ); +} + +static void gs_usb_skb_set_timestamp(struct gs_can *dev, + struct sk_buff *skb, u32 timestamp) +{ + struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); + u64 ns; + + spin_lock_bh(&dev->tc_lock); + ns = timecounter_cyc2time(&dev->tc, timestamp); + spin_unlock_bh(&dev->tc_lock); + + hwtstamps->hwtstamp = ns_to_ktime(ns); +} + +static void gs_usb_timestamp_init(struct gs_can *dev) +{ + struct cyclecounter *cc = &dev->cc; + + cc->read = gs_usb_timestamp_read; + cc->mask = CYCLECOUNTER_MASK(32); + cc->shift = 32 - bits_per(NSEC_PER_SEC / GS_USB_TIMESTAMP_TIMER_HZ); + cc->mult = clocksource_hz2mult(GS_USB_TIMESTAMP_TIMER_HZ, cc->shift); + + spin_lock_init(&dev->tc_lock); + spin_lock_bh(&dev->tc_lock); + timecounter_init(&dev->tc, &dev->cc, ktime_get_real_ns()); + spin_unlock_bh(&dev->tc_lock); + + INIT_DELAYED_WORK(&dev->timestamp, gs_usb_timestamp_work); + schedule_delayed_work(&dev->timestamp, + GS_USB_TIMESTAMP_WORK_DELAY_SEC * HZ); +} + +static void gs_usb_timestamp_stop(struct gs_can *dev) +{ + cancel_delayed_work_sync(&dev->timestamp); } static void gs_update_state(struct gs_can *dev, struct can_frame *cf) @@ -376,6 +502,24 @@ static void gs_update_state(struct gs_can *dev, struct can_frame *cf) } } +static void gs_usb_set_timestamp(struct gs_can *dev, struct sk_buff *skb, + const struct gs_host_frame *hf) +{ + u32 timestamp; + + if (!(dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)) + return; + + if (hf->flags & GS_CAN_FLAG_FD) + timestamp = le32_to_cpu(hf->canfd_ts->timestamp_us); + else + timestamp = le32_to_cpu(hf->classic_can_ts->timestamp_us); + + gs_usb_skb_set_timestamp(dev, skb, timestamp); + + return; +} + static void gs_usb_receive_bulk_callback(struct urb *urb) { struct gs_usb *usbcan = urb->context; @@ -443,6 +587,8 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) gs_update_state(dev, cf); } + gs_usb_set_timestamp(dev, skb, hf); + netdev->stats.rx_packets++; netdev->stats.rx_bytes += hf->can_dlc; @@ -465,6 +611,9 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) goto resubmit_urb; } + skb = dev->can.echo_skb[hf->echo_id]; + gs_usb_set_timestamp(dev, skb, hf); + netdev->stats.tx_packets++; netdev->stats.tx_bytes += can_get_echo_skb(netdev, hf->echo_id, NULL); @@ -491,7 +640,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) resubmit_urb: usb_fill_bulk_urb(urb, usbcan->udev, - usb_rcvbulkpipe(usbcan->udev, GSUSB_ENDPOINT_IN), + usb_rcvbulkpipe(usbcan->udev, GS_USB_ENDPOINT_IN), hf, dev->parent->hf_size_rx, gs_usb_receive_bulk_callback, usbcan); @@ -511,72 +660,44 @@ static int gs_usb_set_bittiming(struct net_device *netdev) { struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.bittiming; - struct usb_interface *intf = dev->iface; - int rc; - struct gs_device_bittiming *dbt; - - dbt = kmalloc(sizeof(*dbt), GFP_KERNEL); - if (!dbt) - return -ENOMEM; - - dbt->prop_seg = cpu_to_le32(bt->prop_seg); - dbt->phase_seg1 = cpu_to_le32(bt->phase_seg1); - dbt->phase_seg2 = cpu_to_le32(bt->phase_seg2); - dbt->sjw = cpu_to_le32(bt->sjw); - dbt->brp = cpu_to_le32(bt->brp); + struct gs_device_bittiming dbt = { + .prop_seg = cpu_to_le32(bt->prop_seg), + .phase_seg1 = cpu_to_le32(bt->phase_seg1), + .phase_seg2 = cpu_to_le32(bt->phase_seg2), + .sjw = cpu_to_le32(bt->sjw), + .brp = cpu_to_le32(bt->brp), + }; /* request bit timings */ - rc = usb_control_msg(interface_to_usbdev(intf), - usb_sndctrlpipe(interface_to_usbdev(intf), 0), - GS_USB_BREQ_BITTIMING, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - dev->channel, 0, dbt, sizeof(*dbt), 1000); - - kfree(dbt); - - if (rc < 0) - dev_err(netdev->dev.parent, "Couldn't set bittimings (err=%d)", - rc); - - return (rc > 0) ? 0 : rc; + return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_BITTIMING, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, &dbt, sizeof(dbt), 1000, + GFP_KERNEL); } static int gs_usb_set_data_bittiming(struct net_device *netdev) { struct gs_can *dev = netdev_priv(netdev); struct can_bittiming *bt = &dev->can.data_bittiming; - struct usb_interface *intf = dev->iface; - struct gs_device_bittiming *dbt; + struct gs_device_bittiming dbt = { + .prop_seg = cpu_to_le32(bt->prop_seg), + .phase_seg1 = cpu_to_le32(bt->phase_seg1), + .phase_seg2 = cpu_to_le32(bt->phase_seg2), + .sjw = cpu_to_le32(bt->sjw), + .brp = cpu_to_le32(bt->brp), + }; u8 request = GS_USB_BREQ_DATA_BITTIMING; - int rc; - - dbt = kmalloc(sizeof(*dbt), GFP_KERNEL); - if (!dbt) - return -ENOMEM; - - dbt->prop_seg = cpu_to_le32(bt->prop_seg); - dbt->phase_seg1 = cpu_to_le32(bt->phase_seg1); - dbt->phase_seg2 = cpu_to_le32(bt->phase_seg2); - dbt->sjw = cpu_to_le32(bt->sjw); - dbt->brp = cpu_to_le32(bt->brp); if (dev->feature & GS_CAN_FEATURE_QUIRK_BREQ_CANTACT_PRO) request = GS_USB_BREQ_QUIRK_CANTACT_PRO_DATA_BITTIMING; - /* request bit timings */ - rc = usb_control_msg(interface_to_usbdev(intf), - usb_sndctrlpipe(interface_to_usbdev(intf), 0), - request, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - dev->channel, 0, dbt, sizeof(*dbt), 1000); - - kfree(dbt); - - if (rc < 0) - dev_err(netdev->dev.parent, - "Couldn't set data bittimings (err=%d)", rc); - - return (rc > 0) ? 0 : rc; + /* request data bit timings */ + return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + request, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, &dbt, sizeof(dbt), 1000, + GFP_KERNEL); } static void gs_usb_xmit_callback(struct urb *urb) @@ -587,9 +708,6 @@ static void gs_usb_xmit_callback(struct urb *urb) if (urb->status) netdev_info(netdev, "usb xmit fail %u\n", txc->echo_id); - - usb_free_coherent(urb->dev, urb->transfer_buffer_length, - urb->transfer_buffer, urb->transfer_dma); } static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, @@ -618,8 +736,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, if (!urb) goto nomem_urb; - hf = usb_alloc_coherent(dev->udev, dev->hf_size_tx, GFP_ATOMIC, - &urb->transfer_dma); + hf = kmalloc(dev->hf_size_tx, GFP_ATOMIC); if (!hf) { netdev_err(netdev, "No memory left for USB buffer\n"); goto nomem_hf; @@ -659,11 +776,11 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, } usb_fill_bulk_urb(urb, dev->udev, - usb_sndbulkpipe(dev->udev, GSUSB_ENDPOINT_OUT), + usb_sndbulkpipe(dev->udev, GS_USB_ENDPOINT_OUT), hf, dev->hf_size_tx, gs_usb_xmit_callback, txc); - urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + urb->transfer_flags |= URB_FREE_BUFFER; usb_anchor_urb(urb, &dev->tx_submitted); can_put_echo_skb(skb, netdev, idx, 0); @@ -678,8 +795,6 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, gs_free_tx_context(txc); usb_unanchor_urb(urb); - usb_free_coherent(dev->udev, urb->transfer_buffer_length, - urb->transfer_buffer, urb->transfer_dma); if (rc == -ENODEV) { netif_device_detach(netdev); @@ -699,8 +814,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; badidx: - usb_free_coherent(dev->udev, urb->transfer_buffer_length, - urb->transfer_buffer, urb->transfer_dma); + kfree(hf); nomem_hf: usb_free_urb(urb); @@ -715,11 +829,13 @@ static int gs_can_open(struct net_device *netdev) { struct gs_can *dev = netdev_priv(netdev); struct gs_usb *parent = dev->parent; - int rc, i; - struct gs_device_mode *dm; + struct gs_device_mode dm = { + .mode = cpu_to_le32(GS_CAN_MODE_START), + }; struct gs_host_frame *hf; u32 ctrlmode; u32 flags = 0; + int rc, i; rc = open_candev(netdev); if (rc) @@ -744,7 +860,6 @@ static int gs_can_open(struct net_device *netdev) for (i = 0; i < GS_MAX_RX_URBS; i++) { struct urb *urb; u8 *buf; - dma_addr_t buf_dma; /* alloc rx urb */ urb = usb_alloc_urb(0, GFP_KERNEL); @@ -752,10 +867,8 @@ static int gs_can_open(struct net_device *netdev) return -ENOMEM; /* alloc rx buffer */ - buf = usb_alloc_coherent(dev->udev, - dev->parent->hf_size_rx, - GFP_KERNEL, - &buf_dma); + buf = kmalloc(dev->parent->hf_size_rx, + GFP_KERNEL); if (!buf) { netdev_err(netdev, "No memory left for USB buffer\n"); @@ -763,17 +876,15 @@ static int gs_can_open(struct net_device *netdev) return -ENOMEM; } - urb->transfer_dma = buf_dma; - /* fill, anchor, and submit rx urb */ usb_fill_bulk_urb(urb, dev->udev, usb_rcvbulkpipe(dev->udev, - GSUSB_ENDPOINT_IN), + GS_USB_ENDPOINT_IN), buf, dev->parent->hf_size_rx, gs_usb_receive_bulk_callback, parent); - urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP; + urb->transfer_flags |= URB_FREE_BUFFER; usb_anchor_urb(urb, &parent->rx_submitted); @@ -786,17 +897,10 @@ static int gs_can_open(struct net_device *netdev) "usb_submit failed (err=%d)\n", rc); usb_unanchor_urb(urb); - usb_free_coherent(dev->udev, - sizeof(struct gs_host_frame), - buf, - buf_dma); usb_free_urb(urb); break; } - dev->rxbuf[i] = buf; - dev->rxbuf_dma[i] = buf_dma; - /* Drop reference, * USB core will take care of freeing it */ @@ -804,10 +908,6 @@ static int gs_can_open(struct net_device *netdev) } } - dm = kmalloc(sizeof(*dm), GFP_KERNEL); - if (!dm) - return -ENOMEM; - /* flags */ if (ctrlmode & CAN_CTRLMODE_LOOPBACK) flags |= GS_CAN_MODE_LOOP_BACK; @@ -823,25 +923,30 @@ static int gs_can_open(struct net_device *netdev) if (ctrlmode & CAN_CTRLMODE_3_SAMPLES) flags |= GS_CAN_MODE_TRIPLE_SAMPLE; + /* if hardware supports timestamps, enable it */ + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + flags |= GS_CAN_MODE_HW_TIMESTAMP; + + /* start polling timestamp */ + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + gs_usb_timestamp_init(dev); + /* finally start device */ - dm->mode = cpu_to_le32(GS_CAN_MODE_START); - dm->flags = cpu_to_le32(flags); - rc = usb_control_msg(interface_to_usbdev(dev->iface), - usb_sndctrlpipe(interface_to_usbdev(dev->iface), 0), - GS_USB_BREQ_MODE, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - dev->channel, 0, dm, sizeof(*dm), 1000); - - if (rc < 0) { + dev->can.state = CAN_STATE_ERROR_ACTIVE; + dm.flags = cpu_to_le32(flags); + rc = usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_MODE, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, &dm, sizeof(dm), 1000, + GFP_KERNEL); + if (rc) { netdev_err(netdev, "Couldn't start device (err=%d)\n", rc); - kfree(dm); + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + gs_usb_timestamp_stop(dev); + dev->can.state = CAN_STATE_STOPPED; return rc; } - kfree(dm); - - dev->can.state = CAN_STATE_ERROR_ACTIVE; - parent->active_channels++; if (!(dev->can.ctrlmode & CAN_CTRLMODE_LISTENONLY)) netif_start_queue(netdev); @@ -854,19 +959,17 @@ static int gs_can_close(struct net_device *netdev) int rc; struct gs_can *dev = netdev_priv(netdev); struct gs_usb *parent = dev->parent; - unsigned int i; netif_stop_queue(netdev); + /* stop polling timestamp */ + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + gs_usb_timestamp_stop(dev); + /* Stop polling */ parent->active_channels--; if (!parent->active_channels) { usb_kill_anchored_urbs(&parent->rx_submitted); - for (i = 0; i < GS_MAX_RX_URBS; i++) - usb_free_coherent(dev->udev, - sizeof(struct gs_host_frame), - dev->rxbuf[i], - dev->rxbuf_dma[i]); } /* Stop sending URBs */ @@ -890,52 +993,57 @@ static int gs_can_close(struct net_device *netdev) return 0; } +static int gs_can_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + const struct gs_can *dev = netdev_priv(netdev); + + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + return can_eth_ioctl_hwts(netdev, ifr, cmd); + + return -EOPNOTSUPP; +} + static const struct net_device_ops gs_usb_netdev_ops = { .ndo_open = gs_can_open, .ndo_stop = gs_can_close, .ndo_start_xmit = gs_can_start_xmit, .ndo_change_mtu = can_change_mtu, + .ndo_eth_ioctl = gs_can_eth_ioctl, }; static int gs_usb_set_identify(struct net_device *netdev, bool do_identify) { struct gs_can *dev = netdev_priv(netdev); - struct gs_identify_mode *imode; - int rc; - - imode = kmalloc(sizeof(*imode), GFP_KERNEL); - - if (!imode) - return -ENOMEM; + struct gs_identify_mode imode; if (do_identify) - imode->mode = cpu_to_le32(GS_CAN_IDENTIFY_ON); + imode.mode = cpu_to_le32(GS_CAN_IDENTIFY_ON); else - imode->mode = cpu_to_le32(GS_CAN_IDENTIFY_OFF); - - rc = usb_control_msg(interface_to_usbdev(dev->iface), - usb_sndctrlpipe(interface_to_usbdev(dev->iface), 0), - GS_USB_BREQ_IDENTIFY, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - dev->channel, 0, imode, sizeof(*imode), 100); + imode.mode = cpu_to_le32(GS_CAN_IDENTIFY_OFF); - kfree(imode); - - return (rc > 0) ? 0 : rc; + return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_IDENTIFY, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, &imode, sizeof(imode), 100, + GFP_KERNEL); } /* blink LED's for finding the this interface */ -static int gs_usb_set_phys_id(struct net_device *dev, +static int gs_usb_set_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) { + const struct gs_can *dev = netdev_priv(netdev); int rc = 0; + if (!(dev->feature & GS_CAN_FEATURE_IDENTIFY)) + return -EOPNOTSUPP; + switch (state) { case ETHTOOL_ID_ACTIVE: - rc = gs_usb_set_identify(dev, GS_CAN_IDENTIFY_ON); + rc = gs_usb_set_identify(netdev, GS_CAN_IDENTIFY_ON); break; case ETHTOOL_ID_INACTIVE: - rc = gs_usb_set_identify(dev, GS_CAN_IDENTIFY_OFF); + rc = gs_usb_set_identify(netdev, GS_CAN_IDENTIFY_OFF); break; default: break; @@ -944,9 +1052,67 @@ static int gs_usb_set_phys_id(struct net_device *dev, return rc; } +static int gs_usb_get_ts_info(struct net_device *netdev, + struct ethtool_ts_info *info) +{ + struct gs_can *dev = netdev_priv(netdev); + + /* report if device supports HW timestamps */ + if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + return can_ethtool_op_get_ts_info_hwts(netdev, info); + + return ethtool_op_get_ts_info(netdev, info); +} + static const struct ethtool_ops gs_usb_ethtool_ops = { .set_phys_id = gs_usb_set_phys_id, - .get_ts_info = ethtool_op_get_ts_info, + .get_ts_info = gs_usb_get_ts_info, +}; + +static int gs_usb_get_termination(struct net_device *netdev, u16 *term) +{ + struct gs_can *dev = netdev_priv(netdev); + struct gs_device_termination_state term_state; + int rc; + + rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_GET_TERMINATION, + USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, + &term_state, sizeof(term_state), 1000, + GFP_KERNEL); + if (rc) + return rc; + + if (term_state.state == cpu_to_le32(GS_CAN_TERMINATION_STATE_ON)) + *term = GS_USB_TERMINATION_ENABLED; + else + *term = GS_USB_TERMINATION_DISABLED; + + return 0; +} + +static int gs_usb_set_termination(struct net_device *netdev, u16 term) +{ + struct gs_can *dev = netdev_priv(netdev); + struct gs_device_termination_state term_state; + + if (term == GS_USB_TERMINATION_ENABLED) + term_state.state = cpu_to_le32(GS_CAN_TERMINATION_STATE_ON); + else + term_state.state = cpu_to_le32(GS_CAN_TERMINATION_STATE_OFF); + + return usb_control_msg_send(interface_to_usbdev(dev->iface), 0, + GS_USB_BREQ_SET_TERMINATION, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + dev->channel, 0, + &term_state, sizeof(term_state), 1000, + GFP_KERNEL); +} + +static const u16 gs_usb_termination_const[] = { + GS_USB_TERMINATION_DISABLED, + GS_USB_TERMINATION_ENABLED }; static struct gs_can *gs_make_candev(unsigned int channel, @@ -956,26 +1122,21 @@ static struct gs_can *gs_make_candev(unsigned int channel, struct gs_can *dev; struct net_device *netdev; int rc; - struct gs_device_bt_const *bt_const; - struct gs_device_bt_const_extended *bt_const_extended; + struct gs_device_bt_const_extended bt_const_extended; + struct gs_device_bt_const bt_const; u32 feature; - bt_const = kmalloc(sizeof(*bt_const), GFP_KERNEL); - if (!bt_const) - return ERR_PTR(-ENOMEM); - /* fetch bit timing constants */ - rc = usb_control_msg(interface_to_usbdev(intf), - usb_rcvctrlpipe(interface_to_usbdev(intf), 0), - GS_USB_BREQ_BT_CONST, - USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - channel, 0, bt_const, sizeof(*bt_const), 1000); + rc = usb_control_msg_recv(interface_to_usbdev(intf), 0, + GS_USB_BREQ_BT_CONST, + USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + channel, 0, &bt_const, sizeof(bt_const), 1000, + GFP_KERNEL); - if (rc < 0) { + if (rc) { dev_err(&intf->dev, - "Couldn't get bit timing const for channel (err=%d)\n", - rc); - kfree(bt_const); + "Couldn't get bit timing const for channel %d (%pe)\n", + channel, ERR_PTR(rc)); return ERR_PTR(rc); } @@ -983,7 +1144,6 @@ static struct gs_can *gs_make_candev(unsigned int channel, netdev = alloc_candev(sizeof(struct gs_can), GS_MAX_TX_URBS); if (!netdev) { dev_err(&intf->dev, "Couldn't allocate candev\n"); - kfree(bt_const); return ERR_PTR(-ENOMEM); } @@ -996,14 +1156,14 @@ static struct gs_can *gs_make_candev(unsigned int channel, /* dev setup */ strcpy(dev->bt_const.name, KBUILD_MODNAME); - dev->bt_const.tseg1_min = le32_to_cpu(bt_const->tseg1_min); - dev->bt_const.tseg1_max = le32_to_cpu(bt_const->tseg1_max); - dev->bt_const.tseg2_min = le32_to_cpu(bt_const->tseg2_min); - dev->bt_const.tseg2_max = le32_to_cpu(bt_const->tseg2_max); - dev->bt_const.sjw_max = le32_to_cpu(bt_const->sjw_max); - dev->bt_const.brp_min = le32_to_cpu(bt_const->brp_min); - dev->bt_const.brp_max = le32_to_cpu(bt_const->brp_max); - dev->bt_const.brp_inc = le32_to_cpu(bt_const->brp_inc); + dev->bt_const.tseg1_min = le32_to_cpu(bt_const.tseg1_min); + dev->bt_const.tseg1_max = le32_to_cpu(bt_const.tseg1_max); + dev->bt_const.tseg2_min = le32_to_cpu(bt_const.tseg2_min); + dev->bt_const.tseg2_max = le32_to_cpu(bt_const.tseg2_max); + dev->bt_const.sjw_max = le32_to_cpu(bt_const.sjw_max); + dev->bt_const.brp_min = le32_to_cpu(bt_const.brp_min); + dev->bt_const.brp_max = le32_to_cpu(bt_const.brp_max); + dev->bt_const.brp_inc = le32_to_cpu(bt_const.brp_inc); dev->udev = interface_to_usbdev(intf); dev->iface = intf; @@ -1020,13 +1180,13 @@ static struct gs_can *gs_make_candev(unsigned int channel, /* can setup */ dev->can.state = CAN_STATE_STOPPED; - dev->can.clock.freq = le32_to_cpu(bt_const->fclk_can); + dev->can.clock.freq = le32_to_cpu(bt_const.fclk_can); dev->can.bittiming_const = &dev->bt_const; dev->can.do_set_bittiming = gs_usb_set_bittiming; dev->can.ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC; - feature = le32_to_cpu(bt_const->feature); + feature = le32_to_cpu(bt_const.feature); dev->feature = FIELD_GET(GS_CAN_FEATURE_MASK, feature); if (feature & GS_CAN_FEATURE_LISTEN_ONLY) dev->can.ctrlmode_supported |= CAN_CTRLMODE_LISTENONLY; @@ -1049,6 +1209,21 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->can.do_set_data_bittiming = gs_usb_set_data_bittiming; } + if (feature & GS_CAN_FEATURE_TERMINATION) { + rc = gs_usb_get_termination(netdev, &dev->can.termination); + if (rc) { + dev->feature &= ~GS_CAN_FEATURE_TERMINATION; + + dev_info(&intf->dev, + "Disabling termination support for channel %d (%pe)\n", + channel, ERR_PTR(rc)); + } else { + dev->can.termination_const = gs_usb_termination_const; + dev->can.termination_const_cnt = ARRAY_SIZE(gs_usb_termination_const); + dev->can.do_set_termination = gs_usb_set_termination; + } + } + /* The CANtact Pro from LinkLayer Labs is based on the * LPC54616 µC, which is affected by the NXP LPC USB transfer * erratum. However, the current firmware (version 2) doesn't @@ -1063,8 +1238,8 @@ static struct gs_can *gs_make_candev(unsigned int channel, * GS_CAN_FEATURE_QUIRK_BREQ_CANTACT_PRO to workaround this * issue. */ - if (dev->udev->descriptor.idVendor == cpu_to_le16(USB_GSUSB_1_VENDOR_ID) && - dev->udev->descriptor.idProduct == cpu_to_le16(USB_GSUSB_1_PRODUCT_ID) && + if (dev->udev->descriptor.idVendor == cpu_to_le16(USB_GS_USB_1_VENDOR_ID) && + dev->udev->descriptor.idProduct == cpu_to_le16(USB_GS_USB_1_PRODUCT_ID) && dev->udev->manufacturer && dev->udev->product && !strcmp(dev->udev->manufacturer, "LinkLayer Labs") && !strcmp(dev->udev->product, "CANtact Pro") && @@ -1072,61 +1247,57 @@ static struct gs_can *gs_make_candev(unsigned int channel, dev->feature |= GS_CAN_FEATURE_REQ_USB_QUIRK_LPC546XX | GS_CAN_FEATURE_QUIRK_BREQ_CANTACT_PRO; - if (le32_to_cpu(dconf->sw_version) > 1) - if (feature & GS_CAN_FEATURE_IDENTIFY) - netdev->ethtool_ops = &gs_usb_ethtool_ops; - - kfree(bt_const); + /* GS_CAN_FEATURE_IDENTIFY is only supported for sw_version > 1 */ + if (!(le32_to_cpu(dconf->sw_version) > 1 && + feature & GS_CAN_FEATURE_IDENTIFY)) + dev->feature &= ~GS_CAN_FEATURE_IDENTIFY; /* fetch extended bit timing constants if device has feature * GS_CAN_FEATURE_FD and GS_CAN_FEATURE_BT_CONST_EXT */ if (feature & GS_CAN_FEATURE_FD && feature & GS_CAN_FEATURE_BT_CONST_EXT) { - bt_const_extended = kmalloc(sizeof(*bt_const_extended), GFP_KERNEL); - if (!bt_const_extended) - return ERR_PTR(-ENOMEM); - - rc = usb_control_msg(interface_to_usbdev(intf), - usb_rcvctrlpipe(interface_to_usbdev(intf), 0), - GS_USB_BREQ_BT_CONST_EXT, - USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - channel, 0, bt_const_extended, - sizeof(*bt_const_extended), - 1000); - if (rc < 0) { + rc = usb_control_msg_recv(interface_to_usbdev(intf), 0, + GS_USB_BREQ_BT_CONST_EXT, + USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + channel, 0, &bt_const_extended, + sizeof(bt_const_extended), + 1000, GFP_KERNEL); + if (rc) { dev_err(&intf->dev, - "Couldn't get extended bit timing const for channel (err=%d)\n", - rc); - kfree(bt_const_extended); - return ERR_PTR(rc); + "Couldn't get extended bit timing const for channel %d (%pe)\n", + channel, ERR_PTR(rc)); + goto out_free_candev; } strcpy(dev->data_bt_const.name, KBUILD_MODNAME); - dev->data_bt_const.tseg1_min = le32_to_cpu(bt_const_extended->dtseg1_min); - dev->data_bt_const.tseg1_max = le32_to_cpu(bt_const_extended->dtseg1_max); - dev->data_bt_const.tseg2_min = le32_to_cpu(bt_const_extended->dtseg2_min); - dev->data_bt_const.tseg2_max = le32_to_cpu(bt_const_extended->dtseg2_max); - dev->data_bt_const.sjw_max = le32_to_cpu(bt_const_extended->dsjw_max); - dev->data_bt_const.brp_min = le32_to_cpu(bt_const_extended->dbrp_min); - dev->data_bt_const.brp_max = le32_to_cpu(bt_const_extended->dbrp_max); - dev->data_bt_const.brp_inc = le32_to_cpu(bt_const_extended->dbrp_inc); + dev->data_bt_const.tseg1_min = le32_to_cpu(bt_const_extended.dtseg1_min); + dev->data_bt_const.tseg1_max = le32_to_cpu(bt_const_extended.dtseg1_max); + dev->data_bt_const.tseg2_min = le32_to_cpu(bt_const_extended.dtseg2_min); + dev->data_bt_const.tseg2_max = le32_to_cpu(bt_const_extended.dtseg2_max); + dev->data_bt_const.sjw_max = le32_to_cpu(bt_const_extended.dsjw_max); + dev->data_bt_const.brp_min = le32_to_cpu(bt_const_extended.dbrp_min); + dev->data_bt_const.brp_max = le32_to_cpu(bt_const_extended.dbrp_max); + dev->data_bt_const.brp_inc = le32_to_cpu(bt_const_extended.dbrp_inc); dev->can.data_bittiming_const = &dev->data_bt_const; - - kfree(bt_const_extended); } SET_NETDEV_DEV(netdev, &intf->dev); rc = register_candev(dev->netdev); if (rc) { - free_candev(dev->netdev); - dev_err(&intf->dev, "Couldn't register candev (err=%d)\n", rc); - return ERR_PTR(rc); + dev_err(&intf->dev, + "Couldn't register candev for channel %d (%pe)\n", + channel, ERR_PTR(rc)); + goto out_free_candev; } return dev; + + out_free_candev: + free_candev(dev->netdev); + return ERR_PTR(rc); } static void gs_destroy_candev(struct gs_can *dev) @@ -1142,76 +1313,61 @@ static int gs_usb_probe(struct usb_interface *intf, struct usb_device *udev = interface_to_usbdev(intf); struct gs_host_frame *hf; struct gs_usb *dev; - int rc = -ENOMEM; + struct gs_host_config hconf = { + .byte_order = cpu_to_le32(0x0000beef), + }; + struct gs_device_config dconf; unsigned int icount, i; - struct gs_host_config *hconf; - struct gs_device_config *dconf; - - hconf = kmalloc(sizeof(*hconf), GFP_KERNEL); - if (!hconf) - return -ENOMEM; - - hconf->byte_order = cpu_to_le32(0x0000beef); + int rc; /* send host config */ - rc = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), - GS_USB_BREQ_HOST_FORMAT, - USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - 1, intf->cur_altsetting->desc.bInterfaceNumber, - hconf, sizeof(*hconf), 1000); - - kfree(hconf); - - if (rc < 0) { + rc = usb_control_msg_send(udev, 0, + GS_USB_BREQ_HOST_FORMAT, + USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + 1, intf->cur_altsetting->desc.bInterfaceNumber, + &hconf, sizeof(hconf), 1000, + GFP_KERNEL); + if (rc) { dev_err(&intf->dev, "Couldn't send data format (err=%d)\n", rc); return rc; } - dconf = kmalloc(sizeof(*dconf), GFP_KERNEL); - if (!dconf) - return -ENOMEM; - /* read device config */ - rc = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), - GS_USB_BREQ_DEVICE_CONFIG, - USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, - 1, intf->cur_altsetting->desc.bInterfaceNumber, - dconf, sizeof(*dconf), 1000); - if (rc < 0) { + rc = usb_control_msg_recv(udev, 0, + GS_USB_BREQ_DEVICE_CONFIG, + USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE, + 1, intf->cur_altsetting->desc.bInterfaceNumber, + &dconf, sizeof(dconf), 1000, + GFP_KERNEL); + if (rc) { dev_err(&intf->dev, "Couldn't get device config: (err=%d)\n", rc); - kfree(dconf); return rc; } - icount = dconf->icount + 1; + icount = dconf.icount + 1; dev_info(&intf->dev, "Configuring for %u interfaces\n", icount); if (icount > GS_MAX_INTF) { dev_err(&intf->dev, "Driver cannot handle more that %u CAN interfaces\n", GS_MAX_INTF); - kfree(dconf); return -EINVAL; } dev = kzalloc(sizeof(*dev), GFP_KERNEL); - if (!dev) { - kfree(dconf); + if (!dev) return -ENOMEM; - } init_usb_anchor(&dev->rx_submitted); - /* default to classic CAN, switch to CAN-FD if at least one of - * our channels support CAN-FD. - */ - dev->hf_size_rx = struct_size(hf, classic_can, 1); usb_set_intfdata(intf, dev); dev->udev = udev; for (i = 0; i < icount; i++) { - dev->canch[i] = gs_make_candev(i, intf, dconf); + unsigned int hf_size_rx = 0; + + dev->canch[i] = gs_make_candev(i, intf, &dconf); if (IS_ERR_OR_NULL(dev->canch[i])) { /* save error code to return later */ rc = PTR_ERR(dev->canch[i]); @@ -1222,18 +1378,28 @@ static int gs_usb_probe(struct usb_interface *intf, gs_destroy_candev(dev->canch[i]); usb_kill_anchored_urbs(&dev->rx_submitted); - kfree(dconf); kfree(dev); return rc; } dev->canch[i]->parent = dev; - if (dev->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD) - dev->hf_size_rx = struct_size(hf, canfd, 1); + /* set RX packet size based on FD and if hardware + * timestamps are supported. + */ + if (dev->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD) { + if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + hf_size_rx = struct_size(hf, canfd_ts, 1); + else + hf_size_rx = struct_size(hf, canfd, 1); + } else { + if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP) + hf_size_rx = struct_size(hf, classic_can_ts, 1); + else + hf_size_rx = struct_size(hf, classic_can, 1); + } + dev->hf_size_rx = max(dev->hf_size_rx, hf_size_rx); } - kfree(dconf); - return 0; } @@ -1258,8 +1424,8 @@ static void gs_usb_disconnect(struct usb_interface *intf) } static const struct usb_device_id gs_usb_table[] = { - { USB_DEVICE_INTERFACE_NUMBER(USB_GSUSB_1_VENDOR_ID, - USB_GSUSB_1_PRODUCT_ID, 0) }, + { USB_DEVICE_INTERFACE_NUMBER(USB_GS_USB_1_VENDOR_ID, + USB_GS_USB_1_PRODUCT_ID, 0) }, { USB_DEVICE_INTERFACE_NUMBER(USB_CANDLELIGHT_VENDOR_ID, USB_CANDLELIGHT_PRODUCT_ID, 0) }, { USB_DEVICE_INTERFACE_NUMBER(USB_CES_CANEXT_FD_VENDOR_ID, diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb.h b/drivers/net/can/usb/kvaser_usb/kvaser_usb.h index 841da29cef939b363be58aaffa36458e9857cb35..f6c0938027ecec65734187a4c90e15a6f7362112 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb.h +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb.h @@ -178,6 +178,8 @@ struct kvaser_usb_dev_cfg { extern const struct kvaser_usb_dev_ops kvaser_usb_hydra_dev_ops; extern const struct kvaser_usb_dev_ops kvaser_usb_leaf_dev_ops; +void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv); + int kvaser_usb_recv_cmd(const struct kvaser_usb *dev, void *cmd, int len, int *actual_len); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index 824cab80aa02fc979a20ff98ca9b24f264e1e9b2..e91648ed73862fe2ac114b391c9c5e233d79c03e 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -477,7 +477,7 @@ static void kvaser_usb_reset_tx_urb_contexts(struct kvaser_usb_net_priv *priv) /* This method might sleep. Do not call it in the atomic context * of URB completions. */ -static void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv) +void kvaser_usb_unlink_tx_urbs(struct kvaser_usb_net_priv *priv) { usb_kill_anchored_urbs(&priv->tx_submitted); kvaser_usb_reset_tx_urb_contexts(priv); @@ -729,6 +729,7 @@ static int kvaser_usb_init_one(struct kvaser_usb *dev, int channel) init_usb_anchor(&priv->tx_submitted); init_completion(&priv->start_comp); init_completion(&priv->stop_comp); + init_completion(&priv->flush_comp); priv->can.ctrlmode_supported = 0; priv->dev = dev; diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index dd65c101bfb8ef19a3b4e144fbc40fda94d3501d..7b52fda73d8277c39a47eb29e806f8f696e99364 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -534,7 +534,7 @@ static int kvaser_usb_hydra_send_simple_cmd(struct kvaser_usb *dev, struct kvaser_cmd *cmd; int err; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -573,7 +573,7 @@ kvaser_usb_hydra_send_simple_cmd_async(struct kvaser_usb_net_priv *priv, struct kvaser_usb *dev = priv->dev; int err; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_ATOMIC); + cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC); if (!cmd) return -ENOMEM; @@ -694,7 +694,7 @@ static int kvaser_usb_hydra_map_channel(struct kvaser_usb *dev, u16 transid, struct kvaser_cmd *cmd; int err; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -735,7 +735,7 @@ static int kvaser_usb_hydra_get_single_capability(struct kvaser_usb *dev, int err; int i; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -1394,7 +1394,7 @@ kvaser_usb_hydra_frame_to_cmd_ext(const struct kvaser_usb_net_priv *priv, u32 kcan_id; u32 kcan_header; - cmd = kcalloc(1, sizeof(struct kvaser_cmd_ext), GFP_ATOMIC); + cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC); if (!cmd) return NULL; @@ -1468,7 +1468,7 @@ kvaser_usb_hydra_frame_to_cmd_std(const struct kvaser_usb_net_priv *priv, u32 flags; u32 id; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_ATOMIC); + cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC); if (!cmd) return NULL; @@ -1533,7 +1533,7 @@ static int kvaser_usb_hydra_set_bittiming(struct net_device *netdev) int sjw = bt->sjw; int err; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -1567,7 +1567,7 @@ static int kvaser_usb_hydra_set_data_bittiming(struct net_device *netdev) int sjw = dbt->sjw; int err; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -1711,7 +1711,7 @@ static int kvaser_usb_hydra_get_software_details(struct kvaser_usb *dev) u32 flags; struct kvaser_usb_dev_card_data *card_data = &dev->card_data; - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -1851,7 +1851,7 @@ static int kvaser_usb_hydra_set_opt_mode(const struct kvaser_usb_net_priv *priv) return -EINVAL; } - cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL); + cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; @@ -1916,7 +1916,7 @@ static int kvaser_usb_hydra_flush_queue(struct kvaser_usb_net_priv *priv) { int err; - init_completion(&priv->flush_comp); + reinit_completion(&priv->flush_comp); err = kvaser_usb_hydra_send_simple_cmd(priv->dev, CMD_FLUSH_QUEUE, priv->channel); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index 07f687f29b3414d76018ff525c4a596bffbefabf..50f2ac8319ff88fc3caeca26706ce1eabb2a6601 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -310,6 +310,38 @@ struct kvaser_cmd { } u; } __packed; +#define CMD_SIZE_ANY 0xff +#define kvaser_fsize(field) sizeof_field(struct kvaser_cmd, field) + +static const u8 kvaser_usb_leaf_cmd_sizes_leaf[] = { + [CMD_START_CHIP_REPLY] = kvaser_fsize(u.simple), + [CMD_STOP_CHIP_REPLY] = kvaser_fsize(u.simple), + [CMD_GET_CARD_INFO_REPLY] = kvaser_fsize(u.cardinfo), + [CMD_TX_ACKNOWLEDGE] = kvaser_fsize(u.tx_acknowledge_header), + [CMD_GET_SOFTWARE_INFO_REPLY] = kvaser_fsize(u.leaf.softinfo), + [CMD_RX_STD_MESSAGE] = kvaser_fsize(u.leaf.rx_can), + [CMD_RX_EXT_MESSAGE] = kvaser_fsize(u.leaf.rx_can), + [CMD_LEAF_LOG_MESSAGE] = kvaser_fsize(u.leaf.log_message), + [CMD_CHIP_STATE_EVENT] = kvaser_fsize(u.leaf.chip_state_event), + [CMD_CAN_ERROR_EVENT] = kvaser_fsize(u.leaf.error_event), + /* ignored events: */ + [CMD_FLUSH_QUEUE_REPLY] = CMD_SIZE_ANY, +}; + +static const u8 kvaser_usb_leaf_cmd_sizes_usbcan[] = { + [CMD_START_CHIP_REPLY] = kvaser_fsize(u.simple), + [CMD_STOP_CHIP_REPLY] = kvaser_fsize(u.simple), + [CMD_GET_CARD_INFO_REPLY] = kvaser_fsize(u.cardinfo), + [CMD_TX_ACKNOWLEDGE] = kvaser_fsize(u.tx_acknowledge_header), + [CMD_GET_SOFTWARE_INFO_REPLY] = kvaser_fsize(u.usbcan.softinfo), + [CMD_RX_STD_MESSAGE] = kvaser_fsize(u.usbcan.rx_can), + [CMD_RX_EXT_MESSAGE] = kvaser_fsize(u.usbcan.rx_can), + [CMD_CHIP_STATE_EVENT] = kvaser_fsize(u.usbcan.chip_state_event), + [CMD_CAN_ERROR_EVENT] = kvaser_fsize(u.usbcan.error_event), + /* ignored events: */ + [CMD_USBCAN_CLOCK_OVERFLOW_EVENT] = CMD_SIZE_ANY, +}; + /* Summary of a kvaser error event, for a unified Leaf/Usbcan error * handling. Some discrepancies between the two families exist: * @@ -397,6 +429,43 @@ static const struct kvaser_usb_dev_cfg kvaser_usb_leaf_imx_dev_cfg_32mhz = { .bittiming_const = &kvaser_usb_flexc_bittiming_const, }; +static int kvaser_usb_leaf_verify_size(const struct kvaser_usb *dev, + const struct kvaser_cmd *cmd) +{ + /* buffer size >= cmd->len ensured by caller */ + u8 min_size = 0; + + switch (dev->driver_info->family) { + case KVASER_LEAF: + if (cmd->id < ARRAY_SIZE(kvaser_usb_leaf_cmd_sizes_leaf)) + min_size = kvaser_usb_leaf_cmd_sizes_leaf[cmd->id]; + break; + case KVASER_USBCAN: + if (cmd->id < ARRAY_SIZE(kvaser_usb_leaf_cmd_sizes_usbcan)) + min_size = kvaser_usb_leaf_cmd_sizes_usbcan[cmd->id]; + break; + } + + if (min_size == CMD_SIZE_ANY) + return 0; + + if (min_size) { + min_size += CMD_HEADER_LEN; + if (cmd->len >= min_size) + return 0; + + dev_err_ratelimited(&dev->intf->dev, + "Received command %u too short (size %u, needed %u)", + cmd->id, cmd->len, min_size); + return -EIO; + } + + dev_warn_ratelimited(&dev->intf->dev, + "Unhandled command (%d, size %d)\n", + cmd->id, cmd->len); + return -EINVAL; +} + static void * kvaser_usb_leaf_frame_to_cmd(const struct kvaser_usb_net_priv *priv, const struct sk_buff *skb, int *cmd_len, @@ -502,6 +571,9 @@ static int kvaser_usb_leaf_wait_cmd(const struct kvaser_usb *dev, u8 id, end: kfree(buf); + if (err == 0) + err = kvaser_usb_leaf_verify_size(dev, cmd); + return err; } @@ -1133,6 +1205,9 @@ static void kvaser_usb_leaf_stop_chip_reply(const struct kvaser_usb *dev, static void kvaser_usb_leaf_handle_command(const struct kvaser_usb *dev, const struct kvaser_cmd *cmd) { + if (kvaser_usb_leaf_verify_size(dev, cmd) < 0) + return; + switch (cmd->id) { case CMD_START_CHIP_REPLY: kvaser_usb_leaf_start_chip_reply(dev, cmd); @@ -1351,9 +1426,13 @@ static int kvaser_usb_leaf_set_mode(struct net_device *netdev, switch (mode) { case CAN_MODE_START: + kvaser_usb_unlink_tx_urbs(priv); + err = kvaser_usb_leaf_simple_cmd_async(priv, CMD_START_CHIP); if (err) return err; + + priv->can.state = CAN_STATE_ERROR_ACTIVE; break; default: return -EOPNOTSUPP; diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 8c9d53f6e24c383a1133feea09de6a4a8b792222..225697d70a9aa3a7c9bb21b1617eecc9228d14b1 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -962,7 +962,7 @@ static void peak_usb_disconnect(struct usb_interface *intf) dev_prev_siblings = dev->prev_siblings; dev->state &= ~PCAN_USB_STATE_CONNECTED; - strlcpy(name, netdev->name, IFNAMSIZ); + strscpy(name, netdev->name, IFNAMSIZ); unregister_netdev(netdev); diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c index 36b6310a2e5b02e3c997f87e1b23180c288e2d9b..285635c234432ba92fb5d7233426c35ab799942e 100644 --- a/drivers/net/can/vcan.c +++ b/drivers/net/can/vcan.c @@ -71,11 +71,10 @@ MODULE_PARM_DESC(echo, "Echo sent frames (for testing). Default: 0 (Off)"); static void vcan_rx(struct sk_buff *skb, struct net_device *dev) { - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct net_device_stats *stats = &dev->stats; stats->rx_packets++; - stats->rx_bytes += cfd->len; + stats->rx_bytes += can_skb_get_data_len(skb); skb->pkt_type = PACKET_BROADCAST; skb->dev = dev; @@ -86,14 +85,14 @@ static void vcan_rx(struct sk_buff *skb, struct net_device *dev) static netdev_tx_t vcan_tx(struct sk_buff *skb, struct net_device *dev) { - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct net_device_stats *stats = &dev->stats; - int loop, len; + unsigned int len; + int loop; if (can_dropped_invalid_skb(dev, skb)) return NETDEV_TX_OK; - len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len; + len = can_skb_get_data_len(skb); stats->tx_packets++; stats->tx_bytes += len; @@ -137,7 +136,8 @@ static int vcan_change_mtu(struct net_device *dev, int new_mtu) if (dev->flags & IFF_UP) return -EBUSY; - if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU) + if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU && + !can_is_canxl_dev_mtu(new_mtu)) return -EINVAL; dev->mtu = new_mtu; diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c index cffd107d8b28293768588e02e9fbbb79c43f28dc..26a472d2ea583cbd5b5a40823d82847276627ea8 100644 --- a/drivers/net/can/vxcan.c +++ b/drivers/net/can/vxcan.c @@ -38,10 +38,9 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev) { struct vxcan_priv *priv = netdev_priv(dev); struct net_device *peer; - struct canfd_frame *cfd = (struct canfd_frame *)oskb->data; struct net_device_stats *peerstats, *srcstats = &dev->stats; struct sk_buff *skb; - u8 len; + unsigned int len; if (can_dropped_invalid_skb(dev, oskb)) return NETDEV_TX_OK; @@ -70,7 +69,7 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev) skb->dev = peer; skb->ip_summed = CHECKSUM_UNNECESSARY; - len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len; + len = can_skb_get_data_len(skb); if (netif_rx(skb) == NET_RX_SUCCESS) { srcstats->tx_packets++; srcstats->tx_bytes += len; @@ -132,7 +131,8 @@ static int vxcan_change_mtu(struct net_device *dev, int new_mtu) if (dev->flags & IFF_UP) return -EBUSY; - if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU) + if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU && + !can_is_canxl_dev_mtu(new_mtu)) return -EINVAL; dev->mtu = new_mtu; diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig index d8ae0e8af2a03c7279579bba0c7d89ba83dd496c..07507b4820d73075bd63c7e6c2fc929ecb984dc2 100644 --- a/drivers/net/dsa/Kconfig +++ b/drivers/net/dsa/Kconfig @@ -76,7 +76,7 @@ config NET_DSA_SMSC_LAN9303 select NET_DSA_TAG_LAN9303 select REGMAP help - This enables support for the SMSC/Microchip LAN9303 3 port ethernet + This enables support for the Microchip LAN9303/LAN9354 3 port ethernet switch chips. config NET_DSA_SMSC_LAN9303_I2C @@ -90,11 +90,11 @@ config NET_DSA_SMSC_LAN9303_I2C for I2C managed mode. config NET_DSA_SMSC_LAN9303_MDIO - tristate "SMSC/Microchip LAN9303 3-ports 10/100 ethernet switch in MDIO managed mode" + tristate "Microchip LAN9303/LAN9354 3-ports 10/100 ethernet switch in MDIO managed mode" select NET_DSA_SMSC_LAN9303 depends on VLAN_8021Q || VLAN_8021Q=n help - Enable access functions if the SMSC/Microchip LAN9303 is configured + Enable access functions if the Microchip LAN9303/LAN9354 is configured for MDIO managed mode. config NET_DSA_VITESSE_VSC73XX diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 48cf344750ffdfce53704de46927bea24d10fd7a..59cdfc51ce06a0de3fb644eae5eebe4fe2dd9b21 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -972,7 +972,7 @@ void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset, if (stringset == ETH_SS_STATS) { for (i = 0; i < mib_size; i++) - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, mibs[i].name, ETH_GSTRING_LEN); } else if (stringset == ETH_SS_PHY_STATS) { phydev = b53_get_phy_device(ds, port); diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c index a7aeb3c132c987297239c178fa0f7e69ea701df2..6ddc03b58b28b6903dc83972cc3311742bf90c1b 100644 --- a/drivers/net/dsa/b53/b53_mdio.c +++ b/drivers/net/dsa/b53/b53_mdio.c @@ -356,8 +356,6 @@ static void b53_mdio_remove(struct mdio_device *mdiodev) return; b53_switch_remove(dev); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void b53_mdio_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c index ae4c79d39bc047e8274cc849078a960df1545527..e968322dfbf0b262e23cd55fa482d5eb268fd2dc 100644 --- a/drivers/net/dsa/b53/b53_mmap.c +++ b/drivers/net/dsa/b53/b53_mmap.c @@ -316,8 +316,6 @@ static int b53_mmap_remove(struct platform_device *pdev) if (dev) b53_switch_remove(dev); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/b53/b53_srab.c b/drivers/net/dsa/b53/b53_srab.c index da0b889880f6af57dd4ed4189e06efb68ee003b5..bcb44034404d390c05b566e090d0adf52e88c3fa 100644 --- a/drivers/net/dsa/b53/b53_srab.c +++ b/drivers/net/dsa/b53/b53_srab.c @@ -667,8 +667,6 @@ static int b53_srab_remove(struct platform_device *pdev) b53_srab_intr_set(dev->priv, false); b53_switch_remove(dev); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index be0edfa093d047fb6f0a4b7b3caeef2ae75b42f4..cde253d27bd0845e079393896d09546d8982c88a 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -94,6 +94,24 @@ static u16 bcm_sf2_reg_led_base(struct bcm_sf2_priv *priv, int port) return REG_SWITCH_STATUS; } +static u32 bcm_sf2_port_override_offset(struct bcm_sf2_priv *priv, int port) +{ + switch (priv->type) { + case BCM4908_DEVICE_ID: + case BCM7445_DEVICE_ID: + return port == 8 ? CORE_STS_OVERRIDE_IMP : + CORE_STS_OVERRIDE_GMIIP_PORT(port); + case BCM7278_DEVICE_ID: + return port == 8 ? CORE_STS_OVERRIDE_IMP2 : + CORE_STS_OVERRIDE_GMIIP2_PORT(port); + default: + WARN_ONCE(1, "Unsupported device: %d\n", priv->type); + } + + /* RO fallback register */ + return REG_SWITCH_STATUS; +} + /* Return the number of active ports, not counting the IMP (CPU) port */ static unsigned int bcm_sf2_num_active_ports(struct dsa_switch *ds) { @@ -141,7 +159,7 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port) { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); unsigned int i; - u32 reg, offset; + u32 reg; /* Enable the port memories */ reg = core_readl(priv, CORE_MEM_PSM_VDD_CTRL); @@ -167,21 +185,6 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port) b53_brcm_hdr_setup(ds, port); if (port == 8) { - if (priv->type == BCM4908_DEVICE_ID || - priv->type == BCM7445_DEVICE_ID) - offset = CORE_STS_OVERRIDE_IMP; - else - offset = CORE_STS_OVERRIDE_IMP2; - - /* Force link status for IMP port */ - reg = core_readl(priv, offset); - reg |= (MII_SW_OR | LINK_STS); - if (priv->type == BCM4908_DEVICE_ID) - reg |= GMII_SPEED_UP_2G; - else - reg &= ~GMII_SPEED_UP_2G; - core_writel(priv, reg, offset); - /* Enable Broadcast, Multicast, Unicast forwarding to IMP port */ reg = core_readl(priv, CORE_IMP_CTL); reg |= (RX_BCST_EN | RX_MCST_EN | RX_UCST_EN); @@ -812,17 +815,10 @@ static void bcm_sf2_sw_mac_link_down(struct dsa_switch *ds, int port, if (priv->wol_ports_mask & BIT(port)) return; - if (port != core_readl(priv, CORE_IMP0_PRT_ID)) { - if (priv->type == BCM4908_DEVICE_ID || - priv->type == BCM7445_DEVICE_ID) - offset = CORE_STS_OVERRIDE_GMIIP_PORT(port); - else - offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port); - - reg = core_readl(priv, offset); - reg &= ~LINK_STS; - core_writel(priv, reg, offset); - } + offset = bcm_sf2_port_override_offset(priv, port); + reg = core_readl(priv, offset); + reg &= ~LINK_STS; + core_writel(priv, reg, offset); bcm_sf2_sw_mac_link_set(ds, port, interface, false); } @@ -836,56 +832,56 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port, { struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); struct ethtool_eee *p = &priv->dev->ports[port].eee; + u32 reg_rgmii_ctrl = 0; + u32 reg, offset; bcm_sf2_sw_mac_link_set(ds, port, interface, true); - if (port != core_readl(priv, CORE_IMP0_PRT_ID)) { - u32 reg_rgmii_ctrl = 0; - u32 reg, offset; + offset = bcm_sf2_port_override_offset(priv, port); - if (priv->type == BCM4908_DEVICE_ID || - priv->type == BCM7445_DEVICE_ID) - offset = CORE_STS_OVERRIDE_GMIIP_PORT(port); - else - offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port); - - if (interface == PHY_INTERFACE_MODE_RGMII || - interface == PHY_INTERFACE_MODE_RGMII_TXID || - interface == PHY_INTERFACE_MODE_MII || - interface == PHY_INTERFACE_MODE_REVMII) { - reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port); - reg = reg_readl(priv, reg_rgmii_ctrl); - reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN); - - if (tx_pause) - reg |= TX_PAUSE_EN; - if (rx_pause) - reg |= RX_PAUSE_EN; - - reg_writel(priv, reg, reg_rgmii_ctrl); - } - - reg = SW_OVERRIDE | LINK_STS; - switch (speed) { - case SPEED_1000: - reg |= SPDSTS_1000 << SPEED_SHIFT; - break; - case SPEED_100: - reg |= SPDSTS_100 << SPEED_SHIFT; - break; - } - - if (duplex == DUPLEX_FULL) - reg |= DUPLX_MODE; + if (phy_interface_mode_is_rgmii(interface) || + interface == PHY_INTERFACE_MODE_MII || + interface == PHY_INTERFACE_MODE_REVMII) { + reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port); + reg = reg_readl(priv, reg_rgmii_ctrl); + reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN); if (tx_pause) - reg |= TXFLOW_CNTL; + reg |= TX_PAUSE_EN; if (rx_pause) - reg |= RXFLOW_CNTL; + reg |= RX_PAUSE_EN; - core_writel(priv, reg, offset); + reg_writel(priv, reg, reg_rgmii_ctrl); } + reg = LINK_STS; + if (port == 8) { + if (priv->type == BCM4908_DEVICE_ID) + reg |= GMII_SPEED_UP_2G; + reg |= MII_SW_OR; + } else { + reg |= SW_OVERRIDE; + } + + switch (speed) { + case SPEED_1000: + reg |= SPDSTS_1000 << SPEED_SHIFT; + break; + case SPEED_100: + reg |= SPDSTS_100 << SPEED_SHIFT; + break; + } + + if (duplex == DUPLEX_FULL) + reg |= DUPLX_MODE; + + if (tx_pause) + reg |= TXFLOW_CNTL; + if (rx_pause) + reg |= RXFLOW_CNTL; + + core_writel(priv, reg, offset); + if (mode == MLO_AN_PHY && phydev) p->eee_enabled = b53_eee_init(ds, port, phydev); } @@ -987,7 +983,7 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds) static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port, struct ethtool_wolinfo *wol) { - struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master; + struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port)); struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); struct ethtool_wolinfo pwol = { }; @@ -1011,7 +1007,7 @@ static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port, static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port, struct ethtool_wolinfo *wol) { - struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master; + struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port)); struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); s8 cpu_port = dsa_to_port(ds, port)->cpu_dp->index; struct ethtool_wolinfo pwol = { }; @@ -1555,8 +1551,6 @@ static int bcm_sf2_sw_remove(struct platform_device *pdev) if (priv->type == BCM7278_DEVICE_ID) reset_control_assert(priv->rcdev); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c index edbe5e7f1cb6b134c9c7b69dc3fdedc1c7dc92b4..c4010b7bf08997d99b374813f7585520e034ee40 100644 --- a/drivers/net/dsa/bcm_sf2_cfp.c +++ b/drivers/net/dsa/bcm_sf2_cfp.c @@ -1102,7 +1102,7 @@ static int bcm_sf2_cfp_rule_get_all(struct bcm_sf2_priv *priv, int bcm_sf2_get_rxnfc(struct dsa_switch *ds, int port, struct ethtool_rxnfc *nfc, u32 *rule_locs) { - struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master; + struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port)); struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); int ret = 0; @@ -1145,7 +1145,7 @@ int bcm_sf2_get_rxnfc(struct dsa_switch *ds, int port, int bcm_sf2_set_rxnfc(struct dsa_switch *ds, int port, struct ethtool_rxnfc *nfc) { - struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master; + struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port)); struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds); int ret = 0; @@ -1296,7 +1296,7 @@ void bcm_sf2_cfp_get_strings(struct dsa_switch *ds, int port, "CFP%03d_%sCntr", i, bcm_sf2_cfp_stats[j].name); iter = (i - 1) * s + j; - strlcpy(data + iter * ETH_GSTRING_LEN, + strscpy(data + iter * ETH_GSTRING_LEN, buf, ETH_GSTRING_LEN); } } diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c index 263e41191c2926c931aac16239740dd35ab5a6f5..b9107fe4002313ae0771dacc244d34be2a88526e 100644 --- a/drivers/net/dsa/dsa_loop.c +++ b/drivers/net/dsa/dsa_loop.c @@ -351,8 +351,6 @@ static void dsa_loop_drv_remove(struct mdio_device *mdiodev) dsa_unregister_switch(ds); dev_put(ps->netdev); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void dsa_loop_drv_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c index 01f90994dedd62c4dde4114a8eb86ba93bc42475..951f7935c872a600e680bebd3905639c43b74df6 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.c +++ b/drivers/net/dsa/hirschmann/hellcreek.c @@ -128,6 +128,16 @@ static void hellcreek_select_prio(struct hellcreek *hellcreek, int prio) hellcreek_write(hellcreek, val, HR_PSEL); } +static void hellcreek_select_port_prio(struct hellcreek *hellcreek, int port, + int prio) +{ + u16 val = port << HR_PSEL_PTWSEL_SHIFT; + + val |= prio << HR_PSEL_PRTCWSEL_SHIFT; + + hellcreek_write(hellcreek, val, HR_PSEL); +} + static void hellcreek_select_counter(struct hellcreek *hellcreek, int counter) { u16 val = counter << HR_CSEL_SHIFT; @@ -288,7 +298,7 @@ static void hellcreek_get_strings(struct dsa_switch *ds, int port, for (i = 0; i < ARRAY_SIZE(hellcreek_counter); ++i) { const struct hellcreek_counter *counter = &hellcreek_counter[i]; - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, counter->name, ETH_GSTRING_LEN); } } @@ -1537,6 +1547,45 @@ out: return ret; } +static void hellcreek_setup_maxsdu(struct hellcreek *hellcreek, int port, + const struct tc_taprio_qopt_offload *schedule) +{ + int tc; + + for (tc = 0; tc < 8; ++tc) { + u32 max_sdu = schedule->max_sdu[tc] + VLAN_ETH_HLEN - ETH_FCS_LEN; + u16 val; + + if (!schedule->max_sdu[tc]) + continue; + + dev_dbg(hellcreek->dev, "Configure max-sdu %u for tc %d on port %d\n", + max_sdu, tc, port); + + hellcreek_select_port_prio(hellcreek, port, tc); + + val = (max_sdu & HR_PTPRTCCFG_MAXSDU_MASK) << HR_PTPRTCCFG_MAXSDU_SHIFT; + + hellcreek_write(hellcreek, val, HR_PTPRTCCFG); + } +} + +static void hellcreek_reset_maxsdu(struct hellcreek *hellcreek, int port) +{ + int tc; + + for (tc = 0; tc < 8; ++tc) { + u16 val; + + hellcreek_select_port_prio(hellcreek, port, tc); + + val = (HELLCREEK_DEFAULT_MAX_SDU & HR_PTPRTCCFG_MAXSDU_MASK) + << HR_PTPRTCCFG_MAXSDU_SHIFT; + + hellcreek_write(hellcreek, val, HR_PTPRTCCFG); + } +} + static void hellcreek_setup_gcl(struct hellcreek *hellcreek, int port, const struct tc_taprio_qopt_offload *schedule) { @@ -1720,7 +1769,10 @@ static int hellcreek_port_set_schedule(struct dsa_switch *ds, int port, } hellcreek_port->current_schedule = taprio_offload_get(taprio); - /* Then select port */ + /* Configure max sdu */ + hellcreek_setup_maxsdu(hellcreek, port, hellcreek_port->current_schedule); + + /* Select tdg */ hellcreek_select_tgd(hellcreek, port); /* Enable gating and keep defaults */ @@ -1772,7 +1824,10 @@ static int hellcreek_port_del_schedule(struct dsa_switch *ds, int port) hellcreek_port->current_schedule = NULL; } - /* Then select port */ + /* Reset max sdu */ + hellcreek_reset_maxsdu(hellcreek, port); + + /* Select tgd */ hellcreek_select_tgd(hellcreek, port); /* Disable gating and return to regular switching flow */ @@ -1809,22 +1864,43 @@ static bool hellcreek_validate_schedule(struct hellcreek *hellcreek, return true; } +static int hellcreek_tc_query_caps(struct tc_query_caps_base *base) +{ + switch (base->type) { + case TC_SETUP_QDISC_TAPRIO: { + struct tc_taprio_caps *caps = base->caps; + + caps->supports_queue_max_sdu = true; + + return 0; + } + default: + return -EOPNOTSUPP; + } +} + static int hellcreek_port_setup_tc(struct dsa_switch *ds, int port, enum tc_setup_type type, void *type_data) { - struct tc_taprio_qopt_offload *taprio = type_data; struct hellcreek *hellcreek = ds->priv; - if (type != TC_SETUP_QDISC_TAPRIO) - return -EOPNOTSUPP; + switch (type) { + case TC_QUERY_CAPS: + return hellcreek_tc_query_caps(type_data); + case TC_SETUP_QDISC_TAPRIO: { + struct tc_taprio_qopt_offload *taprio = type_data; - if (!hellcreek_validate_schedule(hellcreek, taprio)) - return -EOPNOTSUPP; + if (!hellcreek_validate_schedule(hellcreek, taprio)) + return -EOPNOTSUPP; - if (taprio->enable) - return hellcreek_port_set_schedule(ds, port, taprio); + if (taprio->enable) + return hellcreek_port_set_schedule(ds, port, taprio); - return hellcreek_port_del_schedule(ds, port); + return hellcreek_port_del_schedule(ds, port); + } + default: + return -EOPNOTSUPP; + } } static const struct dsa_switch_ops hellcreek_ds_ops = { @@ -1996,7 +2072,6 @@ static int hellcreek_remove(struct platform_device *pdev) hellcreek_hwtstamp_free(hellcreek); hellcreek_ptp_free(hellcreek); dsa_unregister_switch(hellcreek->ds); - platform_set_drvdata(pdev, NULL); return 0; } diff --git a/drivers/net/dsa/hirschmann/hellcreek.h b/drivers/net/dsa/hirschmann/hellcreek.h index 9e303b8ab13c9e5dc6b281642b95b67bb605e435..4a678f7d61ae0fd3ed461f0a2653be3de427d238 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.h +++ b/drivers/net/dsa/hirschmann/hellcreek.h @@ -37,6 +37,7 @@ #define HELLCREEK_VLAN_UNTAGGED_MEMBER 0x1 #define HELLCREEK_VLAN_TAGGED_MEMBER 0x3 #define HELLCREEK_NUM_EGRESS_QUEUES 8 +#define HELLCREEK_DEFAULT_MAX_SDU 1536 /* Register definitions */ #define HR_MODID_C (0 * 2) @@ -72,6 +73,12 @@ #define HR_PRTCCFG_PCP_TC_MAP_SHIFT 0 #define HR_PRTCCFG_PCP_TC_MAP_MASK GENMASK(2, 0) +#define HR_PTPRTCCFG (0xa9 * 2) +#define HR_PTPRTCCFG_SET_QTRACK BIT(15) +#define HR_PTPRTCCFG_REJECT BIT(14) +#define HR_PTPRTCCFG_MAXSDU_SHIFT 0 +#define HR_PTPRTCCFG_MAXSDU_MASK GENMASK(10, 0) + #define HR_CSEL (0x8d * 2) #define HR_CSEL_SHIFT 0 #define HR_CSEL_MASK GENMASK(7, 0) diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c index e03ff1f267bba3b2339dde31eb123812474409b6..438e46af03e9b5a8ae5404f64ccadee305cee45a 100644 --- a/drivers/net/dsa/lan9303-core.c +++ b/drivers/net/dsa/lan9303-core.c @@ -22,6 +22,10 @@ */ #define LAN9303_CHIP_REV 0x14 # define LAN9303_CHIP_ID 0x9303 +# define LAN9352_CHIP_ID 0x9352 +# define LAN9353_CHIP_ID 0x9353 +# define LAN9354_CHIP_ID 0x9354 +# define LAN9355_CHIP_ID 0x9355 #define LAN9303_IRQ_CFG 0x15 # define LAN9303_IRQ_CFG_IRQ_ENABLE BIT(8) # define LAN9303_IRQ_CFG_IRQ_POL BIT(4) @@ -32,6 +36,7 @@ #define LAN9303_INT_EN 0x17 # define LAN9303_INT_EN_PHY_INT2_EN BIT(27) # define LAN9303_INT_EN_PHY_INT1_EN BIT(26) +#define LAN9303_BYTE_ORDER 0x19 #define LAN9303_HW_CFG 0x1D # define LAN9303_HW_CFG_READY BIT(27) # define LAN9303_HW_CFG_AMDX_EN_PORT2 BIT(26) @@ -851,15 +856,12 @@ static int lan9303_check_device(struct lan9303 *chip) if (ret) { dev_err(chip->dev, "failed to read chip revision register: %d\n", ret); - if (!chip->reset_gpio) { - dev_dbg(chip->dev, - "hint: maybe failed due to missing reset GPIO\n"); - } return ret; } - if ((reg >> 16) != LAN9303_CHIP_ID) { - dev_err(chip->dev, "expecting LAN9303 chip, but found: %X\n", + if (((reg >> 16) != LAN9303_CHIP_ID) && + ((reg >> 16) != LAN9354_CHIP_ID)) { + dev_err(chip->dev, "unexpected device found: LAN%4.4X\n", reg >> 16); return -ENODEV; } @@ -875,7 +877,7 @@ static int lan9303_check_device(struct lan9303 *chip) if (ret) dev_warn(chip->dev, "failed to disable switching %d\n", ret); - dev_info(chip->dev, "Found LAN9303 rev. %u\n", reg & 0xffff); + dev_info(chip->dev, "Found LAN%4.4X rev. %u\n", (reg >> 16), reg & 0xffff); ret = lan9303_detect_phy_setup(chip); if (ret) { @@ -1090,7 +1092,7 @@ static int lan9303_port_enable(struct dsa_switch *ds, int port, if (!dsa_port_is_user(dp)) return 0; - vlan_vid_add(dp->cpu_dp->master, htons(ETH_P_8021Q), port); + vlan_vid_add(dsa_port_to_master(dp), htons(ETH_P_8021Q), port); return lan9303_enable_processing_port(chip, port); } @@ -1103,7 +1105,7 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port) if (!dsa_port_is_user(dp)) return; - vlan_vid_del(dp->cpu_dp->master, htons(ETH_P_8021Q), port); + vlan_vid_del(dsa_port_to_master(dp), htons(ETH_P_8021Q), port); lan9303_disable_processing_port(chip, port); lan9303_phy_write(ds, chip->phy_addr_base + port, MII_BMCR, BMCR_PDOWN); @@ -1349,6 +1351,7 @@ static int lan9303_probe_reset_gpio(struct lan9303 *chip, int lan9303_probe(struct lan9303 *chip, struct device_node *np) { int ret; + u32 reg; mutex_init(&chip->indirect_mutex); mutex_init(&chip->alr_mutex); @@ -1359,6 +1362,19 @@ int lan9303_probe(struct lan9303 *chip, struct device_node *np) lan9303_handle_reset(chip); + /* First read to the device. This is a Dummy read to ensure MDIO */ + /* access is in 32-bit sync. */ + ret = lan9303_read(chip->regmap, LAN9303_BYTE_ORDER, ®); + if (ret) { + dev_err(chip->dev, "failed to access the device: %d\n", + ret); + if (!chip->reset_gpio) { + dev_dbg(chip->dev, + "hint: maybe failed due to missing reset GPIO\n"); + } + return ret; + } + ret = lan9303_check_device(chip); if (ret) return ret; diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c index 8ca4713310fa3d821297c6607137209398489ba9..7d746cd9ca1bc6b72b253792c8878d97e48ecd9b 100644 --- a/drivers/net/dsa/lan9303_i2c.c +++ b/drivers/net/dsa/lan9303_i2c.c @@ -65,18 +65,14 @@ static int lan9303_i2c_probe(struct i2c_client *client, return 0; } -static int lan9303_i2c_remove(struct i2c_client *client) +static void lan9303_i2c_remove(struct i2c_client *client) { struct lan9303_i2c *sw_dev = i2c_get_clientdata(client); if (!sw_dev) - return 0; + return; lan9303_remove(&sw_dev->chip); - - i2c_set_clientdata(client, NULL); - - return 0; } static void lan9303_i2c_shutdown(struct i2c_client *client) diff --git a/drivers/net/dsa/lan9303_mdio.c b/drivers/net/dsa/lan9303_mdio.c index bbb7032409baa2305653013713100a2d247ab853..4f33369a2de52f767385976268d826b5d6308d7a 100644 --- a/drivers/net/dsa/lan9303_mdio.c +++ b/drivers/net/dsa/lan9303_mdio.c @@ -138,8 +138,6 @@ static void lan9303_mdio_remove(struct mdio_device *mdiodev) return; lan9303_remove(&sw_dev->chip); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void lan9303_mdio_shutdown(struct mdio_device *mdiodev) @@ -158,6 +156,7 @@ static void lan9303_mdio_shutdown(struct mdio_device *mdiodev) static const struct of_device_id lan9303_mdio_of_match[] = { { .compatible = "smsc,lan9303-mdio" }, + { .compatible = "microchip,lan9354-mdio" }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, lan9303_mdio_of_match); diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c index e531b93f3cb2715e84b5bcd76330e32d0747d4cc..05ecaa007ab18821bfd99e7fe3f1edfc78dca602 100644 --- a/drivers/net/dsa/lantiq_gswip.c +++ b/drivers/net/dsa/lantiq_gswip.c @@ -1989,11 +1989,9 @@ static int gswip_gphy_fw_probe(struct gswip_priv *priv, } gphy_fw->reset = of_reset_control_array_get_exclusive(gphy_fw_np); - if (IS_ERR(gphy_fw->reset)) { - if (PTR_ERR(gphy_fw->reset) != -EPROBE_DEFER) - dev_err(dev, "Failed to lookup gphy reset\n"); - return PTR_ERR(gphy_fw->reset); - } + if (IS_ERR(gphy_fw->reset)) + return dev_err_probe(dev, PTR_ERR(gphy_fw->reset), + "Failed to lookup gphy reset\n"); return gswip_gphy_fw_load(priv, gphy_fw); } @@ -2231,8 +2229,6 @@ static int gswip_remove(struct platform_device *pdev) for (i = 0; i < priv->num_gphy_fw; i++) gswip_gphy_fw_remove(priv, &priv->gphy_fw[i]); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h index 42c50cc4d853613b7ef6595652b7c1c3aded6f64..8582b4b67d989f2fc2dc671adef90caa046c3dd4 100644 --- a/drivers/net/dsa/microchip/ksz8.h +++ b/drivers/net/dsa/microchip/ksz8.h @@ -17,8 +17,8 @@ u32 ksz8_get_port_addr(int port, int offset); void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member); void ksz8_flush_dyn_mac_table(struct ksz_device *dev, int port); void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port); -void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val); -void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val); +int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val); +int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val); int ksz8_r_dyn_mac_table(struct ksz_device *dev, u16 addr, u8 *mac_addr, u8 *fid, u8 *src_port, u8 *timestamp, u16 *entries); int ksz8_r_sta_mac_table(struct ksz_device *dev, u16 addr, diff --git a/drivers/net/dsa/microchip/ksz8795.c b/drivers/net/dsa/microchip/ksz8795.c index c79a5128235f9cc1f1c2479708e6e882ccb90817..bd3b133e7085b8aea692c288f79ccaf39bdd44dd 100644 --- a/drivers/net/dsa/microchip/ksz8795.c +++ b/drivers/net/dsa/microchip/ksz8795.c @@ -552,7 +552,7 @@ static void ksz8_w_vlan_table(struct ksz_device *dev, u16 vid, u16 vlan) ksz8_w_table(dev, TABLE_VLAN, addr, buf); } -void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) +int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) { u8 restart, speed, ctrl, link; int processed = true; @@ -560,14 +560,24 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) u8 val1, val2; u16 data = 0; u8 p = phy; + int ret; regs = dev->info->regs; switch (reg) { case MII_BMCR: - ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart); - ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed); - ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl); + ret = ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart); + if (ret) + return ret; + + ret = ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed); + if (ret) + return ret; + + ret = ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl); + if (ret) + return ret; + if (restart & PORT_PHY_LOOPBACK) data |= BMCR_LOOPBACK; if (ctrl & PORT_FORCE_100_MBIT) @@ -597,7 +607,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) data |= KSZ886X_BMCR_DISABLE_LED; break; case MII_BMSR: - ksz_pread8(dev, p, regs[P_LINK_STATUS], &link); + ret = ksz_pread8(dev, p, regs[P_LINK_STATUS], &link); + if (ret) + return ret; + data = BMSR_100FULL | BMSR_100HALF | BMSR_10FULL | @@ -618,7 +631,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) data = KSZ8795_ID_LO; break; case MII_ADVERTISE: - ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl); + ret = ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl); + if (ret) + return ret; + data = ADVERTISE_CSMA; if (ctrl & PORT_AUTO_NEG_SYM_PAUSE) data |= ADVERTISE_PAUSE_CAP; @@ -632,7 +648,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) data |= ADVERTISE_10HALF; break; case MII_LPA: - ksz_pread8(dev, p, regs[P_REMOTE_STATUS], &link); + ret = ksz_pread8(dev, p, regs[P_REMOTE_STATUS], &link); + if (ret) + return ret; + data = LPA_SLCT; if (link & PORT_REMOTE_SYM_PAUSE) data |= LPA_PAUSE_CAP; @@ -648,8 +667,14 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) data |= LPA_LPACK; break; case PHY_REG_LINK_MD: - ksz_pread8(dev, p, REG_PORT_LINK_MD_CTRL, &val1); - ksz_pread8(dev, p, REG_PORT_LINK_MD_RESULT, &val2); + ret = ksz_pread8(dev, p, REG_PORT_LINK_MD_CTRL, &val1); + if (ret) + return ret; + + ret = ksz_pread8(dev, p, REG_PORT_LINK_MD_RESULT, &val2); + if (ret) + return ret; + if (val1 & PORT_START_CABLE_DIAG) data |= PHY_START_CABLE_DIAG; @@ -664,7 +689,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) FIELD_GET(PORT_CABLE_FAULT_COUNTER_L, val2)); break; case PHY_REG_PHY_CTRL: - ksz_pread8(dev, p, regs[P_LINK_STATUS], &link); + ret = ksz_pread8(dev, p, regs[P_LINK_STATUS], &link); + if (ret) + return ret; + if (link & PORT_MDIX_STATUS) data |= KSZ886X_CTRL_MDIX_STAT; break; @@ -674,13 +702,16 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val) } if (processed) *val = data; + + return 0; } -void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) +int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) { u8 restart, speed, ctrl, data; const u16 *regs; u8 p = phy; + int ret; regs = dev->info->regs; @@ -690,15 +721,26 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) /* Do not support PHY reset function. */ if (val & BMCR_RESET) break; - ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed); + ret = ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed); + if (ret) + return ret; + data = speed; if (val & KSZ886X_BMCR_HP_MDIX) data |= PORT_HP_MDIX; else data &= ~PORT_HP_MDIX; - if (data != speed) - ksz_pwrite8(dev, p, regs[P_SPEED_STATUS], data); - ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl); + + if (data != speed) { + ret = ksz_pwrite8(dev, p, regs[P_SPEED_STATUS], data); + if (ret) + return ret; + } + + ret = ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl); + if (ret) + return ret; + data = ctrl; if (ksz_is_ksz88x3(dev)) { if ((val & BMCR_ANENABLE)) @@ -724,9 +766,17 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) data |= PORT_FORCE_FULL_DUPLEX; else data &= ~PORT_FORCE_FULL_DUPLEX; - if (data != ctrl) - ksz_pwrite8(dev, p, regs[P_FORCE_CTRL], data); - ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart); + + if (data != ctrl) { + ret = ksz_pwrite8(dev, p, regs[P_FORCE_CTRL], data); + if (ret) + return ret; + } + + ret = ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart); + if (ret) + return ret; + data = restart; if (val & KSZ886X_BMCR_DISABLE_LED) data |= PORT_LED_OFF; @@ -756,11 +806,19 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) data |= PORT_PHY_LOOPBACK; else data &= ~PORT_PHY_LOOPBACK; - if (data != restart) - ksz_pwrite8(dev, p, regs[P_NEG_RESTART_CTRL], data); + + if (data != restart) { + ret = ksz_pwrite8(dev, p, regs[P_NEG_RESTART_CTRL], + data); + if (ret) + return ret; + } break; case MII_ADVERTISE: - ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl); + ret = ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl); + if (ret) + return ret; + data = ctrl; data &= ~(PORT_AUTO_NEG_SYM_PAUSE | PORT_AUTO_NEG_100BTX_FD | @@ -777,8 +835,12 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) data |= PORT_AUTO_NEG_10BT_FD; if (val & ADVERTISE_10HALF) data |= PORT_AUTO_NEG_10BT; - if (data != ctrl) - ksz_pwrite8(dev, p, regs[P_LOCAL_CTRL], data); + + if (data != ctrl) { + ret = ksz_pwrite8(dev, p, regs[P_LOCAL_CTRL], data); + if (ret) + return ret; + } break; case PHY_REG_LINK_MD: if (val & PHY_START_CABLE_DIAG) @@ -787,6 +849,8 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val) default: break; } + + return 0; } void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member) @@ -1187,7 +1251,6 @@ void ksz8_config_cpu_port(struct dsa_switch *ds) if (i == dev->phy_port_cnt) break; p->on = 1; - p->phy = 1; } for (i = 0; i < dev->phy_port_cnt; i++) { p = &dev->ports[i]; diff --git a/drivers/net/dsa/microchip/ksz8863_smi.c b/drivers/net/dsa/microchip/ksz8863_smi.c index 5247fdfb964d7b989293f9d18db397cc47a768b3..ddb40838181efdc30db1402a5b8f71e459707656 100644 --- a/drivers/net/dsa/microchip/ksz8863_smi.c +++ b/drivers/net/dsa/microchip/ksz8863_smi.c @@ -180,8 +180,6 @@ static void ksz8863_smi_remove(struct mdio_device *mdiodev) if (dev) ksz_switch_remove(dev); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void ksz8863_smi_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c index e4f446db0ca18da695633da40b4d88c3ba974ef4..a6a0321a89310374b52e2d7dd42bdca2e333814a 100644 --- a/drivers/net/dsa/microchip/ksz9477.c +++ b/drivers/net/dsa/microchip/ksz9477.c @@ -193,6 +193,11 @@ int ksz9477_reset_switch(struct ksz_device *dev) ksz_write32(dev, REG_SW_PORT_INT_MASK__4, 0x7F); ksz_read32(dev, REG_SW_PORT_INT_STATUS__4, &data32); + /* KSZ9893 compatible chips do not support refclk configuration */ + if (dev->chip_id == KSZ9893_CHIP_ID || + dev->chip_id == KSZ8563_CHIP_ID) + return 0; + data8 = SW_ENABLE_REFCLKO; if (dev->synclko_disable) data8 = 0; @@ -264,9 +269,20 @@ void ksz9477_port_init_cnt(struct ksz_device *dev, int port) mutex_unlock(&mib->cnt_mutex); } -void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) +static void ksz9477_r_phy_quirks(struct ksz_device *dev, u16 addr, u16 reg, + u16 *data) +{ + /* KSZ8563R do not have extended registers but BMSR_ESTATEN and + * BMSR_ERCAP bits are set. + */ + if (dev->chip_id == KSZ8563_CHIP_ID && reg == MII_BMSR) + *data &= ~(BMSR_ESTATEN | BMSR_ERCAP); +} + +int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) { u16 val = 0xffff; + int ret; /* No real PHY after this. Simulate the PHY. * A fixed PHY can be setup in the device tree, but this function is @@ -274,7 +290,7 @@ void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) * For RGMII PHY there is no way to access it so the fixed PHY should * be used. For SGMII PHY the supporting code will be added later. */ - if (addr >= dev->phy_port_cnt) { + if (!dev->info->internal_phy[addr]) { struct ksz_port *p = &dev->ports[addr]; switch (reg) { @@ -307,23 +323,25 @@ void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) break; } } else { - ksz_pread16(dev, addr, 0x100 + (reg << 1), &val); + ret = ksz_pread16(dev, addr, 0x100 + (reg << 1), &val); + if (ret) + return ret; + + ksz9477_r_phy_quirks(dev, addr, reg, &val); } *data = val; + + return 0; } -void ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val) +int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val) { /* No real PHY after this. */ - if (addr >= dev->phy_port_cnt) - return; + if (!dev->info->internal_phy[addr]) + return 0; - /* No gigabit support. Do not write to this register. */ - if (!(dev->features & GBIT_SUPPORT) && reg == MII_CTRL1000) - return; - - ksz_pwrite16(dev, addr, 0x100 + (reg << 1), val); + return ksz_pwrite16(dev, addr, 0x100 + (reg << 1), val); } void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member) @@ -869,7 +887,7 @@ static phy_interface_t ksz9477_get_interface(struct ksz_device *dev, int port) phy_interface_t interface; bool gbit; - if (port < dev->phy_port_cnt) + if (dev->info->internal_phy[port]) return PHY_INTERFACE_MODE_NA; gbit = ksz_get_gbit(dev, port); @@ -914,7 +932,7 @@ static void ksz9477_phy_errata_setup(struct ksz_device *dev, int port) /* Energy Efficient Ethernet (EEE) feature select must * be manually disabled (except on KSZ8565 which is 100Mbit) */ - if (dev->features & GBIT_SUPPORT) + if (dev->info->gbit_capable[port]) ksz9477_port_mmd_write(dev, port, 0x07, 0x3c, 0x0000); /* Register settings are required to meet data sheet @@ -941,10 +959,35 @@ void ksz9477_get_caps(struct ksz_device *dev, int port, config->mac_capabilities = MAC_10 | MAC_100 | MAC_ASYM_PAUSE | MAC_SYM_PAUSE; - if (dev->features & GBIT_SUPPORT) + if (dev->info->gbit_capable[port]) config->mac_capabilities |= MAC_1000FD; } +int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs) +{ + u32 secs = msecs / 1000; + u8 value; + u8 data; + int ret; + + value = FIELD_GET(SW_AGE_PERIOD_7_0_M, secs); + + ret = ksz_write8(dev, REG_SW_LUE_CTRL_3, value); + if (ret < 0) + return ret; + + data = FIELD_GET(SW_AGE_PERIOD_10_8_M, secs); + + ret = ksz_read8(dev, REG_SW_LUE_CTRL_0, &value); + if (ret < 0) + return ret; + + value &= ~SW_AGE_CNT_M; + value |= FIELD_PREP(SW_AGE_CNT_M, data); + + return ksz_write8(dev, REG_SW_LUE_CTRL_0, value); +} + void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port) { struct dsa_switch *ds = dev->ds; @@ -976,7 +1019,7 @@ void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port) /* enable 802.1p priority */ ksz_port_cfg(dev, port, P_PRIO_CTRL, PORT_802_1P_PRIO_ENABLE, true); - if (port < dev->phy_port_cnt) { + if (dev->info->internal_phy[port]) { /* do not force flow control */ ksz_port_cfg(dev, port, REG_PORT_CTRL_0, PORT_FORCE_TX_FLOW_CTRL | PORT_FORCE_RX_FLOW_CTRL, @@ -999,7 +1042,7 @@ void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port) ksz9477_cfg_port_member(dev, port, member); /* clear pending interrupts */ - if (port < dev->phy_port_cnt) + if (dev->info->internal_phy[port]) ksz_pread16(dev, port, REG_PORT_PHY_INT_ENABLE, &data16); } @@ -1051,25 +1094,13 @@ void ksz9477_config_cpu_port(struct dsa_switch *ds) /* enable cpu port */ ksz9477_port_setup(dev, i, true); - p->on = 1; } } for (i = 0; i < dev->info->port_cnt; i++) { if (i == dev->cpu_port) continue; - p = &dev->ports[i]; - ksz_port_stp_state_set(ds, i, BR_STATE_DISABLED); - p->on = 1; - if (i < dev->phy_port_cnt) - p->phy = 1; - if (dev->chip_id == 0x00947700 && i == 6) { - p->sgmii = 1; - - /* SGMII PHY detection code is not implemented yet. */ - p->phy = 0; - } } } @@ -1158,29 +1189,6 @@ int ksz9477_switch_init(struct ksz_device *dev) if (ret) return ret; - ret = ksz_read8(dev, REG_GLOBAL_OPTIONS, &data8); - if (ret) - return ret; - - /* Number of ports can be reduced depending on chip. */ - dev->phy_port_cnt = 5; - - /* Default capability is gigabit capable. */ - dev->features = GBIT_SUPPORT; - - if (dev->chip_id == KSZ9893_CHIP_ID) { - dev->features |= IS_9893; - - /* Chip does not support gigabit. */ - if (data8 & SW_QW_ABLE) - dev->features &= ~GBIT_SUPPORT; - dev->phy_port_cnt = 2; - } else { - /* Chip does not support gigabit. */ - if (!(data8 & SW_GIGABIT_ABLE)) - dev->features &= ~GBIT_SUPPORT; - } - return 0; } diff --git a/drivers/net/dsa/microchip/ksz9477.h b/drivers/net/dsa/microchip/ksz9477.h index cd278b307b3c7a60aaaa29f8f3df7784396dfbc6..00862c4cfb7f1e222ac8cc3e237c487cbfbbfed6 100644 --- a/drivers/net/dsa/microchip/ksz9477.h +++ b/drivers/net/dsa/microchip/ksz9477.h @@ -16,8 +16,9 @@ u32 ksz9477_get_port_addr(int port, int offset); void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member); void ksz9477_flush_dyn_mac_table(struct ksz_device *dev, int port); void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port); -void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data); -void ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val); +int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs); +int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data); +int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val); void ksz9477_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt); void ksz9477_r_mib_pkt(struct ksz_device *dev, int port, u16 addr, u64 *dropped, u64 *cnt); diff --git a/drivers/net/dsa/microchip/ksz9477_i2c.c b/drivers/net/dsa/microchip/ksz9477_i2c.c index 99966514d444765e30ad6e04c5ca5fdd1b186f4a..3763930dc6fc4765d08e7c3e6cf1ebe4acc3b24a 100644 --- a/drivers/net/dsa/microchip/ksz9477_i2c.c +++ b/drivers/net/dsa/microchip/ksz9477_i2c.c @@ -52,16 +52,12 @@ static int ksz9477_i2c_probe(struct i2c_client *i2c, return 0; } -static int ksz9477_i2c_remove(struct i2c_client *i2c) +static void ksz9477_i2c_remove(struct i2c_client *i2c) { struct ksz_device *dev = i2c_get_clientdata(i2c); if (dev) ksz_switch_remove(dev); - - i2c_set_clientdata(i2c, NULL); - - return 0; } static void ksz9477_i2c_shutdown(struct i2c_client *i2c) @@ -91,6 +87,10 @@ static const struct of_device_id ksz9477_dt_ids[] = { .compatible = "microchip,ksz9477", .data = &ksz_switch_chips[KSZ9477] }, + { + .compatible = "microchip,ksz9896", + .data = &ksz_switch_chips[KSZ9896] + }, { .compatible = "microchip,ksz9897", .data = &ksz_switch_chips[KSZ9897] diff --git a/drivers/net/dsa/microchip/ksz9477_reg.h b/drivers/net/dsa/microchip/ksz9477_reg.h index ddf99d1e4bbd32812be38778535d92631774e8fc..53c68d286dd3ad893199a6525b0d458d2abe0d1c 100644 --- a/drivers/net/dsa/microchip/ksz9477_reg.h +++ b/drivers/net/dsa/microchip/ksz9477_reg.h @@ -189,8 +189,9 @@ #define SW_VLAN_ENABLE BIT(7) #define SW_DROP_INVALID_VID BIT(6) -#define SW_AGE_CNT_M 0x7 +#define SW_AGE_CNT_M GENMASK(5, 3) #define SW_AGE_CNT_S 3 +#define SW_AGE_PERIOD_10_8_M GENMASK(10, 8) #define SW_RESV_MCAST_ENABLE BIT(2) #define SW_HASH_OPTION_M 0x03 #define SW_HASH_OPTION_CRC 1 @@ -225,6 +226,7 @@ #define SW_PRIO_LOWEST_DA_SA 3 #define REG_SW_LUE_CTRL_3 0x0313 +#define SW_AGE_PERIOD_7_0_M GENMASK(7, 0) #define REG_SW_LUE_INT_STATUS 0x0314 #define REG_SW_LUE_INT_ENABLE 0x0315 diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index ed7d137cba9942a35a238150b586fbbf36a7546e..d612181b3226e36f52a2b80414f1b5e23bf42b18 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -14,6 +14,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -170,12 +173,20 @@ static const struct ksz_dev_ops ksz8_dev_ops = { .exit = ksz8_switch_exit, }; +static void ksz9477_phylink_mac_link_up(struct ksz_device *dev, int port, + unsigned int mode, + phy_interface_t interface, + struct phy_device *phydev, int speed, + int duplex, bool tx_pause, + bool rx_pause); + static const struct ksz_dev_ops ksz9477_dev_ops = { .setup = ksz9477_setup, .get_port_addr = ksz9477_get_port_addr, .cfg_port_member = ksz9477_cfg_port_member, .flush_dyn_mac_table = ksz9477_flush_dyn_mac_table, .port_setup = ksz9477_port_setup, + .set_ageing_time = ksz9477_set_ageing_time, .r_phy = ksz9477_r_phy, .w_phy = ksz9477_w_phy, .r_mib_cnt = ksz9477_r_mib_cnt, @@ -196,6 +207,7 @@ static const struct ksz_dev_ops ksz9477_dev_ops = { .mdb_del = ksz9477_mdb_del, .change_mtu = ksz9477_change_mtu, .max_mtu = ksz9477_max_mtu, + .phylink_mac_link_up = ksz9477_phylink_mac_link_up, .config_cpu_port = ksz9477_config_cpu_port, .enable_stp_addr = ksz9477_enable_stp_addr, .reset = ksz9477_reset_switch, @@ -205,10 +217,12 @@ static const struct ksz_dev_ops ksz9477_dev_ops = { static const struct ksz_dev_ops lan937x_dev_ops = { .setup = lan937x_setup, + .teardown = lan937x_teardown, .get_port_addr = ksz9477_get_port_addr, .cfg_port_member = ksz9477_cfg_port_member, .flush_dyn_mac_table = ksz9477_flush_dyn_mac_table, .port_setup = lan937x_port_setup, + .set_ageing_time = lan937x_set_ageing_time, .r_phy = lan937x_r_phy, .w_phy = lan937x_w_phy, .r_mib_cnt = ksz9477_r_mib_cnt, @@ -230,6 +244,7 @@ static const struct ksz_dev_ops lan937x_dev_ops = { .mdb_del = ksz9477_mdb_del, .change_mtu = lan937x_change_mtu, .max_mtu = ksz9477_max_mtu, + .phylink_mac_link_up = ksz9477_phylink_mac_link_up, .config_cpu_port = lan937x_config_cpu_port, .enable_stp_addr = ksz9477_enable_stp_addr, .reset = lan937x_reset_switch, @@ -412,7 +427,636 @@ static const u8 lan937x_shifts[] = { [ALU_STAT_INDEX] = 8, }; +static const struct regmap_range ksz8563_valid_regs[] = { + regmap_reg_range(0x0000, 0x0003), + regmap_reg_range(0x0006, 0x0006), + regmap_reg_range(0x000f, 0x001f), + regmap_reg_range(0x0100, 0x0100), + regmap_reg_range(0x0104, 0x0107), + regmap_reg_range(0x010d, 0x010d), + regmap_reg_range(0x0110, 0x0113), + regmap_reg_range(0x0120, 0x012b), + regmap_reg_range(0x0201, 0x0201), + regmap_reg_range(0x0210, 0x0213), + regmap_reg_range(0x0300, 0x0300), + regmap_reg_range(0x0302, 0x031b), + regmap_reg_range(0x0320, 0x032b), + regmap_reg_range(0x0330, 0x0336), + regmap_reg_range(0x0338, 0x033e), + regmap_reg_range(0x0340, 0x035f), + regmap_reg_range(0x0370, 0x0370), + regmap_reg_range(0x0378, 0x0378), + regmap_reg_range(0x037c, 0x037d), + regmap_reg_range(0x0390, 0x0393), + regmap_reg_range(0x0400, 0x040e), + regmap_reg_range(0x0410, 0x042f), + regmap_reg_range(0x0500, 0x0519), + regmap_reg_range(0x0520, 0x054b), + regmap_reg_range(0x0550, 0x05b3), + + /* port 1 */ + regmap_reg_range(0x1000, 0x1001), + regmap_reg_range(0x1004, 0x100b), + regmap_reg_range(0x1013, 0x1013), + regmap_reg_range(0x1017, 0x1017), + regmap_reg_range(0x101b, 0x101b), + regmap_reg_range(0x101f, 0x1021), + regmap_reg_range(0x1030, 0x1030), + regmap_reg_range(0x1100, 0x1111), + regmap_reg_range(0x111a, 0x111d), + regmap_reg_range(0x1122, 0x1127), + regmap_reg_range(0x112a, 0x112b), + regmap_reg_range(0x1136, 0x1139), + regmap_reg_range(0x113e, 0x113f), + regmap_reg_range(0x1400, 0x1401), + regmap_reg_range(0x1403, 0x1403), + regmap_reg_range(0x1410, 0x1417), + regmap_reg_range(0x1420, 0x1423), + regmap_reg_range(0x1500, 0x1507), + regmap_reg_range(0x1600, 0x1612), + regmap_reg_range(0x1800, 0x180f), + regmap_reg_range(0x1900, 0x1907), + regmap_reg_range(0x1914, 0x191b), + regmap_reg_range(0x1a00, 0x1a03), + regmap_reg_range(0x1a04, 0x1a08), + regmap_reg_range(0x1b00, 0x1b01), + regmap_reg_range(0x1b04, 0x1b04), + regmap_reg_range(0x1c00, 0x1c05), + regmap_reg_range(0x1c08, 0x1c1b), + + /* port 2 */ + regmap_reg_range(0x2000, 0x2001), + regmap_reg_range(0x2004, 0x200b), + regmap_reg_range(0x2013, 0x2013), + regmap_reg_range(0x2017, 0x2017), + regmap_reg_range(0x201b, 0x201b), + regmap_reg_range(0x201f, 0x2021), + regmap_reg_range(0x2030, 0x2030), + regmap_reg_range(0x2100, 0x2111), + regmap_reg_range(0x211a, 0x211d), + regmap_reg_range(0x2122, 0x2127), + regmap_reg_range(0x212a, 0x212b), + regmap_reg_range(0x2136, 0x2139), + regmap_reg_range(0x213e, 0x213f), + regmap_reg_range(0x2400, 0x2401), + regmap_reg_range(0x2403, 0x2403), + regmap_reg_range(0x2410, 0x2417), + regmap_reg_range(0x2420, 0x2423), + regmap_reg_range(0x2500, 0x2507), + regmap_reg_range(0x2600, 0x2612), + regmap_reg_range(0x2800, 0x280f), + regmap_reg_range(0x2900, 0x2907), + regmap_reg_range(0x2914, 0x291b), + regmap_reg_range(0x2a00, 0x2a03), + regmap_reg_range(0x2a04, 0x2a08), + regmap_reg_range(0x2b00, 0x2b01), + regmap_reg_range(0x2b04, 0x2b04), + regmap_reg_range(0x2c00, 0x2c05), + regmap_reg_range(0x2c08, 0x2c1b), + + /* port 3 */ + regmap_reg_range(0x3000, 0x3001), + regmap_reg_range(0x3004, 0x300b), + regmap_reg_range(0x3013, 0x3013), + regmap_reg_range(0x3017, 0x3017), + regmap_reg_range(0x301b, 0x301b), + regmap_reg_range(0x301f, 0x3021), + regmap_reg_range(0x3030, 0x3030), + regmap_reg_range(0x3300, 0x3301), + regmap_reg_range(0x3303, 0x3303), + regmap_reg_range(0x3400, 0x3401), + regmap_reg_range(0x3403, 0x3403), + regmap_reg_range(0x3410, 0x3417), + regmap_reg_range(0x3420, 0x3423), + regmap_reg_range(0x3500, 0x3507), + regmap_reg_range(0x3600, 0x3612), + regmap_reg_range(0x3800, 0x380f), + regmap_reg_range(0x3900, 0x3907), + regmap_reg_range(0x3914, 0x391b), + regmap_reg_range(0x3a00, 0x3a03), + regmap_reg_range(0x3a04, 0x3a08), + regmap_reg_range(0x3b00, 0x3b01), + regmap_reg_range(0x3b04, 0x3b04), + regmap_reg_range(0x3c00, 0x3c05), + regmap_reg_range(0x3c08, 0x3c1b), +}; + +static const struct regmap_access_table ksz8563_register_set = { + .yes_ranges = ksz8563_valid_regs, + .n_yes_ranges = ARRAY_SIZE(ksz8563_valid_regs), +}; + +static const struct regmap_range ksz9477_valid_regs[] = { + regmap_reg_range(0x0000, 0x0003), + regmap_reg_range(0x0006, 0x0006), + regmap_reg_range(0x0010, 0x001f), + regmap_reg_range(0x0100, 0x0100), + regmap_reg_range(0x0103, 0x0107), + regmap_reg_range(0x010d, 0x010d), + regmap_reg_range(0x0110, 0x0113), + regmap_reg_range(0x0120, 0x012b), + regmap_reg_range(0x0201, 0x0201), + regmap_reg_range(0x0210, 0x0213), + regmap_reg_range(0x0300, 0x0300), + regmap_reg_range(0x0302, 0x031b), + regmap_reg_range(0x0320, 0x032b), + regmap_reg_range(0x0330, 0x0336), + regmap_reg_range(0x0338, 0x033b), + regmap_reg_range(0x033e, 0x033e), + regmap_reg_range(0x0340, 0x035f), + regmap_reg_range(0x0370, 0x0370), + regmap_reg_range(0x0378, 0x0378), + regmap_reg_range(0x037c, 0x037d), + regmap_reg_range(0x0390, 0x0393), + regmap_reg_range(0x0400, 0x040e), + regmap_reg_range(0x0410, 0x042f), + regmap_reg_range(0x0444, 0x044b), + regmap_reg_range(0x0450, 0x046f), + regmap_reg_range(0x0500, 0x0519), + regmap_reg_range(0x0520, 0x054b), + regmap_reg_range(0x0550, 0x05b3), + regmap_reg_range(0x0604, 0x060b), + regmap_reg_range(0x0610, 0x0612), + regmap_reg_range(0x0614, 0x062c), + regmap_reg_range(0x0640, 0x0645), + regmap_reg_range(0x0648, 0x064d), + + /* port 1 */ + regmap_reg_range(0x1000, 0x1001), + regmap_reg_range(0x1013, 0x1013), + regmap_reg_range(0x1017, 0x1017), + regmap_reg_range(0x101b, 0x101b), + regmap_reg_range(0x101f, 0x1020), + regmap_reg_range(0x1030, 0x1030), + regmap_reg_range(0x1100, 0x1115), + regmap_reg_range(0x111a, 0x111f), + regmap_reg_range(0x1122, 0x1127), + regmap_reg_range(0x112a, 0x112b), + regmap_reg_range(0x1136, 0x1139), + regmap_reg_range(0x113e, 0x113f), + regmap_reg_range(0x1400, 0x1401), + regmap_reg_range(0x1403, 0x1403), + regmap_reg_range(0x1410, 0x1417), + regmap_reg_range(0x1420, 0x1423), + regmap_reg_range(0x1500, 0x1507), + regmap_reg_range(0x1600, 0x1613), + regmap_reg_range(0x1800, 0x180f), + regmap_reg_range(0x1820, 0x1827), + regmap_reg_range(0x1830, 0x1837), + regmap_reg_range(0x1840, 0x184b), + regmap_reg_range(0x1900, 0x1907), + regmap_reg_range(0x1914, 0x191b), + regmap_reg_range(0x1920, 0x1920), + regmap_reg_range(0x1923, 0x1927), + regmap_reg_range(0x1a00, 0x1a03), + regmap_reg_range(0x1a04, 0x1a07), + regmap_reg_range(0x1b00, 0x1b01), + regmap_reg_range(0x1b04, 0x1b04), + regmap_reg_range(0x1c00, 0x1c05), + regmap_reg_range(0x1c08, 0x1c1b), + + /* port 2 */ + regmap_reg_range(0x2000, 0x2001), + regmap_reg_range(0x2013, 0x2013), + regmap_reg_range(0x2017, 0x2017), + regmap_reg_range(0x201b, 0x201b), + regmap_reg_range(0x201f, 0x2020), + regmap_reg_range(0x2030, 0x2030), + regmap_reg_range(0x2100, 0x2115), + regmap_reg_range(0x211a, 0x211f), + regmap_reg_range(0x2122, 0x2127), + regmap_reg_range(0x212a, 0x212b), + regmap_reg_range(0x2136, 0x2139), + regmap_reg_range(0x213e, 0x213f), + regmap_reg_range(0x2400, 0x2401), + regmap_reg_range(0x2403, 0x2403), + regmap_reg_range(0x2410, 0x2417), + regmap_reg_range(0x2420, 0x2423), + regmap_reg_range(0x2500, 0x2507), + regmap_reg_range(0x2600, 0x2613), + regmap_reg_range(0x2800, 0x280f), + regmap_reg_range(0x2820, 0x2827), + regmap_reg_range(0x2830, 0x2837), + regmap_reg_range(0x2840, 0x284b), + regmap_reg_range(0x2900, 0x2907), + regmap_reg_range(0x2914, 0x291b), + regmap_reg_range(0x2920, 0x2920), + regmap_reg_range(0x2923, 0x2927), + regmap_reg_range(0x2a00, 0x2a03), + regmap_reg_range(0x2a04, 0x2a07), + regmap_reg_range(0x2b00, 0x2b01), + regmap_reg_range(0x2b04, 0x2b04), + regmap_reg_range(0x2c00, 0x2c05), + regmap_reg_range(0x2c08, 0x2c1b), + + /* port 3 */ + regmap_reg_range(0x3000, 0x3001), + regmap_reg_range(0x3013, 0x3013), + regmap_reg_range(0x3017, 0x3017), + regmap_reg_range(0x301b, 0x301b), + regmap_reg_range(0x301f, 0x3020), + regmap_reg_range(0x3030, 0x3030), + regmap_reg_range(0x3100, 0x3115), + regmap_reg_range(0x311a, 0x311f), + regmap_reg_range(0x3122, 0x3127), + regmap_reg_range(0x312a, 0x312b), + regmap_reg_range(0x3136, 0x3139), + regmap_reg_range(0x313e, 0x313f), + regmap_reg_range(0x3400, 0x3401), + regmap_reg_range(0x3403, 0x3403), + regmap_reg_range(0x3410, 0x3417), + regmap_reg_range(0x3420, 0x3423), + regmap_reg_range(0x3500, 0x3507), + regmap_reg_range(0x3600, 0x3613), + regmap_reg_range(0x3800, 0x380f), + regmap_reg_range(0x3820, 0x3827), + regmap_reg_range(0x3830, 0x3837), + regmap_reg_range(0x3840, 0x384b), + regmap_reg_range(0x3900, 0x3907), + regmap_reg_range(0x3914, 0x391b), + regmap_reg_range(0x3920, 0x3920), + regmap_reg_range(0x3923, 0x3927), + regmap_reg_range(0x3a00, 0x3a03), + regmap_reg_range(0x3a04, 0x3a07), + regmap_reg_range(0x3b00, 0x3b01), + regmap_reg_range(0x3b04, 0x3b04), + regmap_reg_range(0x3c00, 0x3c05), + regmap_reg_range(0x3c08, 0x3c1b), + + /* port 4 */ + regmap_reg_range(0x4000, 0x4001), + regmap_reg_range(0x4013, 0x4013), + regmap_reg_range(0x4017, 0x4017), + regmap_reg_range(0x401b, 0x401b), + regmap_reg_range(0x401f, 0x4020), + regmap_reg_range(0x4030, 0x4030), + regmap_reg_range(0x4100, 0x4115), + regmap_reg_range(0x411a, 0x411f), + regmap_reg_range(0x4122, 0x4127), + regmap_reg_range(0x412a, 0x412b), + regmap_reg_range(0x4136, 0x4139), + regmap_reg_range(0x413e, 0x413f), + regmap_reg_range(0x4400, 0x4401), + regmap_reg_range(0x4403, 0x4403), + regmap_reg_range(0x4410, 0x4417), + regmap_reg_range(0x4420, 0x4423), + regmap_reg_range(0x4500, 0x4507), + regmap_reg_range(0x4600, 0x4613), + regmap_reg_range(0x4800, 0x480f), + regmap_reg_range(0x4820, 0x4827), + regmap_reg_range(0x4830, 0x4837), + regmap_reg_range(0x4840, 0x484b), + regmap_reg_range(0x4900, 0x4907), + regmap_reg_range(0x4914, 0x491b), + regmap_reg_range(0x4920, 0x4920), + regmap_reg_range(0x4923, 0x4927), + regmap_reg_range(0x4a00, 0x4a03), + regmap_reg_range(0x4a04, 0x4a07), + regmap_reg_range(0x4b00, 0x4b01), + regmap_reg_range(0x4b04, 0x4b04), + regmap_reg_range(0x4c00, 0x4c05), + regmap_reg_range(0x4c08, 0x4c1b), + + /* port 5 */ + regmap_reg_range(0x5000, 0x5001), + regmap_reg_range(0x5013, 0x5013), + regmap_reg_range(0x5017, 0x5017), + regmap_reg_range(0x501b, 0x501b), + regmap_reg_range(0x501f, 0x5020), + regmap_reg_range(0x5030, 0x5030), + regmap_reg_range(0x5100, 0x5115), + regmap_reg_range(0x511a, 0x511f), + regmap_reg_range(0x5122, 0x5127), + regmap_reg_range(0x512a, 0x512b), + regmap_reg_range(0x5136, 0x5139), + regmap_reg_range(0x513e, 0x513f), + regmap_reg_range(0x5400, 0x5401), + regmap_reg_range(0x5403, 0x5403), + regmap_reg_range(0x5410, 0x5417), + regmap_reg_range(0x5420, 0x5423), + regmap_reg_range(0x5500, 0x5507), + regmap_reg_range(0x5600, 0x5613), + regmap_reg_range(0x5800, 0x580f), + regmap_reg_range(0x5820, 0x5827), + regmap_reg_range(0x5830, 0x5837), + regmap_reg_range(0x5840, 0x584b), + regmap_reg_range(0x5900, 0x5907), + regmap_reg_range(0x5914, 0x591b), + regmap_reg_range(0x5920, 0x5920), + regmap_reg_range(0x5923, 0x5927), + regmap_reg_range(0x5a00, 0x5a03), + regmap_reg_range(0x5a04, 0x5a07), + regmap_reg_range(0x5b00, 0x5b01), + regmap_reg_range(0x5b04, 0x5b04), + regmap_reg_range(0x5c00, 0x5c05), + regmap_reg_range(0x5c08, 0x5c1b), + + /* port 6 */ + regmap_reg_range(0x6000, 0x6001), + regmap_reg_range(0x6013, 0x6013), + regmap_reg_range(0x6017, 0x6017), + regmap_reg_range(0x601b, 0x601b), + regmap_reg_range(0x601f, 0x6020), + regmap_reg_range(0x6030, 0x6030), + regmap_reg_range(0x6300, 0x6301), + regmap_reg_range(0x6400, 0x6401), + regmap_reg_range(0x6403, 0x6403), + regmap_reg_range(0x6410, 0x6417), + regmap_reg_range(0x6420, 0x6423), + regmap_reg_range(0x6500, 0x6507), + regmap_reg_range(0x6600, 0x6613), + regmap_reg_range(0x6800, 0x680f), + regmap_reg_range(0x6820, 0x6827), + regmap_reg_range(0x6830, 0x6837), + regmap_reg_range(0x6840, 0x684b), + regmap_reg_range(0x6900, 0x6907), + regmap_reg_range(0x6914, 0x691b), + regmap_reg_range(0x6920, 0x6920), + regmap_reg_range(0x6923, 0x6927), + regmap_reg_range(0x6a00, 0x6a03), + regmap_reg_range(0x6a04, 0x6a07), + regmap_reg_range(0x6b00, 0x6b01), + regmap_reg_range(0x6b04, 0x6b04), + regmap_reg_range(0x6c00, 0x6c05), + regmap_reg_range(0x6c08, 0x6c1b), + + /* port 7 */ + regmap_reg_range(0x7000, 0x7001), + regmap_reg_range(0x7013, 0x7013), + regmap_reg_range(0x7017, 0x7017), + regmap_reg_range(0x701b, 0x701b), + regmap_reg_range(0x701f, 0x7020), + regmap_reg_range(0x7030, 0x7030), + regmap_reg_range(0x7200, 0x7203), + regmap_reg_range(0x7206, 0x7207), + regmap_reg_range(0x7300, 0x7301), + regmap_reg_range(0x7400, 0x7401), + regmap_reg_range(0x7403, 0x7403), + regmap_reg_range(0x7410, 0x7417), + regmap_reg_range(0x7420, 0x7423), + regmap_reg_range(0x7500, 0x7507), + regmap_reg_range(0x7600, 0x7613), + regmap_reg_range(0x7800, 0x780f), + regmap_reg_range(0x7820, 0x7827), + regmap_reg_range(0x7830, 0x7837), + regmap_reg_range(0x7840, 0x784b), + regmap_reg_range(0x7900, 0x7907), + regmap_reg_range(0x7914, 0x791b), + regmap_reg_range(0x7920, 0x7920), + regmap_reg_range(0x7923, 0x7927), + regmap_reg_range(0x7a00, 0x7a03), + regmap_reg_range(0x7a04, 0x7a07), + regmap_reg_range(0x7b00, 0x7b01), + regmap_reg_range(0x7b04, 0x7b04), + regmap_reg_range(0x7c00, 0x7c05), + regmap_reg_range(0x7c08, 0x7c1b), +}; + +static const struct regmap_access_table ksz9477_register_set = { + .yes_ranges = ksz9477_valid_regs, + .n_yes_ranges = ARRAY_SIZE(ksz9477_valid_regs), +}; + +static const struct regmap_range ksz9896_valid_regs[] = { + regmap_reg_range(0x0000, 0x0003), + regmap_reg_range(0x0006, 0x0006), + regmap_reg_range(0x0010, 0x001f), + regmap_reg_range(0x0100, 0x0100), + regmap_reg_range(0x0103, 0x0107), + regmap_reg_range(0x010d, 0x010d), + regmap_reg_range(0x0110, 0x0113), + regmap_reg_range(0x0120, 0x0127), + regmap_reg_range(0x0201, 0x0201), + regmap_reg_range(0x0210, 0x0213), + regmap_reg_range(0x0300, 0x0300), + regmap_reg_range(0x0302, 0x030b), + regmap_reg_range(0x0310, 0x031b), + regmap_reg_range(0x0320, 0x032b), + regmap_reg_range(0x0330, 0x0336), + regmap_reg_range(0x0338, 0x033b), + regmap_reg_range(0x033e, 0x033e), + regmap_reg_range(0x0340, 0x035f), + regmap_reg_range(0x0370, 0x0370), + regmap_reg_range(0x0378, 0x0378), + regmap_reg_range(0x037c, 0x037d), + regmap_reg_range(0x0390, 0x0393), + regmap_reg_range(0x0400, 0x040e), + regmap_reg_range(0x0410, 0x042f), + + /* port 1 */ + regmap_reg_range(0x1000, 0x1001), + regmap_reg_range(0x1013, 0x1013), + regmap_reg_range(0x1017, 0x1017), + regmap_reg_range(0x101b, 0x101b), + regmap_reg_range(0x101f, 0x1020), + regmap_reg_range(0x1030, 0x1030), + regmap_reg_range(0x1100, 0x1115), + regmap_reg_range(0x111a, 0x111f), + regmap_reg_range(0x1122, 0x1127), + regmap_reg_range(0x112a, 0x112b), + regmap_reg_range(0x1136, 0x1139), + regmap_reg_range(0x113e, 0x113f), + regmap_reg_range(0x1400, 0x1401), + regmap_reg_range(0x1403, 0x1403), + regmap_reg_range(0x1410, 0x1417), + regmap_reg_range(0x1420, 0x1423), + regmap_reg_range(0x1500, 0x1507), + regmap_reg_range(0x1600, 0x1612), + regmap_reg_range(0x1800, 0x180f), + regmap_reg_range(0x1820, 0x1827), + regmap_reg_range(0x1830, 0x1837), + regmap_reg_range(0x1840, 0x184b), + regmap_reg_range(0x1900, 0x1907), + regmap_reg_range(0x1914, 0x1915), + regmap_reg_range(0x1a00, 0x1a03), + regmap_reg_range(0x1a04, 0x1a07), + regmap_reg_range(0x1b00, 0x1b01), + regmap_reg_range(0x1b04, 0x1b04), + + /* port 2 */ + regmap_reg_range(0x2000, 0x2001), + regmap_reg_range(0x2013, 0x2013), + regmap_reg_range(0x2017, 0x2017), + regmap_reg_range(0x201b, 0x201b), + regmap_reg_range(0x201f, 0x2020), + regmap_reg_range(0x2030, 0x2030), + regmap_reg_range(0x2100, 0x2115), + regmap_reg_range(0x211a, 0x211f), + regmap_reg_range(0x2122, 0x2127), + regmap_reg_range(0x212a, 0x212b), + regmap_reg_range(0x2136, 0x2139), + regmap_reg_range(0x213e, 0x213f), + regmap_reg_range(0x2400, 0x2401), + regmap_reg_range(0x2403, 0x2403), + regmap_reg_range(0x2410, 0x2417), + regmap_reg_range(0x2420, 0x2423), + regmap_reg_range(0x2500, 0x2507), + regmap_reg_range(0x2600, 0x2612), + regmap_reg_range(0x2800, 0x280f), + regmap_reg_range(0x2820, 0x2827), + regmap_reg_range(0x2830, 0x2837), + regmap_reg_range(0x2840, 0x284b), + regmap_reg_range(0x2900, 0x2907), + regmap_reg_range(0x2914, 0x2915), + regmap_reg_range(0x2a00, 0x2a03), + regmap_reg_range(0x2a04, 0x2a07), + regmap_reg_range(0x2b00, 0x2b01), + regmap_reg_range(0x2b04, 0x2b04), + + /* port 3 */ + regmap_reg_range(0x3000, 0x3001), + regmap_reg_range(0x3013, 0x3013), + regmap_reg_range(0x3017, 0x3017), + regmap_reg_range(0x301b, 0x301b), + regmap_reg_range(0x301f, 0x3020), + regmap_reg_range(0x3030, 0x3030), + regmap_reg_range(0x3100, 0x3115), + regmap_reg_range(0x311a, 0x311f), + regmap_reg_range(0x3122, 0x3127), + regmap_reg_range(0x312a, 0x312b), + regmap_reg_range(0x3136, 0x3139), + regmap_reg_range(0x313e, 0x313f), + regmap_reg_range(0x3400, 0x3401), + regmap_reg_range(0x3403, 0x3403), + regmap_reg_range(0x3410, 0x3417), + regmap_reg_range(0x3420, 0x3423), + regmap_reg_range(0x3500, 0x3507), + regmap_reg_range(0x3600, 0x3612), + regmap_reg_range(0x3800, 0x380f), + regmap_reg_range(0x3820, 0x3827), + regmap_reg_range(0x3830, 0x3837), + regmap_reg_range(0x3840, 0x384b), + regmap_reg_range(0x3900, 0x3907), + regmap_reg_range(0x3914, 0x3915), + regmap_reg_range(0x3a00, 0x3a03), + regmap_reg_range(0x3a04, 0x3a07), + regmap_reg_range(0x3b00, 0x3b01), + regmap_reg_range(0x3b04, 0x3b04), + + /* port 4 */ + regmap_reg_range(0x4000, 0x4001), + regmap_reg_range(0x4013, 0x4013), + regmap_reg_range(0x4017, 0x4017), + regmap_reg_range(0x401b, 0x401b), + regmap_reg_range(0x401f, 0x4020), + regmap_reg_range(0x4030, 0x4030), + regmap_reg_range(0x4100, 0x4115), + regmap_reg_range(0x411a, 0x411f), + regmap_reg_range(0x4122, 0x4127), + regmap_reg_range(0x412a, 0x412b), + regmap_reg_range(0x4136, 0x4139), + regmap_reg_range(0x413e, 0x413f), + regmap_reg_range(0x4400, 0x4401), + regmap_reg_range(0x4403, 0x4403), + regmap_reg_range(0x4410, 0x4417), + regmap_reg_range(0x4420, 0x4423), + regmap_reg_range(0x4500, 0x4507), + regmap_reg_range(0x4600, 0x4612), + regmap_reg_range(0x4800, 0x480f), + regmap_reg_range(0x4820, 0x4827), + regmap_reg_range(0x4830, 0x4837), + regmap_reg_range(0x4840, 0x484b), + regmap_reg_range(0x4900, 0x4907), + regmap_reg_range(0x4914, 0x4915), + regmap_reg_range(0x4a00, 0x4a03), + regmap_reg_range(0x4a04, 0x4a07), + regmap_reg_range(0x4b00, 0x4b01), + regmap_reg_range(0x4b04, 0x4b04), + + /* port 5 */ + regmap_reg_range(0x5000, 0x5001), + regmap_reg_range(0x5013, 0x5013), + regmap_reg_range(0x5017, 0x5017), + regmap_reg_range(0x501b, 0x501b), + regmap_reg_range(0x501f, 0x5020), + regmap_reg_range(0x5030, 0x5030), + regmap_reg_range(0x5100, 0x5115), + regmap_reg_range(0x511a, 0x511f), + regmap_reg_range(0x5122, 0x5127), + regmap_reg_range(0x512a, 0x512b), + regmap_reg_range(0x5136, 0x5139), + regmap_reg_range(0x513e, 0x513f), + regmap_reg_range(0x5400, 0x5401), + regmap_reg_range(0x5403, 0x5403), + regmap_reg_range(0x5410, 0x5417), + regmap_reg_range(0x5420, 0x5423), + regmap_reg_range(0x5500, 0x5507), + regmap_reg_range(0x5600, 0x5612), + regmap_reg_range(0x5800, 0x580f), + regmap_reg_range(0x5820, 0x5827), + regmap_reg_range(0x5830, 0x5837), + regmap_reg_range(0x5840, 0x584b), + regmap_reg_range(0x5900, 0x5907), + regmap_reg_range(0x5914, 0x5915), + regmap_reg_range(0x5a00, 0x5a03), + regmap_reg_range(0x5a04, 0x5a07), + regmap_reg_range(0x5b00, 0x5b01), + regmap_reg_range(0x5b04, 0x5b04), + + /* port 6 */ + regmap_reg_range(0x6000, 0x6001), + regmap_reg_range(0x6013, 0x6013), + regmap_reg_range(0x6017, 0x6017), + regmap_reg_range(0x601b, 0x601b), + regmap_reg_range(0x601f, 0x6020), + regmap_reg_range(0x6030, 0x6030), + regmap_reg_range(0x6100, 0x6115), + regmap_reg_range(0x611a, 0x611f), + regmap_reg_range(0x6122, 0x6127), + regmap_reg_range(0x612a, 0x612b), + regmap_reg_range(0x6136, 0x6139), + regmap_reg_range(0x613e, 0x613f), + regmap_reg_range(0x6300, 0x6301), + regmap_reg_range(0x6400, 0x6401), + regmap_reg_range(0x6403, 0x6403), + regmap_reg_range(0x6410, 0x6417), + regmap_reg_range(0x6420, 0x6423), + regmap_reg_range(0x6500, 0x6507), + regmap_reg_range(0x6600, 0x6612), + regmap_reg_range(0x6800, 0x680f), + regmap_reg_range(0x6820, 0x6827), + regmap_reg_range(0x6830, 0x6837), + regmap_reg_range(0x6840, 0x684b), + regmap_reg_range(0x6900, 0x6907), + regmap_reg_range(0x6914, 0x6915), + regmap_reg_range(0x6a00, 0x6a03), + regmap_reg_range(0x6a04, 0x6a07), + regmap_reg_range(0x6b00, 0x6b01), + regmap_reg_range(0x6b04, 0x6b04), +}; + +static const struct regmap_access_table ksz9896_register_set = { + .yes_ranges = ksz9896_valid_regs, + .n_yes_ranges = ARRAY_SIZE(ksz9896_valid_regs), +}; + const struct ksz_chip_data ksz_switch_chips[] = { + [KSZ8563] = { + .chip_id = KSZ8563_CHIP_ID, + .dev_name = "KSZ8563", + .num_vlans = 4096, + .num_alus = 4096, + .num_statics = 16, + .cpu_ports = 0x07, /* can be configured as cpu port */ + .port_cnt = 3, /* total port count */ + .ops = &ksz9477_dev_ops, + .mib_names = ksz9477_mib_names, + .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), + .reg_mib_cnt = MIB_COUNTER_NUM, + .regs = ksz9477_regs, + .masks = ksz9477_masks, + .shifts = ksz9477_shifts, + .xmii_ctrl0 = ksz9477_xmii_ctrl0, + .xmii_ctrl1 = ksz8795_xmii_ctrl1, /* Same as ksz8795 */ + .supports_mii = {false, false, true}, + .supports_rmii = {false, false, true}, + .supports_rgmii = {false, false, true}, + .internal_phy = {true, true, false}, + .gbit_capable = {false, false, true}, + .wr_table = &ksz8563_register_set, + .rd_table = &ksz8563_register_set, + }, + [KSZ8795] = { .chip_id = KSZ8795_CHIP_ID, .dev_name = "KSZ8795", @@ -527,6 +1171,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 16, .cpu_ports = 0x7F, /* can be configured as cpu port */ .port_cnt = 7, /* total physical port count */ + .port_nirqs = 4, .ops = &ksz9477_dev_ops, .phy_errata_9477 = true, .mib_names = ksz9477_mib_names, @@ -545,6 +1190,41 @@ const struct ksz_chip_data ksz_switch_chips[] = { false, true, false}, .internal_phy = {true, true, true, true, true, false, false}, + .gbit_capable = {true, true, true, true, true, true, true}, + .wr_table = &ksz9477_register_set, + .rd_table = &ksz9477_register_set, + }, + + [KSZ9896] = { + .chip_id = KSZ9896_CHIP_ID, + .dev_name = "KSZ9896", + .num_vlans = 4096, + .num_alus = 4096, + .num_statics = 16, + .cpu_ports = 0x3F, /* can be configured as cpu port */ + .port_cnt = 6, /* total physical port count */ + .port_nirqs = 2, + .ops = &ksz9477_dev_ops, + .phy_errata_9477 = true, + .mib_names = ksz9477_mib_names, + .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), + .reg_mib_cnt = MIB_COUNTER_NUM, + .regs = ksz9477_regs, + .masks = ksz9477_masks, + .shifts = ksz9477_shifts, + .xmii_ctrl0 = ksz9477_xmii_ctrl0, + .xmii_ctrl1 = ksz9477_xmii_ctrl1, + .supports_mii = {false, false, false, false, + false, true}, + .supports_rmii = {false, false, false, false, + false, true}, + .supports_rgmii = {false, false, false, false, + false, true}, + .internal_phy = {true, true, true, true, + true, false}, + .gbit_capable = {true, true, true, true, true, true}, + .wr_table = &ksz9896_register_set, + .rd_table = &ksz9896_register_set, }, [KSZ9897] = { @@ -555,6 +1235,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 16, .cpu_ports = 0x7F, /* can be configured as cpu port */ .port_cnt = 7, /* total physical port count */ + .port_nirqs = 2, .ops = &ksz9477_dev_ops, .phy_errata_9477 = true, .mib_names = ksz9477_mib_names, @@ -573,6 +1254,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { false, true, true}, .internal_phy = {true, true, true, true, true, false, false}, + .gbit_capable = {true, true, true, true, true, true, true}, }, [KSZ9893] = { @@ -583,6 +1265,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 16, .cpu_ports = 0x07, /* can be configured as cpu port */ .port_cnt = 3, /* total port count */ + .port_nirqs = 2, .ops = &ksz9477_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -596,6 +1279,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .supports_rmii = {false, false, true}, .supports_rgmii = {false, false, true}, .internal_phy = {true, true, false}, + .gbit_capable = {true, true, true}, }, [KSZ9567] = { @@ -606,6 +1290,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 16, .cpu_ports = 0x7F, /* can be configured as cpu port */ .port_cnt = 7, /* total physical port count */ + .port_nirqs = 3, .ops = &ksz9477_dev_ops, .phy_errata_9477 = true, .mib_names = ksz9477_mib_names, @@ -624,6 +1309,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { false, true, true}, .internal_phy = {true, true, true, true, true, false, false}, + .gbit_capable = {true, true, true, true, true, true, true}, }, [LAN9370] = { @@ -634,6 +1320,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 256, .cpu_ports = 0x10, /* can be configured as cpu port */ .port_cnt = 5, /* total physical port count */ + .port_nirqs = 6, .ops = &lan937x_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -657,6 +1344,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 256, .cpu_ports = 0x30, /* can be configured as cpu port */ .port_cnt = 6, /* total physical port count */ + .port_nirqs = 6, .ops = &lan937x_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -680,6 +1368,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 256, .cpu_ports = 0x30, /* can be configured as cpu port */ .port_cnt = 8, /* total physical port count */ + .port_nirqs = 6, .ops = &lan937x_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -707,6 +1396,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 256, .cpu_ports = 0x38, /* can be configured as cpu port */ .port_cnt = 5, /* total physical port count */ + .port_nirqs = 6, .ops = &lan937x_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -734,6 +1424,7 @@ const struct ksz_chip_data ksz_switch_chips[] = { .num_statics = 256, .cpu_ports = 0x30, /* can be configured as cpu port */ .port_cnt = 8, /* total physical port count */ + .port_nirqs = 6, .ops = &lan937x_dev_ops, .mib_names = ksz9477_mib_names, .mib_cnt = ARRAY_SIZE(ksz9477_mib_names), @@ -803,9 +1494,15 @@ static void ksz_phylink_get_caps(struct dsa_switch *ds, int port, if (dev->info->supports_rgmii[port]) phy_interface_set_rgmii(config->supported_interfaces); - if (dev->info->internal_phy[port]) + if (dev->info->internal_phy[port]) { __set_bit(PHY_INTERFACE_MODE_INTERNAL, config->supported_interfaces); + /* Compatibility for phylib's default interface type when the + * phy-mode property is absent + */ + __set_bit(PHY_INTERFACE_MODE_GMII, + config->supported_interfaces); + } if (dev->dev_ops->get_caps) dev->dev_ops->get_caps(dev, port, config); @@ -959,9 +1656,281 @@ static void ksz_update_port_member(struct ksz_device *dev, int port) dev->dev_ops->cfg_port_member(dev, port, port_member | cpu_port); } +static int ksz_sw_mdio_read(struct mii_bus *bus, int addr, int regnum) +{ + struct ksz_device *dev = bus->priv; + u16 val; + int ret; + + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + + ret = dev->dev_ops->r_phy(dev, addr, regnum, &val); + if (ret < 0) + return ret; + + return val; +} + +static int ksz_sw_mdio_write(struct mii_bus *bus, int addr, int regnum, + u16 val) +{ + struct ksz_device *dev = bus->priv; + + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + + return dev->dev_ops->w_phy(dev, addr, regnum, val); +} + +static int ksz_irq_phy_setup(struct ksz_device *dev) +{ + struct dsa_switch *ds = dev->ds; + int phy; + int irq; + int ret; + + for (phy = 0; phy < KSZ_MAX_NUM_PORTS; phy++) { + if (BIT(phy) & ds->phys_mii_mask) { + irq = irq_find_mapping(dev->ports[phy].pirq.domain, + PORT_SRC_PHY_INT); + if (irq < 0) { + ret = irq; + goto out; + } + ds->slave_mii_bus->irq[phy] = irq; + } + } + return 0; +out: + while (phy--) + if (BIT(phy) & ds->phys_mii_mask) + irq_dispose_mapping(ds->slave_mii_bus->irq[phy]); + + return ret; +} + +static void ksz_irq_phy_free(struct ksz_device *dev) +{ + struct dsa_switch *ds = dev->ds; + int phy; + + for (phy = 0; phy < KSZ_MAX_NUM_PORTS; phy++) + if (BIT(phy) & ds->phys_mii_mask) + irq_dispose_mapping(ds->slave_mii_bus->irq[phy]); +} + +static int ksz_mdio_register(struct ksz_device *dev) +{ + struct dsa_switch *ds = dev->ds; + struct device_node *mdio_np; + struct mii_bus *bus; + int ret; + + mdio_np = of_get_child_by_name(dev->dev->of_node, "mdio"); + if (!mdio_np) + return 0; + + bus = devm_mdiobus_alloc(ds->dev); + if (!bus) { + of_node_put(mdio_np); + return -ENOMEM; + } + + bus->priv = dev; + bus->read = ksz_sw_mdio_read; + bus->write = ksz_sw_mdio_write; + bus->name = "ksz slave smi"; + snprintf(bus->id, MII_BUS_ID_SIZE, "SMI-%d", ds->index); + bus->parent = ds->dev; + bus->phy_mask = ~ds->phys_mii_mask; + + ds->slave_mii_bus = bus; + + if (dev->irq > 0) { + ret = ksz_irq_phy_setup(dev); + if (ret) { + of_node_put(mdio_np); + return ret; + } + } + + ret = devm_of_mdiobus_register(ds->dev, bus, mdio_np); + if (ret) { + dev_err(ds->dev, "unable to register MDIO bus %s\n", + bus->id); + if (dev->irq > 0) + ksz_irq_phy_free(dev); + } + + of_node_put(mdio_np); + + return ret; +} + +static void ksz_irq_mask(struct irq_data *d) +{ + struct ksz_irq *kirq = irq_data_get_irq_chip_data(d); + + kirq->masked |= BIT(d->hwirq); +} + +static void ksz_irq_unmask(struct irq_data *d) +{ + struct ksz_irq *kirq = irq_data_get_irq_chip_data(d); + + kirq->masked &= ~BIT(d->hwirq); +} + +static void ksz_irq_bus_lock(struct irq_data *d) +{ + struct ksz_irq *kirq = irq_data_get_irq_chip_data(d); + + mutex_lock(&kirq->dev->lock_irq); +} + +static void ksz_irq_bus_sync_unlock(struct irq_data *d) +{ + struct ksz_irq *kirq = irq_data_get_irq_chip_data(d); + struct ksz_device *dev = kirq->dev; + int ret; + + ret = ksz_write32(dev, kirq->reg_mask, kirq->masked); + if (ret) + dev_err(dev->dev, "failed to change IRQ mask\n"); + + mutex_unlock(&dev->lock_irq); +} + +static const struct irq_chip ksz_irq_chip = { + .name = "ksz-irq", + .irq_mask = ksz_irq_mask, + .irq_unmask = ksz_irq_unmask, + .irq_bus_lock = ksz_irq_bus_lock, + .irq_bus_sync_unlock = ksz_irq_bus_sync_unlock, +}; + +static int ksz_irq_domain_map(struct irq_domain *d, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_data(irq, d->host_data); + irq_set_chip_and_handler(irq, &ksz_irq_chip, handle_level_irq); + irq_set_noprobe(irq); + + return 0; +} + +static const struct irq_domain_ops ksz_irq_domain_ops = { + .map = ksz_irq_domain_map, + .xlate = irq_domain_xlate_twocell, +}; + +static void ksz_irq_free(struct ksz_irq *kirq) +{ + int irq, virq; + + free_irq(kirq->irq_num, kirq); + + for (irq = 0; irq < kirq->nirqs; irq++) { + virq = irq_find_mapping(kirq->domain, irq); + irq_dispose_mapping(virq); + } + + irq_domain_remove(kirq->domain); +} + +static irqreturn_t ksz_irq_thread_fn(int irq, void *dev_id) +{ + struct ksz_irq *kirq = dev_id; + unsigned int nhandled = 0; + struct ksz_device *dev; + unsigned int sub_irq; + u8 data; + int ret; + u8 n; + + dev = kirq->dev; + + /* Read interrupt status register */ + ret = ksz_read8(dev, kirq->reg_status, &data); + if (ret) + goto out; + + for (n = 0; n < kirq->nirqs; ++n) { + if (data & BIT(n)) { + sub_irq = irq_find_mapping(kirq->domain, n); + handle_nested_irq(sub_irq); + ++nhandled; + } + } +out: + return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE); +} + +static int ksz_irq_common_setup(struct ksz_device *dev, struct ksz_irq *kirq) +{ + int ret, n; + + kirq->dev = dev; + kirq->masked = ~0; + + kirq->domain = irq_domain_add_simple(dev->dev->of_node, kirq->nirqs, 0, + &ksz_irq_domain_ops, kirq); + if (!kirq->domain) + return -ENOMEM; + + for (n = 0; n < kirq->nirqs; n++) + irq_create_mapping(kirq->domain, n); + + ret = request_threaded_irq(kirq->irq_num, NULL, ksz_irq_thread_fn, + IRQF_ONESHOT | IRQF_TRIGGER_FALLING, + kirq->name, kirq); + if (ret) + goto out; + + return 0; + +out: + ksz_irq_free(kirq); + + return ret; +} + +static int ksz_girq_setup(struct ksz_device *dev) +{ + struct ksz_irq *girq = &dev->girq; + + girq->nirqs = dev->info->port_cnt; + girq->reg_mask = REG_SW_PORT_INT_MASK__1; + girq->reg_status = REG_SW_PORT_INT_STATUS__1; + snprintf(girq->name, sizeof(girq->name), "global_port_irq"); + + girq->irq_num = dev->irq; + + return ksz_irq_common_setup(dev, girq); +} + +static int ksz_pirq_setup(struct ksz_device *dev, u8 p) +{ + struct ksz_irq *pirq = &dev->ports[p].pirq; + + pirq->nirqs = dev->info->port_nirqs; + pirq->reg_mask = dev->dev_ops->get_port_addr(p, REG_PORT_INT_MASK); + pirq->reg_status = dev->dev_ops->get_port_addr(p, REG_PORT_INT_STATUS); + snprintf(pirq->name, sizeof(pirq->name), "port_irq-%d", p); + + pirq->irq_num = irq_find_mapping(dev->girq.domain, p); + if (pirq->irq_num < 0) + return pirq->irq_num; + + return ksz_irq_common_setup(dev, pirq); +} + static int ksz_setup(struct dsa_switch *ds) { struct ksz_device *dev = ds->priv; + struct dsa_port *dp; + struct ksz_port *p; const u16 *regs; int ret; @@ -1001,11 +1970,63 @@ static int ksz_setup(struct dsa_switch *ds) return ret; } + /* Start with learning disabled on standalone user ports, and enabled + * on the CPU port. In lack of other finer mechanisms, learning on the + * CPU port will avoid flooding bridge local addresses on the network + * in some cases. + */ + p = &dev->ports[dev->cpu_port]; + p->learning = true; + + if (dev->irq > 0) { + ret = ksz_girq_setup(dev); + if (ret) + return ret; + + dsa_switch_for_each_user_port(dp, dev->ds) { + ret = ksz_pirq_setup(dev, dp->index); + if (ret) + goto out_girq; + } + } + + ret = ksz_mdio_register(dev); + if (ret < 0) { + dev_err(dev->dev, "failed to register the mdio"); + goto out_pirq; + } + /* start switch */ regmap_update_bits(dev->regmap[0], regs[S_START_CTRL], SW_START, SW_START); return 0; + +out_pirq: + if (dev->irq > 0) + dsa_switch_for_each_user_port(dp, dev->ds) + ksz_irq_free(&dev->ports[dp->index].pirq); +out_girq: + if (dev->irq > 0) + ksz_irq_free(&dev->girq); + + return ret; +} + +static void ksz_teardown(struct dsa_switch *ds) +{ + struct ksz_device *dev = ds->priv; + struct dsa_port *dp; + + if (dev->irq > 0) { + dsa_switch_for_each_user_port(dp, dev->ds) + ksz_irq_free(&dev->ports[dp->index].pirq); + + ksz_irq_free(&dev->girq); + } + + if (dev->dev_ops->teardown) + dev->dev_ops->teardown(ds); } static void port_r_cnt(struct ksz_device *dev, int port) @@ -1089,8 +2110,11 @@ static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg) { struct ksz_device *dev = ds->priv; u16 val = 0xffff; + int ret; - dev->dev_ops->r_phy(dev, addr, reg, &val); + ret = dev->dev_ops->r_phy(dev, addr, reg, &val); + if (ret) + return ret; return val; } @@ -1098,8 +2122,11 @@ static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg) static int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val) { struct ksz_device *dev = ds->priv; + int ret; - dev->dev_ops->w_phy(dev, addr, reg, val); + ret = dev->dev_ops->w_phy(dev, addr, reg, val); + if (ret) + return ret; return 0; } @@ -1188,6 +2215,16 @@ static void ksz_port_fast_age(struct dsa_switch *ds, int port) dev->dev_ops->flush_dyn_mac_table(dev, port); } +static int ksz_set_ageing_time(struct dsa_switch *ds, unsigned int msecs) +{ + struct ksz_device *dev = ds->priv; + + if (!dev->dev_ops->set_ageing_time) + return -EOPNOTSUPP; + + return dev->dev_ops->set_ageing_time(dev, msecs); +} + static int ksz_port_fdb_add(struct dsa_switch *ds, int port, const unsigned char *addr, u16 vid, struct dsa_db db) @@ -1277,6 +2314,8 @@ void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) ksz_pread8(dev, port, regs[P_STP_CTRL], &data); data &= ~(PORT_TX_ENABLE | PORT_RX_ENABLE | PORT_LEARN_DISABLE); + p = &dev->ports[port]; + switch (state) { case BR_STATE_DISABLED: data |= PORT_LEARN_DISABLE; @@ -1286,9 +2325,13 @@ void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) break; case BR_STATE_LEARNING: data |= PORT_RX_ENABLE; + if (!p->learning) + data |= PORT_LEARN_DISABLE; break; case BR_STATE_FORWARDING: data |= (PORT_TX_ENABLE | PORT_RX_ENABLE); + if (!p->learning) + data |= PORT_LEARN_DISABLE; break; case BR_STATE_BLOCKING: data |= PORT_LEARN_DISABLE; @@ -1300,12 +2343,38 @@ void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) ksz_pwrite8(dev, port, regs[P_STP_CTRL], data); - p = &dev->ports[port]; p->stp_state = state; ksz_update_port_member(dev, port); } +static int ksz_port_pre_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) +{ + if (flags.mask & ~BR_LEARNING) + return -EINVAL; + + return 0; +} + +static int ksz_port_bridge_flags(struct dsa_switch *ds, int port, + struct switchdev_brport_flags flags, + struct netlink_ext_ack *extack) +{ + struct ksz_device *dev = ds->priv; + struct ksz_port *p = &dev->ports[port]; + + if (flags.mask & BR_LEARNING) { + p->learning = !!(flags.val & BR_LEARNING); + + /* Make the change take effect immediately */ + ksz_port_stp_state_set(ds, port, p->stp_state); + } + + return 0; +} + static enum dsa_tag_protocol ksz_get_tag_protocol(struct dsa_switch *ds, int port, enum dsa_tag_protocol mp) @@ -1319,10 +2388,12 @@ static enum dsa_tag_protocol ksz_get_tag_protocol(struct dsa_switch *ds, proto = DSA_TAG_PROTO_KSZ8795; if (dev->chip_id == KSZ8830_CHIP_ID || + dev->chip_id == KSZ8563_CHIP_ID || dev->chip_id == KSZ9893_CHIP_ID) proto = DSA_TAG_PROTO_KSZ9893; if (dev->chip_id == KSZ9477_CHIP_ID || + dev->chip_id == KSZ9896_CHIP_ID || dev->chip_id == KSZ9897_CHIP_ID || dev->chip_id == KSZ9567_CHIP_ID) proto = DSA_TAG_PROTO_KSZ9477; @@ -1437,7 +2508,8 @@ static void ksz_set_xmii(struct ksz_device *dev, int port, case PHY_INTERFACE_MODE_RGMII_RXID: data8 |= bitval[P_RGMII_SEL]; /* On KSZ9893, disable RGMII in-band status support */ - if (dev->features & IS_9893) + if (dev->chip_id == KSZ9893_CHIP_ID || + dev->chip_id == KSZ8563_CHIP_ID) data8 &= ~P_MII_MAC_MODE; break; default: @@ -1609,13 +2681,13 @@ static void ksz_duplex_flowctrl(struct ksz_device *dev, int port, int duplex, ksz_prmw8(dev, port, regs[P_XMII_CTRL_0], mask, val); } -static void ksz_phylink_mac_link_up(struct dsa_switch *ds, int port, - unsigned int mode, - phy_interface_t interface, - struct phy_device *phydev, int speed, - int duplex, bool tx_pause, bool rx_pause) +static void ksz9477_phylink_mac_link_up(struct ksz_device *dev, int port, + unsigned int mode, + phy_interface_t interface, + struct phy_device *phydev, int speed, + int duplex, bool tx_pause, + bool rx_pause) { - struct ksz_device *dev = ds->priv; struct ksz_port *p; p = &dev->ports[port]; @@ -1629,6 +2701,15 @@ static void ksz_phylink_mac_link_up(struct dsa_switch *ds, int port, ksz_port_set_xmii_speed(dev, port, speed); ksz_duplex_flowctrl(dev, port, duplex, tx_pause, rx_pause); +} + +static void ksz_phylink_mac_link_up(struct dsa_switch *ds, int port, + unsigned int mode, + phy_interface_t interface, + struct phy_device *phydev, int speed, + int duplex, bool tx_pause, bool rx_pause) +{ + struct ksz_device *dev = ds->priv; if (dev->dev_ops->phylink_mac_link_up) dev->dev_ops->phylink_mac_link_up(dev, port, mode, interface, @@ -1638,7 +2719,7 @@ static void ksz_phylink_mac_link_up(struct dsa_switch *ds, int port, static int ksz_switch_detect(struct ksz_device *dev) { - u8 id1, id2; + u8 id1, id2, id4; u16 id16; u32 id32; int ret; @@ -1683,8 +2764,8 @@ static int ksz_switch_detect(struct ksz_device *dev) switch (id32) { case KSZ9477_CHIP_ID: + case KSZ9896_CHIP_ID: case KSZ9897_CHIP_ID: - case KSZ9893_CHIP_ID: case KSZ9567_CHIP_ID: case LAN9370_CHIP_ID: case LAN9371_CHIP_ID: @@ -1692,6 +2773,18 @@ static int ksz_switch_detect(struct ksz_device *dev) case LAN9373_CHIP_ID: case LAN9374_CHIP_ID: dev->chip_id = id32; + break; + case KSZ9893_CHIP_ID: + ret = ksz_read8(dev, REG_CHIP_ID4, + &id4); + if (ret) + return ret; + + if (id4 == SKU_ID_KSZ8563) + dev->chip_id = KSZ8563_CHIP_ID; + else + dev->chip_id = KSZ9893_CHIP_ID; + break; default: dev_err(dev->dev, @@ -1706,6 +2799,7 @@ static const struct dsa_switch_ops ksz_switch_ops = { .get_tag_protocol = ksz_get_tag_protocol, .get_phy_flags = ksz_get_phy_flags, .setup = ksz_setup, + .teardown = ksz_teardown, .phy_read = ksz_phy_read16, .phy_write = ksz_phy_write16, .phylink_get_caps = ksz_phylink_get_caps, @@ -1713,12 +2807,15 @@ static const struct dsa_switch_ops ksz_switch_ops = { .phylink_mac_link_up = ksz_phylink_mac_link_up, .phylink_mac_link_down = ksz_mac_link_down, .port_enable = ksz_enable_port, + .set_ageing_time = ksz_set_ageing_time, .get_strings = ksz_get_strings, .get_ethtool_stats = ksz_get_ethtool_stats, .get_sset_count = ksz_sset_count, .port_bridge_join = ksz_port_bridge_join, .port_bridge_leave = ksz_port_bridge_leave, .port_stp_state_set = ksz_port_stp_state_set, + .port_pre_bridge_flags = ksz_port_pre_bridge_flags, + .port_bridge_flags = ksz_port_bridge_flags, .port_fast_age = ksz_port_fast_age, .port_vlan_filtering = ksz_port_vlan_filtering, .port_vlan_add = ksz_port_vlan_add, @@ -1868,6 +2965,9 @@ int ksz_switch_register(struct ksz_device *dev) GFP_KERNEL); if (!dev->ports[i].mib.counters) return -ENOMEM; + + dev->ports[i].ksz_dev = dev; + dev->ports[i].num = i; } /* set the real number of ports */ diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index 764ada3a0f42acac38f013a7aa87361ea256979f..9cfa179575ce85dc9b92f063e6fd8ec25ac26b0c 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -13,9 +13,12 @@ #include #include #include +#include #define KSZ_MAX_NUM_PORTS 8 +struct ksz_device; + struct vlan_table { u32 table[3]; }; @@ -42,6 +45,7 @@ struct ksz_chip_data { int num_statics; int cpu_ports; int port_cnt; + u8 port_nirqs; const struct ksz_dev_ops *ops; bool phy_errata_9477; bool ksz87xx_eee_link_erratum; @@ -61,17 +65,30 @@ struct ksz_chip_data { bool supports_rmii[KSZ_MAX_NUM_PORTS]; bool supports_rgmii[KSZ_MAX_NUM_PORTS]; bool internal_phy[KSZ_MAX_NUM_PORTS]; + bool gbit_capable[KSZ_MAX_NUM_PORTS]; + const struct regmap_access_table *wr_table; + const struct regmap_access_table *rd_table; +}; + +struct ksz_irq { + u16 masked; + u16 reg_mask; + u16 reg_status; + struct irq_domain *domain; + int nirqs; + int irq_num; + char name[16]; + struct ksz_device *dev; }; struct ksz_port { bool remove_tag; /* Remove Tag flag set, for ksz8795 only */ + bool learning; int stp_state; struct phy_device phydev; u32 on:1; /* port is not disabled by hardware */ - u32 phy:1; /* port has a PHY */ u32 fiber:1; /* port is fiber */ - u32 sgmii:1; /* port is SGMII */ u32 force:1; u32 read:1; /* read MIB counters in background */ u32 freeze:1; /* MIB counter freeze is enabled */ @@ -81,6 +98,9 @@ struct ksz_port { u16 max_frame; u32 rgmii_tx_val; u32 rgmii_rx_val; + struct ksz_device *ksz_dev; + struct ksz_irq pirq; + u8 num; }; struct ksz_device { @@ -98,6 +118,7 @@ struct ksz_device { struct regmap *regmap[3]; void *priv; + int irq; struct gpio_desc *reset_gpio; /* Optional reset GPIO */ @@ -117,17 +138,20 @@ struct ksz_device { unsigned long mib_read_interval; u16 mirror_rx; u16 mirror_tx; - u32 features; /* chip specific features */ u16 port_mask; + struct mutex lock_irq; /* IRQ Access */ + struct ksz_irq girq; }; /* List of supported models */ enum ksz_model { + KSZ8563, KSZ8795, KSZ8794, KSZ8765, KSZ8830, KSZ9477, + KSZ9896, KSZ9897, KSZ9893, KSZ9567, @@ -139,11 +163,13 @@ enum ksz_model { }; enum ksz_chip_id { + KSZ8563_CHIP_ID = 0x8563, KSZ8795_CHIP_ID = 0x8795, KSZ8794_CHIP_ID = 0x8794, KSZ8765_CHIP_ID = 0x8765, KSZ8830_CHIP_ID = 0x8830, KSZ9477_CHIP_ID = 0x00947700, + KSZ9896_CHIP_ID = 0x00989600, KSZ9897_CHIP_ID = 0x00989700, KSZ9893_CHIP_ID = 0x00989300, KSZ9567_CHIP_ID = 0x00956700, @@ -253,13 +279,15 @@ struct alu_struct { struct ksz_dev_ops { int (*setup)(struct dsa_switch *ds); + void (*teardown)(struct dsa_switch *ds); u32 (*get_port_addr)(int port, int offset); void (*cfg_port_member)(struct ksz_device *dev, int port, u8 member); void (*flush_dyn_mac_table)(struct ksz_device *dev, int port); void (*port_cleanup)(struct ksz_device *dev, int port); void (*port_setup)(struct ksz_device *dev, int port, bool cpu_port); - void (*r_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 *val); - void (*w_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 val); + int (*set_ageing_time)(struct ksz_device *dev, unsigned int msecs); + int (*r_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 *val); + int (*w_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 val); void (*r_mib_cnt)(struct ksz_device *dev, int port, u16 addr, u64 *cnt); void (*r_mib_pkt)(struct ksz_device *dev, int port, u16 addr, @@ -329,6 +357,10 @@ static inline int ksz_read8(struct ksz_device *dev, u32 reg, u8 *val) unsigned int value; int ret = regmap_read(dev->regmap[0], reg, &value); + if (ret) + dev_err(dev->dev, "can't read 8bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + *val = value; return ret; } @@ -338,6 +370,10 @@ static inline int ksz_read16(struct ksz_device *dev, u32 reg, u16 *val) unsigned int value; int ret = regmap_read(dev->regmap[1], reg, &value); + if (ret) + dev_err(dev->dev, "can't read 16bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + *val = value; return ret; } @@ -347,6 +383,10 @@ static inline int ksz_read32(struct ksz_device *dev, u32 reg, u32 *val) unsigned int value; int ret = regmap_read(dev->regmap[2], reg, &value); + if (ret) + dev_err(dev->dev, "can't read 32bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + *val = value; return ret; } @@ -357,7 +397,10 @@ static inline int ksz_read64(struct ksz_device *dev, u32 reg, u64 *val) int ret; ret = regmap_bulk_read(dev->regmap[2], reg, value, 2); - if (!ret) + if (ret) + dev_err(dev->dev, "can't read 64bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + else *val = (u64)value[0] << 32 | value[1]; return ret; @@ -365,17 +408,38 @@ static inline int ksz_read64(struct ksz_device *dev, u32 reg, u64 *val) static inline int ksz_write8(struct ksz_device *dev, u32 reg, u8 value) { - return regmap_write(dev->regmap[0], reg, value); + int ret; + + ret = regmap_write(dev->regmap[0], reg, value); + if (ret) + dev_err(dev->dev, "can't write 8bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + + return ret; } static inline int ksz_write16(struct ksz_device *dev, u32 reg, u16 value) { - return regmap_write(dev->regmap[1], reg, value); + int ret; + + ret = regmap_write(dev->regmap[1], reg, value); + if (ret) + dev_err(dev->dev, "can't write 16bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + + return ret; } static inline int ksz_write32(struct ksz_device *dev, u32 reg, u32 value) { - return regmap_write(dev->regmap[2], reg, value); + int ret; + + ret = regmap_write(dev->regmap[2], reg, value); + if (ret) + dev_err(dev->dev, "can't write 32bit reg: 0x%x %pe\n", reg, + ERR_PTR(ret)); + + return ret; } static inline int ksz_write64(struct ksz_device *dev, u32 reg, u64 value) @@ -390,40 +454,42 @@ static inline int ksz_write64(struct ksz_device *dev, u32 reg, u64 value) return regmap_bulk_write(dev->regmap[2], reg, val, 2); } -static inline void ksz_pread8(struct ksz_device *dev, int port, int offset, - u8 *data) +static inline int ksz_pread8(struct ksz_device *dev, int port, int offset, + u8 *data) { - ksz_read8(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_read8(dev, dev->dev_ops->get_port_addr(port, offset), data); } -static inline void ksz_pread16(struct ksz_device *dev, int port, int offset, - u16 *data) +static inline int ksz_pread16(struct ksz_device *dev, int port, int offset, + u16 *data) { - ksz_read16(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_read16(dev, dev->dev_ops->get_port_addr(port, offset), data); } -static inline void ksz_pread32(struct ksz_device *dev, int port, int offset, - u32 *data) +static inline int ksz_pread32(struct ksz_device *dev, int port, int offset, + u32 *data) { - ksz_read32(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_read32(dev, dev->dev_ops->get_port_addr(port, offset), data); } -static inline void ksz_pwrite8(struct ksz_device *dev, int port, int offset, - u8 data) +static inline int ksz_pwrite8(struct ksz_device *dev, int port, int offset, + u8 data) { - ksz_write8(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_write8(dev, dev->dev_ops->get_port_addr(port, offset), data); } -static inline void ksz_pwrite16(struct ksz_device *dev, int port, int offset, - u16 data) +static inline int ksz_pwrite16(struct ksz_device *dev, int port, int offset, + u16 data) { - ksz_write16(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_write16(dev, dev->dev_ops->get_port_addr(port, offset), + data); } -static inline void ksz_pwrite32(struct ksz_device *dev, int port, int offset, - u32 data) +static inline int ksz_pwrite32(struct ksz_device *dev, int port, int offset, + u32 data) { - ksz_write32(dev, dev->dev_ops->get_port_addr(port, offset), data); + return ksz_write32(dev, dev->dev_ops->get_port_addr(port, offset), + data); } static inline void ksz_prmw8(struct ksz_device *dev, int port, int offset, @@ -482,6 +548,10 @@ static inline int is_lan937x(struct ksz_device *dev) #define SW_REV_ID_M GENMASK(7, 4) +/* KSZ9893, KSZ9563, KSZ8563 specific register */ +#define REG_CHIP_ID4 0x0f +#define SKU_ID_KSZ8563 0x3c + /* Driver set switch broadcast storm protection at 10% rate. */ #define BROADCAST_STORM_PROT_RATE 10 @@ -496,10 +566,6 @@ static inline int is_lan937x(struct ksz_device *dev) #define SW_START 0x01 -/* Used with variable features to indicate capabilities. */ -#define GBIT_SUPPORT BIT(0) -#define IS_9893 BIT(2) - /* xMII configuration */ #define P_MII_DUPLEX_M BIT(6) #define P_MII_100MBIT_M BIT(4) @@ -510,6 +576,15 @@ static inline int is_lan937x(struct ksz_device *dev) #define P_MII_MAC_MODE BIT(2) #define P_MII_SEL_M 0x3 +/* Interrupt */ +#define REG_SW_PORT_INT_STATUS__1 0x001B +#define REG_SW_PORT_INT_MASK__1 0x001F + +#define REG_PORT_INT_STATUS 0x001B +#define REG_PORT_INT_MASK 0x001F + +#define PORT_SRC_PHY_INT 1 + /* Regmap tables generation */ #define KSZ_SPI_OP_RD 3 #define KSZ_SPI_OP_WR 2 diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c index 05bd089795f83fc270f28206fd4e4896653da67b..1b6ab891b986e36394e0c603a00ad0c80b973877 100644 --- a/drivers/net/dsa/microchip/ksz_spi.c +++ b/drivers/net/dsa/microchip/ksz_spi.c @@ -66,7 +66,10 @@ static int ksz_spi_probe(struct spi_device *spi) for (i = 0; i < ARRAY_SIZE(ksz8795_regmap_config); i++) { rc = regmap_config[i]; rc.lock_arg = &dev->regmap_mutex; + rc.wr_table = chip->wr_table; + rc.rd_table = chip->rd_table; dev->regmap[i] = devm_regmap_init_spi(spi, &rc); + if (IS_ERR(dev->regmap[i])) { ret = PTR_ERR(dev->regmap[i]); dev_err(&spi->dev, @@ -85,6 +88,8 @@ static int ksz_spi_probe(struct spi_device *spi) if (ret) return ret; + dev->irq = spi->irq; + ret = ksz_switch_register(dev); /* Main DSA driver may not be started yet. */ @@ -102,8 +107,6 @@ static void ksz_spi_remove(struct spi_device *spi) if (dev) ksz_switch_remove(dev); - - spi_set_drvdata(spi, NULL); } static void ksz_spi_shutdown(struct spi_device *spi) @@ -146,6 +149,10 @@ static const struct of_device_id ksz_dt_ids[] = { .compatible = "microchip,ksz9477", .data = &ksz_switch_chips[KSZ9477] }, + { + .compatible = "microchip,ksz9896", + .data = &ksz_switch_chips[KSZ9896] + }, { .compatible = "microchip,ksz9897", .data = &ksz_switch_chips[KSZ9897] @@ -160,7 +167,7 @@ static const struct of_device_id ksz_dt_ids[] = { }, { .compatible = "microchip,ksz8563", - .data = &ksz_switch_chips[KSZ9893] + .data = &ksz_switch_chips[KSZ8563] }, { .compatible = "microchip,ksz9567", @@ -197,6 +204,7 @@ static const struct spi_device_id ksz_spi_ids[] = { { "ksz8863" }, { "ksz8873" }, { "ksz9477" }, + { "ksz9896" }, { "ksz9897" }, { "ksz9893" }, { "ksz9563" }, @@ -226,6 +234,7 @@ static struct spi_driver ksz_spi_driver = { module_spi_driver(ksz_spi_driver); MODULE_ALIAS("spi:ksz9477"); +MODULE_ALIAS("spi:ksz9896"); MODULE_ALIAS("spi:ksz9897"); MODULE_ALIAS("spi:ksz9893"); MODULE_ALIAS("spi:ksz9563"); diff --git a/drivers/net/dsa/microchip/lan937x.h b/drivers/net/dsa/microchip/lan937x.h index 4e0b1dccec270a763c7172f667f35deac4673662..8e9e66d6728d64c3ab93cc4dee8c5716ec6f6ec9 100644 --- a/drivers/net/dsa/microchip/lan937x.h +++ b/drivers/net/dsa/microchip/lan937x.h @@ -8,14 +8,16 @@ int lan937x_reset_switch(struct ksz_device *dev); int lan937x_setup(struct dsa_switch *ds); +void lan937x_teardown(struct dsa_switch *ds); void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port); void lan937x_config_cpu_port(struct dsa_switch *ds); int lan937x_switch_init(struct ksz_device *dev); void lan937x_switch_exit(struct ksz_device *dev); -void lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data); -void lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val); +int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data); +int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val); int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu); void lan937x_phylink_get_caps(struct ksz_device *dev, int port, struct phylink_config *config); void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port); +int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs); #endif diff --git a/drivers/net/dsa/microchip/lan937x_main.c b/drivers/net/dsa/microchip/lan937x_main.c index daedd2bf20c1b6a7746cdd6ed495f2582c389836..7e4f307a0387ebc5b84a4b8f0373cbcb736304ee 100644 --- a/drivers/net/dsa/microchip/lan937x_main.c +++ b/drivers/net/dsa/microchip/lan937x_main.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -128,81 +127,14 @@ static int lan937x_internal_phy_read(struct ksz_device *dev, int addr, int reg, return ksz_read16(dev, REG_VPHY_IND_DATA__2, val); } -void lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) +int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data) { - lan937x_internal_phy_read(dev, addr, reg, data); + return lan937x_internal_phy_read(dev, addr, reg, data); } -void lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val) +int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val) { - lan937x_internal_phy_write(dev, addr, reg, val); -} - -static int lan937x_sw_mdio_read(struct mii_bus *bus, int addr, int regnum) -{ - struct ksz_device *dev = bus->priv; - u16 val; - int ret; - - if (regnum & MII_ADDR_C45) - return -EOPNOTSUPP; - - ret = lan937x_internal_phy_read(dev, addr, regnum, &val); - if (ret < 0) - return ret; - - return val; -} - -static int lan937x_sw_mdio_write(struct mii_bus *bus, int addr, int regnum, - u16 val) -{ - struct ksz_device *dev = bus->priv; - - if (regnum & MII_ADDR_C45) - return -EOPNOTSUPP; - - return lan937x_internal_phy_write(dev, addr, regnum, val); -} - -static int lan937x_mdio_register(struct ksz_device *dev) -{ - struct dsa_switch *ds = dev->ds; - struct device_node *mdio_np; - struct mii_bus *bus; - int ret; - - mdio_np = of_get_child_by_name(dev->dev->of_node, "mdio"); - if (!mdio_np) { - dev_err(ds->dev, "no MDIO bus node\n"); - return -ENODEV; - } - - bus = devm_mdiobus_alloc(ds->dev); - if (!bus) { - of_node_put(mdio_np); - return -ENOMEM; - } - - bus->priv = dev; - bus->read = lan937x_sw_mdio_read; - bus->write = lan937x_sw_mdio_write; - bus->name = "lan937x slave smi"; - snprintf(bus->id, MII_BUS_ID_SIZE, "SMI-%d", ds->index); - bus->parent = ds->dev; - bus->phy_mask = ~ds->phys_mii_mask; - - ds->slave_mii_bus = bus; - - ret = devm_of_mdiobus_register(ds->dev, bus, mdio_np); - if (ret) { - dev_err(ds->dev, "unable to register MDIO bus %s\n", - bus->id); - } - - of_node_put(mdio_np); - - return ret; + return lan937x_internal_phy_write(dev, addr, reg, val); } int lan937x_reset_switch(struct ksz_device *dev) @@ -225,6 +157,10 @@ int lan937x_reset_switch(struct ksz_device *dev) if (ret < 0) return ret; + ret = ksz_write32(dev, REG_SW_INT_STATUS__4, POR_READY_INT); + if (ret < 0) + return ret; + ret = ksz_write32(dev, REG_SW_PORT_INT_MASK__4, 0xFF); if (ret < 0) return ret; @@ -244,10 +180,6 @@ void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port) lan937x_port_cfg(dev, port, REG_PORT_CTRL_0, PORT_TAIL_TAG_ENABLE, true); - /* disable frame check length field */ - lan937x_port_cfg(dev, port, REG_PORT_MAC_CTRL_0, PORT_CHECK_LENGTH, - false); - /* set back pressure for half duplex */ lan937x_port_cfg(dev, port, REG_PORT_MAC_CTRL_1, PORT_BACK_PRESSURE, true); @@ -315,6 +247,23 @@ int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu) return 0; } +int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs) +{ + u32 secs = msecs / 1000; + u32 value; + int ret; + + value = FIELD_GET(SW_AGE_PERIOD_7_0_M, secs); + + ret = ksz_write8(dev, REG_SW_AGE_PERIOD__1, value); + if (ret < 0) + return ret; + + value = FIELD_GET(SW_AGE_PERIOD_19_8_M, secs); + + return ksz_write16(dev, REG_SW_AGE_PERIOD__2, value); +} + static void lan937x_set_tune_adj(struct ksz_device *dev, int port, u16 reg, u8 val) { @@ -383,6 +332,13 @@ void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port) } } +int lan937x_switch_init(struct ksz_device *dev) +{ + dev->port_mask = (1 << dev->info->port_cnt) - 1; + + return 0; +} + int lan937x_setup(struct dsa_switch *ds) { struct ksz_device *dev = ds->priv; @@ -395,12 +351,6 @@ int lan937x_setup(struct dsa_switch *ds) return ret; } - ret = lan937x_mdio_register(dev); - if (ret < 0) { - dev_err(dev->dev, "failed to register the mdio"); - return ret; - } - /* The VLAN aware is a global setting. Mixed vlan * filterings are not supported. */ @@ -426,11 +376,9 @@ int lan937x_setup(struct dsa_switch *ds) return 0; } -int lan937x_switch_init(struct ksz_device *dev) +void lan937x_teardown(struct dsa_switch *ds) { - dev->port_mask = (1 << dev->info->port_cnt) - 1; - return 0; } void lan937x_switch_exit(struct ksz_device *dev) diff --git a/drivers/net/dsa/microchip/lan937x_reg.h b/drivers/net/dsa/microchip/lan937x_reg.h index ba4adaddb3ecce0cbcb2e0d8a79e47a689abd5a4..5bc16a4c444165d20dca63e389aa351ec5085dec 100644 --- a/drivers/net/dsa/microchip/lan937x_reg.h +++ b/drivers/net/dsa/microchip/lan937x_reg.h @@ -62,6 +62,12 @@ #define SW_FAST_AGING BIT(1) #define SW_LINK_AUTO_AGING BIT(0) +#define REG_SW_AGE_PERIOD__1 0x0313 +#define SW_AGE_PERIOD_7_0_M GENMASK(7, 0) + +#define REG_SW_AGE_PERIOD__2 0x0320 +#define SW_AGE_PERIOD_19_8_M GENMASK(19, 8) + #define REG_SW_MAC_CTRL_0 0x0330 #define SW_NEW_BACKOFF BIT(7) #define SW_PAUSE_UNH_MODE BIT(1) @@ -118,6 +124,18 @@ /* Port Registers */ /* 0 - Operation */ +#define REG_PORT_INT_STATUS 0x001B +#define REG_PORT_INT_MASK 0x001F + +#define PORT_TAS_INT BIT(5) +#define PORT_QCI_INT BIT(4) +#define PORT_SGMII_INT BIT(3) +#define PORT_PTP_INT BIT(2) +#define PORT_PHY_INT BIT(1) +#define PORT_ACL_INT BIT(0) + +#define PORT_SRC_PHY_INT 1 + #define REG_PORT_CTRL_0 0x0020 #define PORT_MAC_LOOPBACK BIT(7) diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 835807911be019d78c7b9ac416349f444873ec77..e74c6b4061728e5bbb5d14e5d2c4f903c7c4752e 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -506,14 +506,19 @@ static bool mt7531_dual_sgmii_supported(struct mt7530_priv *priv) static int mt7531_pad_setup(struct dsa_switch *ds, phy_interface_t interface) { - struct mt7530_priv *priv = ds->priv; + return 0; +} + +static void +mt7531_pll_setup(struct mt7530_priv *priv) +{ u32 top_sig; u32 hwstrap; u32 xtal; u32 val; if (mt7531_dual_sgmii_supported(priv)) - return 0; + return; val = mt7530_read(priv, MT7531_CREV); top_sig = mt7530_read(priv, MT7531_TOP_SIG_SR); @@ -592,8 +597,6 @@ mt7531_pad_setup(struct dsa_switch *ds, phy_interface_t interface) val |= EN_COREPLL; mt7530_write(priv, MT7531_PLLGP_EN, val); usleep_range(25, 35); - - return 0; } static void @@ -2326,11 +2329,17 @@ mt7531_setup(struct dsa_switch *ds) return -ENODEV; } + /* all MACs must be forced link-down before sw reset */ + for (i = 0; i < MT7530_NUM_PORTS; i++) + mt7530_write(priv, MT7530_PMCR_P(i), MT7531_FORCE_LNK); + /* Reset the switch through internal reset */ mt7530_write(priv, MT7530_SYS_CTRL, SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST | SYS_CTRL_REG_RST); + mt7531_pll_setup(priv); + if (mt7531_dual_sgmii_supported(priv)) { priv->p5_intf_sel = P5_INTF_SEL_GMAC5_SGMII; @@ -2699,9 +2708,6 @@ mt7531_mac_config(struct dsa_switch *ds, int port, unsigned int mode, case PHY_INTERFACE_MODE_NA: case PHY_INTERFACE_MODE_1000BASEX: case PHY_INTERFACE_MODE_2500BASEX: - if (phylink_autoneg_inband(mode)) - return -EINVAL; - return mt7531_sgmii_setup_mode_force(priv, port, interface); default: return -EINVAL; @@ -2777,13 +2783,6 @@ unsupported: return; } - if (phylink_autoneg_inband(mode) && - state->interface != PHY_INTERFACE_MODE_SGMII) { - dev_err(ds->dev, "%s: in-band negotiation unsupported\n", - __func__); - return; - } - mcr_cur = mt7530_read(priv, MT7530_PMCR_P(port)); mcr_new = mcr_cur; mcr_new &= ~PMCR_LINK_SETTINGS_MASK; @@ -2887,8 +2886,6 @@ mt7531_cpu_port_config(struct dsa_switch *ds, int port) case 6: interface = PHY_INTERFACE_MODE_2500BASEX; - mt7531_pad_setup(ds, interface); - priv->p6_interface = interface; break; default: @@ -2922,6 +2919,9 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port, config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD; + if ((priv->id == ID_MT7531) && mt753x_is_mac_port(port)) + config->mac_capabilities |= MAC_2500FD; + /* This driver does not make use of the speed, duplex, pause or the * advertisement in its mac_config, so it is safe to mark this driver * as non-legacy. @@ -2987,6 +2987,7 @@ mt7531_sgmii_pcs_get_state_an(struct mt7530_priv *priv, int port, status = mt7530_read(priv, MT7531_PCS_CONTROL_1(port)); state->link = !!(status & MT7531_SGMII_LINK_STATUS); + state->an_complete = !!(status & MT7531_SGMII_AN_COMPLETE); if (state->interface == PHY_INTERFACE_MODE_SGMII && (status & MT7531_SGMII_AN_ENABLE)) { val = mt7530_read(priv, MT7531_PCS_SPEED_ABILITY(port)); @@ -3017,16 +3018,44 @@ mt7531_sgmii_pcs_get_state_an(struct mt7530_priv *priv, int port, return 0; } +static void +mt7531_sgmii_pcs_get_state_inband(struct mt7530_priv *priv, int port, + struct phylink_link_state *state) +{ + unsigned int val; + + val = mt7530_read(priv, MT7531_PCS_CONTROL_1(port)); + state->link = !!(val & MT7531_SGMII_LINK_STATUS); + if (!state->link) + return; + + state->an_complete = state->link; + + if (state->interface == PHY_INTERFACE_MODE_2500BASEX) + state->speed = SPEED_2500; + else + state->speed = SPEED_1000; + + state->duplex = DUPLEX_FULL; + state->pause = MLO_PAUSE_NONE; +} + static void mt7531_pcs_get_state(struct phylink_pcs *pcs, struct phylink_link_state *state) { struct mt7530_priv *priv = pcs_to_mt753x_pcs(pcs)->priv; int port = pcs_to_mt753x_pcs(pcs)->port; - if (state->interface == PHY_INTERFACE_MODE_SGMII) + if (state->interface == PHY_INTERFACE_MODE_SGMII) { mt7531_sgmii_pcs_get_state_an(priv, port, state); - else - state->link = false; + return; + } else if ((state->interface == PHY_INTERFACE_MODE_1000BASEX) || + (state->interface == PHY_INTERFACE_MODE_2500BASEX)) { + mt7531_sgmii_pcs_get_state_inband(priv, port, state); + return; + } + + state->link = false; } static int mt753x_pcs_config(struct phylink_pcs *pcs, unsigned int mode, @@ -3067,6 +3096,8 @@ mt753x_setup(struct dsa_switch *ds) priv->pcs[i].pcs.ops = priv->info->pcs_ops; priv->pcs[i].priv = priv; priv->pcs[i].port = i; + if (mt753x_is_mac_port(i)) + priv->pcs[i].pcs.poll = 1; } ret = priv->info->sw_setup(ds); @@ -3300,8 +3331,6 @@ mt7530_remove(struct mdio_device *mdiodev) dsa_unregister_switch(priv->ds); mutex_destroy(&priv->reg_mutex); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void mt7530_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h index e509af95c354140179330ce725f4765050f18616..e8d9664353504f66c7957e7e49b6bbb0645082c9 100644 --- a/drivers/net/dsa/mt7530.h +++ b/drivers/net/dsa/mt7530.h @@ -373,6 +373,7 @@ enum mt7530_vlan_port_acc_frm { #define MT7531_SGMII_LINK_STATUS BIT(18) #define MT7531_SGMII_AN_ENABLE BIT(12) #define MT7531_SGMII_AN_RESTART BIT(9) +#define MT7531_SGMII_AN_COMPLETE BIT(21) /* Register for SGMII PCS_SPPED_ABILITY */ #define MT7531_PCS_SPEED_ABILITY(p) MT7531_SGMII_REG(p, 0x08) diff --git a/drivers/net/dsa/mv88e6060.c b/drivers/net/dsa/mv88e6060.c index 83dca9179aa07a1d4c1a0b89c8e2305cb1b13f22..fdda62d6eb16abca750c2731fc2d060df4374d62 100644 --- a/drivers/net/dsa/mv88e6060.c +++ b/drivers/net/dsa/mv88e6060.c @@ -297,8 +297,6 @@ static void mv88e6060_remove(struct mdio_device *mdiodev) return; dsa_unregister_switch(ds); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void mv88e6060_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 07e9a4da924c5bba8b5e339f8a96167e8a0c2bec..2479be3a1e3588d197e4588d1d0c055721ebdbb6 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -816,6 +816,14 @@ static void mv88e6393x_phylink_get_caps(struct mv88e6xxx_chip *chip, int port, MAC_10000FD; } } + + if (port == 0) { + __set_bit(PHY_INTERFACE_MODE_RMII, supported); + __set_bit(PHY_INTERFACE_MODE_RGMII, supported); + __set_bit(PHY_INTERFACE_MODE_RGMII_ID, supported); + __set_bit(PHY_INTERFACE_MODE_RGMII_RXID, supported); + __set_bit(PHY_INTERFACE_MODE_RGMII_TXID, supported); + } } static void mv88e6xxx_get_caps(struct dsa_switch *ds, int port, @@ -1128,7 +1136,7 @@ static void mv88e6xxx_atu_vtu_get_strings(uint8_t *data) unsigned int i; for (i = 0; i < ARRAY_SIZE(mv88e6xxx_atu_vtu_stats_strings); i++) - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, mv88e6xxx_atu_vtu_stats_strings[i], ETH_GSTRING_LEN); } @@ -6585,14 +6593,17 @@ out: static bool mv88e6xxx_lag_can_offload(struct dsa_switch *ds, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { struct mv88e6xxx_chip *chip = ds->priv; struct dsa_port *dp; int members = 0; - if (!mv88e6xxx_has_lag(chip)) + if (!mv88e6xxx_has_lag(chip)) { + NL_SET_ERR_MSG_MOD(extack, "Chip does not support LAG offload"); return false; + } if (!lag.id) return false; @@ -6601,14 +6612,20 @@ static bool mv88e6xxx_lag_can_offload(struct dsa_switch *ds, /* Includes the port joining the LAG */ members++; - if (members > 8) + if (members > 8) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot offload more than 8 LAG ports"); return false; + } /* We could potentially relax this to include active * backup in the future. */ - if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) + if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) { + NL_SET_ERR_MSG_MOD(extack, + "Can only offload LAG using hash TX type"); return false; + } /* Ideally we would also validate that the hash type matches * the hardware. Alas, this is always set to unknown on team @@ -6761,12 +6778,13 @@ static int mv88e6xxx_port_lag_change(struct dsa_switch *ds, int port) static int mv88e6xxx_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { struct mv88e6xxx_chip *chip = ds->priv; int err, id; - if (!mv88e6xxx_lag_can_offload(ds, lag, info)) + if (!mv88e6xxx_lag_can_offload(ds, lag, info, extack)) return -EOPNOTSUPP; /* DSA LAG IDs are one-based */ @@ -6819,12 +6837,13 @@ static int mv88e6xxx_crosschip_lag_change(struct dsa_switch *ds, int sw_index, static int mv88e6xxx_crosschip_lag_join(struct dsa_switch *ds, int sw_index, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { struct mv88e6xxx_chip *chip = ds->priv; int err; - if (!mv88e6xxx_lag_can_offload(ds, lag, info)) + if (!mv88e6xxx_lag_can_offload(ds, lag, info, extack)) return -EOPNOTSUPP; mv88e6xxx_reg_lock(chip); @@ -7166,8 +7185,6 @@ static void mv88e6xxx_remove(struct mdio_device *mdiodev) mv88e6xxx_g1_irq_free(chip); else mv88e6xxx_irq_poll_free(chip); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void mv88e6xxx_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h index 807aeaad983007d5dee4dcc837e7420f25b854b8..7536b8b0ad011b70395bda7e519af146d20c3319 100644 --- a/drivers/net/dsa/mv88e6xxx/global2.h +++ b/drivers/net/dsa/mv88e6xxx/global2.h @@ -298,7 +298,7 @@ #define MV88E6352_G2_SCRATCH_CONFIG_DATA1 0x71 #define MV88E6352_G2_SCRATCH_CONFIG_DATA1_NO_CPU BIT(2) #define MV88E6352_G2_SCRATCH_CONFIG_DATA2 0x72 -#define MV88E6352_G2_SCRATCH_CONFIG_DATA2_P0_MODE_MASK 0x3 +#define MV88E6352_G2_SCRATCH_CONFIG_DATA2_P0_MODE_MASK 0xf #define MV88E6352_G2_SCRATCH_CONFIG_DATA3 0x73 #define MV88E6352_G2_SCRATCH_CONFIG_DATA3_S_SEL BIT(1) diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c index 90c55f23b7c98fbe312c8e41e828ec76726a2a7e..5c4195c635b0f999a901fc7baed2278542caa5c4 100644 --- a/drivers/net/dsa/mv88e6xxx/port.c +++ b/drivers/net/dsa/mv88e6xxx/port.c @@ -517,6 +517,12 @@ static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port, case PHY_INTERFACE_MODE_RMII: cmode = MV88E6XXX_PORT_STS_CMODE_RMII; break; + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + cmode = MV88E6XXX_PORT_STS_CMODE_RGMII; + break; case PHY_INTERFACE_MODE_1000BASEX: cmode = MV88E6XXX_PORT_STS_CMODE_1000BASEX; break; @@ -634,6 +640,19 @@ int mv88e6393x_port_set_cmode(struct mv88e6xxx_chip *chip, int port, if (port != 0 && port != 9 && port != 10) return -EOPNOTSUPP; + if (port == 9 || port == 10) { + switch (mode) { + case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + return -EINVAL; + default: + break; + } + } + /* mv88e6393x errata 4.5: EEE should be disabled on SERDES ports */ err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_MAC_CTL, ®); if (err) diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index aadb0bd7c24f1296e6e0ce676418dd34aa705314..dd3a18cc89dd2beb88e2fc892694326d5ab76761 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -42,6 +42,25 @@ static struct net_device *felix_classify_db(struct dsa_db db) } } +static int felix_cpu_port_for_master(struct dsa_switch *ds, + struct net_device *master) +{ + struct ocelot *ocelot = ds->priv; + struct dsa_port *cpu_dp; + int lag; + + if (netif_is_lag_master(master)) { + mutex_lock(&ocelot->fwd_domain_lock); + lag = ocelot_bond_get_id(ocelot, master); + mutex_unlock(&ocelot->fwd_domain_lock); + + return lag; + } + + cpu_dp = master->dsa_ptr; + return cpu_dp->index; +} + /* Set up VCAP ES0 rules for pushing a tag_8021q VLAN towards the CPU such that * the tagger can perform RX source port identification. */ @@ -422,6 +441,40 @@ static unsigned long felix_tag_npi_get_host_fwd_mask(struct dsa_switch *ds) return BIT(ocelot->num_phys_ports); } +static int felix_tag_npi_change_master(struct dsa_switch *ds, int port, + struct net_device *master, + struct netlink_ext_ack *extack) +{ + struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; + struct ocelot *ocelot = ds->priv; + + if (netif_is_lag_master(master)) { + NL_SET_ERR_MSG_MOD(extack, + "LAG DSA master only supported using ocelot-8021q"); + return -EOPNOTSUPP; + } + + /* Changing the NPI port breaks user ports still assigned to the old + * one, so only allow it while they're down, and don't allow them to + * come back up until they're all changed to the new one. + */ + dsa_switch_for_each_user_port(other_dp, ds) { + struct net_device *slave = other_dp->slave; + + if (other_dp != dp && (slave->flags & IFF_UP) && + dsa_port_to_master(other_dp) != master) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot change while old master still has users"); + return -EOPNOTSUPP; + } + } + + felix_npi_port_deinit(ocelot, ocelot->npi); + felix_npi_port_init(ocelot, felix_cpu_port_for_master(ds, master)); + + return 0; +} + /* Alternatively to using the NPI functionality, that same hardware MAC * connected internally to the enetc or fman DSA master can be configured to * use the software-defined tag_8021q frame format. As far as the hardware is @@ -433,6 +486,7 @@ static const struct felix_tag_proto_ops felix_tag_npi_proto_ops = { .setup = felix_tag_npi_setup, .teardown = felix_tag_npi_teardown, .get_host_fwd_mask = felix_tag_npi_get_host_fwd_mask, + .change_master = felix_tag_npi_change_master, }; static int felix_tag_8021q_setup(struct dsa_switch *ds) @@ -445,6 +499,9 @@ static int felix_tag_8021q_setup(struct dsa_switch *ds) if (err) return err; + dsa_switch_for_each_cpu_port(dp, ds) + ocelot_port_setup_dsa_8021q_cpu(ocelot, dp->index); + dsa_switch_for_each_user_port(dp, ds) ocelot_port_assign_dsa_8021q_cpu(ocelot, dp->index, dp->cpu_dp->index); @@ -493,6 +550,9 @@ static void felix_tag_8021q_teardown(struct dsa_switch *ds) dsa_switch_for_each_user_port(dp, ds) ocelot_port_unassign_dsa_8021q_cpu(ocelot, dp->index); + dsa_switch_for_each_cpu_port(dp, ds) + ocelot_port_teardown_dsa_8021q_cpu(ocelot, dp->index); + dsa_tag_8021q_unregister(ds); } @@ -501,10 +561,24 @@ static unsigned long felix_tag_8021q_get_host_fwd_mask(struct dsa_switch *ds) return dsa_cpu_ports(ds); } +static int felix_tag_8021q_change_master(struct dsa_switch *ds, int port, + struct net_device *master, + struct netlink_ext_ack *extack) +{ + int cpu = felix_cpu_port_for_master(ds, master); + struct ocelot *ocelot = ds->priv; + + ocelot_port_unassign_dsa_8021q_cpu(ocelot, port); + ocelot_port_assign_dsa_8021q_cpu(ocelot, port, cpu); + + return felix_update_trapping_destinations(ds, true); +} + static const struct felix_tag_proto_ops felix_tag_8021q_proto_ops = { .setup = felix_tag_8021q_setup, .teardown = felix_tag_8021q_teardown, .get_host_fwd_mask = felix_tag_8021q_get_host_fwd_mask, + .change_master = felix_tag_8021q_change_master, }; static void felix_set_host_flood(struct dsa_switch *ds, unsigned long mask, @@ -667,6 +741,16 @@ static void felix_port_set_host_flood(struct dsa_switch *ds, int port, !!felix->host_flood_mc_mask, true); } +static int felix_port_change_master(struct dsa_switch *ds, int port, + struct net_device *master, + struct netlink_ext_ack *extack) +{ + struct ocelot *ocelot = ds->priv; + struct felix *felix = ocelot_to_felix(ocelot); + + return felix->tag_proto_ops->change_master(ds, port, master, extack); +} + static int felix_set_ageing_time(struct dsa_switch *ds, unsigned int ageing_time) { @@ -855,11 +939,21 @@ static void felix_bridge_leave(struct dsa_switch *ds, int port, static int felix_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { struct ocelot *ocelot = ds->priv; + int err; + + err = ocelot_port_lag_join(ocelot, port, lag.dev, info, extack); + if (err) + return err; + + /* Update the logical LAG port that serves as tag_8021q CPU port */ + if (!dsa_is_cpu_port(ds, port)) + return 0; - return ocelot_port_lag_join(ocelot, port, lag.dev, info); + return felix_port_change_master(ds, port, lag.dev, extack); } static int felix_lag_leave(struct dsa_switch *ds, int port, @@ -869,7 +963,11 @@ static int felix_lag_leave(struct dsa_switch *ds, int port, ocelot_port_lag_leave(ocelot, port, lag.dev); - return 0; + /* Update the logical LAG port that serves as tag_8021q CPU port */ + if (!dsa_is_cpu_port(ds, port)) + return 0; + + return felix_port_change_master(ds, port, lag.dev, NULL); } static int felix_lag_change(struct dsa_switch *ds, int port) @@ -1007,6 +1105,27 @@ static void felix_phylink_mac_link_up(struct dsa_switch *ds, int port, felix->info->port_sched_speed_set(ocelot, port, speed); } +static int felix_port_enable(struct dsa_switch *ds, int port, + struct phy_device *phydev) +{ + struct dsa_port *dp = dsa_to_port(ds, port); + struct ocelot *ocelot = ds->priv; + + if (!dsa_port_is_user(dp)) + return 0; + + if (ocelot->npi >= 0) { + struct net_device *master = dsa_port_to_master(dp); + + if (felix_cpu_port_for_master(ds, master) != ocelot->npi) { + dev_err(ds->dev, "Multiple masters are not allowed\n"); + return -EINVAL; + } + } + + return 0; +} + static void felix_port_qos_map_init(struct ocelot *ocelot, int port) { int i; @@ -1028,6 +1147,55 @@ static void felix_port_qos_map_init(struct ocelot *ocelot, int port) } } +static void felix_get_stats64(struct dsa_switch *ds, int port, + struct rtnl_link_stats64 *stats) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_stats64(ocelot, port, stats); +} + +static void felix_get_pause_stats(struct dsa_switch *ds, int port, + struct ethtool_pause_stats *pause_stats) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_pause_stats(ocelot, port, pause_stats); +} + +static void felix_get_rmon_stats(struct dsa_switch *ds, int port, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_rmon_stats(ocelot, port, rmon_stats, ranges); +} + +static void felix_get_eth_ctrl_stats(struct dsa_switch *ds, int port, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_eth_ctrl_stats(ocelot, port, ctrl_stats); +} + +static void felix_get_eth_mac_stats(struct dsa_switch *ds, int port, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_eth_mac_stats(ocelot, port, mac_stats); +} + +static void felix_get_eth_phy_stats(struct dsa_switch *ds, int port, + struct ethtool_eth_phy_stats *phy_stats) +{ + struct ocelot *ocelot = ds->priv; + + ocelot_port_get_eth_phy_stats(ocelot, port, phy_stats); +} + static void felix_get_strings(struct dsa_switch *ds, int port, u32 stringset, u8 *data) { @@ -1144,11 +1312,55 @@ static int felix_parse_dt(struct felix *felix, phy_interface_t *port_phy_modes) return err; } +static struct regmap *felix_request_regmap_by_name(struct felix *felix, + const char *resource_name) +{ + struct ocelot *ocelot = &felix->ocelot; + struct resource res; + int i; + + for (i = 0; i < felix->info->num_resources; i++) { + if (strcmp(resource_name, felix->info->resources[i].name)) + continue; + + memcpy(&res, &felix->info->resources[i], sizeof(res)); + res.start += felix->switch_base; + res.end += felix->switch_base; + + return ocelot_regmap_init(ocelot, &res); + } + + return ERR_PTR(-ENOENT); +} + +static struct regmap *felix_request_regmap(struct felix *felix, + enum ocelot_target target) +{ + const char *resource_name = felix->info->resource_names[target]; + + /* If the driver didn't provide a resource name for the target, + * the resource is optional. + */ + if (!resource_name) + return NULL; + + return felix_request_regmap_by_name(felix, resource_name); +} + +static struct regmap *felix_request_port_regmap(struct felix *felix, int port) +{ + char resource_name[32]; + + sprintf(resource_name, "port%d", port); + + return felix_request_regmap_by_name(felix, resource_name); +} + static int felix_init_structs(struct felix *felix, int num_phys_ports) { struct ocelot *ocelot = &felix->ocelot; phy_interface_t *port_phy_modes; - struct resource res; + struct regmap *target; int port, i, err; ocelot->num_phys_ports = num_phys_ports; @@ -1182,20 +1394,11 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) } for (i = 0; i < TARGET_MAX; i++) { - struct regmap *target; - - if (!felix->info->target_io_res[i].name) - continue; - - memcpy(&res, &felix->info->target_io_res[i], sizeof(res)); - res.flags = IORESOURCE_MEM; - res.start += felix->switch_base; - res.end += felix->switch_base; - - target = felix->info->init_regmap(ocelot, &res); + target = felix_request_regmap(felix, i); if (IS_ERR(target)) { dev_err(ocelot->dev, - "Failed to map device memory space\n"); + "Failed to map device memory space: %pe\n", + target); kfree(port_phy_modes); return PTR_ERR(target); } @@ -1212,7 +1415,6 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) for (port = 0; port < num_phys_ports; port++) { struct ocelot_port *ocelot_port; - struct regmap *target; ocelot_port = devm_kzalloc(ocelot->dev, sizeof(struct ocelot_port), @@ -1224,16 +1426,11 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) return -ENOMEM; } - memcpy(&res, &felix->info->port_io_res[port], sizeof(res)); - res.flags = IORESOURCE_MEM; - res.start += felix->switch_base; - res.end += felix->switch_base; - - target = felix->info->init_regmap(ocelot, &res); + target = felix_request_port_regmap(felix, port); if (IS_ERR(target)) { dev_err(ocelot->dev, - "Failed to map memory space for port %d\n", - port); + "Failed to map memory space for port %d: %pe\n", + port, target); kfree(port_phy_modes); return PTR_ERR(target); } @@ -1842,6 +2039,12 @@ const struct dsa_switch_ops felix_switch_ops = { .setup = felix_setup, .teardown = felix_teardown, .set_ageing_time = felix_set_ageing_time, + .get_stats64 = felix_get_stats64, + .get_pause_stats = felix_get_pause_stats, + .get_rmon_stats = felix_get_rmon_stats, + .get_eth_ctrl_stats = felix_get_eth_ctrl_stats, + .get_eth_mac_stats = felix_get_eth_mac_stats, + .get_eth_phy_stats = felix_get_eth_phy_stats, .get_strings = felix_get_strings, .get_ethtool_stats = felix_get_ethtool_stats, .get_sset_count = felix_get_sset_count, @@ -1851,6 +2054,7 @@ const struct dsa_switch_ops felix_switch_ops = { .phylink_mac_select_pcs = felix_phylink_mac_select_pcs, .phylink_mac_link_down = felix_phylink_mac_link_down, .phylink_mac_link_up = felix_phylink_mac_link_up, + .port_enable = felix_port_enable, .port_fast_age = felix_port_fast_age, .port_fdb_dump = felix_fdb_dump, .port_fdb_add = felix_fdb_add, @@ -1906,6 +2110,7 @@ const struct dsa_switch_ops felix_switch_ops = { .port_add_dscp_prio = felix_port_add_dscp_prio, .port_del_dscp_prio = felix_port_del_dscp_prio, .port_set_host_flood = felix_port_set_host_flood, + .port_change_master = felix_port_change_master, }; struct net_device *felix_port_to_netdev(struct ocelot *ocelot, int port) diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index deb8dde1fc19d169f3d40b789750dd1c79818388..c9c29999c3363cb6719ea203bf7974a4805d81ca 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -16,9 +16,13 @@ /* Platform-specific information */ struct felix_info { - const struct resource *target_io_res; - const struct resource *port_io_res; - const struct resource *imdio_res; + /* Hardcoded resources provided by the hardware instantiation. */ + const struct resource *resources; + size_t num_resources; + /* Names of the mandatory resources that will be requested during + * probe. Must have TARGET_MAX elements, since it is indexed by target. + */ + const char *const *resource_names; const struct reg_field *regfields; const u32 *const *map; const struct ocelot_ops *ops; @@ -56,8 +60,6 @@ struct felix_info { void (*tas_guard_bands_update)(struct ocelot *ocelot, int port); void (*port_sched_speed_set)(struct ocelot *ocelot, int port, u32 speed); - struct regmap *(*init_regmap)(struct ocelot *ocelot, - struct resource *res); }; /* Methods for initializing the hardware resources specific to a tagging @@ -71,6 +73,9 @@ struct felix_tag_proto_ops { int (*setup)(struct dsa_switch *ds); void (*teardown)(struct dsa_switch *ds); unsigned long (*get_host_fwd_mask)(struct dsa_switch *ds); + int (*change_master)(struct dsa_switch *ds, int port, + struct net_device *master, + struct netlink_ext_ack *extack); }; extern const struct dsa_switch_ops felix_switch_ops; @@ -83,7 +88,6 @@ struct felix { struct mii_bus *imdio; struct phylink_pcs **pcs; resource_size_t switch_base; - resource_size_t imdio_base; enum dsa_tag_protocol tag_proto; const struct felix_tag_proto_ops *tag_proto_ops; struct kthread_worker *xmit_worker; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 1cdce8a98d1daa7ee5fc8d069c276c0fa029684d..26a35ae322d10317da3def9ba630fbaedabf45c7 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -22,6 +22,7 @@ #define VSC9959_NUM_PORTS 6 #define VSC9959_TAS_GCL_ENTRY_MAX 63 +#define VSC9959_TAS_MIN_GATE_LEN_NS 33 #define VSC9959_VCAP_POLICER_BASE 63 #define VSC9959_VCAP_POLICER_MAX 383 #define VSC9959_SWITCH_PCI_BAR 4 @@ -347,7 +348,7 @@ static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00026c), REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000270), REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000274), - REG(SYS_COUNT_TX_AGING, 0x000278), + REG(SYS_COUNT_TX_AGED, 0x000278), REG(SYS_COUNT_DROP_LOCAL, 0x000400), REG(SYS_COUNT_DROP_TAIL, 0x000404), REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000408), @@ -366,6 +367,10 @@ static const u32 vsc9959_sys_regmap[] = { REG(SYS_COUNT_DROP_GREEN_PRIO_5, 0x00043c), REG(SYS_COUNT_DROP_GREEN_PRIO_6, 0x000440), REG(SYS_COUNT_DROP_GREEN_PRIO_7, 0x000444), + REG(SYS_COUNT_SF_MATCHING_FRAMES, 0x000800), + REG(SYS_COUNT_SF_NOT_PASSING_FRAMES, 0x000804), + REG(SYS_COUNT_SF_NOT_PASSING_SDU, 0x000808), + REG(SYS_COUNT_SF_RED_FRAMES, 0x00080c), REG(SYS_RESET_CFG, 0x000e00), REG(SYS_SR_ETYPE_CFG, 0x000e04), REG(SYS_VLAN_ETYPE_CFG, 0x000e08), @@ -387,7 +392,6 @@ static const u32 vsc9959_sys_regmap[] = { REG_RESERVED(SYS_MMGT_FAST), REG_RESERVED(SYS_EVENTS_DIF), REG_RESERVED(SYS_EVENTS_CORE), - REG(SYS_CNT, 0x000000), REG(SYS_PTP_STATUS, 0x000f14), REG(SYS_PTP_TXSTAMP, 0x000f18), REG(SYS_PTP_NXT, 0x000f1c), @@ -473,100 +477,43 @@ static const u32 *vsc9959_regmap[TARGET_MAX] = { }; /* Addresses are relative to the PCI device's base address */ -static const struct resource vsc9959_target_io_res[TARGET_MAX] = { - [ANA] = { - .start = 0x0280000, - .end = 0x028ffff, - .name = "ana", - }, - [QS] = { - .start = 0x0080000, - .end = 0x00800ff, - .name = "qs", - }, - [QSYS] = { - .start = 0x0200000, - .end = 0x021ffff, - .name = "qsys", - }, - [REW] = { - .start = 0x0030000, - .end = 0x003ffff, - .name = "rew", - }, - [SYS] = { - .start = 0x0010000, - .end = 0x001ffff, - .name = "sys", - }, - [S0] = { - .start = 0x0040000, - .end = 0x00403ff, - .name = "s0", - }, - [S1] = { - .start = 0x0050000, - .end = 0x00503ff, - .name = "s1", - }, - [S2] = { - .start = 0x0060000, - .end = 0x00603ff, - .name = "s2", - }, - [PTP] = { - .start = 0x0090000, - .end = 0x00900cb, - .name = "ptp", - }, - [GCB] = { - .start = 0x0070000, - .end = 0x00701ff, - .name = "devcpu_gcb", - }, +static const struct resource vsc9959_resources[] = { + DEFINE_RES_MEM_NAMED(0x0010000, 0x0010000, "sys"), + DEFINE_RES_MEM_NAMED(0x0030000, 0x0010000, "rew"), + DEFINE_RES_MEM_NAMED(0x0040000, 0x0000400, "s0"), + DEFINE_RES_MEM_NAMED(0x0050000, 0x0000400, "s1"), + DEFINE_RES_MEM_NAMED(0x0060000, 0x0000400, "s2"), + DEFINE_RES_MEM_NAMED(0x0070000, 0x0000200, "devcpu_gcb"), + DEFINE_RES_MEM_NAMED(0x0080000, 0x0000100, "qs"), + DEFINE_RES_MEM_NAMED(0x0090000, 0x00000cc, "ptp"), + DEFINE_RES_MEM_NAMED(0x0100000, 0x0010000, "port0"), + DEFINE_RES_MEM_NAMED(0x0110000, 0x0010000, "port1"), + DEFINE_RES_MEM_NAMED(0x0120000, 0x0010000, "port2"), + DEFINE_RES_MEM_NAMED(0x0130000, 0x0010000, "port3"), + DEFINE_RES_MEM_NAMED(0x0140000, 0x0010000, "port4"), + DEFINE_RES_MEM_NAMED(0x0150000, 0x0010000, "port5"), + DEFINE_RES_MEM_NAMED(0x0200000, 0x0020000, "qsys"), + DEFINE_RES_MEM_NAMED(0x0280000, 0x0010000, "ana"), }; -static const struct resource vsc9959_port_io_res[] = { - { - .start = 0x0100000, - .end = 0x010ffff, - .name = "port0", - }, - { - .start = 0x0110000, - .end = 0x011ffff, - .name = "port1", - }, - { - .start = 0x0120000, - .end = 0x012ffff, - .name = "port2", - }, - { - .start = 0x0130000, - .end = 0x013ffff, - .name = "port3", - }, - { - .start = 0x0140000, - .end = 0x014ffff, - .name = "port4", - }, - { - .start = 0x0150000, - .end = 0x015ffff, - .name = "port5", - }, +static const char * const vsc9959_resource_names[TARGET_MAX] = { + [SYS] = "sys", + [REW] = "rew", + [S0] = "s0", + [S1] = "s1", + [S2] = "s2", + [GCB] = "devcpu_gcb", + [QS] = "qs", + [PTP] = "ptp", + [QSYS] = "qsys", + [ANA] = "ana", }; /* Port MAC 0 Internal MDIO bus through which the SerDes acting as an * SGMII/QSGMII MAC PCS can be found. */ -static const struct resource vsc9959_imdio_res = { - .start = 0x8030, - .end = 0x8040, - .name = "imdio", -}; +static const struct resource vsc9959_imdio_res = + DEFINE_RES_MEM_NAMED(0x8030, 0x8040, "imdio"); static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = { [ANA_ADVLEARN_VLAN_CHK] = REG_FIELD(ANA_ADVLEARN, 6, 6), @@ -619,378 +566,7 @@ static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = { }; static const struct ocelot_stat_layout vsc9959_stats_layout[OCELOT_NUM_STATS] = { - [OCELOT_STAT_RX_OCTETS] = { - .name = "rx_octets", - .reg = SYS_COUNT_RX_OCTETS, - }, - [OCELOT_STAT_RX_UNICAST] = { - .name = "rx_unicast", - .reg = SYS_COUNT_RX_UNICAST, - }, - [OCELOT_STAT_RX_MULTICAST] = { - .name = "rx_multicast", - .reg = SYS_COUNT_RX_MULTICAST, - }, - [OCELOT_STAT_RX_BROADCAST] = { - .name = "rx_broadcast", - .reg = SYS_COUNT_RX_BROADCAST, - }, - [OCELOT_STAT_RX_SHORTS] = { - .name = "rx_shorts", - .reg = SYS_COUNT_RX_SHORTS, - }, - [OCELOT_STAT_RX_FRAGMENTS] = { - .name = "rx_fragments", - .reg = SYS_COUNT_RX_FRAGMENTS, - }, - [OCELOT_STAT_RX_JABBERS] = { - .name = "rx_jabbers", - .reg = SYS_COUNT_RX_JABBERS, - }, - [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { - .name = "rx_crc_align_errs", - .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, - }, - [OCELOT_STAT_RX_SYM_ERRS] = { - .name = "rx_sym_errs", - .reg = SYS_COUNT_RX_SYM_ERRS, - }, - [OCELOT_STAT_RX_64] = { - .name = "rx_frames_below_65_octets", - .reg = SYS_COUNT_RX_64, - }, - [OCELOT_STAT_RX_65_127] = { - .name = "rx_frames_65_to_127_octets", - .reg = SYS_COUNT_RX_65_127, - }, - [OCELOT_STAT_RX_128_255] = { - .name = "rx_frames_128_to_255_octets", - .reg = SYS_COUNT_RX_128_255, - }, - [OCELOT_STAT_RX_256_511] = { - .name = "rx_frames_256_to_511_octets", - .reg = SYS_COUNT_RX_256_511, - }, - [OCELOT_STAT_RX_512_1023] = { - .name = "rx_frames_512_to_1023_octets", - .reg = SYS_COUNT_RX_512_1023, - }, - [OCELOT_STAT_RX_1024_1526] = { - .name = "rx_frames_1024_to_1526_octets", - .reg = SYS_COUNT_RX_1024_1526, - }, - [OCELOT_STAT_RX_1527_MAX] = { - .name = "rx_frames_over_1526_octets", - .reg = SYS_COUNT_RX_1527_MAX, - }, - [OCELOT_STAT_RX_PAUSE] = { - .name = "rx_pause", - .reg = SYS_COUNT_RX_PAUSE, - }, - [OCELOT_STAT_RX_CONTROL] = { - .name = "rx_control", - .reg = SYS_COUNT_RX_CONTROL, - }, - [OCELOT_STAT_RX_LONGS] = { - .name = "rx_longs", - .reg = SYS_COUNT_RX_LONGS, - }, - [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { - .name = "rx_classified_drops", - .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, - }, - [OCELOT_STAT_RX_RED_PRIO_0] = { - .name = "rx_red_prio_0", - .reg = SYS_COUNT_RX_RED_PRIO_0, - }, - [OCELOT_STAT_RX_RED_PRIO_1] = { - .name = "rx_red_prio_1", - .reg = SYS_COUNT_RX_RED_PRIO_1, - }, - [OCELOT_STAT_RX_RED_PRIO_2] = { - .name = "rx_red_prio_2", - .reg = SYS_COUNT_RX_RED_PRIO_2, - }, - [OCELOT_STAT_RX_RED_PRIO_3] = { - .name = "rx_red_prio_3", - .reg = SYS_COUNT_RX_RED_PRIO_3, - }, - [OCELOT_STAT_RX_RED_PRIO_4] = { - .name = "rx_red_prio_4", - .reg = SYS_COUNT_RX_RED_PRIO_4, - }, - [OCELOT_STAT_RX_RED_PRIO_5] = { - .name = "rx_red_prio_5", - .reg = SYS_COUNT_RX_RED_PRIO_5, - }, - [OCELOT_STAT_RX_RED_PRIO_6] = { - .name = "rx_red_prio_6", - .reg = SYS_COUNT_RX_RED_PRIO_6, - }, - [OCELOT_STAT_RX_RED_PRIO_7] = { - .name = "rx_red_prio_7", - .reg = SYS_COUNT_RX_RED_PRIO_7, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_0] = { - .name = "rx_yellow_prio_0", - .reg = SYS_COUNT_RX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_1] = { - .name = "rx_yellow_prio_1", - .reg = SYS_COUNT_RX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_2] = { - .name = "rx_yellow_prio_2", - .reg = SYS_COUNT_RX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_3] = { - .name = "rx_yellow_prio_3", - .reg = SYS_COUNT_RX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_4] = { - .name = "rx_yellow_prio_4", - .reg = SYS_COUNT_RX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_5] = { - .name = "rx_yellow_prio_5", - .reg = SYS_COUNT_RX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_6] = { - .name = "rx_yellow_prio_6", - .reg = SYS_COUNT_RX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_7] = { - .name = "rx_yellow_prio_7", - .reg = SYS_COUNT_RX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_RX_GREEN_PRIO_0] = { - .name = "rx_green_prio_0", - .reg = SYS_COUNT_RX_GREEN_PRIO_0, - }, - [OCELOT_STAT_RX_GREEN_PRIO_1] = { - .name = "rx_green_prio_1", - .reg = SYS_COUNT_RX_GREEN_PRIO_1, - }, - [OCELOT_STAT_RX_GREEN_PRIO_2] = { - .name = "rx_green_prio_2", - .reg = SYS_COUNT_RX_GREEN_PRIO_2, - }, - [OCELOT_STAT_RX_GREEN_PRIO_3] = { - .name = "rx_green_prio_3", - .reg = SYS_COUNT_RX_GREEN_PRIO_3, - }, - [OCELOT_STAT_RX_GREEN_PRIO_4] = { - .name = "rx_green_prio_4", - .reg = SYS_COUNT_RX_GREEN_PRIO_4, - }, - [OCELOT_STAT_RX_GREEN_PRIO_5] = { - .name = "rx_green_prio_5", - .reg = SYS_COUNT_RX_GREEN_PRIO_5, - }, - [OCELOT_STAT_RX_GREEN_PRIO_6] = { - .name = "rx_green_prio_6", - .reg = SYS_COUNT_RX_GREEN_PRIO_6, - }, - [OCELOT_STAT_RX_GREEN_PRIO_7] = { - .name = "rx_green_prio_7", - .reg = SYS_COUNT_RX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_OCTETS] = { - .name = "tx_octets", - .reg = SYS_COUNT_TX_OCTETS, - }, - [OCELOT_STAT_TX_UNICAST] = { - .name = "tx_unicast", - .reg = SYS_COUNT_TX_UNICAST, - }, - [OCELOT_STAT_TX_MULTICAST] = { - .name = "tx_multicast", - .reg = SYS_COUNT_TX_MULTICAST, - }, - [OCELOT_STAT_TX_BROADCAST] = { - .name = "tx_broadcast", - .reg = SYS_COUNT_TX_BROADCAST, - }, - [OCELOT_STAT_TX_COLLISION] = { - .name = "tx_collision", - .reg = SYS_COUNT_TX_COLLISION, - }, - [OCELOT_STAT_TX_DROPS] = { - .name = "tx_drops", - .reg = SYS_COUNT_TX_DROPS, - }, - [OCELOT_STAT_TX_PAUSE] = { - .name = "tx_pause", - .reg = SYS_COUNT_TX_PAUSE, - }, - [OCELOT_STAT_TX_64] = { - .name = "tx_frames_below_65_octets", - .reg = SYS_COUNT_TX_64, - }, - [OCELOT_STAT_TX_65_127] = { - .name = "tx_frames_65_to_127_octets", - .reg = SYS_COUNT_TX_65_127, - }, - [OCELOT_STAT_TX_128_255] = { - .name = "tx_frames_128_255_octets", - .reg = SYS_COUNT_TX_128_255, - }, - [OCELOT_STAT_TX_256_511] = { - .name = "tx_frames_256_511_octets", - .reg = SYS_COUNT_TX_256_511, - }, - [OCELOT_STAT_TX_512_1023] = { - .name = "tx_frames_512_1023_octets", - .reg = SYS_COUNT_TX_512_1023, - }, - [OCELOT_STAT_TX_1024_1526] = { - .name = "tx_frames_1024_1526_octets", - .reg = SYS_COUNT_TX_1024_1526, - }, - [OCELOT_STAT_TX_1527_MAX] = { - .name = "tx_frames_over_1526_octets", - .reg = SYS_COUNT_TX_1527_MAX, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_0] = { - .name = "tx_yellow_prio_0", - .reg = SYS_COUNT_TX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_1] = { - .name = "tx_yellow_prio_1", - .reg = SYS_COUNT_TX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_2] = { - .name = "tx_yellow_prio_2", - .reg = SYS_COUNT_TX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_3] = { - .name = "tx_yellow_prio_3", - .reg = SYS_COUNT_TX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_4] = { - .name = "tx_yellow_prio_4", - .reg = SYS_COUNT_TX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_5] = { - .name = "tx_yellow_prio_5", - .reg = SYS_COUNT_TX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_6] = { - .name = "tx_yellow_prio_6", - .reg = SYS_COUNT_TX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_7] = { - .name = "tx_yellow_prio_7", - .reg = SYS_COUNT_TX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_TX_GREEN_PRIO_0] = { - .name = "tx_green_prio_0", - .reg = SYS_COUNT_TX_GREEN_PRIO_0, - }, - [OCELOT_STAT_TX_GREEN_PRIO_1] = { - .name = "tx_green_prio_1", - .reg = SYS_COUNT_TX_GREEN_PRIO_1, - }, - [OCELOT_STAT_TX_GREEN_PRIO_2] = { - .name = "tx_green_prio_2", - .reg = SYS_COUNT_TX_GREEN_PRIO_2, - }, - [OCELOT_STAT_TX_GREEN_PRIO_3] = { - .name = "tx_green_prio_3", - .reg = SYS_COUNT_TX_GREEN_PRIO_3, - }, - [OCELOT_STAT_TX_GREEN_PRIO_4] = { - .name = "tx_green_prio_4", - .reg = SYS_COUNT_TX_GREEN_PRIO_4, - }, - [OCELOT_STAT_TX_GREEN_PRIO_5] = { - .name = "tx_green_prio_5", - .reg = SYS_COUNT_TX_GREEN_PRIO_5, - }, - [OCELOT_STAT_TX_GREEN_PRIO_6] = { - .name = "tx_green_prio_6", - .reg = SYS_COUNT_TX_GREEN_PRIO_6, - }, - [OCELOT_STAT_TX_GREEN_PRIO_7] = { - .name = "tx_green_prio_7", - .reg = SYS_COUNT_TX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_AGED] = { - .name = "tx_aged", - .reg = SYS_COUNT_TX_AGING, - }, - [OCELOT_STAT_DROP_LOCAL] = { - .name = "drop_local", - .reg = SYS_COUNT_DROP_LOCAL, - }, - [OCELOT_STAT_DROP_TAIL] = { - .name = "drop_tail", - .reg = SYS_COUNT_DROP_TAIL, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { - .name = "drop_yellow_prio_0", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { - .name = "drop_yellow_prio_1", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { - .name = "drop_yellow_prio_2", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { - .name = "drop_yellow_prio_3", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { - .name = "drop_yellow_prio_4", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { - .name = "drop_yellow_prio_5", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { - .name = "drop_yellow_prio_6", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { - .name = "drop_yellow_prio_7", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_0] = { - .name = "drop_green_prio_0", - .reg = SYS_COUNT_DROP_GREEN_PRIO_0, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_1] = { - .name = "drop_green_prio_1", - .reg = SYS_COUNT_DROP_GREEN_PRIO_1, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_2] = { - .name = "drop_green_prio_2", - .reg = SYS_COUNT_DROP_GREEN_PRIO_2, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_3] = { - .name = "drop_green_prio_3", - .reg = SYS_COUNT_DROP_GREEN_PRIO_3, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_4] = { - .name = "drop_green_prio_4", - .reg = SYS_COUNT_DROP_GREEN_PRIO_4, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_5] = { - .name = "drop_green_prio_5", - .reg = SYS_COUNT_DROP_GREEN_PRIO_5, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_6] = { - .name = "drop_green_prio_6", - .reg = SYS_COUNT_DROP_GREEN_PRIO_6, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_7] = { - .name = "drop_green_prio_7", - .reg = SYS_COUNT_DROP_GREEN_PRIO_7, - }, + OCELOT_COMMON_STATS, }; static const struct vcap_field vsc9959_vcap_es0_keys[] = { @@ -1370,9 +946,11 @@ static void vsc9959_wm_stat(u32 val, u32 *inuse, u32 *maxuse) static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) { + struct pci_dev *pdev = to_pci_dev(ocelot->dev); struct felix *felix = ocelot_to_felix(ocelot); struct enetc_mdio_priv *mdio_priv; struct device *dev = ocelot->dev; + resource_size_t imdio_base; void __iomem *imdio_regs; struct resource res; struct enetc_hw *hw; @@ -1388,10 +966,11 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) return -ENOMEM; } - memcpy(&res, felix->info->imdio_res, sizeof(res)); - res.flags = IORESOURCE_MEM; - res.start += felix->imdio_base; - res.end += felix->imdio_base; + imdio_base = pci_resource_start(pdev, VSC9959_IMDIO_PCI_BAR); + + memcpy(&res, &vsc9959_imdio_res, sizeof(res)); + res.start += imdio_base; + res.end += imdio_base; imdio_regs = devm_ioremap_resource(dev, &res); if (IS_ERR(imdio_regs)) @@ -1478,6 +1057,23 @@ static void vsc9959_mdio_bus_free(struct ocelot *ocelot) mdiobus_free(felix->imdio); } +/* The switch considers any frame (regardless of size) as eligible for + * transmission if the traffic class gate is open for at least 33 ns. + * Overruns are prevented by cropping an interval at the end of the gate time + * slot for which egress scheduling is blocked, but we need to still keep 33 ns + * available for one packet to be transmitted, otherwise the port tc will hang. + * This function returns the size of a gate interval that remains available for + * setting the guard band, after reserving the space for one egress frame. + */ +static u64 vsc9959_tas_remaining_gate_len_ps(u64 gate_len_ns) +{ + /* Gate always open */ + if (gate_len_ns == U64_MAX) + return U64_MAX; + + return (gate_len_ns - VSC9959_TAS_MIN_GATE_LEN_NS) * PSEC_PER_NSEC; +} + /* Extract shortest continuous gate open intervals in ns for each traffic class * of a cyclic tc-taprio schedule. If a gate is always open, the duration is * considered U64_MAX. If the gate is always closed, it is considered 0. @@ -1539,6 +1135,73 @@ static void vsc9959_tas_min_gate_lengths(struct tc_taprio_qopt_offload *taprio, min_gate_len[tc] = 0; } +/* ocelot_write_rix is a macro that concatenates QSYS_MAXSDU_CFG_* with _RSZ, + * so we need to spell out the register access to each traffic class in helper + * functions, to simplify callers + */ +static void vsc9959_port_qmaxsdu_set(struct ocelot *ocelot, int port, int tc, + u32 max_sdu) +{ + switch (tc) { + case 0: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_0, + port); + break; + case 1: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_1, + port); + break; + case 2: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_2, + port); + break; + case 3: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_3, + port); + break; + case 4: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_4, + port); + break; + case 5: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_5, + port); + break; + case 6: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_6, + port); + break; + case 7: + ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_7, + port); + break; + } +} + +static u32 vsc9959_port_qmaxsdu_get(struct ocelot *ocelot, int port, int tc) +{ + switch (tc) { + case 0: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_0, port); + case 1: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_1, port); + case 2: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_2, port); + case 3: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_3, port); + case 4: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_4, port); + case 5: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_5, port); + case 6: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_6, port); + case 7: return ocelot_read_rix(ocelot, QSYS_QMAXSDU_CFG_7, port); + default: + return 0; + } +} + +static u32 vsc9959_tas_tc_max_sdu(struct tc_taprio_qopt_offload *taprio, int tc) +{ + if (!taprio || !taprio->max_sdu[tc]) + return 0; + + return taprio->max_sdu[tc] + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN; +} + /* Update QSYS_PORT_MAX_SDU to make sure the static guard bands added by the * switch (see the ALWAYS_GUARD_BAND_SCH_Q comment) are correct at all MTU * values (the default value is 1518). Also, for traffic class windows smaller @@ -1548,6 +1211,7 @@ static void vsc9959_tas_min_gate_lengths(struct tc_taprio_qopt_offload *taprio, static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) { struct ocelot_port *ocelot_port = ocelot->ports[port]; + struct tc_taprio_qopt_offload *taprio; u64 min_gate_len[OCELOT_NUM_TC]; int speed, picos_per_byte; u64 needed_bit_time_ps; @@ -1557,6 +1221,8 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) lockdep_assert_held(&ocelot->tas_lock); + taprio = ocelot_port->taprio; + val = ocelot_read_rix(ocelot, QSYS_TAG_CONFIG, port); tas_speed = QSYS_TAG_CONFIG_LINK_SPEED_X(val); @@ -1593,17 +1259,23 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) "port %d: max frame size %d needs %llu ps at speed %d\n", port, maxlen, needed_bit_time_ps, speed); - vsc9959_tas_min_gate_lengths(ocelot_port->taprio, min_gate_len); + vsc9959_tas_min_gate_lengths(taprio, min_gate_len); + + mutex_lock(&ocelot->fwd_domain_lock); for (tc = 0; tc < OCELOT_NUM_TC; tc++) { + u32 requested_max_sdu = vsc9959_tas_tc_max_sdu(taprio, tc); + u64 remaining_gate_len_ps; u32 max_sdu; - if (min_gate_len[tc] == U64_MAX /* Gate always open */ || - min_gate_len[tc] * PSEC_PER_NSEC > needed_bit_time_ps) { + remaining_gate_len_ps = + vsc9959_tas_remaining_gate_len_ps(min_gate_len[tc]); + + if (remaining_gate_len_ps > needed_bit_time_ps) { /* Setting QMAXSDU_CFG to 0 disables oversized frame * dropping. */ - max_sdu = 0; + max_sdu = requested_max_sdu; dev_dbg(ocelot->dev, "port %d tc %d min gate len %llu" ", sending all frames\n", @@ -1612,9 +1284,15 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) /* If traffic class doesn't support a full MTU sized * frame, make sure to enable oversize frame dropping * for frames larger than the smallest that would fit. + * + * However, the exact same register, QSYS_QMAXSDU_CFG_*, + * controls not only oversized frame dropping, but also + * per-tc static guard band lengths, so it reduces the + * useful gate interval length. Therefore, be careful + * to calculate a guard band (and therefore max_sdu) + * that still leaves 33 ns available in the time slot. */ - max_sdu = div_u64(min_gate_len[tc] * PSEC_PER_NSEC, - picos_per_byte); + max_sdu = div_u64(remaining_gate_len_ps, picos_per_byte); /* A TC gate may be completely closed, which is a * special case where all packets are oversized. * Any limit smaller than 64 octets accomplishes this @@ -1628,6 +1306,10 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) */ if (max_sdu > 20) max_sdu -= 20; + + if (requested_max_sdu && requested_max_sdu < max_sdu) + max_sdu = requested_max_sdu; + dev_info(ocelot->dev, "port %d tc %d min gate length %llu" " ns not enough for max frame size %d at %d" @@ -1637,47 +1319,14 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port) max_sdu); } - /* ocelot_write_rix is a macro that concatenates - * QSYS_MAXSDU_CFG_* with _RSZ, so we need to spell out - * the writes to each traffic class - */ - switch (tc) { - case 0: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_0, - port); - break; - case 1: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_1, - port); - break; - case 2: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_2, - port); - break; - case 3: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_3, - port); - break; - case 4: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_4, - port); - break; - case 5: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_5, - port); - break; - case 6: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_6, - port); - break; - case 7: - ocelot_write_rix(ocelot, max_sdu, QSYS_QMAXSDU_CFG_7, - port); - break; - } + vsc9959_port_qmaxsdu_set(ocelot, port, tc, max_sdu); } ocelot_write_rix(ocelot, maxlen, QSYS_PORT_MAX_SDU, port); + + ocelot->ops->cut_through_fwd(ocelot); + + mutex_unlock(&ocelot->fwd_domain_lock); } static void vsc9959_sched_speed_set(struct ocelot *ocelot, int port, @@ -1704,13 +1353,13 @@ static void vsc9959_sched_speed_set(struct ocelot *ocelot, int port, break; } + mutex_lock(&ocelot->tas_lock); + ocelot_rmw_rix(ocelot, QSYS_TAG_CONFIG_LINK_SPEED(tas_speed), QSYS_TAG_CONFIG_LINK_SPEED_M, QSYS_TAG_CONFIG, port); - mutex_lock(&ocelot->tas_lock); - if (ocelot_port->taprio) vsc9959_tas_guard_bands_update(ocelot, port); @@ -1950,6 +1599,21 @@ static int vsc9959_qos_port_cbs_set(struct dsa_switch *ds, int port, return 0; } +static int vsc9959_qos_query_caps(struct tc_query_caps_base *base) +{ + switch (base->type) { + case TC_SETUP_QDISC_TAPRIO: { + struct tc_taprio_caps *caps = base->caps; + + caps->supports_queue_max_sdu = true; + + return 0; + } + default: + return -EOPNOTSUPP; + } +} + static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port, enum tc_setup_type type, void *type_data) @@ -1957,6 +1621,8 @@ static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port, struct ocelot *ocelot = ds->priv; switch (type) { + case TC_QUERY_CAPS: + return vsc9959_qos_query_caps(type_data); case TC_SETUP_QDISC_TAPRIO: return vsc9959_qos_port_tas_set(ocelot, port, type_data); case TC_SETUP_QDISC_CBS: @@ -1988,7 +1654,15 @@ struct felix_stream { u32 ssid; }; +struct felix_stream_filter_counters { + u64 match; + u64 not_pass_gate; + u64 not_pass_sdu; + u64 red; +}; + struct felix_stream_filter { + struct felix_stream_filter_counters stats; struct list_head list; refcount_t refcount; u32 index; @@ -2003,13 +1677,6 @@ struct felix_stream_filter { u32 maxsdu; }; -struct felix_stream_filter_counters { - u32 match; - u32 not_pass_gate; - u32 not_pass_sdu; - u32 red; -}; - struct felix_stream_gate { u32 index; u8 enable; @@ -2513,29 +2180,6 @@ static void vsc9959_psfp_sgi_table_del(struct ocelot *ocelot, } } -static void vsc9959_psfp_counters_get(struct ocelot *ocelot, u32 index, - struct felix_stream_filter_counters *counters) -{ - spin_lock(&ocelot->stats_lock); - - ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(index), - SYS_STAT_CFG_STAT_VIEW_M, - SYS_STAT_CFG); - - counters->match = ocelot_read_gix(ocelot, SYS_CNT, 0x200); - counters->not_pass_gate = ocelot_read_gix(ocelot, SYS_CNT, 0x201); - counters->not_pass_sdu = ocelot_read_gix(ocelot, SYS_CNT, 0x202); - counters->red = ocelot_read_gix(ocelot, SYS_CNT, 0x203); - - /* Clear the PSFP counter. */ - ocelot_write(ocelot, - SYS_STAT_CFG_STAT_VIEW(index) | - SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10), - SYS_STAT_CFG); - - spin_unlock(&ocelot->stats_lock); -} - static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, struct flow_cls_offload *f) { @@ -2560,6 +2204,8 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, return ret; } + mutex_lock(&psfp->lock); + flow_action_for_each(i, a, &f->rule->action) { switch (a->id) { case FLOW_ACTION_GATE: @@ -2601,6 +2247,7 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, sfi.maxsdu = a->police.mtu; break; default: + mutex_unlock(&psfp->lock); return -EOPNOTSUPP; } } @@ -2670,6 +2317,8 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, goto err; } + mutex_unlock(&psfp->lock); + return 0; err: @@ -2679,6 +2328,8 @@ err: if (sfi.fm_valid) ocelot_vcap_policer_del(ocelot, sfi.fmid); + mutex_unlock(&psfp->lock); + return ret; } @@ -2686,18 +2337,22 @@ static int vsc9959_psfp_filter_del(struct ocelot *ocelot, struct flow_cls_offload *f) { struct felix_stream *stream, tmp, *stream_entry; + struct ocelot_psfp_list *psfp = &ocelot->psfp; static struct felix_stream_filter *sfi; - struct ocelot_psfp_list *psfp; - psfp = &ocelot->psfp; + mutex_lock(&psfp->lock); stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie); - if (!stream) + if (!stream) { + mutex_unlock(&psfp->lock); return -ENOMEM; + } sfi = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, stream->sfid); - if (!sfi) + if (!sfi) { + mutex_unlock(&psfp->lock); return -ENOMEM; + } if (sfi->sg_valid) vsc9959_psfp_sgi_table_del(ocelot, sfi->sgid); @@ -2723,27 +2378,83 @@ static int vsc9959_psfp_filter_del(struct ocelot *ocelot, stream_entry->ports); } + mutex_unlock(&psfp->lock); + return 0; } +static void vsc9959_update_sfid_stats(struct ocelot *ocelot, + struct felix_stream_filter *sfi) +{ + struct felix_stream_filter_counters *s = &sfi->stats; + u32 match, not_pass_gate, not_pass_sdu, red; + u32 sfid = sfi->index; + + lockdep_assert_held(&ocelot->stat_view_lock); + + ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(sfid), + SYS_STAT_CFG_STAT_VIEW_M, + SYS_STAT_CFG); + + match = ocelot_read(ocelot, SYS_COUNT_SF_MATCHING_FRAMES); + not_pass_gate = ocelot_read(ocelot, SYS_COUNT_SF_NOT_PASSING_FRAMES); + not_pass_sdu = ocelot_read(ocelot, SYS_COUNT_SF_NOT_PASSING_SDU); + red = ocelot_read(ocelot, SYS_COUNT_SF_RED_FRAMES); + + /* Clear the PSFP counter. */ + ocelot_write(ocelot, + SYS_STAT_CFG_STAT_VIEW(sfid) | + SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10), + SYS_STAT_CFG); + + s->match += match; + s->not_pass_gate += not_pass_gate; + s->not_pass_sdu += not_pass_sdu; + s->red += red; +} + +/* Caller must hold &ocelot->stat_view_lock */ +static void vsc9959_update_stats(struct ocelot *ocelot) +{ + struct ocelot_psfp_list *psfp = &ocelot->psfp; + struct felix_stream_filter *sfi; + + mutex_lock(&psfp->lock); + + list_for_each_entry(sfi, &psfp->sfi_list, list) + vsc9959_update_sfid_stats(ocelot, sfi); + + mutex_unlock(&psfp->lock); +} + static int vsc9959_psfp_stats_get(struct ocelot *ocelot, struct flow_cls_offload *f, struct flow_stats *stats) { - struct felix_stream_filter_counters counters; - struct ocelot_psfp_list *psfp; + struct ocelot_psfp_list *psfp = &ocelot->psfp; + struct felix_stream_filter_counters *s; + static struct felix_stream_filter *sfi; struct felix_stream *stream; - psfp = &ocelot->psfp; stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie); if (!stream) return -ENOMEM; - vsc9959_psfp_counters_get(ocelot, stream->sfid, &counters); + sfi = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, stream->sfid); + if (!sfi) + return -EINVAL; + + mutex_lock(&ocelot->stat_view_lock); + + vsc9959_update_sfid_stats(ocelot, sfi); - stats->pkts = counters.match; - stats->drops = counters.not_pass_gate + counters.not_pass_sdu + - counters.red; + s = &sfi->stats; + stats->pkts = s->match; + stats->drops = s->not_pass_gate + s->not_pass_sdu + s->red; + + memset(s, 0, sizeof(*s)); + + mutex_unlock(&ocelot->stat_view_lock); return 0; } @@ -2755,6 +2466,7 @@ static void vsc9959_psfp_init(struct ocelot *ocelot) INIT_LIST_HEAD(&psfp->stream_list); INIT_LIST_HEAD(&psfp->sfi_list); INIT_LIST_HEAD(&psfp->sgi_list); + mutex_init(&psfp->lock); } /* When using cut-through forwarding and the egress port runs at a higher data @@ -2770,7 +2482,7 @@ static void vsc9959_cut_through_fwd(struct ocelot *ocelot) { struct felix *felix = ocelot_to_felix(ocelot); struct dsa_switch *ds = felix->ds; - int port, other_port; + int tc, port, other_port; lockdep_assert_held(&ocelot->fwd_domain_lock); @@ -2814,19 +2526,27 @@ static void vsc9959_cut_through_fwd(struct ocelot *ocelot) min_speed = other_ocelot_port->speed; } - /* Enable cut-through forwarding for all traffic classes. */ - if (ocelot_port->speed == min_speed) + /* Enable cut-through forwarding for all traffic classes that + * don't have oversized dropping enabled, since this check is + * bypassed in cut-through mode. + */ + if (ocelot_port->speed == min_speed) { val = GENMASK(7, 0); + for (tc = 0; tc < OCELOT_NUM_TC; tc++) + if (vsc9959_port_qmaxsdu_get(ocelot, port, tc)) + val &= ~BIT(tc); + } + set: tmp = ocelot_read_rix(ocelot, ANA_CUT_THRU_CFG, port); if (tmp == val) continue; dev_dbg(ocelot->dev, - "port %d fwd mask 0x%lx speed %d min_speed %d, %s cut-through forwarding\n", + "port %d fwd mask 0x%lx speed %d min_speed %d, %s cut-through forwarding on TC mask 0x%x\n", port, mask, ocelot_port->speed, min_speed, - val ? "enabling" : "disabling"); + val ? "enabling" : "disabling", val); ocelot_write_rix(ocelot, val, ANA_CUT_THRU_CFG, port); } @@ -2845,12 +2565,13 @@ static const struct ocelot_ops vsc9959_ops = { .psfp_stats_get = vsc9959_psfp_stats_get, .cut_through_fwd = vsc9959_cut_through_fwd, .tas_clock_adjust = vsc9959_tas_clock_adjust, + .update_stats = vsc9959_update_stats, }; static const struct felix_info felix_info_vsc9959 = { - .target_io_res = vsc9959_target_io_res, - .port_io_res = vsc9959_port_io_res, - .imdio_res = &vsc9959_imdio_res, + .resources = vsc9959_resources, + .num_resources = ARRAY_SIZE(vsc9959_resources), + .resource_names = vsc9959_resource_names, .regfields = vsc9959_regfields, .map = vsc9959_regmap, .ops = &vsc9959_ops, @@ -2872,7 +2593,6 @@ static const struct felix_info felix_info_vsc9959 = { .port_setup_tc = vsc9959_port_setup_tc, .port_sched_speed_set = vsc9959_sched_speed_set, .tas_guard_bands_update = vsc9959_tas_guard_bands_update, - .init_regmap = ocelot_regmap_init, }; static irqreturn_t felix_irq_handler(int irq, void *data) @@ -2924,7 +2644,6 @@ static int felix_pci_probe(struct pci_dev *pdev, ocelot->num_flooding_pgids = OCELOT_NUM_TC; felix->info = &felix_info_vsc9959; felix->switch_base = pci_resource_start(pdev, VSC9959_SWITCH_PCI_BAR); - felix->imdio_base = pci_resource_start(pdev, VSC9959_IMDIO_PCI_BAR); pci_set_master(pdev); @@ -2985,8 +2704,6 @@ static void felix_pci_remove(struct pci_dev *pdev) kfree(felix); pci_disable_device(pdev); - - pci_set_drvdata(pdev, NULL); } static void felix_pci_shutdown(struct pci_dev *pdev) diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index b34f4cdfe814c52ea3ffd8999d2866aee5da69a0..7af33b2c685dae0538550fbd2a78e5a67b6bb728 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -343,7 +343,7 @@ static const u32 vsc9953_sys_regmap[] = { REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00016c), REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000170), REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000174), - REG(SYS_COUNT_TX_AGING, 0x000178), + REG(SYS_COUNT_TX_AGED, 0x000178), REG(SYS_COUNT_DROP_LOCAL, 0x000200), REG(SYS_COUNT_DROP_TAIL, 0x000204), REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000208), @@ -383,7 +383,6 @@ static const u32 vsc9953_sys_regmap[] = { REG_RESERVED(SYS_MMGT_FAST), REG_RESERVED(SYS_EVENTS_DIF), REG_RESERVED(SYS_EVENTS_CORE), - REG_RESERVED(SYS_CNT), REG_RESERVED(SYS_PTP_STATUS), REG_RESERVED(SYS_PTP_TXSTAMP), REG_RESERVED(SYS_PTP_NXT), @@ -459,110 +458,40 @@ static const u32 *vsc9953_regmap[TARGET_MAX] = { }; /* Addresses are relative to the device's base address */ -static const struct resource vsc9953_target_io_res[TARGET_MAX] = { - [ANA] = { - .start = 0x0280000, - .end = 0x028ffff, - .name = "ana", - }, - [QS] = { - .start = 0x0080000, - .end = 0x00800ff, - .name = "qs", - }, - [QSYS] = { - .start = 0x0200000, - .end = 0x021ffff, - .name = "qsys", - }, - [REW] = { - .start = 0x0030000, - .end = 0x003ffff, - .name = "rew", - }, - [SYS] = { - .start = 0x0010000, - .end = 0x001ffff, - .name = "sys", - }, - [S0] = { - .start = 0x0040000, - .end = 0x00403ff, - .name = "s0", - }, - [S1] = { - .start = 0x0050000, - .end = 0x00503ff, - .name = "s1", - }, - [S2] = { - .start = 0x0060000, - .end = 0x00603ff, - .name = "s2", - }, - [PTP] = { - .start = 0x0090000, - .end = 0x00900cb, - .name = "ptp", - }, - [GCB] = { - .start = 0x0070000, - .end = 0x00701ff, - .name = "devcpu_gcb", - }, +static const struct resource vsc9953_resources[] = { + DEFINE_RES_MEM_NAMED(0x0010000, 0x0010000, "sys"), + DEFINE_RES_MEM_NAMED(0x0030000, 0x0010000, "rew"), + DEFINE_RES_MEM_NAMED(0x0040000, 0x0000400, "s0"), + DEFINE_RES_MEM_NAMED(0x0050000, 0x0000400, "s1"), + DEFINE_RES_MEM_NAMED(0x0060000, 0x0000400, "s2"), + DEFINE_RES_MEM_NAMED(0x0070000, 0x0000200, "devcpu_gcb"), + DEFINE_RES_MEM_NAMED(0x0080000, 0x0000100, "qs"), + DEFINE_RES_MEM_NAMED(0x0090000, 0x00000cc, "ptp"), + DEFINE_RES_MEM_NAMED(0x0100000, 0x0010000, "port0"), + DEFINE_RES_MEM_NAMED(0x0110000, 0x0010000, "port1"), + DEFINE_RES_MEM_NAMED(0x0120000, 0x0010000, "port2"), + DEFINE_RES_MEM_NAMED(0x0130000, 0x0010000, "port3"), + DEFINE_RES_MEM_NAMED(0x0140000, 0x0010000, "port4"), + DEFINE_RES_MEM_NAMED(0x0150000, 0x0010000, "port5"), + DEFINE_RES_MEM_NAMED(0x0160000, 0x0010000, "port6"), + DEFINE_RES_MEM_NAMED(0x0170000, 0x0010000, "port7"), + DEFINE_RES_MEM_NAMED(0x0180000, 0x0010000, "port8"), + DEFINE_RES_MEM_NAMED(0x0190000, 0x0010000, "port9"), + DEFINE_RES_MEM_NAMED(0x0200000, 0x0020000, "qsys"), + DEFINE_RES_MEM_NAMED(0x0280000, 0x0010000, "ana"), }; -static const struct resource vsc9953_port_io_res[] = { - { - .start = 0x0100000, - .end = 0x010ffff, - .name = "port0", - }, - { - .start = 0x0110000, - .end = 0x011ffff, - .name = "port1", - }, - { - .start = 0x0120000, - .end = 0x012ffff, - .name = "port2", - }, - { - .start = 0x0130000, - .end = 0x013ffff, - .name = "port3", - }, - { - .start = 0x0140000, - .end = 0x014ffff, - .name = "port4", - }, - { - .start = 0x0150000, - .end = 0x015ffff, - .name = "port5", - }, - { - .start = 0x0160000, - .end = 0x016ffff, - .name = "port6", - }, - { - .start = 0x0170000, - .end = 0x017ffff, - .name = "port7", - }, - { - .start = 0x0180000, - .end = 0x018ffff, - .name = "port8", - }, - { - .start = 0x0190000, - .end = 0x019ffff, - .name = "port9", - }, +static const char * const vsc9953_resource_names[TARGET_MAX] = { + [SYS] = "sys", + [REW] = "rew", + [S0] = "s0", + [S1] = "s1", + [S2] = "s2", + [GCB] = "devcpu_gcb", + [QS] = "qs", + [PTP] = "ptp", + [QSYS] = "qsys", + [ANA] = "ana", }; static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = { @@ -615,378 +544,7 @@ static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = { }; static const struct ocelot_stat_layout vsc9953_stats_layout[OCELOT_NUM_STATS] = { - [OCELOT_STAT_RX_OCTETS] = { - .name = "rx_octets", - .reg = SYS_COUNT_RX_OCTETS, - }, - [OCELOT_STAT_RX_UNICAST] = { - .name = "rx_unicast", - .reg = SYS_COUNT_RX_UNICAST, - }, - [OCELOT_STAT_RX_MULTICAST] = { - .name = "rx_multicast", - .reg = SYS_COUNT_RX_MULTICAST, - }, - [OCELOT_STAT_RX_BROADCAST] = { - .name = "rx_broadcast", - .reg = SYS_COUNT_RX_BROADCAST, - }, - [OCELOT_STAT_RX_SHORTS] = { - .name = "rx_shorts", - .reg = SYS_COUNT_RX_SHORTS, - }, - [OCELOT_STAT_RX_FRAGMENTS] = { - .name = "rx_fragments", - .reg = SYS_COUNT_RX_FRAGMENTS, - }, - [OCELOT_STAT_RX_JABBERS] = { - .name = "rx_jabbers", - .reg = SYS_COUNT_RX_JABBERS, - }, - [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { - .name = "rx_crc_align_errs", - .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, - }, - [OCELOT_STAT_RX_SYM_ERRS] = { - .name = "rx_sym_errs", - .reg = SYS_COUNT_RX_SYM_ERRS, - }, - [OCELOT_STAT_RX_64] = { - .name = "rx_frames_below_65_octets", - .reg = SYS_COUNT_RX_64, - }, - [OCELOT_STAT_RX_65_127] = { - .name = "rx_frames_65_to_127_octets", - .reg = SYS_COUNT_RX_65_127, - }, - [OCELOT_STAT_RX_128_255] = { - .name = "rx_frames_128_to_255_octets", - .reg = SYS_COUNT_RX_128_255, - }, - [OCELOT_STAT_RX_256_511] = { - .name = "rx_frames_256_to_511_octets", - .reg = SYS_COUNT_RX_256_511, - }, - [OCELOT_STAT_RX_512_1023] = { - .name = "rx_frames_512_to_1023_octets", - .reg = SYS_COUNT_RX_512_1023, - }, - [OCELOT_STAT_RX_1024_1526] = { - .name = "rx_frames_1024_to_1526_octets", - .reg = SYS_COUNT_RX_1024_1526, - }, - [OCELOT_STAT_RX_1527_MAX] = { - .name = "rx_frames_over_1526_octets", - .reg = SYS_COUNT_RX_1527_MAX, - }, - [OCELOT_STAT_RX_PAUSE] = { - .name = "rx_pause", - .reg = SYS_COUNT_RX_PAUSE, - }, - [OCELOT_STAT_RX_CONTROL] = { - .name = "rx_control", - .reg = SYS_COUNT_RX_CONTROL, - }, - [OCELOT_STAT_RX_LONGS] = { - .name = "rx_longs", - .reg = SYS_COUNT_RX_LONGS, - }, - [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { - .name = "rx_classified_drops", - .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, - }, - [OCELOT_STAT_RX_RED_PRIO_0] = { - .name = "rx_red_prio_0", - .reg = SYS_COUNT_RX_RED_PRIO_0, - }, - [OCELOT_STAT_RX_RED_PRIO_1] = { - .name = "rx_red_prio_1", - .reg = SYS_COUNT_RX_RED_PRIO_1, - }, - [OCELOT_STAT_RX_RED_PRIO_2] = { - .name = "rx_red_prio_2", - .reg = SYS_COUNT_RX_RED_PRIO_2, - }, - [OCELOT_STAT_RX_RED_PRIO_3] = { - .name = "rx_red_prio_3", - .reg = SYS_COUNT_RX_RED_PRIO_3, - }, - [OCELOT_STAT_RX_RED_PRIO_4] = { - .name = "rx_red_prio_4", - .reg = SYS_COUNT_RX_RED_PRIO_4, - }, - [OCELOT_STAT_RX_RED_PRIO_5] = { - .name = "rx_red_prio_5", - .reg = SYS_COUNT_RX_RED_PRIO_5, - }, - [OCELOT_STAT_RX_RED_PRIO_6] = { - .name = "rx_red_prio_6", - .reg = SYS_COUNT_RX_RED_PRIO_6, - }, - [OCELOT_STAT_RX_RED_PRIO_7] = { - .name = "rx_red_prio_7", - .reg = SYS_COUNT_RX_RED_PRIO_7, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_0] = { - .name = "rx_yellow_prio_0", - .reg = SYS_COUNT_RX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_1] = { - .name = "rx_yellow_prio_1", - .reg = SYS_COUNT_RX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_2] = { - .name = "rx_yellow_prio_2", - .reg = SYS_COUNT_RX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_3] = { - .name = "rx_yellow_prio_3", - .reg = SYS_COUNT_RX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_4] = { - .name = "rx_yellow_prio_4", - .reg = SYS_COUNT_RX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_5] = { - .name = "rx_yellow_prio_5", - .reg = SYS_COUNT_RX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_6] = { - .name = "rx_yellow_prio_6", - .reg = SYS_COUNT_RX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_7] = { - .name = "rx_yellow_prio_7", - .reg = SYS_COUNT_RX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_RX_GREEN_PRIO_0] = { - .name = "rx_green_prio_0", - .reg = SYS_COUNT_RX_GREEN_PRIO_0, - }, - [OCELOT_STAT_RX_GREEN_PRIO_1] = { - .name = "rx_green_prio_1", - .reg = SYS_COUNT_RX_GREEN_PRIO_1, - }, - [OCELOT_STAT_RX_GREEN_PRIO_2] = { - .name = "rx_green_prio_2", - .reg = SYS_COUNT_RX_GREEN_PRIO_2, - }, - [OCELOT_STAT_RX_GREEN_PRIO_3] = { - .name = "rx_green_prio_3", - .reg = SYS_COUNT_RX_GREEN_PRIO_3, - }, - [OCELOT_STAT_RX_GREEN_PRIO_4] = { - .name = "rx_green_prio_4", - .reg = SYS_COUNT_RX_GREEN_PRIO_4, - }, - [OCELOT_STAT_RX_GREEN_PRIO_5] = { - .name = "rx_green_prio_5", - .reg = SYS_COUNT_RX_GREEN_PRIO_5, - }, - [OCELOT_STAT_RX_GREEN_PRIO_6] = { - .name = "rx_green_prio_6", - .reg = SYS_COUNT_RX_GREEN_PRIO_6, - }, - [OCELOT_STAT_RX_GREEN_PRIO_7] = { - .name = "rx_green_prio_7", - .reg = SYS_COUNT_RX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_OCTETS] = { - .name = "tx_octets", - .reg = SYS_COUNT_TX_OCTETS, - }, - [OCELOT_STAT_TX_UNICAST] = { - .name = "tx_unicast", - .reg = SYS_COUNT_TX_UNICAST, - }, - [OCELOT_STAT_TX_MULTICAST] = { - .name = "tx_multicast", - .reg = SYS_COUNT_TX_MULTICAST, - }, - [OCELOT_STAT_TX_BROADCAST] = { - .name = "tx_broadcast", - .reg = SYS_COUNT_TX_BROADCAST, - }, - [OCELOT_STAT_TX_COLLISION] = { - .name = "tx_collision", - .reg = SYS_COUNT_TX_COLLISION, - }, - [OCELOT_STAT_TX_DROPS] = { - .name = "tx_drops", - .reg = SYS_COUNT_TX_DROPS, - }, - [OCELOT_STAT_TX_PAUSE] = { - .name = "tx_pause", - .reg = SYS_COUNT_TX_PAUSE, - }, - [OCELOT_STAT_TX_64] = { - .name = "tx_frames_below_65_octets", - .reg = SYS_COUNT_TX_64, - }, - [OCELOT_STAT_TX_65_127] = { - .name = "tx_frames_65_to_127_octets", - .reg = SYS_COUNT_TX_65_127, - }, - [OCELOT_STAT_TX_128_255] = { - .name = "tx_frames_128_255_octets", - .reg = SYS_COUNT_TX_128_255, - }, - [OCELOT_STAT_TX_256_511] = { - .name = "tx_frames_256_511_octets", - .reg = SYS_COUNT_TX_256_511, - }, - [OCELOT_STAT_TX_512_1023] = { - .name = "tx_frames_512_1023_octets", - .reg = SYS_COUNT_TX_512_1023, - }, - [OCELOT_STAT_TX_1024_1526] = { - .name = "tx_frames_1024_1526_octets", - .reg = SYS_COUNT_TX_1024_1526, - }, - [OCELOT_STAT_TX_1527_MAX] = { - .name = "tx_frames_over_1526_octets", - .reg = SYS_COUNT_TX_1527_MAX, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_0] = { - .name = "tx_yellow_prio_0", - .reg = SYS_COUNT_TX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_1] = { - .name = "tx_yellow_prio_1", - .reg = SYS_COUNT_TX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_2] = { - .name = "tx_yellow_prio_2", - .reg = SYS_COUNT_TX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_3] = { - .name = "tx_yellow_prio_3", - .reg = SYS_COUNT_TX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_4] = { - .name = "tx_yellow_prio_4", - .reg = SYS_COUNT_TX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_5] = { - .name = "tx_yellow_prio_5", - .reg = SYS_COUNT_TX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_6] = { - .name = "tx_yellow_prio_6", - .reg = SYS_COUNT_TX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_7] = { - .name = "tx_yellow_prio_7", - .reg = SYS_COUNT_TX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_TX_GREEN_PRIO_0] = { - .name = "tx_green_prio_0", - .reg = SYS_COUNT_TX_GREEN_PRIO_0, - }, - [OCELOT_STAT_TX_GREEN_PRIO_1] = { - .name = "tx_green_prio_1", - .reg = SYS_COUNT_TX_GREEN_PRIO_1, - }, - [OCELOT_STAT_TX_GREEN_PRIO_2] = { - .name = "tx_green_prio_2", - .reg = SYS_COUNT_TX_GREEN_PRIO_2, - }, - [OCELOT_STAT_TX_GREEN_PRIO_3] = { - .name = "tx_green_prio_3", - .reg = SYS_COUNT_TX_GREEN_PRIO_3, - }, - [OCELOT_STAT_TX_GREEN_PRIO_4] = { - .name = "tx_green_prio_4", - .reg = SYS_COUNT_TX_GREEN_PRIO_4, - }, - [OCELOT_STAT_TX_GREEN_PRIO_5] = { - .name = "tx_green_prio_5", - .reg = SYS_COUNT_TX_GREEN_PRIO_5, - }, - [OCELOT_STAT_TX_GREEN_PRIO_6] = { - .name = "tx_green_prio_6", - .reg = SYS_COUNT_TX_GREEN_PRIO_6, - }, - [OCELOT_STAT_TX_GREEN_PRIO_7] = { - .name = "tx_green_prio_7", - .reg = SYS_COUNT_TX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_AGED] = { - .name = "tx_aged", - .reg = SYS_COUNT_TX_AGING, - }, - [OCELOT_STAT_DROP_LOCAL] = { - .name = "drop_local", - .reg = SYS_COUNT_DROP_LOCAL, - }, - [OCELOT_STAT_DROP_TAIL] = { - .name = "drop_tail", - .reg = SYS_COUNT_DROP_TAIL, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { - .name = "drop_yellow_prio_0", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { - .name = "drop_yellow_prio_1", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { - .name = "drop_yellow_prio_2", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { - .name = "drop_yellow_prio_3", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { - .name = "drop_yellow_prio_4", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { - .name = "drop_yellow_prio_5", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { - .name = "drop_yellow_prio_6", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { - .name = "drop_yellow_prio_7", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_0] = { - .name = "drop_green_prio_0", - .reg = SYS_COUNT_DROP_GREEN_PRIO_0, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_1] = { - .name = "drop_green_prio_1", - .reg = SYS_COUNT_DROP_GREEN_PRIO_1, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_2] = { - .name = "drop_green_prio_2", - .reg = SYS_COUNT_DROP_GREEN_PRIO_2, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_3] = { - .name = "drop_green_prio_3", - .reg = SYS_COUNT_DROP_GREEN_PRIO_3, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_4] = { - .name = "drop_green_prio_4", - .reg = SYS_COUNT_DROP_GREEN_PRIO_4, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_5] = { - .name = "drop_green_prio_5", - .reg = SYS_COUNT_DROP_GREEN_PRIO_5, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_6] = { - .name = "drop_green_prio_6", - .reg = SYS_COUNT_DROP_GREEN_PRIO_6, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_7] = { - .name = "drop_green_prio_7", - .reg = SYS_COUNT_DROP_GREEN_PRIO_7, - }, + OCELOT_COMMON_STATS, }; static const struct vcap_field vsc9953_vcap_es0_keys[] = { @@ -1432,8 +990,9 @@ static void vsc9953_mdio_bus_free(struct ocelot *ocelot) } static const struct felix_info seville_info_vsc9953 = { - .target_io_res = vsc9953_target_io_res, - .port_io_res = vsc9953_port_io_res, + .resources = vsc9953_resources, + .num_resources = ARRAY_SIZE(vsc9953_resources), + .resource_names = vsc9953_resource_names, .regfields = vsc9953_regfields, .map = vsc9953_regmap, .ops = &vsc9953_ops, @@ -1450,7 +1009,6 @@ static const struct felix_info seville_info_vsc9953 = { .mdio_bus_free = vsc9953_mdio_bus_free, .phylink_validate = vsc9953_phylink_validate, .port_modes = vsc9953_port_modes, - .init_regmap = ocelot_regmap_init, }; static int seville_probe(struct platform_device *pdev) @@ -1525,8 +1083,6 @@ static int seville_remove(struct platform_device *pdev) kfree(felix->ds); kfree(felix); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/qca/ar9331.c b/drivers/net/dsa/qca/ar9331.c index 0796b7cf8cae2db8face3e7018efafbbdd4b51c9..e7b98b864fa1690e2cb58c17d60cfca0fa835544 100644 --- a/drivers/net/dsa/qca/ar9331.c +++ b/drivers/net/dsa/qca/ar9331.c @@ -1099,8 +1099,6 @@ static void ar9331_sw_remove(struct mdio_device *mdiodev) dsa_unregister_switch(&priv->ds); reset_control_assert(priv->sw_reset); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void ar9331_sw_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c index 1d3e7782a71f2255d756b9b866875f0331977d1d..5669c92c93f7a5b0aae501fbc2db577edd035457 100644 --- a/drivers/net/dsa/qca/qca8k-8xxx.c +++ b/drivers/net/dsa/qca/qca8k-8xxx.c @@ -1889,9 +1889,9 @@ qca8k_sw_probe(struct mdio_device *mdiodev) if (!priv) return -ENOMEM; - priv->info = of_device_get_match_data(priv->dev); priv->bus = mdiodev->bus; priv->dev = &mdiodev->dev; + priv->info = of_device_get_match_data(priv->dev); priv->reset_gpio = devm_gpiod_get_optional(priv->dev, "reset", GPIOD_ASIS); @@ -1957,8 +1957,6 @@ qca8k_sw_remove(struct mdio_device *mdiodev) qca8k_port_set_status(priv, i, 0); dsa_unregister_switch(priv->ds); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void qca8k_sw_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/qca/qca8k-common.c b/drivers/net/dsa/qca/qca8k-common.c index bba95613e218375ab19b8a746f08288116d9f091..fb45b598847be47c799a642753b4356ba0281ef7 100644 --- a/drivers/net/dsa/qca/qca8k-common.c +++ b/drivers/net/dsa/qca/qca8k-common.c @@ -1017,7 +1017,8 @@ int qca8k_port_vlan_del(struct dsa_switch *ds, int port, static bool qca8k_lag_can_offload(struct dsa_switch *ds, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { struct dsa_port *dp; int members = 0; @@ -1029,15 +1030,24 @@ static bool qca8k_lag_can_offload(struct dsa_switch *ds, /* Includes the port joining the LAG */ members++; - if (members > QCA8K_NUM_PORTS_FOR_LAG) + if (members > QCA8K_NUM_PORTS_FOR_LAG) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot offload more than 4 LAG ports"); return false; + } - if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) + if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) { + NL_SET_ERR_MSG_MOD(extack, + "Can only offload LAG using hash TX type"); return false; + } if (info->hash_type != NETDEV_LAG_HASH_L2 && - info->hash_type != NETDEV_LAG_HASH_L23) + info->hash_type != NETDEV_LAG_HASH_L23) { + NL_SET_ERR_MSG_MOD(extack, + "Can only offload L2 or L2+L3 TX hash"); return false; + } return true; } @@ -1160,11 +1170,12 @@ static int qca8k_lag_refresh_portmap(struct dsa_switch *ds, int port, } int qca8k_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { int ret; - if (!qca8k_lag_can_offload(ds, lag, info)) + if (!qca8k_lag_can_offload(ds, lag, info, extack)) return -EOPNOTSUPP; ret = qca8k_lag_setup_hash(ds, lag, info); diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h index e36ecc9777f437bcfc5f414335bef9ac8937abad..0b7a5cb12321670aba0575e5c5532340bd833c90 100644 --- a/drivers/net/dsa/qca/qca8k.h +++ b/drivers/net/dsa/qca/qca8k.h @@ -512,7 +512,8 @@ int qca8k_port_vlan_del(struct dsa_switch *ds, int port, /* Common port LAG function */ int qca8k_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info); + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack); int qca8k_port_lag_leave(struct dsa_switch *ds, int port, struct dsa_lag lag); diff --git a/drivers/net/dsa/realtek/realtek-mdio.c b/drivers/net/dsa/realtek/realtek-mdio.c index c58f49d558d2415ab69c0a77b43c10a15b57eca9..3e54fac5f9027ca1ae3d11f39f3735a102b0d17d 100644 --- a/drivers/net/dsa/realtek/realtek-mdio.c +++ b/drivers/net/dsa/realtek/realtek-mdio.c @@ -245,8 +245,6 @@ static void realtek_mdio_remove(struct mdio_device *mdiodev) /* leave the device reset asserted */ if (priv->reset) gpiod_set_value(priv->reset, 1); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void realtek_mdio_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dsa/realtek/realtek-smi.c b/drivers/net/dsa/realtek/realtek-smi.c index 45992f79ec8d48da869218970ee96c4bf9546be7..1b447d96b9c46c5d62358ab1bf401533cabe017c 100644 --- a/drivers/net/dsa/realtek/realtek-smi.c +++ b/drivers/net/dsa/realtek/realtek-smi.c @@ -522,8 +522,6 @@ static int realtek_smi_remove(struct platform_device *pdev) if (priv->reset) gpiod_set_value(priv->reset, 1); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/rzn1_a5psw.c b/drivers/net/dsa/rzn1_a5psw.c index 0744e8162e1d593274dde6dfc150bce28e3e4e6a..ed413d555beca84f2e8c79cf31ae8ab47bbcd4de 100644 --- a/drivers/net/dsa/rzn1_a5psw.c +++ b/drivers/net/dsa/rzn1_a5psw.c @@ -1025,8 +1025,6 @@ static int a5psw_remove(struct platform_device *pdev) clk_disable_unprepare(a5psw->hclk); clk_disable_unprepare(a5psw->clk); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index b03d0d0c3dbf5fa7a704c1b51c0b22fdcd6164fd..412666111b0c90cd780f6860e2657643a81e162b 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -3351,8 +3351,6 @@ static void sja1105_remove(struct spi_device *spi) return; dsa_unregister_switch(priv->ds); - - spi_set_drvdata(spi, NULL); } static void sja1105_shutdown(struct spi_device *spi) diff --git a/drivers/net/dsa/vitesse-vsc73xx-platform.c b/drivers/net/dsa/vitesse-vsc73xx-platform.c index fe4b154a0a575f8d363dc72df1aa98eb27d95f7f..bd4206e8f9afcb36f41d885b210faa4439f31a0b 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-platform.c +++ b/drivers/net/dsa/vitesse-vsc73xx-platform.c @@ -121,8 +121,6 @@ static int vsc73xx_platform_remove(struct platform_device *pdev) vsc73xx_remove(&vsc_platform->vsc); - platform_set_drvdata(pdev, NULL); - return 0; } diff --git a/drivers/net/dsa/vitesse-vsc73xx-spi.c b/drivers/net/dsa/vitesse-vsc73xx-spi.c index 97a92e6da60d8b95a9209b71461dd90233c84846..85b9a0f51dd894ce7ca4a0de10d1efb2f3be5d8c 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-spi.c +++ b/drivers/net/dsa/vitesse-vsc73xx-spi.c @@ -167,8 +167,6 @@ static void vsc73xx_spi_remove(struct spi_device *spi) return; vsc73xx_remove(&vsc_spi->vsc); - - spi_set_drvdata(spi, NULL); } static void vsc73xx_spi_shutdown(struct spi_device *spi) diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index 3887ed33c5fe2ab7f3c88701d89fdb2a37461162..fa622639d640138d96404a013ad5328f9bd41649 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -109,6 +109,7 @@ static void xrs700x_read_port_counters(struct xrs700x *priv, int port) { struct xrs700x_port *p = &priv->ports[port]; struct rtnl_link_stats64 stats; + unsigned long flags; int i; memset(&stats, 0, sizeof(stats)); @@ -138,9 +139,9 @@ static void xrs700x_read_port_counters(struct xrs700x *priv, int port) */ stats.rx_packets += stats.multicast; - u64_stats_update_begin(&p->syncp); + flags = u64_stats_update_begin_irqsave(&p->syncp); p->stats64 = stats; - u64_stats_update_end(&p->syncp); + u64_stats_update_end_irqrestore(&p->syncp, flags); mutex_unlock(&p->mib_mutex); } diff --git a/drivers/net/dsa/xrs700x/xrs700x_i2c.c b/drivers/net/dsa/xrs700x/xrs700x_i2c.c index 6deae388a0d6f89809fab4df62478a263fa85231..54065cdedd3572b1040acd9e450ae20523855d43 100644 --- a/drivers/net/dsa/xrs700x/xrs700x_i2c.c +++ b/drivers/net/dsa/xrs700x/xrs700x_i2c.c @@ -105,18 +105,14 @@ static int xrs700x_i2c_probe(struct i2c_client *i2c, return 0; } -static int xrs700x_i2c_remove(struct i2c_client *i2c) +static void xrs700x_i2c_remove(struct i2c_client *i2c) { struct xrs700x *priv = i2c_get_clientdata(i2c); if (!priv) - return 0; + return; xrs700x_switch_remove(priv); - - i2c_set_clientdata(i2c, NULL); - - return 0; } static void xrs700x_i2c_shutdown(struct i2c_client *i2c) diff --git a/drivers/net/dsa/xrs700x/xrs700x_mdio.c b/drivers/net/dsa/xrs700x/xrs700x_mdio.c index 127a677d1f391807109cfc4893ed7365d7c19238..5f7d344b5d7369c3bba7900d74dd6b8ba4f7718d 100644 --- a/drivers/net/dsa/xrs700x/xrs700x_mdio.c +++ b/drivers/net/dsa/xrs700x/xrs700x_mdio.c @@ -140,8 +140,6 @@ static void xrs700x_mdio_remove(struct mdio_device *mdiodev) return; xrs700x_switch_remove(priv); - - dev_set_drvdata(&mdiodev->dev, NULL); } static void xrs700x_mdio_shutdown(struct mdio_device *mdiodev) diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c index f82ad741950824afab584fca668137be76d42c7d..aa0fc00faecbe89f23044a3b8c1176513ccac418 100644 --- a/drivers/net/dummy.c +++ b/drivers/net/dummy.c @@ -102,7 +102,7 @@ static const struct net_device_ops dummy_netdev_ops = { static void dummy_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); } static const struct ethtool_ops dummy_ethtool_ops = { diff --git a/drivers/net/ethernet/3com/3c509.c b/drivers/net/ethernet/3com/3c509.c index 846fa3af45040c18c24f137c460ea287edd822fc..fb68339e1511c4c24ba1dfd3ef26d0d30f6a5202 100644 --- a/drivers/net/ethernet/3com/3c509.c +++ b/drivers/net/ethernet/3com/3c509.c @@ -1135,7 +1135,7 @@ el3_netdev_set_ecmd(struct net_device *dev, static void el3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); } static int el3_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/3com/3c515.c b/drivers/net/ethernet/3com/3c515.c index 1d124b0f65e7cee2042e424c16d7f0d8ec7a6d77..d2f4358cc5503e939d1fd23fbb37933bb494aa36 100644 --- a/drivers/net/ethernet/3com/3c515.c +++ b/drivers/net/ethernet/3com/3c515.c @@ -1527,7 +1527,7 @@ static void set_rx_mode(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); snprintf(info->bus_info, sizeof(info->bus_info), "ISA 0x%lx", dev->base_addr); } diff --git a/drivers/net/ethernet/3com/3c589_cs.c b/drivers/net/ethernet/3com/3c589_cs.c index 4673bc1604e7b263a81bbedbc7abc40af6320dff..82f94b1635bf816150e5f3c4f7d3e71417d81263 100644 --- a/drivers/net/ethernet/3com/3c589_cs.c +++ b/drivers/net/ethernet/3com/3c589_cs.c @@ -480,7 +480,7 @@ static void tc589_reset(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); snprintf(info->bus_info, sizeof(info->bus_info), "PCMCIA 0x%lx", dev->base_addr); } diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c index ccf07667aa5e65b4a7209676acaabbd42b4ae832..082388bb6169fa206514f71f5fae075668537544 100644 --- a/drivers/net/ethernet/3com/3c59x.c +++ b/drivers/net/ethernet/3com/3c59x.c @@ -2959,13 +2959,13 @@ static void vortex_get_drvinfo(struct net_device *dev, { struct vortex_private *vp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); if (VORTEX_PCI(vp)) { - strlcpy(info->bus_info, pci_name(VORTEX_PCI(vp)), + strscpy(info->bus_info, pci_name(VORTEX_PCI(vp)), sizeof(info->bus_info)); } else { if (VORTEX_EISA(vp)) - strlcpy(info->bus_info, dev_name(vp->gendev), + strscpy(info->bus_info, dev_name(vp->gendev), sizeof(info->bus_info)); else snprintf(info->bus_info, sizeof(info->bus_info), diff --git a/drivers/net/ethernet/3com/typhoon.c b/drivers/net/ethernet/3com/typhoon.c index cad4f354cc76f05f01dbbd8e93f04dc4f11389d7..aaaff3ba43ef839e780b88c16fc2d815ee03324d 100644 --- a/drivers/net/ethernet/3com/typhoon.c +++ b/drivers/net/ethernet/3com/typhoon.c @@ -969,12 +969,12 @@ typhoon_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) smp_rmb(); if (tp->card_state == Sleeping) { - strlcpy(info->fw_version, "Sleep image", + strscpy(info->fw_version, "Sleep image", sizeof(info->fw_version)); } else { INIT_COMMAND_WITH_RESPONSE(&xp_cmd, TYPHOON_CMD_READ_VERSIONS); if (typhoon_issue_command(tp, 1, &xp_cmd, 3, xp_resp) < 0) { - strlcpy(info->fw_version, "Unknown runtime", + strscpy(info->fw_version, "Unknown runtime", sizeof(info->fw_version)); } else { u32 sleep_ver = le32_to_cpu(xp_resp[0].parm2); @@ -984,8 +984,8 @@ typhoon_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) } } - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info)); } static int diff --git a/drivers/net/ethernet/8390/ax88796.c b/drivers/net/ethernet/8390/ax88796.c index 1f8acbba5b6b08cf7b8c3117aefa9bba1d919212..af603256b724a2cbf3f206e7c2731ee97053ad4d 100644 --- a/drivers/net/ethernet/8390/ax88796.c +++ b/drivers/net/ethernet/8390/ax88796.c @@ -579,9 +579,9 @@ static void ax_get_drvinfo(struct net_device *dev, { struct platform_device *pdev = to_platform_device(dev->dev.parent); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pdev->name, sizeof(info->bus_info)); } static u32 ax_get_msglevel(struct net_device *dev) diff --git a/drivers/net/ethernet/8390/etherh.c b/drivers/net/ethernet/8390/etherh.c index e7b879123bb19d4f3e69a4ff7c22bab411b6517a..05d39ecb97ffeeeb95c5fa84cbfc487ee6a61b73 100644 --- a/drivers/net/ethernet/8390/etherh.c +++ b/drivers/net/ethernet/8390/etherh.c @@ -555,9 +555,9 @@ static int __init etherm_addr(char *addr) static void etherh_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(dev->dev.parent), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/8390/mcf8390.c b/drivers/net/ethernet/8390/mcf8390.c index 21047ae1bc3d3d49700afe1a59550de0b4ddfdb3..8a7918d33419652c1ebe6474adcd43634858db99 100644 --- a/drivers/net/ethernet/8390/mcf8390.c +++ b/drivers/net/ethernet/8390/mcf8390.c @@ -450,8 +450,7 @@ static int mcf8390_remove(struct platform_device *pdev) unregister_netdev(dev); mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (mem) - release_mem_region(mem->start, resource_size(mem)); + release_mem_region(mem->start, resource_size(mem)); free_netdev(dev); return 0; } diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index 9a55c1d5a0a1aa1f0c02fca04853a506fe4f58e9..1917da784191988ea5c0cbc995ec946f647a4b29 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -121,6 +121,7 @@ config LANTIQ_XRX200 Support for the PMAC of the Gigabit switch (GSWIP) inside the Lantiq / Intel VRX200 VDSL SoC +source "drivers/net/ethernet/adi/Kconfig" source "drivers/net/ethernet/litex/Kconfig" source "drivers/net/ethernet/marvell/Kconfig" source "drivers/net/ethernet/mediatek/Kconfig" diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index c06e75ed4231aa354acbec3edf71a7adc1f80782..0d872d4efcd10b1d29752e7fcd6aede2ec657902 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_NET_VENDOR_8390) += 8390/ obj-$(CONFIG_NET_VENDOR_ACTIONS) += actions/ obj-$(CONFIG_NET_VENDOR_ADAPTEC) += adaptec/ obj-$(CONFIG_GRETH) += aeroflex/ +obj-$(CONFIG_NET_VENDOR_ADI) += adi/ obj-$(CONFIG_NET_VENDOR_AGERE) += agere/ obj-$(CONFIG_NET_VENDOR_ALACRITECH) += alacritech/ obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/ diff --git a/drivers/net/ethernet/actions/owl-emac.c b/drivers/net/ethernet/actions/owl-emac.c index 1cfdd01b4c2e33877ee310af5daa13ce135684a9..cd4d71b83c334211a98aa3e142c48db452a5e125 100644 --- a/drivers/net/ethernet/actions/owl-emac.c +++ b/drivers/net/ethernet/actions/owl-emac.c @@ -1576,7 +1576,7 @@ static int owl_emac_probe(struct platform_device *pdev) netdev->watchdog_timeo = OWL_EMAC_TX_TIMEOUT; netdev->netdev_ops = &owl_emac_netdev_ops; netdev->ethtool_ops = &owl_emac_ethtool_ops; - netif_napi_add(netdev, &priv->napi, owl_emac_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &priv->napi, owl_emac_poll); ret = devm_register_netdev(dev, netdev); if (ret) { diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c index 8f0a6b9c518e40a9c13d58d2a8ea7f922dda7e70..857361c74f5dd51bf6d685cadca9732e64c65ca7 100644 --- a/drivers/net/ethernet/adaptec/starfire.c +++ b/drivers/net/ethernet/adaptec/starfire.c @@ -1844,8 +1844,8 @@ static int check_if_running(struct net_device *dev) static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/adi/Kconfig b/drivers/net/ethernet/adi/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..da3bdd3025022c3dd7286c3b7873e1a108767025 --- /dev/null +++ b/drivers/net/ethernet/adi/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +# +# Analog Devices device configuration +# + +config NET_VENDOR_ADI + bool "Analog Devices devices" + default y + depends on SPI + help + If you have a network (Ethernet) card belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all + the questions about ADI devices. If you say Y, you will be asked + for your specific card in the following questions. + +if NET_VENDOR_ADI + +config ADIN1110 + tristate "Analog Devices ADIN1110 MAC-PHY" + depends on SPI && NET_SWITCHDEV + select CRC8 + help + Say yes here to build support for Analog Devices ADIN1110 + Low Power 10BASE-T1L Ethernet MAC-PHY. + +endif # NET_VENDOR_ADI diff --git a/drivers/net/ethernet/adi/Makefile b/drivers/net/ethernet/adi/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d0383d94303c801328cfc0a9f118b12eb56376f7 --- /dev/null +++ b/drivers/net/ethernet/adi/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +# +# Makefile for the Analog Devices network device drivers. +# + +obj-$(CONFIG_ADIN1110) += adin1110.o diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c new file mode 100644 index 0000000000000000000000000000000000000000..1744d623999d09b33dfbb643b7a0ff9a95610afe --- /dev/null +++ b/drivers/net/ethernet/adi/adin1110.c @@ -0,0 +1,1700 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* ADIN1110 Low Power 10BASE-T1L Ethernet MAC-PHY + * ADIN2111 2-Port Ethernet Switch with Integrated 10BASE-T1L PHY + * + * Copyright 2021 Analog Devices Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#define ADIN1110_PHY_ID 0x1 + +#define ADIN1110_RESET 0x03 +#define ADIN1110_SWRESET BIT(0) + +#define ADIN1110_CONFIG1 0x04 +#define ADIN1110_CONFIG1_SYNC BIT(15) + +#define ADIN1110_CONFIG2 0x06 +#define ADIN2111_P2_FWD_UNK2HOST BIT(12) +#define ADIN2111_PORT_CUT_THRU_EN BIT(11) +#define ADIN1110_CRC_APPEND BIT(5) +#define ADIN1110_FWD_UNK2HOST BIT(2) + +#define ADIN1110_STATUS0 0x08 + +#define ADIN1110_STATUS1 0x09 +#define ADIN2111_P2_RX_RDY BIT(17) +#define ADIN1110_SPI_ERR BIT(10) +#define ADIN1110_RX_RDY BIT(4) + +#define ADIN1110_IMASK1 0x0D +#define ADIN2111_RX_RDY_IRQ BIT(17) +#define ADIN1110_SPI_ERR_IRQ BIT(10) +#define ADIN1110_RX_RDY_IRQ BIT(4) +#define ADIN1110_TX_RDY_IRQ BIT(3) + +#define ADIN1110_MDIOACC 0x20 +#define ADIN1110_MDIO_TRDONE BIT(31) +#define ADIN1110_MDIO_ST GENMASK(29, 28) +#define ADIN1110_MDIO_OP GENMASK(27, 26) +#define ADIN1110_MDIO_PRTAD GENMASK(25, 21) +#define ADIN1110_MDIO_DEVAD GENMASK(20, 16) +#define ADIN1110_MDIO_DATA GENMASK(15, 0) + +#define ADIN1110_TX_FSIZE 0x30 +#define ADIN1110_TX 0x31 +#define ADIN1110_TX_SPACE 0x32 + +#define ADIN1110_MAC_ADDR_FILTER_UPR 0x50 +#define ADIN2111_MAC_ADDR_APPLY2PORT2 BIT(31) +#define ADIN1110_MAC_ADDR_APPLY2PORT BIT(30) +#define ADIN2111_MAC_ADDR_TO_OTHER_PORT BIT(17) +#define ADIN1110_MAC_ADDR_TO_HOST BIT(16) + +#define ADIN1110_MAC_ADDR_FILTER_LWR 0x51 + +#define ADIN1110_MAC_ADDR_MASK_UPR 0x70 +#define ADIN1110_MAC_ADDR_MASK_LWR 0x71 + +#define ADIN1110_RX_FSIZE 0x90 +#define ADIN1110_RX 0x91 + +#define ADIN2111_RX_P2_FSIZE 0xC0 +#define ADIN2111_RX_P2 0xC1 + +#define ADIN1110_CLEAR_STATUS0 0xFFF + +/* MDIO_OP codes */ +#define ADIN1110_MDIO_OP_WR 0x1 +#define ADIN1110_MDIO_OP_RD 0x3 + +#define ADIN1110_CD BIT(7) +#define ADIN1110_WRITE BIT(5) + +#define ADIN1110_MAX_BUFF 2048 +#define ADIN1110_MAX_FRAMES_READ 64 +#define ADIN1110_WR_HEADER_LEN 2 +#define ADIN1110_FRAME_HEADER_LEN 2 +#define ADIN1110_INTERNAL_SIZE_HEADER_LEN 2 +#define ADIN1110_RD_HEADER_LEN 3 +#define ADIN1110_REG_LEN 4 +#define ADIN1110_FEC_LEN 4 + +#define ADIN1110_PHY_ID_VAL 0x0283BC91 +#define ADIN2111_PHY_ID_VAL 0x0283BCA1 + +#define ADIN_MAC_MAX_PORTS 2 +#define ADIN_MAC_MAX_ADDR_SLOTS 16 + +#define ADIN_MAC_MULTICAST_ADDR_SLOT 0 +#define ADIN_MAC_BROADCAST_ADDR_SLOT 1 +#define ADIN_MAC_P1_ADDR_SLOT 2 +#define ADIN_MAC_P2_ADDR_SLOT 3 +#define ADIN_MAC_FDB_ADDR_SLOT 4 + +DECLARE_CRC8_TABLE(adin1110_crc_table); + +enum adin1110_chips_id { + ADIN1110_MAC = 0, + ADIN2111_MAC, +}; + +struct adin1110_cfg { + enum adin1110_chips_id id; + char name[MDIO_NAME_SIZE]; + u32 phy_ids[PHY_MAX_ADDR]; + u32 ports_nr; + u32 phy_id_val; +}; + +struct adin1110_port_priv { + struct adin1110_priv *priv; + struct net_device *netdev; + struct net_device *bridge; + struct phy_device *phydev; + struct work_struct tx_work; + u64 rx_packets; + u64 tx_packets; + u64 rx_bytes; + u64 tx_bytes; + struct work_struct rx_mode_work; + u32 flags; + struct sk_buff_head txq; + u32 nr; + u32 state; + struct adin1110_cfg *cfg; +}; + +struct adin1110_priv { + struct mutex lock; /* protect spi */ + spinlock_t state_lock; /* protect RX mode */ + struct mii_bus *mii_bus; + struct spi_device *spidev; + bool append_crc; + struct adin1110_cfg *cfg; + u32 tx_space; + u32 irq_mask; + bool forwarding; + int irq; + struct adin1110_port_priv *ports[ADIN_MAC_MAX_PORTS]; + char mii_bus_name[MII_BUS_ID_SIZE]; + u8 data[ADIN1110_MAX_BUFF] ____cacheline_aligned; +}; + +struct adin1110_switchdev_event_work { + struct work_struct work; + struct switchdev_notifier_fdb_info fdb_info; + struct adin1110_port_priv *port_priv; + unsigned long event; +}; + +static struct adin1110_cfg adin1110_cfgs[] = { + { + .id = ADIN1110_MAC, + .name = "adin1110", + .phy_ids = {1}, + .ports_nr = 1, + .phy_id_val = ADIN1110_PHY_ID_VAL, + }, + { + .id = ADIN2111_MAC, + .name = "adin2111", + .phy_ids = {1, 2}, + .ports_nr = 2, + .phy_id_val = ADIN2111_PHY_ID_VAL, + }, +}; + +static u8 adin1110_crc_data(u8 *data, u32 len) +{ + return crc8(adin1110_crc_table, data, len, 0); +} + +static int adin1110_read_reg(struct adin1110_priv *priv, u16 reg, u32 *val) +{ + u32 header_len = ADIN1110_RD_HEADER_LEN; + u32 read_len = ADIN1110_REG_LEN; + struct spi_transfer t[2] = {0}; + int ret; + + priv->data[0] = ADIN1110_CD | FIELD_GET(GENMASK(12, 8), reg); + priv->data[1] = FIELD_GET(GENMASK(7, 0), reg); + priv->data[2] = 0x00; + + if (priv->append_crc) { + priv->data[2] = adin1110_crc_data(&priv->data[0], 2); + priv->data[3] = 0x00; + header_len++; + } + + t[0].tx_buf = &priv->data[0]; + t[0].len = header_len; + + if (priv->append_crc) + read_len++; + + memset(&priv->data[header_len], 0, read_len); + t[1].rx_buf = &priv->data[header_len]; + t[1].len = read_len; + + ret = spi_sync_transfer(priv->spidev, t, 2); + if (ret) + return ret; + + if (priv->append_crc) { + u8 recv_crc; + u8 crc; + + crc = adin1110_crc_data(&priv->data[header_len], + ADIN1110_REG_LEN); + recv_crc = priv->data[header_len + ADIN1110_REG_LEN]; + + if (crc != recv_crc) { + dev_err_ratelimited(&priv->spidev->dev, "CRC error."); + return -EBADMSG; + } + } + + *val = get_unaligned_be32(&priv->data[header_len]); + + return ret; +} + +static int adin1110_write_reg(struct adin1110_priv *priv, u16 reg, u32 val) +{ + u32 header_len = ADIN1110_WR_HEADER_LEN; + u32 write_len = ADIN1110_REG_LEN; + + priv->data[0] = ADIN1110_CD | ADIN1110_WRITE | FIELD_GET(GENMASK(12, 8), reg); + priv->data[1] = FIELD_GET(GENMASK(7, 0), reg); + + if (priv->append_crc) { + priv->data[2] = adin1110_crc_data(&priv->data[0], header_len); + header_len++; + } + + put_unaligned_be32(val, &priv->data[header_len]); + if (priv->append_crc) { + priv->data[header_len + write_len] = adin1110_crc_data(&priv->data[header_len], + write_len); + write_len++; + } + + return spi_write(priv->spidev, &priv->data[0], header_len + write_len); +} + +static int adin1110_set_bits(struct adin1110_priv *priv, u16 reg, + unsigned long mask, unsigned long val) +{ + u32 write_val; + int ret; + + ret = adin1110_read_reg(priv, reg, &write_val); + if (ret < 0) + return ret; + + set_mask_bits(&write_val, mask, val); + + return adin1110_write_reg(priv, reg, write_val); +} + +static int adin1110_round_len(int len) +{ + /* can read/write only mutiples of 4 bytes of payload */ + len = ALIGN(len, 4); + + /* NOTE: ADIN1110_WR_HEADER_LEN should be used for write ops. */ + if (len + ADIN1110_RD_HEADER_LEN > ADIN1110_MAX_BUFF) + return -EINVAL; + + return len; +} + +static int adin1110_read_fifo(struct adin1110_port_priv *port_priv) +{ + struct adin1110_priv *priv = port_priv->priv; + u32 header_len = ADIN1110_RD_HEADER_LEN; + struct spi_transfer t[2] = {0}; + u32 frame_size_no_fcs; + struct sk_buff *rxb; + u32 frame_size; + int round_len; + u16 reg; + int ret; + + if (!port_priv->nr) { + reg = ADIN1110_RX; + ret = adin1110_read_reg(priv, ADIN1110_RX_FSIZE, &frame_size); + } else { + reg = ADIN2111_RX_P2; + ret = adin1110_read_reg(priv, ADIN2111_RX_P2_FSIZE, + &frame_size); + } + + if (ret < 0) + return ret; + + /* The read frame size includes the extra 2 bytes + * from the ADIN1110 frame header. + */ + if (frame_size < ADIN1110_FRAME_HEADER_LEN + ADIN1110_FEC_LEN) + return ret; + + round_len = adin1110_round_len(frame_size); + if (round_len < 0) + return ret; + + frame_size_no_fcs = frame_size - ADIN1110_FRAME_HEADER_LEN - ADIN1110_FEC_LEN; + + rxb = netdev_alloc_skb(port_priv->netdev, round_len); + if (!rxb) + return -ENOMEM; + + memset(priv->data, 0, round_len + ADIN1110_RD_HEADER_LEN); + + priv->data[0] = ADIN1110_CD | FIELD_GET(GENMASK(12, 8), reg); + priv->data[1] = FIELD_GET(GENMASK(7, 0), reg); + + if (priv->append_crc) { + priv->data[2] = adin1110_crc_data(&priv->data[0], 2); + header_len++; + } + + skb_put(rxb, frame_size_no_fcs + ADIN1110_FRAME_HEADER_LEN); + + t[0].tx_buf = &priv->data[0]; + t[0].len = header_len; + + t[1].rx_buf = &rxb->data[0]; + t[1].len = round_len; + + ret = spi_sync_transfer(priv->spidev, t, 2); + if (ret) { + kfree_skb(rxb); + return ret; + } + + skb_pull(rxb, ADIN1110_FRAME_HEADER_LEN); + rxb->protocol = eth_type_trans(rxb, port_priv->netdev); + + if ((port_priv->flags & IFF_ALLMULTI && rxb->pkt_type == PACKET_MULTICAST) || + (port_priv->flags & IFF_BROADCAST && rxb->pkt_type == PACKET_BROADCAST)) + rxb->offload_fwd_mark = 1; + + netif_rx(rxb); + + port_priv->rx_bytes += frame_size - ADIN1110_FRAME_HEADER_LEN; + port_priv->rx_packets++; + + return 0; +} + +static int adin1110_write_fifo(struct adin1110_port_priv *port_priv, + struct sk_buff *txb) +{ + struct adin1110_priv *priv = port_priv->priv; + u32 header_len = ADIN1110_WR_HEADER_LEN; + __be16 frame_header; + int padding = 0; + int padded_len; + int round_len; + int ret; + + /* Pad frame to 64 byte length, + * MAC nor PHY will otherwise add the + * required padding. + * The FEC will be added by the MAC internally. + */ + if (txb->len + ADIN1110_FEC_LEN < 64) + padding = 64 - (txb->len + ADIN1110_FEC_LEN); + + padded_len = txb->len + padding + ADIN1110_FRAME_HEADER_LEN; + + round_len = adin1110_round_len(padded_len); + if (round_len < 0) + return round_len; + + ret = adin1110_write_reg(priv, ADIN1110_TX_FSIZE, padded_len); + if (ret < 0) + return ret; + + memset(priv->data, 0, round_len + ADIN1110_WR_HEADER_LEN); + + priv->data[0] = ADIN1110_CD | ADIN1110_WRITE; + priv->data[0] |= FIELD_GET(GENMASK(12, 8), ADIN1110_TX); + priv->data[1] = FIELD_GET(GENMASK(7, 0), ADIN1110_TX); + if (priv->append_crc) { + priv->data[2] = adin1110_crc_data(&priv->data[0], 2); + header_len++; + } + + /* mention the port on which to send the frame in the frame header */ + frame_header = cpu_to_be16(port_priv->nr); + memcpy(&priv->data[header_len], &frame_header, + ADIN1110_FRAME_HEADER_LEN); + + memcpy(&priv->data[header_len + ADIN1110_FRAME_HEADER_LEN], + txb->data, txb->len); + + ret = spi_write(priv->spidev, &priv->data[0], round_len + header_len); + if (ret < 0) + return ret; + + port_priv->tx_bytes += txb->len; + port_priv->tx_packets++; + + return 0; +} + +static int adin1110_read_mdio_acc(struct adin1110_priv *priv) +{ + u32 val; + int ret; + + mutex_lock(&priv->lock); + ret = adin1110_read_reg(priv, ADIN1110_MDIOACC, &val); + mutex_unlock(&priv->lock); + if (ret < 0) + return 0; + + return val; +} + +static int adin1110_mdio_read(struct mii_bus *bus, int phy_id, int reg) +{ + struct adin1110_priv *priv = bus->priv; + u32 val = 0; + int ret; + + if (mdio_phy_id_is_c45(phy_id)) + return -EOPNOTSUPP; + + val |= FIELD_PREP(ADIN1110_MDIO_OP, ADIN1110_MDIO_OP_RD); + val |= FIELD_PREP(ADIN1110_MDIO_ST, 0x1); + val |= FIELD_PREP(ADIN1110_MDIO_PRTAD, phy_id); + val |= FIELD_PREP(ADIN1110_MDIO_DEVAD, reg); + + /* write the clause 22 read command to the chip */ + mutex_lock(&priv->lock); + ret = adin1110_write_reg(priv, ADIN1110_MDIOACC, val); + mutex_unlock(&priv->lock); + if (ret < 0) + return ret; + + /* ADIN1110_MDIO_TRDONE BIT of the ADIN1110_MDIOACC + * register is set when the read is done. + * After the transaction is done, ADIN1110_MDIO_DATA + * bitfield of ADIN1110_MDIOACC register will contain + * the requested register value. + */ + ret = readx_poll_timeout(adin1110_read_mdio_acc, priv, val, + (val & ADIN1110_MDIO_TRDONE), 10000, 30000); + if (ret < 0) + return ret; + + return (val & ADIN1110_MDIO_DATA); +} + +static int adin1110_mdio_write(struct mii_bus *bus, int phy_id, + int reg, u16 reg_val) +{ + struct adin1110_priv *priv = bus->priv; + u32 val = 0; + int ret; + + if (mdio_phy_id_is_c45(phy_id)) + return -EOPNOTSUPP; + + val |= FIELD_PREP(ADIN1110_MDIO_OP, ADIN1110_MDIO_OP_WR); + val |= FIELD_PREP(ADIN1110_MDIO_ST, 0x1); + val |= FIELD_PREP(ADIN1110_MDIO_PRTAD, phy_id); + val |= FIELD_PREP(ADIN1110_MDIO_DEVAD, reg); + val |= FIELD_PREP(ADIN1110_MDIO_DATA, reg_val); + + /* write the clause 22 write command to the chip */ + mutex_lock(&priv->lock); + ret = adin1110_write_reg(priv, ADIN1110_MDIOACC, val); + mutex_unlock(&priv->lock); + if (ret < 0) + return ret; + + return readx_poll_timeout(adin1110_read_mdio_acc, priv, val, + (val & ADIN1110_MDIO_TRDONE), 10000, 30000); +} + +/* ADIN1110 MAC-PHY contains an ADIN1100 PHY. + * ADIN2111 MAC-PHY contains two ADIN1100 PHYs. + * By registering a new MDIO bus we allow the PAL to discover + * the encapsulated PHY and probe the ADIN1100 driver. + */ +static int adin1110_register_mdiobus(struct adin1110_priv *priv, + struct device *dev) +{ + struct mii_bus *mii_bus; + int ret; + + mii_bus = devm_mdiobus_alloc(dev); + if (!mii_bus) + return -ENOMEM; + + snprintf(priv->mii_bus_name, MII_BUS_ID_SIZE, "%s-%u", + priv->cfg->name, priv->spidev->chip_select); + + mii_bus->name = priv->mii_bus_name; + mii_bus->read = adin1110_mdio_read; + mii_bus->write = adin1110_mdio_write; + mii_bus->priv = priv; + mii_bus->parent = dev; + mii_bus->phy_mask = ~((u32)GENMASK(2, 0)); + mii_bus->probe_capabilities = MDIOBUS_C22; + snprintf(mii_bus->id, MII_BUS_ID_SIZE, "%s", dev_name(dev)); + + ret = devm_mdiobus_register(dev, mii_bus); + if (ret) + return ret; + + priv->mii_bus = mii_bus; + + return 0; +} + +static bool adin1110_port_rx_ready(struct adin1110_port_priv *port_priv, + u32 status) +{ + if (!netif_oper_up(port_priv->netdev)) + return false; + + if (!port_priv->nr) + return !!(status & ADIN1110_RX_RDY); + else + return !!(status & ADIN2111_P2_RX_RDY); +} + +static void adin1110_read_frames(struct adin1110_port_priv *port_priv, + unsigned int budget) +{ + struct adin1110_priv *priv = port_priv->priv; + u32 status1; + int ret; + + while (budget) { + ret = adin1110_read_reg(priv, ADIN1110_STATUS1, &status1); + if (ret < 0) + return; + + if (!adin1110_port_rx_ready(port_priv, status1)) + break; + + ret = adin1110_read_fifo(port_priv); + if (ret < 0) + return; + + budget--; + } +} + +static void adin1110_wake_queues(struct adin1110_priv *priv) +{ + int i; + + for (i = 0; i < priv->cfg->ports_nr; i++) + netif_wake_queue(priv->ports[i]->netdev); +} + +static irqreturn_t adin1110_irq(int irq, void *p) +{ + struct adin1110_priv *priv = p; + u32 status1; + u32 val; + int ret; + int i; + + mutex_lock(&priv->lock); + + ret = adin1110_read_reg(priv, ADIN1110_STATUS1, &status1); + if (ret < 0) + goto out; + + if (priv->append_crc && (status1 & ADIN1110_SPI_ERR)) + dev_warn_ratelimited(&priv->spidev->dev, + "SPI CRC error on write.\n"); + + ret = adin1110_read_reg(priv, ADIN1110_TX_SPACE, &val); + if (ret < 0) + goto out; + + /* TX FIFO space is expressed in half-words */ + priv->tx_space = 2 * val; + + for (i = 0; i < priv->cfg->ports_nr; i++) { + if (adin1110_port_rx_ready(priv->ports[i], status1)) + adin1110_read_frames(priv->ports[i], + ADIN1110_MAX_FRAMES_READ); + } + + /* clear IRQ sources */ + adin1110_write_reg(priv, ADIN1110_STATUS0, ADIN1110_CLEAR_STATUS0); + adin1110_write_reg(priv, ADIN1110_STATUS1, priv->irq_mask); + +out: + mutex_unlock(&priv->lock); + + if (priv->tx_space > 0 && ret >= 0) + adin1110_wake_queues(priv); + + return IRQ_HANDLED; +} + +/* ADIN1110 can filter up to 16 MAC addresses, mac_nr here is the slot used */ +static int adin1110_write_mac_address(struct adin1110_port_priv *port_priv, + int mac_nr, const u8 *addr, + u8 *mask, u32 port_rules) +{ + struct adin1110_priv *priv = port_priv->priv; + u32 offset = mac_nr * 2; + u32 port_rules_mask; + int ret; + u32 val; + + if (!port_priv->nr) + port_rules_mask = ADIN1110_MAC_ADDR_APPLY2PORT; + else + port_rules_mask = ADIN2111_MAC_ADDR_APPLY2PORT2; + + if (port_rules & port_rules_mask) + port_rules_mask |= ADIN1110_MAC_ADDR_TO_HOST | ADIN2111_MAC_ADDR_TO_OTHER_PORT; + + port_rules_mask |= GENMASK(15, 0); + val = port_rules | get_unaligned_be16(&addr[0]); + ret = adin1110_set_bits(priv, ADIN1110_MAC_ADDR_FILTER_UPR + offset, + port_rules_mask, val); + if (ret < 0) + return ret; + + val = get_unaligned_be32(&addr[2]); + ret = adin1110_write_reg(priv, + ADIN1110_MAC_ADDR_FILTER_LWR + offset, val); + if (ret < 0) + return ret; + + /* Only the first two MAC address slots support masking. */ + if (mac_nr < ADIN_MAC_P1_ADDR_SLOT) { + val = get_unaligned_be16(&mask[0]); + ret = adin1110_write_reg(priv, + ADIN1110_MAC_ADDR_MASK_UPR + offset, + val); + if (ret < 0) + return ret; + + val = get_unaligned_be32(&mask[2]); + return adin1110_write_reg(priv, + ADIN1110_MAC_ADDR_MASK_LWR + offset, + val); + } + + return 0; +} + +static int adin1110_clear_mac_address(struct adin1110_priv *priv, int mac_nr) +{ + u32 offset = mac_nr * 2; + int ret; + + ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + offset, 0); + if (ret < 0) + return ret; + + ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_FILTER_LWR + offset, 0); + if (ret < 0) + return ret; + + /* only the first two MAC address slots are maskable */ + if (mac_nr <= 1) { + ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_MASK_UPR + offset, 0); + if (ret < 0) + return ret; + + ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_MASK_LWR + offset, 0); + } + + return ret; +} + +static u32 adin1110_port_rules(struct adin1110_port_priv *port_priv, + bool fw_to_host, + bool fw_to_other_port) +{ + u32 port_rules = 0; + + if (!port_priv->nr) + port_rules |= ADIN1110_MAC_ADDR_APPLY2PORT; + else + port_rules |= ADIN2111_MAC_ADDR_APPLY2PORT2; + + if (fw_to_host) + port_rules |= ADIN1110_MAC_ADDR_TO_HOST; + + if (fw_to_other_port && port_priv->priv->forwarding) + port_rules |= ADIN2111_MAC_ADDR_TO_OTHER_PORT; + + return port_rules; +} + +static int adin1110_multicast_filter(struct adin1110_port_priv *port_priv, + int mac_nr, bool accept_multicast) +{ + u8 mask[ETH_ALEN] = {0}; + u8 mac[ETH_ALEN] = {0}; + u32 port_rules = 0; + + mask[0] = BIT(0); + mac[0] = BIT(0); + + if (accept_multicast && port_priv->state == BR_STATE_FORWARDING) + port_rules = adin1110_port_rules(port_priv, true, true); + + return adin1110_write_mac_address(port_priv, mac_nr, mac, + mask, port_rules); +} + +static int adin1110_broadcasts_filter(struct adin1110_port_priv *port_priv, + int mac_nr, bool accept_broadcast) +{ + u32 port_rules = 0; + u8 mask[ETH_ALEN]; + + memset(mask, 0xFF, ETH_ALEN); + + if (accept_broadcast && port_priv->state == BR_STATE_FORWARDING) + port_rules = adin1110_port_rules(port_priv, true, true); + + return adin1110_write_mac_address(port_priv, mac_nr, mask, + mask, port_rules); +} + +static int adin1110_set_mac_address(struct net_device *netdev, + const unsigned char *dev_addr) +{ + struct adin1110_port_priv *port_priv = netdev_priv(netdev); + u8 mask[ETH_ALEN]; + u32 port_rules; + u32 mac_slot; + + if (!is_valid_ether_addr(dev_addr)) + return -EADDRNOTAVAIL; + + eth_hw_addr_set(netdev, dev_addr); + memset(mask, 0xFF, ETH_ALEN); + + mac_slot = (!port_priv->nr) ? ADIN_MAC_P1_ADDR_SLOT : ADIN_MAC_P2_ADDR_SLOT; + port_rules = adin1110_port_rules(port_priv, true, false); + + return adin1110_write_mac_address(port_priv, mac_slot, netdev->dev_addr, + mask, port_rules); +} + +static int adin1110_ndo_set_mac_address(struct net_device *netdev, void *addr) +{ + struct sockaddr *sa = addr; + int ret; + + ret = eth_prepare_mac_addr_change(netdev, addr); + if (ret < 0) + return ret; + + return adin1110_set_mac_address(netdev, sa->sa_data); +} + +static int adin1110_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) +{ + if (!netif_running(netdev)) + return -EINVAL; + + return phy_do_ioctl(netdev, rq, cmd); +} + +static int adin1110_set_promisc_mode(struct adin1110_port_priv *port_priv, + bool promisc) +{ + struct adin1110_priv *priv = port_priv->priv; + u32 mask; + + if (port_priv->state != BR_STATE_FORWARDING) + promisc = false; + + if (!port_priv->nr) + mask = ADIN1110_FWD_UNK2HOST; + else + mask = ADIN2111_P2_FWD_UNK2HOST; + + return adin1110_set_bits(priv, ADIN1110_CONFIG2, + mask, promisc ? mask : 0); +} + +static int adin1110_setup_rx_mode(struct adin1110_port_priv *port_priv) +{ + int ret; + + ret = adin1110_set_promisc_mode(port_priv, + !!(port_priv->flags & IFF_PROMISC)); + if (ret < 0) + return ret; + + ret = adin1110_multicast_filter(port_priv, ADIN_MAC_MULTICAST_ADDR_SLOT, + !!(port_priv->flags & IFF_ALLMULTI)); + if (ret < 0) + return ret; + + ret = adin1110_broadcasts_filter(port_priv, + ADIN_MAC_BROADCAST_ADDR_SLOT, + !!(port_priv->flags & IFF_BROADCAST)); + if (ret < 0) + return ret; + + return adin1110_set_bits(port_priv->priv, ADIN1110_CONFIG1, + ADIN1110_CONFIG1_SYNC, ADIN1110_CONFIG1_SYNC); +} + +static bool adin1110_can_offload_forwarding(struct adin1110_priv *priv) +{ + int i; + + if (priv->cfg->id != ADIN2111_MAC) + return false; + + /* Can't enable forwarding if ports do not belong to the same bridge */ + if (priv->ports[0]->bridge != priv->ports[1]->bridge || !priv->ports[0]->bridge) + return false; + + /* Can't enable forwarding if there is a port + * that has been blocked by STP. + */ + for (i = 0; i < priv->cfg->ports_nr; i++) { + if (priv->ports[i]->state != BR_STATE_FORWARDING) + return false; + } + + return true; +} + +static void adin1110_rx_mode_work(struct work_struct *work) +{ + struct adin1110_port_priv *port_priv; + struct adin1110_priv *priv; + + port_priv = container_of(work, struct adin1110_port_priv, rx_mode_work); + priv = port_priv->priv; + + mutex_lock(&priv->lock); + adin1110_setup_rx_mode(port_priv); + mutex_unlock(&priv->lock); +} + +static void adin1110_set_rx_mode(struct net_device *dev) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + struct adin1110_priv *priv = port_priv->priv; + + spin_lock(&priv->state_lock); + + port_priv->flags = dev->flags; + schedule_work(&port_priv->rx_mode_work); + + spin_unlock(&priv->state_lock); +} + +static int adin1110_net_open(struct net_device *net_dev) +{ + struct adin1110_port_priv *port_priv = netdev_priv(net_dev); + struct adin1110_priv *priv = port_priv->priv; + u32 val; + int ret; + + mutex_lock(&priv->lock); + + /* Configure MAC to compute and append the FCS itself. */ + ret = adin1110_write_reg(priv, ADIN1110_CONFIG2, ADIN1110_CRC_APPEND); + if (ret < 0) + goto out; + + val = ADIN1110_TX_RDY_IRQ | ADIN1110_RX_RDY_IRQ | ADIN1110_SPI_ERR_IRQ; + if (priv->cfg->id == ADIN2111_MAC) + val |= ADIN2111_RX_RDY_IRQ; + + priv->irq_mask = val; + ret = adin1110_write_reg(priv, ADIN1110_IMASK1, ~val); + if (ret < 0) { + netdev_err(net_dev, "Failed to enable chip IRQs: %d\n", ret); + goto out; + } + + ret = adin1110_read_reg(priv, ADIN1110_TX_SPACE, &val); + if (ret < 0) { + netdev_err(net_dev, "Failed to read TX FIFO space: %d\n", ret); + goto out; + } + + priv->tx_space = 2 * val; + + port_priv->state = BR_STATE_FORWARDING; + ret = adin1110_set_mac_address(net_dev, net_dev->dev_addr); + if (ret < 0) { + netdev_err(net_dev, "Could not set MAC address: %pM, %d\n", + net_dev->dev_addr, ret); + goto out; + } + + ret = adin1110_set_bits(priv, ADIN1110_CONFIG1, ADIN1110_CONFIG1_SYNC, + ADIN1110_CONFIG1_SYNC); + +out: + mutex_unlock(&priv->lock); + + if (ret < 0) + return ret; + + phy_start(port_priv->phydev); + + netif_start_queue(net_dev); + + return 0; +} + +static int adin1110_net_stop(struct net_device *net_dev) +{ + struct adin1110_port_priv *port_priv = netdev_priv(net_dev); + struct adin1110_priv *priv = port_priv->priv; + u32 mask; + int ret; + + mask = !port_priv->nr ? ADIN2111_RX_RDY_IRQ : ADIN1110_RX_RDY_IRQ; + + /* Disable RX RDY IRQs */ + mutex_lock(&priv->lock); + ret = adin1110_set_bits(priv, ADIN1110_IMASK1, mask, mask); + mutex_unlock(&priv->lock); + if (ret < 0) + return ret; + + netif_stop_queue(port_priv->netdev); + flush_work(&port_priv->tx_work); + phy_stop(port_priv->phydev); + + return 0; +} + +static void adin1110_tx_work(struct work_struct *work) +{ + struct adin1110_port_priv *port_priv; + struct adin1110_priv *priv; + struct sk_buff *txb; + int ret; + + port_priv = container_of(work, struct adin1110_port_priv, tx_work); + priv = port_priv->priv; + + mutex_lock(&priv->lock); + + while ((txb = skb_dequeue(&port_priv->txq))) { + ret = adin1110_write_fifo(port_priv, txb); + if (ret < 0) + dev_err_ratelimited(&priv->spidev->dev, + "Frame write error: %d\n", ret); + + dev_kfree_skb(txb); + } + + mutex_unlock(&priv->lock); +} + +static netdev_tx_t adin1110_start_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + struct adin1110_priv *priv = port_priv->priv; + netdev_tx_t netdev_ret = NETDEV_TX_OK; + u32 tx_space_needed; + + tx_space_needed = skb->len + ADIN1110_FRAME_HEADER_LEN + ADIN1110_INTERNAL_SIZE_HEADER_LEN; + if (tx_space_needed > priv->tx_space) { + netif_stop_queue(dev); + netdev_ret = NETDEV_TX_BUSY; + } else { + priv->tx_space -= tx_space_needed; + skb_queue_tail(&port_priv->txq, skb); + } + + schedule_work(&port_priv->tx_work); + + return netdev_ret; +} + +static void adin1110_ndo_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *storage) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + + storage->rx_packets = port_priv->rx_packets; + storage->tx_packets = port_priv->tx_packets; + + storage->rx_bytes = port_priv->rx_bytes; + storage->tx_bytes = port_priv->tx_bytes; +} + +static int adin1110_port_get_port_parent_id(struct net_device *dev, + struct netdev_phys_item_id *ppid) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + struct adin1110_priv *priv = port_priv->priv; + + ppid->id_len = strnlen(priv->mii_bus_name, MAX_PHYS_ITEM_ID_LEN); + memcpy(ppid->id, priv->mii_bus_name, ppid->id_len); + + return 0; +} + +static int adin1110_ndo_get_phys_port_name(struct net_device *dev, + char *name, size_t len) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + int err; + + err = snprintf(name, len, "p%d", port_priv->nr); + if (err >= len) + return -EINVAL; + + return 0; +} + +static const struct net_device_ops adin1110_netdev_ops = { + .ndo_open = adin1110_net_open, + .ndo_stop = adin1110_net_stop, + .ndo_eth_ioctl = adin1110_ioctl, + .ndo_start_xmit = adin1110_start_xmit, + .ndo_set_mac_address = adin1110_ndo_set_mac_address, + .ndo_set_rx_mode = adin1110_set_rx_mode, + .ndo_validate_addr = eth_validate_addr, + .ndo_get_stats64 = adin1110_ndo_get_stats64, + .ndo_get_port_parent_id = adin1110_port_get_port_parent_id, + .ndo_get_phys_port_name = adin1110_ndo_get_phys_port_name, +}; + +static void adin1110_get_drvinfo(struct net_device *dev, + struct ethtool_drvinfo *di) +{ + strscpy(di->driver, "ADIN1110", sizeof(di->driver)); + strscpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info)); +} + +static const struct ethtool_ops adin1110_ethtool_ops = { + .get_drvinfo = adin1110_get_drvinfo, + .get_link = ethtool_op_get_link, + .get_link_ksettings = phy_ethtool_get_link_ksettings, + .set_link_ksettings = phy_ethtool_set_link_ksettings, +}; + +static void adin1110_adjust_link(struct net_device *dev) +{ + struct phy_device *phydev = dev->phydev; + + if (!phydev->link) + phy_print_status(phydev); +} + +/* PHY ID is stored in the MAC registers too, + * check spi connection by reading it. + */ +static int adin1110_check_spi(struct adin1110_priv *priv) +{ + int ret; + u32 val; + + ret = adin1110_read_reg(priv, ADIN1110_PHY_ID, &val); + if (ret < 0) + return ret; + + if (val != priv->cfg->phy_id_val) { + dev_err(&priv->spidev->dev, "PHY ID expected: %x, read: %x\n", + priv->cfg->phy_id_val, val); + return -EIO; + } + + return 0; +} + +static int adin1110_hw_forwarding(struct adin1110_priv *priv, bool enable) +{ + int ret; + int i; + + priv->forwarding = enable; + + if (!priv->forwarding) { + for (i = ADIN_MAC_FDB_ADDR_SLOT; i < ADIN_MAC_MAX_ADDR_SLOTS; i++) { + ret = adin1110_clear_mac_address(priv, i); + if (ret < 0) + return ret; + } + } + + /* Forwarding is optimised when MAC runs in Cut Through mode. */ + ret = adin1110_set_bits(priv, ADIN1110_CONFIG2, + ADIN2111_PORT_CUT_THRU_EN, + priv->forwarding ? ADIN2111_PORT_CUT_THRU_EN : 0); + if (ret < 0) + return ret; + + for (i = 0; i < priv->cfg->ports_nr; i++) { + ret = adin1110_setup_rx_mode(priv->ports[i]); + if (ret < 0) + return ret; + } + + return ret; +} + +static int adin1110_port_bridge_join(struct adin1110_port_priv *port_priv, + struct net_device *bridge) +{ + struct adin1110_priv *priv = port_priv->priv; + int ret; + + port_priv->bridge = bridge; + + if (adin1110_can_offload_forwarding(priv)) { + mutex_lock(&priv->lock); + ret = adin1110_hw_forwarding(priv, true); + mutex_unlock(&priv->lock); + + if (ret < 0) + return ret; + } + + return adin1110_set_mac_address(port_priv->netdev, bridge->dev_addr); +} + +static int adin1110_port_bridge_leave(struct adin1110_port_priv *port_priv, + struct net_device *bridge) +{ + struct adin1110_priv *priv = port_priv->priv; + int ret; + + port_priv->bridge = NULL; + + mutex_lock(&priv->lock); + ret = adin1110_hw_forwarding(priv, false); + mutex_unlock(&priv->lock); + + return ret; +} + +static bool adin1110_port_dev_check(const struct net_device *dev) +{ + return dev->netdev_ops == &adin1110_netdev_ops; +} + +static int adin1110_netdevice_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct adin1110_port_priv *port_priv = netdev_priv(dev); + struct netdev_notifier_changeupper_info *info = ptr; + int ret = 0; + + if (!adin1110_port_dev_check(dev)) + return NOTIFY_DONE; + + switch (event) { + case NETDEV_CHANGEUPPER: + if (netif_is_bridge_master(info->upper_dev)) { + if (info->linking) + ret = adin1110_port_bridge_join(port_priv, info->upper_dev); + else + ret = adin1110_port_bridge_leave(port_priv, info->upper_dev); + } + break; + default: + break; + } + + return notifier_from_errno(ret); +} + +static struct notifier_block adin1110_netdevice_nb = { + .notifier_call = adin1110_netdevice_event, +}; + +static void adin1110_disconnect_phy(void *data) +{ + phy_disconnect(data); +} + +static int adin1110_port_set_forwarding_state(struct adin1110_port_priv *port_priv) +{ + struct adin1110_priv *priv = port_priv->priv; + int ret; + + port_priv->state = BR_STATE_FORWARDING; + + mutex_lock(&priv->lock); + ret = adin1110_set_mac_address(port_priv->netdev, + port_priv->netdev->dev_addr); + if (ret < 0) + goto out; + + if (adin1110_can_offload_forwarding(priv)) + ret = adin1110_hw_forwarding(priv, true); + else + ret = adin1110_setup_rx_mode(port_priv); +out: + mutex_unlock(&priv->lock); + + return ret; +} + +static int adin1110_port_set_blocking_state(struct adin1110_port_priv *port_priv) +{ + u8 mac[ETH_ALEN] = {0x01, 0x80, 0xC2, 0x00, 0x00, 0x00}; + struct adin1110_priv *priv = port_priv->priv; + u8 mask[ETH_ALEN]; + u32 port_rules; + int mac_slot; + int ret; + + port_priv->state = BR_STATE_BLOCKING; + + mutex_lock(&priv->lock); + + mac_slot = (!port_priv->nr) ? ADIN_MAC_P1_ADDR_SLOT : ADIN_MAC_P2_ADDR_SLOT; + ret = adin1110_clear_mac_address(priv, mac_slot); + if (ret < 0) + goto out; + + ret = adin1110_hw_forwarding(priv, false); + if (ret < 0) + goto out; + + /* Allow only BPDUs to be passed to the CPU */ + memset(mask, 0xFF, ETH_ALEN); + port_rules = adin1110_port_rules(port_priv, true, false); + ret = adin1110_write_mac_address(port_priv, mac_slot, mac, + mask, port_rules); +out: + mutex_unlock(&priv->lock); + + return ret; +} + +/* ADIN1110/2111 does not have any native STP support. + * Listen for bridge core state changes and + * allow all frames to pass or only the BPDUs. + */ +static int adin1110_port_attr_stp_state_set(struct adin1110_port_priv *port_priv, + u8 state) +{ + switch (state) { + case BR_STATE_FORWARDING: + return adin1110_port_set_forwarding_state(port_priv); + case BR_STATE_LEARNING: + case BR_STATE_LISTENING: + case BR_STATE_DISABLED: + case BR_STATE_BLOCKING: + return adin1110_port_set_blocking_state(port_priv); + default: + return -EINVAL; + } +} + +static int adin1110_port_attr_set(struct net_device *dev, const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack) +{ + struct adin1110_port_priv *port_priv = netdev_priv(dev); + + switch (attr->id) { + case SWITCHDEV_ATTR_ID_PORT_STP_STATE: + return adin1110_port_attr_stp_state_set(port_priv, + attr->u.stp_state); + default: + return -EOPNOTSUPP; + } +} + +static int adin1110_switchdev_blocking_event(struct notifier_block *unused, + unsigned long event, + void *ptr) +{ + struct net_device *netdev = switchdev_notifier_info_to_dev(ptr); + int ret; + + if (event == SWITCHDEV_PORT_ATTR_SET) { + ret = switchdev_handle_port_attr_set(netdev, ptr, + adin1110_port_dev_check, + adin1110_port_attr_set); + + return notifier_from_errno(ret); + } + + return NOTIFY_DONE; +} + +static struct notifier_block adin1110_switchdev_blocking_notifier = { + .notifier_call = adin1110_switchdev_blocking_event, +}; + +static void adin1110_fdb_offload_notify(struct net_device *netdev, + struct switchdev_notifier_fdb_info *rcv) +{ + struct switchdev_notifier_fdb_info info = {}; + + info.addr = rcv->addr; + info.vid = rcv->vid; + info.offloaded = true; + call_switchdev_notifiers(SWITCHDEV_FDB_OFFLOADED, + netdev, &info.info, NULL); +} + +static int adin1110_fdb_add(struct adin1110_port_priv *port_priv, + struct switchdev_notifier_fdb_info *fdb) +{ + struct adin1110_priv *priv = port_priv->priv; + struct adin1110_port_priv *other_port; + u8 mask[ETH_ALEN]; + u32 port_rules; + int mac_nr; + u32 val; + int ret; + + netdev_dbg(port_priv->netdev, + "DEBUG: %s: MACID = %pM vid = %u flags = %u %u -- port %d\n", + __func__, fdb->addr, fdb->vid, fdb->added_by_user, + fdb->offloaded, port_priv->nr); + + if (!priv->forwarding) + return 0; + + if (fdb->is_local) + return -EINVAL; + + /* Find free FDB slot on device. */ + for (mac_nr = ADIN_MAC_FDB_ADDR_SLOT; mac_nr < ADIN_MAC_MAX_ADDR_SLOTS; mac_nr++) { + ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + (mac_nr * 2), &val); + if (ret < 0) + return ret; + if (!val) + break; + } + + if (mac_nr == ADIN_MAC_MAX_ADDR_SLOTS) + return -ENOMEM; + + other_port = priv->ports[!port_priv->nr]; + port_rules = adin1110_port_rules(port_priv, false, true); + memset(mask, 0xFF, ETH_ALEN); + + return adin1110_write_mac_address(other_port, mac_nr, (u8 *)fdb->addr, + mask, port_rules); +} + +static int adin1110_read_mac(struct adin1110_priv *priv, int mac_nr, u8 *addr) +{ + u32 val; + int ret; + + ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + (mac_nr * 2), &val); + if (ret < 0) + return ret; + + put_unaligned_be16(val, addr); + + ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_LWR + (mac_nr * 2), &val); + if (ret < 0) + return ret; + + put_unaligned_be32(val, addr + 2); + + return 0; +} + +static int adin1110_fdb_del(struct adin1110_port_priv *port_priv, + struct switchdev_notifier_fdb_info *fdb) +{ + struct adin1110_priv *priv = port_priv->priv; + u8 addr[ETH_ALEN]; + int mac_nr; + int ret; + + netdev_dbg(port_priv->netdev, + "DEBUG: %s: MACID = %pM vid = %u flags = %u %u -- port %d\n", + __func__, fdb->addr, fdb->vid, fdb->added_by_user, + fdb->offloaded, port_priv->nr); + + if (fdb->is_local) + return -EINVAL; + + for (mac_nr = ADIN_MAC_FDB_ADDR_SLOT; mac_nr < ADIN_MAC_MAX_ADDR_SLOTS; mac_nr++) { + ret = adin1110_read_mac(priv, mac_nr, addr); + if (ret < 0) + return ret; + + if (ether_addr_equal(addr, fdb->addr)) { + ret = adin1110_clear_mac_address(priv, mac_nr); + if (ret < 0) + return ret; + } + } + + return 0; +} + +static void adin1110_switchdev_event_work(struct work_struct *work) +{ + struct adin1110_switchdev_event_work *switchdev_work; + struct adin1110_port_priv *port_priv; + int ret; + + switchdev_work = container_of(work, struct adin1110_switchdev_event_work, work); + port_priv = switchdev_work->port_priv; + + mutex_lock(&port_priv->priv->lock); + + switch (switchdev_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + ret = adin1110_fdb_add(port_priv, &switchdev_work->fdb_info); + if (!ret) + adin1110_fdb_offload_notify(port_priv->netdev, + &switchdev_work->fdb_info); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + adin1110_fdb_del(port_priv, &switchdev_work->fdb_info); + break; + default: + break; + } + + mutex_unlock(&port_priv->priv->lock); + + kfree(switchdev_work->fdb_info.addr); + kfree(switchdev_work); + dev_put(port_priv->netdev); +} + +/* called under rcu_read_lock() */ +static int adin1110_switchdev_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *netdev = switchdev_notifier_info_to_dev(ptr); + struct adin1110_port_priv *port_priv = netdev_priv(netdev); + struct adin1110_switchdev_event_work *switchdev_work; + struct switchdev_notifier_fdb_info *fdb_info = ptr; + + if (!adin1110_port_dev_check(netdev)) + return NOTIFY_DONE; + + switchdev_work = kzalloc(sizeof(*switchdev_work), GFP_ATOMIC); + if (WARN_ON(!switchdev_work)) + return NOTIFY_BAD; + + INIT_WORK(&switchdev_work->work, adin1110_switchdev_event_work); + switchdev_work->port_priv = port_priv; + switchdev_work->event = event; + + switch (event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + case SWITCHDEV_FDB_DEL_TO_DEVICE: + memcpy(&switchdev_work->fdb_info, ptr, + sizeof(switchdev_work->fdb_info)); + switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC); + + if (!switchdev_work->fdb_info.addr) + goto err_addr_alloc; + + ether_addr_copy((u8 *)switchdev_work->fdb_info.addr, + fdb_info->addr); + dev_hold(netdev); + break; + default: + kfree(switchdev_work); + return NOTIFY_DONE; + } + + queue_work(system_long_wq, &switchdev_work->work); + + return NOTIFY_DONE; + +err_addr_alloc: + kfree(switchdev_work); + return NOTIFY_BAD; +} + +static struct notifier_block adin1110_switchdev_notifier = { + .notifier_call = adin1110_switchdev_event, +}; + +static void adin1110_unregister_notifiers(void *data) +{ + unregister_switchdev_blocking_notifier(&adin1110_switchdev_blocking_notifier); + unregister_switchdev_notifier(&adin1110_switchdev_notifier); + unregister_netdevice_notifier(&adin1110_netdevice_nb); +} + +static int adin1110_setup_notifiers(struct adin1110_priv *priv) +{ + struct device *dev = &priv->spidev->dev; + int ret; + + ret = register_netdevice_notifier(&adin1110_netdevice_nb); + if (ret < 0) + return ret; + + ret = register_switchdev_notifier(&adin1110_switchdev_notifier); + if (ret < 0) + goto err_netdev; + + ret = register_switchdev_blocking_notifier(&adin1110_switchdev_blocking_notifier); + if (ret < 0) + goto err_sdev; + + return devm_add_action_or_reset(dev, adin1110_unregister_notifiers, NULL); + +err_sdev: + unregister_switchdev_notifier(&adin1110_switchdev_notifier); + +err_netdev: + unregister_netdevice_notifier(&adin1110_netdevice_nb); + return ret; +} + +static int adin1110_probe_netdevs(struct adin1110_priv *priv) +{ + struct device *dev = &priv->spidev->dev; + struct adin1110_port_priv *port_priv; + struct net_device *netdev; + int ret; + int i; + + for (i = 0; i < priv->cfg->ports_nr; i++) { + netdev = devm_alloc_etherdev(dev, sizeof(*port_priv)); + if (!netdev) + return -ENOMEM; + + port_priv = netdev_priv(netdev); + port_priv->netdev = netdev; + port_priv->priv = priv; + port_priv->cfg = priv->cfg; + port_priv->nr = i; + priv->ports[i] = port_priv; + SET_NETDEV_DEV(netdev, dev); + + ret = device_get_ethdev_address(dev, netdev); + if (ret < 0) + return ret; + + netdev->irq = priv->spidev->irq; + INIT_WORK(&port_priv->tx_work, adin1110_tx_work); + INIT_WORK(&port_priv->rx_mode_work, adin1110_rx_mode_work); + skb_queue_head_init(&port_priv->txq); + + netif_carrier_off(netdev); + + netdev->if_port = IF_PORT_10BASET; + netdev->netdev_ops = &adin1110_netdev_ops; + netdev->ethtool_ops = &adin1110_ethtool_ops; + netdev->priv_flags |= IFF_UNICAST_FLT; + netdev->features |= NETIF_F_NETNS_LOCAL; + + port_priv->phydev = get_phy_device(priv->mii_bus, i + 1, false); + if (IS_ERR(port_priv->phydev)) { + netdev_err(netdev, "Could not find PHY with device address: %d.\n", i); + return PTR_ERR(port_priv->phydev); + } + + port_priv->phydev = phy_connect(netdev, + phydev_name(port_priv->phydev), + adin1110_adjust_link, + PHY_INTERFACE_MODE_INTERNAL); + if (IS_ERR(port_priv->phydev)) { + netdev_err(netdev, "Could not connect PHY with device address: %d.\n", i); + return PTR_ERR(port_priv->phydev); + } + + ret = devm_add_action_or_reset(dev, adin1110_disconnect_phy, + port_priv->phydev); + if (ret < 0) + return ret; + } + + /* ADIN1110 INT_N pin will be used to signal the host */ + ret = devm_request_threaded_irq(dev, priv->spidev->irq, NULL, + adin1110_irq, + IRQF_TRIGGER_LOW | IRQF_ONESHOT, + dev_name(dev), priv); + if (ret < 0) + return ret; + + ret = adin1110_setup_notifiers(priv); + if (ret < 0) + return ret; + + for (i = 0; i < priv->cfg->ports_nr; i++) { + ret = devm_register_netdev(dev, priv->ports[i]->netdev); + if (ret < 0) { + dev_err(dev, "Failed to register network device.\n"); + return ret; + } + } + + return 0; +} + +static int adin1110_probe(struct spi_device *spi) +{ + const struct spi_device_id *dev_id = spi_get_device_id(spi); + struct device *dev = &spi->dev; + struct adin1110_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(struct adin1110_priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->spidev = spi; + priv->cfg = &adin1110_cfgs[dev_id->driver_data]; + spi->bits_per_word = 8; + spi->mode = SPI_MODE_0; + + mutex_init(&priv->lock); + spin_lock_init(&priv->state_lock); + + /* use of CRC on control and data transactions is pin dependent */ + priv->append_crc = device_property_read_bool(dev, "adi,spi-crc"); + if (priv->append_crc) + crc8_populate_msb(adin1110_crc_table, 0x7); + + ret = adin1110_check_spi(priv); + if (ret < 0) { + dev_err(dev, "Probe SPI Read check failed: %d\n", ret); + return ret; + } + + ret = adin1110_write_reg(priv, ADIN1110_RESET, ADIN1110_SWRESET); + if (ret < 0) + return ret; + + ret = adin1110_register_mdiobus(priv, dev); + if (ret < 0) { + dev_err(dev, "Could not register MDIO bus %d\n", ret); + return ret; + } + + return adin1110_probe_netdevs(priv); +} + +static const struct of_device_id adin1110_match_table[] = { + { .compatible = "adi,adin1110" }, + { .compatible = "adi,adin2111" }, + { } +}; +MODULE_DEVICE_TABLE(of, adin1110_match_table); + +static const struct spi_device_id adin1110_spi_id[] = { + { .name = "adin1110", .driver_data = ADIN1110_MAC }, + { .name = "adin2111", .driver_data = ADIN2111_MAC }, + { } +}; +MODULE_DEVICE_TABLE(spi, adin1110_spi_id); + +static struct spi_driver adin1110_driver = { + .driver = { + .name = "adin1110", + .of_match_table = adin1110_match_table, + }, + .probe = adin1110_probe, + .id_table = adin1110_spi_id, +}; +module_spi_driver(adin1110_driver); + +MODULE_DESCRIPTION("ADIN1110 Network driver"); +MODULE_AUTHOR("Alexandru Tachici "); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c index 447dc64a17e5d1aeacbdca7cc77478e76db7d62e..e104fb02817d1d3ba62e38a7364eff15f7fc90d9 100644 --- a/drivers/net/ethernet/aeroflex/greth.c +++ b/drivers/net/ethernet/aeroflex/greth.c @@ -1112,9 +1112,9 @@ static void greth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *in { struct greth_private *greth = netdev_priv(dev); - strlcpy(info->driver, dev_driver_string(greth->dev), + strscpy(info->driver, dev_driver_string(greth->dev), sizeof(info->driver)); - strlcpy(info->bus_info, greth->dev->bus->name, sizeof(info->bus_info)); + strscpy(info->bus_info, greth->dev->bus->name, sizeof(info->bus_info)); } static void greth_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *p) @@ -1507,7 +1507,7 @@ static int greth_of_probe(struct platform_device *ofdev) } /* setup NAPI */ - netif_napi_add(dev, &greth->napi, greth_poll, 64); + netif_napi_add(dev, &greth->napi, greth_poll); return 0; diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c index d19d1579c4153f5ab8ef5eaf08f55303d2671ff4..5fab589b3ddf6fe1909d0a200c4a4737a315d483 100644 --- a/drivers/net/ethernet/agere/et131x.c +++ b/drivers/net/ethernet/agere/et131x.c @@ -2952,8 +2952,8 @@ static void et131x_get_drvinfo(struct net_device *netdev, { struct et131x_adapter *adapter = netdev_priv(netdev); - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); } @@ -3969,7 +3969,7 @@ static int et131x_pci_setup(struct pci_dev *pdev, et131x_init_send(adapter); - netif_napi_add(netdev, &adapter->napi, et131x_poll, 64); + netif_napi_add(netdev, &adapter->napi, et131x_poll); eth_hw_addr_set(netdev, adapter->addr); diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c index ce353b0c02a38e9681e392a7fd502911d6738625..a30d0f172986b8187960465eade0d817bbca1d1b 100644 --- a/drivers/net/ethernet/alacritech/slicoss.c +++ b/drivers/net/ethernet/alacritech/slicoss.c @@ -1531,8 +1531,8 @@ static void slic_get_drvinfo(struct net_device *dev, { struct slic_device *sdev = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(sdev->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(sdev->pdev), sizeof(info->bus_info)); } static const struct ethtool_ops slic_ethtool_ops = { @@ -1803,7 +1803,7 @@ static int slic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto unmap; } - netif_napi_add(dev, &sdev->napi, slic_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &sdev->napi, slic_poll); netif_carrier_off(dev); err = register_netdev(dev); diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c index 621ce742ad21d5f56155953ea31438a69879c711..a94c62956eedb5e1ba8dc03e8cb806f82dac9d12 100644 --- a/drivers/net/ethernet/allwinner/sun4i-emac.c +++ b/drivers/net/ethernet/allwinner/sun4i-emac.c @@ -331,8 +331,8 @@ prepare_err: static void emac_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(&dev->dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(&dev->dev), sizeof(info->bus_info)); } static u32 emac_get_msglevel(struct net_device *dev) diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c index 22fe98555b244a60bf3a2c70941788ffd8ea5b61..d7762da8b2c0fcf613bb3762594f86f4f0351501 100644 --- a/drivers/net/ethernet/alteon/acenic.c +++ b/drivers/net/ethernet/alteon/acenic.c @@ -2691,12 +2691,12 @@ static void ace_get_drvinfo(struct net_device *dev, { struct ace_private *ap = netdev_priv(dev); - strlcpy(info->driver, "acenic", sizeof(info->driver)); + strscpy(info->driver, "acenic", sizeof(info->driver)); snprintf(info->fw_version, sizeof(info->version), "%i.%i.%i", ap->firmware_major, ap->firmware_minor, ap->firmware_fix); if (ap->pdev) - strlcpy(info->bus_info, pci_name(ap->pdev), + strscpy(info->bus_info, pci_name(ap->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/altera/Kconfig b/drivers/net/ethernet/altera/Kconfig index 914e56b91467a59d4e36cc9dca21826a7c8f3636..dd7fd41ccde5bfd087f3850923bbdf3416037678 100644 --- a/drivers/net/ethernet/altera/Kconfig +++ b/drivers/net/ethernet/altera/Kconfig @@ -3,6 +3,8 @@ config ALTERA_TSE tristate "Altera Triple-Speed Ethernet MAC support" depends on HAS_DMA select PHYLIB + select PHYLINK + select PCS_ALTERA_TSE help This driver supports the Altera Triple-Speed (TSE) Ethernet MAC. diff --git a/drivers/net/ethernet/altera/altera_tse.h b/drivers/net/ethernet/altera/altera_tse.h index f17acfb579a0212c7abaa511e12446961a9f5085..db5eed06e92dfa6049894d4ef48804e563c6e8ca 100644 --- a/drivers/net/ethernet/altera/altera_tse.h +++ b/drivers/net/ethernet/altera/altera_tse.h @@ -27,6 +27,7 @@ #include #include #include +#include #define ALTERA_TSE_SW_RESET_WATCHDOG_CNTR 10000 #define ALTERA_TSE_MAC_FIFO_WIDTH 4 /* TX/RX FIFO width in @@ -109,17 +110,6 @@ #define MAC_CMDCFG_DISABLE_READ_TIMEOUT_GET(v) GET_BIT_VALUE(v, 27) #define MAC_CMDCFG_CNT_RESET_GET(v) GET_BIT_VALUE(v, 31) -/* SGMII PCS register addresses - */ -#define SGMII_PCS_SCRATCH 0x10 -#define SGMII_PCS_REV 0x11 -#define SGMII_PCS_LINK_TIMER_0 0x12 -#define SGMII_PCS_LINK_TIMER_1 0x13 -#define SGMII_PCS_IF_MODE 0x14 -#define SGMII_PCS_DIS_READ_TO 0x15 -#define SGMII_PCS_READ_TO 0x16 -#define SGMII_PCS_SW_RESET_TIMEOUT 100 /* usecs */ - /* MDIO registers within MAC register Space */ struct altera_tse_mdio { @@ -423,6 +413,9 @@ struct altera_tse_private { void __iomem *tx_dma_csr; void __iomem *tx_dma_desc; + /* SGMII PCS address space */ + void __iomem *pcs_base; + /* Rx buffers queue */ struct tse_buffer *rx_ring; u32 rx_cons; @@ -480,6 +473,10 @@ struct altera_tse_private { u32 msg_enable; struct altera_dmaops *dmaops; + + struct phylink *phylink; + struct phylink_config phylink_config; + struct phylink_pcs *pcs; }; /* Function prototypes diff --git a/drivers/net/ethernet/altera/altera_tse_ethtool.c b/drivers/net/ethernet/altera/altera_tse_ethtool.c index 4299f130114998632f68e39f291a4047b5f82904..81313c85833eb6a30f0d6ffa19a5797c22d0376c 100644 --- a/drivers/net/ethernet/altera/altera_tse_ethtool.c +++ b/drivers/net/ethernet/altera/altera_tse_ethtool.c @@ -199,9 +199,9 @@ static int tse_reglen(struct net_device *dev) static void tse_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *regbuf) { - int i; struct altera_tse_private *priv = netdev_priv(dev); u32 *buf = regbuf; + int i; /* Set version to a known value, so ethtool knows * how to do any special formatting of this data. @@ -221,6 +221,22 @@ static void tse_get_regs(struct net_device *dev, struct ethtool_regs *regs, buf[i] = csrrd32(priv->mac_dev, i * 4); } +static int tse_ethtool_set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *cmd) +{ + struct altera_tse_private *priv = netdev_priv(dev); + + return phylink_ethtool_ksettings_set(priv->phylink, cmd); +} + +static int tse_ethtool_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *cmd) +{ + struct altera_tse_private *priv = netdev_priv(dev); + + return phylink_ethtool_ksettings_get(priv->phylink, cmd); +} + static const struct ethtool_ops tse_ethtool_ops = { .get_drvinfo = tse_get_drvinfo, .get_regs_len = tse_reglen, @@ -231,8 +247,9 @@ static const struct ethtool_ops tse_ethtool_ops = { .get_ethtool_stats = tse_fill_stats, .get_msglevel = tse_get_msglevel, .set_msglevel = tse_set_msglevel, - .get_link_ksettings = phy_ethtool_get_link_ksettings, - .set_link_ksettings = phy_ethtool_set_link_ksettings, + .get_link_ksettings = tse_ethtool_get_link_ksettings, + .set_link_ksettings = tse_ethtool_set_link_ksettings, + .get_ts_info = ethtool_op_get_ts_info, }; void altera_tse_set_ethtool_ops(struct net_device *netdev) diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c index 8c5828582c21eac53d4fe927599395a319f384a7..7633b227b2ca40ab17167c43ea205b2e11fbead2 100644 --- a/drivers/net/ethernet/altera/altera_tse_main.c +++ b/drivers/net/ethernet/altera/altera_tse_main.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -86,27 +87,6 @@ static inline u32 tse_tx_avail(struct altera_tse_private *priv) return priv->tx_cons + priv->tx_ring_size - priv->tx_prod - 1; } -/* PCS Register read/write functions - */ -static u16 sgmii_pcs_read(struct altera_tse_private *priv, int regnum) -{ - return csrrd32(priv->mac_dev, - tse_csroffs(mdio_phy0) + regnum * 4) & 0xffff; -} - -static void sgmii_pcs_write(struct altera_tse_private *priv, int regnum, - u16 value) -{ - csrwr32(value, priv->mac_dev, tse_csroffs(mdio_phy0) + regnum * 4); -} - -/* Check PCS scratch memory */ -static int sgmii_pcs_scratch_test(struct altera_tse_private *priv, u16 value) -{ - sgmii_pcs_write(priv, SGMII_PCS_SCRATCH, value); - return (sgmii_pcs_read(priv, SGMII_PCS_SCRATCH) == value); -} - /* MDIO specific functions */ static int altera_tse_mdio_read(struct mii_bus *bus, int mii_id, int regnum) @@ -141,10 +121,10 @@ static int altera_tse_mdio_write(struct mii_bus *bus, int mii_id, int regnum, static int altera_tse_mdio_create(struct net_device *dev, unsigned int id) { struct altera_tse_private *priv = netdev_priv(dev); - int ret; struct device_node *mdio_node = NULL; - struct mii_bus *mdio = NULL; struct device_node *child_node = NULL; + struct mii_bus *mdio = NULL; + int ret; for_each_child_of_node(priv->device->of_node, child_node) { if (of_device_is_compatible(child_node, "altr,tse-mdio")) { @@ -236,8 +216,8 @@ static int tse_init_rx_buffer(struct altera_tse_private *priv, static void tse_free_rx_buffer(struct altera_tse_private *priv, struct tse_buffer *rxbuffer) { - struct sk_buff *skb = rxbuffer->skb; dma_addr_t dma_addr = rxbuffer->dma_addr; + struct sk_buff *skb = rxbuffer->skb; if (skb != NULL) { if (dma_addr) @@ -358,6 +338,7 @@ static inline void tse_rx_vlan(struct net_device *dev, struct sk_buff *skb) { struct ethhdr *eth_hdr; u16 vid; + if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && !__vlan_get_tag(skb, &vid)) { eth_hdr = (struct ethhdr *)skb->data; @@ -371,10 +352,10 @@ static inline void tse_rx_vlan(struct net_device *dev, struct sk_buff *skb) */ static int tse_rx(struct altera_tse_private *priv, int limit) { - unsigned int count = 0; + unsigned int entry = priv->rx_cons % priv->rx_ring_size; unsigned int next_entry; + unsigned int count = 0; struct sk_buff *skb; - unsigned int entry = priv->rx_cons % priv->rx_ring_size; u32 rxstatus; u16 pktlength; u16 pktstatus; @@ -448,10 +429,10 @@ static int tse_rx(struct altera_tse_private *priv, int limit) static int tse_tx_complete(struct altera_tse_private *priv) { unsigned int txsize = priv->tx_ring_size; - u32 ready; - unsigned int entry; struct tse_buffer *tx_buff; + unsigned int entry; int txcomplete = 0; + u32 ready; spin_lock(&priv->tx_lock); @@ -497,8 +478,8 @@ static int tse_poll(struct napi_struct *napi, int budget) { struct altera_tse_private *priv = container_of(napi, struct altera_tse_private, napi); - int rxcomplete = 0; unsigned long int flags; + int rxcomplete = 0; tse_tx_complete(priv); @@ -561,13 +542,13 @@ static irqreturn_t altera_isr(int irq, void *dev_id) static netdev_tx_t tse_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct altera_tse_private *priv = netdev_priv(dev); + unsigned int nopaged_len = skb_headlen(skb); unsigned int txsize = priv->tx_ring_size; - unsigned int entry; - struct tse_buffer *buffer = NULL; int nfrags = skb_shinfo(skb)->nr_frags; - unsigned int nopaged_len = skb_headlen(skb); + struct tse_buffer *buffer = NULL; netdev_tx_t ret = NETDEV_TX_OK; dma_addr_t dma_addr; + unsigned int entry; spin_lock_bh(&priv->tx_lock); @@ -619,117 +600,6 @@ out: return ret; } -/* Called every time the controller might need to be made - * aware of new link state. The PHY code conveys this - * information through variables in the phydev structure, and this - * function converts those variables into the appropriate - * register values, and can bring down the device if needed. - */ -static void altera_tse_adjust_link(struct net_device *dev) -{ - struct altera_tse_private *priv = netdev_priv(dev); - struct phy_device *phydev = dev->phydev; - int new_state = 0; - - /* only change config if there is a link */ - spin_lock(&priv->mac_cfg_lock); - if (phydev->link) { - /* Read old config */ - u32 cfg_reg = ioread32(&priv->mac_dev->command_config); - - /* Check duplex */ - if (phydev->duplex != priv->oldduplex) { - new_state = 1; - if (!(phydev->duplex)) - cfg_reg |= MAC_CMDCFG_HD_ENA; - else - cfg_reg &= ~MAC_CMDCFG_HD_ENA; - - netdev_dbg(priv->dev, "%s: Link duplex = 0x%x\n", - dev->name, phydev->duplex); - - priv->oldduplex = phydev->duplex; - } - - /* Check speed */ - if (phydev->speed != priv->oldspeed) { - new_state = 1; - switch (phydev->speed) { - case 1000: - cfg_reg |= MAC_CMDCFG_ETH_SPEED; - cfg_reg &= ~MAC_CMDCFG_ENA_10; - break; - case 100: - cfg_reg &= ~MAC_CMDCFG_ETH_SPEED; - cfg_reg &= ~MAC_CMDCFG_ENA_10; - break; - case 10: - cfg_reg &= ~MAC_CMDCFG_ETH_SPEED; - cfg_reg |= MAC_CMDCFG_ENA_10; - break; - default: - if (netif_msg_link(priv)) - netdev_warn(dev, "Speed (%d) is not 10/100/1000!\n", - phydev->speed); - break; - } - priv->oldspeed = phydev->speed; - } - iowrite32(cfg_reg, &priv->mac_dev->command_config); - - if (!priv->oldlink) { - new_state = 1; - priv->oldlink = 1; - } - } else if (priv->oldlink) { - new_state = 1; - priv->oldlink = 0; - priv->oldspeed = 0; - priv->oldduplex = -1; - } - - if (new_state && netif_msg_link(priv)) - phy_print_status(phydev); - - spin_unlock(&priv->mac_cfg_lock); -} -static struct phy_device *connect_local_phy(struct net_device *dev) -{ - struct altera_tse_private *priv = netdev_priv(dev); - struct phy_device *phydev = NULL; - char phy_id_fmt[MII_BUS_ID_SIZE + 3]; - - if (priv->phy_addr != POLL_PHY) { - snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, - priv->mdio->id, priv->phy_addr); - - netdev_dbg(dev, "trying to attach to %s\n", phy_id_fmt); - - phydev = phy_connect(dev, phy_id_fmt, &altera_tse_adjust_link, - priv->phy_iface); - if (IS_ERR(phydev)) { - netdev_err(dev, "Could not attach to PHY\n"); - phydev = NULL; - } - - } else { - int ret; - phydev = phy_find_first(priv->mdio); - if (phydev == NULL) { - netdev_err(dev, "No PHY found\n"); - return phydev; - } - - ret = phy_connect_direct(dev, phydev, &altera_tse_adjust_link, - priv->phy_iface); - if (ret != 0) { - netdev_err(dev, "Could not attach to PHY\n"); - phydev = NULL; - } - } - return phydev; -} - static int altera_tse_phy_get_addr_mdio_create(struct net_device *dev) { struct altera_tse_private *priv = netdev_priv(dev); @@ -768,91 +638,6 @@ static int altera_tse_phy_get_addr_mdio_create(struct net_device *dev) return 0; } -/* Initialize driver's PHY state, and attach to the PHY - */ -static int init_phy(struct net_device *dev) -{ - struct altera_tse_private *priv = netdev_priv(dev); - struct phy_device *phydev; - struct device_node *phynode; - bool fixed_link = false; - int rc = 0; - - /* Avoid init phy in case of no phy present */ - if (!priv->phy_iface) - return 0; - - priv->oldlink = 0; - priv->oldspeed = 0; - priv->oldduplex = -1; - - phynode = of_parse_phandle(priv->device->of_node, "phy-handle", 0); - - if (!phynode) { - /* check if a fixed-link is defined in device-tree */ - if (of_phy_is_fixed_link(priv->device->of_node)) { - rc = of_phy_register_fixed_link(priv->device->of_node); - if (rc < 0) { - netdev_err(dev, "cannot register fixed PHY\n"); - return rc; - } - - /* In the case of a fixed PHY, the DT node associated - * to the PHY is the Ethernet MAC DT node. - */ - phynode = of_node_get(priv->device->of_node); - fixed_link = true; - - netdev_dbg(dev, "fixed-link detected\n"); - phydev = of_phy_connect(dev, phynode, - &altera_tse_adjust_link, - 0, priv->phy_iface); - } else { - netdev_dbg(dev, "no phy-handle found\n"); - if (!priv->mdio) { - netdev_err(dev, "No phy-handle nor local mdio specified\n"); - return -ENODEV; - } - phydev = connect_local_phy(dev); - } - } else { - netdev_dbg(dev, "phy-handle found\n"); - phydev = of_phy_connect(dev, phynode, - &altera_tse_adjust_link, 0, priv->phy_iface); - } - of_node_put(phynode); - - if (!phydev) { - netdev_err(dev, "Could not find the PHY\n"); - if (fixed_link) - of_phy_deregister_fixed_link(priv->device->of_node); - return -ENODEV; - } - - /* Stop Advertising 1000BASE Capability if interface is not GMII - */ - if ((priv->phy_iface == PHY_INTERFACE_MODE_MII) || - (priv->phy_iface == PHY_INTERFACE_MODE_RMII)) - phy_set_max_speed(phydev, SPEED_100); - - /* Broken HW is sometimes missing the pull-up resistor on the - * MDIO line, which results in reads to non-existent devices returning - * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent - * device as well. If a fixed-link is used the phy_id is always 0. - * Note: phydev->phy_id is the result of reading the UID PHY registers. - */ - if ((phydev->phy_id == 0) && !fixed_link) { - netdev_err(dev, "Bad PHY UID 0x%08x\n", phydev->phy_id); - phy_disconnect(phydev); - return -ENODEV; - } - - netdev_dbg(dev, "attached to PHY %d UID 0x%08x Link = %d\n", - phydev->mdio.addr, phydev->phy_id, phydev->link); - - return 0; -} - static void tse_update_mac_addr(struct altera_tse_private *priv, const u8 *addr) { u32 msb; @@ -1012,8 +797,8 @@ static int tse_change_mtu(struct net_device *dev, int new_mtu) static void altera_tse_set_mcfilter(struct net_device *dev) { struct altera_tse_private *priv = netdev_priv(dev); - int i; struct netdev_hw_addr *ha; + int i; /* clear the hash filter */ for (i = 0; i < 64; i++) @@ -1087,74 +872,14 @@ static void tse_set_rx_mode(struct net_device *dev) spin_unlock(&priv->mac_cfg_lock); } -/* Initialise (if necessary) the SGMII PCS component - */ -static int init_sgmii_pcs(struct net_device *dev) -{ - struct altera_tse_private *priv = netdev_priv(dev); - int n; - unsigned int tmp_reg = 0; - - if (priv->phy_iface != PHY_INTERFACE_MODE_SGMII) - return 0; /* Nothing to do, not in SGMII mode */ - - /* The TSE SGMII PCS block looks a little like a PHY, it is - * mapped into the zeroth MDIO space of the MAC and it has - * ID registers like a PHY would. Sadly this is often - * configured to zeroes, so don't be surprised if it does - * show 0x00000000. - */ - - if (sgmii_pcs_scratch_test(priv, 0x0000) && - sgmii_pcs_scratch_test(priv, 0xffff) && - sgmii_pcs_scratch_test(priv, 0xa5a5) && - sgmii_pcs_scratch_test(priv, 0x5a5a)) { - netdev_info(dev, "PCS PHY ID: 0x%04x%04x\n", - sgmii_pcs_read(priv, MII_PHYSID1), - sgmii_pcs_read(priv, MII_PHYSID2)); - } else { - netdev_err(dev, "SGMII PCS Scratch memory test failed.\n"); - return -ENOMEM; - } - - /* Starting on page 5-29 of the MegaCore Function User Guide - * Set SGMII Link timer to 1.6ms - */ - sgmii_pcs_write(priv, SGMII_PCS_LINK_TIMER_0, 0x0D40); - sgmii_pcs_write(priv, SGMII_PCS_LINK_TIMER_1, 0x03); - - /* Enable SGMII Interface and Enable SGMII Auto Negotiation */ - sgmii_pcs_write(priv, SGMII_PCS_IF_MODE, 0x3); - - /* Enable Autonegotiation */ - tmp_reg = sgmii_pcs_read(priv, MII_BMCR); - tmp_reg |= (BMCR_SPEED1000 | BMCR_FULLDPLX | BMCR_ANENABLE); - sgmii_pcs_write(priv, MII_BMCR, tmp_reg); - - /* Reset PCS block */ - tmp_reg |= BMCR_RESET; - sgmii_pcs_write(priv, MII_BMCR, tmp_reg); - for (n = 0; n < SGMII_PCS_SW_RESET_TIMEOUT; n++) { - if (!(sgmii_pcs_read(priv, MII_BMCR) & BMCR_RESET)) { - netdev_info(dev, "SGMII PCS block initialised OK\n"); - return 0; - } - udelay(1); - } - - /* We failed to reset the block, return a timeout */ - netdev_err(dev, "SGMII PCS block reset failed.\n"); - return -ETIMEDOUT; -} - /* Open and initialize the interface */ static int tse_open(struct net_device *dev) { struct altera_tse_private *priv = netdev_priv(dev); + unsigned long flags; int ret = 0; int i; - unsigned long int flags; /* Reset and configure TSE MAC and probe associated PHY */ ret = priv->dmaops->init_dma(priv); @@ -1171,14 +896,6 @@ static int tse_open(struct net_device *dev) netdev_warn(dev, "TSE revision %x\n", priv->revision); spin_lock(&priv->mac_cfg_lock); - /* no-op if MAC not operating in SGMII mode*/ - ret = init_sgmii_pcs(dev); - if (ret) { - netdev_err(dev, - "Cannot init the SGMII PCS (error: %d)\n", ret); - spin_unlock(&priv->mac_cfg_lock); - goto phy_error; - } ret = reset_mac(priv); /* Note that reset_mac will fail if the clocks are gated by the PHY @@ -1236,8 +953,12 @@ static int tse_open(struct net_device *dev) spin_unlock_irqrestore(&priv->rxdma_irq_lock, flags); - if (dev->phydev) - phy_start(dev->phydev); + ret = phylink_of_phy_connect(priv->phylink, priv->device->of_node, 0); + if (ret) { + netdev_err(dev, "could not connect phylink (%d)\n", ret); + goto tx_request_irq_error; + } + phylink_start(priv->phylink); napi_enable(&priv->napi); netif_start_queue(dev); @@ -1265,13 +986,10 @@ phy_error: static int tse_shutdown(struct net_device *dev) { struct altera_tse_private *priv = netdev_priv(dev); - int ret; unsigned long int flags; + int ret; - /* Stop the PHY */ - if (dev->phydev) - phy_stop(dev->phydev); - + phylink_stop(priv->phylink); netif_stop_queue(dev); napi_disable(&priv->napi); @@ -1317,11 +1035,79 @@ static struct net_device_ops altera_tse_netdev_ops = { .ndo_validate_addr = eth_validate_addr, }; +static void alt_tse_mac_an_restart(struct phylink_config *config) +{ +} + +static void alt_tse_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct altera_tse_private *priv = netdev_priv(ndev); + + spin_lock(&priv->mac_cfg_lock); + reset_mac(priv); + tse_set_mac(priv, true); + spin_unlock(&priv->mac_cfg_lock); +} + +static void alt_tse_mac_link_down(struct phylink_config *config, + unsigned int mode, phy_interface_t interface) +{ +} + +static void alt_tse_mac_link_up(struct phylink_config *config, + struct phy_device *phy, unsigned int mode, + phy_interface_t interface, int speed, + int duplex, bool tx_pause, bool rx_pause) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct altera_tse_private *priv = netdev_priv(ndev); + u32 ctrl; + + ctrl = csrrd32(priv->mac_dev, tse_csroffs(command_config)); + ctrl &= ~(MAC_CMDCFG_ENA_10 | MAC_CMDCFG_ETH_SPEED | MAC_CMDCFG_HD_ENA); + + if (duplex == DUPLEX_HALF) + ctrl |= MAC_CMDCFG_HD_ENA; + + if (speed == SPEED_1000) + ctrl |= MAC_CMDCFG_ETH_SPEED; + else if (speed == SPEED_10) + ctrl |= MAC_CMDCFG_ENA_10; + + spin_lock(&priv->mac_cfg_lock); + csrwr32(ctrl, priv->mac_dev, tse_csroffs(command_config)); + spin_unlock(&priv->mac_cfg_lock); +} + +static struct phylink_pcs *alt_tse_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct altera_tse_private *priv = netdev_priv(ndev); + + if (interface == PHY_INTERFACE_MODE_SGMII || + interface == PHY_INTERFACE_MODE_1000BASEX) + return priv->pcs; + else + return NULL; +} + +static const struct phylink_mac_ops alt_tse_phylink_ops = { + .validate = phylink_generic_validate, + .mac_an_restart = alt_tse_mac_an_restart, + .mac_config = alt_tse_mac_config, + .mac_link_down = alt_tse_mac_link_down, + .mac_link_up = alt_tse_mac_link_up, + .mac_select_pcs = alt_tse_select_pcs, +}; + static int request_and_map(struct platform_device *pdev, const char *name, struct resource **res, void __iomem **ptr) { - struct resource *region; struct device *device = &pdev->dev; + struct resource *region; *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); if (*res == NULL) { @@ -1350,13 +1136,15 @@ static int request_and_map(struct platform_device *pdev, const char *name, */ static int altera_tse_probe(struct platform_device *pdev) { - struct net_device *ndev; - int ret = -ENODEV; + const struct of_device_id *of_id = NULL; + struct altera_tse_private *priv; struct resource *control_port; struct resource *dma_res; - struct altera_tse_private *priv; + struct resource *pcs_res; + struct net_device *ndev; void __iomem *descmap; - const struct of_device_id *of_id = NULL; + int pcs_reg_width = 2; + int ret = -ENODEV; ndev = alloc_etherdev(sizeof(struct altera_tse_private)); if (!ndev) { @@ -1467,6 +1255,17 @@ static int altera_tse_probe(struct platform_device *pdev) if (ret) goto err_free_netdev; + /* SGMII PCS address space. The location can vary depending on how the + * IP is integrated. We can have a resource dedicated to it at a specific + * address space, but if it's not the case, we fallback to the mdiophy0 + * from the MAC's address space + */ + ret = request_and_map(pdev, "pcs", &pcs_res, + &priv->pcs_base); + if (ret) { + priv->pcs_base = priv->mac_dev + tse_csroffs(mdio_phy0); + pcs_reg_width = 4; + } /* Rx IRQ */ priv->rx_irq = platform_get_irq_byname(pdev, "rx_irq"); @@ -1566,7 +1365,7 @@ static int altera_tse_probe(struct platform_device *pdev) ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; /* setup NAPI interface */ - netif_napi_add(ndev, &priv->napi, tse_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, tse_poll); spin_lock_init(&priv->mac_cfg_lock); spin_lock_init(&priv->tx_lock); @@ -1590,11 +1389,32 @@ static int altera_tse_probe(struct platform_device *pdev) (unsigned long) control_port->start, priv->rx_irq, priv->tx_irq); - ret = init_phy(ndev); - if (ret != 0) { - netdev_err(ndev, "Cannot attach to PHY (error: %d)\n", ret); + priv->pcs = alt_tse_pcs_create(ndev, priv->pcs_base, pcs_reg_width); + + priv->phylink_config.dev = &ndev->dev; + priv->phylink_config.type = PHYLINK_NETDEV; + priv->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | + MAC_100 | MAC_1000FD; + + phy_interface_set_rgmii(priv->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_MII, + priv->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + priv->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + priv->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + priv->phylink_config.supported_interfaces); + + priv->phylink = phylink_create(&priv->phylink_config, + of_fwnode_handle(priv->device->of_node), + priv->phy_iface, &alt_tse_phylink_ops); + if (IS_ERR(priv->phylink)) { + dev_err(&pdev->dev, "failed to create phylink\n"); + ret = PTR_ERR(priv->phylink); goto err_init_phy; } + return 0; err_init_phy: @@ -1614,16 +1434,10 @@ static int altera_tse_remove(struct platform_device *pdev) struct net_device *ndev = platform_get_drvdata(pdev); struct altera_tse_private *priv = netdev_priv(ndev); - if (ndev->phydev) { - phy_disconnect(ndev->phydev); - - if (of_phy_is_fixed_link(priv->device->of_node)) - of_phy_deregister_fixed_link(priv->device->of_node); - } - platform_set_drvdata(pdev, NULL); altera_tse_mdio_destroy(ndev); unregister_netdev(ndev); + phylink_destroy(priv->phylink); free_netdev(ndev); return 0; diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 39242c5a1729086349cd34f9df4b7972aba1b1fe..98d6386b7f3987e1ea4a336778dc1b398c35c3d4 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -462,8 +462,8 @@ static void ena_get_drvinfo(struct net_device *dev, { struct ena_adapter *adapter = netdev_priv(dev); - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 6a356a6cee15a29bdb92a2abf1038820c11ec7e5..d350eeec8bad4c23cc6c2ec482bd0dd11677e292 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -2265,10 +2265,8 @@ static void ena_init_napi_in_range(struct ena_adapter *adapter, for (i = first_index; i < first_index + count; i++) { struct ena_napi *napi = &adapter->ena_napi[i]; - netif_napi_add(adapter->netdev, - &napi->napi, - ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(adapter->netdev, &napi->napi, + ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll); if (!ENA_IS_XDP_INDEX(adapter, i)) { napi->rx_ring = &adapter->rx_ring[i]; @@ -3166,7 +3164,7 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pd host_info->bdf = (pdev->bus->number << 8) | pdev->devfn; host_info->os_type = ENA_ADMIN_OS_LINUX; host_info->kernel_ver = LINUX_VERSION_CODE; - strlcpy(host_info->kernel_ver_str, utsname()->version, + strscpy(host_info->kernel_ver_str, utsname()->version, sizeof(host_info->kernel_ver_str) - 1); host_info->os_dist = 0; strncpy(host_info->os_dist_str, utsname()->release, diff --git a/drivers/net/ethernet/amd/a2065.c b/drivers/net/ethernet/amd/a2065.c index 3a351d3396bfea4603152d0f784bc15031dee8c2..68983b7171453f140c60b657c5358f9b76c4e67e 100644 --- a/drivers/net/ethernet/amd/a2065.c +++ b/drivers/net/ethernet/amd/a2065.c @@ -695,7 +695,7 @@ static int a2065_init_one(struct zorro_dev *z, } dev = alloc_etherdev(sizeof(struct lance_private)); - if (dev == NULL) { + if (!dev) { release_mem_region(base_addr, sizeof(struct lance_regs)); release_mem_region(mem_start, A2065_RAM_SIZE); return -ENOMEM; diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c index 5d1baa01360fc519df8be12016fc553ea35838b1..ea6cfc2095e1872dd3a602b330139e164208e4e7 100644 --- a/drivers/net/ethernet/amd/amd8111e.c +++ b/drivers/net/ethernet/amd/amd8111e.c @@ -43,7 +43,7 @@ Revision History: 3.0.4 12/09/2003 1. Added set_mac_address routine for bonding driver support. 2. Tested the driver for bonding support - 3. Bug fix: Fixed mismach in actual receive buffer lenth and lenth + 3. Bug fix: Fixed mismach in actual receive buffer length and length indicated to the h/w. 4. Modified amd8111e_rx() routine to receive all the received packets in the first interrupt. @@ -185,24 +185,23 @@ static void amd8111e_set_ext_phy(struct net_device *dev) advert = amd8111e_mdio_read(dev, lp->ext_phy_addr, MII_ADVERTISE); tmp = advert & ~(ADVERTISE_ALL | ADVERTISE_100BASE4); switch (lp->ext_phy_option) { - - default: - case SPEED_AUTONEG: /* advertise all values */ - tmp |= (ADVERTISE_10HALF | ADVERTISE_10FULL | - ADVERTISE_100HALF | ADVERTISE_100FULL); - break; - case SPEED10_HALF: - tmp |= ADVERTISE_10HALF; - break; - case SPEED10_FULL: - tmp |= ADVERTISE_10FULL; - break; - case SPEED100_HALF: - tmp |= ADVERTISE_100HALF; - break; - case SPEED100_FULL: - tmp |= ADVERTISE_100FULL; - break; + default: + case SPEED_AUTONEG: /* advertise all values */ + tmp |= (ADVERTISE_10HALF | ADVERTISE_10FULL | + ADVERTISE_100HALF | ADVERTISE_100FULL); + break; + case SPEED10_HALF: + tmp |= ADVERTISE_10HALF; + break; + case SPEED10_FULL: + tmp |= ADVERTISE_10FULL; + break; + case SPEED100_HALF: + tmp |= ADVERTISE_100HALF; + break; + case SPEED100_FULL: + tmp |= ADVERTISE_100FULL; + break; } if(advert != tmp) @@ -237,7 +236,7 @@ static int amd8111e_free_skbs(struct net_device *dev) /* Freeing previously allocated receive buffers */ for (i = 0; i < NUM_RX_BUFFERS; i++) { rx_skbuff = lp->rx_skbuff[i]; - if (rx_skbuff != NULL) { + if (rx_skbuff) { dma_unmap_single(&lp->pci_dev->dev, lp->rx_dma_addr[i], lp->rx_buff_len - 2, DMA_FROM_DEVICE); @@ -1084,7 +1083,7 @@ static irqreturn_t amd8111e_interrupt(int irq, void *dev_id) unsigned int intr0, intren0; unsigned int handled = 1; - if (unlikely(dev == NULL)) + if (unlikely(!dev)) return IRQ_NONE; spin_lock(&lp->lock); @@ -1109,7 +1108,7 @@ static irqreturn_t amd8111e_interrupt(int irq, void *dev_id) /* Check if Receive Interrupt has occurred. */ if (intr0 & RINT0) { if (napi_schedule_prep(&lp->napi)) { - /* Disable receive interupts */ + /* Disable receive interrupts */ writel(RINTEN0, mmio + INTEN0); /* Schedule a polling routine */ __napi_schedule(&lp->napi); @@ -1364,10 +1363,10 @@ static void amd8111e_get_drvinfo(struct net_device *dev, { struct amd8111e_priv *lp = netdev_priv(dev); struct pci_dev *pci_dev = lp->pci_dev; - strlcpy(info->driver, MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, MODULE_NAME, sizeof(info->driver)); snprintf(info->fw_version, sizeof(info->fw_version), "%u", chip_version); - strlcpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info)); } static int amd8111e_get_regs_len(struct net_device *dev) @@ -1554,7 +1553,7 @@ static int amd8111e_enable_magicpkt(struct amd8111e_priv *lp) static int amd8111e_enable_link_change(struct amd8111e_priv *lp) { - /* Adapter is already stoped/suspended/interrupt-disabled */ + /* Adapter is already stopped/suspended/interrupt-disabled */ writel(VAL0 | LCMODE_SW, lp->mmio + CMD7); /* To eliminate PCI posting bug */ diff --git a/drivers/net/ethernet/amd/amd8111e.h b/drivers/net/ethernet/amd/amd8111e.h index 37da79da5f5eb8b2966d97e63b88d0223fd5b4b5..9d570adb295b6e2490c8ce31b5f633bf55320007 100644 --- a/drivers/net/ethernet/amd/amd8111e.h +++ b/drivers/net/ethernet/amd/amd8111e.h @@ -600,7 +600,7 @@ typedef enum { #define CSTATE 1 #define SSTATE 2 -/* Assume contoller gets data 10 times the maximum processing time */ +/* Assume controller gets data 10 times the maximum processing time */ #define REPEAT_CNT 10 /* amd8111e descriptor flag definitions */ diff --git a/drivers/net/ethernet/amd/ariadne.c b/drivers/net/ethernet/amd/ariadne.c index 4ea7b9f3c42498e3355d2c946de4e84663557028..38153e63323107413eb0eb9a1f76f8ea5c526b35 100644 --- a/drivers/net/ethernet/amd/ariadne.c +++ b/drivers/net/ethernet/amd/ariadne.c @@ -193,7 +193,7 @@ static int ariadne_rx(struct net_device *dev) struct sk_buff *skb; skb = netdev_alloc_skb(dev, pkt_len + 2); - if (skb == NULL) { + if (!skb) { for (i = 0; i < RX_RING_SIZE; i++) if (lowb(priv->rx_ring[(entry + i) % RX_RING_SIZE]->RMD1) & RF_OWN) break; @@ -731,7 +731,7 @@ static int ariadne_init_one(struct zorro_dev *z, } dev = alloc_etherdev(sizeof(struct ariadne_private)); - if (dev == NULL) { + if (!dev) { release_mem_region(base_addr, sizeof(struct Am79C960)); release_mem_region(mem_start, ARIADNE_RAM_SIZE); return -ENOMEM; diff --git a/drivers/net/ethernet/amd/atarilance.c b/drivers/net/ethernet/amd/atarilance.c index 27869164c6e621636639e9beb80ec9c2cc6131d7..3222c48ce6ae4402c47ee6fa084d862cf261191d 100644 --- a/drivers/net/ethernet/amd/atarilance.c +++ b/drivers/net/ethernet/amd/atarilance.c @@ -581,15 +581,15 @@ static unsigned long __init lance_probe1( struct net_device *dev, /* Get the ethernet address */ switch( lp->cardtype ) { - case OLD_RIEBL: + case OLD_RIEBL: /* No ethernet address! (Set some default address) */ eth_hw_addr_set(dev, OldRieblDefHwaddr); break; - case NEW_RIEBL: + case NEW_RIEBL: lp->memcpy_f(addr, RIEBL_HWADDR_ADDR, ETH_ALEN); eth_hw_addr_set(dev, addr); break; - case PAM_CARD: + case PAM_CARD: i = IO->eeprom; for( i = 0; i < 6; ++i ) addr[i] = @@ -854,7 +854,7 @@ static irqreturn_t lance_interrupt( int irq, void *dev_id ) int csr0, boguscnt = 10; int handled = 0; - if (dev == NULL) { + if (!dev) { DPRINTK( 1, ( "lance_interrupt(): interrupt for unknown device.\n" )); return IRQ_NONE; } @@ -995,7 +995,7 @@ static int lance_rx( struct net_device *dev ) } else { skb = netdev_alloc_skb(dev, pkt_len + 2); - if (skb == NULL) { + if (!skb) { for( i = 0; i < RX_RING_SIZE; i++ ) if (MEM->rx_head[(entry+i) & RX_RING_MOD_MASK].flag & RMD1_OWN_CHIP) diff --git a/drivers/net/ethernet/amd/au1000_eth.c b/drivers/net/ethernet/amd/au1000_eth.c index d5f2c698922102525a767cd44240baf4fd936998..c5cec4e794895d81ae0bf637317a1caf4a4ddfa7 100644 --- a/drivers/net/ethernet/amd/au1000_eth.c +++ b/drivers/net/ethernet/amd/au1000_eth.c @@ -650,7 +650,7 @@ au1000_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct au1000_private *aup = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); snprintf(info->bus_info, sizeof(info->bus_info), "%s %d", DRV_NAME, aup->mac_id); } @@ -786,7 +786,7 @@ static int au1000_rx(struct net_device *dev) frmlen = (status & RX_FRAME_LEN_MASK); frmlen -= 4; /* Remove FCS */ skb = netdev_alloc_skb(dev, frmlen + 2); - if (skb == NULL) { + if (!skb) { dev->stats.rx_dropped++; continue; } @@ -1199,7 +1199,7 @@ static int au1000_probe(struct platform_device *pdev) } aup->mii_bus = mdiobus_alloc(); - if (aup->mii_bus == NULL) { + if (!aup->mii_bus) { dev_err(&pdev->dev, "failed to allocate mdiobus structure\n"); err = -ENOMEM; goto err_mdiobus_alloc; @@ -1284,7 +1284,7 @@ static int au1000_probe(struct platform_device *pdev) return 0; err_out: - if (aup->mii_bus != NULL) + if (aup->mii_bus) mdiobus_unregister(aup->mii_bus); /* here we should have a valid dev plus aup-> register addresses diff --git a/drivers/net/ethernet/amd/lance.c b/drivers/net/ethernet/amd/lance.c index 462016666752c9797aacb21892cee9adcc46debe..fb8686214a327f8036194c6cb67ec898f0769775 100644 --- a/drivers/net/ethernet/amd/lance.c +++ b/drivers/net/ethernet/amd/lance.c @@ -880,7 +880,7 @@ lance_init_ring(struct net_device *dev, gfp_t gfp) rx_buff = skb->data; else rx_buff = kmalloc(PKT_BUF_SZ, GFP_DMA | gfp); - if (rx_buff == NULL) + if (!rx_buff) lp->rx_ring[i].base = 0; else lp->rx_ring[i].base = (u32)isa_virt_to_bus(rx_buff) | 0x80000000; @@ -1186,7 +1186,7 @@ lance_rx(struct net_device *dev) else { skb = dev_alloc_skb(pkt_len+2); - if (skb == NULL) + if (!skb) { printk("%s: Memory squeeze, deferring packet.\n", dev->name); for (i=0; i < RX_RING_SIZE; i++) diff --git a/drivers/net/ethernet/amd/nmclan_cs.c b/drivers/net/ethernet/amd/nmclan_cs.c index 30ee5329bd7cb986275ec09d569e3d8d1e5abab8..823a329a921f466676c64b4d36bdbe81ba48322f 100644 --- a/drivers/net/ethernet/amd/nmclan_cs.c +++ b/drivers/net/ethernet/amd/nmclan_cs.c @@ -485,10 +485,10 @@ static int mace_read(mace_private *lp, unsigned int ioaddr, int reg) unsigned long flags; switch (reg >> 4) { - case 0: /* register 0-15 */ + case 0: /* register 0-15 */ data = inb(ioaddr + AM2150_MACE_BASE + reg); break; - case 1: /* register 16-31 */ + case 1: /* register 16-31 */ spin_lock_irqsave(&lp->bank_lock, flags); MACEBANK(1); data = inb(ioaddr + AM2150_MACE_BASE + (reg & 0x0F)); @@ -512,10 +512,10 @@ static void mace_write(mace_private *lp, unsigned int ioaddr, int reg, unsigned long flags; switch (reg >> 4) { - case 0: /* register 0-15 */ + case 0: /* register 0-15 */ outb(data & 0xFF, ioaddr + AM2150_MACE_BASE + reg); break; - case 1: /* register 16-31 */ + case 1: /* register 16-31 */ spin_lock_irqsave(&lp->bank_lock, flags); MACEBANK(1); outb(data & 0xFF, ioaddr + AM2150_MACE_BASE + (reg & 0x0F)); @@ -567,13 +567,13 @@ static int mace_init(mace_private *lp, unsigned int ioaddr, * Or just set ASEL in PHYCC below! */ switch (if_port) { - case 1: + case 1: mace_write(lp, ioaddr, MACE_PLSCC, 0x02); break; - case 2: + case 2: mace_write(lp, ioaddr, MACE_PLSCC, 0x00); break; - default: + default: mace_write(lp, ioaddr, MACE_PHYCC, /* ASEL */ 4); /* ASEL Auto Select. When set, the PORTSEL[1-0] bits are overridden, and the MACE device will automatically select the operating media @@ -815,7 +815,7 @@ static int mace_close(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); snprintf(info->bus_info, sizeof(info->bus_info), "PCMCIA 0x%lx", dev->base_addr); } @@ -918,7 +918,7 @@ static irqreturn_t mace_interrupt(int irq, void *dev_id) int status; int IntrCnt = MACE_MAX_IR_ITERATIONS; - if (dev == NULL) { + if (!dev) { pr_debug("mace_interrupt(): irq 0x%X for unknown device.\n", irq); return IRQ_NONE; @@ -1102,7 +1102,7 @@ static int mace_rx(struct net_device *dev, unsigned char RxCnt) skb = netdev_alloc_skb(dev, pkt_len + 2); - if (skb != NULL) { + if (skb) { skb_reserve(skb, 2); insw(ioaddr + AM2150_RCV, skb_put(skb, pkt_len), pkt_len>>1); if (pkt_len & 1) diff --git a/drivers/net/ethernet/amd/pcnet32.c b/drivers/net/ethernet/amd/pcnet32.c index b5ff47283cfe0543e3501cc6db4bdd4db313f4ab..72db9f9e7beeae2de11dcedf5de57f53d39564b3 100644 --- a/drivers/net/ethernet/amd/pcnet32.c +++ b/drivers/net/ethernet/amd/pcnet32.c @@ -488,7 +488,7 @@ static void pcnet32_realloc_tx_ring(struct net_device *dev, dma_alloc_coherent(&lp->pci_dev->dev, sizeof(struct pcnet32_tx_head) * entries, &new_ring_dma_addr, GFP_ATOMIC); - if (new_tx_ring == NULL) + if (!new_tx_ring) return; new_dma_addr_list = kcalloc(entries, sizeof(dma_addr_t), GFP_ATOMIC); @@ -547,7 +547,7 @@ static void pcnet32_realloc_rx_ring(struct net_device *dev, dma_alloc_coherent(&lp->pci_dev->dev, sizeof(struct pcnet32_rx_head) * entries, &new_ring_dma_addr, GFP_ATOMIC); - if (new_rx_ring == NULL) + if (!new_rx_ring) return; new_dma_addr_list = kcalloc(entries, sizeof(dma_addr_t), GFP_ATOMIC); @@ -797,9 +797,9 @@ static void pcnet32_get_drvinfo(struct net_device *dev, { struct pcnet32_private *lp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); if (lp->pci_dev) - strlcpy(info->bus_info, pci_name(lp->pci_dev), + strscpy(info->bus_info, pci_name(lp->pci_dev), sizeof(info->bus_info)); else snprintf(info->bus_info, sizeof(info->bus_info), @@ -1249,7 +1249,7 @@ static void pcnet32_rx_entry(struct net_device *dev, } else skb = netdev_alloc_skb(dev, pkt_len + NET_IP_ALIGN); - if (skb == NULL) { + if (!skb) { dev->stats.rx_dropped++; return; } @@ -2018,7 +2018,7 @@ static int pcnet32_alloc_ring(struct net_device *dev, const char *name) lp->tx_ring = dma_alloc_coherent(&lp->pci_dev->dev, sizeof(struct pcnet32_tx_head) * lp->tx_ring_size, &lp->tx_ring_dma_addr, GFP_KERNEL); - if (lp->tx_ring == NULL) { + if (!lp->tx_ring) { netif_err(lp, drv, dev, "Coherent memory allocation failed\n"); return -ENOMEM; } @@ -2026,7 +2026,7 @@ static int pcnet32_alloc_ring(struct net_device *dev, const char *name) lp->rx_ring = dma_alloc_coherent(&lp->pci_dev->dev, sizeof(struct pcnet32_rx_head) * lp->rx_ring_size, &lp->rx_ring_dma_addr, GFP_KERNEL); - if (lp->rx_ring == NULL) { + if (!lp->rx_ring) { netif_err(lp, drv, dev, "Coherent memory allocation failed\n"); return -ENOMEM; } @@ -2365,7 +2365,7 @@ static int pcnet32_init_ring(struct net_device *dev) for (i = 0; i < lp->rx_ring_size; i++) { struct sk_buff *rx_skbuff = lp->rx_skbuff[i]; - if (rx_skbuff == NULL) { + if (!rx_skbuff) { lp->rx_skbuff[i] = netdev_alloc_skb(dev, PKT_BUF_SKB); rx_skbuff = lp->rx_skbuff[i]; if (!rx_skbuff) { diff --git a/drivers/net/ethernet/amd/sun3lance.c b/drivers/net/ethernet/amd/sun3lance.c index 007bd77872916b102c35ba3b2e7936b8e8255438..246f34c4376532dad2d56edd18e5289c159beb5b 100644 --- a/drivers/net/ethernet/amd/sun3lance.c +++ b/drivers/net/ethernet/amd/sun3lance.c @@ -341,7 +341,7 @@ static int __init lance_probe( struct net_device *dev) /* XXX - leak? */ MEM = dvma_malloc_align(sizeof(struct lance_memory), 0x10000); - if (MEM == NULL) { + if (!MEM) { #ifdef CONFIG_SUN3 iounmap((void __iomem *)ioaddr); #endif @@ -796,7 +796,7 @@ static int lance_rx( struct net_device *dev ) } else { skb = netdev_alloc_skb(dev, pkt_len + 2); - if (skb == NULL) { + if (!skb) { dev->stats.rx_dropped++; head->msg_length = 0; head->flag |= RMD1_OWN_CHIP; diff --git a/drivers/net/ethernet/amd/sunlance.c b/drivers/net/ethernet/amd/sunlance.c index 22d609563af83ad1f73ce09b1b534fc1ac822552..68ca1225eedc1656d30b734b438cd573582824b9 100644 --- a/drivers/net/ethernet/amd/sunlance.c +++ b/drivers/net/ethernet/amd/sunlance.c @@ -530,7 +530,7 @@ static void lance_rx_dvma(struct net_device *dev) len = (rd->mblength & 0xfff) - 4; skb = netdev_alloc_skb(dev, len + 2); - if (skb == NULL) { + if (!skb) { dev->stats.rx_dropped++; rd->mblength = 0; rd->rmd1_bits = LE_R1_OWN; @@ -700,7 +700,7 @@ static void lance_rx_pio(struct net_device *dev) len = (sbus_readw(&rd->mblength) & 0xfff) - 4; skb = netdev_alloc_skb(dev, len + 2); - if (skb == NULL) { + if (!skb) { dev->stats.rx_dropped++; sbus_writew(0, &rd->mblength); sbus_writeb(LE_R1_OWN, &rd->rmd1_bits); @@ -1276,7 +1276,7 @@ static void lance_free_hwresources(struct lance_private *lp) /* Ethtool support... */ static void sparc_lance_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "sunlance", sizeof(info->driver)); + strscpy(info->driver, "sunlance", sizeof(info->driver)); } static const struct ethtool_ops sparc_lance_ethtool_ops = { diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index f342bb85318917689d3c1909cdeaf8486602a0fe..7b666106feee935771b9e351f3b70b9d5d8d5cd3 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -952,14 +952,14 @@ static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add) channel = pdata->channel[i]; if (add) netif_napi_add(pdata->netdev, &channel->napi, - xgbe_one_poll, NAPI_POLL_WEIGHT); + xgbe_one_poll); napi_enable(&channel->napi); } } else { if (add) netif_napi_add(pdata->netdev, &pdata->napi, - xgbe_all_poll, NAPI_POLL_WEIGHT); + xgbe_all_poll); napi_enable(&pdata->napi); } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 6ceb1cdf6ebac698f66aa4e8f4864dd41aa1c732..6e83ff59172a36b66d1442dc32b338edfe3d5773 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -402,8 +402,8 @@ static void xgbe_get_drvinfo(struct net_device *netdev, struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_features *hw_feat = &pdata->hw_feat; - strlcpy(drvinfo->driver, XGBE_DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(pdata->dev), + strscpy(drvinfo->driver, XGBE_DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(pdata->dev), sizeof(drvinfo->bus_info)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", XGMAC_GET_BITS(hw_feat->version, MAC_VR, USERVER), diff --git a/drivers/net/ethernet/apm/xgene-v2/main.c b/drivers/net/ethernet/apm/xgene-v2/main.c index d022b6db9e06617fd556b00830596546a4fd5a32..379d19d18dbed0244d8b45ae08c515fe3c546c09 100644 --- a/drivers/net/ethernet/apm/xgene-v2/main.c +++ b/drivers/net/ethernet/apm/xgene-v2/main.c @@ -672,7 +672,7 @@ static int xge_probe(struct platform_device *pdev) if (ret) goto err; - netif_napi_add(ndev, &pdata->napi, xge_napi, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &pdata->napi, xge_napi); ret = register_netdev(ndev); if (ret) { diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c index 53dc8d5fede86fff85f7bc09a78d0f87f4b17dba..d6cfea65a714030d3662915f9103c3ac36724b18 100644 --- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c +++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c @@ -1977,14 +1977,12 @@ static void xgene_enet_napi_add(struct xgene_enet_pdata *pdata) for (i = 0; i < pdata->rxq_cnt; i++) { napi = &pdata->rx_ring[i]->napi; - netif_napi_add(pdata->ndev, napi, xgene_enet_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(pdata->ndev, napi, xgene_enet_napi); } for (i = 0; i < pdata->cq_cnt; i++) { napi = &pdata->tx_ring[i]->cp_ring->napi; - netif_napi_add(pdata->ndev, napi, xgene_enet_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(pdata->ndev, napi, xgene_enet_napi); } } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index 1daecd483b8d632310356d2326a2fab56c02f8fd..a08f221e30d41f85bf4505b250c6d8bd7a281d1c 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -238,7 +238,7 @@ static void aq_ethtool_get_drvinfo(struct net_device *ndev, "%u.%u.%u", firmware_version >> 24, (firmware_version >> 16) & 0xFFU, firmware_version & 0xFFFFU); - strlcpy(drvinfo->bus_info, pdev ? pci_name(pdev) : "", + strscpy(drvinfo->bus_info, pdev ? pci_name(pdev) : "", sizeof(drvinfo->bus_info)); drvinfo->n_stats = aq_ethtool_n_stats(ndev); drvinfo->testinfo_len = 0; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c index 02058fe79f52b15fb0d45552b2861affa3513f1c..3d0e16791e1c931d88682fb10f2777e72d737174 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c @@ -292,9 +292,6 @@ static int aq_mdo_dev_open(struct macsec_context *ctx) struct aq_nic_s *nic = netdev_priv(ctx->netdev); int ret = 0; - if (ctx->prepare) - return 0; - if (netif_carrier_ok(nic->ndev)) ret = aq_apply_secy_cfg(nic, ctx->secy); @@ -306,9 +303,6 @@ static int aq_mdo_dev_stop(struct macsec_context *ctx) struct aq_nic_s *nic = netdev_priv(ctx->netdev); int i; - if (ctx->prepare) - return 0; - for (i = 0; i < AQ_MACSEC_MAX_SC; i++) { if (nic->macsec_cfg->txsc_idx_busy & BIT(i)) aq_clear_secy(nic, nic->macsec_cfg->aq_txsc[i].sw_secy, @@ -466,9 +460,6 @@ static int aq_mdo_add_secy(struct macsec_context *ctx) if (txsc_idx == AQ_MACSEC_MAX_SC) return -ENOSPC; - if (ctx->prepare) - return 0; - cfg->sc_sa = sc_sa; cfg->aq_txsc[txsc_idx].hw_sc_idx = aq_to_hw_sc_idx(txsc_idx, sc_sa); cfg->aq_txsc[txsc_idx].sw_secy = secy; @@ -492,9 +483,6 @@ static int aq_mdo_upd_secy(struct macsec_context *ctx) if (txsc_idx < 0) return -ENOENT; - if (ctx->prepare) - return 0; - if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev)) ret = aq_set_txsc(nic, txsc_idx); @@ -543,9 +531,6 @@ static int aq_mdo_del_secy(struct macsec_context *ctx) struct aq_nic_s *nic = netdev_priv(ctx->netdev); int ret = 0; - if (ctx->prepare) - return 0; - if (!nic->macsec_cfg) return 0; @@ -601,9 +586,6 @@ static int aq_mdo_add_txsa(struct macsec_context *ctx) if (txsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - aq_txsc = &cfg->aq_txsc[txsc_idx]; set_bit(ctx->sa.assoc_num, &aq_txsc->tx_sa_idx_busy); @@ -631,9 +613,6 @@ static int aq_mdo_upd_txsa(struct macsec_context *ctx) if (txsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - aq_txsc = &cfg->aq_txsc[txsc_idx]; if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev)) ret = aq_update_txsa(nic, aq_txsc->hw_sc_idx, secy, @@ -681,9 +660,6 @@ static int aq_mdo_del_txsa(struct macsec_context *ctx) if (txsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - ret = aq_clear_txsa(nic, &cfg->aq_txsc[txsc_idx], ctx->sa.assoc_num, AQ_CLEAR_ALL); @@ -780,9 +756,6 @@ static int aq_mdo_add_rxsc(struct macsec_context *ctx) if (rxsc_idx >= rxsc_idx_max) return -ENOSPC; - if (ctx->prepare) - return 0; - cfg->aq_rxsc[rxsc_idx].hw_sc_idx = aq_to_hw_sc_idx(rxsc_idx, cfg->sc_sa); cfg->aq_rxsc[rxsc_idx].sw_secy = ctx->secy; @@ -809,9 +782,6 @@ static int aq_mdo_upd_rxsc(struct macsec_context *ctx) if (rxsc_idx < 0) return -ENOENT; - if (ctx->prepare) - return 0; - if (netif_carrier_ok(nic->ndev) && netif_running(ctx->secy->netdev)) ret = aq_set_rxsc(nic, rxsc_idx); @@ -876,9 +846,6 @@ static int aq_mdo_del_rxsc(struct macsec_context *ctx) if (rxsc_idx < 0) return -ENOENT; - if (ctx->prepare) - return 0; - if (netif_carrier_ok(nic->ndev)) clear_type = AQ_CLEAR_ALL; @@ -948,9 +915,6 @@ static int aq_mdo_add_rxsa(struct macsec_context *ctx) if (rxsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - aq_rxsc = &nic->macsec_cfg->aq_rxsc[rxsc_idx]; set_bit(ctx->sa.assoc_num, &aq_rxsc->rx_sa_idx_busy); @@ -978,9 +942,6 @@ static int aq_mdo_upd_rxsa(struct macsec_context *ctx) if (rxsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev)) ret = aq_update_rxsa(nic, cfg->aq_rxsc[rxsc_idx].hw_sc_idx, secy, ctx->sa.rx_sa, NULL, @@ -1029,9 +990,6 @@ static int aq_mdo_del_rxsa(struct macsec_context *ctx) if (rxsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - ret = aq_clear_rxsa(nic, &cfg->aq_rxsc[rxsc_idx], ctx->sa.assoc_num, AQ_CLEAR_ALL); @@ -1044,9 +1002,6 @@ static int aq_mdo_get_dev_stats(struct macsec_context *ctx) struct aq_macsec_common_stats *stats = &nic->macsec_cfg->stats; struct aq_hw_s *hw = nic->aq_hw; - if (ctx->prepare) - return 0; - aq_get_macsec_common_stats(hw, stats); ctx->stats.dev_stats->OutPktsUntagged = stats->out.untagged_pkts; @@ -1073,9 +1028,6 @@ static int aq_mdo_get_tx_sc_stats(struct macsec_context *ctx) if (txsc_idx < 0) return -ENOENT; - if (ctx->prepare) - return 0; - aq_txsc = &nic->macsec_cfg->aq_txsc[txsc_idx]; stats = &aq_txsc->stats; aq_get_txsc_stats(hw, aq_txsc->hw_sc_idx, stats); @@ -1106,9 +1058,6 @@ static int aq_mdo_get_tx_sa_stats(struct macsec_context *ctx) if (txsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - aq_txsc = &cfg->aq_txsc[txsc_idx]; sa_idx = aq_txsc->hw_sc_idx | ctx->sa.assoc_num; stats = &aq_txsc->tx_sa_stats[ctx->sa.assoc_num]; @@ -1147,9 +1096,6 @@ static int aq_mdo_get_rx_sc_stats(struct macsec_context *ctx) if (rxsc_idx < 0) return -ENOENT; - if (ctx->prepare) - return 0; - aq_rxsc = &cfg->aq_rxsc[rxsc_idx]; for (i = 0; i < MACSEC_NUM_AN; i++) { if (!test_bit(i, &aq_rxsc->rx_sa_idx_busy)) @@ -1196,9 +1142,6 @@ static int aq_mdo_get_rx_sa_stats(struct macsec_context *ctx) if (rxsc_idx < 0) return -EINVAL; - if (ctx->prepare) - return 0; - aq_rxsc = &cfg->aq_rxsc[rxsc_idx]; stats = &aq_rxsc->rx_sa_stats[ctx->sa.assoc_num]; sa_idx = aq_rxsc->hw_sc_idx | ctx->sa.assoc_num; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_main.c b/drivers/net/ethernet/aquantia/atlantic/aq_main.c index 88595863d8bc6cb7472b0cb859aed1674ddc162b..8a0af371e7dc7a0c865ac0f145b2fe02f114b177 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_main.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_main.c @@ -94,11 +94,8 @@ static int aq_ndev_close(struct net_device *ndev) int err = 0; err = aq_nic_stop(aq_nic); - if (err < 0) - goto err_exit; aq_nic_deinit(aq_nic, true); -err_exit: return err; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c b/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c index 275324c9e51ec227b98ab9a3bd2531c829a25d78..80b44043e6c53f07a1c54b5100f19f7d0d8bee08 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c @@ -1217,8 +1217,7 @@ int aq_ptp_init(struct aq_nic_s *aq_nic, unsigned int idx_vec) atomic_set(&aq_ptp->offset_egress, 0); atomic_set(&aq_ptp->offset_ingress, 0); - netif_napi_add(aq_nic_get_ndev(aq_nic), &aq_ptp->napi, - aq_ptp_poll, NAPI_POLL_WEIGHT); + netif_napi_add(aq_nic_get_ndev(aq_nic), &aq_ptp->napi, aq_ptp_poll); aq_ptp->idx_vector = idx_vec; diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c index f0fdf20f01c1601915eacadea46ac6fc0652f5ce..f5db1c44e9b9171f0ab53d10f99b482a47d2fb0e 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c @@ -119,8 +119,7 @@ struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx, self->tx_rings = 0; self->rx_rings = 0; - netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, - aq_vec_poll, NAPI_POLL_WEIGHT); + netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll); err_exit: return self; diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c index 288e2961823eaf3b008edf709b55bb5b6f7db34b..ba0646b3b122e9a79b05d7c72fb4c441f23a432b 100644 --- a/drivers/net/ethernet/arc/emac_main.c +++ b/drivers/net/ethernet/arc/emac_main.c @@ -91,7 +91,7 @@ static void arc_emac_get_drvinfo(struct net_device *ndev, { struct arc_emac_priv *priv = netdev_priv(ndev); - strlcpy(info->driver, priv->drv_name, sizeof(info->driver)); + strscpy(info->driver, priv->drv_name, sizeof(info->driver)); } static const struct ethtool_ops arc_emac_ethtool_ops = { diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c index 6ba5b024a7be7af0791e9a7687b18d6d85b83781..8b7cdf015a16ee4700a05db36482f5585769bbda 100644 --- a/drivers/net/ethernet/asix/ax88796c_main.c +++ b/drivers/net/ethernet/asix/ax88796c_main.c @@ -293,7 +293,7 @@ ax88796c_tx_fixup(struct net_device *ndev, struct sk_buff_head *q) skb_put(skb, padlen); /* EOP header */ - memcpy(skb_put(skb, TX_EOP_SIZE), &info.eop, TX_EOP_SIZE); + skb_put_data(skb, &info.eop, TX_EOP_SIZE); skb_unlink(skb, q); @@ -381,7 +381,7 @@ static int ax88796c_hard_xmit(struct ax88796c_device *ax_local) return 1; } -static int +static netdev_tx_t ax88796c_start_xmit(struct sk_buff *skb, struct net_device *ndev) { struct ax88796c_device *ax_local = to_ax88796c_device(ndev); diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index e461f47640660ca00365251ac2d88bb80e5f8bd1..cc932b3cf873acbd51f7d36e13b03a021b6d6ea9 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -451,8 +451,8 @@ static void ag71xx_get_drvinfo(struct net_device *ndev, { struct ag71xx *ag = netdev_priv(ndev); - strlcpy(info->driver, "ag71xx", sizeof(info->driver)); - strlcpy(info->bus_info, of_node_full_name(ag->pdev->dev.of_node), + strscpy(info->driver, "ag71xx", sizeof(info->driver)); + strscpy(info->bus_info, of_node_full_name(ag->pdev->dev.of_node), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c index a89b93cb4e26d91c8ab1be848b4a06696b30946b..d30d11872719f7c4eac6c06a0ce61f119174de55 100644 --- a/drivers/net/ethernet/atheros/alx/main.c +++ b/drivers/net/ethernet/atheros/alx/main.c @@ -752,7 +752,7 @@ static int alx_alloc_napis(struct alx_priv *alx) goto err_out; np->alx = alx; - netif_napi_add(alx->dev, &np->napi, alx_poll, 64); + netif_napi_add(alx->dev, &np->napi, alx_poll); alx->qnapi[i] = np; } @@ -1912,11 +1912,14 @@ static int alx_suspend(struct device *dev) if (!netif_running(alx->dev)) return 0; + + rtnl_lock(); netif_device_detach(alx->dev); mutex_lock(&alx->mtx); __alx_stop(alx); mutex_unlock(&alx->mtx); + rtnl_unlock(); return 0; } @@ -1927,6 +1930,7 @@ static int alx_resume(struct device *dev) struct alx_hw *hw = &alx->hw; int err; + rtnl_lock(); mutex_lock(&alx->mtx); alx_reset_phy(hw); @@ -1943,6 +1947,7 @@ static int alx_resume(struct device *dev) unlock: mutex_unlock(&alx->mtx); + rtnl_unlock(); return err; } diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c b/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c index e2eb7b8c63a0d6d0d769a85ea24bdf79429d63af..0bce122c68f1d6ff1bc9ab90edba8011eb38225d 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c @@ -220,8 +220,8 @@ static void atl1c_get_drvinfo(struct net_device *netdev, { struct atl1c_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, atl1c_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, atl1c_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c index be4b1f8eef294f0b59f2afe47a8f54647fffbd18..40c781695d581d2270c19504c7687392ada00fc7 100644 --- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c @@ -2732,7 +2732,7 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_set_threaded(netdev, true); for (i = 0; i < adapter->rx_queue_count; ++i) netif_napi_add(netdev, &adapter->rrd_ring[i].napi, - atl1c_clean_rx, 64); + atl1c_clean_rx); for (i = 0; i < adapter->tx_queue_count; ++i) netif_napi_add_tx(netdev, &adapter->tpd_ring[i].napi, atl1c_clean_tx); diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c b/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c index 0cbde352d1babf05b1993b14abc52f6d569fd624..68f1832a198dd1c52135ff6c0d4b3e0df6293730 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c @@ -306,9 +306,9 @@ static void atl1e_get_drvinfo(struct net_device *netdev, { struct atl1e_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, atl1e_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, "L1e", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, atl1e_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->fw_version, "L1e", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c index 57a51fb7746c8efe1277cf1db0e82419167f014c..5db0f3495a32e647988353de98348c4470d0fb66 100644 --- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c @@ -2354,7 +2354,7 @@ static int atl1e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->mii.phy_id_mask = 0x1f; adapter->mii.reg_num_mask = MDIO_REG_ADDR_MASK; - netif_napi_add(netdev, &adapter->napi, atl1e_clean, 64); + netif_napi_add(netdev, &adapter->napi, atl1e_clean); timer_setup(&adapter->phy_config_timer, atl1e_phy_config, 0); diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index ff1fe09abf9f5659366f4f5816fb782e642dee95..c8444bcdf52707a89bb72c09a3e88fcf08bd2ae7 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -2977,7 +2977,7 @@ static int atl1_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->netdev_ops = &atl1_netdev_ops; netdev->watchdog_timeo = 5 * HZ; - netif_napi_add(netdev, &adapter->napi, atl1_rings_clean, 64); + netif_napi_add(netdev, &adapter->napi, atl1_rings_clean); netdev->ethtool_ops = &atl1_ethtool_ops; adapter->bd_number = cards_found; @@ -3340,8 +3340,8 @@ static void atl1_get_drvinfo(struct net_device *netdev, { struct atl1_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, ATLX_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, ATLX_DRIVER_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c index bbc4d7b08a498f72fd93d626c434bfbcef1578c7..1b487c071cb6281730c20f6481117cc6221e1de4 100644 --- a/drivers/net/ethernet/atheros/atlx/atl2.c +++ b/drivers/net/ethernet/atheros/atlx/atl2.c @@ -1980,9 +1980,9 @@ static void atl2_get_drvinfo(struct net_device *netdev, { struct atl2_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, atl2_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, "L2", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, atl2_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->fw_version, "L2", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index 56e0fb07aec7fde44e11ff8c1b9e64ff8e13b83a..f4e1ca68d831002a250192e5839eb1f657e1b032 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -53,8 +53,8 @@ config B44_PCI config BCM4908_ENET tristate "Broadcom BCM4908 internal mac support" - depends on ARCH_BCM4908 || COMPILE_TEST - default y if ARCH_BCM4908 + depends on ARCH_BCMBCA || COMPILE_TEST + default y if ARCH_BCMBCA help This driver supports Ethernet controller integrated into Broadcom BCM4908 family SoCs. diff --git a/drivers/net/ethernet/broadcom/Makefile b/drivers/net/ethernet/broadcom/Makefile index 2e6c5f258a1ffe523dfeafa90114617ab75b0f6b..0ddfb5b5d53ca83942d74f1a7796587c4c8b9fd0 100644 --- a/drivers/net/ethernet/broadcom/Makefile +++ b/drivers/net/ethernet/broadcom/Makefile @@ -17,8 +17,3 @@ obj-$(CONFIG_BGMAC_BCMA) += bgmac-bcma.o bgmac-bcma-mdio.o obj-$(CONFIG_BGMAC_PLATFORM) += bgmac-platform.o obj-$(CONFIG_SYSTEMPORT) += bcmsysport.o obj-$(CONFIG_BNXT) += bnxt/ - -# FIXME: temporarily silence -Warray-bounds on non W=1+ builds -ifndef KBUILD_EXTRA_WARN -CFLAGS_tg3.o += -Wno-array-bounds -endif diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index e5857e88c20765cfa2f9065311b812aa9b64f04b..7f876721596c1a6067301a635ad08eb055b8a65d 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1790,13 +1790,13 @@ static void b44_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *inf struct b44 *bp = netdev_priv(dev); struct ssb_bus *bus = bp->sdev->bus; - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); switch (bus->bustype) { case SSB_BUSTYPE_PCI: - strlcpy(info->bus_info, pci_name(bus->host_pci), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(bus->host_pci), sizeof(info->bus_info)); break; case SSB_BUSTYPE_SSB: - strlcpy(info->bus_info, "SSB", sizeof(info->bus_info)); + strscpy(info->bus_info, "SSB", sizeof(info->bus_info)); break; case SSB_BUSTYPE_PCMCIA: case SSB_BUSTYPE_SDIO: @@ -2375,7 +2375,7 @@ static int b44_init_one(struct ssb_device *sdev, bp->tx_pending = B44_DEF_TX_RING_PENDING; dev->netdev_ops = &b44_netdev_ops; - netif_napi_add(dev, &bp->napi, b44_poll, 64); + netif_napi_add(dev, &bp->napi, b44_poll); dev->watchdog_timeo = B44_TX_TIMEOUT; dev->min_mtu = B44_MIN_MTU; dev->max_mtu = B44_MAX_MTU; diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c index c131d81184893f75c8a8c2540de645cbcbc09a1e..93ccf549e2ed34e05e88d7c6049089d97936b43d 100644 --- a/drivers/net/ethernet/broadcom/bcm4908_enet.c +++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c @@ -507,7 +507,7 @@ static int bcm4908_enet_stop(struct net_device *netdev) return 0; } -static int bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev) +static netdev_tx_t bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev) { struct bcm4908_enet *enet = netdev_priv(netdev); struct bcm4908_enet_dma_ring *ring = &enet->tx_ring; @@ -716,6 +716,8 @@ static int bcm4908_enet_probe(struct platform_device *pdev) SET_NETDEV_DEV(netdev, &pdev->dev); err = of_get_ethdev_address(dev->of_node, netdev); + if (err == -EPROBE_DEFER) + goto err_dma_free; if (err) eth_hw_addr_random(netdev); netdev->netdev_ops = &bcm4908_enet_netdev_ops; @@ -723,17 +725,20 @@ static int bcm4908_enet_probe(struct platform_device *pdev) netdev->mtu = ETH_DATA_LEN; netdev->max_mtu = ENET_MTU_MAX; netif_napi_add_tx(netdev, &enet->tx_ring.napi, bcm4908_enet_poll_tx); - netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx); err = register_netdev(netdev); - if (err) { - bcm4908_enet_dma_free(enet); - return err; - } + if (err) + goto err_dma_free; platform_set_drvdata(pdev, enet); return 0; + +err_dma_free: + bcm4908_enet_dma_free(enet); + + return err; } static int bcm4908_enet_remove(struct platform_device *pdev) diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index 1c6aea12db721602e9cc103f058ca341db9bd6e4..d91fdb0c2649d895aa7076910482fb24a0e60e79 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -1321,8 +1321,8 @@ static const u32 unused_mib_regs[] = { static void bcm_enet_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, bcm_enet_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, "bcm63xx", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->driver, bcm_enet_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, "bcm63xx", sizeof(drvinfo->bus_info)); } static int bcm_enet_get_sset_count(struct net_device *netdev, diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index 47fc8e6963d599c5712c8102f7403999ba795984..867f14c30e09fbaf618586917948e6dc2ac9e755 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -308,8 +308,8 @@ static const struct bcm_sysport_stats bcm_sysport_gstrings_stats[] = { static void bcm_sysport_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->bus_info, "platform", sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->bus_info, "platform", sizeof(info->bus_info)); } static u32 bcm_sysport_get_msglvl(struct net_device *dev) @@ -2564,7 +2564,7 @@ static int bcm_sysport_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, dev); dev->ethtool_ops = &bcm_sysport_ethtool_ops; dev->netdev_ops = &bcm_sysport_netdev_ops; - netif_napi_add(dev, &priv->napi, bcm_sysport_poll, 64); + netif_napi_add(dev, &priv->napi, bcm_sysport_poll); dev->features |= NETIF_F_RXCSUM | NETIF_F_HIGHDMA | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h index 16b73bb9acc783b41aefb2e6198f354d9e9cc6bd..5af16e5f9ad08934069ebb8df8565cc7881ac114 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.h +++ b/drivers/net/ethernet/broadcom/bcmsysport.h @@ -484,7 +484,7 @@ struct bcm_rsb { /* Number of Receive hardware descriptor words */ #define SP_NUM_HW_RX_DESC_WORDS 1024 -#define SP_LT_NUM_HW_RX_DESC_WORDS 256 +#define SP_LT_NUM_HW_RX_DESC_WORDS 512 /* Internal linked-list RAM size */ #define SP_NUM_TX_DESC 1536 diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c index 93580484a3f4e6a735ed7ce139c7718abdf0b537..5fb3af5670ecbbc80b5045866a70780e653ec7d4 100644 --- a/drivers/net/ethernet/broadcom/bgmac.c +++ b/drivers/net/ethernet/broadcom/bgmac.c @@ -1367,7 +1367,7 @@ static void bgmac_get_strings(struct net_device *dev, u32 stringset, return; for (i = 0; i < BGMAC_STATS_LEN; i++) - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, bgmac_get_strings_stats[i].name, ETH_GSTRING_LEN); } @@ -1395,8 +1395,8 @@ static void bgmac_get_ethtool_stats(struct net_device *dev, static void bgmac_get_drvinfo(struct net_device *net_dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->bus_info, "AXI", sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->bus_info, "AXI", sizeof(info->bus_info)); } static const struct ethtool_ops bgmac_ethtool_ops = { @@ -1527,7 +1527,7 @@ int bgmac_enet_probe(struct bgmac *bgmac) if (bcm47xx_nvram_getenv("et0_no_txint", NULL, 0) == 0) bgmac->int_mask &= ~BGMAC_IS_TX_MASK; - netif_napi_add(net_dev, &bgmac->napi, bgmac_poll, NAPI_POLL_WEIGHT); + netif_napi_add(net_dev, &bgmac->napi, bgmac_poll); err = bgmac_phy_connect(bgmac); if (err) { diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index b97ed9b5f685c3c7dddb3ae2d11c0c32a7263380..fec57f1982c86a0995ff36129c892faee16be866 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -176,12 +176,12 @@ static const struct flash_spec flash_table[] = {0x19000002, 0x5b808201, 0x000500db, 0x03840253, 0xaf020406, NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE, ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*2, - "Entry 0101: ST M45PE10 (128kB non-bufferred)"}, + "Entry 0101: ST M45PE10 (128kB non-buffered)"}, /* Entry 0110: ST M45PE20 (non-buffered flash)*/ {0x15000001, 0x57808201, 0x000500db, 0x03840253, 0xaf020406, NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE, ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*4, - "Entry 0110: ST M45PE20 (256kB non-bufferred)"}, + "Entry 0110: ST M45PE20 (256kB non-buffered)"}, /* Saifun SA25F005 (non-buffered flash) */ /* strap, cfg1, & write1 need updates */ {0x1d000003, 0x5f808201, 0x00050081, 0x03840253, 0xaf020406, @@ -7042,9 +7042,9 @@ bnx2_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct bnx2 *bp = netdev_priv(dev); - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); - strlcpy(info->fw_version, bp->fw_version, sizeof(info->fw_version)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); + strscpy(info->fw_version, bp->fw_version, sizeof(info->fw_version)); } #define BNX2_REGDUMP_LEN (32 * 1024) @@ -8522,7 +8522,7 @@ bnx2_init_napi(struct bnx2 *bp) else poll = bnx2_poll_msix; - netif_napi_add(bp->dev, &bp->bnx2_napi[i].napi, poll, 64); + netif_napi_add(bp->dev, &bp->bnx2_napi[i].napi, poll); bnapi->bp = bp; } } diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index 712b5595bc39385210aac42a9a34b16fa70ce35b..16c490692f4221aa5fa415fa045e3a67fd2c4d2c 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -44,8 +44,7 @@ static void bnx2x_add_all_napi_cnic(struct bnx2x *bp) /* Add NAPI objects */ for_each_rx_queue_cnic(bp, i) { - netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), - bnx2x_poll, NAPI_POLL_WEIGHT); + netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll); } } @@ -55,8 +54,7 @@ static void bnx2x_add_all_napi(struct bnx2x *bp) /* Add NAPI objects */ for_each_eth_queue(bp, i) { - netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), - bnx2x_poll, NAPI_POLL_WEIGHT); + netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll); } } @@ -150,7 +148,7 @@ void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len) phy_fw_ver[0] = '\0'; bnx2x_get_ext_phy_fw_version(&bp->link_params, phy_fw_ver, PHY_FW_VER_LEN); - strlcpy(buf, bp->fw_ver, buf_len); + strscpy(buf, bp->fw_ver, buf_len); snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver), "bc %d.%d.%d%s%s", (bp->common.bc_ver & 0xff0000) >> 16, @@ -789,6 +787,7 @@ static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp, BNX2X_ERR("skb_put is about to fail... pad %d len %d rx_buf_size %d\n", pad, len, fp->rx_buf_size); bnx2x_panic(); + bnx2x_frag_free(fp, new_data); return; } #endif diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 0e319ac7799f93cffeb8fda8da2ff60a912cb5c4..bda3ccc28eca67c9e24ace5c6f094cb4b828a2b1 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -1112,7 +1112,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev, int ext_dev_info_offset; u32 mbi; - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); if (SHMEM2_HAS(bp, extended_dev_info_shared_addr)) { ext_dev_info_offset = SHMEM2_RD(bp, @@ -1126,7 +1126,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev, (mbi & 0xff000000) >> 24, (mbi & 0x00ff0000) >> 16, (mbi & 0x0000ff00) >> 8); - strlcpy(info->fw_version, version, + strscpy(info->fw_version, version, sizeof(info->fw_version)); } } @@ -1135,7 +1135,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev, bnx2x_fill_fw_str(bp, version, ETHTOOL_FWVERS_LEN); strlcat(info->fw_version, version, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); } static void bnx2x_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 962253db25b820ee70fd8578cc90488a6beb7513..51b1690fd0459bb1497c29d9312fa559e2ed0009 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -3385,7 +3385,7 @@ static void bnx2x_drv_info_ether_stat(struct bnx2x *bp) &bp->sp_objs->mac_obj; int i; - strlcpy(ether_stat->version, DRV_MODULE_VERSION, + strscpy(ether_stat->version, DRV_MODULE_VERSION, ETH_STAT_INFO_VERSION_LEN); /* get DRV_INFO_ETH_STAT_NUM_MACS_REQUIRED macs, placing them in the diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h index 2dac704dc3462fa35229d0c785ca2cfc20164545..02a4e557e176d6379bcf601eb9966feb2e117734 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h @@ -518,7 +518,7 @@ int bnx2x_vfpf_storm_rx_mode(struct bnx2x *bp); static inline void bnx2x_vf_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len) { - strlcpy(buf, bp->acquire_resp.pfdev_info.fw_ver, buf_len); + strscpy(buf, bp->acquire_resp.pfdev_info.fw_ver, buf_len); } static inline int bnx2x_vf_ustorm_prods_offset(struct bnx2x *bp, diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c index c9129b9ba446b4d1e950110464919b301b089572..0657a0f5170f46d905b131fcbbf5ad4ca8b35769 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c @@ -380,7 +380,7 @@ int bnx2x_vfpf_acquire(struct bnx2x *bp, u8 tx_count, u8 rx_count) bp->igu_base_sb = bp->acquire_resp.resc.hw_sbs[0].hw_sb_id; bp->vlan_credit = bp->acquire_resp.resc.num_vlan_filters; - strlcpy(bp->fw_ver, bp->acquire_resp.pfdev_info.fw_ver, + strscpy(bp->fw_ver, bp->acquire_resp.pfdev_info.fw_ver, sizeof(bp->fw_ver)); if (is_valid_ether_addr(bp->acquire_resp.resc.current_mac_addr)) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index ba0f1ffac507126179af73feeb42ba581569bd40..04cf7684f1b0c2630c5984b1f7e9f4d78110aa8e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -659,7 +659,6 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) for (i = 0; i < nr_pkts; i++) { struct bnxt_sw_tx_bd *tx_buf; - bool compl_deferred = false; struct sk_buff *skb; int j, last; @@ -668,6 +667,8 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) skb = tx_buf->skb; tx_buf->skb = NULL; + tx_bytes += skb->len; + if (tx_buf->is_push) { tx_buf->is_push = 0; goto next_tx_int; @@ -688,8 +689,9 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) } if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { if (bp->flags & BNXT_FLAG_CHIP_P5) { + /* PTP worker takes ownership of the skb */ if (!bnxt_get_tx_ts_p5(bp, skb)) - compl_deferred = true; + skb = NULL; else atomic_inc(&bp->ptp_cfg->tx_avail); } @@ -698,9 +700,7 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) next_tx_int: cons = NEXT_TX(cons); - tx_bytes += skb->len; - if (!compl_deferred) - dev_kfree_skb_any(skb); + dev_kfree_skb_any(skb); } netdev_tx_completed_queue(txq, nr_pkts, tx_bytes); @@ -3874,7 +3874,7 @@ static void bnxt_init_vnics(struct bnxt *bp) if (bp->vnic_info[i].rss_hash_key) { if (i == 0) - prandom_bytes(vnic->rss_hash_key, + get_random_bytes(vnic->rss_hash_key, HW_HASH_KEY_SIZE); else memcpy(vnic->rss_hash_key, @@ -9366,16 +9366,16 @@ static void bnxt_init_napi(struct bnxt *bp) cp_nr_rings--; for (i = 0; i < cp_nr_rings; i++) { bnapi = bp->bnapi[i]; - netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64); + netif_napi_add(bp->dev, &bnapi->napi, poll_fn); } if (BNXT_CHIP_TYPE_NITRO_A0(bp)) { bnapi = bp->bnapi[cp_nr_rings]; netif_napi_add(bp->dev, &bnapi->napi, - bnxt_poll_nitroa0, 64); + bnxt_poll_nitroa0); } } else { bnapi = bp->bnapi[0]; - netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64); + netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll); } } @@ -11178,10 +11178,7 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev, if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp)) features &= ~NETIF_F_NTUPLE; - if (bp->flags & BNXT_FLAG_NO_AGG_RINGS) - features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); - - if (!(bp->flags & BNXT_FLAG_TPA)) + if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); if (!(features & NETIF_F_GRO)) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 075c6206325ce387052629a00a1da74b0c0c5462..b1b17f911300636983455eb11c6f50116c3c583b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2130,6 +2130,7 @@ struct bnxt { #define BNXT_DUMP_CRASH 1 struct bpf_prog *xdp_prog; + u8 xdp_has_frags; struct bnxt_ptp_cfg *ptp_cfg; u8 ptp_all_rx_tstamp; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c index 059f96f7a96f6495f691118b7ffdf5cb9675db96..a36803e79e92e1b78147bb308020aba27f4384ab 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c @@ -1306,6 +1306,7 @@ int bnxt_dl_register(struct bnxt *bp) if (rc) goto err_dl_port_unreg; + devlink_set_features(dl, DEVLINK_F_RELOAD); out: devlink_register(dl); return 0; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 87eb5362ad70a11397b8ee2100dcd3bab511b6ba..f57e524c7e30b981a17697c875bb73e70038efd5 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -1371,9 +1371,9 @@ static void bnxt_get_drvinfo(struct net_device *dev, { struct bnxt *bp = netdev_priv(dev); - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->fw_version, bp->fw_ver_str, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->fw_version, bp->fw_ver_str, sizeof(info->fw_version)); + strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info)); info->n_stats = bnxt_get_num_stats(bp); info->testinfo_len = bp->num_tests; /* TODO CHIMP_FW: eeprom dump details */ @@ -3876,7 +3876,7 @@ void bnxt_ethtool_init(struct bnxt *bp) } else if (i == BNXT_IRQ_TEST_IDX) { strcpy(str, "Interrupt_test (offline)"); } else { - strlcpy(str, fw_str, ETH_GSTRING_LEN); + strscpy(str, fw_str, ETH_GSTRING_LEN); strncat(str, " test", ETH_GSTRING_LEN - strlen(str)); if (test_info->offline_mask & (1 << i)) strncat(str, " (offline)", diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c index 7f3c0875b6f5834e58223a13c3cfb1d46b02cec4..2132ce63193ce097f917022d2527397d3f4ef5dd 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c @@ -317,9 +317,9 @@ void bnxt_ptp_cfg_tstamp_filters(struct bnxt *bp) if (!(bp->fw_cap & BNXT_FW_CAP_RX_ALL_PKT_TS) && (ptp->tstamp_filters & (PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_ENABLE | - PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE))) { + PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_DISABLE))) { ptp->tstamp_filters &= ~(PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_ENABLE | - PORT_MAC_CFG_REQ_FLAGS_PTP_RX_TS_CAPTURE_DISABLE); + PORT_MAC_CFG_REQ_FLAGS_ALL_RX_TS_CAPTURE_DISABLE); netdev_warn(bp->dev, "Unsupported FW for all RX pkts timestamp filter\n"); } @@ -505,9 +505,13 @@ static int bnxt_hwrm_ptp_cfg(struct bnxt *bp) ptp->tstamp_filters = flags; if (netif_running(bp->dev)) { - rc = bnxt_close_nic(bp, false, false); - if (!rc) - rc = bnxt_open_nic(bp, false, false); + if (ptp->rx_filter == HWTSTAMP_FILTER_ALL) { + rc = bnxt_close_nic(bp, false, false); + if (!rc) + rc = bnxt_open_nic(bp, false, false); + } else { + bnxt_ptp_cfg_tstamp_filters(bp); + } if (!rc && !ptp->tstamp_filters) rc = -EIO; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c index 730febd19330afaa74b25dc2d845a52d79a53bd0..a4cba7cb2783ea70cbcf4b5b40c161b40aa19739 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c @@ -623,7 +623,7 @@ static int bnxt_hwrm_func_vf_resc_cfg(struct bnxt *bp, int num_vfs, bool reset) hw_resc->max_stat_ctxs -= le16_to_cpu(req->min_stat_ctx) * n; hw_resc->max_vnics -= le16_to_cpu(req->min_vnics) * n; if (bp->flags & BNXT_FLAG_CHIP_P5) - hw_resc->max_irqs -= vf_msix * n; + hw_resc->max_nqs -= vf_msix; rc = pf->active_vfs; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c index eb4803b11c0e29e1eddaf12c87f542fc0402fc9d..fcc65890820af62bce5c06ccce018a7781541ee2 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c @@ -222,7 +222,7 @@ static int bnxt_vf_rep_get_phys_port_name(struct net_device *dev, char *buf, static void bnxt_vf_rep_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); } static int bnxt_vf_rep_get_port_parent_id(struct net_device *dev, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index f53387ed0167bb93dbba6659c42c0ff5c2b63cfb..c3065ec0a47981064c133ec9bff25bf44e3eefd0 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -181,6 +181,7 @@ void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp) { struct bnxt_sw_rx_bd *rx_buf; + u32 buflen = PAGE_SIZE; struct pci_dev *pdev; dma_addr_t mapping; u32 offset; @@ -192,7 +193,10 @@ void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, mapping = rx_buf->mapping - bp->rx_dma_offset; dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); - xdp_init_buff(xdp, BNXT_PAGE_MODE_BUF_SIZE + offset, &rxr->xdp_rxq); + if (bp->xdp_has_frags) + buflen = BNXT_PAGE_MODE_BUF_SIZE + offset; + + xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false); } @@ -397,8 +401,10 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n"); return -EOPNOTSUPP; } - if (prog) + if (prog) { tx_xdp = bp->rx_nr_rings; + bp->xdp_has_frags = prog->aux->xdp_has_frags; + } tc = netdev_get_num_tc(dev); if (!tc) diff --git a/drivers/net/ethernet/broadcom/cnic.c b/drivers/net/ethernet/broadcom/cnic.c index e86503d97f32bffc1eb4bc52b710c2b093cf2e89..2198e35d9e18183e96c662a261e17dcc05a193b7 100644 --- a/drivers/net/ethernet/broadcom/cnic.c +++ b/drivers/net/ethernet/broadcom/cnic.c @@ -4105,8 +4105,7 @@ static int cnic_cm_alloc_mem(struct cnic_dev *dev) for (i = 0; i < MAX_CM_SK_TBL_SZ; i++) atomic_set(&cp->csk_tbl[i].ref_count, 0); - port_id = prandom_u32(); - port_id %= CNIC_LOCAL_PORT_RANGE; + port_id = prandom_u32_max(CNIC_LOCAL_PORT_RANGE); if (cnic_init_id_tbl(&cp->csk_port_tbl, CNIC_LOCAL_PORT_RANGE, CNIC_LOCAL_PORT_MIN, port_id)) { cnic_cm_free_mem(dev); @@ -4165,7 +4164,7 @@ static int cnic_cm_init_bnx2_hw(struct cnic_dev *dev) { u32 seed; - seed = prandom_u32(); + seed = get_random_u32(); cnic_ctx_wr(dev, 45, 0, seed); return 0; } diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 8309fb993cdbfb27fbe27c82873cb90e16816c68..25c4506069856bb2dbdcdc703b649f81b6a156f2 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1146,7 +1146,7 @@ static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = { static void bcmgenet_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "bcmgenet", sizeof(info->driver)); + strscpy(info->driver, "bcmgenet", sizeof(info->driver)); } static int bcmgenet_get_sset_count(struct net_device *dev, int string_set) @@ -2707,8 +2707,7 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, bcmgenet_init_rx_coalesce(ring); /* Initialize Rx NAPI */ - netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll); bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX); bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index db1e9d810b4160aff3c3b3fb22938f7d10ea6763..4179a12fc881950a04871faa592e55a98d275cf9 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -7380,9 +7380,9 @@ static void tg3_napi_init(struct tg3 *tp) { int i; - netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64); + netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll); for (i = 1; i < tp->irq_cnt; i++) - netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64); + netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix); } static void tg3_napi_fini(struct tg3 *tp) @@ -12302,9 +12302,9 @@ static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info { struct tg3 *tp = netdev_priv(dev); - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version)); + strscpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info)); } static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) @@ -18076,16 +18076,20 @@ static void tg3_shutdown(struct pci_dev *pdev) struct net_device *dev = pci_get_drvdata(pdev); struct tg3 *tp = netdev_priv(dev); + tg3_reset_task_cancel(tp); + rtnl_lock(); + netif_device_detach(dev); if (netif_running(dev)) dev_close(dev); - if (system_state == SYSTEM_POWER_OFF) - tg3_power_down(tp); + tg3_power_down(tp); rtnl_unlock(); + + pci_disable_device(pdev); } /** diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index 29dd0f93d6c03c5cf16331d35b594680fec56882..d6d90f9722a7e771f24e96ae015fc660cb794019 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -1891,7 +1891,7 @@ bnad_napi_add(struct bnad *bnad, u32 rx_id) for (i = 0; i < bnad->num_rxp_per_rx; i++) { rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i]; netif_napi_add(bnad->netdev, &rx_ctrl->napi, - bnad_napi_poll_rx, NAPI_POLL_WEIGHT); + bnad_napi_poll_rx); } } diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c index 8aca768571b2a0c7c1a8371f8f44786668e3c2e5..df10edff5603aa27d38b88d2b9ed32d5d3701514 100644 --- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c +++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c @@ -114,7 +114,7 @@ static const char *bnad_net_stats_strings[] = { "mac_tx_deferral", "mac_tx_excessive_deferral", "mac_tx_single_collision", - "mac_tx_muliple_collision", + "mac_tx_multiple_collision", "mac_tx_late_collision", "mac_tx_excessive_collision", "mac_tx_total_collision", @@ -283,7 +283,7 @@ bnad_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) struct bfa_ioc_attr *ioc_attr; unsigned long flags; - strlcpy(drvinfo->driver, BNAD_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, BNAD_NAME, sizeof(drvinfo->driver)); ioc_attr = kzalloc(sizeof(*ioc_attr), GFP_KERNEL); if (ioc_attr) { @@ -291,12 +291,12 @@ bnad_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) bfa_nw_ioc_get_attr(&bnad->bna.ioceth.ioc, ioc_attr); spin_unlock_irqrestore(&bnad->bna_lock, flags); - strlcpy(drvinfo->fw_version, ioc_attr->adapter_attr.fw_ver, + strscpy(drvinfo->fw_version, ioc_attr->adapter_attr.fw_ver, sizeof(drvinfo->fw_version)); kfree(ioc_attr); } - strlcpy(drvinfo->bus_info, pci_name(bnad->pcidev), + strscpy(drvinfo->bus_info, pci_name(bnad->pcidev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 66c7d08d376a7ef0c97abf63126cc209b7acefa2..51c9fd6f68a42f295b7386eeba3b1b188ac1005a 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "macb.h" /* This structure is only used for MACB on SiFive FU540 devices */ @@ -3977,8 +3978,8 @@ static int macb_init(struct platform_device *pdev) queue = &bp->queues[q]; queue->bp = bp; spin_lock_init(&queue->tx_ptr_lock); - netif_napi_add(dev, &queue->napi_rx, macb_rx_poll, NAPI_POLL_WEIGHT); - netif_napi_add(dev, &queue->napi_tx, macb_tx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); + netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); @@ -4621,6 +4622,25 @@ static int init_reset_optional(struct platform_device *pdev) "failed to init SGMII PHY\n"); } + ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); + if (!ret) { + u32 pm_info[2]; + + ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", + pm_info, ARRAY_SIZE(pm_info)); + if (ret) { + dev_err(&pdev->dev, "Failed to read power management information\n"); + goto err_out_phy_exit; + } + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); + if (ret) + goto err_out_phy_exit; + + ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); + if (ret) + goto err_out_phy_exit; + } + /* Fully reset controller at hardware level if mapped in device tree */ ret = device_reset_optional(&pdev->dev); if (ret) { @@ -4629,6 +4649,8 @@ static int init_reset_optional(struct platform_device *pdev) } ret = macb_init(pdev); + +err_out_phy_exit: if (ret) phy_exit(bp->sgmii_phy); @@ -5109,6 +5131,7 @@ static int __maybe_unused macb_suspend(struct device *dev) if (!(bp->wol & MACB_WOL_ENABLED)) { rtnl_lock(); phylink_stop(bp->phylink); + phy_exit(bp->sgmii_phy); rtnl_unlock(); spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); @@ -5198,6 +5221,9 @@ static int __maybe_unused macb_resume(struct device *dev) macb_set_rx_mode(netdev); macb_restore_features(bp); rtnl_lock(); + if (!device_may_wakeup(&bp->dev->dev)) + phy_init(bp->sgmii_phy); + phylink_start(bp->phylink); rtnl_unlock(); diff --git a/drivers/net/ethernet/calxeda/xgmac.c b/drivers/net/ethernet/calxeda/xgmac.c index 1281d1565ef84e298fb8270143ba1c8de0ef6bc0..f4f87dfa96874a13a39995651e464bfbc035b7b9 100644 --- a/drivers/net/ethernet/calxeda/xgmac.c +++ b/drivers/net/ethernet/calxeda/xgmac.c @@ -1792,7 +1792,7 @@ static int xgmac_probe(struct platform_device *pdev) netdev_warn(ndev, "MAC address %pM not valid", ndev->dev_addr); - netif_napi_add(ndev, &priv->napi, xgmac_poll, 64); + netif_napi_add(ndev, &priv->napi, xgmac_poll); ret = register_netdev(ndev); if (ret) goto err_reg; diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h index 3f1c189646f4e09f3435677978ab6ce7e1e74c0b..a0fd324762251ba5dc96fe06d01dfe086666edf8 100644 --- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h +++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h @@ -87,8 +87,8 @@ */ #define CN23XX_SLI_PKT_IN_JABBER 0x29170 /* The input jabber is used to determine the TSO max size. - * Due to H/W limitation, this need to be reduced to 60000 - * in order to to H/W TSO and avoid the WQE malfarmation + * Due to H/W limitation, this needs to be reduced to 60000 + * in order to use H/W TSO and avoid the WQE malformation * PKO_BUG_24989_WQE_LEN */ #define CN23XX_DEFAULT_INPUT_JABBER 0xEA60 /*60000*/ diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h index d33dd8f4226f4f5901d0923e7cefa08bc342e4f6..e956109415cd18bec6e8f63d3c4b7664f85db12e 100644 --- a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h +++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h @@ -36,8 +36,8 @@ #define CN23XX_CONFIG_PCIE_FLTMSK 0x720 /* The input jabber is used to determine the TSO max size. - * Due to H/W limitation, this need to be reduced to 60000 - * in order to to H/W TSO and avoid the WQE malfarmation + * Due to H/W limitation, this needs to be reduced to 60000 + * in order to use H/W TSO and avoid the WQE malformation * PKO_BUG_24989_WQE_LEN */ #define CN23XX_DEFAULT_INPUT_JABBER 0xEA60 /*60000*/ diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c index 73cb03266549ca29c1e869fef1ff7c487e9b1c0a..882b2be06ea0e0ac7c9031f95e7757364e12e532 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_core.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c @@ -851,7 +851,7 @@ int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx, napi = &droq->napi; dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx\n", (u64)netdev, (u64)octeon_dev); - netif_napi_add(netdev, napi, liquidio_napi_poll, 64); + netif_napi_add(netdev, napi, liquidio_napi_poll); /* designate a CPU for this droq */ droq->cpu_id = cpu_id; diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c index bee35ce60171006a206dfd3a76c670c65ab324a6..d312bd5949353c6b9fa1a0318a30c2dbcfbd04bb 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_main.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c @@ -92,11 +92,6 @@ static int octeon_console_debug_enabled(u32 console) /* time to wait for possible in-flight requests in milliseconds */ #define WAIT_INFLIGHT_REQUEST msecs_to_jiffies(1000) -struct lio_trusted_vf_ctx { - struct completion complete; - int status; -}; - struct oct_link_status_resp { u64 rh; struct oct_link_info link_info; diff --git a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c index 103591dcea1c3217978069b60c6e44f282d55662..edde0b8fa49cb84f69056d331fc6cc4b10470ea7 100644 --- a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c +++ b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c @@ -1342,7 +1342,7 @@ static void octeon_mgmt_poll_controller(struct net_device *netdev) static void octeon_mgmt_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); } static int octeon_mgmt_nway_reset(struct net_device *dev) @@ -1396,8 +1396,8 @@ static int octeon_mgmt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, netdev); p = netdev_priv(netdev); - netif_napi_add(netdev, &p->napi, octeon_mgmt_napi_poll, - OCTEON_MGMT_NAPI_WEIGHT); + netif_napi_add_weight(netdev, &p->napi, octeon_mgmt_napi_poll, + OCTEON_MGMT_NAPI_WEIGHT); p->netdev = netdev; p->dev = &pdev->dev; diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c index 5a9fad61e9eacd662545d090f28769983100f0f7..e5c71f90785237348ab57924037dea3e85f04f09 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c @@ -191,8 +191,8 @@ static void nicvf_get_drvinfo(struct net_device *netdev, { struct nicvf *nic = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info)); } static u32 nicvf_get_msglevel(struct net_device *netdev) diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index 768ea426d49f413ad4c0df97414334f9b8a7a6ad..98f3dc460ca7ca1a1eb284996a40b5dcdac44a1f 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -1472,8 +1472,7 @@ int nicvf_open(struct net_device *netdev) } cq_poll->cq_idx = qidx; cq_poll->nicvf = nic; - netif_napi_add(netdev, &cq_poll->napi, nicvf_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &cq_poll->napi, nicvf_poll); napi_enable(&cq_poll->napi); nic->napi[qidx] = cq_poll; } diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c index f4054d2553ea036bfaabf5965706a638db8098a1..d2286adf09fed53aea21534179a1f4cfce421a09 100644 --- a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c +++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c @@ -429,8 +429,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct adapter *adapter = dev->ml_priv; - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); } @@ -1053,7 +1053,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->hard_header_len += (netdev->hw_features & NETIF_F_TSO) ? sizeof(struct cpl_tx_pkt_lso) : sizeof(struct cpl_tx_pkt); - netif_napi_add(netdev, &adapter->napi, t1_poll, 64); + netif_napi_add(netdev, &adapter->napi, t1_poll); netdev->ethtool_ops = &t1_ethtool_ops; diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index 174b1e156669e58d59f3071d99a1b6d7fb14e9ee..a52e6b6e2876728431aa752dbe612e605bafcd96 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -609,8 +609,7 @@ static void init_napi(struct adapter *adap) struct sge_qset *qs = &adap->sge.qs[i]; if (qs->adap) - netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll, - 64); + netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll); } /* @@ -1627,8 +1626,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) t3_get_tp_version(adapter, &tp_vers); spin_unlock(&adapter->stats_lock); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); if (fw_vers) snprintf(info->fw_version, sizeof(info->fw_version), diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c index a7f291c89702196c0d244b4eac06732eb3e586df..557c591a6ce3a13b629e48c4bd6176428b3dfce6 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c @@ -14,6 +14,7 @@ #include "cudbg_entity.h" #include "cudbg_lib.h" #include "cudbg_zlib.h" +#include "cxgb4_tc_mqprio.h" static const u32 t6_tp_pio_array[][IREG_NUM_ELEM] = { {0x7e40, 0x7e44, 0x020, 28}, /* t6_tp_pio_regs_20_to_3b */ @@ -3458,7 +3459,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, for (i = 0; i < utxq->ntxq; i++) QDESC_GET_TXQ(&utxq->uldtxq[i].q, cudbg_uld_txq_to_qtype(j), - out_unlock); + out_unlock_uld); } } @@ -3475,7 +3476,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, for (i = 0; i < urxq->nrxq; i++) QDESC_GET_RXQ(&urxq->uldrxq[i].rspq, cudbg_uld_rxq_to_qtype(j), - out_unlock); + out_unlock_uld); } /* ULD FLQ */ @@ -3487,7 +3488,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, for (i = 0; i < urxq->nrxq; i++) QDESC_GET_FLQ(&urxq->uldrxq[i].fl, cudbg_uld_flq_to_qtype(j), - out_unlock); + out_unlock_uld); } /* ULD CIQ */ @@ -3500,29 +3501,34 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init, for (i = 0; i < urxq->nciq; i++) QDESC_GET_RXQ(&urxq->uldrxq[base + i].rspq, cudbg_uld_ciq_to_qtype(j), - out_unlock); + out_unlock_uld); } } + mutex_unlock(&uld_mutex); + + if (!padap->tc_mqprio) + goto out; + mutex_lock(&padap->tc_mqprio->mqprio_mutex); /* ETHOFLD TXQ */ if (s->eohw_txq) for (i = 0; i < s->eoqsets; i++) QDESC_GET_TXQ(&s->eohw_txq[i].q, - CUDBG_QTYPE_ETHOFLD_TXQ, out); + CUDBG_QTYPE_ETHOFLD_TXQ, out_unlock_mqprio); /* ETHOFLD RXQ and FLQ */ if (s->eohw_rxq) { for (i = 0; i < s->eoqsets; i++) QDESC_GET_RXQ(&s->eohw_rxq[i].rspq, - CUDBG_QTYPE_ETHOFLD_RXQ, out); + CUDBG_QTYPE_ETHOFLD_RXQ, out_unlock_mqprio); for (i = 0; i < s->eoqsets; i++) QDESC_GET_FLQ(&s->eohw_rxq[i].fl, - CUDBG_QTYPE_ETHOFLD_FLQ, out); + CUDBG_QTYPE_ETHOFLD_FLQ, out_unlock_mqprio); } -out_unlock: - mutex_unlock(&uld_mutex); +out_unlock_mqprio: + mutex_unlock(&padap->tc_mqprio->mqprio_mutex); out: qdesc_info->qdesc_entry_size = sizeof(*qdesc_entry); @@ -3559,6 +3565,10 @@ out_free: #undef QDESC_GET return rc; + +out_unlock_uld: + mutex_unlock(&uld_mutex); + goto out; } int cudbg_collect_flash(struct cudbg_init *pdbg_init, diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 77897edd2bc0a5a7c58a253e12de2bcc81038cc2..8477a93cee6bd453512ca5b3008c236c8e808b85 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -199,8 +199,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) struct adapter *adapter = netdev2adap(dev); u32 exprom_vers; - strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); info->regdump_len = get_regs_len(dev); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index d0061921529f39455a2f0c35a8ae42e9b1869c2c..9cbce1faab26f96f117728f79f5bdaf9a4e94c2d 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -3903,8 +3903,8 @@ static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, { struct adapter *adapter = netdev2adap(dev); - strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(adapter->pdev), + strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index ee52e3b1d74f73423274061c530b6ea79370d425..46809e2d94ee08bf619398791daf034beb4f8405 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -4467,7 +4467,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, if (ret) goto err; - netif_napi_add(dev, &iq->napi, napi_rx_handler, 64); + netif_napi_add(dev, &iq->napi, napi_rx_handler); iq->cur_desc = iq->desc; iq->cidx = 0; iq->gen = 1; diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index c2822e635f8961f1368c16b8def07d802b3330d9..54db79f4dcfe0909871784facf7ebebd3f6ebaa6 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -1553,8 +1553,8 @@ static void cxgb4vf_get_drvinfo(struct net_device *dev, { struct adapter *adapter = netdev2adap(dev); - strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)), + strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)), sizeof(drvinfo->bus_info)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%u.%u.%u.%u, TP %u.%u.%u.%u", diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c index 43b2ceb6aa3264a24a81210832cd5756c9457437..2d0cf76fb3c5a8cef6294de9053a467d5ecebeaa 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c @@ -2336,7 +2336,7 @@ int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq, if (ret) goto err; - netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64); + netif_napi_add(dev, &rspq->napi, napi_rx_handler); rspq->cur_desc = rspq->desc; rspq->cidx = 0; rspq->gen = 1; diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c index ddfe9208529a5ebebbc168a7aa01919acf15c947..c2e7037c7ba1c40bd41e9d6a8d5f252cee761485 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c @@ -1063,14 +1063,13 @@ static void chtls_pass_accept_rpl(struct sk_buff *skb, opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp)); rpl5->opt0 = cpu_to_be64(opt0); rpl5->opt2 = cpu_to_be32(opt2); - rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1); + rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1); set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure); cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); } -static void inet_inherit_port(struct inet_hashinfo *hash_info, - struct sock *lsk, struct sock *newsk) +static void inet_inherit_port(struct sock *lsk, struct sock *newsk) { local_bh_disable(); __inet_inherit_port(lsk, newsk); @@ -1240,7 +1239,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk, ipv4.sysctl_tcp_window_scaling), tp->window_clamp); neigh_release(n); - inet_inherit_port(&tcp_hashinfo, lsk, newsk); + inet_inherit_port(lsk, newsk); csk_set_flag(csk, CSK_CONN_INLINE); bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */ @@ -1467,7 +1466,7 @@ static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt) tp->write_seq = snd_isn; tp->snd_nxt = snd_isn; tp->snd_una = snd_isn; - inet_sk(sk)->inet_id = prandom_u32(); + inet_sk(sk)->inet_id = get_random_u16(); assign_rxopt(sk, opt); if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10)) diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c index 539992dad8ba3444dc4c68720d7c74882fdf00f4..a4256087ac82867ae00cafdd20f70ccd990aa838 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c @@ -919,8 +919,8 @@ static int csk_wait_memory(struct chtls_dev *cdev, current_timeo = *timeo_p; noblock = (*timeo_p ? false : true); if (csk_mem_free(cdev, sk)) { - current_timeo = (prandom_u32() % (HZ / 5)) + 2; - vm_wait = (prandom_u32() % (HZ / 5)) + 2; + current_timeo = prandom_u32_max(HZ / 5) + 2; + vm_wait = prandom_u32_max(HZ / 5) + 2; } add_wait_queue(sk_sleep(sk), &wait); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c index 9098b3eed4daf252a90aadc28c89a774a951b639..1e55b12fee5168fefb091c39085d5d954d5eefb8 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c @@ -193,7 +193,7 @@ static void chtls_register_dev(struct chtls_dev *cdev) { struct tls_toe_device *tlsdev = &cdev->tlsdev; - strlcpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX); + strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX); strlcat(tlsdev->name, cdev->lldi->ports[0]->name, TLS_TOE_DEVICE_NAME_MAX); tlsdev->feature = chtls_inline_feature; diff --git a/drivers/net/ethernet/cirrus/ep93xx_eth.c b/drivers/net/ethernet/cirrus/ep93xx_eth.c index 21ba6e893072cdae06ff412f01f043bb0c8cf215..8627ab19d47034c07a63649b17c0d148d961ff6e 100644 --- a/drivers/net/ethernet/cirrus/ep93xx_eth.c +++ b/drivers/net/ethernet/cirrus/ep93xx_eth.c @@ -689,7 +689,7 @@ static int ep93xx_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) static void ep93xx_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); } static int ep93xx_get_link_ksettings(struct net_device *dev, @@ -812,7 +812,7 @@ static int ep93xx_eth_probe(struct platform_device *pdev) ep = netdev_priv(dev); ep->dev = dev; SET_NETDEV_DEV(dev, &pdev->dev); - netif_napi_add(dev, &ep->napi, ep93xx_poll, 64); + netif_napi_add(dev, &ep->napi, ep93xx_poll); platform_set_drvdata(pdev, dev); diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index 60d8c0fbc037fed45b4c6e22621b98e7e359bc9d..08b7cc0a1809da329a67f91d544c562aca17ba76 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -131,10 +131,10 @@ static void enic_get_drvinfo(struct net_device *netdev, if (err == -ENOMEM) return; - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, fw_info->fw_version, + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->fw_version, fw_info->fw_version, sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(enic->pdev), + strscpy(drvinfo->bus_info, pci_name(enic->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 372fb7b3a2825a50fe468e2699a0225e2945c773..29500d32e36265de97b704f8a6306dc14ede6e5d 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -2633,16 +2633,17 @@ static int enic_dev_init(struct enic *enic) switch (vnic_dev_get_intr_mode(enic->vdev)) { default: - netif_napi_add(netdev, &enic->napi[0], enic_poll, 64); + netif_napi_add(netdev, &enic->napi[0], enic_poll); break; case VNIC_DEV_INTR_MODE_MSIX: for (i = 0; i < enic->rq_count; i++) { netif_napi_add(netdev, &enic->napi[i], - enic_poll_msix_rq, NAPI_POLL_WEIGHT); + enic_poll_msix_rq); } for (i = 0; i < enic->wq_count; i++) - netif_napi_add(netdev, &enic->napi[enic_cq_wq(enic, i)], - enic_poll_msix_wq, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, + &enic->napi[enic_cq_wq(enic, i)], + enic_poll_msix_wq); break; } diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 9e6de2f968fa39332ffa74b2c19568b933651010..fdf10318758b4ffee8920d1dc52fedcebfc9bf2f 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -1919,7 +1919,7 @@ static void gmac_get_stats64(struct net_device *netdev, /* Racing with RX NAPI */ do { - start = u64_stats_fetch_begin(&port->rx_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); stats->rx_packets = port->stats.rx_packets; stats->rx_bytes = port->stats.rx_bytes; @@ -1931,11 +1931,11 @@ static void gmac_get_stats64(struct net_device *netdev, stats->rx_crc_errors = port->stats.rx_crc_errors; stats->rx_frame_errors = port->stats.rx_frame_errors; - } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); /* Racing with MIB and TX completion interrupts */ do { - start = u64_stats_fetch_begin(&port->ir_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); stats->tx_errors = port->stats.tx_errors; stats->tx_packets = port->stats.tx_packets; @@ -1945,15 +1945,15 @@ static void gmac_get_stats64(struct net_device *netdev, stats->rx_missed_errors = port->stats.rx_missed_errors; stats->rx_fifo_errors = port->stats.rx_fifo_errors; - } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); /* Racing with hard_start_xmit */ do { - start = u64_stats_fetch_begin(&port->tx_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); stats->tx_dropped = port->stats.tx_dropped; - } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); stats->rx_dropped += stats->rx_missed_errors; } @@ -2031,18 +2031,18 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, /* Racing with MIB interrupt */ do { p = values; - start = u64_stats_fetch_begin(&port->ir_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->ir_stats_syncp); for (i = 0; i < RX_STATS_NUM; i++) *p++ = port->hw_stats[i]; - } while (u64_stats_fetch_retry(&port->ir_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->ir_stats_syncp, start)); values = p; /* Racing with RX NAPI */ do { p = values; - start = u64_stats_fetch_begin(&port->rx_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->rx_stats_syncp); for (i = 0; i < RX_STATUS_NUM; i++) *p++ = port->rx_stats[i]; @@ -2050,13 +2050,13 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, *p++ = port->rx_csum_stats[i]; *p++ = port->rx_napi_exits; - } while (u64_stats_fetch_retry(&port->rx_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->rx_stats_syncp, start)); values = p; /* Racing with TX start_xmit */ do { p = values; - start = u64_stats_fetch_begin(&port->tx_stats_syncp); + start = u64_stats_fetch_begin_irq(&port->tx_stats_syncp); for (i = 0; i < TX_MAX_FRAGS; i++) { *values++ = port->tx_frag_stats[i]; @@ -2065,7 +2065,7 @@ static void gmac_get_ethtool_stats(struct net_device *netdev, *values++ = port->tx_frags_linearized; *values++ = port->tx_hw_csummed; - } while (u64_stats_fetch_retry(&port->tx_stats_syncp, start)); + } while (u64_stats_fetch_retry_irq(&port->tx_stats_syncp, start)); } static int gmac_get_ksettings(struct net_device *netdev, @@ -2471,7 +2471,7 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev) netdev->max_mtu = 10236 - VLAN_ETH_HLEN; port->freeq_refill = 0; - netif_napi_add(netdev, &port->napi, gmac_napi_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &port->napi, gmac_napi_poll); ret = of_get_mac_address(np, mac); if (!ret) { diff --git a/drivers/net/ethernet/davicom/dm9000.c b/drivers/net/ethernet/davicom/dm9000.c index 0985ab216566b10cea9275ebf0bb45c485d49f0a..b21e56de61671a90ef3906a03f98ff7c879a988d 100644 --- a/drivers/net/ethernet/davicom/dm9000.c +++ b/drivers/net/ethernet/davicom/dm9000.c @@ -28,8 +28,7 @@ #include #include #include -#include -#include +#include #include #include @@ -540,8 +539,8 @@ static void dm9000_get_drvinfo(struct net_device *dev, { struct board_info *dm = to_dm9000_board(dev); - strlcpy(info->driver, CARDNAME, sizeof(info->driver)); - strlcpy(info->bus_info, to_platform_device(dm->dev)->name, + strscpy(info->driver, CARDNAME, sizeof(info->driver)); + strscpy(info->bus_info, to_platform_device(dm->dev)->name, sizeof(info->bus_info)); } @@ -1012,7 +1011,7 @@ static void dm9000_send_packet(struct net_device *dev, * Hardware start transmission. * Send a packet to media from the upper layer. */ -static int +static netdev_tx_t dm9000_start_xmit(struct sk_buff *skb, struct net_device *dev) { unsigned long flags; @@ -1421,8 +1420,7 @@ dm9000_probe(struct platform_device *pdev) int iosize; int i; u32 id_val; - int reset_gpios; - enum of_gpio_flags flags; + struct gpio_desc *reset_gpio; struct regulator *power; bool inv_mac_addr = false; u8 addr[ETH_ALEN]; @@ -1442,20 +1440,24 @@ dm9000_probe(struct platform_device *pdev) dev_dbg(dev, "regulator enabled\n"); } - reset_gpios = of_get_named_gpio_flags(dev->of_node, "reset-gpios", 0, - &flags); - if (gpio_is_valid(reset_gpios)) { - ret = devm_gpio_request_one(dev, reset_gpios, flags, - "dm9000_reset"); + reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + ret = PTR_ERR_OR_ZERO(reset_gpio); + if (ret) { + dev_err(dev, "failed to request reset gpio: %d\n", ret); + goto out_regulator_disable; + } + + if (reset_gpio) { + ret = gpiod_set_consumer_name(reset_gpio, "dm9000_reset"); if (ret) { - dev_err(dev, "failed to request reset gpio %d: %d\n", - reset_gpios, ret); + dev_err(dev, "failed to set reset gpio name: %d\n", + ret); goto out_regulator_disable; } /* According to manual PWRST# Low Period Min 1ms */ msleep(2); - gpio_set_value(reset_gpios, 1); + gpiod_set_value_cansleep(reset_gpio, 0); /* Needs 3ms to read eeprom when PWRST is deasserted */ msleep(4); } diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c index d51b3d24a0c87b3e2d4afc00eea28cad2b852def..cd3dc4b89518f0849819da8c8b233d1638eb7e08 100644 --- a/drivers/net/ethernet/dec/tulip/de2104x.c +++ b/drivers/net/ethernet/dec/tulip/de2104x.c @@ -1606,8 +1606,8 @@ static void de_get_drvinfo (struct net_device *dev,struct ethtool_drvinfo *info) { struct de_private *de = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(de->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(de->pdev), sizeof(info->bus_info)); } static int de_get_regs_len(struct net_device *dev) diff --git a/drivers/net/ethernet/dec/tulip/dmfe.c b/drivers/net/ethernet/dec/tulip/dmfe.c index 83f1727d1423be483497a97565c34c3adcc6e755..3188ba7b450f47c017648d3a999902f1f791f7fa 100644 --- a/drivers/net/ethernet/dec/tulip/dmfe.c +++ b/drivers/net/ethernet/dec/tulip/dmfe.c @@ -1074,8 +1074,8 @@ static void dmfe_ethtool_get_drvinfo(struct net_device *dev, { struct dmfe_board_info *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); } static int dmfe_ethtool_set_wol(struct net_device *dev, diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c index b8e46c4849ef42dd378d8fb96e2121e3380cdad2..ecfad43df45a7c9872386fe70b009b786b3571bd 100644 --- a/drivers/net/ethernet/dec/tulip/tulip_core.c +++ b/drivers/net/ethernet/dec/tulip/tulip_core.c @@ -858,8 +858,8 @@ static struct net_device_stats *tulip_get_stats(struct net_device *dev) static void tulip_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct tulip_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/dec/tulip/uli526x.c b/drivers/net/ethernet/dec/tulip/uli526x.c index 77d9058431e3a6b0100e23111304a354f8f11f93..ff080ab0f116535ee6261386bdb106475ae85947 100644 --- a/drivers/net/ethernet/dec/tulip/uli526x.c +++ b/drivers/net/ethernet/dec/tulip/uli526x.c @@ -971,8 +971,8 @@ static void netdev_get_drvinfo(struct net_device *dev, { struct uli526x_board_info *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c index 1db19463fd46d022fbed68f654a605e424e46b46..37fba39c005656b8122854aa44c51564b713b039 100644 --- a/drivers/net/ethernet/dec/tulip/winbond-840.c +++ b/drivers/net/ethernet/dec/tulip/winbond-840.c @@ -1374,8 +1374,8 @@ static void netdev_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo * { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/dlink/dl2k.c b/drivers/net/ethernet/dlink/dl2k.c index a301f7e6a4405c4e2ec8a3b819aae19fb7deb53e..2c67a857a42ffa0a9d2cd3906a47f70735ea56ab 100644 --- a/drivers/net/ethernet/dlink/dl2k.c +++ b/drivers/net/ethernet/dlink/dl2k.c @@ -1235,8 +1235,8 @@ static void rio_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, "dl2k", sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); + strscpy(info->driver, "dl2k", sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); } static int rio_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c index 8dd7bf9014ec85f97ba94e4b9bb8073a0fc2e363..43def191f26f23b52f822ae3b42ccc00efa10e04 100644 --- a/drivers/net/ethernet/dlink/sundance.c +++ b/drivers/net/ethernet/dlink/sundance.c @@ -1644,8 +1644,8 @@ static int check_if_running(struct net_device *dev) static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/dnet.c b/drivers/net/ethernet/dnet.c index 92462ed87bc4382a27f6a12c974dbdbeea7b3713..08184f20f5104243babaf7d5c0416ce31de2d26a 100644 --- a/drivers/net/ethernet/dnet.c +++ b/drivers/net/ethernet/dnet.c @@ -725,8 +725,8 @@ static struct net_device_stats *dnet_get_stats(struct net_device *dev) static void dnet_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, "0", sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, "0", sizeof(info->bus_info)); } static const struct ethtool_ops dnet_ethtool_ops = { @@ -788,7 +788,7 @@ static int dnet_probe(struct platform_device *pdev) } dev->netdev_ops = &dnet_netdev_ops; - netif_napi_add(dev, &bp->napi, dnet_poll, 64); + netif_napi_add(dev, &bp->napi, dnet_poll); dev->ethtool_ops = &dnet_ethtool_ops; dev->base_addr = (unsigned long)bp->regs; diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index b4f5e57d0285cb516a26e6d1b7dd54bd9fee0ae7..08ec84cd21c04a9f9ba7c474fb0956a0c5c5c5a1 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -1878,9 +1878,9 @@ int be_cmd_get_fw_ver(struct be_adapter *adapter) if (!status) { struct be_cmd_resp_get_fw_version *resp = embedded_payload(wrb); - strlcpy(adapter->fw_ver, resp->firmware_version_string, + strscpy(adapter->fw_ver, resp->firmware_version_string, sizeof(adapter->fw_ver)); - strlcpy(adapter->fw_on_flash, resp->fw_on_flash_version_string, + strscpy(adapter->fw_on_flash, resp->fw_on_flash_version_string, sizeof(adapter->fw_on_flash)); } err: @@ -2373,7 +2373,7 @@ static int lancer_cmd_write_object(struct be_adapter *adapter, be_dws_cpu_to_le(ctxt, sizeof(req->context)); req->write_offset = cpu_to_le32(data_offset); - strlcpy(req->object_name, obj_name, sizeof(req->object_name)); + strscpy(req->object_name, obj_name, sizeof(req->object_name)); req->descriptor_count = cpu_to_le32(1); req->buf_len = cpu_to_le32(data_size); req->addr_low = cpu_to_le32((cmd->dma + @@ -2442,9 +2442,9 @@ int be_cmd_query_sfp_info(struct be_adapter *adapter) status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0, 0, PAGE_DATA_LEN, page_data); if (!status) { - strlcpy(adapter->phy.vendor_name, page_data + + strscpy(adapter->phy.vendor_name, page_data + SFP_VENDOR_NAME_OFFSET, SFP_VENDOR_NAME_LEN - 1); - strlcpy(adapter->phy.vendor_pn, + strscpy(adapter->phy.vendor_pn, page_data + SFP_VENDOR_PN_OFFSET, SFP_VENDOR_NAME_LEN - 1); } @@ -2473,7 +2473,7 @@ static int lancer_cmd_delete_object(struct be_adapter *adapter, OPCODE_COMMON_DELETE_OBJECT, sizeof(*req), wrb, NULL); - strlcpy(req->object_name, obj_name, sizeof(req->object_name)); + strscpy(req->object_name, obj_name, sizeof(req->object_name)); status = be_mcc_notify_wait(adapter); err: diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index bd0df189d87192f704bb12e63c66618dfa5e800a..77edc3d9b50571f413c1d46cd97b0b9e4303d2a9 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -220,15 +220,15 @@ static void be_get_drvinfo(struct net_device *netdev, { struct be_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); if (!memcmp(adapter->fw_ver, adapter->fw_on_flash, FW_VER_LEN)) - strlcpy(drvinfo->fw_version, adapter->fw_ver, + strscpy(drvinfo->fw_version, adapter->fw_ver, sizeof(drvinfo->fw_version)); else snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%s [%s]", adapter->fw_ver, adapter->fw_on_flash); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index 414362febbb9d67509fc0a479290b59e40937fc3..a92a747615466a522b87c118e815a94f687f7ed6 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c @@ -2982,8 +2982,7 @@ static int be_evt_queues_create(struct be_adapter *adapter) return -ENOMEM; cpumask_set_cpu(cpumask_local_spread(i, numa_node), eqo->affinity_mask); - netif_napi_add(adapter->netdev, &eqo->napi, be_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(adapter->netdev, &eqo->napi, be_poll); } return 0; } diff --git a/drivers/net/ethernet/engleder/Kconfig b/drivers/net/ethernet/engleder/Kconfig index f4e2b1102d8f5ab49604ead4a1a2706508070f18..3df6bf476ae7e2563371dd7d40ab733bbd1a0aa1 100644 --- a/drivers/net/ethernet/engleder/Kconfig +++ b/drivers/net/ethernet/engleder/Kconfig @@ -21,6 +21,7 @@ config TSNEP depends on HAS_IOMEM && HAS_DMA depends on PTP_1588_CLOCK_OPTIONAL select PHYLIB + select PAGE_POOL help Support for the Engleder TSN endpoint Ethernet MAC IP Core. diff --git a/drivers/net/ethernet/engleder/Makefile b/drivers/net/ethernet/engleder/Makefile index cce2191cb889c49b70457616c4a9f96aaefffc91..b6e3b16623deaffed3bbde51f5a57e84f13f3819 100644 --- a/drivers/net/ethernet/engleder/Makefile +++ b/drivers/net/ethernet/engleder/Makefile @@ -6,5 +6,5 @@ obj-$(CONFIG_TSNEP) += tsnep.o tsnep-objs := tsnep_main.o tsnep_ethtool.o tsnep_ptp.o tsnep_tc.o \ - $(tsnep-y) + tsnep_rxnfc.o $(tsnep-y) tsnep-$(CONFIG_TSNEP_SELFTESTS) += tsnep_selftests.o diff --git a/drivers/net/ethernet/engleder/tsnep.h b/drivers/net/ethernet/engleder/tsnep.h index 23bbece6b7dee232ff006c501b7cc90e748c7826..09a723b827c776877173f01c9a143f6b66f6b0d8 100644 --- a/drivers/net/ethernet/engleder/tsnep.h +++ b/drivers/net/ethernet/engleder/tsnep.h @@ -21,8 +21,6 @@ #define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE) #define TSNEP_RING_PAGE_COUNT (TSNEP_RING_SIZE / TSNEP_RING_ENTRIES_PER_PAGE) -#define TSNEP_QUEUES 1 - struct tsnep_gcl { void __iomem *addr; @@ -39,6 +37,24 @@ struct tsnep_gcl { bool change; }; +enum tsnep_rxnfc_filter_type { + TSNEP_RXNFC_ETHER_TYPE, +}; + +struct tsnep_rxnfc_filter { + enum tsnep_rxnfc_filter_type type; + union { + u16 ether_type; + }; +}; + +struct tsnep_rxnfc_rule { + struct list_head list; + struct tsnep_rxnfc_filter filter; + int queue_index; + int location; +}; + struct tsnep_tx_entry { struct tsnep_tx_desc *desc; struct tsnep_tx_desc_wb *desc_wb; @@ -55,6 +71,7 @@ struct tsnep_tx_entry { struct tsnep_tx { struct tsnep_adapter *adapter; void __iomem *addr; + int queue_index; void *page[TSNEP_RING_PAGE_COUNT]; dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT]; @@ -79,14 +96,15 @@ struct tsnep_rx_entry { u32 properties; - struct sk_buff *skb; + struct page *page; size_t len; - DEFINE_DMA_UNMAP_ADDR(dma); + dma_addr_t dma; }; struct tsnep_rx { struct tsnep_adapter *adapter; void __iomem *addr; + int queue_index; void *page[TSNEP_RING_PAGE_COUNT]; dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT]; @@ -95,6 +113,7 @@ struct tsnep_rx { int read; u32 owner_counter; int increment_owner_counter; + struct page_pool *page_pool; u32 packets; u32 bytes; @@ -104,12 +123,14 @@ struct tsnep_rx { struct tsnep_queue { struct tsnep_adapter *adapter; + char name[IFNAMSIZ + 9]; struct tsnep_tx *tx; struct tsnep_rx *rx; struct napi_struct napi; + int irq; u32 irq_mask; }; @@ -125,7 +146,6 @@ struct tsnep_adapter { struct platform_device *pdev; struct device *dmadev; void __iomem *addr; - int irq; bool gate_control; /* gate control lock */ @@ -140,6 +160,12 @@ struct tsnep_adapter { /* ptp clock lock */ spinlock_t ptp_lock; + /* RX flow classification rules lock */ + struct mutex rxnfc_lock; + struct list_head rxnfc_rules; + int rxnfc_count; + int rxnfc_max; + int num_tx_queues; struct tsnep_tx tx[TSNEP_MAX_QUEUES]; int num_rx_queues; @@ -160,6 +186,18 @@ void tsnep_tc_cleanup(struct tsnep_adapter *adapter); int tsnep_tc_setup(struct net_device *netdev, enum tc_setup_type type, void *type_data); +int tsnep_rxnfc_init(struct tsnep_adapter *adapter); +void tsnep_rxnfc_cleanup(struct tsnep_adapter *adapter); +int tsnep_rxnfc_get_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd); +int tsnep_rxnfc_get_all(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd, + u32 *rule_locs); +int tsnep_rxnfc_add_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd); +int tsnep_rxnfc_del_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd); + #if IS_ENABLED(CONFIG_TSNEP_SELFTESTS) int tsnep_ethtool_get_test_count(void); void tsnep_ethtool_get_test_strings(u8 *data); diff --git a/drivers/net/ethernet/engleder/tsnep_ethtool.c b/drivers/net/ethernet/engleder/tsnep_ethtool.c index e6760dc68dddf0cc51c5c3bafc34a95614d54d6e..a713a126b2276ca6e9b894cfe98db375ca7d2987 100644 --- a/drivers/net/ethernet/engleder/tsnep_ethtool.c +++ b/drivers/net/ethernet/engleder/tsnep_ethtool.c @@ -250,6 +250,44 @@ static int tsnep_ethtool_get_sset_count(struct net_device *netdev, int sset) } } +static int tsnep_ethtool_get_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) +{ + struct tsnep_adapter *adapter = netdev_priv(dev); + + switch (cmd->cmd) { + case ETHTOOL_GRXRINGS: + cmd->data = adapter->num_rx_queues; + return 0; + case ETHTOOL_GRXCLSRLCNT: + cmd->rule_cnt = adapter->rxnfc_count; + cmd->data = adapter->rxnfc_max; + cmd->data |= RX_CLS_LOC_SPECIAL; + return 0; + case ETHTOOL_GRXCLSRULE: + return tsnep_rxnfc_get_rule(adapter, cmd); + case ETHTOOL_GRXCLSRLALL: + return tsnep_rxnfc_get_all(adapter, cmd, rule_locs); + default: + return -EOPNOTSUPP; + } +} + +static int tsnep_ethtool_set_rxnfc(struct net_device *dev, + struct ethtool_rxnfc *cmd) +{ + struct tsnep_adapter *adapter = netdev_priv(dev); + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + return tsnep_rxnfc_add_rule(adapter, cmd); + case ETHTOOL_SRXCLSRLDEL: + return tsnep_rxnfc_del_rule(adapter, cmd); + default: + return -EOPNOTSUPP; + } +} + static int tsnep_ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info) { @@ -287,6 +325,8 @@ const struct ethtool_ops tsnep_ethtool_ops = { .get_strings = tsnep_ethtool_get_strings, .get_ethtool_stats = tsnep_ethtool_get_ethtool_stats, .get_sset_count = tsnep_ethtool_get_sset_count, + .get_rxnfc = tsnep_ethtool_get_rxnfc, + .set_rxnfc = tsnep_ethtool_set_rxnfc, .get_ts_info = tsnep_ethtool_get_ts_info, .get_link_ksettings = phy_ethtool_get_link_ksettings, .set_link_ksettings = phy_ethtool_set_link_ksettings, diff --git a/drivers/net/ethernet/engleder/tsnep_hw.h b/drivers/net/ethernet/engleder/tsnep_hw.h index 916ceac3ada233c889a58829fb2b93ca60ac0a42..315dada75323fd885f7f1a6dfd2c274560c3a3f4 100644 --- a/drivers/net/ethernet/engleder/tsnep_hw.h +++ b/drivers/net/ethernet/engleder/tsnep_hw.h @@ -34,6 +34,7 @@ #define ECM_INT_LINK 0x00000020 #define ECM_INT_TX_0 0x00000100 #define ECM_INT_RX_0 0x00000200 +#define ECM_INT_TXRX_SHIFT 2 #define ECM_INT_ALL 0x7FFFFFFF #define ECM_INT_DISABLE 0x80000000 @@ -92,8 +93,7 @@ /* tsnep register */ #define TSNEP_INFO 0x0100 -#define TSNEP_INFO_RX_ASSIGN 0x00010000 -#define TSNEP_INFO_TX_TIME 0x00020000 +#define TSNEP_INFO_TX_TIME 0x00010000 #define TSNEP_CONTROL 0x0108 #define TSNEP_CONTROL_TX_RESET 0x00000001 #define TSNEP_CONTROL_TX_ENABLE 0x00000002 @@ -122,10 +122,6 @@ #define TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL 0x0191 #define TSNEP_RX_STATISTIC_FIFO_OVERFLOW 0x0192 #define TSNEP_RX_STATISTIC_INVALID_FRAME 0x0193 -#define TSNEP_RX_ASSIGN 0x01A0 -#define TSNEP_RX_ASSIGN_ETHER_TYPE_ACTIVE 0x00000001 -#define TSNEP_RX_ASSIGN_ETHER_TYPE_MASK 0xFFFF0000 -#define TSNEP_RX_ASSIGN_ETHER_TYPE_SHIFT 16 #define TSNEP_MAC_ADDRESS_LOW 0x0800 #define TSNEP_MAC_ADDRESS_HIGH 0x0804 #define TSNEP_RX_FILTER 0x0806 @@ -152,6 +148,14 @@ #define TSNEP_GCL_A 0x2000 #define TSNEP_GCL_B 0x2800 #define TSNEP_GCL_SIZE SZ_2K +#define TSNEP_RX_ASSIGN 0x0840 +#define TSNEP_RX_ASSIGN_ACTIVE 0x00000001 +#define TSNEP_RX_ASSIGN_QUEUE_MASK 0x00000006 +#define TSNEP_RX_ASSIGN_QUEUE_SHIFT 1 +#define TSNEP_RX_ASSIGN_OFFSET 1 +#define TSNEP_RX_ASSIGN_ETHER_TYPE 0x0880 +#define TSNEP_RX_ASSIGN_ETHER_TYPE_OFFSET 2 +#define TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT 2 /* tsnep gate control list operation */ struct tsnep_gcl_operation { diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c index a5f7152a17160a7aad88fe4b8256101119c9dd44..48fb391951ddebae381d7efc7b6db9b35eb7449b 100644 --- a/drivers/net/ethernet/engleder/tsnep_main.c +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -27,10 +27,10 @@ #include #include -#define RX_SKB_LENGTH (round_up(TSNEP_RX_INLINE_METADATA_SIZE + ETH_HLEN + \ - TSNEP_MAX_FRAME_SIZE + ETH_FCS_LEN, 4)) -#define RX_SKB_RESERVE ((16 - TSNEP_RX_INLINE_METADATA_SIZE) + NET_IP_ALIGN) -#define RX_SKB_ALLOC_LENGTH (RX_SKB_RESERVE + RX_SKB_LENGTH) +#define TSNEP_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define TSNEP_HEADROOM ALIGN(TSNEP_SKB_PAD, 4) +#define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) @@ -60,22 +60,29 @@ static irqreturn_t tsnep_irq(int irq, void *arg) iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); /* handle link interrupt */ - if ((active & ECM_INT_LINK) != 0) { - if (adapter->netdev->phydev) - phy_mac_interrupt(adapter->netdev->phydev); - } + if ((active & ECM_INT_LINK) != 0) + phy_mac_interrupt(adapter->netdev->phydev); /* handle TX/RX queue 0 interrupt */ if ((active & adapter->queue[0].irq_mask) != 0) { - if (adapter->netdev) { - tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); - napi_schedule(&adapter->queue[0].napi); - } + tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); + napi_schedule(&adapter->queue[0].napi); } return IRQ_HANDLED; } +static irqreturn_t tsnep_irq_txrx(int irq, void *arg) +{ + struct tsnep_queue *queue = arg; + + /* handle TX/RX queue interrupt */ + tsnep_disable_irq(queue->adapter, queue->irq_mask); + napi_schedule(&queue->napi); + + return IRQ_HANDLED; +} + static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) { struct tsnep_adapter *adapter = bus->priv; @@ -124,30 +131,51 @@ static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, return 0; } +static void tsnep_set_link_mode(struct tsnep_adapter *adapter) +{ + u32 mode; + + switch (adapter->phydev->speed) { + case SPEED_100: + mode = ECM_LINK_MODE_100; + break; + case SPEED_1000: + mode = ECM_LINK_MODE_1000; + break; + default: + mode = ECM_LINK_MODE_OFF; + break; + } + iowrite32(mode, adapter->addr + ECM_STATUS); +} + static void tsnep_phy_link_status_change(struct net_device *netdev) { struct tsnep_adapter *adapter = netdev_priv(netdev); struct phy_device *phydev = netdev->phydev; - u32 mode; - if (phydev->link) { - switch (phydev->speed) { - case SPEED_100: - mode = ECM_LINK_MODE_100; - break; - case SPEED_1000: - mode = ECM_LINK_MODE_1000; - break; - default: - mode = ECM_LINK_MODE_OFF; - break; - } - iowrite32(mode, adapter->addr + ECM_STATUS); - } + if (phydev->link) + tsnep_set_link_mode(adapter); phy_print_status(netdev->phydev); } +static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable) +{ + int retval; + + retval = phy_loopback(adapter->phydev, enable); + + /* PHY link state change is not signaled if loopback is enabled, it + * would delay a working loopback anyway, let's ensure that loopback + * is working immediately by setting link mode directly + */ + if (!retval && enable) + tsnep_set_link_mode(adapter); + + return retval; +} + static int tsnep_phy_open(struct tsnep_adapter *adapter) { struct phy_device *phydev; @@ -241,14 +269,14 @@ alloc_failed: return retval; } -static void tsnep_tx_activate(struct tsnep_tx *tx, int index, bool last) +static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length, + bool last) { struct tsnep_tx_entry *entry = &tx->entry[index]; entry->properties = 0; if (entry->skb) { - entry->properties = - skb_pagelen(entry->skb) & TSNEP_DESC_LENGTH_MASK; + entry->properties = length & TSNEP_DESC_LENGTH_MASK; entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; if (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; @@ -313,6 +341,7 @@ static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) struct tsnep_tx_entry *entry; unsigned int len; dma_addr_t dma; + int map_len = 0; int i; for (i = 0; i < count; i++) { @@ -335,15 +364,18 @@ static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) dma_unmap_addr_set(entry, dma, dma); entry->desc->tx = __cpu_to_le64(dma); + + map_len += len; } - return 0; + return map_len; } -static void tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) +static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) { struct device *dmadev = tx->adapter->dmadev; struct tsnep_tx_entry *entry; + int map_len = 0; int i; for (i = 0; i < count; i++) { @@ -360,9 +392,12 @@ static void tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count) dma_unmap_addr(entry, dma), dma_unmap_len(entry, len), DMA_TO_DEVICE); + map_len += entry->len; entry->len = 0; } } + + return map_len; } static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, @@ -371,6 +406,7 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, unsigned long flags; int count = 1; struct tsnep_tx_entry *entry; + int length; int i; int retval; @@ -394,7 +430,7 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, entry->skb = skb; retval = tsnep_tx_map(skb, tx, count); - if (retval != 0) { + if (retval < 0) { tsnep_tx_unmap(tx, tx->write, count); dev_kfree_skb_any(entry->skb); entry->skb = NULL; @@ -407,12 +443,13 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, return NETDEV_TX_OK; } + length = retval; if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; for (i = 0; i < count; i++) - tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, + tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length, i == (count - 1)); tx->write = (tx->write + count) % TSNEP_RING_SIZE; @@ -428,9 +465,6 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, netif_stop_queue(tx->adapter->netdev); } - tx->packets++; - tx->bytes += skb_pagelen(entry->skb) + ETH_FCS_LEN; - spin_unlock_irqrestore(&tx->lock, flags); return NETDEV_TX_OK; @@ -442,6 +476,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) int budget = 128; struct tsnep_tx_entry *entry; int count; + int length; spin_lock_irqsave(&tx->lock, flags); @@ -464,7 +499,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) if (skb_shinfo(entry->skb)->nr_frags > 0) count += skb_shinfo(entry->skb)->nr_frags; - tsnep_tx_unmap(tx, tx->read, count); + length = tsnep_tx_unmap(tx, tx->read, count); if ((skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && (__le32_to_cpu(entry->desc_wb->properties) & @@ -491,6 +526,9 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) tx->read = (tx->read + count) % TSNEP_RING_SIZE; + tx->packets++; + tx->bytes += length + ETH_FCS_LEN; + budget--; } while (likely(budget)); @@ -505,7 +543,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) } static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr, - struct tsnep_tx *tx) + int queue_index, struct tsnep_tx *tx) { dma_addr_t dma; int retval; @@ -513,6 +551,7 @@ static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr, memset(tx, 0, sizeof(*tx)); tx->adapter = adapter; tx->addr = addr; + tx->queue_index = queue_index; retval = tsnep_tx_ring_init(tx); if (retval) @@ -548,14 +587,15 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) for (i = 0; i < TSNEP_RING_SIZE; i++) { entry = &rx->entry[i]; - if (dma_unmap_addr(entry, dma)) - dma_unmap_single(dmadev, dma_unmap_addr(entry, dma), - dma_unmap_len(entry, len), - DMA_FROM_DEVICE); - if (entry->skb) - dev_kfree_skb(entry->skb); + if (entry->page) + page_pool_put_full_page(rx->page_pool, entry->page, + false); + entry->page = NULL; } + if (rx->page_pool) + page_pool_destroy(rx->page_pool); + memset(rx->entry, 0, sizeof(rx->entry)); for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { @@ -568,31 +608,19 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) } } -static int tsnep_rx_alloc_and_map_skb(struct tsnep_rx *rx, - struct tsnep_rx_entry *entry) +static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, + struct tsnep_rx_entry *entry) { - struct device *dmadev = rx->adapter->dmadev; - struct sk_buff *skb; - dma_addr_t dma; + struct page *page; - skb = __netdev_alloc_skb(rx->adapter->netdev, RX_SKB_ALLOC_LENGTH, - GFP_ATOMIC | GFP_DMA); - if (!skb) + page = page_pool_dev_alloc_pages(rx->page_pool); + if (unlikely(!page)) return -ENOMEM; - skb_reserve(skb, RX_SKB_RESERVE); - - dma = dma_map_single(dmadev, skb->data, RX_SKB_LENGTH, - DMA_FROM_DEVICE); - if (dma_mapping_error(dmadev, dma)) { - dev_kfree_skb(skb); - return -ENOMEM; - } - - entry->skb = skb; - entry->len = RX_SKB_LENGTH; - dma_unmap_addr_set(entry, dma, dma); - entry->desc->rx = __cpu_to_le64(dma); + entry->page = page; + entry->len = TSNEP_MAX_RX_BUF_SIZE; + entry->dma = page_pool_get_dma_addr(entry->page); + entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD); return 0; } @@ -601,6 +629,7 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx) { struct device *dmadev = rx->adapter->dmadev; struct tsnep_rx_entry *entry; + struct page_pool_params pp_params = { 0 }; struct tsnep_rx_entry *next_entry; int i, j; int retval; @@ -622,12 +651,28 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx) entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; } } + + pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; + pp_params.order = 0; + pp_params.pool_size = TSNEP_RING_SIZE; + pp_params.nid = dev_to_node(dmadev); + pp_params.dev = dmadev; + pp_params.dma_dir = DMA_FROM_DEVICE; + pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE; + pp_params.offset = TSNEP_SKB_PAD; + rx->page_pool = page_pool_create(&pp_params); + if (IS_ERR(rx->page_pool)) { + retval = PTR_ERR(rx->page_pool); + rx->page_pool = NULL; + goto failed; + } + for (i = 0; i < TSNEP_RING_SIZE; i++) { entry = &rx->entry[i]; next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE]; entry->desc->next = __cpu_to_le64(next_entry->desc_dma); - retval = tsnep_rx_alloc_and_map_skb(rx, entry); + retval = tsnep_rx_alloc_buffer(rx, entry); if (retval) goto failed; } @@ -643,7 +688,7 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index) { struct tsnep_rx_entry *entry = &rx->entry[index]; - /* RX_SKB_LENGTH is a multiple of 4 */ + /* TSNEP_MAX_RX_BUF_SIZE is a multiple of 4 */ entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; if (index == rx->increment_owner_counter) { @@ -666,19 +711,52 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index) entry->desc->properties = __cpu_to_le32(entry->properties); } +static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page, + int length) +{ + struct sk_buff *skb; + + skb = napi_build_skb(page_address(page), PAGE_SIZE); + if (unlikely(!skb)) + return NULL; + + /* update pointers within the skb to store the data */ + skb_reserve(skb, TSNEP_SKB_PAD + TSNEP_RX_INLINE_METADATA_SIZE); + __skb_put(skb, length - TSNEP_RX_INLINE_METADATA_SIZE - ETH_FCS_LEN); + + if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) { + struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb); + struct tsnep_rx_inline *rx_inline = + (struct tsnep_rx_inline *)(page_address(page) + + TSNEP_SKB_PAD); + + skb_shinfo(skb)->tx_flags |= + SKBTX_HW_TSTAMP_NETDEV; + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->netdev_data = rx_inline; + } + + skb_record_rx_queue(skb, rx->queue_index); + skb->protocol = eth_type_trans(skb, rx->adapter->netdev); + + return skb; +} + static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, int budget) { struct device *dmadev = rx->adapter->dmadev; int done = 0; + enum dma_data_direction dma_dir; struct tsnep_rx_entry *entry; + struct page *page; struct sk_buff *skb; - size_t len; - dma_addr_t dma; int length; bool enable = false; int retval; + dma_dir = page_pool_get_dma_dir(rx->page_pool); + while (likely(done < budget)) { entry = &rx->entry[rx->read]; if ((__le32_to_cpu(entry->desc_wb->properties) & @@ -691,42 +769,34 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, */ dma_rmb(); - skb = entry->skb; - len = dma_unmap_len(entry, len); - dma = dma_unmap_addr(entry, dma); + prefetch(page_address(entry->page) + TSNEP_SKB_PAD); + length = __le32_to_cpu(entry->desc_wb->properties) & + TSNEP_DESC_LENGTH_MASK; + dma_sync_single_range_for_cpu(dmadev, entry->dma, TSNEP_SKB_PAD, + length, dma_dir); + page = entry->page; /* forward skb only if allocation is successful, otherwise - * skb is reused and frame dropped + * page is reused and frame dropped */ - retval = tsnep_rx_alloc_and_map_skb(rx, entry); + retval = tsnep_rx_alloc_buffer(rx, entry); if (!retval) { - dma_unmap_single(dmadev, dma, len, DMA_FROM_DEVICE); - - length = __le32_to_cpu(entry->desc_wb->properties) & - TSNEP_DESC_LENGTH_MASK; - skb_put(skb, length - ETH_FCS_LEN); - if (rx->adapter->hwtstamp_config.rx_filter == - HWTSTAMP_FILTER_ALL) { - struct skb_shared_hwtstamps *hwtstamps = - skb_hwtstamps(skb); - struct tsnep_rx_inline *rx_inline = - (struct tsnep_rx_inline *)skb->data; - - skb_shinfo(skb)->tx_flags |= - SKBTX_HW_TSTAMP_NETDEV; - memset(hwtstamps, 0, sizeof(*hwtstamps)); - hwtstamps->netdev_data = rx_inline; - } - skb_pull(skb, TSNEP_RX_INLINE_METADATA_SIZE); - skb->protocol = eth_type_trans(skb, - rx->adapter->netdev); + skb = tsnep_build_skb(rx, page, length); + if (skb) { + page_pool_release_page(rx->page_pool, page); - rx->packets++; - rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE; - if (skb->pkt_type == PACKET_MULTICAST) - rx->multicast++; + rx->packets++; + rx->bytes += length - + TSNEP_RX_INLINE_METADATA_SIZE; + if (skb->pkt_type == PACKET_MULTICAST) + rx->multicast++; - napi_gro_receive(napi, skb); + napi_gro_receive(napi, skb); + } else { + page_pool_recycle_direct(rx->page_pool, page); + + rx->dropped++; + } done++; } else { rx->dropped++; @@ -752,7 +822,7 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, } static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, - struct tsnep_rx *rx) + int queue_index, struct tsnep_rx *rx) { dma_addr_t dma; int i; @@ -761,6 +831,7 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, memset(rx, 0, sizeof(*rx)); rx->adapter = adapter; rx->addr = addr; + rx->queue_index = queue_index; retval = tsnep_rx_ring_init(rx); if (retval) @@ -821,6 +892,56 @@ static int tsnep_poll(struct napi_struct *napi, int budget) return min(done, budget - 1); } +static int tsnep_request_irq(struct tsnep_queue *queue, bool first) +{ + const char *name = netdev_name(queue->adapter->netdev); + irq_handler_t handler; + void *dev; + int retval; + + if (first) { + sprintf(queue->name, "%s-mac", name); + handler = tsnep_irq; + dev = queue->adapter; + } else { + if (queue->tx && queue->rx) + sprintf(queue->name, "%s-txrx-%d", name, + queue->rx->queue_index); + else if (queue->tx) + sprintf(queue->name, "%s-tx-%d", name, + queue->tx->queue_index); + else + sprintf(queue->name, "%s-rx-%d", name, + queue->rx->queue_index); + handler = tsnep_irq_txrx; + dev = queue; + } + + retval = request_irq(queue->irq, handler, 0, queue->name, dev); + if (retval) { + /* if name is empty, then interrupt won't be freed */ + memset(queue->name, 0, sizeof(queue->name)); + } + + return retval; +} + +static void tsnep_free_irq(struct tsnep_queue *queue, bool first) +{ + void *dev; + + if (!strlen(queue->name)) + return; + + if (first) + dev = queue->adapter; + else + dev = queue; + + free_irq(queue->irq, dev); + memset(queue->name, 0, sizeof(queue->name)); +} + static int tsnep_netdev_open(struct net_device *netdev) { struct tsnep_adapter *adapter = netdev_priv(netdev); @@ -830,15 +951,11 @@ static int tsnep_netdev_open(struct net_device *netdev) int rx_queue_index = 0; int retval; - retval = tsnep_phy_open(adapter); - if (retval) - return retval; - for (i = 0; i < adapter->num_queues; i++) { adapter->queue[i].adapter = adapter; if (adapter->queue[i].tx) { addr = adapter->addr + TSNEP_QUEUE(tx_queue_index); - retval = tsnep_tx_open(adapter, addr, + retval = tsnep_tx_open(adapter, addr, tx_queue_index, adapter->queue[i].tx); if (retval) goto failed; @@ -847,11 +964,20 @@ static int tsnep_netdev_open(struct net_device *netdev) if (adapter->queue[i].rx) { addr = adapter->addr + TSNEP_QUEUE(rx_queue_index); retval = tsnep_rx_open(adapter, addr, + rx_queue_index, adapter->queue[i].rx); if (retval) goto failed; rx_queue_index++; } + + retval = tsnep_request_irq(&adapter->queue[i], i == 0); + if (retval) { + netif_err(adapter, drv, adapter->netdev, + "can't get assigned irq %d.\n", + adapter->queue[i].irq); + goto failed; + } } retval = netif_set_real_num_tx_queues(adapter->netdev, @@ -863,9 +989,14 @@ static int tsnep_netdev_open(struct net_device *netdev) if (retval) goto failed; + tsnep_enable_irq(adapter, ECM_INT_LINK); + retval = tsnep_phy_open(adapter); + if (retval) + goto phy_failed; + for (i = 0; i < adapter->num_queues; i++) { netif_napi_add(adapter->netdev, &adapter->queue[i].napi, - tsnep_poll, 64); + tsnep_poll); napi_enable(&adapter->queue[i].napi); tsnep_enable_irq(adapter, adapter->queue[i].irq_mask); @@ -873,14 +1004,18 @@ static int tsnep_netdev_open(struct net_device *netdev) return 0; +phy_failed: + tsnep_disable_irq(adapter, ECM_INT_LINK); + tsnep_phy_close(adapter); failed: for (i = 0; i < adapter->num_queues; i++) { + tsnep_free_irq(&adapter->queue[i], i == 0); + if (adapter->queue[i].rx) tsnep_rx_close(adapter->queue[i].rx); if (adapter->queue[i].tx) tsnep_tx_close(adapter->queue[i].tx); } - tsnep_phy_close(adapter); return retval; } @@ -889,20 +1024,23 @@ static int tsnep_netdev_close(struct net_device *netdev) struct tsnep_adapter *adapter = netdev_priv(netdev); int i; + tsnep_disable_irq(adapter, ECM_INT_LINK); + tsnep_phy_close(adapter); + for (i = 0; i < adapter->num_queues; i++) { tsnep_disable_irq(adapter, adapter->queue[i].irq_mask); napi_disable(&adapter->queue[i].napi); netif_napi_del(&adapter->queue[i].napi); + tsnep_free_irq(&adapter->queue[i], i == 0); + if (adapter->queue[i].rx) tsnep_rx_close(adapter->queue[i].rx); if (adapter->queue[i].tx) tsnep_tx_close(adapter->queue[i].tx); } - tsnep_phy_close(adapter); - return 0; } @@ -1017,6 +1155,22 @@ static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) return 0; } +static int tsnep_netdev_set_features(struct net_device *netdev, + netdev_features_t features) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + netdev_features_t changed = netdev->features ^ features; + bool enable; + int retval = 0; + + if (changed & NETIF_F_LOOPBACK) { + enable = !!(features & NETIF_F_LOOPBACK); + retval = tsnep_phy_loopback(adapter, enable); + } + + return retval; +} + static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev, const struct skb_shared_hwtstamps *hwtstamps, bool cycles) @@ -1038,9 +1192,9 @@ static const struct net_device_ops tsnep_netdev_ops = { .ndo_start_xmit = tsnep_netdev_xmit_frame, .ndo_eth_ioctl = tsnep_netdev_ioctl, .ndo_set_rx_mode = tsnep_netdev_set_multicast, - .ndo_get_stats64 = tsnep_netdev_get_stats64, .ndo_set_mac_address = tsnep_netdev_set_mac_address, + .ndo_set_features = tsnep_netdev_set_features, .ndo_get_tstamp = tsnep_netdev_get_tstamp, .ndo_setup_tc = tsnep_tc_setup, }; @@ -1141,6 +1295,52 @@ static int tsnep_phy_init(struct tsnep_adapter *adapter) return 0; } +static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count) +{ + u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; + char name[8]; + int i; + int retval; + + /* one TX/RX queue pair for netdev is mandatory */ + if (platform_irq_count(adapter->pdev) == 1) + retval = platform_get_irq(adapter->pdev, 0); + else + retval = platform_get_irq_byname(adapter->pdev, "mac"); + if (retval < 0) + return retval; + adapter->num_tx_queues = 1; + adapter->num_rx_queues = 1; + adapter->num_queues = 1; + adapter->queue[0].irq = retval; + adapter->queue[0].tx = &adapter->tx[0]; + adapter->queue[0].rx = &adapter->rx[0]; + adapter->queue[0].irq_mask = irq_mask; + + adapter->netdev->irq = adapter->queue[0].irq; + + /* add additional TX/RX queue pairs only if dedicated interrupt is + * available + */ + for (i = 1; i < queue_count; i++) { + sprintf(name, "txrx-%d", i); + retval = platform_get_irq_byname_optional(adapter->pdev, name); + if (retval < 0) + break; + + adapter->num_tx_queues++; + adapter->num_rx_queues++; + adapter->num_queues++; + adapter->queue[i].irq = retval; + adapter->queue[i].tx = &adapter->tx[i]; + adapter->queue[i].rx = &adapter->rx[i]; + adapter->queue[i].irq_mask = + irq_mask << (ECM_INT_TXRX_SHIFT * i); + } + + return 0; +} + static int tsnep_probe(struct platform_device *pdev) { struct tsnep_adapter *adapter; @@ -1149,6 +1349,7 @@ static int tsnep_probe(struct platform_device *pdev) u32 type; int revision; int version; + int queue_count; int retval; netdev = devm_alloc_etherdev_mqs(&pdev->dev, @@ -1170,41 +1371,39 @@ static int tsnep_probe(struct platform_device *pdev) netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; mutex_init(&adapter->gate_control_lock); + mutex_init(&adapter->rxnfc_lock); + INIT_LIST_HEAD(&adapter->rxnfc_rules); io = platform_get_resource(pdev, IORESOURCE_MEM, 0); adapter->addr = devm_ioremap_resource(&pdev->dev, io); if (IS_ERR(adapter->addr)) return PTR_ERR(adapter->addr); - adapter->irq = platform_get_irq(pdev, 0); netdev->mem_start = io->start; netdev->mem_end = io->end; - netdev->irq = adapter->irq; type = ioread32(adapter->addr + ECM_TYPE); revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; + queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT; adapter->gate_control = type & ECM_GATE_CONTROL; - - adapter->num_tx_queues = TSNEP_QUEUES; - adapter->num_rx_queues = TSNEP_QUEUES; - adapter->num_queues = TSNEP_QUEUES; - adapter->queue[0].tx = &adapter->tx[0]; - adapter->queue[0].rx = &adapter->rx[0]; - adapter->queue[0].irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; + adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT; tsnep_disable_irq(adapter, ECM_INT_ALL); - retval = devm_request_irq(&adapter->pdev->dev, adapter->irq, tsnep_irq, - 0, TSNEP, adapter); - if (retval != 0) { - dev_err(&adapter->pdev->dev, "can't get assigned irq %d.\n", - adapter->irq); + + retval = tsnep_queue_init(adapter, queue_count); + if (retval) + return retval; + + retval = dma_set_mask_and_coherent(&adapter->pdev->dev, + DMA_BIT_MASK(64)); + if (retval) { + dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n"); return retval; } - tsnep_enable_irq(adapter, ECM_INT_LINK); retval = tsnep_mac_init(adapter); if (retval) - goto mac_init_failed; + return retval; retval = tsnep_mdio_init(adapter); if (retval) @@ -1222,10 +1421,14 @@ static int tsnep_probe(struct platform_device *pdev) if (retval) goto tc_init_failed; + retval = tsnep_rxnfc_init(adapter); + if (retval) + goto rxnfc_init_failed; + netdev->netdev_ops = &tsnep_netdev_ops; netdev->ethtool_ops = &tsnep_ethtool_ops; netdev->features = NETIF_F_SG; - netdev->hw_features = netdev->features; + netdev->hw_features = netdev->features | NETIF_F_LOOPBACK; /* carrier off reporting is important to ethtool even BEFORE open */ netif_carrier_off(netdev); @@ -1242,6 +1445,8 @@ static int tsnep_probe(struct platform_device *pdev) return 0; register_failed: + tsnep_rxnfc_cleanup(adapter); +rxnfc_init_failed: tsnep_tc_cleanup(adapter); tc_init_failed: tsnep_ptp_cleanup(adapter); @@ -1250,8 +1455,6 @@ phy_init_failed: if (adapter->mdiobus) mdiobus_unregister(adapter->mdiobus); mdio_init_failed: -mac_init_failed: - tsnep_disable_irq(adapter, ECM_INT_ALL); return retval; } @@ -1261,6 +1464,8 @@ static int tsnep_remove(struct platform_device *pdev) unregister_netdev(adapter->netdev); + tsnep_rxnfc_cleanup(adapter); + tsnep_tc_cleanup(adapter); tsnep_ptp_cleanup(adapter); diff --git a/drivers/net/ethernet/engleder/tsnep_rxnfc.c b/drivers/net/ethernet/engleder/tsnep_rxnfc.c new file mode 100644 index 0000000000000000000000000000000000000000..9ac2a0cf38331e7ee49f31dcd0bcd1b2d0c5c87f --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_rxnfc.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022 Gerhard Engleder */ + +#include "tsnep.h" + +#define ETHER_TYPE_FULL_MASK ((__force __be16)~0) + +static void tsnep_enable_rule(struct tsnep_adapter *adapter, + struct tsnep_rxnfc_rule *rule) +{ + u8 rx_assign; + void __iomem *addr; + + rx_assign = TSNEP_RX_ASSIGN_ACTIVE; + rx_assign |= (rule->queue_index << TSNEP_RX_ASSIGN_QUEUE_SHIFT) & + TSNEP_RX_ASSIGN_QUEUE_MASK; + + addr = adapter->addr + TSNEP_RX_ASSIGN_ETHER_TYPE + + TSNEP_RX_ASSIGN_ETHER_TYPE_OFFSET * rule->location; + iowrite16(rule->filter.ether_type, addr); + + /* enable rule after all settings are done */ + addr = adapter->addr + TSNEP_RX_ASSIGN + + TSNEP_RX_ASSIGN_OFFSET * rule->location; + iowrite8(rx_assign, addr); +} + +static void tsnep_disable_rule(struct tsnep_adapter *adapter, + struct tsnep_rxnfc_rule *rule) +{ + void __iomem *addr; + + addr = adapter->addr + TSNEP_RX_ASSIGN + + TSNEP_RX_ASSIGN_OFFSET * rule->location; + iowrite8(0, addr); +} + +static struct tsnep_rxnfc_rule *tsnep_get_rule(struct tsnep_adapter *adapter, + int location) +{ + struct tsnep_rxnfc_rule *rule; + + list_for_each_entry(rule, &adapter->rxnfc_rules, list) { + if (rule->location == location) + return rule; + if (rule->location > location) + break; + } + + return NULL; +} + +static void tsnep_add_rule(struct tsnep_adapter *adapter, + struct tsnep_rxnfc_rule *rule) +{ + struct tsnep_rxnfc_rule *pred, *cur; + + tsnep_enable_rule(adapter, rule); + + pred = NULL; + list_for_each_entry(cur, &adapter->rxnfc_rules, list) { + if (cur->location >= rule->location) + break; + pred = cur; + } + + list_add(&rule->list, pred ? &pred->list : &adapter->rxnfc_rules); + adapter->rxnfc_count++; +} + +static void tsnep_delete_rule(struct tsnep_adapter *adapter, + struct tsnep_rxnfc_rule *rule) +{ + tsnep_disable_rule(adapter, rule); + + list_del(&rule->list); + adapter->rxnfc_count--; + + kfree(rule); +} + +static void tsnep_flush_rules(struct tsnep_adapter *adapter) +{ + struct tsnep_rxnfc_rule *rule, *tmp; + + mutex_lock(&adapter->rxnfc_lock); + + list_for_each_entry_safe(rule, tmp, &adapter->rxnfc_rules, list) + tsnep_delete_rule(adapter, rule); + + mutex_unlock(&adapter->rxnfc_lock); +} + +int tsnep_rxnfc_get_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp = &cmd->fs; + struct tsnep_rxnfc_rule *rule = NULL; + + cmd->data = adapter->rxnfc_max; + + mutex_lock(&adapter->rxnfc_lock); + + rule = tsnep_get_rule(adapter, fsp->location); + if (!rule) { + mutex_unlock(&adapter->rxnfc_lock); + + return -ENOENT; + } + + fsp->flow_type = ETHER_FLOW; + fsp->ring_cookie = rule->queue_index; + + if (rule->filter.type == TSNEP_RXNFC_ETHER_TYPE) { + fsp->h_u.ether_spec.h_proto = htons(rule->filter.ether_type); + fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK; + } + + mutex_unlock(&adapter->rxnfc_lock); + + return 0; +} + +int tsnep_rxnfc_get_all(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct tsnep_rxnfc_rule *rule; + int count = 0; + + cmd->data = adapter->rxnfc_max; + + mutex_lock(&adapter->rxnfc_lock); + + list_for_each_entry(rule, &adapter->rxnfc_rules, list) { + if (count == cmd->rule_cnt) { + mutex_unlock(&adapter->rxnfc_lock); + + return -EMSGSIZE; + } + + rule_locs[count] = rule->location; + count++; + } + + mutex_unlock(&adapter->rxnfc_lock); + + cmd->rule_cnt = count; + + return 0; +} + +static int tsnep_rxnfc_find_location(struct tsnep_adapter *adapter) +{ + struct tsnep_rxnfc_rule *tmp; + int location = 0; + + list_for_each_entry(tmp, &adapter->rxnfc_rules, list) { + if (tmp->location == location) + location++; + else + return location; + } + + if (location >= adapter->rxnfc_max) + return -ENOSPC; + + return location; +} + +static void tsnep_rxnfc_init_rule(struct tsnep_rxnfc_rule *rule, + const struct ethtool_rx_flow_spec *fsp) +{ + INIT_LIST_HEAD(&rule->list); + + rule->queue_index = fsp->ring_cookie; + rule->location = fsp->location; + + rule->filter.type = TSNEP_RXNFC_ETHER_TYPE; + rule->filter.ether_type = ntohs(fsp->h_u.ether_spec.h_proto); +} + +static int tsnep_rxnfc_check_rule(struct tsnep_adapter *adapter, + struct tsnep_rxnfc_rule *rule) +{ + struct net_device *dev = adapter->netdev; + struct tsnep_rxnfc_rule *tmp; + + list_for_each_entry(tmp, &adapter->rxnfc_rules, list) { + if (!memcmp(&rule->filter, &tmp->filter, sizeof(rule->filter)) && + tmp->location != rule->location) { + netdev_dbg(dev, "rule already exists\n"); + + return -EEXIST; + } + } + + return 0; +} + +int tsnep_rxnfc_add_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd) +{ + struct net_device *netdev = adapter->netdev; + struct ethtool_rx_flow_spec *fsp = + (struct ethtool_rx_flow_spec *)&cmd->fs; + struct tsnep_rxnfc_rule *rule, *old_rule; + int retval; + + /* only EtherType is supported */ + if (fsp->flow_type != ETHER_FLOW || + !is_zero_ether_addr(fsp->m_u.ether_spec.h_dest) || + !is_zero_ether_addr(fsp->m_u.ether_spec.h_source) || + fsp->m_u.ether_spec.h_proto != ETHER_TYPE_FULL_MASK) { + netdev_dbg(netdev, "only ethernet protocol is supported\n"); + + return -EOPNOTSUPP; + } + + if (fsp->ring_cookie > + (TSNEP_RX_ASSIGN_QUEUE_MASK >> TSNEP_RX_ASSIGN_QUEUE_SHIFT)) { + netdev_dbg(netdev, "invalid action\n"); + + return -EINVAL; + } + + if (fsp->location != RX_CLS_LOC_ANY && + fsp->location >= adapter->rxnfc_max) { + netdev_dbg(netdev, "invalid location\n"); + + return -EINVAL; + } + + rule = kzalloc(sizeof(*rule), GFP_KERNEL); + if (!rule) + return -ENOMEM; + + mutex_lock(&adapter->rxnfc_lock); + + if (fsp->location == RX_CLS_LOC_ANY) { + retval = tsnep_rxnfc_find_location(adapter); + if (retval < 0) + goto failed; + fsp->location = retval; + } + + tsnep_rxnfc_init_rule(rule, fsp); + + retval = tsnep_rxnfc_check_rule(adapter, rule); + if (retval) + goto failed; + + old_rule = tsnep_get_rule(adapter, fsp->location); + if (old_rule) + tsnep_delete_rule(adapter, old_rule); + + tsnep_add_rule(adapter, rule); + + mutex_unlock(&adapter->rxnfc_lock); + + return 0; + +failed: + mutex_unlock(&adapter->rxnfc_lock); + kfree(rule); + return retval; +} + +int tsnep_rxnfc_del_rule(struct tsnep_adapter *adapter, + struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp = + (struct ethtool_rx_flow_spec *)&cmd->fs; + struct tsnep_rxnfc_rule *rule; + + mutex_lock(&adapter->rxnfc_lock); + + rule = tsnep_get_rule(adapter, fsp->location); + if (!rule) { + mutex_unlock(&adapter->rxnfc_lock); + + return -ENOENT; + } + + tsnep_delete_rule(adapter, rule); + + mutex_unlock(&adapter->rxnfc_lock); + + return 0; +} + +int tsnep_rxnfc_init(struct tsnep_adapter *adapter) +{ + int i; + + /* disable all rules */ + for (i = 0; i < adapter->rxnfc_max; + i += sizeof(u32) / TSNEP_RX_ASSIGN_OFFSET) + iowrite32(0, adapter->addr + TSNEP_RX_ASSIGN + i); + + return 0; +} + +void tsnep_rxnfc_cleanup(struct tsnep_adapter *adapter) +{ + tsnep_flush_rules(adapter); +} diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c index 437c5acfe222cd86b9c8a46343c4a8882168fcd0..95cbad198b4b6ea113b0b5af7bbb8cde3b93d6fb 100644 --- a/drivers/net/ethernet/ethoc.c +++ b/drivers/net/ethernet/ethoc.c @@ -1224,7 +1224,7 @@ static int ethoc_probe(struct platform_device *pdev) netdev->ethtool_ops = ðoc_ethtool_ops; /* setup NAPI */ - netif_napi_add(netdev, &priv->napi, ethoc_poll, 64); + netif_napi_add(netdev, &priv->napi, ethoc_poll); spin_lock_init(&priv->lock); diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index c03663785a8d4f8e8cc2af95bc3d47d2f5213b49..a03879a27b0416da72abae5e054f056603e91f17 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1063,8 +1063,8 @@ static int ftgmac100_mdiobus_write(struct mii_bus *bus, int phy_addr, static void ftgmac100_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); } static void @@ -1506,7 +1506,7 @@ static int ftgmac100_open(struct net_device *netdev) goto err_hw; /* Initialize NAPI */ - netif_napi_add(netdev, &priv->napi, ftgmac100_poll, 64); + netif_napi_add(netdev, &priv->napi, ftgmac100_poll); /* Grab our interrupt */ err = request_irq(netdev->irq, ftgmac100_interrupt, 0, netdev->name, netdev); @@ -1701,10 +1701,14 @@ err_register_mdiobus: static void ftgmac100_phy_disconnect(struct net_device *netdev) { + struct ftgmac100 *priv = netdev_priv(netdev); + if (!netdev->phydev) return; phy_disconnect(netdev->phydev); + if (of_phy_is_fixed_link(priv->dev->of_node)) + of_phy_deregister_fixed_link(priv->dev->of_node); } static void ftgmac100_destroy_mdio(struct net_device *netdev) @@ -1867,6 +1871,26 @@ static int ftgmac100_probe(struct platform_device *pdev) err = -EINVAL; goto err_phy_connect; } + } else if (np && of_phy_is_fixed_link(np)) { + struct phy_device *phy; + + err = of_phy_register_fixed_link(np); + if (err) { + dev_err(&pdev->dev, "Failed to register fixed PHY\n"); + goto err_phy_connect; + } + + phy = of_phy_get_and_connect(priv->netdev, np, + &ftgmac100_adjust_link); + if (!phy) { + dev_err(&pdev->dev, "Failed to connect to fixed PHY\n"); + of_phy_deregister_fixed_link(np); + err = -EINVAL; + goto err_phy_connect; + } + + /* Display what we found */ + phy_attached_info(phy); } else if (np && of_get_property(np, "phy-handle", NULL)) { struct phy_device *phy; diff --git a/drivers/net/ethernet/faraday/ftmac100.c b/drivers/net/ethernet/faraday/ftmac100.c index 8a341e2d5833435a782f8cdd118fff3ff29b4eca..d95d782308280ae94dba15da7f50e9e9716d6ff4 100644 --- a/drivers/net/ethernet/faraday/ftmac100.c +++ b/drivers/net/ethernet/faraday/ftmac100.c @@ -807,8 +807,8 @@ static void ftmac100_mdio_write(struct net_device *netdev, int phy_id, int reg, static void ftmac100_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); } static int ftmac100_get_link_ksettings(struct net_device *netdev, @@ -1075,6 +1075,11 @@ static int ftmac100_probe(struct platform_device *pdev) SET_NETDEV_DEV(netdev, &pdev->dev); netdev->ethtool_ops = &ftmac100_ethtool_ops; netdev->netdev_ops = &ftmac100_netdev_ops; + netdev->max_mtu = MAX_PKT_SIZE; + + err = platform_get_ethdev_address(&pdev->dev, netdev); + if (err == -EPROBE_DEFER) + goto defer_get_mac; platform_set_drvdata(pdev, netdev); @@ -1086,7 +1091,7 @@ static int ftmac100_probe(struct platform_device *pdev) spin_lock_init(&priv->tx_lock); /* initialize NAPI */ - netif_napi_add(netdev, &priv->napi, ftmac100_poll, 64); + netif_napi_add(netdev, &priv->napi, ftmac100_poll); /* map io memory */ priv->res = request_mem_region(res->start, resource_size(res), @@ -1137,6 +1142,7 @@ err_ioremap: release_resource(priv->res); err_req_mem: netif_napi_del(&priv->napi); +defer_get_mac: free_netdev(netdev); err_alloc_etherdev: return err; diff --git a/drivers/net/ethernet/faraday/ftmac100.h b/drivers/net/ethernet/faraday/ftmac100.h index fe986f1673fc6d162868559ddb7a59954b45c370..8af32f9070f4c8648eb52e57f70aceb6be83bfc7 100644 --- a/drivers/net/ethernet/faraday/ftmac100.h +++ b/drivers/net/ethernet/faraday/ftmac100.h @@ -122,9 +122,9 @@ * Transmit descriptor, aligned to 16 bytes */ struct ftmac100_txdes { - unsigned int txdes0; - unsigned int txdes1; - unsigned int txdes2; /* TXBUF_BADR */ + __le32 txdes0; + __le32 txdes1; + __le32 txdes2; /* TXBUF_BADR */ unsigned int txdes3; /* not used by HW */ } __attribute__ ((aligned(16))); @@ -143,9 +143,9 @@ struct ftmac100_txdes { * Receive descriptor, aligned to 16 bytes */ struct ftmac100_rxdes { - unsigned int rxdes0; - unsigned int rxdes1; - unsigned int rxdes2; /* RXBUF_BADR */ + __le32 rxdes0; + __le32 rxdes1; + __le32 rxdes2; /* RXBUF_BADR */ unsigned int rxdes3; /* not used by HW */ } __attribute__ ((aligned(16))); diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c index b3939a5f7b03320df1a821b741725553e9295a39..ed18450fd2cc5554a7c6e7bdfb56cedcd8f1f457 100644 --- a/drivers/net/ethernet/fealnx.c +++ b/drivers/net/ethernet/fealnx.c @@ -1809,8 +1809,8 @@ static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *i { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig index e04e1c5cb0133c48899c53db79a2c9a7124a5673..ce866ae3df0337beef53175a33c639decdc55b94 100644 --- a/drivers/net/ethernet/freescale/Kconfig +++ b/drivers/net/ethernet/freescale/Kconfig @@ -9,7 +9,7 @@ config NET_VENDOR_FREESCALE depends on FSL_SOC || QUICC_ENGINE || CPM1 || CPM2 || PPC_MPC512x || \ M523x || M527x || M5272 || M528x || M520x || M532x || \ ARCH_MXC || ARCH_MXS || (PPC_MPC52xx && PPC_BESTCOMM) || \ - ARCH_LAYERSCAPE || COMPILE_TEST + ARCH_LAYERSCAPE || ARCH_S32 || COMPILE_TEST help If you have a network (Ethernet) card belonging to this class, say Y. @@ -23,15 +23,16 @@ if NET_VENDOR_FREESCALE config FEC tristate "FEC ethernet controller (of ColdFire and some i.MX CPUs)" depends on (M523x || M527x || M5272 || M528x || M520x || M532x || \ - ARCH_MXC || SOC_IMX28 || COMPILE_TEST) + ARCH_MXC || ARCH_S32 || SOC_IMX28 || COMPILE_TEST) default ARCH_MXC || SOC_IMX28 if ARM depends on PTP_1588_CLOCK_OPTIONAL select CRC32 select PHYLIB + select PAGE_POOL imply NET_SELFTESTS help Say Y here if you want to use the built-in 10/100 Fast ethernet - controller on some Motorola ColdFire and Freescale i.MX processors. + controller on some Motorola ColdFire and Freescale i.MX/S32 processors. config FEC_MPC52xx tristate "FEC MPC52xx driver" diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 45634579adb67814dc3c228f778cf2fda2028695..31cfa121333df3554f41366aa0437b1a20539d7c 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -197,12 +197,15 @@ static int dpaa_rx_extra_headroom; #define dpaa_get_max_mtu() \ (dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN)) +static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed); + static int dpaa_netdev_init(struct net_device *net_dev, const struct net_device_ops *dpaa_ops, u16 tx_timeout) { struct dpaa_priv *priv = netdev_priv(net_dev); struct device *dev = net_dev->dev.parent; + struct mac_device *mac_dev = priv->mac_dev; struct dpaa_percpu_priv *percpu_priv; const u8 *mac_addr; int i, err; @@ -216,10 +219,10 @@ static int dpaa_netdev_init(struct net_device *net_dev, } net_dev->netdev_ops = dpaa_ops; - mac_addr = priv->mac_dev->addr; + mac_addr = mac_dev->addr; - net_dev->mem_start = priv->mac_dev->res->start; - net_dev->mem_end = priv->mac_dev->res->end; + net_dev->mem_start = (unsigned long)mac_dev->vaddr; + net_dev->mem_end = (unsigned long)mac_dev->vaddr_end; net_dev->min_mtu = ETH_MIN_MTU; net_dev->max_mtu = dpaa_get_max_mtu(); @@ -246,7 +249,7 @@ static int dpaa_netdev_init(struct net_device *net_dev, eth_hw_addr_set(net_dev, mac_addr); } else { eth_hw_addr_random(net_dev); - err = priv->mac_dev->change_addr(priv->mac_dev->fman_mac, + err = mac_dev->change_addr(mac_dev->fman_mac, (const enet_addr_t *)net_dev->dev_addr); if (err) { dev_err(dev, "Failed to set random MAC address\n"); @@ -261,6 +264,9 @@ static int dpaa_netdev_init(struct net_device *net_dev, net_dev->needed_headroom = priv->tx_headroom; net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout); + mac_dev->net_dev = net_dev; + mac_dev->update_speed = dpaa_eth_cgr_set_speed; + /* start without the RUNNING flag, phylib controls it later */ netif_carrier_off(net_dev); @@ -288,10 +294,9 @@ static int dpaa_stop(struct net_device *net_dev) */ msleep(200); - err = mac_dev->stop(mac_dev); - if (err < 0) - netif_err(priv, ifdown, net_dev, "mac_dev->stop() = %d\n", - err); + if (mac_dev->phy_dev) + phy_stop(mac_dev->phy_dev); + mac_dev->disable(mac_dev->fman_mac); for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) { error = fman_port_disable(mac_dev->port[i]); @@ -826,10 +831,10 @@ static int dpaa_eth_cgr_init(struct dpaa_priv *priv) initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES); initcgr.cgr.cscn_en = QM_CGR_EN; - /* Set different thresholds based on the MAC speed. - * This may turn suboptimal if the MAC is reconfigured at a speed - * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link. - * In such cases, we ought to reconfigure the threshold, too. + /* Set different thresholds based on the configured MAC speed. + * This may turn suboptimal if the MAC is reconfigured at another + * speed, so MACs must call dpaa_eth_cgr_set_speed in their adjust_link + * callback. */ if (priv->mac_dev->if_support & SUPPORTED_10000baseT_Full) cs_th = DPAA_CS_THRESHOLD_10G; @@ -858,6 +863,31 @@ out_error: return err; } +static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed) +{ + struct net_device *net_dev = mac_dev->net_dev; + struct dpaa_priv *priv = netdev_priv(net_dev); + struct qm_mcc_initcgr opts = { }; + u32 cs_th; + int err; + + opts.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES); + switch (speed) { + case SPEED_10000: + cs_th = DPAA_CS_THRESHOLD_10G; + break; + case SPEED_1000: + default: + cs_th = DPAA_CS_THRESHOLD_1G; + break; + } + qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, cs_th, 1); + + err = qman_update_cgr_safe(&priv->cgr_data.cgr, &opts); + if (err) + netdev_err(net_dev, "could not update speed: %d\n", err); +} + static inline void dpaa_setup_ingress(const struct dpaa_priv *priv, struct dpaa_fq *fq, const struct qman_fq *template) @@ -2886,6 +2916,7 @@ static void dpaa_adjust_link(struct net_device *net_dev) /* The Aquantia PHYs are capable of performing rate adaptation */ #define PHY_VEND_AQUANTIA 0x03a1b400 +#define PHY_VEND_AQUANTIA2 0x31c31c00 static int dpaa_phy_init(struct net_device *net_dev) { @@ -2893,6 +2924,7 @@ static int dpaa_phy_init(struct net_device *net_dev) struct mac_device *mac_dev; struct phy_device *phy_dev; struct dpaa_priv *priv; + u32 phy_vendor; priv = netdev_priv(net_dev); mac_dev = priv->mac_dev; @@ -2905,9 +2937,11 @@ static int dpaa_phy_init(struct net_device *net_dev) return -ENODEV; } + phy_vendor = phy_dev->drv->phy_id & GENMASK(31, 10); /* Unless the PHY is capable of rate adaptation */ if (mac_dev->phy_if != PHY_INTERFACE_MODE_XGMII || - ((phy_dev->drv->phy_id & GENMASK(31, 10)) != PHY_VEND_AQUANTIA)) { + (phy_vendor != PHY_VEND_AQUANTIA && + phy_vendor != PHY_VEND_AQUANTIA2)) { /* remove any features not supported by the controller */ ethtool_convert_legacy_u32_to_link_mode(mask, mac_dev->if_support); @@ -2942,11 +2976,12 @@ static int dpaa_open(struct net_device *net_dev) goto mac_start_failed; } - err = priv->mac_dev->start(mac_dev); + err = priv->mac_dev->enable(mac_dev->fman_mac); if (err < 0) { - netif_err(priv, ifup, net_dev, "mac_dev->start() = %d\n", err); + netif_err(priv, ifup, net_dev, "mac_dev->enable() = %d\n", err); goto mac_start_failed; } + phy_start(priv->mac_dev->phy_dev); netif_tx_start_all_queues(net_dev); @@ -3148,8 +3183,7 @@ static int dpaa_napi_add(struct net_device *net_dev) for_each_possible_cpu(cpu) { percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu); - netif_napi_add(net_dev, &percpu_priv->np.napi, - dpaa_eth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(net_dev, &percpu_priv->np.napi, dpaa_eth_poll); } return 0; diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c index 4fee74c024bd74022f90826e6920b16f6ae27799..258eb6c8f4c060f85576bb357ad5ab734f8d3cae 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c @@ -18,7 +18,7 @@ static ssize_t dpaa_eth_show_addr(struct device *dev, if (mac_dev) return sprintf(buf, "%llx", - (unsigned long long)mac_dev->res->start); + (unsigned long long)mac_dev->vaddr); else return sprintf(buf, "none"); } diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c index 73f07881ce2dc609c725d60e784283880e3b47f1..769e936a263c92092954fba947284f89d2b19780 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c @@ -80,9 +80,9 @@ static int dpaa_set_link_ksettings(struct net_device *net_dev, static void dpaa_get_drvinfo(struct net_device *net_dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, KBUILD_MODNAME, + strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(net_dev->dev.parent->parent), + strscpy(drvinfo->bus_info, dev_name(net_dev->dev.parent->parent), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 75d51572693d64fe42e2ab84483fe25754fe4490..8d029addddad0e5d1f2c28ba574af012c94a092c 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -4565,8 +4565,7 @@ static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv) for (i = 0; i < priv->num_channels; i++) { ch = priv->channel[i]; /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */ - netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll); } } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index c9bee9a0c9b2bd840fe1611fb13ec52137ef3c8b..49ff85633783a362d7d1db9962b5a4c5ad7e17cd 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -549,7 +549,7 @@ void dpaa2_mac_get_strings(u8 *data) int i; for (i = 0; i < DPAA2_MAC_NUM_STATS; i++) { - strlcpy(p, dpaa2_mac_ethtool_stats[i], ETH_GSTRING_LEN); + strscpy(p, dpaa2_mac_ethtool_stats[i], ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index e507e90652148e68c14fced19d1bcac4f6b217db..2b5909fa93cfa47aee2bceffc8089283cd093372 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -3373,9 +3373,8 @@ static int dpaa2_switch_probe(struct fsl_mc_device *sw_dev) * different queues for each switch ports. */ for (i = 0; i < DPAA2_SWITCH_RX_NUM_FQS; i++) - netif_napi_add(ethsw->ports[0]->netdev, - ðsw->fq[i].napi, dpaa2_switch_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(ethsw->ports[0]->netdev, ðsw->fq[i].napi, + dpaa2_switch_poll); /* Setup IRQs */ err = dpaa2_switch_setup_irqs(sw_dev); diff --git a/drivers/net/ethernet/freescale/enetc/Makefile b/drivers/net/ethernet/freescale/enetc/Makefile index a139f2e9d59f058d22251ca9563d809da95c199d..e0e8dfd1379301c8167425a3f46e19b70b1d4a58 100644 --- a/drivers/net/ethernet/freescale/enetc/Makefile +++ b/drivers/net/ethernet/freescale/enetc/Makefile @@ -9,7 +9,6 @@ fsl-enetc-$(CONFIG_FSL_ENETC_QOS) += enetc_qos.o obj-$(CONFIG_FSL_ENETC_VF) += fsl-enetc-vf.o fsl-enetc-vf-y := enetc_vf.o $(common-objs) -fsl-enetc-vf-$(CONFIG_FSL_ENETC_QOS) += enetc_qos.o obj-$(CONFIG_FSL_ENETC_IERB) += fsl-enetc-ierb.o fsl-enetc-ierb-y := enetc_ierb.o diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 4470a4a3e4c3ead28f46d769436bba29e181db83..54bc92fc6bf07ab2a881eb860cbabb0bcfec9e9d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -2116,13 +2116,14 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) static void enetc_setup_bdrs(struct enetc_ndev_priv *priv) { + struct enetc_hw *hw = &priv->si->hw; int i; for (i = 0; i < priv->num_tx_rings; i++) - enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]); + enetc_setup_txbdr(hw, priv->tx_ring[i]); for (i = 0; i < priv->num_rx_rings; i++) - enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]); + enetc_setup_rxbdr(hw, priv->rx_ring[i]); } static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) @@ -2155,13 +2156,14 @@ static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) { + struct enetc_hw *hw = &priv->si->hw; int i; for (i = 0; i < priv->num_tx_rings; i++) - enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]); + enetc_clear_txbdr(hw, priv->tx_ring[i]); for (i = 0; i < priv->num_rx_rings; i++) - enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]); + enetc_clear_rxbdr(hw, priv->rx_ring[i]); udelay(1); } @@ -2169,13 +2171,13 @@ static void enetc_clear_bdrs(struct enetc_ndev_priv *priv) static int enetc_setup_irqs(struct enetc_ndev_priv *priv) { struct pci_dev *pdev = priv->si->pdev; + struct enetc_hw *hw = &priv->si->hw; int i, j, err; for (i = 0; i < priv->bdr_int_num; i++) { int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i); struct enetc_int_vector *v = priv->int_vector[i]; int entry = ENETC_BDR_INT_BASE_IDX + i; - struct enetc_hw *hw = &priv->si->hw; snprintf(v->name, sizeof(v->name), "%s-rxtx%d", priv->ndev->name, i); @@ -2263,13 +2265,14 @@ static void enetc_setup_interrupts(struct enetc_ndev_priv *priv) static void enetc_clear_interrupts(struct enetc_ndev_priv *priv) { + struct enetc_hw *hw = &priv->si->hw; int i; for (i = 0; i < priv->num_tx_rings; i++) - enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0); + enetc_txbdr_wr(hw, i, ENETC_TBIER, 0); for (i = 0; i < priv->num_rx_rings; i++) - enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0); + enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0); } static int enetc_phylink_connect(struct net_device *ndev) @@ -2432,10 +2435,11 @@ int enetc_close(struct net_device *ndev) return 0; } -static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) +int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) { struct enetc_ndev_priv *priv = netdev_priv(ndev); struct tc_mqprio_qopt *mqprio = type_data; + struct enetc_hw *hw = &priv->si->hw; struct enetc_bdr *tx_ring; int num_stack_tx_queues; u8 num_tc; @@ -2452,7 +2456,7 @@ static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) /* Reset all ring priorities to 0 */ for (i = 0; i < priv->num_tx_rings; i++) { tx_ring = priv->tx_ring[i]; - enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0); + enetc_set_bdr_prio(hw, tx_ring->index, 0); } return 0; @@ -2471,7 +2475,7 @@ static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) */ for (i = 0; i < num_tc; i++) { tx_ring = priv->tx_ring[i]; - enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i); + enetc_set_bdr_prio(hw, tx_ring->index, i); } /* Reset the number of netdev queues based on the TC count */ @@ -2486,25 +2490,6 @@ static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data) return 0; } -int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, - void *type_data) -{ - switch (type) { - case TC_SETUP_QDISC_MQPRIO: - return enetc_setup_tc_mqprio(ndev, type_data); - case TC_SETUP_QDISC_TAPRIO: - return enetc_setup_tc_taprio(ndev, type_data); - case TC_SETUP_QDISC_CBS: - return enetc_setup_tc_cbs(ndev, type_data); - case TC_SETUP_QDISC_ETF: - return enetc_setup_tc_txtime(ndev, type_data); - case TC_SETUP_BLOCK: - return enetc_setup_tc_psfp(ndev, type_data); - default: - return -EOPNOTSUPP; - } -} - static int enetc_setup_xdp_prog(struct net_device *dev, struct bpf_prog *prog, struct netlink_ext_ack *extack) { @@ -2600,52 +2585,29 @@ static int enetc_set_rss(struct net_device *ndev, int en) return 0; } -static int enetc_set_psfp(struct net_device *ndev, int en) -{ - struct enetc_ndev_priv *priv = netdev_priv(ndev); - int err; - - if (en) { - err = enetc_psfp_enable(priv); - if (err) - return err; - - priv->active_offloads |= ENETC_F_QCI; - return 0; - } - - err = enetc_psfp_disable(priv); - if (err) - return err; - - priv->active_offloads &= ~ENETC_F_QCI; - - return 0; -} - static void enetc_enable_rxvlan(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; int i; for (i = 0; i < priv->num_rx_rings; i++) - enetc_bdr_enable_rxvlan(&priv->si->hw, i, en); + enetc_bdr_enable_rxvlan(hw, i, en); } static void enetc_enable_txvlan(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; int i; for (i = 0; i < priv->num_tx_rings; i++) - enetc_bdr_enable_txvlan(&priv->si->hw, i, en); + enetc_bdr_enable_txvlan(hw, i, en); } -int enetc_set_features(struct net_device *ndev, - netdev_features_t features) +void enetc_set_features(struct net_device *ndev, netdev_features_t features) { netdev_features_t changed = ndev->features ^ features; - int err = 0; if (changed & NETIF_F_RXHASH) enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH)); @@ -2657,11 +2619,6 @@ int enetc_set_features(struct net_device *ndev, if (changed & NETIF_F_HW_VLAN_CTAG_TX) enetc_enable_txvlan(ndev, !!(features & NETIF_F_HW_VLAN_CTAG_TX)); - - if (changed & NETIF_F_HW_TC) - err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); - - return err; } #ifdef CONFIG_FSL_ENETC_PTP_CLOCK @@ -2808,8 +2765,7 @@ int enetc_alloc_msix(struct enetc_ndev_priv *priv) v->rx_dim_en = true; } INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work); - netif_napi_add(priv->ndev, &v->napi, enetc_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(priv->ndev, &v->napi, enetc_poll); v->count_tx_rings = v_tx_rings; for (j = 0; j < v_tx_rings; j++) { diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 29922c20531f0a43ed1e480dc769b3efb46f71ac..161930a65f61441f7407ae987ac0b5aaa8daa5d0 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -393,11 +393,9 @@ void enetc_start(struct net_device *ndev); void enetc_stop(struct net_device *ndev); netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev); struct net_device_stats *enetc_get_stats(struct net_device *ndev); -int enetc_set_features(struct net_device *ndev, - netdev_features_t features); +void enetc_set_features(struct net_device *ndev, netdev_features_t features); int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd); -int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type, - void *type_data); +int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data); int enetc_setup_bpf(struct net_device *dev, struct netdev_bpf *xdp); int enetc_xdp_xmit(struct net_device *ndev, int num_frames, struct xdp_frame **frames, u32 flags); @@ -455,7 +453,11 @@ static inline void enetc_cbd_free_data_mem(struct enetc_si *si, int size, data, *dma); } +void enetc_reset_ptcmsdur(struct enetc_hw *hw); +void enetc_set_ptcmsdur(struct enetc_hw *hw, u32 *queue_max_sdu); + #ifdef CONFIG_FSL_ENETC_QOS +int enetc_qos_query_caps(struct net_device *ndev, void *type_data); int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data); void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed); int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data); @@ -465,22 +467,24 @@ int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data, int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data); int enetc_psfp_init(struct enetc_ndev_priv *priv); int enetc_psfp_clean(struct enetc_ndev_priv *priv); +int enetc_set_psfp(struct net_device *ndev, bool en); static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv) { + struct enetc_hw *hw = &priv->si->hw; u32 reg; - reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR); + reg = enetc_port_rd(hw, ENETC_PSIDCAPR); priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK; /* Port stream filter capability */ - reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR); + reg = enetc_port_rd(hw, ENETC_PSFCAPR); priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK; /* Port stream gate capability */ - reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR); + reg = enetc_port_rd(hw, ENETC_PSGCAPR); priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK); priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16; /* Port flow meter capability */ - reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR); + reg = enetc_port_rd(hw, ENETC_PFMCAPR); priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK; } @@ -521,6 +525,7 @@ static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv) } #else +#define enetc_qos_query_caps(ndev, type_data) -EOPNOTSUPP #define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP #define enetc_sched_speed_set(priv, speed) (void)0 #define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP @@ -540,4 +545,9 @@ static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv) { return 0; } + +static inline int enetc_set_psfp(struct net_device *ndev, bool en) +{ + return 0; +} #endif diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index ff872e40ce85d7617f21f1587ffe4bd4bf58f441..c8369e3752b0e53bb7713a7d325dc4cddb076cce 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -125,68 +125,68 @@ static const struct { int reg; char name[ETH_GSTRING_LEN]; } enetc_port_counters[] = { - { ENETC_PM0_REOCT, "MAC rx ethernet octets" }, - { ENETC_PM0_RALN, "MAC rx alignment errors" }, - { ENETC_PM0_RXPF, "MAC rx valid pause frames" }, - { ENETC_PM0_RFRM, "MAC rx valid frames" }, - { ENETC_PM0_RFCS, "MAC rx fcs errors" }, - { ENETC_PM0_RVLAN, "MAC rx VLAN frames" }, - { ENETC_PM0_RERR, "MAC rx frame errors" }, - { ENETC_PM0_RUCA, "MAC rx unicast frames" }, - { ENETC_PM0_RMCA, "MAC rx multicast frames" }, - { ENETC_PM0_RBCA, "MAC rx broadcast frames" }, - { ENETC_PM0_RDRP, "MAC rx dropped packets" }, - { ENETC_PM0_RPKT, "MAC rx packets" }, - { ENETC_PM0_RUND, "MAC rx undersized packets" }, - { ENETC_PM0_R64, "MAC rx 64 byte packets" }, - { ENETC_PM0_R127, "MAC rx 65-127 byte packets" }, - { ENETC_PM0_R255, "MAC rx 128-255 byte packets" }, - { ENETC_PM0_R511, "MAC rx 256-511 byte packets" }, - { ENETC_PM0_R1023, "MAC rx 512-1023 byte packets" }, - { ENETC_PM0_R1522, "MAC rx 1024-1522 byte packets" }, - { ENETC_PM0_R1523X, "MAC rx 1523 to max-octet packets" }, - { ENETC_PM0_ROVR, "MAC rx oversized packets" }, - { ENETC_PM0_RJBR, "MAC rx jabber packets" }, - { ENETC_PM0_RFRG, "MAC rx fragment packets" }, - { ENETC_PM0_RCNP, "MAC rx control packets" }, - { ENETC_PM0_RDRNTP, "MAC rx fifo drop" }, - { ENETC_PM0_TEOCT, "MAC tx ethernet octets" }, - { ENETC_PM0_TOCT, "MAC tx octets" }, - { ENETC_PM0_TCRSE, "MAC tx carrier sense errors" }, - { ENETC_PM0_TXPF, "MAC tx valid pause frames" }, - { ENETC_PM0_TFRM, "MAC tx frames" }, - { ENETC_PM0_TFCS, "MAC tx fcs errors" }, - { ENETC_PM0_TVLAN, "MAC tx VLAN frames" }, - { ENETC_PM0_TERR, "MAC tx frame errors" }, - { ENETC_PM0_TUCA, "MAC tx unicast frames" }, - { ENETC_PM0_TMCA, "MAC tx multicast frames" }, - { ENETC_PM0_TBCA, "MAC tx broadcast frames" }, - { ENETC_PM0_TPKT, "MAC tx packets" }, - { ENETC_PM0_TUND, "MAC tx undersized packets" }, - { ENETC_PM0_T64, "MAC tx 64 byte packets" }, - { ENETC_PM0_T127, "MAC tx 65-127 byte packets" }, - { ENETC_PM0_T255, "MAC tx 128-255 byte packets" }, - { ENETC_PM0_T511, "MAC tx 256-511 byte packets" }, - { ENETC_PM0_T1023, "MAC tx 512-1023 byte packets" }, - { ENETC_PM0_T1522, "MAC tx 1024-1522 byte packets" }, - { ENETC_PM0_T1523X, "MAC tx 1523 to max-octet packets" }, - { ENETC_PM0_TCNP, "MAC tx control packets" }, - { ENETC_PM0_TDFR, "MAC tx deferred packets" }, - { ENETC_PM0_TMCOL, "MAC tx multiple collisions" }, - { ENETC_PM0_TSCOL, "MAC tx single collisions" }, - { ENETC_PM0_TLCOL, "MAC tx late collisions" }, - { ENETC_PM0_TECOL, "MAC tx excessive collisions" }, - { ENETC_UFDMF, "SI MAC nomatch u-cast discards" }, - { ENETC_MFDMF, "SI MAC nomatch m-cast discards" }, - { ENETC_PBFDSIR, "SI MAC nomatch b-cast discards" }, - { ENETC_PUFDVFR, "SI VLAN nomatch u-cast discards" }, - { ENETC_PMFDVFR, "SI VLAN nomatch m-cast discards" }, - { ENETC_PBFDVFR, "SI VLAN nomatch b-cast discards" }, - { ENETC_PFDMSAPR, "SI pruning discarded frames" }, - { ENETC_PICDR(0), "ICM DR0 discarded frames" }, - { ENETC_PICDR(1), "ICM DR1 discarded frames" }, - { ENETC_PICDR(2), "ICM DR2 discarded frames" }, - { ENETC_PICDR(3), "ICM DR3 discarded frames" }, + { ENETC_PM_REOCT(0), "MAC rx ethernet octets" }, + { ENETC_PM_RALN(0), "MAC rx alignment errors" }, + { ENETC_PM_RXPF(0), "MAC rx valid pause frames" }, + { ENETC_PM_RFRM(0), "MAC rx valid frames" }, + { ENETC_PM_RFCS(0), "MAC rx fcs errors" }, + { ENETC_PM_RVLAN(0), "MAC rx VLAN frames" }, + { ENETC_PM_RERR(0), "MAC rx frame errors" }, + { ENETC_PM_RUCA(0), "MAC rx unicast frames" }, + { ENETC_PM_RMCA(0), "MAC rx multicast frames" }, + { ENETC_PM_RBCA(0), "MAC rx broadcast frames" }, + { ENETC_PM_RDRP(0), "MAC rx dropped packets" }, + { ENETC_PM_RPKT(0), "MAC rx packets" }, + { ENETC_PM_RUND(0), "MAC rx undersized packets" }, + { ENETC_PM_R64(0), "MAC rx 64 byte packets" }, + { ENETC_PM_R127(0), "MAC rx 65-127 byte packets" }, + { ENETC_PM_R255(0), "MAC rx 128-255 byte packets" }, + { ENETC_PM_R511(0), "MAC rx 256-511 byte packets" }, + { ENETC_PM_R1023(0), "MAC rx 512-1023 byte packets" }, + { ENETC_PM_R1522(0), "MAC rx 1024-1522 byte packets" }, + { ENETC_PM_R1523X(0), "MAC rx 1523 to max-octet packets" }, + { ENETC_PM_ROVR(0), "MAC rx oversized packets" }, + { ENETC_PM_RJBR(0), "MAC rx jabber packets" }, + { ENETC_PM_RFRG(0), "MAC rx fragment packets" }, + { ENETC_PM_RCNP(0), "MAC rx control packets" }, + { ENETC_PM_RDRNTP(0), "MAC rx fifo drop" }, + { ENETC_PM_TEOCT(0), "MAC tx ethernet octets" }, + { ENETC_PM_TOCT(0), "MAC tx octets" }, + { ENETC_PM_TCRSE(0), "MAC tx carrier sense errors" }, + { ENETC_PM_TXPF(0), "MAC tx valid pause frames" }, + { ENETC_PM_TFRM(0), "MAC tx frames" }, + { ENETC_PM_TFCS(0), "MAC tx fcs errors" }, + { ENETC_PM_TVLAN(0), "MAC tx VLAN frames" }, + { ENETC_PM_TERR(0), "MAC tx frame errors" }, + { ENETC_PM_TUCA(0), "MAC tx unicast frames" }, + { ENETC_PM_TMCA(0), "MAC tx multicast frames" }, + { ENETC_PM_TBCA(0), "MAC tx broadcast frames" }, + { ENETC_PM_TPKT(0), "MAC tx packets" }, + { ENETC_PM_TUND(0), "MAC tx undersized packets" }, + { ENETC_PM_T64(0), "MAC tx 64 byte packets" }, + { ENETC_PM_T127(0), "MAC tx 65-127 byte packets" }, + { ENETC_PM_T255(0), "MAC tx 128-255 byte packets" }, + { ENETC_PM_T511(0), "MAC tx 256-511 byte packets" }, + { ENETC_PM_T1023(0), "MAC tx 512-1023 byte packets" }, + { ENETC_PM_T1522(0), "MAC tx 1024-1522 byte packets" }, + { ENETC_PM_T1523X(0), "MAC tx 1523 to max-octet packets" }, + { ENETC_PM_TCNP(0), "MAC tx control packets" }, + { ENETC_PM_TDFR(0), "MAC tx deferred packets" }, + { ENETC_PM_TMCOL(0), "MAC tx multiple collisions" }, + { ENETC_PM_TSCOL(0), "MAC tx single collisions" }, + { ENETC_PM_TLCOL(0), "MAC tx late collisions" }, + { ENETC_PM_TECOL(0), "MAC tx excessive collisions" }, + { ENETC_UFDMF, "SI MAC nomatch u-cast discards" }, + { ENETC_MFDMF, "SI MAC nomatch m-cast discards" }, + { ENETC_PBFDSIR, "SI MAC nomatch b-cast discards" }, + { ENETC_PUFDVFR, "SI VLAN nomatch u-cast discards" }, + { ENETC_PMFDVFR, "SI VLAN nomatch m-cast discards" }, + { ENETC_PBFDVFR, "SI VLAN nomatch b-cast discards" }, + { ENETC_PFDMSAPR, "SI pruning discarded frames" }, + { ENETC_PICDR(0), "ICM DR0 discarded frames" }, + { ENETC_PICDR(1), "ICM DR1 discarded frames" }, + { ENETC_PICDR(2), "ICM DR2 discarded frames" }, + { ENETC_PICDR(3), "ICM DR3 discarded frames" }, }; static const char rx_ring_stats[][ETH_GSTRING_LEN] = { @@ -236,7 +236,7 @@ static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data) switch (stringset) { case ETH_SS_STATS: for (i = 0; i < ARRAY_SIZE(enetc_si_counters); i++) { - strlcpy(p, enetc_si_counters[i].name, ETH_GSTRING_LEN); + strscpy(p, enetc_si_counters[i].name, ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } for (i = 0; i < priv->num_tx_rings; i++) { @@ -258,7 +258,7 @@ static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data) break; for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) { - strlcpy(p, enetc_port_counters[i].name, + strscpy(p, enetc_port_counters[i].name, ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } @@ -301,6 +301,113 @@ static void enetc_get_ethtool_stats(struct net_device *ndev, data[o++] = enetc_port_rd(hw, enetc_port_counters[i].reg); } +static void enetc_get_pause_stats(struct net_device *ndev, + struct ethtool_pause_stats *pause_stats) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; + + pause_stats->tx_pause_frames = enetc_port_rd(hw, ENETC_PM_TXPF(0)); + pause_stats->rx_pause_frames = enetc_port_rd(hw, ENETC_PM_RXPF(0)); +} + +static void enetc_mac_stats(struct enetc_hw *hw, int mac, + struct ethtool_eth_mac_stats *s) +{ + s->FramesTransmittedOK = enetc_port_rd(hw, ENETC_PM_TFRM(mac)); + s->SingleCollisionFrames = enetc_port_rd(hw, ENETC_PM_TSCOL(mac)); + s->MultipleCollisionFrames = enetc_port_rd(hw, ENETC_PM_TMCOL(mac)); + s->FramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RFRM(mac)); + s->FrameCheckSequenceErrors = enetc_port_rd(hw, ENETC_PM_RFCS(mac)); + s->AlignmentErrors = enetc_port_rd(hw, ENETC_PM_RALN(mac)); + s->OctetsTransmittedOK = enetc_port_rd(hw, ENETC_PM_TEOCT(mac)); + s->FramesWithDeferredXmissions = enetc_port_rd(hw, ENETC_PM_TDFR(mac)); + s->LateCollisions = enetc_port_rd(hw, ENETC_PM_TLCOL(mac)); + s->FramesAbortedDueToXSColls = enetc_port_rd(hw, ENETC_PM_TECOL(mac)); + s->FramesLostDueToIntMACXmitError = enetc_port_rd(hw, ENETC_PM_TERR(mac)); + s->CarrierSenseErrors = enetc_port_rd(hw, ENETC_PM_TCRSE(mac)); + s->OctetsReceivedOK = enetc_port_rd(hw, ENETC_PM_REOCT(mac)); + s->FramesLostDueToIntMACRcvError = enetc_port_rd(hw, ENETC_PM_RDRNTP(mac)); + s->MulticastFramesXmittedOK = enetc_port_rd(hw, ENETC_PM_TMCA(mac)); + s->BroadcastFramesXmittedOK = enetc_port_rd(hw, ENETC_PM_TBCA(mac)); + s->MulticastFramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RMCA(mac)); + s->BroadcastFramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RBCA(mac)); +} + +static void enetc_ctrl_stats(struct enetc_hw *hw, int mac, + struct ethtool_eth_ctrl_stats *s) +{ + s->MACControlFramesTransmitted = enetc_port_rd(hw, ENETC_PM_TCNP(mac)); + s->MACControlFramesReceived = enetc_port_rd(hw, ENETC_PM_RCNP(mac)); +} + +static const struct ethtool_rmon_hist_range enetc_rmon_ranges[] = { + { 64, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1522 }, + { 1523, ENETC_MAC_MAXFRM_SIZE }, + {}, +}; + +static void enetc_rmon_stats(struct enetc_hw *hw, int mac, + struct ethtool_rmon_stats *s, + const struct ethtool_rmon_hist_range **ranges) +{ + s->undersize_pkts = enetc_port_rd(hw, ENETC_PM_RUND(mac)); + s->oversize_pkts = enetc_port_rd(hw, ENETC_PM_ROVR(mac)); + s->fragments = enetc_port_rd(hw, ENETC_PM_RFRG(mac)); + s->jabbers = enetc_port_rd(hw, ENETC_PM_RJBR(mac)); + + s->hist[0] = enetc_port_rd(hw, ENETC_PM_R64(mac)); + s->hist[1] = enetc_port_rd(hw, ENETC_PM_R127(mac)); + s->hist[2] = enetc_port_rd(hw, ENETC_PM_R255(mac)); + s->hist[3] = enetc_port_rd(hw, ENETC_PM_R511(mac)); + s->hist[4] = enetc_port_rd(hw, ENETC_PM_R1023(mac)); + s->hist[5] = enetc_port_rd(hw, ENETC_PM_R1522(mac)); + s->hist[6] = enetc_port_rd(hw, ENETC_PM_R1523X(mac)); + + s->hist_tx[0] = enetc_port_rd(hw, ENETC_PM_T64(mac)); + s->hist_tx[1] = enetc_port_rd(hw, ENETC_PM_T127(mac)); + s->hist_tx[2] = enetc_port_rd(hw, ENETC_PM_T255(mac)); + s->hist_tx[3] = enetc_port_rd(hw, ENETC_PM_T511(mac)); + s->hist_tx[4] = enetc_port_rd(hw, ENETC_PM_T1023(mac)); + s->hist_tx[5] = enetc_port_rd(hw, ENETC_PM_T1522(mac)); + s->hist_tx[6] = enetc_port_rd(hw, ENETC_PM_T1523X(mac)); + + *ranges = enetc_rmon_ranges; +} + +static void enetc_get_eth_mac_stats(struct net_device *ndev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; + + enetc_mac_stats(hw, 0, mac_stats); +} + +static void enetc_get_eth_ctrl_stats(struct net_device *ndev, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; + + enetc_ctrl_stats(hw, 0, ctrl_stats); +} + +static void enetc_get_rmon_stats(struct net_device *ndev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; + + enetc_rmon_stats(hw, 0, rmon_stats, ranges); +} + #define ENETC_RSSHASH_L3 (RXH_L2DA | RXH_VLAN | RXH_L3_PROTO | RXH_IP_SRC | \ RXH_IP_DST) #define ENETC_RSSHASH_L4 (ENETC_RSSHASH_L3 | RXH_L4_B_0_1 | RXH_L4_B_2_3) @@ -766,6 +873,10 @@ static const struct ethtool_ops enetc_pf_ethtool_ops = { .get_sset_count = enetc_get_sset_count, .get_strings = enetc_get_strings, .get_ethtool_stats = enetc_get_ethtool_stats, + .get_pause_stats = enetc_get_pause_stats, + .get_rmon_stats = enetc_get_rmon_stats, + .get_eth_ctrl_stats = enetc_get_eth_ctrl_stats, + .get_eth_mac_stats = enetc_get_eth_mac_stats, .get_rxnfc = enetc_get_rxnfc, .set_rxnfc = enetc_set_rxnfc, .get_rxfh_key_size = enetc_get_rxfh_key_size, diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h index 647c87f73bf75791b8d8553a0c9f47149aa4111b..18ca1f42b1f754584dbab8471e16f31e68ee9836 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h @@ -276,58 +276,60 @@ enum enetc_bdr_type {TX, RX}; #define ENETC_PFMCAPR 0x1b38 #define ENETC_PFMCAPR_MSK GENMASK(15, 0) -/* MAC counters */ -#define ENETC_PM0_REOCT 0x8100 -#define ENETC_PM0_RALN 0x8110 -#define ENETC_PM0_RXPF 0x8118 -#define ENETC_PM0_RFRM 0x8120 -#define ENETC_PM0_RFCS 0x8128 -#define ENETC_PM0_RVLAN 0x8130 -#define ENETC_PM0_RERR 0x8138 -#define ENETC_PM0_RUCA 0x8140 -#define ENETC_PM0_RMCA 0x8148 -#define ENETC_PM0_RBCA 0x8150 -#define ENETC_PM0_RDRP 0x8158 -#define ENETC_PM0_RPKT 0x8160 -#define ENETC_PM0_RUND 0x8168 -#define ENETC_PM0_R64 0x8170 -#define ENETC_PM0_R127 0x8178 -#define ENETC_PM0_R255 0x8180 -#define ENETC_PM0_R511 0x8188 -#define ENETC_PM0_R1023 0x8190 -#define ENETC_PM0_R1522 0x8198 -#define ENETC_PM0_R1523X 0x81A0 -#define ENETC_PM0_ROVR 0x81A8 -#define ENETC_PM0_RJBR 0x81B0 -#define ENETC_PM0_RFRG 0x81B8 -#define ENETC_PM0_RCNP 0x81C0 -#define ENETC_PM0_RDRNTP 0x81C8 -#define ENETC_PM0_TEOCT 0x8200 -#define ENETC_PM0_TOCT 0x8208 -#define ENETC_PM0_TCRSE 0x8210 -#define ENETC_PM0_TXPF 0x8218 -#define ENETC_PM0_TFRM 0x8220 -#define ENETC_PM0_TFCS 0x8228 -#define ENETC_PM0_TVLAN 0x8230 -#define ENETC_PM0_TERR 0x8238 -#define ENETC_PM0_TUCA 0x8240 -#define ENETC_PM0_TMCA 0x8248 -#define ENETC_PM0_TBCA 0x8250 -#define ENETC_PM0_TPKT 0x8260 -#define ENETC_PM0_TUND 0x8268 -#define ENETC_PM0_T64 0x8270 -#define ENETC_PM0_T127 0x8278 -#define ENETC_PM0_T255 0x8280 -#define ENETC_PM0_T511 0x8288 -#define ENETC_PM0_T1023 0x8290 -#define ENETC_PM0_T1522 0x8298 -#define ENETC_PM0_T1523X 0x82A0 -#define ENETC_PM0_TCNP 0x82C0 -#define ENETC_PM0_TDFR 0x82D0 -#define ENETC_PM0_TMCOL 0x82D8 -#define ENETC_PM0_TSCOL 0x82E0 -#define ENETC_PM0_TLCOL 0x82E8 -#define ENETC_PM0_TECOL 0x82F0 +/* Port MAC counters: Port MAC 0 corresponds to the eMAC and + * Port MAC 1 to the pMAC. + */ +#define ENETC_PM_REOCT(mac) (0x8100 + 0x1000 * (mac)) +#define ENETC_PM_RALN(mac) (0x8110 + 0x1000 * (mac)) +#define ENETC_PM_RXPF(mac) (0x8118 + 0x1000 * (mac)) +#define ENETC_PM_RFRM(mac) (0x8120 + 0x1000 * (mac)) +#define ENETC_PM_RFCS(mac) (0x8128 + 0x1000 * (mac)) +#define ENETC_PM_RVLAN(mac) (0x8130 + 0x1000 * (mac)) +#define ENETC_PM_RERR(mac) (0x8138 + 0x1000 * (mac)) +#define ENETC_PM_RUCA(mac) (0x8140 + 0x1000 * (mac)) +#define ENETC_PM_RMCA(mac) (0x8148 + 0x1000 * (mac)) +#define ENETC_PM_RBCA(mac) (0x8150 + 0x1000 * (mac)) +#define ENETC_PM_RDRP(mac) (0x8158 + 0x1000 * (mac)) +#define ENETC_PM_RPKT(mac) (0x8160 + 0x1000 * (mac)) +#define ENETC_PM_RUND(mac) (0x8168 + 0x1000 * (mac)) +#define ENETC_PM_R64(mac) (0x8170 + 0x1000 * (mac)) +#define ENETC_PM_R127(mac) (0x8178 + 0x1000 * (mac)) +#define ENETC_PM_R255(mac) (0x8180 + 0x1000 * (mac)) +#define ENETC_PM_R511(mac) (0x8188 + 0x1000 * (mac)) +#define ENETC_PM_R1023(mac) (0x8190 + 0x1000 * (mac)) +#define ENETC_PM_R1522(mac) (0x8198 + 0x1000 * (mac)) +#define ENETC_PM_R1523X(mac) (0x81A0 + 0x1000 * (mac)) +#define ENETC_PM_ROVR(mac) (0x81A8 + 0x1000 * (mac)) +#define ENETC_PM_RJBR(mac) (0x81B0 + 0x1000 * (mac)) +#define ENETC_PM_RFRG(mac) (0x81B8 + 0x1000 * (mac)) +#define ENETC_PM_RCNP(mac) (0x81C0 + 0x1000 * (mac)) +#define ENETC_PM_RDRNTP(mac) (0x81C8 + 0x1000 * (mac)) +#define ENETC_PM_TEOCT(mac) (0x8200 + 0x1000 * (mac)) +#define ENETC_PM_TOCT(mac) (0x8208 + 0x1000 * (mac)) +#define ENETC_PM_TCRSE(mac) (0x8210 + 0x1000 * (mac)) +#define ENETC_PM_TXPF(mac) (0x8218 + 0x1000 * (mac)) +#define ENETC_PM_TFRM(mac) (0x8220 + 0x1000 * (mac)) +#define ENETC_PM_TFCS(mac) (0x8228 + 0x1000 * (mac)) +#define ENETC_PM_TVLAN(mac) (0x8230 + 0x1000 * (mac)) +#define ENETC_PM_TERR(mac) (0x8238 + 0x1000 * (mac)) +#define ENETC_PM_TUCA(mac) (0x8240 + 0x1000 * (mac)) +#define ENETC_PM_TMCA(mac) (0x8248 + 0x1000 * (mac)) +#define ENETC_PM_TBCA(mac) (0x8250 + 0x1000 * (mac)) +#define ENETC_PM_TPKT(mac) (0x8260 + 0x1000 * (mac)) +#define ENETC_PM_TUND(mac) (0x8268 + 0x1000 * (mac)) +#define ENETC_PM_T64(mac) (0x8270 + 0x1000 * (mac)) +#define ENETC_PM_T127(mac) (0x8278 + 0x1000 * (mac)) +#define ENETC_PM_T255(mac) (0x8280 + 0x1000 * (mac)) +#define ENETC_PM_T511(mac) (0x8288 + 0x1000 * (mac)) +#define ENETC_PM_T1023(mac) (0x8290 + 0x1000 * (mac)) +#define ENETC_PM_T1522(mac) (0x8298 + 0x1000 * (mac)) +#define ENETC_PM_T1523X(mac) (0x82A0 + 0x1000 * (mac)) +#define ENETC_PM_TCNP(mac) (0x82C0 + 0x1000 * (mac)) +#define ENETC_PM_TDFR(mac) (0x82D0 + 0x1000 * (mac)) +#define ENETC_PM_TMCOL(mac) (0x82D8 + 0x1000 * (mac)) +#define ENETC_PM_TSCOL(mac) (0x82E0 + 0x1000 * (mac)) +#define ENETC_PM_TLCOL(mac) (0x82E8 + 0x1000 * (mac)) +#define ENETC_PM_TECOL(mac) (0x82F0 + 0x1000 * (mac)) /* Port counters */ #define ENETC_PICDR(n) (0x0700 + (n) * 8) /* n = [0..3] */ @@ -943,13 +945,13 @@ static inline u32 enetc_usecs_to_cycles(u32 usecs) } /* port time gating control register */ -#define ENETC_QBV_PTGCR_OFFSET 0x11a00 -#define ENETC_QBV_TGE BIT(31) -#define ENETC_QBV_TGPE BIT(30) +#define ENETC_PTGCR 0x11a00 +#define ENETC_PTGCR_TGE BIT(31) +#define ENETC_PTGCR_TGPE BIT(30) /* Port time gating capability register */ -#define ENETC_QBV_PTGCAPR_OFFSET 0x11a08 -#define ENETC_QBV_MAX_GCL_LEN_MASK GENMASK(15, 0) +#define ENETC_PTGCAPR 0x11a08 +#define ENETC_PTGCAPR_MAX_GCL_LEN_MASK GENMASK(15, 0) /* Port time specific departure */ #define ENETC_PTCTSDR(n) (0x1210 + 4 * (n)) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index c4a0e836d4f098231249c18d6b06dbde73490a34..bdf94335ee99ccb1b62055694b6a29f939a6f378 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -516,15 +516,34 @@ static void enetc_port_si_configure(struct enetc_si *si) enetc_port_wr(hw, ENETC_PSIVLANFMR, ENETC_PSIVLANFMR_VS); } -static void enetc_configure_port_mac(struct enetc_hw *hw) +void enetc_set_ptcmsdur(struct enetc_hw *hw, u32 *max_sdu) { int tc; - enetc_port_wr(hw, ENETC_PM0_MAXFRM, - ENETC_SET_MAXFRM(ENETC_RX_MAXFRM_SIZE)); + for (tc = 0; tc < 8; tc++) { + u32 val = ENETC_MAC_MAXFRM_SIZE; + + if (max_sdu[tc]) + val = max_sdu[tc] + VLAN_ETH_HLEN; + + enetc_port_wr(hw, ENETC_PTCMSDUR(tc), val); + } +} + +void enetc_reset_ptcmsdur(struct enetc_hw *hw) +{ + int tc; for (tc = 0; tc < 8; tc++) enetc_port_wr(hw, ENETC_PTCMSDUR(tc), ENETC_MAC_MAXFRM_SIZE); +} + +static void enetc_configure_port_mac(struct enetc_hw *hw) +{ + enetc_port_wr(hw, ENETC_PM0_MAXFRM, + ENETC_SET_MAXFRM(ENETC_RX_MAXFRM_SIZE)); + + enetc_reset_ptcmsdur(hw); enetc_port_wr(hw, ENETC_PM0_CMD_CFG, ENETC_PM0_CMD_PHY_TX_EN | ENETC_PM0_CMD_TXP | ENETC_PM0_PROMISC); @@ -709,6 +728,13 @@ static int enetc_pf_set_features(struct net_device *ndev, { netdev_features_t changed = ndev->features ^ features; struct enetc_ndev_priv *priv = netdev_priv(ndev); + int err; + + if (changed & NETIF_F_HW_TC) { + err = enetc_set_psfp(ndev, !!(features & NETIF_F_HW_TC)); + if (err) + return err; + } if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) { struct enetc_pf *pf = enetc_si_priv(priv->si); @@ -722,7 +748,30 @@ static int enetc_pf_set_features(struct net_device *ndev, if (changed & NETIF_F_LOOPBACK) enetc_set_loopback(ndev, !!(features & NETIF_F_LOOPBACK)); - return enetc_set_features(ndev, features); + enetc_set_features(ndev, features); + + return 0; +} + +static int enetc_pf_setup_tc(struct net_device *ndev, enum tc_setup_type type, + void *type_data) +{ + switch (type) { + case TC_QUERY_CAPS: + return enetc_qos_query_caps(ndev, type_data); + case TC_SETUP_QDISC_MQPRIO: + return enetc_setup_tc_mqprio(ndev, type_data); + case TC_SETUP_QDISC_TAPRIO: + return enetc_setup_tc_taprio(ndev, type_data); + case TC_SETUP_QDISC_CBS: + return enetc_setup_tc_cbs(ndev, type_data); + case TC_SETUP_QDISC_ETF: + return enetc_setup_tc_txtime(ndev, type_data); + case TC_SETUP_BLOCK: + return enetc_setup_tc_psfp(ndev, type_data); + default: + return -EOPNOTSUPP; + } } static const struct net_device_ops enetc_ndev_ops = { @@ -739,7 +788,7 @@ static const struct net_device_ops enetc_ndev_ops = { .ndo_set_vf_spoofchk = enetc_pf_set_vf_spoofchk, .ndo_set_features = enetc_pf_set_features, .ndo_eth_ioctl = enetc_ioctl, - .ndo_setup_tc = enetc_setup_tc, + .ndo_setup_tc = enetc_pf_setup_tc, .ndo_bpf = enetc_setup_bpf, .ndo_xdp_xmit = enetc_xdp_xmit, }; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_qos.c b/drivers/net/ethernet/freescale/enetc/enetc_qos.c index 582a663ed0ba429fd04fdf032b144e7ac1121b04..a842e1999122ca2f4e34e2a71ce2fbe126b87b3d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_qos.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_qos.c @@ -11,14 +11,14 @@ static u16 enetc_get_max_gcl_len(struct enetc_hw *hw) { - return enetc_rd(hw, ENETC_QBV_PTGCAPR_OFFSET) - & ENETC_QBV_MAX_GCL_LEN_MASK; + return enetc_rd(hw, ENETC_PTGCAPR) & ENETC_PTGCAPR_MAX_GCL_LEN_MASK; } void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed) { + struct enetc_hw *hw = &priv->si->hw; u32 old_speed = priv->speed; - u32 pspeed; + u32 pspeed, tmp; if (speed == old_speed) return; @@ -39,16 +39,15 @@ void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed) } priv->speed = speed; - enetc_port_wr(&priv->si->hw, ENETC_PMR, - (enetc_port_rd(&priv->si->hw, ENETC_PMR) - & (~ENETC_PMR_PSPEED_MASK)) - | pspeed); + tmp = enetc_port_rd(hw, ENETC_PMR); + enetc_port_wr(hw, ENETC_PMR, (tmp & ~ENETC_PMR_PSPEED_MASK) | pspeed); } static int enetc_setup_taprio(struct net_device *ndev, struct tc_taprio_qopt_offload *admin_conf) { struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; struct enetc_cbd cbd = {.cmd = 0}; struct tgs_gcl_conf *gcl_config; struct tgs_gcl_data *gcl_data; @@ -61,15 +60,14 @@ static int enetc_setup_taprio(struct net_device *ndev, int err; int i; - if (admin_conf->num_entries > enetc_get_max_gcl_len(&priv->si->hw)) + if (admin_conf->num_entries > enetc_get_max_gcl_len(hw)) return -EINVAL; gcl_len = admin_conf->num_entries; - tge = enetc_rd(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET); + tge = enetc_rd(hw, ENETC_PTGCR); if (!admin_conf->enable) { - enetc_wr(&priv->si->hw, - ENETC_QBV_PTGCR_OFFSET, - tge & (~ENETC_QBV_TGE)); + enetc_wr(hw, ENETC_PTGCR, tge & ~ENETC_PTGCR_TGE); + enetc_reset_ptcmsdur(hw); priv->active_offloads &= ~ENETC_F_QBV; @@ -117,27 +115,28 @@ static int enetc_setup_taprio(struct net_device *ndev, cbd.cls = BDCR_CMD_PORT_GCL; cbd.status_flags = 0; - enetc_wr(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET, - tge | ENETC_QBV_TGE); + enetc_wr(hw, ENETC_PTGCR, tge | ENETC_PTGCR_TGE); err = enetc_send_cmd(priv->si, &cbd); if (err) - enetc_wr(&priv->si->hw, - ENETC_QBV_PTGCR_OFFSET, - tge & (~ENETC_QBV_TGE)); + enetc_wr(hw, ENETC_PTGCR, tge & ~ENETC_PTGCR_TGE); enetc_cbd_free_data_mem(priv->si, data_size, tmp, &dma); - if (!err) - priv->active_offloads |= ENETC_F_QBV; + if (err) + return err; - return err; + enetc_set_ptcmsdur(hw, admin_conf->max_sdu); + priv->active_offloads |= ENETC_F_QBV; + + return 0; } int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data) { struct tc_taprio_qopt_offload *taprio = type_data; struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_hw *hw = &priv->si->hw; int err; int i; @@ -147,16 +146,14 @@ int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data) return -EBUSY; for (i = 0; i < priv->num_tx_rings; i++) - enetc_set_bdr_prio(&priv->si->hw, - priv->tx_ring[i]->index, + enetc_set_bdr_prio(hw, priv->tx_ring[i]->index, taprio->enable ? i : 0); err = enetc_setup_taprio(ndev, taprio); if (err) for (i = 0; i < priv->num_tx_rings; i++) - enetc_set_bdr_prio(&priv->si->hw, - priv->tx_ring[i]->index, + enetc_set_bdr_prio(hw, priv->tx_ring[i]->index, taprio->enable ? 0 : i); return err; @@ -178,7 +175,7 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) struct tc_cbs_qopt_offload *cbs = type_data; u32 port_transmit_rate = priv->speed; u8 tc_nums = netdev_get_num_tc(ndev); - struct enetc_si *si = priv->si; + struct enetc_hw *hw = &priv->si->hw; u32 hi_credit_bit, hi_credit_reg; u32 max_interference_size; u32 port_frame_max_size; @@ -199,15 +196,15 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) * lower than this TC have been disabled. */ if (tc == prio_top && - enetc_get_cbs_enable(&si->hw, prio_next)) { + enetc_get_cbs_enable(hw, prio_next)) { dev_err(&ndev->dev, "Disable TC%d before disable TC%d\n", prio_next, tc); return -EINVAL; } - enetc_port_wr(&si->hw, ENETC_PTCCBSR1(tc), 0); - enetc_port_wr(&si->hw, ENETC_PTCCBSR0(tc), 0); + enetc_port_wr(hw, ENETC_PTCCBSR1(tc), 0); + enetc_port_wr(hw, ENETC_PTCCBSR0(tc), 0); return 0; } @@ -224,13 +221,13 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) * higher than this TC have been enabled. */ if (tc == prio_next) { - if (!enetc_get_cbs_enable(&si->hw, prio_top)) { + if (!enetc_get_cbs_enable(hw, prio_top)) { dev_err(&ndev->dev, "Enable TC%d first before enable TC%d\n", prio_top, prio_next); return -EINVAL; } - bw_sum += enetc_get_cbs_bw(&si->hw, prio_top); + bw_sum += enetc_get_cbs_bw(hw, prio_top); } if (bw_sum + bw >= 100) { @@ -239,7 +236,7 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) return -EINVAL; } - enetc_port_rd(&si->hw, ENETC_PTCMSDUR(tc)); + enetc_port_rd(hw, ENETC_PTCMSDUR(tc)); /* For top prio TC, the max_interfrence_size is maxSizedFrame. * @@ -259,8 +256,8 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) u32 m0, ma, r0, ra; m0 = port_frame_max_size * 8; - ma = enetc_port_rd(&si->hw, ENETC_PTCMSDUR(prio_top)) * 8; - ra = enetc_get_cbs_bw(&si->hw, prio_top) * + ma = enetc_port_rd(hw, ENETC_PTCMSDUR(prio_top)) * 8; + ra = enetc_get_cbs_bw(hw, prio_top) * port_transmit_rate * 10000ULL; r0 = port_transmit_rate * 1000000ULL; max_interference_size = m0 + ma + @@ -280,10 +277,10 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data) hi_credit_reg = (u32)div_u64((ENETC_CLK * 100ULL) * hi_credit_bit, port_transmit_rate * 1000000ULL); - enetc_port_wr(&si->hw, ENETC_PTCCBSR1(tc), hi_credit_reg); + enetc_port_wr(hw, ENETC_PTCCBSR1(tc), hi_credit_reg); /* Set bw register and enable this traffic class */ - enetc_port_wr(&si->hw, ENETC_PTCCBSR0(tc), bw | ENETC_CBSE); + enetc_port_wr(hw, ENETC_PTCCBSR0(tc), bw | ENETC_CBSE); return 0; } @@ -293,6 +290,7 @@ int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data) struct enetc_ndev_priv *priv = netdev_priv(ndev); struct tc_etf_qopt_offload *qopt = type_data; u8 tc_nums = netdev_get_num_tc(ndev); + struct enetc_hw *hw = &priv->si->hw; int tc; if (!tc_nums) @@ -304,12 +302,11 @@ int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data) return -EINVAL; /* TSD and Qbv are mutually exclusive in hardware */ - if (enetc_rd(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET) & ENETC_QBV_TGE) + if (enetc_rd(hw, ENETC_PTGCR) & ENETC_PTGCR_TGE) return -EBUSY; priv->tx_ring[tc]->tsd_enable = qopt->enable; - enetc_port_wr(&priv->si->hw, ENETC_PTCTSDR(tc), - qopt->enable ? ENETC_TSDE : 0); + enetc_port_wr(hw, ENETC_PTCTSDR(tc), qopt->enable ? ENETC_TSDE : 0); return 0; } @@ -1517,6 +1514,29 @@ int enetc_setup_tc_block_cb(enum tc_setup_type type, void *type_data, } } +int enetc_set_psfp(struct net_device *ndev, bool en) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + int err; + + if (en) { + err = enetc_psfp_enable(priv); + if (err) + return err; + + priv->active_offloads |= ENETC_F_QCI; + return 0; + } + + err = enetc_psfp_disable(priv); + if (err) + return err; + + priv->active_offloads &= ~ENETC_F_QCI; + + return 0; +} + int enetc_psfp_init(struct enetc_ndev_priv *priv) { if (epsfp.psfp_sfi_bitmap) @@ -1578,3 +1598,23 @@ int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data) return 0; } + +int enetc_qos_query_caps(struct net_device *ndev, void *type_data) +{ + struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct tc_query_caps_base *base = type_data; + struct enetc_si *si = priv->si; + + switch (base->type) { + case TC_SETUP_QDISC_TAPRIO: { + struct tc_taprio_caps *caps = base->caps; + + if (si->hw_features & ENETC_SI_F_QBV) + caps->supports_queue_max_sdu = true; + + return 0; + } + default: + return -EOPNOTSUPP; + } +} diff --git a/drivers/net/ethernet/freescale/enetc/enetc_vf.c b/drivers/net/ethernet/freescale/enetc/enetc_vf.c index 17924305afa2f1a916a8a141219643c2490dedf1..dfcaac302e245179438f8acbe6f80fe2778c7cb3 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_vf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_vf.c @@ -88,7 +88,20 @@ static int enetc_vf_set_mac_addr(struct net_device *ndev, void *addr) static int enetc_vf_set_features(struct net_device *ndev, netdev_features_t features) { - return enetc_set_features(ndev, features); + enetc_set_features(ndev, features); + + return 0; +} + +static int enetc_vf_setup_tc(struct net_device *ndev, enum tc_setup_type type, + void *type_data) +{ + switch (type) { + case TC_SETUP_QDISC_MQPRIO: + return enetc_setup_tc_mqprio(ndev, type_data); + default: + return -EOPNOTSUPP; + } } /* Probing/ Init */ @@ -100,7 +113,7 @@ static const struct net_device_ops enetc_ndev_ops = { .ndo_set_mac_address = enetc_vf_set_mac_addr, .ndo_set_features = enetc_vf_set_features, .ndo_eth_ioctl = enetc_ioctl, - .ndo_setup_tc = enetc_setup_tc, + .ndo_setup_tc = enetc_vf_setup_tc, }; static void enetc_vf_netdev_setup(struct enetc_si *si, struct net_device *ndev, diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h index ed7301b6916941268d1f6ad54adab7c1d3615ca5..33f84a30e1671797b36f2983c4a4d7055761eee1 100644 --- a/drivers/net/ethernet/freescale/fec.h +++ b/drivers/net/ethernet/freescale/fec.h @@ -16,8 +16,12 @@ #include #include +#include +#include #include #include +#include +#include #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ @@ -343,8 +347,11 @@ struct bufdesc_ex { * the skbuffer directly. */ +#define FEC_ENET_XDP_HEADROOM (XDP_PACKET_HEADROOM) + #define FEC_ENET_RX_PAGES 256 -#define FEC_ENET_RX_FRSIZE 2048 +#define FEC_ENET_RX_FRSIZE (PAGE_SIZE - FEC_ENET_XDP_HEADROOM \ + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) #define FEC_ENET_RX_FRPPG (PAGE_SIZE / FEC_ENET_RX_FRSIZE) #define RX_RING_SIZE (FEC_ENET_RX_FRPPG * FEC_ENET_RX_PAGES) #define FEC_ENET_TX_FRSIZE 2048 @@ -498,6 +505,9 @@ struct bufdesc_ex { /* i.MX8MQ SoC integration mix wakeup interrupt signal into "int2" interrupt line. */ #define FEC_QUIRK_WAKEUP_FROM_INT2 (1 << 22) +/* i.MX6Q adds pm_qos support */ +#define FEC_QUIRK_HAS_PMQOS BIT(23) + struct bufdesc_prop { int qid; /* Address of Rx and Tx buffers */ @@ -511,6 +521,12 @@ struct bufdesc_prop { unsigned char dsize_log2; }; +struct fec_enet_priv_txrx_info { + int offset; + struct page *page; + struct sk_buff *skb; +}; + struct fec_enet_priv_tx_q { struct bufdesc_prop bd; unsigned char *tx_bounce[TX_RING_SIZE]; @@ -526,7 +542,14 @@ struct fec_enet_priv_tx_q { struct fec_enet_priv_rx_q { struct bufdesc_prop bd; - struct sk_buff *rx_skbuff[RX_RING_SIZE]; + struct fec_enet_priv_txrx_info rx_skb_info[RX_RING_SIZE]; + + /* page_pool */ + struct page_pool *page_pool; + struct xdp_rxq_info xdp_rxq; + + /* rx queue number, in the range 0-7 */ + u8 id; }; struct fec_stop_mode_gpr { @@ -579,6 +602,7 @@ struct fec_enet_private { struct device_node *phy_node; bool rgmii_txc_dly; bool rgmii_rxc_dly; + bool rpm_active; int link; int full_duplex; int speed; @@ -608,6 +632,7 @@ struct fec_enet_private { struct delayed_work time_keep; struct regulator *reg_phy; struct fec_stop_mode_gpr stop_gpr; + struct pm_qos_request pm_qos_req; unsigned int tx_align; unsigned int rx_align; @@ -634,6 +659,8 @@ struct fec_enet_private { int pps_enable; unsigned int next_counter; + struct imx_sc_ipc *ipc_handle; + u64 ethtool_stats[]; }; diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index e8e2aa1e7f01b783f6e48564fd1c34b698375ef6..98d5cd313fddff19680b59087545bffa5a4d61e2 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -66,6 +66,8 @@ #include #include #include +#include +#include #include @@ -111,7 +113,8 @@ static const struct fec_devinfo fec_imx6q_info = { .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | FEC_QUIRK_HAS_VLAN | FEC_QUIRK_ERR006358 | - FEC_QUIRK_HAS_RACC | FEC_QUIRK_CLEAR_SETUP_MII, + FEC_QUIRK_HAS_RACC | FEC_QUIRK_CLEAR_SETUP_MII | + FEC_QUIRK_HAS_PMQOS, }; static const struct fec_devinfo fec_mvf600_info = { @@ -155,6 +158,13 @@ static const struct fec_devinfo fec_imx8qm_info = { FEC_QUIRK_DELAYED_CLKS_SUPPORT, }; +static const struct fec_devinfo fec_s32v234_info = { + .quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT | + FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM | + FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB | + FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE, +}; + static struct platform_device_id fec_devtype[] = { { /* keep it for coldfire */ @@ -187,6 +197,9 @@ static struct platform_device_id fec_devtype[] = { }, { .name = "imx8qm-fec", .driver_data = (kernel_ulong_t)&fec_imx8qm_info, + }, { + .name = "s32v234-fec", + .driver_data = (kernel_ulong_t)&fec_s32v234_info, }, { /* sentinel */ } @@ -203,6 +216,7 @@ enum imx_fec_type { IMX6UL_FEC, IMX8MQ_FEC, IMX8QM_FEC, + S32V234_FEC, }; static const struct of_device_id fec_dt_ids[] = { @@ -215,6 +229,7 @@ static const struct of_device_id fec_dt_ids[] = { { .compatible = "fsl,imx6ul-fec", .data = &fec_devtype[IMX6UL_FEC], }, { .compatible = "fsl,imx8mq-fec", .data = &fec_devtype[IMX8MQ_FEC], }, { .compatible = "fsl,imx8qm-fec", .data = &fec_devtype[IMX8QM_FEC], }, + { .compatible = "fsl,s32v234-fec", .data = &fec_devtype[S32V234_FEC], }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, fec_dt_ids); @@ -409,6 +424,48 @@ fec_enet_clear_csum(struct sk_buff *skb, struct net_device *ndev) return 0; } +static int +fec_enet_create_page_pool(struct fec_enet_private *fep, + struct fec_enet_priv_rx_q *rxq, int size) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = size, + .nid = dev_to_node(&fep->pdev->dev), + .dev = &fep->pdev->dev, + .dma_dir = DMA_FROM_DEVICE, + .offset = FEC_ENET_XDP_HEADROOM, + .max_len = FEC_ENET_RX_FRSIZE, + }; + int err; + + rxq->page_pool = page_pool_create(&pp_params); + if (IS_ERR(rxq->page_pool)) { + err = PTR_ERR(rxq->page_pool); + rxq->page_pool = NULL; + return err; + } + + err = xdp_rxq_info_reg(&rxq->xdp_rxq, fep->netdev, rxq->id, 0); + if (err < 0) + goto err_free_pp; + + err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, + rxq->page_pool); + if (err) + goto err_unregister_rxq; + + return 0; + +err_unregister_rxq: + xdp_rxq_info_unreg(&rxq->xdp_rxq); +err_free_pp: + page_pool_destroy(rxq->page_pool); + rxq->page_pool = NULL; + return err; +} + static struct bufdesc * fec_enet_txq_submit_frag_skb(struct fec_enet_priv_tx_q *txq, struct sk_buff *skb, @@ -1167,6 +1224,34 @@ fec_restart(struct net_device *ndev) } +static int fec_enet_ipc_handle_init(struct fec_enet_private *fep) +{ + if (!(of_machine_is_compatible("fsl,imx8qm") || + of_machine_is_compatible("fsl,imx8qxp") || + of_machine_is_compatible("fsl,imx8dxl"))) + return 0; + + return imx_scu_get_handle(&fep->ipc_handle); +} + +static void fec_enet_ipg_stop_set(struct fec_enet_private *fep, bool enabled) +{ + struct device_node *np = fep->pdev->dev.of_node; + u32 rsrc_id, val; + int idx; + + if (!np || !fep->ipc_handle) + return; + + idx = of_alias_get_id(np, "ethernet"); + if (idx < 0) + idx = 0; + rsrc_id = idx ? IMX_SC_R_ENET_1 : IMX_SC_R_ENET_0; + + val = enabled ? 1 : 0; + imx_sc_misc_set_control(fep->ipc_handle, rsrc_id, IMX_SC_C_IPG_STOP, val); +} + static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled) { struct fec_platform_data *pdata = fep->pdev->dev.platform_data; @@ -1182,6 +1267,8 @@ static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled) BIT(stop_gpr->bit), 0); } else if (pdata && pdata->sleep_mode_enable) { pdata->sleep_mode_enable(enabled); + } else { + fec_enet_ipg_stop_set(fep, enabled); } } @@ -1407,7 +1494,7 @@ static void fec_enet_tx(struct net_device *ndev) fec_enet_tx_queue(ndev, i); } -static int +static int __maybe_unused fec_enet_new_rxbdp(struct net_device *ndev, struct bufdesc *bdp, struct sk_buff *skb) { struct fec_enet_private *fep = netdev_priv(ndev); @@ -1427,8 +1514,9 @@ fec_enet_new_rxbdp(struct net_device *ndev, struct bufdesc *bdp, struct sk_buff return 0; } -static bool fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb, - struct bufdesc *bdp, u32 length, bool swap) +static bool __maybe_unused +fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb, + struct bufdesc *bdp, u32 length, bool swap) { struct fec_enet_private *fep = netdev_priv(ndev); struct sk_buff *new_skb; @@ -1453,6 +1541,21 @@ static bool fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb, return true; } +static void fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq, + struct bufdesc *bdp, int index) +{ + struct page *new_page; + dma_addr_t phys_addr; + + new_page = page_pool_dev_alloc_pages(rxq->page_pool); + WARN_ON(!new_page); + rxq->rx_skb_info[index].page = new_page; + + rxq->rx_skb_info[index].offset = FEC_ENET_XDP_HEADROOM; + phys_addr = page_pool_get_dma_addr(new_page) + FEC_ENET_XDP_HEADROOM; + bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); +} + /* During a receive, the bd_rx.cur points to the current incoming buffer. * When we update through the ring, if the next incoming buffer has * not been given to the system, we just set the empty indicator, @@ -1465,7 +1568,6 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) struct fec_enet_priv_rx_q *rxq; struct bufdesc *bdp; unsigned short status; - struct sk_buff *skb_new = NULL; struct sk_buff *skb; ushort pkt_len; __u8 *data; @@ -1474,8 +1576,8 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) bool vlan_packet_rcvd = false; u16 vlan_tag; int index = 0; - bool is_copybreak; bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; + struct page *page; #ifdef CONFIG_M532x flush_cache_all(); @@ -1527,31 +1629,25 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) ndev->stats.rx_bytes += pkt_len; index = fec_enet_get_bd_index(bdp, &rxq->bd); - skb = rxq->rx_skbuff[index]; + page = rxq->rx_skb_info[index].page; + dma_sync_single_for_cpu(&fep->pdev->dev, + fec32_to_cpu(bdp->cbd_bufaddr), + pkt_len, + DMA_FROM_DEVICE); + prefetch(page_address(page)); + fec_enet_update_cbd(rxq, bdp, index); /* The packet length includes FCS, but we don't want to * include that when passing upstream as it messes up * bridging applications. */ - is_copybreak = fec_enet_copybreak(ndev, &skb, bdp, pkt_len - 4, - need_swap); - if (!is_copybreak) { - skb_new = netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE); - if (unlikely(!skb_new)) { - ndev->stats.rx_dropped++; - goto rx_processing_done; - } - dma_unmap_single(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), - FEC_ENET_RX_FRSIZE - fep->rx_align, - DMA_FROM_DEVICE); - } - - prefetch(skb->data - NET_IP_ALIGN); + skb = build_skb(page_address(page), PAGE_SIZE); + skb_reserve(skb, FEC_ENET_XDP_HEADROOM); skb_put(skb, pkt_len - 4); + skb_mark_for_recycle(skb); data = skb->data; - if (!is_copybreak && need_swap) + if (need_swap) swap_buffer(data, pkt_len); #if !defined(CONFIG_M5272) @@ -1606,16 +1702,6 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) skb_record_rx_queue(skb, queue_id); napi_gro_receive(&fep->napi, skb); - if (is_copybreak) { - dma_sync_single_for_device(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), - FEC_ENET_RX_FRSIZE - fep->rx_align, - DMA_FROM_DEVICE); - } else { - rxq->rx_skbuff[index] = skb_new; - fec_enet_new_rxbdp(ndev, bdp, skb_new); - } - rx_processing_done: /* Clear the status flags for this buffer */ status &= ~BD_ENET_RX_STATS; @@ -2104,13 +2190,13 @@ static int fec_enet_mii_probe(struct net_device *ndev) continue; if (dev_id--) continue; - strlcpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); + strscpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); break; } if (phy_id >= PHY_MAX_ADDR) { netdev_info(ndev, "no PHY, assuming direct connection to switch\n"); - strlcpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE); + strscpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE); phy_id = 0; } @@ -2294,9 +2380,9 @@ static void fec_enet_get_drvinfo(struct net_device *ndev, { struct fec_enet_private *fep = netdev_priv(ndev); - strlcpy(info->driver, fep->pdev->dev.driver->name, + strscpy(info->driver, fep->pdev->dev.driver->name, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info)); + strscpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info)); } static int fec_enet_get_regs_len(struct net_device *ndev) @@ -2959,26 +3045,19 @@ static void fec_enet_free_buffers(struct net_device *ndev) struct fec_enet_private *fep = netdev_priv(ndev); unsigned int i; struct sk_buff *skb; - struct bufdesc *bdp; struct fec_enet_priv_tx_q *txq; struct fec_enet_priv_rx_q *rxq; unsigned int q; for (q = 0; q < fep->num_rx_queues; q++) { rxq = fep->rx_queue[q]; - bdp = rxq->bd.base; - for (i = 0; i < rxq->bd.ring_size; i++) { - skb = rxq->rx_skbuff[i]; - rxq->rx_skbuff[i] = NULL; - if (skb) { - dma_unmap_single(&fep->pdev->dev, - fec32_to_cpu(bdp->cbd_bufaddr), - FEC_ENET_RX_FRSIZE - fep->rx_align, - DMA_FROM_DEVICE); - dev_kfree_skb(skb); - } - bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); - } + for (i = 0; i < rxq->bd.ring_size; i++) + page_pool_release_page(rxq->page_pool, rxq->rx_skb_info[i].page); + + if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) + xdp_rxq_info_unreg(&rxq->xdp_rxq); + page_pool_destroy(rxq->page_pool); + rxq->page_pool = NULL; } for (q = 0; q < fep->num_tx_queues; q++) { @@ -3068,24 +3147,31 @@ static int fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue) { struct fec_enet_private *fep = netdev_priv(ndev); - unsigned int i; - struct sk_buff *skb; - struct bufdesc *bdp; struct fec_enet_priv_rx_q *rxq; + dma_addr_t phys_addr; + struct bufdesc *bdp; + struct page *page; + int i, err; rxq = fep->rx_queue[queue]; bdp = rxq->bd.base; + + err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size); + if (err < 0) { + netdev_err(ndev, "%s failed queue %d (%d)\n", __func__, queue, err); + return err; + } + for (i = 0; i < rxq->bd.ring_size; i++) { - skb = __netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE, GFP_KERNEL); - if (!skb) + page = page_pool_dev_alloc_pages(rxq->page_pool); + if (!page) goto err_alloc; - if (fec_enet_new_rxbdp(ndev, bdp, skb)) { - dev_kfree_skb(skb); - goto err_alloc; - } + phys_addr = page_pool_get_dma_addr(page) + FEC_ENET_XDP_HEADROOM; + bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); - rxq->rx_skbuff[i] = skb; + rxq->rx_skb_info[i].page = page; + rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM; bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); if (fep->bufdesc_ex) { @@ -3210,6 +3296,9 @@ fec_enet_open(struct net_device *ndev) if (fep->quirks & FEC_QUIRK_ERR006687) imx6q_cpuidle_fec_irqs_used(); + if (fep->quirks & FEC_QUIRK_HAS_PMQOS) + cpu_latency_qos_add_request(&fep->pm_qos_req, 0); + napi_enable(&fep->napi); phy_start(ndev->phydev); netif_tx_start_all_queues(ndev); @@ -3251,6 +3340,9 @@ fec_enet_close(struct net_device *ndev) fec_enet_update_ethtool_stats(ndev); fec_enet_clk_enable(ndev, false); + if (fep->quirks & FEC_QUIRK_HAS_PMQOS) + cpu_latency_qos_remove_request(&fep->pm_qos_req); + pinctrl_pm_select_sleep_state(&fep->pdev->dev); pm_runtime_mark_last_busy(&fep->pdev->dev); pm_runtime_put_autosuspend(&fep->pdev->dev); @@ -3559,7 +3651,7 @@ static int fec_enet_init(struct net_device *ndev) ndev->ethtool_ops = &fec_enet_ethtool_ops; writel(FEC_RX_DISABLED_IMASK, fep->hwp + FEC_IMASK); - netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi); if (fep->quirks & FEC_QUIRK_HAS_VLAN) /* enable hw VLAN support */ @@ -3817,6 +3909,10 @@ fec_probe(struct platform_device *pdev) !of_property_read_bool(np, "fsl,err006687-workaround-present")) fep->quirks |= FEC_QUIRK_ERR006687; + ret = fec_enet_ipc_handle_init(fep); + if (ret) + goto failed_ipc_init; + if (of_get_property(np, "fsl,magic-packet", NULL)) fep->wol_flag |= FEC_WOL_HAS_MAGIC_PACKET; @@ -4014,6 +4110,7 @@ failed_rgmii_delay: of_phy_deregister_fixed_link(np); of_node_put(phy_node); failed_stop_mode: +failed_ipc_init: failed_phy: dev_id--; failed_ioremap: @@ -4058,6 +4155,7 @@ static int __maybe_unused fec_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct fec_enet_private *fep = netdev_priv(ndev); + int ret; rtnl_lock(); if (netif_running(ndev)) { @@ -4082,6 +4180,15 @@ static int __maybe_unused fec_suspend(struct device *dev) } /* It's safe to disable clocks since interrupts are masked */ fec_enet_clk_enable(ndev, false); + + fep->rpm_active = !pm_runtime_status_suspended(dev); + if (fep->rpm_active) { + ret = pm_runtime_force_suspend(dev); + if (ret < 0) { + rtnl_unlock(); + return ret; + } + } } rtnl_unlock(); @@ -4112,6 +4219,9 @@ static int __maybe_unused fec_resume(struct device *dev) rtnl_lock(); if (netif_running(ndev)) { + if (fep->rpm_active) + pm_runtime_force_resume(dev); + ret = fec_enet_clk_enable(ndev, true); if (ret) { rtnl_unlock(); diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c index 3dc3c0b626c21c23681782729ff3b0eb2bf09d6a..cffd9ad499dda4838c4f228cba2df6ffd00076af 100644 --- a/drivers/net/ethernet/freescale/fec_ptp.c +++ b/drivers/net/ethernet/freescale/fec_ptp.c @@ -578,7 +578,7 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx) int ret; fep->ptp_caps.owner = THIS_MODULE; - strlcpy(fep->ptp_caps.name, "fec ptp", sizeof(fep->ptp_caps.name)); + strscpy(fep->ptp_caps.name, "fec ptp", sizeof(fep->ptp_caps.name)); fep->ptp_caps.max_adj = 250000000; fep->ptp_caps.n_alarm = 0; diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c index 8f0db61cb1f6b81b8e5d979ca0231c7a24ca9cd7..9d85fb136e3451ccb867ecd6b70b8922b9ee0643 100644 --- a/drivers/net/ethernet/freescale/fman/fman.c +++ b/drivers/net/ethernet/freescale/fman/fman.c @@ -1,34 +1,7 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* - * Copyright 2008-2015 Freescale Semiconductor Inc. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. * Copyright 2020 NXP - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/net/ethernet/freescale/fman/fman.h b/drivers/net/ethernet/freescale/fman/fman.h index f2ede1360f03a9cc1f312d84c87df854dc5f4847..2ea575a46675b024095d2c7520c977fabdee7246 100644 --- a/drivers/net/ethernet/freescale/fman/fman.h +++ b/drivers/net/ethernet/freescale/fman/fman.h @@ -1,34 +1,7 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* - * Copyright 2008-2015 Freescale Semiconductor Inc. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. * Copyright 2020 NXP - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __FM_H diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c index 1950a8936bc0cbe1d15acd3ac672ec25c5ebe05d..6617932fd3fdc4ba9623eb531449191f76bdfe7e 100644 --- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c +++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c @@ -1,39 +1,13 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include "fman_dtsec.h" #include "fman.h" +#include "mac.h" #include #include @@ -327,7 +301,7 @@ struct fman_mac { /* Ethernet physical interface */ phy_interface_t phy_if; u16 max_speed; - void *dev_id; /* device cookie used by the exception cbs */ + struct mac_device *dev_id; /* device cookie used by the exception cbs */ fman_mac_exception_cb *exception_cb; fman_mac_exception_cb *event_cb; /* Number of individual addresses in registers for this station */ @@ -840,73 +814,45 @@ static void free_init_resources(struct fman_mac *dtsec) dtsec->unicast_addr_hash = NULL; } -int dtsec_cfg_max_frame_len(struct fman_mac *dtsec, u16 new_val) -{ - if (is_init_done(dtsec->dtsec_drv_param)) - return -EINVAL; - - dtsec->dtsec_drv_param->maximum_frame = new_val; - - return 0; -} - -int dtsec_cfg_pad_and_crc(struct fman_mac *dtsec, bool new_val) -{ - if (is_init_done(dtsec->dtsec_drv_param)) - return -EINVAL; - - dtsec->dtsec_drv_param->tx_pad_crc = new_val; - - return 0; -} - -static void graceful_start(struct fman_mac *dtsec, enum comm_mode mode) +static void graceful_start(struct fman_mac *dtsec) { struct dtsec_regs __iomem *regs = dtsec->regs; - if (mode & COMM_MODE_TX) - iowrite32be(ioread32be(®s->tctrl) & - ~TCTRL_GTS, ®s->tctrl); - if (mode & COMM_MODE_RX) - iowrite32be(ioread32be(®s->rctrl) & - ~RCTRL_GRS, ®s->rctrl); + iowrite32be(ioread32be(®s->tctrl) & ~TCTRL_GTS, ®s->tctrl); + iowrite32be(ioread32be(®s->rctrl) & ~RCTRL_GRS, ®s->rctrl); } -static void graceful_stop(struct fman_mac *dtsec, enum comm_mode mode) +static void graceful_stop(struct fman_mac *dtsec) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 tmp; /* Graceful stop - Assert the graceful Rx stop bit */ - if (mode & COMM_MODE_RX) { - tmp = ioread32be(®s->rctrl) | RCTRL_GRS; - iowrite32be(tmp, ®s->rctrl); + tmp = ioread32be(®s->rctrl) | RCTRL_GRS; + iowrite32be(tmp, ®s->rctrl); - if (dtsec->fm_rev_info.major == 2) { - /* Workaround for dTSEC Errata A002 */ - usleep_range(100, 200); - } else { - /* Workaround for dTSEC Errata A004839 */ - usleep_range(10, 50); - } + if (dtsec->fm_rev_info.major == 2) { + /* Workaround for dTSEC Errata A002 */ + usleep_range(100, 200); + } else { + /* Workaround for dTSEC Errata A004839 */ + usleep_range(10, 50); } /* Graceful stop - Assert the graceful Tx stop bit */ - if (mode & COMM_MODE_TX) { - if (dtsec->fm_rev_info.major == 2) { - /* dTSEC Errata A004: Do not use TCTRL[GTS]=1 */ - pr_debug("GTS not supported due to DTSEC_A004 Errata.\n"); - } else { - tmp = ioread32be(®s->tctrl) | TCTRL_GTS; - iowrite32be(tmp, ®s->tctrl); + if (dtsec->fm_rev_info.major == 2) { + /* dTSEC Errata A004: Do not use TCTRL[GTS]=1 */ + pr_debug("GTS not supported due to DTSEC_A004 Errata.\n"); + } else { + tmp = ioread32be(®s->tctrl) | TCTRL_GTS; + iowrite32be(tmp, ®s->tctrl); - /* Workaround for dTSEC Errata A0012, A0014 */ - usleep_range(10, 50); - } + /* Workaround for dTSEC Errata A0012, A0014 */ + usleep_range(10, 50); } } -int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode) +static int dtsec_enable(struct fman_mac *dtsec) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 tmp; @@ -916,58 +862,42 @@ int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode) /* Enable */ tmp = ioread32be(®s->maccfg1); - if (mode & COMM_MODE_RX) - tmp |= MACCFG1_RX_EN; - if (mode & COMM_MODE_TX) - tmp |= MACCFG1_TX_EN; - + tmp |= MACCFG1_RX_EN | MACCFG1_TX_EN; iowrite32be(tmp, ®s->maccfg1); /* Graceful start - clear the graceful Rx/Tx stop bit */ - graceful_start(dtsec, mode); + graceful_start(dtsec); return 0; } -int dtsec_disable(struct fman_mac *dtsec, enum comm_mode mode) +static void dtsec_disable(struct fman_mac *dtsec) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 tmp; - if (!is_init_done(dtsec->dtsec_drv_param)) - return -EINVAL; + WARN_ON_ONCE(!is_init_done(dtsec->dtsec_drv_param)); /* Graceful stop - Assert the graceful Rx/Tx stop bit */ - graceful_stop(dtsec, mode); + graceful_stop(dtsec); tmp = ioread32be(®s->maccfg1); - if (mode & COMM_MODE_RX) - tmp &= ~MACCFG1_RX_EN; - if (mode & COMM_MODE_TX) - tmp &= ~MACCFG1_TX_EN; - + tmp &= ~(MACCFG1_RX_EN | MACCFG1_TX_EN); iowrite32be(tmp, ®s->maccfg1); - - return 0; } -int dtsec_set_tx_pause_frames(struct fman_mac *dtsec, - u8 __maybe_unused priority, - u16 pause_time, u16 __maybe_unused thresh_time) +static int dtsec_set_tx_pause_frames(struct fman_mac *dtsec, + u8 __maybe_unused priority, + u16 pause_time, + u16 __maybe_unused thresh_time) { struct dtsec_regs __iomem *regs = dtsec->regs; - enum comm_mode mode = COMM_MODE_NONE; u32 ptv = 0; if (!is_init_done(dtsec->dtsec_drv_param)) return -EINVAL; - if ((ioread32be(®s->rctrl) & RCTRL_GRS) == 0) - mode |= COMM_MODE_RX; - if ((ioread32be(®s->tctrl) & TCTRL_GTS) == 0) - mode |= COMM_MODE_TX; - - graceful_stop(dtsec, mode); + graceful_stop(dtsec); if (pause_time) { /* FM_BAD_TX_TS_IN_B_2_B_ERRATA_DTSEC_A003 Errata workaround */ @@ -989,26 +919,20 @@ int dtsec_set_tx_pause_frames(struct fman_mac *dtsec, iowrite32be(ioread32be(®s->maccfg1) & ~MACCFG1_TX_FLOW, ®s->maccfg1); - graceful_start(dtsec, mode); + graceful_start(dtsec); return 0; } -int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en) +static int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en) { struct dtsec_regs __iomem *regs = dtsec->regs; - enum comm_mode mode = COMM_MODE_NONE; u32 tmp; if (!is_init_done(dtsec->dtsec_drv_param)) return -EINVAL; - if ((ioread32be(®s->rctrl) & RCTRL_GRS) == 0) - mode |= COMM_MODE_RX; - if ((ioread32be(®s->tctrl) & TCTRL_GTS) == 0) - mode |= COMM_MODE_TX; - - graceful_stop(dtsec, mode); + graceful_stop(dtsec); tmp = ioread32be(®s->maccfg1); if (en) @@ -1017,25 +941,18 @@ int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en) tmp &= ~MACCFG1_RX_FLOW; iowrite32be(tmp, ®s->maccfg1); - graceful_start(dtsec, mode); + graceful_start(dtsec); return 0; } -int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_addr) +static int dtsec_modify_mac_address(struct fman_mac *dtsec, + const enet_addr_t *enet_addr) { - struct dtsec_regs __iomem *regs = dtsec->regs; - enum comm_mode mode = COMM_MODE_NONE; - if (!is_init_done(dtsec->dtsec_drv_param)) return -EINVAL; - if ((ioread32be(®s->rctrl) & RCTRL_GRS) == 0) - mode |= COMM_MODE_RX; - if ((ioread32be(®s->tctrl) & TCTRL_GTS) == 0) - mode |= COMM_MODE_TX; - - graceful_stop(dtsec, mode); + graceful_stop(dtsec); /* Initialize MAC Station Address registers (1 & 2) * Station address have to be swapped (big endian to little endian @@ -1043,12 +960,13 @@ int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_add dtsec->addr = ENET_ADDR_TO_UINT64(*enet_addr); set_mac_address(dtsec->regs, (const u8 *)(*enet_addr)); - graceful_start(dtsec, mode); + graceful_start(dtsec); return 0; } -int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr) +static int dtsec_add_hash_mac_address(struct fman_mac *dtsec, + enet_addr_t *eth_addr) { struct dtsec_regs __iomem *regs = dtsec->regs; struct eth_hash_entry *hash_entry; @@ -1114,7 +1032,7 @@ int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr) return 0; } -int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable) +static int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable) { u32 tmp; struct dtsec_regs __iomem *regs = dtsec->regs; @@ -1133,7 +1051,7 @@ int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable) return 0; } -int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable) +static int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 rctrl, tctrl; @@ -1158,7 +1076,8 @@ int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable) return 0; } -int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr) +static int dtsec_del_hash_mac_address(struct fman_mac *dtsec, + enet_addr_t *eth_addr) { struct dtsec_regs __iomem *regs = dtsec->regs; struct list_head *pos; @@ -1229,7 +1148,7 @@ int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr) return 0; } -int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val) +static int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 tmp; @@ -1258,21 +1177,15 @@ int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val) return 0; } -int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed) +static int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed) { struct dtsec_regs __iomem *regs = dtsec->regs; - enum comm_mode mode = COMM_MODE_NONE; u32 tmp; if (!is_init_done(dtsec->dtsec_drv_param)) return -EINVAL; - if ((ioread32be(®s->rctrl) & RCTRL_GRS) == 0) - mode |= COMM_MODE_RX; - if ((ioread32be(®s->tctrl) & TCTRL_GTS) == 0) - mode |= COMM_MODE_TX; - - graceful_stop(dtsec, mode); + graceful_stop(dtsec); tmp = ioread32be(®s->maccfg2); @@ -1293,12 +1206,12 @@ int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed) tmp &= ~DTSEC_ECNTRL_R100M; iowrite32be(tmp, ®s->ecntrl); - graceful_start(dtsec, mode); + graceful_start(dtsec); return 0; } -int dtsec_restart_autoneg(struct fman_mac *dtsec) +static int dtsec_restart_autoneg(struct fman_mac *dtsec) { u16 tmp_reg16; @@ -1316,20 +1229,31 @@ int dtsec_restart_autoneg(struct fman_mac *dtsec) return 0; } -int dtsec_get_version(struct fman_mac *dtsec, u32 *mac_version) +static void adjust_link_dtsec(struct mac_device *mac_dev) { - struct dtsec_regs __iomem *regs = dtsec->regs; + struct phy_device *phy_dev = mac_dev->phy_dev; + struct fman_mac *fman_mac; + bool rx_pause, tx_pause; + int err; - if (!is_init_done(dtsec->dtsec_drv_param)) - return -EINVAL; + fman_mac = mac_dev->fman_mac; + if (!phy_dev->link) { + dtsec_restart_autoneg(fman_mac); - *mac_version = ioread32be(®s->tsec_id); + return; + } - return 0; + dtsec_adjust_link(fman_mac, phy_dev->speed); + mac_dev->update_speed(mac_dev, phy_dev->speed); + fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause); + err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause); + if (err < 0) + dev_err(mac_dev->dev, "fman_set_mac_active_pause() = %d\n", + err); } -int dtsec_set_exception(struct fman_mac *dtsec, - enum fman_mac_exceptions exception, bool enable) +static int dtsec_set_exception(struct fman_mac *dtsec, + enum fman_mac_exceptions exception, bool enable) { struct dtsec_regs __iomem *regs = dtsec->regs; u32 bit_mask = 0; @@ -1382,7 +1306,7 @@ int dtsec_set_exception(struct fman_mac *dtsec, return 0; } -int dtsec_init(struct fman_mac *dtsec) +static int dtsec_init(struct fman_mac *dtsec) { struct dtsec_regs __iomem *regs = dtsec->regs; struct dtsec_cfg *dtsec_drv_param; @@ -1476,7 +1400,7 @@ int dtsec_init(struct fman_mac *dtsec) return 0; } -int dtsec_free(struct fman_mac *dtsec) +static int dtsec_free(struct fman_mac *dtsec) { free_init_resources(dtsec); @@ -1487,13 +1411,11 @@ int dtsec_free(struct fman_mac *dtsec) return 0; } -struct fman_mac *dtsec_config(struct fman_mac_params *params) +static struct fman_mac *dtsec_config(struct mac_device *mac_dev, + struct fman_mac_params *params) { struct fman_mac *dtsec; struct dtsec_cfg *dtsec_drv_param; - void __iomem *base_addr; - - base_addr = params->base_addr; /* allocate memory for the UCC GETH data structure. */ dtsec = kzalloc(sizeof(*dtsec), GFP_KERNEL); @@ -1510,10 +1432,10 @@ struct fman_mac *dtsec_config(struct fman_mac_params *params) set_dflts(dtsec_drv_param); - dtsec->regs = base_addr; - dtsec->addr = ENET_ADDR_TO_UINT64(params->addr); + dtsec->regs = mac_dev->vaddr; + dtsec->addr = ENET_ADDR_TO_UINT64(mac_dev->addr); dtsec->max_speed = params->max_speed; - dtsec->phy_if = params->phy_if; + dtsec->phy_if = mac_dev->phy_if; dtsec->mac_id = params->mac_id; dtsec->exceptions = (DTSEC_IMASK_BREN | DTSEC_IMASK_RXCEN | @@ -1530,34 +1452,87 @@ struct fman_mac *dtsec_config(struct fman_mac_params *params) DTSEC_IMASK_RDPEEN); dtsec->exception_cb = params->exception_cb; dtsec->event_cb = params->event_cb; - dtsec->dev_id = params->dev_id; + dtsec->dev_id = mac_dev; dtsec->ptp_tsu_enabled = dtsec->dtsec_drv_param->ptp_tsu_en; dtsec->en_tsu_err_exception = dtsec->dtsec_drv_param->ptp_exception_en; dtsec->fm = params->fm; dtsec->basex_if = params->basex_if; - if (!params->internal_phy_node) { + /* Save FMan revision */ + fman_get_revision(dtsec->fm, &dtsec->fm_rev_info); + + return dtsec; + +err_dtsec: + kfree(dtsec); + return NULL; +} + +int dtsec_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params) +{ + int err; + struct fman_mac *dtsec; + struct device_node *phy_node; + + mac_dev->set_promisc = dtsec_set_promiscuous; + mac_dev->change_addr = dtsec_modify_mac_address; + mac_dev->add_hash_mac_addr = dtsec_add_hash_mac_address; + mac_dev->remove_hash_mac_addr = dtsec_del_hash_mac_address; + mac_dev->set_tx_pause = dtsec_set_tx_pause_frames; + mac_dev->set_rx_pause = dtsec_accept_rx_pause_frames; + mac_dev->set_exception = dtsec_set_exception; + mac_dev->set_allmulti = dtsec_set_allmulti; + mac_dev->set_tstamp = dtsec_set_tstamp; + mac_dev->set_multi = fman_set_multi; + mac_dev->adjust_link = adjust_link_dtsec; + mac_dev->enable = dtsec_enable; + mac_dev->disable = dtsec_disable; + + mac_dev->fman_mac = dtsec_config(mac_dev, params); + if (!mac_dev->fman_mac) { + err = -EINVAL; + goto _return; + } + + dtsec = mac_dev->fman_mac; + dtsec->dtsec_drv_param->maximum_frame = fman_get_max_frm(); + dtsec->dtsec_drv_param->tx_pad_crc = true; + + phy_node = of_parse_phandle(mac_node, "tbi-handle", 0); + if (!phy_node) { pr_err("TBI PHY node is not available\n"); - goto err_dtsec_drv_param; + err = -EINVAL; + goto _return_fm_mac_free; } - dtsec->tbiphy = of_phy_find_device(params->internal_phy_node); + dtsec->tbiphy = of_phy_find_device(phy_node); if (!dtsec->tbiphy) { pr_err("of_phy_find_device (TBI PHY) failed\n"); - goto err_dtsec_drv_param; + err = -EINVAL; + goto _return_fm_mac_free; } - put_device(&dtsec->tbiphy->mdio.dev); - /* Save FMan revision */ - fman_get_revision(dtsec->fm, &dtsec->fm_rev_info); + err = dtsec_init(dtsec); + if (err < 0) + goto _return_fm_mac_free; - return dtsec; + /* For 1G MAC, disable by default the MIB counters overflow interrupt */ + err = dtsec_set_exception(dtsec, FM_MAC_EX_1G_RX_MIB_CNT_OVFL, false); + if (err < 0) + goto _return_fm_mac_free; -err_dtsec_drv_param: - kfree(dtsec_drv_param); -err_dtsec: - kfree(dtsec); - return NULL; + dev_info(mac_dev->dev, "FMan dTSEC version: 0x%08x\n", + ioread32be(&dtsec->regs->tsec_id)); + + goto _return; + +_return_fm_mac_free: + dtsec_free(dtsec); + +_return: + return err; } diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.h b/drivers/net/ethernet/freescale/fman/fman_dtsec.h index 68512c3bd6e528323be9423f793af31d09ac07ca..8c72d280c51a5e8ea39d4f6181dfd6a2f0fb91d7 100644 --- a/drivers/net/ethernet/freescale/fman/fman_dtsec.h +++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #ifndef __DTSEC_H @@ -35,27 +8,10 @@ #include "fman_mac.h" -struct fman_mac *dtsec_config(struct fman_mac_params *params); -int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val); -int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_addr); -int dtsec_adjust_link(struct fman_mac *dtsec, - u16 speed); -int dtsec_restart_autoneg(struct fman_mac *dtsec); -int dtsec_cfg_max_frame_len(struct fman_mac *dtsec, u16 new_val); -int dtsec_cfg_pad_and_crc(struct fman_mac *dtsec, bool new_val); -int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode); -int dtsec_disable(struct fman_mac *dtsec, enum comm_mode mode); -int dtsec_init(struct fman_mac *dtsec); -int dtsec_free(struct fman_mac *dtsec); -int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en); -int dtsec_set_tx_pause_frames(struct fman_mac *dtsec, u8 priority, - u16 pause_time, u16 thresh_time); -int dtsec_set_exception(struct fman_mac *dtsec, - enum fman_mac_exceptions exception, bool enable); -int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr); -int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr); -int dtsec_get_version(struct fman_mac *dtsec, u32 *mac_version); -int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable); -int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable); +struct mac_device; + +int dtsec_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params); #endif /* __DTSEC_H */ diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.c b/drivers/net/ethernet/freescale/fman/fman_keygen.c index e1bdfed1613448d19a2b1142bd1777de95fb6d09..e73f6ef3c6ee78115a8827d97ac1910d76bf5a40 100644 --- a/drivers/net/ethernet/freescale/fman/fman_keygen.c +++ b/drivers/net/ethernet/freescale/fman/fman_keygen.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* * Copyright 2017 NXP - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of NXP nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.h b/drivers/net/ethernet/freescale/fman/fman_keygen.h index c4640de3f4cb52013db8a8eaeb9943c576187574..2cb0df45307460ef5abb09415c5089f478988271 100644 --- a/drivers/net/ethernet/freescale/fman/fman_keygen.h +++ b/drivers/net/ethernet/freescale/fman/fman_keygen.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* * Copyright 2017 NXP - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of NXP nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __KEYGEN_H diff --git a/drivers/net/ethernet/freescale/fman/fman_mac.h b/drivers/net/ethernet/freescale/fman/fman_mac.h index 19f327efdaff3e62b5ae484fa372663f3326a965..65887a3160d7fc90f130b80c61b9f75b27e76462 100644 --- a/drivers/net/ethernet/freescale/fman/fman_mac.h +++ b/drivers/net/ethernet/freescale/fman/fman_mac.h @@ -41,6 +41,7 @@ #include struct fman_mac; +struct mac_device; /* Ethernet Address */ typedef u8 enet_addr_t[ETH_ALEN]; @@ -75,16 +76,6 @@ typedef u8 enet_addr_t[ETH_ALEN]; #define ETH_HASH_ENTRY_OBJ(ptr) \ hlist_entry_safe(ptr, struct eth_hash_entry, node) -/* Enumeration (bit flags) of communication modes (Transmit, - * receive or both). - */ -enum comm_mode { - COMM_MODE_NONE = 0, /* No transmit/receive communication */ - COMM_MODE_RX = 1, /* Only receive communication */ - COMM_MODE_TX = 2, /* Only transmit communication */ - COMM_MODE_RX_AND_TX = 3 /* Both transmit and receive communication */ -}; - /* FM MAC Exceptions */ enum fman_mac_exceptions { FM_MAC_EX_10G_MDIO_SCAN_EVENT = 0 @@ -168,30 +159,23 @@ struct eth_hash_entry { struct list_head node; }; -typedef void (fman_mac_exception_cb)(void *dev_id, - enum fman_mac_exceptions exceptions); +typedef void (fman_mac_exception_cb)(struct mac_device *dev_id, + enum fman_mac_exceptions exceptions); /* FMan MAC config input */ struct fman_mac_params { - /* Base of memory mapped FM MAC registers */ - void __iomem *base_addr; - /* MAC address of device; First octet is sent first */ - enet_addr_t addr; /* MAC ID; numbering of dTSEC and 1G-mEMAC: * 0 - FM_MAX_NUM_OF_1G_MACS; * numbering of 10G-MAC (TGEC) and 10G-mEMAC: * 0 - FM_MAX_NUM_OF_10G_MACS */ u8 mac_id; - /* PHY interface */ - phy_interface_t phy_if; /* Note that the speed should indicate the maximum rate that * this MAC should support rather than the actual speed; */ u16 max_speed; /* A handle to the FM object this port related to */ void *fm; - void *dev_id; /* device cookie used by the exception cbs */ fman_mac_exception_cb *event_cb; /* MDIO Events Callback Routine */ fman_mac_exception_cb *exception_cb;/* Exception Callback Routine */ /* SGMII/QSGII interface with 1000BaseX auto-negotiation between MAC @@ -200,8 +184,6 @@ struct fman_mac_params { * synchronize with far-end phy at 10Mbps, 100Mbps or 1000Mbps */ bool basex_if; - /* Pointer to TBI/PCS PHY node, used for TBI/PCS PHY access */ - struct device_node *internal_phy_node; }; struct eth_hash_t { diff --git a/drivers/net/ethernet/freescale/fman/fman_memac.c b/drivers/net/ethernet/freescale/fman/fman_memac.c index 2216b7f51d26eff67a39d16f11219b99ebe0b0c6..32d26cf178437d7a0b7e6577276cf0a62b54a5f4 100644 --- a/drivers/net/ethernet/freescale/fman/fman_memac.c +++ b/drivers/net/ethernet/freescale/fman/fman_memac.c @@ -1,39 +1,13 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include "fman_memac.h" #include "fman.h" +#include "mac.h" #include #include @@ -337,7 +311,7 @@ struct fman_mac { /* Ethernet physical interface */ phy_interface_t phy_if; u16 max_speed; - void *dev_id; /* device cookie used by the exception cbs */ + struct mac_device *dev_id; /* device cookie used by the exception cbs */ fman_mac_exception_cb *exception_cb; fman_mac_exception_cb *event_cb; /* Pointer to driver's global address hash table */ @@ -712,7 +686,7 @@ static bool is_init_done(struct memac_cfg *memac_drv_params) return false; } -int memac_enable(struct fman_mac *memac, enum comm_mode mode) +static int memac_enable(struct fman_mac *memac) { struct memac_regs __iomem *regs = memac->regs; u32 tmp; @@ -721,36 +695,26 @@ int memac_enable(struct fman_mac *memac, enum comm_mode mode) return -EINVAL; tmp = ioread32be(®s->command_config); - if (mode & COMM_MODE_RX) - tmp |= CMD_CFG_RX_EN; - if (mode & COMM_MODE_TX) - tmp |= CMD_CFG_TX_EN; - + tmp |= CMD_CFG_RX_EN | CMD_CFG_TX_EN; iowrite32be(tmp, ®s->command_config); return 0; } -int memac_disable(struct fman_mac *memac, enum comm_mode mode) +static void memac_disable(struct fman_mac *memac) + { struct memac_regs __iomem *regs = memac->regs; u32 tmp; - if (!is_init_done(memac->memac_drv_param)) - return -EINVAL; + WARN_ON_ONCE(!is_init_done(memac->memac_drv_param)); tmp = ioread32be(®s->command_config); - if (mode & COMM_MODE_RX) - tmp &= ~CMD_CFG_RX_EN; - if (mode & COMM_MODE_TX) - tmp &= ~CMD_CFG_TX_EN; - + tmp &= ~(CMD_CFG_RX_EN | CMD_CFG_TX_EN); iowrite32be(tmp, ®s->command_config); - - return 0; } -int memac_set_promiscuous(struct fman_mac *memac, bool new_val) +static int memac_set_promiscuous(struct fman_mac *memac, bool new_val) { struct memac_regs __iomem *regs = memac->regs; u32 tmp; @@ -769,7 +733,7 @@ int memac_set_promiscuous(struct fman_mac *memac, bool new_val) return 0; } -int memac_adjust_link(struct fman_mac *memac, u16 speed) +static int memac_adjust_link(struct fman_mac *memac, u16 speed) { struct memac_regs __iomem *regs = memac->regs; u32 tmp; @@ -809,39 +773,26 @@ int memac_adjust_link(struct fman_mac *memac, u16 speed) return 0; } -int memac_cfg_max_frame_len(struct fman_mac *memac, u16 new_val) -{ - if (is_init_done(memac->memac_drv_param)) - return -EINVAL; - - memac->memac_drv_param->max_frame_length = new_val; - - return 0; -} - -int memac_cfg_reset_on_init(struct fman_mac *memac, bool enable) -{ - if (is_init_done(memac->memac_drv_param)) - return -EINVAL; - - memac->memac_drv_param->reset_on_init = enable; - - return 0; -} - -int memac_cfg_fixed_link(struct fman_mac *memac, - struct fixed_phy_status *fixed_link) +static void adjust_link_memac(struct mac_device *mac_dev) { - if (is_init_done(memac->memac_drv_param)) - return -EINVAL; + struct phy_device *phy_dev = mac_dev->phy_dev; + struct fman_mac *fman_mac; + bool rx_pause, tx_pause; + int err; - memac->memac_drv_param->fixed_link = fixed_link; + fman_mac = mac_dev->fman_mac; + memac_adjust_link(fman_mac, phy_dev->speed); + mac_dev->update_speed(mac_dev, phy_dev->speed); - return 0; + fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause); + err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause); + if (err < 0) + dev_err(mac_dev->dev, "fman_set_mac_active_pause() = %d\n", + err); } -int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority, - u16 pause_time, u16 thresh_time) +static int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority, + u16 pause_time, u16 thresh_time) { struct memac_regs __iomem *regs = memac->regs; u32 tmp; @@ -878,7 +829,7 @@ int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority, return 0; } -int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en) +static int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en) { struct memac_regs __iomem *regs = memac->regs; u32 tmp; @@ -897,7 +848,8 @@ int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en) return 0; } -int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_addr) +static int memac_modify_mac_address(struct fman_mac *memac, + const enet_addr_t *enet_addr) { if (!is_init_done(memac->memac_drv_param)) return -EINVAL; @@ -907,7 +859,8 @@ int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_add return 0; } -int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr) +static int memac_add_hash_mac_address(struct fman_mac *memac, + enet_addr_t *eth_addr) { struct memac_regs __iomem *regs = memac->regs; struct eth_hash_entry *hash_entry; @@ -940,7 +893,7 @@ int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr) return 0; } -int memac_set_allmulti(struct fman_mac *memac, bool enable) +static int memac_set_allmulti(struct fman_mac *memac, bool enable) { u32 entry; struct memac_regs __iomem *regs = memac->regs; @@ -963,12 +916,13 @@ int memac_set_allmulti(struct fman_mac *memac, bool enable) return 0; } -int memac_set_tstamp(struct fman_mac *memac, bool enable) +static int memac_set_tstamp(struct fman_mac *memac, bool enable) { return 0; /* Always enabled. */ } -int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr) +static int memac_del_hash_mac_address(struct fman_mac *memac, + enet_addr_t *eth_addr) { struct memac_regs __iomem *regs = memac->regs; struct eth_hash_entry *hash_entry = NULL; @@ -1001,8 +955,8 @@ int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr) return 0; } -int memac_set_exception(struct fman_mac *memac, - enum fman_mac_exceptions exception, bool enable) +static int memac_set_exception(struct fman_mac *memac, + enum fman_mac_exceptions exception, bool enable) { u32 bit_mask = 0; @@ -1024,13 +978,13 @@ int memac_set_exception(struct fman_mac *memac, return 0; } -int memac_init(struct fman_mac *memac) +static int memac_init(struct fman_mac *memac) { struct memac_cfg *memac_drv_param; u8 i; enet_addr_t eth_addr; bool slow_10g_if = false; - struct fixed_phy_status *fixed_link; + struct fixed_phy_status *fixed_link = NULL; int err; u32 reg32 = 0; @@ -1141,7 +1095,7 @@ int memac_init(struct fman_mac *memac) return 0; } -int memac_free(struct fman_mac *memac) +static int memac_free(struct fman_mac *memac) { free_init_resources(memac); @@ -1154,13 +1108,12 @@ int memac_free(struct fman_mac *memac) return 0; } -struct fman_mac *memac_config(struct fman_mac_params *params) +static struct fman_mac *memac_config(struct mac_device *mac_dev, + struct fman_mac_params *params) { struct fman_mac *memac; struct memac_cfg *memac_drv_param; - void __iomem *base_addr; - base_addr = params->base_addr; /* allocate memory for the m_emac data structure */ memac = kzalloc(sizeof(*memac), GFP_KERNEL); if (!memac) @@ -1178,38 +1131,121 @@ struct fman_mac *memac_config(struct fman_mac_params *params) set_dflts(memac_drv_param); - memac->addr = ENET_ADDR_TO_UINT64(params->addr); + memac->addr = ENET_ADDR_TO_UINT64(mac_dev->addr); - memac->regs = base_addr; + memac->regs = mac_dev->vaddr; memac->max_speed = params->max_speed; - memac->phy_if = params->phy_if; + memac->phy_if = mac_dev->phy_if; memac->mac_id = params->mac_id; memac->exceptions = (MEMAC_IMASK_TSECC_ER | MEMAC_IMASK_TECC_ER | MEMAC_IMASK_RECC_ER | MEMAC_IMASK_MGI); memac->exception_cb = params->exception_cb; memac->event_cb = params->event_cb; - memac->dev_id = params->dev_id; + memac->dev_id = mac_dev; memac->fm = params->fm; memac->basex_if = params->basex_if; /* Save FMan revision */ fman_get_revision(memac->fm, &memac->fm_rev_info); + return memac; +} + +int memac_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params) +{ + int err; + struct device_node *phy_node; + struct fixed_phy_status *fixed_link; + struct fman_mac *memac; + + mac_dev->set_promisc = memac_set_promiscuous; + mac_dev->change_addr = memac_modify_mac_address; + mac_dev->add_hash_mac_addr = memac_add_hash_mac_address; + mac_dev->remove_hash_mac_addr = memac_del_hash_mac_address; + mac_dev->set_tx_pause = memac_set_tx_pause_frames; + mac_dev->set_rx_pause = memac_accept_rx_pause_frames; + mac_dev->set_exception = memac_set_exception; + mac_dev->set_allmulti = memac_set_allmulti; + mac_dev->set_tstamp = memac_set_tstamp; + mac_dev->set_multi = fman_set_multi; + mac_dev->adjust_link = adjust_link_memac; + mac_dev->enable = memac_enable; + mac_dev->disable = memac_disable; + + if (params->max_speed == SPEED_10000) + mac_dev->phy_if = PHY_INTERFACE_MODE_XGMII; + + mac_dev->fman_mac = memac_config(mac_dev, params); + if (!mac_dev->fman_mac) { + err = -EINVAL; + goto _return; + } + + memac = mac_dev->fman_mac; + memac->memac_drv_param->max_frame_length = fman_get_max_frm(); + memac->memac_drv_param->reset_on_init = true; if (memac->phy_if == PHY_INTERFACE_MODE_SGMII || memac->phy_if == PHY_INTERFACE_MODE_QSGMII) { - if (!params->internal_phy_node) { + phy_node = of_parse_phandle(mac_node, "pcsphy-handle", 0); + if (!phy_node) { pr_err("PCS PHY node is not available\n"); - memac_free(memac); - return NULL; + err = -EINVAL; + goto _return_fm_mac_free; } - memac->pcsphy = of_phy_find_device(params->internal_phy_node); + memac->pcsphy = of_phy_find_device(phy_node); if (!memac->pcsphy) { pr_err("of_phy_find_device (PCS PHY) failed\n"); - memac_free(memac); - return NULL; + err = -EINVAL; + goto _return_fm_mac_free; } } - return memac; + if (!mac_dev->phy_node && of_phy_is_fixed_link(mac_node)) { + struct phy_device *phy; + + err = of_phy_register_fixed_link(mac_node); + if (err) + goto _return_fm_mac_free; + + fixed_link = kzalloc(sizeof(*fixed_link), GFP_KERNEL); + if (!fixed_link) { + err = -ENOMEM; + goto _return_fm_mac_free; + } + + mac_dev->phy_node = of_node_get(mac_node); + phy = of_phy_find_device(mac_dev->phy_node); + if (!phy) { + err = -EINVAL; + of_node_put(mac_dev->phy_node); + goto _return_fixed_link_free; + } + + fixed_link->link = phy->link; + fixed_link->speed = phy->speed; + fixed_link->duplex = phy->duplex; + fixed_link->pause = phy->pause; + fixed_link->asym_pause = phy->asym_pause; + + put_device(&phy->mdio.dev); + memac->memac_drv_param->fixed_link = fixed_link; + } + + err = memac_init(mac_dev->fman_mac); + if (err < 0) + goto _return_fixed_link_free; + + dev_info(mac_dev->dev, "FMan MEMAC\n"); + + goto _return; + +_return_fixed_link_free: + kfree(fixed_link); +_return_fm_mac_free: + memac_free(mac_dev->fman_mac); +_return: + return err; } diff --git a/drivers/net/ethernet/freescale/fman/fman_memac.h b/drivers/net/ethernet/freescale/fman/fman_memac.h index 3820f7a229834d18c6acc02639d8b7d9dc06c2a4..5a3a14f9684f8a1325a3b567f93b2b73beec926d 100644 --- a/drivers/net/ethernet/freescale/fman/fman_memac.h +++ b/drivers/net/ethernet/freescale/fman/fman_memac.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #ifndef __MEMAC_H @@ -38,26 +11,10 @@ #include #include -struct fman_mac *memac_config(struct fman_mac_params *params); -int memac_set_promiscuous(struct fman_mac *memac, bool new_val); -int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_addr); -int memac_adjust_link(struct fman_mac *memac, u16 speed); -int memac_cfg_max_frame_len(struct fman_mac *memac, u16 new_val); -int memac_cfg_reset_on_init(struct fman_mac *memac, bool enable); -int memac_cfg_fixed_link(struct fman_mac *memac, - struct fixed_phy_status *fixed_link); -int memac_enable(struct fman_mac *memac, enum comm_mode mode); -int memac_disable(struct fman_mac *memac, enum comm_mode mode); -int memac_init(struct fman_mac *memac); -int memac_free(struct fman_mac *memac); -int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en); -int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority, - u16 pause_time, u16 thresh_time); -int memac_set_exception(struct fman_mac *memac, - enum fman_mac_exceptions exception, bool enable); -int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr); -int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr); -int memac_set_allmulti(struct fman_mac *memac, bool enable); -int memac_set_tstamp(struct fman_mac *memac, bool enable); +struct mac_device; + +int memac_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params); #endif /* __MEMAC_H */ diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.c b/drivers/net/ethernet/freescale/fman/fman_muram.c index 7ad317e622bcea7af5a9c518f6ea82237155ff53..f557d68e5b7601db914a2a4e329b4ce45bef90f9 100644 --- a/drivers/net/ethernet/freescale/fman/fman_muram.c +++ b/drivers/net/ethernet/freescale/fman/fman_muram.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #include "fman_muram.h" diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.h b/drivers/net/ethernet/freescale/fman/fman_muram.h index 453bf849eee10de81bc42184a9f3c74644e3e44a..3643af61bae26c9f275bf874871784559dc29783 100644 --- a/drivers/net/ethernet/freescale/fman/fman_muram.h +++ b/drivers/net/ethernet/freescale/fman/fman_muram.h @@ -1,34 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ + #ifndef __FM_MURAM_EXT #define __FM_MURAM_EXT diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c index 4c9d05c45c033b70712b373f3888a288d71d35fc..ab90fe2bee5e38ab0b61294017f4b7e35e187d32 100644 --- a/drivers/net/ethernet/freescale/fman/fman_port.c +++ b/drivers/net/ethernet/freescale/fman/fman_port.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* * Copyright 2008 - 2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt diff --git a/drivers/net/ethernet/freescale/fman/fman_port.h b/drivers/net/ethernet/freescale/fman/fman_port.h index 82f12661a46dae892e7d374c54b8a6a56d8b2207..4917fe8f06173a47ac13a6e2b244928f6b865ada 100644 --- a/drivers/net/ethernet/freescale/fman/fman_port.h +++ b/drivers/net/ethernet/freescale/fman/fman_port.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* * Copyright 2008 - 2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __FMAN_PORT_H diff --git a/drivers/net/ethernet/freescale/fman/fman_sp.c b/drivers/net/ethernet/freescale/fman/fman_sp.c index 248f5bcca4684a439f9d9c3d513ebd632564d950..0fac60aa5283f4e57480f1278cdd899c1f2cf4b7 100644 --- a/drivers/net/ethernet/freescale/fman/fman_sp.c +++ b/drivers/net/ethernet/freescale/fman/fman_sp.c @@ -1,33 +1,6 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* * Copyright 2008 - 2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "fman_sp.h" diff --git a/drivers/net/ethernet/freescale/fman/fman_sp.h b/drivers/net/ethernet/freescale/fman/fman_sp.h index 820b7f63088f5d44d63ff89b16c574a742ad0f87..a62dd21c81f1cbe1f66d355066e351a070261aac 100644 --- a/drivers/net/ethernet/freescale/fman/fman_sp.h +++ b/drivers/net/ethernet/freescale/fman/fman_sp.h @@ -1,32 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* * Copyright 2008 - 2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __FM_SP_H diff --git a/drivers/net/ethernet/freescale/fman/fman_tgec.c b/drivers/net/ethernet/freescale/fman/fman_tgec.c index 311c1906e0446fc3d506b66bf340a6c3328fb7fa..5a4be54ad459114efe0f3102c50b4c301d52d9d0 100644 --- a/drivers/net/ethernet/freescale/fman/fman_tgec.c +++ b/drivers/net/ethernet/freescale/fman/fman_tgec.c @@ -1,39 +1,13 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include "fman_tgec.h" #include "fman.h" +#include "mac.h" #include #include @@ -206,7 +180,7 @@ struct fman_mac { /* MAC address of device; */ u64 addr; u16 max_speed; - void *dev_id; /* device cookie used by the exception cbs */ + struct mac_device *dev_id; /* device cookie used by the exception cbs */ fman_mac_exception_cb *exception_cb; fman_mac_exception_cb *event_cb; /* pointer to driver's global address hash table */ @@ -419,7 +393,7 @@ static bool is_init_done(struct tgec_cfg *cfg) return false; } -int tgec_enable(struct fman_mac *tgec, enum comm_mode mode) +static int tgec_enable(struct fman_mac *tgec) { struct tgec_regs __iomem *regs = tgec->regs; u32 tmp; @@ -428,34 +402,25 @@ int tgec_enable(struct fman_mac *tgec, enum comm_mode mode) return -EINVAL; tmp = ioread32be(®s->command_config); - if (mode & COMM_MODE_RX) - tmp |= CMD_CFG_RX_EN; - if (mode & COMM_MODE_TX) - tmp |= CMD_CFG_TX_EN; + tmp |= CMD_CFG_RX_EN | CMD_CFG_TX_EN; iowrite32be(tmp, ®s->command_config); return 0; } -int tgec_disable(struct fman_mac *tgec, enum comm_mode mode) +static void tgec_disable(struct fman_mac *tgec) { struct tgec_regs __iomem *regs = tgec->regs; u32 tmp; - if (!is_init_done(tgec->cfg)) - return -EINVAL; + WARN_ON_ONCE(!is_init_done(tgec->cfg)); tmp = ioread32be(®s->command_config); - if (mode & COMM_MODE_RX) - tmp &= ~CMD_CFG_RX_EN; - if (mode & COMM_MODE_TX) - tmp &= ~CMD_CFG_TX_EN; + tmp &= ~(CMD_CFG_RX_EN | CMD_CFG_TX_EN); iowrite32be(tmp, ®s->command_config); - - return 0; } -int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val) +static int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val) { struct tgec_regs __iomem *regs = tgec->regs; u32 tmp; @@ -473,18 +438,9 @@ int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val) return 0; } -int tgec_cfg_max_frame_len(struct fman_mac *tgec, u16 new_val) -{ - if (is_init_done(tgec->cfg)) - return -EINVAL; - - tgec->cfg->max_frame_length = new_val; - - return 0; -} - -int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 __maybe_unused priority, - u16 pause_time, u16 __maybe_unused thresh_time) +static int tgec_set_tx_pause_frames(struct fman_mac *tgec, + u8 __maybe_unused priority, u16 pause_time, + u16 __maybe_unused thresh_time) { struct tgec_regs __iomem *regs = tgec->regs; @@ -496,7 +452,7 @@ int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 __maybe_unused priority, return 0; } -int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en) +static int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en) { struct tgec_regs __iomem *regs = tgec->regs; u32 tmp; @@ -514,7 +470,8 @@ int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en) return 0; } -int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *p_enet_addr) +static int tgec_modify_mac_address(struct fman_mac *tgec, + const enet_addr_t *p_enet_addr) { if (!is_init_done(tgec->cfg)) return -EINVAL; @@ -525,7 +482,8 @@ int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *p_enet_add return 0; } -int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr) +static int tgec_add_hash_mac_address(struct fman_mac *tgec, + enet_addr_t *eth_addr) { struct tgec_regs __iomem *regs = tgec->regs; struct eth_hash_entry *hash_entry; @@ -562,7 +520,7 @@ int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr) return 0; } -int tgec_set_allmulti(struct fman_mac *tgec, bool enable) +static int tgec_set_allmulti(struct fman_mac *tgec, bool enable) { u32 entry; struct tgec_regs __iomem *regs = tgec->regs; @@ -585,7 +543,7 @@ int tgec_set_allmulti(struct fman_mac *tgec, bool enable) return 0; } -int tgec_set_tstamp(struct fman_mac *tgec, bool enable) +static int tgec_set_tstamp(struct fman_mac *tgec, bool enable) { struct tgec_regs __iomem *regs = tgec->regs; u32 tmp; @@ -605,7 +563,8 @@ int tgec_set_tstamp(struct fman_mac *tgec, bool enable) return 0; } -int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr) +static int tgec_del_hash_mac_address(struct fman_mac *tgec, + enet_addr_t *eth_addr) { struct tgec_regs __iomem *regs = tgec->regs; struct eth_hash_entry *hash_entry = NULL; @@ -642,20 +601,15 @@ int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr) return 0; } -int tgec_get_version(struct fman_mac *tgec, u32 *mac_version) +static void tgec_adjust_link(struct mac_device *mac_dev) { - struct tgec_regs __iomem *regs = tgec->regs; - - if (!is_init_done(tgec->cfg)) - return -EINVAL; + struct phy_device *phy_dev = mac_dev->phy_dev; - *mac_version = ioread32be(®s->tgec_id); - - return 0; + mac_dev->update_speed(mac_dev, phy_dev->speed); } -int tgec_set_exception(struct fman_mac *tgec, - enum fman_mac_exceptions exception, bool enable) +static int tgec_set_exception(struct fman_mac *tgec, + enum fman_mac_exceptions exception, bool enable) { struct tgec_regs __iomem *regs = tgec->regs; u32 bit_mask = 0; @@ -681,7 +635,7 @@ int tgec_set_exception(struct fman_mac *tgec, return 0; } -int tgec_init(struct fman_mac *tgec) +static int tgec_init(struct fman_mac *tgec) { struct tgec_cfg *cfg; enet_addr_t eth_addr; @@ -764,7 +718,7 @@ int tgec_init(struct fman_mac *tgec) return 0; } -int tgec_free(struct fman_mac *tgec) +static int tgec_free(struct fman_mac *tgec) { free_init_resources(tgec); @@ -774,13 +728,12 @@ int tgec_free(struct fman_mac *tgec) return 0; } -struct fman_mac *tgec_config(struct fman_mac_params *params) +static struct fman_mac *tgec_config(struct mac_device *mac_dev, + struct fman_mac_params *params) { struct fman_mac *tgec; struct tgec_cfg *cfg; - void __iomem *base_addr; - base_addr = params->base_addr; /* allocate memory for the UCC GETH data structure. */ tgec = kzalloc(sizeof(*tgec), GFP_KERNEL); if (!tgec) @@ -798,8 +751,8 @@ struct fman_mac *tgec_config(struct fman_mac_params *params) set_dflts(cfg); - tgec->regs = base_addr; - tgec->addr = ENET_ADDR_TO_UINT64(params->addr); + tgec->regs = mac_dev->vaddr; + tgec->addr = ENET_ADDR_TO_UINT64(mac_dev->addr); tgec->max_speed = params->max_speed; tgec->mac_id = params->mac_id; tgec->exceptions = (TGEC_IMASK_MDIO_SCAN_EVENT | @@ -819,7 +772,7 @@ struct fman_mac *tgec_config(struct fman_mac_params *params) TGEC_IMASK_RX_ALIGN_ER); tgec->exception_cb = params->exception_cb; tgec->event_cb = params->event_cb; - tgec->dev_id = params->dev_id; + tgec->dev_id = mac_dev; tgec->fm = params->fm; /* Save FMan revision */ @@ -827,3 +780,52 @@ struct fman_mac *tgec_config(struct fman_mac_params *params) return tgec; } + +int tgec_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params) +{ + int err; + struct fman_mac *tgec; + + mac_dev->set_promisc = tgec_set_promiscuous; + mac_dev->change_addr = tgec_modify_mac_address; + mac_dev->add_hash_mac_addr = tgec_add_hash_mac_address; + mac_dev->remove_hash_mac_addr = tgec_del_hash_mac_address; + mac_dev->set_tx_pause = tgec_set_tx_pause_frames; + mac_dev->set_rx_pause = tgec_accept_rx_pause_frames; + mac_dev->set_exception = tgec_set_exception; + mac_dev->set_allmulti = tgec_set_allmulti; + mac_dev->set_tstamp = tgec_set_tstamp; + mac_dev->set_multi = fman_set_multi; + mac_dev->adjust_link = tgec_adjust_link; + mac_dev->enable = tgec_enable; + mac_dev->disable = tgec_disable; + + mac_dev->fman_mac = tgec_config(mac_dev, params); + if (!mac_dev->fman_mac) { + err = -EINVAL; + goto _return; + } + + tgec = mac_dev->fman_mac; + tgec->cfg->max_frame_length = fman_get_max_frm(); + err = tgec_init(tgec); + if (err < 0) + goto _return_fm_mac_free; + + /* For 10G MAC, disable Tx ECC exception */ + err = tgec_set_exception(tgec, FM_MAC_EX_10G_TX_ECC_ER, false); + if (err < 0) + goto _return_fm_mac_free; + + pr_info("FMan XGEC version: 0x%08x\n", + ioread32be(&tgec->regs->tgec_id)); + goto _return; + +_return_fm_mac_free: + tgec_free(mac_dev->fman_mac); + +_return: + return err; +} diff --git a/drivers/net/ethernet/freescale/fman/fman_tgec.h b/drivers/net/ethernet/freescale/fman/fman_tgec.h index b28b20b2614829833fd54569a1f01543fb525a31..768b8d165e058bca2bd8142f7340380d2f2e7c8c 100644 --- a/drivers/net/ethernet/freescale/fman/fman_tgec.h +++ b/drivers/net/ethernet/freescale/fman/fman_tgec.h @@ -1,33 +1,6 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ /* - * Copyright 2008-2015 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #ifndef __TGEC_H @@ -35,23 +8,10 @@ #include "fman_mac.h" -struct fman_mac *tgec_config(struct fman_mac_params *params); -int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val); -int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *enet_addr); -int tgec_cfg_max_frame_len(struct fman_mac *tgec, u16 new_val); -int tgec_enable(struct fman_mac *tgec, enum comm_mode mode); -int tgec_disable(struct fman_mac *tgec, enum comm_mode mode); -int tgec_init(struct fman_mac *tgec); -int tgec_free(struct fman_mac *tgec); -int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en); -int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 priority, - u16 pause_time, u16 thresh_time); -int tgec_set_exception(struct fman_mac *tgec, - enum fman_mac_exceptions exception, bool enable); -int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr); -int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr); -int tgec_get_version(struct fman_mac *tgec, u32 *mac_version); -int tgec_set_allmulti(struct fman_mac *tgec, bool enable); -int tgec_set_tstamp(struct fman_mac *tgec, bool enable); +struct mac_device; + +int tgec_initialization(struct mac_device *mac_dev, + struct device_node *mac_node, + struct fman_mac_params *params); #endif /* __TGEC_H */ diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c index 39ae965cd4f649375defa875884a28cf4334d786..7b7526fd7da3289299022d8e1d3c1a45d4191703 100644 --- a/drivers/net/ethernet/freescale/fman/mac.c +++ b/drivers/net/ethernet/freescale/fman/mac.c @@ -1,32 +1,6 @@ -/* Copyright 2008-2015 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later +/* + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -54,20 +28,12 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_DESCRIPTION("FSL FMan MAC API based driver"); struct mac_priv_s { - struct device *dev; - void __iomem *vaddr; u8 cell_index; struct fman *fman; - struct device_node *internal_phy_node; /* List of multicast addresses */ struct list_head mc_addr_list; struct platform_device *eth_dev; - struct fixed_phy_status *fixed_link; u16 speed; - u16 max_speed; - - int (*enable)(struct fman_mac *mac_dev, enum comm_mode mode); - int (*disable)(struct fman_mac *mac_dev, enum comm_mode mode); }; struct mac_address { @@ -75,222 +41,21 @@ struct mac_address { struct list_head list; }; -static void mac_exception(void *handle, enum fman_mac_exceptions ex) +static void mac_exception(struct mac_device *mac_dev, + enum fman_mac_exceptions ex) { - struct mac_device *mac_dev; - struct mac_priv_s *priv; - - mac_dev = handle; - priv = mac_dev->priv; - if (ex == FM_MAC_EX_10G_RX_FIFO_OVFL) { /* don't flag RX FIFO after the first */ mac_dev->set_exception(mac_dev->fman_mac, FM_MAC_EX_10G_RX_FIFO_OVFL, false); - dev_err(priv->dev, "10G MAC got RX FIFO Error = %x\n", ex); + dev_err(mac_dev->dev, "10G MAC got RX FIFO Error = %x\n", ex); } - dev_dbg(priv->dev, "%s:%s() -> %d\n", KBUILD_BASENAME ".c", + dev_dbg(mac_dev->dev, "%s:%s() -> %d\n", KBUILD_BASENAME ".c", __func__, ex); } -static int set_fman_mac_params(struct mac_device *mac_dev, - struct fman_mac_params *params) -{ - struct mac_priv_s *priv = mac_dev->priv; - - params->base_addr = (typeof(params->base_addr)) - devm_ioremap(priv->dev, mac_dev->res->start, - resource_size(mac_dev->res)); - if (!params->base_addr) - return -ENOMEM; - - memcpy(¶ms->addr, mac_dev->addr, sizeof(mac_dev->addr)); - params->max_speed = priv->max_speed; - params->phy_if = mac_dev->phy_if; - params->basex_if = false; - params->mac_id = priv->cell_index; - params->fm = (void *)priv->fman; - params->exception_cb = mac_exception; - params->event_cb = mac_exception; - params->dev_id = mac_dev; - params->internal_phy_node = priv->internal_phy_node; - - return 0; -} - -static int tgec_initialization(struct mac_device *mac_dev) -{ - int err; - struct mac_priv_s *priv; - struct fman_mac_params params; - u32 version; - - priv = mac_dev->priv; - - err = set_fman_mac_params(mac_dev, ¶ms); - if (err) - goto _return; - - mac_dev->fman_mac = tgec_config(¶ms); - if (!mac_dev->fman_mac) { - err = -EINVAL; - goto _return; - } - - err = tgec_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm()); - if (err < 0) - goto _return_fm_mac_free; - - err = tgec_init(mac_dev->fman_mac); - if (err < 0) - goto _return_fm_mac_free; - - /* For 10G MAC, disable Tx ECC exception */ - err = mac_dev->set_exception(mac_dev->fman_mac, - FM_MAC_EX_10G_TX_ECC_ER, false); - if (err < 0) - goto _return_fm_mac_free; - - err = tgec_get_version(mac_dev->fman_mac, &version); - if (err < 0) - goto _return_fm_mac_free; - - dev_info(priv->dev, "FMan XGEC version: 0x%08x\n", version); - - goto _return; - -_return_fm_mac_free: - tgec_free(mac_dev->fman_mac); - -_return: - return err; -} - -static int dtsec_initialization(struct mac_device *mac_dev) -{ - int err; - struct mac_priv_s *priv; - struct fman_mac_params params; - u32 version; - - priv = mac_dev->priv; - - err = set_fman_mac_params(mac_dev, ¶ms); - if (err) - goto _return; - - mac_dev->fman_mac = dtsec_config(¶ms); - if (!mac_dev->fman_mac) { - err = -EINVAL; - goto _return; - } - - err = dtsec_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm()); - if (err < 0) - goto _return_fm_mac_free; - - err = dtsec_cfg_pad_and_crc(mac_dev->fman_mac, true); - if (err < 0) - goto _return_fm_mac_free; - - err = dtsec_init(mac_dev->fman_mac); - if (err < 0) - goto _return_fm_mac_free; - - /* For 1G MAC, disable by default the MIB counters overflow interrupt */ - err = mac_dev->set_exception(mac_dev->fman_mac, - FM_MAC_EX_1G_RX_MIB_CNT_OVFL, false); - if (err < 0) - goto _return_fm_mac_free; - - err = dtsec_get_version(mac_dev->fman_mac, &version); - if (err < 0) - goto _return_fm_mac_free; - - dev_info(priv->dev, "FMan dTSEC version: 0x%08x\n", version); - - goto _return; - -_return_fm_mac_free: - dtsec_free(mac_dev->fman_mac); - -_return: - return err; -} - -static int memac_initialization(struct mac_device *mac_dev) -{ - int err; - struct mac_priv_s *priv; - struct fman_mac_params params; - - priv = mac_dev->priv; - - err = set_fman_mac_params(mac_dev, ¶ms); - if (err) - goto _return; - - if (priv->max_speed == SPEED_10000) - params.phy_if = PHY_INTERFACE_MODE_XGMII; - - mac_dev->fman_mac = memac_config(¶ms); - if (!mac_dev->fman_mac) { - err = -EINVAL; - goto _return; - } - - err = memac_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm()); - if (err < 0) - goto _return_fm_mac_free; - - err = memac_cfg_reset_on_init(mac_dev->fman_mac, true); - if (err < 0) - goto _return_fm_mac_free; - - err = memac_cfg_fixed_link(mac_dev->fman_mac, priv->fixed_link); - if (err < 0) - goto _return_fm_mac_free; - - err = memac_init(mac_dev->fman_mac); - if (err < 0) - goto _return_fm_mac_free; - - dev_info(priv->dev, "FMan MEMAC\n"); - - goto _return; - -_return_fm_mac_free: - memac_free(mac_dev->fman_mac); - -_return: - return err; -} - -static int start(struct mac_device *mac_dev) -{ - int err; - struct phy_device *phy_dev = mac_dev->phy_dev; - struct mac_priv_s *priv = mac_dev->priv; - - err = priv->enable(mac_dev->fman_mac, COMM_MODE_RX_AND_TX); - if (!err && phy_dev) - phy_start(phy_dev); - - return err; -} - -static int stop(struct mac_device *mac_dev) -{ - struct mac_priv_s *priv = mac_dev->priv; - - if (mac_dev->phy_dev) - phy_stop(mac_dev->phy_dev); - - return priv->disable(mac_dev->fman_mac, COMM_MODE_RX_AND_TX); -} - -static int set_multi(struct net_device *net_dev, struct mac_device *mac_dev) +int fman_set_multi(struct net_device *net_dev, struct mac_device *mac_dev) { struct mac_priv_s *priv; struct mac_address *old_addr, *tmp; @@ -424,109 +189,6 @@ void fman_get_pause_cfg(struct mac_device *mac_dev, bool *rx_pause, } EXPORT_SYMBOL(fman_get_pause_cfg); -static void adjust_link_void(struct mac_device *mac_dev) -{ -} - -static void adjust_link_dtsec(struct mac_device *mac_dev) -{ - struct phy_device *phy_dev = mac_dev->phy_dev; - struct fman_mac *fman_mac; - bool rx_pause, tx_pause; - int err; - - fman_mac = mac_dev->fman_mac; - if (!phy_dev->link) { - dtsec_restart_autoneg(fman_mac); - - return; - } - - dtsec_adjust_link(fman_mac, phy_dev->speed); - fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause); - err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause); - if (err < 0) - dev_err(mac_dev->priv->dev, "fman_set_mac_active_pause() = %d\n", - err); -} - -static void adjust_link_memac(struct mac_device *mac_dev) -{ - struct phy_device *phy_dev = mac_dev->phy_dev; - struct fman_mac *fman_mac; - bool rx_pause, tx_pause; - int err; - - fman_mac = mac_dev->fman_mac; - memac_adjust_link(fman_mac, phy_dev->speed); - - fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause); - err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause); - if (err < 0) - dev_err(mac_dev->priv->dev, "fman_set_mac_active_pause() = %d\n", - err); -} - -static void setup_dtsec(struct mac_device *mac_dev) -{ - mac_dev->init = dtsec_initialization; - mac_dev->set_promisc = dtsec_set_promiscuous; - mac_dev->change_addr = dtsec_modify_mac_address; - mac_dev->add_hash_mac_addr = dtsec_add_hash_mac_address; - mac_dev->remove_hash_mac_addr = dtsec_del_hash_mac_address; - mac_dev->set_tx_pause = dtsec_set_tx_pause_frames; - mac_dev->set_rx_pause = dtsec_accept_rx_pause_frames; - mac_dev->set_exception = dtsec_set_exception; - mac_dev->set_allmulti = dtsec_set_allmulti; - mac_dev->set_tstamp = dtsec_set_tstamp; - mac_dev->set_multi = set_multi; - mac_dev->start = start; - mac_dev->stop = stop; - mac_dev->adjust_link = adjust_link_dtsec; - mac_dev->priv->enable = dtsec_enable; - mac_dev->priv->disable = dtsec_disable; -} - -static void setup_tgec(struct mac_device *mac_dev) -{ - mac_dev->init = tgec_initialization; - mac_dev->set_promisc = tgec_set_promiscuous; - mac_dev->change_addr = tgec_modify_mac_address; - mac_dev->add_hash_mac_addr = tgec_add_hash_mac_address; - mac_dev->remove_hash_mac_addr = tgec_del_hash_mac_address; - mac_dev->set_tx_pause = tgec_set_tx_pause_frames; - mac_dev->set_rx_pause = tgec_accept_rx_pause_frames; - mac_dev->set_exception = tgec_set_exception; - mac_dev->set_allmulti = tgec_set_allmulti; - mac_dev->set_tstamp = tgec_set_tstamp; - mac_dev->set_multi = set_multi; - mac_dev->start = start; - mac_dev->stop = stop; - mac_dev->adjust_link = adjust_link_void; - mac_dev->priv->enable = tgec_enable; - mac_dev->priv->disable = tgec_disable; -} - -static void setup_memac(struct mac_device *mac_dev) -{ - mac_dev->init = memac_initialization; - mac_dev->set_promisc = memac_set_promiscuous; - mac_dev->change_addr = memac_modify_mac_address; - mac_dev->add_hash_mac_addr = memac_add_hash_mac_address; - mac_dev->remove_hash_mac_addr = memac_del_hash_mac_address; - mac_dev->set_tx_pause = memac_set_tx_pause_frames; - mac_dev->set_rx_pause = memac_accept_rx_pause_frames; - mac_dev->set_exception = memac_set_exception; - mac_dev->set_allmulti = memac_set_allmulti; - mac_dev->set_tstamp = memac_set_tstamp; - mac_dev->set_multi = set_multi; - mac_dev->start = start; - mac_dev->stop = stop; - mac_dev->adjust_link = adjust_link_memac; - mac_dev->priv->enable = memac_enable; - mac_dev->priv->disable = memac_disable; -} - #define DTSEC_SUPPORTED \ (SUPPORTED_10baseT_Half \ | SUPPORTED_10baseT_Full \ @@ -577,7 +239,7 @@ static struct platform_device *dpaa_eth_add_device(int fman_id, goto no_mem; } - pdev->dev.parent = priv->dev; + pdev->dev.parent = mac_dev->dev; ret = platform_device_add_data(pdev, &data, sizeof(data)); if (ret) @@ -601,9 +263,9 @@ no_mem: } static const struct of_device_id mac_match[] = { - { .compatible = "fsl,fman-dtsec" }, - { .compatible = "fsl,fman-xgec" }, - { .compatible = "fsl,fman-memac" }, + { .compatible = "fsl,fman-dtsec", .data = dtsec_initialization }, + { .compatible = "fsl,fman-xgec", .data = tgec_initialization }, + { .compatible = "fsl,fman-memac", .data = memac_initialization }, {} }; MODULE_DEVICE_TABLE(of, mac_match); @@ -611,50 +273,33 @@ MODULE_DEVICE_TABLE(of, mac_match); static int mac_probe(struct platform_device *_of_dev) { int err, i, nph; + int (*init)(struct mac_device *mac_dev, struct device_node *mac_node, + struct fman_mac_params *params); struct device *dev; struct device_node *mac_node, *dev_node; struct mac_device *mac_dev; struct platform_device *of_dev; - struct resource res; + struct resource *res; struct mac_priv_s *priv; + struct fman_mac_params params; u32 val; u8 fman_id; phy_interface_t phy_if; dev = &_of_dev->dev; mac_node = dev->of_node; + init = of_device_get_match_data(dev); mac_dev = devm_kzalloc(dev, sizeof(*mac_dev), GFP_KERNEL); - if (!mac_dev) { - err = -ENOMEM; - goto _return; - } + if (!mac_dev) + return -ENOMEM; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); - if (!priv) { - err = -ENOMEM; - goto _return; - } + if (!priv) + return -ENOMEM; /* Save private information */ mac_dev->priv = priv; - priv->dev = dev; - - if (of_device_is_compatible(mac_node, "fsl,fman-dtsec")) { - setup_dtsec(mac_dev); - priv->internal_phy_node = of_parse_phandle(mac_node, - "tbi-handle", 0); - } else if (of_device_is_compatible(mac_node, "fsl,fman-xgec")) { - setup_tgec(mac_dev); - } else if (of_device_is_compatible(mac_node, "fsl,fman-memac")) { - setup_memac(mac_dev); - priv->internal_phy_node = of_parse_phandle(mac_node, - "pcsphy-handle", 0); - } else { - dev_err(dev, "MAC node (%pOF) contains unsupported MAC\n", - mac_node); - err = -EINVAL; - goto _return; - } + mac_dev->dev = dev; INIT_LIST_HEAD(&priv->mc_addr_list); @@ -663,8 +308,7 @@ static int mac_probe(struct platform_device *_of_dev) if (!dev_node) { dev_err(dev, "of_get_parent(%pOF) failed\n", mac_node); - err = -EINVAL; - goto _return_of_get_parent; + return -EINVAL; } of_dev = of_find_device_by_node(dev_node); @@ -694,42 +338,33 @@ static int mac_probe(struct platform_device *_of_dev) of_node_put(dev_node); /* Get the address of the memory mapped registers */ - err = of_address_to_resource(mac_node, 0, &res); - if (err < 0) { - dev_err(dev, "of_address_to_resource(%pOF) = %d\n", - mac_node, err); - goto _return_of_get_parent; + res = platform_get_mem_or_io(_of_dev, 0); + if (!res) { + dev_err(dev, "could not get registers\n"); + return -EINVAL; } - mac_dev->res = __devm_request_region(dev, - fman_get_mem_region(priv->fman), - res.start, resource_size(&res), - "mac"); - if (!mac_dev->res) { - dev_err(dev, "__devm_request_mem_region(mac) failed\n"); - err = -EBUSY; - goto _return_of_get_parent; + err = devm_request_resource(dev, fman_get_mem_region(priv->fman), res); + if (err) { + dev_err_probe(dev, err, "could not request resource\n"); + return err; } - priv->vaddr = devm_ioremap(dev, mac_dev->res->start, - resource_size(mac_dev->res)); - if (!priv->vaddr) { + mac_dev->vaddr = devm_ioremap(dev, res->start, resource_size(res)); + if (!mac_dev->vaddr) { dev_err(dev, "devm_ioremap() failed\n"); - err = -EIO; - goto _return_of_get_parent; + return -EIO; } + mac_dev->vaddr_end = mac_dev->vaddr + resource_size(res); - if (!of_device_is_available(mac_node)) { - err = -ENODEV; - goto _return_of_get_parent; - } + if (!of_device_is_available(mac_node)) + return -ENODEV; /* Get the cell-index */ err = of_property_read_u32(mac_node, "cell-index", &val); if (err) { dev_err(dev, "failed to read cell-index for %pOF\n", mac_node); - err = -EINVAL; - goto _return_of_get_parent; + return -EINVAL; } priv->cell_index = (u8)val; @@ -743,15 +378,13 @@ static int mac_probe(struct platform_device *_of_dev) if (unlikely(nph < 0)) { dev_err(dev, "of_count_phandle_with_args(%pOF, fsl,fman-ports) failed\n", mac_node); - err = nph; - goto _return_of_get_parent; + return nph; } if (nph != ARRAY_SIZE(mac_dev->port)) { dev_err(dev, "Not supported number of fman-ports handles of mac node %pOF from device tree\n", mac_node); - err = -EINVAL; - goto _return_of_get_parent; + return -EINVAL; } for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) { @@ -760,8 +393,7 @@ static int mac_probe(struct platform_device *_of_dev) if (!dev_node) { dev_err(dev, "of_parse_phandle(%pOF, fsl,fman-ports) failed\n", mac_node); - err = -EINVAL; - goto _return_of_node_put; + return -EINVAL; } of_dev = of_find_device_by_node(dev_node); @@ -793,7 +425,7 @@ static int mac_probe(struct platform_device *_of_dev) mac_dev->phy_if = phy_if; priv->speed = phy2speed[mac_dev->phy_if]; - priv->max_speed = priv->speed; + params.max_speed = priv->speed; mac_dev->if_support = DTSEC_SUPPORTED; /* We don't support half-duplex in SGMII mode */ if (mac_dev->phy_if == PHY_INTERFACE_MODE_SGMII) @@ -801,7 +433,7 @@ static int mac_probe(struct platform_device *_of_dev) SUPPORTED_100baseT_Half); /* Gigabit support (no half-duplex) */ - if (priv->max_speed == 1000) + if (params.max_speed == 1000) mac_dev->if_support |= SUPPORTED_1000baseT_Full; /* The 10G interface only supports one mode */ @@ -810,42 +442,18 @@ static int mac_probe(struct platform_device *_of_dev) /* Get the rest of the PHY information */ mac_dev->phy_node = of_parse_phandle(mac_node, "phy-handle", 0); - if (!mac_dev->phy_node && of_phy_is_fixed_link(mac_node)) { - struct phy_device *phy; - - err = of_phy_register_fixed_link(mac_node); - if (err) - goto _return_of_get_parent; - - priv->fixed_link = kzalloc(sizeof(*priv->fixed_link), - GFP_KERNEL); - if (!priv->fixed_link) { - err = -ENOMEM; - goto _return_of_get_parent; - } - - mac_dev->phy_node = of_node_get(mac_node); - phy = of_phy_find_device(mac_dev->phy_node); - if (!phy) { - err = -EINVAL; - of_node_put(mac_dev->phy_node); - goto _return_of_get_parent; - } - priv->fixed_link->link = phy->link; - priv->fixed_link->speed = phy->speed; - priv->fixed_link->duplex = phy->duplex; - priv->fixed_link->pause = phy->pause; - priv->fixed_link->asym_pause = phy->asym_pause; + params.basex_if = false; + params.mac_id = priv->cell_index; + params.fm = (void *)priv->fman; + params.exception_cb = mac_exception; + params.event_cb = mac_exception; - put_device(&phy->mdio.dev); - } - - err = mac_dev->init(mac_dev); + err = init(mac_dev, mac_node, ¶ms); if (err < 0) { dev_err(dev, "mac_dev->init() = %d\n", err); of_node_put(mac_dev->phy_node); - goto _return_of_get_parent; + return err; } /* pause frame autonegotiation enabled */ @@ -872,13 +480,10 @@ static int mac_probe(struct platform_device *_of_dev) priv->eth_dev = NULL; } - goto _return; + return err; _return_of_node_put: of_node_put(dev_node); -_return_of_get_parent: - kfree(priv->fixed_link); -_return: return err; } diff --git a/drivers/net/ethernet/freescale/fman/mac.h b/drivers/net/ethernet/freescale/fman/mac.h index daa285a9b8b25b1d27eeac91c8195ed19c19f0c0..b95d384271bd621919d4190853a08453c429cf79 100644 --- a/drivers/net/ethernet/freescale/fman/mac.h +++ b/drivers/net/ethernet/freescale/fman/mac.h @@ -1,32 +1,6 @@ -/* Copyright 2008-2015 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */ +/* + * Copyright 2008 - 2015 Freescale Semiconductor Inc. */ #ifndef __MAC_H @@ -45,13 +19,16 @@ struct fman_mac; struct mac_priv_s; struct mac_device { - struct resource *res; + void __iomem *vaddr; + void __iomem *vaddr_end; + struct device *dev; u8 addr[ETH_ALEN]; struct fman_port *port[2]; u32 if_support; struct phy_device *phy_dev; phy_interface_t phy_if; struct device_node *phy_node; + struct net_device *net_dev; bool autoneg_pause; bool rx_pause_req; @@ -61,9 +38,8 @@ struct mac_device { bool promisc; bool allmulti; - int (*init)(struct mac_device *mac_dev); - int (*start)(struct mac_device *mac_dev); - int (*stop)(struct mac_device *mac_dev); + int (*enable)(struct fman_mac *mac_dev); + void (*disable)(struct fman_mac *mac_dev); void (*adjust_link)(struct mac_device *mac_dev); int (*set_promisc)(struct fman_mac *mac_dev, bool enable); int (*change_addr)(struct fman_mac *mac_dev, const enet_addr_t *enet_addr); @@ -81,6 +57,8 @@ struct mac_device { int (*remove_hash_mac_addr)(struct fman_mac *mac_dev, enet_addr_t *eth_addr); + void (*update_speed)(struct mac_device *mac_dev, int speed); + struct fman_mac *fman_mac; struct mac_priv_s *priv; }; @@ -97,5 +75,6 @@ int fman_set_mac_active_pause(struct mac_device *mac_dev, bool rx, bool tx); void fman_get_pause_cfg(struct mac_device *mac_dev, bool *rx_pause, bool *tx_pause); +int fman_set_multi(struct net_device *net_dev, struct mac_device *mac_dev); #endif /* __MAC_H */ diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c index b3dae17e067e0798198b7823a8f7b84ace64fc35..8844a9a04fcf95ac6b4de9d05f2f2bf0ae7202ee 100644 --- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c +++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c @@ -791,7 +791,7 @@ static int fs_enet_close(struct net_device *dev) static void fs_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); } static int fs_get_regs_len(struct net_device *dev) @@ -883,9 +883,6 @@ static const struct ethtool_ops fs_ethtool_ops = { .set_tunable = fs_set_tunable, }; -extern int fs_mii_connect(struct net_device *dev); -extern void fs_mii_disconnect(struct net_device *dev); - /**************************************************************************************/ #ifdef CONFIG_FS_ENET_HAS_FEC diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c index 99fe2c210d0f6a52f5a549c256872ee6e4c4d52c..61f4b6e50d29b2ef9f43679c19dab447b4e4dec3 100644 --- a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c +++ b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c @@ -98,7 +98,7 @@ static int do_pd_setup(struct fs_enet_private *fep) return -EINVAL; fep->fec.fecp = of_iomap(ofdev->dev.of_node, 0); - if (!fep->fcc.fccp) + if (!fep->fec.fecp) return -EINVAL; return 0; diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c index e7bf1524b68ee6b17667457e99eac8705840a6c2..b2def295523ab5ce23b547fae1a21b7662c25f17 100644 --- a/drivers/net/ethernet/freescale/gianfar.c +++ b/drivers/net/ethernet/freescale/gianfar.c @@ -3233,7 +3233,7 @@ static int gfar_probe(struct platform_device *ofdev) /* Register for napi ...We are registering NAPI for each grp */ for (i = 0; i < priv->num_grps; i++) { netif_napi_add(dev, &priv->gfargrp[i].napi_rx, - gfar_poll_rx_sq, NAPI_POLL_WEIGHT); + gfar_poll_rx_sq); netif_napi_add_tx_weight(dev, &priv->gfargrp[i].napi_tx, gfar_poll_tx_sq, 2); } diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index 81fb687301381313c9a4968cb38f7f5d5caf174e..b2b0d3c26fcc1d0c9ee5bad5bc534b46fef63977 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -163,7 +163,7 @@ static int gfar_sset_count(struct net_device *dev, int sset) static void gfar_gdrvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); } /* Return the length of the register structure */ diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c index 823221c912ab13161954793949596cb07eaabf7f..7a4cb4f07c32db19addf1217091538060a27b747 100644 --- a/drivers/net/ethernet/freescale/ucc_geth.c +++ b/drivers/net/ethernet/freescale/ucc_geth.c @@ -3712,7 +3712,7 @@ static int ucc_geth_probe(struct platform_device* ofdev) dev->netdev_ops = &ucc_geth_netdev_ops; dev->watchdog_timeo = TX_TIMEOUT; INIT_WORK(&ugeth->timeout_work, ucc_geth_timeout_work); - netif_napi_add(dev, &ugeth->napi, ucc_geth_poll, 64); + netif_napi_add(dev, &ugeth->napi, ucc_geth_poll); dev->mtu = 1500; dev->max_mtu = 1518; diff --git a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c index 69b2b98b15251711c211c9519712a09df6e521d8..601beb93d3b3bc46e8e0dbb53d745fa6f49aaa96 100644 --- a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c +++ b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c @@ -337,8 +337,8 @@ static void uec_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, "QUICC ENGINE", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, "QUICC ENGINE", sizeof(drvinfo->bus_info)); } #ifdef CONFIG_PM diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c index ec90da1de030bf82ff00f8cf3309333a1b6f18e8..d7d39a58cd80c7152a6ad5ff279b65fe2779cd84 100644 --- a/drivers/net/ethernet/freescale/xgmac_mdio.c +++ b/drivers/net/ethernet/freescale/xgmac_mdio.c @@ -355,7 +355,7 @@ static int xgmac_mdio_probe(struct platform_device *pdev) if (ret) return ret; - fwnode = pdev->dev.fwnode; + fwnode = dev_fwnode(&pdev->dev); if (is_of_node(fwnode)) ret = of_mdiobus_register(bus, to_of_node(fwnode)); else if (is_acpi_node(fwnode)) diff --git a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c index b0d733e9a7c66a58a38ff3976701a4a62a456ea2..4859493471db7d80dc5376fe9ead7049ea3950db 100644 --- a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c +++ b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c @@ -1046,8 +1046,8 @@ static void fjn_rx(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); snprintf(info->bus_info, sizeof(info->bus_info), "PCMCIA 0x%lx", dev->base_addr); } diff --git a/drivers/net/ethernet/fungible/funeth/funeth_main.c b/drivers/net/ethernet/fungible/funeth/funeth_main.c index f247b7ad3a8892a0ae780514d641100be5c1b59c..095f51c4d9d94b676923838318d0f37ba7c74766 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_main.c +++ b/drivers/net/ethernet/fungible/funeth/funeth_main.c @@ -339,8 +339,7 @@ static int fun_alloc_queue_irqs(struct net_device *dev, unsigned int ntx, return PTR_ERR(irq); fp->num_rx_irqs++; - netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll); } netif_info(fp, intr, dev, "Reserved %u/%u IRQs for Tx/Rx queues\n", @@ -1802,16 +1801,14 @@ static int fun_create_netdev(struct fun_ethdev *ed, unsigned int portid) if (rc) goto unreg_devlink; - if (fp->dl_port.devlink) - devlink_port_type_eth_set(&fp->dl_port, netdev); + devlink_port_type_eth_set(&fp->dl_port, netdev); return 0; unreg_devlink: ed->netdevs[portid] = NULL; fun_ktls_cleanup(fp); - if (fp->dl_port.devlink) - devlink_port_unregister(&fp->dl_port); + devlink_port_unregister(&fp->dl_port); free_stats: fun_free_stats_area(fp); free_rss: @@ -1830,11 +1827,9 @@ static void fun_destroy_netdev(struct net_device *netdev) struct funeth_priv *fp; fp = netdev_priv(netdev); - if (fp->dl_port.devlink) { - devlink_port_type_clear(&fp->dl_port); - devlink_port_unregister(&fp->dl_port); - } + devlink_port_type_clear(&fp->dl_port); unregister_netdev(netdev); + devlink_port_unregister(&fp->dl_port); fun_ktls_cleanup(fp); fun_free_stats_area(fp); fun_free_rss(fp); diff --git a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h index 53b7e95213a8539831a7dd162802600d51443f8a..671f51135c2696f285dff2cb854af39a7fcf424f 100644 --- a/drivers/net/ethernet/fungible/funeth/funeth_txrx.h +++ b/drivers/net/ethernet/fungible/funeth/funeth_txrx.h @@ -206,9 +206,9 @@ struct funeth_rxq { #define FUN_QSTAT_READ(q, seq, stats_copy) \ do { \ - seq = u64_stats_fetch_begin(&(q)->syncp); \ + seq = u64_stats_fetch_begin_irq(&(q)->syncp); \ stats_copy = (q)->stats; \ - } while (u64_stats_fetch_retry(&(q)->syncp, (seq))) + } while (u64_stats_fetch_retry_irq(&(q)->syncp, (seq))) #define FUN_INT_NAME_LEN (IFNAMSIZ + 16) diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index 50b384910c8399dd9dde1a3ffe4cca8b9f078b13..7b9a2d9d962436d8c99301669768356e90bdb4f7 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -177,14 +177,14 @@ gve_get_ethtool_stats(struct net_device *netdev, struct gve_rx_ring *rx = &priv->rx[ring]; start = - u64_stats_fetch_begin(&priv->rx[ring].statss); + u64_stats_fetch_begin_irq(&priv->rx[ring].statss); tmp_rx_pkts = rx->rpackets; tmp_rx_bytes = rx->rbytes; tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; tmp_rx_desc_err_dropped_pkt = rx->rx_desc_err_dropped_pkt; - } while (u64_stats_fetch_retry(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, start)); rx_pkts += tmp_rx_pkts; rx_bytes += tmp_rx_bytes; @@ -198,10 +198,10 @@ gve_get_ethtool_stats(struct net_device *netdev, if (priv->tx) { do { start = - u64_stats_fetch_begin(&priv->tx[ring].statss); + u64_stats_fetch_begin_irq(&priv->tx[ring].statss); tmp_tx_pkts = priv->tx[ring].pkt_done; tmp_tx_bytes = priv->tx[ring].bytes_done; - } while (u64_stats_fetch_retry(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, start)); tx_pkts += tmp_tx_pkts; tx_bytes += tmp_tx_bytes; @@ -259,13 +259,13 @@ gve_get_ethtool_stats(struct net_device *netdev, data[i++] = rx->fill_cnt - rx->cnt; do { start = - u64_stats_fetch_begin(&priv->rx[ring].statss); + u64_stats_fetch_begin_irq(&priv->rx[ring].statss); tmp_rx_bytes = rx->rbytes; tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; tmp_rx_desc_err_dropped_pkt = rx->rx_desc_err_dropped_pkt; - } while (u64_stats_fetch_retry(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, start)); data[i++] = tmp_rx_bytes; data[i++] = rx->rx_cont_packet_cnt; @@ -331,9 +331,9 @@ gve_get_ethtool_stats(struct net_device *netdev, } do { start = - u64_stats_fetch_begin(&priv->tx[ring].statss); + u64_stats_fetch_begin_irq(&priv->tx[ring].statss); tmp_tx_bytes = tx->bytes_done; - } while (u64_stats_fetch_retry(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, start)); data[i++] = tmp_tx_bytes; data[i++] = tx->wake_queue; diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 6cafee55efc329bed2f94dad51875e2a1464c6c9..d3e3ac242bfc31e5dbaba87ff299d42c17f90f37 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -51,10 +51,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { do { start = - u64_stats_fetch_begin(&priv->rx[ring].statss); + u64_stats_fetch_begin_irq(&priv->rx[ring].statss); packets = priv->rx[ring].rpackets; bytes = priv->rx[ring].rbytes; - } while (u64_stats_fetch_retry(&priv->rx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->rx[ring].statss, start)); s->rx_packets += packets; s->rx_bytes += bytes; @@ -64,10 +64,10 @@ static void gve_get_stats(struct net_device *dev, struct rtnl_link_stats64 *s) for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { do { start = - u64_stats_fetch_begin(&priv->tx[ring].statss); + u64_stats_fetch_begin_irq(&priv->tx[ring].statss); packets = priv->tx[ring].pkt_done; bytes = priv->tx[ring].bytes_done; - } while (u64_stats_fetch_retry(&priv->tx[ring].statss, + } while (u64_stats_fetch_retry_irq(&priv->tx[ring].statss, start)); s->tx_packets += packets; s->tx_bytes += bytes; @@ -526,8 +526,7 @@ static void gve_add_napi(struct gve_priv *priv, int ntfy_idx, { struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; - netif_napi_add(priv->dev, &block->napi, gve_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(priv->dev, &block->napi, gve_poll); } static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx) @@ -1274,9 +1273,9 @@ void gve_handle_report_stats(struct gve_priv *priv) } do { - start = u64_stats_fetch_begin(&priv->tx[idx].statss); + start = u64_stats_fetch_begin_irq(&priv->tx[idx].statss); tx_bytes = priv->tx[idx].bytes_done; - } while (u64_stats_fetch_retry(&priv->tx[idx].statss, start)); + } while (u64_stats_fetch_retry_irq(&priv->tx[idx].statss, start)); stats[stats_idx++] = (struct stats) { .stat_name = cpu_to_be32(TX_WAKE_CNT), .value = cpu_to_be64(priv->tx[idx].wake_queue), diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 8c939628e2d85dadfafdf2acb55a43f69763ed90..2e6461b0ea8bcdd3aad7336f160dac7ef0842dfc 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -157,7 +157,7 @@ static int gve_alloc_page_dqo(struct gve_priv *priv, int err; err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page, - &buf_state->addr, DMA_FROM_DEVICE, GFP_KERNEL); + &buf_state->addr, DMA_FROM_DEVICE, GFP_ATOMIC); if (err) return err; diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c index c84ef494bd60774c6f52f200c723c206b1a1341a..50c3f5d6611f055031804f17a9bfa2cc9fde7247 100644 --- a/drivers/net/ethernet/hisilicon/hip04_eth.c +++ b/drivers/net/ethernet/hisilicon/hip04_eth.c @@ -830,8 +830,8 @@ static int hip04_set_coalesce(struct net_device *netdev, static void hip04_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); } static const struct ethtool_ops hip04_ethtool_ops = { @@ -990,7 +990,7 @@ static int hip04_mac_probe(struct platform_device *pdev) ndev->watchdog_timeo = TX_TIMEOUT; ndev->priv_flags |= IFF_UNICAST_FLT; ndev->irq = irq; - netif_napi_add(ndev, &priv->napi, hip04_rx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, hip04_rx_poll); hip04_reset_dreq(priv); hip04_reset_ppe(priv); diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c index d7e62eca050f461080f2b5ea081991160499b976..ffcf797dfa90291ea2ee79925a207f2be37bcc64 100644 --- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c +++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c @@ -1243,7 +1243,7 @@ static int hix5hd2_dev_probe(struct platform_device *pdev) if (ret) goto out_phy_node; - netif_napi_add(ndev, &priv->napi, hix5hd2_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, hix5hd2_poll); if (HAS_CAP_TSO(priv->hw_cap)) { ret = hix5hd2_init_sg_desc_queue(priv); diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c index d94cc8c6681f7291e9e6bbdc519c25f8eb1516a9..7cf10d1e2b311732ed8c07b815628798a3c9e93d 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c @@ -2109,8 +2109,7 @@ static int hns_nic_init_ring_data(struct hns_nic_priv *priv) rd->fini_process = is_ver1 ? hns_nic_tx_fini_pro : hns_nic_tx_fini_pro_v2; - netif_napi_add(priv->netdev, &rd->napi, - hns_nic_common_poll, NAPI_POLL_WEIGHT); + netif_napi_add(priv->netdev, &rd->napi, hns_nic_common_poll); rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED; } for (i = h->q_num; i < h->q_num * 2; i++) { @@ -2122,8 +2121,7 @@ static int hns_nic_init_ring_data(struct hns_nic_priv *priv) rd->fini_process = is_ver1 ? hns_nic_rx_fini_pro : hns_nic_rx_fini_pro_v2; - netif_napi_add(priv->netdev, &rd->napi, - hns_nic_common_poll, NAPI_POLL_WEIGHT); + netif_napi_add(priv->netdev, &rd->napi, hns_nic_common_poll); rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED; } diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h index 7d4ae467f3ad413c42f39f6b05db3a4ab0d2475a..abcd7877f7d2a585df38e017f1d86107b96a1253 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h +++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h @@ -233,6 +233,17 @@ struct hclgevf_mbx_arq_ring { __le16 msg_q[HCLGE_MBX_MAX_ARQ_MSG_NUM][HCLGE_MBX_MAX_ARQ_MSG_SIZE]; }; +struct hclge_dev; + +#define HCLGE_MBX_OPCODE_MAX 256 +struct hclge_mbx_ops_param { + struct hclge_vport *vport; + struct hclge_mbx_vf_to_pf_cmd *req; + struct hclge_respond_to_vf_msg *resp_msg; +}; + +typedef int (*hclge_mbx_ops_fn)(struct hclge_mbx_ops_param *param); + #define hclge_mbx_ring_ptr_move_crq(crq) \ (crq->next_to_use = (crq->next_to_use + 1) % crq->desc_num) #define hclge_mbx_tail_ptr_move_arq(arq) \ diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 94f80e1c4020cd855823c4701535d37eed13e4c9..0179fc288f5fd72118f5e536d76c98e44f380061 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -97,13 +97,15 @@ enum HNAE3_DEV_CAP_BITS { HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, HNAE3_DEV_SUPPORT_MC_MAC_MNG_B, HNAE3_DEV_SUPPORT_CQ_B, + HNAE3_DEV_SUPPORT_FEC_STATS_B, + HNAE3_DEV_SUPPORT_LANE_NUM_B, }; -#define hnae3_dev_fd_supported(hdev) \ - test_bit(HNAE3_DEV_SUPPORT_FD_B, (hdev)->ae_dev->caps) +#define hnae3_ae_dev_fd_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_FD_B, (ae_dev)->caps) -#define hnae3_dev_gro_supported(hdev) \ - test_bit(HNAE3_DEV_SUPPORT_GRO_B, (hdev)->ae_dev->caps) +#define hnae3_ae_dev_gro_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_GRO_B, (ae_dev)->caps) #define hnae3_dev_fec_supported(hdev) \ test_bit(HNAE3_DEV_SUPPORT_FEC_B, (hdev)->ae_dev->caps) @@ -159,6 +161,12 @@ enum HNAE3_DEV_CAP_BITS { #define hnae3_ae_dev_cq_supported(ae_dev) \ test_bit(HNAE3_DEV_SUPPORT_CQ_B, (ae_dev)->caps) +#define hnae3_ae_dev_fec_stats_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_FEC_STATS_B, (ae_dev)->caps) + +#define hnae3_ae_dev_lane_num_supported(ae_dev) \ + test_bit(HNAE3_DEV_SUPPORT_LANE_NUM_B, (ae_dev)->caps) + enum HNAE3_PF_CAP_BITS { HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B = 0, }; @@ -187,6 +195,7 @@ struct hns3_mac_stats { /* hnae3 loop mode */ enum hnae3_loop { + HNAE3_LOOP_EXTERNAL, HNAE3_LOOP_APP, HNAE3_LOOP_SERIAL_SERDES, HNAE3_LOOP_PARALLEL_SERDES, @@ -223,6 +232,8 @@ enum hnae3_fec_mode { HNAE3_FEC_AUTO = 0, HNAE3_FEC_BASER, HNAE3_FEC_RS, + HNAE3_FEC_LLRS, + HNAE3_FEC_NONE, HNAE3_FEC_USER_DEF, }; @@ -270,6 +281,7 @@ enum hnae3_dbg_cmd { HNAE3_DBG_CMD_TC_SCH_INFO, HNAE3_DBG_CMD_QOS_PAUSE_CFG, HNAE3_DBG_CMD_QOS_PRI_MAP, + HNAE3_DBG_CMD_QOS_DSCP_MAP, HNAE3_DBG_CMD_QOS_BUF_CFG, HNAE3_DBG_CMD_DEV_INFO, HNAE3_DBG_CMD_TX_BD, @@ -308,6 +320,11 @@ enum hnae3_dbg_cmd { HNAE3_DBG_CMD_UNKNOWN, }; +enum hnae3_tc_map_mode { + HNAE3_TC_MAP_MODE_PRIO, + HNAE3_TC_MAP_MODE_DSCP, +}; + struct hnae3_vector_info { u8 __iomem *io_addr; int vector; @@ -560,14 +577,17 @@ struct hnae3_ae_ops { void (*client_stop)(struct hnae3_handle *handle); int (*get_status)(struct hnae3_handle *handle); void (*get_ksettings_an_result)(struct hnae3_handle *handle, - u8 *auto_neg, u32 *speed, u8 *duplex); + u8 *auto_neg, u32 *speed, u8 *duplex, + u32 *lane_num); int (*cfg_mac_speed_dup_h)(struct hnae3_handle *handle, int speed, - u8 duplex); + u8 duplex, u8 lane_num); void (*get_media_type)(struct hnae3_handle *handle, u8 *media_type, u8 *module_type); int (*check_port_speed)(struct hnae3_handle *handle, u32 speed); + void (*get_fec_stats)(struct hnae3_handle *handle, + struct ethtool_fec_stats *fec_stats); void (*get_fec)(struct hnae3_handle *handle, u8 *fec_ability, u8 *fec_mode); int (*set_fec)(struct hnae3_handle *handle, u32 fec_mode); @@ -737,6 +757,8 @@ struct hnae3_ae_ops { int (*get_link_diagnosis_info)(struct hnae3_handle *handle, u32 *status_code); void (*clean_vf_config)(struct hnae3_ae_dev *ae_dev, int num_vfs); + int (*get_dscp_prio)(struct hnae3_handle *handle, u8 dscp, + u8 *tc_map_mode, u8 *priority); }; struct hnae3_dcb_ops { @@ -745,6 +767,8 @@ struct hnae3_dcb_ops { int (*ieee_setets)(struct hnae3_handle *, struct ieee_ets *); int (*ieee_getpfc)(struct hnae3_handle *, struct ieee_pfc *); int (*ieee_setpfc)(struct hnae3_handle *, struct ieee_pfc *); + int (*ieee_setapp)(struct hnae3_handle *h, struct dcb_app *app); + int (*ieee_delapp)(struct hnae3_handle *h, struct dcb_app *app); /* DCBX configuration */ u8 (*getdcbx)(struct hnae3_handle *); @@ -774,6 +798,8 @@ struct hnae3_tc_info { bool mqprio_active; }; +#define HNAE3_MAX_DSCP 64 +#define HNAE3_PRIO_ID_INVALID 0xff struct hnae3_knic_private_info { struct net_device *netdev; /* Set by KNIC client when init instance */ u16 rss_size; /* Allocated RSS queues */ @@ -784,6 +810,9 @@ struct hnae3_knic_private_info { u32 tx_spare_buf_size; struct hnae3_tc_info tc_info; + u8 tc_map_mode; + u8 dscp_app_cnt; + u8 dscp_prio[HNAE3_MAX_DSCP]; u16 num_tqps; /* total number of TQPs in this handle */ struct hnae3_queue **tqp; /* array base of all TQPs in this instance */ @@ -815,6 +844,7 @@ struct hnae3_roce_private_info { #define HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK BIT(2) #define HNAE3_SUPPORT_VF BIT(3) #define HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK BIT(4) +#define HNAE3_SUPPORT_EXTERNAL_LOOPBACK BIT(5) #define HNAE3_USER_UPE BIT(0) /* unicast promisc enabled by user */ #define HNAE3_USER_MPE BIT(1) /* mulitcast promisc enabled by user */ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c index c8b151d29f53ba5df2e269937f25a642951db457..f671a63cecde4824f57054bf176038a261d58a60 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c @@ -52,9 +52,9 @@ void hclge_comm_cmd_reuse_desc(struct hclge_desc *desc, bool is_read) static void hclge_comm_set_default_capability(struct hnae3_ae_dev *ae_dev, bool is_pf) { - set_bit(HNAE3_DEV_SUPPORT_FD_B, ae_dev->caps); set_bit(HNAE3_DEV_SUPPORT_GRO_B, ae_dev->caps); - if (is_pf && ae_dev->dev_version == HNAE3_DEVICE_VERSION_V2) { + if (is_pf) { + set_bit(HNAE3_DEV_SUPPORT_FD_B, ae_dev->caps); set_bit(HNAE3_DEV_SUPPORT_FEC_B, ae_dev->caps); set_bit(HNAE3_DEV_SUPPORT_PAUSE_B, ae_dev->caps); } @@ -91,6 +91,7 @@ int hclge_comm_firmware_compat_config(struct hnae3_ae_dev *ae_dev, hnae3_set_bit(compat, HCLGE_COMM_PHY_IMP_EN_B, 1); hnae3_set_bit(compat, HCLGE_COMM_MAC_STATS_EXT_EN_B, 1); hnae3_set_bit(compat, HCLGE_COMM_SYNC_RX_RING_HEAD_EN_B, 1); + hnae3_set_bit(compat, HCLGE_COMM_LLRS_FEC_EN_B, 1); req->compat = cpu_to_le32(compat); } @@ -150,6 +151,10 @@ static const struct hclge_comm_caps_bit_map hclge_pf_cmd_caps[] = { HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B}, {HCLGE_COMM_CAP_PORT_VLAN_BYPASS_B, HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B}, {HCLGE_COMM_CAP_CQ_B, HNAE3_DEV_SUPPORT_CQ_B}, + {HCLGE_COMM_CAP_GRO_B, HNAE3_DEV_SUPPORT_GRO_B}, + {HCLGE_COMM_CAP_FD_B, HNAE3_DEV_SUPPORT_FD_B}, + {HCLGE_COMM_CAP_FEC_STATS_B, HNAE3_DEV_SUPPORT_FEC_STATS_B}, + {HCLGE_COMM_CAP_LANE_NUM_B, HNAE3_DEV_SUPPORT_LANE_NUM_B}, }; static const struct hclge_comm_caps_bit_map hclge_vf_cmd_caps[] = { @@ -162,6 +167,7 @@ static const struct hclge_comm_caps_bit_map hclge_vf_cmd_caps[] = { {HCLGE_COMM_CAP_TX_PUSH_B, HNAE3_DEV_SUPPORT_TX_PUSH_B}, {HCLGE_COMM_CAP_RXD_ADV_LAYOUT_B, HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B}, {HCLGE_COMM_CAP_CQ_B, HNAE3_DEV_SUPPORT_CQ_B}, + {HCLGE_COMM_CAP_GRO_B, HNAE3_DEV_SUPPORT_GRO_B}, }; static void @@ -220,8 +226,10 @@ int hclge_comm_cmd_query_version_and_capability(struct hnae3_ae_dev *ae_dev, HNAE3_PCI_REVISION_BIT_SIZE; ae_dev->dev_version |= ae_dev->pdev->revision; - if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) + if (ae_dev->dev_version == HNAE3_DEVICE_VERSION_V2) { hclge_comm_set_default_capability(ae_dev, is_pf); + return 0; + } hclge_comm_parse_capability(ae_dev, is_pf, resp); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h index 7a7d4cf9bf35d5c45e8b479633e3b2f040a78a3e..b1f9383b418f4774544dcafecdaed2a7bfe48d3b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h @@ -20,6 +20,7 @@ #define HCLGE_COMM_PHY_IMP_EN_B 2 #define HCLGE_COMM_MAC_STATS_EXT_EN_B 3 #define HCLGE_COMM_SYNC_RX_RING_HEAD_EN_B 4 +#define HCLGE_COMM_LLRS_FEC_EN_B 5 #define hclge_comm_dev_phy_imp_supported(ae_dev) \ test_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, (ae_dev)->caps) @@ -102,6 +103,7 @@ enum hclge_opcode_type { HCLGE_OPC_MAC_TNL_INT_EN = 0x0311, HCLGE_OPC_CLEAR_MAC_TNL_INT = 0x0312, HCLGE_OPC_COMMON_LOOPBACK = 0x0315, + HCLGE_OPC_QUERY_FEC_STATS = 0x0316, HCLGE_OPC_CONFIG_FEC_MODE = 0x031A, HCLGE_OPC_QUERY_ROH_TYPE_INFO = 0x0389, @@ -339,6 +341,10 @@ enum HCLGE_COMM_CAP_BITS { HCLGE_COMM_CAP_RXD_ADV_LAYOUT_B = 15, HCLGE_COMM_CAP_PORT_VLAN_BYPASS_B = 17, HCLGE_COMM_CAP_CQ_B = 18, + HCLGE_COMM_CAP_GRO_B = 20, + HCLGE_COMM_CAP_FD_B = 21, + HCLGE_COMM_CAP_FEC_STATS_B = 25, + HCLGE_COMM_CAP_LANE_NUM_B = 27, }; enum HCLGE_COMM_API_CAP_BITS { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c index d2ec4c573bf86ff98f2556da3036e31cd5d7a28b..3b6dbf158b98db7d9ec97bfa22b34e4e61f0d260 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c @@ -56,6 +56,32 @@ static int hns3_dcbnl_ieee_setpfc(struct net_device *ndev, struct ieee_pfc *pfc) return -EOPNOTSUPP; } +static int hns3_dcbnl_ieee_setapp(struct net_device *ndev, struct dcb_app *app) +{ + struct hnae3_handle *h = hns3_get_handle(ndev); + + if (hns3_nic_resetting(ndev)) + return -EBUSY; + + if (h->kinfo.dcb_ops->ieee_setapp) + return h->kinfo.dcb_ops->ieee_setapp(h, app); + + return -EOPNOTSUPP; +} + +static int hns3_dcbnl_ieee_delapp(struct net_device *ndev, struct dcb_app *app) +{ + struct hnae3_handle *h = hns3_get_handle(ndev); + + if (hns3_nic_resetting(ndev)) + return -EBUSY; + + if (h->kinfo.dcb_ops->ieee_setapp) + return h->kinfo.dcb_ops->ieee_delapp(h, app); + + return -EOPNOTSUPP; +} + /* DCBX configuration */ static u8 hns3_dcbnl_getdcbx(struct net_device *ndev) { @@ -83,6 +109,8 @@ static const struct dcbnl_rtnl_ops hns3_dcbnl_ops = { .ieee_setets = hns3_dcbnl_ieee_setets, .ieee_getpfc = hns3_dcbnl_ieee_getpfc, .ieee_setpfc = hns3_dcbnl_ieee_setpfc, + .ieee_setapp = hns3_dcbnl_ieee_setapp, + .ieee_delapp = hns3_dcbnl_ieee_delapp, .getdcbx = hns3_dcbnl_getdcbx, .setdcbx = hns3_dcbnl_setdcbx, }; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 93aeb615191d9064aee2be7e1f21c0003fb75322..66feb23f7b7b6330683f8c5326e230008782c33f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -105,6 +105,13 @@ static struct hns3_dbg_cmd_info hns3_dbg_cmd[] = { .buf_len = HNS3_DBG_READ_LEN, .init = hns3_dbg_common_file_init, }, + { + .name = "qos_dscp_map", + .cmd = HNAE3_DBG_CMD_QOS_DSCP_MAP, + .dentry = HNS3_DBG_DENTRY_TM, + .buf_len = HNS3_DBG_READ_LEN, + .init = hns3_dbg_common_file_init, + }, { .name = "qos_buf_cfg", .cmd = HNAE3_DBG_CMD_QOS_BUF_CFG, @@ -395,6 +402,12 @@ static struct hns3_dbg_cap_info hns3_dbg_cap[] = { }, { .name = "support modify vlan filter state", .cap_bit = HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, + }, { + .name = "support FEC statistics", + .cap_bit = HNAE3_DEV_SUPPORT_FEC_STATS_B, + }, { + .name = "support lane num", + .cap_bit = HNAE3_DEV_SUPPORT_LANE_NUM_B, } }; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 35d70041b9e8403f7348a2d7abf878c326f67d2c..4cb2421e71a755026a45d3ec01743f673a4adce9 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -2963,6 +2963,48 @@ static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) return h->ae_algo->ops->set_vf_mac(h, vf_id, mac); } +#define HNS3_INVALID_DSCP 0xff +#define HNS3_DSCP_SHIFT 2 + +static u8 hns3_get_skb_dscp(struct sk_buff *skb) +{ + __be16 protocol = skb->protocol; + u8 dscp = HNS3_INVALID_DSCP; + + if (protocol == htons(ETH_P_8021Q)) + protocol = vlan_get_protocol(skb); + + if (protocol == htons(ETH_P_IP)) + dscp = ipv4_get_dsfield(ip_hdr(skb)) >> HNS3_DSCP_SHIFT; + else if (protocol == htons(ETH_P_IPV6)) + dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> HNS3_DSCP_SHIFT; + + return dscp; +} + +static u16 hns3_nic_select_queue(struct net_device *netdev, + struct sk_buff *skb, + struct net_device *sb_dev) +{ + struct hnae3_handle *h = hns3_get_handle(netdev); + u8 dscp; + + if (h->kinfo.tc_map_mode != HNAE3_TC_MAP_MODE_DSCP || + !h->ae_algo->ops->get_dscp_prio) + goto out; + + dscp = hns3_get_skb_dscp(skb); + if (unlikely(dscp >= HNAE3_MAX_DSCP)) + goto out; + + skb->priority = h->kinfo.dscp_prio[dscp]; + if (skb->priority == HNAE3_PRIO_ID_INVALID) + skb->priority = 0; + +out: + return netdev_pick_tx(netdev, skb, sb_dev); +} + static const struct net_device_ops hns3_nic_netdev_ops = { .ndo_open = hns3_nic_net_open, .ndo_stop = hns3_nic_net_stop, @@ -2988,6 +3030,7 @@ static const struct net_device_ops hns3_nic_netdev_ops = { .ndo_set_vf_link_state = hns3_nic_set_vf_link_state, .ndo_set_vf_rate = hns3_nic_set_vf_rate, .ndo_set_vf_mac = hns3_nic_set_vf_mac, + .ndo_select_queue = hns3_nic_select_queue, }; bool hns3_is_phys_func(struct pci_dev *pdev) @@ -3271,12 +3314,11 @@ static void hns3_set_default_feature(struct net_device *netdev) NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL | NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST; - if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) { + if (hnae3_ae_dev_gro_supported(ae_dev)) netdev->features |= NETIF_F_GRO_HW; - if (!(h->flags & HNAE3_SUPPORT_VF)) - netdev->features |= NETIF_F_NTUPLE; - } + if (hnae3_ae_dev_fd_supported(ae_dev)) + netdev->features |= NETIF_F_NTUPLE; if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps)) netdev->features |= NETIF_F_GSO_UDP_L4; @@ -4650,7 +4692,7 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) goto map_ring_fail; netif_napi_add(priv->netdev, &tqp_vector->napi, - hns3_nic_common_poll, NAPI_POLL_WEIGHT); + hns3_nic_common_poll); } return 0; @@ -5782,6 +5824,57 @@ int hns3_set_channels(struct net_device *netdev, return 0; } +void hns3_external_lb_prepare(struct net_device *ndev, bool if_running) +{ + struct hns3_nic_priv *priv = netdev_priv(ndev); + struct hnae3_handle *h = priv->ae_handle; + int i; + + if (!if_running) + return; + + netif_carrier_off(ndev); + netif_tx_disable(ndev); + + for (i = 0; i < priv->vector_num; i++) + hns3_vector_disable(&priv->tqp_vector[i]); + + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_tqp_disable(h->kinfo.tqp[i]); + + /* delay ring buffer clearing to hns3_reset_notify_uninit_enet + * during reset process, because driver may not be able + * to disable the ring through firmware when downing the netdev. + */ + if (!hns3_nic_resetting(ndev)) + hns3_nic_reset_all_ring(priv->ae_handle); + + hns3_reset_tx_queue(priv->ae_handle); +} + +void hns3_external_lb_restore(struct net_device *ndev, bool if_running) +{ + struct hns3_nic_priv *priv = netdev_priv(ndev); + struct hnae3_handle *h = priv->ae_handle; + int i; + + if (!if_running) + return; + + hns3_nic_reset_all_ring(priv->ae_handle); + + for (i = 0; i < priv->vector_num; i++) + hns3_vector_enable(&priv->tqp_vector[i]); + + for (i = 0; i < h->kinfo.num_tqps; i++) + hns3_tqp_enable(h->kinfo.tqp[i]); + + netif_tx_wake_all_queues(ndev); + + if (h->ae_algo->ops->get_status(h)) + netif_carrier_on(ndev); +} + static const struct hns3_hw_error_info hns3_hw_err[] = { { .type = HNAE3_PPU_POISON_ERROR, .msg = "PPU poison" }, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 4a3253692dcc8e167503d497e9807ef8774bb727..133a054af6b7c73e6a312b9f66180503507b8d5b 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -744,4 +744,7 @@ u16 hns3_get_max_available_channels(struct hnae3_handle *h); void hns3_cq_period_mode_init(struct hns3_nic_priv *priv, enum dim_cq_period_mode tx_mode, enum dim_cq_period_mode rx_mode); + +void hns3_external_lb_prepare(struct net_device *ndev, bool if_running); +void hns3_external_lb_restore(struct net_device *ndev, bool if_running); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index 4c7988e308a2ff0f0465ce9cb1c578ef0859e502..cdf76fb58d45ed8364a79cd786a96e24bbc06783 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -69,7 +69,6 @@ static const struct hns3_stats hns3_rxq_stats[] = { #define HNS3_TQP_STATS_COUNT (HNS3_TXQ_STATS_COUNT + HNS3_RXQ_STATS_COUNT) -#define HNS3_SELF_TEST_TYPE_NUM 4 #define HNS3_NIC_LB_TEST_PKT_NUM 1 #define HNS3_NIC_LB_TEST_RING_ID 0 #define HNS3_NIC_LB_TEST_PACKET_SIZE 128 @@ -95,6 +94,7 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop, bool en) case HNAE3_LOOP_PARALLEL_SERDES: case HNAE3_LOOP_APP: case HNAE3_LOOP_PHY: + case HNAE3_LOOP_EXTERNAL: ret = h->ae_algo->ops->set_loopback(h, loop, en); break; default: @@ -304,6 +304,10 @@ out: static void hns3_set_selftest_param(struct hnae3_handle *h, int (*st_param)[2]) { + st_param[HNAE3_LOOP_EXTERNAL][0] = HNAE3_LOOP_EXTERNAL; + st_param[HNAE3_LOOP_EXTERNAL][1] = + h->flags & HNAE3_SUPPORT_EXTERNAL_LOOPBACK; + st_param[HNAE3_LOOP_APP][0] = HNAE3_LOOP_APP; st_param[HNAE3_LOOP_APP][1] = h->flags & HNAE3_SUPPORT_APP_LOOPBACK; @@ -322,17 +326,11 @@ static void hns3_set_selftest_param(struct hnae3_handle *h, int (*st_param)[2]) h->flags & HNAE3_SUPPORT_PHY_LOOPBACK; } -static void hns3_selftest_prepare(struct net_device *ndev, - bool if_running, int (*st_param)[2]) +static void hns3_selftest_prepare(struct net_device *ndev, bool if_running) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = priv->ae_handle; - if (netif_msg_ifdown(h)) - netdev_info(ndev, "self test start\n"); - - hns3_set_selftest_param(h, st_param); - if (if_running) ndev->netdev_ops->ndo_stop(ndev); @@ -371,18 +369,15 @@ static void hns3_selftest_restore(struct net_device *ndev, bool if_running) if (if_running) ndev->netdev_ops->ndo_open(ndev); - - if (netif_msg_ifdown(h)) - netdev_info(ndev, "self test end\n"); } static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2], struct ethtool_test *eth_test, u64 *data) { - int test_index = 0; + int test_index = HNAE3_LOOP_APP; u32 i; - for (i = 0; i < HNS3_SELF_TEST_TYPE_NUM; i++) { + for (i = HNAE3_LOOP_APP; i < HNAE3_LOOP_NONE; i++) { enum hnae3_loop loop_type = (enum hnae3_loop)st_param[i][0]; if (!st_param[i][1]) @@ -401,6 +396,20 @@ static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2], } } +static void hns3_do_external_lb(struct net_device *ndev, + struct ethtool_test *eth_test, u64 *data) +{ + data[HNAE3_LOOP_EXTERNAL] = hns3_lp_up(ndev, HNAE3_LOOP_EXTERNAL); + if (!data[HNAE3_LOOP_EXTERNAL]) + data[HNAE3_LOOP_EXTERNAL] = hns3_lp_run_test(ndev, HNAE3_LOOP_EXTERNAL); + hns3_lp_down(ndev, HNAE3_LOOP_EXTERNAL); + + if (data[HNAE3_LOOP_EXTERNAL]) + eth_test->flags |= ETH_TEST_FL_FAILED; + + eth_test->flags |= ETH_TEST_FL_EXTERNAL_LB_DONE; +} + /** * hns3_self_test - self test * @ndev: net device @@ -410,7 +419,9 @@ static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2], static void hns3_self_test(struct net_device *ndev, struct ethtool_test *eth_test, u64 *data) { - int st_param[HNS3_SELF_TEST_TYPE_NUM][2]; + struct hns3_nic_priv *priv = netdev_priv(ndev); + struct hnae3_handle *h = priv->ae_handle; + int st_param[HNAE3_LOOP_NONE][2]; bool if_running = netif_running(ndev); if (hns3_nic_resetting(ndev)) { @@ -418,13 +429,29 @@ static void hns3_self_test(struct net_device *ndev, return; } - /* Only do offline selftest, or pass by default */ - if (eth_test->flags != ETH_TEST_FL_OFFLINE) + if (!(eth_test->flags & ETH_TEST_FL_OFFLINE)) return; - hns3_selftest_prepare(ndev, if_running, st_param); + if (netif_msg_ifdown(h)) + netdev_info(ndev, "self test start\n"); + + hns3_set_selftest_param(h, st_param); + + /* external loopback test requires that the link is up and the duplex is + * full, do external test first to reduce the whole test time + */ + if (eth_test->flags & ETH_TEST_FL_EXTERNAL_LB) { + hns3_external_lb_prepare(ndev, if_running); + hns3_do_external_lb(ndev, eth_test, data); + hns3_external_lb_restore(ndev, if_running); + } + + hns3_selftest_prepare(ndev, if_running); hns3_do_selftest(ndev, st_param, eth_test, data); hns3_selftest_restore(ndev, if_running); + + if (netif_msg_ifdown(h)) + netdev_info(ndev, "self test end\n"); } static void hns3_update_limit_promisc_mode(struct net_device *netdev, @@ -712,7 +739,8 @@ static void hns3_get_ksettings(struct hnae3_handle *h, ops->get_ksettings_an_result(h, &cmd->base.autoneg, &cmd->base.speed, - &cmd->base.duplex); + &cmd->base.duplex, + &cmd->lanes); /* 2.get link mode */ if (ops->get_link_mode) @@ -794,6 +822,7 @@ static int hns3_check_ksettings_param(const struct net_device *netdev, const struct hnae3_ae_ops *ops = handle->ae_algo->ops; u8 module_type = HNAE3_MODULE_TYPE_UNKNOWN; u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN; + u32 lane_num; u8 autoneg; u32 speed; u8 duplex; @@ -806,9 +835,9 @@ static int hns3_check_ksettings_param(const struct net_device *netdev, return 0; if (ops->get_ksettings_an_result) { - ops->get_ksettings_an_result(handle, &autoneg, &speed, &duplex); + ops->get_ksettings_an_result(handle, &autoneg, &speed, &duplex, &lane_num); if (cmd->base.autoneg == autoneg && cmd->base.speed == speed && - cmd->base.duplex == duplex) + cmd->base.duplex == duplex && cmd->lanes == lane_num) return 0; } @@ -845,10 +874,14 @@ static int hns3_set_link_ksettings(struct net_device *netdev, if (cmd->base.speed == SPEED_1000 && cmd->base.duplex == DUPLEX_HALF) return -EINVAL; + if (cmd->lanes && !hnae3_ae_dev_lane_num_supported(ae_dev)) + return -EOPNOTSUPP; + netif_dbg(handle, drv, netdev, - "set link(%s): autoneg=%u, speed=%u, duplex=%u\n", + "set link(%s): autoneg=%u, speed=%u, duplex=%u, lanes=%u\n", netdev->phydev ? "phy" : "mac", - cmd->base.autoneg, cmd->base.speed, cmd->base.duplex); + cmd->base.autoneg, cmd->base.speed, cmd->base.duplex, + cmd->lanes); /* Only support ksettings_set for netdev with phy attached for now */ if (netdev->phydev) { @@ -886,7 +919,7 @@ static int hns3_set_link_ksettings(struct net_device *netdev, if (ops->cfg_mac_speed_dup_h) ret = ops->cfg_mac_speed_dup_h(handle, cmd->base.speed, - cmd->base.duplex); + cmd->base.duplex, (u8)(cmd->lanes)); return ret; } @@ -1612,6 +1645,19 @@ static void hns3_set_msglevel(struct net_device *netdev, u32 msg_level) h->msg_enable = msg_level; } +static void hns3_get_fec_stats(struct net_device *netdev, + struct ethtool_fec_stats *fec_stats) +{ + struct hnae3_handle *handle = hns3_get_handle(netdev); + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); + const struct hnae3_ae_ops *ops = handle->ae_algo->ops; + + if (!hnae3_ae_dev_fec_stats_supported(ae_dev) || !ops->get_fec_stats) + return; + + ops->get_fec_stats(handle, fec_stats); +} + /* Translate local fec value into ethtool value. */ static unsigned int loc_to_eth_fec(u8 loc_fec) { @@ -1621,12 +1667,12 @@ static unsigned int loc_to_eth_fec(u8 loc_fec) eth_fec |= ETHTOOL_FEC_AUTO; if (loc_fec & BIT(HNAE3_FEC_RS)) eth_fec |= ETHTOOL_FEC_RS; + if (loc_fec & BIT(HNAE3_FEC_LLRS)) + eth_fec |= ETHTOOL_FEC_LLRS; if (loc_fec & BIT(HNAE3_FEC_BASER)) eth_fec |= ETHTOOL_FEC_BASER; - - /* if nothing is set, then FEC is off */ - if (!eth_fec) - eth_fec = ETHTOOL_FEC_OFF; + if (loc_fec & BIT(HNAE3_FEC_NONE)) + eth_fec |= ETHTOOL_FEC_OFF; return eth_fec; } @@ -1637,12 +1683,13 @@ static unsigned int eth_to_loc_fec(unsigned int eth_fec) u32 loc_fec = 0; if (eth_fec & ETHTOOL_FEC_OFF) - return loc_fec; - + loc_fec |= BIT(HNAE3_FEC_NONE); if (eth_fec & ETHTOOL_FEC_AUTO) loc_fec |= BIT(HNAE3_FEC_AUTO); if (eth_fec & ETHTOOL_FEC_RS) loc_fec |= BIT(HNAE3_FEC_RS); + if (eth_fec & ETHTOOL_FEC_LLRS) + loc_fec |= BIT(HNAE3_FEC_LLRS); if (eth_fec & ETHTOOL_FEC_BASER) loc_fec |= BIT(HNAE3_FEC_BASER); @@ -1668,6 +1715,8 @@ static int hns3_get_fecparam(struct net_device *netdev, fec->fec = loc_to_eth_fec(fec_ability); fec->active_fec = loc_to_eth_fec(fec_mode); + if (!fec->active_fec) + fec->active_fec = ETHTOOL_FEC_OFF; return 0; } @@ -2051,6 +2100,7 @@ static const struct ethtool_ops hns3vf_ethtool_ops = { static const struct ethtool_ops hns3_ethtool_ops = { .supported_coalesce_params = HNS3_ETHTOOL_COALESCE, .supported_ring_params = HNS3_ETHTOOL_RING, + .cap_link_lanes_supported = true, .self_test = hns3_self_test, .get_drvinfo = hns3_get_drvinfo, .get_link = hns3_get_link, @@ -2081,6 +2131,7 @@ static const struct ethtool_ops hns3_ethtool_ops = { .set_msglevel = hns3_set_msglevel, .get_fecparam = hns3_get_fecparam, .set_fecparam = hns3_set_fecparam, + .get_fec_stats = hns3_get_fec_stats, .get_module_info = hns3_get_module_info, .get_module_eeprom = hns3_get_module_eeprom, .get_priv_flags = hns3_get_priv_flags, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h index f9d89511eb328421903dec8064d27bd27a5af24d..43cada51d8cb31f6576966a551510a50ba5a0f2e 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h @@ -321,7 +321,9 @@ struct hclge_config_mac_speed_dup_cmd { #define HCLGE_CFG_MAC_SPEED_CHANGE_EN_B 0 u8 mac_change_fec_en; - u8 rsv[22]; + u8 rsv[4]; + u8 lane_num; + u8 rsv1[17]; }; #define HCLGE_TQP_ENABLE_B 0 @@ -347,7 +349,9 @@ struct hclge_sfp_info_cmd { u8 autoneg_ability; /* whether support autoneg */ __le32 speed_ability; /* speed ability for current media */ __le32 module_type; - u8 rsv[8]; + u8 fec_ability; + u8 lane_num; + u8 rsv[6]; }; #define HCLGE_MAC_CFG_FEC_AUTO_EN_B 0 @@ -359,12 +363,27 @@ struct hclge_sfp_info_cmd { #define HCLGE_MAC_FEC_OFF 0 #define HCLGE_MAC_FEC_BASER 1 #define HCLGE_MAC_FEC_RS 2 +#define HCLGE_MAC_FEC_LLRS 3 struct hclge_config_fec_cmd { u8 fec_mode; u8 default_config; u8 rsv[22]; }; +#define HCLGE_FEC_STATS_CMD_NUM 4 + +struct hclge_query_fec_stats_cmd { + /* fec rs mode total stats */ + __le32 rs_fec_corr_blocks; + __le32 rs_fec_uncorr_blocks; + __le32 rs_fec_error_blocks; + /* fec base-r mode per lanes stats */ + u8 base_r_lane_num; + u8 rsv[3]; + __le32 base_r_fec_corr_blocks; + __le32 base_r_fec_uncorr_blocks; +}; + #define HCLGE_MAC_UPLINK_PORT 0x100 struct hclge_config_max_frm_size_cmd { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c index 69b8673436ca9a6da5bc43722c8a770a31307c44..c4aded65e848bff4a2d30aa2b23de555e1fb0158 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c @@ -359,6 +359,93 @@ static int hclge_ieee_setpfc(struct hnae3_handle *h, struct ieee_pfc *pfc) return hclge_notify_client(hdev, HNAE3_UP_CLIENT); } +static int hclge_ieee_setapp(struct hnae3_handle *h, struct dcb_app *app) +{ + struct hclge_vport *vport = hclge_get_vport(h); + struct net_device *netdev = h->kinfo.netdev; + struct hclge_dev *hdev = vport->back; + struct dcb_app old_app; + int ret; + + if (app->selector != IEEE_8021QAZ_APP_SEL_DSCP || + app->protocol >= HNAE3_MAX_DSCP || + app->priority >= HNAE3_MAX_USER_PRIO) + return -EINVAL; + + dev_info(&hdev->pdev->dev, "setapp dscp=%u priority=%u\n", + app->protocol, app->priority); + + if (app->priority == h->kinfo.dscp_prio[app->protocol]) + return 0; + + ret = dcb_ieee_setapp(netdev, app); + if (ret) + return ret; + + old_app.selector = IEEE_8021QAZ_APP_SEL_DSCP; + old_app.protocol = app->protocol; + old_app.priority = h->kinfo.dscp_prio[app->protocol]; + + h->kinfo.dscp_prio[app->protocol] = app->priority; + ret = hclge_dscp_to_tc_map(hdev); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to set dscp to tc map, ret = %d\n", ret); + h->kinfo.dscp_prio[app->protocol] = old_app.priority; + (void)dcb_ieee_delapp(netdev, app); + return ret; + } + + vport->nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_DSCP; + if (old_app.priority == HNAE3_PRIO_ID_INVALID) + h->kinfo.dscp_app_cnt++; + else + ret = dcb_ieee_delapp(netdev, &old_app); + + return ret; +} + +static int hclge_ieee_delapp(struct hnae3_handle *h, struct dcb_app *app) +{ + struct hclge_vport *vport = hclge_get_vport(h); + struct net_device *netdev = h->kinfo.netdev; + struct hclge_dev *hdev = vport->back; + int ret; + + if (app->selector != IEEE_8021QAZ_APP_SEL_DSCP || + app->protocol >= HNAE3_MAX_DSCP || + app->priority >= HNAE3_MAX_USER_PRIO || + app->priority != h->kinfo.dscp_prio[app->protocol]) + return -EINVAL; + + dev_info(&hdev->pdev->dev, "delapp dscp=%u priority=%u\n", + app->protocol, app->priority); + + ret = dcb_ieee_delapp(netdev, app); + if (ret) + return ret; + + h->kinfo.dscp_prio[app->protocol] = HNAE3_PRIO_ID_INVALID; + ret = hclge_dscp_to_tc_map(hdev); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to del dscp to tc map, ret = %d\n", ret); + h->kinfo.dscp_prio[app->protocol] = app->priority; + (void)dcb_ieee_setapp(netdev, app); + return ret; + } + + if (h->kinfo.dscp_app_cnt) + h->kinfo.dscp_app_cnt--; + + if (!h->kinfo.dscp_app_cnt) { + vport->nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_PRIO; + ret = hclge_up_to_tc_map(hdev); + } + + return ret; +} + /* DCBX configuration */ static u8 hclge_getdcbx(struct hnae3_handle *h) { @@ -543,6 +630,8 @@ static const struct hnae3_dcb_ops hns3_dcb_ops = { .ieee_setets = hclge_ieee_setets, .ieee_getpfc = hclge_ieee_getpfc, .ieee_setpfc = hclge_ieee_setpfc, + .ieee_setapp = hclge_ieee_setapp, + .ieee_delapp = hclge_ieee_delapp, .getdcbx = hclge_getdcbx, .setdcbx = hclge_setdcbx, .setup_tc = hclge_setup_tc, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 9b870e79c290c318e7d15943807134730641296d..142415c84c6b2ded65bd1f16f622dd30c53e2f3c 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -14,6 +14,8 @@ static const char * const hclge_mac_state_str[] = { "TO_ADD", "TO_DEL", "ACTIVE" }; +static const char * const tc_map_mode_str[] = { "PRIO", "DSCP" }; + static const struct hclge_dbg_reg_type_info hclge_dbg_reg_info[] = { { .cmd = HNAE3_DBG_CMD_REG_BIOS_COMMON, .dfx_msg = &hclge_dbg_bios_common_reg[0], @@ -1115,10 +1117,11 @@ static int hclge_dbg_dump_qos_pause_cfg(struct hclge_dev *hdev, char *buf, return 0; } +#define HCLGE_DBG_TC_MASK 0x0F + static int hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev, char *buf, int len) { -#define HCLGE_DBG_TC_MASK 0x0F #define HCLGE_DBG_TC_BIT_WIDTH 4 struct hclge_qos_pri_map_cmd *pri_map; @@ -1152,6 +1155,58 @@ static int hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev, char *buf, return 0; } +static int hclge_dbg_dump_qos_dscp_map(struct hclge_dev *hdev, char *buf, + int len) +{ + struct hnae3_knic_private_info *kinfo = &hdev->vport[0].nic.kinfo; + struct hclge_desc desc[HCLGE_DSCP_MAP_TC_BD_NUM]; + u8 *req0 = (u8 *)desc[0].data; + u8 *req1 = (u8 *)desc[1].data; + u8 dscp_tc[HNAE3_MAX_DSCP]; + int pos, ret; + u8 i, j; + + pos = scnprintf(buf, len, "tc map mode: %s\n", + tc_map_mode_str[kinfo->tc_map_mode]); + + if (kinfo->tc_map_mode != HNAE3_TC_MAP_MODE_DSCP) + return 0; + + hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QOS_MAP, true); + desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_QOS_MAP, true); + ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_DSCP_MAP_TC_BD_NUM); + if (ret) { + dev_err(&hdev->pdev->dev, + "failed to dump qos dscp map, ret = %d\n", ret); + return ret; + } + + pos += scnprintf(buf + pos, len - pos, "\nDSCP PRIO TC\n"); + + /* The low 32 dscp setting use bd0, high 32 dscp setting use bd1 */ + for (i = 0; i < HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; i++) { + j = i + HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; + /* Each dscp setting has 4 bits, so each byte saves two dscp + * setting + */ + dscp_tc[i] = req0[i >> 1] >> HCLGE_DSCP_TC_SHIFT(i); + dscp_tc[j] = req1[i >> 1] >> HCLGE_DSCP_TC_SHIFT(i); + dscp_tc[i] &= HCLGE_DBG_TC_MASK; + dscp_tc[j] &= HCLGE_DBG_TC_MASK; + } + + for (i = 0; i < HNAE3_MAX_DSCP; i++) { + if (kinfo->dscp_prio[i] == HNAE3_PRIO_ID_INVALID) + continue; + + pos += scnprintf(buf + pos, len - pos, " %2u %u %u\n", + i, kinfo->dscp_prio[i], dscp_tc[i]); + } + + return 0; +} + static int hclge_dbg_dump_tx_buf_cfg(struct hclge_dev *hdev, char *buf, int len) { struct hclge_tx_buff_alloc_cmd *tx_buf_cmd; @@ -1517,7 +1572,7 @@ static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len) char *tcam_buf; int pos = 0; - if (!hnae3_dev_fd_supported(hdev)) { + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { dev_err(&hdev->pdev->dev, "Only FD-supported dev supports dump fd tcam\n"); return -EOPNOTSUPP; @@ -1585,6 +1640,9 @@ static int hclge_dbg_dump_fd_counter(struct hclge_dev *hdev, char *buf, int len) u64 cnt; u8 i; + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + pos += scnprintf(buf + pos, len - pos, "func_id\thit_times\n"); @@ -2373,6 +2431,10 @@ static const struct hclge_dbg_func hclge_dbg_cmd_func[] = { .cmd = HNAE3_DBG_CMD_QOS_PRI_MAP, .dbg_dump = hclge_dbg_dump_qos_pri_map, }, + { + .cmd = HNAE3_DBG_CMD_QOS_DSCP_MAP, + .dbg_dump = hclge_dbg_dump_qos_dscp_map, + }, { .cmd = HNAE3_DBG_CMD_QOS_BUF_CFG, .dbg_dump = hclge_dbg_dump_qos_buf_cfg, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index fae79764dc4427647189962ea003775ece4f0e78..6962a9d69cf8da937513b9460a4da07dd984760a 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -71,6 +71,7 @@ static void hclge_sync_mac_table(struct hclge_dev *hdev); static void hclge_restore_hw_table(struct hclge_dev *hdev); static void hclge_sync_promisc_mode(struct hclge_dev *hdev); static void hclge_sync_fd_table(struct hclge_dev *hdev); +static void hclge_update_fec_stats(struct hclge_dev *hdev); static struct hnae3_ae_algo ae_algo; @@ -148,10 +149,11 @@ static const u32 tqp_intr_reg_addr_list[] = {HCLGE_TQP_INTR_CTRL_REG, HCLGE_TQP_INTR_RL_REG}; static const char hns3_nic_test_strs[][ETH_GSTRING_LEN] = { - "App Loopback test", - "Serdes serial Loopback test", - "Serdes parallel Loopback test", - "Phy Loopback test" + "External Loopback test", + "App Loopback test", + "Serdes serial Loopback test", + "Serdes parallel Loopback test", + "Phy Loopback test" }; static const struct hclge_comm_stats_str g_mac_stats_string[] = { @@ -679,6 +681,8 @@ static void hclge_update_stats_for_all(struct hclge_dev *hdev) } } + hclge_update_fec_stats(hdev); + status = hclge_mac_update_stats(hdev); if (status) dev_err(&hdev->pdev->dev, @@ -715,7 +719,8 @@ static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset) #define HCLGE_LOOPBACK_TEST_FLAGS (HNAE3_SUPPORT_APP_LOOPBACK | \ HNAE3_SUPPORT_PHY_LOOPBACK | \ HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK | \ - HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK) + HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK | \ + HNAE3_SUPPORT_EXTERNAL_LOOPBACK) struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; @@ -737,9 +742,12 @@ static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset) handle->flags |= HNAE3_SUPPORT_APP_LOOPBACK; } - count += 2; + count += 1; handle->flags |= HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK; + count += 1; handle->flags |= HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK; + count += 1; + handle->flags |= HNAE3_SUPPORT_EXTERNAL_LOOPBACK; if ((hdev->hw.mac.phydev && hdev->hw.mac.phydev->drv && hdev->hw.mac.phydev->drv->set_loopback) || @@ -770,6 +778,11 @@ static void hclge_get_strings(struct hnae3_handle *handle, u32 stringset, size, p); p = hclge_comm_tqps_get_strings(handle, p); } else if (stringset == ETH_SS_TEST) { + if (handle->flags & HNAE3_SUPPORT_EXTERNAL_LOOPBACK) { + memcpy(p, hns3_nic_test_strs[HNAE3_LOOP_EXTERNAL], + ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } if (handle->flags & HNAE3_SUPPORT_APP_LOOPBACK) { memcpy(p, hns3_nic_test_strs[HNAE3_LOOP_APP], ETH_GSTRING_LEN); @@ -1003,6 +1016,27 @@ static int hclge_check_port_speed(struct hnae3_handle *handle, u32 speed) return -EINVAL; } +static void hclge_update_fec_support(struct hclge_mac *mac) +{ + linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, mac->supported); + linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported); + linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, mac->supported); + linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, mac->supported); + + if (mac->fec_ability & BIT(HNAE3_FEC_BASER)) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, + mac->supported); + if (mac->fec_ability & BIT(HNAE3_FEC_RS)) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, + mac->supported); + if (mac->fec_ability & BIT(HNAE3_FEC_LLRS)) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, + mac->supported); + if (mac->fec_ability & BIT(HNAE3_FEC_NONE)) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, + mac->supported); +} + static void hclge_convert_setting_sr(u16 speed_ability, unsigned long *link_mode) { @@ -1101,34 +1135,36 @@ static void hclge_convert_setting_kr(u16 speed_ability, static void hclge_convert_setting_fec(struct hclge_mac *mac) { - linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, mac->supported); - linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported); + /* If firmware has reported fec_ability, don't need to convert by speed */ + if (mac->fec_ability) + goto out; switch (mac->speed) { case HCLGE_MAC_SPEED_10G: case HCLGE_MAC_SPEED_40G: - linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, - mac->supported); - mac->fec_ability = - BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_AUTO); + mac->fec_ability = BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_AUTO) | + BIT(HNAE3_FEC_NONE); break; case HCLGE_MAC_SPEED_25G: case HCLGE_MAC_SPEED_50G: - linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, - mac->supported); - mac->fec_ability = - BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_RS) | - BIT(HNAE3_FEC_AUTO); + mac->fec_ability = BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_RS) | + BIT(HNAE3_FEC_AUTO) | BIT(HNAE3_FEC_NONE); break; case HCLGE_MAC_SPEED_100G: + mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO) | + BIT(HNAE3_FEC_NONE); + break; case HCLGE_MAC_SPEED_200G: - linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported); - mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO); + mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO) | + BIT(HNAE3_FEC_LLRS); break; default: mac->fec_ability = 0; break; } + +out: + hclge_update_fec_support(mac); } static void hclge_parse_fiber_link_mode(struct hclge_dev *hdev, @@ -1574,7 +1610,7 @@ static int hclge_configure(struct hclge_dev *hdev) if (cfg.vlan_fliter_cap == HCLGE_VLAN_FLTR_CAN_MDF) set_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps); - if (hnae3_dev_fd_supported(hdev)) { + if (hnae3_ae_dev_fd_supported(hdev->ae_dev)) { hdev->fd_en = true; hdev->fd_active_type = HCLGE_FD_RULE_NONE; } @@ -1617,7 +1653,7 @@ static int hclge_config_gro(struct hclge_dev *hdev) struct hclge_desc desc; int ret; - if (!hnae3_dev_gro_supported(hdev)) + if (!hnae3_ae_dev_gro_supported(hdev->ae_dev)) return 0; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_GRO_GENERIC_CONFIG, false); @@ -2589,7 +2625,7 @@ static int hclge_convert_to_fw_speed(u32 speed_drv, u32 *speed_fw) } static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, - u8 duplex) + u8 duplex, u8 lane_num) { struct hclge_config_mac_speed_dup_cmd *req; struct hclge_desc desc; @@ -2613,6 +2649,7 @@ static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, speed_fw); hnae3_set_bit(req->mac_change_fec_en, HCLGE_CFG_MAC_SPEED_CHANGE_EN_B, 1); + req->lane_num = lane_num; ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { @@ -2624,33 +2661,35 @@ static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, return 0; } -int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex) +int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex, u8 lane_num) { struct hclge_mac *mac = &hdev->hw.mac; int ret; duplex = hclge_check_speed_dup(duplex, speed); if (!mac->support_autoneg && mac->speed == speed && - mac->duplex == duplex) + mac->duplex == duplex && (mac->lane_num == lane_num || lane_num == 0)) return 0; - ret = hclge_cfg_mac_speed_dup_hw(hdev, speed, duplex); + ret = hclge_cfg_mac_speed_dup_hw(hdev, speed, duplex, lane_num); if (ret) return ret; hdev->hw.mac.speed = speed; hdev->hw.mac.duplex = duplex; + if (!lane_num) + hdev->hw.mac.lane_num = lane_num; return 0; } static int hclge_cfg_mac_speed_dup_h(struct hnae3_handle *handle, int speed, - u8 duplex) + u8 duplex, u8 lane_num) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - return hclge_cfg_mac_speed_dup(hdev, speed, duplex); + return hclge_cfg_mac_speed_dup(hdev, speed, duplex, lane_num); } static int hclge_set_autoneg_en(struct hclge_dev *hdev, bool enable) @@ -2730,6 +2769,157 @@ static int hclge_halt_autoneg(struct hnae3_handle *handle, bool halt) return 0; } +static void hclge_parse_fec_stats_lanes(struct hclge_dev *hdev, + struct hclge_desc *desc, u32 desc_len) +{ + u32 lane_size = HCLGE_FEC_STATS_MAX_LANES * 2; + u32 desc_index = 0; + u32 data_index = 0; + u32 i; + + for (i = 0; i < lane_size; i++) { + if (data_index >= HCLGE_DESC_DATA_LEN) { + desc_index++; + data_index = 0; + } + + if (desc_index >= desc_len) + return; + + hdev->fec_stats.per_lanes[i] += + le32_to_cpu(desc[desc_index].data[data_index]); + data_index++; + } +} + +static void hclge_parse_fec_stats(struct hclge_dev *hdev, + struct hclge_desc *desc, u32 desc_len) +{ + struct hclge_query_fec_stats_cmd *req; + + req = (struct hclge_query_fec_stats_cmd *)desc[0].data; + + hdev->fec_stats.base_r_lane_num = req->base_r_lane_num; + hdev->fec_stats.rs_corr_blocks += + le32_to_cpu(req->rs_fec_corr_blocks); + hdev->fec_stats.rs_uncorr_blocks += + le32_to_cpu(req->rs_fec_uncorr_blocks); + hdev->fec_stats.rs_error_blocks += + le32_to_cpu(req->rs_fec_error_blocks); + hdev->fec_stats.base_r_corr_blocks += + le32_to_cpu(req->base_r_fec_corr_blocks); + hdev->fec_stats.base_r_uncorr_blocks += + le32_to_cpu(req->base_r_fec_uncorr_blocks); + + hclge_parse_fec_stats_lanes(hdev, &desc[1], desc_len - 1); +} + +static int hclge_update_fec_stats_hw(struct hclge_dev *hdev) +{ + struct hclge_desc desc[HCLGE_FEC_STATS_CMD_NUM]; + int ret; + u32 i; + + for (i = 0; i < HCLGE_FEC_STATS_CMD_NUM; i++) { + hclge_cmd_setup_basic_desc(&desc[i], HCLGE_OPC_QUERY_FEC_STATS, + true); + if (i != (HCLGE_FEC_STATS_CMD_NUM - 1)) + desc[i].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + } + + ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_FEC_STATS_CMD_NUM); + if (ret) + return ret; + + hclge_parse_fec_stats(hdev, desc, HCLGE_FEC_STATS_CMD_NUM); + + return 0; +} + +static void hclge_update_fec_stats(struct hclge_dev *hdev) +{ + struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); + int ret; + + if (!hnae3_ae_dev_fec_stats_supported(ae_dev) || + test_and_set_bit(HCLGE_STATE_FEC_STATS_UPDATING, &hdev->state)) + return; + + ret = hclge_update_fec_stats_hw(hdev); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to update fec stats, ret = %d\n", ret); + + clear_bit(HCLGE_STATE_FEC_STATS_UPDATING, &hdev->state); +} + +static void hclge_get_fec_stats_total(struct hclge_dev *hdev, + struct ethtool_fec_stats *fec_stats) +{ + fec_stats->corrected_blocks.total = hdev->fec_stats.rs_corr_blocks; + fec_stats->uncorrectable_blocks.total = + hdev->fec_stats.rs_uncorr_blocks; +} + +static void hclge_get_fec_stats_lanes(struct hclge_dev *hdev, + struct ethtool_fec_stats *fec_stats) +{ + u32 i; + + if (hdev->fec_stats.base_r_lane_num == 0 || + hdev->fec_stats.base_r_lane_num > HCLGE_FEC_STATS_MAX_LANES) { + dev_err(&hdev->pdev->dev, + "fec stats lane number(%llu) is invalid\n", + hdev->fec_stats.base_r_lane_num); + return; + } + + for (i = 0; i < hdev->fec_stats.base_r_lane_num; i++) { + fec_stats->corrected_blocks.lanes[i] = + hdev->fec_stats.base_r_corr_per_lanes[i]; + fec_stats->uncorrectable_blocks.lanes[i] = + hdev->fec_stats.base_r_uncorr_per_lanes[i]; + } +} + +static void hclge_comm_get_fec_stats(struct hclge_dev *hdev, + struct ethtool_fec_stats *fec_stats) +{ + u32 fec_mode = hdev->hw.mac.fec_mode; + + switch (fec_mode) { + case BIT(HNAE3_FEC_RS): + case BIT(HNAE3_FEC_LLRS): + hclge_get_fec_stats_total(hdev, fec_stats); + break; + case BIT(HNAE3_FEC_BASER): + hclge_get_fec_stats_lanes(hdev, fec_stats); + break; + default: + dev_err(&hdev->pdev->dev, + "fec stats is not supported by current fec mode(0x%x)\n", + fec_mode); + break; + } +} + +static void hclge_get_fec_stats(struct hnae3_handle *handle, + struct ethtool_fec_stats *fec_stats) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + struct hclge_dev *hdev = vport->back; + u32 fec_mode = hdev->hw.mac.fec_mode; + + if (fec_mode == BIT(HNAE3_FEC_NONE) || + fec_mode == BIT(HNAE3_FEC_AUTO) || + fec_mode == BIT(HNAE3_FEC_USER_DEF)) + return; + + hclge_update_fec_stats(hdev); + + hclge_comm_get_fec_stats(hdev, fec_stats); +} + static int hclge_set_fec_hw(struct hclge_dev *hdev, u32 fec_mode) { struct hclge_config_fec_cmd *req; @@ -2744,6 +2934,9 @@ static int hclge_set_fec_hw(struct hclge_dev *hdev, u32 fec_mode) if (fec_mode & BIT(HNAE3_FEC_RS)) hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M, HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_RS); + if (fec_mode & BIT(HNAE3_FEC_LLRS)) + hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M, + HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_LLRS); if (fec_mode & BIT(HNAE3_FEC_BASER)) hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M, HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_BASER); @@ -2796,7 +2989,7 @@ static int hclge_mac_init(struct hclge_dev *hdev) hdev->support_sfp_query = true; hdev->hw.mac.duplex = HCLGE_MAC_FULL; ret = hclge_cfg_mac_speed_dup_hw(hdev, hdev->hw.mac.speed, - hdev->hw.mac.duplex); + hdev->hw.mac.duplex, hdev->hw.mac.lane_num); if (ret) return ret; @@ -2988,6 +3181,9 @@ static void hclge_update_fec_advertising(struct hclge_mac *mac) if (mac->fec_mode & BIT(HNAE3_FEC_RS)) linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->advertising); + else if (mac->fec_mode & BIT(HNAE3_FEC_LLRS)) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, + mac->advertising); else if (mac->fec_mode & BIT(HNAE3_FEC_BASER)) linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, mac->advertising); @@ -3037,7 +3233,6 @@ static void hclge_update_port_capability(struct hclge_dev *hdev, struct hclge_mac *mac) { if (hnae3_dev_fec_supported(hdev)) - /* update fec ability by speed */ hclge_convert_setting_fec(mac); /* firmware can not identify back plane type, the media type @@ -3119,10 +3314,12 @@ static int hclge_get_sfp_info(struct hclge_dev *hdev, struct hclge_mac *mac) mac->autoneg = resp->autoneg; mac->support_autoneg = resp->autoneg_ability; mac->speed_type = QUERY_ACTIVE_SPEED; + mac->lane_num = resp->lane_num; if (!resp->active_fec) mac->fec_mode = 0; else mac->fec_mode = BIT(resp->active_fec); + mac->fec_ability = resp->fec_ability; } else { mac->speed_type = QUERY_SFP_SPEED; } @@ -3302,13 +3499,13 @@ static int hclge_update_port_info(struct hclge_dev *hdev) return 0; } return hclge_cfg_mac_speed_dup(hdev, mac->speed, - HCLGE_MAC_FULL); + HCLGE_MAC_FULL, mac->lane_num); } else { if (speed == HCLGE_MAC_SPEED_UNKNOWN) return 0; /* do nothing if no SFP */ /* must config full duplex for SFP */ - return hclge_cfg_mac_speed_dup(hdev, speed, HCLGE_MAC_FULL); + return hclge_cfg_mac_speed_dup(hdev, speed, HCLGE_MAC_FULL, 0); } } @@ -5334,7 +5531,7 @@ static int hclge_init_fd_config(struct hclge_dev *hdev) struct hclge_fd_key_cfg *key_cfg; int ret; - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return 0; ret = hclge_get_fd_mode(hdev, &hdev->fd_cfg.fd_mode); @@ -6339,7 +6536,7 @@ static int hclge_add_fd_entry(struct hnae3_handle *handle, u8 action; int ret; - if (!hnae3_dev_fd_supported(hdev)) { + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { dev_err(&hdev->pdev->dev, "flow table director is not supported\n"); return -EOPNOTSUPP; @@ -6395,7 +6592,7 @@ static int hclge_del_fd_entry(struct hnae3_handle *handle, struct ethtool_rx_flow_spec *fs; int ret; - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return -EOPNOTSUPP; fs = (struct ethtool_rx_flow_spec *)&cmd->fs; @@ -6431,9 +6628,6 @@ static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev, struct hlist_node *node; u16 location; - if (!hnae3_dev_fd_supported(hdev)) - return; - spin_lock_bh(&hdev->fd_rule_lock); for_each_set_bit(location, hdev->fd_bmap, @@ -6458,6 +6652,9 @@ static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev, static void hclge_del_all_fd_entries(struct hclge_dev *hdev) { + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return; + hclge_clear_fd_rules_in_list(hdev, true); hclge_fd_disable_user_def(hdev); } @@ -6473,7 +6670,7 @@ static int hclge_restore_fd_entries(struct hnae3_handle *handle) * return value. If error is returned here, the reset process will * fail. */ - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return 0; /* if fd is disabled, should not restore it when reset */ @@ -6497,7 +6694,7 @@ static int hclge_get_fd_rule_cnt(struct hnae3_handle *handle, struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - if (!hnae3_dev_fd_supported(hdev) || hclge_is_cls_flower_active(handle)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev) || hclge_is_cls_flower_active(handle)) return -EOPNOTSUPP; cmd->rule_cnt = hdev->hclge_fd_rule_num; @@ -6715,7 +6912,7 @@ static int hclge_get_fd_rule_info(struct hnae3_handle *handle, struct hclge_dev *hdev = vport->back; struct ethtool_rx_flow_spec *fs; - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return -EOPNOTSUPP; fs = (struct ethtool_rx_flow_spec *)&cmd->fs; @@ -6778,7 +6975,7 @@ static int hclge_get_all_rules(struct hnae3_handle *handle, struct hlist_node *node2; int cnt = 0; - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return -EOPNOTSUPP; cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]; @@ -6878,7 +7075,7 @@ static int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id, struct hclge_fd_rule *rule; u16 bit_id; - if (!hnae3_dev_fd_supported(hdev)) + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) return -EOPNOTSUPP; /* when there is already fd rule existed add by user, @@ -7167,6 +7364,12 @@ static int hclge_add_cls_flower(struct hnae3_handle *handle, struct hclge_fd_rule *rule; int ret; + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) { + dev_err(&hdev->pdev->dev, + "cls flower is not supported\n"); + return -EOPNOTSUPP; + } + ret = hclge_check_cls_flower(hdev, cls_flower, tc); if (ret) { dev_err(&hdev->pdev->dev, @@ -7220,6 +7423,9 @@ static int hclge_del_cls_flower(struct hnae3_handle *handle, struct hclge_fd_rule *rule; int ret; + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return -EOPNOTSUPP; + spin_lock_bh(&hdev->fd_rule_lock); rule = hclge_find_cls_flower(hdev, cls_flower->cookie); @@ -7282,6 +7488,9 @@ out: static void hclge_sync_fd_table(struct hclge_dev *hdev) { + if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) + return; + if (test_and_clear_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state)) { bool clear_list = hdev->fd_active_type == HCLGE_FD_ARFS_ACTIVE; @@ -7705,7 +7914,7 @@ static int hclge_set_loopback(struct hnae3_handle *handle, { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; - int ret; + int ret = 0; /* Loopback can be enabled in three places: SSU, MAC, and serdes. By * default, SSU loopback is enabled, so if the SMAC and the DMAC are @@ -7732,6 +7941,8 @@ static int hclge_set_loopback(struct hnae3_handle *handle, case HNAE3_LOOP_PHY: ret = hclge_set_phy_loopback(hdev, en); break; + case HNAE3_LOOP_EXTERNAL: + break; default: ret = -ENOTSUPP; dev_err(&hdev->pdev->dev, @@ -10793,7 +11004,7 @@ static int hclge_set_pauseparam(struct hnae3_handle *handle, u32 auto_neg, } static void hclge_get_ksettings_an_result(struct hnae3_handle *handle, - u8 *auto_neg, u32 *speed, u8 *duplex) + u8 *auto_neg, u32 *speed, u8 *duplex, u32 *lane_num) { struct hclge_vport *vport = hclge_get_vport(handle); struct hclge_dev *hdev = vport->back; @@ -10804,6 +11015,8 @@ static void hclge_get_ksettings_an_result(struct hnae3_handle *handle, *duplex = hdev->hw.mac.duplex; if (auto_neg) *auto_neg = hdev->hw.mac.autoneg; + if (lane_num) + *lane_num = hdev->hw.mac.lane_num; } static void hclge_get_media_type(struct hnae3_handle *handle, u8 *media_type, @@ -11443,6 +11656,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) if (ret) goto err_mdiobus_unreg; + ret = hclge_update_port_info(hdev); + if (ret) + goto err_mdiobus_unreg; + INIT_KFIFO(hdev->mac_tnl_log); hclge_dcb_ops_set(hdev); @@ -11510,6 +11727,7 @@ out: static void hclge_stats_clear(struct hclge_dev *hdev) { memset(&hdev->mac_stats, 0, sizeof(hdev->mac_stats)); + memset(&hdev->fec_stats, 0, sizeof(hdev->fec_stats)); } static int hclge_set_mac_spoofchk(struct hclge_dev *hdev, int vf, bool enable) @@ -12763,6 +12981,21 @@ static void hclge_clean_vport_config(struct hnae3_ae_dev *ae_dev, int num_vfs) } } +static int hclge_get_dscp_prio(struct hnae3_handle *h, u8 dscp, u8 *tc_mode, + u8 *priority) +{ + if (dscp >= HNAE3_MAX_DSCP) + return -EINVAL; + + if (tc_mode) + *tc_mode = h->kinfo.tc_map_mode; + if (priority) + *priority = h->kinfo.dscp_prio[dscp] == HNAE3_PRIO_ID_INVALID ? 0 : + h->kinfo.dscp_prio[dscp]; + + return 0; +} + static const struct hnae3_ae_ops hclge_ops = { .init_ae_dev = hclge_init_ae_dev, .uninit_ae_dev = hclge_uninit_ae_dev, @@ -12786,6 +13019,7 @@ static const struct hnae3_ae_ops hclge_ops = { .cfg_mac_speed_dup_h = hclge_cfg_mac_speed_dup_h, .get_media_type = hclge_get_media_type, .check_port_speed = hclge_check_port_speed, + .get_fec_stats = hclge_get_fec_stats, .get_fec = hclge_get_fec, .set_fec = hclge_set_fec, .get_rss_key_size = hclge_comm_get_rss_key_size, @@ -12865,6 +13099,7 @@ static const struct hnae3_ae_ops hclge_ops = { .get_ts_info = hclge_ptp_get_ts_info, .get_link_diagnosis_info = hclge_get_link_diagnosis_info, .clean_vf_config = hclge_clean_vport_config, + .get_dscp_prio = hclge_get_dscp_prio, }; static struct hnae3_ae_algo ae_algo = { @@ -12872,7 +13107,7 @@ static struct hnae3_ae_algo ae_algo = { .pdev_id_table = ae_algo_pci_tbl, }; -static int hclge_init(void) +static int __init hclge_init(void) { pr_info("%s is initializing\n", HCLGE_NAME); @@ -12887,7 +13122,7 @@ static int hclge_init(void) return 0; } -static void hclge_exit(void) +static void __exit hclge_exit(void) { hnae3_unregister_ae_algo_prepare(&ae_algo); hnae3_unregister_ae_algo(&ae_algo); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index 18caddd541f8a6f17e8eba975bfbf76296062517..495b639b0dc249d81d68cb13376fe3f96f6c8564 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -216,6 +216,7 @@ enum HCLGE_DEV_STATE { HCLGE_STATE_FD_USER_DEF_CHANGED, HCLGE_STATE_PTP_EN, HCLGE_STATE_PTP_TX_HANDLING, + HCLGE_STATE_FEC_STATS_UPDATING, HCLGE_STATE_MAX }; @@ -258,6 +259,7 @@ struct hclge_mac { u8 duplex; u8 support_autoneg; u8 speed_type; /* 0: sfp speed, 1: active speed */ + u8 lane_num; u32 speed; u32 max_speed; u32 speed_ability; /* speed ability supported by current media */ @@ -488,6 +490,26 @@ struct hclge_mac_stats { #define HCLGE_STATS_TIMER_INTERVAL 300UL +/* fec stats ,opcode id: 0x0316 */ +#define HCLGE_FEC_STATS_MAX_LANES 8 +struct hclge_fec_stats { + /* fec rs mode total stats */ + u64 rs_corr_blocks; + u64 rs_uncorr_blocks; + u64 rs_error_blocks; + /* fec base-r mode per lanes stats */ + u64 base_r_lane_num; + u64 base_r_corr_blocks; + u64 base_r_uncorr_blocks; + union { + struct { + u64 base_r_corr_per_lanes[HCLGE_FEC_STATS_MAX_LANES]; + u64 base_r_uncorr_per_lanes[HCLGE_FEC_STATS_MAX_LANES]; + }; + u64 per_lanes[HCLGE_FEC_STATS_MAX_LANES * 2]; + }; +}; + struct hclge_vlan_type_cfg { u16 rx_ot_fst_vlan_type; u16 rx_ot_sec_vlan_type; @@ -826,6 +848,7 @@ struct hclge_dev { struct hclge_hw hw; struct hclge_misc_vector misc_vector; struct hclge_mac_stats mac_stats; + struct hclge_fec_stats fec_stats; unsigned long state; unsigned long flr_state; unsigned long last_reset_time; @@ -1070,7 +1093,7 @@ static inline int hclge_get_queue_id(struct hnae3_queue *queue) } int hclge_inform_reset_assert_to_vf(struct hclge_vport *vport); -int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex); +int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex, u8 lane_num); int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto, u16 vlan_id, bool is_kill); int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index e1012f7f9b7349029383cbc13709f34cffd81e37..a7b06c63143cc2082207cfdde8038b4a620a4260 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -779,17 +779,284 @@ static void hclge_handle_vf_tbl(struct hclge_vport *vport, } } +static int +hclge_mbx_map_ring_to_vector_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_map_unmap_ring_to_vf_vector(param->vport, true, + param->req); +} + +static int +hclge_mbx_unmap_ring_to_vector_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_map_unmap_ring_to_vf_vector(param->vport, false, + param->req); +} + +static int +hclge_mbx_get_ring_vector_map_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_get_vf_ring_vector_map(param->vport, param->req, + param->resp_msg); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "PF fail(%d) to get VF ring vector map\n", + ret); + return ret; +} + +static int hclge_mbx_set_promisc_mode_handler(struct hclge_mbx_ops_param *param) +{ + hclge_set_vf_promisc_mode(param->vport, param->req); + return 0; +} + +static int hclge_mbx_set_unicast_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_set_vf_uc_mac_addr(param->vport, param->req); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "PF fail(%d) to set VF UC MAC Addr\n", + ret); + return ret; +} + +static int hclge_mbx_set_multicast_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_set_vf_mc_mac_addr(param->vport, param->req); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "PF fail(%d) to set VF MC MAC Addr\n", + ret); + return ret; +} + +static int hclge_mbx_set_vlan_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_set_vf_vlan_cfg(param->vport, param->req, param->resp_msg); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "PF failed(%d) to config VF's VLAN\n", + ret); + return ret; +} + +static int hclge_mbx_set_alive_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_set_vf_alive(param->vport, param->req); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "PF failed(%d) to set VF's ALIVE\n", + ret); + return ret; +} + +static int hclge_mbx_get_qinfo_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_vf_queue_info(param->vport, param->resp_msg); + return 0; +} + +static int hclge_mbx_get_qdepth_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_vf_queue_depth(param->vport, param->resp_msg); + return 0; +} + +static int hclge_mbx_get_basic_info_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_basic_info(param->vport, param->resp_msg); + return 0; +} + +static int hclge_mbx_get_link_status_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_push_vf_link_status(param->vport); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "failed to inform link stat to VF, ret = %d\n", + ret); + return ret; +} + +static int hclge_mbx_queue_reset_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_mbx_reset_vf_queue(param->vport, param->req, + param->resp_msg); +} + +static int hclge_mbx_reset_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_reset_vf(param->vport); +} + +static int hclge_mbx_keep_alive_handler(struct hclge_mbx_ops_param *param) +{ + hclge_vf_keep_alive(param->vport); + return 0; +} + +static int hclge_mbx_set_mtu_handler(struct hclge_mbx_ops_param *param) +{ + int ret; + + ret = hclge_set_vf_mtu(param->vport, param->req); + if (ret) + dev_err(¶m->vport->back->pdev->dev, + "VF fail(%d) to set mtu\n", ret); + return ret; +} + +static int hclge_mbx_get_qid_in_pf_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_get_queue_id_in_pf(param->vport, param->req, + param->resp_msg); +} + +static int hclge_mbx_get_rss_key_handler(struct hclge_mbx_ops_param *param) +{ + return hclge_get_rss_key(param->vport, param->req, param->resp_msg); +} + +static int hclge_mbx_get_link_mode_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_link_mode(param->vport, param->req); + return 0; +} + +static int +hclge_mbx_get_vf_flr_status_handler(struct hclge_mbx_ops_param *param) +{ + hclge_rm_vport_all_mac_table(param->vport, false, + HCLGE_MAC_ADDR_UC); + hclge_rm_vport_all_mac_table(param->vport, false, + HCLGE_MAC_ADDR_MC); + hclge_rm_vport_all_vlan_table(param->vport, false); + return 0; +} + +static int hclge_mbx_vf_uninit_handler(struct hclge_mbx_ops_param *param) +{ + hclge_rm_vport_all_mac_table(param->vport, true, + HCLGE_MAC_ADDR_UC); + hclge_rm_vport_all_mac_table(param->vport, true, + HCLGE_MAC_ADDR_MC); + hclge_rm_vport_all_vlan_table(param->vport, true); + return 0; +} + +static int hclge_mbx_get_media_type_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_vf_media_type(param->vport, param->resp_msg); + return 0; +} + +static int hclge_mbx_push_link_status_handler(struct hclge_mbx_ops_param *param) +{ + hclge_handle_link_change_event(param->vport->back, param->req); + return 0; +} + +static int hclge_mbx_get_mac_addr_handler(struct hclge_mbx_ops_param *param) +{ + hclge_get_vf_mac_addr(param->vport, param->resp_msg); + return 0; +} + +static int hclge_mbx_ncsi_error_handler(struct hclge_mbx_ops_param *param) +{ + hclge_handle_ncsi_error(param->vport->back); + return 0; +} + +static int hclge_mbx_handle_vf_tbl_handler(struct hclge_mbx_ops_param *param) +{ + hclge_handle_vf_tbl(param->vport, param->req); + return 0; +} + +static const hclge_mbx_ops_fn hclge_mbx_ops_list[HCLGE_MBX_OPCODE_MAX] = { + [HCLGE_MBX_RESET] = hclge_mbx_reset_handler, + [HCLGE_MBX_SET_UNICAST] = hclge_mbx_set_unicast_handler, + [HCLGE_MBX_SET_MULTICAST] = hclge_mbx_set_multicast_handler, + [HCLGE_MBX_SET_VLAN] = hclge_mbx_set_vlan_handler, + [HCLGE_MBX_MAP_RING_TO_VECTOR] = hclge_mbx_map_ring_to_vector_handler, + [HCLGE_MBX_UNMAP_RING_TO_VECTOR] = hclge_mbx_unmap_ring_to_vector_handler, + [HCLGE_MBX_SET_PROMISC_MODE] = hclge_mbx_set_promisc_mode_handler, + [HCLGE_MBX_GET_QINFO] = hclge_mbx_get_qinfo_handler, + [HCLGE_MBX_GET_QDEPTH] = hclge_mbx_get_qdepth_handler, + [HCLGE_MBX_GET_BASIC_INFO] = hclge_mbx_get_basic_info_handler, + [HCLGE_MBX_GET_RSS_KEY] = hclge_mbx_get_rss_key_handler, + [HCLGE_MBX_GET_MAC_ADDR] = hclge_mbx_get_mac_addr_handler, + [HCLGE_MBX_GET_LINK_STATUS] = hclge_mbx_get_link_status_handler, + [HCLGE_MBX_QUEUE_RESET] = hclge_mbx_queue_reset_handler, + [HCLGE_MBX_KEEP_ALIVE] = hclge_mbx_keep_alive_handler, + [HCLGE_MBX_SET_ALIVE] = hclge_mbx_set_alive_handler, + [HCLGE_MBX_SET_MTU] = hclge_mbx_set_mtu_handler, + [HCLGE_MBX_GET_QID_IN_PF] = hclge_mbx_get_qid_in_pf_handler, + [HCLGE_MBX_GET_LINK_MODE] = hclge_mbx_get_link_mode_handler, + [HCLGE_MBX_GET_MEDIA_TYPE] = hclge_mbx_get_media_type_handler, + [HCLGE_MBX_VF_UNINIT] = hclge_mbx_vf_uninit_handler, + [HCLGE_MBX_HANDLE_VF_TBL] = hclge_mbx_handle_vf_tbl_handler, + [HCLGE_MBX_GET_RING_VECTOR_MAP] = hclge_mbx_get_ring_vector_map_handler, + [HCLGE_MBX_GET_VF_FLR_STATUS] = hclge_mbx_get_vf_flr_status_handler, + [HCLGE_MBX_PUSH_LINK_STATUS] = hclge_mbx_push_link_status_handler, + [HCLGE_MBX_NCSI_ERROR] = hclge_mbx_ncsi_error_handler, +}; + +static void hclge_mbx_request_handling(struct hclge_mbx_ops_param *param) +{ + hclge_mbx_ops_fn cmd_func = NULL; + struct hclge_dev *hdev; + int ret = 0; + + hdev = param->vport->back; + cmd_func = hclge_mbx_ops_list[param->req->msg.code]; + if (cmd_func) + ret = cmd_func(param); + else + dev_err(&hdev->pdev->dev, + "un-supported mailbox message, code = %u\n", + param->req->msg.code); + + /* PF driver should not reply IMP */ + if (hnae3_get_bit(param->req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) && + param->req->msg.code < HCLGE_MBX_GET_VF_FLR_STATUS) { + param->resp_msg->status = ret; + if (time_is_before_jiffies(hdev->last_mbx_scheduled + + HCLGE_MBX_SCHED_TIMEOUT)) + dev_warn(&hdev->pdev->dev, + "resp vport%u mbx(%u,%u) late\n", + param->req->mbx_src_vfid, + param->req->msg.code, + param->req->msg.subcode); + + hclge_gen_resp_to_vf(param->vport, param->req, param->resp_msg); + } +} + void hclge_mbx_handler(struct hclge_dev *hdev) { struct hclge_comm_cmq_ring *crq = &hdev->hw.hw.cmq.crq; struct hclge_respond_to_vf_msg resp_msg; struct hclge_mbx_vf_to_pf_cmd *req; - struct hclge_vport *vport; + struct hclge_mbx_ops_param param; struct hclge_desc *desc; - bool is_del = false; unsigned int flag; - int ret = 0; + param.resp_msg = &resp_msg; /* handle all the mailbox requests in the queue */ while (!hclge_cmd_crq_empty(&hdev->hw)) { if (test_bit(HCLGE_COMM_STATE_CMD_DISABLE, @@ -814,152 +1081,16 @@ void hclge_mbx_handler(struct hclge_dev *hdev) continue; } - vport = &hdev->vport[req->mbx_src_vfid]; - trace_hclge_pf_mbx_get(hdev, req); /* clear the resp_msg before processing every mailbox message */ memset(&resp_msg, 0, sizeof(resp_msg)); - - switch (req->msg.code) { - case HCLGE_MBX_MAP_RING_TO_VECTOR: - ret = hclge_map_unmap_ring_to_vf_vector(vport, true, - req); - break; - case HCLGE_MBX_UNMAP_RING_TO_VECTOR: - ret = hclge_map_unmap_ring_to_vf_vector(vport, false, - req); - break; - case HCLGE_MBX_GET_RING_VECTOR_MAP: - ret = hclge_get_vf_ring_vector_map(vport, req, - &resp_msg); - if (ret) - dev_err(&hdev->pdev->dev, - "PF fail(%d) to get VF ring vector map\n", - ret); - break; - case HCLGE_MBX_SET_PROMISC_MODE: - hclge_set_vf_promisc_mode(vport, req); - break; - case HCLGE_MBX_SET_UNICAST: - ret = hclge_set_vf_uc_mac_addr(vport, req); - if (ret) - dev_err(&hdev->pdev->dev, - "PF fail(%d) to set VF UC MAC Addr\n", - ret); - break; - case HCLGE_MBX_SET_MULTICAST: - ret = hclge_set_vf_mc_mac_addr(vport, req); - if (ret) - dev_err(&hdev->pdev->dev, - "PF fail(%d) to set VF MC MAC Addr\n", - ret); - break; - case HCLGE_MBX_SET_VLAN: - ret = hclge_set_vf_vlan_cfg(vport, req, &resp_msg); - if (ret) - dev_err(&hdev->pdev->dev, - "PF failed(%d) to config VF's VLAN\n", - ret); - break; - case HCLGE_MBX_SET_ALIVE: - ret = hclge_set_vf_alive(vport, req); - if (ret) - dev_err(&hdev->pdev->dev, - "PF failed(%d) to set VF's ALIVE\n", - ret); - break; - case HCLGE_MBX_GET_QINFO: - hclge_get_vf_queue_info(vport, &resp_msg); - break; - case HCLGE_MBX_GET_QDEPTH: - hclge_get_vf_queue_depth(vport, &resp_msg); - break; - case HCLGE_MBX_GET_BASIC_INFO: - hclge_get_basic_info(vport, &resp_msg); - break; - case HCLGE_MBX_GET_LINK_STATUS: - ret = hclge_push_vf_link_status(vport); - if (ret) - dev_err(&hdev->pdev->dev, - "failed to inform link stat to VF, ret = %d\n", - ret); - break; - case HCLGE_MBX_QUEUE_RESET: - ret = hclge_mbx_reset_vf_queue(vport, req, &resp_msg); - break; - case HCLGE_MBX_RESET: - ret = hclge_reset_vf(vport); - break; - case HCLGE_MBX_KEEP_ALIVE: - hclge_vf_keep_alive(vport); - break; - case HCLGE_MBX_SET_MTU: - ret = hclge_set_vf_mtu(vport, req); - if (ret) - dev_err(&hdev->pdev->dev, - "VF fail(%d) to set mtu\n", ret); - break; - case HCLGE_MBX_GET_QID_IN_PF: - ret = hclge_get_queue_id_in_pf(vport, req, &resp_msg); - break; - case HCLGE_MBX_GET_RSS_KEY: - ret = hclge_get_rss_key(vport, req, &resp_msg); - break; - case HCLGE_MBX_GET_LINK_MODE: - hclge_get_link_mode(vport, req); - break; - case HCLGE_MBX_GET_VF_FLR_STATUS: - case HCLGE_MBX_VF_UNINIT: - is_del = req->msg.code == HCLGE_MBX_VF_UNINIT; - hclge_rm_vport_all_mac_table(vport, is_del, - HCLGE_MAC_ADDR_UC); - hclge_rm_vport_all_mac_table(vport, is_del, - HCLGE_MAC_ADDR_MC); - hclge_rm_vport_all_vlan_table(vport, is_del); - break; - case HCLGE_MBX_GET_MEDIA_TYPE: - hclge_get_vf_media_type(vport, &resp_msg); - break; - case HCLGE_MBX_PUSH_LINK_STATUS: - hclge_handle_link_change_event(hdev, req); - break; - case HCLGE_MBX_GET_MAC_ADDR: - hclge_get_vf_mac_addr(vport, &resp_msg); - break; - case HCLGE_MBX_NCSI_ERROR: - hclge_handle_ncsi_error(hdev); - break; - case HCLGE_MBX_HANDLE_VF_TBL: - hclge_handle_vf_tbl(vport, req); - break; - default: - dev_err(&hdev->pdev->dev, - "un-supported mailbox message, code = %u\n", - req->msg.code); - break; - } - - /* PF driver should not reply IMP */ - if (hnae3_get_bit(req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) && - req->msg.code < HCLGE_MBX_GET_VF_FLR_STATUS) { - resp_msg.status = ret; - if (time_is_before_jiffies(hdev->last_mbx_scheduled + - HCLGE_MBX_SCHED_TIMEOUT)) - dev_warn(&hdev->pdev->dev, - "resp vport%u mbx(%u,%u) late\n", - req->mbx_src_vfid, - req->msg.code, - req->msg.subcode); - - hclge_gen_resp_to_vf(vport, req, &resp_msg); - } + param.vport = &hdev->vport[req->mbx_src_vfid]; + param.req = req; + hclge_mbx_request_handling(¶m); crq->desc[crq->next_to_use].flag = 0; hclge_mbx_ring_ptr_move_crq(crq); - - /* reinitialize ret after complete the mbx message processing */ - ret = 0; } /* Write back CMDQ_RQ header pointer, M7 need this pointer */ diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c index 03d63b6a9b2bc025d1f180d6d632da42efa12734..85fb11de43a12457879f95c27e7ca4b14355c234 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c @@ -187,7 +187,7 @@ static void hclge_mac_adjust_link(struct net_device *netdev) speed = netdev->phydev->speed; duplex = netdev->phydev->duplex; - ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex); + ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex, 0); if (ret) netdev_err(netdev, "failed to adjust link.\n"); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c index 2f33b036a47a7ece9a99a86a988d50129f7ab2b4..4a33f65190e2b431ea201c42ce81b86ed5c99d54 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c @@ -248,7 +248,7 @@ static int hclge_fill_pri_array(struct hclge_dev *hdev, u8 *pri, u8 pri_id) return 0; } -static int hclge_up_to_tc_map(struct hclge_dev *hdev) +int hclge_up_to_tc_map(struct hclge_dev *hdev) { struct hclge_desc desc; u8 *pri = (u8 *)desc.data; @@ -266,6 +266,47 @@ static int hclge_up_to_tc_map(struct hclge_dev *hdev) return hclge_cmd_send(&hdev->hw, &desc, 1); } +static void hclge_dscp_to_prio_map_init(struct hclge_dev *hdev) +{ + u8 i; + + hdev->vport[0].nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_PRIO; + hdev->vport[0].nic.kinfo.dscp_app_cnt = 0; + for (i = 0; i < HNAE3_MAX_DSCP; i++) + hdev->vport[0].nic.kinfo.dscp_prio[i] = HNAE3_PRIO_ID_INVALID; +} + +int hclge_dscp_to_tc_map(struct hclge_dev *hdev) +{ + struct hclge_desc desc[HCLGE_DSCP_MAP_TC_BD_NUM]; + u8 *req0 = (u8 *)desc[0].data; + u8 *req1 = (u8 *)desc[1].data; + u8 pri_id, tc_id, i, j; + + hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QOS_MAP, false); + desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT); + hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_QOS_MAP, false); + + /* The low 32 dscp setting use bd0, high 32 dscp setting use bd1 */ + for (i = 0; i < HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; i++) { + pri_id = hdev->vport[0].nic.kinfo.dscp_prio[i]; + pri_id = pri_id == HNAE3_PRIO_ID_INVALID ? 0 : pri_id; + tc_id = hdev->tm_info.prio_tc[pri_id]; + /* Each dscp setting has 4 bits, so each byte saves two dscp + * setting + */ + req0[i >> 1] |= tc_id << HCLGE_DSCP_TC_SHIFT(i); + + j = i + HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; + pri_id = hdev->vport[0].nic.kinfo.dscp_prio[j]; + pri_id = pri_id == HNAE3_PRIO_ID_INVALID ? 0 : pri_id; + tc_id = hdev->tm_info.prio_tc[pri_id]; + req1[i >> 1] |= tc_id << HCLGE_DSCP_TC_SHIFT(i); + } + + return hclge_cmd_send(&hdev->hw, desc, HCLGE_DSCP_MAP_TC_BD_NUM); +} + static int hclge_tm_pg_to_pri_map_cfg(struct hclge_dev *hdev, u8 pg_id, u8 pri_bit_map) { @@ -1275,6 +1316,12 @@ static int hclge_tm_map_cfg(struct hclge_dev *hdev) if (ret) return ret; + if (hdev->vport[0].nic.kinfo.tc_map_mode == HNAE3_TC_MAP_MODE_DSCP) { + ret = hclge_dscp_to_tc_map(hdev); + if (ret) + return ret; + } + ret = hclge_tm_pg_to_pri_map(hdev); if (ret) return ret; @@ -1646,6 +1693,7 @@ int hclge_tm_schd_init(struct hclge_dev *hdev) return -EINVAL; hclge_tm_schd_info_init(hdev); + hclge_dscp_to_prio_map_init(hdev); return hclge_tm_init_hw(hdev, true); } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h index d943943912f76522ec340f99b32180a07a224c12..68f28a98e380bc529b1808e39d74a679decbbc43 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h @@ -30,6 +30,9 @@ enum hclge_opcode_type; #define HCLGE_TM_PF_MAX_PRI_NUM 8 #define HCLGE_TM_PF_MAX_QSET_NUM 8 +#define HCLGE_DSCP_MAP_TC_BD_NUM 2 +#define HCLGE_DSCP_TC_SHIFT(n) (((n) & 1) * 4) + struct hclge_pg_to_pri_link_cmd { u8 pg_id; u8 rsvd1[3]; @@ -262,4 +265,6 @@ int hclge_tm_get_pg_shaper(struct hclge_dev *hdev, u8 pg_id, struct hclge_tm_shaper_para *para); int hclge_tm_get_port_shaper(struct hclge_dev *hdev, struct hclge_tm_shaper_para *para); +int hclge_up_to_tc_map(struct hclge_dev *hdev); +int hclge_dscp_to_tc_map(struct hclge_dev *hdev); #endif diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 26f87330173ebc328b7dbd1b905948232b9e2c3d..db6f7cdba9587ebc2121aa3ff48b58dcdf0abe5f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -2125,7 +2125,7 @@ static int hclgevf_config_gro(struct hclgevf_dev *hdev) struct hclge_desc desc; int ret; - if (!hnae3_dev_gro_supported(hdev)) + if (!hnae3_ae_dev_gro_supported(hdev->ae_dev)) return 0; hclgevf_cmd_setup_basic_desc(&desc, HCLGE_OPC_GRO_GENERIC_CONFIG, @@ -3177,7 +3177,7 @@ static int hclgevf_get_status(struct hnae3_handle *handle) static void hclgevf_get_ksettings_an_result(struct hnae3_handle *handle, u8 *auto_neg, u32 *speed, - u8 *duplex) + u8 *duplex, u32 *lane_num) { struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle); @@ -3429,7 +3429,7 @@ static struct hnae3_ae_algo ae_algovf = { .pdev_id_table = ae_algovf_pci_tbl, }; -static int hclgevf_init(void) +static int __init hclgevf_init(void) { pr_info("%s is initializing\n", HCLGEVF_NAME); @@ -3444,7 +3444,7 @@ static int hclgevf_init(void) return 0; } -static void hclgevf_exit(void) +static void __exit hclgevf_exit(void) { hnae3_unregister_ae_algo(&ae_algovf); destroy_workqueue(hclgevf_wq); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h index e9e00cfa1329eb5bc0cbf40664522ceb03b66a7e..e10f739d8339b20459683b7c03d901f42feb1bf2 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h @@ -12,7 +12,6 @@ #define TBL_ID_FUNC_CFG_SM_INST 1 #define HINIC_FUNCTION_CONFIGURE_TABLE_SIZE 64 -#define HINIC_FUNCTION_CONFIGURE_TABLE 1 struct hinic_cmd_lt_rd { u8 status; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index 93192f58ac885c5c7a8650f5f9f001c7be767f03..f4b68028691194614b4d0359e7c19ed6ead66dfb 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -55,7 +55,6 @@ #define COALESCE_ALL_QUEUE 0xFFFF #define COALESCE_MAX_PENDING_LIMIT (255 * COALESCE_PENDING_LIMIT_UNIT) #define COALESCE_MAX_TIMER_CFG (255 * COALESCE_TIMER_CFG_UNIT) -#define OBJ_STR_MAX_LEN 32 struct hw2ethtool_link_mode { enum ethtool_link_mode_bit_indices link_mode_bit; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c index a627237f694bbad6fbf0dea0e56a20df153e78f4..78190e88cd75fa085da5f092c20d6a2fdd9551e9 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c @@ -82,11 +82,6 @@ struct hinic_func_to_io, \ cmdqs) -enum cmdq_wqe_type { - WQE_LCMD_TYPE = 0, - WQE_SCMD_TYPE = 1, -}; - enum completion_format { COMPLETE_DIRECT = 0, COMPLETE_SGE = 1, @@ -509,8 +504,8 @@ int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, * * Return 0 - Success, negative - Failure **/ -int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, - enum hinic_set_arm_qtype q_type, u32 q_id) +static int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, + enum hinic_set_arm_qtype q_type, u32 q_id) { struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC]; struct hinic_hwif *hwif = cmdqs->hwif; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h index 9c413e963a04bebee31345fd5301bcc186c1d9c2..ff09cf0ed52b56d7ba39101352f5ddaeb1a65e19 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h @@ -177,9 +177,6 @@ int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs, enum hinic_mod_type mod, u8 cmd, struct hinic_cmdq_buf *buf_in, u64 *out_param); -int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs, - enum hinic_set_arm_qtype q_type, u32 q_id); - int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, void __iomem **db_area); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h index 7e84e4e33fff1796f1249f7e1b26b6197e029f31..d56e7413ace0c652e0867bc1318cfed3fa784c1c 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h @@ -22,7 +22,6 @@ (HINIC_DMA_ATTR_BASE + (idx) * HINIC_DMA_ATTR_STRIDE) #define HINIC_PPF_ELECTION_STRIDE 0x4 -#define HINIC_CSR_MAX_PORTS 4 #define HINIC_CSR_PPF_ELECTION_ADDR(idx) \ (HINIC_ELECTION_BASE + (idx) * HINIC_PPF_ELECTION_STRIDE) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c index 2127a48749a8c70a1e830de902b8ac3c87be495c..94f470556295b80f02a5acd7f2ccab9c33ced6ca 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c @@ -29,7 +29,6 @@ #include "hinic_hw_io.h" #include "hinic_hw_dev.h" -#define IO_STATUS_TIMEOUT 100 #define OUTBOUND_STATE_TIMEOUT 100 #define DB_STATE_TIMEOUT 100 @@ -42,11 +41,6 @@ enum intr_type { INTR_MSIX_TYPE, }; -enum io_status { - IO_STOPPED = 0, - IO_RUNNING = 1, -}; - /** * parse_capability - convert device capabilities to NIC capabilities * @hwdev: the HW device to set and convert device capabilities for @@ -837,8 +831,8 @@ static int hinic_l2nic_reset(struct hinic_hwdev *hwdev) return 0; } -int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev, - struct hinic_msix_config *interrupt_info) +static int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev, + struct hinic_msix_config *interrupt_info) { u16 out_size = sizeof(*interrupt_info); struct hinic_pfhwdev *pfhwdev; @@ -1041,13 +1035,6 @@ void hinic_free_hwdev(struct hinic_hwdev *hwdev) hinic_free_hwif(hwdev->hwif); } -int hinic_hwdev_max_num_qps(struct hinic_hwdev *hwdev) -{ - struct hinic_cap *nic_cap = &hwdev->nic_cap; - - return nic_cap->max_qps; -} - /** * hinic_hwdev_num_qps - return the number QPs available for use * @hwdev: the NIC HW device diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h index 416492e482740be577858bcfdb0375ff575f879c..d2d89b0a5ef06a72d036f8e460dd2e6a534dd2bf 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h @@ -566,8 +566,6 @@ struct hinic_hwdev *hinic_init_hwdev(struct pci_dev *pdev, struct devlink *devli void hinic_free_hwdev(struct hinic_hwdev *hwdev); -int hinic_hwdev_max_num_qps(struct hinic_hwdev *hwdev); - int hinic_hwdev_num_qps(struct hinic_hwdev *hwdev); struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i); @@ -587,9 +585,6 @@ int hinic_hwdev_hw_ci_addr_set(struct hinic_hwdev *hwdev, struct hinic_sq *sq, void hinic_hwdev_set_msix_state(struct hinic_hwdev *hwdev, u16 msix_index, enum hinic_msix_state flag); -int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev, - struct hinic_msix_config *interrupt_info); - int hinic_set_interrupt_cfg(struct hinic_hwdev *hwdev, struct hinic_msix_config *interrupt_info); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c index 0428faa68e807443beac4a7871d0ec6ffdef6295..88567305d06e7b9f997bf5f6954a8a582a5f41a6 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c @@ -57,39 +57,6 @@ int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index, return 0; } -/** - * hinic_msix_attr_get - get message attribute of msix entry - * @hwif: the HW interface of a pci function device - * @msix_index: msix_index - * @pending_limit: the maximum pending interrupt events (unit 8) - * @coalesc_timer: coalesc period for interrupt (unit 8 us) - * @lli_timer: replenishing period for low latency credit (unit 8 us) - * @lli_credit_limit: maximum credits for low latency msix messages (unit 8) - * @resend_timer: maximum wait for resending msix (unit coalesc period) - * - * Return 0 - Success, negative - Failure - **/ -int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index, - u8 *pending_limit, u8 *coalesc_timer, - u8 *lli_timer, u8 *lli_credit_limit, - u8 *resend_timer) -{ - u32 addr, val; - - if (!VALID_MSIX_IDX(&hwif->attr, msix_index)) - return -EINVAL; - - addr = HINIC_CSR_MSIX_CTRL_ADDR(msix_index); - val = hinic_hwif_read_reg(hwif, addr); - - *pending_limit = HINIC_MSIX_ATTR_GET(val, PENDING_LIMIT); - *coalesc_timer = HINIC_MSIX_ATTR_GET(val, COALESC_TIMER); - *lli_timer = HINIC_MSIX_ATTR_GET(val, LLI_TIMER); - *lli_credit_limit = HINIC_MSIX_ATTR_GET(val, LLI_CREDIT); - *resend_timer = HINIC_MSIX_ATTR_GET(val, RESEND_TIMER); - return 0; -} - /** * hinic_msix_attr_cnt_clear - clear message attribute counters for msix entry * @hwif: the HW interface of a pci function device @@ -115,8 +82,6 @@ int hinic_msix_attr_cnt_clear(struct hinic_hwif *hwif, u16 msix_index) * hinic_set_pf_action - set action on pf channel * @hwif: the HW interface of a pci function device * @action: action on pf channel - * - * Return 0 - Success, negative - Failure **/ void hinic_set_pf_action(struct hinic_hwif *hwif, enum hinic_pf_action action) { diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h index c06f2253151e2f7cf36eac01d65099a79a59e4cb..3d588896a36772e39825a4202e062ff12e43a63b 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h @@ -131,10 +131,6 @@ (((u32)(val) & HINIC_MSIX_##member##_MASK) << \ HINIC_MSIX_##member##_SHIFT) -#define HINIC_MSIX_ATTR_GET(val, member) \ - (((val) >> HINIC_MSIX_##member##_SHIFT) & \ - HINIC_MSIX_##member##_MASK) - #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT 29 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK 0x1 @@ -269,11 +265,6 @@ int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index, u8 lli_timer_cfg, u8 lli_credit_limit, u8 resend_timer); -int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index, - u8 *pending_limit, u8 *coalesc_timer_cfg, - u8 *lli_timer, u8 *lli_credit_limit, - u8 *resend_timer); - void hinic_set_msix_state(struct hinic_hwif *hwif, u16 msix_idx, enum hinic_msix_state flag); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c index 5078c0c7386351175262a829e51423edea20374f..3f9c31d2921586fb3f35f1c2e507f356d885c4a0 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c @@ -117,7 +117,6 @@ enum hinic_mbox_tx_status { #define MBOX_WB_STATUS_MASK 0xFF #define MBOX_WB_ERROR_CODE_MASK 0xFF00 #define MBOX_WB_STATUS_FINISHED_SUCCESS 0xFF -#define MBOX_WB_STATUS_FINISHED_WITH_ERR 0xFE #define MBOX_WB_STATUS_NOT_FINISHED 0x00 #define MBOX_STATUS_FINISHED(wb) \ @@ -130,11 +129,8 @@ enum hinic_mbox_tx_status { #define SEQ_ID_START_VAL 0 #define SEQ_ID_MAX_VAL 42 -#define DST_AEQ_IDX_DEFAULT_VAL 0 -#define SRC_AEQ_IDX_DEFAULT_VAL 0 #define NO_DMA_ATTRIBUTE_VAL 0 -#define HINIC_MGMT_RSP_AEQN 0 #define HINIC_MBOX_RSP_AEQN 2 #define HINIC_MBOX_RECV_AEQN 0 @@ -146,7 +142,6 @@ enum hinic_mbox_tx_status { #define IS_PF_OR_PPF_SRC(src_func_idx) ((src_func_idx) < HINIC_MAX_PF_FUNCS) -#define MBOX_RESPONSE_ERROR 0x1 #define MBOX_MSG_ID_MASK 0xFF #define MBOX_MSG_ID(func_to_func) ((func_to_func)->send_msg_id) #define MBOX_MSG_ID_INC(func_to_func_mbox) (MBOX_MSG_ID(func_to_func_mbox) = \ @@ -621,7 +616,7 @@ static bool check_vf_mbox_random_id(struct hinic_mbox_func_to_func *func_to_func return false; } -void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size) +static void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size) { struct hinic_mbox_func_to_func *func_to_func; u64 mbox_header = *((u64 *)header); @@ -649,7 +644,7 @@ void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size) recv_mbox_handler(func_to_func, (u64 *)header, recv_mbox); } -void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size) +static void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size) { struct hinic_mbox_func_to_func *func_to_func; struct hinic_send_mbox *send_mbox; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h index 46953190d29e8bfa1f83f2df255f40721f9d58d5..33ac7814d3b376aeece54868371e8f13d606ba28 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h @@ -150,10 +150,6 @@ void hinic_unregister_pf_mbox_cb(struct hinic_hwdev *hwdev, void hinic_unregister_vf_mbox_cb(struct hinic_hwdev *hwdev, enum hinic_mod_type mod); -void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size); - -void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size); - int hinic_func_to_func_init(struct hinic_hwdev *hwdev); void hinic_func_to_func_free(struct hinic_hwdev *hwdev); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c index 336248aa2e48b55b85ba8511c5945b86c52e4f1f..537a8098bc4e253ec00755b20cdbaa93c8414cd2 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c @@ -472,8 +472,7 @@ int hinic_get_rq_free_wqebbs(struct hinic_rq *rq) return atomic_read(&wq->delta) - 1; } -static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx, - int nr_descs) +static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs) { u32 ctrl_size, task_size, bufdesc_size; @@ -588,18 +587,16 @@ void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info, /** * hinic_sq_prepare_wqe - prepare wqe before insert to the queue * @sq: send queue - * @prod_idx: pi value * @sq_wqe: wqe to prepare * @sges: sges for use by the wqe for send for buf addresses * @nr_sges: number of sges **/ -void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx, - struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges, - int nr_sges) +void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe, + struct hinic_sge *sges, int nr_sges) { int i; - sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges); + sq_prepare_ctrl(&sq_wqe->ctrl, nr_sges); sq_prepare_task(&sq_wqe->task); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h index 0dfa51ad5855de5b86acd3884d7bbd96c5e38862..178dcc874370ec0650f4f6e7342f5ef1695aaaa8 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h @@ -175,9 +175,8 @@ void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 l4_len, u32 offset, u32 ip_ident, u32 mss); -void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx, - struct hinic_sq_wqe *wqe, struct hinic_sge *sges, - int nr_sges); +void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *wqe, + struct hinic_sge *sges, int nr_sges); void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size, unsigned int cos); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c index 4daf6bf291ecb2ab600392bf09baa1f4c597dbcd..e1a1735c00c1c66a707a546e0cc04689793442af 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c @@ -175,8 +175,6 @@ static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages) /** * cmdq_free_page - free page from cmdq * @cmdq_pages: the pages of the cmdq queue struct that hold the page - * - * Return 0 - Success, negative - Failure **/ static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages) { diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h index f4b6d2c1061ffb26aaa72b25225bca7b7a7bdc12..c6bdeed5606e08e3ff914667114476bbd60a188b 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h @@ -261,23 +261,6 @@ #define HINIC_RSS_TYPE_GET(val, member) \ (((u32)(val) >> HINIC_RSS_TYPE_##member##_SHIFT) & 0x1) -enum hinic_l4offload_type { - HINIC_L4_OFF_DISABLE = 0, - HINIC_TCP_OFFLOAD_ENABLE = 1, - HINIC_SCTP_OFFLOAD_ENABLE = 2, - HINIC_UDP_OFFLOAD_ENABLE = 3, -}; - -enum hinic_vlan_offload { - HINIC_VLAN_OFF_DISABLE = 0, - HINIC_VLAN_OFF_ENABLE = 1, -}; - -enum hinic_pkt_parsed { - HINIC_PKT_NOT_PARSED = 0, - HINIC_PKT_PARSED = 1, -}; - enum hinic_l3_offload_type { L3TYPE_UNKNOWN = 0, IPV6_PKT = 1, @@ -305,18 +288,10 @@ enum hinic_outer_l3type { HINIC_OUTER_L3TYPE_IPV4_CHKSUM = 3, }; -enum hinic_media_type { - HINIC_MEDIA_UNKNOWN = 0, -}; - enum hinic_l2type { HINIC_L2TYPE_ETH = 0, }; -enum hinc_tunnel_l4type { - HINIC_TUNNEL_L4TYPE_UNKNOWN = 0, -}; - struct hinic_cmdq_header { u32 header_info; u32 saved_data; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c index c23ee2ddbce3ef4fd125b6b8044bb8ea6c0103b1..e1f54a2f28b2214a56627fff67030a791b20a250 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_main.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c @@ -960,8 +960,6 @@ static void hinic_refresh_nic_cfg(struct hinic_dev *nic_dev) * @in_size: input size * @buf_out: output buffer * @out_size: returned output size - * - * Return 0 - Success, negative - Failure **/ static void link_status_event_handler(void *handle, void *buf_in, u16 in_size, void *buf_out, u16 *out_size) @@ -1382,8 +1380,6 @@ err_pci_regions: return err; } -#define HINIC_WAIT_SRIOV_CFG_TIMEOUT 15000 - static void wait_sriov_cfg_complete(struct hinic_dev *nic_dev) { struct hinic_sriov_info *sriov_info = &nic_dev->sriov_info; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c index a866bea651103a69614bfb4a3b26cd4469b5f889..d649c6e323c87af8e48db46c5a1c96e60a88d74a 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c @@ -50,7 +50,7 @@ * hinic_rxq_clean_stats - Clean the statistics of specific queue * @rxq: Logical Rx Queue **/ -void hinic_rxq_clean_stats(struct hinic_rxq *rxq) +static void hinic_rxq_clean_stats(struct hinic_rxq *rxq) { struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats; @@ -74,14 +74,14 @@ void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats) unsigned int start; do { - start = u64_stats_fetch_begin(&rxq_stats->syncp); + start = u64_stats_fetch_begin_irq(&rxq_stats->syncp); stats->pkts = rxq_stats->pkts; stats->bytes = rxq_stats->bytes; stats->errors = rxq_stats->csum_errors + rxq_stats->other_errors; stats->csum_errors = rxq_stats->csum_errors; stats->other_errors = rxq_stats->other_errors; - } while (u64_stats_fetch_retry(&rxq_stats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&rxq_stats->syncp, start)); } /** diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.h b/drivers/net/ethernet/huawei/hinic/hinic_rx.h index 507dcbae90858c02afc7137d4de4f28e2ce7608a..8f7bd6a049bdf9f653bc860a7b2ed0d15710e60b 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_rx.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.h @@ -41,8 +41,6 @@ struct hinic_rxq { struct napi_struct napi; }; -void hinic_rxq_clean_stats(struct hinic_rxq *rxq); - void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats); int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq, diff --git a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c index df555847afb56a20ee58c35087e01cb47e18a3d5..a5f08b969e3f68531d9610fc4d626b9657a63bef 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c @@ -24,6 +24,7 @@ MODULE_PARM_DESC(set_vf_link_state, "Set vf link state, 0 represents link auto, #define HINIC_VLAN_PRIORITY_SHIFT 13 #define HINIC_ADD_VLAN_IN_MAC 0x8000 #define HINIC_TX_RATE_TABLE_FULL 12 +#define HINIC_MAX_QOS 7 static int hinic_set_mac(struct hinic_hwdev *hwdev, const u8 *mac_addr, u16 vlan_id, u16 func_id) @@ -774,7 +775,7 @@ int hinic_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos, u16 vlanprio, cur_vlanprio; sriov_info = &nic_dev->sriov_info; - if (vf >= sriov_info->num_vfs || vlan > 4095 || qos > 7) + if (vf >= sriov_info->num_vfs || vlan >= VLAN_N_VID || qos > HINIC_MAX_QOS) return -EINVAL; if (vlan_proto != htons(ETH_P_8021Q)) return -EPROTONOSUPPORT; @@ -820,7 +821,7 @@ int hinic_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting) cur_trust = nic_io->vf_infos[vf].trust; /* same request, so just return success */ - if ((setting && cur_trust) || (!setting && !cur_trust)) + if (setting == cur_trust) return 0; err = hinic_set_vf_trust(adapter->hwdev, vf, setting); @@ -940,7 +941,7 @@ int hinic_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting) cur_spoofchk = nic_dev->hwdev->func_to_io.vf_infos[vf].spoofchk; /* same request, so just return success */ - if ((setting && cur_spoofchk) || (!setting && !cur_spoofchk)) + if (setting == cur_spoofchk) return 0; err = hinic_set_vf_spoofchk(sriov_info->hwdev, @@ -1131,8 +1132,8 @@ static void hinic_clear_vf_infos(struct hinic_dev *nic_dev, u16 vf_id) hinic_init_vf_infos(&nic_dev->hwdev->func_to_io, HW_VF_ID_TO_OS(vf_id)); } -static int hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info, - u16 start_vf_id, u16 end_vf_id) +static void hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info, + u16 start_vf_id, u16 end_vf_id) { struct hinic_dev *nic_dev; u16 func_idx, idx; @@ -1145,8 +1146,6 @@ static int hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info, HINIC_HW_WQ_PAGE_SIZE); hinic_clear_vf_infos(nic_dev, idx); } - - return 0; } int hinic_vf_func_init(struct hinic_hwdev *hwdev) @@ -1293,7 +1292,7 @@ int hinic_pci_sriov_disable(struct pci_dev *pdev) return 0; } -int hinic_pci_sriov_enable(struct pci_dev *pdev, int num_vfs) +static int hinic_pci_sriov_enable(struct pci_dev *pdev, int num_vfs) { struct hinic_sriov_info *sriov_info; int err; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_sriov.h b/drivers/net/ethernet/huawei/hinic/hinic_sriov.h index ba627a362f9ae4ccfd4eec74cc2be1b7cdb8d862..d4d4e63d31ea350ce4c79bc04906fa6d271068e2 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_sriov.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_sriov.h @@ -98,8 +98,6 @@ void hinic_notify_all_vfs_link_changed(struct hinic_hwdev *hwdev, int hinic_pci_sriov_disable(struct pci_dev *dev); -int hinic_pci_sriov_enable(struct pci_dev *dev, int num_vfs); - int hinic_vf_func_init(struct hinic_hwdev *hwdev); void hinic_vf_func_free(struct hinic_hwdev *hwdev); diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c index 5051cdff2384beb3bce4d2e1b271f70f238278fc..e91476c8ff8b07cb02cab562687fbf1e36dea355 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c @@ -74,7 +74,7 @@ enum hinic_offload_type { * hinic_txq_clean_stats - Clean the statistics of specific queue * @txq: Logical Tx Queue **/ -void hinic_txq_clean_stats(struct hinic_txq *txq) +static void hinic_txq_clean_stats(struct hinic_txq *txq) { struct hinic_txq_stats *txq_stats = &txq->txq_stats; @@ -99,14 +99,14 @@ void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats) unsigned int start; do { - start = u64_stats_fetch_begin(&txq_stats->syncp); + start = u64_stats_fetch_begin_irq(&txq_stats->syncp); stats->pkts = txq_stats->pkts; stats->bytes = txq_stats->bytes; stats->tx_busy = txq_stats->tx_busy; stats->tx_wake = txq_stats->tx_wake; stats->tx_dropped = txq_stats->tx_dropped; stats->big_frags_pkts = txq_stats->big_frags_pkts; - } while (u64_stats_fetch_retry(&txq_stats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&txq_stats->syncp, start)); } /** @@ -530,7 +530,7 @@ netdev_tx_t hinic_lb_xmit_frame(struct sk_buff *skb, struct net_device *netdev) } process_sq_wqe: - hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges); + hinic_sq_prepare_wqe(txq->sq, sq_wqe, txq->sges, nr_sges); hinic_sq_write_wqe(txq->sq, prod_idx, sq_wqe, skb, wqe_size); flush_skbs: @@ -614,7 +614,7 @@ netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) } process_sq_wqe: - hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges); + hinic_sq_prepare_wqe(txq->sq, sq_wqe, txq->sges, nr_sges); err = hinic_tx_offload(skb, &sq_wqe->task, &sq_wqe->ctrl.queue_info); if (err) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.h b/drivers/net/ethernet/huawei/hinic/hinic_tx.h index b3c8657774a7f2a3edffdbfd70358a108f50a156..91dc778362f3b487ea7bb775ba9c292c48ff8b5a 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.h @@ -40,8 +40,6 @@ struct hinic_txq { struct napi_struct napi; }; -void hinic_txq_clean_stats(struct hinic_txq *txq); - void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats); netdev_tx_t hinic_lb_xmit_frame(struct sk_buff *skb, struct net_device *netdev); diff --git a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c b/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c index 6cb86032ce46b0ed1dc38824d00494000f7ec300..1db5b6790a41abe37eb95149baec7f4c8d20003d 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c @@ -159,8 +159,8 @@ static int ehea_nway_reset(struct net_device *dev) static void ehea_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static u32 ehea_get_msglevel(struct net_device *dev) diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c index 5dc302880f5f621d8906748191bcd6341c0e3aa1..294bdbbeacc335ff261d85b5c2c17faf2497f87c 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_main.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c @@ -1546,7 +1546,7 @@ static int ehea_init_port_res(struct ehea_port *port, struct ehea_port_res *pr, kfree(init_attr); - netif_napi_add(pr->port->netdev, &pr->napi, ehea_poll, 64); + netif_napi_add(pr->port->netdev, &pr->napi, ehea_poll); ret = 0; goto out; diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index fbea9f7efe8c89c24ae8b0b59a7e7814e08b86ba..9b08e41ccc294dae821d17a438b0f82b6d15182b 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -2284,8 +2284,8 @@ static void emac_ethtool_get_drvinfo(struct net_device *ndev, { struct emac_instance *dev = netdev_priv(ndev); - strlcpy(info->driver, "ibm_emac", sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, "ibm_emac", sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); snprintf(info->bus_info, sizeof(info->bus_info), "PPC 4xx EMAC-%d %pOF", dev->cell_index, dev->ofdev->dev.of_node); } @@ -2979,11 +2979,9 @@ static int emac_init_config(struct emac_instance *dev) /* Read MAC-address */ err = of_get_ethdev_address(np, dev->ndev); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(&dev->ofdev->dev, "Can't get valid [local-]mac-address from OF !\n"); - return err; - } + if (err) + return dev_err_probe(&dev->ofdev->dev, err, + "Can't get valid [local-]mac-address from OF !\n"); /* IAHT and GAHT filter parameterization */ if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) { diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 5c6a04d29f5bad43673687041158ab8f9578a699..3b14dc93f59dd06a5e24240d0434bd6bd18733fe 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -141,6 +141,13 @@ static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter) return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD; } +static unsigned int ibmveth_real_max_tx_queues(void) +{ + unsigned int n_cpu = num_online_cpus(); + + return min(n_cpu, IBMVETH_MAX_QUEUES); +} + /* setup the initial settings for a buffer pool */ static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool, u32 pool_index, u32 pool_size, @@ -456,6 +463,38 @@ static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter) } } +static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx) +{ + dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx], + adapter->tx_ltb_size, DMA_TO_DEVICE); + kfree(adapter->tx_ltb_ptr[idx]); + adapter->tx_ltb_ptr[idx] = NULL; +} + +static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx) +{ + adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size, + GFP_KERNEL); + if (!adapter->tx_ltb_ptr[idx]) { + netdev_err(adapter->netdev, + "unable to allocate tx long term buffer\n"); + return -ENOMEM; + } + adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev, + adapter->tx_ltb_ptr[idx], + adapter->tx_ltb_size, + DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) { + netdev_err(adapter->netdev, + "unable to DMA map tx long term buffer\n"); + kfree(adapter->tx_ltb_ptr[idx]); + adapter->tx_ltb_ptr[idx] = NULL; + return -ENOMEM; + } + + return 0; +} + static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter, union ibmveth_buf_desc rxq_desc, u64 mac_address) { @@ -538,6 +577,11 @@ static int ibmveth_open(struct net_device *netdev) goto out_unmap_buffer_list; } + for (i = 0; i < netdev->real_num_tx_queues; i++) { + if (ibmveth_allocate_tx_ltb(adapter, i)) + goto out_free_tx_ltb; + } + adapter->rx_queue.index = 0; adapter->rx_queue.num_slots = rxq_entries; adapter->rx_queue.toggle = 1; @@ -595,25 +639,15 @@ static int ibmveth_open(struct net_device *netdev) rc = -ENOMEM; - adapter->bounce_buffer = dma_alloc_coherent(&adapter->vdev->dev, - netdev->mtu + IBMVETH_BUFF_OH, - &adapter->bounce_buffer_dma, GFP_KERNEL); - if (!adapter->bounce_buffer) { - netdev_err(netdev, "unable to alloc bounce buffer\n"); - goto out_free_irq; - } - netdev_dbg(netdev, "initial replenish cycle\n"); ibmveth_interrupt(netdev->irq, netdev); - netif_start_queue(netdev); + netif_tx_start_all_queues(netdev); netdev_dbg(netdev, "open complete\n"); return 0; -out_free_irq: - free_irq(netdev->irq, netdev); out_free_buffer_pools: while (--i >= 0) { if (adapter->rx_buff_pool[i].active) @@ -623,6 +657,12 @@ out_free_buffer_pools: out_unmap_filter_list: dma_unmap_single(dev, adapter->filter_list_dma, 4096, DMA_BIDIRECTIONAL); + +out_free_tx_ltb: + while (--i >= 0) { + ibmveth_free_tx_ltb(adapter, i); + } + out_unmap_buffer_list: dma_unmap_single(dev, adapter->buffer_list_dma, 4096, DMA_BIDIRECTIONAL); @@ -651,7 +691,7 @@ static int ibmveth_close(struct net_device *netdev) napi_disable(&adapter->napi); if (!adapter->pool_config) - netif_stop_queue(netdev); + netif_tx_stop_all_queues(netdev); h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE); @@ -685,9 +725,8 @@ static int ibmveth_close(struct net_device *netdev) ibmveth_free_buffer_pool(adapter, &adapter->rx_buff_pool[i]); - dma_free_coherent(&adapter->vdev->dev, - adapter->netdev->mtu + IBMVETH_BUFF_OH, - adapter->bounce_buffer, adapter->bounce_buffer_dma); + for (i = 0; i < netdev->real_num_tx_queues; i++) + ibmveth_free_tx_ltb(adapter, i); netdev_dbg(netdev, "close complete\n"); @@ -727,8 +766,8 @@ static void ibmveth_init_link_settings(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, ibmveth_driver_name, sizeof(info->driver)); - strlcpy(info->version, ibmveth_driver_version, sizeof(info->version)); + strscpy(info->driver, ibmveth_driver_name, sizeof(info->driver)); + strscpy(info->version, ibmveth_driver_version, sizeof(info->version)); } static netdev_features_t ibmveth_fix_features(struct net_device *dev, @@ -953,6 +992,69 @@ static void ibmveth_get_ethtool_stats(struct net_device *dev, data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset); } +static void ibmveth_get_channels(struct net_device *netdev, + struct ethtool_channels *channels) +{ + channels->max_tx = ibmveth_real_max_tx_queues(); + channels->tx_count = netdev->real_num_tx_queues; + + channels->max_rx = netdev->real_num_rx_queues; + channels->rx_count = netdev->real_num_rx_queues; +} + +static int ibmveth_set_channels(struct net_device *netdev, + struct ethtool_channels *channels) +{ + struct ibmveth_adapter *adapter = netdev_priv(netdev); + unsigned int old = netdev->real_num_tx_queues, + goal = channels->tx_count; + int rc, i; + + /* If ndo_open has not been called yet then don't allocate, just set + * desired netdev_queue's and return + */ + if (!(netdev->flags & IFF_UP)) + return netif_set_real_num_tx_queues(netdev, goal); + + /* We have IBMVETH_MAX_QUEUES netdev_queue's allocated + * but we may need to alloc/free the ltb's. + */ + netif_tx_stop_all_queues(netdev); + + /* Allocate any queue that we need */ + for (i = old; i < goal; i++) { + if (adapter->tx_ltb_ptr[i]) + continue; + + rc = ibmveth_allocate_tx_ltb(adapter, i); + if (!rc) + continue; + + /* if something goes wrong, free everything we just allocated */ + netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n", + old); + goal = old; + old = i; + break; + } + rc = netif_set_real_num_tx_queues(netdev, goal); + if (rc) { + netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n", + old); + goal = old; + old = i; + } + /* Free any that are no longer needed */ + for (i = old; i > goal; i--) { + if (adapter->tx_ltb_ptr[i - 1]) + ibmveth_free_tx_ltb(adapter, i - 1); + } + + netif_tx_wake_all_queues(netdev); + + return rc; +} + static const struct ethtool_ops netdev_ethtool_ops = { .get_drvinfo = netdev_get_drvinfo, .get_link = ethtool_op_get_link, @@ -961,6 +1063,8 @@ static const struct ethtool_ops netdev_ethtool_ops = { .get_ethtool_stats = ibmveth_get_ethtool_stats, .get_link_ksettings = ibmveth_get_link_ksettings, .set_link_ksettings = ibmveth_set_link_ksettings, + .get_channels = ibmveth_get_channels, + .set_channels = ibmveth_set_channels }; static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) @@ -969,7 +1073,7 @@ static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) } static int ibmveth_send(struct ibmveth_adapter *adapter, - union ibmveth_buf_desc *descs, unsigned long mss) + unsigned long desc, unsigned long mss) { unsigned long correlator; unsigned int retry_count; @@ -982,12 +1086,9 @@ static int ibmveth_send(struct ibmveth_adapter *adapter, retry_count = 1024; correlator = 0; do { - ret = h_send_logical_lan(adapter->vdev->unit_address, - descs[0].desc, descs[1].desc, - descs[2].desc, descs[3].desc, - descs[4].desc, descs[5].desc, - correlator, &correlator, mss, - adapter->fw_large_send_support); + ret = h_send_logical_lan(adapter->vdev->unit_address, desc, + correlator, &correlator, mss, + adapter->fw_large_send_support); } while ((ret == H_BUSY) && (retry_count--)); if (ret != H_SUCCESS && ret != H_DROPPED) { @@ -1020,34 +1121,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, struct net_device *netdev) { struct ibmveth_adapter *adapter = netdev_priv(netdev); - unsigned int desc_flags; - union ibmveth_buf_desc descs[6]; - int last, i; - int force_bounce = 0; - dma_addr_t dma_addr; + unsigned int desc_flags, total_bytes; + union ibmveth_buf_desc desc; + int i, queue_num = skb_get_queue_mapping(skb); unsigned long mss = 0; if (ibmveth_is_packet_unsupported(skb, netdev)) goto out; - - /* veth doesn't handle frag_list, so linearize the skb. - * When GRO is enabled SKB's can have frag_list. - */ - if (adapter->is_active_trunk && - skb_has_frag_list(skb) && __skb_linearize(skb)) { - netdev->stats.tx_dropped++; - goto out; - } - - /* - * veth handles a maximum of 6 segments including the header, so - * we have to linearize the skb if there are more than this. - */ - if (skb_shinfo(skb)->nr_frags > 5 && __skb_linearize(skb)) { - netdev->stats.tx_dropped++; - goto out; - } - /* veth can't checksum offload UDP */ if (skb->ip_summed == CHECKSUM_PARTIAL && ((skb->protocol == htons(ETH_P_IP) && @@ -1077,56 +1157,6 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, desc_flags |= IBMVETH_BUF_LRG_SND; } -retry_bounce: - memset(descs, 0, sizeof(descs)); - - /* - * If a linear packet is below the rx threshold then - * copy it into the static bounce buffer. This avoids the - * cost of a TCE insert and remove. - */ - if (force_bounce || (!skb_is_nonlinear(skb) && - (skb->len < tx_copybreak))) { - skb_copy_from_linear_data(skb, adapter->bounce_buffer, - skb->len); - - descs[0].fields.flags_len = desc_flags | skb->len; - descs[0].fields.address = adapter->bounce_buffer_dma; - - if (ibmveth_send(adapter, descs, 0)) { - adapter->tx_send_failed++; - netdev->stats.tx_dropped++; - } else { - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += skb->len; - } - - goto out; - } - - /* Map the header */ - dma_addr = dma_map_single(&adapter->vdev->dev, skb->data, - skb_headlen(skb), DMA_TO_DEVICE); - if (dma_mapping_error(&adapter->vdev->dev, dma_addr)) - goto map_failed; - - descs[0].fields.flags_len = desc_flags | skb_headlen(skb); - descs[0].fields.address = dma_addr; - - /* Map the frags */ - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - - dma_addr = skb_frag_dma_map(&adapter->vdev->dev, frag, 0, - skb_frag_size(frag), DMA_TO_DEVICE); - - if (dma_mapping_error(&adapter->vdev->dev, dma_addr)) - goto map_failed_frags; - - descs[i+1].fields.flags_len = desc_flags | skb_frag_size(frag); - descs[i+1].fields.address = dma_addr; - } - if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) { if (adapter->fw_large_send_support) { mss = (unsigned long)skb_shinfo(skb)->gso_size; @@ -1143,7 +1173,36 @@ retry_bounce: } } - if (ibmveth_send(adapter, descs, mss)) { + /* Copy header into mapped buffer */ + if (unlikely(skb->len > adapter->tx_ltb_size)) { + netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n", + skb->len, adapter->tx_ltb_size); + netdev->stats.tx_dropped++; + goto out; + } + memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb)); + total_bytes = skb_headlen(skb); + /* Copy frags into mapped buffers */ + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + memcpy(adapter->tx_ltb_ptr[queue_num] + total_bytes, + skb_frag_address_safe(frag), skb_frag_size(frag)); + total_bytes += skb_frag_size(frag); + } + + if (unlikely(total_bytes != skb->len)) { + netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n", + skb->len, total_bytes); + netdev->stats.tx_dropped++; + goto out; + } + desc.fields.flags_len = desc_flags | skb->len; + desc.fields.address = adapter->tx_ltb_dma[queue_num]; + /* finish writing to long_term_buff before VIOS accessing it */ + dma_wmb(); + + if (ibmveth_send(adapter, desc.desc, mss)) { adapter->tx_send_failed++; netdev->stats.tx_dropped++; } else { @@ -1151,41 +1210,11 @@ retry_bounce: netdev->stats.tx_bytes += skb->len; } - dma_unmap_single(&adapter->vdev->dev, - descs[0].fields.address, - descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK, - DMA_TO_DEVICE); - - for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++) - dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address, - descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK, - DMA_TO_DEVICE); - out: dev_consume_skb_any(skb); return NETDEV_TX_OK; -map_failed_frags: - last = i+1; - for (i = 1; i < last; i++) - dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address, - descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK, - DMA_TO_DEVICE); - dma_unmap_single(&adapter->vdev->dev, - descs[0].fields.address, - descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK, - DMA_TO_DEVICE); -map_failed: - if (!firmware_has_feature(FW_FEATURE_CMO)) - netdev_err(netdev, "tx: unable to map xmit buffer\n"); - adapter->tx_map_failed++; - if (skb_linearize(skb)) { - netdev->stats.tx_dropped++; - goto out; - } - force_bounce = 1; - goto retry_bounce; } static void ibmveth_rx_mss_helper(struct sk_buff *skb, u16 mss, int lrg_pkt) @@ -1568,6 +1597,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev) ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE; ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl); + /* add size of mapped tx buffers */ + ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl); for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { /* add the size of the active receive buffers */ @@ -1660,8 +1691,7 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id) return -EINVAL; } - netdev = alloc_etherdev(sizeof(struct ibmveth_adapter)); - + netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1); if (!netdev) return -ENOMEM; @@ -1727,6 +1757,17 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id) kobject_uevent(kobj, KOBJ_ADD); } + rc = netif_set_real_num_tx_queues(netdev, ibmveth_real_max_tx_queues()); + if (rc) { + netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n", + rc); + free_netdev(netdev); + return rc; + } + adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE); + for (i = 0; i < IBMVETH_MAX_QUEUES; i++) + adapter->tx_ltb_ptr[i] = NULL; + netdev_dbg(netdev, "adapter @ 0x%p\n", adapter); netdev_dbg(netdev, "registering netdev...\n"); diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index 27dfff2001662c6efa49ec037481209e916effd5..daf6f615c03f80d92fbcdeb5d3b01b9fabe981c3 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -46,23 +46,23 @@ #define h_add_logical_lan_buffer(ua, buf) \ plpar_hcall_norets(H_ADD_LOGICAL_LAN_BUFFER, ua, buf) +/* FW allows us to send 6 descriptors but we only use one so mark + * the other 5 as unused (0) + */ static inline long h_send_logical_lan(unsigned long unit_address, - unsigned long desc1, unsigned long desc2, unsigned long desc3, - unsigned long desc4, unsigned long desc5, unsigned long desc6, - unsigned long corellator_in, unsigned long *corellator_out, - unsigned long mss, unsigned long large_send_support) + unsigned long desc, unsigned long corellator_in, + unsigned long *corellator_out, unsigned long mss, + unsigned long large_send_support) { long rc; unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; if (large_send_support) rc = plpar_hcall9(H_SEND_LOGICAL_LAN, retbuf, unit_address, - desc1, desc2, desc3, desc4, desc5, desc6, - corellator_in, mss); + desc, 0, 0, 0, 0, 0, corellator_in, mss); else rc = plpar_hcall9(H_SEND_LOGICAL_LAN, retbuf, unit_address, - desc1, desc2, desc3, desc4, desc5, desc6, - corellator_in); + desc, 0, 0, 0, 0, 0, corellator_in); *corellator_out = retbuf[0]; @@ -98,6 +98,8 @@ static inline long h_illan_attributes(unsigned long unit_address, #define IBMVETH_BUFF_LIST_SIZE 4096 #define IBMVETH_FILT_LIST_SIZE 4096 #define IBMVETH_MAX_BUF_SIZE (1024 * 128) +#define IBMVETH_MAX_TX_BUF_SIZE (1024 * 64) +#define IBMVETH_MAX_QUEUES 16U static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 }; static int pool_count[] = { 256, 512, 256, 256, 256 }; @@ -137,6 +139,9 @@ struct ibmveth_adapter { unsigned int mcastFilterSize; void * buffer_list_addr; void * filter_list_addr; + void *tx_ltb_ptr[IBMVETH_MAX_QUEUES]; + unsigned int tx_ltb_size; + dma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES]; dma_addr_t buffer_list_dma; dma_addr_t filter_list_dma; struct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS]; @@ -145,8 +150,6 @@ struct ibmveth_adapter { int rx_csum; int large_send; bool is_active_trunk; - void *bounce_buffer; - dma_addr_t bounce_buffer_dma; u64 fw_ipv6_csum_support; u64 fw_ipv4_csum_support; diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 5ab7c0f81e9afc9ec6230325cb9bac71ca33f34b..65dbfbec487a34b6af65538434c0d429e9fade46 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -1262,7 +1262,7 @@ static int init_napi(struct ibmvnic_adapter *adapter) for (i = 0; i < adapter->req_rx_queues; i++) { netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i); netif_napi_add(adapter->netdev, &adapter->napi[i], - ibmvnic_poll, NAPI_POLL_WEIGHT); + ibmvnic_poll); } adapter->num_active_rx_napi = adapter->req_rx_queues; diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c index 11a884aa5082c119b2bbe8ca3585b7b7524f4809..560d1d442232660d585d6c6d34e9d6dc307e162a 100644 --- a/drivers/net/ethernet/intel/e100.c +++ b/drivers/net/ethernet/intel/e100.c @@ -2431,8 +2431,8 @@ static void e100_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { struct nic *nic = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(nic->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index 32803b0cf1e8e0131f2d53de9b60724d39fc6f09..d06d29c6c0370d222c91e30cf6dee53c10c7d53a 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -531,10 +531,10 @@ static void e1000_get_drvinfo(struct net_device *netdev, { struct e1000_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, e1000_driver_name, + strscpy(drvinfo->driver, e1000_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 23299fc561999ad435863d5d19f1fffbfb56635b..61e60e4de600fb2059f6a01b47e87cb199a03c09 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -1012,7 +1012,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->netdev_ops = &e1000_netdev_ops; e1000_set_ethtool_ops(netdev); netdev->watchdog_timeo = 5 * HZ; - netif_napi_add(netdev, &adapter->napi, e1000_clean, 64); + netif_napi_add(netdev, &adapter->napi, e1000_clean); strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index b80ae9a82224d4bf76d516ac6d16dfb547de1f35..51a5afe9df2fe9cf07a508861ea60756d128f372 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -639,7 +639,7 @@ static void e1000_get_drvinfo(struct net_device *netdev, { struct e1000_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, e1000e_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, e1000e_driver_name, sizeof(drvinfo->driver)); /* EEPROM image version # is reported as firmware version # for * PCI-E controllers @@ -650,7 +650,7 @@ static void e1000_get_drvinfo(struct net_device *netdev, (adapter->eeprom_vers & 0x0FF0) >> 4, (adapter->eeprom_vers & 0x000F)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 321f2a95ae3ab0c9f4fe3b16b0d35aed34fe43cc..49e926959ad32e28e8850a58fee10bf5bb243954 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -7267,7 +7267,7 @@ static void e1000_print_device_info(struct e1000_adapter *adapter) ret_val = e1000_read_pba_string_generic(hw, pba_str, E1000_PBANUM_LENGTH); if (ret_val) - strlcpy((char *)pba_str, "Unknown", sizeof(pba_str)); + strscpy((char *)pba_str, "Unknown", sizeof(pba_str)); e_info("MAC: %d, PHY: %d, PBA No: %s\n", hw->mac.type, hw->phy.type, pba_str); } @@ -7479,8 +7479,8 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->netdev_ops = &e1000e_netdev_ops; e1000e_set_ethtool_ops(netdev); netdev->watchdog_timeo = 5 * HZ; - netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64); - strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); + netif_napi_add(netdev, &adapter->napi, e1000e_poll); + strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); netdev->mem_start = mmio_start; netdev->mem_end = mmio_start + mmio_len; @@ -7676,7 +7676,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (hw->mac.type >= e1000_pch_cnp) adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS; - strlcpy(netdev->name, "eth%d", sizeof(netdev->name)); + strscpy(netdev->name, "eth%d", sizeof(netdev->name)); err = register_netdev(netdev); if (err) goto err_register; diff --git a/drivers/net/ethernet/intel/e1000e/phy.c b/drivers/net/ethernet/intel/e1000e/phy.c index fd07c3679bb15db73a78c1a77a997abe2fe10e10..060b263348ce8fe1d6ea8bb1ba71a97f78b41d0e 100644 --- a/drivers/net/ethernet/intel/e1000e/phy.c +++ b/drivers/net/ethernet/intel/e1000e/phy.c @@ -2697,9 +2697,14 @@ static s32 e1000_access_phy_wakeup_reg_bm(struct e1000_hw *hw, u32 offset, void e1000_power_up_phy_copper(struct e1000_hw *hw) { u16 mii_reg = 0; + int ret; /* The PHY will retain its settings across a power down/up cycle */ - e1e_rphy(hw, MII_BMCR, &mii_reg); + ret = e1e_rphy(hw, MII_BMCR, &mii_reg); + if (ret) { + e_dbg("Error reading PHY register\n"); + return; + } mii_reg &= ~BMCR_PDOWN; e1e_wphy(hw, MII_BMCR, mii_reg); } @@ -2715,9 +2720,14 @@ void e1000_power_up_phy_copper(struct e1000_hw *hw) void e1000_power_down_phy_copper(struct e1000_hw *hw) { u16 mii_reg = 0; + int ret; /* The PHY will retain its settings across a power down/up cycle */ - e1e_rphy(hw, MII_BMCR, &mii_reg); + ret = e1e_rphy(hw, MII_BMCR, &mii_reg); + if (ret) { + e_dbg("Error reading PHY register\n"); + return; + } mii_reg |= BMCR_PDOWN; e1e_wphy(hw, MII_BMCR, mii_reg); usleep_range(1000, 2000); @@ -3037,7 +3047,11 @@ s32 e1000_link_stall_workaround_hv(struct e1000_hw *hw) return 0; /* Do not apply workaround if in PHY loopback bit 14 set */ - e1e_rphy(hw, MII_BMCR, &data); + ret_val = e1e_rphy(hw, MII_BMCR, &data); + if (ret_val) { + e_dbg("Error reading PHY register\n"); + return ret_val; + } if (data & BMCR_LOOPBACK) return 0; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c index 3362f26d7f9999fd4a9129e92c9ae59400f7fe61..4a6630586ec98aa51d7b8870cc861ea1d0a97d65 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c @@ -1595,8 +1595,7 @@ static int fm10k_alloc_q_vector(struct fm10k_intfc *interface, return -ENOMEM; /* initialize NAPI */ - netif_napi_add(interface->netdev, &q_vector->napi, - fm10k_poll, NAPI_POLL_WEIGHT); + netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll); /* tie q_vector and interface together */ interface->q_vector[v_idx] = q_vector; diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index d86b6d349ea9d5bc5b4c3e62e86a4fab0a8fa31d..9a60d6b207f7ce10b4ec77a5abebc5f7f417aa11 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -399,6 +399,20 @@ struct i40e_ddp_old_profile_list { I40E_FLEX_54_MASK | I40E_FLEX_55_MASK | \ I40E_FLEX_56_MASK | I40E_FLEX_57_MASK) +#define I40E_QINT_TQCTL_VAL(qp, vector, nextq_type) \ + (I40E_QINT_TQCTL_CAUSE_ENA_MASK | \ + (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | \ + ((vector) << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | \ + ((qp) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | \ + (I40E_QUEUE_TYPE_##nextq_type << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT)) + +#define I40E_QINT_RQCTL_VAL(qp, vector, nextq_type) \ + (I40E_QINT_RQCTL_CAUSE_ENA_MASK | \ + (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | \ + ((vector) << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | \ + ((qp) << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | \ + (I40E_QUEUE_TYPE_##nextq_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT)) + struct i40e_flex_pit { struct list_head list; u16 src_offset; diff --git a/drivers/net/ethernet/intel/i40e/i40e_client.c b/drivers/net/ethernet/intel/i40e/i40e_client.c index ea2bb0140a6eb0c7df7397fc374de275ba297e29..10d7a982a5b9b7806a3cd2425ed51726314132f0 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_client.c +++ b/drivers/net/ethernet/intel/i40e/i40e_client.c @@ -177,6 +177,10 @@ void i40e_notify_client_of_netdev_close(struct i40e_vsi *vsi, bool reset) "Cannot locate client instance close routine\n"); return; } + if (!test_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state)) { + dev_dbg(&pf->pdev->dev, "Client is not open, abort close\n"); + return; + } cdev->client->ops->close(&cdev->lan_info, cdev->client, reset); clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); i40e_client_release_qvlist(&cdev->lan_info); @@ -429,7 +433,6 @@ void i40e_client_subtask(struct i40e_pf *pf) /* Remove failed client instance */ clear_bit(__I40E_CLIENT_INSTANCE_OPENED, &cdev->state); - i40e_client_del_instance(pf); return; } } diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index 2819e261a126b6aaf0543697e7de1b7a5051f3cf..4f01e2a6b6bbf1c2c0372bc91139e950eea02515 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -27,6 +27,7 @@ i40e_status i40e_set_mac_type(struct i40e_hw *hw) case I40E_DEV_ID_QSFP_A: case I40E_DEV_ID_QSFP_B: case I40E_DEV_ID_QSFP_C: + case I40E_DEV_ID_1G_BASE_T_BC: case I40E_DEV_ID_5G_BASE_T_BC: case I40E_DEV_ID_10G_BASE_T: case I40E_DEV_ID_10G_BASE_T4: @@ -4974,6 +4975,7 @@ i40e_status i40e_write_phy_register(struct i40e_hw *hw, status = i40e_write_phy_register_clause22(hw, reg, phy_addr, value); break; + case I40E_DEV_ID_1G_BASE_T_BC: case I40E_DEV_ID_5G_BASE_T_BC: case I40E_DEV_ID_10G_BASE_T: case I40E_DEV_ID_10G_BASE_T4: @@ -5012,6 +5014,7 @@ i40e_status i40e_read_phy_register(struct i40e_hw *hw, status = i40e_read_phy_register_clause22(hw, reg, phy_addr, value); break; + case I40E_DEV_ID_1G_BASE_T_BC: case I40E_DEV_ID_5G_BASE_T_BC: case I40E_DEV_ID_10G_BASE_T: case I40E_DEV_ID_10G_BASE_T4: diff --git a/drivers/net/ethernet/intel/i40e/i40e_devids.h b/drivers/net/ethernet/intel/i40e/i40e_devids.h index 2610338002fee7ff0efa054f3eba4d0845b1d77e..d9c51a238dcc6924f4fa9137c46075646c97263c 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_devids.h +++ b/drivers/net/ethernet/intel/i40e/i40e_devids.h @@ -24,8 +24,10 @@ #define I40E_DEV_ID_10G_B 0x104F #define I40E_DEV_ID_10G_SFP 0x104E #define I40E_DEV_ID_5G_BASE_T_BC 0x101F +#define I40E_DEV_ID_1G_BASE_T_BC 0x0DD2 #define I40E_IS_X710TL_DEVICE(d) \ - (((d) == I40E_DEV_ID_5G_BASE_T_BC) || \ + (((d) == I40E_DEV_ID_1G_BASE_T_BC) || \ + ((d) == I40E_DEV_ID_5G_BASE_T_BC) || \ ((d) == I40E_DEV_ID_10G_BASE_T_BC)) #define I40E_DEV_ID_KX_X722 0x37CE #define I40E_DEV_ID_QSFP_X722 0x37CF diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 156e92c437803b30638174b4e54a08abf6f5b081..7e75706f76db26069765baceb4e575bd1455523a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -2001,10 +2001,10 @@ static void i40e_get_drvinfo(struct net_device *netdev, struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; - strlcpy(drvinfo->driver, i40e_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, i40e_nvm_version_str(&pf->hw), + strscpy(drvinfo->driver, i40e_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->fw_version, i40e_nvm_version_str(&pf->hw), sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(pf->pdev), + strscpy(drvinfo->bus_info, pci_name(pf->pdev), sizeof(drvinfo->bus_info)); drvinfo->n_priv_flags = I40E_PRIV_FLAGS_STR_LEN; if (pf->hw.pf_id == 0) @@ -4485,7 +4485,7 @@ static int i40e_check_fdir_input_set(struct i40e_vsi *vsi, (struct in6_addr *)&ipv6_full_mask)) new_mask |= I40E_L3_V6_DST_MASK; else if (ipv6_addr_any((struct in6_addr *) - &usr_ip6_spec->ip6src)) + &usr_ip6_spec->ip6dst)) new_mask &= ~I40E_L3_V6_DST_MASK; else return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 9f1d5de7bf16191dec2c7a939f794666fd4dc776..2c07fa8ecfc8021625f6c33991f0516824e69c2b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -66,6 +66,7 @@ static const struct pci_device_id i40e_pci_tbl[] = { {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, + {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, @@ -3878,7 +3879,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) wr32(hw, I40E_PFINT_RATEN(vector - 1), i40e_intrl_usec_to_reg(vsi->int_rate_limit)); - /* Linked list for the queuepairs assigned to this vector */ + /* begin of linked list for RX queue assigned to this vector */ wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); for (q = 0; q < q_vector->num_ringpairs; q++) { u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; @@ -3894,6 +3895,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) wr32(hw, I40E_QINT_RQCTL(qp), val); if (has_xdp) { + /* TX queue with next queue set to TX */ val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | @@ -3903,7 +3905,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) wr32(hw, I40E_QINT_TQCTL(nextqp), val); } - + /* TX queue with next RX or end of linked list */ val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | @@ -3972,7 +3974,6 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) struct i40e_q_vector *q_vector = vsi->q_vectors[0]; struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; - u32 val; /* set the ITR configuration */ q_vector->rx.next_update = jiffies + 1; @@ -3989,28 +3990,20 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ wr32(hw, I40E_PFINT_LNKLST0, 0); - /* Associate the queue pair to the vector and enable the queue int */ - val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | - (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | - (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| - (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - - wr32(hw, I40E_QINT_RQCTL(0), val); + /* Associate the queue pair to the vector and enable the queue + * interrupt RX queue in linked list with next queue set to TX + */ + wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX)); if (i40e_enabled_xdp_vsi(vsi)) { - val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| - (I40E_QUEUE_TYPE_TX - << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); - - wr32(hw, I40E_QINT_TQCTL(nextqp), val); + /* TX queue in linked list with next queue set to TX */ + wr32(hw, I40E_QINT_TQCTL(nextqp), + I40E_QINT_TQCTL_VAL(nextqp, 0, TX)); } - val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | - (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | - (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); - - wr32(hw, I40E_QINT_TQCTL(0), val); + /* last TX queue so the next RX queue doesn't matter */ + wr32(hw, I40E_QINT_TQCTL(0), + I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX)); i40e_flush(hw); } @@ -5908,6 +5901,26 @@ static int i40e_get_link_speed(struct i40e_vsi *vsi) } } +/** + * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits + * @vsi: Pointer to vsi structure + * @max_tx_rate: max TX rate in bytes to be converted into Mbits + * + * Helper function to convert units before send to set BW limit + **/ +static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate) +{ + if (max_tx_rate < I40E_BW_MBPS_DIVISOR) { + dev_warn(&vsi->back->pdev->dev, + "Setting max tx rate to minimum usable value of 50Mbps.\n"); + max_tx_rate = I40E_BW_CREDIT_DIVISOR; + } else { + do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); + } + + return max_tx_rate; +} + /** * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate * @vsi: VSI to be configured @@ -5930,10 +5943,10 @@ int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) max_tx_rate, seid); return -EINVAL; } - if (max_tx_rate && max_tx_rate < 50) { + if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) { dev_warn(&pf->pdev->dev, "Setting max tx rate to minimum usable value of 50Mbps.\n"); - max_tx_rate = 50; + max_tx_rate = I40E_BW_CREDIT_DIVISOR; } /* Tx rate credits are in values of 50Mbps, 0 is disabled */ @@ -6659,6 +6672,9 @@ static int i40e_configure_queue_channels(struct i40e_vsi *vsi) vsi->tc_seid_map[i] = ch->seid; } } + + /* reset to reconfigure TX queue contexts */ + i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true); return ret; err_free: @@ -8221,9 +8237,9 @@ config_tc: if (i40e_is_tc_mqprio_enabled(pf)) { if (vsi->mqprio_qopt.max_rate[0]) { - u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; + u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, + vsi->mqprio_qopt.max_rate[0]); - do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); if (!ret) { u64 credits = max_tx_rate; @@ -10701,7 +10717,7 @@ static void i40e_send_version(struct i40e_pf *pf) dv.minor_version = 0xff; dv.build_version = 0xff; dv.subbuild_version = 0; - strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); + strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); i40e_aq_send_driver_version(&pf->hw, &dv, NULL); } @@ -10968,10 +10984,10 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) } if (vsi->mqprio_qopt.max_rate[0]) { - u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; + u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, + vsi->mqprio_qopt.max_rate[0]); u64 credits = 0; - do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); if (ret) goto end_unlock; @@ -11925,8 +11941,7 @@ static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); if (vsi->netdev) - netif_napi_add(vsi->netdev, &q_vector->napi, - i40e_napi_poll, NAPI_POLL_WEIGHT); + netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll); /* tie q_vector and vsi together */ vsi->q_vectors[v_idx] = q_vector; @@ -16049,23 +16064,23 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) switch (hw->bus.speed) { case i40e_bus_speed_8000: - strlcpy(speed, "8.0", PCI_SPEED_SIZE); break; + strscpy(speed, "8.0", PCI_SPEED_SIZE); break; case i40e_bus_speed_5000: - strlcpy(speed, "5.0", PCI_SPEED_SIZE); break; + strscpy(speed, "5.0", PCI_SPEED_SIZE); break; case i40e_bus_speed_2500: - strlcpy(speed, "2.5", PCI_SPEED_SIZE); break; + strscpy(speed, "2.5", PCI_SPEED_SIZE); break; default: break; } switch (hw->bus.width) { case i40e_bus_width_pcie_x8: - strlcpy(width, "8", PCI_WIDTH_SIZE); break; + strscpy(width, "8", PCI_WIDTH_SIZE); break; case i40e_bus_width_pcie_x4: - strlcpy(width, "4", PCI_WIDTH_SIZE); break; + strscpy(width, "4", PCI_WIDTH_SIZE); break; case i40e_bus_width_pcie_x2: - strlcpy(width, "2", PCI_WIDTH_SIZE); break; + strscpy(width, "2", PCI_WIDTH_SIZE); break; case i40e_bus_width_pcie_x1: - strlcpy(width, "1", PCI_WIDTH_SIZE); break; + strscpy(width, "1", PCI_WIDTH_SIZE); break; default: break; } diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c index 2d3533f38d7b4f9093db48ae6bf7c983f4deea16..ffea0c9c82f1e755c0a0f17bc0b08f40fefb39c2 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c @@ -1390,7 +1390,7 @@ static long i40e_ptp_create_clock(struct i40e_pf *pf) if (!IS_ERR_OR_NULL(pf->ptp_clock)) return 0; - strlcpy(pf->ptp_caps.name, i40e_driver_name, + strscpy(pf->ptp_caps.name, i40e_driver_name, sizeof(pf->ptp_caps.name) - 1); pf->ptp_caps.owner = THIS_MODULE; pf->ptp_caps.max_adj = 999999999; diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index d4226161a3efc408b96e181d427e46abaa078093..69e67eb6aea7249d8f0a4cd75eb60a4183ae856f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -3688,7 +3688,8 @@ u16 i40e_lan_select_queue(struct net_device *netdev, u8 prio; /* is DCB enabled at all? */ - if (vsi->tc_config.numtc == 1) + if (vsi->tc_config.numtc == 1 || + i40e_is_tc_mqprio_enabled(vsi->back)) return netdev_pick_tx(netdev, skb, sb_dev); prio = skb->priority; diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 4f184c50f6e8b22b66c00ea56200c780c66d6bb4..7e9f6a69eb10ca09ac10667380d312407e22d920 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -2038,6 +2038,25 @@ static void i40e_del_qch(struct i40e_vf *vf) } } +/** + * i40e_vc_get_max_frame_size + * @vf: pointer to the VF + * + * Max frame size is determined based on the current port's max frame size and + * whether a port VLAN is configured on this VF. The VF is not aware whether + * it's in a port VLAN so the PF needs to account for this in max frame size + * checks and sending the max frame size to the VF. + **/ +static u16 i40e_vc_get_max_frame_size(struct i40e_vf *vf) +{ + u16 max_frame_size = vf->pf->hw.phy.link_info.max_frame_size; + + if (vf->port_vlan_id) + max_frame_size -= VLAN_HLEN; + + return max_frame_size; +} + /** * i40e_vc_get_vf_resources_msg * @vf: pointer to the VF info @@ -2139,6 +2158,7 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) vfres->max_vectors = pf->hw.func_caps.num_msix_vectors_vf; vfres->rss_key_size = I40E_HKEY_ARRAY_SIZE; vfres->rss_lut_size = I40E_VF_HLUT_ARRAY_SIZE; + vfres->max_mtu = i40e_vc_get_max_frame_size(vf); if (vf->lan_vsi_idx) { vfres->vsi_res[0].vsi_id = vf->lan_vsi_id; diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index e535d4c3da49d92e80214dc7ca39b4f75f311635..a056e15456153d0fff26654ea5c64f3b7fb64367 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -581,9 +581,9 @@ static void iavf_get_drvinfo(struct net_device *netdev, { struct iavf_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, iavf_driver_name, 32); - strlcpy(drvinfo->fw_version, "N/A", 4); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), 32); + strscpy(drvinfo->driver, iavf_driver_name, 32); + strscpy(drvinfo->fw_version, "N/A", 4); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), 32); drvinfo->n_priv_flags = IAVF_PRIV_FLAGS_STR_LEN; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index f39440ad5c50d6c07e055c145a4ec66e2a293e2f..3fc572341781b0331649ebea83e11c70890bddf9 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -1077,7 +1077,6 @@ static int iavf_set_mac(struct net_device *netdev, void *p) { struct iavf_adapter *adapter = netdev_priv(netdev); struct sockaddr *addr = p; - bool handle_mac = iavf_is_mac_set_handled(netdev, addr->sa_data); int ret; if (!is_valid_ether_addr(addr->sa_data)) @@ -1094,10 +1093,9 @@ static int iavf_set_mac(struct net_device *netdev, void *p) return 0; } - if (handle_mac) - goto done; - - ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, false, msecs_to_jiffies(2500)); + ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, + iavf_is_mac_set_handled(netdev, addr->sa_data), + msecs_to_jiffies(2500)); /* If ret < 0 then it means wait was interrupted. * If ret == 0 then it means we got a timeout. @@ -1111,7 +1109,6 @@ static int iavf_set_mac(struct net_device *netdev, void *p) if (!ret) return -EAGAIN; -done: if (!ether_addr_equal(netdev->dev_addr, addr->sa_data)) return -EACCES; @@ -1270,66 +1267,138 @@ static void iavf_up_complete(struct iavf_adapter *adapter) } /** - * iavf_down - Shutdown the connection processing + * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF + * yet and mark other to be removed. * @adapter: board private structure - * - * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. **/ -void iavf_down(struct iavf_adapter *adapter) +static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter) { - struct net_device *netdev = adapter->netdev; - struct iavf_vlan_filter *vlf; - struct iavf_cloud_filter *cf; - struct iavf_fdir_fltr *fdir; - struct iavf_mac_filter *f; - struct iavf_adv_rss *rss; - - if (adapter->state <= __IAVF_DOWN_PENDING) - return; - - netif_carrier_off(netdev); - netif_tx_disable(netdev); - adapter->link_up = false; - iavf_napi_disable_all(adapter); - iavf_irq_disable(adapter); + struct iavf_vlan_filter *vlf, *vlftmp; + struct iavf_mac_filter *f, *ftmp; spin_lock_bh(&adapter->mac_vlan_list_lock); - /* clear the sync flag on all filters */ __dev_uc_unsync(adapter->netdev, NULL); __dev_mc_unsync(adapter->netdev, NULL); /* remove all MAC filters */ - list_for_each_entry(f, &adapter->mac_filter_list, list) { - f->remove = true; + list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, + list) { + if (f->add) { + list_del(&f->list); + kfree(f); + } else { + f->remove = true; + } } /* remove all VLAN filters */ - list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { - vlf->remove = true; + list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, + list) { + if (vlf->add) { + list_del(&vlf->list); + kfree(vlf); + } else { + vlf->remove = true; + } } - spin_unlock_bh(&adapter->mac_vlan_list_lock); +} + +/** + * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and + * mark other to be removed. + * @adapter: board private structure + **/ +static void iavf_clear_cloud_filters(struct iavf_adapter *adapter) +{ + struct iavf_cloud_filter *cf, *cftmp; /* remove all cloud filters */ spin_lock_bh(&adapter->cloud_filter_list_lock); - list_for_each_entry(cf, &adapter->cloud_filter_list, list) { - cf->del = true; + list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, + list) { + if (cf->add) { + list_del(&cf->list); + kfree(cf); + adapter->num_cloud_filters--; + } else { + cf->del = true; + } } spin_unlock_bh(&adapter->cloud_filter_list_lock); +} + +/** + * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark + * other to be removed. + * @adapter: board private structure + **/ +static void iavf_clear_fdir_filters(struct iavf_adapter *adapter) +{ + struct iavf_fdir_fltr *fdir, *fdirtmp; /* remove all Flow Director filters */ spin_lock_bh(&adapter->fdir_fltr_lock); - list_for_each_entry(fdir, &adapter->fdir_list_head, list) { - fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST; + list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, + list) { + if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) { + list_del(&fdir->list); + kfree(fdir); + adapter->fdir_active_fltr--; + } else { + fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST; + } } spin_unlock_bh(&adapter->fdir_fltr_lock); +} + +/** + * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark + * other to be removed. + * @adapter: board private structure + **/ +static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter) +{ + struct iavf_adv_rss *rss, *rsstmp; /* remove all advance RSS configuration */ spin_lock_bh(&adapter->adv_rss_lock); - list_for_each_entry(rss, &adapter->adv_rss_list_head, list) - rss->state = IAVF_ADV_RSS_DEL_REQUEST; + list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head, + list) { + if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) { + list_del(&rss->list); + kfree(rss); + } else { + rss->state = IAVF_ADV_RSS_DEL_REQUEST; + } + } spin_unlock_bh(&adapter->adv_rss_lock); +} + +/** + * iavf_down - Shutdown the connection processing + * @adapter: board private structure + * + * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. + **/ +void iavf_down(struct iavf_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + + if (adapter->state <= __IAVF_DOWN_PENDING) + return; + + netif_carrier_off(netdev); + netif_tx_disable(netdev); + adapter->link_up = false; + iavf_napi_disable_all(adapter); + iavf_irq_disable(adapter); + + iavf_clear_mac_vlan_filters(adapter); + iavf_clear_cloud_filters(adapter); + iavf_clear_fdir_filters(adapter); + iavf_clear_adv_rss_conf(adapter); if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) { /* cancel any current operation */ @@ -1338,11 +1407,16 @@ void iavf_down(struct iavf_adapter *adapter) * here for this to complete. The watchdog is still running * and it will take care of this. */ - adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER; - adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; - adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; - adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; - adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG; + if (!list_empty(&adapter->mac_filter_list)) + adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; + if (!list_empty(&adapter->vlan_filter_list)) + adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; + if (!list_empty(&adapter->cloud_filter_list)) + adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; + if (!list_empty(&adapter->fdir_list_head)) + adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; + if (!list_empty(&adapter->adv_rss_list_head)) + adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG; adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES; } @@ -1757,7 +1831,7 @@ static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) q_vector->reg_idx = q_idx; cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); netif_napi_add(adapter->netdev, &q_vector->napi, - iavf_napi_poll, NAPI_POLL_WEIGHT); + iavf_napi_poll); } return 0; @@ -2877,6 +2951,11 @@ static void iavf_reset_task(struct work_struct *work) int i = 0, err; bool running; + /* Detach interface to avoid subsequent NDO callbacks */ + rtnl_lock(); + netif_device_detach(netdev); + rtnl_unlock(); + /* When device is being removed it doesn't make sense to run the reset * task, just return in such a case. */ @@ -2884,7 +2963,7 @@ static void iavf_reset_task(struct work_struct *work) if (adapter->state != __IAVF_REMOVE) queue_work(iavf_wq, &adapter->reset_task); - return; + goto reset_finish; } while (!mutex_trylock(&adapter->client_lock)) @@ -2954,7 +3033,6 @@ continue_reset: if (running) { netif_carrier_off(netdev); - netif_tx_stop_all_queues(netdev); adapter->link_up = false; iavf_napi_disable_all(adapter); } @@ -3084,7 +3162,7 @@ continue_reset: mutex_unlock(&adapter->client_lock); mutex_unlock(&adapter->crit_lock); - return; + goto reset_finish; reset_err: if (running) { set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); @@ -3095,6 +3173,10 @@ reset_err: mutex_unlock(&adapter->client_lock); mutex_unlock(&adapter->crit_lock); dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); +reset_finish: + rtnl_lock(); + netif_device_attach(netdev); + rtnl_unlock(); } /** @@ -4173,6 +4255,7 @@ err_unlock: static int iavf_close(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); + u64 aq_to_restore; int status; mutex_lock(&adapter->crit_lock); @@ -4185,6 +4268,29 @@ static int iavf_close(struct net_device *netdev) set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); if (CLIENT_ENABLED(adapter)) adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; + /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before + * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl + * deadlock with adminq_task() until iavf_close timeouts. We must send + * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make + * disable queues possible for vf. Give only necessary flags to + * iavf_down and save other to set them right before iavf_close() + * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and + * iavf will be in DOWN state. + */ + aq_to_restore = adapter->aq_required; + adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG; + + /* Remove flags which we do not want to send after close or we want to + * send before disable queues. + */ + aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG | + IAVF_FLAG_AQ_ENABLE_QUEUES | + IAVF_FLAG_AQ_CONFIGURE_QUEUES | + IAVF_FLAG_AQ_ADD_VLAN_FILTER | + IAVF_FLAG_AQ_ADD_MAC_FILTER | + IAVF_FLAG_AQ_ADD_CLOUD_FILTER | + IAVF_FLAG_AQ_ADD_FDIR_FILTER | + IAVF_FLAG_AQ_ADD_ADV_RSS_CFG); iavf_down(adapter); iavf_change_state(adapter, __IAVF_DOWN_PENDING); @@ -4208,6 +4314,10 @@ static int iavf_close(struct net_device *netdev) msecs_to_jiffies(500)); if (!status) netdev_warn(netdev, "Device resources not yet released\n"); + + mutex_lock(&adapter->crit_lock); + adapter->aq_required |= aq_to_restore; + mutex_unlock(&adapter->crit_lock); return 0; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index 06d18797d25a28c4406f5f3247ab360c3ceb3ef8..18b6a702a1d6dff9e9aeac001526444a6ca38736 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -114,8 +114,11 @@ u32 iavf_get_tx_pending(struct iavf_ring *ring, bool in_sw) { u32 head, tail; + /* underlying hardware might not allow access and/or always return + * 0 for the head/tail registers so just use the cached values + */ head = ring->next_to_clean; - tail = readl(ring->tail); + tail = ring->next_to_use; if (head != tail) return (head < tail) ? @@ -1390,7 +1393,7 @@ static struct sk_buff *iavf_build_skb(struct iavf_ring *rx_ring, #endif struct sk_buff *skb; - if (!rx_buffer) + if (!rx_buffer || !size) return NULL; /* prefetch first cache line of first page */ va = page_address(rx_buffer->page) + rx_buffer->page_offset; @@ -1548,7 +1551,7 @@ static int iavf_clean_rx_irq(struct iavf_ring *rx_ring, int budget) /* exit if we failed to retrieve a buffer */ if (!skb) { rx_ring->rx_stats.alloc_buff_failed++; - if (rx_buffer) + if (rx_buffer && size) rx_buffer->pagecnt_bias++; break; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 15ee85dc33bd817829148401b0b75d7527d3bcd3..5a9e6563923ebd7de870e20faf58df474a961deb 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -269,11 +269,14 @@ int iavf_get_vf_vlan_v2_caps(struct iavf_adapter *adapter) void iavf_configure_queues(struct iavf_adapter *adapter) { struct virtchnl_vsi_queue_config_info *vqci; - struct virtchnl_queue_pair_info *vqpi; + int i, max_frame = adapter->vf_res->max_mtu; int pairs = adapter->num_active_queues; - int i, max_frame = IAVF_MAX_RXBUFFER; + struct virtchnl_queue_pair_info *vqpi; size_t len; + if (max_frame > IAVF_MAX_RXBUFFER || !max_frame) + max_frame = IAVF_MAX_RXBUFFER; + if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { /* bail because we already have a command pending */ dev_err(&adapter->pdev->dev, "Cannot configure queues, command %d pending\n", diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index cc5b85afd437e70b8a43a910cccda29bd16e7603..001500afc4a6e708610868f5c0e9666f505c7f51 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -684,8 +684,8 @@ static inline void ice_set_ring_xdp(struct ice_tx_ring *ring) * ice_xsk_pool - get XSK buffer pool bound to a ring * @ring: Rx ring to use * - * Returns a pointer to xdp_umem structure if there is a buffer pool present, - * NULL otherwise. + * Returns a pointer to xsk_buff_pool structure if there is a buffer pool + * present, NULL otherwise. */ static inline struct xsk_buff_pool *ice_xsk_pool(struct ice_rx_ring *ring) { @@ -699,23 +699,33 @@ static inline struct xsk_buff_pool *ice_xsk_pool(struct ice_rx_ring *ring) } /** - * ice_tx_xsk_pool - get XSK buffer pool bound to a ring - * @ring: Tx ring to use + * ice_tx_xsk_pool - assign XSK buff pool to XDP ring + * @vsi: pointer to VSI + * @qid: index of a queue to look at XSK buff pool presence * - * Returns a pointer to xdp_umem structure if there is a buffer pool present, - * NULL otherwise. Tx equivalent of ice_xsk_pool. + * Sets XSK buff pool pointer on XDP ring. + * + * XDP ring is picked from Rx ring, whereas Rx ring is picked based on provided + * queue id. Reason for doing so is that queue vectors might have assigned more + * than one XDP ring, e.g. when user reduced the queue count on netdev; Rx ring + * carries a pointer to one of these XDP rings for its own purposes, such as + * handling XDP_TX action, therefore we can piggyback here on the + * rx_ring->xdp_ring assignment that was done during XDP rings initialization. */ -static inline struct xsk_buff_pool *ice_tx_xsk_pool(struct ice_tx_ring *ring) +static inline void ice_tx_xsk_pool(struct ice_vsi *vsi, u16 qid) { - struct ice_vsi *vsi = ring->vsi; - u16 qid; + struct ice_tx_ring *ring; - qid = ring->q_index - vsi->alloc_txq; + ring = vsi->rx_rings[qid]->xdp_ring; + if (!ring) + return; - if (!ice_is_xdp_ena_vsi(vsi) || !test_bit(qid, vsi->af_xdp_zc_qps)) - return NULL; + if (!ice_is_xdp_ena_vsi(vsi) || !test_bit(qid, vsi->af_xdp_zc_qps)) { + ring->xsk_pool = NULL; + return; + } - return xsk_get_pool_from_qid(vsi->netdev, qid); + ring->xsk_pool = xsk_get_pool_from_qid(vsi->netdev, qid); } /** @@ -854,6 +864,7 @@ ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, struct ice_q_stats stats, u64 *pkts, u64 *bytes); int ice_up(struct ice_vsi *vsi); int ice_down(struct ice_vsi *vsi); +int ice_down_up(struct ice_vsi *vsi); int ice_vsi_cfg(struct ice_vsi *vsi); struct ice_vsi *ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi); int ice_vsi_determine_xdp_res(struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 9939238573a4f775364786236ed6e2d02483f599..1bdc70aa979dcc8ddb0aa79faf68f40a484d4580 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -1423,6 +1423,56 @@ struct ice_aqc_set_port_id_led { u8 rsvd[13]; }; +/* Get Port Options (indirect, 0x06EA) */ +struct ice_aqc_get_port_options { + u8 lport_num; + u8 lport_num_valid; + u8 port_options_count; +#define ICE_AQC_PORT_OPT_COUNT_M GENMASK(3, 0) +#define ICE_AQC_PORT_OPT_MAX 16 + + u8 innermost_phy_index; + u8 port_options; +#define ICE_AQC_PORT_OPT_ACTIVE_M GENMASK(3, 0) +#define ICE_AQC_PORT_OPT_VALID BIT(7) + + u8 pending_port_option_status; +#define ICE_AQC_PENDING_PORT_OPT_IDX_M GENMASK(3, 0) +#define ICE_AQC_PENDING_PORT_OPT_VALID BIT(7) + + u8 rsvd[2]; + __le32 addr_high; + __le32 addr_low; +}; + +struct ice_aqc_get_port_options_elem { + u8 pmd; +#define ICE_AQC_PORT_OPT_PMD_COUNT_M GENMASK(3, 0) + + u8 max_lane_speed; +#define ICE_AQC_PORT_OPT_MAX_LANE_M GENMASK(3, 0) +#define ICE_AQC_PORT_OPT_MAX_LANE_100M 0 +#define ICE_AQC_PORT_OPT_MAX_LANE_1G 1 +#define ICE_AQC_PORT_OPT_MAX_LANE_2500M 2 +#define ICE_AQC_PORT_OPT_MAX_LANE_5G 3 +#define ICE_AQC_PORT_OPT_MAX_LANE_10G 4 +#define ICE_AQC_PORT_OPT_MAX_LANE_25G 5 +#define ICE_AQC_PORT_OPT_MAX_LANE_50G 6 +#define ICE_AQC_PORT_OPT_MAX_LANE_100G 7 + + u8 global_scid[2]; + u8 phy_scid[2]; + u8 pf2port_cid[2]; +}; + +/* Set Port Option (direct, 0x06EB) */ +struct ice_aqc_set_port_option { + u8 lport_num; + u8 lport_num_valid; + u8 selected_port_option; + u8 rsvd[13]; +}; + /* Set/Get GPIO (direct, 0x06EC/0x06ED) */ struct ice_aqc_gpio { __le16 gpio_ctrl_handle; @@ -1489,6 +1539,12 @@ struct ice_aqc_nvm { #define ICE_AQC_NVM_PERST_FLAG 1 #define ICE_AQC_NVM_EMPR_FLAG 2 #define ICE_AQC_NVM_EMPR_ENA BIT(0) /* Write Activate reply only */ + /* For Write Activate, several flags are sent as part of a separate + * flags2 field using a separate byte. For simplicity of the software + * interface, we pass the flags as a 16 bit value so these flags are + * all offset by 8 bits + */ +#define ICE_AQC_NVM_ACTIV_REQ_EMPR BIT(8) /* NVM Write Activate only */ __le16 module_typeid; __le16 length; #define ICE_AQC_NVM_ERASE_LEN 0xFFFF @@ -2082,6 +2138,8 @@ struct ice_aq_desc { struct ice_aqc_gpio read_write_gpio; struct ice_aqc_sff_eeprom read_write_sff_param; struct ice_aqc_set_port_id_led set_port_id_led; + struct ice_aqc_get_port_options get_port_options; + struct ice_aqc_set_port_option set_port_option; struct ice_aqc_get_sw_cfg get_sw_conf; struct ice_aqc_set_port_params set_port_params; struct ice_aqc_sw_rules sw_rules; @@ -2243,6 +2301,8 @@ enum ice_adminq_opc { ice_aqc_opc_read_i2c = 0x06E2, ice_aqc_opc_write_i2c = 0x06E3, ice_aqc_opc_set_port_id_led = 0x06E9, + ice_aqc_opc_get_port_options = 0x06EA, + ice_aqc_opc_set_port_option = 0x06EB, ice_aqc_opc_set_gpio = 0x06EC, ice_aqc_opc_get_gpio = 0x06ED, ice_aqc_opc_sff_eeprom = 0x06EE, diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index 136d7911adb48915360dacfeedbb4a6fc63dd134..9e36f01dfa4fb70c2d3ec7e3e0876b2bf3deadeb 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -7,18 +7,6 @@ #include "ice_dcb_lib.h" #include "ice_sriov.h" -static bool ice_alloc_rx_buf_zc(struct ice_rx_ring *rx_ring) -{ - rx_ring->xdp_buf = kcalloc(rx_ring->count, sizeof(*rx_ring->xdp_buf), GFP_KERNEL); - return !!rx_ring->xdp_buf; -} - -static bool ice_alloc_rx_buf(struct ice_rx_ring *rx_ring) -{ - rx_ring->rx_buf = kcalloc(rx_ring->count, sizeof(*rx_ring->rx_buf), GFP_KERNEL); - return !!rx_ring->rx_buf; -} - /** * __ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI * @qs_cfg: gathered variables needed for PF->VSI queues assignment @@ -142,8 +130,7 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx) * handler here (i.e. resume, reset/rebuild, etc.) */ if (vsi->netdev) - netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll); out: /* tie q_vector and VSI together */ @@ -417,7 +404,7 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring) /* Strip the Ethernet CRC bytes before the packet is posted to host * memory. */ - rlan_ctx.crcstrip = 1; + rlan_ctx.crcstrip = !(ring->flags & ICE_RX_FLAGS_CRC_STRIP_DIS); /* L2TSEL flag defines the reported L2 Tags in the receive descriptor * and it needs to remain 1 for non-DVM capable configurations to not @@ -519,11 +506,8 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->q_index, ring->q_vector->napi.napi_id); - kfree(ring->rx_buf); ring->xsk_pool = ice_xsk_pool(ring); if (ring->xsk_pool) { - if (!ice_alloc_rx_buf_zc(ring)) - return -ENOMEM; xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); ring->rx_buf_len = @@ -538,8 +522,6 @@ int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) dev_info(dev, "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", ring->q_index); } else { - if (!ice_alloc_rx_buf(ring)) - return -ENOMEM; if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) /* coverity[check_return] */ xdp_rxq_info_reg(&ring->xdp_rxq, diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 27d0cbbd29dad9736b527e3e83570d49095d1ba2..039342a0ed15a3c37794fac3039b17c8d08a705a 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -8,6 +8,108 @@ #define ICE_PF_RESET_WAIT_COUNT 300 +static const char * const ice_link_mode_str_low[] = { + [0] = "100BASE_TX", + [1] = "100M_SGMII", + [2] = "1000BASE_T", + [3] = "1000BASE_SX", + [4] = "1000BASE_LX", + [5] = "1000BASE_KX", + [6] = "1G_SGMII", + [7] = "2500BASE_T", + [8] = "2500BASE_X", + [9] = "2500BASE_KX", + [10] = "5GBASE_T", + [11] = "5GBASE_KR", + [12] = "10GBASE_T", + [13] = "10G_SFI_DA", + [14] = "10GBASE_SR", + [15] = "10GBASE_LR", + [16] = "10GBASE_KR_CR1", + [17] = "10G_SFI_AOC_ACC", + [18] = "10G_SFI_C2C", + [19] = "25GBASE_T", + [20] = "25GBASE_CR", + [21] = "25GBASE_CR_S", + [22] = "25GBASE_CR1", + [23] = "25GBASE_SR", + [24] = "25GBASE_LR", + [25] = "25GBASE_KR", + [26] = "25GBASE_KR_S", + [27] = "25GBASE_KR1", + [28] = "25G_AUI_AOC_ACC", + [29] = "25G_AUI_C2C", + [30] = "40GBASE_CR4", + [31] = "40GBASE_SR4", + [32] = "40GBASE_LR4", + [33] = "40GBASE_KR4", + [34] = "40G_XLAUI_AOC_ACC", + [35] = "40G_XLAUI", + [36] = "50GBASE_CR2", + [37] = "50GBASE_SR2", + [38] = "50GBASE_LR2", + [39] = "50GBASE_KR2", + [40] = "50G_LAUI2_AOC_ACC", + [41] = "50G_LAUI2", + [42] = "50G_AUI2_AOC_ACC", + [43] = "50G_AUI2", + [44] = "50GBASE_CP", + [45] = "50GBASE_SR", + [46] = "50GBASE_FR", + [47] = "50GBASE_LR", + [48] = "50GBASE_KR_PAM4", + [49] = "50G_AUI1_AOC_ACC", + [50] = "50G_AUI1", + [51] = "100GBASE_CR4", + [52] = "100GBASE_SR4", + [53] = "100GBASE_LR4", + [54] = "100GBASE_KR4", + [55] = "100G_CAUI4_AOC_ACC", + [56] = "100G_CAUI4", + [57] = "100G_AUI4_AOC_ACC", + [58] = "100G_AUI4", + [59] = "100GBASE_CR_PAM4", + [60] = "100GBASE_KR_PAM4", + [61] = "100GBASE_CP2", + [62] = "100GBASE_SR2", + [63] = "100GBASE_DR", +}; + +static const char * const ice_link_mode_str_high[] = { + [0] = "100GBASE_KR2_PAM4", + [1] = "100G_CAUI2_AOC_ACC", + [2] = "100G_CAUI2", + [3] = "100G_AUI2_AOC_ACC", + [4] = "100G_AUI2", +}; + +/** + * ice_dump_phy_type - helper function to dump phy_type + * @hw: pointer to the HW structure + * @low: 64 bit value for phy_type_low + * @high: 64 bit value for phy_type_high + * @prefix: prefix string to differentiate multiple dumps + */ +static void +ice_dump_phy_type(struct ice_hw *hw, u64 low, u64 high, const char *prefix) +{ + ice_debug(hw, ICE_DBG_PHY, "%s: phy_type_low: 0x%016llx\n", prefix, low); + + for (u32 i = 0; i < BITS_PER_TYPE(typeof(low)); i++) { + if (low & BIT_ULL(i)) + ice_debug(hw, ICE_DBG_PHY, "%s: bit(%d): %s\n", + prefix, i, ice_link_mode_str_low[i]); + } + + ice_debug(hw, ICE_DBG_PHY, "%s: phy_type_high: 0x%016llx\n", prefix, high); + + for (u32 i = 0; i < BITS_PER_TYPE(typeof(high)); i++) { + if (high & BIT_ULL(i)) + ice_debug(hw, ICE_DBG_PHY, "%s: bit(%d): %s\n", + prefix, i, ice_link_mode_str_high[i]); + } +} + /** * ice_set_mac_type - Sets MAC type * @hw: pointer to the HW structure @@ -80,9 +182,23 @@ bool ice_is_e810t(struct ice_hw *hw) { switch (hw->device_id) { case ICE_DEV_ID_E810C_SFP: - if (hw->subsystem_device_id == ICE_SUBDEV_ID_E810T || - hw->subsystem_device_id == ICE_SUBDEV_ID_E810T2) + switch (hw->subsystem_device_id) { + case ICE_SUBDEV_ID_E810T: + case ICE_SUBDEV_ID_E810T2: + case ICE_SUBDEV_ID_E810T3: + case ICE_SUBDEV_ID_E810T4: + case ICE_SUBDEV_ID_E810T6: + case ICE_SUBDEV_ID_E810T7: + return true; + } + break; + case ICE_DEV_ID_E810C_QSFP: + switch (hw->subsystem_device_id) { + case ICE_SUBDEV_ID_E810T2: + case ICE_SUBDEV_ID_E810T3: + case ICE_SUBDEV_ID_E810T5: return true; + } break; default: break; @@ -183,6 +299,7 @@ ice_aq_get_phy_caps(struct ice_port_info *pi, bool qual_mods, u8 report_mode, struct ice_aqc_get_phy_caps *cmd; u16 pcaps_size = sizeof(*pcaps); struct ice_aq_desc desc; + const char *prefix; struct ice_hw *hw; int status; @@ -204,29 +321,48 @@ ice_aq_get_phy_caps(struct ice_port_info *pi, bool qual_mods, u8 report_mode, cmd->param0 |= cpu_to_le16(report_mode); status = ice_aq_send_cmd(hw, &desc, pcaps, pcaps_size, cd); - ice_debug(hw, ICE_DBG_LINK, "get phy caps - report_mode = 0x%x\n", - report_mode); - ice_debug(hw, ICE_DBG_LINK, " phy_type_low = 0x%llx\n", - (unsigned long long)le64_to_cpu(pcaps->phy_type_low)); - ice_debug(hw, ICE_DBG_LINK, " phy_type_high = 0x%llx\n", - (unsigned long long)le64_to_cpu(pcaps->phy_type_high)); - ice_debug(hw, ICE_DBG_LINK, " caps = 0x%x\n", pcaps->caps); - ice_debug(hw, ICE_DBG_LINK, " low_power_ctrl_an = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "get phy caps dump\n"); + + switch (report_mode) { + case ICE_AQC_REPORT_TOPO_CAP_MEDIA: + prefix = "phy_caps_media"; + break; + case ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA: + prefix = "phy_caps_no_media"; + break; + case ICE_AQC_REPORT_ACTIVE_CFG: + prefix = "phy_caps_active"; + break; + case ICE_AQC_REPORT_DFLT_CFG: + prefix = "phy_caps_default"; + break; + default: + prefix = "phy_caps_invalid"; + } + + ice_dump_phy_type(hw, le64_to_cpu(pcaps->phy_type_low), + le64_to_cpu(pcaps->phy_type_high), prefix); + + ice_debug(hw, ICE_DBG_LINK, "%s: report_mode = 0x%x\n", + prefix, report_mode); + ice_debug(hw, ICE_DBG_LINK, "%s: caps = 0x%x\n", prefix, pcaps->caps); + ice_debug(hw, ICE_DBG_LINK, "%s: low_power_ctrl_an = 0x%x\n", prefix, pcaps->low_power_ctrl_an); - ice_debug(hw, ICE_DBG_LINK, " eee_cap = 0x%x\n", pcaps->eee_cap); - ice_debug(hw, ICE_DBG_LINK, " eeer_value = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "%s: eee_cap = 0x%x\n", prefix, + pcaps->eee_cap); + ice_debug(hw, ICE_DBG_LINK, "%s: eeer_value = 0x%x\n", prefix, pcaps->eeer_value); - ice_debug(hw, ICE_DBG_LINK, " link_fec_options = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "%s: link_fec_options = 0x%x\n", prefix, pcaps->link_fec_options); - ice_debug(hw, ICE_DBG_LINK, " module_compliance_enforcement = 0x%x\n", - pcaps->module_compliance_enforcement); - ice_debug(hw, ICE_DBG_LINK, " extended_compliance_code = 0x%x\n", - pcaps->extended_compliance_code); - ice_debug(hw, ICE_DBG_LINK, " module_type[0] = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "%s: module_compliance_enforcement = 0x%x\n", + prefix, pcaps->module_compliance_enforcement); + ice_debug(hw, ICE_DBG_LINK, "%s: extended_compliance_code = 0x%x\n", + prefix, pcaps->extended_compliance_code); + ice_debug(hw, ICE_DBG_LINK, "%s: module_type[0] = 0x%x\n", prefix, pcaps->module_type[0]); - ice_debug(hw, ICE_DBG_LINK, " module_type[1] = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "%s: module_type[1] = 0x%x\n", prefix, pcaps->module_type[1]); - ice_debug(hw, ICE_DBG_LINK, " module_type[2] = 0x%x\n", + ice_debug(hw, ICE_DBG_LINK, "%s: module_type[2] = 0x%x\n", prefix, pcaps->module_type[2]); if (!status && report_mode == ICE_AQC_REPORT_TOPO_CAP_MEDIA) { @@ -2397,6 +2533,8 @@ ice_parse_1588_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p, info->tmr1_owned = ((number & ICE_TS_TMR1_OWND_M) != 0); info->tmr1_ena = ((number & ICE_TS_TMR1_ENA_M) != 0); + info->ts_ll_read = ((number & ICE_TS_LL_TX_TS_READ_M) != 0); + info->ena_ports = logical_id; info->tmr_own_map = phys_id; @@ -2414,6 +2552,8 @@ ice_parse_1588_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p, info->tmr1_owned); ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr1_ena = %u\n", info->tmr1_ena); + ice_debug(hw, ICE_DBG_INIT, "dev caps: ts_ll_read = %u\n", + info->ts_ll_read); ice_debug(hw, ICE_DBG_INIT, "dev caps: ieee_1588 ena_ports = %u\n", info->ena_ports); ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr_own_map = %u\n", @@ -2775,6 +2915,26 @@ ice_aq_set_port_params(struct ice_port_info *pi, bool double_vlan, return ice_aq_send_cmd(hw, &desc, NULL, 0, cd); } +/** + * ice_is_100m_speed_supported + * @hw: pointer to the HW struct + * + * returns true if 100M speeds are supported by the device, + * false otherwise. + */ +bool ice_is_100m_speed_supported(struct ice_hw *hw) +{ + switch (hw->device_id) { + case ICE_DEV_ID_E822C_SGMII: + case ICE_DEV_ID_E822L_SGMII: + case ICE_DEV_ID_E823L_1GBE: + case ICE_DEV_ID_E823C_SGMII: + return true; + default: + return false; + } +} + /** * ice_get_link_speed_based_on_phy_type - returns link speed * @phy_type_low: lower part of phy_type @@ -3534,6 +3694,121 @@ ice_aq_set_port_id_led(struct ice_port_info *pi, bool is_orig_mode, return ice_aq_send_cmd(hw, &desc, NULL, 0, cd); } +/** + * ice_aq_get_port_options + * @hw: pointer to the HW struct + * @options: buffer for the resultant port options + * @option_count: input - size of the buffer in port options structures, + * output - number of returned port options + * @lport: logical port to call the command with (optional) + * @lport_valid: when false, FW uses port owned by the PF instead of lport, + * when PF owns more than 1 port it must be true + * @active_option_idx: index of active port option in returned buffer + * @active_option_valid: active option in returned buffer is valid + * @pending_option_idx: index of pending port option in returned buffer + * @pending_option_valid: pending option in returned buffer is valid + * + * Calls Get Port Options AQC (0x06ea) and verifies result. + */ +int +ice_aq_get_port_options(struct ice_hw *hw, + struct ice_aqc_get_port_options_elem *options, + u8 *option_count, u8 lport, bool lport_valid, + u8 *active_option_idx, bool *active_option_valid, + u8 *pending_option_idx, bool *pending_option_valid) +{ + struct ice_aqc_get_port_options *cmd; + struct ice_aq_desc desc; + int status; + u8 i; + + /* options buffer shall be able to hold max returned options */ + if (*option_count < ICE_AQC_PORT_OPT_COUNT_M) + return -EINVAL; + + cmd = &desc.params.get_port_options; + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_port_options); + + if (lport_valid) + cmd->lport_num = lport; + cmd->lport_num_valid = lport_valid; + + status = ice_aq_send_cmd(hw, &desc, options, + *option_count * sizeof(*options), NULL); + if (status) + return status; + + /* verify direct FW response & set output parameters */ + *option_count = FIELD_GET(ICE_AQC_PORT_OPT_COUNT_M, + cmd->port_options_count); + ice_debug(hw, ICE_DBG_PHY, "options: %x\n", *option_count); + *active_option_valid = FIELD_GET(ICE_AQC_PORT_OPT_VALID, + cmd->port_options); + if (*active_option_valid) { + *active_option_idx = FIELD_GET(ICE_AQC_PORT_OPT_ACTIVE_M, + cmd->port_options); + if (*active_option_idx > (*option_count - 1)) + return -EIO; + ice_debug(hw, ICE_DBG_PHY, "active idx: %x\n", + *active_option_idx); + } + + *pending_option_valid = FIELD_GET(ICE_AQC_PENDING_PORT_OPT_VALID, + cmd->pending_port_option_status); + if (*pending_option_valid) { + *pending_option_idx = FIELD_GET(ICE_AQC_PENDING_PORT_OPT_IDX_M, + cmd->pending_port_option_status); + if (*pending_option_idx > (*option_count - 1)) + return -EIO; + ice_debug(hw, ICE_DBG_PHY, "pending idx: %x\n", + *pending_option_idx); + } + + /* mask output options fields */ + for (i = 0; i < *option_count; i++) { + options[i].pmd = FIELD_GET(ICE_AQC_PORT_OPT_PMD_COUNT_M, + options[i].pmd); + options[i].max_lane_speed = FIELD_GET(ICE_AQC_PORT_OPT_MAX_LANE_M, + options[i].max_lane_speed); + ice_debug(hw, ICE_DBG_PHY, "pmds: %x max speed: %x\n", + options[i].pmd, options[i].max_lane_speed); + } + + return 0; +} + +/** + * ice_aq_set_port_option + * @hw: pointer to the HW struct + * @lport: logical port to call the command with + * @lport_valid: when false, FW uses port owned by the PF instead of lport, + * when PF owns more than 1 port it must be true + * @new_option: new port option to be written + * + * Calls Set Port Options AQC (0x06eb). + */ +int +ice_aq_set_port_option(struct ice_hw *hw, u8 lport, u8 lport_valid, + u8 new_option) +{ + struct ice_aqc_set_port_option *cmd; + struct ice_aq_desc desc; + + if (new_option > ICE_AQC_PORT_OPT_COUNT_M) + return -EINVAL; + + cmd = &desc.params.set_port_option; + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_port_option); + + if (lport_valid) + cmd->lport_num = lport; + + cmd->lport_num_valid = lport_valid; + cmd->selected_port_option = new_option; + + return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL); +} + /** * ice_aq_sff_eeprom * @hw: pointer to the HW struct @@ -5029,26 +5304,41 @@ ice_aq_get_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, } /** - * ice_fw_supports_link_override + * ice_is_fw_api_min_ver * @hw: pointer to the hardware structure + * @maj: major version + * @min: minor version + * @patch: patch version * - * Checks if the firmware supports link override + * Checks if the firmware API is minimum version */ -bool ice_fw_supports_link_override(struct ice_hw *hw) +static bool ice_is_fw_api_min_ver(struct ice_hw *hw, u8 maj, u8 min, u8 patch) { - if (hw->api_maj_ver == ICE_FW_API_LINK_OVERRIDE_MAJ) { - if (hw->api_min_ver > ICE_FW_API_LINK_OVERRIDE_MIN) + if (hw->api_maj_ver == maj) { + if (hw->api_min_ver > min) return true; - if (hw->api_min_ver == ICE_FW_API_LINK_OVERRIDE_MIN && - hw->api_patch >= ICE_FW_API_LINK_OVERRIDE_PATCH) + if (hw->api_min_ver == min && hw->api_patch >= patch) return true; - } else if (hw->api_maj_ver > ICE_FW_API_LINK_OVERRIDE_MAJ) { + } else if (hw->api_maj_ver > maj) { return true; } return false; } +/** + * ice_fw_supports_link_override + * @hw: pointer to the hardware structure + * + * Checks if the firmware supports link override + */ +bool ice_fw_supports_link_override(struct ice_hw *hw) +{ + return ice_is_fw_api_min_ver(hw, ICE_FW_API_LINK_OVERRIDE_MAJ, + ICE_FW_API_LINK_OVERRIDE_MIN, + ICE_FW_API_LINK_OVERRIDE_PATCH); +} + /** * ice_get_link_default_override * @ldo: pointer to the link default override struct @@ -5179,16 +5469,9 @@ bool ice_fw_supports_lldp_fltr_ctrl(struct ice_hw *hw) if (hw->mac_type != ICE_MAC_E810) return false; - if (hw->api_maj_ver == ICE_FW_API_LLDP_FLTR_MAJ) { - if (hw->api_min_ver > ICE_FW_API_LLDP_FLTR_MIN) - return true; - if (hw->api_min_ver == ICE_FW_API_LLDP_FLTR_MIN && - hw->api_patch >= ICE_FW_API_LLDP_FLTR_PATCH) - return true; - } else if (hw->api_maj_ver > ICE_FW_API_LLDP_FLTR_MAJ) { - return true; - } - return false; + return ice_is_fw_api_min_ver(hw, ICE_FW_API_LLDP_FLTR_MAJ, + ICE_FW_API_LLDP_FLTR_MIN, + ICE_FW_API_LLDP_FLTR_PATCH); } /** @@ -5225,14 +5508,7 @@ ice_lldp_fltr_add_remove(struct ice_hw *hw, u16 vsi_num, bool add) */ bool ice_fw_supports_report_dflt_cfg(struct ice_hw *hw) { - if (hw->api_maj_ver == ICE_FW_API_REPORT_DFLT_CFG_MAJ) { - if (hw->api_min_ver > ICE_FW_API_REPORT_DFLT_CFG_MIN) - return true; - if (hw->api_min_ver == ICE_FW_API_REPORT_DFLT_CFG_MIN && - hw->api_patch >= ICE_FW_API_REPORT_DFLT_CFG_PATCH) - return true; - } else if (hw->api_maj_ver > ICE_FW_API_REPORT_DFLT_CFG_MAJ) { - return true; - } - return false; + return ice_is_fw_api_min_ver(hw, ICE_FW_API_REPORT_DFLT_CFG_MAJ, + ICE_FW_API_REPORT_DFLT_CFG_MIN, + ICE_FW_API_REPORT_DFLT_CFG_PATCH); } diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 61b7c60db689151d0a75d01d7cf6eb691bfd3123..8b6712b92e8482be1ecdca1ad369fe23ff4487ca 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -151,6 +151,15 @@ int ice_aq_set_port_id_led(struct ice_port_info *pi, bool is_orig_mode, struct ice_sq_cd *cd); int +ice_aq_get_port_options(struct ice_hw *hw, + struct ice_aqc_get_port_options_elem *options, + u8 *option_count, u8 lport, bool lport_valid, + u8 *active_option_idx, bool *active_option_valid, + u8 *pending_option_idx, bool *pending_option_valid); +int +ice_aq_set_port_option(struct ice_hw *hw, u8 lport, u8 lport_valid, + u8 new_option); +int ice_aq_sff_eeprom(struct ice_hw *hw, u16 lport, u8 bus_addr, u16 mem_addr, u8 page, u8 set_page, u8 *data, u8 length, bool write, struct ice_sq_cd *cd); @@ -204,6 +213,7 @@ ice_aq_set_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, bool value, int ice_aq_get_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, bool *value, struct ice_sq_cd *cd); +bool ice_is_100m_speed_supported(struct ice_hw *hw); int ice_aq_set_lldp_mib(struct ice_hw *hw, u8 mib_type, void *buf, u16 buf_size, struct ice_sq_cd *cd); diff --git a/drivers/net/ethernet/intel/ice/ice_devids.h b/drivers/net/ethernet/intel/ice/ice_devids.h index b41bc3dc174547cb4f74bf8900f24b164cf680cb..6d560d1c74a4a354db6818a9bf2f2320f991f86f 100644 --- a/drivers/net/ethernet/intel/ice/ice_devids.h +++ b/drivers/net/ethernet/intel/ice/ice_devids.h @@ -24,6 +24,11 @@ #define ICE_DEV_ID_E810C_SFP 0x1593 #define ICE_SUBDEV_ID_E810T 0x000E #define ICE_SUBDEV_ID_E810T2 0x000F +#define ICE_SUBDEV_ID_E810T3 0x0010 +#define ICE_SUBDEV_ID_E810T4 0x0011 +#define ICE_SUBDEV_ID_E810T5 0x0012 +#define ICE_SUBDEV_ID_E810T6 0x02E9 +#define ICE_SUBDEV_ID_E810T7 0x02EA /* Intel(R) Ethernet Controller E810-XXV for backplane */ #define ICE_DEV_ID_E810_XXV_BACKPLANE 0x1599 /* Intel(R) Ethernet Controller E810-XXV for QSFP */ diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index 3337314a7b35eccc460319d6d5a9161d6a3b27d2..e6ec20079ced96910b049cbb91da5480f419f914 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -9,6 +9,8 @@ #include "ice_eswitch.h" #include "ice_fw_update.h" +static int ice_active_port_option = -1; + /* context for devlink info version reporting */ struct ice_info_ctx { char buf[128]; @@ -466,12 +468,259 @@ ice_devlink_reload_empr_finish(struct devlink *devlink, return 0; } +/** + * ice_devlink_port_opt_speed_str - convert speed to a string + * @speed: speed value + */ +static const char *ice_devlink_port_opt_speed_str(u8 speed) +{ + switch (speed & ICE_AQC_PORT_OPT_MAX_LANE_M) { + case ICE_AQC_PORT_OPT_MAX_LANE_100M: + return "0.1"; + case ICE_AQC_PORT_OPT_MAX_LANE_1G: + return "1"; + case ICE_AQC_PORT_OPT_MAX_LANE_2500M: + return "2.5"; + case ICE_AQC_PORT_OPT_MAX_LANE_5G: + return "5"; + case ICE_AQC_PORT_OPT_MAX_LANE_10G: + return "10"; + case ICE_AQC_PORT_OPT_MAX_LANE_25G: + return "25"; + case ICE_AQC_PORT_OPT_MAX_LANE_50G: + return "50"; + case ICE_AQC_PORT_OPT_MAX_LANE_100G: + return "100"; + } + + return "-"; +} + +#define ICE_PORT_OPT_DESC_LEN 50 +/** + * ice_devlink_port_options_print - Print available port split options + * @pf: the PF to print split port options + * + * Prints a table with available port split options and max port speeds + */ +static void ice_devlink_port_options_print(struct ice_pf *pf) +{ + u8 i, j, options_count, cnt, speed, pending_idx, active_idx; + struct ice_aqc_get_port_options_elem *options, *opt; + struct device *dev = ice_pf_to_dev(pf); + bool active_valid, pending_valid; + char desc[ICE_PORT_OPT_DESC_LEN]; + const char *str; + int status; + + options = kcalloc(ICE_AQC_PORT_OPT_MAX * ICE_MAX_PORT_PER_PCI_DEV, + sizeof(*options), GFP_KERNEL); + if (!options) + return; + + for (i = 0; i < ICE_MAX_PORT_PER_PCI_DEV; i++) { + opt = options + i * ICE_AQC_PORT_OPT_MAX; + options_count = ICE_AQC_PORT_OPT_MAX; + active_valid = 0; + + status = ice_aq_get_port_options(&pf->hw, opt, &options_count, + i, true, &active_idx, + &active_valid, &pending_idx, + &pending_valid); + if (status) { + dev_dbg(dev, "Couldn't read port option for port %d, err %d\n", + i, status); + goto err; + } + } + + dev_dbg(dev, "Available port split options and max port speeds (Gbps):\n"); + dev_dbg(dev, "Status Split Quad 0 Quad 1\n"); + dev_dbg(dev, " count L0 L1 L2 L3 L4 L5 L6 L7\n"); + + for (i = 0; i < options_count; i++) { + cnt = 0; + + if (i == ice_active_port_option) + str = "Active"; + else if ((i == pending_idx) && pending_valid) + str = "Pending"; + else + str = ""; + + cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt, + "%-8s", str); + + cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt, + "%-6u", options[i].pmd); + + for (j = 0; j < ICE_MAX_PORT_PER_PCI_DEV; ++j) { + speed = options[i + j * ICE_AQC_PORT_OPT_MAX].max_lane_speed; + str = ice_devlink_port_opt_speed_str(speed); + cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt, + "%3s ", str); + } + + dev_dbg(dev, "%s\n", desc); + } + +err: + kfree(options); +} + +/** + * ice_devlink_aq_set_port_option - Send set port option admin queue command + * @pf: the PF to print split port options + * @option_idx: selected port option + * @extack: extended netdev ack structure + * + * Sends set port option admin queue command with selected port option and + * calls NVM write activate. + */ +static int +ice_devlink_aq_set_port_option(struct ice_pf *pf, u8 option_idx, + struct netlink_ext_ack *extack) +{ + struct device *dev = ice_pf_to_dev(pf); + int status; + + status = ice_aq_set_port_option(&pf->hw, 0, true, option_idx); + if (status) { + dev_dbg(dev, "ice_aq_set_port_option, err %d aq_err %d\n", + status, pf->hw.adminq.sq_last_status); + NL_SET_ERR_MSG_MOD(extack, "Port split request failed"); + return -EIO; + } + + status = ice_acquire_nvm(&pf->hw, ICE_RES_WRITE); + if (status) { + dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", + status, pf->hw.adminq.sq_last_status); + NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); + return -EIO; + } + + status = ice_nvm_write_activate(&pf->hw, ICE_AQC_NVM_ACTIV_REQ_EMPR, NULL); + if (status) { + dev_dbg(dev, "ice_nvm_write_activate failed, err %d aq_err %d\n", + status, pf->hw.adminq.sq_last_status); + NL_SET_ERR_MSG_MOD(extack, "Port split request failed to save data"); + ice_release_nvm(&pf->hw); + return -EIO; + } + + ice_release_nvm(&pf->hw); + + NL_SET_ERR_MSG_MOD(extack, "Reboot required to finish port split"); + return 0; +} + +/** + * ice_devlink_port_split - .port_split devlink handler + * @devlink: devlink instance structure + * @port: devlink port structure + * @count: number of ports to split to + * @extack: extended netdev ack structure + * + * Callback for the devlink .port_split operation. + * + * Unfortunately, the devlink expression of available options is limited + * to just a number, so search for an FW port option which supports + * the specified number. As there could be multiple FW port options with + * the same port split count, allow switching between them. When the same + * port split count request is issued again, switch to the next FW port + * option with the same port split count. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_split(struct devlink *devlink, struct devlink_port *port, + unsigned int count, struct netlink_ext_ack *extack) +{ + struct ice_aqc_get_port_options_elem options[ICE_AQC_PORT_OPT_MAX]; + u8 i, j, active_idx, pending_idx, new_option; + struct ice_pf *pf = devlink_priv(devlink); + u8 option_count = ICE_AQC_PORT_OPT_MAX; + struct device *dev = ice_pf_to_dev(pf); + bool active_valid, pending_valid; + int status; + + status = ice_aq_get_port_options(&pf->hw, options, &option_count, + 0, true, &active_idx, &active_valid, + &pending_idx, &pending_valid); + if (status) { + dev_dbg(dev, "Couldn't read port split options, err = %d\n", + status); + NL_SET_ERR_MSG_MOD(extack, "Failed to get available port split options"); + return -EIO; + } + + new_option = ICE_AQC_PORT_OPT_MAX; + active_idx = pending_valid ? pending_idx : active_idx; + for (i = 1; i <= option_count; i++) { + /* In order to allow switching between FW port options with + * the same port split count, search for a new option starting + * from the active/pending option (with array wrap around). + */ + j = (active_idx + i) % option_count; + + if (count == options[j].pmd) { + new_option = j; + break; + } + } + + if (new_option == active_idx) { + dev_dbg(dev, "request to split: count: %u is already set and there are no other options\n", + count); + NL_SET_ERR_MSG_MOD(extack, "Requested split count is already set"); + ice_devlink_port_options_print(pf); + return -EINVAL; + } + + if (new_option == ICE_AQC_PORT_OPT_MAX) { + dev_dbg(dev, "request to split: count: %u not found\n", count); + NL_SET_ERR_MSG_MOD(extack, "Port split requested unsupported port config"); + ice_devlink_port_options_print(pf); + return -EINVAL; + } + + status = ice_devlink_aq_set_port_option(pf, new_option, extack); + if (status) + return status; + + ice_devlink_port_options_print(pf); + + return 0; +} + +/** + * ice_devlink_port_unsplit - .port_unsplit devlink handler + * @devlink: devlink instance structure + * @port: devlink port structure + * @extack: extended netdev ack structure + * + * Callback for the devlink .port_unsplit operation. + * Calls ice_devlink_port_split with split count set to 1. + * There could be no FW option available with split count 1. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_unsplit(struct devlink *devlink, struct devlink_port *port, + struct netlink_ext_ack *extack) +{ + return ice_devlink_port_split(devlink, port, 1, extack); +} + static const struct devlink_ops ice_devlink_ops = { .supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK, .reload_actions = BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE), /* The ice driver currently does not support driver reinit */ .reload_down = ice_devlink_reload_empr_start, .reload_up = ice_devlink_reload_empr_finish, + .port_split = ice_devlink_port_split, + .port_unsplit = ice_devlink_port_unsplit, .eswitch_mode_get = ice_eswitch_mode_get, .eswitch_mode_set = ice_eswitch_mode_set, .info_get = ice_devlink_info_get, @@ -694,6 +943,39 @@ void ice_devlink_unregister_params(struct ice_pf *pf) ARRAY_SIZE(ice_devlink_params)); } +/** + * ice_devlink_set_port_split_options - Set port split options + * @pf: the PF to set port split options + * @attrs: devlink attributes + * + * Sets devlink port split options based on available FW port options + */ +static void +ice_devlink_set_port_split_options(struct ice_pf *pf, + struct devlink_port_attrs *attrs) +{ + struct ice_aqc_get_port_options_elem options[ICE_AQC_PORT_OPT_MAX]; + u8 i, active_idx, pending_idx, option_count = ICE_AQC_PORT_OPT_MAX; + bool active_valid, pending_valid; + int status; + + status = ice_aq_get_port_options(&pf->hw, options, &option_count, + 0, true, &active_idx, &active_valid, + &pending_idx, &pending_valid); + if (status) { + dev_dbg(ice_pf_to_dev(pf), "Couldn't read port split options, err = %d\n", + status); + return; + } + + /* find the biggest available port split count */ + for (i = 0; i < option_count; i++) + attrs->lanes = max_t(int, attrs->lanes, options[i].pmd); + + attrs->splittable = attrs->lanes ? 1 : 0; + ice_active_port_option = active_idx; +} + /** * ice_devlink_create_pf_port - Create a devlink port for this PF * @pf: the PF to create a devlink port for @@ -722,6 +1004,12 @@ int ice_devlink_create_pf_port(struct ice_pf *pf) attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; attrs.phys.port_number = pf->hw.bus.func; + /* As FW supports only port split options for whole device, + * set port split options only for first PF. + */ + if (pf->hw.pf_id == 0) + ice_devlink_set_port_split_options(pf, &attrs); + ice_devlink_set_switch_id(pf, &attrs.switch_id); devlink_port_attrs_set(devlink_port, &attrs); diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index e35371e61e078538fc8b067506a31e510c368374..f9f15acae90a0536ed024d07c54ee125fde45a96 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -292,8 +292,8 @@ static int ice_eswitch_setup_reprs(struct ice_pf *pf) if (max_vsi_num < vsi->vsi_num) max_vsi_num = vsi->vsi_num; - netif_napi_add(vf->repr->netdev, &vf->repr->q_vector->napi, ice_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(vf->repr->netdev, &vf->repr->q_vector->napi, + ice_napi_poll); netif_keep_dst(vf->repr->netdev); } diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index a6fff8ebaf9d9ebcc5248ddcf63e13861659cbe2..b7be84bbe72d666b4844f372496aeec47ba079ae 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -136,6 +136,11 @@ static const struct ice_stats ice_gstrings_pf_stats[] = { ICE_PF_STAT("mac_remote_faults.nic", stats.mac_remote_faults), ICE_PF_STAT("fdir_sb_match.nic", stats.fd_sb_match), ICE_PF_STAT("fdir_sb_status.nic", stats.fd_sb_status), + ICE_PF_STAT("tx_hwtstamp_skipped", ptp.tx_hwtstamp_skipped), + ICE_PF_STAT("tx_hwtstamp_timeouts", ptp.tx_hwtstamp_timeouts), + ICE_PF_STAT("tx_hwtstamp_flushed", ptp.tx_hwtstamp_flushed), + ICE_PF_STAT("tx_hwtstamp_discarded", ptp.tx_hwtstamp_discarded), + ICE_PF_STAT("late_cached_phc_updates", ptp.late_cached_phc_updates), }; static const u32 ice_regs_dump_list[] = { @@ -1284,10 +1289,7 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags) } if (test_bit(ICE_FLAG_LEGACY_RX, change_flags)) { /* down and up VSI so that changes of Rx cfg are reflected. */ - if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { - ice_down(vsi); - ice_up(vsi); - } + ice_down_up(vsi); } /* don't allow modification of this flag when a single VF is in * promiscuous mode because it's not supported @@ -1468,20 +1470,22 @@ ice_get_ethtool_stats(struct net_device *netdev, /** * ice_mask_min_supported_speeds + * @hw: pointer to the HW structure * @phy_types_high: PHY type high * @phy_types_low: PHY type low to apply minimum supported speeds mask * * Apply minimum supported speeds mask to PHY type low. These are the speeds * for ethtool supported link mode. */ -static -void ice_mask_min_supported_speeds(u64 phy_types_high, u64 *phy_types_low) +static void +ice_mask_min_supported_speeds(struct ice_hw *hw, + u64 phy_types_high, u64 *phy_types_low) { /* if QSFP connection with 100G speed, minimum supported speed is 25G */ if (*phy_types_low & ICE_PHY_TYPE_LOW_MASK_100G || phy_types_high & ICE_PHY_TYPE_HIGH_MASK_100G) *phy_types_low &= ~ICE_PHY_TYPE_LOW_MASK_MIN_25G; - else + else if (!ice_is_100m_speed_supported(hw)) *phy_types_low &= ~ICE_PHY_TYPE_LOW_MASK_MIN_1G; } @@ -1531,7 +1535,8 @@ ice_phy_type_to_ethtool(struct net_device *netdev, phy_types_low = le64_to_cpu(pf->nvm_phy_type_lo); phy_types_high = le64_to_cpu(pf->nvm_phy_type_hi); - ice_mask_min_supported_speeds(phy_types_high, &phy_types_low); + ice_mask_min_supported_speeds(&pf->hw, phy_types_high, + &phy_types_low); /* determine advertised modes based on link override only * if it's supported and if the FW doesn't abstract the * driver from having to account for link overrides @@ -2826,6 +2831,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, tx_rings[i].count = new_tx_cnt; tx_rings[i].desc = NULL; tx_rings[i].tx_buf = NULL; + tx_rings[i].tx_tstamps = &pf->ptp.port.tx; err = ice_setup_tx_ring(&tx_rings[i]); if (err) { while (i--) @@ -2884,6 +2890,7 @@ process_rx: /* clone ring and setup updated count */ rx_rings[i] = *vsi->rx_rings[i]; rx_rings[i].count = new_rx_cnt; + rx_rings[i].cached_phctime = pf->ptp.cached_phc_time; rx_rings[i].desc = NULL; rx_rings[i].rx_buf = NULL; /* this is to allow wr32 to have something to write to diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index c9f7393b783dc37be8cc0b319973052ad9f82828..ee5b36941ba35868c260f36ae430061bea55223f 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -61,13 +61,13 @@ static void ice_lag_set_backup(struct ice_lag *lag) */ static void ice_display_lag_info(struct ice_lag *lag) { - const char *name, *peer, *upper, *role, *bonded, *master; + const char *name, *peer, *upper, *role, *bonded, *primary; struct device *dev = &lag->pf->pdev->dev; name = lag->netdev ? netdev_name(lag->netdev) : "unset"; peer = lag->peer_netdev ? netdev_name(lag->peer_netdev) : "unset"; upper = lag->upper_netdev ? netdev_name(lag->upper_netdev) : "unset"; - master = lag->master ? "TRUE" : "FALSE"; + primary = lag->primary ? "TRUE" : "FALSE"; bonded = lag->bonded ? "BONDED" : "UNBONDED"; switch (lag->role) { @@ -87,8 +87,8 @@ static void ice_display_lag_info(struct ice_lag *lag) role = "ERROR"; } - dev_dbg(dev, "%s %s, peer:%s, upper:%s, role:%s, master:%s\n", name, - bonded, peer, upper, role, master); + dev_dbg(dev, "%s %s, peer:%s, upper:%s, role:%s, primary:%s\n", name, + bonded, peer, upper, role, primary); } /** @@ -119,7 +119,7 @@ static void ice_lag_info_event(struct ice_lag *lag, void *ptr) } if (strcmp(bonding_info->slave.slave_name, lag_netdev_name)) { - netdev_dbg(lag->netdev, "Bonding event recv, but slave info not for us\n"); + netdev_dbg(lag->netdev, "Bonding event recv, but secondary info not for us\n"); goto lag_out; } @@ -164,8 +164,8 @@ ice_lag_link(struct ice_lag *lag, struct netdev_notifier_changeupper_info *info) lag->bonded = true; lag->role = ICE_LAG_UNSET; - /* if this is the first element in an LAG mark as master */ - lag->master = !!(peers == 1); + /* if this is the first element in an LAG mark as primary */ + lag->primary = !!(peers == 1); } /** @@ -264,7 +264,7 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) netdev_dbg(netdev, "bonding %s\n", info->linking ? "LINK" : "UNLINK"); if (!netif_is_lag_master(info->upper_dev)) { - netdev_dbg(netdev, "changeupper rcvd, but not master. bail\n"); + netdev_dbg(netdev, "changeupper rcvd, but not primary. bail\n"); return; } diff --git a/drivers/net/ethernet/intel/ice/ice_lag.h b/drivers/net/ethernet/intel/ice/ice_lag.h index c2e3688dd8fd5898b029c62215e3f13d4698de8f..51b5cf467ce24dba543b6e8ac171f072573df55c 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.h +++ b/drivers/net/ethernet/intel/ice/ice_lag.h @@ -24,7 +24,7 @@ struct ice_lag { struct net_device *upper_netdev; /* upper bonding netdev */ struct notifier_block notif_block; u8 bonded:1; /* currently bonded */ - u8 master:1; /* this is a master */ + u8 primary:1; /* this is primary */ u8 handler:1; /* did we register a rx_netdev_handler */ /* each thing blocking bonding will increment this value by one. * If this value is zero, then bonding is allowed. diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 733c455f65746287b9d7a70edfd7a5f51c357269..938ba8c215cb07f85cb5bc840e3950128ab91983 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -914,7 +914,7 @@ static void ice_set_dflt_vsi_ctx(struct ice_hw *hw, struct ice_vsi_ctx *ctxt) */ static int ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) { - u16 offset = 0, qmap = 0, tx_count = 0, pow = 0; + u16 offset = 0, qmap = 0, tx_count = 0, rx_count = 0, pow = 0; u16 num_txq_per_tc, num_rxq_per_tc; u16 qcount_tx = vsi->alloc_txq; u16 qcount_rx = vsi->alloc_rxq; @@ -981,23 +981,25 @@ static int ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt) * at least 1) */ if (offset) - vsi->num_rxq = offset; + rx_count = offset; else - vsi->num_rxq = num_rxq_per_tc; + rx_count = num_rxq_per_tc; - if (vsi->num_rxq > vsi->alloc_rxq) { + if (rx_count > vsi->alloc_rxq) { dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Rx queues (%u), than were allocated (%u)!\n", - vsi->num_rxq, vsi->alloc_rxq); + rx_count, vsi->alloc_rxq); return -EINVAL; } - vsi->num_txq = tx_count; - if (vsi->num_txq > vsi->alloc_txq) { + if (tx_count > vsi->alloc_txq) { dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Tx queues (%u), than were allocated (%u)!\n", - vsi->num_txq, vsi->alloc_txq); + tx_count, vsi->alloc_txq); return -EINVAL; } + vsi->num_txq = tx_count; + vsi->num_rxq = rx_count; + if (vsi->type == ICE_VSI_VF && vsi->num_txq != vsi->num_rxq) { dev_dbg(ice_pf_to_dev(vsi->back), "VF VSI should have same number of Tx and Rx queues. Hence making them equal\n"); /* since there is a chance that num_rxq could have been changed @@ -1522,6 +1524,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->netdev = vsi->netdev; ring->dev = dev; ring->count = vsi->num_rx_desc; + ring->cached_phctime = pf->ptp.cached_phc_time; WRITE_ONCE(vsi->rx_rings[i], ring); } @@ -1561,6 +1564,22 @@ void ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena) kfree(lut); } +/** + * ice_vsi_cfg_crc_strip - Configure CRC stripping for a VSI + * @vsi: VSI to be configured + * @disable: set to true to have FCS / CRC in the frame data + */ +void ice_vsi_cfg_crc_strip(struct ice_vsi *vsi, bool disable) +{ + int i; + + ice_for_each_rxq(vsi, i) + if (disable) + vsi->rx_rings[i]->flags |= ICE_RX_FLAGS_CRC_STRIP_DIS; + else + vsi->rx_rings[i]->flags &= ~ICE_RX_FLAGS_CRC_STRIP_DIS; +} + /** * ice_vsi_cfg_rss_lut_key - Configure RSS params for a VSI * @vsi: VSI to be configured @@ -1986,8 +2005,8 @@ int ice_vsi_cfg_xdp_txqs(struct ice_vsi *vsi) if (ret) return ret; - ice_for_each_xdp_txq(vsi, i) - vsi->xdp_rings[i]->xsk_pool = ice_tx_xsk_pool(vsi->xdp_rings[i]); + ice_for_each_rxq(vsi, i) + ice_tx_xsk_pool(vsi, i); return ret; } @@ -2969,9 +2988,6 @@ int ice_vsi_release(struct ice_vsi *vsi) clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); } - if (vsi->type == ICE_VSI_PF) - ice_devlink_destroy_pf_port(pf); - if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_rss_clean(vsi); @@ -3029,6 +3045,9 @@ int ice_vsi_release(struct ice_vsi *vsi) } } + if (vsi->type == ICE_VSI_PF) + ice_devlink_destroy_pf_port(pf); + if (vsi->type == ICE_VSI_VF && vsi->agg_node && vsi->agg_node->valid) vsi->agg_node->num_vsis--; @@ -3276,6 +3295,12 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi) */ if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_vsi_cfg_rss_lut_key(vsi); + + /* disable or enable CRC stripping */ + if (vsi->netdev) + ice_vsi_cfg_crc_strip(vsi, !!(vsi->netdev->features & + NETIF_F_RXFCS)); + break; case ICE_VSI_VF: ret = ice_vsi_alloc_q_vectors(vsi); @@ -3490,6 +3515,7 @@ ice_vsi_setup_q_map_mqprio(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt, u16 pow, offset = 0, qcount_tx = 0, qcount_rx = 0, qmap; u16 tc0_offset = vsi->mqprio_qopt.qopt.offset[0]; int tc0_qcount = vsi->mqprio_qopt.qopt.count[0]; + u16 new_txq, new_rxq; u8 netdev_tc = 0; int i; @@ -3530,21 +3556,24 @@ ice_vsi_setup_q_map_mqprio(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt, } } - /* Set actual Tx/Rx queue pairs */ - vsi->num_txq = offset + qcount_tx; - if (vsi->num_txq > vsi->alloc_txq) { + new_txq = offset + qcount_tx; + if (new_txq > vsi->alloc_txq) { dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Tx queues (%u), than were allocated (%u)!\n", - vsi->num_txq, vsi->alloc_txq); + new_txq, vsi->alloc_txq); return -EINVAL; } - vsi->num_rxq = offset + qcount_rx; - if (vsi->num_rxq > vsi->alloc_rxq) { + new_rxq = offset + qcount_rx; + if (new_rxq > vsi->alloc_rxq) { dev_err(ice_pf_to_dev(vsi->back), "Trying to use more Rx queues (%u), than were allocated (%u)!\n", - vsi->num_rxq, vsi->alloc_rxq); + new_rxq, vsi->alloc_rxq); return -EINVAL; } + /* Set actual Tx/Rx queue pairs */ + vsi->num_txq = new_txq; + vsi->num_rxq = new_rxq; + /* Setup queue TC[0].qmap for given VSI context */ ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]); @@ -3576,6 +3605,7 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) { u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; struct ice_pf *pf = vsi->back; + struct ice_tc_cfg old_tc_cfg; struct ice_vsi_ctx *ctx; struct device *dev; int i, ret = 0; @@ -3600,6 +3630,7 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) max_txqs[i] = vsi->num_txq; } + memcpy(&old_tc_cfg, &vsi->tc_cfg, sizeof(old_tc_cfg)); vsi->tc_cfg.ena_tc = ena_tc; vsi->tc_cfg.numtc = num_tc; @@ -3616,8 +3647,10 @@ int ice_vsi_cfg_tc(struct ice_vsi *vsi, u8 ena_tc) else ret = ice_vsi_setup_q_map(vsi, ctx); - if (ret) + if (ret) { + memcpy(&vsi->tc_cfg, &old_tc_cfg, sizeof(vsi->tc_cfg)); goto out; + } /* must to indicate which section of VSI context are being modified */ ctx->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index 8712b1d2ceec9e96544199d87038c38c212eaa20..ec4bf0c89857c6c1ea7c5a81c135d53b2f065b6b 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -89,6 +89,8 @@ void ice_vsi_free_tx_rings(struct ice_vsi *vsi); void ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena); +void ice_vsi_cfg_crc_strip(struct ice_vsi *vsi, bool disable); + void ice_update_tx_ring_stats(struct ice_tx_ring *ring, u64 pkts, u64 bytes); void ice_update_rx_ring_stats(struct ice_rx_ring *ring, u64 pkts, u64 bytes); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 4ecaf40cf946b317779ff856edc7b003765716fe..0f6718719453dd4d00e31db51b4762cbf991da39 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -2399,8 +2399,6 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) return -EBUSY; } - ice_unplug_aux_dev(pf); - switch (reset) { case ICE_RESET_PFR: set_bit(ICE_PFR_REQ, pf->state); @@ -2581,7 +2579,6 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) if (ice_setup_tx_ring(xdp_ring)) goto free_xdp_rings; ice_set_ring_xdp(xdp_ring); - xdp_ring->xsk_pool = ice_tx_xsk_pool(xdp_ring); spin_lock_init(&xdp_ring->tx_lock); for (j = 0; j < xdp_ring->count; j++) { tx_desc = ICE_TX_DESC(xdp_ring, j); @@ -2589,13 +2586,6 @@ static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) } } - ice_for_each_rxq(vsi, i) { - if (static_key_enabled(&ice_xdp_locking_key)) - vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq]; - else - vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i]; - } - return 0; free_xdp_rings: @@ -2685,6 +2675,23 @@ int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) xdp_rings_rem -= xdp_rings_per_v; } + ice_for_each_rxq(vsi, i) { + if (static_key_enabled(&ice_xdp_locking_key)) { + vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq]; + } else { + struct ice_q_vector *q_vector = vsi->rx_rings[i]->q_vector; + struct ice_tx_ring *ring; + + ice_for_each_tx_ring(ring, q_vector->tx) { + if (ice_ring_is_xdp(ring)) { + vsi->rx_rings[i]->xdp_ring = ring; + break; + } + } + } + ice_tx_xsk_pool(vsi, i); + } + /* omit the scheduler update if in reset path; XDP queues will be * taken into account at the end of ice_vsi_rebuild, where * ice_cfg_vsi_lan is being called @@ -2889,10 +2896,18 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, if (xdp_ring_err) NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); } + /* reallocate Rx queues that are used for zero-copy */ + xdp_ring_err = ice_realloc_zc_buf(vsi, true); + if (xdp_ring_err) + NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed"); } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { xdp_ring_err = ice_destroy_xdp_rings(vsi); if (xdp_ring_err) NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); + /* reallocate Rx queues that were used for zero-copy */ + xdp_ring_err = ice_realloc_zc_buf(vsi, false); + if (xdp_ring_err) + NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed"); } else { /* safe to call even when prog == vsi->xdp_prog as * dev_xdp_install in net/core/dev.c incremented prog's @@ -3078,7 +3093,8 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) if (oicr & PFINT_OICR_TSYN_TX_M) { ena_mask &= ~PFINT_OICR_TSYN_TX_M; - ice_ptp_process_ts(pf); + if (!hw->reset_ongoing) + ret = IRQ_WAKE_THREAD; } if (oicr & PFINT_OICR_TSYN_EVNT_M) { @@ -3113,7 +3129,8 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) ice_service_task_schedule(pf); } } - ret = IRQ_HANDLED; + if (!ret) + ret = IRQ_HANDLED; ice_service_task_schedule(pf); ice_irq_dynamic_ena(hw, NULL, NULL); @@ -3121,6 +3138,24 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) return ret; } +/** + * ice_misc_intr_thread_fn - misc interrupt thread function + * @irq: interrupt number + * @data: pointer to a q_vector + */ +static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data) +{ + irqreturn_t ret = IRQ_HANDLED; + struct ice_pf *pf = data; + bool irq_handled; + + irq_handled = ice_ptp_process_ts(pf); + if (!irq_handled) + ret = IRQ_WAKE_THREAD; + + return ret; +} + /** * ice_dis_ctrlq_interrupts - disable control queue interrupts * @hw: pointer to HW structure @@ -3233,10 +3268,12 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf) pf->num_avail_sw_msix -= 1; pf->oicr_idx = (u16)oicr_idx; - err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, - ice_misc_intr, 0, pf->int_name, pf); + err = devm_request_threaded_irq(dev, + pf->msix_entries[pf->oicr_idx].vector, + ice_misc_intr, ice_misc_intr_thread_fn, + 0, pf->int_name, pf); if (err) { - dev_err(dev, "devm_request_irq for %s failed: %d\n", + dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n", pf->int_name, err); ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); pf->num_avail_sw_msix += 1; @@ -3273,7 +3310,7 @@ static void ice_napi_add(struct ice_vsi *vsi) ice_for_each_q_vector(vsi, v_idx) netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, - ice_napi_poll, NAPI_POLL_WEIGHT); + ice_napi_poll); } /** @@ -3376,6 +3413,11 @@ static void ice_set_netdev_features(struct net_device *netdev) if (is_dvm_ena) netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX; + + /* Leave CRC / FCS stripping enabled by default, but allow the value to + * be changed at runtime + */ + netdev->hw_features |= NETIF_F_RXFCS; } /** @@ -3896,7 +3938,7 @@ static int ice_init_pf(struct ice_pf *pf) pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); if (!pf->avail_rxqs) { - devm_kfree(ice_pf_to_dev(pf), pf->avail_txqs); + bitmap_free(pf->avail_txqs); pf->avail_txqs = NULL; return -ENOMEM; } @@ -3907,88 +3949,135 @@ static int ice_init_pf(struct ice_pf *pf) return 0; } +/** + * ice_reduce_msix_usage - Reduce usage of MSI-X vectors + * @pf: board private structure + * @v_remain: number of remaining MSI-X vectors to be distributed + * + * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled. + * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of + * remaining vectors. + */ +static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain) +{ + int v_rdma; + + if (!ice_is_rdma_ena(pf)) { + pf->num_lan_msix = v_remain; + return; + } + + /* RDMA needs at least 1 interrupt in addition to AEQ MSIX */ + v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1; + + if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) { + dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n"); + clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); + + pf->num_rdma_msix = 0; + pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; + } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) || + (v_remain - v_rdma < v_rdma)) { + /* Support minimum RDMA and give remaining vectors to LAN MSIX */ + pf->num_rdma_msix = ICE_MIN_RDMA_MSIX; + pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX; + } else { + /* Split remaining MSIX with RDMA after accounting for AEQ MSIX + */ + pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 + + ICE_RDMA_NUM_AEQ_MSIX; + pf->num_lan_msix = v_remain - pf->num_rdma_msix; + } +} + /** * ice_ena_msix_range - Request a range of MSIX vectors from the OS * @pf: board private structure * - * compute the number of MSIX vectors required (v_budget) and request from - * the OS. Return the number of vectors reserved or negative on failure + * Compute the number of MSIX vectors wanted and request from the OS. Adjust + * device usage if there are not enough vectors. Return the number of vectors + * reserved or negative on failure. */ static int ice_ena_msix_range(struct ice_pf *pf) { - int num_cpus, v_left, v_actual, v_other, v_budget = 0; + int num_cpus, hw_num_msix, v_other, v_wanted, v_actual; struct device *dev = ice_pf_to_dev(pf); - int needed, err, i; + int err, i; - v_left = pf->hw.func_caps.common_cap.num_msix_vectors; + hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors; num_cpus = num_online_cpus(); - /* reserve for LAN miscellaneous handler */ - needed = ICE_MIN_LAN_OICR_MSIX; - if (v_left < needed) - goto no_hw_vecs_left_err; - v_budget += needed; - v_left -= needed; + /* LAN miscellaneous handler */ + v_other = ICE_MIN_LAN_OICR_MSIX; - /* reserve for flow director */ - if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { - needed = ICE_FDIR_MSIX; - if (v_left < needed) - goto no_hw_vecs_left_err; - v_budget += needed; - v_left -= needed; - } - - /* reserve for switchdev */ - needed = ICE_ESWITCH_MSIX; - if (v_left < needed) - goto no_hw_vecs_left_err; - v_budget += needed; - v_left -= needed; - - /* total used for non-traffic vectors */ - v_other = v_budget; - - /* reserve vectors for LAN traffic */ - needed = num_cpus; - if (v_left < needed) - goto no_hw_vecs_left_err; - pf->num_lan_msix = needed; - v_budget += needed; - v_left -= needed; - - /* reserve vectors for RDMA auxiliary driver */ + /* Flow Director */ + if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) + v_other += ICE_FDIR_MSIX; + + /* switchdev */ + v_other += ICE_ESWITCH_MSIX; + + v_wanted = v_other; + + /* LAN traffic */ + pf->num_lan_msix = num_cpus; + v_wanted += pf->num_lan_msix; + + /* RDMA auxiliary driver */ if (ice_is_rdma_ena(pf)) { - needed = num_cpus + ICE_RDMA_NUM_AEQ_MSIX; - if (v_left < needed) - goto no_hw_vecs_left_err; - pf->num_rdma_msix = needed; - v_budget += needed; - v_left -= needed; + pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX; + v_wanted += pf->num_rdma_msix; + } + + if (v_wanted > hw_num_msix) { + int v_remain; + + dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n", + v_wanted, hw_num_msix); + + if (hw_num_msix < ICE_MIN_MSIX) { + err = -ERANGE; + goto exit_err; + } + + v_remain = hw_num_msix - v_other; + if (v_remain < ICE_MIN_LAN_TXRX_MSIX) { + v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX; + v_remain = ICE_MIN_LAN_TXRX_MSIX; + } + + ice_reduce_msix_usage(pf, v_remain); + v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other; + + dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n", + pf->num_lan_msix); + if (ice_is_rdma_ena(pf)) + dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n", + pf->num_rdma_msix); } - pf->msix_entries = devm_kcalloc(dev, v_budget, + pf->msix_entries = devm_kcalloc(dev, v_wanted, sizeof(*pf->msix_entries), GFP_KERNEL); if (!pf->msix_entries) { err = -ENOMEM; goto exit_err; } - for (i = 0; i < v_budget; i++) + for (i = 0; i < v_wanted; i++) pf->msix_entries[i].entry = i; /* actually reserve the vectors */ v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, - ICE_MIN_MSIX, v_budget); + ICE_MIN_MSIX, v_wanted); if (v_actual < 0) { dev_err(dev, "unable to reserve MSI-X vectors\n"); err = v_actual; goto msix_err; } - if (v_actual < v_budget) { + if (v_actual < v_wanted) { dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", - v_budget, v_actual); + v_wanted, v_actual); if (v_actual < ICE_MIN_MSIX) { /* error if we can't get minimum vectors */ @@ -3997,38 +4086,11 @@ static int ice_ena_msix_range(struct ice_pf *pf) goto msix_err; } else { int v_remain = v_actual - v_other; - int v_rdma = 0, v_min_rdma = 0; - if (ice_is_rdma_ena(pf)) { - /* Need at least 1 interrupt in addition to - * AEQ MSIX - */ - v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1; - v_min_rdma = ICE_MIN_RDMA_MSIX; - } + if (v_remain < ICE_MIN_LAN_TXRX_MSIX) + v_remain = ICE_MIN_LAN_TXRX_MSIX; - if (v_actual == ICE_MIN_MSIX || - v_remain < ICE_MIN_LAN_TXRX_MSIX + v_min_rdma) { - dev_warn(dev, "Not enough MSI-X vectors to support RDMA.\n"); - clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); - - pf->num_rdma_msix = 0; - pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; - } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) || - (v_remain - v_rdma < v_rdma)) { - /* Support minimum RDMA and give remaining - * vectors to LAN MSIX - */ - pf->num_rdma_msix = v_min_rdma; - pf->num_lan_msix = v_remain - v_min_rdma; - } else { - /* Split remaining MSIX with RDMA after - * accounting for AEQ MSIX - */ - pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 + - ICE_RDMA_NUM_AEQ_MSIX; - pf->num_lan_msix = v_remain - pf->num_rdma_msix; - } + ice_reduce_msix_usage(pf, v_remain); dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n", pf->num_lan_msix); @@ -4043,12 +4105,7 @@ static int ice_ena_msix_range(struct ice_pf *pf) msix_err: devm_kfree(dev, pf->msix_entries); - goto exit_err; -no_hw_vecs_left_err: - dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n", - needed, v_left); - err = -ERANGE; exit_err: pf->num_rdma_msix = 0; pf->num_lan_msix = 0; @@ -4542,6 +4599,10 @@ static int ice_register_netdev(struct ice_pf *pf) if (!vsi || !vsi->netdev) return -EIO; + err = ice_devlink_create_pf_port(pf); + if (err) + goto err_devlink_create; + err = register_netdev(vsi->netdev); if (err) goto err_register_netdev; @@ -4549,17 +4610,13 @@ static int ice_register_netdev(struct ice_pf *pf) set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); netif_carrier_off(vsi->netdev); netif_tx_stop_all_queues(vsi->netdev); - err = ice_devlink_create_pf_port(pf); - if (err) - goto err_devlink_create; devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev); return 0; -err_devlink_create: - unregister_netdev(vsi->netdev); - clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); err_register_netdev: + ice_devlink_destroy_pf_port(pf); +err_devlink_create: free_netdev(vsi->netdev); vsi->netdev = NULL; clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); @@ -4667,8 +4724,6 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) ice_set_safe_mode_caps(hw); } - hw->ucast_shared = true; - err = ice_init_pf(pf); if (err) { dev_err(dev, "ice_init_pf failed: %d\n", err); @@ -5727,6 +5782,9 @@ ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], NETIF_F_HW_VLAN_STAG_RX | \ NETIF_F_HW_VLAN_STAG_TX) +#define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ + NETIF_F_HW_VLAN_STAG_RX) + #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ NETIF_F_HW_VLAN_STAG_FILTER) @@ -5813,6 +5871,14 @@ ice_fix_features(struct net_device *netdev, netdev_features_t features) NETIF_F_HW_VLAN_STAG_TX); } + if (!(netdev->features & NETIF_F_RXFCS) && + (features & NETIF_F_RXFCS) && + (features & NETIF_VLAN_STRIPPING_FEATURES) && + !ice_vsi_has_non_zero_vlans(np->vsi)) { + netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n"); + features &= ~NETIF_VLAN_STRIPPING_FEATURES; + } + return features; } @@ -5906,6 +5972,13 @@ ice_set_vlan_features(struct net_device *netdev, netdev_features_t features) current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES; requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES; if (current_vlan_features ^ requested_vlan_features) { + if ((features & NETIF_F_RXFCS) && + (features & NETIF_VLAN_STRIPPING_FEATURES)) { + dev_err(ice_pf_to_dev(vsi->back), + "To enable VLAN stripping, you must first enable FCS/CRC stripping\n"); + return -EIO; + } + err = ice_set_vlan_offload_features(vsi, features); if (err) return err; @@ -5987,6 +6060,23 @@ ice_set_features(struct net_device *netdev, netdev_features_t features) if (ret) return ret; + /* Turn on receive of FCS aka CRC, and after setting this + * flag the packet data will have the 4 byte CRC appended + */ + if (changed & NETIF_F_RXFCS) { + if ((features & NETIF_F_RXFCS) && + (features & NETIF_VLAN_STRIPPING_FEATURES)) { + dev_err(ice_pf_to_dev(vsi->back), + "To disable FCS/CRC stripping, you must first disable VLAN stripping\n"); + return -EIO; + } + + ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS)); + ret = ice_down_up(vsi); + if (ret) + return ret; + } + if (changed & NETIF_F_NTUPLE) { bool ena = !!(features & NETIF_F_NTUPLE); @@ -6634,7 +6724,7 @@ static void ice_napi_disable_all(struct ice_vsi *vsi) */ int ice_down(struct ice_vsi *vsi) { - int i, tx_err, rx_err, link_err = 0, vlan_err = 0; + int i, tx_err, rx_err, vlan_err = 0; WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state)); @@ -6668,20 +6758,13 @@ int ice_down(struct ice_vsi *vsi) ice_napi_disable_all(vsi); - if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { - link_err = ice_force_phys_link_state(vsi, false); - if (link_err) - netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", - vsi->vsi_num, link_err); - } - ice_for_each_txq(vsi, i) ice_clean_tx_ring(vsi->tx_rings[i]); ice_for_each_rxq(vsi, i) ice_clean_rx_ring(vsi->rx_rings[i]); - if (tx_err || rx_err || link_err || vlan_err) { + if (tx_err || rx_err || vlan_err) { netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", vsi->vsi_num, vsi->vsw->sw_id); return -EIO; @@ -6690,6 +6773,31 @@ int ice_down(struct ice_vsi *vsi) return 0; } +/** + * ice_down_up - shutdown the VSI connection and bring it up + * @vsi: the VSI to be reconnected + */ +int ice_down_up(struct ice_vsi *vsi) +{ + int ret; + + /* if DOWN already set, nothing to do */ + if (test_and_set_bit(ICE_VSI_DOWN, vsi->state)) + return 0; + + ret = ice_down(vsi); + if (ret) + return ret; + + ret = ice_up(vsi); + if (ret) { + netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n"); + return ret; + } + + return 0; +} + /** * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources * @vsi: VSI having resources allocated @@ -6843,6 +6951,8 @@ int ice_vsi_open(struct ice_vsi *vsi) if (err) goto err_setup_rx; + ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); + if (vsi->type == ICE_VSI_PF) { /* Notify the stack of the actual queue counts. */ err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); @@ -8875,6 +8985,16 @@ int ice_stop(struct net_device *netdev) return -EBUSY; } + if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { + int link_err = ice_force_phys_link_state(vsi, false); + + if (link_err) { + netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", + vsi->vsi_num, link_err); + return -EIO; + } + } + ice_vsi_close(vsi); return 0; diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c index 13cdb5ea594d2f2e22650f397441ed7916e47952..c262dc886e6a6a780b50adfa224bfbb6d2bd3e65 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.c +++ b/drivers/net/ethernet/intel/ice/ice_nvm.c @@ -1114,14 +1114,18 @@ int ice_nvm_validate_checksum(struct ice_hw *hw) * Update the control word with the required banks' validity bits * and dumps the Shadow RAM to flash (0x0707) * - * cmd_flags controls which banks to activate, and the preservation level to - * use when activating the NVM bank. + * cmd_flags controls which banks to activate, the preservation level to use + * when activating the NVM bank, and whether an EMP reset is required for + * activation. + * + * Note that the 16bit cmd_flags value is split between two separate 1 byte + * flag values in the descriptor. * * On successful return of the firmware command, the response_flags variable * is updated with the flags reported by firmware indicating certain status, * such as whether EMP reset is enabled. */ -int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags) +int ice_nvm_write_activate(struct ice_hw *hw, u16 cmd_flags, u8 *response_flags) { struct ice_aqc_nvm *cmd; struct ice_aq_desc desc; @@ -1130,7 +1134,8 @@ int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags) cmd = &desc.params.nvm; ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_nvm_write_activate); - cmd->cmd_flags = cmd_flags; + cmd->cmd_flags = (u8)(cmd_flags & 0xFF); + cmd->offset_high = (u8)((cmd_flags >> 8) & 0xFF); err = ice_aq_send_cmd(hw, &desc, NULL, 0, NULL); if (!err && response_flags) diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h index 856d1ad4398bf70c15ff6c4400d66a9a1b88eb62..774c2317967d733865297e48b75e3ad68879ba72 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.h +++ b/drivers/net/ethernet/intel/ice/ice_nvm.h @@ -34,7 +34,7 @@ ice_aq_update_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, int ice_aq_erase_nvm(struct ice_hw *hw, u16 module_typeid, struct ice_sq_cd *cd); int ice_nvm_validate_checksum(struct ice_hw *hw); -int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags); +int ice_nvm_write_activate(struct ice_hw *hw, u16 cmd_flags, u8 *response_flags); int ice_aq_nvm_update_empr(struct ice_hw *hw); int ice_nvm_set_pkg_data(struct ice_hw *hw, bool del_pkg_data_flag, u8 *data, diff --git a/drivers/net/ethernet/intel/ice/ice_protocol_type.h b/drivers/net/ethernet/intel/ice/ice_protocol_type.h index 560efc7654c779a70952792b01063d925cb0e79e..02a4e1cf624e9ab80f9b0119cf160eede1b1232c 100644 --- a/drivers/net/ethernet/intel/ice/ice_protocol_type.h +++ b/drivers/net/ethernet/intel/ice/ice_protocol_type.h @@ -44,6 +44,7 @@ enum ice_protocol_type { ICE_GTP, ICE_GTP_NO_PAY, ICE_PPPOE, + ICE_L2TPV3, ICE_VLAN_EX, ICE_VLAN_IN, ICE_VXLAN_GPE, @@ -111,6 +112,7 @@ enum ice_prot_id { #define ICE_UDP_ILOS_HW 53 #define ICE_GRE_OF_HW 64 #define ICE_PPPOE_HW 103 +#define ICE_L2TPV3_HW 104 #define ICE_UDP_OF_HW 52 /* UDP Tunnels */ #define ICE_META_DATA_ID_HW 255 /* this is used for tunnel and VLAN type */ @@ -217,6 +219,11 @@ struct ice_pppoe_hdr { __be16 ppp_prot_id; /* control and data only */ }; +struct ice_l2tpv3_sess_hdr { + __be32 session_id; + __be64 cookie; +}; + struct ice_nvgre_hdr { __be16 flags; __be16 protocol; @@ -235,6 +242,7 @@ union ice_prot_hdr { struct ice_nvgre_hdr nvgre_hdr; struct ice_udp_gtp_hdr gtp_hdr; struct ice_pppoe_hdr pppoe_hdr; + struct ice_l2tpv3_sess_hdr l2tpv3_sess_hdr; }; /* This is mapping table entry that maps every word within a given protocol diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 72b663108a4a66d20ada9e1c18ae6ed55b55661d..011b727ab1903f5b183aaa9fd0daeb5eaa86a4f1 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -490,56 +490,6 @@ ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts) return ((u64)hi << 32) | lo; } -/** - * ice_ptp_update_cached_phctime - Update the cached PHC time values - * @pf: Board specific private structure - * - * This function updates the system time values which are cached in the PF - * structure and the Rx rings. - * - * This function must be called periodically to ensure that the cached value - * is never more than 2 seconds old. It must also be called whenever the PHC - * time has been changed. - * - * Return: - * * 0 - OK, successfully updated - * * -EAGAIN - PF was busy, need to reschedule the update - */ -static int ice_ptp_update_cached_phctime(struct ice_pf *pf) -{ - u64 systime; - int i; - - if (test_and_set_bit(ICE_CFG_BUSY, pf->state)) - return -EAGAIN; - - /* Read the current PHC time */ - systime = ice_ptp_read_src_clk_reg(pf, NULL); - - /* Update the cached PHC time stored in the PF structure */ - WRITE_ONCE(pf->ptp.cached_phc_time, systime); - - ice_for_each_vsi(pf, i) { - struct ice_vsi *vsi = pf->vsi[i]; - int j; - - if (!vsi) - continue; - - if (vsi->type != ICE_VSI_PF) - continue; - - ice_for_each_rxq(vsi, j) { - if (!vsi->rx_rings[j]) - continue; - WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime); - } - } - clear_bit(ICE_CFG_BUSY, pf->state); - - return 0; -} - /** * ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b * @cached_phc_time: recently cached copy of PHC time @@ -636,11 +586,399 @@ static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp) static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) { const u64 mask = GENMASK_ULL(31, 0); + unsigned long discard_time; + + /* Discard the hardware timestamp if the cached PHC time is too old */ + discard_time = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000); + if (time_is_before_jiffies(discard_time)) { + pf->ptp.tx_hwtstamp_discarded++; + return 0; + } return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time, (in_tstamp >> 8) & mask); } +/** + * ice_ptp_tx_tstamp - Process Tx timestamps for a port + * @tx: the PTP Tx timestamp tracker + * + * Process timestamps captured by the PHY associated with this port. To do + * this, loop over each index with a waiting skb. + * + * If a given index has a valid timestamp, perform the following steps: + * + * 1) copy the timestamp out of the PHY register + * 4) clear the timestamp valid bit in the PHY register + * 5) unlock the index by clearing the associated in_use bit. + * 2) extend the 40b timestamp value to get a 64bit timestamp + * 3) send that timestamp to the stack + * + * After looping, if we still have waiting SKBs, return true. This may cause us + * effectively poll even when not strictly necessary. We do this because it's + * possible a new timestamp was requested around the same time as the interrupt. + * In some cases hardware might not interrupt us again when the timestamp is + * captured. + * + * Note that we only take the tracking lock when clearing the bit and when + * checking if we need to re-queue this task. The only place where bits can be + * set is the hard xmit routine where an SKB has a request flag set. The only + * places where we clear bits are this work function, or the periodic cleanup + * thread. If the cleanup thread clears a bit we're processing we catch it + * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread + * starts a new timestamp, we might not begin processing it right away but we + * will notice it at the end when we re-queue the task. If a Tx thread starts + * a new timestamp just after this function exits without re-queuing, + * the interrupt when the timestamp finishes should trigger. Avoiding holding + * the lock for the entire function is important in order to ensure that Tx + * threads do not get blocked while waiting for the lock. + */ +static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx) +{ + struct ice_ptp_port *ptp_port; + bool ts_handled = true; + struct ice_pf *pf; + u8 idx; + + if (!tx->init) + return false; + + ptp_port = container_of(tx, struct ice_ptp_port, tx); + pf = ptp_port_to_pf(ptp_port); + + for_each_set_bit(idx, tx->in_use, tx->len) { + struct skb_shared_hwtstamps shhwtstamps = {}; + u8 phy_idx = idx + tx->quad_offset; + u64 raw_tstamp, tstamp; + struct sk_buff *skb; + int err; + + ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx); + + err = ice_read_phy_tstamp(&pf->hw, tx->quad, phy_idx, + &raw_tstamp); + if (err) + continue; + + ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx); + + /* Check if the timestamp is invalid or stale */ + if (!(raw_tstamp & ICE_PTP_TS_VALID) || + raw_tstamp == tx->tstamps[idx].cached_tstamp) + continue; + + /* The timestamp is valid, so we'll go ahead and clear this + * index and then send the timestamp up to the stack. + */ + spin_lock(&tx->lock); + tx->tstamps[idx].cached_tstamp = raw_tstamp; + clear_bit(idx, tx->in_use); + skb = tx->tstamps[idx].skb; + tx->tstamps[idx].skb = NULL; + spin_unlock(&tx->lock); + + /* it's (unlikely but) possible we raced with the cleanup + * thread for discarding old timestamp requests. + */ + if (!skb) + continue; + + /* Extend the timestamp using cached PHC time */ + tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp); + if (tstamp) { + shhwtstamps.hwtstamp = ns_to_ktime(tstamp); + ice_trace(tx_tstamp_complete, skb, idx); + } + + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); + } + + /* Check if we still have work to do. If so, re-queue this task to + * poll for remaining timestamps. + */ + spin_lock(&tx->lock); + if (!bitmap_empty(tx->in_use, tx->len)) + ts_handled = false; + spin_unlock(&tx->lock); + + return ts_handled; +} + +/** + * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps + * @tx: Tx tracking structure to initialize + * + * Assumes that the length has already been initialized. Do not call directly, + * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead. + */ +static int +ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx) +{ + tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL); + if (!tx->tstamps) + return -ENOMEM; + + tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL); + if (!tx->in_use) { + kfree(tx->tstamps); + tx->tstamps = NULL; + return -ENOMEM; + } + + spin_lock_init(&tx->lock); + + tx->init = 1; + + return 0; +} + +/** + * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker + * @pf: Board private structure + * @tx: the tracker to flush + */ +static void +ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + u8 idx; + + for (idx = 0; idx < tx->len; idx++) { + u8 phy_idx = idx + tx->quad_offset; + + spin_lock(&tx->lock); + if (tx->tstamps[idx].skb) { + dev_kfree_skb_any(tx->tstamps[idx].skb); + tx->tstamps[idx].skb = NULL; + pf->ptp.tx_hwtstamp_flushed++; + } + clear_bit(idx, tx->in_use); + spin_unlock(&tx->lock); + + /* Clear any potential residual timestamp in the PHY block */ + if (!pf->hw.reset_ongoing) + ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx); + } +} + +/** + * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker + * @pf: Board private structure + * @tx: Tx tracking structure to release + * + * Free memory associated with the Tx timestamp tracker. + */ +static void +ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + tx->init = 0; + + ice_ptp_flush_tx_tracker(pf, tx); + + kfree(tx->tstamps); + tx->tstamps = NULL; + + bitmap_free(tx->in_use); + tx->in_use = NULL; + + tx->len = 0; +} + +/** + * ice_ptp_init_tx_e822 - Initialize tracking for Tx timestamps + * @pf: Board private structure + * @tx: the Tx tracking structure to initialize + * @port: the port this structure tracks + * + * Initialize the Tx timestamp tracker for this port. For generic MAC devices, + * the timestamp block is shared for all ports in the same quad. To avoid + * ports using the same timestamp index, logically break the block of + * registers into chunks based on the port number. + */ +static int +ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port) +{ + tx->quad = port / ICE_PORTS_PER_QUAD; + tx->quad_offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT; + tx->len = INDEX_PER_PORT; + + return ice_ptp_alloc_tx_tracker(tx); +} + +/** + * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps + * @pf: Board private structure + * @tx: the Tx tracking structure to initialize + * + * Initialize the Tx timestamp tracker for this PF. For E810 devices, each + * port has its own block of timestamps, independent of the other ports. + */ +static int +ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + tx->quad = pf->hw.port_info->lport; + tx->quad_offset = 0; + tx->len = INDEX_PER_QUAD; + + return ice_ptp_alloc_tx_tracker(tx); +} + +/** + * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped + * @pf: pointer to the PF struct + * @tx: PTP Tx tracker to clean up + * + * Loop through the Tx timestamp requests and see if any of them have been + * waiting for a long time. Discard any SKBs that have been waiting for more + * than 2 seconds. This is long enough to be reasonably sure that the + * timestamp will never be captured. This might happen if the packet gets + * discarded before it reaches the PHY timestamping block. + */ +static void ice_ptp_tx_tstamp_cleanup(struct ice_pf *pf, struct ice_ptp_tx *tx) +{ + struct ice_hw *hw = &pf->hw; + u8 idx; + + if (!tx->init) + return; + + for_each_set_bit(idx, tx->in_use, tx->len) { + struct sk_buff *skb; + u64 raw_tstamp; + + /* Check if this SKB has been waiting for too long */ + if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ)) + continue; + + /* Read tstamp to be able to use this register again */ + ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset, + &raw_tstamp); + + spin_lock(&tx->lock); + skb = tx->tstamps[idx].skb; + tx->tstamps[idx].skb = NULL; + clear_bit(idx, tx->in_use); + spin_unlock(&tx->lock); + + /* Count the number of Tx timestamps which have timed out */ + pf->ptp.tx_hwtstamp_timeouts++; + + /* Free the SKB after we've cleared the bit */ + dev_kfree_skb_any(skb); + } +} + +/** + * ice_ptp_update_cached_phctime - Update the cached PHC time values + * @pf: Board specific private structure + * + * This function updates the system time values which are cached in the PF + * structure and the Rx rings. + * + * This function must be called periodically to ensure that the cached value + * is never more than 2 seconds old. + * + * Note that the cached copy in the PF PTP structure is always updated, even + * if we can't update the copy in the Rx rings. + * + * Return: + * * 0 - OK, successfully updated + * * -EAGAIN - PF was busy, need to reschedule the update + */ +static int ice_ptp_update_cached_phctime(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + unsigned long update_before; + u64 systime; + int i; + + update_before = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000); + if (pf->ptp.cached_phc_time && + time_is_before_jiffies(update_before)) { + unsigned long time_taken = jiffies - pf->ptp.cached_phc_jiffies; + + dev_warn(dev, "%u msecs passed between update to cached PHC time\n", + jiffies_to_msecs(time_taken)); + pf->ptp.late_cached_phc_updates++; + } + + /* Read the current PHC time */ + systime = ice_ptp_read_src_clk_reg(pf, NULL); + + /* Update the cached PHC time stored in the PF structure */ + WRITE_ONCE(pf->ptp.cached_phc_time, systime); + WRITE_ONCE(pf->ptp.cached_phc_jiffies, jiffies); + + if (test_and_set_bit(ICE_CFG_BUSY, pf->state)) + return -EAGAIN; + + ice_for_each_vsi(pf, i) { + struct ice_vsi *vsi = pf->vsi[i]; + int j; + + if (!vsi) + continue; + + if (vsi->type != ICE_VSI_PF) + continue; + + ice_for_each_rxq(vsi, j) { + if (!vsi->rx_rings[j]) + continue; + WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime); + } + } + clear_bit(ICE_CFG_BUSY, pf->state); + + return 0; +} + +/** + * ice_ptp_reset_cached_phctime - Reset cached PHC time after an update + * @pf: Board specific private structure + * + * This function must be called when the cached PHC time is no longer valid, + * such as after a time adjustment. It discards any outstanding Tx timestamps, + * and updates the cached PHC time for both the PF and Rx rings. If updating + * the PHC time cannot be done immediately, a warning message is logged and + * the work item is scheduled. + * + * These steps are required in order to ensure that we do not accidentally + * report a timestamp extended by the wrong PHC cached copy. Note that we + * do not directly update the cached timestamp here because it is possible + * this might produce an error when ICE_CFG_BUSY is set. If this occurred, we + * would have to try again. During that time window, timestamps might be + * requested and returned with an invalid extension. Thus, on failure to + * immediately update the cached PHC time we would need to zero the value + * anyways. For this reason, we just zero the value immediately and queue the + * update work item. + */ +static void ice_ptp_reset_cached_phctime(struct ice_pf *pf) +{ + struct device *dev = ice_pf_to_dev(pf); + int err; + + /* Update the cached PHC time immediately if possible, otherwise + * schedule the work item to execute soon. + */ + err = ice_ptp_update_cached_phctime(pf); + if (err) { + /* If another thread is updating the Rx rings, we won't + * properly reset them here. This could lead to reporting of + * invalid timestamps, but there isn't much we can do. + */ + dev_warn(dev, "%s: ICE_CFG_BUSY, unable to immediately update cached PHC time\n", + __func__); + + /* Queue the work item to update the Rx rings when possible */ + kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work, + msecs_to_jiffies(10)); + } + + /* Flush any outstanding Tx timestamps */ + ice_ptp_flush_tx_tracker(pf, &pf->ptp.port.tx); +} + /** * ice_ptp_read_time - Read the time from the device * @pf: Board private structure @@ -900,6 +1238,9 @@ static void ice_ptp_wait_for_offset_valid(struct kthread_work *work) hw = &pf->hw; dev = ice_pf_to_dev(pf); + if (ice_is_reset_in_progress(pf->state)) + return; + if (ice_ptp_check_offset_valid(port)) { /* Offsets not ready yet, try again later */ kthread_queue_delayed_work(pf->ptp.kworker, @@ -1509,7 +1850,7 @@ ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts) ice_ptp_unlock(hw); if (!err) - ice_ptp_update_cached_phctime(pf); + ice_ptp_reset_cached_phctime(pf); /* Reenable periodic outputs */ ice_ptp_enable_all_clkout(pf); @@ -1588,7 +1929,7 @@ static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta) return err; } - ice_ptp_update_cached_phctime(pf); + ice_ptp_reset_cached_phctime(pf); return 0; } @@ -1796,26 +2137,31 @@ void ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) { + struct skb_shared_hwtstamps *hwtstamps; + u64 ts_ns, cached_time; u32 ts_high; - u64 ts_ns; - /* Populate timesync data into skb */ - if (rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID) { - struct skb_shared_hwtstamps *hwtstamps; + if (!(rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID)) + return; - /* Use ice_ptp_extend_32b_ts directly, using the ring-specific - * cached PHC value, rather than accessing the PF. This also - * allows us to simply pass the upper 32bits of nanoseconds - * directly. Calling ice_ptp_extend_40b_ts is unnecessary as - * it would just discard these bits itself. - */ - ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high); - ts_ns = ice_ptp_extend_32b_ts(rx_ring->cached_phctime, ts_high); + cached_time = READ_ONCE(rx_ring->cached_phctime); - hwtstamps = skb_hwtstamps(skb); - memset(hwtstamps, 0, sizeof(*hwtstamps)); - hwtstamps->hwtstamp = ns_to_ktime(ts_ns); - } + /* Do not report a timestamp if we don't have a cached PHC time */ + if (!cached_time) + return; + + /* Use ice_ptp_extend_32b_ts directly, using the ring-specific cached + * PHC value, rather than accessing the PF. This also allows us to + * simply pass the upper 32bits of nanoseconds directly. Calling + * ice_ptp_extend_40b_ts is unnecessary as it would just discard these + * bits itself. + */ + ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high); + ts_ns = ice_ptp_extend_32b_ts(cached_time, ts_high); + + hwtstamps = skb_hwtstamps(skb); + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->hwtstamp = ns_to_ktime(ts_ns); } /** @@ -1870,50 +2216,27 @@ ice_ptp_setup_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info) ice_ptp_disable_sma_pins_e810t(pf, info); } -/** - * ice_ptp_setup_pins_e810t - Setup PTP pins in sysfs - * @pf: pointer to the PF instance - * @info: PTP clock capabilities - */ -static void -ice_ptp_setup_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info) -{ - /* Check if SMA controller is in the netlist */ - if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL) && - !ice_is_pca9575_present(&pf->hw)) - ice_clear_feature_support(pf, ICE_F_SMA_CTRL); - - if (!ice_is_feature_supported(pf, ICE_F_SMA_CTRL)) { - info->n_ext_ts = N_EXT_TS_E810_NO_SMA; - info->n_per_out = N_PER_OUT_E810T_NO_SMA; - return; - } - - info->n_per_out = N_PER_OUT_E810T; - - if (ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) { - info->n_ext_ts = N_EXT_TS_E810; - info->n_pins = NUM_PTP_PINS_E810T; - info->verify = ice_verify_pin_e810t; - } - - /* Complete setup of the SMA pins */ - ice_ptp_setup_sma_pins_e810t(pf, info); -} - /** * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs * @pf: pointer to the PF instance * @info: PTP clock capabilities */ -static void ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info) +static void +ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info) { info->n_per_out = N_PER_OUT_E810; - if (!ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) - return; + if (ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) + info->n_ext_ts = N_EXT_TS_E810; + + if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL)) { + info->n_ext_ts = N_EXT_TS_E810; + info->n_pins = NUM_PTP_PINS_E810T; + info->verify = ice_verify_pin_e810t; - info->n_ext_ts = N_EXT_TS_E810; + /* Complete setup of the SMA pins */ + ice_ptp_setup_sma_pins_e810t(pf, info); + } } /** @@ -1950,11 +2273,7 @@ static void ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info) { info->enable = ice_ptp_gpio_enable_e810; - - if (ice_is_e810t(&pf->hw)) - ice_ptp_setup_pins_e810t(pf, info); - else - ice_ptp_setup_pins_e810(pf, info); + ice_ptp_setup_pins_e810(pf, info); } /** @@ -2015,112 +2334,6 @@ static long ice_ptp_create_clock(struct ice_pf *pf) return 0; } -/** - * ice_ptp_tx_tstamp_work - Process Tx timestamps for a port - * @work: pointer to the kthread_work struct - * - * Process timestamps captured by the PHY associated with this port. To do - * this, loop over each index with a waiting skb. - * - * If a given index has a valid timestamp, perform the following steps: - * - * 1) copy the timestamp out of the PHY register - * 4) clear the timestamp valid bit in the PHY register - * 5) unlock the index by clearing the associated in_use bit. - * 2) extend the 40b timestamp value to get a 64bit timestamp - * 3) send that timestamp to the stack - * - * After looping, if we still have waiting SKBs, then re-queue the work. This - * may cause us effectively poll even when not strictly necessary. We do this - * because it's possible a new timestamp was requested around the same time as - * the interrupt. In some cases hardware might not interrupt us again when the - * timestamp is captured. - * - * Note that we only take the tracking lock when clearing the bit and when - * checking if we need to re-queue this task. The only place where bits can be - * set is the hard xmit routine where an SKB has a request flag set. The only - * places where we clear bits are this work function, or the periodic cleanup - * thread. If the cleanup thread clears a bit we're processing we catch it - * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread - * starts a new timestamp, we might not begin processing it right away but we - * will notice it at the end when we re-queue the work item. If a Tx thread - * starts a new timestamp just after this function exits without re-queuing, - * the interrupt when the timestamp finishes should trigger. Avoiding holding - * the lock for the entire function is important in order to ensure that Tx - * threads do not get blocked while waiting for the lock. - */ -static void ice_ptp_tx_tstamp_work(struct kthread_work *work) -{ - struct ice_ptp_port *ptp_port; - struct ice_ptp_tx *tx; - struct ice_pf *pf; - struct ice_hw *hw; - u8 idx; - - tx = container_of(work, struct ice_ptp_tx, work); - if (!tx->init) - return; - - ptp_port = container_of(tx, struct ice_ptp_port, tx); - pf = ptp_port_to_pf(ptp_port); - hw = &pf->hw; - - for_each_set_bit(idx, tx->in_use, tx->len) { - struct skb_shared_hwtstamps shhwtstamps = {}; - u8 phy_idx = idx + tx->quad_offset; - u64 raw_tstamp, tstamp; - struct sk_buff *skb; - int err; - - ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx); - - err = ice_read_phy_tstamp(hw, tx->quad, phy_idx, - &raw_tstamp); - if (err) - continue; - - ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx); - - /* Check if the timestamp is invalid or stale */ - if (!(raw_tstamp & ICE_PTP_TS_VALID) || - raw_tstamp == tx->tstamps[idx].cached_tstamp) - continue; - - /* The timestamp is valid, so we'll go ahead and clear this - * index and then send the timestamp up to the stack. - */ - spin_lock(&tx->lock); - tx->tstamps[idx].cached_tstamp = raw_tstamp; - clear_bit(idx, tx->in_use); - skb = tx->tstamps[idx].skb; - tx->tstamps[idx].skb = NULL; - spin_unlock(&tx->lock); - - /* it's (unlikely but) possible we raced with the cleanup - * thread for discarding old timestamp requests. - */ - if (!skb) - continue; - - /* Extend the timestamp using cached PHC time */ - tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp); - shhwtstamps.hwtstamp = ns_to_ktime(tstamp); - - ice_trace(tx_tstamp_complete, skb, idx); - - skb_tstamp_tx(skb, &shhwtstamps); - dev_kfree_skb_any(skb); - } - - /* Check if we still have work to do. If so, re-queue this task to - * poll for remaining timestamps. - */ - spin_lock(&tx->lock); - if (!bitmap_empty(tx->in_use, tx->len)) - kthread_queue_work(pf->ptp.kworker, &tx->work); - spin_unlock(&tx->lock); -} - /** * ice_ptp_request_ts - Request an available Tx timestamp index * @tx: the PTP Tx timestamp tracker to request from @@ -2161,177 +2374,17 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) } /** - * ice_ptp_process_ts - Spawn kthread work to handle timestamps + * ice_ptp_process_ts - Process the PTP Tx timestamps * @pf: Board private structure * - * Queue work required to process the PTP Tx timestamps outside of interrupt - * context. + * Returns true if timestamps are processed. */ -void ice_ptp_process_ts(struct ice_pf *pf) +bool ice_ptp_process_ts(struct ice_pf *pf) { if (pf->ptp.port.tx.init) - kthread_queue_work(pf->ptp.kworker, &pf->ptp.port.tx.work); -} - -/** - * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps - * @tx: Tx tracking structure to initialize - * - * Assumes that the length has already been initialized. Do not call directly, - * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead. - */ -static int -ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx) -{ - tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL); - if (!tx->tstamps) - return -ENOMEM; - - tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL); - if (!tx->in_use) { - kfree(tx->tstamps); - tx->tstamps = NULL; - return -ENOMEM; - } - - spin_lock_init(&tx->lock); - kthread_init_work(&tx->work, ice_ptp_tx_tstamp_work); - - tx->init = 1; - - return 0; -} + return ice_ptp_tx_tstamp(&pf->ptp.port.tx); -/** - * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker - * @pf: Board private structure - * @tx: the tracker to flush - */ -static void -ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) -{ - u8 idx; - - for (idx = 0; idx < tx->len; idx++) { - u8 phy_idx = idx + tx->quad_offset; - - spin_lock(&tx->lock); - if (tx->tstamps[idx].skb) { - dev_kfree_skb_any(tx->tstamps[idx].skb); - tx->tstamps[idx].skb = NULL; - } - clear_bit(idx, tx->in_use); - spin_unlock(&tx->lock); - - /* Clear any potential residual timestamp in the PHY block */ - if (!pf->hw.reset_ongoing) - ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx); - } -} - -/** - * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker - * @pf: Board private structure - * @tx: Tx tracking structure to release - * - * Free memory associated with the Tx timestamp tracker. - */ -static void -ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx) -{ - tx->init = 0; - - kthread_cancel_work_sync(&tx->work); - - ice_ptp_flush_tx_tracker(pf, tx); - - kfree(tx->tstamps); - tx->tstamps = NULL; - - bitmap_free(tx->in_use); - tx->in_use = NULL; - - tx->len = 0; -} - -/** - * ice_ptp_init_tx_e822 - Initialize tracking for Tx timestamps - * @pf: Board private structure - * @tx: the Tx tracking structure to initialize - * @port: the port this structure tracks - * - * Initialize the Tx timestamp tracker for this port. For generic MAC devices, - * the timestamp block is shared for all ports in the same quad. To avoid - * ports using the same timestamp index, logically break the block of - * registers into chunks based on the port number. - */ -static int -ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port) -{ - tx->quad = port / ICE_PORTS_PER_QUAD; - tx->quad_offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT; - tx->len = INDEX_PER_PORT; - - return ice_ptp_alloc_tx_tracker(tx); -} - -/** - * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps - * @pf: Board private structure - * @tx: the Tx tracking structure to initialize - * - * Initialize the Tx timestamp tracker for this PF. For E810 devices, each - * port has its own block of timestamps, independent of the other ports. - */ -static int -ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx) -{ - tx->quad = pf->hw.port_info->lport; - tx->quad_offset = 0; - tx->len = INDEX_PER_QUAD; - - return ice_ptp_alloc_tx_tracker(tx); -} - -/** - * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped - * @hw: pointer to the hw struct - * @tx: PTP Tx tracker to clean up - * - * Loop through the Tx timestamp requests and see if any of them have been - * waiting for a long time. Discard any SKBs that have been waiting for more - * than 2 seconds. This is long enough to be reasonably sure that the - * timestamp will never be captured. This might happen if the packet gets - * discarded before it reaches the PHY timestamping block. - */ -static void ice_ptp_tx_tstamp_cleanup(struct ice_hw *hw, struct ice_ptp_tx *tx) -{ - u8 idx; - - if (!tx->init) - return; - - for_each_set_bit(idx, tx->in_use, tx->len) { - struct sk_buff *skb; - u64 raw_tstamp; - - /* Check if this SKB has been waiting for too long */ - if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ)) - continue; - - /* Read tstamp to be able to use this register again */ - ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset, - &raw_tstamp); - - spin_lock(&tx->lock); - skb = tx->tstamps[idx].skb; - tx->tstamps[idx].skb = NULL; - clear_bit(idx, tx->in_use); - spin_unlock(&tx->lock); - - /* Free the SKB after we've cleared the bit */ - dev_kfree_skb_any(skb); - } + return false; } static void ice_ptp_periodic_work(struct kthread_work *work) @@ -2345,7 +2398,7 @@ static void ice_ptp_periodic_work(struct kthread_work *work) err = ice_ptp_update_cached_phctime(pf); - ice_ptp_tx_tstamp_cleanup(&pf->hw, &pf->ptp.port.tx); + ice_ptp_tx_tstamp_cleanup(pf, &pf->ptp.port.tx); /* Run twice a second or reschedule if phc update failed */ kthread_queue_delayed_work(ptp->kworker, &ptp->work, diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h index 10e396abf13094cc633b5baf8a05124b94a18534..028349295b719b6459439b46d5d45ab15fe44389 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp.h @@ -105,7 +105,6 @@ struct ice_tx_tstamp { /** * struct ice_ptp_tx - Tracking structure for all Tx timestamp requests on a port - * @work: work function to handle processing of Tx timestamps * @lock: lock to prevent concurrent write to in_use bitmap * @tstamps: array of len to store outstanding requests * @in_use: bitmap of len to indicate which slots are in use @@ -117,7 +116,6 @@ struct ice_tx_tstamp { * window, timestamps are temporarily disabled. */ struct ice_ptp_tx { - struct kthread_work work; spinlock_t lock; /* lock protecting in_use bitmap */ struct ice_tx_tstamp *tstamps; unsigned long *in_use; @@ -163,6 +161,7 @@ struct ice_ptp_port { * @work: delayed work function for periodic tasks * @extts_work: work function for handling external Tx timestamps * @cached_phc_time: a cached copy of the PHC time for timestamp extension + * @cached_phc_jiffies: jiffies when cached_phc_time was last updated * @ext_ts_chan: the external timestamp channel in use * @ext_ts_irq: the external timestamp IRQ in use * @kworker: kwork thread for handling periodic work @@ -171,12 +170,19 @@ struct ice_ptp_port { * @clock: pointer to registered PTP clock device * @tstamp_config: hardware timestamping configuration * @reset_time: kernel time after clock stop on reset + * @tx_hwtstamp_skipped: number of Tx time stamp requests skipped + * @tx_hwtstamp_timeouts: number of Tx skbs discarded with no time stamp + * @tx_hwtstamp_flushed: number of Tx skbs flushed due to interface closed + * @tx_hwtstamp_discarded: number of Tx skbs discarded due to cached PHC time + * being too old to correctly extend timestamp + * @late_cached_phc_updates: number of times cached PHC update is late */ struct ice_ptp { struct ice_ptp_port port; struct kthread_delayed_work work; struct kthread_work extts_work; u64 cached_phc_time; + unsigned long cached_phc_jiffies; u8 ext_ts_chan; u8 ext_ts_irq; struct kthread_worker *kworker; @@ -185,6 +191,11 @@ struct ice_ptp { struct ptp_clock *clock; struct hwtstamp_config tstamp_config; u64 reset_time; + u32 tx_hwtstamp_skipped; + u32 tx_hwtstamp_timeouts; + u32 tx_hwtstamp_flushed; + u32 tx_hwtstamp_discarded; + u32 late_cached_phc_updates; }; #define __ptp_port_to_ptp(p) \ @@ -224,8 +235,8 @@ struct ice_ptp { #define N_EXT_TS_E810 3 #define N_PER_OUT_E810 4 #define N_PER_OUT_E810T 3 -#define N_PER_OUT_E810T_NO_SMA 2 -#define N_EXT_TS_E810_NO_SMA 2 +#define N_PER_OUT_NO_SMA_E810T 2 +#define N_EXT_TS_NO_SMA_E810T 2 #define ETH_GLTSYN_ENA(_i) (0x03000348 + ((_i) * 4)) #if IS_ENABLED(CONFIG_PTP_1588_CLOCK) @@ -236,7 +247,7 @@ void ice_ptp_cfg_timestamp(struct ice_pf *pf, bool ena); int ice_get_ptp_clock_index(struct ice_pf *pf); s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb); -void ice_ptp_process_ts(struct ice_pf *pf); +bool ice_ptp_process_ts(struct ice_pf *pf); void ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, @@ -269,7 +280,10 @@ ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb) return -1; } -static inline void ice_ptp_process_ts(struct ice_pf *pf) { } +static inline bool ice_ptp_process_ts(struct ice_pf *pf) +{ + return true; +} static inline void ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring, union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) { } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 6dff97d53d81beeac3ddee810063f846384db283..772b1f566d6edcc1326d2ce40e7d2f729473217c 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2021, Intel Corporation. */ +#include #include "ice_common.h" #include "ice_ptp_hw.h" #include "ice_ptp_consts.h" @@ -2587,38 +2588,113 @@ static int ice_write_phy_reg_e810(struct ice_hw *hw, u32 addr, u32 val) } /** - * ice_read_phy_tstamp_e810 - Read a PHY timestamp out of the external PHY + * ice_read_phy_tstamp_ll_e810 - Read a PHY timestamp registers through the FW + * @hw: pointer to the HW struct + * @idx: the timestamp index to read + * @hi: 8 bit timestamp high value + * @lo: 32 bit timestamp low value + * + * Read a 8bit timestamp high value and 32 bit timestamp low value out of the + * timestamp block of the external PHY on the E810 device using the low latency + * timestamp read. + */ +static int +ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo) +{ + u32 val; + u8 i; + + /* Write TS index to read to the PF register so the FW can read it */ + val = FIELD_PREP(TS_LL_READ_TS_IDX, idx) | TS_LL_READ_TS; + wr32(hw, PF_SB_ATQBAL, val); + + /* Read the register repeatedly until the FW provides us the TS */ + for (i = TS_LL_READ_RETRIES; i > 0; i--) { + val = rd32(hw, PF_SB_ATQBAL); + + /* When the bit is cleared, the TS is ready in the register */ + if (!(FIELD_GET(TS_LL_READ_TS, val))) { + /* High 8 bit value of the TS is on the bits 16:23 */ + *hi = FIELD_GET(TS_LL_READ_TS_HIGH, val); + + /* Read the low 32 bit value and set the TS valid bit */ + *lo = rd32(hw, PF_SB_ATQBAH) | TS_VALID; + return 0; + } + + udelay(10); + } + + /* FW failed to provide the TS in time */ + ice_debug(hw, ICE_DBG_PTP, "Failed to read PTP timestamp using low latency read\n"); + return -EINVAL; +} + +/** + * ice_read_phy_tstamp_sbq_e810 - Read a PHY timestamp registers through the sbq * @hw: pointer to the HW struct * @lport: the lport to read from * @idx: the timestamp index to read - * @tstamp: on return, the 40bit timestamp value + * @hi: 8 bit timestamp high value + * @lo: 32 bit timestamp low value * - * Read a 40bit timestamp value out of the timestamp block of the external PHY - * on the E810 device. + * Read a 8bit timestamp high value and 32 bit timestamp low value out of the + * timestamp block of the external PHY on the E810 device using sideband queue. */ static int -ice_read_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx, u64 *tstamp) +ice_read_phy_tstamp_sbq_e810(struct ice_hw *hw, u8 lport, u8 idx, u8 *hi, + u32 *lo) { - u32 lo_addr, hi_addr, lo, hi; + u32 hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx); + u32 lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx); + u32 lo_val, hi_val; int err; - lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx); - hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx); - - err = ice_read_phy_reg_e810(hw, lo_addr, &lo); + err = ice_read_phy_reg_e810(hw, lo_addr, &lo_val); if (err) { ice_debug(hw, ICE_DBG_PTP, "Failed to read low PTP timestamp register, err %d\n", err); return err; } - err = ice_read_phy_reg_e810(hw, hi_addr, &hi); + err = ice_read_phy_reg_e810(hw, hi_addr, &hi_val); if (err) { ice_debug(hw, ICE_DBG_PTP, "Failed to read high PTP timestamp register, err %d\n", err); return err; } + *lo = lo_val; + *hi = (u8)hi_val; + + return 0; +} + +/** + * ice_read_phy_tstamp_e810 - Read a PHY timestamp out of the external PHY + * @hw: pointer to the HW struct + * @lport: the lport to read from + * @idx: the timestamp index to read + * @tstamp: on return, the 40bit timestamp value + * + * Read a 40bit timestamp value out of the timestamp block of the external PHY + * on the E810 device. + */ +static int +ice_read_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx, u64 *tstamp) +{ + u32 lo = 0; + u8 hi = 0; + int err; + + if (hw->dev_caps.ts_dev_info.ts_ll_read) + err = ice_read_phy_tstamp_ll_e810(hw, idx, &hi, &lo); + else + err = ice_read_phy_tstamp_sbq_e810(hw, lport, idx, &hi, &lo); + + if (err) + return err; + /* For E810 devices, the timestamp is reported with the lower 32 bits * in the low register, and the upper 8 bits in the high register. */ diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h index 1246e4ee4b5decc58b34cd0e536ec144c3f9bdb5..2bda64c76abc3ab2d63648177088c670a5c3f0bb 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h @@ -402,6 +402,7 @@ bool ice_is_pca9575_present(struct ice_hw *hw); #define INCVAL_HIGH_M 0xFF /* Timestamp block macros */ +#define TS_VALID BIT(0) #define TS_LOW_M 0xFFFFFFFF #define TS_HIGH_M 0xFF #define TS_HIGH_S 32 @@ -413,6 +414,12 @@ bool ice_is_pca9575_present(struct ice_hw *hw); #define BYTES_PER_IDX_ADDR_L_U 8 #define BYTES_PER_IDX_ADDR_L 4 +/* Tx timestamp low latency read definitions */ +#define TS_LL_READ_RETRIES 200 +#define TS_LL_READ_TS_HIGH GENMASK(23, 16) +#define TS_LL_READ_TS_IDX GENMASK(29, 24) +#define TS_LL_READ_TS BIT(31) + /* Internal PHY timestamp address */ #define TS_L(a, idx) ((a) + ((idx) * BYTES_PER_IDX_ADDR_L_U)) #define TS_H(a, idx) ((a) + ((idx) * BYTES_PER_IDX_ADDR_L_U + \ diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c index 0dac67cd9c77c975d69e71249046035e35b958d9..bd31748aae1b43b44da3de825a35f7124e22d21c 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.c +++ b/drivers/net/ethernet/intel/ice/ice_repr.c @@ -377,10 +377,10 @@ static void ice_repr_rem(struct ice_vf *vf) if (!vf->repr) return; - ice_devlink_destroy_vf_port(vf); kfree(vf->repr->q_vector); vf->repr->q_vector = NULL; unregister_netdev(vf->repr->netdev); + ice_devlink_destroy_vf_port(vf); free_netdev(vf->repr->netdev); vf->repr->netdev = NULL; #ifdef CONFIG_ICE_SWITCHDEV diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index 7947223536e37b4ed984e5dfad83b123dde32ff0..118595763bba3b8c624e90e4ac040ee180f49732 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -1212,7 +1212,7 @@ int ice_sched_init_port(struct ice_port_info *pi) hw = pi->hw; /* Query the Default Topology from FW */ - buf = devm_kzalloc(ice_hw_to_dev(hw), ICE_AQ_MAX_BUF_LEN, GFP_KERNEL); + buf = kzalloc(ICE_AQ_MAX_BUF_LEN, GFP_KERNEL); if (!buf) return -ENOMEM; @@ -1290,7 +1290,7 @@ err_init_port: pi->root = NULL; } - devm_kfree(ice_hw_to_dev(hw), buf); + kfree(buf); return status; } diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index 3808034f7e7e32351a8ea781dd7630ea3071e125..9b762f7972ce5e56d86c27e5116c347382f7ee49 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -42,6 +42,7 @@ enum { ICE_PKT_GTP_NOPAY = BIT(8), ICE_PKT_KMALLOC = BIT(9), ICE_PKT_PPPOE = BIT(10), + ICE_PKT_L2TPV3 = BIT(11), }; struct ice_dummy_pkt_offsets { @@ -1258,6 +1259,65 @@ ICE_DECLARE_PKT_TEMPLATE(pppoe_ipv6_udp) = { 0x00, 0x00, /* 2 bytes for 4 bytes alignment */ }; +ICE_DECLARE_PKT_OFFSETS(ipv4_l2tpv3) = { + { ICE_MAC_OFOS, 0 }, + { ICE_ETYPE_OL, 12 }, + { ICE_IPV4_OFOS, 14 }, + { ICE_L2TPV3, 34 }, + { ICE_PROTOCOL_LAST, 0 }, +}; + +ICE_DECLARE_PKT_TEMPLATE(ipv4_l2tpv3) = { + 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x08, 0x00, /* ICE_ETYPE_OL 12 */ + + 0x45, 0x00, 0x00, 0x20, /* ICE_IPV4_IL 14 */ + 0x00, 0x00, 0x40, 0x00, + 0x40, 0x73, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x00, 0x00, 0x00, 0x00, /* ICE_L2TPV3 34 */ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, /* 2 bytes for 4 bytes alignment */ +}; + +ICE_DECLARE_PKT_OFFSETS(ipv6_l2tpv3) = { + { ICE_MAC_OFOS, 0 }, + { ICE_ETYPE_OL, 12 }, + { ICE_IPV6_OFOS, 14 }, + { ICE_L2TPV3, 54 }, + { ICE_PROTOCOL_LAST, 0 }, +}; + +ICE_DECLARE_PKT_TEMPLATE(ipv6_l2tpv3) = { + 0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x86, 0xDD, /* ICE_ETYPE_OL 12 */ + + 0x60, 0x00, 0x00, 0x00, /* ICE_IPV6_IL 14 */ + 0x00, 0x0c, 0x73, 0x40, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + + 0x00, 0x00, 0x00, 0x00, /* ICE_L2TPV3 54 */ + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, /* 2 bytes for 4 bytes alignment */ +}; + static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = { ICE_PKT_PROFILE(ipv6_gtp, ICE_PKT_TUN_GTPU | ICE_PKT_OUTER_IPV6 | ICE_PKT_GTP_NOPAY), @@ -1297,6 +1357,8 @@ static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = { ICE_PKT_PROFILE(udp_tun_ipv6_tcp, ICE_PKT_TUN_UDP | ICE_PKT_INNER_IPV6 | ICE_PKT_INNER_TCP), + ICE_PKT_PROFILE(ipv6_l2tpv3, ICE_PKT_L2TPV3 | ICE_PKT_OUTER_IPV6), + ICE_PKT_PROFILE(ipv4_l2tpv3, ICE_PKT_L2TPV3), ICE_PKT_PROFILE(udp_tun_tcp, ICE_PKT_TUN_UDP | ICE_PKT_INNER_TCP), ICE_PKT_PROFILE(udp_tun_ipv6_udp, ICE_PKT_TUN_UDP | ICE_PKT_INNER_IPV6), @@ -2274,9 +2336,7 @@ int ice_get_initial_sw_cfg(struct ice_hw *hw) int status; u16 i; - rbuf = devm_kzalloc(ice_hw_to_dev(hw), ICE_SW_CFG_MAX_BUF_LEN, - GFP_KERNEL); - + rbuf = kzalloc(ICE_SW_CFG_MAX_BUF_LEN, GFP_KERNEL); if (!rbuf) return -ENOMEM; @@ -2324,7 +2384,7 @@ int ice_get_initial_sw_cfg(struct ice_hw *hw) } } while (req_desc && !status); - devm_kfree(ice_hw_to_dev(hw), rbuf); + kfree(rbuf); return status; } @@ -3449,31 +3509,15 @@ bool ice_vlan_fltr_exist(struct ice_hw *hw, u16 vlan_id, u16 vsi_handle) * ice_add_mac - Add a MAC address based filter rule * @hw: pointer to the hardware structure * @m_list: list of MAC addresses and forwarding information - * - * IMPORTANT: When the ucast_shared flag is set to false and m_list has - * multiple unicast addresses, the function assumes that all the - * addresses are unique in a given add_mac call. It doesn't - * check for duplicates in this case, removing duplicates from a given - * list should be taken care of in the caller of this function. */ int ice_add_mac(struct ice_hw *hw, struct list_head *m_list) { - struct ice_sw_rule_lkup_rx_tx *s_rule, *r_iter; struct ice_fltr_list_entry *m_list_itr; - struct list_head *rule_head; - u16 total_elem_left, s_rule_size; - struct ice_switch_info *sw; - struct mutex *rule_lock; /* Lock to protect filter rule list */ - u16 num_unicast = 0; int status = 0; - u8 elem_sent; if (!m_list || !hw) return -EINVAL; - s_rule = NULL; - sw = hw->switch_info; - rule_lock = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock; list_for_each_entry(m_list_itr, m_list, list_entry) { u8 *add = &m_list_itr->fltr_info.l_data.mac.mac_addr[0]; u16 vsi_handle; @@ -3492,106 +3536,13 @@ int ice_add_mac(struct ice_hw *hw, struct list_head *m_list) if (m_list_itr->fltr_info.lkup_type != ICE_SW_LKUP_MAC || is_zero_ether_addr(add)) return -EINVAL; - if (is_unicast_ether_addr(add) && !hw->ucast_shared) { - /* Don't overwrite the unicast address */ - mutex_lock(rule_lock); - if (ice_find_rule_entry(hw, ICE_SW_LKUP_MAC, - &m_list_itr->fltr_info)) { - mutex_unlock(rule_lock); - return -EEXIST; - } - mutex_unlock(rule_lock); - num_unicast++; - } else if (is_multicast_ether_addr(add) || - (is_unicast_ether_addr(add) && hw->ucast_shared)) { - m_list_itr->status = - ice_add_rule_internal(hw, ICE_SW_LKUP_MAC, - m_list_itr); - if (m_list_itr->status) - return m_list_itr->status; - } - } - mutex_lock(rule_lock); - /* Exit if no suitable entries were found for adding bulk switch rule */ - if (!num_unicast) { - status = 0; - goto ice_add_mac_exit; - } - - rule_head = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rules; - - /* Allocate switch rule buffer for the bulk update for unicast */ - s_rule_size = ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s_rule); - s_rule = devm_kcalloc(ice_hw_to_dev(hw), num_unicast, s_rule_size, - GFP_KERNEL); - if (!s_rule) { - status = -ENOMEM; - goto ice_add_mac_exit; - } - - r_iter = s_rule; - list_for_each_entry(m_list_itr, m_list, list_entry) { - struct ice_fltr_info *f_info = &m_list_itr->fltr_info; - u8 *mac_addr = &f_info->l_data.mac.mac_addr[0]; - - if (is_unicast_ether_addr(mac_addr)) { - ice_fill_sw_rule(hw, &m_list_itr->fltr_info, r_iter, - ice_aqc_opc_add_sw_rules); - r_iter = (typeof(s_rule))((u8 *)r_iter + s_rule_size); - } - } - - /* Call AQ bulk switch rule update for all unicast addresses */ - r_iter = s_rule; - /* Call AQ switch rule in AQ_MAX chunk */ - for (total_elem_left = num_unicast; total_elem_left > 0; - total_elem_left -= elem_sent) { - struct ice_sw_rule_lkup_rx_tx *entry = r_iter; - - elem_sent = min_t(u8, total_elem_left, - (ICE_AQ_MAX_BUF_LEN / s_rule_size)); - status = ice_aq_sw_rules(hw, entry, elem_sent * s_rule_size, - elem_sent, ice_aqc_opc_add_sw_rules, - NULL); - if (status) - goto ice_add_mac_exit; - r_iter = (typeof(s_rule)) - ((u8 *)r_iter + (elem_sent * s_rule_size)); + m_list_itr->status = ice_add_rule_internal(hw, ICE_SW_LKUP_MAC, + m_list_itr); + if (m_list_itr->status) + return m_list_itr->status; } - /* Fill up rule ID based on the value returned from FW */ - r_iter = s_rule; - list_for_each_entry(m_list_itr, m_list, list_entry) { - struct ice_fltr_info *f_info = &m_list_itr->fltr_info; - u8 *mac_addr = &f_info->l_data.mac.mac_addr[0]; - struct ice_fltr_mgmt_list_entry *fm_entry; - - if (is_unicast_ether_addr(mac_addr)) { - f_info->fltr_rule_id = le16_to_cpu(r_iter->index); - f_info->fltr_act = ICE_FWD_TO_VSI; - /* Create an entry to track this MAC address */ - fm_entry = devm_kzalloc(ice_hw_to_dev(hw), - sizeof(*fm_entry), GFP_KERNEL); - if (!fm_entry) { - status = -ENOMEM; - goto ice_add_mac_exit; - } - fm_entry->fltr_info = *f_info; - fm_entry->vsi_count = 1; - /* The book keeping entries will get removed when - * base driver calls remove filter AQ command - */ - - list_add(&fm_entry->list_entry, rule_head); - r_iter = (typeof(s_rule))((u8 *)r_iter + s_rule_size); - } - } - -ice_add_mac_exit: - mutex_unlock(rule_lock); - if (s_rule) - devm_kfree(ice_hw_to_dev(hw), s_rule); return status; } @@ -3978,38 +3929,6 @@ ice_check_if_dflt_vsi(struct ice_port_info *pi, u16 vsi_handle, return ret; } -/** - * ice_find_ucast_rule_entry - Search for a unicast MAC filter rule entry - * @hw: pointer to the hardware structure - * @recp_id: lookup type for which the specified rule needs to be searched - * @f_info: rule information - * - * Helper function to search for a unicast rule entry - this is to be used - * to remove unicast MAC filter that is not shared with other VSIs on the - * PF switch. - * - * Returns pointer to entry storing the rule if found - */ -static struct ice_fltr_mgmt_list_entry * -ice_find_ucast_rule_entry(struct ice_hw *hw, u8 recp_id, - struct ice_fltr_info *f_info) -{ - struct ice_switch_info *sw = hw->switch_info; - struct ice_fltr_mgmt_list_entry *list_itr; - struct list_head *list_head; - - list_head = &sw->recp_list[recp_id].filt_rules; - list_for_each_entry(list_itr, list_head, list_entry) { - if (!memcmp(&f_info->l_data, &list_itr->fltr_info.l_data, - sizeof(f_info->l_data)) && - f_info->fwd_id.hw_vsi_id == - list_itr->fltr_info.fwd_id.hw_vsi_id && - f_info->flag == list_itr->fltr_info.flag) - return list_itr; - } - return NULL; -} - /** * ice_remove_mac - remove a MAC address based filter rule * @hw: pointer to the hardware structure @@ -4026,15 +3945,12 @@ ice_find_ucast_rule_entry(struct ice_hw *hw, u8 recp_id, int ice_remove_mac(struct ice_hw *hw, struct list_head *m_list) { struct ice_fltr_list_entry *list_itr, *tmp; - struct mutex *rule_lock; /* Lock to protect filter rule list */ if (!m_list) return -EINVAL; - rule_lock = &hw->switch_info->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock; list_for_each_entry_safe(list_itr, tmp, m_list, list_entry) { enum ice_sw_lkup_type l_type = list_itr->fltr_info.lkup_type; - u8 *add = &list_itr->fltr_info.l_data.mac.mac_addr[0]; u16 vsi_handle; if (l_type != ICE_SW_LKUP_MAC) @@ -4046,19 +3962,7 @@ int ice_remove_mac(struct ice_hw *hw, struct list_head *m_list) list_itr->fltr_info.fwd_id.hw_vsi_id = ice_get_hw_vsi_num(hw, vsi_handle); - if (is_unicast_ether_addr(add) && !hw->ucast_shared) { - /* Don't remove the unicast address that belongs to - * another VSI on the switch, since it is not being - * shared... - */ - mutex_lock(rule_lock); - if (!ice_find_ucast_rule_entry(hw, ICE_SW_LKUP_MAC, - &list_itr->fltr_info)) { - mutex_unlock(rule_lock); - return -ENOENT; - } - mutex_unlock(rule_lock); - } + list_itr->status = ice_remove_rule_internal(hw, ICE_SW_LKUP_MAC, list_itr); @@ -4648,6 +4552,7 @@ static const struct ice_prot_ext_tbl_entry ice_prot_ext[ICE_PROTOCOL_LAST] = { { ICE_GTP, { 8, 10, 12, 14, 16, 18, 20, 22 } }, { ICE_GTP_NO_PAY, { 8, 10, 12, 14 } }, { ICE_PPPOE, { 0, 2, 4, 6 } }, + { ICE_L2TPV3, { 0, 2, 4, 6, 8, 10 } }, { ICE_VLAN_EX, { 2, 0 } }, { ICE_VLAN_IN, { 2, 0 } }, }; @@ -4671,6 +4576,7 @@ static struct ice_protocol_entry ice_prot_id_tbl[ICE_PROTOCOL_LAST] = { { ICE_GTP, ICE_UDP_OF_HW }, { ICE_GTP_NO_PAY, ICE_UDP_ILOS_HW }, { ICE_PPPOE, ICE_PPPOE_HW }, + { ICE_L2TPV3, ICE_L2TPV3_HW }, { ICE_VLAN_EX, ICE_VLAN_OF_HW }, { ICE_VLAN_IN, ICE_VLAN_OL_HW }, }; @@ -5754,7 +5660,8 @@ ice_find_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, if (lkups[i].h_u.pppoe_hdr.ppp_prot_id == htons(PPP_IPV6)) match |= ICE_PKT_OUTER_IPV6; - } + } else if (lkups[i].type == ICE_L2TPV3) + match |= ICE_PKT_L2TPV3; } while (ret->match && (match & ret->match) != ret->match) @@ -5855,6 +5762,9 @@ ice_fill_adv_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt, case ICE_PPPOE: len = sizeof(struct ice_pppoe_hdr); break; + case ICE_L2TPV3: + len = sizeof(struct ice_l2tpv3_sess_hdr); + break; default: return -EINVAL; } diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.c b/drivers/net/ethernet/intel/ice/ice_tc_lib.c index a298862857a8a83054489051701b5a91fbb3a1ee..f68c555be4e9a41d859118464550b56f6c8f7606 100644 --- a/drivers/net/ethernet/intel/ice/ice_tc_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.c @@ -36,6 +36,10 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers, ICE_TC_FLWR_FIELD_ENC_DEST_IPV6)) lkups_cnt++; + if (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS | + ICE_TC_FLWR_FIELD_ENC_IP_TTL)) + lkups_cnt++; + if (flags & ICE_TC_FLWR_FIELD_ENC_DEST_L4_PORT) lkups_cnt++; @@ -47,11 +51,11 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers, lkups_cnt++; /* is VLAN specified? */ - if (flags & ICE_TC_FLWR_FIELD_VLAN) + if (flags & (ICE_TC_FLWR_FIELD_VLAN | ICE_TC_FLWR_FIELD_VLAN_PRIO)) lkups_cnt++; /* is CVLAN specified? */ - if (flags & ICE_TC_FLWR_FIELD_CVLAN) + if (flags & (ICE_TC_FLWR_FIELD_CVLAN | ICE_TC_FLWR_FIELD_CVLAN_PRIO)) lkups_cnt++; /* are PPPoE options specified? */ @@ -64,6 +68,13 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers, ICE_TC_FLWR_FIELD_DEST_IPV6 | ICE_TC_FLWR_FIELD_SRC_IPV6)) lkups_cnt++; + if (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL)) + lkups_cnt++; + + /* are L2TPv3 options specified? */ + if (flags & ICE_TC_FLWR_FIELD_L2TPV3_SESSID) + lkups_cnt++; + /* is L4 (TCP/UDP/any other L4 protocol fields) specified? */ if (flags & (ICE_TC_FLWR_FIELD_DEST_L4_PORT | ICE_TC_FLWR_FIELD_SRC_L4_PORT)) @@ -257,6 +268,50 @@ ice_tc_fill_tunnel_outer(u32 flags, struct ice_tc_flower_fltr *fltr, i++; } + if (fltr->inner_headers.l2_key.n_proto == htons(ETH_P_IP) && + (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS | + ICE_TC_FLWR_FIELD_ENC_IP_TTL))) { + list[i].type = ice_proto_type_from_ipv4(false); + + if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TOS) { + list[i].h_u.ipv4_hdr.tos = hdr->l3_key.tos; + list[i].m_u.ipv4_hdr.tos = hdr->l3_mask.tos; + } + + if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TTL) { + list[i].h_u.ipv4_hdr.time_to_live = hdr->l3_key.ttl; + list[i].m_u.ipv4_hdr.time_to_live = hdr->l3_mask.ttl; + } + + i++; + } + + if (fltr->inner_headers.l2_key.n_proto == htons(ETH_P_IPV6) && + (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS | + ICE_TC_FLWR_FIELD_ENC_IP_TTL))) { + struct ice_ipv6_hdr *hdr_h, *hdr_m; + + hdr_h = &list[i].h_u.ipv6_hdr; + hdr_m = &list[i].m_u.ipv6_hdr; + list[i].type = ice_proto_type_from_ipv6(false); + + if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TOS) { + be32p_replace_bits(&hdr_h->be_ver_tc_flow, + hdr->l3_key.tos, + ICE_IPV6_HDR_TC_MASK); + be32p_replace_bits(&hdr_m->be_ver_tc_flow, + hdr->l3_mask.tos, + ICE_IPV6_HDR_TC_MASK); + } + + if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TTL) { + hdr_h->hop_limit = hdr->l3_key.ttl; + hdr_m->hop_limit = hdr->l3_mask.ttl; + } + + i++; + } + if ((flags & ICE_TC_FLWR_FIELD_ENC_DEST_L4_PORT) && hdr->l3_key.ip_proto == IPPROTO_UDP) { list[i].type = ICE_UDP_OF; @@ -334,7 +389,7 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags, } /* copy VLAN info */ - if (flags & ICE_TC_FLWR_FIELD_VLAN) { + if (flags & (ICE_TC_FLWR_FIELD_VLAN | ICE_TC_FLWR_FIELD_VLAN_PRIO)) { vlan_tpid = be16_to_cpu(headers->vlan_hdr.vlan_tpid); rule_info->vlan_type = ice_check_supported_vlan_tpid(vlan_tpid); @@ -343,15 +398,45 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags, list[i].type = ICE_VLAN_EX; else list[i].type = ICE_VLAN_OFOS; - list[i].h_u.vlan_hdr.vlan = headers->vlan_hdr.vlan_id; - list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xFFFF); + + if (flags & ICE_TC_FLWR_FIELD_VLAN) { + list[i].h_u.vlan_hdr.vlan = headers->vlan_hdr.vlan_id; + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0x0FFF); + } + + if (flags & ICE_TC_FLWR_FIELD_VLAN_PRIO) { + if (flags & ICE_TC_FLWR_FIELD_VLAN) { + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xEFFF); + } else { + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xE000); + list[i].h_u.vlan_hdr.vlan = 0; + } + list[i].h_u.vlan_hdr.vlan |= + headers->vlan_hdr.vlan_prio; + } + i++; } - if (flags & ICE_TC_FLWR_FIELD_CVLAN) { + if (flags & (ICE_TC_FLWR_FIELD_CVLAN | ICE_TC_FLWR_FIELD_CVLAN_PRIO)) { list[i].type = ICE_VLAN_IN; - list[i].h_u.vlan_hdr.vlan = headers->cvlan_hdr.vlan_id; - list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xFFFF); + + if (flags & ICE_TC_FLWR_FIELD_CVLAN) { + list[i].h_u.vlan_hdr.vlan = headers->cvlan_hdr.vlan_id; + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0x0FFF); + } + + if (flags & ICE_TC_FLWR_FIELD_CVLAN_PRIO) { + if (flags & ICE_TC_FLWR_FIELD_CVLAN) { + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xEFFF); + } else { + list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xE000); + list[i].h_u.vlan_hdr.vlan = 0; + } + list[i].h_u.vlan_hdr.vlan |= + headers->cvlan_hdr.vlan_prio; + } + i++; } @@ -420,6 +505,61 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags, i++; } + if (headers->l2_key.n_proto == htons(ETH_P_IP) && + (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL))) { + list[i].type = ice_proto_type_from_ipv4(inner); + + if (flags & ICE_TC_FLWR_FIELD_IP_TOS) { + list[i].h_u.ipv4_hdr.tos = headers->l3_key.tos; + list[i].m_u.ipv4_hdr.tos = headers->l3_mask.tos; + } + + if (flags & ICE_TC_FLWR_FIELD_IP_TTL) { + list[i].h_u.ipv4_hdr.time_to_live = + headers->l3_key.ttl; + list[i].m_u.ipv4_hdr.time_to_live = + headers->l3_mask.ttl; + } + + i++; + } + + if (headers->l2_key.n_proto == htons(ETH_P_IPV6) && + (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL))) { + struct ice_ipv6_hdr *hdr_h, *hdr_m; + + hdr_h = &list[i].h_u.ipv6_hdr; + hdr_m = &list[i].m_u.ipv6_hdr; + list[i].type = ice_proto_type_from_ipv6(inner); + + if (flags & ICE_TC_FLWR_FIELD_IP_TOS) { + be32p_replace_bits(&hdr_h->be_ver_tc_flow, + headers->l3_key.tos, + ICE_IPV6_HDR_TC_MASK); + be32p_replace_bits(&hdr_m->be_ver_tc_flow, + headers->l3_mask.tos, + ICE_IPV6_HDR_TC_MASK); + } + + if (flags & ICE_TC_FLWR_FIELD_IP_TTL) { + hdr_h->hop_limit = headers->l3_key.ttl; + hdr_m->hop_limit = headers->l3_mask.ttl; + } + + i++; + } + + if (flags & ICE_TC_FLWR_FIELD_L2TPV3_SESSID) { + list[i].type = ICE_L2TPV3; + + list[i].h_u.l2tpv3_sess_hdr.session_id = + headers->l2tpv3_hdr.session_id; + list[i].m_u.l2tpv3_sess_hdr.session_id = + cpu_to_be32(0xFFFFFFFF); + + i++; + } + /* copy L4 (src, dest) port */ if (flags & (ICE_TC_FLWR_FIELD_DEST_L4_PORT | ICE_TC_FLWR_FIELD_SRC_L4_PORT)) { @@ -838,6 +978,40 @@ ice_tc_set_ipv6(struct flow_match_ipv6_addrs *match, return 0; } +/** + * ice_tc_set_tos_ttl - Parse IP ToS/TTL from TC flower filter + * @match: Pointer to flow match structure + * @fltr: Pointer to filter structure + * @headers: inner or outer header fields + * @is_encap: set true for tunnel + */ +static void +ice_tc_set_tos_ttl(struct flow_match_ip *match, + struct ice_tc_flower_fltr *fltr, + struct ice_tc_flower_lyr_2_4_hdrs *headers, + bool is_encap) +{ + if (match->mask->tos) { + if (is_encap) + fltr->flags |= ICE_TC_FLWR_FIELD_ENC_IP_TOS; + else + fltr->flags |= ICE_TC_FLWR_FIELD_IP_TOS; + + headers->l3_key.tos = match->key->tos; + headers->l3_mask.tos = match->mask->tos; + } + + if (match->mask->ttl) { + if (is_encap) + fltr->flags |= ICE_TC_FLWR_FIELD_ENC_IP_TTL; + else + fltr->flags |= ICE_TC_FLWR_FIELD_IP_TTL; + + headers->l3_key.ttl = match->key->ttl; + headers->l3_mask.ttl = match->mask->ttl; + } +} + /** * ice_tc_set_port - Parse ports from TC flower filter * @match: Flow match structure @@ -967,10 +1141,7 @@ ice_parse_tunnel_attr(struct net_device *dev, struct flow_rule *rule, struct flow_match_ip match; flow_rule_match_enc_ip(rule, &match); - headers->l3_key.tos = match.key->tos; - headers->l3_key.ttl = match.key->ttl; - headers->l3_mask.tos = match.mask->tos; - headers->l3_mask.ttl = match.mask->ttl; + ice_tc_set_tos_ttl(&match, fltr, headers, true); } if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS) && @@ -1039,9 +1210,11 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) | BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) | BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) | + BIT(FLOW_DISSECTOR_KEY_IP) | BIT(FLOW_DISSECTOR_KEY_ENC_IP) | BIT(FLOW_DISSECTOR_KEY_PORTS) | - BIT(FLOW_DISSECTOR_KEY_PPPOE))) { + BIT(FLOW_DISSECTOR_KEY_PPPOE) | + BIT(FLOW_DISSECTOR_KEY_L2TPV3))) { NL_SET_ERR_MSG_MOD(fltr->extack, "Unsupported key used"); return -EOPNOTSUPP; } @@ -1137,16 +1310,22 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, if (match.mask->vlan_id) { if (match.mask->vlan_id == VLAN_VID_MASK) { fltr->flags |= ICE_TC_FLWR_FIELD_VLAN; + headers->vlan_hdr.vlan_id = + cpu_to_be16(match.key->vlan_id & + VLAN_VID_MASK); } else { NL_SET_ERR_MSG_MOD(fltr->extack, "Bad VLAN mask"); return -EINVAL; } } - headers->vlan_hdr.vlan_id = - cpu_to_be16(match.key->vlan_id & VLAN_VID_MASK); - if (match.mask->vlan_priority) - headers->vlan_hdr.vlan_prio = match.key->vlan_priority; + if (match.mask->vlan_priority) { + fltr->flags |= ICE_TC_FLWR_FIELD_VLAN_PRIO; + headers->vlan_hdr.vlan_prio = + cpu_to_be16((match.key->vlan_priority << + VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK); + } + if (match.mask->vlan_tpid) headers->vlan_hdr.vlan_tpid = match.key->vlan_tpid; } @@ -1164,6 +1343,9 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, if (match.mask->vlan_id) { if (match.mask->vlan_id == VLAN_VID_MASK) { fltr->flags |= ICE_TC_FLWR_FIELD_CVLAN; + headers->cvlan_hdr.vlan_id = + cpu_to_be16(match.key->vlan_id & + VLAN_VID_MASK); } else { NL_SET_ERR_MSG_MOD(fltr->extack, "Bad CVLAN mask"); @@ -1171,10 +1353,12 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, } } - headers->cvlan_hdr.vlan_id = - cpu_to_be16(match.key->vlan_id & VLAN_VID_MASK); - if (match.mask->vlan_priority) - headers->cvlan_hdr.vlan_prio = match.key->vlan_priority; + if (match.mask->vlan_priority) { + fltr->flags |= ICE_TC_FLWR_FIELD_CVLAN_PRIO; + headers->cvlan_hdr.vlan_prio = + cpu_to_be16((match.key->vlan_priority << + VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK); + } } if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PPPOE)) { @@ -1217,6 +1401,22 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi, return -EINVAL; } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { + struct flow_match_ip match; + + flow_rule_match_ip(rule, &match); + ice_tc_set_tos_ttl(&match, fltr, headers, false); + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_L2TPV3)) { + struct flow_match_l2tpv3 match; + + flow_rule_match_l2tpv3(rule, &match); + + fltr->flags |= ICE_TC_FLWR_FIELD_L2TPV3_SESSID; + headers->l2tpv3_hdr.session_id = match.key->session_id; + } + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { struct flow_match_ports match; diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.h b/drivers/net/ethernet/intel/ice/ice_tc_lib.h index 91cd3d3778c7cc180fa57b60bfedda1a41f113a0..92642faad595f11cc41b1dadfff77b36d4574293 100644 --- a/drivers/net/ethernet/intel/ice/ice_tc_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.h @@ -26,9 +26,18 @@ #define ICE_TC_FLWR_FIELD_CVLAN BIT(19) #define ICE_TC_FLWR_FIELD_PPPOE_SESSID BIT(20) #define ICE_TC_FLWR_FIELD_PPP_PROTO BIT(21) +#define ICE_TC_FLWR_FIELD_IP_TOS BIT(22) +#define ICE_TC_FLWR_FIELD_IP_TTL BIT(23) +#define ICE_TC_FLWR_FIELD_ENC_IP_TOS BIT(24) +#define ICE_TC_FLWR_FIELD_ENC_IP_TTL BIT(25) +#define ICE_TC_FLWR_FIELD_L2TPV3_SESSID BIT(26) +#define ICE_TC_FLWR_FIELD_VLAN_PRIO BIT(27) +#define ICE_TC_FLWR_FIELD_CVLAN_PRIO BIT(28) #define ICE_TC_FLOWER_MASK_32 0xFFFFFFFF +#define ICE_IPV6_HDR_TC_MASK 0xFF00000 + struct ice_indr_block_priv { struct net_device *netdev; struct ice_netdev_priv *np; @@ -42,7 +51,7 @@ struct ice_tc_flower_action { struct ice_tc_vlan_hdr { __be16 vlan_id; /* Only last 12 bits valid */ - u16 vlan_prio; /* Only last 3 bits valid (valid values: 0..7) */ + __be16 vlan_prio; /* Only last 3 bits valid (valid values: 0..7) */ __be16 vlan_tpid; }; @@ -80,6 +89,10 @@ struct ice_tc_l3_hdr { u8 ttl; }; +struct ice_tc_l2tpv3_hdr { + __be32 session_id; +}; + struct ice_tc_l4_hdr { __be16 dst_port; __be16 src_port; @@ -92,6 +105,7 @@ struct ice_tc_flower_lyr_2_4_hdrs { struct ice_tc_vlan_hdr vlan_hdr; struct ice_tc_vlan_hdr cvlan_hdr; struct ice_tc_pppoe_hdr pppoe_hdr; + struct ice_tc_l2tpv3_hdr l2tpv3_hdr; /* L3 (IPv4[6]) layer fields with their mask */ struct ice_tc_l3_hdr l3_key; struct ice_tc_l3_hdr l3_mask; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 836dce8407124f2f200f36fab42412f80009469e..dbe80e5053a823163ab4e4fe1d9e054a4c1b948e 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -610,7 +610,7 @@ ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, if (test_bit(ICE_VSI_DOWN, vsi->state)) return -ENETDOWN; - if (!ice_is_xdp_ena_vsi(vsi) || queue_index >= vsi->num_xdp_txq) + if (!ice_is_xdp_ena_vsi(vsi)) return -ENXIO; if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) @@ -621,6 +621,9 @@ ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, xdp_ring = vsi->xdp_rings[queue_index]; spin_lock(&xdp_ring->tx_lock); } else { + /* Generally, should not happen */ + if (unlikely(queue_index >= vsi->num_xdp_txq)) + return -ENXIO; xdp_ring = vsi->xdp_rings[queue_index]; } @@ -1464,7 +1467,7 @@ int ice_napi_poll(struct napi_struct *napi, int budget) bool wd; if (tx_ring->xsk_pool) - wd = ice_xmit_zc(tx_ring, ICE_DESC_UNUSED(tx_ring), budget); + wd = ice_xmit_zc(tx_ring); else if (ice_ring_is_xdp(tx_ring)) wd = true; else @@ -2255,8 +2258,10 @@ ice_tstamp(struct ice_tx_ring *tx_ring, struct sk_buff *skb, /* Grab an open timestamp slot */ idx = ice_ptp_request_ts(tx_ring->tx_tstamps, skb); - if (idx < 0) + if (idx < 0) { + tx_ring->vsi->back->ptp.tx_hwtstamp_skipped++; return; + } off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX | (ICE_TX_CTX_DESC_TSYN << ICE_TXD_CTX_QW1_CMD_S) | diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index ca902af54bb4290c9ab009769c5bb79ffc73ced3..932b5661ec4d6cf29dbf34127cc13a649da452f7 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -295,10 +295,11 @@ struct ice_rx_ring { struct xsk_buff_pool *xsk_pool; struct sk_buff *skb; dma_addr_t dma; /* physical address of ring */ -#define ICE_RX_FLAGS_RING_BUILD_SKB BIT(1) u64 cached_phctime; u8 dcb_tc; /* Traffic class of ring */ u8 ptp_rx; +#define ICE_RX_FLAGS_RING_BUILD_SKB BIT(1) +#define ICE_RX_FLAGS_CRC_STRIP_DIS BIT(2) u8 flags; } ____cacheline_internodealigned_in_smp; diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 861b6432295916d541eab130e9eafcf4340291b2..e1abfcee96dcd853035b85af5e29410996bf0306 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -347,6 +347,7 @@ struct ice_ts_func_info { #define ICE_TS_DEV_ENA_M BIT(24) #define ICE_TS_TMR0_ENA_M BIT(25) #define ICE_TS_TMR1_ENA_M BIT(26) +#define ICE_TS_LL_TX_TS_READ_M BIT(28) struct ice_ts_dev_info { /* Device specific info */ @@ -359,6 +360,7 @@ struct ice_ts_dev_info { u8 ena; u8 tmr0_ena; u8 tmr1_ena; + u8 ts_ll_read; }; /* Function specific capabilities */ @@ -564,6 +566,8 @@ enum ice_rl_type { #define ICE_SCHED_INVAL_PROF_ID 0xFFFF #define ICE_SCHED_DFLT_BURST_SIZE (15 * 1024) /* in bytes (15k) */ +#define ICE_MAX_PORT_PER_PCI_DEV 8 + /* Data structure for saving BW information */ enum ice_bw_type { ICE_BW_TYPE_PRIO, @@ -885,8 +889,6 @@ struct ice_hw { /* INTRL granularity in 1 us */ u8 intrl_gran; - u8 ucast_shared; /* true if VSIs can share unicast addr */ - #define ICE_PHY_PER_NAC 1 #define ICE_MAX_QUAD 2 #define ICE_NUM_QUAD_TYPE 2 diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 49ba8bfdbf047a83bad4653cffbdeb389752ba11..056c904b83ccb6ab03902d9907c1a3ef4776cd5a 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -192,6 +192,7 @@ static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) err = ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, true); if (err) return err; + ice_clean_rx_ring(rx_ring); ice_qvec_toggle_napi(vsi, q_vector, false); ice_qp_clean_rings(vsi, q_idx); @@ -243,7 +244,7 @@ static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) if (err) goto free_buf; ice_set_ring_xdp(xdp_ring); - xdp_ring->xsk_pool = ice_tx_xsk_pool(xdp_ring); + ice_tx_xsk_pool(vsi, q_idx); } err = ice_vsi_cfg_rxq(rx_ring); @@ -316,6 +317,62 @@ ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) return 0; } +/** + * ice_realloc_rx_xdp_bufs - reallocate for either XSK or normal buffer + * @rx_ring: Rx ring + * @pool_present: is pool for XSK present + * + * Try allocating memory and return ENOMEM, if failed to allocate. + * If allocation was successful, substitute buffer with allocated one. + * Returns 0 on success, negative on failure + */ +static int +ice_realloc_rx_xdp_bufs(struct ice_rx_ring *rx_ring, bool pool_present) +{ + size_t elem_size = pool_present ? sizeof(*rx_ring->xdp_buf) : + sizeof(*rx_ring->rx_buf); + void *sw_ring = kcalloc(rx_ring->count, elem_size, GFP_KERNEL); + + if (!sw_ring) + return -ENOMEM; + + if (pool_present) { + kfree(rx_ring->rx_buf); + rx_ring->rx_buf = NULL; + rx_ring->xdp_buf = sw_ring; + } else { + kfree(rx_ring->xdp_buf); + rx_ring->xdp_buf = NULL; + rx_ring->rx_buf = sw_ring; + } + + return 0; +} + +/** + * ice_realloc_zc_buf - reallocate XDP ZC queue pairs + * @vsi: Current VSI + * @zc: is zero copy set + * + * Reallocate buffer for rx_rings that might be used by XSK. + * XDP requires more memory, than rx_buf provides. + * Returns 0 on success, negative on failure + */ +int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc) +{ + struct ice_rx_ring *rx_ring; + unsigned long q; + + for_each_set_bit(q, vsi->af_xdp_zc_qps, + max_t(int, vsi->alloc_txq, vsi->alloc_rxq)) { + rx_ring = vsi->rx_rings[q]; + if (ice_realloc_rx_xdp_bufs(rx_ring, zc)) + return -ENOMEM; + } + + return 0; +} + /** * ice_xsk_pool_setup - enable/disable a buffer pool region depending on its state * @vsi: Current VSI @@ -329,9 +386,8 @@ int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) bool if_running, pool_present = !!pool; int ret = 0, pool_failure = 0; - if (!is_power_of_2(vsi->rx_rings[qid]->count) || - !is_power_of_2(vsi->tx_rings[qid]->count)) { - netdev_err(vsi->netdev, "Please align ring sizes to power of 2\n"); + if (qid >= vsi->num_rxq || qid >= vsi->num_txq) { + netdev_err(vsi->netdev, "Please use queue id in scope of combined queues count\n"); pool_failure = -EINVAL; goto failure; } @@ -339,11 +395,17 @@ int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi); if (if_running) { + struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; + ret = ice_qp_dis(vsi, qid); if (ret) { netdev_err(vsi->netdev, "ice_qp_dis error = %d\n", ret); goto xsk_pool_if_up; } + + ret = ice_realloc_rx_xdp_bufs(rx_ring, pool_present); + if (ret) + goto xsk_pool_if_up; } pool_failure = pool_present ? ice_xsk_pool_enable(vsi, pool, qid) : @@ -353,7 +415,7 @@ xsk_pool_if_up: if (if_running) { ret = ice_qp_ena(vsi, qid); if (!ret && pool_present) - napi_schedule(&vsi->xdp_rings[qid]->q_vector->napi); + napi_schedule(&vsi->rx_rings[qid]->xdp_ring->q_vector->napi); else if (ret) netdev_err(vsi->netdev, "ice_qp_ena error = %d\n", ret); } @@ -465,11 +527,10 @@ exit: bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count) { u16 rx_thresh = ICE_RING_QUARTER(rx_ring); - u16 batched, leftover, i, tail_bumps; + u16 leftover, i, tail_bumps; - batched = ALIGN_DOWN(count, rx_thresh); - tail_bumps = batched / rx_thresh; - leftover = count & (rx_thresh - 1); + tail_bumps = count / rx_thresh; + leftover = count - (tail_bumps * rx_thresh); for (i = 0; i < tail_bumps; i++) if (!__ice_alloc_rx_bufs_zc(rx_ring, rx_thresh)) @@ -719,69 +780,57 @@ ice_clean_xdp_tx_buf(struct ice_tx_ring *xdp_ring, struct ice_tx_buf *tx_buf) } /** - * ice_clean_xdp_irq_zc - Reclaim resources after transmit completes on XDP ring - * @xdp_ring: XDP ring to clean - * @napi_budget: amount of descriptors that NAPI allows us to clean - * - * Returns count of cleaned descriptors + * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ + * @xdp_ring: XDP Tx ring */ -static u16 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring, int napi_budget) +static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring) { - u16 tx_thresh = ICE_RING_QUARTER(xdp_ring); - int budget = napi_budget / tx_thresh; - u16 next_dd = xdp_ring->next_dd; - u16 ntc, cleared_dds = 0; - - do { - struct ice_tx_desc *next_dd_desc; - u16 desc_cnt = xdp_ring->count; - struct ice_tx_buf *tx_buf; - u32 xsk_frames; - u16 i; - - next_dd_desc = ICE_TX_DESC(xdp_ring, next_dd); - if (!(next_dd_desc->cmd_type_offset_bsz & - cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) - break; + u16 ntc = xdp_ring->next_to_clean; + struct ice_tx_desc *tx_desc; + u16 cnt = xdp_ring->count; + struct ice_tx_buf *tx_buf; + u16 xsk_frames = 0; + u16 last_rs; + int i; - cleared_dds++; - xsk_frames = 0; - if (likely(!xdp_ring->xdp_tx_active)) { - xsk_frames = tx_thresh; - goto skip; - } + last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1; + tx_desc = ICE_TX_DESC(xdp_ring, last_rs); + if ((tx_desc->cmd_type_offset_bsz & + cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) { + if (last_rs >= ntc) + xsk_frames = last_rs - ntc + 1; + else + xsk_frames = last_rs + cnt - ntc + 1; + } - ntc = xdp_ring->next_to_clean; + if (!xsk_frames) + return; - for (i = 0; i < tx_thresh; i++) { - tx_buf = &xdp_ring->tx_buf[ntc]; + if (likely(!xdp_ring->xdp_tx_active)) + goto skip; - if (tx_buf->raw_buf) { - ice_clean_xdp_tx_buf(xdp_ring, tx_buf); - tx_buf->raw_buf = NULL; - } else { - xsk_frames++; - } + ntc = xdp_ring->next_to_clean; + for (i = 0; i < xsk_frames; i++) { + tx_buf = &xdp_ring->tx_buf[ntc]; - ntc++; - if (ntc >= xdp_ring->count) - ntc = 0; + if (tx_buf->raw_buf) { + ice_clean_xdp_tx_buf(xdp_ring, tx_buf); + tx_buf->raw_buf = NULL; + } else { + xsk_frames++; } + + ntc++; + if (ntc >= xdp_ring->count) + ntc = 0; + } skip: - xdp_ring->next_to_clean += tx_thresh; - if (xdp_ring->next_to_clean >= desc_cnt) - xdp_ring->next_to_clean -= desc_cnt; - if (xsk_frames) - xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); - next_dd_desc->cmd_type_offset_bsz = 0; - next_dd = next_dd + tx_thresh; - if (next_dd >= desc_cnt) - next_dd = tx_thresh - 1; - } while (--budget); - - xdp_ring->next_dd = next_dd; - - return cleared_dds * tx_thresh; + tx_desc->cmd_type_offset_bsz = 0; + xdp_ring->next_to_clean += xsk_frames; + if (xdp_ring->next_to_clean >= cnt) + xdp_ring->next_to_clean -= cnt; + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); } /** @@ -816,7 +865,6 @@ static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc, static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, unsigned int *total_bytes) { - u16 tx_thresh = ICE_RING_QUARTER(xdp_ring); u16 ntu = xdp_ring->next_to_use; struct ice_tx_desc *tx_desc; u32 i; @@ -836,13 +884,6 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de } xdp_ring->next_to_use = ntu; - - if (xdp_ring->next_to_use > xdp_ring->next_rs) { - tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs); - tx_desc->cmd_type_offset_bsz |= - cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S); - xdp_ring->next_rs += tx_thresh; - } } /** @@ -855,7 +896,6 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs, u32 nb_pkts, unsigned int *total_bytes) { - u16 tx_thresh = ICE_RING_QUARTER(xdp_ring); u32 batched, leftover, i; batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH); @@ -864,54 +904,54 @@ static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *d ice_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes); for (; i < batched + leftover; i++) ice_xmit_pkt(xdp_ring, &descs[i], total_bytes); +} - if (xdp_ring->next_to_use > xdp_ring->next_rs) { - struct ice_tx_desc *tx_desc; +/** + * ice_set_rs_bit - set RS bit on last produced descriptor (one behind current NTU) + * @xdp_ring: XDP ring to produce the HW Tx descriptors on + */ +static void ice_set_rs_bit(struct ice_tx_ring *xdp_ring) +{ + u16 ntu = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : xdp_ring->count - 1; + struct ice_tx_desc *tx_desc; - tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs); - tx_desc->cmd_type_offset_bsz |= - cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S); - xdp_ring->next_rs += tx_thresh; - } + tx_desc = ICE_TX_DESC(xdp_ring, ntu); + tx_desc->cmd_type_offset_bsz |= + cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S); } /** * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring * @xdp_ring: XDP ring to produce the HW Tx descriptors on - * @budget: number of free descriptors on HW Tx ring that can be used - * @napi_budget: amount of descriptors that NAPI allows us to clean * * Returns true if there is no more work that needs to be done, false otherwise */ -bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, u32 budget, int napi_budget) +bool ice_xmit_zc(struct ice_tx_ring *xdp_ring) { struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs; - u16 tx_thresh = ICE_RING_QUARTER(xdp_ring); u32 nb_pkts, nb_processed = 0; unsigned int total_bytes = 0; + int budget; + + ice_clean_xdp_irq_zc(xdp_ring); - if (budget < tx_thresh) - budget += ice_clean_xdp_irq_zc(xdp_ring, napi_budget); + budget = ICE_DESC_UNUSED(xdp_ring); + budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring)); nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget); if (!nb_pkts) return true; if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) { - struct ice_tx_desc *tx_desc; - nb_processed = xdp_ring->count - xdp_ring->next_to_use; ice_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes); - tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs); - tx_desc->cmd_type_offset_bsz |= - cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S); - xdp_ring->next_rs = tx_thresh - 1; xdp_ring->next_to_use = 0; } ice_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed, &total_bytes); + ice_set_rs_bit(xdp_ring); ice_xdp_ring_update_tail(xdp_ring); ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes); @@ -944,13 +984,13 @@ ice_xsk_wakeup(struct net_device *netdev, u32 queue_id, if (!ice_is_xdp_ena_vsi(vsi)) return -EINVAL; - if (queue_id >= vsi->num_txq) + if (queue_id >= vsi->num_txq || queue_id >= vsi->num_rxq) return -EINVAL; - if (!vsi->xdp_rings[queue_id]->xsk_pool) - return -EINVAL; + ring = vsi->rx_rings[queue_id]->xdp_ring; - ring = vsi->xdp_rings[queue_id]; + if (!ring->xsk_pool) + return -EINVAL; /* The idea here is that if NAPI is running, mark a miss, so * it will run again. If not, trigger an interrupt and @@ -989,14 +1029,16 @@ bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi) */ void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) { - u16 count_mask = rx_ring->count - 1; u16 ntc = rx_ring->next_to_clean; u16 ntu = rx_ring->next_to_use; - for ( ; ntc != ntu; ntc = (ntc + 1) & count_mask) { + while (ntc != ntu) { struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc); xsk_buff_free(xdp); + ntc++; + if (ntc >= rx_ring->count) + ntc = 0; } } diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h index 21faec8e97db1c0dc1e90cedb1a651b0ffa3c812..6fa181f080ef139f94ae1faee9c3442c0e0371b9 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.h +++ b/drivers/net/ethernet/intel/ice/ice_xsk.h @@ -26,12 +26,10 @@ bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count); bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi); void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring); void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring); -bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, u32 budget, int napi_budget); +bool ice_xmit_zc(struct ice_tx_ring *xdp_ring); +int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc); #else -static inline bool -ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring, - u32 __always_unused budget, - int __always_unused napi_budget) +static inline bool ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring) { return false; } @@ -72,5 +70,12 @@ ice_xsk_wakeup(struct net_device __always_unused *netdev, static inline void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring) { } static inline void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring) { } + +static inline int +ice_realloc_zc_buf(struct ice_vsi __always_unused *vsi, + bool __always_unused zc) +{ + return 0; +} #endif /* CONFIG_XDP_SOCKETS */ #endif /* !_ICE_XSK_H_ */ diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index c14fc871dd417e8a03203d40aa3754279633e0a2..e5f3e7680dc66f3091d5fe08ebde6dfc2350b87b 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -850,14 +850,14 @@ static void igb_get_drvinfo(struct net_device *netdev, { struct igb_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, igb_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, igb_driver_name, sizeof(drvinfo->driver)); /* EEPROM image version # is reported as firmware version # for * 82575 controllers */ - strlcpy(drvinfo->fw_version, adapter->fw_version, + strscpy(drvinfo->fw_version, adapter->fw_version, sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); drvinfo->n_priv_flags = IGB_PRIV_FLAGS_STR_LEN; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 2796e81d27260ee9ac3b8bb2a5bb4564488b32d2..f8e32833226c1b5cb8558ae57ccf8201c2a4fb9b 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -1211,8 +1211,7 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter, return -ENOMEM; /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, - igb_poll, 64); + netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; @@ -3138,7 +3137,7 @@ static s32 igb_init_i2c(struct igb_adapter *adapter) adapter->i2c_algo.data = adapter; adapter->i2c_adap.algo_data = &adapter->i2c_algo; adapter->i2c_adap.dev.parent = &adapter->pdev->dev; - strlcpy(adapter->i2c_adap.name, "igb BB", + strscpy(adapter->i2c_adap.name, "igb BB", sizeof(adapter->i2c_adap.name)); status = i2c_bit_add_bus(&adapter->i2c_adap); return status; diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c index 9d4322b74163680cfd48f267c24b17a601eb13c9..83b97989a6bdc74f8cd46edd930c44ff3abac77b 100644 --- a/drivers/net/ethernet/intel/igbvf/ethtool.c +++ b/drivers/net/ethernet/intel/igbvf/ethtool.c @@ -169,8 +169,8 @@ static void igbvf_get_drvinfo(struct net_device *netdev, { struct igbvf_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, igbvf_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, igbvf_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c index f4e91db89fe5aeb05f7d5ff68c3eaf50939f5e6f..3a32809510fc6b771c2a350e5bed9aa3a5bb1a48 100644 --- a/drivers/net/ethernet/intel/igbvf/netdev.c +++ b/drivers/net/ethernet/intel/igbvf/netdev.c @@ -1109,7 +1109,7 @@ static int igbvf_alloc_queues(struct igbvf_adapter *adapter) return -ENOMEM; } - netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll, 64); + netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll); return 0; } diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 5c66b97c0cfacca6aa7782a8fddc187271ea32e3..4f9d7f013a9581b66668af8a8e7a54920b3326c5 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -610,7 +610,6 @@ #define IGC_MDIC_OP_WRITE 0x04000000 #define IGC_MDIC_OP_READ 0x08000000 #define IGC_MDIC_READY 0x10000000 -#define IGC_MDIC_INT_EN 0x20000000 #define IGC_MDIC_ERROR 0x40000000 #define IGC_N0_QUEUE -1 diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index ebff0e04045d639f0f690340efcd0f63519153f6..34889be63e788f8addc2e4704857c241e8e96f3d 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2129,65 +2129,102 @@ static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count) return ok; } -static int igc_xdp_init_tx_buffer(struct igc_tx_buffer *buffer, - struct xdp_frame *xdpf, - struct igc_ring *ring) -{ - dma_addr_t dma; - - dma = dma_map_single(ring->dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) { - netdev_err_once(ring->netdev, "Failed to map DMA for TX\n"); - return -ENOMEM; - } - - buffer->type = IGC_TX_BUFFER_TYPE_XDP; - buffer->xdpf = xdpf; - buffer->protocol = 0; - buffer->bytecount = xdpf->len; - buffer->gso_segs = 1; - buffer->time_stamp = jiffies; - dma_unmap_len_set(buffer, len, xdpf->len); - dma_unmap_addr_set(buffer, dma, dma); - return 0; -} - /* This function requires __netif_tx_lock is held by the caller. */ static int igc_xdp_init_tx_descriptor(struct igc_ring *ring, struct xdp_frame *xdpf) { - struct igc_tx_buffer *buffer; - union igc_adv_tx_desc *desc; - u32 cmd_type, olinfo_status; - int err; + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); + u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; + u16 count, index = ring->next_to_use; + struct igc_tx_buffer *head = &ring->tx_buffer_info[index]; + struct igc_tx_buffer *buffer = head; + union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index); + u32 olinfo_status, len = xdpf->len, cmd_type; + void *data = xdpf->data; + u16 i; - if (!igc_desc_unused(ring)) - return -EBUSY; + count = TXD_USE_COUNT(len); + for (i = 0; i < nr_frags; i++) + count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i])); - buffer = &ring->tx_buffer_info[ring->next_to_use]; - err = igc_xdp_init_tx_buffer(buffer, xdpf, ring); - if (err) - return err; + if (igc_maybe_stop_tx(ring, count + 3)) { + /* this is a hard error */ + return -EBUSY; + } - cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | - IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD | - buffer->bytecount; - olinfo_status = buffer->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; + i = 0; + head->bytecount = xdp_get_frame_len(xdpf); + head->type = IGC_TX_BUFFER_TYPE_XDP; + head->gso_segs = 1; + head->xdpf = xdpf; - desc = IGC_TX_DESC(ring, ring->next_to_use); - desc->read.cmd_type_len = cpu_to_le32(cmd_type); + olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT; desc->read.olinfo_status = cpu_to_le32(olinfo_status); - desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(buffer, dma)); - netdev_tx_sent_queue(txring_txq(ring), buffer->bytecount); + for (;;) { + dma_addr_t dma; - buffer->next_to_watch = desc; + dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); + if (dma_mapping_error(ring->dev, dma)) { + netdev_err_once(ring->netdev, + "Failed to map DMA for TX\n"); + goto unmap; + } - ring->next_to_use++; - if (ring->next_to_use == ring->count) - ring->next_to_use = 0; + dma_unmap_len_set(buffer, len, len); + dma_unmap_addr_set(buffer, dma, dma); + + cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT | + IGC_ADVTXD_DCMD_IFCS | len; + + desc->read.cmd_type_len = cpu_to_le32(cmd_type); + desc->read.buffer_addr = cpu_to_le64(dma); + + buffer->protocol = 0; + + if (++index == ring->count) + index = 0; + + if (i == nr_frags) + break; + + buffer = &ring->tx_buffer_info[index]; + desc = IGC_TX_DESC(ring, index); + desc->read.olinfo_status = 0; + + data = skb_frag_address(&sinfo->frags[i]); + len = skb_frag_size(&sinfo->frags[i]); + i++; + } + desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD); + + netdev_tx_sent_queue(txring_txq(ring), head->bytecount); + /* set the timestamp */ + head->time_stamp = jiffies; + /* set next_to_watch value indicating a packet is present */ + head->next_to_watch = desc; + ring->next_to_use = index; return 0; + +unmap: + for (;;) { + buffer = &ring->tx_buffer_info[index]; + if (dma_unmap_len(buffer, len)) + dma_unmap_page(ring->dev, + dma_unmap_addr(buffer, dma), + dma_unmap_len(buffer, len), + DMA_TO_DEVICE); + dma_unmap_len_set(buffer, len, 0); + if (buffer == head) + break; + + if (!index) + index += ring->count; + index--; + } + + return -ENOMEM; } static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter, @@ -2369,6 +2406,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring), igc_rx_offset(rx_ring) + pkt_offset, size, true); + xdp_buff_clear_frags_flag(&xdp); skb = igc_xdp_run_prog(adapter, &xdp); } @@ -4356,8 +4394,7 @@ static int igc_alloc_q_vector(struct igc_adapter *adapter, return -ENOMEM; /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, - igc_poll, 64); + netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c index 46efcfab7234aeb3568d87c87ff3f96487444a94..efa980514944233175e013889adda8cab473d0ec 100644 --- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c +++ b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c @@ -456,9 +456,9 @@ ixgb_get_drvinfo(struct net_device *netdev, { struct ixgb_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, ixgb_driver_name, + strscpy(drvinfo->driver, ixgb_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_main.c b/drivers/net/ethernet/intel/ixgb/ixgb_main.c index 45be9a1ab6af2fede53f9c97a01c230bda52d6ed..b4d47e7a76c8cf91add4adbc8f0bce54f68930ec 100644 --- a/drivers/net/ethernet/intel/ixgb/ixgb_main.c +++ b/drivers/net/ethernet/intel/ixgb/ixgb_main.c @@ -414,7 +414,7 @@ ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->netdev_ops = &ixgb_netdev_ops; ixgb_set_ethtool_ops(netdev); netdev->watchdog_timeo = 5 * HZ; - netif_napi_add(netdev, &adapter->napi, ixgb_clean, 64); + netif_napi_add(netdev, &adapter->napi, ixgb_clean); strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 04f453eabef64e91786bd0dbc497512777d81538..e88e3dfac8c21124b8c6a83b69c3c8298392c222 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1106,12 +1106,12 @@ static void ixgbe_get_drvinfo(struct net_device *netdev, { struct ixgbe_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, adapter->eeprom_id, + strscpy(drvinfo->fw_version, adapter->eeprom_id, sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); drvinfo->n_priv_flags = IXGBE_PRIV_FLAGS_STR_LEN; @@ -1964,15 +1964,13 @@ static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer, frame_size >>= 1; - data = kmap(rx_buffer->page) + rx_buffer->page_offset; + data = page_address(rx_buffer->page) + rx_buffer->page_offset; if (data[3] != 0xFF || data[frame_size + 10] != 0xBE || data[frame_size + 12] != 0xAF) match = false; - kunmap(rx_buffer->page); - return match; } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c index 0fcd82036d4e3fc110dccfad0c23c11430388d91..7311bd545acf3eee50b1929fb1698ab05bf8882f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c @@ -1004,7 +1004,7 @@ int ixgbe_fcoe_get_hbainfo(struct net_device *netdev, ixgbe_driver_name, UTS_RELEASE); /* Firmware Version */ - strlcpy(info->firmware_version, adapter->eeprom_id, + strscpy(info->firmware_version, adapter->eeprom_id, sizeof(info->firmware_version)); /* Model */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index 86b11164655e2112c9b885c8e8d34ace093fda9f..f8156fe4b1dc41c56861cb1d670c73a1d70d5bd0 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -874,8 +874,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, #endif /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, - ixgbe_poll, 64); + netif_napi_add(adapter->netdev, &q_vector->napi, ixgbe_poll); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index d1e430b8c8aa13659f2937c0941f761396256871..298cfbfcb7b6fdd35435ac157c8e43965b50a99f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -10849,7 +10849,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->netdev_ops = &ixgbe_netdev_ops; ixgbe_set_ethtool_ops(netdev); netdev->watchdog_timeo = 5 * HZ; - strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); + strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name)); /* Setup hw api */ hw->mac.ops = *ii->mac_ops; @@ -11140,7 +11140,7 @@ skip_sriov: err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str)); if (err) - strlcpy(part_str, "Unknown", sizeof(part_str)); + strscpy(part_str, "Unknown", sizeof(part_str)); if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present) e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n", hw->mac.type, hw->phy.type, hw->phy.sfp_type, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c index 9f06896a049b40470dd91e9bd3ab28710d6b1837..f8605f57bd067d64f44ee028c80f1606107f019f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c @@ -1214,7 +1214,6 @@ void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter) struct cyclecounter cc; unsigned long flags; u32 incval = 0; - u32 tsauxc = 0; u32 fuse0 = 0; /* For some of the boards below this mask is technically incorrect. @@ -1249,18 +1248,6 @@ void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter) case ixgbe_mac_x550em_a: case ixgbe_mac_X550: cc.read = ixgbe_ptp_read_X550; - - /* enable SYSTIME counter */ - IXGBE_WRITE_REG(hw, IXGBE_SYSTIMR, 0); - IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0); - IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0); - tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); - IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, - tsauxc & ~IXGBE_TSAUXC_DISABLE_SYSTIME); - IXGBE_WRITE_REG(hw, IXGBE_TSIM, IXGBE_TSIM_TXTS); - IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_TIMESYNC); - - IXGBE_WRITE_FLUSH(hw); break; case ixgbe_mac_X540: cc.read = ixgbe_ptp_read_82599; @@ -1292,6 +1279,50 @@ void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter) spin_unlock_irqrestore(&adapter->tmreg_lock, flags); } +/** + * ixgbe_ptp_init_systime - Initialize SYSTIME registers + * @adapter: the ixgbe private board structure + * + * Initialize and start the SYSTIME registers. + */ +static void ixgbe_ptp_init_systime(struct ixgbe_adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + u32 tsauxc; + + switch (hw->mac.type) { + case ixgbe_mac_X550EM_x: + case ixgbe_mac_x550em_a: + case ixgbe_mac_X550: + tsauxc = IXGBE_READ_REG(hw, IXGBE_TSAUXC); + + /* Reset SYSTIME registers to 0 */ + IXGBE_WRITE_REG(hw, IXGBE_SYSTIMR, 0); + IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0); + IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0); + + /* Reset interrupt settings */ + IXGBE_WRITE_REG(hw, IXGBE_TSIM, IXGBE_TSIM_TXTS); + IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_TIMESYNC); + + /* Activate the SYSTIME counter */ + IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, + tsauxc & ~IXGBE_TSAUXC_DISABLE_SYSTIME); + break; + case ixgbe_mac_X540: + case ixgbe_mac_82599EB: + /* Reset SYSTIME registers to 0 */ + IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0); + IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0); + break; + default: + /* Other devices aren't supported */ + return; + }; + + IXGBE_WRITE_FLUSH(hw); +} + /** * ixgbe_ptp_reset * @adapter: the ixgbe private board structure @@ -1318,6 +1349,8 @@ void ixgbe_ptp_reset(struct ixgbe_adapter *adapter) ixgbe_ptp_start_cyclecounter(adapter); + ixgbe_ptp_init_systime(adapter); + spin_lock_irqsave(&adapter->tmreg_lock, flags); timecounter_init(&adapter->hw_tc, &adapter->hw_cc, ktime_to_ns(ktime_get_real())); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index 7f7ea468ffa91609c5cc7f964c20f2727a0f89ee..2b00db92b08f51ded3e4f074531f1c7b990d8a73 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -3712,7 +3712,9 @@ struct ixgbe_info { #define IXGBE_KRM_LINK_S1(P) ((P) ? 0x8200 : 0x4200) #define IXGBE_KRM_LINK_CTRL_1(P) ((P) ? 0x820C : 0x420C) #define IXGBE_KRM_AN_CNTL_1(P) ((P) ? 0x822C : 0x422C) +#define IXGBE_KRM_AN_CNTL_4(P) ((P) ? 0x8238 : 0x4238) #define IXGBE_KRM_AN_CNTL_8(P) ((P) ? 0x8248 : 0x4248) +#define IXGBE_KRM_PCS_KX_AN(P) ((P) ? 0x9918 : 0x5918) #define IXGBE_KRM_SGMII_CTRL(P) ((P) ? 0x82A0 : 0x42A0) #define IXGBE_KRM_LP_BASE_PAGE_HIGH(P) ((P) ? 0x836C : 0x436C) #define IXGBE_KRM_DSP_TXFFE_STATE_4(P) ((P) ? 0x8634 : 0x4634) @@ -3722,6 +3724,7 @@ struct ixgbe_info { #define IXGBE_KRM_PMD_FLX_MASK_ST20(P) ((P) ? 0x9054 : 0x5054) #define IXGBE_KRM_TX_COEFF_CTRL_1(P) ((P) ? 0x9520 : 0x5520) #define IXGBE_KRM_RX_ANA_CTL(P) ((P) ? 0x9A00 : 0x5A00) +#define IXGBE_KRM_FLX_TMRS_CTRL_ST31(P) ((P) ? 0x9180 : 0x5180) #define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_DA ~(0x3 << 20) #define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_SR BIT(20) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c index 35c2b9b8bd1910e5505e311ba6fc9cd13f779a77..aa4bf6c9a2f7cd933f096b988c8086d2808b55b9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c @@ -1721,9 +1721,59 @@ static s32 ixgbe_setup_sfi_x550a(struct ixgbe_hw *hw, ixgbe_link_speed *speed) return IXGBE_ERR_LINK_SETUP; } - status = mac->ops.write_iosf_sb_reg(hw, - IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), - IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* change mode enforcement rules to hybrid */ + (void)mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_FLX_TMRS_CTRL_ST31(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + reg_val |= 0x0400; + + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_FLX_TMRS_CTRL_ST31(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* manually control the config */ + (void)mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + reg_val |= 0x20002240; + + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* move the AN base page values */ + (void)mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_PCS_KX_AN(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + reg_val |= 0x1; + + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_PCS_KX_AN(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* set the AN37 over CB mode */ + (void)mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_AN_CNTL_4(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + reg_val |= 0x20000000; + + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_AN_CNTL_4(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); + + /* restart AN manually */ + (void)mac->ops.read_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, ®_val); + reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART; + + (void)mac->ops.write_iosf_sb_reg(hw, + IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id), + IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val); /* Toggle port SW reset by AN reset. */ status = ixgbe_restart_an_internal_phy_x550em(hw); diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index fed46872af2bf8a495469c4e80c111f6889d1fda..ccfa6b91aac631487886a2fca0f4d67bf7b2b7e1 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -213,8 +213,8 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev, { struct ixgbevf_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); drvinfo->n_priv_flags = IXGBEVF_PRIV_FLAGS_STR_LEN; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 2f12fbe229c1588452d7b9fee5bd36eb2c23332a..99933e89717ad9a936bd742e22a8493bcf33fb1f 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -2733,7 +2733,7 @@ static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx, return -ENOMEM; /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64); + netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll); /* tie q_vector and adapter together */ adapter->q_vector[v_idx] = q_vector; diff --git a/drivers/net/ethernet/jme.c b/drivers/net/ethernet/jme.c index f43d6616bc0dd5f96e775989d8780618570aaee7..1732ec3c3dbdc4767a6e48296f9219f588f9e643 100644 --- a/drivers/net/ethernet/jme.c +++ b/drivers/net/ethernet/jme.c @@ -2332,9 +2332,9 @@ jme_get_drvinfo(struct net_device *netdev, { struct jme_adapter *jme = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(jme->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(jme->pdev), sizeof(info->bus_info)); } static int @@ -3009,7 +3009,7 @@ jme_init_one(struct pci_dev *pdev, jwrite32(jme, JME_APMC, apmc); } - netif_napi_add(netdev, &jme->napi, jme_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &jme->napi, jme_poll); spin_lock_init(&jme->phy_lock); spin_lock_init(&jme->macaddr_lock); diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c index df9a8eefa007a36c6348acc76a2a73dbb4c70409..2b9335cb4bb3a1896643268bb1704d3091d08442 100644 --- a/drivers/net/ethernet/korina.c +++ b/drivers/net/ethernet/korina.c @@ -416,7 +416,8 @@ static void korina_abort_rx(struct net_device *dev) } /* transmit packet */ -static int korina_send_packet(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t korina_send_packet(struct sk_buff *skb, + struct net_device *dev) { struct korina_private *lp = netdev_priv(dev); u32 chain_prev, chain_next; @@ -938,9 +939,9 @@ static void netdev_get_drvinfo(struct net_device *dev, { struct korina_private *lp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, lp->dev->name, sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, lp->dev->name, sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, @@ -1354,7 +1355,7 @@ static int korina_probe(struct platform_device *pdev) dev->netdev_ops = &korina_netdev_ops; dev->ethtool_ops = &netdev_ethtool_ops; dev->watchdog_timeo = TX_TIMEOUT; - netif_napi_add(dev, &lp->napi, korina_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &lp->napi, korina_poll); lp->mii_if.dev = dev; lp->mii_if.mdio_read = korina_mdio_read; diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c index 7cedbe1fdfd7721594bcf69f13e7e381ed63eb4b..59aab4086dcce3d1426438bd0683fc9ac26f14c3 100644 --- a/drivers/net/ethernet/lantiq_etop.c +++ b/drivers/net/ethernet/lantiq_etop.c @@ -470,7 +470,7 @@ ltq_etop_stop(struct net_device *dev) return 0; } -static int +static netdev_tx_t ltq_etop_tx(struct sk_buff *skb, struct net_device *dev) { int queue = skb_get_queue_mapping(skb); diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c index 5edb68a8aab1ececb61a652ca6163a4302beabd7..8d646c7f8c823670ace5fedb85cbd25076e51a7e 100644 --- a/drivers/net/ethernet/lantiq_xrx200.c +++ b/drivers/net/ethernet/lantiq_xrx200.c @@ -193,6 +193,7 @@ static int xrx200_alloc_buf(struct xrx200_chan *ch, void *(*alloc)(unsigned int ch->rx_buff[ch->dma.desc] = alloc(priv->rx_skb_size); if (!ch->rx_buff[ch->dma.desc]) { + ch->rx_buff[ch->dma.desc] = buf; ret = -ENOMEM; goto skip; } @@ -239,6 +240,12 @@ static int xrx200_hw_receive(struct xrx200_chan *ch) } skb = build_skb(buf, priv->rx_skb_size); + if (!skb) { + skb_free_frag(buf); + net_dev->stats.rx_dropped++; + return -ENOMEM; + } + skb_reserve(skb, NET_SKB_PAD); skb_put(skb, len); @@ -288,7 +295,7 @@ static int xrx200_poll_rx(struct napi_struct *napi, int budget) if (ret == XRX200_DMA_PACKET_IN_PROGRESS) continue; if (ret != XRX200_DMA_PACKET_COMPLETE) - return ret; + break; rx++; } else { break; @@ -613,8 +620,7 @@ static int xrx200_probe(struct platform_device *pdev) PMAC_HD_CTL); /* setup NAPI */ - netif_napi_add(net_dev, &priv->chan_rx.napi, xrx200_poll_rx, - NAPI_POLL_WEIGHT); + netif_napi_add(net_dev, &priv->chan_rx.napi, xrx200_poll_rx); netif_napi_add_tx(net_dev, &priv->chan_tx.napi, xrx200_tx_housekeeping); diff --git a/drivers/net/ethernet/litex/litex_liteeth.c b/drivers/net/ethernet/litex/litex_liteeth.c index fdd99f0de424dce95aba61410afcca7a8aa30838..35f24e0f0934932fd8feaa53223dbbb5c081f2ee 100644 --- a/drivers/net/ethernet/litex/litex_liteeth.c +++ b/drivers/net/ethernet/litex/litex_liteeth.c @@ -152,7 +152,8 @@ static int liteeth_stop(struct net_device *netdev) return 0; } -static int liteeth_start_xmit(struct sk_buff *skb, struct net_device *netdev) +static netdev_tx_t liteeth_start_xmit(struct sk_buff *skb, + struct net_device *netdev) { struct liteeth *priv = netdev_priv(netdev); void __iomem *txbuffer; diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index b6be0552a6c1d18247ab998f52b13fbc6f717e08..707993b445d1e2ff0baea4d7943949364a56fbc0 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -1603,12 +1603,12 @@ mv643xx_eth_set_link_ksettings(struct net_device *dev, static void mv643xx_eth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, mv643xx_eth_driver_name, + strscpy(drvinfo->driver, mv643xx_eth_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mv643xx_eth_driver_version, + strscpy(drvinfo->version, mv643xx_eth_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); } static int mv643xx_eth_get_coalesce(struct net_device *dev, @@ -3183,7 +3183,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) INIT_WORK(&mp->tx_timeout_task, tx_timeout_task); - netif_napi_add(dev, &mp->napi, mv643xx_eth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &mp->napi, mv643xx_eth_poll); timer_setup(&mp->rx_oom, oom_timer_wrapper, 0); diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 0caa2df87c044f5fefa8e6d11ce9b1cbf1a7f36d..ff3e361e06e781b413d20e1b9b215643e7a15e40 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -4656,11 +4656,11 @@ mvneta_ethtool_get_coalesce(struct net_device *dev, static void mvneta_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, + strscpy(drvinfo->driver, MVNETA_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, + strscpy(drvinfo->version, MVNETA_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } @@ -5600,14 +5600,13 @@ static int mvneta_probe(struct platform_device *pdev) * operation, so only single NAPI should be initialized. */ if (pp->neta_armada3700) { - netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &pp->napi, mvneta_poll); } else { for_each_present_cpu(cpu) { struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); - netif_napi_add(dev, &port->napi, mvneta_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &port->napi, mvneta_poll); port->pp = pp; } } diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h index ad73a488fc5fb6de4ddbf980355e31944b980e08..11e603686a276661ebbf6e3c1ea7545253213eab 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h @@ -1530,6 +1530,7 @@ u32 mvpp2_read(struct mvpp2 *priv, u32 offset); void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name); void mvpp2_dbgfs_cleanup(struct mvpp2 *priv); +void mvpp2_dbgfs_exit(void); void mvpp23_rx_fifo_fc_en(struct mvpp2 *priv, int port, bool en); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c index 4a3baa7e0142416d7ca15d9677362c9c29e5b986..75e83ea2a926e9ab57cd52aeb5f13da06ed87fd0 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c @@ -691,6 +691,13 @@ static int mvpp2_dbgfs_port_init(struct dentry *parent, return 0; } +static struct dentry *mvpp2_root; + +void mvpp2_dbgfs_exit(void) +{ + debugfs_remove(mvpp2_root); +} + void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) { debugfs_remove_recursive(priv->dbgfs_dir); @@ -700,10 +707,9 @@ void mvpp2_dbgfs_cleanup(struct mvpp2 *priv) void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name) { - struct dentry *mvpp2_dir, *mvpp2_root; + struct dentry *mvpp2_dir; int ret, i; - mvpp2_root = debugfs_lookup(MVPP2_DRIVER_NAME, NULL); if (!mvpp2_root) mvpp2_root = debugfs_create_dir(MVPP2_DRIVER_NAME, NULL); diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index b84128b549b44ac464aea2d004623f2de0c41165..eb0fb812809680015575a19c8bd265f6224d7a15 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -5425,11 +5425,11 @@ mvpp2_ethtool_get_coalesce(struct net_device *dev, static void mvpp2_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME, + strscpy(drvinfo->driver, MVPP2_DRIVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION, + strscpy(drvinfo->version, MVPP2_DRIVER_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(&dev->dev), + strscpy(drvinfo->bus_info, dev_name(&dev->dev), sizeof(drvinfo->bus_info)); } @@ -5770,8 +5770,7 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port, v->irq = irq_of_parse_and_map(port_node, 0); if (v->irq <= 0) return -EINVAL; - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); port->nqvecs = 1; @@ -5831,8 +5830,7 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port, goto err; } - netif_napi_add(port->dev, &v->napi, mvpp2_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(port->dev, &v->napi, mvpp2_poll); } return 0; @@ -7706,7 +7704,18 @@ static struct platform_driver mvpp2_driver = { }, }; -module_platform_driver(mvpp2_driver); +static int __init mvpp2_driver_init(void) +{ + return platform_driver_register(&mvpp2_driver); +} +module_init(mvpp2_driver_init); + +static void __exit mvpp2_driver_exit(void) +{ + platform_driver_unregister(&mvpp2_driver); + mvpp2_dbgfs_exit(); +} +module_exit(mvpp2_driver_exit); MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com"); MODULE_AUTHOR("Marcin Wojtas "); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c index 97f080c66dd478814bf047cfe0dede30df5b8e09..9089adcb75f92be857fae13006437d7c27035312 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c @@ -410,7 +410,7 @@ static void octep_napi_add(struct octep_device *oct) for (i = 0; i < oct->num_oqs; i++) { netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i); netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, - octep_napi_poll, 64); + octep_napi_poll); oct->oq[i]->napi = &oct->ioq_vector[i]->napi; } } diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index d9ae0937d17a8232ccb4083a796f1028d53015aa..392d9b0da0d7a8503f3cd9a677c3ddfdb2644e8a 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -158,8 +158,7 @@ static int octep_setup_oq(struct octep_device *oct, int q_no) goto desc_dma_alloc_err; } - oq->buff_info = (struct octep_rx_buffer *) - vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE); + oq->buff_info = vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE); if (unlikely(!oq->buff_info)) { dev_err(&oct->pdev->dev, "Failed to allocate buffer info for OQ-%d\n", q_no); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile index 40203560b29195247b0b22f2590e22821120626a..3cf4c8285c90729b8c0f16a256cfe7b527f5b89c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile @@ -11,4 +11,4 @@ rvu_mbox-y := mbox.o rvu_trace.o rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \ rvu_reg.o rvu_npc.o rvu_debugfs.o ptp.o rvu_npc_fs.o \ rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \ - rvu_sdp.o rvu_npc_hash.o + rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index d7762577e285820720facb011afb05da3d21cf17..8d5d5a0f68c445fce1c80854955f1a50ed5310fc 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -293,20 +293,74 @@ M(NIX_BANDPROF_ALLOC, 0x801d, nix_bandprof_alloc, nix_bandprof_alloc_req, \ M(NIX_BANDPROF_FREE, 0x801e, nix_bandprof_free, nix_bandprof_free_req, \ msg_rsp) \ M(NIX_BANDPROF_GET_HWINFO, 0x801f, nix_bandprof_get_hwinfo, msg_req, \ - nix_bandprof_get_hwinfo_rsp) - -/* Messages initiated by AF (range 0xC00 - 0xDFF) */ + nix_bandprof_get_hwinfo_rsp) \ +/* MCS mbox IDs (range 0xA000 - 0xBFFF) */ \ +M(MCS_ALLOC_RESOURCES, 0xa000, mcs_alloc_resources, mcs_alloc_rsrc_req, \ + mcs_alloc_rsrc_rsp) \ +M(MCS_FREE_RESOURCES, 0xa001, mcs_free_resources, mcs_free_rsrc_req, msg_rsp) \ +M(MCS_FLOWID_ENTRY_WRITE, 0xa002, mcs_flowid_entry_write, mcs_flowid_entry_write_req, \ + msg_rsp) \ +M(MCS_SECY_PLCY_WRITE, 0xa003, mcs_secy_plcy_write, mcs_secy_plcy_write_req, \ + msg_rsp) \ +M(MCS_RX_SC_CAM_WRITE, 0xa004, mcs_rx_sc_cam_write, mcs_rx_sc_cam_write_req, \ + msg_rsp) \ +M(MCS_SA_PLCY_WRITE, 0xa005, mcs_sa_plcy_write, mcs_sa_plcy_write_req, \ + msg_rsp) \ +M(MCS_TX_SC_SA_MAP_WRITE, 0xa006, mcs_tx_sc_sa_map_write, mcs_tx_sc_sa_map, \ + msg_rsp) \ +M(MCS_RX_SC_SA_MAP_WRITE, 0xa007, mcs_rx_sc_sa_map_write, mcs_rx_sc_sa_map, \ + msg_rsp) \ +M(MCS_FLOWID_ENA_ENTRY, 0xa008, mcs_flowid_ena_entry, mcs_flowid_ena_dis_entry, \ + msg_rsp) \ +M(MCS_PN_TABLE_WRITE, 0xa009, mcs_pn_table_write, mcs_pn_table_write_req, \ + msg_rsp) \ +M(MCS_SET_ACTIVE_LMAC, 0xa00a, mcs_set_active_lmac, mcs_set_active_lmac, \ + msg_rsp) \ +M(MCS_GET_HW_INFO, 0xa00b, mcs_get_hw_info, msg_req, mcs_hw_info) \ +M(MCS_GET_FLOWID_STATS, 0xa00c, mcs_get_flowid_stats, mcs_stats_req, \ + mcs_flowid_stats) \ +M(MCS_GET_SECY_STATS, 0xa00d, mcs_get_secy_stats, mcs_stats_req, \ + mcs_secy_stats) \ +M(MCS_GET_SC_STATS, 0xa00e, mcs_get_sc_stats, mcs_stats_req, mcs_sc_stats) \ +M(MCS_GET_SA_STATS, 0xa00f, mcs_get_sa_stats, mcs_stats_req, mcs_sa_stats) \ +M(MCS_GET_PORT_STATS, 0xa010, mcs_get_port_stats, mcs_stats_req, \ + mcs_port_stats) \ +M(MCS_CLEAR_STATS, 0xa011, mcs_clear_stats, mcs_clear_stats, msg_rsp) \ +M(MCS_INTR_CFG, 0xa012, mcs_intr_cfg, mcs_intr_cfg, msg_rsp) \ +M(MCS_SET_LMAC_MODE, 0xa013, mcs_set_lmac_mode, mcs_set_lmac_mode, msg_rsp) \ +M(MCS_SET_PN_THRESHOLD, 0xa014, mcs_set_pn_threshold, mcs_set_pn_threshold, \ + msg_rsp) \ +M(MCS_ALLOC_CTRL_PKT_RULE, 0xa015, mcs_alloc_ctrl_pkt_rule, \ + mcs_alloc_ctrl_pkt_rule_req, \ + mcs_alloc_ctrl_pkt_rule_rsp) \ +M(MCS_FREE_CTRL_PKT_RULE, 0xa016, mcs_free_ctrl_pkt_rule, \ + mcs_free_ctrl_pkt_rule_req, msg_rsp) \ +M(MCS_CTRL_PKT_RULE_WRITE, 0xa017, mcs_ctrl_pkt_rule_write, \ + mcs_ctrl_pkt_rule_write_req, msg_rsp) \ +M(MCS_PORT_RESET, 0xa018, mcs_port_reset, mcs_port_reset_req, msg_rsp) \ +M(MCS_PORT_CFG_SET, 0xa019, mcs_port_cfg_set, mcs_port_cfg_set_req, msg_rsp)\ +M(MCS_PORT_CFG_GET, 0xa020, mcs_port_cfg_get, mcs_port_cfg_get_req, \ + mcs_port_cfg_get_rsp) \ +M(MCS_CUSTOM_TAG_CFG_GET, 0xa021, mcs_custom_tag_cfg_get, \ + mcs_custom_tag_cfg_get_req, \ + mcs_custom_tag_cfg_get_rsp) + +/* Messages initiated by AF (range 0xC00 - 0xEFF) */ #define MBOX_UP_CGX_MESSAGES \ M(CGX_LINK_EVENT, 0xC00, cgx_link_event, cgx_link_info_msg, msg_rsp) #define MBOX_UP_CPT_MESSAGES \ M(CPT_INST_LMTST, 0xD00, cpt_inst_lmtst, cpt_inst_lmtst_req, msg_rsp) +#define MBOX_UP_MCS_MESSAGES \ +M(MCS_INTR_NOTIFY, 0xE00, mcs_intr_notify, mcs_intr_info, msg_rsp) + enum { #define M(_name, _id, _1, _2, _3) MBOX_MSG_ ## _name = _id, MBOX_MESSAGES MBOX_UP_CGX_MESSAGES MBOX_UP_CPT_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M }; @@ -1471,6 +1525,7 @@ enum ptp_op { PTP_OP_GET_CLOCK = 1, PTP_OP_GET_TSTMP = 2, PTP_OP_SET_THRESH = 3, + PTP_OP_EXTTS_ON = 4, }; struct ptp_req { @@ -1478,6 +1533,7 @@ struct ptp_req { u8 op; s64 scaled_ppm; u64 thresh; + int extts_on; }; struct ptp_rsp { @@ -1655,4 +1711,415 @@ enum cgx_af_status { LMAC_AF_ERR_EXACT_MATCH_TBL_LOOK_UP_FAILED = -1110, }; +enum mcs_direction { + MCS_RX, + MCS_TX, +}; + +enum mcs_rsrc_type { + MCS_RSRC_TYPE_FLOWID, + MCS_RSRC_TYPE_SECY, + MCS_RSRC_TYPE_SC, + MCS_RSRC_TYPE_SA, +}; + +struct mcs_alloc_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_type; + u8 rsrc_cnt; /* Resources count */ + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u8 all; /* Allocate all resource type one each */ + u64 rsvd; +}; + +struct mcs_alloc_rsrc_rsp { + struct mbox_msghdr hdr; + u8 flow_ids[128]; /* Index of reserved entries */ + u8 secy_ids[128]; + u8 sc_ids[128]; + u8 sa_ids[256]; + u8 rsrc_type; + u8 rsrc_cnt; /* No of entries reserved */ + u8 mcs_id; + u8 dir; + u8 all; + u8 rsvd[256]; /* reserved fields for future expansion */ +}; + +struct mcs_free_rsrc_req { + struct mbox_msghdr hdr; + u8 rsrc_id; /* Index of the entry to be freed */ + u8 rsrc_type; + u8 mcs_id; + u8 dir; + u8 all; /* Free all the cam resources */ + u64 rsvd; +}; + +struct mcs_flowid_entry_write_req { + struct mbox_msghdr hdr; + u64 data[4]; + u64 mask[4]; + u64 sci; /* CNF10K-B for tx_secy_mem_map */ + u8 flow_id; + u8 secy_id; /* secyid for which flowid is mapped */ + u8 sc_id; /* Valid if dir = MCS_TX, SC_CAM id mapped to flowid */ + u8 ena; /* Enable tcam entry */ + u8 ctrl_pkt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_secy_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy; + u8 secy_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* RX SC_CAM mapping */ +struct mcs_rx_sc_cam_write_req { + struct mbox_msghdr hdr; + u64 sci; /* SCI */ + u64 secy_id; /* secy index mapped to SC */ + u8 sc_id; /* SC CAM entry index */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_sa_plcy_write_req { + struct mbox_msghdr hdr; + u64 plcy[2][9]; /* Support 2 SA policy */ + u8 sa_index[2]; + u8 sa_cnt; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_tx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index0; + u8 sa_index1; + u8 rekey_ena; + u8 sa_index0_vld; + u8 sa_index1_vld; + u8 tx_sa_active; + u64 sectag_sci; + u8 sc_id; /* used as index for SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_rx_sc_sa_map { + struct mbox_msghdr hdr; + u8 sa_index; + u8 sa_in_use; + u8 sc_id; + u8 an; /* value range 0-3, sc_id + an used as index SA_MEM_MAP */ + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_flowid_ena_dis_entry { + struct mbox_msghdr hdr; + u8 flow_id; + u8 ena; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_pn_table_write_req { + struct mbox_msghdr hdr; + u64 next_pn; + u8 pn_id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_hw_info { + struct mbox_msghdr hdr; + u8 num_mcs_blks; /* Number of MCS blocks */ + u8 tcam_entries; /* RX/TX Tcam entries per mcs block */ + u8 secy_entries; /* RX/TX SECY entries per mcs block */ + u8 sc_entries; /* RX/TX SC CAM entries per mcs block */ + u8 sa_entries; /* PN table entries = SA entries */ + u64 rsvd[16]; +}; + +struct mcs_set_active_lmac { + struct mbox_msghdr hdr; + u32 lmac_bmap; /* bitmap of active lmac per mcs block */ + u8 mcs_id; + u16 chan_base; /* MCS channel base */ + u64 rsvd; +}; + +struct mcs_set_lmac_mode { + struct mbox_msghdr hdr; + u8 mode; /* 1:Bypass 0:Operational */ + u8 lmac_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_reset_req { + struct mbox_msghdr hdr; + u8 reset; + u8 mcs_id; + u8 port_id; + u64 rsvd; +}; + +struct mcs_port_cfg_set_req { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_req { + struct mbox_msghdr hdr; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_port_cfg_get_rsp { + struct mbox_msghdr hdr; + u8 cstm_tag_rel_mode_sel; + u8 custom_hdr_enb; + u8 fifo_skid; + u8 port_mode; + u8 port_id; + u8 mcs_id; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_req { + struct mbox_msghdr hdr; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_custom_tag_cfg_get_rsp { + struct mbox_msghdr hdr; + u16 cstm_etype[8]; + u8 cstm_indx[8]; + u8 cstm_etype_en; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +/* MCS mailbox error codes + * Range 1201 - 1300. + */ +enum mcs_af_status { + MCS_AF_ERR_INVALID_MCSID = -1201, + MCS_AF_ERR_NOT_MAPPED = -1202, +}; + +struct mcs_set_pn_threshold { + struct mbox_msghdr hdr; + u64 threshold; + u8 xpn; /* '1' for setting xpn threshold */ + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +enum mcs_ctrl_pkt_rulew_type { + MCS_CTRL_PKT_RULE_TYPE_ETH, + MCS_CTRL_PKT_RULE_TYPE_DA, + MCS_CTRL_PKT_RULE_TYPE_RANGE, + MCS_CTRL_PKT_RULE_TYPE_COMBO, + MCS_CTRL_PKT_RULE_TYPE_MAC, +}; + +struct mcs_alloc_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_type; + u8 mcs_id; /* MCS block ID */ + u8 dir; /* Macsec ingress or egress side */ + u64 rsvd; +}; + +struct mcs_alloc_ctrl_pkt_rule_rsp { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_free_ctrl_pkt_rule_req { + struct mbox_msghdr hdr; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u8 all; + u64 rsvd; +}; + +struct mcs_ctrl_pkt_rule_write_req { + struct mbox_msghdr hdr; + u64 data0; + u64 data1; + u64 data2; + u8 rule_idx; + u8 rule_type; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_stats_req { + struct mbox_msghdr hdr; + u8 id; + u8 mcs_id; + u8 dir; + u64 rsvd; +}; + +struct mcs_flowid_stats { + struct mbox_msghdr hdr; + u64 tcam_hit_cnt; + u64 rsvd; +}; + +struct mcs_secy_stats { + struct mbox_msghdr hdr; + u64 ctl_pkt_bcast_cnt; + u64 ctl_pkt_mcast_cnt; + u64 ctl_pkt_ucast_cnt; + u64 ctl_octet_cnt; + u64 unctl_pkt_bcast_cnt; + u64 unctl_pkt_mcast_cnt; + u64 unctl_pkt_ucast_cnt; + u64 unctl_octet_cnt; + /* Valid only for RX */ + u64 octet_decrypted_cnt; + u64 octet_validated_cnt; + u64 pkt_port_disabled_cnt; + u64 pkt_badtag_cnt; + u64 pkt_nosa_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_tagged_ctl_cnt; + u64 pkt_untaged_cnt; + u64 pkt_ctl_cnt; /* CN10K-B */ + u64 pkt_notag_cnt; /* CNF10K-B */ + /* Valid only for TX */ + u64 octet_encrypted_cnt; + u64 octet_protected_cnt; + u64 pkt_noactivesa_cnt; + u64 pkt_toolong_cnt; + u64 pkt_untagged_cnt; + u64 rsvd[4]; +}; + +struct mcs_port_stats { + struct mbox_msghdr hdr; + u64 tcam_miss_cnt; + u64 parser_err_cnt; + u64 preempt_err_cnt; /* CNF10K-B */ + u64 sectag_insert_err_cnt; + u64 rsvd[4]; +}; + +/* Only for CN10K-B */ +struct mcs_sa_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 pkt_invalid_cnt; + u64 pkt_nosaerror_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_ok_cnt; + u64 pkt_nosa_cnt; + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 rsvd[4]; +}; + +struct mcs_sc_stats { + struct mbox_msghdr hdr; + /* RX */ + u64 hit_cnt; + u64 pkt_invalid_cnt; + u64 pkt_late_cnt; + u64 pkt_notvalid_cnt; + u64 pkt_unchecked_cnt; + u64 pkt_delay_cnt; /* CNF10K-B */ + u64 pkt_ok_cnt; /* CNF10K-B */ + u64 octet_decrypt_cnt; /* CN10K-B */ + u64 octet_validate_cnt; /* CN10K-B */ + /* TX */ + u64 pkt_encrypt_cnt; + u64 pkt_protected_cnt; + u64 octet_encrypt_cnt; /* CN10K-B */ + u64 octet_protected_cnt; /* CN10K-B */ + u64 rsvd[4]; +}; + +struct mcs_clear_stats { + struct mbox_msghdr hdr; +#define MCS_FLOWID_STATS 0 +#define MCS_SECY_STATS 1 +#define MCS_SC_STATS 2 +#define MCS_SA_STATS 3 +#define MCS_PORT_STATS 4 + u8 type; /* FLOWID, SECY, SC, SA, PORT */ + u8 id; /* type = PORT, If id = FF(invalid) port no is derived from pcifunc */ + u8 mcs_id; + u8 dir; + u8 all; /* All resources stats mapped to PF are cleared */ +}; + +struct mcs_intr_cfg { + struct mbox_msghdr hdr; +#define MCS_CPM_RX_SECTAG_V_EQ1_INT BIT_ULL(0) +#define MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT BIT_ULL(1) +#define MCS_CPM_RX_SECTAG_SL_GTE48_INT BIT_ULL(2) +#define MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT BIT_ULL(3) +#define MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT BIT_ULL(4) +#define MCS_CPM_RX_PACKET_XPN_EQ0_INT BIT_ULL(5) +#define MCS_CPM_RX_PN_THRESH_REACHED_INT BIT_ULL(6) +#define MCS_CPM_TX_PACKET_XPN_EQ0_INT BIT_ULL(7) +#define MCS_CPM_TX_PN_THRESH_REACHED_INT BIT_ULL(8) +#define MCS_CPM_TX_SA_NOT_VALID_INT BIT_ULL(9) +#define MCS_BBE_RX_DFIFO_OVERFLOW_INT BIT_ULL(10) +#define MCS_BBE_RX_PLFIFO_OVERFLOW_INT BIT_ULL(11) +#define MCS_BBE_TX_DFIFO_OVERFLOW_INT BIT_ULL(12) +#define MCS_BBE_TX_PLFIFO_OVERFLOW_INT BIT_ULL(13) +#define MCS_PAB_RX_CHAN_OVERFLOW_INT BIT_ULL(14) +#define MCS_PAB_TX_CHAN_OVERFLOW_INT BIT_ULL(15) + u64 intr_mask; /* Interrupt enable mask */ + u8 mcs_id; + u8 lmac_id; + u64 rsvd; +}; + +struct mcs_intr_info { + struct mbox_msghdr hdr; + u64 intr_mask; + int sa_id; + u8 mcs_id; + u8 lmac_id; + u64 rsvd; +}; + #endif /* MBOX_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c new file mode 100644 index 0000000000000000000000000000000000000000..4a343f853b28bd51a3a0937bc8c94b9c2f139309 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -0,0 +1,1603 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include +#include +#include +#include +#include + +#include "mcs.h" +#include "mcs_reg.h" + +#define DRV_NAME "Marvell MCS Driver" + +#define PCI_CFG_REG_BAR_NUM 0 + +static const struct pci_device_id mcs_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_MCS) }, + { 0, } /* end of table */ +}; + +static LIST_HEAD(mcs_list); + +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(id); + stats->octet_encrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(id); + stats->pkt_noactivesa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(id); + stats->pkt_toolong_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(id); + stats->pkt_untagged_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id) +{ + u64 reg; + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(id); + stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(id); + stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(id); + stats->ctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(id); + stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(id); + stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(id); + stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(id); + stats->unctl_octet_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(id); + stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(id); + stats->octet_decrypted_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(id); + stats->octet_validated_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(id); + stats->pkt_port_disabled_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(id); + stats->pkt_badtag_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(id); + stats->pkt_tagged_ctl_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(id); + stats->pkt_untaged_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id); + stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(id); + stats->pkt_notag_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(id); + else + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(id); + + stats->tcam_hit_cnt = mcs_reg_read(mcs, reg); +} + +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(id); + stats->preempt_err_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(id); + stats->tcam_miss_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(id); + stats->parser_err_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(id); + stats->sectag_insert_err_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(id); + stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(id); + stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg); + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + } +} + +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, + int id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(id); + stats->hit_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(id); + stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(id); + stats->pkt_late_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(id); + stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(id); + stats->pkt_unchecked_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(id); + stats->pkt_delay_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(id); + stats->pkt_ok_cnt = mcs_reg_read(mcs, reg); + } + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(id); + stats->octet_decrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(id); + stats->octet_validate_cnt = mcs_reg_read(mcs, reg); + } + } else { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(id); + stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(id); + stats->pkt_protected_cnt = mcs_reg_read(mcs, reg); + + if (mcs->hw->mcs_blks == 1) { + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(id); + stats->octet_encrypt_cnt = mcs_reg_read(mcs, reg); + + reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(id); + stats->octet_protected_cnt = mcs_reg_read(mcs, reg); + } + } +} + +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir) +{ + struct mcs_flowid_stats flowid_st; + struct mcs_port_stats port_st; + struct mcs_secy_stats secy_st; + struct mcs_sc_stats sc_st; + struct mcs_sa_stats sa_st; + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CSE_RX_SLAVE_CTRL; + else + reg = MCSX_CSE_TX_SLAVE_CTRL; + + mcs_reg_write(mcs, reg, BIT_ULL(0)); + + switch (type) { + case MCS_FLOWID_STATS: + mcs_get_flowid_stats(mcs, &flowid_st, id, dir); + break; + case MCS_SECY_STATS: + if (dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, &secy_st, id); + else + mcs_get_tx_secy_stats(mcs, &secy_st, id); + break; + case MCS_SC_STATS: + mcs_get_sc_stats(mcs, &sc_st, id, dir); + break; + case MCS_SA_STATS: + mcs_get_sa_stats(mcs, &sa_st, id, dir); + break; + case MCS_PORT_STATS: + mcs_get_port_stats(mcs, &port_st, id, dir); + break; + } + + mcs_reg_write(mcs, reg, 0x0); +} + +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear FLOWID stats */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_FLOWID_STATS, id, dir); + } + + /* Clear SECY stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SECY_STATS, id, dir); + } + + /* Clear SC stats */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SC_STATS, id, dir); + } + + /* Clear SA stats */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_clear_stats(mcs, MCS_SA_STATS, id, dir); + } + return 0; +} + +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + else + reg = MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(pn_id); + mcs_reg_write(mcs, reg, next_pn); +} + +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0xFF) | + (map->sa_index1 & 0xFF) << 9 | + (map->rekey_ena & 0x1) << 18 | + (map->sa_index0_vld & 0x1) << 19 | + (map->sa_index1_vld & 0x1) << 20 | + (map->tx_sa_active & 0x1) << 21 | + map->sectag_sci << 22; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + val = map->sectag_sci >> 42; + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(map->sc_id); + mcs_reg_write(mcs, reg, val); +} + +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0xFF) | map->sa_in_use << 9; + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 8; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 9; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id); + mcs_reg_write(mcs, reg, plcy[reg_id]); + } + } +} + +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int sc_id, int ena) +{ + u64 reg, val; + + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(0); + if (sc_id > 63) + reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(1); + + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(sc_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(sc_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id) +{ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(0, sc_id), sci); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(1, sc_id), secy); + /* Enable SC CAM */ + mcs_ena_dis_sc_cam_entry(mcs, sc_id, true); +} + +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int secy_id, int dir) +{ + u64 reg; + + if (dir == MCS_RX) + reg = MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(secy_id); + else + reg = MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(secy_id); + + mcs_reg_write(mcs, reg, plcy); + + if (mcs->hw->mcs_blks == 1 && dir == MCS_RX) + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(secy_id), 0x0ull); +} + +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x7F) | (map->ctrl_pkt & 0x1) << 8; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + val |= (map->sc & 0x7F) << 9; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int flow_id, int dir, int ena) +{ + u64 reg, val; + + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1; + } else { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0; + if (flow_id > 63) + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1; + } + + /* Enable/Disable the tcam entry */ + if (ena) + val = mcs_reg_read(mcs, reg) | BIT_ULL(flow_id); + else + val = mcs_reg_read(mcs, reg) & ~BIT_ULL(flow_id); + + mcs_reg_write(mcs, reg, val); +} + +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int flow_id, int dir) +{ + int reg_id; + u64 reg; + + if (dir == MCS_RX) { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } else { + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id); + mcs_reg_write(mcs, reg, data[reg_id]); + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, mask[reg_id]); + } + } +} + +int mcs_install_flowid_bypass_entry(struct mcs *mcs) +{ + int flow_id, secy_id, reg_id; + struct secy_mem_map map; + u64 reg, plcy = 0; + + /* Flow entry */ + flow_id = mcs->hw->tcam_entries - MCS_RSRC_RSVD_CNT; + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + for (reg_id = 0; reg_id < 4; reg_id++) { + reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id); + mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0)); + } + /* secy */ + secy_id = mcs->hw->secy_entries - MCS_RSRC_RSVD_CNT; + + /* Set validate frames to NULL and enable control port */ + plcy = 0x7ull; + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | 0x3ull << 4; + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_RX); + + /* Enable control port and set mtu to max */ + plcy = BIT_ULL(0) | GENMASK_ULL(43, 28); + if (mcs->hw->mcs_blks > 1) + plcy = BIT_ULL(0) | GENMASK_ULL(63, 48); + mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_TX); + + /* Map flowid to secy */ + map.secy = secy_id; + map.ctrl_pkt = 0; + map.flow_id = flow_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_RX); + map.sc = secy_id; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_TX); + + /* Enable Flowid entry */ + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_RX, true); + mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_TX, true); + return 0; +} + +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir) +{ + struct mcs_rsrc_map *map; + int flow_id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* Clear secy memory to zero */ + mcs_secy_plcy_write(mcs, 0, secy_id, dir); + + /* Disable the tcam entry using this secy */ + for (flow_id = 0; flow_id < map->flow_ids.max; flow_id++) { + if (map->flowid2secy_map[flow_id] != secy_id) + continue; + mcs_ena_dis_flowid_entry(mcs, flow_id, dir, false); + } +} + +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc) +{ + int rsrc_id; + + if (!rsrc->bmap) + return -EINVAL; + + rsrc_id = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, offset, 1, 0); + if (rsrc_id >= rsrc->max) + return -ENOSPC; + + bitmap_set(rsrc->bmap, rsrc_id, 1); + pf_map[rsrc_id] = pcifunc; + + return rsrc_id; +} + +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + u64 dis, reg; + int id, rc; + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + if (req->all) { + for (id = 0; id < map->ctrlpktrule.max; id++) { + if (map->ctrlpktrule2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, id, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(id); + mcs_reg_write(mcs, reg, dis); + } + return 0; + } + + rc = mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, req->rule_idx, pcifunc); + dis = mcs_reg_read(mcs, reg); + dis &= ~BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, dis); + + return rc; +} + +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req) +{ + u64 reg, enb; + u64 idx; + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + req->data0 &= GENMASK(15, 0); + if (req->data0 != ETH_P_PAE) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_ETYPE_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(idx) : + MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(idx); + + mcs_reg_write(mcs, reg, req->data0); + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_DAX(idx) : + MCSX_PEX_TX_SLAVE_RULE_DAX(idx); + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + if (!(req->data0 & BIT_ULL(40)) || !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + req->data2 &= GENMASK(15, 0); + if (req->data2 != ETH_P_PAE || !(req->data0 & BIT_ULL(40)) || + !(req->data1 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_COMBO_RULE_OFFSET; + if (req->dir == MCS_RX) { + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } else { + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(idx); + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(idx); + mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0)); + reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(idx); + mcs_reg_write(mcs, reg, req->data2); + } + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + if (!(req->data0 & BIT_ULL(40))) + return -EINVAL; + + idx = req->rule_idx - MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_MAC : + MCSX_PEX_TX_SLAVE_RULE_MAC; + + mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0)); + break; + } + + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE; + + enb = mcs_reg_read(mcs, reg); + enb |= BIT_ULL(req->rule_idx); + mcs_reg_write(mcs, reg, enb); + + return 0; +} + +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc) +{ + /* Check if the rsrc_id is mapped to PF/VF */ + if (pf_map[rsrc_id] != pcifunc) + return -EINVAL; + + rvu_free_rsrc(rsrc, rsrc_id); + pf_map[rsrc_id] = 0; + return 0; +} + +/* Free all the cam resources mapped to pf */ +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + /* free tcam entries */ + for (id = 0; id < map->flow_ids.max; id++) { + if (map->flowid2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, + id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, id, dir, false); + } + + /* free secy entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->secy2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->secy, map->secy2pf_map, + id, pcifunc); + mcs_clear_secy_plcy(mcs, id, dir); + } + + /* free sc entries */ + for (id = 0; id < map->secy.max; id++) { + if (map->sc2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc); + + /* Disable SC CAM only on RX side */ + if (dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, id, false); + } + + /* free sa entries */ + for (id = 0; id < map->sa.max; id++) { + if (map->sa2pf_map[id] != pcifunc) + continue; + mcs_free_rsrc(&map->sa, map->sa2pf_map, id, pcifunc); + } + return 0; +} + +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc) +{ + int rsrc_id; + + rsrc_id = rvu_alloc_rsrc(rsrc); + if (rsrc_id < 0) + return -ENOMEM; + pf_map[rsrc_id] = pcifunc; + return rsrc_id; +} + +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flow_id, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir) +{ + struct mcs_rsrc_map *map; + int id; + + if (dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *flow_id = id; + + id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *secy_id = id; + + id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sc_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa1_id = id; + + id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (id < 0) + return -ENOMEM; + *sa2_id = id; + + return 0; +} + +static void cn10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 9) & 0xFF; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void cn10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val, status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + /* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + /* Auto rekey is enable */ + if (!((val >> 18) & 0x1)) + continue; + + status = (val >> 21) & 0x1; + + /* Check if tx_sa_active status had changed */ + if (status == mcs->tx_sa_active[sc]) + continue; + /* SA_index0 is expired */ + if (status) + event.sa_id = val & 0xFF; + else + event.sa_id = (val >> 9) & 0xFF; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_rx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + int sa, reg; + u64 intr; + + /* Check expired SAs */ + for (reg = 0; reg < (mcs->hw->sa_entries / 64); reg++) { + /* Bit high in *PN_THRESH_REACHEDX implies + * corresponding SAs are expired. + */ + intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(reg)); + for (sa = 0; sa < 64; sa++) { + if (!(intr & BIT_ULL(sa))) + continue; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_RX_PN_THRESH_REACHED_INT; + event.sa_id = sa + (reg * 64); + event.pcifunc = mcs->rx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } + } +} + +static void mcs_rx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + if (intr & MCS_CPM_RX_INT_SECTAG_V_EQ1) + event.intr_mask = MCS_CPM_RX_SECTAG_V_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SL_GTE48) + event.intr_mask |= MCS_CPM_RX_SECTAG_SL_GTE48_INT; + if (intr & MCS_CPM_RX_INT_ES_EQ1_SC_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT; + if (intr & MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1) + event.intr_mask |= MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT; + if (intr & MCS_CPM_RX_INT_PACKET_XPN_EQ0) + event.intr_mask |= MCS_CPM_RX_PACKET_XPN_EQ0_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_tx_misc_intr_handler(struct mcs *mcs, u64 intr) +{ + struct mcs_intr_event event = { 0 }; + + if (!(intr & MCS_CPM_TX_INT_SA_NOT_VALID)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + event.intr_mask = MCS_CPM_TX_SA_NOT_VALID_INT; + + mcs_add_intr_wq_entry(mcs, &event); +} + +static void mcs_bbe_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_BBE_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_BBE_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + /* Lower nibble denotes data fifo overflow interrupts and + * upper nibble indicates policy fifo overflow interrupts. + */ + if (intr & 0xFULL) + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_DFIFO_OVERFLOW_INT : + MCS_BBE_TX_DFIFO_OVERFLOW_INT; + else + event.intr_mask = (dir == MCS_RX) ? + MCS_BBE_RX_PLFIFO_OVERFLOW_INT : + MCS_BBE_RX_PLFIFO_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into BBE fatal error */ + event.lmac_id = i & 0x3ULL; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static void mcs_pab_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir) +{ + struct mcs_intr_event event = { 0 }; + int i; + + if (!(intr & MCS_PAB_INT_MASK)) + return; + + event.mcs_id = mcs->mcs_id; + event.pcifunc = mcs->pf_map[0]; + + for (i = 0; i < MCS_MAX_PAB_INT; i++) { + if (!(intr & BIT_ULL(i))) + continue; + + event.intr_mask = (dir == MCS_RX) ? MCS_PAB_RX_CHAN_OVERFLOW_INT : + MCS_PAB_TX_CHAN_OVERFLOW_INT; + + /* Notify the lmac_id info which ran into PAB fatal error */ + event.lmac_id = i; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +static irqreturn_t mcs_ip_intr_handler(int irq, void *mcs_irq) +{ + struct mcs *mcs = (struct mcs *)mcs_irq; + u64 intr, cpm_intr, bbe_intr, pab_intr; + + /* Disable and clear the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1C, BIT_ULL(0)); + mcs_reg_write(mcs, MCSX_IP_INT, BIT_ULL(0)); + + /* Check which block has interrupt*/ + intr = mcs_reg_read(mcs, MCSX_TOP_SLAVE_INT_SUM); + + /* CPM RX */ + if (intr & MCS_CPM_RX_INT_ENA) { + /* Check for PN thresh interrupt bit */ + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_RX_INT); + + if (cpm_intr & MCS_CPM_RX_INT_PN_THRESH_REACHED) + mcs_rx_pn_thresh_reached_handler(mcs); + + if (cpm_intr & MCS_CPM_RX_INT_ALL) + mcs_rx_misc_intr_handler(mcs, cpm_intr); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT, cpm_intr); + } + + /* CPM TX */ + if (intr & MCS_CPM_TX_INT_ENA) { + cpm_intr = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_INT); + + if (cpm_intr & MCS_CPM_TX_INT_PN_THRESH_REACHED) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_thresh_reached_handler(mcs); + else + cn10kb_mcs_tx_pn_thresh_reached_handler(mcs); + } + + if (cpm_intr & MCS_CPM_TX_INT_SA_NOT_VALID) + mcs_tx_misc_intr_handler(mcs, cpm_intr); + + if (cpm_intr & MCS_CPM_TX_INT_PACKET_XPN_EQ0) { + if (mcs->hw->mcs_blks > 1) + cnf10kb_mcs_tx_pn_wrapped_handler(mcs); + else + cn10kb_mcs_tx_pn_wrapped_handler(mcs); + } + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT, cpm_intr); + } + + /* BBE RX */ + if (intr & MCS_BBE_RX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_RX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT, bbe_intr); + } + + /* BBE TX */ + if (intr & MCS_BBE_TX_INT_ENA) { + bbe_intr = mcs_reg_read(mcs, MCSX_BBE_TX_SLAVE_BBE_INT); + mcs_bbe_intr_handler(mcs, bbe_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT, bbe_intr); + } + + /* PAB RX */ + if (intr & MCS_PAB_RX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_RX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT, pab_intr); + } + + /* PAB TX */ + if (intr & MCS_PAB_TX_INT_ENA) { + pab_intr = mcs_reg_read(mcs, MCSX_PAB_TX_SLAVE_PAB_INT); + mcs_pab_intr_handler(mcs, pab_intr, MCS_TX); + + /* Clear the interrupt */ + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW, 0); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT, pab_intr); + } + + /* Enable the interrupt */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + return IRQ_HANDLED; +} + +static void *alloc_mem(struct mcs *mcs, int n) +{ + return devm_kcalloc(mcs->dev, n, sizeof(u16), GFP_KERNEL); +} + +static int mcs_alloc_struct_mem(struct mcs *mcs, struct mcs_rsrc_map *res) +{ + struct hwinfo *hw = mcs->hw; + int err; + + res->flowid2pf_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2pf_map) + return -ENOMEM; + + res->secy2pf_map = alloc_mem(mcs, hw->secy_entries); + if (!res->secy2pf_map) + return -ENOMEM; + + res->sc2pf_map = alloc_mem(mcs, hw->sc_entries); + if (!res->sc2pf_map) + return -ENOMEM; + + res->sa2pf_map = alloc_mem(mcs, hw->sa_entries); + if (!res->sa2pf_map) + return -ENOMEM; + + res->flowid2secy_map = alloc_mem(mcs, hw->tcam_entries); + if (!res->flowid2secy_map) + return -ENOMEM; + + res->ctrlpktrule2pf_map = alloc_mem(mcs, MCS_MAX_CTRLPKT_RULES); + if (!res->ctrlpktrule2pf_map) + return -ENOMEM; + + res->flow_ids.max = hw->tcam_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->flow_ids); + if (err) + return err; + + res->secy.max = hw->secy_entries - MCS_RSRC_RSVD_CNT; + err = rvu_alloc_bitmap(&res->secy); + if (err) + return err; + + res->sc.max = hw->sc_entries; + err = rvu_alloc_bitmap(&res->sc); + if (err) + return err; + + res->sa.max = hw->sa_entries; + err = rvu_alloc_bitmap(&res->sa); + if (err) + return err; + + res->ctrlpktrule.max = MCS_MAX_CTRLPKT_RULES; + err = rvu_alloc_bitmap(&res->ctrlpktrule); + if (err) + return err; + + return 0; +} + +static int mcs_register_interrupts(struct mcs *mcs) +{ + int ret = 0; + + mcs->num_vec = pci_msix_vec_count(mcs->pdev); + + ret = pci_alloc_irq_vectors(mcs->pdev, mcs->num_vec, + mcs->num_vec, PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(mcs->dev, "MCS Request for %d msix vector failed err:%d\n", + mcs->num_vec, ret); + return ret; + } + + ret = request_irq(pci_irq_vector(mcs->pdev, MCS_INT_VEC_IP), + mcs_ip_intr_handler, 0, "MCS_IP", mcs); + if (ret) { + dev_err(mcs->dev, "MCS IP irq registration failed\n"); + goto exit; + } + + /* MCS enable IP interrupts */ + mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0)); + + /* Enable CPM Rx/Tx interrupts */ + mcs_reg_write(mcs, MCSX_TOP_SLAVE_INT_SUM_ENB, + MCS_CPM_RX_INT_ENA | MCS_CPM_TX_INT_ENA | + MCS_BBE_RX_INT_ENA | MCS_BBE_TX_INT_ENA | + MCS_PAB_RX_INT_ENA | MCS_PAB_TX_INT_ENA); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT_ENB, 0x7ULL); + mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT_ENB, 0x7FULL); + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_ENB, 0xff); + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_ENB, 0xff); + mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_ENB, 0xff); + + mcs->tx_sa_active = alloc_mem(mcs, mcs->hw->sc_entries); + if (!mcs->tx_sa_active) { + ret = -ENOMEM; + goto exit; + } + + return ret; +exit: + pci_free_irq_vectors(mcs->pdev); + mcs->num_vec = 0; + return ret; +} + +int mcs_get_blkcnt(void) +{ + struct mcs *mcs; + int idmax = -ENODEV; + + /* Check MCS block is present in hardware */ + if (!pci_dev_present(mcs_id_table)) + return 0; + + list_for_each_entry(mcs, &mcs_list, mcs_list) + if (mcs->mcs_id > idmax) + idmax = mcs->mcs_id; + + if (idmax < 0) + return 0; + + return idmax + 1; +} + +struct mcs *mcs_get_pdata(int mcs_id) +{ + struct mcs *mcs_dev; + + list_for_each_entry(mcs_dev, &mcs_list, mcs_list) { + if (mcs_dev->mcs_id == mcs_id) + return mcs_dev; + } + return NULL; +} + +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req) +{ + u64 val = 0; + + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id), + req->port_mode & MCS_PORT_MODE_MASK); + + req->cstm_tag_rel_mode_sel &= 0x3; + + if (mcs->hw->mcs_blks > 1) { + req->fifo_skid &= MCS_PORT_FIFO_SKID_MASK; + val = (u32)req->fifo_skid << 0x10; + val |= req->fifo_skid; + mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id), val); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id), + req->cstm_tag_rel_mode_sel); + val = mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION); + + if (req->custom_hdr_enb) + val |= BIT_ULL(req->port_id); + else + val &= ~BIT_ULL(req->port_id); + + mcs_reg_write(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION, val); + } else { + val = mcs_reg_read(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id)); + val |= (req->cstm_tag_rel_mode_sel << 2); + mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id), val); + } +} + +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + u64 reg = 0; + + rsp->port_mode = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id)) & + MCS_PORT_MODE_MASK; + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id); + rsp->fifo_skid = mcs_reg_read(mcs, reg) & MCS_PORT_FIFO_SKID_MASK; + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) & 0x3; + if (mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION) & BIT_ULL(req->port_id)) + rsp->custom_hdr_enb = 1; + } else { + reg = MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id); + rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) >> 2; + } + + rsp->port_id = req->port_id; + rsp->mcs_id = req->mcs_id; +} + +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + u64 reg = 0, val = 0; + u8 idx; + + for (idx = 0; idx < MCS_MAX_CUSTOM_TAGS; idx++) { + if (mcs->hw->mcs_blks > 1) + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(idx) : + MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(idx); + else + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_VLAN_CFGX(idx) : + MCSX_PEX_TX_SLAVE_VLAN_CFGX(idx); + + val = mcs_reg_read(mcs, reg); + if (mcs->hw->mcs_blks > 1) { + rsp->cstm_etype[idx] = val & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x16) & 0x3; + reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_ETYPE_ENABLE : + MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + rsp->cstm_etype_en = mcs_reg_read(mcs, reg) & 0xFF; + } else { + rsp->cstm_etype[idx] = (val >> 0x1) & GENMASK(15, 0); + rsp->cstm_indx[idx] = (val >> 0x11) & 0x3; + rsp->cstm_etype_en |= (val & 0x1) << idx; + } + } + + rsp->mcs_id = req->mcs_id; + rsp->dir = req->dir; +} + +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset) +{ + u64 reg = MCSX_MCS_TOP_SLAVE_PORT_RESET(port_id); + + mcs_reg_write(mcs, reg, reset & 0x1); +} + +/* Set lmac to bypass/operational mode */ +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode) +{ + u64 reg; + + reg = MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(lmac_id * 2); + mcs_reg_write(mcs, reg, (u64)mode); +} + +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn) +{ + u64 reg; + + if (pn->dir == MCS_RX) + reg = pn->xpn ? MCSX_CPM_RX_SLAVE_XPN_THRESHOLD : MCSX_CPM_RX_SLAVE_PN_THRESHOLD; + else + reg = pn->xpn ? MCSX_CPM_TX_SLAVE_XPN_THRESHOLD : MCSX_CPM_TX_SLAVE_PN_THRESHOLD; + + mcs_reg_write(mcs, reg, pn->threshold); +} + +void cn10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN CTag */ + val = BIT_ULL(0) | (0x8100ull & 0xFFFF) << 1 | BIT_ULL(17); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = BIT_ULL(0) | (0x88a8ull & 0xFFFF) << 1 | BIT_ULL(18); + /* RX */ + reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(1); + mcs_reg_write(mcs, reg, val); +} + +static void mcs_lmac_init(struct mcs *mcs, int lmac_id) +{ + u64 reg; + + /* Port mode 25GB */ + reg = MCSX_PAB_RX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); + + if (mcs->hw->mcs_blks > 1) { + reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0xe000e); + return; + } + + reg = MCSX_PAB_TX_SLAVE_PORT_CFGX(lmac_id); + mcs_reg_write(mcs, reg, 0); +} + +int mcs_set_lmac_channels(int mcs_id, u16 base) +{ + struct mcs *mcs; + int lmac; + u64 cfg; + + mcs = mcs_get_pdata(mcs_id); + if (!mcs) + return -ENODEV; + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) { + cfg = mcs_reg_read(mcs, MCSX_LINK_LMACX_CFG(lmac)); + cfg &= ~(MCSX_LINK_LMAC_BASE_MASK | MCSX_LINK_LMAC_RANGE_MASK); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_RANGE_MASK, ilog2(16)); + cfg |= FIELD_PREP(MCSX_LINK_LMAC_BASE_MASK, base); + mcs_reg_write(mcs, MCSX_LINK_LMACX_CFG(lmac), cfg); + base += 16; + } + return 0; +} + +static int mcs_x2p_calibration(struct mcs *mcs) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(20000); + int i, err = 0; + u64 val; + + /* set X2P calibration */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + val |= BIT_ULL(5); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Wait for calibration to complete */ + while (!(mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS) & BIT_ULL(0))) { + if (time_before(jiffies, timeout)) { + usleep_range(80, 100); + continue; + } else { + err = -EBUSY; + dev_err(mcs->dev, "MCS X2P calibration failed..ignoring\n"); + return err; + } + } + + val = mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS); + for (i = 0; i < mcs->hw->mcs_x2p_intf; i++) { + if (val & BIT_ULL(1 + i)) + continue; + err = -EBUSY; + dev_err(mcs->dev, "MCS:%d didn't respond to X2P calibration\n", i); + } + /* Clear X2P calibrate */ + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, mcs_reg_read(mcs, MCSX_MIL_GLOBAL) & ~BIT_ULL(5)); + + return err; +} + +static void mcs_set_external_bypass(struct mcs *mcs, u8 bypass) +{ + u64 val; + + /* Set MCS to external bypass */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + if (bypass) + val |= BIT_ULL(6); + else + val &= ~BIT_ULL(6); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); +} + +static void mcs_global_cfg(struct mcs *mcs) +{ + /* Disable external bypass */ + mcs_set_external_bypass(mcs, false); + + /* Reset TX/RX stats memory */ + mcs_reg_write(mcs, MCSX_CSE_RX_SLAVE_STATS_CLEAR, 0x1F); + mcs_reg_write(mcs, MCSX_CSE_TX_SLAVE_STATS_CLEAR, 0x1F); + + /* Set MCS to perform standard IEEE802.1AE macsec processing */ + if (mcs->hw->mcs_blks == 1) { + mcs_reg_write(mcs, MCSX_IP_MODE, BIT_ULL(3)); + return; + } + + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_ENTRY, 0xe4); + mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_LEN, 4); +} + +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 128; /* TCAM entries */ + hw->secy_entries = 128; /* SecY entries */ + hw->sc_entries = 128; /* SC CAM entries */ + hw->sa_entries = 256; /* SA entries */ + hw->lmac_cnt = 20; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 5; /* x2p clabration intf */ + hw->mcs_blks = 1; /* MCS blocks */ +} + +static struct mcs_ops cn10kb_mcs_ops = { + .mcs_set_hw_capabilities = cn10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cn10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cn10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cn10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cn10kb_mcs_flowid_secy_map, +}; + +static int mcs_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + int lmac, err = 0; + struct mcs *mcs; + + mcs = devm_kzalloc(dev, sizeof(*mcs), GFP_KERNEL); + if (!mcs) + return -ENOMEM; + + mcs->hw = devm_kzalloc(dev, sizeof(struct hwinfo), GFP_KERNEL); + if (!mcs->hw) + return -ENOMEM; + + err = pci_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + pci_set_drvdata(pdev, NULL); + return err; + } + + err = pci_request_regions(pdev, DRV_NAME); + if (err) { + dev_err(dev, "PCI request regions failed 0x%x\n", err); + goto exit; + } + + mcs->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); + if (!mcs->reg_base) { + dev_err(dev, "mcs: Cannot map CSR memory space, aborting\n"); + err = -ENOMEM; + goto exit; + } + + pci_set_drvdata(pdev, mcs); + mcs->pdev = pdev; + mcs->dev = &pdev->dev; + + if (pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B) + mcs->mcs_ops = &cn10kb_mcs_ops; + else + mcs->mcs_ops = cnf10kb_get_mac_ops(); + + /* Set hardware capabilities */ + mcs->mcs_ops->mcs_set_hw_capabilities(mcs); + + mcs_global_cfg(mcs); + + /* Perform X2P clibration */ + err = mcs_x2p_calibration(mcs); + if (err) + goto err_x2p; + + mcs->mcs_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24) + & MCS_ID_MASK; + + /* Set mcs tx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->tx); + if (err) + goto err_x2p; + + /* Set mcs rx side resources */ + err = mcs_alloc_struct_mem(mcs, &mcs->rx); + if (err) + goto err_x2p; + + /* per port config */ + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_lmac_init(mcs, lmac); + + /* Parser configuration */ + mcs->mcs_ops->mcs_parser_cfg(mcs); + + err = mcs_register_interrupts(mcs); + if (err) + goto exit; + + list_add(&mcs->mcs_list, &mcs_list); + mutex_init(&mcs->stats_lock); + + return 0; + +err_x2p: + /* Enable external bypass */ + mcs_set_external_bypass(mcs, true); +exit: + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); + return err; +} + +static void mcs_remove(struct pci_dev *pdev) +{ + struct mcs *mcs = pci_get_drvdata(pdev); + + /* Set MCS to external bypass */ + mcs_set_external_bypass(mcs, true); + pci_free_irq_vectors(pdev); + pci_release_regions(pdev); + pci_disable_device(pdev); + pci_set_drvdata(pdev, NULL); +} + +struct pci_driver mcs_driver = { + .name = DRV_NAME, + .id_table = mcs_id_table, + .probe = mcs_probe, + .remove = mcs_remove, +}; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h new file mode 100644 index 0000000000000000000000000000000000000000..64dc2b80e15dde7b12e95b744acdac3d2e10480d --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h @@ -0,0 +1,246 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_H +#define MCS_H + +#include +#include "rvu.h" + +#define PCI_DEVID_CN10K_MCS 0xA096 + +#define MCSX_LINK_LMAC_RANGE_MASK GENMASK_ULL(19, 16) +#define MCSX_LINK_LMAC_BASE_MASK GENMASK_ULL(11, 0) + +#define MCS_ID_MASK 0x7 +#define MCS_MAX_PFS 128 + +#define MCS_PORT_MODE_MASK 0x3 +#define MCS_PORT_FIFO_SKID_MASK 0x3F +#define MCS_MAX_CUSTOM_TAGS 0x8 + +#define MCS_CTRLPKT_ETYPE_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RULE_MAX 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_MAX 4 +#define MCS_CTRLPKT_COMBO_RULE_MAX 4 +#define MCS_CTRLPKT_MAC_RULE_MAX 1 + +#define MCS_MAX_CTRLPKT_RULES (MCS_CTRLPKT_ETYPE_RULE_MAX + \ + MCS_CTRLPKT_DA_RULE_MAX + \ + MCS_CTRLPKT_DA_RANGE_RULE_MAX + \ + MCS_CTRLPKT_COMBO_RULE_MAX + \ + MCS_CTRLPKT_MAC_RULE_MAX) + +#define MCS_CTRLPKT_ETYPE_RULE_OFFSET 0 +#define MCS_CTRLPKT_DA_RULE_OFFSET 8 +#define MCS_CTRLPKT_DA_RANGE_RULE_OFFSET 16 +#define MCS_CTRLPKT_COMBO_RULE_OFFSET 20 +#define MCS_CTRLPKT_MAC_EN_RULE_OFFSET 24 + +/* Reserved resources for default bypass entry */ +#define MCS_RSRC_RSVD_CNT 1 + +/* MCS Interrupt Vector Enumeration */ +enum mcs_int_vec_e { + MCS_INT_VEC_MIL_RX_GBL = 0x0, + MCS_INT_VEC_MIL_RX_LMACX = 0x1, + MCS_INT_VEC_MIL_TX_LMACX = 0x5, + MCS_INT_VEC_HIL_RX_GBL = 0x9, + MCS_INT_VEC_HIL_RX_LMACX = 0xa, + MCS_INT_VEC_HIL_TX_GBL = 0xe, + MCS_INT_VEC_HIL_TX_LMACX = 0xf, + MCS_INT_VEC_IP = 0x13, + MCS_INT_VEC_CNT = 0x14, +}; + +#define MCS_MAX_BBE_INT 8ULL +#define MCS_BBE_INT_MASK 0xFFULL + +#define MCS_MAX_PAB_INT 4ULL +#define MCS_PAB_INT_MASK 0xFULL + +#define MCS_BBE_RX_INT_ENA BIT_ULL(0) +#define MCS_BBE_TX_INT_ENA BIT_ULL(1) +#define MCS_CPM_RX_INT_ENA BIT_ULL(2) +#define MCS_CPM_TX_INT_ENA BIT_ULL(3) +#define MCS_PAB_RX_INT_ENA BIT_ULL(4) +#define MCS_PAB_TX_INT_ENA BIT_ULL(5) + +#define MCS_CPM_TX_INT_PACKET_XPN_EQ0 BIT_ULL(0) +#define MCS_CPM_TX_INT_PN_THRESH_REACHED BIT_ULL(1) +#define MCS_CPM_TX_INT_SA_NOT_VALID BIT_ULL(2) + +#define MCS_CPM_RX_INT_SECTAG_V_EQ1 BIT_ULL(0) +#define MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 BIT_ULL(1) +#define MCS_CPM_RX_INT_SL_GTE48 BIT_ULL(2) +#define MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 BIT_ULL(3) +#define MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 BIT_ULL(4) +#define MCS_CPM_RX_INT_PACKET_XPN_EQ0 BIT_ULL(5) +#define MCS_CPM_RX_INT_PN_THRESH_REACHED BIT_ULL(6) + +#define MCS_CPM_RX_INT_ALL (MCS_CPM_RX_INT_SECTAG_V_EQ1 | \ + MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 | \ + MCS_CPM_RX_INT_SL_GTE48 | \ + MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 | \ + MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 | \ + MCS_CPM_RX_INT_PACKET_XPN_EQ0 | \ + MCS_CPM_RX_INT_PN_THRESH_REACHED) + +struct mcs_pfvf { + u64 intr_mask; /* Enabled Interrupt mask */ +}; + +struct mcs_intr_event { + u16 pcifunc; + u64 intr_mask; + u64 sa_id; + u8 mcs_id; + u8 lmac_id; +}; + +struct mcs_intrq_entry { + struct list_head node; + struct mcs_intr_event intr_event; +}; + +struct secy_mem_map { + u8 flow_id; + u8 secy; + u8 ctrl_pkt; + u8 sc; + u64 sci; +}; + +struct mcs_rsrc_map { + u16 *flowid2pf_map; + u16 *secy2pf_map; + u16 *sc2pf_map; + u16 *sa2pf_map; + u16 *flowid2secy_map; /* bitmap flowid mapped to secy*/ + u16 *ctrlpktrule2pf_map; + struct rsrc_bmap flow_ids; + struct rsrc_bmap secy; + struct rsrc_bmap sc; + struct rsrc_bmap sa; + struct rsrc_bmap ctrlpktrule; +}; + +struct hwinfo { + u8 tcam_entries; + u8 secy_entries; + u8 sc_entries; + u16 sa_entries; + u8 mcs_x2p_intf; + u8 lmac_cnt; + u8 mcs_blks; + unsigned long lmac_bmap; /* bitmap of enabled mcs lmac */ +}; + +struct mcs { + void __iomem *reg_base; + struct pci_dev *pdev; + struct device *dev; + struct hwinfo *hw; + struct mcs_rsrc_map tx; + struct mcs_rsrc_map rx; + u16 pf_map[MCS_MAX_PFS]; /* List of PCIFUNC mapped to MCS */ + u8 mcs_id; + struct mcs_ops *mcs_ops; + struct list_head mcs_list; + /* Lock for mcs stats */ + struct mutex stats_lock; + struct mcs_pfvf *pf; + struct mcs_pfvf *vf; + u16 num_vec; + void *rvu; + u16 *tx_sa_active; +}; + +struct mcs_ops { + void (*mcs_set_hw_capabilities)(struct mcs *mcs); + void (*mcs_parser_cfg)(struct mcs *mcs); + void (*mcs_tx_sa_mem_map_write)(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); + void (*mcs_rx_sa_mem_map_write)(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); + void (*mcs_flowid_secy_map)(struct mcs *mcs, struct secy_mem_map *map, int dir); +}; + +extern struct pci_driver mcs_driver; + +static inline void mcs_reg_write(struct mcs *mcs, u64 offset, u64 val) +{ + writeq(val, mcs->reg_base + offset); +} + +static inline u64 mcs_reg_read(struct mcs *mcs, u64 offset) +{ + return readq(mcs->reg_base + offset); +} + +/* MCS APIs */ +struct mcs *mcs_get_pdata(int mcs_id); +int mcs_get_blkcnt(void); +int mcs_set_lmac_channels(int mcs_id, u16 base); +int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc); +int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc); +int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flowid, u8 *secy_id, + u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir); +int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc); +void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir); +void mcs_ena_dis_flowid_entry(struct mcs *mcs, int id, int dir, int ena); +void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int id, int ena); +void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int id, int dir); +void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int id, int dir); +void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id); +void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa, int dir); +void mcs_map_sc_to_sa(struct mcs *mcs, u64 *sa_map, int sc, int dir); +void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir); +void mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn); +int mcs_install_flowid_bypass_entry(struct mcs *mcs); +void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode); +void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset); +void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req); +void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp); +void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp); +int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc); +int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req); +int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req); + +/* CN10K-B APIs */ +void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cn10kb_mcs_parser_cfg(struct mcs *mcs); + +/* CNF10K-B APIs */ +struct mcs_ops *cnf10kb_get_mac_ops(void); +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs); +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map); +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir); +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map); +void cnf10kb_mcs_parser_cfg(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs); +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs); + +/* Stats APIs */ +void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, int id, int dir); +void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir); +void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, int id, int dir); +void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, int id, int dir); +void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id); +void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir); +int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir); +int mcs_set_force_clk_en(struct mcs *mcs, bool set); + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event); + +#endif /* MCS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c new file mode 100644 index 0000000000000000000000000000000000000000..7b620541442863fa2ba416571d3ed36d80f6f6a0 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include "mcs.h" +#include "mcs_reg.h" + +static struct mcs_ops cnf10kb_mcs_ops = { + .mcs_set_hw_capabilities = cnf10kb_mcs_set_hw_capabilities, + .mcs_parser_cfg = cnf10kb_mcs_parser_cfg, + .mcs_tx_sa_mem_map_write = cnf10kb_mcs_tx_sa_mem_map_write, + .mcs_rx_sa_mem_map_write = cnf10kb_mcs_rx_sa_mem_map_write, + .mcs_flowid_secy_map = cnf10kb_mcs_flowid_secy_map, +}; + +struct mcs_ops *cnf10kb_get_mac_ops(void) +{ + return &cnf10kb_mcs_ops; +} + +void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs) +{ + struct hwinfo *hw = mcs->hw; + + hw->tcam_entries = 64; /* TCAM entries */ + hw->secy_entries = 64; /* SecY entries */ + hw->sc_entries = 64; /* SC CAM entries */ + hw->sa_entries = 128; /* SA entries */ + hw->lmac_cnt = 4; /* lmacs/ports per mcs block */ + hw->mcs_x2p_intf = 1; /* x2p clabration intf */ + hw->mcs_blks = 7; /* MCS blocks */ +} + +void cnf10kb_mcs_parser_cfg(struct mcs *mcs) +{ + u64 reg, val; + + /* VLAN Ctag */ + val = (0x8100ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(22); + + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(0); + mcs_reg_write(mcs, reg, val); + + /* VLAN STag */ + val = (0x88a8ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(23); + + /* RX */ + reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* TX */ + reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(1); + mcs_reg_write(mcs, reg, val); + + /* Enable custom tage 0 and 1 and sectag */ + val = BIT_ULL(0) | BIT_ULL(1) | BIT_ULL(12); + + reg = MCSX_PEX_RX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); + + reg = MCSX_PEX_TX_SLAVE_ETYPE_ENABLE; + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir) +{ + u64 reg, val; + + val = (map->secy & 0x3F) | (map->ctrl_pkt & 0x1) << 6; + if (dir == MCS_RX) { + reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id); + } else { + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id); + mcs_reg_write(mcs, reg, map->sci); + val |= (map->sc & 0x3F) << 7; + reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(map->flow_id); + } + + mcs_reg_write(mcs, reg, val); +} + +void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map) +{ + u64 reg, val; + + val = (map->sa_index0 & 0x7F) | (map->sa_index1 & 0x7F) << 7; + + reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id); + mcs_reg_write(mcs, reg, val); + + reg = MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0; + val = mcs_reg_read(mcs, reg); + + if (map->rekey_ena) + val |= BIT_ULL(map->sc_id); + else + val &= ~BIT_ULL(map->sc_id); + + mcs_reg_write(mcs, reg, val); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(map->sc_id), map->sa_index0_vld); + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(map->sc_id), map->sa_index1_vld); + + mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(map->sc_id), map->tx_sa_active); +} + +void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map) +{ + u64 val, reg; + + val = (map->sa_index & 0x7F) | (map->sa_in_use << 7); + + reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an); + mcs_reg_write(mcs, reg, val); +} + +int mcs_set_force_clk_en(struct mcs *mcs, bool set) +{ + unsigned long timeout = jiffies + usecs_to_jiffies(2000); + u64 val; + + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + + if (set) { + val |= BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + + /* Poll till mcsx_mil_ip_gbl_status.mcs_ip_stats_ready value is 1 */ + while (!(mcs_reg_read(mcs, MCSX_MIL_IP_GBL_STATUS) & BIT_ULL(0))) { + if (time_after(jiffies, timeout)) { + dev_err(mcs->dev, "MCS set force clk enable failed\n"); + break; + } + } + } else { + val &= ~BIT_ULL(4); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + } + + return 0; +} + +/* TX SA interrupt is raised only if autorekey is enabled. + * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if + * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies + * SA in SA_index1 got expired else SA in SA_index0 got expired. + */ +void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs) +{ + struct mcs_intr_event event; + struct rsrc_bmap *sc_bmap; + unsigned long rekey_ena; + u64 val, sa_status; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT; + + rekey_ena = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0); + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + /* Auto rekey is enable */ + if (!test_bit(sc, &rekey_ena)) + continue; + sa_status = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(sc)); + /* Check if tx_sa_active status had changed */ + if (sa_status == mcs->tx_sa_active[sc]) + continue; + + /* SA_index0 is expired */ + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + if (sa_status) + event.sa_id = val & 0x7F; + else + event.sa_id = (val >> 7) & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} + +void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs) +{ + struct mcs_intr_event event = { 0 }; + struct rsrc_bmap *sc_bmap; + u64 val; + int sc; + + sc_bmap = &mcs->tx.sc; + + event.mcs_id = mcs->mcs_id; + event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) { + val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc)); + + if (mcs->tx_sa_active[sc]) + /* SA_index1 was used and got expired */ + event.sa_id = (val >> 7) & 0x7F; + else + /* SA_index0 was used and got expired */ + event.sa_id = val & 0x7F; + + event.pcifunc = mcs->tx.sa2pf_map[event.sa_id]; + mcs_add_intr_wq_entry(mcs, &event); + } +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h new file mode 100644 index 0000000000000000000000000000000000000000..c95a8b8f5eaf7c50513e1c7592bcc1080637c8a2 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h @@ -0,0 +1,1102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#ifndef MCS_REG_H +#define MCS_REG_H + +#include + +/* Registers */ +#define MCSX_IP_MODE 0x900c8ull +#define MCSX_MCS_TOP_SLAVE_PORT_RESET(a) ({ \ + u64 offset; \ + \ + offset = 0x408ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa28ull; \ + offset += (a) * 0x8ull; \ + offset; }) + + +#define MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x808ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa68ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_MIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0x80000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60000ull; \ + offset; }) + +#define MCSX_MIL_RX_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x900a8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x700a8ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_HIL_GLOBAL ({ \ + u64 offset; \ + \ + offset = 0xc0000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa0000ull; \ + offset; }) + +#define MCSX_LINK_LMACX_CFG(a) ({ \ + u64 offset; \ + \ + offset = 0x90000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x70000ull; \ + offset += (a) * 0x800ull; \ + offset; }) + +#define MCSX_MIL_RX_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800c8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600c8ull; \ + offset; }) + +#define MCSX_MIL_IP_GBL_STATUS ({ \ + u64 offset; \ + \ + offset = 0x800d0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x600d0ull; \ + offset; }) + +/* PAB */ +#define MCSX_PAB_RX_SLAVE_PORT_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1718ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x280ull; \ + offset += (a) * 0x40ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PORT_CFGX(a) (0x2930ull + (a) * 0x40ull) + +/* PEX registers */ +#define MCSX_PEX_RX_SLAVE_VLAN_CFGX(a) (0x3b58ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_VLAN_CFGX(a) (0x46f8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(a) (0x788ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_PORT_CONFIG(a) (0x4738ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x3fc0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x558ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4000ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x598ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4048ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5e0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4080ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x648ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4088ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x650ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4090ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x658ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x40e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6d8ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x40e8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6e0ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(a) ({ \ + u64 offset; \ + \ + offset = 0x4b60ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x7d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DAX(a) ({ \ + u64 offset; \ + \ + offset = 0x4ba0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x858ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4be8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x860ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8c8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(a) ({ \ + u64 offset; \ + \ + offset = 0x4c30ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x8d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_MAC ({ \ + u64 offset; \ + \ + offset = 0x4c80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x958ull; \ + offset; }) + +#define MCSX_PEX_TX_SLAVE_RULE_ENABLE ({ \ + u64 offset; \ + \ + offset = 0x4c88ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x960ull; \ + offset; }) + +#define MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION ({ \ + u64 offset; \ + \ + offset = 0x3b50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c0ull; \ + offset; }) + +/* CNF10K-B */ +#define MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(a) (0x4c8ull + (a) * 0x8ull) +#define MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(a) (0x748ull + (a) * 0x8ull) +#define MCSX_PEX_RX_SLAVE_ETYPE_ENABLE 0x6e8ull +#define MCSX_PEX_TX_SLAVE_ETYPE_ENABLE 0x968ull + +/* BEE */ +#define MCSX_BBE_RX_SLAVE_PADDING_CTL 0xe08ull +#define MCSX_BBE_TX_SLAVE_PADDING_CTL 0x12f8ull +#define MCSX_BBE_RX_SLAVE_CAL_ENTRY 0x180ull +#define MCSX_BBE_RX_SLAVE_CAL_LEN 0x188ull +#define MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(a) (0x290ull + (a) * 0x40ull) + +#define MCSX_BBE_RX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0xe00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x160ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x168ull; \ + offset; }) + +#define MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0xe08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x178ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e0ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x1278ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1f8ull; \ + offset; }) + +#define MCSX_BBE_TX_SLAVE_BBE_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x1280ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x1e8ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x16f0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x260ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x268ull; \ + offset; }) + +#define MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x278ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT ({ \ + u64 offset; \ + \ + offset = 0x2908ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x380ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x2910ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x388ull; \ + offset; }) + +#define MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW ({ \ + u64 offset; \ + \ + offset = 0x16f8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x398ull; \ + offset; }) + +/* CPM registers */ +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x30740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bf8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x34740ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x43f8ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x30700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x3bd8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x38780ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4c08ull; \ + offset += (a) * 0x8ull + (b) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SC_CAM_ENA(a) ({ \ + u64 offset; \ + \ + offset = 0x38740ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x4bf8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x23ee0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = (0x246e0ull + (a) * 0x10ull); \ + if (mcs->hw->mcs_blks > 1) \ + offset = (0xdd0ull + (a) * 0x8ull); \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x23E90ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbb0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_MAP_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x256e0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfd0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x27700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x17d8ull; \ + offset += (a) * 0x8ull + (b) * 0x40ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x2f700ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x37d8; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb90ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x23e48ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xb98ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x23e50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xba0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1 0x30708ull +#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(a) (0x246e8ull + (a) * 0x10ull) + +/* TX registers */ +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x51d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7c0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x55d50ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xafc0ull; \ + offset += (a) * 0x8ull + (b) * 0x20ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0 ({ \ + u64 offset; \ + \ + offset = 0x51d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa7a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3e508ull + (a) * 0x8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5550ull + (a) * 0x10ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x3ed08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5950ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_KEY_LOCKOUTX(a) ({ \ + u64 offset; \ + \ + offset = 0x3e4c0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5538ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(a) ({ \ + u64 offset; \ + \ + offset = 0x3fd10ull + (a) * 0x10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x6150ull + (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(a, b) ({ \ + u64 offset; \ + \ + offset = 0x40d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x63a0ull; \ + offset += (a) * 0x8ull + (b) * 0x80ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(a) ({ \ + u64 offset; \ + \ + offset = 0x50d10ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa3a0ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_XPN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5528ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_PN_THRESHOLD ({ \ + u64 offset; \ + \ + offset = 0x3e4b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x5530ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(a) (0x3fd18ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(a) (0x5558ull + (a) * 0x10ull) +#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1 0x51d18ull +#define MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(a) (0x5b50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(a) (0x5d50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(a) (0x5f50 + (a) * 0x8ull) +#define MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0 0x5500ull + +/* CSE */ +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x9680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc018ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xbe18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x8680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xca18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x6680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x7680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xc618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x5e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xdc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(a)({ \ + u64 offset; \ + \ + offset = 0x5680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xda18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0xd680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xce18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x16a80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x16680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec38ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x16880ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xec18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0xfe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xde18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(a) ({ \ + u64 offset; \ + \ + offset = 0x10e80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xe218ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(a) ({ \ + u64 offset; \ + \ + offset = 0xae80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd418ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(a) ({ \ + u64 offset; \ + \ + offset = 0xc680ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd618ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(a) ({ \ + u64 offset; \ + \ + offset = 0xce80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xd818ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(a) ({ \ + u64 offset; \ + \ + offset = 0xbe80ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xcc18ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x52a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9c0ull; \ + offset; }) + +#define MCSX_CSE_RX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x52b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x9d8ull; \ + offset; }) + +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(a) (0xe680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(a) (0xde80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(a) (0xa680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(a) (0xd218 + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDX(a) (0xd018ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(a) (0xee80ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(a) (0xb680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(a) (0xf680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(a) (0x12680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(a) (0x15680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(a) (0x13680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(a) (0x11680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(a) (0x14680ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(a) (0xec58ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(a) (0xea18ull + (a) * 0x8ull) +#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(a) (0xe618ull + (a) * 0x8ull) + +/* CSE TX */ +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCOMMONOCTETSX(a) (0x18440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1c440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1bc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xee78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1b440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1ac40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfc78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x1a440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfa78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(a) ({ \ + u64 offset; \ + \ + offset = 0x18c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(a) ({ \ + u64 offset; \ + \ + offset = 0x19c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xf878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10878ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x17440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10678ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSCTRLPORTDISABLEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1e440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xfe78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(a) ({ \ + u64 offset; \ + \ + offset = 0x23240ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10ed8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(a) ({ \ + u64 offset; \ + \ + offset = 0x22c40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e98ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x22e40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10e78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x20440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10c78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1fc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10a78ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(a) ({ \ + u64 offset; \ + \ + offset = 0x23040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x110d8ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(a) ({ \ + u64 offset; \ + \ + offset = 0x1dc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10278ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(a) ({ \ + u64 offset; \ + \ + offset = 0x1d440ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10478ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(a) ({ \ + u64 offset; \ + \ + offset = 0x1cc40ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x10078ull; \ + offset += (a) * 0x8ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_CTRL ({ \ + u64 offset; \ + \ + offset = 0x54a0ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa00ull; \ + offset; }) + +#define MCSX_CSE_TX_SLAVE_STATS_CLEAR ({ \ + u64 offset; \ + \ + offset = 0x54b8ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xa18ull; \ + offset; }) + +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(a) (0x1f440ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(a) (0x1ec40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSEARLYPREEMPTERRX(a) (0x10eb8ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(a) (0x21c40ull + (a) * 0x8ull) +#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(a) (0x20c40ull + (a) * 0x8ull) + +#define MCSX_IP_INT ({ \ + u64 offset; \ + \ + offset = 0x80028ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60028ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1S ({ \ + u64 offset; \ + \ + offset = 0x80040ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60040ull; \ + offset; }) + +#define MCSX_IP_INT_ENA_W1C ({ \ + u64 offset; \ + \ + offset = 0x80038ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x60038ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM ({ \ + u64 offset; \ + \ + offset = 0xc20ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xab8ull; \ + offset; }) + +#define MCSX_TOP_SLAVE_INT_SUM_ENB ({ \ + u64 offset; \ + \ + offset = 0xc28ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xac0ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT ({ \ + u64 offset; \ + \ + offset = 0x23c00ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x0ad8ull; \ + offset; }) + +#define MCSX_CPM_RX_SLAVE_RX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x23c08ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0xae0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT ({ \ + u64 offset; \ + \ + offset = 0x3d490ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a0ull; \ + offset; }) + +#define MCSX_CPM_TX_SLAVE_TX_INT_ENB ({ \ + u64 offset; \ + \ + offset = 0x3d498ull; \ + if (mcs->hw->mcs_blks > 1) \ + offset = 0x54a8ull; \ + offset; }) + +#endif diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c new file mode 100644 index 0000000000000000000000000000000000000000..fa8029a940689eb605c24dbe1dcbfaeeed58d0b4 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell CN10K MCS driver + * + * Copyright (C) 2022 Marvell. + */ + +#include +#include +#include +#include + +#include "mcs.h" +#include "rvu.h" +#include "lmac_common.h" + +#define M(_name, _id, _fn_name, _req_type, _rsp_type) \ +static struct _req_type __maybe_unused \ +*otx2_mbox_alloc_msg_ ## _fn_name(struct rvu *rvu, int devid) \ +{ \ + struct _req_type *req; \ + \ + req = (struct _req_type *)otx2_mbox_alloc_msg_rsp( \ + &rvu->afpf_wq_info.mbox_up, devid, sizeof(struct _req_type), \ + sizeof(struct _rsp_type)); \ + if (!req) \ + return NULL; \ + req->hdr.sig = OTX2_MBOX_REQ_SIG; \ + req->hdr.id = _id; \ + return req; \ +} + +MBOX_UP_MCS_MESSAGES +#undef M + +int rvu_mbox_handler_mcs_set_lmac_mode(struct rvu *rvu, + struct mcs_set_lmac_mode *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (BIT_ULL(req->lmac_id) & mcs->hw->lmac_bmap) + mcs_set_lmac_mode(mcs, req->lmac_id, req->mode); + + return 0; +} + +int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event) +{ + struct mcs_intrq_entry *qentry; + u16 pcifunc = event->pcifunc; + struct rvu *rvu = mcs->rvu; + struct mcs_pfvf *pfvf; + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + event->intr_mask &= pfvf->intr_mask; + + /* Check PF/VF interrupt notification is enabled */ + if (!(pfvf->intr_mask && event->intr_mask)) + return 0; + + qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); + if (!qentry) + return -ENOMEM; + + qentry->intr_event = *event; + spin_lock(&rvu->mcs_intrq_lock); + list_add_tail(&qentry->node, &rvu->mcs_intrq_head); + spin_unlock(&rvu->mcs_intrq_lock); + queue_work(rvu->mcs_intr_wq, &rvu->mcs_intr_work); + + return 0; +} + +static int mcs_notify_pfvf(struct mcs_intr_event *event, struct rvu *rvu) +{ + struct mcs_intr_info *req; + int err, pf; + + pf = rvu_get_pf(event->pcifunc); + + req = otx2_mbox_alloc_msg_mcs_intr_notify(rvu, pf); + if (!req) + return -ENOMEM; + + req->mcs_id = event->mcs_id; + req->intr_mask = event->intr_mask; + req->sa_id = event->sa_id; + req->hdr.pcifunc = event->pcifunc; + req->lmac_id = event->lmac_id; + + otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pf); + err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pf); + if (err) + dev_warn(rvu->dev, "MCS notification to pf %d failed\n", pf); + + return 0; +} + +static void mcs_intr_handler_task(struct work_struct *work) +{ + struct rvu *rvu = container_of(work, struct rvu, mcs_intr_work); + struct mcs_intrq_entry *qentry; + struct mcs_intr_event *event; + unsigned long flags; + + do { + spin_lock_irqsave(&rvu->mcs_intrq_lock, flags); + qentry = list_first_entry_or_null(&rvu->mcs_intrq_head, + struct mcs_intrq_entry, + node); + if (qentry) + list_del(&qentry->node); + + spin_unlock_irqrestore(&rvu->mcs_intrq_lock, flags); + if (!qentry) + break; /* nothing more to process */ + + event = &qentry->intr_event; + + mcs_notify_pfvf(event, rvu); + kfree(qentry); + } while (1); +} + +int rvu_mbox_handler_mcs_intr_cfg(struct rvu *rvu, + struct mcs_intr_cfg *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_pfvf *pfvf; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* Check if it is PF or VF */ + if (pcifunc & RVU_PFVF_FUNC_MASK) + pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)]; + else + pfvf = &mcs->pf[rvu_get_pf(pcifunc)]; + + mcs->pf_map[0] = pcifunc; + pfvf->intr_mask = req->intr_mask; + + return 0; +} + +int rvu_mbox_handler_mcs_get_hw_info(struct rvu *rvu, + struct msg_req *req, + struct mcs_hw_info *rsp) +{ + struct mcs *mcs; + + if (!rvu->mcs_blk_cnt) + return MCS_AF_ERR_NOT_MAPPED; + + /* MCS resources are same across all blocks */ + mcs = mcs_get_pdata(0); + rsp->num_mcs_blks = rvu->mcs_blk_cnt; + rsp->tcam_entries = mcs->hw->tcam_entries; + rsp->secy_entries = mcs->hw->secy_entries; + rsp->sc_entries = mcs->hw->sc_entries; + rsp->sa_entries = mcs->hw->sa_entries; + return 0; +} + +int rvu_mbox_handler_mcs_port_reset(struct rvu *rvu, struct mcs_port_reset_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_reset_port(mcs, req->port_id, req->reset); + + return 0; +} + +int rvu_mbox_handler_mcs_clear_stats(struct rvu *rvu, + struct mcs_clear_stats *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&mcs->stats_lock); + if (req->all) + mcs_clear_all_stats(mcs, pcifunc, req->dir); + else + mcs_clear_stats(mcs, req->type, req->id, req->dir); + + mutex_unlock(&mcs->stats_lock); + return 0; +} + +int rvu_mbox_handler_mcs_get_flowid_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_flowid_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* In CNF10K-B, before reading the statistics, + * MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP needs to be set + * to get accurate statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_flowid_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + /* Clear MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP after reading + * the statistics + */ + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_secy_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_secy_stats *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + + if (req->dir == MCS_RX) + mcs_get_rx_secy_stats(mcs, rsp, req->id); + else + mcs_get_tx_secy_stats(mcs, rsp, req->id); + + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sc_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sc_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sc_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_sa_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_sa_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_sa_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_get_port_stats(struct rvu *rvu, + struct mcs_stats_req *req, + struct mcs_port_stats *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, true); + + mutex_lock(&mcs->stats_lock); + mcs_get_port_stats(mcs, rsp, req->id, req->dir); + mutex_unlock(&mcs->stats_lock); + + if (mcs->hw->mcs_blks > 1) + mcs_set_force_clk_en(mcs, false); + + return 0; +} + +int rvu_mbox_handler_mcs_set_active_lmac(struct rvu *rvu, + struct mcs_set_active_lmac *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + if (!mcs) + return MCS_AF_ERR_NOT_MAPPED; + + mcs->hw->lmac_bmap = req->lmac_bmap; + mcs_set_lmac_channels(req->mcs_id, req->chan_base); + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_set(struct rvu *rvu, struct mcs_port_cfg_set_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_set_port_cfg(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_port_cfg_get(struct rvu *rvu, struct mcs_port_cfg_get_req *req, + struct mcs_port_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id))) + return -EINVAL; + + mcs_get_port_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mbox_handler_mcs_custom_tag_cfg_get(struct rvu *rvu, struct mcs_custom_tag_cfg_get_req *req, + struct mcs_custom_tag_cfg_get_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_get_custom_tag_cfg(mcs, req, rsp); + + return 0; +} + +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc) +{ + struct mcs *mcs; + int mcs_id; + + /* CNF10K-B mcs0-6 are mapped to RPM2-8*/ + if (rvu->mcs_blk_cnt > 1) { + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + } else { + /* CN10K-B has only one mcs block */ + mcs = mcs_get_pdata(0); + mcs_free_all_rsrc(mcs, MCS_RX, pcifunc); + mcs_free_all_rsrc(mcs, MCS_TX, pcifunc); + } + return 0; +} + +int rvu_mbox_handler_mcs_flowid_ena_entry(struct rvu *rvu, + struct mcs_flowid_ena_dis_entry *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_ena_dis_flowid_entry(mcs, req->flow_id, req->dir, req->ena); + return 0; +} + +int rvu_mbox_handler_mcs_pn_table_write(struct rvu *rvu, + struct mcs_pn_table_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_pn_table_write(mcs, req->pn_id, req->next_pn, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_set_pn_threshold(struct rvu *rvu, + struct mcs_set_pn_threshold *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_pn_threshold_set(mcs, req); + + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_sa_map_write(struct rvu *rvu, + struct mcs_rx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_rx_sa_mem_map_write(mcs, req); + return 0; +} + +int rvu_mbox_handler_mcs_tx_sc_sa_map_write(struct rvu *rvu, + struct mcs_tx_sc_sa_map *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs->mcs_ops->mcs_tx_sa_mem_map_write(mcs, req); + mcs->tx_sa_active[req->sc_id] = req->tx_sa_active; + + return 0; +} + +int rvu_mbox_handler_mcs_sa_plcy_write(struct rvu *rvu, + struct mcs_sa_plcy_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + for (i = 0; i < req->sa_cnt; i++) + mcs_sa_plcy_write(mcs, &req->plcy[i][0], + req->sa_index[i], req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_rx_sc_cam_write(struct rvu *rvu, + struct mcs_rx_sc_cam_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + mcs_rx_sc_cam_write(mcs, req->sci, req->secy_id, req->sc_id); + return 0; +} + +int rvu_mbox_handler_mcs_secy_plcy_write(struct rvu *rvu, + struct mcs_secy_plcy_write_req *req, + struct msg_rsp *rsp) +{ struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mcs_secy_plcy_write(mcs, req->plcy, + req->secy_id, req->dir); + return 0; +} + +int rvu_mbox_handler_mcs_flowid_entry_write(struct rvu *rvu, + struct mcs_flowid_entry_write_req *req, + struct msg_rsp *rsp) +{ + struct secy_mem_map map; + struct mcs *mcs; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + /* TODO validate the flowid */ + mcs_flowid_entry_write(mcs, req->data, req->mask, + req->flow_id, req->dir); + map.secy = req->secy_id; + map.sc = req->sc_id; + map.ctrl_pkt = req->ctrl_pkt; + map.flow_id = req->flow_id; + map.sci = req->sci; + mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, req->dir); + if (req->ena) + mcs_ena_dis_flowid_entry(mcs, req->flow_id, + req->dir, true); + return 0; +} + +int rvu_mbox_handler_mcs_free_resources(struct rvu *rvu, + struct mcs_free_rsrc_req *req, + struct msg_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + /* Free all the cam resources mapped to PF/VF */ + if (req->all) { + rc = mcs_free_all_rsrc(mcs, req->dir, pcifunc); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + rc = mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, req->rsrc_id, pcifunc); + mcs_ena_dis_flowid_entry(mcs, req->rsrc_id, req->dir, false); + break; + case MCS_RSRC_TYPE_SECY: + rc = mcs_free_rsrc(&map->secy, map->secy2pf_map, req->rsrc_id, pcifunc); + mcs_clear_secy_plcy(mcs, req->rsrc_id, req->dir); + break; + case MCS_RSRC_TYPE_SC: + rc = mcs_free_rsrc(&map->sc, map->sc2pf_map, req->rsrc_id, pcifunc); + /* Disable SC CAM only on RX side */ + if (req->dir == MCS_RX) + mcs_ena_dis_sc_cam_entry(mcs, req->rsrc_id, false); + break; + case MCS_RSRC_TYPE_SA: + rc = mcs_free_rsrc(&map->sa, map->sa2pf_map, req->rsrc_id, pcifunc); + break; + } +exit: + mutex_unlock(&rvu->rsrc_lock); + return rc; +} + +int rvu_mbox_handler_mcs_alloc_resources(struct rvu *rvu, + struct mcs_alloc_rsrc_req *req, + struct mcs_alloc_rsrc_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id, i; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + if (req->dir == MCS_RX) + map = &mcs->rx; + else + map = &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + if (req->all) { + rsrc_id = mcs_alloc_all_rsrc(mcs, &rsp->flow_ids[0], + &rsp->secy_ids[0], + &rsp->sc_ids[0], + &rsp->sa_ids[0], + &rsp->sa_ids[1], + pcifunc, req->dir); + goto exit; + } + + switch (req->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->flow_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SECY: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->secy_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SC: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sc_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + case MCS_RSRC_TYPE_SA: + for (i = 0; i < req->rsrc_cnt; i++) { + rsrc_id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc); + if (rsrc_id < 0) + goto exit; + rsp->sa_ids[i] = rsrc_id; + rsp->rsrc_cnt++; + } + break; + } + + rsp->rsrc_type = req->rsrc_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + rsp->all = req->all; + +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs resources for PCIFUNC:%d\n", pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return 0; +} + +int rvu_mbox_handler_mcs_alloc_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_alloc_ctrl_pkt_rule_req *req, + struct mcs_alloc_ctrl_pkt_rule_rsp *rsp) +{ + u16 pcifunc = req->hdr.pcifunc; + struct mcs_rsrc_map *map; + struct mcs *mcs; + int rsrc_id; + u16 offset; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx; + + mutex_lock(&rvu->rsrc_lock); + + switch (req->rule_type) { + case MCS_CTRL_PKT_RULE_TYPE_ETH: + offset = MCS_CTRLPKT_ETYPE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_DA: + offset = MCS_CTRLPKT_DA_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_RANGE: + offset = MCS_CTRLPKT_DA_RANGE_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_COMBO: + offset = MCS_CTRLPKT_COMBO_RULE_OFFSET; + break; + case MCS_CTRL_PKT_RULE_TYPE_MAC: + offset = MCS_CTRLPKT_MAC_EN_RULE_OFFSET; + break; + } + + rsrc_id = mcs_alloc_ctrlpktrule(&map->ctrlpktrule, map->ctrlpktrule2pf_map, offset, + pcifunc); + if (rsrc_id < 0) + goto exit; + + rsp->rule_idx = rsrc_id; + rsp->rule_type = req->rule_type; + rsp->dir = req->dir; + rsp->mcs_id = req->mcs_id; + + mutex_unlock(&rvu->rsrc_lock); + return 0; +exit: + if (rsrc_id < 0) + dev_err(rvu->dev, "Failed to allocate the mcs ctrl pkt rule for PCIFUNC:%d\n", + pcifunc); + mutex_unlock(&rvu->rsrc_lock); + return rsrc_id; +} + +int rvu_mbox_handler_mcs_free_ctrl_pkt_rule(struct rvu *rvu, + struct mcs_free_ctrl_pkt_rule_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + mutex_lock(&rvu->rsrc_lock); + + rc = mcs_free_ctrlpktrule(mcs, req); + + mutex_unlock(&rvu->rsrc_lock); + + return rc; +} + +int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, + struct mcs_ctrl_pkt_rule_write_req *req, + struct msg_rsp *rsp) +{ + struct mcs *mcs; + int rc; + + if (req->mcs_id >= rvu->mcs_blk_cnt) + return MCS_AF_ERR_INVALID_MCSID; + + mcs = mcs_get_pdata(req->mcs_id); + + rc = mcs_ctrlpktrule_write(mcs, req); + + return rc; +} + +static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) +{ + struct mcs *mcs = mcs_get_pdata(0); + unsigned long lmac_bmap; + int cgx, lmac, port; + + for (port = 0; port < mcs->hw->lmac_cnt; port++) { + cgx = port / rvu->hw->lmac_per_cgx; + lmac = port % rvu->hw->lmac_per_cgx; + if (!is_lmac_valid(rvu_cgx_pdata(cgx, rvu), lmac)) + continue; + set_bit(port, &lmac_bmap); + } + mcs->hw->lmac_bmap = lmac_bmap; +} + +int rvu_mcs_init(struct rvu *rvu) +{ + struct rvu_hwinfo *hw = rvu->hw; + int lmac, err = 0, mcs_id; + struct mcs *mcs; + + rvu->mcs_blk_cnt = mcs_get_blkcnt(); + + if (!rvu->mcs_blk_cnt) + return 0; + + /* Needed only for CN10K-B */ + if (rvu->mcs_blk_cnt == 1) { + err = mcs_set_lmac_channels(0, hw->cgx_chan_base); + if (err) + return err; + /* Set active lmacs */ + rvu_mcs_set_lmac_bmap(rvu); + } + + /* Install default tcam bypass entry and set port to operational mode */ + for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) { + mcs = mcs_get_pdata(mcs_id); + mcs_install_flowid_bypass_entry(mcs); + for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) + mcs_set_lmac_mode(mcs, lmac, 0); + + mcs->rvu = rvu; + + /* Allocated memory for PFVF data */ + mcs->pf = devm_kcalloc(mcs->dev, hw->total_pfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->pf) + return -ENOMEM; + + mcs->vf = devm_kcalloc(mcs->dev, hw->total_vfs, + sizeof(struct mcs_pfvf), GFP_KERNEL); + if (!mcs->vf) + return -ENOMEM; + } + + /* Initialize the wq for handling mcs interrupts */ + INIT_LIST_HEAD(&rvu->mcs_intrq_head); + INIT_WORK(&rvu->mcs_intr_work, mcs_intr_handler_task); + rvu->mcs_intr_wq = alloc_workqueue("mcs_intr_wq", 0, 0); + if (!rvu->mcs_intr_wq) { + dev_err(rvu->dev, "mcs alloc workqueue failed\n"); + return -ENOMEM; + } + + return err; +} + +void rvu_mcs_exit(struct rvu *rvu) +{ + if (!rvu->mcs_intr_wq) + return; + + flush_workqueue(rvu->mcs_intr_wq); + destroy_workqueue(rvu->mcs_intr_wq); + rvu->mcs_intr_wq = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c index 67a6821d2dff0f6c3ef3c3ca79e6172375844a46..3411e2e47d46b70976c14eff1411c11394539036 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "ptp.h" #include "mbox.h" @@ -50,12 +52,23 @@ #define PTP_CLOCK_COMP 0xF18ULL #define PTP_TIMESTAMP 0xF20ULL #define PTP_CLOCK_SEC 0xFD0ULL +#define PTP_SEC_ROLLOVER 0xFD8ULL #define CYCLE_MULT 1000 static struct ptp *first_ptp_block; static const struct pci_device_id ptp_id_table[]; +static bool is_ptp_dev_cnf10kb(struct ptp *ptp) +{ + return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false; +} + +static bool is_ptp_dev_cn10k(struct ptp *ptp) +{ + return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false; +} + static bool cn10k_ptp_errata(struct ptp *ptp) { if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP || @@ -72,6 +85,43 @@ static bool is_ptp_tsfmt_sec_nsec(struct ptp *ptp) return false; } +static enum hrtimer_restart ptp_reset_thresh(struct hrtimer *hrtimer) +{ + struct ptp *ptp = container_of(hrtimer, struct ptp, hrtimer); + ktime_t curr_ts = ktime_get(); + ktime_t delta_ns, period_ns; + u64 ptp_clock_hi; + + /* calculate the elapsed time since last restart */ + delta_ns = ktime_to_ns(ktime_sub(curr_ts, ptp->last_ts)); + + /* if the ptp clock value has crossed 0.5 seconds, + * its too late to update pps threshold value, so + * update threshold after 1 second. + */ + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + if (ptp_clock_hi > 500000000) { + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - ptp_clock_hi)); + } else { + writeq(500000000, ptp->reg_base + PTP_PPS_THRESH_HI); + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - delta_ns)); + } + + hrtimer_forward_now(hrtimer, period_ns); + ptp->last_ts = curr_ts; + + return HRTIMER_RESTART; +} + +static void ptp_hrtimer_start(struct ptp *ptp, ktime_t start_ns) +{ + ktime_t period_ns; + + period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - start_ns)); + hrtimer_start(&ptp->hrtimer, period_ns, HRTIMER_MODE_REL); + ptp->last_ts = ktime_get(); +} + static u64 read_ptp_tstmp_sec_nsec(struct ptp *ptp) { u64 sec, sec1, nsec; @@ -246,6 +296,10 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* sclk is in MHz */ ptp->clock_rate = sclk * 1000000; + /* Program the seconds rollover value to 1 second */ + if (is_ptp_dev_cnf10kb(ptp)) + writeq(0x3b9aca00, ptp->reg_base + PTP_SEC_ROLLOVER); + /* Enable PTP clock */ clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG); @@ -270,6 +324,18 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) /* Set 50% duty cycle for 1Hz output */ writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_HI_INCR); writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_LO_INCR); + if (cn10k_ptp_errata(ptp)) { + /* The ptp_clock_hi rollsover to zero once clock cycle before it + * reaches one second boundary. so, program the pps_lo_incr in + * such a way that the pps threshold value comparison at one + * second boundary will succeed and pps edge changes. After each + * one second boundary, the hrtimer handler will be invoked and + * reprograms the pps threshold value. + */ + ptp->clock_period = NSEC_PER_SEC / ptp->clock_rate; + writeq((0x1dcd6500ULL - ptp->clock_period) << 32, + ptp->reg_base + PTP_PPS_LO_INCR); + } if (cn10k_ptp_errata(ptp)) clock_comp = ptp_calc_adjusted_comp(ptp->clock_rate); @@ -282,14 +348,39 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts) static int ptp_get_tstmp(struct ptp *ptp, u64 *clk) { - *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + u64 timestamp; + + if (is_ptp_dev_cn10k(ptp)) { + timestamp = readq(ptp->reg_base + PTP_TIMESTAMP); + *clk = (timestamp >> 32) * NSEC_PER_SEC + (timestamp & 0xFFFFFFFF); + } else { + *clk = readq(ptp->reg_base + PTP_TIMESTAMP); + } return 0; } static int ptp_set_thresh(struct ptp *ptp, u64 thresh) { - writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + if (!cn10k_ptp_errata(ptp)) + writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI); + + return 0; +} + +static int ptp_extts_on(struct ptp *ptp, int on) +{ + u64 ptp_clock_hi; + + if (cn10k_ptp_errata(ptp)) { + if (on) { + ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI); + ptp_hrtimer_start(ptp, (ktime_t)ptp_clock_hi); + } else { + if (hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + } + } return 0; } @@ -329,6 +420,11 @@ static int ptp_probe(struct pci_dev *pdev, else ptp->read_ptp_tstmp = &read_ptp_tstmp_nsec; + if (cn10k_ptp_errata(ptp)) { + hrtimer_init(&ptp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + ptp->hrtimer.function = ptp_reset_thresh; + } + return 0; error_free: @@ -353,6 +449,9 @@ static void ptp_remove(struct pci_dev *pdev) struct ptp *ptp = pci_get_drvdata(pdev); u64 clock_cfg; + if (cn10k_ptp_errata(ptp) && hrtimer_active(&ptp->hrtimer)) + hrtimer_cancel(&ptp->hrtimer); + if (IS_ERR_OR_NULL(ptp)) return; @@ -420,6 +519,9 @@ int rvu_mbox_handler_ptp_op(struct rvu *rvu, struct ptp_req *req, case PTP_OP_SET_THRESH: err = ptp_set_thresh(rvu->ptp, req->thresh); break; + case PTP_OP_EXTTS_ON: + err = ptp_extts_on(rvu->ptp, req->extts_on); + break; default: err = -EINVAL; break; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h index 95a955159f40aa7586090c37d5cfff7751e39ae9..b9d92abc38440a452e57172f7e59f0d83cd8e422 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h @@ -17,7 +17,10 @@ struct ptp { void __iomem *reg_base; u64 (*read_ptp_tstmp)(struct ptp *ptp); spinlock_t ptp_lock; /* lock */ + struct hrtimer hrtimer; + ktime_t last_ts; u32 clock_rate; + u32 clock_period; }; struct ptp *ptp_get(void); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c index ef59de43b11e6dfbf62cae368a79b76597b1df2a..a70e1153fa04bb4520aae88ad376252ee8098a11 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c @@ -415,11 +415,26 @@ void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable) return; cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_CFG); - if (enable) + if (enable) { cfg |= RPMX_RX_TS_PREPEND; - else + cfg |= RPMX_TX_PTP_1S_SUPPORT; + } else { cfg &= ~RPMX_RX_TS_PREPEND; + cfg &= ~RPMX_TX_PTP_1S_SUPPORT; + } + rpm_write(rpm, lmac_id, RPMX_CMRX_CFG, cfg); + + cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE); + + if (enable) { + cfg |= RPMX_ONESTEP_ENABLE; + cfg &= ~RPMX_TS_BINARY_MODE; + } else { + cfg &= ~RPMX_ONESTEP_ENABLE; + } + + rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE, cfg); } int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h index c2bd6e54ea51e42c655feca962c506250a4bddc6..77f2ef9e142526507dd2342fa00c9d6c12f67595 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h @@ -16,6 +16,7 @@ /* Registers */ #define RPMX_CMRX_CFG 0x00 #define RPMX_RX_TS_PREPEND BIT_ULL(22) +#define RPMX_TX_PTP_1S_SUPPORT BIT_ULL(17) #define RPMX_CMRX_SW_INT 0x180 #define RPMX_CMRX_SW_INT_W1S 0x188 #define RPMX_CMRX_SW_INT_ENA_W1S 0x198 @@ -72,6 +73,10 @@ #define RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA 0x8108 #define RPM_DEFAULT_PAUSE_TIME 0x7FF +#define RPMX_MTI_MAC100X_XIF_MODE 0x8100 +#define RPMX_ONESTEP_ENABLE BIT_ULL(5) +#define RPMX_TS_BINARY_MODE BIT_ULL(11) + /* Function Declarations */ int rpm_get_nr_lmacs(void *rpmd); u8 rpm_get_lmac_type(void *rpmd, int lmac_id); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 7282a826d81e0f6a9ffcf7d2802e3a450f6e44ab..3f5e09b77d4bd680510593a6571483dbaa332c02 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -16,6 +16,7 @@ #include "rvu.h" #include "rvu_reg.h" #include "ptp.h" +#include "mcs.h" #include "rvu_trace.h" #include "rvu_npc_hash.h" @@ -23,8 +24,6 @@ #define DRV_NAME "rvu_af" #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver" -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc); - static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, struct rvu_block *block, int lf); static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf, @@ -418,7 +417,7 @@ void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf) *hwvf = cfg & 0xFFF; } -static int rvu_get_hwvf(struct rvu *rvu, int pcifunc) +int rvu_get_hwvf(struct rvu *rvu, int pcifunc) { int pf, func; u64 cfg; @@ -1159,6 +1158,12 @@ cpt: rvu_program_channels(rvu); + err = rvu_mcs_init(rvu); + if (err) { + dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__); + goto nix_err; + } + return 0; nix_err: @@ -3293,6 +3298,7 @@ err_mbox: err_hwsetup: rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_reset_all_blocks(rvu); rvu_free_hw_resources(rvu); rvu_clear_rvum_blk_revid(rvu); @@ -3319,6 +3325,7 @@ static void rvu_remove(struct pci_dev *pdev) rvu_flr_wq_destroy(rvu); rvu_cgx_exit(rvu); rvu_fwdata_exit(rvu); + rvu_mcs_exit(rvu); rvu_mbox_destroy(&rvu->afpf_wq_info); rvu_disable_sriov(rvu); rvu_reset_all_blocks(rvu); @@ -3354,12 +3361,18 @@ static int __init rvu_init_module(void) if (err < 0) goto ptp_err; + err = pci_register_driver(&mcs_driver); + if (err < 0) + goto mcs_err; + err = pci_register_driver(&rvu_driver); if (err < 0) goto rvu_err; return 0; rvu_err: + pci_unregister_driver(&mcs_driver); +mcs_err: pci_unregister_driver(&ptp_driver); ptp_err: pci_unregister_driver(&cgx_driver); @@ -3370,6 +3383,7 @@ ptp_err: static void __exit rvu_cleanup_module(void) { pci_unregister_driver(&rvu_driver); + pci_unregister_driver(&mcs_driver); pci_unregister_driver(&ptp_driver); pci_unregister_driver(&cgx_driver); } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index d15bc443335dba20c2fcb07a62bbdc2a5b9bf4af..76474385a6027c8cf62df877cc1530880e694bed 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -25,6 +25,8 @@ /* Subsystem Device ID */ #define PCI_SUBSYS_DEVID_96XX 0xB200 #define PCI_SUBSYS_DEVID_CN10K_A 0xB900 +#define PCI_SUBSYS_DEVID_CNF10K_B 0xBC00 +#define PCI_SUBSYS_DEVID_CN10K_B 0xBD00 /* PCI BAR nos */ #define PCI_AF_REG_BAR_NUM 0 @@ -62,6 +64,10 @@ struct rvu_debugfs { struct dentry *nix; struct dentry *npc; struct dentry *cpt; + struct dentry *mcs_root; + struct dentry *mcs; + struct dentry *mcs_rx; + struct dentry *mcs_tx; struct dump_ctx npa_aura_ctx; struct dump_ctx npa_pool_ctx; struct dump_ctx nix_cq_ctx; @@ -497,6 +503,8 @@ struct rvu { struct ptp *ptp; + int mcs_blk_cnt; + #ifdef CONFIG_DEBUG_FS struct rvu_debugfs rvu_dbg; #endif @@ -504,6 +512,12 @@ struct rvu { /* RVU switch implementation over NPC with DMAC rules */ struct rvu_switch rswitch; + + struct work_struct mcs_intr_work; + struct workqueue_struct *mcs_intr_wq; + struct list_head mcs_intrq_head; + /* mcs interrupt queue lock */ + spinlock_t mcs_intrq_lock; }; static inline void rvu_write64(struct rvu *rvu, u64 block, u64 offset, u64 val) @@ -868,4 +882,11 @@ void rvu_switch_update_rules(struct rvu *rvu, u16 pcifunc); int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, u64 pkind, u8 var_len_off, u8 var_len_off_mask, u8 shift_dir); +int rvu_get_hwvf(struct rvu *rvu, int pcifunc); + +/* CN10K MCS */ +int rvu_mcs_init(struct rvu *rvu); +int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc); +void rvu_mcs_exit(struct rvu *rvu); + #endif /* RVU_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index f42a09f04b256cd1958d53915c130467fe14b766..a1970ebedf950cf6e69262abb42ab26ff25764df 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -19,6 +19,7 @@ #include "lmac_common.h" #include "npc.h" #include "rvu_npc_hash.h" +#include "mcs.h" #define DEBUGFS_DIR_NAME "octeontx2" @@ -227,6 +228,350 @@ static const struct file_operations rvu_dbg_##name##_fops = { \ static void print_nix_qsize(struct seq_file *filp, struct rvu_pfvf *pfvf); +static int rvu_dbg_mcs_port_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_port_stats stats; + int lmac; + + seq_puts(filp, "\n port stats\n"); + mutex_lock(&mcs->stats_lock); + for_each_set_bit(lmac, &mcs->hw->lmac_bmap, mcs->hw->lmac_cnt) { + mcs_get_port_stats(mcs, &stats, lmac, dir); + seq_printf(filp, "port%d: Tcam Miss: %lld\n", lmac, stats.tcam_miss_cnt); + seq_printf(filp, "port%d: Parser errors: %lld\n", lmac, stats.parser_err_cnt); + + if (dir == MCS_RX && mcs->hw->mcs_blks > 1) + seq_printf(filp, "port%d: Preempt error: %lld\n", lmac, + stats.preempt_err_cnt); + if (dir == MCS_TX) + seq_printf(filp, "port%d: Sectag insert error: %lld\n", lmac, + stats.sectag_insert_err_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_port_stats, mcs_rx_port_stats_display, NULL); + +static int rvu_dbg_mcs_tx_port_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_port_stats, mcs_tx_port_stats_display, NULL); + +static int rvu_dbg_mcs_sa_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_sa_stats stats; + struct rsrc_bmap *map; + int sa_id; + + if (dir == MCS_TX) { + map = &mcs->tx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n TX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_TX); + seq_printf(filp, "sa%d: Pkts encrypted: %lld\n", sa_id, + stats.pkt_encrypt_cnt); + + seq_printf(filp, "sa%d: Pkts protected: %lld\n", sa_id, + stats.pkt_protected_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; + } + + /* RX stats */ + map = &mcs->rx.sa; + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) { + seq_puts(filp, "\n RX SA stats\n"); + mcs_get_sa_stats(mcs, &stats, sa_id, MCS_RX); + seq_printf(filp, "sa%d: Invalid pkts: %lld\n", sa_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sa%d: Pkts no sa error: %lld\n", sa_id, stats.pkt_nosaerror_cnt); + seq_printf(filp, "sa%d: Pkts not valid: %lld\n", sa_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sa%d: Pkts ok: %lld\n", sa_id, stats.pkt_ok_cnt); + seq_printf(filp, "sa%d: Pkts no sa: %lld\n", sa_id, stats.pkt_nosa_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_rx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sa_stats, mcs_rx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sa_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sa_stats, mcs_tx_sa_stats_display, NULL); + +static int rvu_dbg_mcs_tx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->tx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_TX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Pkts encrypted: %lld\n", sc_id, stats.pkt_encrypt_cnt); + seq_printf(filp, "sc%d: Pkts protected: %lld\n", sc_id, stats.pkt_protected_cnt); + + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets encrypted: %lld\n", sc_id, + stats.octet_encrypt_cnt); + seq_printf(filp, "sc%d: Octets protected: %lld\n", sc_id, + stats.octet_protected_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_sc_stats, mcs_tx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_rx_sc_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_sc_stats stats; + struct rsrc_bmap *map; + int sc_id; + + map = &mcs->rx.sc; + seq_puts(filp, "\n SC stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) { + mcs_get_sc_stats(mcs, &stats, sc_id, MCS_RX); + seq_printf(filp, "\n=======sc%d======\n\n", sc_id); + seq_printf(filp, "sc%d: Cam hits: %lld\n", sc_id, stats.hit_cnt); + seq_printf(filp, "sc%d: Invalid pkts: %lld\n", sc_id, stats.pkt_invalid_cnt); + seq_printf(filp, "sc%d: Late pkts: %lld\n", sc_id, stats.pkt_late_cnt); + seq_printf(filp, "sc%d: Notvalid pkts: %lld\n", sc_id, stats.pkt_notvalid_cnt); + seq_printf(filp, "sc%d: Unchecked pkts: %lld\n", sc_id, stats.pkt_unchecked_cnt); + + if (mcs->hw->mcs_blks > 1) { + seq_printf(filp, "sc%d: Delay pkts: %lld\n", sc_id, stats.pkt_delay_cnt); + seq_printf(filp, "sc%d: Pkts ok: %lld\n", sc_id, stats.pkt_ok_cnt); + } + if (mcs->hw->mcs_blks == 1) { + seq_printf(filp, "sc%d: Octets decrypted: %lld\n", sc_id, + stats.octet_decrypt_cnt); + seq_printf(filp, "sc%d: Octets validated: %lld\n", sc_id, + stats.octet_validate_cnt); + } + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_sc_stats, mcs_rx_sc_stats_display, NULL); + +static int rvu_dbg_mcs_flowid_stats_display(struct seq_file *filp, void *unused, int dir) +{ + struct mcs *mcs = filp->private; + struct mcs_flowid_stats stats; + struct rsrc_bmap *map; + int flow_id; + + seq_puts(filp, "\n Flowid stats\n"); + + if (dir == MCS_RX) + map = &mcs->rx.flow_ids; + else + map = &mcs->tx.flow_ids; + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(flow_id, map->bmap, mcs->hw->tcam_entries) { + mcs_get_flowid_stats(mcs, &stats, flow_id, dir); + seq_printf(filp, "Flowid%d: Hit:%lld\n", flow_id, stats.tcam_hit_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +static int rvu_dbg_mcs_tx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_TX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_flowid_stats, mcs_tx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_rx_flowid_stats_display(struct seq_file *filp, void *unused) +{ + return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_RX); +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_flowid_stats, mcs_rx_flowid_stats_display, NULL); + +static int rvu_dbg_mcs_tx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->tx.secy; + seq_puts(filp, "\n MCS TX secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_tx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet encrypted: %lld\n", secy_id, + stats.octet_encrypted_cnt); + seq_printf(filp, "secy%d: octet protected: %lld\n", secy_id, + stats.octet_protected_cnt); + seq_printf(filp, "secy%d: Pkts on active sa: %lld\n", secy_id, + stats.pkt_noactivesa_cnt); + seq_printf(filp, "secy%d: Pkts too long: %lld\n", secy_id, stats.pkt_toolong_cnt); + seq_printf(filp, "secy%d: Pkts untagged: %lld\n", secy_id, stats.pkt_untagged_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_tx_secy_stats, mcs_tx_secy_stats_display, NULL); + +static int rvu_dbg_mcs_rx_secy_stats_display(struct seq_file *filp, void *unused) +{ + struct mcs *mcs = filp->private; + struct mcs_secy_stats stats; + struct rsrc_bmap *map; + int secy_id; + + map = &mcs->rx.secy; + seq_puts(filp, "\n MCS secy stats\n"); + + mutex_lock(&mcs->stats_lock); + for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) { + mcs_get_rx_secy_stats(mcs, &stats, secy_id); + seq_printf(filp, "\n=======Secy%d======\n\n", secy_id); + seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id, + stats.ctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id, + stats.ctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id, + stats.ctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt); + seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id, + stats.unctl_pkt_bcast_cnt); + seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id, + stats.unctl_pkt_mcast_cnt); + seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id, + stats.unctl_pkt_ucast_cnt); + seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt); + seq_printf(filp, "secy%d: Octet decrypted: %lld\n", secy_id, + stats.octet_decrypted_cnt); + seq_printf(filp, "secy%d: octet validated: %lld\n", secy_id, + stats.octet_validated_cnt); + seq_printf(filp, "secy%d: Pkts on disable port: %lld\n", secy_id, + stats.pkt_port_disabled_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_badtag_cnt); + seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_nosa_cnt); + seq_printf(filp, "secy%d: Pkts with nosaerror: %lld\n", secy_id, + stats.pkt_nosaerror_cnt); + seq_printf(filp, "secy%d: Tagged ctrl pkts: %lld\n", secy_id, + stats.pkt_tagged_ctl_cnt); + seq_printf(filp, "secy%d: Untaged pkts: %lld\n", secy_id, stats.pkt_untaged_cnt); + seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt); + if (mcs->hw->mcs_blks > 1) + seq_printf(filp, "secy%d: pkts notag: %lld\n", secy_id, + stats.pkt_notag_cnt); + } + mutex_unlock(&mcs->stats_lock); + return 0; +} + +RVU_DEBUG_SEQ_FOPS(mcs_rx_secy_stats, mcs_rx_secy_stats_display, NULL); + +static void rvu_dbg_mcs_init(struct rvu *rvu) +{ + struct mcs *mcs; + char dname[10]; + int i; + + if (!rvu->mcs_blk_cnt) + return; + + rvu->rvu_dbg.mcs_root = debugfs_create_dir("mcs", rvu->rvu_dbg.root); + + for (i = 0; i < rvu->mcs_blk_cnt; i++) { + mcs = mcs_get_pdata(i); + + sprintf(dname, "mcs%d", i); + rvu->rvu_dbg.mcs = debugfs_create_dir(dname, + rvu->rvu_dbg.mcs_root); + + rvu->rvu_dbg.mcs_rx = debugfs_create_dir("rx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_rx, mcs, + &rvu_dbg_mcs_rx_port_stats_fops); + + rvu->rvu_dbg.mcs_tx = debugfs_create_dir("tx_stats", rvu->rvu_dbg.mcs); + + debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_flowid_stats_fops); + + debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_secy_stats_fops); + + debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sc_stats_fops); + + debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_sa_stats_fops); + + debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_tx, mcs, + &rvu_dbg_mcs_tx_port_stats_fops); + } +} + #define LMT_MAPTBL_ENTRY_SIZE 16 /* Dump LMTST map table */ static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp, @@ -3053,6 +3398,7 @@ create: rvu_dbg_npc_init(rvu); rvu_dbg_cpt_init(rvu, BLKADDR_CPT0); rvu_dbg_cpt_init(rvu, BLKADDR_CPT1); + rvu_dbg_mcs_init(rvu); } void rvu_dbg_exit(struct rvu *rvu) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 0879a48411f31487597510370965680c2fd763d0..7646bb2ec89b9e5bfa6e658506cf9796e5b59c34 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -4296,8 +4296,14 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) /* Restore CINT timer delay to HW reset values */ rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); + cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); + /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ - rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, 0x1ULL); + cfg |= 1ULL; + if (!is_rvu_otx2(rvu)) + cfg |= NIX_PTP_1STEP_EN; + + rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); if (is_block_implemented(hw, blkaddr)) { err = nix_setup_txschq(rvu, nix_hw, blkaddr); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h index 77a9ade91f3e85574ed4a66489d42b515bd8cebe..0e0d536645ac7f93ceec4f1c847684144f2db701 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h @@ -266,6 +266,7 @@ #define NIX_AF_TX_NPC_CAPTURE_CONFIG (0x0660) #define NIX_AF_TX_NPC_CAPTURE_INFO (0x0670) #define NIX_AF_SEB_CFG (0x05F0) +#define NIX_PTP_1STEP_EN BIT_ULL(2) #define NIX_AF_DEBUG_NPC_RESP_DATAX(a) (0x680 | (a) << 3) #define NIX_AF_SMQX_CFG(a) (0x700 | (a) << 16) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile index d463dc72d80af08f0963793b6cbeb2921a5c1c66..73fdb8798614835a8438b60d561d04d79c4b0926 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile @@ -13,5 +13,6 @@ rvu_nicvf-y := otx2_vf.o otx2_devlink.o rvu_nicpf-$(CONFIG_DCB) += otx2_dcbnl.o rvu_nicvf-$(CONFIG_DCB) += otx2_dcbnl.o +rvu_nicpf-$(CONFIG_MACSEC) += cn10k_macsec.o ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c index fd4f083c699ecbea5830485cb40b06840b780ecc..826f691de2595bb67ab7e5499deacd9c481aab26 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c @@ -86,8 +86,7 @@ int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c new file mode 100644 index 0000000000000000000000000000000000000000..9809f551fc2e3db0176f9115b78fc20be4539e9e --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -0,0 +1,1669 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell MACSEC hardware offload driver + * + * Copyright (C) 2022 Marvell. + */ + +#include +#include +#include +#include "otx2_common.h" + +#define MCS_TCAM0_MAC_SA_MASK GENMASK_ULL(63, 48) +#define MCS_TCAM1_MAC_SA_MASK GENMASK_ULL(31, 0) +#define MCS_TCAM1_ETYPE_MASK GENMASK_ULL(47, 32) + +#define MCS_SA_MAP_MEM_SA_USE BIT_ULL(9) + +#define MCS_RX_SECY_PLCY_RW_MASK GENMASK_ULL(49, 18) +#define MCS_RX_SECY_PLCY_RP BIT_ULL(17) +#define MCS_RX_SECY_PLCY_AUTH_ENA BIT_ULL(16) +#define MCS_RX_SECY_PLCY_CIP GENMASK_ULL(8, 5) +#define MCS_RX_SECY_PLCY_VAL GENMASK_ULL(2, 1) +#define MCS_RX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_TX_SECY_PLCY_MTU GENMASK_ULL(43, 28) +#define MCS_TX_SECY_PLCY_ST_TCI GENMASK_ULL(27, 22) +#define MCS_TX_SECY_PLCY_ST_OFFSET GENMASK_ULL(21, 15) +#define MCS_TX_SECY_PLCY_INS_MODE BIT_ULL(14) +#define MCS_TX_SECY_PLCY_AUTH_ENA BIT_ULL(13) +#define MCS_TX_SECY_PLCY_CIP GENMASK_ULL(5, 2) +#define MCS_TX_SECY_PLCY_PROTECT BIT_ULL(1) +#define MCS_TX_SECY_PLCY_ENA BIT_ULL(0) + +#define MCS_GCM_AES_128 0 +#define MCS_GCM_AES_256 1 +#define MCS_GCM_AES_XPN_128 2 +#define MCS_GCM_AES_XPN_256 3 + +#define MCS_TCI_ES 0x40 /* end station */ +#define MCS_TCI_SC 0x20 /* SCI present */ +#define MCS_TCI_SCB 0x10 /* epon */ +#define MCS_TCI_E 0x08 /* encryption */ +#define MCS_TCI_C 0x04 /* changed text */ + +static struct cn10k_mcs_txsc *cn10k_mcs_get_txsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy) +{ + struct cn10k_mcs_txsc *txsc; + + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + if (txsc->sw_secy == secy) + return txsc; + } + + return NULL; +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_get_rxsc(struct cn10k_mcs_cfg *cfg, + struct macsec_secy *secy, + struct macsec_rx_sc *rx_sc) +{ + struct cn10k_mcs_rxsc *rxsc; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + if (rxsc->sw_rxsc == rx_sc && rxsc->sw_secy == secy) + return rxsc; + } + + return NULL; +} + +static const char *rsrc_name(enum mcs_rsrc_type rsrc_type) +{ + switch (rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + return "FLOW"; + case MCS_RSRC_TYPE_SC: + return "SC"; + case MCS_RSRC_TYPE_SECY: + return "SECY"; + case MCS_RSRC_TYPE_SA: + return "SA"; + default: + return "Unknown"; + }; + + return "Unknown"; +} + +static int cn10k_mcs_alloc_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 *rsrc_id) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_alloc_rsrc_req *req; + struct mcs_alloc_rsrc_rsp *rsp; + int ret = -ENOMEM; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_alloc_resources(mbox); + if (!req) + goto fail; + + req->rsrc_type = type; + req->rsrc_cnt = 1; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_alloc_rsrc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp) || req->rsrc_cnt != rsp->rsrc_cnt || + req->rsrc_type != rsp->rsrc_type || req->dir != rsp->dir) { + ret = -EINVAL; + goto fail; + } + + switch (rsp->rsrc_type) { + case MCS_RSRC_TYPE_FLOWID: + *rsrc_id = rsp->flow_ids[0]; + break; + case MCS_RSRC_TYPE_SC: + *rsrc_id = rsp->sc_ids[0]; + break; + case MCS_RSRC_TYPE_SECY: + *rsrc_id = rsp->secy_ids[0]; + break; + case MCS_RSRC_TYPE_SA: + *rsrc_id = rsp->sa_ids[0]; + break; + default: + ret = -EINVAL; + goto fail; + } + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Failed to allocate %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); + return ret; +} + +static void cn10k_mcs_free_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir, + enum mcs_rsrc_type type, u16 hw_rsrc_id, + bool all) +{ + struct mbox *mbox = &pfvf->mbox; + struct mcs_free_rsrc_req *req; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_free_resources(mbox); + if (!req) + goto fail; + + req->rsrc_id = hw_rsrc_id; + req->rsrc_type = type; + req->dir = dir; + if (all) + req->all = 1; + + if (otx2_sync_mbox_msg(&pfvf->mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return; +fail: + dev_err(pfvf->dev, "Failed to free %s %s resource\n", + dir == MCS_TX ? "TX" : "RX", rsrc_name(type)); + mutex_unlock(&mbox->lock); +} + +static int cn10k_mcs_alloc_txsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static int cn10k_mcs_alloc_rxsa(struct otx2_nic *pfvf, u16 *hw_sa_id) +{ + return cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id); +} + +static void cn10k_mcs_free_txsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static void cn10k_mcs_free_rxsa(struct otx2_nic *pfvf, u16 hw_sa_id) +{ + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id, false); +} + +static int cn10k_mcs_write_rx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 policy; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + policy = FIELD_PREP(MCS_RX_SECY_PLCY_RW_MASK, secy->replay_window); + if (secy->replay_protect) + policy |= MCS_RX_SECY_PLCY_RP; + + policy |= MCS_RX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_CIP, MCS_GCM_AES_128); + policy |= FIELD_PREP(MCS_RX_SECY_PLCY_VAL, secy->validate_frames); + + policy |= MCS_RX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = hw_secy_id; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_flowid(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->data[1] = FIELD_PREP(MCS_TCAM1_ETYPE_MASK, ETH_P_MACSEC); + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_ETYPE_MASK; + + req->mask[0] = ~0ULL; + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = rxsc->hw_flow_id; + req->secy_id = hw_secy_id; + req->sc_id = rxsc->hw_sc_id; + req->dir = MCS_RX; + + if (sw_rx_sc->active) + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_sc_cam(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id) +{ + struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc; + struct mcs_rx_sc_cam_write_req *sc_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + sc_req = otx2_mbox_alloc_msg_mcs_rx_sc_cam_write(mbox); + if (!sc_req) { + ret = -ENOMEM; + goto fail; + } + + sc_req->sci = (__force u64)cpu_to_be64((__force u64)sw_rx_sc->sci); + sc_req->sc_id = rxsc->hw_sc_id; + sc_req->secy_id = hw_secy_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, bool sa_in_use) +{ + unsigned char *src = rxsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mcs_rx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + map_req = otx2_mbox_alloc_msg_mcs_rx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], + (src + reg * 8), 8); + reg++; + } + + plcy_req->sa_index[0] = rxsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_RX; + + map_req->sa_index = rxsc->hw_sa_id[assoc_num]; + map_req->sa_in_use = sa_in_use; + map_req->sc_id = rxsc->hw_sc_id; + map_req->an = assoc_num; + + /* Send two messages together */ + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_rx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = rxsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_RX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_secy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_secy_plcy_write_req *req; + struct mbox *mbox = &pfvf->mbox; + struct macsec_tx_sc *sw_tx_sc; + /* Insert SecTag after 12 bytes (DA+SA)*/ + u8 tag_offset = 12; + u8 sectag_tci = 0; + u64 policy; + int ret; + + sw_tx_sc = &secy->tx_sc; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + if (sw_tx_sc->send_sci) { + sectag_tci |= MCS_TCI_SC; + } else { + if (sw_tx_sc->end_station) + sectag_tci |= MCS_TCI_ES; + if (sw_tx_sc->scb) + sectag_tci |= MCS_TCI_SCB; + } + + if (sw_tx_sc->encrypt) + sectag_tci |= (MCS_TCI_E | MCS_TCI_C); + + policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, secy->netdev->mtu); + /* Write SecTag excluding AN bits(1..0) */ + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_TCI, sectag_tci >> 2); + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_OFFSET, tag_offset); + policy |= MCS_TX_SECY_PLCY_INS_MODE; + policy |= MCS_TX_SECY_PLCY_AUTH_ENA; + policy |= FIELD_PREP(MCS_TX_SECY_PLCY_CIP, MCS_GCM_AES_128); + + if (secy->protect_frames) + policy |= MCS_TX_SECY_PLCY_PROTECT; + + /* If the encodingsa does not exist/active and protect is + * not set then frames can be sent out as it is. Hence enable + * the policy irrespective of secy operational when !protect. + */ + if (!secy->protect_frames || secy->operational) + policy |= MCS_TX_SECY_PLCY_ENA; + + req->plcy = policy; + req->secy_id = txsc->hw_secy_id_tx; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_flowid(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct mcs_flowid_entry_write_req *req; + struct mbox *mbox = &pfvf->mbox; + u64 mac_sa; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + mac_sa = ether_addr_to_u64(secy->netdev->dev_addr); + + req->data[0] = FIELD_PREP(MCS_TCAM0_MAC_SA_MASK, mac_sa); + req->data[1] = FIELD_PREP(MCS_TCAM1_MAC_SA_MASK, mac_sa >> 16); + + req->mask[0] = ~0ULL; + req->mask[0] &= ~MCS_TCAM0_MAC_SA_MASK; + + req->mask[1] = ~0ULL; + req->mask[1] &= ~MCS_TCAM1_MAC_SA_MASK; + + req->mask[2] = ~0ULL; + req->mask[3] = ~0ULL; + + req->flow_id = txsc->hw_flow_id; + req->secy_id = txsc->hw_secy_id_tx; + req->sc_id = txsc->hw_sc_id; + req->sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + req->dir = MCS_TX; + /* This can be enabled since stack xmits packets only when interface is up */ + req->ena = 1; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_link_tx_sa2sc(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 sa_num, bool sa_active) +{ + struct mcs_tx_sc_sa_map *map_req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + /* Link the encoding_sa only to SC out of all SAs */ + if (txsc->encoding_sa != sa_num) + return 0; + + mutex_lock(&mbox->lock); + + map_req = otx2_mbox_alloc_msg_mcs_tx_sc_sa_map_write(mbox); + if (!map_req) { + otx2_mbox_reset(&mbox->mbox, 0); + ret = -ENOMEM; + goto fail; + } + + map_req->sa_index0 = txsc->hw_sa_id[sa_num]; + map_req->sa_index0_vld = sa_active; + map_req->sectag_sci = (__force u64)cpu_to_be64((__force u64)secy->sci); + map_req->sc_id = txsc->hw_sc_id; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_write_tx_sa_plcy(struct otx2_nic *pfvf, + struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num) +{ + unsigned char *src = txsc->sa_key[assoc_num]; + struct mcs_sa_plcy_write_req *plcy_req; + struct mbox *mbox = &pfvf->mbox; + u8 reg, key_len; + int ret; + + mutex_lock(&mbox->lock); + + plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox); + if (!plcy_req) { + ret = -ENOMEM; + goto fail; + } + + for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) { + memcpy((u8 *)&plcy_req->plcy[0][reg], (src + reg * 8), 8); + reg++; + } + + plcy_req->plcy[0][8] = assoc_num; + plcy_req->sa_index[0] = txsc->hw_sa_id[assoc_num]; + plcy_req->sa_cnt = 1; + plcy_req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_write_tx_sa_pn(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc, + u8 assoc_num, u64 next_pn) +{ + struct mcs_pn_table_write_req *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->pn_id = txsc->hw_sa_id[assoc_num]; + req->next_pn = next_pn; + req->dir = MCS_TX; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_ena_dis_flowid(struct otx2_nic *pfvf, u16 hw_flow_id, + bool enable, enum mcs_direction dir) +{ + struct mcs_flowid_ena_dis_entry *req; + struct mbox *mbox = &pfvf->mbox; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_flowid_ena_entry(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->flow_id = hw_flow_id; + req->ena = enable; + req->dir = dir; + + ret = otx2_sync_mbox_msg(mbox); + +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sa_stats(struct otx2_nic *pfvf, u8 hw_sa_id, + struct mcs_sa_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sa_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sa_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sa_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sa_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SA; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sa_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_sc_stats(struct otx2_nic *pfvf, u8 hw_sc_id, + struct mcs_sc_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_stats_req *req; + struct mcs_sc_stats *rsp; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_sc_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_sc_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_sc_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SC; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_sc_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static int cn10k_mcs_secy_stats(struct otx2_nic *pfvf, u8 hw_secy_id, + struct mcs_secy_stats *rsp_p, + enum mcs_direction dir, bool clear) +{ + struct mcs_clear_stats *clear_req; + struct mbox *mbox = &pfvf->mbox; + struct mcs_secy_stats *rsp; + struct mcs_stats_req *req; + int ret; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_get_secy_stats(mbox); + if (!req) { + ret = -ENOMEM; + goto fail; + } + + req->id = hw_secy_id; + req->dir = dir; + + if (!clear) + goto send_msg; + + clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox); + if (!clear_req) { + ret = -ENOMEM; + goto fail; + } + clear_req->id = hw_secy_id; + clear_req->dir = dir; + clear_req->type = MCS_RSRC_TYPE_SECY; + +send_msg: + ret = otx2_sync_mbox_msg(mbox); + if (ret) + goto fail; + + rsp = (struct mcs_secy_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) { + ret = PTR_ERR(rsp); + goto fail; + } + + memcpy(rsp_p, rsp, sizeof(*rsp_p)); + + mutex_unlock(&mbox->lock); + + return 0; +fail: + mutex_unlock(&mbox->lock); + return ret; +} + +static struct cn10k_mcs_txsc *cn10k_mcs_create_txsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_txsc *txsc; + int ret; + + txsc = kzalloc(sizeof(*txsc), GFP_KERNEL); + if (!txsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + &txsc->hw_flow_id); + if (ret) + goto fail; + + /* For a SecY, one TX secy and one RX secy HW resources are needed */ + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_tx); + if (ret) + goto free_flowid; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + &txsc->hw_secy_id_rx); + if (ret) + goto free_tx_secy; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + &txsc->hw_sc_id); + if (ret) + goto free_rx_secy; + + return txsc; +free_rx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); +free_tx_secy: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +fail: + return ERR_PTR(ret); +} + +/* Free Tx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_txsc(struct otx2_nic *pfvf, + struct cn10k_mcs_txsc *txsc) +{ + u8 sa_bmap = txsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_tx_sa_plcy(pfvf, txsc->sw_secy, + txsc, sa_num); + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC, + txsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_rx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, + txsc->hw_secy_id_tx, false); + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID, + txsc->hw_flow_id, false); +} + +static struct cn10k_mcs_rxsc *cn10k_mcs_create_rxsc(struct otx2_nic *pfvf) +{ + struct cn10k_mcs_rxsc *rxsc; + int ret; + + rxsc = kzalloc(sizeof(*rxsc), GFP_KERNEL); + if (!rxsc) + return ERR_PTR(-ENOMEM); + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + &rxsc->hw_flow_id); + if (ret) + goto fail; + + ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + &rxsc->hw_sc_id); + if (ret) + goto free_flowid; + + return rxsc; +free_flowid: + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +fail: + return ERR_PTR(ret); +} + +/* Free Rx SC and its SAs(if any) resources to AF + */ +static void cn10k_mcs_delete_rxsc(struct otx2_nic *pfvf, + struct cn10k_mcs_rxsc *rxsc) +{ + u8 sa_bmap = rxsc->sa_bmap; + u8 sa_num = 0; + + while (sa_bmap) { + if (sa_bmap & 1) { + cn10k_mcs_write_rx_sa_plcy(pfvf, rxsc->sw_secy, rxsc, + sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + } + sa_num++; + sa_bmap >>= 1; + } + + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC, + rxsc->hw_sc_id, false); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID, + rxsc->hw_flow_id, false); +} + +static int cn10k_mcs_secy_tx_cfg(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc, + struct macsec_tx_sa *sw_tx_sa, u8 sa_num) +{ + if (sw_tx_sa) { + cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, sa_num, + sw_tx_sa->active); + } + + cn10k_mcs_write_tx_secy(pfvf, secy, txsc); + cn10k_mcs_write_tx_flowid(pfvf, secy, txsc); + /* When updating secy, change RX secy also */ + cn10k_mcs_write_rx_secy(pfvf, secy, txsc->hw_secy_id_rx); + + return 0; +} + +static int cn10k_mcs_secy_rx_cfg(struct otx2_nic *pfvf, + struct macsec_secy *secy, u8 hw_secy_id) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + struct macsec_rx_sa *sw_rx_sa; + u8 sa_num; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + for (sa_num = 0; sa_num < CN10K_MCS_SA_PER_SC; sa_num++) { + sw_rx_sa = rcu_dereference_bh(sw_rx_sc->sa[sa_num]); + if (!sw_rx_sa) + continue; + + cn10k_mcs_write_rx_sa_plcy(pfvf, secy, mcs_rx_sc, + sa_num, sw_rx_sa->active); + cn10k_mcs_write_rx_sa_pn(pfvf, mcs_rx_sc, sa_num, + sw_rx_sa->next_pn_halves.lower); + } + + cn10k_mcs_write_rx_flowid(pfvf, mcs_rx_sc, hw_secy_id); + cn10k_mcs_write_sc_cam(pfvf, mcs_rx_sc, hw_secy_id); + } + + return 0; +} + +static int cn10k_mcs_disable_rxscs(struct otx2_nic *pfvf, + struct macsec_secy *secy, + bool delete) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *mcs_rx_sc; + struct macsec_rx_sc *sw_rx_sc; + int ret; + + for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active; + sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) { + mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (unlikely(!mcs_rx_sc)) + continue; + + ret = cn10k_mcs_ena_dis_flowid(pfvf, mcs_rx_sc->hw_flow_id, + false, MCS_RX); + if (ret) + dev_err(pfvf->dev, "Failed to disable TCAM for SC %d\n", + mcs_rx_sc->hw_sc_id); + if (delete) { + cn10k_mcs_delete_rxsc(pfvf, mcs_rx_sc); + list_del(&mcs_rx_sc->entry); + kfree(mcs_rx_sc); + } + } + + return 0; +} + +static void cn10k_mcs_sync_stats(struct otx2_nic *pfvf, struct macsec_secy *secy, + struct cn10k_mcs_txsc *txsc) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_secy_stats rx_rsp = { 0 }; + struct mcs_sc_stats sc_rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + /* Because of shared counters for some stats in the hardware, when + * updating secy policy take a snapshot of current stats and reset them. + * Below are the effected stats because of shared counters. + */ + + /* Check if sync is really needed */ + if (secy->validate_frames == txsc->last_validate_frames && + secy->protect_frames == txsc->last_protect_frames) + return; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (txsc->last_validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + + list_for_each_entry(rxsc, &cfg->rxsc_list, entry) { + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &sc_rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += sc_rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += sc_rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += sc_rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += sc_rsp.pkt_notvalid_cnt; + + if (txsc->last_protect_frames) + rxsc->stats.InPktsLate += sc_rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += sc_rsp.pkt_late_cnt; + + if (txsc->last_validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += sc_rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += sc_rsp.pkt_unchecked_cnt; + } + + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; +} + +static int cn10k_mdo_open(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + + return cn10k_mcs_secy_rx_cfg(pfvf, secy, txsc->hw_secy_id_rx); +} + +static int cn10k_mdo_stop(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + err = cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + if (err) + return err; + + return cn10k_mcs_disable_rxscs(pfvf, ctx->secy, false); +} + +static int cn10k_mdo_add_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) + return -EOPNOTSUPP; + + /* Stick to 16 bytes key len until XPN support is added */ + if (secy->key_len != 16) + return -EOPNOTSUPP; + + if (secy->xpn) + return -EOPNOTSUPP; + + txsc = cn10k_mcs_create_txsc(pfvf); + if (IS_ERR(txsc)) + return -ENOSPC; + + txsc->sw_secy = secy; + txsc->encoding_sa = secy->tx_sc.encoding_sa; + txsc->last_validate_frames = secy->validate_frames; + txsc->last_protect_frames = secy->protect_frames; + + list_add(&txsc->entry, &cfg->txsc_list); + + if (netif_running(secy->netdev)) + return cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, NULL, 0); + + return 0; +} + +static int cn10k_mdo_upd_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct macsec_tx_sa *sw_tx_sa; + struct cn10k_mcs_txsc *txsc; + u8 sa_num; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + txsc->encoding_sa = secy->tx_sc.encoding_sa; + + sa_num = txsc->encoding_sa; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]); + + if (netif_running(secy->netdev)) { + cn10k_mcs_sync_stats(pfvf, secy, txsc); + + err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_secy(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX); + cn10k_mcs_disable_rxscs(pfvf, ctx->secy, true); + cn10k_mcs_delete_txsc(pfvf, txsc); + list_del(&txsc->entry); + kfree(txsc); + + return 0; +} + +static int cn10k_mdo_add_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_txsa(pfvf, &txsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&txsc->sa_key[sa_num], ctx->sa.key, secy->key_len); + txsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num); + if (err) + return err; + + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa; + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + /* Keys cannot be changed after creation */ + err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num, + sw_tx_sa->next_pn_halves.lower); + if (err) + return err; + + err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, + sa_num, sw_tx_sa->active); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_txsa(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]); + txsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_add_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_rxsc *rxsc; + struct cn10k_mcs_txsc *txsc; + int err; + + txsc = cn10k_mcs_get_txsc(cfg, secy); + if (!txsc) + return -ENOENT; + + rxsc = cn10k_mcs_create_rxsc(pfvf); + if (IS_ERR(rxsc)) + return -ENOSPC; + + rxsc->sw_secy = ctx->secy; + rxsc->sw_rxsc = ctx->rx_sc; + list_add(&rxsc->entry, &cfg->rxsc_list); + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_flowid(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + + err = cn10k_mcs_write_sc_cam(pfvf, rxsc, txsc->hw_secy_id_rx); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + bool enable = ctx->rx_sc->active; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + if (netif_running(secy->netdev)) + return cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, + enable, MCS_RX); + + return 0; +} + +static int cn10k_mdo_del_rxsc(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, false, MCS_RX); + cn10k_mcs_delete_rxsc(pfvf, rxsc); + list_del(&rxsc->entry); + kfree(rxsc); + + return 0; +} + +static int cn10k_mdo_add_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (cn10k_mcs_alloc_rxsa(pfvf, &rxsc->hw_sa_id[sa_num])) + return -ENOSPC; + + memcpy(&rxsc->sa_key[sa_num], ctx->sa.key, ctx->secy->key_len); + rxsc->sa_bmap |= 1 << sa_num; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, + sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_upd_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa; + u64 next_pn = rx_sa->next_pn_halves.lower; + struct macsec_secy *secy = ctx->secy; + bool sa_in_use = rx_sa->active; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + int err; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + if (netif_running(secy->netdev)) { + err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, sa_num, sa_in_use); + if (err) + return err; + + err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn); + if (err) + return err; + } + + return 0; +} + +static int cn10k_mdo_del_rxsa(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_write_rx_sa_plcy(pfvf, ctx->secy, rxsc, sa_num, false); + cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]); + + rxsc->sa_bmap &= ~(1 << sa_num); + + return 0; +} + +static int cn10k_mdo_get_dev_stats(struct macsec_context *ctx) +{ + struct mcs_secy_stats tx_rsp = { 0 }, rx_rsp = { 0 }; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_tx, &tx_rsp, MCS_TX, false); + ctx->stats.dev_stats->OutPktsUntagged = tx_rsp.pkt_untagged_cnt; + ctx->stats.dev_stats->OutPktsTooLong = tx_rsp.pkt_toolong_cnt; + + cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true); + txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt; + txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt; + txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt; + if (secy->validate_frames == MACSEC_VALIDATE_STRICT) + txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt; + else + txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt; + txsc->stats.InPktsOverrun = 0; + + ctx->stats.dev_stats->InPktsNoTag = txsc->stats.InPktsNoTag; + ctx->stats.dev_stats->InPktsUntagged = txsc->stats.InPktsUntagged; + ctx->stats.dev_stats->InPktsBadTag = txsc->stats.InPktsBadTag; + ctx->stats.dev_stats->InPktsUnknownSCI = txsc->stats.InPktsUnknownSCI; + ctx->stats.dev_stats->InPktsNoSCI = txsc->stats.InPktsNoSCI; + ctx->stats.dev_stats->InPktsOverrun = txsc->stats.InPktsOverrun; + + return 0; +} + +static int cn10k_mdo_get_tx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, txsc->hw_sc_id, &rsp, MCS_TX, false); + + ctx->stats.tx_sc_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sc_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + ctx->stats.tx_sc_stats->OutOctetsProtected = rsp.octet_protected_cnt; + ctx->stats.tx_sc_stats->OutOctetsEncrypted = rsp.octet_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_tx_sa_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_txsc *txsc; + + txsc = cn10k_mcs_get_txsc(cfg, ctx->secy); + if (!txsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, txsc->hw_sa_id[sa_num], &rsp, MCS_TX, false); + + ctx->stats.tx_sa_stats->OutPktsProtected = rsp.pkt_protected_cnt; + ctx->stats.tx_sa_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt; + + return 0; +} + +static int cn10k_mdo_get_rx_sc_stats(struct macsec_context *ctx) +{ + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_secy *secy = ctx->secy; + struct mcs_sc_stats rsp = { 0 }; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc); + if (!rxsc) + return -ENOENT; + + cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &rsp, MCS_RX, true); + + rxsc->stats.InOctetsValidated += rsp.octet_validate_cnt; + rxsc->stats.InOctetsDecrypted += rsp.octet_decrypt_cnt; + + rxsc->stats.InPktsInvalid += rsp.pkt_invalid_cnt; + rxsc->stats.InPktsNotValid += rsp.pkt_notvalid_cnt; + + if (secy->protect_frames) + rxsc->stats.InPktsLate += rsp.pkt_late_cnt; + else + rxsc->stats.InPktsDelayed += rsp.pkt_late_cnt; + + if (secy->validate_frames == MACSEC_VALIDATE_CHECK) + rxsc->stats.InPktsUnchecked += rsp.pkt_unchecked_cnt; + else + rxsc->stats.InPktsOK += rsp.pkt_unchecked_cnt; + + ctx->stats.rx_sc_stats->InOctetsValidated = rxsc->stats.InOctetsValidated; + ctx->stats.rx_sc_stats->InOctetsDecrypted = rxsc->stats.InOctetsDecrypted; + ctx->stats.rx_sc_stats->InPktsInvalid = rxsc->stats.InPktsInvalid; + ctx->stats.rx_sc_stats->InPktsNotValid = rxsc->stats.InPktsNotValid; + ctx->stats.rx_sc_stats->InPktsLate = rxsc->stats.InPktsLate; + ctx->stats.rx_sc_stats->InPktsDelayed = rxsc->stats.InPktsDelayed; + ctx->stats.rx_sc_stats->InPktsUnchecked = rxsc->stats.InPktsUnchecked; + ctx->stats.rx_sc_stats->InPktsOK = rxsc->stats.InPktsOK; + + return 0; +} + +static int cn10k_mdo_get_rx_sa_stats(struct macsec_context *ctx) +{ + struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc; + struct otx2_nic *pfvf = netdev_priv(ctx->netdev); + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct mcs_sa_stats rsp = { 0 }; + u8 sa_num = ctx->sa.assoc_num; + struct cn10k_mcs_rxsc *rxsc; + + rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc); + if (!rxsc) + return -ENOENT; + + if (sa_num >= CN10K_MCS_SA_PER_SC) + return -EOPNOTSUPP; + + cn10k_mcs_sa_stats(pfvf, rxsc->hw_sa_id[sa_num], &rsp, MCS_RX, false); + + ctx->stats.rx_sa_stats->InPktsOK = rsp.pkt_ok_cnt; + ctx->stats.rx_sa_stats->InPktsInvalid = rsp.pkt_invalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotValid = rsp.pkt_notvalid_cnt; + ctx->stats.rx_sa_stats->InPktsNotUsingSA = rsp.pkt_nosaerror_cnt; + ctx->stats.rx_sa_stats->InPktsUnusedSA = rsp.pkt_nosa_cnt; + + return 0; +} + +static const struct macsec_ops cn10k_mcs_ops = { + .mdo_dev_open = cn10k_mdo_open, + .mdo_dev_stop = cn10k_mdo_stop, + .mdo_add_secy = cn10k_mdo_add_secy, + .mdo_upd_secy = cn10k_mdo_upd_secy, + .mdo_del_secy = cn10k_mdo_del_secy, + .mdo_add_rxsc = cn10k_mdo_add_rxsc, + .mdo_upd_rxsc = cn10k_mdo_upd_rxsc, + .mdo_del_rxsc = cn10k_mdo_del_rxsc, + .mdo_add_rxsa = cn10k_mdo_add_rxsa, + .mdo_upd_rxsa = cn10k_mdo_upd_rxsa, + .mdo_del_rxsa = cn10k_mdo_del_rxsa, + .mdo_add_txsa = cn10k_mdo_add_txsa, + .mdo_upd_txsa = cn10k_mdo_upd_txsa, + .mdo_del_txsa = cn10k_mdo_del_txsa, + .mdo_get_dev_stats = cn10k_mdo_get_dev_stats, + .mdo_get_tx_sc_stats = cn10k_mdo_get_tx_sc_stats, + .mdo_get_tx_sa_stats = cn10k_mdo_get_tx_sa_stats, + .mdo_get_rx_sc_stats = cn10k_mdo_get_rx_sc_stats, + .mdo_get_rx_sa_stats = cn10k_mdo_get_rx_sa_stats, +}; + +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event) +{ + struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg; + struct macsec_tx_sa *sw_tx_sa = NULL; + struct macsec_secy *secy = NULL; + struct cn10k_mcs_txsc *txsc; + u8 an; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + if (!(event->intr_mask & MCS_CPM_TX_PACKET_XPN_EQ0_INT)) + return; + + /* Find the SecY to which the expired hardware SA is mapped */ + list_for_each_entry(txsc, &cfg->txsc_list, entry) { + for (an = 0; an < CN10K_MCS_SA_PER_SC; an++) + if (txsc->hw_sa_id[an] == event->sa_id) { + secy = txsc->sw_secy; + sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[an]); + } + } + + if (secy && sw_tx_sa) + macsec_pn_wrapped(secy, sw_tx_sa); +} + +int cn10k_mcs_init(struct otx2_nic *pfvf) +{ + struct mbox *mbox = &pfvf->mbox; + struct cn10k_mcs_cfg *cfg; + struct mcs_intr_cfg *req; + + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return 0; + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return -ENOMEM; + + INIT_LIST_HEAD(&cfg->txsc_list); + INIT_LIST_HEAD(&cfg->rxsc_list); + pfvf->macsec_cfg = cfg; + + pfvf->netdev->features |= NETIF_F_HW_MACSEC; + pfvf->netdev->macsec_ops = &cn10k_mcs_ops; + + mutex_lock(&mbox->lock); + + req = otx2_mbox_alloc_msg_mcs_intr_cfg(mbox); + if (!req) + goto fail; + + req->intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT; + + if (otx2_sync_mbox_msg(mbox)) + goto fail; + + mutex_unlock(&mbox->lock); + + return 0; +fail: + dev_err(pfvf->dev, "Cannot notify PN wrapped event\n"); + mutex_unlock(&mbox->lock); + return 0; +} + +void cn10k_mcs_free(struct otx2_nic *pfvf) +{ + if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag)) + return; + + cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true); + cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true); + kfree(pfvf->macsec_cfg); + pfvf->macsec_cfg = NULL; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index d686c7b6252f41946282dd99bf85905064e92e43..9ac9e6615ae7172310d553814d5a697bda9331a5 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -586,8 +586,9 @@ void otx2_get_mac_from_af(struct net_device *netdev) } EXPORT_SYMBOL(otx2_get_mac_from_af); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc) { + u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC]; struct otx2_hw *hw = &pfvf->hw; struct nix_txschq_config *req; u64 schq, parent; @@ -602,7 +603,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->lvl = lvl; req->num_regs = 1; - schq = hw->txschq_list[lvl][0]; + schq_list = hw->txschq_list; +#ifdef CONFIG_DCB + if (txschq_for_pfc) + schq_list = pfvf->pfc_schq_list; +#endif + + schq = schq_list[lvl][prio]; /* Set topology e.t.c configuration */ if (lvl == NIX_TXSCH_LVL_SMQ) { req->reg[0] = NIX_AF_SMQX_CFG(schq); @@ -611,7 +618,7 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) (0x2ULL << 36); req->num_regs++; /* MDQ config */ - parent = hw->txschq_list[NIX_TXSCH_LVL_TL4][0]; + parent = schq_list[NIX_TXSCH_LVL_TL4][prio]; req->reg[1] = NIX_AF_MDQX_PARENT(schq); req->regval[1] = parent << 16; req->num_regs++; @@ -619,14 +626,14 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq); req->regval[2] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL4) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL3][0]; + parent = schq_list[NIX_TXSCH_LVL_TL3][prio]; req->reg[0] = NIX_AF_TL4X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq); req->regval[1] = dwrr_val; } else if (lvl == NIX_TXSCH_LVL_TL3) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL2][0]; + parent = schq_list[NIX_TXSCH_LVL_TL2][prio]; req->reg[0] = NIX_AF_TL3X_PARENT(schq); req->regval[0] = parent << 16; req->num_regs++; @@ -635,11 +642,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) if (lvl == hw->txschq_link_cfg_lvl) { req->num_regs++; req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); - /* Enable this queue and backpressure */ - req->regval[2] = BIT_ULL(13) | BIT_ULL(12); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; } } else if (lvl == NIX_TXSCH_LVL_TL2) { - parent = hw->txschq_list[NIX_TXSCH_LVL_TL1][0]; + parent = schq_list[NIX_TXSCH_LVL_TL1][prio]; req->reg[0] = NIX_AF_TL2X_PARENT(schq); req->regval[0] = parent << 16; @@ -650,8 +659,10 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) if (lvl == hw->txschq_link_cfg_lvl) { req->num_regs++; req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link); - /* Enable this queue and backpressure */ - req->regval[2] = BIT_ULL(13) | BIT_ULL(12); + /* Enable this queue and backpressure + * and set relative channel + */ + req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio; } } else if (lvl == NIX_TXSCH_LVL_TL1) { /* Default config for TL1. @@ -676,6 +687,31 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl) return otx2_sync_mbox_msg(&pfvf->mbox); } +EXPORT_SYMBOL(otx2_txschq_config); + +int otx2_smq_flush(struct otx2_nic *pfvf, int smq) +{ + struct nix_txschq_config *req; + int rc; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + req->lvl = NIX_TXSCH_LVL_SMQ; + req->reg[0] = NIX_AF_SMQX_CFG(smq); + req->regval[0] |= BIT_ULL(49); + req->num_regs++; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + return rc; +} +EXPORT_SYMBOL(otx2_smq_flush); int otx2_txsch_alloc(struct otx2_nic *pfvf) { @@ -806,8 +842,7 @@ int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ aq->sq.cq_ena = 1; aq->sq.ena = 1; - /* Only one SMQ is allocated, map all SQ's to that SMQ */ - aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; + aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); aq->sq.default_chan = pfvf->hw.tx_chan_base; aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ @@ -1792,4 +1827,5 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ } \ EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name); MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index b28029cc4316c0675abac1fc724adffe9f0659c4..282db6fe3b08a91d85fc5a0ef23165e05b16af6c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,7 @@ #define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF 0xB200 +#define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF 0xBD00 /* PCI BAR nos */ #define PCI_CFG_REG_BAR_NUM 2 @@ -40,6 +42,11 @@ #define NAME_SIZE 32 +#ifdef CONFIG_DCB +/* Max priority supported for PFC */ +#define NIX_PF_PFC_PRIO_MAX 8 +#endif + enum arua_mapped_qtypes { AURA_NIX_RQ, AURA_NIX_SQ, @@ -196,7 +203,7 @@ struct otx2_hw { /* NIX */ u8 txschq_link_cfg_lvl; - u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + u16 txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; u16 matchall_ipolicer; u32 dwrr_mtu; @@ -238,6 +245,8 @@ struct otx2_hw { #define CN10K_MBOX 1 #define CN10K_LMTST 2 #define CN10K_RPM 3 +#define CN10K_PTP_ONESTEP 4 +#define CN10K_HW_MACSEC 5 unsigned long cap_flag; #define LMT_LINE_SIZE 128 @@ -271,6 +280,13 @@ struct refill_work { struct otx2_nic *pf; }; +/* PTPv2 originTimestamp structure */ +struct ptpv2_tstamp { + __be16 seconds_msb; /* 16 bits + */ + __be32 seconds_lsb; /* 32 bits = 48 bits*/ + __be32 nanoseconds; +} __packed; + struct otx2_ptp { struct ptp_clock_info ptp_info; struct ptp_clock *ptp_clock; @@ -286,6 +302,9 @@ struct otx2_ptp { struct ptp_pin_desc extts_config; u64 (*convert_rx_ptp_tstmp)(u64 timestamp); u64 (*convert_tx_ptp_tstmp)(u64 timestamp); + struct delayed_work synctstamp_work; + u64 tstamp; + u32 base_ns; }; #define OTX2_HW_TIMESTAMP_LEN 8 @@ -335,6 +354,66 @@ struct dev_hw_ops { void (*aura_freeptr)(void *dev, int aura, u64 buf); }; +#define CN10K_MCS_SA_PER_SC 4 + +/* Stats which need to be accumulated in software because + * of shared counters in hardware. + */ +struct cn10k_txsc_stats { + u64 InPktsUntagged; + u64 InPktsNoTag; + u64 InPktsBadTag; + u64 InPktsUnknownSCI; + u64 InPktsNoSCI; + u64 InPktsOverrun; +}; + +struct cn10k_rxsc_stats { + u64 InOctetsValidated; + u64 InOctetsDecrypted; + u64 InPktsUnchecked; + u64 InPktsDelayed; + u64 InPktsOK; + u64 InPktsInvalid; + u64 InPktsLate; + u64 InPktsNotValid; + u64 InPktsNotUsingSA; + u64 InPktsUnusedSA; +}; + +struct cn10k_mcs_txsc { + struct macsec_secy *sw_secy; + struct cn10k_txsc_stats stats; + struct list_head entry; + enum macsec_validation_type last_validate_frames; + bool last_protect_frames; + u16 hw_secy_id_tx; + u16 hw_secy_id_rx; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; + u8 encoding_sa; +}; + +struct cn10k_mcs_rxsc { + struct macsec_secy *sw_secy; + struct macsec_rx_sc *sw_rxsc; + struct cn10k_rxsc_stats stats; + struct list_head entry; + u16 hw_flow_id; + u16 hw_sc_id; + u16 hw_sa_id[CN10K_MCS_SA_PER_SC]; + u8 sa_bmap; + u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN]; +}; + +struct cn10k_mcs_cfg { + struct list_head txsc_list; + struct list_head rxsc_list; +}; + struct otx2_nic { void __iomem *reg_base; struct net_device *netdev; @@ -358,6 +437,7 @@ struct otx2_nic { #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED BIT_ULL(12) #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED BIT_ULL(13) #define OTX2_FLAG_DMACFLTR_SUPPORT BIT_ULL(14) +#define OTX2_FLAG_PTP_ONESTEP_SYNC BIT_ULL(15) #define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16) u64 flags; u64 *cq_op_addr; @@ -415,10 +495,16 @@ struct otx2_nic { /* PFC */ u8 pfc_en; u8 *queue_to_pfc_map; + u16 pfc_schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC]; + bool pfc_alloc_status[NIX_PF_PFC_PRIO_MAX]; #endif /* napi event count. It is needed for adaptive irq coalescing. */ u32 napi_events; + +#if IS_ENABLED(CONFIG_MACSEC) + struct cn10k_mcs_cfg *macsec_cfg; +#endif }; static inline bool is_otx2_lbkvf(struct pci_dev *pdev) @@ -458,6 +544,11 @@ static inline bool is_dev_otx2(struct pci_dev *pdev) midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO); } +static inline bool is_dev_cn10kb(struct pci_dev *pdev) +{ + return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF; +} + static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) { struct otx2_hw *hw = &pfvf->hw; @@ -487,7 +578,11 @@ static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf) __set_bit(CN10K_MBOX, &hw->cap_flag); __set_bit(CN10K_LMTST, &hw->cap_flag); __set_bit(CN10K_RPM, &hw->cap_flag); + __set_bit(CN10K_PTP_ONESTEP, &hw->cap_flag); } + + if (is_dev_cn10kb(pfvf->pdev)) + __set_bit(CN10K_HW_MACSEC, &hw->cap_flag); } /* Register read/write APIs */ @@ -743,6 +838,7 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf, \ struct _rsp_type *rsp); \ MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M /* Time to wait before watchdog kicks off */ @@ -785,6 +881,16 @@ static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf, dir, DMA_ATTR_SKIP_CPU_SYNC); } +static inline u16 otx2_get_smq_idx(struct otx2_nic *pfvf, u16 qidx) +{ +#ifdef CONFIG_DCB + if (pfvf->pfc_alloc_status[qidx]) + return pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][qidx]; +#endif + + return pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; +} + /* MSI-X APIs */ void otx2_free_cints(struct otx2_nic *pfvf, int n); void otx2_set_cints_affinity(struct otx2_nic *pfvf); @@ -807,7 +913,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type); void otx2_sq_free_sqbs(struct otx2_nic *pfvf); int otx2_config_nix(struct otx2_nic *pfvf); int otx2_config_nix_queues(struct otx2_nic *pfvf); -int otx2_txschq_config(struct otx2_nic *pfvf, int lvl); +int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool pfc_en); int otx2_txsch_alloc(struct otx2_nic *pfvf); int otx2_txschq_stop(struct otx2_nic *pfvf); void otx2_sqb_flush(struct otx2_nic *pfvf); @@ -888,6 +994,8 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx); u16 otx2_get_max_mtu(struct otx2_nic *pfvf); int otx2_handle_ntuple_tc_features(struct net_device *netdev, netdev_features_t features); +int otx2_smq_flush(struct otx2_nic *pfvf, int smq); + /* tc support */ int otx2_init_tc(struct otx2_nic *nic); void otx2_shutdown_tc(struct otx2_nic *nic); @@ -907,5 +1015,24 @@ void otx2_dmacflt_update_pfmac_flow(struct otx2_nic *pfvf); void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, bool pfc_enable); int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf); int otx2_dcbnl_set_ops(struct net_device *dev); +/* PFC support */ +int otx2_pfc_txschq_config(struct otx2_nic *pfvf); +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf); +int otx2_pfc_txschq_update(struct otx2_nic *pfvf); +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf); #endif + +#if IS_ENABLED(CONFIG_MACSEC) +/* MACSEC offload support */ +int cn10k_mcs_init(struct otx2_nic *pfvf); +void cn10k_mcs_free(struct otx2_nic *pfvf); +void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event); +#else +static inline int cn10k_mcs_init(struct otx2_nic *pfvf) { return 0; } +static inline void cn10k_mcs_free(struct otx2_nic *pfvf) {} +static inline void cn10k_handle_mcs_event(struct otx2_nic *pfvf, + struct mcs_intr_info *event) +{} +#endif /* CONFIG_MACSEC */ + #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c index 723d2506d309d24f317507f887e2c533e8353b4d..ccaf97bb1ce03e84dc4edd3a7e76469eb5d80768 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c @@ -7,6 +7,289 @@ #include "otx2_common.h" +static int otx2_check_pfc_config(struct otx2_nic *pfvf) +{ + u8 tx_queues = pfvf->hw.tx_queues, prio; + u8 pfc_en = pfvf->pfc_en; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + if ((pfc_en & (1 << prio)) && + prio > tx_queues - 1) { + dev_warn(pfvf->dev, + "Increase number of tx queues from %d to %d to support PFC.\n", + tx_queues, prio + 1); + return -EINVAL; + } + } + + return 0; +} + +int otx2_pfc_txschq_config(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, lvl, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* Either PFC bit is not set + * or tx scheduler is not allocated for the priority + */ + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* configure the scheduler for the tls*/ + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { + err = otx2_txschq_config(pfvf, lvl, prio, true); + if (err) { + dev_err(pfvf->dev, + "%s configure PFC tx schq for lvl:%d, prio:%d failed!\n", + __func__, lvl, prio); + return err; + } + } + } + + return 0; +} + +static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_alloc_req *req; + struct nix_txsch_alloc_rsp *rsp; + int lvl, rc; + + /* Get memory to put this msg */ + req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox); + if (!req) + return -ENOMEM; + + /* Request one schq per level upto max level as configured + * link config level. These rest of the scheduler can be + * same as hw.txschq_list. + */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) + req->schq[lvl] = 1; + + rc = otx2_sync_mbox_msg(&pfvf->mbox); + if (rc) + return rc; + + rsp = (struct nix_txsch_alloc_rsp *) + otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); + if (IS_ERR(rsp)) + return PTR_ERR(rsp); + + /* Setup transmit scheduler list */ + for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) { + if (!rsp->schq[lvl]) + return -ENOSPC; + + pfvf->pfc_schq_list[lvl][prio] = rsp->schq_list[lvl][0]; + } + + /* Set the Tx schedulers for rest of the levels same as + * hw.txschq_list as those will be common for all. + */ + for (; lvl < NIX_TXSCH_LVL_CNT; lvl++) + pfvf->pfc_schq_list[lvl][prio] = pfvf->hw.txschq_list[lvl][0]; + + pfvf->pfc_alloc_status[prio] = true; + return 0; +} + +int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf) +{ + u8 pfc_en = pfvf->pfc_en; + u8 pfc_bit_set; + int err, prio; + + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to allocate PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + +static int otx2_pfc_txschq_stop_one(struct otx2_nic *pfvf, u8 prio) +{ + struct nix_txsch_free_req *free_req; + + mutex_lock(&pfvf->mbox.lock); + /* free PFC TLx nodes */ + free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox); + if (!free_req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + free_req->flags = TXSCHQ_FREE_ALL; + otx2_sync_mbox_msg(&pfvf->mbox); + mutex_unlock(&pfvf->mbox.lock); + + pfvf->pfc_alloc_status[prio] = false; + return 0; +} + +static int otx2_pfc_update_sq_smq_mapping(struct otx2_nic *pfvf, int prio) +{ + struct nix_cn10k_aq_enq_req *cn10k_sq_aq; + struct net_device *dev = pfvf->netdev; + bool if_up = netif_running(dev); + struct nix_aq_enq_req *sq_aq; + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_stop_all_queues(pfvf->netdev); + else + netif_tx_stop_queue(netdev_get_tx_queue(dev, prio)); + } + + if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { + cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); + if (!cn10k_sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + cn10k_sq_aq->qidx = prio; + cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ; + cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + cn10k_sq_aq->sq.ena = 1; + cn10k_sq_aq->sq_mask.ena = 1; + cn10k_sq_aq->sq_mask.smq = GENMASK(9, 0); + cn10k_sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } else { + sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); + if (!sq_aq) + return -ENOMEM; + + /* Fill AQ info */ + sq_aq->qidx = prio; + sq_aq->ctype = NIX_AQ_CTYPE_SQ; + sq_aq->op = NIX_AQ_INSTOP_WRITE; + + /* Fill fields to update */ + sq_aq->sq.ena = 1; + sq_aq->sq_mask.ena = 1; + sq_aq->sq_mask.smq = GENMASK(8, 0); + sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio); + } + + otx2_sync_mbox_msg(&pfvf->mbox); + + if (if_up) { + if (pfvf->pfc_alloc_status[prio]) + netif_tx_start_all_queues(pfvf->netdev); + else + netif_tx_start_queue(netdev_get_tx_queue(dev, prio)); + } + + return 0; +} + +int otx2_pfc_txschq_update(struct otx2_nic *pfvf) +{ + bool if_up = netif_running(pfvf->netdev); + u8 pfc_en = pfvf->pfc_en, pfc_bit_set; + struct mbox *mbox = &pfvf->mbox; + int err, prio; + + mutex_lock(&mbox->lock); + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + + /* tx scheduler was created but user wants to disable now */ + if (!pfc_bit_set && pfvf->pfc_alloc_status[prio]) { + mutex_unlock(&mbox->lock); + if (if_up) + netif_tx_stop_all_queues(pfvf->netdev); + + otx2_smq_flush(pfvf, pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][prio]); + if (if_up) + netif_tx_start_all_queues(pfvf->netdev); + + /* delete the schq */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, + "%s failed to stop PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + + mutex_lock(&mbox->lock); + goto update_sq_smq_map; + } + + /* Either PFC bit is not set + * or Tx scheduler is already mapped for the priority + */ + if (!pfc_bit_set || pfvf->pfc_alloc_status[prio]) + continue; + + /* Add new scheduler to the priority */ + err = otx2_pfc_txschq_alloc_one(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, + "%s failed to allocate PFC tx schedulers for priority: %d\n", + __func__, prio); + return err; + } + +update_sq_smq_map: + err = otx2_pfc_update_sq_smq_mapping(pfvf, prio); + if (err) { + mutex_unlock(&mbox->lock); + dev_err(pfvf->dev, "%s failed PFC Tx schq sq:%d mapping", __func__, prio); + return err; + } + } + + err = otx2_pfc_txschq_config(pfvf); + mutex_unlock(&mbox->lock); + if (err) + return err; + + return 0; +} + +int otx2_pfc_txschq_stop(struct otx2_nic *pfvf) +{ + u8 pfc_en, pfc_bit_set; + int prio, err; + + pfc_en = pfvf->pfc_en; + for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) { + pfc_bit_set = pfc_en & (1 << prio); + if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio]) + continue; + + /* Delete the existing scheduler */ + err = otx2_pfc_txschq_stop_one(pfvf, prio); + if (err) { + dev_err(pfvf->dev, "%s failed to stop PFC TX schedulers\n", __func__); + return err; + } + } + + return 0; +} + int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf) { struct cgx_pfc_cfg *req; @@ -128,6 +411,17 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc) /* Save PFC configuration to interface */ pfvf->pfc_en = pfc->pfc_en; + if (pfvf->hw.tx_queues >= NIX_PF_PFC_PRIO_MAX) + goto process_pfc; + + /* Check if the PFC configuration can be + * supported by the tx queue configuration + */ + err = otx2_check_pfc_config(pfvf); + if (err) + return err; + +process_pfc: err = otx2_config_priority_flow_ctrl(pfvf); if (err) return err; @@ -136,6 +430,12 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc) if (pfc->pfc_en) otx2_nix_config_bp(pfvf, true); + err = otx2_pfc_txschq_update(pfvf); + if (err) { + dev_err(pfvf->dev, "%s failed to update TX schedulers\n", __func__); + return err; + } + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 3f60a80e34c822433beb4386cca26f2064bf0405..0eb74e8c553ddb349b5f26c7fee178ee31b23837 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -76,8 +76,8 @@ static void otx2_get_drvinfo(struct net_device *netdev, { struct otx2_nic *pfvf = netdev_priv(netdev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info)); } static void otx2_get_qset_strings(struct otx2_nic *pfvf, u8 **data, int qset) @@ -963,10 +963,12 @@ static int otx2_get_ts_info(struct net_device *netdev, info->phc_index = otx2_ptp_clock_index(pfvf); - info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON); + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); + if (test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_SYNC); - info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | - (1 << HWTSTAMP_FILTER_ALL); + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_ALL); return 0; } @@ -1313,8 +1315,8 @@ static void otx2vf_get_drvinfo(struct net_device *netdev, { struct otx2_nic *vf = netdev_priv(netdev); - strlcpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_VF_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info)); } static void otx2vf_get_strings(struct net_device *netdev, u32 sset, u8 *data) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 9376d0e62914b644b243a0fadfdd911aac7db5d6..892ca88e0cf43be6cb590b1925a8f5b065ce9575 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -858,6 +858,15 @@ static void otx2_handle_link_event(struct otx2_nic *pf) } } +int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf, + struct mcs_intr_info *event, + struct msg_rsp *rsp) +{ + cn10k_handle_mcs_event(pf, event); + + return 0; +} + int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf, struct cgx_link_info_msg *msg, struct msg_rsp *rsp) @@ -917,6 +926,7 @@ static int otx2_process_mbox_msg_up(struct otx2_nic *pf, return err; \ } MBOX_UP_CGX_MESSAGES +MBOX_UP_MCS_MESSAGES #undef M break; default: @@ -1389,18 +1399,40 @@ static int otx2_init_hw_resources(struct otx2_nic *pf) goto err_free_sq_ptrs; } +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_alloc(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_sq_ptrs; + } + } +#endif + err = otx2_config_nix_queues(pf); if (err) { mutex_unlock(&mbox->lock); goto err_free_txsch; } + for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { - err = otx2_txschq_config(pf, lvl); + err = otx2_txschq_config(pf, lvl, 0, false); if (err) { mutex_unlock(&mbox->lock); goto err_free_nix_queues; } } + +#ifdef CONFIG_DCB + if (pf->pfc_en) { + err = otx2_pfc_txschq_config(pf); + if (err) { + mutex_unlock(&mbox->lock); + goto err_free_nix_queues; + } + } +#endif + mutex_unlock(&mbox->lock); return err; @@ -1455,6 +1487,11 @@ static void otx2_free_hw_resources(struct otx2_nic *pf) if (err) dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n"); +#ifdef CONFIG_DCB + if (pf->pfc_en) + otx2_pfc_txschq_stop(pf); +#endif + mutex_lock(&mbox->lock); /* Disable backpressure */ if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK)) @@ -1634,8 +1671,7 @@ int otx2_open(struct net_device *netdev) cq_poll->dev = (void *)pf; cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; INIT_WORK(&cq_poll->dim.work, otx2_dim_work); - netif_napi_add(netdev, &cq_poll->napi, - otx2_napi_handler, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler); napi_enable(&cq_poll->napi); } @@ -1853,6 +1889,30 @@ static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +static u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb, + struct net_device *sb_dev) +{ +#ifdef CONFIG_DCB + struct otx2_nic *pf = netdev_priv(netdev); + u8 vlan_prio; +#endif + +#ifdef CONFIG_DCB + if (!skb->vlan_present) + goto pick_tx; + + vlan_prio = skb->vlan_tci >> 13; + if ((vlan_prio > pf->hw.tx_queues - 1) || + !pf->pfc_alloc_status[vlan_prio]) + goto pick_tx; + + return vlan_prio; + +pick_tx: +#endif + return netdev_pick_tx(netdev, skb, NULL); +} + static netdev_features_t otx2_fix_features(struct net_device *dev, netdev_features_t features) { @@ -1987,8 +2047,19 @@ int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) switch (config.tx_type) { case HWTSTAMP_TX_OFF: + if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC) + pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC; + + cancel_delayed_work(&pfvf->ptp->synctstamp_work); otx2_config_hw_tx_tstamp(pfvf, false); break; + case HWTSTAMP_TX_ONESTEP_SYNC: + if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag)) + return -ERANGE; + pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC; + schedule_delayed_work(&pfvf->ptp->synctstamp_work, + msecs_to_jiffies(500)); + fallthrough; case HWTSTAMP_TX_ON: otx2_config_hw_tx_tstamp(pfvf, true); break; @@ -2447,6 +2518,7 @@ static const struct net_device_ops otx2_netdev_ops = { .ndo_open = otx2_open, .ndo_stop = otx2_stop, .ndo_start_xmit = otx2_xmit, + .ndo_select_queue = otx2_select_queue, .ndo_fix_features = otx2_fix_features, .ndo_set_mac_address = otx2_set_mac_address, .ndo_change_mtu = otx2_change_mtu, @@ -2702,6 +2774,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = cn10k_mcs_init(pf); + if (err) + goto err_del_mcam_entries; + if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT) netdev->hw_features |= NETIF_F_NTUPLE; @@ -2734,7 +2810,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_del_mcam_entries; + goto err_mcs_free; } err = otx2_wq_init(pf); @@ -2773,6 +2849,8 @@ err_mcam_flow_del: otx2_mcam_flow_del(pf); err_unreg_netdev: unregister_netdev(netdev); +err_mcs_free: + cn10k_mcs_free(pf); err_del_mcam_entries: otx2_mcam_flow_del(pf); err_ptp_destroy: @@ -2916,6 +2994,8 @@ static void otx2_remove(struct pci_dev *pdev) otx2_config_pause_frm(pf); } + cn10k_mcs_free(pf); + #ifdef CONFIG_DCB /* Disable PFC config */ if (pf->pfc_en) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c index fdc2c9315b91fb2561d50372e83298a5159940b4..896b2f9bac34416e202aee9bca357fa258796c47 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c @@ -10,6 +10,33 @@ #include "otx2_common.h" #include "otx2_ptp.h" +static u64 otx2_ptp_get_clock(struct otx2_ptp *ptp) +{ + struct ptp_req *req; + struct ptp_rsp *rsp; + int err; + + if (!ptp->nic) + return 0; + + req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); + if (!req) + return 0; + + req->op = PTP_OP_GET_CLOCK; + + err = otx2_sync_mbox_msg(&ptp->nic->mbox); + if (err) + return 0; + + rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, + &req->hdr); + if (IS_ERR(rsp)) + return 0; + + return rsp->clk; +} + static int otx2_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -46,32 +73,28 @@ static int ptp_set_thresh(struct otx2_ptp *ptp, u64 thresh) return otx2_sync_mbox_msg(&ptp->nic->mbox); } -static u64 ptp_cc_read(const struct cyclecounter *cc) +static int ptp_extts_on(struct otx2_ptp *ptp, int on) { - struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); struct ptp_req *req; - struct ptp_rsp *rsp; - int err; if (!ptp->nic) - return 0; + return -ENODEV; req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox); if (!req) - return 0; + return -ENOMEM; - req->op = PTP_OP_GET_CLOCK; + req->op = PTP_OP_EXTTS_ON; + req->extts_on = on; - err = otx2_sync_mbox_msg(&ptp->nic->mbox); - if (err) - return 0; + return otx2_sync_mbox_msg(&ptp->nic->mbox); +} - rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0, - &req->hdr); - if (IS_ERR(rsp)) - return 0; +static u64 ptp_cc_read(const struct cyclecounter *cc) +{ + struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter); - return rsp->clk; + return otx2_ptp_get_clock(ptp); } static u64 ptp_tstmp_read(struct otx2_ptp *ptp) @@ -101,6 +124,15 @@ static u64 ptp_tstmp_read(struct otx2_ptp *ptp) return rsp->clk; } +static void otx2_get_ptpclock(struct otx2_ptp *ptp, u64 *tstamp) +{ + struct otx2_nic *pfvf = ptp->nic; + + mutex_lock(&pfvf->mbox.lock); + *tstamp = timecounter_read(&ptp->time_counter); + mutex_unlock(&pfvf->mbox.lock); +} + static int otx2_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta) { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, @@ -119,14 +151,10 @@ static int otx2_ptp_gettime(struct ptp_clock_info *ptp_info, { struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp, ptp_info); - struct otx2_nic *pfvf = ptp->nic; - u64 nsec; + u64 tstamp; - mutex_lock(&pfvf->mbox.lock); - nsec = timecounter_read(&ptp->time_counter); - mutex_unlock(&pfvf->mbox.lock); - - *ts = ns_to_timespec64(nsec); + otx2_get_ptpclock(ptp, &tstamp); + *ts = ns_to_timespec64(tstamp); return 0; } @@ -178,8 +206,6 @@ static void otx2_ptp_extts_check(struct work_struct *work) event.index = 0; event.timestamp = timecounter_cyc2time(&ptp->time_counter, tstmp); ptp_clock_event(ptp->ptp_clock, &event); - ptp->last_extts = tstmp; - new_thresh = tstmp % 500000000; if (ptp->thresh != new_thresh) { mutex_lock(&ptp->nic->mbox.lock); @@ -187,10 +213,28 @@ static void otx2_ptp_extts_check(struct work_struct *work) mutex_unlock(&ptp->nic->mbox.lock); ptp->thresh = new_thresh; } + ptp->last_extts = tstmp; } schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); } +static void otx2_sync_tstamp(struct work_struct *work) +{ + struct otx2_ptp *ptp = container_of(work, struct otx2_ptp, + synctstamp_work.work); + struct otx2_nic *pfvf = ptp->nic; + u64 tstamp; + + mutex_lock(&pfvf->mbox.lock); + tstamp = otx2_ptp_get_clock(ptp); + mutex_unlock(&pfvf->mbox.lock); + + ptp->tstamp = timecounter_cyc2time(&pfvf->ptp->time_counter, tstamp); + ptp->base_ns = tstamp % NSEC_PER_SEC; + + schedule_delayed_work(&ptp->synctstamp_work, msecs_to_jiffies(250)); +} + static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, struct ptp_clock_request *rq, int on) { @@ -207,10 +251,13 @@ static int otx2_ptp_enable(struct ptp_clock_info *ptp_info, rq->extts.index); if (pin < 0) return -EBUSY; - if (on) + if (on) { + ptp_extts_on(ptp, on); schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200)); - else + } else { + ptp_extts_on(ptp, on); cancel_delayed_work_sync(&ptp->extts_work); + } return 0; default: break; @@ -302,6 +349,8 @@ int otx2_ptp_init(struct otx2_nic *pfvf) ptp_ptr->convert_tx_ptp_tstmp = &cn10k_ptp_convert_timestamp; } + INIT_DELAYED_WORK(&ptp_ptr->synctstamp_work, otx2_sync_tstamp); + pfvf->ptp = ptp_ptr; error: @@ -316,6 +365,8 @@ void otx2_ptp_destroy(struct otx2_nic *pfvf) if (!ptp) return; + cancel_delayed_work(&pfvf->ptp->synctstamp_work); + ptp_clock_unregister(ptp->ptp_clock); kfree(ptp); pfvf->ptp = NULL; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h index 4bbd12ff26e64557f9b4abdd3586f6eb8c106e22..aa205a0d158f6495f279665a36b7c017f646401a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h @@ -236,8 +236,15 @@ struct nix_sqe_sg_s { /* NIX send memory subdescriptor structure */ struct nix_sqe_mem_s { - u64 offset : 16; /* W0 */ - u64 rsvd_51_16 : 36; + u64 start_offset : 8; + u64 rsvd_11_8 : 4; + u64 rsvd_12 : 1; + u64 udp_csum_crt : 1; + u64 update64 : 1; + u64 rsvd_15_16 : 1; + u64 base_ns : 32; + u64 step_type : 1; + u64 rsvd_51_49 : 3; u64 per_lso_seg : 1; u64 wmem : 1; u64 dsz : 2; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index a18e8efd0f1ee53d891e9fd8faaef39aa2df1774..5ec11d71bf606686ae66adc746e02ea20f1f59ab 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -19,6 +19,12 @@ #include "cn10k.h" #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx))) +#define PTP_PORT 0x13F +/* PTPv2 header Original Timestamp starts at byte offset 34 and + * contains 6 byte seconds field and 4 byte nano seconds field. + */ +#define PTP_SYNC_SEC_OFFSET 34 + static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf, struct bpf_prog *prog, struct nix_cqe_rx_s *cqe, @@ -686,7 +692,8 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, } static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, - int alg, u64 iova) + int alg, u64 iova, int ptp_offset, + u64 base_ns, int udp_csum) { struct nix_sqe_mem_s *mem; @@ -696,6 +703,13 @@ static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset, mem->wmem = 1; /* wait for the memory operation */ mem->addr = iova; + if (ptp_offset) { + mem->start_offset = ptp_offset; + mem->udp_csum_crt = udp_csum; + mem->base_ns = base_ns; + mem->step_type = 1; + } + *offset += sizeof(*mem); } @@ -952,16 +966,102 @@ static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb) return skb_shinfo(skb)->gso_segs; } +static bool otx2_validate_network_transport(struct sk_buff *skb) +{ + if ((ip_hdr(skb)->protocol == IPPROTO_UDP) || + (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) { + struct udphdr *udph = udp_hdr(skb); + + if (udph->source == htons(PTP_PORT) && + udph->dest == htons(PTP_PORT)) + return true; + } + + return false; +} + +static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, int *udp_csum) +{ + struct ethhdr *eth = (struct ethhdr *)(skb->data); + u16 nix_offload_hlen = 0, inner_vhlen = 0; + u8 *data = skb->data, *msgtype; + __be16 proto = eth->h_proto; + int network_depth = 0; + + /* NIX is programmed to offload outer VLAN header + * in case of single vlan protocol field holds Network header ETH_IP/V6 + * in case of stacked vlan protocol field holds Inner vlan (8100) + */ + if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX && + skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) { + if (skb->vlan_proto == htons(ETH_P_8021AD)) { + /* Get vlan protocol */ + proto = __vlan_get_protocol(skb, eth->h_proto, NULL); + /* SKB APIs like skb_transport_offset does not include + * offloaded vlan header length. Need to explicitly add + * the length + */ + nix_offload_hlen = VLAN_HLEN; + inner_vhlen = VLAN_HLEN; + } else if (skb->vlan_proto == htons(ETH_P_8021Q)) { + nix_offload_hlen = VLAN_HLEN; + } + } else if (eth_type_vlan(eth->h_proto)) { + proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth); + } + + switch (ntohs(proto)) { + case ETH_P_1588: + if (network_depth) + *offset = network_depth; + else + *offset = ETH_HLEN + nix_offload_hlen + + inner_vhlen; + break; + case ETH_P_IP: + case ETH_P_IPV6: + if (!otx2_validate_network_transport(skb)) + return false; + + *udp_csum = 1; + *offset = nix_offload_hlen + skb_transport_offset(skb) + + sizeof(struct udphdr); + } + + msgtype = data + *offset; + + /* Check PTP messageId is SYNC or not */ + return (*msgtype & 0xf) == 0; +} + static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb, struct otx2_snd_queue *sq, int *offset) { + struct ptpv2_tstamp *origin_tstamp; + int ptp_offset = 0, udp_csum = 0; + struct timespec64 ts; u64 iova; - if (!skb_shinfo(skb)->gso_size && - skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + if (unlikely(!skb_shinfo(skb)->gso_size && + (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) { + if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)) { + if (otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum)) { + origin_tstamp = (struct ptpv2_tstamp *) + ((u8 *)skb->data + ptp_offset + + PTP_SYNC_SEC_OFFSET); + ts = ns_to_timespec64(pfvf->ptp->tstamp); + origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff); + origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff); + origin_tstamp->nanoseconds = htonl(ts.tv_nsec); + /* Point to correction field in PTP packet */ + ptp_offset += 8; + } + } else { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + } iova = sq->timestamps->iova + (sq->head * sizeof(u64)); - otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova); + otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova, + ptp_offset, pfvf->ptp->base_ns, udp_csum); } else { skb_tx_timestamp(skb); } diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile index d395f4131648b484fb4de691b26d5e94c430dc19..df14cee801530e0d37d85c5c4de17d486b253f69 100644 --- a/drivers/net/ethernet/marvell/prestera/Makefile +++ b/drivers/net/ethernet/marvell/prestera/Makefile @@ -4,6 +4,6 @@ prestera-objs := prestera_main.o prestera_hw.o prestera_dsa.o \ prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \ prestera_switchdev.o prestera_acl.o prestera_flow.o \ prestera_flower.o prestera_span.o prestera_counter.o \ - prestera_router.o prestera_router_hw.o + prestera_router.o prestera_router_hw.o prestera_matchall.o obj-$(CONFIG_PRESTERA_PCI) += prestera_pci.o diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index 2f84d0fb40941e9e125e6b8fc7149f6899bc79d2..35554ee805cd0db7d3dd2efe588eaf3ea1371277 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -306,17 +306,27 @@ struct prestera_switch { struct prestera_counter *counter; u8 lag_member_max; u8 lag_max; + u32 size_tbl_router_nexthop; }; struct prestera_router { struct prestera_switch *sw; struct list_head vr_list; struct list_head rif_entry_list; + struct rhashtable nh_neigh_ht; + struct rhashtable nexthop_group_ht; struct rhashtable fib_ht; + struct rhashtable kern_neigh_cache_ht; struct rhashtable kern_fib_cache_ht; struct notifier_block inetaddr_nb; struct notifier_block inetaddr_valid_nb; struct notifier_block fib_nb; + struct notifier_block netevent_nb; + u8 *nhgrp_hw_state_cache; /* Bitmap cached hw state of nhs */ + unsigned long nhgrp_hw_cache_kick; /* jiffies */ + struct { + struct delayed_work dw; + } neighs_update; }; struct prestera_rxtx_params { @@ -362,11 +372,15 @@ int prestera_port_cfg_mac_write(struct prestera_port *port, struct prestera_port *prestera_port_dev_lower_find(struct net_device *dev); void prestera_queue_work(struct work_struct *work); +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay); +void prestera_queue_drain(void); int prestera_port_learning_set(struct prestera_port *port, bool learn_enable); int prestera_port_uc_flood_set(struct prestera_port *port, bool flood); int prestera_port_mc_flood_set(struct prestera_port *port, bool flood); +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked); + int prestera_port_pvid_set(struct prestera_port *port, u16 vid); bool prestera_netdev_check(const struct net_device *dev); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c index 3d4b85f2d54147b3659091e54dee58f38c064e28..cba89fda504bbb6f8f506776a1b6da1da6c6af3b 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -54,6 +54,10 @@ struct prestera_acl_ruleset { struct prestera_acl_ruleset_ht_key ht_key; struct rhashtable rule_ht; struct prestera_acl *acl; + struct { + u32 min; + u32 max; + } prio; unsigned long rule_count; refcount_t refcount; void *keymask; @@ -162,6 +166,9 @@ prestera_acl_ruleset_create(struct prestera_acl *acl, ruleset->pcl_id = PRESTERA_ACL_PCL_ID_MAKE((u8)uid, chain_index); ruleset->index = uid; + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node, prestera_acl_ruleset_ht_params); if (err) @@ -178,10 +185,14 @@ err_rhashtable_init: return ERR_PTR(err); } -void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, - void *keymask) +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask) { ruleset->keymask = kmemdup(keymask, ACL_KEYMASK_SIZE, GFP_KERNEL); + if (!ruleset->keymask) + return -ENOMEM; + + return 0; } int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset) @@ -365,6 +376,26 @@ prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset, block->ruleset_zero = NULL; } +static void +prestera_acl_ruleset_prio_refresh(struct prestera_acl *acl, + struct prestera_acl_ruleset *ruleset) +{ + struct prestera_acl_rule *rule; + + ruleset->prio.min = UINT_MAX; + ruleset->prio.max = 0; + + list_for_each_entry(rule, &acl->rules, list) { + if (ruleset->ingress != rule->ruleset->ingress) + continue; + if (ruleset->ht_key.chain_index != rule->chain_index) + continue; + + ruleset->prio.min = min(ruleset->prio.min, rule->priority); + ruleset->prio.max = max(ruleset->prio.max, rule->priority); + } +} + void prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id) { @@ -389,6 +420,13 @@ u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset) return ruleset->index; } +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max) +{ + *prio_min = ruleset->prio.min; + *prio_max = ruleset->prio.max; +} + bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset) { return ruleset->offload; @@ -429,6 +467,13 @@ void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) kfree(rule); } +static void prestera_acl_ruleset_prio_update(struct prestera_acl_ruleset *ruleset, + u32 prio) +{ + ruleset->prio.min = min(ruleset->prio.min, prio); + ruleset->prio.max = max(ruleset->prio.max, prio); +} + int prestera_acl_rule_add(struct prestera_switch *sw, struct prestera_acl_rule *rule) { @@ -468,6 +513,7 @@ int prestera_acl_rule_add(struct prestera_switch *sw, list_add_tail(&rule->list, &sw->acl->rules); ruleset->rule_count++; + prestera_acl_ruleset_prio_update(ruleset, rule->priority); return 0; err_acl_block_bind: @@ -492,6 +538,7 @@ void prestera_acl_rule_del(struct prestera_switch *sw, list_del(&rule->list); prestera_acl_rule_entry_destroy(sw->acl, rule->re); + prestera_acl_ruleset_prio_refresh(sw->acl, ruleset); /* unbind block (all ports) */ if (!ruleset->ht_key.chain_index && !ruleset->rule_count) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h index 03fc5b9dc9258686944d1fd41def03d59fb3e795..a35cc0609a1d2149308ca1654a847b710eb67580 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h @@ -185,8 +185,8 @@ struct prestera_acl_ruleset * prestera_acl_ruleset_lookup(struct prestera_acl *acl, struct prestera_flow_block *block, u32 chain_index); -void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, - void *keymask); +int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset, + void *keymask); bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset); int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset); void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset); @@ -195,6 +195,8 @@ int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, struct prestera_port *port); u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset); +void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset, + u32 *prio_min, u32 *prio_max); void prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c index 1da7ff889417f89ec84bb76d28165ea4c2d34a6b..2f52daba58e6c1369a2db5b8479bc3b81b24b653 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c @@ -300,8 +300,8 @@ static void prestera_ethtool_get_drvinfo(struct net_device *dev, struct prestera_port *port = netdev_priv(dev); struct prestera_switch *sw = port->sw; - strlcpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), + strscpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(prestera_dev(sw)), sizeof(drvinfo->bus_info)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c index 2262693bd5cfb56cb051b94c5902072f89649a77..9f4267f326b0114ac64ba47fc424e5aeb6de6170 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c @@ -7,8 +7,9 @@ #include "prestera.h" #include "prestera_acl.h" #include "prestera_flow.h" -#include "prestera_span.h" #include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" static LIST_HEAD(prestera_block_cb_list); @@ -17,9 +18,9 @@ static int prestera_flow_block_mall_cb(struct prestera_flow_block *block, { switch (f->command) { case TC_CLSMATCHALL_REPLACE: - return prestera_span_replace(block, f); + return prestera_mall_replace(block, f); case TC_CLSMATCHALL_DESTROY: - prestera_span_destroy(block); + prestera_mall_destroy(block); return 0; default: return -EOPNOTSUPP; @@ -89,6 +90,9 @@ prestera_flow_block_create(struct prestera_switch *sw, INIT_LIST_HEAD(&block->template_list); block->net = net; block->sw = sw; + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; block->ingress = ingress; return block; @@ -263,7 +267,7 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port, block = flow_block_cb_priv(block_cb); - prestera_span_destroy(block); + prestera_mall_destroy(block); err = prestera_flow_block_unbind(block, port); if (err) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h index 0c9e13263261df73fe932811352e25a7370a82e7..a85a3eb40279acb319fabc53e2f8392370ea6838 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h @@ -22,6 +22,11 @@ struct prestera_flow_block { struct prestera_acl_ruleset *ruleset_zero; struct flow_block_cb *block_cb; struct list_head template_list; + struct { + u32 prio_min; + u32 prio_max; + bool bound; + } mall; unsigned int rule_count; bool ingress; }; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index 19d3b55c578eedd370aee0e62ec7bb99c75fa64d..91a478b75cbf009ff5caa8ccb550e38da1dd7836 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -5,6 +5,7 @@ #include "prestera_acl.h" #include "prestera_flow.h" #include "prestera_flower.h" +#include "prestera_matchall.h" struct prestera_flower_template { struct prestera_acl_ruleset *ruleset; @@ -360,6 +361,49 @@ static int prestera_flower_parse(struct prestera_flow_block *block, f->common.extack); } +static int prestera_flower_prio_check(struct prestera_flow_block *block, + struct flow_cls_offload *f) +{ + u32 mall_prio_min; + u32 mall_prio_max; + int err; + + err = prestera_mall_prio_get(block, &mall_prio_min, &mall_prio_max); + if (err == -ENOENT) + /* No matchall filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get matchall priorities"); + return err; + } + + if (f->common.prio <= mall_prio_max && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, + "Failed to add in front of existing matchall rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= mall_prio_min && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing matchall rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max) +{ + struct prestera_acl_ruleset *ruleset; + + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, chain_index); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + prestera_acl_ruleset_prio_get(ruleset, prio_min, prio_max); + return 0; +} + int prestera_flower_replace(struct prestera_flow_block *block, struct flow_cls_offload *f) { @@ -368,6 +412,10 @@ int prestera_flower_replace(struct prestera_flow_block *block, struct prestera_acl_rule *rule; int err; + err = prestera_flower_prio_check(block, f); + if (err) + return err; + ruleset = prestera_acl_ruleset_get(acl, block, f->common.chain_index); if (IS_ERR(ruleset)) return PTR_ERR(ruleset); @@ -452,7 +500,9 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, } /* preserve keymask/template to this ruleset */ - prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask); + err = prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask); + if (err) + goto err_ruleset_keymask_set; /* skip error, as it is not possible to reject template operation, * so, keep the reference to the ruleset for rules to be added @@ -468,6 +518,8 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, list_add_rcu(&template->list, &block->template_list); return 0; +err_ruleset_keymask_set: + prestera_acl_ruleset_put(ruleset); err_ruleset_get: kfree(template); err_malloc: diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h index 495f151e6fa9cbfbecc1d1cff2145472904c3ee0..1181115fe6fadadf4b9676d5bc4e62bf8b20fa11 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h @@ -19,5 +19,7 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block, void prestera_flower_tmplt_destroy(struct prestera_flow_block *block, struct flow_cls_offload *f); void prestera_flower_template_cleanup(struct prestera_flow_block *block); +int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index, + u32 *prio_min, u32 *prio_max); #endif /* _PRESTERA_FLOWER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c index 962d7e0c0cb5819150e61d0cf0fe6fffbd14f49c..fc6f7d2746e887157ab4ea978914be6c7b1de483 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c @@ -10,11 +10,14 @@ #include "prestera_hw.h" #include "prestera_acl.h" #include "prestera_counter.h" +#include "prestera_router_hw.h" #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000) #define PRESTERA_MIN_MTU 64 +#define PRESTERA_MSG_CHUNK_SIZE 1024 + enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_SWITCH_INIT = 0x1, PRESTERA_CMD_TYPE_SWITCH_ATTR_SET = 0x2, @@ -57,6 +60,10 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_ROUTER_RIF_DELETE = 0x601, PRESTERA_CMD_TYPE_ROUTER_LPM_ADD = 0x610, PRESTERA_CMD_TYPE_ROUTER_LPM_DELETE = 0x611, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET = 0x622, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET = 0x645, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD = 0x623, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE = 0x624, PRESTERA_CMD_TYPE_ROUTER_VR_CREATE = 0x630, PRESTERA_CMD_TYPE_ROUTER_VR_DELETE = 0x631, @@ -78,9 +85,11 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_STP_PORT_SET = 0x1000, PRESTERA_CMD_TYPE_SPAN_GET = 0x1100, - PRESTERA_CMD_TYPE_SPAN_BIND = 0x1101, - PRESTERA_CMD_TYPE_SPAN_UNBIND = 0x1102, + PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND = 0x1101, + PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND = 0x1102, PRESTERA_CMD_TYPE_SPAN_RELEASE = 0x1103, + PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND = 0x1104, + PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND = 0x1105, PRESTERA_CMD_TYPE_POLICER_CREATE = 0x1500, PRESTERA_CMD_TYPE_POLICER_RELEASE = 0x1501, @@ -101,6 +110,7 @@ enum { PRESTERA_CMD_PORT_ATTR_LEARNING = 7, PRESTERA_CMD_PORT_ATTR_FLOOD = 8, PRESTERA_CMD_PORT_ATTR_CAPABILITY = 9, + PRESTERA_CMD_PORT_ATTR_LOCKED = 10, PRESTERA_CMD_PORT_ATTR_PHY_MODE = 12, PRESTERA_CMD_PORT_ATTR_TYPE = 13, PRESTERA_CMD_PORT_ATTR_STATS = 17, @@ -285,6 +295,7 @@ union prestera_msg_port_param { u8 duplex; u8 fec; u8 fc; + u8 br_locked; union { struct { u8 admin; @@ -538,6 +549,14 @@ struct prestera_msg_ip_addr { u8 __pad[3]; }; +struct prestera_msg_nh { + struct prestera_msg_iface oif; + __le32 hw_id; + u8 mac[ETH_ALEN]; + u8 is_active; + u8 pad; +}; + struct prestera_msg_rif_req { struct prestera_msg_cmd cmd; struct prestera_msg_iface iif; @@ -563,6 +582,34 @@ struct prestera_msg_lpm_req { u8 __pad[2]; }; +struct prestera_msg_nh_req { + struct prestera_msg_cmd cmd; + struct prestera_msg_nh nh[PRESTERA_NHGR_SIZE_MAX]; + __le32 size; + __le32 grp_id; +}; + +struct prestera_msg_nh_chunk_req { + struct prestera_msg_cmd cmd; + __le32 offset; +}; + +struct prestera_msg_nh_chunk_resp { + struct prestera_msg_ret ret; + u8 hw_state[PRESTERA_MSG_CHUNK_SIZE]; +}; + +struct prestera_msg_nh_grp_req { + struct prestera_msg_cmd cmd; + __le32 grp_id; + __le32 size; +}; + +struct prestera_msg_nh_grp_resp { + struct prestera_msg_ret ret; + __le32 grp_id; +}; + struct prestera_msg_vr_req { struct prestera_msg_cmd cmd; __le16 vr_id; @@ -725,11 +772,15 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_ports_reset_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_create_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_destroy_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_req) != 124); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_req) != 12); /* structure that are part of req/resp fw messages */ BUILD_BUG_ON(sizeof(struct prestera_msg_iface) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_ip_addr) != 20); BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_port) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh) != 28); /* check responses */ BUILD_BUG_ON(sizeof(struct prestera_msg_common_resp) != 8); @@ -745,6 +796,9 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_rif_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_vr_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_policer_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_create_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_resp) != 1032); + BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_resp) != 12); /* check events */ BUILD_BUG_ON(sizeof(struct prestera_msg_event_port) != 20); @@ -1022,6 +1076,8 @@ int prestera_hw_switch_init(struct prestera_switch *sw) sw->id = resp.switch_id; sw->lag_member_max = resp.lag_member_max; sw->lag_max = resp.lag_max; + sw->size_tbl_router_nexthop = + __le32_to_cpu(resp.size_tbl_router_nexthop); return 0; } @@ -1431,27 +1487,39 @@ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id) return 0; } -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id) +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), .id = span_id, }; + enum prestera_cmd_type_t cmd_type; + + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_BIND, - &req.cmd, sizeof(req)); } -int prestera_hw_span_unbind(const struct prestera_port *port) +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress) { struct prestera_msg_span_req req = { .port = __cpu_to_le32(port->hw_id), .dev = __cpu_to_le32(port->dev_id), }; + enum prestera_cmd_type_t cmd_type; - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_UNBIND, - &req.cmd, sizeof(req)); + if (ingress) + cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND; + else + cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND; + + return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req)); } int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id) @@ -1639,6 +1707,22 @@ int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood) &req.cmd, sizeof(req)); } +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked) +{ + struct prestera_msg_port_attr_req req = { + .attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_LOCKED), + .port = __cpu_to_le32(port->hw_id), + .dev = __cpu_to_le32(port->dev_id), + .param = { + .br_locked = br_locked, + } + }; + + return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_PORT_ATTR_SET, + &req.cmd, sizeof(req)); +} + int prestera_hw_vlan_create(struct prestera_switch *sw, u16 vid) { struct prestera_msg_vlan_req req = { @@ -2004,6 +2088,85 @@ int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, sizeof(req)); } +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id) +{ + struct prestera_msg_nh_req req = { .size = __cpu_to_le32((u32)count), + .grp_id = __cpu_to_le32(grp_id) }; + int i, err; + + for (i = 0; i < count; i++) { + req.nh[i].is_active = nhs[i].connected; + memcpy(&req.nh[i].mac, nhs[i].ha, ETH_ALEN); + err = prestera_iface_to_msg(&nhs[i].iface, &req.nh[i].oif); + if (err) + return err; + } + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET, &req.cmd, + sizeof(req)); +} + +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */) +{ + static struct prestera_msg_nh_chunk_resp resp; + struct prestera_msg_nh_chunk_req req; + u32 buf_offset; + int err; + + memset(&hw_state[0], 0, buf_size); + buf_offset = 0; + while (1) { + if (buf_offset >= buf_size) + break; + + memset(&req, 0, sizeof(req)); + req.offset = __cpu_to_le32(buf_offset * 8); /* 8 bits in u8 */ + err = prestera_cmd_ret(sw, + PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET, + &req.cmd, sizeof(req), &resp.ret, + sizeof(resp)); + if (err) + return err; + + memcpy(&hw_state[buf_offset], &resp.hw_state[0], + buf_offset + PRESTERA_MSG_CHUNK_SIZE > buf_size ? + buf_size - buf_offset : PRESTERA_MSG_CHUNK_SIZE); + buf_offset += PRESTERA_MSG_CHUNK_SIZE; + } + + return 0; +} + +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id) +{ + struct prestera_msg_nh_grp_req req = { .size = __cpu_to_le32((u32)nh_count) }; + struct prestera_msg_nh_grp_resp resp; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *grp_id = __le32_to_cpu(resp.grp_id); + return err; +} + +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id) +{ + struct prestera_msg_nh_grp_req req = { + .grp_id = __cpu_to_le32(grp_id), + .size = __cpu_to_le32(nh_count) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE, + &req.cmd, sizeof(req)); +} + int prestera_hw_rxtx_init(struct prestera_switch *sw, struct prestera_rxtx_params *params) { diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h index 56e043146dd256423625ba3b757a8172f3af2002..0a929279e1cea482f95337788c3d0078474774dd 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h @@ -146,6 +146,7 @@ struct prestera_counter_stats; struct prestera_iface; struct prestera_flood_domain; struct prestera_mdb_entry; +struct prestera_neigh_info; /* Switch API */ int prestera_hw_switch_init(struct prestera_switch *sw); @@ -183,6 +184,8 @@ int prestera_hw_port_speed_get(const struct prestera_port *port, u32 *speed); int prestera_hw_port_learning_set(struct prestera_port *port, bool enable); int prestera_hw_port_uc_flood_set(const struct prestera_port *port, bool flood); int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood); +int prestera_hw_port_br_locked_set(const struct prestera_port *port, + bool br_locked); int prestera_hw_port_accept_frm_type(struct prestera_port *port, enum prestera_accept_frm_type type); /* Vlan API */ @@ -243,8 +246,9 @@ int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, /* SPAN API */ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id); -int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id); -int prestera_hw_span_unbind(const struct prestera_port *port); +int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id, + bool ingress); +int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress); int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id); /* Router API */ @@ -263,6 +267,16 @@ int prestera_hw_lpm_add(struct prestera_switch *sw, u16 vr_id, int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id, __be32 dst, u32 dst_len); +/* NH API */ +int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count, + struct prestera_neigh_info *nhs, u32 grp_id); +int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw, + u8 *hw_state, u32 buf_size /* Buffer in bytes */); +int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count, + u32 *grp_id); +int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count, + u32 grp_id); + /* Event handlers */ int prestera_hw_event_handler_register(struct prestera_switch *sw, enum prestera_event_type type, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index ede3e53b97901a16c6c347036920058a27abda6b..24f9d60247453107f79f425e1918928d27a92568 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -36,6 +36,17 @@ void prestera_queue_work(struct work_struct *work) queue_work(prestera_owq, work); } +void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay) +{ + queue_delayed_work(prestera_wq, work, delay); +} + +void prestera_queue_drain(void) +{ + drain_workqueue(prestera_wq); + drain_workqueue(prestera_owq); +} + int prestera_port_learning_set(struct prestera_port *port, bool learn) { return prestera_hw_port_learning_set(port, learn); @@ -51,6 +62,11 @@ int prestera_port_mc_flood_set(struct prestera_port *port, bool flood) return prestera_hw_port_mc_flood_set(port, flood); } +int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked) +{ + return prestera_hw_port_br_locked_set(port, br_locked); +} + int prestera_port_pvid_set(struct prestera_port *port, u16 vid) { enum prestera_accept_frm_type frm_type; @@ -368,6 +384,7 @@ static int prestera_port_sfp_bind(struct prestera_port *port) if (!sw->np) return 0; + of_node_get(sw->np); ports = of_find_node_by_name(sw->np, "ports"); for_each_child_of_node(ports, node) { @@ -417,6 +434,7 @@ static int prestera_port_sfp_bind(struct prestera_port *port) } out: + of_node_put(node); of_node_put(ports); return err; } @@ -797,32 +815,30 @@ static void prestera_port_handle_event(struct prestera_switch *sw, caching_dw = &port->cached_hw_stats.caching_dw; - if (port->phy_link) { - memset(&smac, 0, sizeof(smac)); - smac.valid = true; - smac.oper = pevt->data.mac.oper; - if (smac.oper) { - smac.mode = pevt->data.mac.mode; - smac.speed = pevt->data.mac.speed; - smac.duplex = pevt->data.mac.duplex; - smac.fc = pevt->data.mac.fc; - smac.fec = pevt->data.mac.fec; - phylink_mac_change(port->phy_link, true); - } else { - phylink_mac_change(port->phy_link, false); - } - prestera_port_mac_state_cache_write(port, &smac); + memset(&smac, 0, sizeof(smac)); + smac.valid = true; + smac.oper = pevt->data.mac.oper; + if (smac.oper) { + smac.mode = pevt->data.mac.mode; + smac.speed = pevt->data.mac.speed; + smac.duplex = pevt->data.mac.duplex; + smac.fc = pevt->data.mac.fc; + smac.fec = pevt->data.mac.fec; } + prestera_port_mac_state_cache_write(port, &smac); if (port->state_mac.oper) { - if (!port->phy_link) + if (port->phy_link) + phylink_mac_change(port->phy_link, true); + else netif_carrier_on(port->dev); if (!delayed_work_pending(caching_dw)) queue_delayed_work(prestera_wq, caching_dw, 0); - } else if (netif_running(port->dev) && - netif_carrier_ok(port->dev)) { - if (!port->phy_link) + } else { + if (port->phy_link) + phylink_mac_change(port->phy_link, false); + else if (netif_running(port->dev) && netif_carrier_ok(port->dev)) netif_carrier_off(port->dev); if (delayed_work_pending(caching_dw)) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.c b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c new file mode 100644 index 0000000000000000000000000000000000000000..1da9c1bc1ee9a23ccecb68a32b9a742f8eaef1f1 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2019-2022 Marvell International Ltd. All rights reserved */ + +#include +#include + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_flow.h" +#include "prestera_flower.h" +#include "prestera_matchall.h" +#include "prestera_span.h" + +static int prestera_mall_prio_check(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + u32 flower_prio_min; + u32 flower_prio_max; + int err; + + err = prestera_flower_prio_get(block, f->common.chain_index, + &flower_prio_min, &flower_prio_max); + if (err == -ENOENT) + /* No flower filters installed on this chain. */ + return 0; + + if (err) { + NL_SET_ERR_MSG(f->common.extack, "Failed to get flower priorities"); + return err; + } + + if (f->common.prio <= flower_prio_max && !block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add in front of existing flower rules"); + return -EOPNOTSUPP; + } + if (f->common.prio >= flower_prio_min && block->ingress) { + NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing flower rules"); + return -EOPNOTSUPP; + } + + return 0; +} + +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max) +{ + if (!block->mall.bound) + return -ENOENT; + + *prio_min = block->mall.prio_min; + *prio_max = block->mall.prio_max; + return 0; +} + +static void prestera_mall_prio_update(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + block->mall.prio_min = min(block->mall.prio_min, f->common.prio); + block->mall.prio_max = max(block->mall.prio_max, f->common.prio); +} + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f) +{ + struct prestera_flow_block_binding *binding; + __be16 protocol = f->common.protocol; + struct flow_action_entry *act; + struct prestera_port *port; + int err; + + if (!flow_offload_has_one_action(&f->rule->action)) { + NL_SET_ERR_MSG(f->common.extack, + "Only singular actions are supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + + if (!prestera_netdev_check(act->dev)) { + NL_SET_ERR_MSG(f->common.extack, + "Only Marvell Prestera port is supported"); + return -EINVAL; + } + if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) + return -EOPNOTSUPP; + if (act->id != FLOW_ACTION_MIRRED) + return -EOPNOTSUPP; + if (protocol != htons(ETH_P_ALL)) + return -EOPNOTSUPP; + + err = prestera_mall_prio_check(block, f); + if (err) + return err; + + port = netdev_priv(act->dev); + + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_span_rule_add(binding, port, block->ingress); + if (err == -EEXIST) + return err; + if (err) + goto rollback; + } + + prestera_mall_prio_update(block, f); + + block->mall.bound = true; + return 0; + +rollback: + list_for_each_entry_continue_reverse(binding, + &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + return err; +} + +void prestera_mall_destroy(struct prestera_flow_block *block) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + prestera_span_rule_del(binding, block->ingress); + + block->mall.prio_min = UINT_MAX; + block->mall.prio_max = 0; + block->mall.bound = false; +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.h b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h new file mode 100644 index 0000000000000000000000000000000000000000..fed08be80257bcc57a7487d3f14b1b9fe96b1370 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2022 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_MATCHALL_H_ +#define _PRESTERA_MATCHALL_H_ + +#include + +struct prestera_flow_block; + +int prestera_mall_replace(struct prestera_flow_block *block, + struct tc_cls_matchall_offload *f); +void prestera_mall_destroy(struct prestera_flow_block *block); +int prestera_mall_prio_get(struct prestera_flow_block *block, + u32 *prio_min, u32 *prio_max); + +#endif /* _PRESTERA_MATCHALL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_pci.c b/drivers/net/ethernet/marvell/prestera/prestera_pci.c index f538a749ebd4d21d0e4ce4e74ce7f1d7d1663987..59470d99f522892ee1042cf59b2fc813612b34ab 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_pci.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_pci.c @@ -872,6 +872,7 @@ static void prestera_pci_remove(struct pci_dev *pdev) static const struct pci_device_id prestera_pci_devices[] = { { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC804) }, { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xC80C) }, + { PCI_DEVICE(PCI_VENDOR_ID_MARVELL, 0xCC1E) }, { } }; MODULE_DEVICE_TABLE(pci, prestera_pci_devices); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c index 58f4e44d5ad7607ddafae57a55ac2dc6ecd8f671..4046be0e86ff4a60aeb793cd59746446c477fb65 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c @@ -7,10 +7,35 @@ #include #include #include +#include +#include +#include +#include +#include #include "prestera.h" #include "prestera_router_hw.h" +#define PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH +#define PRESTERA_NH_PROBE_INTERVAL 5000 /* ms */ + +struct prestera_kern_neigh_cache_key { + struct prestera_ip_addr addr; + struct net_device *dev; +}; + +struct prestera_kern_neigh_cache { + struct prestera_kern_neigh_cache_key key; + struct rhash_head ht_node; + struct list_head kern_fib_cache_list; + /* Hold prepared nh_neigh info if is in_kernel */ + struct prestera_neigh_info nh_neigh_info; + /* Indicate if neighbour is reachable by direct route */ + bool reachable; + /* Lock cache if neigh is present in kernel */ + bool in_kernel; +}; + struct prestera_kern_fib_cache_key { struct prestera_ip_addr addr; u32 prefix_len; @@ -23,15 +48,29 @@ struct prestera_kern_fib_cache { struct { struct prestera_fib_key fib_key; enum prestera_fib_type fib_type; + struct prestera_nexthop_group_key nh_grp_key; } lpm_info; /* hold prepared lpm info */ /* Indicate if route is not overlapped by another table */ struct rhash_head ht_node; /* node of prestera_router */ - struct fib_info *fi; - dscp_t kern_dscp; - u8 kern_type; + struct prestera_kern_neigh_cache_head { + struct prestera_kern_fib_cache *this; + struct list_head head; + struct prestera_kern_neigh_cache *n_cache; + } kern_neigh_cache_head[PRESTERA_NHGR_SIZE_MAX]; + union { + struct fib_notifier_info info; /* point to any of 4/6 */ + struct fib_entry_notifier_info fen4_info; + }; bool reachable; }; +static const struct rhashtable_params __prestera_kern_neigh_cache_ht_params = { + .key_offset = offsetof(struct prestera_kern_neigh_cache, key), + .head_offset = offsetof(struct prestera_kern_neigh_cache, ht_node), + .key_len = sizeof(struct prestera_kern_neigh_cache_key), + .automatic_shrinking = true, +}; + static const struct rhashtable_params __prestera_kern_fib_cache_ht_params = { .key_offset = offsetof(struct prestera_kern_fib_cache, key), .head_offset = offsetof(struct prestera_kern_fib_cache, ht_node), @@ -51,15 +90,450 @@ static u32 prestera_fix_tb_id(u32 tb_id) } static void -prestera_util_fen_info2fib_cache_key(struct fib_entry_notifier_info *fen_info, +prestera_util_fen_info2fib_cache_key(struct fib_notifier_info *info, struct prestera_kern_fib_cache_key *key) { + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + memset(key, 0, sizeof(*key)); + key->addr.v = PRESTERA_IPV4; key->addr.u.ipv4 = cpu_to_be32(fen_info->dst); key->prefix_len = fen_info->dst_len; key->kern_tb_id = fen_info->tb_id; } +static int prestera_util_nhc2nc_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + if (nhc->nhc_gw_family == AF_INET) { + nk->addr.v = PRESTERA_IPV4; + nk->addr.u.ipv4 = nhc->nhc_gw.ipv4; + } else { + nk->addr.v = PRESTERA_IPV6; + nk->addr.u.ipv6 = nhc->nhc_gw.ipv6; + } + + nk->dev = nhc->nhc_dev; + return 0; +} + +static void +prestera_util_nc_key2nh_key(struct prestera_kern_neigh_cache_key *ck, + struct prestera_nh_neigh_key *nk) +{ + memset(nk, 0, sizeof(*nk)); + nk->addr = ck->addr; + nk->rif = (void *)ck->dev; +} + +static bool +prestera_util_nhc_eq_n_cache_key(struct prestera_switch *sw, + struct fib_nh_common *nhc, + struct prestera_kern_neigh_cache_key *nk) +{ + struct prestera_kern_neigh_cache_key tk; + int err; + + err = prestera_util_nhc2nc_key(sw, nhc, &tk); + if (err) + return false; + + if (memcmp(&tk, nk, sizeof(tk))) + return false; + + return true; +} + +static int +prestera_util_neigh2nc_key(struct prestera_switch *sw, struct neighbour *n, + struct prestera_kern_neigh_cache_key *key) +{ + memset(key, 0, sizeof(*key)); + if (n->tbl->family == AF_INET) { + key->addr.v = PRESTERA_IPV4; + key->addr.u.ipv4 = *(__be32 *)n->primary_key; + } else { + return -ENOENT; + } + + key->dev = n->dev; + + return 0; +} + +static bool __prestera_fi_is_direct(struct fib_info *fi) +{ + struct fib_nh *fib_nh; + + if (fib_info_num_path(fi) == 1) { + fib_nh = fib_info_nh(fi, 0); + if (fib_nh->fib_nh_gw_family == AF_UNSPEC) + return true; + } + + return false; +} + +static bool prestera_fi_is_direct(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return __prestera_fi_is_direct(fi); +} + +static bool prestera_fi_is_nh(struct fib_info *fi) +{ + if (fi->fib_type != RTN_UNICAST) + return false; + + return !__prestera_fi_is_direct(fi); +} + +static bool __prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (!fi->fib6_nh->nh_common.nhc_gw_family) + return true; + + return false; +} + +static bool prestera_fi6_is_direct(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return __prestera_fi6_is_direct(fi); +} + +static bool prestera_fi6_is_nh(struct fib6_info *fi) +{ + if (fi->fib6_type != RTN_UNICAST) + return false; + + return !__prestera_fi6_is_direct(fi); +} + +static bool prestera_fib_info_is_direct(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_direct(fen_info->fi); + else + return prestera_fi6_is_direct(fen6_info->rt); +} + +static bool prestera_fib_info_is_nh(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info = + container_of(info, struct fib6_entry_notifier_info, info); + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); + + if (info->family == AF_INET) + return prestera_fi_is_nh(fen_info->fi); + else + return prestera_fi6_is_nh(fen6_info->rt); +} + +/* must be called with rcu_read_lock() */ +static int prestera_util_kern_get_route(struct fib_result *res, u32 tb_id, + __be32 *addr) +{ + struct flowi4 fl4; + + /* TODO: walkthrough appropriate tables in kernel + * to know if the same prefix exists in several tables + */ + memset(&fl4, 0, sizeof(fl4)); + fl4.daddr = *addr; + return fib_lookup(&init_net, &fl4, res, 0 /* FIB_LOOKUP_NOREF */); +} + +static bool +__prestera_util_kern_n_is_reachable_v4(u32 tb_id, __be32 *addr, + struct net_device *dev) +{ + struct fib_nh *fib_nh; + struct fib_result res; + bool reachable; + + reachable = false; + + if (!prestera_util_kern_get_route(&res, tb_id, addr)) + if (prestera_fi_is_direct(res.fi)) { + fib_nh = fib_info_nh(res.fi, 0); + if (dev == fib_nh->fib_nh_dev) + reachable = true; + } + + return reachable; +} + +/* Check if neigh route is reachable */ +static bool +prestera_util_kern_n_is_reachable(u32 tb_id, + struct prestera_ip_addr *addr, + struct net_device *dev) +{ + if (addr->v == PRESTERA_IPV4) + return __prestera_util_kern_n_is_reachable_v4(tb_id, + &addr->u.ipv4, + dev); + else + return false; +} + +static void prestera_util_kern_set_neigh_offload(struct neighbour *n, + bool offloaded) +{ + if (offloaded) + n->flags |= NTF_OFFLOADED; + else + n->flags &= ~NTF_OFFLOADED; +} + +static void +prestera_util_kern_set_nh_offload(struct fib_nh_common *nhc, bool offloaded, bool trap) +{ + if (offloaded) + nhc->nhc_flags |= RTNH_F_OFFLOAD; + else + nhc->nhc_flags &= ~RTNH_F_OFFLOAD; + + if (trap) + nhc->nhc_flags |= RTNH_F_TRAP; + else + nhc->nhc_flags &= ~RTNH_F_TRAP; +} + +static struct fib_nh_common * +prestera_kern_fib_info_nhc(struct fib_notifier_info *info, int n) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + struct fib6_info *iter; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return &fib_info_nh(fen4_info->fi, n)->nh_common; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + if (!n) + return &fen6_info->rt->fib6_nh->nh_common; + + list_for_each_entry(iter, &fen6_info->rt->fib6_siblings, + fib6_siblings) { + if (!--n) + return &iter->fib6_nh->nh_common; + } + } + + /* if family is incorrect - than upper functions has BUG */ + /* if doesn't find requested index - there is alsi bug, because + * valid index must be produced by nhs, which checks list length + */ + WARN(1, "Invalid parameters passed to %s n=%d i=%p", + __func__, n, info); + return NULL; +} + +static int prestera_kern_fib_info_nhs(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fib_info_num_path(fen4_info->fi); + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + return fen6_info->rt->fib6_nsiblings + 1; + } + + return 0; +} + +static unsigned char +prestera_kern_fib_info_type(struct fib_notifier_info *info) +{ + struct fib6_entry_notifier_info *fen6_info; + struct fib_entry_notifier_info *fen4_info; + + if (info->family == AF_INET) { + fen4_info = container_of(info, struct fib_entry_notifier_info, + info); + return fen4_info->fi->fib_type; + } else if (info->family == AF_INET6) { + fen6_info = container_of(info, struct fib6_entry_notifier_info, + info); + /* TODO: ECMP in ipv6 is several routes. + * Every route has single nh. + */ + return fen6_info->rt->fib6_type; + } + + return RTN_UNSPEC; +} + +/* Decided, that uc_nh route with key==nh is obviously neighbour route */ +static bool +prestera_fib_node_util_is_neighbour(struct prestera_fib_node *fib_node) +{ + if (fib_node->info.type != PRESTERA_FIB_TYPE_UC_NH) + return false; + + if (fib_node->info.nh_grp->nh_neigh_head[1].neigh) + return false; + + if (!fib_node->info.nh_grp->nh_neigh_head[0].neigh) + return false; + + if (memcmp(&fib_node->info.nh_grp->nh_neigh_head[0].neigh->key.addr, + &fib_node->key.addr, sizeof(struct prestera_ip_addr))) + return false; + + return true; +} + +static int prestera_dev_if_type(const struct net_device *dev) +{ + struct macvlan_dev *vlan; + + if (is_vlan_dev(dev) && + netif_is_bridge_master(vlan_dev_real_dev(dev))) { + return PRESTERA_IF_VID_E; + } else if (netif_is_bridge_master(dev)) { + return PRESTERA_IF_VID_E; + } else if (netif_is_lag_master(dev)) { + return PRESTERA_IF_LAG_E; + } else if (netif_is_macvlan(dev)) { + vlan = netdev_priv(dev); + return prestera_dev_if_type(vlan->lowerdev); + } else { + return PRESTERA_IF_PORT_E; + } +} + +static int +prestera_neigh_iface_init(struct prestera_switch *sw, + struct prestera_iface *iface, + struct neighbour *n) +{ + struct prestera_port *port; + + iface->vlan_id = 0; /* TODO: vlan egress */ + iface->type = prestera_dev_if_type(n->dev); + if (iface->type != PRESTERA_IF_PORT_E) + return -EINVAL; + + if (!prestera_netdev_check(n->dev)) + return -EINVAL; + + port = netdev_priv(n->dev); + iface->dev_port.hw_dev_num = port->dev_id; + iface->dev_port.port_num = port->hw_id; + + return 0; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_find(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = + rhashtable_lookup_fast(&sw->router->kern_neigh_cache_ht, key, + __prestera_kern_neigh_cache_ht_params); + return IS_ERR(n_cache) ? NULL : n_cache; +} + +static void +__prestera_kern_neigh_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + dev_put(n_cache->key.dev); +} + +static void +__prestera_kern_neigh_cache_destroy(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + rhashtable_remove_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static struct prestera_kern_neigh_cache * +__prestera_kern_neigh_cache_create(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + int err; + + n_cache = kzalloc(sizeof(*n_cache), GFP_KERNEL); + if (!n_cache) + goto err_kzalloc; + + memcpy(&n_cache->key, key, sizeof(*key)); + dev_hold(n_cache->key.dev); + + INIT_LIST_HEAD(&n_cache->kern_fib_cache_list); + err = rhashtable_insert_fast(&sw->router->kern_neigh_cache_ht, + &n_cache->ht_node, + __prestera_kern_neigh_cache_ht_params); + if (err) + goto err_ht_insert; + + return n_cache; + +err_ht_insert: + dev_put(n_cache->key.dev); + kfree(n_cache); +err_kzalloc: + return NULL; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_get(struct prestera_switch *sw, + struct prestera_kern_neigh_cache_key *key) +{ + struct prestera_kern_neigh_cache *n_cache; + + n_cache = prestera_kern_neigh_cache_find(sw, key); + if (!n_cache) + n_cache = __prestera_kern_neigh_cache_create(sw, key); + + return n_cache; +} + +static struct prestera_kern_neigh_cache * +prestera_kern_neigh_cache_put(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache) +{ + if (!n_cache->in_kernel && + list_empty(&n_cache->kern_fib_cache_list)) { + __prestera_kern_neigh_cache_destroy(sw, n_cache); + return NULL; + } + + return n_cache; +} + static struct prestera_kern_fib_cache * prestera_kern_fib_cache_find(struct prestera_switch *sw, struct prestera_kern_fib_cache_key *key) @@ -72,25 +546,80 @@ prestera_kern_fib_cache_find(struct prestera_switch *sw, return fib_cache; } +static void +__prestera_kern_fib_cache_destruct(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fib_cache) +{ + struct prestera_kern_neigh_cache *n_cache; + int i; + + for (i = 0; i < PRESTERA_NHGR_SIZE_MAX; i++) { + n_cache = fib_cache->kern_neigh_cache_head[i].n_cache; + if (n_cache) { + list_del(&fib_cache->kern_neigh_cache_head[i].head); + prestera_kern_neigh_cache_put(sw, n_cache); + } + } + + fib_info_put(fib_cache->fen4_info.fi); +} + static void prestera_kern_fib_cache_destroy(struct prestera_switch *sw, struct prestera_kern_fib_cache *fib_cache) { - fib_info_put(fib_cache->fi); rhashtable_remove_fast(&sw->router->kern_fib_cache_ht, &fib_cache->ht_node, __prestera_kern_fib_cache_ht_params); + __prestera_kern_fib_cache_destruct(sw, fib_cache); kfree(fib_cache); } +static int +__prestera_kern_fib_cache_create_nhs(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fc) +{ + struct prestera_kern_neigh_cache_key nc_key; + struct prestera_kern_neigh_cache *n_cache; + struct fib_nh_common *nhc; + int i, nhs, err; + + if (!prestera_fib_info_is_nh(&fc->info)) + return 0; + + nhs = prestera_kern_fib_info_nhs(&fc->info); + if (nhs > PRESTERA_NHGR_SIZE_MAX) + return 0; + + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fc->fen4_info.info, i); + err = prestera_util_nhc2nc_key(sw, nhc, &nc_key); + if (err) + return 0; + + n_cache = prestera_kern_neigh_cache_get(sw, &nc_key); + if (!n_cache) + return 0; + + fc->kern_neigh_cache_head[i].this = fc; + fc->kern_neigh_cache_head[i].n_cache = n_cache; + list_add(&fc->kern_neigh_cache_head[i].head, + &n_cache->kern_fib_cache_list); + } + + return 0; +} + /* Operations on fi (offload, etc) must be wrapped in utils. * This function just create storage. */ static struct prestera_kern_fib_cache * prestera_kern_fib_cache_create(struct prestera_switch *sw, struct prestera_kern_fib_cache_key *key, - struct fib_info *fi, dscp_t dscp, u8 type) + struct fib_notifier_info *info) { + struct fib_entry_notifier_info *fen_info = + container_of(info, struct fib_entry_notifier_info, info); struct prestera_kern_fib_cache *fib_cache; int err; @@ -99,10 +628,8 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw, goto err_kzalloc; memcpy(&fib_cache->key, key, sizeof(*key)); - fib_info_hold(fi); - fib_cache->fi = fi; - fib_cache->kern_dscp = dscp; - fib_cache->kern_type = type; + fib_info_hold(fen_info->fi); + memcpy(&fib_cache->fen4_info, fen_info, sizeof(*fen_info)); err = rhashtable_insert_fast(&sw->router->kern_fib_cache_ht, &fib_cache->ht_node, @@ -110,15 +637,61 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw, if (err) goto err_ht_insert; + /* Handle nexthops */ + err = __prestera_kern_fib_cache_create_nhs(sw, fib_cache); + if (err) + goto out; /* Not critical */ + +out: return fib_cache; err_ht_insert: - fib_info_put(fi); + fib_info_put(fen_info->fi); kfree(fib_cache); err_kzalloc: return NULL; } +static void +__prestera_k_arb_fib_nh_offload_set(struct prestera_switch *sw, + struct prestera_kern_fib_cache *fibc, + struct prestera_kern_neigh_cache *nc, + bool offloaded, bool trap) +{ + struct fib_nh_common *nhc; + int i, nhs; + + nhs = prestera_kern_fib_info_nhs(&fibc->info); + for (i = 0; i < nhs; i++) { + nhc = prestera_kern_fib_info_nhc(&fibc->info, i); + if (!nc) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + continue; + } + + if (prestera_util_nhc_eq_n_cache_key(sw, nhc, &nc->key)) { + prestera_util_kern_set_nh_offload(nhc, offloaded, trap); + break; + } + } +} + +static void +__prestera_k_arb_n_offload_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc, + bool offloaded) +{ + struct neighbour *n; + + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + return; + + prestera_util_kern_set_neigh_offload(n, offloaded); + neigh_release(n); +} + static void __prestera_k_arb_fib_lpm_offload_set(struct prestera_switch *sw, struct prestera_kern_fib_cache *fc, @@ -126,32 +699,208 @@ __prestera_k_arb_fib_lpm_offload_set(struct prestera_switch *sw, { struct fib_rt_info fri; - if (fc->key.addr.v != PRESTERA_IPV4) + switch (fc->key.addr.v) { + case PRESTERA_IPV4: + fri.fi = fc->fen4_info.fi; + fri.tb_id = fc->key.kern_tb_id; + fri.dst = fc->key.addr.u.ipv4; + fri.dst_len = fc->key.prefix_len; + fri.dscp = fc->fen4_info.dscp; + fri.type = fc->fen4_info.type; + /* flags begin */ + fri.offload = offload; + fri.trap = trap; + fri.offload_failed = fail; + /* flags end */ + fib_alias_hw_flags_set(&init_net, &fri); + return; + case PRESTERA_IPV6: + /* TODO */ return; + } +} + +static void +__prestera_k_arb_n_lpm_set(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *n_cache, + bool enabled) +{ + struct prestera_nexthop_group_key nh_grp_key; + struct prestera_kern_fib_cache_key fc_key; + struct prestera_kern_fib_cache *fib_cache; + struct prestera_fib_node *fib_node; + struct prestera_fib_key fib_key; + + /* Exception for fc with prefix 32: LPM entry is already used by fib */ + memset(&fc_key, 0, sizeof(fc_key)); + fc_key.addr = n_cache->key.addr; + fc_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + /* But better to use tb_id of route, which pointed to this neighbour. */ + /* We take it from rif, because rif inconsistent. + * Must be separated in_rif and out_rif. + * Also note: for each fib pointed to this neigh should be separated + * neigh lpm entry (for each ingress vr) + */ + fc_key.kern_tb_id = l3mdev_fib_table(n_cache->key.dev); + fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); + memset(&fib_key, 0, sizeof(fib_key)); + fib_key.addr = n_cache->key.addr; + fib_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v); + fib_key.tb_id = prestera_fix_tb_id(fc_key.kern_tb_id); + fib_node = prestera_fib_node_find(sw, &fib_key); + if (!fib_cache || !fib_cache->reachable) { + if (!enabled && fib_node) { + if (prestera_fib_node_util_is_neighbour(fib_node)) + prestera_fib_node_destroy(sw, fib_node); + return; + } + } + + if (enabled && !fib_node) { + memset(&nh_grp_key, 0, sizeof(nh_grp_key)); + prestera_util_nc_key2nh_key(&n_cache->key, + &nh_grp_key.neigh[0]); + fib_node = prestera_fib_node_create(sw, &fib_key, + PRESTERA_FIB_TYPE_UC_NH, + &nh_grp_key); + if (!fib_node) + pr_err("%s failed ip=%pI4n", "prestera_fib_node_create", + &fib_key.addr.u.ipv4); + return; + } +} + +static void +__prestera_k_arb_nc_kern_fib_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + if (prestera_util_kern_n_is_reachable(l3mdev_fib_table(nc->key.dev), + &nc->key.addr, nc->key.dev)) + nc->reachable = true; + else + nc->reachable = false; +} + +/* Kernel neighbour -> neigh_cache info */ +static void +__prestera_k_arb_nc_kern_n_fetch(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct neighbour *n; + int err; + + memset(&nc->nh_neigh_info, 0, sizeof(nc->nh_neigh_info)); + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, nc->key.dev); + if (!n) + goto out; + + read_lock_bh(&n->lock); + if (n->nud_state & NUD_VALID && !n->dead) { + err = prestera_neigh_iface_init(sw, &nc->nh_neigh_info.iface, + n); + if (err) + goto n_read_out; - fri.fi = fc->fi; - fri.tb_id = fc->key.kern_tb_id; - fri.dst = fc->key.addr.u.ipv4; - fri.dst_len = fc->key.prefix_len; - fri.dscp = fc->kern_dscp; - fri.type = fc->kern_type; - /* flags begin */ - fri.offload = offload; - fri.trap = trap; - fri.offload_failed = fail; - /* flags end */ - fib_alias_hw_flags_set(&init_net, &fri); + memcpy(&nc->nh_neigh_info.ha[0], &n->ha[0], ETH_ALEN); + nc->nh_neigh_info.connected = true; + } +n_read_out: + read_unlock_bh(&n->lock); +out: + nc->in_kernel = nc->nh_neigh_info.connected; + if (n) + neigh_release(n); +} + +/* neigh_cache info -> lpm update */ +static void +__prestera_k_arb_nc_apply(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_kern_neigh_cache_head *nhead; + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + int err; + + __prestera_k_arb_n_lpm_set(sw, nc, nc->reachable && nc->in_kernel); + __prestera_k_arb_n_offload_set(sw, nc, nc->reachable && nc->in_kernel); + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) + goto out; + + /* Do hw update only if something changed to prevent nh flap */ + if (memcmp(&nc->nh_neigh_info, &nh_neigh->info, + sizeof(nh_neigh->info))) { + memcpy(&nh_neigh->info, &nc->nh_neigh_info, + sizeof(nh_neigh->info)); + err = prestera_nh_neigh_set(sw, nh_neigh); + if (err) { + pr_err("%s failed with err=%d ip=%pI4n mac=%pM", + "prestera_nh_neigh_set", err, + &nh_neigh->key.addr.u.ipv4, + &nh_neigh->info.ha[0]); + goto out; + } + } + +out: + list_for_each_entry(nhead, &nc->kern_fib_cache_list, head) { + __prestera_k_arb_fib_nh_offload_set(sw, nhead->this, nc, + nc->in_kernel, + !nc->in_kernel); + } } static int __prestera_pr_k_arb_fc_lpm_info_calc(struct prestera_switch *sw, struct prestera_kern_fib_cache *fc) { + struct fib_nh_common *nhc; + int nh_cnt; + memset(&fc->lpm_info, 0, sizeof(fc->lpm_info)); - switch (fc->fi->fib_type) { + switch (prestera_kern_fib_info_type(&fc->info)) { case RTN_UNICAST: - fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_TRAP; + if (prestera_fib_info_is_direct(&fc->info) && + fc->key.prefix_len == + PRESTERA_IP_ADDR_PLEN(fc->key.addr.v)) { + /* This is special case. + * When prefix is 32. Than we will have conflict in lpm + * for direct route - once TRAP added, there is no + * place for neighbour entry. So represent direct route + * with prefix 32, as NH. So neighbour will be resolved + * as nexthop of this route. + */ + nhc = prestera_kern_fib_info_nhc(&fc->info, 0); + fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_UC_NH; + fc->lpm_info.nh_grp_key.neigh[0].addr = + fc->key.addr; + fc->lpm_info.nh_grp_key.neigh[0].rif = + nhc->nhc_dev; + + break; + } + + /* We can also get nh_grp_key from fi. This will be correct to + * because cache not always represent, what actually written to + * lpm. But we use nh cache, as well for now (for this case). + */ + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!fc->kern_neigh_cache_head[nh_cnt].n_cache) + break; + + fc->lpm_info.nh_grp_key.neigh[nh_cnt].addr = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.addr; + fc->lpm_info.nh_grp_key.neigh[nh_cnt].rif = + fc->kern_neigh_cache_head[nh_cnt].n_cache->key.dev; + } + + fc->lpm_info.fib_type = nh_cnt ? + PRESTERA_FIB_TYPE_UC_NH : + PRESTERA_FIB_TYPE_TRAP; break; /* Unsupported. Leave it for kernel: */ case RTN_BROADCAST: @@ -191,7 +940,8 @@ static int __prestera_k_arb_f_lpm_set(struct prestera_switch *sw, return 0; fib_node = prestera_fib_node_create(sw, &fc->lpm_info.fib_key, - fc->lpm_info.fib_type); + fc->lpm_info.fib_type, + &fc->lpm_info.nh_grp_key); if (!fib_node) { dev_err(sw->dev->dev, "fib_node=NULL %pI4n/%d kern_tb_id = %d", @@ -220,6 +970,10 @@ static int __prestera_k_arb_fc_apply(struct prestera_switch *sw, } switch (fc->lpm_info.fib_type) { + case PRESTERA_FIB_TYPE_UC_NH: + __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, + fc->reachable, false); + break; case PRESTERA_FIB_TYPE_TRAP: __prestera_k_arb_fib_lpm_offload_set(sw, fc, false, false, fc->reachable); @@ -271,17 +1025,140 @@ __prestera_k_arb_util_fib_overlapped(struct prestera_switch *sw, return rfc; } +static void __prestera_k_arb_hw_state_upd(struct prestera_switch *sw, + struct prestera_kern_neigh_cache *nc) +{ + struct prestera_nh_neigh_key nh_key; + struct prestera_nh_neigh *nh_neigh; + struct neighbour *n; + bool hw_active; + + prestera_util_nc_key2nh_key(&nc->key, &nh_key); + nh_neigh = prestera_nh_neigh_find(sw, &nh_key); + if (!nh_neigh) { + pr_err("Cannot find nh_neigh for cached %pI4n", + &nc->key.addr.u.ipv4); + return; + } + + hw_active = prestera_nh_neigh_util_hw_state(sw, nh_neigh); + +#ifdef PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH + if (!hw_active && nc->in_kernel) + goto out; +#else /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + if (!hw_active) + goto out; +#endif /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */ + + if (nc->key.addr.v == PRESTERA_IPV4) { + n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + if (!n) + n = neigh_create(&arp_tbl, &nc->key.addr.u.ipv4, + nc->key.dev); + } else { + n = NULL; + } + + if (!IS_ERR(n) && n) { + neigh_event_send(n, NULL); + neigh_release(n); + } else { + pr_err("Cannot create neighbour %pI4n", &nc->key.addr.u.ipv4); + } + +out: + return; +} + +/* Propagate hw state to kernel */ +static void prestera_k_arb_hw_evt(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_hw_state_upd(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + +/* Propagate kernel event to hw */ +static void prestera_k_arb_n_evt(struct prestera_switch *sw, + struct neighbour *n) +{ + struct prestera_kern_neigh_cache_key n_key; + struct prestera_kern_neigh_cache *n_cache; + int err; + + err = prestera_util_neigh2nc_key(sw, n, &n_key); + if (err) + return; + + n_cache = prestera_kern_neigh_cache_find(sw, &n_key); + if (!n_cache) { + n_cache = prestera_kern_neigh_cache_get(sw, &n_key); + if (!n_cache) + return; + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + } + + __prestera_k_arb_nc_kern_n_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + + prestera_kern_neigh_cache_put(sw, n_cache); +} + +static void __prestera_k_arb_fib_evt2nc(struct prestera_switch *sw) +{ + struct prestera_kern_neigh_cache *n_cache; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter); + rhashtable_walk_start(&iter); + while (1) { + n_cache = rhashtable_walk_next(&iter); + + if (!n_cache) + break; + + if (IS_ERR(n_cache)) + continue; + + rhashtable_walk_stop(&iter); + __prestera_k_arb_nc_kern_fib_fetch(sw, n_cache); + __prestera_k_arb_nc_apply(sw, n_cache); + rhashtable_walk_start(&iter); + } + rhashtable_walk_stop(&iter); + rhashtable_walk_exit(&iter); +} + static int prestera_k_arb_fib_evt(struct prestera_switch *sw, bool replace, /* replace or del */ - struct fib_entry_notifier_info *fen_info) + struct fib_notifier_info *info) { struct prestera_kern_fib_cache *tfib_cache, *bfib_cache; /* top/btm */ struct prestera_kern_fib_cache_key fc_key; struct prestera_kern_fib_cache *fib_cache; int err; - prestera_util_fen_info2fib_cache_key(fen_info, &fc_key); + prestera_util_fen_info2fib_cache_key(info, &fc_key); fib_cache = prestera_kern_fib_cache_find(sw, &fc_key); if (fib_cache) { fib_cache->reachable = false; @@ -304,10 +1181,7 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw, } if (replace) { - fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, - fen_info->fi, - fen_info->dscp, - fen_info->type); + fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, info); if (!fib_cache) { dev_err(sw->dev->dev, "fib_cache == NULL"); return -ENOENT; @@ -331,9 +1205,65 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw, dev_err(sw->dev->dev, "Applying fib_cache failed"); } + /* Update all neighs to resolve overlapped and apply related */ + __prestera_k_arb_fib_evt2nc(sw); + return 0; } +static void __prestera_k_arb_abort_neigh_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_neigh_cache *n_cache = ptr; + struct prestera_switch *sw = arg; + + if (!list_empty(&n_cache->kern_fib_cache_list)) { + WARN_ON(1); /* BUG */ + return; + } + __prestera_k_arb_n_offload_set(sw, n_cache, false); + n_cache->in_kernel = false; + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_neigh_cache_destruct(sw, n_cache); + kfree(n_cache); +} + +static void __prestera_k_arb_abort_fib_ht_cb(void *ptr, void *arg) +{ + struct prestera_kern_fib_cache *fib_cache = ptr; + struct prestera_switch *sw = arg; + + __prestera_k_arb_fib_lpm_offload_set(sw, fib_cache, + false, false, + false); + __prestera_k_arb_fib_nh_offload_set(sw, fib_cache, NULL, + false, false); + /* No need to destroy lpm. + * It will be aborted by destroy_ht + */ + __prestera_kern_fib_cache_destruct(sw, fib_cache); + kfree(fib_cache); +} + +static void prestera_k_arb_abort(struct prestera_switch *sw) +{ + /* Function to remove all arbiter entries and related hw objects. */ + /* Sequence: + * 1) Clear arbiter tables, but don't touch hw + * 2) Clear hw + * We use such approach, because arbiter object is not directly mapped + * to hw. So deletion of one arbiter object may even lead to creation of + * hw object (e.g. in case of overlapped routes). + */ + rhashtable_free_and_destroy(&sw->router->kern_fib_cache_ht, + __prestera_k_arb_abort_fib_ht_cb, + sw); + rhashtable_free_and_destroy(&sw->router->kern_neigh_cache_ht, + __prestera_k_arb_abort_neigh_ht_cb, + sw); +} + static int __prestera_inetaddr_port_event(struct net_device *port_dev, unsigned long event, struct netlink_ext_ack *extack) @@ -469,13 +1399,15 @@ static void __prestera_router_fib_event_work(struct work_struct *work) switch (fib_work->event) { case FIB_EVENT_ENTRY_REPLACE: - err = prestera_k_arb_fib_evt(sw, true, &fib_work->fen_info); + err = prestera_k_arb_fib_evt(sw, true, + &fib_work->fen_info.info); if (err) goto err_out; break; case FIB_EVENT_ENTRY_DEL: - err = prestera_k_arb_fib_evt(sw, false, &fib_work->fen_info); + err = prestera_k_arb_fib_evt(sw, false, + &fib_work->fen_info.info); if (err) goto err_out; @@ -534,10 +1466,89 @@ static int __prestera_router_fib_event(struct notifier_block *nb, return NOTIFY_DONE; } +struct prestera_netevent_work { + struct work_struct work; + struct prestera_switch *sw; + struct neighbour *n; +}; + +static void prestera_router_neigh_event_work(struct work_struct *work) +{ + struct prestera_netevent_work *net_work = + container_of(work, struct prestera_netevent_work, work); + struct prestera_switch *sw = net_work->sw; + struct neighbour *n = net_work->n; + + /* neigh - its not hw related object. It stored only in kernel. So... */ + rtnl_lock(); + + prestera_k_arb_n_evt(sw, n); + + neigh_release(n); + rtnl_unlock(); + kfree(net_work); +} + +static int prestera_router_netevent_event(struct notifier_block *nb, + unsigned long event, void *ptr) +{ + struct prestera_netevent_work *net_work; + struct prestera_router *router; + struct neighbour *n = ptr; + + router = container_of(nb, struct prestera_router, netevent_nb); + + switch (event) { + case NETEVENT_NEIGH_UPDATE: + if (n->tbl->family != AF_INET) + return NOTIFY_DONE; + + net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); + if (WARN_ON(!net_work)) + return NOTIFY_BAD; + + neigh_clone(n); + net_work->n = n; + net_work->sw = router->sw; + INIT_WORK(&net_work->work, prestera_router_neigh_event_work); + prestera_queue_work(&net_work->work); + } + + return NOTIFY_DONE; +} + +static void prestera_router_update_neighs_work(struct work_struct *work) +{ + struct prestera_router *router; + + router = container_of(work, struct prestera_router, + neighs_update.dw.work); + rtnl_lock(); + + prestera_k_arb_hw_evt(router->sw); + + rtnl_unlock(); + prestera_queue_delayed_work(&router->neighs_update.dw, + msecs_to_jiffies(PRESTERA_NH_PROBE_INTERVAL)); +} + +static int prestera_neigh_work_init(struct prestera_switch *sw) +{ + INIT_DELAYED_WORK(&sw->router->neighs_update.dw, + prestera_router_update_neighs_work); + prestera_queue_delayed_work(&sw->router->neighs_update.dw, 0); + return 0; +} + +static void prestera_neigh_work_fini(struct prestera_switch *sw) +{ + cancel_delayed_work_sync(&sw->router->neighs_update.dw); +} + int prestera_router_init(struct prestera_switch *sw) { struct prestera_router *router; - int err; + int err, nhgrp_cache_bytes; router = kzalloc(sizeof(*sw->router), GFP_KERNEL); if (!router) @@ -555,6 +1566,22 @@ int prestera_router_init(struct prestera_switch *sw) if (err) goto err_kern_fib_cache_ht_init; + err = rhashtable_init(&router->kern_neigh_cache_ht, + &__prestera_kern_neigh_cache_ht_params); + if (err) + goto err_kern_neigh_cache_ht_init; + + nhgrp_cache_bytes = sw->size_tbl_router_nexthop / 8 + 1; + router->nhgrp_hw_state_cache = kzalloc(nhgrp_cache_bytes, GFP_KERNEL); + if (!router->nhgrp_hw_state_cache) { + err = -ENOMEM; + goto err_nh_state_cache_alloc; + } + + err = prestera_neigh_work_init(sw); + if (err) + goto err_neigh_work_init; + router->inetaddr_valid_nb.notifier_call = __prestera_inetaddr_valid_cb; err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); if (err) @@ -565,6 +1592,11 @@ int prestera_router_init(struct prestera_switch *sw) if (err) goto err_register_inetaddr_notifier; + router->netevent_nb.notifier_call = prestera_router_netevent_event; + err = register_netevent_notifier(&router->netevent_nb); + if (err) + goto err_register_netevent_notifier; + router->fib_nb.notifier_call = __prestera_router_fib_event; err = register_fib_notifier(&init_net, &router->fib_nb, /* TODO: flush fib entries */ NULL, NULL); @@ -574,10 +1606,18 @@ int prestera_router_init(struct prestera_switch *sw) return 0; err_register_fib_notifier: + unregister_netevent_notifier(&router->netevent_nb); +err_register_netevent_notifier: unregister_inetaddr_notifier(&router->inetaddr_nb); err_register_inetaddr_notifier: unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); err_register_inetaddr_validator_notifier: + prestera_neigh_work_fini(sw); +err_neigh_work_init: + kfree(router->nhgrp_hw_state_cache); +err_nh_state_cache_alloc: + rhashtable_destroy(&router->kern_neigh_cache_ht); +err_kern_neigh_cache_ht_init: rhashtable_destroy(&router->kern_fib_cache_ht); err_kern_fib_cache_ht_init: prestera_router_hw_fini(sw); @@ -589,8 +1629,15 @@ err_router_lib_init: void prestera_router_fini(struct prestera_switch *sw) { unregister_fib_notifier(&init_net, &sw->router->fib_nb); + unregister_netevent_notifier(&sw->router->netevent_nb); unregister_inetaddr_notifier(&sw->router->inetaddr_nb); unregister_inetaddr_validator_notifier(&sw->router->inetaddr_valid_nb); + prestera_neigh_work_fini(sw); + prestera_queue_drain(); + + prestera_k_arb_abort(sw); + + kfree(sw->router->nhgrp_hw_state_cache); rhashtable_destroy(&sw->router->kern_fib_cache_ht); prestera_router_hw_fini(sw); kfree(sw->router); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c index 5b0cf3be9a9edf7df4b6f512799518e2353efc68..aa080dc57ff0050a216c096eb9b585fdf56aaf8f 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c @@ -8,10 +8,16 @@ #include "prestera_router_hw.h" #include "prestera_acl.h" -/* +--+ - * +------->|vr|<-+ - * | +--+ | - * | | +/* Nexthop is pointed + * to port (not rif) + * +-------+ + * +>|nexthop| + * | +-------+ + * | + * +--+ +-----++ + * +------->|vr|<-+ +>|nh_grp| + * | +--+ | | +------+ + * | | | * +-+-------+ +--+---+-+ * |rif_entry| |fib_node| * +---------+ +--------+ @@ -23,6 +29,8 @@ #define PRESTERA_NHGR_UNUSED (0) #define PRESTERA_NHGR_DROP (0xFFFFFFFF) +/* Need to merge it with router_manager */ +#define PRESTERA_NH_ACTIVE_JIFFER_FILTER 3000 /* ms */ static const struct rhashtable_params __prestera_fib_ht_params = { .key_offset = offsetof(struct prestera_fib_node, key), @@ -31,10 +39,45 @@ static const struct rhashtable_params __prestera_fib_ht_params = { .automatic_shrinking = true, }; +static const struct rhashtable_params __prestera_nh_neigh_ht_params = { + .key_offset = offsetof(struct prestera_nh_neigh, key), + .key_len = sizeof(struct prestera_nh_neigh_key), + .head_offset = offsetof(struct prestera_nh_neigh, ht_node), +}; + +static const struct rhashtable_params __prestera_nexthop_group_ht_params = { + .key_offset = offsetof(struct prestera_nexthop_group, key), + .key_len = sizeof(struct prestera_nexthop_group_key), + .head_offset = offsetof(struct prestera_nexthop_group, ht_node), +}; + +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp); +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg); + +/* TODO: move to router.h as macros */ +static bool prestera_nh_neigh_key_is_valid(struct prestera_nh_neigh_key *key) +{ + return memchr_inv(key, 0, sizeof(*key)) ? true : false; +} + int prestera_router_hw_init(struct prestera_switch *sw) { int err; + err = rhashtable_init(&sw->router->nh_neigh_ht, + &__prestera_nh_neigh_ht_params); + if (err) + goto err_nh_neigh_ht_init; + + err = rhashtable_init(&sw->router->nexthop_group_ht, + &__prestera_nexthop_group_ht_params); + if (err) + goto err_nexthop_grp_ht_init; + err = rhashtable_init(&sw->router->fib_ht, &__prestera_fib_ht_params); if (err) @@ -43,15 +86,25 @@ int prestera_router_hw_init(struct prestera_switch *sw) INIT_LIST_HEAD(&sw->router->vr_list); INIT_LIST_HEAD(&sw->router->rif_entry_list); + return 0; + err_fib_ht_init: + rhashtable_destroy(&sw->router->nexthop_group_ht); +err_nexthop_grp_ht_init: + rhashtable_destroy(&sw->router->nh_neigh_ht); +err_nh_neigh_ht_init: return 0; } void prestera_router_hw_fini(struct prestera_switch *sw) { + rhashtable_free_and_destroy(&sw->router->fib_ht, + prestera_fib_node_destroy_ht_cb, sw); WARN_ON(!list_empty(&sw->router->vr_list)); WARN_ON(!list_empty(&sw->router->rif_entry_list)); rhashtable_destroy(&sw->router->fib_ht); + rhashtable_destroy(&sw->router->nexthop_group_ht); + rhashtable_destroy(&sw->router->nh_neigh_ht); } static struct prestera_vr *__prestera_vr_find(struct prestera_switch *sw, @@ -232,6 +285,286 @@ err_kzalloc: return NULL; } +static void __prestera_nh_neigh_destroy(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + rhashtable_remove_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + kfree(neigh); +} + +static struct prestera_nh_neigh * +__prestera_nh_neigh_create(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + int err; + + neigh = kzalloc(sizeof(*neigh), GFP_KERNEL); + if (!neigh) + goto err_kzalloc; + + memcpy(&neigh->key, key, sizeof(*key)); + neigh->info.connected = false; + INIT_LIST_HEAD(&neigh->nexthop_group_list); + err = rhashtable_insert_fast(&sw->router->nh_neigh_ht, + &neigh->ht_node, + __prestera_nh_neigh_ht_params); + if (err) + goto err_rhashtable_insert; + + return neigh; + +err_rhashtable_insert: + kfree(neigh); +err_kzalloc: + return NULL; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *nh_neigh; + + nh_neigh = rhashtable_lookup_fast(&sw->router->nh_neigh_ht, + key, __prestera_nh_neigh_ht_params); + return IS_ERR(nh_neigh) ? NULL : nh_neigh; +} + +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key) +{ + struct prestera_nh_neigh *neigh; + + neigh = prestera_nh_neigh_find(sw, key); + if (!neigh) + return __prestera_nh_neigh_create(sw, key); + + return neigh; +} + +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + if (list_empty(&neigh->nexthop_group_list)) + __prestera_nh_neigh_destroy(sw, neigh); +} + +/* Updates new prestera_neigh_info */ +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh) +{ + struct prestera_nh_neigh_head *nh_head; + struct prestera_nexthop_group *nh_grp; + int err; + + list_for_each_entry(nh_head, &neigh->nexthop_group_list, head) { + nh_grp = nh_head->this; + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + return err; + } + + return 0; +} + +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh) +{ + bool state; + struct prestera_nh_neigh_head *nh_head, *tmp; + + state = false; + list_for_each_entry_safe(nh_head, tmp, + &nh_neigh->nexthop_group_list, head) { + state = prestera_nexthop_group_util_hw_state(sw, nh_head->this); + if (state) + goto out; + } + +out: + return state; +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_create(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + struct prestera_nh_neigh *nh_neigh; + int nh_cnt, err, gid; + + nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); + if (!nh_grp) + goto err_kzalloc; + + memcpy(&nh_grp->key, key, sizeof(*key)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + if (!prestera_nh_neigh_key_is_valid(&nh_grp->key.neigh[nh_cnt])) + break; + + nh_neigh = prestera_nh_neigh_get(sw, + &nh_grp->key.neigh[nh_cnt]); + if (!nh_neigh) + goto err_nh_neigh_get; + + nh_grp->nh_neigh_head[nh_cnt].neigh = nh_neigh; + nh_grp->nh_neigh_head[nh_cnt].this = nh_grp; + list_add(&nh_grp->nh_neigh_head[nh_cnt].head, + &nh_neigh->nexthop_group_list); + } + + err = prestera_hw_nh_group_create(sw, nh_cnt, &nh_grp->grp_id); + if (err) + goto err_nh_group_create; + + err = prestera_nexthop_group_set(sw, nh_grp); + if (err) + goto err_nexthop_group_set; + + err = rhashtable_insert_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + if (err) + goto err_ht_insert; + + /* reset cache for created group */ + gid = nh_grp->grp_id; + sw->router->nhgrp_hw_state_cache[gid / 8] &= ~BIT(gid % 8); + + return nh_grp; + +err_ht_insert: +err_nexthop_group_set: + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); +err_nh_group_create: +err_nh_neigh_get: + for (nh_cnt--; nh_cnt >= 0; nh_cnt--) { + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_grp->nh_neigh_head[nh_cnt].neigh); + } + + kfree(nh_grp); +err_kzalloc: + return NULL; +} + +static void +__prestera_nexthop_group_destroy(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_nh_neigh *nh_neigh; + int nh_cnt; + + rhashtable_remove_fast(&sw->router->nexthop_group_ht, + &nh_grp->ht_node, + __prestera_nexthop_group_ht_params); + + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + nh_neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!nh_neigh) + break; + + list_del(&nh_grp->nh_neigh_head[nh_cnt].head); + prestera_nh_neigh_put(sw, nh_neigh); + } + + prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id); + kfree(nh_grp); +} + +static struct prestera_nexthop_group * +__prestera_nexthop_group_find(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = rhashtable_lookup_fast(&sw->router->nexthop_group_ht, + key, __prestera_nexthop_group_ht_params); + return IS_ERR(nh_grp) ? NULL : nh_grp; +} + +static struct prestera_nexthop_group * +prestera_nexthop_group_get(struct prestera_switch *sw, + struct prestera_nexthop_group_key *key) +{ + struct prestera_nexthop_group *nh_grp; + + nh_grp = __prestera_nexthop_group_find(sw, key); + if (nh_grp) { + refcount_inc(&nh_grp->refcount); + } else { + nh_grp = __prestera_nexthop_group_create(sw, key); + if (!nh_grp) + return ERR_PTR(-ENOMEM); + + refcount_set(&nh_grp->refcount, 1); + } + + return nh_grp; +} + +static void prestera_nexthop_group_put(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + if (refcount_dec_and_test(&nh_grp->refcount)) + __prestera_nexthop_group_destroy(sw, nh_grp); +} + +/* Updates with new nh_neigh's info */ +static int prestera_nexthop_group_set(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + struct prestera_neigh_info info[PRESTERA_NHGR_SIZE_MAX]; + struct prestera_nh_neigh *neigh; + int nh_cnt; + + memset(&info[0], 0, sizeof(info)); + for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) { + neigh = nh_grp->nh_neigh_head[nh_cnt].neigh; + if (!neigh) + break; + + memcpy(&info[nh_cnt], &neigh->info, sizeof(neigh->info)); + } + + return prestera_hw_nh_entries_set(sw, nh_cnt, &info[0], nh_grp->grp_id); +} + +static bool +prestera_nexthop_group_util_hw_state(struct prestera_switch *sw, + struct prestera_nexthop_group *nh_grp) +{ + int err; + u32 buf_size = sw->size_tbl_router_nexthop / 8 + 1; + u32 gid = nh_grp->grp_id; + u8 *cache = sw->router->nhgrp_hw_state_cache; + + /* Antijitter + * Prevent situation, when we read state of nh_grp twice in short time, + * and state bit is still cleared on second call. So just stuck active + * state for PRESTERA_NH_ACTIVE_JIFFER_FILTER, after last occurred. + */ + if (!time_before(jiffies, sw->router->nhgrp_hw_cache_kick + + msecs_to_jiffies(PRESTERA_NH_ACTIVE_JIFFER_FILTER))) { + err = prestera_hw_nhgrp_blk_get(sw, cache, buf_size); + if (err) { + pr_err("Failed to get hw state nh_grp's"); + return false; + } + + sw->router->nhgrp_hw_cache_kick = jiffies; + } + + if (cache[gid / 8] & BIT(gid % 8)) + return true; + + return false; +} + struct prestera_fib_node * prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key) { @@ -251,6 +584,9 @@ static void __prestera_fib_node_destruct(struct prestera_switch *sw, prestera_hw_lpm_del(sw, vr->hw_vr_id, fib_node->key.addr.u.ipv4, fib_node->key.prefix_len); switch (fib_node->info.type) { + case PRESTERA_FIB_TYPE_UC_NH: + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); + break; case PRESTERA_FIB_TYPE_TRAP: break; case PRESTERA_FIB_TYPE_DROP: @@ -272,10 +608,20 @@ void prestera_fib_node_destroy(struct prestera_switch *sw, kfree(fib_node); } +static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg) +{ + struct prestera_fib_node *node = ptr; + struct prestera_switch *sw = arg; + + __prestera_fib_node_destruct(sw, node); + kfree(node); +} + struct prestera_fib_node * prestera_fib_node_create(struct prestera_switch *sw, struct prestera_fib_key *key, - enum prestera_fib_type fib_type) + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key) { struct prestera_fib_node *fib_node; u32 grp_id; @@ -302,6 +648,14 @@ prestera_fib_node_create(struct prestera_switch *sw, case PRESTERA_FIB_TYPE_DROP: grp_id = PRESTERA_NHGR_DROP; break; + case PRESTERA_FIB_TYPE_UC_NH: + fib_node->info.nh_grp = prestera_nexthop_group_get(sw, + nh_grp_key); + if (IS_ERR(fib_node->info.nh_grp)) + goto err_nh_grp_get; + + grp_id = fib_node->info.nh_grp->grp_id; + break; default: pr_err("Unsupported fib_type %d", fib_type); goto err_nh_grp_get; @@ -323,6 +677,8 @@ err_ht_insert: prestera_hw_lpm_del(sw, vr->hw_vr_id, key->addr.u.ipv4, key->prefix_len); err_lpm_add: + if (fib_type == PRESTERA_FIB_TYPE_UC_NH) + prestera_nexthop_group_put(sw, fib_node->info.nh_grp); err_nh_grp_get: prestera_vr_put(sw, vr); err_vr_get: diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h index 67dbb49c8bd42887a05aa5d4ef2ca293ffb9f38e..9ca97919c863893ed5f5cd82fd8960a4a7246405 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h @@ -31,6 +31,63 @@ struct prestera_ip_addr { PRESTERA_IPV4 = 0, PRESTERA_IPV6 } v; +#define PRESTERA_IP_ADDR_PLEN(V) ((V) == PRESTERA_IPV4 ? 32 : \ + /* (V) == PRESTERA_IPV6 ? */ 128 /* : 0 */) +}; + +struct prestera_nh_neigh_key { + struct prestera_ip_addr addr; + /* Seems like rif is obsolete, because there is iface in info ? + * Key can contain functional fields, or fields, which is used to + * filter duplicate objects on logical level (before you pass it to + * HW)... also key can be used to cover hardware restrictions. + * In our case rif - is logical interface (even can be VLAN), which + * is used in combination with IP address (which is also not related to + * hardware nexthop) to provide logical compression of created nexthops. + * You even can imagine, that rif+IPaddr is just cookie. + */ + /* struct prestera_rif *rif; */ + /* Use just as cookie, to divide ARP domains (in order with addr) */ + void *rif; +}; + +/* Used for hw call */ +struct prestera_neigh_info { + struct prestera_iface iface; + unsigned char ha[ETH_ALEN]; + u8 connected; /* bool. indicate, if mac/oif valid */ + u8 __pad[1]; +}; + +/* Used to notify nh about neigh change */ +struct prestera_nh_neigh { + struct prestera_nh_neigh_key key; + struct prestera_neigh_info info; + struct rhash_head ht_node; /* node of prestera_vr */ + struct list_head nexthop_group_list; +}; + +#define PRESTERA_NHGR_SIZE_MAX 4 + +struct prestera_nexthop_group { + struct prestera_nexthop_group_key { + struct prestera_nh_neigh_key neigh[PRESTERA_NHGR_SIZE_MAX]; + } key; + /* Store intermediate object here. + * This prevent overhead kzalloc call. + */ + /* nh_neigh is used only to notify nexthop_group */ + struct prestera_nh_neigh_head { + struct prestera_nexthop_group *this; + struct list_head head; + /* ptr to neigh is not necessary. + * It used to prevent lookup of nh_neigh by key (n) on destroy + */ + struct prestera_nh_neigh *neigh; + } nh_neigh_head[PRESTERA_NHGR_SIZE_MAX]; + struct rhash_head ht_node; /* node of prestera_vr */ + refcount_t refcount; + u32 grp_id; /* hw */ }; struct prestera_fib_key { @@ -44,12 +101,16 @@ struct prestera_fib_info { struct list_head vr_node; enum prestera_fib_type { PRESTERA_FIB_TYPE_INVALID = 0, + /* must be pointer to nh_grp id */ + PRESTERA_FIB_TYPE_UC_NH, /* It can be connected route * and will be overlapped with neighbours */ PRESTERA_FIB_TYPE_TRAP, PRESTERA_FIB_TYPE_DROP } type; + /* Valid only if type = UC_NH*/ + struct prestera_nexthop_group *nh_grp; }; struct prestera_fib_node { @@ -67,6 +128,18 @@ struct prestera_rif_entry * prestera_rif_entry_create(struct prestera_switch *sw, struct prestera_rif_entry_key *k, u32 tb_id, const unsigned char *addr); +struct prestera_nh_neigh * +prestera_nh_neigh_find(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +struct prestera_nh_neigh * +prestera_nh_neigh_get(struct prestera_switch *sw, + struct prestera_nh_neigh_key *key); +void prestera_nh_neigh_put(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +int prestera_nh_neigh_set(struct prestera_switch *sw, + struct prestera_nh_neigh *neigh); +bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw, + struct prestera_nh_neigh *nh_neigh); struct prestera_fib_node *prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key); void prestera_fib_node_destroy(struct prestera_switch *sw, @@ -74,7 +147,8 @@ void prestera_fib_node_destroy(struct prestera_switch *sw, struct prestera_fib_node * prestera_fib_node_create(struct prestera_switch *sw, struct prestera_fib_key *key, - enum prestera_fib_type fib_type); + enum prestera_fib_type fib_type, + struct prestera_nexthop_group_key *nh_grp_key); int prestera_router_hw_init(struct prestera_switch *sw); void prestera_router_hw_fini(struct prestera_switch *sw); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index dc3e3ddc60bf5d5f39c3880bcb3bebd188e266e0..42ee963e9f75967a2392defa36f60baef4a2de3d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -659,7 +659,7 @@ static int prestera_sdma_switch_init(struct prestera_switch *sw) init_dummy_netdev(&sdma->napi_dev); - netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll, 64); + netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll); napi_enable(&sdma->rx_napi); return 0; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c index 845e9d8c8cc7ebf321ef53fc38d3bae4b8247163..1005182ce3bc1d09545b006ac42b76e1e5ec0c1b 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c @@ -107,7 +107,7 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) entry = prestera_span_entry_find_by_id(sw->span, span_id); if (!entry) - return false; + return -ENOENT; if (!refcount_dec_and_test(&entry->ref_count)) return 0; @@ -120,8 +120,9 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id) return 0; } -static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, - struct prestera_port *to_port) +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress) { struct prestera_switch *sw = binding->port->sw; u8 span_id; @@ -135,7 +136,7 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, if (err) return err; - err = prestera_hw_span_bind(binding->port, span_id); + err = prestera_hw_span_bind(binding->port, span_id, ingress); if (err) { prestera_span_put(sw, span_id); return err; @@ -145,11 +146,15 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding, return 0; } -static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress) { int err; - err = prestera_hw_span_unbind(binding->port); + if (binding->span_id == PRESTERA_SPAN_INVALID_ID) + return -ENOENT; + + err = prestera_hw_span_unbind(binding->port, ingress); if (err) return err; @@ -161,60 +166,6 @@ static int prestera_span_rule_del(struct prestera_flow_block_binding *binding) return 0; } -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f) -{ - struct prestera_flow_block_binding *binding; - __be16 protocol = f->common.protocol; - struct flow_action_entry *act; - struct prestera_port *port; - int err; - - if (!flow_offload_has_one_action(&f->rule->action)) { - NL_SET_ERR_MSG(f->common.extack, - "Only singular actions are supported"); - return -EOPNOTSUPP; - } - - act = &f->rule->action.entries[0]; - - if (!prestera_netdev_check(act->dev)) { - NL_SET_ERR_MSG(f->common.extack, - "Only Marvell Prestera port is supported"); - return -EINVAL; - } - if (!tc_cls_can_offload_and_chain0(act->dev, &f->common)) - return -EOPNOTSUPP; - if (act->id != FLOW_ACTION_MIRRED) - return -EOPNOTSUPP; - if (protocol != htons(ETH_P_ALL)) - return -EOPNOTSUPP; - - port = netdev_priv(act->dev); - - list_for_each_entry(binding, &block->binding_list, list) { - err = prestera_span_rule_add(binding, port); - if (err) - goto rollback; - } - - return 0; - -rollback: - list_for_each_entry_continue_reverse(binding, - &block->binding_list, list) - prestera_span_rule_del(binding); - return err; -} - -void prestera_span_destroy(struct prestera_flow_block *block) -{ - struct prestera_flow_block_binding *binding; - - list_for_each_entry(binding, &block->binding_list, list) - prestera_span_rule_del(binding); -} - int prestera_span_init(struct prestera_switch *sw) { struct prestera_span *span; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.h b/drivers/net/ethernet/marvell/prestera/prestera_span.h index f0644521f78a73fb58e50a63871fff80c001dd0d..493b68524bcbcfa5cdaa8dac9441fb86280e40cf 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.h @@ -8,13 +8,17 @@ #define PRESTERA_SPAN_INVALID_ID -1 +struct prestera_port; struct prestera_switch; -struct prestera_flow_block; +struct prestera_flow_block_binding; int prestera_span_init(struct prestera_switch *sw); void prestera_span_fini(struct prestera_switch *sw); -int prestera_span_replace(struct prestera_flow_block *block, - struct tc_cls_matchall_offload *f); -void prestera_span_destroy(struct prestera_flow_block *block); + +int prestera_span_rule_add(struct prestera_flow_block_binding *binding, + struct prestera_port *to_port, + bool ingress); +int prestera_span_rule_del(struct prestera_flow_block_binding *binding, + bool ingress); #endif /* _PRESTERA_SPAN_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c index 71cde97d85c837ca0644d1ae091c428903a0d868..e548cd32582ebf4b5ad3a91fe9ca87b929eac728 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c @@ -143,6 +143,7 @@ prestera_br_port_flags_reset(struct prestera_bridge_port *br_port, prestera_port_uc_flood_set(port, false); prestera_port_mc_flood_set(port, false); prestera_port_learning_set(port, false); + prestera_port_br_locked_set(port, false); } static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port, @@ -162,6 +163,11 @@ static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port, if (err) goto err_out; + err = prestera_port_br_locked_set(port, + br_port->flags & BR_PORT_LOCKED); + if (err) + goto err_out; + return 0; err_out: @@ -1163,7 +1169,7 @@ static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx, break; case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS: if (attr->u.brport_flags.mask & - ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD)) + ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_PORT_LOCKED)) err = -EINVAL; break; case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS: diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c index 349b8a94e9391b48afff2d0721cea5a0404fd0fa..cf456d62677feba30556201586d86c3f2ff8858d 100644 --- a/drivers/net/ethernet/marvell/pxa168_eth.c +++ b/drivers/net/ethernet/marvell/pxa168_eth.c @@ -1354,10 +1354,10 @@ static void pxa168_eth_netpoll(struct net_device *dev) static void pxa168_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); - strlcpy(info->bus_info, "N/A", sizeof(info->bus_info)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->bus_info, "N/A", sizeof(info->bus_info)); } static const struct ethtool_ops pxa168_ethtool_ops = { diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index c1e985416c0e6b9d4dde3d7d496536e141622d17..1b43704baceb5284a20ed323a06ec81d5e9318c7 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -394,9 +394,9 @@ static void skge_get_drvinfo(struct net_device *dev, { struct skge_port *skge = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(skge->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(skge->hw->pdev), sizeof(info->bus_info)); } @@ -3832,7 +3832,7 @@ static struct net_device *skge_devinit(struct skge_hw *hw, int port, dev->features |= NETIF_F_HIGHDMA; skge = netdev_priv(dev); - netif_napi_add(dev, &skge->napi, skge_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &skge->napi, skge_poll); skge->netdev = dev; skge->hw = hw; skge->msg_enable = netif_msg_init(debug, default_msg); diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index bbea5458000bf44a16b84568137b610f2c1bf284..ab33ba1c3023c47f8fb9e7574a8a5ab12e8918d7 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -3687,9 +3687,9 @@ static void sky2_get_drvinfo(struct net_device *dev, { struct sky2_port *sky2 = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(sky2->hw->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(sky2->hw->pdev), sizeof(info->bus_info)); } @@ -4937,7 +4937,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } } - netif_napi_add(dev, &hw->napi, sky2_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &hw->napi, sky2_poll); err = register_netdev(dev); if (err) { diff --git a/drivers/net/ethernet/mediatek/Makefile b/drivers/net/ethernet/mediatek/Makefile index fe66ba8793cf8c53e09fc123003b9b5778a3bcc6..45ba0970504a4ced639c1a133a3c23505e0ddbc6 100644 --- a/drivers/net/ethernet/mediatek/Makefile +++ b/drivers/net/ethernet/mediatek/Makefile @@ -11,8 +11,3 @@ mtk_eth-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_debugfs.o endif obj-$(CONFIG_NET_MEDIATEK_SOC_WED) += mtk_wed_ops.o obj-$(CONFIG_NET_MEDIATEK_STAR_EMAC) += mtk_star_emac.o - -# FIXME: temporarily silence -Warray-bounds on non W=1+ builds -ifndef KBUILD_EXTRA_WARN -CFLAGS_mtk_ppe.o += -Wno-array-bounds -endif diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 8aff4c0c28bd7c8d7f80390dd83b97b77d4896fb..4fba7cb0144bae7e4f4b942a3832a325700bffff 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -73,6 +73,12 @@ static const struct mtk_reg_map mtk_reg_map = { .fq_blen = 0x1b2c, }, .gdm1_cnt = 0x2400, + .gdma_to_ppe = 0x4444, + .ppe_base = 0x0c00, + .wdma_base = { + [0] = 0x2800, + [1] = 0x2c00, + }, }; static const struct mtk_reg_map mt7628_reg_map = { @@ -126,6 +132,12 @@ static const struct mtk_reg_map mt7986_reg_map = { .fq_blen = 0x472c, }, .gdm1_cnt = 0x1c00, + .gdma_to_ppe = 0x3333, + .ppe_base = 0x2000, + .wdma_base = { + [0] = 0x4800, + [1] = 0x4c00, + }, }; /* strings used by ethtool */ @@ -1458,7 +1470,7 @@ static void mtk_update_rx_cpu_idx(struct mtk_eth *eth) static bool mtk_page_pool_enabled(struct mtk_eth *eth) { - return !eth->hwlro; + return MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2); } static struct page_pool *mtk_create_page_pool(struct mtk_eth *eth, @@ -1573,8 +1585,8 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, .last = !xdp_frame_has_frags(xdpf), }; int err, index = 0, n_desc = 1, nr_frags; - struct mtk_tx_dma *htxd, *txd, *txd_pdma; struct mtk_tx_buf *htx_buf, *tx_buf; + struct mtk_tx_dma *htxd, *txd; void *data = xdpf->data; if (unlikely(test_bit(MTK_RESETTING, ð->state))) @@ -1608,7 +1620,6 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) { txd = mtk_qdma_phys_to_virt(ring, txd->txd2); - txd_pdma = qdma_to_pdma(ring, txd); if (txd == ring->last_free) goto unmap; @@ -1629,7 +1640,8 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, htx_buf->data = xdpf; if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { - txd_pdma = qdma_to_pdma(ring, txd); + struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, txd); + if (index & 1) txd_pdma->txd2 |= TX_DMA_LS0; else @@ -1660,13 +1672,15 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf, unmap: while (htxd != txd) { - txd_pdma = qdma_to_pdma(ring, htxd); tx_buf = mtk_desc_to_tx_buf(ring, htxd, soc->txrx.txd_size); mtk_tx_unmap(eth, tx_buf, NULL, false); htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU; - if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) + if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) { + struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, htxd); + txd_pdma->txd2 = TX_DMA_DESP2_DEF; + } htxd = mtk_qdma_phys_to_virt(ring, htxd->txd2); } @@ -1891,10 +1905,21 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skb->dev = netdev; bytes += skb->len; - if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + reason = FIELD_GET(MTK_RXD5_PPE_CPU_REASON, trxd.rxd5); + hash = trxd.rxd5 & MTK_RXD5_FOE_ENTRY; + if (hash != MTK_RXD5_FOE_ENTRY) + skb_set_hash(skb, jhash_1word(hash, 0), + PKT_HASH_TYPE_L4); rxdcsum = &trxd.rxd3; - else + } else { + reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4); + hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY; + if (hash != MTK_RXD4_FOE_ENTRY) + skb_set_hash(skb, jhash_1word(hash, 0), + PKT_HASH_TYPE_L4); rxdcsum = &trxd.rxd4; + } if (*rxdcsum & eth->soc->txrx.rx_dma_l4_valid) skb->ip_summed = CHECKSUM_UNNECESSARY; @@ -1902,16 +1927,8 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget, skb_checksum_none_assert(skb); skb->protocol = eth_type_trans(skb, netdev); - hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY; - if (hash != MTK_RXD4_FOE_ENTRY) { - hash = jhash_1word(hash, 0); - skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); - } - - reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4); if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) - mtk_ppe_check_skb(eth->ppe, skb, - trxd.rxd4 & MTK_RXD4_FOE_ENTRY); + mtk_ppe_check_skb(eth->ppe[0], skb, hash); if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) { if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { @@ -2974,21 +2991,25 @@ static int mtk_open(struct net_device *dev) /* we run 2 netdevs on the same dma ring so we only bring it up once */ if (!refcount_read(ð->dma_refcnt)) { - u32 gdm_config = MTK_GDMA_TO_PDMA; + const struct mtk_soc_data *soc = eth->soc; + u32 gdm_config; + int i; err = mtk_start_dma(eth); if (err) return err; - if (eth->soc->offload_version && mtk_ppe_start(eth->ppe) == 0) - gdm_config = MTK_GDMA_TO_PPE; + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_start(eth->ppe[i]); + gdm_config = soc->offload_version ? soc->reg_map->gdma_to_ppe + : MTK_GDMA_TO_PDMA; mtk_gdm_config(eth, gdm_config); napi_enable(ð->tx_napi); napi_enable(ð->rx_napi); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); - mtk_rx_irq_enable(eth, eth->soc->txrx.rx_irq_done_mask); + mtk_rx_irq_enable(eth, soc->txrx.rx_irq_done_mask); refcount_set(ð->dma_refcnt, 1); } else @@ -3026,6 +3047,7 @@ static int mtk_stop(struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; + int i; phylink_stop(mac->phylink); @@ -3053,8 +3075,8 @@ static int mtk_stop(struct net_device *dev) mtk_dma_free(eth); - if (eth->soc->offload_version) - mtk_ppe_stop(eth->ppe); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_stop(eth->ppe[i]); return 0; } @@ -3554,8 +3576,8 @@ static void mtk_get_drvinfo(struct net_device *dev, { struct mtk_mac *mac = netdev_priv(dev); - strlcpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info)); + strscpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info)); info->n_stats = ARRAY_SIZE(mtk_ethtool_stats); } @@ -3923,6 +3945,7 @@ void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev) static int mtk_probe(struct platform_device *pdev) { + struct resource *res = NULL; struct device_node *mac_np; struct mtk_eth *eth; int err, i; @@ -4003,20 +4026,31 @@ static int mtk_probe(struct platform_device *pdev) } } - for (i = 0;; i++) { - struct device_node *np = of_parse_phandle(pdev->dev.of_node, - "mediatek,wed", i); - static const u32 wdma_regs[] = { - MTK_WDMA0_BASE, - MTK_WDMA1_BASE - }; - void __iomem *wdma; - - if (!np || i >= ARRAY_SIZE(wdma_regs)) - break; + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; + } - wdma = eth->base + wdma_regs[i]; - mtk_wed_add_hw(np, eth, wdma, i); + if (eth->soc->offload_version) { + for (i = 0;; i++) { + struct device_node *np; + phys_addr_t wdma_phy; + u32 wdma_base; + + if (i >= ARRAY_SIZE(eth->soc->reg_map->wdma_base)) + break; + + np = of_parse_phandle(pdev->dev.of_node, + "mediatek,wed", i); + if (!np) + break; + + wdma_base = eth->soc->reg_map->wdma_base[i]; + wdma_phy = res ? res->start + wdma_base : 0; + mtk_wed_add_hw(np, eth, eth->base + wdma_base, + wdma_phy, i); + } } for (i = 0; i < 3; i++) { @@ -4094,10 +4128,19 @@ static int mtk_probe(struct platform_device *pdev) } if (eth->soc->offload_version) { - eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2); - if (!eth->ppe) { - err = -ENOMEM; - goto err_free_dev; + u32 num_ppe; + + num_ppe = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1; + num_ppe = min_t(u32, ARRAY_SIZE(eth->ppe), num_ppe); + for (i = 0; i < num_ppe; i++) { + u32 ppe_addr = eth->soc->reg_map->ppe_base + i * 0x400; + + eth->ppe[i] = mtk_ppe_init(eth, eth->base + ppe_addr, + eth->soc->offload_version, i); + if (!eth->ppe[i]) { + err = -ENOMEM; + goto err_free_dev; + } } err = mtk_eth_offload_init(eth); @@ -4123,10 +4166,8 @@ static int mtk_probe(struct platform_device *pdev) * for NAPI to work */ init_dummy_netdev(ð->dummy_dev); - netif_napi_add(ð->dummy_dev, ð->tx_napi, mtk_napi_tx, - NAPI_POLL_WEIGHT); - netif_napi_add(ð->dummy_dev, ð->rx_napi, mtk_napi_rx, - NAPI_POLL_WEIGHT); + netif_napi_add(ð->dummy_dev, ð->tx_napi, mtk_napi_tx); + netif_napi_add(ð->dummy_dev, ð->rx_napi, mtk_napi_rx); platform_set_drvdata(pdev, eth); @@ -4190,6 +4231,8 @@ static const struct mtk_soc_data mt7621_data = { .required_clks = MT7621_CLKS_BITMAP, .required_pctl = false, .offload_version = 2, + .hash_offset = 2, + .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4208,6 +4251,8 @@ static const struct mtk_soc_data mt7622_data = { .required_clks = MT7622_CLKS_BITMAP, .required_pctl = false, .offload_version = 2, + .hash_offset = 2, + .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4225,6 +4270,8 @@ static const struct mtk_soc_data mt7623_data = { .required_clks = MT7623_CLKS_BITMAP, .required_pctl = true, .offload_version = 2, + .hash_offset = 2, + .foe_entry_size = sizeof(struct mtk_foe_entry) - 16, .txrx = { .txd_size = sizeof(struct mtk_tx_dma), .rxd_size = sizeof(struct mtk_rx_dma), @@ -4256,8 +4303,11 @@ static const struct mtk_soc_data mt7986_data = { .reg_map = &mt7986_reg_map, .ana_rgc3 = 0x128, .caps = MT7986_CAPS, + .hw_features = MTK_HW_FEATURES, .required_clks = MT7986_CLKS_BITMAP, .required_pctl = false, + .hash_offset = 4, + .foe_entry_size = sizeof(struct mtk_foe_entry), .txrx = { .txd_size = sizeof(struct mtk_tx_dma_v2), .rxd_size = sizeof(struct mtk_rx_dma_v2), diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 7405c97cda660eae1d777efdcd5a22414def0f1e..b52f3b0177efb9ed4db04567d5e58211c9b83f84 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -105,7 +105,6 @@ #define MTK_GDMA_TCS_EN BIT(21) #define MTK_GDMA_UCS_EN BIT(20) #define MTK_GDMA_TO_PDMA 0x0 -#define MTK_GDMA_TO_PPE 0x4444 #define MTK_GDMA_DROP_ALL 0x7777 /* Unicast Filter MAC Address Register - Low */ @@ -269,9 +268,6 @@ #define TX_DMA_FPORT_MASK_V2 0xf #define TX_DMA_SWC_V2 BIT(30) -#define MTK_WDMA0_BASE 0x2800 -#define MTK_WDMA1_BASE 0x2c00 - /* QDMA descriptor txd4 */ #define TX_DMA_CHKSUM (0x7 << 29) #define TX_DMA_TSO BIT(28) @@ -314,8 +310,13 @@ #define RX_DMA_L4_VALID_PDMA BIT(30) /* when PDMA is used */ #define RX_DMA_SPECIAL_TAG BIT(22) -#define RX_DMA_GET_SPORT(x) (((x) >> 19) & 0xf) -#define RX_DMA_GET_SPORT_V2(x) (((x) >> 26) & 0x7) +/* PDMA descriptor rxd5 */ +#define MTK_RXD5_FOE_ENTRY GENMASK(14, 0) +#define MTK_RXD5_PPE_CPU_REASON GENMASK(22, 18) +#define MTK_RXD5_SRC_PORT GENMASK(29, 26) + +#define RX_DMA_GET_SPORT(x) (((x) >> 19) & 0x7) +#define RX_DMA_GET_SPORT_V2(x) (((x) >> 26) & 0xf) /* PDMA V2 descriptor rxd3 */ #define RX_DMA_VTAG_V2 BIT(0) @@ -950,6 +951,9 @@ struct mtk_reg_map { u32 fq_blen; /* fq free page buffer length */ } qdma; u32 gdm1_cnt; + u32 gdma_to_ppe; + u32 ppe_base; + u32 wdma_base[2]; }; /* struct mtk_eth_data - This is the structure holding all differences @@ -963,6 +967,8 @@ struct mtk_reg_map { * the target SoC * @required_pctl A bool value to show whether the SoC requires * the extra setup for those pins used by GMAC. + * @hash_offset Flow table hash offset. + * @foe_entry_size Foe table entry size. * @txd_size Tx DMA descriptor size. * @rxd_size Rx DMA descriptor size. * @rx_irq_done_mask Rx irq done register mask. @@ -977,6 +983,8 @@ struct mtk_soc_data { u32 required_clks; bool required_pctl; u8 offload_version; + u8 hash_offset; + u16 foe_entry_size; netdev_features_t hw_features; struct { u32 txd_size; @@ -1106,7 +1114,7 @@ struct mtk_eth { int ip_align; - struct mtk_ppe *ppe; + struct mtk_ppe *ppe[2]; struct rhashtable flow_table; struct bpf_prog __rcu *prog; @@ -1137,6 +1145,86 @@ struct mtk_mac { /* the struct describing the SoC. these are declared in the soc_xyz.c files */ extern const struct of_device_id of_mtk_match[]; +static inline struct mtk_foe_entry * +mtk_foe_get_entry(struct mtk_ppe *ppe, u16 hash) +{ + const struct mtk_soc_data *soc = ppe->eth->soc; + + return ppe->foe_table + hash * soc->foe_entry_size; +} + +static inline u32 mtk_get_ib1_ts_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB1_BIND_TIMESTAMP_V2; + + return MTK_FOE_IB1_BIND_TIMESTAMP; +} + +static inline u32 mtk_get_ib1_ppoe_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB1_BIND_PPPOE_V2; + + return MTK_FOE_IB1_BIND_PPPOE; +} + +static inline u32 mtk_get_ib1_vlan_tag_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB1_BIND_VLAN_TAG_V2; + + return MTK_FOE_IB1_BIND_VLAN_TAG; +} + +static inline u32 mtk_get_ib1_vlan_layer_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB1_BIND_VLAN_LAYER_V2; + + return MTK_FOE_IB1_BIND_VLAN_LAYER; +} + +static inline u32 mtk_prep_ib1_vlan_layer(struct mtk_eth *eth, u32 val) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val); + + return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, val); +} + +static inline u32 mtk_get_ib1_vlan_layer(struct mtk_eth *eth, u32 val) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val); + + return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, val); +} + +static inline u32 mtk_get_ib1_pkt_type_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB1_PACKET_TYPE_V2; + + return MTK_FOE_IB1_PACKET_TYPE; +} + +static inline u32 mtk_get_ib1_pkt_type(struct mtk_eth *eth, u32 val) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE_V2, val); + + return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, val); +} + +static inline u32 mtk_get_ib2_multicast_mask(struct mtk_eth *eth) +{ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) + return MTK_FOE_IB2_MULTICAST_V2; + + return MTK_FOE_IB2_MULTICAST; +} + /* read the hardware status register */ void mtk_stats_update_mac(struct mtk_mac *mac); diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index dab8f3f771f848079c6858b3491c3a9c98770fd3..ae00e572390d7ba705f0a865d174c1b161ec7c36 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -56,7 +56,7 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val) static u32 mtk_eth_timestamp(struct mtk_eth *eth) { - return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP; + return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth); } static int mtk_ppe_wait_busy(struct mtk_ppe *ppe) @@ -88,12 +88,12 @@ static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable) enable * MTK_PPE_CACHE_CTL_EN); } -static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e) +static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e) { u32 hv1, hv2, hv3; u32 hash; - switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) { + switch (mtk_get_ib1_pkt_type(eth, e->ib1)) { case MTK_PPE_PKT_TYPE_IPV4_ROUTE: case MTK_PPE_PKT_TYPE_IPV4_HNAPT: hv1 = e->ipv4.orig.ports; @@ -122,16 +122,16 @@ static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e) hash = (hash >> 24) | ((hash & 0xffffff) << 8); hash ^= hv1 ^ hv2 ^ hv3; hash ^= hash >> 16; - hash <<= 1; + hash <<= (ffs(eth->soc->hash_offset) - 1); hash &= MTK_PPE_ENTRIES - 1; return hash; } static inline struct mtk_foe_mac_info * -mtk_foe_entry_l2(struct mtk_foe_entry *entry) +mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); if (type == MTK_PPE_PKT_TYPE_BRIDGE) return &entry->bridge.l2; @@ -143,9 +143,9 @@ mtk_foe_entry_l2(struct mtk_foe_entry *entry) } static inline u32 * -mtk_foe_entry_ib2(struct mtk_foe_entry *entry) +mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); if (type == MTK_PPE_PKT_TYPE_BRIDGE) return &entry->bridge.ib2; @@ -156,27 +156,38 @@ mtk_foe_entry_ib2(struct mtk_foe_entry *entry) return &entry->ipv4.ib2; } -int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, - u8 pse_port, u8 *src_mac, u8 *dest_mac) +int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int type, int l4proto, u8 pse_port, u8 *src_mac, + u8 *dest_mac) { struct mtk_foe_mac_info *l2; u32 ports_pad, val; memset(entry, 0, sizeof(*entry)); - val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | - FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | - FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | - MTK_FOE_IB1_BIND_TTL | - MTK_FOE_IB1_BIND_CACHE; - entry->ib1 = val; + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | + FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) | + FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | + MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2; + entry->ib1 = val; - val = FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) | - FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f) | - FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port); + val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) | + FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf); + } else { + val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) | + FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) | + FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) | + MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL; + entry->ib1 = val; + + val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) | + FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) | + FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f); + } if (is_multicast_ether_addr(dest_mac)) - val |= MTK_FOE_IB2_MULTICAST; + val |= mtk_get_ib2_multicast_mask(eth); ports_pad = 0xa5a5a500 | (l4proto & 0xff); if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE) @@ -210,24 +221,30 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, return 0; } -int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port) +int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, + struct mtk_foe_entry *entry, u8 port) { - u32 *ib2 = mtk_foe_entry_ib2(entry); - u32 val; + u32 *ib2 = mtk_foe_entry_ib2(eth, entry); + u32 val = *ib2; - val = *ib2; - val &= ~MTK_FOE_IB2_DEST_PORT; - val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + val &= ~MTK_FOE_IB2_DEST_PORT_V2; + val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port); + } else { + val &= ~MTK_FOE_IB2_DEST_PORT; + val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port); + } *ib2 = val; return 0; } -int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress, +int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, bool egress, __be32 src_addr, __be16 src_port, __be32 dest_addr, __be16 dest_port) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); struct mtk_ipv4_tuple *t; switch (type) { @@ -262,11 +279,12 @@ int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress, return 0; } -int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, +int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, __be32 *src_addr, __be16 src_port, __be32 *dest_addr, __be16 dest_port) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1); + int type = mtk_get_ib1_pkt_type(eth, entry->ib1); u32 *src, *dest; int i; @@ -297,39 +315,41 @@ int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, return 0; } -int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port) +int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int port) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); l2->etype = BIT(port); - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER)) - entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth))) + entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1); else l2->etype |= BIT(8); - entry->ib1 &= ~MTK_FOE_IB1_BIND_VLAN_TAG; + entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth); return 0; } -int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid) +int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int vid) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); - switch (FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, entry->ib1)) { + switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) { case 0: - entry->ib1 |= MTK_FOE_IB1_BIND_VLAN_TAG | - FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) | + mtk_prep_ib1_vlan_layer(eth, 1); l2->vlan1 = vid; return 0; case 1: - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) { + if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) { l2->vlan1 = vid; l2->etype |= BIT(8); } else { l2->vlan2 = vid; - entry->ib1 += FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1); + entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1); } return 0; default: @@ -337,34 +357,42 @@ int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid) } } -int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid) +int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int sid) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); - if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER) || - (entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) + if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) || + (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) l2->etype = ETH_P_PPP_SES; - entry->ib1 |= MTK_FOE_IB1_BIND_PPPOE; + entry->ib1 |= mtk_get_ib1_ppoe_mask(eth); l2->pppoe_id = sid; return 0; } -int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq, - int bss, int wcid) +int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int wdma_idx, int txq, int bss, int wcid) { - struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry); - u32 *ib2 = mtk_foe_entry_ib2(entry); - - *ib2 &= ~MTK_FOE_IB2_PORT_MG; - *ib2 |= MTK_FOE_IB2_WDMA_WINFO; - if (wdma_idx) - *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; + struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry); + u32 *ib2 = mtk_foe_entry_ib2(eth, entry); - l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | - FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | - FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + *ib2 &= ~MTK_FOE_IB2_PORT_MG_V2; + *ib2 |= FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) | + MTK_FOE_IB2_WDMA_WINFO_V2; + l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) | + FIELD_PREP(MTK_FOE_WINFO_BSS, bss); + } else { + *ib2 &= ~MTK_FOE_IB2_PORT_MG; + *ib2 |= MTK_FOE_IB2_WDMA_WINFO; + if (wdma_idx) + *ib2 |= MTK_FOE_IB2_WDMA_DEVIDX; + l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) | + FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq); + } return 0; } @@ -376,14 +404,15 @@ static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry) } static bool -mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data) +mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry, + struct mtk_foe_entry *data) { int type, len; if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP) return false; - type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1); + type = mtk_get_ib1_pkt_type(eth, entry->data.ib1); if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE) len = offsetof(struct mtk_foe_entry, ipv6._rsv); else @@ -410,9 +439,10 @@ __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) hlist_del_init(&entry->list); if (entry->hash != 0xffff) { - ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE; - ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, - MTK_FOE_STATE_BIND); + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash); + + hwe->ib1 &= ~MTK_FOE_IB1_STATE; + hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID); dma_wmb(); } entry->hash = 0xffff; @@ -426,14 +456,12 @@ __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) { - u16 timestamp; - u16 now; - - now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP; - timestamp = ib1 & MTK_FOE_IB1_BIND_TIMESTAMP; + u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); + u16 now = mtk_eth_timestamp(ppe->eth); + u16 timestamp = ib1 & ib1_ts_mask; if (timestamp > now) - return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now; + return ib1_ts_mask + 1 - timestamp + now; else return now - timestamp; } @@ -441,6 +469,7 @@ static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1) static void mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) { + u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth); struct mtk_flow_entry *cur; struct mtk_foe_entry *hwe; struct hlist_node *tmp; @@ -451,7 +480,7 @@ mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) int cur_idle; u32 ib1; - hwe = &ppe->foe_table[cur->hash]; + hwe = mtk_foe_get_entry(ppe, cur->hash); ib1 = READ_ONCE(hwe->ib1); if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) { @@ -465,16 +494,16 @@ mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) continue; idle = cur_idle; - entry->data.ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; - entry->data.ib1 |= hwe->ib1 & MTK_FOE_IB1_BIND_TIMESTAMP; + entry->data.ib1 &= ~ib1_ts_mask; + entry->data.ib1 |= hwe->ib1 & ib1_ts_mask; } } static void mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) { + struct mtk_foe_entry foe = {}; struct mtk_foe_entry *hwe; - struct mtk_foe_entry foe; spin_lock_bh(&ppe_lock); @@ -486,9 +515,9 @@ mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) if (entry->hash == 0xffff) goto out; - hwe = &ppe->foe_table[entry->hash]; - memcpy(&foe, hwe, sizeof(foe)); - if (!mtk_flow_entry_match(entry, &foe)) { + hwe = mtk_foe_get_entry(ppe, entry->hash); + memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size); + if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) { entry->hash = 0xffff; goto out; } @@ -503,16 +532,22 @@ static void __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry, u16 hash) { + struct mtk_eth *eth = ppe->eth; + u16 timestamp = mtk_eth_timestamp(eth); struct mtk_foe_entry *hwe; - u16 timestamp; - timestamp = mtk_eth_timestamp(ppe->eth); - timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP; - entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; - entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2; + entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2, + timestamp); + } else { + entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP; + entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, + timestamp); + } - hwe = &ppe->foe_table[hash]; - memcpy(&hwe->data, &entry->data, sizeof(hwe->data)); + hwe = mtk_foe_get_entry(ppe, hash); + memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1)); wmb(); hwe->ib1 = entry->ib1; @@ -539,16 +574,17 @@ mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) { - int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1); + const struct mtk_soc_data *soc = ppe->eth->soc; + int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1); u32 hash; if (type == MTK_PPE_PKT_TYPE_BRIDGE) return mtk_foe_entry_commit_l2(ppe, entry); - hash = mtk_ppe_hash_entry(&entry->data); + hash = mtk_ppe_hash_entry(ppe->eth, &entry->data); entry->hash = 0xffff; spin_lock_bh(&ppe_lock); - hlist_add_head(&entry->list, &ppe->foe_flow[hash / 2]); + hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]); spin_unlock_bh(&ppe_lock); return 0; @@ -558,10 +594,11 @@ static void mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, u16 hash) { + const struct mtk_soc_data *soc = ppe->eth->soc; struct mtk_flow_entry *flow_info; - struct mtk_foe_entry foe, *hwe; + struct mtk_foe_entry foe = {}, *hwe; struct mtk_foe_mac_info *l2; - u32 ib1_mask = MTK_FOE_IB1_PACKET_TYPE | MTK_FOE_IB1_UDP; + u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP; int type; flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end), @@ -572,32 +609,34 @@ mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry, flow_info->l2_data.base_flow = entry; flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW; flow_info->hash = hash; - hlist_add_head(&flow_info->list, &ppe->foe_flow[hash / 2]); + hlist_add_head(&flow_info->list, + &ppe->foe_flow[hash / soc->hash_offset]); hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows); - hwe = &ppe->foe_table[hash]; - memcpy(&foe, hwe, sizeof(foe)); + hwe = mtk_foe_get_entry(ppe, hash); + memcpy(&foe, hwe, soc->foe_entry_size); foe.ib1 &= ib1_mask; foe.ib1 |= entry->data.ib1 & ~ib1_mask; - l2 = mtk_foe_entry_l2(&foe); + l2 = mtk_foe_entry_l2(ppe->eth, &foe); memcpy(l2, &entry->data.bridge.l2, sizeof(*l2)); - type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, foe.ib1); + type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1); if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT) memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new)); else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP) l2->etype = ETH_P_IPV6; - *mtk_foe_entry_ib2(&foe) = entry->data.bridge.ib2; + *mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2; __mtk_foe_entry_commit(ppe, &foe, hash); } void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) { - struct hlist_head *head = &ppe->foe_flow[hash / 2]; - struct mtk_foe_entry *hwe = &ppe->foe_table[hash]; + const struct mtk_soc_data *soc = ppe->eth->soc; + struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset]; + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash); struct mtk_flow_entry *entry; struct mtk_foe_bridge key = {}; struct hlist_node *n; @@ -621,7 +660,7 @@ void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) continue; } - if (found || !mtk_flow_entry_match(entry, hwe)) { + if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) { if (entry->hash != 0xffff) entry->hash = 0xffff; continue; @@ -678,11 +717,13 @@ int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry) } struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, - int version) + int version, int index) { + const struct mtk_soc_data *soc = eth->soc; struct device *dev = eth->dev; - struct mtk_foe_entry *foe; struct mtk_ppe *ppe; + u32 foe_flow_size; + void *foe; ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL); if (!ppe) @@ -698,14 +739,21 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, ppe->dev = dev; ppe->version = version; - foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe), + foe = dmam_alloc_coherent(ppe->dev, + MTK_PPE_ENTRIES * soc->foe_entry_size, &ppe->foe_phys, GFP_KERNEL); if (!foe) return NULL; ppe->foe_table = foe; - mtk_ppe_debugfs_init(ppe); + foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) * + sizeof(*ppe->foe_flow); + ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL); + if (!ppe->foe_flow) + return NULL; + + mtk_ppe_debugfs_init(ppe, index); return ppe; } @@ -715,21 +763,30 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 }; int i, k; - memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(*ppe->foe_table)); + memset(ppe->foe_table, 0, + MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size); if (!IS_ENABLED(CONFIG_SOC_MT7621)) return; /* skip all entries that cross the 1024 byte boundary */ - for (i = 0; i < MTK_PPE_ENTRIES; i += 128) - for (k = 0; k < ARRAY_SIZE(skip); k++) - ppe->foe_table[i + skip[k]].ib1 |= MTK_FOE_IB1_STATIC; + for (i = 0; i < MTK_PPE_ENTRIES; i += 128) { + for (k = 0; k < ARRAY_SIZE(skip); k++) { + struct mtk_foe_entry *hwe; + + hwe = mtk_foe_get_entry(ppe, i + skip[k]); + hwe->ib1 |= MTK_FOE_IB1_STATIC; + } + } } -int mtk_ppe_start(struct mtk_ppe *ppe) +void mtk_ppe_start(struct mtk_ppe *ppe) { u32 val; + if (!ppe) + return; + mtk_ppe_init_foe_table(ppe); ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys); @@ -748,6 +805,8 @@ int mtk_ppe_start(struct mtk_ppe *ppe) MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) | FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM, MTK_PPE_ENTRIES_SHIFT); + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + val |= MTK_PPE_TB_CFG_INFO_SEL; ppe_w32(ppe, MTK_PPE_TB_CFG, val); ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK, @@ -755,15 +814,21 @@ int mtk_ppe_start(struct mtk_ppe *ppe) mtk_ppe_cache_enable(ppe, true); - val = MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | - MTK_PPE_FLOW_CFG_IP4_UDP_FRAG | - MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | + val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE | MTK_PPE_FLOW_CFG_IP6_5T_ROUTE | MTK_PPE_FLOW_CFG_IP6_6RD | MTK_PPE_FLOW_CFG_IP4_NAT | MTK_PPE_FLOW_CFG_IP4_NAPT | MTK_PPE_FLOW_CFG_IP4_DSLITE | MTK_PPE_FLOW_CFG_IP4_NAT_FRAG; + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) + val |= MTK_PPE_MD_TOAP_BYP_CRSN0 | + MTK_PPE_MD_TOAP_BYP_CRSN1 | + MTK_PPE_MD_TOAP_BYP_CRSN2 | + MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY; + else + val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG | + MTK_PPE_FLOW_CFG_IP4_UDP_FRAG; ppe_w32(ppe, MTK_PPE_FLOW_CFG, val); val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) | @@ -798,7 +863,10 @@ int mtk_ppe_start(struct mtk_ppe *ppe) ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0); - return 0; + if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) { + ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777); + ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f); + } } int mtk_ppe_stop(struct mtk_ppe *ppe) @@ -806,9 +874,15 @@ int mtk_ppe_stop(struct mtk_ppe *ppe) u32 val; int i; - for (i = 0; i < MTK_PPE_ENTRIES; i++) - ppe->foe_table[i].ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, - MTK_FOE_STATE_INVALID); + if (!ppe) + return 0; + + for (i = 0; i < MTK_PPE_ENTRIES; i++) { + struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i); + + hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE, + MTK_FOE_STATE_INVALID); + } mtk_ppe_cache_enable(ppe, false); diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index 1f5cf1c9a9475ce0f762ca46df28fa31979f330f..0b7a67a958e4ca9d218ebf33b3e8a48503ff61ff 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -8,8 +8,6 @@ #include #include -#define MTK_ETH_PPE_BASE 0xc00 - #define MTK_PPE_ENTRIES_SHIFT 3 #define MTK_PPE_ENTRIES (1024 << MTK_PPE_ENTRIES_SHIFT) #define MTK_PPE_HASH_MASK (MTK_PPE_ENTRIES - 1) @@ -34,6 +32,15 @@ #define MTK_FOE_IB1_UDP BIT(30) #define MTK_FOE_IB1_STATIC BIT(31) +/* CONFIG_MEDIATEK_NETSYS_V2 */ +#define MTK_FOE_IB1_BIND_TIMESTAMP_V2 GENMASK(7, 0) +#define MTK_FOE_IB1_BIND_VLAN_LAYER_V2 GENMASK(16, 14) +#define MTK_FOE_IB1_BIND_PPPOE_V2 BIT(17) +#define MTK_FOE_IB1_BIND_VLAN_TAG_V2 BIT(18) +#define MTK_FOE_IB1_BIND_CACHE_V2 BIT(20) +#define MTK_FOE_IB1_BIND_TTL_V2 BIT(22) +#define MTK_FOE_IB1_PACKET_TYPE_V2 GENMASK(27, 23) + enum { MTK_PPE_PKT_TYPE_IPV4_HNAPT = 0, MTK_PPE_PKT_TYPE_IPV4_ROUTE = 1, @@ -55,14 +62,25 @@ enum { #define MTK_FOE_IB2_PORT_MG GENMASK(17, 12) +#define MTK_FOE_IB2_RX_IDX GENMASK(18, 17) #define MTK_FOE_IB2_PORT_AG GENMASK(23, 18) #define MTK_FOE_IB2_DSCP GENMASK(31, 24) +/* CONFIG_MEDIATEK_NETSYS_V2 */ +#define MTK_FOE_IB2_PORT_MG_V2 BIT(7) +#define MTK_FOE_IB2_DEST_PORT_V2 GENMASK(12, 9) +#define MTK_FOE_IB2_MULTICAST_V2 BIT(13) +#define MTK_FOE_IB2_WDMA_WINFO_V2 BIT(19) +#define MTK_FOE_IB2_PORT_AG_V2 GENMASK(23, 20) + #define MTK_FOE_VLAN2_WINFO_BSS GENMASK(5, 0) #define MTK_FOE_VLAN2_WINFO_WCID GENMASK(13, 6) #define MTK_FOE_VLAN2_WINFO_RING GENMASK(15, 14) +#define MTK_FOE_WINFO_BSS GENMASK(5, 0) +#define MTK_FOE_WINFO_WCID GENMASK(15, 6) + enum { MTK_FOE_STATE_INVALID, MTK_FOE_STATE_UNBIND, @@ -83,6 +101,9 @@ struct mtk_foe_mac_info { u16 pppoe_id; u16 src_mac_lo; + + u16 minfo; + u16 winfo; }; /* software-only entry type */ @@ -200,7 +221,7 @@ struct mtk_foe_entry { struct mtk_foe_ipv4_dslite dslite; struct mtk_foe_ipv6 ipv6; struct mtk_foe_ipv6_6rd ipv6_6rd; - u32 data[19]; + u32 data[23]; }; }; @@ -249,6 +270,7 @@ struct mtk_flow_entry { }; u8 type; s8 wed_index; + u8 ppe_index; u16 hash; union { struct mtk_foe_entry data; @@ -267,20 +289,22 @@ struct mtk_ppe { struct device *dev; void __iomem *base; int version; + char dirname[5]; - struct mtk_foe_entry *foe_table; + void *foe_table; dma_addr_t foe_phys; u16 foe_check_time[MTK_PPE_ENTRIES]; - struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2]; + struct hlist_head *foe_flow; struct rhashtable l2_flows; void *acct_table; }; -struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version); -int mtk_ppe_start(struct mtk_ppe *ppe); +struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, + int version, int index); +void mtk_ppe_start(struct mtk_ppe *ppe); int mtk_ppe_stop(struct mtk_ppe *ppe); void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash); @@ -293,6 +317,9 @@ mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) if (!ppe) return; + if (hash > MTK_PPE_HASH_MASK) + return; + now = (u16)jiffies; diff = now - ppe->foe_check_time[hash]; if (diff < HZ / 10) @@ -302,34 +329,30 @@ mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash) __mtk_ppe_check_skb(ppe, skb, hash); } -static inline int -mtk_foe_entry_timestamp(struct mtk_ppe *ppe, u16 hash) -{ - u32 ib1 = READ_ONCE(ppe->foe_table[hash].ib1); - - if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) - return -1; - - return FIELD_GET(MTK_FOE_IB1_BIND_TIMESTAMP, ib1); -} - -int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto, - u8 pse_port, u8 *src_mac, u8 *dest_mac); -int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port); -int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool orig, +int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int type, int l4proto, u8 pse_port, u8 *src_mac, + u8 *dest_mac); +int mtk_foe_entry_set_pse_port(struct mtk_eth *eth, + struct mtk_foe_entry *entry, u8 port); +int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, bool orig, __be32 src_addr, __be16 src_port, __be32 dest_addr, __be16 dest_port); -int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry, +int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth, + struct mtk_foe_entry *entry, __be32 *src_addr, __be16 src_port, __be32 *dest_addr, __be16 dest_port); -int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port); -int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid); -int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid); -int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq, - int bss, int wcid); +int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int port); +int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int vid); +int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int sid); +int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry, + int wdma_idx, int txq, int bss, int wcid); int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry); -int mtk_ppe_debugfs_init(struct mtk_ppe *ppe); +int mtk_ppe_debugfs_init(struct mtk_ppe *ppe, int index); #endif diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c index eb0b598f14e4bc2b1f132cb48ad914a707ed71c6..391b071bcff38d2d3bec4530da31b9504228528e 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c @@ -79,7 +79,7 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind) int i; for (i = 0; i < MTK_PPE_ENTRIES; i++) { - struct mtk_foe_entry *entry = &ppe->foe_table[i]; + struct mtk_foe_entry *entry = mtk_foe_get_entry(ppe, i); struct mtk_foe_mac_info *l2; struct mtk_flow_addr_info ai = {}; unsigned char h_source[ETH_ALEN]; @@ -162,52 +162,28 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind) } static int -mtk_ppe_debugfs_foe_show_all(struct seq_file *m, void *private) +mtk_ppe_debugfs_foe_all_show(struct seq_file *m, void *private) { return mtk_ppe_debugfs_foe_show(m, private, false); } +DEFINE_SHOW_ATTRIBUTE(mtk_ppe_debugfs_foe_all); static int -mtk_ppe_debugfs_foe_show_bind(struct seq_file *m, void *private) +mtk_ppe_debugfs_foe_bind_show(struct seq_file *m, void *private) { return mtk_ppe_debugfs_foe_show(m, private, true); } +DEFINE_SHOW_ATTRIBUTE(mtk_ppe_debugfs_foe_bind); -static int -mtk_ppe_debugfs_foe_open_all(struct inode *inode, struct file *file) +int mtk_ppe_debugfs_init(struct mtk_ppe *ppe, int index) { - return single_open(file, mtk_ppe_debugfs_foe_show_all, - inode->i_private); -} - -static int -mtk_ppe_debugfs_foe_open_bind(struct inode *inode, struct file *file) -{ - return single_open(file, mtk_ppe_debugfs_foe_show_bind, - inode->i_private); -} - -int mtk_ppe_debugfs_init(struct mtk_ppe *ppe) -{ - static const struct file_operations fops_all = { - .open = mtk_ppe_debugfs_foe_open_all, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - }; - - static const struct file_operations fops_bind = { - .open = mtk_ppe_debugfs_foe_open_bind, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, - }; - struct dentry *root; - root = debugfs_create_dir("mtk_ppe", NULL); - debugfs_create_file("entries", S_IRUGO, root, ppe, &fops_all); - debugfs_create_file("bind", S_IRUGO, root, ppe, &fops_bind); + snprintf(ppe->dirname, sizeof(ppe->dirname), "ppe%d", index); + + root = debugfs_create_dir(ppe->dirname, NULL); + debugfs_create_file("entries", S_IRUGO, root, ppe, &mtk_ppe_debugfs_foe_all_fops); + debugfs_create_file("bind", S_IRUGO, root, ppe, &mtk_ppe_debugfs_foe_bind_fops); return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c index 25dc3c3aa31d943b5a6c4ad5cb244e1656e9c1c0..28bbd1df3e3059c3356f2b1d704602abf1ede7fd 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c @@ -52,18 +52,19 @@ static const struct rhashtable_params mtk_flow_ht_params = { }; static int -mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data, - bool egress) +mtk_flow_set_ipv4_addr(struct mtk_eth *eth, struct mtk_foe_entry *foe, + struct mtk_flow_data *data, bool egress) { - return mtk_foe_entry_set_ipv4_tuple(foe, egress, + return mtk_foe_entry_set_ipv4_tuple(eth, foe, egress, data->v4.src_addr, data->src_port, data->v4.dst_addr, data->dst_port); } static int -mtk_flow_set_ipv6_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data) +mtk_flow_set_ipv6_addr(struct mtk_eth *eth, struct mtk_foe_entry *foe, + struct mtk_flow_data *data) { - return mtk_foe_entry_set_ipv6_tuple(foe, + return mtk_foe_entry_set_ipv6_tuple(eth, foe, data->v6.src_addr.s6_addr32, data->src_port, data->v6.dst_addr.s6_addr32, data->dst_port); } @@ -173,7 +174,7 @@ mtk_flow_get_dsa_port(struct net_device **dev) if (dp->cpu_dp->tag_ops->proto != DSA_TAG_PROTO_MTK) return -ENODEV; - *dev = dp->cpu_dp->master; + *dev = dsa_port_to_master(dp); return dp->index; #else @@ -190,16 +191,29 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, int pse_port, dsa_port; if (mtk_flow_get_wdma_info(dev, dest_mac, &info) == 0) { - mtk_foe_entry_set_wdma(foe, info.wdma_idx, info.queue, info.bss, - info.wcid); - pse_port = 3; + mtk_foe_entry_set_wdma(eth, foe, info.wdma_idx, info.queue, + info.bss, info.wcid); + if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) { + switch (info.wdma_idx) { + case 0: + pse_port = 8; + break; + case 1: + pse_port = 9; + break; + default: + return -EINVAL; + } + } else { + pse_port = 3; + } *wed_index = info.wdma_idx; goto out; } dsa_port = mtk_flow_get_dsa_port(&dev); if (dsa_port >= 0) - mtk_foe_entry_set_dsa(foe, dsa_port); + mtk_foe_entry_set_dsa(eth, foe, dsa_port); if (dev == eth->netdev[0]) pse_port = 1; @@ -209,7 +223,7 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe, return -EOPNOTSUPP; out: - mtk_foe_entry_set_pse_port(foe, pse_port); + mtk_foe_entry_set_pse_port(eth, foe, pse_port); return 0; } @@ -333,9 +347,8 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) !is_valid_ether_addr(data.eth.h_dest)) return -EINVAL; - err = mtk_foe_entry_prepare(&foe, offload_type, l4proto, 0, - data.eth.h_source, - data.eth.h_dest); + err = mtk_foe_entry_prepare(eth, &foe, offload_type, l4proto, 0, + data.eth.h_source, data.eth.h_dest); if (err) return err; @@ -360,7 +373,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) data.v4.src_addr = addrs.key->src; data.v4.dst_addr = addrs.key->dst; - mtk_flow_set_ipv4_addr(&foe, &data, false); + mtk_flow_set_ipv4_addr(eth, &foe, &data, false); } if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { @@ -371,7 +384,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) data.v6.src_addr = addrs.key->src; data.v6.dst_addr = addrs.key->dst; - mtk_flow_set_ipv6_addr(&foe, &data); + mtk_flow_set_ipv6_addr(eth, &foe, &data); } flow_action_for_each(i, act, &rule->action) { @@ -401,7 +414,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) } if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { - err = mtk_flow_set_ipv4_addr(&foe, &data, true); + err = mtk_flow_set_ipv4_addr(eth, &foe, &data, true); if (err) return err; } @@ -413,10 +426,10 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) if (data.vlan.proto != htons(ETH_P_8021Q)) return -EOPNOTSUPP; - mtk_foe_entry_set_vlan(&foe, data.vlan.id); + mtk_foe_entry_set_vlan(eth, &foe, data.vlan.id); } if (data.pppoe.num == 1) - mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid); + mtk_foe_entry_set_pppoe(eth, &foe, data.pppoe.sid); err = mtk_flow_set_output_device(eth, &foe, odev, data.eth.h_dest, &wed_index); @@ -434,7 +447,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) memcpy(&entry->data, &foe, sizeof(entry->data)); entry->wed_index = wed_index; - err = mtk_foe_entry_commit(eth->ppe, entry); + err = mtk_foe_entry_commit(eth->ppe[entry->ppe_index], entry); if (err < 0) goto free; @@ -446,7 +459,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f) return 0; clear: - mtk_foe_entry_clear(eth->ppe, entry); + mtk_foe_entry_clear(eth->ppe[entry->ppe_index], entry); free: kfree(entry); if (wed_index >= 0) @@ -464,7 +477,7 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f) if (!entry) return -ENOENT; - mtk_foe_entry_clear(eth->ppe, entry); + mtk_foe_entry_clear(eth->ppe[entry->ppe_index], entry); rhashtable_remove_fast(ð->flow_table, &entry->node, mtk_flow_ht_params); if (entry->wed_index >= 0) @@ -485,7 +498,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f) if (!entry) return -ENOENT; - idle = mtk_foe_entry_idle_time(eth->ppe, entry); + idle = mtk_foe_entry_idle_time(eth->ppe[entry->ppe_index], entry); f->stats.lastused = jiffies - idle * HZ; return 0; @@ -537,7 +550,7 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f) struct flow_block_cb *block_cb; flow_setup_cb_t *cb; - if (!eth->ppe || !eth->ppe->foe_table) + if (!eth->soc->offload_version) return -EOPNOTSUPP; if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) @@ -589,8 +602,5 @@ int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type, int mtk_eth_offload_init(struct mtk_eth *eth) { - if (!eth->ppe || !eth->ppe->foe_table) - return 0; - return rhashtable_init(ð->flow_table, &mtk_flow_ht_params); } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h index 0c45ea0900f16aba01e4e1d2422e0519b825c5f9..59596d823d8b8a36ed63bdb5b4f882e74a2d4b8f 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h @@ -21,6 +21,9 @@ #define MTK_PPE_GLO_CFG_BUSY BIT(31) #define MTK_PPE_FLOW_CFG 0x204 +#define MTK_PPE_MD_TOAP_BYP_CRSN0 BIT(1) +#define MTK_PPE_MD_TOAP_BYP_CRSN1 BIT(2) +#define MTK_PPE_MD_TOAP_BYP_CRSN2 BIT(3) #define MTK_PPE_FLOW_CFG_IP4_TCP_FRAG BIT(6) #define MTK_PPE_FLOW_CFG_IP4_UDP_FRAG BIT(7) #define MTK_PPE_FLOW_CFG_IP6_3T_ROUTE BIT(8) @@ -54,6 +57,7 @@ #define MTK_PPE_TB_CFG_HASH_MODE GENMASK(15, 14) #define MTK_PPE_TB_CFG_SCAN_MODE GENMASK(17, 16) #define MTK_PPE_TB_CFG_HASH_DEBUG GENMASK(19, 18) +#define MTK_PPE_TB_CFG_INFO_SEL BIT(20) enum { MTK_PPE_SCAN_MODE_DISABLED, @@ -112,6 +116,8 @@ enum { #define MTK_PPE_DEFAULT_CPU_PORT 0x248 #define MTK_PPE_DEFAULT_CPU_PORT_MASK(_n) (GENMASK(2, 0) << ((_n) * 4)) +#define MTK_PPE_DEFAULT_CPU_PORT1 0x24c + #define MTK_PPE_MTU_DROP 0x308 #define MTK_PPE_VLAN_MTU0 0x30c @@ -141,4 +147,6 @@ enum { #define MTK_PPE_MIB_CACHE_CTL_EN BIT(0) #define MTK_PPE_MIB_CACHE_CTL_FLUSH BIT(2) +#define MTK_PPE_SBW_CTRL 0x374 + #endif diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c index 3f0e5e64de50595cedfa4c54e79dcb71e947679e..7e890f81148e6a8f7c625b4399d9fa44f8e81c0f 100644 --- a/drivers/net/ethernet/mediatek/mtk_star_emac.c +++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c @@ -1255,7 +1255,7 @@ static const struct net_device_ops mtk_star_netdev_ops = { static void mtk_star_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, MTK_STAR_DRVNAME, sizeof(info->driver)); + strscpy(info->driver, MTK_STAR_DRVNAME, sizeof(info->driver)); } /* TODO Add ethtool stats. */ @@ -1651,8 +1651,7 @@ static int mtk_star_probe(struct platform_device *pdev) ndev->netdev_ops = &mtk_star_netdev_ops; ndev->ethtool_ops = &mtk_star_ethtool_ops; - netif_napi_add(ndev, &priv->rx_napi, mtk_star_rx_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->rx_napi, mtk_star_rx_poll); netif_napi_add_tx(ndev, &priv->tx_napi, mtk_star_tx_poll); return devm_register_netdev(dev, ndev); diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index 29be2fcafea3bc3907fe5b90a8bdb6e7e3997578..099b6e0df619a9cd580d69058504e2a32393a529 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -25,6 +25,11 @@ #define MTK_WED_TX_RING_SIZE 2048 #define MTK_WED_WDMA_RING_SIZE 1024 +#define MTK_WED_MAX_GROUP_SIZE 0x100 +#define MTK_WED_VLD_GROUP_SIZE 0x40 +#define MTK_WED_PER_GROUP_PKT 128 + +#define MTK_WED_FBUF_SIZE 128 static struct mtk_wed_hw *hw_list[2]; static DEFINE_MUTEX(hw_lock); @@ -80,11 +85,31 @@ static struct mtk_wed_hw * mtk_wed_assign(struct mtk_wed_device *dev) { struct mtk_wed_hw *hw; + int i; + + if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { + hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)]; + if (!hw) + return NULL; + + if (!hw->wed_dev) + goto out; + + if (hw->version == 1) + return NULL; + + /* MT7986 WED devices do not have any pcie slot restrictions */ + } + /* MT7986 PCIE or AXI */ + for (i = 0; i < ARRAY_SIZE(hw_list); i++) { + hw = hw_list[i]; + if (hw && !hw->wed_dev) + goto out; + } - hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)]; - if (!hw || hw->wed_dev) - return NULL; + return NULL; +out: hw->wed_dev = dev; return hw; } @@ -150,10 +175,17 @@ mtk_wed_buffer_alloc(struct mtk_wed_device *dev) desc->buf0 = cpu_to_le32(buf_phys); desc->buf1 = cpu_to_le32(buf_phys + txd_size); - ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) | - FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1, - MTK_WED_BUF_SIZE - txd_size) | - MTK_WDMA_DESC_CTRL_LAST_SEG1; + + if (dev->hw->version == 1) + ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) | + FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1, + MTK_WED_BUF_SIZE - txd_size) | + MTK_WDMA_DESC_CTRL_LAST_SEG1; + else + ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) | + FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2, + MTK_WED_BUF_SIZE - txd_size) | + MTK_WDMA_DESC_CTRL_LAST_SEG0; desc->ctrl = cpu_to_le32(ctrl); desc->info = 0; desc++; @@ -209,7 +241,7 @@ mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring) if (!ring->desc) return; - dma_free_coherent(dev->hw->dev, ring->size * sizeof(*ring->desc), + dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size, ring->desc, ring->desc_phys); } @@ -229,6 +261,14 @@ mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en) { u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK; + if (dev->hw->version == 1) + mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR; + else + mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH | + MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH | + MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT | + MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR; + if (!dev->hw->num_flows) mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; @@ -236,10 +276,55 @@ mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en) wed_r32(dev, MTK_WED_EXT_INT_MASK); } +static void +mtk_wed_set_512_support(struct mtk_wed_device *dev, bool enable) +{ + if (enable) { + wed_w32(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR); + wed_w32(dev, MTK_WED_TXP_DW1, + FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0103)); + } else { + wed_w32(dev, MTK_WED_TXP_DW1, + FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0100)); + wed_clr(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR); + } +} + +static void +mtk_wed_dma_disable(struct mtk_wed_device *dev) +{ + wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | + MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); + + wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); + + wed_clr(dev, MTK_WED_GLO_CFG, + MTK_WED_GLO_CFG_TX_DMA_EN | + MTK_WED_GLO_CFG_RX_DMA_EN); + + wdma_m32(dev, MTK_WDMA_GLO_CFG, + MTK_WDMA_GLO_CFG_TX_DMA_EN | + MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | + MTK_WDMA_GLO_CFG_RX_INFO2_PRERES, 0); + + if (dev->hw->version == 1) { + regmap_write(dev->hw->mirror, dev->hw->index * 4, 0); + wdma_m32(dev, MTK_WDMA_GLO_CFG, + MTK_WDMA_GLO_CFG_RX_INFO3_PRERES, 0); + } else { + wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC | + MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC); + + mtk_wed_set_512_support(dev, false); + } +} + static void mtk_wed_stop(struct mtk_wed_device *dev) { - regmap_write(dev->hw->mirror, dev->hw->index * 4, 0); + mtk_wed_dma_disable(dev); mtk_wed_set_ext_int(dev, false); wed_clr(dev, MTK_WED_CTRL, @@ -252,21 +337,11 @@ mtk_wed_stop(struct mtk_wed_device *dev) wdma_w32(dev, MTK_WDMA_INT_MASK, 0); wdma_w32(dev, MTK_WDMA_INT_GRP2, 0); wed_w32(dev, MTK_WED_WPDMA_INT_MASK, 0); - - wed_clr(dev, MTK_WED_GLO_CFG, - MTK_WED_GLO_CFG_TX_DMA_EN | - MTK_WED_GLO_CFG_RX_DMA_EN); - wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, - MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN | - MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN); - wed_clr(dev, MTK_WED_WDMA_GLO_CFG, - MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); } static void mtk_wed_detach(struct mtk_wed_device *dev) { - struct device_node *wlan_node = dev->wlan.pci_dev->dev.of_node; struct mtk_wed_hw *hw = dev->hw; mutex_lock(&hw_lock); @@ -281,9 +356,14 @@ mtk_wed_detach(struct mtk_wed_device *dev) mtk_wed_free_buffer(dev); mtk_wed_free_tx_rings(dev); - if (of_dma_is_coherent(wlan_node)) - regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, - BIT(hw->index), BIT(hw->index)); + if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { + struct device_node *wlan_node; + + wlan_node = dev->wlan.pci_dev->dev.of_node; + if (of_dma_is_coherent(wlan_node) && hw->hifsys) + regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, + BIT(hw->index), BIT(hw->index)); + } if (!hw_list[!hw->index]->wed_dev && hw->eth->dma_dev != hw->eth->dev) @@ -296,14 +376,76 @@ mtk_wed_detach(struct mtk_wed_device *dev) mutex_unlock(&hw_lock); } +#define PCIE_BASE_ADDR0 0x11280000 +static void +mtk_wed_bus_init(struct mtk_wed_device *dev) +{ + switch (dev->wlan.bus_type) { + case MTK_WED_BUS_PCIE: { + struct device_node *np = dev->hw->eth->dev->of_node; + struct regmap *regs; + + regs = syscon_regmap_lookup_by_phandle(np, + "mediatek,wed-pcie"); + if (IS_ERR(regs)) + break; + + regmap_update_bits(regs, 0, BIT(0), BIT(0)); + + wed_w32(dev, MTK_WED_PCIE_INT_CTRL, + FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 2)); + + /* pcie interrupt control: pola/source selection */ + wed_set(dev, MTK_WED_PCIE_INT_CTRL, + MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA | + FIELD_PREP(MTK_WED_PCIE_INT_CTRL_SRC_SEL, 1)); + wed_r32(dev, MTK_WED_PCIE_INT_CTRL); + + wed_w32(dev, MTK_WED_PCIE_CFG_INTM, PCIE_BASE_ADDR0 | 0x180); + wed_w32(dev, MTK_WED_PCIE_CFG_BASE, PCIE_BASE_ADDR0 | 0x184); + + /* pcie interrupt status trigger register */ + wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(24)); + wed_r32(dev, MTK_WED_PCIE_INT_TRIGGER); + + /* pola setting */ + wed_set(dev, MTK_WED_PCIE_INT_CTRL, + MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA); + break; + } + case MTK_WED_BUS_AXI: + wed_set(dev, MTK_WED_WPDMA_INT_CTRL, + MTK_WED_WPDMA_INT_CTRL_SIG_SRC | + FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_SRC_SEL, 0)); + break; + default: + break; + } +} + +static void +mtk_wed_set_wpdma(struct mtk_wed_device *dev) +{ + if (dev->hw->version == 1) { + wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys); + } else { + mtk_wed_bus_init(dev); + + wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_int); + wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK, dev->wlan.wpdma_mask); + wed_w32(dev, MTK_WED_WPDMA_CFG_TX, dev->wlan.wpdma_tx); + wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE, dev->wlan.wpdma_txfree); + } +} + static void mtk_wed_hw_init_early(struct mtk_wed_device *dev) { u32 mask, set; - u32 offset; mtk_wed_stop(dev); mtk_wed_reset(dev, MTK_WED_RESET_WED); + mtk_wed_set_wpdma(dev); mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE | MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE | @@ -313,14 +455,33 @@ mtk_wed_hw_init_early(struct mtk_wed_device *dev) MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY; wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set); - wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_INFO_PRERES); + if (dev->hw->version == 1) { + u32 offset = dev->hw->index ? 0x04000400 : 0; - offset = dev->hw->index ? 0x04000400 : 0; - wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset); - wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset); + wdma_set(dev, MTK_WDMA_GLO_CFG, + MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | + MTK_WDMA_GLO_CFG_RX_INFO2_PRERES | + MTK_WDMA_GLO_CFG_RX_INFO3_PRERES); - wed_w32(dev, MTK_WED_PCIE_CFG_BASE, MTK_PCIE_BASE(dev->hw->index)); - wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys); + wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset); + wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset); + wed_w32(dev, MTK_WED_PCIE_CFG_BASE, + MTK_PCIE_BASE(dev->hw->index)); + } else { + wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy); + wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_ETH_DMAD_FMT); + wed_w32(dev, MTK_WED_WDMA_OFFSET0, + FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_INTS, + MTK_WDMA_INT_STATUS) | + FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_CFG, + MTK_WDMA_GLO_CFG)); + + wed_w32(dev, MTK_WED_WDMA_OFFSET1, + FIELD_PREP(MTK_WED_WDMA_OFST1_TX_CTRL, + MTK_WDMA_RING_TX(0)) | + FIELD_PREP(MTK_WED_WDMA_OFST1_RX_CTRL, + MTK_WDMA_RING_RX(0))); + } } static void @@ -340,37 +501,65 @@ mtk_wed_hw_init(struct mtk_wed_device *dev) wed_w32(dev, MTK_WED_TX_BM_BASE, dev->buf_ring.desc_phys); - wed_w32(dev, MTK_WED_TX_BM_TKID, - FIELD_PREP(MTK_WED_TX_BM_TKID_START, - dev->wlan.token_start) | - FIELD_PREP(MTK_WED_TX_BM_TKID_END, - dev->wlan.token_start + dev->wlan.nbuf - 1)); - wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE); - wed_w32(dev, MTK_WED_TX_BM_DYN_THR, - FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) | - MTK_WED_TX_BM_DYN_THR_HI); + if (dev->hw->version == 1) { + wed_w32(dev, MTK_WED_TX_BM_TKID, + FIELD_PREP(MTK_WED_TX_BM_TKID_START, + dev->wlan.token_start) | + FIELD_PREP(MTK_WED_TX_BM_TKID_END, + dev->wlan.token_start + + dev->wlan.nbuf - 1)); + wed_w32(dev, MTK_WED_TX_BM_DYN_THR, + FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) | + MTK_WED_TX_BM_DYN_THR_HI); + } else { + wed_w32(dev, MTK_WED_TX_BM_TKID_V2, + FIELD_PREP(MTK_WED_TX_BM_TKID_START, + dev->wlan.token_start) | + FIELD_PREP(MTK_WED_TX_BM_TKID_END, + dev->wlan.token_start + + dev->wlan.nbuf - 1)); + wed_w32(dev, MTK_WED_TX_BM_DYN_THR, + FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO_V2, 0) | + MTK_WED_TX_BM_DYN_THR_HI_V2); + wed_w32(dev, MTK_WED_TX_TKID_CTRL, + MTK_WED_TX_TKID_CTRL_PAUSE | + FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM, + dev->buf_ring.size / 128) | + FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM, + dev->buf_ring.size / 128)); + wed_w32(dev, MTK_WED_TX_TKID_DYN_THR, + FIELD_PREP(MTK_WED_TX_TKID_DYN_THR_LO, 0) | + MTK_WED_TX_TKID_DYN_THR_HI); + } mtk_wed_reset(dev, MTK_WED_RESET_TX_BM); - wed_set(dev, MTK_WED_CTRL, - MTK_WED_CTRL_WED_TX_BM_EN | - MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); + if (dev->hw->version == 1) + wed_set(dev, MTK_WED_CTRL, + MTK_WED_CTRL_WED_TX_BM_EN | + MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); + else + wed_clr(dev, MTK_WED_TX_TKID_CTRL, MTK_WED_TX_TKID_CTRL_PAUSE); wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE); } static void -mtk_wed_ring_reset(struct mtk_wdma_desc *desc, int size) +mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size) { + void *head = (void *)ring->desc; int i; for (i = 0; i < size; i++) { - desc[i].buf0 = 0; - desc[i].ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); - desc[i].buf1 = 0; - desc[i].info = 0; + struct mtk_wdma_desc *desc; + + desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size); + desc->buf0 = 0; + desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); + desc->buf1 = 0; + desc->info = 0; } } @@ -421,12 +610,10 @@ mtk_wed_reset_dma(struct mtk_wed_device *dev) int i; for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) { - struct mtk_wdma_desc *desc = dev->tx_ring[i].desc; - - if (!desc) + if (!dev->tx_ring[i].desc) continue; - mtk_wed_ring_reset(desc, MTK_WED_TX_RING_SIZE); + mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE); } if (mtk_wed_poll_busy(dev)) @@ -483,16 +670,16 @@ mtk_wed_reset_dma(struct mtk_wed_device *dev) static int mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring, - int size) + int size, u32 desc_size) { - ring->desc = dma_alloc_coherent(dev->hw->dev, - size * sizeof(*ring->desc), + ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size, &ring->desc_phys, GFP_KERNEL); if (!ring->desc) return -ENOMEM; + ring->desc_size = desc_size; ring->size = size; - mtk_wed_ring_reset(ring->desc, size); + mtk_wed_ring_reset(ring, size); return 0; } @@ -500,9 +687,10 @@ mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring, static int mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size) { + u32 desc_size = sizeof(struct mtk_wdma_desc) * dev->hw->version; struct mtk_wed_ring *wdma = &dev->tx_wdma[idx]; - if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE)) + if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, desc_size)) return -ENOMEM; wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE, @@ -520,43 +708,63 @@ mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size) } static void -mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask) +mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask) { - u32 wdma_mask; - u32 val; - int i; - - for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) - if (!dev->tx_wdma[i].desc) - mtk_wed_wdma_ring_setup(dev, i, 16); - - wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0)); - - mtk_wed_hw_init(dev); + u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0)); + /* wed control cr set */ wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WDMA_INT_AGENT_EN | MTK_WED_CTRL_WPDMA_INT_AGENT_EN | MTK_WED_CTRL_WED_TX_BM_EN | MTK_WED_CTRL_WED_TX_FREE_AGENT_EN); - wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, MTK_WED_PCIE_INT_TRIGGER_STATUS); + if (dev->hw->version == 1) { + wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, + MTK_WED_PCIE_INT_TRIGGER_STATUS); - wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, - MTK_WED_WPDMA_INT_TRIGGER_RX_DONE | - MTK_WED_WPDMA_INT_TRIGGER_TX_DONE); + wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, + MTK_WED_WPDMA_INT_TRIGGER_RX_DONE | + MTK_WED_WPDMA_INT_TRIGGER_TX_DONE); - wed_set(dev, MTK_WED_WPDMA_INT_CTRL, - MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV); + wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask); + } else { + /* initail tx interrupt trigger */ + wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX, + MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN | + MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR | + MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN | + MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR | + FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG, + dev->wlan.tx_tbit[0]) | + FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG, + dev->wlan.tx_tbit[1])); + + /* initail txfree interrupt trigger */ + wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX_FREE, + MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN | + MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR | + FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG, + dev->wlan.txfree_tbit)); + + wed_w32(dev, MTK_WED_WDMA_INT_CLR, wdma_mask); + wed_set(dev, MTK_WED_WDMA_INT_CTRL, + FIELD_PREP(MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL, + dev->wdma_idx)); + } wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask); - wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask); wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask); wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask); - wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask); wed_w32(dev, MTK_WED_INT_MASK, irq_mask); +} + +static void +mtk_wed_dma_enable(struct mtk_wed_device *dev) +{ + wed_set(dev, MTK_WED_WPDMA_INT_CTRL, MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV); wed_set(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_TX_DMA_EN | @@ -567,16 +775,54 @@ mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask) wed_set(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN); + wdma_set(dev, MTK_WDMA_GLO_CFG, + MTK_WDMA_GLO_CFG_TX_DMA_EN | + MTK_WDMA_GLO_CFG_RX_INFO1_PRERES | + MTK_WDMA_GLO_CFG_RX_INFO2_PRERES); + + if (dev->hw->version == 1) { + wdma_set(dev, MTK_WDMA_GLO_CFG, + MTK_WDMA_GLO_CFG_RX_INFO3_PRERES); + } else { + wed_set(dev, MTK_WED_WPDMA_CTRL, + MTK_WED_WPDMA_CTRL_SDL1_FIXED); + + wed_set(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC | + MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC); + + wed_clr(dev, MTK_WED_WPDMA_GLO_CFG, + MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP | + MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV); + } +} + +static void +mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) + if (!dev->tx_wdma[i].desc) + mtk_wed_wdma_ring_setup(dev, i, 16); + + mtk_wed_hw_init(dev); + mtk_wed_configure_irq(dev, irq_mask); + mtk_wed_set_ext_int(dev, true); - val = dev->wlan.wpdma_phys | - MTK_PCIE_MIRROR_MAP_EN | - FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, dev->hw->index); - if (dev->hw->index) - val |= BIT(1); - val |= BIT(0); - regmap_write(dev->hw->mirror, dev->hw->index * 4, val); + if (dev->hw->version == 1) { + u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN | + FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, + dev->hw->index); + + val |= BIT(0) | (BIT(1) * !!dev->hw->index); + regmap_write(dev->hw->mirror, dev->hw->index * 4, val); + } else { + mtk_wed_set_512_support(dev, true); + } + mtk_wed_dma_enable(dev); dev->running = true; } @@ -585,12 +831,14 @@ mtk_wed_attach(struct mtk_wed_device *dev) __releases(RCU) { struct mtk_wed_hw *hw; + struct device *device; int ret = 0; RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "mtk_wed_attach without holding the RCU read lock"); - if (pci_domain_nr(dev->wlan.pci_dev->bus) > 1 || + if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE && + pci_domain_nr(dev->wlan.pci_dev->bus) > 1) || !try_module_get(THIS_MODULE)) ret = -ENODEV; @@ -608,7 +856,11 @@ mtk_wed_attach(struct mtk_wed_device *dev) goto out; } - dev_info(&dev->wlan.pci_dev->dev, "attaching wed device %d\n", hw->index); + device = dev->wlan.bus_type == MTK_WED_BUS_PCIE + ? &dev->wlan.pci_dev->dev + : &dev->wlan.platform_dev->dev; + dev_info(device, "attaching wed device %d version %d\n", + hw->index, hw->version); dev->hw = hw; dev->dev = hw->dev; @@ -626,7 +878,9 @@ mtk_wed_attach(struct mtk_wed_device *dev) } mtk_wed_hw_init_early(dev); - regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, BIT(hw->index), 0); + if (hw->hifsys) + regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, + BIT(hw->index), 0); out: mutex_unlock(&hw_lock); @@ -653,7 +907,8 @@ mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs) BUG_ON(idx >= ARRAY_SIZE(dev->tx_ring)); - if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE)) + if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE, + sizeof(*ring->desc))) return -ENOMEM; if (mtk_wed_wdma_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE)) @@ -680,21 +935,21 @@ static int mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) { struct mtk_wed_ring *ring = &dev->txfree_ring; - int i; + int i, index = dev->hw->version == 1; /* * For txfree event handling, the same DMA ring is shared between WED * and WLAN. The WLAN driver accesses the ring index registers through * WED */ - ring->reg_base = MTK_WED_RING_RX(1); + ring->reg_base = MTK_WED_RING_RX(index); ring->wpdma = regs; for (i = 0; i < 12; i += 4) { u32 val = readl(regs + i); - wed_w32(dev, MTK_WED_RING_RX(1) + i, val); - wed_w32(dev, MTK_WED_WPDMA_RING_RX(1) + i, val); + wed_w32(dev, MTK_WED_RING_RX(index) + i, val); + wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val); } return 0; @@ -703,11 +958,19 @@ mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs) static u32 mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask) { - u32 val; + u32 val, ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK; + + if (dev->hw->version == 1) + ext_mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR; + else + ext_mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH | + MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH | + MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT | + MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR; val = wed_r32(dev, MTK_WED_EXT_INT_STATUS); wed_w32(dev, MTK_WED_EXT_INT_STATUS, val); - val &= MTK_WED_EXT_INT_STATUS_ERROR_MASK; + val &= ext_mask; if (!dev->hw->num_flows) val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD; if (val && net_ratelimit()) @@ -782,7 +1045,8 @@ out: } void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, - void __iomem *wdma, int index) + void __iomem *wdma, phys_addr_t wdma_phy, + int index) { static const struct mtk_wed_ops wed_ops = { .attach = mtk_wed_attach, @@ -829,26 +1093,33 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, hw = kzalloc(sizeof(*hw), GFP_KERNEL); if (!hw) goto unlock; + hw->node = np; hw->regs = regs; hw->eth = eth; hw->dev = &pdev->dev; + hw->wdma_phy = wdma_phy; hw->wdma = wdma; hw->index = index; hw->irq = irq; - hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, - "mediatek,pcie-mirror"); - hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np, - "mediatek,hifsys"); - if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) { - kfree(hw); - goto unlock; - } + hw->version = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1; + + if (hw->version == 1) { + hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, + "mediatek,pcie-mirror"); + hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np, + "mediatek,hifsys"); + if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) { + kfree(hw); + goto unlock; + } - if (!index) { - regmap_write(hw->mirror, 0, 0); - regmap_write(hw->mirror, 4, 0); + if (!index) { + regmap_write(hw->mirror, 0, 0); + regmap_write(hw->mirror, 4, 0); + } } + mtk_wed_hw_add_debugfs(hw); hw_list[index] = hw; diff --git a/drivers/net/ethernet/mediatek/mtk_wed.h b/drivers/net/ethernet/mediatek/mtk_wed.h index 981ec613f4b0b589ea67a4d076eba5ed095db8e3..ae420ca01a488047fe99978df54c22d68fd7427a 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.h +++ b/drivers/net/ethernet/mediatek/mtk_wed.h @@ -18,11 +18,13 @@ struct mtk_wed_hw { struct regmap *hifsys; struct device *dev; void __iomem *wdma; + phys_addr_t wdma_phy; struct regmap *mirror; struct dentry *debugfs_dir; struct mtk_wed_device *wed_dev; u32 debugfs_reg; u32 num_flows; + u8 version; char dirname[5]; int irq; int index; @@ -101,14 +103,16 @@ wpdma_txfree_w32(struct mtk_wed_device *dev, u32 reg, u32 val) } void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, - void __iomem *wdma, int index); + void __iomem *wdma, phys_addr_t wdma_phy, + int index); void mtk_wed_exit(void); int mtk_wed_flow_add(int index); void mtk_wed_flow_remove(int index); #else static inline void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, - void __iomem *wdma, int index) + void __iomem *wdma, phys_addr_t wdma_phy, + int index) { } static inline void diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c index a81d3fd1a439de0f7da8ab644cb2258a89d79269..f420f187e837352afb05ef9a2df42e1ed340c138 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c +++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c @@ -116,6 +116,9 @@ wed_txinfo_show(struct seq_file *s, void *data) DUMP_WDMA(WDMA_GLO_CFG), DUMP_WDMA_RING(WDMA_RING_RX(0)), DUMP_WDMA_RING(WDMA_RING_RX(1)), + + DUMP_STR("TX FREE"), + DUMP_WED(WED_RX_MIB(0)), }; struct mtk_wed_hw *hw = s->private; struct mtk_wed_device *dev = hw->wed_dev; diff --git a/drivers/net/ethernet/mediatek/mtk_wed_regs.h b/drivers/net/ethernet/mediatek/mtk_wed_regs.h index 0a0465ea58b4682098a3d9e0e7f3d5bcec360d4f..e270fb33614320dbcfa9eb1a355d853084367ec9 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed_regs.h +++ b/drivers/net/ethernet/mediatek/mtk_wed_regs.h @@ -5,6 +5,7 @@ #define __MTK_WED_REGS_H #define MTK_WDMA_DESC_CTRL_LEN1 GENMASK(14, 0) +#define MTK_WDMA_DESC_CTRL_LEN1_V2 GENMASK(13, 0) #define MTK_WDMA_DESC_CTRL_LAST_SEG1 BIT(15) #define MTK_WDMA_DESC_CTRL_BURST BIT(16) #define MTK_WDMA_DESC_CTRL_LEN0 GENMASK(29, 16) @@ -41,6 +42,7 @@ struct mtk_wdma_desc { #define MTK_WED_CTRL_RESERVE_EN BIT(12) #define MTK_WED_CTRL_RESERVE_BUSY BIT(13) #define MTK_WED_CTRL_FINAL_DIDX_READ BIT(24) +#define MTK_WED_CTRL_ETH_DMAD_FMT BIT(25) #define MTK_WED_CTRL_MIB_READ_CLEAR BIT(28) #define MTK_WED_EXT_INT_STATUS 0x020 @@ -57,7 +59,8 @@ struct mtk_wdma_desc { #define MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN BIT(19) #define MTK_WED_EXT_INT_STATUS_RX_DRV_BM_DMAD_COHERENT BIT(20) #define MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR BIT(21) -#define MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR BIT(22) +#define MTK_WED_EXT_INT_STATUS_TX_DMA_R_RESP_ERR BIT(22) +#define MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR BIT(23) #define MTK_WED_EXT_INT_STATUS_RX_DRV_DMA_RECYCLE BIT(24) #define MTK_WED_EXT_INT_STATUS_ERROR_MASK (MTK_WED_EXT_INT_STATUS_TF_LEN_ERR | \ MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD | \ @@ -65,8 +68,7 @@ struct mtk_wdma_desc { MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR | \ MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR | \ MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN | \ - MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR | \ - MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR) + MTK_WED_EXT_INT_STATUS_TX_DMA_R_RESP_ERR) #define MTK_WED_EXT_INT_MASK 0x028 @@ -81,6 +83,7 @@ struct mtk_wdma_desc { #define MTK_WED_TX_BM_BASE 0x084 #define MTK_WED_TX_BM_TKID 0x088 +#define MTK_WED_TX_BM_TKID_V2 0x0c8 #define MTK_WED_TX_BM_TKID_START GENMASK(15, 0) #define MTK_WED_TX_BM_TKID_END GENMASK(31, 16) @@ -94,7 +97,25 @@ struct mtk_wdma_desc { #define MTK_WED_TX_BM_DYN_THR 0x0a0 #define MTK_WED_TX_BM_DYN_THR_LO GENMASK(6, 0) +#define MTK_WED_TX_BM_DYN_THR_LO_V2 GENMASK(8, 0) #define MTK_WED_TX_BM_DYN_THR_HI GENMASK(22, 16) +#define MTK_WED_TX_BM_DYN_THR_HI_V2 GENMASK(24, 16) + +#define MTK_WED_TX_TKID_CTRL 0x0c0 +#define MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM GENMASK(6, 0) +#define MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM GENMASK(22, 16) +#define MTK_WED_TX_TKID_CTRL_PAUSE BIT(28) + +#define MTK_WED_TX_TKID_DYN_THR 0x0e0 +#define MTK_WED_TX_TKID_DYN_THR_LO GENMASK(6, 0) +#define MTK_WED_TX_TKID_DYN_THR_HI GENMASK(22, 16) + +#define MTK_WED_TXP_DW0 0x120 +#define MTK_WED_TXP_DW1 0x124 +#define MTK_WED_WPDMA_WRITE_TXP GENMASK(31, 16) +#define MTK_WED_TXDP_CTRL 0x130 +#define MTK_WED_TXDP_DW9_OVERWR BIT(9) +#define MTK_WED_RX_BM_TKID_MIB 0x1cc #define MTK_WED_INT_STATUS 0x200 #define MTK_WED_INT_MASK 0x204 @@ -125,6 +146,7 @@ struct mtk_wdma_desc { #define MTK_WED_RESET_IDX_RX GENMASK(17, 16) #define MTK_WED_TX_MIB(_n) (0x2a0 + (_n) * 4) +#define MTK_WED_RX_MIB(_n) (0x2e0 + (_n) * 4) #define MTK_WED_RING_TX(_n) (0x300 + (_n) * 0x10) @@ -155,21 +177,64 @@ struct mtk_wdma_desc { #define MTK_WED_WPDMA_GLO_CFG_BYTE_SWAP BIT(29) #define MTK_WED_WPDMA_GLO_CFG_RX_2B_OFFSET BIT(31) +/* CONFIG_MEDIATEK_NETSYS_V2 */ +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC BIT(4) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R1_PKT_PROC BIT(5) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC BIT(6) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R1_CRX_SYNC BIT(7) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_EVENT_PKT_FMT_VER GENMASK(18, 16) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_UNSUPPORT_FMT BIT(19) +#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_UEVENT_PKT_FMT_CHK BIT(20) +#define MTK_WED_WPDMA_GLO_CFG_RX_DDONE2_WR BIT(21) +#define MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP BIT(24) +#define MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV BIT(28) + #define MTK_WED_WPDMA_RESET_IDX 0x50c #define MTK_WED_WPDMA_RESET_IDX_TX GENMASK(3, 0) #define MTK_WED_WPDMA_RESET_IDX_RX GENMASK(17, 16) +#define MTK_WED_WPDMA_CTRL 0x518 +#define MTK_WED_WPDMA_CTRL_SDL1_FIXED BIT(31) + #define MTK_WED_WPDMA_INT_CTRL 0x520 #define MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV BIT(21) +#define MTK_WED_WPDMA_INT_CTRL_SIG_SRC BIT(22) +#define MTK_WED_WPDMA_INT_CTRL_SRC_SEL GENMASK(17, 16) #define MTK_WED_WPDMA_INT_MASK 0x524 +#define MTK_WED_WPDMA_INT_CTRL_TX 0x530 +#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN BIT(0) +#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR BIT(1) +#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG GENMASK(6, 2) +#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN BIT(8) +#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR BIT(9) +#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG GENMASK(14, 10) + +#define MTK_WED_WPDMA_INT_CTRL_RX 0x534 + +#define MTK_WED_WPDMA_INT_CTRL_TX_FREE 0x538 +#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN BIT(0) +#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR BIT(1) +#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG GENMASK(6, 2) + #define MTK_WED_PCIE_CFG_BASE 0x560 +#define MTK_WED_PCIE_CFG_BASE 0x560 +#define MTK_WED_PCIE_CFG_INTM 0x564 +#define MTK_WED_PCIE_CFG_MSIS 0x568 #define MTK_WED_PCIE_INT_TRIGGER 0x570 #define MTK_WED_PCIE_INT_TRIGGER_STATUS BIT(16) +#define MTK_WED_PCIE_INT_CTRL 0x57c +#define MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA BIT(20) +#define MTK_WED_PCIE_INT_CTRL_SRC_SEL GENMASK(17, 16) +#define MTK_WED_PCIE_INT_CTRL_POLL_EN GENMASK(13, 12) + #define MTK_WED_WPDMA_CFG_BASE 0x580 +#define MTK_WED_WPDMA_CFG_INT_MASK 0x584 +#define MTK_WED_WPDMA_CFG_TX 0x588 +#define MTK_WED_WPDMA_CFG_TX_FREE 0x58c #define MTK_WED_WPDMA_TX_MIB(_n) (0x5a0 + (_n) * 4) #define MTK_WED_WPDMA_TX_COHERENT_MIB(_n) (0x5d0 + (_n) * 4) @@ -203,15 +268,24 @@ struct mtk_wdma_desc { #define MTK_WED_WDMA_RESET_IDX_RX GENMASK(17, 16) #define MTK_WED_WDMA_RESET_IDX_DRV GENMASK(25, 24) +#define MTK_WED_WDMA_INT_CLR 0xa24 +#define MTK_WED_WDMA_INT_CLR_RX_DONE GENMASK(17, 16) + #define MTK_WED_WDMA_INT_TRIGGER 0xa28 #define MTK_WED_WDMA_INT_TRIGGER_RX_DONE GENMASK(17, 16) #define MTK_WED_WDMA_INT_CTRL 0xa2c #define MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL GENMASK(17, 16) +#define MTK_WED_WDMA_CFG_BASE 0xaa0 #define MTK_WED_WDMA_OFFSET0 0xaa4 #define MTK_WED_WDMA_OFFSET1 0xaa8 +#define MTK_WED_WDMA_OFST0_GLO_INTS GENMASK(15, 0) +#define MTK_WED_WDMA_OFST0_GLO_CFG GENMASK(31, 16) +#define MTK_WED_WDMA_OFST1_TX_CTRL GENMASK(15, 0) +#define MTK_WED_WDMA_OFST1_RX_CTRL GENMASK(31, 16) + #define MTK_WED_WDMA_RX_MIB(_n) (0xae0 + (_n) * 4) #define MTK_WED_WDMA_RX_RECYCLE_MIB(_n) (0xae8 + (_n) * 4) #define MTK_WED_WDMA_RX_PROCESSED_MIB(_n) (0xaf0 + (_n) * 4) @@ -221,15 +295,22 @@ struct mtk_wdma_desc { #define MTK_WED_RING_OFS_CPU_IDX 0x08 #define MTK_WED_RING_OFS_DMA_IDX 0x0c +#define MTK_WDMA_RING_TX(_n) (0x000 + (_n) * 0x10) #define MTK_WDMA_RING_RX(_n) (0x100 + (_n) * 0x10) #define MTK_WDMA_GLO_CFG 0x204 -#define MTK_WDMA_GLO_CFG_RX_INFO_PRERES GENMASK(28, 26) +#define MTK_WDMA_GLO_CFG_TX_DMA_EN BIT(0) +#define MTK_WDMA_GLO_CFG_RX_DMA_EN BIT(2) +#define MTK_WDMA_GLO_CFG_RX_INFO3_PRERES BIT(26) +#define MTK_WDMA_GLO_CFG_RX_INFO2_PRERES BIT(27) +#define MTK_WDMA_GLO_CFG_RX_INFO1_PRERES BIT(28) #define MTK_WDMA_RESET_IDX 0x208 #define MTK_WDMA_RESET_IDX_TX GENMASK(3, 0) #define MTK_WDMA_RESET_IDX_RX GENMASK(17, 16) +#define MTK_WDMA_INT_STATUS 0x220 + #define MTK_WDMA_INT_MASK 0x228 #define MTK_WDMA_INT_MASK_TX_DONE GENMASK(3, 0) #define MTK_WDMA_INT_MASK_RX_DONE GENMASK(17, 16) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c index 6affbd241264a1493e04d93f01f1ec0e78756170..1184ac5751e1becb8c50ab9cbe9358aea50ba5c0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c @@ -152,7 +152,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, break; case RX: cq->mcq.comp = mlx4_en_rx_irq; - netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64); + netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq); napi_enable(&cq->napi); break; case TX_XDP: diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 6400a827173cfa898173c9f258859c05de238e42..7d45f1d55f7999131a93612f6d0914f3c53b9115 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -89,15 +89,15 @@ mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, DRV_VERSION, + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", (u16) (mdev->dev->caps.fw_ver >> 32), (u16) ((mdev->dev->caps.fw_ver >> 16) & 0xffff), (u16) (mdev->dev->caps.fw_ver & 0xffff)); - strlcpy(drvinfo->bus_info, pci_name(mdev->dev->persist->pdev), + strscpy(drvinfo->bus_info, pci_name(mdev->dev->persist->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c index dcb9eb1899ce27ac413a5431398f837564575458..fe48d20d6118694eff7330d7e0c4a4b98fa4e419 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.c +++ b/drivers/net/ethernet/mellanox/mlx4/fw.c @@ -1779,7 +1779,7 @@ static void get_board_id(void *vsd, char *board_id) if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { - strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN); + strscpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN); } else { /* * The board ID is a string but the firmware byte diff --git a/drivers/net/ethernet/mellanox/mlx4/icm.c b/drivers/net/ethernet/mellanox/mlx4/icm.c index d89a3da89e5aa0ee304481bd165a7179e52f72e9..59b8b3c73582812b0a6dfa03e7fdf46d13d810f6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/icm.c +++ b/drivers/net/ethernet/mellanox/mlx4/icm.c @@ -208,7 +208,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, chunk->sg, chunk->npages, DMA_BIDIRECTIONAL); - if (chunk->nsg <= 0) + if (!chunk->nsg) goto fail; } @@ -222,7 +222,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, chunk->nsg = dma_map_sg(&dev->persist->pdev->dev, chunk->sg, chunk->npages, DMA_BIDIRECTIONAL); - if (chunk->nsg <= 0) + if (!chunk->nsg) goto fail; } diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 78c5f40382c964cb197a65025053cc2aeb425eda..d3fc86cd3c1d3e97b827cb4dde2210cb5e891ea1 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -3071,6 +3071,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) err = device_create_file(&dev->persist->pdev->dev, &info->port_attr); if (err) { mlx4_err(dev, "Failed to create file for port %d\n", port); + devlink_port_type_clear(&info->devlink_port); devl_port_unregister(&info->devlink_port); info->port = -1; return err; @@ -3093,6 +3094,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port) mlx4_err(dev, "Failed to create mtu file for port %d\n", port); device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); + devlink_port_type_clear(&info->devlink_port); devl_port_unregister(&info->devlink_port); info->port = -1; return err; @@ -3109,6 +3111,7 @@ static void mlx4_cleanup_port_info(struct mlx4_port_info *info) device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr); device_remove_file(&info->dev->persist->pdev->dev, &info->port_mtu_attr); + devlink_port_type_clear(&info->devlink_port); devl_port_unregister(&info->devlink_port); #ifdef CONFIG_RFS_ACCEL diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index bfc0cd5ec423116a6587c20ade5fcb0a3e3b0ff4..26685fd0fdaa472bee67e3ca207efd8de8c0b56e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -139,6 +139,14 @@ config MLX5_CORE_IPOIB help MLX5 IPoIB offloads & acceleration support. +config MLX5_EN_MACSEC + bool "Connect-X support for MACSec offload" + depends on MLX5_CORE_EN + depends on MACSEC + default n + help + Build support for MACsec cryptography-offload acceleration in the NIC. + config MLX5_EN_IPSEC bool "Mellanox Technologies IPsec Connect-X support" depends on MLX5_CORE_EN diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index a3773a8177edb6834d194840f91d3844c456183d..a22c32aabf111eadf79f29660ef6bf2a5504cf59 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -92,6 +92,9 @@ mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o ipoib/ipoib # mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o +mlx5_core-$(CONFIG_MLX5_EN_MACSEC) += en_accel/macsec.o en_accel/macsec_fs.o \ + en_accel/macsec_stats.o + mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \ en_accel/ipsec_stats.o en_accel/ipsec_fs.o \ en_accel/ipsec_offload.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index a560df446bac55896564e9704ebae00ed7c62f51..26a23047f1f3b5e722517fec703e2a9f82278787 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -93,29 +93,26 @@ struct page_pool; #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \ MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD)) -#define MLX5_MPWRQ_LOG_WQE_SZ 18 -#define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \ - MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0) -#define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER) - -#define MLX5_ALIGN_MTTS(mtts) (ALIGN(mtts, 8)) -#define MLX5_ALIGNED_MTTS_OCTW(mtts) ((mtts) / 2) -#define MLX5_MTT_OCTW(mtts) (MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts))) -/* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between - * WQEs, This page will absorb write overflow by the hardware, when - * receiving packets larger than MTU. These oversize packets are - * dropped by the driver at a later stage. +#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 + +/* Keep in sync with mlx5e_mpwrq_log_wqe_sz. + * These are theoretical maximums, which can be further restricted by + * capabilities. These values are used for static resource allocations and + * sanity checks. + * MLX5_SEND_WQE_MAX_SIZE is a bit bigger than the maximum cacheline-aligned WQE + * size actually used at runtime, but it's not a problem when calculating static + * array sizes. */ -#define MLX5E_REQUIRED_WQE_MTTS (MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1)) -#define MLX5E_REQUIRED_MTTS(wqes) (wqes * MLX5E_REQUIRED_WQE_MTTS) +#define MLX5_UMR_MAX_MTT_SPACE \ + (ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \ + MLX5_UMR_MTT_ALIGNMENT)) +#define MLX5_MPWRQ_MAX_PAGES_PER_WQE \ + rounddown_pow_of_two(MLX5_UMR_MAX_MTT_SPACE / sizeof(struct mlx5_mtt)) + #define MLX5E_MAX_RQ_NUM_MTTS \ - (ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */ + (ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */ +#define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */ #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024)) -#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW \ - (ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS)) -#define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \ - (MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \ - (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU)) #define MLX5E_MIN_SKB_FRAG_SZ (MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM)) #define MLX5E_LOG_MAX_RX_WQE_BULK \ @@ -127,8 +124,7 @@ struct page_pool; #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK) #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE 0xa -#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd, \ - MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW) +#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2 @@ -150,13 +146,6 @@ struct page_pool; #define MLX5E_TX_XSK_POLL_BUDGET 64 #define MLX5E_SQ_RECOVER_MIN_INTERVAL 500 /* msecs */ -#define MLX5E_UMR_WQE_INLINE_SZ \ - (sizeof(struct mlx5e_umr_wqe) + \ - ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \ - MLX5_UMR_MTT_ALIGNMENT)) -#define MLX5E_UMR_WQEBBS \ - (DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB)) - #define MLX5E_KLM_UMR_WQE_SZ(sgl_len)\ (sizeof(struct mlx5e_umr_wqe) +\ (sizeof(struct mlx5_klm) * (sgl_len))) @@ -174,8 +163,7 @@ struct page_pool; ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_ALIGNMENT) #define MLX5E_MAX_KLM_PER_WQE(mdev) \ - MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * \ - mlx5e_get_sw_max_sq_mpw_wqebbs(mlx5e_get_max_sq_wqebbs(mdev))) + MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * mlx5e_get_max_sq_aligned_wqebbs(mdev)) #define MLX5E_MSG_LEVEL NETIF_MSG_LINK @@ -189,12 +177,6 @@ do { \ #define mlx5e_state_dereference(priv, p) \ rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock)) -enum mlx5e_rq_group { - MLX5E_RQ_GROUP_REGULAR, - MLX5E_RQ_GROUP_XSK, -#define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g) -}; - static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev) { if (mlx5_lag_is_lacp_owner(mdev)) @@ -227,13 +209,15 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev) * bytes units. Driver hardens the limitation to 1KB (16 * WQEBBs), unless firmware capability is stricter. */ -static inline u16 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev) +static inline u8 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev) { - return min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS, - MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB); + BUILD_BUG_ON(MLX5_SEND_WQE_MAX_WQEBBS > U8_MAX); + + return (u8)min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS, + MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB); } -static inline u8 mlx5e_get_sw_max_sq_mpw_wqebbs(u8 max_sq_wqebbs) +static inline u8 mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev *mdev) { /* The return value will be multiplied by MLX5_SEND_WQEBB_NUM_DS. * Since max_sq_wqebbs may be up to MLX5_SEND_WQE_MAX_WQEBBS == 16, @@ -242,8 +226,9 @@ static inline u8 mlx5e_get_sw_max_sq_mpw_wqebbs(u8 max_sq_wqebbs) * than MLX5_SEND_WQE_MAX_WQEBBS to let a full-session WQE be * cache-aligned. */ - u8 wqebbs = min_t(u8, max_sq_wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1); + u8 wqebbs = mlx5e_get_max_sq_wqebbs(mdev); + wqebbs = min_t(u8, wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1); #if L1_CACHE_BYTES >= 128 wqebbs = ALIGN_DOWN(wqebbs, 2); #endif @@ -272,6 +257,7 @@ struct mlx5e_umr_wqe { union { DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts); DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms); + DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms); }; }; @@ -476,15 +462,11 @@ struct mlx5e_txqsq { struct work_struct recover_work; struct mlx5e_ptpsq *ptpsq; cqe_ts_to_ns ptp_cyc2time; - u16 max_sq_wqebbs; } ____cacheline_aligned_in_smp; -struct mlx5e_dma_info { - dma_addr_t addr; - union { - struct page *page; - struct xdp_buff *xsk; - }; +union mlx5e_alloc_unit { + struct page *page; + struct xdp_buff *xsk; }; /* XDP packets can be transmitted in different ways. On completion, we need to @@ -580,7 +562,6 @@ struct mlx5e_xdpsq { /* control path */ struct mlx5_wq_ctrl wq_ctrl; struct mlx5e_channel *channel; - u16 max_sq_wqebbs; } ____cacheline_aligned_in_smp; struct mlx5e_ktls_resync_resp; @@ -609,25 +590,20 @@ struct mlx5e_icosq { /* control path */ struct mlx5_wq_ctrl wq_ctrl; struct mlx5e_channel *channel; - u16 max_sq_wqebbs; struct work_struct recover_work; } ____cacheline_aligned_in_smp; struct mlx5e_wqe_frag_info { - struct mlx5e_dma_info *di; + union mlx5e_alloc_unit *au; u32 offset; bool last_in_page; }; -struct mlx5e_umr_dma_info { - struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE]; -}; - struct mlx5e_mpw_info { - struct mlx5e_umr_dma_info umr; u16 consumed_strides; - DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE); + DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE); + union mlx5e_alloc_unit alloc_units[]; }; #define MLX5E_MAX_RX_FRAGS 4 @@ -635,13 +611,13 @@ struct mlx5e_mpw_info { /* a single cache unit is capable to serve one napi call (for non-striding rq) * or a MPWQE (for striding rq). */ -#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ - MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT) +#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_MAX_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \ + MLX5_MPWRQ_MAX_PAGES_PER_WQE : NAPI_POLL_WEIGHT) #define MLX5E_CACHE_SIZE (4 * roundup_pow_of_two(MLX5E_CACHE_UNIT)) struct mlx5e_page_cache { u32 head; u32 tail; - struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE]; + struct page *page_cache[MLX5E_CACHE_SIZE]; }; struct mlx5e_rq; @@ -674,6 +650,12 @@ struct mlx5e_rq_frags_info { u8 num_frags; u8 log_num_frags; u8 wqe_bulk; + u8 wqe_index_mask; +}; + +struct mlx5e_dma_info { + dma_addr_t addr; + struct page *page; }; struct mlx5e_shampo_hd { @@ -695,13 +677,20 @@ struct mlx5e_hw_gro_data { int second_ip_id; }; +enum mlx5e_mpwrq_umr_mode { + MLX5E_MPWRQ_UMR_MODE_ALIGNED, + MLX5E_MPWRQ_UMR_MODE_UNALIGNED, + MLX5E_MPWRQ_UMR_MODE_OVERSIZED, + MLX5E_MPWRQ_UMR_MODE_TRIPLE, +}; + struct mlx5e_rq { /* data path */ union { struct { struct mlx5_wq_cyc wq; struct mlx5e_wqe_frag_info *frags; - struct mlx5e_dma_info *di; + union mlx5e_alloc_unit *alloc_units; struct mlx5e_rq_frags_info info; mlx5e_fp_skb_from_cqe skb_from_cqe; } wqe; @@ -710,6 +699,7 @@ struct mlx5e_rq { struct mlx5e_umr_wqe umr_wqe; struct mlx5e_mpw_info *info; mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq; + __be32 umr_mkey_be; u16 num_strides; u16 actual_wq_head; u8 log_stride_sz; @@ -717,6 +707,11 @@ struct mlx5e_rq { u8 umr_last_bulk; u8 umr_completed; u8 min_wqe_bulk; + u8 page_shift; + u8 pages_per_wqe; + u8 umr_wqebbs; + u8 mtts_per_wqe; + u8 umr_mode; struct mlx5e_shampo_hd *shampo; } mpwqe; }; @@ -767,7 +762,6 @@ struct mlx5e_rq { u32 rqn; struct mlx5_core_dev *mdev; struct mlx5e_channel *channel; - u32 umr_mkey; struct mlx5e_dma_info wqe_overflow; /* XDP read-mostly */ @@ -856,11 +850,6 @@ enum { MLX5E_STATE_XDP_ACTIVE, }; -enum { - MLX5E_TC_PRIO = 0, - MLX5E_NIC_PRIO -}; - struct mlx5e_modify_sq_param { int curr_state; int next_state; @@ -959,6 +948,9 @@ struct mlx5e_priv { const struct mlx5e_profile *profile; void *ppriv; +#ifdef CONFIG_MLX5_EN_MACSEC + struct mlx5e_macsec *macsec; +#endif #ifdef CONFIG_MLX5_EN_IPSEC struct mlx5e_ipsec *ipsec; #endif @@ -1010,7 +1002,6 @@ struct mlx5e_profile { mlx5e_stats_grp_t *stats_grps; const struct mlx5e_rx_handlers *rx_handlers; int max_tc; - u8 rq_groups; u32 features; }; @@ -1019,7 +1010,8 @@ struct mlx5e_profile { void mlx5e_build_ptys2ethtool_map(void); -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev); +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close); void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); @@ -1047,6 +1039,7 @@ struct mlx5e_rq_param; int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, int node, struct mlx5e_rq *rq); +#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_close_rq(struct mlx5e_rq *rq); int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param); @@ -1101,7 +1094,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv); void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv); int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx); -int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state); +int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state); void mlx5e_activate_rq(struct mlx5e_rq *rq); void mlx5e_deactivate_rq(struct mlx5e_rq *rq); void mlx5e_activate_icosq(struct mlx5e_icosq *icosq); @@ -1136,6 +1129,7 @@ static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev) extern const struct ethtool_ops mlx5e_ethtool_ops; +int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey); int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev); void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev); int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb, @@ -1148,8 +1142,6 @@ void mlx5e_destroy_q_counters(struct mlx5e_priv *priv); int mlx5e_open_drop_rq(struct mlx5e_priv *priv, struct mlx5e_rq *drop_rq); void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq); -int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node); -void mlx5e_free_di_list(struct mlx5e_rq *rq); int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn); void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c index e7c14c0de0a7e3da3a9ae91fb0467f1f84675aac..48581ea3adcb8e33430a22323f90551a0e4763a7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c @@ -10,28 +10,33 @@ unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs) return chs->num; } -void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) +static struct mlx5e_channel *mlx5e_channels_get(struct mlx5e_channels *chs, unsigned int ix) { - struct mlx5e_channel *c; + WARN_ON_ONCE(ix >= mlx5e_channels_get_num(chs)); + return chs->c[ix]; +} - WARN_ON(ix >= mlx5e_channels_get_num(chs)); - c = chs->c[ix]; +bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix) +{ + struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); - *rqn = c->rq.rqn; + return test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); } -bool mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) +void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) { - struct mlx5e_channel *c; + struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); - WARN_ON(ix >= mlx5e_channels_get_num(chs)); - c = chs->c[ix]; + *rqn = c->rq.rqn; +} - if (!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) - return false; +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn) +{ + struct mlx5e_channel *c = mlx5e_channels_get(chs, ix); + + WARN_ON_ONCE(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)); *rqn = c->xskrq.rqn; - return true; } bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h index ca00cbc827cbfd4f81c441fbf505b3df6aa2b3de..637ca90daaa81e5a88fce08741c6cc459b5d8f31 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h @@ -9,8 +9,9 @@ struct mlx5e_channels; unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs); +bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix); void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); -bool mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); +void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn); bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn); #endif /* __MLX5_EN_CHANNELS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 9b8cdf2e68adb0be17a75e470a733df1c1c757a9..bf2741eb7f9b7527f075ce1388303d1ae358fd89 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -8,6 +8,7 @@ #include "lib/fs_ttc.h" struct mlx5e_post_act; +struct mlx5e_tc_table; enum { MLX5E_TC_FT_LEVEL = 0, @@ -15,6 +16,11 @@ enum { MLX5E_TC_MISS_LEVEL, }; +enum { + MLX5E_TC_PRIO = 0, + MLX5E_NIC_PRIO +}; + struct mlx5e_flow_table { int num_groups; struct mlx5_flow_table *t; @@ -83,54 +89,28 @@ enum { #endif }; -struct mlx5e_priv; - -#ifdef CONFIG_MLX5_EN_RXNFC - -struct mlx5e_ethtool_table { - struct mlx5_flow_table *ft; - int num_rules; -}; - -#define ETHTOOL_NUM_L3_L4_FTS 7 -#define ETHTOOL_NUM_L2_FTS 4 - -struct mlx5e_ethtool_steering { - struct mlx5e_ethtool_table l3_l4_ft[ETHTOOL_NUM_L3_L4_FTS]; - struct mlx5e_ethtool_table l2_ft[ETHTOOL_NUM_L2_FTS]; - struct list_head rules; - int tot_num_rules; -}; - -void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv); -void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv); -int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd); -int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, - struct ethtool_rxnfc *info, u32 *rule_locs); -#else -static inline void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv) { } -static inline void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv) { } -static inline int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd) -{ return -EOPNOTSUPP; } -static inline int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, - struct ethtool_rxnfc *info, u32 *rule_locs) -{ return -EOPNOTSUPP; } -#endif /* CONFIG_MLX5_EN_RXNFC */ +struct mlx5e_flow_steering; +struct mlx5e_rx_res; #ifdef CONFIG_MLX5_EN_ARFS struct mlx5e_arfs_tables; -int mlx5e_arfs_create_tables(struct mlx5e_priv *priv); -void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv); -int mlx5e_arfs_enable(struct mlx5e_priv *priv); -int mlx5e_arfs_disable(struct mlx5e_priv *priv); +int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, bool ntuple); +void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple); +int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs); +int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs); int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, u16 rxq_index, u32 flow_id); #else -static inline int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) { return 0; } -static inline void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) {} -static inline int mlx5e_arfs_enable(struct mlx5e_priv *priv) { return -EOPNOTSUPP; } -static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv) { return -EOPNOTSUPP; } +static inline int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, bool ntuple) +{ return 0; } +static inline void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple) {} +static inline int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs) +{ return -EOPNOTSUPP; } +static inline int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs) +{ return -EOPNOTSUPP; } #endif #ifdef CONFIG_MLX5_EN_TLS @@ -142,54 +122,63 @@ struct mlx5e_fs_udp; struct mlx5e_fs_any; struct mlx5e_ptp_fs; -struct mlx5e_flow_steering { - bool state_destroy; - bool vlan_strip_disable; - struct mlx5_core_dev *mdev; - struct mlx5_flow_namespace *ns; -#ifdef CONFIG_MLX5_EN_RXNFC - struct mlx5e_ethtool_steering ethtool; -#endif - struct mlx5e_tc_table *tc; - struct mlx5e_promisc_table promisc; - struct mlx5e_vlan_table *vlan; - struct mlx5e_l2_table l2; - struct mlx5_ttc_table *ttc; - struct mlx5_ttc_table *inner_ttc; -#ifdef CONFIG_MLX5_EN_ARFS - struct mlx5e_arfs_tables *arfs; -#endif -#ifdef CONFIG_MLX5_EN_TLS - struct mlx5e_accel_fs_tcp *accel_tcp; -#endif - struct mlx5e_fs_udp *udp; - struct mlx5e_fs_any *any; - struct mlx5e_ptp_fs *ptp_fs; -}; - -void mlx5e_set_ttc_params(struct mlx5e_priv *priv, +void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, struct ttc_params *ttc_params, bool tunnel); -void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv); -int mlx5e_create_ttc_table(struct mlx5e_priv *priv); +void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs); +int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res); void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft); -void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv); -void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv); +void mlx5e_enable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc); +void mlx5e_disable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc); -int mlx5e_create_flow_steering(struct mlx5e_priv *priv); -void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv); +int mlx5e_create_flow_steering(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, + const struct mlx5e_profile *profile, + struct net_device *netdev); +void mlx5e_destroy_flow_steering(struct mlx5e_flow_steering *fs, bool ntuple, + const struct mlx5e_profile *profile); struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile, struct mlx5_core_dev *mdev, bool state_destroy); void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs); - -int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int trap_id, int tir_num); -void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv); -int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int trap_id, int tir_num); -void mlx5e_remove_mac_trap(struct mlx5e_priv *priv); +struct mlx5e_vlan_table *mlx5e_fs_get_vlan(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_tc(struct mlx5e_flow_steering *fs, struct mlx5e_tc_table *tc); +struct mlx5e_tc_table *mlx5e_fs_get_tc(struct mlx5e_flow_steering *fs); +struct mlx5e_l2_table *mlx5e_fs_get_l2(struct mlx5e_flow_steering *fs); +struct mlx5_flow_namespace *mlx5e_fs_get_ns(struct mlx5e_flow_steering *fs, bool egress); +void mlx5e_fs_set_ns(struct mlx5e_flow_steering *fs, struct mlx5_flow_namespace *ns, bool egress); +#ifdef CONFIG_MLX5_EN_RXNFC +struct mlx5e_ethtool_steering *mlx5e_fs_get_ethtool(struct mlx5e_flow_steering *fs); +#endif +struct mlx5_ttc_table *mlx5e_fs_get_ttc(struct mlx5e_flow_steering *fs, bool inner); +void mlx5e_fs_set_ttc(struct mlx5e_flow_steering *fs, struct mlx5_ttc_table *ttc, bool inner); +#ifdef CONFIG_MLX5_EN_ARFS +struct mlx5e_arfs_tables *mlx5e_fs_get_arfs(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_arfs(struct mlx5e_flow_steering *fs, struct mlx5e_arfs_tables *arfs); +#endif +struct mlx5e_ptp_fs *mlx5e_fs_get_ptp(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_ptp(struct mlx5e_flow_steering *fs, struct mlx5e_ptp_fs *ptp_fs); +struct mlx5e_fs_any *mlx5e_fs_get_any(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any); +struct mlx5e_fs_udp *mlx5e_fs_get_udp(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_udp(struct mlx5e_flow_steering *fs, struct mlx5e_fs_udp *udp); +#ifdef CONFIG_MLX5_EN_TLS +struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs); +void mlx5e_fs_set_accel_tcp(struct mlx5e_flow_steering *fs, struct mlx5e_accel_fs_tcp *accel_tcp); +#endif +void mlx5e_fs_set_state_destroy(struct mlx5e_flow_steering *fs, bool state_destroy); +void mlx5e_fs_set_vlan_strip_disable(struct mlx5e_flow_steering *fs, bool vlan_strip_disable); + +struct mlx5_core_dev *mlx5e_fs_get_mdev(struct mlx5e_flow_steering *fs); +int mlx5e_add_vlan_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num); +void mlx5e_remove_vlan_trap(struct mlx5e_flow_steering *fs); +int mlx5e_add_mac_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num); +void mlx5e_remove_mac_trap(struct mlx5e_flow_steering *fs); void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, struct net_device *netdev); int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs, struct net_device *netdev, @@ -198,5 +187,18 @@ int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs, struct net_device *netdev, __be16 proto, u16 vid); void mlx5e_fs_init_l2_addr(struct mlx5e_flow_steering *fs, struct net_device *netdev); + +#define fs_err(fs, fmt, ...) \ + mlx5_core_err(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__) + +#define fs_dbg(fs, fmt, ...) \ + mlx5_core_dbg(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__) + +#define fs_warn(fs, fmt, ...) \ + mlx5_core_warn(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__) + +#define fs_warn_once(fs, fmt, ...) \ + mlx5_core_warn_once(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__) + #endif /* __MLX5E_FLOW_STEER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h new file mode 100644 index 0000000000000000000000000000000000000000..9e276fd3c0cf112229892c46234f798f87da034b --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */ + +#ifndef __MLX5E_FS_ETHTOOL_H__ +#define __MLX5E_FS_ETHTOOL_H__ + +struct mlx5e_priv; +struct mlx5e_ethtool_steering; +#ifdef CONFIG_MLX5_EN_RXNFC +int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool); +void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool); +void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs); +void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs); +int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd); +int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, + struct ethtool_rxnfc *info, u32 *rule_locs); +#else +static inline int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool) +{ return 0; } +static inline void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool) { } +static inline void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs) { } +static inline void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs) { } +static inline int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd) +{ return -EOPNOTSUPP; } +static inline int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, + struct ethtool_rxnfc *info, u32 *rule_locs) +{ return -EOPNOTSUPP; } +#endif +#endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c index e153d6119e02a96582dbae0a6337d383c1a9a79e..03cb79adf912f70e703007c888a91cc97f611f37 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ -#include #include "en/fs_tt_redirect.h" #include "fs_core.h" +#include "mlx5_core.h" enum fs_udp_type { FS_IPV4_UDP, @@ -74,17 +74,17 @@ static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type } struct mlx5_flow_handle * -mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, +mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_flow_steering *fs, enum mlx5_traffic_types ttc_type, u32 tir_num, u16 d_port) { + struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs); enum fs_udp_type type = tt2fs_udp(ttc_type); struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft = NULL; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; - struct mlx5e_fs_udp *fs_udp; int err; if (type == FS_UDP_NUM_TYPES) @@ -94,7 +94,6 @@ mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, if (!spec) return ERR_PTR(-ENOMEM); - fs_udp = priv->fs->udp; ft = fs_udp->tables[type].t; fs_udp_set_dport_flow(spec, type, d_port); @@ -106,31 +105,30 @@ mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n", - __func__, fs_udp_type2str(type), err); + fs_err(fs, "%s: add %s rule failed, err %d\n", + __func__, fs_udp_type2str(type), err); } return rule; } -static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type) +static int fs_udp_add_default_rule(struct mlx5e_flow_steering *fs, enum fs_udp_type type) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); + struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs); struct mlx5e_flow_table *fs_udp_t; struct mlx5_flow_destination dest; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; - struct mlx5e_fs_udp *fs_udp; int err; - fs_udp = priv->fs->udp; fs_udp_t = &fs_udp->tables[type]; - dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_udp2tt(type)); + dest = mlx5_ttc_get_default_dest(ttc, fs_udp2tt(type)); rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, - "%s: add default rule failed, fs type=%d, err %d\n", - __func__, type, err); + fs_err(fs, "%s: add default rule failed, fs type=%d, err %d\n", + __func__, type, err); return err; } @@ -206,33 +204,36 @@ out: return err; } -static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type) +static int fs_udp_create_table(struct mlx5e_flow_steering *fs, enum fs_udp_type type) { - struct mlx5e_flow_table *ft = &priv->fs->udp->tables[type]; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false); + struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs); struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5e_flow_table *ft; int err; + ft = &fs_udp->tables[type]; ft->num_groups = 0; ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE; ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); + ft->t = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; return err; } - netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n", - fs_udp_type2str(type), ft->t->id, ft->t->level); + mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs %s table id %u level %u\n", + fs_udp_type2str(type), ft->t->id, ft->t->level); err = fs_udp_create_groups(ft, type); if (err) goto err; - err = fs_udp_add_default_rule(priv, type); + err = fs_udp_add_default_rule(fs, type); if (err) goto err; @@ -253,17 +254,17 @@ static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i) fs_udp->tables[i].t = NULL; } -static int fs_udp_disable(struct mlx5e_priv *priv) +static int fs_udp_disable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); int err, i; for (i = 0; i < FS_UDP_NUM_TYPES; i++) { /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_udp2tt(i)); + err = mlx5_ttc_fwd_default_dest(ttc, fs_udp2tt(i)); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] default destination failed, err(%d)\n", - __func__, fs_udp2tt(i), err); + fs_err(fs, "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, fs_udp2tt(i), err); return err; } } @@ -271,30 +272,31 @@ static int fs_udp_disable(struct mlx5e_priv *priv) return 0; } -static int fs_udp_enable(struct mlx5e_priv *priv) +static int fs_udp_enable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); + struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs); struct mlx5_flow_destination dest = {}; int err, i; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < FS_UDP_NUM_TYPES; i++) { - dest.ft = priv->fs->udp->tables[i].t; + dest.ft = udp->tables[i].t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_udp2tt(i), &dest); + err = mlx5_ttc_fwd_dest(ttc, fs_udp2tt(i), &dest); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] destination to accel failed, err(%d)\n", - __func__, fs_udp2tt(i), err); + fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n", + __func__, fs_udp2tt(i), err); return err; } } return 0; } -void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv) +void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_flow_steering *fs) { - struct mlx5e_fs_udp *fs_udp = priv->fs->udp; + struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs); int i; if (!fs_udp) @@ -303,48 +305,50 @@ void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv) if (--fs_udp->ref_cnt) return; - fs_udp_disable(priv); + fs_udp_disable(fs); for (i = 0; i < FS_UDP_NUM_TYPES; i++) fs_udp_destroy_table(fs_udp, i); kfree(fs_udp); - priv->fs->udp = NULL; + mlx5e_fs_set_udp(fs, NULL); } -int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv) +int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_flow_steering *fs) { + struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs); int i, err; - if (priv->fs->udp) { - priv->fs->udp->ref_cnt++; + if (udp) { + udp->ref_cnt++; return 0; } - priv->fs->udp = kzalloc(sizeof(*priv->fs->udp), GFP_KERNEL); - if (!priv->fs->udp) + udp = kzalloc(sizeof(*udp), GFP_KERNEL); + if (!udp) return -ENOMEM; + mlx5e_fs_set_udp(fs, udp); for (i = 0; i < FS_UDP_NUM_TYPES; i++) { - err = fs_udp_create_table(priv, i); + err = fs_udp_create_table(fs, i); if (err) goto err_destroy_tables; } - err = fs_udp_enable(priv); + err = fs_udp_enable(fs); if (err) goto err_destroy_tables; - priv->fs->udp->ref_cnt = 1; + udp->ref_cnt = 1; return 0; err_destroy_tables: while (--i >= 0) - fs_udp_destroy_table(priv->fs->udp, i); + fs_udp_destroy_table(udp, i); - kfree(priv->fs->udp); - priv->fs->udp = NULL; + kfree(udp); + mlx5e_fs_set_udp(fs, NULL); return err; } @@ -356,22 +360,21 @@ static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_typ } struct mlx5_flow_handle * -mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv, +mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_flow_steering *fs, u32 tir_num, u16 ether_type) { + struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs); struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft = NULL; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; - struct mlx5e_fs_any *fs_any; int err; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return ERR_PTR(-ENOMEM); - fs_any = priv->fs->any; ft = fs_any->table.t; fs_any_set_ethertype_flow(spec, ether_type); @@ -383,31 +386,29 @@ mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv, if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n", - __func__, err); + fs_err(fs, "%s: add ANY rule failed, err %d\n", + __func__, err); } return rule; } -static int fs_any_add_default_rule(struct mlx5e_priv *priv) +static int fs_any_add_default_rule(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); + struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs); struct mlx5e_flow_table *fs_any_t; struct mlx5_flow_destination dest; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; - struct mlx5e_fs_any *fs_any; int err; - fs_any = priv->fs->any; fs_any_t = &fs_any->table; - - dest = mlx5_ttc_get_default_dest(priv->fs->ttc, MLX5_TT_ANY); + dest = mlx5_ttc_get_default_dest(ttc, MLX5_TT_ANY); rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, - "%s: add default rule failed, fs type=ANY, err %d\n", - __func__, err); + fs_err(fs, "%s: add default rule failed, fs type=ANY, err %d\n", + __func__, err); return err; } @@ -472,9 +473,11 @@ err: return err; } -static int fs_any_create_table(struct mlx5e_priv *priv) +static int fs_any_create_table(struct mlx5e_flow_steering *fs) { - struct mlx5e_flow_table *ft = &priv->fs->any->table; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false); + struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs); + struct mlx5e_flow_table *ft = &fs_any->table; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -484,21 +487,21 @@ static int fs_any_create_table(struct mlx5e_priv *priv) ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); + ft->t = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; return err; } - netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n", - ft->t->id, ft->t->level); + mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs ANY table id %u level %u\n", + ft->t->id, ft->t->level); err = fs_any_create_groups(ft); if (err) goto err; - err = fs_any_add_default_rule(priv); + err = fs_any_add_default_rule(fs); if (err) goto err; @@ -509,35 +512,38 @@ err: return err; } -static int fs_any_disable(struct mlx5e_priv *priv) +static int fs_any_disable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); int err; /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, MLX5_TT_ANY); + err = mlx5_ttc_fwd_default_dest(ttc, MLX5_TT_ANY); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] default destination failed, err(%d)\n", - __func__, MLX5_TT_ANY, err); + fs_err(fs, + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, MLX5_TT_ANY, err); return err; } return 0; } -static int fs_any_enable(struct mlx5e_priv *priv) +static int fs_any_enable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); + struct mlx5e_fs_any *any = mlx5e_fs_get_any(fs); struct mlx5_flow_destination dest = {}; int err; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = priv->fs->any->table.t; + dest.ft = any->table.t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs->ttc, MLX5_TT_ANY, &dest); + err = mlx5_ttc_fwd_dest(ttc, MLX5_TT_ANY, &dest); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] destination to accel failed, err(%d)\n", - __func__, MLX5_TT_ANY, err); + fs_err(fs, + "%s: modify ttc[%d] destination to accel failed, err(%d)\n", + __func__, MLX5_TT_ANY, err); return err; } return 0; @@ -553,9 +559,9 @@ static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any) fs_any->table.t = NULL; } -void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv) +void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_flow_steering *fs) { - struct mlx5e_fs_any *fs_any = priv->fs->any; + struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs); if (!fs_any) return; @@ -563,43 +569,45 @@ void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv) if (--fs_any->ref_cnt) return; - fs_any_disable(priv); + fs_any_disable(fs); fs_any_destroy_table(fs_any); kfree(fs_any); - priv->fs->any = NULL; + mlx5e_fs_set_any(fs, NULL); } -int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv) +int mlx5e_fs_tt_redirect_any_create(struct mlx5e_flow_steering *fs) { + struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs); int err; - if (priv->fs->any) { - priv->fs->any->ref_cnt++; + if (fs_any) { + fs_any->ref_cnt++; return 0; } - priv->fs->any = kzalloc(sizeof(*priv->fs->any), GFP_KERNEL); - if (!priv->fs->any) + fs_any = kzalloc(sizeof(*fs_any), GFP_KERNEL); + if (!fs_any) return -ENOMEM; + mlx5e_fs_set_any(fs, fs_any); - err = fs_any_create_table(priv); + err = fs_any_create_table(fs); if (err) return err; - err = fs_any_enable(priv); + err = fs_any_enable(fs); if (err) goto err_destroy_table; - priv->fs->any->ref_cnt = 1; + fs_any->ref_cnt = 1; return 0; err_destroy_table: - fs_any_destroy_table(priv->fs->any); + fs_any_destroy_table(fs_any); - kfree(priv->fs->any); - priv->fs->any = NULL; + kfree(fs_any); + mlx5e_fs_set_any(fs, NULL); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h index 7a70c4f38fdadab6bfd27b4a2a4000e0440d2471..5780fd7ad5079725de987acb67a8a493c4bc2fe5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h @@ -4,23 +4,22 @@ #ifndef __MLX5E_FS_TT_REDIRECT_H__ #define __MLX5E_FS_TT_REDIRECT_H__ -#include "en.h" #include "en/fs.h" void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule); /* UDP traffic type redirect */ struct mlx5_flow_handle * -mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, +mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_flow_steering *fs, enum mlx5_traffic_types ttc_type, u32 tir_num, u16 d_port); -void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv); -int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv); +void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_flow_steering *fs); +int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_flow_steering *fs); /* ANY traffic type redirect*/ struct mlx5_flow_handle * -mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv, +mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_flow_steering *fs, u32 tir_num, u16 ether_type); -void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv); -int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv); +void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_flow_steering *fs); +int mlx5e_fs_tt_redirect_any_create(struct mlx5e_flow_steering *fs); #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index e025040350baba552f7eade5b202fe8b7c6e05ee..29dd3a04c15456a49e62caabd49361b4fb2f9c23 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -6,11 +6,212 @@ #include "en/port.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec.h" +#include -static bool mlx5e_rx_is_xdp(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) +static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) +{ + u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size); + + return min_page_shift ? : 12; +} + +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +{ + u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT; + u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev); + + /* Regular RQ uses order-0 pages, the NIC must be able to map them. */ + if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift)) + min_page_shift = req_page_shift; + + return max(req_page_shift, min_page_shift); +} + +enum mlx5e_mpwrq_umr_mode +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +{ + /* Different memory management schemes use different mechanisms to map + * user-mode memory. The stricter guarantees we have, the faster + * mechanisms we use: + * 1. MTT - direct mapping in page granularity. + * 2. KSM - indirect mapping to another MKey to arbitrary addresses, but + * all mappings have the same size. + * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and + * mappings can have different sizes. + */ + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; + bool oversized = false; + + if (xsk) { + oversized = xsk->chunk_size < (1 << page_shift); + WARN_ON_ONCE(xsk->chunk_size > (1 << page_shift)); + } + + /* XSK frame size doesn't match the UMR page size, either because the + * frame size is not a power of two, or it's smaller than the minimal + * page size supported by the firmware. + * It's possible to receive packets bigger than MTU in certain setups. + * To avoid writing over the XSK frame boundary, the top region of each + * stride is mapped to a garbage page, resulting in two mappings of + * different sizes per frame. + */ + if (oversized) { + /* An optimization for frame sizes equal to 3 * power_of_two. + * 3 KSMs point to the frame, and one KSM points to the garbage + * page, which works faster than KLM. + */ + if (xsk->chunk_size % 3 == 0 && is_power_of_2(xsk->chunk_size / 3)) + return MLX5E_MPWRQ_UMR_MODE_TRIPLE; + + return MLX5E_MPWRQ_UMR_MODE_OVERSIZED; + } + + /* XSK frames can start at arbitrary unaligned locations, but they all + * have the same size which is a power of two. It allows to optimize to + * one KSM per frame. + */ + if (unaligned) + return MLX5E_MPWRQ_UMR_MODE_UNALIGNED; + + /* XSK: frames are naturally aligned, MTT can be used. + * Non-XSK: Allocations happen in units of CPU pages, therefore, the + * mappings are naturally aligned. + */ + return MLX5E_MPWRQ_UMR_MODE_ALIGNED; +} + +u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode) +{ + switch (mode) { + case MLX5E_MPWRQ_UMR_MODE_ALIGNED: + return sizeof(struct mlx5_mtt); + case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: + return sizeof(struct mlx5_ksm); + case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: + return sizeof(struct mlx5_klm) * 2; + case MLX5E_MPWRQ_UMR_MODE_TRIPLE: + return sizeof(struct mlx5_ksm) * 4; + } + WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode); + return 0; +} + +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); + u8 max_pages_per_wqe, max_log_mpwqe_size; + u16 max_wqe_size; + + /* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */ + max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB; + max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe), + MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size; + max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift; + + WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU); + + return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ); +} + +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) { - return params->xdp_prog || xsk; + u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); + u8 pages_per_wqe; + + pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1; + + /* Two MTTs are needed to form an octword. The number of MTTs is encoded + * in octwords in a UMR WQE, so we need at least two to avoid mapping + * garbage addresses. + */ + if (WARN_ON_ONCE(pages_per_wqe < 2 && umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) + pages_per_wqe = 2; + + /* Sanity check for further calculations to succeed. */ + BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64); + if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE)) + return MLX5_MPWRQ_MAX_PAGES_PER_WQE; + + return pages_per_wqe; +} + +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode); + u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); + u16 umr_wqe_sz; + + umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) + + ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT); + + WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK); + + return umr_wqe_sz; +} + +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, umr_mode), + MLX5_SEND_WQE_BB); +} + +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode); + + /* Add another page as a buffer between WQEs. This page will absorb + * write overflow by the hardware, when receiving packets larger than + * MTU. These oversize packets are dropped by the driver at a later + * stage. + */ + return ALIGN(pages_per_wqe + 1, + MLX5_SEND_WQE_BB / mlx5e_mpwrq_umr_entry_size(umr_mode)); +} + +u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + /* Same limits apply to KSMs and KLMs. */ + u32 klm_limit = min(MLX5E_MAX_RQ_NUM_KSMS, + 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size)); + + switch (umr_mode) { + case MLX5E_MPWRQ_UMR_MODE_ALIGNED: + return MLX5E_MAX_RQ_NUM_MTTS; + case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: + return klm_limit; + case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: + /* Each entry is two KLMs. */ + return klm_limit / 2; + case MLX5E_MPWRQ_UMR_MODE_TRIPLE: + /* Each entry is four KSMs. */ + return klm_limit / 4; + } + WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); + return 0; +} + +static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, umr_mode); + u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, umr_mode); + + return ilog2(max_entries / mtts_per_wqe); +} + +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) +{ + return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode) + + mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) - + MLX5E_ORDER2_MAX_PACKET_MTU; } u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, @@ -22,7 +223,7 @@ u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, return xsk->headroom; headroom = NET_IP_ALIGN; - if (mlx5e_rx_is_xdp(params, xsk)) + if (params->xdp_prog) headroom += XDP_PACKET_HEADROOM; else headroom += MLX5_RX_HEADROOM; @@ -30,70 +231,80 @@ u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, return headroom; } -u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) +static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk) { u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); - u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk); - return linear_rq_headroom + hw_mtu; + return xsk->headroom + hw_mtu; } -static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) +static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk) { - u32 frag_sz = mlx5e_rx_get_min_frag_sz(params, xsk); - - /* AF_XDP doesn't build SKBs in place. */ - if (!xsk) - frag_sz = MLX5_SKB_FRAG_SZ(frag_sz); + /* SKBs built on XDP_PASS on XSK RQs don't have headroom. */ + u16 headroom = xsk ? 0 : mlx5e_get_linear_rq_headroom(params, NULL); + u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); - /* XDP in mlx5e doesn't support multiple packets per page. AF_XDP is a - * special case. It can run with frames smaller than a page, as it - * doesn't allocate pages dynamically. However, here we pretend that - * fragments are page-sized: it allows to treat XSK frames like pages - * by redirecting alloc and free operations to XSK rings and by using - * the fact there are no multiple packets per "page" (which is a frame). - * The latter is important, because frames may come in a random order, - * and we will have trouble assemblying a real page of multiple frames. - */ - if (mlx5e_rx_is_xdp(params, xsk)) - frag_sz = max_t(u32, frag_sz, PAGE_SIZE); + return MLX5_SKB_FRAG_SZ(headroom + hw_mtu); +} - /* Even if we can go with a smaller fragment size, we must not put - * multiple packets into a single frame. +static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + bool mpwqe) +{ + /* XSK frames are mapped as individual pages, because frames may come in + * an arbitrary order from random locations in the UMEM. */ if (xsk) - frag_sz = max_t(u32, frag_sz, xsk->chunk_size); + return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE; + + /* XDP in mlx5e doesn't support multiple packets per page. */ + if (params->xdp_prog) + return PAGE_SIZE; - return frag_sz; + return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false)); } -u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) +static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk) { - u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk); + u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz); + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) - + order_base_2(linear_stride_sz); } -bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, +bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - /* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more - * than one page. For this, check both with and without xsk. + if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) + return false; + + /* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data + * must fit into a CPU page. */ - u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk), - mlx5e_rx_get_linear_frag_sz(params, NULL)); + if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE) + return false; + + /* XSK frames must be big enough to hold the packet data. */ + if (xsk && mlx5e_rx_get_linear_sz_xsk(params, xsk) > xsk->chunk_size) + return false; - return params->packet_merge.type == MLX5E_PACKET_MERGE_NONE && - linear_frag_sz <= PAGE_SIZE; + return true; } -bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, - u8 log_stride_sz, u8 log_num_strides) +static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, + u8 log_stride_sz, u8 log_num_strides, + u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) { - if (log_stride_sz + log_num_strides != MLX5_MPWRQ_LOG_WQE_SZ) + if (log_stride_sz + log_num_strides != + mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode)) return false; if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE || @@ -113,28 +324,53 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - s8 log_num_strides; + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + u8 log_num_strides; u8 log_stride_sz; + u8 log_wqe_sz; - if (!mlx5e_rx_is_linear_skb(params, xsk)) + if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) return false; - log_stride_sz = order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk)); - log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - log_stride_sz; + log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); + log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); - return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, log_num_strides); + if (log_wqe_sz < log_stride_sz) + return false; + + log_num_strides = log_wqe_sz - log_stride_sz; + + return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, + log_num_strides, page_shift, + umr_mode); } -u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 log_pkts_per_wqe, page_shift, max_log_rq_size; + + log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); + page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode); /* Numbers are unsigned, don't subtract to avoid underflow. */ if (params->log_rq_mtu_frames < log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW) return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW; + /* Ethtool's rx_max_pending is calculated for regular RQ, that uses + * pages of PAGE_SIZE. Max length of an XSK RQ might differ if it uses a + * frame size not equal to PAGE_SIZE. + * A stricter condition is checked in mlx5e_mpwrq_validate_xsk, WARN on + * unexpected failure. + */ + if (WARN_ON_ONCE(params->log_rq_mtu_frames > log_pkts_per_wqe + max_log_rq_size)) + return max_log_rq_size; + return params->log_rq_mtu_frames - log_pkts_per_wqe; } @@ -164,7 +400,7 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) { if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) - return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk)); + return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); } @@ -173,7 +409,10 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) { - return MLX5_MPWRQ_LOG_WQE_SZ - + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + + return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) - mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); } @@ -209,11 +448,11 @@ u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *par stop_room = mlx5e_ktls_get_stop_room(mdev, params); stop_room += mlx5e_stop_room_for_max_wqe(mdev); if (is_mpwqe) - /* A MPWQE can take up to the maximum-sized WQE + all the normal - * stop room can be taken if a new packet breaks the active - * MPWQE session and allocates its WQEs right away. + /* A MPWQE can take up to the maximum cacheline-aligned WQE + + * all the normal stop room can be taken if a new packet breaks + * the active MPWQE session and allocates its WQEs right away. */ - stop_room += mlx5e_stop_room_for_max_wqe(mdev); + stop_room += mlx5e_stop_room_for_mpwqe(mdev); return stop_room; } @@ -320,22 +559,46 @@ bool slow_pci_heuristic(struct mlx5_core_dev *mdev) link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw; } -bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, - struct mlx5e_params *params) +int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { - if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) - return false; + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, NULL); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL); - if (params->xdp_prog) { - /* XSK params are not considered here. If striding RQ is in use, - * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will - * be called with the known XSK params. - */ - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) - return false; + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) + return -EOPNOTSUPP; + + if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) + return -EINVAL; + + return 0; +} + +int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk) +{ + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + bool unaligned = xsk ? xsk->unaligned : false; + u16 max_mtu_pkts; + + if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) + return -EOPNOTSUPP; + + if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) + return -EINVAL; + + /* Current RQ length is too big for the given frame size, the + * needed number of WQEs exceeds the maximum. + */ + max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, + mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, unaligned)); + if (params->log_rq_mtu_frames > max_mtu_pkts) { + mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", + 1 << params->log_rq_mtu_frames, xsk->chunk_size); + return -EINVAL; } - return true; + return 0; } void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, @@ -348,7 +611,7 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? - BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) : + BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) : BIT(params->log_rq_mtu_frames), BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); @@ -356,8 +619,7 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { - params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) && - MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? + params->rq_wq_type = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC; } @@ -374,9 +636,9 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, */ if ((!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) || MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) && - mlx5e_striding_rq_possible(mdev, params) && + !mlx5e_mpwrq_validate_regular(mdev, params) && (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) || - !mlx5e_rx_is_linear_skb(params, NULL))) + !mlx5e_rx_is_linear_skb(mdev, params, NULL))) MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); mlx5e_set_rq_type(mdev, params); mlx5e_init_rq_type_params(mdev, params); @@ -419,16 +681,22 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, int max_mtu; int i; - if (mlx5e_rx_is_linear_skb(params, xsk)) { + if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) { int frag_stride; - frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk); - frag_stride = roundup_pow_of_two(frag_stride); + frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false); info->arr[0].frag_size = byte_count; info->arr[0].frag_stride = frag_stride; info->num_frags = 1; - info->wqe_bulk = PAGE_SIZE / frag_stride; + + /* N WQEs share the same page, N = PAGE_SIZE / frag_stride. The + * first WQE in the page is responsible for allocation of this + * page, this WQE's index is k*N. If WQEs [k*N+1; k*N+N-1] are + * still not completed, the allocation must stop before k*N. + */ + info->wqe_index_mask = (PAGE_SIZE / frag_stride) - 1; + goto out; } @@ -477,11 +745,40 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, i++; } info->num_frags = i; - /* number of different wqes sharing a page */ - info->wqe_bulk = 1 + (info->num_frags % 2); + + /* The last fragment of WQE with index 2*N may share the page with the + * first fragment of WQE with index 2*N+1 in certain cases. If WQE 2*N+1 + * is not completed yet, WQE 2*N must not be allocated, as it's + * responsible for allocating a new page. + */ + if (frag_size_max == PAGE_SIZE) { + /* No WQE can start in the middle of a page. */ + info->wqe_index_mask = 0; + } else { + /* PAGE_SIZEs starting from 8192 don't use 2K-sized fragments, + * because there would be more than MLX5E_MAX_RX_FRAGS of them. + */ + WARN_ON(PAGE_SIZE != 2 * DEFAULT_FRAG_SIZE); + + /* Odd number of fragments allows to pack the last fragment of + * the previous WQE and the first fragment of the next WQE into + * the same page. + * As long as DEFAULT_FRAG_SIZE is 2048, and MLX5E_MAX_RX_FRAGS + * is 4, the last fragment can be bigger than the rest only if + * it's the fourth one, so WQEs consisting of 3 fragments will + * always share a page. + * When a page is shared, WQE bulk size is 2, otherwise just 1. + */ + info->wqe_index_mask = info->num_frags % 2; + } out: - info->wqe_bulk = max_t(u8, info->wqe_bulk, 8); + /* Bulking optimization to skip allocation until at least 8 WQEs can be + * allocated in a row. At the same time, never start allocation when + * the page is still used by older WQEs. + */ + info->wqe_bulk = max_t(u8, info->wqe_index_mask + 1, 8); + info->log_num_frags = order_base_2(info->num_frags); return 0; @@ -520,7 +817,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params)); u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(params, xsk)); + int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); int wqe_size = BIT(log_stride_sz) * num_strides; /* +1 is for the case that the pkt_per_rsrv dont consume the reservation @@ -544,7 +841,7 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk); else - log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + + log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) + mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); break; default: /* MLX5_WQ_TYPE_CYCLIC */ @@ -587,12 +884,16 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: { u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, - log_wqe_num_of_strides)) { + log_wqe_num_of_strides, + page_shift, umr_mode)) { mlx5_core_err(mdev, - "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n", - log_wqe_stride_size, log_wqe_num_of_strides); + "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n", + log_wqe_stride_size, log_wqe_num_of_strides, + umr_mode); return -EINVAL; } @@ -600,7 +901,7 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); MLX5_SET(wq, wq, log_wqe_stride_size, log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); - MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); + MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { MLX5_SET(wq, wq, shampo_enable, true); MLX5_SET(wq, wq, log_reservation_size, @@ -712,13 +1013,6 @@ static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev, param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; } -static u8 mlx5e_get_rq_log_wq_sz(void *rqc) -{ - void *wq = MLX5_ADDR_OF(rqc, rqc, wq); - - return MLX5_GET(wq, wq, log_wq_sz); -} - /* This function calculates the maximum number of headers entries that are needed * per WQE, the formula is based on the size of the reservations and the * restriction we have about max packets for reservation that is equal to max @@ -779,31 +1073,92 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, return wqebbs; } +static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk) +{ + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + u8 umr_wqebbs; + + umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); + + return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); +} + static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rqp) { - u32 wqebbs; + u32 wqebbs, total_pages, useful_space; /* MLX5_WQ_TYPE_CYCLIC */ if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; - wqebbs = MLX5E_UMR_WQEBBS * BIT(mlx5e_get_rq_log_wq_sz(rqp->rqc)); + /* UMR WQEs for the regular RQ. */ + wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); /* If XDP program is attached, XSK may be turned on at any time without * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of * both regular RQ and XSK RQ. - * Although mlx5e_mpwqe_get_log_rq_size accepts mlx5e_xsk_param, it - * doesn't affect its return value, as long as params->xdp_prog != NULL, - * so we can just multiply by 2. + * + * XSK uses different values of page_shift, and the total number of UMR + * WQEBBs depends on it. This dependency is complex and not monotonic, + * especially taking into consideration that some of the parameters come + * from capabilities. Hence, we have to try all valid values of XSK + * frame size (and page_shift) to find the maximum. */ - if (params->xdp_prog) - wqebbs *= 2; + if (params->xdp_prog) { + u32 max_xsk_wqebbs = 0; + u8 frame_shift; + + for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; + frame_shift <= PAGE_SHIFT; frame_shift++) { + /* The headroom doesn't affect the calculation. */ + struct mlx5e_xsk_param xsk = { + .chunk_size = 1 << frame_shift, + .unaligned = false, + }; + + /* XSK aligned mode. */ + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + + /* XSK unaligned mode, frame size is a power of two. */ + xsk.unaligned = true; + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + + /* XSK unaligned mode, frame size is not equal to stride size. */ + xsk.chunk_size -= 1; + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + + /* XSK unaligned mode, frame size is a triple power of two. */ + xsk.chunk_size = (1 << frame_shift) / 4 * 3; + max_xsk_wqebbs = max(max_xsk_wqebbs, + mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + } + + wqebbs += max_xsk_wqebbs; + } if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp); + /* UMR WQEs don't cross the page boundary, they are padded with NOPs. + * This padding is always smaller than the max WQE size. That gives us + * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes + * per page. The number of pages is estimated as the total size of WQEs + * divided by the useful space in page, rounding up. If some WQEs don't + * fully fit into the useful space, they can occupy part of the padding, + * which proves this estimation to be correct (reserve enough space). + */ + useful_space = PAGE_SIZE - mlx5e_get_max_sq_wqebbs(mdev) + MLX5_SEND_WQE_BB; + total_pages = DIV_ROUND_UP(wqebbs * MLX5_SEND_WQE_BB, useful_space); + wqebbs = total_pages * (PAGE_SIZE / MLX5_SEND_WQE_BB); + return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs)); } @@ -857,7 +1212,7 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, mlx5e_build_sq_param_common(mdev, param); MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); - param->is_xdp_mb = !mlx5e_rx_is_linear_skb(params, xsk); + param->is_xdp_mb = !mlx5e_rx_is_linear_skb(mdev, params, xsk); mlx5e_build_tx_cq_param(mdev, params, ¶m->cqp); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index f5c46e78eebc51992eb7a97896459ff4018ab11a..034debd140bcede3941c6f0cf2ad8869a1cd5b62 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -9,6 +9,7 @@ struct mlx5e_xsk_param { u16 headroom; u16 chunk_size; + bool unaligned; }; struct mlx5e_cq_param { @@ -52,37 +53,26 @@ struct mlx5e_create_sq_param { u8 min_inline_mode; }; -static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params, - u16 qid, - enum mlx5e_rq_group group, - u16 *ix) -{ - int nch = params->num_channels; - int ch = qid - nch * group; - - if (ch < 0 || ch >= nch) - return false; - - *ix = ch; - return true; -} - -static inline void mlx5e_qid_get_ch_and_group(struct mlx5e_params *params, - u16 qid, - u16 *ix, - enum mlx5e_rq_group *group) -{ - u16 nch = params->num_channels; - - *ix = qid % nch; - *group = qid / nch; -} - -static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile, - struct mlx5e_params *params, u64 qid) -{ - return qid < params->num_channels * profile->rq_groups; -} +/* Striding RQ dynamic parameters */ + +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +enum mlx5e_mpwrq_umr_mode +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode); +u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); +u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); +u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); +u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); +u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); +u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev, + enum mlx5e_mpwrq_umr_mode umr_mode); +u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode); /* Parameter calculations */ @@ -92,25 +82,23 @@ void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode); bool slow_pci_heuristic(struct mlx5_core_dev *mdev); -bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, struct mlx5e_params *params); +int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params); +int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk); void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); -bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, - u8 log_stride_sz, u8 log_num_strides); u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); -u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); -u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); -bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params, +bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); -u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params, +u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); u8 mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 903de88bab5334c3f3488da1fda6494df258eb51..8469e9c38670653499a2355c50b319491ae39460 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -622,37 +622,39 @@ static int mlx5e_ptp_set_state(struct mlx5e_ptp *c, struct mlx5e_params *params) return bitmap_empty(c->state, MLX5E_PTP_STATE_NUM_STATES) ? -EINVAL : 0; } -static void mlx5e_ptp_rx_unset_fs(struct mlx5e_priv *priv) +static void mlx5e_ptp_rx_unset_fs(struct mlx5e_flow_steering *fs) { - struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; + struct mlx5e_ptp_fs *ptp_fs = mlx5e_fs_get_ptp(fs); if (!ptp_fs->valid) return; mlx5e_fs_tt_redirect_del_rule(ptp_fs->l2_rule); - mlx5e_fs_tt_redirect_any_destroy(priv); + mlx5e_fs_tt_redirect_any_destroy(fs); mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule); mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule); - mlx5e_fs_tt_redirect_udp_destroy(priv); + mlx5e_fs_tt_redirect_udp_destroy(fs); ptp_fs->valid = false; } static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv) { u32 tirn = mlx5e_rx_res_get_tirn_ptp(priv->rx_res); - struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; + struct mlx5e_flow_steering *fs = priv->fs; struct mlx5_flow_handle *rule; + struct mlx5e_ptp_fs *ptp_fs; int err; + ptp_fs = mlx5e_fs_get_ptp(fs); if (ptp_fs->valid) return 0; - err = mlx5e_fs_tt_redirect_udp_create(priv); + err = mlx5e_fs_tt_redirect_udp_create(fs); if (err) goto out_free; - rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV4_UDP, + rule = mlx5e_fs_tt_redirect_udp_add_rule(fs, MLX5_TT_IPV4_UDP, tirn, PTP_EV_PORT); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -660,7 +662,7 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv) } ptp_fs->udp_v4_rule = rule; - rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV6_UDP, + rule = mlx5e_fs_tt_redirect_udp_add_rule(fs, MLX5_TT_IPV6_UDP, tirn, PTP_EV_PORT); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -668,11 +670,11 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv) } ptp_fs->udp_v6_rule = rule; - err = mlx5e_fs_tt_redirect_any_create(priv); + err = mlx5e_fs_tt_redirect_any_create(fs); if (err) goto out_destroy_udp_v6_rule; - rule = mlx5e_fs_tt_redirect_any_add_rule(priv, tirn, ETH_P_1588); + rule = mlx5e_fs_tt_redirect_any_add_rule(fs, tirn, ETH_P_1588); if (IS_ERR(rule)) { err = PTR_ERR(rule); goto out_destroy_fs_any; @@ -683,13 +685,13 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv) return 0; out_destroy_fs_any: - mlx5e_fs_tt_redirect_any_destroy(priv); + mlx5e_fs_tt_redirect_any_destroy(fs); out_destroy_udp_v6_rule: mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule); out_destroy_udp_v4_rule: mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule); out_destroy_fs_udp: - mlx5e_fs_tt_redirect_udp_destroy(priv); + mlx5e_fs_tt_redirect_udp_destroy(fs); out_free: return err; } @@ -723,7 +725,7 @@ int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params, if (err) goto err_free; - netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll, 64); + netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll); mlx5e_ptp_build_params(c, cparams, params); @@ -797,29 +799,31 @@ int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn) return 0; } -int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv) +int mlx5e_ptp_alloc_rx_fs(struct mlx5e_flow_steering *fs, + const struct mlx5e_profile *profile) { struct mlx5e_ptp_fs *ptp_fs; - if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) + if (!mlx5e_profile_feature_cap(profile, PTP_RX)) return 0; ptp_fs = kzalloc(sizeof(*ptp_fs), GFP_KERNEL); if (!ptp_fs) return -ENOMEM; + mlx5e_fs_set_ptp(fs, ptp_fs); - priv->fs->ptp_fs = ptp_fs; return 0; } -void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv) +void mlx5e_ptp_free_rx_fs(struct mlx5e_flow_steering *fs, + const struct mlx5e_profile *profile) { - struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; + struct mlx5e_ptp_fs *ptp_fs = mlx5e_fs_get_ptp(fs); - if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) + if (!mlx5e_profile_feature_cap(profile, PTP_RX)) return; - mlx5e_ptp_rx_unset_fs(priv); + mlx5e_ptp_rx_unset_fs(fs); kfree(ptp_fs); } @@ -845,6 +849,6 @@ int mlx5e_ptp_rx_manage_fs(struct mlx5e_priv *priv, bool set) netdev_WARN_ONCE(priv->netdev, "Don't try to remove PTP RX-FS rules"); return -EINVAL; } - mlx5e_ptp_rx_unset_fs(priv); + mlx5e_ptp_rx_unset_fs(priv->fs); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index 92dbbec472ec639eedaa2cfeead95338a2136b5b..5bce554e131a9c8d526f3c8292fe51ff9441f0fc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -74,8 +74,10 @@ void mlx5e_ptp_close(struct mlx5e_ptp *c); void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c); void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c); int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn); -int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv); -void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv); +int mlx5e_ptp_alloc_rx_fs(struct mlx5e_flow_steering *fs, + const struct mlx5e_profile *profile); +void mlx5e_ptp_free_rx_fs(struct mlx5e_flow_steering *fs, + const struct mlx5e_profile *profile); int mlx5e_ptp_rx_manage_fs(struct mlx5e_priv *priv, bool set); enum { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c index fc366e66d0b0f98ed431845522fa82f6b3b01389..5f6f95ad6888c433ae333a9fb006fbf6f9393a3f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c @@ -134,38 +134,17 @@ out: return err; } -static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) -{ - struct net_device *dev = rq->netdev; - int err; - - err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); - if (err) { - netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); - return err; - } - err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); - if (err) { - netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); - return err; - } - - return 0; -} - static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) { struct mlx5e_rq *rq = ctx; int err; mlx5e_deactivate_rq(rq); - mlx5e_free_rx_descs(rq); - - err = mlx5e_rq_to_ready(rq, MLX5_RQC_STATE_ERR); + err = mlx5e_flush_rq(rq, MLX5_RQC_STATE_ERR); + clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); if (err) - goto out; + return err; - clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); mlx5e_activate_rq(rq); rq->stats->recover++; if (rq->channel) @@ -173,9 +152,6 @@ static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) else mlx5e_trigger_napi_sched(rq->cq.napi); return 0; -out: - clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); - return err; } static int mlx5e_rx_reporter_timeout_recover(void *ctx) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index 24c32f73040adf16358dfc3ff94131949e18cab2..e1095bc36543659ef0175946a5fa321c4d9c60ec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -24,8 +24,6 @@ struct mlx5e_rx_res { struct { struct mlx5e_rqt direct_rqt; struct mlx5e_tir direct_tir; - struct mlx5e_rqt xsk_rqt; - struct mlx5e_tir xsk_tir; } *channels; struct { @@ -320,48 +318,8 @@ static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res) mlx5e_tir_builder_clear(builder); } - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - goto out; - - for (ix = 0; ix < res->max_nch; ix++) { - err = mlx5e_rqt_init_direct(&res->channels[ix].xsk_rqt, - res->mdev, false, res->drop_rqn); - if (err) { - mlx5_core_warn(res->mdev, "Failed to create an XSK RQT: err = %d, ix = %u\n", - err, ix); - goto err_destroy_xsk_rqts; - } - } - - for (ix = 0; ix < res->max_nch; ix++) { - mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - inner_ft_support); - mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param); - mlx5e_tir_builder_build_direct(builder); - - err = mlx5e_tir_init(&res->channels[ix].xsk_tir, builder, res->mdev, true); - if (err) { - mlx5_core_warn(res->mdev, "Failed to create an XSK TIR: err = %d, ix = %u\n", - err, ix); - goto err_destroy_xsk_tirs; - } - - mlx5e_tir_builder_clear(builder); - } - goto out; -err_destroy_xsk_tirs: - while (--ix >= 0) - mlx5e_tir_destroy(&res->channels[ix].xsk_tir); - - ix = res->max_nch; -err_destroy_xsk_rqts: - while (--ix >= 0) - mlx5e_rqt_destroy(&res->channels[ix].xsk_rqt); - - ix = res->max_nch; err_destroy_direct_tirs: while (--ix >= 0) mlx5e_tir_destroy(&res->channels[ix].direct_tir); @@ -420,12 +378,6 @@ static void mlx5e_rx_res_channels_destroy(struct mlx5e_rx_res *res) for (ix = 0; ix < res->max_nch; ix++) { mlx5e_tir_destroy(&res->channels[ix].direct_tir); mlx5e_rqt_destroy(&res->channels[ix].direct_rqt); - - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; - - mlx5e_tir_destroy(&res->channels[ix].xsk_tir); - mlx5e_rqt_destroy(&res->channels[ix].xsk_rqt); } kvfree(res->channels); @@ -491,13 +443,6 @@ u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix) return mlx5e_tir_get_tirn(&res->channels[ix].direct_tir); } -u32 mlx5e_rx_res_get_tirn_xsk(struct mlx5e_rx_res *res, unsigned int ix) -{ - WARN_ON(!(res->features & MLX5E_RX_RES_FEATURE_XSK)); - - return mlx5e_tir_get_tirn(&res->channels[ix].xsk_tir); -} - u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) { struct mlx5e_rss *rss = res->rss[0]; @@ -523,56 +468,53 @@ static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int i return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt); } -void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs) +static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, + struct mlx5e_channels *chs, + unsigned int ix) { - unsigned int nch, ix; + u32 rqn = res->rss_rqns[ix]; int err; - nch = mlx5e_channels_get_num(chs); - - for (ix = 0; ix < chs->num; ix++) - mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); - res->rss_nch = chs->num; + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), + rqn, ix, err); +} - mlx5e_rx_res_rss_enable(res); +static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, + unsigned int ix) +{ + int err; - for (ix = 0; ix < nch; ix++) { - u32 rqn; + err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); + if (err) + mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", + mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), + res->drop_rqn, ix, err); +} - mlx5e_channels_get_regular_rqn(chs, ix, &rqn); - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - rqn, ix, err); +void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs) +{ + unsigned int nch, ix; + int err; - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; + nch = mlx5e_channels_get_num(chs); - if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn)) - rqn = res->drop_rqn; - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - rqn, ix, err); + for (ix = 0; ix < chs->num; ix++) { + if (mlx5e_channels_is_xsk(chs, ix)) + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); + else + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); } - for (ix = nch; ix < res->max_nch; ix++) { - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - res->drop_rqn, ix, err); + res->rss_nch = chs->num; - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; + mlx5e_rx_res_rss_enable(res); - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - res->drop_rqn, ix, err); - } + for (ix = 0; ix < nch; ix++) + mlx5e_rx_res_channel_activate_direct(res, chs, ix); + for (ix = nch; ix < res->max_nch; ix++) + mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { u32 rqn; @@ -595,22 +537,8 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) mlx5e_rx_res_rss_disable(res); - for (ix = 0; ix < res->max_nch; ix++) { - err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), - res->drop_rqn, ix, err); - - if (!(res->features & MLX5E_RX_RES_FEATURE_XSK)) - continue; - - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - res->drop_rqn, ix, err); - } + for (ix = 0; ix < res->max_nch; ix++) + mlx5e_rx_res_channel_deactivate_direct(res, ix); if (res->features & MLX5E_RX_RES_FEATURE_PTP) { err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); @@ -621,33 +549,17 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) } } -int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, - unsigned int ix) +void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, + unsigned int ix, bool xsk) { - u32 rqn; - int err; - - if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn)) - return -EINVAL; - - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to XSK RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - rqn, ix, err); - return err; -} + if (xsk) + mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); + else + mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); -int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix) -{ - int err; + mlx5e_rx_res_rss_enable(res); - err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn); - if (err) - mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n", - mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt), - res->drop_rqn, ix, err); - return err; + mlx5e_rx_res_channel_activate_direct(res, chs, ix); } int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index b39b20a720e0fa244b2047d2ee3ad969f18bfdc9..5d5f64fab60ffe6a993090c93e065413e5d74cbc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -17,8 +17,7 @@ struct mlx5e_rss_params_hash; enum mlx5e_rx_res_features { MLX5E_RX_RES_FEATURE_INNER_FT = BIT(0), - MLX5E_RX_RES_FEATURE_XSK = BIT(1), - MLX5E_RX_RES_FEATURE_PTP = BIT(2), + MLX5E_RX_RES_FEATURE_PTP = BIT(1), }; /* Setup */ @@ -32,7 +31,6 @@ void mlx5e_rx_res_free(struct mlx5e_rx_res *res); /* TIRN getters for flow steering */ u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix); -u32 mlx5e_rx_res_get_tirn_xsk(struct mlx5e_rx_res *res, unsigned int ix); u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt); u32 mlx5e_rx_res_get_tirn_rss_inner(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt); u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res); @@ -40,9 +38,8 @@ u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res); /* Activate/deactivate API */ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs); void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res); -int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, - unsigned int ix); -int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix); +void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, + unsigned int ix, bool xsk); /* Configuration API */ void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c index 69949ab830b6ff388accd4c1dddb9c5cab349c6a..25174f68613ede4943c9ede955309963f726b893 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c @@ -12,6 +12,7 @@ validate_goto_chain(struct mlx5e_priv *priv, const struct flow_action_entry *act, struct netlink_ext_ack *extack) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); bool is_esw = mlx5e_is_eswitch_flow(flow); bool ft_flow = mlx5e_is_ft_flow(flow); u32 dest_chain = act->chain_index; @@ -21,7 +22,7 @@ validate_goto_chain(struct mlx5e_priv *priv, u32 max_chain; esw = priv->mdev->priv.eswitch; - chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(priv->fs->tc); + chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(tc); max_chain = mlx5_chains_get_chain_range(chains); reformat_and_fwd = is_esw ? MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, reformat_and_fwd_to_table) : diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c index 37522352e4b23011d17fd8225b7438452e77e868..c8e5ca65bb6ec395116352f0a5816b4e03026f1f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c @@ -79,6 +79,10 @@ tc_act_police_offload(struct mlx5e_priv *priv, struct mlx5e_flow_meter_handle *meter; int err = 0; + err = mlx5e_policer_validate(&fl_act->action, act, fl_act->extack); + if (err) + return err; + err = fill_meter_params_from_act(act, ¶ms); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index a53e205f4a8953b4a9777eb65783128e206a2e69..be74e14033283cba8b42fceaf5012394a680cb43 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -115,6 +115,7 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, struct mlx5e_flow_meters *flow_meters; u8 cir_man, cir_exp, cbs_man, cbs_exp; struct mlx5_aso_wqe *aso_wqe; + unsigned long expires; struct mlx5_aso *aso; u64 rate, burst; u8 ds_cnt; @@ -187,7 +188,12 @@ mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ - err = mlx5_aso_poll_cq(aso, true, 10); + expires = jiffies + msecs_to_jiffies(10); + do { + err = mlx5_aso_poll_cq(aso, true); + if (err) + usleep_range(2, 10); + } while (err && time_is_after_jiffies(expires)); mutex_unlock(&flow_meters->aso_lock); return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c index 11f2a7fb72a9e620e6d37da3037baddcee5a7552..201ac7dd338f06181f778000a3e43e363af6484d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c @@ -147,7 +147,7 @@ static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv) t->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey); t->stats = &priv->trap_stats.ch; - netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll, 64); + netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll); err = mlx5e_open_trap_rq(priv, t); if (unlikely(err)) @@ -230,12 +230,12 @@ static int mlx5e_handle_action_trap(struct mlx5e_priv *priv, int trap_id) switch (trap_id) { case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER: - err = mlx5e_add_vlan_trap(priv, trap_id, mlx5e_trap_get_tirn(priv->en_trap)); + err = mlx5e_add_vlan_trap(priv->fs, trap_id, mlx5e_trap_get_tirn(priv->en_trap)); if (err) goto err_out; break; case DEVLINK_TRAP_GENERIC_ID_DMAC_FILTER: - err = mlx5e_add_mac_trap(priv, trap_id, mlx5e_trap_get_tirn(priv->en_trap)); + err = mlx5e_add_mac_trap(priv->fs, trap_id, mlx5e_trap_get_tirn(priv->en_trap)); if (err) goto err_out; break; @@ -256,10 +256,10 @@ static int mlx5e_handle_action_drop(struct mlx5e_priv *priv, int trap_id) { switch (trap_id) { case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER: - mlx5e_remove_vlan_trap(priv); + mlx5e_remove_vlan_trap(priv->fs); break; case DEVLINK_TRAP_GENERIC_ID_DMAC_FILTER: - mlx5e_remove_mac_trap(priv); + mlx5e_remove_mac_trap(priv->fs); break; default: netdev_warn(priv->netdev, "%s: Unknown trap id %d\n", __func__, trap_id); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index c208ea307bffbae3d226b66c92c192736b219490..4456ad5cedf1ee536d813f5ac49822ed1e70e089 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -439,16 +439,24 @@ static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev) return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev)); } -static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size) +static inline u16 mlx5e_stop_room_for_mpwqe(struct mlx5_core_dev *mdev) { - u16 room = sq->reserved_room; + u8 mpwqe_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); - WARN_ONCE(wqe_size > sq->max_sq_wqebbs, - "wqe_size %u is greater than max SQ WQEBBs %u", - wqe_size, sq->max_sq_wqebbs); + return mlx5e_stop_room_for_wqe(mdev, mpwqe_wqebbs); +} - room += MLX5E_STOP_ROOM(wqe_size); +static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size) +{ + u16 room = sq->reserved_room + MLX5E_STOP_ROOM(wqe_size); return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room); } + +static inline struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i) +{ + size_t isz = struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe); + + return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz)); +} #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 8f321a6c080903c347eb61d95d7612d32aab19d7..4685c652c97e10635a211ada7b844bfc1203fa42 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -333,7 +333,7 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx mlx5e_xdp_mpwqe_add_dseg(sq, xdptxd, stats); - if (unlikely(mlx5e_xdp_mpqwe_is_full(session, sq->max_sq_mpw_wqebbs))) + if (unlikely(mlx5e_xdp_mpwqe_is_full(session, sq->max_sq_mpw_wqebbs))) mlx5e_xdp_mpwqe_complete(sq); stats->xmit++; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index 287e17911251bf5e460840fc543564c2bdea17fd..bc2d9034af5bbcd16b5a5fcec36ba64143c27b40 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -122,7 +122,7 @@ static inline bool mlx5e_xdp_get_inline_state(struct mlx5e_xdpsq *sq, bool cur) return cur; } -static inline bool mlx5e_xdp_mpqwe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs) +static inline bool mlx5e_xdp_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs) { if (session->inline_on) return session->ds_count + MLX5E_XDP_INLINE_WQE_MAX_DS_CNT > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index 2c520394aa1d646c4bb3f30ee46fa98bf4bd5027..ebada0c5af3cd42601bc322f57d9381e01e7d899 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -72,6 +72,7 @@ void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *x { xsk->headroom = xsk_pool_get_headroom(pool); xsk->chunk_size = xsk_pool_get_chunk_size(pool); + xsk->unaligned = pool->unaligned; } static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, @@ -98,6 +99,15 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, mlx5e_build_xsk_param(pool, &xsk); + if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && + mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { + const char *recommendation = is_power_of_2(xsk.chunk_size) ? + "Upgrade firmware" : "Disable striding RQ"; + + mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n", + xsk.chunk_size, recommendation); + } + if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { /* XSK objects will be created on open. */ goto validate_closed; @@ -123,15 +133,12 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, * any Fill Ring entries at the setup stage. */ - err = mlx5e_rx_res_xsk_activate(priv->rx_res, &priv->channels, ix); - if (unlikely(err)) - goto err_deactivate; + mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true); - return 0; + mlx5e_deactivate_rq(&c->rq); + mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY); -err_deactivate: - mlx5e_deactivate_xsk(c); - mlx5e_close_xsk(c); + return 0; err_remove_pool: mlx5e_xsk_remove_pool(&priv->xsk, ix); @@ -170,7 +177,13 @@ static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix) goto remove_pool; c = priv->channels.c[ix]; - mlx5e_rx_res_xsk_deactivate(priv->rx_res, ix); + + mlx5e_activate_rq(&c->rq); + mlx5e_trigger_napi_icosq(c); + mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT); + + mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false); + mlx5e_deactivate_xsk(c); mlx5e_close_xsk(c); @@ -208,11 +221,10 @@ int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 { struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_params *params = &priv->channels.params; - u16 ix; - if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix))) + if (unlikely(qid >= params->num_channels)) return -EINVAL; - return pool ? mlx5e_xsk_enable_pool(priv, pool, ix) : - mlx5e_xsk_disable_pool(priv, ix); + return pool ? mlx5e_xsk_enable_pool(priv, pool, qid) : + mlx5e_xsk_disable_pool(priv, qid); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c index 9a1553598a7c3db813a8f0562133dfb2df86af8d..c91b54d9ff27a774b11b6633058c8e8f1b603157 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c @@ -8,18 +8,221 @@ /* RX data path */ -static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data, - u32 cqe_bcnt) +int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { + struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); + struct mlx5e_icosq *icosq = rq->icosq; + struct mlx5_wq_cyc *wq = &icosq->wq; + struct mlx5e_umr_wqe *umr_wqe; + int batch, i; + u32 offset; /* 17-bit value with MTT. */ + u16 pi; + + if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, rq->mpwqe.pages_per_wqe))) + goto err; + + BUILD_BUG_ON(sizeof(wi->alloc_units[0]) != sizeof(wi->alloc_units[0].xsk)); + batch = xsk_buff_alloc_batch(rq->xsk_pool, (struct xdp_buff **)wi->alloc_units, + rq->mpwqe.pages_per_wqe); + + /* If batch < pages_per_wqe, either: + * 1. Some (or all) descriptors were invalid. + * 2. dma_need_sync is true, and it fell back to allocating one frame. + * In either case, try to continue allocating frames one by one, until + * the first error, which will mean there are no more valid descriptors. + */ + for (; batch < rq->mpwqe.pages_per_wqe; batch++) { + wi->alloc_units[batch].xsk = xsk_buff_alloc(rq->xsk_pool); + if (unlikely(!wi->alloc_units[batch].xsk)) + goto err_reuse_batch; + } + + pi = mlx5e_icosq_get_next_pi(icosq, rq->mpwqe.umr_wqebbs); + umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); + memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); + + if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) { + for (i = 0; i < batch; i++) { + dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk); + + umr_wqe->inline_mtts[i] = (struct mlx5_mtt) { + .ptag = cpu_to_be64(addr | MLX5_EN_WR), + }; + } + } else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) { + for (i = 0; i < batch; i++) { + dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk); + + umr_wqe->inline_ksms[i] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(addr), + }; + } + } else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) { + u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2); + + for (i = 0; i < batch; i++) { + dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk); + + umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(addr), + }; + umr_wqe->inline_ksms[(i << 2) + 1] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(addr + mapping_size), + }; + umr_wqe->inline_ksms[(i << 2) + 2] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(addr + mapping_size * 2), + }; + umr_wqe->inline_ksms[(i << 2) + 3] = (struct mlx5_ksm) { + .key = rq->mkey_be, + .va = cpu_to_be64(rq->wqe_overflow.addr), + }; + } + } else { + __be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) - + rq->xsk_pool->chunk_size); + __be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size); + + for (i = 0; i < batch; i++) { + dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk); + + umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) { + .key = rq->mkey_be, + .va = cpu_to_be64(addr), + .bcount = frame_size, + }; + umr_wqe->inline_klms[(i << 1) + 1] = (struct mlx5_klm) { + .key = rq->mkey_be, + .va = cpu_to_be64(rq->wqe_overflow.addr), + .bcount = pad_size, + }; + } + } + + bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); + wi->consumed_strides = 0; + + umr_wqe->ctrl.opmod_idx_opcode = + cpu_to_be32((icosq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR); + + /* Optimized for speed: keep in sync with mlx5e_mpwrq_umr_entry_size. */ + offset = ix * rq->mpwqe.mtts_per_wqe; + if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) + offset = offset * sizeof(struct mlx5_mtt) / MLX5_OCTWORD; + else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED)) + offset = offset * sizeof(struct mlx5_klm) * 2 / MLX5_OCTWORD; + else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) + offset = offset * sizeof(struct mlx5_ksm) * 4 / MLX5_OCTWORD; + umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset); + + icosq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { + .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX, + .num_wqebbs = rq->mpwqe.umr_wqebbs, + .umr.rq = rq, + }; + + icosq->pc += rq->mpwqe.umr_wqebbs; + + icosq->doorbell_cseg = &umr_wqe->ctrl; + + return 0; + +err_reuse_batch: + while (--batch >= 0) + xsk_buff_free(wi->alloc_units[batch].xsk); + +err: + rq->stats->buff_alloc_err++; + return -ENOMEM; +} + +int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk) +{ + struct mlx5_wq_cyc *wq = &rq->wqe.wq; + struct xdp_buff **buffs; + u32 contig, alloc; + int i; + + /* mlx5e_init_frags_partition creates a 1:1 mapping between + * rq->wqe.frags and rq->wqe.alloc_units, which allows us to + * allocate XDP buffers straight into alloc_units. + */ + BUILD_BUG_ON(sizeof(rq->wqe.alloc_units[0]) != + sizeof(rq->wqe.alloc_units[0].xsk)); + buffs = (struct xdp_buff **)rq->wqe.alloc_units; + contig = mlx5_wq_cyc_get_size(wq) - ix; + if (wqe_bulk <= contig) { + alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, wqe_bulk); + } else { + alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, contig); + if (likely(alloc == contig)) + alloc += xsk_buff_alloc_batch(rq->xsk_pool, buffs, wqe_bulk - contig); + } + + for (i = 0; i < alloc; i++) { + int j = mlx5_wq_cyc_ctr2ix(wq, ix + i); + struct mlx5e_wqe_frag_info *frag; + struct mlx5e_rx_wqe_cyc *wqe; + dma_addr_t addr; + + wqe = mlx5_wq_cyc_get_wqe(wq, j); + /* Assumes log_num_frags == 0. */ + frag = &rq->wqe.frags[j]; + + addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk); + wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom); + } + + return alloc; +} + +int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk) +{ + struct mlx5_wq_cyc *wq = &rq->wqe.wq; + int i; + + for (i = 0; i < wqe_bulk; i++) { + int j = mlx5_wq_cyc_ctr2ix(wq, ix + i); + struct mlx5e_wqe_frag_info *frag; + struct mlx5e_rx_wqe_cyc *wqe; + dma_addr_t addr; + + wqe = mlx5_wq_cyc_get_wqe(wq, j); + /* Assumes log_num_frags == 0. */ + frag = &rq->wqe.frags[j]; + + frag->au->xsk = xsk_buff_alloc(rq->xsk_pool); + if (unlikely(!frag->au->xsk)) + return i; + + addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk); + wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom); + } + + return wqe_bulk; +} + +static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp) +{ + u32 totallen = xdp->data_end - xdp->data_meta; + u32 metalen = xdp->data - xdp->data_meta; struct sk_buff *skb; - skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt); + skb = napi_alloc_skb(rq->cq.napi, totallen); if (unlikely(!skb)) { rq->stats->buff_alloc_err++; return NULL; } - skb_put_data(skb, data, cqe_bcnt); + skb_put_data(skb, xdp->data_meta, totallen); + + if (metalen) { + skb_metadata_set(skb, metalen); + __skb_pull(skb, metalen); + } return skb; } @@ -30,7 +233,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, u32 head_offset, u32 page_idx) { - struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk; + struct xdp_buff *xdp = wi->alloc_units[page_idx].xsk; struct bpf_prog *prog; /* Check packet size. Note LRO doesn't use linear SKB */ @@ -46,8 +249,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, */ WARN_ON_ONCE(head_offset); - xdp->data_end = xdp->data + cqe_bcnt; - xdp_set_data_meta_invalid(xdp); + xsk_buff_set_size(xdp, cqe_bcnt); xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); net_prefetch(xdp->data); @@ -76,14 +278,14 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, /* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the * frame. On SKB allocation failure, NULL is returned. */ - return mlx5e_xsk_construct_skb(rq, xdp->data, xdp->data_end - xdp->data); + return mlx5e_xsk_construct_skb(rq, xdp); } struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { - struct xdp_buff *xdp = wi->di->xsk; + struct xdp_buff *xdp = wi->au->xsk; struct bpf_prog *prog; /* wi->offset is not used in this function, because xdp->data and the @@ -93,8 +295,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, */ WARN_ON_ONCE(wi->offset); - xdp->data_end = xdp->data + cqe_bcnt; - xdp_set_data_meta_invalid(xdp); + xsk_buff_set_size(xdp, cqe_bcnt); xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool); net_prefetch(xdp->data); @@ -103,8 +304,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, return NULL; /* page/packet was consumed by XDP */ /* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse - * will be handled by mlx5e_put_rx_frag. + * will be handled by mlx5e_free_rx_wqe. * On SKB allocation failure, NULL is returned. */ - return mlx5e_xsk_construct_skb(rq, xdp->data, xdp->data_end - xdp->data); + return mlx5e_xsk_construct_skb(rq, xdp); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h index cc18d97d8ee06743f182fc314e16fc188cc96e37..087c943bd8e90544c43d449765eac79a9cc42d4b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h @@ -5,12 +5,12 @@ #define __MLX5_EN_XSK_RX_H__ #include "en.h" -#include - -#define MLX5E_MTT_PTAG_MASK 0xfffffffffffffff8ULL /* RX data path */ +int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); +int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk); +int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk); struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, @@ -20,46 +20,4 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt); -static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) -{ -retry: - dma_info->xsk = xsk_buff_alloc(rq->xsk_pool); - if (!dma_info->xsk) - return -ENOMEM; - - /* Store the DMA address without headroom. In striding RQ case, we just - * provide pages for UMR, and headroom is counted at the setup stage - * when creating a WQE. In non-striding RQ case, headroom is accounted - * in mlx5e_alloc_rx_wqe. - */ - dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk); - - /* MTT page mapping has alignment requirements. If they are not - * satisfied, leak the descriptor so that it won't come again, and try - * to allocate a new one. - */ - if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { - if (unlikely(dma_info->addr & ~MLX5E_MTT_PTAG_MASK)) { - xsk_buff_discard(dma_info->xsk); - goto retry; - } - } - - return 0; -} - -static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err) -{ - if (!xsk_uses_need_wakeup(rq->xsk_pool)) - return alloc_err; - - if (unlikely(alloc_err)) - xsk_set_rx_need_wakeup(rq->xsk_pool); - else - xsk_clear_rx_need_wakeup(rq->xsk_pool); - - return false; -} - #endif /* __MLX5_EN_XSK_RX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 98ed9ef3a6bdd02277eabdccade2ba380ca09b4a..ff03c43833bbf0148b6b1243a407a19224cbbeab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -5,24 +5,19 @@ #include "en/params.h" #include "en/txrx.h" #include "en/health.h" +#include -/* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may - * change unexpectedly, and mlx5e has a minimum valid stride size for striding - * RQ, keep this check in the driver. +/* The limitation of 2048 can be altered, but shouldn't go beyond the minimal + * stride size of striding RQ. */ -#define MLX5E_MIN_XSK_CHUNK_SIZE 2048 +#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE) bool mlx5e_validate_xsk_param(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk, struct mlx5_core_dev *mdev) { /* AF_XDP doesn't support frames larger than PAGE_SIZE. */ - if (xsk->chunk_size > PAGE_SIZE || - xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) - return false; - - /* Current MTU and XSK headroom don't allow packets to fit the frames. */ - if (mlx5e_rx_get_min_frag_sz(params, xsk) > xsk->chunk_size) + if (xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) return false; /* frag_sz is different for regular and XSK RQs, so ensure that linear @@ -30,9 +25,9 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, */ switch (params->rq_wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk); + return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk); default: /* MLX5_WQ_TYPE_CYCLIC */ - return mlx5e_rx_is_linear_skb(params, xsk); + return mlx5e_rx_is_linear_skb(mdev, params, xsk); } } @@ -71,7 +66,7 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, rq->xsk_pool = pool; rq->stats = &c->priv->channel_stats[c->ix]->xskrq; rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); - rq_xdp_ix = c->ix + params->num_channels * MLX5E_RQ_GROUP_XSK; + rq_xdp_ix = c->ix; err = mlx5e_rq_set_handlers(rq, params, xsk); if (err) return err; @@ -159,7 +154,7 @@ err_free_cparam: void mlx5e_close_xsk(struct mlx5e_channel *c) { clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state); - synchronize_net(); /* Sync with the XSK wakeup and with NAPI. */ + synchronize_net(); /* Sync with NAPI. */ mlx5e_close_rq(&c->xskrq); mlx5e_close_cq(&c->xskrq.cq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index 4902ef74fedfb06dd6d8d186dc66a7da08b79059..367a9505ca4f998b97e316ae17dc01111f3b04e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -12,18 +12,14 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_params *params = &priv->channels.params; struct mlx5e_channel *c; - u16 ix; if (unlikely(!mlx5e_xdp_is_active(priv))) return -ENETDOWN; - if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix))) + if (unlikely(qid >= params->num_channels)) return -EINVAL; - c = priv->channels.c[ix]; - - if (unlikely(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))) - return -EINVAL; + c = priv->channels.c[qid]; if (!napi_if_scheduled_mark_missed(&c->napi)) { /* To avoid WQE overrun, don't post a NOP if async_icosq is not @@ -36,9 +32,7 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) return 0; - spin_lock_bh(&c->async_icosq_lock); - mlx5e_trigger_irq(&c->async_icosq); - spin_unlock_bh(&c->async_icosq_lock); + mlx5e_trigger_napi_icosq(c); } return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h index a05085035f23515bb6a113a0475f2b9ff79931e3..9c505158b975b5ac0063c70d511d6753dbf136b4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h @@ -5,7 +5,6 @@ #define __MLX5_EN_XSK_TX_H__ #include "en.h" -#include /* TX data path */ @@ -13,15 +12,4 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags); bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget); -static inline void mlx5e_xsk_update_tx_wakeup(struct mlx5e_xdpsq *sq) -{ - if (!xsk_uses_need_wakeup(sq->xsk_pool)) - return; - - if (sq->pc != sq->cc) - xsk_clear_tx_need_wakeup(sq->xsk_pool); - else - xsk_set_tx_need_wakeup(sq->xsk_pool); -} - #endif /* __MLX5_EN_XSK_TX_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 1839f1ab1dddea27c39a86fffd087f4bfa15a83d..07187028f0d3587d57291bf18cac38d8e0b05184 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -39,6 +39,7 @@ #include "en_accel/ipsec_rxtx.h" #include "en_accel/ktls.h" #include "en_accel/ktls_txrx.h" +#include #include "en.h" #include "en/txrx.h" @@ -137,6 +138,15 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev, } #endif +#ifdef CONFIG_MLX5_EN_MACSEC + if (unlikely(mlx5e_macsec_skb_is_offload(skb))) { + struct mlx5e_priv *priv = netdev_priv(dev); + + if (unlikely(!mlx5e_macsec_handle_tx_skb(priv->macsec, skb))) + return false; + } +#endif + return true; } @@ -163,6 +173,11 @@ static inline void mlx5e_accel_tx_eseg(struct mlx5e_priv *priv, mlx5e_ipsec_tx_build_eseg(priv, skb, eseg); #endif +#ifdef CONFIG_MLX5_EN_MACSEC + if (unlikely(mlx5e_macsec_skb_is_offload(skb))) + mlx5e_macsec_tx_build_eseg(priv->macsec, skb, eseg); +#endif + #if IS_ENABLED(CONFIG_GENEVE) if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) mlx5e_tx_tunnel_accel(skb, eseg, ihs); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c index 20a4f1e585aff33354cb95e0f2d4b4fdf90c5cb6..285d32d2fd08ab1518dd9cdcfad6042fa28e60da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ -#include +#include #include "en_accel/fs_tcp.h" #include "fs_core.h" @@ -71,13 +71,13 @@ void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule) mlx5_del_flow_rules(rule); } -struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs, struct sock *sk, u32 tirn, uint32_t flow_tag) { + struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs); struct mlx5_flow_destination dest = {}; struct mlx5e_flow_table *ft = NULL; - struct mlx5e_accel_fs_tcp *fs_tcp; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *flow; struct mlx5_flow_spec *spec; @@ -86,19 +86,17 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, if (!spec) return ERR_PTR(-ENOMEM); - fs_tcp = priv->fs->accel_tcp; - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; switch (sk->sk_family) { case AF_INET: accel_fs_tcp_set_ipv4_flow(spec, sk); ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP]; - mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__, - &inet_sk(sk)->inet_rcv_saddr, - inet_sk(sk)->inet_sport, - &inet_sk(sk)->inet_daddr, - inet_sk(sk)->inet_dport); + fs_dbg(fs, "%s flow is %pI4:%d -> %pI4:%d\n", __func__, + &inet_sk(sk)->inet_rcv_saddr, + inet_sk(sk)->inet_sport, + &inet_sk(sk)->inet_daddr, + inet_sk(sk)->inet_dport); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: @@ -140,34 +138,32 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1); if (IS_ERR(flow)) - netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n", - PTR_ERR(flow)); + fs_err(fs, "mlx5_add_flow_rules() failed, flow is %ld\n", PTR_ERR(flow)); out: kvfree(spec); return flow; } -static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, +static int accel_fs_tcp_add_default_rule(struct mlx5e_flow_steering *fs, enum accel_fs_tcp_type type) { + struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); struct mlx5e_flow_table *accel_fs_t; struct mlx5_flow_destination dest; - struct mlx5e_accel_fs_tcp *fs_tcp; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; int err = 0; - fs_tcp = priv->fs->accel_tcp; accel_fs_t = &fs_tcp->tables[type]; - dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_accel2tt(type)); + dest = mlx5_ttc_get_default_dest(ttc, fs_accel2tt(type)); rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, - "%s: add default rule failed, accel_fs type=%d, err %d\n", - __func__, type, err); + fs_err(fs, "%s: add default rule failed, accel_fs type=%d, err %d\n", + __func__, type, err); return err; } @@ -265,9 +261,11 @@ out: return err; } -static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) +static int accel_fs_tcp_create_table(struct mlx5e_flow_steering *fs, enum accel_fs_tcp_type type) { - struct mlx5e_flow_table *ft = &priv->fs->accel_tcp->tables[type]; + struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs); + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false); + struct mlx5e_flow_table *ft = &accel_tcp->tables[type]; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -277,21 +275,21 @@ static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_ ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); + ft->t = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; return err; } - netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n", - ft->t->id, ft->t->level); + fs_dbg(fs, "Created fs accel table id %u level %u\n", + ft->t->id, ft->t->level); err = accel_fs_tcp_create_groups(ft, type); if (err) goto err; - err = accel_fs_tcp_add_default_rule(priv, type); + err = accel_fs_tcp_add_default_rule(fs, type); if (err) goto err; @@ -301,17 +299,18 @@ err: return err; } -static int accel_fs_tcp_disable(struct mlx5e_priv *priv) +static int accel_fs_tcp_disable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); int err, i; for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_accel2tt(i)); + err = mlx5_ttc_fwd_default_dest(ttc, fs_accel2tt(i)); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] default destination failed, err(%d)\n", - __func__, fs_accel2tt(i), err); + fs_err(fs, + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, fs_accel2tt(i), err); return err; } } @@ -319,32 +318,32 @@ static int accel_fs_tcp_disable(struct mlx5e_priv *priv) return 0; } -static int accel_fs_tcp_enable(struct mlx5e_priv *priv) +static int accel_fs_tcp_enable(struct mlx5e_flow_steering *fs) { + struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); struct mlx5_flow_destination dest = {}; int err, i; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { - dest.ft = priv->fs->accel_tcp->tables[i].t; + dest.ft = accel_tcp->tables[i].t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_accel2tt(i), &dest); + err = mlx5_ttc_fwd_dest(ttc, fs_accel2tt(i), &dest); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] destination to accel failed, err(%d)\n", - __func__, fs_accel2tt(i), err); + fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n", + __func__, fs_accel2tt(i), err); return err; } } return 0; } -static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) +static void accel_fs_tcp_destroy_table(struct mlx5e_flow_steering *fs, int i) { - struct mlx5e_accel_fs_tcp *fs_tcp; + struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs); - fs_tcp = priv->fs->accel_tcp; if (IS_ERR_OR_NULL(fs_tcp->tables[i].t)) return; @@ -353,40 +352,43 @@ static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) fs_tcp->tables[i].t = NULL; } -void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs) { + struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs); int i; - if (!priv->fs->accel_tcp) + if (!accel_tcp) return; - accel_fs_tcp_disable(priv); + accel_fs_tcp_disable(fs); for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) - accel_fs_tcp_destroy_table(priv, i); + accel_fs_tcp_destroy_table(fs, i); - kfree(priv->fs->accel_tcp); - priv->fs->accel_tcp = NULL; + kfree(accel_tcp); + mlx5e_fs_set_accel_tcp(fs, NULL); } -int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) +int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs) { + struct mlx5e_accel_fs_tcp *accel_tcp; int i, err; - if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) + if (!MLX5_CAP_FLOWTABLE_NIC_RX(mlx5e_fs_get_mdev(fs), ft_field_support.outer_ip_version)) return -EOPNOTSUPP; - priv->fs->accel_tcp = kzalloc(sizeof(*priv->fs->accel_tcp), GFP_KERNEL); - if (!priv->fs->accel_tcp) + accel_tcp = kvzalloc(sizeof(*accel_tcp), GFP_KERNEL); + if (!accel_tcp) return -ENOMEM; + mlx5e_fs_set_accel_tcp(fs, accel_tcp); for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { - err = accel_fs_tcp_create_table(priv, i); + err = accel_fs_tcp_create_table(fs, i); if (err) goto err_destroy_tables; } - err = accel_fs_tcp_enable(priv); + err = accel_fs_tcp_enable(fs); if (err) goto err_destroy_tables; @@ -394,9 +396,8 @@ int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) err_destroy_tables: while (--i >= 0) - accel_fs_tcp_destroy_table(priv, i); - - kfree(priv->fs->accel_tcp); - priv->fs->accel_tcp = NULL; + accel_fs_tcp_destroy_table(fs, i); + kfree(accel_tcp); + mlx5e_fs_set_accel_tcp(fs, NULL); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h index 5892358245437a136b2667cb189efd2657fd7041..a032bff482a6f79113ae9b387563b7256a258676 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h @@ -4,19 +4,19 @@ #ifndef __MLX5E_ACCEL_FS_TCP_H__ #define __MLX5E_ACCEL_FS_TCP_H__ -#include "en.h" +#include "en/fs.h" #ifdef CONFIG_MLX5_EN_TLS -int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv); -void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv); -struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, +int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs); +void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs); +struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs, struct sock *sk, u32 tirn, uint32_t flow_tag); void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule); #else -static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) { return 0; } -static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) {} -static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, +static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs) { return 0; } +static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs) {} +static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs, struct sock *sk, u32 tirn, uint32_t flow_tag) { return ERR_PTR(-EOPNOTSUPP); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index f8113fd23265daa7b2eca5e7d2aa296b1f135010..b859e4a4c744ff80f58e2680c2acd19dec6128b1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -174,6 +174,8 @@ static void rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type) static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) { + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); struct mlx5_flow_table_attr ft_attr = {}; struct mlx5e_accel_fs_esp_prot *fs_prot; struct mlx5e_accel_fs_esp *accel_esp; @@ -182,15 +184,14 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) accel_esp = priv->ipsec->rx_fs; fs_prot = &accel_esp->fs_prot[type]; - fs_prot->default_dest = - mlx5_ttc_get_default_dest(priv->fs->ttc, fs_esp2tt(type)); + mlx5_ttc_get_default_dest(ttc, fs_esp2tt(type)); ft_attr.max_fte = 1; ft_attr.autogroup.max_num_groups = 1; ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); if (IS_ERR(ft)) return PTR_ERR(ft); @@ -205,7 +206,7 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) ft_attr.prio = MLX5E_NIC_PRIO; ft_attr.autogroup.num_reserved_entries = 1; ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; @@ -230,6 +231,7 @@ err_add: static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); struct mlx5e_accel_fs_esp_prot *fs_prot; struct mlx5_flow_destination dest = {}; struct mlx5e_accel_fs_esp *accel_esp; @@ -249,7 +251,7 @@ static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(priv->fs->ttc, fs_esp2tt(type), &dest); + mlx5_ttc_fwd_dest(ttc, fs_esp2tt(type), &dest); skip: fs_prot->refcnt++; @@ -260,6 +262,7 @@ out: static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); struct mlx5e_accel_fs_esp_prot *fs_prot; struct mlx5e_accel_fs_esp *accel_esp; @@ -271,7 +274,7 @@ static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) goto out; /* disconnect */ - mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_esp2tt(type)); + mlx5_ttc_fwd_default_dest(ttc, fs_esp2tt(type)); /* remove FT */ rx_destroy(priv, type); @@ -385,7 +388,8 @@ static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, 0xff, 16); } - flow_act->ipsec_obj_id = ipsec_obj_id; + flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act->crypto.obj_id = ipsec_obj_id; flow_act->flags |= FLOW_ACT_NO_APPEND; } @@ -441,7 +445,7 @@ static int rx_add_rule(struct mlx5e_priv *priv, } flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; flow_act.modify_hdr = modify_hdr; @@ -497,7 +501,7 @@ static int tx_add_rule(struct mlx5e_priv *priv, MLX5_ETH_WQE_FT_META_IPSEC); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | - MLX5_FLOW_CONTEXT_ACTION_IPSEC_ENCRYPT; + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -573,7 +577,7 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) int err = -ENOMEM; ns = mlx5_get_flow_namespace(ipsec->mdev, - MLX5_FLOW_NAMESPACE_EGRESS_KERNEL); + MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); if (!ns) return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h index 0ae4e12ce5285bfef594c4359a026acb4a2ca480..1878a70b9031d9fb10dae48efcd377c1cd8a4729 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h @@ -39,9 +39,9 @@ #include "en.h" #include "en/txrx.h" -/* Bit31: IPsec marker, Bit30-24: IPsec syndrome, Bit23-0: IPsec obj id */ +/* Bit31: IPsec marker, Bit30: reserved, Bit29-24: IPsec syndrome, Bit23-0: IPsec obj id */ #define MLX5_IPSEC_METADATA_MARKER(metadata) (((metadata) >> 31) & 0x1) -#define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(6, 0)) +#define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(5, 0)) #define MLX5_IPSEC_METADATA_HANDLE(metadata) ((metadata) & GENMASK(23, 0)) struct mlx5e_accel_tx_ipsec_state { @@ -77,11 +77,6 @@ static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata)); } -static inline bool mlx5e_ipsec_is_tx_flow(struct mlx5e_accel_tx_ipsec_state *ipsec_st) -{ - return ipsec_st->x; -} - static inline bool mlx5e_ipsec_eseg_meta(struct mlx5_wqe_eth_seg *eseg) { return eseg->flow_table_metadata & cpu_to_be32(MLX5_ETH_WQE_FT_META_IPSEC); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index 30a70d13904688abbc0d25d5e883564b59917f79..da2184c94203654c7bf9d288c15f022844b76eb3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -92,6 +92,24 @@ static const struct tlsdev_ops mlx5e_ktls_ops = { .tls_dev_resync = mlx5e_ktls_resync, }; +bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) +{ + u8 max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); + + if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx)) + return false; + + /* Check the possibility to post the required ICOSQ WQEs. */ + if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS)) + return false; + if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS)) + return false; + if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_KTLS_GET_PROGRESS_WQEBBS)) + return false; + + return true; +} + void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; @@ -118,9 +136,9 @@ int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable) mutex_lock(&priv->state_lock); if (enable) - err = mlx5e_accel_fs_tcp_create(priv); + err = mlx5e_accel_fs_tcp_create(priv->fs); else - mlx5e_accel_fs_tcp_destroy(priv); + mlx5e_accel_fs_tcp_destroy(priv->fs); mutex_unlock(&priv->state_lock); return err; @@ -138,7 +156,7 @@ int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) return -ENOMEM; if (priv->netdev->features & NETIF_F_HW_TLS_RX) { - err = mlx5e_accel_fs_tcp_create(priv); + err = mlx5e_accel_fs_tcp_create(priv->fs); if (err) { destroy_workqueue(priv->tls->rx_wq); return err; @@ -154,7 +172,7 @@ void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv) return; if (priv->netdev->features & NETIF_F_HW_TLS_RX) - mlx5e_accel_fs_tcp_destroy(priv); + mlx5e_accel_fs_tcp_destroy(priv->fs); destroy_workqueue(priv->tls->rx_wq); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index 948400dee525a95ab72f9260f554150ee07b7f80..1c35045e41fb6d6def9f6f06481b10d4bdfe280d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -25,7 +25,8 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev) if (!MLX5_CAP_GEN(mdev, log_max_dek)) return false; - return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128); + return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) || + MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256)); } static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, @@ -36,6 +37,10 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, if (crypto_info->version == TLS_1_2_VERSION) return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128); break; + case TLS_CIPHER_AES_GCM_256: + if (crypto_info->version == TLS_1_2_VERSION) + return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256); + break; } return false; @@ -56,10 +61,7 @@ static inline bool mlx5e_is_ktls_tx(struct mlx5_core_dev *mdev) return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx); } -static inline bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) -{ - return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_rx); -} +bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev); struct mlx5e_tls_sw_stats { atomic64_t tx_tls_ctx; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 27483aa7be8a8e649cb69edf08a179400cfa3622..3e54834747ce6d697c28df34c7a9de66556c7550 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -43,7 +43,7 @@ struct mlx5e_ktls_rx_resync_ctx { }; struct mlx5e_ktls_offload_context_rx { - struct tls12_crypto_info_aes_gcm_128 crypto_info; + union mlx5e_crypto_info crypto_info; struct accel_rule rule; struct sock *sk; struct mlx5e_rq_stats *rq_stats; @@ -111,7 +111,7 @@ static void accel_rule_handle_work(struct work_struct *work) if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) goto out; - rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk, + rule = mlx5e_accel_fs_add_sk(accel_rule->priv->fs, priv_rx->sk, mlx5e_tir_get_tirn(&priv_rx->tir), MLX5_FS_DEFAULT_FLOW_TAG); if (!IS_ERR_OR_NULL(rule)) @@ -362,7 +362,6 @@ static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync, static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx, struct mlx5e_channel *c) { - struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info; struct mlx5e_ktls_resync_resp *ktls_resync; struct mlx5e_icosq *sq; bool trigger_poll; @@ -373,7 +372,31 @@ static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_r spin_lock_bh(&ktls_resync->lock); spin_lock_bh(&priv_rx->lock); - memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq)); + switch (priv_rx->crypto_info.crypto_info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: { + struct tls12_crypto_info_aes_gcm_128 *info = + &priv_rx->crypto_info.crypto_info_128; + + memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, + sizeof(info->rec_seq)); + break; + } + case TLS_CIPHER_AES_GCM_256: { + struct tls12_crypto_info_aes_gcm_256 *info = + &priv_rx->crypto_info.crypto_info_256; + + memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, + sizeof(info->rec_seq)); + break; + } + default: + WARN_ONCE(1, "Unsupported cipher type %u\n", + priv_rx->crypto_info.crypto_info.cipher_type); + spin_unlock_bh(&priv_rx->lock); + spin_unlock_bh(&ktls_resync->lock); + return; + } + if (list_empty(&priv_rx->list)) { list_add_tail(&priv_rx->list, &ktls_resync->list); trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); @@ -461,6 +484,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) { struct ethhdr *eth = (struct ethhdr *)(skb->data); struct net_device *netdev = rq->netdev; + struct net *net = dev_net(netdev); struct sock *sk = NULL; unsigned int datalen; struct iphdr *iph; @@ -475,7 +499,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) depth += sizeof(struct iphdr); th = (void *)iph + sizeof(struct iphdr); - sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, + sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, iph->saddr, th->source, iph->daddr, th->dest, netdev->ifindex); #if IS_ENABLED(CONFIG_IPV6) @@ -485,7 +509,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) depth += sizeof(struct ipv6hdr); th = (void *)ipv6h + sizeof(struct ipv6hdr); - sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &ipv6h->saddr, th->source, &ipv6h->daddr, ntohs(th->dest), netdev->ifindex, 0); @@ -603,8 +627,20 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, INIT_LIST_HEAD(&priv_rx->list); spin_lock_init(&priv_rx->lock); - priv_rx->crypto_info = - *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + switch (crypto_info->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + priv_rx->crypto_info.crypto_info_128 = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + break; + case TLS_CIPHER_AES_GCM_256: + priv_rx->crypto_info.crypto_info_256 = + *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info; + break; + default: + WARN_ONCE(1, "Unsupported cipher type %u\n", + crypto_info->cipher_type); + return -EOPNOTSUPP; + } rxq = mlx5e_ktls_sk_get_rxq(sk); priv_rx->rxq = rxq; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 0aef69527226443f6b15d3127c4155d246ff093e..2e0335246967b16d7130ec1c172455c4a287fd14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -93,7 +93,7 @@ struct mlx5e_ktls_offload_context_tx { bool ctx_post_pending; /* control / resync */ struct list_head list_node; /* member of the pool */ - struct tls12_crypto_info_aes_gcm_128 crypto_info; + union mlx5e_crypto_info crypto_info; struct tls_offload_context_tx *tx_ctx; struct mlx5_core_dev *mdev; struct mlx5e_tls_sw_stats *sw_stats; @@ -246,7 +246,7 @@ static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, struct list_head *list, int size) { - struct mlx5e_ktls_offload_context_tx *obj; + struct mlx5e_ktls_offload_context_tx *obj, *n; struct mlx5e_async_ctx *bulk_async; int i; @@ -255,7 +255,7 @@ static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, return; i = 0; - list_for_each_entry(obj, list, list_node) { + list_for_each_entry_safe(obj, n, list, list_node) { mlx5e_tls_priv_tx_cleanup(obj, &bulk_async[i]); i++; } @@ -485,8 +485,20 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, goto err_create_key; priv_tx->expected_seq = start_offload_tcp_sn; - priv_tx->crypto_info = - *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + switch (crypto_info->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + priv_tx->crypto_info.crypto_info_128 = + *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + break; + case TLS_CIPHER_AES_GCM_256: + priv_tx->crypto_info.crypto_info_256 = + *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info; + break; + default: + WARN_ONCE(1, "Unsupported cipher type %u\n", + crypto_info->cipher_type); + return -EOPNOTSUPP; + } priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx); mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); @@ -671,14 +683,31 @@ tx_post_resync_params(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, u64 rcd_sn) { - struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info; __be64 rn_be = cpu_to_be64(rcd_sn); bool skip_static_post; u16 rec_seq_sz; char *rec_seq; - rec_seq = info->rec_seq; - rec_seq_sz = sizeof(info->rec_seq); + switch (priv_tx->crypto_info.crypto_info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: { + struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info.crypto_info_128; + + rec_seq = info->rec_seq; + rec_seq_sz = sizeof(info->rec_seq); + break; + } + case TLS_CIPHER_AES_GCM_256: { + struct tls12_crypto_info_aes_gcm_256 *info = &priv_tx->crypto_info.crypto_info_256; + + rec_seq = info->rec_seq; + rec_seq_sz = sizeof(info->rec_seq); + break; + } + default: + WARN_ONCE(1, "Unsupported cipher type %u\n", + priv_tx->crypto_info.crypto_info.cipher_type); + return; + } skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz); if (!skip_static_post) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c index ac29aeb8af492fbbb4c093d74dd69951fee922b0..570a912dd6faf470c687511be3621cb17b090e51 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c @@ -21,7 +21,7 @@ enum { static void fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, - struct tls12_crypto_info_aes_gcm_128 *info, + union mlx5e_crypto_info *crypto_info, u32 key_id, u32 resync_tcp_sn) { char *initial_rn, *gcm_iv; @@ -32,7 +32,26 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, ctx = params->ctx; - EXTRACT_INFO_FIELDS; + switch (crypto_info->crypto_info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: { + struct tls12_crypto_info_aes_gcm_128 *info = + &crypto_info->crypto_info_128; + + EXTRACT_INFO_FIELDS; + break; + } + case TLS_CIPHER_AES_GCM_256: { + struct tls12_crypto_info_aes_gcm_256 *info = + &crypto_info->crypto_info_256; + + EXTRACT_INFO_FIELDS; + break; + } + default: + WARN_ONCE(1, "Unsupported cipher type %u\n", + crypto_info->crypto_info.cipher_type); + return; + } gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv); initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number); @@ -54,7 +73,7 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params, void mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, u16 pc, u32 sqn, - struct tls12_crypto_info_aes_gcm_128 *info, + union mlx5e_crypto_info *crypto_info, u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, bool fence, enum tls_offload_ctx_dir direction) { @@ -75,7 +94,7 @@ mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, ucseg->flags = MLX5_UMR_INLINE; ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16); - fill_static_params(&wqe->params, info, key_id, resync_tcp_sn); + fill_static_params(&wqe->params, crypto_info, key_id, resync_tcp_sn); } static void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h index 0dc715c4c10d75bc9ee187b8b7f2be4781625e72..3d79cd37989079b1a1414127fe89e721047003d2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h @@ -27,6 +27,12 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx); void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn); +union mlx5e_crypto_info { + struct tls_crypto_info crypto_info; + struct tls12_crypto_info_aes_gcm_128 crypto_info_128; + struct tls12_crypto_info_aes_gcm_256 crypto_info_256; +}; + struct mlx5e_set_tls_static_params_wqe { struct mlx5_wqe_ctrl_seg ctrl; struct mlx5_wqe_umr_ctrl_seg uctrl; @@ -72,7 +78,7 @@ struct mlx5e_get_tls_progress_params_wqe { void mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe, u16 pc, u32 sqn, - struct tls12_crypto_info_aes_gcm_128 *info, + union mlx5e_crypto_info *crypto_info, u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn, bool fence, enum tls_offload_ctx_dir direction); void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c new file mode 100644 index 0000000000000000000000000000000000000000..41970067917bf23f603d6faf6c53bfd8e25d8dec --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -0,0 +1,1870 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include +#include +#include + +#include "en.h" +#include "lib/aso.h" +#include "lib/mlx5.h" +#include "en_accel/macsec.h" +#include "en_accel/macsec_fs.h" + +#define MLX5_MACSEC_EPN_SCOPE_MID 0x80000000L +#define MLX5E_MACSEC_ASO_CTX_SZ MLX5_ST_SZ_BYTES(macsec_aso) + +enum mlx5_macsec_aso_event_arm { + MLX5E_ASO_EPN_ARM = BIT(0), +}; + +enum { + MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET, +}; + +struct mlx5e_macsec_handle { + struct mlx5e_macsec *macsec; + u32 obj_id; + u8 idx; +}; + +enum { + MLX5_MACSEC_EPN, +}; + +struct mlx5e_macsec_aso_out { + u8 event_arm; + u32 mode_param; +}; + +struct mlx5e_macsec_aso_in { + u8 mode; + u32 obj_id; +}; + +struct mlx5e_macsec_epn_state { + u32 epn_msb; + u8 epn_enabled; + u8 overlap; +}; + +struct mlx5e_macsec_async_work { + struct mlx5e_macsec *macsec; + struct mlx5_core_dev *mdev; + struct work_struct work; + u32 obj_id; +}; + +struct mlx5e_macsec_sa { + bool active; + u8 assoc_num; + u32 macsec_obj_id; + u32 enc_key_id; + u32 next_pn; + sci_t sci; + salt_t salt; + + struct rhash_head hash; + u32 fs_id; + union mlx5e_macsec_rule *macsec_rule; + struct rcu_head rcu_head; + struct mlx5e_macsec_epn_state epn_state; +}; + +struct mlx5e_macsec_rx_sc; +struct mlx5e_macsec_rx_sc_xarray_element { + u32 fs_id; + struct mlx5e_macsec_rx_sc *rx_sc; +}; + +struct mlx5e_macsec_rx_sc { + bool active; + sci_t sci; + struct mlx5e_macsec_sa *rx_sa[MACSEC_NUM_AN]; + struct list_head rx_sc_list_element; + struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element; + struct metadata_dst *md_dst; + struct rcu_head rcu_head; +}; + +struct mlx5e_macsec_umr { + dma_addr_t dma_addr; + u8 ctx[MLX5_ST_SZ_BYTES(macsec_aso)]; + u32 mkey; +}; + +struct mlx5e_macsec_aso { + /* ASO */ + struct mlx5_aso *maso; + /* Protects macsec ASO */ + struct mutex aso_lock; + /* UMR */ + struct mlx5e_macsec_umr *umr; + + u32 pdn; +}; + +static const struct rhashtable_params rhash_sci = { + .key_len = sizeof_field(struct mlx5e_macsec_sa, sci), + .key_offset = offsetof(struct mlx5e_macsec_sa, sci), + .head_offset = offsetof(struct mlx5e_macsec_sa, hash), + .automatic_shrinking = true, + .min_size = 1, +}; + +struct mlx5e_macsec_device { + const struct net_device *netdev; + struct mlx5e_macsec_sa *tx_sa[MACSEC_NUM_AN]; + struct list_head macsec_rx_sc_list_head; + unsigned char *dev_addr; + struct list_head macsec_device_list_element; +}; + +struct mlx5e_macsec { + struct list_head macsec_device_list_head; + int num_of_devices; + struct mlx5e_macsec_fs *macsec_fs; + struct mutex lock; /* Protects mlx5e_macsec internal contexts */ + + /* Tx sci -> fs id mapping handling */ + struct rhashtable sci_hash; /* sci -> mlx5e_macsec_sa */ + + /* Rx fs_id -> rx_sc mapping */ + struct xarray sc_xarray; + + struct mlx5_core_dev *mdev; + + /* Stats manage */ + struct mlx5e_macsec_stats stats; + + /* ASO */ + struct mlx5e_macsec_aso aso; + + struct notifier_block nb; + struct workqueue_struct *wq; +}; + +struct mlx5_macsec_obj_attrs { + u32 aso_pdn; + u32 next_pn; + __be64 sci; + u32 enc_key_id; + bool encrypt; + struct mlx5e_macsec_epn_state epn_state; + salt_t salt; + __be32 ssci; + bool replay_protect; + u32 replay_window; +}; + +struct mlx5_aso_ctrl_param { + u8 data_mask_mode; + u8 condition_0_operand; + u8 condition_1_operand; + u8 condition_0_offset; + u8 condition_1_offset; + u8 data_offset; + u8 condition_operand; + u32 condition_0_data; + u32 condition_0_mask; + u32 condition_1_data; + u32 condition_1_mask; + u64 bitwise_data; + u64 data_mask; +}; + +static int mlx5e_macsec_aso_reg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso) +{ + struct mlx5e_macsec_umr *umr; + struct device *dma_device; + dma_addr_t dma_addr; + int err; + + umr = kzalloc(sizeof(*umr), GFP_KERNEL); + if (!umr) { + err = -ENOMEM; + return err; + } + + dma_device = &mdev->pdev->dev; + dma_addr = dma_map_single(dma_device, umr->ctx, sizeof(umr->ctx), DMA_BIDIRECTIONAL); + err = dma_mapping_error(dma_device, dma_addr); + if (err) { + mlx5_core_err(mdev, "Can't map dma device, err=%d\n", err); + goto out_dma; + } + + err = mlx5e_create_mkey(mdev, aso->pdn, &umr->mkey); + if (err) { + mlx5_core_err(mdev, "Can't create mkey, err=%d\n", err); + goto out_mkey; + } + + umr->dma_addr = dma_addr; + + aso->umr = umr; + + return 0; + +out_mkey: + dma_unmap_single(dma_device, dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL); +out_dma: + kfree(umr); + return err; +} + +static void mlx5e_macsec_aso_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso) +{ + struct mlx5e_macsec_umr *umr = aso->umr; + + mlx5_core_destroy_mkey(mdev, umr->mkey); + dma_unmap_single(&mdev->pdev->dev, umr->dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL); + kfree(umr); +} + +static int macsec_set_replay_protection(struct mlx5_macsec_obj_attrs *attrs, void *aso_ctx) +{ + u8 window_sz; + + if (!attrs->replay_protect) + return 0; + + switch (attrs->replay_window) { + case 256: + window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_256BIT; + break; + case 128: + window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_128BIT; + break; + case 64: + window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_64BIT; + break; + case 32: + window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_32BIT; + break; + default: + return -EINVAL; + } + MLX5_SET(macsec_aso, aso_ctx, window_size, window_sz); + MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_REPLAY_PROTECTION); + + return 0; +} + +static int mlx5e_macsec_create_object(struct mlx5_core_dev *mdev, + struct mlx5_macsec_obj_attrs *attrs, + bool is_tx, + u32 *macsec_obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_macsec_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *aso_ctx; + void *obj; + int err; + + obj = MLX5_ADDR_OF(create_macsec_obj_in, in, macsec_object); + aso_ctx = MLX5_ADDR_OF(macsec_offload_obj, obj, macsec_aso); + + MLX5_SET(macsec_offload_obj, obj, confidentiality_en, attrs->encrypt); + MLX5_SET(macsec_offload_obj, obj, dekn, attrs->enc_key_id); + MLX5_SET(macsec_offload_obj, obj, aso_return_reg, MLX5_MACSEC_ASO_REG_C_4_5); + MLX5_SET(macsec_offload_obj, obj, macsec_aso_access_pd, attrs->aso_pdn); + MLX5_SET(macsec_aso, aso_ctx, mode_parameter, attrs->next_pn); + + /* Epn */ + if (attrs->epn_state.epn_enabled) { + void *salt_p; + int i; + + MLX5_SET(macsec_aso, aso_ctx, epn_event_arm, 1); + MLX5_SET(macsec_offload_obj, obj, epn_en, 1); + MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb); + MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap); + MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)attrs->ssci); + salt_p = MLX5_ADDR_OF(macsec_offload_obj, obj, salt); + for (i = 0; i < 3 ; i++) + memcpy((u32 *)salt_p + i, &attrs->salt.bytes[4 * (2 - i)], 4); + } else { + MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)(attrs->sci)); + } + + MLX5_SET(macsec_aso, aso_ctx, valid, 0x1); + if (is_tx) { + MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_INC_SN); + } else { + err = macsec_set_replay_protection(attrs, aso_ctx); + if (err) + return err; + } + + /* general object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) { + mlx5_core_err(mdev, + "MACsec offload: Failed to create MACsec object (err = %d)\n", + err); + return err; + } + + *macsec_obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + + return err; +} + +static void mlx5e_macsec_destroy_object(struct mlx5_core_dev *mdev, u32 macsec_obj_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_obj_id); + + mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static void mlx5e_macsec_cleanup_sa(struct mlx5e_macsec *macsec, + struct mlx5e_macsec_sa *sa, + bool is_tx) +{ + int action = (is_tx) ? MLX5_ACCEL_MACSEC_ACTION_ENCRYPT : + MLX5_ACCEL_MACSEC_ACTION_DECRYPT; + + if ((is_tx) && sa->fs_id) { + /* Make sure ongoing datapath readers sees a valid SA */ + rhashtable_remove_fast(&macsec->sci_hash, &sa->hash, rhash_sci); + sa->fs_id = 0; + } + + if (!sa->macsec_rule) + return; + + mlx5e_macsec_fs_del_rule(macsec->macsec_fs, sa->macsec_rule, action); + mlx5e_macsec_destroy_object(macsec->mdev, sa->macsec_obj_id); + sa->macsec_rule = NULL; +} + +static int mlx5e_macsec_init_sa(struct macsec_context *ctx, + struct mlx5e_macsec_sa *sa, + bool encrypt, + bool is_tx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec *macsec = priv->macsec; + struct mlx5_macsec_rule_attrs rule_attrs; + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5_macsec_obj_attrs obj_attrs; + union mlx5e_macsec_rule *macsec_rule; + struct macsec_key *key; + int err; + + obj_attrs.next_pn = sa->next_pn; + obj_attrs.sci = cpu_to_be64((__force u64)sa->sci); + obj_attrs.enc_key_id = sa->enc_key_id; + obj_attrs.encrypt = encrypt; + obj_attrs.aso_pdn = macsec->aso.pdn; + obj_attrs.epn_state = sa->epn_state; + + if (is_tx) { + obj_attrs.ssci = cpu_to_be32((__force u32)ctx->sa.tx_sa->ssci); + key = &ctx->sa.tx_sa->key; + } else { + obj_attrs.ssci = cpu_to_be32((__force u32)ctx->sa.rx_sa->ssci); + key = &ctx->sa.rx_sa->key; + } + + memcpy(&obj_attrs.salt, &key->salt, sizeof(key->salt)); + obj_attrs.replay_window = ctx->secy->replay_window; + obj_attrs.replay_protect = ctx->secy->replay_protect; + + err = mlx5e_macsec_create_object(mdev, &obj_attrs, is_tx, &sa->macsec_obj_id); + if (err) + return err; + + rule_attrs.macsec_obj_id = sa->macsec_obj_id; + rule_attrs.sci = sa->sci; + rule_attrs.assoc_num = sa->assoc_num; + rule_attrs.action = (is_tx) ? MLX5_ACCEL_MACSEC_ACTION_ENCRYPT : + MLX5_ACCEL_MACSEC_ACTION_DECRYPT; + + macsec_rule = mlx5e_macsec_fs_add_rule(macsec->macsec_fs, ctx, &rule_attrs, &sa->fs_id); + if (!macsec_rule) { + err = -ENOMEM; + goto destroy_macsec_object; + } + + sa->macsec_rule = macsec_rule; + + if (is_tx) { + err = rhashtable_insert_fast(&macsec->sci_hash, &sa->hash, rhash_sci); + if (err) + goto destroy_macsec_object_and_rule; + } + + return 0; + +destroy_macsec_object_and_rule: + mlx5e_macsec_cleanup_sa(macsec, sa, is_tx); +destroy_macsec_object: + mlx5e_macsec_destroy_object(mdev, sa->macsec_obj_id); + + return err; +} + +static struct mlx5e_macsec_rx_sc * +mlx5e_macsec_get_rx_sc_from_sc_list(const struct list_head *list, sci_t sci) +{ + struct mlx5e_macsec_rx_sc *iter; + + list_for_each_entry_rcu(iter, list, rx_sc_list_element) { + if (iter->sci == sci) + return iter; + } + + return NULL; +} + +static int mlx5e_macsec_update_rx_sa(struct mlx5e_macsec *macsec, + struct mlx5e_macsec_sa *rx_sa, + bool active) +{ + struct mlx5_core_dev *mdev = macsec->mdev; + struct mlx5_macsec_obj_attrs attrs; + int err = 0; + + if (rx_sa->active != active) + return 0; + + rx_sa->active = active; + if (!active) { + mlx5e_macsec_cleanup_sa(macsec, rx_sa, false); + return 0; + } + + attrs.sci = rx_sa->sci; + attrs.enc_key_id = rx_sa->enc_key_id; + err = mlx5e_macsec_create_object(mdev, &attrs, false, &rx_sa->macsec_obj_id); + if (err) + return err; + + return 0; +} + +static bool mlx5e_macsec_secy_features_validate(struct macsec_context *ctx) +{ + const struct net_device *netdev = ctx->netdev; + const struct macsec_secy *secy = ctx->secy; + + if (secy->validate_frames != MACSEC_VALIDATE_STRICT) { + netdev_err(netdev, + "MACsec offload is supported only when validate_frame is in strict mode\n"); + return false; + } + + if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) { + netdev_err(netdev, "MACsec offload is supported only when icv_len is %d\n", + MACSEC_DEFAULT_ICV_LEN); + return false; + } + + if (!secy->protect_frames) { + netdev_err(netdev, + "MACsec offload is supported only when protect_frames is set\n"); + return false; + } + + return true; +} + +static struct mlx5e_macsec_device * +mlx5e_macsec_get_macsec_device_context(const struct mlx5e_macsec *macsec, + const struct macsec_context *ctx) +{ + struct mlx5e_macsec_device *iter; + const struct list_head *list; + + list = &macsec->macsec_device_list_head; + list_for_each_entry_rcu(iter, list, macsec_device_list_element) { + if (iter->netdev == ctx->secy->netdev) + return iter; + } + + return NULL; +} + +static void update_macsec_epn(struct mlx5e_macsec_sa *sa, const struct macsec_key *key, + const pn_t *next_pn_halves) +{ + struct mlx5e_macsec_epn_state *epn_state = &sa->epn_state; + + sa->salt = key->salt; + epn_state->epn_enabled = 1; + epn_state->epn_msb = next_pn_halves->upper; + epn_state->overlap = next_pn_halves->lower < MLX5_MACSEC_EPN_SCOPE_MID ? 0 : 1; +} + +static int mlx5e_macsec_add_txsa(struct macsec_context *ctx) +{ + const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc; + const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct macsec_secy *secy = ctx->secy; + struct mlx5e_macsec_device *macsec_device; + struct mlx5_core_dev *mdev = priv->mdev; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_sa *tx_sa; + struct mlx5e_macsec *macsec; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EEXIST; + goto out; + } + + if (macsec_device->tx_sa[assoc_num]) { + netdev_err(ctx->netdev, "MACsec offload tx_sa: %d already exist\n", assoc_num); + err = -EEXIST; + goto out; + } + + tx_sa = kzalloc(sizeof(*tx_sa), GFP_KERNEL); + if (!tx_sa) { + err = -ENOMEM; + goto out; + } + + tx_sa->active = ctx_tx_sa->active; + tx_sa->next_pn = ctx_tx_sa->next_pn_halves.lower; + tx_sa->sci = secy->sci; + tx_sa->assoc_num = assoc_num; + + if (secy->xpn) + update_macsec_epn(tx_sa, &ctx_tx_sa->key, &ctx_tx_sa->next_pn_halves); + + err = mlx5_create_encryption_key(mdev, ctx->sa.key, secy->key_len, + MLX5_ACCEL_OBJ_MACSEC_KEY, + &tx_sa->enc_key_id); + if (err) + goto destroy_sa; + + macsec_device->tx_sa[assoc_num] = tx_sa; + if (!secy->operational || + assoc_num != tx_sc->encoding_sa || + !tx_sa->active) + goto out; + + err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true); + if (err) + goto destroy_encryption_key; + + mutex_unlock(&macsec->lock); + + return 0; + +destroy_encryption_key: + macsec_device->tx_sa[assoc_num] = NULL; + mlx5_destroy_encryption_key(mdev, tx_sa->enc_key_id); +destroy_sa: + kfree(tx_sa); +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_upd_txsa(struct macsec_context *ctx) +{ + const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc; + const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_sa *tx_sa; + struct mlx5e_macsec *macsec; + struct net_device *netdev; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + netdev = ctx->netdev; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + tx_sa = macsec_device->tx_sa[assoc_num]; + if (!tx_sa) { + netdev_err(netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num); + err = -EEXIST; + goto out; + } + + if (tx_sa->next_pn != ctx_tx_sa->next_pn_halves.lower) { + netdev_err(netdev, "MACsec offload: update TX sa %d PN isn't supported\n", + assoc_num); + err = -EINVAL; + goto out; + } + + if (tx_sa->active == ctx_tx_sa->active) + goto out; + + if (tx_sa->assoc_num != tx_sc->encoding_sa) + goto out; + + if (ctx_tx_sa->active) { + err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true); + if (err) + goto out; + } else { + if (!tx_sa->macsec_rule) { + err = -EINVAL; + goto out; + } + + mlx5e_macsec_cleanup_sa(macsec, tx_sa, true); + } + + tx_sa->active = ctx_tx_sa->active; +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_del_txsa(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_sa *tx_sa; + struct mlx5e_macsec *macsec; + int err = 0; + + mutex_lock(&priv->macsec->lock); + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + tx_sa = macsec_device->tx_sa[assoc_num]; + if (!tx_sa) { + netdev_err(ctx->netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num); + err = -EEXIST; + goto out; + } + + mlx5e_macsec_cleanup_sa(macsec, tx_sa, true); + mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id); + kfree_rcu(tx_sa); + macsec_device->tx_sa[assoc_num] = NULL; + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static u32 mlx5e_macsec_get_sa_from_hashtable(struct rhashtable *sci_hash, sci_t *sci) +{ + struct mlx5e_macsec_sa *macsec_sa; + u32 fs_id = 0; + + rcu_read_lock(); + macsec_sa = rhashtable_lookup(sci_hash, sci, rhash_sci); + if (macsec_sa) + fs_id = macsec_sa->fs_id; + rcu_read_unlock(); + + return fs_id; +} + +static int mlx5e_macsec_add_rxsc(struct macsec_context *ctx) +{ + struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc; + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec_rx_sc *rx_sc; + struct list_head *rx_sc_list; + struct mlx5e_macsec *macsec; + int err = 0; + + mutex_lock(&priv->macsec->lock); + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + rx_sc_list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(rx_sc_list, ctx_rx_sc->sci); + if (rx_sc) { + netdev_err(ctx->netdev, "MACsec offload: rx_sc (sci %lld) already exists\n", + ctx_rx_sc->sci); + err = -EEXIST; + goto out; + } + + rx_sc = kzalloc(sizeof(*rx_sc), GFP_KERNEL); + if (!rx_sc) { + err = -ENOMEM; + goto out; + } + + sc_xarray_element = kzalloc(sizeof(*sc_xarray_element), GFP_KERNEL); + if (!sc_xarray_element) { + err = -ENOMEM; + goto destroy_rx_sc; + } + + sc_xarray_element->rx_sc = rx_sc; + err = xa_alloc(&macsec->sc_xarray, &sc_xarray_element->fs_id, sc_xarray_element, + XA_LIMIT(1, USHRT_MAX), GFP_KERNEL); + if (err) + goto destroy_sc_xarray_elemenet; + + rx_sc->md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL); + if (!rx_sc->md_dst) { + err = -ENOMEM; + goto erase_xa_alloc; + } + + rx_sc->sci = ctx_rx_sc->sci; + rx_sc->active = ctx_rx_sc->active; + list_add_rcu(&rx_sc->rx_sc_list_element, rx_sc_list); + + rx_sc->sc_xarray_element = sc_xarray_element; + rx_sc->md_dst->u.macsec_info.sci = rx_sc->sci; + mutex_unlock(&macsec->lock); + + return 0; + +erase_xa_alloc: + xa_erase(&macsec->sc_xarray, sc_xarray_element->fs_id); +destroy_sc_xarray_elemenet: + kfree(sc_xarray_element); +destroy_rx_sc: + kfree(rx_sc); + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_upd_rxsc(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc; + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec_rx_sc *rx_sc; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int i; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx_rx_sc->sci); + if (!rx_sc) { + err = -EINVAL; + goto out; + } + + rx_sc->active = ctx_rx_sc->active; + if (rx_sc->active == ctx_rx_sc->active) + goto out; + + for (i = 0; i < MACSEC_NUM_AN; ++i) { + rx_sa = rx_sc->rx_sa[i]; + if (!rx_sa) + continue; + + err = mlx5e_macsec_update_rx_sa(macsec, rx_sa, rx_sa->active && ctx_rx_sc->active); + if (err) + goto out; + } + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_del_rxsc(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec_rx_sc *rx_sc; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int err = 0; + int i; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx->rx_sc->sci); + if (!rx_sc) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld doesn't exist\n", + ctx->sa.rx_sa->sc->sci); + err = -EINVAL; + goto out; + } + + for (i = 0; i < MACSEC_NUM_AN; ++i) { + rx_sa = rx_sc->rx_sa[i]; + if (!rx_sa) + continue; + + mlx5e_macsec_cleanup_sa(macsec, rx_sa, false); + mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id); + + kfree(rx_sa); + rx_sc->rx_sa[i] = NULL; + } + +/* + * At this point the relevant MACsec offload Rx rule already removed at + * mlx5e_macsec_cleanup_sa need to wait for datapath to finish current + * Rx related data propagating using xa_erase which uses rcu to sync, + * once fs_id is erased then this rx_sc is hidden from datapath. + */ + list_del_rcu(&rx_sc->rx_sc_list_element); + xa_erase(&macsec->sc_xarray, rx_sc->sc_xarray_element->fs_id); + metadata_dst_free(rx_sc->md_dst); + kfree(rx_sc->sc_xarray_element); + + kfree_rcu(rx_sc); + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_add_rxsa(struct macsec_context *ctx) +{ + const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + struct mlx5_core_dev *mdev = priv->mdev; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_rx_sc *rx_sc; + sci_t sci = ctx_rx_sa->sc->sci; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci); + if (!rx_sc) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld doesn't exist\n", + ctx->sa.rx_sa->sc->sci); + err = -EINVAL; + goto out; + } + + if (rx_sc->rx_sa[assoc_num]) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld rx_sa %d already exist\n", + sci, assoc_num); + err = -EEXIST; + goto out; + } + + rx_sa = kzalloc(sizeof(*rx_sa), GFP_KERNEL); + if (!rx_sa) { + err = -ENOMEM; + goto out; + } + + rx_sa->active = ctx_rx_sa->active; + rx_sa->next_pn = ctx_rx_sa->next_pn; + rx_sa->sci = sci; + rx_sa->assoc_num = assoc_num; + rx_sa->fs_id = rx_sc->sc_xarray_element->fs_id; + + if (ctx->secy->xpn) + update_macsec_epn(rx_sa, &ctx_rx_sa->key, &ctx_rx_sa->next_pn_halves); + + err = mlx5_create_encryption_key(mdev, ctx->sa.key, ctx->secy->key_len, + MLX5_ACCEL_OBJ_MACSEC_KEY, + &rx_sa->enc_key_id); + if (err) + goto destroy_sa; + + rx_sc->rx_sa[assoc_num] = rx_sa; + if (!rx_sa->active) + goto out; + + //TODO - add support for both authentication and encryption flows + err = mlx5e_macsec_init_sa(ctx, rx_sa, true, false); + if (err) + goto destroy_encryption_key; + + goto out; + +destroy_encryption_key: + rx_sc->rx_sa[assoc_num] = NULL; + mlx5_destroy_encryption_key(mdev, rx_sa->enc_key_id); +destroy_sa: + kfree(rx_sa); +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_upd_rxsa(struct macsec_context *ctx) +{ + const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_rx_sc *rx_sc; + sci_t sci = ctx_rx_sa->sc->sci; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci); + if (!rx_sc) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld doesn't exist\n", + ctx->sa.rx_sa->sc->sci); + err = -EINVAL; + goto out; + } + + rx_sa = rx_sc->rx_sa[assoc_num]; + if (rx_sa) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld rx_sa %d already exist\n", + sci, assoc_num); + err = -EEXIST; + goto out; + } + + if (rx_sa->next_pn != ctx_rx_sa->next_pn_halves.lower) { + netdev_err(ctx->netdev, + "MACsec offload update RX sa %d PN isn't supported\n", + assoc_num); + err = -EINVAL; + goto out; + } + + err = mlx5e_macsec_update_rx_sa(macsec, rx_sa, ctx_rx_sa->active); +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_del_rxsa(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + sci_t sci = ctx->sa.rx_sa->sc->sci; + struct mlx5e_macsec_rx_sc *rx_sc; + u8 assoc_num = ctx->sa.assoc_num; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int err = 0; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + list = &macsec_device->macsec_rx_sc_list_head; + rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci); + if (!rx_sc) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld doesn't exist\n", + ctx->sa.rx_sa->sc->sci); + err = -EINVAL; + goto out; + } + + rx_sa = rx_sc->rx_sa[assoc_num]; + if (rx_sa) { + netdev_err(ctx->netdev, + "MACsec offload rx_sc sci %lld rx_sa %d already exist\n", + sci, assoc_num); + err = -EEXIST; + goto out; + } + + mlx5e_macsec_cleanup_sa(macsec, rx_sa, false); + mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id); + kfree(rx_sa); + rx_sc->rx_sa[assoc_num] = NULL; + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_add_secy(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct net_device *dev = ctx->secy->netdev; + const struct net_device *netdev = ctx->netdev; + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec *macsec; + int err = 0; + + if (!mlx5e_macsec_secy_features_validate(ctx)) + return -EINVAL; + + mutex_lock(&priv->macsec->lock); + macsec = priv->macsec; + if (mlx5e_macsec_get_macsec_device_context(macsec, ctx)) { + netdev_err(netdev, "MACsec offload: MACsec net_device already exist\n"); + goto out; + } + + if (macsec->num_of_devices >= MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES) { + netdev_err(netdev, "Currently, only %d MACsec offload devices can be set\n", + MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES); + err = -EBUSY; + goto out; + } + + macsec_device = kzalloc(sizeof(*macsec_device), GFP_KERNEL); + if (!macsec_device) { + err = -ENOMEM; + goto out; + } + + macsec_device->dev_addr = kmemdup(dev->dev_addr, dev->addr_len, GFP_KERNEL); + if (!macsec_device->dev_addr) { + kfree(macsec_device); + err = -ENOMEM; + goto out; + } + + macsec_device->netdev = dev; + + INIT_LIST_HEAD_RCU(&macsec_device->macsec_rx_sc_list_head); + list_add_rcu(&macsec_device->macsec_device_list_element, &macsec->macsec_device_list_head); + + ++macsec->num_of_devices; +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int macsec_upd_secy_hw_address(struct macsec_context *ctx, + struct mlx5e_macsec_device *macsec_device) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct net_device *dev = ctx->secy->netdev; + struct mlx5e_macsec *macsec = priv->macsec; + struct mlx5e_macsec_rx_sc *rx_sc, *tmp; + struct mlx5e_macsec_sa *rx_sa; + struct list_head *list; + int i, err = 0; + + + list = &macsec_device->macsec_rx_sc_list_head; + list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) { + for (i = 0; i < MACSEC_NUM_AN; ++i) { + rx_sa = rx_sc->rx_sa[i]; + if (!rx_sa || !rx_sa->macsec_rule) + continue; + + mlx5e_macsec_cleanup_sa(macsec, rx_sa, false); + } + } + + list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) { + for (i = 0; i < MACSEC_NUM_AN; ++i) { + rx_sa = rx_sc->rx_sa[i]; + if (!rx_sa) + continue; + + if (rx_sa->active) { + err = mlx5e_macsec_init_sa(ctx, rx_sa, false, false); + if (err) + goto out; + } + } + } + + memcpy(macsec_device->dev_addr, dev->dev_addr, dev->addr_len); +out: + return err; +} + +/* this function is called from 2 macsec ops functions: + * macsec_set_mac_address – MAC address was changed, therefore we need to destroy + * and create new Tx contexts(macsec object + steering). + * macsec_changelink – in this case the tx SC or SecY may be changed, therefore need to + * destroy Tx and Rx contexts(macsec object + steering) + */ +static int mlx5e_macsec_upd_secy(struct macsec_context *ctx) +{ + const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc; + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + const struct net_device *dev = ctx->secy->netdev; + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec_sa *tx_sa; + struct mlx5e_macsec *macsec; + int i, err = 0; + + if (!mlx5e_macsec_secy_features_validate(ctx)) + return -EINVAL; + + mutex_lock(&priv->macsec->lock); + + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + goto out; + } + + /* if the dev_addr hasn't change, it mean the callback is from macsec_changelink */ + if (!memcmp(macsec_device->dev_addr, dev->dev_addr, dev->addr_len)) { + err = macsec_upd_secy_hw_address(ctx, macsec_device); + if (err) + goto out; + } + + for (i = 0; i < MACSEC_NUM_AN; ++i) { + tx_sa = macsec_device->tx_sa[i]; + if (!tx_sa) + continue; + + mlx5e_macsec_cleanup_sa(macsec, tx_sa, true); + } + + for (i = 0; i < MACSEC_NUM_AN; ++i) { + tx_sa = macsec_device->tx_sa[i]; + if (!tx_sa) + continue; + + if (tx_sa->assoc_num == tx_sc->encoding_sa && tx_sa->active) { + err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true); + if (err) + goto out; + } + } + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static int mlx5e_macsec_del_secy(struct macsec_context *ctx) +{ + struct mlx5e_priv *priv = netdev_priv(ctx->netdev); + struct mlx5e_macsec_device *macsec_device; + struct mlx5e_macsec_rx_sc *rx_sc, *tmp; + struct mlx5e_macsec_sa *rx_sa; + struct mlx5e_macsec_sa *tx_sa; + struct mlx5e_macsec *macsec; + struct list_head *list; + int err = 0; + int i; + + mutex_lock(&priv->macsec->lock); + macsec = priv->macsec; + macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx); + if (!macsec_device) { + netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n"); + err = -EINVAL; + + goto out; + } + + for (i = 0; i < MACSEC_NUM_AN; ++i) { + tx_sa = macsec_device->tx_sa[i]; + if (!tx_sa) + continue; + + mlx5e_macsec_cleanup_sa(macsec, tx_sa, true); + mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id); + kfree(tx_sa); + macsec_device->tx_sa[i] = NULL; + } + + list = &macsec_device->macsec_rx_sc_list_head; + list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) { + for (i = 0; i < MACSEC_NUM_AN; ++i) { + rx_sa = rx_sc->rx_sa[i]; + if (!rx_sa) + continue; + + mlx5e_macsec_cleanup_sa(macsec, rx_sa, false); + mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id); + kfree(rx_sa); + rx_sc->rx_sa[i] = NULL; + } + + list_del_rcu(&rx_sc->rx_sc_list_element); + + kfree_rcu(rx_sc); + } + + kfree(macsec_device->dev_addr); + macsec_device->dev_addr = NULL; + + list_del_rcu(&macsec_device->macsec_device_list_element); + --macsec->num_of_devices; + +out: + mutex_unlock(&macsec->lock); + + return err; +} + +static void macsec_build_accel_attrs(struct mlx5e_macsec_sa *sa, + struct mlx5_macsec_obj_attrs *attrs) +{ + attrs->epn_state.epn_msb = sa->epn_state.epn_msb; + attrs->epn_state.overlap = sa->epn_state.overlap; +} + +static void macsec_aso_build_wqe_ctrl_seg(struct mlx5e_macsec_aso *macsec_aso, + struct mlx5_wqe_aso_ctrl_seg *aso_ctrl, + struct mlx5_aso_ctrl_param *param) +{ + memset(aso_ctrl, 0, sizeof(*aso_ctrl)); + if (macsec_aso->umr->dma_addr) { + aso_ctrl->va_l = cpu_to_be32(macsec_aso->umr->dma_addr | ASO_CTRL_READ_EN); + aso_ctrl->va_h = cpu_to_be32((u64)macsec_aso->umr->dma_addr >> 32); + aso_ctrl->l_key = cpu_to_be32(macsec_aso->umr->mkey); + } + + if (!param) + return; + + aso_ctrl->data_mask_mode = param->data_mask_mode << 6; + aso_ctrl->condition_1_0_operand = param->condition_1_operand | + param->condition_0_operand << 4; + aso_ctrl->condition_1_0_offset = param->condition_1_offset | + param->condition_0_offset << 4; + aso_ctrl->data_offset_condition_operand = param->data_offset | + param->condition_operand << 6; + aso_ctrl->condition_0_data = cpu_to_be32(param->condition_0_data); + aso_ctrl->condition_0_mask = cpu_to_be32(param->condition_0_mask); + aso_ctrl->condition_1_data = cpu_to_be32(param->condition_1_data); + aso_ctrl->condition_1_mask = cpu_to_be32(param->condition_1_mask); + aso_ctrl->bitwise_data = cpu_to_be64(param->bitwise_data); + aso_ctrl->data_mask = cpu_to_be64(param->data_mask); +} + +static int mlx5e_macsec_modify_obj(struct mlx5_core_dev *mdev, struct mlx5_macsec_obj_attrs *attrs, + u32 macsec_id) +{ + u32 in[MLX5_ST_SZ_DW(modify_macsec_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(query_macsec_obj_out)]; + u64 modify_field_select = 0; + void *obj; + int err; + + /* General object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_id); + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) { + mlx5_core_err(mdev, "Query MACsec object failed (Object id %d), err = %d\n", + macsec_id, err); + return err; + } + + obj = MLX5_ADDR_OF(query_macsec_obj_out, out, macsec_object); + modify_field_select = MLX5_GET64(macsec_offload_obj, obj, modify_field_select); + + /* EPN */ + if (!(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP) || + !(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB)) { + mlx5_core_dbg(mdev, "MACsec object field is not modifiable (Object id %d)\n", + macsec_id); + return -EOPNOTSUPP; + } + + obj = MLX5_ADDR_OF(modify_macsec_obj_in, in, macsec_object); + MLX5_SET64(macsec_offload_obj, obj, modify_field_select, + MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP | MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB); + MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb); + MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap); + + /* General object fields set */ + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static void macsec_aso_build_ctrl(struct mlx5e_macsec_aso *aso, + struct mlx5_wqe_aso_ctrl_seg *aso_ctrl, + struct mlx5e_macsec_aso_in *in) +{ + struct mlx5_aso_ctrl_param param = {}; + + param.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT; + param.condition_0_operand = MLX5_ASO_ALWAYS_TRUE; + param.condition_1_operand = MLX5_ASO_ALWAYS_TRUE; + if (in->mode == MLX5_MACSEC_EPN) { + param.data_offset = MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET; + param.bitwise_data = BIT_ULL(54); + param.data_mask = param.bitwise_data; + } + macsec_aso_build_wqe_ctrl_seg(aso, aso_ctrl, ¶m); +} + +static int macsec_aso_set_arm_event(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec, + struct mlx5e_macsec_aso_in *in) +{ + struct mlx5e_macsec_aso *aso; + struct mlx5_aso_wqe *aso_wqe; + struct mlx5_aso *maso; + int err; + + aso = &macsec->aso; + maso = aso->maso; + + mutex_lock(&aso->aso_lock); + aso_wqe = mlx5_aso_get_wqe(maso); + mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id, + MLX5_ACCESS_ASO_OPC_MOD_MACSEC); + macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in); + mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); + err = mlx5_aso_poll_cq(maso, false); + mutex_unlock(&aso->aso_lock); + + return err; +} + +static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec, + struct mlx5e_macsec_aso_in *in, struct mlx5e_macsec_aso_out *out) +{ + struct mlx5e_macsec_aso *aso; + struct mlx5_aso_wqe *aso_wqe; + struct mlx5_aso *maso; + int err; + + aso = &macsec->aso; + maso = aso->maso; + + mutex_lock(&aso->aso_lock); + + aso_wqe = mlx5_aso_get_wqe(maso); + mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id, + MLX5_ACCESS_ASO_OPC_MOD_MACSEC); + macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL); + + mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl); + err = mlx5_aso_poll_cq(maso, false); + if (err) + goto err_out; + + if (MLX5_GET(macsec_aso, aso->umr->ctx, epn_event_arm)) + out->event_arm |= MLX5E_ASO_EPN_ARM; + + out->mode_param = MLX5_GET(macsec_aso, aso->umr->ctx, mode_parameter); + +err_out: + mutex_unlock(&aso->aso_lock); + return err; +} + +static struct mlx5e_macsec_sa *get_macsec_tx_sa_from_obj_id(const struct mlx5e_macsec *macsec, + const u32 obj_id) +{ + const struct list_head *device_list; + struct mlx5e_macsec_sa *macsec_sa; + struct mlx5e_macsec_device *iter; + int i; + + device_list = &macsec->macsec_device_list_head; + + list_for_each_entry(iter, device_list, macsec_device_list_element) { + for (i = 0; i < MACSEC_NUM_AN; ++i) { + macsec_sa = iter->tx_sa[i]; + if (!macsec_sa || !macsec_sa->active) + continue; + if (macsec_sa->macsec_obj_id == obj_id) + return macsec_sa; + } + } + + return NULL; +} + +static struct mlx5e_macsec_sa *get_macsec_rx_sa_from_obj_id(const struct mlx5e_macsec *macsec, + const u32 obj_id) +{ + const struct list_head *device_list, *sc_list; + struct mlx5e_macsec_rx_sc *mlx5e_rx_sc; + struct mlx5e_macsec_sa *macsec_sa; + struct mlx5e_macsec_device *iter; + int i; + + device_list = &macsec->macsec_device_list_head; + + list_for_each_entry(iter, device_list, macsec_device_list_element) { + sc_list = &iter->macsec_rx_sc_list_head; + list_for_each_entry(mlx5e_rx_sc, sc_list, rx_sc_list_element) { + for (i = 0; i < MACSEC_NUM_AN; ++i) { + macsec_sa = mlx5e_rx_sc->rx_sa[i]; + if (!macsec_sa || !macsec_sa->active) + continue; + if (macsec_sa->macsec_obj_id == obj_id) + return macsec_sa; + } + } + } + + return NULL; +} + +static void macsec_epn_update(struct mlx5e_macsec *macsec, struct mlx5_core_dev *mdev, + struct mlx5e_macsec_sa *sa, u32 obj_id, u32 mode_param) +{ + struct mlx5_macsec_obj_attrs attrs = {}; + struct mlx5e_macsec_aso_in in = {}; + + /* When the bottom of the replay protection window (mode_param) crosses 2^31 (half sequence + * number wraparound) hence mode_param > MLX5_MACSEC_EPN_SCOPE_MID the SW should update the + * esn_overlap to OLD (1). + * When the bottom of the replay protection window (mode_param) crosses 2^32 (full sequence + * number wraparound) hence mode_param < MLX5_MACSEC_EPN_SCOPE_MID since it did a + * wraparound, the SW should update the esn_overlap to NEW (0), and increment the esn_msb. + */ + + if (mode_param < MLX5_MACSEC_EPN_SCOPE_MID) { + sa->epn_state.epn_msb++; + sa->epn_state.overlap = 0; + } else { + sa->epn_state.overlap = 1; + } + + macsec_build_accel_attrs(sa, &attrs); + mlx5e_macsec_modify_obj(mdev, &attrs, obj_id); + + /* Re-set EPN arm event */ + in.obj_id = obj_id; + in.mode = MLX5_MACSEC_EPN; + macsec_aso_set_arm_event(mdev, macsec, &in); +} + +static void macsec_async_event(struct work_struct *work) +{ + struct mlx5e_macsec_async_work *async_work; + struct mlx5e_macsec_aso_out out = {}; + struct mlx5e_macsec_aso_in in = {}; + struct mlx5e_macsec_sa *macsec_sa; + struct mlx5e_macsec *macsec; + struct mlx5_core_dev *mdev; + u32 obj_id; + + async_work = container_of(work, struct mlx5e_macsec_async_work, work); + macsec = async_work->macsec; + mdev = async_work->mdev; + obj_id = async_work->obj_id; + macsec_sa = get_macsec_tx_sa_from_obj_id(macsec, obj_id); + if (!macsec_sa) { + macsec_sa = get_macsec_rx_sa_from_obj_id(macsec, obj_id); + if (!macsec_sa) { + mlx5_core_dbg(mdev, "MACsec SA is not found (SA object id %d)\n", obj_id); + goto out_async_work; + } + } + + /* Query MACsec ASO context */ + in.obj_id = obj_id; + macsec_aso_query(mdev, macsec, &in, &out); + + /* EPN case */ + if (macsec_sa->epn_state.epn_enabled && !(out.event_arm & MLX5E_ASO_EPN_ARM)) + macsec_epn_update(macsec, mdev, macsec_sa, obj_id, out.mode_param); + +out_async_work: + kfree(async_work); +} + +static int macsec_obj_change_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct mlx5e_macsec *macsec = container_of(nb, struct mlx5e_macsec, nb); + struct mlx5e_macsec_async_work *async_work; + struct mlx5_eqe_obj_change *obj_change; + struct mlx5_eqe *eqe = data; + u16 obj_type; + u32 obj_id; + + if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE) + return NOTIFY_DONE; + + obj_change = &eqe->data.obj_change; + obj_type = be16_to_cpu(obj_change->obj_type); + obj_id = be32_to_cpu(obj_change->obj_id); + + if (obj_type != MLX5_GENERAL_OBJECT_TYPES_MACSEC) + return NOTIFY_DONE; + + async_work = kzalloc(sizeof(*async_work), GFP_ATOMIC); + if (!async_work) + return NOTIFY_DONE; + + async_work->macsec = macsec; + async_work->mdev = macsec->mdev; + async_work->obj_id = obj_id; + + INIT_WORK(&async_work->work, macsec_async_event); + + WARN_ON(!queue_work(macsec->wq, &async_work->work)); + + return NOTIFY_OK; +} + +static int mlx5e_macsec_aso_init(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev) +{ + struct mlx5_aso *maso; + int err; + + err = mlx5_core_alloc_pd(mdev, &aso->pdn); + if (err) { + mlx5_core_err(mdev, + "MACsec offload: Failed to alloc pd for MACsec ASO, err=%d\n", + err); + return err; + } + + maso = mlx5_aso_create(mdev, aso->pdn); + if (IS_ERR(maso)) { + err = PTR_ERR(maso); + goto err_aso; + } + + err = mlx5e_macsec_aso_reg_mr(mdev, aso); + if (err) + goto err_aso_reg; + + mutex_init(&aso->aso_lock); + + aso->maso = maso; + + return 0; + +err_aso_reg: + mlx5_aso_destroy(maso); +err_aso: + mlx5_core_dealloc_pd(mdev, aso->pdn); + return err; +} + +static void mlx5e_macsec_aso_cleanup(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev) +{ + if (!aso) + return; + + mlx5e_macsec_aso_dereg_mr(mdev, aso); + + mlx5_aso_destroy(aso->maso); + + mlx5_core_dealloc_pd(mdev, aso->pdn); +} + +bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev) +{ + if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD)) + return false; + + if (!MLX5_CAP_GEN(mdev, log_max_dek)) + return false; + + if (!MLX5_CAP_MACSEC(mdev, log_max_macsec_offload)) + return false; + + if (!MLX5_CAP_FLOWTABLE_NIC_RX(mdev, macsec_decrypt) || + !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_remove_macsec)) + return false; + + if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, macsec_encrypt) || + !MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_macsec)) + return false; + + if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_encrypt) && + !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_encrypt)) + return false; + + if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_decrypt) && + !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_decrypt)) + return false; + + return true; +} + +void mlx5e_macsec_get_stats_fill(struct mlx5e_macsec *macsec, void *macsec_stats) +{ + mlx5e_macsec_fs_get_stats_fill(macsec->macsec_fs, macsec_stats); +} + +struct mlx5e_macsec_stats *mlx5e_macsec_get_stats(struct mlx5e_macsec *macsec) +{ + if (!macsec) + return NULL; + + return &macsec->stats; +} + +static const struct macsec_ops macsec_offload_ops = { + .mdo_add_txsa = mlx5e_macsec_add_txsa, + .mdo_upd_txsa = mlx5e_macsec_upd_txsa, + .mdo_del_txsa = mlx5e_macsec_del_txsa, + .mdo_add_rxsc = mlx5e_macsec_add_rxsc, + .mdo_upd_rxsc = mlx5e_macsec_upd_rxsc, + .mdo_del_rxsc = mlx5e_macsec_del_rxsc, + .mdo_add_rxsa = mlx5e_macsec_add_rxsa, + .mdo_upd_rxsa = mlx5e_macsec_upd_rxsa, + .mdo_del_rxsa = mlx5e_macsec_del_rxsa, + .mdo_add_secy = mlx5e_macsec_add_secy, + .mdo_upd_secy = mlx5e_macsec_upd_secy, + .mdo_del_secy = mlx5e_macsec_del_secy, +}; + +bool mlx5e_macsec_handle_tx_skb(struct mlx5e_macsec *macsec, struct sk_buff *skb) +{ + struct metadata_dst *md_dst = skb_metadata_dst(skb); + u32 fs_id; + + fs_id = mlx5e_macsec_get_sa_from_hashtable(&macsec->sci_hash, &md_dst->u.macsec_info.sci); + if (!fs_id) + goto err_out; + + return true; + +err_out: + dev_kfree_skb_any(skb); + return false; +} + +void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec, + struct sk_buff *skb, + struct mlx5_wqe_eth_seg *eseg) +{ + struct metadata_dst *md_dst = skb_metadata_dst(skb); + u32 fs_id; + + fs_id = mlx5e_macsec_get_sa_from_hashtable(&macsec->sci_hash, &md_dst->u.macsec_info.sci); + if (!fs_id) + return; + + eseg->flow_table_metadata = cpu_to_be32(MLX5_ETH_WQE_FT_META_MACSEC | fs_id << 2); +} + +void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{ + struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element; + u32 macsec_meta_data = be32_to_cpu(cqe->ft_metadata); + struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_macsec_rx_sc *rx_sc; + struct mlx5e_macsec *macsec; + u32 fs_id; + + macsec = priv->macsec; + if (!macsec) + return; + + fs_id = MLX5_MACSEC_METADATA_HANDLE(macsec_meta_data); + + rcu_read_lock(); + sc_xarray_element = xa_load(&macsec->sc_xarray, fs_id); + rx_sc = sc_xarray_element->rx_sc; + if (rx_sc) { + dst_hold(&rx_sc->md_dst->dst); + skb_dst_set(skb, &rx_sc->md_dst->dst); + } + + rcu_read_unlock(); +} + +void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv) +{ + struct net_device *netdev = priv->netdev; + + if (!mlx5e_is_macsec_device(priv->mdev)) + return; + + /* Enable MACsec */ + mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n"); + netdev->macsec_ops = &macsec_offload_ops; + netdev->features |= NETIF_F_HW_MACSEC; + netif_keep_dst(netdev); +} + +int mlx5e_macsec_init(struct mlx5e_priv *priv) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_macsec *macsec = NULL; + struct mlx5e_macsec_fs *macsec_fs; + int err; + + if (!mlx5e_is_macsec_device(priv->mdev)) { + mlx5_core_dbg(mdev, "Not a MACsec offload device\n"); + return 0; + } + + macsec = kzalloc(sizeof(*macsec), GFP_KERNEL); + if (!macsec) + return -ENOMEM; + + INIT_LIST_HEAD(&macsec->macsec_device_list_head); + mutex_init(&macsec->lock); + + err = rhashtable_init(&macsec->sci_hash, &rhash_sci); + if (err) { + mlx5_core_err(mdev, "MACsec offload: Failed to init SCI hash table, err=%d\n", + err); + goto err_hash; + } + + err = mlx5e_macsec_aso_init(&macsec->aso, priv->mdev); + if (err) { + mlx5_core_err(mdev, "MACsec offload: Failed to init aso, err=%d\n", err); + goto err_aso; + } + + macsec->wq = alloc_ordered_workqueue("mlx5e_macsec_%s", 0, priv->netdev->name); + if (!macsec->wq) { + err = -ENOMEM; + goto err_wq; + } + + xa_init_flags(&macsec->sc_xarray, XA_FLAGS_ALLOC1); + + priv->macsec = macsec; + + macsec->mdev = mdev; + + macsec_fs = mlx5e_macsec_fs_init(mdev, priv->netdev); + if (!macsec_fs) { + err = -ENOMEM; + goto err_out; + } + + macsec->macsec_fs = macsec_fs; + + macsec->nb.notifier_call = macsec_obj_change_event; + mlx5_notifier_register(mdev, &macsec->nb); + + mlx5_core_dbg(mdev, "MACsec attached to netdevice\n"); + + return 0; + +err_out: + destroy_workqueue(macsec->wq); +err_wq: + mlx5e_macsec_aso_cleanup(&macsec->aso, priv->mdev); +err_aso: + rhashtable_destroy(&macsec->sci_hash); +err_hash: + kfree(macsec); + priv->macsec = NULL; + return err; +} + +void mlx5e_macsec_cleanup(struct mlx5e_priv *priv) +{ + struct mlx5e_macsec *macsec = priv->macsec; + struct mlx5_core_dev *mdev = macsec->mdev; + + if (!macsec) + return; + + mlx5_notifier_unregister(mdev, &macsec->nb); + + mlx5e_macsec_fs_cleanup(macsec->macsec_fs); + + /* Cleanup workqueue */ + destroy_workqueue(macsec->wq); + + mlx5e_macsec_aso_cleanup(&macsec->aso, mdev); + + priv->macsec = NULL; + + rhashtable_destroy(&macsec->sci_hash); + + mutex_destroy(&macsec->lock); + + kfree(macsec); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h new file mode 100644 index 0000000000000000000000000000000000000000..d580b4a91253c80c612b5093d68cfc4487743044 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_ACCEL_MACSEC_H__ +#define __MLX5_EN_ACCEL_MACSEC_H__ + +#ifdef CONFIG_MLX5_EN_MACSEC + +#include +#include +#include + +/* Bit31 - 30: MACsec marker, Bit3-0: MACsec id */ +#define MLX5_MACSEC_METADATA_MARKER(metadata) ((((metadata) >> 30) & 0x3) == 0x1) +#define MLX5_MACSEC_METADATA_HANDLE(metadata) ((metadata) & GENMASK(3, 0)) + +struct mlx5e_priv; +struct mlx5e_macsec; + +struct mlx5e_macsec_stats { + u64 macsec_rx_pkts; + u64 macsec_rx_bytes; + u64 macsec_rx_pkts_drop; + u64 macsec_rx_bytes_drop; + u64 macsec_tx_pkts; + u64 macsec_tx_bytes; + u64 macsec_tx_pkts_drop; + u64 macsec_tx_bytes_drop; +}; + +void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv); +int mlx5e_macsec_init(struct mlx5e_priv *priv); +void mlx5e_macsec_cleanup(struct mlx5e_priv *priv); +bool mlx5e_macsec_handle_tx_skb(struct mlx5e_macsec *macsec, struct sk_buff *skb); +void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec, + struct sk_buff *skb, + struct mlx5_wqe_eth_seg *eseg); + +static inline bool mlx5e_macsec_skb_is_offload(struct sk_buff *skb) +{ + struct metadata_dst *md_dst = skb_metadata_dst(skb); + + return md_dst && (md_dst->type == METADATA_MACSEC); +} + +static inline bool mlx5e_macsec_is_rx_flow(struct mlx5_cqe64 *cqe) +{ + return MLX5_MACSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata)); +} + +void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, + struct mlx5_cqe64 *cqe); +bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev); +void mlx5e_macsec_get_stats_fill(struct mlx5e_macsec *macsec, void *macsec_stats); +struct mlx5e_macsec_stats *mlx5e_macsec_get_stats(struct mlx5e_macsec *macsec); + +#else + +static inline void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv) {} +static inline int mlx5e_macsec_init(struct mlx5e_priv *priv) { return 0; } +static inline void mlx5e_macsec_cleanup(struct mlx5e_priv *priv) {} +static inline bool mlx5e_macsec_skb_is_offload(struct sk_buff *skb) { return false; } +static inline bool mlx5e_macsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; } +static inline void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, + struct sk_buff *skb, + struct mlx5_cqe64 *cqe) +{} +static inline bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev) { return false; } +#endif /* CONFIG_MLX5_EN_MACSEC */ + +#endif /* __MLX5_ACCEL_EN_MACSEC_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c new file mode 100644 index 0000000000000000000000000000000000000000..13dc628b988a1d3c19eee87b244907d3f3f20d59 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c @@ -0,0 +1,1384 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include +#include +#include +#include "fs_core.h" +#include "en/fs.h" +#include "en_accel/macsec_fs.h" +#include "mlx5_core.h" + +/* MACsec TX flow steering */ +#define CRYPTO_NUM_MAXSEC_FTE BIT(15) +#define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1 + +#define TX_CRYPTO_TABLE_LEVEL 0 +#define TX_CRYPTO_TABLE_NUM_GROUPS 3 +#define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1 +#define TX_CRYPTO_TABLE_SA_GROUP_SIZE \ + (CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \ + CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE)) +#define TX_CHECK_TABLE_LEVEL 1 +#define TX_CHECK_TABLE_NUM_FTE 2 +#define RX_CRYPTO_TABLE_LEVEL 0 +#define RX_CHECK_TABLE_LEVEL 1 +#define RX_CHECK_TABLE_NUM_FTE 3 +#define RX_CRYPTO_TABLE_NUM_GROUPS 3 +#define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \ + ((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2) +#define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \ + (CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE) +#define RX_NUM_OF_RULES_PER_SA 2 + +#define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */ +#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23 +#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8 +#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5 +#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) +#define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8 +#define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN) + +/* MACsec RX flow steering */ +#define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E + +struct mlx5_sectag_header { + __be16 ethertype; + u8 tci_an; + u8 sl; + u32 pn; + u8 sci[MACSEC_SCI_LEN]; /* optional */ +} __packed; + +struct mlx5e_macsec_tx_rule { + struct mlx5_flow_handle *rule; + struct mlx5_pkt_reformat *pkt_reformat; + u32 fs_id; +}; + +struct mlx5e_macsec_tables { + struct mlx5e_flow_table ft_crypto; + struct mlx5_flow_handle *crypto_miss_rule; + + struct mlx5_flow_table *ft_check; + struct mlx5_flow_group *ft_check_group; + struct mlx5_fc *check_miss_rule_counter; + struct mlx5_flow_handle *check_miss_rule; + struct mlx5_fc *check_rule_counter; + + u32 refcnt; +}; + +struct mlx5e_macsec_tx { + struct mlx5_flow_handle *crypto_mke_rule; + struct mlx5_flow_handle *check_rule; + + struct ida tx_halloc; + + struct mlx5e_macsec_tables tables; +}; + +struct mlx5e_macsec_rx_rule { + struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA]; + struct mlx5_modify_hdr *meta_modhdr; +}; + +struct mlx5e_macsec_rx { + struct mlx5_flow_handle *check_rule[2]; + struct mlx5_pkt_reformat *check_rule_pkt_reformat[2]; + + struct mlx5e_macsec_tables tables; +}; + +union mlx5e_macsec_rule { + struct mlx5e_macsec_tx_rule tx_rule; + struct mlx5e_macsec_rx_rule rx_rule; +}; + +struct mlx5e_macsec_fs { + struct mlx5_core_dev *mdev; + struct net_device *netdev; + struct mlx5e_macsec_tx *tx_fs; + struct mlx5e_macsec_rx *rx_fs; +}; + +static void macsec_fs_tx_destroy(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + struct mlx5e_macsec_tables *tx_tables; + + tx_tables = &tx_fs->tables; + + /* Tx check table */ + if (tx_fs->check_rule) { + mlx5_del_flow_rules(tx_fs->check_rule); + tx_fs->check_rule = NULL; + } + + if (tx_tables->check_miss_rule) { + mlx5_del_flow_rules(tx_tables->check_miss_rule); + tx_tables->check_miss_rule = NULL; + } + + if (tx_tables->ft_check_group) { + mlx5_destroy_flow_group(tx_tables->ft_check_group); + tx_tables->ft_check_group = NULL; + } + + if (tx_tables->ft_check) { + mlx5_destroy_flow_table(tx_tables->ft_check); + tx_tables->ft_check = NULL; + } + + /* Tx crypto table */ + if (tx_fs->crypto_mke_rule) { + mlx5_del_flow_rules(tx_fs->crypto_mke_rule); + tx_fs->crypto_mke_rule = NULL; + } + + if (tx_tables->crypto_miss_rule) { + mlx5_del_flow_rules(tx_tables->crypto_miss_rule); + tx_tables->crypto_miss_rule = NULL; + } + + mlx5e_destroy_flow_table(&tx_tables->ft_crypto); +} + +static int macsec_fs_tx_create_crypto_table_groups(struct mlx5e_flow_table *ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + int mclen = MLX5_ST_SZ_BYTES(fte_match_param); + int ix = 0; + u32 *in; + int err; + u8 *mc; + + ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); + if (!ft->g) + return -ENOMEM; + in = kvzalloc(inlen, GFP_KERNEL); + + if (!in) { + kfree(ft->g); + return -ENOMEM; + } + + mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + /* Flow Group for MKE match */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); + + MLX5_SET_CFG(in, start_flow_index, ix); + ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Flow Group for SA rules */ + memset(in, 0, inlen); + memset(mc, 0, mclen); + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2); + MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a, + MLX5_ETH_WQE_FT_META_MACSEC_MASK); + + MLX5_SET_CFG(in, start_flow_index, ix); + ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Flow Group for l2 traps */ + memset(in, 0, inlen); + memset(mc, 0, mclen); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + kvfree(in); + return 0; + +err: + err = PTR_ERR(ft->g[ft->num_groups]); + ft->g[ft->num_groups] = NULL; + kvfree(in); + + return err; +} + +static struct mlx5_flow_table + *macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags, + int level, int max_fte) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_table *fdb = NULL; + + /* reserve entry for the match all miss group and rule */ + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = 1; + ft_attr.prio = 0; + ft_attr.flags = flags; + ft_attr.level = level; + ft_attr.max_fte = max_fte; + + fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + + return fdb; +} + +static int macsec_fs_tx_create(struct mlx5e_macsec_fs *macsec_fs) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + struct net_device *netdev = macsec_fs->netdev; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_destination dest = {}; + struct mlx5e_macsec_tables *tx_tables; + struct mlx5_flow_act flow_act = {}; + struct mlx5e_flow_table *ft_crypto; + struct mlx5_flow_table *flow_table; + struct mlx5_flow_group *flow_group; + struct mlx5_flow_namespace *ns; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + u32 *flow_group_in; + int err = 0; + + ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); + if (!ns) + return -ENOMEM; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + goto out_spec; + + tx_tables = &tx_fs->tables; + ft_crypto = &tx_tables->ft_crypto; + + /* Tx crypto table */ + ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; + ft_attr.level = TX_CRYPTO_TABLE_LEVEL; + ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; + + flow_table = mlx5_create_flow_table(ns, &ft_attr); + if (IS_ERR(flow_table)) { + err = PTR_ERR(flow_table); + netdev_err(netdev, "Failed to create MACsec Tx crypto table err(%d)\n", err); + goto out_flow_group; + } + ft_crypto->t = flow_table; + + /* Tx crypto table groups */ + err = macsec_fs_tx_create_crypto_table_groups(ft_crypto); + if (err) { + netdev_err(netdev, + "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", + err); + goto err; + } + + /* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */ + memset(&flow_act, 0, sizeof(flow_act)); + memset(spec, 0, sizeof(*spec)); + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; + + rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, NULL, 0); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to add MACsec TX MKE rule, err=%d\n", err); + goto err; + } + tx_fs->crypto_mke_rule = rule; + + /* Tx crypto table Default miss rule */ + memset(&flow_act, 0, sizeof(flow_act)); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; + rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to add MACsec Tx table default miss rule %d\n", err); + goto err; + } + tx_tables->crypto_miss_rule = rule; + + /* Tx check table */ + flow_table = macsec_fs_auto_group_table_create(ns, 0, TX_CHECK_TABLE_LEVEL, + TX_CHECK_TABLE_NUM_FTE); + if (IS_ERR(flow_table)) { + err = PTR_ERR(flow_table); + netdev_err(netdev, "fail to create MACsec TX check table, err(%d)\n", err); + goto err; + } + tx_tables->ft_check = flow_table; + + /* Tx check table Default miss group/rule */ + memset(flow_group_in, 0, inlen); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); + flow_group = mlx5_create_flow_group(tx_tables->ft_check, flow_group_in); + if (IS_ERR(flow_group)) { + err = PTR_ERR(flow_group); + netdev_err(netdev, + "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", + err); + goto err; + } + tx_tables->ft_check_group = flow_group; + + /* Tx check table default drop rule */ + memset(&dest, 0, sizeof(struct mlx5_flow_destination)); + memset(&flow_act, 0, sizeof(flow_act)); + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(tx_tables->check_miss_rule_counter); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + rule = mlx5_add_flow_rules(tx_tables->ft_check, NULL, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err); + goto err; + } + tx_tables->check_miss_rule = rule; + + /* Tx check table rule */ + memset(spec, 0, sizeof(struct mlx5_flow_spec)); + memset(&dest, 0, sizeof(struct mlx5_flow_destination)); + memset(&flow_act, 0, sizeof(flow_act)); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; + + flow_act.flags = FLOW_ACT_NO_APPEND; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(tx_tables->check_rule_counter); + rule = mlx5_add_flow_rules(tx_tables->ft_check, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to add MACsec check rule, err=%d\n", err); + goto err; + } + tx_fs->check_rule = rule; + + goto out_flow_group; + +err: + macsec_fs_tx_destroy(macsec_fs); +out_flow_group: + kvfree(flow_group_in); +out_spec: + kvfree(spec); + return err; +} + +static int macsec_fs_tx_ft_get(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + struct mlx5e_macsec_tables *tx_tables; + int err = 0; + + tx_tables = &tx_fs->tables; + if (tx_tables->refcnt) + goto out; + + err = macsec_fs_tx_create(macsec_fs); + if (err) + return err; + +out: + tx_tables->refcnt++; + return err; +} + +static void macsec_fs_tx_ft_put(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; + + if (--tx_tables->refcnt) + return; + + macsec_fs_tx_destroy(macsec_fs); +} + +static int macsec_fs_tx_setup_fte(struct mlx5e_macsec_fs *macsec_fs, + struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + u32 macsec_obj_id, + u32 *fs_id) +{ + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + int err = 0; + u32 id; + + err = ida_alloc_range(&tx_fs->tx_halloc, 1, + MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES, + GFP_KERNEL); + if (err < 0) + return err; + + id = err; + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + /* Metadata match */ + MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, + MLX5_ETH_WQE_FT_META_MACSEC_MASK); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, + MLX5_ETH_WQE_FT_META_MACSEC | id << 2); + + *fs_id = id; + flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; + flow_act->crypto.obj_id = macsec_obj_id; + + mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id); + return 0; +} + +static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx, + char *reformatbf, + size_t *reformat_size) +{ + const struct macsec_secy *secy = ctx->secy; + bool sci_present = macsec_send_sci(secy); + struct mlx5_sectag_header sectag = {}; + const struct macsec_tx_sc *tx_sc; + + tx_sc = &secy->tx_sc; + sectag.ethertype = htons(ETH_P_MACSEC); + + if (sci_present) { + sectag.tci_an |= MACSEC_TCI_SC; + memcpy(§ag.sci, &secy->sci, + sizeof(sectag.sci)); + } else { + if (tx_sc->end_station) + sectag.tci_an |= MACSEC_TCI_ES; + if (tx_sc->scb) + sectag.tci_an |= MACSEC_TCI_SCB; + } + + /* With GCM, C/E clear for !encrypt, both set for encrypt */ + if (tx_sc->encrypt) + sectag.tci_an |= MACSEC_TCI_CONFID; + else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) + sectag.tci_an |= MACSEC_TCI_C; + + sectag.tci_an |= tx_sc->encoding_sa; + + *reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0); + + memcpy(reformatbf, §ag, *reformat_size); +} + +static void macsec_fs_tx_del_rule(struct mlx5e_macsec_fs *macsec_fs, + struct mlx5e_macsec_tx_rule *tx_rule) +{ + if (tx_rule->rule) { + mlx5_del_flow_rules(tx_rule->rule); + tx_rule->rule = NULL; + } + + if (tx_rule->pkt_reformat) { + mlx5_packet_reformat_dealloc(macsec_fs->mdev, tx_rule->pkt_reformat); + tx_rule->pkt_reformat = NULL; + } + + if (tx_rule->fs_id) { + ida_free(&macsec_fs->tx_fs->tx_halloc, tx_rule->fs_id); + tx_rule->fs_id = 0; + } + + kfree(tx_rule); + + macsec_fs_tx_ft_put(macsec_fs); +} + +static union mlx5e_macsec_rule * +macsec_fs_tx_add_rule(struct mlx5e_macsec_fs *macsec_fs, + const struct macsec_context *macsec_ctx, + struct mlx5_macsec_rule_attrs *attrs, + u32 *sa_fs_id) +{ + char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN]; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + struct net_device *netdev = macsec_fs->netdev; + union mlx5e_macsec_rule *macsec_rule = NULL; + struct mlx5_flow_destination dest = {}; + struct mlx5e_macsec_tables *tx_tables; + struct mlx5e_macsec_tx_rule *tx_rule; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + size_t reformat_size; + int err = 0; + u32 fs_id; + + tx_tables = &tx_fs->tables; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return NULL; + + err = macsec_fs_tx_ft_get(macsec_fs); + if (err) + goto out_spec; + + macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); + if (!macsec_rule) { + macsec_fs_tx_ft_put(macsec_fs); + goto out_spec; + } + + tx_rule = &macsec_rule->tx_rule; + + /* Tx crypto table crypto rule */ + macsec_fs_tx_create_sectag_header(macsec_ctx, reformatbf, &reformat_size); + + reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC; + reformat_params.size = reformat_size; + reformat_params.data = reformatbf; + flow_act.pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev, + &reformat_params, + MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); + if (IS_ERR(flow_act.pkt_reformat)) { + err = PTR_ERR(flow_act.pkt_reformat); + netdev_err(netdev, "Failed to allocate MACsec Tx reformat context err=%d\n", err); + goto err; + } + tx_rule->pkt_reformat = flow_act.pkt_reformat; + + err = macsec_fs_tx_setup_fte(macsec_fs, spec, &flow_act, attrs->macsec_obj_id, &fs_id); + if (err) { + netdev_err(netdev, + "Failed to add packet reformat for MACsec TX crypto rule, err=%d\n", + err); + goto err; + } + + tx_rule->fs_id = fs_id; + *sa_fs_id = fs_id; + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = tx_tables->ft_check; + rule = mlx5_add_flow_rules(tx_tables->ft_crypto.t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to add MACsec TX crypto rule, err=%d\n", err); + goto err; + } + tx_rule->rule = rule; + + goto out_spec; + +err: + macsec_fs_tx_del_rule(macsec_fs, tx_rule); + macsec_rule = NULL; +out_spec: + kvfree(spec); + + return macsec_rule; +} + +static void macsec_fs_tx_cleanup(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs; + struct mlx5_core_dev *mdev = macsec_fs->mdev; + struct mlx5e_macsec_tables *tx_tables; + + if (!tx_fs) + return; + + tx_tables = &tx_fs->tables; + if (tx_tables->refcnt) { + netdev_err(macsec_fs->netdev, + "Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n", + tx_tables->refcnt); + return; + } + + ida_destroy(&tx_fs->tx_halloc); + + if (tx_tables->check_miss_rule_counter) { + mlx5_fc_destroy(mdev, tx_tables->check_miss_rule_counter); + tx_tables->check_miss_rule_counter = NULL; + } + + if (tx_tables->check_rule_counter) { + mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); + tx_tables->check_rule_counter = NULL; + } + + kfree(tx_fs); + macsec_fs->tx_fs = NULL; +} + +static int macsec_fs_tx_init(struct mlx5e_macsec_fs *macsec_fs) +{ + struct net_device *netdev = macsec_fs->netdev; + struct mlx5_core_dev *mdev = macsec_fs->mdev; + struct mlx5e_macsec_tables *tx_tables; + struct mlx5e_macsec_tx *tx_fs; + struct mlx5_fc *flow_counter; + int err; + + tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL); + if (!tx_fs) + return -ENOMEM; + + tx_tables = &tx_fs->tables; + + flow_counter = mlx5_fc_create(mdev, false); + if (IS_ERR(flow_counter)) { + err = PTR_ERR(flow_counter); + netdev_err(netdev, + "Failed to create MACsec Tx encrypt flow counter, err(%d)\n", + err); + goto err_encrypt_counter; + } + tx_tables->check_rule_counter = flow_counter; + + flow_counter = mlx5_fc_create(mdev, false); + if (IS_ERR(flow_counter)) { + err = PTR_ERR(flow_counter); + netdev_err(netdev, + "Failed to create MACsec Tx drop flow counter, err(%d)\n", + err); + goto err_drop_counter; + } + tx_tables->check_miss_rule_counter = flow_counter; + + ida_init(&tx_fs->tx_halloc); + + macsec_fs->tx_fs = tx_fs; + + return 0; + +err_drop_counter: + mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); + tx_tables->check_rule_counter = NULL; + +err_encrypt_counter: + kfree(tx_fs); + macsec_fs->tx_fs = NULL; + + return err; +} + +static void macsec_fs_rx_destroy(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs; + struct mlx5e_macsec_tables *rx_tables; + int i; + + /* Rx check table */ + for (i = 1; i >= 0; --i) { + if (rx_fs->check_rule[i]) { + mlx5_del_flow_rules(rx_fs->check_rule[i]); + rx_fs->check_rule[i] = NULL; + } + + if (rx_fs->check_rule_pkt_reformat[i]) { + mlx5_packet_reformat_dealloc(macsec_fs->mdev, + rx_fs->check_rule_pkt_reformat[i]); + rx_fs->check_rule_pkt_reformat[i] = NULL; + } + } + + rx_tables = &rx_fs->tables; + + if (rx_tables->check_miss_rule) { + mlx5_del_flow_rules(rx_tables->check_miss_rule); + rx_tables->check_miss_rule = NULL; + } + + if (rx_tables->ft_check_group) { + mlx5_destroy_flow_group(rx_tables->ft_check_group); + rx_tables->ft_check_group = NULL; + } + + if (rx_tables->ft_check) { + mlx5_destroy_flow_table(rx_tables->ft_check); + rx_tables->ft_check = NULL; + } + + /* Rx crypto table */ + if (rx_tables->crypto_miss_rule) { + mlx5_del_flow_rules(rx_tables->crypto_miss_rule); + rx_tables->crypto_miss_rule = NULL; + } + + mlx5e_destroy_flow_table(&rx_tables->ft_crypto); +} + +static int macsec_fs_rx_create_crypto_table_groups(struct mlx5e_flow_table *ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + int mclen = MLX5_ST_SZ_BYTES(fte_match_param); + int ix = 0; + u32 *in; + int err; + u8 *mc; + + ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); + if (!ft->g) + return -ENOMEM; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + kfree(ft->g); + return -ENOMEM; + } + + mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); + + /* Flow group for SA rule with SCI */ + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | + MLX5_MATCH_MISC_PARAMETERS_5); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); + + MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, + MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << + MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2); + MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3); + + MLX5_SET_CFG(in, start_flow_index, ix); + ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Flow group for SA rule without SCI */ + memset(in, 0, inlen); + memset(mc, 0, mclen); + MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | + MLX5_MATCH_MISC_PARAMETERS_5); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0); + MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); + + MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, + MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + + MLX5_SET_CFG(in, start_flow_index, ix); + ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + /* Flow Group for l2 traps */ + memset(in, 0, inlen); + memset(mc, 0, mclen); + MLX5_SET_CFG(in, start_flow_index, ix); + ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; + MLX5_SET_CFG(in, end_flow_index, ix - 1); + ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); + if (IS_ERR(ft->g[ft->num_groups])) + goto err; + ft->num_groups++; + + kvfree(in); + return 0; + +err: + err = PTR_ERR(ft->g[ft->num_groups]); + ft->g[ft->num_groups] = NULL; + kvfree(in); + + return err; +} + +static int macsec_fs_rx_create_check_decap_rule(struct mlx5e_macsec_fs *macsec_fs, + struct mlx5_flow_destination *dest, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_spec *spec, + int reformat_param_size) +{ + int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1; + u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI]; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs; + struct net_device *netdev = macsec_fs->netdev; + struct mlx5e_macsec_tables *rx_tables; + struct mlx5_flow_handle *rule; + int err = 0; + + rx_tables = &rx_fs->tables; + + /* Rx check table decap 16B rule */ + memset(dest, 0, sizeof(*dest)); + memset(flow_act, 0, sizeof(*flow_act)); + memset(spec, 0, sizeof(*spec)); + + reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC; + reformat_params.size = reformat_param_size; + reformat_params.data = mlx5_reformat_buf; + flow_act->pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev, + &reformat_params, + MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); + if (IS_ERR(flow_act->pkt_reformat)) { + err = PTR_ERR(flow_act->pkt_reformat); + netdev_err(netdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err); + return err; + } + rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat; + + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; + /* MACsec syndrome match */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0); + /* ASO return reg syndrome match */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); + + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; + /* Sectag TCI SC present bit*/ + MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, + MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + + if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, + MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << + MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + + flow_act->flags = FLOW_ACT_NO_APPEND; + flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO | + MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest->type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest->counter_id = mlx5_fc_id(rx_tables->check_rule_counter); + rule = mlx5_add_flow_rules(rx_tables->ft_check, spec, flow_act, dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to add MACsec Rx check rule, err=%d\n", err); + return err; + } + + rx_fs->check_rule[rule_index] = rule; + + return 0; +} + +static int macsec_fs_rx_create(struct mlx5e_macsec_fs *macsec_fs) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs; + struct net_device *netdev = macsec_fs->netdev; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_destination dest = {}; + struct mlx5e_macsec_tables *rx_tables; + struct mlx5e_flow_table *ft_crypto; + struct mlx5_flow_table *flow_table; + struct mlx5_flow_group *flow_group; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_namespace *ns; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + u32 *flow_group_in; + int err = 0; + + ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); + if (!ns) + return -ENOMEM; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + goto free_spec; + + rx_tables = &rx_fs->tables; + ft_crypto = &rx_tables->ft_crypto; + + /* Rx crypto table */ + ft_attr.level = RX_CRYPTO_TABLE_LEVEL; + ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; + + flow_table = mlx5_create_flow_table(ns, &ft_attr); + if (IS_ERR(flow_table)) { + err = PTR_ERR(flow_table); + netdev_err(netdev, "Failed to create MACsec Rx crypto table err(%d)\n", err); + goto out_flow_group; + } + ft_crypto->t = flow_table; + + /* Rx crypto table groups */ + err = macsec_fs_rx_create_crypto_table_groups(ft_crypto); + if (err) { + netdev_err(netdev, + "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", + err); + goto err; + } + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; + rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, + "Failed to add MACsec Rx crypto table default miss rule %d\n", + err); + goto err; + } + rx_tables->crypto_miss_rule = rule; + + /* Rx check table */ + flow_table = macsec_fs_auto_group_table_create(ns, + MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT, + RX_CHECK_TABLE_LEVEL, + RX_CHECK_TABLE_NUM_FTE); + if (IS_ERR(flow_table)) { + err = PTR_ERR(flow_table); + netdev_err(netdev, "fail to create MACsec RX check table, err(%d)\n", err); + goto err; + } + rx_tables->ft_check = flow_table; + + /* Rx check table Default miss group/rule */ + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); + flow_group = mlx5_create_flow_group(rx_tables->ft_check, flow_group_in); + if (IS_ERR(flow_group)) { + err = PTR_ERR(flow_group); + netdev_err(netdev, + "Failed to create default flow group for MACsec Rx check table err(%d)\n", + err); + goto err; + } + rx_tables->ft_check_group = flow_group; + + /* Rx check table default drop rule */ + memset(&flow_act, 0, sizeof(flow_act)); + + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(rx_tables->check_miss_rule_counter); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + rule = mlx5_add_flow_rules(rx_tables->ft_check, NULL, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err); + goto err; + } + rx_tables->check_miss_rule = rule; + + /* Rx check table decap rules */ + err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, + MLX5_SECTAG_HEADER_SIZE_WITH_SCI); + if (err) + goto err; + + err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, + MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI); + if (err) + goto err; + + goto out_flow_group; + +err: + macsec_fs_rx_destroy(macsec_fs); +out_flow_group: + kvfree(flow_group_in); +free_spec: + kvfree(spec); + return err; +} + +static int macsec_fs_rx_ft_get(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; + int err = 0; + + if (rx_tables->refcnt) + goto out; + + err = macsec_fs_rx_create(macsec_fs); + if (err) + return err; + +out: + rx_tables->refcnt++; + return err; +} + +static void macsec_fs_rx_ft_put(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; + + if (--rx_tables->refcnt) + return; + + macsec_fs_rx_destroy(macsec_fs); +} + +static void macsec_fs_rx_del_rule(struct mlx5e_macsec_fs *macsec_fs, + struct mlx5e_macsec_rx_rule *rx_rule) +{ + int i; + + for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) { + if (rx_rule->rule[i]) { + mlx5_del_flow_rules(rx_rule->rule[i]); + rx_rule->rule[i] = NULL; + } + } + + if (rx_rule->meta_modhdr) { + mlx5_modify_header_dealloc(macsec_fs->mdev, rx_rule->meta_modhdr); + rx_rule->meta_modhdr = NULL; + } + + kfree(rx_rule); + + macsec_fs_rx_ft_put(macsec_fs); +} + +static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_macsec_rule_attrs *attrs, + bool sci_present) +{ + u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num; + struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto; + __be32 *sci_p = (__be32 *)(&attrs->sci); + + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + + /* MACsec ethertype */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC); + + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; + + /* Sectag AN + TCI SC present bit*/ + MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, + MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, + tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); + + if (sci_present) { + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_5.macsec_tag_2); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2, + be32_to_cpu(sci_p[0])); + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + misc_parameters_5.macsec_tag_3); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3, + be32_to_cpu(sci_p[1])); + } else { + /* When SCI isn't present in the Sectag, need to match the source */ + /* MAC address only if the SCI contains the default MACsec PORT */ + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16), + sci_p, ETH_ALEN); + } + + crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; + crypto_params->obj_id = attrs->macsec_obj_id; +} + +static union mlx5e_macsec_rule * +macsec_fs_rx_add_rule(struct mlx5e_macsec_fs *macsec_fs, + const struct macsec_context *macsec_ctx, + struct mlx5_macsec_rule_attrs *attrs, + u32 fs_id) +{ + u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; + struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs; + struct net_device *netdev = macsec_fs->netdev; + union mlx5e_macsec_rule *macsec_rule = NULL; + struct mlx5_modify_hdr *modify_hdr = NULL; + struct mlx5_flow_destination dest = {}; + struct mlx5e_macsec_tables *rx_tables; + struct mlx5e_macsec_rx_rule *rx_rule; + struct mlx5_flow_act flow_act = {}; + struct mlx5e_flow_table *ft_crypto; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int err = 0; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return NULL; + + err = macsec_fs_rx_ft_get(macsec_fs); + if (err) + goto out_spec; + + macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); + if (!macsec_rule) { + macsec_fs_rx_ft_put(macsec_fs); + goto out_spec; + } + + rx_rule = &macsec_rule->rx_rule; + rx_tables = &rx_fs->tables; + ft_crypto = &rx_tables->ft_crypto; + + /* Set bit[31 - 30] macsec marker - 0x01 */ + /* Set bit[3-0] fs id */ + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); + MLX5_SET(set_action_in, action, data, fs_id | BIT(30)); + MLX5_SET(set_action_in, action, offset, 0); + MLX5_SET(set_action_in, action, length, 32); + + modify_hdr = mlx5_modify_header_alloc(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, + 1, action); + if (IS_ERR(modify_hdr)) { + err = PTR_ERR(modify_hdr); + netdev_err(netdev, "fail to alloc MACsec set modify_header_id err=%d\n", err); + modify_hdr = NULL; + goto err; + } + rx_rule->meta_modhdr = modify_hdr; + + /* Rx crypto table with SCI rule */ + macsec_fs_rx_setup_fte(spec, &flow_act, attrs, true); + + flow_act.modify_hdr = modify_hdr; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | + MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = rx_tables->ft_check; + rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, + "Failed to add SA with SCI rule to Rx crypto rule, err=%d\n", + err); + goto err; + } + rx_rule->rule[0] = rule; + + /* Rx crypto table without SCI rule */ + if (cpu_to_be64((__force u64)attrs->sci) & ntohs(MACSEC_PORT_ES)) { + memset(spec, 0, sizeof(struct mlx5_flow_spec)); + memset(&dest, 0, sizeof(struct mlx5_flow_destination)); + memset(&flow_act, 0, sizeof(flow_act)); + + macsec_fs_rx_setup_fte(spec, &flow_act, attrs, false); + + flow_act.modify_hdr = modify_hdr; + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | + MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = rx_tables->ft_check; + rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + netdev_err(netdev, + "Failed to add SA without SCI rule to Rx crypto rule, err=%d\n", + err); + goto err; + } + rx_rule->rule[1] = rule; + } + + return macsec_rule; + +err: + macsec_fs_rx_del_rule(macsec_fs, rx_rule); + macsec_rule = NULL; +out_spec: + kvfree(spec); + return macsec_rule; +} + +static int macsec_fs_rx_init(struct mlx5e_macsec_fs *macsec_fs) +{ + struct net_device *netdev = macsec_fs->netdev; + struct mlx5_core_dev *mdev = macsec_fs->mdev; + struct mlx5e_macsec_tables *rx_tables; + struct mlx5e_macsec_rx *rx_fs; + struct mlx5_fc *flow_counter; + int err; + + rx_fs = kzalloc(sizeof(*rx_fs), GFP_KERNEL); + if (!rx_fs) + return -ENOMEM; + + flow_counter = mlx5_fc_create(mdev, false); + if (IS_ERR(flow_counter)) { + err = PTR_ERR(flow_counter); + netdev_err(netdev, + "Failed to create MACsec Rx encrypt flow counter, err(%d)\n", + err); + goto err_encrypt_counter; + } + + rx_tables = &rx_fs->tables; + rx_tables->check_rule_counter = flow_counter; + + flow_counter = mlx5_fc_create(mdev, false); + if (IS_ERR(flow_counter)) { + err = PTR_ERR(flow_counter); + netdev_err(netdev, + "Failed to create MACsec Rx drop flow counter, err(%d)\n", + err); + goto err_drop_counter; + } + rx_tables->check_miss_rule_counter = flow_counter; + + macsec_fs->rx_fs = rx_fs; + + return 0; + +err_drop_counter: + mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); + rx_tables->check_rule_counter = NULL; + +err_encrypt_counter: + kfree(rx_fs); + macsec_fs->rx_fs = NULL; + + return err; +} + +static void macsec_fs_rx_cleanup(struct mlx5e_macsec_fs *macsec_fs) +{ + struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs; + struct mlx5_core_dev *mdev = macsec_fs->mdev; + struct mlx5e_macsec_tables *rx_tables; + + if (!rx_fs) + return; + + rx_tables = &rx_fs->tables; + + if (rx_tables->refcnt) { + netdev_err(macsec_fs->netdev, + "Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n", + rx_tables->refcnt); + return; + } + + if (rx_tables->check_miss_rule_counter) { + mlx5_fc_destroy(mdev, rx_tables->check_miss_rule_counter); + rx_tables->check_miss_rule_counter = NULL; + } + + if (rx_tables->check_rule_counter) { + mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); + rx_tables->check_rule_counter = NULL; + } + + kfree(rx_fs); + macsec_fs->rx_fs = NULL; +} + +void mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs *macsec_fs, void *macsec_stats) +{ + struct mlx5e_macsec_stats *stats = (struct mlx5e_macsec_stats *)macsec_stats; + struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; + struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; + struct mlx5_core_dev *mdev = macsec_fs->mdev; + + if (tx_tables->check_rule_counter) + mlx5_fc_query(mdev, tx_tables->check_rule_counter, + &stats->macsec_tx_pkts, &stats->macsec_tx_bytes); + + if (tx_tables->check_miss_rule_counter) + mlx5_fc_query(mdev, tx_tables->check_miss_rule_counter, + &stats->macsec_tx_pkts_drop, &stats->macsec_tx_bytes_drop); + + if (rx_tables->check_rule_counter) + mlx5_fc_query(mdev, rx_tables->check_rule_counter, + &stats->macsec_rx_pkts, &stats->macsec_rx_bytes); + + if (rx_tables->check_miss_rule_counter) + mlx5_fc_query(mdev, rx_tables->check_miss_rule_counter, + &stats->macsec_rx_pkts_drop, &stats->macsec_rx_bytes_drop); +} + +union mlx5e_macsec_rule * +mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs, + const struct macsec_context *macsec_ctx, + struct mlx5_macsec_rule_attrs *attrs, + u32 *sa_fs_id) +{ + return (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? + macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, sa_fs_id) : + macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id); +} + +void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs, + union mlx5e_macsec_rule *macsec_rule, + int action) +{ + (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? + macsec_fs_tx_del_rule(macsec_fs, &macsec_rule->tx_rule) : + macsec_fs_rx_del_rule(macsec_fs, &macsec_rule->rx_rule); +} + +void mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs *macsec_fs) +{ + macsec_fs_rx_cleanup(macsec_fs); + macsec_fs_tx_cleanup(macsec_fs); + kfree(macsec_fs); +} + +struct mlx5e_macsec_fs * +mlx5e_macsec_fs_init(struct mlx5_core_dev *mdev, + struct net_device *netdev) +{ + struct mlx5e_macsec_fs *macsec_fs; + int err; + + macsec_fs = kzalloc(sizeof(*macsec_fs), GFP_KERNEL); + if (!macsec_fs) + return NULL; + + macsec_fs->mdev = mdev; + macsec_fs->netdev = netdev; + + err = macsec_fs_tx_init(macsec_fs); + if (err) { + netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); + goto err; + } + + err = macsec_fs_rx_init(macsec_fs); + if (err) { + netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); + goto tx_cleanup; + } + + return macsec_fs; + +tx_cleanup: + macsec_fs_tx_cleanup(macsec_fs); +err: + kfree(macsec_fs); + return NULL; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h new file mode 100644 index 0000000000000000000000000000000000000000..b429648d4ee7111d60b5640be132ba73313a8f0e --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_MACSEC_STEERING_H__ +#define __MLX5_MACSEC_STEERING_H__ + +#ifdef CONFIG_MLX5_EN_MACSEC + +#include "en_accel/macsec.h" + +#define MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES 16 + +struct mlx5e_macsec_fs; +union mlx5e_macsec_rule; + +struct mlx5_macsec_rule_attrs { + sci_t sci; + u32 macsec_obj_id; + u8 assoc_num; + int action; +}; + +enum mlx5_macsec_action { + MLX5_ACCEL_MACSEC_ACTION_ENCRYPT, + MLX5_ACCEL_MACSEC_ACTION_DECRYPT, +}; + +void mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs *macsec_fs); + +struct mlx5e_macsec_fs * +mlx5e_macsec_fs_init(struct mlx5_core_dev *mdev, struct net_device *netdev); + +union mlx5e_macsec_rule * +mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs, + const struct macsec_context *ctx, + struct mlx5_macsec_rule_attrs *attrs, + u32 *sa_fs_id); + +void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs, + union mlx5e_macsec_rule *macsec_rule, + int action); + +void mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs *macsec_fs, void *macsec_stats); + +#endif + +#endif /* __MLX5_MACSEC_STEERING_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c new file mode 100644 index 0000000000000000000000000000000000000000..e50a2e3f3d18e9199c502eaa585d35db9aac7020 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include +#include + +#include "en.h" +#include "en_accel/macsec.h" + +static const struct counter_desc mlx5e_macsec_hw_stats_desc[] = { + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_pkts_drop) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_bytes_drop) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_pkts) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_bytes) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_pkts_drop) }, + { MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_bytes_drop) }, +}; + +#define NUM_MACSEC_HW_COUNTERS ARRAY_SIZE(mlx5e_macsec_hw_stats_desc) + +static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(macsec_hw) +{ + if (!priv->macsec) + return 0; + + if (mlx5e_is_macsec_device(priv->mdev)) + return NUM_MACSEC_HW_COUNTERS; + + return 0; +} + +static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(macsec_hw) {} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(macsec_hw) +{ + unsigned int i; + + if (!priv->macsec) + return idx; + + if (!mlx5e_is_macsec_device(priv->mdev)) + return idx; + + for (i = 0; i < NUM_MACSEC_HW_COUNTERS; i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + mlx5e_macsec_hw_stats_desc[i].format); + + return idx; +} + +static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(macsec_hw) +{ + int i; + + if (!priv->macsec) + return idx; + + if (!mlx5e_is_macsec_device(priv->mdev)) + return idx; + + mlx5e_macsec_get_stats_fill(priv->macsec, mlx5e_macsec_get_stats(priv->macsec)); + for (i = 0; i < NUM_MACSEC_HW_COUNTERS; i++) + data[idx++] = MLX5E_READ_CTR64_CPU(mlx5e_macsec_get_stats(priv->macsec), + mlx5e_macsec_hw_stats_desc, + i); + + return idx; +} + +MLX5E_DEFINE_STATS_GRP(macsec_hw, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index cd7f245dcf14a72733d4a800fba49935f0b4e5e7..0ae1865086ff132ecddcc9e1944b95a1a1a1fb00 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -114,47 +114,49 @@ static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type) } } -static int arfs_disable(struct mlx5e_priv *priv) +static int arfs_disable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); int err, i; for (i = 0; i < ARFS_NUM_TYPES; i++) { /* Modify ttc rules destination back to their default */ - err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, arfs_get_tt(i)); + err = mlx5_ttc_fwd_default_dest(ttc, arfs_get_tt(i)); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] default destination failed, err(%d)\n", - __func__, arfs_get_tt(i), err); + fs_err(fs, + "%s: modify ttc[%d] default destination failed, err(%d)\n", + __func__, arfs_get_tt(i), err); return err; } } return 0; } -static void arfs_del_rules(struct mlx5e_priv *priv); +static void arfs_del_rules(struct mlx5e_flow_steering *fs); -int mlx5e_arfs_disable(struct mlx5e_priv *priv) +int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs) { - arfs_del_rules(priv); + arfs_del_rules(fs); - return arfs_disable(priv); + return arfs_disable(fs); } -int mlx5e_arfs_enable(struct mlx5e_priv *priv) +int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs) { + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false); + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); struct mlx5_flow_destination dest = {}; int err, i; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ARFS_NUM_TYPES; i++) { - dest.ft = priv->fs->arfs->arfs_tables[i].ft.t; + dest.ft = arfs->arfs_tables[i].ft.t; /* Modify ttc rules destination to point on the aRFS FTs */ - err = mlx5_ttc_fwd_dest(priv->fs->ttc, arfs_get_tt(i), &dest); + err = mlx5_ttc_fwd_dest(ttc, arfs_get_tt(i), &dest); if (err) { - netdev_err(priv->netdev, - "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", - __func__, arfs_get_tt(i), err); - arfs_disable(priv); + fs_err(fs, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", + __func__, arfs_get_tt(i), err); + arfs_disable(fs); return err; } } @@ -167,31 +169,37 @@ static void arfs_destroy_table(struct arfs_table *arfs_t) mlx5e_destroy_flow_table(&arfs_t->ft); } -static void _mlx5e_cleanup_tables(struct mlx5e_priv *priv) +static void _mlx5e_cleanup_tables(struct mlx5e_flow_steering *fs) { + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); int i; - arfs_del_rules(priv); - destroy_workqueue(priv->fs->arfs->wq); + arfs_del_rules(fs); + destroy_workqueue(arfs->wq); for (i = 0; i < ARFS_NUM_TYPES; i++) { - if (!IS_ERR_OR_NULL(priv->fs->arfs->arfs_tables[i].ft.t)) - arfs_destroy_table(&priv->fs->arfs->arfs_tables[i]); + if (!IS_ERR_OR_NULL(arfs->arfs_tables[i].ft.t)) + arfs_destroy_table(&arfs->arfs_tables[i]); } } -void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) +void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple) { - if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); + + if (!ntuple) return; - _mlx5e_cleanup_tables(priv); - kvfree(priv->fs->arfs); + _mlx5e_cleanup_tables(fs); + mlx5e_fs_set_arfs(fs, NULL); + kvfree(arfs); } -static int arfs_add_default_rule(struct mlx5e_priv *priv, +static int arfs_add_default_rule(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, enum arfs_type type) { - struct arfs_table *arfs_t = &priv->fs->arfs->arfs_tables[type]; + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); + struct arfs_table *arfs_t = &arfs->arfs_tables[type]; struct mlx5_flow_destination dest = {}; MLX5_DECLARE_FLOW_ACT(flow_act); enum mlx5_traffic_types tt; @@ -200,23 +208,21 @@ static int arfs_add_default_rule(struct mlx5e_priv *priv, dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; tt = arfs_get_tt(type); if (tt == -EINVAL) { - netdev_err(priv->netdev, "%s: bad arfs_type: %d\n", - __func__, type); + fs_err(fs, "%s: bad arfs_type: %d\n", __func__, type); return -EINVAL; } /* FIXME: Must use mlx5_ttc_get_default_dest(), * but can't since TTC default is not setup yet ! */ - dest.tir_num = mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt); + dest.tir_num = mlx5e_rx_res_get_tirn_rss(rx_res, tt); arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL, &flow_act, &dest, 1); if (IS_ERR(arfs_t->default_rule)) { err = PTR_ERR(arfs_t->default_rule); arfs_t->default_rule = NULL; - netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n", - __func__, type); + fs_err(fs, "%s: add rule failed, arfs type=%d\n", __func__, type); } return err; @@ -318,10 +324,12 @@ out: return err; } -static int arfs_create_table(struct mlx5e_priv *priv, +static int arfs_create_table(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, enum arfs_type type) { - struct mlx5e_arfs_tables *arfs = priv->fs->arfs; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false); + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -332,7 +340,7 @@ static int arfs_create_table(struct mlx5e_priv *priv, ft_attr.level = MLX5E_ARFS_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); + ft->t = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -343,7 +351,7 @@ static int arfs_create_table(struct mlx5e_priv *priv, if (err) goto err; - err = arfs_add_default_rule(priv, type); + err = arfs_add_default_rule(fs, rx_res, type); if (err) goto err; @@ -353,35 +361,40 @@ err: return err; } -int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) +int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, bool ntuple) { + struct mlx5e_arfs_tables *arfs; int err = -ENOMEM; int i; - if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) + if (!ntuple) return 0; - priv->fs->arfs = kvzalloc(sizeof(*priv->fs->arfs), GFP_KERNEL); - if (!priv->fs->arfs) + arfs = kvzalloc(sizeof(*arfs), GFP_KERNEL); + if (!arfs) return -ENOMEM; - spin_lock_init(&priv->fs->arfs->arfs_lock); - INIT_LIST_HEAD(&priv->fs->arfs->rules); - priv->fs->arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); - if (!priv->fs->arfs->wq) + spin_lock_init(&arfs->arfs_lock); + INIT_LIST_HEAD(&arfs->rules); + arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); + if (!arfs->wq) goto err; + mlx5e_fs_set_arfs(fs, arfs); + for (i = 0; i < ARFS_NUM_TYPES; i++) { - err = arfs_create_table(priv, i); + err = arfs_create_table(fs, rx_res, i); if (err) goto err_des; } return 0; err_des: - _mlx5e_cleanup_tables(priv); + _mlx5e_cleanup_tables(fs); err: - kvfree(priv->fs->arfs); + mlx5e_fs_set_arfs(fs, NULL); + kvfree(arfs); return err; } @@ -389,6 +402,7 @@ err: static void arfs_may_expire_flow(struct mlx5e_priv *priv) { + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); struct arfs_rule *arfs_rule; struct hlist_node *htmp; HLIST_HEAD(del_list); @@ -396,8 +410,8 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv) int i; int j; - spin_lock_bh(&priv->fs->arfs->arfs_lock); - mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs->arfs->arfs_tables, i, j) { + spin_lock_bh(&arfs->arfs_lock); + mlx5e_for_each_arfs_rule(arfs_rule, htmp, arfs->arfs_tables, i, j) { if (!work_pending(&arfs_rule->arfs_work) && rps_may_expire_flow(priv->netdev, arfs_rule->rxq, arfs_rule->flow_id, @@ -408,7 +422,7 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv) break; } } - spin_unlock_bh(&priv->fs->arfs->arfs_lock); + spin_unlock_bh(&arfs->arfs_lock); hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { if (arfs_rule->rule) mlx5_del_flow_rules(arfs_rule->rule); @@ -417,20 +431,21 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv) } } -static void arfs_del_rules(struct mlx5e_priv *priv) +static void arfs_del_rules(struct mlx5e_flow_steering *fs) { + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); struct hlist_node *htmp; struct arfs_rule *rule; HLIST_HEAD(del_list); int i; int j; - spin_lock_bh(&priv->fs->arfs->arfs_lock); - mlx5e_for_each_arfs_rule(rule, htmp, priv->fs->arfs->arfs_tables, i, j) { + spin_lock_bh(&arfs->arfs_lock); + mlx5e_for_each_arfs_rule(rule, htmp, arfs->arfs_tables, i, j) { hlist_del_init(&rule->hlist); hlist_add_head(&rule->hlist, &del_list); } - spin_unlock_bh(&priv->fs->arfs->arfs_lock); + spin_unlock_bh(&arfs->arfs_lock); hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { cancel_work_sync(&rule->arfs_work); @@ -474,7 +489,7 @@ static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, struct arfs_rule *arfs_rule) { - struct mlx5e_arfs_tables *arfs = priv->fs->arfs; + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); struct arfs_tuple *tuple = &arfs_rule->tuple; struct mlx5_flow_handle *rule = NULL; struct mlx5_flow_destination dest = {}; @@ -588,13 +603,15 @@ static void arfs_handle_work(struct work_struct *work) struct arfs_rule, arfs_work); struct mlx5e_priv *priv = arfs_rule->priv; + struct mlx5e_arfs_tables *arfs; struct mlx5_flow_handle *rule; + arfs = mlx5e_fs_get_arfs(priv->fs); mutex_lock(&priv->state_lock); if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { - spin_lock_bh(&priv->fs->arfs->arfs_lock); + spin_lock_bh(&arfs->arfs_lock); hlist_del(&arfs_rule->hlist); - spin_unlock_bh(&priv->fs->arfs->arfs_lock); + spin_unlock_bh(&arfs->arfs_lock); mutex_unlock(&priv->state_lock); kfree(arfs_rule); @@ -620,6 +637,7 @@ static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, const struct flow_keys *fk, u16 rxq, u32 flow_id) { + struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs); struct arfs_rule *rule; struct arfs_tuple *tuple; @@ -647,7 +665,7 @@ static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, tuple->dst_port = fk->ports.dst; rule->flow_id = flow_id; - rule->filter_id = priv->fs->arfs->last_filter_id++ % RPS_NO_FILTER; + rule->filter_id = arfs->last_filter_id++ % RPS_NO_FILTER; hlist_add_head(&rule->hlist, arfs_hash_bucket(arfs_t, tuple->src_port, @@ -691,11 +709,12 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, u16 rxq_index, u32 flow_id) { struct mlx5e_priv *priv = netdev_priv(dev); - struct mlx5e_arfs_tables *arfs = priv->fs->arfs; - struct arfs_table *arfs_t; + struct mlx5e_arfs_tables *arfs; struct arfs_rule *arfs_rule; + struct arfs_table *arfs_t; struct flow_keys fk; + arfs = mlx5e_fs_get_arfs(priv->fs); if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) return -EPROTONOSUPPORT; @@ -725,7 +744,7 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, return -ENOMEM; } } - queue_work(priv->fs->arfs->wq, &arfs_rule->arfs_work); + queue_work(arfs->wq, &arfs_rule->arfs_work); spin_unlock_bh(&arfs->arfs_lock); return arfs_rule->filter_id; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index c0f409c195bf8dc20a9b7433469af8397f110b1a..68f19324db93ce345707109c8d65438cf566c4c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -46,8 +46,7 @@ void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc) MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write); } -static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, - u32 *mkey) +int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey) { int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); void *mkc; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index b811207fe5ed54a3afbc75fbdb5306e24a1b1026..24aa25da482b5a788bf3d76e1c51fb7925dbd7f4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -38,18 +38,19 @@ #include "en/xsk/pool.h" #include "en/ptp.h" #include "lib/clock.h" +#include "en/fs_ethtool.h" void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv, struct ethtool_drvinfo *drvinfo) { struct mlx5_core_dev *mdev = priv->mdev; - strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%04d (%.16s)", fw_rev_maj(mdev), fw_rev_min(mdev), fw_rev_sub(mdev), mdev->board_id); - strlcpy(drvinfo->bus_info, dev_name(mdev->device), + strscpy(drvinfo->bus_info, dev_name(mdev->device), sizeof(drvinfo->bus_info)); } @@ -310,7 +311,15 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, struct ethtool_ringparam *param, struct kernel_ethtool_ringparam *kernel_param) { - param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; + /* Limitation for regular RQ. XSK RQ may clamp the queue length in + * mlx5e_mpwqe_get_log_rq_size. + */ + u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev, + PAGE_SHIFT, + MLX5E_MPWRQ_UMR_MODE_ALIGNED); + + param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE, + max_log_mpwrq_pkts); param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; @@ -494,14 +503,14 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, arfs_enabled = opened && (priv->netdev->features & NETIF_F_NTUPLE); if (arfs_enabled) - mlx5e_arfs_disable(priv); + mlx5e_arfs_disable(priv->fs); /* Switch to new channels, set new parameters and close old ones */ err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_num_channels_changed_ctx, NULL, true); if (arfs_enabled) { - int err2 = mlx5e_arfs_enable(priv); + int err2 = mlx5e_arfs_enable(priv->fs); if (err2) netdev_err(priv->netdev, "%s: mlx5e_arfs_enable failed: %d\n", @@ -1996,10 +2005,14 @@ static int set_pflag_rx_striding_rq(struct net_device *netdev, bool enable) struct mlx5e_params new_params; if (enable) { - if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) - return -EOPNOTSUPP; - if (!mlx5e_striding_rq_possible(mdev, &priv->channels.params)) - return -EINVAL; + /* Checking the regular RQ here; mlx5e_validate_xsk_param called + * from mlx5e_open_xsk will check for each XSK queue, and + * mlx5e_safe_switch_params will be reverted if any check fails. + */ + int err = mlx5e_mpwrq_validate_regular(mdev, &priv->channels.params); + + if (err) + return err; } else if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) { netdev_warn(netdev, "Can't set legacy RQ with HW-GRO/LRO, disable them first\n"); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index e2a9b9be5c1fba84499005133feb14e7dc133258..1892ccb889b3fe87e6722619f7ddfe86a776ae90 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -36,10 +36,38 @@ #include #include #include -#include "en.h" #include "en_tc.h" #include "lib/mpfs.h" #include "en/ptp.h" +#include "en/fs_ethtool.h" + +struct mlx5e_flow_steering { + struct work_struct set_rx_mode_work; + bool state_destroy; + bool vlan_strip_disable; + struct mlx5_core_dev *mdev; + struct net_device *netdev; + struct mlx5_flow_namespace *ns; + struct mlx5_flow_namespace *egress_ns; +#ifdef CONFIG_MLX5_EN_RXNFC + struct mlx5e_ethtool_steering *ethtool; +#endif + struct mlx5e_tc_table *tc; + struct mlx5e_promisc_table promisc; + struct mlx5e_vlan_table *vlan; + struct mlx5e_l2_table l2; + struct mlx5_ttc_table *ttc; + struct mlx5_ttc_table *inner_ttc; +#ifdef CONFIG_MLX5_EN_ARFS + struct mlx5e_arfs_tables *arfs; +#endif +#ifdef CONFIG_MLX5_EN_TLS + struct mlx5e_accel_fs_tcp *accel_tcp; +#endif + struct mlx5e_fs_udp *udp; + struct mlx5e_fs_any *any; + struct mlx5e_ptp_fs *ptp_fs; +}; static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs, struct mlx5e_l2_rule *ai, int type); @@ -148,9 +176,8 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_flow_steering *fs) max_list_size = 1 << MLX5_CAP_GEN(fs->mdev, log_max_vlan_list); if (list_size > max_list_size) { - mlx5_core_warn(fs->mdev, - "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n", - list_size, max_list_size); + fs_warn(fs, "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n", + list_size, max_list_size); list_size = max_list_size; } @@ -167,8 +194,8 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_flow_steering *fs) err = mlx5_modify_nic_vport_vlans(fs->mdev, vlans, list_size); if (err) - mlx5_core_err(fs->mdev, "Failed to modify vport vlans list err(%d)\n", - err); + fs_err(fs, "Failed to modify vport vlans list err(%d)\n", + err); kvfree(vlans); return err; @@ -249,7 +276,7 @@ static int __mlx5e_add_vlan_rule(struct mlx5e_flow_steering *fs, if (IS_ERR(*rule_p)) { err = PTR_ERR(*rule_p); *rule_p = NULL; - mlx5_core_err(fs->mdev, "%s: add rule failed\n", __func__); + fs_err(fs, "%s: add rule failed\n", __func__); } return err; @@ -351,78 +378,78 @@ mlx5e_add_trap_rule(struct mlx5_flow_table *ft, int trap_id, int tir_num) return rule; } -int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int trap_id, int tir_num) +int mlx5e_add_vlan_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num) { - struct mlx5_flow_table *ft = priv->fs->vlan->ft.t; + struct mlx5_flow_table *ft = fs->vlan->ft.t; struct mlx5_flow_handle *rule; int err; rule = mlx5e_add_trap_rule(ft, trap_id, tir_num); if (IS_ERR(rule)) { err = PTR_ERR(rule); - priv->fs->vlan->trap_rule = NULL; - mlx5_core_err(priv->fs->mdev, "%s: add VLAN trap rule failed, err %d\n", - __func__, err); + fs->vlan->trap_rule = NULL; + fs_err(fs, "%s: add VLAN trap rule failed, err %d\n", + __func__, err); return err; } - priv->fs->vlan->trap_rule = rule; + fs->vlan->trap_rule = rule; return 0; } -void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv) +void mlx5e_remove_vlan_trap(struct mlx5e_flow_steering *fs) { - if (priv->fs->vlan->trap_rule) { - mlx5_del_flow_rules(priv->fs->vlan->trap_rule); - priv->fs->vlan->trap_rule = NULL; + if (fs->vlan->trap_rule) { + mlx5_del_flow_rules(fs->vlan->trap_rule); + fs->vlan->trap_rule = NULL; } } -int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int trap_id, int tir_num) +int mlx5e_add_mac_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num) { - struct mlx5_flow_table *ft = priv->fs->l2.ft.t; + struct mlx5_flow_table *ft = fs->l2.ft.t; struct mlx5_flow_handle *rule; int err; rule = mlx5e_add_trap_rule(ft, trap_id, tir_num); if (IS_ERR(rule)) { err = PTR_ERR(rule); - priv->fs->l2.trap_rule = NULL; - mlx5_core_err(priv->fs->mdev, "%s: add MAC trap rule failed, err %d\n", - __func__, err); + fs->l2.trap_rule = NULL; + fs_err(fs, "%s: add MAC trap rule failed, err %d\n", + __func__, err); return err; } - priv->fs->l2.trap_rule = rule; + fs->l2.trap_rule = rule; return 0; } -void mlx5e_remove_mac_trap(struct mlx5e_priv *priv) +void mlx5e_remove_mac_trap(struct mlx5e_flow_steering *fs) { - if (priv->fs->l2.trap_rule) { - mlx5_del_flow_rules(priv->fs->l2.trap_rule); - priv->fs->l2.trap_rule = NULL; + if (fs->l2.trap_rule) { + mlx5_del_flow_rules(fs->l2.trap_rule); + fs->l2.trap_rule = NULL; } } -void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv) +void mlx5e_enable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc) { - if (!priv->fs->vlan->cvlan_filter_disabled) + if (!fs->vlan->cvlan_filter_disabled) return; - priv->fs->vlan->cvlan_filter_disabled = false; - if (priv->netdev->flags & IFF_PROMISC) + fs->vlan->cvlan_filter_disabled = false; + if (promisc) return; - mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); } -void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv) +void mlx5e_disable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc) { - if (priv->fs->vlan->cvlan_filter_disabled) + if (fs->vlan->cvlan_filter_disabled) return; - priv->fs->vlan->cvlan_filter_disabled = true; - if (priv->netdev->flags & IFF_PROMISC) + fs->vlan->cvlan_filter_disabled = true; + if (promisc) return; - mlx5e_add_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); } static int mlx5e_vlan_rx_add_cvid(struct mlx5e_flow_steering *fs, u16 vid) @@ -462,7 +489,7 @@ int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs, { if (!fs->vlan) { - mlx5_core_err(fs->mdev, "Vlan doesn't exist\n"); + fs_err(fs, "Vlan doesn't exist\n"); return -EINVAL; } @@ -479,7 +506,7 @@ int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs, __be16 proto, u16 vid) { if (!fs->vlan) { - mlx5_core_err(fs->mdev, "Vlan doesn't exist\n"); + fs_err(fs, "Vlan doesn't exist\n"); return -EINVAL; } @@ -512,28 +539,28 @@ static void mlx5e_fs_add_vlan_rules(struct mlx5e_flow_steering *fs) mlx5e_fs_add_any_vid_rules(fs); } -static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv) +static void mlx5e_del_vlan_rules(struct mlx5e_flow_steering *fs) { int i; - mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); - for_each_set_bit(i, priv->fs->vlan->active_cvlans, VLAN_N_VID) { - mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); + for_each_set_bit(i, fs->vlan->active_cvlans, VLAN_N_VID) { + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); } - for_each_set_bit(i, priv->fs->vlan->active_svlans, VLAN_N_VID) - mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); + for_each_set_bit(i, fs->vlan->active_svlans, VLAN_N_VID) + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); - WARN_ON_ONCE(priv->fs->state_destroy); + WARN_ON_ONCE(fs->state_destroy); - mlx5e_remove_vlan_trap(priv); + mlx5e_remove_vlan_trap(fs); /* must be called after DESTROY bit is set and * set_rx_mode is called and flushed */ - if (priv->fs->vlan->cvlan_filter_disabled) - mlx5e_fs_del_any_vid_rules(priv->fs); + if (fs->vlan->cvlan_filter_disabled) + mlx5e_fs_del_any_vid_rules(fs); } #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \ @@ -568,8 +595,9 @@ static void mlx5e_execute_l2_action(struct mlx5e_flow_steering *fs, } if (l2_err) - mlx5_core_warn(fs->mdev, "MPFS, failed to %s mac %pM, err(%d)\n", - action == MLX5E_ACTION_ADD ? "add" : "del", mac_addr, l2_err); + fs_warn(fs, "MPFS, failed to %s mac %pM, err(%d)\n", + action == MLX5E_ACTION_ADD ? "add" : "del", + mac_addr, l2_err); } static void mlx5e_sync_netdev_addr(struct mlx5e_flow_steering *fs, @@ -640,9 +668,8 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_flow_steering *fs, size++; if (size > max_size) { - mlx5_core_warn(fs->mdev, - "mdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n", - is_uc ? "UC" : "MC", size, max_size); + fs_warn(fs, "mdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n", + is_uc ? "UC" : "MC", size, max_size); size = max_size; } @@ -658,9 +685,8 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_flow_steering *fs, err = mlx5_modify_nic_vport_mac_list(fs->mdev, list_type, addr_array, size); out: if (err) - mlx5_core_err(fs->mdev, - "Failed to modify vport %s list err(%d)\n", - is_uc ? "UC" : "MC", err); + fs_err(fs, "Failed to modify vport %s list err(%d)\n", + is_uc ? "UC" : "MC", err); kfree(addr_array); } @@ -730,7 +756,7 @@ static int mlx5e_add_promisc_rule(struct mlx5e_flow_steering *fs) if (IS_ERR(*rule_p)) { err = PTR_ERR(*rule_p); *rule_p = NULL; - mlx5_core_err(fs->mdev, "%s: add promiscuous rule failed\n", __func__); + fs_err(fs, "%s: add promiscuous rule failed\n", __func__); } kvfree(spec); return err; @@ -750,7 +776,7 @@ static int mlx5e_create_promisc_table(struct mlx5e_flow_steering *fs) ft->t = mlx5_create_auto_grouped_flow_table(fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); - mlx5_core_err(fs->mdev, "fail to create promisc table err=%d\n", err); + fs_err(fs, "fail to create promisc table err=%d\n", err); return err; } @@ -807,8 +833,8 @@ void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, if (err) enable_promisc = false; if (!fs->vlan_strip_disable && !err) - mlx5_core_warn_once(fs->mdev, - "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n"); + fs_warn_once(fs, + "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n"); } if (enable_allmulti) mlx5e_add_l2_flow_rule(fs, &ea->allmulti, MLX5E_ALLMULTI); @@ -856,14 +882,15 @@ void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft) ft->t = NULL; } -static void mlx5e_set_inner_ttc_params(struct mlx5e_priv *priv, +static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, struct ttc_params *ttc_params) { struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr; int tt; memset(ttc_params, 0, sizeof(*ttc_params)); - ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev, + ttc_params->ns = mlx5_get_flow_namespace(fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL); ft_attr->level = MLX5E_INNER_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; @@ -872,13 +899,14 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_priv *priv, ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? - mlx5e_rx_res_get_tirn_direct(priv->rx_res, 0) : - mlx5e_rx_res_get_tirn_rss_inner(priv->rx_res, + mlx5e_rx_res_get_tirn_direct(rx_res, 0) : + mlx5e_rx_res_get_tirn_rss_inner(rx_res, tt); } } -void mlx5e_set_ttc_params(struct mlx5e_priv *priv, +void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, struct ttc_params *ttc_params, bool tunnel) { @@ -886,7 +914,7 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv, int tt; memset(ttc_params, 0, sizeof(*ttc_params)); - ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev, + ttc_params->ns = mlx5_get_flow_namespace(fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL); ft_attr->level = MLX5E_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; @@ -895,19 +923,19 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv, ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR; ttc_params->dests[tt].tir_num = tt == MLX5_TT_ANY ? - mlx5e_rx_res_get_tirn_direct(priv->rx_res, 0) : - mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt); + mlx5e_rx_res_get_tirn_direct(rx_res, 0) : + mlx5e_rx_res_get_tirn_rss(rx_res, tt); } ttc_params->inner_ttc = tunnel; - if (!tunnel || !mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) + if (!tunnel || !mlx5_tunnel_inner_ft_supported(fs->mdev)) return; for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { ttc_params->tunnel_dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; ttc_params->tunnel_dests[tt].ft = - mlx5_get_ttc_flow_table(priv->fs->inner_ttc); + mlx5_get_ttc_flow_table(fs->inner_ttc); } } @@ -959,8 +987,7 @@ static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs, ai->rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); if (IS_ERR(ai->rule)) { - mlx5_core_err(fs->mdev, "%s: add l2 rule(mac:%pM) failed\n", - __func__, mv_dmac); + fs_err(fs, "%s: add l2 rule(mac:%pM) failed\n", __func__, mv_dmac); err = PTR_ERR(ai->rule); ai->rule = NULL; } @@ -1044,14 +1071,14 @@ err_destroy_groups: return err; } -static void mlx5e_destroy_l2_table(struct mlx5e_priv *priv) +static void mlx5e_destroy_l2_table(struct mlx5e_flow_steering *fs) { - mlx5e_destroy_flow_table(&priv->fs->l2.ft); + mlx5e_destroy_flow_table(&fs->l2.ft); } -static int mlx5e_create_l2_table(struct mlx5e_priv *priv) +static int mlx5e_create_l2_table(struct mlx5e_flow_steering *fs) { - struct mlx5e_l2_table *l2_table = &priv->fs->l2; + struct mlx5e_l2_table *l2_table = &fs->l2; struct mlx5e_flow_table *ft = &l2_table->ft; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -1062,7 +1089,7 @@ static int mlx5e_create_l2_table(struct mlx5e_priv *priv) ft_attr.level = MLX5E_L2_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); + ft->t = mlx5_create_flow_table(fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -1221,126 +1248,128 @@ err_destroy_vlan_table: return err; } -static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv) +static void mlx5e_destroy_vlan_table(struct mlx5e_flow_steering *fs) { - mlx5e_del_vlan_rules(priv); - mlx5e_destroy_flow_table(&priv->fs->vlan->ft); + mlx5e_del_vlan_rules(fs); + mlx5e_destroy_flow_table(&fs->vlan->ft); } -static void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv) +static void mlx5e_destroy_inner_ttc_table(struct mlx5e_flow_steering *fs) { - if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) + if (!mlx5_tunnel_inner_ft_supported(fs->mdev)) return; - mlx5_destroy_ttc_table(priv->fs->inner_ttc); + mlx5_destroy_ttc_table(fs->inner_ttc); } -void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv) +void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs) { - mlx5_destroy_ttc_table(priv->fs->ttc); + mlx5_destroy_ttc_table(fs->ttc); } -static int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv) +static int mlx5e_create_inner_ttc_table(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res) { struct ttc_params ttc_params = {}; - if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) + if (!mlx5_tunnel_inner_ft_supported(fs->mdev)) return 0; - mlx5e_set_inner_ttc_params(priv, &ttc_params); - priv->fs->inner_ttc = mlx5_create_inner_ttc_table(priv->fs->mdev, - &ttc_params); - if (IS_ERR(priv->fs->inner_ttc)) - return PTR_ERR(priv->fs->inner_ttc); + mlx5e_set_inner_ttc_params(fs, rx_res, &ttc_params); + fs->inner_ttc = mlx5_create_inner_ttc_table(fs->mdev, + &ttc_params); + if (IS_ERR(fs->inner_ttc)) + return PTR_ERR(fs->inner_ttc); return 0; } -int mlx5e_create_ttc_table(struct mlx5e_priv *priv) +int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res) { struct ttc_params ttc_params = {}; - mlx5e_set_ttc_params(priv, &ttc_params, true); - priv->fs->ttc = mlx5_create_ttc_table(priv->fs->mdev, &ttc_params); - if (IS_ERR(priv->fs->ttc)) - return PTR_ERR(priv->fs->ttc); + mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true); + fs->ttc = mlx5_create_ttc_table(fs->mdev, &ttc_params); + if (IS_ERR(fs->ttc)) + return PTR_ERR(fs->ttc); return 0; } -int mlx5e_create_flow_steering(struct mlx5e_priv *priv) +int mlx5e_create_flow_steering(struct mlx5e_flow_steering *fs, + struct mlx5e_rx_res *rx_res, + const struct mlx5e_profile *profile, + struct net_device *netdev) { + struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(fs->mdev, + MLX5_FLOW_NAMESPACE_KERNEL); int err; - priv->fs->ns = mlx5_get_flow_namespace(priv->fs->mdev, - MLX5_FLOW_NAMESPACE_KERNEL); - - if (!priv->fs->ns) + if (!ns) return -EOPNOTSUPP; - err = mlx5e_arfs_create_tables(priv); + mlx5e_fs_set_ns(fs, ns, false); + err = mlx5e_arfs_create_tables(fs, rx_res, + !!(netdev->hw_features & NETIF_F_NTUPLE)); if (err) { - mlx5_core_err(priv->fs->mdev, "Failed to create arfs tables, err=%d\n", - err); - priv->netdev->hw_features &= ~NETIF_F_NTUPLE; + fs_err(fs, "Failed to create arfs tables, err=%d\n", err); + netdev->hw_features &= ~NETIF_F_NTUPLE; } - err = mlx5e_create_inner_ttc_table(priv); + err = mlx5e_create_inner_ttc_table(fs, rx_res); if (err) { - mlx5_core_err(priv->fs->mdev, - "Failed to create inner ttc table, err=%d\n", err); + fs_err(fs, "Failed to create inner ttc table, err=%d\n", err); goto err_destroy_arfs_tables; } - err = mlx5e_create_ttc_table(priv); + err = mlx5e_create_ttc_table(fs, rx_res); if (err) { - mlx5_core_err(priv->fs->mdev, "Failed to create ttc table, err=%d\n", - err); + fs_err(fs, "Failed to create ttc table, err=%d\n", err); goto err_destroy_inner_ttc_table; } - err = mlx5e_create_l2_table(priv); + err = mlx5e_create_l2_table(fs); if (err) { - mlx5_core_err(priv->fs->mdev, "Failed to create l2 table, err=%d\n", - err); + fs_err(fs, "Failed to create l2 table, err=%d\n", err); goto err_destroy_ttc_table; } - err = mlx5e_fs_create_vlan_table(priv->fs); + err = mlx5e_fs_create_vlan_table(fs); if (err) { - mlx5_core_err(priv->fs->mdev, "Failed to create vlan table, err=%d\n", - err); + fs_err(fs, "Failed to create vlan table, err=%d\n", err); goto err_destroy_l2_table; } - err = mlx5e_ptp_alloc_rx_fs(priv); + err = mlx5e_ptp_alloc_rx_fs(fs, profile); if (err) goto err_destory_vlan_table; - mlx5e_ethtool_init_steering(priv); + mlx5e_ethtool_init_steering(fs); return 0; err_destory_vlan_table: - mlx5e_destroy_vlan_table(priv); + mlx5e_destroy_vlan_table(fs); err_destroy_l2_table: - mlx5e_destroy_l2_table(priv); + mlx5e_destroy_l2_table(fs); err_destroy_ttc_table: - mlx5e_destroy_ttc_table(priv); + mlx5e_destroy_ttc_table(fs); err_destroy_inner_ttc_table: - mlx5e_destroy_inner_ttc_table(priv); + mlx5e_destroy_inner_ttc_table(fs); err_destroy_arfs_tables: - mlx5e_arfs_destroy_tables(priv); + mlx5e_arfs_destroy_tables(fs, !!(netdev->hw_features & NETIF_F_NTUPLE)); return err; } -void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv) +void mlx5e_destroy_flow_steering(struct mlx5e_flow_steering *fs, bool ntuple, + const struct mlx5e_profile *profile) { - mlx5e_ptp_free_rx_fs(priv); - mlx5e_destroy_vlan_table(priv); - mlx5e_destroy_l2_table(priv); - mlx5e_destroy_ttc_table(priv); - mlx5e_destroy_inner_ttc_table(priv); - mlx5e_arfs_destroy_tables(priv); - mlx5e_ethtool_cleanup_steering(priv); + mlx5e_ptp_free_rx_fs(fs, profile); + mlx5e_destroy_vlan_table(fs); + mlx5e_destroy_l2_table(fs); + mlx5e_destroy_ttc_table(fs); + mlx5e_destroy_inner_ttc_table(fs); + mlx5e_arfs_destroy_tables(fs, ntuple); + mlx5e_ethtool_cleanup_steering(fs); } static int mlx5e_fs_vlan_alloc(struct mlx5e_flow_steering *fs) @@ -1356,6 +1385,11 @@ static void mlx5e_fs_vlan_free(struct mlx5e_flow_steering *fs) kvfree(fs->vlan); } +struct mlx5e_vlan_table *mlx5e_fs_get_vlan(struct mlx5e_flow_steering *fs) +{ + return fs->vlan; +} + static int mlx5e_fs_tc_alloc(struct mlx5e_flow_steering *fs) { fs->tc = mlx5e_tc_table_alloc(); @@ -1369,6 +1403,32 @@ static void mlx5e_fs_tc_free(struct mlx5e_flow_steering *fs) mlx5e_tc_table_free(fs->tc); } +struct mlx5e_tc_table *mlx5e_fs_get_tc(struct mlx5e_flow_steering *fs) +{ + return fs->tc; +} + +#ifdef CONFIG_MLX5_EN_RXNFC +static int mlx5e_fs_ethtool_alloc(struct mlx5e_flow_steering *fs) +{ + return mlx5e_ethtool_alloc(&fs->ethtool); +} + +static void mlx5e_fs_ethtool_free(struct mlx5e_flow_steering *fs) +{ + mlx5e_ethtool_free(fs->ethtool); +} + +struct mlx5e_ethtool_steering *mlx5e_fs_get_ethtool(struct mlx5e_flow_steering *fs) +{ + return fs->ethtool; +} +#else +static int mlx5e_fs_ethtool_alloc(struct mlx5e_flow_steering *fs) +{ return 0; } +static void mlx5e_fs_ethtool_free(struct mlx5e_flow_steering *fs) { } +#endif + struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile, struct mlx5_core_dev *mdev, bool state_destroy) @@ -1394,18 +1454,126 @@ struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile, goto err_free_vlan; } + err = mlx5e_fs_ethtool_alloc(fs); + if (err) + goto err_free_tc; + return fs; -err_free_fs: - kvfree(fs); +err_free_tc: + mlx5e_fs_tc_free(fs); err_free_vlan: mlx5e_fs_vlan_free(fs); +err_free_fs: + kvfree(fs); err: return NULL; } void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs) { + mlx5e_fs_ethtool_free(fs); mlx5e_fs_tc_free(fs); mlx5e_fs_vlan_free(fs); kvfree(fs); } + +struct mlx5e_l2_table *mlx5e_fs_get_l2(struct mlx5e_flow_steering *fs) +{ + return &fs->l2; +} + +struct mlx5_flow_namespace *mlx5e_fs_get_ns(struct mlx5e_flow_steering *fs, bool egress) +{ + return egress ? fs->egress_ns : fs->ns; +} + +void mlx5e_fs_set_ns(struct mlx5e_flow_steering *fs, struct mlx5_flow_namespace *ns, bool egress) +{ + if (!egress) + fs->ns = ns; + else + fs->egress_ns = ns; +} + +struct mlx5_ttc_table *mlx5e_fs_get_ttc(struct mlx5e_flow_steering *fs, bool inner) +{ + return inner ? fs->inner_ttc : fs->ttc; +} + +void mlx5e_fs_set_ttc(struct mlx5e_flow_steering *fs, struct mlx5_ttc_table *ttc, bool inner) +{ + if (!inner) + fs->ttc = ttc; + else + fs->inner_ttc = ttc; +} + +#ifdef CONFIG_MLX5_EN_ARFS +struct mlx5e_arfs_tables *mlx5e_fs_get_arfs(struct mlx5e_flow_steering *fs) +{ + return fs->arfs; +} + +void mlx5e_fs_set_arfs(struct mlx5e_flow_steering *fs, struct mlx5e_arfs_tables *arfs) +{ + fs->arfs = arfs; +} +#endif + +struct mlx5e_ptp_fs *mlx5e_fs_get_ptp(struct mlx5e_flow_steering *fs) +{ + return fs->ptp_fs; +} + +void mlx5e_fs_set_ptp(struct mlx5e_flow_steering *fs, struct mlx5e_ptp_fs *ptp_fs) +{ + fs->ptp_fs = ptp_fs; +} + +struct mlx5e_fs_any *mlx5e_fs_get_any(struct mlx5e_flow_steering *fs) +{ + return fs->any; +} + +void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any) +{ + fs->any = any; +} + +#ifdef CONFIG_MLX5_EN_TLS +struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs) +{ + return fs->accel_tcp; +} + +void mlx5e_fs_set_accel_tcp(struct mlx5e_flow_steering *fs, struct mlx5e_accel_fs_tcp *accel_tcp) +{ + fs->accel_tcp = accel_tcp; +} +#endif + +void mlx5e_fs_set_state_destroy(struct mlx5e_flow_steering *fs, bool state_destroy) +{ + fs->state_destroy = state_destroy; +} + +void mlx5e_fs_set_vlan_strip_disable(struct mlx5e_flow_steering *fs, + bool vlan_strip_disable) +{ + fs->vlan_strip_disable = vlan_strip_disable; +} + +struct mlx5e_fs_udp *mlx5e_fs_get_udp(struct mlx5e_flow_steering *fs) +{ + return fs->udp; +} + +void mlx5e_fs_set_udp(struct mlx5e_flow_steering *fs, struct mlx5e_fs_udp *udp) +{ + fs->udp = udp; +} + +struct mlx5_core_dev *mlx5e_fs_get_mdev(struct mlx5e_flow_steering *fs) +{ + return fs->mdev; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c index 3e4bc7836ef4c4a7dad053605eb6214e4e624ea6..aac32e505c14fe0c5b451cf58e8dd4f42393ed6b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c @@ -34,6 +34,22 @@ #include "en.h" #include "en/params.h" #include "en/xsk/pool.h" +#include "en/fs_ethtool.h" + +struct mlx5e_ethtool_table { + struct mlx5_flow_table *ft; + int num_rules; +}; + +#define ETHTOOL_NUM_L3_L4_FTS 7 +#define ETHTOOL_NUM_L2_FTS 4 + +struct mlx5e_ethtool_steering { + struct mlx5e_ethtool_table l3_l4_ft[ETHTOOL_NUM_L3_L4_FTS]; + struct mlx5e_ethtool_table l2_ft[ETHTOOL_NUM_L2_FTS]; + struct list_head rules; + int tot_num_rules; +}; static int flow_type_to_traffic_type(u32 flow_type); @@ -66,6 +82,7 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv, struct ethtool_rx_flow_spec *fs, int num_tuples) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs); struct mlx5_flow_table_attr ft_attr = {}; struct mlx5e_ethtool_table *eth_ft; struct mlx5_flow_namespace *ns; @@ -81,18 +98,18 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv, case UDP_V6_FLOW: max_tuples = ETHTOOL_NUM_L3_L4_FTS; prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples); - eth_ft = &priv->fs->ethtool.l3_l4_ft[prio]; + eth_ft = ðtool->l3_l4_ft[prio]; break; case IP_USER_FLOW: case IPV6_USER_FLOW: max_tuples = ETHTOOL_NUM_L3_L4_FTS; prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples); - eth_ft = &priv->fs->ethtool.l3_l4_ft[prio]; + eth_ft = ðtool->l3_l4_ft[prio]; break; case ETHER_FLOW: max_tuples = ETHTOOL_NUM_L2_FTS; prio = max_tuples - num_tuples; - eth_ft = &priv->fs->ethtool.l2_ft[prio]; + eth_ft = ðtool->l2_ft[prio]; prio += MLX5E_ETHTOOL_L2_PRIO; break; default: @@ -382,15 +399,16 @@ static int set_flow_attrs(u32 *match_c, u32 *match_v, static void add_rule_to_list(struct mlx5e_priv *priv, struct mlx5e_ethtool_rule *rule) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs); + struct list_head *head = ðtool->rules; struct mlx5e_ethtool_rule *iter; - struct list_head *head = &priv->fs->ethtool.rules; - list_for_each_entry(iter, &priv->fs->ethtool.rules, list) { + list_for_each_entry(iter, ðtool->rules, list) { if (iter->flow_spec.location > rule->flow_spec.location) break; head = &iter->list; } - priv->fs->ethtool.tot_num_rules++; + ethtool->tot_num_rules++; list_add(&rule->list, head); } @@ -433,15 +451,7 @@ static int flow_get_tirn(struct mlx5e_priv *priv, eth_rule->rss = rss; mlx5e_rss_refcnt_inc(eth_rule->rss); } else { - struct mlx5e_params *params = &priv->channels.params; - enum mlx5e_rq_group group; - u16 ix; - - mlx5e_qid_get_ch_and_group(params, fs->ring_cookie, &ix, &group); - - *tirn = group == MLX5E_RQ_GROUP_XSK ? - mlx5e_rx_res_get_tirn_xsk(priv->rx_res, ix) : - mlx5e_rx_res_get_tirn_direct(priv->rx_res, ix); + *tirn = mlx5e_rx_res_get_tirn_direct(priv->rx_res, fs->ring_cookie); } return 0; @@ -499,15 +509,16 @@ free: return err ? ERR_PTR(err) : rule; } -static void del_ethtool_rule(struct mlx5e_priv *priv, +static void del_ethtool_rule(struct mlx5e_flow_steering *fs, struct mlx5e_ethtool_rule *eth_rule) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs); if (eth_rule->rule) mlx5_del_flow_rules(eth_rule->rule); if (eth_rule->rss) mlx5e_rss_refcnt_dec(eth_rule->rss); list_del(ð_rule->list); - priv->fs->ethtool.tot_num_rules--; + ethtool->tot_num_rules--; put_flow_table(eth_rule->eth_ft); kfree(eth_rule); } @@ -515,9 +526,10 @@ static void del_ethtool_rule(struct mlx5e_priv *priv, static struct mlx5e_ethtool_rule *find_ethtool_rule(struct mlx5e_priv *priv, int location) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs); struct mlx5e_ethtool_rule *iter; - list_for_each_entry(iter, &priv->fs->ethtool.rules, list) { + list_for_each_entry(iter, ðtool->rules, list) { if (iter->flow_spec.location == location) return iter; } @@ -531,7 +543,7 @@ static struct mlx5e_ethtool_rule *get_ethtool_rule(struct mlx5e_priv *priv, eth_rule = find_ethtool_rule(priv, location); if (eth_rule) - del_ethtool_rule(priv, eth_rule); + del_ethtool_rule(priv->fs, eth_rule); eth_rule = kzalloc(sizeof(*eth_rule), GFP_KERNEL); if (!eth_rule) @@ -662,8 +674,7 @@ static int validate_flow(struct mlx5e_priv *priv, return -ENOSPC; if (fs->ring_cookie != RX_CLS_FLOW_DISC) - if (!mlx5e_qid_validate(priv->profile, &priv->channels.params, - fs->ring_cookie)) + if (fs->ring_cookie >= priv->channels.params.num_channels) return -EINVAL; switch (flow_type_mask(fs->flow_type)) { @@ -754,7 +765,7 @@ mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv, return 0; del_ethtool_rule: - del_ethtool_rule(priv, eth_rule); + del_ethtool_rule(priv->fs, eth_rule); return err; } @@ -774,7 +785,7 @@ mlx5e_ethtool_flow_remove(struct mlx5e_priv *priv, int location) goto out; } - del_ethtool_rule(priv, eth_rule); + del_ethtool_rule(priv->fs, eth_rule); out: return err; } @@ -783,12 +794,13 @@ static int mlx5e_ethtool_get_flow(struct mlx5e_priv *priv, struct ethtool_rxnfc *info, int location) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs); struct mlx5e_ethtool_rule *eth_rule; if (location < 0 || location >= MAX_NUM_OF_ETHTOOL_RULES) return -EINVAL; - list_for_each_entry(eth_rule, &priv->fs->ethtool.rules, list) { + list_for_each_entry(eth_rule, ðtool->rules, list) { int index; if (eth_rule->flow_spec.location != location) @@ -826,18 +838,34 @@ mlx5e_ethtool_get_all_flows(struct mlx5e_priv *priv, return err; } -void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv) +int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool) { + *ethtool = kvzalloc(sizeof(**ethtool), GFP_KERNEL); + if (!*ethtool) + return -ENOMEM; + return 0; +} + +void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool) +{ + kvfree(ethtool); +} + +void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs) +{ + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs); struct mlx5e_ethtool_rule *iter; struct mlx5e_ethtool_rule *temp; - list_for_each_entry_safe(iter, temp, &priv->fs->ethtool.rules, list) - del_ethtool_rule(priv, iter); + list_for_each_entry_safe(iter, temp, ðtool->rules, list) + del_ethtool_rule(fs, iter); } -void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv) +void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs) { - INIT_LIST_HEAD(&priv->fs->ethtool.rules); + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs); + + INIT_LIST_HEAD(ðtool->rules); } static int flow_type_to_traffic_type(u32 flow_type) @@ -959,11 +987,12 @@ int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd) int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *info, u32 *rule_locs) { + struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs); int err = 0; switch (info->cmd) { case ETHTOOL_GRXCLSRLCNT: - info->rule_cnt = priv->fs->ethtool.tot_num_rules; + info->rule_cnt = ethtool->tot_num_rules; break; case ETHTOOL_GRXCLSRULE: err = mlx5e_ethtool_get_flow(priv, info, info->fs.location); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index d858667736a32c8cf0cbabea5f231f7d5f370a36..364f04309149225b3a8795d315cd619fb838b62b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -45,6 +45,7 @@ #include "en_tc.h" #include "en_rep.h" #include "en_accel/ipsec.h" +#include "en_accel/macsec.h" #include "en_accel/en_accel.h" #include "en_accel/ktls.h" #include "lib/vxlan.h" @@ -67,22 +68,25 @@ #include "qos.h" #include "en/trap.h" -bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) +bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, + enum mlx5e_mpwrq_umr_mode umr_mode) { - bool striding_rq_umr, inline_umr; - u16 max_wqe_sz_cap; + u16 umr_wqebbs, max_wqebbs; + bool striding_rq_umr; striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && MLX5_CAP_ETH(mdev, reg_umr_sq); - max_wqe_sz_cap = mlx5e_get_max_sq_wqebbs(mdev) * MLX5_SEND_WQE_BB; - inline_umr = max_wqe_sz_cap >= MLX5E_UMR_WQE_INLINE_SZ; if (!striding_rq_umr) return false; - if (!inline_umr) { - mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n", - (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap); + + umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); + max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); + /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is + * calculated from mlx5e_get_max_sq_aligned_wqebbs. + */ + if (WARN_ON(umr_wqebbs > max_wqebbs)) return false; - } + return true; } @@ -199,21 +203,35 @@ static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); } +static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) +{ + u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); + + WARN_ON_ONCE(entries * umr_entry_size % MLX5_OCTWORD); + + return entries * umr_entry_size / MLX5_OCTWORD; +} + static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_icosq *sq, struct mlx5e_umr_wqe *wqe) { struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; - u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS); + u16 octowords; + u8 ds_cnt; + + ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, + rq->mpwqe.umr_mode), + MLX5_SEND_WQE_DS); cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); - cseg->umr_mkey = rq->mkey_be; + cseg->umr_mkey = rq->mpwqe.umr_mkey_be; ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; - ucseg->xlt_octowords = - cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE)); + octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); + ucseg->xlt_octowords = cpu_to_be16(octowords); ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); } @@ -259,10 +277,12 @@ static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq) static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) { int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); + size_t alloc_size; + + alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units, + rq->mpwqe.pages_per_wqe)); - rq->mpwqe.info = kvzalloc_node(array_size(wq_sz, - sizeof(*rq->mpwqe.info)), - GFP_KERNEL, node); + rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); if (!rq->mpwqe.info) return -ENOMEM; @@ -271,18 +291,52 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) return 0; } -static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, - u64 npages, u8 page_shift, u32 *umr_mkey, - dma_addr_t filler_addr) + +static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) +{ + switch (umr_mode) { + case MLX5E_MPWRQ_UMR_MODE_ALIGNED: + return MLX5_MKC_ACCESS_MODE_MTT; + case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: + return MLX5_MKC_ACCESS_MODE_KSM; + case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: + return MLX5_MKC_ACCESS_MODE_KLMS; + case MLX5E_MPWRQ_UMR_MODE_TRIPLE: + return MLX5_MKC_ACCESS_MODE_KSM; + } + WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); + return 0; +} + +static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, + u32 npages, u8 page_shift, u32 *umr_mkey, + dma_addr_t filler_addr, + enum mlx5e_mpwrq_umr_mode umr_mode, + u32 xsk_chunk_size) { struct mlx5_mtt *mtt; + struct mlx5_ksm *ksm; + struct mlx5_klm *klm; + u32 octwords; int inlen; void *mkc; u32 *in; int err; int i; - inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages; + if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || + umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && + !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { + mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); + return -EINVAL; + } + + octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); + + inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), + MLX5_OCTWORD, octwords); + if (inlen < 0) + return inlen; in = kvzalloc(inlen, GFP_KERNEL); if (!in) @@ -294,16 +348,17 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, MLX5_SET(mkc, mkc, umr_en, 1); MLX5_SET(mkc, mkc, lw, 1); MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); + MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); mlx5e_mkey_set_relaxed_ordering(mdev, mkc); MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); MLX5_SET64(mkc, mkc, len, npages << page_shift); - MLX5_SET(mkc, mkc, translations_octword_size, - MLX5_MTT_OCTW(npages)); - MLX5_SET(mkc, mkc, log_page_size, page_shift); - MLX5_SET(create_mkey_in, in, translations_octword_actual_size, - MLX5_MTT_OCTW(npages)); + MLX5_SET(mkc, mkc, translations_octword_size, octwords); + if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) + MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); + else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) + MLX5_SET(mkc, mkc, log_page_size, page_shift); + MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); /* Initialize the mkey with all MTTs pointing to a default * page (filler_addr). When the channels are activated, UMR @@ -311,9 +366,47 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev, * the RQ's pool, while the gaps (wqe_overflow) remain mapped * to the default page. */ - mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); - for (i = 0 ; i < npages ; i++) - mtt[i].ptag = cpu_to_be64(filler_addr); + switch (umr_mode) { + case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: + klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages; i++) { + klm[i << 1] = (struct mlx5_klm) { + .va = cpu_to_be64(filler_addr), + .bcount = cpu_to_be32(xsk_chunk_size), + .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), + }; + klm[(i << 1) + 1] = (struct mlx5_klm) { + .va = cpu_to_be64(filler_addr), + .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), + .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), + }; + } + break; + case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: + ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages; i++) + ksm[i] = (struct mlx5_ksm) { + .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), + .va = cpu_to_be64(filler_addr), + }; + break; + case MLX5E_MPWRQ_UMR_MODE_ALIGNED: + mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages; i++) + mtt[i] = (struct mlx5_mtt) { + .ptag = cpu_to_be64(filler_addr), + }; + break; + case MLX5E_MPWRQ_UMR_MODE_TRIPLE: + ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); + for (i = 0; i < npages * 4; i++) { + ksm[i] = (struct mlx5_ksm) { + .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), + .va = cpu_to_be64(filler_addr), + }; + } + break; + } err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); @@ -356,10 +449,27 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev, static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) { - u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq)); + u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; + u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); + u32 num_entries, max_num_entries; + u32 umr_mkey; + int err; + + max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); + + /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ + if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, + &num_entries) || + num_entries > max_num_entries)) + mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", + __func__, wq_size, rq->mpwqe.mtts_per_wqe, + max_num_entries); - return mlx5e_create_umr_mtt_mkey(mdev, num_mtts, PAGE_SHIFT, - &rq->umr_mkey, rq->wqe_overflow.addr); + err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, + &umr_mkey, rq->wqe_overflow.addr, + rq->mpwqe.umr_mode, xsk_chunk_size); + rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); + return err; } static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, @@ -376,18 +486,20 @@ static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev, &rq->mpwqe.shampo->mkey); } -static u64 mlx5e_get_mpwqe_offset(u16 wqe_ix) -{ - return MLX5E_REQUIRED_MTTS(wqe_ix) << PAGE_SHIFT; -} - static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) { struct mlx5e_wqe_frag_info next_frag = {}; struct mlx5e_wqe_frag_info *prev = NULL; int i; - next_frag.di = &rq->wqe.di[0]; + if (rq->xsk_pool) { + /* Assumptions used by XSK batched allocator. */ + WARN_ON(rq->wqe.info.num_frags != 1); + WARN_ON(rq->wqe.info.log_num_frags != 0); + WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); + } + + next_frag.au = &rq->wqe.alloc_units[0]; for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; @@ -397,7 +509,7 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { - next_frag.di++; + next_frag.au++; next_frag.offset = 0; if (prev) prev->last_in_page = true; @@ -414,12 +526,13 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) prev->last_in_page = true; } -int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) +static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node) { int len = wq_sz << rq->wqe.info.log_num_frags; - rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node); - if (!rq->wqe.di) + rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)), + GFP_KERNEL, node); + if (!rq->wqe.alloc_units) return -ENOMEM; mlx5e_init_frags_partition(rq); @@ -427,9 +540,9 @@ int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node) return 0; } -void mlx5e_free_di_list(struct mlx5e_rq *rq) +static void mlx5e_free_au_list(struct mlx5e_rq *rq) { - kvfree(rq->wqe.di); + kvfree(rq->wqe.alloc_units); } static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) @@ -485,7 +598,7 @@ static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *param if (err) return err; - return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, 0); + return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id); } static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, @@ -572,6 +685,8 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); pool_size = 1 << params->log_rq_mtu_frames; + rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); + switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, @@ -587,8 +702,20 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); - pool_size = MLX5_MPWRQ_PAGES_PER_WQE << - mlx5e_mpwqe_get_log_rq_size(params, xsk); + rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + rq->mpwqe.pages_per_wqe = + mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, + rq->mpwqe.umr_mode); + rq->mpwqe.umr_wqebbs = + mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, + rq->mpwqe.umr_mode); + rq->mpwqe.mtts_per_wqe = + mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, + rq->mpwqe.umr_mode); + + pool_size = rq->mpwqe.pages_per_wqe << + mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); rq->mpwqe.num_strides = @@ -600,7 +727,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, err = mlx5e_create_rq_umr_mkey(mdev, rq); if (err) goto err_rq_drop_page; - rq->mkey_be = cpu_to_be32(rq->umr_mkey); err = mlx5e_rq_alloc_mpwqe_info(rq, node); if (err) @@ -608,7 +734,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node); if (err) - goto err_free_by_rq_type; + goto err_free_mpwqe_info; break; default: /* MLX5_WQ_TYPE_CYCLIC */ @@ -633,11 +759,9 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, goto err_rq_wq_destroy; } - err = mlx5e_init_di_list(rq, wq_sz, node); + err = mlx5e_init_au_list(rq, wq_sz, node); if (err) goto err_rq_frags; - - rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); } if (xsk) { @@ -662,14 +786,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, if (IS_ERR(rq->page_pool)) { err = PTR_ERR(rq->page_pool); rq->page_pool = NULL; - goto err_free_shampo; + goto err_free_by_rq_type; } if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, rq->page_pool); } if (err) - goto err_free_shampo; + goto err_destroy_page_pool; for (i = 0; i < wq_sz; i++) { if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { @@ -677,13 +801,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); u32 byte_count = rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; - u64 dma_offset = mlx5e_get_mpwqe_offset(i); + u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << + rq->mpwqe.page_shift; u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 0 : rq->buff.headroom; wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); wqe->data[0].byte_count = cpu_to_be32(byte_count); - wqe->data[0].lkey = rq->mkey_be; + wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; } else { struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); @@ -721,19 +846,21 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, return 0; -err_free_shampo: - mlx5e_rq_free_shampo(rq); +err_destroy_page_pool: + page_pool_destroy(rq->page_pool); err_free_by_rq_type: switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + mlx5e_rq_free_shampo(rq); +err_free_mpwqe_info: kvfree(rq->mpwqe.info); err_rq_mkey: - mlx5_core_destroy_mkey(mdev, rq->umr_mkey); + mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); err_rq_drop_page: mlx5e_free_mpwqe_rq_drop_page(rq); break; default: /* MLX5_WQ_TYPE_CYCLIC */ - mlx5e_free_di_list(rq); + mlx5e_free_au_list(rq); err_rq_frags: kvfree(rq->wqe.frags); } @@ -761,24 +888,22 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: kvfree(rq->mpwqe.info); - mlx5_core_destroy_mkey(rq->mdev, rq->umr_mkey); + mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); mlx5e_free_mpwqe_rq_drop_page(rq); mlx5e_rq_free_shampo(rq); break; default: /* MLX5_WQ_TYPE_CYCLIC */ kvfree(rq->wqe.frags); - mlx5e_free_di_list(rq); + mlx5e_free_au_list(rq); } for (i = rq->page_cache.head; i != rq->page_cache.tail; i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { - struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i]; - /* With AF_XDP, page_cache is not used, so this loop is not * entered, and it's safe to call mlx5e_page_release_dynamic * directly. */ - mlx5e_page_release_dynamic(rq, dma_info->page, false); + mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false); } xdp_rxq_info_unreg(&rq->xdp_rxq); @@ -833,7 +958,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param) return err; } -int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) +static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) { struct mlx5_core_dev *mdev = rq->mdev; @@ -862,6 +987,32 @@ int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) return err; } +static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) +{ + struct net_device *dev = rq->netdev; + int err; + + err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); + if (err) { + netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); + return err; + } + err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); + if (err) { + netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); + return err; + } + + return 0; +} + +int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) +{ + mlx5e_free_rx_descs(rq); + + return mlx5e_rq_to_ready(rq, curr_state); +} + static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1154,9 +1305,9 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, is_redirect ? &c->priv->channel_stats[c->ix]->xdpsq : &c->priv->channel_stats[c->ix]->rq_xdpsq; - sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); - sq->stop_room = MLX5E_STOP_ROOM(sq->max_sq_wqebbs); - sq->max_sq_mpw_wqebbs = mlx5e_get_sw_max_sq_mpw_wqebbs(sq->max_sq_wqebbs); + sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : + mlx5e_stop_room_for_max_wqe(mdev); + sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); param->wq.db_numa_node = cpu_to_node(c->cpu); err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); @@ -1231,7 +1382,6 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c, sq->channel = c; sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; sq->reserved_room = param->stop_room; - sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); param->wq.db_numa_node = cpu_to_node(c->cpu); err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); @@ -1317,8 +1467,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; sq->min_inline_mode = params->tx_min_inline_mode; sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); - sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev); - sq->max_sq_mpw_wqebbs = mlx5e_get_sw_max_sq_mpw_wqebbs(sq->max_sq_wqebbs); + sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); @@ -2280,7 +2429,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, c->aff_mask = irq_get_effective_affinity_mask(irq); c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); - netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64); + netif_napi_add(netdev, &c->napi, mlx5e_napi_poll); err = mlx5e_open_queues(c, params, cparam); if (unlikely(err)) @@ -2318,10 +2467,11 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c) mlx5e_activate_txqsq(&c->sq[tc]); mlx5e_activate_icosq(&c->icosq); mlx5e_activate_icosq(&c->async_icosq); - mlx5e_activate_rq(&c->rq); if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) mlx5e_activate_xsk(c); + else + mlx5e_activate_rq(&c->rq); mlx5e_trigger_napi_icosq(c); } @@ -2332,8 +2482,9 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c) if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) mlx5e_deactivate_xsk(c); + else + mlx5e_deactivate_rq(&c->rq); - mlx5e_deactivate_rq(&c->rq); mlx5e_deactivate_icosq(&c->async_icosq); mlx5e_deactivate_icosq(&c->icosq); for (tc = 0; tc < c->num_tc; tc++) @@ -2425,8 +2576,6 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs) mlx5e_ptp_activate_channel(chs->ptp); } -#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ - static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) { int err = 0; @@ -2434,8 +2583,12 @@ static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) for (i = 0; i < chs->num; i++) { int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; + struct mlx5e_channel *c = chs->c[i]; - err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout); + if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) + continue; + + err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); /* Don't wait on the XSK RQ, because the newer xdpsock sample * doesn't provide any Fill Ring entries at the setup stage. @@ -2600,7 +2753,7 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; struct net_device *netdev = priv->netdev; int old_num_txqs, old_ntc; - int num_rxqs, nch, ntc; + int nch, ntc; int err; int i; @@ -2611,7 +2764,6 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) nch = priv->channels.params.num_channels; ntc = priv->channels.params.mqprio.num_tc; - num_rxqs = nch * priv->profile->rq_groups; tc_to_txq = priv->channels.params.mqprio.tc_to_txq; err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); @@ -2620,7 +2772,7 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) err = mlx5e_update_tx_netdev_queues(priv); if (err) goto err_tcs; - err = netif_set_real_num_rx_queues(netdev, num_rxqs); + err = netif_set_real_num_rx_queues(netdev, nch); if (err) { netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); goto err_txqs; @@ -2738,7 +2890,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) netif_tx_start_all_queues(priv->netdev); if (mlx5e_is_vport_rep(priv)) - mlx5e_add_sqs_fwd_rules(priv); + mlx5e_rep_activate_channels(priv); mlx5e_wait_channels_min_rx_wqes(&priv->channels); @@ -2752,7 +2904,7 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) mlx5e_rx_res_channels_deactivate(priv->rx_res); if (mlx5e_is_vport_rep(priv)) - mlx5e_remove_sqs_fwd_rules(priv); + mlx5e_rep_deactivate_channels(priv); /* The results of ndo_select_queue are unreliable, while netdev config * is being changed (real_num_tx_queues, num_tc). Stop all queues to @@ -3547,7 +3699,8 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) stats->rx_length_errors = PPORT_802_3_GET(pstats, a_in_range_length_errors) + PPORT_802_3_GET(pstats, a_out_of_range_length_field) + - PPORT_802_3_GET(pstats, a_frame_too_long_errors); + PPORT_802_3_GET(pstats, a_frame_too_long_errors) + + VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); stats->rx_crc_errors = PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); @@ -3669,9 +3822,11 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) struct mlx5e_priv *priv = netdev_priv(netdev); if (enable) - mlx5e_enable_cvlan_filter(priv); + mlx5e_enable_cvlan_filter(priv->fs, + !!(priv->netdev->flags & IFF_PROMISC)); else - mlx5e_disable_cvlan_filter(priv); + mlx5e_disable_cvlan_filter(priv->fs, + !!(priv->netdev->flags & IFF_PROMISC)); return 0; } @@ -3682,7 +3837,9 @@ static int set_feature_hw_tc(struct net_device *netdev, bool enable) int err = 0; #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) - if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { + int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : + MLX5_TC_FLAG(NIC_OFFLOAD); + if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { netdev_err(netdev, "Active offloaded tc filters, can't turn hw_tc_offload off\n"); return -EINVAL; @@ -3778,7 +3935,7 @@ static int set_feature_rx_vlan(struct net_device *netdev, bool enable) mutex_lock(&priv->state_lock); - priv->fs->vlan_strip_disable = !enable; + mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); priv->channels.params.vlan_strip_disable = !enable; if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) @@ -3786,7 +3943,7 @@ static int set_feature_rx_vlan(struct net_device *netdev, bool enable) err = mlx5e_modify_channels_vsd(&priv->channels, !enable); if (err) { - priv->fs->vlan_strip_disable = enable; + mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); priv->channels.params.vlan_strip_disable = enable; } unlock: @@ -3824,9 +3981,9 @@ static int set_feature_arfs(struct net_device *netdev, bool enable) int err; if (enable) - err = mlx5e_arfs_enable(priv); + err = mlx5e_arfs_enable(priv->fs); else - err = mlx5e_arfs_disable(priv); + err = mlx5e_arfs_disable(priv->fs); return err; } @@ -3910,12 +4067,14 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, netdev_features_t features) { struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_vlan_table *vlan; struct mlx5e_params *params; + vlan = mlx5e_fs_get_vlan(priv->fs); mutex_lock(&priv->state_lock); params = &priv->channels.params; - if (!priv->fs->vlan || - !bitmap_empty(mlx5e_vlan_get_active_svlans(priv->fs->vlan), VLAN_N_VID)) { + if (!vlan || + !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { /* HW strips the outer C-tag header, this is a problem * for S-tag traffic. */ @@ -4004,7 +4163,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. */ max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); - max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk); + max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); max_mtu = min(max_mtu_frame, max_mtu_page); netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n", @@ -4016,14 +4175,16 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, return true; } -static bool mlx5e_params_validate_xdp(struct net_device *netdev, struct mlx5e_params *params) +static bool mlx5e_params_validate_xdp(struct net_device *netdev, + struct mlx5_core_dev *mdev, + struct mlx5e_params *params) { bool is_linear; /* No XSK params: AF_XDP can't be enabled yet at the point of setting * the XDP program. */ - is_linear = mlx5e_rx_is_linear_skb(params, NULL); + is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL); if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", @@ -4060,7 +4221,8 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, if (err) goto out; - if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, &new_params)) { + if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, + &new_params)) { err = -EINVAL; goto out; } @@ -4075,19 +4237,21 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO) reset = false; - if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { + if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && + params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) { bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL); bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, &new_params, NULL); - u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL); - u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_params, NULL); + u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL); + u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL); /* Always reset in linear mode - hw_mtu is used in data path. * Check that the mode was non-linear and didn't change. * If XSK is active, XSK RQs are linear. + * Reset if the RQ size changed, even if it's non-linear. */ if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt && - ppw_old == ppw_new) + sz_old == sz_new) reset = false; } @@ -4537,7 +4701,7 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) new_params = priv->channels.params; new_params.xdp_prog = prog; - if (!mlx5e_params_validate_xdp(netdev, &new_params)) + if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params)) return -EINVAL; return 0; @@ -4575,8 +4739,20 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) new_params = priv->channels.params; new_params.xdp_prog = prog; - if (reset) - mlx5e_set_rq_type(priv->mdev, &new_params); + + /* XDP affects striding RQ parameters. Block XDP if striding RQ won't be + * supported with the new parameters: if PAGE_SIZE is bigger than + * MLX5_MPWQE_LOG_STRIDE_SZ_MAX, striding RQ can't be used, even though + * the MTU is small enough for the linear mode, because XDP uses strides + * of PAGE_SIZE on regular RQs. + */ + if (reset && MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { + /* Checking for regular RQs here; XSK RQs were checked on XSK bind. */ + err = mlx5e_mpwrq_validate_regular(priv->mdev, &new_params); + if (err) + goto unlock; + } + old_prog = priv->channels.params.xdp_prog; err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); @@ -4769,14 +4945,6 @@ void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 /* RQ */ mlx5e_build_rq_params(mdev, params); - /* HW LRO */ - if (MLX5_CAP_ETH(mdev, lro_cap) && - params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { - /* No XSK params: checking the availability of striding RQ in general. */ - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) - params->packet_merge.type = slow_pci_heuristic(mdev) ? - MLX5E_PACKET_MERGE_NONE : MLX5E_PACKET_MERGE_LRO; - } params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); /* CQ moderation params */ @@ -4904,7 +5072,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) if (!!MLX5_CAP_ETH(mdev, lro_cap) && !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && - mlx5e_check_fragmented_striding_rq_cap(mdev)) + mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, + MLX5E_MPWRQ_UMR_MODE_ALIGNED)) netdev->vlan_features |= NETIF_F_LRO; netdev->hw_features = netdev->vlan_features; @@ -4992,6 +5161,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev) netif_set_tso_max_size(netdev, GSO_MAX_SIZE); mlx5e_set_netdev_dev_addr(netdev); + mlx5e_macsec_build_netdev(priv); mlx5e_ipsec_build_netdev(priv); mlx5e_ktls_build_netdev(priv); } @@ -5093,7 +5263,7 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) goto err_destroy_q_counters; } - features = MLX5E_RX_RES_FEATURE_XSK | MLX5E_RX_RES_FEATURE_PTP; + features = MLX5E_RX_RES_FEATURE_PTP; if (priv->channels.params.tunneled_offload_en) features |= MLX5E_RX_RES_FEATURE_INNER_FT; err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features, @@ -5103,7 +5273,8 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) if (err) goto err_close_drop_rq; - err = mlx5e_create_flow_steering(priv); + err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, + priv->netdev); if (err) { mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); goto err_destroy_rx_res; @@ -5126,7 +5297,8 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) err_tc_nic_cleanup: mlx5e_tc_nic_cleanup(priv); err_destroy_flow_steering: - mlx5e_destroy_flow_steering(priv); + mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), + priv->profile); err_destroy_rx_res: mlx5e_rx_res_destroy(priv->rx_res); err_close_drop_rq: @@ -5142,7 +5314,8 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) { mlx5e_accel_cleanup_rx(priv); mlx5e_tc_nic_cleanup(priv); - mlx5e_destroy_flow_steering(priv); + mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE), + priv->profile); mlx5e_rx_res_destroy(priv->rx_res); mlx5e_close_drop_rq(&priv->drop_rq); mlx5e_destroy_q_counters(priv); @@ -5194,9 +5367,14 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; + int err; mlx5e_fs_init_l2_addr(priv->fs, netdev); + err = mlx5e_macsec_init(priv); + if (err) + mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); + /* Marking the link as currently not needed by the Driver */ if (!netif_running(netdev)) mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); @@ -5254,6 +5432,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv) mlx5e_disable_async_events(priv); mlx5_lag_remove_netdev(mdev, priv->netdev); mlx5_vxlan_reset_to_default(mdev->vxlan); + mlx5e_macsec_cleanup(priv); } int mlx5e_update_nic_rx(struct mlx5e_priv *priv) @@ -5275,7 +5454,6 @@ static const struct mlx5e_profile mlx5e_nic_profile = { .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_nic, .max_tc = MLX5E_MAX_NUM_TC, - .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), .stats_grps = mlx5e_nic_stats_grps, .stats_grps_num = mlx5e_nic_stats_grps_num, .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | @@ -5308,8 +5486,7 @@ mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, max_nch = mlx5e_profile_max_num_channels(mdev, profile); /* netdev rx queues */ - tmp = netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1); - max_nch = min_t(unsigned int, max_nch, tmp); + max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); /* netdev tx queues */ tmp = netdev->num_tx_queues; @@ -5453,11 +5630,7 @@ static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) { - unsigned int nch; - - nch = mlx5e_profile_max_num_channels(mdev, profile); - - return nch * profile->rq_groups; + return mlx5e_profile_max_num_channels(mdev, profile); } struct net_device * @@ -5518,7 +5691,8 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) clear_bit(MLX5E_STATE_DESTROYING, &priv->state); if (priv->fs) - priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); + mlx5e_fs_set_state_destroy(priv->fs, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); /* max number of channels may have changed */ max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); @@ -5579,7 +5753,8 @@ out: mlx5e_reset_channels(priv->netdev); set_bit(MLX5E_STATE_DESTROYING, &priv->state); if (priv->fs) - priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); + mlx5e_fs_set_state_destroy(priv->fs, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); cancel_work_sync(&priv->update_stats_work); return err; } @@ -5590,7 +5765,8 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv) set_bit(MLX5E_STATE_DESTROYING, &priv->state); if (priv->fs) - priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); + mlx5e_fs_set_state_destroy(priv->fs, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); if (profile->disable) profile->disable(priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 0c66774a1720cc6117e55cefeb888de76d2aedca..794cd8dfe9c911f7c6480c877cb81bc6beac96fa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -56,6 +56,7 @@ #include "en_accel/ipsec.h" #include "en/tc/int_port.h" #include "en/ptp.h" +#include "en/fs_ethtool.h" #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) @@ -69,7 +70,7 @@ static void mlx5e_rep_get_drvinfo(struct net_device *dev, struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5_core_dev *mdev = priv->mdev; - strlcpy(drvinfo->driver, mlx5e_rep_driver_name, + strscpy(drvinfo->driver, mlx5e_rep_driver_name, sizeof(drvinfo->driver)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%04d (%.16s)", @@ -397,7 +398,8 @@ out_err: return err; } -int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) +static int +mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) { int sqs_per_channel = mlx5e_get_dcb_num_tc(&priv->channels.params); struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; @@ -451,7 +453,8 @@ out: return err; } -void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) +static void +mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5e_rep_priv *rpriv = priv->ppriv; @@ -460,6 +463,49 @@ void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) mlx5e_sqs2vport_stop(esw, rep); } +static int +mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv) +{ + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + struct mlx5e_rep_priv *rpriv = priv->ppriv; + struct mlx5_eswitch_rep *rep = rpriv->rep; + struct mlx5_flow_handle *flow_rule; + struct mlx5_flow_group *g; + + g = esw->fdb_table.offloads.send_to_vport_meta_grp; + if (!g) + return 0; + + flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, rep->vport); + if (IS_ERR(flow_rule)) + return PTR_ERR(flow_rule); + + rpriv->send_to_vport_meta_rule = flow_rule; + + return 0; +} + +static void +mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv) +{ + struct mlx5e_rep_priv *rpriv = priv->ppriv; + + if (rpriv->send_to_vport_meta_rule) + mlx5_eswitch_del_send_to_vport_meta_rule(rpriv->send_to_vport_meta_rule); +} + +void mlx5e_rep_activate_channels(struct mlx5e_priv *priv) +{ + mlx5e_add_sqs_fwd_rules(priv); + mlx5e_rep_add_meta_tunnel_rule(priv); +} + +void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv) +{ + mlx5e_rep_del_meta_tunnel_rule(priv); + mlx5e_remove_sqs_fwd_rules(priv); +} + static int mlx5e_rep_open(struct net_device *dev) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -662,6 +708,8 @@ static void mlx5e_build_rep_params(struct net_device *netdev) params->mqprio.num_tc = 1; params->tunneled_offload_en = false; + if (rep->vport != MLX5_VPORT_UPLINK) + params->vlan_strip_disable = true; mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); } @@ -745,19 +793,20 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) struct ttc_params ttc_params = {}; int err; - priv->fs->ns = mlx5_get_flow_namespace(priv->mdev, - MLX5_FLOW_NAMESPACE_KERNEL); + mlx5e_fs_set_ns(priv->fs, + mlx5_get_flow_namespace(priv->mdev, + MLX5_FLOW_NAMESPACE_KERNEL), false); /* The inner_ttc in the ttc params is intentionally not set */ - mlx5e_set_ttc_params(priv, &ttc_params, false); + mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false); if (rep->vport != MLX5_VPORT_UPLINK) /* To give uplik rep TTC a lower level for chaining from root ft */ ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1; - priv->fs->ttc = mlx5_create_ttc_table(priv->mdev, &ttc_params); - if (IS_ERR(priv->fs->ttc)) { - err = PTR_ERR(priv->fs->ttc); + mlx5e_fs_set_ttc(priv->fs, mlx5_create_ttc_table(priv->mdev, &ttc_params), false); + if (IS_ERR(mlx5e_fs_get_ttc(priv->fs, false))) { + err = PTR_ERR(mlx5e_fs_get_ttc(priv->fs, false)); netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", err); return err; @@ -777,7 +826,7 @@ static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv) /* non uplik reps will skip any bypass tables and go directly to * their own ttc */ - rpriv->root_ft = mlx5_get_ttc_flow_table(priv->fs->ttc); + rpriv->root_ft = mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv->fs, false)); return 0; } @@ -885,14 +934,14 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) if (err) goto err_destroy_root_ft; - mlx5e_ethtool_init_steering(priv); + mlx5e_ethtool_init_steering(priv->fs); return 0; err_destroy_root_ft: mlx5e_destroy_rep_root_ft(priv); err_destroy_ttc_table: - mlx5_destroy_ttc_table(priv->fs->ttc); + mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); err_destroy_rx_res: mlx5e_rx_res_destroy(priv->rx_res); err_close_drop_rq: @@ -906,10 +955,10 @@ err_free_fs: static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) { - mlx5e_ethtool_cleanup_steering(priv); + mlx5e_ethtool_cleanup_steering(priv->fs); rep_vport_rx_rule_destroy(priv); mlx5e_destroy_rep_root_ft(priv); - mlx5_destroy_ttc_table(priv->fs->ttc); + mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); mlx5e_rx_res_destroy(priv->rx_res); mlx5e_close_drop_rq(&priv->drop_rq); mlx5e_rx_res_free(priv->rx_res); @@ -1175,7 +1224,6 @@ static const struct mlx5e_profile mlx5e_rep_profile = { .update_stats = mlx5e_stats_update_ndo_stats, .rx_handlers = &mlx5e_rx_handlers_rep, .max_tc = 1, - .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .stats_grps = mlx5e_rep_stats_grps, .stats_grps_num = mlx5e_rep_stats_grps_num, .max_nch_limit = mlx5e_rep_max_nch_limit, @@ -1195,8 +1243,6 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = { .update_carrier = mlx5e_update_carrier, .rx_handlers = &mlx5e_rx_handlers_rep, .max_tc = MLX5E_MAX_NUM_TC, - /* XSK is needed so we can replace profile with NIC netdev */ - .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), .stats_grps = mlx5e_ul_rep_stats_grps, .stats_grps_num = mlx5e_ul_rep_stats_grps_num, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index dec183ccd4acbc9240aed92f0b3c8fd8529d6fe6..b4e691760da9f964d5851b1289f891ae14a9c688 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -111,6 +111,7 @@ struct mlx5e_rep_priv { struct list_head vport_sqs_list; struct mlx5_rep_uplink_priv uplink_priv; /* valid for uplink rep */ struct rtnl_link_stats64 prev_vf_vport_stats; + struct mlx5_flow_handle *send_to_vport_meta_rule; struct rhashtable tc_ht; }; @@ -241,8 +242,8 @@ int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev, void *sp); bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv); -int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv); -void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv); +void mlx5e_rep_activate_channels(struct mlx5e_priv *priv); +void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv); void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv); @@ -256,8 +257,8 @@ static inline bool mlx5e_eswitch_rep(const struct net_device *netdev) #else /* CONFIG_MLX5_ESWITCH */ static inline bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) { return false; } -static inline int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) { return 0; } -static inline void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) {} +static inline void mlx5e_rep_activate_channels(struct mlx5e_priv *priv) {} +static inline void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv) {} static inline int mlx5e_rep_init(void) { return 0; }; static inline void mlx5e_rep_cleanup(void) {}; static inline bool mlx5e_rep_has_offload_stats(const struct net_device *dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 24de37b79f5a917b304c011fcebcd09748ee5c6a..58084650151f836102b0b09518a0525016aed3b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "en.h" #include "en/txrx.h" #include "en_tc.h" @@ -49,6 +50,7 @@ #include "en/rep/tc.h" #include "ipoib/ipoib.h" #include "en_accel/ipsec.h" +#include "en_accel/macsec.h" #include "en_accel/ipsec_rxtx.h" #include "en_accel/ktls_txrx.h" #include "en/xdp.h" @@ -237,69 +239,61 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) return false; } - cache->page_cache[cache->tail].page = page; - cache->page_cache[cache->tail].addr = page_pool_get_dma_addr(page); + cache->page_cache[cache->tail] = page; cache->tail = tail_next; return true; } -static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) +static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { struct mlx5e_page_cache *cache = &rq->page_cache; struct mlx5e_rq_stats *stats = rq->stats; + dma_addr_t addr; if (unlikely(cache->head == cache->tail)) { stats->cache_empty++; return false; } - if (page_ref_count(cache->page_cache[cache->head].page) != 1) { + if (page_ref_count(cache->page_cache[cache->head]) != 1) { stats->cache_busy++; return false; } - *dma_info = cache->page_cache[cache->head]; + au->page = cache->page_cache[cache->head]; cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1); stats->cache_reuse++; - dma_sync_single_for_device(rq->pdev, dma_info->addr, - PAGE_SIZE, - DMA_FROM_DEVICE); + addr = page_pool_get_dma_addr(au->page); + /* Non-XSK always uses PAGE_SIZE. */ + dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, DMA_FROM_DEVICE); return true; } -static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) +static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au) { - if (mlx5e_rx_cache_get(rq, dma_info)) + dma_addr_t addr; + + if (mlx5e_rx_cache_get(rq, au)) return 0; - dma_info->page = page_pool_dev_alloc_pages(rq->page_pool); - if (unlikely(!dma_info->page)) + au->page = page_pool_dev_alloc_pages(rq->page_pool); + if (unlikely(!au->page)) return -ENOMEM; - dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE, - rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); - if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) { - page_pool_recycle_direct(rq->page_pool, dma_info->page); - dma_info->page = NULL; + /* Non-XSK always uses PAGE_SIZE. */ + addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE, + rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); + if (unlikely(dma_mapping_error(rq->pdev, addr))) { + page_pool_recycle_direct(rq->page_pool, au->page); + au->page = NULL; return -ENOMEM; } - page_pool_set_dma_addr(dma_info->page, dma_info->addr); + page_pool_set_dma_addr(au->page, addr); return 0; } -static inline int mlx5e_page_alloc(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info) -{ - if (rq->xsk_pool) - return mlx5e_xsk_page_alloc_pool(rq, dma_info); - else - return mlx5e_page_alloc_pool(rq, dma_info); -} - void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page) { dma_addr_t dma_addr = page_pool_get_dma_addr(page); @@ -324,32 +318,18 @@ void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool rec } } -static inline void mlx5e_page_release(struct mlx5e_rq *rq, - struct mlx5e_dma_info *dma_info, - bool recycle) -{ - if (rq->xsk_pool) - /* The `recycle` parameter is ignored, and the page is always - * put into the Reuse Ring, because there is no way to return - * the page to the userspace when the interface goes down. - */ - xsk_buff_free(dma_info->xsk); - else - mlx5e_page_release_dynamic(rq, dma_info->page, recycle); -} - static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *frag) { int err = 0; if (!frag->offset) - /* On first frag (offset == 0), replenish page (dma_info actually). - * Other frags that point to the same dma_info (with a different + /* On first frag (offset == 0), replenish page (alloc_unit actually). + * Other frags that point to the same alloc_unit (with a different * offset) should just use the new one without replenishing again * by themselves. */ - err = mlx5e_page_alloc(rq, frag->di); + err = mlx5e_page_alloc_pool(rq, frag->au); return err; } @@ -359,7 +339,7 @@ static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq, bool recycle) { if (frag->last_in_page) - mlx5e_page_release(rq, frag->di, recycle); + mlx5e_page_release_dynamic(rq, frag->au->page, recycle); } static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix) @@ -375,6 +355,7 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, int i; for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) { + dma_addr_t addr; u16 headroom; err = mlx5e_get_rx_frag(rq, frag); @@ -382,8 +363,8 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe, goto free_frags; headroom = i == 0 ? rq->buff.headroom : 0; - wqe->data[i].addr = cpu_to_be64(frag->di->addr + - frag->offset + headroom); + addr = page_pool_get_dma_addr(frag->au->page); + wqe->data[i].addr = cpu_to_be64(addr + frag->offset + headroom); } return 0; @@ -401,6 +382,15 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq, { int i; + if (rq->xsk_pool) { + /* The `recycle` parameter is ignored, and the page is always + * put into the Reuse Ring, because there is no way to return + * the page to the userspace when the interface goes down. + */ + xsk_buff_free(wi->au->xsk); + return; + } + for (i = 0; i < rq->wqe.info.num_frags; i++, wi++) mlx5e_put_rx_frag(rq, wi, recycle); } @@ -412,84 +402,76 @@ static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix) mlx5e_free_rx_wqe(rq, wi, false); } -static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk) +static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; - int err; int i; - if (rq->xsk_pool) { - int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags; - - /* Check in advance that we have enough frames, instead of - * allocating one-by-one, failing and moving frames to the - * Reuse Ring. - */ - if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, pages_desired))) - return -ENOMEM; - } - for (i = 0; i < wqe_bulk; i++) { - struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i); - - err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i); - if (unlikely(err)) - goto free_wqes; - } + int j = mlx5_wq_cyc_ctr2ix(wq, ix + i); + struct mlx5e_rx_wqe_cyc *wqe; - return 0; + wqe = mlx5_wq_cyc_get_wqe(wq, j); -free_wqes: - while (--i >= 0) - mlx5e_dealloc_rx_wqe(rq, ix + i); + if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, j))) + break; + } - return err; + return i; } static inline void mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, - struct mlx5e_dma_info *di, u32 frag_offset, u32 len, + union mlx5e_alloc_unit *au, u32 frag_offset, u32 len, unsigned int truesize) { - dma_sync_single_for_cpu(rq->pdev, - di->addr + frag_offset, - len, DMA_FROM_DEVICE); - page_ref_inc(di->page); + dma_addr_t addr = page_pool_get_dma_addr(au->page); + + dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, DMA_FROM_DEVICE); + page_ref_inc(au->page); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - di->page, frag_offset, len, truesize); + au->page, frag_offset, len, truesize); } static inline void mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb, - struct mlx5e_dma_info *dma_info, + struct page *page, dma_addr_t addr, int offset_from, int dma_offset, u32 headlen) { - const void *from = page_address(dma_info->page) + offset_from; + const void *from = page_address(page) + offset_from; /* Aligning len to sizeof(long) optimizes memcpy performance */ unsigned int len = ALIGN(headlen, sizeof(long)); - dma_sync_single_for_cpu(pdev, dma_info->addr + dma_offset, len, - DMA_FROM_DEVICE); + dma_sync_single_for_cpu(pdev, addr + dma_offset, len, DMA_FROM_DEVICE); skb_copy_to_linear_data(skb, from, len); } static void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle) { + union mlx5e_alloc_unit *alloc_units = wi->alloc_units; bool no_xdp_xmit; - struct mlx5e_dma_info *dma_info = wi->umr.dma_info; int i; /* A common case for AF_XDP. */ - if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE)) + if (bitmap_full(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe)) return; - no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap, - MLX5_MPWRQ_PAGES_PER_WQE); + no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); - for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) - if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap)) - mlx5e_page_release(rq, &dma_info[i], recycle); + if (rq->xsk_pool) { + /* The `recycle` parameter is ignored, and the page is always + * put into the Reuse Ring, because there is no way to return + * the page to the userspace when the interface goes down. + */ + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++) + if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap)) + xsk_buff_free(alloc_units[i].xsk); + } else { + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++) + if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap)) + mlx5e_page_release_dynamic(rq, alloc_units[i].page, recycle); + } } static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n) @@ -574,11 +556,13 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq, header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) << MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE; if (!(header_offset & (PAGE_SIZE - 1))) { - err = mlx5e_page_alloc(rq, dma_info); + union mlx5e_alloc_unit au; + + err = mlx5e_page_alloc_pool(rq, &au); if (unlikely(err)) goto err_unmap; - addr = dma_info->addr; - page = dma_info->page; + page = dma_info->page = au.page; + addr = dma_info->addr = page_pool_get_dma_addr(au.page); } else { dma_info->addr = addr + header_offset; dma_info->page = page; @@ -611,7 +595,7 @@ err_unmap: dma_info = &shampo->info[--index]; if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) { dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE); - mlx5e_page_release(rq, dma_info, true); + mlx5e_page_release_dynamic(rq, dma_info->page, true); } } rq->stats->buff_alloc_err++; @@ -659,57 +643,55 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { - struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; - struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0]; + struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); + union mlx5e_alloc_unit *au = &wi->alloc_units[0]; struct mlx5e_icosq *sq = rq->icosq; struct mlx5_wq_cyc *wq = &sq->wq; struct mlx5e_umr_wqe *umr_wqe; + u32 offset; /* 17-bit value with MTT. */ u16 pi; int err; int i; - /* Check in advance that we have enough frames, instead of allocating - * one-by-one, failing and moving frames to the Reuse Ring. - */ - if (rq->xsk_pool && - unlikely(!xsk_buff_can_alloc(rq->xsk_pool, MLX5_MPWRQ_PAGES_PER_WQE))) { - err = -ENOMEM; - goto err; - } - if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { err = mlx5e_alloc_rx_hd_mpwqe(rq); if (unlikely(err)) goto err; } - pi = mlx5e_icosq_get_next_pi(sq, MLX5E_UMR_WQEBBS); + pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs); umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi); - memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts)); + memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe)); + + for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) { + dma_addr_t addr; - for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) { - err = mlx5e_page_alloc(rq, dma_info); + err = mlx5e_page_alloc_pool(rq, au); if (unlikely(err)) goto err_unmap; - umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR); + addr = page_pool_get_dma_addr(au->page); + umr_wqe->inline_mtts[i] = (struct mlx5_mtt) { + .ptag = cpu_to_be64(addr | MLX5_EN_WR), + }; } - bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE); + bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe); wi->consumed_strides = 0; umr_wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR); - umr_wqe->uctrl.xlt_offset = - cpu_to_be16(MLX5_ALIGNED_MTTS_OCTW(MLX5E_REQUIRED_MTTS(ix))); + + offset = (ix * rq->mpwqe.mtts_per_wqe) * sizeof(struct mlx5_mtt) / MLX5_OCTWORD; + umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset); sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) { .wqe_type = MLX5E_ICOSQ_WQE_UMR_RX, - .num_wqebbs = MLX5E_UMR_WQEBBS, + .num_wqebbs = rq->mpwqe.umr_wqebbs, .umr.rq = rq, }; - sq->pc += MLX5E_UMR_WQEBBS; + sq->pc += rq->mpwqe.umr_wqebbs; sq->doorbell_cseg = &umr_wqe->ctrl; @@ -717,8 +699,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) err_unmap: while (--i >= 0) { - dma_info--; - mlx5e_page_release(rq, dma_info, true); + au--; + mlx5e_page_release_dynamic(rq, au->page, true); } err: @@ -752,7 +734,7 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE); if (hd_info->page != deleted_page) { deleted_page = hd_info->page; - mlx5e_page_release(rq, hd_info, false); + mlx5e_page_release_dynamic(rq, hd_info->page, false); } } @@ -767,7 +749,7 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) { - struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix]; + struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix); /* Don't recycle, this function is called on rq/netdev close */ mlx5e_free_rx_mpwqe(rq, wi, false); } @@ -775,38 +757,51 @@ static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; - u8 wqe_bulk; - int err; + int wqe_bulk, count; + bool busy = false; + u16 head; if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) return false; - wqe_bulk = rq->wqe.info.wqe_bulk; - - if (mlx5_wq_cyc_missing(wq) < wqe_bulk) + if (mlx5_wq_cyc_missing(wq) < rq->wqe.info.wqe_bulk) return false; if (rq->page_pool) page_pool_nid_changed(rq->page_pool, numa_mem_id()); - do { - u16 head = mlx5_wq_cyc_get_head(wq); + wqe_bulk = mlx5_wq_cyc_missing(wq); + head = mlx5_wq_cyc_get_head(wq); - err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk); - if (unlikely(err)) { - rq->stats->buff_alloc_err++; - break; - } + /* Don't allow any newly allocated WQEs to share the same page with old + * WQEs that aren't completed yet. Stop earlier. + */ + wqe_bulk -= (head + wqe_bulk) & rq->wqe.info.wqe_index_mask; - mlx5_wq_cyc_push_n(wq, wqe_bulk); - } while (mlx5_wq_cyc_missing(wq) >= wqe_bulk); + if (!rq->xsk_pool) + count = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk); + else if (likely(!rq->xsk_pool->dma_need_sync)) + count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk); + else + /* If dma_need_sync is true, it's more efficient to call + * xsk_buff_alloc in a loop, rather than xsk_buff_alloc_batch, + * because the latter does the same check and returns only one + * frame. + */ + count = mlx5e_xsk_alloc_rx_wqes(rq, head, wqe_bulk); + + mlx5_wq_cyc_push_n(wq, count); + if (unlikely(count != wqe_bulk)) { + rq->stats->buff_alloc_err++; + busy = true; + } /* ensure wqes are visible to device before updating doorbell record */ dma_wmb(); mlx5_wq_cyc_update_db_record(wq); - return !!err; + return busy; } void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq) @@ -974,7 +969,8 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq) head = rq->mpwqe.actual_wq_head; i = missing; do { - alloc_err = mlx5e_alloc_rx_mpwqe(rq, head); + alloc_err = rq->xsk_pool ? mlx5e_xsk_alloc_rx_mpwqe(rq, head) : + mlx5e_alloc_rx_mpwqe(rq, head); if (unlikely(alloc_err)) break; @@ -1421,6 +1417,9 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, if (unlikely(mlx5_ipsec_is_rx_flow(cqe))) mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe); + if (unlikely(mlx5e_macsec_is_rx_flow(cqe))) + mlx5e_macsec_offload_handle_rx_skb(netdev, skb, cqe); + if (lro_num_seg > 1) { mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt); skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg); @@ -1524,19 +1523,21 @@ static struct sk_buff * mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, u32 cqe_bcnt) { - struct mlx5e_dma_info *di = wi->di; + union mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; u32 metasize = 0; void *va, *data; + dma_addr_t addr; u32 frag_size; - va = page_address(di->page) + wi->offset; + va = page_address(au->page) + wi->offset; data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); @@ -1546,7 +1547,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp); - if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) + if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) return NULL; /* page/packet was consumed by XDP */ rx_headroom = xdp.data - xdp.data_hard_start; @@ -1559,7 +1560,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, return NULL; /* queue up for recycling/reuse */ - page_ref_inc(di->page); + page_ref_inc(au->page); return skb; } @@ -1570,20 +1571,22 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi { struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; struct mlx5e_wqe_frag_info *head_wi = wi; + union mlx5e_alloc_unit *au = wi->au; u16 rx_headroom = rq->buff.headroom; - struct mlx5e_dma_info *di = wi->di; struct skb_shared_info *sinfo; u32 frag_consumed_bytes; struct bpf_prog *prog; struct xdp_buff xdp; struct sk_buff *skb; + dma_addr_t addr; u32 truesize; void *va; - va = page_address(di->page) + wi->offset; + va = page_address(au->page) + wi->offset; frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, rq->buff.frame0_sz, DMA_FROM_DEVICE); net_prefetchw(va); /* xdp_frame data area */ net_prefetch(va + rx_headroom); @@ -1599,11 +1602,12 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi while (cqe_bcnt) { skb_frag_t *frag; - di = wi->di; + au = wi->au; frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt); - dma_sync_single_for_cpu(rq->pdev, di->addr + wi->offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_for_cpu(rq->pdev, addr + wi->offset, frag_consumed_bytes, DMA_FROM_DEVICE); if (!xdp_buff_has_frags(&xdp)) { @@ -1616,11 +1620,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi } frag = &sinfo->frags[sinfo->nr_frags++]; - __skb_frag_set_page(frag, di->page); + __skb_frag_set_page(frag, au->page); skb_frag_off_set(frag, wi->offset); skb_frag_size_set(frag, frag_consumed_bytes); - if (page_is_pfmemalloc(di->page)) + if (page_is_pfmemalloc(au->page)) xdp_buff_set_frag_pfmemalloc(&xdp); sinfo->xdp_frags_size += frag_consumed_bytes; @@ -1631,10 +1635,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi wi++; } - di = head_wi->di; + au = head_wi->au; prog = rcu_dereference(rq->xdp_prog); - if (prog && mlx5e_xdp_handle(rq, di->page, prog, &xdp)) { + if (prog && mlx5e_xdp_handle(rq, au->page, prog, &xdp)) { if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) { int i; @@ -1651,7 +1655,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi if (unlikely(!skb)) return NULL; - page_ref_inc(di->page); + page_ref_inc(au->page); if (unlikely(xdp_buff_has_frags(&xdp))) { int i; @@ -1706,9 +1710,10 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) goto free_wqe; } - skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe, + skb = INDIRECT_CALL_3(rq->wqe.skb_from_cqe, mlx5e_skb_from_cqe_linear, mlx5e_skb_from_cqe_nonlinear, + mlx5e_xsk_skb_from_cqe_linear, rq, wi, cqe_bcnt); if (!skb) { /* probably for XDP */ @@ -1791,11 +1796,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64 { u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); - struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id]; + struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id); u16 stride_ix = mpwrq_get_cqe_stride_index(cqe); u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz; - u32 head_offset = wqe_offset & (PAGE_SIZE - 1); - u32 page_idx = wqe_offset >> PAGE_SHIFT; + u32 head_offset = wqe_offset & ((1 << rq->mpwqe.page_shift) - 1); + u32 page_idx = wqe_offset >> rq->mpwqe.page_shift; struct mlx5e_rx_wqe_ll *wqe; struct mlx5_wq_ll *wq; struct sk_buff *skb; @@ -1846,12 +1851,13 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = { #endif static void -mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_info *di, - u32 data_bcnt, u32 data_offset) +mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, + union mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset) { net_prefetchw(skb->data); while (data_bcnt) { + /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt); unsigned int truesize; @@ -1860,12 +1866,12 @@ mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_i else truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); - mlx5e_add_skb_frag(rq, skb, di, data_offset, + mlx5e_add_skb_frag(rq, skb, au, data_offset, pg_consumed_bytes, truesize); data_bcnt -= pg_consumed_bytes; data_offset = 0; - di++; + au++; } } @@ -1873,12 +1879,13 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { + union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); - struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx]; u32 frag_offset = head_offset + headlen; u32 byte_cnt = cqe_bcnt - headlen; - struct mlx5e_dma_info *head_di = di; + union mlx5e_alloc_unit *head_au = au; struct sk_buff *skb; + dma_addr_t addr; skb = napi_alloc_skb(rq->cq.napi, ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long))); @@ -1889,14 +1896,17 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w net_prefetchw(skb->data); + /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ if (unlikely(frag_offset >= PAGE_SIZE)) { - di++; + au++; frag_offset -= PAGE_SIZE; } - mlx5e_fill_skb_data(skb, rq, di, byte_cnt, frag_offset); + mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset); /* copy header */ - mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, head_offset, headlen); + addr = page_pool_get_dma_addr(head_au->page); + mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, addr, + head_offset, head_offset, headlen); /* skb linear part was allocated with headlen and aligned to long */ skb->tail += headlen; skb->len += headlen; @@ -1908,12 +1918,13 @@ static struct sk_buff * mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, u16 cqe_bcnt, u32 head_offset, u32 page_idx) { - struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx]; + union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx]; u16 rx_headroom = rq->buff.headroom; struct bpf_prog *prog; struct sk_buff *skb; u32 metasize = 0; void *va, *data; + dma_addr_t addr; u32 frag_size; /* Check packet size. Note LRO doesn't use linear SKB */ @@ -1922,11 +1933,12 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return NULL; } - va = page_address(di->page) + head_offset; + va = page_address(au->page) + head_offset; data = va + rx_headroom; frag_size = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt); - dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset, + addr = page_pool_get_dma_addr(au->page); + dma_sync_single_range_for_cpu(rq->pdev, addr, head_offset, frag_size, DMA_FROM_DEVICE); net_prefetch(data); @@ -1936,7 +1948,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, net_prefetchw(va); /* xdp_frame data area */ mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp); - if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) { + if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) { if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) __set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */ return NULL; /* page/packet was consumed by XDP */ @@ -1952,7 +1964,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, return NULL; /* queue up for recycling/reuse */ - page_ref_inc(di->page); + page_ref_inc(au->page); return skb; } @@ -1997,7 +2009,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, } prefetchw(skb->data); - mlx5e_copy_skb_header(rq->pdev, skb, head, + mlx5e_copy_skb_header(rq->pdev, skb, head->page, head->addr, head_offset + rx_headroom, rx_headroom, head_size); /* skb linear part was allocated with headlen and aligned to long */ @@ -2049,7 +2061,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index) if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) { shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE); - mlx5e_page_release(rq, &shampo->info[header_index], true); + mlx5e_page_release_dynamic(rq, shampo->info[header_index].page, true); } bitmap_clear(shampo->bitmap, header_index, 1); } @@ -2070,11 +2082,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq bool match = cqe->shampo.match; struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; - struct mlx5e_dma_info *di; + union mlx5e_alloc_unit *au; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; - wi = &rq->mpwqe.info[wqe_id]; + wi = mlx5e_get_mpw_info(rq, wqe_id); wi->consumed_strides += cstrides; if (unlikely(MLX5E_RX_ERR_CQE(cqe))) { @@ -2120,8 +2132,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq } if (likely(head_size)) { - di = &wi->umr.dma_info[page_idx]; - mlx5e_fill_skb_data(*skb, rq, di, data_bcnt, data_offset); + au = &wi->alloc_units[page_idx]; + mlx5e_fill_skb_data(*skb, rq, au, data_bcnt, data_offset); } mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb); @@ -2143,11 +2155,11 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq { u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); - struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id]; + struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id); u16 stride_ix = mpwrq_get_cqe_stride_index(cqe); u32 wqe_offset = stride_ix << rq->mpwqe.log_stride_sz; - u32 head_offset = wqe_offset & (PAGE_SIZE - 1); - u32 page_idx = wqe_offset >> PAGE_SHIFT; + u32 head_offset = wqe_offset & ((1 << rq->mpwqe.page_shift) - 1); + u32 page_idx = wqe_offset >> rq->mpwqe.page_shift; struct mlx5e_rx_wqe_ll *wqe; struct mlx5_wq_ll *wq; struct sk_buff *skb; @@ -2170,9 +2182,10 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe); - skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq, + skb = INDIRECT_CALL_3(rq->mpwqe.skb_from_cqe_mpwrq, mlx5e_skb_from_cqe_mpwrq_linear, mlx5e_skb_from_cqe_mpwrq_nonlinear, + mlx5e_xsk_skb_from_cqe_mpwrq_linear, rq, wi, cqe_bcnt, head_offset, page_idx); if (!skb) goto mpwrq_cqe_out; @@ -2417,7 +2430,7 @@ int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool default: /* MLX5_WQ_TYPE_CYCLIC */ rq->wqe.skb_from_cqe = xsk ? mlx5e_xsk_skb_from_cqe_linear : - mlx5e_rx_is_linear_skb(params, NULL) ? + mlx5e_rx_is_linear_skb(mdev, params, NULL) ? mlx5e_skb_from_cqe_linear : mlx5e_skb_from_cqe_nonlinear; rq->post_wqes = mlx5e_post_rx_wqes; @@ -2471,7 +2484,7 @@ free_wqe: void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params) { - rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ? + rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(rq->mdev, params, NULL) ? mlx5e_skb_from_cqe_linear : mlx5e_skb_from_cqe_nonlinear; rq->post_wqes = mlx5e_post_rx_wqes; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 7409829d12012e58abde0929ae073a139e0ed39e..03c1841970f14b053cd2f3b1711d8df79bf9cc37 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -641,17 +641,26 @@ static const struct counter_desc vnic_env_stats_dev_oob_desc[] = { VNIC_ENV_OFF(vport_env.internal_rq_out_of_buffer) }, }; +static const struct counter_desc vnic_env_stats_drop_desc[] = { + { "rx_oversize_pkts_buffer", + VNIC_ENV_OFF(vport_env.eth_wqe_too_small) }, +}; + #define NUM_VNIC_ENV_STEER_COUNTERS(dev) \ (MLX5_CAP_GEN(dev, nic_receive_steering_discard) ? \ ARRAY_SIZE(vnic_env_stats_steer_desc) : 0) #define NUM_VNIC_ENV_DEV_OOB_COUNTERS(dev) \ (MLX5_CAP_GEN(dev, vnic_env_int_rq_oob) ? \ ARRAY_SIZE(vnic_env_stats_dev_oob_desc) : 0) +#define NUM_VNIC_ENV_DROP_COUNTERS(dev) \ + (MLX5_CAP_GEN(dev, eth_wqe_too_small) ? \ + ARRAY_SIZE(vnic_env_stats_drop_desc) : 0) static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vnic_env) { return NUM_VNIC_ENV_STEER_COUNTERS(priv->mdev) + - NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); + NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev) + + NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vnic_env) @@ -665,6 +674,11 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vnic_env) for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++) strcpy(data + (idx++) * ETH_GSTRING_LEN, vnic_env_stats_dev_oob_desc[i].format); + + for (i = 0; i < NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev); i++) + strcpy(data + (idx++) * ETH_GSTRING_LEN, + vnic_env_stats_drop_desc[i].format); + return idx; } @@ -679,6 +693,11 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vnic_env) for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++) data[idx++] = MLX5E_READ_CTR32_BE(priv->stats.vnic.query_vnic_env_out, vnic_env_stats_dev_oob_desc, i); + + for (i = 0; i < NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev); i++) + data[idx++] = MLX5E_READ_CTR32_BE(priv->stats.vnic.query_vnic_env_out, + vnic_env_stats_drop_desc, i); + return idx; } @@ -2451,6 +2470,9 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = { &MLX5E_STATS_GRP(per_port_buff_congest), &MLX5E_STATS_GRP(ptp), &MLX5E_STATS_GRP(qos), +#ifdef CONFIG_MLX5_EN_MACSEC + &MLX5E_STATS_GRP(macsec_hw), +#endif }; unsigned int mlx5e_nic_stats_grps_num(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index ed4fc940e4efbaa71aa9b38022514ff493ce4b57..9f781085be4711336fa03c8092b234f80d272651 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -273,6 +273,10 @@ struct mlx5e_qcounter_stats { u32 rx_if_down_packets; }; +#define VNIC_ENV_GET(vnic_env_stats, c) \ + MLX5_GET(query_vnic_env_out, (vnic_env_stats)->query_vnic_env_out, \ + vport_env.c) + struct mlx5e_vnic_env_stats { __be64 query_vnic_env_out[MLX5_ST_SZ_QW(query_vnic_env_out)]; }; @@ -486,5 +490,6 @@ extern MLX5E_DECLARE_STATS_GRP(channels); extern MLX5E_DECLARE_STATS_GRP(per_port_buff_congest); extern MLX5E_DECLARE_STATS_GRP(ipsec_sw); extern MLX5E_DECLARE_STATS_GRP(ptp); +extern MLX5E_DECLARE_STATS_GRP(macsec_hw); #endif /* __MLX5_EN_STATS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index f154bda668adf6a0a19d3ff9a5c570ff2018c2f6..70a7a61f97087a22991448978af56164b2353fb5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -311,6 +311,7 @@ mlx5e_get_flow_meters(struct mlx5_core_dev *dev) static struct mlx5_tc_ct_priv * get_ct_priv(struct mlx5e_priv *priv) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5_rep_uplink_priv *uplink_priv; struct mlx5e_rep_priv *uplink_rpriv; @@ -322,7 +323,7 @@ get_ct_priv(struct mlx5e_priv *priv) return uplink_priv->ct_priv; } - return priv->fs->tc->ct; + return tc->ct; } static struct mlx5e_tc_psample * @@ -345,6 +346,7 @@ get_sample_priv(struct mlx5e_priv *priv) static struct mlx5e_post_act * get_post_action(struct mlx5e_priv *priv) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5_rep_uplink_priv *uplink_priv; struct mlx5e_rep_priv *uplink_rpriv; @@ -356,7 +358,7 @@ get_post_action(struct mlx5e_priv *priv) return uplink_priv->post_act; } - return priv->fs->tc->post_act; + return tc->post_act; } struct mlx5_flow_handle * @@ -607,11 +609,12 @@ int mlx5e_get_flow_namespace(struct mlx5e_tc_flow *flow) static struct mod_hdr_tbl * get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; return mlx5e_get_flow_namespace(flow) == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr : - &priv->fs->tc->mod_hdr; + &tc->mod_hdr; } static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv, @@ -810,6 +813,7 @@ static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp) { struct mlx5e_priv *priv = hp->func_priv; struct ttc_params ttc_params; + struct mlx5_ttc_table *ttc; int err; err = mlx5e_hairpin_create_indirect_rqt(hp); @@ -827,9 +831,10 @@ static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp) goto err_create_ttc_table; } + ttc = mlx5e_fs_get_ttc(priv->fs, false); netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n", hp->num_channels, - mlx5_get_ttc_flow_table(priv->fs->ttc)->id); + mlx5_get_ttc_flow_table(ttc)->id); return 0; @@ -916,10 +921,11 @@ static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio) static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv, u16 peer_vhca_id, u8 prio) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5e_hairpin_entry *hpe; u32 hash_key = hash_hairpin_info(peer_vhca_id, prio); - hash_for_each_possible(priv->fs->tc->hairpin_tbl, hpe, + hash_for_each_possible(tc->hairpin_tbl, hpe, hairpin_hlist, hash_key) { if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio) { refcount_inc(&hpe->refcnt); @@ -933,11 +939,12 @@ static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv, static void mlx5e_hairpin_put(struct mlx5e_priv *priv, struct mlx5e_hairpin_entry *hpe) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); /* no more hairpin flows for us, release the hairpin pair */ - if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &priv->fs->tc->hairpin_tbl_lock)) + if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &tc->hairpin_tbl_lock)) return; hash_del(&hpe->hairpin_hlist); - mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); + mutex_unlock(&tc->hairpin_tbl_lock); if (!IS_ERR_OR_NULL(hpe->hp)) { netdev_dbg(priv->netdev, "del hairpin: peer %s\n", @@ -993,6 +1000,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, struct mlx5e_tc_flow_parse_attr *parse_attr, struct netlink_ext_ack *extack) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); int peer_ifindex = parse_attr->mirred_ifindex[0]; struct mlx5_hairpin_params params; struct mlx5_core_dev *peer_mdev; @@ -1021,10 +1029,10 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, if (err) return err; - mutex_lock(&priv->fs->tc->hairpin_tbl_lock); + mutex_lock(&tc->hairpin_tbl_lock); hpe = mlx5e_hairpin_get(priv, peer_id, match_prio); if (hpe) { - mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); + mutex_unlock(&tc->hairpin_tbl_lock); wait_for_completion(&hpe->res_ready); if (IS_ERR(hpe->hp)) { @@ -1036,7 +1044,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, hpe = kzalloc(sizeof(*hpe), GFP_KERNEL); if (!hpe) { - mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); + mutex_unlock(&tc->hairpin_tbl_lock); return -ENOMEM; } @@ -1048,9 +1056,9 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, refcount_set(&hpe->refcnt, 1); init_completion(&hpe->res_ready); - hash_add(priv->fs->tc->hairpin_tbl, &hpe->hairpin_hlist, + hash_add(tc->hairpin_tbl, &hpe->hairpin_hlist, hash_hairpin_info(peer_id, match_prio)); - mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); + mutex_unlock(&tc->hairpin_tbl_lock); params.log_data_size = 16; params.log_data_size = min_t(u8, params.log_data_size, @@ -1126,8 +1134,9 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr) { struct mlx5_flow_context *flow_context = &spec->flow_context; + struct mlx5e_vlan_table *vlan = mlx5e_fs_get_vlan(priv->fs); + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_nic_flow_attr *nic_attr = attr->nic_attr; - struct mlx5e_tc_table *tc = priv->fs->tc; struct mlx5_flow_destination dest[2] = {}; struct mlx5_fs_chains *nic_chains; struct mlx5_flow_act flow_act = { @@ -1163,7 +1172,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, if (IS_ERR(dest[dest_ix].ft)) return ERR_CAST(dest[dest_ix].ft); } else { - dest[dest_ix].ft = mlx5e_vlan_get_flowtable(priv->fs->vlan); + dest[dest_ix].ft = mlx5e_vlan_get_flowtable(vlan); } dest_ix++; } @@ -1191,7 +1200,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, mutex_unlock(&tc->t_lock); netdev_err(priv->netdev, "Failed to create tc offload table\n"); - rule = ERR_CAST(priv->fs->tc->t); + rule = ERR_CAST(tc->t); goto err_ft_get; } } @@ -1293,8 +1302,10 @@ void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv, struct mlx5_flow_handle *rule, struct mlx5_flow_attr *attr) { - struct mlx5_fs_chains *nic_chains = mlx5e_nic_chains(priv->fs->tc); + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); + struct mlx5_fs_chains *nic_chains; + nic_chains = mlx5e_nic_chains(tc); mlx5_del_flow_rules(rule); if (attr->chain || attr->prio) @@ -1309,8 +1320,8 @@ void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv, static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_flow_attr *attr = flow->attr; - struct mlx5e_tc_table *tc = priv->fs->tc; flow_flag_clear(flow, OFFLOADED); @@ -1322,13 +1333,13 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, /* Remove root table if no rules are left to avoid * extra steering hops. */ - mutex_lock(&priv->fs->tc->t_lock); + mutex_lock(&tc->t_lock); if (!mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD)) && !IS_ERR_OR_NULL(tc->t)) { mlx5_chains_put_table(mlx5e_nic_chains(tc), 0, 1, MLX5E_TC_FT_LEVEL); - priv->fs->tc->t = NULL; + tc->t = NULL; } - mutex_unlock(&priv->fs->tc->t_lock); + mutex_unlock(&tc->t_lock); if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) mlx5e_detach_mod_hdr(priv, flow); @@ -1494,8 +1505,11 @@ bool mlx5e_tc_is_vf_tunnel(struct net_device *out_dev, struct net_device *route_ route_priv = netdev_priv(route_dev); route_mdev = route_priv->mdev; - if (out_mdev->coredev_type != MLX5_COREDEV_PF || - route_mdev->coredev_type != MLX5_COREDEV_VF) + if (out_mdev->coredev_type != MLX5_COREDEV_PF) + return false; + + if (route_mdev->coredev_type != MLX5_COREDEV_VF && + route_mdev->coredev_type != MLX5_COREDEV_SF) return false; return mlx5e_same_hw_devs(out_priv, route_priv); @@ -4058,13 +4072,14 @@ static const struct rhashtable_params tc_ht_params = { static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv, unsigned long flags) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5e_rep_priv *rpriv; if (flags & MLX5_TC_FLAG(ESW_OFFLOAD)) { rpriv = priv->ppriv; return &rpriv->tc_ht; } else /* NIC offload */ - return &priv->fs->tc->ht; + return &tc->ht; } static bool is_peer_flow_needed(struct mlx5e_tc_flow *flow) @@ -4448,7 +4463,7 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv, int err = 0; if (!mlx5_esw_hold(priv->mdev)) - return -EAGAIN; + return -EBUSY; mlx5_esw_get(priv->mdev); @@ -4772,6 +4787,7 @@ void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv, static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv) { + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_core_dev *peer_mdev = peer_priv->mdev; struct mlx5e_hairpin_entry *hpe, *tmp; LIST_HEAD(init_wait_list); @@ -4783,11 +4799,11 @@ static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv, peer_vhca_id = MLX5_CAP_GEN(peer_mdev, vhca_id); - mutex_lock(&priv->fs->tc->hairpin_tbl_lock); - hash_for_each(priv->fs->tc->hairpin_tbl, bkt, hpe, hairpin_hlist) + mutex_lock(&tc->hairpin_tbl_lock); + hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist) if (refcount_inc_not_zero(&hpe->refcnt)) list_add(&hpe->dead_peer_wait_list, &init_wait_list); - mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); + mutex_unlock(&tc->hairpin_tbl_lock); list_for_each_entry_safe(hpe, tmp, &init_wait_list, dead_peer_wait_list) { wait_for_completion(&hpe->res_ready); @@ -4841,7 +4857,8 @@ static int mlx5e_tc_nic_get_ft_size(struct mlx5_core_dev *dev) static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv) { - struct mlx5_flow_table **ft = &priv->fs->tc->miss_t; + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); + struct mlx5_flow_table **ft = &tc->miss_t; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_namespace *ns; int err = 0; @@ -4863,12 +4880,14 @@ static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv) static void mlx5e_tc_nic_destroy_miss_table(struct mlx5e_priv *priv) { - mlx5_destroy_flow_table(priv->fs->tc->miss_t); + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); + + mlx5_destroy_flow_table(tc->miss_t); } int mlx5e_tc_nic_init(struct mlx5e_priv *priv) { - struct mlx5e_tc_table *tc = priv->fs->tc; + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); struct mlx5_core_dev *dev = priv->mdev; struct mapping_ctx *chains_mapping; struct mlx5_chains_attr attr = {}; @@ -4909,7 +4928,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) attr.ns = MLX5_FLOW_NAMESPACE_KERNEL; attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev); attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS; - attr.default_ft = priv->fs->tc->miss_t; + attr.default_ft = tc->miss_t; attr.mapping = chains_mapping; tc->chains = mlx5_chains_create(dev, &attr); @@ -4958,7 +4977,7 @@ static void _mlx5e_tc_del_flow(void *ptr, void *arg) void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) { - struct mlx5e_tc_table *tc = priv->fs->tc; + struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs); if (tc->netdevice_nb.notifier_call) unregister_netdevice_notifier_dev_net(priv->netdev, @@ -5163,13 +5182,13 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) u32 chain = 0, chain_tag, reg_b, zone_restore_id; struct mlx5e_priv *priv = netdev_priv(skb->dev); - struct mlx5e_tc_table *tc = priv->fs->tc; struct mlx5_mapped_obj mapped_obj; struct tc_skb_ext *tc_skb_ext; + struct mlx5e_tc_table *tc; int err; reg_b = be32_to_cpu(cqe->ft_metadata); - + tc = mlx5e_fs_get_tc(priv->fs); chain_tag = reg_b & MLX5E_TC_TABLE_CHAIN_TAG_MASK; err = mapping_find(tc->mapping, chain_tag, &mapped_obj); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index 6ce1ab6b86b7ab3f002beb78860d8ecf0982b867..48241317a53545ce7c46e40c9a60aafbcf301650 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -54,6 +54,7 @@ ESW_FLOW_ATTR_SZ :\ NIC_FLOW_ATTR_SZ) +struct mlx5_fs_chains *mlx5e_nic_chains(struct mlx5e_tc_table *tc); int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags); struct mlx5e_tc_update_priv { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 27f791feb5174151406ad68618532f1c26900089..bf2232a2a836bb8ca55fb1e92de74ce4d8452e8f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -39,6 +39,7 @@ #include "ipoib/ipoib.h" #include "en_accel/en_accel.h" #include "en_accel/ipsec_rxtx.h" +#include "en_accel/macsec.h" #include "en/ptp.h" #include @@ -485,7 +486,7 @@ err_drop: static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr) { return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs && - !attr->insz; + !attr->insz && !mlx5e_macsec_skb_is_offload(skb); } static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index 833be29170a135c54ae011e9d959279564ccf3f6..9a458a5d985394fcf5c9963278939b3a1e94246f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -31,6 +31,7 @@ */ #include +#include #include "en.h" #include "en/txrx.h" #include "en/xdp.h" @@ -86,26 +87,36 @@ void mlx5e_trigger_irq(struct mlx5e_icosq *sq) static bool mlx5e_napi_xsk_post(struct mlx5e_xdpsq *xsksq, struct mlx5e_rq *xskrq) { + bool need_wakeup = xsk_uses_need_wakeup(xskrq->xsk_pool); bool busy_xsk = false, xsk_rx_alloc_err; - /* Handle the race between the application querying need_wakeup and the - * driver setting it: - * 1. Update need_wakeup both before and after the TX. If it goes to - * "yes", it can only happen with the first update. - * 2. If the application queried need_wakeup before we set it, the - * packets will be transmitted anyway, even w/o a wakeup. - * 3. Give a chance to clear need_wakeup after new packets were queued - * for TX. + /* If SQ is empty, there are no TX completions to trigger NAPI, so set + * need_wakeup. Do it before queuing packets for TX to avoid race + * condition with userspace. */ - mlx5e_xsk_update_tx_wakeup(xsksq); + if (need_wakeup && xsksq->pc == xsksq->cc) + xsk_set_tx_need_wakeup(xsksq->xsk_pool); busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET); - mlx5e_xsk_update_tx_wakeup(xsksq); + /* If we queued some packets for TX, no need for wakeup anymore. */ + if (need_wakeup && xsksq->pc != xsksq->cc) + xsk_clear_tx_need_wakeup(xsksq->xsk_pool); + /* If WQ is empty, RX won't trigger NAPI, so set need_wakeup. Do it + * before refilling to avoid race condition with userspace. + */ + if (need_wakeup && !mlx5e_rqwq_get_cur_sz(xskrq)) + xsk_set_rx_need_wakeup(xskrq->xsk_pool); xsk_rx_alloc_err = INDIRECT_CALL_2(xskrq->post_wqes, mlx5e_post_rx_mpwqes, mlx5e_post_rx_wqes, xskrq); - busy_xsk |= mlx5e_xsk_update_rx_wakeup(xskrq, xsk_rx_alloc_err); + /* Ask for wakeup if WQ is not full after refill. */ + if (!need_wakeup) + busy_xsk |= xsk_rx_alloc_err; + else if (xsk_rx_alloc_err) + xsk_set_rx_need_wakeup(xskrq->xsk_pool); + else + xsk_clear_rx_need_wakeup(xskrq->xsk_pool); return busy_xsk; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 229728c802335403eae620cd3a15fb5a9eb9fd35..a0242dc15741c7853e5dffaad1629cff1f07050e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -575,6 +575,9 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4]) if (MLX5_CAP_GEN_MAX(dev, vhca_state)) async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE); + if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload)) + async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE); + mask[0] = async_event_mask; if (MLX5_CAP_GEN(dev, event_cap)) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c index 0abef71cb839b81ae803d8bfe42b0d6c39c8e8e0..c9a91158e99c99199c40181d41a21639932d4aab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c @@ -78,12 +78,16 @@ mlx5_esw_indir_table_needed(struct mlx5_eswitch *esw, struct mlx5_core_dev *dest_mdev) { struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; + bool vf_sf_vport; + + vf_sf_vport = mlx5_eswitch_is_vf_vport(esw, vport_num) || + mlx5_esw_is_sf_vport(esw, vport_num); /* Use indirect table for all IP traffic from UL to VF with vport * destination when source rewrite flag is set. */ return esw_attr->in_rep->vport == MLX5_VPORT_UPLINK && - mlx5_eswitch_is_vf_vport(esw, vport_num) && + vf_sf_vport && esw->dev == dest_mdev && attr->ip_version && attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index 694c540669550a2c6f37656932d1a414c97b1803..4f8a24d84a86a2ba516fc3cc0787b20fe76df2ce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -924,12 +924,16 @@ int mlx5_esw_qos_vport_update_group(struct mlx5_eswitch *esw, struct mlx5_esw_rate_group *group, struct netlink_ext_ack *extack) { - int err; + int err = 0; mutex_lock(&esw->state_lock); + if (!vport->qos.enabled && !group) + goto unlock; + err = esw_qos_vport_enable(esw, vport, 0, 0, extack); if (!err) err = esw_qos_vport_update_group(esw, vport, group, extack); +unlock: mutex_unlock(&esw->state_lock); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 6aa58044b949bfe9c718f0464ece6f96215d0d46..c59107fa9e6d2d8f87031ea12ef2d345461c2d7d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1360,7 +1360,6 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) if (esw->mode == MLX5_ESWITCH_OFFLOADS) { struct devlink *devlink = priv_to_devlink(esw->dev); - esw_offloads_del_send_to_vport_meta_rules(esw); devl_rate_nodes_destroy(devlink); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 87ce5a208cb52e4b7fbecc5b66be0543c0f777a4..f68dc2d0dbe6598d923c564be7234b1e5724f79a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -244,6 +244,8 @@ struct mlx5_esw_offload { struct mlx5_flow_table *ft_offloads; struct mlx5_flow_group *vport_rx_group; + struct mlx5_flow_group *vport_rx_drop_group; + struct mlx5_flow_handle *vport_rx_drop_rule; struct xarray vport_reps; struct list_head peer_flows; struct mutex peer_mutex; @@ -344,7 +346,10 @@ void esw_offloads_disable(struct mlx5_eswitch *esw); int esw_offloads_enable(struct mlx5_eswitch *esw); void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw); int esw_offloads_init_reps(struct mlx5_eswitch *esw); -void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw); + +struct mlx5_flow_handle * +mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num); +void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule); bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw); int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index ed73132129aae8f0c81fedb2d42ff6637a3faca0..4e50df3139c68e7b02f5dc88f2b5506d4b022745 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -70,6 +70,8 @@ #define MLX5_ESW_VPORT_TBL_SIZE 128 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 +#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) + static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { .max_fte = MLX5_ESW_VPORT_TBL_SIZE, .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, @@ -427,7 +429,8 @@ esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *f dest[dest_idx].vport.vhca_id = MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; - if (mlx5_lag_mpesw_is_activated(esw->dev)) + if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && + mlx5_lag_mpesw_is_activated(esw->dev)) dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; } if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) { @@ -480,25 +483,27 @@ esw_setup_dests(struct mlx5_flow_destination *dest, !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) { esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); (*i)++; - } else if (attr->dest_ft) { - esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i); - (*i)++; } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { esw_setup_slow_path_dest(dest, flow_act, esw, *i); (*i)++; } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { esw_setup_accept_dest(dest, flow_act, chains, *i); (*i)++; - } else if (attr->dest_chain) { - err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, - 1, 0, *i); - (*i)++; } else if (esw_is_indir_table(esw, attr)) { err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i); } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); } else { *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); + + if (attr->dest_ft) { + err = esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i); + (*i)++; + } else if (attr->dest_chain) { + err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, + 1, 0, *i); + (*i)++; + } } return err; @@ -1057,52 +1062,23 @@ void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) mlx5_del_flow_rules(rule); } -static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) { - struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules; - int i = 0, num_vfs = esw->esw_funcs.num_vfs; - - if (!num_vfs || !flows) - return; - - for (i = 0; i < num_vfs; i++) - mlx5_del_flow_rules(flows[i]); - - kvfree(flows); - /* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0, - * meta rules could be freed again. So set it to NULL. - */ - esw->fdb_table.offloads.send_to_vport_meta_rules = NULL; + if (rule) + mlx5_del_flow_rules(rule); } -void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) -{ - mlx5_eswitch_del_send_to_vport_meta_rules(esw); -} - -static int -mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +struct mlx5_flow_handle * +mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {0}; - int num_vfs, rule_idx = 0, err = 0; struct mlx5_flow_handle *flow_rule; - struct mlx5_flow_handle **flows; struct mlx5_flow_spec *spec; - struct mlx5_vport *vport; - unsigned long i; - u16 vport_num; - - num_vfs = esw->esw_funcs.num_vfs; - flows = kvcalloc(num_vfs, sizeof(*flows), GFP_KERNEL); - if (!flows) - return -ENOMEM; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); - if (!spec) { - err = -ENOMEM; - goto alloc_err; - } + if (!spec) + return ERR_PTR(-ENOMEM); MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); @@ -1115,34 +1091,18 @@ mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw) dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; - mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) { - vport_num = vport->vport; - MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, - mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); - dest.vport.num = vport_num; - - flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, - spec, &flow_act, &dest, 1); - if (IS_ERR(flow_rule)) { - err = PTR_ERR(flow_rule); - esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n", - rule_idx, PTR_ERR(flow_rule)); - goto rule_err; - } - flows[rule_idx++] = flow_rule; - } + MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); + dest.vport.num = vport_num; - esw->fdb_table.offloads.send_to_vport_meta_rules = flows; - kvfree(spec); - return 0; + flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, + spec, &flow_act, &dest, 1); + if (IS_ERR(flow_rule)) + esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", + vport_num, PTR_ERR(flow_rule)); -rule_err: - while (--rule_idx >= 0) - mlx5_del_flow_rules(flows[rule_idx]); kvfree(spec); -alloc_err: - kvfree(flows); - return err; + return flow_rule; } static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) @@ -1667,18 +1627,200 @@ esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) #endif +static int +esw_create_send_to_vport_group(struct mlx5_eswitch *esw, + struct mlx5_flow_table *fdb, + u32 *flow_group_in, + int *ix) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *g; + void *match_criteria; + int count, err = 0; + + memset(flow_group_in, 0, inlen); + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS); + + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); + MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); + if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + misc_parameters.source_eswitch_owner_vhca_id); + MLX5_SET(create_flow_group_in, flow_group_in, + source_eswitch_owner_vhca_id_valid, 1); + } + + /* See comment at table_size calculation */ + count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); + *ix += count; + + g = mlx5_create_flow_group(fdb, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); + goto out; + } + esw->fdb_table.offloads.send_to_vport_grp = g; + +out: + return err; +} + +static int +esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, + struct mlx5_flow_table *fdb, + u32 *flow_group_in, + int *ix) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *g; + void *match_criteria; + int err = 0; + + if (!esw_src_port_rewrite_supported(esw)) + return 0; + + memset(flow_group_in, 0, inlen); + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS_2); + + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); + + MLX5_SET(fte_match_param, match_criteria, + misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + MLX5_SET(fte_match_param, match_criteria, + misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); + MLX5_SET(create_flow_group_in, flow_group_in, + end_flow_index, *ix + esw->total_vports - 1); + *ix += esw->total_vports; + + g = mlx5_create_flow_group(fdb, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + esw_warn(esw->dev, + "Failed to create send-to-vport meta flow group err(%d)\n", err); + goto send_vport_meta_err; + } + esw->fdb_table.offloads.send_to_vport_meta_grp = g; + + return 0; + +send_vport_meta_err: + return err; +} + +static int +esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, + struct mlx5_flow_table *fdb, + u32 *flow_group_in, + int *ix) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *g; + void *match_criteria; + int err = 0; + + if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) + return 0; + + memset(flow_group_in, 0, inlen); + + esw_set_flow_group_source_port(esw, flow_group_in); + + if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { + match_criteria = MLX5_ADDR_OF(create_flow_group_in, + flow_group_in, + match_criteria); + + MLX5_SET_TO_ONES(fte_match_param, match_criteria, + misc_parameters.source_eswitch_owner_vhca_id); + + MLX5_SET(create_flow_group_in, flow_group_in, + source_eswitch_owner_vhca_id_valid, 1); + } + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, + *ix + esw->total_vports - 1); + *ix += esw->total_vports; + + g = mlx5_create_flow_group(fdb, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); + goto out; + } + esw->fdb_table.offloads.peer_miss_grp = g; + +out: + return err; +} + +static int +esw_create_miss_group(struct mlx5_eswitch *esw, + struct mlx5_flow_table *fdb, + u32 *flow_group_in, + int *ix) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *g; + void *match_criteria; + int err = 0; + u8 *dmac; + + memset(flow_group_in, 0, inlen); + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_OUTER_HEADERS); + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, + match_criteria); + dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, + outer_headers.dmac_47_16); + dmac[0] = 0x01; + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, + *ix + MLX5_ESW_MISS_FLOWS); + + g = mlx5_create_flow_group(fdb, flow_group_in); + if (IS_ERR(g)) { + err = PTR_ERR(g); + esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); + goto miss_err; + } + esw->fdb_table.offloads.miss_grp = g; + + err = esw_add_fdb_miss_rule(esw); + if (err) + goto miss_rule_err; + + return 0; + +miss_rule_err: + mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); +miss_err: + return err; +} + static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table_attr ft_attr = {}; - int num_vfs, table_size, ix, err = 0; struct mlx5_core_dev *dev = esw->dev; struct mlx5_flow_namespace *root_ns; struct mlx5_flow_table *fdb = NULL; + int table_size, ix = 0, err = 0; u32 flags = 0, *flow_group_in; - struct mlx5_flow_group *g; - void *match_criteria; - u8 *dmac; esw_debug(esw->dev, "Create offloads FDB Tables\n"); @@ -1712,7 +1854,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) * total vports of the peer (currently is also uses esw->total_vports). */ table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + - MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs; + esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS; /* create the slow path fdb with encap set, so further table instances * can be created at run time while VFs are probed if the FW allows that. @@ -1753,139 +1895,29 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) goto fdb_chains_err; } - /* create send-to-vport group */ - MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, - MLX5_MATCH_MISC_PARAMETERS); - - match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); - - MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); - MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); - if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { - MLX5_SET_TO_ONES(fte_match_param, match_criteria, - misc_parameters.source_eswitch_owner_vhca_id); - MLX5_SET(create_flow_group_in, flow_group_in, - source_eswitch_owner_vhca_id_valid, 1); - } - - /* See comment above table_size calculation */ - ix = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); - MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); - MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); - - g = mlx5_create_flow_group(fdb, flow_group_in); - if (IS_ERR(g)) { - err = PTR_ERR(g); - esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); + err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); + if (err) goto send_vport_err; - } - esw->fdb_table.offloads.send_to_vport_grp = g; - - if (esw_src_port_rewrite_supported(esw)) { - /* meta send to vport */ - memset(flow_group_in, 0, inlen); - MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, - MLX5_MATCH_MISC_PARAMETERS_2); - - match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); - - MLX5_SET(fte_match_param, match_criteria, - misc_parameters_2.metadata_reg_c_0, - mlx5_eswitch_get_vport_metadata_mask()); - MLX5_SET(fte_match_param, match_criteria, - misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); - - num_vfs = esw->esw_funcs.num_vfs; - if (num_vfs) { - MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); - MLX5_SET(create_flow_group_in, flow_group_in, - end_flow_index, ix + num_vfs - 1); - ix += num_vfs; - - g = mlx5_create_flow_group(fdb, flow_group_in); - if (IS_ERR(g)) { - err = PTR_ERR(g); - esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n", - err); - goto send_vport_meta_err; - } - esw->fdb_table.offloads.send_to_vport_meta_grp = g; - - err = mlx5_eswitch_add_send_to_vport_meta_rules(esw); - if (err) - goto meta_rule_err; - } - } - - if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { - /* create peer esw miss group */ - memset(flow_group_in, 0, inlen); - - esw_set_flow_group_source_port(esw, flow_group_in); - - if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { - match_criteria = MLX5_ADDR_OF(create_flow_group_in, - flow_group_in, - match_criteria); - - MLX5_SET_TO_ONES(fte_match_param, match_criteria, - misc_parameters.source_eswitch_owner_vhca_id); - - MLX5_SET(create_flow_group_in, flow_group_in, - source_eswitch_owner_vhca_id_valid, 1); - } - - MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); - MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, - ix + esw->total_vports - 1); - ix += esw->total_vports; - - g = mlx5_create_flow_group(fdb, flow_group_in); - if (IS_ERR(g)) { - err = PTR_ERR(g); - esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err); - goto peer_miss_err; - } - esw->fdb_table.offloads.peer_miss_grp = g; - } - - /* create miss group */ - memset(flow_group_in, 0, inlen); - MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, - MLX5_MATCH_OUTER_HEADERS); - match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, - match_criteria); - dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, - outer_headers.dmac_47_16); - dmac[0] = 0x01; - MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); - MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, - ix + MLX5_ESW_MISS_FLOWS); + err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); + if (err) + goto send_vport_meta_err; - g = mlx5_create_flow_group(fdb, flow_group_in); - if (IS_ERR(g)) { - err = PTR_ERR(g); - esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); - goto miss_err; - } - esw->fdb_table.offloads.miss_grp = g; + err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); + if (err) + goto peer_miss_err; - err = esw_add_fdb_miss_rule(esw); + err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); if (err) - goto miss_rule_err; + goto miss_err; kvfree(flow_group_in); return 0; -miss_rule_err: - mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); miss_err: if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); peer_miss_err: - mlx5_eswitch_del_send_to_vport_meta_rules(esw); -meta_rule_err: if (esw->fdb_table.offloads.send_to_vport_meta_grp) mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); send_vport_meta_err: @@ -1912,7 +1944,6 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); - mlx5_eswitch_del_send_to_vport_meta_rules(esw); mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); if (esw->fdb_table.offloads.send_to_vport_meta_grp) mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); @@ -1930,7 +1961,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) atomic64_set(&esw->user_count, 0); } -static int esw_get_offloads_ft_size(struct mlx5_eswitch *esw) +static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) { int nvports; @@ -1955,7 +1986,8 @@ static int esw_create_offloads_table(struct mlx5_eswitch *esw) return -EOPNOTSUPP; } - ft_attr.max_fte = esw_get_offloads_ft_size(esw); + ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + + MLX5_ESW_FT_OFFLOADS_DROP_RULE; ft_attr.prio = 1; ft_offloads = mlx5_create_flow_table(ns, &ft_attr); @@ -1984,7 +2016,7 @@ static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) int nvports; int err = 0; - nvports = esw_get_offloads_ft_size(esw); + nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); flow_group_in = kvzalloc(inlen, GFP_KERNEL); if (!flow_group_in) return -ENOMEM; @@ -2014,6 +2046,52 @@ static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) mlx5_destroy_flow_group(esw->offloads.vport_rx_group); } +static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) +{ + /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) + * for the drop rule, which is placed at the end of the table. + * So return the total of vport and int_port as rule index. + */ + return esw_get_nr_ft_offloads_steering_src_ports(esw); +} + +static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *g; + u32 *flow_group_in; + int flow_index; + int err = 0; + + flow_index = esw_create_vport_rx_drop_rule_index(esw); + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return -ENOMEM; + + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); + + g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); + + if (IS_ERR(g)) { + err = PTR_ERR(g); + mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); + goto out; + } + + esw->offloads.vport_rx_drop_group = g; +out: + kvfree(flow_group_in); + return err; +} + +static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) +{ + if (esw->offloads.vport_rx_drop_group) + mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); +} + struct mlx5_flow_handle * mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, struct mlx5_flow_destination *dest) @@ -2062,6 +2140,32 @@ out: return flow_rule; } +static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) +{ + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *flow_rule; + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, + &flow_act, NULL, 0); + if (IS_ERR(flow_rule)) { + esw_warn(esw->dev, + "fs offloads: Failed to add vport rx drop rule err %ld\n", + PTR_ERR(flow_rule)); + return PTR_ERR(flow_rule); + } + + esw->offloads.vport_rx_drop_rule = flow_rule; + + return 0; +} + +static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) +{ + if (esw->offloads.vport_rx_drop_rule) + mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); +} + static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) { u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; @@ -3062,8 +3166,20 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) if (err) goto create_fg_err; + err = esw_create_vport_rx_drop_group(esw); + if (err) + goto create_rx_drop_fg_err; + + err = esw_create_vport_rx_drop_rule(esw); + if (err) + goto create_rx_drop_rule_err; + return 0; +create_rx_drop_rule_err: + esw_destroy_vport_rx_drop_group(esw); +create_rx_drop_fg_err: + esw_destroy_vport_rx_group(esw); create_fg_err: esw_destroy_offloads_fdb_tables(esw); create_fdb_err: @@ -3081,6 +3197,8 @@ create_indir_err: static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) { + esw_destroy_vport_rx_drop_rule(esw); + esw_destroy_vport_rx_drop_group(esw); esw_destroy_vport_rx_group(esw); esw_destroy_offloads_fdb_tables(esw); esw_destroy_restore_table(esw); @@ -3115,8 +3233,10 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, MLX5_VPORT_UC_ADDR_CHANGE); - if (err) + if (err) { + devl_unlock(devlink); return; + } } esw->esw_funcs.num_vfs = new_num_vfs; devl_unlock(devlink); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c index a1ac3a654962e445448bbba95cce1e67e81857c6..9459e56ee90a64dfee01cc019a7a415fa7fa72ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/events.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c @@ -36,6 +36,7 @@ static struct mlx5_nb events_nbs_ref[] = { /* Events to be forwarded (as is) to mlx5 core interfaces (mlx5e/mlx5_ib) */ {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_PORT_CHANGE }, {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_GENERAL_EVENT }, + {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_OBJECT_CHANGE }, /* QP/WQ resource events to forward */ {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_DCT_DRAINED }, {.nb.notifier_call = forward_event, .event_type = MLX5_EVENT_TYPE_PATH_MIG }, @@ -132,6 +133,8 @@ static const char *eqe_type_str(u8 type) return "MLX5_EVENT_TYPE_MONITOR_COUNTER"; case MLX5_EVENT_TYPE_DEVICE_TRACER: return "MLX5_EVENT_TYPE_DEVICE_TRACER"; + case MLX5_EVENT_TYPE_OBJECT_CHANGE: + return "MLX5_EVENT_TYPE_OBJECT_CHANGE"; default: return "Unrecognized event"; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index e735e19461bab60ca799cae788fe6d931a406e29..32d4c967469ccc75eceb28d2d6e7701aecacce3a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -577,7 +577,10 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, MLX5_SET(flow_context, in_flow_context, modify_header_id, fte->action.modify_hdr->id); - MLX5_SET(flow_context, in_flow_context, ipsec_obj_id, fte->action.ipsec_obj_id); + MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type, + fte->action.crypto.type); + MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id, + fte->action.crypto.obj_id); vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan); @@ -919,13 +922,15 @@ static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns, max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions); table_type = FS_FT_FDB; break; + case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC: case MLX5_FLOW_NAMESPACE_KERNEL: case MLX5_FLOW_NAMESPACE_BYPASS: max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions); table_type = FS_FT_NIC_RX; break; case MLX5_FLOW_NAMESPACE_EGRESS: - case MLX5_FLOW_NAMESPACE_EGRESS_KERNEL: + case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC: + case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC: max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions); table_type = FS_FT_NIC_TX; break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index e3960cdf51318fd2df0d123ee0fa2f215897bdd0..d53749248fa09404cc6696213bc770962b83be5c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -104,6 +104,10 @@ #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\ LEFTOVERS_NUM_PRIOS) +#define KERNEL_RX_MACSEC_NUM_PRIOS 1 +#define KERNEL_RX_MACSEC_NUM_LEVELS 2 +#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS) + #define ETHTOOL_PRIO_NUM_LEVELS 1 #define ETHTOOL_NUM_PRIOS 11 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS) @@ -126,11 +130,15 @@ #define LAG_PRIO_NUM_LEVELS 1 #define LAG_NUM_PRIOS 1 -#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1) +#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1) #define KERNEL_TX_IPSEC_NUM_PRIOS 1 #define KERNEL_TX_IPSEC_NUM_LEVELS 1 -#define KERNEL_TX_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS) +#define KERNEL_TX_IPSEC_MIN_LEVEL (KERNEL_TX_IPSEC_NUM_LEVELS) + +#define KERNEL_TX_MACSEC_NUM_PRIOS 1 +#define KERNEL_TX_MACSEC_NUM_LEVELS 2 +#define KERNEL_TX_MACSEC_MIN_LEVEL (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS) struct node_caps { size_t arr_sz; @@ -149,12 +157,16 @@ static struct init_tree_node { enum mlx5_flow_table_miss_action def_miss_action; } root_fs = { .type = FS_TYPE_NAMESPACE, - .ar_size = 7, + .ar_size = 8, .children = (struct init_tree_node[]){ ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, BY_PASS_PRIO_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS, + KERNEL_RX_MACSEC_NUM_LEVELS))), ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS, @@ -186,18 +198,23 @@ static struct init_tree_node { static struct init_tree_node egress_root_fs = { .type = FS_TYPE_NAMESPACE, - .ar_size = 2, + .ar_size = 3, .children = (struct init_tree_node[]) { ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0, FS_CHAINING_CAPS_EGRESS, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS, BY_PASS_PRIO_NUM_LEVELS))), - ADD_PRIO(0, KERNEL_TX_MIN_LEVEL, 0, + ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS_EGRESS, ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS, KERNEL_TX_IPSEC_NUM_LEVELS))), + ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0, + FS_CHAINING_CAPS_EGRESS, + ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF, + ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS, + KERNEL_TX_MACSEC_NUM_LEVELS))), } }; @@ -2269,6 +2286,7 @@ static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type) { switch (type) { case MLX5_FLOW_NAMESPACE_BYPASS: + case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC: case MLX5_FLOW_NAMESPACE_LAG: case MLX5_FLOW_NAMESPACE_OFFLOADS: case MLX5_FLOW_NAMESPACE_ETHTOOL: @@ -2315,7 +2333,8 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev, prio = FDB_BYPASS_PATH; break; case MLX5_FLOW_NAMESPACE_EGRESS: - case MLX5_FLOW_NAMESPACE_EGRESS_KERNEL: + case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC: + case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC: root_ns = steering->egress_root_ns; prio = type - MLX5_FLOW_NAMESPACE_EGRESS; break; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index 079fa44ada71ee5b13b4004f292b863569ed17f0..f34e758a2f1f60101c35179dc53b0a940a102e01 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -273,6 +273,19 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN_64(dev, general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD) { + err = mlx5_core_get_caps(dev, MLX5_CAP_MACSEC); + if (err) + return err; + } + + if (MLX5_CAP_GEN(dev, adv_virtualization)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_ADV_VIRTUALIZATION); + if (err) + return err; + } + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 2cf2c99484467ad79e1812a06e019d99cee42c48..86ed87d704f7d7a81a8cc69653d3a52e0b184d61 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -601,7 +601,7 @@ static void mlx5_fw_reporter_err_work(struct work_struct *work) fw_reporter_ctx.miss_counter = health->miss_counter; if (fw_reporter_ctx.err_synd) { devlink_health_report(health->fw_reporter, - "FW syndrom reported", &fw_reporter_ctx); + "FW syndrome reported", &fw_reporter_ctx); return; } if (fw_reporter_ctx.miss_counter) @@ -702,11 +702,25 @@ static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { .dump = mlx5_fw_fatal_reporter_dump, }; -#define MLX5_REPORTER_FW_GRACEFUL_PERIOD 1200000 +#define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000 +#define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000 +#define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000 +#define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD + static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; struct devlink *devlink = priv_to_devlink(dev); + u64 grace_period; + + if (mlx5_core_is_ecpf(dev)) { + grace_period = MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD; + } else if (mlx5_core_is_pf(dev)) { + grace_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD; + } else { + /* VF or SF */ + grace_period = MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD; + } health->fw_reporter = devlink_health_reporter_create(devlink, &mlx5_fw_reporter_ops, @@ -718,7 +732,7 @@ static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) health->fw_fatal_reporter = devlink_health_reporter_create(devlink, &mlx5_fw_fatal_reporter_ops, - MLX5_REPORTER_FW_GRACEFUL_PERIOD, + grace_period, dev); if (IS_ERR(health->fw_fatal_reporter)) mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n", @@ -843,9 +857,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) health->timer.expires = jiffies + msecs_to_jiffies(poll_interval_ms); add_timer(&health->timer); - - if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc)) - queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0); } void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) @@ -862,6 +873,14 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) del_timer_sync(&health->timer); } +void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev) +{ + struct mlx5_core_health *health = &dev->priv.health; + + if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc)) + queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0); +} + void mlx5_drain_health_wq(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; @@ -875,13 +894,6 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) cancel_work_sync(&health->fatal_report_work); } -void mlx5_health_flush(struct mlx5_core_dev *dev) -{ - struct mlx5_core_health *health = &dev->priv.health; - - flush_workqueue(health->wq); -} - void mlx5_health_cleanup(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index ac3757beaea20ab4f008a4676979f665c5d63d29..c247cca154e9c62ea5a478390fb9a40970fe83da 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -32,6 +32,7 @@ #include "en.h" #include "ipoib.h" +#include "en/fs_ethtool.h" static void mlx5i_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) @@ -39,7 +40,7 @@ static void mlx5i_get_drvinfo(struct net_device *dev, struct mlx5e_priv *priv = mlx5i_epriv(dev); mlx5e_ethtool_get_drvinfo(priv, drvinfo); - strlcpy(drvinfo->driver, KBUILD_MODNAME "[ib_ipoib]", + strscpy(drvinfo->driver, KBUILD_MODNAME "[ib_ipoib]", sizeof(drvinfo->driver)); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index c02b7b08fb4c1da8b53eed6901a41b04f78303b9..4e3a75496dd9ac213d53b6962d6dee6e92d70b9d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -35,6 +35,7 @@ #include "en.h" #include "en/params.h" #include "ipoib.h" +#include "en/fs_ethtool.h" #define IB_DEFAULT_Q_KEY 0xb1b #define MLX5I_PARAMS_DEFAULT_LOG_RQ_SIZE 9 @@ -320,43 +321,47 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv) static int mlx5i_create_flow_steering(struct mlx5e_priv *priv) { + struct mlx5_flow_namespace *ns = + mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL); int err; - priv->fs->ns = mlx5_get_flow_namespace(priv->mdev, - MLX5_FLOW_NAMESPACE_KERNEL); - if (!priv->fs->ns) + if (!ns) return -EINVAL; - err = mlx5e_arfs_create_tables(priv); + mlx5e_fs_set_ns(priv->fs, ns, false); + err = mlx5e_arfs_create_tables(priv->fs, priv->rx_res, + !!(priv->netdev->hw_features & NETIF_F_NTUPLE)); if (err) { netdev_err(priv->netdev, "Failed to create arfs tables, err=%d\n", err); priv->netdev->hw_features &= ~NETIF_F_NTUPLE; } - err = mlx5e_create_ttc_table(priv); + err = mlx5e_create_ttc_table(priv->fs, priv->rx_res); if (err) { netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n", err); goto err_destroy_arfs_tables; } - mlx5e_ethtool_init_steering(priv); + mlx5e_ethtool_init_steering(priv->fs); return 0; err_destroy_arfs_tables: - mlx5e_arfs_destroy_tables(priv); + mlx5e_arfs_destroy_tables(priv->fs, + !!(priv->netdev->hw_features & NETIF_F_NTUPLE)); return err; } static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv) { - mlx5e_destroy_ttc_table(priv); - mlx5e_arfs_destroy_tables(priv); - mlx5e_ethtool_cleanup_steering(priv); + mlx5e_destroy_ttc_table(priv->fs); + mlx5e_arfs_destroy_tables(priv->fs, + !!(priv->netdev->hw_features & NETIF_F_NTUPLE)); + mlx5e_ethtool_cleanup_steering(priv->fs); } static int mlx5i_init_rx(struct mlx5e_priv *priv) @@ -458,7 +463,6 @@ static const struct mlx5e_profile mlx5i_nic_profile = { .update_carrier = NULL, /* no HW update in IB link */ .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, - .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), .stats_grps = mlx5i_stats_grps, .stats_grps_num = mlx5i_stats_grps_num, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c index 0b86e78dbc0e211952e16655bfc4a1db0122392b..0227a521d301e5f4410be751dc904952dd9d7175 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c @@ -349,7 +349,6 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = { .update_stats = NULL, .rx_handlers = &mlx5i_rx_handlers, .max_tc = MLX5I_MAX_NUM_TC, - .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), }; const struct mlx5e_profile *mlx5i_pkey_get_profile(void) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 0f34e3c80d1f412a15d30027d286d11363dae889..a9f4ede4a9bf8d610a07fd89cb129a4e06b7ba6c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -65,6 +65,21 @@ static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags) return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY; } +static u8 lag_active_port_bits(struct mlx5_lag *ldev) +{ + u8 enabled_ports[MLX5_MAX_PORTS] = {}; + u8 active_port = 0; + int num_enabled; + int idx; + + mlx5_infer_tx_enabled(&ldev->tracker, ldev->ports, enabled_ports, + &num_enabled); + for (idx = 0; idx < num_enabled; idx++) + active_port |= BIT_MASK(enabled_ports[idx]); + + return active_port; +} + static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 *ports, int mode, unsigned long flags) { @@ -77,9 +92,21 @@ static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 *ports, int mode, lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx); MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG); MLX5_SET(lagc, lag_ctx, fdb_selection_mode, fdb_sel_mode); - if (port_sel_mode == MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY) { + + switch (port_sel_mode) { + case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY: MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[0]); MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[1]); + break; + case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT: + if (!MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass)) + break; + + MLX5_SET(lagc, lag_ctx, active_port, + lag_active_port_bits(mlx5_lag_dev(dev))); + break; + default: + break; } MLX5_SET(lagc, lag_ctx, port_select_mode, port_sel_mode); @@ -386,12 +413,37 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev, } } +static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports) +{ + u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {}; + void *lag_ctx; + + lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx); + + MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG); + MLX5_SET(modify_lag_in, in, field_select, 0x2); + + MLX5_SET(lagc, lag_ctx, active_port, ports); + + return mlx5_cmd_exec_in(dev, modify_lag, in); +} + static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) { struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev; + u8 active_ports; + int ret; - if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) - return mlx5_lag_port_sel_modify(ldev, ports); + if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) { + ret = mlx5_lag_port_sel_modify(ldev, ports); + if (ret || + !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass)) + return ret; + + active_ports = lag_active_port_bits(ldev); + + return mlx5_cmd_modify_active_port(dev0, active_ports); + } return mlx5_cmd_modify_lag(dev0, ldev->ports, ports); } @@ -432,21 +484,22 @@ void mlx5_modify_lag(struct mlx5_lag *ldev, mlx5_lag_drop_rule_setup(ldev, tracker); } -#define MLX5_LAG_ROCE_HASH_PORTS_SUPPORTED 4 static int mlx5_lag_set_port_sel_mode_roce(struct mlx5_lag *ldev, unsigned long *flags) { - struct lag_func *dev0 = &ldev->pf[MLX5_LAG_P1]; + struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev; - if (ldev->ports == MLX5_LAG_ROCE_HASH_PORTS_SUPPORTED) { - /* Four ports are support only in hash mode */ - if (!MLX5_CAP_PORT_SELECTION(dev0->dev, port_select_flow_table)) - return -EINVAL; - set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags); + if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) { if (ldev->ports > 2) - ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS; + return -EINVAL; + return 0; } + if (ldev->ports > 2) + ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS; + + set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags); + return 0; } @@ -1067,30 +1120,32 @@ static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev, struct net_device *netdev) { unsigned int fn = mlx5_get_dev_index(dev); + unsigned long flags; if (fn >= ldev->ports) return; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev->pf[fn].netdev = netdev; ldev->tracker.netdev_state[fn].link_up = 0; ldev->tracker.netdev_state[fn].tx_enabled = 0; - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); } static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev, struct net_device *netdev) { + unsigned long flags; int i; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); for (i = 0; i < ldev->ports; i++) { if (ldev->pf[i].netdev == netdev) { ldev->pf[i].netdev = NULL; break; } } - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); } static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev, @@ -1234,7 +1289,7 @@ void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, mlx5_ldev_add_netdev(ldev, dev, netdev); for (i = 0; i < ldev->ports; i++) - if (!ldev->pf[i].dev) + if (!ldev->pf[i].netdev) break; if (i >= ldev->ports) @@ -1246,12 +1301,13 @@ void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, bool mlx5_lag_is_roce(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; + unsigned long flags; bool res; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_roce(ldev); - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return res; } @@ -1260,27 +1316,45 @@ EXPORT_SYMBOL(mlx5_lag_is_roce); bool mlx5_lag_is_active(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; + unsigned long flags; bool res; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_active(ldev); - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return res; } EXPORT_SYMBOL(mlx5_lag_is_active); +bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev) +{ + struct mlx5_lag *ldev; + unsigned long flags; + bool res = 0; + + spin_lock_irqsave(&lag_lock, flags); + ldev = mlx5_lag_dev(dev); + if (ldev) + res = test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags); + spin_unlock_irqrestore(&lag_lock, flags); + + return res; +} +EXPORT_SYMBOL(mlx5_lag_mode_is_hash); + bool mlx5_lag_is_master(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; + unsigned long flags; bool res; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_active(ldev) && dev == ldev->pf[MLX5_LAG_P1].dev; - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return res; } @@ -1289,12 +1363,13 @@ EXPORT_SYMBOL(mlx5_lag_is_master); bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; + unsigned long flags; bool res; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_sriov(ldev); - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return res; } @@ -1303,13 +1378,14 @@ EXPORT_SYMBOL(mlx5_lag_is_sriov); bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; + unsigned long flags; bool res; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); res = ldev && __mlx5_lag_is_sriov(ldev) && test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &ldev->mode_flags); - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return res; } @@ -1352,9 +1428,10 @@ struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev) { struct net_device *ndev = NULL; struct mlx5_lag *ldev; + unsigned long flags; int i; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); if (!(ldev && __mlx5_lag_is_roce(ldev))) @@ -1373,7 +1450,7 @@ struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev) dev_hold(ndev); unlock: - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return ndev; } @@ -1383,10 +1460,11 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, struct net_device *slave) { struct mlx5_lag *ldev; + unsigned long flags; u8 port = 0; int i; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); if (!(ldev && __mlx5_lag_is_roce(ldev))) goto unlock; @@ -1401,7 +1479,7 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, port = ldev->v2p_map[port * ldev->buckets]; unlock: - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return port; } EXPORT_SYMBOL(mlx5_lag_get_slave_port); @@ -1422,8 +1500,9 @@ struct mlx5_core_dev *mlx5_lag_get_peer_mdev(struct mlx5_core_dev *dev) { struct mlx5_core_dev *peer_dev = NULL; struct mlx5_lag *ldev; + unsigned long flags; - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); if (!ldev) goto unlock; @@ -1433,7 +1512,7 @@ struct mlx5_core_dev *mlx5_lag_get_peer_mdev(struct mlx5_core_dev *dev) ldev->pf[MLX5_LAG_P1].dev; unlock: - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); return peer_dev; } EXPORT_SYMBOL(mlx5_lag_get_peer_mdev); @@ -1446,6 +1525,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, int outlen = MLX5_ST_SZ_BYTES(query_cong_statistics_out); struct mlx5_core_dev **mdev; struct mlx5_lag *ldev; + unsigned long flags; int num_ports; int ret, i, j; void *out; @@ -1462,7 +1542,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, memset(values, 0, sizeof(*values) * num_counters); - spin_lock(&lag_lock); + spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); if (ldev && __mlx5_lag_is_active(ldev)) { num_ports = ldev->ports; @@ -1472,7 +1552,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, num_ports = 1; mdev[MLX5_LAG_P1] = dev; } - spin_unlock(&lag_lock); + spin_unlock_irqrestore(&lag_lock, flags); for (i = 0; i < num_ports; ++i) { u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)] = {}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c index 21e14507ff5c0293098dc262bc83e7cbcdc57cc8..baa8092f335e365d60f8bc074316c892ea6635d5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -381,20 +381,12 @@ void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, WRITE_ONCE(doorbell_cseg, NULL); } -int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms) +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data) { struct mlx5_aso_cq *cq = &aso->cq; struct mlx5_cqe64 *cqe; - unsigned long expires; cqe = mlx5_cqwq_get_cqe(&cq->wq); - - expires = jiffies + msecs_to_jiffies(interval_ms); - while (!cqe && time_is_after_jiffies(expires)) { - usleep_range(2, 10); - cqe = mlx5_cqwq_get_cqe(&cq->wq); - } - if (!cqe) return -ETIMEDOUT; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h index b3bbf284fe71527aaf55278a4194dbca7e11d453..2d40dcf9d42ed9f8e7372ba87cdbaaefe81dc17a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -11,7 +11,9 @@ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB)) #define MLX5_ASO_WQEBBS_DATA \ (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB)) +#define ASO_CTRL_READ_EN BIT(0) #define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24 +#define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS)) struct mlx5_wqe_aso_ctrl_seg { __be32 va_h; @@ -70,6 +72,7 @@ enum { enum { MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2, + MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5, }; struct mlx5_aso; @@ -80,7 +83,7 @@ void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, u32 obj_id, u32 opc_mode); void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, struct mlx5_wqe_ctrl_seg *doorbell_cseg); -int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms); +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data); struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn); void mlx5_aso_destroy(struct mlx5_aso *aso); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index 91e806c1aa21126abe7bcdc9e8817a8efd1bc79c..d3a9ae80fd30e80571dd510cb6dfa84a320a6e4b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -65,6 +65,8 @@ enum { MLX5_MTPPS_FS_TIME_STAMP = BIT(0x4), MLX5_MTPPS_FS_OUT_PULSE_DURATION = BIT(0x5), MLX5_MTPPS_FS_ENH_OUT_PER_ADJ = BIT(0x7), + MLX5_MTPPS_FS_NPPS_PERIOD = BIT(0x9), + MLX5_MTPPS_FS_OUT_PULSE_DURATION_NS = BIT(0xa), }; static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev) @@ -72,6 +74,13 @@ static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev) return (mlx5_is_real_time_rq(mdev) || mlx5_is_real_time_sq(mdev)); } +static bool mlx5_npps_real_time_supported(struct mlx5_core_dev *mdev) +{ + return (mlx5_real_time_mode(mdev) && + MLX5_CAP_MCAM_FEATURE(mdev, npps_period) && + MLX5_CAP_MCAM_FEATURE(mdev, out_pulse_duration_ns)); +} + static bool mlx5_modify_mtutc_allowed(struct mlx5_core_dev *mdev) { return MLX5_CAP_MCAM_FEATURE(mdev, ptpcyc2realtime_modify); @@ -459,9 +468,95 @@ static u64 perout_conf_internal_timer(struct mlx5_core_dev *mdev, s64 sec) return find_target_cycles(mdev, target_ns); } -static u64 perout_conf_real_time(s64 sec) +static u64 perout_conf_real_time(s64 sec, u32 nsec) +{ + return (u64)nsec | (u64)sec << 32; +} + +static int perout_conf_1pps(struct mlx5_core_dev *mdev, struct ptp_clock_request *rq, + u64 *time_stamp, bool real_time) +{ + struct timespec64 ts; + s64 ns; + + ts.tv_nsec = rq->perout.period.nsec; + ts.tv_sec = rq->perout.period.sec; + ns = timespec64_to_ns(&ts); + + if ((ns >> 1) != 500000000LL) + return -EINVAL; + + *time_stamp = real_time ? perout_conf_real_time(rq->perout.start.sec, 0) : + perout_conf_internal_timer(mdev, rq->perout.start.sec); + + return 0; +} + +#define MLX5_MAX_PULSE_DURATION (BIT(__mlx5_bit_sz(mtpps_reg, out_pulse_duration_ns)) - 1) +static int mlx5_perout_conf_out_pulse_duration(struct mlx5_core_dev *mdev, + struct ptp_clock_request *rq, + u32 *out_pulse_duration_ns) { - return (u64)sec << 32; + struct mlx5_pps *pps_info = &mdev->clock.pps_info; + u32 out_pulse_duration; + struct timespec64 ts; + + if (rq->perout.flags & PTP_PEROUT_DUTY_CYCLE) { + ts.tv_sec = rq->perout.on.sec; + ts.tv_nsec = rq->perout.on.nsec; + out_pulse_duration = (u32)timespec64_to_ns(&ts); + } else { + /* out_pulse_duration_ns should be up to 50% of the + * pulse period as default + */ + ts.tv_sec = rq->perout.period.sec; + ts.tv_nsec = rq->perout.period.nsec; + out_pulse_duration = (u32)timespec64_to_ns(&ts) >> 1; + } + + if (out_pulse_duration < pps_info->min_out_pulse_duration_ns || + out_pulse_duration > MLX5_MAX_PULSE_DURATION) { + mlx5_core_err(mdev, "NPPS pulse duration %u is not in [%llu, %lu]\n", + out_pulse_duration, pps_info->min_out_pulse_duration_ns, + MLX5_MAX_PULSE_DURATION); + return -EINVAL; + } + *out_pulse_duration_ns = out_pulse_duration; + + return 0; +} + +static int perout_conf_npps_real_time(struct mlx5_core_dev *mdev, struct ptp_clock_request *rq, + u32 *field_select, u32 *out_pulse_duration_ns, + u64 *period, u64 *time_stamp) +{ + struct mlx5_pps *pps_info = &mdev->clock.pps_info; + struct ptp_clock_time *time = &rq->perout.start; + struct timespec64 ts; + + ts.tv_sec = rq->perout.period.sec; + ts.tv_nsec = rq->perout.period.nsec; + if (timespec64_to_ns(&ts) < pps_info->min_npps_period) { + mlx5_core_err(mdev, "NPPS period is lower than minimal npps period %llu\n", + pps_info->min_npps_period); + return -EINVAL; + } + *period = perout_conf_real_time(rq->perout.period.sec, rq->perout.period.nsec); + + if (mlx5_perout_conf_out_pulse_duration(mdev, rq, out_pulse_duration_ns)) + return -EINVAL; + + *time_stamp = perout_conf_real_time(time->sec, time->nsec); + *field_select |= MLX5_MTPPS_FS_NPPS_PERIOD | + MLX5_MTPPS_FS_OUT_PULSE_DURATION_NS; + + return 0; +} + +static bool mlx5_perout_verify_flags(struct mlx5_core_dev *mdev, unsigned int flags) +{ + return ((!mlx5_npps_real_time_supported(mdev) && flags) || + (mlx5_npps_real_time_supported(mdev) && flags & ~PTP_PEROUT_DUTY_CYCLE)); } static int mlx5_perout_configure(struct ptp_clock_info *ptp, @@ -474,20 +569,20 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, container_of(clock, struct mlx5_core_dev, clock); bool rt_mode = mlx5_real_time_mode(mdev); u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0}; - struct timespec64 ts; + u32 out_pulse_duration_ns = 0; u32 field_select = 0; + u64 npps_period = 0; u64 time_stamp = 0; u8 pin_mode = 0; u8 pattern = 0; int pin = -1; int err = 0; - s64 ns; if (!MLX5_PPS_CAP(mdev)) return -EOPNOTSUPP; /* Reject requests with unsupported flags */ - if (rq->perout.flags) + if (mlx5_perout_verify_flags(mdev, rq->perout.flags)) return -EOPNOTSUPP; if (rq->perout.index >= clock->ptp_info.n_pins) @@ -500,29 +595,25 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, if (on) { bool rt_mode = mlx5_real_time_mode(mdev); - s64 sec = rq->perout.start.sec; - - if (rq->perout.start.nsec) - return -EINVAL; pin_mode = MLX5_PIN_MODE_OUT; pattern = MLX5_OUT_PATTERN_PERIODIC; - ts.tv_sec = rq->perout.period.sec; - ts.tv_nsec = rq->perout.period.nsec; - ns = timespec64_to_ns(&ts); - if ((ns >> 1) != 500000000LL) + if (rt_mode && rq->perout.start.sec > U32_MAX) return -EINVAL; - if (rt_mode && sec > U32_MAX) - return -EINVAL; - - time_stamp = rt_mode ? perout_conf_real_time(sec) : - perout_conf_internal_timer(mdev, sec); - field_select |= MLX5_MTPPS_FS_PIN_MODE | MLX5_MTPPS_FS_PATTERN | MLX5_MTPPS_FS_TIME_STAMP; + + if (mlx5_npps_real_time_supported(mdev)) + err = perout_conf_npps_real_time(mdev, rq, &field_select, + &out_pulse_duration_ns, &npps_period, + &time_stamp); + else + err = perout_conf_1pps(mdev, rq, &time_stamp, rt_mode); + if (err) + return err; } MLX5_SET(mtpps_reg, in, pin, pin); @@ -531,7 +622,8 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp, MLX5_SET(mtpps_reg, in, enable, on); MLX5_SET64(mtpps_reg, in, time_stamp, time_stamp); MLX5_SET(mtpps_reg, in, field_select, field_select); - + MLX5_SET64(mtpps_reg, in, npps_period, npps_period); + MLX5_SET(mtpps_reg, in, out_pulse_duration_ns, out_pulse_duration_ns); err = mlx5_set_mtpps(mdev, in, sizeof(in)); if (err) return err; @@ -687,6 +779,13 @@ static void mlx5_get_pps_caps(struct mlx5_core_dev *mdev) clock->ptp_info.n_per_out = MLX5_GET(mtpps_reg, out, cap_max_num_of_pps_out_pins); + if (MLX5_CAP_MCAM_FEATURE(mdev, npps_period)) + clock->pps_info.min_npps_period = 1 << MLX5_GET(mtpps_reg, out, + cap_log_min_npps_period); + if (MLX5_CAP_MCAM_FEATURE(mdev, out_pulse_duration_ns)) + clock->pps_info.min_out_pulse_duration_ns = 1 << MLX5_GET(mtpps_reg, out, + cap_log_min_out_pulse_duration_ns); + clock->pps_info.pin_caps[0] = MLX5_GET(mtpps_reg, out, cap_pin_0_mode); clock->pps_info.pin_caps[1] = MLX5_GET(mtpps_reg, out, cap_pin_1_mode); clock->pps_info.pin_caps[2] = MLX5_GET(mtpps_reg, out, cap_pin_2_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 2f536c5d30b195dd9986045c8f331a79f77c0c31..032adb21ad4b663cd669dea857b8f7cff088fad2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -83,6 +83,7 @@ int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, voi enum { MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, + MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC, }; int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index bec8d6d0b5f67690513237f03bfc525045dc2319..0b459d841c3a33d5c758c9cbd48ac47a847d3a66 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -494,6 +494,24 @@ static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) return err; } +bool mlx5_is_roce_on(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + union devlink_param_value val; + int err; + + err = devlink_param_driverinit_value_get(devlink, + DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, + &val); + + if (!err) + return val.vbool; + + mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err); + return MLX5_CAP_GEN(dev, roce); +} +EXPORT_SYMBOL(mlx5_is_roce_on); + static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) { void *set_hca_cap; @@ -597,7 +615,8 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix)); if (MLX5_CAP_GEN(dev, roce_rw_supported)) - MLX5_SET(cmd_hca_cap, set_hca_cap, roce, mlx5_is_roce_init_enabled(dev)); + MLX5_SET(cmd_hca_cap, set_hca_cap, roce, + mlx5_is_roce_on(dev)); max_uc_list = max_uc_list_get_devlink_param(dev); if (max_uc_list > 0) @@ -623,7 +642,7 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) */ static bool is_roce_fw_disabled(struct mlx5_core_dev *dev) { - return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_init_enabled(dev)) || + return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) || (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce)); } @@ -652,6 +671,33 @@ static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx) return err; } +static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev, + void *set_ctx) +{ + void *set_hca_cap; + int err; + + if (!MLX5_CAP_GEN(dev, port_selection_cap)) + return 0; + + err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION); + if (err) + return err; + + if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) || + !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass)) + return 0; + + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability); + memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur, + MLX5_ST_SZ_BYTES(port_selection_cap)); + MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1); + + err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION); + + return err; +} + static int set_hca_cap(struct mlx5_core_dev *dev) { int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); @@ -696,6 +742,13 @@ static int set_hca_cap(struct mlx5_core_dev *dev) goto out; } + memset(set_ctx, 0, set_sz); + err = handle_hca_cap_port_selection(dev, set_ctx); + if (err) { + mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n"); + goto out; + } + out: kfree(set_ctx); return err; @@ -1039,7 +1092,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) mlx5_devcom_unregister_device(dev->priv.devcom); } -static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout) +static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout) { int err; @@ -1077,10 +1130,12 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout) mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP); + mlx5_start_health_poll(dev); + err = mlx5_core_enable_hca(dev, 0); if (err) { mlx5_core_err(dev, "enable hca failed\n"); - goto err_cmd_cleanup; + goto stop_health_poll; } err = mlx5_core_set_issi(dev); @@ -1132,8 +1187,7 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout) mlx5_core_err(dev, "query hca failed\n"); goto reclaim_boot_pages; } - - mlx5_start_health_poll(dev); + mlx5_start_health_fw_log_up(dev); return 0; @@ -1141,6 +1195,8 @@ reclaim_boot_pages: mlx5_reclaim_startup_pages(dev); err_disable_hca: mlx5_core_disable_hca(dev, 0); +stop_health_poll: + mlx5_stop_health_poll(dev, boot); err_cmd_cleanup: mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); mlx5_cmd_cleanup(dev); @@ -1152,7 +1208,6 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) { int err; - mlx5_stop_health_poll(dev, boot); err = mlx5_cmd_teardown_hca(dev); if (err) { mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n"); @@ -1160,6 +1215,7 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot) } mlx5_reclaim_startup_pages(dev); mlx5_core_disable_hca(dev, 0); + mlx5_stop_health_poll(dev, boot); mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN); mlx5_cmd_cleanup(dev); @@ -1309,7 +1365,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev) mutex_lock(&dev->intf_state_mutex); dev->state = MLX5_DEVICE_STATE_UP; - err = mlx5_function_setup(dev, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); + err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT)); if (err) goto err_function; @@ -1397,7 +1453,7 @@ int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT); else timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT); - err = mlx5_function_setup(dev, timeout); + err = mlx5_function_setup(dev, false, timeout); if (err) goto err_function; @@ -1488,6 +1544,8 @@ static const int types[] = { MLX5_CAP_IPSEC, MLX5_CAP_PORT_SELECTION, MLX5_CAP_DEV_SHAMPO, + MLX5_CAP_MACSEC, + MLX5_CAP_ADV_VIRTUALIZATION, }; static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) @@ -1530,7 +1588,9 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile)); INIT_LIST_HEAD(&priv->ctx_list); spin_lock_init(&priv->ctx_lock); + lockdep_register_key(&dev->lock_key); mutex_init(&dev->intf_state_mutex); + lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key); mutex_init(&priv->bfregs.reg_head.lock); mutex_init(&priv->bfregs.wc_head.lock); @@ -1597,6 +1657,7 @@ err_timeout_init: mutex_destroy(&priv->bfregs.wc_head.lock); mutex_destroy(&priv->bfregs.reg_head.lock); mutex_destroy(&dev->intf_state_mutex); + lockdep_unregister_key(&dev->lock_key); return err; } @@ -1618,6 +1679,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) mutex_destroy(&priv->bfregs.wc_head.lock); mutex_destroy(&priv->bfregs.reg_head.lock); mutex_destroy(&dev->intf_state_mutex); + lockdep_unregister_key(&dev->lock_key); } static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index ad61b86d57699b1d38d13e4d8b1cbab5c0134c53..a806e3de7b7c321ddb17861951e91d47c92bd881 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -143,6 +143,36 @@ enum mlx5_semaphore_space_address { #define MLX5_DEFAULT_PROF 2 +static inline int mlx5_flexible_inlen(struct mlx5_core_dev *dev, size_t fixed, + size_t item_size, size_t num_items, + const char *func, int line) +{ + int inlen; + + if (fixed > INT_MAX || item_size > INT_MAX || num_items > INT_MAX) { + mlx5_core_err(dev, "%s: %s:%d: input values too big: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + if (check_mul_overflow((int)item_size, (int)num_items, &inlen)) { + mlx5_core_err(dev, "%s: %s:%d: multiplication overflow: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + if (check_add_overflow((int)fixed, inlen, &inlen)) { + mlx5_core_err(dev, "%s: %s:%d: addition overflow: %zu + %zu * %zu\n", + __func__, func, line, fixed, item_size, num_items); + return -ENOMEM; + } + + return inlen; +} + +#define MLX5_FLEXIBLE_INLEN(dev, fixed, item_size, num_items) \ + mlx5_flexible_inlen(dev, fixed, item_size, num_items, __func__, __LINE__) + int mlx5_query_hca_caps(struct mlx5_core_dev *dev); int mlx5_query_board_id(struct mlx5_core_dev *dev); int mlx5_cmd_init(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index ec76a8b1acc1cccaace705fd0ba424123ce467c7..60596357bfc7ae94f7201037a8014f3b81f941c7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -376,8 +376,8 @@ retry: goto out_dropped; } } + err = mlx5_cmd_check(dev, err, in, out); if (err) { - err = mlx5_cmd_check(dev, err, in, out); mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", func_id, npages, err); goto out_dropped; @@ -524,10 +524,13 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, dev->priv.reclaim_pages_discard += npages; } /* if triggered by FW event and failed by FW then ignore */ - if (event && err == -EREMOTEIO) + if (event && err == -EREMOTEIO) { err = 0; + goto out_free; + } + + err = mlx5_cmd_check(dev, err, in, out); if (err) { - err = mlx5_cmd_check(dev, err, in, out); mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err); goto out_free; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index e1bd54574ea52d6d841090ef0f938ea209bb486e..a1548e6bfb35dadc73b18863da27c89417a9fc3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -493,29 +493,6 @@ int mlx5_query_port_vl_hw_cap(struct mlx5_core_dev *dev, } EXPORT_SYMBOL_GPL(mlx5_query_port_vl_hw_cap); -int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev, - u8 port_num, void *out, size_t sz) -{ - u32 *in; - int err; - - in = kvzalloc(sz, GFP_KERNEL); - if (!in) { - err = -ENOMEM; - return err; - } - - MLX5_SET(ppcnt_reg, in, local_port, port_num); - - MLX5_SET(ppcnt_reg, in, grp, MLX5_INFINIBAND_PORT_COUNTERS_GROUP); - err = mlx5_core_access_reg(dev, in, sz, out, - sz, MLX5_REG_PPCNT, 0, 0); - - kvfree(in); - return err; -} -EXPORT_SYMBOL_GPL(mlx5_core_query_ib_ppcnt); - static int mlx5_query_pfcc_reg(struct mlx5_core_dev *dev, u32 *out, u32 out_size) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index ee2e1b7c1310d51ce3ba91b7cf6991c634747559..c0e6c487c63c1ef105c92c4dcc6c552f5a95202f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -159,11 +159,11 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) devl_lock(devlink); err = mlx5_device_enable_sriov(dev, num_vfs); + devl_unlock(devlink); if (err) { mlx5_core_warn(dev, "mlx5_device_enable_sriov failed : %d\n", err); return err; } - devl_unlock(devlink); err = pci_enable_sriov(pdev, num_vfs); if (err) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 062c7c74a1f3f336189ea4998bd933cafed38ff7..1777a1e508e7a3362113e2225f948dd09df7e174 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -1294,20 +1294,6 @@ struct mlx5dr_cmd_gid_attr { u32 roce_ver; }; -struct mlx5dr_cmd_qp_create_attr { - u32 page_id; - u32 pdn; - u32 cqn; - u32 pm_state; - u32 service_type; - u32 buff_umem_id; - u32 db_umem_id; - u32 sq_wqe_cnt; - u32 rq_wqe_cnt; - u32 rq_wqe_shift; - u8 isolate_vl_tc:1; -}; - int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num, u16 index, struct mlx5dr_cmd_gid_attr *attr); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h index 1fb185d6ac7f7ca9ab92404122996609256ccd93..d168622063d559d06b69656190c085411f5f8153 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h @@ -14,10 +14,6 @@ struct mlx5_fs_dr_action { struct mlx5dr_action *dr_action; }; -struct mlx5_fs_dr_ns { - struct mlx5_dr_ns *dr_ns; -}; - struct mlx5_fs_dr_rule { struct mlx5dr_rule *dr_rule; /* Only actions created by fs_dr */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h index e5c4dcd1425e982e71f4306d41ea09c69ec5b69b..4d629e5ddbc74a1b49dfbbc2506a3808c8722772 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h @@ -123,7 +123,7 @@ static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq) wq->cur_sz++; } -static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u8 n) +static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u16 n) { wq->wqe_ctr += n; wq->cur_sz += n; diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h index 5fdf9b7179f55a8264782dfc49c462436715ed9d..5a1027b0721556f7445700dc15786696ec3e2ac5 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige.h @@ -75,6 +75,7 @@ struct mlxbf_gige { struct net_device *netdev; struct platform_device *pdev; void __iomem *mdio_io; + void __iomem *clk_io; struct mii_bus *mdiobus; spinlock_t lock; /* for packet processing indices */ u16 rx_q_entries; @@ -137,7 +138,8 @@ enum mlxbf_gige_res { MLXBF_GIGE_RES_MDIO9, MLXBF_GIGE_RES_GPIO0, MLXBF_GIGE_RES_LLU, - MLXBF_GIGE_RES_PLU + MLXBF_GIGE_RES_PLU, + MLXBF_GIGE_RES_CLK }; /* Version of register data returned by mlxbf_gige_get_regs() */ diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c index b03e1c66bac0d9be9aa0154e52fb344bde7f82c4..2292d63a279c19a29117bd436760819dcfac6faf 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c @@ -156,7 +156,7 @@ static int mlxbf_gige_open(struct net_device *netdev) phy_start(phydev); - netif_napi_add(netdev, &priv->napi, mlxbf_gige_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &priv->napi, mlxbf_gige_poll); napi_enable(&priv->napi); netif_start_queue(netdev); diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c index 2e6c1b7af0964c230d60f3a12509bedff98d40a4..aa780b1614a3d63f4be25117fa79931fed5f08d8 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c @@ -22,10 +22,23 @@ #include #include "mlxbf_gige.h" +#include "mlxbf_gige_regs.h" #define MLXBF_GIGE_MDIO_GW_OFFSET 0x0 #define MLXBF_GIGE_MDIO_CFG_OFFSET 0x4 +#define MLXBF_GIGE_MDIO_FREQ_REFERENCE 156250000ULL +#define MLXBF_GIGE_MDIO_COREPLL_CONST 16384ULL +#define MLXBF_GIGE_MDC_CLK_NS 400 +#define MLXBF_GIGE_MDIO_PLL_I1CLK_REG1 0x4 +#define MLXBF_GIGE_MDIO_PLL_I1CLK_REG2 0x8 +#define MLXBF_GIGE_MDIO_CORE_F_SHIFT 0 +#define MLXBF_GIGE_MDIO_CORE_F_MASK GENMASK(25, 0) +#define MLXBF_GIGE_MDIO_CORE_R_SHIFT 26 +#define MLXBF_GIGE_MDIO_CORE_R_MASK GENMASK(31, 26) +#define MLXBF_GIGE_MDIO_CORE_OD_SHIFT 0 +#define MLXBF_GIGE_MDIO_CORE_OD_MASK GENMASK(3, 0) + /* Support clause 22 */ #define MLXBF_GIGE_MDIO_CL22_ST1 0x1 #define MLXBF_GIGE_MDIO_CL22_WRITE 0x1 @@ -50,27 +63,76 @@ #define MLXBF_GIGE_MDIO_CFG_MDIO_IN_SAMP_MASK GENMASK(23, 16) #define MLXBF_GIGE_MDIO_CFG_MDIO_OUT_SAMP_MASK GENMASK(31, 24) +#define MLXBF_GIGE_MDIO_CFG_VAL (FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_MODE_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO3_3_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_FULL_DRIVE_MASK, 1) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_IN_SAMP_MASK, 6) | \ + FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_OUT_SAMP_MASK, 13)) + +#define MLXBF_GIGE_BF2_COREPLL_ADDR 0x02800c30 +#define MLXBF_GIGE_BF2_COREPLL_SIZE 0x0000000c + +static struct resource corepll_params[] = { + [MLXBF_GIGE_VERSION_BF2] = { + .start = MLXBF_GIGE_BF2_COREPLL_ADDR, + .end = MLXBF_GIGE_BF2_COREPLL_ADDR + MLXBF_GIGE_BF2_COREPLL_SIZE - 1, + .name = "COREPLL_RES" + }, +}; + +/* Returns core clock i1clk in Hz */ +static u64 calculate_i1clk(struct mlxbf_gige *priv) +{ + u8 core_od, core_r; + u64 freq_output; + u32 reg1, reg2; + u32 core_f; + + reg1 = readl(priv->clk_io + MLXBF_GIGE_MDIO_PLL_I1CLK_REG1); + reg2 = readl(priv->clk_io + MLXBF_GIGE_MDIO_PLL_I1CLK_REG2); + + core_f = (reg1 & MLXBF_GIGE_MDIO_CORE_F_MASK) >> + MLXBF_GIGE_MDIO_CORE_F_SHIFT; + core_r = (reg1 & MLXBF_GIGE_MDIO_CORE_R_MASK) >> + MLXBF_GIGE_MDIO_CORE_R_SHIFT; + core_od = (reg2 & MLXBF_GIGE_MDIO_CORE_OD_MASK) >> + MLXBF_GIGE_MDIO_CORE_OD_SHIFT; + + /* Compute PLL output frequency as follow: + * + * CORE_F / 16384 + * freq_output = freq_reference * ---------------------------- + * (CORE_R + 1) * (CORE_OD + 1) + */ + freq_output = div_u64((MLXBF_GIGE_MDIO_FREQ_REFERENCE * core_f), + MLXBF_GIGE_MDIO_COREPLL_CONST); + freq_output = div_u64(freq_output, (core_r + 1) * (core_od + 1)); + + return freq_output; +} + /* Formula for encoding the MDIO period. The encoded value is * passed to the MDIO config register. * - * mdc_clk = 2*(val + 1)*i1clk + * mdc_clk = 2*(val + 1)*(core clock in sec) * - * 400 ns = 2*(val + 1)*(((1/430)*1000) ns) + * i1clk is in Hz: + * 400 ns = 2*(val + 1)*(1/i1clk) * - * val = (((400 * 430 / 1000) / 2) - 1) + * val = (((400/10^9) / (1/i1clk) / 2) - 1) + * val = (400/2 * i1clk)/10^9 - 1 */ -#define MLXBF_GIGE_I1CLK_MHZ 430 -#define MLXBF_GIGE_MDC_CLK_NS 400 +static u8 mdio_period_map(struct mlxbf_gige *priv) +{ + u8 mdio_period; + u64 i1clk; -#define MLXBF_GIGE_MDIO_PERIOD (((MLXBF_GIGE_MDC_CLK_NS * MLXBF_GIGE_I1CLK_MHZ / 1000) / 2) - 1) + i1clk = calculate_i1clk(priv); -#define MLXBF_GIGE_MDIO_CFG_VAL (FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_MODE_MASK, 1) | \ - FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO3_3_MASK, 1) | \ - FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_FULL_DRIVE_MASK, 1) | \ - FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDC_PERIOD_MASK, \ - MLXBF_GIGE_MDIO_PERIOD) | \ - FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_IN_SAMP_MASK, 6) | \ - FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDIO_OUT_SAMP_MASK, 13)) + mdio_period = div_u64((MLXBF_GIGE_MDC_CLK_NS >> 1) * i1clk, 1000000000) - 1; + + return mdio_period; +} static u32 mlxbf_gige_mdio_create_cmd(u16 data, int phy_add, int phy_reg, u32 opcode) @@ -117,6 +179,9 @@ static int mlxbf_gige_mdio_read(struct mii_bus *bus, int phy_add, int phy_reg) /* Only return ad bits of the gw register */ ret &= MLXBF_GIGE_MDIO_GW_AD_MASK; + /* The MDIO lock is set on read. To release it, clear gw register */ + writel(0, priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + return ret; } @@ -124,9 +189,9 @@ static int mlxbf_gige_mdio_write(struct mii_bus *bus, int phy_add, int phy_reg, u16 val) { struct mlxbf_gige *priv = bus->priv; + u32 temp; u32 cmd; int ret; - u32 temp; if (phy_reg & MII_ADDR_C45) return -EOPNOTSUPP; @@ -141,21 +206,50 @@ static int mlxbf_gige_mdio_write(struct mii_bus *bus, int phy_add, temp, !(temp & MLXBF_GIGE_MDIO_GW_BUSY_MASK), 5, 1000000); + /* The MDIO lock is set on read. To release it, clear gw register */ + writel(0, priv->mdio_io + MLXBF_GIGE_MDIO_GW_OFFSET); + return ret; } +static void mlxbf_gige_mdio_cfg(struct mlxbf_gige *priv) +{ + u8 mdio_period; + u32 val; + + mdio_period = mdio_period_map(priv); + + val = MLXBF_GIGE_MDIO_CFG_VAL; + val |= FIELD_PREP(MLXBF_GIGE_MDIO_CFG_MDC_PERIOD_MASK, mdio_period); + writel(val, priv->mdio_io + MLXBF_GIGE_MDIO_CFG_OFFSET); +} + int mlxbf_gige_mdio_probe(struct platform_device *pdev, struct mlxbf_gige *priv) { struct device *dev = &pdev->dev; + struct resource *res; int ret; priv->mdio_io = devm_platform_ioremap_resource(pdev, MLXBF_GIGE_RES_MDIO9); if (IS_ERR(priv->mdio_io)) return PTR_ERR(priv->mdio_io); - /* Configure mdio parameters */ - writel(MLXBF_GIGE_MDIO_CFG_VAL, - priv->mdio_io + MLXBF_GIGE_MDIO_CFG_OFFSET); + /* clk resource shared with other drivers so cannot use + * devm_platform_ioremap_resource + */ + res = platform_get_resource(pdev, IORESOURCE_MEM, MLXBF_GIGE_RES_CLK); + if (!res) { + /* For backward compatibility with older ACPI tables, also keep + * CLK resource internal to the driver. + */ + res = &corepll_params[MLXBF_GIGE_VERSION_BF2]; + } + + priv->clk_io = devm_ioremap(dev, res->start, resource_size(res)); + if (!priv->clk_io) + return -ENOMEM; + + mlxbf_gige_mdio_cfg(priv); priv->mdiobus = devm_mdiobus_alloc(dev); if (!priv->mdiobus) { diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h index 5fb33c9294bf935086256138ff9934d69e18de19..7be3a793984d5d696be5546e7c089c42a171cb01 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_regs.h @@ -8,6 +8,8 @@ #ifndef __MLXBF_GIGE_REGS_H__ #define __MLXBF_GIGE_REGS_H__ +#define MLXBF_GIGE_VERSION 0x0000 +#define MLXBF_GIGE_VERSION_BF2 0x0 #define MLXBF_GIGE_STATUS 0x0010 #define MLXBF_GIGE_STATUS_READY BIT(0) #define MLXBF_GIGE_INT_STATUS 0x0028 diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h index 60232fb8ccd7abc87d468ac0484505c801b63f54..09bef04b11d16087795c0c5870241354ea57cbdd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h +++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h @@ -703,6 +703,9 @@ MLXSW_ITEM32(cmd_mbox, config_profile, max_vepa_channels, 0x10, 0, 8); /* cmd_mbox_config_profile_max_lag * Maximum number of LAG IDs requested. + * Reserved when Spectrum-1/2/3, supported from Spectrum-4 and above. + * For Spectrum-4, firmware sets 128 for values between 1-128 and 256 for values + * between 129-256. */ MLXSW_ITEM32(cmd_mbox, config_profile, max_lag, 0x14, 0, 16); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 75553eb2c7f2b6be18fdcf76ebd17e74442c1c4c..e2a985ec2c765a36269d20ca60c3b186fa87c30d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -70,6 +70,8 @@ struct mlxsw_core { struct workqueue_struct *emad_wq; struct list_head rx_listener_list; struct list_head event_listener_list; + struct list_head irq_event_handler_list; + struct mutex irq_event_handler_lock; /* Locks access to handlers list */ struct { atomic64_t tid; struct list_head trans_list; @@ -184,6 +186,23 @@ unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core) } EXPORT_SYMBOL(mlxsw_core_max_ports); +int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag) +{ + struct mlxsw_driver *driver = mlxsw_core->driver; + + if (driver->profile->used_max_lag) { + *p_max_lag = driver->profile->max_lag; + return 0; + } + + if (!MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG)) + return -EIO; + + *p_max_lag = MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG); + return 0; +} +EXPORT_SYMBOL(mlxsw_core_max_lag); + void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core) { return mlxsw_core->driver_priv; @@ -633,7 +652,7 @@ static void mlxsw_emad_process_string_tlv(const struct sk_buff *skb, return; string = mlxsw_emad_string_tlv_string_data(string_tlv); - strlcpy(trans->emad_err_string, string, + strscpy(trans->emad_err_string, string, MLXSW_EMAD_STRING_TLV_STRING_LEN); } @@ -1305,21 +1324,6 @@ mlxsw_devlink_sb_pool_set(struct devlink *devlink, extack); } -static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port, - enum devlink_port_type port_type) -{ - struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink); - struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver; - struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port); - - if (!mlxsw_driver->port_type_set) - return -EOPNOTSUPP; - - return mlxsw_driver->port_type_set(mlxsw_core, - mlxsw_core_port->local_port, - port_type); -} - static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port, unsigned int sb_index, u16 pool_index, u32 *p_threshold) @@ -1650,7 +1654,6 @@ static const struct devlink_ops mlxsw_devlink_ops = { BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE), .reload_down = mlxsw_devlink_core_bus_device_reload_down, .reload_up = mlxsw_devlink_core_bus_device_reload_up, - .port_type_set = mlxsw_devlink_port_type_set, .port_split = mlxsw_devlink_port_split, .port_unsplit = mlxsw_devlink_port_unsplit, .sb_pool_get = mlxsw_devlink_sb_pool_get, @@ -2090,6 +2093,18 @@ static void mlxsw_core_health_fini(struct mlxsw_core *mlxsw_core) devlink_health_reporter_destroy(mlxsw_core->health.fw_fatal); } +static void mlxsw_core_irq_event_handler_init(struct mlxsw_core *mlxsw_core) +{ + INIT_LIST_HEAD(&mlxsw_core->irq_event_handler_list); + mutex_init(&mlxsw_core->irq_event_handler_lock); +} + +static void mlxsw_core_irq_event_handler_fini(struct mlxsw_core *mlxsw_core) +{ + mutex_destroy(&mlxsw_core->irq_event_handler_lock); + WARN_ON(!list_empty(&mlxsw_core->irq_event_handler_list)); +} + static int __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, const struct mlxsw_bus *mlxsw_bus, @@ -2101,6 +2116,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, struct mlxsw_core *mlxsw_core; struct mlxsw_driver *mlxsw_driver; size_t alloc_size; + u16 max_lag; int err; mlxsw_driver = mlxsw_core_driver_get(device_kind); @@ -2125,6 +2141,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, mlxsw_core->bus = mlxsw_bus; mlxsw_core->bus_priv = bus_priv; mlxsw_core->bus_info = mlxsw_bus_info; + mlxsw_core_irq_event_handler_init(mlxsw_core); err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile, &mlxsw_core->res); @@ -2141,10 +2158,9 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (err) goto err_ports_init; - if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) && - MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) { - alloc_size = sizeof(*mlxsw_core->lag.mapping) * - MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) * + err = mlxsw_core_max_lag(mlxsw_core, &max_lag); + if (!err && MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) { + alloc_size = sizeof(*mlxsw_core->lag.mapping) * max_lag * MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL); if (!mlxsw_core->lag.mapping) { @@ -2233,6 +2249,7 @@ err_ports_init: err_register_resources: mlxsw_bus->fini(bus_priv); err_bus_init: + mlxsw_core_irq_event_handler_fini(mlxsw_core); if (!reload) { devl_unlock(devlink); devlink_free(devlink); @@ -2302,6 +2319,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, if (!reload) devl_resources_unregister(devlink); mlxsw_core->bus->fini(mlxsw_core->bus_priv); + mlxsw_core_irq_event_handler_fini(mlxsw_core); if (!reload) { devl_unlock(devlink); devlink_free(devlink); @@ -2772,6 +2790,57 @@ int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list) } EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait); +struct mlxsw_core_irq_event_handler_item { + struct list_head list; + void (*cb)(struct mlxsw_core *mlxsw_core); +}; + +int mlxsw_core_irq_event_handler_register(struct mlxsw_core *mlxsw_core, + mlxsw_irq_event_cb_t cb) +{ + struct mlxsw_core_irq_event_handler_item *item; + + item = kzalloc(sizeof(*item), GFP_KERNEL); + if (!item) + return -ENOMEM; + item->cb = cb; + mutex_lock(&mlxsw_core->irq_event_handler_lock); + list_add_tail(&item->list, &mlxsw_core->irq_event_handler_list); + mutex_unlock(&mlxsw_core->irq_event_handler_lock); + return 0; +} +EXPORT_SYMBOL(mlxsw_core_irq_event_handler_register); + +void mlxsw_core_irq_event_handler_unregister(struct mlxsw_core *mlxsw_core, + mlxsw_irq_event_cb_t cb) +{ + struct mlxsw_core_irq_event_handler_item *item, *tmp; + + mutex_lock(&mlxsw_core->irq_event_handler_lock); + list_for_each_entry_safe(item, tmp, + &mlxsw_core->irq_event_handler_list, list) { + if (item->cb == cb) { + list_del(&item->list); + kfree(item); + } + } + mutex_unlock(&mlxsw_core->irq_event_handler_lock); +} +EXPORT_SYMBOL(mlxsw_core_irq_event_handler_unregister); + +void mlxsw_core_irq_event_handlers_call(struct mlxsw_core *mlxsw_core) +{ + struct mlxsw_core_irq_event_handler_item *item; + + mutex_lock(&mlxsw_core->irq_event_handler_lock); + list_for_each_entry(item, &mlxsw_core->irq_event_handler_list, list) { + if (item->cb) + item->cb(mlxsw_core); + } + mutex_unlock(&mlxsw_core->irq_event_handler_lock); +} +EXPORT_SYMBOL(mlxsw_core_irq_event_handlers_call); + static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core, const struct mlxsw_reg_info *reg, char *payload, @@ -3115,18 +3184,6 @@ void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port, } EXPORT_SYMBOL(mlxsw_core_port_eth_set); -void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port, - void *port_driver_priv) -{ - struct mlxsw_core_port *mlxsw_core_port = - &mlxsw_core->ports[local_port]; - struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port; - - mlxsw_core_port->port_driver_priv = port_driver_priv; - devlink_port_type_ib_set(devlink_port, NULL); -} -EXPORT_SYMBOL(mlxsw_core_port_ib_set); - void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv) { @@ -3139,18 +3196,6 @@ void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port, } EXPORT_SYMBOL(mlxsw_core_port_clear); -enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, - u16 local_port) -{ - struct mlxsw_core_port *mlxsw_core_port = - &mlxsw_core->ports[local_port]; - struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port; - - return devlink_port->type; -} -EXPORT_SYMBOL(mlxsw_core_port_type_get); - - struct devlink_port * mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, u16 local_port) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 02d9cc2ef0c8e7414573e271449e2e73af50e107..ca0c3d2bee6b31e247997557a138477c662dab94 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -35,6 +35,8 @@ struct mlxsw_fw_rev; unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core); +int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag); + void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core); struct mlxsw_linecards *mlxsw_core_linecards(struct mlxsw_core *mlxsw_core); @@ -215,6 +217,14 @@ int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core, mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv); int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list); +typedef void mlxsw_irq_event_cb_t(struct mlxsw_core *mlxsw_core); + +int mlxsw_core_irq_event_handler_register(struct mlxsw_core *mlxsw_core, + mlxsw_irq_event_cb_t cb); +void mlxsw_core_irq_event_handler_unregister(struct mlxsw_core *mlxsw_core, + mlxsw_irq_event_cb_t cb); +void mlxsw_core_irq_event_handlers_call(struct mlxsw_core *mlxsw_core); + int mlxsw_reg_query(struct mlxsw_core *mlxsw_core, const struct mlxsw_reg_info *reg, char *payload); int mlxsw_reg_write(struct mlxsw_core *mlxsw_core, @@ -256,12 +266,8 @@ int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core, void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core); void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv, struct net_device *dev); -void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port, - void *port_driver_priv); void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv); -enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, - u16 local_port); struct devlink_port * mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, u16 local_port); @@ -291,6 +297,7 @@ struct mlxsw_swid_config { struct mlxsw_config_profile { u16 used_max_vepa_channels:1, + used_max_lag:1, used_max_mid:1, used_max_pgt:1, used_max_system_port:1, @@ -306,6 +313,7 @@ struct mlxsw_config_profile { used_kvd_sizes:1, used_cqe_time_stamp_type:1; u8 max_vepa_channels; + u16 max_lag; u16 max_mid; u16 max_pgt; u16 max_system_port; @@ -341,8 +349,6 @@ struct mlxsw_driver { const struct mlxsw_bus_info *mlxsw_bus_info, struct netlink_ext_ack *extack); void (*fini)(struct mlxsw_core *mlxsw_core); - int (*port_type_set)(struct mlxsw_core *mlxsw_core, u16 local_port, - enum devlink_port_type new_type); int (*port_split)(struct mlxsw_core *mlxsw_core, u16 local_port, unsigned int count, struct netlink_ext_ack *extack); int (*port_unsplit)(struct mlxsw_core *mlxsw_core, u16 local_port, diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c index 636db9a8745774adc42d97946b3166209c4f41e9..9dfe7148199f9ec4d79e8aa23ea8d8e7bdfe69d7 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c @@ -737,8 +737,9 @@ mlxsw_afa_cookie_create(struct mlxsw_afa *mlxsw_afa, if (!cookie) return ERR_PTR(-ENOMEM); refcount_set(&cookie->ref_count, 1); - memcpy(&cookie->fa_cookie, fa_cookie, - sizeof(*fa_cookie) + fa_cookie->cookie_len); + cookie->fa_cookie = *fa_cookie; + memcpy(cookie->fa_cookie.cookie, fa_cookie->cookie, + fa_cookie->cookie_len); err = rhashtable_insert_fast(&mlxsw_afa->cookie_ht, &cookie->ht_node, mlxsw_afa_cookie_ht_params); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c index ca59f0b946da96f89e1c7bb905ecc9b6b83c6a9b..83d2dc91ba2c8a814c2750754aa5ea398dc4f47b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c @@ -785,6 +785,21 @@ static int mlxsw_linecard_status_get_and_process(struct mlxsw_core *mlxsw_core, return mlxsw_linecard_status_process(linecards, linecard, mddq_pl); } +static void mlxsw_linecards_irq_event_handler(struct mlxsw_core *mlxsw_core) +{ + struct mlxsw_linecards *linecards = mlxsw_core_linecards(mlxsw_core); + int i; + + /* Handle change of line card active state. */ + for (i = 0; i < linecards->count; i++) { + struct mlxsw_linecard *linecard = mlxsw_linecard_get(linecards, + i + 1); + + mlxsw_linecard_status_get_and_process(mlxsw_core, linecards, + linecard); + } +} + static const char * const mlxsw_linecard_status_event_type_name[] = { [MLXSW_LINECARD_STATUS_EVENT_TYPE_PROVISION] = "provision", [MLXSW_LINECARD_STATUS_EVENT_TYPE_UNPROVISION] = "unprovision", @@ -1238,7 +1253,6 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core, { struct devlink_linecard *devlink_linecard; struct mlxsw_linecard *linecard; - int err; linecard = mlxsw_linecard_get(linecards, slot_index); linecard->slot_index = slot_index; @@ -1248,17 +1262,45 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core, devlink_linecard = devlink_linecard_create(priv_to_devlink(mlxsw_core), slot_index, &mlxsw_linecard_ops, linecard); - if (IS_ERR(devlink_linecard)) { - err = PTR_ERR(devlink_linecard); - goto err_devlink_linecard_create; - } + if (IS_ERR(devlink_linecard)) + return PTR_ERR(devlink_linecard); + linecard->devlink_linecard = devlink_linecard; INIT_DELAYED_WORK(&linecard->status_event_to_dw, &mlxsw_linecard_status_event_to_work); + return 0; +} + +static void mlxsw_linecard_fini(struct mlxsw_core *mlxsw_core, + struct mlxsw_linecards *linecards, + u8 slot_index) +{ + struct mlxsw_linecard *linecard; + + linecard = mlxsw_linecard_get(linecards, slot_index); + cancel_delayed_work_sync(&linecard->status_event_to_dw); + /* Make sure all scheduled events are processed */ + mlxsw_core_flush_owq(); + if (linecard->active) + mlxsw_linecard_active_clear(linecard); + mlxsw_linecard_bdev_del(linecard); + devlink_linecard_destroy(linecard->devlink_linecard); + mutex_destroy(&linecard->lock); +} + +static int +mlxsw_linecard_event_delivery_init(struct mlxsw_core *mlxsw_core, + struct mlxsw_linecards *linecards, + u8 slot_index) +{ + struct mlxsw_linecard *linecard; + int err; + + linecard = mlxsw_linecard_get(linecards, slot_index); err = mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, true); if (err) - goto err_event_delivery_set; + return err; err = mlxsw_linecard_status_get_and_process(mlxsw_core, linecards, linecard); @@ -1269,29 +1311,18 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core, err_status_get_and_process: mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, false); -err_event_delivery_set: - devlink_linecard_destroy(linecard->devlink_linecard); -err_devlink_linecard_create: - mutex_destroy(&linecard->lock); return err; } -static void mlxsw_linecard_fini(struct mlxsw_core *mlxsw_core, - struct mlxsw_linecards *linecards, - u8 slot_index) +static void +mlxsw_linecard_event_delivery_fini(struct mlxsw_core *mlxsw_core, + struct mlxsw_linecards *linecards, + u8 slot_index) { struct mlxsw_linecard *linecard; linecard = mlxsw_linecard_get(linecards, slot_index); mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, false); - cancel_delayed_work_sync(&linecard->status_event_to_dw); - /* Make sure all scheduled events are processed */ - mlxsw_core_flush_owq(); - if (linecard->active) - mlxsw_linecard_active_clear(linecard); - mlxsw_linecard_bdev_del(linecard); - devlink_linecard_destroy(linecard->devlink_linecard); - mutex_destroy(&linecard->lock); } /* LINECARDS INI BUNDLE FILE @@ -1505,6 +1536,11 @@ int mlxsw_linecards_init(struct mlxsw_core *mlxsw_core, if (err) goto err_traps_register; + err = mlxsw_core_irq_event_handler_register(mlxsw_core, + mlxsw_linecards_irq_event_handler); + if (err) + goto err_irq_event_handler_register; + mlxsw_core_linecards_set(mlxsw_core, linecards); for (i = 0; i < linecards->count; i++) { @@ -1513,11 +1549,25 @@ int mlxsw_linecards_init(struct mlxsw_core *mlxsw_core, goto err_linecard_init; } + for (i = 0; i < linecards->count; i++) { + err = mlxsw_linecard_event_delivery_init(mlxsw_core, linecards, + i + 1); + if (err) + goto err_linecard_event_delivery_init; + } + return 0; +err_linecard_event_delivery_init: + for (i--; i >= 0; i--) + mlxsw_linecard_event_delivery_fini(mlxsw_core, linecards, i + 1); + i = linecards->count; err_linecard_init: for (i--; i >= 0; i--) mlxsw_linecard_fini(mlxsw_core, linecards, i + 1); + mlxsw_core_irq_event_handler_unregister(mlxsw_core, + mlxsw_linecards_irq_event_handler); +err_irq_event_handler_register: mlxsw_core_traps_unregister(mlxsw_core, mlxsw_linecard_listener, ARRAY_SIZE(mlxsw_linecard_listener), mlxsw_core); @@ -1535,8 +1585,12 @@ void mlxsw_linecards_fini(struct mlxsw_core *mlxsw_core) if (!linecards) return; + for (i = 0; i < linecards->count; i++) + mlxsw_linecard_event_delivery_fini(mlxsw_core, linecards, i + 1); for (i = 0; i < linecards->count; i++) mlxsw_linecard_fini(mlxsw_core, linecards, i + 1); + mlxsw_core_irq_event_handler_unregister(mlxsw_core, + mlxsw_linecards_irq_event_handler); mlxsw_core_traps_unregister(mlxsw_core, mlxsw_linecard_listener, ARRAY_SIZE(mlxsw_linecard_listener), mlxsw_core); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c index 3548fe1df7c87df609668734ef611d45e63817c3..987fe5c9d5a362f3f5c0c3a8610a3f3c1e918f1f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c @@ -21,7 +21,6 @@ #define MLXSW_THERMAL_ASIC_TEMP_HOT 105000 /* 105C */ #define MLXSW_THERMAL_HYSTERESIS_TEMP 5000 /* 5C */ #define MLXSW_THERMAL_MODULE_TEMP_SHIFT (MLXSW_THERMAL_HYSTERESIS_TEMP * 2) -#define MLXSW_THERMAL_TEMP_SCORE_MAX GENMASK(31, 0) #define MLXSW_THERMAL_MAX_STATE 10 #define MLXSW_THERMAL_MIN_STATE 2 #define MLXSW_THERMAL_MAX_DUTY 255 @@ -101,8 +100,6 @@ struct mlxsw_thermal { struct thermal_cooling_device *cdevs[MLXSW_MFCR_PWMS_MAX]; u8 cooling_levels[MLXSW_THERMAL_MAX_STATE + 1]; struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS]; - unsigned int tz_highest_score; - struct thermal_zone_device *tz_highest_dev; struct mlxsw_thermal_area line_cards[]; }; @@ -193,34 +190,6 @@ mlxsw_thermal_module_trips_update(struct device *dev, struct mlxsw_core *core, return 0; } -static void mlxsw_thermal_tz_score_update(struct mlxsw_thermal *thermal, - struct thermal_zone_device *tzdev, - struct mlxsw_thermal_trip *trips, - int temp) -{ - struct mlxsw_thermal_trip *trip = trips; - unsigned int score, delta, i, shift = 1; - - /* Calculate thermal zone score, if temperature is above the hot - * threshold score is set to MLXSW_THERMAL_TEMP_SCORE_MAX. - */ - score = MLXSW_THERMAL_TEMP_SCORE_MAX; - for (i = MLXSW_THERMAL_TEMP_TRIP_NORM; i < MLXSW_THERMAL_NUM_TRIPS; - i++, trip++) { - if (temp < trip->temp) { - delta = DIV_ROUND_CLOSEST(temp, trip->temp - temp); - score = delta * shift; - break; - } - shift *= 256; - } - - if (score > thermal->tz_highest_score) { - thermal->tz_highest_score = score; - thermal->tz_highest_dev = tzdev; - } -} - static int mlxsw_thermal_bind(struct thermal_zone_device *tzdev, struct thermal_cooling_device *cdev) { @@ -286,9 +255,6 @@ static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev, return err; } mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL); - if (temp > 0) - mlxsw_thermal_tz_score_update(thermal, tzdev, thermal->trips, - temp); *p_temp = temp; return 0; @@ -349,21 +315,6 @@ static int mlxsw_thermal_set_trip_hyst(struct thermal_zone_device *tzdev, return 0; } -static int mlxsw_thermal_trend_get(struct thermal_zone_device *tzdev, - int trip, enum thermal_trend *trend) -{ - struct mlxsw_thermal *thermal = tzdev->devdata; - - if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS) - return -EINVAL; - - if (tzdev == thermal->tz_highest_dev) - return 1; - - *trend = THERMAL_TREND_STABLE; - return 0; -} - static struct thermal_zone_params mlxsw_thermal_params = { .no_hwmon = true, }; @@ -377,7 +328,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_ops = { .set_trip_temp = mlxsw_thermal_set_trip_temp, .get_trip_hyst = mlxsw_thermal_get_trip_hyst, .set_trip_hyst = mlxsw_thermal_set_trip_hyst, - .get_trend = mlxsw_thermal_trend_get, }; static int mlxsw_thermal_module_bind(struct thermal_zone_device *tzdev, @@ -463,7 +413,6 @@ static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev, int temp, crit_temp, emerg_temp; struct device *dev; u16 sensor_index; - int err; dev = thermal->bus_info->dev; sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + tz->module; @@ -479,10 +428,8 @@ static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev, return 0; /* Update trip points. */ - err = mlxsw_thermal_module_trips_update(dev, thermal->core, tz, - crit_temp, emerg_temp); - if (!err && temp > 0) - mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp); + mlxsw_thermal_module_trips_update(dev, thermal->core, tz, + crit_temp, emerg_temp); return 0; } @@ -546,22 +493,6 @@ mlxsw_thermal_module_trip_hyst_set(struct thermal_zone_device *tzdev, int trip, return 0; } -static int mlxsw_thermal_module_trend_get(struct thermal_zone_device *tzdev, - int trip, enum thermal_trend *trend) -{ - struct mlxsw_thermal_module *tz = tzdev->devdata; - struct mlxsw_thermal *thermal = tz->parent; - - if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS) - return -EINVAL; - - if (tzdev == thermal->tz_highest_dev) - return 1; - - *trend = THERMAL_TREND_STABLE; - return 0; -} - static struct thermal_zone_device_ops mlxsw_thermal_module_ops = { .bind = mlxsw_thermal_module_bind, .unbind = mlxsw_thermal_module_unbind, @@ -571,7 +502,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_module_ops = { .set_trip_temp = mlxsw_thermal_module_trip_temp_set, .get_trip_hyst = mlxsw_thermal_module_trip_hyst_get, .set_trip_hyst = mlxsw_thermal_module_trip_hyst_set, - .get_trend = mlxsw_thermal_module_trend_get, }; static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev, @@ -592,8 +522,6 @@ static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev, return err; mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL); - if (temp > 0) - mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp); *p_temp = temp; return 0; @@ -608,7 +536,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_gearbox_ops = { .set_trip_temp = mlxsw_thermal_module_trip_temp_set, .get_trip_hyst = mlxsw_thermal_module_trip_hyst_get, .set_trip_hyst = mlxsw_thermal_module_trip_hyst_set, - .get_trend = mlxsw_thermal_module_trend_get, }; static int mlxsw_thermal_get_max_state(struct thermal_cooling_device *cdev, diff --git a/drivers/net/ethernet/mellanox/mlxsw/i2c.c b/drivers/net/ethernet/mellanox/mlxsw/i2c.c index ce843ea9146466d402a97f4e249aec548bcd06ed..f5f5f8dc3d190c5298ce8b3c9272a8155062d4b3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/i2c.c +++ b/drivers/net/ethernet/mellanox/mlxsw/i2c.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "cmd.h" @@ -51,6 +52,15 @@ #define MLXSW_I2C_TIMEOUT_MSECS 5000 #define MLXSW_I2C_MAX_DATA_SIZE 256 +/* Driver can be initialized by kernel platform driver or from the user + * space. In the first case IRQ line number is passed through the platform + * data, otherwise default IRQ line is to be used. Default IRQ is relevant + * only for specific I2C slave address, allowing 3.4 MHz I2C path to the chip + * (special hardware feature for I2C acceleration). + */ +#define MLXSW_I2C_DEFAULT_IRQ 17 +#define MLXSW_FAST_I2C_SLAVE 0x37 + /** * struct mlxsw_i2c - device private data: * @cmd: command attributes; @@ -63,6 +73,9 @@ * @core: switch core pointer; * @bus_info: bus info block; * @block_size: maximum block size allowed to pass to under layer; + * @pdata: device platform data; + * @irq_work: interrupts work item; + * @irq: IRQ line number; */ struct mlxsw_i2c { struct { @@ -76,6 +89,9 @@ struct mlxsw_i2c { struct mlxsw_core *core; struct mlxsw_bus_info bus_info; u16 block_size; + struct mlxreg_core_hotplug_platform_data *pdata; + struct work_struct irq_work; + int irq; }; #define MLXSW_I2C_READ_MSG(_client, _addr_buf, _buf, _len) { \ @@ -546,6 +562,67 @@ static void mlxsw_i2c_fini(void *bus_priv) mlxsw_i2c->core = NULL; } +static void mlxsw_i2c_work_handler(struct work_struct *work) +{ + struct mlxsw_i2c *mlxsw_i2c; + + mlxsw_i2c = container_of(work, struct mlxsw_i2c, irq_work); + mlxsw_core_irq_event_handlers_call(mlxsw_i2c->core); +} + +static irqreturn_t mlxsw_i2c_irq_handler(int irq, void *dev) +{ + struct mlxsw_i2c *mlxsw_i2c = dev; + + mlxsw_core_schedule_work(&mlxsw_i2c->irq_work); + + /* Interrupt handler shares IRQ line with 'main' interrupt handler. + * Return here IRQ_NONE, while main handler will return IRQ_HANDLED. + */ + return IRQ_NONE; +} + +static int mlxsw_i2c_irq_init(struct mlxsw_i2c *mlxsw_i2c, u8 addr) +{ + int err; + + /* Initialize interrupt handler if system hotplug driver is reachable, + * otherwise interrupt line is not enabled and interrupts will not be + * raised to CPU. Also request_irq() call will be not valid. + */ + if (!IS_REACHABLE(CONFIG_MLXREG_HOTPLUG)) + return 0; + + /* Set default interrupt line. */ + if (mlxsw_i2c->pdata && mlxsw_i2c->pdata->irq) + mlxsw_i2c->irq = mlxsw_i2c->pdata->irq; + else if (addr == MLXSW_FAST_I2C_SLAVE) + mlxsw_i2c->irq = MLXSW_I2C_DEFAULT_IRQ; + + if (!mlxsw_i2c->irq) + return 0; + + INIT_WORK(&mlxsw_i2c->irq_work, mlxsw_i2c_work_handler); + err = request_irq(mlxsw_i2c->irq, mlxsw_i2c_irq_handler, + IRQF_TRIGGER_FALLING | IRQF_SHARED, "mlxsw-i2c", + mlxsw_i2c); + if (err) { + dev_err(mlxsw_i2c->bus_info.dev, "Failed to request irq: %d\n", + err); + return err; + } + + return 0; +} + +static void mlxsw_i2c_irq_fini(struct mlxsw_i2c *mlxsw_i2c) +{ + if (!IS_REACHABLE(CONFIG_MLXREG_HOTPLUG) || !mlxsw_i2c->irq) + return; + cancel_work_sync(&mlxsw_i2c->irq_work); + free_irq(mlxsw_i2c->irq, mlxsw_i2c); +} + static const struct mlxsw_bus mlxsw_i2c_bus = { .kind = "i2c", .init = mlxsw_i2c_init, @@ -638,17 +715,24 @@ static int mlxsw_i2c_probe(struct i2c_client *client, mlxsw_i2c->bus_info.dev = &client->dev; mlxsw_i2c->bus_info.low_frequency = true; mlxsw_i2c->dev = &client->dev; + mlxsw_i2c->pdata = client->dev.platform_data; + + err = mlxsw_i2c_irq_init(mlxsw_i2c, client->addr); + if (err) + goto errout; err = mlxsw_core_bus_device_register(&mlxsw_i2c->bus_info, &mlxsw_i2c_bus, mlxsw_i2c, false, NULL, NULL); if (err) { dev_err(&client->dev, "Fail to register core bus\n"); - return err; + goto err_bus_device_register; } return 0; +err_bus_device_register: + mlxsw_i2c_irq_fini(mlxsw_i2c); errout: mutex_destroy(&mlxsw_i2c->cmd.lock); i2c_set_clientdata(client, NULL); @@ -656,14 +740,13 @@ errout: return err; } -static int mlxsw_i2c_remove(struct i2c_client *client) +static void mlxsw_i2c_remove(struct i2c_client *client) { struct mlxsw_i2c *mlxsw_i2c = i2c_get_clientdata(client); mlxsw_core_bus_device_unregister(mlxsw_i2c->core, false); + mlxsw_i2c_irq_fini(mlxsw_i2c); mutex_destroy(&mlxsw_i2c->cmd.lock); - - return 0; } int mlxsw_i2c_driver_register(struct i2c_driver *i2c_driver) diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index bb1cd4bae82eab56b8920ffb9e4658e07e6b61c0..55b3c42bb007199b63ee3c9b282156a55b5e46a2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -26,20 +26,29 @@ static const struct mlxsw_fw_rev mlxsw_m_fw_rev = { struct mlxsw_m_port; +struct mlxsw_m_line_card { + bool active; + int module_to_port[]; +}; + struct mlxsw_m { struct mlxsw_m_port **ports; - int *module_to_port; struct mlxsw_core *core; const struct mlxsw_bus_info *bus_info; u8 base_mac[ETH_ALEN]; u8 max_ports; + u8 max_modules_per_slot; /* Maximum number of modules per-slot. */ + u8 num_of_slots; /* Including the main board. */ + struct mlxsw_m_line_card **line_cards; }; struct mlxsw_m_port { struct net_device *dev; struct mlxsw_m *mlxsw_m; u16 local_port; + u8 slot_index; u8 module; + u8 module_offset; }; static int mlxsw_m_base_mac_get(struct mlxsw_m *mlxsw_m) @@ -94,14 +103,14 @@ static void mlxsw_m_module_get_drvinfo(struct net_device *dev, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev); struct mlxsw_m *mlxsw_m = mlxsw_m_port->mlxsw_m; - strlcpy(drvinfo->driver, mlxsw_m->bus_info->device_kind, + strscpy(drvinfo->driver, mlxsw_m->bus_info->device_kind, sizeof(drvinfo->driver)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", mlxsw_m->bus_info->fw_rev.major, mlxsw_m->bus_info->fw_rev.minor, mlxsw_m->bus_info->fw_rev.subminor); - strlcpy(drvinfo->bus_info, mlxsw_m->bus_info->device_name, + strscpy(drvinfo->bus_info, mlxsw_m->bus_info->device_name, sizeof(drvinfo->bus_info)); } @@ -111,8 +120,9 @@ static int mlxsw_m_get_module_info(struct net_device *netdev, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_get_module_info(netdev, core, 0, mlxsw_m_port->module, - modinfo); + return mlxsw_env_get_module_info(netdev, core, + mlxsw_m_port->slot_index, + mlxsw_m_port->module, modinfo); } static int @@ -122,7 +132,8 @@ mlxsw_m_get_module_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_get_module_eeprom(netdev, core, 0, + return mlxsw_env_get_module_eeprom(netdev, core, + mlxsw_m_port->slot_index, mlxsw_m_port->module, ee, data); } @@ -134,7 +145,8 @@ mlxsw_m_get_module_eeprom_by_page(struct net_device *netdev, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_get_module_eeprom_by_page(core, 0, + return mlxsw_env_get_module_eeprom_by_page(core, + mlxsw_m_port->slot_index, mlxsw_m_port->module, page, extack); } @@ -144,7 +156,8 @@ static int mlxsw_m_reset(struct net_device *netdev, u32 *flags) struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_reset_module(netdev, core, 0, mlxsw_m_port->module, + return mlxsw_env_reset_module(netdev, core, mlxsw_m_port->slot_index, + mlxsw_m_port->module, flags); } @@ -156,7 +169,8 @@ mlxsw_m_get_module_power_mode(struct net_device *netdev, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_get_module_power_mode(core, 0, mlxsw_m_port->module, + return mlxsw_env_get_module_power_mode(core, mlxsw_m_port->slot_index, + mlxsw_m_port->module, params, extack); } @@ -168,7 +182,8 @@ mlxsw_m_set_module_power_mode(struct net_device *netdev, struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; - return mlxsw_env_set_module_power_mode(core, 0, mlxsw_m_port->module, + return mlxsw_env_set_module_power_mode(core, mlxsw_m_port->slot_index, + mlxsw_m_port->module, params->policy, extack); } @@ -184,7 +199,7 @@ static const struct ethtool_ops mlxsw_m_port_ethtool_ops = { static int mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u16 local_port, - u8 *p_module, u8 *p_width) + u8 *p_module, u8 *p_width, u8 *p_slot_index) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; int err; @@ -195,6 +210,7 @@ mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u16 local_port, return err; *p_module = mlxsw_reg_pmlp_module_get(pmlp_pl, 0); *p_width = mlxsw_reg_pmlp_width_get(pmlp_pl); + *p_slot_index = mlxsw_reg_pmlp_slot_index_get(pmlp_pl, 0); return 0; } @@ -212,18 +228,25 @@ mlxsw_m_port_dev_addr_get(struct mlxsw_m_port *mlxsw_m_port) if (err) return err; mlxsw_reg_ppad_mac_memcpy_from(ppad_pl, addr); - eth_hw_addr_gen(mlxsw_m_port->dev, addr, mlxsw_m_port->module + 1); + eth_hw_addr_gen(mlxsw_m_port->dev, addr, mlxsw_m_port->module + 1 + + mlxsw_m_port->module_offset); return 0; } +static bool mlxsw_m_port_created(struct mlxsw_m *mlxsw_m, u16 local_port) +{ + return mlxsw_m->ports[local_port]; +} + static int -mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 module) +mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 slot_index, + u8 module) { struct mlxsw_m_port *mlxsw_m_port; struct net_device *dev; int err; - err = mlxsw_core_port_init(mlxsw_m->core, local_port, 0, + err = mlxsw_core_port_init(mlxsw_m->core, local_port, slot_index, module + 1, false, 0, false, 0, mlxsw_m->base_mac, sizeof(mlxsw_m->base_mac)); @@ -246,6 +269,15 @@ mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 module) mlxsw_m_port->mlxsw_m = mlxsw_m; mlxsw_m_port->local_port = local_port; mlxsw_m_port->module = module; + mlxsw_m_port->slot_index = slot_index; + /* Add module offset for line card. Offset for main board iz zero. + * For line card in slot #n offset is calculated as (#n - 1) + * multiplied by maximum modules number, which could be found on a line + * card. + */ + mlxsw_m_port->module_offset = mlxsw_m_port->slot_index ? + (mlxsw_m_port->slot_index - 1) * + mlxsw_m->max_modules_per_slot : 0; dev->netdev_ops = &mlxsw_m_port_netdev_ops; dev->ethtool_ops = &mlxsw_m_port_ethtool_ops; @@ -291,19 +323,29 @@ static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u16 local_port) mlxsw_core_port_fini(mlxsw_m->core, local_port); } +static int* +mlxsw_m_port_mapping_get(struct mlxsw_m *mlxsw_m, u8 slot_index, u8 module) +{ + return &mlxsw_m->line_cards[slot_index]->module_to_port[module]; +} + static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u16 local_port, u8 *last_module) { unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core); - u8 module, width; + u8 module, width, slot_index; + int *module_to_port; int err; /* Fill out to local port mapping array */ err = mlxsw_m_port_module_info_get(mlxsw_m, local_port, &module, - &width); + &width, &slot_index); if (err) return err; + /* Skip if line card has been already configured */ + if (mlxsw_m->line_cards[slot_index]->active) + return 0; if (!width) return 0; /* Skip, if port belongs to the cluster */ @@ -313,91 +355,220 @@ static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u16 local_port, if (WARN_ON_ONCE(module >= max_ports)) return -EINVAL; - mlxsw_env_module_port_map(mlxsw_m->core, 0, module); - mlxsw_m->module_to_port[module] = ++mlxsw_m->max_ports; + mlxsw_env_module_port_map(mlxsw_m->core, slot_index, module); + module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, module); + *module_to_port = local_port; return 0; } -static void mlxsw_m_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 module) +static void +mlxsw_m_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 slot_index, u8 module) { - mlxsw_m->module_to_port[module] = -1; - mlxsw_env_module_port_unmap(mlxsw_m->core, 0, module); + int *module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, + module); + *module_to_port = -1; + mlxsw_env_module_port_unmap(mlxsw_m->core, slot_index, module); } -static int mlxsw_m_ports_create(struct mlxsw_m *mlxsw_m) +static int mlxsw_m_linecards_init(struct mlxsw_m *mlxsw_m) { unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core); - u8 last_module = max_ports; - int i; - int err; + char mgpir_pl[MLXSW_REG_MGPIR_LEN]; + u8 num_of_modules; + int i, j, err; + + mlxsw_reg_mgpir_pack(mgpir_pl, 0); + err = mlxsw_reg_query(mlxsw_m->core, MLXSW_REG(mgpir), mgpir_pl); + if (err) + return err; + + mlxsw_reg_mgpir_unpack(mgpir_pl, NULL, NULL, NULL, &num_of_modules, + &mlxsw_m->num_of_slots); + /* If the system is modular, get the maximum number of modules per-slot. + * Otherwise, get the maximum number of modules on the main board. + */ + if (mlxsw_m->num_of_slots) + mlxsw_m->max_modules_per_slot = + mlxsw_reg_mgpir_max_modules_per_slot_get(mgpir_pl); + else + mlxsw_m->max_modules_per_slot = num_of_modules; + /* Add slot for main board. */ + mlxsw_m->num_of_slots += 1; mlxsw_m->ports = kcalloc(max_ports, sizeof(*mlxsw_m->ports), GFP_KERNEL); if (!mlxsw_m->ports) return -ENOMEM; - mlxsw_m->module_to_port = kmalloc_array(max_ports, sizeof(int), - GFP_KERNEL); - if (!mlxsw_m->module_to_port) { + mlxsw_m->line_cards = kcalloc(mlxsw_m->num_of_slots, + sizeof(*mlxsw_m->line_cards), + GFP_KERNEL); + if (!mlxsw_m->line_cards) { err = -ENOMEM; - goto err_module_to_port_alloc; + goto err_kcalloc; } - /* Invalidate the entries of module to local port mapping array */ - for (i = 0; i < max_ports; i++) - mlxsw_m->module_to_port[i] = -1; + for (i = 0; i < mlxsw_m->num_of_slots; i++) { + mlxsw_m->line_cards[i] = + kzalloc(struct_size(mlxsw_m->line_cards[i], + module_to_port, + mlxsw_m->max_modules_per_slot), + GFP_KERNEL); + if (!mlxsw_m->line_cards[i]) { + err = -ENOMEM; + goto err_kmalloc_array; + } - /* Fill out module to local port mapping array */ - for (i = 1; i < max_ports; i++) { - err = mlxsw_m_port_module_map(mlxsw_m, i, &last_module); - if (err) - goto err_module_to_port_map; + /* Invalidate the entries of module to local port mapping array. */ + for (j = 0; j < mlxsw_m->max_modules_per_slot; j++) + mlxsw_m->line_cards[i]->module_to_port[j] = -1; } - /* Create port objects for each valid entry */ - for (i = 0; i < mlxsw_m->max_ports; i++) { - if (mlxsw_m->module_to_port[i] > 0) { - err = mlxsw_m_port_create(mlxsw_m, - mlxsw_m->module_to_port[i], - i); + return 0; + +err_kmalloc_array: + for (i--; i >= 0; i--) + kfree(mlxsw_m->line_cards[i]); +err_kcalloc: + kfree(mlxsw_m->ports); + return err; +} + +static void mlxsw_m_linecards_fini(struct mlxsw_m *mlxsw_m) +{ + int i = mlxsw_m->num_of_slots; + + for (i--; i >= 0; i--) + kfree(mlxsw_m->line_cards[i]); + kfree(mlxsw_m->line_cards); + kfree(mlxsw_m->ports); +} + +static void +mlxsw_m_linecard_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 slot_index) +{ + int i; + + for (i = mlxsw_m->max_modules_per_slot - 1; i >= 0; i--) { + int *module_to_port; + + module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i); + if (*module_to_port > 0) + mlxsw_m_port_module_unmap(mlxsw_m, slot_index, i); + } +} + +static int +mlxsw_m_linecard_ports_create(struct mlxsw_m *mlxsw_m, u8 slot_index) +{ + int *module_to_port; + int i, err; + + for (i = 0; i < mlxsw_m->max_modules_per_slot; i++) { + module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i); + if (*module_to_port > 0) { + err = mlxsw_m_port_create(mlxsw_m, *module_to_port, + slot_index, i); if (err) - goto err_module_to_port_create; + goto err_port_create; + /* Mark slot as active */ + if (!mlxsw_m->line_cards[slot_index]->active) + mlxsw_m->line_cards[slot_index]->active = true; } } - return 0; -err_module_to_port_create: +err_port_create: for (i--; i >= 0; i--) { - if (mlxsw_m->module_to_port[i] > 0) - mlxsw_m_port_remove(mlxsw_m, - mlxsw_m->module_to_port[i]); + module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i); + if (*module_to_port > 0 && + mlxsw_m_port_created(mlxsw_m, *module_to_port)) { + mlxsw_m_port_remove(mlxsw_m, *module_to_port); + /* Mark slot as inactive */ + if (mlxsw_m->line_cards[slot_index]->active) + mlxsw_m->line_cards[slot_index]->active = false; + } } - i = max_ports; -err_module_to_port_map: - for (i--; i > 0; i--) - mlxsw_m_port_module_unmap(mlxsw_m, i); - kfree(mlxsw_m->module_to_port); -err_module_to_port_alloc: - kfree(mlxsw_m->ports); return err; } -static void mlxsw_m_ports_remove(struct mlxsw_m *mlxsw_m) +static void +mlxsw_m_linecard_ports_remove(struct mlxsw_m *mlxsw_m, u8 slot_index) { int i; - for (i = 0; i < mlxsw_m->max_ports; i++) { - if (mlxsw_m->module_to_port[i] > 0) { - mlxsw_m_port_remove(mlxsw_m, - mlxsw_m->module_to_port[i]); - mlxsw_m_port_module_unmap(mlxsw_m, i); + for (i = 0; i < mlxsw_m->max_modules_per_slot; i++) { + int *module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, + slot_index, i); + + if (*module_to_port > 0 && + mlxsw_m_port_created(mlxsw_m, *module_to_port)) { + mlxsw_m_port_remove(mlxsw_m, *module_to_port); + mlxsw_m_port_module_unmap(mlxsw_m, slot_index, i); } } +} - kfree(mlxsw_m->module_to_port); - kfree(mlxsw_m->ports); +static int mlxsw_m_ports_module_map(struct mlxsw_m *mlxsw_m) +{ + unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core); + u8 last_module = max_ports; + int i, err; + + for (i = 1; i < max_ports; i++) { + err = mlxsw_m_port_module_map(mlxsw_m, i, &last_module); + if (err) + return err; + } + + return 0; +} + +static int mlxsw_m_ports_create(struct mlxsw_m *mlxsw_m) +{ + int err; + + /* Fill out module to local port mapping array */ + err = mlxsw_m_ports_module_map(mlxsw_m); + if (err) + goto err_ports_module_map; + + /* Create port objects for each valid entry */ + err = mlxsw_m_linecard_ports_create(mlxsw_m, 0); + if (err) + goto err_linecard_ports_create; + + return 0; + +err_linecard_ports_create: +err_ports_module_map: + mlxsw_m_linecard_port_module_unmap(mlxsw_m, 0); + + return err; +} + +static void mlxsw_m_ports_remove(struct mlxsw_m *mlxsw_m) +{ + mlxsw_m_linecard_ports_remove(mlxsw_m, 0); +} + +static void +mlxsw_m_ports_remove_selected(struct mlxsw_core *mlxsw_core, + bool (*selector)(void *priv, u16 local_port), + void *priv) +{ + struct mlxsw_m *mlxsw_m = mlxsw_core_driver_priv(mlxsw_core); + struct mlxsw_linecard *linecard_priv = priv; + struct mlxsw_m_line_card *linecard; + + linecard = mlxsw_m->line_cards[linecard_priv->slot_index]; + + if (WARN_ON(!linecard->active)) + return; + + mlxsw_m_linecard_ports_remove(mlxsw_m, linecard_priv->slot_index); + linecard->active = false; } static int mlxsw_m_fw_rev_validate(struct mlxsw_m *mlxsw_m) @@ -418,6 +589,60 @@ static int mlxsw_m_fw_rev_validate(struct mlxsw_m *mlxsw_m) return -EINVAL; } +static void +mlxsw_m_got_active(struct mlxsw_core *mlxsw_core, u8 slot_index, void *priv) +{ + struct mlxsw_m_line_card *linecard; + struct mlxsw_m *mlxsw_m = priv; + int err; + + linecard = mlxsw_m->line_cards[slot_index]; + /* Skip if line card has been already configured during init */ + if (linecard->active) + return; + + /* Fill out module to local port mapping array */ + err = mlxsw_m_ports_module_map(mlxsw_m); + if (err) + goto err_ports_module_map; + + /* Create port objects for each valid entry */ + err = mlxsw_m_linecard_ports_create(mlxsw_m, slot_index); + if (err) { + dev_err(mlxsw_m->bus_info->dev, "Failed to create port for line card at slot %d\n", + slot_index); + goto err_linecard_ports_create; + } + + linecard->active = true; + + return; + +err_linecard_ports_create: +err_ports_module_map: + mlxsw_m_linecard_port_module_unmap(mlxsw_m, slot_index); +} + +static void +mlxsw_m_got_inactive(struct mlxsw_core *mlxsw_core, u8 slot_index, void *priv) +{ + struct mlxsw_m_line_card *linecard; + struct mlxsw_m *mlxsw_m = priv; + + linecard = mlxsw_m->line_cards[slot_index]; + + if (WARN_ON(!linecard->active)) + return; + + mlxsw_m_linecard_ports_remove(mlxsw_m, slot_index); + linecard->active = false; +} + +static struct mlxsw_linecards_event_ops mlxsw_m_event_ops = { + .got_active = mlxsw_m_got_active, + .got_inactive = mlxsw_m_got_inactive, +}; + static int mlxsw_m_init(struct mlxsw_core *mlxsw_core, const struct mlxsw_bus_info *mlxsw_bus_info, struct netlink_ext_ack *extack) @@ -438,13 +663,33 @@ static int mlxsw_m_init(struct mlxsw_core *mlxsw_core, return err; } + err = mlxsw_m_linecards_init(mlxsw_m); + if (err) { + dev_err(mlxsw_m->bus_info->dev, "Failed to create line cards\n"); + return err; + } + + err = mlxsw_linecards_event_ops_register(mlxsw_core, + &mlxsw_m_event_ops, mlxsw_m); + if (err) { + dev_err(mlxsw_m->bus_info->dev, "Failed to register line cards operations\n"); + goto linecards_event_ops_register; + } + err = mlxsw_m_ports_create(mlxsw_m); if (err) { dev_err(mlxsw_m->bus_info->dev, "Failed to create ports\n"); - return err; + goto err_ports_create; } return 0; + +err_ports_create: + mlxsw_linecards_event_ops_unregister(mlxsw_core, + &mlxsw_m_event_ops, mlxsw_m); +linecards_event_ops_register: + mlxsw_m_linecards_fini(mlxsw_m); + return err; } static void mlxsw_m_fini(struct mlxsw_core *mlxsw_core) @@ -452,6 +697,9 @@ static void mlxsw_m_fini(struct mlxsw_core *mlxsw_core) struct mlxsw_m *mlxsw_m = mlxsw_core_driver_priv(mlxsw_core); mlxsw_m_ports_remove(mlxsw_m); + mlxsw_linecards_event_ops_unregister(mlxsw_core, + &mlxsw_m_event_ops, mlxsw_m); + mlxsw_m_linecards_fini(mlxsw_m); } static const struct mlxsw_config_profile mlxsw_m_config_profile; @@ -461,6 +709,7 @@ static struct mlxsw_driver mlxsw_m_driver = { .priv_size = sizeof(struct mlxsw_m), .init = mlxsw_m_init, .fini = mlxsw_m_fini, + .ports_remove_selected = mlxsw_m_ports_remove_selected, .profile = &mlxsw_m_config_profile, }; diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 50527adc5b5ad6391c02c2ede3318b2351081941..c968309657dd124318df3a9c4db4d2c0304e0945 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1187,6 +1187,11 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox, mlxsw_cmd_mbox_config_profile_max_vepa_channels_set( mbox, profile->max_vepa_channels); } + if (profile->used_max_lag) { + mlxsw_cmd_mbox_config_profile_set_max_lag_set(mbox, 1); + mlxsw_cmd_mbox_config_profile_max_lag_set(mbox, + profile->max_lag); + } if (profile->used_max_mid) { mlxsw_cmd_mbox_config_profile_set_max_mid_set( mbox, 1); diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index f27bdecdf95219af959f4522e673c0a143636aad..0777bed5bb1af83c4cd2efaeef4226e947164685 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -2218,76 +2218,6 @@ static inline void mlxsw_reg_smpe_pack(char *payload, u16 local_port, mlxsw_reg_smpe_evid_set(payload, evid); } -/* SFTR-V2 - Switch Flooding Table Version 2 Register - * -------------------------------------------------- - * The switch flooding table is used for flooding packet replication. The table - * defines a bit mask of ports for packet replication. - */ -#define MLXSW_REG_SFTR2_ID 0x202F -#define MLXSW_REG_SFTR2_LEN 0x120 - -MLXSW_REG_DEFINE(sftr2, MLXSW_REG_SFTR2_ID, MLXSW_REG_SFTR2_LEN); - -/* reg_sftr2_swid - * Switch partition ID with which to associate the port. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr2, swid, 0x00, 24, 8); - -/* reg_sftr2_flood_table - * Flooding table index to associate with the specific type on the specific - * switch partition. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr2, flood_table, 0x00, 16, 6); - -/* reg_sftr2_index - * Index. Used as an index into the Flooding Table in case the table is - * configured to use VID / FID or FID Offset. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr2, index, 0x00, 0, 16); - -/* reg_sftr2_table_type - * See mlxsw_flood_table_type - * Access: RW - */ -MLXSW_ITEM32(reg, sftr2, table_type, 0x04, 16, 3); - -/* reg_sftr2_range - * Range of entries to update - * Access: Index - */ -MLXSW_ITEM32(reg, sftr2, range, 0x04, 0, 16); - -/* reg_sftr2_port - * Local port membership (1 bit per port). - * Access: RW - */ -MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port, 0x20, 0x80, 1); - -/* reg_sftr2_port_mask - * Local port mask (1 bit per port). - * Access: WO - */ -MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port_mask, 0xA0, 0x80, 1); - -static inline void mlxsw_reg_sftr2_pack(char *payload, - unsigned int flood_table, - unsigned int index, - enum mlxsw_flood_table_type table_type, - unsigned int range, u16 port, bool set) -{ - MLXSW_REG_ZERO(sftr2, payload); - mlxsw_reg_sftr2_swid_set(payload, 0); - mlxsw_reg_sftr2_flood_table_set(payload, flood_table); - mlxsw_reg_sftr2_index_set(payload, index); - mlxsw_reg_sftr2_table_type_set(payload, table_type); - mlxsw_reg_sftr2_range_set(payload, range); - mlxsw_reg_sftr2_port_set(payload, port, set); - mlxsw_reg_sftr2_port_mask_set(payload, port, 1); -} - /* SMID-V2 - Switch Multicast ID Version 2 Register * ------------------------------------------------ * The MID record maps from a MID (Multicast ID), which is a unique identifier @@ -4729,25 +4659,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_cap, 0x08, 0, 32); */ MLXSW_ITEM32(reg, ptys, eth_proto_cap, 0x0C, 0, 32); -/* reg_ptys_ib_link_width_cap - * IB port supported widths. - * Access: RO - */ -MLXSW_ITEM32(reg, ptys, ib_link_width_cap, 0x10, 16, 16); - -#define MLXSW_REG_PTYS_IB_SPEED_SDR BIT(0) -#define MLXSW_REG_PTYS_IB_SPEED_DDR BIT(1) -#define MLXSW_REG_PTYS_IB_SPEED_QDR BIT(2) -#define MLXSW_REG_PTYS_IB_SPEED_FDR10 BIT(3) -#define MLXSW_REG_PTYS_IB_SPEED_FDR BIT(4) -#define MLXSW_REG_PTYS_IB_SPEED_EDR BIT(5) - -/* reg_ptys_ib_proto_cap - * IB port supported speeds and protocols. - * Access: RO - */ -MLXSW_ITEM32(reg, ptys, ib_proto_cap, 0x10, 0, 16); - /* reg_ptys_ext_eth_proto_admin * Extended speed and protocol to set port to. * Access: RW @@ -4760,18 +4671,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_admin, 0x14, 0, 32); */ MLXSW_ITEM32(reg, ptys, eth_proto_admin, 0x18, 0, 32); -/* reg_ptys_ib_link_width_admin - * IB width to set port to. - * Access: RW - */ -MLXSW_ITEM32(reg, ptys, ib_link_width_admin, 0x1C, 16, 16); - -/* reg_ptys_ib_proto_admin - * IB speeds and protocols to set port to. - * Access: RW - */ -MLXSW_ITEM32(reg, ptys, ib_proto_admin, 0x1C, 0, 16); - /* reg_ptys_ext_eth_proto_oper * The extended current speed and protocol configured for the port. * Access: RO @@ -4784,18 +4683,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_oper, 0x20, 0, 32); */ MLXSW_ITEM32(reg, ptys, eth_proto_oper, 0x24, 0, 32); -/* reg_ptys_ib_link_width_oper - * The current IB width to set port to. - * Access: RO - */ -MLXSW_ITEM32(reg, ptys, ib_link_width_oper, 0x28, 16, 16); - -/* reg_ptys_ib_proto_oper - * The current IB speed and protocol. - * Access: RO - */ -MLXSW_ITEM32(reg, ptys, ib_proto_oper, 0x28, 0, 16); - enum mlxsw_reg_ptys_connector_type { MLXSW_REG_PTYS_CONNECTOR_TYPE_UNKNOWN_OR_NO_CONNECTOR, MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_NONE, @@ -4866,33 +4753,6 @@ static inline void mlxsw_reg_ptys_ext_eth_unpack(char *payload, mlxsw_reg_ptys_ext_eth_proto_oper_get(payload); } -static inline void mlxsw_reg_ptys_ib_pack(char *payload, u16 local_port, - u16 proto_admin, u16 link_width) -{ - MLXSW_REG_ZERO(ptys, payload); - mlxsw_reg_ptys_local_port_set(payload, local_port); - mlxsw_reg_ptys_proto_mask_set(payload, MLXSW_REG_PTYS_PROTO_MASK_IB); - mlxsw_reg_ptys_ib_proto_admin_set(payload, proto_admin); - mlxsw_reg_ptys_ib_link_width_admin_set(payload, link_width); -} - -static inline void mlxsw_reg_ptys_ib_unpack(char *payload, u16 *p_ib_proto_cap, - u16 *p_ib_link_width_cap, - u16 *p_ib_proto_oper, - u16 *p_ib_link_width_oper) -{ - if (p_ib_proto_cap) - *p_ib_proto_cap = mlxsw_reg_ptys_ib_proto_cap_get(payload); - if (p_ib_link_width_cap) - *p_ib_link_width_cap = - mlxsw_reg_ptys_ib_link_width_cap_get(payload); - if (p_ib_proto_oper) - *p_ib_proto_oper = mlxsw_reg_ptys_ib_proto_oper_get(payload); - if (p_ib_link_width_oper) - *p_ib_link_width_oper = - mlxsw_reg_ptys_ib_link_width_oper_get(payload); -} - /* PPAD - Port Physical Address Register * ------------------------------------- * The PPAD register configures the per port physical MAC address. @@ -5666,27 +5526,6 @@ static inline void mlxsw_reg_ppcnt_pack(char *payload, u16 local_port, mlxsw_reg_ppcnt_prio_tc_set(payload, prio_tc); } -/* PLIB - Port Local to InfiniBand Port - * ------------------------------------ - * The PLIB register performs mapping from Local Port into InfiniBand Port. - */ -#define MLXSW_REG_PLIB_ID 0x500A -#define MLXSW_REG_PLIB_LEN 0x10 - -MLXSW_REG_DEFINE(plib, MLXSW_REG_PLIB_ID, MLXSW_REG_PLIB_LEN); - -/* reg_plib_local_port - * Local port number. - * Access: Index - */ -MLXSW_ITEM32_LP(reg, plib, 0x00, 16, 0x00, 12); - -/* reg_plib_ib_port - * InfiniBand port remapping for local_port. - * Access: RW - */ -MLXSW_ITEM32(reg, plib, ib_port, 0x00, 0, 8); - /* PPTB - Port Prio To Buffer Register * ----------------------------------- * Configures the switch priority to buffer table. @@ -12924,7 +12763,6 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(spvc), MLXSW_REG(spevet), MLXSW_REG(smpe), - MLXSW_REG(sftr2), MLXSW_REG(smid2), MLXSW_REG(cwtp), MLXSW_REG(cwtpm), @@ -12962,7 +12800,6 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(paos), MLXSW_REG(pfcc), MLXSW_REG(ppcnt), - MLXSW_REG(plib), MLXSW_REG(pptb), MLXSW_REG(pbmc), MLXSW_REG(pspa), diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 30c7b0e1572181b76e87e3597a28e7652ed9d4aa..5bcf5bceff7107d0a1d714d65a6b5bf9c2b3b3ba 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -2691,6 +2691,7 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp) static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) { char slcr_pl[MLXSW_REG_SLCR_LEN]; + u16 max_lag; u32 seed; int err; @@ -2709,12 +2710,14 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) if (err) return err; - if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG) || - !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS)) + err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); + if (err) + return err; + + if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS)) return -EIO; - mlxsw_sp->lags = kcalloc(MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG), - sizeof(struct mlxsw_sp_upper), + mlxsw_sp->lags = kcalloc(max_lag, sizeof(struct mlxsw_sp_upper), GFP_KERNEL); if (!mlxsw_sp->lags) return -ENOMEM; @@ -3509,6 +3512,33 @@ static const struct mlxsw_config_profile mlxsw_sp2_config_profile = { .cqe_time_stamp_type = MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, }; +/* Reduce number of LAGs from full capacity (256) to the maximum supported LAGs + * in Spectrum-2/3, to avoid regression in number of free entries in the PGT + * table. + */ +#define MLXSW_SP4_CONFIG_PROFILE_MAX_LAG 128 + +static const struct mlxsw_config_profile mlxsw_sp4_config_profile = { + .used_max_lag = 1, + .max_lag = MLXSW_SP4_CONFIG_PROFILE_MAX_LAG, + .used_flood_mode = 1, + .flood_mode = MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CONTROLLED, + .used_max_ib_mc = 1, + .max_ib_mc = 0, + .used_max_pkey = 1, + .max_pkey = 0, + .used_ubridge = 1, + .ubridge = 1, + .swid_config = { + { + .used_type = 1, + .type = MLXSW_PORT_SWID_TYPE_ETH, + } + }, + .used_cqe_time_stamp_type = 1, + .cqe_time_stamp_type = MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, +}; + static void mlxsw_sp_resource_size_params_prepare(struct mlxsw_core *mlxsw_core, struct devlink_resource_size_params *kvd_size_params, @@ -4039,7 +4069,7 @@ static struct mlxsw_driver mlxsw_sp4_driver = { .params_unregister = mlxsw_sp2_params_unregister, .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, - .profile = &mlxsw_sp2_config_profile, + .profile = &mlxsw_sp4_config_profile, .sdq_supports_cqe_v2 = true, }; @@ -4263,10 +4293,13 @@ static int mlxsw_sp_lag_index_get(struct mlxsw_sp *mlxsw_sp, { struct mlxsw_sp_upper *lag; int free_lag_id = -1; - u64 max_lag; - int i; + u16 max_lag; + int err, i; + + err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); + if (err) + return err; - max_lag = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG); for (i = 0; i < max_lag; i++) { lag = mlxsw_sp_lag_get(mlxsw_sp, i); if (lag->ref_count) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c index 915dffb85a1c580ca6196396bb5034935258e115..dcd79d7e2af4c0b7c9a6cdbb787af1c1a67a6846 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c @@ -14,16 +14,16 @@ static void mlxsw_sp_port_get_drvinfo(struct net_device *dev, struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - strlcpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind, + strscpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, mlxsw_sp_driver_version, + strscpy(drvinfo->version, mlxsw_sp_driver_version, sizeof(drvinfo->version)); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", mlxsw_sp->bus_info->fw_rev.major, mlxsw_sp->bus_info->fw_rev.minor, mlxsw_sp->bus_info->fw_rev.subminor); - strlcpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name, + strscpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name, sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 2c4443c6b964c606d5017395ce15f66c506cf039..48f1fa62a4fd492e68018e918afc193181ce215b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -1819,7 +1819,7 @@ void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, /* The configuration where several tunnels have the same local address in the * same underlay table needs special treatment in the HW. That is currently not * implemented in the driver. This function finds and demotes the first tunnel - * with a given source address, except the one passed in in the argument + * with a given source address, except the one passed in the argument * `except'. */ bool diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index 39904dacf4f0d6cec38acb49c16caa1314d1a857..b3472fb946177778b47ac29fe738dd5029b35daf 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -423,7 +423,8 @@ mlxsw_sp_span_gretap4_route(const struct net_device *to_dev, parms = mlxsw_sp_ipip_netdev_parms4(to_dev); ip_tunnel_init_flow(&fl4, parms.iph.protocol, *daddrp, *saddrp, - 0, 0, dev_net(to_dev), parms.link, tun->fwmark, 0); + 0, 0, dev_net(to_dev), parms.link, tun->fwmark, 0, + 0); rt = ip_route_output_key(tun->net, &fl4); if (IS_ERR(rt)) diff --git a/drivers/net/ethernet/micrel/ks8851.h b/drivers/net/ethernet/micrel/ks8851.h index 6f34a61739b6622d86df007767374193528859f9..fecd43754cead10662344962d0e63e359ee27b20 100644 --- a/drivers/net/ethernet/micrel/ks8851.h +++ b/drivers/net/ethernet/micrel/ks8851.h @@ -403,7 +403,7 @@ struct ks8851_net { struct eeprom_93cx6 eeprom; struct regulator *vdd_reg; struct regulator *vdd_io; - int gpio; + struct gpio_desc *gpio; struct mii_bus *mii_bus; void (*lock)(struct ks8851_net *ks, diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 691206f19ea7bc43df299a8a6409ab9c5bcbd37f..cfbc900d4aeb9ef7763230d7eca9256912c8a00b 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -17,10 +17,9 @@ #include #include #include +#include #include -#include -#include #include #include @@ -703,9 +702,9 @@ static const struct net_device_ops ks8851_netdev_ops = { static void ks8851_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *di) { - strlcpy(di->driver, "KS8851", sizeof(di->driver)); - strlcpy(di->version, "1.00", sizeof(di->version)); - strlcpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info)); + strscpy(di->driver, "KS8851", sizeof(di->driver)); + strscpy(di->version, "1.00", sizeof(di->version)); + strscpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info)); } static u32 ks8851_get_msglevel(struct net_device *dev) @@ -1117,24 +1116,23 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev, { struct ks8851_net *ks = netdev_priv(netdev); unsigned cider; - int gpio; int ret; ks->netdev = netdev; ks->tx_space = 6144; - gpio = of_get_named_gpio_flags(dev->of_node, "reset-gpios", 0, NULL); - if (gpio == -EPROBE_DEFER) - return gpio; - - ks->gpio = gpio; - if (gpio_is_valid(gpio)) { - ret = devm_gpio_request_one(dev, gpio, - GPIOF_OUT_INIT_LOW, "ks8851_rst_n"); - if (ret) { - dev_err(dev, "reset gpio request failed\n"); - return ret; - } + ks->gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + ret = PTR_ERR_OR_ZERO(ks->gpio); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "reset gpio request failed: %d\n", ret); + return ret; + } + + ret = gpiod_set_consumer_name(ks->gpio, "ks8851_rst_n"); + if (ret) { + dev_err(dev, "failed to set reset gpio name: %d\n", ret); + return ret; } ks->vdd_io = devm_regulator_get(dev, "vdd-io"); @@ -1161,9 +1159,9 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev, goto err_reg; } - if (gpio_is_valid(gpio)) { + if (ks->gpio) { usleep_range(10000, 11000); - gpio_set_value(gpio, 1); + gpiod_set_value_cansleep(ks->gpio, 0); } spin_lock_init(&ks->statelock); @@ -1239,8 +1237,8 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev, err_id: ks8851_unregister_mdiobus(ks); err_mdio: - if (gpio_is_valid(gpio)) - gpio_set_value(gpio, 0); + if (ks->gpio) + gpiod_set_value_cansleep(ks->gpio, 1); regulator_disable(ks->vdd_reg); err_reg: regulator_disable(ks->vdd_io); @@ -1259,8 +1257,8 @@ void ks8851_remove_common(struct device *dev) dev_info(dev, "remove\n"); unregister_netdev(priv->netdev); - if (gpio_is_valid(priv->gpio)) - gpio_set_value(priv->gpio, 0); + if (priv->gpio) + gpiod_set_value_cansleep(priv->gpio, 1); regulator_disable(priv->vdd_reg); regulator_disable(priv->vdd_io); } diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c index 82d55fc27edc67d6c89623665dd243fa923dbec8..70bc7253454f6b6b2794684b2a9fdd9b430bc279 100644 --- a/drivers/net/ethernet/micrel/ks8851_spi.c +++ b/drivers/net/ethernet/micrel/ks8851_spi.c @@ -413,7 +413,8 @@ static int ks8851_probe_spi(struct spi_device *spi) spi->bits_per_word = 8; - ks = netdev_priv(netdev); + kss = netdev_priv(netdev); + ks = &kss->ks8851; ks->lock = ks8851_lock_spi; ks->unlock = ks8851_unlock_spi; @@ -433,8 +434,6 @@ static int ks8851_probe_spi(struct spi_device *spi) IRQ_RXPSI) /* RX process stop */ ks->rc_ier = STD_IRQ; - kss = to_ks8851_spi(ks); - kss->spidev = spi; mutex_init(&kss->lock); INIT_WORK(&kss->tx_work, ks8851_tx_work); diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index 2b3eb5ed823385fc22bfba6b4c82aee3de842d0b..468520079c65e5c10200b3bd29ba89c256353a90 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -5998,9 +5998,9 @@ static void netdev_get_drvinfo(struct net_device *dev, struct dev_priv *priv = netdev_priv(dev); struct dev_info *hw_priv = priv->adapter; - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(hw_priv->pdev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(hw_priv->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/microchip/enc28j60.c b/drivers/net/ethernet/microchip/enc28j60.c index 559ad94a44d03f4cfb9b2b72a53664af47aa933d..176efbeae12735712835e48da7baed4ee7c8edc6 100644 --- a/drivers/net/ethernet/microchip/enc28j60.c +++ b/drivers/net/ethernet/microchip/enc28j60.c @@ -1467,9 +1467,9 @@ static void enc28j60_restart_work_handler(struct work_struct *work) static void enc28j60_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/microchip/encx24j600.c b/drivers/net/ethernet/microchip/encx24j600.c index dc1840cb5b10fe12b05980b6dcbc94eecafc9b3d..d7c8aa77ec75c328c962b9cc444919b9a86b4d3e 100644 --- a/drivers/net/ethernet/microchip/encx24j600.c +++ b/drivers/net/ethernet/microchip/encx24j600.c @@ -925,9 +925,9 @@ static void encx24j600_get_regs(struct net_device *dev, static void encx24j600_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(dev->dev.parent), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index b1c74e6cb01259d7cdea675bacee15499d8809b4..c739d60ee17d5cfc0975d07ecf704eb5eb2ac559 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -579,8 +579,8 @@ static void lan743x_ethtool_get_drvinfo(struct net_device *netdev, { struct lan743x_adapter *adapter = netdev_priv(netdev); - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(adapter->pdev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index a9a1dea6d7311104ed2d9d2fc4eaea43a5838e4d..50eeecba1f18d1586c77642437f4a7f8f12585d5 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -1585,6 +1585,9 @@ static void lan743x_rfe_set_multicast(struct lan743x_adapter *adapter) rfctl |= RFE_CTL_AM_; } + if (netdev->features & NETIF_F_RXCSUM) + rfctl |= RFE_CTL_IP_COE_ | RFE_CTL_TCP_UDP_COE_; + memset(hash_table, 0, DP_SEL_VHF_HASH_LEN * sizeof(u32)); if (netdev_mc_count(netdev)) { struct netdev_hw_addr *ha; @@ -2066,11 +2069,13 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, { int required_number_of_descriptors = 0; unsigned int start_frame_length = 0; + netdev_tx_t retval = NETDEV_TX_OK; unsigned int frame_length = 0; unsigned int head_length = 0; unsigned long irq_flags = 0; bool do_timestamp = false; bool ignore_sync = false; + struct netdev_queue *txq; int nr_frags = 0; bool gso = false; int j; @@ -2083,9 +2088,12 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx, if (required_number_of_descriptors > (tx->ring_size - 1)) { dev_kfree_skb_irq(skb); } else { - /* save to overflow buffer */ - tx->overflow_skb = skb; - netif_stop_queue(tx->adapter->netdev); + /* save how many descriptors we needed to restart the queue */ + tx->rqd_descriptors = required_number_of_descriptors; + retval = NETDEV_TX_BUSY; + txq = netdev_get_tx_queue(tx->adapter->netdev, + tx->channel_number); + netif_tx_stop_queue(txq); } goto unlock; } @@ -2144,15 +2152,15 @@ finish: unlock: spin_unlock_irqrestore(&tx->ring_lock, irq_flags); - return NETDEV_TX_OK; + return retval; } static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight) { struct lan743x_tx *tx = container_of(napi, struct lan743x_tx, napi); struct lan743x_adapter *adapter = tx->adapter; - bool start_transmitter = false; unsigned long irq_flags = 0; + struct netdev_queue *txq; u32 ioc_bit = 0; ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number); @@ -2163,24 +2171,20 @@ static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight) /* clean up tx ring */ lan743x_tx_release_completed_descriptors(tx); - if (netif_queue_stopped(adapter->netdev)) { - if (tx->overflow_skb) { - if (lan743x_tx_get_desc_cnt(tx, tx->overflow_skb) <= - lan743x_tx_get_avail_desc(tx)) - start_transmitter = true; + txq = netdev_get_tx_queue(adapter->netdev, tx->channel_number); + if (netif_tx_queue_stopped(txq)) { + if (tx->rqd_descriptors) { + if (tx->rqd_descriptors <= + lan743x_tx_get_avail_desc(tx)) { + tx->rqd_descriptors = 0; + netif_tx_wake_queue(txq); + } } else { - netif_wake_queue(adapter->netdev); + netif_tx_wake_queue(txq); } } spin_unlock_irqrestore(&tx->ring_lock, irq_flags); - if (start_transmitter) { - /* space is now available, transmit overflow skb */ - lan743x_tx_xmit_frame(tx, tx->overflow_skb); - tx->overflow_skb = NULL; - netif_wake_queue(adapter->netdev); - } - if (!napi_complete(napi)) goto done; @@ -2304,10 +2308,7 @@ static void lan743x_tx_close(struct lan743x_tx *tx) lan743x_tx_release_all_descriptors(tx); - if (tx->overflow_skb) { - dev_kfree_skb(tx->overflow_skb); - tx->overflow_skb = NULL; - } + tx->rqd_descriptors = 0; lan743x_tx_ring_cleanup(tx); } @@ -2387,7 +2388,7 @@ static int lan743x_tx_open(struct lan743x_tx *tx) (tx->channel_number)); netif_napi_add_tx_weight(adapter->netdev, &tx->napi, lan743x_tx_napi_poll, - tx->ring_size - 1); + NAPI_POLL_WEIGHT); napi_enable(&tx->napi); data = 0; @@ -2549,6 +2550,7 @@ static int lan743x_rx_process_buffer(struct lan743x_rx *rx) int result = RX_PROCESS_RESULT_NOTHING_TO_DO; struct lan743x_rx_buffer_info *buffer_info; int frame_length, buffer_length; + bool is_ice, is_tce, is_icsm; int extension_index = -1; bool is_last, is_first; struct sk_buff *skb; @@ -2595,6 +2597,9 @@ static int lan743x_rx_process_buffer(struct lan743x_rx *rx) frame_length = RX_DESC_DATA0_FRAME_LENGTH_GET_(le32_to_cpu(descriptor->data0)); buffer_length = buffer_info->buffer_length; + is_ice = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICE_; + is_tce = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_TCE_; + is_icsm = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICSM_; netdev_dbg(netdev, "%s%schunk: %d/%d", is_first ? "first " : " ", @@ -2663,6 +2668,10 @@ process_extension: if (is_last && rx->skb_head) { rx->skb_head->protocol = eth_type_trans(rx->skb_head, rx->adapter->netdev); + if (rx->adapter->netdev->features & NETIF_F_RXCSUM) { + if (!is_ice && !is_tce && !is_icsm) + skb->ip_summed = CHECKSUM_UNNECESSARY; + } netdev_dbg(netdev, "sending %d byte frame to OS", rx->skb_head->len); napi_gro_receive(&rx->napi, rx->skb_head); @@ -2866,9 +2875,7 @@ static int lan743x_rx_open(struct lan743x_rx *rx) if (ret) goto return_error; - netif_napi_add(adapter->netdev, - &rx->napi, lan743x_rx_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(adapter->netdev, &rx->napi, lan743x_rx_napi_poll); lan743x_csr_write(adapter, DMAC_CMD, DMAC_CMD_RX_SWR_(rx->channel_number)); @@ -3347,8 +3354,10 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev, PCI11X1X_USED_TX_CHANNELS, LAN743X_USED_RX_CHANNELS); } else { - netdev = devm_alloc_etherdev(&pdev->dev, - sizeof(struct lan743x_adapter)); + netdev = devm_alloc_etherdev_mqs(&pdev->dev, + sizeof(struct lan743x_adapter), + LAN743X_USED_TX_CHANNELS, + LAN743X_USED_RX_CHANNELS); } if (!netdev) @@ -3383,7 +3392,8 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev, adapter->netdev->netdev_ops = &lan743x_netdev_ops; adapter->netdev->ethtool_ops = &lan743x_ethtool_ops; - adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_CSUM; + adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO | + NETIF_F_HW_CSUM | NETIF_F_RXCSUM; adapter->netdev->hw_features = adapter->netdev->features; /* carrier off reporting is important to ethtool even BEFORE open */ diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 72adae4f2aa08568d69b7a784b42c3fd8ed79c22..67877d3b6dd9842308401063e20ffc00aeea4df7 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -266,6 +266,8 @@ #define RFE_ADDR_FILT_LO(x) (0x404 + (8 * (x))) #define RFE_CTL (0x508) +#define RFE_CTL_TCP_UDP_COE_ BIT(12) +#define RFE_CTL_IP_COE_ BIT(11) #define RFE_CTL_AB_ BIT(10) #define RFE_CTL_AM_ BIT(9) #define RFE_CTL_AU_ BIT(8) @@ -954,8 +956,7 @@ struct lan743x_tx { struct napi_struct napi; u32 frame_count; - - struct sk_buff *overflow_skb; + u32 rqd_descriptors; }; void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx, @@ -1110,7 +1111,7 @@ struct lan743x_tx_buffer_info { unsigned int buffer_length; }; -#define LAN743X_TX_RING_SIZE (50) +#define LAN743X_TX_RING_SIZE (128) /* OWN bit is set. ie, Descs are owned by RX DMAC */ #define RX_DESC_DATA0_OWN_ (0x00008000) @@ -1122,6 +1123,9 @@ struct lan743x_tx_buffer_info { (((data0) & RX_DESC_DATA0_FRAME_LENGTH_MASK_) >> 16) #define RX_DESC_DATA0_EXT_ (0x00004000) #define RX_DESC_DATA0_BUF_LENGTH_MASK_ (0x00003FFF) +#define RX_DESC_DATA1_STATUS_ICE_ (0x00020000) +#define RX_DESC_DATA1_STATUS_TCE_ (0x00010000) +#define RX_DESC_DATA1_STATUS_ICSM_ (0x00000001) #define RX_DESC_DATA2_TS_NS_MASK_ (0x3FFFFFFF) #if ((NET_IP_ALIGN != 0) && (NET_IP_ALIGN != 2)) diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.c b/drivers/net/ethernet/microchip/lan743x_ptp.c index 6a11e2ceb013b65e0043fc4137b599d60c292bdc..da3ea905adbb866ed7464393ff6f6e33ffaf5036 100644 --- a/drivers/net/ethernet/microchip/lan743x_ptp.c +++ b/drivers/net/ethernet/microchip/lan743x_ptp.c @@ -1049,6 +1049,10 @@ static int lan743x_ptpci_verify_pin_config(struct ptp_clock_info *ptp, enum ptp_pin_function func, unsigned int chan) { + struct lan743x_ptp *lan_ptp = + container_of(ptp, struct lan743x_ptp, ptp_clock_info); + struct lan743x_adapter *adapter = + container_of(lan_ptp, struct lan743x_adapter, ptp); int result = 0; /* Confirm the requested function is supported. Parameter @@ -1057,7 +1061,10 @@ static int lan743x_ptpci_verify_pin_config(struct ptp_clock_info *ptp, switch (func) { case PTP_PF_NONE: case PTP_PF_PEROUT: + break; case PTP_PF_EXTTS: + if (!adapter->is_pci11x1x) + result = -1; break; case PTP_PF_PHYSYNC: default: diff --git a/drivers/net/ethernet/microchip/lan966x/Kconfig b/drivers/net/ethernet/microchip/lan966x/Kconfig index 4241ff0e509846d97b9509944ad7906290f8cae3..49e1464a431392711c72040b6b9a8e9d01722a68 100644 --- a/drivers/net/ethernet/microchip/lan966x/Kconfig +++ b/drivers/net/ethernet/microchip/lan966x/Kconfig @@ -4,6 +4,7 @@ config LAN966X_SWITCH depends on HAS_IOMEM depends on OF depends on NET_SWITCHDEV + depends on BRIDGE || BRIDGE=n select PHYLINK select PACKING help diff --git a/drivers/net/ethernet/microchip/lan966x/Makefile b/drivers/net/ethernet/microchip/lan966x/Makefile index fd2e0ebb2427ad2f40ef3746c23ff74a493c3b8b..962f7c5f9e7dd7ecb099f859e64ddaabd8c3a2bb 100644 --- a/drivers/net/ethernet/microchip/lan966x/Makefile +++ b/drivers/net/ethernet/microchip/lan966x/Makefile @@ -8,4 +8,7 @@ obj-$(CONFIG_LAN966X_SWITCH) += lan966x-switch.o lan966x-switch-objs := lan966x_main.o lan966x_phylink.o lan966x_port.o \ lan966x_mac.o lan966x_ethtool.o lan966x_switchdev.o \ lan966x_vlan.o lan966x_fdb.o lan966x_mdb.o \ - lan966x_ptp.o lan966x_fdma.o + lan966x_ptp.o lan966x_fdma.o lan966x_lag.o \ + lan966x_tc.o lan966x_mqprio.o lan966x_taprio.o \ + lan966x_tbf.o lan966x_cbs.o lan966x_ets.o \ + lan966x_tc_matchall.o lan966x_police.o lan966x_mirror.o diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c b/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c new file mode 100644 index 0000000000000000000000000000000000000000..70cbbf8d2b67b685fb81dcd3a8d67e251819cd7a --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +int lan966x_cbs_add(struct lan966x_port *port, + struct tc_cbs_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + u32 cir, cbs; + u8 se_idx; + + /* Check for invalid values */ + if (qopt->idleslope <= 0 || + qopt->sendslope >= 0 || + qopt->locredit >= qopt->hicredit) + return -EINVAL; + + se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + qopt->queue; + cir = qopt->idleslope; + cbs = (qopt->idleslope - qopt->sendslope) * + (qopt->hicredit - qopt->locredit) / + -qopt->sendslope; + + /* Rate unit is 100 kbps */ + cir = DIV_ROUND_UP(cir, 100); + /* Avoid using zero rate */ + cir = cir ?: 1; + /* Burst unit is 4kB */ + cbs = DIV_ROUND_UP(cbs, 4096); + /* Avoid using zero burst */ + cbs = cbs ?: 1; + + /* Check that actually the result can be written */ + if (cir > GENMASK(15, 0) || + cbs > GENMASK(6, 0)) + return -EINVAL; + + lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(1) | + QSYS_SE_CFG_SE_FRM_MODE_SET(1), + QSYS_SE_CFG_SE_AVB_ENA | + QSYS_SE_CFG_SE_FRM_MODE, + lan966x, QSYS_SE_CFG(se_idx)); + + lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(cir) | + QSYS_CIR_CFG_CIR_BURST_SET(cbs), + lan966x, QSYS_CIR_CFG(se_idx)); + + return 0; +} + +int lan966x_cbs_del(struct lan966x_port *port, + struct tc_cbs_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + u8 se_idx; + + se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + qopt->queue; + + lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(1) | + QSYS_SE_CFG_SE_FRM_MODE_SET(0), + QSYS_SE_CFG_SE_AVB_ENA | + QSYS_SE_CFG_SE_FRM_MODE, + lan966x, QSYS_SE_CFG(se_idx)); + + lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(0) | + QSYS_CIR_CFG_CIR_BURST_SET(0), + lan966x, QSYS_CIR_CFG(se_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c new file mode 100644 index 0000000000000000000000000000000000000000..8310d3f35404e4ae4444619b65005d6f70184dc7 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +#define DWRR_COST_BIT_WIDTH BIT(5) + +static u32 lan966x_ets_hw_cost(u32 w_min, u32 weight) +{ + u32 res; + + /* Round half up: Multiply with 16 before division, + * add 8 and divide result with 16 again + */ + res = (((DWRR_COST_BIT_WIDTH << 4) * w_min / weight) + 8) >> 4; + return max_t(u32, 1, res) - 1; +} + +int lan966x_ets_add(struct lan966x_port *port, + struct tc_ets_qopt_offload *qopt) +{ + struct tc_ets_qopt_offload_replace_params *params; + struct lan966x *lan966x = port->lan966x; + u32 w_min = 100; + u8 count = 0; + u32 se_idx; + u8 i; + + /* Check the input */ + if (qopt->parent != TC_H_ROOT) + return -EINVAL; + + params = &qopt->replace_params; + if (params->bands != NUM_PRIO_QUEUES) + return -EINVAL; + + for (i = 0; i < params->bands; ++i) { + /* In the switch the DWRR is always on the lowest consecutive + * priorities. Due to this, the first priority must map to the + * first DWRR band. + */ + if (params->priomap[i] != (7 - i)) + return -EINVAL; + + if (params->quanta[i] && params->weights[i] == 0) + return -EINVAL; + } + + se_idx = SE_IDX_PORT + port->chip_port; + + /* Find minimum weight */ + for (i = 0; i < params->bands; ++i) { + if (params->quanta[i] == 0) + continue; + + w_min = min(w_min, params->weights[i]); + } + + for (i = 0; i < params->bands; ++i) { + if (params->quanta[i] == 0) + continue; + + ++count; + + lan_wr(lan966x_ets_hw_cost(w_min, params->weights[i]), + lan966x, QSYS_SE_DWRR_CFG(se_idx, 7 - i)); + } + + lan_rmw(QSYS_SE_CFG_SE_DWRR_CNT_SET(count) | + QSYS_SE_CFG_SE_RR_ENA_SET(0), + QSYS_SE_CFG_SE_DWRR_CNT | + QSYS_SE_CFG_SE_RR_ENA, + lan966x, QSYS_SE_CFG(se_idx)); + + return 0; +} + +int lan966x_ets_del(struct lan966x_port *port, + struct tc_ets_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + u32 se_idx; + int i; + + se_idx = SE_IDX_PORT + port->chip_port; + + for (i = 0; i < NUM_PRIO_QUEUES; ++i) + lan_wr(0, lan966x, QSYS_SE_DWRR_CFG(se_idx, i)); + + lan_rmw(QSYS_SE_CFG_SE_DWRR_CNT_SET(0) | + QSYS_SE_CFG_SE_RR_ENA_SET(0), + QSYS_SE_CFG_SE_DWRR_CNT | + QSYS_SE_CFG_SE_RR_ENA, + lan966x, QSYS_SE_CFG(se_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c index da5ca718867915b74e1b55ab9e1afbc21a563578..2ea263e893ee64483ddbd14c53e1b75a40b6ec05 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c @@ -8,6 +8,7 @@ struct lan966x_fdb_event_work { struct work_struct work; struct switchdev_notifier_fdb_info fdb_info; struct net_device *dev; + struct net_device *orig_dev; struct lan966x *lan966x; unsigned long event; }; @@ -127,75 +128,119 @@ void lan966x_fdb_deinit(struct lan966x *lan966x) lan966x_fdb_purge_entries(lan966x); } -static void lan966x_fdb_event_work(struct work_struct *work) +void lan966x_fdb_flush_workqueue(struct lan966x *lan966x) +{ + flush_workqueue(lan966x->fdb_work); +} + +static void lan966x_fdb_port_event_work(struct lan966x_fdb_event_work *fdb_work) { - struct lan966x_fdb_event_work *fdb_work = - container_of(work, struct lan966x_fdb_event_work, work); struct switchdev_notifier_fdb_info *fdb_info; - struct net_device *dev = fdb_work->dev; struct lan966x_port *port; struct lan966x *lan966x; - int ret; - fdb_info = &fdb_work->fdb_info; lan966x = fdb_work->lan966x; + port = netdev_priv(fdb_work->orig_dev); + fdb_info = &fdb_work->fdb_info; - if (lan966x_netdevice_check(dev)) { - port = netdev_priv(dev); + switch (fdb_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + if (!fdb_info->added_by_user) + break; + lan966x_mac_add_entry(lan966x, port, fdb_info->addr, + fdb_info->vid); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + if (!fdb_info->added_by_user) + break; + lan966x_mac_del_entry(lan966x, fdb_info->addr, + fdb_info->vid); + break; + } +} + +static void lan966x_fdb_bridge_event_work(struct lan966x_fdb_event_work *fdb_work) +{ + struct switchdev_notifier_fdb_info *fdb_info; + struct lan966x *lan966x; + int ret; - switch (fdb_work->event) { - case SWITCHDEV_FDB_ADD_TO_DEVICE: - if (!fdb_info->added_by_user) - break; - lan966x_mac_add_entry(lan966x, port, fdb_info->addr, - fdb_info->vid); + lan966x = fdb_work->lan966x; + fdb_info = &fdb_work->fdb_info; + + /* In case the bridge is called */ + switch (fdb_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + /* If there is no front port in this vlan, there is no + * point to copy the frame to CPU because it would be + * just dropped at later point. So add it only if + * there is a port but it is required to store the fdb + * entry for later point when a port actually gets in + * the vlan. + */ + lan966x_fdb_add_entry(lan966x, fdb_info); + if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x, + fdb_info->vid)) break; - case SWITCHDEV_FDB_DEL_TO_DEVICE: - if (!fdb_info->added_by_user) - break; - lan966x_mac_del_entry(lan966x, fdb_info->addr, - fdb_info->vid); + + lan966x_mac_cpu_learn(lan966x, fdb_info->addr, + fdb_info->vid); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + ret = lan966x_fdb_del_entry(lan966x, fdb_info); + if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x, + fdb_info->vid)) break; - } - } else { - if (!netif_is_bridge_master(dev)) - goto out; - - /* In case the bridge is called */ - switch (fdb_work->event) { - case SWITCHDEV_FDB_ADD_TO_DEVICE: - /* If there is no front port in this vlan, there is no - * point to copy the frame to CPU because it would be - * just dropped at later point. So add it only if - * there is a port but it is required to store the fdb - * entry for later point when a port actually gets in - * the vlan. - */ - lan966x_fdb_add_entry(lan966x, fdb_info); - if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x, - fdb_info->vid)) - break; - - lan966x_mac_cpu_learn(lan966x, fdb_info->addr, - fdb_info->vid); + + if (ret) + lan966x_mac_cpu_forget(lan966x, fdb_info->addr, + fdb_info->vid); + break; + } +} + +static void lan966x_fdb_lag_event_work(struct lan966x_fdb_event_work *fdb_work) +{ + struct switchdev_notifier_fdb_info *fdb_info; + struct lan966x_port *port; + struct lan966x *lan966x; + + if (!lan966x_lag_first_port(fdb_work->orig_dev, fdb_work->dev)) + return; + + lan966x = fdb_work->lan966x; + port = netdev_priv(fdb_work->dev); + fdb_info = &fdb_work->fdb_info; + + switch (fdb_work->event) { + case SWITCHDEV_FDB_ADD_TO_DEVICE: + if (!fdb_info->added_by_user) break; - case SWITCHDEV_FDB_DEL_TO_DEVICE: - ret = lan966x_fdb_del_entry(lan966x, fdb_info); - if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x, - fdb_info->vid)) - break; - - if (ret) - lan966x_mac_cpu_forget(lan966x, fdb_info->addr, - fdb_info->vid); + lan966x_mac_add_entry(lan966x, port, fdb_info->addr, + fdb_info->vid); + break; + case SWITCHDEV_FDB_DEL_TO_DEVICE: + if (!fdb_info->added_by_user) break; - } + lan966x_mac_del_entry(lan966x, fdb_info->addr, fdb_info->vid); + break; } +} + +static void lan966x_fdb_event_work(struct work_struct *work) +{ + struct lan966x_fdb_event_work *fdb_work = + container_of(work, struct lan966x_fdb_event_work, work); + + if (lan966x_netdevice_check(fdb_work->orig_dev)) + lan966x_fdb_port_event_work(fdb_work); + else if (netif_is_bridge_master(fdb_work->orig_dev)) + lan966x_fdb_bridge_event_work(fdb_work); + else if (netif_is_lag_master(fdb_work->orig_dev)) + lan966x_fdb_lag_event_work(fdb_work); -out: kfree(fdb_work->fdb_info.addr); kfree(fdb_work); - dev_put(dev); } int lan966x_handle_fdb(struct net_device *dev, @@ -221,7 +266,8 @@ int lan966x_handle_fdb(struct net_device *dev, if (!fdb_work) return -ENOMEM; - fdb_work->dev = orig_dev; + fdb_work->dev = dev; + fdb_work->orig_dev = orig_dev; fdb_work->lan966x = lan966x; fdb_work->event = event; INIT_WORK(&fdb_work->work, lan966x_fdb_event_work); @@ -231,7 +277,6 @@ int lan966x_handle_fdb(struct net_device *dev, goto err_addr_alloc; ether_addr_copy((u8 *)fdb_work->fdb_info.addr, fdb_info->addr); - dev_hold(orig_dev); queue_work(lan966x->fdb_work, &fdb_work->work); break; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 6dea7f8c14814fb70c28c79406c576d8712c2e6a..7e4061c854f0e2115a3c67765aa4647f545986ac 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -425,7 +425,8 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) lan966x_ifh_get_src_port(skb->data, &src_port); lan966x_ifh_get_timestamp(skb->data, ×tamp); - WARN_ON(src_port >= lan966x->num_phys_ports); + if (WARN_ON(src_port >= lan966x->num_phys_ports)) + goto free_skb; skb->dev = lan966x->ports[src_port]->dev; skb_pull(skb, IFH_LEN * sizeof(u32)); @@ -449,6 +450,8 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) return skb; +free_skb: + kfree_skb(skb); unmap_page: dma_unmap_page(lan966x->dev, (dma_addr_t)db->dataptr, FDMA_DCB_STATUS_BLOCKL(db->status), @@ -784,8 +787,7 @@ void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev) return; lan966x->fdma_ndev = dev; - netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll); napi_enable(&lan966x->napi); } diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c b/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c new file mode 100644 index 0000000000000000000000000000000000000000..41fa2523d91d3bf57479dd7d66c1903786aa98b2 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include + +#include "lan966x_main.h" + +static void lan966x_lag_set_aggr_pgids(struct lan966x *lan966x) +{ + u32 visited = GENMASK(lan966x->num_phys_ports - 1, 0); + int p, lag, i; + + /* Reset destination and aggregation PGIDS */ + for (p = 0; p < lan966x->num_phys_ports; ++p) + lan_wr(ANA_PGID_PGID_SET(BIT(p)), + lan966x, ANA_PGID(p)); + + for (p = PGID_AGGR; p < PGID_SRC; ++p) + lan_wr(ANA_PGID_PGID_SET(visited), + lan966x, ANA_PGID(p)); + + /* The visited ports bitmask holds the list of ports offloading any + * bonding interface. Initially we mark all these ports as unvisited, + * then every time we visit a port in this bitmask, we know that it is + * the lowest numbered port, i.e. the one whose logical ID == physical + * port ID == LAG ID. So we mark as visited all further ports in the + * bitmask that are offloading the same bonding interface. This way, + * we set up the aggregation PGIDs only once per bonding interface. + */ + for (p = 0; p < lan966x->num_phys_ports; ++p) { + struct lan966x_port *port = lan966x->ports[p]; + + if (!port || !port->bond) + continue; + + visited &= ~BIT(p); + } + + /* Now, set PGIDs for each active LAG */ + for (lag = 0; lag < lan966x->num_phys_ports; ++lag) { + struct net_device *bond = lan966x->ports[lag]->bond; + int num_active_ports = 0; + unsigned long bond_mask; + u8 aggr_idx[16]; + + if (!bond || (visited & BIT(lag))) + continue; + + bond_mask = lan966x_lag_get_mask(lan966x, bond); + + for_each_set_bit(p, &bond_mask, lan966x->num_phys_ports) { + struct lan966x_port *port = lan966x->ports[p]; + + lan_wr(ANA_PGID_PGID_SET(bond_mask), + lan966x, ANA_PGID(p)); + if (port->lag_tx_active) + aggr_idx[num_active_ports++] = p; + } + + for (i = PGID_AGGR; i < PGID_SRC; ++i) { + u32 ac; + + ac = lan_rd(lan966x, ANA_PGID(i)); + ac &= ~bond_mask; + /* Don't do division by zero if there was no active + * port. Just make all aggregation codes zero. + */ + if (num_active_ports) + ac |= BIT(aggr_idx[i % num_active_ports]); + lan_wr(ANA_PGID_PGID_SET(ac), + lan966x, ANA_PGID(i)); + } + + /* Mark all ports in the same LAG as visited to avoid applying + * the same config again. + */ + for (p = lag; p < lan966x->num_phys_ports; p++) { + struct lan966x_port *port = lan966x->ports[p]; + + if (!port) + continue; + + if (port->bond == bond) + visited |= BIT(p); + } + } +} + +static void lan966x_lag_set_port_ids(struct lan966x *lan966x) +{ + struct lan966x_port *port; + u32 bond_mask; + u32 lag_id; + int p; + + for (p = 0; p < lan966x->num_phys_ports; ++p) { + port = lan966x->ports[p]; + if (!port) + continue; + + lag_id = port->chip_port; + + bond_mask = lan966x_lag_get_mask(lan966x, port->bond); + if (bond_mask) + lag_id = __ffs(bond_mask); + + lan_rmw(ANA_PORT_CFG_PORTID_VAL_SET(lag_id), + ANA_PORT_CFG_PORTID_VAL, + lan966x, ANA_PORT_CFG(port->chip_port)); + } +} + +static void lan966x_lag_update_ids(struct lan966x *lan966x) +{ + lan966x_lag_set_port_ids(lan966x); + lan966x_update_fwd_mask(lan966x); + lan966x_lag_set_aggr_pgids(lan966x); +} + +int lan966x_lag_port_join(struct lan966x_port *port, + struct net_device *brport_dev, + struct net_device *bond, + struct netlink_ext_ack *extack) +{ + struct lan966x *lan966x = port->lan966x; + struct net_device *dev = port->dev; + u32 lag_id = -1; + u32 bond_mask; + int err; + + bond_mask = lan966x_lag_get_mask(lan966x, bond); + if (bond_mask) + lag_id = __ffs(bond_mask); + + port->bond = bond; + lan966x_lag_update_ids(lan966x); + + err = switchdev_bridge_port_offload(brport_dev, dev, port, + &lan966x_switchdev_nb, + &lan966x_switchdev_blocking_nb, + false, extack); + if (err) + goto out; + + lan966x_port_stp_state_set(port, br_port_get_stp_state(brport_dev)); + + if (lan966x_lag_first_port(port->bond, port->dev) && + lag_id != -1) + lan966x_mac_lag_replace_port_entry(lan966x, + lan966x->ports[lag_id], + port); + + return 0; + +out: + port->bond = NULL; + lan966x_lag_update_ids(lan966x); + + return err; +} + +void lan966x_lag_port_leave(struct lan966x_port *port, struct net_device *bond) +{ + struct lan966x *lan966x = port->lan966x; + u32 bond_mask; + u32 lag_id; + + if (lan966x_lag_first_port(port->bond, port->dev)) { + bond_mask = lan966x_lag_get_mask(lan966x, port->bond); + bond_mask &= ~BIT(port->chip_port); + if (bond_mask) { + lag_id = __ffs(bond_mask); + lan966x_mac_lag_replace_port_entry(lan966x, port, + lan966x->ports[lag_id]); + } else { + lan966x_mac_lag_remove_port_entry(lan966x, port); + } + } + + port->bond = NULL; + lan966x_lag_update_ids(lan966x); + lan966x_port_stp_state_set(port, BR_STATE_FORWARDING); +} + +static bool lan966x_lag_port_check_hash_types(struct lan966x *lan966x, + enum netdev_lag_hash hash_type) +{ + int p; + + for (p = 0; p < lan966x->num_phys_ports; ++p) { + struct lan966x_port *port = lan966x->ports[p]; + + if (!port || !port->bond) + continue; + + if (port->hash_type != hash_type) + return false; + } + + return true; +} + +int lan966x_lag_port_prechangeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + struct netdev_lag_upper_info *lui; + struct netlink_ext_ack *extack; + + extack = netdev_notifier_info_to_extack(&info->info); + lui = info->upper_info; + if (!lui) { + port->hash_type = NETDEV_LAG_HASH_NONE; + return NOTIFY_DONE; + } + + if (lui->tx_type != NETDEV_LAG_TX_TYPE_HASH) { + NL_SET_ERR_MSG_MOD(extack, + "LAG device using unsupported Tx type"); + return -EINVAL; + } + + if (!lan966x_lag_port_check_hash_types(lan966x, lui->hash_type)) { + NL_SET_ERR_MSG_MOD(extack, + "LAG devices can have only the same hash_type"); + return -EINVAL; + } + + switch (lui->hash_type) { + case NETDEV_LAG_HASH_L2: + lan_wr(ANA_AGGR_CFG_AC_DMAC_ENA_SET(1) | + ANA_AGGR_CFG_AC_SMAC_ENA_SET(1), + lan966x, ANA_AGGR_CFG); + break; + case NETDEV_LAG_HASH_L34: + lan_wr(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(1) | + ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(1) | + ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_SET(1), + lan966x, ANA_AGGR_CFG); + break; + case NETDEV_LAG_HASH_L23: + lan_wr(ANA_AGGR_CFG_AC_DMAC_ENA_SET(1) | + ANA_AGGR_CFG_AC_SMAC_ENA_SET(1) | + ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(1) | + ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(1), + lan966x, ANA_AGGR_CFG); + break; + default: + NL_SET_ERR_MSG_MOD(extack, + "LAG device using unsupported hash type"); + return -EINVAL; + } + + port->hash_type = lui->hash_type; + + return NOTIFY_OK; +} + +int lan966x_lag_port_changelowerstate(struct net_device *dev, + struct netdev_notifier_changelowerstate_info *info) +{ + struct netdev_lag_lower_state_info *lag = info->lower_state_info; + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + bool is_active; + + if (!port->bond) + return NOTIFY_DONE; + + is_active = lag->link_up && lag->tx_enabled; + if (port->lag_tx_active == is_active) + return NOTIFY_DONE; + + port->lag_tx_active = is_active; + lan966x_lag_set_aggr_pgids(lan966x); + + return NOTIFY_OK; +} + +int lan966x_lag_netdev_prechangeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct lan966x_port *port; + struct net_device *lower; + struct list_head *iter; + int err; + + netdev_for_each_lower_dev(dev, lower, iter) { + if (!lan966x_netdevice_check(lower)) + continue; + + port = netdev_priv(lower); + if (port->bond != dev) + continue; + + err = lan966x_port_prechangeupper(lower, dev, info); + if (err) + return err; + } + + return NOTIFY_DONE; +} + +int lan966x_lag_netdev_changeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct lan966x_port *port; + struct net_device *lower; + struct list_head *iter; + int err; + + netdev_for_each_lower_dev(dev, lower, iter) { + if (!lan966x_netdevice_check(lower)) + continue; + + port = netdev_priv(lower); + if (port->bond != dev) + continue; + + err = lan966x_port_changeupper(lower, dev, info); + if (err) + return err; + } + + return NOTIFY_DONE; +} + +bool lan966x_lag_first_port(struct net_device *lag, struct net_device *dev) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + unsigned long bond_mask; + + if (port->bond != lag) + return false; + + bond_mask = lan966x_lag_get_mask(lan966x, lag); + if (bond_mask && port->chip_port == __ffs(bond_mask)) + return true; + + return false; +} + +u32 lan966x_lag_get_mask(struct lan966x *lan966x, struct net_device *bond) +{ + struct lan966x_port *port; + u32 mask = 0; + int p; + + if (!bond) + return mask; + + for (p = 0; p < lan966x->num_phys_ports; p++) { + port = lan966x->ports[p]; + if (!port) + continue; + + if (port->bond == bond) + mask |= BIT(p); + } + + return mask; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c index 5893770bfd94627139cdca7d61b88a224040ad48..baa3a30c039f49a1960d835ad356fdddac8cb1b6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c @@ -22,6 +22,7 @@ struct lan966x_mac_entry { u16 vid; u16 port_index; int row; + bool lag; }; struct lan966x_mac_raw_entry { @@ -69,15 +70,14 @@ static void lan966x_mac_select(struct lan966x *lan966x, lan_wr(mach, lan966x, ANA_MACHDATA); } -static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid, - bool cpu_copy, - const unsigned char mac[ETH_ALEN], - unsigned int vid, - enum macaccess_entry_type type) +static int __lan966x_mac_learn_locked(struct lan966x *lan966x, int pgid, + bool cpu_copy, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) { - int ret; + lockdep_assert_held(&lan966x->mac_lock); - spin_lock(&lan966x->mac_lock); lan966x_mac_select(lan966x, mac, vid); /* Issue a write command */ @@ -89,7 +89,19 @@ static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid, ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_LEARN), lan966x, ANA_MACACCESS); - ret = lan966x_mac_wait_for_completion(lan966x); + return lan966x_mac_wait_for_completion(lan966x); +} + +static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid, + bool cpu_copy, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) +{ + int ret; + + spin_lock(&lan966x->mac_lock); + ret = __lan966x_mac_learn_locked(lan966x, pgid, cpu_copy, mac, vid, type); spin_unlock(&lan966x->mac_lock); return ret; @@ -119,6 +131,16 @@ int lan966x_mac_learn(struct lan966x *lan966x, int port, return __lan966x_mac_learn(lan966x, port, false, mac, vid, type); } +static int lan966x_mac_learn_locked(struct lan966x *lan966x, int port, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) +{ + WARN_ON(type != ENTRYTYPE_NORMAL && type != ENTRYTYPE_LOCKED); + + return __lan966x_mac_learn_locked(lan966x, port, false, mac, vid, type); +} + static int lan966x_mac_forget_locked(struct lan966x *lan966x, const unsigned char mac[ETH_ALEN], unsigned int vid, @@ -178,8 +200,9 @@ void lan966x_mac_init(struct lan966x *lan966x) INIT_LIST_HEAD(&lan966x->mac_entries); } -static struct lan966x_mac_entry *lan966x_mac_alloc_entry(const unsigned char *mac, - u16 vid, u16 port_index) +static struct lan966x_mac_entry *lan966x_mac_alloc_entry(struct lan966x_port *port, + const unsigned char *mac, + u16 vid) { struct lan966x_mac_entry *mac_entry; @@ -189,8 +212,9 @@ static struct lan966x_mac_entry *lan966x_mac_alloc_entry(const unsigned char *ma memcpy(mac_entry->mac, mac, ETH_ALEN); mac_entry->vid = vid; - mac_entry->port_index = port_index; + mac_entry->port_index = port->chip_port; mac_entry->row = LAN966X_MAC_INVALID_ROW; + mac_entry->lag = port->bond ? true : false; return mac_entry; } @@ -269,7 +293,7 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port, goto mac_learn; } - mac_entry = lan966x_mac_alloc_entry(addr, vid, port->chip_port); + mac_entry = lan966x_mac_alloc_entry(port, addr, vid); if (!mac_entry) { spin_unlock(&lan966x->mac_lock); return -ENOMEM; @@ -278,7 +302,8 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port, list_add_tail(&mac_entry->list, &lan966x->mac_entries); spin_unlock(&lan966x->mac_lock); - lan966x_fdb_call_notifiers(SWITCHDEV_FDB_OFFLOADED, addr, vid, port->dev); + lan966x_fdb_call_notifiers(SWITCHDEV_FDB_OFFLOADED, addr, vid, + port->bond ?: port->dev); mac_learn: lan966x_mac_learn(lan966x, port->chip_port, addr, vid, ENTRYTYPE_LOCKED); @@ -309,6 +334,50 @@ int lan966x_mac_del_entry(struct lan966x *lan966x, const unsigned char *addr, return 0; } +void lan966x_mac_lag_replace_port_entry(struct lan966x *lan966x, + struct lan966x_port *src, + struct lan966x_port *dst) +{ + struct lan966x_mac_entry *mac_entry; + + spin_lock(&lan966x->mac_lock); + list_for_each_entry(mac_entry, &lan966x->mac_entries, list) { + if (mac_entry->port_index == src->chip_port && + mac_entry->lag) { + lan966x_mac_forget_locked(lan966x, mac_entry->mac, + mac_entry->vid, + ENTRYTYPE_LOCKED); + + lan966x_mac_learn_locked(lan966x, dst->chip_port, + mac_entry->mac, mac_entry->vid, + ENTRYTYPE_LOCKED); + mac_entry->port_index = dst->chip_port; + } + } + spin_unlock(&lan966x->mac_lock); +} + +void lan966x_mac_lag_remove_port_entry(struct lan966x *lan966x, + struct lan966x_port *src) +{ + struct lan966x_mac_entry *mac_entry, *tmp; + + spin_lock(&lan966x->mac_lock); + list_for_each_entry_safe(mac_entry, tmp, &lan966x->mac_entries, + list) { + if (mac_entry->port_index == src->chip_port && + mac_entry->lag) { + lan966x_mac_forget_locked(lan966x, mac_entry->mac, + mac_entry->vid, + ENTRYTYPE_LOCKED); + + list_del(&mac_entry->list); + kfree(mac_entry); + } + } + spin_unlock(&lan966x->mac_lock); +} + void lan966x_mac_purge_entries(struct lan966x *lan966x) { struct lan966x_mac_entry *mac_entry, *tmp; @@ -354,6 +423,7 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, struct lan966x_mac_entry *mac_entry, *tmp; unsigned char mac[ETH_ALEN] __aligned(2); struct list_head mac_deleted_entries; + struct lan966x_port *port; u32 dest_idx; u32 column; u16 vid; @@ -406,9 +476,10 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, /* Notify the bridge that the entry doesn't exist * anymore in the HW */ + port = lan966x->ports[mac_entry->port_index]; lan966x_mac_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, mac_entry->mac, mac_entry->vid, - lan966x->ports[mac_entry->port_index]->dev); + port->bond ?: port->dev); list_del(&mac_entry->list); kfree(mac_entry); } @@ -440,7 +511,8 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, continue; } - mac_entry = lan966x_mac_alloc_entry(mac, vid, dest_idx); + port = lan966x->ports[dest_idx]; + mac_entry = lan966x_mac_alloc_entry(port, mac, vid); if (!mac_entry) { spin_unlock(&lan966x->mac_lock); return; @@ -451,7 +523,7 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row, spin_unlock(&lan966x->mac_lock); lan966x_mac_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE, - mac, vid, lan966x->ports[dest_idx]->dev); + mac, vid, port->bond ?: port->dev); } } diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index d928b75f37803992b87298753d894e546dd4d65f..be2fd030cccbe0427cabd136f30f42904ebb709f 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -344,7 +344,8 @@ static void lan966x_ifh_set_timestamp(void *ifh, u64 timestamp) IFH_POS_TIMESTAMP, IFH_LEN * 4, PACK, 0); } -static int lan966x_port_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t lan966x_port_xmit(struct sk_buff *skb, + struct net_device *dev) { struct lan966x_port *port = netdev_priv(dev); struct lan966x *lan966x = port->lan966x; @@ -466,6 +467,7 @@ static const struct net_device_ops lan966x_port_netdev_ops = { .ndo_set_mac_address = lan966x_port_set_mac_address, .ndo_get_port_parent_id = lan966x_port_get_parent_id, .ndo_eth_ioctl = lan966x_port_ioctl, + .ndo_setup_tc = lan966x_tc_setup, }; bool lan966x_netdevice_check(const struct net_device *dev) @@ -738,7 +740,8 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, return -EINVAL; dev = devm_alloc_etherdev_mqs(lan966x->dev, - sizeof(struct lan966x_port), 8, 1); + sizeof(struct lan966x_port), + NUM_PRIO_QUEUES, 1); if (!dev) return -ENOMEM; @@ -754,7 +757,9 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, dev->netdev_ops = &lan966x_port_netdev_ops; dev->ethtool_ops = &lan966x_ethtool_ops; dev->features |= NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_HW_VLAN_STAG_TX; + NETIF_F_HW_VLAN_STAG_TX | + NETIF_F_HW_TC; + dev->hw_features |= NETIF_F_HW_TC; dev->needed_headroom = IFH_LEN * sizeof(u32); eth_hw_addr_gen(dev, lan966x->base_mac, p + 1); @@ -770,6 +775,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, port->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD | MAC_2500FD; + phy_interface_set_rgmii(port->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_MII, port->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_GMII, @@ -778,6 +784,8 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, port->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_QSGMII, port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_QUSGMII, + port->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_1000BASEX, port->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_2500BASEX, @@ -956,6 +964,8 @@ static void lan966x_init(struct lan966x *lan966x) lan966x, ANA_ANAINTR); spin_lock_init(&lan966x->tx_lock); + + lan966x_taprio_init(lan966x); } static int lan966x_ram_init(struct lan966x *lan966x) @@ -969,7 +979,8 @@ static int lan966x_reset_switch(struct lan966x *lan966x) int val = 0; int ret; - switch_reset = devm_reset_control_get_shared(lan966x->dev, "switch"); + switch_reset = devm_reset_control_get_optional_shared(lan966x->dev, + "switch"); if (IS_ERR(switch_reset)) return dev_err_probe(lan966x->dev, PTR_ERR(switch_reset), "Could not obtain switch reset"); @@ -1164,6 +1175,7 @@ static int lan966x_remove(struct platform_device *pdev) { struct lan966x *lan966x = platform_get_drvdata(pdev); + lan966x_taprio_deinit(lan966x); lan966x_fdma_deinit(lan966x); lan966x_cleanup_ports(lan966x); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index 2787055c1847535c09a1eb5a3086ffe42305b3ea..9656071b8289e076be984a5b5ebd77181e39042a 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include "lan966x_regs.h" @@ -36,6 +38,7 @@ #define NUM_PHYS_PORTS 8 #define CPU_PORT 8 +#define NUM_PRIO_QUEUES 8 /* Reserved PGIDs */ #define PGID_CPU (PGID_AGGR - 6) @@ -79,6 +82,9 @@ #define FDMA_INJ_CHANNEL 0 #define FDMA_DCB_MAX 512 +#define SE_IDX_QUEUE 0 /* 0-79 : Queue scheduler elements */ +#define SE_IDX_PORT 80 /* 80-89 : Port schedular elements */ + /* MAC table entry types. * ENTRYTYPE_NORMAL is subject to aging. * ENTRYTYPE_LOCKED is not subject to aging. @@ -258,6 +264,11 @@ struct lan966x { struct lan966x_rx rx; struct lan966x_tx tx; struct napi_struct napi; + + /* Mirror */ + struct lan966x_port *mirror_monitor; + u32 mirror_mask[2]; + u32 mirror_count; }; struct lan966x_port_config { @@ -270,6 +281,15 @@ struct lan966x_port_config { bool autoneg; }; +struct lan966x_port_tc { + bool ingress_shared_block; + unsigned long police_id; + unsigned long ingress_mirror_id; + unsigned long egress_mirror_id; + struct flow_stats police_stat; + struct flow_stats mirror_stat; +}; + struct lan966x_port { struct net_device *dev; struct lan966x *lan966x; @@ -292,11 +312,19 @@ struct lan966x_port { u8 ptp_cmd; u16 ts_id; struct sk_buff_head tx_skbs; + + struct net_device *bond; + bool lag_tx_active; + enum netdev_lag_hash hash_type; + + struct lan966x_port_tc tc; }; extern const struct phylink_mac_ops lan966x_phylink_mac_ops; extern const struct phylink_pcs_ops lan966x_phylink_pcs_ops; extern const struct ethtool_ops lan966x_ethtool_ops; +extern struct notifier_block lan966x_switchdev_nb __read_mostly; +extern struct notifier_block lan966x_switchdev_blocking_nb __read_mostly; bool lan966x_netdevice_check(const struct net_device *dev); @@ -345,6 +373,11 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port, const unsigned char *addr, u16 vid); +void lan966x_mac_lag_replace_port_entry(struct lan966x *lan966x, + struct lan966x_port *src, + struct lan966x_port *dst); +void lan966x_mac_lag_remove_port_entry(struct lan966x *lan966x, + struct lan966x_port *src); void lan966x_mac_purge_entries(struct lan966x *lan966x); irqreturn_t lan966x_mac_irq_handler(struct lan966x *lan966x); @@ -369,6 +402,7 @@ void lan966x_fdb_write_entries(struct lan966x *lan966x, u16 vid); void lan966x_fdb_erase_entries(struct lan966x *lan966x, u16 vid); int lan966x_fdb_init(struct lan966x *lan966x); void lan966x_fdb_deinit(struct lan966x *lan966x); +void lan966x_fdb_flush_workqueue(struct lan966x *lan966x); int lan966x_handle_fdb(struct net_device *dev, struct net_device *orig_dev, unsigned long event, const void *ctx, @@ -397,6 +431,8 @@ void lan966x_ptp_txtstamp_release(struct lan966x_port *port, struct sk_buff *skb); irqreturn_t lan966x_ptp_irq_handler(int irq, void *args); irqreturn_t lan966x_ptp_ext_irq_handler(int irq, void *args); +u32 lan966x_ptp_get_period_ps(void); +int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts); int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev); int lan966x_fdma_change_mtu(struct lan966x *lan966x); @@ -406,6 +442,89 @@ int lan966x_fdma_init(struct lan966x *lan966x); void lan966x_fdma_deinit(struct lan966x *lan966x); irqreturn_t lan966x_fdma_irq_handler(int irq, void *args); +int lan966x_lag_port_join(struct lan966x_port *port, + struct net_device *brport_dev, + struct net_device *bond, + struct netlink_ext_ack *extack); +void lan966x_lag_port_leave(struct lan966x_port *port, struct net_device *bond); +int lan966x_lag_port_prechangeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info); +int lan966x_lag_port_changelowerstate(struct net_device *dev, + struct netdev_notifier_changelowerstate_info *info); +int lan966x_lag_netdev_prechangeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info); +int lan966x_lag_netdev_changeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info); +bool lan966x_lag_first_port(struct net_device *lag, struct net_device *dev); +u32 lan966x_lag_get_mask(struct lan966x *lan966x, struct net_device *bond); + +int lan966x_port_changeupper(struct net_device *dev, + struct net_device *brport_dev, + struct netdev_notifier_changeupper_info *info); +int lan966x_port_prechangeupper(struct net_device *dev, + struct net_device *brport_dev, + struct netdev_notifier_changeupper_info *info); +void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state); +void lan966x_port_ageing_set(struct lan966x_port *port, + unsigned long ageing_clock_t); +void lan966x_update_fwd_mask(struct lan966x *lan966x); + +int lan966x_tc_setup(struct net_device *dev, enum tc_setup_type type, + void *type_data); + +int lan966x_mqprio_add(struct lan966x_port *port, u8 num_tc); +int lan966x_mqprio_del(struct lan966x_port *port); + +void lan966x_taprio_init(struct lan966x *lan966x); +void lan966x_taprio_deinit(struct lan966x *lan966x); +int lan966x_taprio_add(struct lan966x_port *port, + struct tc_taprio_qopt_offload *qopt); +int lan966x_taprio_del(struct lan966x_port *port); +int lan966x_taprio_speed_set(struct lan966x_port *port, int speed); + +int lan966x_tbf_add(struct lan966x_port *port, + struct tc_tbf_qopt_offload *qopt); +int lan966x_tbf_del(struct lan966x_port *port, + struct tc_tbf_qopt_offload *qopt); + +int lan966x_cbs_add(struct lan966x_port *port, + struct tc_cbs_qopt_offload *qopt); +int lan966x_cbs_del(struct lan966x_port *port, + struct tc_cbs_qopt_offload *qopt); + +int lan966x_ets_add(struct lan966x_port *port, + struct tc_ets_qopt_offload *qopt); +int lan966x_ets_del(struct lan966x_port *port, + struct tc_ets_qopt_offload *qopt); + +int lan966x_tc_matchall(struct lan966x_port *port, + struct tc_cls_matchall_offload *f, + bool ingress); + +int lan966x_police_port_add(struct lan966x_port *port, + struct flow_action *action, + struct flow_action_entry *act, + unsigned long police_id, + bool ingress, + struct netlink_ext_ack *extack); +int lan966x_police_port_del(struct lan966x_port *port, + unsigned long police_id, + struct netlink_ext_ack *extack); +void lan966x_police_port_stats(struct lan966x_port *port, + struct flow_stats *stats); + +int lan966x_mirror_port_add(struct lan966x_port *port, + struct flow_action_entry *action, + unsigned long mirror_id, + bool ingress, + struct netlink_ext_ack *extack); +int lan966x_mirror_port_del(struct lan966x_port *port, + bool ingress, + struct netlink_ext_ack *extack); +void lan966x_mirror_port_stats(struct lan966x_port *port, + struct flow_stats *stats, + bool ingress); + static inline void __iomem *lan_addr(void __iomem *base[], int id, int tinst, int tcnt, int gbase, int ginst, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c new file mode 100644 index 0000000000000000000000000000000000000000..7e1ba3f40c35ea7dc46a89ad554cb01ad7280c35 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +int lan966x_mirror_port_add(struct lan966x_port *port, + struct flow_action_entry *action, + unsigned long mirror_id, + bool ingress, + struct netlink_ext_ack *extack) +{ + struct lan966x *lan966x = port->lan966x; + struct lan966x_port *monitor_port; + + if (!lan966x_netdevice_check(action->dev)) { + NL_SET_ERR_MSG_MOD(extack, + "Destination not an lan966x port"); + return -EOPNOTSUPP; + } + + monitor_port = netdev_priv(action->dev); + + if (lan966x->mirror_mask[ingress] & BIT(port->chip_port)) { + NL_SET_ERR_MSG_MOD(extack, + "Mirror already exists"); + return -EEXIST; + } + + if (lan966x->mirror_monitor && + lan966x->mirror_monitor != monitor_port) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot change mirror port while in use"); + return -EBUSY; + } + + if (port == monitor_port) { + NL_SET_ERR_MSG_MOD(extack, + "Cannot mirror the monitor port"); + return -EINVAL; + } + + lan966x->mirror_mask[ingress] |= BIT(port->chip_port); + + lan966x->mirror_monitor = monitor_port; + lan_wr(BIT(monitor_port->chip_port), lan966x, ANA_MIRRORPORTS); + + if (ingress) { + lan_rmw(ANA_PORT_CFG_SRC_MIRROR_ENA_SET(1), + ANA_PORT_CFG_SRC_MIRROR_ENA, + lan966x, ANA_PORT_CFG(port->chip_port)); + } else { + lan_wr(lan966x->mirror_mask[0], lan966x, + ANA_EMIRRORPORTS); + } + + lan966x->mirror_count++; + + if (ingress) + port->tc.ingress_mirror_id = mirror_id; + else + port->tc.egress_mirror_id = mirror_id; + + return 0; +} + +int lan966x_mirror_port_del(struct lan966x_port *port, + bool ingress, + struct netlink_ext_ack *extack) +{ + struct lan966x *lan966x = port->lan966x; + + if (!(lan966x->mirror_mask[ingress] & BIT(port->chip_port))) { + NL_SET_ERR_MSG_MOD(extack, + "There is no mirroring for this port"); + return -ENOENT; + } + + lan966x->mirror_mask[ingress] &= ~BIT(port->chip_port); + + if (ingress) { + lan_rmw(ANA_PORT_CFG_SRC_MIRROR_ENA_SET(0), + ANA_PORT_CFG_SRC_MIRROR_ENA, + lan966x, ANA_PORT_CFG(port->chip_port)); + } else { + lan_wr(lan966x->mirror_mask[0], lan966x, + ANA_EMIRRORPORTS); + } + + lan966x->mirror_count--; + + if (lan966x->mirror_count == 0) { + lan966x->mirror_monitor = NULL; + lan_wr(0, lan966x, ANA_MIRRORPORTS); + } + + if (ingress) + port->tc.ingress_mirror_id = 0; + else + port->tc.egress_mirror_id = 0; + + return 0; +} + +void lan966x_mirror_port_stats(struct lan966x_port *port, + struct flow_stats *stats, + bool ingress) +{ + struct rtnl_link_stats64 new_stats; + struct flow_stats *old_stats; + + old_stats = &port->tc.mirror_stat; + lan966x_stats_get(port->dev, &new_stats); + + if (ingress) { + flow_stats_update(stats, + new_stats.rx_bytes - old_stats->bytes, + new_stats.rx_packets - old_stats->pkts, + new_stats.rx_dropped - old_stats->drops, + old_stats->lastused, + FLOW_ACTION_HW_STATS_IMMEDIATE); + + old_stats->bytes = new_stats.rx_bytes; + old_stats->pkts = new_stats.rx_packets; + old_stats->drops = new_stats.rx_dropped; + old_stats->lastused = jiffies; + } else { + flow_stats_update(stats, + new_stats.tx_bytes - old_stats->bytes, + new_stats.tx_packets - old_stats->pkts, + new_stats.tx_dropped - old_stats->drops, + old_stats->lastused, + FLOW_ACTION_HW_STATS_IMMEDIATE); + + old_stats->bytes = new_stats.tx_bytes; + old_stats->pkts = new_stats.tx_packets; + old_stats->drops = new_stats.tx_dropped; + old_stats->lastused = jiffies; + } +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c new file mode 100644 index 0000000000000000000000000000000000000000..7fa76e74f9e23154cad5d9da9d9a2934803f85da --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +int lan966x_mqprio_add(struct lan966x_port *port, u8 num_tc) +{ + u8 i; + + if (num_tc != NUM_PRIO_QUEUES) { + netdev_err(port->dev, "Only %d traffic classes supported\n", + NUM_PRIO_QUEUES); + return -EINVAL; + } + + netdev_set_num_tc(port->dev, num_tc); + + for (i = 0; i < num_tc; ++i) + netdev_set_tc_queue(port->dev, i, 1, i); + + return 0; +} + +int lan966x_mqprio_del(struct lan966x_port *port) +{ + netdev_reset_tc(port->dev); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c index 38a7e95d69b473a1c91e1f331f1931fdcb3e50f1..e4ac59480514b6037192aebefafb705a58ffe7d3 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c @@ -28,11 +28,12 @@ static int lan966x_phylink_mac_prepare(struct phylink_config *config, phy_interface_t iface) { struct lan966x_port *port = netdev_priv(to_net_dev(config->dev)); + phy_interface_t serdes_mode = iface; int err; if (port->serdes) { err = phy_set_mode_ext(port->serdes, PHY_MODE_ETHERNET, - iface); + serdes_mode); if (err) { netdev_err(to_net_dev(config->dev), "Could not set mode of SerDes\n"); @@ -59,6 +60,9 @@ static void lan966x_phylink_mac_link_up(struct phylink_config *config, port_config->pause |= tx_pause ? MLO_PAUSE_TX : 0; port_config->pause |= rx_pause ? MLO_PAUSE_RX : 0; + if (phy_interface_mode_is_rgmii(interface)) + phy_set_speed(port->serdes, speed); + lan966x_port_config_up(port); } diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_police.c b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c new file mode 100644 index 0000000000000000000000000000000000000000..a9aec900d608dd426496c642477b6250d4469f78 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +/* 0-8 : 9 port policers */ +#define POL_IDX_PORT 0 + +/* Policer order: Serial (QoS -> Port -> VCAP) */ +#define POL_ORDER 0x1d3 + +struct lan966x_tc_policer { + /* kilobit per second */ + u32 rate; + /* bytes */ + u32 burst; +}; + +static int lan966x_police_add(struct lan966x_port *port, + struct lan966x_tc_policer *pol, + u16 pol_idx) +{ + struct lan966x *lan966x = port->lan966x; + + /* Rate unit is 33 1/3 kpps */ + pol->rate = DIV_ROUND_UP(pol->rate * 3, 100); + /* Avoid zero burst size */ + pol->burst = pol->burst ?: 1; + /* Unit is 4kB */ + pol->burst = DIV_ROUND_UP(pol->burst, 4096); + + if (pol->rate > GENMASK(15, 0) || + pol->burst > GENMASK(6, 0)) + return -EINVAL; + + lan_wr(ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(0) | + ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(0) | + ANA_POL_MODE_IPG_SIZE_SET(20) | + ANA_POL_MODE_FRM_MODE_SET(1) | + ANA_POL_MODE_OVERSHOOT_ENA_SET(1), + lan966x, ANA_POL_MODE(pol_idx)); + + lan_wr(ANA_POL_PIR_STATE_PIR_LVL_SET(0), + lan966x, ANA_POL_PIR_STATE(pol_idx)); + + lan_wr(ANA_POL_PIR_CFG_PIR_RATE_SET(pol->rate) | + ANA_POL_PIR_CFG_PIR_BURST_SET(pol->burst), + lan966x, ANA_POL_PIR_CFG(pol_idx)); + + return 0; +} + +static int lan966x_police_del(struct lan966x_port *port, + u16 pol_idx) +{ + struct lan966x *lan966x = port->lan966x; + + lan_wr(ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(0) | + ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(0) | + ANA_POL_MODE_IPG_SIZE_SET(20) | + ANA_POL_MODE_FRM_MODE_SET(2) | + ANA_POL_MODE_OVERSHOOT_ENA_SET(1), + lan966x, ANA_POL_MODE(pol_idx)); + + lan_wr(ANA_POL_PIR_STATE_PIR_LVL_SET(0), + lan966x, ANA_POL_PIR_STATE(pol_idx)); + + lan_wr(ANA_POL_PIR_CFG_PIR_RATE_SET(GENMASK(14, 0)) | + ANA_POL_PIR_CFG_PIR_BURST_SET(0), + lan966x, ANA_POL_PIR_CFG(pol_idx)); + + return 0; +} + +static int lan966x_police_validate(struct lan966x_port *port, + const struct flow_action *action, + const struct flow_action_entry *act, + unsigned long police_id, + bool ingress, + struct netlink_ext_ack *extack) +{ + if (act->police.exceed.act_id != FLOW_ACTION_DROP) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when exceed action is not drop"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not pipe or ok"); + return -EOPNOTSUPP; + } + + if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && + !flow_action_is_last_entry(action, act)) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is ok, but action is not last"); + return -EOPNOTSUPP; + } + + if (act->police.peakrate_bytes_ps || + act->police.avrate || act->police.overhead) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when peakrate/avrate/overhead is configured"); + return -EOPNOTSUPP; + } + + if (act->police.rate_pkt_ps) { + NL_SET_ERR_MSG_MOD(extack, + "QoS offload not support packets per second"); + return -EOPNOTSUPP; + } + + if (!ingress) { + NL_SET_ERR_MSG_MOD(extack, + "Policer is not supported on egress"); + return -EOPNOTSUPP; + } + + if (port->tc.ingress_shared_block) { + NL_SET_ERR_MSG_MOD(extack, + "Policer is not supported on shared ingress blocks"); + return -EOPNOTSUPP; + } + + if (port->tc.police_id && port->tc.police_id != police_id) { + NL_SET_ERR_MSG_MOD(extack, + "Only one policer per port is supported"); + return -EEXIST; + } + + return 0; +} + +int lan966x_police_port_add(struct lan966x_port *port, + struct flow_action *action, + struct flow_action_entry *act, + unsigned long police_id, + bool ingress, + struct netlink_ext_ack *extack) +{ + struct lan966x *lan966x = port->lan966x; + struct rtnl_link_stats64 new_stats; + struct lan966x_tc_policer pol; + struct flow_stats *old_stats; + int err; + + err = lan966x_police_validate(port, action, act, police_id, ingress, + extack); + if (err) + return err; + + memset(&pol, 0, sizeof(pol)); + + pol.rate = div_u64(act->police.rate_bytes_ps, 1000) * 8; + pol.burst = act->police.burst; + + err = lan966x_police_add(port, &pol, POL_IDX_PORT + port->chip_port); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to add policer to port"); + return err; + } + + lan_rmw(ANA_POL_CFG_PORT_POL_ENA_SET(1) | + ANA_POL_CFG_POL_ORDER_SET(POL_ORDER), + ANA_POL_CFG_PORT_POL_ENA | + ANA_POL_CFG_POL_ORDER, + lan966x, ANA_POL_CFG(port->chip_port)); + + port->tc.police_id = police_id; + + /* Setup initial stats */ + old_stats = &port->tc.police_stat; + lan966x_stats_get(port->dev, &new_stats); + old_stats->bytes = new_stats.rx_bytes; + old_stats->pkts = new_stats.rx_packets; + old_stats->drops = new_stats.rx_dropped; + old_stats->lastused = jiffies; + + return 0; +} + +int lan966x_police_port_del(struct lan966x_port *port, + unsigned long police_id, + struct netlink_ext_ack *extack) +{ + struct lan966x *lan966x = port->lan966x; + int err; + + if (port->tc.police_id != police_id) { + NL_SET_ERR_MSG_MOD(extack, + "Invalid policer id"); + return -EINVAL; + } + + err = lan966x_police_del(port, port->tc.police_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to add policer to port"); + return err; + } + + lan_rmw(ANA_POL_CFG_PORT_POL_ENA_SET(0) | + ANA_POL_CFG_POL_ORDER_SET(POL_ORDER), + ANA_POL_CFG_PORT_POL_ENA | + ANA_POL_CFG_POL_ORDER, + lan966x, ANA_POL_CFG(port->chip_port)); + + port->tc.police_id = 0; + + return 0; +} + +void lan966x_police_port_stats(struct lan966x_port *port, + struct flow_stats *stats) +{ + struct rtnl_link_stats64 new_stats; + struct flow_stats *old_stats; + + old_stats = &port->tc.police_stat; + lan966x_stats_get(port->dev, &new_stats); + + flow_stats_update(stats, + new_stats.rx_bytes - old_stats->bytes, + new_stats.rx_packets - old_stats->pkts, + new_stats.rx_dropped - old_stats->drops, + old_stats->lastused, + FLOW_ACTION_HW_STATS_IMMEDIATE); + + old_stats->bytes = new_stats.rx_bytes; + old_stats->pkts = new_stats.rx_packets; + old_stats->drops = new_stats.rx_dropped; + old_stats->lastused = jiffies; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_port.c b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c index f141644e4372ff7bad8450e4260a5844f659dad4..1a61c6cdb0779f84e90b765050a83ab4432ac6af 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_port.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c @@ -165,10 +165,12 @@ static void lan966x_port_link_up(struct lan966x_port *port) break; } + lan966x_taprio_speed_set(port, config->speed); + /* Also the GIGA_MODE_ENA(1) needs to be set regardless of the * port speed for QSGMII ports. */ - if (config->portmode == PHY_INTERFACE_MODE_QSGMII) + if (phy_interface_num_ports(config->portmode) == 4) mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(1); lan_wr(config->duplex | mode, @@ -331,10 +333,14 @@ int lan966x_port_pcs_set(struct lan966x_port *port, struct lan966x *lan966x = port->lan966x; bool inband_aneg = false; bool outband; + bool full_preamble = false; + + if (config->portmode == PHY_INTERFACE_MODE_QUSGMII) + full_preamble = true; if (config->inband) { if (config->portmode == PHY_INTERFACE_MODE_SGMII || - config->portmode == PHY_INTERFACE_MODE_QSGMII) + phy_interface_num_ports(config->portmode) == 4) inband_aneg = true; /* Cisco-SGMII in-band-aneg */ else if (config->portmode == PHY_INTERFACE_MODE_1000BASEX && config->autoneg) @@ -345,9 +351,15 @@ int lan966x_port_pcs_set(struct lan966x_port *port, outband = true; } - /* Disable or enable inband */ - lan_rmw(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(outband), - DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, + /* Disable or enable inband. + * For QUSGMII, we rely on the preamble to transmit data such as + * timestamps, therefore force full preamble transmission, and prevent + * premable shortening + */ + lan_rmw(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(outband) | + DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_SET(full_preamble), + DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA | + DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, lan966x, DEV_PCS1G_MODE_CFG(port->chip_port)); /* Enable PCS */ @@ -396,7 +408,7 @@ void lan966x_port_init(struct lan966x_port *port) if (lan966x->fdma) lan966x_fdma_netdev_init(lan966x, port->dev); - if (config->portmode != PHY_INTERFACE_MODE_QSGMII) + if (phy_interface_num_ports(config->portmode) != 4) return; lan_rmw(DEV_CLOCK_CFG_PCS_RX_RST_SET(0) | diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c index 3a621c5165bc5eedd3ba804f0730adc569d3e939..e5a2bbe064f8f70b91743bf70d429204bcc6b2c7 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c @@ -464,8 +464,7 @@ static int lan966x_ptp_settime64(struct ptp_clock_info *ptp, return 0; } -static int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, - struct timespec64 *ts) +int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts) { struct lan966x_phc *phc = container_of(ptp, struct lan966x_phc, info); struct lan966x *lan966x = phc->lan966x; @@ -890,3 +889,9 @@ void lan966x_ptp_rxtstamp(struct lan966x *lan966x, struct sk_buff *skb, shhwtstamps = skb_hwtstamps(skb); shhwtstamps->hwtstamp = full_ts_in_ns; } + +u32 lan966x_ptp_get_period_ps(void) +{ + /* This represents the system clock period in picoseconds */ + return 15125; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h index 8265ad89f0bcb82ec7f2f2484b021915d7433022..1d90b93dd417a15c2b86dbfd79de2be0693b30a6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h @@ -90,6 +90,24 @@ enum lan966x_target { #define ANA_AUTOAGE_AGE_PERIOD_GET(x)\ FIELD_GET(ANA_AUTOAGE_AGE_PERIOD, x) +/* ANA:ANA:MIRRORPORTS */ +#define ANA_MIRRORPORTS __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 60, 0, 1, 4) + +#define ANA_MIRRORPORTS_MIRRORPORTS GENMASK(8, 0) +#define ANA_MIRRORPORTS_MIRRORPORTS_SET(x)\ + FIELD_PREP(ANA_MIRRORPORTS_MIRRORPORTS, x) +#define ANA_MIRRORPORTS_MIRRORPORTS_GET(x)\ + FIELD_GET(ANA_MIRRORPORTS_MIRRORPORTS, x) + +/* ANA:ANA:EMIRRORPORTS */ +#define ANA_EMIRRORPORTS __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 64, 0, 1, 4) + +#define ANA_EMIRRORPORTS_EMIRRORPORTS GENMASK(8, 0) +#define ANA_EMIRRORPORTS_EMIRRORPORTS_SET(x)\ + FIELD_PREP(ANA_EMIRRORPORTS_EMIRRORPORTS, x) +#define ANA_EMIRRORPORTS_EMIRRORPORTS_GET(x)\ + FIELD_GET(ANA_EMIRRORPORTS_EMIRRORPORTS, x) + /* ANA:ANA:FLOODING */ #define ANA_FLOODING(r) __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 68, r, 8, 4) @@ -330,6 +348,12 @@ enum lan966x_target { /* ANA:PORT:PORT_CFG */ #define ANA_PORT_CFG(g) __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 112, 0, 1, 4) +#define ANA_PORT_CFG_SRC_MIRROR_ENA BIT(13) +#define ANA_PORT_CFG_SRC_MIRROR_ENA_SET(x)\ + FIELD_PREP(ANA_PORT_CFG_SRC_MIRROR_ENA, x) +#define ANA_PORT_CFG_SRC_MIRROR_ENA_GET(x)\ + FIELD_GET(ANA_PORT_CFG_SRC_MIRROR_ENA, x) + #define ANA_PORT_CFG_LEARNAUTO BIT(6) #define ANA_PORT_CFG_LEARNAUTO_SET(x)\ FIELD_PREP(ANA_PORT_CFG_LEARNAUTO, x) @@ -354,6 +378,21 @@ enum lan966x_target { #define ANA_PORT_CFG_PORTID_VAL_GET(x)\ FIELD_GET(ANA_PORT_CFG_PORTID_VAL, x) +/* ANA:PORT:POL_CFG */ +#define ANA_POL_CFG(g) __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 116, 0, 1, 4) + +#define ANA_POL_CFG_PORT_POL_ENA BIT(17) +#define ANA_POL_CFG_PORT_POL_ENA_SET(x)\ + FIELD_PREP(ANA_POL_CFG_PORT_POL_ENA, x) +#define ANA_POL_CFG_PORT_POL_ENA_GET(x)\ + FIELD_GET(ANA_POL_CFG_PORT_POL_ENA, x) + +#define ANA_POL_CFG_POL_ORDER GENMASK(8, 0) +#define ANA_POL_CFG_POL_ORDER_SET(x)\ + FIELD_PREP(ANA_POL_CFG_POL_ORDER, x) +#define ANA_POL_CFG_POL_ORDER_GET(x)\ + FIELD_GET(ANA_POL_CFG_POL_ORDER, x) + /* ANA:PFC:PFC_CFG */ #define ANA_PFC_CFG(g) __REG(TARGET_ANA, 0, 1, 30720, g, 8, 64, 0, 0, 1, 4) @@ -363,6 +402,108 @@ enum lan966x_target { #define ANA_PFC_CFG_FC_LINK_SPEED_GET(x)\ FIELD_GET(ANA_PFC_CFG_FC_LINK_SPEED, x) +/* ANA:COMMON:AGGR_CFG */ +#define ANA_AGGR_CFG __REG(TARGET_ANA, 0, 1, 31232, 0, 1, 552, 0, 0, 1, 4) + +#define ANA_AGGR_CFG_AC_RND_ENA BIT(6) +#define ANA_AGGR_CFG_AC_RND_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_RND_ENA, x) +#define ANA_AGGR_CFG_AC_RND_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_RND_ENA, x) + +#define ANA_AGGR_CFG_AC_DMAC_ENA BIT(5) +#define ANA_AGGR_CFG_AC_DMAC_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_DMAC_ENA, x) +#define ANA_AGGR_CFG_AC_DMAC_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_DMAC_ENA, x) + +#define ANA_AGGR_CFG_AC_SMAC_ENA BIT(4) +#define ANA_AGGR_CFG_AC_SMAC_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_SMAC_ENA, x) +#define ANA_AGGR_CFG_AC_SMAC_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_SMAC_ENA, x) + +#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA BIT(3) +#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA, x) +#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA, x) + +#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA BIT(2) +#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA, x) +#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA, x) + +#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA BIT(1) +#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA, x) +#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA, x) + +#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA BIT(0) +#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(x)\ + FIELD_PREP(ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA, x) +#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_GET(x)\ + FIELD_GET(ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA, x) + +/* ANA:POL:POL_PIR_CFG */ +#define ANA_POL_PIR_CFG(g) __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 0, 0, 1, 4) + +#define ANA_POL_PIR_CFG_PIR_RATE GENMASK(20, 6) +#define ANA_POL_PIR_CFG_PIR_RATE_SET(x)\ + FIELD_PREP(ANA_POL_PIR_CFG_PIR_RATE, x) +#define ANA_POL_PIR_CFG_PIR_RATE_GET(x)\ + FIELD_GET(ANA_POL_PIR_CFG_PIR_RATE, x) + +#define ANA_POL_PIR_CFG_PIR_BURST GENMASK(5, 0) +#define ANA_POL_PIR_CFG_PIR_BURST_SET(x)\ + FIELD_PREP(ANA_POL_PIR_CFG_PIR_BURST, x) +#define ANA_POL_PIR_CFG_PIR_BURST_GET(x)\ + FIELD_GET(ANA_POL_PIR_CFG_PIR_BURST, x) + +/* ANA:POL:POL_MODE_CFG */ +#define ANA_POL_MODE(g) __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 8, 0, 1, 4) + +#define ANA_POL_MODE_DROP_ON_YELLOW_ENA BIT(11) +#define ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(x)\ + FIELD_PREP(ANA_POL_MODE_DROP_ON_YELLOW_ENA, x) +#define ANA_POL_MODE_DROP_ON_YELLOW_ENA_GET(x)\ + FIELD_GET(ANA_POL_MODE_DROP_ON_YELLOW_ENA, x) + +#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA BIT(10) +#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(x)\ + FIELD_PREP(ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA, x) +#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_GET(x)\ + FIELD_GET(ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA, x) + +#define ANA_POL_MODE_IPG_SIZE GENMASK(9, 5) +#define ANA_POL_MODE_IPG_SIZE_SET(x)\ + FIELD_PREP(ANA_POL_MODE_IPG_SIZE, x) +#define ANA_POL_MODE_IPG_SIZE_GET(x)\ + FIELD_GET(ANA_POL_MODE_IPG_SIZE, x) + +#define ANA_POL_MODE_FRM_MODE GENMASK(4, 3) +#define ANA_POL_MODE_FRM_MODE_SET(x)\ + FIELD_PREP(ANA_POL_MODE_FRM_MODE, x) +#define ANA_POL_MODE_FRM_MODE_GET(x)\ + FIELD_GET(ANA_POL_MODE_FRM_MODE, x) + +#define ANA_POL_MODE_OVERSHOOT_ENA BIT(0) +#define ANA_POL_MODE_OVERSHOOT_ENA_SET(x)\ + FIELD_PREP(ANA_POL_MODE_OVERSHOOT_ENA, x) +#define ANA_POL_MODE_OVERSHOOT_ENA_GET(x)\ + FIELD_GET(ANA_POL_MODE_OVERSHOOT_ENA, x) + +/* ANA:POL:POL_PIR_STATE */ +#define ANA_POL_PIR_STATE(g) __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 12, 0, 1, 4) + +#define ANA_POL_PIR_STATE_PIR_LVL GENMASK(21, 0) +#define ANA_POL_PIR_STATE_PIR_LVL_SET(x)\ + FIELD_PREP(ANA_POL_PIR_STATE_PIR_LVL, x) +#define ANA_POL_PIR_STATE_PIR_LVL_GET(x)\ + FIELD_GET(ANA_POL_PIR_STATE_PIR_LVL, x) + /* CHIP_TOP:CUPHY_CFG:CUPHY_PORT_CFG */ #define CHIP_TOP_CUPHY_PORT_CFG(r) __REG(TARGET_CHIP_TOP, 0, 1, 16, 0, 1, 20, 8, r, 2, 4) @@ -504,6 +645,12 @@ enum lan966x_target { #define DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_GET(x)\ FIELD_GET(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, x) +#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA BIT(1) +#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x) +#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x) + /* DEV:PCS1G_CFG_STATUS:PCS1G_SD_CFG */ #define DEV_PCS1G_SD_CFG(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 8, 0, 1, 4) @@ -967,6 +1114,215 @@ enum lan966x_target { /* QSYS:RES_CTRL:RES_CFG */ #define QSYS_RES_CFG(g) __REG(TARGET_QSYS, 0, 1, 32768, g, 1024, 8, 0, 0, 1, 4) +/* QSYS:HSCH:CIR_CFG */ +#define QSYS_CIR_CFG(g) __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 0, 0, 1, 4) + +#define QSYS_CIR_CFG_CIR_RATE GENMASK(20, 6) +#define QSYS_CIR_CFG_CIR_RATE_SET(x)\ + FIELD_PREP(QSYS_CIR_CFG_CIR_RATE, x) +#define QSYS_CIR_CFG_CIR_RATE_GET(x)\ + FIELD_GET(QSYS_CIR_CFG_CIR_RATE, x) + +#define QSYS_CIR_CFG_CIR_BURST GENMASK(5, 0) +#define QSYS_CIR_CFG_CIR_BURST_SET(x)\ + FIELD_PREP(QSYS_CIR_CFG_CIR_BURST, x) +#define QSYS_CIR_CFG_CIR_BURST_GET(x)\ + FIELD_GET(QSYS_CIR_CFG_CIR_BURST, x) + +/* QSYS:HSCH:SE_CFG */ +#define QSYS_SE_CFG(g) __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 8, 0, 1, 4) + +#define QSYS_SE_CFG_SE_DWRR_CNT GENMASK(9, 6) +#define QSYS_SE_CFG_SE_DWRR_CNT_SET(x)\ + FIELD_PREP(QSYS_SE_CFG_SE_DWRR_CNT, x) +#define QSYS_SE_CFG_SE_DWRR_CNT_GET(x)\ + FIELD_GET(QSYS_SE_CFG_SE_DWRR_CNT, x) + +#define QSYS_SE_CFG_SE_RR_ENA BIT(5) +#define QSYS_SE_CFG_SE_RR_ENA_SET(x)\ + FIELD_PREP(QSYS_SE_CFG_SE_RR_ENA, x) +#define QSYS_SE_CFG_SE_RR_ENA_GET(x)\ + FIELD_GET(QSYS_SE_CFG_SE_RR_ENA, x) + +#define QSYS_SE_CFG_SE_AVB_ENA BIT(4) +#define QSYS_SE_CFG_SE_AVB_ENA_SET(x)\ + FIELD_PREP(QSYS_SE_CFG_SE_AVB_ENA, x) +#define QSYS_SE_CFG_SE_AVB_ENA_GET(x)\ + FIELD_GET(QSYS_SE_CFG_SE_AVB_ENA, x) + +#define QSYS_SE_CFG_SE_FRM_MODE GENMASK(3, 2) +#define QSYS_SE_CFG_SE_FRM_MODE_SET(x)\ + FIELD_PREP(QSYS_SE_CFG_SE_FRM_MODE, x) +#define QSYS_SE_CFG_SE_FRM_MODE_GET(x)\ + FIELD_GET(QSYS_SE_CFG_SE_FRM_MODE, x) + +#define QSYS_SE_DWRR_CFG(g, r) __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 12, r, 12, 4) + +#define QSYS_SE_DWRR_CFG_DWRR_COST GENMASK(4, 0) +#define QSYS_SE_DWRR_CFG_DWRR_COST_SET(x)\ + FIELD_PREP(QSYS_SE_DWRR_CFG_DWRR_COST, x) +#define QSYS_SE_DWRR_CFG_DWRR_COST_GET(x)\ + FIELD_GET(QSYS_SE_DWRR_CFG_DWRR_COST, x) + +/* QSYS:TAS_CONFIG:TAS_CFG_CTRL */ +#define QSYS_TAS_CFG_CTRL __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 0, 0, 1, 4) + +#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX GENMASK(27, 23) +#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_SET(x)\ + FIELD_PREP(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX, x) +#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_GET(x)\ + FIELD_GET(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX, x) + +#define QSYS_TAS_CFG_CTRL_LIST_NUM GENMASK(22, 18) +#define QSYS_TAS_CFG_CTRL_LIST_NUM_SET(x)\ + FIELD_PREP(QSYS_TAS_CFG_CTRL_LIST_NUM, x) +#define QSYS_TAS_CFG_CTRL_LIST_NUM_GET(x)\ + FIELD_GET(QSYS_TAS_CFG_CTRL_LIST_NUM, x) + +#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q BIT(17) +#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_SET(x)\ + FIELD_PREP(QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q, x) +#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_GET(x)\ + FIELD_GET(QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q, x) + +#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM GENMASK(16, 5) +#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(x)\ + FIELD_PREP(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, x) +#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_GET(x)\ + FIELD_GET(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, x) + +/* QSYS:TAS_CONFIG:TAS_GATE_STATE_CTRL */ +#define QSYS_TAS_GS_CTRL __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 4, 0, 1, 4) + +#define QSYS_TAS_GS_CTRL_HSCH_POS GENMASK(2, 0) +#define QSYS_TAS_GS_CTRL_HSCH_POS_SET(x)\ + FIELD_PREP(QSYS_TAS_GS_CTRL_HSCH_POS, x) +#define QSYS_TAS_GS_CTRL_HSCH_POS_GET(x)\ + FIELD_GET(QSYS_TAS_GS_CTRL_HSCH_POS, x) + +/* QSYS:TAS_CONFIG:TAS_STATEMACHINE_CFG */ +#define QSYS_TAS_STM_CFG __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 8, 0, 1, 4) + +#define QSYS_TAS_STM_CFG_REVISIT_DLY GENMASK(7, 0) +#define QSYS_TAS_STM_CFG_REVISIT_DLY_SET(x)\ + FIELD_PREP(QSYS_TAS_STM_CFG_REVISIT_DLY, x) +#define QSYS_TAS_STM_CFG_REVISIT_DLY_GET(x)\ + FIELD_GET(QSYS_TAS_STM_CFG_REVISIT_DLY, x) + +/* QSYS:TAS_PROFILE_CFG:TAS_PROFILE_CONFIG */ +#define QSYS_TAS_PROFILE_CFG(g) __REG(TARGET_QSYS, 0, 1, 30720, g, 16, 64, 32, 0, 1, 4) + +#define QSYS_TAS_PROFILE_CFG_PORT_NUM GENMASK(21, 19) +#define QSYS_TAS_PROFILE_CFG_PORT_NUM_SET(x)\ + FIELD_PREP(QSYS_TAS_PROFILE_CFG_PORT_NUM, x) +#define QSYS_TAS_PROFILE_CFG_PORT_NUM_GET(x)\ + FIELD_GET(QSYS_TAS_PROFILE_CFG_PORT_NUM, x) + +#define QSYS_TAS_PROFILE_CFG_LINK_SPEED GENMASK(18, 16) +#define QSYS_TAS_PROFILE_CFG_LINK_SPEED_SET(x)\ + FIELD_PREP(QSYS_TAS_PROFILE_CFG_LINK_SPEED, x) +#define QSYS_TAS_PROFILE_CFG_LINK_SPEED_GET(x)\ + FIELD_GET(QSYS_TAS_PROFILE_CFG_LINK_SPEED, x) + +/* QSYS:TAS_LIST_CFG:TAS_BASE_TIME_NSEC */ +#define QSYS_TAS_BT_NSEC __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 0, 0, 1, 4) + +#define QSYS_TAS_BT_NSEC_NSEC GENMASK(29, 0) +#define QSYS_TAS_BT_NSEC_NSEC_SET(x)\ + FIELD_PREP(QSYS_TAS_BT_NSEC_NSEC, x) +#define QSYS_TAS_BT_NSEC_NSEC_GET(x)\ + FIELD_GET(QSYS_TAS_BT_NSEC_NSEC, x) + +/* QSYS:TAS_LIST_CFG:TAS_BASE_TIME_SEC_LSB */ +#define QSYS_TAS_BT_SEC_LSB __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 4, 0, 1, 4) + +/* QSYS:TAS_LIST_CFG:TAS_BASE_TIME_SEC_MSB */ +#define QSYS_TAS_BT_SEC_MSB __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 8, 0, 1, 4) + +#define QSYS_TAS_BT_SEC_MSB_SEC_MSB GENMASK(15, 0) +#define QSYS_TAS_BT_SEC_MSB_SEC_MSB_SET(x)\ + FIELD_PREP(QSYS_TAS_BT_SEC_MSB_SEC_MSB, x) +#define QSYS_TAS_BT_SEC_MSB_SEC_MSB_GET(x)\ + FIELD_GET(QSYS_TAS_BT_SEC_MSB_SEC_MSB, x) + +/* QSYS:TAS_LIST_CFG:TAS_CYCLE_TIME_CFG */ +#define QSYS_TAS_CT_CFG __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 24, 0, 1, 4) + +/* QSYS:TAS_LIST_CFG:TAS_STARTUP_CFG */ +#define QSYS_TAS_STARTUP_CFG __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 28, 0, 1, 4) + +#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX GENMASK(27, 23) +#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_SET(x)\ + FIELD_PREP(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX, x) +#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_GET(x)\ + FIELD_GET(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX, x) + +/* QSYS:TAS_LIST_CFG:TAS_LIST_CFG */ +#define QSYS_TAS_LIST_CFG __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 32, 0, 1, 4) + +#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR GENMASK(11, 0) +#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_SET(x)\ + FIELD_PREP(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR, x) +#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_GET(x)\ + FIELD_GET(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR, x) + +/* QSYS:TAS_LIST_CFG:TAS_LIST_STATE */ +#define QSYS_TAS_LST __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 36, 0, 1, 4) + +#define QSYS_TAS_LST_LIST_STATE GENMASK(2, 0) +#define QSYS_TAS_LST_LIST_STATE_SET(x)\ + FIELD_PREP(QSYS_TAS_LST_LIST_STATE, x) +#define QSYS_TAS_LST_LIST_STATE_GET(x)\ + FIELD_GET(QSYS_TAS_LST_LIST_STATE, x) + +/* QSYS:TAS_GCL_CFG:TAS_GCL_CTRL_CFG */ +#define QSYS_TAS_GCL_CT_CFG __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 0, 0, 1, 4) + +#define QSYS_TAS_GCL_CT_CFG_HSCH_POS GENMASK(12, 10) +#define QSYS_TAS_GCL_CT_CFG_HSCH_POS_SET(x)\ + FIELD_PREP(QSYS_TAS_GCL_CT_CFG_HSCH_POS, x) +#define QSYS_TAS_GCL_CT_CFG_HSCH_POS_GET(x)\ + FIELD_GET(QSYS_TAS_GCL_CT_CFG_HSCH_POS, x) + +#define QSYS_TAS_GCL_CT_CFG_GATE_STATE GENMASK(9, 2) +#define QSYS_TAS_GCL_CT_CFG_GATE_STATE_SET(x)\ + FIELD_PREP(QSYS_TAS_GCL_CT_CFG_GATE_STATE, x) +#define QSYS_TAS_GCL_CT_CFG_GATE_STATE_GET(x)\ + FIELD_GET(QSYS_TAS_GCL_CT_CFG_GATE_STATE, x) + +#define QSYS_TAS_GCL_CT_CFG_OP_TYPE GENMASK(1, 0) +#define QSYS_TAS_GCL_CT_CFG_OP_TYPE_SET(x)\ + FIELD_PREP(QSYS_TAS_GCL_CT_CFG_OP_TYPE, x) +#define QSYS_TAS_GCL_CT_CFG_OP_TYPE_GET(x)\ + FIELD_GET(QSYS_TAS_GCL_CT_CFG_OP_TYPE, x) + +/* QSYS:TAS_GCL_CFG:TAS_GCL_CTRL_CFG2 */ +#define QSYS_TAS_GCL_CT_CFG2 __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 4, 0, 1, 4) + +#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE GENMASK(15, 12) +#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_SET(x)\ + FIELD_PREP(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE, x) +#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_GET(x)\ + FIELD_GET(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE, x) + +#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL GENMASK(11, 0) +#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_SET(x)\ + FIELD_PREP(QSYS_TAS_GCL_CT_CFG2_NEXT_GCL, x) +#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_GET(x)\ + FIELD_GET(QSYS_TAS_GCL_CT_CFG2_NEXT_GCL, x) + +/* QSYS:TAS_GCL_CFG:TAS_GCL_TIME_CFG */ +#define QSYS_TAS_GCL_TM_CFG __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 8, 0, 1, 4) + +/* QSYS:HSCH_TAS_STATE:TAS_GATE_STATE */ +#define QSYS_TAS_GATE_STATE __REG(TARGET_QSYS, 0, 1, 28004, 0, 1, 4, 0, 0, 1, 4) + +#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE GENMASK(7, 0) +#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE_SET(x)\ + FIELD_PREP(QSYS_TAS_GATE_STATE_TAS_GATE_STATE, x) +#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE_GET(x)\ + FIELD_GET(QSYS_TAS_GATE_STATE_TAS_GATE_STATE, x) + /* REW:PORT:PORT_VLAN_CFG */ #define REW_PORT_VLAN_CFG(g) __REG(TARGET_REW, 0, 1, 0, g, 10, 128, 0, 0, 1, 4) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c index df2bee6785598ac0fb5b2eade1a99d2223e9da79..1c88120eb291a2bc8efc7982c09af4fab5975c0e 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c @@ -6,8 +6,6 @@ #include "lan966x_main.h" static struct notifier_block lan966x_netdevice_nb __read_mostly; -static struct notifier_block lan966x_switchdev_nb __read_mostly; -static struct notifier_block lan966x_switchdev_blocking_nb __read_mostly; static void lan966x_port_set_mcast_ip_flood(struct lan966x_port *port, u32 pgid_ip) @@ -132,7 +130,7 @@ static int lan966x_port_pre_bridge_flags(struct lan966x_port *port, return 0; } -static void lan966x_update_fwd_mask(struct lan966x *lan966x) +void lan966x_update_fwd_mask(struct lan966x *lan966x) { int i; @@ -140,9 +138,14 @@ static void lan966x_update_fwd_mask(struct lan966x *lan966x) struct lan966x_port *port = lan966x->ports[i]; unsigned long mask = 0; - if (port && lan966x->bridge_fwd_mask & BIT(i)) + if (port && lan966x->bridge_fwd_mask & BIT(i)) { mask = lan966x->bridge_fwd_mask & ~BIT(i); + if (port->bond) + mask &= ~lan966x_lag_get_mask(lan966x, + port->bond); + } + mask |= BIT(CPU_PORT); lan_wr(ANA_PGID_PGID_SET(mask), @@ -150,7 +153,7 @@ static void lan966x_update_fwd_mask(struct lan966x *lan966x) } } -static void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state) +void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state) { struct lan966x *lan966x = port->lan966x; bool learn_ena = false; @@ -171,8 +174,8 @@ static void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state) lan966x_update_fwd_mask(lan966x); } -static void lan966x_port_ageing_set(struct lan966x_port *port, - unsigned long ageing_clock_t) +void lan966x_port_ageing_set(struct lan966x_port *port, + unsigned long ageing_clock_t) { unsigned long ageing_jiffies = clock_t_to_jiffies(ageing_clock_t); u32 ageing_time = jiffies_to_msecs(ageing_jiffies) / 1000; @@ -241,6 +244,7 @@ static int lan966x_port_attr_set(struct net_device *dev, const void *ctx, } static int lan966x_port_bridge_join(struct lan966x_port *port, + struct net_device *brport_dev, struct net_device *bridge, struct netlink_ext_ack *extack) { @@ -258,7 +262,7 @@ static int lan966x_port_bridge_join(struct lan966x_port *port, } } - err = switchdev_bridge_port_offload(dev, dev, port, + err = switchdev_bridge_port_offload(brport_dev, dev, port, &lan966x_switchdev_nb, &lan966x_switchdev_blocking_nb, false, extack); @@ -295,8 +299,9 @@ static void lan966x_port_bridge_leave(struct lan966x_port *port, lan966x_vlan_port_apply(port); } -static int lan966x_port_changeupper(struct net_device *dev, - struct netdev_notifier_changeupper_info *info) +int lan966x_port_changeupper(struct net_device *dev, + struct net_device *brport_dev, + struct netdev_notifier_changeupper_info *info) { struct lan966x_port *port = netdev_priv(dev); struct netlink_ext_ack *extack; @@ -306,44 +311,68 @@ static int lan966x_port_changeupper(struct net_device *dev, if (netif_is_bridge_master(info->upper_dev)) { if (info->linking) - err = lan966x_port_bridge_join(port, info->upper_dev, + err = lan966x_port_bridge_join(port, brport_dev, + info->upper_dev, extack); else lan966x_port_bridge_leave(port, info->upper_dev); } + if (netif_is_lag_master(info->upper_dev)) { + if (info->linking) + err = lan966x_lag_port_join(port, info->upper_dev, + info->upper_dev, + extack); + else + lan966x_lag_port_leave(port, info->upper_dev); + } + return err; } -static int lan966x_port_prechangeupper(struct net_device *dev, - struct netdev_notifier_changeupper_info *info) +int lan966x_port_prechangeupper(struct net_device *dev, + struct net_device *brport_dev, + struct netdev_notifier_changeupper_info *info) { struct lan966x_port *port = netdev_priv(dev); + int err = NOTIFY_DONE; - if (netif_is_bridge_master(info->upper_dev) && !info->linking) - switchdev_bridge_port_unoffload(port->dev, port, - NULL, NULL); + if (netif_is_bridge_master(info->upper_dev) && !info->linking) { + switchdev_bridge_port_unoffload(port->dev, port, NULL, NULL); + lan966x_fdb_flush_workqueue(port->lan966x); + } - return NOTIFY_DONE; + if (netif_is_lag_master(info->upper_dev)) { + err = lan966x_lag_port_prechangeupper(dev, info); + if (err || info->linking) + return err; + + switchdev_bridge_port_unoffload(brport_dev, port, NULL, NULL); + lan966x_fdb_flush_workqueue(port->lan966x); + } + + return err; } -static int lan966x_foreign_bridging_check(struct net_device *bridge, +static int lan966x_foreign_bridging_check(struct net_device *upper, + bool *has_foreign, + bool *seen_lan966x, struct netlink_ext_ack *extack) { struct lan966x *lan966x = NULL; - bool has_foreign = false; struct net_device *dev; struct list_head *iter; - if (!netif_is_bridge_master(bridge)) + if (!netif_is_bridge_master(upper) && + !netif_is_lag_master(upper)) return 0; - netdev_for_each_lower_dev(bridge, dev, iter) { + netdev_for_each_lower_dev(upper, dev, iter) { if (lan966x_netdevice_check(dev)) { struct lan966x_port *port = netdev_priv(dev); if (lan966x) { - /* Bridge already has at least one port of a + /* Upper already has at least one port of a * lan966x switch inside it, check that it's * the same instance of the driver. */ @@ -354,15 +383,24 @@ static int lan966x_foreign_bridging_check(struct net_device *bridge, } } else { /* This is the first lan966x port inside this - * bridge + * upper device */ lan966x = port->lan966x; + *seen_lan966x = true; } + } else if (netif_is_lag_master(dev)) { + /* Allow to have bond interfaces that have only lan966x + * devices + */ + if (lan966x_foreign_bridging_check(dev, has_foreign, + seen_lan966x, + extack)) + return -EINVAL; } else { - has_foreign = true; + *has_foreign = true; } - if (lan966x && has_foreign) { + if (*seen_lan966x && *has_foreign) { NL_SET_ERR_MSG_MOD(extack, "Bridging lan966x ports with foreign interfaces disallowed"); return -EINVAL; @@ -375,7 +413,12 @@ static int lan966x_foreign_bridging_check(struct net_device *bridge, static int lan966x_bridge_check(struct net_device *dev, struct netdev_notifier_changeupper_info *info) { + bool has_foreign = false; + bool seen_lan966x = false; + return lan966x_foreign_bridging_check(info->upper_dev, + &has_foreign, + &seen_lan966x, info->info.extack); } @@ -386,21 +429,44 @@ static int lan966x_netdevice_port_event(struct net_device *dev, int err = 0; if (!lan966x_netdevice_check(dev)) { - if (event == NETDEV_CHANGEUPPER) - return lan966x_bridge_check(dev, ptr); + switch (event) { + case NETDEV_CHANGEUPPER: + case NETDEV_PRECHANGEUPPER: + err = lan966x_bridge_check(dev, ptr); + if (err) + return err; + + if (netif_is_lag_master(dev)) { + if (event == NETDEV_CHANGEUPPER) + err = lan966x_lag_netdev_changeupper(dev, + ptr); + else + err = lan966x_lag_netdev_prechangeupper(dev, + ptr); + + return err; + } + break; + default: + return 0; + } + return 0; } switch (event) { case NETDEV_PRECHANGEUPPER: - err = lan966x_port_prechangeupper(dev, ptr); + err = lan966x_port_prechangeupper(dev, dev, ptr); break; case NETDEV_CHANGEUPPER: err = lan966x_bridge_check(dev, ptr); if (err) return err; - err = lan966x_port_changeupper(dev, ptr); + err = lan966x_port_changeupper(dev, dev, ptr); + break; + case NETDEV_CHANGELOWERSTATE: + err = lan966x_lag_port_changelowerstate(dev, ptr); break; } @@ -418,19 +484,23 @@ static int lan966x_netdevice_event(struct notifier_block *nb, return notifier_from_errno(ret); } -/* We don't offload uppers such as LAG as bridge ports, so every device except - * the bridge itself is foreign. - */ static bool lan966x_foreign_dev_check(const struct net_device *dev, const struct net_device *foreign_dev) { struct lan966x_port *port = netdev_priv(dev); struct lan966x *lan966x = port->lan966x; + int i; if (netif_is_bridge_master(foreign_dev)) if (lan966x->bridge == foreign_dev) return false; + if (netif_is_lag_master(foreign_dev)) + for (i = 0; i < lan966x->num_phys_ports; ++i) + if (lan966x->ports[i] && + lan966x->ports[i]->bond == foreign_dev) + return false; + return true; } @@ -571,11 +641,11 @@ static struct notifier_block lan966x_netdevice_nb __read_mostly = { .notifier_call = lan966x_netdevice_event, }; -static struct notifier_block lan966x_switchdev_nb __read_mostly = { +struct notifier_block lan966x_switchdev_nb __read_mostly = { .notifier_call = lan966x_switchdev_event, }; -static struct notifier_block lan966x_switchdev_blocking_nb __read_mostly = { +struct notifier_block lan966x_switchdev_blocking_nb __read_mostly = { .notifier_call = lan966x_switchdev_blocking_event, }; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c b/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c new file mode 100644 index 0000000000000000000000000000000000000000..3f5b212066c5c6957af022b12c038bba2bd8b342 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +#define LAN966X_TAPRIO_TIMEOUT_MS 1000 +#define LAN966X_TAPRIO_ENTRIES_PER_PORT 2 + +/* Minimum supported cycle time in nanoseconds */ +#define LAN966X_TAPRIO_MIN_CYCLE_TIME_NS NSEC_PER_USEC + +/* Maximum supported cycle time in nanoseconds */ +#define LAN966X_TAPRIO_MAX_CYCLE_TIME_NS (NSEC_PER_SEC - 1) + +/* Total number of TAS GCL entries */ +#define LAN966X_TAPRIO_NUM_GCL 256 + +/* TAPRIO link speeds for calculation of guard band */ +enum lan966x_taprio_link_speed { + LAN966X_TAPRIO_SPEED_NO_GB, + LAN966X_TAPRIO_SPEED_10, + LAN966X_TAPRIO_SPEED_100, + LAN966X_TAPRIO_SPEED_1000, + LAN966X_TAPRIO_SPEED_2500, +}; + +/* TAPRIO list states */ +enum lan966x_taprio_state { + LAN966X_TAPRIO_STATE_ADMIN, + LAN966X_TAPRIO_STATE_ADVANCING, + LAN966X_TAPRIO_STATE_PENDING, + LAN966X_TAPRIO_STATE_OPERATING, + LAN966X_TAPRIO_STATE_TERMINATING, + LAN966X_TAPRIO_STATE_MAX, +}; + +/* TAPRIO GCL command */ +enum lan966x_taprio_gcl_cmd { + LAN966X_TAPRIO_GCL_CMD_SET_GATE_STATES = 0, +}; + +static u32 lan966x_taprio_list_index(struct lan966x_port *port, u8 entry) +{ + return port->chip_port * LAN966X_TAPRIO_ENTRIES_PER_PORT + entry; +} + +static u32 lan966x_taprio_list_state_get(struct lan966x_port *port) +{ + struct lan966x *lan966x = port->lan966x; + u32 val; + + val = lan_rd(lan966x, QSYS_TAS_LST); + return QSYS_TAS_LST_LIST_STATE_GET(val); +} + +static u32 lan966x_taprio_list_index_state_get(struct lan966x_port *port, + u32 list) +{ + struct lan966x *lan966x = port->lan966x; + + lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_SET(list), + QSYS_TAS_CFG_CTRL_LIST_NUM, + lan966x, QSYS_TAS_CFG_CTRL); + + return lan966x_taprio_list_state_get(port); +} + +static void lan966x_taprio_list_state_set(struct lan966x_port *port, + u32 state) +{ + struct lan966x *lan966x = port->lan966x; + + lan_rmw(QSYS_TAS_LST_LIST_STATE_SET(state), + QSYS_TAS_LST_LIST_STATE, + lan966x, QSYS_TAS_LST); +} + +static int lan966x_taprio_list_shutdown(struct lan966x_port *port, + u32 list) +{ + struct lan966x *lan966x = port->lan966x; + bool pending, operating; + unsigned long end; + u32 state; + + end = jiffies + msecs_to_jiffies(LAN966X_TAPRIO_TIMEOUT_MS); + /* It is required to try multiple times to set the state of list, + * because the HW can overwrite this. + */ + do { + state = lan966x_taprio_list_state_get(port); + + pending = false; + operating = false; + + if (state == LAN966X_TAPRIO_STATE_ADVANCING || + state == LAN966X_TAPRIO_STATE_PENDING) { + lan966x_taprio_list_state_set(port, + LAN966X_TAPRIO_STATE_ADMIN); + pending = true; + } + + if (state == LAN966X_TAPRIO_STATE_OPERATING) { + lan966x_taprio_list_state_set(port, + LAN966X_TAPRIO_STATE_TERMINATING); + operating = true; + } + + /* If the entry was in pending and now gets in admin, then there + * is nothing else to do, so just bail out + */ + state = lan966x_taprio_list_state_get(port); + if (pending && + state == LAN966X_TAPRIO_STATE_ADMIN) + return 0; + + /* If the list was in operating and now is in terminating or + * admin, then is OK to exit but it needs to wait until the list + * will get in admin. It is not required to set the state + * again. + */ + if (operating && + (state == LAN966X_TAPRIO_STATE_TERMINATING || + state == LAN966X_TAPRIO_STATE_ADMIN)) + break; + + } while (!time_after(jiffies, end)); + + end = jiffies + msecs_to_jiffies(LAN966X_TAPRIO_TIMEOUT_MS); + do { + state = lan966x_taprio_list_state_get(port); + if (state == LAN966X_TAPRIO_STATE_ADMIN) + break; + + } while (!time_after(jiffies, end)); + + /* If the list was in operating mode, it could be stopped while some + * queues where closed, so make sure to restore "all-queues-open" + */ + if (operating) { + lan_wr(QSYS_TAS_GS_CTRL_HSCH_POS_SET(port->chip_port), + lan966x, QSYS_TAS_GS_CTRL); + + lan_wr(QSYS_TAS_GATE_STATE_TAS_GATE_STATE_SET(0xff), + lan966x, QSYS_TAS_GATE_STATE); + } + + return 0; +} + +static int lan966x_taprio_shutdown(struct lan966x_port *port) +{ + u32 i, list, state; + int err; + + for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) { + list = lan966x_taprio_list_index(port, i); + state = lan966x_taprio_list_index_state_get(port, list); + if (state == LAN966X_TAPRIO_STATE_ADMIN) + continue; + + err = lan966x_taprio_list_shutdown(port, list); + if (err) + return err; + } + + return 0; +} + +/* Find a suitable list for a new schedule. First priority is a list in state + * pending. Second priority is a list in state admin. + */ +static int lan966x_taprio_find_list(struct lan966x_port *port, + struct tc_taprio_qopt_offload *qopt, + int *new_list, int *obs_list) +{ + int state[LAN966X_TAPRIO_ENTRIES_PER_PORT]; + int list[LAN966X_TAPRIO_ENTRIES_PER_PORT]; + int err, oper = -1; + u32 i; + + *new_list = -1; + *obs_list = -1; + + /* If there is already an entry in operating mode, return this list in + * obs_list, such that when the new list will get activated the + * operating list will be stopped. In this way is possible to have + * smooth transitions between the lists + */ + for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) { + list[i] = lan966x_taprio_list_index(port, i); + state[i] = lan966x_taprio_list_index_state_get(port, list[i]); + if (state[i] == LAN966X_TAPRIO_STATE_OPERATING) + oper = list[i]; + } + + for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) { + if (state[i] == LAN966X_TAPRIO_STATE_PENDING) { + err = lan966x_taprio_shutdown(port); + if (err) + return err; + + *new_list = list[i]; + *obs_list = (oper == -1) ? *new_list : oper; + return 0; + } + } + + for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) { + if (state[i] == LAN966X_TAPRIO_STATE_ADMIN) { + *new_list = list[i]; + *obs_list = (oper == -1) ? *new_list : oper; + return 0; + } + } + + return -ENOSPC; +} + +static int lan966x_taprio_check(struct tc_taprio_qopt_offload *qopt) +{ + u64 total_time = 0; + u32 i; + + /* This is not supported by th HW */ + if (qopt->cycle_time_extension) + return -EOPNOTSUPP; + + /* There is a limited number of gcl entries that can be used, they are + * shared by all ports + */ + if (qopt->num_entries > LAN966X_TAPRIO_NUM_GCL) + return -EINVAL; + + /* Don't allow cycle times bigger than 1 sec or smaller than 1 usec */ + if (qopt->cycle_time < LAN966X_TAPRIO_MIN_CYCLE_TIME_NS || + qopt->cycle_time > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS) + return -EINVAL; + + for (i = 0; i < qopt->num_entries; ++i) { + struct tc_taprio_sched_entry *entry = &qopt->entries[i]; + + /* Don't allow intervals bigger than 1 sec or smaller than 1 + * usec + */ + if (entry->interval < LAN966X_TAPRIO_MIN_CYCLE_TIME_NS || + entry->interval > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS) + return -EINVAL; + + if (qopt->entries[i].command != TC_TAPRIO_CMD_SET_GATES) + return -EINVAL; + + total_time += qopt->entries[i].interval; + } + + /* Don't allow the total time of intervals be bigger than 1 sec */ + if (total_time > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS) + return -EINVAL; + + /* The HW expects that the cycle time to be at least as big as sum of + * each interval of gcl + */ + if (qopt->cycle_time < total_time) + return -EINVAL; + + return 0; +} + +static int lan966x_taprio_gcl_free_get(struct lan966x_port *port, + unsigned long *free_list) +{ + struct lan966x *lan966x = port->lan966x; + u32 num_free, state, list; + u32 base, next, max_list; + + /* By default everything is free */ + bitmap_fill(free_list, LAN966X_TAPRIO_NUM_GCL); + num_free = LAN966X_TAPRIO_NUM_GCL; + + /* Iterate over all gcl entries and find out which are free. And mark + * those that are not free. + */ + max_list = lan966x->num_phys_ports * LAN966X_TAPRIO_ENTRIES_PER_PORT; + for (list = 0; list < max_list; ++list) { + state = lan966x_taprio_list_index_state_get(port, list); + if (state == LAN966X_TAPRIO_STATE_ADMIN) + continue; + + base = lan_rd(lan966x, QSYS_TAS_LIST_CFG); + base = QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_GET(base); + next = base; + + do { + clear_bit(next, free_list); + num_free--; + + lan_rmw(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(next), + QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, + lan966x, QSYS_TAS_CFG_CTRL); + + next = lan_rd(lan966x, QSYS_TAS_GCL_CT_CFG2); + next = QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_GET(next); + } while (base != next); + } + + return num_free; +} + +static void lan966x_taprio_gcl_setup_entry(struct lan966x_port *port, + struct tc_taprio_sched_entry *entry, + u32 next_entry) +{ + struct lan966x *lan966x = port->lan966x; + + /* Setup a single gcl entry */ + lan_wr(QSYS_TAS_GCL_CT_CFG_GATE_STATE_SET(entry->gate_mask) | + QSYS_TAS_GCL_CT_CFG_HSCH_POS_SET(port->chip_port) | + QSYS_TAS_GCL_CT_CFG_OP_TYPE_SET(LAN966X_TAPRIO_GCL_CMD_SET_GATE_STATES), + lan966x, QSYS_TAS_GCL_CT_CFG); + + lan_wr(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_SET(port->chip_port) | + QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_SET(next_entry), + lan966x, QSYS_TAS_GCL_CT_CFG2); + + lan_wr(entry->interval, lan966x, QSYS_TAS_GCL_TM_CFG); +} + +static int lan966x_taprio_gcl_setup(struct lan966x_port *port, + struct tc_taprio_qopt_offload *qopt, + int list) +{ + DECLARE_BITMAP(free_list, LAN966X_TAPRIO_NUM_GCL); + struct lan966x *lan966x = port->lan966x; + u32 i, base, next; + + if (lan966x_taprio_gcl_free_get(port, free_list) < qopt->num_entries) + return -ENOSPC; + + /* Select list */ + lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_SET(list), + QSYS_TAS_CFG_CTRL_LIST_NUM, + lan966x, QSYS_TAS_CFG_CTRL); + + /* Setup the address of the first gcl entry */ + base = find_first_bit(free_list, LAN966X_TAPRIO_NUM_GCL); + lan_rmw(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_SET(base), + QSYS_TAS_LIST_CFG_LIST_BASE_ADDR, + lan966x, QSYS_TAS_LIST_CFG); + + /* Iterate over entries and add them to the gcl list */ + next = base; + for (i = 0; i < qopt->num_entries; ++i) { + lan_rmw(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(next), + QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, + lan966x, QSYS_TAS_CFG_CTRL); + + /* If the entry is last, point back to the start of the list */ + if (i == qopt->num_entries - 1) + next = base; + else + next = find_next_bit(free_list, LAN966X_TAPRIO_NUM_GCL, + next + 1); + + lan966x_taprio_gcl_setup_entry(port, &qopt->entries[i], next); + } + + return 0; +} + +/* Calculate new base_time based on cycle_time. The HW recommends to have the + * new base time at least 2 * cycle type + current time + */ +static void lan966x_taprio_new_base_time(struct lan966x *lan966x, + const u32 cycle_time, + const ktime_t org_base_time, + ktime_t *new_base_time) +{ + ktime_t current_time, threshold_time; + struct timespec64 ts; + + /* Get the current time and calculate the threshold_time */ + lan966x_ptp_gettime64(&lan966x->phc[LAN966X_PHC_PORT].info, &ts); + current_time = timespec64_to_ktime(ts); + threshold_time = current_time + (2 * cycle_time); + + /* If the org_base_time is in enough in future just use it */ + if (org_base_time >= threshold_time) { + *new_base_time = org_base_time; + return; + } + + /* If the org_base_time is smaller than current_time, calculate the new + * base time as following. + */ + if (org_base_time <= current_time) { + u64 tmp = current_time - org_base_time; + u32 rem = 0; + + if (tmp > cycle_time) + div_u64_rem(tmp, cycle_time, &rem); + rem = cycle_time - rem; + *new_base_time = threshold_time + rem; + return; + } + + /* The only left place for org_base_time is between current_time and + * threshold_time. In this case the new_base_time is calculated like + * org_base_time + 2 * cycletime + */ + *new_base_time = org_base_time + 2 * cycle_time; +} + +int lan966x_taprio_speed_set(struct lan966x_port *port, int speed) +{ + struct lan966x *lan966x = port->lan966x; + u8 taprio_speed; + + switch (speed) { + case SPEED_10: + taprio_speed = LAN966X_TAPRIO_SPEED_10; + break; + case SPEED_100: + taprio_speed = LAN966X_TAPRIO_SPEED_100; + break; + case SPEED_1000: + taprio_speed = LAN966X_TAPRIO_SPEED_1000; + break; + case SPEED_2500: + taprio_speed = LAN966X_TAPRIO_SPEED_2500; + break; + default: + return -EINVAL; + } + + lan_rmw(QSYS_TAS_PROFILE_CFG_LINK_SPEED_SET(taprio_speed), + QSYS_TAS_PROFILE_CFG_LINK_SPEED, + lan966x, QSYS_TAS_PROFILE_CFG(port->chip_port)); + + return 0; +} + +int lan966x_taprio_add(struct lan966x_port *port, + struct tc_taprio_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + int err, new_list, obs_list; + struct timespec64 ts; + ktime_t base_time; + + err = lan966x_taprio_check(qopt); + if (err) + return err; + + err = lan966x_taprio_find_list(port, qopt, &new_list, &obs_list); + if (err) + return err; + + err = lan966x_taprio_gcl_setup(port, qopt, new_list); + if (err) + return err; + + lan966x_taprio_new_base_time(lan966x, qopt->cycle_time, + qopt->base_time, &base_time); + + ts = ktime_to_timespec64(base_time); + lan_wr(QSYS_TAS_BT_NSEC_NSEC_SET(ts.tv_nsec), + lan966x, QSYS_TAS_BT_NSEC); + + lan_wr(lower_32_bits(ts.tv_sec), + lan966x, QSYS_TAS_BT_SEC_LSB); + + lan_wr(QSYS_TAS_BT_SEC_MSB_SEC_MSB_SET(upper_32_bits(ts.tv_sec)), + lan966x, QSYS_TAS_BT_SEC_MSB); + + lan_wr(qopt->cycle_time, lan966x, QSYS_TAS_CT_CFG); + + lan_rmw(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_SET(obs_list), + QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX, + lan966x, QSYS_TAS_STARTUP_CFG); + + /* Start list processing */ + lan_rmw(QSYS_TAS_LST_LIST_STATE_SET(LAN966X_TAPRIO_STATE_ADVANCING), + QSYS_TAS_LST_LIST_STATE, + lan966x, QSYS_TAS_LST); + + return err; +} + +int lan966x_taprio_del(struct lan966x_port *port) +{ + return lan966x_taprio_shutdown(port); +} + +void lan966x_taprio_init(struct lan966x *lan966x) +{ + int num_taprio_lists; + int p; + + lan_wr(QSYS_TAS_STM_CFG_REVISIT_DLY_SET((256 * 1000) / + lan966x_ptp_get_period_ps()), + lan966x, QSYS_TAS_STM_CFG); + + num_taprio_lists = lan966x->num_phys_ports * + LAN966X_TAPRIO_ENTRIES_PER_PORT; + + /* For now we always use guard band on all queues */ + lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_SET(num_taprio_lists) | + QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_SET(1), + QSYS_TAS_CFG_CTRL_LIST_NUM_MAX | + QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q, + lan966x, QSYS_TAS_CFG_CTRL); + + for (p = 0; p < lan966x->num_phys_ports; p++) + lan_rmw(QSYS_TAS_PROFILE_CFG_PORT_NUM_SET(p), + QSYS_TAS_PROFILE_CFG_PORT_NUM, + lan966x, QSYS_TAS_PROFILE_CFG(p)); +} + +void lan966x_taprio_deinit(struct lan966x *lan966x) +{ + int p; + + for (p = 0; p < lan966x->num_phys_ports; ++p) { + if (!lan966x->ports[p]) + continue; + + lan966x_taprio_del(lan966x->ports[p]); + } +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c new file mode 100644 index 0000000000000000000000000000000000000000..4555a35d0d288fbad22840c0a9d91fac3739de97 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +int lan966x_tbf_add(struct lan966x_port *port, + struct tc_tbf_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + bool root = qopt->parent == TC_H_ROOT; + u32 queue = 0; + u32 cir, cbs; + u32 se_idx; + + if (!root) { + queue = TC_H_MIN(qopt->parent) - 1; + if (queue >= NUM_PRIO_QUEUES) + return -EOPNOTSUPP; + } + + if (root) + se_idx = SE_IDX_PORT + port->chip_port; + else + se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + queue; + + cir = div_u64(qopt->replace_params.rate.rate_bytes_ps, 1000) * 8; + cbs = qopt->replace_params.max_size; + + /* Rate unit is 100 kbps */ + cir = DIV_ROUND_UP(cir, 100); + /* Avoid using zero rate */ + cir = cir ?: 1; + /* Burst unit is 4kB */ + cbs = DIV_ROUND_UP(cbs, 4096); + /* Avoid using zero burst */ + cbs = cbs ?: 1; + + /* Check that actually the result can be written */ + if (cir > GENMASK(15, 0) || + cbs > GENMASK(6, 0)) + return -EINVAL; + + lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(0) | + QSYS_SE_CFG_SE_FRM_MODE_SET(1), + QSYS_SE_CFG_SE_AVB_ENA | + QSYS_SE_CFG_SE_FRM_MODE, + lan966x, QSYS_SE_CFG(se_idx)); + + lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(cir) | + QSYS_CIR_CFG_CIR_BURST_SET(cbs), + lan966x, QSYS_CIR_CFG(se_idx)); + + return 0; +} + +int lan966x_tbf_del(struct lan966x_port *port, + struct tc_tbf_qopt_offload *qopt) +{ + struct lan966x *lan966x = port->lan966x; + bool root = qopt->parent == TC_H_ROOT; + u32 queue = 0; + u32 se_idx; + + if (!root) { + queue = TC_H_MIN(qopt->parent) - 1; + if (queue >= NUM_PRIO_QUEUES) + return -EOPNOTSUPP; + } + + if (root) + se_idx = SE_IDX_PORT + port->chip_port; + else + se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + queue; + + lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(0) | + QSYS_SE_CFG_SE_FRM_MODE_SET(0), + QSYS_SE_CFG_SE_AVB_ENA | + QSYS_SE_CFG_SE_FRM_MODE, + lan966x, QSYS_SE_CFG(se_idx)); + + lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(0) | + QSYS_CIR_CFG_CIR_BURST_SET(0), + lan966x, QSYS_CIR_CFG(se_idx)); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c new file mode 100644 index 0000000000000000000000000000000000000000..651d5493ae55b21149b2748bef4345bb61c05c96 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include + +#include "lan966x_main.h" + +static LIST_HEAD(lan966x_tc_block_cb_list); + +static int lan966x_tc_setup_qdisc_mqprio(struct lan966x_port *port, + struct tc_mqprio_qopt_offload *mqprio) +{ + u8 num_tc = mqprio->qopt.num_tc; + + mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS; + + return num_tc ? lan966x_mqprio_add(port, num_tc) : + lan966x_mqprio_del(port); +} + +static int lan966x_tc_setup_qdisc_taprio(struct lan966x_port *port, + struct tc_taprio_qopt_offload *taprio) +{ + return taprio->enable ? lan966x_taprio_add(port, taprio) : + lan966x_taprio_del(port); +} + +static int lan966x_tc_setup_qdisc_tbf(struct lan966x_port *port, + struct tc_tbf_qopt_offload *qopt) +{ + switch (qopt->command) { + case TC_TBF_REPLACE: + return lan966x_tbf_add(port, qopt); + case TC_TBF_DESTROY: + return lan966x_tbf_del(port, qopt); + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; +} + +static int lan966x_tc_setup_qdisc_cbs(struct lan966x_port *port, + struct tc_cbs_qopt_offload *qopt) +{ + return qopt->enable ? lan966x_cbs_add(port, qopt) : + lan966x_cbs_del(port, qopt); +} + +static int lan966x_tc_setup_qdisc_ets(struct lan966x_port *port, + struct tc_ets_qopt_offload *qopt) +{ + switch (qopt->command) { + case TC_ETS_REPLACE: + return lan966x_ets_add(port, qopt); + case TC_ETS_DESTROY: + return lan966x_ets_del(port, qopt); + default: + return -EOPNOTSUPP; + }; + + return -EOPNOTSUPP; +} + +static int lan966x_tc_block_cb(enum tc_setup_type type, void *type_data, + void *cb_priv, bool ingress) +{ + struct lan966x_port *port = cb_priv; + + switch (type) { + case TC_SETUP_CLSMATCHALL: + return lan966x_tc_matchall(port, type_data, ingress); + default: + return -EOPNOTSUPP; + } +} + +static int lan966x_tc_block_cb_ingress(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + return lan966x_tc_block_cb(type, type_data, cb_priv, true); +} + +static int lan966x_tc_block_cb_egress(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + return lan966x_tc_block_cb(type, type_data, cb_priv, false); +} + +static int lan966x_tc_setup_block(struct lan966x_port *port, + struct flow_block_offload *f) +{ + flow_setup_cb_t *cb; + bool ingress; + + if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) { + cb = lan966x_tc_block_cb_ingress; + port->tc.ingress_shared_block = f->block_shared; + ingress = true; + } else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) { + cb = lan966x_tc_block_cb_egress; + ingress = false; + } else { + return -EOPNOTSUPP; + } + + return flow_block_cb_setup_simple(f, &lan966x_tc_block_cb_list, + cb, port, port, ingress); +} + +int lan966x_tc_setup(struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + struct lan966x_port *port = netdev_priv(dev); + + switch (type) { + case TC_SETUP_QDISC_MQPRIO: + return lan966x_tc_setup_qdisc_mqprio(port, type_data); + case TC_SETUP_QDISC_TAPRIO: + return lan966x_tc_setup_qdisc_taprio(port, type_data); + case TC_SETUP_QDISC_TBF: + return lan966x_tc_setup_qdisc_tbf(port, type_data); + case TC_SETUP_QDISC_CBS: + return lan966x_tc_setup_qdisc_cbs(port, type_data); + case TC_SETUP_QDISC_ETS: + return lan966x_tc_setup_qdisc_ets(port, type_data); + case TC_SETUP_BLOCK: + return lan966x_tc_setup_block(port, type_data); + default: + return -EOPNOTSUPP; + } + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c new file mode 100644 index 0000000000000000000000000000000000000000..7368433b9277a9809b8e8676feb6a35c0be77a03 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +static int lan966x_tc_matchall_add(struct lan966x_port *port, + struct tc_cls_matchall_offload *f, + bool ingress) +{ + struct flow_action_entry *act; + + if (!flow_offload_has_one_action(&f->rule->action)) { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Only once action per filter is supported"); + return -EOPNOTSUPP; + } + + act = &f->rule->action.entries[0]; + switch (act->id) { + case FLOW_ACTION_POLICE: + return lan966x_police_port_add(port, &f->rule->action, act, + f->cookie, ingress, + f->common.extack); + case FLOW_ACTION_MIRRED: + return lan966x_mirror_port_add(port, act, f->cookie, + ingress, f->common.extack); + default: + NL_SET_ERR_MSG_MOD(f->common.extack, + "Unsupported action"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int lan966x_tc_matchall_del(struct lan966x_port *port, + struct tc_cls_matchall_offload *f, + bool ingress) +{ + if (f->cookie == port->tc.police_id) { + return lan966x_police_port_del(port, f->cookie, + f->common.extack); + } else if (f->cookie == port->tc.ingress_mirror_id || + f->cookie == port->tc.egress_mirror_id) { + return lan966x_mirror_port_del(port, ingress, + f->common.extack); + } else { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Unsupported action"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int lan966x_tc_matchall_stats(struct lan966x_port *port, + struct tc_cls_matchall_offload *f, + bool ingress) +{ + if (f->cookie == port->tc.police_id) { + lan966x_police_port_stats(port, &f->stats); + } else if (f->cookie == port->tc.ingress_mirror_id || + f->cookie == port->tc.egress_mirror_id) { + lan966x_mirror_port_stats(port, &f->stats, ingress); + } else { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Unsupported action"); + return -EOPNOTSUPP; + } + + return 0; +} + +int lan966x_tc_matchall(struct lan966x_port *port, + struct tc_cls_matchall_offload *f, + bool ingress) +{ + if (!tc_cls_can_offload_and_chain0(port->dev, &f->common)) { + NL_SET_ERR_MSG_MOD(f->common.extack, + "Only chain zero is supported"); + return -EOPNOTSUPP; + } + + switch (f->command) { + case TC_CLSMATCHALL_REPLACE: + return lan966x_tc_matchall_add(port, f, ingress); + case TC_CLSMATCHALL_DESTROY: + return lan966x_tc_matchall_del(port, f, ingress); + case TC_CLSMATCHALL_STATS: + return lan966x_tc_matchall_stats(port, f, ingress); + default: + return -EOPNOTSUPP; + } + + return 0; +} diff --git a/drivers/net/ethernet/microchip/sparx5/Makefile b/drivers/net/ethernet/microchip/sparx5/Makefile index 4402c3ed1dc51852df38f0d45dddbb30341ecad2..d1c6ad96674789b21ac2bbb388f2294bbc532508 100644 --- a/drivers/net/ethernet/microchip/sparx5/Makefile +++ b/drivers/net/ethernet/microchip/sparx5/Makefile @@ -8,4 +8,4 @@ obj-$(CONFIG_SPARX5_SWITCH) += sparx5-switch.o sparx5-switch-objs := sparx5_main.o sparx5_packet.o \ sparx5_netdev.o sparx5_phylink.o sparx5_port.o sparx5_mactable.o sparx5_vlan.o \ sparx5_switchdev.o sparx5_calendar.o sparx5_ethtool.o sparx5_fdma.o \ - sparx5_ptp.o sparx5_pgid.o + sparx5_ptp.o sparx5_pgid.o sparx5_tc.o sparx5_qos.o diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c index a5837dbe0c7e2e9fd564fd61e0a9809ff3853347..4af285918ea2a45cae16ca01ec1ab8fd3f5370c9 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c @@ -186,8 +186,8 @@ bool sparx5_mact_getnext(struct sparx5 *sparx5, return ret == 0; } -bool sparx5_mact_find(struct sparx5 *sparx5, - const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2) +int sparx5_mact_find(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2) { int ret; u32 cfg2; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c index 01be7bd841813e709cba4a8a37904f588f05fd47..62a325e963450b393e1003dcabed9a650a431aca 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -27,6 +27,7 @@ #include "sparx5_main_regs.h" #include "sparx5_main.h" #include "sparx5_port.h" +#include "sparx5_qos.h" #define QLIM_WM(fraction) \ ((SPX5_BUFFER_MEMORY / SPX5_BUFFER_CELL_SZ - 100) * (fraction) / 100) @@ -277,6 +278,7 @@ static int sparx5_create_port(struct sparx5 *sparx5, spx5_port->custom_etype = 0x8880; /* Vitesse */ spx5_port->phylink_pcs.poll = true; spx5_port->phylink_pcs.ops = &sparx5_phylink_pcs_ops; + spx5_port->is_mrouter = false; sparx5->ports[config->portno] = spx5_port; err = sparx5_port_init(sparx5, spx5_port, &config->conf); @@ -661,6 +663,9 @@ static int sparx5_start(struct sparx5 *sparx5) queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work, SPX5_MACT_PULL_DELAY); + mutex_init(&sparx5->mdb_lock); + INIT_LIST_HEAD(&sparx5->mdb_entries); + err = sparx5_register_netdevs(sparx5); if (err) return err; @@ -864,6 +869,12 @@ static int mchp_sparx5_probe(struct platform_device *pdev) goto cleanup_ports; } + err = sparx5_qos_init(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to initialize QoS\n"); + goto cleanup_ports; + } + err = sparx5_ptp_init(sparx5); if (err) { dev_err(sparx5->dev, "PTP failed\n"); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h index b197129044b526f486dd085e93b746c41f9db8dd..7a83222caa737bcb1cc13f6a32cb0522659241d0 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h @@ -190,6 +190,7 @@ struct sparx5_port { u8 ptp_cmd; u16 ts_id; struct sk_buff_head tx_skbs; + bool is_mrouter; }; enum sparx5_core_clockfreq { @@ -215,6 +216,15 @@ struct sparx5_skb_cb { unsigned long jiffies; }; +struct sparx5_mdb_entry { + struct list_head list; + DECLARE_BITMAP(port_mask, SPX5_PORTS); + unsigned char addr[ETH_ALEN]; + bool cpu_copy; + u16 vid; + u16 pgid_idx; +}; + #define SPARX5_PTP_TIMEOUT msecs_to_jiffies(10) #define SPARX5_SKB_CB(skb) \ ((struct sparx5_skb_cb *)((skb)->cb)) @@ -256,6 +266,10 @@ struct sparx5 { struct list_head mact_entries; /* mac table list (mact_entries) mutex */ struct mutex mact_lock; + /* SW MDB table */ + struct list_head mdb_entries; + /* mdb list mutex */ + struct mutex mdb_lock; struct delayed_work mact_work; struct workqueue_struct *mact_queue; /* Board specifics */ @@ -291,7 +305,7 @@ struct frame_info { void sparx5_xtr_flush(struct sparx5 *sparx5, u8 grp); void sparx5_ifh_parse(u32 *ifh, struct frame_info *info); irqreturn_t sparx5_xtr_handler(int irq, void *_priv); -int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev); +netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev); int sparx5_manual_injection_mode(struct sparx5 *sparx5); void sparx5_port_inj_timer_setup(struct sparx5_port *port); @@ -307,8 +321,8 @@ int sparx5_mact_learn(struct sparx5 *sparx5, int port, const unsigned char mac[ETH_ALEN], u16 vid); bool sparx5_mact_getnext(struct sparx5 *sparx5, unsigned char mac[ETH_ALEN], u16 *vid, u32 *pcfg2); -bool sparx5_mact_find(struct sparx5 *sparx5, - const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2); +int sparx5_mact_find(struct sparx5 *sparx5, + const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2); int sparx5_mact_forget(struct sparx5 *sparx5, const unsigned char mac[ETH_ALEN], u16 vid); int sparx5_add_mact_entry(struct sparx5 *sparx5, @@ -325,6 +339,7 @@ void sparx5_mact_init(struct sparx5 *sparx5); /* sparx5_vlan.c */ void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable); +void sparx5_pgid_clear(struct sparx5 *spx5, int pgid); void sparx5_pgid_read_mask(struct sparx5 *sparx5, int pgid, u32 portmask[3]); void sparx5_update_fwd(struct sparx5 *sparx5); void sparx5_vlan_init(struct sparx5 *sparx5); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h index c94de436b2817cea289e327b3ad11f29cd1a96bf..fa2eb70f487ab2f34cc9e68765f607d075056415 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h @@ -2993,6 +2993,147 @@ enum sparx5_target { #define GCB_SIO_CLOCK_SYS_CLK_PERIOD_GET(x)\ FIELD_GET(GCB_SIO_CLOCK_SYS_CLK_PERIOD, x) +/* HSCH:HSCH_CFG:CIR_CFG */ +#define HSCH_CIR_CFG(g) __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 0, 0, 1, 4) + +#define HSCH_CIR_CFG_CIR_RATE GENMASK(22, 6) +#define HSCH_CIR_CFG_CIR_RATE_SET(x)\ + FIELD_PREP(HSCH_CIR_CFG_CIR_RATE, x) +#define HSCH_CIR_CFG_CIR_RATE_GET(x)\ + FIELD_GET(HSCH_CIR_CFG_CIR_RATE, x) + +#define HSCH_CIR_CFG_CIR_BURST GENMASK(5, 0) +#define HSCH_CIR_CFG_CIR_BURST_SET(x)\ + FIELD_PREP(HSCH_CIR_CFG_CIR_BURST, x) +#define HSCH_CIR_CFG_CIR_BURST_GET(x)\ + FIELD_GET(HSCH_CIR_CFG_CIR_BURST, x) + +/* HSCH:HSCH_CFG:EIR_CFG */ +#define HSCH_EIR_CFG(g) __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 4, 0, 1, 4) + +#define HSCH_EIR_CFG_EIR_RATE GENMASK(22, 6) +#define HSCH_EIR_CFG_EIR_RATE_SET(x)\ + FIELD_PREP(HSCH_EIR_CFG_EIR_RATE, x) +#define HSCH_EIR_CFG_EIR_RATE_GET(x)\ + FIELD_GET(HSCH_EIR_CFG_EIR_RATE, x) + +#define HSCH_EIR_CFG_EIR_BURST GENMASK(5, 0) +#define HSCH_EIR_CFG_EIR_BURST_SET(x)\ + FIELD_PREP(HSCH_EIR_CFG_EIR_BURST, x) +#define HSCH_EIR_CFG_EIR_BURST_GET(x)\ + FIELD_GET(HSCH_EIR_CFG_EIR_BURST, x) + +/* HSCH:HSCH_CFG:SE_CFG */ +#define HSCH_SE_CFG(g) __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 8, 0, 1, 4) + +#define HSCH_SE_CFG_SE_DWRR_CNT GENMASK(12, 6) +#define HSCH_SE_CFG_SE_DWRR_CNT_SET(x)\ + FIELD_PREP(HSCH_SE_CFG_SE_DWRR_CNT, x) +#define HSCH_SE_CFG_SE_DWRR_CNT_GET(x)\ + FIELD_GET(HSCH_SE_CFG_SE_DWRR_CNT, x) + +#define HSCH_SE_CFG_SE_AVB_ENA BIT(5) +#define HSCH_SE_CFG_SE_AVB_ENA_SET(x)\ + FIELD_PREP(HSCH_SE_CFG_SE_AVB_ENA, x) +#define HSCH_SE_CFG_SE_AVB_ENA_GET(x)\ + FIELD_GET(HSCH_SE_CFG_SE_AVB_ENA, x) + +#define HSCH_SE_CFG_SE_FRM_MODE GENMASK(4, 3) +#define HSCH_SE_CFG_SE_FRM_MODE_SET(x)\ + FIELD_PREP(HSCH_SE_CFG_SE_FRM_MODE, x) +#define HSCH_SE_CFG_SE_FRM_MODE_GET(x)\ + FIELD_GET(HSCH_SE_CFG_SE_FRM_MODE, x) + +#define HSCH_SE_CFG_SE_DWRR_FRM_MODE GENMASK(2, 1) +#define HSCH_SE_CFG_SE_DWRR_FRM_MODE_SET(x)\ + FIELD_PREP(HSCH_SE_CFG_SE_DWRR_FRM_MODE, x) +#define HSCH_SE_CFG_SE_DWRR_FRM_MODE_GET(x)\ + FIELD_GET(HSCH_SE_CFG_SE_DWRR_FRM_MODE, x) + +#define HSCH_SE_CFG_SE_STOP BIT(0) +#define HSCH_SE_CFG_SE_STOP_SET(x)\ + FIELD_PREP(HSCH_SE_CFG_SE_STOP, x) +#define HSCH_SE_CFG_SE_STOP_GET(x)\ + FIELD_GET(HSCH_SE_CFG_SE_STOP, x) + +/* HSCH:HSCH_CFG:SE_CONNECT */ +#define HSCH_SE_CONNECT(g) __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 12, 0, 1, 4) + +#define HSCH_SE_CONNECT_SE_LEAK_LINK GENMASK(15, 0) +#define HSCH_SE_CONNECT_SE_LEAK_LINK_SET(x)\ + FIELD_PREP(HSCH_SE_CONNECT_SE_LEAK_LINK, x) +#define HSCH_SE_CONNECT_SE_LEAK_LINK_GET(x)\ + FIELD_GET(HSCH_SE_CONNECT_SE_LEAK_LINK, x) + +/* HSCH:HSCH_CFG:SE_DLB_SENSE */ +#define HSCH_SE_DLB_SENSE(g) __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 16, 0, 1, 4) + +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO GENMASK(12, 10) +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_SET(x)\ + FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_PRIO, x) +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_GET(x)\ + FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_PRIO, x) + +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT GENMASK(9, 3) +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_SET(x)\ + FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_DPORT, x) +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_GET(x)\ + FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_DPORT, x) + +#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA BIT(2) +#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA_SET(x)\ + FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA, x) +#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA_GET(x)\ + FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA, x) + +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA BIT(1) +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA_SET(x)\ + FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA, x) +#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA_GET(x)\ + FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA, x) + +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA BIT(0) +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA_SET(x)\ + FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA, x) +#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA_GET(x)\ + FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA, x) + +/* HSCH:HSCH_DWRR:DWRR_ENTRY */ +#define HSCH_DWRR_ENTRY(g) __REG(TARGET_HSCH, 0, 1, 162816, g, 72, 4, 0, 0, 1, 4) + +#define HSCH_DWRR_ENTRY_DWRR_COST GENMASK(24, 20) +#define HSCH_DWRR_ENTRY_DWRR_COST_SET(x)\ + FIELD_PREP(HSCH_DWRR_ENTRY_DWRR_COST, x) +#define HSCH_DWRR_ENTRY_DWRR_COST_GET(x)\ + FIELD_GET(HSCH_DWRR_ENTRY_DWRR_COST, x) + +#define HSCH_DWRR_ENTRY_DWRR_BALANCE GENMASK(19, 0) +#define HSCH_DWRR_ENTRY_DWRR_BALANCE_SET(x)\ + FIELD_PREP(HSCH_DWRR_ENTRY_DWRR_BALANCE, x) +#define HSCH_DWRR_ENTRY_DWRR_BALANCE_GET(x)\ + FIELD_GET(HSCH_DWRR_ENTRY_DWRR_BALANCE, x) + +/* HSCH:HSCH_MISC:HSCH_CFG_CFG */ +#define HSCH_HSCH_CFG_CFG __REG(TARGET_HSCH, 0, 1, 163104, 0, 1, 648, 284, 0, 1, 4) + +#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX GENMASK(26, 14) +#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX_SET(x)\ + FIELD_PREP(HSCH_HSCH_CFG_CFG_CFG_SE_IDX, x) +#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX_GET(x)\ + FIELD_GET(HSCH_HSCH_CFG_CFG_CFG_SE_IDX, x) + +#define HSCH_HSCH_CFG_CFG_HSCH_LAYER GENMASK(13, 12) +#define HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(x)\ + FIELD_PREP(HSCH_HSCH_CFG_CFG_HSCH_LAYER, x) +#define HSCH_HSCH_CFG_CFG_HSCH_LAYER_GET(x)\ + FIELD_GET(HSCH_HSCH_CFG_CFG_HSCH_LAYER, x) + +#define HSCH_HSCH_CFG_CFG_CSR_GRANT GENMASK(11, 0) +#define HSCH_HSCH_CFG_CFG_CSR_GRANT_SET(x)\ + FIELD_PREP(HSCH_HSCH_CFG_CFG_CSR_GRANT, x) +#define HSCH_HSCH_CFG_CFG_CSR_GRANT_GET(x)\ + FIELD_GET(HSCH_HSCH_CFG_CFG_CSR_GRANT, x) + /* HSCH:HSCH_MISC:SYS_CLK_PER */ #define HSCH_SYS_CLK_PER __REG(TARGET_HSCH, 0, 1, 163104, 0, 1, 648, 640, 0, 1, 4) @@ -3002,6 +3143,30 @@ enum sparx5_target { #define HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS_GET(x)\ FIELD_GET(HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS, x) +/* HSCH:HSCH_LEAK_LISTS:HSCH_TIMER_CFG */ +#define HSCH_HSCH_TIMER_CFG(g, r) __REG(TARGET_HSCH, 0, 1, 161664, g, 4, 32, 0, r, 4, 4) + +#define HSCH_HSCH_TIMER_CFG_LEAK_TIME GENMASK(17, 0) +#define HSCH_HSCH_TIMER_CFG_LEAK_TIME_SET(x)\ + FIELD_PREP(HSCH_HSCH_TIMER_CFG_LEAK_TIME, x) +#define HSCH_HSCH_TIMER_CFG_LEAK_TIME_GET(x)\ + FIELD_GET(HSCH_HSCH_TIMER_CFG_LEAK_TIME, x) + +/* HSCH:HSCH_LEAK_LISTS:HSCH_LEAK_CFG */ +#define HSCH_HSCH_LEAK_CFG(g, r) __REG(TARGET_HSCH, 0, 1, 161664, g, 4, 32, 16, r, 4, 4) + +#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST GENMASK(16, 1) +#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST_SET(x)\ + FIELD_PREP(HSCH_HSCH_LEAK_CFG_LEAK_FIRST, x) +#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST_GET(x)\ + FIELD_GET(HSCH_HSCH_LEAK_CFG_LEAK_FIRST, x) + +#define HSCH_HSCH_LEAK_CFG_LEAK_ERR BIT(0) +#define HSCH_HSCH_LEAK_CFG_LEAK_ERR_SET(x)\ + FIELD_PREP(HSCH_HSCH_LEAK_CFG_LEAK_ERR, x) +#define HSCH_HSCH_LEAK_CFG_LEAK_ERR_GET(x)\ + FIELD_GET(HSCH_HSCH_LEAK_CFG_LEAK_ERR, x) + /* HSCH:SYSTEM:FLUSH_CTRL */ #define HSCH_FLUSH_CTRL __REG(TARGET_HSCH, 0, 1, 184000, 0, 1, 312, 4, 0, 1, 4) diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c index af4d3e1f1a6d92e71430801b2c28439a8240c841..19516ccad5338c511c8e2ea132cb55681034a468 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c @@ -7,6 +7,7 @@ #include "sparx5_main_regs.h" #include "sparx5_main.h" #include "sparx5_port.h" +#include "sparx5_tc.h" /* The IFH bit position of the first VSTAX bit. This is because the * VSTAX bit positions in Data sheet is starting from zero. @@ -228,6 +229,7 @@ static const struct net_device_ops sparx5_port_netdev_ops = { .ndo_get_stats64 = sparx5_get_stats64, .ndo_get_port_parent_id = sparx5_get_port_parent_id, .ndo_eth_ioctl = sparx5_port_ioctl, + .ndo_setup_tc = sparx5_port_setup_tc, }; bool sparx5_netdevice_check(const struct net_device *dev) @@ -240,10 +242,14 @@ struct net_device *sparx5_create_netdev(struct sparx5 *sparx5, u32 portno) struct sparx5_port *spx5_port; struct net_device *ndev; - ndev = devm_alloc_etherdev(sparx5->dev, sizeof(struct sparx5_port)); + ndev = devm_alloc_etherdev_mqs(sparx5->dev, sizeof(struct sparx5_port), + SPX5_PRIOS, 1); if (!ndev) return ERR_PTR(-ENOMEM); + ndev->hw_features |= NETIF_F_HW_TC; + ndev->features |= NETIF_F_HW_TC; + SET_NETDEV_DEV(ndev, sparx5->dev); spx5_port = netdev_priv(ndev); spx5_port->ndev = ndev; diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c index 304f84aadc36bb9949d15aa2dbfead1d4d0340b8..83c16ca5b30f08722ebc69a2e470e811a509b687 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c @@ -113,6 +113,8 @@ static void sparx5_xtr_grp(struct sparx5 *sparx5, u8 grp, bool byte_swap) /* This assumes STATUS_WORD_POS == 1, Status * just after last data */ + if (!byte_swap) + val = ntohl((__force __be32)val); byte_cnt -= (4 - XTR_VALID_BYTES(val)); eof_flag = true; break; @@ -220,13 +222,13 @@ static int sparx5_inject(struct sparx5 *sparx5, return NETDEV_TX_OK; } -int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev) +netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev) { struct net_device_stats *stats = &dev->stats; struct sparx5_port *port = netdev_priv(dev); struct sparx5 *sparx5 = port->sparx5; u32 ifh[IFH_LEN]; - int ret; + netdev_tx_t ret; memset(ifh, 0, IFH_LEN * 4); sparx5_set_port_ifh(ifh, port->portno); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c new file mode 100644 index 0000000000000000000000000000000000000000..1e79d0ef0cb8dbb3ce751e40a2677d237676bc30 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries. + */ + +#include + +#include "sparx5_main.h" +#include "sparx5_qos.h" + +/* Max rates for leak groups */ +static const u32 spx5_hsch_max_group_rate[SPX5_HSCH_LEAK_GRP_CNT] = { + 1048568, /* 1.049 Gbps */ + 2621420, /* 2.621 Gbps */ + 10485680, /* 10.486 Gbps */ + 26214200 /* 26.214 Gbps */ +}; + +static struct sparx5_layer layers[SPX5_HSCH_LAYER_CNT]; + +static u32 sparx5_lg_get_leak_time(struct sparx5 *sparx5, u32 layer, u32 group) +{ + u32 value; + + value = spx5_rd(sparx5, HSCH_HSCH_TIMER_CFG(layer, group)); + return HSCH_HSCH_TIMER_CFG_LEAK_TIME_GET(value); +} + +static void sparx5_lg_set_leak_time(struct sparx5 *sparx5, u32 layer, u32 group, + u32 leak_time) +{ + spx5_wr(HSCH_HSCH_TIMER_CFG_LEAK_TIME_SET(leak_time), sparx5, + HSCH_HSCH_TIMER_CFG(layer, group)); +} + +static u32 sparx5_lg_get_first(struct sparx5 *sparx5, u32 layer, u32 group) +{ + u32 value; + + value = spx5_rd(sparx5, HSCH_HSCH_LEAK_CFG(layer, group)); + return HSCH_HSCH_LEAK_CFG_LEAK_FIRST_GET(value); +} + +static u32 sparx5_lg_get_next(struct sparx5 *sparx5, u32 layer, u32 group, + u32 idx) + +{ + u32 value; + + value = spx5_rd(sparx5, HSCH_SE_CONNECT(idx)); + return HSCH_SE_CONNECT_SE_LEAK_LINK_GET(value); +} + +static u32 sparx5_lg_get_last(struct sparx5 *sparx5, u32 layer, u32 group) +{ + u32 itr, next; + + itr = sparx5_lg_get_first(sparx5, layer, group); + + for (;;) { + next = sparx5_lg_get_next(sparx5, layer, group, itr); + if (itr == next) + return itr; + + itr = next; + } +} + +static bool sparx5_lg_is_last(struct sparx5 *sparx5, u32 layer, u32 group, + u32 idx) +{ + return idx == sparx5_lg_get_next(sparx5, layer, group, idx); +} + +static bool sparx5_lg_is_first(struct sparx5 *sparx5, u32 layer, u32 group, + u32 idx) +{ + return idx == sparx5_lg_get_first(sparx5, layer, group); +} + +static bool sparx5_lg_is_empty(struct sparx5 *sparx5, u32 layer, u32 group) +{ + return sparx5_lg_get_leak_time(sparx5, layer, group) == 0; +} + +static bool sparx5_lg_is_singular(struct sparx5 *sparx5, u32 layer, u32 group) +{ + if (sparx5_lg_is_empty(sparx5, layer, group)) + return false; + + return sparx5_lg_get_first(sparx5, layer, group) == + sparx5_lg_get_last(sparx5, layer, group); +} + +static void sparx5_lg_enable(struct sparx5 *sparx5, u32 layer, u32 group, + u32 leak_time) +{ + sparx5_lg_set_leak_time(sparx5, layer, group, leak_time); +} + +static void sparx5_lg_disable(struct sparx5 *sparx5, u32 layer, u32 group) +{ + sparx5_lg_set_leak_time(sparx5, layer, group, 0); +} + +static int sparx5_lg_get_group_by_index(struct sparx5 *sparx5, u32 layer, + u32 idx, u32 *group) +{ + u32 itr, next; + int i; + + for (i = 0; i < SPX5_HSCH_LEAK_GRP_CNT; i++) { + if (sparx5_lg_is_empty(sparx5, layer, i)) + continue; + + itr = sparx5_lg_get_first(sparx5, layer, i); + + for (;;) { + next = sparx5_lg_get_next(sparx5, layer, i, itr); + + if (itr == idx) { + *group = i; + return 0; /* Found it */ + } + if (itr == next) + break; /* Was not found */ + + itr = next; + } + } + + return -1; +} + +static int sparx5_lg_get_group_by_rate(u32 layer, u32 rate, u32 *group) +{ + struct sparx5_layer *l = &layers[layer]; + struct sparx5_lg *lg; + u32 i; + + for (i = 0; i < SPX5_HSCH_LEAK_GRP_CNT; i++) { + lg = &l->leak_groups[i]; + if (rate <= lg->max_rate) { + *group = i; + return 0; + } + } + + return -1; +} + +static int sparx5_lg_get_adjacent(struct sparx5 *sparx5, u32 layer, u32 group, + u32 idx, u32 *prev, u32 *next, u32 *first) +{ + u32 itr; + + *first = sparx5_lg_get_first(sparx5, layer, group); + *prev = *first; + *next = *first; + itr = *first; + + for (;;) { + *next = sparx5_lg_get_next(sparx5, layer, group, itr); + + if (itr == idx) + return 0; /* Found it */ + + if (itr == *next) + return -1; /* Was not found */ + + *prev = itr; + itr = *next; + } + + return -1; +} + +static int sparx5_lg_conf_set(struct sparx5 *sparx5, u32 layer, u32 group, + u32 se_first, u32 idx, u32 idx_next, bool empty) +{ + u32 leak_time = layers[layer].leak_groups[group].leak_time; + + /* Stop leaking */ + sparx5_lg_disable(sparx5, layer, group); + + if (empty) + return 0; + + /* Select layer */ + spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(layer), + HSCH_HSCH_CFG_CFG_HSCH_LAYER, sparx5, HSCH_HSCH_CFG_CFG); + + /* Link elements */ + spx5_wr(HSCH_SE_CONNECT_SE_LEAK_LINK_SET(idx_next), sparx5, + HSCH_SE_CONNECT(idx)); + + /* Set the first element. */ + spx5_rmw(HSCH_HSCH_LEAK_CFG_LEAK_FIRST_SET(se_first), + HSCH_HSCH_LEAK_CFG_LEAK_FIRST, sparx5, + HSCH_HSCH_LEAK_CFG(layer, group)); + + /* Start leaking */ + sparx5_lg_enable(sparx5, layer, group, leak_time); + + return 0; +} + +static int sparx5_lg_del(struct sparx5 *sparx5, u32 layer, u32 group, u32 idx) +{ + u32 first, next, prev; + bool empty = false; + + /* idx *must* be present in the leak group */ + WARN_ON(sparx5_lg_get_adjacent(sparx5, layer, group, idx, &prev, &next, + &first) < 0); + + if (sparx5_lg_is_singular(sparx5, layer, group)) { + empty = true; + } else if (sparx5_lg_is_last(sparx5, layer, group, idx)) { + /* idx is removed, prev is now last */ + idx = prev; + next = prev; + } else if (sparx5_lg_is_first(sparx5, layer, group, idx)) { + /* idx is removed and points to itself, first is next */ + first = next; + next = idx; + } else { + /* Next is not touched */ + idx = prev; + } + + return sparx5_lg_conf_set(sparx5, layer, group, first, idx, next, + empty); +} + +static int sparx5_lg_add(struct sparx5 *sparx5, u32 layer, u32 new_group, + u32 idx) +{ + u32 first, next, old_group; + + pr_debug("ADD: layer: %d, new_group: %d, idx: %d", layer, new_group, + idx); + + /* Is this SE already shaping ? */ + if (sparx5_lg_get_group_by_index(sparx5, layer, idx, &old_group) >= 0) { + if (old_group != new_group) { + /* Delete from old group */ + sparx5_lg_del(sparx5, layer, old_group, idx); + } else { + /* Nothing to do here */ + return 0; + } + } + + /* We always add to head of the list */ + first = idx; + + if (sparx5_lg_is_empty(sparx5, layer, new_group)) + next = idx; + else + next = sparx5_lg_get_first(sparx5, layer, new_group); + + return sparx5_lg_conf_set(sparx5, layer, new_group, first, idx, next, + false); +} + +static int sparx5_shaper_conf_set(struct sparx5_port *port, + const struct sparx5_shaper *sh, u32 layer, + u32 idx, u32 group) +{ + int (*sparx5_lg_action)(struct sparx5 *, u32, u32, u32); + struct sparx5 *sparx5 = port->sparx5; + + if (!sh->rate && !sh->burst) + sparx5_lg_action = &sparx5_lg_del; + else + sparx5_lg_action = &sparx5_lg_add; + + /* Select layer */ + spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(layer), + HSCH_HSCH_CFG_CFG_HSCH_LAYER, sparx5, HSCH_HSCH_CFG_CFG); + + /* Set frame mode */ + spx5_rmw(HSCH_SE_CFG_SE_FRM_MODE_SET(sh->mode), HSCH_SE_CFG_SE_FRM_MODE, + sparx5, HSCH_SE_CFG(idx)); + + /* Set committed rate and burst */ + spx5_wr(HSCH_CIR_CFG_CIR_RATE_SET(sh->rate) | + HSCH_CIR_CFG_CIR_BURST_SET(sh->burst), + sparx5, HSCH_CIR_CFG(idx)); + + /* This has to be done after the shaper configuration has been set */ + sparx5_lg_action(sparx5, layer, group, idx); + + return 0; +} + +static u32 sparx5_weight_to_hw_cost(u32 weight_min, u32 weight) +{ + return ((((SPX5_DWRR_COST_MAX << 4) * weight_min / weight) + 8) >> 4) - + 1; +} + +static int sparx5_dwrr_conf_set(struct sparx5_port *port, + struct sparx5_dwrr *dwrr) +{ + int i; + + spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(2) | + HSCH_HSCH_CFG_CFG_CFG_SE_IDX_SET(port->portno), + HSCH_HSCH_CFG_CFG_HSCH_LAYER | HSCH_HSCH_CFG_CFG_CFG_SE_IDX, + port->sparx5, HSCH_HSCH_CFG_CFG); + + /* Number of *lower* indexes that are arbitrated dwrr */ + spx5_rmw(HSCH_SE_CFG_SE_DWRR_CNT_SET(dwrr->count), + HSCH_SE_CFG_SE_DWRR_CNT, port->sparx5, + HSCH_SE_CFG(port->portno)); + + for (i = 0; i < dwrr->count; i++) { + spx5_rmw(HSCH_DWRR_ENTRY_DWRR_COST_SET(dwrr->cost[i]), + HSCH_DWRR_ENTRY_DWRR_COST, port->sparx5, + HSCH_DWRR_ENTRY(i)); + } + + return 0; +} + +static int sparx5_leak_groups_init(struct sparx5 *sparx5) +{ + struct sparx5_layer *layer; + u32 sys_clk_per_100ps; + struct sparx5_lg *lg; + u32 leak_time_us; + int i, ii; + + sys_clk_per_100ps = spx5_rd(sparx5, HSCH_SYS_CLK_PER); + + for (i = 0; i < SPX5_HSCH_LAYER_CNT; i++) { + layer = &layers[i]; + for (ii = 0; ii < SPX5_HSCH_LEAK_GRP_CNT; ii++) { + lg = &layer->leak_groups[ii]; + lg->max_rate = spx5_hsch_max_group_rate[ii]; + + /* Calculate the leak time in us, to serve a maximum + * rate of 'max_rate' for this group + */ + leak_time_us = (SPX5_SE_RATE_MAX * 1000) / lg->max_rate; + + /* Hardware wants leak time in ns */ + lg->leak_time = 1000 * leak_time_us; + + /* Calculate resolution */ + lg->resolution = 1000 / leak_time_us; + + /* Maximum number of shapers that can be served by + * this leak group + */ + lg->max_ses = (1000 * leak_time_us) / sys_clk_per_100ps; + + /* Example: + * Wanted bandwidth is 100Mbit: + * + * 100 mbps can be served by leak group zero. + * + * leak_time is 125000 ns. + * resolution is: 8 + * + * cir = 100000 / 8 = 12500 + * leaks_pr_sec = 125000 / 10^9 = 8000 + * bw = 12500 * 8000 = 10^8 (100 Mbit) + */ + + /* Disable by default - this also indicates an empty + * leak group + */ + sparx5_lg_disable(sparx5, i, ii); + } + } + + return 0; +} + +int sparx5_qos_init(struct sparx5 *sparx5) +{ + int ret; + + ret = sparx5_leak_groups_init(sparx5); + if (ret < 0) + return ret; + + return 0; +} + +int sparx5_tc_mqprio_add(struct net_device *ndev, u8 num_tc) +{ + int i; + + if (num_tc != SPX5_PRIOS) { + netdev_err(ndev, "Only %d traffic classes supported\n", + SPX5_PRIOS); + return -EINVAL; + } + + netdev_set_num_tc(ndev, num_tc); + + for (i = 0; i < num_tc; i++) + netdev_set_tc_queue(ndev, i, 1, i); + + netdev_dbg(ndev, "dev->num_tc %u dev->real_num_tx_queues %u\n", + ndev->num_tc, ndev->real_num_tx_queues); + + return 0; +} + +int sparx5_tc_mqprio_del(struct net_device *ndev) +{ + netdev_reset_tc(ndev); + + netdev_dbg(ndev, "dev->num_tc %u dev->real_num_tx_queues %u\n", + ndev->num_tc, ndev->real_num_tx_queues); + + return 0; +} + +int sparx5_tc_tbf_add(struct sparx5_port *port, + struct tc_tbf_qopt_offload_replace_params *params, + u32 layer, u32 idx) +{ + struct sparx5_shaper sh = { + .mode = SPX5_SE_MODE_DATARATE, + .rate = div_u64(params->rate.rate_bytes_ps, 1000) * 8, + .burst = params->max_size, + }; + struct sparx5_lg *lg; + u32 group; + + /* Find suitable group for this se */ + if (sparx5_lg_get_group_by_rate(layer, sh.rate, &group) < 0) { + pr_debug("Could not find leak group for se with rate: %d", + sh.rate); + return -EINVAL; + } + + lg = &layers[layer].leak_groups[group]; + + pr_debug("Found matching group (speed: %d)\n", lg->max_rate); + + if (sh.rate < SPX5_SE_RATE_MIN || sh.burst < SPX5_SE_BURST_MIN) + return -EINVAL; + + /* Calculate committed rate and burst */ + sh.rate = DIV_ROUND_UP(sh.rate, lg->resolution); + sh.burst = DIV_ROUND_UP(sh.burst, SPX5_SE_BURST_UNIT); + + if (sh.rate > SPX5_SE_RATE_MAX || sh.burst > SPX5_SE_BURST_MAX) + return -EINVAL; + + return sparx5_shaper_conf_set(port, &sh, layer, idx, group); +} + +int sparx5_tc_tbf_del(struct sparx5_port *port, u32 layer, u32 idx) +{ + struct sparx5_shaper sh = {0}; + u32 group; + + sparx5_lg_get_group_by_index(port->sparx5, layer, idx, &group); + + return sparx5_shaper_conf_set(port, &sh, layer, idx, group); +} + +int sparx5_tc_ets_add(struct sparx5_port *port, + struct tc_ets_qopt_offload_replace_params *params) +{ + struct sparx5_dwrr dwrr = {0}; + /* Minimum weight for each iteration */ + unsigned int w_min = 100; + int i; + + /* Find minimum weight for all dwrr bands */ + for (i = 0; i < SPX5_PRIOS; i++) { + if (params->quanta[i] == 0) + continue; + w_min = min(w_min, params->weights[i]); + } + + for (i = 0; i < SPX5_PRIOS; i++) { + /* Strict band; skip */ + if (params->quanta[i] == 0) + continue; + + dwrr.count++; + + /* On the sparx5, bands with higher indexes are preferred and + * arbitrated strict. Strict bands are put in the lower indexes, + * by tc, so we reverse the bands here. + * + * Also convert the weight to something the hardware + * understands. + */ + dwrr.cost[SPX5_PRIOS - i - 1] = + sparx5_weight_to_hw_cost(w_min, params->weights[i]); + } + + return sparx5_dwrr_conf_set(port, &dwrr); +} + +int sparx5_tc_ets_del(struct sparx5_port *port) +{ + struct sparx5_dwrr dwrr = {0}; + + return sparx5_dwrr_conf_set(port, &dwrr); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h new file mode 100644 index 0000000000000000000000000000000000000000..ced35033a6c5ddbb0e7ca75553527fe5d3d03984 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries. + */ + +#ifndef __SPARX5_QOS_H__ +#define __SPARX5_QOS_H__ + +#include + +/* Number of Layers */ +#define SPX5_HSCH_LAYER_CNT 3 + +/* Scheduling elements per layer */ +#define SPX5_HSCH_L0_SE_CNT 5040 +#define SPX5_HSCH_L1_SE_CNT 64 +#define SPX5_HSCH_L2_SE_CNT 64 + +/* Calculate Layer 0 Scheduler Element when using normal hierarchy */ +#define SPX5_HSCH_L0_GET_IDX(port, queue) ((64 * (port)) + (8 * (queue))) + +/* Number of leak groups */ +#define SPX5_HSCH_LEAK_GRP_CNT 4 + +/* Scheduler modes */ +#define SPX5_SE_MODE_LINERATE 0 +#define SPX5_SE_MODE_DATARATE 1 + +/* Rate and burst */ +#define SPX5_SE_RATE_MAX 262143 +#define SPX5_SE_BURST_MAX 127 +#define SPX5_SE_RATE_MIN 1 +#define SPX5_SE_BURST_MIN 1 +#define SPX5_SE_BURST_UNIT 4096 + +/* Dwrr */ +#define SPX5_DWRR_COST_MAX 63 + +struct sparx5_shaper { + u32 mode; + u32 rate; + u32 burst; +}; + +struct sparx5_lg { + u32 max_rate; + u32 resolution; + u32 leak_time; + u32 max_ses; +}; + +struct sparx5_layer { + struct sparx5_lg leak_groups[SPX5_HSCH_LEAK_GRP_CNT]; +}; + +struct sparx5_dwrr { + u32 count; /* Number of inputs running dwrr */ + u8 cost[SPX5_PRIOS]; +}; + +int sparx5_qos_init(struct sparx5 *sparx5); + +/* Multi-Queue Priority */ +int sparx5_tc_mqprio_add(struct net_device *ndev, u8 num_tc); +int sparx5_tc_mqprio_del(struct net_device *ndev); + +/* Token Bucket Filter */ +struct tc_tbf_qopt_offload_replace_params; +int sparx5_tc_tbf_add(struct sparx5_port *port, + struct tc_tbf_qopt_offload_replace_params *params, + u32 layer, u32 idx); +int sparx5_tc_tbf_del(struct sparx5_port *port, u32 layer, u32 idx); + +/* Enhanced Transmission Selection */ +struct tc_ets_qopt_offload_replace_params; +int sparx5_tc_ets_add(struct sparx5_port *port, + struct tc_ets_qopt_offload_replace_params *params); + +int sparx5_tc_ets_del(struct sparx5_port *port); + +#endif /* __SPARX5_QOS_H__ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c index ec07f7d0528ce0a0074d5dc02715444b9d6c7365..4af85d108a0652a105000976d87333812d64f29c 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c @@ -29,14 +29,23 @@ static int sparx5_port_attr_pre_bridge_flags(struct sparx5_port *port, return 0; } +static void sparx5_port_update_mcast_ip_flood(struct sparx5_port *port, bool flood_flag) +{ + bool should_flood = flood_flag || port->is_mrouter; + int pgid; + + for (pgid = PGID_IPV4_MC_DATA; pgid <= PGID_IPV6_MC_CTRL; pgid++) + sparx5_pgid_update_mask(port, pgid, should_flood); +} + static void sparx5_port_attr_bridge_flags(struct sparx5_port *port, struct switchdev_brport_flags flags) { - int pgid; + if (flags.mask & BR_MCAST_FLOOD) { + sparx5_pgid_update_mask(port, PGID_MC_FLOOD, !!(flags.val & BR_MCAST_FLOOD)); + sparx5_port_update_mcast_ip_flood(port, !!(flags.val & BR_MCAST_FLOOD)); + } - if (flags.mask & BR_MCAST_FLOOD) - for (pgid = PGID_MC_FLOOD; pgid <= PGID_IPV6_MC_CTRL; pgid++) - sparx5_pgid_update_mask(port, pgid, !!(flags.val & BR_MCAST_FLOOD)); if (flags.mask & BR_FLOOD) sparx5_pgid_update_mask(port, PGID_UC_FLOOD, !!(flags.val & BR_FLOOD)); if (flags.mask & BR_BCAST_FLOOD) @@ -82,6 +91,37 @@ static void sparx5_port_attr_ageing_set(struct sparx5_port *port, sparx5_set_ageing(port->sparx5, ageing_time); } +static void sparx5_port_attr_mrouter_set(struct sparx5_port *port, + struct net_device *orig_dev, + bool enable) +{ + struct sparx5 *sparx5 = port->sparx5; + struct sparx5_mdb_entry *e; + bool flood_flag; + + if ((enable && port->is_mrouter) || (!enable && !port->is_mrouter)) + return; + + /* Add/del mrouter port on all active mdb entries in HW. + * Don't change entry port mask, since that represents + * ports that actually joined that group. + */ + mutex_lock(&sparx5->mdb_lock); + list_for_each_entry(e, &sparx5->mdb_entries, list) { + if (!test_bit(port->portno, e->port_mask) && + ether_addr_is_ip_mcast(e->addr)) + sparx5_pgid_update_mask(port, e->pgid_idx, enable); + } + mutex_unlock(&sparx5->mdb_lock); + + /* Enable/disable flooding depending on if port is mrouter port + * or if mcast flood is enabled. + */ + port->is_mrouter = enable; + flood_flag = br_port_flag_is_set(port->ndev, BR_MCAST_FLOOD); + sparx5_port_update_mcast_ip_flood(port, flood_flag); +} + static int sparx5_port_attr_set(struct net_device *dev, const void *ctx, const struct switchdev_attr *attr, struct netlink_ext_ack *extack) @@ -110,6 +150,11 @@ static int sparx5_port_attr_set(struct net_device *dev, const void *ctx, port->vlan_aware = attr->u.vlan_filtering; sparx5_vlan_port_apply(port->sparx5, port); break; + case SWITCHDEV_ATTR_ID_PORT_MROUTER: + sparx5_port_attr_mrouter_set(port, + attr->orig_dev, + attr->u.mrouter); + break; default: return -EOPNOTSUPP; } @@ -386,16 +431,95 @@ static int sparx5_handle_port_vlan_add(struct net_device *dev, v->flags & BRIDGE_VLAN_INFO_UNTAGGED); } +static int sparx5_alloc_mdb_entry(struct sparx5 *sparx5, + const unsigned char *addr, + u16 vid, + struct sparx5_mdb_entry **entry_out) +{ + struct sparx5_mdb_entry *entry; + u16 pgid_idx; + int err; + + entry = kzalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + err = sparx5_pgid_alloc_mcast(sparx5, &pgid_idx); + if (err) { + kfree(entry); + return err; + } + + memcpy(entry->addr, addr, ETH_ALEN); + entry->vid = vid; + entry->pgid_idx = pgid_idx; + + mutex_lock(&sparx5->mdb_lock); + list_add_tail(&entry->list, &sparx5->mdb_entries); + mutex_unlock(&sparx5->mdb_lock); + + *entry_out = entry; + return 0; +} + +static void sparx5_free_mdb_entry(struct sparx5 *sparx5, + const unsigned char *addr, + u16 vid) +{ + struct sparx5_mdb_entry *entry, *tmp; + + mutex_lock(&sparx5->mdb_lock); + list_for_each_entry_safe(entry, tmp, &sparx5->mdb_entries, list) { + if ((vid == 0 || entry->vid == vid) && + ether_addr_equal(addr, entry->addr)) { + list_del(&entry->list); + + sparx5_pgid_free(sparx5, entry->pgid_idx); + kfree(entry); + goto out; + } + } + +out: + mutex_unlock(&sparx5->mdb_lock); +} + +static struct sparx5_mdb_entry *sparx5_mdb_get_entry(struct sparx5 *sparx5, + const unsigned char *addr, + u16 vid) +{ + struct sparx5_mdb_entry *e, *found = NULL; + + mutex_lock(&sparx5->mdb_lock); + list_for_each_entry(e, &sparx5->mdb_entries, list) { + if (ether_addr_equal(e->addr, addr) && e->vid == vid) { + found = e; + goto out; + } + } + +out: + mutex_unlock(&sparx5->mdb_lock); + return found; +} + +static void sparx5_cpu_copy_ena(struct sparx5 *spx5, u16 pgid, bool enable) +{ + spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(enable), + ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5, + ANA_AC_PGID_MISC_CFG(pgid)); +} + static int sparx5_handle_port_mdb_add(struct net_device *dev, struct notifier_block *nb, const struct switchdev_obj_port_mdb *v) { struct sparx5_port *port = netdev_priv(dev); struct sparx5 *spx5 = port->sparx5; - u16 pgid_idx, vid; - u32 mact_entry; - bool is_host; - int res, err; + struct sparx5_mdb_entry *entry; + bool is_host, is_new; + int err, i; + u16 vid; if (!sparx5_netdevice_check(dev)) return -EOPNOTSUPP; @@ -410,66 +534,36 @@ static int sparx5_handle_port_mdb_add(struct net_device *dev, else vid = v->vid; - res = sparx5_mact_find(spx5, v->addr, vid, &mact_entry); - - if (res == 0) { - pgid_idx = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(mact_entry); - - /* MC_IDX starts after the port masks in the PGID table */ - pgid_idx += SPX5_PORTS; - - if (is_host) - spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1), - ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5, - ANA_AC_PGID_MISC_CFG(pgid_idx)); - else - sparx5_pgid_update_mask(port, pgid_idx, true); - - } else { - err = sparx5_pgid_alloc_mcast(spx5, &pgid_idx); - if (err) { - netdev_warn(dev, "multicast pgid table full\n"); + is_new = false; + entry = sparx5_mdb_get_entry(spx5, v->addr, vid); + if (!entry) { + err = sparx5_alloc_mdb_entry(spx5, v->addr, vid, &entry); + is_new = true; + if (err) return err; - } - - if (is_host) - spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1), - ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5, - ANA_AC_PGID_MISC_CFG(pgid_idx)); - else - sparx5_pgid_update_mask(port, pgid_idx, true); - - err = sparx5_mact_learn(spx5, pgid_idx, v->addr, vid); - - if (err) { - netdev_warn(dev, "could not learn mac address %pM\n", v->addr); - sparx5_pgid_free(spx5, pgid_idx); - sparx5_pgid_update_mask(port, pgid_idx, false); - return err; - } } - return 0; -} + mutex_lock(&spx5->mdb_lock); + + /* Add any mrouter ports to the new entry */ + if (is_new && ether_addr_is_ip_mcast(v->addr)) + for (i = 0; i < SPX5_PORTS; i++) + if (spx5->ports[i] && spx5->ports[i]->is_mrouter) + sparx5_pgid_update_mask(spx5->ports[i], + entry->pgid_idx, + true); + + if (is_host && !entry->cpu_copy) { + sparx5_cpu_copy_ena(spx5, entry->pgid_idx, true); + entry->cpu_copy = true; + } else if (!is_host) { + sparx5_pgid_update_mask(port, entry->pgid_idx, true); + set_bit(port->portno, entry->port_mask); + } + mutex_unlock(&spx5->mdb_lock); -static int sparx5_mdb_del_entry(struct net_device *dev, - struct sparx5 *spx5, - const unsigned char mac[ETH_ALEN], - const u16 vid, - u16 pgid_idx) -{ - int err; + sparx5_mact_learn(spx5, entry->pgid_idx, entry->addr, entry->vid); - err = sparx5_mact_forget(spx5, mac, vid); - if (err) { - netdev_warn(dev, "could not forget mac address %pM", mac); - return err; - } - err = sparx5_pgid_free(spx5, pgid_idx); - if (err) { - netdev_err(dev, "attempted to free already freed pgid\n"); - return err; - } return 0; } @@ -479,42 +573,45 @@ static int sparx5_handle_port_mdb_del(struct net_device *dev, { struct sparx5_port *port = netdev_priv(dev); struct sparx5 *spx5 = port->sparx5; - u16 pgid_idx, vid; - u32 mact_entry, res, pgid_entry[3], misc_cfg; - bool host_ena; + struct sparx5_mdb_entry *entry; + bool is_host; + u16 vid; if (!sparx5_netdevice_check(dev)) return -EOPNOTSUPP; + is_host = netif_is_bridge_master(v->obj.orig_dev); + if (!br_vlan_enabled(spx5->hw_bridge_dev)) vid = 1; else vid = v->vid; - res = sparx5_mact_find(spx5, v->addr, vid, &mact_entry); - - if (res == 0) { - pgid_idx = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(mact_entry); - - /* MC_IDX starts after the port masks in the PGID table */ - pgid_idx += SPX5_PORTS; - - if (netif_is_bridge_master(v->obj.orig_dev)) - spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(0), - ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5, - ANA_AC_PGID_MISC_CFG(pgid_idx)); - else - sparx5_pgid_update_mask(port, pgid_idx, false); + entry = sparx5_mdb_get_entry(spx5, v->addr, vid); + if (!entry) + return 0; - misc_cfg = spx5_rd(spx5, ANA_AC_PGID_MISC_CFG(pgid_idx)); - host_ena = ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_GET(misc_cfg); + mutex_lock(&spx5->mdb_lock); + if (is_host && entry->cpu_copy) { + sparx5_cpu_copy_ena(spx5, entry->pgid_idx, false); + entry->cpu_copy = false; + } else if (!is_host) { + clear_bit(port->portno, entry->port_mask); - sparx5_pgid_read_mask(spx5, pgid_idx, pgid_entry); - if (bitmap_empty((unsigned long *)pgid_entry, SPX5_PORTS) && !host_ena) - /* No ports or CPU are in MC group. Remove entry */ - return sparx5_mdb_del_entry(dev, spx5, v->addr, vid, pgid_idx); + /* Port not mrouter port or addr is L2 mcast, remove port from mask. */ + if (!port->is_mrouter || !ether_addr_is_ip_mcast(v->addr)) + sparx5_pgid_update_mask(port, entry->pgid_idx, false); + } + mutex_unlock(&spx5->mdb_lock); + + if (bitmap_empty(entry->port_mask, SPX5_PORTS) && !entry->cpu_copy) { + /* Clear pgid in case mrouter ports exists + * that are not part of the group. + */ + sparx5_pgid_clear(spx5, entry->pgid_idx); + sparx5_mact_forget(spx5, entry->addr, entry->vid); + sparx5_free_mdb_entry(spx5, entry->addr, entry->vid); } - return 0; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c new file mode 100644 index 0000000000000000000000000000000000000000..e05429c751eefb150930d8f60a33f3573c392cad --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries. + */ + +#include + +#include "sparx5_tc.h" +#include "sparx5_main.h" +#include "sparx5_qos.h" + +static void sparx5_tc_get_layer_and_idx(u32 parent, u32 portno, u32 *layer, + u32 *idx) +{ + if (parent == TC_H_ROOT) { + *layer = 2; + *idx = portno; + } else { + u32 queue = TC_H_MIN(parent) - 1; + *layer = 0; + *idx = SPX5_HSCH_L0_GET_IDX(portno, queue); + } +} + +static int sparx5_tc_setup_qdisc_mqprio(struct net_device *ndev, + struct tc_mqprio_qopt_offload *m) +{ + m->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS; + + if (m->qopt.num_tc == 0) + return sparx5_tc_mqprio_del(ndev); + else + return sparx5_tc_mqprio_add(ndev, m->qopt.num_tc); +} + +static int sparx5_tc_setup_qdisc_tbf(struct net_device *ndev, + struct tc_tbf_qopt_offload *qopt) +{ + struct sparx5_port *port = netdev_priv(ndev); + u32 layer, se_idx; + + sparx5_tc_get_layer_and_idx(qopt->parent, port->portno, &layer, + &se_idx); + + switch (qopt->command) { + case TC_TBF_REPLACE: + return sparx5_tc_tbf_add(port, &qopt->replace_params, layer, + se_idx); + case TC_TBF_DESTROY: + return sparx5_tc_tbf_del(port, layer, se_idx); + case TC_TBF_STATS: + return -EOPNOTSUPP; + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; +} + +static int sparx5_tc_setup_qdisc_ets(struct net_device *ndev, + struct tc_ets_qopt_offload *qopt) +{ + struct tc_ets_qopt_offload_replace_params *params = + &qopt->replace_params; + struct sparx5_port *port = netdev_priv(ndev); + int i; + + /* Only allow ets on ports */ + if (qopt->parent != TC_H_ROOT) + return -EOPNOTSUPP; + + switch (qopt->command) { + case TC_ETS_REPLACE: + + /* We support eight priorities */ + if (params->bands != SPX5_PRIOS) + return -EOPNOTSUPP; + + /* Sanity checks */ + for (i = 0; i < SPX5_PRIOS; ++i) { + /* Priority map is *always* reverse e.g: 7 6 5 .. 0 */ + if (params->priomap[i] != (7 - i)) + return -EOPNOTSUPP; + /* Throw an error if we receive zero weights by tc */ + if (params->quanta[i] && params->weights[i] == 0) { + pr_err("Invalid ets configuration; band %d has weight zero", + i); + return -EINVAL; + } + } + + sparx5_tc_ets_add(port, params); + break; + case TC_ETS_DESTROY: + + sparx5_tc_ets_del(port); + + break; + case TC_ETS_GRAFT: + return -EOPNOTSUPP; + + default: + return -EOPNOTSUPP; + } + + return -EOPNOTSUPP; +} + +int sparx5_port_setup_tc(struct net_device *ndev, enum tc_setup_type type, + void *type_data) +{ + switch (type) { + case TC_SETUP_QDISC_MQPRIO: + return sparx5_tc_setup_qdisc_mqprio(ndev, type_data); + case TC_SETUP_QDISC_TBF: + return sparx5_tc_setup_qdisc_tbf(ndev, type_data); + case TC_SETUP_QDISC_ETS: + return sparx5_tc_setup_qdisc_ets(ndev, type_data); + default: + return -EOPNOTSUPP; + } + + return 0; +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h new file mode 100644 index 0000000000000000000000000000000000000000..5b55e11b77e11509c0e27f3a4aaf9989e5f9ce73 --- /dev/null +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* Microchip Sparx5 Switch driver + * + * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries. + */ + +#ifndef __SPARX5_TC_H__ +#define __SPARX5_TC_H__ + +#include + +int sparx5_port_setup_tc(struct net_device *ndev, enum tc_setup_type type, + void *type_data); + +#endif /* __SPARX5_TC_H__ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c index 37e4ac965849620c2283a85b71c5ee19f6e589ac..34f954bbf815155f6bd680d4b50e0bd969a3441f 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c @@ -138,6 +138,13 @@ void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable) } } +void sparx5_pgid_clear(struct sparx5 *spx5, int pgid) +{ + spx5_wr(0, spx5, ANA_AC_PGID_CFG(pgid)); + spx5_wr(0, spx5, ANA_AC_PGID_CFG1(pgid)); + spx5_wr(0, spx5, ANA_AC_PGID_CFG2(pgid)); +} + void sparx5_pgid_read_mask(struct sparx5 *spx5, int pgid, u32 portmask[3]) { portmask[0] = spx5_rd(spx5, ANA_AC_PGID_CFG(pgid)); diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 5f924018235174ffd7f21488dc21146da3b97715..a6f99b4344d93fa813c6300aa904d3fb4c5e44da 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -397,6 +397,11 @@ static void mana_gd_process_eq_events(void *arg) break; } + /* Per GDMA spec, rmb is necessary after checking owner_bits, before + * reading eqe. + */ + rmb(); + mana_gd_process_eqe(eq); eq->head++; @@ -1134,6 +1139,11 @@ static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) if (WARN_ON_ONCE(owner_bits != new_bits)) return -1; + /* Per GDMA spec, rmb is necessary after checking owner_bits, before + * reading completion info + */ + rmb(); + comp->wq_num = cqe->cqe_info.wq_num; comp->is_sq = cqe->cqe_info.is_sq; memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); @@ -1465,10 +1475,6 @@ static void mana_gd_shutdown(struct pci_dev *pdev) pci_disable_device(pdev); } -#ifndef PCI_VENDOR_ID_MICROSOFT -#define PCI_VENDOR_ID_MICROSOFT 0x1414 -#endif - static const struct pci_device_id mana_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, diff --git a/drivers/net/ethernet/moxa/moxart_ether.c b/drivers/net/ethernet/moxa/moxart_ether.c index 19009a6bd33aedcf1912b927e367d6b64f2edb85..3da99b62797de4fff88e0b5b8d6e4f62e9a2c9f5 100644 --- a/drivers/net/ethernet/moxa/moxart_ether.c +++ b/drivers/net/ethernet/moxa/moxart_ether.c @@ -29,12 +29,12 @@ #include "moxart_ether.h" -static inline void moxart_desc_write(u32 data, u32 *desc) +static inline void moxart_desc_write(u32 data, __le32 *desc) { *desc = cpu_to_le32(data); } -static inline u32 moxart_desc_read(u32 *desc) +static inline u32 moxart_desc_read(__le32 *desc) { return le32_to_cpu(*desc); } @@ -71,11 +71,6 @@ static int moxart_set_mac_address(struct net_device *ndev, void *addr) static void moxart_mac_free_memory(struct net_device *ndev) { struct moxart_mac_priv_t *priv = netdev_priv(ndev); - int i; - - for (i = 0; i < RX_DESC_NUM; i++) - dma_unmap_single(&priv->pdev->dev, priv->rx_mapping[i], - priv->rx_buf_size, DMA_FROM_DEVICE); if (priv->tx_desc_base) dma_free_coherent(&priv->pdev->dev, @@ -187,6 +182,7 @@ static int moxart_mac_open(struct net_device *ndev) static int moxart_mac_stop(struct net_device *ndev) { struct moxart_mac_priv_t *priv = netdev_priv(ndev); + int i; napi_disable(&priv->napi); @@ -198,6 +194,11 @@ static int moxart_mac_stop(struct net_device *ndev) /* disable all functions */ writel(0, priv->base + REG_MAC_CTRL); + /* unmap areas mapped in moxart_mac_setup_desc_ring() */ + for (i = 0; i < RX_DESC_NUM; i++) + dma_unmap_single(&priv->pdev->dev, priv->rx_mapping[i], + priv->rx_buf_size, DMA_FROM_DEVICE); + return 0; } diff --git a/drivers/net/ethernet/mscc/Makefile b/drivers/net/ethernet/mscc/Makefile index 41b34a50930877e03e3d25a1fbd347fe465d119f..5d435a565d4c2a34cfd9933d76c944bd2fbc60a6 100644 --- a/drivers/net/ethernet/mscc/Makefile +++ b/drivers/net/ethernet/mscc/Makefile @@ -2,16 +2,17 @@ obj-$(CONFIG_MSCC_OCELOT_SWITCH_LIB) += mscc_ocelot_switch_lib.o mscc_ocelot_switch_lib-y := \ ocelot.o \ + ocelot_devlink.o \ + ocelot_flower.o \ ocelot_io.o \ ocelot_police.o \ - ocelot_vcap.o \ - ocelot_flower.o \ ocelot_ptp.o \ - ocelot_devlink.o \ + ocelot_stats.o \ + ocelot_vcap.o \ vsc7514_regs.o mscc_ocelot_switch_lib-$(CONFIG_BRIDGE_MRP) += ocelot_mrp.o obj-$(CONFIG_MSCC_OCELOT_SWITCH) += mscc_ocelot.o mscc_ocelot-y := \ ocelot_fdma.o \ - ocelot_vsc7514.o \ - ocelot_net.o + ocelot_net.o \ + ocelot_vsc7514.o diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 306026e6aa111b392e7be765b262d83a88078bbe..13b14110a06034990b385edd3f31b830664c2117 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -6,7 +6,6 @@ */ #include #include -#include #include #include "ocelot.h" #include "ocelot_vcap.h" @@ -290,6 +289,13 @@ static int ocelot_port_num_untagged_vlans(struct ocelot *ocelot, int port) if (!(vlan->portmask & BIT(port))) continue; + /* Ignore the VLAN added by ocelot_add_vlan_unaware_pvid(), + * because this is never active in hardware at the same time as + * the bridge VLANs, which only matter in VLAN-aware mode. + */ + if (vlan->vid >= OCELOT_RSV_VLAN_RANGE_START) + continue; + if (vlan->untagged & BIT(port)) num_untagged++; } @@ -910,211 +916,6 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port, } EXPORT_SYMBOL_GPL(ocelot_phylink_mac_link_up); -static int ocelot_port_add_txtstamp_skb(struct ocelot *ocelot, int port, - struct sk_buff *clone) -{ - struct ocelot_port *ocelot_port = ocelot->ports[port]; - unsigned long flags; - - spin_lock_irqsave(&ocelot->ts_id_lock, flags); - - if (ocelot_port->ptp_skbs_in_flight == OCELOT_MAX_PTP_ID || - ocelot->ptp_skbs_in_flight == OCELOT_PTP_FIFO_SIZE) { - spin_unlock_irqrestore(&ocelot->ts_id_lock, flags); - return -EBUSY; - } - - skb_shinfo(clone)->tx_flags |= SKBTX_IN_PROGRESS; - /* Store timestamp ID in OCELOT_SKB_CB(clone)->ts_id */ - OCELOT_SKB_CB(clone)->ts_id = ocelot_port->ts_id; - - ocelot_port->ts_id++; - if (ocelot_port->ts_id == OCELOT_MAX_PTP_ID) - ocelot_port->ts_id = 0; - - ocelot_port->ptp_skbs_in_flight++; - ocelot->ptp_skbs_in_flight++; - - skb_queue_tail(&ocelot_port->tx_skbs, clone); - - spin_unlock_irqrestore(&ocelot->ts_id_lock, flags); - - return 0; -} - -static bool ocelot_ptp_is_onestep_sync(struct sk_buff *skb, - unsigned int ptp_class) -{ - struct ptp_header *hdr; - u8 msgtype, twostep; - - hdr = ptp_parse_header(skb, ptp_class); - if (!hdr) - return false; - - msgtype = ptp_get_msgtype(hdr, ptp_class); - twostep = hdr->flag_field[0] & 0x2; - - if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) - return true; - - return false; -} - -int ocelot_port_txtstamp_request(struct ocelot *ocelot, int port, - struct sk_buff *skb, - struct sk_buff **clone) -{ - struct ocelot_port *ocelot_port = ocelot->ports[port]; - u8 ptp_cmd = ocelot_port->ptp_cmd; - unsigned int ptp_class; - int err; - - /* Don't do anything if PTP timestamping not enabled */ - if (!ptp_cmd) - return 0; - - ptp_class = ptp_classify_raw(skb); - if (ptp_class == PTP_CLASS_NONE) - return -EINVAL; - - /* Store ptp_cmd in OCELOT_SKB_CB(skb)->ptp_cmd */ - if (ptp_cmd == IFH_REW_OP_ORIGIN_PTP) { - if (ocelot_ptp_is_onestep_sync(skb, ptp_class)) { - OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd; - return 0; - } - - /* Fall back to two-step timestamping */ - ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; - } - - if (ptp_cmd == IFH_REW_OP_TWO_STEP_PTP) { - *clone = skb_clone_sk(skb); - if (!(*clone)) - return -ENOMEM; - - err = ocelot_port_add_txtstamp_skb(ocelot, port, *clone); - if (err) - return err; - - OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd; - OCELOT_SKB_CB(*clone)->ptp_class = ptp_class; - } - - return 0; -} -EXPORT_SYMBOL(ocelot_port_txtstamp_request); - -static void ocelot_get_hwtimestamp(struct ocelot *ocelot, - struct timespec64 *ts) -{ - unsigned long flags; - u32 val; - - spin_lock_irqsave(&ocelot->ptp_clock_lock, flags); - - /* Read current PTP time to get seconds */ - val = ocelot_read_rix(ocelot, PTP_PIN_CFG, TOD_ACC_PIN); - - val &= ~(PTP_PIN_CFG_SYNC | PTP_PIN_CFG_ACTION_MASK | PTP_PIN_CFG_DOM); - val |= PTP_PIN_CFG_ACTION(PTP_PIN_ACTION_SAVE); - ocelot_write_rix(ocelot, val, PTP_PIN_CFG, TOD_ACC_PIN); - ts->tv_sec = ocelot_read_rix(ocelot, PTP_PIN_TOD_SEC_LSB, TOD_ACC_PIN); - - /* Read packet HW timestamp from FIFO */ - val = ocelot_read(ocelot, SYS_PTP_TXSTAMP); - ts->tv_nsec = SYS_PTP_TXSTAMP_PTP_TXSTAMP(val); - - /* Sec has incremented since the ts was registered */ - if ((ts->tv_sec & 0x1) != !!(val & SYS_PTP_TXSTAMP_PTP_TXSTAMP_SEC)) - ts->tv_sec--; - - spin_unlock_irqrestore(&ocelot->ptp_clock_lock, flags); -} - -static bool ocelot_validate_ptp_skb(struct sk_buff *clone, u16 seqid) -{ - struct ptp_header *hdr; - - hdr = ptp_parse_header(clone, OCELOT_SKB_CB(clone)->ptp_class); - if (WARN_ON(!hdr)) - return false; - - return seqid == ntohs(hdr->sequence_id); -} - -void ocelot_get_txtstamp(struct ocelot *ocelot) -{ - int budget = OCELOT_PTP_QUEUE_SZ; - - while (budget--) { - struct sk_buff *skb, *skb_tmp, *skb_match = NULL; - struct skb_shared_hwtstamps shhwtstamps; - u32 val, id, seqid, txport; - struct ocelot_port *port; - struct timespec64 ts; - unsigned long flags; - - val = ocelot_read(ocelot, SYS_PTP_STATUS); - - /* Check if a timestamp can be retrieved */ - if (!(val & SYS_PTP_STATUS_PTP_MESS_VLD)) - break; - - WARN_ON(val & SYS_PTP_STATUS_PTP_OVFL); - - /* Retrieve the ts ID and Tx port */ - id = SYS_PTP_STATUS_PTP_MESS_ID_X(val); - txport = SYS_PTP_STATUS_PTP_MESS_TXPORT_X(val); - seqid = SYS_PTP_STATUS_PTP_MESS_SEQ_ID(val); - - port = ocelot->ports[txport]; - - spin_lock(&ocelot->ts_id_lock); - port->ptp_skbs_in_flight--; - ocelot->ptp_skbs_in_flight--; - spin_unlock(&ocelot->ts_id_lock); - - /* Retrieve its associated skb */ -try_again: - spin_lock_irqsave(&port->tx_skbs.lock, flags); - - skb_queue_walk_safe(&port->tx_skbs, skb, skb_tmp) { - if (OCELOT_SKB_CB(skb)->ts_id != id) - continue; - __skb_unlink(skb, &port->tx_skbs); - skb_match = skb; - break; - } - - spin_unlock_irqrestore(&port->tx_skbs.lock, flags); - - if (WARN_ON(!skb_match)) - continue; - - if (!ocelot_validate_ptp_skb(skb_match, seqid)) { - dev_err_ratelimited(ocelot->dev, - "port %d received stale TX timestamp for seqid %d, discarding\n", - txport, seqid); - dev_kfree_skb_any(skb); - goto try_again; - } - - /* Get the h/w timestamp */ - ocelot_get_hwtimestamp(ocelot, &ts); - - /* Set the timestamp into the skb */ - memset(&shhwtstamps, 0, sizeof(shhwtstamps)); - shhwtstamps.hwtstamp = ktime_set(ts.tv_sec, ts.tv_nsec); - skb_complete_tx_timestamp(skb_match, &shhwtstamps); - - /* Next ts */ - ocelot_write(ocelot, SYS_PTP_NXT_PTP_NXT, SYS_PTP_NXT); - } -} -EXPORT_SYMBOL(ocelot_get_txtstamp); - static int ocelot_rx_frame_word(struct ocelot *ocelot, u8 grp, bool ifh, u32 *rval) { @@ -1366,50 +1167,6 @@ int ocelot_fdb_del(struct ocelot *ocelot, int port, const unsigned char *addr, } EXPORT_SYMBOL(ocelot_fdb_del); -int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid, - bool is_static, void *data) -{ - struct ocelot_dump_ctx *dump = data; - u32 portid = NETLINK_CB(dump->cb->skb).portid; - u32 seq = dump->cb->nlh->nlmsg_seq; - struct nlmsghdr *nlh; - struct ndmsg *ndm; - - if (dump->idx < dump->cb->args[2]) - goto skip; - - nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH, - sizeof(*ndm), NLM_F_MULTI); - if (!nlh) - return -EMSGSIZE; - - ndm = nlmsg_data(nlh); - ndm->ndm_family = AF_BRIDGE; - ndm->ndm_pad1 = 0; - ndm->ndm_pad2 = 0; - ndm->ndm_flags = NTF_SELF; - ndm->ndm_type = 0; - ndm->ndm_ifindex = dump->dev->ifindex; - ndm->ndm_state = is_static ? NUD_NOARP : NUD_REACHABLE; - - if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, addr)) - goto nla_put_failure; - - if (vid && nla_put_u16(dump->skb, NDA_VLAN, vid)) - goto nla_put_failure; - - nlmsg_end(dump->skb, nlh); - -skip: - dump->idx++; - return 0; - -nla_put_failure: - nlmsg_cancel(dump->skb, nlh); - return -EMSGSIZE; -} -EXPORT_SYMBOL(ocelot_port_fdb_do_dump); - /* Caller must hold &ocelot->mact_lock */ static int ocelot_mact_read(struct ocelot *ocelot, int port, int row, int col, struct ocelot_mact_entry *entry) @@ -1541,53 +1298,6 @@ int ocelot_fdb_dump(struct ocelot *ocelot, int port, } EXPORT_SYMBOL(ocelot_fdb_dump); -static void ocelot_populate_l2_ptp_trap_key(struct ocelot_vcap_filter *trap) -{ - trap->key_type = OCELOT_VCAP_KEY_ETYPE; - *(__be16 *)trap->key.etype.etype.value = htons(ETH_P_1588); - *(__be16 *)trap->key.etype.etype.mask = htons(0xffff); -} - -static void -ocelot_populate_ipv4_ptp_event_trap_key(struct ocelot_vcap_filter *trap) -{ - trap->key_type = OCELOT_VCAP_KEY_IPV4; - trap->key.ipv4.proto.value[0] = IPPROTO_UDP; - trap->key.ipv4.proto.mask[0] = 0xff; - trap->key.ipv4.dport.value = PTP_EV_PORT; - trap->key.ipv4.dport.mask = 0xffff; -} - -static void -ocelot_populate_ipv6_ptp_event_trap_key(struct ocelot_vcap_filter *trap) -{ - trap->key_type = OCELOT_VCAP_KEY_IPV6; - trap->key.ipv4.proto.value[0] = IPPROTO_UDP; - trap->key.ipv4.proto.mask[0] = 0xff; - trap->key.ipv6.dport.value = PTP_EV_PORT; - trap->key.ipv6.dport.mask = 0xffff; -} - -static void -ocelot_populate_ipv4_ptp_general_trap_key(struct ocelot_vcap_filter *trap) -{ - trap->key_type = OCELOT_VCAP_KEY_IPV4; - trap->key.ipv4.proto.value[0] = IPPROTO_UDP; - trap->key.ipv4.proto.mask[0] = 0xff; - trap->key.ipv4.dport.value = PTP_GEN_PORT; - trap->key.ipv4.dport.mask = 0xffff; -} - -static void -ocelot_populate_ipv6_ptp_general_trap_key(struct ocelot_vcap_filter *trap) -{ - trap->key_type = OCELOT_VCAP_KEY_IPV6; - trap->key.ipv4.proto.value[0] = IPPROTO_UDP; - trap->key.ipv4.proto.mask[0] = 0xff; - trap->key.ipv6.dport.value = PTP_GEN_PORT; - trap->key.ipv6.dport.mask = 0xffff; -} - int ocelot_trap_add(struct ocelot *ocelot, int port, unsigned long cookie, bool take_ts, void (*populate)(struct ocelot_vcap_filter *f)) @@ -1656,381 +1366,6 @@ int ocelot_trap_del(struct ocelot *ocelot, int port, unsigned long cookie) return ocelot_vcap_filter_replace(ocelot, trap); } -static int ocelot_l2_ptp_trap_add(struct ocelot *ocelot, int port) -{ - unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot); - - return ocelot_trap_add(ocelot, port, l2_cookie, true, - ocelot_populate_l2_ptp_trap_key); -} - -static int ocelot_l2_ptp_trap_del(struct ocelot *ocelot, int port) -{ - unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot); - - return ocelot_trap_del(ocelot, port, l2_cookie); -} - -static int ocelot_ipv4_ptp_trap_add(struct ocelot *ocelot, int port) -{ - unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot); - unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot); - int err; - - err = ocelot_trap_add(ocelot, port, ipv4_ev_cookie, true, - ocelot_populate_ipv4_ptp_event_trap_key); - if (err) - return err; - - err = ocelot_trap_add(ocelot, port, ipv4_gen_cookie, false, - ocelot_populate_ipv4_ptp_general_trap_key); - if (err) - ocelot_trap_del(ocelot, port, ipv4_ev_cookie); - - return err; -} - -static int ocelot_ipv4_ptp_trap_del(struct ocelot *ocelot, int port) -{ - unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot); - unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot); - int err; - - err = ocelot_trap_del(ocelot, port, ipv4_ev_cookie); - err |= ocelot_trap_del(ocelot, port, ipv4_gen_cookie); - return err; -} - -static int ocelot_ipv6_ptp_trap_add(struct ocelot *ocelot, int port) -{ - unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot); - unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot); - int err; - - err = ocelot_trap_add(ocelot, port, ipv6_ev_cookie, true, - ocelot_populate_ipv6_ptp_event_trap_key); - if (err) - return err; - - err = ocelot_trap_add(ocelot, port, ipv6_gen_cookie, false, - ocelot_populate_ipv6_ptp_general_trap_key); - if (err) - ocelot_trap_del(ocelot, port, ipv6_ev_cookie); - - return err; -} - -static int ocelot_ipv6_ptp_trap_del(struct ocelot *ocelot, int port) -{ - unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot); - unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot); - int err; - - err = ocelot_trap_del(ocelot, port, ipv6_ev_cookie); - err |= ocelot_trap_del(ocelot, port, ipv6_gen_cookie); - return err; -} - -static int ocelot_setup_ptp_traps(struct ocelot *ocelot, int port, - bool l2, bool l4) -{ - int err; - - if (l2) - err = ocelot_l2_ptp_trap_add(ocelot, port); - else - err = ocelot_l2_ptp_trap_del(ocelot, port); - if (err) - return err; - - if (l4) { - err = ocelot_ipv4_ptp_trap_add(ocelot, port); - if (err) - goto err_ipv4; - - err = ocelot_ipv6_ptp_trap_add(ocelot, port); - if (err) - goto err_ipv6; - } else { - err = ocelot_ipv4_ptp_trap_del(ocelot, port); - - err |= ocelot_ipv6_ptp_trap_del(ocelot, port); - } - if (err) - return err; - - return 0; - -err_ipv6: - ocelot_ipv4_ptp_trap_del(ocelot, port); -err_ipv4: - if (l2) - ocelot_l2_ptp_trap_del(ocelot, port); - return err; -} - -int ocelot_hwstamp_get(struct ocelot *ocelot, int port, struct ifreq *ifr) -{ - return copy_to_user(ifr->ifr_data, &ocelot->hwtstamp_config, - sizeof(ocelot->hwtstamp_config)) ? -EFAULT : 0; -} -EXPORT_SYMBOL(ocelot_hwstamp_get); - -int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr) -{ - struct ocelot_port *ocelot_port = ocelot->ports[port]; - bool l2 = false, l4 = false; - struct hwtstamp_config cfg; - int err; - - if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) - return -EFAULT; - - /* Tx type sanity check */ - switch (cfg.tx_type) { - case HWTSTAMP_TX_ON: - ocelot_port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; - break; - case HWTSTAMP_TX_ONESTEP_SYNC: - /* IFH_REW_OP_ONE_STEP_PTP updates the correctional field, we - * need to update the origin time. - */ - ocelot_port->ptp_cmd = IFH_REW_OP_ORIGIN_PTP; - break; - case HWTSTAMP_TX_OFF: - ocelot_port->ptp_cmd = 0; - break; - default: - return -ERANGE; - } - - mutex_lock(&ocelot->ptp_lock); - - switch (cfg.rx_filter) { - case HWTSTAMP_FILTER_NONE: - break; - case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: - l4 = true; - break; - case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: - l2 = true; - break; - case HWTSTAMP_FILTER_PTP_V2_EVENT: - case HWTSTAMP_FILTER_PTP_V2_SYNC: - case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: - l2 = true; - l4 = true; - break; - default: - mutex_unlock(&ocelot->ptp_lock); - return -ERANGE; - } - - err = ocelot_setup_ptp_traps(ocelot, port, l2, l4); - if (err) { - mutex_unlock(&ocelot->ptp_lock); - return err; - } - - if (l2 && l4) - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - else if (l2) - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; - else if (l4) - cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; - else - cfg.rx_filter = HWTSTAMP_FILTER_NONE; - - /* Commit back the result & save it */ - memcpy(&ocelot->hwtstamp_config, &cfg, sizeof(cfg)); - mutex_unlock(&ocelot->ptp_lock); - - return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; -} -EXPORT_SYMBOL(ocelot_hwstamp_set); - -void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data) -{ - int i; - - if (sset != ETH_SS_STATS) - return; - - for (i = 0; i < OCELOT_NUM_STATS; i++) { - if (ocelot->stats_layout[i].name[0] == '\0') - continue; - - memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name, - ETH_GSTRING_LEN); - } -} -EXPORT_SYMBOL(ocelot_get_strings); - -/* Caller must hold &ocelot->stats_lock */ -static int ocelot_port_update_stats(struct ocelot *ocelot, int port) -{ - unsigned int idx = port * OCELOT_NUM_STATS; - struct ocelot_stats_region *region; - int err, j; - - /* Configure the port to read the stats from */ - ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port), SYS_STAT_CFG); - - list_for_each_entry(region, &ocelot->stats_regions, node) { - err = ocelot_bulk_read(ocelot, region->base, region->buf, - region->count); - if (err) - return err; - - for (j = 0; j < region->count; j++) { - u64 *stat = &ocelot->stats[idx + j]; - u64 val = region->buf[j]; - - if (val < (*stat & U32_MAX)) - *stat += (u64)1 << 32; - - *stat = (*stat & ~(u64)U32_MAX) + val; - } - - idx += region->count; - } - - return err; -} - -static void ocelot_check_stats_work(struct work_struct *work) -{ - struct delayed_work *del_work = to_delayed_work(work); - struct ocelot *ocelot = container_of(del_work, struct ocelot, - stats_work); - int i, err; - - spin_lock(&ocelot->stats_lock); - for (i = 0; i < ocelot->num_phys_ports; i++) { - err = ocelot_port_update_stats(ocelot, i); - if (err) - break; - } - spin_unlock(&ocelot->stats_lock); - - if (err) - dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err); - - queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work, - OCELOT_STATS_CHECK_DELAY); -} - -void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data) -{ - int i, err; - - spin_lock(&ocelot->stats_lock); - - /* check and update now */ - err = ocelot_port_update_stats(ocelot, port); - - /* Copy all supported counters */ - for (i = 0; i < OCELOT_NUM_STATS; i++) { - int index = port * OCELOT_NUM_STATS + i; - - if (ocelot->stats_layout[i].name[0] == '\0') - continue; - - *data++ = ocelot->stats[index]; - } - - spin_unlock(&ocelot->stats_lock); - - if (err) - dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err); -} -EXPORT_SYMBOL(ocelot_get_ethtool_stats); - -int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset) -{ - int i, num_stats = 0; - - if (sset != ETH_SS_STATS) - return -EOPNOTSUPP; - - for (i = 0; i < OCELOT_NUM_STATS; i++) - if (ocelot->stats_layout[i].name[0] != '\0') - num_stats++; - - return num_stats; -} -EXPORT_SYMBOL(ocelot_get_sset_count); - -static int ocelot_prepare_stats_regions(struct ocelot *ocelot) -{ - struct ocelot_stats_region *region = NULL; - unsigned int last; - int i; - - INIT_LIST_HEAD(&ocelot->stats_regions); - - for (i = 0; i < OCELOT_NUM_STATS; i++) { - if (ocelot->stats_layout[i].name[0] == '\0') - continue; - - if (region && ocelot->stats_layout[i].reg == last + 4) { - region->count++; - } else { - region = devm_kzalloc(ocelot->dev, sizeof(*region), - GFP_KERNEL); - if (!region) - return -ENOMEM; - - region->base = ocelot->stats_layout[i].reg; - region->count = 1; - list_add_tail(®ion->node, &ocelot->stats_regions); - } - - last = ocelot->stats_layout[i].reg; - } - - list_for_each_entry(region, &ocelot->stats_regions, node) { - region->buf = devm_kcalloc(ocelot->dev, region->count, - sizeof(*region->buf), GFP_KERNEL); - if (!region->buf) - return -ENOMEM; - } - - return 0; -} - -int ocelot_get_ts_info(struct ocelot *ocelot, int port, - struct ethtool_ts_info *info) -{ - info->phc_index = ocelot->ptp_clock ? - ptp_clock_index(ocelot->ptp_clock) : -1; - if (info->phc_index == -1) { - info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE; - return 0; - } - info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | - SOF_TIMESTAMPING_TX_HARDWARE | - SOF_TIMESTAMPING_RX_HARDWARE | - SOF_TIMESTAMPING_RAW_HARDWARE; - info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON) | - BIT(HWTSTAMP_TX_ONESTEP_SYNC); - info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | - BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | - BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | - BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT); - - return 0; -} -EXPORT_SYMBOL(ocelot_get_ts_info); - static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond) { u32 mask = 0; @@ -2054,7 +1389,7 @@ static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond) /* The logical port number of a LAG is equal to the lowest numbered physical * port ID present in that LAG. It may change if that port ever leaves the LAG. */ -static int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond) +int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond) { int bond_mask = ocelot_get_bond_mask(ocelot, bond); @@ -2063,7 +1398,18 @@ static int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond) return __ffs(bond_mask); } +EXPORT_SYMBOL_GPL(ocelot_bond_get_id); +/* Returns the mask of user ports assigned to this DSA tag_8021q CPU port. + * Note that when CPU ports are in a LAG, the user ports are assigned to the + * 'primary' CPU port, the one whose physical port number gives the logical + * port number of the LAG. + * + * We leave PGID_SRC poorly configured for the 'secondary' CPU port in the LAG + * (to which no user port is assigned), but it appears that forwarding from + * this secondary CPU port looks at the PGID_SRC associated with the logical + * port ID that it's assigned to, which *is* configured properly. + */ static u32 ocelot_dsa_8021q_cpu_assigned_ports(struct ocelot *ocelot, struct ocelot_port *cpu) { @@ -2080,9 +1426,15 @@ static u32 ocelot_dsa_8021q_cpu_assigned_ports(struct ocelot *ocelot, mask |= BIT(port); } + if (cpu->bond) + mask &= ~ocelot_get_bond_mask(ocelot, cpu->bond); + return mask; } +/* Returns the DSA tag_8021q CPU port that the given port is assigned to, + * or the bit mask of CPU ports if said CPU port is in a LAG. + */ u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port) { struct ocelot_port *ocelot_port = ocelot->ports[port]; @@ -2091,6 +1443,9 @@ u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port) if (!cpu_port) return 0; + if (cpu_port->bond) + return ocelot_get_bond_mask(ocelot, cpu_port->bond); + return BIT(cpu_port->index); } EXPORT_SYMBOL_GPL(ocelot_port_assigned_dsa_8021q_cpu_mask); @@ -2214,61 +1569,61 @@ static void ocelot_update_pgid_cpu(struct ocelot *ocelot) ocelot_write_rix(ocelot, pgid_cpu, ANA_PGID_PGID, PGID_CPU); } -void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port, - int cpu) +void ocelot_port_setup_dsa_8021q_cpu(struct ocelot *ocelot, int cpu) { struct ocelot_port *cpu_port = ocelot->ports[cpu]; u16 vid; mutex_lock(&ocelot->fwd_domain_lock); - ocelot->ports[port]->dsa_8021q_cpu = cpu_port; - - if (!cpu_port->is_dsa_8021q_cpu) { - cpu_port->is_dsa_8021q_cpu = true; + cpu_port->is_dsa_8021q_cpu = true; - for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++) - ocelot_vlan_member_add(ocelot, cpu, vid, true); + for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++) + ocelot_vlan_member_add(ocelot, cpu, vid, true); - ocelot_update_pgid_cpu(ocelot); - } - - ocelot_apply_bridge_fwd_mask(ocelot, true); + ocelot_update_pgid_cpu(ocelot); mutex_unlock(&ocelot->fwd_domain_lock); } -EXPORT_SYMBOL_GPL(ocelot_port_assign_dsa_8021q_cpu); +EXPORT_SYMBOL_GPL(ocelot_port_setup_dsa_8021q_cpu); -void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port) +void ocelot_port_teardown_dsa_8021q_cpu(struct ocelot *ocelot, int cpu) { - struct ocelot_port *cpu_port = ocelot->ports[port]->dsa_8021q_cpu; - bool keep = false; + struct ocelot_port *cpu_port = ocelot->ports[cpu]; u16 vid; - int p; mutex_lock(&ocelot->fwd_domain_lock); - ocelot->ports[port]->dsa_8021q_cpu = NULL; + cpu_port->is_dsa_8021q_cpu = false; - for (p = 0; p < ocelot->num_phys_ports; p++) { - if (!ocelot->ports[p]) - continue; + for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++) + ocelot_vlan_member_del(ocelot, cpu_port->index, vid); - if (ocelot->ports[p]->dsa_8021q_cpu == cpu_port) { - keep = true; - break; - } - } + ocelot_update_pgid_cpu(ocelot); - if (!keep) { - cpu_port->is_dsa_8021q_cpu = false; + mutex_unlock(&ocelot->fwd_domain_lock); +} +EXPORT_SYMBOL_GPL(ocelot_port_teardown_dsa_8021q_cpu); - for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++) - ocelot_vlan_member_del(ocelot, cpu_port->index, vid); +void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port, + int cpu) +{ + struct ocelot_port *cpu_port = ocelot->ports[cpu]; - ocelot_update_pgid_cpu(ocelot); - } + mutex_lock(&ocelot->fwd_domain_lock); + + ocelot->ports[port]->dsa_8021q_cpu = cpu_port; + ocelot_apply_bridge_fwd_mask(ocelot, true); + + mutex_unlock(&ocelot->fwd_domain_lock); +} +EXPORT_SYMBOL_GPL(ocelot_port_assign_dsa_8021q_cpu); + +void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port) +{ + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ports[port]->dsa_8021q_cpu = NULL; ocelot_apply_bridge_fwd_mask(ocelot, true); mutex_unlock(&ocelot->fwd_domain_lock); @@ -2785,10 +2140,14 @@ static void ocelot_migrate_lag_fdbs(struct ocelot *ocelot, int ocelot_port_lag_join(struct ocelot *ocelot, int port, struct net_device *bond, - struct netdev_lag_upper_info *info) + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack) { - if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) + if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) { + NL_SET_ERR_MSG_MOD(extack, + "Can only offload LAG using hash TX type"); return -EOPNOTSUPP; + } mutex_lock(&ocelot->fwd_domain_lock); @@ -3358,7 +2717,6 @@ static void ocelot_detect_features(struct ocelot *ocelot) int ocelot_init(struct ocelot *ocelot) { - char queue_name[32]; int i, ret; u32 port; @@ -3370,29 +2728,21 @@ int ocelot_init(struct ocelot *ocelot) } } - ocelot->stats = devm_kcalloc(ocelot->dev, - ocelot->num_phys_ports * OCELOT_NUM_STATS, - sizeof(u64), GFP_KERNEL); - if (!ocelot->stats) - return -ENOMEM; - - spin_lock_init(&ocelot->stats_lock); mutex_init(&ocelot->ptp_lock); mutex_init(&ocelot->mact_lock); mutex_init(&ocelot->fwd_domain_lock); mutex_init(&ocelot->tas_lock); spin_lock_init(&ocelot->ptp_clock_lock); spin_lock_init(&ocelot->ts_id_lock); - snprintf(queue_name, sizeof(queue_name), "%s-stats", - dev_name(ocelot->dev)); - ocelot->stats_queue = create_singlethread_workqueue(queue_name); - if (!ocelot->stats_queue) - return -ENOMEM; ocelot->owq = alloc_ordered_workqueue("ocelot-owq", 0); - if (!ocelot->owq) { - destroy_workqueue(ocelot->stats_queue); + if (!ocelot->owq) return -ENOMEM; + + ret = ocelot_stats_init(ocelot); + if (ret) { + destroy_workqueue(ocelot->owq); + return ret; } INIT_LIST_HEAD(&ocelot->multicast); @@ -3504,25 +2854,13 @@ int ocelot_init(struct ocelot *ocelot) ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL(6), ANA_CPUQ_8021_CFG, i); - ret = ocelot_prepare_stats_regions(ocelot); - if (ret) { - destroy_workqueue(ocelot->stats_queue); - destroy_workqueue(ocelot->owq); - return ret; - } - - INIT_DELAYED_WORK(&ocelot->stats_work, ocelot_check_stats_work); - queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work, - OCELOT_STATS_CHECK_DELAY); - return 0; } EXPORT_SYMBOL(ocelot_init); void ocelot_deinit(struct ocelot *ocelot) { - cancel_delayed_work(&ocelot->stats_work); - destroy_workqueue(ocelot->stats_queue); + ocelot_stats_deinit(ocelot); destroy_workqueue(ocelot->owq); } EXPORT_SYMBOL(ocelot_deinit); diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h index 6d65cc87d757ac432a88ae09079b4f8450f4b12b..70dbd9c4e5123eba948b7733ef2731969b4b00d9 100644 --- a/drivers/net/ethernet/mscc/ocelot.h +++ b/drivers/net/ethernet/mscc/ocelot.h @@ -51,13 +51,6 @@ struct ocelot_port_private { struct ocelot_port_tc tc; }; -struct ocelot_dump_ctx { - struct net_device *dev; - struct sk_buff *skb; - struct netlink_callback *cb; - int idx; -}; - /* A (PGID) port mask structure, encoding the 2^ocelot->num_phys_ports * possibilities of egress port masks for L2 multicast traffic. * For a switch with 9 user ports, there are 512 possible port masks, but the @@ -84,8 +77,6 @@ struct ocelot_multicast { int ocelot_bridge_num_find(struct ocelot *ocelot, const struct net_device *bridge); -int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid, - bool is_static, void *data); int ocelot_mact_learn(struct ocelot *ocelot, int port, const unsigned char mac[ETH_ALEN], unsigned int vid, enum macaccess_entry_type type); @@ -115,6 +106,9 @@ struct ocelot_mirror *ocelot_mirror_get(struct ocelot *ocelot, int to, struct netlink_ext_ack *extack); void ocelot_mirror_put(struct ocelot *ocelot); +int ocelot_stats_init(struct ocelot *ocelot); +void ocelot_stats_deinit(struct ocelot *ocelot); + extern struct notifier_block ocelot_netdevice_nb; extern struct notifier_block ocelot_switchdev_nb; extern struct notifier_block ocelot_switchdev_blocking_nb; diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index 330d30841cdc47c351531e26c34cb339c1c41c93..50858cc10fef6b77791c9e682cbd7d2bbfc6a285 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -20,6 +20,13 @@ #define OCELOT_MAC_QUIRKS OCELOT_QUIRK_QSGMII_PORTS_MUST_BE_UP +struct ocelot_dump_ctx { + struct net_device *dev; + struct sk_buff *skb; + struct netlink_callback *cb; + int idx; +}; + static bool ocelot_netdevice_dev_check(const struct net_device *dev); static struct ocelot *devlink_port_to_ocelot(struct devlink_port *dlp) @@ -725,42 +732,8 @@ static void ocelot_get_stats64(struct net_device *dev, struct ocelot_port_private *priv = netdev_priv(dev); struct ocelot *ocelot = priv->port.ocelot; int port = priv->port.index; - u64 *s; - - spin_lock(&ocelot->stats_lock); - - s = &ocelot->stats[port * OCELOT_NUM_STATS]; - - /* Get Rx stats */ - stats->rx_bytes = s[OCELOT_STAT_RX_OCTETS]; - stats->rx_packets = s[OCELOT_STAT_RX_SHORTS] + - s[OCELOT_STAT_RX_FRAGMENTS] + - s[OCELOT_STAT_RX_JABBERS] + - s[OCELOT_STAT_RX_LONGS] + - s[OCELOT_STAT_RX_64] + - s[OCELOT_STAT_RX_65_127] + - s[OCELOT_STAT_RX_128_255] + - s[OCELOT_STAT_RX_256_511] + - s[OCELOT_STAT_RX_512_1023] + - s[OCELOT_STAT_RX_1024_1526] + - s[OCELOT_STAT_RX_1527_MAX]; - stats->multicast = s[OCELOT_STAT_RX_MULTICAST]; - stats->rx_dropped = dev->stats.rx_dropped; - - /* Get Tx stats */ - stats->tx_bytes = s[OCELOT_STAT_TX_OCTETS]; - stats->tx_packets = s[OCELOT_STAT_TX_64] + - s[OCELOT_STAT_TX_65_127] + - s[OCELOT_STAT_TX_128_255] + - s[OCELOT_STAT_TX_256_511] + - s[OCELOT_STAT_TX_512_1023] + - s[OCELOT_STAT_TX_1024_1526] + - s[OCELOT_STAT_TX_1527_MAX]; - stats->tx_dropped = s[OCELOT_STAT_TX_DROPS] + - s[OCELOT_STAT_TX_AGED]; - stats->collisions = s[OCELOT_STAT_TX_COLLISION]; - - spin_unlock(&ocelot->stats_lock); + + return ocelot_port_get_stats64(ocelot, port, stats); } static int ocelot_port_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], @@ -790,6 +763,49 @@ static int ocelot_port_fdb_del(struct ndmsg *ndm, struct nlattr *tb[], return ocelot_fdb_del(ocelot, port, addr, vid, ocelot_port->bridge); } +static int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid, + bool is_static, void *data) +{ + struct ocelot_dump_ctx *dump = data; + u32 portid = NETLINK_CB(dump->cb->skb).portid; + u32 seq = dump->cb->nlh->nlmsg_seq; + struct nlmsghdr *nlh; + struct ndmsg *ndm; + + if (dump->idx < dump->cb->args[2]) + goto skip; + + nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH, + sizeof(*ndm), NLM_F_MULTI); + if (!nlh) + return -EMSGSIZE; + + ndm = nlmsg_data(nlh); + ndm->ndm_family = AF_BRIDGE; + ndm->ndm_pad1 = 0; + ndm->ndm_pad2 = 0; + ndm->ndm_flags = NTF_SELF; + ndm->ndm_type = 0; + ndm->ndm_ifindex = dump->dev->ifindex; + ndm->ndm_state = is_static ? NUD_NOARP : NUD_REACHABLE; + + if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, addr)) + goto nla_put_failure; + + if (vid && nla_put_u16(dump->skb, NDA_VLAN, vid)) + goto nla_put_failure; + + nlmsg_end(dump->skb, nlh); + +skip: + dump->idx++; + return 0; + +nla_put_failure: + nlmsg_cancel(dump->skb, nlh); + return -EMSGSIZE; +} + static int ocelot_port_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb, struct net_device *dev, @@ -1396,11 +1412,10 @@ static int ocelot_netdevice_lag_join(struct net_device *dev, int port = priv->port.index; int err; - err = ocelot_port_lag_join(ocelot, port, bond, info); - if (err == -EOPNOTSUPP) { - NL_SET_ERR_MSG_MOD(extack, "Offloading not supported"); + err = ocelot_port_lag_join(ocelot, port, bond, info, extack); + if (err == -EOPNOTSUPP) + /* Offloading not supported, fall back to software LAG */ return 0; - } bridge_dev = netdev_master_upper_dev_get(bond); if (!bridge_dev || !netif_is_bridge_master(bridge_dev)) diff --git a/drivers/net/ethernet/mscc/ocelot_ptp.c b/drivers/net/ethernet/mscc/ocelot_ptp.c index 09c703efe946c055b4b46a6711c6deb9299869c2..1a82f10c8853943b141fbc6fbe29d0e7f6f4b0f4 100644 --- a/drivers/net/ethernet/mscc/ocelot_ptp.c +++ b/drivers/net/ethernet/mscc/ocelot_ptp.c @@ -6,9 +6,13 @@ */ #include +#include +#include #include #include +#include #include +#include "ocelot.h" int ocelot_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts) { @@ -310,6 +314,483 @@ int ocelot_ptp_enable(struct ptp_clock_info *ptp, } EXPORT_SYMBOL(ocelot_ptp_enable); +static void ocelot_populate_l2_ptp_trap_key(struct ocelot_vcap_filter *trap) +{ + trap->key_type = OCELOT_VCAP_KEY_ETYPE; + *(__be16 *)trap->key.etype.etype.value = htons(ETH_P_1588); + *(__be16 *)trap->key.etype.etype.mask = htons(0xffff); +} + +static void +ocelot_populate_ipv4_ptp_event_trap_key(struct ocelot_vcap_filter *trap) +{ + trap->key_type = OCELOT_VCAP_KEY_IPV4; + trap->key.ipv4.proto.value[0] = IPPROTO_UDP; + trap->key.ipv4.proto.mask[0] = 0xff; + trap->key.ipv4.dport.value = PTP_EV_PORT; + trap->key.ipv4.dport.mask = 0xffff; +} + +static void +ocelot_populate_ipv6_ptp_event_trap_key(struct ocelot_vcap_filter *trap) +{ + trap->key_type = OCELOT_VCAP_KEY_IPV6; + trap->key.ipv4.proto.value[0] = IPPROTO_UDP; + trap->key.ipv4.proto.mask[0] = 0xff; + trap->key.ipv6.dport.value = PTP_EV_PORT; + trap->key.ipv6.dport.mask = 0xffff; +} + +static void +ocelot_populate_ipv4_ptp_general_trap_key(struct ocelot_vcap_filter *trap) +{ + trap->key_type = OCELOT_VCAP_KEY_IPV4; + trap->key.ipv4.proto.value[0] = IPPROTO_UDP; + trap->key.ipv4.proto.mask[0] = 0xff; + trap->key.ipv4.dport.value = PTP_GEN_PORT; + trap->key.ipv4.dport.mask = 0xffff; +} + +static void +ocelot_populate_ipv6_ptp_general_trap_key(struct ocelot_vcap_filter *trap) +{ + trap->key_type = OCELOT_VCAP_KEY_IPV6; + trap->key.ipv4.proto.value[0] = IPPROTO_UDP; + trap->key.ipv4.proto.mask[0] = 0xff; + trap->key.ipv6.dport.value = PTP_GEN_PORT; + trap->key.ipv6.dport.mask = 0xffff; +} + +static int ocelot_l2_ptp_trap_add(struct ocelot *ocelot, int port) +{ + unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot); + + return ocelot_trap_add(ocelot, port, l2_cookie, true, + ocelot_populate_l2_ptp_trap_key); +} + +static int ocelot_l2_ptp_trap_del(struct ocelot *ocelot, int port) +{ + unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot); + + return ocelot_trap_del(ocelot, port, l2_cookie); +} + +static int ocelot_ipv4_ptp_trap_add(struct ocelot *ocelot, int port) +{ + unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot); + unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot); + int err; + + err = ocelot_trap_add(ocelot, port, ipv4_ev_cookie, true, + ocelot_populate_ipv4_ptp_event_trap_key); + if (err) + return err; + + err = ocelot_trap_add(ocelot, port, ipv4_gen_cookie, false, + ocelot_populate_ipv4_ptp_general_trap_key); + if (err) + ocelot_trap_del(ocelot, port, ipv4_ev_cookie); + + return err; +} + +static int ocelot_ipv4_ptp_trap_del(struct ocelot *ocelot, int port) +{ + unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot); + unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot); + int err; + + err = ocelot_trap_del(ocelot, port, ipv4_ev_cookie); + err |= ocelot_trap_del(ocelot, port, ipv4_gen_cookie); + return err; +} + +static int ocelot_ipv6_ptp_trap_add(struct ocelot *ocelot, int port) +{ + unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot); + unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot); + int err; + + err = ocelot_trap_add(ocelot, port, ipv6_ev_cookie, true, + ocelot_populate_ipv6_ptp_event_trap_key); + if (err) + return err; + + err = ocelot_trap_add(ocelot, port, ipv6_gen_cookie, false, + ocelot_populate_ipv6_ptp_general_trap_key); + if (err) + ocelot_trap_del(ocelot, port, ipv6_ev_cookie); + + return err; +} + +static int ocelot_ipv6_ptp_trap_del(struct ocelot *ocelot, int port) +{ + unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot); + unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot); + int err; + + err = ocelot_trap_del(ocelot, port, ipv6_ev_cookie); + err |= ocelot_trap_del(ocelot, port, ipv6_gen_cookie); + return err; +} + +static int ocelot_setup_ptp_traps(struct ocelot *ocelot, int port, + bool l2, bool l4) +{ + int err; + + if (l2) + err = ocelot_l2_ptp_trap_add(ocelot, port); + else + err = ocelot_l2_ptp_trap_del(ocelot, port); + if (err) + return err; + + if (l4) { + err = ocelot_ipv4_ptp_trap_add(ocelot, port); + if (err) + goto err_ipv4; + + err = ocelot_ipv6_ptp_trap_add(ocelot, port); + if (err) + goto err_ipv6; + } else { + err = ocelot_ipv4_ptp_trap_del(ocelot, port); + + err |= ocelot_ipv6_ptp_trap_del(ocelot, port); + } + if (err) + return err; + + return 0; + +err_ipv6: + ocelot_ipv4_ptp_trap_del(ocelot, port); +err_ipv4: + if (l2) + ocelot_l2_ptp_trap_del(ocelot, port); + return err; +} + +int ocelot_hwstamp_get(struct ocelot *ocelot, int port, struct ifreq *ifr) +{ + return copy_to_user(ifr->ifr_data, &ocelot->hwtstamp_config, + sizeof(ocelot->hwtstamp_config)) ? -EFAULT : 0; +} +EXPORT_SYMBOL(ocelot_hwstamp_get); + +int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + bool l2 = false, l4 = false; + struct hwtstamp_config cfg; + int err; + + if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg))) + return -EFAULT; + + /* Tx type sanity check */ + switch (cfg.tx_type) { + case HWTSTAMP_TX_ON: + ocelot_port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; + break; + case HWTSTAMP_TX_ONESTEP_SYNC: + /* IFH_REW_OP_ONE_STEP_PTP updates the correctional field, we + * need to update the origin time. + */ + ocelot_port->ptp_cmd = IFH_REW_OP_ORIGIN_PTP; + break; + case HWTSTAMP_TX_OFF: + ocelot_port->ptp_cmd = 0; + break; + default: + return -ERANGE; + } + + mutex_lock(&ocelot->ptp_lock); + + switch (cfg.rx_filter) { + case HWTSTAMP_FILTER_NONE: + break; + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + l4 = true; + break; + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + l2 = true; + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + l2 = true; + l4 = true; + break; + default: + mutex_unlock(&ocelot->ptp_lock); + return -ERANGE; + } + + err = ocelot_setup_ptp_traps(ocelot, port, l2, l4); + if (err) { + mutex_unlock(&ocelot->ptp_lock); + return err; + } + + if (l2 && l4) + cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + else if (l2) + cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; + else if (l4) + cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; + else + cfg.rx_filter = HWTSTAMP_FILTER_NONE; + + /* Commit back the result & save it */ + memcpy(&ocelot->hwtstamp_config, &cfg, sizeof(cfg)); + mutex_unlock(&ocelot->ptp_lock); + + return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0; +} +EXPORT_SYMBOL(ocelot_hwstamp_set); + +int ocelot_get_ts_info(struct ocelot *ocelot, int port, + struct ethtool_ts_info *info) +{ + info->phc_index = ocelot->ptp_clock ? + ptp_clock_index(ocelot->ptp_clock) : -1; + if (info->phc_index == -1) { + info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE; + return 0; + } + info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON) | + BIT(HWTSTAMP_TX_ONESTEP_SYNC); + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT); + + return 0; +} +EXPORT_SYMBOL(ocelot_get_ts_info); + +static int ocelot_port_add_txtstamp_skb(struct ocelot *ocelot, int port, + struct sk_buff *clone) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + unsigned long flags; + + spin_lock_irqsave(&ocelot->ts_id_lock, flags); + + if (ocelot_port->ptp_skbs_in_flight == OCELOT_MAX_PTP_ID || + ocelot->ptp_skbs_in_flight == OCELOT_PTP_FIFO_SIZE) { + spin_unlock_irqrestore(&ocelot->ts_id_lock, flags); + return -EBUSY; + } + + skb_shinfo(clone)->tx_flags |= SKBTX_IN_PROGRESS; + /* Store timestamp ID in OCELOT_SKB_CB(clone)->ts_id */ + OCELOT_SKB_CB(clone)->ts_id = ocelot_port->ts_id; + + ocelot_port->ts_id++; + if (ocelot_port->ts_id == OCELOT_MAX_PTP_ID) + ocelot_port->ts_id = 0; + + ocelot_port->ptp_skbs_in_flight++; + ocelot->ptp_skbs_in_flight++; + + skb_queue_tail(&ocelot_port->tx_skbs, clone); + + spin_unlock_irqrestore(&ocelot->ts_id_lock, flags); + + return 0; +} + +static bool ocelot_ptp_is_onestep_sync(struct sk_buff *skb, + unsigned int ptp_class) +{ + struct ptp_header *hdr; + u8 msgtype, twostep; + + hdr = ptp_parse_header(skb, ptp_class); + if (!hdr) + return false; + + msgtype = ptp_get_msgtype(hdr, ptp_class); + twostep = hdr->flag_field[0] & 0x2; + + if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0) + return true; + + return false; +} + +int ocelot_port_txtstamp_request(struct ocelot *ocelot, int port, + struct sk_buff *skb, + struct sk_buff **clone) +{ + struct ocelot_port *ocelot_port = ocelot->ports[port]; + u8 ptp_cmd = ocelot_port->ptp_cmd; + unsigned int ptp_class; + int err; + + /* Don't do anything if PTP timestamping not enabled */ + if (!ptp_cmd) + return 0; + + ptp_class = ptp_classify_raw(skb); + if (ptp_class == PTP_CLASS_NONE) + return -EINVAL; + + /* Store ptp_cmd in OCELOT_SKB_CB(skb)->ptp_cmd */ + if (ptp_cmd == IFH_REW_OP_ORIGIN_PTP) { + if (ocelot_ptp_is_onestep_sync(skb, ptp_class)) { + OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd; + return 0; + } + + /* Fall back to two-step timestamping */ + ptp_cmd = IFH_REW_OP_TWO_STEP_PTP; + } + + if (ptp_cmd == IFH_REW_OP_TWO_STEP_PTP) { + *clone = skb_clone_sk(skb); + if (!(*clone)) + return -ENOMEM; + + err = ocelot_port_add_txtstamp_skb(ocelot, port, *clone); + if (err) + return err; + + OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd; + OCELOT_SKB_CB(*clone)->ptp_class = ptp_class; + } + + return 0; +} +EXPORT_SYMBOL(ocelot_port_txtstamp_request); + +static void ocelot_get_hwtimestamp(struct ocelot *ocelot, + struct timespec64 *ts) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&ocelot->ptp_clock_lock, flags); + + /* Read current PTP time to get seconds */ + val = ocelot_read_rix(ocelot, PTP_PIN_CFG, TOD_ACC_PIN); + + val &= ~(PTP_PIN_CFG_SYNC | PTP_PIN_CFG_ACTION_MASK | PTP_PIN_CFG_DOM); + val |= PTP_PIN_CFG_ACTION(PTP_PIN_ACTION_SAVE); + ocelot_write_rix(ocelot, val, PTP_PIN_CFG, TOD_ACC_PIN); + ts->tv_sec = ocelot_read_rix(ocelot, PTP_PIN_TOD_SEC_LSB, TOD_ACC_PIN); + + /* Read packet HW timestamp from FIFO */ + val = ocelot_read(ocelot, SYS_PTP_TXSTAMP); + ts->tv_nsec = SYS_PTP_TXSTAMP_PTP_TXSTAMP(val); + + /* Sec has incremented since the ts was registered */ + if ((ts->tv_sec & 0x1) != !!(val & SYS_PTP_TXSTAMP_PTP_TXSTAMP_SEC)) + ts->tv_sec--; + + spin_unlock_irqrestore(&ocelot->ptp_clock_lock, flags); +} + +static bool ocelot_validate_ptp_skb(struct sk_buff *clone, u16 seqid) +{ + struct ptp_header *hdr; + + hdr = ptp_parse_header(clone, OCELOT_SKB_CB(clone)->ptp_class); + if (WARN_ON(!hdr)) + return false; + + return seqid == ntohs(hdr->sequence_id); +} + +void ocelot_get_txtstamp(struct ocelot *ocelot) +{ + int budget = OCELOT_PTP_QUEUE_SZ; + + while (budget--) { + struct sk_buff *skb, *skb_tmp, *skb_match = NULL; + struct skb_shared_hwtstamps shhwtstamps; + u32 val, id, seqid, txport; + struct ocelot_port *port; + struct timespec64 ts; + unsigned long flags; + + val = ocelot_read(ocelot, SYS_PTP_STATUS); + + /* Check if a timestamp can be retrieved */ + if (!(val & SYS_PTP_STATUS_PTP_MESS_VLD)) + break; + + WARN_ON(val & SYS_PTP_STATUS_PTP_OVFL); + + /* Retrieve the ts ID and Tx port */ + id = SYS_PTP_STATUS_PTP_MESS_ID_X(val); + txport = SYS_PTP_STATUS_PTP_MESS_TXPORT_X(val); + seqid = SYS_PTP_STATUS_PTP_MESS_SEQ_ID(val); + + port = ocelot->ports[txport]; + + spin_lock(&ocelot->ts_id_lock); + port->ptp_skbs_in_flight--; + ocelot->ptp_skbs_in_flight--; + spin_unlock(&ocelot->ts_id_lock); + + /* Retrieve its associated skb */ +try_again: + spin_lock_irqsave(&port->tx_skbs.lock, flags); + + skb_queue_walk_safe(&port->tx_skbs, skb, skb_tmp) { + if (OCELOT_SKB_CB(skb)->ts_id != id) + continue; + __skb_unlink(skb, &port->tx_skbs); + skb_match = skb; + break; + } + + spin_unlock_irqrestore(&port->tx_skbs.lock, flags); + + if (WARN_ON(!skb_match)) + continue; + + if (!ocelot_validate_ptp_skb(skb_match, seqid)) { + dev_err_ratelimited(ocelot->dev, + "port %d received stale TX timestamp for seqid %d, discarding\n", + txport, seqid); + dev_kfree_skb_any(skb); + goto try_again; + } + + /* Get the h/w timestamp */ + ocelot_get_hwtimestamp(ocelot, &ts); + + /* Set the timestamp into the skb */ + memset(&shhwtstamps, 0, sizeof(shhwtstamps)); + shhwtstamps.hwtstamp = ktime_set(ts.tv_sec, ts.tv_nsec); + skb_complete_tx_timestamp(skb_match, &shhwtstamps); + + /* Next ts */ + ocelot_write(ocelot, SYS_PTP_NXT_PTP_NXT, SYS_PTP_NXT); + } +} +EXPORT_SYMBOL(ocelot_get_txtstamp); + int ocelot_init_timestamp(struct ocelot *ocelot, const struct ptp_clock_info *info) { diff --git a/drivers/net/ethernet/mscc/ocelot_stats.c b/drivers/net/ethernet/mscc/ocelot_stats.c new file mode 100644 index 0000000000000000000000000000000000000000..dbd20b125ceaf63e8d8d431c4895e4ee6cffa6d1 --- /dev/null +++ b/drivers/net/ethernet/mscc/ocelot_stats.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Statistics for Ocelot switch family + * + * Copyright (c) 2017 Microsemi Corporation + * Copyright 2022 NXP + */ +#include +#include +#include +#include "ocelot.h" + +/* Read the counters from hardware and keep them in region->buf. + * Caller must hold &ocelot->stat_view_lock. + */ +static int ocelot_port_update_stats(struct ocelot *ocelot, int port) +{ + struct ocelot_stats_region *region; + int err; + + /* Configure the port to read the stats from */ + ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port), SYS_STAT_CFG); + + list_for_each_entry(region, &ocelot->stats_regions, node) { + err = ocelot_bulk_read(ocelot, region->base, region->buf, + region->count); + if (err) + return err; + } + + return 0; +} + +/* Transfer the counters from region->buf to ocelot->stats. + * Caller must hold &ocelot->stat_view_lock and &ocelot->stats_lock. + */ +static void ocelot_port_transfer_stats(struct ocelot *ocelot, int port) +{ + unsigned int idx = port * OCELOT_NUM_STATS; + struct ocelot_stats_region *region; + int j; + + list_for_each_entry(region, &ocelot->stats_regions, node) { + for (j = 0; j < region->count; j++) { + u64 *stat = &ocelot->stats[idx + j]; + u64 val = region->buf[j]; + + if (val < (*stat & U32_MAX)) + *stat += (u64)1 << 32; + + *stat = (*stat & ~(u64)U32_MAX) + val; + } + + idx += region->count; + } +} + +static void ocelot_check_stats_work(struct work_struct *work) +{ + struct delayed_work *del_work = to_delayed_work(work); + struct ocelot *ocelot = container_of(del_work, struct ocelot, + stats_work); + int port, err; + + mutex_lock(&ocelot->stat_view_lock); + + for (port = 0; port < ocelot->num_phys_ports; port++) { + err = ocelot_port_update_stats(ocelot, port); + if (err) + break; + + spin_lock(&ocelot->stats_lock); + ocelot_port_transfer_stats(ocelot, port); + spin_unlock(&ocelot->stats_lock); + } + + if (!err && ocelot->ops->update_stats) + ocelot->ops->update_stats(ocelot); + + mutex_unlock(&ocelot->stat_view_lock); + + if (err) + dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err); + + queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work, + OCELOT_STATS_CHECK_DELAY); +} + +void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data) +{ + int i; + + if (sset != ETH_SS_STATS) + return; + + for (i = 0; i < OCELOT_NUM_STATS; i++) { + if (ocelot->stats_layout[i].name[0] == '\0') + continue; + + memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name, + ETH_GSTRING_LEN); + } +} +EXPORT_SYMBOL(ocelot_get_strings); + +/* Update ocelot->stats for the given port and run the given callback */ +static void ocelot_port_stats_run(struct ocelot *ocelot, int port, void *priv, + void (*cb)(struct ocelot *ocelot, int port, + void *priv)) +{ + int err; + + mutex_lock(&ocelot->stat_view_lock); + + err = ocelot_port_update_stats(ocelot, port); + if (err) { + dev_err(ocelot->dev, "Failed to update port %d stats: %pe\n", + port, ERR_PTR(err)); + goto out_unlock; + } + + spin_lock(&ocelot->stats_lock); + + ocelot_port_transfer_stats(ocelot, port); + cb(ocelot, port, priv); + + spin_unlock(&ocelot->stats_lock); + +out_unlock: + mutex_unlock(&ocelot->stat_view_lock); +} + +int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset) +{ + int i, num_stats = 0; + + if (sset != ETH_SS_STATS) + return -EOPNOTSUPP; + + for (i = 0; i < OCELOT_NUM_STATS; i++) + if (ocelot->stats_layout[i].name[0] != '\0') + num_stats++; + + return num_stats; +} +EXPORT_SYMBOL(ocelot_get_sset_count); + +static void ocelot_port_ethtool_stats_cb(struct ocelot *ocelot, int port, + void *priv) +{ + u64 *data = priv; + int i; + + /* Copy all supported counters */ + for (i = 0; i < OCELOT_NUM_STATS; i++) { + int index = port * OCELOT_NUM_STATS + i; + + if (ocelot->stats_layout[i].name[0] == '\0') + continue; + + *data++ = ocelot->stats[index]; + } +} + +void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data) +{ + ocelot_port_stats_run(ocelot, port, data, ocelot_port_ethtool_stats_cb); +} +EXPORT_SYMBOL(ocelot_get_ethtool_stats); + +static void ocelot_port_pause_stats_cb(struct ocelot *ocelot, int port, void *priv) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + struct ethtool_pause_stats *pause_stats = priv; + + pause_stats->tx_pause_frames = s[OCELOT_STAT_TX_PAUSE]; + pause_stats->rx_pause_frames = s[OCELOT_STAT_RX_PAUSE]; +} + +void ocelot_port_get_pause_stats(struct ocelot *ocelot, int port, + struct ethtool_pause_stats *pause_stats) +{ + ocelot_port_stats_run(ocelot, port, pause_stats, + ocelot_port_pause_stats_cb); +} +EXPORT_SYMBOL_GPL(ocelot_port_get_pause_stats); + +static const struct ethtool_rmon_hist_range ocelot_rmon_ranges[] = { + { 64, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1526 }, + { 1527, 65535 }, + {}, +}; + +static void ocelot_port_rmon_stats_cb(struct ocelot *ocelot, int port, void *priv) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + struct ethtool_rmon_stats *rmon_stats = priv; + + rmon_stats->undersize_pkts = s[OCELOT_STAT_RX_SHORTS]; + rmon_stats->oversize_pkts = s[OCELOT_STAT_RX_LONGS]; + rmon_stats->fragments = s[OCELOT_STAT_RX_FRAGMENTS]; + rmon_stats->jabbers = s[OCELOT_STAT_RX_JABBERS]; + + rmon_stats->hist[0] = s[OCELOT_STAT_RX_64]; + rmon_stats->hist[1] = s[OCELOT_STAT_RX_65_127]; + rmon_stats->hist[2] = s[OCELOT_STAT_RX_128_255]; + rmon_stats->hist[3] = s[OCELOT_STAT_RX_256_511]; + rmon_stats->hist[4] = s[OCELOT_STAT_RX_512_1023]; + rmon_stats->hist[5] = s[OCELOT_STAT_RX_1024_1526]; + rmon_stats->hist[6] = s[OCELOT_STAT_RX_1527_MAX]; + + rmon_stats->hist_tx[0] = s[OCELOT_STAT_TX_64]; + rmon_stats->hist_tx[1] = s[OCELOT_STAT_TX_65_127]; + rmon_stats->hist_tx[2] = s[OCELOT_STAT_TX_128_255]; + rmon_stats->hist_tx[3] = s[OCELOT_STAT_TX_128_255]; + rmon_stats->hist_tx[4] = s[OCELOT_STAT_TX_256_511]; + rmon_stats->hist_tx[5] = s[OCELOT_STAT_TX_512_1023]; + rmon_stats->hist_tx[6] = s[OCELOT_STAT_TX_1024_1526]; +} + +void ocelot_port_get_rmon_stats(struct ocelot *ocelot, int port, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + *ranges = ocelot_rmon_ranges; + + ocelot_port_stats_run(ocelot, port, rmon_stats, + ocelot_port_rmon_stats_cb); +} +EXPORT_SYMBOL_GPL(ocelot_port_get_rmon_stats); + +static void ocelot_port_ctrl_stats_cb(struct ocelot *ocelot, int port, void *priv) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + struct ethtool_eth_ctrl_stats *ctrl_stats = priv; + + ctrl_stats->MACControlFramesReceived = s[OCELOT_STAT_RX_CONTROL]; +} + +void ocelot_port_get_eth_ctrl_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + ocelot_port_stats_run(ocelot, port, ctrl_stats, + ocelot_port_ctrl_stats_cb); +} +EXPORT_SYMBOL_GPL(ocelot_port_get_eth_ctrl_stats); + +static void ocelot_port_mac_stats_cb(struct ocelot *ocelot, int port, void *priv) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + struct ethtool_eth_mac_stats *mac_stats = priv; + + mac_stats->OctetsTransmittedOK = s[OCELOT_STAT_TX_OCTETS]; + mac_stats->FramesTransmittedOK = s[OCELOT_STAT_TX_64] + + s[OCELOT_STAT_TX_65_127] + + s[OCELOT_STAT_TX_128_255] + + s[OCELOT_STAT_TX_256_511] + + s[OCELOT_STAT_TX_512_1023] + + s[OCELOT_STAT_TX_1024_1526] + + s[OCELOT_STAT_TX_1527_MAX]; + mac_stats->OctetsReceivedOK = s[OCELOT_STAT_RX_OCTETS]; + mac_stats->FramesReceivedOK = s[OCELOT_STAT_RX_GREEN_PRIO_0] + + s[OCELOT_STAT_RX_GREEN_PRIO_1] + + s[OCELOT_STAT_RX_GREEN_PRIO_2] + + s[OCELOT_STAT_RX_GREEN_PRIO_3] + + s[OCELOT_STAT_RX_GREEN_PRIO_4] + + s[OCELOT_STAT_RX_GREEN_PRIO_5] + + s[OCELOT_STAT_RX_GREEN_PRIO_6] + + s[OCELOT_STAT_RX_GREEN_PRIO_7] + + s[OCELOT_STAT_RX_YELLOW_PRIO_0] + + s[OCELOT_STAT_RX_YELLOW_PRIO_1] + + s[OCELOT_STAT_RX_YELLOW_PRIO_2] + + s[OCELOT_STAT_RX_YELLOW_PRIO_3] + + s[OCELOT_STAT_RX_YELLOW_PRIO_4] + + s[OCELOT_STAT_RX_YELLOW_PRIO_5] + + s[OCELOT_STAT_RX_YELLOW_PRIO_6] + + s[OCELOT_STAT_RX_YELLOW_PRIO_7]; + mac_stats->MulticastFramesXmittedOK = s[OCELOT_STAT_TX_MULTICAST]; + mac_stats->BroadcastFramesXmittedOK = s[OCELOT_STAT_TX_BROADCAST]; + mac_stats->MulticastFramesReceivedOK = s[OCELOT_STAT_RX_MULTICAST]; + mac_stats->BroadcastFramesReceivedOK = s[OCELOT_STAT_RX_BROADCAST]; + mac_stats->FrameTooLongErrors = s[OCELOT_STAT_RX_LONGS]; + /* Sadly, C_RX_CRC is the sum of FCS and alignment errors, they are not + * counted individually. + */ + mac_stats->FrameCheckSequenceErrors = s[OCELOT_STAT_RX_CRC_ALIGN_ERRS]; + mac_stats->AlignmentErrors = s[OCELOT_STAT_RX_CRC_ALIGN_ERRS]; +} + +void ocelot_port_get_eth_mac_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_mac_stats *mac_stats) +{ + ocelot_port_stats_run(ocelot, port, mac_stats, + ocelot_port_mac_stats_cb); +} +EXPORT_SYMBOL_GPL(ocelot_port_get_eth_mac_stats); + +static void ocelot_port_phy_stats_cb(struct ocelot *ocelot, int port, void *priv) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + struct ethtool_eth_phy_stats *phy_stats = priv; + + phy_stats->SymbolErrorDuringCarrier = s[OCELOT_STAT_RX_SYM_ERRS]; +} + +void ocelot_port_get_eth_phy_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_phy_stats *phy_stats) +{ + ocelot_port_stats_run(ocelot, port, phy_stats, + ocelot_port_phy_stats_cb); +} +EXPORT_SYMBOL_GPL(ocelot_port_get_eth_phy_stats); + +void ocelot_port_get_stats64(struct ocelot *ocelot, int port, + struct rtnl_link_stats64 *stats) +{ + u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS]; + + spin_lock(&ocelot->stats_lock); + + /* Get Rx stats */ + stats->rx_bytes = s[OCELOT_STAT_RX_OCTETS]; + stats->rx_packets = s[OCELOT_STAT_RX_SHORTS] + + s[OCELOT_STAT_RX_FRAGMENTS] + + s[OCELOT_STAT_RX_JABBERS] + + s[OCELOT_STAT_RX_LONGS] + + s[OCELOT_STAT_RX_64] + + s[OCELOT_STAT_RX_65_127] + + s[OCELOT_STAT_RX_128_255] + + s[OCELOT_STAT_RX_256_511] + + s[OCELOT_STAT_RX_512_1023] + + s[OCELOT_STAT_RX_1024_1526] + + s[OCELOT_STAT_RX_1527_MAX]; + stats->multicast = s[OCELOT_STAT_RX_MULTICAST]; + stats->rx_missed_errors = s[OCELOT_STAT_DROP_TAIL]; + stats->rx_dropped = s[OCELOT_STAT_RX_RED_PRIO_0] + + s[OCELOT_STAT_RX_RED_PRIO_1] + + s[OCELOT_STAT_RX_RED_PRIO_2] + + s[OCELOT_STAT_RX_RED_PRIO_3] + + s[OCELOT_STAT_RX_RED_PRIO_4] + + s[OCELOT_STAT_RX_RED_PRIO_5] + + s[OCELOT_STAT_RX_RED_PRIO_6] + + s[OCELOT_STAT_RX_RED_PRIO_7] + + s[OCELOT_STAT_DROP_LOCAL] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_0] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_1] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_2] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_3] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_4] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_5] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_6] + + s[OCELOT_STAT_DROP_YELLOW_PRIO_7] + + s[OCELOT_STAT_DROP_GREEN_PRIO_0] + + s[OCELOT_STAT_DROP_GREEN_PRIO_1] + + s[OCELOT_STAT_DROP_GREEN_PRIO_2] + + s[OCELOT_STAT_DROP_GREEN_PRIO_3] + + s[OCELOT_STAT_DROP_GREEN_PRIO_4] + + s[OCELOT_STAT_DROP_GREEN_PRIO_5] + + s[OCELOT_STAT_DROP_GREEN_PRIO_6] + + s[OCELOT_STAT_DROP_GREEN_PRIO_7]; + + /* Get Tx stats */ + stats->tx_bytes = s[OCELOT_STAT_TX_OCTETS]; + stats->tx_packets = s[OCELOT_STAT_TX_64] + + s[OCELOT_STAT_TX_65_127] + + s[OCELOT_STAT_TX_128_255] + + s[OCELOT_STAT_TX_256_511] + + s[OCELOT_STAT_TX_512_1023] + + s[OCELOT_STAT_TX_1024_1526] + + s[OCELOT_STAT_TX_1527_MAX]; + stats->tx_dropped = s[OCELOT_STAT_TX_DROPS] + + s[OCELOT_STAT_TX_AGED]; + stats->collisions = s[OCELOT_STAT_TX_COLLISION]; + + spin_unlock(&ocelot->stats_lock); +} +EXPORT_SYMBOL(ocelot_port_get_stats64); + +static int ocelot_prepare_stats_regions(struct ocelot *ocelot) +{ + struct ocelot_stats_region *region = NULL; + unsigned int last; + int i; + + INIT_LIST_HEAD(&ocelot->stats_regions); + + for (i = 0; i < OCELOT_NUM_STATS; i++) { + if (!ocelot->stats_layout[i].reg) + continue; + + if (region && ocelot->stats_layout[i].reg == last + 4) { + region->count++; + } else { + region = devm_kzalloc(ocelot->dev, sizeof(*region), + GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->base = ocelot->stats_layout[i].reg; + region->count = 1; + list_add_tail(®ion->node, &ocelot->stats_regions); + } + + last = ocelot->stats_layout[i].reg; + } + + list_for_each_entry(region, &ocelot->stats_regions, node) { + region->buf = devm_kcalloc(ocelot->dev, region->count, + sizeof(*region->buf), GFP_KERNEL); + if (!region->buf) + return -ENOMEM; + } + + return 0; +} + +int ocelot_stats_init(struct ocelot *ocelot) +{ + char queue_name[32]; + int ret; + + ocelot->stats = devm_kcalloc(ocelot->dev, + ocelot->num_phys_ports * OCELOT_NUM_STATS, + sizeof(u64), GFP_KERNEL); + if (!ocelot->stats) + return -ENOMEM; + + snprintf(queue_name, sizeof(queue_name), "%s-stats", + dev_name(ocelot->dev)); + ocelot->stats_queue = create_singlethread_workqueue(queue_name); + if (!ocelot->stats_queue) + return -ENOMEM; + + spin_lock_init(&ocelot->stats_lock); + mutex_init(&ocelot->stat_view_lock); + + ret = ocelot_prepare_stats_regions(ocelot); + if (ret) { + destroy_workqueue(ocelot->stats_queue); + return ret; + } + + INIT_DELAYED_WORK(&ocelot->stats_work, ocelot_check_stats_work); + queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work, + OCELOT_STATS_CHECK_DELAY); + + return 0; +} + +void ocelot_stats_deinit(struct ocelot *ocelot) +{ + cancel_delayed_work(&ocelot->stats_work); + destroy_workqueue(ocelot->stats_queue); +} diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index 9c488953f541daa12e5c9d26676efae241ed1002..6f22aea08a644648db42179d602f76e6607a6606 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -25,6 +26,9 @@ #define VSC7514_VCAP_POLICER_BASE 128 #define VSC7514_VCAP_POLICER_MAX 191 +#define MEM_INIT_SLEEP_US 1000 +#define MEM_INIT_TIMEOUT_US 100000 + static const u32 *ocelot_regmap[TARGET_MAX] = { [ANA] = vsc7514_ana_regmap, [QS] = vsc7514_qs_regmap, @@ -97,378 +101,7 @@ static const struct reg_field ocelot_regfields[REGFIELD_MAX] = { }; static const struct ocelot_stat_layout ocelot_stats_layout[OCELOT_NUM_STATS] = { - [OCELOT_STAT_RX_OCTETS] = { - .name = "rx_octets", - .reg = SYS_COUNT_RX_OCTETS, - }, - [OCELOT_STAT_RX_UNICAST] = { - .name = "rx_unicast", - .reg = SYS_COUNT_RX_UNICAST, - }, - [OCELOT_STAT_RX_MULTICAST] = { - .name = "rx_multicast", - .reg = SYS_COUNT_RX_MULTICAST, - }, - [OCELOT_STAT_RX_BROADCAST] = { - .name = "rx_broadcast", - .reg = SYS_COUNT_RX_BROADCAST, - }, - [OCELOT_STAT_RX_SHORTS] = { - .name = "rx_shorts", - .reg = SYS_COUNT_RX_SHORTS, - }, - [OCELOT_STAT_RX_FRAGMENTS] = { - .name = "rx_fragments", - .reg = SYS_COUNT_RX_FRAGMENTS, - }, - [OCELOT_STAT_RX_JABBERS] = { - .name = "rx_jabbers", - .reg = SYS_COUNT_RX_JABBERS, - }, - [OCELOT_STAT_RX_CRC_ALIGN_ERRS] = { - .name = "rx_crc_align_errs", - .reg = SYS_COUNT_RX_CRC_ALIGN_ERRS, - }, - [OCELOT_STAT_RX_SYM_ERRS] = { - .name = "rx_sym_errs", - .reg = SYS_COUNT_RX_SYM_ERRS, - }, - [OCELOT_STAT_RX_64] = { - .name = "rx_frames_below_65_octets", - .reg = SYS_COUNT_RX_64, - }, - [OCELOT_STAT_RX_65_127] = { - .name = "rx_frames_65_to_127_octets", - .reg = SYS_COUNT_RX_65_127, - }, - [OCELOT_STAT_RX_128_255] = { - .name = "rx_frames_128_to_255_octets", - .reg = SYS_COUNT_RX_128_255, - }, - [OCELOT_STAT_RX_256_511] = { - .name = "rx_frames_256_to_511_octets", - .reg = SYS_COUNT_RX_256_511, - }, - [OCELOT_STAT_RX_512_1023] = { - .name = "rx_frames_512_to_1023_octets", - .reg = SYS_COUNT_RX_512_1023, - }, - [OCELOT_STAT_RX_1024_1526] = { - .name = "rx_frames_1024_to_1526_octets", - .reg = SYS_COUNT_RX_1024_1526, - }, - [OCELOT_STAT_RX_1527_MAX] = { - .name = "rx_frames_over_1526_octets", - .reg = SYS_COUNT_RX_1527_MAX, - }, - [OCELOT_STAT_RX_PAUSE] = { - .name = "rx_pause", - .reg = SYS_COUNT_RX_PAUSE, - }, - [OCELOT_STAT_RX_CONTROL] = { - .name = "rx_control", - .reg = SYS_COUNT_RX_CONTROL, - }, - [OCELOT_STAT_RX_LONGS] = { - .name = "rx_longs", - .reg = SYS_COUNT_RX_LONGS, - }, - [OCELOT_STAT_RX_CLASSIFIED_DROPS] = { - .name = "rx_classified_drops", - .reg = SYS_COUNT_RX_CLASSIFIED_DROPS, - }, - [OCELOT_STAT_RX_RED_PRIO_0] = { - .name = "rx_red_prio_0", - .reg = SYS_COUNT_RX_RED_PRIO_0, - }, - [OCELOT_STAT_RX_RED_PRIO_1] = { - .name = "rx_red_prio_1", - .reg = SYS_COUNT_RX_RED_PRIO_1, - }, - [OCELOT_STAT_RX_RED_PRIO_2] = { - .name = "rx_red_prio_2", - .reg = SYS_COUNT_RX_RED_PRIO_2, - }, - [OCELOT_STAT_RX_RED_PRIO_3] = { - .name = "rx_red_prio_3", - .reg = SYS_COUNT_RX_RED_PRIO_3, - }, - [OCELOT_STAT_RX_RED_PRIO_4] = { - .name = "rx_red_prio_4", - .reg = SYS_COUNT_RX_RED_PRIO_4, - }, - [OCELOT_STAT_RX_RED_PRIO_5] = { - .name = "rx_red_prio_5", - .reg = SYS_COUNT_RX_RED_PRIO_5, - }, - [OCELOT_STAT_RX_RED_PRIO_6] = { - .name = "rx_red_prio_6", - .reg = SYS_COUNT_RX_RED_PRIO_6, - }, - [OCELOT_STAT_RX_RED_PRIO_7] = { - .name = "rx_red_prio_7", - .reg = SYS_COUNT_RX_RED_PRIO_7, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_0] = { - .name = "rx_yellow_prio_0", - .reg = SYS_COUNT_RX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_1] = { - .name = "rx_yellow_prio_1", - .reg = SYS_COUNT_RX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_2] = { - .name = "rx_yellow_prio_2", - .reg = SYS_COUNT_RX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_3] = { - .name = "rx_yellow_prio_3", - .reg = SYS_COUNT_RX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_4] = { - .name = "rx_yellow_prio_4", - .reg = SYS_COUNT_RX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_5] = { - .name = "rx_yellow_prio_5", - .reg = SYS_COUNT_RX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_6] = { - .name = "rx_yellow_prio_6", - .reg = SYS_COUNT_RX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_RX_YELLOW_PRIO_7] = { - .name = "rx_yellow_prio_7", - .reg = SYS_COUNT_RX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_RX_GREEN_PRIO_0] = { - .name = "rx_green_prio_0", - .reg = SYS_COUNT_RX_GREEN_PRIO_0, - }, - [OCELOT_STAT_RX_GREEN_PRIO_1] = { - .name = "rx_green_prio_1", - .reg = SYS_COUNT_RX_GREEN_PRIO_1, - }, - [OCELOT_STAT_RX_GREEN_PRIO_2] = { - .name = "rx_green_prio_2", - .reg = SYS_COUNT_RX_GREEN_PRIO_2, - }, - [OCELOT_STAT_RX_GREEN_PRIO_3] = { - .name = "rx_green_prio_3", - .reg = SYS_COUNT_RX_GREEN_PRIO_3, - }, - [OCELOT_STAT_RX_GREEN_PRIO_4] = { - .name = "rx_green_prio_4", - .reg = SYS_COUNT_RX_GREEN_PRIO_4, - }, - [OCELOT_STAT_RX_GREEN_PRIO_5] = { - .name = "rx_green_prio_5", - .reg = SYS_COUNT_RX_GREEN_PRIO_5, - }, - [OCELOT_STAT_RX_GREEN_PRIO_6] = { - .name = "rx_green_prio_6", - .reg = SYS_COUNT_RX_GREEN_PRIO_6, - }, - [OCELOT_STAT_RX_GREEN_PRIO_7] = { - .name = "rx_green_prio_7", - .reg = SYS_COUNT_RX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_OCTETS] = { - .name = "tx_octets", - .reg = SYS_COUNT_TX_OCTETS, - }, - [OCELOT_STAT_TX_UNICAST] = { - .name = "tx_unicast", - .reg = SYS_COUNT_TX_UNICAST, - }, - [OCELOT_STAT_TX_MULTICAST] = { - .name = "tx_multicast", - .reg = SYS_COUNT_TX_MULTICAST, - }, - [OCELOT_STAT_TX_BROADCAST] = { - .name = "tx_broadcast", - .reg = SYS_COUNT_TX_BROADCAST, - }, - [OCELOT_STAT_TX_COLLISION] = { - .name = "tx_collision", - .reg = SYS_COUNT_TX_COLLISION, - }, - [OCELOT_STAT_TX_DROPS] = { - .name = "tx_drops", - .reg = SYS_COUNT_TX_DROPS, - }, - [OCELOT_STAT_TX_PAUSE] = { - .name = "tx_pause", - .reg = SYS_COUNT_TX_PAUSE, - }, - [OCELOT_STAT_TX_64] = { - .name = "tx_frames_below_65_octets", - .reg = SYS_COUNT_TX_64, - }, - [OCELOT_STAT_TX_65_127] = { - .name = "tx_frames_65_to_127_octets", - .reg = SYS_COUNT_TX_65_127, - }, - [OCELOT_STAT_TX_128_255] = { - .name = "tx_frames_128_255_octets", - .reg = SYS_COUNT_TX_128_255, - }, - [OCELOT_STAT_TX_256_511] = { - .name = "tx_frames_256_511_octets", - .reg = SYS_COUNT_TX_256_511, - }, - [OCELOT_STAT_TX_512_1023] = { - .name = "tx_frames_512_1023_octets", - .reg = SYS_COUNT_TX_512_1023, - }, - [OCELOT_STAT_TX_1024_1526] = { - .name = "tx_frames_1024_1526_octets", - .reg = SYS_COUNT_TX_1024_1526, - }, - [OCELOT_STAT_TX_1527_MAX] = { - .name = "tx_frames_over_1526_octets", - .reg = SYS_COUNT_TX_1527_MAX, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_0] = { - .name = "tx_yellow_prio_0", - .reg = SYS_COUNT_TX_YELLOW_PRIO_0, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_1] = { - .name = "tx_yellow_prio_1", - .reg = SYS_COUNT_TX_YELLOW_PRIO_1, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_2] = { - .name = "tx_yellow_prio_2", - .reg = SYS_COUNT_TX_YELLOW_PRIO_2, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_3] = { - .name = "tx_yellow_prio_3", - .reg = SYS_COUNT_TX_YELLOW_PRIO_3, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_4] = { - .name = "tx_yellow_prio_4", - .reg = SYS_COUNT_TX_YELLOW_PRIO_4, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_5] = { - .name = "tx_yellow_prio_5", - .reg = SYS_COUNT_TX_YELLOW_PRIO_5, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_6] = { - .name = "tx_yellow_prio_6", - .reg = SYS_COUNT_TX_YELLOW_PRIO_6, - }, - [OCELOT_STAT_TX_YELLOW_PRIO_7] = { - .name = "tx_yellow_prio_7", - .reg = SYS_COUNT_TX_YELLOW_PRIO_7, - }, - [OCELOT_STAT_TX_GREEN_PRIO_0] = { - .name = "tx_green_prio_0", - .reg = SYS_COUNT_TX_GREEN_PRIO_0, - }, - [OCELOT_STAT_TX_GREEN_PRIO_1] = { - .name = "tx_green_prio_1", - .reg = SYS_COUNT_TX_GREEN_PRIO_1, - }, - [OCELOT_STAT_TX_GREEN_PRIO_2] = { - .name = "tx_green_prio_2", - .reg = SYS_COUNT_TX_GREEN_PRIO_2, - }, - [OCELOT_STAT_TX_GREEN_PRIO_3] = { - .name = "tx_green_prio_3", - .reg = SYS_COUNT_TX_GREEN_PRIO_3, - }, - [OCELOT_STAT_TX_GREEN_PRIO_4] = { - .name = "tx_green_prio_4", - .reg = SYS_COUNT_TX_GREEN_PRIO_4, - }, - [OCELOT_STAT_TX_GREEN_PRIO_5] = { - .name = "tx_green_prio_5", - .reg = SYS_COUNT_TX_GREEN_PRIO_5, - }, - [OCELOT_STAT_TX_GREEN_PRIO_6] = { - .name = "tx_green_prio_6", - .reg = SYS_COUNT_TX_GREEN_PRIO_6, - }, - [OCELOT_STAT_TX_GREEN_PRIO_7] = { - .name = "tx_green_prio_7", - .reg = SYS_COUNT_TX_GREEN_PRIO_7, - }, - [OCELOT_STAT_TX_AGED] = { - .name = "tx_aged", - .reg = SYS_COUNT_TX_AGING, - }, - [OCELOT_STAT_DROP_LOCAL] = { - .name = "drop_local", - .reg = SYS_COUNT_DROP_LOCAL, - }, - [OCELOT_STAT_DROP_TAIL] = { - .name = "drop_tail", - .reg = SYS_COUNT_DROP_TAIL, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_0] = { - .name = "drop_yellow_prio_0", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_0, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_1] = { - .name = "drop_yellow_prio_1", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_1, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_2] = { - .name = "drop_yellow_prio_2", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_2, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_3] = { - .name = "drop_yellow_prio_3", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_3, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_4] = { - .name = "drop_yellow_prio_4", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_4, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_5] = { - .name = "drop_yellow_prio_5", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_5, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_6] = { - .name = "drop_yellow_prio_6", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_6, - }, - [OCELOT_STAT_DROP_YELLOW_PRIO_7] = { - .name = "drop_yellow_prio_7", - .reg = SYS_COUNT_DROP_YELLOW_PRIO_7, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_0] = { - .name = "drop_green_prio_0", - .reg = SYS_COUNT_DROP_GREEN_PRIO_0, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_1] = { - .name = "drop_green_prio_1", - .reg = SYS_COUNT_DROP_GREEN_PRIO_1, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_2] = { - .name = "drop_green_prio_2", - .reg = SYS_COUNT_DROP_GREEN_PRIO_2, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_3] = { - .name = "drop_green_prio_3", - .reg = SYS_COUNT_DROP_GREEN_PRIO_3, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_4] = { - .name = "drop_green_prio_4", - .reg = SYS_COUNT_DROP_GREEN_PRIO_4, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_5] = { - .name = "drop_green_prio_5", - .reg = SYS_COUNT_DROP_GREEN_PRIO_5, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_6] = { - .name = "drop_green_prio_6", - .reg = SYS_COUNT_DROP_GREEN_PRIO_6, - }, - [OCELOT_STAT_DROP_GREEN_PRIO_7] = { - .name = "drop_green_prio_7", - .reg = SYS_COUNT_DROP_GREEN_PRIO_7, - }, + OCELOT_COMMON_STATS, }; static void ocelot_pll5_init(struct ocelot *ocelot) @@ -562,27 +195,43 @@ static const struct of_device_id mscc_ocelot_match[] = { }; MODULE_DEVICE_TABLE(of, mscc_ocelot_match); +static int ocelot_mem_init_status(struct ocelot *ocelot) +{ + unsigned int val; + int err; + + err = regmap_field_read(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], + &val); + + return err ?: val; +} + static int ocelot_reset(struct ocelot *ocelot) { - int retries = 100; + int err; u32 val; - regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], 1); - regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1); + err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], 1); + if (err) + return err; - do { - msleep(1); - regmap_field_read(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], - &val); - } while (val && --retries); + err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1); + if (err) + return err; - if (!retries) - return -ETIMEDOUT; + /* MEM_INIT is a self-clearing bit. Wait for it to be cleared (should be + * 100us) before enabling the switch core. + */ + err = readx_poll_timeout(ocelot_mem_init_status, ocelot, val, !val, + MEM_INIT_SLEEP_US, MEM_INIT_TIMEOUT_US); + if (err) + return err; - regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1); - regmap_field_write(ocelot->regfields[SYS_RESET_CFG_CORE_ENA], 1); + err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1); + if (err) + return err; - return 0; + return regmap_field_write(ocelot->regfields[SYS_RESET_CFG_CORE_ENA], 1); } /* Watermark encode diff --git a/drivers/net/ethernet/mscc/vsc7514_regs.c b/drivers/net/ethernet/mscc/vsc7514_regs.c index 9cf82ecf191cd0727300d9d5df94b584b08a7e44..9d2d3e13cacfa21d9fc3f97120a50bbedd77e7f8 100644 --- a/drivers/net/ethernet/mscc/vsc7514_regs.c +++ b/drivers/net/ethernet/mscc/vsc7514_regs.c @@ -242,7 +242,7 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_COUNT_TX_GREEN_PRIO_5, 0x00016c), REG(SYS_COUNT_TX_GREEN_PRIO_6, 0x000170), REG(SYS_COUNT_TX_GREEN_PRIO_7, 0x000174), - REG(SYS_COUNT_TX_AGING, 0x000178), + REG(SYS_COUNT_TX_AGED, 0x000178), REG(SYS_COUNT_DROP_LOCAL, 0x000200), REG(SYS_COUNT_DROP_TAIL, 0x000204), REG(SYS_COUNT_DROP_YELLOW_PRIO_0, 0x000208), @@ -283,7 +283,6 @@ const u32 vsc7514_sys_regmap[] = { REG(SYS_MMGT_FAST, 0x0006a0), REG(SYS_EVENTS_DIF, 0x0006a4), REG(SYS_EVENTS_CORE, 0x0006b4), - REG(SYS_CNT, 0x000000), REG(SYS_PTP_STATUS, 0x0006b8), REG(SYS_PTP_TXSTAMP, 0x0006bc), REG(SYS_PTP_NXT, 0x0006c0), diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 971dde8c328643fc2f456b30671a31f2dc94e15e..9063e2e22cd5c4e4d56bb3aa7e049865345327ad 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -1647,10 +1647,10 @@ myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { struct myri10ge_priv *mgp = netdev_priv(netdev); - strlcpy(info->driver, "myri10ge", sizeof(info->driver)); - strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); - strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, "myri10ge", sizeof(info->driver)); + strscpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); + strscpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); + strscpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); } static int myri10ge_get_coalesce(struct net_device *netdev, diff --git a/drivers/net/ethernet/natsemi/natsemi.c b/drivers/net/ethernet/natsemi/natsemi.c index 9aae7f1eb5d266308832f414888e6c01787b246d..650a5a16607012b832c678a0e7417e93770c0c97 100644 --- a/drivers/net/ethernet/natsemi/natsemi.c +++ b/drivers/net/ethernet/natsemi/natsemi.c @@ -869,7 +869,7 @@ static int natsemi_probe1(struct pci_dev *pdev, const struct pci_device_id *ent) np = netdev_priv(dev); np->ioaddr = ioaddr; - netif_napi_add(dev, &np->napi, natsemi_poll, 64); + netif_napi_add(dev, &np->napi, natsemi_poll); np->dev = dev; np->pci_dev = pdev; @@ -2564,9 +2564,9 @@ static void set_rx_mode(struct net_device *dev) static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct netdev_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int get_regs_len(struct net_device *dev) diff --git a/drivers/net/ethernet/natsemi/ns83820.c b/drivers/net/ethernet/natsemi/ns83820.c index 49ea130c906712ccb4c842ed8bb8945247ffaab7..998586872599b39e67592b3ddbaa2e54829f916c 100644 --- a/drivers/net/ethernet/natsemi/ns83820.c +++ b/drivers/net/ethernet/natsemi/ns83820.c @@ -1351,9 +1351,9 @@ static int ns83820_set_link_ksettings(struct net_device *ndev, static void ns83820_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *info) { struct ns83820 *dev = PRIV(ndev); - strlcpy(info->driver, "ns83820", sizeof(info->driver)); - strlcpy(info->version, VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(dev->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, "ns83820", sizeof(info->driver)); + strscpy(info->version, VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(dev->pci_dev), sizeof(info->bus_info)); } static u32 ns83820_get_link(struct net_device *ndev) diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index 30f955efa83086d0389bb0421379ab3b1656aaaa..dcf8212119f9cd8415a175c115018f0c6097b525 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -5348,9 +5348,9 @@ static void s2io_ethtool_gdrvinfo(struct net_device *dev, { struct s2io_nic *sp = netdev_priv(dev); - strlcpy(info->driver, s2io_driver_name, sizeof(info->driver)); - strlcpy(info->version, s2io_driver_version, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(sp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, s2io_driver_name, sizeof(info->driver)); + strscpy(info->version, s2io_driver_version, sizeof(info->version)); + strscpy(info->bus_info, pci_name(sp->pdev), sizeof(info->bus_info)); } /** @@ -7359,10 +7359,9 @@ static int rx_osm_handler(struct ring_info *ring_data, struct RxD_t * rxdp) int get_off = ring_data->rx_curr_get_info.offset; int buf0_len = RXD_GET_BUFFER0_SIZE_3(rxdp->Control_2); int buf2_len = RXD_GET_BUFFER2_SIZE_3(rxdp->Control_2); - unsigned char *buff = skb_push(skb, buf0_len); struct buffAdd *ba = &ring_data->ba[get_block][get_off]; - memcpy(buff, ba->ba_0, buf0_len); + skb_put_data(skb, ba->ba_0, buf0_len); skb_put(skb, buf2_len); } @@ -7905,10 +7904,10 @@ s2io_init_nic(struct pci_dev *pdev, const struct pci_device_id *pre) for (i = 0; i < config->rx_ring_num ; i++) { struct ring_info *ring = &mac_control->rings[i]; - netif_napi_add(dev, &ring->napi, s2io_poll_msix, 64); + netif_napi_add(dev, &ring->napi, s2io_poll_msix); } } else { - netif_napi_add(dev, &sp->napi, s2io_poll_inta, 64); + netif_napi_add(dev, &sp->napi, s2io_poll_inta); } /* Not needed for Herc */ diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c index 78368e71ce836e90972f0fe503a9ff51e10ada67..f80f1a6953fa29cbffe622f59b502a499f5de9c5 100644 --- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c +++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c @@ -474,6 +474,7 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev, { struct nfp_net *nn = netdev_priv(netdev); struct nfp_net_tls_offload_ctx *ntls; + struct net *net = dev_net(netdev); struct ipv6hdr *ipv6h; struct tcphdr *th; struct iphdr *iph; @@ -494,13 +495,13 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev, switch (ipv6h->version) { case 4: - sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, + sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, iph->saddr, th->source, iph->daddr, th->dest, netdev->ifindex); break; #if IS_ENABLED(CONFIG_IPV6) case 6: - sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &ipv6h->saddr, th->source, &ipv6h->daddr, ntohs(th->dest), netdev->ifindex, 0); diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c index b3b2a23b8d89513b1ceaddca50e96eaa97cf9e9c..f693119541d55a034ff6dd2a4dfe6d559b3d75c4 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c +++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) /* Copyright (C) 2021 Corigine, Inc. */ +#include +#include + #include "conntrack.h" #include "../nfp_port.h" @@ -56,9 +59,17 @@ bool is_pre_ct_flow(struct flow_cls_offload *flow) int i; flow_action_for_each(i, act, &flow->rule->action) { - if (act->id == FLOW_ACTION_CT && !act->ct.action) - return true; + if (act->id == FLOW_ACTION_CT) { + /* The pre_ct rule only have the ct or ct nat action, cannot + * contains other ct action e.g ct commit and so on. + */ + if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT)) + return true; + else + return false; + } } + return false; } @@ -66,13 +77,37 @@ bool is_post_ct_flow(struct flow_cls_offload *flow) { struct flow_rule *rule = flow_cls_offload_flow_rule(flow); struct flow_dissector *dissector = rule->match.dissector; + struct flow_action_entry *act; + bool exist_ct_clear = false; struct flow_match_ct ct; + int i; + + /* post ct entry cannot contains any ct action except ct_clear. */ + flow_action_for_each(i, act, &flow->rule->action) { + if (act->id == FLOW_ACTION_CT) { + /* ignore ct clear action. */ + if (act->ct.action == TCA_CT_ACT_CLEAR) { + exist_ct_clear = true; + continue; + } + + return false; + } + } if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { flow_rule_match_ct(rule, &ct); if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) return true; + } else { + /* when do nat with ct, the post ct entry ignore the ct status, + * will match the nat field(sip/dip) instead. In this situation, + * the flow chain index is not zero and contains ct clear action. + */ + if (flow->common.chain_index && exist_ct_clear) + return true; } + return false; } @@ -168,6 +203,20 @@ static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf, return buf; } +/* Note entry1 and entry2 are not swappable. only skip ip and + * tport merge check for pre_ct and post_ct when pre_ct do nat. + */ +static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1, + struct nfp_fl_ct_flow_entry *entry2) +{ + /* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */ + if ((entry1->flags & NFP_FL_ACTION_DO_NAT) && + entry2->type == CT_TYPE_POST_CT) + return false; + + return true; +} + /* Note entry1 and entry2 are not swappable, entry1 should be * the former flow whose mangle action need be taken into account * if existed, and entry2 should be the latter flow whose action @@ -225,7 +274,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { + /* if pre ct entry do nat, the nat ip exists in nft entry, + * will be do merge check when do nft and post ct merge, + * so skip this ip merge check here. + */ + if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) && + nfp_ct_merge_check_cannot_skip(entry1, entry2)) { struct flow_match_ipv4_addrs match1, match2; flow_rule_match_ipv4_addrs(entry1->rule, &match1); @@ -242,7 +296,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { + /* if pre ct entry do nat, the nat ip exists in nft entry, + * will be do merge check when do nft and post ct merge, + * so skip this ip merge check here. + */ + if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) && + nfp_ct_merge_check_cannot_skip(entry1, entry2)) { struct flow_match_ipv6_addrs match1, match2; flow_rule_match_ipv6_addrs(entry1->rule, &match1); @@ -259,7 +318,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, goto check_failed; } - if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) { + /* if pre ct entry do nat, the nat tport exists in nft entry, + * will be do merge check when do nft and post ct merge, + * so skip this tport merge check here. + */ + if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) && + nfp_ct_merge_check_cannot_skip(entry1, entry2)) { enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC; struct flow_match_ports match1, match2; @@ -404,12 +468,55 @@ check_failed: return -EINVAL; } +static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in, + struct flow_rule *rule) +{ + struct flow_match_vlan match; + + if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN))) + return -EOPNOTSUPP; + + /* post_ct does not match VLAN KEY, can be merged. */ + if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN))) + return 0; + + switch (a_in->id) { + /* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */ + case FLOW_ACTION_VLAN_POP: + return -EOPNOTSUPP; + + case FLOW_ACTION_VLAN_PUSH: + case FLOW_ACTION_VLAN_MANGLE: + flow_rule_match_vlan(rule, &match); + /* different vlan id, cannot be merged. */ + if ((match.key->vlan_id & match.mask->vlan_id) ^ + (a_in->vlan.vid & match.mask->vlan_id)) + return -EOPNOTSUPP; + + /* different tpid, cannot be merged. */ + if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^ + (a_in->vlan.proto & match.mask->vlan_tpid)) + return -EOPNOTSUPP; + + /* different priority, cannot be merged. */ + if ((match.key->vlan_priority & match.mask->vlan_priority) ^ + (a_in->vlan.prio & match.mask->vlan_priority)) + return -EOPNOTSUPP; + + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry, struct nfp_fl_ct_flow_entry *post_ct_entry, struct nfp_fl_ct_flow_entry *nft_entry) { struct flow_action_entry *act; - int i; + int i, err; /* Check for pre_ct->action conflicts */ flow_action_for_each(i, act, &pre_ct_entry->rule->action) { @@ -417,6 +524,10 @@ static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry, case FLOW_ACTION_VLAN_PUSH: case FLOW_ACTION_VLAN_POP: case FLOW_ACTION_VLAN_MANGLE: + err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule); + if (err) + return err; + break; case FLOW_ACTION_MPLS_PUSH: case FLOW_ACTION_MPLS_POP: case FLOW_ACTION_MPLS_MANGLE: @@ -468,6 +579,12 @@ static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry, return -EINVAL; return 0; + } else { + /* post_ct with ct clear action will not match the + * ct status when nft is nat entry. + */ + if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE) + return 0; } return -EINVAL; @@ -537,11 +654,37 @@ nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map) return key_size; } +/* get the csum flag according the ip proto and mangle action. */ +static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum) +{ + if (a_in->id != FLOW_ACTION_MANGLE) + return; + + switch (a_in->mangle.htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + *csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR; + if (ip_proto == IPPROTO_TCP) + *csum |= TCA_CSUM_UPDATE_FLAG_TCP; + else if (ip_proto == IPPROTO_UDP) + *csum |= TCA_CSUM_UPDATE_FLAG_UDP; + break; + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + *csum |= TCA_CSUM_UPDATE_FLAG_TCP; + break; + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + *csum |= TCA_CSUM_UPDATE_FLAG_UDP; + break; + default: + break; + } +} + static int nfp_fl_merge_actions_offload(struct flow_rule **rules, struct nfp_flower_priv *priv, struct net_device *netdev, struct nfp_fl_payload *flow_pay) { + enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE; struct flow_action_entry *a_in; int i, j, num_actions, id; struct flow_rule *a_rule; @@ -551,15 +694,25 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules, rules[CT_TYPE_NFT]->action.num_entries + rules[CT_TYPE_POST_CT]->action.num_entries; - a_rule = flow_rule_alloc(num_actions); + /* Add one action to make sure there is enough room to add an checksum action + * when do nat. + */ + a_rule = flow_rule_alloc(num_actions + 1); if (!a_rule) return -ENOMEM; /* Actions need a BASIC dissector. */ a_rule->match = rules[CT_TYPE_PRE_CT]->match; + /* post_ct entry have one action at least. */ + if (rules[CT_TYPE_POST_CT]->action.num_entries != 0) { + tmp_stats = rules[CT_TYPE_POST_CT]->action.entries[0].hw_stats; + } /* Copy actions */ for (j = 0; j < _CT_TYPE_MAX; j++) { + u32 csum_updated = 0; + u8 ip_proto = 0; + if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) { struct flow_match_basic match; @@ -571,8 +724,10 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules, * through the subflows and assign the proper subflow to a_rule */ flow_rule_match_basic(rules[j], &match); - if (match.mask->ip_proto) + if (match.mask->ip_proto) { a_rule->match = rules[j]->match; + ip_proto = match.key->ip_proto; + } } for (i = 0; i < rules[j]->action.num_entries; i++) { @@ -589,11 +744,32 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules, case FLOW_ACTION_CT_METADATA: continue; default: + /* nft entry is generated by tc ct, which mangle action do not care + * the stats, inherit the post entry stats to meet the + * flow_action_hw_stats_check. + */ + if (j == CT_TYPE_NFT) { + if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE) + a_in->hw_stats = tmp_stats; + nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated); + } memcpy(&a_rule->action.entries[offset++], a_in, sizeof(struct flow_action_entry)); break; } } + /* nft entry have mangle action, but do not have checksum action when do NAT, + * hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action + * to meet csum action check. + */ + if (csum_updated) { + struct flow_action_entry *csum_action; + + csum_action = &a_rule->action.entries[offset++]; + csum_action->id = FLOW_ACTION_CSUM; + csum_action->csum_flags = csum_updated; + csum_action->hw_stats = tmp_stats; + } } /* Some actions would have been ignored, so update the num_entries field */ @@ -1191,6 +1367,49 @@ static struct net_device *get_netdev_from_rule(struct flow_rule *rule) return NULL; } +static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action) +{ + if (mangle_action->id != FLOW_ACTION_MANGLE) + return; + + switch (mangle_action->mangle.htype) { + case FLOW_ACT_MANGLE_HDR_TYPE_IP4: + case FLOW_ACT_MANGLE_HDR_TYPE_IP6: + mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val); + mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask); + return; + + case FLOW_ACT_MANGLE_HDR_TYPE_TCP: + case FLOW_ACT_MANGLE_HDR_TYPE_UDP: + mangle_action->mangle.val = (__force u16)cpu_to_be16(mangle_action->mangle.val); + mangle_action->mangle.mask = (__force u16)cpu_to_be16(mangle_action->mangle.mask); + return; + + default: + return; + } +} + +static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act, + struct nfp_fl_ct_flow_entry *entry) +{ + switch (act->id) { + case FLOW_ACTION_CT: + if (act->ct.action == TCA_CT_ACT_NAT) + entry->flags |= NFP_FL_ACTION_DO_NAT; + break; + + case FLOW_ACTION_MANGLE: + entry->flags |= NFP_FL_ACTION_DO_MANGLE; + break; + + default: + break; + } + + return 0; +} + static struct nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt, struct net_device *netdev, @@ -1257,6 +1476,13 @@ nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt, new_act = &entry->rule->action.entries[i]; memcpy(new_act, act, sizeof(struct flow_action_entry)); + /* nft entry mangle field is host byte order, need translate to + * network byte order. + */ + if (is_nft) + nfp_nft_ct_translate_mangle_action(new_act); + + nfp_nft_ct_set_flow_flag(new_act, entry); /* Entunnel is a special case, need to allocate and copy * tunnel info. */ diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h index beb6cceff9d8e88cbd1246beacca61f7368d4b2c..762c0b36e269b973b812ab03db34f91da7960f9e 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h +++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h @@ -103,6 +103,10 @@ enum nfp_nfp_layer_name { _FLOW_PAY_LAYERS_MAX }; +/* NFP flow entry flags. */ +#define NFP_FL_ACTION_DO_NAT BIT(0) +#define NFP_FL_ACTION_DO_MANGLE BIT(1) + /** * struct nfp_fl_ct_flow_entry - Flow entry containing conntrack flow information * @cookie: Flow cookie, same as original TC flow, used as key @@ -115,6 +119,7 @@ enum nfp_nfp_layer_name { * @rule: Reference to the original TC flow rule * @stats: Used to cache stats for updating * @tun_offset: Used to indicate tunnel action offset in action list + * @flags: Used to indicate flow flag like NAT which used by merge. */ struct nfp_fl_ct_flow_entry { unsigned long cookie; @@ -127,6 +132,7 @@ struct nfp_fl_ct_flow_entry { struct flow_rule *rule; struct flow_stats stats; u8 tun_offset; // Set to NFP_FL_CT_NO_TUN if no tun + u8 flags; }; /** diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index 83c97154c0c72d74762594a6f1f24772123d4f32..8593cafa6368365e787e61e72a8a72773c5ad3b4 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -373,10 +373,10 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, if (ipv6_tun) { key_layer_two |= NFP_FLOWER_LAYER2_TUN_IPV6; key_size += - sizeof(struct nfp_flower_ipv6_udp_tun); + sizeof(struct nfp_flower_ipv6_gre_tun); } else { key_size += - sizeof(struct nfp_flower_ipv4_udp_tun); + sizeof(struct nfp_flower_ipv4_gre_tun); } if (enc_op.key) { @@ -1301,9 +1301,14 @@ static bool offload_pre_check(struct flow_cls_offload *flow) { struct flow_rule *rule = flow_cls_offload_flow_rule(flow); struct flow_dissector *dissector = rule->match.dissector; + struct flow_match_ct ct; - if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) - return false; + if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { + flow_rule_match_ct(rule, &ct); + /* Allow special case where CT match is all 0 */ + if (memchr_inv(ct.key, 0, sizeof(*ct.key))) + return false; + } if (flow->common.chain_index) return false; diff --git a/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c index 4e5df9f2c37224378d802e704821434bb83fc89a..99052a925d9ecac75afdca263becf70b005a891d 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c @@ -119,7 +119,8 @@ int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress, static int nfp_policer_validate(const struct flow_action *action, const struct flow_action_entry *act, - struct netlink_ext_ack *extack) + struct netlink_ext_ack *extack, + bool ingress) { if (act->police.exceed.act_id != FLOW_ACTION_DROP) { NL_SET_ERR_MSG_MOD(extack, @@ -127,11 +128,20 @@ static int nfp_policer_validate(const struct flow_action *action, return -EOPNOTSUPP; } - if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && - act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { - NL_SET_ERR_MSG_MOD(extack, - "Offload not supported when conform action is not pipe or ok"); - return -EOPNOTSUPP; + if (ingress) { + if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not continue or ok"); + return -EOPNOTSUPP; + } + } else { + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(extack, + "Offload not supported when conform action is not pipe or ok"); + return -EOPNOTSUPP; + } } if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT && @@ -217,7 +227,7 @@ nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev, return -EOPNOTSUPP; } - err = nfp_policer_validate(&flow->rule->action, action, extack); + err = nfp_policer_validate(&flow->rule->action, action, extack, true); if (err) return err; @@ -686,6 +696,7 @@ nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act, bool pps_support, pps; bool add = false; u64 rate; + int err; pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS); @@ -697,6 +708,11 @@ nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act, "unsupported offload: qos rate limit offload requires police action"); continue; } + + err = nfp_policer_validate(&fl_act->action, action, extack, false); + if (err) + return err; + if (action->police.rate_bytes_ps > 0) { rate = action->police.rate_bytes_ps; burst = action->police.burst; diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c index 65e24316876551b0262950a21adbe5a0c3741afa..5d9db8c2a5b437c5e2032e4a17d055bf620d9c82 100644 --- a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c +++ b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c @@ -84,7 +84,7 @@ static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring, nfp_net_xsk_rx_drop(r_vec, xrxbuf); return; } - memcpy(skb_put(skb, pkt_len), xrxbuf->xdp->data, pkt_len); + skb_put_data(skb, xrxbuf->xdp->data, pkt_len); skb->mark = meta->mark; skb_set_hash(skb, meta->hash, meta->hash_type); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c index 873429f7a6daeca8ad6db3c0590eb098e032f23f..e66e548919d4899fe3f0a089f3cdbb2228815d6e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c @@ -691,6 +691,71 @@ static int nfp_pf_find_rtsyms(struct nfp_pf *pf) return 0; } +int nfp_net_pf_get_app_id(struct nfp_pf *pf) +{ + return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id", + NFP_APP_CORE_NIC); +} + +static u64 nfp_net_pf_get_app_cap(struct nfp_pf *pf) +{ + char name[32]; + int err = 0; + u64 val; + + snprintf(name, sizeof(name), "_pf%u_net_app_cap", nfp_cppcore_pcie_unit(pf->cpp)); + + val = nfp_rtsym_read_le(pf->rtbl, name, &err); + if (err) { + if (err != -ENOENT) + nfp_err(pf->cpp, "Unable to read symbol %s\n", name); + + return 0; + } + + return val; +} + +static int nfp_pf_cfg_hwinfo(struct nfp_pf *pf, bool sp_indiff) +{ + struct nfp_nsp *nsp; + char hwinfo[32]; + int err; + + nsp = nfp_nsp_open(pf->cpp); + if (IS_ERR(nsp)) + return PTR_ERR(nsp); + + snprintf(hwinfo, sizeof(hwinfo), "sp_indiff=%d", sp_indiff); + err = nfp_nsp_hwinfo_set(nsp, hwinfo, sizeof(hwinfo)); + /* Not a fatal error, no need to return error to stop driver from loading */ + if (err) { + nfp_warn(pf->cpp, "HWinfo(sp_indiff=%d) set failed: %d\n", sp_indiff, err); + } else { + /* Need reinit eth_tbl since the eth table state may change + * after sp_indiff is configured. + */ + kfree(pf->eth_tbl); + pf->eth_tbl = __nfp_eth_read_ports(pf->cpp, nsp); + } + + nfp_nsp_close(nsp); + return 0; +} + +static int nfp_pf_nsp_cfg(struct nfp_pf *pf) +{ + bool sp_indiff = (nfp_net_pf_get_app_id(pf) == NFP_APP_FLOWER_NIC) || + (nfp_net_pf_get_app_cap(pf) & NFP_NET_APP_CAP_SP_INDIFF); + + return nfp_pf_cfg_hwinfo(pf, sp_indiff); +} + +static void nfp_pf_nsp_clean(struct nfp_pf *pf) +{ + nfp_pf_cfg_hwinfo(pf, false); +} + static int nfp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) { @@ -791,10 +856,14 @@ static int nfp_pci_probe(struct pci_dev *pdev, goto err_fw_unload; } - err = nfp_net_pci_probe(pf); + err = nfp_pf_nsp_cfg(pf); if (err) goto err_fw_unload; + err = nfp_net_pci_probe(pf); + if (err) + goto err_nsp_clean; + err = nfp_hwmon_register(pf); if (err) { dev_err(&pdev->dev, "Failed to register hwmon info\n"); @@ -805,6 +874,8 @@ static int nfp_pci_probe(struct pci_dev *pdev, err_net_remove: nfp_net_pci_remove(pf); +err_nsp_clean: + nfp_pf_nsp_clean(pf); err_fw_unload: kfree(pf->rtbl); nfp_mip_close(pf->mip); @@ -844,6 +915,7 @@ static void __nfp_pci_shutdown(struct pci_dev *pdev, bool unload_fw) nfp_net_pci_remove(pf); + nfp_pf_nsp_clean(pf); vfree(pf->dumpspec); kfree(pf->rtbl); nfp_mip_close(pf->mip); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h index f56ca11de13401d27b3bd74649794ee0b9dccbf4..afd3edfa242832a9a415982b34da5514f59f9df7 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_main.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h @@ -161,6 +161,7 @@ bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb); int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format, unsigned int default_val); +int nfp_net_pf_get_app_id(struct nfp_pf *pf); u8 __iomem * nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt, unsigned int min_size, struct nfp_cpp_area **area); @@ -190,4 +191,7 @@ int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb, int nfp_devlink_params_register(struct nfp_pf *pf); void nfp_devlink_params_unregister(struct nfp_pf *pf); + +unsigned int nfp_net_lr2speed(unsigned int linkrate); +unsigned int nfp_net_speed2lr(unsigned int speed); #endif /* NFP_MAIN_H */ diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index cf4d6f1129fa29c0ca007bb3ecf0f8d246dfea85..27f4786ace4fbbb38253273808146c4ca8668c22 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -474,19 +474,22 @@ static void nfp_net_read_link_status(struct nfp_net *nn) { unsigned long flags; bool link_up; - u32 sts; + u16 sts; spin_lock_irqsave(&nn->link_status_lock, flags); - sts = nn_readl(nn, NFP_NET_CFG_STS); + sts = nn_readw(nn, NFP_NET_CFG_STS); link_up = !!(sts & NFP_NET_CFG_STS_LINK); if (nn->link_up == link_up) goto out; nn->link_up = link_up; - if (nn->port) + if (nn->port) { set_bit(NFP_PORT_CHANGED, &nn->port->flags); + if (nn->port->link_cb) + nn->port->link_cb(nn->port); + } if (nn->link_up) { netif_carrier_on(nn->dp.netdev); @@ -768,9 +771,7 @@ nfp_net_napi_add(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx) { if (dp->netdev) netif_napi_add(dp->netdev, &r_vec->napi, - nfp_net_has_xsk_pool_slow(dp, idx) ? - dp->ops->xsk_poll : dp->ops->poll, - NAPI_POLL_WEIGHT); + nfp_net_has_xsk_pool_slow(dp, idx) ? dp->ops->xsk_poll : dp->ops->poll); else tasklet_enable(&r_vec->tasklet); } @@ -1630,21 +1631,21 @@ static void nfp_net_stat64(struct net_device *netdev, unsigned int start; do { - start = u64_stats_fetch_begin(&r_vec->rx_sync); + start = u64_stats_fetch_begin_irq(&r_vec->rx_sync); data[0] = r_vec->rx_pkts; data[1] = r_vec->rx_bytes; data[2] = r_vec->rx_drops; - } while (u64_stats_fetch_retry(&r_vec->rx_sync, start)); + } while (u64_stats_fetch_retry_irq(&r_vec->rx_sync, start)); stats->rx_packets += data[0]; stats->rx_bytes += data[1]; stats->rx_dropped += data[2]; do { - start = u64_stats_fetch_begin(&r_vec->tx_sync); + start = u64_stats_fetch_begin_irq(&r_vec->tx_sync); data[0] = r_vec->tx_pkts; data[1] = r_vec->tx_bytes; data[2] = r_vec->tx_errors; - } while (u64_stats_fetch_retry(&r_vec->tx_sync, start)); + } while (u64_stats_fetch_retry_irq(&r_vec->tx_sync, start)); stats->tx_packets += data[0]; stats->tx_bytes += data[1]; stats->tx_errors += data[2]; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h index ac05ec34d69ede56bf6ebb3b5d975b54b2236a69..6714d5e8fdab2117f1d0cab15274ce78e0bee8c9 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h @@ -14,6 +14,9 @@ #include +/* 64-bit per app capabilities */ +#define NFP_NET_APP_CAP_SP_INDIFF BIT_ULL(0) /* indifferent to port speed */ + /* Configuration BAR size. * * The configuration BAR is 8K in size, but due to @@ -193,6 +196,10 @@ #define NFP_NET_CFG_STS_LINK_RATE_40G 5 #define NFP_NET_CFG_STS_LINK_RATE_50G 6 #define NFP_NET_CFG_STS_LINK_RATE_100G 7 +/* NSP Link rate is a 16-bit word. It's determined by NSP and + * written to CFG BAR by NFP driver. + */ +#define NFP_NET_CFG_STS_NSP_LINK_RATE 0x0036 #define NFP_NET_CFG_CAP 0x0038 #define NFP_NET_CFG_MAX_TXRINGS 0x003c #define NFP_NET_CFG_MAX_RXRINGS 0x0040 diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index eeb1455a4e5dbb05069aba8c409cb9ecd3220c5d..22a5d241908404d94a4c26e25ff614dea11fabca 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -205,7 +205,7 @@ nfp_get_drvinfo(struct nfp_app *app, struct pci_dev *pdev, { char nsp_version[ETHTOOL_FWVERS_LEN] = {}; - strlcpy(drvinfo->driver, dev_driver_string(&pdev->dev), + strscpy(drvinfo->driver, dev_driver_string(&pdev->dev), sizeof(drvinfo->driver)); nfp_net_get_nspinfo(app, nsp_version); snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), @@ -222,18 +222,49 @@ nfp_net_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) snprintf(vnic_version, sizeof(vnic_version), "%d.%d.%d.%d", nn->fw_ver.extend, nn->fw_ver.class, nn->fw_ver.major, nn->fw_ver.minor); - strlcpy(drvinfo->bus_info, pci_name(nn->pdev), + strscpy(drvinfo->bus_info, pci_name(nn->pdev), sizeof(drvinfo->bus_info)); nfp_get_drvinfo(nn->app, nn->pdev, vnic_version, drvinfo); } +static int +nfp_net_nway_reset(struct net_device *netdev) +{ + struct nfp_eth_table_port *eth_port; + struct nfp_port *port; + int err; + + port = nfp_port_from_netdev(netdev); + eth_port = nfp_port_get_eth_port(port); + if (!eth_port) + return -EOPNOTSUPP; + + if (!netif_running(netdev)) + return 0; + + err = nfp_eth_set_configured(port->app->cpp, eth_port->index, false); + if (err) { + netdev_info(netdev, "Link down failed: %d\n", err); + return err; + } + + err = nfp_eth_set_configured(port->app->cpp, eth_port->index, true); + if (err) { + netdev_info(netdev, "Link up failed: %d\n", err); + return err; + } + + netdev_info(netdev, "Link reset succeeded\n"); + return 0; +} + static void nfp_app_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { struct nfp_app *app = nfp_app_from_netdev(netdev); - strlcpy(drvinfo->bus_info, pci_name(app->pdev), + strscpy(drvinfo->bus_info, pci_name(app->pdev), sizeof(drvinfo->bus_info)); nfp_get_drvinfo(app, app->pdev, "*", drvinfo); } @@ -273,20 +304,11 @@ static int nfp_net_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd) { - static const u32 ls_to_ethtool[] = { - [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = 0, - [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = SPEED_UNKNOWN, - [NFP_NET_CFG_STS_LINK_RATE_1G] = SPEED_1000, - [NFP_NET_CFG_STS_LINK_RATE_10G] = SPEED_10000, - [NFP_NET_CFG_STS_LINK_RATE_25G] = SPEED_25000, - [NFP_NET_CFG_STS_LINK_RATE_40G] = SPEED_40000, - [NFP_NET_CFG_STS_LINK_RATE_50G] = SPEED_50000, - [NFP_NET_CFG_STS_LINK_RATE_100G] = SPEED_100000, - }; struct nfp_eth_table_port *eth_port; struct nfp_port *port; struct nfp_net *nn; - u32 sts, ls; + unsigned int speed; + u16 sts; /* Init to unknowns */ ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); @@ -299,8 +321,13 @@ nfp_net_get_link_ksettings(struct net_device *netdev, if (eth_port) { ethtool_link_ksettings_add_link_mode(cmd, supported, Pause); ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause); - cmd->base.autoneg = eth_port->aneg != NFP_ANEG_DISABLED ? - AUTONEG_ENABLE : AUTONEG_DISABLE; + if (eth_port->supp_aneg) { + ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg); + if (eth_port->aneg == NFP_ANEG_AUTO) { + ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg); + cmd->base.autoneg = AUTONEG_ENABLE; + } + } nfp_net_set_fec_link_mode(eth_port, cmd); } @@ -319,18 +346,15 @@ nfp_net_get_link_ksettings(struct net_device *netdev, return -EOPNOTSUPP; nn = netdev_priv(netdev); - sts = nn_readl(nn, NFP_NET_CFG_STS); - - ls = FIELD_GET(NFP_NET_CFG_STS_LINK_RATE, sts); - if (ls == NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED) + sts = nn_readw(nn, NFP_NET_CFG_STS); + speed = nfp_net_lr2speed(FIELD_GET(NFP_NET_CFG_STS_LINK_RATE, sts)); + if (!speed) return -EOPNOTSUPP; - if (ls == NFP_NET_CFG_STS_LINK_RATE_UNKNOWN || - ls >= ARRAY_SIZE(ls_to_ethtool)) - return 0; - - cmd->base.speed = ls_to_ethtool[ls]; - cmd->base.duplex = DUPLEX_FULL; + if (speed != SPEED_UNKNOWN) { + cmd->base.speed = speed; + cmd->base.duplex = DUPLEX_FULL; + } return 0; } @@ -339,6 +363,7 @@ static int nfp_net_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *cmd) { + bool req_aneg = (cmd->base.autoneg == AUTONEG_ENABLE); struct nfp_eth_table_port *eth_port; struct nfp_port *port; struct nfp_nsp *nsp; @@ -358,13 +383,25 @@ nfp_net_set_link_ksettings(struct net_device *netdev, if (IS_ERR(nsp)) return PTR_ERR(nsp); - err = __nfp_eth_set_aneg(nsp, cmd->base.autoneg == AUTONEG_ENABLE ? - NFP_ANEG_AUTO : NFP_ANEG_DISABLED); + if (req_aneg && !eth_port->supp_aneg) { + netdev_warn(netdev, "Autoneg is not supported.\n"); + err = -EOPNOTSUPP; + goto err_bad_set; + } + + err = __nfp_eth_set_aneg(nsp, req_aneg ? NFP_ANEG_AUTO : NFP_ANEG_DISABLED); if (err) goto err_bad_set; + if (cmd->base.speed != SPEED_UNKNOWN) { u32 speed = cmd->base.speed / eth_port->lanes; + if (req_aneg) { + netdev_err(netdev, "Speed changing is not allowed when working on autoneg mode.\n"); + err = -EINVAL; + goto err_bad_set; + } + err = __nfp_eth_set_speed(nsp, speed); if (err) goto err_bad_set; @@ -649,7 +686,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) unsigned int start; do { - start = u64_stats_fetch_begin(&nn->r_vecs[i].rx_sync); + start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].rx_sync); data[0] = nn->r_vecs[i].rx_pkts; tmp[0] = nn->r_vecs[i].hw_csum_rx_ok; tmp[1] = nn->r_vecs[i].hw_csum_rx_inner_ok; @@ -657,10 +694,10 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) tmp[3] = nn->r_vecs[i].hw_csum_rx_error; tmp[4] = nn->r_vecs[i].rx_replace_buf_alloc_fail; tmp[5] = nn->r_vecs[i].hw_tls_rx; - } while (u64_stats_fetch_retry(&nn->r_vecs[i].rx_sync, start)); + } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].rx_sync, start)); do { - start = u64_stats_fetch_begin(&nn->r_vecs[i].tx_sync); + start = u64_stats_fetch_begin_irq(&nn->r_vecs[i].tx_sync); data[1] = nn->r_vecs[i].tx_pkts; data[2] = nn->r_vecs[i].tx_busy; tmp[6] = nn->r_vecs[i].hw_csum_tx; @@ -670,7 +707,7 @@ static u64 *nfp_vnic_get_sw_stats(struct net_device *netdev, u64 *data) tmp[10] = nn->r_vecs[i].hw_tls_tx; tmp[11] = nn->r_vecs[i].tls_tx_fallback; tmp[12] = nn->r_vecs[i].tls_tx_no_fallback; - } while (u64_stats_fetch_retry(&nn->r_vecs[i].tx_sync, start)); + } while (u64_stats_fetch_retry_irq(&nn->r_vecs[i].tx_sync, start)); data += NN_RVEC_PER_Q_STATS; @@ -1008,7 +1045,7 @@ nfp_port_get_fecparam(struct net_device *netdev, return 0; param->fec = nfp_port_fec_nsp_to_ethtool(eth_port->fec_modes_supported); - param->active_fec = nfp_port_fec_nsp_to_ethtool(eth_port->fec); + param->active_fec = nfp_port_fec_nsp_to_ethtool(BIT(eth_port->act_fec)); return 0; } @@ -1676,11 +1713,166 @@ static int nfp_net_set_phys_id(struct net_device *netdev, return err; } +#define NFP_EEPROM_LEN ETH_ALEN + +static int +nfp_net_get_eeprom_len(struct net_device *netdev) +{ + struct nfp_eth_table_port *eth_port; + struct nfp_port *port; + + port = nfp_port_from_netdev(netdev); + eth_port = __nfp_port_get_eth_port(port); + if (!eth_port) + return 0; + + return NFP_EEPROM_LEN; +} + +static int +nfp_net_get_nsp_hwindex(struct net_device *netdev, + struct nfp_nsp **nspptr, + u32 *index) +{ + struct nfp_eth_table_port *eth_port; + struct nfp_port *port; + struct nfp_nsp *nsp; + int err; + + port = nfp_port_from_netdev(netdev); + eth_port = __nfp_port_get_eth_port(port); + if (!eth_port) + return -EOPNOTSUPP; + + nsp = nfp_nsp_open(port->app->cpp); + if (IS_ERR(nsp)) { + err = PTR_ERR(nsp); + netdev_err(netdev, "Failed to access the NSP: %d\n", err); + return err; + } + + if (!nfp_nsp_has_hwinfo_lookup(nsp)) { + netdev_err(netdev, "NSP doesn't support PF MAC generation\n"); + nfp_nsp_close(nsp); + return -EOPNOTSUPP; + } + + *nspptr = nsp; + *index = eth_port->eth_index; + + return 0; +} + +static int +nfp_net_get_port_mac_by_hwinfo(struct net_device *netdev, + u8 *mac_addr) +{ + char hwinfo[32] = {}; + struct nfp_nsp *nsp; + u32 index; + int err; + + err = nfp_net_get_nsp_hwindex(netdev, &nsp, &index); + if (err) + return err; + + snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac", index); + err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); + nfp_nsp_close(nsp); + if (err) { + netdev_err(netdev, "Reading persistent MAC address failed: %d\n", + err); + return -EOPNOTSUPP; + } + + if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", + &mac_addr[0], &mac_addr[1], &mac_addr[2], + &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { + netdev_err(netdev, "Can't parse persistent MAC address (%s)\n", + hwinfo); + return -EOPNOTSUPP; + } + + return 0; +} + +static int +nfp_net_set_port_mac_by_hwinfo(struct net_device *netdev, + u8 *mac_addr) +{ + char hwinfo[32] = {}; + struct nfp_nsp *nsp; + u32 index; + int err; + + err = nfp_net_get_nsp_hwindex(netdev, &nsp, &index); + if (err) + return err; + + snprintf(hwinfo, sizeof(hwinfo), + "eth%u.mac=%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", + index, mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3], + mac_addr[4], mac_addr[5]); + + err = nfp_nsp_hwinfo_set(nsp, hwinfo, sizeof(hwinfo)); + nfp_nsp_close(nsp); + if (err) { + netdev_err(netdev, "HWinfo set failed: %d, hwinfo: %s\n", + err, hwinfo); + return -EOPNOTSUPP; + } + + return 0; +} + +static int +nfp_net_get_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) +{ + struct nfp_net *nn = netdev_priv(netdev); + u8 buf[NFP_EEPROM_LEN] = {}; + + if (eeprom->len == 0) + return -EINVAL; + + if (nfp_net_get_port_mac_by_hwinfo(netdev, buf)) + return -EOPNOTSUPP; + + eeprom->magic = nn->pdev->vendor | (nn->pdev->device << 16); + memcpy(bytes, buf + eeprom->offset, eeprom->len); + + return 0; +} + +static int +nfp_net_set_eeprom(struct net_device *netdev, + struct ethtool_eeprom *eeprom, u8 *bytes) +{ + struct nfp_net *nn = netdev_priv(netdev); + u8 buf[NFP_EEPROM_LEN] = {}; + + if (eeprom->len == 0) + return -EINVAL; + + if (eeprom->magic != (nn->pdev->vendor | nn->pdev->device << 16)) + return -EINVAL; + + if (nfp_net_get_port_mac_by_hwinfo(netdev, buf)) + return -EOPNOTSUPP; + + memcpy(buf + eeprom->offset, bytes, eeprom->len); + if (nfp_net_set_port_mac_by_hwinfo(netdev, buf)) + return -EOPNOTSUPP; + + return 0; +} + static const struct ethtool_ops nfp_net_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE, .get_drvinfo = nfp_net_get_drvinfo, + .nway_reset = nfp_net_nway_reset, .get_link = ethtool_op_get_link, .get_ringparam = nfp_net_get_ringparam, .set_ringparam = nfp_net_set_ringparam, @@ -1699,6 +1891,9 @@ static const struct ethtool_ops nfp_net_ethtool_ops = { .set_dump = nfp_app_set_dump, .get_dump_flag = nfp_app_get_dump_flag, .get_dump_data = nfp_app_get_dump_data, + .get_eeprom_len = nfp_net_get_eeprom_len, + .get_eeprom = nfp_net_get_eeprom, + .set_eeprom = nfp_net_set_eeprom, .get_module_info = nfp_port_get_module_info, .get_module_eeprom = nfp_port_get_module_eeprom, .get_coalesce = nfp_net_get_coalesce, @@ -1715,6 +1910,7 @@ static const struct ethtool_ops nfp_net_ethtool_ops = { const struct ethtool_ops nfp_port_ethtool_ops = { .get_drvinfo = nfp_app_get_drvinfo, + .nway_reset = nfp_net_nway_reset, .get_link = ethtool_op_get_link, .get_strings = nfp_port_get_strings, .get_ethtool_stats = nfp_port_get_stats, diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c index ca4e05650fe6446231d59b935c5f0d3ee70d6a59..3bae92dc899ed03930f30b9de3fb0b0ce6b84f4a 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c @@ -77,12 +77,6 @@ static int nfp_net_pf_get_num_ports(struct nfp_pf *pf) return nfp_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1); } -static int nfp_net_pf_get_app_id(struct nfp_pf *pf) -{ - return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id", - NFP_APP_CORE_NIC); -} - static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn) { if (nfp_net_is_data_vnic(nn)) @@ -202,6 +196,9 @@ nfp_net_pf_alloc_vnics(struct nfp_pf *pf, void __iomem *ctrl_bar, goto err_free_prev; } + if (nn->port) + nn->port->link_cb = nfp_net_refresh_port_table; + ctrl_bar += NFP_PF_CSR_SLICE_SIZE; /* Kill the vNIC if app init marked it as invalid */ @@ -523,6 +520,57 @@ err_unmap_ctrl: return err; } +static const unsigned int lr_to_speed[] = { + [NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED] = 0, + [NFP_NET_CFG_STS_LINK_RATE_UNKNOWN] = SPEED_UNKNOWN, + [NFP_NET_CFG_STS_LINK_RATE_1G] = SPEED_1000, + [NFP_NET_CFG_STS_LINK_RATE_10G] = SPEED_10000, + [NFP_NET_CFG_STS_LINK_RATE_25G] = SPEED_25000, + [NFP_NET_CFG_STS_LINK_RATE_40G] = SPEED_40000, + [NFP_NET_CFG_STS_LINK_RATE_50G] = SPEED_50000, + [NFP_NET_CFG_STS_LINK_RATE_100G] = SPEED_100000, +}; + +unsigned int nfp_net_lr2speed(unsigned int linkrate) +{ + if (linkrate < ARRAY_SIZE(lr_to_speed)) + return lr_to_speed[linkrate]; + + return SPEED_UNKNOWN; +} + +unsigned int nfp_net_speed2lr(unsigned int speed) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(lr_to_speed); i++) { + if (speed == lr_to_speed[i]) + return i; + } + + return NFP_NET_CFG_STS_LINK_RATE_UNKNOWN; +} + +static void nfp_net_notify_port_speed(struct nfp_port *port) +{ + struct net_device *netdev = port->netdev; + struct nfp_net *nn; + u16 sts; + + if (!nfp_netdev_is_nfp_net(netdev)) + return; + + nn = netdev_priv(netdev); + sts = nn_readw(nn, NFP_NET_CFG_STS); + + if (!(sts & NFP_NET_CFG_STS_LINK)) { + nn_writew(nn, NFP_NET_CFG_STS_NSP_LINK_RATE, NFP_NET_CFG_STS_LINK_RATE_UNKNOWN); + return; + } + + nn_writew(nn, NFP_NET_CFG_STS_NSP_LINK_RATE, nfp_net_speed2lr(port->eth_port->speed)); +} + static int nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port, struct nfp_eth_table *eth_table) @@ -544,6 +592,7 @@ nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port, } memcpy(port->eth_port, eth_port, sizeof(*eth_port)); + nfp_net_notify_port_speed(port); return 0; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h index d1ebe6c72f7fbc83b1ba82ac1fe337a28dfff59a..6793cdf9ff1158b48e5fa9e727fc6b9f73e0680e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_port.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h @@ -46,6 +46,7 @@ enum nfp_port_flags { * @tc_offload_cnt: number of active TC offloads, how offloads are counted * is not defined, use as a boolean * @app: backpointer to the app structure + * @link_cb: callback when link status changed * @dl_port: devlink port structure * @eth_id: for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme * @eth_forced: for %NFP_PORT_PHYS_PORT port is forced UP or DOWN, don't change @@ -66,6 +67,7 @@ struct nfp_port { unsigned long tc_offload_cnt; struct nfp_app *app; + void (*link_cb)(struct nfp_port *port); struct devlink_port dl_port; diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h index 77d66855be42e12ea4065611567f16850fa2bf96..992d72ac98d38926d8e18b66e79b28a3e8bd6153 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h @@ -132,6 +132,7 @@ enum nfp_eth_fec { * @ports.interface: interface (module) plugged in * @ports.media: media type of the @interface * @ports.fec: forward error correction mode + * @ports.act_fec: active forward error correction mode * @ports.aneg: auto negotiation mode * @ports.mac_addr: interface MAC address * @ports.label_port: port id @@ -162,6 +163,7 @@ struct nfp_eth_table { enum nfp_eth_media media; enum nfp_eth_fec fec; + enum nfp_eth_fec act_fec; enum nfp_eth_aneg aneg; u8 mac_addr[ETH_ALEN]; @@ -172,6 +174,7 @@ struct nfp_eth_table { bool enabled; bool tx_enabled; bool rx_enabled; + bool supp_aneg; bool override_changed; diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c index edd300033735219c901373ea85983d210ab522c2..bb64efec4c46b336fc0ebb64d7b665cd1bd5aeec 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c @@ -27,6 +27,7 @@ #define NSP_ETH_PORT_PHYLABEL GENMASK_ULL(59, 54) #define NSP_ETH_PORT_FEC_SUPP_BASER BIT_ULL(60) #define NSP_ETH_PORT_FEC_SUPP_RS BIT_ULL(61) +#define NSP_ETH_PORT_SUPP_ANEG BIT_ULL(63) #define NSP_ETH_PORT_LANES_MASK cpu_to_le64(NSP_ETH_PORT_LANES) @@ -40,6 +41,7 @@ #define NSP_ETH_STATE_OVRD_CHNG BIT_ULL(22) #define NSP_ETH_STATE_ANEG GENMASK_ULL(25, 23) #define NSP_ETH_STATE_FEC GENMASK_ULL(27, 26) +#define NSP_ETH_STATE_ACT_FEC GENMASK_ULL(29, 28) #define NSP_ETH_CTRL_CONFIGURED BIT_ULL(0) #define NSP_ETH_CTRL_ENABLED BIT_ULL(1) @@ -170,7 +172,14 @@ nfp_eth_port_translate(struct nfp_nsp *nsp, const union eth_table_entry *src, if (dst->fec_modes_supported) dst->fec_modes_supported |= NFP_FEC_AUTO | NFP_FEC_DISABLED; - dst->fec = 1 << FIELD_GET(NSP_ETH_STATE_FEC, state); + dst->fec = FIELD_GET(NSP_ETH_STATE_FEC, state); + dst->act_fec = dst->fec; + + if (nfp_nsp_get_abi_ver_minor(nsp) < 33) + return; + + dst->act_fec = FIELD_GET(NSP_ETH_STATE_ACT_FEC, state); + dst->supp_aneg = FIELD_GET(NSP_ETH_PORT_SUPP_ANEG, port); } static void @@ -507,6 +516,7 @@ int nfp_eth_set_idmode(struct nfp_cpp *cpp, unsigned int idx, bool state) if (nfp_nsp_get_abi_ver_minor(nsp) < 32) { nfp_err(nfp_nsp_cpp(nsp), "set id mode operation not supported, please update flash\n"); + nfp_eth_config_cleanup_end(nsp); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c index 4b3482ce90a1c14377fad2882c2d0bf7c62cf031..3db4a243174194bdc25f880f4cd6a4f6c939aac5 100644 --- a/drivers/net/ethernet/ni/nixge.c +++ b/drivers/net/ethernet/ni/nixge.c @@ -990,8 +990,8 @@ static const struct net_device_ops nixge_netdev_ops = { static void nixge_ethtools_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *ed) { - strlcpy(ed->driver, "nixge", sizeof(ed->driver)); - strlcpy(ed->bus_info, "platform", sizeof(ed->bus_info)); + strscpy(ed->driver, "nixge", sizeof(ed->driver)); + strscpy(ed->bus_info, "platform", sizeof(ed->bus_info)); } static int @@ -1294,7 +1294,7 @@ static int nixge_probe(struct platform_device *pdev) priv->ndev = ndev; priv->dev = &pdev->dev; - netif_napi_add(ndev, &priv->napi, nixge_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, nixge_poll); err = nixge_of_get_resources(pdev); if (err) goto free_netdev; diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 5116badaf0919748aeaaca5c5cceb9dfb7a8d11a..daa028729d444a7b51f056868d4b12e92acf6adf 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -4291,9 +4291,9 @@ static void nv_do_stats_poll(struct timer_list *t) static void nv_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct fe_priv *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, FORCEDETH_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, FORCEDETH_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static void nv_get_wol(struct net_device *dev, struct ethtool_wolinfo *wolinfo) @@ -5876,7 +5876,7 @@ static int nv_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) else dev->netdev_ops = &nv_netdev_ops_optimized; - netif_napi_add(dev, &np->napi, nv_napi_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &np->napi, nv_napi_poll); dev->ethtool_ops = &ops; dev->watchdog_timeo = NV_WATCHDOG_TIMEO; diff --git a/drivers/net/ethernet/nxp/lpc_eth.c b/drivers/net/ethernet/nxp/lpc_eth.c index f606d75b33b4f59dab952b7b3d0fe2a394b2aba6..1a4a272f4c5cf8c3d8a619502fa96eaba30bb445 100644 --- a/drivers/net/ethernet/nxp/lpc_eth.c +++ b/drivers/net/ethernet/nxp/lpc_eth.c @@ -1184,9 +1184,9 @@ static int lpc_eth_open(struct net_device *ndev) static void lpc_eth_ethtool_getdrvinfo(struct net_device *ndev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, MODNAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(ndev->dev.parent), + strscpy(info->driver, MODNAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(ndev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c index 84cc79e928c8bfdbd422301552ba5a46cfca1642..541b8bcd3223da642cc2e63e53ba302015ac29d9 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c @@ -169,9 +169,9 @@ static void pch_gbe_get_drvinfo(struct net_device *netdev, { struct pch_gbe_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, pch_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, pch_driver_version, sizeof(drvinfo->version)); + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c index 46da937ad27f80e65b717286705aea93099d1dfc..3f2c30184752d88fa170c0a71cf083ba75b84312 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c @@ -2516,8 +2516,7 @@ static int pch_gbe_probe(struct pci_dev *pdev, netdev->netdev_ops = &pch_gbe_netdev_ops; netdev->watchdog_timeo = PCH_GBE_WATCHDOG_PERIOD; - netif_napi_add(netdev, &adapter->napi, - pch_gbe_napi_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &adapter->napi, pch_gbe_napi_poll); netdev->hw_features = NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; netdev->features = netdev->hw_features; diff --git a/drivers/net/ethernet/packetengines/hamachi.c b/drivers/net/ethernet/packetengines/hamachi.c index 9c408328be0db40ea4c925f9db7c0479b0d7dcf7..1cc0010871932baa4f9c2d405e03d5105614d4df 100644 --- a/drivers/net/ethernet/packetengines/hamachi.c +++ b/drivers/net/ethernet/packetengines/hamachi.c @@ -1819,9 +1819,9 @@ static void hamachi_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo * { struct hamachi_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int hamachi_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/packetengines/yellowfin.c b/drivers/net/ethernet/packetengines/yellowfin.c index 03650022d44412f9c81efc1eabc59b3f96505ee0..640ac01689fb5eb71aaa5239ef5db03a547194dd 100644 --- a/drivers/net/ethernet/packetengines/yellowfin.c +++ b/drivers/net/ethernet/packetengines/yellowfin.c @@ -1340,9 +1340,9 @@ static void yellowfin_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo { struct yellowfin_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static const struct ethtool_ops ethtool_ops = { diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c index f0ace3a0e85cb70b42554498283cfaf58c3b05a1..aaab590ef548de1a63734c57171f066868c1e85d 100644 --- a/drivers/net/ethernet/pasemi/pasemi_mac.c +++ b/drivers/net/ethernet/pasemi/pasemi_mac.c @@ -1697,7 +1697,7 @@ pasemi_mac_probe(struct pci_dev *pdev, const struct pci_device_id *ent) mac->pdev = pdev; mac->netdev = dev; - netif_napi_add(dev, &mac->napi, pasemi_mac_poll, 64); + netif_napi_add(dev, &mac->napi, pasemi_mac_poll); dev->features = NETIF_F_IP_CSUM | NETIF_F_LLTX | NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_GSO; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c index 0a7a757494bc5f8448a03e4a82ad10aff462d0f5..ce436e97324ac31bfd41e42fb98e992f2fac159a 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c @@ -320,16 +320,16 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_err(dev, "Cannot enable existing VFs: %d\n", err); } - err = ionic_lif_register(ionic->lif); + err = ionic_devlink_register(ionic); if (err) { - dev_err(dev, "Cannot register LIF: %d, aborting\n", err); + dev_err(dev, "Cannot register devlink: %d\n", err); goto err_out_deinit_lifs; } - err = ionic_devlink_register(ionic); + err = ionic_lif_register(ionic->lif); if (err) { - dev_err(dev, "Cannot register devlink: %d\n", err); - goto err_out_deregister_lifs; + dev_err(dev, "Cannot register LIF: %d, aborting\n", err); + goto err_out_deregister_devlink; } mod_timer(&ionic->watchdog_timer, @@ -337,8 +337,8 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; -err_out_deregister_lifs: - ionic_lif_unregister(ionic->lif); +err_out_deregister_devlink: + ionic_devlink_unregister(ionic); err_out_deinit_lifs: ionic_vf_dealloc(ionic); ionic_lif_deinit(ionic->lif); @@ -380,8 +380,8 @@ static void ionic_remove(struct pci_dev *pdev) del_timer_sync(&ionic->watchdog_timer); if (ionic->lif) { - ionic_devlink_unregister(ionic); ionic_lif_unregister(ionic->lif); + ionic_devlink_unregister(ionic); ionic_lif_deinit(ionic->lif); ionic_lif_free(ionic->lif); ionic->lif = NULL; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index 1443f788ee37cf911a0c28ae3c0ef6b8a11a1398..5d58fd99be3cf2ecf0e03fb544ae58ae493cdc2c 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -774,8 +774,7 @@ static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) dev_dbg(dev, "txq->hw_index %d\n", q->hw_index); if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) - netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi); qcq->flags |= IONIC_QCQ_F_INITED; @@ -830,11 +829,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index); if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) - netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi); else - netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi); qcq->flags |= IONIC_QCQ_F_INITED; @@ -1564,8 +1561,67 @@ static int ionic_set_features(struct net_device *netdev, return err; } +static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac) +{ + struct ionic_admin_ctx ctx = { + .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work), + .cmd.lif_setattr = { + .opcode = IONIC_CMD_LIF_SETATTR, + .index = cpu_to_le16(lif->index), + .attr = IONIC_LIF_ATTR_MAC, + }, + }; + + ether_addr_copy(ctx.cmd.lif_setattr.mac, mac); + return ionic_adminq_post_wait(lif, &ctx); +} + +static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr) +{ + struct ionic_admin_ctx ctx = { + .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work), + .cmd.lif_getattr = { + .opcode = IONIC_CMD_LIF_GETATTR, + .index = cpu_to_le16(lif->index), + .attr = IONIC_LIF_ATTR_MAC, + }, + }; + int err; + + err = ionic_adminq_post_wait(lif, &ctx); + if (err) + return err; + + ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac); + return 0; +} + +static int ionic_program_mac(struct ionic_lif *lif, u8 *mac) +{ + u8 get_mac[ETH_ALEN]; + int err; + + err = ionic_set_attr_mac(lif, mac); + if (err) + return err; + + err = ionic_get_attr_mac(lif, get_mac); + if (err) + return err; + + /* To deal with older firmware that silently ignores the set attr mac: + * doesn't actually change the mac and doesn't return an error, so we + * do the get attr to verify whether or not the set actually happened + */ + if (!ether_addr_equal(get_mac, mac)) + return 1; + + return 0; +} + static int ionic_set_mac_address(struct net_device *netdev, void *sa) { + struct ionic_lif *lif = netdev_priv(netdev); struct sockaddr *addr = sa; u8 *mac; int err; @@ -1574,6 +1630,14 @@ static int ionic_set_mac_address(struct net_device *netdev, void *sa) if (ether_addr_equal(netdev->dev_addr, mac)) return 0; + err = ionic_program_mac(lif, mac); + if (err < 0) + return err; + + if (err > 0) + netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n", + __func__); + err = eth_prepare_mac_addr_change(netdev, addr); if (err) return err; @@ -2963,6 +3027,9 @@ static void ionic_lif_handle_fw_up(struct ionic_lif *lif) mutex_lock(&lif->queue_lock); + if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state)) + dev_info(ionic->dev, "FW Up: clearing broken state\n"); + err = ionic_qcqs_alloc(lif); if (err) goto err_unlock; @@ -3095,8 +3162,7 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif) dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type); dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index); - netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi, - NAPI_POLL_WEIGHT); + netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi); napi_enable(&qcq->napi); @@ -3169,6 +3235,7 @@ static int ionic_station_set(struct ionic_lif *lif) .attr = IONIC_LIF_ATTR_MAC, }, }; + u8 mac_address[ETH_ALEN]; struct sockaddr addr; int err; @@ -3177,8 +3244,23 @@ static int ionic_station_set(struct ionic_lif *lif) return err; netdev_dbg(lif->netdev, "found initial MAC addr %pM\n", ctx.comp.lif_getattr.mac); - if (is_zero_ether_addr(ctx.comp.lif_getattr.mac)) - return 0; + ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac); + + if (is_zero_ether_addr(mac_address)) { + eth_hw_addr_random(netdev); + netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr); + ether_addr_copy(mac_address, netdev->dev_addr); + + err = ionic_program_mac(lif, mac_address); + if (err < 0) + return err; + + if (err > 0) { + netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n", + __func__); + return 0; + } + } if (!is_zero_ether_addr(netdev->dev_addr)) { /* If the netdev mac is non-zero and doesn't match the default @@ -3186,12 +3268,11 @@ static int ionic_station_set(struct ionic_lif *lif) * likely here again after a fw-upgrade reset. We need to be * sure the netdev mac is in our filter list. */ - if (!ether_addr_equal(ctx.comp.lif_getattr.mac, - netdev->dev_addr)) + if (!ether_addr_equal(mac_address, netdev->dev_addr)) ionic_lif_addr_add(lif, netdev->dev_addr); } else { /* Update the netdev mac with the device's mac */ - memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len); + ether_addr_copy(addr.sa_data, mac_address); addr.sa_family = AF_INET; err = eth_prepare_mac_addr_change(netdev, &addr); if (err) { diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c index 4029b4e021f868bab3b45743294cd004640e13f3..56f93b030551909339e6027153326d67e9104199 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_main.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c @@ -474,8 +474,8 @@ try_again: ionic_opcode_to_str(opcode), opcode, ionic_error_to_str(err), err); - msleep(1000); iowrite32(0, &idev->dev_cmd_regs->done); + msleep(1000); iowrite32(1, &idev->dev_cmd_regs->doorbell); goto try_again; } @@ -488,6 +488,8 @@ try_again: return ionic_error_to_errno(err); } + ionic_dev_cmd_clean(ionic); + return 0; } diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c index 3c4a84ea632117b71d8697af65cf701a050c53fb..8c4cb910e09b160f4dbcc8de2b8cd36b40786442 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c @@ -65,9 +65,9 @@ netxen_nic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) u32 fw_minor = 0; u32 fw_build = 0; - strlcpy(drvinfo->driver, netxen_nic_driver_name, + strscpy(drvinfo->driver, netxen_nic_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, NETXEN_NIC_LINUX_VERSIONID, + strscpy(drvinfo->version, NETXEN_NIC_LINUX_VERSIONID, sizeof(drvinfo->version)); fw_major = NXRD32(adapter, NETXEN_FW_VERSION_MAJOR); fw_minor = NXRD32(adapter, NETXEN_FW_VERSION_MINOR); @@ -75,7 +75,7 @@ netxen_nic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", fw_major, fw_minor, fw_build); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c index 4e6f00af17d92ed2fcac1c8df5d90cb3c55219a6..de8d54b23f7385238196c684fa030c98756ff933 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c @@ -173,8 +173,7 @@ netxen_napi_add(struct netxen_adapter *adapter, struct net_device *netdev) for (ring = 0; ring < adapter->max_sds_rings; ring++) { sds_ring = &recv_ctx->sds_rings[ring]; - netif_napi_add(netdev, &sds_ring->napi, - netxen_nic_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &sds_ring->napi, netxen_nic_poll); } return 0; diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c index d701ecd3ba003d572cd86df3e5b1800ace96adbd..2661c483c67ef150bbc6b0b8058bdb3684bdd1a5 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.c +++ b/drivers/net/ethernet/qlogic/qed/qed_int.c @@ -1119,7 +1119,7 @@ static int qed_int_deassertion(struct qed_hwfn *p_hwfn, snprintf(bit_name, 30, p_aeu->bit_name, num); else - strlcpy(bit_name, + strscpy(bit_name, p_aeu->bit_name, 30); /* We now need to pass bitmask in its diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 97a7ab0826eded1caf092c5de9312b5788888460..8034d812d5a001dbb26eb5fd49ff433e3311c288 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -624,7 +624,7 @@ static void qede_get_drvinfo(struct net_device *ndev, struct qede_dev *edev = netdev_priv(ndev); char mbi[ETHTOOL_FWVERS_LEN]; - strlcpy(info->driver, "qede", sizeof(info->driver)); + strscpy(info->driver, "qede", sizeof(info->driver)); snprintf(storm, ETHTOOL_FWVERS_LEN, "%d.%d.%d.%d", edev->dev_info.common.fw_major, @@ -661,7 +661,7 @@ static void qede_get_drvinfo(struct net_device *ndev, "mfw %s", mfw); } - strlcpy(info->bus_info, pci_name(edev->pdev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(edev->pdev), sizeof(info->bus_info)); } static void qede_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index f56b679adb4b6c7bcdde3f398005af0ada6bbead..953f304b8588c58c35eda919b385621ee2951ba1 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -1214,7 +1214,7 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level, /* Start the Slowpath-process */ memset(&sp_params, 0, sizeof(sp_params)); sp_params.int_mode = QED_INT_MODE_MSIX; - strlcpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE); + strscpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE); rc = qed_ops->common->slowpath_start(cdev, &sp_params); if (rc) { pr_notice("Cannot start slowpath\n"); @@ -1904,8 +1904,7 @@ static void qede_napi_add_enable(struct qede_dev *edev) /* Add NAPI objects */ for_each_queue(i) { - netif_napi_add(edev->ndev, &edev->fp_array[i].napi, - qede_poll, NAPI_POLL_WEIGHT); + netif_napi_add(edev->ndev, &edev->fp_array[i].napi, qede_poll); napi_enable(&edev->fp_array[i].napi); } } diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c index 06f4d9a9e9388634ca3d08e3911efa4cd09e4d25..76072f8c3d2fbcdbecd3a35aee734cc2cc9b26ff 100644 --- a/drivers/net/ethernet/qlogic/qla3xxx.c +++ b/drivers/net/ethernet/qlogic/qla3xxx.c @@ -1736,10 +1736,10 @@ static void ql_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *drvinfo) { struct ql3_adapter *qdev = netdev_priv(ndev); - strlcpy(drvinfo->driver, ql3xxx_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, ql3xxx_driver_version, + strscpy(drvinfo->driver, ql3xxx_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, ql3xxx_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, pci_name(qdev->pdev), + strscpy(drvinfo->bus_info, pci_name(qdev->pdev), sizeof(drvinfo->bus_info)); } @@ -3813,7 +3813,7 @@ static int ql3xxx_probe(struct pci_dev *pdev, ndev->ethtool_ops = &ql3xxx_ethtool_ops; ndev->watchdog_timeo = 5 * HZ; - netif_napi_add(ndev, &qdev->napi, ql_poll, 64); + netif_napi_add(ndev, &qdev->napi, ql_poll); ndev->irq = pdev->irq; diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c index 54a2d653be633e3dcf867f53a116de0939bc566e..1ee491f78c6b2767d6e073f4ba21b779c8b37337 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c @@ -277,10 +277,10 @@ qlcnic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d", fw_major, fw_minor, fw_build); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); - strlcpy(drvinfo->driver, qlcnic_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, QLCNIC_LINUX_VERSIONID, + strscpy(drvinfo->driver, qlcnic_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, QLCNIC_LINUX_VERSIONID, sizeof(drvinfo->version)); } diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c index 9da5e97f8a0ada4a1f38f3255ec024665f288173..92930a055cbcc84a9f60e6acb3903d99f046c171 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c @@ -1586,17 +1586,15 @@ int qlcnic_82xx_napi_add(struct qlcnic_adapter *adapter, sds_ring = &recv_ctx->sds_rings[ring]; if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) { - netif_napi_add(netdev, &sds_ring->napi, qlcnic_rx_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_rx_poll); } else { if (ring == (adapter->drv_sds_rings - 1)) netif_napi_add(netdev, &sds_ring->napi, - qlcnic_poll, - NAPI_POLL_WEIGHT); + qlcnic_poll); else netif_napi_add(netdev, &sds_ring->napi, - qlcnic_rx_poll, - NAPI_POLL_WEIGHT); + qlcnic_rx_poll); } } @@ -2115,17 +2113,14 @@ int qlcnic_83xx_napi_add(struct qlcnic_adapter *adapter, if (adapter->flags & QLCNIC_MSIX_ENABLED) { if (!(adapter->flags & QLCNIC_TX_INTR_SHARED)) netif_napi_add(netdev, &sds_ring->napi, - qlcnic_83xx_rx_poll, - NAPI_POLL_WEIGHT); + qlcnic_83xx_rx_poll); else netif_napi_add(netdev, &sds_ring->napi, - qlcnic_83xx_msix_sriov_vf_poll, - NAPI_POLL_WEIGHT); + qlcnic_83xx_msix_sriov_vf_poll); } else { netif_napi_add(netdev, &sds_ring->napi, - qlcnic_83xx_poll, - NAPI_POLL_WEIGHT); + qlcnic_83xx_poll); } } diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c index a55c52696d49f6a97a8532bd2108896a7609f0e4..3115b2c128980db23bf85851fa9724a5359f0509 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac.c @@ -684,8 +684,7 @@ static int emac_probe(struct platform_device *pdev) /* Initialize queues */ emac_mac_rx_tx_ring_init_all(pdev, adpt); - netif_napi_add(netdev, &adpt->rx_q.napi, emac_napi_rtx, - NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &adpt->rx_q.napi, emac_napi_rtx); ret = register_netdev(netdev); if (ret) { diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c index 792ce9a323cded79090c7fafec4c370b75ae0e44..f62c39544e08601b62f3056e89ad4587b1c913a1 100644 --- a/drivers/net/ethernet/qualcomm/qca_debug.c +++ b/drivers/net/ethernet/qualcomm/qca_debug.c @@ -164,10 +164,10 @@ qcaspi_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *p) { struct qcaspi *qca = netdev_priv(dev); - strlcpy(p->driver, QCASPI_DRV_NAME, sizeof(p->driver)); - strlcpy(p->version, QCASPI_DRV_VERSION, sizeof(p->version)); - strlcpy(p->fw_version, "QCA7000", sizeof(p->fw_version)); - strlcpy(p->bus_info, dev_name(&qca->spi_dev->dev), + strscpy(p->driver, QCASPI_DRV_NAME, sizeof(p->driver)); + strscpy(p->version, QCASPI_DRV_VERSION, sizeof(p->version)); + strscpy(p->fw_version, "QCA7000", sizeof(p->fw_version)); + strscpy(p->bus_info, dev_name(&qca->spi_dev->dev), sizeof(p->bus_info)); } diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h index e5a0b38f7dbeada39928655885b52d66069401af..2b033060fc2059f1f1417f7e820bd10fc78b6865 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h @@ -19,7 +19,7 @@ struct rmnet_map_control_command { __be16 flow_control_seq_num; __be32 qos_id; } flow_control; - u8 data[0]; + DECLARE_FLEX_ARRAY(u8, data); }; } __aligned(1); diff --git a/drivers/net/ethernet/rdc/r6040.c b/drivers/net/ethernet/rdc/r6040.c index a6bf7d50517862859c9d34fa39d08ffebf30a361..eecd52ed1ed21869c073ed74db8e192866e026cf 100644 --- a/drivers/net/ethernet/rdc/r6040.c +++ b/drivers/net/ethernet/rdc/r6040.c @@ -939,9 +939,9 @@ static void netdev_get_drvinfo(struct net_device *dev, { struct r6040_private *rp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(rp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(rp->pdev), sizeof(info->bus_info)); } static const struct ethtool_ops netdev_ethtool_ops = { @@ -1127,7 +1127,7 @@ static int r6040_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->ethtool_ops = &netdev_ethtool_ops; dev->watchdog_timeo = TX_TIMEOUT; - netif_napi_add(dev, &lp->napi, r6040_poll, 64); + netif_napi_add(dev, &lp->napi, r6040_poll); lp->mii_bus = mdiobus_alloc(); if (!lp->mii_bus) { diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index e0feeec13da686be419edc3fa3e7309c7d44fcf1..f5786d78ed2306a735e78f8a2c925d2170e93232 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -1382,9 +1382,9 @@ static void cp_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *info { struct cp_private *cp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info)); } static void cp_get_ringparam(struct net_device *dev, diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c index 15b40fd93cd2e75e20783a7aa26a1459e603aa4b..469e2e229c6e7e2781bb09b4b45fece809c76c51 100644 --- a/drivers/net/ethernet/realtek/8139too.c +++ b/drivers/net/ethernet/realtek/8139too.c @@ -1002,7 +1002,7 @@ static int rtl8139_init_one(struct pci_dev *pdev, dev->netdev_ops = &rtl8139_netdev_ops; dev->ethtool_ops = &rtl8139_ethtool_ops; dev->watchdog_timeo = TX_TIMEOUT; - netif_napi_add(dev, &tp->napi, rtl8139_poll, 64); + netif_napi_add(dev, &tp->napi, rtl8139_poll); /* note: the hardware is not capable of sg/csum/highdma, however * through the use of skb_copy_and_csum_dev we enable these @@ -2380,9 +2380,9 @@ static int rtl8139_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) static void rtl8139_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct rtl8139_private *tp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); } static int rtl8139_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/realtek/r8169.h b/drivers/net/ethernet/realtek/r8169.h index 8da4b66b71b5de25a488f6c889302bd7088969c6..55ef8251feb585cd4f76c3d2cff9abc62416a4bb 100644 --- a/drivers/net/ethernet/realtek/r8169.h +++ b/drivers/net/ethernet/realtek/r8169.h @@ -23,10 +23,10 @@ enum mac_version { RTL_GIGA_MAC_VER_09, RTL_GIGA_MAC_VER_10, RTL_GIGA_MAC_VER_11, - RTL_GIGA_MAC_VER_12, - RTL_GIGA_MAC_VER_13, + /* RTL_GIGA_MAC_VER_12 was handled the same as VER_17 */ + /* RTL_GIGA_MAC_VER_13 was merged with VER_10 */ RTL_GIGA_MAC_VER_14, - RTL_GIGA_MAC_VER_16, + /* RTL_GIGA_MAC_VER_16 was merged with VER_10 */ RTL_GIGA_MAC_VER_17, RTL_GIGA_MAC_VER_18, RTL_GIGA_MAC_VER_19, @@ -51,20 +51,20 @@ enum mac_version { RTL_GIGA_MAC_VER_38, RTL_GIGA_MAC_VER_39, RTL_GIGA_MAC_VER_40, - RTL_GIGA_MAC_VER_41, + /* support for RTL_GIGA_MAC_VER_41 has been removed */ RTL_GIGA_MAC_VER_42, RTL_GIGA_MAC_VER_43, RTL_GIGA_MAC_VER_44, - RTL_GIGA_MAC_VER_45, + /* support for RTL_GIGA_MAC_VER_45 has been removed */ RTL_GIGA_MAC_VER_46, - RTL_GIGA_MAC_VER_47, + /* support for RTL_GIGA_MAC_VER_47 has been removed */ RTL_GIGA_MAC_VER_48, - RTL_GIGA_MAC_VER_49, - RTL_GIGA_MAC_VER_50, + /* support for RTL_GIGA_MAC_VER_49 has been removed */ + /* support for RTL_GIGA_MAC_VER_50 has been removed */ RTL_GIGA_MAC_VER_51, RTL_GIGA_MAC_VER_52, RTL_GIGA_MAC_VER_53, - RTL_GIGA_MAC_VER_60, + /* support for RTL_GIGA_MAC_VER_60 has been removed */ RTL_GIGA_MAC_VER_61, RTL_GIGA_MAC_VER_63, RTL_GIGA_MAC_NONE diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 1b7fdb4f056bdabb918637b0dcf644d9571e6aa5..a73d061d9fcb18df928ba4d26248c16c2f29c17b 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -49,10 +49,8 @@ #define FIRMWARE_8106E_2 "rtl_nic/rtl8106e-2.fw" #define FIRMWARE_8168G_2 "rtl_nic/rtl8168g-2.fw" #define FIRMWARE_8168G_3 "rtl_nic/rtl8168g-3.fw" -#define FIRMWARE_8168H_1 "rtl_nic/rtl8168h-1.fw" #define FIRMWARE_8168H_2 "rtl_nic/rtl8168h-2.fw" #define FIRMWARE_8168FP_3 "rtl_nic/rtl8168fp-3.fw" -#define FIRMWARE_8107E_1 "rtl_nic/rtl8107e-1.fw" #define FIRMWARE_8107E_2 "rtl_nic/rtl8107e-2.fw" #define FIRMWARE_8125A_3 "rtl_nic/rtl8125a-3.fw" #define FIRMWARE_8125B_2 "rtl_nic/rtl8125b-2.fw" @@ -102,12 +100,9 @@ static const struct { [RTL_GIGA_MAC_VER_07] = {"RTL8102e" }, [RTL_GIGA_MAC_VER_08] = {"RTL8102e" }, [RTL_GIGA_MAC_VER_09] = {"RTL8102e/RTL8103e" }, - [RTL_GIGA_MAC_VER_10] = {"RTL8101e" }, + [RTL_GIGA_MAC_VER_10] = {"RTL8101e/RTL8100e" }, [RTL_GIGA_MAC_VER_11] = {"RTL8168b/8111b" }, - [RTL_GIGA_MAC_VER_12] = {"RTL8168b/8111b" }, - [RTL_GIGA_MAC_VER_13] = {"RTL8101e/RTL8100e" }, [RTL_GIGA_MAC_VER_14] = {"RTL8401" }, - [RTL_GIGA_MAC_VER_16] = {"RTL8101e" }, [RTL_GIGA_MAC_VER_17] = {"RTL8168b/8111b" }, [RTL_GIGA_MAC_VER_18] = {"RTL8168cp/8111cp" }, [RTL_GIGA_MAC_VER_19] = {"RTL8168c/8111c" }, @@ -131,20 +126,14 @@ static const struct { [RTL_GIGA_MAC_VER_38] = {"RTL8411", FIRMWARE_8411_1 }, [RTL_GIGA_MAC_VER_39] = {"RTL8106e", FIRMWARE_8106E_1}, [RTL_GIGA_MAC_VER_40] = {"RTL8168g/8111g", FIRMWARE_8168G_2}, - [RTL_GIGA_MAC_VER_41] = {"RTL8168g/8111g" }, [RTL_GIGA_MAC_VER_42] = {"RTL8168gu/8111gu", FIRMWARE_8168G_3}, [RTL_GIGA_MAC_VER_43] = {"RTL8106eus", FIRMWARE_8106E_2}, [RTL_GIGA_MAC_VER_44] = {"RTL8411b", FIRMWARE_8411_2 }, - [RTL_GIGA_MAC_VER_45] = {"RTL8168h/8111h", FIRMWARE_8168H_1}, [RTL_GIGA_MAC_VER_46] = {"RTL8168h/8111h", FIRMWARE_8168H_2}, - [RTL_GIGA_MAC_VER_47] = {"RTL8107e", FIRMWARE_8107E_1}, [RTL_GIGA_MAC_VER_48] = {"RTL8107e", FIRMWARE_8107E_2}, - [RTL_GIGA_MAC_VER_49] = {"RTL8168ep/8111ep" }, - [RTL_GIGA_MAC_VER_50] = {"RTL8168ep/8111ep" }, [RTL_GIGA_MAC_VER_51] = {"RTL8168ep/8111ep" }, [RTL_GIGA_MAC_VER_52] = {"RTL8168fp/RTL8117", FIRMWARE_8168FP_3}, [RTL_GIGA_MAC_VER_53] = {"RTL8168fp/RTL8117", }, - [RTL_GIGA_MAC_VER_60] = {"RTL8125A" }, [RTL_GIGA_MAC_VER_61] = {"RTL8125A", FIRMWARE_8125A_3}, /* reserve 62 for CFG_METHOD_4 in the vendor driver */ [RTL_GIGA_MAC_VER_63] = {"RTL8125B", FIRMWARE_8125B_2}, @@ -658,10 +647,8 @@ MODULE_FIRMWARE(FIRMWARE_8106E_1); MODULE_FIRMWARE(FIRMWARE_8106E_2); MODULE_FIRMWARE(FIRMWARE_8168G_2); MODULE_FIRMWARE(FIRMWARE_8168G_3); -MODULE_FIRMWARE(FIRMWARE_8168H_1); MODULE_FIRMWARE(FIRMWARE_8168H_2); MODULE_FIRMWARE(FIRMWARE_8168FP_3); -MODULE_FIRMWARE(FIRMWARE_8107E_1); MODULE_FIRMWARE(FIRMWARE_8107E_2); MODULE_FIRMWARE(FIRMWARE_8125A_3); MODULE_FIRMWARE(FIRMWARE_8125B_2); @@ -689,7 +676,7 @@ static void rtl_pci_commit(struct rtl8169_private *tp) static bool rtl_is_8125(struct rtl8169_private *tp) { - return tp->mac_version >= RTL_GIGA_MAC_VER_60; + return tp->mac_version >= RTL_GIGA_MAC_VER_61; } static bool rtl_is_8168evl_up(struct rtl8169_private *tp) @@ -892,8 +879,6 @@ static void rtl8168g_phy_suspend_quirk(struct rtl8169_private *tp, int value) { switch (tp->mac_version) { case RTL_GIGA_MAC_VER_40: - case RTL_GIGA_MAC_VER_41: - case RTL_GIGA_MAC_VER_49: if (value & BMCR_RESET || !(value & BMCR_PDOWN)) rtl_eri_set_bits(tp, 0x1a8, 0xfc000000); else @@ -1207,7 +1192,7 @@ static enum rtl_dash_type rtl_check_dash(struct rtl8169_private *tp) case RTL_GIGA_MAC_VER_28: case RTL_GIGA_MAC_VER_31: return r8168dp_check_dash(tp) ? RTL_DASH_DP : RTL_DASH_NONE; - case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53: + case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53: return r8168ep_check_dash(tp) ? RTL_DASH_EP : RTL_DASH_NONE; default: return RTL_DASH_NONE; @@ -1423,11 +1408,11 @@ static void rtl8169_get_drvinfo(struct net_device *dev, struct rtl8169_private *tp = netdev_priv(dev); struct rtl_fw *rtl_fw = tp->rtl_fw; - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version)); if (rtl_fw) - strlcpy(info->fw_version, rtl_fw->version, + strscpy(info->fw_version, rtl_fw->version, sizeof(info->fw_version)); } @@ -2011,7 +1996,10 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) /* 8168F family. */ { 0x7c8, 0x488, RTL_GIGA_MAC_VER_38 }, - { 0x7cf, 0x481, RTL_GIGA_MAC_VER_36 }, + /* It seems this chip version never made it to + * the wild. Let's disable detection. + * { 0x7cf, 0x481, RTL_GIGA_MAC_VER_36 }, + */ { 0x7cf, 0x480, RTL_GIGA_MAC_VER_35 }, /* 8168E family. */ @@ -2041,7 +2029,6 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) { 0x7c8, 0x3c0, RTL_GIGA_MAC_VER_22 }, /* 8168B family. */ - { 0x7cf, 0x380, RTL_GIGA_MAC_VER_12 }, { 0x7c8, 0x380, RTL_GIGA_MAC_VER_17 }, { 0x7c8, 0x300, RTL_GIGA_MAC_VER_11 }, @@ -2054,19 +2041,10 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) { 0x7cf, 0x249, RTL_GIGA_MAC_VER_08 }, { 0x7cf, 0x348, RTL_GIGA_MAC_VER_07 }, { 0x7cf, 0x248, RTL_GIGA_MAC_VER_07 }, - { 0x7cf, 0x340, RTL_GIGA_MAC_VER_13 }, { 0x7cf, 0x240, RTL_GIGA_MAC_VER_14 }, - { 0x7cf, 0x343, RTL_GIGA_MAC_VER_10 }, - { 0x7cf, 0x342, RTL_GIGA_MAC_VER_16 }, { 0x7c8, 0x348, RTL_GIGA_MAC_VER_09 }, { 0x7c8, 0x248, RTL_GIGA_MAC_VER_09 }, - { 0x7c8, 0x340, RTL_GIGA_MAC_VER_16 }, - /* FIXME: where did these entries come from ? -- FR - * Not even r8101 vendor driver knows these id's, - * so let's disable detection for now. -- HK - * { 0xfc8, 0x388, RTL_GIGA_MAC_VER_13 }, - * { 0xfc8, 0x308, RTL_GIGA_MAC_VER_13 }, - */ + { 0x7c8, 0x340, RTL_GIGA_MAC_VER_10 }, /* 8110 family. */ { 0xfc8, 0x980, RTL_GIGA_MAC_VER_06 }, @@ -2088,8 +2066,6 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) if (ver != RTL_GIGA_MAC_NONE && !gmii) { if (ver == RTL_GIGA_MAC_VER_42) ver = RTL_GIGA_MAC_VER_43; - else if (ver == RTL_GIGA_MAC_VER_45) - ver = RTL_GIGA_MAC_VER_47; else if (ver == RTL_GIGA_MAC_VER_46) ver = RTL_GIGA_MAC_VER_48; } @@ -2271,7 +2247,7 @@ static void rtl_init_rxcfg(struct rtl8169_private *tp) case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53: RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF); break; - case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63: + case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63: RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST); break; default: @@ -2338,7 +2314,6 @@ static void rtl_jumbo_config(struct rtl8169_private *tp) rtl_unlock_config_regs(tp); switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_12: case RTL_GIGA_MAC_VER_17: if (jumbo) { readrq = 512; @@ -2455,7 +2430,7 @@ static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp) rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 42); rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42); break; - case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_61: + case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_61: rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42); break; case RTL_GIGA_MAC_VER_63: @@ -2468,6 +2443,11 @@ static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp) } } +static void rtl_disable_rxdvgate(struct rtl8169_private *tp) +{ + RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); +} + static void rtl_enable_rxdvgate(struct rtl8169_private *tp) { RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN); @@ -2700,8 +2680,8 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable) RTL_W8(tp, Config2, RTL_R8(tp, Config2) | ClkReqEn); switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48: - case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63: + case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48: + case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63: /* reset ephy tx/rx disable timer */ r8168_mac_ocp_modify(tp, 0xe094, 0xff00, 0); /* chip can trigger L1.2 */ @@ -2712,8 +2692,8 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable) } } else { switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48: - case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63: + case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48: + case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63: r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, 0); break; default: @@ -2985,7 +2965,7 @@ static void rtl_hw_start_8168g(struct rtl8169_private *tp) rtl_reset_packet_filter(tp); rtl_eri_write(tp, 0x2f8, ERIAR_MASK_0011, 0x1d8f); - RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); + rtl_disable_rxdvgate(tp); rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000); rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000); @@ -3223,7 +3203,7 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp) rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87); - RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); + rtl_disable_rxdvgate(tp); rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000); rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000); @@ -3274,7 +3254,7 @@ static void rtl_hw_start_8168ep(struct rtl8169_private *tp) rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87); - RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); + rtl_disable_rxdvgate(tp); rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000); rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000); @@ -3288,45 +3268,6 @@ static void rtl_hw_start_8168ep(struct rtl8169_private *tp) rtl_pcie_state_l2l3_disable(tp); } -static void rtl_hw_start_8168ep_1(struct rtl8169_private *tp) -{ - static const struct ephy_info e_info_8168ep_1[] = { - { 0x00, 0xffff, 0x10ab }, - { 0x06, 0xffff, 0xf030 }, - { 0x08, 0xffff, 0x2006 }, - { 0x0d, 0xffff, 0x1666 }, - { 0x0c, 0x3ff0, 0x0000 } - }; - - /* disable aspm and clock request before access ephy */ - rtl_hw_aspm_clkreq_enable(tp, false); - rtl_ephy_init(tp, e_info_8168ep_1); - - rtl_hw_start_8168ep(tp); - - rtl_hw_aspm_clkreq_enable(tp, true); -} - -static void rtl_hw_start_8168ep_2(struct rtl8169_private *tp) -{ - static const struct ephy_info e_info_8168ep_2[] = { - { 0x00, 0xffff, 0x10a3 }, - { 0x19, 0xffff, 0xfc00 }, - { 0x1e, 0xffff, 0x20ea } - }; - - /* disable aspm and clock request before access ephy */ - rtl_hw_aspm_clkreq_enable(tp, false); - rtl_ephy_init(tp, e_info_8168ep_2); - - rtl_hw_start_8168ep(tp); - - RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN); - RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN); - - rtl_hw_aspm_clkreq_enable(tp, true); -} - static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp) { static const struct ephy_info e_info_8168ep_3[] = { @@ -3377,7 +3318,7 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp) rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87); - RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); + rtl_disable_rxdvgate(tp); rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000); rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000); @@ -3621,48 +3562,7 @@ static void rtl_hw_start_8125_common(struct rtl8169_private *tp) else rtl8125a_config_eee_mac(tp); - RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN); - udelay(10); -} - -static void rtl_hw_start_8125a_1(struct rtl8169_private *tp) -{ - static const struct ephy_info e_info_8125a_1[] = { - { 0x01, 0xffff, 0xa812 }, - { 0x09, 0xffff, 0x520c }, - { 0x04, 0xffff, 0xd000 }, - { 0x0d, 0xffff, 0xf702 }, - { 0x0a, 0xffff, 0x8653 }, - { 0x06, 0xffff, 0x001e }, - { 0x08, 0xffff, 0x3595 }, - { 0x20, 0xffff, 0x9455 }, - { 0x21, 0xffff, 0x99ff }, - { 0x02, 0xffff, 0x6046 }, - { 0x29, 0xffff, 0xfe00 }, - { 0x23, 0xffff, 0xab62 }, - - { 0x41, 0xffff, 0xa80c }, - { 0x49, 0xffff, 0x520c }, - { 0x44, 0xffff, 0xd000 }, - { 0x4d, 0xffff, 0xf702 }, - { 0x4a, 0xffff, 0x8653 }, - { 0x46, 0xffff, 0x001e }, - { 0x48, 0xffff, 0x3595 }, - { 0x60, 0xffff, 0x9455 }, - { 0x61, 0xffff, 0x99ff }, - { 0x42, 0xffff, 0x6046 }, - { 0x69, 0xffff, 0xfe00 }, - { 0x63, 0xffff, 0xab62 }, - }; - - rtl_set_def_aspm_entry_latency(tp); - - /* disable aspm and clock request before access ephy */ - rtl_hw_aspm_clkreq_enable(tp, false); - rtl_ephy_init(tp, e_info_8125a_1); - - rtl_hw_start_8125_common(tp); - rtl_hw_aspm_clkreq_enable(tp, true); + rtl_disable_rxdvgate(tp); } static void rtl_hw_start_8125a_2(struct rtl8169_private *tp) @@ -3721,10 +3621,7 @@ static void rtl_hw_config(struct rtl8169_private *tp) [RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2, [RTL_GIGA_MAC_VER_10] = NULL, [RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b, - [RTL_GIGA_MAC_VER_12] = rtl_hw_start_8168b, - [RTL_GIGA_MAC_VER_13] = NULL, [RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401, - [RTL_GIGA_MAC_VER_16] = NULL, [RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b, [RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1, [RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1, @@ -3748,20 +3645,14 @@ static void rtl_hw_config(struct rtl8169_private *tp) [RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411, [RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106, [RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1, - [RTL_GIGA_MAC_VER_41] = rtl_hw_start_8168g_1, [RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2, [RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2, [RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2, - [RTL_GIGA_MAC_VER_45] = rtl_hw_start_8168h_1, [RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1, - [RTL_GIGA_MAC_VER_47] = rtl_hw_start_8168h_1, [RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1, - [RTL_GIGA_MAC_VER_49] = rtl_hw_start_8168ep_1, - [RTL_GIGA_MAC_VER_50] = rtl_hw_start_8168ep_2, [RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3, [RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117, [RTL_GIGA_MAC_VER_53] = rtl_hw_start_8117, - [RTL_GIGA_MAC_VER_60] = rtl_hw_start_8125a_1, [RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2, [RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b, }; @@ -4156,7 +4047,6 @@ static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp, switch (tp->mac_version) { case RTL_GIGA_MAC_VER_34: - case RTL_GIGA_MAC_VER_60: case RTL_GIGA_MAC_VER_61: case RTL_GIGA_MAC_VER_63: padto = max_t(unsigned int, padto, ETH_ZLEN); @@ -4677,8 +4567,7 @@ static void r8169_phylink_handler(struct net_device *ndev) pm_runtime_idle(&tp->pci_dev->dev); } - if (net_ratelimit()) - phy_print_status(tp->phydev); + phy_print_status(tp->phydev); } static int r8169_phy_connect(struct rtl8169_private *tp) @@ -4954,23 +4843,6 @@ static const struct dev_pm_ops rtl8169_pm_ops = { rtl8169_runtime_idle) }; -static void rtl_wol_shutdown_quirk(struct rtl8169_private *tp) -{ - /* WoL fails with 8168b when the receiver is disabled. */ - switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_11: - case RTL_GIGA_MAC_VER_12: - case RTL_GIGA_MAC_VER_17: - pci_clear_master(tp->pci_dev); - - RTL_W8(tp, ChipCmd, CmdRxEnb); - rtl_pci_commit(tp); - break; - default: - break; - } -} - static void rtl_shutdown(struct pci_dev *pdev) { struct rtl8169_private *tp = pci_get_drvdata(pdev); @@ -4984,9 +4856,6 @@ static void rtl_shutdown(struct pci_dev *pdev) if (system_state == SYSTEM_POWER_OFF && tp->dash_type == RTL_DASH_NONE) { - if (tp->saved_wolopts) - rtl_wol_shutdown_quirk(tp); - pci_wake_from_d3(pdev, tp->saved_wolopts); pci_set_power_state(pdev, PCI_D3hot); } @@ -5194,13 +5063,13 @@ static void rtl_hw_init_8125(struct rtl8169_private *tp) static void rtl_hw_initialize(struct rtl8169_private *tp) { switch (tp->mac_version) { - case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53: + case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53: rtl8168ep_stop_cmac(tp); fallthrough; case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48: rtl_hw_init_8168g(tp); break; - case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63: + case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63: rtl_hw_init_8125(tp); break; default: @@ -5220,7 +5089,6 @@ static int rtl_jumbo_max(struct rtl8169_private *tp) return JUMBO_7K; /* RTL8168b */ case RTL_GIGA_MAC_VER_11: - case RTL_GIGA_MAC_VER_12: case RTL_GIGA_MAC_VER_17: return JUMBO_4K; /* RTL8168c */ @@ -5231,37 +5099,6 @@ static int rtl_jumbo_max(struct rtl8169_private *tp) } } -static void rtl_disable_clk(void *data) -{ - clk_disable_unprepare(data); -} - -static int rtl_get_ether_clk(struct rtl8169_private *tp) -{ - struct device *d = tp_to_dev(tp); - struct clk *clk; - int rc; - - clk = devm_clk_get(d, "ether_clk"); - if (IS_ERR(clk)) { - rc = PTR_ERR(clk); - if (rc == -ENOENT) - /* clk-core allows NULL (for suspend / resume) */ - rc = 0; - else - dev_err_probe(d, rc, "failed to get clk\n"); - } else { - tp->clk = clk; - rc = clk_prepare_enable(clk); - if (rc) - dev_err(d, "failed to enable clk: %d\n", rc); - else - rc = devm_add_action_or_reset(d, rtl_disable_clk, clk); - } - - return rc; -} - static void rtl_init_mac_address(struct rtl8169_private *tp) { u8 mac_addr[ETH_ALEN] __aligned(2) = {}; @@ -5291,7 +5128,7 @@ done: /* register is set if system vendor successfully tested ASPM 1.2 */ static bool rtl_aspm_is_safe(struct rtl8169_private *tp) { - if (tp->mac_version >= RTL_GIGA_MAC_VER_60 && + if (tp->mac_version >= RTL_GIGA_MAC_VER_61 && r8168_mac_ocp_read(tp, 0xc0b2) & 0xf) return true; @@ -5325,9 +5162,9 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; /* Get the *optional* external "ether_clk" used on some boards */ - rc = rtl_get_ether_clk(tp); - if (rc) - return rc; + tp->clk = devm_clk_get_optional_enabled(&pdev->dev, "ether_clk"); + if (IS_ERR(tp->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(tp->clk), "failed to get ether_clk\n"); /* enable device (incl. PCI PM wakeup and hotplug setup) */ rc = pcim_enable_device(pdev); @@ -5346,12 +5183,6 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENODEV; } - /* check for weird/broken PCI region reporting */ - if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) { - dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n"); - return -ENODEV; - } - rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME); if (rc < 0) { dev_err(&pdev->dev, "cannot remap MMIO, aborting\n"); @@ -5378,7 +5209,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (rtl_aspm_is_safe(tp)) rc = 0; - else if (tp->mac_version >= RTL_GIGA_MAC_VER_45) + else if (tp->mac_version >= RTL_GIGA_MAC_VER_46) rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); else rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1); @@ -5413,7 +5244,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->ethtool_ops = &rtl8169_ethtool_ops; - netif_napi_add(dev, &tp->napi, rtl8169_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &tp->napi, rtl8169_poll); dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; diff --git a/drivers/net/ethernet/realtek/r8169_phy_config.c b/drivers/net/ethernet/realtek/r8169_phy_config.c index 15c295f9019670619004339cf76b6d9d5a0ebcf6..930496cd34ed00c61d4b37d11b9008d777fe0609 100644 --- a/drivers/net/ethernet/realtek/r8169_phy_config.c +++ b/drivers/net/ethernet/realtek/r8169_phy_config.c @@ -793,71 +793,6 @@ static void rtl8168g_2_hw_phy_config(struct rtl8169_private *tp, rtl8168g_config_eee_phy(phydev); } -static void rtl8168h_1_hw_phy_config(struct rtl8169_private *tp, - struct phy_device *phydev) -{ - u16 dout_tapbin; - u32 data; - - r8169_apply_firmware(tp); - - /* CHN EST parameters adjust - giga master */ - r8168g_phy_param(phydev, 0x809b, 0xf800, 0x8000); - r8168g_phy_param(phydev, 0x80a2, 0xff00, 0x8000); - r8168g_phy_param(phydev, 0x80a4, 0xff00, 0x8500); - r8168g_phy_param(phydev, 0x809c, 0xff00, 0xbd00); - - /* CHN EST parameters adjust - giga slave */ - r8168g_phy_param(phydev, 0x80ad, 0xf800, 0x7000); - r8168g_phy_param(phydev, 0x80b4, 0xff00, 0x5000); - r8168g_phy_param(phydev, 0x80ac, 0xff00, 0x4000); - - /* CHN EST parameters adjust - fnet */ - r8168g_phy_param(phydev, 0x808e, 0xff00, 0x1200); - r8168g_phy_param(phydev, 0x8090, 0xff00, 0xe500); - r8168g_phy_param(phydev, 0x8092, 0xff00, 0x9f00); - - /* enable R-tune & PGA-retune function */ - dout_tapbin = 0; - data = phy_read_paged(phydev, 0x0a46, 0x13); - data &= 3; - data <<= 2; - dout_tapbin |= data; - data = phy_read_paged(phydev, 0x0a46, 0x12); - data &= 0xc000; - data >>= 14; - dout_tapbin |= data; - dout_tapbin = ~(dout_tapbin ^ 0x08); - dout_tapbin <<= 12; - dout_tapbin &= 0xf000; - - r8168g_phy_param(phydev, 0x827a, 0xf000, dout_tapbin); - r8168g_phy_param(phydev, 0x827b, 0xf000, dout_tapbin); - r8168g_phy_param(phydev, 0x827c, 0xf000, dout_tapbin); - r8168g_phy_param(phydev, 0x827d, 0xf000, dout_tapbin); - r8168g_phy_param(phydev, 0x0811, 0x0000, 0x0800); - phy_modify_paged(phydev, 0x0a42, 0x16, 0x0000, 0x0002); - - rtl8168g_enable_gphy_10m(phydev); - - /* SAR ADC performance */ - phy_modify_paged(phydev, 0x0bca, 0x17, BIT(12) | BIT(13), BIT(14)); - - r8168g_phy_param(phydev, 0x803f, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x8047, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x804f, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x8057, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x805f, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x8067, 0x3000, 0x0000); - r8168g_phy_param(phydev, 0x806f, 0x3000, 0x0000); - - /* disable phy pfm mode */ - phy_modify_paged(phydev, 0x0a44, 0x11, BIT(7), 0); - - rtl8168g_disable_aldps(phydev); - rtl8168h_config_eee_phy(phydev); -} - static void rtl8168h_2_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev) { @@ -895,27 +830,6 @@ static void rtl8168h_2_hw_phy_config(struct rtl8169_private *tp, rtl8168g_config_eee_phy(phydev); } -static void rtl8168ep_1_hw_phy_config(struct rtl8169_private *tp, - struct phy_device *phydev) -{ - /* Enable PHY auto speed down */ - phy_modify_paged(phydev, 0x0a44, 0x11, 0, BIT(3) | BIT(2)); - - rtl8168g_phy_adjust_10m_aldps(phydev); - - /* Enable EEE auto-fallback function */ - phy_modify_paged(phydev, 0x0a4b, 0x11, 0, BIT(2)); - - /* Enable UC LPF tune function */ - r8168g_phy_param(phydev, 0x8012, 0x0000, 0x8000); - - /* set rg_sel_sdm_rate */ - phy_modify_paged(phydev, 0x0c42, 0x11, BIT(13), BIT(14)); - - rtl8168g_disable_aldps(phydev); - rtl8168g_config_eee_phy(phydev); -} - static void rtl8168ep_2_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev) { @@ -1081,44 +995,6 @@ static void rtl8125_legacy_force_mode(struct phy_device *phydev) phy_modify_paged(phydev, 0xa5b, 0x12, BIT(15), 0); } -static void rtl8125a_1_hw_phy_config(struct rtl8169_private *tp, - struct phy_device *phydev) -{ - phy_modify_paged(phydev, 0xad4, 0x10, 0x03ff, 0x0084); - phy_modify_paged(phydev, 0xad4, 0x17, 0x0000, 0x0010); - phy_modify_paged(phydev, 0xad1, 0x13, 0x03ff, 0x0006); - phy_modify_paged(phydev, 0xad3, 0x11, 0x003f, 0x0006); - phy_modify_paged(phydev, 0xac0, 0x14, 0x0000, 0x1100); - phy_modify_paged(phydev, 0xac8, 0x15, 0xf000, 0x7000); - phy_modify_paged(phydev, 0xad1, 0x14, 0x0000, 0x0400); - phy_modify_paged(phydev, 0xad1, 0x15, 0x0000, 0x03ff); - phy_modify_paged(phydev, 0xad1, 0x16, 0x0000, 0x03ff); - - r8168g_phy_param(phydev, 0x80ea, 0xff00, 0xc400); - r8168g_phy_param(phydev, 0x80eb, 0x0700, 0x0300); - r8168g_phy_param(phydev, 0x80f8, 0xff00, 0x1c00); - r8168g_phy_param(phydev, 0x80f1, 0xff00, 0x3000); - r8168g_phy_param(phydev, 0x80fe, 0xff00, 0xa500); - r8168g_phy_param(phydev, 0x8102, 0xff00, 0x5000); - r8168g_phy_param(phydev, 0x8105, 0xff00, 0x3300); - r8168g_phy_param(phydev, 0x8100, 0xff00, 0x7000); - r8168g_phy_param(phydev, 0x8104, 0xff00, 0xf000); - r8168g_phy_param(phydev, 0x8106, 0xff00, 0x6500); - r8168g_phy_param(phydev, 0x80dc, 0xff00, 0xed00); - r8168g_phy_param(phydev, 0x80df, 0x0000, 0x0100); - r8168g_phy_param(phydev, 0x80e1, 0x0100, 0x0000); - - phy_modify_paged(phydev, 0xbf0, 0x13, 0x003f, 0x0038); - r8168g_phy_param(phydev, 0x819f, 0xffff, 0xd0b6); - - phy_write_paged(phydev, 0xbc3, 0x12, 0x5555); - phy_modify_paged(phydev, 0xbf0, 0x15, 0x0e00, 0x0a00); - phy_modify_paged(phydev, 0xa5c, 0x10, 0x0400, 0x0000); - rtl8168g_enable_gphy_10m(phydev); - - rtl8125a_config_eee_phy(phydev); -} - static void rtl8125a_2_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev) { @@ -1239,10 +1115,7 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev, [RTL_GIGA_MAC_VER_09] = rtl8102e_hw_phy_config, [RTL_GIGA_MAC_VER_10] = NULL, [RTL_GIGA_MAC_VER_11] = rtl8168bb_hw_phy_config, - [RTL_GIGA_MAC_VER_12] = rtl8168bef_hw_phy_config, - [RTL_GIGA_MAC_VER_13] = NULL, [RTL_GIGA_MAC_VER_14] = rtl8401_hw_phy_config, - [RTL_GIGA_MAC_VER_16] = NULL, [RTL_GIGA_MAC_VER_17] = rtl8168bef_hw_phy_config, [RTL_GIGA_MAC_VER_18] = rtl8168cp_1_hw_phy_config, [RTL_GIGA_MAC_VER_19] = rtl8168c_1_hw_phy_config, @@ -1266,20 +1139,14 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev, [RTL_GIGA_MAC_VER_38] = rtl8411_hw_phy_config, [RTL_GIGA_MAC_VER_39] = rtl8106e_hw_phy_config, [RTL_GIGA_MAC_VER_40] = rtl8168g_1_hw_phy_config, - [RTL_GIGA_MAC_VER_41] = NULL, [RTL_GIGA_MAC_VER_42] = rtl8168g_2_hw_phy_config, [RTL_GIGA_MAC_VER_43] = rtl8168g_2_hw_phy_config, [RTL_GIGA_MAC_VER_44] = rtl8168g_2_hw_phy_config, - [RTL_GIGA_MAC_VER_45] = rtl8168h_1_hw_phy_config, [RTL_GIGA_MAC_VER_46] = rtl8168h_2_hw_phy_config, - [RTL_GIGA_MAC_VER_47] = rtl8168h_1_hw_phy_config, [RTL_GIGA_MAC_VER_48] = rtl8168h_2_hw_phy_config, - [RTL_GIGA_MAC_VER_49] = rtl8168ep_1_hw_phy_config, - [RTL_GIGA_MAC_VER_50] = rtl8168ep_2_hw_phy_config, [RTL_GIGA_MAC_VER_51] = rtl8168ep_2_hw_phy_config, [RTL_GIGA_MAC_VER_52] = rtl8117_hw_phy_config, [RTL_GIGA_MAC_VER_53] = rtl8117_hw_phy_config, - [RTL_GIGA_MAC_VER_60] = rtl8125a_1_hw_phy_config, [RTL_GIGA_MAC_VER_61] = rtl8125a_2_hw_phy_config, [RTL_GIGA_MAC_VER_63] = rtl8125b_hw_phy_config, }; diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h index b980bce763d3c0a0fe32b7493cb6a98ab2863eeb..e0f8276cffeddeb242217f3111e1211115e5ff98 100644 --- a/drivers/net/ethernet/renesas/ravb.h +++ b/drivers/net/ethernet/renesas/ravb.h @@ -189,6 +189,7 @@ enum ravb_reg { PSR = 0x0528, PIPR = 0x052c, CXR31 = 0x0530, /* RZ/G2L only */ + CXR35 = 0x0540, /* RZ/G2L only */ MPR = 0x0558, PFTCR = 0x055c, PFRCR = 0x0560, @@ -965,6 +966,13 @@ enum CXR31_BIT { CXR31_SEL_LINK1 = 0x00000008, }; +enum CXR35_BIT { + CXR35_SEL_XMII = 0x00000003, + CXR35_SEL_XMII_RGMII = 0x00000000, + CXR35_SEL_XMII_MII = 0x00000002, + CXR35_HALFCYC_CLKSW = 0xffff0000, +}; + enum CSR0_BIT { CSR0_TPE = 0x00000010, CSR0_RPE = 0x00000020, diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index b357ac4c56c599ad4a73ad8670228fc9fec77722..36324126db6db5998846031bf2ca0d2da6217ae1 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -540,7 +540,13 @@ static void ravb_emac_init_gbeth(struct net_device *ndev) /* E-MAC interrupt enable register */ ravb_write(ndev, ECSIPR_ICDIP, ECSIPR); - ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, CXR31_SEL_LINK0); + if (priv->phy_interface == PHY_INTERFACE_MODE_MII) { + ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, 0); + ravb_write(ndev, (1000 << 16) | CXR35_SEL_XMII_MII, CXR35); + } else { + ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, + CXR31_SEL_LINK0); + } } static void ravb_emac_init_rcar(struct net_device *ndev) @@ -1449,6 +1455,8 @@ static int ravb_phy_init(struct net_device *ndev) phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); } + /* Indicate that the MAC is responsible for managing PHY PM */ + phydev->mac_managed_pm = true; phy_attached_info(phydev); return 0; @@ -2512,6 +2520,7 @@ static const struct of_device_id ravb_match_table[] = { { .compatible = "renesas,etheravb-rcar-gen2", .data = &ravb_gen2_hw_info }, { .compatible = "renesas,etheravb-r8a7795", .data = &ravb_gen3_hw_info }, { .compatible = "renesas,etheravb-rcar-gen3", .data = &ravb_gen3_hw_info }, + { .compatible = "renesas,etheravb-rcar-gen4", .data = &ravb_gen3_hw_info }, { .compatible = "renesas,etheravb-rzv2m", .data = &ravb_rzv2m_hw_info }, { .compatible = "renesas,rzg2l-gbeth", .data = &gbeth_hw_info }, { } @@ -2832,9 +2841,9 @@ static int ravb_probe(struct platform_device *pdev) goto out_dma_free; } - netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll, 64); + netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll); if (info->nc_queues) - netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll, 64); + netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll); /* Network device register */ error = register_netdev(ndev); diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index 67ade78fb7671c4a8bbaae0e02b067bd351a3beb..71a4991133080fac9bbbbc20437a09025795fc6d 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -2029,6 +2029,8 @@ static int sh_eth_phy_init(struct net_device *ndev) if (mdp->cd->register_type != SH_ETH_REG_GIGABIT) phy_set_max_speed(phydev, SPEED_100); + /* Indicate that the MAC is responsible for managing PHY PM */ + phydev->mac_managed_pm = true; phy_attached_info(phydev); return 0; @@ -3366,7 +3368,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev) goto out_release; } - netif_napi_add(ndev, &mdp->napi, sh_eth_poll, 64); + netif_napi_add(ndev, &mdp->napi, sh_eth_poll); /* network device register */ ret = register_netdev(ndev); diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index fc83ec23bd1d8b3a85c0e6f9166aa771b7e47de7..9e59669a93dd3988ca6c8198995f95a9886d8cf7 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -129,7 +129,7 @@ static int rocker_reg_test(const struct rocker *rocker) u64 test_reg; u64 rnd; - rnd = prandom_u32(); + rnd = get_random_u32(); rnd >>= 1; rocker_write32(rocker, TEST_REG, rnd); test_reg = rocker_read32(rocker, TEST_REG); @@ -139,9 +139,9 @@ static int rocker_reg_test(const struct rocker *rocker) return -EIO; } - rnd = prandom_u32(); + rnd = get_random_u32(); rnd <<= 31; - rnd |= prandom_u32(); + rnd |= get_random_u32(); rocker_write64(rocker, TEST_REG64, rnd); test_reg = rocker_read64(rocker, TEST_REG64); if (test_reg != rnd * 2) { @@ -224,7 +224,7 @@ static int rocker_dma_test_offset(const struct rocker *rocker, if (err) goto unmap; - prandom_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE); + get_random_bytes(buf, ROCKER_TEST_DMA_BUF_SIZE); for (i = 0; i < ROCKER_TEST_DMA_BUF_SIZE; i++) expect[i] = ~buf[i]; err = rocker_dma_test_one(rocker, wait, ROCKER_TEST_DMA_CTRL_INVERT, @@ -2226,8 +2226,8 @@ rocker_port_set_link_ksettings(struct net_device *dev, static void rocker_port_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version)); } static struct rocker_port_stats { @@ -2574,8 +2574,7 @@ static int rocker_probe_port(struct rocker *rocker, unsigned int port_number) dev->netdev_ops = &rocker_port_netdev_ops; dev->ethtool_ops = &rocker_port_ethtool_ops; netif_napi_add_tx(dev, &rocker_port->napi_tx, rocker_port_poll_tx); - netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx); rocker_carrier_init(rocker_port); dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG; diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c index bc70c6abd6a5bfddc72bdec26e2aaee5b5313705..58cf7cc54f4084e1b96b8c1c6f8a17d314bf7541 100644 --- a/drivers/net/ethernet/rocker/rocker_ofdpa.c +++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c @@ -1273,7 +1273,7 @@ static int ofdpa_port_ipv4_neigh(struct ofdpa_port *ofdpa_port, bool removing; int err = 0; - entry = kzalloc(sizeof(*entry), GFP_KERNEL); + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); if (!entry) return -ENOMEM; diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c index 98edb01024f0be6cc791d68dd965966326dbcb63..8ba017ec9849325cc9506cb5a32c1f0858a147d5 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c @@ -175,8 +175,8 @@ static int sxgbe_set_eee(struct net_device *dev, static void sxgbe_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static u32 sxgbe_getmsglevel(struct net_device *dev) diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c index a1c10b61269b0a3eee29b2743b5977b5463635c8..9664f029fa161c08d941500d47fe25957cc72cb5 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c @@ -2143,7 +2143,7 @@ struct sxgbe_priv_data *sxgbe_drv_probe(struct device *device, pr_info("Enable RX Mitigation via HW Watchdog Timer\n"); } - netif_napi_add(ndev, &priv->napi, sxgbe_poll, 64); + netif_napi_add(ndev, &priv->napi, sxgbe_poll); spin_lock_init(&priv->stats_lock); diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile index bb06fa228367e4a75ec40e0027548688a346b807..b5e45fc6337e84eda40beb3b66176ca2f4692fa7 100644 --- a/drivers/net/ethernet/sfc/Makefile +++ b/drivers/net/ethernet/sfc/Makefile @@ -9,7 +9,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \ ef100_ethtool.o ef100_rx.o ef100_tx.o sfc-$(CONFIG_SFC_MTD) += mtd.o sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \ - mae.o tc.o + mae.o tc.o tc_bindings.o obj-$(CONFIG_SFC) += sfc.o diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c index ee734b69150f1b49391ef3a6408801e065e413a3..d1e1aa19a68edb49a589a9041351a484392cf83d 100644 --- a/drivers/net/ethernet/sfc/ef10.c +++ b/drivers/net/ethernet/sfc/ef10.c @@ -4213,7 +4213,7 @@ const struct efx_nic_type efx_hunt_a0_nic_type = { .ev_test_generate = efx_ef10_ev_test_generate, .filter_table_probe = efx_ef10_filter_table_probe, .filter_table_restore = efx_mcdi_filter_table_restore, - .filter_table_remove = efx_mcdi_filter_table_remove, + .filter_table_remove = efx_ef10_filter_table_remove, .filter_update_rx_scatter = efx_mcdi_update_rx_scatter, .filter_insert = efx_mcdi_filter_insert, .filter_remove_safe = efx_mcdi_filter_remove_safe, diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c index 702abbe59b7600551241becfe105f809d0f01df3..135ece2f1375fefffe875745760c3d49e3597028 100644 --- a/drivers/net/ethernet/sfc/ef100_ethtool.c +++ b/drivers/net/ethernet/sfc/ef100_ethtool.c @@ -43,6 +43,8 @@ const struct ethtool_ops ef100_ethtool_ops = { .get_pauseparam = efx_ethtool_get_pauseparam, .set_pauseparam = efx_ethtool_set_pauseparam, .get_sset_count = efx_ethtool_get_sset_count, + .get_priv_flags = efx_ethtool_get_priv_flags, + .set_priv_flags = efx_ethtool_set_priv_flags, .self_test = efx_ethtool_self_test, .get_strings = efx_ethtool_get_strings, .get_link_ksettings = efx_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/sfc/ef100_netdev.c b/drivers/net/ethernet/sfc/ef100_netdev.c index 17b9d37218cbce019c657db2a245834b7c137bd2..88fa29572e230ec0691a7ca61f30c406b21d72cd 100644 --- a/drivers/net/ethernet/sfc/ef100_netdev.c +++ b/drivers/net/ethernet/sfc/ef100_netdev.c @@ -23,6 +23,7 @@ #include "mcdi_filters.h" #include "rx_common.h" #include "ef100_sriov.h" +#include "tc_bindings.h" static void ef100_update_name(struct efx_nic *efx) { @@ -246,6 +247,9 @@ static const struct net_device_ops ef100_netdev_ops = { #ifdef CONFIG_RFS_ACCEL .ndo_rx_flow_steer = efx_filter_rfs, #endif +#ifdef CONFIG_SFC_SRIOV + .ndo_setup_tc = efx_tc_setup, +#endif }; /* Netdev registration diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index 8061efdaf82ce184f2a38e398609c39cbdf39d9a..ad686c671ab893693c93f9394bc3a3597d2b3264 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -1137,6 +1137,9 @@ int ef100_probe_netdev_pf(struct efx_nic *efx) */ netif_warn(efx, probe, net_dev, "Failed to probe MAE rc %d\n", rc); + } else { + net_dev->features |= NETIF_F_HW_TC; + efx->fixed_features |= NETIF_F_HW_TC; } #endif return 0; diff --git a/drivers/net/ethernet/sfc/ef100_rep.c b/drivers/net/ethernet/sfc/ef100_rep.c index 73ae4656a6e7359251febb49411ec26ad018003b..81ab22c746355280ff31a01254810f1ff0c123ea 100644 --- a/drivers/net/ethernet/sfc/ef100_rep.c +++ b/drivers/net/ethernet/sfc/ef100_rep.c @@ -14,6 +14,7 @@ #include "ef100_nic.h" #include "mae.h" #include "rx_common.h" +#include "tc_bindings.h" #define EFX_EF100_REP_DRIVER "efx_ef100_rep" @@ -42,8 +43,7 @@ static int efx_ef100_rep_open(struct net_device *net_dev) { struct efx_rep *efv = netdev_priv(net_dev); - netif_napi_add(net_dev, &efv->napi, efx_ef100_rep_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(net_dev, &efv->napi, efx_ef100_rep_poll); napi_enable(&efv->napi); return 0; } @@ -107,6 +107,20 @@ static int efx_ef100_rep_get_phys_port_name(struct net_device *dev, return 0; } +static int efx_ef100_rep_setup_tc(struct net_device *net_dev, + enum tc_setup_type type, void *type_data) +{ + struct efx_rep *efv = netdev_priv(net_dev); + struct efx_nic *efx = efv->parent; + + if (type == TC_SETUP_CLSFLOWER) + return efx_tc_flower(efx, net_dev, type_data, efv); + if (type == TC_SETUP_BLOCK) + return efx_tc_setup_block(net_dev, efx, type_data, efv); + + return -EOPNOTSUPP; +} + static void efx_ef100_rep_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) { @@ -120,13 +134,14 @@ static void efx_ef100_rep_get_stats64(struct net_device *dev, stats->tx_errors = atomic64_read(&efv->stats.tx_errors); } -static const struct net_device_ops efx_ef100_rep_netdev_ops = { +const struct net_device_ops efx_ef100_rep_netdev_ops = { .ndo_open = efx_ef100_rep_open, .ndo_stop = efx_ef100_rep_close, .ndo_start_xmit = efx_ef100_rep_xmit, .ndo_get_port_parent_id = efx_ef100_rep_get_port_parent_id, .ndo_get_phys_port_name = efx_ef100_rep_get_phys_port_name, .ndo_get_stats64 = efx_ef100_rep_get_stats64, + .ndo_setup_tc = efx_ef100_rep_setup_tc, }; static void efx_ef100_rep_get_drvinfo(struct net_device *dev, diff --git a/drivers/net/ethernet/sfc/ef100_rep.h b/drivers/net/ethernet/sfc/ef100_rep.h index 070f700893c100d22e42ec8e02ce3f48780dbf2d..c21bc716f847bba39a296f1887630aace17aff6f 100644 --- a/drivers/net/ethernet/sfc/ef100_rep.h +++ b/drivers/net/ethernet/sfc/ef100_rep.h @@ -66,4 +66,5 @@ void efx_ef100_rep_rx_packet(struct efx_rep *efv, struct efx_rx_buffer *rx_buf); * Caller must hold rcu_read_lock(). */ struct efx_rep *efx_ef100_find_rep_by_mport(struct efx_nic *efx, u16 mport); +extern const struct net_device_ops efx_ef100_rep_netdev_ops; #endif /* EF100_REP_H */ diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 153d68e29b8b31e44c9acd2f57c3aab19dca9929..054d5ce6029e413df454106c464e331bf4d5ba94 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -778,7 +778,7 @@ static void efx_unregister_netdev(struct efx_nic *efx) return; if (efx_dev_registered(efx)) { - strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); efx_fini_mcdi_logging(efx); device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); unregister_netdev(efx->net_dev); @@ -1175,6 +1175,17 @@ static int efx_pm_freeze(struct device *dev) return 0; } +static void efx_pci_shutdown(struct pci_dev *pci_dev) +{ + struct efx_nic *efx = pci_get_drvdata(pci_dev); + + if (!efx) + return; + + efx_pm_freeze(&pci_dev->dev); + pci_disable_device(pci_dev); +} + static int efx_pm_thaw(struct device *dev) { int rc; @@ -1279,6 +1290,7 @@ static struct pci_driver efx_pci_driver = { .probe = efx_pci_probe, .remove = efx_pci_remove, .driver.pm = &efx_pm_ops, + .shutdown = efx_pci_shutdown, .err_handler = &efx_err_handlers, #ifdef CONFIG_SFC_SRIOV .sriov_configure = efx_pci_sriov_configure, diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c index 032b8c0bd78890d6de890fa2cf1fa9405c56ba9f..aaa381743bca38a8adb21ce072f73d2a592f3bfc 100644 --- a/drivers/net/ethernet/sfc/efx_channels.c +++ b/drivers/net/ethernet/sfc/efx_channels.c @@ -319,7 +319,7 @@ int efx_probe_interrupts(struct efx_nic *efx) efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); efx->n_rx_channels = 1; efx->n_tx_channels = 1; - efx->tx_channel_offset = 1; + efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0; efx->n_xdp_channels = 0; efx->xdp_channel_offset = efx->n_channels; efx->legacy_irq = efx->pci_dev->irq; @@ -1313,7 +1313,7 @@ void efx_init_napi_channel(struct efx_channel *channel) struct efx_nic *efx = channel->efx; channel->napi_dev = efx->net_dev; - netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64); + netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll); } void efx_init_napi(struct efx_nic *efx) diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c index a929a1aaba9291e5b840416b4849447d1f598340..c2224e41a694d849076f43395f42362a74c9e0cd 100644 --- a/drivers/net/ethernet/sfc/efx_common.c +++ b/drivers/net/ethernet/sfc/efx_common.c @@ -996,7 +996,7 @@ int efx_init_struct(struct efx_nic *efx, struct pci_dev *pci_dev) efx->pci_dev = pci_dev; efx->msg_enable = debug; efx->state = STATE_UNINIT; - strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); efx->rx_prefix_size = efx->type->rx_prefix_size; efx->rx_ip_align = diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c index bc840ede3053cd112e9f363729052b9562397e01..6649a2327d038d04ce24dd33bc48e34b511c6767 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.c +++ b/drivers/net/ethernet/sfc/ethtool_common.c @@ -101,15 +101,23 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = { #define EFX_ETHTOOL_SW_STAT_COUNT ARRAY_SIZE(efx_sw_stat_desc) +static const char efx_ethtool_priv_flags_strings[][ETH_GSTRING_LEN] = { + "log-tc-errors", +}; + +#define EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS BIT(0) + +#define EFX_ETHTOOL_PRIV_FLAGS_COUNT ARRAY_SIZE(efx_ethtool_priv_flags_strings) + void efx_ethtool_get_drvinfo(struct net_device *net_dev, struct ethtool_drvinfo *info) { struct efx_nic *efx = efx_netdev_priv(net_dev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); efx_mcdi_print_fwver(efx, info->fw_version, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); } u32 efx_ethtool_get_msglevel(struct net_device *net_dev) @@ -452,6 +460,8 @@ int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set) efx_ptp_describe_stats(efx, NULL); case ETH_SS_TEST: return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL); + case ETH_SS_PRIV_FLAGS: + return EFX_ETHTOOL_PRIV_FLAGS_COUNT; default: return -EINVAL; } @@ -468,7 +478,7 @@ void efx_ethtool_get_strings(struct net_device *net_dev, strings += (efx->type->describe_stats(efx, strings) * ETH_GSTRING_LEN); for (i = 0; i < EFX_ETHTOOL_SW_STAT_COUNT; i++) - strlcpy(strings + i * ETH_GSTRING_LEN, + strscpy(strings + i * ETH_GSTRING_LEN, efx_sw_stat_desc[i].name, ETH_GSTRING_LEN); strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN; strings += (efx_describe_per_queue_stats(efx, strings) * @@ -478,12 +488,39 @@ void efx_ethtool_get_strings(struct net_device *net_dev, case ETH_SS_TEST: efx_ethtool_fill_self_tests(efx, NULL, strings, NULL); break; + case ETH_SS_PRIV_FLAGS: + for (i = 0; i < EFX_ETHTOOL_PRIV_FLAGS_COUNT; i++) + strscpy(strings + i * ETH_GSTRING_LEN, + efx_ethtool_priv_flags_strings[i], + ETH_GSTRING_LEN); + break; default: /* No other string sets */ break; } } +u32 efx_ethtool_get_priv_flags(struct net_device *net_dev) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + u32 ret_flags = 0; + + if (efx->log_tc_errs) + ret_flags |= EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS; + + return ret_flags; +} + +int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + + efx->log_tc_errs = + !!(flags & EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS); + + return 0; +} + void efx_ethtool_get_stats(struct net_device *net_dev, struct ethtool_stats *stats, u64 *data) diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h index 6594919321010e7bee223fb7aa5cee88984baf7b..0afc74021a5ec71180297def9ba228d141620d8f 100644 --- a/drivers/net/ethernet/sfc/ethtool_common.h +++ b/drivers/net/ethernet/sfc/ethtool_common.h @@ -27,6 +27,8 @@ int efx_ethtool_fill_self_tests(struct efx_nic *efx, int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set); void efx_ethtool_get_strings(struct net_device *net_dev, u32 string_set, u8 *strings); +u32 efx_ethtool_get_priv_flags(struct net_device *net_dev); +int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags); void efx_ethtool_get_stats(struct net_device *net_dev, struct ethtool_stats *stats __attribute__ ((unused)), u64 *data); diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c index a63f40b098563af1e32d9fdd1566159ebf4e2f0e..e151b0957751741f96870d9184f9049f51272016 100644 --- a/drivers/net/ethernet/sfc/falcon/efx.c +++ b/drivers/net/ethernet/sfc/falcon/efx.c @@ -2012,7 +2012,7 @@ static void ef4_init_napi_channel(struct ef4_channel *channel) struct ef4_nic *efx = channel->efx; channel->napi_dev = efx->net_dev; - netif_napi_add(channel->napi_dev, &channel->napi_str, ef4_poll, 64); + netif_napi_add(channel->napi_dev, &channel->napi_str, ef4_poll); } static void ef4_init_napi(struct ef4_nic *efx) @@ -2329,7 +2329,7 @@ static void ef4_unregister_netdev(struct ef4_nic *efx) BUG_ON(netdev_priv(efx->net_dev) != efx); if (ef4_dev_registered(efx)) { - strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); unregister_netdev(efx->net_dev); } @@ -2640,7 +2640,7 @@ static int ef4_init_struct(struct ef4_nic *efx, efx->pci_dev = pci_dev; efx->msg_enable = debug; efx->state = STATE_UNINIT; - strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); efx->net_dev = net_dev; efx->rx_prefix_size = efx->type->rx_prefix_size; diff --git a/drivers/net/ethernet/sfc/falcon/ethtool.c b/drivers/net/ethernet/sfc/falcon/ethtool.c index 907254b36663b22a6615f8c2e10f355e145b3f6f..3976a333f7e37dd1d0f71ae1c36d4d346109f040 100644 --- a/drivers/net/ethernet/sfc/falcon/ethtool.c +++ b/drivers/net/ethernet/sfc/falcon/ethtool.c @@ -162,9 +162,9 @@ static void ef4_ethtool_get_drvinfo(struct net_device *net_dev, { struct ef4_nic *efx = netdev_priv(net_dev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->version, EF4_DRIVER_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->version, EF4_DRIVER_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); } static int ef4_ethtool_get_regs_len(struct net_device *net_dev) @@ -412,7 +412,7 @@ static void ef4_ethtool_get_strings(struct net_device *net_dev, strings += (efx->type->describe_stats(efx, strings) * ETH_GSTRING_LEN); for (i = 0; i < EF4_ETHTOOL_SW_STAT_COUNT; i++) - strlcpy(strings + i * ETH_GSTRING_LEN, + strscpy(strings + i * ETH_GSTRING_LEN, ef4_sw_stat_desc[i].name, ETH_GSTRING_LEN); strings += EF4_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN; strings += (ef4_describe_per_queue_stats(efx, strings) * diff --git a/drivers/net/ethernet/sfc/falcon/falcon.c b/drivers/net/ethernet/sfc/falcon/falcon.c index 3324a6219a0902851810987de71756a03ea5dee2..7a1c9337081b52d783a95e3cb0cdb1b5d3bd1b19 100644 --- a/drivers/net/ethernet/sfc/falcon/falcon.c +++ b/drivers/net/ethernet/sfc/falcon/falcon.c @@ -2387,7 +2387,7 @@ static int falcon_probe_nic(struct ef4_nic *efx) board->i2c_data.data = efx; board->i2c_adap.algo_data = &board->i2c_data; board->i2c_adap.dev.parent = &efx->pci_dev->dev; - strlcpy(board->i2c_adap.name, "SFC4000 GPIO", + strscpy(board->i2c_adap.name, "SFC4000 GPIO", sizeof(board->i2c_adap.name)); rc = i2c_bit_add_bus(&board->i2c_adap); if (rc) diff --git a/drivers/net/ethernet/sfc/falcon/nic.c b/drivers/net/ethernet/sfc/falcon/nic.c index 156da315ec8951d7fb301f52b85d7a8076fdd163..78c851b5a56f21b7a5756103e330d5417a2dc262 100644 --- a/drivers/net/ethernet/sfc/falcon/nic.c +++ b/drivers/net/ethernet/sfc/falcon/nic.c @@ -452,7 +452,7 @@ size_t ef4_nic_describe_stats(const struct ef4_hw_stat_desc *desc, size_t count, for_each_set_bit(index, mask, count) { if (desc[index].name) { if (names) { - strlcpy(names, desc[index].name, + strscpy(names, desc[index].name, ETH_GSTRING_LEN); names += ETH_GSTRING_LEN; } diff --git a/drivers/net/ethernet/sfc/filter.h b/drivers/net/ethernet/sfc/filter.h index 4d928839d2922e3647aac606f46f2c15b4accd70..be72e71da0277f039bd60358c5505b74c691754d 100644 --- a/drivers/net/ethernet/sfc/filter.h +++ b/drivers/net/ethernet/sfc/filter.h @@ -9,6 +9,7 @@ #include #include +#include #include /** @@ -223,6 +224,27 @@ efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto, return 0; } +/** + * efx_filter_set_ipv6_local - specify IPv6 host, transport protocol and port + * @spec: Specification to initialise + * @proto: Transport layer protocol number + * @host: Local host address (network byte order) + * @port: Local port (network byte order) + */ +static inline int +efx_filter_set_ipv6_local(struct efx_filter_spec *spec, u8 proto, + const struct in6_addr *host, __be16 port) +{ + spec->match_flags |= + EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | + EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT; + spec->ether_type = htons(ETH_P_IPV6); + spec->ip_proto = proto; + memcpy(spec->loc_host, host, sizeof(spec->loc_host)); + spec->loc_port = port; + return 0; +} + /** * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports * @spec: Specification to initialise diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c index 97627f5e36741f029b2b9788e93c262df3a5dae5..874c765b2465105d3e9d41ad192b826529ba1cb8 100644 --- a/drivers/net/ethernet/sfc/mae.c +++ b/drivers/net/ethernet/sfc/mae.c @@ -112,6 +112,167 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) return 0; } +static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN); + size_t outlen; + int rc; + + BUILD_BUG_ON(MC_CMD_MAE_GET_CAPS_IN_LEN); + + rc = efx_mcdi_rpc(efx, MC_CMD_MAE_GET_CAPS, NULL, 0, outbuf, + sizeof(outbuf), &outlen); + if (rc) + return rc; + if (outlen < sizeof(outbuf)) + return -EIO; + caps->match_field_count = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT); + caps->action_prios = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_ACTION_PRIOS); + return 0; +} + +static int efx_mae_get_rule_fields(struct efx_nic *efx, u32 cmd, + u8 *field_support) +{ + MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_AR_CAPS_OUT_LEN(MAE_NUM_FIELDS)); + MCDI_DECLARE_STRUCT_PTR(caps); + unsigned int count; + size_t outlen; + int rc, i; + + BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_IN_LEN); + + rc = efx_mcdi_rpc(efx, cmd, NULL, 0, outbuf, sizeof(outbuf), &outlen); + if (rc) + return rc; + count = MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_COUNT); + memset(field_support, MAE_FIELD_UNSUPPORTED, MAE_NUM_FIELDS); + caps = _MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_FIELD_FLAGS); + /* We're only interested in the support status enum, not any other + * flags, so just extract that from each entry. + */ + for (i = 0; i < count; i++) + if (i * sizeof(*outbuf) + MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_OFST < outlen) + field_support[i] = EFX_DWORD_FIELD(caps[i], MAE_FIELD_FLAGS_SUPPORT_STATUS); + return 0; +} + +int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps) +{ + int rc; + + rc = efx_mae_get_basic_caps(efx, caps); + if (rc) + return rc; + return efx_mae_get_rule_fields(efx, MC_CMD_MAE_GET_AR_CAPS, + caps->action_rule_fields); +} + +/* Bit twiddling: + * Prefix: 1...110...0 + * ~: 0...001...1 + * + 1: 0...010...0 is power of two + * so (~x) & ((~x) + 1) == 0. Converse holds also. + */ +#define is_prefix_byte(_x) !(((_x) ^ 0xff) & (((_x) ^ 0xff) + 1)) + +enum mask_type { MASK_ONES, MASK_ZEROES, MASK_PREFIX, MASK_OTHER }; + +static const char *mask_type_name(enum mask_type typ) +{ + switch (typ) { + case MASK_ONES: + return "all-1s"; + case MASK_ZEROES: + return "all-0s"; + case MASK_PREFIX: + return "prefix"; + case MASK_OTHER: + return "arbitrary"; + default: /* can't happen */ + return "unknown"; + } +} + +/* Checks a (big-endian) bytestring is a bit prefix */ +static enum mask_type classify_mask(const u8 *mask, size_t len) +{ + bool zeroes = true; /* All bits seen so far are zeroes */ + bool ones = true; /* All bits seen so far are ones */ + bool prefix = true; /* Valid prefix so far */ + size_t i; + + for (i = 0; i < len; i++) { + if (ones) { + if (!is_prefix_byte(mask[i])) + prefix = false; + } else if (mask[i]) { + prefix = false; + } + if (mask[i] != 0xff) + ones = false; + if (mask[i]) + zeroes = false; + } + if (ones) + return MASK_ONES; + if (zeroes) + return MASK_ZEROES; + if (prefix) + return MASK_PREFIX; + return MASK_OTHER; +} + +static int efx_mae_match_check_cap_typ(u8 support, enum mask_type typ) +{ + switch (support) { + case MAE_FIELD_UNSUPPORTED: + case MAE_FIELD_SUPPORTED_MATCH_NEVER: + if (typ == MASK_ZEROES) + return 0; + return -EOPNOTSUPP; + case MAE_FIELD_SUPPORTED_MATCH_OPTIONAL: + if (typ == MASK_ZEROES) + return 0; + fallthrough; + case MAE_FIELD_SUPPORTED_MATCH_ALWAYS: + if (typ == MASK_ONES) + return 0; + return -EINVAL; + case MAE_FIELD_SUPPORTED_MATCH_PREFIX: + if (typ == MASK_OTHER) + return -EOPNOTSUPP; + return 0; + case MAE_FIELD_SUPPORTED_MATCH_MASK: + return 0; + default: + return -EIO; + } +} + +int efx_mae_match_check_caps(struct efx_nic *efx, + const struct efx_tc_match_fields *mask, + struct netlink_ext_ack *extack) +{ + const u8 *supported_fields = efx->tc->caps->action_rule_fields; + __be32 ingress_port = cpu_to_be32(mask->ingress_port); + enum mask_type ingress_port_mask_type; + int rc; + + /* Check for _PREFIX assumes big-endian, so we need to convert */ + ingress_port_mask_type = classify_mask((const u8 *)&ingress_port, + sizeof(ingress_port)); + rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_INGRESS_PORT], + ingress_port_mask_type); + if (rc) { + efx_tc_err(efx, "No support for %s mask in field ingress_port\n", + mask_type_name(ingress_port_mask_type)); + NL_SET_ERR_MSG_MOD(extack, "Unsupported mask type for ingress_port"); + return rc; + } + return 0; +} + static bool efx_mae_asl_id(u32 id) { return !!(id & BIT(31)); @@ -279,6 +440,10 @@ static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit), } MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK, match->mask.ingress_port); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID, + match->value.recirc_id); + MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK, + match->mask.recirc_id); return 0; } diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h index 0369be4d8983f32476d04cca51caf1d9d58915be..3e0cd238d523e0af85d7abe5a3785bd94dac4582 100644 --- a/drivers/net/ethernet/sfc/mae.h +++ b/drivers/net/ethernet/sfc/mae.h @@ -27,6 +27,20 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out); int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id); +#define MAE_NUM_FIELDS (MAE_FIELD_ENC_VNET_ID + 1) + +struct mae_caps { + u32 match_field_count; + u32 action_prios; + u8 action_rule_fields[MAE_NUM_FIELDS]; +}; + +int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps); + +int efx_mae_match_check_caps(struct efx_nic *efx, + const struct efx_tc_match_fields *mask, + struct netlink_ext_ack *extack); + int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act); int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id); diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h index 26bc69f768016ad8d0c6689178656a103b34cc36..1f18e9dc62e8d00e6bac063d0e8dc9b4a7868046 100644 --- a/drivers/net/ethernet/sfc/mcdi.h +++ b/drivers/net/ethernet/sfc/mcdi.h @@ -201,6 +201,12 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev); ((u8 *)(_buf) + (_offset)) #define MCDI_PTR(_buf, _field) \ _MCDI_PTR(_buf, MC_CMD_ ## _field ## _OFST) +/* Use MCDI_STRUCT_ functions to access members of MCDI structuredefs. + * _buf should point to the start of the structure, typically obtained with + * MCDI_DECLARE_STRUCT_PTR(structure) = _MCDI_DWORD(mcdi_buf, FIELD_WHICH_IS_STRUCT); + */ +#define MCDI_STRUCT_PTR(_buf, _field) \ + _MCDI_PTR(_buf, _field ## _OFST) #define _MCDI_CHECK_ALIGN(_ofst, _align) \ ((_ofst) + BUILD_BUG_ON_ZERO((_ofst) & (_align - 1))) #define _MCDI_DWORD(_buf, _field) \ @@ -208,6 +214,10 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev); #define _MCDI_STRUCT_DWORD(_buf, _field) \ ((_buf) + (_MCDI_CHECK_ALIGN(_field ## _OFST, 4) >> 2)) +#define MCDI_STRUCT_SET_BYTE(_buf, _field, _value) do { \ + BUILD_BUG_ON(_field ## _LEN != 1); \ + *(u8 *)MCDI_STRUCT_PTR(_buf, _field) = _value; \ + } while (0) #define MCDI_BYTE(_buf, _field) \ ((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1), \ *MCDI_PTR(_buf, _field)) diff --git a/drivers/net/ethernet/sfc/mcdi_mon.c b/drivers/net/ethernet/sfc/mcdi_mon.c index 5954fcfee2b16f354ebfa15ef5c21de7c7a3874f..f5128db7c7e7e0a2d004bd31e38f8a799b5ce3cd 100644 --- a/drivers/net/ethernet/sfc/mcdi_mon.c +++ b/drivers/net/ethernet/sfc/mcdi_mon.c @@ -285,7 +285,7 @@ efx_mcdi_mon_add_attr(struct efx_nic *efx, const char *name, struct efx_mcdi_mon *hwmon = efx_mcdi_mon(efx); struct efx_mcdi_mon_attribute *attr = &hwmon->attrs[hwmon->n_attrs]; - strlcpy(attr->name, name, sizeof(attr->name)); + strscpy(attr->name, name, sizeof(attr->name)); attr->index = index; attr->type = type; if (type < ARRAY_SIZE(efx_mcdi_sensor_type)) diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h index 7ef823d7a89a6140bab0007ebb60958802e745cd..2e9ba0cfe8487a82490e57208e443d40e09729a5 100644 --- a/drivers/net/ethernet/sfc/net_driver.h +++ b/drivers/net/ethernet/sfc/net_driver.h @@ -855,6 +855,7 @@ enum efx_xdp_tx_queues_mode { * @timer_max_ns: Interrupt timer maximum value, in nanoseconds * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues * @irqs_hooked: Channel interrupts are hooked + * @log_tc_errs: Error logging for TC filter insertion is enabled * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues * @irq_rx_moderation_us: IRQ moderation time for RX event queues * @msg_enable: Log message enable flags @@ -1017,6 +1018,7 @@ struct efx_nic { unsigned int timer_max_ns; bool irq_rx_adaptive; bool irqs_hooked; + bool log_tc_errs; unsigned int irq_mod_step_us; unsigned int irq_rx_moderation_us; u32 msg_enable; diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c index 22fbb0ae77fba7681bee74ed0726ab6dbb8d7a2c..63e2394382bb7cae126b11c2fc62bf725a8c1c31 100644 --- a/drivers/net/ethernet/sfc/nic.c +++ b/drivers/net/ethernet/sfc/nic.c @@ -465,7 +465,7 @@ size_t efx_nic_describe_stats(const struct efx_hw_stat_desc *desc, size_t count, for_each_set_bit(index, mask, count) { if (desc[index].name) { if (names) { - strlcpy(names, desc[index].name, + strscpy(names, desc[index].name, ETH_GSTRING_LEN); names += ETH_GSTRING_LEN; } diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c index 10ad0b93d283b01a98d3dca215023d334d257ee3..eaef4a15008a3e3bef67270c0658de9fd0c7b0a3 100644 --- a/drivers/net/ethernet/sfc/ptp.c +++ b/drivers/net/ethernet/sfc/ptp.c @@ -118,9 +118,14 @@ #define PTP_MIN_LENGTH 63 -#define PTP_ADDRESS 0xe0000181 /* 224.0.1.129 */ +#define PTP_RXFILTERS_LEN 5 + +#define PTP_ADDR_IPV4 0xe0000181 /* 224.0.1.129 */ +#define PTP_ADDR_IPV6 {0xff, 0x0e, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0x01, 0x81} /* ff0e::181 */ #define PTP_EVENT_PORT 319 #define PTP_GENERAL_PORT 320 +#define PTP_ADDR_ETHER {0x01, 0x1b, 0x19, 0, 0, 0} /* 01-1B-19-00-00-00 */ /* Annoyingly the format of the version numbers are different between * versions 1 and 2 so it isn't possible to simply look for 1 or 2. @@ -224,9 +229,8 @@ struct efx_ptp_timeset { * @work: Work task * @reset_required: A serious error has occurred and the PTP task needs to be * reset (disable, enable). - * @rxfilter_event: Receive filter when operating - * @rxfilter_general: Receive filter when operating - * @rxfilter_installed: Receive filter installed + * @rxfilters: Receive filters when operating + * @rxfilters_count: Num of installed rxfilters, should be == PTP_RXFILTERS_LEN * @config: Current timestamp configuration * @enabled: PTP operation enabled * @mode: Mode in which PTP operating (PTP version) @@ -295,9 +299,8 @@ struct efx_ptp_data { struct workqueue_struct *workwq; struct work_struct work; bool reset_required; - u32 rxfilter_event; - u32 rxfilter_general; - bool rxfilter_installed; + u32 rxfilters[PTP_RXFILTERS_LEN]; + size_t rxfilters_count; struct hwtstamp_config config; bool enabled; unsigned int mode; @@ -1290,61 +1293,108 @@ static void efx_ptp_remove_multicast_filters(struct efx_nic *efx) { struct efx_ptp_data *ptp = efx->ptp_data; - if (ptp->rxfilter_installed) { - efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, - ptp->rxfilter_general); + while (ptp->rxfilters_count) { + ptp->rxfilters_count--; efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, - ptp->rxfilter_event); - ptp->rxfilter_installed = false; + ptp->rxfilters[ptp->rxfilters_count]); } } -static int efx_ptp_insert_multicast_filters(struct efx_nic *efx) +static void efx_ptp_init_filter(struct efx_nic *efx, + struct efx_filter_spec *rxfilter) +{ + struct efx_channel *channel = efx->ptp_data->channel; + struct efx_rx_queue *queue = efx_channel_get_rx_queue(channel); + + efx_filter_init_rx(rxfilter, EFX_FILTER_PRI_REQUIRED, 0, + efx_rx_queue_index(queue)); +} + +static int efx_ptp_insert_filter(struct efx_nic *efx, + struct efx_filter_spec *rxfilter) { struct efx_ptp_data *ptp = efx->ptp_data; + + int rc = efx_filter_insert_filter(efx, rxfilter, true); + if (rc < 0) + return rc; + ptp->rxfilters[ptp->rxfilters_count] = rc; + ptp->rxfilters_count++; + return 0; +} + +static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx, u16 port) +{ struct efx_filter_spec rxfilter; + + efx_ptp_init_filter(efx, &rxfilter); + efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP, htonl(PTP_ADDR_IPV4), + htons(port)); + return efx_ptp_insert_filter(efx, &rxfilter); +} + +static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx, u16 port) +{ + const struct in6_addr addr = {{PTP_ADDR_IPV6}}; + struct efx_filter_spec rxfilter; + + efx_ptp_init_filter(efx, &rxfilter); + efx_filter_set_ipv6_local(&rxfilter, IPPROTO_UDP, &addr, htons(port)); + return efx_ptp_insert_filter(efx, &rxfilter); +} + +static int efx_ptp_insert_eth_filter(struct efx_nic *efx) +{ + const u8 addr[ETH_ALEN] = PTP_ADDR_ETHER; + struct efx_filter_spec rxfilter; + + efx_ptp_init_filter(efx, &rxfilter); + efx_filter_set_eth_local(&rxfilter, EFX_FILTER_VID_UNSPEC, addr); + rxfilter.match_flags |= EFX_FILTER_MATCH_ETHER_TYPE; + rxfilter.ether_type = htons(ETH_P_1588); + return efx_ptp_insert_filter(efx, &rxfilter); +} + +static int efx_ptp_insert_multicast_filters(struct efx_nic *efx) +{ + struct efx_ptp_data *ptp = efx->ptp_data; int rc; - if (!ptp->channel || ptp->rxfilter_installed) + if (!ptp->channel || ptp->rxfilters_count) return 0; /* Must filter on both event and general ports to ensure * that there is no packet re-ordering. */ - efx_filter_init_rx(&rxfilter, EFX_FILTER_PRI_REQUIRED, 0, - efx_rx_queue_index( - efx_channel_get_rx_queue(ptp->channel))); - rc = efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP, - htonl(PTP_ADDRESS), - htons(PTP_EVENT_PORT)); - if (rc != 0) - return rc; - - rc = efx_filter_insert_filter(efx, &rxfilter, true); + rc = efx_ptp_insert_ipv4_filter(efx, PTP_EVENT_PORT); if (rc < 0) - return rc; - ptp->rxfilter_event = rc; - - efx_filter_init_rx(&rxfilter, EFX_FILTER_PRI_REQUIRED, 0, - efx_rx_queue_index( - efx_channel_get_rx_queue(ptp->channel))); - rc = efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP, - htonl(PTP_ADDRESS), - htons(PTP_GENERAL_PORT)); - if (rc != 0) goto fail; - rc = efx_filter_insert_filter(efx, &rxfilter, true); + rc = efx_ptp_insert_ipv4_filter(efx, PTP_GENERAL_PORT); if (rc < 0) goto fail; - ptp->rxfilter_general = rc; - ptp->rxfilter_installed = true; + /* if the NIC supports hw timestamps by the MAC, we can support + * PTP over IPv6 and Ethernet + */ + if (efx_ptp_use_mac_tx_timestamps(efx)) { + rc = efx_ptp_insert_ipv6_filter(efx, PTP_EVENT_PORT); + if (rc < 0) + goto fail; + + rc = efx_ptp_insert_ipv6_filter(efx, PTP_GENERAL_PORT); + if (rc < 0) + goto fail; + + rc = efx_ptp_insert_eth_filter(efx); + if (rc < 0) + goto fail; + } + return 0; fail: - efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, - ptp->rxfilter_event); + efx_ptp_remove_multicast_filters(efx); return rc; } diff --git a/drivers/net/ethernet/sfc/siena/efx.c b/drivers/net/ethernet/sfc/siena/efx.c index 63d999e639604932c6f618ad002b1e0345639fe2..60e5b7c8ccf95ded5558bd7d6c90d40b04353a70 100644 --- a/drivers/net/ethernet/sfc/siena/efx.c +++ b/drivers/net/ethernet/sfc/siena/efx.c @@ -775,7 +775,7 @@ static void efx_unregister_netdev(struct efx_nic *efx) BUG_ON(netdev_priv(efx->net_dev) != efx); if (efx_dev_registered(efx)) { - strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name)); efx_siena_fini_mcdi_logging(efx); device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type); unregister_netdev(efx->net_dev); @@ -1148,6 +1148,17 @@ static int efx_pm_freeze(struct device *dev) return 0; } +static void efx_pci_shutdown(struct pci_dev *pci_dev) +{ + struct efx_nic *efx = pci_get_drvdata(pci_dev); + + if (!efx) + return; + + efx_pm_freeze(&pci_dev->dev); + pci_disable_device(pci_dev); +} + static int efx_pm_thaw(struct device *dev) { int rc; @@ -1252,6 +1263,7 @@ static struct pci_driver efx_pci_driver = { .probe = efx_pci_probe, .remove = efx_pci_remove, .driver.pm = &efx_pm_ops, + .shutdown = efx_pci_shutdown, .err_handler = &efx_siena_err_handlers, #ifdef CONFIG_SFC_SIENA_SRIOV .sriov_configure = efx_pci_sriov_configure, diff --git a/drivers/net/ethernet/sfc/siena/efx_channels.c b/drivers/net/ethernet/sfc/siena/efx_channels.c index 017212a40df388ecfc6d119f5808545c3c249779..06ed74994e366201ec468e2fd05ff14f474b8e7f 100644 --- a/drivers/net/ethernet/sfc/siena/efx_channels.c +++ b/drivers/net/ethernet/sfc/siena/efx_channels.c @@ -320,7 +320,7 @@ int efx_siena_probe_interrupts(struct efx_nic *efx) efx->n_channels = 1 + (efx_siena_separate_tx_channels ? 1 : 0); efx->n_rx_channels = 1; efx->n_tx_channels = 1; - efx->tx_channel_offset = 1; + efx->tx_channel_offset = efx_siena_separate_tx_channels ? 1 : 0; efx->n_xdp_channels = 0; efx->xdp_channel_offset = efx->n_channels; efx->legacy_irq = efx->pci_dev->irq; @@ -1317,7 +1317,7 @@ static void efx_init_napi_channel(struct efx_channel *channel) struct efx_nic *efx = channel->efx; channel->napi_dev = efx->net_dev; - netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64); + netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll); } void efx_siena_init_napi(struct efx_nic *efx) diff --git a/drivers/net/ethernet/sfc/siena/efx_common.c b/drivers/net/ethernet/sfc/siena/efx_common.c index 954daf464abbdcd5ddd980b95c64afa282bcc085..1fd396b00bfbb84a00e3a47d504911b800270605 100644 --- a/drivers/net/ethernet/sfc/siena/efx_common.c +++ b/drivers/net/ethernet/sfc/siena/efx_common.c @@ -1006,7 +1006,7 @@ int efx_siena_init_struct(struct efx_nic *efx, efx->pci_dev = pci_dev; efx->msg_enable = debug; efx->state = STATE_UNINIT; - strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); + strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name)); efx->net_dev = net_dev; efx->rx_prefix_size = efx->type->rx_prefix_size; diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.c b/drivers/net/ethernet/sfc/siena/ethtool_common.c index 0207d07f54e3902d00284e89e256cbe5033b97bc..f590e87e5a23a6187a0297d5981f84bb58188d6e 100644 --- a/drivers/net/ethernet/sfc/siena/ethtool_common.c +++ b/drivers/net/ethernet/sfc/siena/ethtool_common.c @@ -105,10 +105,10 @@ void efx_siena_ethtool_get_drvinfo(struct net_device *net_dev, { struct efx_nic *efx = netdev_priv(net_dev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); efx_siena_mcdi_print_fwver(efx, info->fw_version, sizeof(info->fw_version)); - strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); } u32 efx_siena_ethtool_get_msglevel(struct net_device *net_dev) @@ -467,7 +467,7 @@ void efx_siena_ethtool_get_strings(struct net_device *net_dev, strings += (efx->type->describe_stats(efx, strings) * ETH_GSTRING_LEN); for (i = 0; i < EFX_ETHTOOL_SW_STAT_COUNT; i++) - strlcpy(strings + i * ETH_GSTRING_LEN, + strscpy(strings + i * ETH_GSTRING_LEN, efx_sw_stat_desc[i].name, ETH_GSTRING_LEN); strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN; strings += (efx_describe_per_queue_stats(efx, strings) * diff --git a/drivers/net/ethernet/sfc/siena/mcdi_mon.c b/drivers/net/ethernet/sfc/siena/mcdi_mon.c index c7ea703c5d7ad5b048aca3578be844e6c3c5b334..56a9c56ed9e3b9b44c2286fc6c3859bee8e9ae30 100644 --- a/drivers/net/ethernet/sfc/siena/mcdi_mon.c +++ b/drivers/net/ethernet/sfc/siena/mcdi_mon.c @@ -285,7 +285,7 @@ efx_mcdi_mon_add_attr(struct efx_nic *efx, const char *name, struct efx_mcdi_mon *hwmon = efx_mcdi_mon(efx); struct efx_mcdi_mon_attribute *attr = &hwmon->attrs[hwmon->n_attrs]; - strlcpy(attr->name, name, sizeof(attr->name)); + strscpy(attr->name, name, sizeof(attr->name)); attr->index = index; attr->type = type; if (type < ARRAY_SIZE(efx_mcdi_sensor_type)) diff --git a/drivers/net/ethernet/sfc/siena/nic.c b/drivers/net/ethernet/sfc/siena/nic.c index abf9a4adf139a8dead5eb6d3bfa91e29e946d3ae..0ea0433a62301bbd343b987ae4a2be55d5928c34 100644 --- a/drivers/net/ethernet/sfc/siena/nic.c +++ b/drivers/net/ethernet/sfc/siena/nic.c @@ -458,7 +458,7 @@ size_t efx_siena_describe_stats(const struct efx_hw_stat_desc *desc, size_t coun for_each_set_bit(index, mask, count) { if (desc[index].name) { if (names) { - strlcpy(names, desc[index].name, + strscpy(names, desc[index].name, ETH_GSTRING_LEN); names += ETH_GSTRING_LEN; } diff --git a/drivers/net/ethernet/sfc/siena/tx.c b/drivers/net/ethernet/sfc/siena/tx.c index e166dcb9b99ceaa055a880f57785ee13bfb2a6f9..91e87594ed1eac18e8dbec04c9b29161a3ef0b54 100644 --- a/drivers/net/ethernet/sfc/siena/tx.c +++ b/drivers/net/ethernet/sfc/siena/tx.c @@ -336,7 +336,7 @@ netdev_tx_t efx_siena_hard_start_xmit(struct sk_buff *skb, * previous packets out. */ if (!netdev_xmit_more()) - efx_tx_send_pending(tx_queue->channel); + efx_tx_send_pending(efx_get_tx_channel(efx, index)); return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c index 0c0aeb91f5009dc4a1731f0215a07f9647635359..3478860d40232c07997b4699513534b6a0ed952a 100644 --- a/drivers/net/ethernet/sfc/tc.c +++ b/drivers/net/ethernet/sfc/tc.c @@ -9,11 +9,60 @@ * by the Free Software Foundation, incorporated herein by reference. */ +#include #include "tc.h" +#include "tc_bindings.h" #include "mae.h" #include "ef100_rep.h" #include "efx.h" +#define EFX_EFV_PF NULL +/* Look up the representor information (efv) for a device. + * May return NULL for the PF (us), or an error pointer for a device that + * isn't supported as a TC offload endpoint + */ +static struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx, + struct net_device *dev) +{ + struct efx_rep *efv; + + if (!dev) + return ERR_PTR(-EOPNOTSUPP); + /* Is it us (the PF)? */ + if (dev == efx->net_dev) + return EFX_EFV_PF; + /* Is it an efx vfrep at all? */ + if (dev->netdev_ops != &efx_ef100_rep_netdev_ops) + return ERR_PTR(-EOPNOTSUPP); + /* Is it ours? We don't support TC rules that include another + * EF100's netdevices (not even on another port of the same NIC). + */ + efv = netdev_priv(dev); + if (efv->parent != efx) + return ERR_PTR(-EOPNOTSUPP); + return efv; +} + +/* Convert a driver-internal vport ID into an external device (wire or VF) */ +static s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv) +{ + u32 mport; + + if (IS_ERR(efv)) + return PTR_ERR(efv); + if (!efv) /* device is PF (us) */ + efx_mae_mport_wire(efx, &mport); + else /* device is repr */ + efx_mae_mport_mport(efx, efv->mport, &mport); + return mport; +} + +static const struct rhashtable_params efx_tc_match_action_ht_params = { + .key_len = sizeof(unsigned long), + .key_offset = offsetof(struct efx_tc_flow_rule, cookie), + .head_offset = offsetof(struct efx_tc_flow_rule, linkage), +}; + static void efx_tc_free_action_set(struct efx_nic *efx, struct efx_tc_action_set *act, bool in_hw) { @@ -58,6 +107,333 @@ static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rul rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; } +static void efx_tc_flow_free(void *ptr, void *arg) +{ + struct efx_tc_flow_rule *rule = ptr; + struct efx_nic *efx = arg; + + netif_err(efx, drv, efx->net_dev, + "tc rule %lx still present at teardown, removing\n", + rule->cookie); + + efx_mae_delete_rule(efx, rule->fw_id); + + /* Release entries in subsidiary tables */ + efx_tc_free_action_set_list(efx, &rule->acts, true); + + kfree(rule); +} + +static int efx_tc_flower_parse_match(struct efx_nic *efx, + struct flow_rule *rule, + struct efx_tc_match *match, + struct netlink_ext_ack *extack) +{ + struct flow_dissector *dissector = rule->match.dissector; + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { + struct flow_match_control fm; + + flow_rule_match_control(rule, &fm); + + if (fm.mask->flags) { + efx_tc_err(efx, "Unsupported match on control.flags %#x\n", + fm.mask->flags); + NL_SET_ERR_MSG_MOD(extack, "Unsupported match on control.flags"); + return -EOPNOTSUPP; + } + } + if (dissector->used_keys & + ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | + BIT(FLOW_DISSECTOR_KEY_BASIC))) { + efx_tc_err(efx, "Unsupported flower keys %#x\n", dissector->used_keys); + NL_SET_ERR_MSG_MOD(extack, "Unsupported flower keys encountered"); + return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { + struct flow_match_basic fm; + + flow_rule_match_basic(rule, &fm); + if (fm.mask->n_proto) { + EFX_TC_ERR_MSG(efx, extack, "Unsupported eth_proto match\n"); + return -EOPNOTSUPP; + } + if (fm.mask->ip_proto) { + EFX_TC_ERR_MSG(efx, extack, "Unsupported ip_proto match\n"); + return -EOPNOTSUPP; + } + } + + return 0; +} + +static int efx_tc_flower_replace(struct efx_nic *efx, + struct net_device *net_dev, + struct flow_cls_offload *tc, + struct efx_rep *efv) +{ + struct flow_rule *fr = flow_cls_offload_flow_rule(tc); + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_flow_rule *rule = NULL, *old; + struct efx_tc_action_set *act = NULL; + const struct flow_action_entry *fa; + struct efx_rep *from_efv, *to_efv; + struct efx_tc_match match; + s64 rc; + int i; + + if (!tc_can_offload_extack(efx->net_dev, extack)) + return -EOPNOTSUPP; + if (WARN_ON(!efx->tc)) + return -ENETDOWN; + if (WARN_ON(!efx->tc->up)) + return -ENETDOWN; + + from_efv = efx_tc_flower_lookup_efv(efx, net_dev); + if (IS_ERR(from_efv)) { + /* Might be a tunnel decap rule from an indirect block. + * Support for those not implemented yet. + */ + return -EOPNOTSUPP; + } + + if (efv != from_efv) { + /* can't happen */ + efx_tc_err(efx, "for %s efv is %snull but from_efv is %snull\n", + netdev_name(net_dev), efv ? "non-" : "", + from_efv ? "non-" : ""); + if (efv) + NL_SET_ERR_MSG_MOD(extack, "vfrep filter has PF net_dev (can't happen)"); + else + NL_SET_ERR_MSG_MOD(extack, "PF filter has vfrep net_dev (can't happen)"); + return -EINVAL; + } + + /* Parse match */ + memset(&match, 0, sizeof(match)); + rc = efx_tc_flower_external_mport(efx, from_efv); + if (rc < 0) { + EFX_TC_ERR_MSG(efx, extack, "Failed to identify ingress m-port"); + return rc; + } + match.value.ingress_port = rc; + match.mask.ingress_port = ~0; + rc = efx_tc_flower_parse_match(efx, fr, &match, extack); + if (rc) + return rc; + + if (tc->common.chain_index) { + EFX_TC_ERR_MSG(efx, extack, "No support for nonzero chain_index"); + return -EOPNOTSUPP; + } + match.mask.recirc_id = 0xff; + + rc = efx_mae_match_check_caps(efx, &match.mask, extack); + if (rc) + return rc; + + rule = kzalloc(sizeof(*rule), GFP_USER); + if (!rule) + return -ENOMEM; + INIT_LIST_HEAD(&rule->acts.list); + rule->cookie = tc->cookie; + old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + if (old) { + netif_dbg(efx, drv, efx->net_dev, + "Already offloaded rule (cookie %lx)\n", tc->cookie); + rc = -EEXIST; + NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded"); + goto release; + } + + /* Parse actions */ + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + + flow_action_for_each(i, fa, &fr->action) { + struct efx_tc_action_set save; + + if (!act) { + /* more actions after a non-pipe action */ + EFX_TC_ERR_MSG(efx, extack, "Action follows non-pipe action"); + rc = -EINVAL; + goto release; + } + + switch (fa->id) { + case FLOW_ACTION_DROP: + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (drop)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; /* end of the line */ + break; + case FLOW_ACTION_REDIRECT: + case FLOW_ACTION_MIRRED: + save = *act; + to_efv = efx_tc_flower_lookup_efv(efx, fa->dev); + if (IS_ERR(to_efv)) { + EFX_TC_ERR_MSG(efx, extack, "Mirred egress device not on switch"); + rc = PTR_ERR(to_efv); + goto release; + } + rc = efx_tc_flower_external_mport(efx, to_efv); + if (rc < 0) { + EFX_TC_ERR_MSG(efx, extack, "Failed to identify egress m-port"); + goto release; + } + act->dest_mport = rc; + act->deliver = 1; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (mirred)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; + if (fa->id == FLOW_ACTION_REDIRECT) + break; /* end of the line */ + /* Mirror, so continue on with saved act */ + act = kzalloc(sizeof(*act), GFP_USER); + if (!act) { + rc = -ENOMEM; + goto release; + } + *act = save; + break; + default: + efx_tc_err(efx, "Unhandled action %u\n", fa->id); + rc = -EOPNOTSUPP; + NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); + goto release; + } + } + + if (act) { + /* Not shot/redirected, so deliver to default dest */ + if (from_efv == EFX_EFV_PF) + /* Rule applies to traffic from the wire, + * and default dest is thus the PF + */ + efx_mae_mport_uplink(efx, &act->dest_mport); + else + /* Representor, so rule applies to traffic from + * representee, and default dest is thus the rep. + * All reps use the same mport for delivery + */ + efx_mae_mport_mport(efx, efx->tc->reps_mport_id, + &act->dest_mport); + act->deliver = 1; + rc = efx_mae_alloc_action_set(efx, act); + if (rc) { + EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (deliver)"); + goto release; + } + list_add_tail(&act->list, &rule->acts.list); + act = NULL; /* Prevent double-free in error path */ + } + + netif_dbg(efx, drv, efx->net_dev, + "Successfully parsed filter (cookie %lx)\n", + tc->cookie); + + rule->match = match; + + rc = efx_mae_alloc_action_set_list(efx, &rule->acts); + if (rc) { + EFX_TC_ERR_MSG(efx, extack, "Failed to write action set list to hw"); + goto release; + } + rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC, + rule->acts.fw_id, &rule->fw_id); + if (rc) { + EFX_TC_ERR_MSG(efx, extack, "Failed to insert rule in hw"); + goto release_acts; + } + return 0; + +release_acts: + efx_mae_free_action_set_list(efx, &rule->acts); +release: + /* We failed to insert the rule, so free up any entries we created in + * subsidiary tables. + */ + if (act) + efx_tc_free_action_set(efx, act, false); + if (rule) { + rhashtable_remove_fast(&efx->tc->match_action_ht, + &rule->linkage, + efx_tc_match_action_ht_params); + efx_tc_free_action_set_list(efx, &rule->acts, false); + } + kfree(rule); + return rc; +} + +static int efx_tc_flower_destroy(struct efx_nic *efx, + struct net_device *net_dev, + struct flow_cls_offload *tc) +{ + struct netlink_ext_ack *extack = tc->common.extack; + struct efx_tc_flow_rule *rule; + + rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie, + efx_tc_match_action_ht_params); + if (!rule) { + /* Only log a message if we're the ingress device. Otherwise + * it's a foreign filter and we might just not have been + * interested (e.g. we might not have been the egress device + * either). + */ + if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev))) + netif_warn(efx, drv, efx->net_dev, + "Filter %lx not found to remove\n", tc->cookie); + NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules"); + return -ENOENT; + } + + /* Remove it from HW */ + efx_tc_delete_rule(efx, rule); + /* Delete it from SW */ + rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage, + efx_tc_match_action_ht_params); + netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie); + kfree(rule); + return 0; +} + +int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, + struct flow_cls_offload *tc, struct efx_rep *efv) +{ + int rc; + + if (!efx->tc) + return -EOPNOTSUPP; + + mutex_lock(&efx->tc->mutex); + switch (tc->command) { + case FLOW_CLS_REPLACE: + rc = efx_tc_flower_replace(efx, net_dev, tc, efv); + break; + case FLOW_CLS_DESTROY: + rc = efx_tc_flower_destroy(efx, net_dev, tc); + break; + default: + rc = -EOPNOTSUPP; + break; + } + mutex_unlock(&efx->tc->mutex); + return rc; +} + static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port, u32 eg_port, struct efx_tc_flow_rule *rule) { @@ -201,13 +577,37 @@ int efx_init_tc(struct efx_nic *efx) { int rc; + rc = efx_mae_get_caps(efx, efx->tc->caps); + if (rc) + return rc; + if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS) + /* Firmware supports some match fields the driver doesn't know + * about. Not fatal, unless any of those fields are required + * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know. + */ + netif_warn(efx, probe, efx->net_dev, + "FW reports additional match fields %u\n", + efx->tc->caps->match_field_count); + if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) { + netif_err(efx, probe, efx->net_dev, + "Too few action prios supported (have %u, need %u)\n", + efx->tc->caps->action_prios, EFX_TC_PRIO__NUM); + return -EIO; + } rc = efx_tc_configure_default_rule_pf(efx); if (rc) return rc; rc = efx_tc_configure_default_rule_wire(efx); if (rc) return rc; - return efx_tc_configure_rep_mport(efx); + rc = efx_tc_configure_rep_mport(efx); + if (rc) + return rc; + efx->tc->up = true; + rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx); + if (rc) + return rc; + return 0; } void efx_fini_tc(struct efx_nic *efx) @@ -215,20 +615,35 @@ void efx_fini_tc(struct efx_nic *efx) /* We can get called even if efx_init_struct_tc() failed */ if (!efx->tc) return; + if (efx->tc->up) + flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind); efx_tc_deconfigure_rep_mport(efx); efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf); efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire); + efx->tc->up = false; } int efx_init_struct_tc(struct efx_nic *efx) { + int rc; + if (efx->type->is_vf) return 0; efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL); if (!efx->tc) return -ENOMEM; + efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL); + if (!efx->tc->caps) { + rc = -ENOMEM; + goto fail_alloc_caps; + } + INIT_LIST_HEAD(&efx->tc->block_list); + mutex_init(&efx->tc->mutex); + rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params); + if (rc < 0) + goto fail_match_action_ht; efx->tc->reps_filter_uc = -1; efx->tc->reps_filter_mc = -1; INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list); @@ -236,6 +651,13 @@ int efx_init_struct_tc(struct efx_nic *efx) INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list); efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL; return 0; +fail_match_action_ht: + mutex_destroy(&efx->tc->mutex); + kfree(efx->tc->caps); +fail_alloc_caps: + kfree(efx->tc); + efx->tc = NULL; + return rc; } void efx_fini_struct_tc(struct efx_nic *efx) @@ -243,10 +665,16 @@ void efx_fini_struct_tc(struct efx_nic *efx) if (!efx->tc) return; + mutex_lock(&efx->tc->mutex); EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL); + rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free, + efx); + mutex_unlock(&efx->tc->mutex); + mutex_destroy(&efx->tc->mutex); + kfree(efx->tc->caps); kfree(efx->tc); efx->tc = NULL; } diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h index 309123c6b3867059e56c1d768f3d9c5dd1e20feb..196fd74ed97338d5a4e13ff188411d97d86475ae 100644 --- a/drivers/net/ethernet/sfc/tc.h +++ b/drivers/net/ethernet/sfc/tc.h @@ -11,8 +11,28 @@ #ifndef EFX_TC_H #define EFX_TC_H +#include +#include #include "net_driver.h" +/* Error reporting: convenience macros. For indicating why a given filter + * insertion is not supported; errors in internal operation or in the + * hardware should be netif_err()s instead. + */ +/* Used when error message is constant. */ +#define EFX_TC_ERR_MSG(efx, extack, message) do { \ + NL_SET_ERR_MSG_MOD(extack, message); \ + if (efx->log_tc_errs) \ + netif_info(efx, drv, efx->net_dev, "%s\n", message); \ +} while (0) +/* Used when error message is not constant; caller should also supply a + * constant extack message with NL_SET_ERR_MSG_MOD(). + */ +#define efx_tc_err(efx, fmt, args...) do { \ +if (efx->log_tc_errs) \ + netif_info(efx, drv, efx->net_dev, fmt, ##args);\ +} while (0) + struct efx_tc_action_set { u16 deliver:1; u32 dest_mport; @@ -23,6 +43,7 @@ struct efx_tc_action_set { struct efx_tc_match_fields { /* L1 */ u32 ingress_port; + u8 recirc_id; }; struct efx_tc_match { @@ -36,12 +57,15 @@ struct efx_tc_action_set_list { }; struct efx_tc_flow_rule { + unsigned long cookie; + struct rhash_head linkage; struct efx_tc_match match; struct efx_tc_action_set_list acts; u32 fw_id; }; enum efx_tc_rule_prios { + EFX_TC_PRIO_TC, /* Rule inserted by TC */ EFX_TC_PRIO_DFLT, /* Default switch rule; one of efx_tc_default_rules */ EFX_TC_PRIO__NUM }; @@ -49,6 +73,10 @@ enum efx_tc_rule_prios { /** * struct efx_tc_state - control plane data for TC offload * + * @caps: MAE capabilities reported by MCDI + * @block_list: List of &struct efx_tc_block_binding + * @mutex: Used to serialise operations on TC hashtables + * @match_action_ht: Hashtable of TC match-action rules * @reps_mport_id: MAE port allocated for representor RX * @reps_filter_uc: VNIC filter for representor unicast RX (promisc) * @reps_filter_mc: VNIC filter for representor multicast RX (allmulti) @@ -57,14 +85,20 @@ enum efx_tc_rule_prios { * %EFX_TC_PRIO_DFLT. Named by *ingress* port * @dflt.pf: rule for traffic ingressing from PF (egresses to wire) * @dflt.wire: rule for traffic ingressing from wire (egresses to PF) + * @up: have TC datastructures been set up? */ struct efx_tc_state { + struct mae_caps *caps; + struct list_head block_list; + struct mutex mutex; + struct rhashtable match_action_ht; u32 reps_mport_id, reps_mport_vport_id; s32 reps_filter_uc, reps_filter_mc; struct { struct efx_tc_flow_rule pf; struct efx_tc_flow_rule wire; } dflt; + bool up; }; struct efx_rep; @@ -72,6 +106,8 @@ struct efx_rep; int efx_tc_configure_default_rule_rep(struct efx_rep *efv); void efx_tc_deconfigure_default_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule); +int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev, + struct flow_cls_offload *tc, struct efx_rep *efv); int efx_tc_insert_rep_filters(struct efx_nic *efx); void efx_tc_remove_rep_filters(struct efx_nic *efx); diff --git a/drivers/net/ethernet/sfc/tc_bindings.c b/drivers/net/ethernet/sfc/tc_bindings.c new file mode 100644 index 0000000000000000000000000000000000000000..c18d64519c2db92faf371e321a18ea0a46752488 --- /dev/null +++ b/drivers/net/ethernet/sfc/tc_bindings.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include "tc_bindings.h" +#include "tc.h" + +struct efx_tc_block_binding { + struct list_head list; + struct efx_nic *efx; + struct efx_rep *efv; + struct net_device *otherdev; /* may actually be us */ + struct flow_block *block; +}; + +static struct efx_tc_block_binding *efx_tc_find_binding(struct efx_nic *efx, + struct net_device *otherdev) +{ + struct efx_tc_block_binding *binding; + + ASSERT_RTNL(); + list_for_each_entry(binding, &efx->tc->block_list, list) + if (binding->otherdev == otherdev) + return binding; + return NULL; +} + +static int efx_tc_block_cb(enum tc_setup_type type, void *type_data, + void *cb_priv) +{ + struct efx_tc_block_binding *binding = cb_priv; + struct flow_cls_offload *tcf = type_data; + + switch (type) { + case TC_SETUP_CLSFLOWER: + return efx_tc_flower(binding->efx, binding->otherdev, + tcf, binding->efv); + default: + return -EOPNOTSUPP; + } +} + +void efx_tc_block_unbind(void *cb_priv) +{ + struct efx_tc_block_binding *binding = cb_priv; + + list_del(&binding->list); + kfree(binding); +} + +static struct efx_tc_block_binding *efx_tc_create_binding( + struct efx_nic *efx, struct efx_rep *efv, + struct net_device *otherdev, struct flow_block *block) +{ + struct efx_tc_block_binding *binding = kmalloc(sizeof(*binding), GFP_KERNEL); + + if (!binding) + return ERR_PTR(-ENOMEM); + binding->efx = efx; + binding->efv = efv; + binding->otherdev = otherdev; + binding->block = block; + list_add(&binding->list, &efx->tc->block_list); + return binding; +} + +int efx_tc_setup_block(struct net_device *net_dev, struct efx_nic *efx, + struct flow_block_offload *tcb, struct efx_rep *efv) +{ + struct efx_tc_block_binding *binding; + struct flow_block_cb *block_cb; + int rc; + + if (tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) + return -EOPNOTSUPP; + + if (WARN_ON(!efx->tc)) + return -ENETDOWN; + + switch (tcb->command) { + case FLOW_BLOCK_BIND: + binding = efx_tc_create_binding(efx, efv, net_dev, tcb->block); + if (IS_ERR(binding)) + return PTR_ERR(binding); + block_cb = flow_block_cb_alloc(efx_tc_block_cb, binding, + binding, efx_tc_block_unbind); + rc = PTR_ERR_OR_ZERO(block_cb); + netif_dbg(efx, drv, efx->net_dev, + "bind %sdirect block for device %s, rc %d\n", + net_dev == efx->net_dev ? "" : + efv ? "semi" : "in", + net_dev ? net_dev->name : NULL, rc); + if (rc) { + list_del(&binding->list); + kfree(binding); + } else { + flow_block_cb_add(block_cb, tcb); + } + return rc; + case FLOW_BLOCK_UNBIND: + binding = efx_tc_find_binding(efx, net_dev); + if (binding) { + block_cb = flow_block_cb_lookup(tcb->block, + efx_tc_block_cb, + binding); + if (block_cb) { + flow_block_cb_remove(block_cb, tcb); + netif_dbg(efx, drv, efx->net_dev, + "unbound %sdirect block for device %s\n", + net_dev == efx->net_dev ? "" : + binding->efv ? "semi" : "in", + net_dev ? net_dev->name : NULL); + return 0; + } + } + /* If we're in driver teardown, then we expect to have + * already unbound all our blocks (we did it early while + * we still had MCDI to remove the filters), so getting + * unbind callbacks now isn't a problem. + */ + netif_cond_dbg(efx, drv, efx->net_dev, + !efx->tc->up, warn, + "%sdirect block unbind for device %s, was never bound\n", + net_dev == efx->net_dev ? "" : "in", + net_dev ? net_dev->name : NULL); + return -ENOENT; + default: + return -EOPNOTSUPP; + } +} + +int efx_tc_indr_setup_cb(struct net_device *net_dev, struct Qdisc *sch, + void *cb_priv, enum tc_setup_type type, + void *type_data, void *data, + void (*cleanup)(struct flow_block_cb *block_cb)) +{ + struct flow_block_offload *tcb = type_data; + struct efx_tc_block_binding *binding; + struct flow_block_cb *block_cb; + struct efx_nic *efx = cb_priv; + bool is_ovs_int_port; + int rc; + + if (!net_dev) + return -EOPNOTSUPP; + + if (tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS && + tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) + return -EOPNOTSUPP; + + is_ovs_int_port = netif_is_ovs_master(net_dev); + if (tcb->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS && + !is_ovs_int_port) + return -EOPNOTSUPP; + + if (is_ovs_int_port) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_BLOCK: + switch (tcb->command) { + case FLOW_BLOCK_BIND: + binding = efx_tc_create_binding(efx, NULL, net_dev, tcb->block); + if (IS_ERR(binding)) + return PTR_ERR(binding); + block_cb = flow_indr_block_cb_alloc(efx_tc_block_cb, binding, + binding, efx_tc_block_unbind, + tcb, net_dev, sch, data, binding, + cleanup); + rc = PTR_ERR_OR_ZERO(block_cb); + netif_dbg(efx, drv, efx->net_dev, + "bind indr block for device %s, rc %d\n", + net_dev ? net_dev->name : NULL, rc); + if (rc) { + list_del(&binding->list); + kfree(binding); + } else { + flow_block_cb_add(block_cb, tcb); + } + return rc; + case FLOW_BLOCK_UNBIND: + binding = efx_tc_find_binding(efx, net_dev); + if (!binding) + return -ENOENT; + block_cb = flow_block_cb_lookup(tcb->block, + efx_tc_block_cb, + binding); + if (!block_cb) + return -ENOENT; + flow_indr_block_cb_remove(block_cb, tcb); + netif_dbg(efx, drv, efx->net_dev, + "unbind indr block for device %s\n", + net_dev ? net_dev->name : NULL); + return 0; + default: + return -EOPNOTSUPP; + } + default: + return -EOPNOTSUPP; + } +} + +/* .ndo_setup_tc implementation + * Entry point for flower block and filter management. + */ +int efx_tc_setup(struct net_device *net_dev, enum tc_setup_type type, + void *type_data) +{ + struct efx_nic *efx = efx_netdev_priv(net_dev); + + if (efx->type->is_vf) + return -EOPNOTSUPP; + if (!efx->tc) + return -EOPNOTSUPP; + + if (type == TC_SETUP_CLSFLOWER) + return efx_tc_flower(efx, net_dev, type_data, NULL); + if (type == TC_SETUP_BLOCK) + return efx_tc_setup_block(net_dev, efx, type_data, NULL); + + return -EOPNOTSUPP; +} diff --git a/drivers/net/ethernet/sfc/tc_bindings.h b/drivers/net/ethernet/sfc/tc_bindings.h new file mode 100644 index 0000000000000000000000000000000000000000..c210bb09150ee632734a89456512ec1348c362f6 --- /dev/null +++ b/drivers/net/ethernet/sfc/tc_bindings.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/**************************************************************************** + * Driver for Solarflare network controllers and boards + * Copyright 2022 Xilinx Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#ifndef EFX_TC_BINDINGS_H +#define EFX_TC_BINDINGS_H +#include "net_driver.h" + +#include + +struct efx_rep; + +void efx_tc_block_unbind(void *cb_priv); +int efx_tc_setup_block(struct net_device *net_dev, struct efx_nic *efx, + struct flow_block_offload *tcb, struct efx_rep *efv); +int efx_tc_setup(struct net_device *net_dev, enum tc_setup_type type, + void *type_data); + +int efx_tc_indr_setup_cb(struct net_device *net_dev, struct Qdisc *sch, + void *cb_priv, enum tc_setup_type type, + void *type_data, void *data, + void (*cleanup)(struct flow_block_cb *block_cb)); +#endif /* EFX_TC_BINDINGS_H */ diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index d12474042c842c186d88f42ba9baa2f7b69abdf1..c5f88f7a7a047db15d67803f2658b9a71eff7a68 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -549,7 +549,7 @@ netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb, * previous packets out. */ if (!netdev_xmit_more()) - efx_tx_send_pending(tx_queue->channel); + efx_tx_send_pending(efx_get_tx_channel(efx, index)); return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/sgi/ioc3-eth.c b/drivers/net/ethernet/sgi/ioc3-eth.c index e2d009866a7b34036d276dda15dd9555e13763bf..8fc3f5272fa70e619e8b3a36151686ea52c94358 100644 --- a/drivers/net/ethernet/sgi/ioc3-eth.c +++ b/drivers/net/ethernet/sgi/ioc3-eth.c @@ -1158,9 +1158,9 @@ static inline unsigned int ioc3_hash(const unsigned char *addr) static void ioc3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, IOC3_NAME, sizeof(info->driver)); - strlcpy(info->version, IOC3_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(to_pci_dev(dev->dev.parent)), + strscpy(info->driver, IOC3_NAME, sizeof(info->driver)); + strscpy(info->version, IOC3_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(to_pci_dev(dev->dev.parent)), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/sis/sis190.c b/drivers/net/ethernet/sis/sis190.c index 216bb2d34d7c8f1607b7d3824204afae82185129..dda4e488c77a103c0bfe8508e72864d00d97b434 100644 --- a/drivers/net/ethernet/sis/sis190.c +++ b/drivers/net/ethernet/sis/sis190.c @@ -1769,9 +1769,9 @@ static void sis190_get_drvinfo(struct net_device *dev, { struct sis190_private *tp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(tp->pci_dev), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/sis/sis900.c b/drivers/net/ethernet/sis/sis900.c index 23a336c5096ed7c2f553a93bb7ac4874f4a34b89..cb7fec226cab63451f1c5e31287d67181f4f8a95 100644 --- a/drivers/net/ethernet/sis/sis900.c +++ b/drivers/net/ethernet/sis/sis900.c @@ -2027,9 +2027,9 @@ static void sis900_get_drvinfo(struct net_device *net_dev, { struct sis900_private *sis_priv = netdev_priv(net_dev); - strlcpy(info->driver, SIS900_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->version, SIS900_DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(sis_priv->pci_dev), + strscpy(info->driver, SIS900_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, SIS900_DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(sis_priv->pci_dev), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/smsc/epic100.c b/drivers/net/ethernet/smsc/epic100.c index 0329caf63279c54b6d6614336aec107edd6b972c..013e90d691829410cd1b6027f6dbef2980983e23 100644 --- a/drivers/net/ethernet/smsc/epic100.c +++ b/drivers/net/ethernet/smsc/epic100.c @@ -482,7 +482,7 @@ static int epic_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->netdev_ops = &epic_netdev_ops; dev->ethtool_ops = &netdev_ethtool_ops; dev->watchdog_timeo = TX_TIMEOUT; - netif_napi_add(dev, &ep->napi, epic_poll, 64); + netif_napi_add(dev, &ep->napi, epic_poll); ret = register_netdev(dev); if (ret < 0) @@ -1392,9 +1392,9 @@ static void netdev_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo * { struct epic_private *np = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/smsc/smc911x.c b/drivers/net/ethernet/smsc/smc911x.c index 24d66af797d465ca0b5d28bc74ad3fa24888a707..52ecfb461c4195002902ddb38a939d68a9410553 100644 --- a/drivers/net/ethernet/smsc/smc911x.c +++ b/drivers/net/ethernet/smsc/smc911x.c @@ -1509,9 +1509,9 @@ smc911x_ethtool_set_link_ksettings(struct net_device *dev, static void smc911x_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, CARDNAME, sizeof(info->driver)); - strlcpy(info->version, version, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(dev->dev.parent), + strscpy(info->driver, CARDNAME, sizeof(info->driver)); + strscpy(info->version, version, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/smsc/smc91c92_cs.c b/drivers/net/ethernet/smsc/smc91c92_cs.c index 37c822e27207a000e81e5ae9c6349a4bdaa78b77..29bb19f42de9f58661d5af3550ef78603fa373b0 100644 --- a/drivers/net/ethernet/smsc/smc91c92_cs.c +++ b/drivers/net/ethernet/smsc/smc91c92_cs.c @@ -1909,8 +1909,8 @@ static int check_if_running(struct net_device *dev) static void smc_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static int smc_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c index a31c159e96eaa46bf492a8785659689d1423de87..35e99bf0c40153da155327b322c5c45bade71c6f 100644 --- a/drivers/net/ethernet/smsc/smc91x.c +++ b/drivers/net/ethernet/smsc/smc91x.c @@ -1588,9 +1588,9 @@ smc_ethtool_set_link_ksettings(struct net_device *dev, static void smc_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, CARDNAME, sizeof(info->driver)); - strlcpy(info->version, version, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(dev->dev.parent), + strscpy(info->driver, CARDNAME, sizeof(info->driver)); + strscpy(info->version, version, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c index 3bf20211cceb4529ee0063e310457343dbe7ae07..a2e511912e6a9f0d1476170f4adbfff5b6a9a07d 100644 --- a/drivers/net/ethernet/smsc/smsc911x.c +++ b/drivers/net/ethernet/smsc/smsc911x.c @@ -1037,6 +1037,8 @@ static int smsc911x_mii_probe(struct net_device *dev) return ret; } + /* Indicate that the MAC is responsible for managing PHY PM */ + phydev->mac_managed_pm = true; phy_attached_info(phydev); phy_set_max_speed(phydev, SPEED_100); @@ -1953,9 +1955,9 @@ static int smsc911x_set_mac_address(struct net_device *dev, void *p) static void smsc911x_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, SMSC_CHIPNAME, sizeof(info->driver)); - strlcpy(info->version, SMSC_DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(dev->dev.parent), + strscpy(info->driver, SMSC_CHIPNAME, sizeof(info->driver)); + strscpy(info->version, SMSC_DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(dev->dev.parent), sizeof(info->bus_info)); } @@ -2587,6 +2589,8 @@ static int smsc911x_suspend(struct device *dev) if (netif_running(ndev)) { netif_stop_queue(ndev); netif_device_detach(ndev); + if (!device_may_wakeup(dev)) + phy_stop(ndev->phydev); } /* enable wake on LAN, energy detection and the external PME @@ -2628,6 +2632,8 @@ static int smsc911x_resume(struct device *dev) if (netif_running(ndev)) { netif_device_attach(ndev); netif_start_queue(ndev); + if (!device_may_wakeup(dev)) + phy_start(ndev->phydev); } return 0; diff --git a/drivers/net/ethernet/smsc/smsc9420.c b/drivers/net/ethernet/smsc/smsc9420.c index 0c68c7f8056dc2b1772a9eaf8cf6483d8caeeaa9..71fbb358bb7d2bd7395b7ac0dd50d9cb8a250bf9 100644 --- a/drivers/net/ethernet/smsc/smsc9420.c +++ b/drivers/net/ethernet/smsc/smsc9420.c @@ -215,10 +215,10 @@ static void smsc9420_ethtool_get_drvinfo(struct net_device *netdev, { struct smsc9420_pdata *pd = netdev_priv(netdev); - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(pd->pdev), + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, pci_name(pd->pdev), sizeof(drvinfo->bus_info)); - strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); } static u32 smsc9420_ethtool_get_msglevel(struct net_device *netdev) @@ -1585,7 +1585,7 @@ smsc9420_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev->netdev_ops = &smsc9420_netdev_ops; dev->ethtool_ops = &smsc9420_ethtool_ops; - netif_napi_add(dev, &pd->napi, smsc9420_rx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &pd->napi, smsc9420_rx_poll); result = register_netdev(dev); if (result) { diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index b0c5a44785fa4733e50ab64e8af69f71499518e5..2240f6d0b89badbeafc4323efce6e2faa19e54ca 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -526,8 +526,8 @@ static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr) static void netsec_et_get_drvinfo(struct net_device *net_device, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "netsec", sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(net_device->dev.parent), + strscpy(info->driver, "netsec", sizeof(info->driver)); + strscpy(info->bus_info, dev_name(net_device->dev.parent), sizeof(info->bus_info)); } @@ -2093,7 +2093,7 @@ static int netsec_probe(struct platform_device *pdev) dev_info(&pdev->dev, "hardware revision %d.%d\n", hw_ver >> 16, hw_ver & 0xffff); - netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, netsec_napi_poll); ndev->netdev_ops = &netsec_netdev_ops; ndev->ethtool_ops = &netsec_ethtool_ops; diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index f0c8de2c607550f6f4818cb0e7e6746291790f3f..1fa09b49ba7fa8bcae6ca2cae26daa710a6ae349 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -395,8 +395,8 @@ static void ave_ethtool_get_drvinfo(struct net_device *ndev, { struct device *dev = ndev->dev.parent; - strlcpy(info->driver, dev->driver->name, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(dev), sizeof(info->bus_info)); + strscpy(info->driver, dev->driver->name, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(dev), sizeof(info->bus_info)); ave_hw_read_version(ndev, info->fw_version, sizeof(info->fw_version)); } @@ -1687,8 +1687,7 @@ static int ave_probe(struct platform_device *pdev) pdev->name, pdev->id); /* Register as a NAPI supported driver */ - netif_napi_add(ndev, &priv->napi_rx, ave_napi_poll_rx, - NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi_rx, ave_napi_poll_rx); netif_napi_add_tx(ndev, &priv->napi_tx, ave_napi_poll_tx); platform_set_drvdata(pdev, ndev); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index 358fc26f8d1fcd6b9cb17170fbb9e132162064ac..80efdeeb0b594c2be222e85e711b9bf60559af03 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -445,9 +445,7 @@ static int dwc_eth_dwmac_probe(struct platform_device *pdev) ret = data->probe(pdev, plat_dat, &stmmac_res); if (ret < 0) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to probe subdriver: %d\n", - ret); + dev_err_probe(&pdev->dev, ret, "failed to probe subdriver\n"); goto remove_config; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 4f2b82a884b9161e4af3880ebaf080956a35a4af..0a2afc1a3124e9d1b1fa53f143daa448ebd09d37 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -610,7 +610,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->int_snapshot_num = AUX_SNAPSHOT1; plat->ext_snapshot_num = AUX_SNAPSHOT0; - plat->has_crossts = true; plat->crosststamp = intel_crosststamp; plat->int_snapshot_en = 0; @@ -1136,8 +1135,6 @@ static void intel_eth_pci_remove(struct pci_dev *pdev) clk_disable_unprepare(priv->plat->stmmac_clk); clk_unregister_fixed_rate(priv->plat->stmmac_clk); - - pcim_iounmap_regions(pdev, BIT(0)); } static int __maybe_unused intel_eth_pci_suspend(struct device *dev) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index c469abc91fa1b04966f276042c60e1fa0f8696a3..f7269d79a3851d41779d96c510f23c2b1dba75c6 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -32,6 +32,8 @@ struct rk_gmac_ops { void (*set_to_rmii)(struct rk_priv_data *bsp_priv); void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed); void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed); + void (*set_clock_selection)(struct rk_priv_data *bsp_priv, bool input, + bool enable); void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv); bool regs_valid; u32 regs[]; @@ -66,6 +68,7 @@ struct rk_priv_data { int rx_delay; struct regmap *grf; + struct regmap *php_grf; }; #define HIWORD_UPDATE(val, mask, shift) \ @@ -1101,6 +1104,147 @@ static const struct rk_gmac_ops rk3568_ops = { }, }; +/* sys_grf */ +#define RK3588_GRF_GMAC_CON7 0X031c +#define RK3588_GRF_GMAC_CON8 0X0320 +#define RK3588_GRF_GMAC_CON9 0X0324 + +#define RK3588_GMAC_RXCLK_DLY_ENABLE(id) GRF_BIT(2 * (id) + 3) +#define RK3588_GMAC_RXCLK_DLY_DISABLE(id) GRF_CLR_BIT(2 * (id) + 3) +#define RK3588_GMAC_TXCLK_DLY_ENABLE(id) GRF_BIT(2 * (id) + 2) +#define RK3588_GMAC_TXCLK_DLY_DISABLE(id) GRF_CLR_BIT(2 * (id) + 2) + +#define RK3588_GMAC_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0xFF, 8) +#define RK3588_GMAC_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0xFF, 0) + +/* php_grf */ +#define RK3588_GRF_GMAC_CON0 0X0008 +#define RK3588_GRF_CLK_CON1 0X0070 + +#define RK3588_GMAC_PHY_INTF_SEL_RGMII(id) \ + (GRF_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_CLR_BIT(5 + (id) * 6)) +#define RK3588_GMAC_PHY_INTF_SEL_RMII(id) \ + (GRF_CLR_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_BIT(5 + (id) * 6)) + +#define RK3588_GMAC_CLK_RMII_MODE(id) GRF_BIT(5 * (id)) +#define RK3588_GMAC_CLK_RGMII_MODE(id) GRF_CLR_BIT(5 * (id)) + +#define RK3588_GMAC_CLK_SELET_CRU(id) GRF_BIT(5 * (id) + 4) +#define RK3588_GMAC_CLK_SELET_IO(id) GRF_CLR_BIT(5 * (id) + 4) + +#define RK3588_GMA_CLK_RMII_DIV2(id) GRF_BIT(5 * (id) + 2) +#define RK3588_GMA_CLK_RMII_DIV20(id) GRF_CLR_BIT(5 * (id) + 2) + +#define RK3588_GMAC_CLK_RGMII_DIV1(id) \ + (GRF_CLR_BIT(5 * (id) + 2) | GRF_CLR_BIT(5 * (id) + 3)) +#define RK3588_GMAC_CLK_RGMII_DIV5(id) \ + (GRF_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3)) +#define RK3588_GMAC_CLK_RGMII_DIV50(id) \ + (GRF_CLR_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3)) + +#define RK3588_GMAC_CLK_RMII_GATE(id) GRF_BIT(5 * (id) + 1) +#define RK3588_GMAC_CLK_RMII_NOGATE(id) GRF_CLR_BIT(5 * (id) + 1) + +static void rk3588_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + u32 offset_con, id = bsp_priv->id; + + if (IS_ERR(bsp_priv->grf) || IS_ERR(bsp_priv->php_grf)) { + dev_err(dev, "Missing rockchip,grf or rockchip,php_grf property\n"); + return; + } + + offset_con = bsp_priv->id == 1 ? RK3588_GRF_GMAC_CON9 : + RK3588_GRF_GMAC_CON8; + + regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0, + RK3588_GMAC_PHY_INTF_SEL_RGMII(id)); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, + RK3588_GMAC_CLK_RGMII_MODE(id)); + + regmap_write(bsp_priv->grf, RK3588_GRF_GMAC_CON7, + RK3588_GMAC_RXCLK_DLY_ENABLE(id) | + RK3588_GMAC_TXCLK_DLY_ENABLE(id)); + + regmap_write(bsp_priv->grf, offset_con, + RK3588_GMAC_CLK_RX_DL_CFG(rx_delay) | + RK3588_GMAC_CLK_TX_DL_CFG(tx_delay)); +} + +static void rk3588_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->php_grf)) { + dev_err(dev, "%s: Missing rockchip,php_grf property\n", __func__); + return; + } + + regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0, + RK3588_GMAC_PHY_INTF_SEL_RMII(bsp_priv->id)); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, + RK3588_GMAC_CLK_RMII_MODE(bsp_priv->id)); +} + +static void rk3588_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned int val = 0, id = bsp_priv->id; + + switch (speed) { + case 10: + if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII) + val = RK3588_GMA_CLK_RMII_DIV20(id); + else + val = RK3588_GMAC_CLK_RGMII_DIV50(id); + break; + case 100: + if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII) + val = RK3588_GMA_CLK_RMII_DIV2(id); + else + val = RK3588_GMAC_CLK_RGMII_DIV5(id); + break; + case 1000: + if (bsp_priv->phy_iface != PHY_INTERFACE_MODE_RMII) + val = RK3588_GMAC_CLK_RGMII_DIV1(id); + else + goto err; + break; + default: + goto err; + } + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val); + + return; +err: + dev_err(dev, "unknown speed value for GMAC speed=%d", speed); +} + +static void rk3588_set_clock_selection(struct rk_priv_data *bsp_priv, bool input, + bool enable) +{ + unsigned int val = input ? RK3588_GMAC_CLK_SELET_IO(bsp_priv->id) : + RK3588_GMAC_CLK_SELET_CRU(bsp_priv->id); + + val |= enable ? RK3588_GMAC_CLK_RMII_NOGATE(bsp_priv->id) : + RK3588_GMAC_CLK_RMII_GATE(bsp_priv->id); + + regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val); +} + +static const struct rk_gmac_ops rk3588_ops = { + .set_to_rgmii = rk3588_set_to_rgmii, + .set_to_rmii = rk3588_set_to_rmii, + .set_rgmii_speed = rk3588_set_gmac_speed, + .set_rmii_speed = rk3588_set_gmac_speed, + .set_clock_selection = rk3588_set_clock_selection, +}; + #define RV1108_GRF_GMAC_CON0 0X0900 /* RV1108_GRF_GMAC_CON0 */ @@ -1153,6 +1297,130 @@ static const struct rk_gmac_ops rv1108_ops = { .set_rmii_speed = rv1108_set_rmii_speed, }; +#define RV1126_GRF_GMAC_CON0 0X0070 +#define RV1126_GRF_GMAC_CON1 0X0074 +#define RV1126_GRF_GMAC_CON2 0X0078 + +/* RV1126_GRF_GMAC_CON0 */ +#define RV1126_GMAC_PHY_INTF_SEL_RGMII \ + (GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6)) +#define RV1126_GMAC_PHY_INTF_SEL_RMII \ + (GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6)) +#define RV1126_GMAC_FLOW_CTRL GRF_BIT(7) +#define RV1126_GMAC_FLOW_CTRL_CLR GRF_CLR_BIT(7) +#define RV1126_GMAC_M0_RXCLK_DLY_ENABLE GRF_BIT(1) +#define RV1126_GMAC_M0_RXCLK_DLY_DISABLE GRF_CLR_BIT(1) +#define RV1126_GMAC_M0_TXCLK_DLY_ENABLE GRF_BIT(0) +#define RV1126_GMAC_M0_TXCLK_DLY_DISABLE GRF_CLR_BIT(0) +#define RV1126_GMAC_M1_RXCLK_DLY_ENABLE GRF_BIT(3) +#define RV1126_GMAC_M1_RXCLK_DLY_DISABLE GRF_CLR_BIT(3) +#define RV1126_GMAC_M1_TXCLK_DLY_ENABLE GRF_BIT(2) +#define RV1126_GMAC_M1_TXCLK_DLY_DISABLE GRF_CLR_BIT(2) + +/* RV1126_GRF_GMAC_CON1 */ +#define RV1126_GMAC_M0_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RV1126_GMAC_M0_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) +/* RV1126_GRF_GMAC_CON2 */ +#define RV1126_GMAC_M1_CLK_RX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 8) +#define RV1126_GMAC_M1_CLK_TX_DL_CFG(val) HIWORD_UPDATE(val, 0x7F, 0) + +static void rv1126_set_to_rgmii(struct rk_priv_data *bsp_priv, + int tx_delay, int rx_delay) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "Missing rockchip,grf property\n"); + return; + } + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0, + RV1126_GMAC_PHY_INTF_SEL_RGMII | + RV1126_GMAC_M0_RXCLK_DLY_ENABLE | + RV1126_GMAC_M0_TXCLK_DLY_ENABLE | + RV1126_GMAC_M1_RXCLK_DLY_ENABLE | + RV1126_GMAC_M1_TXCLK_DLY_ENABLE); + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON1, + RV1126_GMAC_M0_CLK_RX_DL_CFG(rx_delay) | + RV1126_GMAC_M0_CLK_TX_DL_CFG(tx_delay)); + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON2, + RV1126_GMAC_M1_CLK_RX_DL_CFG(rx_delay) | + RV1126_GMAC_M1_CLK_TX_DL_CFG(tx_delay)); +} + +static void rv1126_set_to_rmii(struct rk_priv_data *bsp_priv) +{ + struct device *dev = &bsp_priv->pdev->dev; + + if (IS_ERR(bsp_priv->grf)) { + dev_err(dev, "%s: Missing rockchip,grf property\n", __func__); + return; + } + + regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0, + RV1126_GMAC_PHY_INTF_SEL_RMII); +} + +static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + case 1000: + rate = 125000000; + break; + default: + dev_err(dev, "unknown speed value for RGMII speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static void rv1126_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed) +{ + struct device *dev = &bsp_priv->pdev->dev; + unsigned long rate; + int ret; + + switch (speed) { + case 10: + rate = 2500000; + break; + case 100: + rate = 25000000; + break; + default: + dev_err(dev, "unknown speed value for RGMII speed=%d", speed); + return; + } + + ret = clk_set_rate(bsp_priv->clk_mac_speed, rate); + if (ret) + dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n", + __func__, rate, ret); +} + +static const struct rk_gmac_ops rv1126_ops = { + .set_to_rgmii = rv1126_set_to_rgmii, + .set_to_rmii = rv1126_set_to_rmii, + .set_rgmii_speed = rv1126_set_rgmii_speed, + .set_rmii_speed = rv1126_set_rmii_speed, +}; + #define RK_GRF_MACPHY_CON0 0xb00 #define RK_GRF_MACPHY_CON1 0xb04 #define RK_GRF_MACPHY_CON2 0xb08 @@ -1304,6 +1572,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) if (!IS_ERR(bsp_priv->clk_mac_speed)) clk_prepare_enable(bsp_priv->clk_mac_speed); + if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) + bsp_priv->ops->set_clock_selection(bsp_priv, + bsp_priv->clock_input, true); + /** * if (!IS_ERR(bsp_priv->clk_mac)) * clk_prepare_enable(bsp_priv->clk_mac); @@ -1330,6 +1602,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable) clk_disable_unprepare(bsp_priv->mac_clk_tx); clk_disable_unprepare(bsp_priv->clk_mac_speed); + + if (bsp_priv->ops && bsp_priv->ops->set_clock_selection) + bsp_priv->ops->set_clock_selection(bsp_priv, + bsp_priv->clock_input, false); /** * if (!IS_ERR(bsp_priv->clk_mac)) * clk_disable_unprepare(bsp_priv->clk_mac); @@ -1444,6 +1720,8 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev, bsp_priv->grf = syscon_regmap_lookup_by_phandle(dev->of_node, "rockchip,grf"); + bsp_priv->php_grf = syscon_regmap_lookup_by_phandle(dev->of_node, + "rockchip,php-grf"); if (plat->phy_node) { bsp_priv->integrated_phy = of_property_read_bool(plat->phy_node, @@ -1680,7 +1958,9 @@ static const struct of_device_id rk_gmac_dwmac_match[] = { { .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops }, { .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops }, { .compatible = "rockchip,rk3568-gmac", .data = &rk3568_ops }, + { .compatible = "rockchip,rk3588-gmac", .data = &rk3588_ops }, { .compatible = "rockchip,rv1108-gmac", .data = &rv1108_ops }, + { .compatible = "rockchip,rv1126-gmac", .data = &rv1126_ops }, { } }; MODULE_DEVICE_TABLE(of, rk_gmac_dwmac_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h index 35ab8d0bdce71c6223f6905b876e230177a66db8..7ab791c8d355fb056a9f92b1747ddeee579a97d7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h @@ -56,7 +56,7 @@ #define MAC_CONTROL_TE 0x00000008 /* Transmitter Enable */ #define MAC_CONTROL_RE 0x00000004 /* Receiver Enable */ -#define MAC_CORE_INIT (MAC_CONTROL_HBD | MAC_CONTROL_ASTP) +#define MAC_CORE_INIT (MAC_CONTROL_HBD) /* MAC FLOW CTRL defines */ #define MAC_FLOW_CTRL_PT_MASK 0xffff0000 /* Pause Time Mask */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index 3c73453725f946b00d6d15aedbc934efe5cfa583..4296ddda8aaa6c91b44e3c0a1fddd13d1405e8e9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -126,7 +126,7 @@ enum inter_frame_gap { #define GMAC_CONTROL_TE 0x00000008 /* Transmitter Enable */ #define GMAC_CONTROL_RE 0x00000004 /* Receiver Enable */ -#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | GMAC_CONTROL_ACS | \ +#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | \ GMAC_CONTROL_BE | GMAC_CONTROL_DCRS) /* GMAC Frame Filter defines */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index 76edb9b726756ef7d91de344a8d01eee391bb110..0e00dd83d027afcfec6267857a5e28816d314350 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include "stmmac.h" #include "stmmac_pcs.h" @@ -24,7 +23,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw, struct net_device *dev) { - struct stmmac_priv *priv = netdev_priv(dev); void __iomem *ioaddr = hw->pcsr; u32 value = readl(ioaddr + GMAC_CONTROL); int mtu = dev->mtu; @@ -32,13 +30,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw, /* Configure GMAC core */ value |= GMAC_CORE_INIT; - /* Clear ACS bit because Ethernet switch tagging formats such as - * Broadcom tags can look like invalid LLC/SNAP packets and cause the - * hardware to truncate packets on reception. - */ - if (netdev_uses_dsa(dev) || !priv->plat->enh_desc) - value &= ~GMAC_CONTROL_ACS; - if (mtu > 1500) value |= GMAC_CONTROL_2K; if (mtu > 2000) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c index 75071a7d551a828e652c00c53f689b4f05ce278a..a6e8d7bd95886fc277c7e22c896ddf618e0fca97 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c @@ -15,7 +15,6 @@ *******************************************************************************/ #include -#include #include #include "stmmac.h" #include "dwmac100.h" @@ -28,13 +27,6 @@ static void dwmac100_core_init(struct mac_device_info *hw, value |= MAC_CORE_INIT; - /* Clear ASTP bit because Ethernet switch tagging formats such as - * Broadcom tags can look like invalid LLC/SNAP packets and cause the - * hardware to truncate packets on reception. - */ - if (netdev_uses_dsa(dev)) - value &= ~MAC_CONTROL_ASTP; - writel(value, ioaddr + MAC_CONTROL); #ifdef STMMAC_VLAN_TAG_USED diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index d8f1fbc25bdd3e7a7e66e9f7d983137492997e1a..c25bfecb4a2dfeff441293e75fda4e22ea0b1147 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -14,7 +14,6 @@ #include #include #include -#include #include "stmmac.h" #include "stmmac_pcs.h" #include "dwmac4.h" diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c index caa4bfc4c1d62effb20b08de3e7d495743d9eb53..9b6138b11776656a91e3fa0c293c5d13177fda7c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c @@ -258,14 +258,18 @@ EXPORT_SYMBOL_GPL(stmmac_set_mac_addr); /* Enable disable MAC RX/TX */ void stmmac_set_mac(void __iomem *ioaddr, bool enable) { - u32 value = readl(ioaddr + MAC_CTRL_REG); + u32 old_val, value; + + old_val = readl(ioaddr + MAC_CTRL_REG); + value = old_val; if (enable) value |= MAC_ENABLE_RX | MAC_ENABLE_TX; else value &= ~(MAC_ENABLE_TX | MAC_ENABLE_RX); - writel(value, ioaddr + MAC_CTRL_REG); + if (value != old_val) + writel(value, ioaddr + MAC_CTRL_REG); } void stmmac_get_mac_addr(void __iomem *ioaddr, unsigned char *addr, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index d6a44d53fe087b74ab14f4bbdf4272cbb9a0687a..f453b0d093663d5db152d217f8376943e3ce5cc1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -287,15 +287,15 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, struct stmmac_priv *priv = netdev_priv(dev); if (priv->plat->has_gmac || priv->plat->has_gmac4) - strlcpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); + strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver)); else if (priv->plat->has_xgmac) - strlcpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); + strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver)); else - strlcpy(info->driver, MAC100_ETHTOOL_NAME, + strscpy(info->driver, MAC100_ETHTOOL_NAME, sizeof(info->driver)); if (priv->plat->pdev) { - strlcpy(info->bus_info, pci_name(priv->plat->pdev), + strscpy(info->bus_info, pci_name(priv->plat->pdev), sizeof(info->bus_info)); } } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 070b5ef165eba4026d25a7d4f2476cbb5a42ed31..65c96773c6d2b2972cea2c6cc80128675e8ffa54 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -986,10 +986,10 @@ static void stmmac_mac_link_up(struct phylink_config *config, bool tx_pause, bool rx_pause) { struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); - u32 ctrl; + u32 old_ctrl, ctrl; - ctrl = readl(priv->ioaddr + MAC_CTRL_REG); - ctrl &= ~priv->hw->link.speed_mask; + old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); + ctrl = old_ctrl & ~priv->hw->link.speed_mask; if (interface == PHY_INTERFACE_MODE_USXGMII) { switch (speed) { @@ -1064,7 +1064,8 @@ static void stmmac_mac_link_up(struct phylink_config *config, if (tx_pause && rx_pause) stmmac_mac_flow_ctrl(priv, duplex); - writel(ctrl, priv->ioaddr + MAC_CTRL_REG); + if (ctrl != old_ctrl) + writel(ctrl, priv->ioaddr + MAC_CTRL_REG); stmmac_mac_set(priv, priv->ioaddr, true); if (phy && priv->dma_cap.eee) { @@ -3800,6 +3801,15 @@ static int __stmmac_open(struct net_device *dev, stmmac_reset_queues_param(priv); + if (priv->plat->serdes_powerup) { + ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); + if (ret < 0) { + netdev_err(priv->dev, "%s: Serdes powerup failed\n", + __func__); + goto init_error; + } + } + ret = stmmac_hw_setup(dev, true); if (ret < 0) { netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); @@ -3903,6 +3913,10 @@ static int stmmac_release(struct net_device *dev) /* Disable the MAC Rx/Tx */ stmmac_mac_set(priv, priv->ioaddr, false); + /* Powerdown Serdes if there is */ + if (priv->plat->serdes_powerdown) + priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); + netif_carrier_off(dev); stmmac_release_ptp(priv); @@ -5075,16 +5089,8 @@ read_again: buf1_len = stmmac_rx_buf1_len(priv, p, status, len); len += buf1_len; - /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 - * Type frames (LLC/LLC-SNAP) - * - * llc_snap is never checked in GMAC >= 4, so this ACS - * feature is always disabled and packets need to be - * stripped manually. - */ - if (likely(!(status & rx_not_ls)) && - (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || - unlikely(status != llc_snap))) { + /* ACS is disabled; strip manually. */ + if (likely(!(status & rx_not_ls))) { buf1_len -= ETH_FCS_LEN; len -= ETH_FCS_LEN; } @@ -5261,16 +5267,8 @@ read_again: buf2_len = stmmac_rx_buf2_len(priv, p, status, len); len += buf2_len; - /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3 - * Type frames (LLC/LLC-SNAP) - * - * llc_snap is never checked in GMAC >= 4, so this ACS - * feature is always disabled and packets need to be - * stripped manually. - */ - if (likely(!(status & rx_not_ls)) && - (likely(priv->synopsys_id >= DWMAC_CORE_4_00) || - unlikely(status != llc_snap))) { + /* ACS is disabled; strip manually. */ + if (likely(!(status & rx_not_ls))) { if (buf2_len) { buf2_len -= ETH_FCS_LEN; len -= ETH_FCS_LEN; @@ -6889,8 +6887,7 @@ static void stmmac_napi_add(struct net_device *dev) spin_lock_init(&ch->lock); if (queue < priv->plat->rx_queues_to_use) { - netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); } if (queue < priv->plat->tx_queues_to_use) { netif_napi_add_tx(dev, &ch->tx_napi, @@ -6899,8 +6896,7 @@ static void stmmac_napi_add(struct net_device *dev) if (queue < priv->plat->rx_queues_to_use && queue < priv->plat->tx_queues_to_use) { netif_napi_add(dev, &ch->rxtx_napi, - stmmac_napi_poll_rxtx, - NAPI_POLL_WEIGHT); + stmmac_napi_poll_rxtx); } } } @@ -7292,14 +7288,6 @@ int stmmac_dvr_probe(struct device *device, goto error_netdev_register; } - if (priv->plat->serdes_powerup) { - ret = priv->plat->serdes_powerup(ndev, - priv->plat->bsp_priv); - - if (ret < 0) - goto error_serdes_powerup; - } - #ifdef CONFIG_DEBUG_FS stmmac_init_fs(ndev); #endif @@ -7314,8 +7302,6 @@ int stmmac_dvr_probe(struct device *device, return ret; -error_serdes_powerup: - unregister_netdev(ndev); error_netdev_register: phylink_destroy(priv->phylink); error_xpcs_setup: diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 9f5cac4000da680d061c37171817fb8642abee55..50f6b4a14be4cbc027ee0844a5db896767f83873 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -440,11 +440,12 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) /* Default to phy auto-detection */ plat->phy_addr = -1; - /* Default to get clk_csr from stmmac_clk_crs_set(), + /* Default to get clk_csr from stmmac_clk_csr_set(), * or get clk_csr from device tree. */ plat->clk_csr = -1; - of_property_read_u32(np, "clk_csr", &plat->clk_csr); + if (of_property_read_u32(np, "snps,clk-csr", &plat->clk_csr)) + of_property_read_u32(np, "clk_csr", &plat->clk_csr); /* "snps,phy-addr" is not a standard property. Mark it as deprecated * and warn of its use. Remove this when phy node support is added. diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c index 0b08b0e085e8274adaca87fd36c0fa11e145755f..0aca193d9550d40f43a8536566d417cadb29faa9 100644 --- a/drivers/net/ethernet/sun/cassini.c +++ b/drivers/net/ethernet/sun/cassini.c @@ -4484,9 +4484,9 @@ static void cas_set_multicast(struct net_device *dev) static void cas_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct cas *cp = netdev_priv(dev); - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info)); } static int cas_get_link_ksettings(struct net_device *dev, @@ -5050,7 +5050,7 @@ static int cas_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->watchdog_timeo = CAS_TX_TIMEOUT; #ifdef USE_NAPI - netif_napi_add(dev, &cp->napi, cas_poll, 64); + netif_napi_add(dev, &cp->napi, cas_poll); #endif dev->irq = pdev->irq; dev->dma = 0; diff --git a/drivers/net/ethernet/sun/ldmvsw.c b/drivers/net/ethernet/sun/ldmvsw.c index 0cd8493b810fa7a2b81389f8ec09c5ae49a4eb43..8addee6d04bd803636c0e14577220332371632c2 100644 --- a/drivers/net/ethernet/sun/ldmvsw.c +++ b/drivers/net/ethernet/sun/ldmvsw.c @@ -63,8 +63,8 @@ static struct vio_version vsw_versions[] = { static void vsw_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); } static u32 vsw_get_msglevel(struct net_device *dev) @@ -354,8 +354,7 @@ static int vsw_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) dev_set_drvdata(&vdev->dev, port); - netif_napi_add(dev, &port->napi, sunvnet_poll_common, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &port->napi, sunvnet_poll_common); spin_lock_irqsave(&vp->lock, flags); list_add_rcu(&port->list, &vp->port_list); diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c index df70df29deeaa9228e23baf1005ff785c104ac80..e6144d963eaaadb3343906fac020b38f1734533f 100644 --- a/drivers/net/ethernet/sun/niu.c +++ b/drivers/net/ethernet/sun/niu.c @@ -6798,12 +6798,12 @@ static void niu_get_drvinfo(struct net_device *dev, struct niu *np = netdev_priv(dev); struct niu_vpd *vpd = &np->vpd; - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); snprintf(info->fw_version, sizeof(info->fw_version), "%d.%d", vpd->fcode_major, vpd->fcode_minor); if (np->parent->plat_type != PLAT_TYPE_NIU) - strlcpy(info->bus_info, pci_name(np->pdev), + strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info)); } @@ -9115,7 +9115,7 @@ static int niu_ldg_init(struct niu *np) for (i = 0; i < np->num_ldg; i++) { struct niu_ldg *lp = &np->ldg[i]; - netif_napi_add(np->dev, &lp->napi, niu_poll, 64); + netif_napi_add(np->dev, &lp->napi, niu_poll); lp->np = np; lp->ldg_num = ldg_num_map[i]; diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c index 531a6f449afaddab10d692407be0edd405c68ea9..34b94153bf0ce2989b067a6c6d9dace4c41a9cb9 100644 --- a/drivers/net/ethernet/sun/sunbmac.c +++ b/drivers/net/ethernet/sun/sunbmac.c @@ -1038,8 +1038,8 @@ static void bigmac_set_multicast(struct net_device *dev) /* Ethtool support... */ static void bigmac_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "sunbmac", sizeof(info->driver)); - strlcpy(info->version, "2.0", sizeof(info->version)); + strscpy(info->driver, "sunbmac", sizeof(info->driver)); + strscpy(info->version, "2.0", sizeof(info->version)); } static u32 bigmac_get_link(struct net_device *dev) diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c index a14591b41acbc016522505b8803644c0dc17dc0b..4154e68639aced155b9ce1e171b0ed8e7189562d 100644 --- a/drivers/net/ethernet/sun/sungem.c +++ b/drivers/net/ethernet/sun/sungem.c @@ -2521,9 +2521,9 @@ static void gem_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info { struct gem *gp = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(gp->pdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(gp->pdev), sizeof(info->bus_info)); } static int gem_get_link_ksettings(struct net_device *dev, @@ -2980,7 +2980,7 @@ static int gem_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_out_free_consistent; dev->netdev_ops = &gem_netdev_ops; - netif_napi_add(dev, &gp->napi, gem_poll, 64); + netif_napi_add(dev, &gp->napi, gem_poll); dev->ethtool_ops = &gem_ethtool_ops; dev->watchdog_timeo = 5 * HZ; dev->dma = 0; diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c index 8594ee839628bd9fa80f1d8a4f8476ba2eab6ef2..91f10f746dffdf446586adc35a0c23413c2561ea 100644 --- a/drivers/net/ethernet/sun/sunhme.c +++ b/drivers/net/ethernet/sun/sunhme.c @@ -61,15 +61,8 @@ #include "sunhme.h" #define DRV_NAME "sunhme" -#define DRV_VERSION "3.10" -#define DRV_RELDATE "August 26, 2008" -#define DRV_AUTHOR "David S. Miller (davem@davemloft.net)" -static char version[] = - DRV_NAME ".c:v" DRV_VERSION " " DRV_RELDATE " " DRV_AUTHOR "\n"; - -MODULE_VERSION(DRV_VERSION); -MODULE_AUTHOR(DRV_AUTHOR); +MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); MODULE_DESCRIPTION("Sun HappyMealEthernet(HME) 10/100baseT ethernet driver"); MODULE_LICENSE("GPL"); @@ -87,13 +80,17 @@ static struct quattro *qfe_sbus_list; static struct quattro *qfe_pci_list; #endif -#undef HMEDEBUG -#undef SXDEBUG -#undef RXDEBUG -#undef TXDEBUG -#undef TXLOGGING +#define hme_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__) +#define HMD hme_debug + +/* "Auto Switch Debug" aka phy debug */ +#if 1 +#define ASD hme_debug +#else +#define ASD(...) +#endif -#ifdef TXLOGGING +#if 0 struct hme_tx_logent { unsigned int tstamp; int tx_new, tx_old; @@ -128,46 +125,16 @@ static __inline__ void tx_dump_log(void) this = txlog_cur_entry; for (i = 0; i < TX_LOG_LEN; i++) { - printk("TXLOG[%d]: j[%08x] tx[N(%d)O(%d)] action[%08x] stat[%08x]\n", i, + pr_err("TXLOG[%d]: j[%08x] tx[N(%d)O(%d)] action[%08x] stat[%08x]\n", i, tx_log[this].tstamp, tx_log[this].tx_new, tx_log[this].tx_old, tx_log[this].action, tx_log[this].status); this = (this + 1) & (TX_LOG_LEN - 1); } } -static __inline__ void tx_dump_ring(struct happy_meal *hp) -{ - struct hmeal_init_block *hb = hp->happy_block; - struct happy_meal_txd *tp = &hb->happy_meal_txd[0]; - int i; - - for (i = 0; i < TX_RING_SIZE; i+=4) { - printk("TXD[%d..%d]: [%08x:%08x] [%08x:%08x] [%08x:%08x] [%08x:%08x]\n", - i, i + 4, - le32_to_cpu(tp[i].tx_flags), le32_to_cpu(tp[i].tx_addr), - le32_to_cpu(tp[i + 1].tx_flags), le32_to_cpu(tp[i + 1].tx_addr), - le32_to_cpu(tp[i + 2].tx_flags), le32_to_cpu(tp[i + 2].tx_addr), - le32_to_cpu(tp[i + 3].tx_flags), le32_to_cpu(tp[i + 3].tx_addr)); - } -} -#else -#define tx_add_log(hp, a, s) do { } while(0) -#define tx_dump_log() do { } while(0) -#define tx_dump_ring(hp) do { } while(0) -#endif - -#ifdef HMEDEBUG -#define HMD(x) printk x -#else -#define HMD(x) -#endif - -/* #define AUTO_SWITCH_DEBUG */ - -#ifdef AUTO_SWITCH_DEBUG -#define ASD(x) printk x #else -#define ASD(x) +#define tx_add_log(hp, a, s) +#define tx_dump_log() #endif #define DEFAULT_IPG0 16 /* For lance-mode only */ @@ -343,8 +310,6 @@ static int happy_meal_bb_read(struct happy_meal *hp, int retval = 0; int i; - ASD(("happy_meal_bb_read: reg=%d ", reg)); - /* Enable the MIF BitBang outputs. */ hme_write32(hp, tregs + TCVR_BBOENAB, 1); @@ -378,7 +343,7 @@ static int happy_meal_bb_read(struct happy_meal *hp, (void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal)); (void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal)); (void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal)); - ASD(("value=%x\n", retval)); + ASD("reg=%d value=%x\n", reg, retval); return retval; } @@ -389,7 +354,7 @@ static void happy_meal_bb_write(struct happy_meal *hp, u32 tmp; int i; - ASD(("happy_meal_bb_write: reg=%d value=%x\n", reg, value)); + ASD("reg=%d value=%x\n", reg, value); /* Enable the MIF BitBang outputs. */ hme_write32(hp, tregs + TCVR_BBOENAB, 1); @@ -433,14 +398,13 @@ static int happy_meal_tcvr_read(struct happy_meal *hp, int tries = TCVR_READ_TRIES; int retval; - ASD(("happy_meal_tcvr_read: reg=0x%02x ", reg)); if (hp->tcvr_type == none) { - ASD(("no transceiver, value=TCVR_FAILURE\n")); + ASD("no transceiver, value=TCVR_FAILURE\n"); return TCVR_FAILURE; } if (!(hp->happy_flags & HFLAG_FENABLE)) { - ASD(("doing bit bang\n")); + ASD("doing bit bang\n"); return happy_meal_bb_read(hp, tregs, reg); } @@ -449,11 +413,11 @@ static int happy_meal_tcvr_read(struct happy_meal *hp, while (!(hme_read32(hp, tregs + TCVR_FRAME) & 0x10000) && --tries) udelay(20); if (!tries) { - printk(KERN_ERR "happy meal: Aieee, transceiver MIF read bolixed\n"); + netdev_err(hp->dev, "Aieee, transceiver MIF read bolixed\n"); return TCVR_FAILURE; } retval = hme_read32(hp, tregs + TCVR_FRAME) & 0xffff; - ASD(("value=%04x\n", retval)); + ASD("reg=0x%02x value=%04x\n", reg, retval); return retval; } @@ -465,7 +429,7 @@ static void happy_meal_tcvr_write(struct happy_meal *hp, { int tries = TCVR_WRITE_TRIES; - ASD(("happy_meal_tcvr_write: reg=0x%02x value=%04x\n", reg, value)); + ASD("reg=0x%02x value=%04x\n", reg, value); /* Welcome to Sun Microsystems, can I take your order please? */ if (!(hp->happy_flags & HFLAG_FENABLE)) { @@ -482,7 +446,7 @@ static void happy_meal_tcvr_write(struct happy_meal *hp, /* Anything else? */ if (!tries) - printk(KERN_ERR "happy meal: Aieee, transceiver MIF write bolixed\n"); + netdev_err(hp->dev, "Aieee, transceiver MIF write bolixed\n"); /* Fifty-two cents is your change, have a nice day. */ } @@ -660,8 +624,8 @@ static void happy_meal_timer(struct timer_list *t) /* Enter force mode. */ do_force_mode: hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR); - printk(KERN_NOTICE "%s: Auto-Negotiation unsuccessful, trying force link mode\n", - hp->dev->name); + netdev_notice(hp->dev, + "Auto-Negotiation unsuccessful, trying force link mode\n"); hp->sw_bmcr = BMCR_SPEED100; happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr); @@ -720,8 +684,8 @@ static void happy_meal_timer(struct timer_list *t) restart_timer = 0; } else { if (hp->timer_ticks >= 10) { - printk(KERN_NOTICE "%s: Auto negotiation successful, link still " - "not completely up.\n", hp->dev->name); + netdev_notice(hp->dev, + "Auto negotiation successful, link still not completely up.\n"); hp->timer_ticks = 0; restart_timer = 1; } else { @@ -776,14 +740,14 @@ static void happy_meal_timer(struct timer_list *t) */ /* Let the user know... */ - printk(KERN_NOTICE "%s: Link down, cable problem?\n", - hp->dev->name); + netdev_notice(hp->dev, + "Link down, cable problem?\n"); ret = happy_meal_init(hp); if (ret) { /* ho hum... */ - printk(KERN_ERR "%s: Error, cannot re-init the " - "Happy Meal.\n", hp->dev->name); + netdev_err(hp->dev, + "Error, cannot re-init the Happy Meal.\n"); } goto out; } @@ -805,8 +769,8 @@ static void happy_meal_timer(struct timer_list *t) case asleep: default: /* Can't happens.... */ - printk(KERN_ERR "%s: Aieee, link timer is asleep but we got one anyways!\n", - hp->dev->name); + netdev_err(hp->dev, + "Aieee, link timer is asleep but we got one anyways!\n"); restart_timer = 0; hp->timer_ticks = 0; hp->timer_state = asleep; /* foo on you */ @@ -830,7 +794,7 @@ static void happy_meal_tx_reset(struct happy_meal *hp, void __iomem *bregs) { int tries = TX_RESET_TRIES; - HMD(("happy_meal_tx_reset: reset, ")); + HMD("reset...\n"); /* Would you like to try our SMCC Delux? */ hme_write32(hp, bregs + BMAC_TXSWRESET, 0); @@ -839,10 +803,10 @@ static void happy_meal_tx_reset(struct happy_meal *hp, void __iomem *bregs) /* Lettuce, tomato, buggy hardware (no extra charge)? */ if (!tries) - printk(KERN_ERR "happy meal: Transceiver BigMac ATTACK!"); + netdev_err(hp->dev, "Transceiver BigMac ATTACK!"); /* Take care. */ - HMD(("done\n")); + HMD("done\n"); } /* hp->happy_lock must be held */ @@ -850,7 +814,7 @@ static void happy_meal_rx_reset(struct happy_meal *hp, void __iomem *bregs) { int tries = RX_RESET_TRIES; - HMD(("happy_meal_rx_reset: reset, ")); + HMD("reset...\n"); /* We have a special on GNU/Viking hardware bugs today. */ hme_write32(hp, bregs + BMAC_RXSWRESET, 0); @@ -859,10 +823,10 @@ static void happy_meal_rx_reset(struct happy_meal *hp, void __iomem *bregs) /* Will that be all? */ if (!tries) - printk(KERN_ERR "happy meal: Receiver BigMac ATTACK!"); + netdev_err(hp->dev, "Receiver BigMac ATTACK!\n"); /* Don't forget your vik_1137125_wa. Have a nice day. */ - HMD(("done\n")); + HMD("done\n"); } #define STOP_TRIES 16 @@ -872,7 +836,7 @@ static void happy_meal_stop(struct happy_meal *hp, void __iomem *gregs) { int tries = STOP_TRIES; - HMD(("happy_meal_stop: reset, ")); + HMD("reset...\n"); /* We're consolidating our STB products, it's your lucky day. */ hme_write32(hp, gregs + GREG_SWRESET, GREG_RESET_ALL); @@ -881,10 +845,10 @@ static void happy_meal_stop(struct happy_meal *hp, void __iomem *gregs) /* Come back next week when we are "Sun Microelectronics". */ if (!tries) - printk(KERN_ERR "happy meal: Fry guys."); + netdev_err(hp->dev, "Fry guys.\n"); /* Remember: "Different name, same old buggy as shit hardware." */ - HMD(("done\n")); + HMD("done\n"); } /* hp->happy_lock must be held */ @@ -913,21 +877,18 @@ static void happy_meal_get_counters(struct happy_meal *hp, void __iomem *bregs) /* hp->happy_lock must be held */ static void happy_meal_poll_stop(struct happy_meal *hp, void __iomem *tregs) { - ASD(("happy_meal_poll_stop: ")); - /* If polling disabled or not polling already, nothing to do. */ if ((hp->happy_flags & (HFLAG_POLLENABLE | HFLAG_POLL)) != (HFLAG_POLLENABLE | HFLAG_POLL)) { - HMD(("not polling, return\n")); + ASD("not polling, return\n"); return; } /* Shut up the MIF. */ - ASD(("were polling, mif ints off, ")); + ASD("were polling, mif ints off, polling off\n"); hme_write32(hp, tregs + TCVR_IMASK, 0xffff); /* Turn off polling. */ - ASD(("polling off, ")); hme_write32(hp, tregs + TCVR_CFG, hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_PENABLE)); @@ -936,7 +897,7 @@ static void happy_meal_poll_stop(struct happy_meal *hp, void __iomem *tregs) /* Let the bits set. */ udelay(200); - ASD(("done\n")); + ASD("done\n"); } /* Only Sun can take such nice parts and fuck up the programming interface @@ -952,44 +913,40 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs) int result, tries = TCVR_RESET_TRIES; tconfig = hme_read32(hp, tregs + TCVR_CFG); - ASD(("happy_meal_tcvr_reset: tcfg<%08lx> ", tconfig)); + ASD("tcfg=%08x\n", tconfig); if (hp->tcvr_type == external) { - ASD(("external<")); hme_write32(hp, tregs + TCVR_CFG, tconfig & ~(TCV_CFG_PSELECT)); hp->tcvr_type = internal; hp->paddr = TCV_PADDR_ITX; - ASD(("ISOLATE,")); happy_meal_tcvr_write(hp, tregs, MII_BMCR, (BMCR_LOOPBACK|BMCR_PDOWN|BMCR_ISOLATE)); result = happy_meal_tcvr_read(hp, tregs, MII_BMCR); if (result == TCVR_FAILURE) { - ASD(("phyread_fail>\n")); + ASD("phyread_fail\n"); return -1; } - ASD(("phyread_ok,PSELECT>")); + ASD("external: ISOLATE, phyread_ok, PSELECT\n"); hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT); hp->tcvr_type = external; hp->paddr = TCV_PADDR_ETX; } else { if (tconfig & TCV_CFG_MDIO1) { - ASD(("internal\n")); + ASD("phyread_fail>\n"); return -1; } - ASD(("phyread_ok,~PSELECT>")); + ASD("internal: PSELECT, ISOLATE, phyread_ok, ~PSELECT\n"); hme_write32(hp, tregs + TCVR_CFG, (tconfig & ~(TCV_CFG_PSELECT))); hp->tcvr_type = internal; hp->paddr = TCV_PADDR_ITX; } } - ASD(("BMCR_RESET ")); + ASD("BMCR_RESET...\n"); happy_meal_tcvr_write(hp, tregs, MII_BMCR, BMCR_RESET); while (--tries) { @@ -1002,10 +959,10 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs) udelay(20); } if (!tries) { - ASD(("BMCR RESET FAILED!\n")); + ASD("BMCR RESET FAILED!\n"); return -1; } - ASD(("RESET_OK\n")); + ASD("RESET_OK\n"); /* Get fresh copies of the PHY registers. */ hp->sw_bmsr = happy_meal_tcvr_read(hp, tregs, MII_BMSR); @@ -1013,7 +970,7 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs) hp->sw_physid2 = happy_meal_tcvr_read(hp, tregs, MII_PHYSID2); hp->sw_advertise = happy_meal_tcvr_read(hp, tregs, MII_ADVERTISE); - ASD(("UNISOLATE")); + ASD("UNISOLATE...\n"); hp->sw_bmcr &= ~(BMCR_ISOLATE); happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr); @@ -1027,10 +984,10 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs) udelay(20); } if (!tries) { - ASD((" FAILED!\n")); + ASD("UNISOLATE FAILED!\n"); return -1; } - ASD((" SUCCESS and CSCONFIG_DFBYPASS\n")); + ASD("SUCCESS and CSCONFIG_DFBYPASS\n"); if (!is_lucent_phy(hp)) { result = happy_meal_tcvr_read(hp, tregs, DP83840_CSCONFIG); @@ -1048,60 +1005,55 @@ static void happy_meal_transceiver_check(struct happy_meal *hp, void __iomem *tr { unsigned long tconfig = hme_read32(hp, tregs + TCVR_CFG); - ASD(("happy_meal_transceiver_check: tcfg=%08lx ", tconfig)); + ASD("tcfg=%08lx\n", tconfig); if (hp->happy_flags & HFLAG_POLL) { /* If we are polling, we must stop to get the transceiver type. */ - ASD((" ")); if (hp->tcvr_type == internal) { if (tconfig & TCV_CFG_MDIO1) { - ASD((" ")); happy_meal_poll_stop(hp, tregs); hp->paddr = TCV_PADDR_ETX; hp->tcvr_type = external; - ASD(("\n")); tconfig &= ~(TCV_CFG_PENABLE); tconfig |= TCV_CFG_PSELECT; hme_write32(hp, tregs + TCVR_CFG, tconfig); + ASD("poll stop, internal->external\n"); } } else { if (hp->tcvr_type == external) { - ASD((" ")); if (!(hme_read32(hp, tregs + TCVR_STATUS) >> 16)) { - ASD((" ")); happy_meal_poll_stop(hp, tregs); hp->paddr = TCV_PADDR_ITX; hp->tcvr_type = internal; - ASD(("\n")); hme_write32(hp, tregs + TCVR_CFG, hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_PSELECT)); + ASD("poll stop, external->internal\n"); } - ASD(("\n")); } else { - ASD(("\n")); + ASD("polling, none\n"); } } } else { u32 reread = hme_read32(hp, tregs + TCVR_CFG); /* Else we can just work off of the MDIO bits. */ - ASD((" ")); if (reread & TCV_CFG_MDIO1) { hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT); hp->paddr = TCV_PADDR_ETX; hp->tcvr_type = external; - ASD(("\n")); + ASD("not polling, external\n"); } else { if (reread & TCV_CFG_MDIO0) { hme_write32(hp, tregs + TCVR_CFG, tconfig & ~(TCV_CFG_PSELECT)); hp->paddr = TCV_PADDR_ITX; hp->tcvr_type = internal; - ASD(("\n")); + ASD("not polling, internal\n"); } else { - printk(KERN_ERR "happy meal: Transceiver and a coke please."); + netdev_err(hp->dev, + "Transceiver and a coke please."); hp->tcvr_type = none; /* Grrr... */ - ASD(("\n")); + ASD("not polling, none\n"); } } } @@ -1208,15 +1160,14 @@ static void happy_meal_init_rings(struct happy_meal *hp) struct hmeal_init_block *hb = hp->happy_block; int i; - HMD(("happy_meal_init_rings: counters to zero, ")); + HMD("counters to zero\n"); hp->rx_new = hp->rx_old = hp->tx_new = hp->tx_old = 0; /* Free any skippy bufs left around in the rings. */ - HMD(("clean, ")); happy_meal_clean_rings(hp); /* Now get new skippy bufs for the receive ring. */ - HMD(("init rxring, ")); + HMD("init rxring\n"); for (i = 0; i < RX_RING_SIZE; i++) { struct sk_buff *skb; u32 mapping; @@ -1243,11 +1194,11 @@ static void happy_meal_init_rings(struct happy_meal *hp) skb_reserve(skb, RX_OFFSET); } - HMD(("init txring, ")); + HMD("init txring\n"); for (i = 0; i < TX_RING_SIZE; i++) hme_write_txd(hp, &hb->happy_meal_txd[i], 0, 0); - HMD(("done\n")); + HMD("done\n"); } /* hp->happy_lock must be held */ @@ -1294,17 +1245,11 @@ happy_meal_begin_auto_negotiation(struct happy_meal *hp, * XXX so I completely skip checking for it in the BMSR for now. */ -#ifdef AUTO_SWITCH_DEBUG - ASD(("%s: Advertising [ ", hp->dev->name)); - if (hp->sw_advertise & ADVERTISE_10HALF) - ASD(("10H ")); - if (hp->sw_advertise & ADVERTISE_10FULL) - ASD(("10F ")); - if (hp->sw_advertise & ADVERTISE_100HALF) - ASD(("100H ")); - if (hp->sw_advertise & ADVERTISE_100FULL) - ASD(("100F ")); -#endif + ASD("Advertising [ %s%s%s%s]\n", + hp->sw_advertise & ADVERTISE_10HALF ? "10H " : "", + hp->sw_advertise & ADVERTISE_10FULL ? "10F " : "", + hp->sw_advertise & ADVERTISE_100HALF ? "100H " : "", + hp->sw_advertise & ADVERTISE_100FULL ? "100F " : ""); /* Enable Auto-Negotiation, this is usually on already... */ hp->sw_bmcr |= BMCR_ANENABLE; @@ -1324,10 +1269,11 @@ happy_meal_begin_auto_negotiation(struct happy_meal *hp, udelay(10); } if (!timeout) { - printk(KERN_ERR "%s: Happy Meal would not start auto negotiation " - "BMCR=0x%04x\n", hp->dev->name, hp->sw_bmcr); - printk(KERN_NOTICE "%s: Performing force link detection.\n", - hp->dev->name); + netdev_err(hp->dev, + "Happy Meal would not start auto negotiation BMCR=0x%04x\n", + hp->sw_bmcr); + netdev_notice(hp->dev, + "Performing force link detection.\n"); goto force_link; } else { hp->timer_state = arbwait; @@ -1382,70 +1328,69 @@ static int happy_meal_init(struct happy_meal *hp) void __iomem *erxregs = hp->erxregs; void __iomem *bregs = hp->bigmacregs; void __iomem *tregs = hp->tcvregs; + const char *bursts; u32 regtmp, rxcfg; /* If auto-negotiation timer is running, kill it. */ del_timer(&hp->happy_timer); - HMD(("happy_meal_init: happy_flags[%08x] ", - hp->happy_flags)); + HMD("happy_flags[%08x]\n", hp->happy_flags); if (!(hp->happy_flags & HFLAG_INIT)) { - HMD(("set HFLAG_INIT, ")); + HMD("set HFLAG_INIT\n"); hp->happy_flags |= HFLAG_INIT; happy_meal_get_counters(hp, bregs); } /* Stop polling. */ - HMD(("to happy_meal_poll_stop\n")); + HMD("to happy_meal_poll_stop\n"); happy_meal_poll_stop(hp, tregs); /* Stop transmitter and receiver. */ - HMD(("happy_meal_init: to happy_meal_stop\n")); + HMD("to happy_meal_stop\n"); happy_meal_stop(hp, gregs); /* Alloc and reset the tx/rx descriptor chains. */ - HMD(("happy_meal_init: to happy_meal_init_rings\n")); + HMD("to happy_meal_init_rings\n"); happy_meal_init_rings(hp); /* Shut up the MIF. */ - HMD(("happy_meal_init: Disable all MIF irqs (old[%08x]), ", - hme_read32(hp, tregs + TCVR_IMASK))); + HMD("Disable all MIF irqs (old[%08x])\n", + hme_read32(hp, tregs + TCVR_IMASK)); hme_write32(hp, tregs + TCVR_IMASK, 0xffff); /* See if we can enable the MIF frame on this card to speak to the DP83840. */ if (hp->happy_flags & HFLAG_FENABLE) { - HMD(("use frame old[%08x], ", - hme_read32(hp, tregs + TCVR_CFG))); + HMD("use frame old[%08x]\n", + hme_read32(hp, tregs + TCVR_CFG)); hme_write32(hp, tregs + TCVR_CFG, hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_BENABLE)); } else { - HMD(("use bitbang old[%08x], ", - hme_read32(hp, tregs + TCVR_CFG))); + HMD("use bitbang old[%08x]\n", + hme_read32(hp, tregs + TCVR_CFG)); hme_write32(hp, tregs + TCVR_CFG, hme_read32(hp, tregs + TCVR_CFG) | TCV_CFG_BENABLE); } /* Check the state of the transceiver. */ - HMD(("to happy_meal_transceiver_check\n")); + HMD("to happy_meal_transceiver_check\n"); happy_meal_transceiver_check(hp, tregs); /* Put the Big Mac into a sane state. */ - HMD(("happy_meal_init: ")); switch(hp->tcvr_type) { case none: /* Cannot operate if we don't know the transceiver type! */ - HMD(("AAIEEE no transceiver type, EAGAIN")); + HMD("AAIEEE no transceiver type, EAGAIN\n"); return -EAGAIN; case internal: /* Using the MII buffers. */ - HMD(("internal, using MII, ")); + HMD("internal, using MII\n"); hme_write32(hp, bregs + BMAC_XIFCFG, 0); break; case external: /* Not using the MII, disable it. */ - HMD(("external, disable MII, ")); + HMD("external, disable MII\n"); hme_write32(hp, bregs + BMAC_XIFCFG, BIGMAC_XCFG_MIIDISAB); break; } @@ -1454,18 +1399,16 @@ static int happy_meal_init(struct happy_meal *hp) return -EAGAIN; /* Reset the Happy Meal Big Mac transceiver and the receiver. */ - HMD(("tx/rx reset, ")); + HMD("tx/rx reset\n"); happy_meal_tx_reset(hp, bregs); happy_meal_rx_reset(hp, bregs); /* Set jam size and inter-packet gaps to reasonable defaults. */ - HMD(("jsize/ipg1/ipg2, ")); hme_write32(hp, bregs + BMAC_JSIZE, DEFAULT_JAMSIZE); hme_write32(hp, bregs + BMAC_IGAP1, DEFAULT_IPG1); hme_write32(hp, bregs + BMAC_IGAP2, DEFAULT_IPG2); /* Load up the MAC address and random seed. */ - HMD(("rseed/macaddr, ")); /* The docs recommend to use the 10LSB of our MAC here. */ hme_write32(hp, bregs + BMAC_RSEED, ((e[5] | e[4]<<8)&0x3ff)); @@ -1474,7 +1417,6 @@ static int happy_meal_init(struct happy_meal *hp) hme_write32(hp, bregs + BMAC_MACADDR1, ((e[2] << 8) | e[3])); hme_write32(hp, bregs + BMAC_MACADDR0, ((e[0] << 8) | e[1])); - HMD(("htable, ")); if ((hp->dev->flags & IFF_ALLMULTI) || (netdev_mc_count(hp->dev) > 64)) { hme_write32(hp, bregs + BMAC_HTABLE0, 0xffff); @@ -1504,9 +1446,9 @@ static int happy_meal_init(struct happy_meal *hp) } /* Set the RX and TX ring ptrs. */ - HMD(("ring ptrs rxr[%08x] txr[%08x]\n", - ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0)), - ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_txd, 0)))); + HMD("ring ptrs rxr[%08x] txr[%08x]\n", + ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0)), + ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_txd, 0))); hme_write32(hp, erxregs + ERX_RING, ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0))); hme_write32(hp, etxregs + ETX_RING, @@ -1524,9 +1466,6 @@ static int happy_meal_init(struct happy_meal *hp) | 0x4); /* Set the supported burst sizes. */ - HMD(("happy_meal_init: old[%08x] bursts<", - hme_read32(hp, gregs + GREG_CFG))); - #ifndef CONFIG_SPARC /* It is always PCI and can handle 64byte bursts. */ hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST64); @@ -1554,34 +1493,35 @@ static int happy_meal_init(struct happy_meal *hp) } #endif - HMD(("64>")); + bursts = "64"; hme_write32(hp, gregs + GREG_CFG, gcfg); } else if (hp->happy_bursts & DMA_BURST32) { - HMD(("32>")); + bursts = "32"; hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST32); } else if (hp->happy_bursts & DMA_BURST16) { - HMD(("16>")); + bursts = "16"; hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST16); } else { - HMD(("XXX>")); + bursts = "XXX"; hme_write32(hp, gregs + GREG_CFG, 0); } #endif /* CONFIG_SPARC */ + HMD("old[%08x] bursts<%s>\n", + hme_read32(hp, gregs + GREG_CFG), bursts); + /* Turn off interrupts we do not want to hear. */ - HMD((", enable global interrupts, ")); hme_write32(hp, gregs + GREG_IMASK, (GREG_IMASK_GOTFRAME | GREG_IMASK_RCNTEXP | GREG_IMASK_SENTFRAME | GREG_IMASK_TXPERR)); /* Set the transmit ring buffer size. */ - HMD(("tx rsize=%d oreg[%08x], ", (int)TX_RING_SIZE, - hme_read32(hp, etxregs + ETX_RSIZE))); + HMD("tx rsize=%d oreg[%08x]\n", (int)TX_RING_SIZE, + hme_read32(hp, etxregs + ETX_RSIZE)); hme_write32(hp, etxregs + ETX_RSIZE, (TX_RING_SIZE >> ETX_RSIZE_SHIFT) - 1); /* Enable transmitter DVMA. */ - HMD(("tx dma enable old[%08x], ", - hme_read32(hp, etxregs + ETX_CFG))); + HMD("tx dma enable old[%08x]\n", hme_read32(hp, etxregs + ETX_CFG)); hme_write32(hp, etxregs + ETX_CFG, hme_read32(hp, etxregs + ETX_CFG) | ETX_CFG_DMAENABLE); @@ -1590,21 +1530,23 @@ static int happy_meal_init(struct happy_meal *hp) * properly. I cannot think of a sane way to provide complete * coverage for this hardware bug yet. */ - HMD(("erx regs bug old[%08x]\n", - hme_read32(hp, erxregs + ERX_CFG))); + HMD("erx regs bug old[%08x]\n", + hme_read32(hp, erxregs + ERX_CFG)); hme_write32(hp, erxregs + ERX_CFG, ERX_CFG_DEFAULT(RX_OFFSET)); regtmp = hme_read32(hp, erxregs + ERX_CFG); hme_write32(hp, erxregs + ERX_CFG, ERX_CFG_DEFAULT(RX_OFFSET)); if (hme_read32(hp, erxregs + ERX_CFG) != ERX_CFG_DEFAULT(RX_OFFSET)) { - printk(KERN_ERR "happy meal: Eieee, rx config register gets greasy fries.\n"); - printk(KERN_ERR "happy meal: Trying to set %08x, reread gives %08x\n", - ERX_CFG_DEFAULT(RX_OFFSET), regtmp); + netdev_err(hp->dev, + "Eieee, rx config register gets greasy fries.\n"); + netdev_err(hp->dev, + "Trying to set %08x, reread gives %08x\n", + ERX_CFG_DEFAULT(RX_OFFSET), regtmp); /* XXX Should return failure here... */ } /* Enable Big Mac hash table filter. */ - HMD(("happy_meal_init: enable hash rx_cfg_old[%08x], ", - hme_read32(hp, bregs + BMAC_RXCFG))); + HMD("enable hash rx_cfg_old[%08x]\n", + hme_read32(hp, bregs + BMAC_RXCFG)); rxcfg = BIGMAC_RXCFG_HENABLE | BIGMAC_RXCFG_REJME; if (hp->dev->flags & IFF_PROMISC) rxcfg |= BIGMAC_RXCFG_PMISC; @@ -1614,7 +1556,7 @@ static int happy_meal_init(struct happy_meal *hp) udelay(10); /* Ok, configure the Big Mac transmitter. */ - HMD(("BIGMAC init, ")); + HMD("BIGMAC init\n"); regtmp = 0; if (hp->happy_flags & HFLAG_FULL) regtmp |= BIGMAC_TXCFG_FULLDPLX; @@ -1638,14 +1580,13 @@ static int happy_meal_init(struct happy_meal *hp) if (hp->tcvr_type == external) regtmp |= BIGMAC_XCFG_MIIDISAB; - HMD(("XIF config old[%08x], ", - hme_read32(hp, bregs + BMAC_XIFCFG))); + HMD("XIF config old[%08x]\n", hme_read32(hp, bregs + BMAC_XIFCFG)); hme_write32(hp, bregs + BMAC_XIFCFG, regtmp); /* Start things up. */ - HMD(("tx old[%08x] and rx [%08x] ON!\n", - hme_read32(hp, bregs + BMAC_TXCFG), - hme_read32(hp, bregs + BMAC_RXCFG))); + HMD("tx old[%08x] and rx [%08x] ON!\n", + hme_read32(hp, bregs + BMAC_TXCFG), + hme_read32(hp, bregs + BMAC_RXCFG)); /* Set larger TX/RX size to allow for 802.1q */ hme_write32(hp, bregs + BMAC_TXMAX, ETH_FRAME_LEN + 8); @@ -1735,25 +1676,26 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status) GREG_STAT_MIFIRQ | GREG_STAT_TXEACK | GREG_STAT_TXLERR | GREG_STAT_TXPERR | GREG_STAT_TXTERR | GREG_STAT_SLVERR | GREG_STAT_SLVPERR)) - printk(KERN_ERR "%s: Error interrupt for happy meal, status = %08x\n", - hp->dev->name, status); + netdev_err(hp->dev, + "Error interrupt for happy meal, status = %08x\n", + status); if (status & GREG_STAT_RFIFOVF) { /* Receive FIFO overflow is harmless and the hardware will take care of it, just some packets are lost. Who cares. */ - printk(KERN_DEBUG "%s: Happy Meal receive FIFO overflow.\n", hp->dev->name); + netdev_dbg(hp->dev, "Happy Meal receive FIFO overflow.\n"); } if (status & GREG_STAT_STSTERR) { /* BigMAC SQE link test failed. */ - printk(KERN_ERR "%s: Happy Meal BigMAC SQE test failed.\n", hp->dev->name); + netdev_err(hp->dev, "Happy Meal BigMAC SQE test failed.\n"); reset = 1; } if (status & GREG_STAT_TFIFO_UND) { /* Transmit FIFO underrun, again DMA error likely. */ - printk(KERN_ERR "%s: Happy Meal transmitter FIFO underrun, DMA error.\n", - hp->dev->name); + netdev_err(hp->dev, + "Happy Meal transmitter FIFO underrun, DMA error.\n"); reset = 1; } @@ -1761,7 +1703,7 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status) /* Driver error, tried to transmit something larger * than ethernet max mtu. */ - printk(KERN_ERR "%s: Happy Meal MAX Packet size error.\n", hp->dev->name); + netdev_err(hp->dev, "Happy Meal MAX Packet size error.\n"); reset = 1; } @@ -1771,21 +1713,16 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status) * faster than the interrupt handler could keep up * with. */ - printk(KERN_INFO "%s: Happy Meal out of receive " - "descriptors, packet dropped.\n", - hp->dev->name); + netdev_info(hp->dev, + "Happy Meal out of receive descriptors, packet dropped.\n"); } if (status & (GREG_STAT_RXERR|GREG_STAT_RXPERR|GREG_STAT_RXTERR)) { /* All sorts of DMA receive errors. */ - printk(KERN_ERR "%s: Happy Meal rx DMA errors [ ", hp->dev->name); - if (status & GREG_STAT_RXERR) - printk("GenericError "); - if (status & GREG_STAT_RXPERR) - printk("ParityError "); - if (status & GREG_STAT_RXTERR) - printk("RxTagBotch "); - printk("]\n"); + netdev_err(hp->dev, "Happy Meal rx DMA errors [ %s%s%s]\n", + status & GREG_STAT_RXERR ? "GenericError " : "", + status & GREG_STAT_RXPERR ? "ParityError " : "", + status & GREG_STAT_RXTERR ? "RxTagBotch " : ""); reset = 1; } @@ -1793,29 +1730,24 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status) /* Driver bug, didn't set EOP bit in tx descriptor given * to the happy meal. */ - printk(KERN_ERR "%s: EOP not set in happy meal transmit descriptor!\n", - hp->dev->name); + netdev_err(hp->dev, + "EOP not set in happy meal transmit descriptor!\n"); reset = 1; } if (status & GREG_STAT_MIFIRQ) { /* MIF signalled an interrupt, were we polling it? */ - printk(KERN_ERR "%s: Happy Meal MIF interrupt.\n", hp->dev->name); + netdev_err(hp->dev, "Happy Meal MIF interrupt.\n"); } if (status & (GREG_STAT_TXEACK|GREG_STAT_TXLERR|GREG_STAT_TXPERR|GREG_STAT_TXTERR)) { /* All sorts of transmit DMA errors. */ - printk(KERN_ERR "%s: Happy Meal tx DMA errors [ ", hp->dev->name); - if (status & GREG_STAT_TXEACK) - printk("GenericError "); - if (status & GREG_STAT_TXLERR) - printk("LateError "); - if (status & GREG_STAT_TXPERR) - printk("ParityError "); - if (status & GREG_STAT_TXTERR) - printk("TagBotch "); - printk("]\n"); + netdev_err(hp->dev, "Happy Meal tx DMA errors [ %s%s%s%s]\n", + status & GREG_STAT_TXEACK ? "GenericError " : "", + status & GREG_STAT_TXLERR ? "LateError " : "", + status & GREG_STAT_TXPERR ? "ParityError " : "", + status & GREG_STAT_TXTERR ? "TagBotch " : ""); reset = 1; } @@ -1823,14 +1755,14 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status) /* Bus or parity error when cpu accessed happy meal registers * or it's internal FIFO's. Should never see this. */ - printk(KERN_ERR "%s: Happy Meal register access SBUS slave (%s) error.\n", - hp->dev->name, - (status & GREG_STAT_SLVPERR) ? "parity" : "generic"); + netdev_err(hp->dev, + "Happy Meal register access SBUS slave (%s) error.\n", + (status & GREG_STAT_SLVPERR) ? "parity" : "generic"); reset = 1; } if (reset) { - printk(KERN_NOTICE "%s: Resetting...\n", hp->dev->name); + netdev_notice(hp->dev, "Resetting...\n"); happy_meal_init(hp); return 1; } @@ -1842,22 +1774,22 @@ static void happy_meal_mif_interrupt(struct happy_meal *hp) { void __iomem *tregs = hp->tcvregs; - printk(KERN_INFO "%s: Link status change.\n", hp->dev->name); + netdev_info(hp->dev, "Link status change.\n"); hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR); hp->sw_lpa = happy_meal_tcvr_read(hp, tregs, MII_LPA); /* Use the fastest transmission protocol possible. */ if (hp->sw_lpa & LPA_100FULL) { - printk(KERN_INFO "%s: Switching to 100Mbps at full duplex.", hp->dev->name); + netdev_info(hp->dev, "Switching to 100Mbps at full duplex.\n"); hp->sw_bmcr |= (BMCR_FULLDPLX | BMCR_SPEED100); } else if (hp->sw_lpa & LPA_100HALF) { - printk(KERN_INFO "%s: Switching to 100MBps at half duplex.", hp->dev->name); + netdev_info(hp->dev, "Switching to 100MBps at half duplex.\n"); hp->sw_bmcr |= BMCR_SPEED100; } else if (hp->sw_lpa & LPA_10FULL) { - printk(KERN_INFO "%s: Switching to 10MBps at full duplex.", hp->dev->name); + netdev_info(hp->dev, "Switching to 10MBps at full duplex.\n"); hp->sw_bmcr |= BMCR_FULLDPLX; } else { - printk(KERN_INFO "%s: Using 10Mbps at half duplex.", hp->dev->name); + netdev_info(hp->dev, "Using 10Mbps at half duplex.\n"); } happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr); @@ -1865,12 +1797,6 @@ static void happy_meal_mif_interrupt(struct happy_meal *hp) happy_meal_poll_stop(hp, tregs); } -#ifdef TXDEBUG -#define TXD(x) printk x -#else -#define TXD(x) -#endif - /* hp->happy_lock must be held */ static void happy_meal_tx(struct happy_meal *hp) { @@ -1880,13 +1806,12 @@ static void happy_meal_tx(struct happy_meal *hp) int elem; elem = hp->tx_old; - TXD(("TX<")); while (elem != hp->tx_new) { struct sk_buff *skb; u32 flags, dma_addr, dma_len; int frag; - TXD(("[%d]", elem)); + netdev_vdbg(hp->dev, "TX[%d]\n", elem); this = &txbase[elem]; flags = hme_read_desc32(hp, &this->tx_flags); if (flags & TXFLAG_OWN) @@ -1922,19 +1847,12 @@ static void happy_meal_tx(struct happy_meal *hp) dev->stats.tx_packets++; } hp->tx_old = elem; - TXD((">")); if (netif_queue_stopped(dev) && TX_BUFFS_AVAIL(hp) > (MAX_SKB_FRAGS + 1)) netif_wake_queue(dev); } -#ifdef RXDEBUG -#define RXD(x) printk x -#else -#define RXD(x) -#endif - /* Originally I used to handle the allocation failure by just giving back just * that one ring buffer to the happy meal. Problem is that usually when that * condition is triggered, the happy meal expects you to do something reasonable @@ -1951,7 +1869,6 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev) int elem = hp->rx_new, drops = 0; u32 flags; - RXD(("RX<")); this = &rxbase[elem]; while (!((flags = hme_read_desc32(hp, &this->rx_flags)) & RXFLAG_OWN)) { struct sk_buff *skb; @@ -1959,11 +1876,9 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev) u16 csum = flags & RXFLAG_CSUM; u32 dma_addr = hme_read_desc32(hp, &this->rx_addr); - RXD(("[%d ", elem)); - /* Check for errors. */ if ((len < ETH_ZLEN) || (flags & RXFLAG_OVERFLOW)) { - RXD(("ERR(%08x)]", flags)); + netdev_vdbg(dev, "RX[%d ERR(%08x)]", elem, flags); dev->stats.rx_errors++; if (len < ETH_ZLEN) dev->stats.rx_length_errors++; @@ -2020,9 +1935,9 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev) skb_reserve(copy_skb, 2); skb_put(copy_skb, len); - dma_sync_single_for_cpu(hp->dma_dev, dma_addr, len, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(hp->dma_dev, dma_addr, len + 2, DMA_FROM_DEVICE); skb_copy_from_linear_data(skb, copy_skb->data, len); - dma_sync_single_for_device(hp->dma_dev, dma_addr, len, DMA_FROM_DEVICE); + dma_sync_single_for_device(hp->dma_dev, dma_addr, len + 2, DMA_FROM_DEVICE); /* Reuse original ring buffer. */ hme_write_rxd(hp, this, (RXFLAG_OWN|((RX_BUF_ALLOC_SIZE-RX_OFFSET)<<16)), @@ -2035,7 +1950,7 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev) skb->csum = csum_unfold(~(__force __sum16)htons(csum)); skb->ip_summed = CHECKSUM_COMPLETE; - RXD(("len=%d csum=%4x]", len, csum)); + netdev_vdbg(dev, "RX[%d len=%d csum=%4x]", elem, len, csum); skb->protocol = eth_type_trans(skb, dev); netif_rx(skb); @@ -2047,8 +1962,7 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev) } hp->rx_new = elem; if (drops) - printk(KERN_INFO "%s: Memory squeeze, deferring packet.\n", hp->dev->name); - RXD((">")); + netdev_info(hp->dev, "Memory squeeze, deferring packet.\n"); } static irqreturn_t happy_meal_interrupt(int irq, void *dev_id) @@ -2057,32 +1971,25 @@ static irqreturn_t happy_meal_interrupt(int irq, void *dev_id) struct happy_meal *hp = netdev_priv(dev); u32 happy_status = hme_read32(hp, hp->gregs + GREG_STAT); - HMD(("happy_meal_interrupt: status=%08x ", happy_status)); + HMD("status=%08x\n", happy_status); spin_lock(&hp->happy_lock); if (happy_status & GREG_STAT_ERRORS) { - HMD(("ERRORS ")); if (happy_meal_is_not_so_happy(hp, /* un- */ happy_status)) goto out; } - if (happy_status & GREG_STAT_MIFIRQ) { - HMD(("MIFIRQ ")); + if (happy_status & GREG_STAT_MIFIRQ) happy_meal_mif_interrupt(hp); - } - if (happy_status & GREG_STAT_TXALL) { - HMD(("TXALL ")); + if (happy_status & GREG_STAT_TXALL) happy_meal_tx(hp); - } - if (happy_status & GREG_STAT_RXTOHOST) { - HMD(("RXTOHOST ")); + if (happy_status & GREG_STAT_RXTOHOST) happy_meal_rx(hp, dev); - } - HMD(("done\n")); + HMD("done\n"); out: spin_unlock(&hp->happy_lock); @@ -2100,7 +2007,7 @@ static irqreturn_t quattro_sbus_interrupt(int irq, void *cookie) struct happy_meal *hp = netdev_priv(dev); u32 happy_status = hme_read32(hp, hp->gregs + GREG_STAT); - HMD(("quattro_interrupt: status=%08x ", happy_status)); + HMD("status=%08x\n", happy_status); if (!(happy_status & (GREG_STAT_ERRORS | GREG_STAT_MIFIRQ | @@ -2110,31 +2017,23 @@ static irqreturn_t quattro_sbus_interrupt(int irq, void *cookie) spin_lock(&hp->happy_lock); - if (happy_status & GREG_STAT_ERRORS) { - HMD(("ERRORS ")); + if (happy_status & GREG_STAT_ERRORS) if (happy_meal_is_not_so_happy(hp, happy_status)) goto next; - } - if (happy_status & GREG_STAT_MIFIRQ) { - HMD(("MIFIRQ ")); + if (happy_status & GREG_STAT_MIFIRQ) happy_meal_mif_interrupt(hp); - } - if (happy_status & GREG_STAT_TXALL) { - HMD(("TXALL ")); + if (happy_status & GREG_STAT_TXALL) happy_meal_tx(hp); - } - if (happy_status & GREG_STAT_RXTOHOST) { - HMD(("RXTOHOST ")); + if (happy_status & GREG_STAT_RXTOHOST) happy_meal_rx(hp, dev); - } next: spin_unlock(&hp->happy_lock); } - HMD(("done\n")); + HMD("done\n"); return IRQ_HANDLED; } @@ -2145,8 +2044,6 @@ static int happy_meal_open(struct net_device *dev) struct happy_meal *hp = netdev_priv(dev); int res; - HMD(("happy_meal_open: ")); - /* On SBUS Quattro QFE cards, all hme interrupts are concentrated * into a single source which we register handling at probe time. */ @@ -2154,15 +2051,14 @@ static int happy_meal_open(struct net_device *dev) res = request_irq(hp->irq, happy_meal_interrupt, IRQF_SHARED, dev->name, dev); if (res) { - HMD(("EAGAIN\n")); - printk(KERN_ERR "happy_meal(SBUS): Can't order irq %d to go.\n", - hp->irq); + HMD("EAGAIN\n"); + netdev_err(dev, "Can't order irq %d to go.\n", hp->irq); return -EAGAIN; } } - HMD(("to happy_meal_init\n")); + HMD("to happy_meal_init\n"); spin_lock_irq(&hp->happy_lock); res = happy_meal_init(hp); @@ -2196,22 +2092,16 @@ static int happy_meal_close(struct net_device *dev) return 0; } -#ifdef SXDEBUG -#define SXD(x) printk x -#else -#define SXD(x) -#endif - static void happy_meal_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct happy_meal *hp = netdev_priv(dev); - printk (KERN_ERR "%s: transmit timed out, resetting\n", dev->name); + netdev_err(dev, "transmit timed out, resetting\n"); tx_dump_log(); - printk (KERN_ERR "%s: Happy Status %08x TX[%08x:%08x]\n", dev->name, - hme_read32(hp, hp->gregs + GREG_STAT), - hme_read32(hp, hp->etxregs + ETX_CFG), - hme_read32(hp, hp->bigmacregs + BMAC_TXCFG)); + netdev_err(dev, "Happy Status %08x TX[%08x:%08x]\n", + hme_read32(hp, hp->gregs + GREG_STAT), + hme_read32(hp, hp->etxregs + ETX_CFG), + hme_read32(hp, hp->bigmacregs + BMAC_TXCFG)); spin_lock_irq(&hp->happy_lock); happy_meal_init(hp); @@ -2261,13 +2151,12 @@ static netdev_tx_t happy_meal_start_xmit(struct sk_buff *skb, if (TX_BUFFS_AVAIL(hp) <= (skb_shinfo(skb)->nr_frags + 1)) { netif_stop_queue(dev); spin_unlock_irq(&hp->happy_lock); - printk(KERN_ERR "%s: BUG! Tx Ring full when queue awake!\n", - dev->name); + netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); return NETDEV_TX_BUSY; } entry = hp->tx_new; - SXD(("SX", len, entry)); + netdev_vdbg(dev, "SX\n", skb->len, entry); hp->tx_skbs[entry] = skb; if (skb_shinfo(skb)->nr_frags == 0) { @@ -2467,11 +2356,10 @@ static void hme_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info { struct happy_meal *hp = netdev_priv(dev); - strlcpy(info->driver, "sunhme", sizeof(info->driver)); - strlcpy(info->version, "2.02", sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); if (hp->happy_flags & HFLAG_PCI) { struct pci_dev *pdev = hp->happy_dev; - strlcpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info)); + strscpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info)); } #ifdef CONFIG_SBUS else { @@ -2504,8 +2392,6 @@ static const struct ethtool_ops hme_ethtool_ops = { .set_link_ksettings = hme_set_link_ksettings, }; -static int hme_version_printed; - #ifdef CONFIG_SBUS /* Given a happy meal sbus device, find it's quattro parent. * If none exist, allocate and return a new one. @@ -2523,19 +2409,15 @@ static struct quattro *quattro_sbus_find(struct platform_device *child) if (qp) return qp; - qp = kmalloc(sizeof(struct quattro), GFP_KERNEL); - if (qp != NULL) { - int i; - - for (i = 0; i < 4; i++) - qp->happy_meals[i] = NULL; + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return NULL; - qp->quattro_dev = child; - qp->next = qfe_sbus_list; - qfe_sbus_list = qp; + qp->quattro_dev = child; + qp->next = qfe_sbus_list; + qfe_sbus_list = qp; - platform_set_drvdata(op, qp); - } + platform_set_drvdata(op, qp); return qp; } @@ -2563,8 +2445,9 @@ static int __init quattro_sbus_register_irqs(void) IRQF_SHARED, "Quattro", qp); if (err != 0) { - printk(KERN_ERR "Quattro HME: IRQ registration " - "error %d.\n", err); + dev_err(&op->dev, + "Quattro HME: IRQ registration error %d.\n", + err); return err; } } @@ -2595,30 +2478,33 @@ static void quattro_sbus_free_irqs(void) #ifdef CONFIG_PCI static struct quattro *quattro_pci_find(struct pci_dev *pdev) { + int i; struct pci_dev *bdev = pdev->bus->self; struct quattro *qp; - if (!bdev) return NULL; + if (!bdev) + return ERR_PTR(-ENODEV); + for (qp = qfe_pci_list; qp != NULL; qp = qp->next) { struct pci_dev *qpdev = qp->quattro_dev; if (qpdev == bdev) return qp; } + qp = kmalloc(sizeof(struct quattro), GFP_KERNEL); - if (qp != NULL) { - int i; + if (!qp) + return ERR_PTR(-ENOMEM); - for (i = 0; i < 4; i++) - qp->happy_meals[i] = NULL; + for (i = 0; i < 4; i++) + qp->happy_meals[i] = NULL; - qp->quattro_dev = bdev; - qp->next = qfe_pci_list; - qfe_pci_list = qp; + qp->quattro_dev = bdev; + qp->next = qfe_pci_list; + qfe_pci_list = qp; - /* No range tricks necessary on PCI. */ - qp->nranges = 0; - } + /* No range tricks necessary on PCI. */ + qp->nranges = 0; return qp; } #endif /* CONFIG_PCI */ @@ -2668,9 +2554,6 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) goto err_out; SET_NETDEV_DEV(dev, &op->dev); - if (hme_version_printed++ == 0) - printk(KERN_INFO "%s", version); - /* If user did not specify a MAC address specifically, use * the Quattro local-mac-address property... */ @@ -2712,35 +2595,35 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) hp->gregs = of_ioremap(&op->resource[0], 0, GREG_REG_SIZE, "HME Global Regs"); if (!hp->gregs) { - printk(KERN_ERR "happymeal: Cannot map global registers.\n"); + dev_err(&op->dev, "Cannot map global registers.\n"); goto err_out_free_netdev; } hp->etxregs = of_ioremap(&op->resource[1], 0, ETX_REG_SIZE, "HME TX Regs"); if (!hp->etxregs) { - printk(KERN_ERR "happymeal: Cannot map MAC TX registers.\n"); + dev_err(&op->dev, "Cannot map MAC TX registers.\n"); goto err_out_iounmap; } hp->erxregs = of_ioremap(&op->resource[2], 0, ERX_REG_SIZE, "HME RX Regs"); if (!hp->erxregs) { - printk(KERN_ERR "happymeal: Cannot map MAC RX registers.\n"); + dev_err(&op->dev, "Cannot map MAC RX registers.\n"); goto err_out_iounmap; } hp->bigmacregs = of_ioremap(&op->resource[3], 0, BMAC_REG_SIZE, "HME BIGMAC Regs"); if (!hp->bigmacregs) { - printk(KERN_ERR "happymeal: Cannot map BIGMAC registers.\n"); + dev_err(&op->dev, "Cannot map BIGMAC registers.\n"); goto err_out_iounmap; } hp->tcvregs = of_ioremap(&op->resource[4], 0, TCVR_REG_SIZE, "HME Tranceiver Regs"); if (!hp->tcvregs) { - printk(KERN_ERR "happymeal: Cannot map TCVR registers.\n"); + dev_err(&op->dev, "Cannot map TCVR registers.\n"); goto err_out_iounmap; } @@ -2807,21 +2690,19 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe) err = register_netdev(hp->dev); if (err) { - printk(KERN_ERR "happymeal: Cannot register net device, " - "aborting.\n"); + dev_err(&op->dev, "Cannot register net device, aborting.\n"); goto err_out_free_coherent; } platform_set_drvdata(op, hp); if (qfe_slot != -1) - printk(KERN_INFO "%s: Quattro HME slot %d (SBUS) 10/100baseT Ethernet ", - dev->name, qfe_slot); + netdev_info(dev, + "Quattro HME slot %d (SBUS) 10/100baseT Ethernet %pM\n", + qfe_slot, dev->dev_addr); else - printk(KERN_INFO "%s: HAPPY MEAL (SBUS) 10/100baseT Ethernet ", - dev->name); - - printk("%pM\n", dev->dev_addr); + netdev_info(dev, "HAPPY MEAL (SBUS) 10/100baseT Ethernet %pM\n", + dev->dev_addr); return 0; @@ -2949,7 +2830,7 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, struct happy_meal *hp; struct net_device *dev; void __iomem *hpreg_base; - unsigned long hpreg_res; + struct resource *hpreg_res; int i, qfe_slot = -1; char prom_name[64]; u8 addr[ETH_ALEN]; @@ -2966,32 +2847,33 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, strcpy(prom_name, "SUNW,hme"); #endif - err = -ENODEV; - - if (pci_enable_device(pdev)) + err = pcim_enable_device(pdev); + if (err) goto err_out; pci_set_master(pdev); if (!strcmp(prom_name, "SUNW,qfe") || !strcmp(prom_name, "qfe")) { qp = quattro_pci_find(pdev); - if (qp == NULL) + if (IS_ERR(qp)) { + err = PTR_ERR(qp); goto err_out; + } + for (qfe_slot = 0; qfe_slot < 4; qfe_slot++) - if (qp->happy_meals[qfe_slot] == NULL) + if (!qp->happy_meals[qfe_slot]) break; + if (qfe_slot == 4) goto err_out; } - dev = alloc_etherdev(sizeof(struct happy_meal)); - err = -ENOMEM; - if (!dev) + dev = devm_alloc_etherdev(&pdev->dev, sizeof(struct happy_meal)); + if (!dev) { + err = -ENOMEM; goto err_out; + } SET_NETDEV_DEV(dev, &pdev->dev); - if (hme_version_printed++ == 0) - printk(KERN_INFO "%s", version); - hp = netdev_priv(dev); hp->happy_dev = pdev; @@ -3005,21 +2887,26 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, qp->happy_meals[qfe_slot] = dev; } - hpreg_res = pci_resource_start(pdev, 0); - err = -ENODEV; + err = -EINVAL; if ((pci_resource_flags(pdev, 0) & IORESOURCE_IO) != 0) { - printk(KERN_ERR "happymeal(PCI): Cannot find proper PCI device base address.\n"); + dev_err(&pdev->dev, + "Cannot find proper PCI device base address.\n"); goto err_out_clear_quattro; } - if (pci_request_regions(pdev, DRV_NAME)) { - printk(KERN_ERR "happymeal(PCI): Cannot obtain PCI resources, " - "aborting.\n"); + + hpreg_res = devm_request_region(&pdev->dev, pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0), DRV_NAME); + if (!hpreg_res) { + err = -EBUSY; + dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting.\n"); goto err_out_clear_quattro; } - if ((hpreg_base = ioremap(hpreg_res, 0x8000)) == NULL) { - printk(KERN_ERR "happymeal(PCI): Unable to remap card memory.\n"); - goto err_out_free_res; + hpreg_base = pcim_iomap(pdev, 0, 0x8000); + if (!hpreg_base) { + err = -ENOMEM; + dev_err(&pdev->dev, "Unable to remap card memory.\n"); + goto err_out_clear_quattro; } for (i = 0; i < 6; i++) { @@ -3085,11 +2972,12 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, hp->happy_bursts = DMA_BURSTBITS; #endif - hp->happy_block = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, - &hp->hblock_dvma, GFP_KERNEL); - err = -ENODEV; - if (!hp->happy_block) - goto err_out_iounmap; + hp->happy_block = dmam_alloc_coherent(&pdev->dev, PAGE_SIZE, + &hp->hblock_dvma, GFP_KERNEL); + if (!hp->happy_block) { + err = -ENOMEM; + goto err_out_clear_quattro; + } hp->linkcheck = 0; hp->timer_state = asleep; @@ -3123,11 +3011,10 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, happy_meal_set_initial_advertisement(hp); spin_unlock_irq(&hp->happy_lock); - err = register_netdev(hp->dev); + err = devm_register_netdev(&pdev->dev, dev); if (err) { - printk(KERN_ERR "happymeal(PCI): Cannot register net device, " - "aborting.\n"); - goto err_out_free_coherent; + dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); + goto err_out_clear_quattro; } pci_set_drvdata(pdev, hp); @@ -3140,61 +3027,30 @@ static int happy_meal_pci_probe(struct pci_dev *pdev, int i = simple_strtoul(dev->name + 3, NULL, 10); sprintf(prom_name, "-%d", i + 3); } - printk(KERN_INFO "%s%s: Quattro HME (PCI/CheerIO) 10/100baseT Ethernet ", dev->name, prom_name); - if (qpdev->vendor == PCI_VENDOR_ID_DEC && - qpdev->device == PCI_DEVICE_ID_DEC_21153) - printk("DEC 21153 PCI Bridge\n"); - else - printk("unknown bridge %04x.%04x\n", - qpdev->vendor, qpdev->device); + netdev_info(dev, + "%s: Quattro HME (PCI/CheerIO) 10/100baseT Ethernet bridge %04x.%04x\n", + prom_name, qpdev->vendor, qpdev->device); } if (qfe_slot != -1) - printk(KERN_INFO "%s: Quattro HME slot %d (PCI/CheerIO) 10/100baseT Ethernet ", - dev->name, qfe_slot); + netdev_info(dev, + "Quattro HME slot %d (PCI/CheerIO) 10/100baseT Ethernet %pM\n", + qfe_slot, dev->dev_addr); else - printk(KERN_INFO "%s: HAPPY MEAL (PCI/CheerIO) 10/100BaseT Ethernet ", - dev->name); - - printk("%pM\n", dev->dev_addr); + netdev_info(dev, + "HAPPY MEAL (PCI/CheerIO) 10/100BaseT Ethernet %pM\n", + dev->dev_addr); return 0; -err_out_free_coherent: - dma_free_coherent(hp->dma_dev, PAGE_SIZE, - hp->happy_block, hp->hblock_dvma); - -err_out_iounmap: - iounmap(hp->gregs); - -err_out_free_res: - pci_release_regions(pdev); - err_out_clear_quattro: if (qp != NULL) qp->happy_meals[qfe_slot] = NULL; - free_netdev(dev); - err_out: return err; } -static void happy_meal_pci_remove(struct pci_dev *pdev) -{ - struct happy_meal *hp = pci_get_drvdata(pdev); - struct net_device *net_dev = hp->dev; - - unregister_netdev(net_dev); - - dma_free_coherent(hp->dma_dev, PAGE_SIZE, - hp->happy_block, hp->hblock_dvma); - iounmap(hp->gregs); - pci_release_regions(hp->happy_dev); - - free_netdev(net_dev); -} - static const struct pci_device_id happymeal_pci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_SUN, PCI_DEVICE_ID_SUN_HAPPYMEAL) }, { } /* Terminating entry */ @@ -3206,7 +3062,6 @@ static struct pci_driver hme_pci_driver = { .name = "hme", .id_table = happymeal_pci_ids, .probe = happy_meal_pci_probe, - .remove = happy_meal_pci_remove, }; static int __init happy_meal_pci_init(void) diff --git a/drivers/net/ethernet/sun/sunqe.c b/drivers/net/ethernet/sun/sunqe.c index efe0d33f6024e28930f4585cb65a6ef950302579..6418fcc3139f4563210506dad39180ce8e7ed707 100644 --- a/drivers/net/ethernet/sun/sunqe.c +++ b/drivers/net/ethernet/sun/sunqe.c @@ -684,8 +684,8 @@ static void qe_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) struct sunqe *qep = netdev_priv(dev); struct platform_device *op; - strlcpy(info->driver, "sunqe", sizeof(info->driver)); - strlcpy(info->version, "3.0", sizeof(info->version)); + strscpy(info->driver, "sunqe", sizeof(info->driver)); + strscpy(info->version, "3.0", sizeof(info->version)); op = qep->op; regs = of_get_property(op->dev.of_node, "reg", NULL); diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c index da8119625cf33484812053033f2b16e8977e5f7e..acda6cbd0238da735bec48f465bf6880a1def7cb 100644 --- a/drivers/net/ethernet/sun/sunvnet.c +++ b/drivers/net/ethernet/sun/sunvnet.c @@ -60,8 +60,8 @@ static struct vio_version vnet_versions[] = { static void vnet_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); } static u32 vnet_get_msglevel(struct net_device *dev) @@ -467,8 +467,7 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) if (err) goto err_out_free_port; - netif_napi_add(port->vp->dev, &port->napi, sunvnet_poll_common, - NAPI_POLL_WEIGHT); + netif_napi_add(port->vp->dev, &port->napi, sunvnet_poll_common); INIT_HLIST_NODE(&port->hash); INIT_LIST_HEAD(&port->list); diff --git a/drivers/net/ethernet/sunplus/spl2sw_driver.c b/drivers/net/ethernet/sunplus/spl2sw_driver.c index 546206640492b318f57fd3c5b5c8d1bbad9c757b..9be5852372773f13c3facc868e0f7e014492e519 100644 --- a/drivers/net/ethernet/sunplus/spl2sw_driver.c +++ b/drivers/net/ethernet/sunplus/spl2sw_driver.c @@ -62,7 +62,8 @@ static int spl2sw_ethernet_stop(struct net_device *ndev) return 0; } -static int spl2sw_ethernet_start_xmit(struct sk_buff *skb, struct net_device *ndev) +static netdev_tx_t spl2sw_ethernet_start_xmit(struct sk_buff *skb, + struct net_device *ndev) { struct spl2sw_mac *mac = netdev_priv(ndev); struct spl2sw_common *comm = mac->comm; @@ -248,8 +249,8 @@ static int spl2sw_nvmem_get_mac_address(struct device *dev, struct device_node * /* Check if mac address is valid */ if (!is_valid_ether_addr(mac)) { - kfree(mac); dev_info(dev, "Invalid mac address in nvmem (%pM)!\n", mac); + kfree(mac); return -EINVAL; } @@ -492,7 +493,7 @@ static int spl2sw_probe(struct platform_device *pdev) } /* Add and enable napi. */ - netif_napi_add(ndev, &comm->rx_napi, spl2sw_rx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &comm->rx_napi, spl2sw_rx_poll); napi_enable(&comm->rx_napi); netif_napi_add_tx(ndev, &comm->tx_napi, spl2sw_tx_poll); napi_enable(&comm->tx_napi); diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c index 5c9b6c90942b64e906902d89f8da39f66ebd9ecf..f8e133604146450712df42c571a4a1e96939727f 100644 --- a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c +++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c @@ -54,8 +54,8 @@ static void xlgmac_default_config(struct xlgmac_pdata *pdata) pdata->phy_speed = SPEED_25000; pdata->sysclk_rate = XLGMAC_SYSCLOCK; - strlcpy(pdata->drv_name, XLGMAC_DRV_NAME, sizeof(pdata->drv_name)); - strlcpy(pdata->drv_ver, XLGMAC_DRV_VERSION, sizeof(pdata->drv_ver)); + strscpy(pdata->drv_name, XLGMAC_DRV_NAME, sizeof(pdata->drv_name)); + strscpy(pdata->drv_ver, XLGMAC_DRV_VERSION, sizeof(pdata->drv_ver)); } static void xlgmac_init_all_ops(struct xlgmac_pdata *pdata) diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c index 49f8c6be9459ec6d1e1e8803fa1b3e03bfd79bc1..e794da727fe000832f8e662c0b463b0532028aa7 100644 --- a/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c +++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c @@ -102,9 +102,9 @@ static void xlgmac_ethtool_get_drvinfo(struct net_device *netdev, u32 ver = pdata->hw_feat.version; u32 snpsver, devid, userver; - strlcpy(drvinfo->driver, pdata->drv_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, pdata->drv_ver, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, dev_name(pdata->dev), + strscpy(drvinfo->driver, pdata->drv_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, pdata->drv_ver, sizeof(drvinfo->version)); + strscpy(drvinfo->bus_info, dev_name(pdata->dev), sizeof(drvinfo->bus_info)); /* S|SNPSVER: Synopsys-defined Version * D|DEVID: Indicates the Device family diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c index e54ce73396ee7845d7d89f0ca38501ca5936b6b8..36b948820c1e56bc338a6317bf3da5b9010b270a 100644 --- a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c +++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c @@ -419,15 +419,14 @@ static void xlgmac_napi_enable(struct xlgmac_pdata *pdata, unsigned int add) for (i = 0; i < pdata->channel_count; i++, channel++) { if (add) netif_napi_add(pdata->netdev, &channel->napi, - xlgmac_one_poll, - NAPI_POLL_WEIGHT); + xlgmac_one_poll); napi_enable(&channel->napi); } } else { if (add) netif_napi_add(pdata->netdev, &pdata->napi, - xlgmac_all_poll, NAPI_POLL_WEIGHT); + xlgmac_all_poll); napi_enable(&pdata->napi); } diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c index 985073eba3bd7258c4813e524e16d35998938783..ca409515ead5a43adcfd5bc84bea70bb21cd470d 100644 --- a/drivers/net/ethernet/tehuti/tehuti.c +++ b/drivers/net/ethernet/tehuti/tehuti.c @@ -1994,7 +1994,7 @@ bdx_probe(struct pci_dev *pdev, const struct pci_device_id *ent) priv->nic = nic; priv->msg_enable = BDX_DEF_MSG_ENABLE; - netif_napi_add(ndev, &priv->napi, bdx_poll, 64); + netif_napi_add(ndev, &priv->napi, bdx_poll); if ((readl(nic->regs + FPGA_VER) & 0xFFF) == 308) { DBG("HW statistics not supported\n"); @@ -2133,10 +2133,10 @@ bdx_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { struct bdx_priv *priv = netdev_priv(netdev); - strlcpy(drvinfo->driver, BDX_DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, BDX_DRV_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(priv->pdev), + strscpy(drvinfo->driver, BDX_DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, BDX_DRV_VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, pci_name(priv->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig index fb30bc5d56cb7de1b5ed23320396e97932e39cde..fce06663e1e110fa13a071a5b8fdaef1d04429e0 100644 --- a/drivers/net/ethernet/ti/Kconfig +++ b/drivers/net/ethernet/ti/Kconfig @@ -33,6 +33,7 @@ config TI_DAVINCI_MDIO tristate "TI DaVinci MDIO Support" depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST select PHYLIB + select MDIO_BITBANG help This driver supports TI's DaVinci MDIO module. diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c index abc1e4276cf088f51a46bb725f05956814979a96..c51e2af91f6943b3ee58c4c3cbdc29e5dd0640b3 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c +++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c @@ -402,9 +402,9 @@ static void am65_cpsw_get_drvinfo(struct net_device *ndev, { struct am65_cpsw_common *common = am65_ndev_to_common(ndev); - strlcpy(info->driver, dev_driver_string(common->dev), + strscpy(info->driver, dev_driver_string(common->dev), sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info)); + strscpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info)); } static u32 am65_cpsw_get_msglevel(struct net_device *ndev) diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index f4a6b590a1e3995a6eb51ba2c8d0c27b9b9af641..7f86068f3ff6319cdfcc333c067b4e70e815d45b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -74,6 +74,9 @@ #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C +#define AM65_CPSW_SGMII_CONTROL_REG 0x010 +#define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE BIT(0) + #define AM65_CPSW_CTL_VLAN_AWARE BIT(1) #define AM65_CPSW_CTL_P0_ENABLE BIT(2) #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13) @@ -360,8 +363,7 @@ static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common); static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port); static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port); -static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common, - netdev_features_t features) +static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common) { struct am65_cpsw_host *host_p = am65_common_get_host(common); int port_idx, i, ret; @@ -574,7 +576,7 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) for (i = 0; i < common->tx_ch_num; i++) netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i)); - ret = am65_cpsw_nuss_common_open(common, ndev->features); + ret = am65_cpsw_nuss_common_open(common); if (ret) return ret; @@ -590,11 +592,6 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev) /* mac_sl should be configured via phy-link interface */ am65_cpsw_sl_ctl_reset(port); - ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, - port->slave.phy_if); - if (ret) - goto error_cleanup; - ret = phylink_of_phy_connect(port->slave.phylink, port->slave.phy_node, 0); if (ret) goto error_cleanup; @@ -1409,7 +1406,14 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = { static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { - /* Currently not used */ + struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data, + phylink_config); + struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave); + struct am65_cpsw_common *common = port->common; + + if (common->pdata.extra_modes & BIT(state->interface)) + writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE, + port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG); } static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode, @@ -1847,6 +1851,8 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common) port->common = common; port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE + AM65_CPSW_NU_PORTS_OFFSET * (port_id); + if (common->pdata.extra_modes) + port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id); port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE + (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id); port->name = of_get_property(port_np, "label", NULL); @@ -1886,6 +1892,10 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common) goto of_node_put; } + ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if); + if (ret) + goto of_node_put; + ret = of_get_mac_address(port_np, port->slave.mac_addr); if (ret) { am65_cpsw_am654_get_efuse_macid(port_np, @@ -1981,7 +1991,18 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx) port->slave.phylink_config.type = PHYLINK_NETDEV; port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD; - phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces); + if (phy_interface_mode_is_rgmii(port->slave.phy_if)) { + phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces); + } else if (port->slave.phy_if == PHY_INTERFACE_MODE_RMII) { + __set_bit(PHY_INTERFACE_MODE_RMII, + port->slave.phylink_config.supported_interfaces); + } else if (common->pdata.extra_modes & BIT(port->slave.phy_if)) { + __set_bit(PHY_INTERFACE_MODE_QSGMII, + port->slave.phylink_config.supported_interfaces); + } else { + dev_err(dev, "selected phy-mode is not supported\n"); + return -EOPNOTSUPP; + } phylink = phylink_create(&port->slave.phylink_config, of_node_to_fwnode(port->slave.phy_node), @@ -2023,7 +2044,7 @@ static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common) } netif_napi_add(common->dma_ndev, &common->napi_rx, - am65_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT); + am65_cpsw_nuss_rx_poll); return ret; } @@ -2455,7 +2476,10 @@ static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common) port = am65_common_get_port(common, i); dl_port = &port->devlink_port; - attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + if (port->ndev) + attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + else + attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED; attrs.phys.port_number = port->port_id; attrs.switch_id.id_len = sizeof(resource_size_t); memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len); @@ -2611,10 +2635,18 @@ static const struct am65_cpsw_pdata am64x_cpswxg_pdata = { .fdqring_mode = K3_RINGACC_RING_MODE_RING, }; +static const struct am65_cpsw_pdata j7200_cpswxg_pdata = { + .quirks = 0, + .ale_dev_id = "am64-cpswxg", + .fdqring_mode = K3_RINGACC_RING_MODE_RING, + .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII), +}; + static const struct of_device_id am65_cpsw_nuss_of_mtable[] = { { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0}, { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata}, { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata}, + { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata}, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable); diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h index ac945631bf2fdc8b43256df7e0b1ebe556279f90..2c9850fdfcb6dc7489fb072038b9574bf66ce417 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h @@ -46,6 +46,7 @@ struct am65_cpsw_port { const char *name; u32 port_id; void __iomem *port_base; + void __iomem *sgmii_base; void __iomem *stat_base; void __iomem *fetch_ram_base; bool disabled; @@ -88,6 +89,7 @@ struct am65_cpsw_rx_chn { struct am65_cpsw_pdata { u32 quirks; + u64 extra_modes; enum k3_ring_mode fdqring_mode; const char *ale_dev_id; }; diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c index c30a6e510aa3237b6ffcc97a261db297a033c0b3..e2f0fb286143b3d8b6cd731ece27695f3a48d92b 100644 --- a/drivers/net/ethernet/ti/am65-cpts.c +++ b/drivers/net/ethernet/ti/am65-cpts.c @@ -943,9 +943,7 @@ struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs, cpts->irq = of_irq_get_byname(node, "cpts"); if (cpts->irq <= 0) { ret = cpts->irq ?: -ENXIO; - if (ret != -EPROBE_DEFER) - dev_err(dev, "Failed to get IRQ number (err = %d)\n", - ret); + dev_err_probe(dev, ret, "Failed to get IRQ number\n"); return ERR_PTR(ret); } @@ -965,8 +963,7 @@ struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs, cpts->refclk = devm_get_clk_from_child(dev, node, "cpts"); if (IS_ERR(cpts->refclk)) { ret = PTR_ERR(cpts->refclk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Failed to get refclk %d\n", ret); + dev_err_probe(dev, ret, "Failed to get refclk\n"); return ERR_PTR(ret); } diff --git a/drivers/net/ethernet/ti/cpmac.c b/drivers/net/ethernet/ti/cpmac.c index bef5e68dac31167b6f3880bcb6c0ae8cdcb8bee3..80eeeb463c4f5e8734b7f31f55a44123e185d495 100644 --- a/drivers/net/ethernet/ti/cpmac.c +++ b/drivers/net/ethernet/ti/cpmac.c @@ -851,8 +851,8 @@ static int cpmac_set_ringparam(struct net_device *dev, static void cpmac_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "cpmac", sizeof(info->driver)); - strlcpy(info->version, CPMAC_VERSION, sizeof(info->version)); + strscpy(info->driver, "cpmac", sizeof(info->driver)); + strscpy(info->version, CPMAC_VERSION, sizeof(info->version)); snprintf(info->bus_info, sizeof(info->bus_info), "%s", "cpmac"); } @@ -1109,7 +1109,7 @@ static int cpmac_probe(struct platform_device *pdev) dev->netdev_ops = &cpmac_netdev_ops; dev->ethtool_ops = &cpmac_ethtool_ops; - netif_napi_add(dev, &priv->napi, cpmac_poll, 64); + netif_napi_add(dev, &priv->napi, cpmac_poll); spin_lock_init(&priv->lock); spin_lock_init(&priv->rx_lock); @@ -1169,7 +1169,7 @@ static struct platform_driver cpmac_driver = { .remove = cpmac_remove, }; -int cpmac_init(void) +int __init cpmac_init(void) { u32 mask; int i, res; @@ -1239,7 +1239,7 @@ fail_alloc: return res; } -void cpmac_exit(void) +void __exit cpmac_exit(void) { platform_driver_unregister(&cpmac_driver); mdiobus_unregister(cpmac_mii); diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index ed66c4d4d830126e38a38b6ed2c874247b7f7c64..709ca6dd6ecb82255f691ad2eda8a6a160cc8b1c 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -1172,9 +1172,9 @@ static void cpsw_get_drvinfo(struct net_device *ndev, struct cpsw_common *cpsw = ndev_to_cpsw(ndev); struct platform_device *pdev = to_platform_device(cpsw->dev); - strlcpy(info->driver, "cpsw", sizeof(info->driver)); - strlcpy(info->version, "1.0", sizeof(info->version)); - strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info)); + strscpy(info->driver, "cpsw", sizeof(info->driver)); + strscpy(info->version, "1.0", sizeof(info->version)); + strscpy(info->bus_info, pdev->name, sizeof(info->bus_info)); } static int cpsw_set_pauseparam(struct net_device *ndev, @@ -1319,8 +1319,7 @@ static int cpsw_probe_dt(struct cpsw_platform_data *data, */ ret = of_phy_register_fixed_link(slave_node); if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret); + dev_err_probe(&pdev->dev, ret, "failed to register fixed-link phy\n"); goto err_node_put; } slave_data->phy_node = of_node_get(slave_node); @@ -1638,8 +1637,7 @@ static int cpsw_probe(struct platform_device *pdev) ndev->netdev_ops = &cpsw_netdev_ops; ndev->ethtool_ops = &cpsw_ethtool_ops; netif_napi_add(ndev, &cpsw->napi_rx, - cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll, - NAPI_POLL_WEIGHT); + cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll); netif_napi_add_tx(ndev, &cpsw->napi_tx, cpsw->quirk_irq ? cpsw_tx_poll : cpsw_tx_mq_poll); diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 353e58b22c512354157c51f2a484c66399f0a2b8..83596ec0c7cb9b0527ebff58f43928b36dbcc01f 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -1146,9 +1146,9 @@ static void cpsw_get_drvinfo(struct net_device *ndev, struct platform_device *pdev; pdev = to_platform_device(cpsw->dev); - strlcpy(info->driver, "cpsw-switch", sizeof(info->driver)); - strlcpy(info->version, "2.0", sizeof(info->version)); - strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info)); + strscpy(info->driver, "cpsw-switch", sizeof(info->driver)); + strscpy(info->version, "2.0", sizeof(info->version)); + strscpy(info->bus_info, pdev->name, sizeof(info->bus_info)); } static int cpsw_set_pauseparam(struct net_device *ndev, @@ -1288,9 +1288,8 @@ static int cpsw_probe_dt(struct cpsw_common *cpsw) if (of_phy_is_fixed_link(port_np)) { ret = of_phy_register_fixed_link(port_np); if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "%pOF failed to register fixed-link phy: %d\n", - port_np, ret); + dev_err_probe(dev, ret, "%pOF failed to register fixed-link phy\n", + port_np); goto err_node_put; } slave_data->phy_node = of_node_get(port_np); @@ -1417,9 +1416,7 @@ static int cpsw_create_ports(struct cpsw_common *cpsw) * accordingly. */ netif_napi_add(ndev, &cpsw->napi_rx, - cpsw->quirk_irq ? - cpsw_rx_poll : cpsw_rx_mq_poll, - NAPI_POLL_WEIGHT); + cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll); netif_napi_add_tx(ndev, &cpsw->napi_tx, cpsw->quirk_irq ? cpsw_tx_poll : cpsw_tx_mq_poll); diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c index 2a3e4e842fa5fde48e96a104f93b672174d1455a..2eb9d5a32588a0532bdfa957986584a16172e514 100644 --- a/drivers/net/ethernet/ti/davinci_emac.c +++ b/drivers/net/ethernet/ti/davinci_emac.c @@ -374,8 +374,8 @@ static char *emac_rxhost_errcodes[16] = { static void emac_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, emac_version_string, sizeof(info->driver)); - strlcpy(info->version, EMAC_MODULE_VERSION, sizeof(info->version)); + strscpy(info->driver, emac_version_string, sizeof(info->driver)); + strscpy(info->version, EMAC_MODULE_VERSION, sizeof(info->version)); } /** @@ -949,7 +949,7 @@ static void emac_tx_handler(void *token, int len, int status) * * Returns success(NETDEV_TX_OK) or error code (typically out of desc's) */ -static int emac_dev_xmit(struct sk_buff *skb, struct net_device *ndev) +static netdev_tx_t emac_dev_xmit(struct sk_buff *skb, struct net_device *ndev) { struct device *emac_dev = &ndev->dev; int ret_code; @@ -1948,7 +1948,7 @@ static int davinci_emac_probe(struct platform_device *pdev) ndev->netdev_ops = &emac_netdev_ops; ndev->ethtool_ops = ðtool_ops; - netif_napi_add(ndev, &priv->napi, emac_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &priv->napi, emac_poll); pm_runtime_enable(&pdev->dev); rc = pm_runtime_resume_and_get(&pdev->dev); diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c index ea3772618043164b6130e2db1f1bf261b59eb87e..946b9753ccfb3837de38f117c0b32fea43f797e7 100644 --- a/drivers/net/ethernet/ti/davinci_mdio.c +++ b/drivers/net/ethernet/ti/davinci_mdio.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include /* * This timeout definition is a worst-case ultra defensive measure against @@ -41,6 +43,7 @@ struct davinci_mdio_of_param { int autosuspend_delay_ms; + bool manual_mode; }; struct davinci_mdio_regs { @@ -49,6 +52,15 @@ struct davinci_mdio_regs { #define CONTROL_IDLE BIT(31) #define CONTROL_ENABLE BIT(30) #define CONTROL_MAX_DIV (0xffff) +#define CONTROL_CLKDIV GENMASK(15, 0) + +#define MDIO_MAN_MDCLK_O BIT(2) +#define MDIO_MAN_OE BIT(1) +#define MDIO_MAN_PIN BIT(0) +#define MDIO_MANUALMODE BIT(31) + +#define MDIO_PIN 0 + u32 alive; u32 link; @@ -59,7 +71,9 @@ struct davinci_mdio_regs { u32 userintmasked; u32 userintmaskset; u32 userintmaskclr; - u32 __reserved_1[20]; + u32 manualif; + u32 poll; + u32 __reserved_1[18]; struct { u32 access; @@ -79,6 +93,7 @@ static const struct mdio_platform_data default_pdata = { struct davinci_mdio_data { struct mdio_platform_data pdata; + struct mdiobb_ctrl bb_ctrl; struct davinci_mdio_regs __iomem *regs; struct clk *clk; struct device *dev; @@ -90,6 +105,7 @@ struct davinci_mdio_data { */ bool skip_scan; u32 clk_div; + bool manual_mode; }; static void davinci_mdio_init_clk(struct davinci_mdio_data *data) @@ -128,9 +144,122 @@ static void davinci_mdio_enable(struct davinci_mdio_data *data) writel(data->clk_div | CONTROL_ENABLE, &data->regs->control); } -static int davinci_mdio_reset(struct mii_bus *bus) +static void davinci_mdio_disable(struct davinci_mdio_data *data) +{ + u32 reg; + + /* Disable MDIO state machine */ + reg = readl(&data->regs->control); + + reg &= ~CONTROL_CLKDIV; + reg |= data->clk_div; + + reg &= ~CONTROL_ENABLE; + writel(reg, &data->regs->control); +} + +static void davinci_mdio_enable_manual_mode(struct davinci_mdio_data *data) +{ + u32 reg; + /* set manual mode */ + reg = readl(&data->regs->poll); + reg |= MDIO_MANUALMODE; + writel(reg, &data->regs->poll); +} + +static void davinci_set_mdc(struct mdiobb_ctrl *ctrl, int level) +{ + struct davinci_mdio_data *data; + u32 reg; + + data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl); + reg = readl(&data->regs->manualif); + + if (level) + reg |= MDIO_MAN_MDCLK_O; + else + reg &= ~MDIO_MAN_MDCLK_O; + + writel(reg, &data->regs->manualif); +} + +static void davinci_set_mdio_dir(struct mdiobb_ctrl *ctrl, int output) +{ + struct davinci_mdio_data *data; + u32 reg; + + data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl); + reg = readl(&data->regs->manualif); + + if (output) + reg |= MDIO_MAN_OE; + else + reg &= ~MDIO_MAN_OE; + + writel(reg, &data->regs->manualif); +} + +static void davinci_set_mdio_data(struct mdiobb_ctrl *ctrl, int value) +{ + struct davinci_mdio_data *data; + u32 reg; + + data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl); + reg = readl(&data->regs->manualif); + + if (value) + reg |= MDIO_MAN_PIN; + else + reg &= ~MDIO_MAN_PIN; + + writel(reg, &data->regs->manualif); +} + +static int davinci_get_mdio_data(struct mdiobb_ctrl *ctrl) +{ + struct davinci_mdio_data *data; + unsigned long reg; + + data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl); + reg = readl(&data->regs->manualif); + return test_bit(MDIO_PIN, ®); +} + +static int davinci_mdiobb_read(struct mii_bus *bus, int phy, int reg) +{ + int ret; + + ret = pm_runtime_resume_and_get(bus->parent); + if (ret < 0) + return ret; + + ret = mdiobb_read(bus, phy, reg); + + pm_runtime_mark_last_busy(bus->parent); + pm_runtime_put_autosuspend(bus->parent); + + return ret; +} + +static int davinci_mdiobb_write(struct mii_bus *bus, int phy, int reg, + u16 val) +{ + int ret; + + ret = pm_runtime_resume_and_get(bus->parent); + if (ret < 0) + return ret; + + ret = mdiobb_write(bus, phy, reg, val); + + pm_runtime_mark_last_busy(bus->parent); + pm_runtime_put_autosuspend(bus->parent); + + return ret; +} + +static int davinci_mdio_common_reset(struct davinci_mdio_data *data) { - struct davinci_mdio_data *data = bus->priv; u32 phy_mask, ver; int ret; @@ -138,6 +267,11 @@ static int davinci_mdio_reset(struct mii_bus *bus) if (ret < 0) return ret; + if (data->manual_mode) { + davinci_mdio_disable(data); + davinci_mdio_enable_manual_mode(data); + } + /* wait for scan logic to settle */ msleep(PHY_MAX_ADDR * data->access_time); @@ -171,6 +305,23 @@ done: return 0; } +static int davinci_mdio_reset(struct mii_bus *bus) +{ + struct davinci_mdio_data *data = bus->priv; + + return davinci_mdio_common_reset(data); +} + +static int davinci_mdiobb_reset(struct mii_bus *bus) +{ + struct mdiobb_ctrl *ctrl = bus->priv; + struct davinci_mdio_data *data; + + data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl); + + return davinci_mdio_common_reset(data); +} + /* wait until hardware is ready for another user access */ static inline int wait_for_user_access(struct davinci_mdio_data *data) { @@ -318,6 +469,28 @@ static int davinci_mdio_probe_dt(struct mdio_platform_data *data, return 0; } +struct k3_mdio_soc_data { + bool manual_mode; +}; + +static const struct k3_mdio_soc_data am65_mdio_soc_data = { + .manual_mode = true, +}; + +static const struct soc_device_attribute k3_mdio_socinfo[] = { + { .family = "AM62X", .revision = "SR1.0", .data = &am65_mdio_soc_data }, + { .family = "AM64X", .revision = "SR1.0", .data = &am65_mdio_soc_data }, + { .family = "AM64X", .revision = "SR2.0", .data = &am65_mdio_soc_data }, + { .family = "AM65X", .revision = "SR1.0", .data = &am65_mdio_soc_data }, + { .family = "AM65X", .revision = "SR2.0", .data = &am65_mdio_soc_data }, + { .family = "J7200", .revision = "SR1.0", .data = &am65_mdio_soc_data }, + { .family = "J7200", .revision = "SR2.0", .data = &am65_mdio_soc_data }, + { .family = "J721E", .revision = "SR1.0", .data = &am65_mdio_soc_data }, + { .family = "J721E", .revision = "SR2.0", .data = &am65_mdio_soc_data }, + { .family = "J721S2", .revision = "SR1.0", .data = &am65_mdio_soc_data}, + { /* sentinel */ }, +}; + #if IS_ENABLED(CONFIG_OF) static const struct davinci_mdio_of_param of_cpsw_mdio_data = { .autosuspend_delay_ms = 100, @@ -331,6 +504,14 @@ static const struct of_device_id davinci_mdio_of_mtable[] = { MODULE_DEVICE_TABLE(of, davinci_mdio_of_mtable); #endif +static const struct mdiobb_ops davinci_mdiobb_ops = { + .owner = THIS_MODULE, + .set_mdc = davinci_set_mdc, + .set_mdio_dir = davinci_set_mdio_dir, + .set_mdio_data = davinci_set_mdio_data, + .get_mdio_data = davinci_get_mdio_data, +}; + static int davinci_mdio_probe(struct platform_device *pdev) { struct mdio_platform_data *pdata = dev_get_platdata(&pdev->dev); @@ -345,7 +526,26 @@ static int davinci_mdio_probe(struct platform_device *pdev) if (!data) return -ENOMEM; - data->bus = devm_mdiobus_alloc(dev); + data->manual_mode = false; + data->bb_ctrl.ops = &davinci_mdiobb_ops; + + if (IS_ENABLED(CONFIG_OF) && dev->of_node) { + const struct soc_device_attribute *soc_match_data; + + soc_match_data = soc_device_match(k3_mdio_socinfo); + if (soc_match_data && soc_match_data->data) { + const struct k3_mdio_soc_data *socdata = + soc_match_data->data; + + data->manual_mode = socdata->manual_mode; + } + } + + if (data->manual_mode) + data->bus = alloc_mdio_bitbang(&data->bb_ctrl); + else + data->bus = devm_mdiobus_alloc(dev); + if (!data->bus) { dev_err(dev, "failed to alloc mii bus\n"); return -ENOMEM; @@ -371,11 +571,20 @@ static int davinci_mdio_probe(struct platform_device *pdev) } data->bus->name = dev_name(dev); - data->bus->read = davinci_mdio_read; - data->bus->write = davinci_mdio_write; - data->bus->reset = davinci_mdio_reset; + + if (data->manual_mode) { + data->bus->read = davinci_mdiobb_read; + data->bus->write = davinci_mdiobb_write; + data->bus->reset = davinci_mdiobb_reset; + + dev_info(dev, "Configuring MDIO in manual mode\n"); + } else { + data->bus->read = davinci_mdio_read; + data->bus->write = davinci_mdio_write; + data->bus->reset = davinci_mdio_reset; + data->bus->priv = data; + } data->bus->parent = dev; - data->bus->priv = data; data->clk = devm_clk_get(dev, "fck"); if (IS_ERR(data->clk)) { @@ -433,9 +642,13 @@ static int davinci_mdio_remove(struct platform_device *pdev) { struct davinci_mdio_data *data = platform_get_drvdata(pdev); - if (data->bus) + if (data->bus) { mdiobus_unregister(data->bus); + if (data->manual_mode) + free_mdio_bitbang(data->bus); + } + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_disable(&pdev->dev); @@ -452,7 +665,9 @@ static int davinci_mdio_runtime_suspend(struct device *dev) ctrl = readl(&data->regs->control); ctrl &= ~CONTROL_ENABLE; writel(ctrl, &data->regs->control); - wait_for_idle(data); + + if (!data->manual_mode) + wait_for_idle(data); return 0; } @@ -461,7 +676,12 @@ static int davinci_mdio_runtime_resume(struct device *dev) { struct davinci_mdio_data *data = dev_get_drvdata(dev); - davinci_mdio_enable(data); + if (data->manual_mode) { + davinci_mdio_disable(data); + davinci_mdio_enable_manual_mode(data); + } else { + davinci_mdio_enable(data); + } return 0; } #endif diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c index b15d44261e766a5be5e6cf65ca9d026586747f6a..aba70bef489459b7df22e0c799744c570ca1c77c 100644 --- a/drivers/net/ethernet/ti/netcp_core.c +++ b/drivers/net/ethernet/ti/netcp_core.c @@ -2095,7 +2095,7 @@ static int netcp_create_interface(struct netcp_device *netcp_device, } /* NAPI register */ - netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll); netif_napi_add_tx(ndev, &netcp->tx_napi, netcp_tx_poll); /* Register the network device */ diff --git a/drivers/net/ethernet/ti/tlan.c b/drivers/net/ethernet/ti/tlan.c index 741c42c6a41776f6256c44e9210adbf170f7df00..b3da76efa8f5b70716b34c8a68b914ab7d1025ac 100644 --- a/drivers/net/ethernet/ti/tlan.c +++ b/drivers/net/ethernet/ti/tlan.c @@ -762,12 +762,12 @@ static void tlan_get_drvinfo(struct net_device *dev, { struct tlan_priv *priv = netdev_priv(dev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); if (priv->pci_dev) - strlcpy(info->bus_info, pci_name(priv->pci_dev), + strscpy(info->bus_info, pci_name(priv->pci_dev), sizeof(info->bus_info)); else - strlcpy(info->bus_info, "EISA", sizeof(info->bus_info)); + strscpy(info->bus_info, "EISA", sizeof(info->bus_info)); } static int tlan_get_eeprom_len(struct net_device *dev) diff --git a/drivers/net/ethernet/toshiba/ps3_gelic_net.c b/drivers/net/ethernet/toshiba/ps3_gelic_net.c index 3dbfb1b20649748e0e4b1eddb8aefe1cfe1b39d5..cf8de8a7a8a1eb0ef0c79ebd69daeb6305783ffc 100644 --- a/drivers/net/ethernet/toshiba/ps3_gelic_net.c +++ b/drivers/net/ethernet/toshiba/ps3_gelic_net.c @@ -1187,8 +1187,8 @@ int gelic_net_open(struct net_device *netdev) void gelic_net_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static int gelic_ether_get_link_ksettings(struct net_device *netdev, @@ -1441,7 +1441,7 @@ static void gelic_ether_setup_netdev_ops(struct net_device *netdev, { netdev->watchdog_timeo = GELIC_NET_WATCHDOG_TIMEOUT; /* NAPI */ - netif_napi_add(netdev, napi, gelic_net_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, napi, gelic_net_poll); netdev->ethtool_ops = &gelic_ether_ethtool_ops; netdev->netdev_ops = &gelic_netdevice_ops; } diff --git a/drivers/net/ethernet/toshiba/spider_net.c b/drivers/net/ethernet/toshiba/spider_net.c index bc4914c758ad25941d3f179ccecbcb143062c364..50d7eacfec5827a141b6a46be9d12b9b34359e64 100644 --- a/drivers/net/ethernet/toshiba/spider_net.c +++ b/drivers/net/ethernet/toshiba/spider_net.c @@ -2270,8 +2270,7 @@ spider_net_setup_netdev(struct spider_net_card *card) card->aneg_count = 0; timer_setup(&card->aneg_timer, spider_net_link_phy, 0); - netif_napi_add(netdev, &card->napi, - spider_net_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &card->napi, spider_net_poll); spider_net_setup_netdev_ops(netdev); diff --git a/drivers/net/ethernet/toshiba/spider_net_ethtool.c b/drivers/net/ethernet/toshiba/spider_net_ethtool.c index 93110dba0bfa0139af38c5cf437fee3c692e80cb..fef9fd127b5e35a9369832a44a323b79c26e165e 100644 --- a/drivers/net/ethernet/toshiba/spider_net_ethtool.c +++ b/drivers/net/ethernet/toshiba/spider_net_ethtool.c @@ -63,12 +63,12 @@ spider_net_ethtool_get_drvinfo(struct net_device *netdev, card = netdev_priv(netdev); /* clear and fill out info */ - strlcpy(drvinfo->driver, spider_net_driver_name, + strscpy(drvinfo->driver, spider_net_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "no information", + strscpy(drvinfo->version, VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->fw_version, "no information", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, pci_name(card->pdev), + strscpy(drvinfo->bus_info, pci_name(card->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/ethernet/toshiba/tc35815.c b/drivers/net/ethernet/toshiba/tc35815.c index 47aab9c132c81e9342629d96adabdd9a71394618..b50be67b398b141a0ae14ab84a70da9625a09ba9 100644 --- a/drivers/net/ethernet/toshiba/tc35815.c +++ b/drivers/net/ethernet/toshiba/tc35815.c @@ -1956,9 +1956,9 @@ static void tc35815_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo * { struct tc35815_local *lp = netdev_priv(dev); - strlcpy(info->driver, MODNAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(lp->pci_dev), sizeof(info->bus_info)); + strscpy(info->driver, MODNAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, pci_name(lp->pci_dev), sizeof(info->bus_info)); } static u32 tc35815_get_msglevel(struct net_device *dev) diff --git a/drivers/net/ethernet/tundra/tsi108_eth.c b/drivers/net/ethernet/tundra/tsi108_eth.c index 5251fc32422192916954a398d3c3456a9f497f5e..2cd2afc3fff0370543ebe0e82ee4c14e94e6185d 100644 --- a/drivers/net/ethernet/tundra/tsi108_eth.c +++ b/drivers/net/ethernet/tundra/tsi108_eth.c @@ -59,9 +59,6 @@ /* Check the phy status every half a second. */ #define CHECK_PHY_INTERVAL (HZ/2) -static int tsi108_init_one(struct platform_device *pdev); -static int tsi108_ether_remove(struct platform_device *pdev); - struct tsi108_prv_data { void __iomem *regs; /* Base of normal regs */ void __iomem *phyregs; /* Base of register bank used for PHY access */ @@ -144,16 +141,6 @@ struct tsi108_prv_data { struct platform_device *pdev; }; -/* Structure for a device driver */ - -static struct platform_driver tsi_eth_driver = { - .probe = tsi108_init_one, - .remove = tsi108_ether_remove, - .driver = { - .name = "tsi-ethernet", - }, -}; - static void tsi108_timed_checker(struct timer_list *t); #ifdef DEBUG @@ -1598,7 +1585,7 @@ tsi108_init_one(struct platform_device *pdev) data->phy_type = einfo->phy_type; data->irq_num = einfo->irq_num; data->id = pdev->id; - netif_napi_add(dev, &data->napi, tsi108_poll, 64); + netif_napi_add(dev, &data->napi, tsi108_poll); dev->netdev_ops = &tsi108_netdev_ops; dev->ethtool_ops = &tsi108_ethtool_ops; @@ -1683,6 +1670,16 @@ static int tsi108_ether_remove(struct platform_device *pdev) return 0; } + +/* Structure for a device driver */ + +static struct platform_driver tsi_eth_driver = { + .probe = tsi108_init_one, + .remove = tsi108_ether_remove, + .driver = { + .name = "tsi-ethernet", + }, +}; module_platform_driver(tsi_eth_driver); MODULE_AUTHOR("Tundra Semiconductor Corporation"); diff --git a/drivers/net/ethernet/vertexcom/mse102x.c b/drivers/net/ethernet/vertexcom/mse102x.c index eb39a45de0121b65ec5057e39e86dc74aa54e219..aeed2a093e34111987aa4223cedf189d77f324aa 100644 --- a/drivers/net/ethernet/vertexcom/mse102x.c +++ b/drivers/net/ethernet/vertexcom/mse102x.c @@ -750,6 +750,13 @@ static const struct of_device_id mse102x_match_table[] = { }; MODULE_DEVICE_TABLE(of, mse102x_match_table); +static const struct spi_device_id mse102x_ids[] = { + { "mse1021" }, + { "mse1022" }, + { } +}; +MODULE_DEVICE_TABLE(spi, mse102x_ids); + static struct spi_driver mse102x_driver = { .driver = { .name = DRV_NAME, @@ -758,10 +765,11 @@ static struct spi_driver mse102x_driver = { }, .probe = mse102x_probe_spi, .remove = mse102x_remove_spi, + .id_table = mse102x_ids, }; module_spi_driver(mse102x_driver); MODULE_DESCRIPTION("MSE102x Network driver"); -MODULE_AUTHOR("Stefan Wahren "); +MODULE_AUTHOR("Stefan Wahren "); MODULE_LICENSE("GPL"); MODULE_ALIAS("spi:" DRV_NAME); diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c index 509c5e9b29dfa4174b8c3b8597ffe1bafede9294..0fb15a17b547213c8cbae9088225b4735b885dae 100644 --- a/drivers/net/ethernet/via/via-rhine.c +++ b/drivers/net/ethernet/via/via-rhine.c @@ -965,7 +965,7 @@ static int rhine_init_one_common(struct device *hwdev, u32 quirks, dev->ethtool_ops = &netdev_ethtool_ops; dev->watchdog_timeo = TX_TIMEOUT; - netif_napi_add(dev, &rp->napi, rhine_napipoll, 64); + netif_napi_add(dev, &rp->napi, rhine_napipoll); if (rp->quirks & rqRhineI) dev->features |= NETIF_F_SG|NETIF_F_HW_CSUM; @@ -2281,8 +2281,8 @@ static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *i { struct device *hwdev = dev->dev.parent; - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info)); } static int netdev_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/via/via-velocity.c b/drivers/net/ethernet/via/via-velocity.c index ff0c102cb5787d1e0dcc06debd2a77e4a5c3c256..a502812ac418a6e26aab5d03a4ead85a0ce38900 100644 --- a/drivers/net/ethernet/via/via-velocity.c +++ b/drivers/net/ethernet/via/via-velocity.c @@ -2846,7 +2846,7 @@ static int velocity_probe(struct device *dev, int irq, netdev->netdev_ops = &velocity_netdev_ops; netdev->ethtool_ops = &velocity_ethtool_ops; - netif_napi_add(netdev, &vptr->napi, velocity_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &vptr->napi, velocity_poll); netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG | NETIF_F_HW_VLAN_CTAG_TX; @@ -3419,13 +3419,13 @@ static void velocity_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo { struct velocity_info *vptr = netdev_priv(dev); - strlcpy(info->driver, VELOCITY_NAME, sizeof(info->driver)); - strlcpy(info->version, VELOCITY_VERSION, sizeof(info->version)); + strscpy(info->driver, VELOCITY_NAME, sizeof(info->driver)); + strscpy(info->version, VELOCITY_VERSION, sizeof(info->version)); if (vptr->pdev) - strlcpy(info->bus_info, pci_name(vptr->pdev), + strscpy(info->bus_info, pci_name(vptr->pdev), sizeof(info->bus_info)); else - strlcpy(info->bus_info, "platform", sizeof(info->bus_info)); + strscpy(info->bus_info, "platform", sizeof(info->bus_info)); } static void velocity_ethtool_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) diff --git a/drivers/net/ethernet/wangxun/Kconfig b/drivers/net/ethernet/wangxun/Kconfig index b4a4fa0a58f88c43c1b8fc72c2287b472ccc3e3d..f5d43d8c962934980cb1f09faccaac8c79d8cc49 100644 --- a/drivers/net/ethernet/wangxun/Kconfig +++ b/drivers/net/ethernet/wangxun/Kconfig @@ -16,6 +16,19 @@ config NET_VENDOR_WANGXUN if NET_VENDOR_WANGXUN +config NGBE + tristate "Wangxun(R) GbE PCI Express adapters support" + depends on PCI + help + This driver supports Wangxun(R) GbE PCI Express family of + adapters. + + More specific information on configuring the driver is in + . + + To compile this driver as a module, choose M here. The module + will be called ngbe. + config TXGBE tristate "Wangxun(R) 10GbE PCI Express adapters support" depends on PCI diff --git a/drivers/net/ethernet/wangxun/Makefile b/drivers/net/ethernet/wangxun/Makefile index c34db1bead25b4339ae3ea8f01dda6bcaea31f08..ac3fb06b233ca4ff9e5164f84fdfc13bcd80ff9c 100644 --- a/drivers/net/ethernet/wangxun/Makefile +++ b/drivers/net/ethernet/wangxun/Makefile @@ -4,3 +4,4 @@ # obj-$(CONFIG_TXGBE) += txgbe/ +obj-$(CONFIG_NGBE) += ngbe/ diff --git a/drivers/net/ethernet/wangxun/ngbe/Makefile b/drivers/net/ethernet/wangxun/ngbe/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0baf75907496872fc01425613e3e037180199087 --- /dev/null +++ b/drivers/net/ethernet/wangxun/ngbe/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. +# +# Makefile for the Wangxun(R) GbE PCI Express ethernet driver +# + +obj-$(CONFIG_NGBE) += ngbe.o + +ngbe-objs := ngbe_main.o diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe.h b/drivers/net/ethernet/wangxun/ngbe/ngbe.h new file mode 100644 index 0000000000000000000000000000000000000000..f5fa6e5238cc85ef088ccee04e2640ac3c7f534f --- /dev/null +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */ + +#ifndef _NGBE_H_ +#define _NGBE_H_ + +#include "ngbe_type.h" + +#define NGBE_MAX_FDIR_INDICES 7 + +#define NGBE_MAX_RX_QUEUES (NGBE_MAX_FDIR_INDICES + 1) +#define NGBE_MAX_TX_QUEUES (NGBE_MAX_FDIR_INDICES + 1) + +/* board specific private data structure */ +struct ngbe_adapter { + u8 __iomem *io_addr; /* Mainly for iounmap use */ + /* OS defined structs */ + struct net_device *netdev; + struct pci_dev *pdev; +}; + +extern char ngbe_driver_name[]; + +#endif /* _NGBE_H_ */ diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c new file mode 100644 index 0000000000000000000000000000000000000000..7674cb6e5700af3d61fb0cc5b76152577c6a98c4 --- /dev/null +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "ngbe.h" +char ngbe_driver_name[] = "ngbe"; + +/* ngbe_pci_tbl - PCI Device ID Table + * + * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, + * Class, Class Mask, private data (not used) } + */ +static const struct pci_device_id ngbe_pci_tbl[] = { + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2S), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4S), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860LC), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1), 0}, + { PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1L), 0}, + /* required last entry */ + { .device = 0 } +}; + +static void ngbe_dev_shutdown(struct pci_dev *pdev, bool *enable_wake) +{ + struct ngbe_adapter *adapter = pci_get_drvdata(pdev); + struct net_device *netdev = adapter->netdev; + + netif_device_detach(netdev); + + pci_disable_device(pdev); +} + +static void ngbe_shutdown(struct pci_dev *pdev) +{ + bool wake; + + ngbe_dev_shutdown(pdev, &wake); + + if (system_state == SYSTEM_POWER_OFF) { + pci_wake_from_d3(pdev, wake); + pci_set_power_state(pdev, PCI_D3hot); + } +} + +/** + * ngbe_probe - Device Initialization Routine + * @pdev: PCI device information struct + * @ent: entry in ngbe_pci_tbl + * + * Returns 0 on success, negative on failure + * + * ngbe_probe initializes an adapter identified by a pci_dev structure. + * The OS initialization, configuring of the adapter private structure, + * and a hardware reset occur. + **/ +static int ngbe_probe(struct pci_dev *pdev, + const struct pci_device_id __always_unused *ent) +{ + struct ngbe_adapter *adapter = NULL; + struct net_device *netdev; + int err; + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, + "No usable DMA configuration, aborting\n"); + goto err_pci_disable_dev; + } + + err = pci_request_selected_regions(pdev, + pci_select_bars(pdev, IORESOURCE_MEM), + ngbe_driver_name); + if (err) { + dev_err(&pdev->dev, + "pci_request_selected_regions failed %d\n", err); + goto err_pci_disable_dev; + } + + pci_enable_pcie_error_reporting(pdev); + pci_set_master(pdev); + + netdev = devm_alloc_etherdev_mqs(&pdev->dev, + sizeof(struct ngbe_adapter), + NGBE_MAX_TX_QUEUES, + NGBE_MAX_RX_QUEUES); + if (!netdev) { + err = -ENOMEM; + goto err_pci_release_regions; + } + + SET_NETDEV_DEV(netdev, &pdev->dev); + + adapter = netdev_priv(netdev); + adapter->netdev = netdev; + adapter->pdev = pdev; + + adapter->io_addr = devm_ioremap(&pdev->dev, + pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!adapter->io_addr) { + err = -EIO; + goto err_pci_release_regions; + } + + netdev->features |= NETIF_F_HIGHDMA; + + pci_set_drvdata(pdev, adapter); + + return 0; + +err_pci_release_regions: + pci_disable_pcie_error_reporting(pdev); + pci_release_selected_regions(pdev, + pci_select_bars(pdev, IORESOURCE_MEM)); +err_pci_disable_dev: + pci_disable_device(pdev); + return err; +} + +/** + * ngbe_remove - Device Removal Routine + * @pdev: PCI device information struct + * + * ngbe_remove is called by the PCI subsystem to alert the driver + * that it should release a PCI device. The could be caused by a + * Hot-Plug event, or because the driver is going to be removed from + * memory. + **/ +static void ngbe_remove(struct pci_dev *pdev) +{ + pci_release_selected_regions(pdev, + pci_select_bars(pdev, IORESOURCE_MEM)); + + pci_disable_pcie_error_reporting(pdev); + + pci_disable_device(pdev); +} + +static struct pci_driver ngbe_driver = { + .name = ngbe_driver_name, + .id_table = ngbe_pci_tbl, + .probe = ngbe_probe, + .remove = ngbe_remove, + .shutdown = ngbe_shutdown, +}; + +module_pci_driver(ngbe_driver); + +MODULE_DEVICE_TABLE(pci, ngbe_pci_tbl); +MODULE_AUTHOR("Beijing WangXun Technology Co., Ltd, "); +MODULE_DESCRIPTION("WangXun(R) Gigabit PCI Express Network Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h new file mode 100644 index 0000000000000000000000000000000000000000..26e776c3539a07d5ceff3265a1cd6844668574bc --- /dev/null +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */ + +#ifndef _NGBE_TYPE_H_ +#define _NGBE_TYPE_H_ + +#include +#include + +/************ NGBE_register.h ************/ +/* Vendor ID */ +#ifndef PCI_VENDOR_ID_WANGXUN +#define PCI_VENDOR_ID_WANGXUN 0x8088 +#endif + +/* Device IDs */ +#define NGBE_DEV_ID_EM_WX1860AL_W 0x0100 +#define NGBE_DEV_ID_EM_WX1860A2 0x0101 +#define NGBE_DEV_ID_EM_WX1860A2S 0x0102 +#define NGBE_DEV_ID_EM_WX1860A4 0x0103 +#define NGBE_DEV_ID_EM_WX1860A4S 0x0104 +#define NGBE_DEV_ID_EM_WX1860AL2 0x0105 +#define NGBE_DEV_ID_EM_WX1860AL2S 0x0106 +#define NGBE_DEV_ID_EM_WX1860AL4 0x0107 +#define NGBE_DEV_ID_EM_WX1860AL4S 0x0108 +#define NGBE_DEV_ID_EM_WX1860LC 0x0109 +#define NGBE_DEV_ID_EM_WX1860A1 0x010a +#define NGBE_DEV_ID_EM_WX1860A1L 0x010b + +/* Subsystem ID */ +#define NGBE_SUBID_M88E1512_SFP 0x0003 +#define NGBE_SUBID_OCP_CARD 0x0040 +#define NGBE_SUBID_LY_M88E1512_SFP 0x0050 +#define NGBE_SUBID_M88E1512_RJ45 0x0051 +#define NGBE_SUBID_M88E1512_MIX 0x0052 +#define NGBE_SUBID_YT8521S_SFP 0x0060 +#define NGBE_SUBID_INTERNAL_YT8521S_SFP 0x0061 +#define NGBE_SUBID_YT8521S_SFP_GPIO 0x0062 +#define NGBE_SUBID_INTERNAL_YT8521S_SFP_GPIO 0x0064 +#define NGBE_SUBID_LY_YT8521S_SFP 0x0070 +#define NGBE_SUBID_RGMII_FPGA 0x0080 + +#define NGBE_OEM_MASK 0x00FF + +#define NGBE_NCSI_SUP 0x8000 +#define NGBE_NCSI_MASK 0x8000 +#define NGBE_WOL_SUP 0x4000 +#define NGBE_WOL_MASK 0x4000 + +#endif /* _NGBE_TYPE_H_ */ diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c index acd78120e53c9b5ef1e4fea5adee23eae15c6dbe..634946e87e5f7a5fb6ea274229674ffd30d03b9d 100644 --- a/drivers/net/ethernet/wiznet/w5100.c +++ b/drivers/net/ethernet/wiznet/w5100.c @@ -719,9 +719,9 @@ static void w5100_hw_close(struct w5100_priv *priv) static void w5100_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(ndev->dev.parent), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(ndev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/wiznet/w5300.c b/drivers/net/ethernet/wiznet/w5300.c index 773f8c77909a5c190c0994829799dacaecfc2361..b0958fe8111ef7942d12750be86e64e8f405e878 100644 --- a/drivers/net/ethernet/wiznet/w5300.c +++ b/drivers/net/ethernet/wiznet/w5300.c @@ -282,9 +282,9 @@ static void w5300_hw_close(struct w5300_priv *priv) static void w5300_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, dev_name(ndev->dev.parent), + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->bus_info, dev_name(ndev->dev.parent), sizeof(info->bus_info)); } diff --git a/drivers/net/ethernet/xilinx/ll_temac.h b/drivers/net/ethernet/xilinx/ll_temac.h index c6395c406418e1fc9d0cda3e37b7330101e87aa2..6668d1b760d8de8472a404cbdcf3e27638fa41f5 100644 --- a/drivers/net/ethernet/xilinx/ll_temac.h +++ b/drivers/net/ethernet/xilinx/ll_temac.h @@ -21,36 +21,45 @@ /* Configuration options */ /* Accept all incoming packets. - * This option defaults to disabled (cleared) */ + * This option defaults to disabled (cleared) + */ #define XTE_OPTION_PROMISC (1 << 0) /* Jumbo frame support for Tx & Rx. - * This option defaults to disabled (cleared) */ + * This option defaults to disabled (cleared) + */ #define XTE_OPTION_JUMBO (1 << 1) /* VLAN Rx & Tx frame support. - * This option defaults to disabled (cleared) */ + * This option defaults to disabled (cleared) + */ #define XTE_OPTION_VLAN (1 << 2) /* Enable recognition of flow control frames on Rx - * This option defaults to enabled (set) */ + * This option defaults to enabled (set) + */ #define XTE_OPTION_FLOW_CONTROL (1 << 4) /* Strip FCS and PAD from incoming frames. * Note: PAD from VLAN frames is not stripped. - * This option defaults to disabled (set) */ + * This option defaults to disabled (set) + */ #define XTE_OPTION_FCS_STRIP (1 << 5) /* Generate FCS field and add PAD automatically for outgoing frames. - * This option defaults to enabled (set) */ + * This option defaults to enabled (set) + */ #define XTE_OPTION_FCS_INSERT (1 << 6) /* Enable Length/Type error checking for incoming frames. When this option is -set, the MAC will filter frames that have a mismatched type/length field -and if XTE_OPTION_REPORT_RXERR is set, the user is notified when these -types of frames are encountered. When this option is cleared, the MAC will -allow these types of frames to be received. -This option defaults to enabled (set) */ + * set, the MAC will filter frames that have a mismatched type/length field + * and if XTE_OPTION_REPORT_RXERR is set, the user is notified when these + * types of frames are encountered. When this option is cleared, the MAC will + * allow these types of frames to be received. + * This option defaults to enabled (set) + */ #define XTE_OPTION_LENTYPE_ERR (1 << 7) /* Enable the transmitter. - * This option defaults to enabled (set) */ + * This option defaults to enabled (set) + */ #define XTE_OPTION_TXEN (1 << 11) /* Enable the receiver -* This option defaults to enabled (set) */ + * This option defaults to enabled (set) + */ #define XTE_OPTION_RXEN (1 << 12) /* Default options set when device is initialized or reset */ @@ -68,18 +77,18 @@ This option defaults to enabled (set) */ #define TX_TAILDESC_PTR 0x04 /* rw */ #define TX_CHNL_CTRL 0x05 /* rw */ /* - 0:7 24:31 IRQTimeout - 8:15 16:23 IRQCount - 16:20 11:15 Reserved - 21 10 0 - 22 9 UseIntOnEnd - 23 8 LdIRQCnt - 24 7 IRQEn - 25:28 3:6 Reserved - 29 2 IrqErrEn - 30 1 IrqDlyEn - 31 0 IrqCoalEn -*/ + * 0:7 24:31 IRQTimeout + * 8:15 16:23 IRQCount + * 16:20 11:15 Reserved + * 21 10 0 + * 22 9 UseIntOnEnd + * 23 8 LdIRQCnt + * 24 7 IRQEn + * 25:28 3:6 Reserved + * 29 2 IrqErrEn + * 30 1 IrqDlyEn + * 31 0 IrqCoalEn + */ #define CHNL_CTRL_IRQ_IOE (1 << 9) #define CHNL_CTRL_IRQ_EN (1 << 7) #define CHNL_CTRL_IRQ_ERR_EN (1 << 2) @@ -87,35 +96,35 @@ This option defaults to enabled (set) */ #define CHNL_CTRL_IRQ_COAL_EN (1 << 0) #define TX_IRQ_REG 0x06 /* rw */ /* - 0:7 24:31 DltTmrValue - 8:15 16:23 ClscCntrValue - 16:17 14:15 Reserved - 18:21 10:13 ClscCnt - 22:23 8:9 DlyCnt - 24:28 3::7 Reserved - 29 2 ErrIrq - 30 1 DlyIrq - 31 0 CoalIrq + * 0:7 24:31 DltTmrValue + * 8:15 16:23 ClscCntrValue + * 16:17 14:15 Reserved + * 18:21 10:13 ClscCnt + * 22:23 8:9 DlyCnt + * 24:28 3::7 Reserved + * 29 2 ErrIrq + * 30 1 DlyIrq + * 31 0 CoalIrq */ #define TX_CHNL_STS 0x07 /* r */ /* - 0:9 22:31 Reserved - 10 21 TailPErr - 11 20 CmpErr - 12 19 AddrErr - 13 18 NxtPErr - 14 17 CurPErr - 15 16 BsyWr - 16:23 8:15 Reserved - 24 7 Error - 25 6 IOE - 26 5 SOE - 27 4 Cmplt - 28 3 SOP - 29 2 EOP - 30 1 EngBusy - 31 0 Reserved -*/ + * 0:9 22:31 Reserved + * 10 21 TailPErr + * 11 20 CmpErr + * 12 19 AddrErr + * 13 18 NxtPErr + * 14 17 CurPErr + * 15 16 BsyWr + * 16:23 8:15 Reserved + * 24 7 Error + * 25 6 IOE + * 26 5 SOE + * 27 4 Cmplt + * 28 3 SOP + * 29 2 EOP + * 30 1 EngBusy + * 31 0 Reserved + */ #define RX_NXTDESC_PTR 0x08 /* r */ #define RX_CURBUF_ADDR 0x09 /* r */ @@ -124,17 +133,17 @@ This option defaults to enabled (set) */ #define RX_TAILDESC_PTR 0x0c /* rw */ #define RX_CHNL_CTRL 0x0d /* rw */ /* - 0:7 24:31 IRQTimeout - 8:15 16:23 IRQCount - 16:20 11:15 Reserved - 21 10 0 - 22 9 UseIntOnEnd - 23 8 LdIRQCnt - 24 7 IRQEn - 25:28 3:6 Reserved - 29 2 IrqErrEn - 30 1 IrqDlyEn - 31 0 IrqCoalEn + * 0:7 24:31 IRQTimeout + * 8:15 16:23 IRQCount + * 16:20 11:15 Reserved + * 21 10 0 + * 22 9 UseIntOnEnd + * 23 8 LdIRQCnt + * 24 7 IRQEn + * 25:28 3:6 Reserved + * 29 2 IrqErrEn + * 30 1 IrqDlyEn + * 31 0 IrqCoalEn */ #define RX_IRQ_REG 0x0e /* rw */ #define IRQ_COAL (1 << 0) @@ -142,13 +151,13 @@ This option defaults to enabled (set) */ #define IRQ_ERR (1 << 2) #define IRQ_DMAERR (1 << 7) /* this is not documented ??? */ /* - 0:7 24:31 DltTmrValue - 8:15 16:23 ClscCntrValue - 16:17 14:15 Reserved - 18:21 10:13 ClscCnt - 22:23 8:9 DlyCnt - 24:28 3::7 Reserved -*/ + * 0:7 24:31 DltTmrValue + * 8:15 16:23 ClscCntrValue + * 16:17 14:15 Reserved + * 18:21 10:13 ClscCnt + * 22:23 8:9 DlyCnt + * 24:28 3::7 Reserved + */ #define RX_CHNL_STS 0x0f /* r */ #define CHNL_STS_ENGBUSY (1 << 1) #define CHNL_STS_EOP (1 << 2) @@ -165,23 +174,23 @@ This option defaults to enabled (set) */ #define CHNL_STS_CMPERR (1 << 20) #define CHNL_STS_TAILERR (1 << 21) /* - 0:9 22:31 Reserved - 10 21 TailPErr - 11 20 CmpErr - 12 19 AddrErr - 13 18 NxtPErr - 14 17 CurPErr - 15 16 BsyWr - 16:23 8:15 Reserved - 24 7 Error - 25 6 IOE - 26 5 SOE - 27 4 Cmplt - 28 3 SOP - 29 2 EOP - 30 1 EngBusy - 31 0 Reserved -*/ + * 0:9 22:31 Reserved + * 10 21 TailPErr + * 11 20 CmpErr + * 12 19 AddrErr + * 13 18 NxtPErr + * 14 17 CurPErr + * 15 16 BsyWr + * 16:23 8:15 Reserved + * 24 7 Error + * 25 6 IOE + * 26 5 SOE + * 27 4 Cmplt + * 28 3 SOP + * 29 2 EOP + * 30 1 EngBusy + * 31 0 Reserved + */ #define DMA_CONTROL_REG 0x10 /* rw */ #define DMA_CONTROL_RST (1 << 0) diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index 3f6b9dfca095cf93251df3d812ac0e30cdc9b365..1066420d6a83aa55ff6689329b44c0b57ffa37a4 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -117,8 +117,8 @@ int temac_indirect_busywait(struct temac_local *lp) spin_until_cond(hard_acs_rdy_or_timeout(lp, timeout)); if (WARN_ON(!hard_acs_rdy(lp))) return -ETIMEDOUT; - else - return 0; + + return 0; } /* @@ -261,7 +261,7 @@ static void temac_dma_dcr_out(struct temac_local *lp, int reg, u32 value) * I/O functions */ static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op, - struct device_node *np) + struct device_node *np) { unsigned int dcrs; @@ -286,7 +286,7 @@ static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op, * such as with MicroBlaze and x86 */ static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op, - struct device_node *np) + struct device_node *np) { return -1; } @@ -307,11 +307,9 @@ static void temac_dma_bd_release(struct net_device *ndev) for (i = 0; i < lp->rx_bd_num; i++) { if (!lp->rx_skb[i]) break; - else { - dma_unmap_single(ndev->dev.parent, lp->rx_bd_v[i].phys, - XTE_MAX_JUMBO_FRAME_SIZE, DMA_FROM_DEVICE); - dev_kfree_skb(lp->rx_skb[i]); - } + dma_unmap_single(ndev->dev.parent, lp->rx_bd_v[i].phys, + XTE_MAX_JUMBO_FRAME_SIZE, DMA_FROM_DEVICE); + dev_kfree_skb(lp->rx_skb[i]); } if (lp->rx_bd_v) dma_free_coherent(ndev->dev.parent, @@ -430,7 +428,8 @@ static void temac_do_set_mac_address(struct net_device *ndev) (ndev->dev_addr[2] << 16) | (ndev->dev_addr[3] << 24)); /* There are reserved bits in EUAW1 - * so don't affect them Set MAC bits [47:32] in EUAW1 */ + * so don't affect them Set MAC bits [47:32] in EUAW1 + */ temac_indirect_out32_locked(lp, XTE_UAW1_OFFSET, (ndev->dev_addr[4] & 0x000000ff) | (ndev->dev_addr[5] << 8)); @@ -530,66 +529,66 @@ static struct temac_option { { .opt = XTE_OPTION_JUMBO, .reg = XTE_RXC1_OFFSET, - .m_or =XTE_RXC1_RXJMBO_MASK, + .m_or = XTE_RXC1_RXJMBO_MASK, }, /* Turn on VLAN packet support for both Rx and Tx */ { .opt = XTE_OPTION_VLAN, .reg = XTE_TXC_OFFSET, - .m_or =XTE_TXC_TXVLAN_MASK, + .m_or = XTE_TXC_TXVLAN_MASK, }, { .opt = XTE_OPTION_VLAN, .reg = XTE_RXC1_OFFSET, - .m_or =XTE_RXC1_RXVLAN_MASK, + .m_or = XTE_RXC1_RXVLAN_MASK, }, /* Turn on FCS stripping on receive packets */ { .opt = XTE_OPTION_FCS_STRIP, .reg = XTE_RXC1_OFFSET, - .m_or =XTE_RXC1_RXFCS_MASK, + .m_or = XTE_RXC1_RXFCS_MASK, }, /* Turn on FCS insertion on transmit packets */ { .opt = XTE_OPTION_FCS_INSERT, .reg = XTE_TXC_OFFSET, - .m_or =XTE_TXC_TXFCS_MASK, + .m_or = XTE_TXC_TXFCS_MASK, }, /* Turn on length/type field checking on receive packets */ { .opt = XTE_OPTION_LENTYPE_ERR, .reg = XTE_RXC1_OFFSET, - .m_or =XTE_RXC1_RXLT_MASK, + .m_or = XTE_RXC1_RXLT_MASK, }, /* Turn on flow control */ { .opt = XTE_OPTION_FLOW_CONTROL, .reg = XTE_FCC_OFFSET, - .m_or =XTE_FCC_RXFLO_MASK, + .m_or = XTE_FCC_RXFLO_MASK, }, /* Turn on flow control */ { .opt = XTE_OPTION_FLOW_CONTROL, .reg = XTE_FCC_OFFSET, - .m_or =XTE_FCC_TXFLO_MASK, + .m_or = XTE_FCC_TXFLO_MASK, }, /* Turn on promiscuous frame filtering (all frames are received ) */ { .opt = XTE_OPTION_PROMISC, .reg = XTE_AFM_OFFSET, - .m_or =XTE_AFM_EPPRM_MASK, + .m_or = XTE_AFM_EPPRM_MASK, }, /* Enable transmitter if not already enabled */ { .opt = XTE_OPTION_TXEN, .reg = XTE_TXC_OFFSET, - .m_or =XTE_TXC_TXEN_MASK, + .m_or = XTE_TXC_TXEN_MASK, }, /* Enable receiver? */ { .opt = XTE_OPTION_RXEN, .reg = XTE_RXC1_OFFSET, - .m_or =XTE_RXC1_RXEN_MASK, + .m_or = XTE_RXC1_RXEN_MASK, }, {} }; @@ -641,7 +640,7 @@ static void temac_device_reset(struct net_device *ndev) udelay(1); if (--timeout == 0) { dev_err(&ndev->dev, - "temac_device_reset RX reset timeout!!\n"); + "%s RX reset timeout!!\n", __func__); break; } } @@ -653,7 +652,7 @@ static void temac_device_reset(struct net_device *ndev) udelay(1); if (--timeout == 0) { dev_err(&ndev->dev, - "temac_device_reset TX reset timeout!!\n"); + "%s TX reset timeout!!\n", __func__); break; } } @@ -672,7 +671,7 @@ static void temac_device_reset(struct net_device *ndev) udelay(1); if (--timeout == 0) { dev_err(&ndev->dev, - "temac_device_reset DMA reset timeout!!\n"); + "%s DMA reset timeout!!\n", __func__); break; } } @@ -680,7 +679,7 @@ static void temac_device_reset(struct net_device *ndev) if (temac_dma_bd_init(ndev)) { dev_err(&ndev->dev, - "temac_device_reset descriptor allocation failed\n"); + "%s descriptor allocation failed\n", __func__); } spin_lock_irqsave(lp->indirect_lock, flags); @@ -691,7 +690,8 @@ static void temac_device_reset(struct net_device *ndev) spin_unlock_irqrestore(lp->indirect_lock, flags); /* Sync default options with HW - * but leave receiver and transmitter disabled. */ + * but leave receiver and transmitter disabled. + */ temac_setoptions(ndev, lp->options & ~(XTE_OPTION_TXEN | XTE_OPTION_RXEN)); @@ -723,9 +723,15 @@ static void temac_adjust_link(struct net_device *ndev) mii_speed &= ~XTE_EMCFG_LINKSPD_MASK; switch (phy->speed) { - case SPEED_1000: mii_speed |= XTE_EMCFG_LINKSPD_1000; break; - case SPEED_100: mii_speed |= XTE_EMCFG_LINKSPD_100; break; - case SPEED_10: mii_speed |= XTE_EMCFG_LINKSPD_10; break; + case SPEED_1000: + mii_speed |= XTE_EMCFG_LINKSPD_1000; + break; + case SPEED_100: + mii_speed |= XTE_EMCFG_LINKSPD_100; + break; + case SPEED_10: + mii_speed |= XTE_EMCFG_LINKSPD_10; + break; } /* Write new speed setting out to TEMAC */ @@ -1007,7 +1013,6 @@ static void ll_temac_recv(struct net_device *ndev) if (((lp->temac_features & TEMAC_FEATURE_RX_CSUM) != 0) && (skb->protocol == htons(ETH_P_IP)) && (skb->len > 64)) { - /* Convert from device endianness (be32) to cpu * endianness, and if necessary swap the bytes * (back) for proper IP checksum byte order @@ -1563,16 +1568,12 @@ static int temac_probe(struct platform_device *pdev) } /* Error handle returned DMA RX and TX interrupts */ - if (lp->rx_irq < 0) { - if (lp->rx_irq != -EPROBE_DEFER) - dev_err(&pdev->dev, "could not get DMA RX irq\n"); - return lp->rx_irq; - } - if (lp->tx_irq < 0) { - if (lp->tx_irq != -EPROBE_DEFER) - dev_err(&pdev->dev, "could not get DMA TX irq\n"); - return lp->tx_irq; - } + if (lp->rx_irq < 0) + return dev_err_probe(&pdev->dev, lp->rx_irq, + "could not get DMA RX irq\n"); + if (lp->tx_irq < 0) + return dev_err_probe(&pdev->dev, lp->tx_irq, + "could not get DMA TX irq\n"); if (temac_np) { /* Retrieve the MAC address */ diff --git a/drivers/net/ethernet/xilinx/ll_temac_mdio.c b/drivers/net/ethernet/xilinx/ll_temac_mdio.c index 6fd2dea4e60f75c01c666525449ea7328b741493..2371c072b53f3ad239063f842988cc1d5b41b9de 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_mdio.c +++ b/drivers/net/ethernet/xilinx/ll_temac_mdio.c @@ -29,7 +29,8 @@ static int temac_mdio_read(struct mii_bus *bus, int phy_id, int reg) /* Write the PHY address to the MIIM Access Initiator register. * When the transfer completes, the PHY register value will appear - * in the LSW0 register */ + * in the LSW0 register + */ spin_lock_irqsave(lp->indirect_lock, flags); temac_iow(lp, XTE_LSW0_OFFSET, (phy_id << 5) | reg); rc = temac_indirect_in32_locked(lp, XTE_MIIMAI_OFFSET); @@ -88,7 +89,8 @@ int temac_mdio_setup(struct temac_local *lp, struct platform_device *pdev) } /* Enable the MDIO bus by asserting the enable bit and writing - * in the clock config */ + * in the clock config + */ temac_indirect_out32(lp, XTE_MC_OFFSET, 1 << 6 | clk_div); bus = devm_mdiobus_alloc(&pdev->dev); diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h index f2e2261b4b7d90078a1a4b3707971c2000aff72e..6370c447ac5cacfb8ef7e6144b0cb2d88a4204f0 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet.h +++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h @@ -402,6 +402,9 @@ struct axidma_bd { * @rx_bd_num: Size of RX buffer descriptor ring * @rx_bd_ci: Stores the index of the Rx buffer descriptor in the ring being * accessed currently. + * @rx_packets: RX packet count for statistics + * @rx_bytes: RX byte count for statistics + * @rx_stat_sync: Synchronization object for RX stats * @napi_tx: NAPI TX control structure * @tx_dma_cr: Nominal content of TX DMA control register * @tx_bd_v: Virtual address of the TX buffer descriptor ring @@ -411,6 +414,9 @@ struct axidma_bd { * complete. Only updated at runtime by TX NAPI poll. * @tx_bd_tail: Stores the index of the next Tx buffer descriptor in the ring * to be populated. + * @tx_packets: TX packet count for statistics + * @tx_bytes: TX byte count for statistics + * @tx_stat_sync: Synchronization object for TX stats * @dma_err_task: Work structure to process Axi DMA errors * @tx_irq: Axidma TX IRQ number * @rx_irq: Axidma RX IRQ number @@ -458,6 +464,9 @@ struct axienet_local { dma_addr_t rx_bd_p; u32 rx_bd_num; u32 rx_bd_ci; + u64_stats_t rx_packets; + u64_stats_t rx_bytes; + struct u64_stats_sync rx_stat_sync; struct napi_struct napi_tx; u32 tx_dma_cr; @@ -466,6 +475,9 @@ struct axienet_local { u32 tx_bd_num; u32 tx_bd_ci; u32 tx_bd_tail; + u64_stats_t tx_packets; + u64_stats_t tx_bytes; + struct u64_stats_sync tx_stat_sync; struct work_struct dma_err_task; @@ -591,7 +603,7 @@ static inline void axienet_dma_out_addr(struct axienet_local *lp, off_t reg, #else /* CONFIG_64BIT */ static inline void axienet_dma_out_addr(struct axienet_local *lp, off_t reg, - dma_addr_t addr) + dma_addr_t addr) { axienet_dma_out32(lp, reg, lower_32_bits(addr)); } diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 1760930ec0c49ddaf95bcb72fbc0046ba46badcf..d1d772580da98e576d51c91ea0633f283f44d64e 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -597,7 +597,7 @@ static int axienet_device_reset(struct net_device *ndev) lp->options &= (~XAE_OPTION_JUMBO); if ((ndev->mtu > XAE_MTU) && - (ndev->mtu <= XAE_JUMBO_MTU)) { + (ndev->mtu <= XAE_JUMBO_MTU)) { lp->max_frm_size = ndev->mtu + VLAN_ETH_HLEN + XAE_TRL_SIZE; @@ -645,7 +645,7 @@ static int axienet_device_reset(struct net_device *ndev) * @nr_bds: Max number of descriptors to clean up * @force: Whether to clean descriptors even if not complete * @sizep: Pointer to a u32 filled with the total sum of all bytes - * in all cleaned-up descriptors. Ignored if NULL. + * in all cleaned-up descriptors. Ignored if NULL. * @budget: NAPI budget (use 0 when not called from NAPI poll) * * Would either be called after a successful transmit operation, or after @@ -752,8 +752,10 @@ static int axienet_tx_poll(struct napi_struct *napi, int budget) if (lp->tx_bd_ci >= lp->tx_bd_num) lp->tx_bd_ci %= lp->tx_bd_num; - ndev->stats.tx_packets += packets; - ndev->stats.tx_bytes += size; + u64_stats_update_begin(&lp->tx_stat_sync); + u64_stats_add(&lp->tx_packets, packets); + u64_stats_add(&lp->tx_bytes, size); + u64_stats_update_end(&lp->tx_stat_sync); /* Matches barrier in axienet_start_xmit */ smp_mb(); @@ -984,8 +986,10 @@ static int axienet_rx_poll(struct napi_struct *napi, int budget) cur_p = &lp->rx_bd_v[lp->rx_bd_ci]; } - lp->ndev->stats.rx_packets += packets; - lp->ndev->stats.rx_bytes += size; + u64_stats_update_begin(&lp->rx_stat_sync); + u64_stats_add(&lp->rx_packets, packets); + u64_stats_add(&lp->rx_bytes, size); + u64_stats_update_end(&lp->rx_stat_sync); if (tail_p) axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p); @@ -1292,10 +1296,32 @@ static int axienet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) return phylink_mii_ioctl(lp->phylink, rq, cmd); } +static void +axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) +{ + struct axienet_local *lp = netdev_priv(dev); + unsigned int start; + + netdev_stats_to_stats64(stats, &dev->stats); + + do { + start = u64_stats_fetch_begin_irq(&lp->rx_stat_sync); + stats->rx_packets = u64_stats_read(&lp->rx_packets); + stats->rx_bytes = u64_stats_read(&lp->rx_bytes); + } while (u64_stats_fetch_retry_irq(&lp->rx_stat_sync, start)); + + do { + start = u64_stats_fetch_begin_irq(&lp->tx_stat_sync); + stats->tx_packets = u64_stats_read(&lp->tx_packets); + stats->tx_bytes = u64_stats_read(&lp->tx_bytes); + } while (u64_stats_fetch_retry_irq(&lp->tx_stat_sync, start)); +} + static const struct net_device_ops axienet_netdev_ops = { .ndo_open = axienet_open, .ndo_stop = axienet_stop, .ndo_start_xmit = axienet_start_xmit, + .ndo_get_stats64 = axienet_get_stats64, .ndo_change_mtu = axienet_change_mtu, .ndo_set_mac_address = netdev_set_mac_address, .ndo_validate_addr = eth_validate_addr, @@ -1317,8 +1343,8 @@ static const struct net_device_ops axienet_netdev_ops = { static void axienet_ethtools_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *ed) { - strlcpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); - strlcpy(ed->version, DRIVER_VERSION, sizeof(ed->version)); + strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); + strscpy(ed->version, DRIVER_VERSION, sizeof(ed->version)); } /** @@ -1349,7 +1375,7 @@ static int axienet_ethtools_get_regs_len(struct net_device *ndev) static void axienet_ethtools_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *ret) { - u32 *data = (u32 *) ret; + u32 *data = (u32 *)ret; size_t len = sizeof(u32) * AXIENET_REGS_N; struct axienet_local *lp = netdev_priv(ndev); @@ -1850,8 +1876,11 @@ static int axienet_probe(struct platform_device *pdev) lp->rx_bd_num = RX_BD_NUM_DEFAULT; lp->tx_bd_num = TX_BD_NUM_DEFAULT; - netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll, NAPI_POLL_WEIGHT); - netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll, NAPI_POLL_WEIGHT); + u64_stats_init(&lp->rx_stat_sync); + u64_stats_init(&lp->tx_stat_sync); + + netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll); + netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll); lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk"); if (!lp->axi_clk) { diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c b/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c index 2772a79cd3eddd1ec6b2958c6e40b096663af30e..0b3b6935c55816071cf00f0b71a5234217e5ac71 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c @@ -126,7 +126,7 @@ static int axienet_mdio_write(struct mii_bus *bus, int phy_id, int reg, return ret; } - axienet_iow(lp, XAE_MDIO_MWD_OFFSET, (u32) val); + axienet_iow(lp, XAE_MDIO_MWD_OFFSET, (u32)val); axienet_iow(lp, XAE_MDIO_MCR_OFFSET, (((phy_id << XAE_MDIO_MCR_PHYAD_SHIFT) & XAE_MDIO_MCR_PHYAD_MASK) | diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c index 016a9c4f2c6c734b37290aa39d40f0bd2edd9f02..05848ff15fb51a7b64d777517b5d4855dd56cf38 100644 --- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c +++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c @@ -1060,7 +1060,7 @@ static bool get_bool(struct platform_device *ofdev, const char *s) static void xemaclite_ethtools_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *ed) { - strlcpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); + strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver)); } static const struct ethtool_ops xemaclite_ethtool_ops = { diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c index f9587e55b842831a1e820392e6f2930ffe2cf22e..894e92ef415b98b761af99fa917838b0f640e8fb 100644 --- a/drivers/net/ethernet/xircom/xirc2ps_cs.c +++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c @@ -1402,7 +1402,7 @@ do_open(struct net_device *dev) static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "xirc2ps_cs", sizeof(info->driver)); + strscpy(info->driver, "xirc2ps_cs", sizeof(info->driver)); snprintf(info->bus_info, sizeof(info->bus_info), "PCMCIA 0x%lx", dev->base_addr); } diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c index 3591b9edc9a124c710d7179c7f48c44b01f64f9e..3b0c5f177447bf2e0bd8ba73920e41c2d2b8a3c9 100644 --- a/drivers/net/ethernet/xscale/ixp4xx_eth.c +++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c @@ -841,7 +841,7 @@ static void eth_txdone_irq(void *unused) } } -static int eth_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t eth_xmit(struct sk_buff *skb, struct net_device *dev) { struct port *port = netdev_priv(dev); unsigned int txreadyq = port->plat->txreadyq; @@ -999,11 +999,11 @@ static void ixp4xx_get_drvinfo(struct net_device *dev, { struct port *port = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); snprintf(info->fw_version, sizeof(info->fw_version), "%u:%u:%u:%u", port->firmware[0], port->firmware[1], port->firmware[2], port->firmware[3]); - strlcpy(info->bus_info, "internal", sizeof(info->bus_info)); + strscpy(info->bus_info, "internal", sizeof(info->bus_info)); } static int ixp4xx_get_ts_info(struct net_device *dev, diff --git a/drivers/net/fjes/fjes_ethtool.c b/drivers/net/fjes/fjes_ethtool.c index 746736c83873702ee30114f5bce45b32ce7352eb..19c99529566b3d574a9e47aa0f9c92abe6219153 100644 --- a/drivers/net/fjes/fjes_ethtool.c +++ b/drivers/net/fjes/fjes_ethtool.c @@ -151,11 +151,11 @@ static void fjes_get_drvinfo(struct net_device *netdev, plat_dev = adapter->plat_dev; - strlcpy(drvinfo->driver, fjes_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, fjes_driver_version, + strscpy(drvinfo->driver, fjes_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, fjes_driver_version, sizeof(drvinfo->version)); - strlcpy(drvinfo->fw_version, "none", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->fw_version, "none", sizeof(drvinfo->fw_version)); snprintf(drvinfo->bus_info, sizeof(drvinfo->bus_info), "platform:%s", plat_dev->name); } diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c index 5805e4a563859bd4e23ad2eca98301ba6eb28f30..1eff202f6a1fa2dd6590aa7313f70d3ffe65ebda 100644 --- a/drivers/net/fjes/fjes_main.c +++ b/drivers/net/fjes/fjes_main.c @@ -32,68 +32,12 @@ MODULE_VERSION(DRV_VERSION); #define ACPI_MOTHERBOARD_RESOURCE_HID "PNP0C02" -static int fjes_request_irq(struct fjes_adapter *); -static void fjes_free_irq(struct fjes_adapter *); - -static int fjes_open(struct net_device *); -static int fjes_close(struct net_device *); -static int fjes_setup_resources(struct fjes_adapter *); -static void fjes_free_resources(struct fjes_adapter *); -static netdev_tx_t fjes_xmit_frame(struct sk_buff *, struct net_device *); -static void fjes_raise_intr_rxdata_task(struct work_struct *); -static void fjes_tx_stall_task(struct work_struct *); -static void fjes_force_close_task(struct work_struct *); -static irqreturn_t fjes_intr(int, void*); -static void fjes_get_stats64(struct net_device *, struct rtnl_link_stats64 *); -static int fjes_change_mtu(struct net_device *, int); -static int fjes_vlan_rx_add_vid(struct net_device *, __be16 proto, u16); -static int fjes_vlan_rx_kill_vid(struct net_device *, __be16 proto, u16); -static void fjes_tx_retry(struct net_device *, unsigned int txqueue); - -static int fjes_acpi_add(struct acpi_device *); -static int fjes_acpi_remove(struct acpi_device *); -static acpi_status fjes_get_acpi_resource(struct acpi_resource *, void*); - -static int fjes_probe(struct platform_device *); -static int fjes_remove(struct platform_device *); - -static int fjes_sw_init(struct fjes_adapter *); -static void fjes_netdev_setup(struct net_device *); -static void fjes_irq_watch_task(struct work_struct *); -static void fjes_watch_unshare_task(struct work_struct *); -static void fjes_rx_irq(struct fjes_adapter *, int); -static int fjes_poll(struct napi_struct *, int); - static const struct acpi_device_id fjes_acpi_ids[] = { {ACPI_MOTHERBOARD_RESOURCE_HID, 0}, {"", 0}, }; MODULE_DEVICE_TABLE(acpi, fjes_acpi_ids); -static struct acpi_driver fjes_acpi_driver = { - .name = DRV_NAME, - .class = DRV_NAME, - .owner = THIS_MODULE, - .ids = fjes_acpi_ids, - .ops = { - .add = fjes_acpi_add, - .remove = fjes_acpi_remove, - }, -}; - -static struct platform_driver fjes_driver = { - .driver = { - .name = DRV_NAME, - }, - .probe = fjes_probe, - .remove = fjes_remove, -}; - -static struct resource fjes_resource[] = { - DEFINE_RES_MEM(0, 1), - DEFINE_RES_IRQ(0) -}; - static bool is_extended_socket_device(struct acpi_device *device) { struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL}; @@ -139,43 +83,6 @@ static int acpi_check_extended_socket_status(struct acpi_device *device) return 0; } -static int fjes_acpi_add(struct acpi_device *device) -{ - struct platform_device *plat_dev; - acpi_status status; - - if (!is_extended_socket_device(device)) - return -ENODEV; - - if (acpi_check_extended_socket_status(device)) - return -ENODEV; - - status = acpi_walk_resources(device->handle, METHOD_NAME__CRS, - fjes_get_acpi_resource, fjes_resource); - if (ACPI_FAILURE(status)) - return -ENODEV; - - /* create platform_device */ - plat_dev = platform_device_register_simple(DRV_NAME, 0, fjes_resource, - ARRAY_SIZE(fjes_resource)); - if (IS_ERR(plat_dev)) - return PTR_ERR(plat_dev); - - device->driver_data = plat_dev; - - return 0; -} - -static int fjes_acpi_remove(struct acpi_device *device) -{ - struct platform_device *plat_dev; - - plat_dev = (struct platform_device *)acpi_driver_data(device); - platform_device_unregister(plat_dev); - - return 0; -} - static acpi_status fjes_get_acpi_resource(struct acpi_resource *acpi_res, void *data) { @@ -206,143 +113,59 @@ fjes_get_acpi_resource(struct acpi_resource *acpi_res, void *data) return AE_OK; } -static int fjes_request_irq(struct fjes_adapter *adapter) -{ - struct net_device *netdev = adapter->netdev; - int result = -1; - - adapter->interrupt_watch_enable = true; - if (!delayed_work_pending(&adapter->interrupt_watch_task)) { - queue_delayed_work(adapter->control_wq, - &adapter->interrupt_watch_task, - FJES_IRQ_WATCH_DELAY); - } - - if (!adapter->irq_registered) { - result = request_irq(adapter->hw.hw_res.irq, fjes_intr, - IRQF_SHARED, netdev->name, adapter); - if (result) - adapter->irq_registered = false; - else - adapter->irq_registered = true; - } - - return result; -} - -static void fjes_free_irq(struct fjes_adapter *adapter) -{ - struct fjes_hw *hw = &adapter->hw; - - adapter->interrupt_watch_enable = false; - cancel_delayed_work_sync(&adapter->interrupt_watch_task); - - fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, true); - - if (adapter->irq_registered) { - free_irq(adapter->hw.hw_res.irq, adapter); - adapter->irq_registered = false; - } -} - -static const struct net_device_ops fjes_netdev_ops = { - .ndo_open = fjes_open, - .ndo_stop = fjes_close, - .ndo_start_xmit = fjes_xmit_frame, - .ndo_get_stats64 = fjes_get_stats64, - .ndo_change_mtu = fjes_change_mtu, - .ndo_tx_timeout = fjes_tx_retry, - .ndo_vlan_rx_add_vid = fjes_vlan_rx_add_vid, - .ndo_vlan_rx_kill_vid = fjes_vlan_rx_kill_vid, +static struct resource fjes_resource[] = { + DEFINE_RES_MEM(0, 1), + DEFINE_RES_IRQ(0) }; -/* fjes_open - Called when a network interface is made active */ -static int fjes_open(struct net_device *netdev) +static int fjes_acpi_add(struct acpi_device *device) { - struct fjes_adapter *adapter = netdev_priv(netdev); - struct fjes_hw *hw = &adapter->hw; - int result; - - if (adapter->open_guard) - return -ENXIO; - - result = fjes_setup_resources(adapter); - if (result) - goto err_setup_res; - - hw->txrx_stop_req_bit = 0; - hw->epstop_req_bit = 0; + struct platform_device *plat_dev; + acpi_status status; - napi_enable(&adapter->napi); + if (!is_extended_socket_device(device)) + return -ENODEV; - fjes_hw_capture_interrupt_status(hw); + if (acpi_check_extended_socket_status(device)) + return -ENODEV; - result = fjes_request_irq(adapter); - if (result) - goto err_req_irq; + status = acpi_walk_resources(device->handle, METHOD_NAME__CRS, + fjes_get_acpi_resource, fjes_resource); + if (ACPI_FAILURE(status)) + return -ENODEV; - fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, false); + /* create platform_device */ + plat_dev = platform_device_register_simple(DRV_NAME, 0, fjes_resource, + ARRAY_SIZE(fjes_resource)); + if (IS_ERR(plat_dev)) + return PTR_ERR(plat_dev); - netif_tx_start_all_queues(netdev); - netif_carrier_on(netdev); + device->driver_data = plat_dev; return 0; - -err_req_irq: - fjes_free_irq(adapter); - napi_disable(&adapter->napi); - -err_setup_res: - fjes_free_resources(adapter); - return result; } -/* fjes_close - Disables a network interface */ -static int fjes_close(struct net_device *netdev) +static int fjes_acpi_remove(struct acpi_device *device) { - struct fjes_adapter *adapter = netdev_priv(netdev); - struct fjes_hw *hw = &adapter->hw; - unsigned long flags; - int epidx; - - netif_tx_stop_all_queues(netdev); - netif_carrier_off(netdev); - - fjes_hw_raise_epstop(hw); - - napi_disable(&adapter->napi); - - spin_lock_irqsave(&hw->rx_status_lock, flags); - for (epidx = 0; epidx < hw->max_epid; epidx++) { - if (epidx == hw->my_epid) - continue; - - if (fjes_hw_get_partner_ep_status(hw, epidx) == - EP_PARTNER_SHARED) - adapter->hw.ep_shm_info[epidx] - .tx.info->v1i.rx_status &= - ~FJES_RX_POLL_WORK; - } - spin_unlock_irqrestore(&hw->rx_status_lock, flags); - - fjes_free_irq(adapter); - - cancel_delayed_work_sync(&adapter->interrupt_watch_task); - cancel_work_sync(&adapter->unshare_watch_task); - adapter->unshare_watch_bitmask = 0; - cancel_work_sync(&adapter->raise_intr_rxdata_task); - cancel_work_sync(&adapter->tx_stall_task); - - cancel_work_sync(&hw->update_zone_task); - cancel_work_sync(&hw->epstop_task); - - fjes_hw_wait_epstop(hw); + struct platform_device *plat_dev; - fjes_free_resources(adapter); + plat_dev = (struct platform_device *)acpi_driver_data(device); + platform_device_unregister(plat_dev); return 0; } +static struct acpi_driver fjes_acpi_driver = { + .name = DRV_NAME, + .class = DRV_NAME, + .owner = THIS_MODULE, + .ids = fjes_acpi_ids, + .ops = { + .add = fjes_acpi_add, + .remove = fjes_acpi_remove, + }, +}; + static int fjes_setup_resources(struct fjes_adapter *adapter) { struct net_device *netdev = adapter->netdev; @@ -421,38 +244,220 @@ static int fjes_setup_resources(struct fjes_adapter *adapter) return 0; } -static void fjes_free_resources(struct fjes_adapter *adapter) +static void fjes_rx_irq(struct fjes_adapter *adapter, int src_epid) { - struct net_device *netdev = adapter->netdev; - struct fjes_device_command_param param; - struct ep_share_mem_info *buf_pair; struct fjes_hw *hw = &adapter->hw; - bool reset_flag = false; - unsigned long flags; - int result; - int epidx; - - for (epidx = 0; epidx < hw->max_epid; epidx++) { - if (epidx == hw->my_epid) - continue; - mutex_lock(&hw->hw_info.lock); - result = fjes_hw_unregister_buff_addr(hw, epidx); - mutex_unlock(&hw->hw_info.lock); + fjes_hw_set_irqmask(hw, REG_ICTL_MASK_RX_DATA, true); - hw->ep_shm_info[epidx].ep_stats.com_unregist_buf_exec += 1; + adapter->unset_rx_last = true; + napi_schedule(&adapter->napi); +} - if (result) - reset_flag = true; +static void fjes_stop_req_irq(struct fjes_adapter *adapter, int src_epid) +{ + struct fjes_hw *hw = &adapter->hw; + enum ep_partner_status status; + unsigned long flags; - buf_pair = &hw->ep_shm_info[epidx]; + set_bit(src_epid, &hw->hw_info.buffer_unshare_reserve_bit); + status = fjes_hw_get_partner_ep_status(hw, src_epid); + trace_fjes_stop_req_irq_pre(hw, src_epid, status); + switch (status) { + case EP_PARTNER_WAITING: spin_lock_irqsave(&hw->rx_status_lock, flags); - fjes_hw_setup_epbuf(&buf_pair->tx, - netdev->dev_addr, netdev->mtu); + hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |= + FJES_RX_STOP_REQ_DONE; spin_unlock_irqrestore(&hw->rx_status_lock, flags); - - clear_bit(epidx, &hw->txrx_stop_req_bit); + clear_bit(src_epid, &hw->txrx_stop_req_bit); + fallthrough; + case EP_PARTNER_UNSHARE: + case EP_PARTNER_COMPLETE: + default: + set_bit(src_epid, &adapter->unshare_watch_bitmask); + if (!work_pending(&adapter->unshare_watch_task)) + queue_work(adapter->control_wq, + &adapter->unshare_watch_task); + break; + case EP_PARTNER_SHARED: + set_bit(src_epid, &hw->epstop_req_bit); + + if (!work_pending(&hw->epstop_task)) + queue_work(adapter->control_wq, &hw->epstop_task); + break; + } + trace_fjes_stop_req_irq_post(hw, src_epid); +} + +static void fjes_txrx_stop_req_irq(struct fjes_adapter *adapter, + int src_epid) +{ + struct fjes_hw *hw = &adapter->hw; + enum ep_partner_status status; + unsigned long flags; + + status = fjes_hw_get_partner_ep_status(hw, src_epid); + trace_fjes_txrx_stop_req_irq_pre(hw, src_epid, status); + switch (status) { + case EP_PARTNER_UNSHARE: + case EP_PARTNER_COMPLETE: + default: + break; + case EP_PARTNER_WAITING: + if (src_epid < hw->my_epid) { + spin_lock_irqsave(&hw->rx_status_lock, flags); + hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |= + FJES_RX_STOP_REQ_DONE; + spin_unlock_irqrestore(&hw->rx_status_lock, flags); + + clear_bit(src_epid, &hw->txrx_stop_req_bit); + set_bit(src_epid, &adapter->unshare_watch_bitmask); + + if (!work_pending(&adapter->unshare_watch_task)) + queue_work(adapter->control_wq, + &adapter->unshare_watch_task); + } + break; + case EP_PARTNER_SHARED: + if (hw->ep_shm_info[src_epid].rx.info->v1i.rx_status & + FJES_RX_STOP_REQ_REQUEST) { + set_bit(src_epid, &hw->epstop_req_bit); + if (!work_pending(&hw->epstop_task)) + queue_work(adapter->control_wq, + &hw->epstop_task); + } + break; + } + trace_fjes_txrx_stop_req_irq_post(hw, src_epid); +} + +static void fjes_update_zone_irq(struct fjes_adapter *adapter, + int src_epid) +{ + struct fjes_hw *hw = &adapter->hw; + + if (!work_pending(&hw->update_zone_task)) + queue_work(adapter->control_wq, &hw->update_zone_task); +} + +static irqreturn_t fjes_intr(int irq, void *data) +{ + struct fjes_adapter *adapter = data; + struct fjes_hw *hw = &adapter->hw; + irqreturn_t ret; + u32 icr; + + icr = fjes_hw_capture_interrupt_status(hw); + + if (icr & REG_IS_MASK_IS_ASSERT) { + if (icr & REG_ICTL_MASK_RX_DATA) { + fjes_rx_irq(adapter, icr & REG_IS_MASK_EPID); + hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats + .recv_intr_rx += 1; + } + + if (icr & REG_ICTL_MASK_DEV_STOP_REQ) { + fjes_stop_req_irq(adapter, icr & REG_IS_MASK_EPID); + hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats + .recv_intr_stop += 1; + } + + if (icr & REG_ICTL_MASK_TXRX_STOP_REQ) { + fjes_txrx_stop_req_irq(adapter, icr & REG_IS_MASK_EPID); + hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats + .recv_intr_unshare += 1; + } + + if (icr & REG_ICTL_MASK_TXRX_STOP_DONE) + fjes_hw_set_irqmask(hw, + REG_ICTL_MASK_TXRX_STOP_DONE, true); + + if (icr & REG_ICTL_MASK_INFO_UPDATE) { + fjes_update_zone_irq(adapter, icr & REG_IS_MASK_EPID); + hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats + .recv_intr_zoneupdate += 1; + } + + ret = IRQ_HANDLED; + } else { + ret = IRQ_NONE; + } + + return ret; +} + +static int fjes_request_irq(struct fjes_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + int result = -1; + + adapter->interrupt_watch_enable = true; + if (!delayed_work_pending(&adapter->interrupt_watch_task)) { + queue_delayed_work(adapter->control_wq, + &adapter->interrupt_watch_task, + FJES_IRQ_WATCH_DELAY); + } + + if (!adapter->irq_registered) { + result = request_irq(adapter->hw.hw_res.irq, fjes_intr, + IRQF_SHARED, netdev->name, adapter); + if (result) + adapter->irq_registered = false; + else + adapter->irq_registered = true; + } + + return result; +} + +static void fjes_free_irq(struct fjes_adapter *adapter) +{ + struct fjes_hw *hw = &adapter->hw; + + adapter->interrupt_watch_enable = false; + cancel_delayed_work_sync(&adapter->interrupt_watch_task); + + fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, true); + + if (adapter->irq_registered) { + free_irq(adapter->hw.hw_res.irq, adapter); + adapter->irq_registered = false; + } +} + +static void fjes_free_resources(struct fjes_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + struct fjes_device_command_param param; + struct ep_share_mem_info *buf_pair; + struct fjes_hw *hw = &adapter->hw; + bool reset_flag = false; + unsigned long flags; + int result; + int epidx; + + for (epidx = 0; epidx < hw->max_epid; epidx++) { + if (epidx == hw->my_epid) + continue; + + mutex_lock(&hw->hw_info.lock); + result = fjes_hw_unregister_buff_addr(hw, epidx); + mutex_unlock(&hw->hw_info.lock); + + hw->ep_shm_info[epidx].ep_stats.com_unregist_buf_exec += 1; + + if (result) + reset_flag = true; + + buf_pair = &hw->ep_shm_info[epidx]; + + spin_lock_irqsave(&hw->rx_status_lock, flags); + fjes_hw_setup_epbuf(&buf_pair->tx, + netdev->dev_addr, netdev->mtu); + spin_unlock_irqrestore(&hw->rx_status_lock, flags); + + clear_bit(epidx, &hw->txrx_stop_req_bit); } if (reset_flag || adapter->force_reset) { @@ -477,121 +482,91 @@ static void fjes_free_resources(struct fjes_adapter *adapter) } } -static void fjes_tx_stall_task(struct work_struct *work) +/* fjes_open - Called when a network interface is made active */ +static int fjes_open(struct net_device *netdev) { - struct fjes_adapter *adapter = container_of(work, - struct fjes_adapter, tx_stall_task); - struct net_device *netdev = adapter->netdev; + struct fjes_adapter *adapter = netdev_priv(netdev); struct fjes_hw *hw = &adapter->hw; - int all_queue_available, sendable; - enum ep_partner_status pstatus; - int max_epid, my_epid, epid; - union ep_buffer_info *info; - int i; - - if (((long)jiffies - - dev_trans_start(netdev)) > FJES_TX_TX_STALL_TIMEOUT) { - netif_wake_queue(netdev); - return; - } - - my_epid = hw->my_epid; - max_epid = hw->max_epid; + int result; - for (i = 0; i < 5; i++) { - all_queue_available = 1; + if (adapter->open_guard) + return -ENXIO; - for (epid = 0; epid < max_epid; epid++) { - if (my_epid == epid) - continue; + result = fjes_setup_resources(adapter); + if (result) + goto err_setup_res; - pstatus = fjes_hw_get_partner_ep_status(hw, epid); - sendable = (pstatus == EP_PARTNER_SHARED); - if (!sendable) - continue; + hw->txrx_stop_req_bit = 0; + hw->epstop_req_bit = 0; - info = adapter->hw.ep_shm_info[epid].tx.info; + napi_enable(&adapter->napi); - if (!(info->v1i.rx_status & FJES_RX_MTU_CHANGING_DONE)) - return; + fjes_hw_capture_interrupt_status(hw); - if (EP_RING_FULL(info->v1i.head, info->v1i.tail, - info->v1i.count_max)) { - all_queue_available = 0; - break; - } - } + result = fjes_request_irq(adapter); + if (result) + goto err_req_irq; - if (all_queue_available) { - netif_wake_queue(netdev); - return; - } - } + fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, false); - usleep_range(50, 100); + netif_tx_start_all_queues(netdev); + netif_carrier_on(netdev); - queue_work(adapter->txrx_wq, &adapter->tx_stall_task); -} + return 0; -static void fjes_force_close_task(struct work_struct *work) -{ - struct fjes_adapter *adapter = container_of(work, - struct fjes_adapter, force_close_task); - struct net_device *netdev = adapter->netdev; +err_req_irq: + fjes_free_irq(adapter); + napi_disable(&adapter->napi); - rtnl_lock(); - dev_close(netdev); - rtnl_unlock(); +err_setup_res: + fjes_free_resources(adapter); + return result; } -static void fjes_raise_intr_rxdata_task(struct work_struct *work) +/* fjes_close - Disables a network interface */ +static int fjes_close(struct net_device *netdev) { - struct fjes_adapter *adapter = container_of(work, - struct fjes_adapter, raise_intr_rxdata_task); + struct fjes_adapter *adapter = netdev_priv(netdev); struct fjes_hw *hw = &adapter->hw; - enum ep_partner_status pstatus; - int max_epid, my_epid, epid; + unsigned long flags; + int epidx; - my_epid = hw->my_epid; - max_epid = hw->max_epid; + netif_tx_stop_all_queues(netdev); + netif_carrier_off(netdev); - for (epid = 0; epid < max_epid; epid++) - hw->ep_shm_info[epid].tx_status_work = 0; + fjes_hw_raise_epstop(hw); - for (epid = 0; epid < max_epid; epid++) { - if (epid == my_epid) - continue; + napi_disable(&adapter->napi); - pstatus = fjes_hw_get_partner_ep_status(hw, epid); - if (pstatus == EP_PARTNER_SHARED) { - hw->ep_shm_info[epid].tx_status_work = - hw->ep_shm_info[epid].tx.info->v1i.tx_status; + spin_lock_irqsave(&hw->rx_status_lock, flags); + for (epidx = 0; epidx < hw->max_epid; epidx++) { + if (epidx == hw->my_epid) + continue; - if (hw->ep_shm_info[epid].tx_status_work == - FJES_TX_DELAY_SEND_PENDING) { - hw->ep_shm_info[epid].tx.info->v1i.tx_status = - FJES_TX_DELAY_SEND_NONE; - } - } + if (fjes_hw_get_partner_ep_status(hw, epidx) == + EP_PARTNER_SHARED) + adapter->hw.ep_shm_info[epidx] + .tx.info->v1i.rx_status &= + ~FJES_RX_POLL_WORK; } + spin_unlock_irqrestore(&hw->rx_status_lock, flags); - for (epid = 0; epid < max_epid; epid++) { - if (epid == my_epid) - continue; + fjes_free_irq(adapter); - pstatus = fjes_hw_get_partner_ep_status(hw, epid); - if ((hw->ep_shm_info[epid].tx_status_work == - FJES_TX_DELAY_SEND_PENDING) && - (pstatus == EP_PARTNER_SHARED) && - !(hw->ep_shm_info[epid].rx.info->v1i.rx_status & - FJES_RX_POLL_WORK)) { - fjes_hw_raise_interrupt(hw, epid, - REG_ICTL_MASK_RX_DATA); - hw->ep_shm_info[epid].ep_stats.send_intr_rx += 1; - } - } + cancel_delayed_work_sync(&adapter->interrupt_watch_task); + cancel_work_sync(&adapter->unshare_watch_task); + adapter->unshare_watch_bitmask = 0; + cancel_work_sync(&adapter->raise_intr_rxdata_task); + cancel_work_sync(&adapter->tx_stall_task); - usleep_range(500, 1000); + cancel_work_sync(&hw->update_zone_task); + cancel_work_sync(&hw->epstop_task); + + fjes_hw_wait_epstop(hw); + + fjes_free_resources(adapter); + + return 0; } static int fjes_tx_send(struct fjes_adapter *adapter, int dest, @@ -787,13 +762,6 @@ fjes_xmit_frame(struct sk_buff *skb, struct net_device *netdev) return ret; } -static void fjes_tx_retry(struct net_device *netdev, unsigned int txqueue) -{ - struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0); - - netif_tx_wake_queue(queue); -} - static void fjes_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { @@ -865,179 +833,78 @@ static int fjes_change_mtu(struct net_device *netdev, int new_mtu) netif_tx_wake_all_queues(netdev); netif_carrier_on(netdev); napi_enable(&adapter->napi); - napi_schedule(&adapter->napi); - } - - return ret; -} - -static int fjes_vlan_rx_add_vid(struct net_device *netdev, - __be16 proto, u16 vid) -{ - struct fjes_adapter *adapter = netdev_priv(netdev); - bool ret = true; - int epid; - - for (epid = 0; epid < adapter->hw.max_epid; epid++) { - if (epid == adapter->hw.my_epid) - continue; - - if (!fjes_hw_check_vlan_id( - &adapter->hw.ep_shm_info[epid].tx, vid)) - ret = fjes_hw_set_vlan_id( - &adapter->hw.ep_shm_info[epid].tx, vid); - } - - return ret ? 0 : -ENOSPC; -} - -static int fjes_vlan_rx_kill_vid(struct net_device *netdev, - __be16 proto, u16 vid) -{ - struct fjes_adapter *adapter = netdev_priv(netdev); - int epid; - - for (epid = 0; epid < adapter->hw.max_epid; epid++) { - if (epid == adapter->hw.my_epid) - continue; - - fjes_hw_del_vlan_id(&adapter->hw.ep_shm_info[epid].tx, vid); - } - - return 0; -} - -static void fjes_txrx_stop_req_irq(struct fjes_adapter *adapter, - int src_epid) -{ - struct fjes_hw *hw = &adapter->hw; - enum ep_partner_status status; - unsigned long flags; - - status = fjes_hw_get_partner_ep_status(hw, src_epid); - trace_fjes_txrx_stop_req_irq_pre(hw, src_epid, status); - switch (status) { - case EP_PARTNER_UNSHARE: - case EP_PARTNER_COMPLETE: - default: - break; - case EP_PARTNER_WAITING: - if (src_epid < hw->my_epid) { - spin_lock_irqsave(&hw->rx_status_lock, flags); - hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |= - FJES_RX_STOP_REQ_DONE; - spin_unlock_irqrestore(&hw->rx_status_lock, flags); - - clear_bit(src_epid, &hw->txrx_stop_req_bit); - set_bit(src_epid, &adapter->unshare_watch_bitmask); - - if (!work_pending(&adapter->unshare_watch_task)) - queue_work(adapter->control_wq, - &adapter->unshare_watch_task); - } - break; - case EP_PARTNER_SHARED: - if (hw->ep_shm_info[src_epid].rx.info->v1i.rx_status & - FJES_RX_STOP_REQ_REQUEST) { - set_bit(src_epid, &hw->epstop_req_bit); - if (!work_pending(&hw->epstop_task)) - queue_work(adapter->control_wq, - &hw->epstop_task); - } - break; - } - trace_fjes_txrx_stop_req_irq_post(hw, src_epid); -} - -static void fjes_stop_req_irq(struct fjes_adapter *adapter, int src_epid) -{ - struct fjes_hw *hw = &adapter->hw; - enum ep_partner_status status; - unsigned long flags; - - set_bit(src_epid, &hw->hw_info.buffer_unshare_reserve_bit); - - status = fjes_hw_get_partner_ep_status(hw, src_epid); - trace_fjes_stop_req_irq_pre(hw, src_epid, status); - switch (status) { - case EP_PARTNER_WAITING: - spin_lock_irqsave(&hw->rx_status_lock, flags); - hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |= - FJES_RX_STOP_REQ_DONE; - spin_unlock_irqrestore(&hw->rx_status_lock, flags); - clear_bit(src_epid, &hw->txrx_stop_req_bit); - fallthrough; - case EP_PARTNER_UNSHARE: - case EP_PARTNER_COMPLETE: - default: - set_bit(src_epid, &adapter->unshare_watch_bitmask); - if (!work_pending(&adapter->unshare_watch_task)) - queue_work(adapter->control_wq, - &adapter->unshare_watch_task); - break; - case EP_PARTNER_SHARED: - set_bit(src_epid, &hw->epstop_req_bit); - - if (!work_pending(&hw->epstop_task)) - queue_work(adapter->control_wq, &hw->epstop_task); - break; + napi_schedule(&adapter->napi); } - trace_fjes_stop_req_irq_post(hw, src_epid); + + return ret; } -static void fjes_update_zone_irq(struct fjes_adapter *adapter, - int src_epid) +static void fjes_tx_retry(struct net_device *netdev, unsigned int txqueue) { - struct fjes_hw *hw = &adapter->hw; + struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0); - if (!work_pending(&hw->update_zone_task)) - queue_work(adapter->control_wq, &hw->update_zone_task); + netif_tx_wake_queue(queue); } -static irqreturn_t fjes_intr(int irq, void *data) +static int fjes_vlan_rx_add_vid(struct net_device *netdev, + __be16 proto, u16 vid) { - struct fjes_adapter *adapter = data; - struct fjes_hw *hw = &adapter->hw; - irqreturn_t ret; - u32 icr; - - icr = fjes_hw_capture_interrupt_status(hw); + struct fjes_adapter *adapter = netdev_priv(netdev); + bool ret = true; + int epid; - if (icr & REG_IS_MASK_IS_ASSERT) { - if (icr & REG_ICTL_MASK_RX_DATA) { - fjes_rx_irq(adapter, icr & REG_IS_MASK_EPID); - hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats - .recv_intr_rx += 1; - } + for (epid = 0; epid < adapter->hw.max_epid; epid++) { + if (epid == adapter->hw.my_epid) + continue; - if (icr & REG_ICTL_MASK_DEV_STOP_REQ) { - fjes_stop_req_irq(adapter, icr & REG_IS_MASK_EPID); - hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats - .recv_intr_stop += 1; - } + if (!fjes_hw_check_vlan_id( + &adapter->hw.ep_shm_info[epid].tx, vid)) + ret = fjes_hw_set_vlan_id( + &adapter->hw.ep_shm_info[epid].tx, vid); + } - if (icr & REG_ICTL_MASK_TXRX_STOP_REQ) { - fjes_txrx_stop_req_irq(adapter, icr & REG_IS_MASK_EPID); - hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats - .recv_intr_unshare += 1; - } + return ret ? 0 : -ENOSPC; +} - if (icr & REG_ICTL_MASK_TXRX_STOP_DONE) - fjes_hw_set_irqmask(hw, - REG_ICTL_MASK_TXRX_STOP_DONE, true); +static int fjes_vlan_rx_kill_vid(struct net_device *netdev, + __be16 proto, u16 vid) +{ + struct fjes_adapter *adapter = netdev_priv(netdev); + int epid; - if (icr & REG_ICTL_MASK_INFO_UPDATE) { - fjes_update_zone_irq(adapter, icr & REG_IS_MASK_EPID); - hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats - .recv_intr_zoneupdate += 1; - } + for (epid = 0; epid < adapter->hw.max_epid; epid++) { + if (epid == adapter->hw.my_epid) + continue; - ret = IRQ_HANDLED; - } else { - ret = IRQ_NONE; + fjes_hw_del_vlan_id(&adapter->hw.ep_shm_info[epid].tx, vid); } - return ret; + return 0; +} + +static const struct net_device_ops fjes_netdev_ops = { + .ndo_open = fjes_open, + .ndo_stop = fjes_close, + .ndo_start_xmit = fjes_xmit_frame, + .ndo_get_stats64 = fjes_get_stats64, + .ndo_change_mtu = fjes_change_mtu, + .ndo_tx_timeout = fjes_tx_retry, + .ndo_vlan_rx_add_vid = fjes_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = fjes_vlan_rx_kill_vid, +}; + +/* fjes_netdev_setup - netdevice initialization routine */ +static void fjes_netdev_setup(struct net_device *netdev) +{ + ether_setup(netdev); + + netdev->watchdog_timeo = FJES_TX_RETRY_INTERVAL; + netdev->netdev_ops = &fjes_netdev_ops; + fjes_set_ethtool_ops(netdev); + netdev->mtu = fjes_support_mtu[3]; + netdev->min_mtu = fjes_support_mtu[0]; + netdev->max_mtu = fjes_support_mtu[3]; + netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; } static int fjes_rxframe_search_exist(struct fjes_adapter *adapter, @@ -1087,16 +954,6 @@ static void fjes_rxframe_release(struct fjes_adapter *adapter, int cur_epid) fjes_hw_epbuf_rx_curpkt_drop(&adapter->hw.ep_shm_info[cur_epid].rx); } -static void fjes_rx_irq(struct fjes_adapter *adapter, int src_epid) -{ - struct fjes_hw *hw = &adapter->hw; - - fjes_hw_set_irqmask(hw, REG_ICTL_MASK_RX_DATA, true); - - adapter->unset_rx_last = true; - napi_schedule(&adapter->napi); -} - static int fjes_poll(struct napi_struct *napi, int budget) { struct fjes_adapter *adapter = @@ -1196,182 +1053,130 @@ static int fjes_poll(struct napi_struct *napi, int budget) return work_done; } -/* fjes_probe - Device Initialization Routine */ -static int fjes_probe(struct platform_device *plat_dev) +static int fjes_sw_init(struct fjes_adapter *adapter) { - struct fjes_adapter *adapter; - struct net_device *netdev; - struct resource *res; - struct fjes_hw *hw; - u8 addr[ETH_ALEN]; - int err; - - err = -ENOMEM; - netdev = alloc_netdev_mq(sizeof(struct fjes_adapter), "es%d", - NET_NAME_UNKNOWN, fjes_netdev_setup, - FJES_MAX_QUEUES); - - if (!netdev) - goto err_out; - - SET_NETDEV_DEV(netdev, &plat_dev->dev); - - dev_set_drvdata(&plat_dev->dev, netdev); - adapter = netdev_priv(netdev); - adapter->netdev = netdev; - adapter->plat_dev = plat_dev; - hw = &adapter->hw; - hw->back = adapter; - - /* setup the private structure */ - err = fjes_sw_init(adapter); - if (err) - goto err_free_netdev; + struct net_device *netdev = adapter->netdev; - INIT_WORK(&adapter->force_close_task, fjes_force_close_task); - adapter->force_reset = false; - adapter->open_guard = false; + netif_napi_add(netdev, &adapter->napi, fjes_poll); - adapter->txrx_wq = alloc_workqueue(DRV_NAME "/txrx", WQ_MEM_RECLAIM, 0); - if (unlikely(!adapter->txrx_wq)) { - err = -ENOMEM; - goto err_free_netdev; - } + return 0; +} - adapter->control_wq = alloc_workqueue(DRV_NAME "/control", - WQ_MEM_RECLAIM, 0); - if (unlikely(!adapter->control_wq)) { - err = -ENOMEM; - goto err_free_txrx_wq; - } +static void fjes_force_close_task(struct work_struct *work) +{ + struct fjes_adapter *adapter = container_of(work, + struct fjes_adapter, force_close_task); + struct net_device *netdev = adapter->netdev; - INIT_WORK(&adapter->tx_stall_task, fjes_tx_stall_task); - INIT_WORK(&adapter->raise_intr_rxdata_task, - fjes_raise_intr_rxdata_task); - INIT_WORK(&adapter->unshare_watch_task, fjes_watch_unshare_task); - adapter->unshare_watch_bitmask = 0; + rtnl_lock(); + dev_close(netdev); + rtnl_unlock(); +} - INIT_DELAYED_WORK(&adapter->interrupt_watch_task, fjes_irq_watch_task); - adapter->interrupt_watch_enable = false; +static void fjes_tx_stall_task(struct work_struct *work) +{ + struct fjes_adapter *adapter = container_of(work, + struct fjes_adapter, tx_stall_task); + struct net_device *netdev = adapter->netdev; + struct fjes_hw *hw = &adapter->hw; + int all_queue_available, sendable; + enum ep_partner_status pstatus; + int max_epid, my_epid, epid; + union ep_buffer_info *info; + int i; - res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0); - if (!res) { - err = -EINVAL; - goto err_free_control_wq; - } - hw->hw_res.start = res->start; - hw->hw_res.size = resource_size(res); - hw->hw_res.irq = platform_get_irq(plat_dev, 0); - if (hw->hw_res.irq < 0) { - err = hw->hw_res.irq; - goto err_free_control_wq; + if (((long)jiffies - + dev_trans_start(netdev)) > FJES_TX_TX_STALL_TIMEOUT) { + netif_wake_queue(netdev); + return; } - err = fjes_hw_init(&adapter->hw); - if (err) - goto err_free_control_wq; - - /* setup MAC address (02:00:00:00:00:[epid])*/ - addr[0] = 2; - addr[1] = 0; - addr[2] = 0; - addr[3] = 0; - addr[4] = 0; - addr[5] = hw->my_epid; /* EPID */ - eth_hw_addr_set(netdev, addr); - - err = register_netdev(netdev); - if (err) - goto err_hw_exit; - - netif_carrier_off(netdev); - - fjes_dbg_adapter_init(adapter); - - return 0; - -err_hw_exit: - fjes_hw_exit(&adapter->hw); -err_free_control_wq: - destroy_workqueue(adapter->control_wq); -err_free_txrx_wq: - destroy_workqueue(adapter->txrx_wq); -err_free_netdev: - free_netdev(netdev); -err_out: - return err; -} + my_epid = hw->my_epid; + max_epid = hw->max_epid; -/* fjes_remove - Device Removal Routine */ -static int fjes_remove(struct platform_device *plat_dev) -{ - struct net_device *netdev = dev_get_drvdata(&plat_dev->dev); - struct fjes_adapter *adapter = netdev_priv(netdev); - struct fjes_hw *hw = &adapter->hw; + for (i = 0; i < 5; i++) { + all_queue_available = 1; - fjes_dbg_adapter_exit(adapter); + for (epid = 0; epid < max_epid; epid++) { + if (my_epid == epid) + continue; - cancel_delayed_work_sync(&adapter->interrupt_watch_task); - cancel_work_sync(&adapter->unshare_watch_task); - cancel_work_sync(&adapter->raise_intr_rxdata_task); - cancel_work_sync(&adapter->tx_stall_task); - if (adapter->control_wq) - destroy_workqueue(adapter->control_wq); - if (adapter->txrx_wq) - destroy_workqueue(adapter->txrx_wq); + pstatus = fjes_hw_get_partner_ep_status(hw, epid); + sendable = (pstatus == EP_PARTNER_SHARED); + if (!sendable) + continue; - unregister_netdev(netdev); + info = adapter->hw.ep_shm_info[epid].tx.info; - fjes_hw_exit(hw); + if (!(info->v1i.rx_status & FJES_RX_MTU_CHANGING_DONE)) + return; - netif_napi_del(&adapter->napi); + if (EP_RING_FULL(info->v1i.head, info->v1i.tail, + info->v1i.count_max)) { + all_queue_available = 0; + break; + } + } - free_netdev(netdev); + if (all_queue_available) { + netif_wake_queue(netdev); + return; + } + } - return 0; + usleep_range(50, 100); + + queue_work(adapter->txrx_wq, &adapter->tx_stall_task); } -static int fjes_sw_init(struct fjes_adapter *adapter) +static void fjes_raise_intr_rxdata_task(struct work_struct *work) { - struct net_device *netdev = adapter->netdev; - - netif_napi_add(netdev, &adapter->napi, fjes_poll, 64); + struct fjes_adapter *adapter = container_of(work, + struct fjes_adapter, raise_intr_rxdata_task); + struct fjes_hw *hw = &adapter->hw; + enum ep_partner_status pstatus; + int max_epid, my_epid, epid; - return 0; -} + my_epid = hw->my_epid; + max_epid = hw->max_epid; -/* fjes_netdev_setup - netdevice initialization routine */ -static void fjes_netdev_setup(struct net_device *netdev) -{ - ether_setup(netdev); + for (epid = 0; epid < max_epid; epid++) + hw->ep_shm_info[epid].tx_status_work = 0; - netdev->watchdog_timeo = FJES_TX_RETRY_INTERVAL; - netdev->netdev_ops = &fjes_netdev_ops; - fjes_set_ethtool_ops(netdev); - netdev->mtu = fjes_support_mtu[3]; - netdev->min_mtu = fjes_support_mtu[0]; - netdev->max_mtu = fjes_support_mtu[3]; - netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; -} + for (epid = 0; epid < max_epid; epid++) { + if (epid == my_epid) + continue; -static void fjes_irq_watch_task(struct work_struct *work) -{ - struct fjes_adapter *adapter = container_of(to_delayed_work(work), - struct fjes_adapter, interrupt_watch_task); + pstatus = fjes_hw_get_partner_ep_status(hw, epid); + if (pstatus == EP_PARTNER_SHARED) { + hw->ep_shm_info[epid].tx_status_work = + hw->ep_shm_info[epid].tx.info->v1i.tx_status; - local_irq_disable(); - fjes_intr(adapter->hw.hw_res.irq, adapter); - local_irq_enable(); + if (hw->ep_shm_info[epid].tx_status_work == + FJES_TX_DELAY_SEND_PENDING) { + hw->ep_shm_info[epid].tx.info->v1i.tx_status = + FJES_TX_DELAY_SEND_NONE; + } + } + } - if (fjes_rxframe_search_exist(adapter, 0) >= 0) - napi_schedule(&adapter->napi); + for (epid = 0; epid < max_epid; epid++) { + if (epid == my_epid) + continue; - if (adapter->interrupt_watch_enable) { - if (!delayed_work_pending(&adapter->interrupt_watch_task)) - queue_delayed_work(adapter->control_wq, - &adapter->interrupt_watch_task, - FJES_IRQ_WATCH_DELAY); + pstatus = fjes_hw_get_partner_ep_status(hw, epid); + if ((hw->ep_shm_info[epid].tx_status_work == + FJES_TX_DELAY_SEND_PENDING) && + (pstatus == EP_PARTNER_SHARED) && + !(hw->ep_shm_info[epid].rx.info->v1i.rx_status & + FJES_RX_POLL_WORK)) { + fjes_hw_raise_interrupt(hw, epid, + REG_ICTL_MASK_RX_DATA); + hw->ep_shm_info[epid].ep_stats.send_intr_rx += 1; + } } + + usleep_range(500, 1000); } static void fjes_watch_unshare_task(struct work_struct *work) @@ -1508,6 +1313,169 @@ static void fjes_watch_unshare_task(struct work_struct *work) } } +static void fjes_irq_watch_task(struct work_struct *work) +{ + struct fjes_adapter *adapter = container_of(to_delayed_work(work), + struct fjes_adapter, interrupt_watch_task); + + local_irq_disable(); + fjes_intr(adapter->hw.hw_res.irq, adapter); + local_irq_enable(); + + if (fjes_rxframe_search_exist(adapter, 0) >= 0) + napi_schedule(&adapter->napi); + + if (adapter->interrupt_watch_enable) { + if (!delayed_work_pending(&adapter->interrupt_watch_task)) + queue_delayed_work(adapter->control_wq, + &adapter->interrupt_watch_task, + FJES_IRQ_WATCH_DELAY); + } +} + +/* fjes_probe - Device Initialization Routine */ +static int fjes_probe(struct platform_device *plat_dev) +{ + struct fjes_adapter *adapter; + struct net_device *netdev; + struct resource *res; + struct fjes_hw *hw; + u8 addr[ETH_ALEN]; + int err; + + err = -ENOMEM; + netdev = alloc_netdev_mq(sizeof(struct fjes_adapter), "es%d", + NET_NAME_UNKNOWN, fjes_netdev_setup, + FJES_MAX_QUEUES); + + if (!netdev) + goto err_out; + + SET_NETDEV_DEV(netdev, &plat_dev->dev); + + dev_set_drvdata(&plat_dev->dev, netdev); + adapter = netdev_priv(netdev); + adapter->netdev = netdev; + adapter->plat_dev = plat_dev; + hw = &adapter->hw; + hw->back = adapter; + + /* setup the private structure */ + err = fjes_sw_init(adapter); + if (err) + goto err_free_netdev; + + INIT_WORK(&adapter->force_close_task, fjes_force_close_task); + adapter->force_reset = false; + adapter->open_guard = false; + + adapter->txrx_wq = alloc_workqueue(DRV_NAME "/txrx", WQ_MEM_RECLAIM, 0); + if (unlikely(!adapter->txrx_wq)) { + err = -ENOMEM; + goto err_free_netdev; + } + + adapter->control_wq = alloc_workqueue(DRV_NAME "/control", + WQ_MEM_RECLAIM, 0); + if (unlikely(!adapter->control_wq)) { + err = -ENOMEM; + goto err_free_txrx_wq; + } + + INIT_WORK(&adapter->tx_stall_task, fjes_tx_stall_task); + INIT_WORK(&adapter->raise_intr_rxdata_task, + fjes_raise_intr_rxdata_task); + INIT_WORK(&adapter->unshare_watch_task, fjes_watch_unshare_task); + adapter->unshare_watch_bitmask = 0; + + INIT_DELAYED_WORK(&adapter->interrupt_watch_task, fjes_irq_watch_task); + adapter->interrupt_watch_enable = false; + + res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0); + if (!res) { + err = -EINVAL; + goto err_free_control_wq; + } + hw->hw_res.start = res->start; + hw->hw_res.size = resource_size(res); + hw->hw_res.irq = platform_get_irq(plat_dev, 0); + if (hw->hw_res.irq < 0) { + err = hw->hw_res.irq; + goto err_free_control_wq; + } + + err = fjes_hw_init(&adapter->hw); + if (err) + goto err_free_control_wq; + + /* setup MAC address (02:00:00:00:00:[epid])*/ + addr[0] = 2; + addr[1] = 0; + addr[2] = 0; + addr[3] = 0; + addr[4] = 0; + addr[5] = hw->my_epid; /* EPID */ + eth_hw_addr_set(netdev, addr); + + err = register_netdev(netdev); + if (err) + goto err_hw_exit; + + netif_carrier_off(netdev); + + fjes_dbg_adapter_init(adapter); + + return 0; + +err_hw_exit: + fjes_hw_exit(&adapter->hw); +err_free_control_wq: + destroy_workqueue(adapter->control_wq); +err_free_txrx_wq: + destroy_workqueue(adapter->txrx_wq); +err_free_netdev: + free_netdev(netdev); +err_out: + return err; +} + +/* fjes_remove - Device Removal Routine */ +static int fjes_remove(struct platform_device *plat_dev) +{ + struct net_device *netdev = dev_get_drvdata(&plat_dev->dev); + struct fjes_adapter *adapter = netdev_priv(netdev); + struct fjes_hw *hw = &adapter->hw; + + fjes_dbg_adapter_exit(adapter); + + cancel_delayed_work_sync(&adapter->interrupt_watch_task); + cancel_work_sync(&adapter->unshare_watch_task); + cancel_work_sync(&adapter->raise_intr_rxdata_task); + cancel_work_sync(&adapter->tx_stall_task); + if (adapter->control_wq) + destroy_workqueue(adapter->control_wq); + if (adapter->txrx_wq) + destroy_workqueue(adapter->txrx_wq); + + unregister_netdev(netdev); + + fjes_hw_exit(hw); + + netif_napi_del(&adapter->napi); + + free_netdev(netdev); + + return 0; +} + +static struct platform_driver fjes_driver = { + .driver = { + .name = DRV_NAME, + }, + .probe = fjes_probe, + .remove = fjes_remove, +}; + static acpi_status acpi_find_extended_socket_device(acpi_handle obj_handle, u32 level, void *context, void **return_value) diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 7962c37b3f14bc244db5725a0578132601d892aa..f393e454f45ca1cda064f112cb2ba1742569d4e7 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -503,12 +503,9 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk, off_gnv = skb_gro_offset(skb); hlen = off_gnv + sizeof(*gh); - gh = skb_gro_header_fast(skb, off_gnv); - if (skb_gro_header_hard(skb, hlen)) { - gh = skb_gro_header_slow(skb, hlen, off_gnv); - if (unlikely(!gh)) - goto out; - } + gh = skb_gro_header(skb, hlen, off_gnv); + if (unlikely(!gh)) + goto out; if (gh->ver != GENEVE_VER || gh->oam) goto out; @@ -1200,8 +1197,8 @@ static const struct net_device_ops geneve_netdev_ops = { static void geneve_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version)); - strlcpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver)); + strscpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver)); } static const struct ethtool_ops geneve_ethtool_ops = { diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index a208e2b1a9af224e3a9d51facdcd75f02a8e8e55..15c7dc82107f45fc068be92b43d29d3ff9d9ee72 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -1859,6 +1859,7 @@ static struct genl_family gtp_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = gtp_genl_ops, .n_small_ops = ARRAY_SIZE(gtp_genl_ops), + .resv_start_op = GTP_CMD_ECHOREQ + 1, .mcgrps = gtp_genl_mcgrps, .n_mcgrps = ARRAY_SIZE(gtp_genl_mcgrps), }; diff --git a/drivers/net/hamradio/baycom_epp.c b/drivers/net/hamradio/baycom_epp.c index 3e69079ed694b33c9b70fd94747281f647722ef3..791b4a53d69fdf20679780c02bb580bbd78aa3b9 100644 --- a/drivers/net/hamradio/baycom_epp.c +++ b/drivers/net/hamradio/baycom_epp.c @@ -438,7 +438,7 @@ static int transmit(struct baycom_state *bc, int cnt, unsigned char stat) if ((--bc->hdlctx.slotcnt) > 0) return 0; bc->hdlctx.slotcnt = bc->ch_params.slottime; - if ((prandom_u32() % 256) > bc->ch_params.ppersist) + if (get_random_u8() > bc->ch_params.ppersist) return 0; } } diff --git a/drivers/net/hamradio/hdlcdrv.c b/drivers/net/hamradio/hdlcdrv.c index 8297411e87ea06914a5ee2c466d7a0dd1346f948..2263029d1a20e614b0d08da400285f36af913c3d 100644 --- a/drivers/net/hamradio/hdlcdrv.c +++ b/drivers/net/hamradio/hdlcdrv.c @@ -377,7 +377,7 @@ void hdlcdrv_arbitrate(struct net_device *dev, struct hdlcdrv_state *s) if ((--s->hdlctx.slotcnt) > 0) return; s->hdlctx.slotcnt = s->ch_params.slottime; - if ((prandom_u32() % 256) > s->ch_params.ppersist) + if (get_random_u8() > s->ch_params.ppersist) return; start_tx(dev, s); } @@ -600,7 +600,7 @@ static int hdlcdrv_siocdevprivate(struct net_device *dev, struct ifreq *ifr, case HDLCDRVCTL_DRIVERNAME: if (s->ops && s->ops->drvname) { - strlcpy(bi.data.drivername, s->ops->drvname, + strscpy(bi.data.drivername, s->ops->drvname, sizeof(bi.data.drivername)); break; } diff --git a/drivers/net/hamradio/yam.c b/drivers/net/hamradio/yam.c index 980f2be32f05a1974c13aba7d44f257901ed8cd6..2ed2f836f09af5a0bf8dbecce1d77e2f48042dc6 100644 --- a/drivers/net/hamradio/yam.c +++ b/drivers/net/hamradio/yam.c @@ -626,7 +626,7 @@ static void yam_arbitrate(struct net_device *dev) yp->slotcnt = yp->slot / 10; /* is random > persist ? */ - if ((prandom_u32() % 256) > yp->pers) + if (get_random_u8() > yp->pers) return; yam_start_tx(dev, yp); diff --git a/drivers/net/hippi/rrunner.c b/drivers/net/hippi/rrunner.c index 74e845fa2e07ed8e2430c57c2d24b830be361be8..aa8f828a0ae7f271a04bace4bc921269cc1cfbd7 100644 --- a/drivers/net/hippi/rrunner.c +++ b/drivers/net/hippi/rrunner.c @@ -213,6 +213,7 @@ static int rr_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) pci_iounmap(pdev, rrpriv->regs); if (pdev) pci_release_regions(pdev); + pci_disable_device(pdev); out2: free_netdev(dev); out3: diff --git a/drivers/net/hyperv/hyperv_net.h b/drivers/net/hyperv/hyperv_net.h index 25b38a374e3c3db8509ce34c1addf739d47f77ef..dd5919ec408bf6ec9f37d44727c9e4d19416f5d0 100644 --- a/drivers/net/hyperv/hyperv_net.h +++ b/drivers/net/hyperv/hyperv_net.h @@ -1051,7 +1051,8 @@ struct net_device_context { u32 vf_alloc; /* Serial number of the VF to team with */ u32 vf_serial; - + /* completion variable to confirm vf association */ + struct completion vf_add; /* Is the current data path through the VF NIC? */ bool data_path_is_vf; diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 6e42cb03e226abeae15180521d40db3e80c02f7b..9352dad58996d831e596939c94e86daf0524d9a2 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -1580,6 +1580,10 @@ static void netvsc_send_vf(struct net_device *ndev, net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; + + if (net_device_ctx->vf_alloc) + complete(&net_device_ctx->vf_add); + netdev_info(ndev, "VF slot %u %s\n", net_device_ctx->vf_serial, net_device_ctx->vf_alloc ? "added" : "removed"); @@ -1779,8 +1783,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, } /* Enable NAPI handler before init callbacks */ - netif_napi_add(ndev, &net_device->chan_table[0].napi, - netvsc_poll, NAPI_POLL_WEIGHT); + netif_napi_add(ndev, &net_device->chan_table[0].napi, netvsc_poll); /* Open the channel */ device->channel->next_request_id_callback = vmbus_next_request_id; diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 15ebd54266049794f913e11a58bc40785c56f161..89eb4f179a3ceff333617a224e71e968c7ad272a 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -935,8 +935,8 @@ int netvsc_recv_callback(struct net_device *net, static void netvsc_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info) { - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); } static void netvsc_get_channels(struct net_device *net, @@ -2313,6 +2313,18 @@ static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev) } + /* Fallback path to check synthetic vf with + * help of mac addr + */ + list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) { + ndev = hv_get_drvdata(ndev_ctx->device_ctx); + if (ether_addr_equal(vf_netdev->perm_addr, ndev->perm_addr)) { + netdev_notice(vf_netdev, + "falling back to mac addr based matching\n"); + return ndev; + } + } + netdev_notice(vf_netdev, "no netdev found for vf serial:%u\n", serial); return NULL; @@ -2409,6 +2421,11 @@ static int netvsc_vf_changed(struct net_device *vf_netdev, unsigned long event) if (net_device_ctx->data_path_is_vf == vf_is_up) return NOTIFY_OK; + if (vf_is_up && !net_device_ctx->vf_alloc) { + netdev_info(ndev, "Waiting for the VF association from host\n"); + wait_for_completion(&net_device_ctx->vf_add); + } + ret = netvsc_switch_datapath(ndev, vf_is_up); if (ret) { @@ -2440,6 +2457,7 @@ static int netvsc_unregister_vf(struct net_device *vf_netdev) netvsc_vf_setxdp(vf_netdev, NULL); + reinit_completion(&net_device_ctx->vf_add); netdev_rx_handler_unregister(vf_netdev); netdev_upper_dev_unlink(vf_netdev, ndev); RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL); @@ -2479,6 +2497,7 @@ static int netvsc_probe(struct hv_device *dev, INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); + init_completion(&net_device_ctx->vf_add); spin_lock_init(&net_device_ctx->lock); INIT_LIST_HEAD(&net_device_ctx->reconfig_events); INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup); diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c index 6da36cb8af8055eba338490b6bc7493181e8644c..11f767a20444328875cf5991eb4657553fb37dc9 100644 --- a/drivers/net/hyperv/rndis_filter.c +++ b/drivers/net/hyperv/rndis_filter.c @@ -1575,7 +1575,7 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev, for (i = 1; i < net_device->num_chn; i++) netif_napi_add(net, &net_device->chan_table[i].napi, - netvsc_poll, NAPI_POLL_WEIGHT); + netvsc_poll); return net_device; diff --git a/drivers/net/ieee802154/adf7242.c b/drivers/net/ieee802154/adf7242.c index 6afdf1622944e4efc9d888dbb93b86c901dddf6f..5cf218c674a5ac0ae856785c5292f4690d8580c2 100644 --- a/drivers/net/ieee802154/adf7242.c +++ b/drivers/net/ieee802154/adf7242.c @@ -1310,10 +1310,11 @@ static void adf7242_remove(struct spi_device *spi) debugfs_remove_recursive(lp->debugfs_root); + ieee802154_unregister_hw(lp->hw); + cancel_delayed_work_sync(&lp->work); destroy_workqueue(lp->wqueue); - ieee802154_unregister_hw(lp->hw); mutex_destroy(&lp->bmux); ieee802154_free_hw(lp->hw); } diff --git a/drivers/net/ieee802154/ca8210.c b/drivers/net/ieee802154/ca8210.c index 42c0b451088dce86a1f6c3a12c58fe6e339e13eb..450b16ad40a41556ceb0153ae4fe667de62c3d05 100644 --- a/drivers/net/ieee802154/ca8210.c +++ b/drivers/net/ieee802154/ca8210.c @@ -2293,7 +2293,7 @@ static int ca8210_set_csma_params( * @retries: Number of retries * * Sets the number of times to retry a transmission if no acknowledgment was - * was received from the other end when one was requested. + * received from the other end when one was requested. * * Return: 0 or linux error code */ diff --git a/drivers/net/ieee802154/cc2520.c b/drivers/net/ieee802154/cc2520.c index 1e1f40f628a028a234c017f9f75c81afaf37aa52..c69b87d3837dad4e4f34c9936274013671fdfb42 100644 --- a/drivers/net/ieee802154/cc2520.c +++ b/drivers/net/ieee802154/cc2520.c @@ -504,6 +504,7 @@ cc2520_tx(struct ieee802154_hw *hw, struct sk_buff *skb) goto err_tx; if (status & CC2520_STATUS_TX_UNDERFLOW) { + rc = -EINVAL; dev_err(&priv->spi->dev, "cc2520 tx underflow exception\n"); goto err_tx; } diff --git a/drivers/net/ieee802154/mac802154_hwsim.c b/drivers/net/ieee802154/mac802154_hwsim.c index 38c217bd7c8226c174a6f35a20047f7b8f027b69..2f0544dd7c2ad918480b4f00c3c24962412687b0 100644 --- a/drivers/net/ieee802154/mac802154_hwsim.c +++ b/drivers/net/ieee802154/mac802154_hwsim.c @@ -630,6 +630,7 @@ static struct genl_family hwsim_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = hwsim_nl_ops, .n_small_ops = ARRAY_SIZE(hwsim_nl_ops), + .resv_start_op = MAC802154_HWSIM_CMD_NEW_EDGE + 1, .mcgrps = hwsim_mcgrps, .n_mcgrps = ARRAY_SIZE(hwsim_mcgrps), }; diff --git a/drivers/net/ipa/Makefile b/drivers/net/ipa/Makefile index 8b2220eb6b92d439b813c721219846e15f616232..48255fc4b25c3e0660130be2b48106c00612f5ff 100644 --- a/drivers/net/ipa/Makefile +++ b/drivers/net/ipa/Makefile @@ -13,4 +13,6 @@ ipa-y := ipa_main.o ipa_power.o ipa_reg.o ipa_mem.o \ ipa_resource.o ipa_qmi.o ipa_qmi_msg.o \ ipa_sysfs.o +ipa-y += $(IPA_VERSIONS:%=reg/ipa_reg-v%.o) + ipa-y += $(IPA_VERSIONS:%=data/ipa_data-v%.o) diff --git a/drivers/net/ipa/data/ipa_data-v3.1.c b/drivers/net/ipa/data/ipa_data-v3.1.c index 1c1895aea81180c94b8d52b7ced5149f42ceef93..e0d71f609272991cbfb6429834bdf4ea85f2db82 100644 --- a/drivers/net/ipa/data/ipa_data-v3.1.c +++ b/drivers/net/ipa/data/ipa_data-v3.1.c @@ -526,7 +526,7 @@ static const struct ipa_power_data ipa_power_data = { /* Configuration data for an SoC having IPA v3.1 */ const struct ipa_data ipa_data_v3_1 = { .version = IPA_VERSION_3_1, - .backward_compat = BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK, + .backward_compat = BIT(BCR_CMDQ_L_LACK_ONE_ENTRY), .qsb_count = ARRAY_SIZE(ipa_qsb_data), .qsb_data = ipa_qsb_data, .endpoint_count = ARRAY_SIZE(ipa_gsi_endpoint_data), diff --git a/drivers/net/ipa/data/ipa_data-v3.5.1.c b/drivers/net/ipa/data/ipa_data-v3.5.1.c index 58b708d2fc75da0ad7490758a2a6bfc21c21bff2..383ef18900654505bf83b361006fd97ef36dfc29 100644 --- a/drivers/net/ipa/data/ipa_data-v3.5.1.c +++ b/drivers/net/ipa/data/ipa_data-v3.5.1.c @@ -407,11 +407,11 @@ static const struct ipa_power_data ipa_power_data = { /* Configuration data for an SoC having IPA v3.5.1 */ const struct ipa_data ipa_data_v3_5_1 = { .version = IPA_VERSION_3_5_1, - .backward_compat = BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK | - BCR_TX_NOT_USING_BRESP_FMASK | - BCR_SUSPEND_L2_IRQ_FMASK | - BCR_HOLB_DROP_L2_IRQ_FMASK | - BCR_DUAL_TX_FMASK, + .backward_compat = BIT(BCR_CMDQ_L_LACK_ONE_ENTRY) | + BIT(BCR_TX_NOT_USING_BRESP) | + BIT(BCR_SUSPEND_L2_IRQ) | + BIT(BCR_HOLB_DROP_L2_IRQ) | + BIT(BCR_DUAL_TX), .qsb_count = ARRAY_SIZE(ipa_qsb_data), .qsb_data = ipa_qsb_data, .endpoint_count = ARRAY_SIZE(ipa_gsi_endpoint_data), diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 9e307eebd33f9bd88f79c069fbffd1a73d30fc69..bea2da1c4c51d302f0fb86c2bd247621b2ac1d6f 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -56,9 +56,9 @@ * element can also contain an immediate command, requesting the IPA perform * actions other than data transfer. * - * Each TRE refers to a block of data--also located DRAM. After writing one - * or more TREs to a channel, the writer (either the IPA or an EE) writes a - * doorbell register to inform the receiving side how many elements have + * Each TRE refers to a block of data--also located in DRAM. After writing + * one or more TREs to a channel, the writer (either the IPA or an EE) writes + * a doorbell register to inform the receiving side how many elements have * been written. * * Each channel has a GSI "event ring" associated with it. An event ring @@ -710,43 +710,32 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id) static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel) { struct gsi_trans_info *trans_info = &channel->trans_info; - const struct list_head *list; + u32 pending_id = trans_info->pending_id; struct gsi_trans *trans; - - spin_lock_bh(&trans_info->spinlock); - - /* There is a small chance a TX transaction got allocated just - * before we disabled transmits, so check for that. - */ - if (channel->toward_ipa) { - list = &trans_info->alloc; - if (!list_empty(list)) - goto done; - list = &trans_info->committed; - if (!list_empty(list)) - goto done; - list = &trans_info->pending; - if (!list_empty(list)) - goto done; + u16 trans_id; + + if (channel->toward_ipa && pending_id != trans_info->free_id) { + /* There is a small chance a TX transaction got allocated + * just before we disabled transmits, so check for that. + * The last allocated, committed, or pending transaction + * precedes the first free transaction. + */ + trans_id = trans_info->free_id - 1; + } else if (trans_info->polled_id != pending_id) { + /* Otherwise (TX or RX) we want to wait for anything that + * has completed, or has been polled but not released yet. + * + * The last completed or polled transaction precedes the + * first pending transaction. + */ + trans_id = pending_id - 1; + } else { + return NULL; } - /* Otherwise (TX or RX) we want to wait for anything that - * has completed, or has been polled but not released yet. - */ - list = &trans_info->complete; - if (!list_empty(list)) - goto done; - list = &trans_info->polled; - if (list_empty(list)) - list = NULL; -done: - trans = list ? list_last_entry(list, struct gsi_trans, links) : NULL; - /* Caller will wait for this, so take a reference */ - if (trans) - refcount_inc(&trans->refcount); - - spin_unlock_bh(&trans_info->spinlock); + trans = &trans_info->trans[trans_id % channel->tre_count]; + refcount_inc(&trans->refcount); return trans; } @@ -1358,8 +1347,8 @@ gsi_event_trans(struct gsi *gsi, struct gsi_event *event) * we update transactions to record their actual received lengths. * * When an event for a TX channel arrives we use information in the - * transaction to report the number of requests and bytes have been - * transferred. + * transaction to report the number of requests and bytes that have + * been transferred. * * This function is called whenever we learn that the GSI hardware has filled * new events since the last time we checked. The ring's index field tells @@ -1485,8 +1474,8 @@ void gsi_channel_doorbell(struct gsi_channel *channel) iowrite32(val, gsi->virt + GSI_CH_C_DOORBELL_0_OFFSET(channel_id)); } -/* Consult hardware, move any newly completed transactions to completed list */ -static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel) +/* Consult hardware, move newly completed transactions to completed state */ +void gsi_channel_update(struct gsi_channel *channel) { u32 evt_ring_id = channel->evt_ring_id; struct gsi *gsi = channel->gsi; @@ -1505,12 +1494,12 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel) offset = GSI_EV_CH_E_CNTXT_4_OFFSET(evt_ring_id); index = gsi_ring_index(ring, ioread32(gsi->virt + offset)); if (index == ring->index % ring->count) - return NULL; + return; /* Get the transaction for the latest completed event. */ trans = gsi_event_trans(gsi, gsi_ring_virt(ring, index - 1)); if (!trans) - return NULL; + return; /* For RX channels, update each completed transaction with the number * of bytes that were actually received. For TX channels, report @@ -1518,8 +1507,6 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel) * up the network stack. */ gsi_evt_ring_update(gsi, evt_ring_id, index); - - return gsi_channel_trans_complete(channel); } /** @@ -1528,21 +1515,18 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel) * * Return: Transaction pointer, or null if none are available * - * This function returns the first entry on a channel's completed transaction - * list. If that list is empty, the hardware is consulted to determine - * whether any new transactions have completed. If so, they're moved to the - * completed list and the new first entry is returned. If there are no more - * completed transactions, a null pointer is returned. + * This function returns the first of a channel's completed transactions. + * If no transactions are in completed state, the hardware is consulted to + * determine whether any new transactions have completed. If so, they're + * moved to completed state and the first such transaction is returned. + * If there are no more completed transactions, a null pointer is returned. */ static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel) { struct gsi_trans *trans; - /* Get the first transaction from the completed list */ + /* Get the first completed transaction */ trans = gsi_channel_trans_complete(channel); - if (!trans) /* List is empty; see if there's more to do */ - trans = gsi_channel_update(channel); - if (trans) gsi_trans_move_polled(trans); @@ -1623,7 +1607,7 @@ static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id) gsi_channel_poll); else netif_napi_add(&gsi->dummy_dev, &channel->napi, - gsi_channel_poll, NAPI_POLL_WEIGHT); + gsi_channel_poll); return 0; diff --git a/drivers/net/ipa/gsi.h b/drivers/net/ipa/gsi.h index 23de5f67374cf2daed9ad135d7c41a931d3fc7ec..49dcadba4e0b9a772d3d5a99af1a35dd56d969ce 100644 --- a/drivers/net/ipa/gsi.h +++ b/drivers/net/ipa/gsi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _GSI_H_ #define _GSI_H_ @@ -31,14 +31,6 @@ struct gsi_trans; struct gsi_channel_data; struct ipa_gsi_endpoint_data; -/* Execution environment IDs */ -enum gsi_ee_id { - GSI_EE_AP = 0x0, - GSI_EE_MODEM = 0x1, - GSI_EE_UC = 0x2, - GSI_EE_TZ = 0x3, -}; - struct gsi_ring { void *virt; /* ring array base address */ dma_addr_t addr; /* primarily low 32 bits used */ @@ -82,18 +74,18 @@ struct gsi_trans_pool { struct gsi_trans_info { atomic_t tre_avail; /* TREs available for allocation */ - struct gsi_trans_pool pool; /* transaction pool */ + + u16 free_id; /* first free trans in array */ + u16 allocated_id; /* first allocated transaction */ + u16 committed_id; /* first committed transaction */ + u16 pending_id; /* first pending transaction */ + u16 completed_id; /* first completed transaction */ + u16 polled_id; /* first polled transaction */ + struct gsi_trans *trans; /* transaction array */ struct gsi_trans **map; /* TRE -> transaction map */ struct gsi_trans_pool sg_pool; /* scatterlist pool */ struct gsi_trans_pool cmd_pool; /* command payload DMA pool */ - - spinlock_t spinlock; /* protects updates to the lists */ - struct list_head alloc; /* allocated, not committed */ - struct list_head committed; /* committed, awaiting doorbell */ - struct list_head pending; /* pending, awaiting completion */ - struct list_head complete; /* completed, awaiting poll */ - struct list_head polled; /* returned by gsi_channel_poll_one() */ }; /* Hardware values signifying the state of a channel */ diff --git a/drivers/net/ipa/gsi_private.h b/drivers/net/ipa/gsi_private.h index 0b2516fa21b5deb057a38aee8f16babab66532cd..c65f7c5cdc8d0eb3d0fd8d45eeea35c1aaa6a5c3 100644 --- a/drivers/net/ipa/gsi_private.h +++ b/drivers/net/ipa/gsi_private.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _GSI_PRIVATE_H_ #define _GSI_PRIVATE_H_ @@ -18,13 +18,13 @@ struct gsi_channel; /** * gsi_trans_move_complete() - Mark a GSI transaction completed - * @trans: Transaction to commit + * @trans: Transaction whose state is to be updated */ void gsi_trans_move_complete(struct gsi_trans *trans); /** * gsi_trans_move_polled() - Mark a transaction polled - * @trans: Transaction to update + * @trans: Transaction whose state is to be updated */ void gsi_trans_move_polled(struct gsi_trans *trans); @@ -94,6 +94,14 @@ void gsi_channel_trans_exit(struct gsi_channel *channel); */ void gsi_channel_doorbell(struct gsi_channel *channel); +/* gsi_channel_update() - Update knowledge of channel hardware state + * @channel: Channel to be updated + * + * Consult hardware, change the state of any newly-completed transactions + * on a channel. + */ +void gsi_channel_update(struct gsi_channel *channel); + /** * gsi_ring_virt() - Return virtual address for a ring entry * @ring: Ring whose address is to be translated diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h index 5bd8b31656d30efa419fa640a9be061ed08e5c55..3763359f208f7cba4d33593619e7c69d123262f4 100644 --- a/drivers/net/ipa/gsi_reg.h +++ b/drivers/net/ipa/gsi_reg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _GSI_REG_H_ #define _GSI_REG_H_ @@ -55,14 +55,10 @@ /* The inter-EE IRQ registers are relative to gsi->virt_raw (IPA v3.5+) */ #define GSI_INTER_EE_SRC_CH_IRQ_MSK_OFFSET \ - GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(ee) \ - (0x0000c020 + 0x1000 * (ee)) + (0x0000c020 + 0x1000 * GSI_EE_AP) #define GSI_INTER_EE_SRC_EV_CH_IRQ_MSK_OFFSET \ - GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(ee) \ - (0x0000c024 + 0x1000 * (ee)) + (0x0000c024 + 0x1000 * GSI_EE_AP) /* All other register offsets are relative to gsi->virt */ @@ -81,9 +77,7 @@ enum gsi_channel_type { }; #define GSI_CH_C_CNTXT_0_OFFSET(ch) \ - GSI_EE_N_CH_C_CNTXT_0_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_CNTXT_0_OFFSET(ch, ee) \ - (0x0001c000 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c000 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define CHTYPE_PROTOCOL_FMASK GENMASK(2, 0) #define CHTYPE_DIR_FMASK GENMASK(3, 3) #define EE_FMASK GENMASK(7, 4) @@ -112,9 +106,7 @@ chtype_protocol_encoded(enum ipa_version version, enum gsi_channel_type type) } #define GSI_CH_C_CNTXT_1_OFFSET(ch) \ - GSI_EE_N_CH_C_CNTXT_1_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_CNTXT_1_OFFSET(ch, ee) \ - (0x0001c004 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c004 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) /* Encoded value for CH_C_CNTXT_1 register R_LENGTH field */ static inline u32 r_length_encoded(enum ipa_version version, u32 length) @@ -125,19 +117,13 @@ static inline u32 r_length_encoded(enum ipa_version version, u32 length) } #define GSI_CH_C_CNTXT_2_OFFSET(ch) \ - GSI_EE_N_CH_C_CNTXT_2_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_CNTXT_2_OFFSET(ch, ee) \ - (0x0001c008 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c008 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_CH_C_CNTXT_3_OFFSET(ch) \ - GSI_EE_N_CH_C_CNTXT_3_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_CNTXT_3_OFFSET(ch, ee) \ - (0x0001c00c + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c00c + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_CH_C_QOS_OFFSET(ch) \ - GSI_EE_N_CH_C_QOS_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_QOS_OFFSET(ch, ee) \ - (0x0001c05c + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c05c + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define WRR_WEIGHT_FMASK GENMASK(3, 0) #define MAX_PREFETCH_FMASK GENMASK(8, 8) #define USE_DB_ENG_FMASK GENMASK(9, 9) @@ -158,29 +144,19 @@ enum gsi_prefetch_mode { }; #define GSI_CH_C_SCRATCH_0_OFFSET(ch) \ - GSI_EE_N_CH_C_SCRATCH_0_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_SCRATCH_0_OFFSET(ch, ee) \ - (0x0001c060 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c060 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_CH_C_SCRATCH_1_OFFSET(ch) \ - GSI_EE_N_CH_C_SCRATCH_1_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_SCRATCH_1_OFFSET(ch, ee) \ - (0x0001c064 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c064 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_CH_C_SCRATCH_2_OFFSET(ch) \ - GSI_EE_N_CH_C_SCRATCH_2_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_SCRATCH_2_OFFSET(ch, ee) \ - (0x0001c068 + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c068 + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_CH_C_SCRATCH_3_OFFSET(ch) \ - GSI_EE_N_CH_C_SCRATCH_3_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_SCRATCH_3_OFFSET(ch, ee) \ - (0x0001c06c + 0x4000 * (ee) + 0x80 * (ch)) + (0x0001c06c + 0x4000 * GSI_EE_AP + 0x80 * (ch)) #define GSI_EV_CH_E_CNTXT_0_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_0_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_0_OFFSET(ev, ee) \ - (0x0001d000 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d000 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) /* enum gsi_channel_type defines EV_CHTYPE field values in EV_CH_E_CNTXT_0 */ #define EV_CHTYPE_FMASK GENMASK(3, 0) #define EV_EE_FMASK GENMASK(7, 4) @@ -190,9 +166,7 @@ enum gsi_prefetch_mode { #define EV_ELEMENT_SIZE_FMASK GENMASK(31, 24) #define GSI_EV_CH_E_CNTXT_1_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_1_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_1_OFFSET(ev, ee) \ - (0x0001d004 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d004 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) /* Encoded value for EV_CH_C_CNTXT_1 register EV_R_LENGTH field */ static inline u32 ev_r_length_encoded(enum ipa_version version, u32 length) { @@ -202,83 +176,53 @@ static inline u32 ev_r_length_encoded(enum ipa_version version, u32 length) } #define GSI_EV_CH_E_CNTXT_2_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_2_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_2_OFFSET(ev, ee) \ - (0x0001d008 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d008 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_3_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_3_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_3_OFFSET(ev, ee) \ - (0x0001d00c + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d00c + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_4_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_4_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_4_OFFSET(ev, ee) \ - (0x0001d010 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d010 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_8_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_8_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_8_OFFSET(ev, ee) \ - (0x0001d020 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d020 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define MODT_FMASK GENMASK(15, 0) #define MODC_FMASK GENMASK(23, 16) #define MOD_CNT_FMASK GENMASK(31, 24) #define GSI_EV_CH_E_CNTXT_9_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_9_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_9_OFFSET(ev, ee) \ - (0x0001d024 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d024 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_10_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_10_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_10_OFFSET(ev, ee) \ - (0x0001d028 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d028 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_11_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_11_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_11_OFFSET(ev, ee) \ - (0x0001d02c + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d02c + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_12_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_12_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_12_OFFSET(ev, ee) \ - (0x0001d030 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d030 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_CNTXT_13_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_CNTXT_13_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_CNTXT_13_OFFSET(ev, ee) \ - (0x0001d034 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d034 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_SCRATCH_0_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_SCRATCH_0_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_SCRATCH_0_OFFSET(ev, ee) \ - (0x0001d048 + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d048 + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_EV_CH_E_SCRATCH_1_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_SCRATCH_1_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_SCRATCH_1_OFFSET(ev, ee) \ - (0x0001d04c + 0x4000 * (ee) + 0x80 * (ev)) + (0x0001d04c + 0x4000 * GSI_EE_AP + 0x80 * (ev)) #define GSI_CH_C_DOORBELL_0_OFFSET(ch) \ - GSI_EE_N_CH_C_DOORBELL_0_OFFSET((ch), GSI_EE_AP) -#define GSI_EE_N_CH_C_DOORBELL_0_OFFSET(ch, ee) \ - (0x0001e000 + 0x4000 * (ee) + 0x08 * (ch)) + (0x0001e000 + 0x4000 * GSI_EE_AP + 0x08 * (ch)) #define GSI_EV_CH_E_DOORBELL_0_OFFSET(ev) \ - GSI_EE_N_EV_CH_E_DOORBELL_0_OFFSET((ev), GSI_EE_AP) -#define GSI_EE_N_EV_CH_E_DOORBELL_0_OFFSET(ev, ee) \ - (0x0001e100 + 0x4000 * (ee) + 0x08 * (ev)) + (0x0001e100 + 0x4000 * GSI_EE_AP + 0x08 * (ev)) #define GSI_GSI_STATUS_OFFSET \ - GSI_EE_N_GSI_STATUS_OFFSET(GSI_EE_AP) -#define GSI_EE_N_GSI_STATUS_OFFSET(ee) \ - (0x0001f000 + 0x4000 * (ee)) + (0x0001f000 + 0x4000 * GSI_EE_AP) #define ENABLED_FMASK GENMASK(0, 0) #define GSI_CH_CMD_OFFSET \ - GSI_EE_N_CH_CMD_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CH_CMD_OFFSET(ee) \ - (0x0001f008 + 0x4000 * (ee)) + (0x0001f008 + 0x4000 * GSI_EE_AP) #define CH_CHID_FMASK GENMASK(7, 0) #define CH_OPCODE_FMASK GENMASK(31, 24) @@ -293,9 +237,7 @@ enum gsi_ch_cmd_opcode { }; #define GSI_EV_CH_CMD_OFFSET \ - GSI_EE_N_EV_CH_CMD_OFFSET(GSI_EE_AP) -#define GSI_EE_N_EV_CH_CMD_OFFSET(ee) \ - (0x0001f010 + 0x4000 * (ee)) + (0x0001f010 + 0x4000 * GSI_EE_AP) #define EV_CHID_FMASK GENMASK(7, 0) #define EV_OPCODE_FMASK GENMASK(31, 24) @@ -307,9 +249,7 @@ enum gsi_evt_cmd_opcode { }; #define GSI_GENERIC_CMD_OFFSET \ - GSI_EE_N_GENERIC_CMD_OFFSET(GSI_EE_AP) -#define GSI_EE_N_GENERIC_CMD_OFFSET(ee) \ - (0x0001f018 + 0x4000 * (ee)) + (0x0001f018 + 0x4000 * GSI_EE_AP) #define GENERIC_OPCODE_FMASK GENMASK(4, 0) #define GENERIC_CHID_FMASK GENMASK(9, 5) #define GENERIC_EE_FMASK GENMASK(13, 10) @@ -326,9 +266,7 @@ enum gsi_generic_cmd_opcode { /* The next register is present for IPA v3.5.1 and above */ #define GSI_GSI_HW_PARAM_2_OFFSET \ - GSI_EE_N_GSI_HW_PARAM_2_OFFSET(GSI_EE_AP) -#define GSI_EE_N_GSI_HW_PARAM_2_OFFSET(ee) \ - (0x0001f040 + 0x4000 * (ee)) + (0x0001f040 + 0x4000 * GSI_EE_AP) #define IRAM_SIZE_FMASK GENMASK(2, 0) #define NUM_CH_PER_EE_FMASK GENMASK(7, 3) #define NUM_EV_PER_EE_FMASK GENMASK(12, 8) @@ -357,13 +295,9 @@ enum gsi_iram_size { /* IRQ condition for each type is cleared by writing type-specific register */ #define GSI_CNTXT_TYPE_IRQ_OFFSET \ - GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(ee) \ - (0x0001f080 + 0x4000 * (ee)) + (0x0001f080 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_TYPE_IRQ_MSK_OFFSET \ - GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(ee) \ - (0x0001f088 + 0x4000 * (ee)) + (0x0001f088 + 0x4000 * GSI_EE_AP) /* Values here are bit positions in the TYPE_IRQ and TYPE_IRQ_MSK registers */ enum gsi_irq_type_id { @@ -377,62 +311,38 @@ enum gsi_irq_type_id { }; #define GSI_CNTXT_SRC_CH_IRQ_OFFSET \ - GSI_EE_N_CNTXT_SRC_CH_IRQ_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_CH_IRQ_OFFSET(ee) \ - (0x0001f090 + 0x4000 * (ee)) + (0x0001f090 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET \ - GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_OFFSET(ee) \ - (0x0001f094 + 0x4000 * (ee)) + (0x0001f094 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET \ - GSI_EE_N_CNTXT_SRC_CH_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_CH_IRQ_MSK_OFFSET(ee) \ - (0x0001f098 + 0x4000 * (ee)) + (0x0001f098 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET \ - GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET(ee) \ - (0x0001f09c + 0x4000 * (ee)) + (0x0001f09c + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET \ - GSI_EE_N_CNTXT_SRC_CH_IRQ_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_CH_IRQ_CLR_OFFSET(ee) \ - (0x0001f0a0 + 0x4000 * (ee)) + (0x0001f0a0 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET \ - GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET(ee) \ - (0x0001f0a4 + 0x4000 * (ee)) + (0x0001f0a4 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_IEOB_IRQ_OFFSET \ - GSI_EE_N_CNTXT_SRC_IEOB_IRQ_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_OFFSET(ee) \ - (0x0001f0b0 + 0x4000 * (ee)) + (0x0001f0b0 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET \ - GSI_EE_N_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET(ee) \ - (0x0001f0b8 + 0x4000 * (ee)) + (0x0001f0b8 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET \ - GSI_EE_N_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET(ee) \ - (0x0001f0c0 + 0x4000 * (ee)) + (0x0001f0c0 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_GLOB_IRQ_STTS_OFFSET \ - GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(ee) \ - (0x0001f100 + 0x4000 * (ee)) + (0x0001f100 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_GLOB_IRQ_EN_OFFSET \ - GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(ee) \ - (0x0001f108 + 0x4000 * (ee)) + (0x0001f108 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_GLOB_IRQ_CLR_OFFSET \ - GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(ee) \ - (0x0001f110 + 0x4000 * (ee)) + (0x0001f110 + 0x4000 * GSI_EE_AP) /* Values here are bit positions in the GLOB_IRQ_* registers */ enum gsi_global_irq_id { ERROR_INT = 0x0, @@ -442,17 +352,11 @@ enum gsi_global_irq_id { }; #define GSI_CNTXT_GSI_IRQ_STTS_OFFSET \ - GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(ee) \ - (0x0001f118 + 0x4000 * (ee)) + (0x0001f118 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_GSI_IRQ_EN_OFFSET \ - GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(ee) \ - (0x0001f120 + 0x4000 * (ee)) + (0x0001f120 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_GSI_IRQ_CLR_OFFSET \ - GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(ee) \ - (0x0001f128 + 0x4000 * (ee)) + (0x0001f128 + 0x4000 * GSI_EE_AP) /* Values here are bit positions in the (general) GSI_IRQ_* registers */ enum gsi_general_id { BREAK_POINT = 0x0, @@ -462,15 +366,11 @@ enum gsi_general_id { }; #define GSI_CNTXT_INTSET_OFFSET \ - GSI_EE_N_CNTXT_INTSET_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_INTSET_OFFSET(ee) \ - (0x0001f180 + 0x4000 * (ee)) + (0x0001f180 + 0x4000 * GSI_EE_AP) #define INTYPE_FMASK GENMASK(0, 0) #define GSI_ERROR_LOG_OFFSET \ - GSI_EE_N_ERROR_LOG_OFFSET(GSI_EE_AP) -#define GSI_EE_N_ERROR_LOG_OFFSET(ee) \ - (0x0001f200 + 0x4000 * (ee)) + (0x0001f200 + 0x4000 * GSI_EE_AP) /* Fields below are present for IPA v3.5.1 and above */ #define ERR_ARG3_FMASK GENMASK(3, 0) @@ -501,14 +401,10 @@ enum gsi_err_type { }; #define GSI_ERROR_LOG_CLR_OFFSET \ - GSI_EE_N_ERROR_LOG_CLR_OFFSET(GSI_EE_AP) -#define GSI_EE_N_ERROR_LOG_CLR_OFFSET(ee) \ - (0x0001f210 + 0x4000 * (ee)) + (0x0001f210 + 0x4000 * GSI_EE_AP) #define GSI_CNTXT_SCRATCH_0_OFFSET \ - GSI_EE_N_CNTXT_SCRATCH_0_OFFSET(GSI_EE_AP) -#define GSI_EE_N_CNTXT_SCRATCH_0_OFFSET(ee) \ - (0x0001f400 + 0x4000 * (ee)) + (0x0001f400 + 0x4000 * GSI_EE_AP) #define INTER_EE_RESULT_FMASK GENMASK(2, 0) #define GENERIC_EE_RESULT_FMASK GENMASK(7, 5) diff --git a/drivers/net/ipa/gsi_trans.c b/drivers/net/ipa/gsi_trans.c index 18e7e8c405bea4c09388adfe9f7ea317d54c5932..26b7f683a3e17454181694775066f16836a98e43 100644 --- a/drivers/net/ipa/gsi_trans.c +++ b/drivers/net/ipa/gsi_trans.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include @@ -22,37 +22,36 @@ * DOC: GSI Transactions * * A GSI transaction abstracts the behavior of a GSI channel by representing - * everything about a related group of IPA commands in a single structure. - * (A "command" in this sense is either a data transfer or an IPA immediate + * everything about a related group of IPA operations in a single structure. + * (A "operation" in this sense is either a data transfer or an IPA immediate * command.) Most details of interaction with the GSI hardware are managed - * by the GSI transaction core, allowing users to simply describe commands + * by the GSI transaction core, allowing users to simply describe operations * to be performed. When a transaction has completed a callback function * (dependent on the type of endpoint associated with the channel) allows * cleanup of resources associated with the transaction. * - * To perform a command (or set of them), a user of the GSI transaction + * To perform an operation (or set of them), a user of the GSI transaction * interface allocates a transaction, indicating the number of TREs required - * (one per command). If sufficient TREs are available, they are reserved + * (one per operation). If sufficient TREs are available, they are reserved * for use in the transaction and the allocation succeeds. This way - * exhaustion of the available TREs in a channel ring is detected - * as early as possible. All resources required to complete a transaction - * are allocated at transaction allocation time. + * exhaustion of the available TREs in a channel ring is detected as early + * as possible. Any other resources that might be needed to complete a + * transaction are also allocated when the transaction is allocated. * - * Commands performed as part of a transaction are represented in an array - * of Linux scatterlist structures. This array is allocated with the - * transaction, and its entries are initialized using standard scatterlist - * functions (such as sg_set_buf() or skb_to_sgvec()). + * Operations performed as part of a transaction are represented in an array + * of Linux scatterlist structures, allocated with the transaction. These + * scatterlist structures are initialized by "adding" operations to the + * transaction. If a buffer in an operation must be mapped for DMA, this is + * done at the time it is added to the transaction. It is possible for a + * mapping error to occur when an operation is added. In this case the + * transaction should simply be freed; this correctly releases resources + * associated with the transaction. * - * Once a transaction's scatterlist structures have been initialized, the - * transaction is committed. The caller is responsible for mapping buffers - * for DMA if necessary, and this should be done *before* allocating - * the transaction. Between a successful allocation and commit of a - * transaction no errors should occur. - * - * Committing transfers ownership of the entire transaction to the GSI - * transaction core. The GSI transaction code formats the content of - * the scatterlist array into the channel ring buffer and informs the - * hardware that new TREs are available to process. + * Once all operations have been successfully added to a transaction, the + * transaction is committed. Committing transfers ownership of the entire + * transaction to the GSI transaction core. The GSI transaction code + * formats the content of the scatterlist array into the channel ring + * buffer and informs the hardware that new TREs are available to process. * * The last TRE in each transaction is marked to interrupt the AP when the * GSI hardware has completed it. Because transfers described by TREs are @@ -125,11 +124,10 @@ void gsi_trans_pool_exit(struct gsi_trans_pool *pool) memset(pool, 0, sizeof(*pool)); } -/* Allocate the requested number of (zeroed) entries from the pool */ -/* Home-grown DMA pool. This way we can preallocate and use the tre_count - * to guarantee allocations will succeed. Even though we specify max_alloc - * (and it can be more than one), we only allow allocation of a single - * element from a DMA pool. +/* Home-grown DMA pool. This way we can preallocate the pool, and guarantee + * allocations will succeed. The immediate commands in a transaction can + * require up to max_alloc elements from the pool. But we only allow + * allocation of a single element from a DMA pool at a time. */ int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool, size_t size, u32 count, u32 max_alloc) @@ -237,68 +235,63 @@ gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index) /* Return the oldest completed transaction for a channel (or null) */ struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel) { - return list_first_entry_or_null(&channel->trans_info.complete, - struct gsi_trans, links); + struct gsi_trans_info *trans_info = &channel->trans_info; + u16 trans_id = trans_info->completed_id; + + if (trans_id == trans_info->pending_id) { + gsi_channel_update(channel); + if (trans_id == trans_info->pending_id) + return NULL; + } + + return &trans_info->trans[trans_id %= channel->tre_count]; } -/* Move a transaction from the allocated list to the committed list */ +/* Move a transaction from allocated to committed state */ static void gsi_trans_move_committed(struct gsi_trans *trans) { struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; struct gsi_trans_info *trans_info = &channel->trans_info; - spin_lock_bh(&trans_info->spinlock); - - list_move_tail(&trans->links, &trans_info->committed); - - spin_unlock_bh(&trans_info->spinlock); + /* This allocated transaction is now committed */ + trans_info->allocated_id++; } -/* Move transactions from the committed list to the pending list */ +/* Move committed transactions to pending state */ static void gsi_trans_move_pending(struct gsi_trans *trans) { struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; struct gsi_trans_info *trans_info = &channel->trans_info; - struct list_head list; - - spin_lock_bh(&trans_info->spinlock); + u16 trans_index = trans - &trans_info->trans[0]; + u16 delta; - /* Move this transaction and all predecessors to the pending list */ - list_cut_position(&list, &trans_info->committed, &trans->links); - list_splice_tail(&list, &trans_info->pending); - - spin_unlock_bh(&trans_info->spinlock); + /* These committed transactions are now pending */ + delta = trans_index - trans_info->committed_id + 1; + trans_info->committed_id += delta % channel->tre_count; } -/* Move a transaction and all of its predecessors from the pending list - * to the completed list. - */ +/* Move pending transactions to completed state */ void gsi_trans_move_complete(struct gsi_trans *trans) { struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; struct gsi_trans_info *trans_info = &channel->trans_info; - struct list_head list; + u16 trans_index = trans - trans_info->trans; + u16 delta; - spin_lock_bh(&trans_info->spinlock); - - /* Move this transaction and all predecessors to completed list */ - list_cut_position(&list, &trans_info->pending, &trans->links); - list_splice_tail(&list, &trans_info->complete); - - spin_unlock_bh(&trans_info->spinlock); + /* These pending transactions are now completed */ + delta = trans_index - trans_info->pending_id + 1; + delta %= channel->tre_count; + trans_info->pending_id += delta; } -/* Move a transaction from the completed list to the polled list */ +/* Move a transaction from completed to polled state */ void gsi_trans_move_polled(struct gsi_trans *trans) { struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id]; struct gsi_trans_info *trans_info = &channel->trans_info; - spin_lock_bh(&trans_info->spinlock); - - list_move_tail(&trans->links, &trans_info->polled); - - spin_unlock_bh(&trans_info->spinlock); + /* This completed transaction is now polled */ + trans_info->completed_id++; } /* Reserve some number of TREs on a channel. Returns true if successful */ @@ -343,20 +336,22 @@ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, struct gsi_channel *channel = &gsi->channel[channel_id]; struct gsi_trans_info *trans_info; struct gsi_trans *trans; + u16 trans_index; if (WARN_ON(tre_count > channel->trans_tre_max)) return NULL; trans_info = &channel->trans_info; - /* We reserve the TREs now, but consume them at commit time. - * If there aren't enough available, we're done. - */ + /* If we can't reserve the TREs for the transaction, we're done */ if (!gsi_trans_tre_reserve(trans_info, tre_count)) return NULL; - /* Allocate and initialize non-zero fields in the transaction */ - trans = gsi_trans_pool_alloc(&trans_info->pool, 1); + trans_index = trans_info->free_id % channel->tre_count; + trans = &trans_info->trans[trans_index]; + memset(trans, 0, sizeof(*trans)); + + /* Initialize non-zero fields in the transaction */ trans->gsi = gsi; trans->channel_id = channel_id; trans->rsvd_count = tre_count; @@ -367,45 +362,37 @@ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id, sg_init_marker(trans->sgl, tre_count); trans->direction = direction; - - spin_lock_bh(&trans_info->spinlock); - - list_add_tail(&trans->links, &trans_info->alloc); - - spin_unlock_bh(&trans_info->spinlock); - refcount_set(&trans->refcount, 1); + /* This free transaction is now allocated */ + trans_info->free_id++; + return trans; } /* Free a previously-allocated transaction */ void gsi_trans_free(struct gsi_trans *trans) { - refcount_t *refcount = &trans->refcount; struct gsi_trans_info *trans_info; - bool last; - /* We must hold the lock to release the last reference */ - if (refcount_dec_not_one(refcount)) + if (!refcount_dec_and_test(&trans->refcount)) return; + /* Unused transactions are allocated but never committed, pending, + * completed, or polled. + */ trans_info = &trans->gsi->channel[trans->channel_id].trans_info; - - spin_lock_bh(&trans_info->spinlock); - - /* Reference might have been added before we got the lock */ - last = refcount_dec_and_test(refcount); - if (last) - list_del(&trans->links); - - spin_unlock_bh(&trans_info->spinlock); - - if (!last) - return; - - if (trans->used_count) + if (!trans->used_count) { + trans_info->allocated_id++; + trans_info->committed_id++; + trans_info->pending_id++; + trans_info->completed_id++; + } else { ipa_gsi_trans_release(trans); + } + + /* This transaction is now free */ + trans_info->polled_id++; /* Releasing the reserved TREs implicitly frees the sgl[] and * (if present) info[] arrays, plus the transaction itself. @@ -548,8 +535,8 @@ static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr, * * Formats channel ring TRE entries based on the content of the scatterlist. * Maps a transaction pointer to the last ring entry used for the transaction, - * so it can be recovered when it completes. Moves the transaction to the - * pending list. Finally, updates the channel ring pointer and optionally + * so it can be recovered when it completes. Moves the transaction to + * pending state. Finally, updates the channel ring pointer and optionally * rings the doorbell. */ static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db) @@ -654,23 +641,27 @@ void gsi_trans_complete(struct gsi_trans *trans) void gsi_channel_trans_cancel_pending(struct gsi_channel *channel) { struct gsi_trans_info *trans_info = &channel->trans_info; - struct gsi_trans *trans; - bool cancelled; + u16 trans_id = trans_info->pending_id; /* channel->gsi->mutex is held by caller */ - spin_lock_bh(&trans_info->spinlock); - cancelled = !list_empty(&trans_info->pending); - list_for_each_entry(trans, &trans_info->pending, links) - trans->cancelled = true; + /* If there are no pending transactions, we're done */ + if (trans_id == trans_info->committed_id) + return; - list_splice_tail_init(&trans_info->pending, &trans_info->complete); + /* Mark all pending transactions cancelled */ + do { + struct gsi_trans *trans; + + trans = &trans_info->trans[trans_id % channel->tre_count]; + trans->cancelled = true; + } while (++trans_id != trans_info->committed_id); - spin_unlock_bh(&trans_info->spinlock); + /* All pending transactions are now completed */ + trans_info->pending_id = trans_info->committed_id; /* Schedule NAPI polling to complete the cancelled transactions */ - if (cancelled) - napi_schedule(&channel->napi); + napi_schedule(&channel->napi); } /* Issue a command to read a single byte from a channel */ @@ -736,10 +727,16 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) * modulo that number to determine the next one that's free. * Transactions are allocated one at a time. */ - ret = gsi_trans_pool_init(&trans_info->pool, sizeof(struct gsi_trans), - tre_max, 1); - if (ret) + trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans), + GFP_KERNEL); + if (!trans_info->trans) return -ENOMEM; + trans_info->free_id = 0; /* all modulo channel->tre_count */ + trans_info->allocated_id = 0; + trans_info->committed_id = 0; + trans_info->pending_id = 0; + trans_info->completed_id = 0; + trans_info->polled_id = 0; /* A completion event contains a pointer to the TRE that caused * the event (which will be the last one used by the transaction). @@ -765,19 +762,13 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id) if (ret) goto err_map_free; - spin_lock_init(&trans_info->spinlock); - INIT_LIST_HEAD(&trans_info->alloc); - INIT_LIST_HEAD(&trans_info->committed); - INIT_LIST_HEAD(&trans_info->pending); - INIT_LIST_HEAD(&trans_info->complete); - INIT_LIST_HEAD(&trans_info->polled); return 0; err_map_free: kfree(trans_info->map); err_trans_free: - gsi_trans_pool_exit(&trans_info->pool); + kfree(trans_info->trans); dev_err(gsi->dev, "error %d initializing channel %u transactions\n", ret, channel_id); @@ -791,6 +782,6 @@ void gsi_channel_trans_exit(struct gsi_channel *channel) struct gsi_trans_info *trans_info = &channel->trans_info; gsi_trans_pool_exit(&trans_info->sg_pool); - gsi_trans_pool_exit(&trans_info->pool); + kfree(trans_info->trans); kfree(trans_info->map); } diff --git a/drivers/net/ipa/gsi_trans.h b/drivers/net/ipa/gsi_trans.h index 7084507830c21d18375c03d60dadc15fd9b9b557..30c1c2dc77c67fb6547cef1cb723cf4741aab975 100644 --- a/drivers/net/ipa/gsi_trans.h +++ b/drivers/net/ipa/gsi_trans.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _GSI_TRANS_H_ #define _GSI_TRANS_H_ @@ -29,7 +29,6 @@ struct gsi_trans_pool; * struct gsi_trans - a GSI transaction * * Most fields in this structure for internal use by the transaction core code: - * @links: Links for channel transaction lists by state * @gsi: GSI pointer * @channel_id: Channel number transaction is associated with * @cancelled: If set by the core code, transaction was cancelled @@ -50,8 +49,6 @@ struct gsi_trans_pool; * received. */ struct gsi_trans { - struct list_head links; /* gsi_channel lists */ - struct gsi *gsi; u8 channel_id; @@ -77,7 +74,7 @@ struct gsi_trans { /** * gsi_trans_pool_init() - Initialize a pool of structures for transactions - * @pool: GSI transaction poll pointer + * @pool: GSI transaction pool pointer * @size: Size of elements in the pool * @count: Minimum number of elements in the pool * @max_alloc: Maximum number of elements allocated at a time from pool diff --git a/drivers/net/ipa/ipa.h b/drivers/net/ipa/ipa.h index 4fc3c72359f5ef00ee60c04bc0fbbd745a1d38ee..09ead433ec38ed66f127ff73665767674dc86d89 100644 --- a/drivers/net/ipa/ipa.h +++ b/drivers/net/ipa/ipa.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_H_ #define _IPA_H_ @@ -44,6 +44,7 @@ struct ipa_interrupt; * @uc_loaded: true after microcontroller has reported it's ready * @reg_addr: DMA address used for IPA register access * @reg_virt: Virtual address used for IPA register access + * @regs: IPA register definitions * @mem_addr: DMA address of IPA-local memory space * @mem_virt: Virtual address of IPA-local memory space * @mem_offset: Offset from @mem_virt used for access to IPA memory @@ -90,6 +91,7 @@ struct ipa { dma_addr_t reg_addr; void __iomem *reg_virt; + const struct ipa_regs *regs; dma_addr_t mem_addr; void *mem_virt; diff --git a/drivers/net/ipa/ipa_cmd.c b/drivers/net/ipa/ipa_cmd.c index 6dea40259b6048d22ed5191a450448440530f289..26c3db9f52b18a3f3e74481a1c9b080d205818b0 100644 --- a/drivers/net/ipa/ipa_cmd.c +++ b/drivers/net/ipa/ipa_cmd.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include @@ -32,7 +32,7 @@ * immediate command's opcode. The payload for a command resides in AP * memory and is described by a single scatterlist entry in its transaction. * Commands do not require a transaction completion callback, and are - * (currently) always issued using gsi_trans_commit_wait(). + * always issued using gsi_trans_commit_wait(). */ /* Some commands can wait until indicated pipeline stages are clear */ @@ -305,6 +305,7 @@ static bool ipa_cmd_register_write_offset_valid(struct ipa *ipa, /* Check whether offsets passed to register_write are valid */ static bool ipa_cmd_register_write_valid(struct ipa *ipa) { + const struct ipa_reg *reg; const char *name; u32 offset; @@ -312,7 +313,8 @@ static bool ipa_cmd_register_write_valid(struct ipa *ipa) * offset will fit in a register write IPA immediate command. */ if (ipa_table_hash_support(ipa)) { - offset = ipa_reg_filt_rout_hash_flush_offset(ipa->version); + reg = ipa_reg(ipa, FILT_ROUT_HASH_FLUSH); + offset = ipa_reg_offset(reg); name = "filter/route hash flush"; if (!ipa_cmd_register_write_offset_valid(ipa, name, offset)) return false; @@ -325,7 +327,8 @@ static bool ipa_cmd_register_write_valid(struct ipa *ipa) * worst case (highest endpoint number) offset of that endpoint * fits in the register write command field(s) that must hold it. */ - offset = IPA_REG_ENDP_STATUS_N_OFFSET(IPA_ENDPOINT_COUNT - 1); + reg = ipa_reg(ipa, ENDP_STATUS); + offset = ipa_reg_n_offset(reg, IPA_ENDPOINT_COUNT - 1); name = "maximal endpoint status"; if (!ipa_cmd_register_write_offset_valid(ipa, name, offset)) return false; diff --git a/drivers/net/ipa/ipa_cmd.h b/drivers/net/ipa/ipa_cmd.h index 9215ddad101079129e58e240b19cfa385421bcb2..8e4243c1f0bbef911be79c93bb9045e989626c84 100644 --- a/drivers/net/ipa/ipa_cmd.h +++ b/drivers/net/ipa/ipa_cmd.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_CMD_H_ #define _IPA_CMD_H_ diff --git a/drivers/net/ipa/ipa_data.h b/drivers/net/ipa/ipa_data.h index e15eb3cd3e333bc925f3c3e8bd91d63151f85577..e5a6ce75c7ddd50c1eaf7bcbe57b002fe9630278 100644 --- a/drivers/net/ipa/ipa_data.h +++ b/drivers/net/ipa/ipa_data.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_DATA_H_ #define _IPA_DATA_H_ @@ -31,7 +31,7 @@ * communication path between the IPA and a particular execution environment * (EE), such as the AP or Modem. Each EE has a set of channels associated * with it, and each channel has an ID unique for that EE. For the most part - * the only GSI channels of concern to this driver belong to the AP + * the only GSI channels of concern to this driver belong to the AP. * * An endpoint is an IPA construct representing a single channel anywhere * in the system. An IPA endpoint ID maps directly to an (EE, channel_id) diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c index 66d2bfdf9e423e1c89c27a449db90ba5cd0d5181..093e11ec7c2d1b3aec3757772cf5558493bf8a4d 100644 --- a/drivers/net/ipa/ipa_endpoint.c +++ b/drivers/net/ipa/ipa_endpoint.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include @@ -23,8 +23,6 @@ #include "ipa_gsi.h" #include "ipa_power.h" -#define atomic_dec_not_zero(v) atomic_add_unless((v), -1, 0) - /* Hardware is told about receive buffers once a "batch" has been queued */ #define IPA_REPLENISH_BATCH 16 /* Must be non-zero */ @@ -72,14 +70,6 @@ struct ipa_status { #define IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK GENMASK(31, 22) #define IPA_STATUS_FLAGS2_TAG_FMASK GENMASK_ULL(63, 16) -static u32 aggr_byte_limit_max(enum ipa_version version) -{ - if (version < IPA_VERSION_4_5) - return field_max(aggr_byte_limit_fmask(true)); - - return field_max(aggr_byte_limit_fmask(false)); -} - /* Compute the aggregation size value to use for a given buffer size */ static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit) { @@ -111,6 +101,7 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, if (!data->toward_ipa) { const struct ipa_endpoint_rx *rx_config; + const struct ipa_reg *reg; u32 buffer_size; u32 aggr_size; u32 limit; @@ -171,7 +162,9 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, */ aggr_size = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD, rx_config->aggr_hard_limit); - limit = aggr_byte_limit_max(ipa->version); + reg = ipa_reg(ipa, ENDP_INIT_AGGR); + + limit = ipa_reg_field_max(reg, BYTE_LIMIT); if (aggr_size > limit) { dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n", data->endpoint_id, aggr_size, limit); @@ -182,6 +175,15 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count, return true; /* Nothing more to check for RX */ } + /* Starting with IPA v4.5 sequencer replication is obsolete */ + if (ipa->version >= IPA_VERSION_4_5) { + if (data->endpoint.config.tx.seq_rep_type) { + dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n", + data->endpoint_id); + return false; + } + } + if (data->endpoint.config.status_enable) { other_name = data->endpoint.config.tx.status_endpoint; if (other_name >= count) { @@ -299,8 +301,10 @@ static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint, static bool ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) { - u32 offset = IPA_REG_ENDP_INIT_CTRL_N_OFFSET(endpoint->endpoint_id); struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; + u32 field_id; + u32 offset; bool state; u32 mask; u32 val; @@ -310,9 +314,13 @@ ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) else WARN_ON(ipa->version >= IPA_VERSION_4_0); - mask = endpoint->toward_ipa ? ENDP_DELAY_FMASK : ENDP_SUSPEND_FMASK; - + reg = ipa_reg(ipa, ENDP_INIT_CTRL); + offset = ipa_reg_n_offset(reg, endpoint->endpoint_id); val = ioread32(ipa->reg_virt + offset); + + field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND; + mask = ipa_reg_bit(reg, field_id); + state = !!(val & mask); /* Don't bother if it's already in the requested state */ @@ -339,13 +347,13 @@ static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint) { u32 mask = BIT(endpoint->endpoint_id); struct ipa *ipa = endpoint->ipa; - u32 offset; + const struct ipa_reg *reg; u32 val; WARN_ON(!(mask & ipa->available)); - offset = ipa_reg_state_aggr_active_offset(ipa->version); - val = ioread32(ipa->reg_virt + offset); + reg = ipa_reg(ipa, STATE_AGGR_ACTIVE); + val = ioread32(ipa->reg_virt + ipa_reg_offset(reg)); return !!(val & mask); } @@ -354,10 +362,12 @@ static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint) { u32 mask = BIT(endpoint->endpoint_id); struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; WARN_ON(!(mask & ipa->available)); - iowrite32(mask, ipa->reg_virt + IPA_REG_AGGR_FORCE_CLOSE_OFFSET); + reg = ipa_reg(ipa, AGGR_FORCE_CLOSE); + iowrite32(mask, ipa->reg_virt + ipa_reg_offset(reg)); } /** @@ -456,6 +466,7 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) while (initialized) { u32 endpoint_id = __ffs(initialized); struct ipa_endpoint *endpoint; + const struct ipa_reg *reg; u32 offset; initialized ^= BIT(endpoint_id); @@ -465,7 +476,8 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa)) continue; - offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id); + reg = ipa_reg(ipa, ENDP_STATUS); + offset = ipa_reg_n_offset(reg, endpoint_id); /* Value written is 0, and all bits are updated. That * means status is disabled on the endpoint, and as a @@ -485,22 +497,23 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa) static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_CFG_N_OFFSET(endpoint->endpoint_id); + u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; enum ipa_cs_offload_en enabled; + const struct ipa_reg *reg; u32 val = 0; + reg = ipa_reg(ipa, ENDP_INIT_CFG); /* FRAG_OFFLOAD_EN is 0 */ if (endpoint->config.checksum) { - enum ipa_version version = endpoint->ipa->version; + enum ipa_version version = ipa->version; if (endpoint->toward_ipa) { - u32 checksum_offset; + u32 off; /* Checksum header offset is in 4-byte units */ - checksum_offset = sizeof(struct rmnet_map_header); - checksum_offset /= sizeof(u32); - val |= u32_encode_bits(checksum_offset, - CS_METADATA_HDR_OFFSET_FMASK); + off = sizeof(struct rmnet_map_header) / sizeof(u32); + val |= ipa_reg_encode(reg, CS_METADATA_HDR_OFFSET, off); enabled = version < IPA_VERSION_4_5 ? IPA_CS_OFFLOAD_UL @@ -513,24 +526,26 @@ static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint) } else { enabled = IPA_CS_OFFLOAD_NONE; } - val |= u32_encode_bits(enabled, CS_OFFLOAD_EN_FMASK); + val |= ipa_reg_encode(reg, CS_OFFLOAD_EN, enabled); /* CS_GEN_QMB_MASTER_SEL is 0 */ - iowrite32(val, endpoint->ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint) { - u32 offset; + u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val; if (!endpoint->toward_ipa) return; - offset = IPA_REG_ENDP_INIT_NAT_N_OFFSET(endpoint->endpoint_id); - val = u32_encode_bits(IPA_NAT_BYPASS, NAT_EN_FMASK); + reg = ipa_reg(ipa, ENDP_INIT_NAT); + val = ipa_reg_encode(reg, NAT_EN, IPA_NAT_BYPASS); - iowrite32(val, endpoint->ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static u32 @@ -554,6 +569,50 @@ ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint) return header_size; } +/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */ +static u32 ipa_header_size_encode(enum ipa_version version, + const struct ipa_reg *reg, u32 header_size) +{ + u32 field_max = ipa_reg_field_max(reg, HDR_LEN); + u32 val; + + /* We know field_max can be used as a mask (2^n - 1) */ + val = ipa_reg_encode(reg, HDR_LEN, header_size & field_max); + if (version < IPA_VERSION_4_5) { + WARN_ON(header_size > field_max); + return val; + } + + /* IPA v4.5 adds a few more most-significant bits */ + header_size >>= hweight32(field_max); + WARN_ON(header_size > ipa_reg_field_max(reg, HDR_LEN_MSB)); + val |= ipa_reg_encode(reg, HDR_LEN_MSB, header_size); + + return val; +} + +/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */ +static u32 ipa_metadata_offset_encode(enum ipa_version version, + const struct ipa_reg *reg, u32 offset) +{ + u32 field_max = ipa_reg_field_max(reg, HDR_OFST_METADATA); + u32 val; + + /* We know field_max can be used as a mask (2^n - 1) */ + val = ipa_reg_encode(reg, HDR_OFST_METADATA, offset); + if (version < IPA_VERSION_4_5) { + WARN_ON(offset > field_max); + return val; + } + + /* IPA v4.5 adds a few more most-significant bits */ + offset >>= hweight32(field_max); + WARN_ON(offset > ipa_reg_field_max(reg, HDR_OFST_METADATA_MSB)); + val |= ipa_reg_encode(reg, HDR_OFST_METADATA_MSB, offset); + + return val; +} + /** * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register * @endpoint: Endpoint pointer @@ -577,36 +636,38 @@ ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint) */ static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_HDR_N_OFFSET(endpoint->endpoint_id); + u32 endpoint_id = endpoint->endpoint_id; struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; + reg = ipa_reg(ipa, ENDP_INIT_HDR); if (endpoint->config.qmap) { enum ipa_version version = ipa->version; size_t header_size; header_size = ipa_qmap_header_size(version, endpoint); - val = ipa_header_size_encoded(version, header_size); + val = ipa_header_size_encode(version, reg, header_size); /* Define how to fill fields in a received QMAP header */ if (!endpoint->toward_ipa) { - u32 offset; /* Field offset within header */ + u32 off; /* Field offset within header */ /* Where IPA will write the metadata value */ - offset = offsetof(struct rmnet_map_header, mux_id); - val |= ipa_metadata_offset_encoded(version, offset); + off = offsetof(struct rmnet_map_header, mux_id); + val |= ipa_metadata_offset_encode(version, reg, off); /* Where IPA will write the length */ - offset = offsetof(struct rmnet_map_header, pkt_len); + off = offsetof(struct rmnet_map_header, pkt_len); /* Upper bits are stored in HDR_EXT with IPA v4.5 */ if (version >= IPA_VERSION_4_5) - offset &= field_mask(HDR_OFST_PKT_SIZE_FMASK); + off &= ipa_reg_field_max(reg, HDR_OFST_PKT_SIZE); - val |= HDR_OFST_PKT_SIZE_VALID_FMASK; - val |= u32_encode_bits(offset, HDR_OFST_PKT_SIZE_FMASK); + val |= ipa_reg_bit(reg, HDR_OFST_PKT_SIZE_VALID); + val |= ipa_reg_encode(reg, HDR_OFST_PKT_SIZE, off); } /* For QMAP TX, metadata offset is 0 (modem assumes this) */ - val |= HDR_OFST_METADATA_VALID_FMASK; + val |= ipa_reg_bit(reg, HDR_OFST_METADATA_VALID); /* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */ /* HDR_A5_MUX is 0 */ @@ -614,19 +675,21 @@ static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint) /* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */ } - iowrite32(val, ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_HDR_EXT_N_OFFSET(endpoint->endpoint_id); u32 pad_align = endpoint->config.rx.pad_align; + u32 endpoint_id = endpoint->endpoint_id; struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; + reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT); if (endpoint->config.qmap) { /* We have a header, so we must specify its endianness */ - val |= HDR_ENDIANNESS_FMASK; /* big endian */ + val |= ipa_reg_bit(reg, HDR_ENDIANNESS); /* big endian */ /* A QMAP header contains a 6 bit pad field at offset 0. * The RMNet driver assumes this field is meaningful in @@ -636,16 +699,16 @@ static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint) * (although 0) should be ignored. */ if (!endpoint->toward_ipa) { - val |= HDR_TOTAL_LEN_OR_PAD_VALID_FMASK; + val |= ipa_reg_bit(reg, HDR_TOTAL_LEN_OR_PAD_VALID); /* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */ - val |= HDR_PAYLOAD_LEN_INC_PADDING_FMASK; + val |= ipa_reg_bit(reg, HDR_PAYLOAD_LEN_INC_PADDING); /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */ } } /* HDR_PAYLOAD_LEN_INC_PADDING is 0 */ if (!endpoint->toward_ipa) - val |= u32_encode_bits(pad_align, HDR_PAD_TO_ALIGNMENT_FMASK); + val |= ipa_reg_encode(reg, HDR_PAD_TO_ALIGNMENT, pad_align); /* IPA v4.5 adds some most-significant bits to a few fields, * two of which are defined in the HDR (not HDR_EXT) register. @@ -653,191 +716,170 @@ static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint) if (ipa->version >= IPA_VERSION_4_5) { /* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */ if (endpoint->config.qmap && !endpoint->toward_ipa) { - u32 offset; + u32 mask = ipa_reg_field_max(reg, HDR_OFST_PKT_SIZE); + u32 off; /* Field offset within header */ - offset = offsetof(struct rmnet_map_header, pkt_len); - offset >>= hweight32(HDR_OFST_PKT_SIZE_FMASK); - val |= u32_encode_bits(offset, - HDR_OFST_PKT_SIZE_MSB_FMASK); + off = offsetof(struct rmnet_map_header, pkt_len); + /* Low bits are in the ENDP_INIT_HDR register */ + off >>= hweight32(mask); + val |= ipa_reg_encode(reg, HDR_OFST_PKT_SIZE_MSB, off); /* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */ } } - iowrite32(val, ipa->reg_virt + offset); + + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint) { u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; u32 offset; if (endpoint->toward_ipa) return; /* Register not valid for TX endpoints */ - offset = IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(endpoint_id); + reg = ipa_reg(ipa, ENDP_INIT_HDR_METADATA_MASK); + offset = ipa_reg_n_offset(reg, endpoint_id); /* Note that HDR_ENDIANNESS indicates big endian header fields */ if (endpoint->config.qmap) val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK); - iowrite32(val, endpoint->ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + offset); } static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_MODE_N_OFFSET(endpoint->endpoint_id); + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; + u32 offset; u32 val; if (!endpoint->toward_ipa) return; /* Register not valid for RX endpoints */ + reg = ipa_reg(ipa, ENDP_INIT_MODE); if (endpoint->config.dma_mode) { enum ipa_endpoint_name name = endpoint->config.dma_endpoint; - u32 dma_endpoint_id; + u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id; - dma_endpoint_id = endpoint->ipa->name_map[name]->endpoint_id; - - val = u32_encode_bits(IPA_DMA, MODE_FMASK); - val |= u32_encode_bits(dma_endpoint_id, DEST_PIPE_INDEX_FMASK); + val = ipa_reg_encode(reg, ENDP_MODE, IPA_DMA); + val |= ipa_reg_encode(reg, DEST_PIPE_INDEX, dma_endpoint_id); } else { - val = u32_encode_bits(IPA_BASIC, MODE_FMASK); + val = ipa_reg_encode(reg, ENDP_MODE, IPA_BASIC); } /* All other bits unspecified (and 0) */ - iowrite32(val, endpoint->ipa->reg_virt + offset); + offset = ipa_reg_n_offset(reg, endpoint->endpoint_id); + iowrite32(val, ipa->reg_virt + offset); } -/* Encoded values for AGGR endpoint register fields */ -static u32 aggr_byte_limit_encoded(enum ipa_version version, u32 limit) +/* For IPA v4.5+, times are expressed using Qtime. The AP uses one of two + * pulse generators (0 and 1) to measure elapsed time. In ipa_qtime_config() + * they're configured to have granularity 100 usec and 1 msec, respectively. + * + * The return value is the positive or negative Qtime value to use to + * express the (microsecond) time provided. A positive return value + * means pulse generator 0 can be used; otherwise use pulse generator 1. + */ +static int ipa_qtime_val(u32 microseconds, u32 max) { - if (version < IPA_VERSION_4_5) - return u32_encode_bits(limit, aggr_byte_limit_fmask(true)); + u32 val; - return u32_encode_bits(limit, aggr_byte_limit_fmask(false)); + /* Use 100 microsecond granularity if possible */ + val = DIV_ROUND_CLOSEST(microseconds, 100); + if (val <= max) + return (int)val; + + /* Have to use pulse generator 1 (millisecond granularity) */ + val = DIV_ROUND_CLOSEST(microseconds, 1000); + WARN_ON(val > max); + + return (int)-val; } /* Encode the aggregation timer limit (microseconds) based on IPA version */ -static u32 aggr_time_limit_encoded(enum ipa_version version, u32 limit) +static u32 aggr_time_limit_encode(struct ipa *ipa, const struct ipa_reg *reg, + u32 microseconds) { - u32 gran_sel; - u32 fmask; + u32 max; u32 val; - if (version < IPA_VERSION_4_5) { - /* We set aggregation granularity in ipa_hardware_config() */ - fmask = aggr_time_limit_fmask(true); - val = DIV_ROUND_CLOSEST(limit, IPA_AGGR_GRANULARITY); - WARN(val > field_max(fmask), - "aggr_time_limit too large (%u > %u usec)\n", - val, field_max(fmask) * IPA_AGGR_GRANULARITY); - - return u32_encode_bits(val, fmask); - } - - /* IPA v4.5 expresses the time limit using Qtime. The AP has - * pulse generators 0 and 1 available, which were configured - * in ipa_qtime_config() to have granularity 100 usec and - * 1 msec, respectively. Use pulse generator 0 if possible, - * otherwise fall back to pulse generator 1. - */ - fmask = aggr_time_limit_fmask(false); - val = DIV_ROUND_CLOSEST(limit, 100); - if (val > field_max(fmask)) { - /* Have to use pulse generator 1 (millisecond granularity) */ - gran_sel = AGGR_GRAN_SEL_FMASK; - val = DIV_ROUND_CLOSEST(limit, 1000); - WARN(val > field_max(fmask), - "aggr_time_limit too large (%u > %u usec)\n", - limit, field_max(fmask) * 1000); - } else { - /* We can use pulse generator 0 (100 usec granularity) */ - gran_sel = 0; - } + if (!microseconds) + return 0; /* Nothing to compute if time limit is 0 */ - return gran_sel | u32_encode_bits(val, fmask); -} + max = ipa_reg_field_max(reg, TIME_LIMIT); + if (ipa->version >= IPA_VERSION_4_5) { + u32 gran_sel; + int ret; + + /* Compute the Qtime limit value to use */ + ret = ipa_qtime_val(microseconds, max); + if (ret < 0) { + val = -ret; + gran_sel = ipa_reg_bit(reg, AGGR_GRAN_SEL); + } else { + val = ret; + gran_sel = 0; + } -static u32 aggr_sw_eof_active_encoded(enum ipa_version version, bool enabled) -{ - u32 val = enabled ? 1 : 0; + return gran_sel | ipa_reg_encode(reg, TIME_LIMIT, val); + } - if (version < IPA_VERSION_4_5) - return u32_encode_bits(val, aggr_sw_eof_active_fmask(true)); + /* We program aggregation granularity in ipa_hardware_config() */ + val = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY); + WARN(val > max, "aggr_time_limit too large (%u > %u usec)\n", + microseconds, max * IPA_AGGR_GRANULARITY); - return u32_encode_bits(val, aggr_sw_eof_active_fmask(false)); + return ipa_reg_encode(reg, TIME_LIMIT, val); } static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_AGGR_N_OFFSET(endpoint->endpoint_id); - enum ipa_version version = endpoint->ipa->version; + u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; + reg = ipa_reg(ipa, ENDP_INIT_AGGR); if (endpoint->config.aggregation) { if (!endpoint->toward_ipa) { const struct ipa_endpoint_rx *rx_config; u32 buffer_size; - bool close_eof; u32 limit; rx_config = &endpoint->config.rx; - val |= u32_encode_bits(IPA_ENABLE_AGGR, AGGR_EN_FMASK); - val |= u32_encode_bits(IPA_GENERIC, AGGR_TYPE_FMASK); + val |= ipa_reg_encode(reg, AGGR_EN, IPA_ENABLE_AGGR); + val |= ipa_reg_encode(reg, AGGR_TYPE, IPA_GENERIC); buffer_size = rx_config->buffer_size; limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD, rx_config->aggr_hard_limit); - val |= aggr_byte_limit_encoded(version, limit); + val |= ipa_reg_encode(reg, BYTE_LIMIT, limit); limit = rx_config->aggr_time_limit; - val |= aggr_time_limit_encoded(version, limit); + val |= aggr_time_limit_encode(ipa, reg, limit); /* AGGR_PKT_LIMIT is 0 (unlimited) */ - close_eof = rx_config->aggr_close_eof; - val |= aggr_sw_eof_active_encoded(version, close_eof); + if (rx_config->aggr_close_eof) + val |= ipa_reg_bit(reg, SW_EOF_ACTIVE); } else { - val |= u32_encode_bits(IPA_ENABLE_DEAGGR, - AGGR_EN_FMASK); - val |= u32_encode_bits(IPA_QCMAP, AGGR_TYPE_FMASK); + val |= ipa_reg_encode(reg, AGGR_EN, IPA_ENABLE_DEAGGR); + val |= ipa_reg_encode(reg, AGGR_TYPE, IPA_QCMAP); /* other fields ignored */ } /* AGGR_FORCE_CLOSE is 0 */ /* AGGR_GRAN_SEL is 0 for IPA v4.5 */ } else { - val |= u32_encode_bits(IPA_BYPASS_AGGR, AGGR_EN_FMASK); + val |= ipa_reg_encode(reg, AGGR_EN, IPA_BYPASS_AGGR); /* other fields ignored */ } - iowrite32(val, endpoint->ipa->reg_virt + offset); -} - -/* Return the Qtime-based head-of-line blocking timer value that - * represents the given number of microseconds. The result - * includes both the timer value and the selected timer granularity. - */ -static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds) -{ - u32 gran_sel; - u32 val; - - /* IPA v4.5 expresses time limits using Qtime. The AP has - * pulse generators 0 and 1 available, which were configured - * in ipa_qtime_config() to have granularity 100 usec and - * 1 msec, respectively. Use pulse generator 0 if possible, - * otherwise fall back to pulse generator 1. - */ - val = DIV_ROUND_CLOSEST(microseconds, 100); - if (val > field_max(TIME_LIMIT_FMASK)) { - /* Have to use pulse generator 1 (millisecond granularity) */ - gran_sel = GRAN_SEL_FMASK; - val = DIV_ROUND_CLOSEST(microseconds, 1000); - } else { - /* We can use pulse generator 0 (100 usec granularity) */ - gran_sel = 0; - } - - return gran_sel | u32_encode_bits(val, TIME_LIMIT_FMASK); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } /* The head-of-line blocking timer is defined as a tick count. For @@ -845,12 +887,11 @@ static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds) * derived from the 19.2 MHz SoC XO clock. For older IPA versions * each tick represents 128 cycles of the IPA core clock. * - * Return the encoded value that should be written to that register - * that represents the timeout period provided. For IPA v4.2 this - * encodes a base and scale value, while for earlier versions the - * value is a simple tick count. + * Return the encoded value representing the timeout period provided + * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register. */ -static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds) +static u32 hol_block_timer_encode(struct ipa *ipa, const struct ipa_reg *reg, + u32 microseconds) { u32 width; u32 scale; @@ -862,18 +903,34 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds) if (!microseconds) return 0; /* Nothing to compute if timer period is 0 */ - if (ipa->version >= IPA_VERSION_4_5) - return hol_block_timer_qtime_val(ipa, microseconds); + if (ipa->version >= IPA_VERSION_4_5) { + u32 max = ipa_reg_field_max(reg, TIMER_LIMIT); + u32 gran_sel; + int ret; + + /* Compute the Qtime limit value to use */ + ret = ipa_qtime_val(microseconds, max); + if (ret < 0) { + val = -ret; + gran_sel = ipa_reg_bit(reg, TIMER_GRAN_SEL); + } else { + val = ret; + gran_sel = 0; + } - /* Use 64 bit arithmetic to avoid overflow... */ + return gran_sel | ipa_reg_encode(reg, TIMER_LIMIT, val); + } + + /* Use 64 bit arithmetic to avoid overflow */ rate = ipa_core_clock_rate(ipa); ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC); - /* ...but we still need to fit into a 32-bit register */ - WARN_ON(ticks > U32_MAX); + + /* We still need the result to fit into the field */ + WARN_ON(ticks > ipa_reg_field_max(reg, TIMER_BASE_VALUE)); /* IPA v3.5.1 through v4.1 just record the tick count */ if (ipa->version < IPA_VERSION_4_2) - return (u32)ticks; + return ipa_reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks); /* For IPA v4.2, the tick count is represented by base and * scale fields within the 32-bit timer register, where: @@ -883,8 +940,8 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds) * count, and extract the number of bits in the base field * such that high bit is included. */ - high = fls(ticks); /* 1..32 */ - width = HWEIGHT32(BASE_VALUE_FMASK); + high = fls(ticks); /* 1..32 (or warning above) */ + width = hweight32(ipa_reg_fmask(reg, TIMER_BASE_VALUE)); scale = high > width ? high - width : 0; if (scale) { /* If we're scaling, round up to get a closer result */ @@ -894,8 +951,8 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds) scale++; } - val = u32_encode_bits(scale, SCALE_FMASK); - val |= u32_encode_bits(ticks >> scale, BASE_VALUE_FMASK); + val = ipa_reg_encode(reg, TIMER_SCALE, scale); + val |= ipa_reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks >> scale); return val; } @@ -906,28 +963,34 @@ static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint, { u32 endpoint_id = endpoint->endpoint_id; struct ipa *ipa = endpoint->ipa; - u32 offset; + const struct ipa_reg *reg; u32 val; /* This should only be changed when HOL_BLOCK_EN is disabled */ - offset = IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(endpoint_id); - val = hol_block_timer_val(ipa, microseconds); - iowrite32(val, ipa->reg_virt + offset); + reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER); + val = hol_block_timer_encode(ipa, reg, microseconds); + + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable) { u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 offset; u32 val; - val = enable ? HOL_BLOCK_EN_FMASK : 0; - offset = IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(endpoint_id); - iowrite32(val, endpoint->ipa->reg_virt + offset); + reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN); + offset = ipa_reg_n_offset(reg, endpoint_id); + val = enable ? ipa_reg_bit(reg, HOL_BLOCK_EN) : 0; + + iowrite32(val, ipa->reg_virt + offset); + /* When enabling, the register must be written twice for IPA v4.5+ */ - if (enable && endpoint->ipa->version >= IPA_VERSION_4_5) - iowrite32(val, endpoint->ipa->reg_virt + offset); + if (enable && ipa->version >= IPA_VERSION_4_5) + iowrite32(val, ipa->reg_virt + offset); } /* Assumes HOL_BLOCK is in disabled state */ @@ -960,46 +1023,58 @@ void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_DEAGGR_N_OFFSET(endpoint->endpoint_id); + u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; if (!endpoint->toward_ipa) return; /* Register not valid for RX endpoints */ + reg = ipa_reg(ipa, ENDP_INIT_DEAGGR); /* DEAGGR_HDR_LEN is 0 */ /* PACKET_OFFSET_VALID is 0 */ /* PACKET_OFFSET_LOCATION is ignored (not valid) */ /* MAX_PACKET_LEN is 0 (not enforced) */ - iowrite32(val, endpoint->ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_RSRC_GRP_N_OFFSET(endpoint->endpoint_id); + u32 resource_group = endpoint->config.resource_group; + u32 endpoint_id = endpoint->endpoint_id; struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val; - val = rsrc_grp_encoded(ipa->version, endpoint->config.resource_group); - iowrite32(val, ipa->reg_virt + offset); + reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP); + val = ipa_reg_encode(reg, ENDP_RSRC_GRP, resource_group); + + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint) { - u32 offset = IPA_REG_ENDP_INIT_SEQ_N_OFFSET(endpoint->endpoint_id); - u32 val = 0; + u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; + u32 val; if (!endpoint->toward_ipa) return; /* Register not valid for RX endpoints */ + reg = ipa_reg(ipa, ENDP_INIT_SEQ); + /* Low-order byte configures primary packet processing */ - val |= u32_encode_bits(endpoint->config.tx.seq_type, SEQ_TYPE_FMASK); + val = ipa_reg_encode(reg, SEQ_TYPE, endpoint->config.tx.seq_type); - /* Second byte configures replicated packet processing */ - val |= u32_encode_bits(endpoint->config.tx.seq_rep_type, - SEQ_REP_TYPE_FMASK); + /* Second byte (if supported) configures replicated packet processing */ + if (ipa->version < IPA_VERSION_4_5) + val |= ipa_reg_encode(reg, SEQ_REP_TYPE, + endpoint->config.tx.seq_rep_type); - iowrite32(val, endpoint->ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } /** @@ -1049,13 +1124,12 @@ static void ipa_endpoint_status(struct ipa_endpoint *endpoint) { u32 endpoint_id = endpoint->endpoint_id; struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 val = 0; - u32 offset; - - offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id); + reg = ipa_reg(ipa, ENDP_STATUS); if (endpoint->config.status_enable) { - val |= STATUS_EN_FMASK; + val |= ipa_reg_bit(reg, STATUS_EN); if (endpoint->toward_ipa) { enum ipa_endpoint_name name; u32 status_endpoint_id; @@ -1063,16 +1137,16 @@ static void ipa_endpoint_status(struct ipa_endpoint *endpoint) name = endpoint->config.tx.status_endpoint; status_endpoint_id = ipa->name_map[name]->endpoint_id; - val |= u32_encode_bits(status_endpoint_id, - STATUS_ENDP_FMASK); + val |= ipa_reg_encode(reg, STATUS_ENDP, + status_endpoint_id); } /* STATUS_LOCATION is 0, meaning status element precedes - * packet (not present for IPA v4.5) + * packet (not present for IPA v4.5+) */ - /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v3.5.1) */ + /* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */ } - iowrite32(val, ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id)); } static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint, @@ -1412,16 +1486,18 @@ void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint, void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id) { + const struct ipa_reg *reg; u32 val; + reg = ipa_reg(ipa, ROUTE); /* ROUTE_DIS is 0 */ - val = u32_encode_bits(endpoint_id, ROUTE_DEF_PIPE_FMASK); - val |= ROUTE_DEF_HDR_TABLE_FMASK; - val |= u32_encode_bits(0, ROUTE_DEF_HDR_OFST_FMASK); - val |= u32_encode_bits(endpoint_id, ROUTE_FRAG_DEF_PIPE_FMASK); - val |= ROUTE_DEF_RETAIN_HDR_FMASK; + val = ipa_reg_encode(reg, ROUTE_DEF_PIPE, endpoint_id); + val |= ipa_reg_bit(reg, ROUTE_DEF_HDR_TABLE); + /* ROUTE_DEF_HDR_OFST is 0 */ + val |= ipa_reg_encode(reg, ROUTE_FRAG_DEF_PIPE, endpoint_id); + val |= ipa_reg_bit(reg, ROUTE_DEF_RETAIN_HDR); - iowrite32(val, ipa->reg_virt + IPA_REG_ROUTE_OFFSET); + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); } void ipa_endpoint_default_route_clear(struct ipa *ipa) @@ -1765,6 +1841,7 @@ void ipa_endpoint_teardown(struct ipa *ipa) int ipa_endpoint_config(struct ipa *ipa) { struct device *dev = &ipa->pdev->dev; + const struct ipa_reg *reg; u32 initialized; u32 rx_base; u32 rx_mask; @@ -1791,11 +1868,12 @@ int ipa_endpoint_config(struct ipa *ipa) /* Find out about the endpoints supplied by the hardware, and ensure * the highest one doesn't exceed the number we support. */ - val = ioread32(ipa->reg_virt + IPA_REG_FLAVOR_0_OFFSET); + reg = ipa_reg(ipa, FLAVOR_0); + val = ioread32(ipa->reg_virt + ipa_reg_offset(reg)); /* Our RX is an IPA producer */ - rx_base = u32_get_bits(val, IPA_PROD_LOWEST_FMASK); - max = rx_base + u32_get_bits(val, IPA_MAX_PROD_PIPES_FMASK); + rx_base = ipa_reg_decode(reg, PROD_LOWEST, val); + max = rx_base + ipa_reg_decode(reg, MAX_PROD_PIPES, val); if (max > IPA_ENDPOINT_MAX) { dev_err(dev, "too many endpoints (%u > %u)\n", max, IPA_ENDPOINT_MAX); @@ -1804,7 +1882,7 @@ int ipa_endpoint_config(struct ipa *ipa) rx_mask = GENMASK(max - 1, rx_base); /* Our TX is an IPA consumer */ - max = u32_get_bits(val, IPA_MAX_CONS_PIPES_FMASK); + max = ipa_reg_decode(reg, MAX_CONS_PIPES, val); tx_mask = GENMASK(max - 1, 0); ipa->available = rx_mask | tx_mask; diff --git a/drivers/net/ipa/ipa_endpoint.h b/drivers/net/ipa/ipa_endpoint.h index 28e0a7386fd723d0cb2003913859f30b7bc2c944..d8dfa24f521409435f486833d15fd398c943a8e5 100644 --- a/drivers/net/ipa/ipa_endpoint.h +++ b/drivers/net/ipa/ipa_endpoint.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_ENDPOINT_H_ #define _IPA_ENDPOINT_H_ diff --git a/drivers/net/ipa/ipa_interrupt.c b/drivers/net/ipa/ipa_interrupt.c index 307bed2ee707863d120b805d54caabf7aee4eae5..c269432f9c2ee3d45fc1b31ba31f60cbdf11742a 100644 --- a/drivers/net/ipa/ipa_interrupt.c +++ b/drivers/net/ipa/ipa_interrupt.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ /* DOC: IPA Interrupts @@ -53,13 +53,15 @@ static void ipa_interrupt_process(struct ipa_interrupt *interrupt, u32 irq_id) { bool uc_irq = ipa_interrupt_uc(interrupt, irq_id); struct ipa *ipa = interrupt->ipa; + const struct ipa_reg *reg; u32 mask = BIT(irq_id); u32 offset; /* For microcontroller interrupts, clear the interrupt right away, * "to avoid clearing unhandled interrupts." */ - offset = ipa_reg_irq_clr_offset(ipa->version); + reg = ipa_reg(ipa, IPA_IRQ_CLR); + offset = ipa_reg_offset(reg); if (uc_irq) iowrite32(mask, ipa->reg_virt + offset); @@ -80,6 +82,7 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id) struct ipa_interrupt *interrupt = dev_id; struct ipa *ipa = interrupt->ipa; u32 enabled = interrupt->enabled; + const struct ipa_reg *reg; struct device *dev; u32 pending; u32 offset; @@ -95,7 +98,8 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id) * including conditions whose interrupt is not enabled. Handle * only the enabled ones. */ - offset = ipa_reg_irq_stts_offset(ipa->version); + reg = ipa_reg(ipa, IPA_IRQ_STTS); + offset = ipa_reg_offset(reg); pending = ioread32(ipa->reg_virt + offset); while ((mask = pending & enabled)) { do { @@ -112,7 +116,8 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id) if (pending) { dev_dbg(dev, "clearing disabled IPA interrupts 0x%08x\n", pending); - offset = ipa_reg_irq_clr_offset(ipa->version); + reg = ipa_reg(ipa, IPA_IRQ_CLR); + offset = ipa_reg_offset(reg); iowrite32(pending, ipa->reg_virt + offset); } out_power_put: @@ -128,6 +133,7 @@ static void ipa_interrupt_suspend_control(struct ipa_interrupt *interrupt, { struct ipa *ipa = interrupt->ipa; u32 mask = BIT(endpoint_id); + const struct ipa_reg *reg; u32 offset; u32 val; @@ -137,7 +143,8 @@ static void ipa_interrupt_suspend_control(struct ipa_interrupt *interrupt, if (ipa->version == IPA_VERSION_3_0) return; - offset = ipa_reg_irq_suspend_en_offset(ipa->version); + reg = ipa_reg(ipa, IRQ_SUSPEND_EN); + offset = ipa_reg_offset(reg); val = ioread32(ipa->reg_virt + offset); if (enable) val |= mask; @@ -164,18 +171,18 @@ ipa_interrupt_suspend_disable(struct ipa_interrupt *interrupt, u32 endpoint_id) void ipa_interrupt_suspend_clear_all(struct ipa_interrupt *interrupt) { struct ipa *ipa = interrupt->ipa; - u32 offset; + const struct ipa_reg *reg; u32 val; - offset = ipa_reg_irq_suspend_info_offset(ipa->version); - val = ioread32(ipa->reg_virt + offset); + reg = ipa_reg(ipa, IRQ_SUSPEND_INFO); + val = ioread32(ipa->reg_virt + ipa_reg_offset(reg)); /* SUSPEND interrupt status isn't cleared on IPA version 3.0 */ if (ipa->version == IPA_VERSION_3_0) return; - offset = ipa_reg_irq_suspend_clr_offset(ipa->version); - iowrite32(val, ipa->reg_virt + offset); + reg = ipa_reg(ipa, IRQ_SUSPEND_CLR); + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); } /* Simulate arrival of an IPA TX_SUSPEND interrupt */ @@ -189,7 +196,7 @@ void ipa_interrupt_add(struct ipa_interrupt *interrupt, enum ipa_irq_id ipa_irq, ipa_irq_handler_t handler) { struct ipa *ipa = interrupt->ipa; - u32 offset; + const struct ipa_reg *reg; if (WARN_ON(ipa_irq >= IPA_IRQ_COUNT)) return; @@ -198,8 +205,9 @@ void ipa_interrupt_add(struct ipa_interrupt *interrupt, /* Update the IPA interrupt mask to enable it */ interrupt->enabled |= BIT(ipa_irq); - offset = ipa_reg_irq_en_offset(ipa->version); - iowrite32(interrupt->enabled, ipa->reg_virt + offset); + + reg = ipa_reg(ipa, IPA_IRQ_EN); + iowrite32(interrupt->enabled, ipa->reg_virt + ipa_reg_offset(reg)); } /* Remove the handler for an IPA interrupt type */ @@ -207,15 +215,16 @@ void ipa_interrupt_remove(struct ipa_interrupt *interrupt, enum ipa_irq_id ipa_irq) { struct ipa *ipa = interrupt->ipa; - u32 offset; + const struct ipa_reg *reg; if (WARN_ON(ipa_irq >= IPA_IRQ_COUNT)) return; /* Update the IPA interrupt mask to disable it */ interrupt->enabled &= ~BIT(ipa_irq); - offset = ipa_reg_irq_en_offset(ipa->version); - iowrite32(interrupt->enabled, ipa->reg_virt + offset); + + reg = ipa_reg(ipa, IPA_IRQ_EN); + iowrite32(interrupt->enabled, ipa->reg_virt + ipa_reg_offset(reg)); interrupt->handler[ipa_irq] = NULL; } @@ -225,8 +234,8 @@ struct ipa_interrupt *ipa_interrupt_config(struct ipa *ipa) { struct device *dev = &ipa->pdev->dev; struct ipa_interrupt *interrupt; + const struct ipa_reg *reg; unsigned int irq; - u32 offset; int ret; ret = platform_get_irq_byname(ipa->pdev, "ipa"); @@ -244,8 +253,8 @@ struct ipa_interrupt *ipa_interrupt_config(struct ipa *ipa) interrupt->irq = irq; /* Start with all IPA interrupts disabled */ - offset = ipa_reg_irq_en_offset(ipa->version); - iowrite32(0, ipa->reg_virt + offset); + reg = ipa_reg(ipa, IPA_IRQ_EN); + iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg)); ret = request_threaded_irq(irq, NULL, ipa_isr_thread, IRQF_ONESHOT, "ipa", interrupt); diff --git a/drivers/net/ipa/ipa_interrupt.h b/drivers/net/ipa/ipa_interrupt.h index 231390cea52a2f56e9c01880a30778ef18f33bfc..f31fd9965fdc678a38e5f1223d28233a773007e1 100644 --- a/drivers/net/ipa/ipa_interrupt.h +++ b/drivers/net/ipa/ipa_interrupt.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_INTERRUPT_H_ #define _IPA_INTERRUPT_H_ diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 32962d885acd501b67d8e2e5439abd823368d12e..3461ad3029ab825c3b6e58002a6885a10436642e 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -183,31 +183,97 @@ static void ipa_teardown(struct ipa *ipa) gsi_teardown(&ipa->gsi); } +static void +ipa_hardware_config_bcr(struct ipa *ipa, const struct ipa_data *data) +{ + const struct ipa_reg *reg; + u32 val; + + /* IPA v4.5+ has no backward compatibility register */ + if (ipa->version >= IPA_VERSION_4_5) + return; + + reg = ipa_reg(ipa, IPA_BCR); + val = data->backward_compat; + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); +} + +static void ipa_hardware_config_tx(struct ipa *ipa) +{ + enum ipa_version version = ipa->version; + const struct ipa_reg *reg; + u32 offset; + u32 val; + + if (version <= IPA_VERSION_4_0 || version >= IPA_VERSION_4_5) + return; + + /* Disable PA mask to allow HOLB drop */ + reg = ipa_reg(ipa, IPA_TX_CFG); + offset = ipa_reg_offset(reg); + + val = ioread32(ipa->reg_virt + offset); + + val &= ~ipa_reg_bit(reg, PA_MASK_EN); + + iowrite32(val, ipa->reg_virt + offset); +} + +static void ipa_hardware_config_clkon(struct ipa *ipa) +{ + enum ipa_version version = ipa->version; + const struct ipa_reg *reg; + u32 val; + + if (version >= IPA_VERSION_4_5) + return; + + if (version < IPA_VERSION_4_0 && version != IPA_VERSION_3_1) + return; + + /* Implement some hardware workarounds */ + reg = ipa_reg(ipa, CLKON_CFG); + if (version == IPA_VERSION_3_1) { + /* Disable MISC clock gating */ + val = ipa_reg_bit(reg, CLKON_MISC); + } else { /* IPA v4.0+ */ + /* Enable open global clocks in the CLKON configuration */ + val = ipa_reg_bit(reg, CLKON_GLOBAL); + val |= ipa_reg_bit(reg, GLOBAL_2X_CLK); + } + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); +} + /* Configure bus access behavior for IPA components */ static void ipa_hardware_config_comp(struct ipa *ipa) { + const struct ipa_reg *reg; + u32 offset; u32 val; /* Nothing to configure prior to IPA v4.0 */ if (ipa->version < IPA_VERSION_4_0) return; - val = ioread32(ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET); + reg = ipa_reg(ipa, COMP_CFG); + offset = ipa_reg_offset(reg); + val = ioread32(ipa->reg_virt + offset); if (ipa->version == IPA_VERSION_4_0) { - val &= ~IPA_QMB_SELECT_CONS_EN_FMASK; - val &= ~IPA_QMB_SELECT_PROD_EN_FMASK; - val &= ~IPA_QMB_SELECT_GLOBAL_EN_FMASK; + val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_CONS_EN); + val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_PROD_EN); + val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_GLOBAL_EN); } else if (ipa->version < IPA_VERSION_4_5) { - val |= GSI_MULTI_AXI_MASTERS_DIS_FMASK; + val |= ipa_reg_bit(reg, GSI_MULTI_AXI_MASTERS_DIS); } else { - /* For IPA v4.5 IPA_FULL_FLUSH_WAIT_RSC_CLOSE_EN is 0 */ + /* For IPA v4.5 FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */ } - val |= GSI_MULTI_INORDER_RD_DIS_FMASK; - val |= GSI_MULTI_INORDER_WR_DIS_FMASK; + val |= ipa_reg_bit(reg, GSI_MULTI_INORDER_RD_DIS); + val |= ipa_reg_bit(reg, GSI_MULTI_INORDER_WR_DIS); - iowrite32(val, ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET); + iowrite32(val, ipa->reg_virt + offset); } /* Configure DDR and (possibly) PCIe max read/write QSB values */ @@ -216,6 +282,7 @@ ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data) { const struct ipa_qsb_data *data0; const struct ipa_qsb_data *data1; + const struct ipa_reg *reg; u32 val; /* QMB 0 represents DDR; QMB 1 (if present) represents PCIe */ @@ -224,25 +291,31 @@ ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data) data1 = &data->qsb_data[IPA_QSB_MASTER_PCIE]; /* Max outstanding write accesses for QSB masters */ - val = u32_encode_bits(data0->max_writes, GEN_QMB_0_MAX_WRITES_FMASK); + reg = ipa_reg(ipa, QSB_MAX_WRITES); + + val = ipa_reg_encode(reg, GEN_QMB_0_MAX_WRITES, data0->max_writes); if (data->qsb_count > 1) - val |= u32_encode_bits(data1->max_writes, - GEN_QMB_1_MAX_WRITES_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_WRITES_OFFSET); + val |= ipa_reg_encode(reg, GEN_QMB_1_MAX_WRITES, + data1->max_writes); + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); /* Max outstanding read accesses for QSB masters */ - val = u32_encode_bits(data0->max_reads, GEN_QMB_0_MAX_READS_FMASK); + reg = ipa_reg(ipa, QSB_MAX_READS); + + val = ipa_reg_encode(reg, GEN_QMB_0_MAX_READS, data0->max_reads); if (ipa->version >= IPA_VERSION_4_0) - val |= u32_encode_bits(data0->max_reads_beats, - GEN_QMB_0_MAX_READS_BEATS_FMASK); + val |= ipa_reg_encode(reg, GEN_QMB_0_MAX_READS_BEATS, + data0->max_reads_beats); if (data->qsb_count > 1) { - val |= u32_encode_bits(data1->max_reads, - GEN_QMB_1_MAX_READS_FMASK); + val = ipa_reg_encode(reg, GEN_QMB_1_MAX_READS, + data1->max_reads); if (ipa->version >= IPA_VERSION_4_0) - val |= u32_encode_bits(data1->max_reads_beats, - GEN_QMB_1_MAX_READS_BEATS_FMASK); + val |= ipa_reg_encode(reg, GEN_QMB_1_MAX_READS_BEATS, + data1->max_reads_beats); } - iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_READS_OFFSET); + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); } /* The internal inactivity timer clock is used for the aggregation timer */ @@ -278,48 +351,96 @@ static __always_inline u32 ipa_aggr_granularity_val(u32 usec) */ static void ipa_qtime_config(struct ipa *ipa) { + const struct ipa_reg *reg; + u32 offset; u32 val; /* Timer clock divider must be disabled when we change the rate */ - iowrite32(0, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET); + reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG); + iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg)); + reg = ipa_reg(ipa, QTIME_TIMESTAMP_CFG); /* Set DPL time stamp resolution to use Qtime (instead of 1 msec) */ - val = u32_encode_bits(DPL_TIMESTAMP_SHIFT, DPL_TIMESTAMP_LSB_FMASK); - val |= u32_encode_bits(1, DPL_TIMESTAMP_SEL_FMASK); + val = ipa_reg_encode(reg, DPL_TIMESTAMP_LSB, DPL_TIMESTAMP_SHIFT); + val |= ipa_reg_bit(reg, DPL_TIMESTAMP_SEL); /* Configure tag and NAT Qtime timestamp resolution as well */ - val |= u32_encode_bits(TAG_TIMESTAMP_SHIFT, TAG_TIMESTAMP_LSB_FMASK); - val |= u32_encode_bits(NAT_TIMESTAMP_SHIFT, NAT_TIMESTAMP_LSB_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_QTIME_TIMESTAMP_CFG_OFFSET); + val = ipa_reg_encode(reg, TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT); + val = ipa_reg_encode(reg, NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT); + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); /* Set granularity of pulse generators used for other timers */ - val = u32_encode_bits(IPA_GRAN_100_US, GRAN_0_FMASK); - val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_1_FMASK); - val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_2_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_PULSE_GRAN_CFG_OFFSET); + reg = ipa_reg(ipa, TIMERS_PULSE_GRAN_CFG); + val = ipa_reg_encode(reg, PULSE_GRAN_0, IPA_GRAN_100_US); + val |= ipa_reg_encode(reg, PULSE_GRAN_1, IPA_GRAN_1_MS); + val |= ipa_reg_encode(reg, PULSE_GRAN_2, IPA_GRAN_1_MS); + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); /* Actual divider is 1 more than value supplied here */ - val = u32_encode_bits(IPA_XO_CLOCK_DIVIDER - 1, DIV_VALUE_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET); + reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG); + offset = ipa_reg_offset(reg); + val = ipa_reg_encode(reg, DIV_VALUE, IPA_XO_CLOCK_DIVIDER - 1); + + iowrite32(val, ipa->reg_virt + offset); /* Divider value is set; re-enable the common timer clock divider */ - val |= u32_encode_bits(1, DIV_ENABLE_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET); + val |= ipa_reg_bit(reg, DIV_ENABLE); + + iowrite32(val, ipa->reg_virt + offset); +} + +/* Before IPA v4.5 timing is controlled by a counter register */ +static void ipa_hardware_config_counter(struct ipa *ipa) +{ + u32 granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY); + const struct ipa_reg *reg; + u32 val; + + reg = ipa_reg(ipa, COUNTER_CFG); + /* If defined, EOT_COAL_GRANULARITY is 0 */ + val = ipa_reg_encode(reg, AGGR_GRANULARITY, granularity); + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); +} + +static void ipa_hardware_config_timing(struct ipa *ipa) +{ + if (ipa->version < IPA_VERSION_4_5) + ipa_hardware_config_counter(ipa); + else + ipa_qtime_config(ipa); +} + +static void ipa_hardware_config_hashing(struct ipa *ipa) +{ + const struct ipa_reg *reg; + + if (ipa->version != IPA_VERSION_4_2) + return; + + /* IPA v4.2 does not support hashed tables, so disable them */ + reg = ipa_reg(ipa, FILT_ROUT_HASH_EN); + + /* IPV6_ROUTER_HASH, IPV6_FILTER_HASH, IPV4_ROUTER_HASH, + * IPV4_FILTER_HASH are all zero. + */ + iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg)); } static void ipa_idle_indication_cfg(struct ipa *ipa, u32 enter_idle_debounce_thresh, bool const_non_idle_enable) { - u32 offset; + const struct ipa_reg *reg; u32 val; - val = u32_encode_bits(enter_idle_debounce_thresh, - ENTER_IDLE_DEBOUNCE_THRESH_FMASK); + reg = ipa_reg(ipa, IDLE_INDICATION_CFG); + val = ipa_reg_encode(reg, ENTER_IDLE_DEBOUNCE_THRESH, + enter_idle_debounce_thresh); if (const_non_idle_enable) - val |= CONST_NON_IDLE_ENABLE_FMASK; + val |= ipa_reg_bit(reg, CONST_NON_IDLE_ENABLE); - offset = ipa_reg_idle_indication_cfg_offset(ipa->version); - iowrite32(val, ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); } /** @@ -349,55 +470,13 @@ static void ipa_hardware_dcd_deconfig(struct ipa *ipa) */ static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data) { - enum ipa_version version = ipa->version; - u32 granularity; - u32 val; - - /* IPA v4.5+ has no backward compatibility register */ - if (version < IPA_VERSION_4_5) { - val = data->backward_compat; - iowrite32(val, ipa->reg_virt + IPA_REG_BCR_OFFSET); - } - - /* Implement some hardware workarounds */ - if (version >= IPA_VERSION_4_0 && version < IPA_VERSION_4_5) { - /* Disable PA mask to allow HOLB drop */ - val = ioread32(ipa->reg_virt + IPA_REG_TX_CFG_OFFSET); - val &= ~PA_MASK_EN_FMASK; - iowrite32(val, ipa->reg_virt + IPA_REG_TX_CFG_OFFSET); - - /* Enable open global clocks in the CLKON configuration */ - val = GLOBAL_FMASK | GLOBAL_2X_CLK_FMASK; - } else if (version == IPA_VERSION_3_1) { - val = MISC_FMASK; /* Disable MISC clock gating */ - } else { - val = 0; /* No CLKON configuration needed */ - } - if (val) - iowrite32(val, ipa->reg_virt + IPA_REG_CLKON_CFG_OFFSET); - + ipa_hardware_config_bcr(ipa, data); + ipa_hardware_config_tx(ipa); + ipa_hardware_config_clkon(ipa); ipa_hardware_config_comp(ipa); - - /* Configure system bus limits */ ipa_hardware_config_qsb(ipa, data); - - if (version < IPA_VERSION_4_5) { - /* Configure aggregation timer granularity */ - granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY); - val = u32_encode_bits(granularity, AGGR_GRANULARITY_FMASK); - iowrite32(val, ipa->reg_virt + IPA_REG_COUNTER_CFG_OFFSET); - } else { - ipa_qtime_config(ipa); - } - - /* IPA v4.2 does not support hashed tables, so disable them */ - if (version == IPA_VERSION_4_2) { - u32 offset = ipa_reg_filt_rout_hash_en_offset(version); - - iowrite32(0, ipa->reg_virt + offset); - } - - /* Enable dynamic clock division */ + ipa_hardware_config_timing(ipa); + ipa_hardware_config_hashing(ipa); ipa_hardware_dcd_config(ipa); } @@ -612,29 +691,6 @@ static void ipa_validate_build(void) /* Aggregation granularity value can't be 0, and must fit */ BUILD_BUG_ON(!ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY)); - BUILD_BUG_ON(ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY) > - field_max(AGGR_GRANULARITY_FMASK)); -} - -static bool ipa_version_valid(enum ipa_version version) -{ - switch (version) { - case IPA_VERSION_3_0: - case IPA_VERSION_3_1: - case IPA_VERSION_3_5: - case IPA_VERSION_3_5_1: - case IPA_VERSION_4_0: - case IPA_VERSION_4_1: - case IPA_VERSION_4_2: - case IPA_VERSION_4_5: - case IPA_VERSION_4_7: - case IPA_VERSION_4_9: - case IPA_VERSION_4_11: - return true; - - default: - return false; - } } /** @@ -678,8 +734,8 @@ static int ipa_probe(struct platform_device *pdev) return -ENODEV; } - if (!ipa_version_valid(data->version)) { - dev_err(dev, "invalid IPA version\n"); + if (!ipa_version_supported(data->version)) { + dev_err(dev, "unsupported IPA version %u\n", data->version); return -EINVAL; } diff --git a/drivers/net/ipa/ipa_mem.c b/drivers/net/ipa/ipa_mem.c index 1e9eae208e44f89d61e9fb8a280a81fd63724567..f84c6830495a4c295489300b8edbdc0aa93a4dbb 100644 --- a/drivers/net/ipa/ipa_mem.c +++ b/drivers/net/ipa/ipa_mem.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include @@ -75,6 +75,7 @@ ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id) int ipa_mem_setup(struct ipa *ipa) { dma_addr_t addr = ipa->zero_addr; + const struct ipa_reg *reg; const struct ipa_mem *mem; struct gsi_trans *trans; u32 offset; @@ -112,8 +113,10 @@ int ipa_mem_setup(struct ipa *ipa) /* Tell the hardware where the processing context area is located */ mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX); offset = ipa->mem_offset + mem->offset; - val = proc_cntxt_base_addr_encoded(ipa->version, offset); - iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET); + + reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT); + val = ipa_reg_encode(reg, IPA_BASE_ADDR, offset); + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); return 0; } @@ -306,6 +309,7 @@ static bool ipa_mem_size_valid(struct ipa *ipa) int ipa_mem_config(struct ipa *ipa) { struct device *dev = &ipa->pdev->dev; + const struct ipa_reg *reg; const struct ipa_mem *mem; dma_addr_t addr; u32 mem_size; @@ -314,12 +318,14 @@ int ipa_mem_config(struct ipa *ipa) u32 i; /* Check the advertised location and size of the shared memory area */ - val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET); + reg = ipa_reg(ipa, SHARED_MEM_SIZE); + val = ioread32(ipa->reg_virt + ipa_reg_offset(reg)); /* The fields in the register are in 8 byte units */ - ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK); + ipa->mem_offset = 8 * ipa_reg_decode(reg, MEM_BADDR, val); + /* Make sure the end is within the region's mapped space */ - mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK); + mem_size = 8 * ipa_reg_decode(reg, MEM_SIZE, val); /* If the sizes don't match, issue a warning */ if (ipa->mem_offset + mem_size < ipa->mem_size) { @@ -568,7 +574,7 @@ static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size) } /* Align the address down and the size up to a page boundary */ - addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK; + addr = qcom_smem_virt_to_phys(virt); phys = addr & PAGE_MASK; size = PAGE_ALIGN(size + addr - phys); iova = phys; /* We just want a direct mapping */ diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c index c8b1c4d9c5073fb270aaa3374b627c7e34c48aff..423422a2a445f91e4181fe1cd5e978f00c5f1595 100644 --- a/drivers/net/ipa/ipa_modem.c +++ b/drivers/net/ipa/ipa_modem.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include diff --git a/drivers/net/ipa/ipa_modem.h b/drivers/net/ipa/ipa_modem.h index e64ccc2402e9d954518c56adbaff61f6b103b5bf..d85718db9a575c2ef65572c5b83e65c67a58fb29 100644 --- a/drivers/net/ipa/ipa_modem.h +++ b/drivers/net/ipa/ipa_modem.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_MODEM_H_ #define _IPA_MODEM_H_ diff --git a/drivers/net/ipa/ipa_power.c b/drivers/net/ipa/ipa_power.c index db5ac7552286e354a70f2dacefd47bcdf7866f80..8420f93128a268cce757f93dcaaf6a7979a3c0cd 100644 --- a/drivers/net/ipa/ipa_power.c +++ b/drivers/net/ipa/ipa_power.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include diff --git a/drivers/net/ipa/ipa_power.h b/drivers/net/ipa/ipa_power.h index 6f84f057a2095b92d869ba0c8d626ab85b9c9c34..896f052e51a1ca67ec89dfd011b554437a06c8d0 100644 --- a/drivers/net/ipa/ipa_power.h +++ b/drivers/net/ipa/ipa_power.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_POWER_H_ #define _IPA_POWER_H_ diff --git a/drivers/net/ipa/ipa_qmi.c b/drivers/net/ipa/ipa_qmi.c index ec010cf2e816a9f46336a66a3d52f4bb827c20fd..8295fd4b70d16ce79fd2b5149dadc8ef2f96b09c 100644 --- a/drivers/net/ipa/ipa_qmi.c +++ b/drivers/net/ipa/ipa_qmi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -308,12 +308,12 @@ init_modem_driver_req(struct ipa_qmi *ipa_qmi) mem = ipa_mem_find(ipa, IPA_MEM_V4_ROUTE); req.v4_route_tbl_info_valid = 1; req.v4_route_tbl_info.start = ipa->mem_offset + mem->offset; - req.v4_route_tbl_info.count = mem->size / sizeof(__le64); + req.v4_route_tbl_info.end = IPA_ROUTE_MODEM_COUNT - 1; mem = ipa_mem_find(ipa, IPA_MEM_V6_ROUTE); req.v6_route_tbl_info_valid = 1; req.v6_route_tbl_info.start = ipa->mem_offset + mem->offset; - req.v6_route_tbl_info.count = mem->size / sizeof(__le64); + req.v6_route_tbl_info.end = IPA_ROUTE_MODEM_COUNT - 1; mem = ipa_mem_find(ipa, IPA_MEM_V4_FILTER); req.v4_filter_tbl_start_valid = 1; @@ -352,7 +352,7 @@ init_modem_driver_req(struct ipa_qmi *ipa_qmi) req.v4_hash_route_tbl_info_valid = 1; req.v4_hash_route_tbl_info.start = ipa->mem_offset + mem->offset; - req.v4_hash_route_tbl_info.count = mem->size / sizeof(__le64); + req.v4_hash_route_tbl_info.end = IPA_ROUTE_MODEM_COUNT - 1; } mem = ipa_mem_find(ipa, IPA_MEM_V6_ROUTE_HASHED); @@ -360,7 +360,7 @@ init_modem_driver_req(struct ipa_qmi *ipa_qmi) req.v6_hash_route_tbl_info_valid = 1; req.v6_hash_route_tbl_info.start = ipa->mem_offset + mem->offset; - req.v6_hash_route_tbl_info.count = mem->size / sizeof(__le64); + req.v6_hash_route_tbl_info.end = IPA_ROUTE_MODEM_COUNT - 1; } mem = ipa_mem_find(ipa, IPA_MEM_V4_FILTER_HASHED); diff --git a/drivers/net/ipa/ipa_qmi.h b/drivers/net/ipa/ipa_qmi.h index 856ef629ccc8dc47cf37cf0c81d3629f334d582e..1c236826c17ab6dca053bcba38990bf4da30617a 100644 --- a/drivers/net/ipa/ipa_qmi.h +++ b/drivers/net/ipa/ipa_qmi.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_QMI_H_ #define _IPA_QMI_H_ diff --git a/drivers/net/ipa/ipa_qmi_msg.c b/drivers/net/ipa/ipa_qmi_msg.c index 6838e8065072ba7afe3fecd5abbab92b825a098f..97c0befe8d86f6204fb037ff8f6daab6e6e73262 100644 --- a/drivers/net/ipa/ipa_qmi_msg.c +++ b/drivers/net/ipa/ipa_qmi_msg.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include #include @@ -311,7 +311,7 @@ struct qmi_elem_info ipa_init_modem_driver_req_ei[] = { .tlv_type = 0x12, .offset = offsetof(struct ipa_init_modem_driver_req, v4_route_tbl_info), - .ei_array = ipa_mem_array_ei, + .ei_array = ipa_mem_bounds_ei, }, { .data_type = QMI_OPT_FLAG, @@ -332,7 +332,7 @@ struct qmi_elem_info ipa_init_modem_driver_req_ei[] = { .tlv_type = 0x13, .offset = offsetof(struct ipa_init_modem_driver_req, v6_route_tbl_info), - .ei_array = ipa_mem_array_ei, + .ei_array = ipa_mem_bounds_ei, }, { .data_type = QMI_OPT_FLAG, @@ -496,7 +496,7 @@ struct qmi_elem_info ipa_init_modem_driver_req_ei[] = { .tlv_type = 0x1b, .offset = offsetof(struct ipa_init_modem_driver_req, v4_hash_route_tbl_info), - .ei_array = ipa_mem_array_ei, + .ei_array = ipa_mem_bounds_ei, }, { .data_type = QMI_OPT_FLAG, @@ -517,7 +517,7 @@ struct qmi_elem_info ipa_init_modem_driver_req_ei[] = { .tlv_type = 0x1c, .offset = offsetof(struct ipa_init_modem_driver_req, v6_hash_route_tbl_info), - .ei_array = ipa_mem_array_ei, + .ei_array = ipa_mem_bounds_ei, }, { .data_type = QMI_OPT_FLAG, diff --git a/drivers/net/ipa/ipa_qmi_msg.h b/drivers/net/ipa/ipa_qmi_msg.h index 495e85abe50bdb8a0e5b7f7dd3fc022449b73469..e29663965f43416a603765f3c5c4eae0efa3386c 100644 --- a/drivers/net/ipa/ipa_qmi_msg.h +++ b/drivers/net/ipa/ipa_qmi_msg.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_QMI_MSG_H_ #define _IPA_QMI_MSG_H_ @@ -86,9 +86,11 @@ enum ipa_platform_type { IPA_QMI_PLATFORM_TYPE_MSM_QNX_V01 = 0x5, /* QNX MSM */ }; -/* This defines the start and end offset of a range of memory. Both - * fields are offsets relative to the start of IPA shared memory. - * The end value is the last addressable byte *within* the range. +/* This defines the start and end offset of a range of memory. The start + * value is a byte offset relative to the start of IPA shared memory. The + * end value is the last addressable unit *within* the range. Typically + * the end value is in units of bytes, however it can also be a maximum + * array index value. */ struct ipa_mem_bounds { u32 start; @@ -129,18 +131,19 @@ struct ipa_init_modem_driver_req { u8 hdr_tbl_info_valid; struct ipa_mem_bounds hdr_tbl_info; - /* Routing table information. These define the location and size of - * non-hashable IPv4 and IPv6 filter tables. The start values are - * offsets relative to the start of IPA shared memory. + /* Routing table information. These define the location and maximum + * *index* (not byte) for the modem portion of non-hashable IPv4 and + * IPv6 routing tables. The start values are byte offsets relative + * to the start of IPA shared memory. */ u8 v4_route_tbl_info_valid; - struct ipa_mem_array v4_route_tbl_info; + struct ipa_mem_bounds v4_route_tbl_info; u8 v6_route_tbl_info_valid; - struct ipa_mem_array v6_route_tbl_info; + struct ipa_mem_bounds v6_route_tbl_info; /* Filter table information. These define the location of the * non-hashable IPv4 and IPv6 filter tables. The start values are - * offsets relative to the start of IPA shared memory. + * byte offsets relative to the start of IPA shared memory. */ u8 v4_filter_tbl_start_valid; u32 v4_filter_tbl_start; @@ -181,18 +184,20 @@ struct ipa_init_modem_driver_req { u8 zip_tbl_info_valid; struct ipa_mem_bounds zip_tbl_info; - /* Routing table information. These define the location and size - * of hashable IPv4 and IPv6 filter tables. The start values are - * offsets relative to the start of IPA shared memory. + /* Routing table information. These define the location and maximum + * *index* (not byte) for the modem portion of hashable IPv4 and IPv6 + * routing tables (if supported by hardware). The start values are + * byte offsets relative to the start of IPA shared memory. */ u8 v4_hash_route_tbl_info_valid; - struct ipa_mem_array v4_hash_route_tbl_info; + struct ipa_mem_bounds v4_hash_route_tbl_info; u8 v6_hash_route_tbl_info_valid; - struct ipa_mem_array v6_hash_route_tbl_info; + struct ipa_mem_bounds v6_hash_route_tbl_info; /* Filter table information. These define the location and size - * of hashable IPv4 and IPv6 filter tables. The start values are - * offsets relative to the start of IPA shared memory. + * of hashable IPv4 and IPv6 filter tables (if supported by hardware). + * The start values are byte offsets relative to the start of IPA + * shared memory. */ u8 v4_hash_filter_tbl_start_valid; u32 v4_hash_filter_tbl_start; diff --git a/drivers/net/ipa/ipa_reg.c b/drivers/net/ipa/ipa_reg.c index e6147a1cd787b661f6e5591c2758fd617c0e74d1..22f067741d9b26d342d7c11cbb1d4ea7e713d740 100644 --- a/drivers/net/ipa/ipa_reg.c +++ b/drivers/net/ipa/ipa_reg.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include @@ -9,11 +9,105 @@ #include "ipa.h" #include "ipa_reg.h" +/* Is this register valid and defined for the current IPA version? */ +static bool ipa_reg_valid(struct ipa *ipa, enum ipa_reg_id reg_id) +{ + enum ipa_version version = ipa->version; + bool valid; + + /* Check for bogus (out of range) register IDs */ + if ((u32)reg_id >= ipa->regs->reg_count) + return false; + + switch (reg_id) { + case IPA_BCR: + case COUNTER_CFG: + valid = version < IPA_VERSION_4_5; + break; + + case IPA_TX_CFG: + case FLAVOR_0: + case IDLE_INDICATION_CFG: + valid = version >= IPA_VERSION_3_5; + break; + + case QTIME_TIMESTAMP_CFG: + case TIMERS_XO_CLK_DIV_CFG: + case TIMERS_PULSE_GRAN_CFG: + valid = version >= IPA_VERSION_4_5; + break; + + case SRC_RSRC_GRP_45_RSRC_TYPE: + case DST_RSRC_GRP_45_RSRC_TYPE: + valid = version <= IPA_VERSION_3_1 || + version == IPA_VERSION_4_5; + break; + + case SRC_RSRC_GRP_67_RSRC_TYPE: + case DST_RSRC_GRP_67_RSRC_TYPE: + valid = version <= IPA_VERSION_3_1; + break; + + case ENDP_FILTER_ROUTER_HSH_CFG: + valid = version != IPA_VERSION_4_2; + break; + + case IRQ_SUSPEND_EN: + case IRQ_SUSPEND_CLR: + valid = version >= IPA_VERSION_3_1; + break; + + default: + valid = true; /* Others should be defined for all versions */ + break; + } + + /* To be valid, it must be defined */ + + return valid && ipa->regs->reg[reg_id]; +} + +const struct ipa_reg *ipa_reg(struct ipa *ipa, enum ipa_reg_id reg_id) +{ + if (WARN_ON(!ipa_reg_valid(ipa, reg_id))) + return NULL; + + return ipa->regs->reg[reg_id]; +} + +static const struct ipa_regs *ipa_regs(enum ipa_version version) +{ + switch (version) { + case IPA_VERSION_3_1: + return &ipa_regs_v3_1; + case IPA_VERSION_3_5_1: + return &ipa_regs_v3_5_1; + case IPA_VERSION_4_2: + return &ipa_regs_v4_2; + case IPA_VERSION_4_5: + return &ipa_regs_v4_5; + case IPA_VERSION_4_9: + return &ipa_regs_v4_9; + case IPA_VERSION_4_11: + return &ipa_regs_v4_11; + default: + return NULL; + } +} + int ipa_reg_init(struct ipa *ipa) { struct device *dev = &ipa->pdev->dev; + const struct ipa_regs *regs; struct resource *res; + regs = ipa_regs(ipa->version); + if (!regs) + return -EINVAL; + + if (WARN_ON(regs->reg_count > IPA_REG_ID_COUNT)) + return -EINVAL; + /* Setup IPA register memory */ res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM, "ipa-reg"); @@ -28,6 +122,7 @@ int ipa_reg_init(struct ipa *ipa) return -ENOMEM; } ipa->reg_addr = res->start; + ipa->regs = regs; return 0; } diff --git a/drivers/net/ipa/ipa_reg.h b/drivers/net/ipa/ipa_reg.h index 6f35438cda890ededc18de5b4fdf8d411c5ca1c7..7bf70f70f63fec99ee0c921be809cb44619af263 100644 --- a/drivers/net/ipa/ipa_reg.h +++ b/drivers/net/ipa/ipa_reg.h @@ -1,12 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #ifndef _IPA_REG_H_ #define _IPA_REG_H_ #include +#include #include "ipa_version.h" @@ -16,304 +17,325 @@ struct ipa; * DOC: IPA Registers * * IPA registers are located within the "ipa-reg" address space defined by - * Device Tree. The offset of each register within that space is specified - * by symbols defined below. The address space is mapped to virtual memory - * space in ipa_mem_init(). All IPA registers are 32 bits wide. + * Device Tree. Each register has a specified offset within that space, + * which is mapped into virtual memory space in ipa_mem_init(). Each + * has a unique identifer, taken from the ipa_reg_id enumerated type. + * All IPA registers are 32 bits wide. * - * Certain register types are duplicated for a number of instances of - * something. For example, each IPA endpoint has an set of registers - * defining its configuration. The offset to an endpoint's set of registers - * is computed based on an "base" offset, plus an endpoint's ID multiplied - * and a "stride" value for the register. For such registers, the offset is - * computed by a function-like macro that takes a parameter used in the - * computation. + * Certain "parameterized" register types are duplicated for a number of + * instances of something. For example, each IPA endpoint has an set of + * registers defining its configuration. The offset to an endpoint's set + * of registers is computed based on an "base" offset, plus an endpoint's + * ID multiplied and a "stride" value for the register. Similarly, some + * registers have an offset that depends on execution environment. In + * this case, the stride is multiplied by a member of the gsi_ee_id + * enumerated type. * - * Some register offsets depend on execution environment. For these an "ee" - * parameter is supplied to the offset macro. The "ee" value is a member of - * the gsi_ee enumerated type. + * Each version of IPA implements an array of ipa_reg structures indexed + * by register ID. Each entry in the array specifies the base offset and + * (for parameterized registers) a non-zero stride value. Not all versions + * of IPA define all registers. The offset for a register is returned by + * ipa_reg_offset() when the register's ipa_reg structure is supplied; + * zero is returned for an undefined register (this should never happen). * - * The offset of a register dependent on endpoint ID is computed by a macro - * that is supplied a parameter "ep", "txep", or "rxep". A register with an - * "ep" parameter is valid for any endpoint; a register with a "txep" or - * "rxep" parameter is valid only for TX or RX endpoints, respectively. The - * "*ep" value is assumed to be less than the maximum valid endpoint ID - * for the current hardware, and that will not exceed IPA_ENDPOINT_MAX. - * - * The offset of registers related to filter and route tables is computed - * by a macro that is supplied a parameter "er". The "er" represents an - * endpoint ID for filters, or a route ID for routes. For filters, the - * endpoint ID must be less than IPA_ENDPOINT_MAX, but is further restricted - * because not all endpoints support filtering. For routes, the route ID - * must be less than IPA_ROUTE_MAX. - * - * The offset of registers related to resource types is computed by a macro - * that is supplied a parameter "rt". The "rt" represents a resource type, - * which is a member of the ipa_resource_type_src enumerated type for - * source endpoint resources or the ipa_resource_type_dst enumerated type - * for destination endpoint resources. - * - * Some registers encode multiple fields within them. For these, each field - * has a symbol below defining a field mask that encodes both the position - * and width of the field within its register. - * - * In some cases, different versions of IPA hardware use different offset or - * field mask values. In such cases an inline_function(ipa) is used rather - * than a MACRO to define the offset or field mask to use. - * - * Finally, some registers hold bitmasks representing endpoints. In such - * cases the @available field in the @ipa structure defines the "full" set - * of valid bits for the register. + * Some registers encode multiple fields within them. Each field in + * such a register has a unique identifier (from an enumerated type). + * The position and width of the fields in a register are defined by + * an array of field masks, indexed by field ID. Two functions are + * used to access register fields; both take an ipa_reg structure as + * argument. To encode a value to be represented in a register field, + * the value and field ID are passed to ipa_reg_encode(). To extract + * a value encoded in a register field, the field ID is passed to + * ipa_reg_decode(). In addition, for single-bit fields, ipa_reg_bit() + * can be used to either encode the bit value, or to generate a mask + * used to extract the bit value. */ -#define IPA_REG_COMP_CFG_OFFSET 0x0000003c -/* The next field is not supported for IPA v4.0+, not present for IPA v4.5+ */ -#define ENABLE_FMASK GENMASK(0, 0) -/* The next field is present for IPA v4.7+ */ -#define RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS_FMASK GENMASK(0, 0) -#define GSI_SNOC_BYPASS_DIS_FMASK GENMASK(1, 1) -#define GEN_QMB_0_SNOC_BYPASS_DIS_FMASK GENMASK(2, 2) -#define GEN_QMB_1_SNOC_BYPASS_DIS_FMASK GENMASK(3, 3) -/* The next field is not present for IPA v4.5+ */ -#define IPA_DCMP_FAST_CLK_EN_FMASK GENMASK(4, 4) -/* The next twelve fields are present for IPA v4.0+ */ -#define IPA_QMB_SELECT_CONS_EN_FMASK GENMASK(5, 5) -#define IPA_QMB_SELECT_PROD_EN_FMASK GENMASK(6, 6) -#define GSI_MULTI_INORDER_RD_DIS_FMASK GENMASK(7, 7) -#define GSI_MULTI_INORDER_WR_DIS_FMASK GENMASK(8, 8) -#define GEN_QMB_0_MULTI_INORDER_RD_DIS_FMASK GENMASK(9, 9) -#define GEN_QMB_1_MULTI_INORDER_RD_DIS_FMASK GENMASK(10, 10) -#define GEN_QMB_0_MULTI_INORDER_WR_DIS_FMASK GENMASK(11, 11) -#define GEN_QMB_1_MULTI_INORDER_WR_DIS_FMASK GENMASK(12, 12) -#define GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS_FMASK GENMASK(13, 13) -#define GSI_SNOC_CNOC_LOOP_PROT_DISABLE_FMASK GENMASK(14, 14) -#define GSI_MULTI_AXI_MASTERS_DIS_FMASK GENMASK(15, 15) -#define IPA_QMB_SELECT_GLOBAL_EN_FMASK GENMASK(16, 16) -/* The next five fields are present for IPA v4.9+ */ -#define QMB_RAM_RD_CACHE_DISABLE_FMASK GENMASK(19, 19) -#define GENQMB_AOOOWR_FMASK GENMASK(20, 20) -#define IF_OUT_OF_BUF_STOP_RESET_MASK_EN_FMASK GENMASK(21, 21) -#define GEN_QMB_1_DYNAMIC_ASIZE_FMASK GENMASK(30, 30) -#define GEN_QMB_0_DYNAMIC_ASIZE_FMASK GENMASK(31, 31) - -/* Encoded value for COMP_CFG register ATOMIC_FETCHER_ARB_LOCK_DIS field */ -static inline u32 arbitration_lock_disable_encoded(enum ipa_version version, - u32 mask) -{ - WARN_ON(version < IPA_VERSION_4_0); +/* enum ipa_reg_id - IPA register IDs */ +enum ipa_reg_id { + COMP_CFG, + CLKON_CFG, + ROUTE, + SHARED_MEM_SIZE, + QSB_MAX_WRITES, + QSB_MAX_READS, + FILT_ROUT_HASH_EN, + FILT_ROUT_HASH_FLUSH, + STATE_AGGR_ACTIVE, + IPA_BCR, /* Not IPA v4.5+ */ + LOCAL_PKT_PROC_CNTXT, + AGGR_FORCE_CLOSE, + COUNTER_CFG, /* Not IPA v4.5+ */ + IPA_TX_CFG, /* IPA v3.5+ */ + FLAVOR_0, /* IPA v3.5+ */ + IDLE_INDICATION_CFG, /* IPA v3.5+ */ + QTIME_TIMESTAMP_CFG, /* IPA v4.5+ */ + TIMERS_XO_CLK_DIV_CFG, /* IPA v4.5+ */ + TIMERS_PULSE_GRAN_CFG, /* IPA v4.5+ */ + SRC_RSRC_GRP_01_RSRC_TYPE, + SRC_RSRC_GRP_23_RSRC_TYPE, + SRC_RSRC_GRP_45_RSRC_TYPE, /* Not IPA v3.5+, IPA v4.5 */ + SRC_RSRC_GRP_67_RSRC_TYPE, /* Not IPA v3.5+ */ + DST_RSRC_GRP_01_RSRC_TYPE, + DST_RSRC_GRP_23_RSRC_TYPE, + DST_RSRC_GRP_45_RSRC_TYPE, /* Not IPA v3.5+, IPA v4.5 */ + DST_RSRC_GRP_67_RSRC_TYPE, /* Not IPA v3.5+ */ + ENDP_INIT_CTRL, /* Not IPA v4.2+ for TX, not IPA v4.0+ for RX */ + ENDP_INIT_CFG, + ENDP_INIT_NAT, /* TX only */ + ENDP_INIT_HDR, + ENDP_INIT_HDR_EXT, + ENDP_INIT_HDR_METADATA_MASK, /* RX only */ + ENDP_INIT_MODE, /* TX only */ + ENDP_INIT_AGGR, + ENDP_INIT_HOL_BLOCK_EN, /* RX only */ + ENDP_INIT_HOL_BLOCK_TIMER, /* RX only */ + ENDP_INIT_DEAGGR, /* TX only */ + ENDP_INIT_RSRC_GRP, + ENDP_INIT_SEQ, /* TX only */ + ENDP_STATUS, + ENDP_FILTER_ROUTER_HSH_CFG, /* Not IPA v4.2 */ + /* The IRQ registers are only used for GSI_EE_AP */ + IPA_IRQ_STTS, + IPA_IRQ_EN, + IPA_IRQ_CLR, + IPA_IRQ_UC, + IRQ_SUSPEND_INFO, + IRQ_SUSPEND_EN, /* IPA v3.1+ */ + IRQ_SUSPEND_CLR, /* IPA v3.1+ */ + IPA_REG_ID_COUNT, /* Last; not an ID */ +}; - if (version < IPA_VERSION_4_9) - return u32_encode_bits(mask, GENMASK(20, 17)); +/** + * struct ipa_reg - An IPA register descriptor + * @offset: Register offset relative to base of the "ipa-reg" memory + * @stride: Distance between two instances, if parameterized + * @fcount: Number of entries in the @fmask array + * @fmask: Array of mask values defining position and width of fields + * @name: Upper-case name of the IPA register + */ +struct ipa_reg { + u32 offset; + u32 stride; + u32 fcount; + const u32 *fmask; /* BIT(nr) or GENMASK(h, l) */ + const char *name; +}; - if (version == IPA_VERSION_4_9) - return u32_encode_bits(mask, GENMASK(24, 22)); +/* Helper macro for defining "simple" (non-parameterized) registers */ +#define IPA_REG(__NAME, __reg_id, __offset) \ + IPA_REG_STRIDE(__NAME, __reg_id, __offset, 0) - return u32_encode_bits(mask, GENMASK(23, 22)); -} +/* Helper macro for defining parameterized registers, specifying stride */ +#define IPA_REG_STRIDE(__NAME, __reg_id, __offset, __stride) \ + static const struct ipa_reg ipa_reg_ ## __reg_id = { \ + .name = #__NAME, \ + .offset = __offset, \ + .stride = __stride, \ + } -/* Encoded value for COMP_CFG register FULL_FLUSH_WAIT_RS_CLOSURE_EN field */ -static inline u32 full_flush_rsc_closure_en_encoded(enum ipa_version version, - bool enable) -{ - u32 val = enable ? 1 : 0; +#define IPA_REG_FIELDS(__NAME, __name, __offset) \ + IPA_REG_STRIDE_FIELDS(__NAME, __name, __offset, 0) - WARN_ON(version < IPA_VERSION_4_5); +#define IPA_REG_STRIDE_FIELDS(__NAME, __name, __offset, __stride) \ + static const struct ipa_reg ipa_reg_ ## __name = { \ + .name = #__NAME, \ + .offset = __offset, \ + .stride = __stride, \ + .fcount = ARRAY_SIZE(ipa_reg_ ## __name ## _fmask), \ + .fmask = ipa_reg_ ## __name ## _fmask, \ + } - if (version == IPA_VERSION_4_5 || version == IPA_VERSION_4_7) - return u32_encode_bits(val, GENMASK(21, 21)); +/** + * struct ipa_regs - Description of registers supported by hardware + * @reg_count: Number of registers in the @reg[] array + * @reg: Array of register descriptors + */ +struct ipa_regs { + u32 reg_count; + const struct ipa_reg **reg; +}; - return u32_encode_bits(val, GENMASK(17, 17)); -} +/* COMP_CFG register */ +enum ipa_reg_comp_cfg_field_id { + COMP_CFG_ENABLE, /* Not IPA v4.0+ */ + RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS, /* IPA v4.7+ */ + GSI_SNOC_BYPASS_DIS, + GEN_QMB_0_SNOC_BYPASS_DIS, + GEN_QMB_1_SNOC_BYPASS_DIS, + IPA_DCMP_FAST_CLK_EN, /* Not IPA v4.5+ */ + IPA_QMB_SELECT_CONS_EN, /* IPA v4.0+ */ + IPA_QMB_SELECT_PROD_EN, /* IPA v4.0+ */ + GSI_MULTI_INORDER_RD_DIS, /* IPA v4.0+ */ + GSI_MULTI_INORDER_WR_DIS, /* IPA v4.0+ */ + GEN_QMB_0_MULTI_INORDER_RD_DIS, /* IPA v4.0+ */ + GEN_QMB_1_MULTI_INORDER_RD_DIS, /* IPA v4.0+ */ + GEN_QMB_0_MULTI_INORDER_WR_DIS, /* IPA v4.0+ */ + GEN_QMB_1_MULTI_INORDER_WR_DIS, /* IPA v4.0+ */ + GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS, /* IPA v4.0+ */ + GSI_SNOC_CNOC_LOOP_PROT_DISABLE, /* IPA v4.0+ */ + GSI_MULTI_AXI_MASTERS_DIS, /* IPA v4.0+ */ + IPA_QMB_SELECT_GLOBAL_EN, /* IPA v4.0+ */ + QMB_RAM_RD_CACHE_DISABLE, /* IPA v4.9+ */ + GENQMB_AOOOWR, /* IPA v4.9+ */ + IF_OUT_OF_BUF_STOP_RESET_MASK_EN, /* IPA v4.9+ */ + GEN_QMB_1_DYNAMIC_ASIZE, /* IPA v4.9+ */ + GEN_QMB_0_DYNAMIC_ASIZE, /* IPA v4.9+ */ + ATOMIC_FETCHER_ARB_LOCK_DIS, /* IPA v4.0+ */ + FULL_FLUSH_WAIT_RS_CLOSURE_EN, /* IPA v4.5+ */ +}; -#define IPA_REG_CLKON_CFG_OFFSET 0x00000044 -#define RX_FMASK GENMASK(0, 0) -#define PROC_FMASK GENMASK(1, 1) -#define TX_WRAPPER_FMASK GENMASK(2, 2) -#define MISC_FMASK GENMASK(3, 3) -#define RAM_ARB_FMASK GENMASK(4, 4) -#define FTCH_HPS_FMASK GENMASK(5, 5) -#define FTCH_DPS_FMASK GENMASK(6, 6) -#define HPS_FMASK GENMASK(7, 7) -#define DPS_FMASK GENMASK(8, 8) -#define RX_HPS_CMDQS_FMASK GENMASK(9, 9) -#define HPS_DPS_CMDQS_FMASK GENMASK(10, 10) -#define DPS_TX_CMDQS_FMASK GENMASK(11, 11) -#define RSRC_MNGR_FMASK GENMASK(12, 12) -#define CTX_HANDLER_FMASK GENMASK(13, 13) -#define ACK_MNGR_FMASK GENMASK(14, 14) -#define D_DCPH_FMASK GENMASK(15, 15) -#define H_DCPH_FMASK GENMASK(16, 16) -/* The next field is not present for IPA v4.5+ */ -#define DCMP_FMASK GENMASK(17, 17) -/* The next three fields are present for IPA v3.5+ */ -#define NTF_TX_CMDQS_FMASK GENMASK(18, 18) -#define TX_0_FMASK GENMASK(19, 19) -#define TX_1_FMASK GENMASK(20, 20) -/* The next field is present for IPA v3.5.1+ */ -#define FNR_FMASK GENMASK(21, 21) -/* The next eight fields are present for IPA v4.0+ */ -#define QSB2AXI_CMDQ_L_FMASK GENMASK(22, 22) -#define AGGR_WRAPPER_FMASK GENMASK(23, 23) -#define RAM_SLAVEWAY_FMASK GENMASK(24, 24) -#define QMB_FMASK GENMASK(25, 25) -#define WEIGHT_ARB_FMASK GENMASK(26, 26) -#define GSI_IF_FMASK GENMASK(27, 27) -#define GLOBAL_FMASK GENMASK(28, 28) -#define GLOBAL_2X_CLK_FMASK GENMASK(29, 29) -/* The next field is present for IPA v4.5+ */ -#define DPL_FIFO_FMASK GENMASK(30, 30) -/* The next field is present for IPA v4.7+ */ -#define DRBIP_FMASK GENMASK(31, 31) - -#define IPA_REG_ROUTE_OFFSET 0x00000048 -#define ROUTE_DIS_FMASK GENMASK(0, 0) -#define ROUTE_DEF_PIPE_FMASK GENMASK(5, 1) -#define ROUTE_DEF_HDR_TABLE_FMASK GENMASK(6, 6) -#define ROUTE_DEF_HDR_OFST_FMASK GENMASK(16, 7) -#define ROUTE_FRAG_DEF_PIPE_FMASK GENMASK(21, 17) -#define ROUTE_DEF_RETAIN_HDR_FMASK GENMASK(24, 24) - -#define IPA_REG_SHARED_MEM_SIZE_OFFSET 0x00000054 -#define SHARED_MEM_SIZE_FMASK GENMASK(15, 0) -#define SHARED_MEM_BADDR_FMASK GENMASK(31, 16) - -#define IPA_REG_QSB_MAX_WRITES_OFFSET 0x00000074 -#define GEN_QMB_0_MAX_WRITES_FMASK GENMASK(3, 0) -#define GEN_QMB_1_MAX_WRITES_FMASK GENMASK(7, 4) - -#define IPA_REG_QSB_MAX_READS_OFFSET 0x00000078 -#define GEN_QMB_0_MAX_READS_FMASK GENMASK(3, 0) -#define GEN_QMB_1_MAX_READS_FMASK GENMASK(7, 4) -/* The next two fields are present for IPA v4.0+ */ -#define GEN_QMB_0_MAX_READS_BEATS_FMASK GENMASK(23, 16) -#define GEN_QMB_1_MAX_READS_BEATS_FMASK GENMASK(31, 24) - -static inline u32 ipa_reg_filt_rout_hash_en_offset(enum ipa_version version) -{ - if (version < IPA_VERSION_4_0) - return 0x000008c; +/* CLKON_CFG register */ +enum ipa_reg_clkon_cfg_field_id { + CLKON_RX, + CLKON_PROC, + TX_WRAPPER, + CLKON_MISC, + RAM_ARB, + FTCH_HPS, + FTCH_DPS, + CLKON_HPS, + CLKON_DPS, + RX_HPS_CMDQS, + HPS_DPS_CMDQS, + DPS_TX_CMDQS, + RSRC_MNGR, + CTX_HANDLER, + ACK_MNGR, + D_DCPH, + H_DCPH, + CLKON_DCMP, /* IPA v4.5+ */ + NTF_TX_CMDQS, /* IPA v3.5+ */ + CLKON_TX_0, /* IPA v3.5+ */ + CLKON_TX_1, /* IPA v3.5+ */ + CLKON_FNR, /* IPA v3.5.1+ */ + QSB2AXI_CMDQ_L, /* IPA v4.0+ */ + AGGR_WRAPPER, /* IPA v4.0+ */ + RAM_SLAVEWAY, /* IPA v4.0+ */ + CLKON_QMB, /* IPA v4.0+ */ + WEIGHT_ARB, /* IPA v4.0+ */ + GSI_IF, /* IPA v4.0+ */ + CLKON_GLOBAL, /* IPA v4.0+ */ + GLOBAL_2X_CLK, /* IPA v4.0+ */ + DPL_FIFO, /* IPA v4.5+ */ + DRBIP, /* IPA v4.7+ */ +}; - return 0x0000148; -} +/* ROUTE register */ +enum ipa_reg_route_field_id { + ROUTE_DIS, + ROUTE_DEF_PIPE, + ROUTE_DEF_HDR_TABLE, + ROUTE_DEF_HDR_OFST, + ROUTE_FRAG_DEF_PIPE, + ROUTE_DEF_RETAIN_HDR, +}; -static inline u32 ipa_reg_filt_rout_hash_flush_offset(enum ipa_version version) -{ - if (version < IPA_VERSION_4_0) - return 0x0000090; +/* SHARED_MEM_SIZE register */ +enum ipa_reg_shared_mem_size_field_id { + MEM_SIZE, + MEM_BADDR, +}; - return 0x000014c; -} +/* QSB_MAX_WRITES register */ +enum ipa_reg_qsb_max_writes_field_id { + GEN_QMB_0_MAX_WRITES, + GEN_QMB_1_MAX_WRITES, +}; -/* The next four fields are used for the hash enable and flush registers */ -#define IPV6_ROUTER_HASH_FMASK GENMASK(0, 0) -#define IPV6_FILTER_HASH_FMASK GENMASK(4, 4) -#define IPV4_ROUTER_HASH_FMASK GENMASK(8, 8) -#define IPV4_FILTER_HASH_FMASK GENMASK(12, 12) +/* QSB_MAX_READS register */ +enum ipa_reg_qsb_max_reads_field_id { + GEN_QMB_0_MAX_READS, + GEN_QMB_1_MAX_READS, + GEN_QMB_0_MAX_READS_BEATS, /* IPA v4.0+ */ + GEN_QMB_1_MAX_READS_BEATS, /* IPA v4.0+ */ +}; -/* ipa->available defines the valid bits in the STATE_AGGR_ACTIVE register */ -static inline u32 ipa_reg_state_aggr_active_offset(enum ipa_version version) -{ - if (version < IPA_VERSION_4_0) - return 0x0000010c; +/* FILT_ROUT_HASH_EN and FILT_ROUT_HASH_FLUSH registers */ +enum ipa_reg_rout_hash_field_id { + IPV6_ROUTER_HASH, + IPV6_FILTER_HASH, + IPV4_ROUTER_HASH, + IPV4_FILTER_HASH, +}; - return 0x000000b4; -} +/* BCR register */ +enum ipa_bcr_compat { + BCR_CMDQ_L_LACK_ONE_ENTRY = 0x0, /* Not IPA v4.2+ */ + BCR_TX_NOT_USING_BRESP = 0x1, /* Not IPA v4.2+ */ + BCR_TX_SUSPEND_IRQ_ASSERT_ONCE = 0x2, /* Not IPA v4.0+ */ + BCR_SUSPEND_L2_IRQ = 0x3, /* Not IPA v4.2+ */ + BCR_HOLB_DROP_L2_IRQ = 0x4, /* Not IPA v4.2+ */ + BCR_DUAL_TX = 0x5, /* IPA v3.5+ */ + BCR_ENABLE_FILTER_DATA_CACHE = 0x6, /* IPA v3.5+ */ + BCR_NOTIF_PRIORITY_OVER_ZLT = 0x7, /* IPA v3.5+ */ + BCR_FILTER_PREFETCH_EN = 0x8, /* IPA v3.5+ */ + BCR_ROUTER_PREFETCH_EN = 0x9, /* IPA v3.5+ */ +}; -/* The next register is not present for IPA v4.5+ */ -#define IPA_REG_BCR_OFFSET 0x000001d0 -/* The next two fields are not present for IPA v4.2+ */ -#define BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK GENMASK(0, 0) -#define BCR_TX_NOT_USING_BRESP_FMASK GENMASK(1, 1) -/* The next field is invalid for IPA v4.0+ */ -#define BCR_TX_SUSPEND_IRQ_ASSERT_ONCE_FMASK GENMASK(2, 2) -/* The next two fields are not present for IPA v4.2+ */ -#define BCR_SUSPEND_L2_IRQ_FMASK GENMASK(3, 3) -#define BCR_HOLB_DROP_L2_IRQ_FMASK GENMASK(4, 4) -/* The next five fields are present for IPA v3.5+ */ -#define BCR_DUAL_TX_FMASK GENMASK(5, 5) -#define BCR_ENABLE_FILTER_DATA_CACHE_FMASK GENMASK(6, 6) -#define BCR_NOTIF_PRIORITY_OVER_ZLT_FMASK GENMASK(7, 7) -#define BCR_FILTER_PREFETCH_EN_FMASK GENMASK(8, 8) -#define BCR_ROUTER_PREFETCH_EN_FMASK GENMASK(9, 9) - -/* The value of the next register must be a multiple of 8 (bottom 3 bits 0) */ -#define IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET 0x000001e8 - -/* Encoded value for LOCAL_PKT_PROC_CNTXT register BASE_ADDR field */ -static inline u32 proc_cntxt_base_addr_encoded(enum ipa_version version, - u32 addr) -{ - if (version < IPA_VERSION_4_5) - return u32_encode_bits(addr, GENMASK(16, 0)); +/* LOCAL_PKT_PROC_CNTXT register */ +enum ipa_reg_local_pkt_proc_cntxt_field_id { + IPA_BASE_ADDR, +}; - return u32_encode_bits(addr, GENMASK(17, 0)); -} +/* COUNTER_CFG register */ +enum ipa_reg_counter_cfg_field_id { + EOT_COAL_GRANULARITY, /* Not v3.5+ */ + AGGR_GRANULARITY, +}; -/* ipa->available defines the valid bits in the AGGR_FORCE_CLOSE register */ -#define IPA_REG_AGGR_FORCE_CLOSE_OFFSET 0x000001ec - -/* The next register is not present for IPA v4.5+ */ -#define IPA_REG_COUNTER_CFG_OFFSET 0x000001f0 -/* The next field is not present for IPA v3.5+ */ -#define EOT_COAL_GRANULARITY GENMASK(3, 0) -#define AGGR_GRANULARITY_FMASK GENMASK(8, 4) - -/* The next register is present for IPA v3.5+ */ -#define IPA_REG_TX_CFG_OFFSET 0x000001fc -/* The next three fields are not present for IPA v4.0+ */ -#define TX0_PREFETCH_DISABLE_FMASK GENMASK(0, 0) -#define TX1_PREFETCH_DISABLE_FMASK GENMASK(1, 1) -#define PREFETCH_ALMOST_EMPTY_SIZE_FMASK GENMASK(4, 2) -/* The next six fields are present for IPA v4.0+ */ -#define PREFETCH_ALMOST_EMPTY_SIZE_TX0_FMASK GENMASK(5, 2) -#define DMAW_SCND_OUTSD_PRED_THRESHOLD_FMASK GENMASK(9, 6) -#define DMAW_SCND_OUTSD_PRED_EN_FMASK GENMASK(10, 10) -#define DMAW_MAX_BEATS_256_DIS_FMASK GENMASK(11, 11) -#define PA_MASK_EN_FMASK GENMASK(12, 12) -#define PREFETCH_ALMOST_EMPTY_SIZE_TX1_FMASK GENMASK(16, 13) -/* The next field is present for IPA v4.5+ */ -#define DUAL_TX_ENABLE_FMASK GENMASK(17, 17) -/* The next field is present for IPA v4.2+, but not IPA v4.5 */ -#define SSPND_PA_NO_START_STATE_FMASK GENMASK(18, 18) -/* The next field is present for IPA v4.2 only */ -#define SSPND_PA_NO_BQ_STATE_FMASK GENMASK(19, 19) - -/* The next register is present for IPA v3.5+ */ -#define IPA_REG_FLAVOR_0_OFFSET 0x00000210 -#define IPA_MAX_PIPES_FMASK GENMASK(3, 0) -#define IPA_MAX_CONS_PIPES_FMASK GENMASK(12, 8) -#define IPA_MAX_PROD_PIPES_FMASK GENMASK(20, 16) -#define IPA_PROD_LOWEST_FMASK GENMASK(27, 24) - -/* The next register is present for IPA v3.5+ */ -static inline u32 ipa_reg_idle_indication_cfg_offset(enum ipa_version version) -{ - if (version >= IPA_VERSION_4_2) - return 0x00000240; +/* IPA_TX_CFG register */ +enum ipa_reg_ipa_tx_cfg_field_id { + TX0_PREFETCH_DISABLE, /* Not v4.0+ */ + TX1_PREFETCH_DISABLE, /* Not v4.0+ */ + PREFETCH_ALMOST_EMPTY_SIZE, /* Not v4.0+ */ + PREFETCH_ALMOST_EMPTY_SIZE_TX0, /* v4.0+ */ + DMAW_SCND_OUTSD_PRED_THRESHOLD, /* v4.0+ */ + DMAW_SCND_OUTSD_PRED_EN, /* v4.0+ */ + DMAW_MAX_BEATS_256_DIS, /* v4.0+ */ + PA_MASK_EN, /* v4.0+ */ + PREFETCH_ALMOST_EMPTY_SIZE_TX1, /* v4.0+ */ + DUAL_TX_ENABLE, /* v4.5+ */ + SSPND_PA_NO_START_STATE, /* v4,2+, not v4.5 */ + SSPND_PA_NO_BQ_STATE, /* v4.2 only */ +}; - return 0x00000220; -} +/* FLAVOR_0 register */ +enum ipa_reg_flavor_0_field_id { + MAX_PIPES, + MAX_CONS_PIPES, + MAX_PROD_PIPES, + PROD_LOWEST, +}; + +/* IDLE_INDICATION_CFG register */ +enum ipa_reg_idle_indication_cfg_field_id { + ENTER_IDLE_DEBOUNCE_THRESH, + CONST_NON_IDLE_ENABLE, +}; + +/* QTIME_TIMESTAMP_CFG register */ +enum ipa_reg_qtime_timestamp_cfg_field_id { + DPL_TIMESTAMP_LSB, + DPL_TIMESTAMP_SEL, + TAG_TIMESTAMP_LSB, + NAT_TIMESTAMP_LSB, +}; -#define ENTER_IDLE_DEBOUNCE_THRESH_FMASK GENMASK(15, 0) -#define CONST_NON_IDLE_ENABLE_FMASK GENMASK(16, 16) - -/* The next register is present for IPA v4.5+ */ -#define IPA_REG_QTIME_TIMESTAMP_CFG_OFFSET 0x0000024c -#define DPL_TIMESTAMP_LSB_FMASK GENMASK(4, 0) -#define DPL_TIMESTAMP_SEL_FMASK GENMASK(7, 7) -#define TAG_TIMESTAMP_LSB_FMASK GENMASK(12, 8) -#define NAT_TIMESTAMP_LSB_FMASK GENMASK(20, 16) - -/* The next register is present for IPA v4.5+ */ -#define IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET 0x00000250 -#define DIV_VALUE_FMASK GENMASK(8, 0) -#define DIV_ENABLE_FMASK GENMASK(31, 31) - -/* The next register is present for IPA v4.5+ */ -#define IPA_REG_TIMERS_PULSE_GRAN_CFG_OFFSET 0x00000254 -#define GRAN_0_FMASK GENMASK(2, 0) -#define GRAN_1_FMASK GENMASK(5, 3) -#define GRAN_2_FMASK GENMASK(8, 6) -/* Values for GRAN_x fields of TIMERS_PULSE_GRAN_CFG */ +/* TIMERS_XO_CLK_DIV_CFG register */ +enum ipa_reg_timers_xo_clk_div_cfg_field_id { + DIV_VALUE, + DIV_ENABLE, +}; + +/* TIMERS_PULSE_GRAN_CFG register */ +enum ipa_reg_timers_pulse_gran_cfg_field_id { + PULSE_GRAN_0, + PULSE_GRAN_1, + PULSE_GRAN_2, +}; + +/* Values for IPA_GRAN_x fields of TIMERS_PULSE_GRAN_CFG */ enum ipa_pulse_gran { IPA_GRAN_10_US = 0x0, IPA_GRAN_20_US = 0x1, @@ -325,267 +347,160 @@ enum ipa_pulse_gran { IPA_GRAN_655350_US = 0x7, }; -/* Not all of the following are present (depends on IPA version) */ -#define IPA_REG_SRC_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000400 + 0x0020 * (rt)) -#define IPA_REG_SRC_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000404 + 0x0020 * (rt)) -#define IPA_REG_SRC_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000408 + 0x0020 * (rt)) -#define IPA_REG_SRC_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(rt) \ - (0x0000040c + 0x0020 * (rt)) -#define IPA_REG_DST_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000500 + 0x0020 * (rt)) -#define IPA_REG_DST_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000504 + 0x0020 * (rt)) -#define IPA_REG_DST_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(rt) \ - (0x00000508 + 0x0020 * (rt)) -#define IPA_REG_DST_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(rt) \ - (0x0000050c + 0x0020 * (rt)) -/* The next four fields are used for all resource group registers */ -#define X_MIN_LIM_FMASK GENMASK(5, 0) -#define X_MAX_LIM_FMASK GENMASK(13, 8) -/* The next two fields are not always present (if resource count is odd) */ -#define Y_MIN_LIM_FMASK GENMASK(21, 16) -#define Y_MAX_LIM_FMASK GENMASK(29, 24) - -#define IPA_REG_ENDP_INIT_CTRL_N_OFFSET(ep) \ - (0x00000800 + 0x0070 * (ep)) -/* Valid only for RX (IPA producer) endpoints (do not use for IPA v4.0+) */ -#define ENDP_SUSPEND_FMASK GENMASK(0, 0) -/* Valid only for TX (IPA consumer) endpoints */ -#define ENDP_DELAY_FMASK GENMASK(1, 1) - -#define IPA_REG_ENDP_INIT_CFG_N_OFFSET(ep) \ - (0x00000808 + 0x0070 * (ep)) -#define FRAG_OFFLOAD_EN_FMASK GENMASK(0, 0) -#define CS_OFFLOAD_EN_FMASK GENMASK(2, 1) -#define CS_METADATA_HDR_OFFSET_FMASK GENMASK(6, 3) -#define CS_GEN_QMB_MASTER_SEL_FMASK GENMASK(8, 8) +/* {SRC,DST}_RSRC_GRP_{01,23,45,67}_RSRC_TYPE registers */ +enum ipa_reg_rsrc_grp_rsrc_type_field_id { + X_MIN_LIM, + X_MAX_LIM, + Y_MIN_LIM, + Y_MAX_LIM, +}; + +/* ENDP_INIT_CTRL register */ +enum ipa_reg_endp_init_ctrl_field_id { + ENDP_SUSPEND, /* Not v4.0+ */ + ENDP_DELAY, /* Not v4.2+ */ +}; + +/* ENDP_INIT_CFG register */ +enum ipa_reg_endp_init_cfg_field_id { + FRAG_OFFLOAD_EN, + CS_OFFLOAD_EN, + CS_METADATA_HDR_OFFSET, + CS_GEN_QMB_MASTER_SEL, +}; /** enum ipa_cs_offload_en - ENDP_INIT_CFG register CS_OFFLOAD_EN field value */ enum ipa_cs_offload_en { - IPA_CS_OFFLOAD_NONE = 0x0, - IPA_CS_OFFLOAD_UL = 0x1, /* Before IPA v4.5 (TX) */ - IPA_CS_OFFLOAD_DL = 0x2, /* Before IPA v4.5 (RX) */ - IPA_CS_OFFLOAD_INLINE = 0x1, /* IPA v4.5 (TX and RX) */ + IPA_CS_OFFLOAD_NONE = 0x0, + IPA_CS_OFFLOAD_UL /* TX */ = 0x1, /* Not IPA v4.5+ */ + IPA_CS_OFFLOAD_DL /* RX */ = 0x2, /* Not IPA v4.5+ */ + IPA_CS_OFFLOAD_INLINE /* TX and RX */ = 0x1, /* IPA v4.5+ */ }; -/* Valid only for TX (IPA consumer) endpoints */ -#define IPA_REG_ENDP_INIT_NAT_N_OFFSET(ep) \ - (0x0000080c + 0x0070 * (ep)) -#define NAT_EN_FMASK GENMASK(1, 0) +/* ENDP_INIT_NAT register */ +enum ipa_reg_endp_init_nat_field_id { + NAT_EN, +}; /** enum ipa_nat_en - ENDP_INIT_NAT register NAT_EN field value */ enum ipa_nat_en { - IPA_NAT_BYPASS = 0x0, - IPA_NAT_SRC = 0x1, - IPA_NAT_DST = 0x2, -}; - -#define IPA_REG_ENDP_INIT_HDR_N_OFFSET(ep) \ - (0x00000810 + 0x0070 * (ep)) -#define HDR_LEN_FMASK GENMASK(5, 0) -#define HDR_OFST_METADATA_VALID_FMASK GENMASK(6, 6) -#define HDR_OFST_METADATA_FMASK GENMASK(12, 7) -#define HDR_ADDITIONAL_CONST_LEN_FMASK GENMASK(18, 13) -#define HDR_OFST_PKT_SIZE_VALID_FMASK GENMASK(19, 19) -#define HDR_OFST_PKT_SIZE_FMASK GENMASK(25, 20) -/* The next field is not present for IPA v4.9+ */ -#define HDR_A5_MUX_FMASK GENMASK(26, 26) -#define HDR_LEN_INC_DEAGG_HDR_FMASK GENMASK(27, 27) -/* The next field is not present for IPA v4.5+ */ -#define HDR_METADATA_REG_VALID_FMASK GENMASK(28, 28) -/* The next two fields are present for IPA v4.5+ */ -#define HDR_LEN_MSB_FMASK GENMASK(29, 28) -#define HDR_OFST_METADATA_MSB_FMASK GENMASK(31, 30) - -/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */ -static inline u32 ipa_header_size_encoded(enum ipa_version version, - u32 header_size) -{ - u32 size = header_size & field_mask(HDR_LEN_FMASK); - u32 val; - - val = u32_encode_bits(size, HDR_LEN_FMASK); - if (version < IPA_VERSION_4_5) { - WARN_ON(header_size != size); - return val; - } - - /* IPA v4.5 adds a few more most-significant bits */ - size = header_size >> hweight32(HDR_LEN_FMASK); - val |= u32_encode_bits(size, HDR_LEN_MSB_FMASK); - - return val; -} - -/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */ -static inline u32 ipa_metadata_offset_encoded(enum ipa_version version, - u32 offset) -{ - u32 off = offset & field_mask(HDR_OFST_METADATA_FMASK); - u32 val; - - val = u32_encode_bits(off, HDR_OFST_METADATA_FMASK); - if (version < IPA_VERSION_4_5) { - WARN_ON(offset != off); - return val; - } + IPA_NAT_BYPASS = 0x0, + IPA_NAT_SRC = 0x1, + IPA_NAT_DST = 0x2, +}; - /* IPA v4.5 adds a few more most-significant bits */ - off = offset >> hweight32(HDR_OFST_METADATA_FMASK); - val |= u32_encode_bits(off, HDR_OFST_METADATA_MSB_FMASK); +/* ENDP_INIT_HDR register */ +enum ipa_reg_endp_init_hdr_field_id { + HDR_LEN, + HDR_OFST_METADATA_VALID, + HDR_OFST_METADATA, + HDR_ADDITIONAL_CONST_LEN, + HDR_OFST_PKT_SIZE_VALID, + HDR_OFST_PKT_SIZE, + HDR_A5_MUX, /* Not v4.9+ */ + HDR_LEN_INC_DEAGG_HDR, + HDR_METADATA_REG_VALID, /* Not v4.5+ */ + HDR_LEN_MSB, /* v4.5+ */ + HDR_OFST_METADATA_MSB, /* v4.5+ */ +}; - return val; -} +/* ENDP_INIT_HDR_EXT register */ +enum ipa_reg_endp_init_hdr_ext_field_id { + HDR_ENDIANNESS, + HDR_TOTAL_LEN_OR_PAD_VALID, + HDR_TOTAL_LEN_OR_PAD, + HDR_PAYLOAD_LEN_INC_PADDING, + HDR_TOTAL_LEN_OR_PAD_OFFSET, + HDR_PAD_TO_ALIGNMENT, + HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB, /* v4.5+ */ + HDR_OFST_PKT_SIZE_MSB, /* v4.5+ */ + HDR_ADDITIONAL_CONST_LEN_MSB, /* v4.5+ */ +}; -#define IPA_REG_ENDP_INIT_HDR_EXT_N_OFFSET(ep) \ - (0x00000814 + 0x0070 * (ep)) -#define HDR_ENDIANNESS_FMASK GENMASK(0, 0) -#define HDR_TOTAL_LEN_OR_PAD_VALID_FMASK GENMASK(1, 1) -#define HDR_TOTAL_LEN_OR_PAD_FMASK GENMASK(2, 2) -#define HDR_PAYLOAD_LEN_INC_PADDING_FMASK GENMASK(3, 3) -#define HDR_TOTAL_LEN_OR_PAD_OFFSET_FMASK GENMASK(9, 4) -#define HDR_PAD_TO_ALIGNMENT_FMASK GENMASK(13, 10) -/* The next three fields are present for IPA v4.5+ */ -#define HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB_FMASK GENMASK(17, 16) -#define HDR_OFST_PKT_SIZE_MSB_FMASK GENMASK(19, 18) -#define HDR_ADDITIONAL_CONST_LEN_MSB_FMASK GENMASK(21, 20) - -/* Valid only for RX (IPA producer) endpoints */ -#define IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(rxep) \ - (0x00000818 + 0x0070 * (rxep)) - -/* Valid only for TX (IPA consumer) endpoints */ -#define IPA_REG_ENDP_INIT_MODE_N_OFFSET(txep) \ - (0x00000820 + 0x0070 * (txep)) -#define MODE_FMASK GENMASK(2, 0) -/* The next field is present for IPA v4.5+ */ -#define DCPH_ENABLE_FMASK GENMASK(3, 3) -#define DEST_PIPE_INDEX_FMASK GENMASK(8, 4) -#define BYTE_THRESHOLD_FMASK GENMASK(27, 12) -#define PIPE_REPLICATION_EN_FMASK GENMASK(28, 28) -#define PAD_EN_FMASK GENMASK(29, 29) -/* The next field is not present for IPA v4.5+ */ -#define HDR_FTCH_DISABLE_FMASK GENMASK(30, 30) -/* The next field is present for IPA v4.9+ */ -#define DRBIP_ACL_ENABLE GENMASK(30, 30) +/* ENDP_INIT_MODE register */ +enum ipa_reg_endp_init_mode_field_id { + ENDP_MODE, + DCPH_ENABLE, /* v4.5+ */ + DEST_PIPE_INDEX, + BYTE_THRESHOLD, + PIPE_REPLICATION_EN, + PAD_EN, + HDR_FTCH_DISABLE, /* v4.5+ */ + DRBIP_ACL_ENABLE, /* v4.9+ */ +}; /** enum ipa_mode - ENDP_INIT_MODE register MODE field value */ enum ipa_mode { - IPA_BASIC = 0x0, - IPA_ENABLE_FRAMING_HDLC = 0x1, - IPA_ENABLE_DEFRAMING_HDLC = 0x2, - IPA_DMA = 0x3, + IPA_BASIC = 0x0, + IPA_ENABLE_FRAMING_HDLC = 0x1, + IPA_ENABLE_DEFRAMING_HDLC = 0x2, + IPA_DMA = 0x3, }; -#define IPA_REG_ENDP_INIT_AGGR_N_OFFSET(ep) \ - (0x00000824 + 0x0070 * (ep)) -#define AGGR_EN_FMASK GENMASK(1, 0) -#define AGGR_TYPE_FMASK GENMASK(4, 2) - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_byte_limit_fmask(bool legacy) -{ - return legacy ? GENMASK(9, 5) : GENMASK(10, 5); -} - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_time_limit_fmask(bool legacy) -{ - return legacy ? GENMASK(14, 10) : GENMASK(16, 12); -} - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_pkt_limit_fmask(bool legacy) -{ - return legacy ? GENMASK(20, 15) : GENMASK(22, 17); -} - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_sw_eof_active_fmask(bool legacy) -{ - return legacy ? GENMASK(21, 21) : GENMASK(23, 23); -} - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_force_close_fmask(bool legacy) -{ - return legacy ? GENMASK(22, 22) : GENMASK(24, 24); -} - -/* The legacy value is used for IPA hardware before IPA v4.5 */ -static inline u32 aggr_hard_byte_limit_enable_fmask(bool legacy) -{ - return legacy ? GENMASK(24, 24) : GENMASK(26, 26); -} - -/* The next field is present for IPA v4.5+ */ -#define AGGR_GRAN_SEL_FMASK GENMASK(27, 27) +/* ENDP_INIT_AGGR register */ +enum ipa_reg_endp_init_aggr_field_id { + AGGR_EN, + AGGR_TYPE, + BYTE_LIMIT, + TIME_LIMIT, + PKT_LIMIT, + SW_EOF_ACTIVE, + FORCE_CLOSE, + HARD_BYTE_LIMIT_EN, + AGGR_GRAN_SEL, +}; /** enum ipa_aggr_en - ENDP_INIT_AGGR register AGGR_EN field value */ enum ipa_aggr_en { - IPA_BYPASS_AGGR = 0x0, /* (TX, RX) */ - IPA_ENABLE_AGGR = 0x1, /* (RX) */ - IPA_ENABLE_DEAGGR = 0x2, /* (TX) */ + IPA_BYPASS_AGGR /* TX and RX */ = 0x0, + IPA_ENABLE_AGGR /* RX */ = 0x1, + IPA_ENABLE_DEAGGR /* TX */ = 0x2, }; /** enum ipa_aggr_type - ENDP_INIT_AGGR register AGGR_TYPE field value */ enum ipa_aggr_type { - IPA_MBIM_16 = 0x0, - IPA_HDLC = 0x1, - IPA_TLP = 0x2, - IPA_RNDIS = 0x3, - IPA_GENERIC = 0x4, - IPA_COALESCE = 0x5, - IPA_QCMAP = 0x6, -}; - -/* Valid only for RX (IPA producer) endpoints */ -#define IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(rxep) \ - (0x0000082c + 0x0070 * (rxep)) -#define HOL_BLOCK_EN_FMASK GENMASK(0, 0) - -/* Valid only for RX (IPA producer) endpoints */ -#define IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(rxep) \ - (0x00000830 + 0x0070 * (rxep)) -/* The next two fields are present for IPA v4.2 only */ -#define BASE_VALUE_FMASK GENMASK(4, 0) -#define SCALE_FMASK GENMASK(12, 8) -/* The next two fields are present for IPA v4.5 */ -#define TIME_LIMIT_FMASK GENMASK(4, 0) -#define GRAN_SEL_FMASK GENMASK(8, 8) - -/* Valid only for TX (IPA consumer) endpoints */ -#define IPA_REG_ENDP_INIT_DEAGGR_N_OFFSET(txep) \ - (0x00000834 + 0x0070 * (txep)) -#define DEAGGR_HDR_LEN_FMASK GENMASK(5, 0) -#define SYSPIPE_ERR_DETECTION_FMASK GENMASK(6, 6) -#define PACKET_OFFSET_VALID_FMASK GENMASK(7, 7) -#define PACKET_OFFSET_LOCATION_FMASK GENMASK(13, 8) -#define IGNORE_MIN_PKT_ERR_FMASK GENMASK(14, 14) -#define MAX_PACKET_LEN_FMASK GENMASK(31, 16) - -#define IPA_REG_ENDP_INIT_RSRC_GRP_N_OFFSET(ep) \ - (0x00000838 + 0x0070 * (ep)) -/* Encoded value for ENDP_INIT_RSRC_GRP register RSRC_GRP field */ -static inline u32 rsrc_grp_encoded(enum ipa_version version, u32 rsrc_grp) -{ - if (version < IPA_VERSION_3_5 || version == IPA_VERSION_4_5) - return u32_encode_bits(rsrc_grp, GENMASK(2, 0)); + IPA_MBIM_16 = 0x0, + IPA_HDLC = 0x1, + IPA_TLP = 0x2, + IPA_RNDIS = 0x3, + IPA_GENERIC = 0x4, + IPA_COALESCE = 0x5, + IPA_QCMAP = 0x6, +}; - if (version == IPA_VERSION_4_2 || version == IPA_VERSION_4_7) - return u32_encode_bits(rsrc_grp, GENMASK(0, 0)); +/* ENDP_INIT_HOL_BLOCK_EN register */ +enum ipa_reg_endp_init_hol_block_en_field_id { + HOL_BLOCK_EN, +}; - return u32_encode_bits(rsrc_grp, GENMASK(1, 0)); -} +/* ENDP_INIT_HOL_BLOCK_TIMER register */ +enum ipa_reg_endp_init_hol_block_timer_field_id { + TIMER_BASE_VALUE, /* Not v4.5+ */ + TIMER_SCALE, /* v4.2 only */ + TIMER_LIMIT, /* v4.5+ */ + TIMER_GRAN_SEL, /* v4.5+ */ +}; -/* Valid only for TX (IPA consumer) endpoints */ -#define IPA_REG_ENDP_INIT_SEQ_N_OFFSET(txep) \ - (0x0000083c + 0x0070 * (txep)) -#define SEQ_TYPE_FMASK GENMASK(7, 0) -#define SEQ_REP_TYPE_FMASK GENMASK(15, 8) +/* ENDP_INIT_DEAGGR register */ +enum ipa_reg_endp_deaggr_field_id { + DEAGGR_HDR_LEN, + SYSPIPE_ERR_DETECTION, + PACKET_OFFSET_VALID, + PACKET_OFFSET_LOCATION, + IGNORE_MIN_PKT_ERR, + MAX_PACKET_LEN, +}; + +/* ENDP_INIT_RSRC_GRP register */ +enum ipa_reg_endp_init_rsrc_grp_field_id { + ENDP_RSRC_GRP, +}; + +/* ENDP_INIT_SEQ register */ +enum ipa_reg_endp_init_seq_field_id { + SEQ_TYPE, + SEQ_REP_TYPE, /* Not v4.5+ */ +}; /** * enum ipa_seq_type - HPS and DPS sequencer type @@ -629,76 +544,36 @@ enum ipa_seq_rep_type { IPA_SEQ_REP_DMA_PARSER = 0x08, }; -#define IPA_REG_ENDP_STATUS_N_OFFSET(ep) \ - (0x00000840 + 0x0070 * (ep)) -#define STATUS_EN_FMASK GENMASK(0, 0) -#define STATUS_ENDP_FMASK GENMASK(5, 1) -/* The next field is not present for IPA v4.5+ */ -#define STATUS_LOCATION_FMASK GENMASK(8, 8) -/* The next field is present for IPA v4.0+ */ -#define STATUS_PKT_SUPPRESS_FMASK GENMASK(9, 9) - -/* The next register is not present for IPA v4.2 (which no hashing support) */ -#define IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(er) \ - (0x0000085c + 0x0070 * (er)) -#define FILTER_HASH_MSK_SRC_ID_FMASK GENMASK(0, 0) -#define FILTER_HASH_MSK_SRC_IP_FMASK GENMASK(1, 1) -#define FILTER_HASH_MSK_DST_IP_FMASK GENMASK(2, 2) -#define FILTER_HASH_MSK_SRC_PORT_FMASK GENMASK(3, 3) -#define FILTER_HASH_MSK_DST_PORT_FMASK GENMASK(4, 4) -#define FILTER_HASH_MSK_PROTOCOL_FMASK GENMASK(5, 5) -#define FILTER_HASH_MSK_METADATA_FMASK GENMASK(6, 6) -#define IPA_REG_ENDP_FILTER_HASH_MSK_ALL GENMASK(6, 0) - -#define ROUTER_HASH_MSK_SRC_ID_FMASK GENMASK(16, 16) -#define ROUTER_HASH_MSK_SRC_IP_FMASK GENMASK(17, 17) -#define ROUTER_HASH_MSK_DST_IP_FMASK GENMASK(18, 18) -#define ROUTER_HASH_MSK_SRC_PORT_FMASK GENMASK(19, 19) -#define ROUTER_HASH_MSK_DST_PORT_FMASK GENMASK(20, 20) -#define ROUTER_HASH_MSK_PROTOCOL_FMASK GENMASK(21, 21) -#define ROUTER_HASH_MSK_METADATA_FMASK GENMASK(22, 22) -#define IPA_REG_ENDP_ROUTER_HASH_MSK_ALL GENMASK(22, 16) - -static inline u32 ipa_reg_irq_stts_ee_n_offset(enum ipa_version version, - u32 ee) -{ - if (version < IPA_VERSION_4_9) - return 0x00003008 + 0x1000 * ee; - - return 0x00004008 + 0x1000 * ee; -} - -static inline u32 ipa_reg_irq_stts_offset(enum ipa_version version) -{ - return ipa_reg_irq_stts_ee_n_offset(version, GSI_EE_AP); -} - -static inline u32 ipa_reg_irq_en_ee_n_offset(enum ipa_version version, u32 ee) -{ - if (version < IPA_VERSION_4_9) - return 0x0000300c + 0x1000 * ee; - - return 0x0000400c + 0x1000 * ee; -} - -static inline u32 ipa_reg_irq_en_offset(enum ipa_version version) -{ - return ipa_reg_irq_en_ee_n_offset(version, GSI_EE_AP); -} - -static inline u32 ipa_reg_irq_clr_ee_n_offset(enum ipa_version version, u32 ee) -{ - if (version < IPA_VERSION_4_9) - return 0x00003010 + 0x1000 * ee; - - return 0x00004010 + 0x1000 * ee; -} +/* ENDP_STATUS register */ +enum ipa_reg_endp_status_field_id { + STATUS_EN, + STATUS_ENDP, + STATUS_LOCATION, /* Not v4.5+ */ + STATUS_PKT_SUPPRESS, /* v4.0+ */ +}; -static inline u32 ipa_reg_irq_clr_offset(enum ipa_version version) -{ - return ipa_reg_irq_clr_ee_n_offset(version, GSI_EE_AP); -} +/* ENDP_FILTER_ROUTER_HSH_CFG register */ +enum ipa_reg_endp_filter_router_hsh_cfg_field_id { + FILTER_HASH_MSK_SRC_ID, + FILTER_HASH_MSK_SRC_IP, + FILTER_HASH_MSK_DST_IP, + FILTER_HASH_MSK_SRC_PORT, + FILTER_HASH_MSK_DST_PORT, + FILTER_HASH_MSK_PROTOCOL, + FILTER_HASH_MSK_METADATA, + FILTER_HASH_MSK_ALL, /* Bitwise OR of the above 6 fields */ + + ROUTER_HASH_MSK_SRC_ID, + ROUTER_HASH_MSK_SRC_IP, + ROUTER_HASH_MSK_DST_IP, + ROUTER_HASH_MSK_SRC_PORT, + ROUTER_HASH_MSK_DST_PORT, + ROUTER_HASH_MSK_PROTOCOL, + ROUTER_HASH_MSK_METADATA, + ROUTER_HASH_MSK_ALL, /* Bitwise OR of the above 6 fields */ +}; +/* IPA_IRQ_STTS, IPA_IRQ_EN, and IPA_IRQ_CLR registers */ /** * enum ipa_irq_id - Bit positions representing type of IPA IRQ * @IPA_IRQ_UC_0: Microcontroller event interrupt @@ -774,74 +649,82 @@ enum ipa_irq_id { IPA_IRQ_COUNT, /* Last; not an id */ }; -static inline u32 ipa_reg_irq_uc_ee_n_offset(enum ipa_version version, u32 ee) -{ - if (version < IPA_VERSION_4_9) - return 0x0000301c + 0x1000 * ee; +/* IPA_IRQ_UC register */ +enum ipa_reg_ipa_irq_uc_field_id { + UC_INTR, +}; - return 0x0000401c + 0x1000 * ee; -} +extern const struct ipa_regs ipa_regs_v3_1; +extern const struct ipa_regs ipa_regs_v3_5_1; +extern const struct ipa_regs ipa_regs_v4_2; +extern const struct ipa_regs ipa_regs_v4_5; +extern const struct ipa_regs ipa_regs_v4_9; +extern const struct ipa_regs ipa_regs_v4_11; -static inline u32 ipa_reg_irq_uc_offset(enum ipa_version version) +/* Return the field mask for a field in a register */ +static inline u32 ipa_reg_fmask(const struct ipa_reg *reg, u32 field_id) { - return ipa_reg_irq_uc_ee_n_offset(version, GSI_EE_AP); -} + if (!reg || WARN_ON(field_id >= reg->fcount)) + return 0; -#define UC_INTR_FMASK GENMASK(0, 0) + return reg->fmask[field_id]; +} -/* ipa->available defines the valid bits in the SUSPEND_INFO register */ -static inline u32 -ipa_reg_irq_suspend_info_ee_n_offset(enum ipa_version version, u32 ee) +/* Return the mask for a single-bit field in a register */ +static inline u32 ipa_reg_bit(const struct ipa_reg *reg, u32 field_id) { - if (version == IPA_VERSION_3_0) - return 0x00003098 + 0x1000 * ee; + u32 fmask = ipa_reg_fmask(reg, field_id); - if (version < IPA_VERSION_4_9) - return 0x00003030 + 0x1000 * ee; + WARN_ON(!is_power_of_2(fmask)); - return 0x00004030 + 0x1000 * ee; + return fmask; } +/* Encode a value into the given field of a register */ static inline u32 -ipa_reg_irq_suspend_info_offset(enum ipa_version version) +ipa_reg_encode(const struct ipa_reg *reg, u32 field_id, u32 val) { - return ipa_reg_irq_suspend_info_ee_n_offset(version, GSI_EE_AP); -} + u32 fmask = ipa_reg_fmask(reg, field_id); -/* ipa->available defines the valid bits in the SUSPEND_EN register */ -static inline u32 -ipa_reg_irq_suspend_en_ee_n_offset(enum ipa_version version, u32 ee) -{ - WARN_ON(version == IPA_VERSION_3_0); + if (!fmask) + return 0; - if (version < IPA_VERSION_4_9) - return 0x00003034 + 0x1000 * ee; + val <<= __ffs(fmask); + if (WARN_ON(val & ~fmask)) + return 0; - return 0x00004034 + 0x1000 * ee; + return val; } +/* Given a register value, decode (extract) the value in the given field */ static inline u32 -ipa_reg_irq_suspend_en_offset(enum ipa_version version) +ipa_reg_decode(const struct ipa_reg *reg, u32 field_id, u32 val) { - return ipa_reg_irq_suspend_en_ee_n_offset(version, GSI_EE_AP); + u32 fmask = ipa_reg_fmask(reg, field_id); + + return fmask ? (val & fmask) >> __ffs(fmask) : 0; } -/* ipa->available defines the valid bits in the SUSPEND_CLR register */ -static inline u32 -ipa_reg_irq_suspend_clr_ee_n_offset(enum ipa_version version, u32 ee) +/* Return the maximum value representable by the given field; always 2^n - 1 */ +static inline u32 ipa_reg_field_max(const struct ipa_reg *reg, u32 field_id) { - WARN_ON(version == IPA_VERSION_3_0); + u32 fmask = ipa_reg_fmask(reg, field_id); - if (version < IPA_VERSION_4_9) - return 0x00003038 + 0x1000 * ee; + return fmask ? fmask >> __ffs(fmask) : 0; +} + +const struct ipa_reg *ipa_reg(struct ipa *ipa, enum ipa_reg_id reg_id); - return 0x00004038 + 0x1000 * ee; +/* Returns 0 for NULL reg; warning will have already been issued */ +static inline u32 ipa_reg_offset(const struct ipa_reg *reg) +{ + return reg ? reg->offset : 0; } -static inline u32 -ipa_reg_irq_suspend_clr_offset(enum ipa_version version) +/* Returns 0 for NULL reg; warning will have already been issued */ +static inline u32 ipa_reg_n_offset(const struct ipa_reg *reg, u32 n) { - return ipa_reg_irq_suspend_clr_ee_n_offset(version, GSI_EE_AP); + return reg ? reg->offset + n * reg->stride : 0; } int ipa_reg_init(struct ipa *ipa); diff --git a/drivers/net/ipa/ipa_resource.c b/drivers/net/ipa/ipa_resource.c index 06cec71993823deeadf98756bc95e4b43ae63d01..a257f0e5e361883b841916448cc31b8d435a6975 100644 --- a/drivers/net/ipa/ipa_resource.c +++ b/drivers/net/ipa/ipa_resource.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -69,20 +69,21 @@ static bool ipa_resource_limits_valid(struct ipa *ipa, } static void -ipa_resource_config_common(struct ipa *ipa, u32 offset, +ipa_resource_config_common(struct ipa *ipa, u32 resource_type, + const struct ipa_reg *reg, const struct ipa_resource_limits *xlimits, const struct ipa_resource_limits *ylimits) { u32 val; - val = u32_encode_bits(xlimits->min, X_MIN_LIM_FMASK); - val |= u32_encode_bits(xlimits->max, X_MAX_LIM_FMASK); + val = ipa_reg_encode(reg, X_MIN_LIM, xlimits->min); + val |= ipa_reg_encode(reg, X_MAX_LIM, xlimits->max); if (ylimits) { - val |= u32_encode_bits(ylimits->min, Y_MIN_LIM_FMASK); - val |= u32_encode_bits(ylimits->max, Y_MAX_LIM_FMASK); + val |= ipa_reg_encode(reg, Y_MIN_LIM, ylimits->min); + val |= ipa_reg_encode(reg, Y_MAX_LIM, ylimits->max); } - iowrite32(val, ipa->reg_virt + offset); + iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, resource_type)); } static void ipa_resource_config_src(struct ipa *ipa, u32 resource_type, @@ -91,34 +92,35 @@ static void ipa_resource_config_src(struct ipa *ipa, u32 resource_type, u32 group_count = data->rsrc_group_src_count; const struct ipa_resource_limits *ylimits; const struct ipa_resource *resource; - u32 offset; + const struct ipa_reg *reg; resource = &data->resource_src[resource_type]; - offset = IPA_REG_SRC_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, SRC_RSRC_GRP_01_RSRC_TYPE); ylimits = group_count == 1 ? NULL : &resource->limits[1]; - ipa_resource_config_common(ipa, offset, &resource->limits[0], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[0], ylimits); if (group_count < 3) return; - offset = IPA_REG_SRC_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, SRC_RSRC_GRP_23_RSRC_TYPE); ylimits = group_count == 3 ? NULL : &resource->limits[3]; - ipa_resource_config_common(ipa, offset, &resource->limits[2], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[2], ylimits); if (group_count < 5) return; - offset = IPA_REG_SRC_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, SRC_RSRC_GRP_45_RSRC_TYPE); ylimits = group_count == 5 ? NULL : &resource->limits[5]; - ipa_resource_config_common(ipa, offset, &resource->limits[4], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[4], ylimits); if (group_count < 7) return; - offset = IPA_REG_SRC_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, SRC_RSRC_GRP_67_RSRC_TYPE); ylimits = group_count == 7 ? NULL : &resource->limits[7]; - ipa_resource_config_common(ipa, offset, &resource->limits[6], ylimits); + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[6], ylimits); } static void ipa_resource_config_dst(struct ipa *ipa, u32 resource_type, @@ -127,34 +129,35 @@ static void ipa_resource_config_dst(struct ipa *ipa, u32 resource_type, u32 group_count = data->rsrc_group_dst_count; const struct ipa_resource_limits *ylimits; const struct ipa_resource *resource; - u32 offset; + const struct ipa_reg *reg; resource = &data->resource_dst[resource_type]; - offset = IPA_REG_DST_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, DST_RSRC_GRP_01_RSRC_TYPE); ylimits = group_count == 1 ? NULL : &resource->limits[1]; - ipa_resource_config_common(ipa, offset, &resource->limits[0], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[0], ylimits); if (group_count < 3) return; - offset = IPA_REG_DST_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, DST_RSRC_GRP_23_RSRC_TYPE); ylimits = group_count == 3 ? NULL : &resource->limits[3]; - ipa_resource_config_common(ipa, offset, &resource->limits[2], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[2], ylimits); if (group_count < 5) return; - offset = IPA_REG_DST_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, DST_RSRC_GRP_45_RSRC_TYPE); ylimits = group_count == 5 ? NULL : &resource->limits[5]; - ipa_resource_config_common(ipa, offset, &resource->limits[4], ylimits); - + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[4], ylimits); if (group_count < 7) return; - offset = IPA_REG_DST_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(resource_type); + reg = ipa_reg(ipa, DST_RSRC_GRP_67_RSRC_TYPE); ylimits = group_count == 7 ? NULL : &resource->limits[7]; - ipa_resource_config_common(ipa, offset, &resource->limits[6], ylimits); + ipa_resource_config_common(ipa, resource_type, reg, + &resource->limits[6], ylimits); } /* Configure resources; there is no ipa_resource_deconfig() */ diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index 2112336120391c21b26a6bf5df5cf334deee1ee2..5620dc271fac33faac4b1e9a6bd35f55224d1975 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #include diff --git a/drivers/net/ipa/ipa_smp2p.h b/drivers/net/ipa/ipa_smp2p.h index 59cee31a738365dda7c50649a0f70fa892bc40d9..9b969b03d1a4bbdc67cc58b9ed5a453eccadb0f0 100644 --- a/drivers/net/ipa/ipa_smp2p.h +++ b/drivers/net/ipa/ipa_smp2p.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_SMP2P_H_ #define _IPA_SMP2P_H_ diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c index c0c8641cdd14a2be7c7eb656e42641e8885abf4c..5cbc15a971f9dbd95d644eeef0c7d199cc0aa2bf 100644 --- a/drivers/net/ipa/ipa_sysfs.c +++ b/drivers/net/ipa/ipa_sysfs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -/* Copyright (C) 2021 Linaro Ltd. */ +/* Copyright (C) 2021-2022 Linaro Ltd. */ #include #include diff --git a/drivers/net/ipa/ipa_sysfs.h b/drivers/net/ipa/ipa_sysfs.h index 4a3ffd1e4e3fb9b13e62e93f2ef8b0e474500894..58ba22810bab424926461289ea88f9a9f0b1bda6 100644 --- a/drivers/net/ipa/ipa_sysfs.h +++ b/drivers/net/ipa/ipa_sysfs.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_SYSFS_H_ #define _IPA_SYSFS_H_ diff --git a/drivers/net/ipa/ipa_table.c b/drivers/net/ipa/ipa_table.c index 2f5a58bfc529a3148bce114b1fcf4f32f0db07bc..510ff2dc8999a8a6dc724a483ff35b2755525f32 100644 --- a/drivers/net/ipa/ipa_table.c +++ b/drivers/net/ipa/ipa_table.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2021 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -108,8 +108,6 @@ /* Assignment of route table entries to the modem and AP */ #define IPA_ROUTE_MODEM_MIN 0 -#define IPA_ROUTE_MODEM_COUNT 8 - #define IPA_ROUTE_AP_MIN IPA_ROUTE_MODEM_COUNT #define IPA_ROUTE_AP_COUNT \ (IPA_ROUTE_COUNT_MAX - IPA_ROUTE_MODEM_COUNT) @@ -386,8 +384,9 @@ void ipa_table_reset(struct ipa *ipa, bool modem) int ipa_table_hash_flush(struct ipa *ipa) { - u32 offset = ipa_reg_filt_rout_hash_flush_offset(ipa->version); + const struct ipa_reg *reg; struct gsi_trans *trans; + u32 offset; u32 val; if (!ipa_table_hash_support(ipa)) @@ -399,8 +398,13 @@ int ipa_table_hash_flush(struct ipa *ipa) return -EBUSY; } - val = IPV4_FILTER_HASH_FMASK | IPV6_FILTER_HASH_FMASK; - val |= IPV6_ROUTER_HASH_FMASK | IPV4_ROUTER_HASH_FMASK; + reg = ipa_reg(ipa, FILT_ROUT_HASH_FLUSH); + offset = ipa_reg_offset(reg); + + val = ipa_reg_bit(reg, IPV6_ROUTER_HASH); + val |= ipa_reg_bit(reg, IPV6_FILTER_HASH); + val |= ipa_reg_bit(reg, IPV4_ROUTER_HASH); + val |= ipa_reg_bit(reg, IPV4_FILTER_HASH); ipa_cmd_register_write_add(trans, offset, val, val, false); @@ -518,15 +522,18 @@ int ipa_table_setup(struct ipa *ipa) static void ipa_filter_tuple_zero(struct ipa_endpoint *endpoint) { u32 endpoint_id = endpoint->endpoint_id; + struct ipa *ipa = endpoint->ipa; + const struct ipa_reg *reg; u32 offset; u32 val; - offset = IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(endpoint_id); + reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG); + offset = ipa_reg_n_offset(reg, endpoint_id); val = ioread32(endpoint->ipa->reg_virt + offset); /* Zero all filter-related fields, preserving the rest */ - u32p_replace_bits(&val, 0, IPA_REG_ENDP_FILTER_HASH_MSK_ALL); + val &= ~ipa_reg_fmask(reg, FILTER_HASH_MSK_ALL); iowrite32(val, endpoint->ipa->reg_virt + offset); } @@ -567,13 +574,17 @@ static bool ipa_route_id_modem(u32 route_id) */ static void ipa_route_tuple_zero(struct ipa *ipa, u32 route_id) { - u32 offset = IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(route_id); + const struct ipa_reg *reg; + u32 offset; u32 val; + reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG); + offset = ipa_reg_n_offset(reg, route_id); + val = ioread32(ipa->reg_virt + offset); /* Zero all route-related fields, preserving the rest */ - u32p_replace_bits(&val, 0, IPA_REG_ENDP_ROUTER_HASH_MSK_ALL); + val &= ~ipa_reg_fmask(reg, ROUTER_HASH_MSK_ALL); iowrite32(val, ipa->reg_virt + offset); } diff --git a/drivers/net/ipa/ipa_table.h b/drivers/net/ipa/ipa_table.h index b6a9a0d79d68e9669b50ca9079bbf6e0d8ad1df5..395189f75d784ab126ef051f3429c4487ca3d36e 100644 --- a/drivers/net/ipa/ipa_table.h +++ b/drivers/net/ipa/ipa_table.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2021 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_TABLE_H_ #define _IPA_TABLE_H_ @@ -13,6 +13,9 @@ struct ipa; /* The maximum number of filter table entries (IPv4, IPv6; hashed or not) */ #define IPA_FILTER_COUNT_MAX 14 +/* The number of route table entries allotted to the modem */ +#define IPA_ROUTE_MODEM_COUNT 8 + /* The maximum number of route table entries (IPv4, IPv6; hashed or not) */ #define IPA_ROUTE_COUNT_MAX 15 diff --git a/drivers/net/ipa/ipa_uc.c b/drivers/net/ipa/ipa_uc.c index fe11910518d95f1ee352eb1ce1cf8cdb60d91c45..f0ee472810153dbce71250b81db4e01640bfbe78 100644 --- a/drivers/net/ipa/ipa_uc.c +++ b/drivers/net/ipa/ipa_uc.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2018-2020 Linaro Ltd. + * Copyright (C) 2018-2022 Linaro Ltd. */ #include @@ -222,7 +222,7 @@ void ipa_uc_power(struct ipa *ipa) static void send_uc_command(struct ipa *ipa, u32 command, u32 command_param) { struct ipa_uc_mem_area *shared = ipa_uc_shared(ipa); - u32 offset; + const struct ipa_reg *reg; u32 val; /* Fill in the command data */ @@ -233,9 +233,10 @@ static void send_uc_command(struct ipa *ipa, u32 command, u32 command_param) shared->response_param = 0; /* Use an interrupt to tell the microcontroller the command is ready */ - val = u32_encode_bits(1, UC_INTR_FMASK); - offset = ipa_reg_irq_uc_offset(ipa->version); - iowrite32(val, ipa->reg_virt + offset); + reg = ipa_reg(ipa, IPA_IRQ_UC); + val = ipa_reg_bit(reg, UC_INTR); + + iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); } /* Tell the microcontroller the AP is shutting down */ diff --git a/drivers/net/ipa/ipa_uc.h b/drivers/net/ipa/ipa_uc.h index 23847f934d64e97d78c78adc2abf3c16b2f8c5e4..8514096e6f36f277e9e8555dd17cd91bca67662f 100644 --- a/drivers/net/ipa/ipa_uc.h +++ b/drivers/net/ipa/ipa_uc.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_UC_H_ #define _IPA_UC_H_ diff --git a/drivers/net/ipa/ipa_version.h b/drivers/net/ipa/ipa_version.h index 6c16c895d84296733730857cdea39564939e31a4..7870e0cc3d7c913a58c1503fb51aa01701f96e22 100644 --- a/drivers/net/ipa/ipa_version.h +++ b/drivers/net/ipa/ipa_version.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. - * Copyright (C) 2019-2020 Linaro Ltd. + * Copyright (C) 2019-2022 Linaro Ltd. */ #ifndef _IPA_VERSION_H_ #define _IPA_VERSION_H_ @@ -19,10 +19,10 @@ * @IPA_VERSION_4_7: IPA version 4.7/GSI version 2.7 * @IPA_VERSION_4_9: IPA version 4.9/GSI version 2.9 * @IPA_VERSION_4_11: IPA version 4.11/GSI version 2.11 (2.1.1) + * @IPA_VERSION_COUNT: Number of defined IPA versions * * Defines the version of IPA (and GSI) hardware present on the platform. - * Please update ipa_version_valid() and ipa_version_string() whenever a - * new version is added. + * Please update ipa_version_string() whenever a new version is added. */ enum ipa_version { IPA_VERSION_3_0, @@ -36,6 +36,30 @@ enum ipa_version { IPA_VERSION_4_7, IPA_VERSION_4_9, IPA_VERSION_4_11, + IPA_VERSION_COUNT, /* Last; not a version */ +}; + +static inline bool ipa_version_supported(enum ipa_version version) +{ + switch (version) { + case IPA_VERSION_3_1: + case IPA_VERSION_3_5_1: + case IPA_VERSION_4_2: + case IPA_VERSION_4_5: + case IPA_VERSION_4_9: + case IPA_VERSION_4_11: + return true; + default: + return false; + } +} + +/* Execution environment IDs */ +enum gsi_ee_id { + GSI_EE_AP = 0x0, + GSI_EE_MODEM = 0x1, + GSI_EE_UC = 0x2, + GSI_EE_TZ = 0x3, }; #endif /* _IPA_VERSION_H_ */ diff --git a/drivers/net/ipa/reg/ipa_reg-v3.1.c b/drivers/net/ipa/reg/ipa_reg-v3.1.c new file mode 100644 index 0000000000000000000000000000000000000000..116b27717e3d785542480cd76ab4b2fd8bb9e403 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v3.1.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + [COMP_CFG_ENABLE] = BIT(0), + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + [IPA_DCMP_FAST_CLK_EN] = BIT(4), + /* Bits 5-31 reserved */ +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x000008c); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x0000090); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x0000010c); + +IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(16, 0), + /* Bits 17-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_counter_cfg_fmask[] = { + [EOT_COAL_GRANULARITY] = GENMASK(3, 0), + [AGGR_GRANULARITY] = GENMASK(8, 4), + /* Bits 5-31 reserved */ +}; + +IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_45_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type, + 0x00000408, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_67_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_67_RSRC_TYPE, src_rsrc_grp_67_rsrc_type, + 0x0000040c, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_45_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type, + 0x00000508, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_67_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_67_RSRC_TYPE, dst_rsrc_grp_67_rsrc_type, + 0x0000050c, 0x0020); + +static const u32 ipa_reg_endp_init_ctrl_fmask[] = { + [ENDP_SUSPEND] = BIT(0), + [ENDP_DELAY] = BIT(1), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CTRL, endp_init_ctrl, 0x00000800, 0x0070); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + [HDR_A5_MUX] = BIT(26), + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_METADATA_REG_VALID] = BIT(28), + /* Bits 29-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + /* Bit 3 reserved */ + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + [HDR_FTCH_DISABLE] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(9, 5), + [TIME_LIMIT] = GENMASK(14, 10), + [PKT_LIMIT] = GENMASK(20, 15), + [SW_EOF_ACTIVE] = BIT(21), + [FORCE_CLOSE] = BIT(22), + /* Bit 23 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +/* Entire register is a tick count */ +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_BASE_VALUE] = GENMASK(31, 0), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = GENMASK(2, 0), + /* Bits 3-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + [SEQ_REP_TYPE] = GENMASK(15, 8), + /* Bits 16-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-7 reserved */ + [STATUS_LOCATION] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = { + [FILTER_HASH_MSK_SRC_ID] = BIT(0), + [FILTER_HASH_MSK_SRC_IP] = BIT(1), + [FILTER_HASH_MSK_DST_IP] = BIT(2), + [FILTER_HASH_MSK_SRC_PORT] = BIT(3), + [FILTER_HASH_MSK_DST_PORT] = BIT(4), + [FILTER_HASH_MSK_PROTOCOL] = BIT(5), + [FILTER_HASH_MSK_METADATA] = BIT(6), + [FILTER_HASH_MSK_ALL] = GENMASK(6, 0), + /* Bits 7-15 reserved */ + [ROUTER_HASH_MSK_SRC_ID] = BIT(16), + [ROUTER_HASH_MSK_SRC_IP] = BIT(17), + [ROUTER_HASH_MSK_DST_IP] = BIT(18), + [ROUTER_HASH_MSK_SRC_PORT] = BIT(19), + [ROUTER_HASH_MSK_DST_PORT] = BIT(20), + [ROUTER_HASH_MSK_PROTOCOL] = BIT(21), + [ROUTER_HASH_MSK_METADATA] = BIT(22), + [ROUTER_HASH_MSK_ALL] = GENMASK(22, 16), + /* Bits 23-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg, + 0x0000085c, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [IPA_BCR] = &ipa_reg_ipa_bcr, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [COUNTER_CFG] = &ipa_reg_counter_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [SRC_RSRC_GRP_45_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_45_rsrc_type, + [SRC_RSRC_GRP_67_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_67_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_45_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_45_rsrc_type, + [DST_RSRC_GRP_67_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_67_rsrc_type, + [ENDP_INIT_CTRL] = &ipa_reg_endp_init_ctrl, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [ENDP_FILTER_ROUTER_HSH_CFG] = &ipa_reg_endp_filter_router_hsh_cfg, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v3_1 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipa/reg/ipa_reg-v3.5.1.c b/drivers/net/ipa/reg/ipa_reg-v3.5.1.c new file mode 100644 index 0000000000000000000000000000000000000000..6e2f939b18f1984b7d966117fb0b33140564eab9 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v3.5.1.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + [COMP_CFG_ENABLE] = BIT(0), + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + [IPA_DCMP_FAST_CLK_EN] = BIT(4), + /* Bits 5-31 reserved */ +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + /* Bit 17 reserved */ + [NTF_TX_CMDQS] = BIT(18), + [CLKON_TX_0] = BIT(19), + [CLKON_TX_1] = BIT(20), + [CLKON_FNR] = BIT(21), + /* Bits 22-31 reserved */ +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x000008c); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x0000090); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x0000010c); + +IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(16, 0), + /* Bits 17-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_counter_cfg_fmask[] = { + /* Bits 0-3 reserved */ + [AGGR_GRANULARITY] = GENMASK(8, 4), + /* Bits 5-31 reserved */ +}; + +IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0); + +static const u32 ipa_reg_ipa_tx_cfg_fmask[] = { + [TX0_PREFETCH_DISABLE] = BIT(0), + [TX1_PREFETCH_DISABLE] = BIT(1), + [PREFETCH_ALMOST_EMPTY_SIZE] = GENMASK(4, 2), + /* Bits 5-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc); + +static const u32 ipa_reg_flavor_0_fmask[] = { + [MAX_PIPES] = GENMASK(3, 0), + /* Bits 4-7 reserved */ + [MAX_CONS_PIPES] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [MAX_PROD_PIPES] = GENMASK(20, 16), + /* Bits 21-23 reserved */ + [PROD_LOWEST] = GENMASK(27, 24), + /* Bits 28-31 reserved */ +}; + +IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210); + +static const u32 ipa_reg_idle_indication_cfg_fmask[] = { + [ENTER_IDLE_DEBOUNCE_THRESH] = GENMASK(15, 0), + [CONST_NON_IDLE_ENABLE] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000220); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_endp_init_ctrl_fmask[] = { + [ENDP_SUSPEND] = BIT(0), + [ENDP_DELAY] = BIT(1), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CTRL, endp_init_ctrl, 0x00000800, 0x0070); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + [HDR_A5_MUX] = BIT(26), + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_METADATA_REG_VALID] = BIT(28), + /* Bits 29-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + /* Bit 3 reserved */ + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + [HDR_FTCH_DISABLE] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(9, 5), + [TIME_LIMIT] = GENMASK(14, 10), + [PKT_LIMIT] = GENMASK(20, 15), + [SW_EOF_ACTIVE] = BIT(21), + [FORCE_CLOSE] = BIT(22), + /* Bit 23 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +/* Entire register is a tick count */ +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_BASE_VALUE] = GENMASK(31, 0), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + [SEQ_REP_TYPE] = GENMASK(15, 8), + /* Bits 16-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-7 reserved */ + [STATUS_LOCATION] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = { + [FILTER_HASH_MSK_SRC_ID] = BIT(0), + [FILTER_HASH_MSK_SRC_IP] = BIT(1), + [FILTER_HASH_MSK_DST_IP] = BIT(2), + [FILTER_HASH_MSK_SRC_PORT] = BIT(3), + [FILTER_HASH_MSK_DST_PORT] = BIT(4), + [FILTER_HASH_MSK_PROTOCOL] = BIT(5), + [FILTER_HASH_MSK_METADATA] = BIT(6), + [FILTER_HASH_MSK_ALL] = GENMASK(6, 0), + /* Bits 7-15 reserved */ + [ROUTER_HASH_MSK_SRC_ID] = BIT(16), + [ROUTER_HASH_MSK_SRC_IP] = BIT(17), + [ROUTER_HASH_MSK_DST_IP] = BIT(18), + [ROUTER_HASH_MSK_SRC_PORT] = BIT(19), + [ROUTER_HASH_MSK_DST_PORT] = BIT(20), + [ROUTER_HASH_MSK_PROTOCOL] = BIT(21), + [ROUTER_HASH_MSK_METADATA] = BIT(22), + [ROUTER_HASH_MSK_ALL] = GENMASK(22, 16), + /* Bits 23-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg, + 0x0000085c, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [IPA_BCR] = &ipa_reg_ipa_bcr, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [COUNTER_CFG] = &ipa_reg_counter_cfg, + [IPA_TX_CFG] = &ipa_reg_ipa_tx_cfg, + [FLAVOR_0] = &ipa_reg_flavor_0, + [IDLE_INDICATION_CFG] = &ipa_reg_idle_indication_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [ENDP_INIT_CTRL] = &ipa_reg_endp_init_ctrl, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [ENDP_FILTER_ROUTER_HSH_CFG] = &ipa_reg_endp_filter_router_hsh_cfg, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v3_5_1 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipa/reg/ipa_reg-v4.11.c b/drivers/net/ipa/reg/ipa_reg-v4.11.c new file mode 100644 index 0000000000000000000000000000000000000000..8fd36569bb9f8e7fc0c0df43dfc34d989f067b01 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v4.11.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + [RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS] = BIT(0), + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + /* Bit 4 reserved */ + [IPA_QMB_SELECT_CONS_EN] = BIT(5), + [IPA_QMB_SELECT_PROD_EN] = BIT(6), + [GSI_MULTI_INORDER_RD_DIS] = BIT(7), + [GSI_MULTI_INORDER_WR_DIS] = BIT(8), + [GEN_QMB_0_MULTI_INORDER_RD_DIS] = BIT(9), + [GEN_QMB_1_MULTI_INORDER_RD_DIS] = BIT(10), + [GEN_QMB_0_MULTI_INORDER_WR_DIS] = BIT(11), + [GEN_QMB_1_MULTI_INORDER_WR_DIS] = BIT(12), + [GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS] = BIT(13), + [GSI_SNOC_CNOC_LOOP_PROT_DISABLE] = BIT(14), + [GSI_MULTI_AXI_MASTERS_DIS] = BIT(15), + [IPA_QMB_SELECT_GLOBAL_EN] = BIT(16), + [FULL_FLUSH_WAIT_RS_CLOSURE_EN] = BIT(17), + /* Bit 18 reserved */ + [QMB_RAM_RD_CACHE_DISABLE] = BIT(19), + [GENQMB_AOOOWR] = BIT(20), + [IF_OUT_OF_BUF_STOP_RESET_MASK_EN] = BIT(21), + [ATOMIC_FETCHER_ARB_LOCK_DIS] = GENMASK(23, 22), + /* Bits 24-29 reserved */ + [GEN_QMB_1_DYNAMIC_ASIZE] = BIT(30), + [GEN_QMB_0_DYNAMIC_ASIZE] = BIT(31), +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + /* Bit 17 reserved */ + [NTF_TX_CMDQS] = BIT(18), + [CLKON_TX_0] = BIT(19), + [CLKON_TX_1] = BIT(20), + [CLKON_FNR] = BIT(21), + [QSB2AXI_CMDQ_L] = BIT(22), + [AGGR_WRAPPER] = BIT(23), + [RAM_SLAVEWAY] = BIT(24), + [CLKON_QMB] = BIT(25), + [WEIGHT_ARB] = BIT(26), + [GSI_IF] = BIT(27), + [CLKON_GLOBAL] = BIT(28), + [GLOBAL_2X_CLK] = BIT(29), + [DPL_FIFO] = BIT(30), + [DRBIP] = BIT(31), +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), + /* Bits 8-15 reserved */ + [GEN_QMB_0_MAX_READS_BEATS] = GENMASK(23, 16), + [GEN_QMB_1_MAX_READS_BEATS] = GENMASK(31, 24), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(17, 0), + /* Bits 18-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_ipa_tx_cfg_fmask[] = { + /* Bits 0-1 reserved */ + [PREFETCH_ALMOST_EMPTY_SIZE_TX0] = GENMASK(5, 2), + [DMAW_SCND_OUTSD_PRED_THRESHOLD] = GENMASK(9, 6), + [DMAW_SCND_OUTSD_PRED_EN] = BIT(10), + [DMAW_MAX_BEATS_256_DIS] = BIT(11), + [PA_MASK_EN] = BIT(12), + [PREFETCH_ALMOST_EMPTY_SIZE_TX1] = GENMASK(16, 13), + [DUAL_TX_ENABLE] = BIT(17), + [SSPND_PA_NO_START_STATE] = BIT(18), + /* Bits 19-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc); + +static const u32 ipa_reg_flavor_0_fmask[] = { + [MAX_PIPES] = GENMASK(4, 0), + /* Bits 5-7 reserved */ + [MAX_CONS_PIPES] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [MAX_PROD_PIPES] = GENMASK(20, 16), + /* Bits 21-23 reserved */ + [PROD_LOWEST] = GENMASK(27, 24), + /* Bits 28-31 reserved */ +}; + +IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210); + +static const u32 ipa_reg_idle_indication_cfg_fmask[] = { + [ENTER_IDLE_DEBOUNCE_THRESH] = GENMASK(15, 0), + [CONST_NON_IDLE_ENABLE] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240); + +static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = { + [DPL_TIMESTAMP_LSB] = GENMASK(4, 0), + /* Bits 5-6 reserved */ + [DPL_TIMESTAMP_SEL] = BIT(7), + [TAG_TIMESTAMP_LSB] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [NAT_TIMESTAMP_LSB] = GENMASK(20, 16), + /* Bits 21-31 reserved */ +}; + +IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c); + +static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = { + [DIV_VALUE] = GENMASK(8, 0), + /* Bits 9-30 reserved */ + [DIV_ENABLE] = BIT(31), +}; + +IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250); + +static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = { + [PULSE_GRAN_0] = GENMASK(2, 0), + [PULSE_GRAN_1] = GENMASK(5, 3), + [PULSE_GRAN_2] = GENMASK(8, 6), + /* Bits 9-31 reserved */ +}; + +IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + /* Bit 26 reserved */ + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_LEN_MSB] = GENMASK(29, 28), + [HDR_OFST_METADATA_MSB] = GENMASK(31, 30), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-15 reserved */ + [HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB] = GENMASK(17, 16), + [HDR_OFST_PKT_SIZE_MSB] = GENMASK(19, 18), + [HDR_ADDITIONAL_CONST_LEN_MSB] = GENMASK(21, 20), + /* Bits 22-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + [DCPH_ENABLE] = BIT(3), + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + [DRBIP_ACL_ENABLE] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(10, 5), + /* Bit 11 reserved */ + [TIME_LIMIT] = GENMASK(16, 12), + [PKT_LIMIT] = GENMASK(22, 17), + [SW_EOF_ACTIVE] = BIT(23), + [FORCE_CLOSE] = BIT(24), + /* Bit 25 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(26), + [AGGR_GRAN_SEL] = BIT(27), + /* Bits 28-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_LIMIT] = GENMASK(4, 0), + /* Bits 5-7 reserved */ + [TIMER_GRAN_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + /* Bits 8-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-8 reserved */ + [STATUS_PKT_SUPPRESS] = BIT(9), + /* Bits 10-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = { + [FILTER_HASH_MSK_SRC_ID] = BIT(0), + [FILTER_HASH_MSK_SRC_IP] = BIT(1), + [FILTER_HASH_MSK_DST_IP] = BIT(2), + [FILTER_HASH_MSK_SRC_PORT] = BIT(3), + [FILTER_HASH_MSK_DST_PORT] = BIT(4), + [FILTER_HASH_MSK_PROTOCOL] = BIT(5), + [FILTER_HASH_MSK_METADATA] = BIT(6), + [FILTER_HASH_MSK_ALL] = GENMASK(6, 0), + /* Bits 7-15 reserved */ + [ROUTER_HASH_MSK_SRC_ID] = BIT(16), + [ROUTER_HASH_MSK_SRC_IP] = BIT(17), + [ROUTER_HASH_MSK_DST_IP] = BIT(18), + [ROUTER_HASH_MSK_SRC_PORT] = BIT(19), + [ROUTER_HASH_MSK_DST_PORT] = BIT(20), + [ROUTER_HASH_MSK_PROTOCOL] = BIT(21), + [ROUTER_HASH_MSK_METADATA] = BIT(22), + [ROUTER_HASH_MSK_ALL] = GENMASK(22, 16), + /* Bits 23-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg, + 0x0000085c, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00004008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000400c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00004010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000401c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00004030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00004034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00004038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [IPA_TX_CFG] = &ipa_reg_ipa_tx_cfg, + [FLAVOR_0] = &ipa_reg_flavor_0, + [IDLE_INDICATION_CFG] = &ipa_reg_idle_indication_cfg, + [QTIME_TIMESTAMP_CFG] = &ipa_reg_qtime_timestamp_cfg, + [TIMERS_XO_CLK_DIV_CFG] = &ipa_reg_timers_xo_clk_div_cfg, + [TIMERS_PULSE_GRAN_CFG] = &ipa_reg_timers_pulse_gran_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [ENDP_FILTER_ROUTER_HSH_CFG] = &ipa_reg_endp_filter_router_hsh_cfg, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v4_11 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipa/reg/ipa_reg-v4.2.c b/drivers/net/ipa/reg/ipa_reg-v4.2.c new file mode 100644 index 0000000000000000000000000000000000000000..f8e78e1907c836168d72528b1cfcd94b60681f68 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v4.2.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + /* Bit 0 reserved */ + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + [IPA_DCMP_FAST_CLK_EN] = BIT(4), + [IPA_QMB_SELECT_CONS_EN] = BIT(5), + [IPA_QMB_SELECT_PROD_EN] = BIT(6), + [GSI_MULTI_INORDER_RD_DIS] = BIT(7), + [GSI_MULTI_INORDER_WR_DIS] = BIT(8), + [GEN_QMB_0_MULTI_INORDER_RD_DIS] = BIT(9), + [GEN_QMB_1_MULTI_INORDER_RD_DIS] = BIT(10), + [GEN_QMB_0_MULTI_INORDER_WR_DIS] = BIT(11), + [GEN_QMB_1_MULTI_INORDER_WR_DIS] = BIT(12), + [GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS] = BIT(13), + [GSI_SNOC_CNOC_LOOP_PROT_DISABLE] = BIT(14), + [GSI_MULTI_AXI_MASTERS_DIS] = BIT(15), + [IPA_QMB_SELECT_GLOBAL_EN] = BIT(16), + [ATOMIC_FETCHER_ARB_LOCK_DIS] = GENMASK(20, 17), + /* Bits 21-31 reserved */ +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + /* Bit 17 reserved */ + [NTF_TX_CMDQS] = BIT(18), + [CLKON_TX_0] = BIT(19), + [CLKON_TX_1] = BIT(20), + [CLKON_FNR] = BIT(21), + [QSB2AXI_CMDQ_L] = BIT(22), + [AGGR_WRAPPER] = BIT(23), + [RAM_SLAVEWAY] = BIT(24), + [CLKON_QMB] = BIT(25), + [WEIGHT_ARB] = BIT(26), + [GSI_IF] = BIT(27), + [CLKON_GLOBAL] = BIT(28), + [GLOBAL_2X_CLK] = BIT(29), + /* Bits 30-31 reserved */ +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), + /* Bits 8-15 reserved */ + [GEN_QMB_0_MAX_READS_BEATS] = GENMASK(23, 16), + [GEN_QMB_1_MAX_READS_BEATS] = GENMASK(31, 24), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4); + +IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(16, 0), + /* Bits 17-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_counter_cfg_fmask[] = { + /* Bits 0-3 reserved */ + [AGGR_GRANULARITY] = GENMASK(8, 4), + /* Bits 9-31 reserved */ +}; + +IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0); + +static const u32 ipa_reg_ipa_tx_cfg_fmask[] = { + /* Bits 0-1 reserved */ + [PREFETCH_ALMOST_EMPTY_SIZE_TX0] = GENMASK(5, 2), + [DMAW_SCND_OUTSD_PRED_THRESHOLD] = GENMASK(9, 6), + [DMAW_SCND_OUTSD_PRED_EN] = BIT(10), + [DMAW_MAX_BEATS_256_DIS] = BIT(11), + [PA_MASK_EN] = BIT(12), + [PREFETCH_ALMOST_EMPTY_SIZE_TX1] = GENMASK(16, 13), + /* Bit 17 reserved */ + [SSPND_PA_NO_START_STATE] = BIT(18), + [SSPND_PA_NO_BQ_STATE] = BIT(19), + /* Bits 20-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc); + +static const u32 ipa_reg_flavor_0_fmask[] = { + [MAX_PIPES] = GENMASK(3, 0), + /* Bits 4-7 reserved */ + [MAX_CONS_PIPES] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [MAX_PROD_PIPES] = GENMASK(20, 16), + /* Bits 21-23 reserved */ + [PROD_LOWEST] = GENMASK(27, 24), + /* Bits 28-31 reserved */ +}; + +IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210); + +static const u32 ipa_reg_idle_indication_cfg_fmask[] = { + [ENTER_IDLE_DEBOUNCE_THRESH] = GENMASK(15, 0), + [CONST_NON_IDLE_ENABLE] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + [HDR_A5_MUX] = BIT(26), + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_METADATA_REG_VALID] = BIT(28), + /* Bits 29-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + /* Bit 3 reserved */ + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + [HDR_FTCH_DISABLE] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(9, 5), + [TIME_LIMIT] = GENMASK(14, 10), + [PKT_LIMIT] = GENMASK(20, 15), + [SW_EOF_ACTIVE] = BIT(21), + [FORCE_CLOSE] = BIT(22), + /* Bit 23 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_BASE_VALUE] = GENMASK(4, 0), + /* Bits 5-7 reserved */ + [TIMER_SCALE] = GENMASK(12, 8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + [SEQ_REP_TYPE] = GENMASK(15, 8), + /* Bits 16-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-7 reserved */ + [STATUS_LOCATION] = BIT(8), + [STATUS_PKT_SUPPRESS] = BIT(9), + /* Bits 10-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [IPA_BCR] = &ipa_reg_ipa_bcr, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [COUNTER_CFG] = &ipa_reg_counter_cfg, + [IPA_TX_CFG] = &ipa_reg_ipa_tx_cfg, + [FLAVOR_0] = &ipa_reg_flavor_0, + [IDLE_INDICATION_CFG] = &ipa_reg_idle_indication_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v4_2 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipa/reg/ipa_reg-v4.5.c b/drivers/net/ipa/reg/ipa_reg-v4.5.c new file mode 100644 index 0000000000000000000000000000000000000000..d32b805abb11aa84b1f2cd1c65a64d631d0473b9 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v4.5.c @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + /* Bit 0 reserved */ + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + /* Bit 4 reserved */ + [IPA_QMB_SELECT_CONS_EN] = BIT(5), + [IPA_QMB_SELECT_PROD_EN] = BIT(6), + [GSI_MULTI_INORDER_RD_DIS] = BIT(7), + [GSI_MULTI_INORDER_WR_DIS] = BIT(8), + [GEN_QMB_0_MULTI_INORDER_RD_DIS] = BIT(9), + [GEN_QMB_1_MULTI_INORDER_RD_DIS] = BIT(10), + [GEN_QMB_0_MULTI_INORDER_WR_DIS] = BIT(11), + [GEN_QMB_1_MULTI_INORDER_WR_DIS] = BIT(12), + [GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS] = BIT(13), + [GSI_SNOC_CNOC_LOOP_PROT_DISABLE] = BIT(14), + [GSI_MULTI_AXI_MASTERS_DIS] = BIT(15), + [IPA_QMB_SELECT_GLOBAL_EN] = BIT(16), + [ATOMIC_FETCHER_ARB_LOCK_DIS] = GENMASK(20, 17), + [FULL_FLUSH_WAIT_RS_CLOSURE_EN] = BIT(21), + /* Bits 22-31 reserved */ +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + [CLKON_DCMP] = BIT(17), + [NTF_TX_CMDQS] = BIT(18), + [CLKON_TX_0] = BIT(19), + [CLKON_TX_1] = BIT(20), + [CLKON_FNR] = BIT(21), + [QSB2AXI_CMDQ_L] = BIT(22), + [AGGR_WRAPPER] = BIT(23), + [RAM_SLAVEWAY] = BIT(24), + [CLKON_QMB] = BIT(25), + [WEIGHT_ARB] = BIT(26), + [GSI_IF] = BIT(27), + [CLKON_GLOBAL] = BIT(28), + [GLOBAL_2X_CLK] = BIT(29), + [DPL_FIFO] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), + /* Bits 8-15 reserved */ + [GEN_QMB_0_MAX_READS_BEATS] = GENMASK(23, 16), + [GEN_QMB_1_MAX_READS_BEATS] = GENMASK(31, 24), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(17, 0), + /* Bits 18-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_ipa_tx_cfg_fmask[] = { + /* Bits 0-1 reserved */ + [PREFETCH_ALMOST_EMPTY_SIZE_TX0] = GENMASK(5, 2), + [DMAW_SCND_OUTSD_PRED_THRESHOLD] = GENMASK(9, 6), + [DMAW_SCND_OUTSD_PRED_EN] = BIT(10), + [DMAW_MAX_BEATS_256_DIS] = BIT(11), + [PA_MASK_EN] = BIT(12), + [PREFETCH_ALMOST_EMPTY_SIZE_TX1] = GENMASK(16, 13), + [DUAL_TX_ENABLE] = BIT(17), + /* Bits 18-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc); + +static const u32 ipa_reg_flavor_0_fmask[] = { + [MAX_PIPES] = GENMASK(3, 0), + /* Bits 4-7 reserved */ + [MAX_CONS_PIPES] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [MAX_PROD_PIPES] = GENMASK(20, 16), + /* Bits 21-23 reserved */ + [PROD_LOWEST] = GENMASK(27, 24), + /* Bits 28-31 reserved */ +}; + +IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210); + +static const u32 ipa_reg_idle_indication_cfg_fmask[] = { + [ENTER_IDLE_DEBOUNCE_THRESH] = GENMASK(15, 0), + [CONST_NON_IDLE_ENABLE] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240); + +static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = { + [DPL_TIMESTAMP_LSB] = GENMASK(4, 0), + /* Bits 5-6 reserved */ + [DPL_TIMESTAMP_SEL] = BIT(7), + [TAG_TIMESTAMP_LSB] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [NAT_TIMESTAMP_LSB] = GENMASK(20, 16), + /* Bits 21-31 reserved */ +}; + +IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c); + +static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = { + [DIV_VALUE] = GENMASK(8, 0), + /* Bits 9-30 reserved */ + [DIV_ENABLE] = BIT(31), +}; + +IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250); + +static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = { + [PULSE_GRAN_0] = GENMASK(2, 0), + [PULSE_GRAN_1] = GENMASK(5, 3), + [PULSE_GRAN_2] = GENMASK(8, 6), +}; + +IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_45_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type, + 0x00000408, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_45_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type, + 0x00000508, 0x0020); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + [HDR_A5_MUX] = BIT(26), + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_LEN_MSB] = GENMASK(29, 28), + [HDR_OFST_METADATA_MSB] = GENMASK(31, 30), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-15 reserved */ + [HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB] = GENMASK(17, 16), + [HDR_OFST_PKT_SIZE_MSB] = GENMASK(19, 18), + [HDR_ADDITIONAL_CONST_LEN_MSB] = GENMASK(21, 20), + /* Bits 22-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + [DCPH_ENABLE] = BIT(3), + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(10, 5), + /* Bit 11 reserved */ + [TIME_LIMIT] = GENMASK(16, 12), + [PKT_LIMIT] = GENMASK(22, 17), + [SW_EOF_ACTIVE] = BIT(23), + [FORCE_CLOSE] = BIT(24), + /* Bit 25 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(26), + [AGGR_GRAN_SEL] = BIT(27), + /* Bits 28-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_LIMIT] = GENMASK(4, 0), + /* Bits 5-7 reserved */ + [TIMER_GRAN_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = GENMASK(2, 0), + /* Bits 3-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + /* Bits 8-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-8 reserved */ + [STATUS_PKT_SUPPRESS] = BIT(9), + /* Bits 10-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = { + [FILTER_HASH_MSK_SRC_ID] = BIT(0), + [FILTER_HASH_MSK_SRC_IP] = BIT(1), + [FILTER_HASH_MSK_DST_IP] = BIT(2), + [FILTER_HASH_MSK_SRC_PORT] = BIT(3), + [FILTER_HASH_MSK_DST_PORT] = BIT(4), + [FILTER_HASH_MSK_PROTOCOL] = BIT(5), + [FILTER_HASH_MSK_METADATA] = BIT(6), + [FILTER_HASH_MSK_ALL] = GENMASK(6, 0), + /* Bits 7-15 reserved */ + [ROUTER_HASH_MSK_SRC_ID] = BIT(16), + [ROUTER_HASH_MSK_SRC_IP] = BIT(17), + [ROUTER_HASH_MSK_DST_IP] = BIT(18), + [ROUTER_HASH_MSK_SRC_PORT] = BIT(19), + [ROUTER_HASH_MSK_DST_PORT] = BIT(20), + [ROUTER_HASH_MSK_PROTOCOL] = BIT(21), + [ROUTER_HASH_MSK_METADATA] = BIT(22), + [ROUTER_HASH_MSK_ALL] = GENMASK(22, 16), + /* Bits 23-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg, + 0x0000085c, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [IPA_TX_CFG] = &ipa_reg_ipa_tx_cfg, + [FLAVOR_0] = &ipa_reg_flavor_0, + [IDLE_INDICATION_CFG] = &ipa_reg_idle_indication_cfg, + [QTIME_TIMESTAMP_CFG] = &ipa_reg_qtime_timestamp_cfg, + [TIMERS_XO_CLK_DIV_CFG] = &ipa_reg_timers_xo_clk_div_cfg, + [TIMERS_PULSE_GRAN_CFG] = &ipa_reg_timers_pulse_gran_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [SRC_RSRC_GRP_45_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_45_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_45_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_45_rsrc_type, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [ENDP_FILTER_ROUTER_HSH_CFG] = &ipa_reg_endp_filter_router_hsh_cfg, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v4_5 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipa/reg/ipa_reg-v4.9.c b/drivers/net/ipa/reg/ipa_reg-v4.9.c new file mode 100644 index 0000000000000000000000000000000000000000..eabbc5451937bf79f73e040d5698e9e9a7eb10d0 --- /dev/null +++ b/drivers/net/ipa/reg/ipa_reg-v4.9.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (C) 2022 Linaro Ltd. */ + +#include + +#include "../ipa.h" +#include "../ipa_reg.h" + +static const u32 ipa_reg_comp_cfg_fmask[] = { + [RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS] = BIT(0), + [GSI_SNOC_BYPASS_DIS] = BIT(1), + [GEN_QMB_0_SNOC_BYPASS_DIS] = BIT(2), + [GEN_QMB_1_SNOC_BYPASS_DIS] = BIT(3), + /* Bit 4 reserved */ + [IPA_QMB_SELECT_CONS_EN] = BIT(5), + [IPA_QMB_SELECT_PROD_EN] = BIT(6), + [GSI_MULTI_INORDER_RD_DIS] = BIT(7), + [GSI_MULTI_INORDER_WR_DIS] = BIT(8), + [GEN_QMB_0_MULTI_INORDER_RD_DIS] = BIT(9), + [GEN_QMB_1_MULTI_INORDER_RD_DIS] = BIT(10), + [GEN_QMB_0_MULTI_INORDER_WR_DIS] = BIT(11), + [GEN_QMB_1_MULTI_INORDER_WR_DIS] = BIT(12), + [GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS] = BIT(13), + [GSI_SNOC_CNOC_LOOP_PROT_DISABLE] = BIT(14), + [GSI_MULTI_AXI_MASTERS_DIS] = BIT(15), + [IPA_QMB_SELECT_GLOBAL_EN] = BIT(16), + [FULL_FLUSH_WAIT_RS_CLOSURE_EN] = BIT(17), + [QMB_RAM_RD_CACHE_DISABLE] = BIT(19), + [GENQMB_AOOOWR] = BIT(20), + [IF_OUT_OF_BUF_STOP_RESET_MASK_EN] = BIT(21), + [ATOMIC_FETCHER_ARB_LOCK_DIS] = GENMASK(24, 22), + /* Bits 25-29 reserved */ + [GEN_QMB_1_DYNAMIC_ASIZE] = BIT(30), + [GEN_QMB_0_DYNAMIC_ASIZE] = BIT(31), +}; + +IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c); + +static const u32 ipa_reg_clkon_cfg_fmask[] = { + [CLKON_RX] = BIT(0), + [CLKON_PROC] = BIT(1), + [TX_WRAPPER] = BIT(2), + [CLKON_MISC] = BIT(3), + [RAM_ARB] = BIT(4), + [FTCH_HPS] = BIT(5), + [FTCH_DPS] = BIT(6), + [CLKON_HPS] = BIT(7), + [CLKON_DPS] = BIT(8), + [RX_HPS_CMDQS] = BIT(9), + [HPS_DPS_CMDQS] = BIT(10), + [DPS_TX_CMDQS] = BIT(11), + [RSRC_MNGR] = BIT(12), + [CTX_HANDLER] = BIT(13), + [ACK_MNGR] = BIT(14), + [D_DCPH] = BIT(15), + [H_DCPH] = BIT(16), + [CLKON_DCMP] = BIT(17), + [NTF_TX_CMDQS] = BIT(18), + [CLKON_TX_0] = BIT(19), + [CLKON_TX_1] = BIT(20), + [CLKON_FNR] = BIT(21), + [QSB2AXI_CMDQ_L] = BIT(22), + [AGGR_WRAPPER] = BIT(23), + [RAM_SLAVEWAY] = BIT(24), + [CLKON_QMB] = BIT(25), + [WEIGHT_ARB] = BIT(26), + [GSI_IF] = BIT(27), + [CLKON_GLOBAL] = BIT(28), + [GLOBAL_2X_CLK] = BIT(29), + [DPL_FIFO] = BIT(30), + [DRBIP] = BIT(31), +}; + +IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044); + +static const u32 ipa_reg_route_fmask[] = { + [ROUTE_DIS] = BIT(0), + [ROUTE_DEF_PIPE] = GENMASK(5, 1), + [ROUTE_DEF_HDR_TABLE] = BIT(6), + [ROUTE_DEF_HDR_OFST] = GENMASK(16, 7), + [ROUTE_FRAG_DEF_PIPE] = GENMASK(21, 17), + /* Bits 22-23 reserved */ + [ROUTE_DEF_RETAIN_HDR] = BIT(24), + /* Bits 25-31 reserved */ +}; + +IPA_REG_FIELDS(ROUTE, route, 0x00000048); + +static const u32 ipa_reg_shared_mem_size_fmask[] = { + [MEM_SIZE] = GENMASK(15, 0), + [MEM_BADDR] = GENMASK(31, 16), +}; + +IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054); + +static const u32 ipa_reg_qsb_max_writes_fmask[] = { + [GEN_QMB_0_MAX_WRITES] = GENMASK(3, 0), + [GEN_QMB_1_MAX_WRITES] = GENMASK(7, 4), + /* Bits 8-31 reserved */ +}; + +IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074); + +static const u32 ipa_reg_qsb_max_reads_fmask[] = { + [GEN_QMB_0_MAX_READS] = GENMASK(3, 0), + [GEN_QMB_1_MAX_READS] = GENMASK(7, 4), + /* Bits 8-15 reserved */ + [GEN_QMB_0_MAX_READS_BEATS] = GENMASK(23, 16), + [GEN_QMB_1_MAX_READS_BEATS] = GENMASK(31, 24), +}; + +IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078); + +static const u32 ipa_reg_filt_rout_hash_en_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148); + +static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = { + [IPV6_ROUTER_HASH] = BIT(0), + /* Bits 1-3 reserved */ + [IPV6_FILTER_HASH] = BIT(4), + /* Bits 5-7 reserved */ + [IPV4_ROUTER_HASH] = BIT(8), + /* Bits 9-11 reserved */ + [IPV4_FILTER_HASH] = BIT(12), + /* Bits 13-31 reserved */ +}; + +IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c); + +/* Valid bits defined by ipa->available */ +IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4); + +static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = { + [IPA_BASE_ADDR] = GENMASK(17, 0), + /* Bits 18-31 reserved */ +}; + +/* Offset must be a multiple of 8 */ +IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8); + +/* Valid bits defined by ipa->available */ +IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec); + +static const u32 ipa_reg_ipa_tx_cfg_fmask[] = { + /* Bits 0-1 reserved */ + [PREFETCH_ALMOST_EMPTY_SIZE_TX0] = GENMASK(5, 2), + [DMAW_SCND_OUTSD_PRED_THRESHOLD] = GENMASK(9, 6), + [DMAW_SCND_OUTSD_PRED_EN] = BIT(10), + [DMAW_MAX_BEATS_256_DIS] = BIT(11), + [PA_MASK_EN] = BIT(12), + [PREFETCH_ALMOST_EMPTY_SIZE_TX1] = GENMASK(16, 13), + [DUAL_TX_ENABLE] = BIT(17), + [SSPND_PA_NO_START_STATE] = BIT(18), + /* Bits 19-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc); + +static const u32 ipa_reg_flavor_0_fmask[] = { + [MAX_PIPES] = GENMASK(3, 0), + /* Bits 4-7 reserved */ + [MAX_CONS_PIPES] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [MAX_PROD_PIPES] = GENMASK(20, 16), + /* Bits 21-23 reserved */ + [PROD_LOWEST] = GENMASK(27, 24), + /* Bits 28-31 reserved */ +}; + +IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210); + +static const u32 ipa_reg_idle_indication_cfg_fmask[] = { + [ENTER_IDLE_DEBOUNCE_THRESH] = GENMASK(15, 0), + [CONST_NON_IDLE_ENABLE] = BIT(16), + /* Bits 17-31 reserved */ +}; + +IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240); + +static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = { + [DPL_TIMESTAMP_LSB] = GENMASK(4, 0), + /* Bits 5-6 reserved */ + [DPL_TIMESTAMP_SEL] = BIT(7), + [TAG_TIMESTAMP_LSB] = GENMASK(12, 8), + /* Bits 13-15 reserved */ + [NAT_TIMESTAMP_LSB] = GENMASK(20, 16), + /* Bits 21-31 reserved */ +}; + +IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c); + +static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = { + [DIV_VALUE] = GENMASK(8, 0), + /* Bits 9-30 reserved */ + [DIV_ENABLE] = BIT(31), +}; + +IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250); + +static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = { + [PULSE_GRAN_0] = GENMASK(2, 0), + [PULSE_GRAN_1] = GENMASK(5, 3), + [PULSE_GRAN_2] = GENMASK(8, 6), +}; + +IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254); + +static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, + 0x00000400, 0x0020); + +static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, + 0x00000404, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, + 0x00000500, 0x0020); + +static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { + [X_MIN_LIM] = GENMASK(5, 0), + /* Bits 6-7 reserved */ + [X_MAX_LIM] = GENMASK(13, 8), + /* Bits 14-15 reserved */ + [Y_MIN_LIM] = GENMASK(21, 16), + /* Bits 22-23 reserved */ + [Y_MAX_LIM] = GENMASK(29, 24), + /* Bits 30-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, + 0x00000504, 0x0020); + +static const u32 ipa_reg_endp_init_cfg_fmask[] = { + [FRAG_OFFLOAD_EN] = BIT(0), + [CS_OFFLOAD_EN] = GENMASK(2, 1), + [CS_METADATA_HDR_OFFSET] = GENMASK(6, 3), + /* Bit 7 reserved */ + [CS_GEN_QMB_MASTER_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070); + +static const u32 ipa_reg_endp_init_nat_fmask[] = { + [NAT_EN] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_fmask[] = { + [HDR_LEN] = GENMASK(5, 0), + [HDR_OFST_METADATA_VALID] = BIT(6), + [HDR_OFST_METADATA] = GENMASK(12, 7), + [HDR_ADDITIONAL_CONST_LEN] = GENMASK(18, 13), + [HDR_OFST_PKT_SIZE_VALID] = BIT(19), + [HDR_OFST_PKT_SIZE] = GENMASK(25, 20), + [HDR_LEN_INC_DEAGG_HDR] = BIT(27), + [HDR_LEN_MSB] = GENMASK(29, 28), + [HDR_OFST_METADATA_MSB] = GENMASK(31, 30), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070); + +static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = { + [HDR_ENDIANNESS] = BIT(0), + [HDR_TOTAL_LEN_OR_PAD_VALID] = BIT(1), + [HDR_TOTAL_LEN_OR_PAD] = BIT(2), + [HDR_PAYLOAD_LEN_INC_PADDING] = BIT(3), + [HDR_TOTAL_LEN_OR_PAD_OFFSET] = GENMASK(9, 4), + [HDR_PAD_TO_ALIGNMENT] = GENMASK(13, 10), + /* Bits 14-15 reserved */ + [HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB] = GENMASK(17, 16), + [HDR_OFST_PKT_SIZE_MSB] = GENMASK(19, 18), + [HDR_ADDITIONAL_CONST_LEN_MSB] = GENMASK(21, 20), + /* Bits 22-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070); + +IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask, + 0x00000818, 0x0070); + +static const u32 ipa_reg_endp_init_mode_fmask[] = { + [ENDP_MODE] = GENMASK(2, 0), + [DCPH_ENABLE] = BIT(3), + [DEST_PIPE_INDEX] = GENMASK(8, 4), + /* Bits 9-11 reserved */ + [BYTE_THRESHOLD] = GENMASK(27, 12), + [PIPE_REPLICATION_EN] = BIT(28), + [PAD_EN] = BIT(29), + [DRBIP_ACL_ENABLE] = BIT(30), + /* Bit 31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070); + +static const u32 ipa_reg_endp_init_aggr_fmask[] = { + [AGGR_EN] = GENMASK(1, 0), + [AGGR_TYPE] = GENMASK(4, 2), + [BYTE_LIMIT] = GENMASK(10, 5), + /* Bit 11 reserved */ + [TIME_LIMIT] = GENMASK(16, 12), + [PKT_LIMIT] = GENMASK(22, 17), + [SW_EOF_ACTIVE] = BIT(23), + [FORCE_CLOSE] = BIT(24), + /* Bit 25 reserved */ + [HARD_BYTE_LIMIT_EN] = BIT(26), + [AGGR_GRAN_SEL] = BIT(27), + /* Bits 28-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = { + [HOL_BLOCK_EN] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en, + 0x0000082c, 0x0070); + +static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = { + [TIMER_LIMIT] = GENMASK(4, 0), + /* Bits 5-7 reserved */ + [TIMER_GRAN_SEL] = BIT(8), + /* Bits 9-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer, + 0x00000830, 0x0070); + +static const u32 ipa_reg_endp_init_deaggr_fmask[] = { + [DEAGGR_HDR_LEN] = GENMASK(5, 0), + [SYSPIPE_ERR_DETECTION] = BIT(6), + [PACKET_OFFSET_VALID] = BIT(7), + [PACKET_OFFSET_LOCATION] = GENMASK(13, 8), + [IGNORE_MIN_PKT_ERR] = BIT(14), + /* Bit 15 reserved */ + [MAX_PACKET_LEN] = GENMASK(31, 16), +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070); + +static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = { + [ENDP_RSRC_GRP] = GENMASK(1, 0), + /* Bits 2-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp, + 0x00000838, 0x0070); + +static const u32 ipa_reg_endp_init_seq_fmask[] = { + [SEQ_TYPE] = GENMASK(7, 0), + /* Bits 8-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070); + +static const u32 ipa_reg_endp_status_fmask[] = { + [STATUS_EN] = BIT(0), + [STATUS_ENDP] = GENMASK(5, 1), + /* Bits 6-8 reserved */ + [STATUS_PKT_SUPPRESS] = BIT(9), + /* Bits 10-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070); + +static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = { + [FILTER_HASH_MSK_SRC_ID] = BIT(0), + [FILTER_HASH_MSK_SRC_IP] = BIT(1), + [FILTER_HASH_MSK_DST_IP] = BIT(2), + [FILTER_HASH_MSK_SRC_PORT] = BIT(3), + [FILTER_HASH_MSK_DST_PORT] = BIT(4), + [FILTER_HASH_MSK_PROTOCOL] = BIT(5), + [FILTER_HASH_MSK_METADATA] = BIT(6), + [FILTER_HASH_MSK_ALL] = GENMASK(6, 0), + /* Bits 7-15 reserved */ + [ROUTER_HASH_MSK_SRC_ID] = BIT(16), + [ROUTER_HASH_MSK_SRC_IP] = BIT(17), + [ROUTER_HASH_MSK_DST_IP] = BIT(18), + [ROUTER_HASH_MSK_SRC_PORT] = BIT(19), + [ROUTER_HASH_MSK_DST_PORT] = BIT(20), + [ROUTER_HASH_MSK_PROTOCOL] = BIT(21), + [ROUTER_HASH_MSK_METADATA] = BIT(22), + [ROUTER_HASH_MSK_ALL] = GENMASK(22, 16), + /* Bits 23-31 reserved */ +}; + +IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg, + 0x0000085c, 0x0070); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00004008 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000400c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */ +IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00004010 + 0x1000 * GSI_EE_AP); + +static const u32 ipa_reg_ipa_irq_uc_fmask[] = { + [UC_INTR] = BIT(0), + /* Bits 1-31 reserved */ +}; + +IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000401c + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00004030 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00004034 + 0x1000 * GSI_EE_AP); + +/* Valid bits defined by ipa->available */ +IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00004038 + 0x1000 * GSI_EE_AP); + +static const struct ipa_reg *ipa_reg_array[] = { + [COMP_CFG] = &ipa_reg_comp_cfg, + [CLKON_CFG] = &ipa_reg_clkon_cfg, + [ROUTE] = &ipa_reg_route, + [SHARED_MEM_SIZE] = &ipa_reg_shared_mem_size, + [QSB_MAX_WRITES] = &ipa_reg_qsb_max_writes, + [QSB_MAX_READS] = &ipa_reg_qsb_max_reads, + [FILT_ROUT_HASH_EN] = &ipa_reg_filt_rout_hash_en, + [FILT_ROUT_HASH_FLUSH] = &ipa_reg_filt_rout_hash_flush, + [STATE_AGGR_ACTIVE] = &ipa_reg_state_aggr_active, + [LOCAL_PKT_PROC_CNTXT] = &ipa_reg_local_pkt_proc_cntxt, + [AGGR_FORCE_CLOSE] = &ipa_reg_aggr_force_close, + [IPA_TX_CFG] = &ipa_reg_ipa_tx_cfg, + [FLAVOR_0] = &ipa_reg_flavor_0, + [IDLE_INDICATION_CFG] = &ipa_reg_idle_indication_cfg, + [QTIME_TIMESTAMP_CFG] = &ipa_reg_qtime_timestamp_cfg, + [TIMERS_XO_CLK_DIV_CFG] = &ipa_reg_timers_xo_clk_div_cfg, + [TIMERS_PULSE_GRAN_CFG] = &ipa_reg_timers_pulse_gran_cfg, + [SRC_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_01_rsrc_type, + [SRC_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_src_rsrc_grp_23_rsrc_type, + [DST_RSRC_GRP_01_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_01_rsrc_type, + [DST_RSRC_GRP_23_RSRC_TYPE] = &ipa_reg_dst_rsrc_grp_23_rsrc_type, + [ENDP_INIT_CFG] = &ipa_reg_endp_init_cfg, + [ENDP_INIT_NAT] = &ipa_reg_endp_init_nat, + [ENDP_INIT_HDR] = &ipa_reg_endp_init_hdr, + [ENDP_INIT_HDR_EXT] = &ipa_reg_endp_init_hdr_ext, + [ENDP_INIT_HDR_METADATA_MASK] = &ipa_reg_endp_init_hdr_metadata_mask, + [ENDP_INIT_MODE] = &ipa_reg_endp_init_mode, + [ENDP_INIT_AGGR] = &ipa_reg_endp_init_aggr, + [ENDP_INIT_HOL_BLOCK_EN] = &ipa_reg_endp_init_hol_block_en, + [ENDP_INIT_HOL_BLOCK_TIMER] = &ipa_reg_endp_init_hol_block_timer, + [ENDP_INIT_DEAGGR] = &ipa_reg_endp_init_deaggr, + [ENDP_INIT_RSRC_GRP] = &ipa_reg_endp_init_rsrc_grp, + [ENDP_INIT_SEQ] = &ipa_reg_endp_init_seq, + [ENDP_STATUS] = &ipa_reg_endp_status, + [ENDP_FILTER_ROUTER_HSH_CFG] = &ipa_reg_endp_filter_router_hsh_cfg, + [IPA_IRQ_STTS] = &ipa_reg_ipa_irq_stts, + [IPA_IRQ_EN] = &ipa_reg_ipa_irq_en, + [IPA_IRQ_CLR] = &ipa_reg_ipa_irq_clr, + [IPA_IRQ_UC] = &ipa_reg_ipa_irq_uc, + [IRQ_SUSPEND_INFO] = &ipa_reg_irq_suspend_info, + [IRQ_SUSPEND_EN] = &ipa_reg_irq_suspend_en, + [IRQ_SUSPEND_CLR] = &ipa_reg_irq_suspend_clr, +}; + +const struct ipa_regs ipa_regs_v4_9 = { + .reg_count = ARRAY_SIZE(ipa_reg_array), + .reg = ipa_reg_array, +}; diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c index dfeb5b392e642885c3897018337f94fb41d10eab..bb1c298c1e78b8b5721b2bb6fb1de1311b11850a 100644 --- a/drivers/net/ipvlan/ipvlan_core.c +++ b/drivers/net/ipvlan/ipvlan_core.c @@ -495,7 +495,6 @@ static int ipvlan_process_v6_outbound(struct sk_buff *skb) static int ipvlan_process_outbound(struct sk_buff *skb) { - struct ethhdr *ethh = eth_hdr(skb); int ret = NET_XMIT_DROP; /* The ipvlan is a pseudo-L2 device, so the packets that we receive @@ -505,6 +504,8 @@ static int ipvlan_process_outbound(struct sk_buff *skb) if (skb_mac_header_was_set(skb)) { /* In this mode we dont care about * multicast and broadcast traffic */ + struct ethhdr *ethh = eth_hdr(skb); + if (is_multicast_ether_addr(ethh->h_dest)) { pr_debug_ratelimited( "Dropped {multi|broad}cast of type=[%x]\n", @@ -589,7 +590,7 @@ out: static int ipvlan_xmit_mode_l2(struct sk_buff *skb, struct net_device *dev) { const struct ipvl_dev *ipvlan = netdev_priv(dev); - struct ethhdr *eth = eth_hdr(skb); + struct ethhdr *eth = skb_eth_hdr(skb); struct ipvl_addr *addr; void *lyr3h; int addr_type; @@ -619,6 +620,7 @@ static int ipvlan_xmit_mode_l2(struct sk_buff *skb, struct net_device *dev) return dev_forward_skb(ipvlan->phy_dev, skb); } else if (is_multicast_ether_addr(eth->h_dest)) { + skb_reset_mac_header(skb); ipvlan_skb_crossing_ns(skb, NULL); ipvlan_multicast_enqueue(ipvlan->port, skb, true); return NET_XMIT_SUCCESS; diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 49ba8a50dfb1e169beb137296fb3d3c1cfde4c19..54c94a69c2bb8df851f03bb0effe950fd72a93bb 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -408,8 +408,8 @@ static int ipvlan_ethtool_get_link_ksettings(struct net_device *dev, static void ipvlan_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, IPVLAN_DRV, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, IPV_DRV_VER, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, IPVLAN_DRV, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, IPV_DRV_VER, sizeof(drvinfo->version)); } static u32 ipvlan_ethtool_get_msglevel(struct net_device *dev) diff --git a/drivers/net/ipvlan/ipvtap.c b/drivers/net/ipvlan/ipvtap.c index ef02f2cf5ce13d9a3db210f83b801f1a3fde0976..cbabca167a0785d136f59c11935f4c934cf8ad51 100644 --- a/drivers/net/ipvlan/ipvtap.c +++ b/drivers/net/ipvlan/ipvtap.c @@ -194,7 +194,7 @@ static struct notifier_block ipvtap_notifier_block __read_mostly = { .notifier_call = ipvtap_device_event, }; -static int ipvtap_init(void) +static int __init ipvtap_init(void) { int err; @@ -228,7 +228,7 @@ out1: } module_init(ipvtap_init); -static void ipvtap_exit(void) +static void __exit ipvtap_exit(void) { rtnl_link_unregister(&ipvtap_link_ops); unregister_netdevice_notifier(&ipvtap_notifier_block); diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index ee6087e7b2bfb5db7cd22103285a9a28aa2b8995..c891b60937a7f0118147fc6b1e27f78f0df6ab0d 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -18,14 +18,13 @@ #include #include #include +#include #include #include #include #include -#define MACSEC_SCI_LEN 8 - /* SecTAG length = macsec_eth_header without the optional SCI */ #define MACSEC_TAG_LEN 6 @@ -46,20 +45,10 @@ struct macsec_eth_header { u8 secure_channel_id[8]; /* optional */ } __packed; -#define MACSEC_TCI_VERSION 0x80 -#define MACSEC_TCI_ES 0x40 /* end station */ -#define MACSEC_TCI_SC 0x20 /* SCI present */ -#define MACSEC_TCI_SCB 0x10 /* epon */ -#define MACSEC_TCI_E 0x08 /* encryption */ -#define MACSEC_TCI_C 0x04 /* changed text */ -#define MACSEC_AN_MASK 0x03 /* association number */ -#define MACSEC_TCI_CONFID (MACSEC_TCI_E | MACSEC_TCI_C) - /* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */ #define MIN_NON_SHORT_LEN 48 #define GCM_AES_IV_LEN 12 -#define DEFAULT_ICV_LEN 16 #define for_each_rxsc(secy, sc) \ for (sc = rcu_dereference_bh(secy->rx_sc); \ @@ -243,7 +232,6 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb) return (struct macsec_cb *)skb->cb; } -#define MACSEC_PORT_ES (htons(0x0001)) #define MACSEC_PORT_SCB (0x0000) #define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL) #define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff) @@ -258,14 +246,6 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb) #define DEFAULT_ENCODING_SA 0 #define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1)) -static bool send_sci(const struct macsec_secy *secy) -{ - const struct macsec_tx_sc *tx_sc = &secy->tx_sc; - - return tx_sc->send_sci || - (secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb); -} - static sci_t make_sci(const u8 *addr, __be16 port) { sci_t sci; @@ -330,7 +310,7 @@ static void macsec_fill_sectag(struct macsec_eth_header *h, /* with GCM, C/E clear for !encrypt, both set for encrypt */ if (tx_sc->encrypt) h->tci_an |= MACSEC_TCI_CONFID; - else if (secy->icv_len != DEFAULT_ICV_LEN) + else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) h->tci_an |= MACSEC_TCI_C; h->tci_an |= tx_sc->encoding_sa; @@ -462,11 +442,6 @@ static struct macsec_eth_header *macsec_ethhdr(struct sk_buff *skb) return (struct macsec_eth_header *)skb_mac_header(skb); } -static sci_t dev_to_sci(struct net_device *dev, __be16 port) -{ - return make_sci(dev->dev_addr, port); -} - static void __macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa) { @@ -659,7 +634,7 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, unprotected_len = skb->len; eth = eth_hdr(skb); - sci_present = send_sci(secy); + sci_present = macsec_send_sci(secy); hh = skb_push(skb, macsec_extra_len(sci_present)); memmove(hh, eth, 2 * ETH_ALEN); @@ -1029,11 +1004,13 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb) /* Deliver to the uncontrolled port by default */ enum rx_handler_result ret = RX_HANDLER_PASS; struct ethhdr *hdr = eth_hdr(skb); + struct metadata_dst *md_dst; struct macsec_rxh_data *rxd; struct macsec_dev *macsec; rcu_read_lock(); rxd = macsec_data_rcu(skb->dev); + md_dst = skb_metadata_dst(skb); list_for_each_entry_rcu(macsec, &rxd->secys, secys) { struct sk_buff *nskb; @@ -1044,6 +1021,10 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb) * the SecTAG, so we have to deduce which port to deliver to. */ if (macsec_is_offloaded(macsec) && netif_running(ndev)) { + if (md_dst && md_dst->type == METADATA_MACSEC && + (!find_rx_sc(&macsec->secy, md_dst->u.macsec_info.sci))) + continue; + if (ether_addr_equal_64bits(hdr->h_dest, ndev->dev_addr)) { /* exact match, divert skb to this port */ @@ -1301,7 +1282,7 @@ nosci: /* 10.6.1 if the SC is not found */ cbit = !!(hdr->tci_an & MACSEC_TCI_C); if (!cbit) - macsec_finalize_skb(skb, DEFAULT_ICV_LEN, + macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN, macsec_extra_len(macsec_skb_cb(skb)->has_sci)); list_for_each_entry_rcu(macsec, &rxd->secys, secys) { @@ -1682,22 +1663,8 @@ static int macsec_offload(int (* const func)(struct macsec_context *), if (ctx->offload == MACSEC_OFFLOAD_PHY) mutex_lock(&ctx->phydev->lock); - /* Phase I: prepare. The drive should fail here if there are going to be - * issues in the commit phase. - */ - ctx->prepare = true; ret = (*func)(ctx); - if (ret) - goto phy_unlock; - /* Phase II: commit. This step cannot fail. */ - ctx->prepare = false; - ret = (*func)(ctx); - /* This should never happen: commit is not allowed to fail */ - if (unlikely(ret)) - WARN(1, "MACsec offloading commit failed (%d)\n", ret); - -phy_unlock: if (ctx->offload == MACSEC_OFFLOAD_PHY) mutex_unlock(&ctx->phydev->lock); @@ -1847,6 +1814,12 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info) rx_sa->sc = rx_sc; + if (secy->xpn) { + rx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]); + nla_memcpy(rx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT], + MACSEC_SALT_LEN); + } + /* If h/w offloading is available, propagate to the device */ if (macsec_is_offloaded(netdev_priv(dev))) { const struct macsec_ops *ops; @@ -1869,12 +1842,6 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info) goto cleanup; } - if (secy->xpn) { - rx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]); - nla_memcpy(rx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT], - MACSEC_SALT_LEN); - } - nla_memcpy(rx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN); rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa); @@ -2089,6 +2056,12 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info) if (assoc_num == tx_sc->encoding_sa && tx_sa->active) secy->operational = true; + if (secy->xpn) { + tx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]); + nla_memcpy(tx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT], + MACSEC_SALT_LEN); + } + /* If h/w offloading is available, propagate to the device */ if (macsec_is_offloaded(netdev_priv(dev))) { const struct macsec_ops *ops; @@ -2111,12 +2084,6 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info) goto cleanup; } - if (secy->xpn) { - tx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]); - nla_memcpy(tx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT], - MACSEC_SALT_LEN); - } - nla_memcpy(tx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN); rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa); @@ -3409,6 +3376,7 @@ static struct genl_family macsec_fam __ro_after_init = { .module = THIS_MODULE, .small_ops = macsec_genl_ops, .n_small_ops = ARRAY_SIZE(macsec_genl_ops), + .resv_start_op = MACSEC_CMD_UPD_OFFLOAD + 1, }; static netdev_tx_t macsec_start_xmit(struct sk_buff *skb, @@ -3420,6 +3388,11 @@ static netdev_tx_t macsec_start_xmit(struct sk_buff *skb, int ret, len; if (macsec_is_offloaded(netdev_priv(dev))) { + struct metadata_dst *md_dst = secy->tx_sc.md_dst; + + skb_dst_drop(skb); + dst_hold(&md_dst->dst); + skb_dst_set(skb, &md_dst->dst); skb->dev = macsec->real_dev; return dev_queue_xmit(skb); } @@ -3661,7 +3634,6 @@ static int macsec_set_mac_address(struct net_device *dev, void *p) out: eth_hw_addr_set(dev, addr->sa_data); - macsec->secy.sci = dev_to_sci(dev, MACSEC_PORT_ES); /* If h/w offloading is available, propagate to the device */ if (macsec_is_offloaded(macsec)) { @@ -3748,6 +3720,8 @@ static void macsec_free_netdev(struct net_device *dev) { struct macsec_dev *macsec = macsec_priv(dev); + if (macsec->secy.tx_sc.md_dst) + metadata_dst_free(macsec->secy.tx_sc.md_dst); free_percpu(macsec->stats); free_percpu(macsec->secy.tx_sc.stats); @@ -4000,6 +3974,11 @@ static bool sci_exists(struct net_device *dev, sci_t sci) return false; } +static sci_t dev_to_sci(struct net_device *dev, __be16 port) +{ + return make_sci(dev->dev_addr, port); +} + static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) { struct macsec_dev *macsec = macsec_priv(dev); @@ -4015,6 +3994,13 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) return -ENOMEM; } + secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL); + if (!secy->tx_sc.md_dst) { + free_percpu(secy->tx_sc.stats); + free_percpu(macsec->stats); + return -ENOMEM; + } + if (sci == MACSEC_UNDEF_SCI) sci = dev_to_sci(dev, MACSEC_PORT_ES); @@ -4028,6 +4014,7 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len) secy->xpn = DEFAULT_XPN; secy->sci = sci; + secy->tx_sc.md_dst->u.macsec_info.sci = sci; secy->tx_sc.active = true; secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA; secy->tx_sc.encrypt = DEFAULT_ENCRYPT; @@ -4046,7 +4033,7 @@ static int macsec_newlink(struct net *net, struct net_device *dev, { struct macsec_dev *macsec = macsec_priv(dev); rx_handler_func_t *rx_handler; - u8 icv_len = DEFAULT_ICV_LEN; + u8 icv_len = MACSEC_DEFAULT_ICV_LEN; struct net_device *real_dev; int err, mtu; sci_t sci; @@ -4170,7 +4157,7 @@ static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) { u64 csid = MACSEC_DEFAULT_CIPHER_ID; - u8 icv_len = DEFAULT_ICV_LEN; + u8 icv_len = MACSEC_DEFAULT_ICV_LEN; int flag; bool es, scb, sci; @@ -4182,7 +4169,7 @@ static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[], if (data[IFLA_MACSEC_ICV_LEN]) { icv_len = nla_get_u8(data[IFLA_MACSEC_ICV_LEN]); - if (icv_len != DEFAULT_ICV_LEN) { + if (icv_len != MACSEC_DEFAULT_ICV_LEN) { char dummy_key[DEFAULT_SAK_LEN] = { 0 }; struct crypto_aead *dummy_tfm; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 1080d6ebff63b6ffb89dfec1b298c5fc2bcbbd8d..8f8f73099de8d8308b45b7e3bf9179e8311182ee 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1043,8 +1043,8 @@ static int macvlan_fdb_del(struct ndmsg *ndm, struct nlattr *tb[], static void macvlan_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, "macvlan", sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, "0.1", sizeof(drvinfo->version)); + strscpy(drvinfo->driver, "macvlan", sizeof(drvinfo->driver)); + strscpy(drvinfo->version, "0.1", sizeof(drvinfo->version)); } static int macvlan_ethtool_get_link_ksettings(struct net_device *dev, @@ -1192,7 +1192,7 @@ void macvlan_common_setup(struct net_device *dev) { ether_setup(dev); - dev->min_mtu = 0; + /* ether_setup() has set dev->min_mtu to ETH_MIN_MTU. */ dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; netif_keep_dst(dev); diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c index cecf8c63096cd602f4c740431152ed56964a2495..d1f435788e9021277f4e904db5f240128d22c38b 100644 --- a/drivers/net/macvtap.c +++ b/drivers/net/macvtap.c @@ -207,7 +207,7 @@ static struct notifier_block macvtap_notifier_block __read_mostly = { .notifier_call = macvtap_device_event, }; -static int macvtap_init(void) +static int __init macvtap_init(void) { int err; @@ -241,7 +241,7 @@ out1: } module_init(macvtap_init); -static void macvtap_exit(void) +static void __exit macvtap_exit(void) { rtnl_link_unregister(&macvtap_link_ops); unregister_netdevice_notifier(&macvtap_notifier_block); diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index 53846c6b56ca2fc3b631d875e2aa02156140ddc4..0762c735dd8ae3c49db9a2dc45450ee84c25de7f 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -986,7 +986,7 @@ out: return rc; } -static int mctp_i2c_remove(struct i2c_client *client) +static void mctp_i2c_remove(struct i2c_client *client) { struct mctp_i2c_client *mcli = i2c_get_clientdata(client); struct mctp_i2c_dev *midev = NULL, *tmp = NULL; @@ -999,8 +999,6 @@ static int mctp_i2c_remove(struct i2c_client *client) mctp_i2c_free_client(mcli); mutex_unlock(&driver_clients_lock); - /* Callers ignore return code */ - return 0; } /* We look for a 'mctp-controller' property on I2C busses as they are diff --git a/drivers/net/mdio/fwnode_mdio.c b/drivers/net/mdio/fwnode_mdio.c index 3e79c2c519298287aae2e66957d6cef2715ae19d..689e728345ce38b317d2071afcc003c041c5a098 100644 --- a/drivers/net/mdio/fwnode_mdio.c +++ b/drivers/net/mdio/fwnode_mdio.c @@ -10,10 +10,31 @@ #include #include #include +#include MODULE_AUTHOR("Calvin Johnson "); MODULE_LICENSE("GPL"); +static struct pse_control * +fwnode_find_pse_control(struct fwnode_handle *fwnode) +{ + struct pse_control *psec; + struct device_node *np; + + if (!IS_ENABLED(CONFIG_PSE_CONTROLLER)) + return NULL; + + np = to_of_node(fwnode); + if (!np) + return NULL; + + psec = of_pse_control_get(np); + if (PTR_ERR(psec) == -ENOENT) + return NULL; + + return psec; +} + static struct mii_timestamper * fwnode_find_mii_timestamper(struct fwnode_handle *fwnode) { @@ -47,7 +68,9 @@ int fwnode_mdiobus_phy_device_register(struct mii_bus *mdio, * just fall back to poll mode */ if (rc == -EPROBE_DEFER) - rc = -ENODEV; + rc = driver_deferred_probe_check_state(&phy->mdio.dev); + if (rc == -EPROBE_DEFER) + return rc; if (rc > 0) { phy->irq = rc; @@ -89,14 +112,21 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus, struct fwnode_handle *child, u32 addr) { struct mii_timestamper *mii_ts = NULL; + struct pse_control *psec = NULL; struct phy_device *phy; bool is_c45 = false; u32 phy_id; int rc; + psec = fwnode_find_pse_control(child); + if (IS_ERR(psec)) + return PTR_ERR(psec); + mii_ts = fwnode_find_mii_timestamper(child); - if (IS_ERR(mii_ts)) - return PTR_ERR(mii_ts); + if (IS_ERR(mii_ts)) { + rc = PTR_ERR(mii_ts); + goto clean_pse; + } rc = fwnode_property_match_string(child, "compatible", "ethernet-phy-ieee802.3-c45"); @@ -108,8 +138,8 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus, else phy = phy_device_create(bus, addr, phy_id, 0, NULL); if (IS_ERR(phy)) { - unregister_mii_timestamper(mii_ts); - return PTR_ERR(phy); + rc = PTR_ERR(phy); + goto clean_mii_ts; } if (is_acpi_node(child)) { @@ -123,25 +153,33 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus, /* All data is now stored in the phy struct, so register it */ rc = phy_device_register(phy); if (rc) { - phy_device_free(phy); fwnode_handle_put(phy->mdio.dev.fwnode); - return rc; + goto clean_phy; } } else if (is_of_node(child)) { rc = fwnode_mdiobus_phy_device_register(bus, phy, child, addr); - if (rc) { - unregister_mii_timestamper(mii_ts); - phy_device_free(phy); - return rc; - } + if (rc) + goto clean_phy; } + phy->psec = psec; + /* phy->mii_ts may already be defined by the PHY driver. A * mii_timestamper probed via the device tree will still have * precedence. */ if (mii_ts) phy->mii_ts = mii_ts; + return 0; + +clean_phy: + phy_device_free(phy); +clean_mii_ts: + unregister_mii_timestamper(mii_ts); +clean_pse: + pse_control_put(psec); + + return rc; } EXPORT_SYMBOL(fwnode_mdiobus_register_phy); diff --git a/drivers/net/mdio/mdio-i2c.c b/drivers/net/mdio/mdio-i2c.c index 09200a70b315110cfcca634f38bb4afa29f68604..bf8bf5e20fafe12aa472a4480bde55b111f295ad 100644 --- a/drivers/net/mdio/mdio-i2c.c +++ b/drivers/net/mdio/mdio-i2c.c @@ -3,6 +3,7 @@ * MDIO I2C bridge * * Copyright (C) 2015-2016 Russell King + * Copyright (C) 2021 Marek Behun * * Network PHYs can appear on I2C buses when they are part of SFP module. * This driver exposes these PHYs to the networking PHY code, allowing @@ -12,6 +13,7 @@ #include #include #include +#include /* * I2C bus addresses 0x50 and 0x51 are normally an EEPROM, which is @@ -28,7 +30,7 @@ static unsigned int i2c_mii_phy_addr(int phy_id) return phy_id + 0x40; } -static int i2c_mii_read(struct mii_bus *bus, int phy_id, int reg) +static int i2c_mii_read_default(struct mii_bus *bus, int phy_id, int reg) { struct i2c_adapter *i2c = bus->priv; struct i2c_msg msgs[2]; @@ -62,7 +64,8 @@ static int i2c_mii_read(struct mii_bus *bus, int phy_id, int reg) return data[0] << 8 | data[1]; } -static int i2c_mii_write(struct mii_bus *bus, int phy_id, int reg, u16 val) +static int i2c_mii_write_default(struct mii_bus *bus, int phy_id, int reg, + u16 val) { struct i2c_adapter *i2c = bus->priv; struct i2c_msg msg; @@ -91,9 +94,288 @@ static int i2c_mii_write(struct mii_bus *bus, int phy_id, int reg, u16 val) return ret < 0 ? ret : 0; } -struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c) +/* RollBall SFPs do not access internal PHY via I2C address 0x56, but + * instead via address 0x51, when SFP page is set to 0x03 and password to + * 0xffffffff. + * + * address size contents description + * ------- ---- -------- ----------- + * 0x80 1 CMD 0x01/0x02/0x04 for write/read/done + * 0x81 1 DEV Clause 45 device + * 0x82 2 REG Clause 45 register + * 0x84 2 VAL Register value + */ +#define ROLLBALL_PHY_I2C_ADDR 0x51 + +#define ROLLBALL_PASSWORD (SFP_VSL + 3) + +#define ROLLBALL_CMD_ADDR 0x80 +#define ROLLBALL_DATA_ADDR 0x81 + +#define ROLLBALL_CMD_WRITE 0x01 +#define ROLLBALL_CMD_READ 0x02 +#define ROLLBALL_CMD_DONE 0x04 + +#define SFP_PAGE_ROLLBALL_MDIO 3 + +static int __i2c_transfer_err(struct i2c_adapter *i2c, struct i2c_msg *msgs, + int num) +{ + int ret; + + ret = __i2c_transfer(i2c, msgs, num); + if (ret < 0) + return ret; + else if (ret != num) + return -EIO; + else + return 0; +} + +static int __i2c_rollball_get_page(struct i2c_adapter *i2c, int bus_addr, + u8 *page) +{ + struct i2c_msg msgs[2]; + u8 addr = SFP_PAGE; + + msgs[0].addr = bus_addr; + msgs[0].flags = 0; + msgs[0].len = 1; + msgs[0].buf = &addr; + + msgs[1].addr = bus_addr; + msgs[1].flags = I2C_M_RD; + msgs[1].len = 1; + msgs[1].buf = page; + + return __i2c_transfer_err(i2c, msgs, 2); +} + +static int __i2c_rollball_set_page(struct i2c_adapter *i2c, int bus_addr, + u8 page) +{ + struct i2c_msg msg; + u8 buf[2]; + + buf[0] = SFP_PAGE; + buf[1] = page; + + msg.addr = bus_addr; + msg.flags = 0; + msg.len = 2; + msg.buf = buf; + + return __i2c_transfer_err(i2c, &msg, 1); +} + +/* In order to not interfere with other SFP code (which possibly may manipulate + * SFP_PAGE), for every transfer we do this: + * 1. lock the bus + * 2. save content of SFP_PAGE + * 3. set SFP_PAGE to 3 + * 4. do the transfer + * 5. restore original SFP_PAGE + * 6. unlock the bus + * Note that one might think that steps 2 to 5 could be theoretically done all + * in one call to i2c_transfer (by constructing msgs array in such a way), but + * unfortunately tests show that this does not work :-( Changed SFP_PAGE does + * not take into account until i2c_transfer() is done. + */ +static int i2c_transfer_rollball(struct i2c_adapter *i2c, + struct i2c_msg *msgs, int num) +{ + int ret, main_err = 0; + u8 saved_page; + + i2c_lock_bus(i2c, I2C_LOCK_SEGMENT); + + /* save original page */ + ret = __i2c_rollball_get_page(i2c, msgs->addr, &saved_page); + if (ret) + goto unlock; + + /* change to RollBall MDIO page */ + ret = __i2c_rollball_set_page(i2c, msgs->addr, SFP_PAGE_ROLLBALL_MDIO); + if (ret) + goto unlock; + + /* do the transfer; we try to restore original page if this fails */ + ret = __i2c_transfer_err(i2c, msgs, num); + if (ret) + main_err = ret; + + /* restore original page */ + ret = __i2c_rollball_set_page(i2c, msgs->addr, saved_page); + +unlock: + i2c_unlock_bus(i2c, I2C_LOCK_SEGMENT); + + return main_err ? : ret; +} + +static int i2c_rollball_mii_poll(struct mii_bus *bus, int bus_addr, u8 *buf, + size_t len) +{ + struct i2c_adapter *i2c = bus->priv; + struct i2c_msg msgs[2]; + u8 cmd_addr, tmp, *res; + int i, ret; + + cmd_addr = ROLLBALL_CMD_ADDR; + + res = buf ? buf : &tmp; + len = buf ? len : 1; + + msgs[0].addr = bus_addr; + msgs[0].flags = 0; + msgs[0].len = 1; + msgs[0].buf = &cmd_addr; + + msgs[1].addr = bus_addr; + msgs[1].flags = I2C_M_RD; + msgs[1].len = len; + msgs[1].buf = res; + + /* By experiment it takes up to 70 ms to access a register for these + * SFPs. Sleep 20ms between iterations and try 10 times. + */ + i = 10; + do { + msleep(20); + + ret = i2c_transfer_rollball(i2c, msgs, ARRAY_SIZE(msgs)); + if (ret) + return ret; + + if (*res == ROLLBALL_CMD_DONE) + return 0; + } while (i-- > 0); + + dev_dbg(&bus->dev, "poll timed out\n"); + + return -ETIMEDOUT; +} + +static int i2c_rollball_mii_cmd(struct mii_bus *bus, int bus_addr, u8 cmd, + u8 *data, size_t len) +{ + struct i2c_adapter *i2c = bus->priv; + struct i2c_msg msgs[2]; + u8 cmdbuf[2]; + + cmdbuf[0] = ROLLBALL_CMD_ADDR; + cmdbuf[1] = cmd; + + msgs[0].addr = bus_addr; + msgs[0].flags = 0; + msgs[0].len = len; + msgs[0].buf = data; + + msgs[1].addr = bus_addr; + msgs[1].flags = 0; + msgs[1].len = sizeof(cmdbuf); + msgs[1].buf = cmdbuf; + + return i2c_transfer_rollball(i2c, msgs, ARRAY_SIZE(msgs)); +} + +static int i2c_mii_read_rollball(struct mii_bus *bus, int phy_id, int reg) +{ + u8 buf[4], res[6]; + int bus_addr, ret; + u16 val; + + if (!(reg & MII_ADDR_C45)) + return -EOPNOTSUPP; + + bus_addr = i2c_mii_phy_addr(phy_id); + if (bus_addr != ROLLBALL_PHY_I2C_ADDR) + return 0xffff; + + buf[0] = ROLLBALL_DATA_ADDR; + buf[1] = (reg >> 16) & 0x1f; + buf[2] = (reg >> 8) & 0xff; + buf[3] = reg & 0xff; + + ret = i2c_rollball_mii_cmd(bus, bus_addr, ROLLBALL_CMD_READ, buf, + sizeof(buf)); + if (ret < 0) + return ret; + + ret = i2c_rollball_mii_poll(bus, bus_addr, res, sizeof(res)); + if (ret == -ETIMEDOUT) + return 0xffff; + else if (ret < 0) + return ret; + + val = res[4] << 8 | res[5]; + + return val; +} + +static int i2c_mii_write_rollball(struct mii_bus *bus, int phy_id, int reg, + u16 val) +{ + int bus_addr, ret; + u8 buf[6]; + + if (!(reg & MII_ADDR_C45)) + return -EOPNOTSUPP; + + bus_addr = i2c_mii_phy_addr(phy_id); + if (bus_addr != ROLLBALL_PHY_I2C_ADDR) + return 0; + + buf[0] = ROLLBALL_DATA_ADDR; + buf[1] = (reg >> 16) & 0x1f; + buf[2] = (reg >> 8) & 0xff; + buf[3] = reg & 0xff; + buf[4] = val >> 8; + buf[5] = val & 0xff; + + ret = i2c_rollball_mii_cmd(bus, bus_addr, ROLLBALL_CMD_WRITE, buf, + sizeof(buf)); + if (ret < 0) + return ret; + + ret = i2c_rollball_mii_poll(bus, bus_addr, NULL, 0); + if (ret < 0) + return ret; + + return 0; +} + +static int i2c_mii_init_rollball(struct i2c_adapter *i2c) +{ + struct i2c_msg msg; + u8 pw[5]; + int ret; + + pw[0] = ROLLBALL_PASSWORD; + pw[1] = 0xff; + pw[2] = 0xff; + pw[3] = 0xff; + pw[4] = 0xff; + + msg.addr = ROLLBALL_PHY_I2C_ADDR; + msg.flags = 0; + msg.len = sizeof(pw); + msg.buf = pw; + + ret = i2c_transfer(i2c, &msg, 1); + if (ret < 0) + return ret; + else if (ret != 1) + return -EIO; + else + return 0; +} + +struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c, + enum mdio_i2c_proto protocol) { struct mii_bus *mii; + int ret; if (!i2c_check_functionality(i2c, I2C_FUNC_I2C)) return ERR_PTR(-EINVAL); @@ -104,10 +386,28 @@ struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c) snprintf(mii->id, MII_BUS_ID_SIZE, "i2c:%s", dev_name(parent)); mii->parent = parent; - mii->read = i2c_mii_read; - mii->write = i2c_mii_write; mii->priv = i2c; + switch (protocol) { + case MDIO_I2C_ROLLBALL: + ret = i2c_mii_init_rollball(i2c); + if (ret < 0) { + dev_err(parent, + "Cannot initialize RollBall MDIO I2C protocol: %d\n", + ret); + mdiobus_free(mii); + return ERR_PTR(ret); + } + + mii->read = i2c_mii_read_rollball; + mii->write = i2c_mii_write_rollball; + break; + default: + mii->read = i2c_mii_read_default; + mii->write = i2c_mii_write_default; + break; + } + return mii; } EXPORT_SYMBOL_GPL(mdio_i2c_alloc); diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c index 08541007b18aad7d09a02b0a675e5c41e3221b86..51f68daac152f6936508240d9dcebe4ae05a069c 100644 --- a/drivers/net/mdio/mdio-mscc-miim.c +++ b/drivers/net/mdio/mdio-mscc-miim.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -270,44 +271,25 @@ static int mscc_miim_clk_set(struct mii_bus *bus) static int mscc_miim_probe(struct platform_device *pdev) { - struct regmap *mii_regmap, *phy_regmap = NULL; struct device_node *np = pdev->dev.of_node; + struct regmap *mii_regmap, *phy_regmap; struct device *dev = &pdev->dev; - void __iomem *regs, *phy_regs; struct mscc_miim_dev *miim; - struct resource *res; struct mii_bus *bus; int ret; - regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(regs)) { - dev_err(dev, "Unable to map MIIM registers\n"); - return PTR_ERR(regs); - } - - mii_regmap = devm_regmap_init_mmio(dev, regs, &mscc_miim_regmap_config); - - if (IS_ERR(mii_regmap)) { - dev_err(dev, "Unable to create MIIM regmap\n"); - return PTR_ERR(mii_regmap); - } + mii_regmap = ocelot_regmap_from_resource(pdev, 0, + &mscc_miim_regmap_config); + if (IS_ERR(mii_regmap)) + return dev_err_probe(dev, PTR_ERR(mii_regmap), + "Unable to create MIIM regmap\n"); /* This resource is optional */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (res) { - phy_regs = devm_ioremap_resource(dev, res); - if (IS_ERR(phy_regs)) { - dev_err(dev, "Unable to map internal phy registers\n"); - return PTR_ERR(phy_regs); - } - - phy_regmap = devm_regmap_init_mmio(dev, phy_regs, - &mscc_miim_phy_regmap_config); - if (IS_ERR(phy_regmap)) { - dev_err(dev, "Unable to create phy register regmap\n"); - return PTR_ERR(phy_regmap); - } - } + phy_regmap = ocelot_regmap_from_resource_optional(pdev, 1, + &mscc_miim_phy_regmap_config); + if (IS_ERR(phy_regmap)) + return dev_err_probe(dev, PTR_ERR(phy_regmap), + "Unable to create phy register regmap\n"); ret = mscc_miim_setup(dev, &bus, "mscc_miim", mii_regmap, 0); if (ret < 0) { diff --git a/drivers/net/mdio/mdio-mux-meson-g12a.c b/drivers/net/mdio/mdio-mux-meson-g12a.c index b8866bc3f2e8b7b77f5d3639f81e771ddd9c6779..4a2e94faf57e2f37980dbedc620743ce1a9fcbbe 100644 --- a/drivers/net/mdio/mdio-mux-meson-g12a.c +++ b/drivers/net/mdio/mdio-mux-meson-g12a.c @@ -233,11 +233,9 @@ static int g12a_ephy_glue_clk_register(struct device *dev) snprintf(in_name, sizeof(in_name), "clkin%d", i); clk = devm_clk_get(dev, in_name); - if (IS_ERR(clk)) { - if (PTR_ERR(clk) != -EPROBE_DEFER) - dev_err(dev, "Missing clock %s\n", in_name); - return PTR_ERR(clk); - } + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "Missing clock %s\n", in_name); parent_names[i] = __clk_get_name(clk); } @@ -317,12 +315,9 @@ static int g12a_mdio_mux_probe(struct platform_device *pdev) return PTR_ERR(priv->regs); priv->pclk = devm_clk_get(dev, "pclk"); - if (IS_ERR(priv->pclk)) { - ret = PTR_ERR(priv->pclk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get peripheral clock\n"); - return ret; - } + if (IS_ERR(priv->pclk)) + return dev_err_probe(dev, PTR_ERR(priv->pclk), + "failed to get peripheral clock\n"); /* Make sure the device registers are clocked */ ret = clk_prepare_enable(priv->pclk); @@ -339,8 +334,7 @@ static int g12a_mdio_mux_probe(struct platform_device *pdev) ret = mdio_mux_init(dev, dev->of_node, g12a_mdio_switch_fn, &priv->mux_handle, dev, NULL); if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "mdio multiplexer init failed: %d", ret); + dev_err_probe(dev, ret, "mdio multiplexer init failed\n"); goto err; } diff --git a/drivers/net/mdio/mdio-mux-mmioreg.c b/drivers/net/mdio/mdio-mux-mmioreg.c index c02fb2a067eef22dc34cee77fa372aef72382d86..c02c9c66001696c709e354f35f4a875cdf6869ae 100644 --- a/drivers/net/mdio/mdio-mux-mmioreg.c +++ b/drivers/net/mdio/mdio-mux-mmioreg.c @@ -159,12 +159,9 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev) ret = mdio_mux_init(&pdev->dev, pdev->dev.of_node, mdio_mux_mmioreg_switch_fn, &s->mux_handle, s, NULL); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, - "failed to register mdio-mux bus %pOF\n", np); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register mdio-mux bus %pOF\n", np); pdev->dev.platform_data = s; diff --git a/drivers/net/mdio/mdio-mux-multiplexer.c b/drivers/net/mdio/mdio-mux-multiplexer.c index 527acfc3c045a94b613a97b3d4e727f76c6b4a82..bfa5af577b0a2a056f21119223f4ac6909a48f25 100644 --- a/drivers/net/mdio/mdio-mux-multiplexer.c +++ b/drivers/net/mdio/mdio-mux-multiplexer.c @@ -72,12 +72,9 @@ static int mdio_mux_multiplexer_probe(struct platform_device *pdev) return -ENOMEM; s->muxc = devm_mux_control_get(dev, NULL); - if (IS_ERR(s->muxc)) { - ret = PTR_ERR(s->muxc); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Failed to get mux: %d\n", ret); - return ret; - } + if (IS_ERR(s->muxc)) + return dev_err_probe(&pdev->dev, PTR_ERR(s->muxc), + "Failed to get mux\n"); platform_set_drvdata(pdev, s); diff --git a/drivers/net/mdio/of_mdio.c b/drivers/net/mdio/of_mdio.c index 9e3c815a070f14c4c46f16b84dcfc705d9cd5865..796e9c7857d09428632649dde3d3d172657d8fec 100644 --- a/drivers/net/mdio/of_mdio.c +++ b/drivers/net/mdio/of_mdio.c @@ -231,6 +231,7 @@ int of_mdiobus_register(struct mii_bus *mdio, struct device_node *np) return 0; unregister: + of_node_put(child); mdiobus_unregister(mdio); return rc; } diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c index 21a0435c02de6c44e5175d0f204c255c0e22ddd5..7a28e082436e4a3a5a9e11a3711b666e5735c406 100644 --- a/drivers/net/net_failover.c +++ b/drivers/net/net_failover.c @@ -324,8 +324,8 @@ static const struct net_device_ops failover_dev_ops = { static void nfo_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, FAILOVER_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, FAILOVER_VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, FAILOVER_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, FAILOVER_VERSION, sizeof(drvinfo->version)); } static int nfo_ethtool_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index ddac61d79145ead755124358a22f7feebad3b1e2..bdff9ac5056dd6500b1ba4d4186ac3b22a08d892 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -55,7 +55,7 @@ MODULE_PARM_DESC(oops_only, "Only log oops messages"); #ifndef MODULE static int __init option_setup(char *opt) { - strlcpy(config, opt, MAX_PARAM_LENGTH); + strscpy(config, opt, MAX_PARAM_LENGTH); return 1; } __setup("netconsole=", option_setup); @@ -178,7 +178,7 @@ static struct netconsole_target *alloc_param_target(char *target_config) goto fail; nt->np.name = "netconsole"; - strlcpy(nt->np.dev_name, "eth0", IFNAMSIZ); + strscpy(nt->np.dev_name, "eth0", IFNAMSIZ); nt->np.local_port = 6665; nt->np.remote_port = 6666; eth_broadcast_addr(nt->np.remote_mac); @@ -414,7 +414,7 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf, return -EINVAL; } - strlcpy(nt->np.dev_name, buf, IFNAMSIZ); + strscpy(nt->np.dev_name, buf, IFNAMSIZ); /* Get rid of possible trailing newline from echo(1) */ len = strnlen(nt->np.dev_name, IFNAMSIZ); @@ -630,7 +630,7 @@ static struct config_item *make_netconsole_target(struct config_group *group, return ERR_PTR(-ENOMEM); nt->np.name = "netconsole"; - strlcpy(nt->np.dev_name, "eth0", IFNAMSIZ); + strscpy(nt->np.dev_name, "eth0", IFNAMSIZ); nt->np.local_port = 6665; nt->np.remote_port = 6666; eth_broadcast_addr(nt->np.remote_mac); @@ -708,7 +708,7 @@ restart: if (nt->np.dev == dev) { switch (event) { case NETDEV_CHANGENAME: - strlcpy(nt->np.dev_name, dev->name, IFNAMSIZ); + strscpy(nt->np.dev_name, dev->name, IFNAMSIZ); break; case NETDEV_RELEASE: case NETDEV_JOIN: diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index e88f783c297ebcdd3fd71528eeb4939ed20b525e..794fc0cc73b8870ce744dea2ef1b35d90cea2888 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -965,7 +965,6 @@ static int nsim_dev_reload_up(struct devlink *devlink, enum devlink_reload_actio struct netlink_ext_ack *extack) { struct nsim_dev *nsim_dev = devlink_priv(devlink); - int ret; if (nsim_dev->fail_reload) { /* For testing purposes, user set debugfs fail_reload @@ -976,15 +975,25 @@ static int nsim_dev_reload_up(struct devlink *devlink, enum devlink_reload_actio } *actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); - ret = nsim_dev_reload_create(nsim_dev, extack); - return ret; + + return nsim_dev_reload_create(nsim_dev, extack); } static int nsim_dev_info_get(struct devlink *devlink, struct devlink_info_req *req, struct netlink_ext_ack *extack) { - return devlink_info_driver_name_put(req, DRV_NAME); + int err; + + err = devlink_info_driver_name_put(req, DRV_NAME); + if (err) + return err; + err = devlink_info_version_stored_put_ext(req, "fw.mgmt", "10.20.30", + DEVLINK_INFO_VERSION_TYPE_COMPONENT); + if (err) + return err; + return devlink_info_version_running_put_ext(req, "fw.mgmt", "10.20.30", + DEVLINK_INFO_VERSION_TYPE_COMPONENT); } #define NSIM_DEV_FLASH_SIZE 500000 @@ -1312,8 +1321,7 @@ nsim_dev_devlink_trap_drop_counter_get(struct devlink *devlink, static const struct devlink_ops nsim_dev_devlink_ops = { .eswitch_mode_set = nsim_devlink_eswitch_mode_set, .eswitch_mode_get = nsim_devlink_eswitch_mode_get, - .supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT | - DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK, + .supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK, .reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT), .reload_down = nsim_dev_reload_down, .reload_up = nsim_dev_reload_up, diff --git a/drivers/net/netdevsim/hwstats.c b/drivers/net/netdevsim/hwstats.c index 605a38e16db056e0c35481075a5d438063f32272..0e58aa7f0374eefefe583b10f73266b19ba2a79c 100644 --- a/drivers/net/netdevsim/hwstats.c +++ b/drivers/net/netdevsim/hwstats.c @@ -433,11 +433,11 @@ int nsim_dev_hwstats_init(struct nsim_dev *nsim_dev) goto err_remove_hwstats_recursive; } - debugfs_create_file("enable_ifindex", 0600, hwstats->l3_ddir, hwstats, + debugfs_create_file("enable_ifindex", 0200, hwstats->l3_ddir, hwstats, &nsim_dev_hwstats_l3_enable_fops.fops); - debugfs_create_file("disable_ifindex", 0600, hwstats->l3_ddir, hwstats, + debugfs_create_file("disable_ifindex", 0200, hwstats->l3_ddir, hwstats, &nsim_dev_hwstats_l3_disable_fops.fops); - debugfs_create_file("fail_next_enable", 0600, hwstats->l3_ddir, hwstats, + debugfs_create_file("fail_next_enable", 0200, hwstats->l3_ddir, hwstats, &nsim_dev_hwstats_l3_fail_fops.fops); INIT_DELAYED_WORK(&hwstats->traffic_dw, diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index e470e3398abc27a20aaa573ce6f05bb2a8c0d244..9a1a5b20362408cb4b3ff43b11f2f9f8ebb3db99 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -67,10 +67,10 @@ nsim_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) unsigned int start; do { - start = u64_stats_fetch_begin(&ns->syncp); + start = u64_stats_fetch_begin_irq(&ns->syncp); stats->tx_bytes = ns->tx_bytes; stats->tx_packets = ns->tx_packets; - } while (u64_stats_fetch_retry(&ns->syncp, start)); + } while (u64_stats_fetch_retry_irq(&ns->syncp, start)); } static int diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c index 80bdc07f2cd33c8b68827db83d38d96ccba03709..464d88ca8ab0af9e5625a6d640629a80a037e09b 100644 --- a/drivers/net/ntb_netdev.c +++ b/drivers/net/ntb_netdev.c @@ -364,9 +364,9 @@ static void ntb_get_drvinfo(struct net_device *ndev, { struct ntb_netdev *dev = netdev_priv(ndev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->version, NTB_NETDEV_VER, sizeof(info->version)); - strlcpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->version, NTB_NETDEV_VER, sizeof(info->version)); + strscpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info)); } static int ntb_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/pcs/Kconfig b/drivers/net/pcs/Kconfig index 6289b7c765f13be49f4903ad3811b8d094f4725d..6e7e6c346a3ec551c91cf889b2902b7ef922d5a5 100644 --- a/drivers/net/pcs/Kconfig +++ b/drivers/net/pcs/Kconfig @@ -26,4 +26,10 @@ config PCS_RZN1_MIIC on RZ/N1 SoCs. This PCS converts MII to RMII/RGMII or can be set in pass-through mode for MII. +config PCS_ALTERA_TSE + tristate + help + This module provides helper functions for the Altera Triple Speed + Ethernet SGMII PCS, that can be found on the Intel Socfpga family. + endmenu diff --git a/drivers/net/pcs/Makefile b/drivers/net/pcs/Makefile index 0ff5388fcdeaf8f61457fc1dbc2eee90eba34c13..4c780d8f2e98453943329cf4277c4370759caacc 100644 --- a/drivers/net/pcs/Makefile +++ b/drivers/net/pcs/Makefile @@ -6,3 +6,4 @@ pcs_xpcs-$(CONFIG_PCS_XPCS) := pcs-xpcs.o pcs-xpcs-nxp.o obj-$(CONFIG_PCS_XPCS) += pcs_xpcs.o obj-$(CONFIG_PCS_LYNX) += pcs-lynx.o obj-$(CONFIG_PCS_RZN1_MIIC) += pcs-rzn1-miic.o +obj-$(CONFIG_PCS_ALTERA_TSE) += pcs-altera-tse.o diff --git a/drivers/net/pcs/pcs-altera-tse.c b/drivers/net/pcs/pcs-altera-tse.c new file mode 100644 index 0000000000000000000000000000000000000000..97a7cabff962a4db891c148a20ed77e5589a7775 --- /dev/null +++ b/drivers/net/pcs/pcs-altera-tse.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Bootlin + * + * Maxime Chevallier + */ + +#include +#include +#include +#include + +/* SGMII PCS register addresses + */ +#define SGMII_PCS_SCRATCH 0x10 +#define SGMII_PCS_REV 0x11 +#define SGMII_PCS_LINK_TIMER_0 0x12 +#define SGMII_PCS_LINK_TIMER_REG(x) (0x12 + (x)) +#define SGMII_PCS_LINK_TIMER_1 0x13 +#define SGMII_PCS_IF_MODE 0x14 +#define PCS_IF_MODE_SGMII_ENA BIT(0) +#define PCS_IF_MODE_USE_SGMII_AN BIT(1) +#define PCS_IF_MODE_SGMI_SPEED_MASK GENMASK(3, 2) +#define PCS_IF_MODE_SGMI_SPEED_10 (0 << 2) +#define PCS_IF_MODE_SGMI_SPEED_100 (1 << 2) +#define PCS_IF_MODE_SGMI_SPEED_1000 (2 << 2) +#define PCS_IF_MODE_SGMI_HALF_DUPLEX BIT(4) +#define PCS_IF_MODE_SGMI_PHY_AN BIT(5) +#define SGMII_PCS_DIS_READ_TO 0x15 +#define SGMII_PCS_READ_TO 0x16 +#define SGMII_PCS_SW_RESET_TIMEOUT 100 /* usecs */ + +struct altera_tse_pcs { + struct phylink_pcs pcs; + void __iomem *base; + int reg_width; +}; + +static struct altera_tse_pcs *phylink_pcs_to_tse_pcs(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct altera_tse_pcs, pcs); +} + +static u16 tse_pcs_read(struct altera_tse_pcs *tse_pcs, int regnum) +{ + if (tse_pcs->reg_width == 4) + return readl(tse_pcs->base + regnum * 4); + else + return readw(tse_pcs->base + regnum * 2); +} + +static void tse_pcs_write(struct altera_tse_pcs *tse_pcs, int regnum, + u16 value) +{ + if (tse_pcs->reg_width == 4) + writel(value, tse_pcs->base + regnum * 4); + else + writew(value, tse_pcs->base + regnum * 2); +} + +static int tse_pcs_reset(struct altera_tse_pcs *tse_pcs) +{ + int i = 0; + u16 bmcr; + + /* Reset PCS block */ + bmcr = tse_pcs_read(tse_pcs, MII_BMCR); + bmcr |= BMCR_RESET; + tse_pcs_write(tse_pcs, MII_BMCR, bmcr); + + for (i = 0; i < SGMII_PCS_SW_RESET_TIMEOUT; i++) { + if (!(tse_pcs_read(tse_pcs, MII_BMCR) & BMCR_RESET)) + return 0; + udelay(1); + } + + return -ETIMEDOUT; +} + +static int alt_tse_pcs_validate(struct phylink_pcs *pcs, + unsigned long *supported, + const struct phylink_link_state *state) +{ + if (state->interface == PHY_INTERFACE_MODE_SGMII || + state->interface == PHY_INTERFACE_MODE_1000BASEX) + return 1; + + return -EINVAL; +} + +static int alt_tse_pcs_config(struct phylink_pcs *pcs, unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs); + u32 ctrl, if_mode; + + ctrl = tse_pcs_read(tse_pcs, MII_BMCR); + if_mode = tse_pcs_read(tse_pcs, SGMII_PCS_IF_MODE); + + /* Set link timer to 1.6ms, as per the MegaCore Function User Guide */ + tse_pcs_write(tse_pcs, SGMII_PCS_LINK_TIMER_0, 0x0D40); + tse_pcs_write(tse_pcs, SGMII_PCS_LINK_TIMER_1, 0x03); + + if (interface == PHY_INTERFACE_MODE_SGMII) { + if_mode |= PCS_IF_MODE_USE_SGMII_AN | PCS_IF_MODE_SGMII_ENA; + } else if (interface == PHY_INTERFACE_MODE_1000BASEX) { + if_mode &= ~(PCS_IF_MODE_USE_SGMII_AN | PCS_IF_MODE_SGMII_ENA); + if_mode |= PCS_IF_MODE_SGMI_SPEED_1000; + } + + ctrl |= (BMCR_SPEED1000 | BMCR_FULLDPLX | BMCR_ANENABLE); + + tse_pcs_write(tse_pcs, MII_BMCR, ctrl); + tse_pcs_write(tse_pcs, SGMII_PCS_IF_MODE, if_mode); + + return tse_pcs_reset(tse_pcs); +} + +static void alt_tse_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs); + u16 bmsr, lpa; + + bmsr = tse_pcs_read(tse_pcs, MII_BMSR); + lpa = tse_pcs_read(tse_pcs, MII_LPA); + + phylink_mii_c22_pcs_decode_state(state, bmsr, lpa); +} + +static void alt_tse_pcs_an_restart(struct phylink_pcs *pcs) +{ + struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs); + u16 bmcr; + + bmcr = tse_pcs_read(tse_pcs, MII_BMCR); + bmcr |= BMCR_ANRESTART; + tse_pcs_write(tse_pcs, MII_BMCR, bmcr); + + /* This PCS seems to require a soft reset to re-sync the AN logic */ + tse_pcs_reset(tse_pcs); +} + +static const struct phylink_pcs_ops alt_tse_pcs_ops = { + .pcs_validate = alt_tse_pcs_validate, + .pcs_get_state = alt_tse_pcs_get_state, + .pcs_config = alt_tse_pcs_config, + .pcs_an_restart = alt_tse_pcs_an_restart, +}; + +struct phylink_pcs *alt_tse_pcs_create(struct net_device *ndev, + void __iomem *pcs_base, int reg_width) +{ + struct altera_tse_pcs *tse_pcs; + + if (reg_width != 4 && reg_width != 2) + return ERR_PTR(-EINVAL); + + tse_pcs = devm_kzalloc(&ndev->dev, sizeof(*tse_pcs), GFP_KERNEL); + if (!tse_pcs) + return ERR_PTR(-ENOMEM); + + tse_pcs->pcs.ops = &alt_tse_pcs_ops; + tse_pcs->base = pcs_base; + tse_pcs->reg_width = reg_width; + + return &tse_pcs->pcs; +} +EXPORT_SYMBOL_GPL(alt_tse_pcs_create); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Altera TSE PCS driver"); +MODULE_AUTHOR("Maxime Chevallier "); diff --git a/drivers/net/phy/adin.c b/drivers/net/phy/adin.c index ee374a85544a635f3711de9bca8083ff5221b2f3..134637584a83856ebce07e59a634d489655e2ed3 100644 --- a/drivers/net/phy/adin.c +++ b/drivers/net/phy/adin.c @@ -749,7 +749,7 @@ static void adin_get_strings(struct phy_device *phydev, u8 *data) int i; for (i = 0; i < ARRAY_SIZE(adin_hw_stats); i++) { - strlcpy(&data[i * ETH_GSTRING_LEN], + strscpy(&data[i * ETH_GSTRING_LEN], adin_hw_stats[i].string, ETH_GSTRING_LEN); } } diff --git a/drivers/net/phy/adin1100.c b/drivers/net/phy/adin1100.c index b6d139501199016e3271add67f69aca5b7f47010..7619d6185801c021fa76cb0ad4a9e56f1dbef6d3 100644 --- a/drivers/net/phy/adin1100.c +++ b/drivers/net/phy/adin1100.c @@ -15,6 +15,8 @@ #include #define PHY_ID_ADIN1100 0x0283bc81 +#define PHY_ID_ADIN1110 0x0283bc91 +#define PHY_ID_ADIN2111 0x0283bca1 #define ADIN_FORCED_MODE 0x8000 #define ADIN_FORCED_MODE_EN BIT(0) @@ -265,7 +267,8 @@ static int adin_probe(struct phy_device *phydev) static struct phy_driver adin_driver[] = { { - PHY_ID_MATCH_MODEL(PHY_ID_ADIN1100), + .phy_id = PHY_ID_ADIN1100, + .phy_id_mask = 0xffffffcf, .name = "ADIN1100", .get_features = adin_get_features, .soft_reset = adin_soft_reset, @@ -284,6 +287,8 @@ module_phy_driver(adin_driver); static struct mdio_device_id __maybe_unused adin_tbl[] = { { PHY_ID_MATCH_MODEL(PHY_ID_ADIN1100) }, + { PHY_ID_MATCH_MODEL(PHY_ID_ADIN1110) }, + { PHY_ID_MATCH_MODEL(PHY_ID_ADIN2111) }, { } }; diff --git a/drivers/net/phy/aquantia_main.c b/drivers/net/phy/aquantia_main.c index 8b7a46db30e0c39a5afd8ea85ac367bf722eee6e..47a76df36b74776b4fe60ff1096a8085c8fcbeb9 100644 --- a/drivers/net/phy/aquantia_main.c +++ b/drivers/net/phy/aquantia_main.c @@ -27,9 +27,12 @@ #define MDIO_PHYXS_VEND_IF_STATUS 0xe812 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK GENMASK(7, 3) #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR 0 +#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX 1 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI 2 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII 3 +#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI 4 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII 6 +#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI 7 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII 10 #define MDIO_AN_VEND_PROV 0xc400 @@ -91,6 +94,22 @@ #define VEND1_GLOBAL_FW_ID_MAJOR GENMASK(15, 8) #define VEND1_GLOBAL_FW_ID_MINOR GENMASK(7, 0) +#define VEND1_GLOBAL_GEN_STAT2 0xc831 +#define VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG BIT(15) + +/* The following registers all have similar layouts; first the registers... */ +#define VEND1_GLOBAL_CFG_10M 0x0310 +#define VEND1_GLOBAL_CFG_100M 0x031b +#define VEND1_GLOBAL_CFG_1G 0x031c +#define VEND1_GLOBAL_CFG_2_5G 0x031d +#define VEND1_GLOBAL_CFG_5G 0x031e +#define VEND1_GLOBAL_CFG_10G 0x031f +/* ...and now the fields */ +#define VEND1_GLOBAL_CFG_RATE_ADAPT GENMASK(8, 7) +#define VEND1_GLOBAL_CFG_RATE_ADAPT_NONE 0 +#define VEND1_GLOBAL_CFG_RATE_ADAPT_USX 1 +#define VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE 2 + #define VEND1_GLOBAL_RSVD_STAT1 0xc885 #define VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID GENMASK(7, 4) #define VEND1_GLOBAL_RSVD_STAT1_PROV_ID GENMASK(3, 0) @@ -125,6 +144,12 @@ #define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL2 BIT(1) #define VEND1_GLOBAL_INT_VEND_MASK_GLOBAL3 BIT(0) +/* Sleep and timeout for checking if the Processor-Intensive + * MDIO operation is finished + */ +#define AQR107_OP_IN_PROG_SLEEP 1000 +#define AQR107_OP_IN_PROG_TIMEOUT 100000 + struct aqr107_hw_stat { const char *name; int reg; @@ -335,40 +360,57 @@ static int aqr_read_status(struct phy_device *phydev) static int aqr107_read_rate(struct phy_device *phydev) { + u32 config_reg; int val; val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_STATUS1); if (val < 0) return val; + if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX) + phydev->duplex = DUPLEX_FULL; + else + phydev->duplex = DUPLEX_HALF; + switch (FIELD_GET(MDIO_AN_TX_VEND_STATUS1_RATE_MASK, val)) { case MDIO_AN_TX_VEND_STATUS1_10BASET: phydev->speed = SPEED_10; + config_reg = VEND1_GLOBAL_CFG_10M; break; case MDIO_AN_TX_VEND_STATUS1_100BASETX: phydev->speed = SPEED_100; + config_reg = VEND1_GLOBAL_CFG_100M; break; case MDIO_AN_TX_VEND_STATUS1_1000BASET: phydev->speed = SPEED_1000; + config_reg = VEND1_GLOBAL_CFG_1G; break; case MDIO_AN_TX_VEND_STATUS1_2500BASET: phydev->speed = SPEED_2500; + config_reg = VEND1_GLOBAL_CFG_2_5G; break; case MDIO_AN_TX_VEND_STATUS1_5000BASET: phydev->speed = SPEED_5000; + config_reg = VEND1_GLOBAL_CFG_5G; break; case MDIO_AN_TX_VEND_STATUS1_10GBASET: phydev->speed = SPEED_10000; + config_reg = VEND1_GLOBAL_CFG_10G; break; default: phydev->speed = SPEED_UNKNOWN; - break; + return 0; } - if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX) - phydev->duplex = DUPLEX_FULL; + val = phy_read_mmd(phydev, MDIO_MMD_VEND1, config_reg); + if (val < 0) + return val; + + if (FIELD_GET(VEND1_GLOBAL_CFG_RATE_ADAPT, val) == + VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE) + phydev->rate_matching = RATE_MATCH_PAUSE; else - phydev->duplex = DUPLEX_HALF; + phydev->rate_matching = RATE_MATCH_NONE; return 0; } @@ -392,15 +434,24 @@ static int aqr107_read_status(struct phy_device *phydev) case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR: phydev->interface = PHY_INTERFACE_MODE_10GKR; break; + case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX: + phydev->interface = PHY_INTERFACE_MODE_1000BASEKX; + break; case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI: phydev->interface = PHY_INTERFACE_MODE_10GBASER; break; case MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII: phydev->interface = PHY_INTERFACE_MODE_USXGMII; break; + case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI: + phydev->interface = PHY_INTERFACE_MODE_XAUI; + break; case MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII: phydev->interface = PHY_INTERFACE_MODE_SGMII; break; + case MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI: + phydev->interface = PHY_INTERFACE_MODE_RXAUI; + break; case MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII: phydev->interface = PHY_INTERFACE_MODE_2500BASEX; break; @@ -513,11 +564,14 @@ static int aqr107_config_init(struct phy_device *phydev) /* Check that the PHY interface type is compatible */ if (phydev->interface != PHY_INTERFACE_MODE_SGMII && + phydev->interface != PHY_INTERFACE_MODE_1000BASEKX && phydev->interface != PHY_INTERFACE_MODE_2500BASEX && phydev->interface != PHY_INTERFACE_MODE_XGMII && phydev->interface != PHY_INTERFACE_MODE_USXGMII && phydev->interface != PHY_INTERFACE_MODE_10GKR && - phydev->interface != PHY_INTERFACE_MODE_10GBASER) + phydev->interface != PHY_INTERFACE_MODE_10GBASER && + phydev->interface != PHY_INTERFACE_MODE_XAUI && + phydev->interface != PHY_INTERFACE_MODE_RXAUI) return -ENODEV; WARN(phydev->interface == PHY_INTERFACE_MODE_XGMII, @@ -597,16 +651,62 @@ static void aqr107_link_change_notify(struct phy_device *phydev) phydev_info(phydev, "Aquantia 1000Base-T2 mode active\n"); } +static int aqr107_wait_processor_intensive_op(struct phy_device *phydev) +{ + int val, err; + + /* The datasheet notes to wait at least 1ms after issuing a + * processor intensive operation before checking. + * We cannot use the 'sleep_before_read' parameter of read_poll_timeout + * because that just determines the maximum time slept, not the minimum. + */ + usleep_range(1000, 5000); + + err = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_VEND1, + VEND1_GLOBAL_GEN_STAT2, val, + !(val & VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG), + AQR107_OP_IN_PROG_SLEEP, + AQR107_OP_IN_PROG_TIMEOUT, false); + if (err) { + phydev_err(phydev, "timeout: processor-intensive MDIO operation\n"); + return err; + } + + return 0; +} + +static int aqr107_get_rate_matching(struct phy_device *phydev, + phy_interface_t iface) +{ + if (iface == PHY_INTERFACE_MODE_10GBASER || + iface == PHY_INTERFACE_MODE_2500BASEX || + iface == PHY_INTERFACE_MODE_NA) + return RATE_MATCH_PAUSE; + return RATE_MATCH_NONE; +} + static int aqr107_suspend(struct phy_device *phydev) { - return phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1, - MDIO_CTRL1_LPOWER); + int err; + + err = phy_set_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1, + MDIO_CTRL1_LPOWER); + if (err) + return err; + + return aqr107_wait_processor_intensive_op(phydev); } static int aqr107_resume(struct phy_device *phydev) { - return phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1, - MDIO_CTRL1_LPOWER); + int err; + + err = phy_clear_bits_mmd(phydev, MDIO_MMD_VEND1, MDIO_CTRL1, + MDIO_CTRL1_LPOWER); + if (err) + return err; + + return aqr107_wait_processor_intensive_op(phydev); } static int aqr107_probe(struct phy_device *phydev) @@ -658,6 +758,7 @@ static struct phy_driver aqr_driver[] = { PHY_ID_MATCH_MODEL(PHY_ID_AQR107), .name = "Aquantia AQR107", .probe = aqr107_probe, + .get_rate_matching = aqr107_get_rate_matching, .config_init = aqr107_config_init, .config_aneg = aqr_config_aneg, .config_intr = aqr_config_intr, @@ -676,6 +777,7 @@ static struct phy_driver aqr_driver[] = { PHY_ID_MATCH_MODEL(PHY_ID_AQCS109), .name = "Aquantia AQCS109", .probe = aqr107_probe, + .get_rate_matching = aqr107_get_rate_matching, .config_init = aqcs109_config_init, .config_aneg = aqr_config_aneg, .config_intr = aqr_config_intr, @@ -702,6 +804,7 @@ static struct phy_driver aqr_driver[] = { PHY_ID_MATCH_MODEL(PHY_ID_AQR113C), .name = "Aquantia AQR113C", .probe = aqr107_probe, + .get_rate_matching = aqr107_get_rate_matching, .config_init = aqr107_config_init, .config_aneg = aqr_config_aneg, .config_intr = aqr_config_intr, diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index 59fe356942b51e18c8c763d455d4b4d1da091e0b..349b7b1dbbf292fccee52f355d0a194b334f5a07 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -115,6 +115,7 @@ #define AT803X_DEBUG_REG_HIB_CTRL 0x0b #define AT803X_DEBUG_HIB_CTRL_SEL_RST_80U BIT(10) #define AT803X_DEBUG_HIB_CTRL_EN_ANY_CHANGE BIT(13) +#define AT803X_DEBUG_HIB_CTRL_PS_HIB_EN BIT(15) #define AT803X_DEBUG_REG_3C 0x3C @@ -192,6 +193,9 @@ #define AT803X_KEEP_PLL_ENABLED BIT(0) #define AT803X_DISABLE_SMARTEEE BIT(1) +/* disable hibernation mode */ +#define AT803X_DISABLE_HIBERNATION_MODE BIT(2) + /* ADC threshold */ #define QCA808X_PHY_DEBUG_ADC_THRESHOLD 0x2c80 #define QCA808X_ADC_THRESHOLD_MASK GENMASK(7, 0) @@ -672,6 +676,7 @@ static int at803x_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) struct phy_device *phydev = upstream; __ETHTOOL_DECLARE_LINK_MODE_MASK(phy_support); __ETHTOOL_DECLARE_LINK_MODE_MASK(sfp_support); + DECLARE_PHY_INTERFACE_MASK(interfaces); phy_interface_t iface; linkmode_zero(phy_support); @@ -682,7 +687,7 @@ static int at803x_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) phylink_set(phy_support, Asym_Pause); linkmode_zero(sfp_support); - sfp_parse_support(phydev->sfp_bus, id, sfp_support); + sfp_parse_support(phydev->sfp_bus, id, sfp_support, interfaces); /* Some modules support 10G modes as well as others we support. * Mask out non-supported modes so the correct interface is picked. */ @@ -730,6 +735,9 @@ static int at803x_parse_dt(struct phy_device *phydev) if (of_property_read_bool(node, "qca,disable-smarteee")) priv->flags |= AT803X_DISABLE_SMARTEEE; + if (of_property_read_bool(node, "qca,disable-hibernation-mode")) + priv->flags |= AT803X_DISABLE_HIBERNATION_MODE; + if (!of_property_read_u32(node, "qca,smarteee-tw-us-1g", &tw)) { if (!tw || tw > 255) { phydev_err(phydev, "invalid qca,smarteee-tw-us-1g\n"); @@ -999,6 +1007,20 @@ static int at8031_pll_config(struct phy_device *phydev) AT803X_DEBUG_PLL_ON, 0); } +static int at803x_hibernation_mode_config(struct phy_device *phydev) +{ + struct at803x_priv *priv = phydev->priv; + + /* The default after hardware reset is hibernation mode enabled. After + * software reset, the value is retained. + */ + if (!(priv->flags & AT803X_DISABLE_HIBERNATION_MODE)) + return 0; + + return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_HIB_CTRL, + AT803X_DEBUG_HIB_CTRL_PS_HIB_EN, 0); +} + static int at803x_config_init(struct phy_device *phydev) { struct at803x_priv *priv = phydev->priv; @@ -1051,6 +1073,10 @@ static int at803x_config_init(struct phy_device *phydev) if (ret < 0) return ret; + ret = at803x_hibernation_mode_config(phydev); + if (ret < 0) + return ret; + /* Ar803x extended next page bit is enabled by default. Cisco * multigig switches read this bit and attempt to negotiate 10Gbps * rates even if the next page bit is disabled. This is incorrect @@ -1732,7 +1758,7 @@ static int qca808x_phy_fast_retrain_config(struct phy_device *phydev) static int qca808x_phy_ms_random_seed_set(struct phy_device *phydev) { - u16 seed_value = (prandom_u32() % QCA808X_MASTER_SLAVE_SEED_RANGE); + u16 seed_value = prandom_u32_max(QCA808X_MASTER_SLAVE_SEED_RANGE); return at803x_debug_reg_mask(phydev, QCA808X_PHY_DEBUG_LOCAL_SEED, QCA808X_MASTER_SLAVE_SEED_CFG, diff --git a/drivers/net/phy/bcm-phy-lib.c b/drivers/net/phy/bcm-phy-lib.c index 287cccf8f7f4e5417d2750609e40ec9e393cee2a..b2c0baa51f39ebad65a51d65e0e9c0f12bc03133 100644 --- a/drivers/net/phy/bcm-phy-lib.c +++ b/drivers/net/phy/bcm-phy-lib.c @@ -519,7 +519,7 @@ void bcm_phy_get_strings(struct phy_device *phydev, u8 *data) unsigned int i; for (i = 0; i < ARRAY_SIZE(bcm_phy_hw_stats); i++) - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, bcm_phy_hw_stats[i].string, ETH_GSTRING_LEN); } EXPORT_SYMBOL_GPL(bcm_phy_get_strings); diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c index 31fbcdddc9ade625fd2713c34c655ff5c6c78a85..ad71c88c87e78d0185e0dd309787061895bd68f8 100644 --- a/drivers/net/phy/broadcom.c +++ b/drivers/net/phy/broadcom.c @@ -766,6 +766,41 @@ static irqreturn_t brcm_fet_handle_interrupt(struct phy_device *phydev) return IRQ_HANDLED; } +static int brcm_fet_suspend(struct phy_device *phydev) +{ + int reg, err, err2, brcmtest; + + /* We cannot use a read/modify/write here otherwise the PHY continues + * to drive LEDs which defeats the purpose of low power mode. + */ + err = phy_write(phydev, MII_BMCR, BMCR_PDOWN); + if (err < 0) + return err; + + /* Enable shadow register access */ + brcmtest = phy_read(phydev, MII_BRCM_FET_BRCMTEST); + if (brcmtest < 0) + return brcmtest; + + reg = brcmtest | MII_BRCM_FET_BT_SRE; + + err = phy_write(phydev, MII_BRCM_FET_BRCMTEST, reg); + if (err < 0) + return err; + + /* Set standby mode */ + err = phy_modify(phydev, MII_BRCM_FET_SHDW_AUXMODE4, + MII_BRCM_FET_SHDW_AM4_STANDBY, + MII_BRCM_FET_SHDW_AM4_STANDBY); + + /* Disable shadow register access */ + err2 = phy_write(phydev, MII_BRCM_FET_BRCMTEST, brcmtest); + if (!err) + err = err2; + + return err; +} + static int bcm54xx_phy_probe(struct phy_device *phydev) { struct bcm54xx_phy_priv *priv; @@ -1033,6 +1068,8 @@ static struct phy_driver broadcom_drivers[] = { .config_init = brcm_fet_config_init, .config_intr = brcm_fet_config_intr, .handle_interrupt = brcm_fet_handle_interrupt, + .suspend = brcm_fet_suspend, + .resume = brcm_fet_config_init, }, { .phy_id = PHY_ID_BCM5241, .phy_id_mask = 0xfffffff0, @@ -1041,6 +1078,8 @@ static struct phy_driver broadcom_drivers[] = { .config_init = brcm_fet_config_init, .config_intr = brcm_fet_config_intr, .handle_interrupt = brcm_fet_handle_interrupt, + .suspend = brcm_fet_suspend, + .resume = brcm_fet_config_init, }, { .phy_id = PHY_ID_BCM5395, .phy_id_mask = 0xfffffff0, diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c index f070776ca904a0c430f7b4414d3a94601c588b15..fd9ad482019242f37f1136842b516a22d0aa6054 100644 --- a/drivers/net/phy/marvell-88x2222.c +++ b/drivers/net/phy/marvell-88x2222.c @@ -478,6 +478,7 @@ static int mv2222_config_init(struct phy_device *phydev) static int mv2222_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) { + DECLARE_PHY_INTERFACE_MASK(interfaces); struct phy_device *phydev = upstream; phy_interface_t sfp_interface; struct mv2222_data *priv; @@ -489,7 +490,7 @@ static int mv2222_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) priv = (struct mv2222_data *)phydev->priv; dev = &phydev->mdio.dev; - sfp_parse_support(phydev->sfp_bus, id, sfp_supported); + sfp_parse_support(phydev->sfp_bus, id, sfp_supported, interfaces); phydev->port = sfp_parse_port(phydev->sfp_bus, id, sfp_supported); sfp_interface = sfp_select_interface(phydev->sfp_bus, sfp_supported); diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index a714150f5e8cda1c0610b7f40b817992537e4cdd..2810f4f9da0ccc8f5c3aa183f06edc336b64c5c1 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -1952,7 +1952,7 @@ static void marvell_get_strings(struct phy_device *phydev, u8 *data) int i; for (i = 0; i < count; i++) { - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, marvell_hw_stats[i].string, ETH_GSTRING_LEN); } } @@ -2845,6 +2845,7 @@ static int marvell_probe(struct phy_device *phydev) static int m88e1510_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) { + DECLARE_PHY_INTERFACE_MASK(interfaces); struct phy_device *phydev = upstream; phy_interface_t interface; struct device *dev; @@ -2856,7 +2857,7 @@ static int m88e1510_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) dev = &phydev->mdio.dev; - sfp_parse_support(phydev->sfp_bus, id, supported); + sfp_parse_support(phydev->sfp_bus, id, supported, interfaces); interface = sfp_select_interface(phydev->sfp_bus, supported); dev_info(dev, "%s SFP module inserted\n", phy_modes(interface)); diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c index 2b7d0720720b6b2aa38d28139e6db013909beb99..383a9c9f36e54499a7688c601b73a418f9367c6e 100644 --- a/drivers/net/phy/marvell10g.c +++ b/drivers/net/phy/marvell10g.c @@ -96,6 +96,11 @@ enum { MV_PCS_PORT_INFO_NPORTS_MASK = 0x0380, MV_PCS_PORT_INFO_NPORTS_SHIFT = 7, + /* SerDes reinitialization 88E21X0 */ + MV_AN_21X0_SERDES_CTRL2 = 0x800f, + MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS = BIT(13), + MV_AN_21X0_SERDES_CTRL2_RUN_INIT = BIT(15), + /* These registers appear at 0x800X and 0xa00X - the 0xa00X control * registers appear to set themselves to the 0x800X when AN is * restarted, but status registers appear readable from either. @@ -117,16 +122,16 @@ enum { MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_NO_SGMII_AN = 0x5, MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH = 0x6, MV_V2_33X0_PORT_CTRL_MACTYPE_USXGMII = 0x7, - MV_V2_PORT_INTR_STS = 0xf040, - MV_V2_PORT_INTR_MASK = 0xf043, - MV_V2_PORT_INTR_STS_WOL_EN = BIT(8), - MV_V2_MAGIC_PKT_WORD0 = 0xf06b, - MV_V2_MAGIC_PKT_WORD1 = 0xf06c, - MV_V2_MAGIC_PKT_WORD2 = 0xf06d, + MV_V2_PORT_INTR_STS = 0xf040, + MV_V2_PORT_INTR_MASK = 0xf043, + MV_V2_PORT_INTR_STS_WOL_EN = BIT(8), + MV_V2_MAGIC_PKT_WORD0 = 0xf06b, + MV_V2_MAGIC_PKT_WORD1 = 0xf06c, + MV_V2_MAGIC_PKT_WORD2 = 0xf06d, /* Wake on LAN registers */ - MV_V2_WOL_CTRL = 0xf06e, - MV_V2_WOL_CTRL_CLEAR_STS = BIT(15), - MV_V2_WOL_CTRL_MAGIC_PKT_EN = BIT(0), + MV_V2_WOL_CTRL = 0xf06e, + MV_V2_WOL_CTRL_CLEAR_STS = BIT(15), + MV_V2_WOL_CTRL_MAGIC_PKT_EN = BIT(0), /* Temperature control/read registers (88X3310 only) */ MV_V2_TEMP_CTRL = 0xf08a, MV_V2_TEMP_CTRL_MASK = 0xc000, @@ -140,6 +145,8 @@ struct mv3310_chip { bool (*has_downshift)(struct phy_device *phydev); void (*init_supported_interfaces)(unsigned long *mask); int (*get_mactype)(struct phy_device *phydev); + int (*set_mactype)(struct phy_device *phydev, int mactype); + int (*select_mactype)(unsigned long *interfaces); int (*init_interface)(struct phy_device *phydev, int mactype); #ifdef CONFIG_HWMON @@ -466,9 +473,10 @@ static int mv3310_sfp_insert(void *upstream, const struct sfp_eeprom_id *id) { struct phy_device *phydev = upstream; __ETHTOOL_DECLARE_LINK_MODE_MASK(support) = { 0, }; + DECLARE_PHY_INTERFACE_MASK(interfaces); phy_interface_t iface; - sfp_parse_support(phydev->sfp_bus, id, support); + sfp_parse_support(phydev->sfp_bus, id, support, interfaces); iface = sfp_select_interface(phydev->sfp_bus, support); if (iface != PHY_INTERFACE_MODE_10GBASER) { @@ -593,6 +601,49 @@ static int mv2110_get_mactype(struct phy_device *phydev) return mactype & MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK; } +static int mv2110_set_mactype(struct phy_device *phydev, int mactype) +{ + int err, val; + + mactype &= MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK; + err = phy_modify_mmd(phydev, MDIO_MMD_PMAPMD, MV_PMA_21X0_PORT_CTRL, + MV_PMA_21X0_PORT_CTRL_SWRST | + MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK, + MV_PMA_21X0_PORT_CTRL_SWRST | mactype); + if (err) + return err; + + err = phy_set_bits_mmd(phydev, MDIO_MMD_AN, MV_AN_21X0_SERDES_CTRL2, + MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS | + MV_AN_21X0_SERDES_CTRL2_RUN_INIT); + if (err) + return err; + + err = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_AN, + MV_AN_21X0_SERDES_CTRL2, val, + !(val & + MV_AN_21X0_SERDES_CTRL2_RUN_INIT), + 5000, 100000, true); + if (err) + return err; + + return phy_clear_bits_mmd(phydev, MDIO_MMD_AN, MV_AN_21X0_SERDES_CTRL2, + MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS); +} + +static int mv2110_select_mactype(unsigned long *interfaces) +{ + if (test_bit(PHY_INTERFACE_MODE_USXGMII, interfaces)) + return MV_PMA_21X0_PORT_CTRL_MACTYPE_USXGMII; + else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) && + !test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces)) + return MV_PMA_21X0_PORT_CTRL_MACTYPE_5GBASER; + else if (test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces)) + return MV_PMA_21X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH; + else + return -1; +} + static int mv3310_get_mactype(struct phy_device *phydev) { int mactype; @@ -604,6 +655,46 @@ static int mv3310_get_mactype(struct phy_device *phydev) return mactype & MV_V2_33X0_PORT_CTRL_MACTYPE_MASK; } +static int mv3310_set_mactype(struct phy_device *phydev, int mactype) +{ + int ret; + + mactype &= MV_V2_33X0_PORT_CTRL_MACTYPE_MASK; + ret = phy_modify_mmd_changed(phydev, MDIO_MMD_VEND2, MV_V2_PORT_CTRL, + MV_V2_33X0_PORT_CTRL_MACTYPE_MASK, + mactype); + if (ret <= 0) + return ret; + + return phy_set_bits_mmd(phydev, MDIO_MMD_VEND2, MV_V2_PORT_CTRL, + MV_V2_33X0_PORT_CTRL_SWRST); +} + +static int mv3310_select_mactype(unsigned long *interfaces) +{ + if (test_bit(PHY_INTERFACE_MODE_USXGMII, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_USXGMII; + else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) && + test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER; + else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) && + test_bit(PHY_INTERFACE_MODE_RXAUI, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_RXAUI; + else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) && + test_bit(PHY_INTERFACE_MODE_XAUI, interfaces)) + return MV_V2_3310_PORT_CTRL_MACTYPE_XAUI; + else if (test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH; + else if (test_bit(PHY_INTERFACE_MODE_RXAUI, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_RXAUI_RATE_MATCH; + else if (test_bit(PHY_INTERFACE_MODE_XAUI, interfaces)) + return MV_V2_3310_PORT_CTRL_MACTYPE_XAUI_RATE_MATCH; + else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces)) + return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER; + else + return -1; +} + static int mv2110_init_interface(struct phy_device *phydev, int mactype) { struct mv3310_priv *priv = dev_get_drvdata(&phydev->mdio.dev); @@ -687,6 +778,20 @@ static int mv3310_config_init(struct phy_device *phydev) if (err) return err; + /* If host provided host supported interface modes, try to select the + * best one + */ + if (!phy_interface_empty(phydev->host_interfaces)) { + mactype = chip->select_mactype(phydev->host_interfaces); + if (mactype >= 0) { + phydev_info(phydev, "Changing MACTYPE to %i\n", + mactype); + err = chip->set_mactype(phydev, mactype); + if (err) + return err; + } + } + mactype = chip->get_mactype(phydev); if (mactype < 0) return mactype; @@ -1049,6 +1154,8 @@ static const struct mv3310_chip mv3310_type = { .has_downshift = mv3310_has_downshift, .init_supported_interfaces = mv3310_init_supported_interfaces, .get_mactype = mv3310_get_mactype, + .set_mactype = mv3310_set_mactype, + .select_mactype = mv3310_select_mactype, .init_interface = mv3310_init_interface, #ifdef CONFIG_HWMON @@ -1060,6 +1167,8 @@ static const struct mv3310_chip mv3340_type = { .has_downshift = mv3310_has_downshift, .init_supported_interfaces = mv3340_init_supported_interfaces, .get_mactype = mv3310_get_mactype, + .set_mactype = mv3310_set_mactype, + .select_mactype = mv3310_select_mactype, .init_interface = mv3340_init_interface, #ifdef CONFIG_HWMON @@ -1070,6 +1179,8 @@ static const struct mv3310_chip mv3340_type = { static const struct mv3310_chip mv2110_type = { .init_supported_interfaces = mv2110_init_supported_interfaces, .get_mactype = mv2110_get_mactype, + .set_mactype = mv2110_set_mactype, + .select_mactype = mv2110_select_mactype, .init_interface = mv2110_init_interface, #ifdef CONFIG_HWMON @@ -1080,6 +1191,8 @@ static const struct mv3310_chip mv2110_type = { static const struct mv3310_chip mv2111_type = { .init_supported_interfaces = mv2111_init_supported_interfaces, .get_mactype = mv2110_get_mactype, + .set_mactype = mv2110_set_mactype, + .select_mactype = mv2110_select_mactype, .init_interface = mv2110_init_interface, #ifdef CONFIG_HWMON diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index 8a2dbe849866d1eb643719e04350745aea358837..f82090bdf7ab81633d51693caccf132596ee8903 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -232,7 +232,7 @@ static ssize_t mdio_bus_stat_field_show(struct device *dev, val = mdio_bus_get_stat(&bus->stats[sattr->addr], sattr->field_offset); - return sprintf(buf, "%llu\n", val); + return sysfs_emit(buf, "%llu\n", val); } static ssize_t mdio_bus_device_stat_field_show(struct device *dev, @@ -251,7 +251,7 @@ static ssize_t mdio_bus_device_stat_field_show(struct device *dev, val = mdio_bus_get_stat(&bus->stats[addr], sattr->field_offset); - return sprintf(buf, "%llu\n", val); + return sysfs_emit(buf, "%llu\n", val); } #define MDIO_BUS_STATS_ATTR_DECL(field, file) \ diff --git a/drivers/net/phy/meson-gxl.c b/drivers/net/phy/meson-gxl.c index 73f7962a37d335508b4a09e82d01132822c3b98f..c49062ad72c6c6922e18a26c68c2d96627da906e 100644 --- a/drivers/net/phy/meson-gxl.c +++ b/drivers/net/phy/meson-gxl.c @@ -243,13 +243,7 @@ static irqreturn_t meson_gxl_handle_interrupt(struct phy_device *phydev) irq_status == INTSRC_ENERGY_DETECT) return IRQ_HANDLED; - /* Give PHY some time before MAC starts sending data. This works - * around an issue where network doesn't come up properly. - */ - if (!(irq_status & INTSRC_LINK_DOWN)) - phy_queue_state_machine(phydev, msecs_to_jiffies(100)); - else - phy_trigger_machine(phydev); + phy_trigger_machine(phydev); return IRQ_HANDLED; } diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c index e78d0bf69bc3eec1259e802cb2fdf244cf82b2f6..54a17b576eac0a74320cfb0586979c2da2919cef 100644 --- a/drivers/net/phy/micrel.c +++ b/drivers/net/phy/micrel.c @@ -92,6 +92,15 @@ #define KSZ9x31_LMD_VCT_DATA_HI_PULSE_MASK GENMASK(1, 0) #define KSZ9x31_LMD_VCT_DATA_MASK GENMASK(7, 0) +#define KSZPHY_WIRE_PAIR_MASK 0x3 + +#define LAN8814_CABLE_DIAG 0x12 +#define LAN8814_CABLE_DIAG_STAT_MASK GENMASK(9, 8) +#define LAN8814_CABLE_DIAG_VCT_DATA_MASK GENMASK(7, 0) +#define LAN8814_PAIR_BIT_SHIFT 12 + +#define LAN8814_WIRE_PAIR_MASK 0xF + /* Lan8814 general Interrupt control/status reg in GPHY specific block. */ #define LAN8814_INTC 0x18 #define LAN8814_INTS 0x1B @@ -257,6 +266,8 @@ static struct kszphy_hw_stat kszphy_hw_stats[] = { struct kszphy_type { u32 led_mode_reg; u16 interrupt_level_mask; + u16 cable_diag_reg; + unsigned long pair_mask; bool has_broadcast_disable; bool has_nand_tree_disable; bool has_rmii_ref_clk_sel; @@ -313,6 +324,13 @@ struct kszphy_priv { static const struct kszphy_type lan8814_type = { .led_mode_reg = ~LAN8814_LED_CTRL_1, + .cable_diag_reg = LAN8814_CABLE_DIAG, + .pair_mask = LAN8814_WIRE_PAIR_MASK, +}; + +static const struct kszphy_type ksz886x_type = { + .cable_diag_reg = KSZ8081_LMD, + .pair_mask = KSZPHY_WIRE_PAIR_MASK, }; static const struct kszphy_type ksz8021_type = { @@ -1650,7 +1668,7 @@ static void kszphy_get_strings(struct phy_device *phydev, u8 *data) int i; for (i = 0; i < ARRAY_SIZE(kszphy_hw_stats); i++) { - strlcpy(data + i * ETH_GSTRING_LEN, + strscpy(data + i * ETH_GSTRING_LEN, kszphy_hw_stats[i].string, ETH_GSTRING_LEN); } } @@ -1796,6 +1814,17 @@ static int kszphy_probe(struct phy_device *phydev) return 0; } +static int lan8814_cable_test_start(struct phy_device *phydev) +{ + /* If autoneg is enabled, we won't be able to test cross pair + * short. In this case, the PHY will "detect" a link and + * confuse the internal state machine - disable auto neg here. + * Set the speed to 1000mbit and full duplex. + */ + return phy_modify(phydev, MII_BMCR, BMCR_ANENABLE | BMCR_SPEED100, + BMCR_SPEED1000 | BMCR_FULLDPLX); +} + static int ksz886x_cable_test_start(struct phy_device *phydev) { if (phydev->dev_flags & MICREL_KSZ8_P1_ERRATA) @@ -1809,9 +1838,9 @@ static int ksz886x_cable_test_start(struct phy_device *phydev) return phy_clear_bits(phydev, MII_BMCR, BMCR_ANENABLE | BMCR_SPEED100); } -static int ksz886x_cable_test_result_trans(u16 status) +static __always_inline int ksz886x_cable_test_result_trans(u16 status, u16 mask) { - switch (FIELD_GET(KSZ8081_LMD_STAT_MASK, status)) { + switch (FIELD_GET(mask, status)) { case KSZ8081_LMD_STAT_NORMAL: return ETHTOOL_A_CABLE_RESULT_CODE_OK; case KSZ8081_LMD_STAT_SHORT: @@ -1825,15 +1854,15 @@ static int ksz886x_cable_test_result_trans(u16 status) } } -static bool ksz886x_cable_test_failed(u16 status) +static __always_inline bool ksz886x_cable_test_failed(u16 status, u16 mask) { - return FIELD_GET(KSZ8081_LMD_STAT_MASK, status) == + return FIELD_GET(mask, status) == KSZ8081_LMD_STAT_FAIL; } -static bool ksz886x_cable_test_fault_length_valid(u16 status) +static __always_inline bool ksz886x_cable_test_fault_length_valid(u16 status, u16 mask) { - switch (FIELD_GET(KSZ8081_LMD_STAT_MASK, status)) { + switch (FIELD_GET(mask, status)) { case KSZ8081_LMD_STAT_OPEN: fallthrough; case KSZ8081_LMD_STAT_SHORT: @@ -1842,29 +1871,80 @@ static bool ksz886x_cable_test_fault_length_valid(u16 status) return false; } -static int ksz886x_cable_test_fault_length(u16 status) +static __always_inline int ksz886x_cable_test_fault_length(struct phy_device *phydev, + u16 status, u16 data_mask) { int dt; /* According to the data sheet the distance to the fault is - * DELTA_TIME * 0.4 meters. + * DELTA_TIME * 0.4 meters for ksz phys. + * (DELTA_TIME - 22) * 0.8 for lan8814 phy. */ - dt = FIELD_GET(KSZ8081_LMD_DELTA_TIME_MASK, status); + dt = FIELD_GET(data_mask, status); - return (dt * 400) / 10; + if ((phydev->phy_id & MICREL_PHY_ID_MASK) == PHY_ID_LAN8814) + return ((dt - 22) * 800) / 10; + else + return (dt * 400) / 10; } static int ksz886x_cable_test_wait_for_completion(struct phy_device *phydev) { + const struct kszphy_type *type = phydev->drv->driver_data; int val, ret; - ret = phy_read_poll_timeout(phydev, KSZ8081_LMD, val, + ret = phy_read_poll_timeout(phydev, type->cable_diag_reg, val, !(val & KSZ8081_LMD_ENABLE_TEST), 30000, 100000, true); return ret < 0 ? ret : 0; } +static int lan8814_cable_test_one_pair(struct phy_device *phydev, int pair) +{ + static const int ethtool_pair[] = { ETHTOOL_A_CABLE_PAIR_A, + ETHTOOL_A_CABLE_PAIR_B, + ETHTOOL_A_CABLE_PAIR_C, + ETHTOOL_A_CABLE_PAIR_D, + }; + u32 fault_length; + int ret; + int val; + + val = KSZ8081_LMD_ENABLE_TEST; + val = val | (pair << LAN8814_PAIR_BIT_SHIFT); + + ret = phy_write(phydev, LAN8814_CABLE_DIAG, val); + if (ret < 0) + return ret; + + ret = ksz886x_cable_test_wait_for_completion(phydev); + if (ret) + return ret; + + val = phy_read(phydev, LAN8814_CABLE_DIAG); + if (val < 0) + return val; + + if (ksz886x_cable_test_failed(val, LAN8814_CABLE_DIAG_STAT_MASK)) + return -EAGAIN; + + ret = ethnl_cable_test_result(phydev, ethtool_pair[pair], + ksz886x_cable_test_result_trans(val, + LAN8814_CABLE_DIAG_STAT_MASK + )); + if (ret) + return ret; + + if (!ksz886x_cable_test_fault_length_valid(val, LAN8814_CABLE_DIAG_STAT_MASK)) + return 0; + + fault_length = ksz886x_cable_test_fault_length(phydev, val, + LAN8814_CABLE_DIAG_VCT_DATA_MASK); + + return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair], fault_length); +} + static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair) { static const int ethtool_pair[] = { @@ -1872,6 +1952,7 @@ static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair) ETHTOOL_A_CABLE_PAIR_B, }; int ret, val, mdix; + u32 fault_length; /* There is no way to choice the pair, like we do one ksz9031. * We can workaround this limitation by using the MDI-X functionality. @@ -1910,25 +1991,27 @@ static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair) if (val < 0) return val; - if (ksz886x_cable_test_failed(val)) + if (ksz886x_cable_test_failed(val, KSZ8081_LMD_STAT_MASK)) return -EAGAIN; ret = ethnl_cable_test_result(phydev, ethtool_pair[pair], - ksz886x_cable_test_result_trans(val)); + ksz886x_cable_test_result_trans(val, KSZ8081_LMD_STAT_MASK)); if (ret) return ret; - if (!ksz886x_cable_test_fault_length_valid(val)) + if (!ksz886x_cable_test_fault_length_valid(val, KSZ8081_LMD_STAT_MASK)) return 0; - return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair], - ksz886x_cable_test_fault_length(val)); + fault_length = ksz886x_cable_test_fault_length(phydev, val, KSZ8081_LMD_DELTA_TIME_MASK); + + return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair], fault_length); } static int ksz886x_cable_test_get_status(struct phy_device *phydev, bool *finished) { - unsigned long pair_mask = 0x3; + const struct kszphy_type *type = phydev->drv->driver_data; + unsigned long pair_mask = type->pair_mask; int retries = 20; int pair, ret; @@ -1937,7 +2020,10 @@ static int ksz886x_cable_test_get_status(struct phy_device *phydev, /* Try harder if link partner is active */ while (pair_mask && retries--) { for_each_set_bit(pair, &pair_mask, 4) { - ret = ksz886x_cable_test_one_pair(phydev, pair); + if (type->cable_diag_reg == LAN8814_CABLE_DIAG) + ret = lan8814_cable_test_one_pair(phydev, pair); + else + ret = ksz886x_cable_test_one_pair(phydev, pair); if (ret == -EAGAIN) continue; if (ret < 0) @@ -2676,19 +2762,82 @@ static int lan8804_config_init(struct phy_device *phydev) return 0; } +static irqreturn_t lan8804_handle_interrupt(struct phy_device *phydev) +{ + int status; + + status = phy_read(phydev, LAN8814_INTS); + if (status < 0) { + phy_error(phydev); + return IRQ_NONE; + } + + if (status > 0) + phy_trigger_machine(phydev); + + return IRQ_HANDLED; +} + +#define LAN8804_OUTPUT_CONTROL 25 +#define LAN8804_OUTPUT_CONTROL_INTR_BUFFER BIT(14) +#define LAN8804_CONTROL 31 +#define LAN8804_CONTROL_INTR_POLARITY BIT(14) + +static int lan8804_config_intr(struct phy_device *phydev) +{ + int err; + + /* This is an internal PHY of lan966x and is not possible to change the + * polarity on the GIC found in lan966x, therefore change the polarity + * of the interrupt in the PHY from being active low instead of active + * high. + */ + phy_write(phydev, LAN8804_CONTROL, LAN8804_CONTROL_INTR_POLARITY); + + /* By default interrupt buffer is open-drain in which case the interrupt + * can be active only low. Therefore change the interrupt buffer to be + * push-pull to be able to change interrupt polarity + */ + phy_write(phydev, LAN8804_OUTPUT_CONTROL, + LAN8804_OUTPUT_CONTROL_INTR_BUFFER); + + if (phydev->interrupts == PHY_INTERRUPT_ENABLED) { + err = phy_read(phydev, LAN8814_INTS); + if (err < 0) + return err; + + err = phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK); + if (err) + return err; + } else { + err = phy_write(phydev, LAN8814_INTC, 0); + if (err) + return err; + + err = phy_read(phydev, LAN8814_INTS); + if (err < 0) + return err; + } + + return 0; +} + static irqreturn_t lan8814_handle_interrupt(struct phy_device *phydev) { int irq_status, tsu_irq_status; + int ret = IRQ_NONE; irq_status = phy_read(phydev, LAN8814_INTS); - if (irq_status > 0 && (irq_status & LAN8814_INT_LINK)) - phy_trigger_machine(phydev); - if (irq_status < 0) { phy_error(phydev); return IRQ_NONE; } + if (irq_status & LAN8814_INT_LINK) { + phy_trigger_machine(phydev); + ret = IRQ_HANDLED; + } + while (1) { tsu_irq_status = lanphy_read_page_reg(phydev, 4, LAN8814_INTR_STS_REG); @@ -2697,12 +2846,15 @@ static irqreturn_t lan8814_handle_interrupt(struct phy_device *phydev) (tsu_irq_status & (LAN8814_INTR_STS_REG_1588_TSU0_ | LAN8814_INTR_STS_REG_1588_TSU1_ | LAN8814_INTR_STS_REG_1588_TSU2_ | - LAN8814_INTR_STS_REG_1588_TSU3_))) + LAN8814_INTR_STS_REG_1588_TSU3_))) { lan8814_handle_ptp_interrupt(phydev); - else + ret = IRQ_HANDLED; + } else { break; + } } - return IRQ_HANDLED; + + return ret; } static int lan8814_ack_interrupt(struct phy_device *phydev) @@ -2729,9 +2881,9 @@ static int lan8814_config_intr(struct phy_device *phydev) if (err) return err; - err = phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK); + err = phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK); } else { - err = phy_write(phydev, LAN8814_INTC, 0); + err = phy_write(phydev, LAN8814_INTC, 0); if (err) return err; @@ -2873,12 +3025,18 @@ static int lan8814_config_init(struct phy_device *phydev) return 0; } +/* It is expected that there will not be any 'lan8814_take_coma_mode' + * function called in suspend. Because the GPIO line can be shared, so if one of + * the phys goes back in coma mode, then all the other PHYs will go, which is + * wrong. + */ static int lan8814_release_coma_mode(struct phy_device *phydev) { struct gpio_desc *gpiod; gpiod = devm_gpiod_get_optional(&phydev->mdio.dev, "coma-mode", - GPIOD_OUT_HIGH_OPEN_DRAIN); + GPIOD_OUT_HIGH_OPEN_DRAIN | + GPIOD_FLAGS_BIT_NONEXCLUSIVE); if (IS_ERR(gpiod)) return PTR_ERR(gpiod); @@ -3105,6 +3263,7 @@ static struct phy_driver ksphy_driver[] = { .phy_id = PHY_ID_LAN8814, .phy_id_mask = MICREL_PHY_ID_MASK, .name = "Microchip INDY Gigabit Quad PHY", + .flags = PHY_POLL_CABLE_TEST, .config_init = lan8814_config_init, .driver_data = &lan8814_type, .probe = lan8814_probe, @@ -3117,6 +3276,8 @@ static struct phy_driver ksphy_driver[] = { .resume = kszphy_resume, .config_intr = lan8814_config_intr, .handle_interrupt = lan8814_handle_interrupt, + .cable_test_start = lan8814_cable_test_start, + .cable_test_get_status = ksz886x_cable_test_get_status, }, { .phy_id = PHY_ID_LAN8804, .phy_id_mask = MICREL_PHY_ID_MASK, @@ -3131,6 +3292,8 @@ static struct phy_driver ksphy_driver[] = { .get_stats = kszphy_get_stats, .suspend = genphy_suspend, .resume = kszphy_resume, + .config_intr = lan8804_config_intr, + .handle_interrupt = lan8804_handle_interrupt, }, { .phy_id = PHY_ID_KSZ9131, .phy_id_mask = MICREL_PHY_ID_MASK, @@ -3163,6 +3326,7 @@ static struct phy_driver ksphy_driver[] = { .phy_id = PHY_ID_KSZ886X, .phy_id_mask = MICREL_PHY_ID_MASK, .name = "Micrel KSZ8851 Ethernet MAC or KSZ886X Switch", + .driver_data = &ksz886x_type, /* PHY_BASIC_FEATURES */ .flags = PHY_POLL_CABLE_TEST, .config_init = kszphy_config_init, @@ -3185,6 +3349,8 @@ static struct phy_driver ksphy_driver[] = { .name = "Microchip KSZ9477", /* PHY_GBIT_FEATURES */ .config_init = kszphy_config_init, + .config_intr = kszphy_config_intr, + .handle_interrupt = kszphy_handle_interrupt, .suspend = genphy_suspend, .resume = genphy_resume, } }; diff --git a/drivers/net/phy/microchip_t1.c b/drivers/net/phy/microchip_t1.c index d4c93d59bc5390593c89b4b0315fbdab4ae84dce..8569a545e0a3fa5e9fda6ae38e801a90c8308ff4 100644 --- a/drivers/net/phy/microchip_t1.c +++ b/drivers/net/phy/microchip_t1.c @@ -28,12 +28,16 @@ /* Interrupt Source Register */ #define LAN87XX_INTERRUPT_SOURCE (0x18) +#define LAN87XX_INTERRUPT_SOURCE_2 (0x08) /* Interrupt Mask Register */ #define LAN87XX_INTERRUPT_MASK (0x19) #define LAN87XX_MASK_LINK_UP (0x0004) #define LAN87XX_MASK_LINK_DOWN (0x0002) +#define LAN87XX_INTERRUPT_MASK_2 (0x09) +#define LAN87XX_MASK_COMM_RDY BIT(10) + /* MISC Control 1 Register */ #define LAN87XX_CTRL_1 (0x11) #define LAN87XX_MASK_RGMII_TXC_DLY_EN (0x4000) @@ -424,17 +428,55 @@ static int lan87xx_phy_config_intr(struct phy_device *phydev) int rc, val = 0; if (phydev->interrupts == PHY_INTERRUPT_ENABLED) { - /* unmask all source and clear them before enable */ - rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, 0x7FFF); + /* clear all interrupt */ + rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, val); + if (rc < 0) + return rc; + rc = phy_read(phydev, LAN87XX_INTERRUPT_SOURCE); - val = LAN87XX_MASK_LINK_UP | LAN87XX_MASK_LINK_DOWN; + if (rc < 0) + return rc; + + rc = access_ereg(phydev, PHYACC_ATTR_MODE_WRITE, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_MASK_2, val); + if (rc < 0) + return rc; + + rc = access_ereg(phydev, PHYACC_ATTR_MODE_READ, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_SOURCE_2, 0); + if (rc < 0) + return rc; + + /* enable link down and comm ready interrupt */ + val = LAN87XX_MASK_LINK_DOWN; rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, val); + if (rc < 0) + return rc; + + val = LAN87XX_MASK_COMM_RDY; + rc = access_ereg(phydev, PHYACC_ATTR_MODE_WRITE, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_MASK_2, val); } else { rc = phy_write(phydev, LAN87XX_INTERRUPT_MASK, val); - if (rc) + if (rc < 0) return rc; rc = phy_read(phydev, LAN87XX_INTERRUPT_SOURCE); + if (rc < 0) + return rc; + + rc = access_ereg(phydev, PHYACC_ATTR_MODE_WRITE, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_MASK_2, val); + if (rc < 0) + return rc; + + rc = access_ereg(phydev, PHYACC_ATTR_MODE_READ, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_SOURCE_2, 0); } return rc < 0 ? rc : 0; @@ -444,6 +486,14 @@ static irqreturn_t lan87xx_handle_interrupt(struct phy_device *phydev) { int irq_status; + irq_status = access_ereg(phydev, PHYACC_ATTR_MODE_READ, + PHYACC_ATTR_BANK_MISC, + LAN87XX_INTERRUPT_SOURCE_2, 0); + if (irq_status < 0) { + phy_error(phydev); + return IRQ_NONE; + } + irq_status = phy_read(phydev, LAN87XX_INTERRUPT_SOURCE); if (irq_status < 0) { phy_error(phydev); diff --git a/drivers/net/phy/mscc/mscc_macsec.c b/drivers/net/phy/mscc/mscc_macsec.c index b7b2521c73fb629b341ea89a48dc43fb6458d930..ee5b17edca39e518e74081f9c0abe4b63e47cddd 100644 --- a/drivers/net/phy/mscc/mscc_macsec.c +++ b/drivers/net/phy/mscc/mscc_macsec.c @@ -706,14 +706,6 @@ static int __vsc8584_macsec_add_rxsa(struct macsec_context *ctx, struct phy_device *phydev = ctx->phydev; struct vsc8531_private *priv = phydev->priv; - if (!flow) { - flow = vsc8584_macsec_alloc_flow(priv, MACSEC_INGR); - if (IS_ERR(flow)) - return PTR_ERR(flow); - - memcpy(flow->key, ctx->sa.key, priv->secy->key_len); - } - flow->assoc_num = ctx->sa.assoc_num; flow->rx_sa = ctx->sa.rx_sa; @@ -730,24 +722,13 @@ static int __vsc8584_macsec_add_rxsa(struct macsec_context *ctx, static int __vsc8584_macsec_add_txsa(struct macsec_context *ctx, struct macsec_flow *flow, bool update) { - struct phy_device *phydev = ctx->phydev; - struct vsc8531_private *priv = phydev->priv; - - if (!flow) { - flow = vsc8584_macsec_alloc_flow(priv, MACSEC_EGR); - if (IS_ERR(flow)) - return PTR_ERR(flow); - - memcpy(flow->key, ctx->sa.key, priv->secy->key_len); - } - flow->assoc_num = ctx->sa.assoc_num; flow->tx_sa = ctx->sa.tx_sa; /* Always match untagged packets on egress */ flow->match.untagged = 1; - return vsc8584_macsec_add_flow(phydev, flow, update); + return vsc8584_macsec_add_flow(ctx->phydev, flow, update); } static int vsc8584_macsec_dev_open(struct macsec_context *ctx) @@ -755,10 +736,6 @@ static int vsc8584_macsec_dev_open(struct macsec_context *ctx) struct vsc8531_private *priv = ctx->phydev->priv; struct macsec_flow *flow, *tmp; - /* No operation to perform before the commit step */ - if (ctx->prepare) - return 0; - list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list) vsc8584_macsec_flow_enable(ctx->phydev, flow); @@ -770,10 +747,6 @@ static int vsc8584_macsec_dev_stop(struct macsec_context *ctx) struct vsc8531_private *priv = ctx->phydev->priv; struct macsec_flow *flow, *tmp; - /* No operation to perform before the commit step */ - if (ctx->prepare) - return 0; - list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list) vsc8584_macsec_flow_disable(ctx->phydev, flow); @@ -785,12 +758,8 @@ static int vsc8584_macsec_add_secy(struct macsec_context *ctx) struct vsc8531_private *priv = ctx->phydev->priv; struct macsec_secy *secy = ctx->secy; - if (ctx->prepare) { - if (priv->secy) - return -EEXIST; - - return 0; - } + if (priv->secy) + return -EEXIST; priv->secy = secy; @@ -807,10 +776,6 @@ static int vsc8584_macsec_del_secy(struct macsec_context *ctx) struct vsc8531_private *priv = ctx->phydev->priv; struct macsec_flow *flow, *tmp; - /* No operation to perform before the commit step */ - if (ctx->prepare) - return 0; - list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list) vsc8584_macsec_del_flow(ctx->phydev, flow); @@ -823,10 +788,6 @@ static int vsc8584_macsec_del_secy(struct macsec_context *ctx) static int vsc8584_macsec_upd_secy(struct macsec_context *ctx) { - /* No operation to perform before the commit step */ - if (ctx->prepare) - return 0; - vsc8584_macsec_del_secy(ctx); return vsc8584_macsec_add_secy(ctx); } @@ -847,10 +808,6 @@ static int vsc8584_macsec_del_rxsc(struct macsec_context *ctx) struct vsc8531_private *priv = ctx->phydev->priv; struct macsec_flow *flow, *tmp; - /* No operation to perform before the commit step */ - if (ctx->prepare) - return 0; - list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list) { if (flow->bank == MACSEC_INGR && flow->rx_sa && flow->rx_sa->sc->sci == ctx->rx_sc->sci) @@ -862,33 +819,40 @@ static int vsc8584_macsec_del_rxsc(struct macsec_context *ctx) static int vsc8584_macsec_add_rxsa(struct macsec_context *ctx) { - struct macsec_flow *flow = NULL; - - if (ctx->prepare) - return __vsc8584_macsec_add_rxsa(ctx, flow, false); + struct phy_device *phydev = ctx->phydev; + struct vsc8531_private *priv = phydev->priv; + struct macsec_flow *flow; + int ret; - flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR); + flow = vsc8584_macsec_alloc_flow(priv, MACSEC_INGR); if (IS_ERR(flow)) return PTR_ERR(flow); - vsc8584_macsec_flow_enable(ctx->phydev, flow); + memcpy(flow->key, ctx->sa.key, priv->secy->key_len); + + ret = __vsc8584_macsec_add_rxsa(ctx, flow, false); + if (ret) + return ret; + + vsc8584_macsec_flow_enable(phydev, flow); return 0; } static int vsc8584_macsec_upd_rxsa(struct macsec_context *ctx) { struct macsec_flow *flow; + int ret; flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR); if (IS_ERR(flow)) return PTR_ERR(flow); - if (ctx->prepare) { - /* Make sure the flow is disabled before updating it */ - vsc8584_macsec_flow_disable(ctx->phydev, flow); + /* Make sure the flow is disabled before updating it */ + vsc8584_macsec_flow_disable(ctx->phydev, flow); - return __vsc8584_macsec_add_rxsa(ctx, flow, true); - } + ret = __vsc8584_macsec_add_rxsa(ctx, flow, true); + if (ret) + return ret; vsc8584_macsec_flow_enable(ctx->phydev, flow); return 0; @@ -899,11 +863,8 @@ static int vsc8584_macsec_del_rxsa(struct macsec_context *ctx) struct macsec_flow *flow; flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR); - if (IS_ERR(flow)) return PTR_ERR(flow); - if (ctx->prepare) - return 0; vsc8584_macsec_del_flow(ctx->phydev, flow); return 0; @@ -911,33 +872,40 @@ static int vsc8584_macsec_del_rxsa(struct macsec_context *ctx) static int vsc8584_macsec_add_txsa(struct macsec_context *ctx) { - struct macsec_flow *flow = NULL; - - if (ctx->prepare) - return __vsc8584_macsec_add_txsa(ctx, flow, false); + struct phy_device *phydev = ctx->phydev; + struct vsc8531_private *priv = phydev->priv; + struct macsec_flow *flow; + int ret; - flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR); + flow = vsc8584_macsec_alloc_flow(priv, MACSEC_EGR); if (IS_ERR(flow)) return PTR_ERR(flow); - vsc8584_macsec_flow_enable(ctx->phydev, flow); + memcpy(flow->key, ctx->sa.key, priv->secy->key_len); + + ret = __vsc8584_macsec_add_txsa(ctx, flow, false); + if (ret) + return ret; + + vsc8584_macsec_flow_enable(phydev, flow); return 0; } static int vsc8584_macsec_upd_txsa(struct macsec_context *ctx) { struct macsec_flow *flow; + int ret; flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR); if (IS_ERR(flow)) return PTR_ERR(flow); - if (ctx->prepare) { - /* Make sure the flow is disabled before updating it */ - vsc8584_macsec_flow_disable(ctx->phydev, flow); + /* Make sure the flow is disabled before updating it */ + vsc8584_macsec_flow_disable(ctx->phydev, flow); - return __vsc8584_macsec_add_txsa(ctx, flow, true); - } + ret = __vsc8584_macsec_add_txsa(ctx, flow, true); + if (ret) + return ret; vsc8584_macsec_flow_enable(ctx->phydev, flow); return 0; @@ -948,11 +916,8 @@ static int vsc8584_macsec_del_txsa(struct macsec_context *ctx) struct macsec_flow *flow; flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR); - if (IS_ERR(flow)) return PTR_ERR(flow); - if (ctx->prepare) - return 0; vsc8584_macsec_del_flow(ctx->phydev, flow); return 0; diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c index 7e3017e7a1c0324746f1e53a068a70eddacdbea3..8a13b1ad9a330b599742fc40a5d540ec3dcc0c3f 100644 --- a/drivers/net/phy/mscc/mscc_main.c +++ b/drivers/net/phy/mscc/mscc_main.c @@ -136,7 +136,7 @@ static void vsc85xx_get_strings(struct phy_device *phydev, u8 *data) return; for (i = 0; i < priv->nstats; i++) - strlcpy(data + i * ETH_GSTRING_LEN, priv->hw_stats[i].string, + strscpy(data + i * ETH_GSTRING_LEN, priv->hw_stats[i].string, ETH_GSTRING_LEN); } diff --git a/drivers/net/phy/nxp-tja11xx.c b/drivers/net/phy/nxp-tja11xx.c index 2a8195c50d147860325c1e4e972a46204efb489b..ec91e671f8aa6a169e7e7931bb356346f6da425a 100644 --- a/drivers/net/phy/nxp-tja11xx.c +++ b/drivers/net/phy/nxp-tja11xx.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,11 @@ #define MII_CFG1 18 #define MII_CFG1_MASTER_SLAVE BIT(15) #define MII_CFG1_AUTO_OP BIT(14) +#define MII_CFG1_INTERFACE_MODE_MASK GENMASK(9, 8) +#define MII_CFG1_MII_MODE (0x0 << 8) +#define MII_CFG1_RMII_MODE_REFCLK_IN BIT(8) +#define MII_CFG1_RMII_MODE_REFCLK_OUT BIT(9) +#define MII_CFG1_REVMII_MODE GENMASK(9, 8) #define MII_CFG1_SLEEP_CONFIRM BIT(6) #define MII_CFG1_LED_MODE_MASK GENMASK(5, 4) #define MII_CFG1_LED_MODE_LINKUP 0 @@ -72,11 +78,15 @@ #define MII_COMMCFG 27 #define MII_COMMCFG_AUTO_OP BIT(15) +/* Configure REF_CLK as input in RMII mode */ +#define TJA110X_RMII_MODE_REFCLK_IN BIT(0) + struct tja11xx_priv { char *hwmon_name; struct device *hwmon_dev; struct phy_device *phydev; struct work_struct phy_register_work; + u32 flags; }; struct tja11xx_phy_stats { @@ -251,8 +261,34 @@ do_test: return __genphy_config_aneg(phydev, changed); } +static int tja11xx_get_interface_mode(struct phy_device *phydev) +{ + struct tja11xx_priv *priv = phydev->priv; + int mii_mode; + + switch (phydev->interface) { + case PHY_INTERFACE_MODE_MII: + mii_mode = MII_CFG1_MII_MODE; + break; + case PHY_INTERFACE_MODE_REVMII: + mii_mode = MII_CFG1_REVMII_MODE; + break; + case PHY_INTERFACE_MODE_RMII: + if (priv->flags & TJA110X_RMII_MODE_REFCLK_IN) + mii_mode = MII_CFG1_RMII_MODE_REFCLK_IN; + else + mii_mode = MII_CFG1_RMII_MODE_REFCLK_OUT; + break; + default: + return -EINVAL; + } + + return mii_mode; +} + static int tja11xx_config_init(struct phy_device *phydev) { + u16 reg_mask, reg_val; int ret; ret = tja11xx_enable_reg_write(phydev); @@ -265,15 +301,32 @@ static int tja11xx_config_init(struct phy_device *phydev) switch (phydev->phy_id & PHY_ID_MASK) { case PHY_ID_TJA1100: - ret = phy_modify(phydev, MII_CFG1, - MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_MASK | - MII_CFG1_LED_ENABLE, - MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_LINKUP | - MII_CFG1_LED_ENABLE); + reg_mask = MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_MASK | + MII_CFG1_LED_ENABLE; + reg_val = MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_LINKUP | + MII_CFG1_LED_ENABLE; + + reg_mask |= MII_CFG1_INTERFACE_MODE_MASK; + ret = tja11xx_get_interface_mode(phydev); + if (ret < 0) + return ret; + + reg_val |= (ret & 0xffff); + ret = phy_modify(phydev, MII_CFG1, reg_mask, reg_val); if (ret) return ret; break; case PHY_ID_TJA1101: + reg_mask = MII_CFG1_INTERFACE_MODE_MASK; + ret = tja11xx_get_interface_mode(phydev); + if (ret < 0) + return ret; + + reg_val = ret & 0xffff; + ret = phy_modify(phydev, MII_CFG1, reg_mask, reg_val); + if (ret) + return ret; + fallthrough; case PHY_ID_TJA1102: ret = phy_set_bits(phydev, MII_COMMCFG, MII_COMMCFG_AUTO_OP); if (ret) @@ -458,16 +511,36 @@ static int tja11xx_hwmon_register(struct phy_device *phydev, return PTR_ERR_OR_ZERO(priv->hwmon_dev); } +static int tja11xx_parse_dt(struct phy_device *phydev) +{ + struct device_node *node = phydev->mdio.dev.of_node; + struct tja11xx_priv *priv = phydev->priv; + + if (!IS_ENABLED(CONFIG_OF_MDIO)) + return 0; + + if (of_property_read_bool(node, "nxp,rmii-refclk-in")) + priv->flags |= TJA110X_RMII_MODE_REFCLK_IN; + + return 0; +} + static int tja11xx_probe(struct phy_device *phydev) { struct device *dev = &phydev->mdio.dev; struct tja11xx_priv *priv; + int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; priv->phydev = phydev; + phydev->priv = priv; + + ret = tja11xx_parse_dt(phydev); + if (ret) + return ret; return tja11xx_hwmon_register(phydev, priv); } diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c index 1f2531a1a87672cf0e59fffa15f59415728c3a02..2c8bf438ea617aeae2e0d247a3d84ab29dd897df 100644 --- a/drivers/net/phy/phy-core.c +++ b/drivers/net/phy/phy-core.c @@ -74,6 +74,80 @@ const char *phy_duplex_to_str(unsigned int duplex) } EXPORT_SYMBOL_GPL(phy_duplex_to_str); +/** + * phy_rate_matching_to_str - Return a string describing the rate matching + * + * @rate_matching: Type of rate matching to describe + */ +const char *phy_rate_matching_to_str(int rate_matching) +{ + switch (rate_matching) { + case RATE_MATCH_NONE: + return "none"; + case RATE_MATCH_PAUSE: + return "pause"; + case RATE_MATCH_CRS: + return "crs"; + case RATE_MATCH_OPEN_LOOP: + return "open-loop"; + } + return "Unsupported (update phy-core.c)"; +} +EXPORT_SYMBOL_GPL(phy_rate_matching_to_str); + +/** + * phy_interface_num_ports - Return the number of links that can be carried by + * a given MAC-PHY physical link. Returns 0 if this is + * unknown, the number of links else. + * + * @interface: The interface mode we want to get the number of ports + */ +int phy_interface_num_ports(phy_interface_t interface) +{ + switch (interface) { + case PHY_INTERFACE_MODE_NA: + return 0; + case PHY_INTERFACE_MODE_INTERNAL: + case PHY_INTERFACE_MODE_MII: + case PHY_INTERFACE_MODE_GMII: + case PHY_INTERFACE_MODE_TBI: + case PHY_INTERFACE_MODE_REVMII: + case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_REVRMII: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RTBI: + case PHY_INTERFACE_MODE_XGMII: + case PHY_INTERFACE_MODE_XLGMII: + case PHY_INTERFACE_MODE_MOCA: + case PHY_INTERFACE_MODE_TRGMII: + case PHY_INTERFACE_MODE_USXGMII: + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_SMII: + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_2500BASEX: + case PHY_INTERFACE_MODE_5GBASER: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_25GBASER: + case PHY_INTERFACE_MODE_10GKR: + case PHY_INTERFACE_MODE_100BASEX: + case PHY_INTERFACE_MODE_RXAUI: + case PHY_INTERFACE_MODE_XAUI: + case PHY_INTERFACE_MODE_1000BASEKX: + return 1; + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: + return 4; + case PHY_INTERFACE_MODE_MAX: + WARN_ONCE(1, "PHY_INTERFACE_MODE_MAX isn't a valid interface mode"); + return 0; + } + return 0; +} +EXPORT_SYMBOL_GPL(phy_interface_num_ports); + /* A mapping of all SUPPORTED settings to speed/duplex. This table * must be grouped by speed and sorted in descending match priority * - iow, descending speed. diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c index 8d3ee3a6495b45754095c8c8a0c842ced8a26434..e741d8aebffe187fe2973cdef2e462b1eebeed82 100644 --- a/drivers/net/phy/phy.c +++ b/drivers/net/phy/phy.c @@ -114,6 +114,33 @@ void phy_print_status(struct phy_device *phydev) } EXPORT_SYMBOL(phy_print_status); +/** + * phy_get_rate_matching - determine if rate matching is supported + * @phydev: The phy device to return rate matching for + * @iface: The interface mode to use + * + * This determines the type of rate matching (if any) that @phy supports + * using @iface. @iface may be %PHY_INTERFACE_MODE_NA to determine if any + * interface supports rate matching. + * + * Return: The type of rate matching @phy supports for @iface, or + * %RATE_MATCH_NONE. + */ +int phy_get_rate_matching(struct phy_device *phydev, + phy_interface_t iface) +{ + int ret = RATE_MATCH_NONE; + + if (phydev->drv->get_rate_matching) { + mutex_lock(&phydev->lock); + ret = phydev->drv->get_rate_matching(phydev, iface); + mutex_unlock(&phydev->lock); + } + + return ret; +} +EXPORT_SYMBOL_GPL(phy_get_rate_matching); + /** * phy_config_interrupt - configure the PHY device for the requested interrupts * @phydev: the phy_device struct @@ -256,6 +283,7 @@ void phy_ethtool_ksettings_get(struct phy_device *phydev, cmd->base.duplex = phydev->duplex; cmd->base.master_slave_cfg = phydev->master_slave_get; cmd->base.master_slave_state = phydev->master_slave_state; + cmd->base.rate_matching = phydev->rate_matching; if (phydev->interface == PHY_INTERFACE_MODE_MOCA) cmd->base.port = PORT_BNC; else diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 0c6efd79269071ec5844940aa0c9dbcf1e09b49f..57849ac0384eff810c034855f1dbba5c0554a88e 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -316,11 +317,13 @@ static __maybe_unused int mdio_bus_phy_resume(struct device *dev) phydev->suspended_by_mdio_bus = 0; - /* If we managed to get here with the PHY state machine in a state other - * than PHY_HALTED this is an indication that something went wrong and - * we should most likely be using MAC managed PM and we are not. + /* If we managed to get here with the PHY state machine in a state + * neither PHY_HALTED, PHY_READY nor PHY_UP, this is an indication + * that something went wrong and we should most likely be using + * MAC managed PM, but we are not. */ - WARN_ON(phydev->state != PHY_HALTED && !phydev->mac_managed_pm); + WARN_ON(phydev->state != PHY_HALTED && phydev->state != PHY_READY && + phydev->state != PHY_UP); ret = phy_init_hw(phydev); if (ret < 0) @@ -370,7 +373,7 @@ int phy_register_fixup(const char *bus_id, u32 phy_uid, u32 phy_uid_mask, if (!fixup) return -ENOMEM; - strlcpy(fixup->bus_id, bus_id, sizeof(fixup->bus_id)); + strscpy(fixup->bus_id, bus_id, sizeof(fixup->bus_id)); fixup->phy_uid = phy_uid; fixup->phy_uid_mask = phy_uid_mask; fixup->run = run; @@ -520,7 +523,7 @@ phy_id_show(struct device *dev, struct device_attribute *attr, char *buf) { struct phy_device *phydev = to_phy_device(dev); - return sprintf(buf, "0x%.8lx\n", (unsigned long)phydev->phy_id); + return sysfs_emit(buf, "0x%.8lx\n", (unsigned long)phydev->phy_id); } static DEVICE_ATTR_RO(phy_id); @@ -535,7 +538,7 @@ phy_interface_show(struct device *dev, struct device_attribute *attr, char *buf) else mode = phy_modes(phydev->interface); - return sprintf(buf, "%s\n", mode); + return sysfs_emit(buf, "%s\n", mode); } static DEVICE_ATTR_RO(phy_interface); @@ -545,7 +548,7 @@ phy_has_fixups_show(struct device *dev, struct device_attribute *attr, { struct phy_device *phydev = to_phy_device(dev); - return sprintf(buf, "%d\n", phydev->has_fixups); + return sysfs_emit(buf, "%d\n", phydev->has_fixups); } static DEVICE_ATTR_RO(phy_has_fixups); @@ -555,7 +558,7 @@ static ssize_t phy_dev_flags_show(struct device *dev, { struct phy_device *phydev = to_phy_device(dev); - return sprintf(buf, "0x%08x\n", phydev->dev_flags); + return sysfs_emit(buf, "0x%08x\n", phydev->dev_flags); } static DEVICE_ATTR_RO(phy_dev_flags); @@ -989,6 +992,7 @@ EXPORT_SYMBOL(phy_device_register); void phy_device_remove(struct phy_device *phydev) { unregister_mii_timestamper(phydev->mii_ts); + pse_control_put(phydev->psec); device_del(&phydev->mdio.dev); @@ -1310,7 +1314,7 @@ phy_standalone_show(struct device *dev, struct device_attribute *attr, { struct phy_device *phydev = to_phy_device(dev); - return sprintf(buf, "%d\n", !phydev->attached_dev); + return sysfs_emit(buf, "%d\n", !phydev->attached_dev); } static DEVICE_ATTR_RO(phy_standalone); diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 9bd69328dc4d4c14d4833cecadb8147e4dc7905e..75464df191ef71dcb0ecb8870275f800f6b15e27 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -77,6 +77,7 @@ struct phylink { struct sfp_bus *sfp_bus; bool sfp_may_have_phy; + DECLARE_PHY_INTERFACE_MASK(sfp_interfaces); __ETHTOOL_DECLARE_LINK_MODE_MASK(sfp_support); u8 sfp_port; }; @@ -155,8 +156,84 @@ static const char *phylink_an_mode_str(unsigned int mode) return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown"; } -static void phylink_caps_to_linkmodes(unsigned long *linkmodes, - unsigned long caps) +/** + * phylink_interface_max_speed() - get the maximum speed of a phy interface + * @interface: phy interface mode defined by &typedef phy_interface_t + * + * Determine the maximum speed of a phy interface. This is intended to help + * determine the correct speed to pass to the MAC when the phy is performing + * rate matching. + * + * Return: The maximum speed of @interface + */ +static int phylink_interface_max_speed(phy_interface_t interface) +{ + switch (interface) { + case PHY_INTERFACE_MODE_100BASEX: + case PHY_INTERFACE_MODE_REVRMII: + case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_SMII: + case PHY_INTERFACE_MODE_REVMII: + case PHY_INTERFACE_MODE_MII: + return SPEED_100; + + case PHY_INTERFACE_MODE_TBI: + case PHY_INTERFACE_MODE_MOCA: + case PHY_INTERFACE_MODE_RTBI: + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_1000BASEKX: + case PHY_INTERFACE_MODE_TRGMII: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_GMII: + return SPEED_1000; + + case PHY_INTERFACE_MODE_2500BASEX: + return SPEED_2500; + + case PHY_INTERFACE_MODE_5GBASER: + return SPEED_5000; + + case PHY_INTERFACE_MODE_XGMII: + case PHY_INTERFACE_MODE_RXAUI: + case PHY_INTERFACE_MODE_XAUI: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_10GKR: + case PHY_INTERFACE_MODE_USXGMII: + case PHY_INTERFACE_MODE_QUSGMII: + return SPEED_10000; + + case PHY_INTERFACE_MODE_25GBASER: + return SPEED_25000; + + case PHY_INTERFACE_MODE_XLGMII: + return SPEED_40000; + + case PHY_INTERFACE_MODE_INTERNAL: + case PHY_INTERFACE_MODE_NA: + case PHY_INTERFACE_MODE_MAX: + /* No idea! Garbage in, unknown out */ + return SPEED_UNKNOWN; + } + + /* If we get here, someone forgot to add an interface mode above */ + WARN_ON_ONCE(1); + return SPEED_UNKNOWN; +} + +/** + * phylink_caps_to_linkmodes() - Convert capabilities to ethtool link modes + * @linkmodes: ethtool linkmode mask (must be already initialised) + * @caps: bitmask of MAC capabilities + * + * Set all possible pause, speed and duplex linkmodes in @linkmodes that are + * supported by the @caps. @linkmodes must have been initialised previously. + */ +void phylink_caps_to_linkmodes(unsigned long *linkmodes, unsigned long caps) { if (caps & MAC_SYM_PAUSE) __set_bit(ETHTOOL_LINK_MODE_Pause_BIT, linkmodes); @@ -295,21 +372,72 @@ static void phylink_caps_to_linkmodes(unsigned long *linkmodes, __set_bit(ETHTOOL_LINK_MODE_400000baseCR4_Full_BIT, linkmodes); } } +EXPORT_SYMBOL_GPL(phylink_caps_to_linkmodes); + +static struct { + unsigned long mask; + int speed; + unsigned int duplex; +} phylink_caps_params[] = { + { MAC_400000FD, SPEED_400000, DUPLEX_FULL }, + { MAC_200000FD, SPEED_200000, DUPLEX_FULL }, + { MAC_100000FD, SPEED_100000, DUPLEX_FULL }, + { MAC_56000FD, SPEED_56000, DUPLEX_FULL }, + { MAC_50000FD, SPEED_50000, DUPLEX_FULL }, + { MAC_40000FD, SPEED_40000, DUPLEX_FULL }, + { MAC_25000FD, SPEED_25000, DUPLEX_FULL }, + { MAC_20000FD, SPEED_20000, DUPLEX_FULL }, + { MAC_10000FD, SPEED_10000, DUPLEX_FULL }, + { MAC_5000FD, SPEED_5000, DUPLEX_FULL }, + { MAC_2500FD, SPEED_2500, DUPLEX_FULL }, + { MAC_1000FD, SPEED_1000, DUPLEX_FULL }, + { MAC_1000HD, SPEED_1000, DUPLEX_HALF }, + { MAC_100FD, SPEED_100, DUPLEX_FULL }, + { MAC_100HD, SPEED_100, DUPLEX_HALF }, + { MAC_10FD, SPEED_10, DUPLEX_FULL }, + { MAC_10HD, SPEED_10, DUPLEX_HALF }, +}; /** - * phylink_get_linkmodes() - get acceptable link modes - * @linkmodes: ethtool linkmode mask (must be already initialised) + * phylink_cap_from_speed_duplex - Get mac capability from speed/duplex + * @speed: the speed to search for + * @duplex: the duplex to search for + * + * Find the mac capability for a given speed and duplex. + * + * Return: A mask with the mac capability patching @speed and @duplex, or 0 if + * there were no matches. + */ +static unsigned long phylink_cap_from_speed_duplex(int speed, + unsigned int duplex) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(phylink_caps_params); i++) { + if (speed == phylink_caps_params[i].speed && + duplex == phylink_caps_params[i].duplex) + return phylink_caps_params[i].mask; + } + + return 0; +} + +/** + * phylink_get_capabilities() - get capabilities for a given MAC * @interface: phy interface mode defined by &typedef phy_interface_t * @mac_capabilities: bitmask of MAC capabilities + * @rate_matching: type of rate matching being performed * - * Set all possible pause, speed and duplex linkmodes in @linkmodes that - * are supported by the @interface mode and @mac_capabilities. @linkmodes - * must have been initialised previously. + * Get the MAC capabilities that are supported by the @interface mode and + * @mac_capabilities. */ -void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, - unsigned long mac_capabilities) +unsigned long phylink_get_capabilities(phy_interface_t interface, + unsigned long mac_capabilities, + int rate_matching) { + int max_speed = phylink_interface_max_speed(interface); unsigned long caps = MAC_SYM_PAUSE | MAC_ASYM_PAUSE; + unsigned long matched_caps = 0; switch (interface) { case PHY_INTERFACE_MODE_USXGMII: @@ -321,6 +449,7 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, case PHY_INTERFACE_MODE_RGMII_ID: case PHY_INTERFACE_MODE_RGMII: case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: case PHY_INTERFACE_MODE_SGMII: case PHY_INTERFACE_MODE_GMII: caps |= MAC_1000HD | MAC_1000FD; @@ -344,6 +473,7 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, case PHY_INTERFACE_MODE_1000BASEX: caps |= MAC_1000HD; fallthrough; + case PHY_INTERFACE_MODE_1000BASEKX: case PHY_INTERFACE_MODE_TRGMII: caps |= MAC_1000FD; break; @@ -381,9 +511,55 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, break; } - phylink_caps_to_linkmodes(linkmodes, caps & mac_capabilities); + switch (rate_matching) { + case RATE_MATCH_OPEN_LOOP: + /* TODO */ + fallthrough; + case RATE_MATCH_NONE: + matched_caps = 0; + break; + case RATE_MATCH_PAUSE: { + /* The MAC must support asymmetric pause towards the local + * device for this. We could allow just symmetric pause, but + * then we might have to renegotiate if the link partner + * doesn't support pause. This is because there's no way to + * accept pause frames without transmitting them if we only + * support symmetric pause. + */ + if (!(mac_capabilities & MAC_SYM_PAUSE) || + !(mac_capabilities & MAC_ASYM_PAUSE)) + break; + + /* We can't adapt if the MAC doesn't support the interface's + * max speed at full duplex. + */ + if (mac_capabilities & + phylink_cap_from_speed_duplex(max_speed, DUPLEX_FULL)) { + /* Although a duplex-matching phy might exist, we + * conservatively remove these modes because the MAC + * will not be aware of the half-duplex nature of the + * link. + */ + matched_caps = GENMASK(__fls(caps), __fls(MAC_10HD)); + matched_caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD); + } + break; + } + case RATE_MATCH_CRS: + /* The MAC must support half duplex at the interface's max + * speed. + */ + if (mac_capabilities & + phylink_cap_from_speed_duplex(max_speed, DUPLEX_HALF)) { + matched_caps = GENMASK(__fls(caps), __fls(MAC_10HD)); + matched_caps &= mac_capabilities; + } + break; + } + + return (caps & mac_capabilities) | matched_caps; } -EXPORT_SYMBOL_GPL(phylink_get_linkmodes); +EXPORT_SYMBOL_GPL(phylink_get_capabilities); /** * phylink_generic_validate() - generic validate() callback implementation @@ -400,10 +576,14 @@ void phylink_generic_validate(struct phylink_config *config, struct phylink_link_state *state) { __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; + unsigned long caps; phylink_set_port_modes(mask); phylink_set(mask, Autoneg); - phylink_get_linkmodes(mask, state->interface, config->mac_capabilities); + caps = phylink_get_capabilities(state->interface, + config->mac_capabilities, + state->rate_matching); + phylink_caps_to_linkmodes(mask, caps); linkmode_and(supported, supported, mask); linkmode_and(state->advertising, state->advertising, mask); @@ -458,8 +638,9 @@ static int phylink_validate_mac_and_pcs(struct phylink *pl, return phylink_is_empty_linkmode(supported) ? -EINVAL : 0; } -static int phylink_validate_any(struct phylink *pl, unsigned long *supported, - struct phylink_link_state *state) +static int phylink_validate_mask(struct phylink *pl, unsigned long *supported, + struct phylink_link_state *state, + const unsigned long *interfaces) { __ETHTOOL_DECLARE_LINK_MODE_MASK(all_adv) = { 0, }; __ETHTOOL_DECLARE_LINK_MODE_MASK(all_s) = { 0, }; @@ -468,7 +649,7 @@ static int phylink_validate_any(struct phylink *pl, unsigned long *supported, int intf; for (intf = 0; intf < PHY_INTERFACE_MODE_MAX; intf++) { - if (test_bit(intf, pl->config->supported_interfaces)) { + if (test_bit(intf, interfaces)) { linkmode_copy(s, supported); t = *state; @@ -489,12 +670,14 @@ static int phylink_validate_any(struct phylink *pl, unsigned long *supported, static int phylink_validate(struct phylink *pl, unsigned long *supported, struct phylink_link_state *state) { - if (!phy_interface_empty(pl->config->supported_interfaces)) { + const unsigned long *interfaces = pl->config->supported_interfaces; + + if (!phy_interface_empty(interfaces)) { if (state->interface == PHY_INTERFACE_MODE_NA) - return phylink_validate_any(pl, supported, state); + return phylink_validate_mask(pl, supported, state, + interfaces); - if (!test_bit(state->interface, - pl->config->supported_interfaces)) + if (!test_bit(state->interface, interfaces)) return -EINVAL; } @@ -632,6 +815,12 @@ static int phylink_parse_mode(struct phylink *pl, struct fwnode_handle *fwnode) switch (pl->link_config.interface) { case PHY_INTERFACE_MODE_SGMII: case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RTBI: phylink_set(pl->supported, 10baseT_Half); phylink_set(pl->supported, 10baseT_Full); phylink_set(pl->supported, 100baseT_Half); @@ -774,11 +963,12 @@ static void phylink_mac_config(struct phylink *pl, const struct phylink_link_state *state) { phylink_dbg(pl, - "%s: mode=%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n", + "%s: mode=%s/%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n", __func__, phylink_an_mode_str(pl->cur_link_an_mode), phy_modes(state->interface), phy_speed_to_str(state->speed), phy_duplex_to_str(state->duplex), + phy_rate_matching_to_str(state->rate_matching), __ETHTOOL_LINK_MODE_MASK_NBITS, state->advertising, state->pause, state->link, state->an_enabled); @@ -915,7 +1105,8 @@ static void phylink_mac_pcs_get_state(struct phylink *pl, linkmode_zero(state->lp_advertising); state->interface = pl->link_config.interface; state->an_enabled = pl->link_config.an_enabled; - if (state->an_enabled) { + state->rate_matching = pl->link_config.rate_matching; + if (state->an_enabled) { state->speed = SPEED_UNKNOWN; state->duplex = DUPLEX_UNKNOWN; state->pause = MLO_PAUSE_NONE; @@ -998,19 +1189,43 @@ static void phylink_link_up(struct phylink *pl, struct phylink_link_state link_state) { struct net_device *ndev = pl->netdev; + int speed, duplex; + bool rx_pause; + + speed = link_state.speed; + duplex = link_state.duplex; + rx_pause = !!(link_state.pause & MLO_PAUSE_RX); + + switch (link_state.rate_matching) { + case RATE_MATCH_PAUSE: + /* The PHY is doing rate matchion from the media rate (in + * the link_state) to the interface speed, and will send + * pause frames to the MAC to limit its transmission speed. + */ + speed = phylink_interface_max_speed(link_state.interface); + duplex = DUPLEX_FULL; + rx_pause = true; + break; + + case RATE_MATCH_CRS: + /* The PHY is doing rate matchion from the media rate (in + * the link_state) to the interface speed, and will cause + * collisions to the MAC to limit its transmission speed. + */ + speed = phylink_interface_max_speed(link_state.interface); + duplex = DUPLEX_HALF; + break; + } pl->cur_interface = link_state.interface; if (pl->pcs && pl->pcs->ops->pcs_link_up) pl->pcs->ops->pcs_link_up(pl->pcs, pl->cur_link_an_mode, - pl->cur_interface, - link_state.speed, link_state.duplex); + pl->cur_interface, speed, duplex); - pl->mac_ops->mac_link_up(pl->config, pl->phydev, - pl->cur_link_an_mode, pl->cur_interface, - link_state.speed, link_state.duplex, - !!(link_state.pause & MLO_PAUSE_TX), - !!(link_state.pause & MLO_PAUSE_RX)); + pl->mac_ops->mac_link_up(pl->config, pl->phydev, pl->cur_link_an_mode, + pl->cur_interface, speed, duplex, + !!(link_state.pause & MLO_PAUSE_TX), rx_pause); if (ndev) netif_carrier_on(ndev); @@ -1102,6 +1317,17 @@ static void phylink_resolve(struct work_struct *w) } link_state.interface = pl->phy_state.interface; + /* If we are doing rate matching, then the + * link speed/duplex comes from the PHY + */ + if (pl->phy_state.rate_matching) { + link_state.rate_matching = + pl->phy_state.rate_matching; + link_state.speed = pl->phy_state.speed; + link_state.duplex = + pl->phy_state.duplex; + } + /* If we have a PHY, we need to update with * the PHY flow control bits. */ @@ -1336,6 +1562,7 @@ static void phylink_phy_change(struct phy_device *phydev, bool up) mutex_lock(&pl->state_mutex); pl->phy_state.speed = phydev->speed; pl->phy_state.duplex = phydev->duplex; + pl->phy_state.rate_matching = phydev->rate_matching; pl->phy_state.pause = MLO_PAUSE_NONE; if (tx_pause) pl->phy_state.pause |= MLO_PAUSE_TX; @@ -1347,10 +1574,11 @@ static void phylink_phy_change(struct phy_device *phydev, bool up) phylink_run_resolve(pl); - phylink_dbg(pl, "phy link %s %s/%s/%s/%s\n", up ? "up" : "down", + phylink_dbg(pl, "phy link %s %s/%s/%s/%s/%s\n", up ? "up" : "down", phy_modes(phydev->interface), phy_speed_to_str(phydev->speed), phy_duplex_to_str(phydev->duplex), + phy_rate_matching_to_str(phydev->rate_matching), phylink_pause_to_str(pl->phy_state.pause)); } @@ -1387,6 +1615,7 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, config.interface = PHY_INTERFACE_MODE_NA; else config.interface = interface; + config.rate_matching = phy_get_rate_matching(phy, config.interface); ret = phylink_validate(pl, supported, &config); if (ret) { @@ -1414,6 +1643,7 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, pl->phy_state.pause = MLO_PAUSE_NONE; pl->phy_state.speed = SPEED_UNKNOWN; pl->phy_state.duplex = DUPLEX_UNKNOWN; + pl->phy_state.rate_matching = RATE_MATCH_NONE; linkmode_copy(pl->supported, supported); linkmode_copy(pl->link_config.advertising, config.advertising); @@ -1439,7 +1669,7 @@ static int phylink_attach_phy(struct phylink *pl, struct phy_device *phy, { if (WARN_ON(pl->cfg_link_an_mode == MLO_AN_FIXED || (pl->cfg_link_an_mode == MLO_AN_INBAND && - phy_interface_mode_is_8023z(interface)))) + phy_interface_mode_is_8023z(interface) && !pl->sfp_bus))) return -EINVAL; if (pl->phydev) @@ -1856,8 +2086,10 @@ static void phylink_get_ksettings(const struct phylink_link_state *state, { phylink_merge_link_mode(kset->link_modes.advertising, state->advertising); linkmode_copy(kset->link_modes.lp_advertising, state->lp_advertising); - kset->base.speed = state->speed; - kset->base.duplex = state->duplex; + if (kset->base.rate_matching == RATE_MATCH_NONE) { + kset->base.speed = state->speed; + kset->base.duplex = state->duplex; + } kset->base.autoneg = state->an_enabled ? AUTONEG_ENABLE : AUTONEG_DISABLE; } @@ -2571,21 +2803,85 @@ static void phylink_sfp_detach(void *upstream, struct sfp_bus *bus) pl->netdev->sfp_bus = NULL; } -static int phylink_sfp_config(struct phylink *pl, u8 mode, - const unsigned long *supported, - const unsigned long *advertising) +static const phy_interface_t phylink_sfp_interface_preference[] = { + PHY_INTERFACE_MODE_25GBASER, + PHY_INTERFACE_MODE_USXGMII, + PHY_INTERFACE_MODE_10GBASER, + PHY_INTERFACE_MODE_5GBASER, + PHY_INTERFACE_MODE_2500BASEX, + PHY_INTERFACE_MODE_SGMII, + PHY_INTERFACE_MODE_1000BASEX, + PHY_INTERFACE_MODE_100BASEX, +}; + +static DECLARE_PHY_INTERFACE_MASK(phylink_sfp_interfaces); + +static phy_interface_t phylink_choose_sfp_interface(struct phylink *pl, + const unsigned long *intf) +{ + phy_interface_t interface; + size_t i; + + interface = PHY_INTERFACE_MODE_NA; + for (i = 0; i < ARRAY_SIZE(phylink_sfp_interface_preference); i++) + if (test_bit(phylink_sfp_interface_preference[i], intf)) { + interface = phylink_sfp_interface_preference[i]; + break; + } + + return interface; +} + +static void phylink_sfp_set_config(struct phylink *pl, u8 mode, + unsigned long *supported, + struct phylink_link_state *state) +{ + bool changed = false; + + phylink_dbg(pl, "requesting link mode %s/%s with support %*pb\n", + phylink_an_mode_str(mode), phy_modes(state->interface), + __ETHTOOL_LINK_MODE_MASK_NBITS, supported); + + if (!linkmode_equal(pl->supported, supported)) { + linkmode_copy(pl->supported, supported); + changed = true; + } + + if (!linkmode_equal(pl->link_config.advertising, state->advertising)) { + linkmode_copy(pl->link_config.advertising, state->advertising); + changed = true; + } + + if (pl->cur_link_an_mode != mode || + pl->link_config.interface != state->interface) { + pl->cur_link_an_mode = mode; + pl->link_config.interface = state->interface; + + changed = true; + + phylink_info(pl, "switched to %s/%s link mode\n", + phylink_an_mode_str(mode), + phy_modes(state->interface)); + } + + if (changed && !test_bit(PHYLINK_DISABLE_STOPPED, + &pl->phylink_disable_state)) + phylink_mac_initial_config(pl, false); +} + +static int phylink_sfp_config_phy(struct phylink *pl, u8 mode, + struct phy_device *phy) { __ETHTOOL_DECLARE_LINK_MODE_MASK(support1); __ETHTOOL_DECLARE_LINK_MODE_MASK(support); struct phylink_link_state config; phy_interface_t iface; - bool changed; int ret; - linkmode_copy(support, supported); + linkmode_copy(support, phy->supported); memset(&config, 0, sizeof(config)); - linkmode_copy(config.advertising, advertising); + linkmode_copy(config.advertising, phy->advertising); config.interface = PHY_INTERFACE_MODE_NA; config.speed = SPEED_UNKNOWN; config.duplex = DUPLEX_UNKNOWN; @@ -2622,60 +2918,100 @@ static int phylink_sfp_config(struct phylink *pl, u8 mode, return ret; } - phylink_dbg(pl, "requesting link mode %s/%s with support %*pb\n", - phylink_an_mode_str(mode), phy_modes(config.interface), - __ETHTOOL_LINK_MODE_MASK_NBITS, support); + pl->link_port = pl->sfp_port; + + phylink_sfp_set_config(pl, mode, support, &config); + + return 0; +} + +static int phylink_sfp_config_optical(struct phylink *pl) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(support); + DECLARE_PHY_INTERFACE_MASK(interfaces); + struct phylink_link_state config; + phy_interface_t interface; + int ret; + + phylink_dbg(pl, "optical SFP: interfaces=[mac=%*pbl, sfp=%*pbl]\n", + (int)PHY_INTERFACE_MODE_MAX, + pl->config->supported_interfaces, + (int)PHY_INTERFACE_MODE_MAX, + pl->sfp_interfaces); - if (phy_interface_mode_is_8023z(iface) && pl->phydev) + /* Find the union of the supported interfaces by the PCS/MAC and + * the SFP module. + */ + phy_interface_and(interfaces, pl->config->supported_interfaces, + pl->sfp_interfaces); + if (phy_interface_empty(interfaces)) { + phylink_err(pl, "unsupported SFP module: no common interface modes\n"); return -EINVAL; + } - changed = !linkmode_equal(pl->supported, support) || - !linkmode_equal(pl->link_config.advertising, - config.advertising); - if (changed) { - linkmode_copy(pl->supported, support); - linkmode_copy(pl->link_config.advertising, config.advertising); + memset(&config, 0, sizeof(config)); + linkmode_copy(support, pl->sfp_support); + linkmode_copy(config.advertising, pl->sfp_support); + config.speed = SPEED_UNKNOWN; + config.duplex = DUPLEX_UNKNOWN; + config.pause = MLO_PAUSE_AN; + config.an_enabled = true; + + /* For all the interfaces that are supported, reduce the sfp_support + * mask to only those link modes that can be supported. + */ + ret = phylink_validate_mask(pl, pl->sfp_support, &config, interfaces); + if (ret) { + phylink_err(pl, "unsupported SFP module: validation with support %*pb failed\n", + __ETHTOOL_LINK_MODE_MASK_NBITS, support); + return ret; } - if (pl->cur_link_an_mode != mode || - pl->link_config.interface != config.interface) { - pl->link_config.interface = config.interface; - pl->cur_link_an_mode = mode; + interface = phylink_choose_sfp_interface(pl, interfaces); + if (interface == PHY_INTERFACE_MODE_NA) { + phylink_err(pl, "failed to select SFP interface\n"); + return -EINVAL; + } - changed = true; + phylink_dbg(pl, "optical SFP: chosen %s interface\n", + phy_modes(interface)); - phylink_info(pl, "switched to %s/%s link mode\n", - phylink_an_mode_str(mode), - phy_modes(config.interface)); + config.interface = interface; + + /* Ignore errors if we're expecting a PHY to attach later */ + ret = phylink_validate(pl, support, &config); + if (ret) { + phylink_err(pl, "validation with support %*pb failed: %pe\n", + __ETHTOOL_LINK_MODE_MASK_NBITS, support, + ERR_PTR(ret)); + return ret; } pl->link_port = pl->sfp_port; - if (changed && !test_bit(PHYLINK_DISABLE_STOPPED, - &pl->phylink_disable_state)) - phylink_mac_initial_config(pl, false); + phylink_sfp_set_config(pl, MLO_AN_INBAND, pl->sfp_support, &config); - return ret; + return 0; } static int phylink_sfp_module_insert(void *upstream, const struct sfp_eeprom_id *id) { struct phylink *pl = upstream; - unsigned long *support = pl->sfp_support; ASSERT_RTNL(); - linkmode_zero(support); - sfp_parse_support(pl->sfp_bus, id, support); - pl->sfp_port = sfp_parse_port(pl->sfp_bus, id, support); + linkmode_zero(pl->sfp_support); + phy_interface_zero(pl->sfp_interfaces); + sfp_parse_support(pl->sfp_bus, id, pl->sfp_support, pl->sfp_interfaces); + pl->sfp_port = sfp_parse_port(pl->sfp_bus, id, pl->sfp_support); /* If this module may have a PHY connecting later, defer until later */ pl->sfp_may_have_phy = sfp_may_have_phy(pl->sfp_bus, id); if (pl->sfp_may_have_phy) return 0; - return phylink_sfp_config(pl, MLO_AN_INBAND, support, support); + return phylink_sfp_config_optical(pl); } static int phylink_sfp_module_start(void *upstream) @@ -2694,8 +3030,7 @@ static int phylink_sfp_module_start(void *upstream) if (!pl->sfp_may_have_phy) return 0; - return phylink_sfp_config(pl, MLO_AN_INBAND, - pl->sfp_support, pl->sfp_support); + return phylink_sfp_config_optical(pl); } static void phylink_sfp_module_stop(void *upstream) @@ -2755,8 +3090,12 @@ static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy) else mode = MLO_AN_INBAND; + /* Set the PHY's host supported interfaces */ + phy_interface_and(phy->host_interfaces, phylink_sfp_interfaces, + pl->config->supported_interfaces); + /* Do the initial configuration */ - ret = phylink_sfp_config(pl, mode, phy->supported, phy->advertising); + ret = phylink_sfp_config_phy(pl, mode, phy); if (ret < 0) return ret; @@ -2929,6 +3268,7 @@ void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, case PHY_INTERFACE_MODE_SGMII: case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_QUSGMII: phylink_decode_sgmii_word(state, lpa); break; @@ -3107,4 +3447,15 @@ void phylink_mii_c45_pcs_get_state(struct mdio_device *pcs, } EXPORT_SYMBOL_GPL(phylink_mii_c45_pcs_get_state); +static int __init phylink_init(void) +{ + for (int i = 0; i < ARRAY_SIZE(phylink_sfp_interface_preference); ++i) + __set_bit(phylink_sfp_interface_preference[i], + phylink_sfp_interfaces); + + return 0; +} + +module_init(phylink_init); + MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c index a5671ab896b385d4d5b152aa828883aa3a783783..3d99fd6664d7a3a95877333b937a85bb283f9aab 100644 --- a/drivers/net/phy/realtek.c +++ b/drivers/net/phy/realtek.c @@ -70,6 +70,7 @@ #define RTLGEN_SPEED_MASK 0x0630 #define RTL_GENERIC_PHYID 0x001cc800 +#define RTL_8211FVD_PHYID 0x001cc878 MODULE_DESCRIPTION("Realtek PHY driver"); MODULE_AUTHOR("Johnson Leung"); @@ -78,6 +79,7 @@ MODULE_LICENSE("GPL"); struct rtl821x_priv { u16 phycr1; u16 phycr2; + bool has_phycr2; }; static int rtl821x_read_page(struct phy_device *phydev) @@ -94,6 +96,7 @@ static int rtl821x_probe(struct phy_device *phydev) { struct device *dev = &phydev->mdio.dev; struct rtl821x_priv *priv; + u32 phy_id = phydev->drv->phy_id; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -108,13 +111,16 @@ static int rtl821x_probe(struct phy_device *phydev) if (of_property_read_bool(dev->of_node, "realtek,aldps-enable")) priv->phycr1 |= RTL8211F_ALDPS_PLL_OFF | RTL8211F_ALDPS_ENABLE | RTL8211F_ALDPS_XTAL_OFF; - ret = phy_read_paged(phydev, 0xa43, RTL8211F_PHYCR2); - if (ret < 0) - return ret; + priv->has_phycr2 = !(phy_id == RTL_8211FVD_PHYID); + if (priv->has_phycr2) { + ret = phy_read_paged(phydev, 0xa43, RTL8211F_PHYCR2); + if (ret < 0) + return ret; - priv->phycr2 = ret & RTL8211F_CLKOUT_EN; - if (of_property_read_bool(dev->of_node, "realtek,clkout-disable")) - priv->phycr2 &= ~RTL8211F_CLKOUT_EN; + priv->phycr2 = ret & RTL8211F_CLKOUT_EN; + if (of_property_read_bool(dev->of_node, "realtek,clkout-disable")) + priv->phycr2 &= ~RTL8211F_CLKOUT_EN; + } phydev->priv = priv; @@ -400,12 +406,14 @@ static int rtl8211f_config_init(struct phy_device *phydev) val_rxdly ? "enabled" : "disabled"); } - ret = phy_modify_paged(phydev, 0xa43, RTL8211F_PHYCR2, - RTL8211F_CLKOUT_EN, priv->phycr2); - if (ret < 0) { - dev_err(dev, "clkout configuration failed: %pe\n", - ERR_PTR(ret)); - return ret; + if (priv->has_phycr2) { + ret = phy_modify_paged(phydev, 0xa43, RTL8211F_PHYCR2, + RTL8211F_CLKOUT_EN, priv->phycr2); + if (ret < 0) { + dev_err(dev, "clkout configuration failed: %pe\n", + ERR_PTR(ret)); + return ret; + } } return genphy_soft_reset(phydev); @@ -923,6 +931,18 @@ static struct phy_driver realtek_drvs[] = { .resume = rtl821x_resume, .read_page = rtl821x_read_page, .write_page = rtl821x_write_page, + }, { + PHY_ID_MATCH_EXACT(RTL_8211FVD_PHYID), + .name = "RTL8211F-VD Gigabit Ethernet", + .probe = rtl821x_probe, + .config_init = &rtl8211f_config_init, + .read_status = rtlgen_read_status, + .config_intr = &rtl8211f_config_intr, + .handle_interrupt = rtl8211f_handle_interrupt, + .suspend = genphy_suspend, + .resume = rtl821x_resume, + .read_page = rtl821x_read_page, + .write_page = rtl821x_write_page, }, { .name = "Generic FE-GE Realtek PHY", .match_phy_device = rtlgen_match_phy_device, diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c index 15aa5ac1ff49ccb779eb7abfe8474500f1aaae7e..daac293e8edece7e594ca35582ff2483a817b6a3 100644 --- a/drivers/net/phy/sfp-bus.c +++ b/drivers/net/phy/sfp-bus.c @@ -10,12 +10,6 @@ #include "sfp.h" -struct sfp_quirk { - const char *vendor; - const char *part; - void (*modes)(const struct sfp_eeprom_id *id, unsigned long *modes); -}; - /** * struct sfp_bus - internal representation of a sfp bus */ @@ -38,93 +32,6 @@ struct sfp_bus { bool started; }; -static void sfp_quirk_2500basex(const struct sfp_eeprom_id *id, - unsigned long *modes) -{ - phylink_set(modes, 2500baseX_Full); -} - -static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id, - unsigned long *modes) -{ - /* Ubiquiti U-Fiber Instant module claims that support all transceiver - * types including 10G Ethernet which is not truth. So clear all claimed - * modes and set only one mode which module supports: 1000baseX_Full. - */ - phylink_zero(modes); - phylink_set(modes, 1000baseX_Full); -} - -static const struct sfp_quirk sfp_quirks[] = { - { - // Alcatel Lucent G-010S-P can operate at 2500base-X, but - // incorrectly report 2500MBd NRZ in their EEPROM - .vendor = "ALCATELLUCENT", - .part = "G010SP", - .modes = sfp_quirk_2500basex, - }, { - // Alcatel Lucent G-010S-A can operate at 2500base-X, but - // report 3.2GBd NRZ in their EEPROM - .vendor = "ALCATELLUCENT", - .part = "3FE46541AA", - .modes = sfp_quirk_2500basex, - }, { - // Huawei MA5671A can operate at 2500base-X, but report 1.2GBd - // NRZ in their EEPROM - .vendor = "HUAWEI", - .part = "MA5671A", - .modes = sfp_quirk_2500basex, - }, { - // Lantech 8330-262D-E can operate at 2500base-X, but - // incorrectly report 2500MBd NRZ in their EEPROM - .vendor = "Lantech", - .part = "8330-262D-E", - .modes = sfp_quirk_2500basex, - }, { - .vendor = "UBNT", - .part = "UF-INSTANT", - .modes = sfp_quirk_ubnt_uf_instant, - }, -}; - -static size_t sfp_strlen(const char *str, size_t maxlen) -{ - size_t size, i; - - /* Trailing characters should be filled with space chars */ - for (i = 0, size = 0; i < maxlen; i++) - if (str[i] != ' ') - size = i + 1; - - return size; -} - -static bool sfp_match(const char *qs, const char *str, size_t len) -{ - if (!qs) - return true; - if (strlen(qs) != len) - return false; - return !strncmp(qs, str, len); -} - -static const struct sfp_quirk *sfp_lookup_quirk(const struct sfp_eeprom_id *id) -{ - const struct sfp_quirk *q; - unsigned int i; - size_t vs, ps; - - vs = sfp_strlen(id->base.vendor_name, ARRAY_SIZE(id->base.vendor_name)); - ps = sfp_strlen(id->base.vendor_pn, ARRAY_SIZE(id->base.vendor_pn)); - - for (i = 0, q = sfp_quirks; i < ARRAY_SIZE(sfp_quirks); i++, q++) - if (sfp_match(q->vendor, id->base.vendor_name, vs) && - sfp_match(q->part, id->base.vendor_pn, ps)) - return q; - - return NULL; -} - /** * sfp_parse_port() - Parse the EEPROM base ID, setting the port type * @bus: a pointer to the &struct sfp_bus structure for the sfp module @@ -232,12 +139,14 @@ EXPORT_SYMBOL_GPL(sfp_may_have_phy); * @bus: a pointer to the &struct sfp_bus structure for the sfp module * @id: a pointer to the module's &struct sfp_eeprom_id * @support: pointer to an array of unsigned long for the ethtool support mask + * @interfaces: pointer to an array of unsigned long for phy interface modes + * mask * * Parse the EEPROM identification information and derive the supported * ethtool link modes for the module. */ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, - unsigned long *support) + unsigned long *support, unsigned long *interfaces) { unsigned int br_min, br_nom, br_max; __ETHTOOL_DECLARE_LINK_MODE_MASK(modes) = { 0, }; @@ -264,54 +173,81 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, } /* Set ethtool support from the compliance fields. */ - if (id->base.e10g_base_sr) + if (id->base.e10g_base_sr) { phylink_set(modes, 10000baseSR_Full); - if (id->base.e10g_base_lr) + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } + if (id->base.e10g_base_lr) { phylink_set(modes, 10000baseLR_Full); - if (id->base.e10g_base_lrm) + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } + if (id->base.e10g_base_lrm) { phylink_set(modes, 10000baseLRM_Full); - if (id->base.e10g_base_er) + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } + if (id->base.e10g_base_er) { phylink_set(modes, 10000baseER_Full); + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } if (id->base.e1000_base_sx || id->base.e1000_base_lx || - id->base.e1000_base_cx) + id->base.e1000_base_cx) { phylink_set(modes, 1000baseX_Full); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + } if (id->base.e1000_base_t) { phylink_set(modes, 1000baseT_Half); phylink_set(modes, 1000baseT_Full); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, interfaces); } /* 1000Base-PX or 1000Base-BX10 */ if ((id->base.e_base_px || id->base.e_base_bx10) && - br_min <= 1300 && br_max >= 1200) + br_min <= 1300 && br_max >= 1200) { phylink_set(modes, 1000baseX_Full); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + } /* 100Base-FX, 100Base-LX, 100Base-PX, 100Base-BX10 */ - if (id->base.e100_base_fx || id->base.e100_base_lx) + if (id->base.e100_base_fx || id->base.e100_base_lx) { phylink_set(modes, 100baseFX_Full); - if ((id->base.e_base_px || id->base.e_base_bx10) && br_nom == 100) + __set_bit(PHY_INTERFACE_MODE_100BASEX, interfaces); + } + if ((id->base.e_base_px || id->base.e_base_bx10) && br_nom == 100) { phylink_set(modes, 100baseFX_Full); + __set_bit(PHY_INTERFACE_MODE_100BASEX, interfaces); + } /* For active or passive cables, select the link modes * based on the bit rates and the cable compliance bytes. */ if ((id->base.sfp_ct_passive || id->base.sfp_ct_active) && br_nom) { /* This may look odd, but some manufacturers use 12000MBd */ - if (br_min <= 12000 && br_max >= 10300) + if (br_min <= 12000 && br_max >= 10300) { phylink_set(modes, 10000baseCR_Full); - if (br_min <= 3200 && br_max >= 3100) + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } + if (br_min <= 3200 && br_max >= 3100) { phylink_set(modes, 2500baseX_Full); - if (br_min <= 1300 && br_max >= 1200) + __set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces); + } + if (br_min <= 1300 && br_max >= 1200) { phylink_set(modes, 1000baseX_Full); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + } } if (id->base.sfp_ct_passive) { - if (id->base.passive.sff8431_app_e) + if (id->base.passive.sff8431_app_e) { phylink_set(modes, 10000baseCR_Full); + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); + } } if (id->base.sfp_ct_active) { if (id->base.active.sff8431_app_e || id->base.active.sff8431_lim) { phylink_set(modes, 10000baseCR_Full); + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); } } @@ -321,6 +257,7 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, case SFF8024_ECC_100GBASE_SR4_25GBASE_SR: phylink_set(modes, 100000baseSR4_Full); phylink_set(modes, 25000baseSR_Full); + __set_bit(PHY_INTERFACE_MODE_25GBASER, interfaces); break; case SFF8024_ECC_100GBASE_LR4_25GBASE_LR: case SFF8024_ECC_100GBASE_ER4_25GBASE_ER: @@ -332,16 +269,20 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, case SFF8024_ECC_25GBASE_CR_S: case SFF8024_ECC_25GBASE_CR_N: phylink_set(modes, 25000baseCR_Full); + __set_bit(PHY_INTERFACE_MODE_25GBASER, interfaces); break; case SFF8024_ECC_10GBASE_T_SFI: case SFF8024_ECC_10GBASE_T_SR: phylink_set(modes, 10000baseT_Full); + __set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces); break; case SFF8024_ECC_5GBASE_T: phylink_set(modes, 5000baseT_Full); + __set_bit(PHY_INTERFACE_MODE_5GBASER, interfaces); break; case SFF8024_ECC_2_5GBASE_T: phylink_set(modes, 2500baseT_Full); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces); break; default: dev_warn(bus->sfp_dev, @@ -354,10 +295,14 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, if (id->base.fc_speed_100 || id->base.fc_speed_200 || id->base.fc_speed_400) { - if (id->base.br_nominal >= 31) + if (id->base.br_nominal >= 31) { phylink_set(modes, 2500baseX_Full); - if (id->base.br_nominal >= 12) + __set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces); + } + if (id->base.br_nominal >= 12) { phylink_set(modes, 1000baseX_Full); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + } } /* If we haven't discovered any modes that this module supports, try @@ -370,14 +315,18 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, * 2500BASE-X, so we allow some slack here. */ if (bitmap_empty(modes, __ETHTOOL_LINK_MODE_MASK_NBITS) && br_nom) { - if (br_min <= 1300 && br_max >= 1200) + if (br_min <= 1300 && br_max >= 1200) { phylink_set(modes, 1000baseX_Full); - if (br_min <= 3200 && br_max >= 2500) + __set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces); + } + if (br_min <= 3200 && br_max >= 2500) { phylink_set(modes, 2500baseX_Full); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces); + } } - if (bus->sfp_quirk) - bus->sfp_quirk->modes(id, modes); + if (bus->sfp_quirk && bus->sfp_quirk->modes) + bus->sfp_quirk->modes(id, modes, interfaces); linkmode_or(support, support, modes); @@ -786,12 +735,13 @@ void sfp_link_down(struct sfp_bus *bus) } EXPORT_SYMBOL_GPL(sfp_link_down); -int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id) +int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id, + const struct sfp_quirk *quirk) { const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus); int ret = 0; - bus->sfp_quirk = sfp_lookup_quirk(id); + bus->sfp_quirk = quirk; if (ops && ops->module_insert) ret = ops->module_insert(bus->upstream, id); diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 63f90fe9a4d2a21ae9b17d392bc386ba04740b5d..40c9a64c5e30109222365f2d5f9142b7f2dc7c84 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -166,6 +166,7 @@ static const enum gpiod_flags gpio_flags[] = { * on board (for a copper SFP) time to initialise. */ #define T_WAIT msecs_to_jiffies(50) +#define T_WAIT_ROLLBALL msecs_to_jiffies(25000) #define T_START_UP msecs_to_jiffies(300) #define T_START_UP_BAD_GPON msecs_to_jiffies(60000) @@ -205,8 +206,11 @@ static const enum gpiod_flags gpio_flags[] = { /* SFP modules appear to always have their PHY configured for bus address * 0x56 (which with mdio-i2c, translates to a PHY address of 22). + * RollBall SFPs access phy via SFP Enhanced Digital Diagnostic Interface + * via address 0x51 (mdio-i2c will use RollBall protocol on this address). */ -#define SFP_PHY_ADDR 22 +#define SFP_PHY_ADDR 22 +#define SFP_PHY_ADDR_ROLLBALL 17 struct sff_data { unsigned int gpios; @@ -218,6 +222,7 @@ struct sfp { struct i2c_adapter *i2c; struct mii_bus *i2c_mii; struct sfp_bus *sfp_bus; + enum mdio_i2c_proto mdio_protocol; struct phy_device *mod_phy; const struct sff_data *type; size_t i2c_block_size; @@ -234,6 +239,7 @@ struct sfp { bool need_poll; struct mutex st_mutex; /* Protects state */ + unsigned int state_hw_mask; unsigned int state_soft_mask; unsigned int state; struct delayed_work poll; @@ -250,8 +256,11 @@ struct sfp { struct sfp_eeprom_id id; unsigned int module_power_mW; unsigned int module_t_start_up; + unsigned int module_t_wait; bool tx_fault_ignore; + const struct sfp_quirk *quirk; + #if IS_ENABLED(CONFIG_HWMON) struct sfp_diag diag; struct delayed_work hwmon_probe; @@ -308,6 +317,136 @@ static const struct of_device_id sfp_of_match[] = { }; MODULE_DEVICE_TABLE(of, sfp_of_match); +static void sfp_fixup_long_startup(struct sfp *sfp) +{ + sfp->module_t_start_up = T_START_UP_BAD_GPON; +} + +static void sfp_fixup_ignore_tx_fault(struct sfp *sfp) +{ + sfp->tx_fault_ignore = true; +} + +static void sfp_fixup_halny_gsfp(struct sfp *sfp) +{ + /* Ignore the TX_FAULT and LOS signals on this module. + * these are possibly used for other purposes on this + * module, e.g. a serial port. + */ + sfp->state_hw_mask &= ~(SFP_F_TX_FAULT | SFP_F_LOS); +} + +static void sfp_fixup_rollball(struct sfp *sfp) +{ + sfp->mdio_protocol = MDIO_I2C_ROLLBALL; + sfp->module_t_wait = T_WAIT_ROLLBALL; +} + +static void sfp_fixup_rollball_cc(struct sfp *sfp) +{ + sfp_fixup_rollball(sfp); + + /* Some RollBall SFPs may have wrong (zero) extended compliance code + * burned in EEPROM. For PHY probing we need the correct one. + */ + sfp->id.base.extended_cc = SFF8024_ECC_10GBASE_T_SFI; +} + +static void sfp_quirk_2500basex(const struct sfp_eeprom_id *id, + unsigned long *modes, + unsigned long *interfaces) +{ + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, modes); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces); +} + +static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id, + unsigned long *modes, + unsigned long *interfaces) +{ + /* Ubiquiti U-Fiber Instant module claims that support all transceiver + * types including 10G Ethernet which is not truth. So clear all claimed + * modes and set only one mode which module supports: 1000baseX_Full. + */ + linkmode_zero(modes); + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, modes); +} + +#define SFP_QUIRK(_v, _p, _m, _f) \ + { .vendor = _v, .part = _p, .modes = _m, .fixup = _f, } +#define SFP_QUIRK_M(_v, _p, _m) SFP_QUIRK(_v, _p, _m, NULL) +#define SFP_QUIRK_F(_v, _p, _f) SFP_QUIRK(_v, _p, NULL, _f) + +static const struct sfp_quirk sfp_quirks[] = { + // Alcatel Lucent G-010S-P can operate at 2500base-X, but incorrectly + // report 2500MBd NRZ in their EEPROM + SFP_QUIRK_M("ALCATELLUCENT", "G010SP", sfp_quirk_2500basex), + + // Alcatel Lucent G-010S-A can operate at 2500base-X, but report 3.2GBd + // NRZ in their EEPROM + SFP_QUIRK("ALCATELLUCENT", "3FE46541AA", sfp_quirk_2500basex, + sfp_fixup_long_startup), + + SFP_QUIRK_F("HALNy", "HL-GSFP", sfp_fixup_halny_gsfp), + + // Huawei MA5671A can operate at 2500base-X, but report 1.2GBd NRZ in + // their EEPROM + SFP_QUIRK("HUAWEI", "MA5671A", sfp_quirk_2500basex, + sfp_fixup_ignore_tx_fault), + + // Lantech 8330-262D-E can operate at 2500base-X, but incorrectly report + // 2500MBd NRZ in their EEPROM + SFP_QUIRK_M("Lantech", "8330-262D-E", sfp_quirk_2500basex), + + SFP_QUIRK_M("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant), + + SFP_QUIRK_F("OEM", "SFP-10G-T", sfp_fixup_rollball_cc), + SFP_QUIRK_F("OEM", "RTSFP-10", sfp_fixup_rollball_cc), + SFP_QUIRK_F("OEM", "RTSFP-10G", sfp_fixup_rollball_cc), + SFP_QUIRK_F("Turris", "RTSFP-10", sfp_fixup_rollball), + SFP_QUIRK_F("Turris", "RTSFP-10G", sfp_fixup_rollball), +}; + +static size_t sfp_strlen(const char *str, size_t maxlen) +{ + size_t size, i; + + /* Trailing characters should be filled with space chars, but + * some manufacturers can't read SFF-8472 and use NUL. + */ + for (i = 0, size = 0; i < maxlen; i++) + if (str[i] != ' ' && str[i] != '\0') + size = i + 1; + + return size; +} + +static bool sfp_match(const char *qs, const char *str, size_t len) +{ + if (!qs) + return true; + if (strlen(qs) != len) + return false; + return !strncmp(qs, str, len); +} + +static const struct sfp_quirk *sfp_lookup_quirk(const struct sfp_eeprom_id *id) +{ + const struct sfp_quirk *q; + unsigned int i; + size_t vs, ps; + + vs = sfp_strlen(id->base.vendor_name, ARRAY_SIZE(id->base.vendor_name)); + ps = sfp_strlen(id->base.vendor_pn, ARRAY_SIZE(id->base.vendor_pn)); + + for (i = 0, q = sfp_quirks; i < ARRAY_SIZE(sfp_quirks); i++, q++) + if (sfp_match(q->vendor, id->base.vendor_name, vs) && + sfp_match(q->part, id->base.vendor_pn, ps)) + return q; + + return NULL; +} + static unsigned long poll_jiffies; static unsigned int sfp_gpio_get_state(struct sfp *sfp) @@ -419,9 +558,6 @@ static int sfp_i2c_write(struct sfp *sfp, bool a2, u8 dev_addr, void *buf, static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c) { - struct mii_bus *i2c_mii; - int ret; - if (!i2c_check_functionality(i2c, I2C_FUNC_I2C)) return -EINVAL; @@ -429,7 +565,15 @@ static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c) sfp->read = sfp_i2c_read; sfp->write = sfp_i2c_write; - i2c_mii = mdio_i2c_alloc(sfp->dev, i2c); + return 0; +} + +static int sfp_i2c_mdiobus_create(struct sfp *sfp) +{ + struct mii_bus *i2c_mii; + int ret; + + i2c_mii = mdio_i2c_alloc(sfp->dev, sfp->i2c, sfp->mdio_protocol); if (IS_ERR(i2c_mii)) return PTR_ERR(i2c_mii); @@ -447,6 +591,12 @@ static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c) return 0; } +static void sfp_i2c_mdiobus_destroy(struct sfp *sfp) +{ + mdiobus_unregister(sfp->i2c_mii); + sfp->i2c_mii = NULL; +} + /* Interface */ static int sfp_read(struct sfp *sfp, bool a2, u8 addr, void *buf, size_t len) { @@ -499,17 +649,18 @@ static void sfp_soft_set_state(struct sfp *sfp, unsigned int state) static void sfp_soft_start_poll(struct sfp *sfp) { const struct sfp_eeprom_id *id = &sfp->id; + unsigned int mask = 0; sfp->state_soft_mask = 0; - if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_DISABLE && - !sfp->gpio[GPIO_TX_DISABLE]) - sfp->state_soft_mask |= SFP_F_TX_DISABLE; - if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_FAULT && - !sfp->gpio[GPIO_TX_FAULT]) - sfp->state_soft_mask |= SFP_F_TX_FAULT; - if (id->ext.enhopts & SFP_ENHOPTS_SOFT_RX_LOS && - !sfp->gpio[GPIO_LOS]) - sfp->state_soft_mask |= SFP_F_LOS; + if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_DISABLE) + mask |= SFP_F_TX_DISABLE; + if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_FAULT) + mask |= SFP_F_TX_FAULT; + if (id->ext.enhopts & SFP_ENHOPTS_SOFT_RX_LOS) + mask |= SFP_F_LOS; + + // Poll the soft state for hardware pins we want to ignore + sfp->state_soft_mask = ~sfp->state_hw_mask & mask; if (sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT) && !sfp->need_poll) @@ -523,10 +674,11 @@ static void sfp_soft_stop_poll(struct sfp *sfp) static unsigned int sfp_get_state(struct sfp *sfp) { - unsigned int state = sfp->get_state(sfp); + unsigned int soft = sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT); + unsigned int state; - if (state & SFP_F_PRESENT && - sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT)) + state = sfp->get_state(sfp) & sfp->state_hw_mask; + if (state & SFP_F_PRESENT && soft) state |= sfp_soft_get_state(sfp); return state; @@ -1195,90 +1347,45 @@ static const struct hwmon_ops sfp_hwmon_ops = { .read_string = sfp_hwmon_read_string, }; -static u32 sfp_hwmon_chip_config[] = { - HWMON_C_REGISTER_TZ, - 0, -}; - -static const struct hwmon_channel_info sfp_hwmon_chip = { - .type = hwmon_chip, - .config = sfp_hwmon_chip_config, -}; - -static u32 sfp_hwmon_temp_config[] = { - HWMON_T_INPUT | - HWMON_T_MAX | HWMON_T_MIN | - HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM | - HWMON_T_CRIT | HWMON_T_LCRIT | - HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM | - HWMON_T_LABEL, - 0, -}; - -static const struct hwmon_channel_info sfp_hwmon_temp_channel_info = { - .type = hwmon_temp, - .config = sfp_hwmon_temp_config, -}; - -static u32 sfp_hwmon_vcc_config[] = { - HWMON_I_INPUT | - HWMON_I_MAX | HWMON_I_MIN | - HWMON_I_MAX_ALARM | HWMON_I_MIN_ALARM | - HWMON_I_CRIT | HWMON_I_LCRIT | - HWMON_I_CRIT_ALARM | HWMON_I_LCRIT_ALARM | - HWMON_I_LABEL, - 0, -}; - -static const struct hwmon_channel_info sfp_hwmon_vcc_channel_info = { - .type = hwmon_in, - .config = sfp_hwmon_vcc_config, -}; - -static u32 sfp_hwmon_bias_config[] = { - HWMON_C_INPUT | - HWMON_C_MAX | HWMON_C_MIN | - HWMON_C_MAX_ALARM | HWMON_C_MIN_ALARM | - HWMON_C_CRIT | HWMON_C_LCRIT | - HWMON_C_CRIT_ALARM | HWMON_C_LCRIT_ALARM | - HWMON_C_LABEL, - 0, -}; - -static const struct hwmon_channel_info sfp_hwmon_bias_channel_info = { - .type = hwmon_curr, - .config = sfp_hwmon_bias_config, -}; - -static u32 sfp_hwmon_power_config[] = { - /* Transmit power */ - HWMON_P_INPUT | - HWMON_P_MAX | HWMON_P_MIN | - HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | - HWMON_P_CRIT | HWMON_P_LCRIT | - HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM | - HWMON_P_LABEL, - /* Receive power */ - HWMON_P_INPUT | - HWMON_P_MAX | HWMON_P_MIN | - HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | - HWMON_P_CRIT | HWMON_P_LCRIT | - HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM | - HWMON_P_LABEL, - 0, -}; - -static const struct hwmon_channel_info sfp_hwmon_power_channel_info = { - .type = hwmon_power, - .config = sfp_hwmon_power_config, -}; - static const struct hwmon_channel_info *sfp_hwmon_info[] = { - &sfp_hwmon_chip, - &sfp_hwmon_vcc_channel_info, - &sfp_hwmon_temp_channel_info, - &sfp_hwmon_bias_channel_info, - &sfp_hwmon_power_channel_info, + HWMON_CHANNEL_INFO(chip, + HWMON_C_REGISTER_TZ), + HWMON_CHANNEL_INFO(in, + HWMON_I_INPUT | + HWMON_I_MAX | HWMON_I_MIN | + HWMON_I_MAX_ALARM | HWMON_I_MIN_ALARM | + HWMON_I_CRIT | HWMON_I_LCRIT | + HWMON_I_CRIT_ALARM | HWMON_I_LCRIT_ALARM | + HWMON_I_LABEL), + HWMON_CHANNEL_INFO(temp, + HWMON_T_INPUT | + HWMON_T_MAX | HWMON_T_MIN | + HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM | + HWMON_T_CRIT | HWMON_T_LCRIT | + HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM | + HWMON_T_LABEL), + HWMON_CHANNEL_INFO(curr, + HWMON_C_INPUT | + HWMON_C_MAX | HWMON_C_MIN | + HWMON_C_MAX_ALARM | HWMON_C_MIN_ALARM | + HWMON_C_CRIT | HWMON_C_LCRIT | + HWMON_C_CRIT_ALARM | HWMON_C_LCRIT_ALARM | + HWMON_C_LABEL), + HWMON_CHANNEL_INFO(power, + /* Transmit power */ + HWMON_P_INPUT | + HWMON_P_MAX | HWMON_P_MIN | + HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | + HWMON_P_CRIT | HWMON_P_LCRIT | + HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM | + HWMON_P_LABEL, + /* Receive power */ + HWMON_P_INPUT | + HWMON_P_MAX | HWMON_P_MIN | + HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM | + HWMON_P_CRIT | HWMON_P_LCRIT | + HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM | + HWMON_P_LABEL), NULL, }; @@ -1505,12 +1612,12 @@ static void sfp_sm_phy_detach(struct sfp *sfp) sfp->mod_phy = NULL; } -static int sfp_sm_probe_phy(struct sfp *sfp, bool is_c45) +static int sfp_sm_probe_phy(struct sfp *sfp, int addr, bool is_c45) { struct phy_device *phy; int err; - phy = get_phy_device(sfp->i2c_mii, SFP_PHY_ADDR, is_c45); + phy = get_phy_device(sfp->i2c_mii, addr, is_c45); if (phy == ERR_PTR(-ENODEV)) return PTR_ERR(phy); if (IS_ERR(phy)) { @@ -1606,6 +1713,14 @@ static void sfp_sm_fault(struct sfp *sfp, unsigned int next_state, bool warn) } } +static int sfp_sm_add_mdio_bus(struct sfp *sfp) +{ + if (sfp->mdio_protocol != MDIO_I2C_NONE) + return sfp_i2c_mdiobus_create(sfp); + + return 0; +} + /* Probe a SFP for a PHY device if the module supports copper - the PHY * normally sits at I2C bus address 0x56, and may either be a clause 22 * or clause 45 PHY. @@ -1621,19 +1736,23 @@ static int sfp_sm_probe_for_phy(struct sfp *sfp) { int err = 0; - switch (sfp->id.base.extended_cc) { - case SFF8024_ECC_10GBASE_T_SFI: - case SFF8024_ECC_10GBASE_T_SR: - case SFF8024_ECC_5GBASE_T: - case SFF8024_ECC_2_5GBASE_T: - err = sfp_sm_probe_phy(sfp, true); + switch (sfp->mdio_protocol) { + case MDIO_I2C_NONE: break; - default: - if (sfp->id.base.e1000_base_t) - err = sfp_sm_probe_phy(sfp, false); + case MDIO_I2C_MARVELL_C22: + err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR, false); + break; + + case MDIO_I2C_C45: + err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR, true); + break; + + case MDIO_I2C_ROLLBALL: + err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR_ROLLBALL, true); break; } + return err; } @@ -1947,17 +2066,33 @@ static int sfp_sm_mod_probe(struct sfp *sfp, bool report) if (ret < 0) return ret; - if (!memcmp(id.base.vendor_name, "ALCATELLUCENT ", 16) && - !memcmp(id.base.vendor_pn, "3FE46541AA ", 16)) - sfp->module_t_start_up = T_START_UP_BAD_GPON; + /* Initialise state bits to use from hardware */ + sfp->state_hw_mask = SFP_F_PRESENT; + if (sfp->gpio[GPIO_TX_DISABLE]) + sfp->state_hw_mask |= SFP_F_TX_DISABLE; + if (sfp->gpio[GPIO_TX_FAULT]) + sfp->state_hw_mask |= SFP_F_TX_FAULT; + if (sfp->gpio[GPIO_LOS]) + sfp->state_hw_mask |= SFP_F_LOS; + + sfp->module_t_start_up = T_START_UP; + sfp->module_t_wait = T_WAIT; + + sfp->tx_fault_ignore = false; + + if (sfp->id.base.extended_cc == SFF8024_ECC_10GBASE_T_SFI || + sfp->id.base.extended_cc == SFF8024_ECC_10GBASE_T_SR || + sfp->id.base.extended_cc == SFF8024_ECC_5GBASE_T || + sfp->id.base.extended_cc == SFF8024_ECC_2_5GBASE_T) + sfp->mdio_protocol = MDIO_I2C_C45; + else if (sfp->id.base.e1000_base_t) + sfp->mdio_protocol = MDIO_I2C_MARVELL_C22; else - sfp->module_t_start_up = T_START_UP; + sfp->mdio_protocol = MDIO_I2C_NONE; - if (!memcmp(id.base.vendor_name, "HUAWEI ", 16) && - !memcmp(id.base.vendor_pn, "MA5671A ", 16)) - sfp->tx_fault_ignore = true; - else - sfp->tx_fault_ignore = false; + sfp->quirk = sfp_lookup_quirk(&id); + if (sfp->quirk && sfp->quirk->fixup) + sfp->quirk->fixup(sfp); return 0; } @@ -2071,7 +2206,8 @@ static void sfp_sm_module(struct sfp *sfp, unsigned int event) break; /* Report the module insertion to the upstream device */ - err = sfp_module_insert(sfp->sfp_bus, &sfp->id); + err = sfp_module_insert(sfp->sfp_bus, &sfp->id, + sfp->quirk); if (err < 0) { sfp_sm_mod_next(sfp, SFP_MOD_ERROR, 0); break; @@ -2130,6 +2266,8 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event) sfp_module_stop(sfp->sfp_bus); if (sfp->mod_phy) sfp_sm_phy_detach(sfp); + if (sfp->i2c_mii) + sfp_i2c_mdiobus_destroy(sfp); sfp_module_tx_disable(sfp); sfp_soft_stop_poll(sfp); sfp_sm_next(sfp, SFP_S_DOWN, 0); @@ -2153,9 +2291,10 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event) /* We need to check the TX_FAULT state, which is not defined * while TX_DISABLE is asserted. The earliest we want to do - * anything (such as probe for a PHY) is 50ms. + * anything (such as probe for a PHY) is 50ms (or more on + * specific modules). */ - sfp_sm_next(sfp, SFP_S_WAIT, T_WAIT); + sfp_sm_next(sfp, SFP_S_WAIT, sfp->module_t_wait); break; case SFP_S_WAIT: @@ -2169,8 +2308,8 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event) * deasserting. */ timeout = sfp->module_t_start_up; - if (timeout > T_WAIT) - timeout -= T_WAIT; + if (timeout > sfp->module_t_wait) + timeout -= sfp->module_t_wait; else timeout = 1; @@ -2192,6 +2331,12 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event) sfp->sm_fault_retries == N_FAULT_INIT); } else if (event == SFP_E_TIMEOUT || event == SFP_E_TX_CLEAR) { init_done: + /* Create mdiobus and start trying for PHY */ + ret = sfp_sm_add_mdio_bus(sfp); + if (ret < 0) { + sfp_sm_next(sfp, SFP_S_FAIL, 0); + break; + } sfp->sm_phy_retries = R_PHY_RETRY; goto phy_probe; } @@ -2573,6 +2718,8 @@ static int sfp_probe(struct platform_device *pdev) return PTR_ERR(sfp->gpio[i]); } + sfp->state_hw_mask = SFP_F_PRESENT; + sfp->get_state = sfp_gpio_get_state; sfp->set_state = sfp_gpio_set_state; diff --git a/drivers/net/phy/sfp.h b/drivers/net/phy/sfp.h index 27226535c72b79d2ebdd6bb39697cc42a62ed619..6cf1643214d3ef34c0dfdb716be9b76823f0d3d1 100644 --- a/drivers/net/phy/sfp.h +++ b/drivers/net/phy/sfp.h @@ -6,6 +6,14 @@ struct sfp; +struct sfp_quirk { + const char *vendor; + const char *part; + void (*modes)(const struct sfp_eeprom_id *id, unsigned long *modes, + unsigned long *interfaces); + void (*fixup)(struct sfp *sfp); +}; + struct sfp_socket_ops { void (*attach)(struct sfp *sfp); void (*detach)(struct sfp *sfp); @@ -23,7 +31,8 @@ int sfp_add_phy(struct sfp_bus *bus, struct phy_device *phydev); void sfp_remove_phy(struct sfp_bus *bus); void sfp_link_up(struct sfp_bus *bus); void sfp_link_down(struct sfp_bus *bus); -int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id); +int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id, + const struct sfp_quirk *quirk); void sfp_module_remove(struct sfp_bus *bus); int sfp_module_start(struct sfp_bus *bus); void sfp_module_stop(struct sfp_bus *bus); diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c index 69423b8965b3c84dfdd46d95258637b83a1045c6..ac7481ce2fc16c1a68d964396cb43d861f32cc4b 100644 --- a/drivers/net/phy/smsc.c +++ b/drivers/net/phy/smsc.c @@ -46,7 +46,6 @@ static struct smsc_hw_stat smsc_hw_stats[] = { struct smsc_phy_priv { u16 intmask; bool energy_enable; - struct clk *refclk; }; static int smsc_phy_ack_interrupt(struct phy_device *phydev) @@ -285,20 +284,12 @@ static void smsc_get_stats(struct phy_device *phydev, data[i] = smsc_get_stat(phydev, i); } -static void smsc_phy_remove(struct phy_device *phydev) -{ - struct smsc_phy_priv *priv = phydev->priv; - - clk_disable_unprepare(priv->refclk); - clk_put(priv->refclk); -} - static int smsc_phy_probe(struct phy_device *phydev) { struct device *dev = &phydev->mdio.dev; struct device_node *of_node = dev->of_node; struct smsc_phy_priv *priv; - int ret; + struct clk *refclk; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -312,22 +303,12 @@ static int smsc_phy_probe(struct phy_device *phydev) phydev->priv = priv; /* Make clk optional to keep DTB backward compatibility. */ - priv->refclk = clk_get_optional(dev, NULL); - if (IS_ERR(priv->refclk)) - return dev_err_probe(dev, PTR_ERR(priv->refclk), + refclk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(refclk)) + return dev_err_probe(dev, PTR_ERR(refclk), "Failed to request clock\n"); - ret = clk_prepare_enable(priv->refclk); - if (ret) - return ret; - - ret = clk_set_rate(priv->refclk, 50 * 1000 * 1000); - if (ret) { - clk_disable_unprepare(priv->refclk); - return ret; - } - - return 0; + return clk_set_rate(refclk, 50 * 1000 * 1000); } static struct phy_driver smsc_phy_driver[] = { @@ -429,7 +410,6 @@ static struct phy_driver smsc_phy_driver[] = { /* PHY_BASIC_FEATURES */ .probe = smsc_phy_probe, - .remove = smsc_phy_remove, /* basic functions */ .read_status = lan87xx_read_status, diff --git a/drivers/net/phy/spi_ks8995.c b/drivers/net/phy/spi_ks8995.c index ff37f8ba6758bfc6ac798c3a7dea31f921a0c7ee..d4202d40d47aa674ddd2f6ecbdae0637259123b6 100644 --- a/drivers/net/phy/spi_ks8995.c +++ b/drivers/net/phy/spi_ks8995.c @@ -17,7 +17,6 @@ #include #include #include -#include #include @@ -137,15 +136,10 @@ static const struct ks8995_chip_params ks8995_chip[] = { }, }; -struct ks8995_pdata { - int reset_gpio; - enum of_gpio_flags reset_gpio_flags; -}; - struct ks8995_switch { struct spi_device *spi; struct mutex lock; - struct ks8995_pdata *pdata; + struct gpio_desc *reset_gpio; struct bin_attribute regs_attr; const struct ks8995_chip_params *chip; int revision_id; @@ -401,24 +395,6 @@ err_out: return err; } -/* ks8995_parse_dt - setup platform data from devicetree - * @ks: pointer to switch instance - * - * Parses supported DT properties and sets up platform data - * accordingly. - */ -static void ks8995_parse_dt(struct ks8995_switch *ks) -{ - struct device_node *np = ks->spi->dev.of_node; - struct ks8995_pdata *pdata = ks->pdata; - - if (!np) - return; - - pdata->reset_gpio = of_get_named_gpio_flags(np, "reset-gpios", 0, - &pdata->reset_gpio_flags); -} - static const struct bin_attribute ks8995_registers_attr = { .attr = { .name = "registers", @@ -449,38 +425,22 @@ static int ks8995_probe(struct spi_device *spi) ks->spi = spi; ks->chip = &ks8995_chip[variant]; - if (ks->spi->dev.of_node) { - ks->pdata = devm_kzalloc(&spi->dev, sizeof(*ks->pdata), - GFP_KERNEL); - if (!ks->pdata) - return -ENOMEM; - - ks->pdata->reset_gpio = -1; - - ks8995_parse_dt(ks); + ks->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset", + GPIOD_OUT_HIGH); + err = PTR_ERR_OR_ZERO(ks->reset_gpio); + if (err) { + dev_err(&spi->dev, + "failed to get reset gpio: %d\n", err); + return err; } - if (!ks->pdata) - ks->pdata = spi->dev.platform_data; + err = gpiod_set_consumer_name(ks->reset_gpio, "switch-reset"); + if (err) + return err; /* de-assert switch reset */ - if (ks->pdata && gpio_is_valid(ks->pdata->reset_gpio)) { - unsigned long flags; - - flags = (ks->pdata->reset_gpio_flags == OF_GPIO_ACTIVE_LOW ? - GPIOF_ACTIVE_LOW : 0); - - err = devm_gpio_request_one(&spi->dev, - ks->pdata->reset_gpio, - flags, "switch-reset"); - if (err) { - dev_err(&spi->dev, - "failed to get reset-gpios: %d\n", err); - return -EIO; - } - - gpiod_set_value(gpio_to_desc(ks->pdata->reset_gpio), 0); - } + /* FIXME: this likely requires a delay */ + gpiod_set_value_cansleep(ks->reset_gpio, 0); spi_set_drvdata(spi, ks); @@ -524,8 +484,7 @@ static void ks8995_remove(struct spi_device *spi) sysfs_remove_bin_file(&spi->dev.kobj, &ks->regs_attr); /* assert reset */ - if (ks->pdata && gpio_is_valid(ks->pdata->reset_gpio)) - gpiod_set_value(gpio_to_desc(ks->pdata->reset_gpio), 1); + gpiod_set_value_cansleep(ks->reset_gpio, 1); } /* ------------------------------------------------------------------------ */ diff --git a/drivers/net/pse-pd/Kconfig b/drivers/net/pse-pd/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..687dec49c1e13fa079c10f75936e543a8db14396 --- /dev/null +++ b/drivers/net/pse-pd/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Ethernet Power Sourcing Equipment drivers +# + +menuconfig PSE_CONTROLLER + bool "Ethernet Power Sourcing Equipment Support" + help + Generic Power Sourcing Equipment Controller support. + + If unsure, say no. + +if PSE_CONTROLLER + +config PSE_REGULATOR + tristate "Regulator based PSE controller" + depends on REGULATOR || COMPILE_TEST + help + This module provides support for simple regulator based Ethernet Power + Sourcing Equipment without automatic classification support. For + example for basic implementation of PoDL (802.3bu) specification. + +endif diff --git a/drivers/net/pse-pd/Makefile b/drivers/net/pse-pd/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..1b8aa4c70f0b94d36afbc84feed0646b03207f28 --- /dev/null +++ b/drivers/net/pse-pd/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Makefile for Linux PSE drivers + +obj-$(CONFIG_PSE_CONTROLLER) += pse_core.o + +obj-$(CONFIG_PSE_REGULATOR) += pse_regulator.o diff --git a/drivers/net/pse-pd/pse_core.c b/drivers/net/pse-pd/pse_core.c new file mode 100644 index 0000000000000000000000000000000000000000..146b81f08a890ca93efda3c46e37d28e6c36c1ad --- /dev/null +++ b/drivers/net/pse-pd/pse_core.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Framework for Ethernet Power Sourcing Equipment +// +// Copyright (c) 2022 Pengutronix, Oleksij Rempel +// + +#include +#include +#include + +static DEFINE_MUTEX(pse_list_mutex); +static LIST_HEAD(pse_controller_list); + +/** + * struct pse_control - a PSE control + * @pcdev: a pointer to the PSE controller device + * this PSE control belongs to + * @list: list entry for the pcdev's PSE controller list + * @id: ID of the PSE line in the PSE controller device + * @refcnt: Number of gets of this pse_control + */ +struct pse_control { + struct pse_controller_dev *pcdev; + struct list_head list; + unsigned int id; + struct kref refcnt; +}; + +/** + * of_pse_zero_xlate - dummy function for controllers with one only control + * @pcdev: a pointer to the PSE controller device + * @pse_spec: PSE line specifier as found in the device tree + * + * This static translation function is used by default if of_xlate in + * :c:type:`pse_controller_dev` is not set. It is useful for all PSE + * controllers with #pse-cells = <0>. + */ +static int of_pse_zero_xlate(struct pse_controller_dev *pcdev, + const struct of_phandle_args *pse_spec) +{ + return 0; +} + +/** + * of_pse_simple_xlate - translate pse_spec to the PSE line number + * @pcdev: a pointer to the PSE controller device + * @pse_spec: PSE line specifier as found in the device tree + * + * This static translation function is used by default if of_xlate in + * :c:type:`pse_controller_dev` is not set. It is useful for all PSE + * controllers with 1:1 mapping, where PSE lines can be indexed by number + * without gaps. + */ +static int of_pse_simple_xlate(struct pse_controller_dev *pcdev, + const struct of_phandle_args *pse_spec) +{ + if (pse_spec->args[0] >= pcdev->nr_lines) + return -EINVAL; + + return pse_spec->args[0]; +} + +/** + * pse_controller_register - register a PSE controller device + * @pcdev: a pointer to the initialized PSE controller device + */ +int pse_controller_register(struct pse_controller_dev *pcdev) +{ + if (!pcdev->of_xlate) { + if (pcdev->of_pse_n_cells == 0) + pcdev->of_xlate = of_pse_zero_xlate; + else if (pcdev->of_pse_n_cells == 1) + pcdev->of_xlate = of_pse_simple_xlate; + } + + mutex_init(&pcdev->lock); + INIT_LIST_HEAD(&pcdev->pse_control_head); + + mutex_lock(&pse_list_mutex); + list_add(&pcdev->list, &pse_controller_list); + mutex_unlock(&pse_list_mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(pse_controller_register); + +/** + * pse_controller_unregister - unregister a PSE controller device + * @pcdev: a pointer to the PSE controller device + */ +void pse_controller_unregister(struct pse_controller_dev *pcdev) +{ + mutex_lock(&pse_list_mutex); + list_del(&pcdev->list); + mutex_unlock(&pse_list_mutex); +} +EXPORT_SYMBOL_GPL(pse_controller_unregister); + +static void devm_pse_controller_release(struct device *dev, void *res) +{ + pse_controller_unregister(*(struct pse_controller_dev **)res); +} + +/** + * devm_pse_controller_register - resource managed pse_controller_register() + * @dev: device that is registering this PSE controller + * @pcdev: a pointer to the initialized PSE controller device + * + * Managed pse_controller_register(). For PSE controllers registered by + * this function, pse_controller_unregister() is automatically called on + * driver detach. See pse_controller_register() for more information. + */ +int devm_pse_controller_register(struct device *dev, + struct pse_controller_dev *pcdev) +{ + struct pse_controller_dev **pcdevp; + int ret; + + pcdevp = devres_alloc(devm_pse_controller_release, sizeof(*pcdevp), + GFP_KERNEL); + if (!pcdevp) + return -ENOMEM; + + ret = pse_controller_register(pcdev); + if (ret) { + devres_free(pcdevp); + return ret; + } + + *pcdevp = pcdev; + devres_add(dev, pcdevp); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_pse_controller_register); + +/* PSE control section */ + +static void __pse_control_release(struct kref *kref) +{ + struct pse_control *psec = container_of(kref, struct pse_control, + refcnt); + + lockdep_assert_held(&pse_list_mutex); + + module_put(psec->pcdev->owner); + + list_del(&psec->list); + kfree(psec); +} + +static void __pse_control_put_internal(struct pse_control *psec) +{ + lockdep_assert_held(&pse_list_mutex); + + kref_put(&psec->refcnt, __pse_control_release); +} + +/** + * pse_control_put - free the PSE control + * @psec: PSE control pointer + */ +void pse_control_put(struct pse_control *psec) +{ + if (IS_ERR_OR_NULL(psec)) + return; + + mutex_lock(&pse_list_mutex); + __pse_control_put_internal(psec); + mutex_unlock(&pse_list_mutex); +} +EXPORT_SYMBOL_GPL(pse_control_put); + +static struct pse_control * +pse_control_get_internal(struct pse_controller_dev *pcdev, unsigned int index) +{ + struct pse_control *psec; + + lockdep_assert_held(&pse_list_mutex); + + list_for_each_entry(psec, &pcdev->pse_control_head, list) { + if (psec->id == index) { + kref_get(&psec->refcnt); + return psec; + } + } + + psec = kzalloc(sizeof(*psec), GFP_KERNEL); + if (!psec) + return ERR_PTR(-ENOMEM); + + if (!try_module_get(pcdev->owner)) { + kfree(psec); + return ERR_PTR(-ENODEV); + } + + psec->pcdev = pcdev; + list_add(&psec->list, &pcdev->pse_control_head); + psec->id = index; + kref_init(&psec->refcnt); + + return psec; +} + +struct pse_control * +of_pse_control_get(struct device_node *node) +{ + struct pse_controller_dev *r, *pcdev; + struct of_phandle_args args; + struct pse_control *psec; + int psec_id; + int ret; + + if (!node) + return ERR_PTR(-EINVAL); + + ret = of_parse_phandle_with_args(node, "pses", "#pse-cells", 0, &args); + if (ret) + return ERR_PTR(ret); + + mutex_lock(&pse_list_mutex); + pcdev = NULL; + list_for_each_entry(r, &pse_controller_list, list) { + if (args.np == r->dev->of_node) { + pcdev = r; + break; + } + } + + if (!pcdev) { + psec = ERR_PTR(-EPROBE_DEFER); + goto out; + } + + if (WARN_ON(args.args_count != pcdev->of_pse_n_cells)) { + psec = ERR_PTR(-EINVAL); + goto out; + } + + psec_id = pcdev->of_xlate(pcdev, &args); + if (psec_id < 0) { + psec = ERR_PTR(psec_id); + goto out; + } + + /* pse_list_mutex also protects the pcdev's pse_control list */ + psec = pse_control_get_internal(pcdev, psec_id); + +out: + mutex_unlock(&pse_list_mutex); + of_node_put(args.np); + + return psec; +} +EXPORT_SYMBOL_GPL(of_pse_control_get); + +/** + * pse_ethtool_get_status - get status of PSE control + * @psec: PSE control pointer + * @extack: extack for reporting useful error messages + * @status: struct to store PSE status + */ +int pse_ethtool_get_status(struct pse_control *psec, + struct netlink_ext_ack *extack, + struct pse_control_status *status) +{ + const struct pse_controller_ops *ops; + int err; + + ops = psec->pcdev->ops; + + if (!ops->ethtool_get_status) { + NL_SET_ERR_MSG(extack, + "PSE driver does not support status report"); + return -EOPNOTSUPP; + } + + mutex_lock(&psec->pcdev->lock); + err = ops->ethtool_get_status(psec->pcdev, psec->id, extack, status); + mutex_unlock(&psec->pcdev->lock); + + return err; +} +EXPORT_SYMBOL_GPL(pse_ethtool_get_status); + +/** + * pse_ethtool_set_config - set PSE control configuration + * @psec: PSE control pointer + * @extack: extack for reporting useful error messages + * @config: Configuration of the test to run + */ +int pse_ethtool_set_config(struct pse_control *psec, + struct netlink_ext_ack *extack, + const struct pse_control_config *config) +{ + const struct pse_controller_ops *ops; + int err; + + ops = psec->pcdev->ops; + + if (!ops->ethtool_set_config) { + NL_SET_ERR_MSG(extack, + "PSE driver does not configuration"); + return -EOPNOTSUPP; + } + + mutex_lock(&psec->pcdev->lock); + err = ops->ethtool_set_config(psec->pcdev, psec->id, extack, config); + mutex_unlock(&psec->pcdev->lock); + + return err; +} +EXPORT_SYMBOL_GPL(pse_ethtool_set_config); diff --git a/drivers/net/pse-pd/pse_regulator.c b/drivers/net/pse-pd/pse_regulator.c new file mode 100644 index 0000000000000000000000000000000000000000..e2bf8306ca90b96de86bc800afcf3d14809fc472 --- /dev/null +++ b/drivers/net/pse-pd/pse_regulator.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Driver for the regulator based Ethernet Power Sourcing Equipment, without +// auto classification support. +// +// Copyright (c) 2022 Pengutronix, Oleksij Rempel +// + +#include +#include +#include +#include +#include + +struct pse_reg_priv { + struct pse_controller_dev pcdev; + struct regulator *ps; /*power source */ + enum ethtool_podl_pse_admin_state admin_state; +}; + +static struct pse_reg_priv *to_pse_reg(struct pse_controller_dev *pcdev) +{ + return container_of(pcdev, struct pse_reg_priv, pcdev); +} + +static int +pse_reg_ethtool_set_config(struct pse_controller_dev *pcdev, unsigned long id, + struct netlink_ext_ack *extack, + const struct pse_control_config *config) +{ + struct pse_reg_priv *priv = to_pse_reg(pcdev); + int ret; + + if (priv->admin_state == config->admin_cotrol) + return 0; + + switch (config->admin_cotrol) { + case ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED: + ret = regulator_enable(priv->ps); + break; + case ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED: + ret = regulator_disable(priv->ps); + break; + default: + dev_err(pcdev->dev, "Unknown admin state %i\n", + config->admin_cotrol); + ret = -ENOTSUPP; + } + + if (ret) + return ret; + + priv->admin_state = config->admin_cotrol; + + return 0; +} + +static int +pse_reg_ethtool_get_status(struct pse_controller_dev *pcdev, unsigned long id, + struct netlink_ext_ack *extack, + struct pse_control_status *status) +{ + struct pse_reg_priv *priv = to_pse_reg(pcdev); + int ret; + + ret = regulator_is_enabled(priv->ps); + if (ret < 0) + return ret; + + if (!ret) + status->podl_pw_status = ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED; + else + status->podl_pw_status = + ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING; + + status->podl_admin_state = priv->admin_state; + + return 0; +} + +static const struct pse_controller_ops pse_reg_ops = { + .ethtool_get_status = pse_reg_ethtool_get_status, + .ethtool_set_config = pse_reg_ethtool_set_config, +}; + +static int +pse_reg_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pse_reg_priv *priv; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + if (!pdev->dev.of_node) + return -ENOENT; + + priv->ps = devm_regulator_get_exclusive(dev, "pse"); + if (IS_ERR(priv->ps)) + return dev_err_probe(dev, PTR_ERR(priv->ps), + "failed to get PSE regulator.\n"); + + platform_set_drvdata(pdev, priv); + + ret = regulator_is_enabled(priv->ps); + if (ret < 0) + return ret; + + if (ret) + priv->admin_state = ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED; + else + priv->admin_state = ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED; + + priv->pcdev.owner = THIS_MODULE; + priv->pcdev.ops = &pse_reg_ops; + priv->pcdev.dev = dev; + ret = devm_pse_controller_register(dev, &priv->pcdev); + if (ret) { + dev_err(dev, "failed to register PSE controller (%pe)\n", + ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static const __maybe_unused struct of_device_id pse_reg_of_match[] = { + { .compatible = "podl-pse-regulator", }, + { }, +}; +MODULE_DEVICE_TABLE(of, pse_reg_of_match); + +static struct platform_driver pse_reg_driver = { + .probe = pse_reg_probe, + .driver = { + .name = "PSE regulator", + .of_match_table = of_match_ptr(pse_reg_of_match), + }, +}; +module_platform_driver(pse_reg_driver); + +MODULE_AUTHOR("Oleksij Rempel "); +MODULE_DESCRIPTION("regulator based Ethernet Power Sourcing Equipment"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:pse-regulator"); diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c index 39e61e07e4894aa3eb6bffc9710a853a9da0aca3..fbcb9d05da649ebda37aa2440cf5893ab838aa43 100644 --- a/drivers/net/rionet.c +++ b/drivers/net/rionet.c @@ -443,10 +443,10 @@ static void rionet_get_drvinfo(struct net_device *ndev, { struct rionet_private *rnet = netdev_priv(ndev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); - strlcpy(info->fw_version, "n/a", sizeof(info->fw_version)); - strlcpy(info->bus_info, rnet->mport->name, sizeof(info->bus_info)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->fw_version, "n/a", sizeof(info->fw_version)); + strscpy(info->bus_info, rnet->mport->name, sizeof(info->bus_info)); } static u32 rionet_get_msglevel(struct net_device *ndev) diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index aac133a1e27a5f64fe8f83a456aa0598fad6824c..62ade69295a94a3fd0fce6475b37116e37b9d6ec 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c @@ -1275,10 +1275,12 @@ static int team_port_add(struct team *team, struct net_device *port_dev, } } - netif_addr_lock_bh(dev); - dev_uc_sync_multiple(port_dev, dev); - dev_mc_sync_multiple(port_dev, dev); - netif_addr_unlock_bh(dev); + if (dev->flags & IFF_UP) { + netif_addr_lock_bh(dev); + dev_uc_sync_multiple(port_dev, dev); + dev_mc_sync_multiple(port_dev, dev); + netif_addr_unlock_bh(dev); + } port->index = -1; list_add_tail_rcu(&port->list, &team->port_list); @@ -1349,8 +1351,10 @@ static int team_port_del(struct team *team, struct net_device *port_dev) netdev_rx_handler_unregister(port_dev); team_port_disable_netpoll(port); vlan_vids_del_by_dev(port_dev, dev); - dev_uc_unsync(port_dev, dev); - dev_mc_unsync(port_dev, dev); + if (dev->flags & IFF_UP) { + dev_uc_unsync(port_dev, dev); + dev_mc_unsync(port_dev, dev); + } dev_close(port_dev); team_port_leave(team, port); @@ -1700,6 +1704,14 @@ static int team_open(struct net_device *dev) static int team_close(struct net_device *dev) { + struct team *team = netdev_priv(dev); + struct team_port *port; + + list_for_each_entry(port, &team->port_list, list) { + dev_uc_unsync(port->dev, dev); + dev_mc_unsync(port->dev, dev); + } + return 0; } @@ -2070,8 +2082,8 @@ static const struct net_device_ops team_netdev_ops = { static void team_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); + strscpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version)); } static int team_ethtool_get_link_ksettings(struct net_device *dev, @@ -2840,6 +2852,7 @@ static struct genl_family team_nl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = team_nl_ops, .n_small_ops = ARRAY_SIZE(team_nl_ops), + .resv_start_op = TEAM_CMD_PORT_LIST_GET + 1, .mcgrps = team_nl_mcgrps, .n_mcgrps = ARRAY_SIZE(team_nl_mcgrps), }; diff --git a/drivers/net/thunderbolt.c b/drivers/net/thunderbolt.c index ff5d0e98a0881643be3607a83d067301da2a354a..83fcaeb2ac5e81d0e9b1e80b3667e355b1ec7bf9 100644 --- a/drivers/net/thunderbolt.c +++ b/drivers/net/thunderbolt.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Networking over Thunderbolt cable using Apple ThunderboltIP protocol + * Networking over Thunderbolt/USB4 cables using USB4NET protocol + * (formerly Apple ThunderboltIP). * * Copyright (C) 2017, Intel Corporation * Authors: Amir Levy @@ -30,6 +31,7 @@ #define TBNET_RING_SIZE 256 #define TBNET_LOGIN_RETRIES 60 #define TBNET_LOGOUT_RETRIES 10 +#define TBNET_E2E BIT(0) #define TBNET_MATCH_FRAGS_ID BIT(1) #define TBNET_64K_FRAMES BIT(2) #define TBNET_MAX_MTU SZ_64K @@ -209,6 +211,10 @@ static const uuid_t tbnet_svc_uuid = static struct tb_property_dir *tbnet_dir; +static bool tbnet_e2e = true; +module_param_named(e2e, tbnet_e2e, bool, 0444); +MODULE_PARM_DESC(e2e, "USB4NET full end-to-end flow control (default: true)"); + static void tbnet_fill_header(struct thunderbolt_ip_header *hdr, u64 route, u8 sequence, const uuid_t *initiator_uuid, const uuid_t *target_uuid, enum thunderbolt_ip_type type, size_t size, u32 command_id) @@ -612,18 +618,13 @@ static void tbnet_connected_work(struct work_struct *work) return; } - /* Both logins successful so enable the high-speed DMA paths and - * start the network device queue. + /* Both logins successful so enable the rings, high-speed DMA + * paths and start the network device queue. + * + * Note we enable the DMA paths last to make sure we have primed + * the Rx ring before any incoming packets are allowed to + * arrive. */ - ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path, - net->rx_ring.ring->hop, - net->remote_transmit_path, - net->tx_ring.ring->hop); - if (ret) { - netdev_err(net->dev, "failed to enable DMA paths\n"); - return; - } - tb_ring_start(net->tx_ring.ring); tb_ring_start(net->rx_ring.ring); @@ -635,10 +636,21 @@ static void tbnet_connected_work(struct work_struct *work) if (ret) goto err_free_rx_buffers; + ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path, + net->rx_ring.ring->hop, + net->remote_transmit_path, + net->tx_ring.ring->hop); + if (ret) { + netdev_err(net->dev, "failed to enable DMA paths\n"); + goto err_free_tx_buffers; + } + netif_carrier_on(net->dev); netif_start_queue(net->dev); return; +err_free_tx_buffers: + tbnet_free_buffers(&net->tx_ring); err_free_rx_buffers: tbnet_free_buffers(&net->rx_ring); err_stop_rings: @@ -867,6 +879,7 @@ static int tbnet_open(struct net_device *dev) struct tb_xdomain *xd = net->xd; u16 sof_mask, eof_mask; struct tb_ring *ring; + unsigned int flags; int hopid; netif_carrier_off(dev); @@ -891,9 +904,14 @@ static int tbnet_open(struct net_device *dev) sof_mask = BIT(TBIP_PDF_FRAME_START); eof_mask = BIT(TBIP_PDF_FRAME_END); - ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, - RING_FLAG_FRAME, 0, sof_mask, eof_mask, - tbnet_start_poll, net); + flags = RING_FLAG_FRAME; + /* Only enable full E2E if the other end supports it too */ + if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E) + flags |= RING_FLAG_E2E; + + ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags, + net->tx_ring.ring->hop, sof_mask, + eof_mask, tbnet_start_poll, net); if (!ring) { netdev_err(dev, "failed to allocate Rx ring\n"); tb_ring_free(net->tx_ring.ring); @@ -1264,7 +1282,7 @@ static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id) dev->features = dev->hw_features | NETIF_F_HIGHDMA; dev->hard_header_len += sizeof(struct thunderbolt_ip_frame_header); - netif_napi_add(dev, &net->napi, tbnet_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &net->napi, tbnet_poll); /* MTU range: 68 - 65522 */ dev->min_mtu = ETH_MIN_MTU; @@ -1356,6 +1374,7 @@ static struct tb_service_driver tbnet_driver = { static int __init tbnet_init(void) { + unsigned int flags; int ret; tbnet_dir = tb_property_create_dir(&tbnet_dir_uuid); @@ -1365,12 +1384,11 @@ static int __init tbnet_init(void) tb_property_add_immediate(tbnet_dir, "prtcid", 1); tb_property_add_immediate(tbnet_dir, "prtcvers", 1); tb_property_add_immediate(tbnet_dir, "prtcrevs", 1); - /* Currently only announce support for match frags ID (bit 1). Bit 0 - * is reserved for full E2E flow control which we do not support at - * the moment. - */ - tb_property_add_immediate(tbnet_dir, "prtcstns", - TBNET_MATCH_FRAGS_ID | TBNET_64K_FRAMES); + + flags = TBNET_MATCH_FRAGS_ID | TBNET_64K_FRAMES; + if (tbnet_e2e) + flags |= TBNET_E2E; + tb_property_add_immediate(tbnet_dir, "prtcstns", flags); ret = tb_register_property_dir("network", tbnet_dir); if (ret) { @@ -1393,5 +1411,5 @@ module_exit(tbnet_exit); MODULE_AUTHOR("Amir Levy "); MODULE_AUTHOR("Michael Jamet "); MODULE_AUTHOR("Mika Westerberg "); -MODULE_DESCRIPTION("Thunderbolt network driver"); +MODULE_DESCRIPTION("Thunderbolt/USB4 network driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 259b2b84b2b39453cb3de27c632cf9bc0f9ce2b7..27c6d235cbda32f61a00d6879d0b611bd3ff384b 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -2664,7 +2664,7 @@ static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr, char *buf) { struct tun_struct *tun = netdev_priv(to_net_dev(dev)); - return sprintf(buf, "0x%x\n", tun_flags(tun)); + return sysfs_emit(buf, "0x%x\n", tun_flags(tun)); } static ssize_t owner_show(struct device *dev, struct device_attribute *attr, @@ -2672,9 +2672,9 @@ static ssize_t owner_show(struct device *dev, struct device_attribute *attr, { struct tun_struct *tun = netdev_priv(to_net_dev(dev)); return uid_valid(tun->owner)? - sprintf(buf, "%u\n", - from_kuid_munged(current_user_ns(), tun->owner)): - sprintf(buf, "-1\n"); + sysfs_emit(buf, "%u\n", + from_kuid_munged(current_user_ns(), tun->owner)) : + sysfs_emit(buf, "-1\n"); } static ssize_t group_show(struct device *dev, struct device_attribute *attr, @@ -2682,9 +2682,9 @@ static ssize_t group_show(struct device *dev, struct device_attribute *attr, { struct tun_struct *tun = netdev_priv(to_net_dev(dev)); return gid_valid(tun->group) ? - sprintf(buf, "%u\n", - from_kgid_munged(current_user_ns(), tun->group)): - sprintf(buf, "-1\n"); + sysfs_emit(buf, "%u\n", + from_kgid_munged(current_user_ns(), tun->group)) : + sysfs_emit(buf, "-1\n"); } static DEVICE_ATTR_RO(tun_flags); @@ -2828,7 +2828,10 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) rcu_assign_pointer(tfile->tun, tun); } - netif_carrier_on(tun->dev); + if (ifr->ifr_flags & IFF_NO_CARRIER) + netif_carrier_off(tun->dev); + else + netif_carrier_on(tun->dev); /* Make sure persistent devices do not get stuck in * xoff state. @@ -3056,8 +3059,8 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd, * This is needed because we never checked for invalid flags on * TUNSETIFF. */ - return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES, - (unsigned int __user*)argp); + return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER | + TUN_FEATURES, (unsigned int __user*)argp); } else if (cmd == TUNSETQUEUE) { return tun_set_queue(file, &ifr); } else if (cmd == SIOCGSKNS) { @@ -3540,15 +3543,15 @@ static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info { struct tun_struct *tun = netdev_priv(dev); - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); switch (tun->flags & TUN_TYPE_MASK) { case IFF_TUN: - strlcpy(info->bus_info, "tun", sizeof(info->bus_info)); + strscpy(info->bus_info, "tun", sizeof(info->bus_info)); break; case IFF_TAP: - strlcpy(info->bus_info, "tap", sizeof(info->bus_info)); + strscpy(info->bus_info, "tap", sizeof(info->bus_info)); break; } } diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig index 76659c1c525a2c5d4ca8df7ab080b97cc5a8a84c..4402eedb3d1a24e10561006a233d784cb06b869b 100644 --- a/drivers/net/usb/Kconfig +++ b/drivers/net/usb/Kconfig @@ -168,7 +168,7 @@ config USB_NET_AX8817X tristate "ASIX AX88xxx Based USB 2.0 Ethernet Adapters" depends on USB_USBNET select CRC32 - select PHYLIB + select PHYLINK select AX88796B_PHY imply NET_SELFTESTS default y diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c index 3020e81159d0223f1e088a394ffed3ed2fcdb632..a017e9de2119d5f5163981b9777e13e94f523f86 100644 --- a/drivers/net/usb/aqc111.c +++ b/drivers/net/usb/aqc111.c @@ -201,7 +201,7 @@ static void aqc111_get_drvinfo(struct net_device *net, /* Inherit standard device info */ usbnet_get_drvinfo(net, info); - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); snprintf(info->fw_version, sizeof(info->fw_version), "%u.%u.%u", aqc111_data->fw_ver.major, aqc111_data->fw_ver.minor, diff --git a/drivers/net/usb/asix.h b/drivers/net/usb/asix.h index 21c1ca275cc447c6686381ca00ce0f29693e855e..74162190bccc10e37232bbc3b698b112a96b5af6 100644 --- a/drivers/net/usb/asix.h +++ b/drivers/net/usb/asix.h @@ -27,6 +27,7 @@ #include #include #include +#include #define DRIVER_VERSION "22-Dec-2011" #define DRIVER_NAME "asix" @@ -185,6 +186,8 @@ struct asix_common_private { struct mii_bus *mdio; struct phy_device *phydev; struct phy_device *phydev_int; + struct phylink *phylink; + struct phylink_config phylink_config; u16 phy_addr; bool embd_phy; u8 chipcode; diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c index 9ea91c3ff0458ca39a32564c45ca0e90f6b24b14..72ffc89b477ad81b2cd810b43f3cccedd8044b1e 100644 --- a/drivers/net/usb/asix_common.c +++ b/drivers/net/usb/asix_common.c @@ -752,8 +752,8 @@ void asix_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info) { /* Inherit standard device info */ usbnet_get_drvinfo(net, info); - strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->driver, DRIVER_NAME, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); } int asix_set_mac_address(struct net_device *net, void *p) diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c index 5b5eb630c4b7918371a48b098a15dcdb6c070a68..11f60d32be82eba670ed88a2402e59f9bd3521bd 100644 --- a/drivers/net/usb/asix_devices.c +++ b/drivers/net/usb/asix_devices.c @@ -303,6 +303,24 @@ static int ax88772_ethtool_get_sset_count(struct net_device *ndev, int sset) } } +static void ax88772_ethtool_get_pauseparam(struct net_device *ndev, + struct ethtool_pauseparam *pause) +{ + struct usbnet *dev = netdev_priv(ndev); + struct asix_common_private *priv = dev->driver_priv; + + phylink_ethtool_get_pauseparam(priv->phylink, pause); +} + +static int ax88772_ethtool_set_pauseparam(struct net_device *ndev, + struct ethtool_pauseparam *pause) +{ + struct usbnet *dev = netdev_priv(ndev); + struct asix_common_private *priv = dev->driver_priv; + + return phylink_ethtool_set_pauseparam(priv->phylink, pause); +} + static const struct ethtool_ops ax88772_ethtool_ops = { .get_drvinfo = asix_get_drvinfo, .get_link = usbnet_get_link, @@ -319,6 +337,8 @@ static const struct ethtool_ops ax88772_ethtool_ops = { .self_test = net_selftest, .get_strings = ax88772_ethtool_get_strings, .get_sset_count = ax88772_ethtool_get_sset_count, + .get_pauseparam = ax88772_ethtool_get_pauseparam, + .set_pauseparam = ax88772_ethtool_set_pauseparam, }; static int ax88772_reset(struct usbnet *dev) @@ -343,7 +363,7 @@ static int ax88772_reset(struct usbnet *dev) if (ret < 0) goto out; - phy_start(priv->phydev); + phylink_start(priv->phylink); return 0; @@ -590,8 +610,11 @@ static void ax88772_suspend(struct usbnet *dev) struct asix_common_private *priv = dev->driver_priv; u16 medium; - if (netif_running(dev->net)) - phy_stop(priv->phydev); + if (netif_running(dev->net)) { + rtnl_lock(); + phylink_suspend(priv->phylink, false); + rtnl_unlock(); + } /* Stop MAC operation */ medium = asix_read_medium_status(dev, 1); @@ -622,8 +645,11 @@ static void ax88772_resume(struct usbnet *dev) if (!priv->reset(dev, 1)) break; - if (netif_running(dev->net)) - phy_start(priv->phydev); + if (netif_running(dev->net)) { + rtnl_lock(); + phylink_resume(priv->phylink); + rtnl_unlock(); + } } static int asix_resume(struct usb_interface *intf) @@ -667,8 +693,7 @@ static int ax88772_init_phy(struct usbnet *dev) return -ENODEV; } - ret = phy_connect_direct(dev->net, priv->phydev, &asix_adjust_link, - PHY_INTERFACE_MODE_INTERNAL); + ret = phylink_connect_phy(priv->phylink, priv->phydev); if (ret) { netdev_err(dev->net, "Could not connect PHY\n"); return ret; @@ -688,6 +713,9 @@ static int ax88772_init_phy(struct usbnet *dev) */ priv->phydev_int = mdiobus_get_phy(priv->mdio, AX_EMBD_PHY_ADDR); if (!priv->phydev_int) { + rtnl_lock(); + phylink_disconnect_phy(priv->phylink); + rtnl_unlock(); netdev_err(dev->net, "Could not find internal PHY\n"); return -ENODEV; } @@ -698,6 +726,89 @@ static int ax88772_init_phy(struct usbnet *dev) return 0; } +static void ax88772_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ + /* Nothing to do */ +} + +static void ax88772_mac_link_down(struct phylink_config *config, + unsigned int mode, phy_interface_t interface) +{ + struct usbnet *dev = netdev_priv(to_net_dev(config->dev)); + + asix_write_medium_mode(dev, 0, 0); + usbnet_link_change(dev, false, false); +} + +static void ax88772_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct usbnet *dev = netdev_priv(to_net_dev(config->dev)); + u16 m = AX_MEDIUM_AC | AX_MEDIUM_RE; + + m |= duplex ? AX_MEDIUM_FD : 0; + + switch (speed) { + case SPEED_100: + m |= AX_MEDIUM_PS; + break; + case SPEED_10: + break; + default: + return; + } + + if (tx_pause) + m |= AX_MEDIUM_TFC; + + if (rx_pause) + m |= AX_MEDIUM_RFC; + + asix_write_medium_mode(dev, m, 0); + usbnet_link_change(dev, true, false); +} + +static const struct phylink_mac_ops ax88772_phylink_mac_ops = { + .validate = phylink_generic_validate, + .mac_config = ax88772_mac_config, + .mac_link_down = ax88772_mac_link_down, + .mac_link_up = ax88772_mac_link_up, +}; + +static int ax88772_phylink_setup(struct usbnet *dev) +{ + struct asix_common_private *priv = dev->driver_priv; + phy_interface_t phy_if_mode; + struct phylink *phylink; + + priv->phylink_config.dev = &dev->net->dev; + priv->phylink_config.type = PHYLINK_NETDEV; + priv->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | + MAC_10 | MAC_100; + + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + priv->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + priv->phylink_config.supported_interfaces); + + if (priv->embd_phy) + phy_if_mode = PHY_INTERFACE_MODE_INTERNAL; + else + phy_if_mode = PHY_INTERFACE_MODE_RMII; + + phylink = phylink_create(&priv->phylink_config, dev->net->dev.fwnode, + phy_if_mode, &ax88772_phylink_mac_ops); + if (IS_ERR(phylink)) + return PTR_ERR(phylink); + + priv->phylink = phylink; + return 0; +} + static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf) { struct asix_common_private *priv; @@ -788,14 +899,22 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf) if (ret) return ret; - return ax88772_init_phy(dev); + ret = ax88772_phylink_setup(dev); + if (ret) + return ret; + + ret = ax88772_init_phy(dev); + if (ret) + phylink_destroy(priv->phylink); + + return ret; } static int ax88772_stop(struct usbnet *dev) { struct asix_common_private *priv = dev->driver_priv; - phy_stop(priv->phydev); + phylink_stop(priv->phylink); return 0; } @@ -804,7 +923,10 @@ static void ax88772_unbind(struct usbnet *dev, struct usb_interface *intf) { struct asix_common_private *priv = dev->driver_priv; - phy_disconnect(priv->phydev); + rtnl_lock(); + phylink_disconnect_phy(priv->phylink); + rtnl_unlock(); + phylink_destroy(priv->phylink); asix_rx_fixup_common_free(dev->driver_priv); } diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c index 843893482abd637b21c5c273fdbaa7effe97e646..ff439ef535ac9bd7ce0d1160544df591c34478aa 100644 --- a/drivers/net/usb/catc.c +++ b/drivers/net/usb/catc.c @@ -672,8 +672,8 @@ static void catc_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct catc *catc = netdev_priv(dev); - strlcpy(info->driver, driver_name, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->driver, driver_name, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); usb_make_path(catc->usbdev, info->bus_info, sizeof(info->bus_info)); } diff --git a/drivers/net/usb/cdc_ether.c b/drivers/net/usb/cdc_ether.c index 2de09ad5bac0318d0ec0edda7231a50e062e05c0..e11f70911acc10b11d22bed672649b3cb5c63f65 100644 --- a/drivers/net/usb/cdc_ether.c +++ b/drivers/net/usb/cdc_ether.c @@ -777,6 +777,13 @@ static const struct usb_device_id products[] = { }, #endif +/* Lenovo ThinkPad OneLink+ Dock (based on Realtek RTL8153) */ +{ + USB_DEVICE_AND_INTERFACE_INFO(LENOVO_VENDOR_ID, 0x3054, USB_CLASS_COMM, + USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE), + .driver_info = 0, +}, + /* ThinkPad USB-C Dock (based on Realtek RTL8153) */ { USB_DEVICE_AND_INTERFACE_INFO(LENOVO_VENDOR_ID, 0x3062, USB_CLASS_COMM, diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c index f8221a7acf620a2d657d0bce838b99ae8c84fd91..ce1f6081d582f4319bb961a87904b8a6a143db0c 100644 --- a/drivers/net/usb/hso.c +++ b/drivers/net/usb/hso.c @@ -1380,7 +1380,8 @@ static void hso_serial_cleanup(struct tty_struct *tty) } /* setup the term */ -static void hso_serial_set_termios(struct tty_struct *tty, struct ktermios *old) +static void hso_serial_set_termios(struct tty_struct *tty, + const struct ktermios *old) { struct hso_serial *serial = tty->driver_data; unsigned long flags; diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 3226ab33afae5d5f93961a233f0b40b30dd84cf1..f18ab8e220db72787bffef62c598e7c65e1e1c6e 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -4374,7 +4374,7 @@ static int lan78xx_probe(struct usb_interface *intf, netif_set_tso_max_size(netdev, LAN78XX_TSO_SIZE(dev)); - netif_napi_add(netdev, &dev->napi, lan78xx_poll, NAPI_POLL_WEIGHT); + netif_napi_add(netdev, &dev->napi, lan78xx_poll); INIT_DELAYED_WORK(&dev->wq, lan78xx_delayedwork); init_usb_anchor(&dev->deferred); diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index feb247e355f74c04319484a068bfd767c54ffb28..81ca64debc5b9441a100846265d99550ed5b7307 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -894,7 +894,7 @@ static void pegasus_get_drvinfo(struct net_device *dev, { pegasus_t *pegasus = netdev_priv(dev); - strlcpy(info->driver, driver_name, sizeof(info->driver)); + strscpy(info->driver, driver_name, sizeof(info->driver)); usb_make_path(pegasus->usb, info->bus_info, sizeof(info->bus_info)); } diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 709e3c59e3405ac9a21a6747c02f904034af6962..26c34a7c21bdd1fc56819c00cdc1ad050799e221 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -1087,6 +1087,7 @@ static const struct usb_device_id products[] = { {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0512)}, /* Quectel EG12/EM12 */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0620)}, /* Quectel EM160R-GL */ {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0800)}, /* Quectel RM500Q-GL */ + {QMI_MATCH_FF_FF_FF(0x2c7c, 0x0801)}, /* Quectel RM520N */ /* 3. Combined interface devices matching on interface number */ {QMI_FIXED_INTF(0x0408, 0xea42, 4)}, /* Yota / Megafon M100-1 */ @@ -1401,6 +1402,7 @@ static const struct usb_device_id products[] = { {QMI_FIXED_INTF(0x413c, 0x81b3, 8)}, /* Dell Wireless 5809e Gobi(TM) 4G LTE Mobile Broadband Card (rev3) */ {QMI_FIXED_INTF(0x413c, 0x81b6, 8)}, /* Dell Wireless 5811e */ {QMI_FIXED_INTF(0x413c, 0x81b6, 10)}, /* Dell Wireless 5811e */ + {QMI_FIXED_INTF(0x413c, 0x81c2, 8)}, /* Dell Wireless 5811e */ {QMI_FIXED_INTF(0x413c, 0x81cc, 8)}, /* Dell Wireless 5816e */ {QMI_FIXED_INTF(0x413c, 0x81d7, 0)}, /* Dell Wireless 5821e */ {QMI_FIXED_INTF(0x413c, 0x81d7, 1)}, /* Dell Wireless 5821e preproduction config */ diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 0f6efaabaa32b892ccb57355bb9db3941ec216ec..a481a1d831e2f4fe45562e76c5fd034c0d361ac1 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -770,6 +770,8 @@ enum rtl8152_flags { RX_EPROTO, }; +#define DEVICE_ID_LENOVO_USB_C_TRAVEL_HUB 0x721e +#define DEVICE_ID_THINKPAD_ONELINK_PLUS_DOCK 0x3054 #define DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2 0x3082 #define DEVICE_ID_THINKPAD_USB_C_DONGLE 0x720c #define DEVICE_ID_THINKPAD_USB_C_DOCK_GEN2 0xa387 @@ -1873,7 +1875,9 @@ static void intr_callback(struct urb *urb) "Stop submitting intr, status %d\n", status); return; case -EOVERFLOW: - netif_info(tp, intr, tp->netdev, "intr status -EOVERFLOW\n"); + if (net_ratelimit()) + netif_info(tp, intr, tp->netdev, + "intr status -EOVERFLOW\n"); goto resubmit; /* -EPIPE: should clear the halt */ default: @@ -2726,22 +2730,26 @@ static void _rtl8152_set_rx_mode(struct net_device *netdev) ocp_data |= RCR_AM | RCR_AAP; mc_filter[1] = 0xffffffff; mc_filter[0] = 0xffffffff; - } else if ((netdev_mc_count(netdev) > multicast_filter_limit) || - (netdev->flags & IFF_ALLMULTI)) { + } else if ((netdev->flags & IFF_MULTICAST && + netdev_mc_count(netdev) > multicast_filter_limit) || + (netdev->flags & IFF_ALLMULTI)) { /* Too many to filter perfectly -- accept all multicasts. */ ocp_data |= RCR_AM; mc_filter[1] = 0xffffffff; mc_filter[0] = 0xffffffff; } else { - struct netdev_hw_addr *ha; - mc_filter[1] = 0; mc_filter[0] = 0; - netdev_for_each_mc_addr(ha, netdev) { - int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26; - mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); - ocp_data |= RCR_AM; + if (netdev->flags & IFF_MULTICAST) { + struct netdev_hw_addr *ha; + + netdev_for_each_mc_addr(ha, netdev) { + int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26; + + mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31); + ocp_data |= RCR_AM; + } } } @@ -5906,6 +5914,11 @@ static void r8153_enter_oob(struct r8152 *tp) ocp_data &= ~NOW_IS_OOB; ocp_write_byte(tp, MCU_TYPE_PLA, PLA_OOB_CTRL, ocp_data); + /* RX FIFO settings for OOB */ + ocp_write_dword(tp, MCU_TYPE_PLA, PLA_RXFIFO_CTRL0, RXFIFO_THR1_OOB); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RXFIFO_CTRL1, RXFIFO_THR2_OOB); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RXFIFO_CTRL2, RXFIFO_THR3_OOB); + rtl_disable(tp); rtl_reset_bmu(tp); @@ -6431,21 +6444,8 @@ static void r8156_fc_parameter(struct r8152 *tp) u32 pause_on = tp->fc_pause_on ? tp->fc_pause_on : fc_pause_on_auto(tp); u32 pause_off = tp->fc_pause_off ? tp->fc_pause_off : fc_pause_off_auto(tp); - switch (tp->version) { - case RTL_VER_10: - case RTL_VER_11: - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_FULL, pause_on / 8); - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_EMPTY, pause_off / 8); - break; - case RTL_VER_12: - case RTL_VER_13: - case RTL_VER_15: - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_FULL, pause_on / 16); - ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_EMPTY, pause_off / 16); - break; - default: - break; - } + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_FULL, pause_on / 16); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_EMPTY, pause_off / 16); } static void rtl8156_change_mtu(struct r8152 *tp) @@ -6557,6 +6557,11 @@ static void rtl8156_down(struct r8152 *tp) ocp_data &= ~NOW_IS_OOB; ocp_write_byte(tp, MCU_TYPE_PLA, PLA_OOB_CTRL, ocp_data); + /* RX FIFO settings for OOB */ + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RXFIFO_FULL, 64 / 16); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_FULL, 1024 / 16); + ocp_write_word(tp, MCU_TYPE_PLA, PLA_RX_FIFO_EMPTY, 4096 / 16); + rtl_disable(tp); rtl_reset_bmu(tp); @@ -8604,11 +8609,11 @@ static void rtl8152_get_drvinfo(struct net_device *netdev, { struct r8152 *tp = netdev_priv(netdev); - strlcpy(info->driver, MODULENAME, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->driver, MODULENAME, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); usb_make_path(tp->udev, info->bus_info, sizeof(info->bus_info)); if (!IS_ERR_OR_NULL(tp->rtl_fw.fw)) - strlcpy(info->fw_version, tp->rtl_fw.version, + strscpy(info->fw_version, tp->rtl_fw.version, sizeof(info->fw_version)); } @@ -9584,6 +9589,8 @@ static bool rtl8152_supports_lenovo_macpassthru(struct usb_device *udev) if (vendor_id == VENDOR_ID_LENOVO) { switch (product_id) { + case DEVICE_ID_LENOVO_USB_C_TRAVEL_HUB: + case DEVICE_ID_THINKPAD_ONELINK_PLUS_DOCK: case DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2: case DEVICE_ID_THINKPAD_USB_C_DOCK_GEN2: case DEVICE_ID_THINKPAD_USB_C_DOCK_GEN3: @@ -9831,6 +9838,7 @@ static const struct usb_device_id rtl8152_table[] = { REALTEK_USB_DEVICE(VENDOR_ID_MICROSOFT, 0x0927), REALTEK_USB_DEVICE(VENDOR_ID_SAMSUNG, 0xa101), REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x304f), + REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x3054), REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x3062), REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x3069), REALTEK_USB_DEVICE(VENDOR_ID_LENOVO, 0x3082), diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c index 3d2bf2acca942dc94b6cd6c335fd6fe0b9711ea3..97afd7335d8685581b317c3a5311bccfd1679770 100644 --- a/drivers/net/usb/rtl8150.c +++ b/drivers/net/usb/rtl8150.c @@ -769,8 +769,8 @@ static void rtl8150_get_drvinfo(struct net_device *netdev, struct ethtool_drvinf { rtl8150_t *dev = netdev_priv(netdev); - strlcpy(info->driver, driver_name, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->driver, driver_name, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); usb_make_path(dev->udev, info->bus_info, sizeof(info->bus_info)); } diff --git a/drivers/net/usb/sierra_net.c b/drivers/net/usb/sierra_net.c index bb4cbe8fc846bd866e51f7991ff6145995c655c5..b3ae949e6f1c54e11e082f19435913cec0a0fccf 100644 --- a/drivers/net/usb/sierra_net.c +++ b/drivers/net/usb/sierra_net.c @@ -612,8 +612,8 @@ static void sierra_net_get_drvinfo(struct net_device *net, { /* Inherit standard device info */ usbnet_get_drvinfo(net, info); - strlcpy(info->driver, driver_name, sizeof(info->driver)); - strlcpy(info->version, DRIVER_VERSION, sizeof(info->version)); + strscpy(info->driver, driver_name, sizeof(info->driver)); + strscpy(info->version, DRIVER_VERSION, sizeof(info->version)); } static u32 sierra_net_get_link(struct net_device *net) diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index aaa89b4cfd507b51c6c339bc1b16d0cf40d34181..64a9a80b23094ea57948fbc8181b9dff2a373523 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1050,9 +1050,9 @@ void usbnet_get_drvinfo (struct net_device *net, struct ethtool_drvinfo *info) { struct usbnet *dev = netdev_priv(net); - strlcpy (info->driver, dev->driver_name, sizeof info->driver); - strlcpy (info->fw_version, dev->driver_info->description, - sizeof info->fw_version); + strscpy(info->driver, dev->driver_name, sizeof(info->driver)); + strscpy(info->fw_version, dev->driver_info->description, + sizeof(info->fw_version)); usb_make_path (dev->udev, info->bus_info, sizeof info->bus_info); } EXPORT_SYMBOL_GPL(usbnet_get_drvinfo); @@ -1598,6 +1598,7 @@ void usbnet_disconnect (struct usb_interface *intf) struct usbnet *dev; struct usb_device *xdev; struct net_device *net; + struct urb *urb; dev = usb_get_intfdata(intf); usb_set_intfdata(intf, NULL); @@ -1614,7 +1615,11 @@ void usbnet_disconnect (struct usb_interface *intf) net = dev->net; unregister_netdev (net); - usb_scuttle_anchored_urbs(&dev->deferred); + while ((urb = usb_get_from_anchor(&dev->deferred))) { + dev_kfree_skb(urb->context); + kfree(urb->sg); + usb_free_urb(urb); + } if (dev->driver_info->unbind) dev->driver_info->unbind(dev, intf); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 466da01ba2e3e97ba9eb16586b6d5d9f092b3d76..09682ea3354e92b05c2f42a1a37da7cc060c89a7 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -128,8 +128,8 @@ static int veth_get_link_ksettings(struct net_device *dev, static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) @@ -1070,7 +1070,7 @@ static int veth_enable_xdp_range(struct net_device *dev, int start, int end, struct veth_rq *rq = &priv->rq[i]; if (!napi_already_on) - netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &rq->xdp_napi, veth_poll); err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); if (err < 0) goto err_rxq_reg; @@ -1184,7 +1184,7 @@ static int veth_napi_enable_range(struct net_device *dev, int start, int end) for (i = start; i < end; i++) { struct veth_rq *rq = &priv->rq[i]; - netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(dev, &rq->xdp_napi, veth_poll); } err = __veth_napi_enable_range(dev, start, end); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 9cce7dec7366d41816f0e09ce9342d9869ea9ac9..7106932c6f8877f83f2ba6e5aee5ec1caa9857ea 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -225,6 +225,9 @@ struct virtnet_info { /* I like... big packets and I cannot lie! */ bool big_packets; + /* number of sg entries allocated for big packets */ + unsigned int big_packets_num_skbfrags; + /* Host will merge rx buffers for big packets (shake it! shake it!) */ bool mergeable_rx_bufs; @@ -1331,10 +1334,10 @@ static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, char *p; int i, err, offset; - sg_init_table(rq->sg, MAX_SKB_FRAGS + 2); + sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); - /* page in rq->sg[MAX_SKB_FRAGS + 1] is list tail */ - for (i = MAX_SKB_FRAGS + 1; i > 1; --i) { + /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ + for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { first = get_a_page(rq, gfp); if (!first) { if (list) @@ -1365,7 +1368,7 @@ static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, /* chain first in list head */ first->private = (unsigned long)list; - err = virtqueue_add_inbuf(rq->vq, rq->sg, MAX_SKB_FRAGS + 2, + err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, first, gfp); if (err < 0) give_pages(rq, first); @@ -2594,9 +2597,9 @@ static void virtnet_get_drvinfo(struct net_device *dev, struct virtnet_info *vi = netdev_priv(dev); struct virtio_device *vdev = vi->vdev; - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); - strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); + strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); } @@ -3682,13 +3685,35 @@ static int virtnet_validate(struct virtio_device *vdev) return 0; } +static bool virtnet_check_guest_gso(const struct virtnet_info *vi) +{ + return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || + virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO); +} + +static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) +{ + bool guest_gso = virtnet_check_guest_gso(vi); + + /* If device can receive ANY guest GSO packets, regardless of mtu, + * allocate packets of maximum size, otherwise limit it to only + * mtu size worth only. + */ + if (mtu > ETH_DATA_LEN || guest_gso) { + vi->big_packets = true; + vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); + } +} + static int virtnet_probe(struct virtio_device *vdev) { int i, err = -ENOMEM; struct net_device *dev; struct virtnet_info *vi; u16 max_queue_pairs; - int mtu; + int mtu = 0; /* Find if host supports multiqueue/rss virtio_net device */ max_queue_pairs = 1; @@ -3776,13 +3801,6 @@ static int virtnet_probe(struct virtio_device *vdev) INIT_WORK(&vi->config_work, virtnet_config_changed_work); spin_lock_init(&vi->refill_lock); - /* If we can receive ANY GSO packets, we must allocate large ones. */ - if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || - virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6) || - virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_ECN) || - virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_UFO)) - vi->big_packets = true; - if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; @@ -3848,12 +3866,10 @@ static int virtnet_probe(struct virtio_device *vdev) dev->mtu = mtu; dev->max_mtu = mtu; - - /* TODO: size buffers correctly in this case. */ - if (dev->mtu > ETH_DATA_LEN) - vi->big_packets = true; } + virtnet_set_big_packets(vi, mtu); + if (vi->any_header_sg) dev->needed_headroom = vi->hdr_len; diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index 53b3b241e027817c7859b414fe944bf97fcba35f..d3e7b27eb9332ae4e5f856f9eee10ca4d790c859 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -3882,11 +3882,11 @@ vmxnet3_probe_device(struct pci_dev *pdev, for (i = 0; i < adapter->num_rx_queues; i++) { netif_napi_add(adapter->netdev, &adapter->rx_queue[i].napi, - vmxnet3_poll_rx_only, 64); + vmxnet3_poll_rx_only); } } else { netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, - vmxnet3_poll, 64); + vmxnet3_poll); } netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c index e2034adc3a1a77c63310bfd1786b0101bad2d993..18cf7c7232010d8985ee4b4a00c181a26c4bd5a1 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethtool.c +++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c @@ -209,12 +209,12 @@ vmxnet3_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); - strlcpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver)); + strscpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver)); - strlcpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT, + strscpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT, sizeof(drvinfo->version)); - strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + strscpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info)); } diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 5df7a0abc39d5f123bdb51891455551fc623b6c4..badf6f09ae51ceaa0abfef0a3551d0060c8b8d5d 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1541,8 +1541,8 @@ static const struct l3mdev_ops vrf_l3mdev_ops = { static void vrf_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); } static const struct ethtool_ops vrf_ethtool_ops = { diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index c3285242f74fbdb9efe03712fe680f24faefecb3..6ab669dcd1c6c2a17e3fe134b2c30633088f1fe8 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -713,12 +713,9 @@ static struct sk_buff *vxlan_gro_receive(struct sock *sk, off_vx = skb_gro_offset(skb); hlen = off_vx + sizeof(*vh); - vh = skb_gro_header_fast(skb, off_vx); - if (skb_gro_header_hard(skb, hlen)) { - vh = skb_gro_header_slow(skb, hlen, off_vx); - if (unlikely(!vh)) - goto out; - } + vh = skb_gro_header(skb, hlen, off_vx); + if (unlikely(!vh)) + goto out; skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr)); @@ -3313,8 +3310,8 @@ static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[], static void vxlan_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version)); - strlcpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver)); + strscpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version)); + strscpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver)); } static int vxlan_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/wireguard/netlink.c b/drivers/net/wireguard/netlink.c index d0f3b6d7f40894fdee515c471295d3516da9b13d..43c8c84e7ea82377c4d2a532380fcc5c4554fab2 100644 --- a/drivers/net/wireguard/netlink.c +++ b/drivers/net/wireguard/netlink.c @@ -436,14 +436,13 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs) if (attrs[WGPEER_A_ENDPOINT]) { struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); + struct endpoint endpoint = { { { 0 } } }; - if ((len == sizeof(struct sockaddr_in) && - addr->sa_family == AF_INET) || - (len == sizeof(struct sockaddr_in6) && - addr->sa_family == AF_INET6)) { - struct endpoint endpoint = { { { 0 } } }; - - memcpy(&endpoint.addr, addr, len); + if (len == sizeof(struct sockaddr_in) && addr->sa_family == AF_INET) { + endpoint.addr4 = *(struct sockaddr_in *)addr; + wg_socket_set_peer_endpoint(peer, &endpoint); + } else if (len == sizeof(struct sockaddr_in6) && addr->sa_family == AF_INET6) { + endpoint.addr6 = *(struct sockaddr_in6 *)addr; wg_socket_set_peer_endpoint(peer, &endpoint); } } @@ -621,6 +620,7 @@ static const struct genl_ops genl_ops[] = { static struct genl_family genl_family __ro_after_init = { .ops = genl_ops, .n_ops = ARRAY_SIZE(genl_ops), + .resv_start_op = WG_CMD_SET_DEVICE + 1, .name = WG_GENL_NAME, .version = WG_GENL_VERSION, .maxattr = WGDEVICE_A_MAX, diff --git a/drivers/net/wireguard/peer.c b/drivers/net/wireguard/peer.c index 1acd00ab2fbcbf0ac5faabae474bf6ec16dcf30c..1cb502a932e07c16ecf91121a294ccafc7005516 100644 --- a/drivers/net/wireguard/peer.c +++ b/drivers/net/wireguard/peer.c @@ -54,8 +54,7 @@ struct wg_peer *wg_peer_create(struct wg_device *wg, skb_queue_head_init(&peer->staged_packet_queue); wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); - netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll); napi_enable(&peer->napi); list_add_tail(&peer->peer_list, &wg->peer_list); INIT_LIST_HEAD(&peer->allowedips_list); diff --git a/drivers/net/wireguard/selftest/allowedips.c b/drivers/net/wireguard/selftest/allowedips.c index 41db10f9be4985619b21a0922159ad91f87a803b..19eac00b2381446763f306398399528026a8df23 100644 --- a/drivers/net/wireguard/selftest/allowedips.c +++ b/drivers/net/wireguard/selftest/allowedips.c @@ -284,7 +284,7 @@ static __init bool randomized_test(void) mutex_lock(&mutex); for (i = 0; i < NUM_RAND_ROUTES; ++i) { - prandom_bytes(ip, 4); + get_random_bytes(ip, 4); cidr = prandom_u32_max(32) + 1; peer = peers[prandom_u32_max(NUM_PEERS)]; if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, @@ -299,7 +299,7 @@ static __init bool randomized_test(void) } for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { memcpy(mutated, ip, 4); - prandom_bytes(mutate_mask, 4); + get_random_bytes(mutate_mask, 4); mutate_amount = prandom_u32_max(32); for (k = 0; k < mutate_amount / 8; ++k) mutate_mask[k] = 0xff; @@ -310,7 +310,7 @@ static __init bool randomized_test(void) for (k = 0; k < 4; ++k) mutated[k] = (mutated[k] & mutate_mask[k]) | (~mutate_mask[k] & - prandom_u32_max(256)); + get_random_u8()); cidr = prandom_u32_max(32) + 1; peer = peers[prandom_u32_max(NUM_PEERS)]; if (wg_allowedips_insert_v4(&t, @@ -328,7 +328,7 @@ static __init bool randomized_test(void) } for (i = 0; i < NUM_RAND_ROUTES; ++i) { - prandom_bytes(ip, 16); + get_random_bytes(ip, 16); cidr = prandom_u32_max(128) + 1; peer = peers[prandom_u32_max(NUM_PEERS)]; if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, @@ -343,7 +343,7 @@ static __init bool randomized_test(void) } for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { memcpy(mutated, ip, 16); - prandom_bytes(mutate_mask, 16); + get_random_bytes(mutate_mask, 16); mutate_amount = prandom_u32_max(128); for (k = 0; k < mutate_amount / 8; ++k) mutate_mask[k] = 0xff; @@ -354,7 +354,7 @@ static __init bool randomized_test(void) for (k = 0; k < 4; ++k) mutated[k] = (mutated[k] & mutate_mask[k]) | (~mutate_mask[k] & - prandom_u32_max(256)); + get_random_u8()); cidr = prandom_u32_max(128) + 1; peer = peers[prandom_u32_max(NUM_PEERS)]; if (wg_allowedips_insert_v6(&t, @@ -381,13 +381,13 @@ static __init bool randomized_test(void) for (j = 0;; ++j) { for (i = 0; i < NUM_QUERIES; ++i) { - prandom_bytes(ip, 4); + get_random_bytes(ip, 4); if (lookup(t.root4, 32, ip) != horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip); pr_err("allowedips random v4 self-test: FAIL\n"); goto free; } - prandom_bytes(ip, 16); + get_random_bytes(ip, 16); if (lookup(t.root6, 128, ip) != horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { pr_err("allowedips random v6 self-test: FAIL\n"); goto free; diff --git a/drivers/net/wireguard/selftest/ratelimiter.c b/drivers/net/wireguard/selftest/ratelimiter.c index ba87d294604fe436e9a672dce65a37139a7a32bf..d4bb40a695ab6785968babc1060a8b1010651e8b 100644 --- a/drivers/net/wireguard/selftest/ratelimiter.c +++ b/drivers/net/wireguard/selftest/ratelimiter.c @@ -6,29 +6,28 @@ #ifdef DEBUG #include -#include static const struct { bool result; - u64 nsec_to_sleep_before; + unsigned int msec_to_sleep_before; } expected_results[] __initconst = { [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, [PACKETS_BURSTABLE] = { false, 0 }, - [PACKETS_BURSTABLE + 1] = { true, NSEC_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, [PACKETS_BURSTABLE + 2] = { false, 0 }, - [PACKETS_BURSTABLE + 3] = { true, (NSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, + [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, [PACKETS_BURSTABLE + 4] = { true, 0 }, [PACKETS_BURSTABLE + 5] = { false, 0 } }; static __init unsigned int maximum_jiffies_at_index(int index) { - u64 total_nsecs = 2 * NSEC_PER_SEC / PACKETS_PER_SECOND / 3; + unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; int i; for (i = 0; i <= index; ++i) - total_nsecs += expected_results[i].nsec_to_sleep_before; - return nsecs_to_jiffies(total_nsecs); + total_msecs += expected_results[i].msec_to_sleep_before; + return msecs_to_jiffies(total_msecs); } static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, @@ -43,12 +42,8 @@ static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, loop_start_time = jiffies; for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { - if (expected_results[i].nsec_to_sleep_before) { - ktime_t timeout = ktime_add(ktime_add_ns(ktime_get_coarse_boottime(), TICK_NSEC * 4 / 3), - ns_to_ktime(expected_results[i].nsec_to_sleep_before)); - set_current_state(TASK_UNINTERRUPTIBLE); - schedule_hrtimeout_range_clock(&timeout, 0, HRTIMER_MODE_ABS, CLOCK_BOOTTIME); - } + if (expected_results[i].msec_to_sleep_before) + msleep(expected_results[i].msec_to_sleep_before); if (time_is_before_jiffies(loop_start_time + maximum_jiffies_at_index(i))) @@ -132,7 +127,7 @@ bool __init wg_ratelimiter_selftest(void) if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) return true; - BUILD_BUG_ON(NSEC_PER_SEC % PACKETS_PER_SECOND != 0); + BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); if (wg_ratelimiter_init()) goto out; @@ -172,7 +167,7 @@ bool __init wg_ratelimiter_selftest(void) ++test; #endif - for (trials = TRIALS_BEFORE_GIVING_UP;;) { + for (trials = TRIALS_BEFORE_GIVING_UP; IS_ENABLED(DEBUG_RATELIMITER_TIMINGS);) { int test_count = 0, ret; ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); diff --git a/drivers/net/wireless/ath/ath10k/bmi.c b/drivers/net/wireless/ath/ath10k/bmi.c index 4481ed375f55d95143222a461905e37c4111fbeb..af6546572df26b5e43fe819987d46dc5ffc032a7 100644 --- a/drivers/net/wireless/ath/ath10k/bmi.c +++ b/drivers/net/wireless/ath/ath10k/bmi.c @@ -101,7 +101,7 @@ int ath10k_bmi_get_target_info_sdio(struct ath10k *ar, cmd.id = __cpu_to_le32(BMI_GET_TARGET_INFO); /* Step 1: Read 4 bytes of the target info and check if it is - * the special sentinal version word or the first word in the + * the special sentinel version word or the first word in the * version response. */ resplen = sizeof(u32); @@ -111,7 +111,7 @@ int ath10k_bmi_get_target_info_sdio(struct ath10k *ar, return ret; } - /* Some SDIO boards have a special sentinal byte before the real + /* Some SDIO boards have a special sentinel byte before the real * version response. */ if (__le32_to_cpu(tmp) == TARGET_VERSION_SENTINAL) { diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c index c45c814fd1229a61368425a43b9c8933e34f7ae8..59926227bd49ef1388beaa937db9abd7f8e5a51b 100644 --- a/drivers/net/wireless/ath/ath10k/ce.c +++ b/drivers/net/wireless/ath/ath10k/ce.c @@ -1323,7 +1323,7 @@ EXPORT_SYMBOL(ath10k_ce_per_engine_service); /* * Handler for per-engine interrupts on ALL active CEs. * This is used in cases where the system is sharing a - * single interrput for all CEs + * single interrupt for all CEs */ void ath10k_ce_per_engine_service_any(struct ath10k *ar) diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 276954b70d6305e554699ebfa197e3c77c16c6d1..400f332a7ff015fe4aaaf213d615142e420cc5ad 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -98,6 +98,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = true, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA988X_HW_2_0_VERSION, @@ -136,6 +137,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = true, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9887_HW_1_0_VERSION, @@ -175,6 +177,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA6174_HW_3_2_VERSION, @@ -209,6 +212,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .supports_peer_stats_info = true, .dynamic_sar_support = true, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA6174_HW_2_1_VERSION, @@ -247,6 +251,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA6174_HW_2_1_VERSION, @@ -285,6 +290,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA6174_HW_3_0_VERSION, @@ -323,6 +329,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA6174_HW_3_2_VERSION, @@ -365,6 +372,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .supports_peer_stats_info = true, .dynamic_sar_support = true, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA99X0_HW_2_0_DEV_VERSION, @@ -409,6 +417,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9984_HW_1_0_DEV_VERSION, @@ -460,6 +469,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9888_HW_2_0_DEV_VERSION, @@ -508,6 +518,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9377_HW_1_0_DEV_VERSION, @@ -546,6 +557,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9377_HW_1_1_DEV_VERSION, @@ -586,6 +598,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA9377_HW_1_1_DEV_VERSION, @@ -617,6 +630,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .credit_size_workaround = true, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = QCA4019_HW_1_0_DEV_VERSION, @@ -662,6 +676,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = false, .hw_restart_disconnect = false, + .use_fw_tx_credits = true, }, { .id = WCN3990_HW_1_0_DEV_VERSION, @@ -693,6 +708,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = { .tx_stats_over_pktlog = false, .dynamic_sar_support = true, .hw_restart_disconnect = true, + .use_fw_tx_credits = false, }, }; @@ -3080,7 +3096,7 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode, * enabled always. * * We can still enable BTCOEX if firmware has the support - * eventhough btceox_support value is + * even though btceox_support value is * ATH10K_DT_BTCOEX_NOT_FOUND */ diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index d70d7d088a2bf488b65c50e0b5be5fd8d0b11f9f..f5de8ce8fb4561221f3aa0972284490c5f6640d0 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -76,7 +76,7 @@ /* The magic used by QCA spec */ #define ATH10K_SMBIOS_BDF_EXT_MAGIC "BDF_" -/* Default Airtime weight multipler (Tuned for multiclient performance) */ +/* Default Airtime weight multiplier (Tuned for multiclient performance) */ #define ATH10K_AIRTIME_WEIGHT_MULTIPLIER 4 #define ATH10K_MAX_RETRY_COUNT 30 @@ -857,7 +857,7 @@ enum ath10k_dev_flags { /* Disable HW crypto engine */ ATH10K_FLAG_HW_CRYPTO_DISABLED, - /* Bluetooth coexistance enabled */ + /* Bluetooth coexistence enabled */ ATH10K_FLAG_BTCOEX, /* Per Station statistics service */ diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c index fe6b6f97a91631005acad93b6dc195003e508e54..2d1634a890dde3e8d7487ba3db025d744f43ebc8 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.c +++ b/drivers/net/wireless/ath/ath10k/coredump.c @@ -531,7 +531,7 @@ static const struct ath10k_mem_section qca6174_hw30_sdio_register_sections[] = { {0x40000, 0x400A4}, - /* SI register is skiped here. + /* SI register is skipped here. * Because it will cause bus hang * * {0x50000, 0x50018}, diff --git a/drivers/net/wireless/ath/ath10k/coredump.h b/drivers/net/wireless/ath/ath10k/coredump.h index 240d70515088890fabcd2d8d5e4cb11d41a46e52..437b9759f05d3d8da26b6bb6e01e80fda9754bc7 100644 --- a/drivers/net/wireless/ath/ath10k/coredump.h +++ b/drivers/net/wireless/ath/ath10k/coredump.h @@ -125,7 +125,7 @@ enum ath10k_mem_region_type { * To minimize the size of the array, the list must obey the format: * '{start0,stop0},{start1,stop1},{start2,stop2}....' The values below must * also obey to 'start0 < stop0 < start1 < stop1 < start2 < ...', otherwise - * we may encouter error in the dump processing. + * we may encounter error in the dump processing. */ struct ath10k_mem_section { u32 start; diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c index 39378e3f9b2bbb9e90d84df669d20ce36e6a6387..c861e66ef6bc54b16d76b4d3dff15b9c372e8ee1 100644 --- a/drivers/net/wireless/ath/ath10k/debug.c +++ b/drivers/net/wireless/ath/ath10k/debug.c @@ -1081,7 +1081,7 @@ exit: * struct available.. */ -/* This generally cooresponds to the debugfs fw_stats file */ +/* This generally corresponds to the debugfs fw_stats file */ static const char ath10k_gstrings_stats[][ETH_GSTRING_LEN] = { "tx_pkts_nic", "tx_bytes_nic", diff --git a/drivers/net/wireless/ath/ath10k/debugfs_sta.c b/drivers/net/wireless/ath/ath10k/debugfs_sta.c index 367539f2c3700612cb69f72c94653d8c012c9477..87a3365330ff80c880663f384abfded6f2fc3171 100644 --- a/drivers/net/wireless/ath/ath10k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath10k/debugfs_sta.c @@ -498,7 +498,7 @@ static char *get_num_ampdu_subfrm_str(enum ath10k_ampdu_subfrm_num i) { switch (i) { case ATH10K_AMPDU_SUBFRM_NUM_10: - return "upto 10"; + return "up to 10"; case ATH10K_AMPDU_SUBFRM_NUM_20: return "11-20"; case ATH10K_AMPDU_SUBFRM_NUM_30: diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c index fab398046a3f231c38b20e60d771f3548ddaba57..6d1784f74bea497e7ac1c34a70504e6d4b023960 100644 --- a/drivers/net/wireless/ath/ath10k/htc.c +++ b/drivers/net/wireless/ath/ath10k/htc.c @@ -947,13 +947,18 @@ int ath10k_htc_wait_target(struct ath10k_htc *htc) return -ECOMM; } - htc->total_transmit_credits = __le16_to_cpu(msg->ready.credit_count); + if (ar->hw_params.use_fw_tx_credits) + htc->total_transmit_credits = __le16_to_cpu(msg->ready.credit_count); + else + htc->total_transmit_credits = 1; + htc->target_credit_size = __le16_to_cpu(msg->ready.credit_size); ath10k_dbg(ar, ATH10K_DBG_HTC, - "Target ready! transmit resources: %d size:%d\n", + "Target ready! transmit resources: %d size:%d actual credits:%d\n", htc->total_transmit_credits, - htc->target_credit_size); + htc->target_credit_size, + msg->ready.credit_count); if ((htc->total_transmit_credits == 0) || (htc->target_credit_size == 0)) { diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c index 8a075a711b713f9e279ef6b2784f062e8b8082e3..e76aab9733208ff70bb0ea19317ba3c093ef36d7 100644 --- a/drivers/net/wireless/ath/ath10k/htt_rx.c +++ b/drivers/net/wireless/ath/ath10k/htt_rx.c @@ -301,12 +301,16 @@ void ath10k_htt_rx_free(struct ath10k_htt *htt) ath10k_htt_get_vaddr_ring(htt), htt->rx_ring.base_paddr); + ath10k_htt_config_paddrs_ring(htt, NULL); + dma_free_coherent(htt->ar->dev, sizeof(*htt->rx_ring.alloc_idx.vaddr), htt->rx_ring.alloc_idx.vaddr, htt->rx_ring.alloc_idx.paddr); + htt->rx_ring.alloc_idx.vaddr = NULL; kfree(htt->rx_ring.netbufs_ring); + htt->rx_ring.netbufs_ring = NULL; } static inline struct sk_buff *ath10k_htt_rx_netbuf_pop(struct ath10k_htt *htt) @@ -846,8 +850,10 @@ err_dma_idx: ath10k_htt_get_rx_ring_size(htt), vaddr_ring, htt->rx_ring.base_paddr); + ath10k_htt_config_paddrs_ring(htt, NULL); err_dma_ring: kfree(htt->rx_ring.netbufs_ring); + htt->rx_ring.netbufs_ring = NULL; err_netbuf: return -ENOMEM; } @@ -2496,7 +2502,7 @@ static bool ath10k_htt_rx_proc_rx_ind_hl(struct ath10k_htt *htt, /* I have not yet seen any case where num_mpdu_ranges > 1. * qcacld does not seem handle that case either, so we introduce the - * same limitiation here as well. + * same limitation here as well. */ if (num_mpdu_ranges > 1) ath10k_warn(ar, diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c index a19b0795c86d52e0dfc38fae13bdaf2cb8d0fef7..bd603feb7953148acc79cc0876874165b35ef53f 100644 --- a/drivers/net/wireless/ath/ath10k/htt_tx.c +++ b/drivers/net/wireless/ath/ath10k/htt_tx.c @@ -1112,7 +1112,7 @@ int ath10k_htt_tx_fetch_resp(struct ath10k *ar, int len = 0; int ret; - /* Response IDs are echo-ed back only for host driver convienence + /* Response IDs are echo-ed back only for host driver convenience * purposes. They aren't used for anything in the driver yet so use 0. */ diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c index e52e41a70321045987040195ca84e70c6c8709eb..6d32b43a4da65e620df4cacddd41c8e5bcdb4805 100644 --- a/drivers/net/wireless/ath/ath10k/hw.c +++ b/drivers/net/wireless/ath/ath10k/hw.c @@ -84,7 +84,7 @@ const struct ath10k_hw_regs qca99x0_regs = { .ce5_base_address = 0x0004b400, .ce6_base_address = 0x0004b800, .ce7_base_address = 0x0004bc00, - /* Note: qca99x0 supports upto 12 Copy Engines. Other than address of + /* Note: qca99x0 supports up to 12 Copy Engines. Other than address of * CE0 and CE1 no other copy engine is directly referred in the code. * It is not really necessary to assign address for newly supported * CEs in this address table. @@ -120,7 +120,7 @@ const struct ath10k_hw_regs qca4019_regs = { .ce5_base_address = 0x0004b400, .ce6_base_address = 0x0004b800, .ce7_base_address = 0x0004bc00, - /* qca4019 supports upto 12 copy engines. Since base address + /* qca4019 supports up to 12 copy engines. Since base address * of ce8 to ce11 are not directly referred in the code, * no need have them in separate members in this table. * Copy Engine Address @@ -924,7 +924,7 @@ static void ath10k_hw_map_target_mem(struct ath10k *ar, u32 msb) ath10k_hif_write32(ar, address, msb); } -/* 1. Write to memory region of target, such as IRAM adn DRAM. +/* 1. Write to memory region of target, such as IRAM and DRAM. * 2. Target address( 0 ~ 00100000 & 0x00400000~0x00500000) * can be written directly. See ath10k_pci_targ_cpu_to_ce_addr() too. * 3. In order to access the region other than the above, diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h index 93acf0dd580a6f0f5e076fb6d6106887c39791ef..1b99f3a39a1134bcca75fe729c0fe5f1acdbf3ac 100644 --- a/drivers/net/wireless/ath/ath10k/hw.h +++ b/drivers/net/wireless/ath/ath10k/hw.h @@ -635,6 +635,8 @@ struct ath10k_hw_params { bool dynamic_sar_support; bool hw_restart_disconnect; + + bool use_fw_tx_credits; }; struct htt_resp; diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c index 9dd3b8fba4b0e5d078d5b20637c0a3ce54451e93..ec8d5b29bc72c8721c503a5b2eddabace35e4083 100644 --- a/drivers/net/wireless/ath/ath10k/mac.c +++ b/drivers/net/wireless/ath/ath10k/mac.c @@ -864,11 +864,36 @@ static int ath10k_peer_delete(struct ath10k *ar, u32 vdev_id, const u8 *addr) return 0; } +static void ath10k_peer_map_cleanup(struct ath10k *ar, struct ath10k_peer *peer) +{ + int peer_id, i; + + lockdep_assert_held(&ar->conf_mutex); + + for_each_set_bit(peer_id, peer->peer_ids, + ATH10K_MAX_NUM_PEER_IDS) { + ar->peer_map[peer_id] = NULL; + } + + /* Double check that peer is properly un-referenced from + * the peer_map + */ + for (i = 0; i < ARRAY_SIZE(ar->peer_map); i++) { + if (ar->peer_map[i] == peer) { + ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %pK idx %d)\n", + peer->addr, peer, i); + ar->peer_map[i] = NULL; + } + } + + list_del(&peer->list); + kfree(peer); + ar->num_peers--; +} + static void ath10k_peer_cleanup(struct ath10k *ar, u32 vdev_id) { struct ath10k_peer *peer, *tmp; - int peer_id; - int i; lockdep_assert_held(&ar->conf_mutex); @@ -880,25 +905,7 @@ static void ath10k_peer_cleanup(struct ath10k *ar, u32 vdev_id) ath10k_warn(ar, "removing stale peer %pM from vdev_id %d\n", peer->addr, vdev_id); - for_each_set_bit(peer_id, peer->peer_ids, - ATH10K_MAX_NUM_PEER_IDS) { - ar->peer_map[peer_id] = NULL; - } - - /* Double check that peer is properly un-referenced from - * the peer_map - */ - for (i = 0; i < ARRAY_SIZE(ar->peer_map); i++) { - if (ar->peer_map[i] == peer) { - ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %pK idx %d)\n", - peer->addr, peer, i); - ar->peer_map[i] = NULL; - } - } - - list_del(&peer->list); - kfree(peer); - ar->num_peers--; + ath10k_peer_map_cleanup(ar, peer); } spin_unlock_bh(&ar->data_lock); } @@ -4044,7 +4051,7 @@ static int ath10k_mac_tx(struct ath10k *ar, ath10k_tx_h_seq_no(vif, skb); break; case ATH10K_HW_TXRX_ETHERNET: - /* Convert 802.11->802.3 header only if the frame was erlier + /* Convert 802.11->802.3 header only if the frame was earlier * encapsulated to 802.11 by mac80211. Otherwise pass it as is. */ if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP)) @@ -7621,10 +7628,7 @@ static int ath10k_sta_state(struct ieee80211_hw *hw, /* Clean up the peer object as well since we * must have failed to do this above. */ - list_del(&peer->list); - ar->peer_map[i] = NULL; - kfree(peer); - ar->num_peers--; + ath10k_peer_map_cleanup(ar, peer); } } spin_unlock_bh(&ar->data_lock); @@ -8093,7 +8097,7 @@ static void ath10k_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, /* TODO: Implement this function properly * For now it is needed to reply to Probe Requests in IBSS mode. - * Propably we need this information from FW. + * Probably we need this information from FW. */ static int ath10k_tx_last_beacon(struct ieee80211_hw *hw) { @@ -8516,7 +8520,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw, "mac sta rc update for %pM changed %08x bw %d nss %d smps %d\n", sta->addr, changed, sta->deflink.bandwidth, sta->deflink.rx_nss, - sta->smps_mode); + sta->deflink.smps_mode); if (changed & IEEE80211_RC_BW_CHANGED) { bw = WMI_PEER_CHWIDTH_20MHZ; @@ -8550,7 +8554,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw, if (changed & IEEE80211_RC_SMPS_CHANGED) { smps = WMI_PEER_SMPS_PS_NONE; - switch (sta->smps_mode) { + switch (sta->deflink.smps_mode) { case IEEE80211_SMPS_AUTOMATIC: case IEEE80211_SMPS_OFF: smps = WMI_PEER_SMPS_PS_NONE; @@ -8563,7 +8567,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw, break; case IEEE80211_SMPS_NUM_MODES: ath10k_warn(ar, "Invalid smps %d in sta rc update for %pM\n", - sta->smps_mode, sta->addr); + sta->deflink.smps_mode, sta->addr); smps = WMI_PEER_SMPS_PS_NONE; break; } @@ -9682,7 +9686,7 @@ static const struct ieee80211_iface_limit ath10k_tlv_if_limit_ibss[] = { }, }; -/* FIXME: This is not thouroughly tested. These combinations may over- or +/* FIXME: This is not thoroughly tested. These combinations may over- or * underestimate hw/fw capabilities. */ static struct ieee80211_iface_combination ath10k_tlv_if_comb[] = { @@ -9922,7 +9926,7 @@ int ath10k_mac_register(struct ath10k *ar) WLAN_CIPHER_SUITE_BIP_GMAC_128, WLAN_CIPHER_SUITE_BIP_GMAC_256, - /* Only QCA99x0 and QCA4019 varients support GCMP-128, GCMP-256 + /* Only QCA99x0 and QCA4019 variants support GCMP-128, GCMP-256 * and CCMP-256 in hardware. */ WLAN_CIPHER_SUITE_GCMP, diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c index bf1c938be7d0a2889d8181c8219142ce0d5e0232..e56c6a6b137914bf4ea535087524f7a531a46204 100644 --- a/drivers/net/wireless/ath/ath10k/pci.c +++ b/drivers/net/wireless/ath/ath10k/pci.c @@ -1244,7 +1244,7 @@ static void ath10k_pci_process_htt_rx_cb(struct ath10k_ce_pipe *ce_state, unsigned int nbytes, max_nbytes, nentries; int orig_len; - /* No need to aquire ce_lock for CE5, since this is the only place CE5 + /* No need to acquire ce_lock for CE5, since this is the only place CE5 * is processed other than init and deinit. Before releasing CE5 * buffers, interrupts are disabled. Thus CE5 access is serialized. */ @@ -3215,8 +3215,7 @@ static void ath10k_pci_free_irq(struct ath10k *ar) void ath10k_pci_init_napi(struct ath10k *ar) { - netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_pci_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_pci_napi_poll); } static int ath10k_pci_init_irq(struct ath10k *ar) diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h index cf64898b9447af03fcdcbe9d9a791c05cc5715b9..480cd97ab739de0c0405ae01e1c62209dd5509ef 100644 --- a/drivers/net/wireless/ath/ath10k/pci.h +++ b/drivers/net/wireless/ath/ath10k/pci.h @@ -81,7 +81,7 @@ struct ath10k_pci_pipe { /* Handle of underlying Copy Engine */ struct ath10k_ce_pipe *ce_hdl; - /* Our pipe number; facilitiates use of pipe_info ptrs. */ + /* Our pipe number; facilitates use of pipe_info ptrs. */ u8 pipe_num; /* Convenience back pointer to hif_ce_state. */ diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c index d7e406916bc881cfc82eaafb54529320ce650a3d..66cb7a1e628a48322a318d54e55dda130d8186a2 100644 --- a/drivers/net/wireless/ath/ath10k/qmi.c +++ b/drivers/net/wireless/ath/ath10k/qmi.c @@ -792,7 +792,7 @@ static void ath10k_qmi_event_server_arrive(struct ath10k_qmi *qmi) return; /* - * HACK: sleep for a while inbetween receiving the msa info response + * HACK: sleep for a while between receiving the msa info response * and the XPU update to prevent SDM845 from crashing due to a security * violation, when running MPSS.AT.4.0.c2-01184-SDM845_GEN_PACK-1. */ diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h index 6ce2a8b1060d936fca2b1abb7a472bb44dd5ec8d..777e53aa69dc86bdf4b1b9e61956bb0b21fbeb0d 100644 --- a/drivers/net/wireless/ath/ath10k/rx_desc.h +++ b/drivers/net/wireless/ath/ath10k/rx_desc.h @@ -448,7 +448,7 @@ struct rx_mpdu_end { * - 4 bytes for WEP * - 8 bytes for TKIP, AES * [padding to 4 bytes] - * c) A-MSDU subframe header (14 bytes) if appliable + * c) A-MSDU subframe header (14 bytes) if applicable * d) LLC/SNAP (RFC1042, 8 bytes) * * In case of A-MSDU only first frame in sequence contains (a) and (b). diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c index 24283c02a5ef6abc21b45b1313727bafe78cf0ab..79e09c7a82b3d61f9a20ab177e4345245eb22584 100644 --- a/drivers/net/wireless/ath/ath10k/sdio.c +++ b/drivers/net/wireless/ath/ath10k/sdio.c @@ -1057,7 +1057,7 @@ static int ath10k_sdio_mbox_proc_pending_irqs(struct ath10k *ar, out: /* An optimization to bypass reading the IRQ status registers - * unecessarily which can re-wake the target, if upper layers + * unnecessarily which can re-wake the target, if upper layers * determine that we are in a low-throughput mode, we can rely on * taking another interrupt rather than re-checking the status * registers which can re-wake the target. @@ -2531,8 +2531,7 @@ static int ath10k_sdio_probe(struct sdio_func *func, return -ENOMEM; } - netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_sdio_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_sdio_napi_poll); ath10k_dbg(ar, ATH10K_DBG_BOOT, "sdio new func %d vendor 0x%x device 0x%x block 0x%x/0x%x\n", diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c index 5576ad9fd11610c72a9edc7309c63c6dc9e1477a..cfcb759a87deac499b6b15b208aeb16cfa934271 100644 --- a/drivers/net/wireless/ath/ath10k/snoc.c +++ b/drivers/net/wireless/ath/ath10k/snoc.c @@ -1242,8 +1242,7 @@ static int ath10k_snoc_napi_poll(struct napi_struct *ctx, int budget) static void ath10k_snoc_init_napi(struct ath10k *ar) { - netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_snoc_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_snoc_napi_poll); } static int ath10k_snoc_request_irq(struct ath10k *ar) diff --git a/drivers/net/wireless/ath/ath10k/thermal.c b/drivers/net/wireless/ath/ath10k/thermal.c index 36c9a1364253f806df8eeed03bdb48c2992d3a83..cefd97323dfe5071a325a8d17830e1559d462564 100644 --- a/drivers/net/wireless/ath/ath10k/thermal.c +++ b/drivers/net/wireless/ath/ath10k/thermal.c @@ -98,7 +98,7 @@ static ssize_t ath10k_thermal_show_temp(struct device *dev, temperature = ar->thermal.temperature; spin_unlock_bh(&ar->data_lock); - /* display in millidegree celcius */ + /* display in millidegree celsius */ ret = snprintf(buf, PAGE_SIZE, "%d\n", temperature * 1000); out: mutex_unlock(&ar->conf_mutex); diff --git a/drivers/net/wireless/ath/ath10k/thermal.h b/drivers/net/wireless/ath/ath10k/thermal.h index 5fdb020f4da38780c9105a06714547718523b880..1f4de9fbf2b30be22e0b657062115b9c463a8326 100644 --- a/drivers/net/wireless/ath/ath10k/thermal.h +++ b/drivers/net/wireless/ath/ath10k/thermal.h @@ -19,7 +19,7 @@ struct ath10k_thermal { /* protected by conf_mutex */ u32 throttle_state; u32 quiet_period; - /* temperature value in Celcius degree + /* temperature value in Celsius degree * protected by data_lock */ int temperature; diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c index ad6471b217964b2a9a5c1da457deda059a95f4c0..b0067af685b1683d71820151d84093360456bc58 100644 --- a/drivers/net/wireless/ath/ath10k/usb.c +++ b/drivers/net/wireless/ath/ath10k/usb.c @@ -1014,8 +1014,7 @@ static int ath10k_usb_probe(struct usb_interface *interface, return -ENOMEM; } - netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_usb_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_usb_napi_poll); usb_get_dev(dev); vendor_id = le16_to_cpu(dev->descriptor.idVendor); diff --git a/drivers/net/wireless/ath/ath10k/usb.h b/drivers/net/wireless/ath/ath10k/usb.h index 34d683e8fc1836bf1bc0db137c71aab8da27b0ed..48e066ba816249a746a1170d993049ea5bb705ec 100644 --- a/drivers/net/wireless/ath/ath10k/usb.h +++ b/drivers/net/wireless/ath/ath10k/usb.h @@ -26,7 +26,7 @@ #define ATH10K_USB_EP_ADDR_APP_DATA_MP_OUT 0x03 #define ATH10K_USB_EP_ADDR_APP_DATA_HP_OUT 0x04 -/* diagnostic command defnitions */ +/* diagnostic command definitions */ #define ATH10K_USB_CONTROL_REQ_SEND_BMI_CMD 1 #define ATH10K_USB_CONTROL_REQ_RECV_BMI_RESP 2 #define ATH10K_USB_CONTROL_REQ_DIAG_CMD 3 diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h index b39c9b78b32bdf49ba6d80cecee921e277a59b2c..dbb48d70f2e93e95fa6a51ceea65c156b82a9493 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h @@ -1813,7 +1813,7 @@ struct wmi_tlv_pdev_get_temp_cmd { struct wmi_tlv_pdev_temperature_event { __le32 tlv_hdr; - /* temperature value in Celcius degree */ + /* temperature value in Celsius degree */ __le32 temperature; __le32 pdev_id; } __packed; @@ -2548,7 +2548,7 @@ struct nlo_channel_prediction_cfg { /* Preconfigured stationary threshold. * Lesser value means more conservative. Bigger value means more aggressive. - * Maximum is 100 and mininum is 0. + * Maximum is 100 and minimum is 0. */ __le32 stationary_threshold; diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c index 074d8ba5072adfb4683db212055947783f6fde35..980d4124fa287d5072fc2353bff7edb2c342af73 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.c +++ b/drivers/net/wireless/ath/ath10k/wmi.c @@ -3555,7 +3555,7 @@ static void ath10k_wmi_update_tim(struct ath10k *ar, __le32 t; u32 v, tim_len; - /* When FW reports 0 in tim_len, ensure atleast first byte + /* When FW reports 0 in tim_len, ensure at least first byte * in tim_bitmap is considered for pvm calculation. */ tim_len = tim_info->tim_len ? __le32_to_cpu(tim_info->tim_len) : 1; diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h index 4abd12e78028a3052ee52a4626aab67b6afa4098..6de3cc4640a0b6825179cadaa8b3e757b5942778 100644 --- a/drivers/net/wireless/ath/ath10k/wmi.h +++ b/drivers/net/wireless/ath/ath10k/wmi.h @@ -3170,7 +3170,7 @@ struct wmi_start_scan_common { /* dwell time in msec on passive channels */ __le32 dwell_time_passive; /* - * min time in msec on the BSS channel,only valid if atleast one + * min time in msec on the BSS channel,only valid if at least one * VDEV is active */ __le32 min_rest_time; @@ -3196,7 +3196,7 @@ struct wmi_start_scan_common { * and bssid_list */ __le32 repeat_probe_time; - /* time in msec between 2 consequetive probe requests with in a set. */ + /* time in msec between 2 consecutive probe requests with in a set. */ __le32 probe_spacing_time; /* * data inactivity time in msec on bss channel that will be used by @@ -4397,7 +4397,7 @@ struct wmi_pdev_stats_tx { /* wal pdev continuous xretry */ __le32 pdev_cont_xretry; - /* wal pdev continous xretry */ + /* wal pdev continuous xretry */ __le32 pdev_tx_timeout; /* wal pdev resets */ @@ -5240,7 +5240,7 @@ enum wmi_vdev_param { * scheduler. */ WMI_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, - /* enable/dsiable WDS for this VDEV */ + /* enable/disable WDS for this VDEV */ WMI_VDEV_PARAM_WDS, /* ATIM Window */ WMI_VDEV_PARAM_ATIM_WINDOW, @@ -5372,7 +5372,7 @@ enum wmi_10x_vdev_param { * scheduler. */ WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, - /* enable/dsiable WDS for this VDEV */ + /* enable/disable WDS for this VDEV */ WMI_10X_VDEV_PARAM_WDS, /* ATIM Window */ WMI_10X_VDEV_PARAM_ATIM_WINDOW, @@ -5904,7 +5904,7 @@ enum wmi_sta_ps_param_tx_wake_threshold { enum wmi_sta_ps_param_pspoll_count { WMI_STA_PS_PSPOLL_COUNT_NO_MAX = 0, /* - * Values greater than 0 indicate the maximum numer of PS-Poll frames + * Values greater than 0 indicate the maximum number of PS-Poll frames * FW will send before waking up. */ @@ -6947,7 +6947,7 @@ struct wmi_echo_ev_arg { }; struct wmi_pdev_temperature_event { - /* temperature value in Celcius degree */ + /* temperature value in Celsius degree */ __le32 temperature; } __packed; diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c index c4741471013822f5f870f1ed407f1c74b9117ab1..d34a4d6325b2b4700dcba866e9d7ded1e8d13d7a 100644 --- a/drivers/net/wireless/ath/ath11k/ahb.c +++ b/drivers/net/wireless/ath/ath11k/ahb.c @@ -16,6 +16,8 @@ #include "hif.h" #include #include "pcic.h" +#include +#include static const struct of_device_id ath11k_ahb_of_match[] = { /* TODO: Should we change the compatible string to something similar @@ -359,6 +361,7 @@ static void ath11k_ahb_ext_irq_enable(struct ath11k_base *ab) struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; if (!irq_grp->napi_enabled) { + dev_set_threaded(&irq_grp->napi_ndev, true); napi_enable(&irq_grp->napi); irq_grp->napi_enabled = true; } @@ -406,7 +409,8 @@ static int ath11k_ahb_fwreset_from_cold_boot(struct ath11k_base *ab) int timeout; if (ath11k_cold_boot_cal == 0 || ab->qmi.cal_done || - ab->hw_params.cold_boot_calib == 0) + ab->hw_params.cold_boot_calib == 0 || + ab->hw_params.cbcal_restart_fw == 0) return 0; ath11k_dbg(ab, ATH11K_DBG_AHB, "wait for cold boot done\n"); @@ -541,7 +545,7 @@ static int ath11k_ahb_config_ext_irq(struct ath11k_base *ab) irq_grp->grp_id = i; init_dummy_netdev(&irq_grp->napi_ndev); netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi, - ath11k_ahb_ext_grp_napi_poll, NAPI_POLL_WEIGHT); + ath11k_ahb_ext_grp_napi_poll); for (j = 0; j < ATH11K_EXT_IRQ_NUM_MAX; j++) { if (ab->hw_params.ring_mask->tx[i] & BIT(j)) { @@ -685,11 +689,90 @@ static int ath11k_ahb_map_service_to_pipe(struct ath11k_base *ab, u16 service_id return 0; } +static int ath11k_ahb_hif_suspend(struct ath11k_base *ab) +{ + struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab); + u32 wake_irq; + u32 value = 0; + int ret; + + if (!device_may_wakeup(ab->dev)) + return -EPERM; + + wake_irq = ab->irq_num[ATH11K_PCI_IRQ_CE0_OFFSET + ATH11K_PCI_CE_WAKE_IRQ]; + + ret = enable_irq_wake(wake_irq); + if (ret) { + ath11k_err(ab, "failed to enable wakeup irq :%d\n", ret); + return ret; + } + + value = u32_encode_bits(ab_ahb->smp2p_info.seq_no++, + ATH11K_AHB_SMP2P_SMEM_SEQ_NO); + value |= u32_encode_bits(ATH11K_AHB_POWER_SAVE_ENTER, + ATH11K_AHB_SMP2P_SMEM_MSG); + + ret = qcom_smem_state_update_bits(ab_ahb->smp2p_info.smem_state, + ATH11K_AHB_SMP2P_SMEM_VALUE_MASK, value); + if (ret) { + ath11k_err(ab, "failed to send smp2p power save enter cmd :%d\n", ret); + return ret; + } + + ath11k_dbg(ab, ATH11K_DBG_AHB, "ahb device suspended\n"); + + return ret; +} + +static int ath11k_ahb_hif_resume(struct ath11k_base *ab) +{ + struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab); + u32 wake_irq; + u32 value = 0; + int ret; + + if (!device_may_wakeup(ab->dev)) + return -EPERM; + + wake_irq = ab->irq_num[ATH11K_PCI_IRQ_CE0_OFFSET + ATH11K_PCI_CE_WAKE_IRQ]; + + ret = disable_irq_wake(wake_irq); + if (ret) { + ath11k_err(ab, "failed to disable wakeup irq: %d\n", ret); + return ret; + } + + reinit_completion(&ab->wow.wakeup_completed); + + value = u32_encode_bits(ab_ahb->smp2p_info.seq_no++, + ATH11K_AHB_SMP2P_SMEM_SEQ_NO); + value |= u32_encode_bits(ATH11K_AHB_POWER_SAVE_EXIT, + ATH11K_AHB_SMP2P_SMEM_MSG); + + ret = qcom_smem_state_update_bits(ab_ahb->smp2p_info.smem_state, + ATH11K_AHB_SMP2P_SMEM_VALUE_MASK, value); + if (ret) { + ath11k_err(ab, "failed to send smp2p power save enter cmd :%d\n", ret); + return ret; + } + + ret = wait_for_completion_timeout(&ab->wow.wakeup_completed, 3 * HZ); + if (ret == 0) { + ath11k_warn(ab, "timed out while waiting for wow wakeup completion\n"); + return -ETIMEDOUT; + } + + ath11k_dbg(ab, ATH11K_DBG_AHB, "ahb device resumed\n"); + + return 0; +} + static const struct ath11k_hif_ops ath11k_ahb_hif_ops_ipq8074 = { .start = ath11k_ahb_start, .stop = ath11k_ahb_stop, .read32 = ath11k_ahb_read32, .write32 = ath11k_ahb_write32, + .read = NULL, .irq_enable = ath11k_ahb_ext_irq_enable, .irq_disable = ath11k_ahb_ext_irq_disable, .map_service_to_pipe = ath11k_ahb_map_service_to_pipe, @@ -702,6 +785,7 @@ static const struct ath11k_hif_ops ath11k_ahb_hif_ops_wcn6750 = { .stop = ath11k_pcic_stop, .read32 = ath11k_pcic_read32, .write32 = ath11k_pcic_write32, + .read = NULL, .irq_enable = ath11k_pcic_ext_irq_enable, .irq_disable = ath11k_pcic_ext_irq_disable, .get_msi_address = ath11k_pcic_get_msi_address, @@ -709,6 +793,10 @@ static const struct ath11k_hif_ops ath11k_ahb_hif_ops_wcn6750 = { .map_service_to_pipe = ath11k_pcic_map_service_to_pipe, .power_down = ath11k_ahb_power_down, .power_up = ath11k_ahb_power_up, + .suspend = ath11k_ahb_hif_suspend, + .resume = ath11k_ahb_hif_resume, + .ce_irq_enable = ath11k_pci_enable_ce_irqs_except_wake_irq, + .ce_irq_disable = ath11k_pci_disable_ce_irqs_except_wake_irq, }; static int ath11k_core_get_rproc(struct ath11k_base *ab) @@ -783,6 +871,34 @@ static int ath11k_ahb_setup_msi_resources(struct ath11k_base *ab) return 0; } +static int ath11k_ahb_setup_smp2p_handle(struct ath11k_base *ab) +{ + struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab); + + if (!ab->hw_params.smp2p_wow_exit) + return 0; + + ab_ahb->smp2p_info.smem_state = qcom_smem_state_get(ab->dev, "wlan-smp2p-out", + &ab_ahb->smp2p_info.smem_bit); + if (IS_ERR(ab_ahb->smp2p_info.smem_state)) { + ath11k_err(ab, "failed to fetch smem state: %ld\n", + PTR_ERR(ab_ahb->smp2p_info.smem_state)); + return PTR_ERR(ab_ahb->smp2p_info.smem_state); + } + + return 0; +} + +static void ath11k_ahb_release_smp2p_handle(struct ath11k_base *ab) +{ + struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab); + + if (!ab->hw_params.smp2p_wow_exit) + return; + + qcom_smem_state_put(ab_ahb->smp2p_info.smem_state); +} + static int ath11k_ahb_setup_resources(struct ath11k_base *ab) { struct platform_device *pdev = ab->pdev; @@ -1038,10 +1154,14 @@ static int ath11k_ahb_probe(struct platform_device *pdev) if (ret) goto err_core_free; - ret = ath11k_hal_srng_init(ab); + ret = ath11k_ahb_setup_smp2p_handle(ab); if (ret) goto err_fw_deinit; + ret = ath11k_hal_srng_init(ab); + if (ret) + goto err_release_smp2p_handle; + ret = ath11k_ce_alloc_pipes(ab); if (ret) { ath11k_err(ab, "failed to allocate ce pipes: %d\n", ret); @@ -1078,6 +1198,9 @@ err_ce_free: err_hal_srng_deinit: ath11k_hal_srng_deinit(ab); +err_release_smp2p_handle: + ath11k_ahb_release_smp2p_handle(ab); + err_fw_deinit: ath11k_ahb_fw_resource_deinit(ab); @@ -1088,20 +1211,10 @@ err_core_free: return ret; } -static int ath11k_ahb_remove(struct platform_device *pdev) +static void ath11k_ahb_remove_prepare(struct ath11k_base *ab) { - struct ath11k_base *ab = platform_get_drvdata(pdev); unsigned long left; - if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) { - ath11k_ahb_power_down(ab); - ath11k_debugfs_soc_destroy(ab); - ath11k_qmi_deinit_service(ab); - goto qmi_fail; - } - - reinit_completion(&ab->driver_recovery); - if (test_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags)) { left = wait_for_completion_timeout(&ab->driver_recovery, ATH11K_AHB_RECOVERY_TIMEOUT); @@ -1111,19 +1224,61 @@ static int ath11k_ahb_remove(struct platform_device *pdev) set_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags); cancel_work_sync(&ab->restart_work); + cancel_work_sync(&ab->qmi.event_work); +} + +static void ath11k_ahb_free_resources(struct ath11k_base *ab) +{ + struct platform_device *pdev = ab->pdev; - ath11k_core_deinit(ab); -qmi_fail: ath11k_ahb_free_irq(ab); ath11k_hal_srng_deinit(ab); + ath11k_ahb_release_smp2p_handle(ab); ath11k_ahb_fw_resource_deinit(ab); ath11k_ce_free_pipes(ab); ath11k_core_free(ab); platform_set_drvdata(pdev, NULL); +} + +static int ath11k_ahb_remove(struct platform_device *pdev) +{ + struct ath11k_base *ab = platform_get_drvdata(pdev); + + if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) { + ath11k_ahb_power_down(ab); + ath11k_debugfs_soc_destroy(ab); + ath11k_qmi_deinit_service(ab); + goto qmi_fail; + } + + ath11k_ahb_remove_prepare(ab); + ath11k_core_deinit(ab); + +qmi_fail: + ath11k_ahb_free_resources(ab); return 0; } +static void ath11k_ahb_shutdown(struct platform_device *pdev) +{ + struct ath11k_base *ab = platform_get_drvdata(pdev); + + /* platform shutdown() & remove() are mutually exclusive. + * remove() is invoked during rmmod & shutdown() during + * system reboot/shutdown. + */ + ath11k_ahb_remove_prepare(ab); + + if (!(test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags))) + goto free_resources; + + ath11k_core_deinit(ab); + +free_resources: + ath11k_ahb_free_resources(ab); +} + static struct platform_driver ath11k_ahb_driver = { .driver = { .name = "ath11k", @@ -1131,6 +1286,7 @@ static struct platform_driver ath11k_ahb_driver = { }, .probe = ath11k_ahb_probe, .remove = ath11k_ahb_remove, + .shutdown = ath11k_ahb_shutdown, }; static int ath11k_ahb_init(void) diff --git a/drivers/net/wireless/ath/ath11k/ahb.h b/drivers/net/wireless/ath/ath11k/ahb.h index 58a945411c5b3cc6ff555f4db35bf43b890ec8c4..415ddfd2665497bb80e772b7ffce1c71f1fd8763 100644 --- a/drivers/net/wireless/ath/ath11k/ahb.h +++ b/drivers/net/wireless/ath/ath11k/ahb.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_AHB_H #define ATH11K_AHB_H @@ -8,6 +9,16 @@ #include "core.h" #define ATH11K_AHB_RECOVERY_TIMEOUT (3 * HZ) + +#define ATH11K_AHB_SMP2P_SMEM_MSG GENMASK(15, 0) +#define ATH11K_AHB_SMP2P_SMEM_SEQ_NO GENMASK(31, 16) +#define ATH11K_AHB_SMP2P_SMEM_VALUE_MASK 0xFFFFFFFF + +enum ath11k_ahb_smp2p_msg_id { + ATH11K_AHB_POWER_SAVE_ENTER = 1, + ATH11K_AHB_POWER_SAVE_EXIT, +}; + struct ath11k_base; struct ath11k_ahb { @@ -21,6 +32,11 @@ struct ath11k_ahb { u32 ce_size; bool use_tz; } fw; + struct { + unsigned short seq_no; + unsigned int smem_bit; + struct qcom_smem_state *smem_state; + } smp2p_info; }; static inline struct ath11k_ahb *ath11k_ahb_priv(struct ath11k_base *ab) diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c index c14c51f3870944f2dbc806edcf35c8e4f2ecea79..f2da95fd4253a2575e165d111308e613879fe0d4 100644 --- a/drivers/net/wireless/ath/ath11k/ce.c +++ b/drivers/net/wireless/ath/ath11k/ce.c @@ -250,7 +250,7 @@ const struct ce_attr ath11k_host_ce_config_qcn9074[] = { static bool ath11k_ce_need_shadow_fix(int ce_id) { - /* only ce4 needs shadow workaroud*/ + /* only ce4 needs shadow workaround */ if (ce_id == 4) return true; return false; @@ -1042,7 +1042,7 @@ int ath11k_ce_alloc_pipes(struct ath11k_base *ab) ret = ath11k_ce_alloc_pipe(ab, i); if (ret) { - /* Free any parial successful allocation */ + /* Free any partial successful allocation */ ath11k_ce_free_pipes(ab); return ret; } diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index c3e9e4f7bc24e75830a29015d55de6ba1ad994ca..b99180bc81723b150476942667080bad6d80555b 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -70,6 +70,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 16, .max_fft_bins = 512, + .fragment_160mhz = true, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -81,6 +82,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = false, .supports_sta_ps = false, .cold_boot_calib = true, + .cbcal_restart_fw = true, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -106,6 +108,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = false, + .supports_multi_bssid = false, + + .sram_dump = {}, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .hw_rev = ATH11K_HW_IPQ6018_HW10, @@ -141,6 +150,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 16, .max_fft_bins = 512, + .fragment_160mhz = true, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -152,6 +162,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = false, .supports_sta_ps = false, .cold_boot_calib = true, + .cbcal_restart_fw = true, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -177,6 +188,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = false, + .supports_multi_bssid = false, + + .sram_dump = {}, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .name = "qca6390 hw2.0", @@ -212,6 +230,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 0, .max_fft_bins = 0, + .fragment_160mhz = false, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -222,6 +241,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = true, .supports_sta_ps = true, .cold_boot_calib = false, + .cbcal_restart_fw = false, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -247,6 +267,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = true, + .supports_multi_bssid = true, + + .sram_dump = { + .start = 0x01400000, + .end = 0x0171ffff, + }, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .name = "qcn9074 hw1.0", @@ -281,6 +311,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 16, .fft_hdr_len = 24, .max_fft_bins = 1024, + .fragment_160mhz = false, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -292,6 +323,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = false, .supports_sta_ps = false, .cold_boot_calib = false, + .cbcal_restart_fw = false, .fw_mem_mode = 2, .num_vdevs = 8, .num_peers = 128, @@ -317,6 +349,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = false, + .supports_multi_bssid = false, + + .sram_dump = {}, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .name = "wcn6855 hw2.0", @@ -352,6 +391,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 0, .max_fft_bins = 0, + .fragment_160mhz = false, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -362,6 +402,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = true, .supports_sta_ps = true, .cold_boot_calib = false, + .cbcal_restart_fw = false, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -387,6 +428,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = true, + .supports_multi_bssid = true, + + .sram_dump = { + .start = 0x01400000, + .end = 0x0177ffff, + }, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .name = "wcn6855 hw2.1", @@ -422,6 +473,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 0, .max_fft_bins = 0, + .fragment_160mhz = false, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -431,6 +483,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .idle_ps = true, .supports_sta_ps = true, .cold_boot_calib = false, + .cbcal_restart_fw = false, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -456,6 +509,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .hybrid_bus_type = false, .fixed_fw_mem = false, .support_off_channel_tx = true, + .supports_multi_bssid = true, + + .sram_dump = { + .start = 0x01400000, + .end = 0x0177ffff, + }, + + .tcl_ring_retry = true, + .tx_ring_size = DP_TCL_DATA_RING_SIZE, + .smp2p_wow_exit = false, }, { .name = "wcn6750 hw1.0", @@ -468,7 +531,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .max_radios = 1, .bdf_addr = 0x4B0C0000, .hw_ops = &wcn6750_ops, - .ring_mask = &ath11k_hw_ring_mask_qca6390, + .ring_mask = &ath11k_hw_ring_mask_wcn6750, .internal_sleep_clock = false, .regs = &wcn6750_regs, .qmi_service_ins_id = ATH11K_QMI_WLFW_SERVICE_INS_ID_V01_WCN6750, @@ -491,6 +554,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .summary_pad_sz = 0, .fft_hdr_len = 0, .max_fft_bins = 0, + .fragment_160mhz = false, }, .interface_modes = BIT(NL80211_IFTYPE_STATION) | @@ -499,7 +563,8 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_shadow_regs = true, .idle_ps = true, .supports_sta_ps = true, - .cold_boot_calib = false, + .cold_boot_calib = true, + .cbcal_restart_fw = false, .fw_mem_mode = 0, .num_vdevs = 16 + 1, .num_peers = 512, @@ -508,8 +573,8 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_regdb = true, .fix_l1ss = false, .credit_flow = true, - .max_tx_ring = DP_TCL_NUM_RING_MAX_QCA6390, - .hal_params = &ath11k_hw_hal_params_qca6390, + .max_tx_ring = DP_TCL_NUM_RING_MAX, + .hal_params = &ath11k_hw_hal_params_wcn6750, .supports_dynamic_smps_6ghz = false, .alloc_cacheable_memory = false, .supports_rssi_stats = true, @@ -524,7 +589,14 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .static_window_map = true, .hybrid_bus_type = true, .fixed_fw_mem = true, - .support_off_channel_tx = false, + .support_off_channel_tx = true, + .supports_multi_bssid = true, + + .sram_dump = {}, + + .tcl_ring_retry = false, + .tx_ring_size = DP_TCL_DATA_RING_SIZE_WCN6750, + .smp2p_wow_exit = true, }, }; @@ -535,6 +607,52 @@ static inline struct ath11k_pdev *ath11k_core_get_single_pdev(struct ath11k_base return &ab->pdevs[0]; } +void ath11k_fw_stats_pdevs_free(struct list_head *head) +{ + struct ath11k_fw_stats_pdev *i, *tmp; + + list_for_each_entry_safe(i, tmp, head, list) { + list_del(&i->list); + kfree(i); + } +} + +void ath11k_fw_stats_vdevs_free(struct list_head *head) +{ + struct ath11k_fw_stats_vdev *i, *tmp; + + list_for_each_entry_safe(i, tmp, head, list) { + list_del(&i->list); + kfree(i); + } +} + +void ath11k_fw_stats_bcn_free(struct list_head *head) +{ + struct ath11k_fw_stats_bcn *i, *tmp; + + list_for_each_entry_safe(i, tmp, head, list) { + list_del(&i->list); + kfree(i); + } +} + +void ath11k_fw_stats_init(struct ath11k *ar) +{ + INIT_LIST_HEAD(&ar->fw_stats.pdevs); + INIT_LIST_HEAD(&ar->fw_stats.vdevs); + INIT_LIST_HEAD(&ar->fw_stats.bcn); + + init_completion(&ar->fw_stats_complete); +} + +void ath11k_fw_stats_free(struct ath11k_fw_stats *stats) +{ + ath11k_fw_stats_pdevs_free(&stats->pdevs); + ath11k_fw_stats_vdevs_free(&stats->vdevs); + ath11k_fw_stats_bcn_free(&stats->bcn); +} + int ath11k_core_suspend(struct ath11k_base *ab) { int ret; @@ -1544,7 +1662,7 @@ static void ath11k_core_pre_reconfigure_recovery(struct ath11k_base *ab) ar->state_11d = ATH11K_11D_IDLE; complete(&ar->completed_11d_scan); complete(&ar->scan.started); - complete(&ar->scan.completed); + complete_all(&ar->scan.completed); complete(&ar->scan.on_channel); complete(&ar->peer_assoc_done); complete(&ar->peer_delete_done); @@ -1563,6 +1681,8 @@ static void ath11k_core_pre_reconfigure_recovery(struct ath11k_base *ab) wake_up(&ab->wmi_ab.tx_credits_wq); wake_up(&ab->peer_mapping_wq); + + reinit_completion(&ab->driver_recovery); } static void ath11k_core_post_reconfigure_recovery(struct ath11k_base *ab) diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index afad8f55e43316af45e16408eae04ffb16b6f948..cf2f52cc4e30de528391dc9c4a9678e01ca810fa 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -498,6 +498,13 @@ struct ath11k_sta { bool use_4addr_set; u16 tcl_metadata; + + /* Protected with ar->data_lock */ + enum ath11k_wmi_peer_ps_state peer_ps_state; + u64 ps_start_time; + u64 ps_start_jiffies; + u64 ps_total_duration; + bool peer_current_ps_valid; }; #define ATH11K_MIN_5G_FREQ 4150 @@ -545,9 +552,6 @@ struct ath11k_debug { struct dentry *debugfs_pdev; struct ath11k_dbg_htt_stats htt_stats; u32 extd_tx_stats; - struct ath11k_fw_stats fw_stats; - struct completion fw_stats_complete; - bool fw_stats_done; u32 extd_rx_stats; u32 pktlog_filter; u32 pktlog_mode; @@ -710,6 +714,13 @@ struct ath11k { u8 twt_enabled; bool nlo_enabled; u8 alpha2[REG_ALPHA2_LEN + 1]; + struct ath11k_fw_stats fw_stats; + struct completion fw_stats_complete; + bool fw_stats_done; + + /* protected by conf_mutex */ + bool ps_state_enable; + bool ps_timekeeper_enable; }; struct ath11k_band_cap { @@ -887,7 +898,7 @@ struct ath11k_base { /* Below regd's are protected by ab->data_lock */ /* This is the regd set for every radio - * by the firmware during initializatin + * by the firmware during initialization */ struct ieee80211_regdomain *default_regd[MAX_RADIOS]; /* This regd is set during dynamic country setting @@ -1112,6 +1123,12 @@ struct ath11k_fw_stats_bcn { u32 tx_bcn_outage_cnt; }; +void ath11k_fw_stats_init(struct ath11k *ar); +void ath11k_fw_stats_pdevs_free(struct list_head *head); +void ath11k_fw_stats_vdevs_free(struct list_head *head); +void ath11k_fw_stats_bcn_free(struct list_head *head); +void ath11k_fw_stats_free(struct ath11k_fw_stats *stats); + extern const struct ce_pipe_config ath11k_target_ce_config_wlan_ipq8074[]; extern const struct service_to_pipe ath11k_target_service_to_ce_map_wlan_ipq8074[]; extern const struct service_to_pipe ath11k_target_service_to_ce_map_wlan_ipq6018[]; diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c index 9648e00173936b7f8e203986681ec52cc490cccb..ccdf3d5ba1ab6286a5a3f06e2a93d2590f2bb562 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.c +++ b/drivers/net/wireless/ath/ath11k/debugfs.c @@ -14,6 +14,7 @@ #include "dp_tx.h" #include "debugfs_htt_stats.h" #include "peer.h" +#include "hif.h" static const char *htt_bp_umac_ring[HTT_SW_UMAC_RING_IDX_MAX] = { "REO2SW1_RING", @@ -91,91 +92,35 @@ void ath11k_debugfs_add_dbring_entry(struct ath11k *ar, spin_unlock_bh(&dbr_data->lock); } -static void ath11k_fw_stats_pdevs_free(struct list_head *head) -{ - struct ath11k_fw_stats_pdev *i, *tmp; - - list_for_each_entry_safe(i, tmp, head, list) { - list_del(&i->list); - kfree(i); - } -} - -static void ath11k_fw_stats_vdevs_free(struct list_head *head) -{ - struct ath11k_fw_stats_vdev *i, *tmp; - - list_for_each_entry_safe(i, tmp, head, list) { - list_del(&i->list); - kfree(i); - } -} - -static void ath11k_fw_stats_bcn_free(struct list_head *head) -{ - struct ath11k_fw_stats_bcn *i, *tmp; - - list_for_each_entry_safe(i, tmp, head, list) { - list_del(&i->list); - kfree(i); - } -} - static void ath11k_debugfs_fw_stats_reset(struct ath11k *ar) { spin_lock_bh(&ar->data_lock); - ar->debug.fw_stats_done = false; - ath11k_fw_stats_pdevs_free(&ar->debug.fw_stats.pdevs); - ath11k_fw_stats_vdevs_free(&ar->debug.fw_stats.vdevs); + ar->fw_stats_done = false; + ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs); + ath11k_fw_stats_vdevs_free(&ar->fw_stats.vdevs); spin_unlock_bh(&ar->data_lock); } -void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb) +void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats *stats) { - struct ath11k_fw_stats stats = {}; - struct ath11k *ar; + struct ath11k_base *ab = ar->ab; struct ath11k_pdev *pdev; bool is_end; static unsigned int num_vdev, num_bcn; size_t total_vdevs_started = 0; - int i, ret; - - INIT_LIST_HEAD(&stats.pdevs); - INIT_LIST_HEAD(&stats.vdevs); - INIT_LIST_HEAD(&stats.bcn); - - ret = ath11k_wmi_pull_fw_stats(ab, skb, &stats); - if (ret) { - ath11k_warn(ab, "failed to pull fw stats: %d\n", ret); - goto free; - } - - rcu_read_lock(); - ar = ath11k_mac_get_ar_by_pdev_id(ab, stats.pdev_id); - if (!ar) { - rcu_read_unlock(); - ath11k_warn(ab, "failed to get ar for pdev_id %d: %d\n", - stats.pdev_id, ret); - goto free; - } + int i; - spin_lock_bh(&ar->data_lock); + /* WMI_REQUEST_PDEV_STAT request has been already processed */ - if (stats.stats_id == WMI_REQUEST_PDEV_STAT) { - list_splice_tail_init(&stats.pdevs, &ar->debug.fw_stats.pdevs); - ar->debug.fw_stats_done = true; - goto complete; - } - - if (stats.stats_id == WMI_REQUEST_RSSI_PER_CHAIN_STAT) { - ar->debug.fw_stats_done = true; - goto complete; + if (stats->stats_id == WMI_REQUEST_RSSI_PER_CHAIN_STAT) { + ar->fw_stats_done = true; + return; } - if (stats.stats_id == WMI_REQUEST_VDEV_STAT) { - if (list_empty(&stats.vdevs)) { + if (stats->stats_id == WMI_REQUEST_VDEV_STAT) { + if (list_empty(&stats->vdevs)) { ath11k_warn(ab, "empty vdev stats"); - goto complete; + return; } /* FW sends all the active VDEV stats irrespective of PDEV, * hence limit until the count of all VDEVs started @@ -188,43 +133,34 @@ void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb is_end = ((++num_vdev) == total_vdevs_started); - list_splice_tail_init(&stats.vdevs, - &ar->debug.fw_stats.vdevs); + list_splice_tail_init(&stats->vdevs, + &ar->fw_stats.vdevs); if (is_end) { - ar->debug.fw_stats_done = true; + ar->fw_stats_done = true; num_vdev = 0; } - goto complete; + return; } - if (stats.stats_id == WMI_REQUEST_BCN_STAT) { - if (list_empty(&stats.bcn)) { + if (stats->stats_id == WMI_REQUEST_BCN_STAT) { + if (list_empty(&stats->bcn)) { ath11k_warn(ab, "empty bcn stats"); - goto complete; + return; } /* Mark end until we reached the count of all started VDEVs * within the PDEV */ is_end = ((++num_bcn) == ar->num_started_vdevs); - list_splice_tail_init(&stats.bcn, - &ar->debug.fw_stats.bcn); + list_splice_tail_init(&stats->bcn, + &ar->fw_stats.bcn); if (is_end) { - ar->debug.fw_stats_done = true; + ar->fw_stats_done = true; num_bcn = 0; } } -complete: - complete(&ar->debug.fw_stats_complete); - rcu_read_unlock(); - spin_unlock_bh(&ar->data_lock); - -free: - ath11k_fw_stats_pdevs_free(&stats.pdevs); - ath11k_fw_stats_vdevs_free(&stats.vdevs); - ath11k_fw_stats_bcn_free(&stats.bcn); } static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, @@ -245,7 +181,7 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, ath11k_debugfs_fw_stats_reset(ar); - reinit_completion(&ar->debug.fw_stats_complete); + reinit_completion(&ar->fw_stats_complete); ret = ath11k_wmi_send_stats_request_cmd(ar, req_param); @@ -255,9 +191,8 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, return ret; } - time_left = - wait_for_completion_timeout(&ar->debug.fw_stats_complete, - 1 * HZ); + time_left = wait_for_completion_timeout(&ar->fw_stats_complete, 1 * HZ); + if (!time_left) return -ETIMEDOUT; @@ -266,7 +201,7 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, break; spin_lock_bh(&ar->data_lock); - if (ar->debug.fw_stats_done) { + if (ar->fw_stats_done) { spin_unlock_bh(&ar->data_lock); break; } @@ -338,8 +273,7 @@ static int ath11k_open_pdev_stats(struct inode *inode, struct file *file) goto err_free; } - ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id, - buf); + ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf); file->private_data = buf; @@ -410,8 +344,7 @@ static int ath11k_open_vdev_stats(struct inode *inode, struct file *file) goto err_free; } - ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id, - buf); + ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf); file->private_data = buf; @@ -488,14 +421,13 @@ static int ath11k_open_bcn_stats(struct inode *inode, struct file *file) } } - ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id, - buf); + ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf); /* since beacon stats request is looped for all active VDEVs, saved fw * stats is not freed for each request until done for all active VDEVs */ spin_lock_bh(&ar->data_lock); - ath11k_fw_stats_bcn_free(&ar->debug.fw_stats.bcn); + ath11k_fw_stats_bcn_free(&ar->fw_stats.bcn); spin_unlock_bh(&ar->data_lock); file->private_data = buf; @@ -982,6 +914,63 @@ static const struct file_operations fops_fw_dbglog = { .llseek = default_llseek, }; +static int ath11k_open_sram_dump(struct inode *inode, struct file *file) +{ + struct ath11k_base *ab = inode->i_private; + u8 *buf; + u32 start, end; + int ret; + + start = ab->hw_params.sram_dump.start; + end = ab->hw_params.sram_dump.end; + + buf = vmalloc(end - start + 1); + if (!buf) + return -ENOMEM; + + ret = ath11k_hif_read(ab, buf, start, end); + if (ret) { + ath11k_warn(ab, "failed to dump sram: %d\n", ret); + vfree(buf); + return ret; + } + + file->private_data = buf; + return 0; +} + +static ssize_t ath11k_read_sram_dump(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k_base *ab = file->f_inode->i_private; + const char *buf = file->private_data; + int len; + u32 start, end; + + start = ab->hw_params.sram_dump.start; + end = ab->hw_params.sram_dump.end; + len = end - start + 1; + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static int ath11k_release_sram_dump(struct inode *inode, struct file *file) +{ + vfree(file->private_data); + file->private_data = NULL; + + return 0; +} + +static const struct file_operations fops_sram_dump = { + .open = ath11k_open_sram_dump, + .read = ath11k_read_sram_dump, + .release = ath11k_release_sram_dump, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + int ath11k_debugfs_pdev_create(struct ath11k_base *ab) { if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) @@ -997,6 +986,10 @@ int ath11k_debugfs_pdev_create(struct ath11k_base *ab) debugfs_create_file("soc_dp_stats", 0600, ab->debugfs_soc, ab, &fops_soc_dp_stats); + if (ab->hw_params.sram_dump.start != 0) + debugfs_create_file("sram", 0400, ab->debugfs_soc, ab, + &fops_sram_dump); + return 0; } @@ -1025,7 +1018,7 @@ void ath11k_debugfs_fw_stats_init(struct ath11k *ar) struct dentry *fwstats_dir = debugfs_create_dir("fw_stats", ar->debug.debugfs_pdev); - ar->debug.fw_stats.debugfs_fwstats = fwstats_dir; + ar->fw_stats.debugfs_fwstats = fwstats_dir; /* all stats debugfs files created are under "fw_stats" directory * created per PDEV @@ -1036,12 +1029,6 @@ void ath11k_debugfs_fw_stats_init(struct ath11k *ar) &fops_vdev_stats); debugfs_create_file("beacon_stats", 0600, fwstats_dir, ar, &fops_bcn_stats); - - INIT_LIST_HEAD(&ar->debug.fw_stats.pdevs); - INIT_LIST_HEAD(&ar->debug.fw_stats.vdevs); - INIT_LIST_HEAD(&ar->debug.fw_stats.bcn); - - init_completion(&ar->debug.fw_stats_complete); } static ssize_t ath11k_write_pktlog_filter(struct file *file, @@ -1382,6 +1369,193 @@ static const struct file_operations fops_dbr_debug = { .llseek = default_llseek, }; +static ssize_t ath11k_write_ps_timekeeper_enable(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + ssize_t ret; + u8 ps_timekeeper_enable; + + if (kstrtou8_from_user(user_buf, count, 0, &ps_timekeeper_enable)) + return -EINVAL; + + mutex_lock(&ar->conf_mutex); + + if (ar->state != ATH11K_STATE_ON) { + ret = -ENETDOWN; + goto exit; + } + + if (!ar->ps_state_enable) { + ret = -EINVAL; + goto exit; + } + + ar->ps_timekeeper_enable = !!ps_timekeeper_enable; + ret = count; +exit: + mutex_unlock(&ar->conf_mutex); + + return ret; +} + +static ssize_t ath11k_read_ps_timekeeper_enable(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + char buf[32]; + int len; + + mutex_lock(&ar->conf_mutex); + len = scnprintf(buf, sizeof(buf), "%d\n", ar->ps_timekeeper_enable); + mutex_unlock(&ar->conf_mutex); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_ps_timekeeper_enable = { + .read = ath11k_read_ps_timekeeper_enable, + .write = ath11k_write_ps_timekeeper_enable, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static void ath11k_reset_peer_ps_duration(void *data, + struct ieee80211_sta *sta) +{ + struct ath11k *ar = data; + struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + + spin_lock_bh(&ar->data_lock); + arsta->ps_total_duration = 0; + spin_unlock_bh(&ar->data_lock); +} + +static ssize_t ath11k_write_reset_ps_duration(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + int ret; + u8 reset_ps_duration; + + if (kstrtou8_from_user(user_buf, count, 0, &reset_ps_duration)) + return -EINVAL; + + mutex_lock(&ar->conf_mutex); + + if (ar->state != ATH11K_STATE_ON) { + ret = -ENETDOWN; + goto exit; + } + + if (!ar->ps_state_enable) { + ret = -EINVAL; + goto exit; + } + + ieee80211_iterate_stations_atomic(ar->hw, + ath11k_reset_peer_ps_duration, + ar); + + ret = count; +exit: + mutex_unlock(&ar->conf_mutex); + return ret; +} + +static const struct file_operations fops_reset_ps_duration = { + .write = ath11k_write_reset_ps_duration, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static void ath11k_peer_ps_state_disable(void *data, + struct ieee80211_sta *sta) +{ + struct ath11k *ar = data; + struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + + spin_lock_bh(&ar->data_lock); + arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED; + arsta->ps_start_time = 0; + arsta->ps_total_duration = 0; + spin_unlock_bh(&ar->data_lock); +} + +static ssize_t ath11k_write_ps_state_enable(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + struct ath11k_pdev *pdev = ar->pdev; + int ret; + u32 param; + u8 ps_state_enable; + + if (kstrtou8_from_user(user_buf, count, 0, &ps_state_enable)) + return -EINVAL; + + mutex_lock(&ar->conf_mutex); + + ps_state_enable = !!ps_state_enable; + + if (ar->ps_state_enable == ps_state_enable) { + ret = count; + goto exit; + } + + param = WMI_PDEV_PEER_STA_PS_STATECHG_ENABLE; + ret = ath11k_wmi_pdev_set_param(ar, param, ps_state_enable, pdev->pdev_id); + if (ret) { + ath11k_warn(ar->ab, "failed to enable ps_state_enable: %d\n", + ret); + goto exit; + } + ar->ps_state_enable = ps_state_enable; + + if (!ar->ps_state_enable) { + ar->ps_timekeeper_enable = false; + ieee80211_iterate_stations_atomic(ar->hw, + ath11k_peer_ps_state_disable, + ar); + } + + ret = count; + +exit: + mutex_unlock(&ar->conf_mutex); + + return ret; +} + +static ssize_t ath11k_read_ps_state_enable(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ath11k *ar = file->private_data; + char buf[32]; + int len; + + mutex_lock(&ar->conf_mutex); + len = scnprintf(buf, sizeof(buf), "%d\n", ar->ps_state_enable); + mutex_unlock(&ar->conf_mutex); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_ps_state_enable = { + .read = ath11k_read_ps_state_enable, + .write = ath11k_write_ps_state_enable, + .open = simple_open, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + int ath11k_debugfs_register(struct ath11k *ar) { struct ath11k_base *ab = ar->ab; @@ -1428,6 +1602,20 @@ int ath11k_debugfs_register(struct ath11k *ar) debugfs_create_file("enable_dbr_debug", 0200, ar->debug.debugfs_pdev, ar, &fops_dbr_debug); + debugfs_create_file("ps_state_enable", 0600, ar->debug.debugfs_pdev, ar, + &fops_ps_state_enable); + + if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT, + ar->ab->wmi_ab.svc_map)) { + debugfs_create_file("ps_timekeeper_enable", 0600, + ar->debug.debugfs_pdev, ar, + &fops_ps_timekeeper_enable); + + debugfs_create_file("reset_ps_duration", 0200, + ar->debug.debugfs_pdev, ar, + &fops_reset_ps_duration); + } + return 0; } @@ -1456,11 +1644,13 @@ static ssize_t ath11k_write_twt_add_dialog(struct file *file, { struct ath11k_vif *arvif = file->private_data; struct wmi_twt_add_dialog_params params = { 0 }; + struct wmi_twt_enable_params twt_params = {0}; + struct ath11k *ar = arvif->ar; u8 buf[128] = {0}; int ret; - if (arvif->ar->twt_enabled == 0) { - ath11k_err(arvif->ar->ab, "twt support is not enabled\n"); + if (ar->twt_enabled == 0) { + ath11k_err(ar->ab, "twt support is not enabled\n"); return -EOPNOTSUPP; } @@ -1490,13 +1680,38 @@ static ssize_t ath11k_write_twt_add_dialog(struct file *file, if (ret != 16) return -EINVAL; + /* In the case of station vif, TWT is entirely handled by + * the firmware based on the input parameters in the TWT enable + * WMI command that is sent to the target during assoc. + * For manually testing the TWT feature, we need to first disable + * TWT and send enable command again with TWT input parameter + * sta_cong_timer_ms set to 0. + */ + if (arvif->vif->type == NL80211_IFTYPE_STATION) { + ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id); + + ath11k_wmi_fill_default_twt_params(&twt_params); + twt_params.sta_cong_timer_ms = 0; + + ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params); + } + params.vdev_id = arvif->vdev_id; ret = ath11k_wmi_send_twt_add_dialog_cmd(arvif->ar, ¶ms); if (ret) - return ret; + goto err_twt_add_dialog; return count; + +err_twt_add_dialog: + if (arvif->vif->type == NL80211_IFTYPE_STATION) { + ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id); + ath11k_wmi_fill_default_twt_params(&twt_params); + ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params); + } + + return ret; } static ssize_t ath11k_write_twt_del_dialog(struct file *file, @@ -1505,11 +1720,13 @@ static ssize_t ath11k_write_twt_del_dialog(struct file *file, { struct ath11k_vif *arvif = file->private_data; struct wmi_twt_del_dialog_params params = { 0 }; + struct wmi_twt_enable_params twt_params = {0}; + struct ath11k *ar = arvif->ar; u8 buf[64] = {0}; int ret; - if (arvif->ar->twt_enabled == 0) { - ath11k_err(arvif->ar->ab, "twt support is not enabled\n"); + if (ar->twt_enabled == 0) { + ath11k_err(ar->ab, "twt support is not enabled\n"); return -EOPNOTSUPP; } @@ -1535,6 +1752,12 @@ static ssize_t ath11k_write_twt_del_dialog(struct file *file, if (ret) return ret; + if (arvif->vif->type == NL80211_IFTYPE_STATION) { + ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id); + ath11k_wmi_fill_default_twt_params(&twt_params); + ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params); + } + return count; } @@ -1638,36 +1861,35 @@ static const struct file_operations ath11k_fops_twt_resume_dialog = { .open = simple_open }; -int ath11k_debugfs_add_interface(struct ath11k_vif *arvif) +void ath11k_debugfs_add_interface(struct ath11k_vif *arvif) { - if (arvif->vif->type == NL80211_IFTYPE_AP && !arvif->debugfs_twt) { - arvif->debugfs_twt = debugfs_create_dir("twt", - arvif->vif->debugfs_dir); - if (!arvif->debugfs_twt || IS_ERR(arvif->debugfs_twt)) { - ath11k_warn(arvif->ar->ab, - "failed to create directory %p\n", - arvif->debugfs_twt); - arvif->debugfs_twt = NULL; - return -1; - } + struct ath11k_base *ab = arvif->ar->ab; - debugfs_create_file("add_dialog", 0200, arvif->debugfs_twt, - arvif, &ath11k_fops_twt_add_dialog); + if (arvif->vif->type != NL80211_IFTYPE_AP && + !(arvif->vif->type == NL80211_IFTYPE_STATION && + test_bit(WMI_TLV_SERVICE_STA_TWT, ab->wmi_ab.svc_map))) + return; - debugfs_create_file("del_dialog", 0200, arvif->debugfs_twt, - arvif, &ath11k_fops_twt_del_dialog); + arvif->debugfs_twt = debugfs_create_dir("twt", + arvif->vif->debugfs_dir); + debugfs_create_file("add_dialog", 0200, arvif->debugfs_twt, + arvif, &ath11k_fops_twt_add_dialog); - debugfs_create_file("pause_dialog", 0200, arvif->debugfs_twt, - arvif, &ath11k_fops_twt_pause_dialog); + debugfs_create_file("del_dialog", 0200, arvif->debugfs_twt, + arvif, &ath11k_fops_twt_del_dialog); - debugfs_create_file("resume_dialog", 0200, arvif->debugfs_twt, - arvif, &ath11k_fops_twt_resume_dialog); - } - return 0; + debugfs_create_file("pause_dialog", 0200, arvif->debugfs_twt, + arvif, &ath11k_fops_twt_pause_dialog); + + debugfs_create_file("resume_dialog", 0200, arvif->debugfs_twt, + arvif, &ath11k_fops_twt_resume_dialog); } void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif) { + if (!arvif->debugfs_twt) + return; + debugfs_remove_recursive(arvif->debugfs_twt); arvif->debugfs_twt = NULL; } diff --git a/drivers/net/wireless/ath/ath11k/debugfs.h b/drivers/net/wireless/ath/ath11k/debugfs.h index 30c00cb283118872f2ce923a7952618c0e005caa..3af0169f6cf218b0732a8c77bc665c2088550543 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.h +++ b/drivers/net/wireless/ath/ath11k/debugfs.h @@ -269,7 +269,7 @@ int ath11k_debugfs_pdev_create(struct ath11k_base *ab); void ath11k_debugfs_pdev_destroy(struct ath11k_base *ab); int ath11k_debugfs_register(struct ath11k *ar); void ath11k_debugfs_unregister(struct ath11k *ar); -void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb); +void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats *stats); void ath11k_debugfs_fw_stats_init(struct ath11k *ar); int ath11k_debugfs_get_fw_stats(struct ath11k *ar, u32 pdev_id, @@ -306,7 +306,7 @@ static inline int ath11k_debugfs_rx_filter(struct ath11k *ar) return ar->debug.rx_filter; } -int ath11k_debugfs_add_interface(struct ath11k_vif *arvif); +void ath11k_debugfs_add_interface(struct ath11k_vif *arvif); void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif); void ath11k_debugfs_add_dbring_entry(struct ath11k *ar, enum wmi_direct_buffer_module id, @@ -341,8 +341,8 @@ static inline void ath11k_debugfs_unregister(struct ath11k *ar) { } -static inline void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, - struct sk_buff *skb) +static inline void ath11k_debugfs_fw_stats_process(struct ath11k *ar, + struct ath11k_fw_stats *stats) { } @@ -386,9 +386,8 @@ static inline int ath11k_debugfs_get_fw_stats(struct ath11k *ar, return 0; } -static inline int ath11k_debugfs_add_interface(struct ath11k_vif *arvif) +static inline void ath11k_debugfs_add_interface(struct ath11k_vif *arvif) { - return 0; } static inline void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif) diff --git a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h index 5d722b51b12584cbffb62d10a05d78e7165d0777..2b97cbbd28cb791fa9969a449b0d19d179cf1877 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h +++ b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h @@ -630,7 +630,7 @@ struct htt_tx_hwq_tried_mpdu_cnt_hist_tlv_v { * completing the burst, we identify the txop used in the burst and * incr the corresponding bin. * Each bin represents 1ms & we have 10 bins in this histogram. - * they are deined in FW using the following macros + * they are defined in FW using the following macros * #define WAL_MAX_TXOP_USED_CNT_HISTOGRAM 10 * #define WAL_TXOP_USED_HISTOGRAM_INTERVAL 1000 ( 1 ms ) */ @@ -1897,7 +1897,7 @@ struct htt_phy_counters_tlv { u32 phytx_abort_cnt; /* number of times rx abort initiated by phy */ u32 phyrx_abort_cnt; - /* number of rx defered count initiated by phy */ + /* number of rx deferred count initiated by phy */ u32 phyrx_defer_abort_cnt; /* number of sizing events generated at LSTF */ u32 rx_gain_adj_lstf_event_cnt; diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.c b/drivers/net/wireless/ath/ath11k/debugfs_sta.c index 1b1acbdf837a30e519aaab3e9ddd480b7249f4ac..9cc4ef28e7519bab4d9886b3e7df0f39360ada07 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs_sta.c +++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.c @@ -751,6 +751,102 @@ static const struct file_operations fops_htt_peer_stats_reset = { .llseek = default_llseek, }; +static ssize_t ath11k_dbg_sta_read_peer_ps_state(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ieee80211_sta *sta = file->private_data; + struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k *ar = arsta->arvif->ar; + char buf[20]; + int len; + + spin_lock_bh(&ar->data_lock); + + len = scnprintf(buf, sizeof(buf), "%d\n", arsta->peer_ps_state); + + spin_unlock_bh(&ar->data_lock); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_peer_ps_state = { + .open = simple_open, + .read = ath11k_dbg_sta_read_peer_ps_state, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static ssize_t ath11k_dbg_sta_read_current_ps_duration(struct file *file, + char __user *user_buf, + size_t count, + loff_t *ppos) +{ + struct ieee80211_sta *sta = file->private_data; + struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k *ar = arsta->arvif->ar; + u64 time_since_station_in_power_save; + char buf[20]; + int len; + + spin_lock_bh(&ar->data_lock); + + if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON && + arsta->peer_current_ps_valid) + time_since_station_in_power_save = jiffies_to_msecs(jiffies + - arsta->ps_start_jiffies); + else + time_since_station_in_power_save = 0; + + len = scnprintf(buf, sizeof(buf), "%llu\n", + time_since_station_in_power_save); + spin_unlock_bh(&ar->data_lock); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_current_ps_duration = { + .open = simple_open, + .read = ath11k_dbg_sta_read_current_ps_duration, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static ssize_t ath11k_dbg_sta_read_total_ps_duration(struct file *file, + char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct ieee80211_sta *sta = file->private_data; + struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv; + struct ath11k *ar = arsta->arvif->ar; + char buf[20]; + u64 power_save_duration; + int len; + + spin_lock_bh(&ar->data_lock); + + if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON && + arsta->peer_current_ps_valid) + power_save_duration = jiffies_to_msecs(jiffies + - arsta->ps_start_jiffies) + + arsta->ps_total_duration; + else + power_save_duration = arsta->ps_total_duration; + + len = scnprintf(buf, sizeof(buf), "%llu\n", power_save_duration); + + spin_unlock_bh(&ar->data_lock); + + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static const struct file_operations fops_total_ps_duration = { + .open = simple_open, + .read = ath11k_dbg_sta_read_total_ps_duration, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + void ath11k_debugfs_sta_op_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct dentry *dir) { @@ -778,4 +874,15 @@ void ath11k_debugfs_sta_op_add(struct ieee80211_hw *hw, struct ieee80211_vif *vi ar->ab->wmi_ab.svc_map)) debugfs_create_file("htt_peer_stats_reset", 0600, dir, sta, &fops_htt_peer_stats_reset); + + debugfs_create_file("peer_ps_state", 0400, dir, sta, + &fops_peer_ps_state); + + if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT, + ar->ab->wmi_ab.svc_map)) { + debugfs_create_file("current_ps_duration", 0440, dir, sta, + &fops_current_ps_duration); + debugfs_create_file("total_ps_duration", 0440, dir, sta, + &fops_total_ps_duration); + } } diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c index 8b790ce72e5df2a0bcd5945c2631cec884db33e7..f5156a7fbdd7a36918d908dc1df27a3beccf1ddb 100644 --- a/drivers/net/wireless/ath/ath11k/dp.c +++ b/drivers/net/wireless/ath/ath11k/dp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -131,13 +132,11 @@ static int ath11k_dp_srng_calculate_msi_group(struct ath11k_base *ab, switch (type) { case HAL_WBM2SW_RELEASE: - if (ring_num < 3) { - grp_mask = &ab->hw_params.ring_mask->tx[0]; - } else if (ring_num == 3) { + if (ring_num == DP_RX_RELEASE_RING_NUM) { grp_mask = &ab->hw_params.ring_mask->rx_wbm_rel[0]; ring_num = 0; } else { - return -ENOENT; + grp_mask = &ab->hw_params.ring_mask->tx[0]; } break; case HAL_REO_EXCEPTION: @@ -371,6 +370,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab) struct ath11k_dp *dp = &ab->dp; struct hal_srng *srng; int i, ret; + u8 tcl_num, wbm_num; ret = ath11k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring, HAL_SW2WBM_RELEASE, 0, 0, @@ -396,9 +396,12 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab) } for (i = 0; i < ab->hw_params.max_tx_ring; i++) { + tcl_num = ab->hw_params.hal_params->tcl2wbm_rbm_map[i].tcl_ring_num; + wbm_num = ab->hw_params.hal_params->tcl2wbm_rbm_map[i].wbm_ring_num; + ret = ath11k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring, - HAL_TCL_DATA, i, 0, - DP_TCL_DATA_RING_SIZE); + HAL_TCL_DATA, tcl_num, 0, + ab->hw_params.tx_ring_size); if (ret) { ath11k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n", i, ret); @@ -406,7 +409,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab) } ret = ath11k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring, - HAL_WBM2SW_RELEASE, i, 0, + HAL_WBM2SW_RELEASE, wbm_num, 0, DP_TX_COMP_RING_SIZE); if (ret) { ath11k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n", @@ -431,7 +434,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab) } ret = ath11k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE, - 3, 0, DP_RX_RELEASE_RING_SIZE); + DP_RX_RELEASE_RING_NUM, 0, DP_RX_RELEASE_RING_SIZE); if (ret) { ath11k_warn(ab, "failed to set up rx_rel ring :%d\n", ret); goto err; @@ -774,9 +777,10 @@ int ath11k_dp_service_srng(struct ath11k_base *ab, int i, j; int tot_work_done = 0; - if (ab->hw_params.ring_mask->tx[grp_id]) { - i = __fls(ab->hw_params.ring_mask->tx[grp_id]); - ath11k_dp_tx_completion_handler(ab, i); + for (i = 0; i < ab->hw_params.max_tx_ring; i++) { + if (BIT(ab->hw_params.hal_params->tcl2wbm_rbm_map[i].wbm_ring_num) & + ab->hw_params.ring_mask->tx[grp_id]) + ath11k_dp_tx_completion_handler(ab, i); } if (ab->hw_params.ring_mask->rx_err[grp_id]) { @@ -963,7 +967,7 @@ static void ath11k_dp_update_vdev_search(struct ath11k_vif *arvif) { /* When v2_map_support is true:for STA mode, enable address * search index, tcl uses ast_hash value in the descriptor. - * When v2_map_support is false: for STA mode, dont' enable + * When v2_map_support is false: for STA mode, don't enable * address search index. */ switch (arvif->vdev_type) { diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h index e9dfa209098b1fe8385114e880a7ea02ddd679a1..be9eafc872b3be6ef978975dd7787f24705683fa 100644 --- a/drivers/net/wireless/ath/ath11k/dp.h +++ b/drivers/net/wireless/ath/ath11k/dp.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_DP_H @@ -203,6 +204,7 @@ struct ath11k_pdev_dp { #define DP_WBM_RELEASE_RING_SIZE 64 #define DP_TCL_DATA_RING_SIZE 512 +#define DP_TCL_DATA_RING_SIZE_WCN6750 2048 #define DP_TX_COMP_RING_SIZE 32768 #define DP_TX_IDR_SIZE DP_TX_COMP_RING_SIZE #define DP_TCL_CMD_RING_SIZE 32 @@ -222,6 +224,8 @@ struct ath11k_pdev_dp { #define DP_RXDMA_MONITOR_DST_RING_SIZE 2048 #define DP_RXDMA_MONITOR_DESC_RING_SIZE 4096 +#define DP_RX_RELEASE_RING_NUM 3 + #define DP_RX_BUFFER_SIZE 2048 #define DP_RX_BUFFER_SIZE_LITE 1024 #define DP_RX_BUFFER_ALIGN_SIZE 128 @@ -299,7 +303,7 @@ struct ath11k_dp { #define HTT_TX_WBM_COMP_STATUS_OFFSET 8 -/* HTT tx completion is overlayed in wbm_release_ring */ +/* HTT tx completion is overlaid in wbm_release_ring */ #define HTT_TX_WBM_COMP_INFO0_STATUS GENMASK(12, 9) #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON GENMASK(16, 13) #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON GENMASK(16, 13) @@ -466,7 +470,7 @@ enum htt_srng_ring_id { * 3'b010: 4 usec * 3'b011: 8 usec (default) * 3'b100: 16 usec - * Others: Reserverd + * Others: Reserved * b'19 - response_required: * Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response * b'20:31 - reserved: reserved for future use @@ -993,8 +997,7 @@ struct htt_rx_ring_tlv_filter { #define HTT_RX_FULL_MON_MODE_CFG_CMD_CFG_NON_ZERO_MPDUS_END BIT(2) #define HTT_RX_FULL_MON_MODE_CFG_CMD_CFG_RELEASE_RING GENMASK(10, 3) -/** - * Enumeration for full monitor mode destination ring select +/* Enumeration for full monitor mode destination ring select * 0 - REO destination ring select * 1 - FW destination ring select * 2 - SW destination ring select @@ -1391,8 +1394,7 @@ struct htt_ppdu_stats_info { struct list_head list; }; -/** - * @brief target -> host packet log message +/* @brief target -> host packet log message * * @details * The following field definitions describe the format of the packet log @@ -1430,8 +1432,7 @@ struct htt_pktlog_msg { u8 payload[]; }; -/** - * @brief host -> target FW extended statistics retrieve +/* @brief host -> target FW extended statistics retrieve * * @details * The following field definitions describe the format of the HTT host @@ -1566,8 +1567,7 @@ struct htt_ext_stats_cfg_params { u32 cfg3; }; -/** - * @brief target -> host extended statistics upload +/* @brief target -> host extended statistics upload * * @details * The following field definitions describe the format of the HTT target diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 2148acf37071ea3bbcc791682fbf5abd5a9da5b2..c5a4c34d77499ac11e3f95f0a589ce29edf16b89 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -2499,7 +2499,7 @@ static void ath11k_dp_rx_deliver_msdu(struct ath11k *ar, struct napi_struct *nap /* PN for multicast packets are not validate in HW, * so skip 802.3 rx path - * Also, fast_rx expectes the STA to be authorized, hence + * Also, fast_rx expects the STA to be authorized, hence * eapol packets are sent in slow path. */ if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol && @@ -5197,7 +5197,8 @@ int ath11k_dp_rx_process_mon_status(struct ath11k_base *ab, int mac_id, if (log_type != ATH11K_PKTLOG_TYPE_INVALID) trace_ath11k_htt_rxdesc(ar, skb->data, log_type, rx_buf_sz); - memset(ppdu_info, 0, sizeof(struct hal_rx_mon_ppdu_info)); + memset(ppdu_info, 0, sizeof(*ppdu_info)); + ppdu_info->peer_id = HAL_INVALID_PEERID; hal_status = ath11k_hal_rx_parse_mon_status(ab, ppdu_info, skb); if (test_bit(ATH11K_FLAG_MONITOR_STARTED, &ar->monitor_flags) && diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c index c17a2620aad74fc9f07af1440870bc0a277d2106..8afbba2369354bb7a9a249920d3989e81e4e035d 100644 --- a/drivers/net/wireless/ath/ath11k/dp_tx.c +++ b/drivers/net/wireless/ath/ath11k/dp_tx.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "core.h" @@ -93,7 +94,8 @@ int ath11k_dp_tx(struct ath11k *ar, struct ath11k_vif *arvif, u8 pool_id; u8 hal_ring_id; int ret; - u8 ring_selector = 0, ring_map = 0; + u32 ring_selector = 0; + u8 ring_map = 0; bool tcl_ring_retry; if (unlikely(test_bit(ATH11K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags))) @@ -105,19 +107,13 @@ int ath11k_dp_tx(struct ath11k *ar, struct ath11k_vif *arvif, pool_id = skb_get_queue_mapping(skb) & (ATH11K_HW_MAX_QUEUES - 1); - /* Let the default ring selection be based on current processor - * number, where one of the 3 tcl rings are selected based on - * the smp_processor_id(). In case that ring - * is full/busy, we resort to other available rings. - * If all rings are full, we drop the packet. - * //TODO Add throttling logic when all rings are full - */ - ring_selector = smp_processor_id(); + ring_selector = ab->hw_params.hw_ops->get_ring_selector(skb); tcl_ring_sel: tcl_ring_retry = false; ti.ring_id = ring_selector % ab->hw_params.max_tx_ring; + ti.rbm_id = ab->hw_params.hal_params->tcl2wbm_rbm_map[ti.ring_id].rbm_id; ring_map |= BIT(ti.ring_id); @@ -129,7 +125,8 @@ tcl_ring_sel: spin_unlock_bh(&tx_ring->tx_idr_lock); if (unlikely(ret < 0)) { - if (ring_map == (BIT(ab->hw_params.max_tx_ring) - 1)) { + if (ring_map == (BIT(ab->hw_params.max_tx_ring) - 1) || + !ab->hw_params.tcl_ring_retry) { atomic_inc(&ab->soc_stats.tx_err.misc_fail); return -ENOSPC; } @@ -247,7 +244,7 @@ tcl_ring_sel: * Restart ring selection if some rings are not checked yet. */ if (unlikely(ring_map != (BIT(ab->hw_params.max_tx_ring)) - 1) && - ab->hw_params.max_tx_ring > 1) { + ab->hw_params.tcl_ring_retry && ab->hw_params.max_tx_ring > 1) { tcl_ring_retry = true; ring_selector++; } @@ -755,7 +752,7 @@ int ath11k_dp_tx_send_reo_cmd(struct ath11k_base *ab, struct dp_rx_tid *rx_tid, return 0; /* Can this be optimized so that we keep the pending command list only - * for tid delete command to free up the resoruce on the command status + * for tid delete command to free up the resource on the command status * indication? */ dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC); diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c index bda71ab5a1f29a40df06f86156f099f50b2be4e8..2fd224480d4532d7d37211b07b576f2c1ad0234a 100644 --- a/drivers/net/wireless/ath/ath11k/hal.c +++ b/drivers/net/wireless/ath/ath11k/hal.c @@ -126,7 +126,7 @@ static const struct hal_srng_config hw_srng_config_template[] = { }, { /* WBM2SW_RELEASE */ .start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE, - .max_rings = 4, + .max_rings = 5, .entry_size = sizeof(struct hal_wbm_release_ring) >> 2, .lmac_ring = false, .ring_dir = HAL_SRNG_DIR_DST, @@ -1164,7 +1164,7 @@ void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab, { lockdep_assert_held(&srng->lock); - /* check whether the ring is emptry. Update the shadow + /* check whether the ring is empty. Update the shadow * HP only when then ring isn't empty. */ if (srng->ring_dir == HAL_SRNG_DIR_SRC && diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h index 110c337ddf337a9bf61ce049b93f70805e6cd6df..6a1f78ee6eb66ccdbfaaa262d92891c915649f3b 100644 --- a/drivers/net/wireless/ath/ath11k/hal.h +++ b/drivers/net/wireless/ath/ath11k/hal.h @@ -243,7 +243,7 @@ struct ath11k_base; #define HAL_WBM0_RELEASE_RING_HP 0x000030c0 #define HAL_WBM1_RELEASE_RING_HP 0x000030c8 -/* TCL ring feild mask and offset */ +/* TCL ring field mask and offset */ #define HAL_TCL1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) #define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) #define HAL_TCL1_RING_ID_ENTRY_SIZE GENMASK(7, 0) @@ -268,7 +268,7 @@ struct ath11k_base; #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6 GENMASK(20, 18) #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7 GENMASK(23, 21) -/* REO ring feild mask and offset */ +/* REO ring field mask and offset */ #define HAL_REO1_RING_BASE_MSB_RING_SIZE GENMASK(27, 8) #define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB GENMASK(7, 0) #define HAL_REO1_RING_ID_RING_ID GENMASK(15, 8) @@ -389,6 +389,7 @@ enum hal_srng_ring_id { HAL_SRNG_RING_ID_WBM2SW1_RELEASE, HAL_SRNG_RING_ID_WBM2SW2_RELEASE, HAL_SRNG_RING_ID_WBM2SW3_RELEASE, + HAL_SRNG_RING_ID_WBM2SW4_RELEASE, HAL_SRNG_RING_ID_UMAC_ID_END = 127, HAL_SRNG_RING_ID_LMAC1_ID_START, @@ -450,13 +451,13 @@ enum hal_ring_type { /** * enum hal_reo_cmd_type: Enum for REO command type - * @CMD_GET_QUEUE_STATS: Get REO queue status/stats - * @CMD_FLUSH_QUEUE: Flush all frames in REO queue - * @CMD_FLUSH_CACHE: Flush descriptor entries in the cache - * @CMD_UNBLOCK_CACHE: Unblock a descriptor's address that was blocked + * @HAL_REO_CMD_GET_QUEUE_STATS: Get REO queue status/stats + * @HAL_REO_CMD_FLUSH_QUEUE: Flush all frames in REO queue + * @HAL_REO_CMD_FLUSH_CACHE: Flush descriptor entries in the cache + * @HAL_REO_CMD_UNBLOCK_CACHE: Unblock a descriptor's address that was blocked * earlier with a 'REO_FLUSH_CACHE' command - * @CMD_FLUSH_TIMEOUT_LIST: Flush buffers/descriptors from timeout list - * @CMD_UPDATE_RX_REO_QUEUE: Update REO queue settings + * @HAL_REO_CMD_FLUSH_TIMEOUT_LIST: Flush buffers/descriptors from timeout list + * @HAL_REO_CMD_UPDATE_RX_QUEUE: Update REO queue settings */ enum hal_reo_cmd_type { HAL_REO_CMD_GET_QUEUE_STATS = 0, @@ -635,7 +636,7 @@ struct hal_srng { } u; }; -/* Interrupt mitigation - Batch threshold in terms of numer of frames */ +/* Interrupt mitigation - Batch threshold in terms of number of frames */ #define HAL_SRNG_INT_BATCH_THRESHOLD_TX 256 #define HAL_SRNG_INT_BATCH_THRESHOLD_RX 128 #define HAL_SRNG_INT_BATCH_THRESHOLD_OTHER 1 @@ -678,6 +679,7 @@ enum hal_rx_buf_return_buf_manager { HAL_RX_BUF_RBM_SW1_BM, HAL_RX_BUF_RBM_SW2_BM, HAL_RX_BUF_RBM_SW3_BM, + HAL_RX_BUF_RBM_SW4_BM, }; #define HAL_SRNG_DESC_LOOP_CNT 0xf0000000 @@ -873,8 +875,7 @@ struct hal_reo_status { } u; }; -/** - * HAL context to be used to access SRNG APIs (currently used by data path +/* HAL context to be used to access SRNG APIs (currently used by data path * and transport (CE) modules) */ struct ath11k_hal { diff --git a/drivers/net/wireless/ath/ath11k/hal_desc.h b/drivers/net/wireless/ath/ath11k/hal_desc.h index 24e72e75a8c733dd599f1cc4dd9b774ffb4a83ce..d895ea878d9f03af56f9092ea887869dc7ddbd77 100644 --- a/drivers/net/wireless/ath/ath11k/hal_desc.h +++ b/drivers/net/wireless/ath/ath11k/hal_desc.h @@ -607,7 +607,7 @@ struct rx_msdu_desc { * * msdu_continuation * When set, this MSDU buffer was not able to hold the entire MSDU. - * The next buffer will therefor contain additional information + * The next buffer will therefore contain additional information * related to this MSDU. * * msdu_length @@ -643,7 +643,7 @@ struct rx_msdu_desc { * * da_idx_timeout * Indicates, an unsuccessful MAC destination address search due - * to the expiration of search timer fot this MSDU. + * to the expiration of search timer for this MSDU. */ enum hal_reo_dest_ring_buffer_type { @@ -1678,7 +1678,7 @@ struct hal_wbm_release_ring { * Producer: SW/TQM/RXDMA/REO/SWITCH * Consumer: WBM/SW/FW * - * HTT tx status is overlayed on wbm_release ring on 4-byte words 2, 3, 4 and 5 + * HTT tx status is overlaid on wbm_release ring on 4-byte words 2, 3, 4 and 5 * for software based completions. * * buf_addr_info @@ -2159,7 +2159,7 @@ struct hal_reo_status_hdr { * commands. * * execution_time (in us) - * The amount of time REO took to excecute the command. Note that + * The amount of time REO took to execute the command. Note that * this time does not include the duration of the command waiting * in the command ring, before the execution started. * diff --git a/drivers/net/wireless/ath/ath11k/hal_tx.c b/drivers/net/wireless/ath/ath11k/hal_tx.c index c8929de8ce6c830161b39f3168a3ef8c57f2febe..d1b0e36e04a9c20b2a3324e5dc93b83502b2a3e2 100644 --- a/drivers/net/wireless/ath/ath11k/hal_tx.c +++ b/drivers/net/wireless/ath/ath11k/hal_tx.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include "hal_desc.h" @@ -44,8 +45,7 @@ void ath11k_hal_tx_cmd_desc_setup(struct ath11k_base *ab, void *cmd, FIELD_PREP(BUFFER_ADDR_INFO1_ADDR, ((uint64_t)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT)); tcl_cmd->buf_addr_info.info1 |= - FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, - (ti->ring_id + HAL_RX_BUF_RBM_SW0_BM)) | + FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, ti->rbm_id) | FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, ti->desc_id); tcl_cmd->info0 = diff --git a/drivers/net/wireless/ath/ath11k/hal_tx.h b/drivers/net/wireless/ath/ath11k/hal_tx.h index 36f4f6f6cbc2154b03ac7ec74ab4db6a3e94d966..c5e88364afe56d5002ccabdd5a25847f3c37b8ef 100644 --- a/drivers/net/wireless/ath/ath11k/hal_tx.h +++ b/drivers/net/wireless/ath/ath11k/hal_tx.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause-Clear */ /* * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #ifndef ATH11K_HAL_TX_H @@ -35,6 +36,7 @@ struct hal_tx_info { u8 lmac_id; u8 dscp_tid_tbl_idx; bool enable_mesh; + u8 rbm_id; }; /* TODO: Check if the actual desc macros can be used instead */ diff --git a/drivers/net/wireless/ath/ath11k/hif.h b/drivers/net/wireless/ath/ath11k/hif.h index e9366f786fbb6ab059d910efdf72c63a8b36bbed..659b80d2abd4de878c2aa78531569466e1fb3927 100644 --- a/drivers/net/wireless/ath/ath11k/hif.h +++ b/drivers/net/wireless/ath/ath11k/hif.h @@ -11,6 +11,7 @@ struct ath11k_hif_ops { u32 (*read32)(struct ath11k_base *sc, u32 address); void (*write32)(struct ath11k_base *sc, u32 address, u32 data); + int (*read)(struct ath11k_base *ab, void *buf, u32 start, u32 end); void (*irq_enable)(struct ath11k_base *sc); void (*irq_disable)(struct ath11k_base *sc); int (*start)(struct ath11k_base *sc); @@ -99,6 +100,15 @@ static inline void ath11k_hif_write32(struct ath11k_base *sc, u32 address, u32 d sc->hif.ops->write32(sc, address, data); } +static inline int ath11k_hif_read(struct ath11k_base *ab, void *buf, + u32 start, u32 end) +{ + if (!ab->hif.ops->read) + return -EOPNOTSUPP; + + return ab->hif.ops->read(ab, buf, start, end); +} + static inline int ath11k_hif_map_service_to_pipe(struct ath11k_base *sc, u16 service_id, u8 *ul_pipe, u8 *dl_pipe) { @@ -134,4 +144,5 @@ static inline void ath11k_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id, else *msi_data_idx = ce_id; } + #endif /* _HIF_H_ */ diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c index 96db85c555851a8e55088be8f39c4e6cf9289fb5..dbcc0c4035b62e9aa0d470dbec4667a2f3660c0a 100644 --- a/drivers/net/wireless/ath/ath11k/hw.c +++ b/drivers/net/wireless/ath/ath11k/hw.c @@ -820,6 +820,30 @@ static bool ath11k_hw_wcn6855_rx_desc_get_ldpc_support(struct hal_rx_desc *desc) __le32_to_cpu(desc->u.wcn6855.msdu_start.info2)); } +static u32 ath11k_hw_ipq8074_get_tcl_ring_selector(struct sk_buff *skb) +{ + /* Let the default ring selection be based on current processor + * number, where one of the 3 tcl rings are selected based on + * the smp_processor_id(). In case that ring + * is full/busy, we resort to other available rings. + * If all rings are full, we drop the packet. + * + * TODO: Add throttling logic when all rings are full + */ + return smp_processor_id(); +} + +static u32 ath11k_hw_wcn6750_get_tcl_ring_selector(struct sk_buff *skb) +{ + /* Select the TCL ring based on the flow hash of the SKB instead + * of CPU ID. Since applications pumping the traffic can be scheduled + * on multiple CPUs, there is a chance that packets of the same flow + * could end on different TCL rings, this could sometimes results in + * an out of order arrival of the packets at the receiver. + */ + return skb_get_hash(skb); +} + const struct ath11k_hw_ops ipq8074_ops = { .get_hw_mac_from_pdev_id = ath11k_hw_ipq8074_mac_from_pdev_id, .wmi_init_config = ath11k_init_wmi_config_ipq8074, @@ -857,6 +881,7 @@ const struct ath11k_hw_ops ipq8074_ops = { .mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector, }; const struct ath11k_hw_ops ipq6018_ops = { @@ -896,6 +921,7 @@ const struct ath11k_hw_ops ipq6018_ops = { .mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector, }; const struct ath11k_hw_ops qca6390_ops = { @@ -935,6 +961,7 @@ const struct ath11k_hw_ops qca6390_ops = { .mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector, }; const struct ath11k_hw_ops qcn9074_ops = { @@ -974,6 +1001,7 @@ const struct ath11k_hw_ops qcn9074_ops = { .mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_ipq9074_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_ipq9074_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector, }; const struct ath11k_hw_ops wcn6855_ops = { @@ -1013,6 +1041,7 @@ const struct ath11k_hw_ops wcn6855_ops = { .mpdu_info_get_peerid = ath11k_hw_wcn6855_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_wcn6855_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_wcn6855_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector, }; const struct ath11k_hw_ops wcn6750_ops = { @@ -1052,11 +1081,14 @@ const struct ath11k_hw_ops wcn6750_ops = { .mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid, .rx_desc_mac_addr2_valid = ath11k_hw_ipq9074_rx_desc_mac_addr2_valid, .rx_desc_mpdu_start_addr2 = ath11k_hw_ipq9074_rx_desc_mpdu_start_addr2, + .get_ring_selector = ath11k_hw_wcn6750_get_tcl_ring_selector, }; -#define ATH11K_TX_RING_MASK_0 0x1 -#define ATH11K_TX_RING_MASK_1 0x2 -#define ATH11K_TX_RING_MASK_2 0x4 +#define ATH11K_TX_RING_MASK_0 BIT(0) +#define ATH11K_TX_RING_MASK_1 BIT(1) +#define ATH11K_TX_RING_MASK_2 BIT(2) +#define ATH11K_TX_RING_MASK_3 BIT(3) +#define ATH11K_TX_RING_MASK_4 BIT(4) #define ATH11K_RX_RING_MASK_0 0x1 #define ATH11K_RX_RING_MASK_1 0x2 @@ -1903,6 +1935,43 @@ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qcn9074 = { }, }; +const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_wcn6750 = { + .tx = { + ATH11K_TX_RING_MASK_0, + 0, + ATH11K_TX_RING_MASK_2, + 0, + ATH11K_TX_RING_MASK_4, + }, + .rx_mon_status = { + 0, 0, 0, 0, 0, 0, + ATH11K_RX_MON_STATUS_RING_MASK_0, + }, + .rx = { + 0, 0, 0, 0, 0, 0, 0, + ATH11K_RX_RING_MASK_0, + ATH11K_RX_RING_MASK_1, + ATH11K_RX_RING_MASK_2, + ATH11K_RX_RING_MASK_3, + }, + .rx_err = { + 0, ATH11K_RX_ERR_RING_MASK_0, + }, + .rx_wbm_rel = { + 0, ATH11K_RX_WBM_REL_RING_MASK_0, + }, + .reo_status = { + 0, ATH11K_REO_STATUS_RING_MASK_0, + }, + .rxdma2host = { + ATH11K_RXDMA2HOST_RING_MASK_0, + ATH11K_RXDMA2HOST_RING_MASK_1, + ATH11K_RXDMA2HOST_RING_MASK_2, + }, + .host2rxdma = { + }, +}; + const struct ath11k_hw_regs ipq8074_regs = { /* SW2TCL(x) R0 ring configuration address */ .hal_tcl1_ring_base_lsb = 0x00000510, @@ -2332,12 +2401,55 @@ const struct ath11k_hw_regs wcn6750_regs = { .hal_reo1_misc_ctl = 0x000005d8, }; +static const struct ath11k_hw_tcl2wbm_rbm_map ath11k_hw_tcl2wbm_rbm_map_ipq8074[] = { + { + .tcl_ring_num = 0, + .wbm_ring_num = 0, + .rbm_id = HAL_RX_BUF_RBM_SW0_BM, + }, + { + .tcl_ring_num = 1, + .wbm_ring_num = 1, + .rbm_id = HAL_RX_BUF_RBM_SW1_BM, + }, + { + .tcl_ring_num = 2, + .wbm_ring_num = 2, + .rbm_id = HAL_RX_BUF_RBM_SW2_BM, + }, +}; + +static const struct ath11k_hw_tcl2wbm_rbm_map ath11k_hw_tcl2wbm_rbm_map_wcn6750[] = { + { + .tcl_ring_num = 0, + .wbm_ring_num = 0, + .rbm_id = HAL_RX_BUF_RBM_SW0_BM, + }, + { + .tcl_ring_num = 1, + .wbm_ring_num = 4, + .rbm_id = HAL_RX_BUF_RBM_SW4_BM, + }, + { + .tcl_ring_num = 2, + .wbm_ring_num = 2, + .rbm_id = HAL_RX_BUF_RBM_SW2_BM, + }, +}; + const struct ath11k_hw_hal_params ath11k_hw_hal_params_ipq8074 = { .rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM, + .tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_ipq8074, }; const struct ath11k_hw_hal_params ath11k_hw_hal_params_qca6390 = { .rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM, + .tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_ipq8074, +}; + +const struct ath11k_hw_hal_params ath11k_hw_hal_params_wcn6750 = { + .rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM, + .tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_wcn6750, }; static const struct cfg80211_sar_freq_ranges ath11k_hw_sar_freq_ranges_wcn6855[] = { diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h index bb5ac940e4709194889c87a1c6d65d139660fd55..8a3f24862edc46936a7861f488728f6aaeeb4cef 100644 --- a/drivers/net/wireless/ath/ath11k/hw.h +++ b/drivers/net/wireless/ath/ath11k/hw.h @@ -122,8 +122,15 @@ struct ath11k_hw_ring_mask { u8 host2rxdma[ATH11K_EXT_IRQ_GRP_NUM_MAX]; }; +struct ath11k_hw_tcl2wbm_rbm_map { + u8 tcl_ring_num; + u8 wbm_ring_num; + u8 rbm_id; +}; + struct ath11k_hw_hal_params { enum hal_rx_buf_return_buf_manager rx_buf_rbm; + const struct ath11k_hw_tcl2wbm_rbm_map *tcl2wbm_rbm_map; }; struct ath11k_hw_params { @@ -166,6 +173,7 @@ struct ath11k_hw_params { u8 summary_pad_sz; u8 fft_hdr_len; u16 max_fft_bins; + bool fragment_160mhz; } spectral; u16 interface_modes; @@ -175,6 +183,7 @@ struct ath11k_hw_params { bool idle_ps; bool supports_sta_ps; bool cold_boot_calib; + bool cbcal_restart_fw; int fw_mem_mode; u32 num_vdevs; u32 num_peers; @@ -200,6 +209,16 @@ struct ath11k_hw_params { bool hybrid_bus_type; bool fixed_fw_mem; bool support_off_channel_tx; + bool supports_multi_bssid; + + struct { + u32 start; + u32 end; + } sram_dump; + + bool tcl_ring_retry; + u32 tx_ring_size; + bool smp2p_wow_exit; }; struct ath11k_hw_ops { @@ -242,6 +261,7 @@ struct ath11k_hw_ops { u16 (*mpdu_info_get_peerid)(u8 *tlv_data); bool (*rx_desc_mac_addr2_valid)(struct hal_rx_desc *desc); u8* (*rx_desc_mpdu_start_addr2)(struct hal_rx_desc *desc); + u32 (*get_ring_selector)(struct sk_buff *skb); }; extern const struct ath11k_hw_ops ipq8074_ops; @@ -254,9 +274,11 @@ extern const struct ath11k_hw_ops wcn6750_ops; extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_ipq8074; extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qca6390; extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qcn9074; +extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_wcn6750; extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_ipq8074; extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_qca6390; +extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_wcn6750; static inline int ath11k_hw_get_mac_from_pdev_id(struct ath11k_hw_params *hw, @@ -397,4 +419,5 @@ static inline const char *ath11k_bd_ie_type_str(enum ath11k_bd_ie_type type) } extern const struct cfg80211_sar_capa ath11k_hw_sar_capa_wcn6855; + #endif diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 7e91e347c9ff2d5c13a0e1610e65c8aae483d786..2d1e3fd9b526c16c190bcc02af249261e624aece 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -2081,7 +2081,7 @@ static void ath11k_peer_assoc_h_he(struct ath11k *ar, struct cfg80211_chan_def def; const struct ieee80211_sta_he_cap *he_cap = &sta->deflink.he_cap; enum nl80211_band band; - u16 *he_mcs_mask; + u16 he_mcs_mask[NL80211_HE_NSS_MAX]; u8 max_nss, he_mcs; u16 he_tx_mcs = 0, v = 0; int i, he_nss, nss_idx; @@ -2098,7 +2098,8 @@ static void ath11k_peer_assoc_h_he(struct ath11k *ar, return; band = def.chan->band; - he_mcs_mask = arvif->bitrate_mask.control[band].he_mcs; + memcpy(he_mcs_mask, arvif->bitrate_mask.control[band].he_mcs, + sizeof(he_mcs_mask)); if (ath11k_peer_assoc_h_he_masked(he_mcs_mask)) return; @@ -3059,7 +3060,7 @@ static int ath11k_mac_config_obss_pd(struct ath11k *ar, return ret; } - /* Enable all patial BSSID mask for SRG */ + /* Enable all partial BSSID mask for SRG */ ret = ath11k_wmi_pdev_srg_obss_bssid_enable_bitmap(ar, bitmap); if (ret) { ath11k_warn(ar->ab, @@ -3077,7 +3078,7 @@ static int ath11k_mac_config_obss_pd(struct ath11k *ar, return ret; } - /* Enable all patial BSSID mask for non-SRG */ + /* Enable all partial BSSID mask for non-SRG */ ret = ath11k_wmi_pdev_non_srg_obss_bssid_enable_bitmap(ar, bitmap); if (ret) { ath11k_warn(ar->ab, @@ -3350,10 +3351,15 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw, ath11k_recalculate_mgmt_rate(ar, vif, &def); if (changed & BSS_CHANGED_TWT) { - if (info->twt_requester || info->twt_responder) - ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id); - else + struct wmi_twt_enable_params twt_params = {0}; + + if (info->twt_requester || info->twt_responder) { + ath11k_wmi_fill_default_twt_params(&twt_params); + ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, + &twt_params); + } else { ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id); + } } if (changed & BSS_CHANGED_HE_OBSS_PD) @@ -3451,7 +3457,7 @@ void __ath11k_mac_scan_finish(struct ath11k *ar) ar->scan_channel = NULL; ar->scan.roc_freq = 0; cancel_delayed_work(&ar->scan.timeout); - complete(&ar->scan.completed); + complete_all(&ar->scan.completed); break; } } @@ -4524,6 +4530,7 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, new_state == IEEE80211_STA_NONE) { memset(arsta, 0, sizeof(*arsta)); arsta->arvif = arvif; + arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED; INIT_WORK(&arsta->update_wk, ath11k_sta_rc_update_wk); INIT_WORK(&arsta->set_4addr_wk, ath11k_sta_set_4addr_wk); @@ -4701,7 +4708,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw, "mac sta rc update for %pM changed %08x bw %d nss %d smps %d\n", sta->addr, changed, sta->deflink.bandwidth, sta->deflink.rx_nss, - sta->smps_mode); + sta->deflink.smps_mode); spin_lock_bh(&ar->data_lock); @@ -4737,7 +4744,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw, if (changed & IEEE80211_RC_SMPS_CHANGED) { smps = WMI_PEER_SMPS_PS_NONE; - switch (sta->smps_mode) { + switch (sta->deflink.smps_mode) { case IEEE80211_SMPS_AUTOMATIC: case IEEE80211_SMPS_OFF: smps = WMI_PEER_SMPS_PS_NONE; @@ -4750,7 +4757,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw, break; default: ath11k_warn(ar->ab, "Invalid smps %d in sta rc update for %pM\n", - sta->smps_mode, sta->addr); + sta->deflink.smps_mode, sta->addr); smps = WMI_PEER_SMPS_PS_NONE; break; } @@ -4954,6 +4961,8 @@ static int ath11k_mac_set_txbf_conf(struct ath11k_vif *arvif) if (vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE)) { nsts = vht_cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK; nsts >>= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT; + if (nsts > (ar->num_rx_chains - 1)) + nsts = ar->num_rx_chains - 1; value |= SM(nsts, WMI_TXBF_STS_CAP_OFFSET); } @@ -4994,7 +5003,7 @@ static int ath11k_mac_set_txbf_conf(struct ath11k_vif *arvif) static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap) { bool subfer, subfee; - int sound_dim = 0; + int sound_dim = 0, nsts = 0; subfer = !!(*vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE)); subfee = !!(*vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE)); @@ -5004,6 +5013,11 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap) subfer = false; } + if (ar->num_rx_chains < 2) { + *vht_cap &= ~(IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE); + subfee = false; + } + /* If SU Beaformer is not set, then disable MU Beamformer Capability */ if (!subfer) *vht_cap &= ~(IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE); @@ -5016,7 +5030,9 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap) sound_dim >>= IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT; *vht_cap &= ~IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK; - /* TODO: Need to check invalid STS and Sound_dim values set by FW? */ + nsts = (*vht_cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK); + nsts >>= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT; + *vht_cap &= ~IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK; /* Enable Sounding Dimension Field only if SU BF is enabled */ if (subfer) { @@ -5028,9 +5044,15 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap) *vht_cap |= sound_dim; } - /* Use the STS advertised by FW unless SU Beamformee is not supported*/ - if (!subfee) - *vht_cap &= ~(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK); + /* Enable Beamformee STS Field only if SU BF is enabled */ + if (subfee) { + if (nsts > (ar->num_rx_chains - 1)) + nsts = ar->num_rx_chains - 1; + + nsts <<= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT; + nsts &= IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK; + *vht_cap |= nsts; + } } static struct ieee80211_sta_vht_cap @@ -6173,6 +6195,13 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw, goto err; } + /* In the case of hardware recovery, debugfs files are + * not deleted since ieee80211_ops.remove_interface() is + * not invoked. In such cases, try to delete the files. + * These will be re-created later. + */ + ath11k_debugfs_remove_interface(arvif); + memset(arvif, 0, sizeof(*arvif)); arvif->ar = ar; @@ -6354,9 +6383,7 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw, } } - ret = ath11k_debugfs_add_interface(arvif); - if (ret) - goto err_peer_del; + ath11k_debugfs_add_interface(arvif); mutex_unlock(&ar->conf_mutex); @@ -8421,6 +8448,95 @@ exit: return ret; } +static int ath11k_fw_stats_request(struct ath11k *ar, + struct stats_request_params *req_param) +{ + struct ath11k_base *ab = ar->ab; + unsigned long time_left; + int ret; + + lockdep_assert_held(&ar->conf_mutex); + + spin_lock_bh(&ar->data_lock); + ar->fw_stats_done = false; + ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs); + spin_unlock_bh(&ar->data_lock); + + reinit_completion(&ar->fw_stats_complete); + + ret = ath11k_wmi_send_stats_request_cmd(ar, req_param); + if (ret) { + ath11k_warn(ab, "could not request fw stats (%d)\n", + ret); + return ret; + } + + time_left = wait_for_completion_timeout(&ar->fw_stats_complete, + 1 * HZ); + + if (!time_left) + return -ETIMEDOUT; + + return 0; +} + +static int ath11k_mac_op_get_txpower(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + int *dbm) +{ + struct ath11k *ar = hw->priv; + struct ath11k_base *ab = ar->ab; + struct stats_request_params req_param = {0}; + struct ath11k_fw_stats_pdev *pdev; + int ret; + + /* Final Tx power is minimum of Target Power, CTL power, Regulatory + * Power, PSD EIRP Power. We just know the Regulatory power from the + * regulatory rules obtained. FW knows all these power and sets the min + * of these. Hence, we request the FW pdev stats in which FW reports + * the minimum of all vdev's channel Tx power. + */ + mutex_lock(&ar->conf_mutex); + + if (ar->state != ATH11K_STATE_ON) + goto err_fallback; + + req_param.pdev_id = ar->pdev->pdev_id; + req_param.stats_id = WMI_REQUEST_PDEV_STAT; + + ret = ath11k_fw_stats_request(ar, &req_param); + if (ret) { + ath11k_warn(ab, "failed to request fw pdev stats: %d\n", ret); + goto err_fallback; + } + + spin_lock_bh(&ar->data_lock); + pdev = list_first_entry_or_null(&ar->fw_stats.pdevs, + struct ath11k_fw_stats_pdev, list); + if (!pdev) { + spin_unlock_bh(&ar->data_lock); + goto err_fallback; + } + + /* tx power is set as 2 units per dBm in FW. */ + *dbm = pdev->chan_tx_power / 2; + + spin_unlock_bh(&ar->data_lock); + mutex_unlock(&ar->conf_mutex); + + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, "txpower from firmware %d, reported %d dBm\n", + pdev->chan_tx_power, *dbm); + return 0; + +err_fallback: + mutex_unlock(&ar->conf_mutex); + /* We didn't get txpower from FW. Hence, relying on vif->bss_conf.txpower */ + *dbm = vif->bss_conf.txpower; + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, "txpower from firmware NaN, reported %d dBm\n", + *dbm); + return 0; +} + static const struct ieee80211_ops ath11k_ops = { .tx = ath11k_mac_op_tx, .start = ath11k_mac_op_start, @@ -8471,6 +8587,7 @@ static const struct ieee80211_ops ath11k_ops = { #if IS_ENABLED(CONFIG_IPV6) .ipv6_addr_change = ath11k_mac_op_ipv6_changed, #endif + .get_txpower = ath11k_mac_op_get_txpower, .set_sar_specs = ath11k_mac_op_set_bios_sar_specs, .remain_on_channel = ath11k_mac_op_remain_on_channel, @@ -8777,6 +8894,11 @@ static int __ath11k_mac_register(struct ath11k *ar) if (ab->hw_params.single_pdev_only && ar->supports_6ghz) ieee80211_hw_set(ar->hw, SINGLE_SCAN_ON_ALL_BANDS); + if (ab->hw_params.supports_multi_bssid) { + ieee80211_hw_set(ar->hw, SUPPORTS_MULTI_BSSID); + ieee80211_hw_set(ar->hw, SUPPORTS_ONLY_HE_MULTI_BSSID); + } + ieee80211_hw_set(ar->hw, SIGNAL_DBM); ieee80211_hw_set(ar->hw, SUPPORTS_PS); ieee80211_hw_set(ar->hw, SUPPORTS_DYNAMIC_PS); @@ -8967,6 +9089,7 @@ int ath11k_mac_register(struct ath11k_base *ab) struct ath11k_pdev *pdev; int i; int ret; + u8 mac_addr[ETH_ALEN] = {0}; if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) return 0; @@ -8979,13 +9102,18 @@ int ath11k_mac_register(struct ath11k_base *ab) if (ret) return ret; + device_get_mac_address(ab->dev, mac_addr); + for (i = 0; i < ab->num_radios; i++) { pdev = &ab->pdevs[i]; ar = pdev->ar; if (ab->pdevs_macaddr_valid) { ether_addr_copy(ar->mac_addr, pdev->mac_addr); } else { - ether_addr_copy(ar->mac_addr, ab->mac_addr); + if (is_zero_ether_addr(mac_addr)) + ether_addr_copy(ar->mac_addr, ab->mac_addr); + else + ether_addr_copy(ar->mac_addr, mac_addr); ar->mac_addr[4] += i; } @@ -9079,6 +9207,8 @@ int ath11k_mac_allocate(struct ath11k_base *ab) clear_bit(ATH11K_FLAG_MONITOR_VDEV_CREATED, &ar->monitor_flags); ar->vdev_id_11d_scan = ATH11K_11D_INVALID_VDEV_ID; init_completion(&ar->completed_11d_scan); + + ath11k_fw_stats_init(ar); } return 0; diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index c44df17719f64aafd37512020b6af5a5c575c917..86995e8dc9135162233959920827b615534845df 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -402,8 +402,7 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci) ret = ath11k_mhi_get_msi(ab_pci); if (ret) { ath11k_err(ab, "failed to get msi for mhi\n"); - mhi_free_controller(mhi_ctrl); - return ret; + goto free_controller; } if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags)) @@ -412,7 +411,7 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci) if (test_bit(ATH11K_FLAG_FIXED_MEM_RGN, &ab->dev_flags)) { ret = ath11k_mhi_read_addr_from_dt(mhi_ctrl); if (ret < 0) - return ret; + goto free_controller; } else { mhi_ctrl->iova_start = 0; mhi_ctrl->iova_stop = 0xFFFFFFFF; @@ -440,18 +439,22 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci) default: ath11k_err(ab, "failed assign mhi_config for unknown hw rev %d\n", ab->hw_rev); - mhi_free_controller(mhi_ctrl); - return -EINVAL; + ret = -EINVAL; + goto free_controller; } ret = mhi_register_controller(mhi_ctrl, ath11k_mhi_config); if (ret) { ath11k_err(ab, "failed to register to mhi bus, err = %d\n", ret); - mhi_free_controller(mhi_ctrl); - return ret; + goto free_controller; } return 0; + +free_controller: + mhi_free_controller(mhi_ctrl); + ab_pci->mhi_ctrl = NULL; + return ret; } void ath11k_mhi_unregister(struct ath11k_pci *ab_pci) diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index 5bd34a6273d99c2b5affc7bcc69c02cfe3e08053..99cf3357c66e16bbd91a0fb8e56c3f156a993383 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -685,6 +685,7 @@ static const struct ath11k_hif_ops ath11k_pci_hif_ops = { .stop = ath11k_pcic_stop, .read32 = ath11k_pcic_read32, .write32 = ath11k_pcic_write32, + .read = ath11k_pcic_read, .power_down = ath11k_pci_power_down, .power_up = ath11k_pci_power_up, .suspend = ath11k_pci_hif_suspend, diff --git a/drivers/net/wireless/ath/ath11k/pcic.c b/drivers/net/wireless/ath/ath11k/pcic.c index 1adf20ebef27ce8f58fc8a50588199f5ab6ae8cf..380f9d37b6449f0dd76fc3668c6d051605911851 100644 --- a/drivers/net/wireless/ath/ath11k/pcic.c +++ b/drivers/net/wireless/ath/ath11k/pcic.c @@ -140,55 +140,100 @@ int ath11k_pcic_init_msi_config(struct ath11k_base *ab) } EXPORT_SYMBOL(ath11k_pcic_init_msi_config); +static void __ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value) +{ + if (offset < ATH11K_PCI_WINDOW_START) + iowrite32(value, ab->mem + offset); + else + ab->pci.ops->window_write32(ab, offset, value); +} + void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value) { int ret = 0; + bool wakeup_required; /* for offset beyond BAR + 4K - 32, may * need to wakeup the device to access. */ - if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && - offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup) + wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && + offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF; + if (wakeup_required && ab->pci.ops->wakeup) ret = ab->pci.ops->wakeup(ab); - if (offset < ATH11K_PCI_WINDOW_START) - iowrite32(value, ab->mem + offset); - else - ab->pci.ops->window_write32(ab, offset, value); + __ath11k_pcic_write32(ab, offset, value); - if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && - offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release && - !ret) + if (wakeup_required && !ret && ab->pci.ops->release) ab->pci.ops->release(ab); } EXPORT_SYMBOL(ath11k_pcic_write32); +static u32 __ath11k_pcic_read32(struct ath11k_base *ab, u32 offset) +{ + u32 val; + + if (offset < ATH11K_PCI_WINDOW_START) + val = ioread32(ab->mem + offset); + else + val = ab->pci.ops->window_read32(ab, offset); + + return val; +} + u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset) { int ret = 0; u32 val; + bool wakeup_required; /* for offset beyond BAR + 4K - 32, may * need to wakeup the device to access. */ - if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && - offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup) + wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && + offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF; + if (wakeup_required && ab->pci.ops->wakeup) ret = ab->pci.ops->wakeup(ab); - if (offset < ATH11K_PCI_WINDOW_START) - val = ioread32(ab->mem + offset); - else - val = ab->pci.ops->window_read32(ab, offset); + val = __ath11k_pcic_read32(ab, offset); - if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && - offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release && - !ret) + if (wakeup_required && !ret && ab->pci.ops->release) ab->pci.ops->release(ab); return val; } EXPORT_SYMBOL(ath11k_pcic_read32); +int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end) +{ + int ret = 0; + bool wakeup_required; + u32 *data = buf; + u32 i; + + /* for offset beyond BAR + 4K - 32, may + * need to wakeup the device to access. + */ + wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) && + end >= ATH11K_PCI_ACCESS_ALWAYS_OFF; + if (wakeup_required && ab->pci.ops->wakeup) { + ret = ab->pci.ops->wakeup(ab); + if (ret) { + ath11k_warn(ab, "failed to wakeup for read from 0x%x: %d\n", + start, ret); + return ret; + } + } + + for (i = start; i < end + 1; i += 4) + *data++ = __ath11k_pcic_read32(ab, i); + + if (wakeup_required && ab->pci.ops->release) + ab->pci.ops->release(ab); + + return 0; +} +EXPORT_SYMBOL(ath11k_pcic_read); + void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo, u32 *msi_addr_hi) { @@ -414,6 +459,7 @@ void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab) struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i]; if (!irq_grp->napi_enabled) { + dev_set_threaded(&irq_grp->napi_ndev, true); napi_enable(&irq_grp->napi); irq_grp->napi_enabled = true; } @@ -517,7 +563,7 @@ static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab) irq_grp->grp_id = i; init_dummy_netdev(&irq_grp->napi_ndev); netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi, - ath11k_pcic_ext_grp_napi_poll, NAPI_POLL_WEIGHT); + ath11k_pcic_ext_grp_napi_poll); if (ab->hw_params.ring_mask->tx[i] || ab->hw_params.ring_mask->rx[i] || @@ -731,3 +777,37 @@ int ath11k_pcic_register_pci_ops(struct ath11k_base *ab, return 0; } EXPORT_SYMBOL(ath11k_pcic_register_pci_ops); + +void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab) +{ + int i; + + for (i = 0; i < ab->hw_params.ce_count; i++) { + if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR || + i == ATH11K_PCI_CE_WAKE_IRQ) + continue; + ath11k_pcic_ce_irq_enable(ab, i); + } +} +EXPORT_SYMBOL(ath11k_pci_enable_ce_irqs_except_wake_irq); + +void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab) +{ + int i; + int irq_idx; + struct ath11k_ce_pipe *ce_pipe; + + for (i = 0; i < ab->hw_params.ce_count; i++) { + ce_pipe = &ab->ce.ce_pipe[i]; + irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i; + + if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR || + i == ATH11K_PCI_CE_WAKE_IRQ) + continue; + + disable_irq_nosync(ab->irq_num[irq_idx]); + synchronize_irq(ab->irq_num[irq_idx]); + tasklet_kill(&ce_pipe->intr_tq); + } +} +EXPORT_SYMBOL(ath11k_pci_disable_ce_irqs_except_wake_irq); diff --git a/drivers/net/wireless/ath/ath11k/pcic.h b/drivers/net/wireless/ath/ath11k/pcic.h index 0afbb34510dbb1cfc43027ab9a97ff93c1262cc2..ac012e88bf6d4f65be3116196f010719787135eb 100644 --- a/drivers/net/wireless/ath/ath11k/pcic.h +++ b/drivers/net/wireless/ath/ath11k/pcic.h @@ -12,6 +12,8 @@ #define ATH11K_PCI_IRQ_CE0_OFFSET 3 #define ATH11K_PCI_IRQ_DP_OFFSET 14 +#define ATH11K_PCI_CE_WAKE_IRQ 2 + #define ATH11K_PCI_WINDOW_ENABLE_BIT 0x40000000 #define ATH11K_PCI_WINDOW_REG_ADDRESS 0x310c #define ATH11K_PCI_WINDOW_VALUE_MASK GENMASK(24, 19) @@ -45,4 +47,8 @@ void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab); int ath11k_pcic_init_msi_config(struct ath11k_base *ab); int ath11k_pcic_register_pci_ops(struct ath11k_base *ab, const struct ath11k_pci_ops *pci_ops); +int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end); +void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab); +void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab); + #endif diff --git a/drivers/net/wireless/ath/ath11k/peer.c b/drivers/net/wireless/ath/ath11k/peer.c index 9e22aaf34b88c151b940a097539c21994c673903..1ae7af02c364ec0559a30adf082f470ca4e6223b 100644 --- a/drivers/net/wireless/ath/ath11k/peer.c +++ b/drivers/net/wireless/ath/ath11k/peer.c @@ -302,6 +302,21 @@ static int __ath11k_peer_delete(struct ath11k *ar, u32 vdev_id, const u8 *addr) spin_lock_bh(&ab->base_lock); peer = ath11k_peer_find_by_addr(ab, addr); + /* Check if the found peer is what we want to remove. + * While the sta is transitioning to another band we may + * have 2 peer with the same addr assigned to different + * vdev_id. Make sure we are deleting the correct peer. + */ + if (peer && peer->vdev_id == vdev_id) + ath11k_peer_rhash_delete(ab, peer); + + /* Fallback to peer list search if the correct peer can't be found. + * Skip the deletion of the peer from the rhash since it has already + * been deleted in peer add. + */ + if (!peer) + peer = ath11k_peer_find(ab, vdev_id, addr); + if (!peer) { spin_unlock_bh(&ab->base_lock); mutex_unlock(&ab->tbl_mtx_lock); @@ -312,8 +327,6 @@ static int __ath11k_peer_delete(struct ath11k *ar, u32 vdev_id, const u8 *addr) return -EINVAL; } - ath11k_peer_rhash_delete(ab, peer); - spin_unlock_bh(&ab->base_lock); mutex_unlock(&ab->tbl_mtx_lock); @@ -372,8 +385,17 @@ int ath11k_peer_create(struct ath11k *ar, struct ath11k_vif *arvif, spin_lock_bh(&ar->ab->base_lock); peer = ath11k_peer_find_by_addr(ar->ab, param->peer_addr); if (peer) { - spin_unlock_bh(&ar->ab->base_lock); - return -EINVAL; + if (peer->vdev_id == param->vdev_id) { + spin_unlock_bh(&ar->ab->base_lock); + return -EINVAL; + } + + /* Assume sta is transitioning to another band. + * Remove here the peer from rhash. + */ + mutex_lock(&ar->ab->tbl_mtx_lock); + ath11k_peer_rhash_delete(ar->ab, peer); + mutex_unlock(&ar->ab->tbl_mtx_lock); } spin_unlock_bh(&ar->ab->base_lock); diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index 00136601cb7dbe88d6a7153de181bbb1bc71a2fa..51de2208b7899cf6eab025299498113ba221e269 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -1696,6 +1696,13 @@ static struct qmi_elem_info qmi_wlanfw_wlan_ini_resp_msg_v01_ei[] = { }, }; +static struct qmi_elem_info qmi_wlfw_fw_init_done_ind_msg_v01_ei[] = { + { + .data_type = QMI_EOTI, + .array_type = NO_ARRAY, + }, +}; + static int ath11k_qmi_host_cap_send(struct ath11k_base *ab) { struct qmi_wlanfw_host_cap_req_msg_v01 req; @@ -1872,7 +1879,7 @@ static int ath11k_qmi_respond_fw_mem_request(struct ath11k_base *ab) /* For QCA6390 by default FW requests a block of ~4M contiguous * DMA memory, it's hard to allocate from OS. So host returns - * failure to FW and FW will then request mulitple blocks of small + * failure to FW and FW will then request multiple blocks of small * chunk size memory. */ if (!(ab->hw_params.fixed_mem_region || @@ -3006,6 +3013,12 @@ static void ath11k_qmi_msg_fw_ready_cb(struct qmi_handle *qmi_hdl, struct ath11k_base *ab = qmi->ab; ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi firmware ready\n"); + + if (!ab->qmi.cal_done) { + ab->qmi.cal_done = 1; + wake_up(&ab->qmi.cold_boot_waitq); + } + ath11k_qmi_driver_event_post(qmi, ATH11K_QMI_EVENT_FW_READY, NULL); } @@ -3023,6 +3036,19 @@ static void ath11k_qmi_msg_cold_boot_cal_done_cb(struct qmi_handle *qmi_hdl, ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi cold boot calibration done\n"); } +static void ath11k_qmi_msg_fw_init_done_cb(struct qmi_handle *qmi_hdl, + struct sockaddr_qrtr *sq, + struct qmi_txn *txn, + const void *decoded) +{ + struct ath11k_qmi *qmi = container_of(qmi_hdl, + struct ath11k_qmi, handle); + struct ath11k_base *ab = qmi->ab; + + ath11k_qmi_driver_event_post(qmi, ATH11K_QMI_EVENT_FW_INIT_DONE, NULL); + ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi firmware init done\n"); +} + static const struct qmi_msg_handler ath11k_qmi_msg_handlers[] = { { .type = QMI_INDICATION, @@ -3053,6 +3079,14 @@ static const struct qmi_msg_handler ath11k_qmi_msg_handlers[] = { sizeof(struct qmi_wlanfw_fw_cold_cal_done_ind_msg_v01), .fn = ath11k_qmi_msg_cold_boot_cal_done_cb, }, + { + .type = QMI_INDICATION, + .msg_id = QMI_WLFW_FW_INIT_DONE_IND_V01, + .ei = qmi_wlfw_fw_init_done_ind_msg_v01_ei, + .decoded_size = + sizeof(struct qmi_wlfw_fw_init_done_ind_msg_v01), + .fn = ath11k_qmi_msg_fw_init_done_cb, + }, }; static int ath11k_qmi_ops_new_server(struct qmi_handle *qmi_hdl, @@ -3145,7 +3179,7 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work) } break; - case ATH11K_QMI_EVENT_FW_READY: + case ATH11K_QMI_EVENT_FW_INIT_DONE: clear_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags); if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) { ath11k_hal_dump_srng_stats(ab); @@ -3168,6 +3202,22 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work) set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags); } + break; + case ATH11K_QMI_EVENT_FW_READY: + /* For targets requiring a FW restart upon cold + * boot completion, there is no need to process + * FW ready; such targets will receive FW init + * done message after FW restart. + */ + if (ab->hw_params.cbcal_restart_fw) + break; + + clear_bit(ATH11K_FLAG_CRASH_FLUSH, + &ab->dev_flags); + clear_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags); + ath11k_core_qmi_firmware_ready(ab); + set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags); + break; case ATH11K_QMI_EVENT_COLD_BOOT_CAL_DONE: break; diff --git a/drivers/net/wireless/ath/ath11k/qmi.h b/drivers/net/wireless/ath/ath11k/qmi.h index c83cf822be81a144da44aa7ccd34ed4cd18db6bc..2ec56a34fa810970469f299491ae39a32af9b747 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.h +++ b/drivers/net/wireless/ath/ath11k/qmi.h @@ -31,8 +31,9 @@ #define QMI_WLFW_REQUEST_MEM_IND_V01 0x0035 #define QMI_WLFW_FW_MEM_READY_IND_V01 0x0037 -#define QMI_WLFW_COLD_BOOT_CAL_DONE_IND_V01 0x0021 -#define QMI_WLFW_FW_READY_IND_V01 0x0038 +#define QMI_WLFW_COLD_BOOT_CAL_DONE_IND_V01 0x003E +#define QMI_WLFW_FW_READY_IND_V01 0x0021 +#define QMI_WLFW_FW_INIT_DONE_IND_V01 0x0038 #define QMI_WLANFW_MAX_DATA_SIZE_V01 6144 #define ATH11K_FIRMWARE_MODE_OFF 4 @@ -69,6 +70,7 @@ enum ath11k_qmi_event_type { ATH11K_QMI_EVENT_FORCE_FW_ASSERT, ATH11K_QMI_EVENT_POWER_UP, ATH11K_QMI_EVENT_POWER_DOWN, + ATH11K_QMI_EVENT_FW_INIT_DONE, ATH11K_QMI_EVENT_MAX, }; @@ -291,6 +293,10 @@ struct qmi_wlanfw_fw_cold_cal_done_ind_msg_v01 { char placeholder; }; +struct qmi_wlfw_fw_init_done_ind_msg_v01 { + char placeholder; +}; + #define QMI_WLANFW_CAP_REQ_MSG_V01_MAX_LEN 0 #define QMI_WLANFW_CAP_RESP_MSG_V01_MAX_LEN 235 #define QMI_WLANFW_CAP_REQ_V01 0x0024 diff --git a/drivers/net/wireless/ath/ath11k/rx_desc.h b/drivers/net/wireless/ath/ath11k/rx_desc.h index 26ecc1bcd9d5735ab30fb965c43d29adc0d307f8..786d5f36f5e54783518a76bd0721a075a6049328 100644 --- a/drivers/net/wireless/ath/ath11k/rx_desc.h +++ b/drivers/net/wireless/ath/ath11k/rx_desc.h @@ -877,7 +877,7 @@ struct rx_msdu_start_wcn6855 { * * l4_offset * Depending upon mode bit, this field either indicates the - * L4 offset nin bytes from the start of RX_HEADER (only valid + * L4 offset in bytes from the start of RX_HEADER (only valid * if either ipv4_proto or ipv6_proto is set to 1) or indicates * the offset in bytes to the start of TCP or UDP header from * the start of the IP header after decapsulation (Only valid if diff --git a/drivers/net/wireless/ath/ath11k/spectral.c b/drivers/net/wireless/ath/ath11k/spectral.c index 516a7b4cd180577ea9cf153d0088505e78082595..705868198df4bc7d7b2e5a15470c4b341c52fd42 100644 --- a/drivers/net/wireless/ath/ath11k/spectral.c +++ b/drivers/net/wireless/ath/ath11k/spectral.c @@ -30,6 +30,7 @@ #define ATH11K_SPECTRAL_20MHZ 20 #define ATH11K_SPECTRAL_40MHZ 40 #define ATH11K_SPECTRAL_80MHZ 80 +#define ATH11K_SPECTRAL_160MHZ 160 #define ATH11K_SPECTRAL_SIGNATURE 0xFA @@ -183,6 +184,8 @@ static int ath11k_spectral_scan_trigger(struct ath11k *ar) if (ar->spectral.mode == ATH11K_SPECTRAL_DISABLED) return 0; + ar->spectral.is_primary = true; + ret = ath11k_wmi_vdev_spectral_enable(ar, arvif->vdev_id, ATH11K_WMI_SPECTRAL_TRIGGER_CMD_CLEAR, ATH11K_WMI_SPECTRAL_ENABLE_CMD_ENABLE); @@ -585,6 +588,7 @@ int ath11k_spectral_process_fft(struct ath11k *ar, u8 chan_width_mhz, bin_sz; int ret; u32 check_length; + bool fragment_sample = false; lockdep_assert_held(&ar->spectral.lock); @@ -639,6 +643,13 @@ int ath11k_spectral_process_fft(struct ath11k *ar, case ATH11K_SPECTRAL_80MHZ: fft_sample->chan_width_mhz = chan_width_mhz; break; + case ATH11K_SPECTRAL_160MHZ: + if (ab->hw_params.spectral.fragment_160mhz) { + chan_width_mhz /= 2; + fragment_sample = true; + } + fft_sample->chan_width_mhz = chan_width_mhz; + break; default: ath11k_warn(ab, "invalid channel width %d\n", chan_width_mhz); return -EINVAL; @@ -663,6 +674,17 @@ int ath11k_spectral_process_fft(struct ath11k *ar, freq = summary->meta.freq2; fft_sample->freq2 = __cpu_to_be16(freq); + /* If freq2 is available then the spectral scan results are fragmented + * as primary and secondary + */ + if (fragment_sample && freq) { + if (!ar->spectral.is_primary) + fft_sample->freq1 = cpu_to_be16(freq); + + /* We have to toggle the is_primary to handle the next report */ + ar->spectral.is_primary = !ar->spectral.is_primary; + } + ath11k_spectral_parse_fft(fft_sample->data, fft_report->bins, num_bins, ab->hw_params.spectral.fft_sz); diff --git a/drivers/net/wireless/ath/ath11k/spectral.h b/drivers/net/wireless/ath/ath11k/spectral.h index 081744265f2ab05c8a38c32977968ca0d22295b1..96bfa16e18e96d6160cb2aa393c2da98d7bb9812 100644 --- a/drivers/net/wireless/ath/ath11k/spectral.h +++ b/drivers/net/wireless/ath/ath11k/spectral.h @@ -35,6 +35,7 @@ struct ath11k_spectral { u16 count; u8 fft_size; bool enabled; + bool is_primary; }; #ifdef CONFIG_ATH11K_SPECTRAL diff --git a/drivers/net/wireless/ath/ath11k/thermal.c b/drivers/net/wireless/ath/ath11k/thermal.c index c96b26f39a2569399a81b0a286c8d12509a1bce9..23ed01bd44f9aab8b5a84d4c40d31188b8818fd2 100644 --- a/drivers/net/wireless/ath/ath11k/thermal.c +++ b/drivers/net/wireless/ath/ath11k/thermal.c @@ -99,7 +99,7 @@ static ssize_t ath11k_thermal_show_temp(struct device *dev, temperature = ar->thermal.temperature; spin_unlock_bh(&ar->data_lock); - /* display in millidegree celcius */ + /* display in millidegree Celsius */ ret = snprintf(buf, PAGE_SIZE, "%d\n", temperature * 1000); out: mutex_unlock(&ar->conf_mutex); diff --git a/drivers/net/wireless/ath/ath11k/thermal.h b/drivers/net/wireless/ath/ath11k/thermal.h index f9af55f3682def093b461eb33530bda4719d3f3f..3e39675ef7f5772dd191b4f9d0e6ee2f673f4cc3 100644 --- a/drivers/net/wireless/ath/ath11k/thermal.h +++ b/drivers/net/wireless/ath/ath11k/thermal.h @@ -19,7 +19,7 @@ struct ath11k_thermal { /* protected by conf_mutex */ u32 throttle_state; - /* temperature value in Celcius degree + /* temperature value in Celsius degree * protected by data_lock */ int temperature; diff --git a/drivers/net/wireless/ath/ath11k/trace.h b/drivers/net/wireless/ath/ath11k/trace.h index 76560587bea06c2ca68660286fdd073cf3d46509..9535745fe026c259c328ec28182ff155dc23ccd8 100644 --- a/drivers/net/wireless/ath/ath11k/trace.h +++ b/drivers/net/wireless/ath/ath11k/trace.h @@ -305,6 +305,34 @@ TRACE_EVENT(ath11k_wmi_diag, ) ); +TRACE_EVENT(ath11k_ps_timekeeper, + TP_PROTO(struct ath11k *ar, const void *peer_addr, + u32 peer_ps_timestamp, u8 peer_ps_state), + TP_ARGS(ar, peer_addr, peer_ps_timestamp, peer_ps_state), + + TP_STRUCT__entry(__string(device, dev_name(ar->ab->dev)) + __string(driver, dev_driver_string(ar->ab->dev)) + __dynamic_array(u8, peer_addr, ETH_ALEN) + __field(u8, peer_ps_state) + __field(u32, peer_ps_timestamp) + ), + + TP_fast_assign(__assign_str(device, dev_name(ar->ab->dev)); + __assign_str(driver, dev_driver_string(ar->ab->dev)); + memcpy(__get_dynamic_array(peer_addr), peer_addr, + ETH_ALEN); + __entry->peer_ps_state = peer_ps_state; + __entry->peer_ps_timestamp = peer_ps_timestamp; + ), + + TP_printk("%s %s %u %u", + __get_str(driver), + __get_str(device), + __entry->peer_ps_state, + __entry->peer_ps_timestamp + ) +); + #endif /* _TRACE_H_ || TRACE_HEADER_MULTI_READ*/ /* we don't want to use include/trace/events */ diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index 88ee4f9d19da589c12d970b10b3b1e4c942ca5af..fad9f8d308a207ab35a402af24deb61fd66bb2df 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -416,7 +416,7 @@ ath11k_pull_mac_phy_cap_svc_ready_ext(struct ath11k_pdev_wmi *wmi_handle, /* tx/rx chainmask reported from fw depends on the actual hw chains used, * For example, for 4x4 capable macphys, first 4 chains can be used for first - * mac and the remaing 4 chains can be used for the second mac or vice-versa. + * mac and the remaining 4 chains can be used for the second mac or vice-versa. * In this case, tx/rx chainmask 0xf will be advertised for first mac and 0xf0 * will be advertised for second mac or vice-versa. Compute the shift value * for tx/rx chainmask which will be used to advertise supported ht/vht rates to @@ -991,9 +991,13 @@ int ath11k_wmi_vdev_up(struct ath11k *ar, u32 vdev_id, u32 aid, const u8 *bssid) { struct ath11k_pdev_wmi *wmi = ar->wmi; struct wmi_vdev_up_cmd *cmd; + struct ieee80211_bss_conf *bss_conf; + struct ath11k_vif *arvif; struct sk_buff *skb; int ret; + arvif = ath11k_mac_get_arvif(ar, vdev_id); + skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, sizeof(*cmd)); if (!skb) return -ENOMEM; @@ -1007,6 +1011,17 @@ int ath11k_wmi_vdev_up(struct ath11k *ar, u32 vdev_id, u32 aid, const u8 *bssid) ether_addr_copy(cmd->vdev_bssid.addr, bssid); + if (arvif && arvif->vif->type == NL80211_IFTYPE_STATION) { + bss_conf = &arvif->vif->bss_conf; + + if (bss_conf->nontransmitted) { + ether_addr_copy(cmd->trans_bssid.addr, + bss_conf->transmitter_bssid); + cmd->profile_idx = bss_conf->bssid_index; + cmd->profile_num = bss_conf->bssid_indicator; + } + } + ret = ath11k_wmi_cmd_send(wmi, skb, WMI_VDEV_UP_CMDID); if (ret) { ath11k_warn(ar->ab, "failed to submit WMI_VDEV_UP cmd\n"); @@ -3064,8 +3079,34 @@ int ath11k_wmi_pdev_pktlog_disable(struct ath11k *ar) return ret; } -int -ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id) +void ath11k_wmi_fill_default_twt_params(struct wmi_twt_enable_params *twt_params) +{ + twt_params->sta_cong_timer_ms = ATH11K_TWT_DEF_STA_CONG_TIMER_MS; + twt_params->default_slot_size = ATH11K_TWT_DEF_DEFAULT_SLOT_SIZE; + twt_params->congestion_thresh_setup = ATH11K_TWT_DEF_CONGESTION_THRESH_SETUP; + twt_params->congestion_thresh_teardown = + ATH11K_TWT_DEF_CONGESTION_THRESH_TEARDOWN; + twt_params->congestion_thresh_critical = + ATH11K_TWT_DEF_CONGESTION_THRESH_CRITICAL; + twt_params->interference_thresh_teardown = + ATH11K_TWT_DEF_INTERFERENCE_THRESH_TEARDOWN; + twt_params->interference_thresh_setup = + ATH11K_TWT_DEF_INTERFERENCE_THRESH_SETUP; + twt_params->min_no_sta_setup = ATH11K_TWT_DEF_MIN_NO_STA_SETUP; + twt_params->min_no_sta_teardown = ATH11K_TWT_DEF_MIN_NO_STA_TEARDOWN; + twt_params->no_of_bcast_mcast_slots = ATH11K_TWT_DEF_NO_OF_BCAST_MCAST_SLOTS; + twt_params->min_no_twt_slots = ATH11K_TWT_DEF_MIN_NO_TWT_SLOTS; + twt_params->max_no_sta_twt = ATH11K_TWT_DEF_MAX_NO_STA_TWT; + twt_params->mode_check_interval = ATH11K_TWT_DEF_MODE_CHECK_INTERVAL; + twt_params->add_sta_slot_interval = ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL; + twt_params->remove_sta_slot_interval = + ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL; + /* TODO add MBSSID support */ + twt_params->mbss_support = 0; +} + +int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id, + struct wmi_twt_enable_params *params) { struct ath11k_pdev_wmi *wmi = ar->wmi; struct ath11k_base *ab = wmi->wmi_ab->ab; @@ -3083,28 +3124,22 @@ ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id) cmd->tlv_header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_TWT_ENABLE_CMD) | FIELD_PREP(WMI_TLV_LEN, len - TLV_HDR_SIZE); cmd->pdev_id = pdev_id; - cmd->sta_cong_timer_ms = ATH11K_TWT_DEF_STA_CONG_TIMER_MS; - cmd->default_slot_size = ATH11K_TWT_DEF_DEFAULT_SLOT_SIZE; - cmd->congestion_thresh_setup = ATH11K_TWT_DEF_CONGESTION_THRESH_SETUP; - cmd->congestion_thresh_teardown = - ATH11K_TWT_DEF_CONGESTION_THRESH_TEARDOWN; - cmd->congestion_thresh_critical = - ATH11K_TWT_DEF_CONGESTION_THRESH_CRITICAL; - cmd->interference_thresh_teardown = - ATH11K_TWT_DEF_INTERFERENCE_THRESH_TEARDOWN; - cmd->interference_thresh_setup = - ATH11K_TWT_DEF_INTERFERENCE_THRESH_SETUP; - cmd->min_no_sta_setup = ATH11K_TWT_DEF_MIN_NO_STA_SETUP; - cmd->min_no_sta_teardown = ATH11K_TWT_DEF_MIN_NO_STA_TEARDOWN; - cmd->no_of_bcast_mcast_slots = ATH11K_TWT_DEF_NO_OF_BCAST_MCAST_SLOTS; - cmd->min_no_twt_slots = ATH11K_TWT_DEF_MIN_NO_TWT_SLOTS; - cmd->max_no_sta_twt = ATH11K_TWT_DEF_MAX_NO_STA_TWT; - cmd->mode_check_interval = ATH11K_TWT_DEF_MODE_CHECK_INTERVAL; - cmd->add_sta_slot_interval = ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL; - cmd->remove_sta_slot_interval = - ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL; - /* TODO add MBSSID support */ - cmd->mbss_support = 0; + cmd->sta_cong_timer_ms = params->sta_cong_timer_ms; + cmd->default_slot_size = params->default_slot_size; + cmd->congestion_thresh_setup = params->congestion_thresh_setup; + cmd->congestion_thresh_teardown = params->congestion_thresh_teardown; + cmd->congestion_thresh_critical = params->congestion_thresh_critical; + cmd->interference_thresh_teardown = params->interference_thresh_teardown; + cmd->interference_thresh_setup = params->interference_thresh_setup; + cmd->min_no_sta_setup = params->min_no_sta_setup; + cmd->min_no_sta_teardown = params->min_no_sta_teardown; + cmd->no_of_bcast_mcast_slots = params->no_of_bcast_mcast_slots; + cmd->min_no_twt_slots = params->min_no_twt_slots; + cmd->max_no_sta_twt = params->max_no_sta_twt; + cmd->mode_check_interval = params->mode_check_interval; + cmd->add_sta_slot_interval = params->add_sta_slot_interval; + cmd->remove_sta_slot_interval = params->remove_sta_slot_interval; + cmd->mbss_support = params->mbss_support; ret = ath11k_wmi_cmd_send(wmi, skb, WMI_TWT_ENABLE_CMDID); if (ret) { @@ -6767,6 +6802,107 @@ static void ath11k_bcn_tx_status_event(struct ath11k_base *ab, struct sk_buff *s rcu_read_unlock(); } +static void ath11k_wmi_event_peer_sta_ps_state_chg(struct ath11k_base *ab, + struct sk_buff *skb) +{ + const struct wmi_peer_sta_ps_state_chg_event *ev; + struct ieee80211_sta *sta; + struct ath11k_peer *peer; + struct ath11k *ar; + struct ath11k_sta *arsta; + const void **tb; + enum ath11k_wmi_peer_ps_state peer_previous_ps_state; + int ret; + + tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC); + if (IS_ERR(tb)) { + ret = PTR_ERR(tb); + ath11k_warn(ab, "failed to parse tlv: %d\n", ret); + return; + } + + ev = tb[WMI_TAG_PEER_STA_PS_STATECHANGE_EVENT]; + if (!ev) { + ath11k_warn(ab, "failed to fetch sta ps change ev"); + kfree(tb); + return; + } + + ath11k_dbg(ab, ATH11K_DBG_WMI, + "peer sta ps chnange ev addr %pM state %u sup_bitmap %x ps_valid %u ts %u\n", + ev->peer_macaddr.addr, ev->peer_ps_state, + ev->ps_supported_bitmap, ev->peer_ps_valid, + ev->peer_ps_timestamp); + + rcu_read_lock(); + + spin_lock_bh(&ab->base_lock); + + peer = ath11k_peer_find_by_addr(ab, ev->peer_macaddr.addr); + + if (!peer) { + spin_unlock_bh(&ab->base_lock); + ath11k_warn(ab, "peer not found %pM\n", ev->peer_macaddr.addr); + goto exit; + } + + ar = ath11k_mac_get_ar_by_vdev_id(ab, peer->vdev_id); + + if (!ar) { + spin_unlock_bh(&ab->base_lock); + ath11k_warn(ab, "invalid vdev id in peer sta ps state change ev %d", + peer->vdev_id); + + goto exit; + } + + sta = peer->sta; + + spin_unlock_bh(&ab->base_lock); + + if (!sta) { + ath11k_warn(ab, "failed to find station entry %pM\n", + ev->peer_macaddr.addr); + goto exit; + } + + arsta = (struct ath11k_sta *)sta->drv_priv; + + spin_lock_bh(&ar->data_lock); + + peer_previous_ps_state = arsta->peer_ps_state; + arsta->peer_ps_state = ev->peer_ps_state; + arsta->peer_current_ps_valid = !!ev->peer_ps_valid; + + if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT, + ar->ab->wmi_ab.svc_map)) { + if (!(ev->ps_supported_bitmap & WMI_PEER_PS_VALID) || + !(ev->ps_supported_bitmap & WMI_PEER_PS_STATE_TIMESTAMP) || + !ev->peer_ps_valid) + goto out; + + if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON) { + arsta->ps_start_time = ev->peer_ps_timestamp; + arsta->ps_start_jiffies = jiffies; + } else if (arsta->peer_ps_state == WMI_PEER_PS_STATE_OFF && + peer_previous_ps_state == WMI_PEER_PS_STATE_ON) { + arsta->ps_total_duration = arsta->ps_total_duration + + (ev->peer_ps_timestamp - arsta->ps_start_time); + } + + if (ar->ps_timekeeper_enable) + trace_ath11k_ps_timekeeper(ar, ev->peer_macaddr.addr, + ev->peer_ps_timestamp, + arsta->peer_ps_state); + } + +out: + spin_unlock_bh(&ar->data_lock); +exit: + rcu_read_unlock(); + kfree(tb); +} + static void ath11k_vdev_stopped_event(struct ath11k_base *ab, struct sk_buff *skb) { struct ath11k *ar; @@ -7409,7 +7545,53 @@ static void ath11k_peer_assoc_conf_event(struct ath11k_base *ab, struct sk_buff static void ath11k_update_stats_event(struct ath11k_base *ab, struct sk_buff *skb) { - ath11k_debugfs_fw_stats_process(ab, skb); + struct ath11k_fw_stats stats = {}; + struct ath11k *ar; + int ret; + + INIT_LIST_HEAD(&stats.pdevs); + INIT_LIST_HEAD(&stats.vdevs); + INIT_LIST_HEAD(&stats.bcn); + + ret = ath11k_wmi_pull_fw_stats(ab, skb, &stats); + if (ret) { + ath11k_warn(ab, "failed to pull fw stats: %d\n", ret); + goto free; + } + + rcu_read_lock(); + ar = ath11k_mac_get_ar_by_pdev_id(ab, stats.pdev_id); + if (!ar) { + rcu_read_unlock(); + ath11k_warn(ab, "failed to get ar for pdev_id %d: %d\n", + stats.pdev_id, ret); + goto free; + } + + spin_lock_bh(&ar->data_lock); + + /* WMI_REQUEST_PDEV_STAT can be requested via .get_txpower mac ops or via + * debugfs fw stats. Therefore, processing it separately. + */ + if (stats.stats_id == WMI_REQUEST_PDEV_STAT) { + list_splice_tail_init(&stats.pdevs, &ar->fw_stats.pdevs); + ar->fw_stats_done = true; + goto complete; + } + + /* WMI_REQUEST_VDEV_STAT, WMI_REQUEST_BCN_STAT and WMI_REQUEST_RSSI_PER_CHAIN_STAT + * are currently requested only via debugfs fw stats. Hence, processing these + * in debugfs context + */ + ath11k_debugfs_fw_stats_process(ar, &stats); + +complete: + complete(&ar->fw_stats_complete); + rcu_read_unlock(); + spin_unlock_bh(&ar->data_lock); + +free: + ath11k_fw_stats_free(&stats); } /* PDEV_CTL_FAILSAFE_CHECK_EVENT is received from FW when the frequency scanned @@ -7960,6 +8142,9 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) case WMI_DIAG_EVENTID: ath11k_wmi_diag_event(ab, skb); break; + case WMI_PEER_STA_PS_STATECHG_EVENTID: + ath11k_wmi_event_peer_sta_ps_state_chg(ab, skb); + break; case WMI_GTK_OFFLOAD_STATUS_EVENTID: ath11k_wmi_gtk_offload_status_event(ab, skb); break; @@ -8962,12 +9147,13 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar, cmd->interval = arg->interval; cmd->method = arg->method; + arp = (struct wmi_sta_keepalive_arp_resp *)(cmd + 1); + arp->tlv_header = FIELD_PREP(WMI_TLV_TAG, + WMI_TAG_STA_KEEPALIVE_ARP_RESPONSE) | + FIELD_PREP(WMI_TLV_LEN, sizeof(*arp) - TLV_HDR_SIZE); + if (arg->method == WMI_STA_KEEPALIVE_METHOD_UNSOLICITED_ARP_RESPONSE || arg->method == WMI_STA_KEEPALIVE_METHOD_GRATUITOUS_ARP_REQUEST) { - arp = (struct wmi_sta_keepalive_arp_resp *)(cmd + 1); - arp->tlv_header = FIELD_PREP(WMI_TLV_TAG, - WMI_TAG_STA_KEEPALVE_ARP_RESPONSE) | - FIELD_PREP(WMI_TLV_LEN, sizeof(*arp) - TLV_HDR_SIZE); arp->src_ip4_addr = arg->src_ip4_addr; arp->dest_ip4_addr = arg->dest_ip4_addr; ether_addr_copy(arp->dest_mac_addr.addr, arg->dest_mac_addr); diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h index 4da248ffa3186e29b1c58773cfdc536701a4039e..8f2c07d70a4a2462a2dc56afa26974654ff755cf 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.h +++ b/drivers/net/wireless/ath/ath11k/wmi.h @@ -17,7 +17,7 @@ struct ath11k_vif; #define PSOC_HOST_MAX_NUM_SS (8) -/* defines to set Packet extension values whic can be 0 us, 8 usec or 16 usec */ +/* defines to set Packet extension values which can be 0 us, 8 usec or 16 usec */ #define MAX_HE_NSS 8 #define MAX_HE_MODULATION 8 #define MAX_HE_RU 4 @@ -1214,7 +1214,7 @@ enum wmi_tlv_tag { WMI_TAG_NS_OFFLOAD_TUPLE, WMI_TAG_FTM_INTG_CMD, WMI_TAG_STA_KEEPALIVE_CMD, - WMI_TAG_STA_KEEPALVE_ARP_RESPONSE, + WMI_TAG_STA_KEEPALIVE_ARP_RESPONSE, WMI_TAG_P2P_SET_VENDOR_IE_DATA_CMD, WMI_TAG_AP_PS_PEER_CMD, WMI_TAG_PEER_RATE_RETRY_SCHED_CMD, @@ -2090,6 +2090,7 @@ enum wmi_tlv_service { WMI_TLV_SERVICE_PER_PEER_HTT_STATS_RESET = 213, WMI_TLV_SERVICE_FREQINFO_IN_METADATA = 219, WMI_TLV_SERVICE_EXT2_MSG = 220, + WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT = 246, WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT = 249, /* The second 128 bits */ @@ -4482,7 +4483,7 @@ struct wmi_pdev_radar_ev { } __packed; struct wmi_pdev_temperature_event { - /* temperature value in Celcius degree */ + /* temperature value in Celsius degree */ s32 temp; u32 pdev_id; } __packed; @@ -4708,7 +4709,7 @@ enum wmi_sta_ps_param_tx_wake_threshold { */ enum wmi_sta_ps_param_pspoll_count { WMI_STA_PS_PSPOLL_COUNT_NO_MAX = 0, - /* Values greater than 0 indicate the maximum numer of PS-Poll frames + /* Values greater than 0 indicate the maximum number of PS-Poll frames * FW will send before waking up. */ }; @@ -4820,9 +4821,9 @@ enum wmi_rate_preamble { /** * enum wmi_rtscts_prot_mode - Enable/Disable RTS/CTS and CTS2Self Protection. - * @WMI_RTS_CTS_DISABLED : RTS/CTS protection is disabled. - * @WMI_USE_RTS_CTS : RTS/CTS Enabled. - * @WMI_USE_CTS2SELF : CTS to self protection Enabled. + * @WMI_RTS_CTS_DISABLED: RTS/CTS protection is disabled. + * @WMI_USE_RTS_CTS: RTS/CTS Enabled. + * @WMI_USE_CTS2SELF: CTS to self protection Enabled. */ enum wmi_rtscts_prot_mode { WMI_RTS_CTS_DISABLED = 0, @@ -4833,13 +4834,13 @@ enum wmi_rtscts_prot_mode { /** * enum wmi_rtscts_profile - Selection of RTS CTS profile along with enabling * protection mode. - * @WMI_RTSCTS_FOR_NO_RATESERIES - Neither of rate-series should use RTS-CTS - * @WMI_RTSCTS_FOR_SECOND_RATESERIES - Only second rate-series will use RTS-CTS - * @WMI_RTSCTS_ACROSS_SW_RETRIES - Only the second rate-series will use RTS-CTS, - * but if there's a sw retry, both the rate - * series will use RTS-CTS. - * @WMI_RTSCTS_ERP - RTS/CTS used for ERP protection for every PPDU. - * @WMI_RTSCTS_FOR_ALL_RATESERIES - Enable RTS-CTS for all rate series. + * @WMI_RTSCTS_FOR_NO_RATESERIES: Neither of rate-series should use RTS-CTS + * @WMI_RTSCTS_FOR_SECOND_RATESERIES: Only second rate-series will use RTS-CTS + * @WMI_RTSCTS_ACROSS_SW_RETRIES: Only the second rate-series will use RTS-CTS, + * but if there's a sw retry, both the rate + * series will use RTS-CTS. + * @WMI_RTSCTS_ERP: RTS/CTS used for ERP protection for every PPDU. + * @WMI_RTSCTS_FOR_ALL_RATESERIES: Enable RTS-CTS for all rate series. */ enum wmi_rtscts_profile { WMI_RTSCTS_FOR_NO_RATESERIES = 0, @@ -4933,6 +4934,25 @@ struct wmi_wmm_params_all_arg { #define ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL 1000 #define ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL 5000 +struct wmi_twt_enable_params { + u32 sta_cong_timer_ms; + u32 mbss_support; + u32 default_slot_size; + u32 congestion_thresh_setup; + u32 congestion_thresh_teardown; + u32 congestion_thresh_critical; + u32 interference_thresh_teardown; + u32 interference_thresh_setup; + u32 min_no_sta_setup; + u32 min_no_sta_teardown; + u32 no_of_bcast_mcast_slots; + u32 min_no_twt_slots; + u32 max_no_sta_twt; + u32 mode_check_interval; + u32 add_sta_slot_interval; + u32 remove_sta_slot_interval; +}; + struct wmi_twt_enable_params_cmd { u32 tlv_header; u32 pdev_id; @@ -5350,6 +5370,26 @@ struct wmi_debug_log_config_cmd_fixed_param { #define WMI_SERVICE_READY_TIMEOUT_HZ (5 * HZ) #define WMI_SEND_TIMEOUT_HZ (3 * HZ) +enum ath11k_wmi_peer_ps_state { + WMI_PEER_PS_STATE_OFF, + WMI_PEER_PS_STATE_ON, + WMI_PEER_PS_STATE_DISABLED, +}; + +enum wmi_peer_ps_supported_bitmap { + /* Used to indicate that power save state change is valid */ + WMI_PEER_PS_VALID = 0x1, + WMI_PEER_PS_STATE_TIMESTAMP = 0x2, +}; + +struct wmi_peer_sta_ps_state_chg_event { + struct wmi_mac_addr peer_macaddr; + u32 peer_ps_state; + u32 ps_supported_bitmap; + u32 peer_ps_valid; + u32 peer_ps_timestamp; +} __packed; + struct ath11k_wmi_base { struct ath11k_base *ab; struct ath11k_pdev_wmi wmi[MAX_RADIOS]; @@ -6039,7 +6079,9 @@ void ath11k_wmi_fw_stats_fill(struct ath11k *ar, struct ath11k_fw_stats *fw_stats, u32 stats_id, char *buf); int ath11k_wmi_simulate_radar(struct ath11k *ar); -int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id); +void ath11k_wmi_fill_default_twt_params(struct wmi_twt_enable_params *twt_params); +int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id, + struct wmi_twt_enable_params *params); int ath11k_wmi_send_twt_disable_cmd(struct ath11k *ar, u32 pdev_id); int ath11k_wmi_send_twt_add_dialog_cmd(struct ath11k *ar, struct wmi_twt_add_dialog_params *params); diff --git a/drivers/net/wireless/ath/ath11k/wow.c b/drivers/net/wireless/ath/ath11k/wow.c index b3e65cd13d8349cab54413abcf3dc6cd44a36e7e..1dec23b0699c86e2ae957b523ec286dc157d0abc 100644 --- a/drivers/net/wireless/ath/ath11k/wow.c +++ b/drivers/net/wireless/ath/ath11k/wow.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -67,6 +68,13 @@ int ath11k_wow_wakeup(struct ath11k_base *ab) struct ath11k *ar = ath11k_ab_to_ar(ab, 0); int ret; + /* In the case of WCN6750, WoW wakeup is done + * by sending SMP2P power save exit message + * to the target processor. + */ + if (ab->hw_params.smp2p_wow_exit) + return 0; + reinit_completion(&ab->wow.wakeup_completed); ret = ath11k_wmi_wow_host_wakeup_ind(ar); @@ -664,6 +672,12 @@ int ath11k_wow_op_suspend(struct ieee80211_hw *hw, struct ath11k *ar = hw->priv; int ret; + ret = ath11k_mac_wait_tx_complete(ar); + if (ret) { + ath11k_warn(ar->ab, "failed to wait tx complete: %d\n", ret); + return ret; + } + mutex_lock(&ar->conf_mutex); ret = ath11k_dp_rx_pktlog_stop(ar->ab, true); @@ -695,13 +709,6 @@ int ath11k_wow_op_suspend(struct ieee80211_hw *hw, goto cleanup; } - ath11k_mac_drain_tx(ar); - ret = ath11k_mac_wait_tx_complete(ar); - if (ret) { - ath11k_warn(ar->ab, "failed to wait tx complete: %d\n", ret); - goto cleanup; - } - ret = ath11k_wow_set_hw_filter(ar); if (ret) { ath11k_warn(ar->ab, "failed to set hw filter: %d\n", diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c index e11c7e9accc03e536a612c0e73ee3fee9f85dd70..a20e0aeae284ca8d3a6273aaf0be1561a20e2fe3 100644 --- a/drivers/net/wireless/ath/ath6kl/cfg80211.c +++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c @@ -1124,7 +1124,7 @@ void ath6kl_cfg80211_ch_switch_notify(struct ath6kl_vif *vif, int freq, } static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, + int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, struct key_params *params) { @@ -1249,7 +1249,7 @@ static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, } static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, + int link_id, u8 key_index, bool pairwise, const u8 *mac_addr) { struct ath6kl *ar = ath6kl_priv(ndev); @@ -1279,7 +1279,7 @@ static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev, } static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, + int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, void *cookie, void (*callback) (void *cookie, struct key_params *)) @@ -1314,7 +1314,7 @@ static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev, } static int ath6kl_cfg80211_set_default_key(struct wiphy *wiphy, - struct net_device *ndev, + struct net_device *ndev, int link_id, u8 key_index, bool unicast, bool multicast) { diff --git a/drivers/net/wireless/ath/ath6kl/init.c b/drivers/net/wireless/ath/ath6kl/init.c index 9b5c7d8f2b95e00bb69410756a114559a66c42c3..201e45554070602292587be0e9b3068a6960ceba 100644 --- a/drivers/net/wireless/ath/ath6kl/init.c +++ b/drivers/net/wireless/ath/ath6kl/init.c @@ -1014,7 +1014,7 @@ static int ath6kl_fetch_fw_apin(struct ath6kl *ar, const char *name) switch (ie_id) { case ATH6KL_FW_IE_FW_VERSION: - strlcpy(ar->wiphy->fw_version, data, + strscpy(ar->wiphy->fw_version, data, min(sizeof(ar->wiphy->fw_version), ie_len+1)); ath6kl_dbg(ATH6KL_DBG_BOOT, diff --git a/drivers/net/wireless/ath/ath9k/ar9003_phy.c b/drivers/net/wireless/ath/ath9k/ar9003_phy.c index dc0e5ea25673764ff7ba97f61c0e20c210bbd146..090ff0600c81d26a0a645c53825a93f300516671 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_phy.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_phy.c @@ -1744,7 +1744,7 @@ static void ar9003_hw_spectral_scan_config(struct ath_hw *ah, REG_SET_BIT(ah, AR_PHY_RADAR_0, AR_PHY_RADAR_0_FFT_ENA); REG_SET_BIT(ah, AR_PHY_SPECTRAL_SCAN, AR_PHY_SPECTRAL_SCAN_ENABLE); - /* on AR93xx and newer, count = 0 will make the the chip send + /* on AR93xx and newer, count = 0 will make the chip send * spectral samples endlessly. Check if this really was intended, * and fix otherwise. */ diff --git a/drivers/net/wireless/ath/ath9k/channel.c b/drivers/net/wireless/ath/ath9k/channel.c index 6cf08752215725fdcf47a16de94241c2132e216a..571062f2e82a7ba39f3bffb569168f3a5007e016 100644 --- a/drivers/net/wireless/ath/ath9k/channel.c +++ b/drivers/net/wireless/ath/ath9k/channel.c @@ -1113,7 +1113,7 @@ ath_chanctx_send_vif_ps_frame(struct ath_softc *sc, struct ath_vif *avp, if (!avp->assoc) return false; - skb = ieee80211_nullfunc_get(sc->hw, vif, false); + skb = ieee80211_nullfunc_get(sc->hw, vif, -1, false); if (!skb) return false; diff --git a/drivers/net/wireless/ath/ath9k/htc_hst.c b/drivers/net/wireless/ath/ath9k/htc_hst.c index 994ec48b2f669588dbfe46c69611aad47aaebfbe..ca05b07a45e67c714e58ac5f9c708acfb1b8f1c8 100644 --- a/drivers/net/wireless/ath/ath9k/htc_hst.c +++ b/drivers/net/wireless/ath/ath9k/htc_hst.c @@ -364,33 +364,27 @@ ret: } static void ath9k_htc_fw_panic_report(struct htc_target *htc_handle, - struct sk_buff *skb) + struct sk_buff *skb, u32 len) { uint32_t *pattern = (uint32_t *)skb->data; - switch (*pattern) { - case 0x33221199: - { + if (*pattern == 0x33221199 && len >= sizeof(struct htc_panic_bad_vaddr)) { struct htc_panic_bad_vaddr *htc_panic; htc_panic = (struct htc_panic_bad_vaddr *) skb->data; dev_err(htc_handle->dev, "ath: firmware panic! " "exccause: 0x%08x; pc: 0x%08x; badvaddr: 0x%08x.\n", htc_panic->exccause, htc_panic->pc, htc_panic->badvaddr); - break; - } - case 0x33221299: - { + return; + } + if (*pattern == 0x33221299) { struct htc_panic_bad_epid *htc_panic; htc_panic = (struct htc_panic_bad_epid *) skb->data; dev_err(htc_handle->dev, "ath: firmware panic! " "bad epid: 0x%08x\n", htc_panic->epid); - break; - } - default: - dev_err(htc_handle->dev, "ath: unknown panic pattern!\n"); - break; + return; } + dev_err(htc_handle->dev, "ath: unknown panic pattern!\n"); } /* @@ -411,16 +405,26 @@ void ath9k_htc_rx_msg(struct htc_target *htc_handle, if (!htc_handle || !skb) return; + /* A valid message requires len >= 8. + * + * sizeof(struct htc_frame_hdr) == 8 + * sizeof(struct htc_ready_msg) == 8 + * sizeof(struct htc_panic_bad_vaddr) == 16 + * sizeof(struct htc_panic_bad_epid) == 8 + */ + if (unlikely(len < sizeof(struct htc_frame_hdr))) + goto invalid; htc_hdr = (struct htc_frame_hdr *) skb->data; epid = htc_hdr->endpoint_id; if (epid == 0x99) { - ath9k_htc_fw_panic_report(htc_handle, skb); + ath9k_htc_fw_panic_report(htc_handle, skb, len); kfree_skb(skb); return; } if (epid < 0 || epid >= ENDPOINT_MAX) { +invalid: if (pipe_id != USB_REG_IN_PIPE) dev_kfree_skb_any(skb); else @@ -432,21 +436,30 @@ void ath9k_htc_rx_msg(struct htc_target *htc_handle, /* Handle trailer */ if (htc_hdr->flags & HTC_FLAGS_RECV_TRAILER) { - if (be32_to_cpu(*(__be32 *) skb->data) == 0x00C60000) + if (be32_to_cpu(*(__be32 *) skb->data) == 0x00C60000) { /* Move past the Watchdog pattern */ htc_hdr = (struct htc_frame_hdr *)(skb->data + 4); + len -= 4; + } } /* Get the message ID */ + if (unlikely(len < sizeof(struct htc_frame_hdr) + sizeof(__be16))) + goto invalid; msg_id = (__be16 *) ((void *) htc_hdr + sizeof(struct htc_frame_hdr)); /* Now process HTC messages */ switch (be16_to_cpu(*msg_id)) { case HTC_MSG_READY_ID: + if (unlikely(len < sizeof(struct htc_ready_msg))) + goto invalid; htc_process_target_rdy(htc_handle, htc_hdr); break; case HTC_MSG_CONNECT_SERVICE_RESPONSE_ID: + if (unlikely(len < sizeof(struct htc_frame_hdr) + + sizeof(struct htc_conn_svc_rspmsg))) + goto invalid; htc_process_conn_rsp(htc_handle, htc_hdr); break; default: diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h index 096a206f49ed5ec3dc833588829d6a241139109d..450ab19b1d4e8436b2db7828fd89f53e17e07b05 100644 --- a/drivers/net/wireless/ath/ath9k/hw.h +++ b/drivers/net/wireless/ath/ath9k/hw.h @@ -710,7 +710,7 @@ struct ath_spec_scan { /** * struct ath_hw_ops - callbacks used by hardware code and driver code * - * This structure contains callbacks designed to to be used internally by + * This structure contains callbacks designed to be used internally by * hardware code and also by the lower level driver. * * @config_pci_powersave: diff --git a/drivers/net/wireless/ath/ath9k/rng.c b/drivers/net/wireless/ath/ath9k/rng.c index cb5414265a9b53577f33c02edbb3a935f0321921..58c0ab01771b0483182fe6d77d47a944f83e4cd5 100644 --- a/drivers/net/wireless/ath/ath9k/rng.c +++ b/drivers/net/wireless/ath/ath9k/rng.c @@ -83,7 +83,8 @@ static int ath9k_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) if (!wait || !max || likely(bytes_read) || fail_stats > 110) break; - msleep_interruptible(ath9k_rng_delay_get(++fail_stats)); + if (hwrng_msleep(rng, ath9k_rng_delay_get(++fail_stats))) + break; } if (wait && !bytes_read && max) diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c index ba16a7f3e23dc59454161abe235595a336c328dc..ba271a10d4ab17e49829c5ee3eac632f59378db8 100644 --- a/drivers/net/wireless/ath/ath9k/xmit.c +++ b/drivers/net/wireless/ath/ath9k/xmit.c @@ -2160,7 +2160,7 @@ static void setup_frame_info(struct ieee80211_hw *hw, fi->keyix = an->ps_key; else fi->keyix = ATH9K_TXKEYIX_INVALID; - fi->dyn_smps = sta && sta->smps_mode == IEEE80211_SMPS_DYNAMIC; + fi->dyn_smps = sta && sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC; fi->keytype = keytype; fi->framelen = framelen; fi->tx_power = txpower; diff --git a/drivers/net/wireless/ath/carl9170/fw.c b/drivers/net/wireless/ath/carl9170/fw.c index 1ab09e1c9ec57a0ff91f7af398849e95858c073f..4c1aecd1163cee707f576902a255fc8afee03784 100644 --- a/drivers/net/wireless/ath/carl9170/fw.c +++ b/drivers/net/wireless/ath/carl9170/fw.c @@ -105,7 +105,7 @@ static void carl9170_fw_info(struct ar9170 *ar) CARL9170FW_GET_MONTH(fw_date), CARL9170FW_GET_DAY(fw_date)); - strlcpy(ar->hw->wiphy->fw_version, motd_desc->release, + strscpy(ar->hw->wiphy->fw_version, motd_desc->release, sizeof(ar->hw->wiphy->fw_version)); } } diff --git a/drivers/net/wireless/ath/wcn36xx/hal.h b/drivers/net/wireless/ath/wcn36xx/hal.h index f1a43fd1d957dfcd324e7723fbff5f564ef23b50..d3a9d00e65e13255d758aaead3661530e3af0192 100644 --- a/drivers/net/wireless/ath/wcn36xx/hal.h +++ b/drivers/net/wireless/ath/wcn36xx/hal.h @@ -2677,7 +2677,7 @@ struct ani_global_security_stats { * management information base (MIB) object is enabled */ u32 rx_wep_unencrypted_frm_cnt; - /* The number of received MSDU packets that that the 802.11 station + /* The number of received MSDU packets that the 802.11 station * discarded because of MIC failures */ u32 rx_mic_fail_cnt; diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.c b/drivers/net/wireless/ath/wcn36xx/txrx.c index 8da3955995b6ea3e4431c8f07c9662ff00330a29..0802ed728824948e7278f622cbdce06e12f97c62 100644 --- a/drivers/net/wireless/ath/wcn36xx/txrx.c +++ b/drivers/net/wireless/ath/wcn36xx/txrx.c @@ -16,6 +16,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include "txrx.h" static inline int get_rssi0(struct wcn36xx_rx_bd *bd) @@ -278,6 +279,7 @@ static void wcn36xx_update_survey(struct wcn36xx *wcn, int rssi, int snr, struct ieee80211_supported_band *sband; int idx; int i; + u8 snr_sample = snr & 0xff; idx = 0; if (band == NL80211_BAND_5GHZ) @@ -297,6 +299,8 @@ static void wcn36xx_update_survey(struct wcn36xx *wcn, int rssi, int snr, wcn->chan_survey[idx].rssi = rssi; wcn->chan_survey[idx].snr = snr; spin_unlock(&wcn->survey_lock); + + add_device_randomness(&snr_sample, sizeof(snr_sample)); } int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb) diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c index f93bdffa4d1dd74c572651ed4bf55b3ccec09f38..40f9a7ef8980e8ec047d5588cc2fbb6ea6030a41 100644 --- a/drivers/net/wireless/ath/wil6210/cfg80211.c +++ b/drivers/net/wireless/ath/wil6210/cfg80211.c @@ -1620,7 +1620,7 @@ static void wil_del_rx_key(u8 key_index, enum wmi_key_usage key_usage, } static int wil_cfg80211_add_key(struct wiphy *wiphy, - struct net_device *ndev, + struct net_device *ndev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, struct key_params *params) @@ -1696,7 +1696,7 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy, } static int wil_cfg80211_del_key(struct wiphy *wiphy, - struct net_device *ndev, + struct net_device *ndev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr) { @@ -1723,7 +1723,7 @@ static int wil_cfg80211_del_key(struct wiphy *wiphy, /* Need to be present or wiphy_new() will WARN */ static int wil_cfg80211_set_default_key(struct wiphy *wiphy, - struct net_device *ndev, + struct net_device *ndev, int link_id, u8 key_index, bool unicast, bool multicast) { @@ -2072,8 +2072,8 @@ void wil_cfg80211_ap_recovery(struct wil6210_priv *wil) key_params.key = vif->gtk; key_params.key_len = vif->gtk_len; key_params.seq_len = IEEE80211_GCMP_PN_LEN; - rc = wil_cfg80211_add_key(wiphy, ndev, vif->gtk_index, false, - NULL, &key_params); + rc = wil_cfg80211_add_key(wiphy, ndev, -1, vif->gtk_index, + false, NULL, &key_params); if (rc) wil_err(wil, "vif %d recovery add key failed (%d)\n", i, rc); diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c index 7da87c9f363fdc437e409e02ec8dad3fe78f5e92..94e61dbe94f83eb5433763648464cfcdf7702297 100644 --- a/drivers/net/wireless/ath/wil6210/main.c +++ b/drivers/net/wireless/ath/wil6210/main.c @@ -1305,7 +1305,7 @@ void wil_get_board_file(struct wil6210_priv *wil, char *buf, size_t len) board_file = WIL_BOARD_FILE_NAME; } - strlcpy(buf, board_file, len); + strscpy(buf, board_file, len); } static int wil_get_bl_info(struct wil6210_priv *wil) diff --git a/drivers/net/wireless/ath/wil6210/netdev.c b/drivers/net/wireless/ath/wil6210/netdev.c index 87a88f26233e02d593d5755877f5340bbdff51c9..ee7d7e9c27184b425956bb8f148ea72e6a2d5c54 100644 --- a/drivers/net/wireless/ath/wil6210/netdev.c +++ b/drivers/net/wireless/ath/wil6210/netdev.c @@ -445,7 +445,7 @@ int wil_if_add(struct wil6210_priv *wil) wil_dbg_misc(wil, "entered"); - strlcpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version)); + strscpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version)); rc = wiphy_register(wiphy); if (rc < 0) { @@ -456,14 +456,12 @@ int wil_if_add(struct wil6210_priv *wil) init_dummy_netdev(&wil->napi_ndev); if (wil->use_enhanced_dma_hw) { netif_napi_add(&wil->napi_ndev, &wil->napi_rx, - wil6210_netdev_poll_rx_edma, - NAPI_POLL_WEIGHT); + wil6210_netdev_poll_rx_edma); netif_napi_add_tx(&wil->napi_ndev, &wil->napi_tx, wil6210_netdev_poll_tx_edma); } else { netif_napi_add(&wil->napi_ndev, &wil->napi_rx, - wil6210_netdev_poll_rx, - NAPI_POLL_WEIGHT); + wil6210_netdev_poll_rx); netif_napi_add_tx(&wil->napi_ndev, &wil->napi_tx, wil6210_netdev_poll_tx); } diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c index ea7bd403e70674365db3cb153f3ed24339c04858..6a5976a2944c9a6e3ad0fda5b0944d79b7557c47 100644 --- a/drivers/net/wireless/ath/wil6210/wmi.c +++ b/drivers/net/wireless/ath/wil6210/wmi.c @@ -780,7 +780,7 @@ static void wmi_evt_ready(struct wil6210_vif *vif, int id, void *d, int len) return; /* FW load will fail after timeout */ } /* ignore MAC address, we already have it from the boot loader */ - strlcpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version)); + strscpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version)); if (len > offsetof(struct wmi_ready_event, rfc_read_calib_result)) { wil_dbg_wmi(wil, "rfc calibration result %d\n", diff --git a/drivers/net/wireless/atmel/atmel.c b/drivers/net/wireless/atmel/atmel.c index 0361c8eb20088fdf389ca4c76682936663715f7a..45d079b933844a783b4069a959318770a58a9e38 100644 --- a/drivers/net/wireless/atmel/atmel.c +++ b/drivers/net/wireless/atmel/atmel.c @@ -1518,7 +1518,7 @@ struct net_device *init_atmel_card(unsigned short irq, unsigned long port, priv->firmware = NULL; priv->firmware_type = fw_type; if (firmware) /* module parameter */ - strlcpy(priv->firmware_id, firmware, sizeof(priv->firmware_id)); + strscpy(priv->firmware_id, firmware, sizeof(priv->firmware_id)); priv->bus_type = card_present ? BUS_TYPE_PCCARD : BUS_TYPE_PCI; priv->station_state = STATION_STATE_DOWN; priv->do_rx_crc = 0; diff --git a/drivers/net/wireless/broadcom/b43/leds.c b/drivers/net/wireless/broadcom/b43/leds.c index 982a772a9d879331c7d922fa2b163b26a6307cb7..bfe1be345844da1da67aeb65c9d359ed52b1659f 100644 --- a/drivers/net/wireless/broadcom/b43/leds.c +++ b/drivers/net/wireless/broadcom/b43/leds.c @@ -118,7 +118,7 @@ static int b43_register_led(struct b43_wldev *dev, struct b43_led *led, led->wl = dev->wl; led->index = led_index; led->activelow = activelow; - strlcpy(led->name, name, sizeof(led->name)); + strscpy(led->name, name, sizeof(led->name)); atomic_set(&led->state, 0); led->led_dev.name = led->name; diff --git a/drivers/net/wireless/broadcom/b43/phy_n.c b/drivers/net/wireless/broadcom/b43/phy_n.c index aa5c994656749afb4efb737a862cc9a2f339a251..2c0c019a815d613df753aa51949985f12f52af51 100644 --- a/drivers/net/wireless/broadcom/b43/phy_n.c +++ b/drivers/net/wireless/broadcom/b43/phy_n.c @@ -2479,11 +2479,7 @@ static void b43_nphy_gain_ctl_workarounds_rev19(struct b43_wldev *dev) static void b43_nphy_gain_ctl_workarounds_rev7(struct b43_wldev *dev) { - struct b43_phy *phy = &dev->phy; - - switch (phy->rev) { - /* TODO */ - } + /* TODO - should depend on phy->rev */ } static void b43_nphy_gain_ctl_workarounds_rev3(struct b43_wldev *dev) diff --git a/drivers/net/wireless/broadcom/b43legacy/leds.c b/drivers/net/wireless/broadcom/b43legacy/leds.c index 38b5be3a84e2c0dd22386ea288ae2687d98a2d74..79e6fd205bfb70e4c5763716451b239829067ef1 100644 --- a/drivers/net/wireless/broadcom/b43legacy/leds.c +++ b/drivers/net/wireless/broadcom/b43legacy/leds.c @@ -88,7 +88,7 @@ static int b43legacy_register_led(struct b43legacy_wldev *dev, led->dev = dev; led->index = led_index; led->activelow = activelow; - strlcpy(led->name, name, sizeof(led->name)); + strscpy(led->name, name, sizeof(led->name)); led->led_dev.name = led->name; led->led_dev.default_trigger = default_trigger; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 2c95a08a5871175394e3353cbac4476f43d4ee2a..9ec0c60b6da10aa9b7dbfd39a28a9cc18dd5f3f6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -87,6 +87,8 @@ struct brcmf_proto_bcdc_header { * plus any space that might be needed * for bus alignment padding. */ +#define ROUND_UP_MARGIN 2048 + struct brcmf_bcdc { u16 reqid; u8 bus_header[BUS_HEADER_LEN]; @@ -368,8 +370,7 @@ brcmf_proto_bcdc_txcomplete(struct device *dev, struct sk_buff *txp, /* await txstatus signal for firmware if active */ if (brcmf_fws_fc_active(bcdc->fws)) { - if (!success) - brcmf_fws_bustxfail(bcdc->fws, txp); + brcmf_fws_bustxcomplete(bcdc->fws, txp, success); } else { if (brcmf_proto_bcdc_hdrpull(bus_if->drvr, false, txp, &ifp)) brcmu_pkt_buf_free_skb(txp); @@ -471,7 +472,7 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES; drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN + - sizeof(struct brcmf_proto_bcdc_dcmd); + sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN; return 0; fail: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index d639bb8b51ae42ac8c90ad0451394817f7f20c73..d0daef674e7289a7c4c397afe9eec81b9e357211 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -983,6 +983,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_4373), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43012), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43752), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_89359), { /* end: all zeroes */ } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index ae5af76e256840b4a96ac9c567eb5349db67439b..2208ab3aa79598b3d122333b2958c1acc5375f78 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -6,6 +6,8 @@ #ifndef BRCMFMAC_BUS_H #define BRCMFMAC_BUS_H +#include +#include #include "debug.h" /* IDs of the 6 default common rings of msgbuf protocol */ @@ -34,6 +36,11 @@ enum brcmf_bus_protocol_type { BRCMF_PROTO_MSGBUF }; +/* Firmware blobs that may be available */ +enum brcmf_blob_type { + BRCMF_BLOB_CLM, +}; + struct brcmf_mp_device; struct brcmf_bus_dcmd { @@ -60,7 +67,7 @@ struct brcmf_bus_dcmd { * @wowl_config: specify if dongle is configured for wowl when going to suspend * @get_ramsize: obtain size of device memory. * @get_memdump: obtain device memory dump in provided buffer. - * @get_fwname: obtain firmware name. + * @get_blob: obtain a firmware blob. * * This structure provides an abstract interface towards the * bus specific driver. For control messages to common driver @@ -77,8 +84,8 @@ struct brcmf_bus_ops { void (*wowl_config)(struct device *dev, bool enabled); size_t (*get_ramsize)(struct device *dev); int (*get_memdump)(struct device *dev, void *data, size_t len); - int (*get_fwname)(struct device *dev, const char *ext, - unsigned char *fw_name); + int (*get_blob)(struct device *dev, const struct firmware **fw, + enum brcmf_blob_type type); void (*debugfs_create)(struct device *dev); int (*reset)(struct device *dev); }; @@ -220,10 +227,10 @@ int brcmf_bus_get_memdump(struct brcmf_bus *bus, void *data, size_t len) } static inline -int brcmf_bus_get_fwname(struct brcmf_bus *bus, const char *ext, - unsigned char *fw_name) +int brcmf_bus_get_blob(struct brcmf_bus *bus, const struct firmware **fw, + enum brcmf_blob_type type) { - return bus->ops->get_fwname(bus->dev, ext, fw_name); + return bus->ops->get_blob(bus->dev, fw, type); } static inline diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index db45da33adfd3763cf85e4ad606ee15ab68cc5ab..dfcfb333336906d9941ad3b968c91dd1519cb7c1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2361,7 +2361,8 @@ done: static s32 brcmf_cfg80211_config_default_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_idx, bool unicast, bool multicast) + int link_id, u8 key_idx, bool unicast, + bool multicast) { struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_pub *drvr = ifp->drvr; @@ -2395,7 +2396,8 @@ done: static s32 brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_idx, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr) { struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_wsec_key *key; @@ -2432,8 +2434,8 @@ brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev, static s32 brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_idx, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); @@ -2457,8 +2459,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, } if (params->key_len == 0) - return brcmf_cfg80211_del_key(wiphy, ndev, key_idx, pairwise, - mac_addr); + return brcmf_cfg80211_del_key(wiphy, ndev, -1, key_idx, + pairwise, mac_addr); if (params->key_len > sizeof(key->data)) { bphy_err(drvr, "Too long key length (%u)\n", params->key_len); @@ -2553,8 +2555,9 @@ done: } static s32 -brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev, u8 key_idx, - bool pairwise, const u8 *mac_addr, void *cookie, +brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev, + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params *params)) { @@ -2610,7 +2613,8 @@ done: static s32 brcmf_cfg80211_config_default_mgmt_key(struct wiphy *wiphy, - struct net_device *ndev, u8 key_idx) + struct net_device *ndev, int link_id, + u8 key_idx) { struct brcmf_if *ifp = netdev_priv(ndev); @@ -3160,10 +3164,7 @@ static s32 brcmf_update_bss_info(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) { struct brcmf_pub *drvr = cfg->pub; - struct brcmf_bss_info_le *bi; - const struct brcmf_tlv *tim; - size_t ie_len; - u8 *ie; + struct brcmf_bss_info_le *bi = NULL; s32 err = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -3177,29 +3178,8 @@ static s32 brcmf_update_bss_info(struct brcmf_cfg80211_info *cfg, bphy_err(drvr, "Could not get bss info %d\n", err); goto update_bss_info_out; } - bi = (struct brcmf_bss_info_le *)(cfg->extra_buf + 4); err = brcmf_inform_single_bss(cfg, bi); - if (err) - goto update_bss_info_out; - - ie = ((u8 *)bi) + le16_to_cpu(bi->ie_offset); - ie_len = le32_to_cpu(bi->ie_length); - - tim = brcmf_parse_tlvs(ie, ie_len, WLAN_EID_TIM); - if (!tim) { - /* - * active scan was done so we could not get dtim - * information out of probe response. - * so we speficially query dtim information to dongle. - */ - u32 var; - err = brcmf_fil_iovar_int_get(ifp, "dtim_assoc", &var); - if (err) { - bphy_err(drvr, "wl dtim_assoc failed (%d)\n", err); - goto update_bss_info_out; - } - } update_bss_info_out: brcmf_dbg(TRACE, "Exit"); @@ -3984,7 +3964,6 @@ brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) struct brcmf_pmk_list_le *pmk_list; int i; u32 npmk; - s32 err; pmk_list = &cfg->pmk_list; npmk = le32_to_cpu(pmk_list->npmk); @@ -3993,10 +3972,8 @@ brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) for (i = 0; i < npmk; i++) brcmf_dbg(CONN, "PMK[%d]: %pM\n", i, &pmk_list->pmk[i].bssid); - err = brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_list, - sizeof(*pmk_list)); - - return err; + return brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_list, + sizeof(*pmk_list)); } static s32 @@ -5042,13 +5019,10 @@ brcmf_cfg80211_change_beacon(struct wiphy *wiphy, struct net_device *ndev, struct cfg80211_beacon_data *info) { struct brcmf_if *ifp = netdev_priv(ndev); - s32 err; brcmf_dbg(TRACE, "Enter\n"); - err = brcmf_config_ap_mgmt_ie(ifp->vif, info); - - return err; + return brcmf_config_ap_mgmt_ie(ifp->vif, info); } static int @@ -6431,6 +6405,7 @@ static void wl_deinit_priv(struct brcmf_cfg80211_info *cfg) cfg->dongle_up = false; /* dongle down */ brcmf_abort_scanning(cfg); brcmf_deinit_priv_mem(cfg); + brcmf_clear_assoc_ies(cfg); } static void init_vif_event(struct brcmf_cfg80211_vif_event *event) @@ -7485,6 +7460,7 @@ static bool brmcf_use_iso3166_ccode_fallback(struct brcmf_pub *drvr) return true; switch (drvr->bus_if->chip) { + case BRCM_CC_43430_CHIP_ID: case BRCM_CC_4345_CHIP_ID: case BRCM_CC_43602_CHIP_ID: return true; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 4ec7773b690645daef40307e7d5d06c95da47c5d..121893bbaa1d7c41ef01f914b717d5d750598553 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -641,6 +641,7 @@ static void brcmf_chip_socram_ramsize(struct brcmf_core_priv *sr, u32 *ramsize, *srsize = (32 * 1024); break; case BRCM_CC_43430_CHIP_ID: + case CY_CC_43439_CHIP_ID: /* assume sr for now as we can not check * firmware sr capability at this point. */ @@ -732,6 +733,10 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) return 0x160000; case CY_CC_43752_CHIP_ID: return 0x170000; + case BRCM_CC_4378_CHIP_ID: + return 0x352000; + case CY_CC_89459_CHIP_ID: + return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000); default: brcmf_err("unknown chip: %s\n", ci->pub.name); break; @@ -1258,7 +1263,8 @@ brcmf_chip_cm3_set_passive(struct brcmf_chip_priv *chip) brcmf_chip_resetcore(core, 0, 0, 0); /* disable bank #3 remap for this device */ - if (chip->pub.chip == BRCM_CC_43430_CHIP_ID) { + if (chip->pub.chip == BRCM_CC_43430_CHIP_ID || + chip->pub.chip == CY_CC_43439_CHIP_ID) { sr = container_of(core, struct brcmf_core_priv, pub); brcmf_chip_core_write32(sr, SOCRAMREGOFFS(bankidx), 3); brcmf_chip_core_write32(sr, SOCRAMREGOFFS(bankpda), 0); @@ -1416,10 +1422,12 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) reg = chip->ops->read32(chip->ctx, addr); return (reg & pmu_cc3_mask) != 0; case BRCM_CC_43430_CHIP_ID: + case CY_CC_43439_CHIP_ID: addr = CORE_CC_REG(base, sr_control1); reg = chip->ops->read32(chip->ctx, addr); return reg != 0; case CY_CC_4373_CHIP_ID: + case CY_CC_89459_CHIP_ID: /* explicitly check SR engine enable bit */ addr = CORE_CC_REG(base, sr_control0); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 7485e784be2a08a3764d8665ae9b8462f5096e25..74020fa100659e554574173d8ba77db7ff8b0de7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -123,7 +123,6 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) struct brcmf_bus *bus = drvr->bus_if; struct brcmf_dload_data_le *chunk_buf; const struct firmware *clm = NULL; - u8 clm_name[BRCMF_FW_NAME_LEN]; u32 chunk_len; u32 datalen; u32 cumulative_len; @@ -133,15 +132,8 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) brcmf_dbg(TRACE, "Enter\n"); - memset(clm_name, 0, sizeof(clm_name)); - err = brcmf_bus_get_fwname(bus, ".clm_blob", clm_name); - if (err) { - bphy_err(drvr, "get CLM blob file name failed (%d)\n", err); - return err; - } - - err = firmware_request_nowarn(&clm, clm_name, bus->dev); - if (err) { + err = brcmf_bus_get_blob(bus, &clm, BRCMF_BLOB_CLM); + if (err || !clm) { brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n", err); return 0; @@ -261,7 +253,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) &revinfo, sizeof(revinfo)); if (err < 0) { bphy_err(drvr, "retrieving revision info failed, %d\n", err); - strlcpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname)); + strscpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname)); } else { ri->vendorid = le32_to_cpu(revinfo.vendorid); ri->deviceid = le32_to_cpu(revinfo.deviceid); @@ -314,7 +306,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) /* locate firmware version number for ethtool */ ptr = strrchr(buf, ' ') + 1; - strlcpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver)); + strscpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver)); /* Query for 'clmver' to get CLM version info from firmware */ memset(buf, 0, sizeof(buf)); @@ -424,11 +416,11 @@ static void brcmf_mp_attach(void) * if not set then if available use the platform data version. To make * sure it gets initialized at all, always copy the module param version */ - strlcpy(brcmf_mp_global.firmware_path, brcmf_firmware_path, + strscpy(brcmf_mp_global.firmware_path, brcmf_firmware_path, BRCMF_FW_ALTPATH_LEN); if ((brcmfmac_pdata) && (brcmfmac_pdata->fw_alternative_path) && (brcmf_mp_global.firmware_path[0] == '\0')) { - strlcpy(brcmf_mp_global.firmware_path, + strscpy(brcmf_mp_global.firmware_path, brcmfmac_pdata->fw_alternative_path, BRCMF_FW_ALTPATH_LEN); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 6c5a22a32a96c8f3a838145cf42f56f05ecb4e11..aa25abffcc7dbc4556b084770af9e07769187678 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -53,6 +53,7 @@ struct brcmf_mp_device { struct brcmfmac_pd_cc *country_codes; const char *board_type; unsigned char mac[ETH_ALEN]; + const char *antenna_sku; union { struct brcmfmac_sdio_pd sdio; } bus; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index bd164a0821f9ffb725b7f38d302556371fffc755..595ae3ae561ef20d69da0388813adfbd963ef560 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -292,6 +292,7 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct brcmf_pub *drvr = ifp->drvr; struct ethhdr *eh; int head_delta; + unsigned int tx_bytes = skb->len; brcmf_dbg(DATA, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx); @@ -366,7 +367,7 @@ done: ndev->stats.tx_dropped++; } else { ndev->stats.tx_packets++; - ndev->stats.tx_bytes += skb->len; + ndev->stats.tx_bytes += tx_bytes; } /* Return ok: we always eat the packet */ @@ -561,10 +562,10 @@ static void brcmf_ethtool_get_drvinfo(struct net_device *ndev, if (drvr->revinfo.result == 0) brcmu_dotrev_str(drvr->revinfo.driverrev, drev); - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); - strlcpy(info->version, drev, sizeof(info->version)); - strlcpy(info->fw_version, drvr->fwver, sizeof(info->fw_version)); - strlcpy(info->bus_info, dev_name(drvr->bus_if->dev), + strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); + strscpy(info->version, drev, sizeof(info->version)); + strscpy(info->fw_version, drvr->fwver, sizeof(info->fw_version)); + strscpy(info->bus_info, dev_name(drvr->bus_if->dev), sizeof(info->bus_info)); } @@ -1480,8 +1481,10 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp) !brcmf_get_pend_8021x_cnt(ifp), MAX_WAIT_FOR_8021X_TX); - if (!err) + if (!err) { bphy_err(drvr, "Timed out waiting for no pending 802.1x packets\n"); + atomic_set(&ifp->pend_8021x_cnt, 0); + } return !err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c index 0af452dca7664e5d9c14ede702278e8bace35fd2..86ff174936a9a0e0ae1874be113eef0b2de77e14 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c @@ -24,6 +24,13 @@ static const struct brcmf_dmi_data acepc_t8_data = { BRCM_CC_4345_CHIP_ID, 6, "acepc-t8" }; +/* The Chuwi Hi8 Pro uses the same Ampak AP6212 module as the Chuwi Vi8 Plus + * and the nvram for the Vi8 Plus is already in linux-firmware, so use that. + */ +static const struct brcmf_dmi_data chuwi_hi8_pro_data = { + BRCM_CC_43430_CHIP_ID, 0, "ilife-S806" +}; + static const struct brcmf_dmi_data gpd_win_pocket_data = { BRCM_CC_4356_CHIP_ID, 2, "gpd-win-pocket" }; @@ -75,6 +82,17 @@ static const struct dmi_system_id dmi_platform_data[] = { }, .driver_data = (void *)&acepc_t8_data, }, + { + /* Chuwi Hi8 Pro with D2D3_Hi8Pro.233 BIOS */ + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Hampoo"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "MRD"), + /* Above strings are too generic, also match on BIOS date */ + DMI_MATCH(DMI_BIOS_DATE, "05/10/2016"), + }, + .driver_data = (void *)&chuwi_hi8_pro_data, + }, { /* Cyberbook T116 rugged tablet */ .matches = { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index d2ac844e1e9ffe3f1111eaf945903a3f1e7dfe24..2c2f3e026c136672f714605bf2f632c6953052f5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -249,7 +249,8 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) memset(&gscan_cfg, 0, sizeof(gscan_cfg)); if (drvr->bus_if->chip != BRCM_CC_43430_CHIP_ID && drvr->bus_if->chip != BRCM_CC_4345_CHIP_ID && - drvr->bus_if->chip != BRCM_CC_43454_CHIP_ID) + drvr->bus_if->chip != BRCM_CC_43454_CHIP_ID && + drvr->bus_if->chip != CY_CC_43439_CHIP_ID) brcmf_feat_iovar_data_set(ifp, BRCMF_FEAT_GSCAN, "pfn_gscan_cfg", &gscan_cfg, sizeof(gscan_cfg)); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c index b8379e4034a4c9c91dc7495de13e1b1c3152dff9..f2207793f6e279764e22b5734b3b983f28769d3f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c @@ -21,6 +21,8 @@ #define BRCMF_FW_NVRAM_DEVPATH_LEN 19 /* devpath0=pcie/1/4/ */ #define BRCMF_FW_NVRAM_PCIEDEV_LEN 10 /* pcie/1/4/ + \0 */ #define BRCMF_FW_DEFAULT_BOARDREV "boardrev=0xff" +#define BRCMF_FW_MACADDR_FMT "macaddr=%pM" +#define BRCMF_FW_MACADDR_LEN (7 + ETH_ALEN * 3) enum nvram_parser_state { IDLE, @@ -44,6 +46,7 @@ enum nvram_parser_state { * @multi_dev_v1: detect pcie multi device v1 (compressed). * @multi_dev_v2: detect pcie multi device v2. * @boardrev_found: nvram contains boardrev information. + * @strip_mac: strip the MAC address. */ struct nvram_parser { enum nvram_parser_state state; @@ -57,6 +60,7 @@ struct nvram_parser { bool multi_dev_v1; bool multi_dev_v2; bool boardrev_found; + bool strip_mac; }; /* @@ -121,6 +125,10 @@ static enum nvram_parser_state brcmf_nvram_handle_key(struct nvram_parser *nvp) nvp->multi_dev_v2 = true; if (strncmp(&nvp->data[nvp->entry], "boardrev", 8) == 0) nvp->boardrev_found = true; + /* strip macaddr if platform MAC overrides */ + if (nvp->strip_mac && + strncmp(&nvp->data[nvp->entry], "macaddr", 7) == 0) + st = COMMENT; } else if (!is_nvram_char(c) || c == ' ') { brcmf_dbg(INFO, "warning: ln=%d:col=%d: '=' expected, skip invalid key entry\n", nvp->line, nvp->column); @@ -209,6 +217,7 @@ static int brcmf_init_nvram_parser(struct nvram_parser *nvp, size = data_len; /* Add space for properties we may add */ size += strlen(BRCMF_FW_DEFAULT_BOARDREV) + 1; + size += BRCMF_FW_MACADDR_LEN + 1; /* Alloc for extra 0 byte + roundup by 4 + length field */ size += 1 + 3 + sizeof(u32); nvp->nvram = kzalloc(size, GFP_KERNEL); @@ -368,22 +377,37 @@ static void brcmf_fw_add_defaults(struct nvram_parser *nvp) nvp->nvram_len++; } +static void brcmf_fw_add_macaddr(struct nvram_parser *nvp, u8 *mac) +{ + int len; + + len = scnprintf(&nvp->nvram[nvp->nvram_len], BRCMF_FW_MACADDR_LEN + 1, + BRCMF_FW_MACADDR_FMT, mac); + WARN_ON(len != BRCMF_FW_MACADDR_LEN); + nvp->nvram_len += len + 1; +} + /* brcmf_nvram_strip :Takes a buffer of "=\n" lines read from a fil * and ending in a NUL. Removes carriage returns, empty lines, comment lines, * and converts newlines to NULs. Shortens buffer as needed and pads with NULs. * End of buffer is completed with token identifying length of buffer. */ static void *brcmf_fw_nvram_strip(const u8 *data, size_t data_len, - u32 *new_length, u16 domain_nr, u16 bus_nr) + u32 *new_length, u16 domain_nr, u16 bus_nr, + struct device *dev) { struct nvram_parser nvp; u32 pad; u32 token; __le32 token_le; + u8 mac[ETH_ALEN]; if (brcmf_init_nvram_parser(&nvp, data, data_len) < 0) return NULL; + if (eth_platform_get_mac_address(dev, mac) == 0) + nvp.strip_mac = true; + while (nvp.pos < data_len) { nvp.state = nv_parser_states[nvp.state](&nvp); if (nvp.state == END) @@ -404,6 +428,9 @@ static void *brcmf_fw_nvram_strip(const u8 *data, size_t data_len, brcmf_fw_add_defaults(&nvp); + if (nvp.strip_mac) + brcmf_fw_add_macaddr(&nvp, mac); + pad = nvp.nvram_len; *new_length = roundup(nvp.nvram_len + 1, 4); while (pad != *new_length) { @@ -430,6 +457,7 @@ struct brcmf_fw { struct device *dev; struct brcmf_fw_request *req; u32 curpos; + unsigned int board_index; void (*done)(struct device *dev, int err, struct brcmf_fw_request *req); }; @@ -537,7 +565,8 @@ static int brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx) if (data) nvram = brcmf_fw_nvram_strip(data, data_len, &nvram_length, fwctx->req->domain_nr, - fwctx->req->bus_nr); + fwctx->req->bus_nr, + fwctx->dev); if (free_bcm47xx_nvram) bcm47xx_nvram_release_contents(data); @@ -587,39 +616,50 @@ static int brcmf_fw_complete_request(const struct firmware *fw, static char *brcm_alt_fw_path(const char *path, const char *board_type) { - char alt_path[BRCMF_FW_NAME_LEN]; - char suffix[5]; + char base[BRCMF_FW_NAME_LEN]; + const char *suffix; + char *ret; + + if (!board_type) + return NULL; - strscpy(alt_path, path, BRCMF_FW_NAME_LEN); - /* At least one character + suffix */ - if (strlen(alt_path) < 5) + suffix = strrchr(path, '.'); + if (!suffix || suffix == path) return NULL; - /* strip .txt or .bin at the end */ - strscpy(suffix, alt_path + strlen(alt_path) - 4, 5); - alt_path[strlen(alt_path) - 4] = 0; - strlcat(alt_path, ".", BRCMF_FW_NAME_LEN); - strlcat(alt_path, board_type, BRCMF_FW_NAME_LEN); - strlcat(alt_path, suffix, BRCMF_FW_NAME_LEN); + /* strip extension at the end */ + strscpy(base, path, BRCMF_FW_NAME_LEN); + base[suffix - path] = 0; - return kstrdup(alt_path, GFP_KERNEL); + ret = kasprintf(GFP_KERNEL, "%s.%s%s", base, board_type, suffix); + if (!ret) + brcmf_err("out of memory allocating firmware path for '%s'\n", + path); + + brcmf_dbg(TRACE, "FW alt path: %s\n", ret); + + return ret; } static int brcmf_fw_request_firmware(const struct firmware **fw, struct brcmf_fw *fwctx) { struct brcmf_fw_item *cur = &fwctx->req->items[fwctx->curpos]; + unsigned int i; int ret; - /* Files can be board-specific, first try a board-specific path */ - if (cur->type == BRCMF_FW_TYPE_NVRAM && fwctx->req->board_type) { + /* Files can be board-specific, first try board-specific paths */ + for (i = 0; i < ARRAY_SIZE(fwctx->req->board_types); i++) { char *alt_path; - alt_path = brcm_alt_fw_path(cur->path, fwctx->req->board_type); + if (!fwctx->req->board_types[i]) + goto fallback; + alt_path = brcm_alt_fw_path(cur->path, + fwctx->req->board_types[i]); if (!alt_path) goto fallback; - ret = request_firmware(fw, alt_path, fwctx->dev); + ret = firmware_request_nowarn(fw, alt_path, fwctx->dev); kfree(alt_path); if (ret == 0) return ret; @@ -653,15 +693,40 @@ static void brcmf_fw_request_done_alt_path(const struct firmware *fw, void *ctx) { struct brcmf_fw *fwctx = ctx; struct brcmf_fw_item *first = &fwctx->req->items[0]; + const char *board_type, *alt_path; int ret = 0; - /* Fall back to canonical path if board firmware not found */ - if (!fw) - ret = request_firmware_nowait(THIS_MODULE, true, first->path, + if (fw) { + brcmf_fw_request_done(fw, ctx); + return; + } + + /* Try next board firmware */ + if (fwctx->board_index < ARRAY_SIZE(fwctx->req->board_types)) { + board_type = fwctx->req->board_types[fwctx->board_index++]; + if (!board_type) + goto fallback; + alt_path = brcm_alt_fw_path(first->path, board_type); + if (!alt_path) + goto fallback; + + ret = request_firmware_nowait(THIS_MODULE, true, alt_path, fwctx->dev, GFP_KERNEL, fwctx, - brcmf_fw_request_done); + brcmf_fw_request_done_alt_path); + kfree(alt_path); + + if (ret < 0) + brcmf_fw_request_done(fw, ctx); + return; + } - if (fw || ret < 0) +fallback: + /* Fall back to canonical path if board firmware not found */ + ret = request_firmware_nowait(THIS_MODULE, true, first->path, + fwctx->dev, GFP_KERNEL, fwctx, + brcmf_fw_request_done); + + if (ret < 0) brcmf_fw_request_done(fw, ctx); } @@ -705,10 +770,11 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req, fwctx->done = fw_cb; /* First try alternative board-specific path if any */ - if (fwctx->req->board_type) + if (fwctx->req->board_types[0]) alt_path = brcm_alt_fw_path(first->path, - fwctx->req->board_type); + fwctx->req->board_types[0]); if (alt_path) { + fwctx->board_index++; ret = request_firmware_nowait(THIS_MODULE, true, alt_path, fwctx->dev, GFP_KERNEL, fwctx, brcmf_fw_request_done_alt_path); @@ -769,7 +835,7 @@ brcmf_fw_alloc_request(u32 chip, u32 chiprev, fwnames[j].path[0] = '\0'; /* check if firmware path is provided by module parameter */ if (brcmf_mp_global.firmware_path[0] != '\0') { - strlcpy(fwnames[j].path, mp_path, + strscpy(fwnames[j].path, mp_path, BRCMF_FW_NAME_LEN); if (end != '/') { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h index e290dec9c53d7013bac1c60b0727bfd27c14bc40..1266cbaee0729451ea358058dffdee631927d2b0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h @@ -11,6 +11,8 @@ #define BRCMF_FW_DEFAULT_PATH "brcm/" +#define BRCMF_FW_MAX_BOARD_TYPES 8 + /** * struct brcmf_firmware_mapping - Used to map chipid/revmask to firmware * filename and nvram filename. Each bus type implementation should create @@ -66,7 +68,7 @@ struct brcmf_fw_request { u16 domain_nr; u16 bus_nr; u32 n_items; - const char *board_type; + const char *board_types[BRCMF_FW_MAX_BOARD_TYPES]; struct brcmf_fw_item items[]; }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index 096f6b969dd83b0dc20b7f4423a8d803b494a054..e1127d7e086d53bf76c559d2c757aa02275f4a37 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -419,7 +419,6 @@ void brcmf_flowring_configure_addr_mode(struct brcmf_flowring *flow, int ifidx, flowid = flow->hash[i].flowid; if (flow->rings[flowid]->status != RING_OPEN) continue; - flow->rings[flowid]->status = RING_CLOSING; brcmf_msgbuf_delete_flowring(drvr, flowid); } } @@ -458,10 +457,8 @@ void brcmf_flowring_delete_peer(struct brcmf_flowring *flow, int ifidx, if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) && (hash[i].ifidx == ifidx)) { flowid = flow->hash[i].flowid; - if (flow->rings[flowid]->status == RING_OPEN) { - flow->rings[flowid]->status = RING_CLOSING; + if (flow->rings[flowid]->status == RING_OPEN) brcmf_msgbuf_delete_flowring(drvr, flowid); - } } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index c87b829adb0d06017ef4943f1d1e220033cb7a08..f518e025d6e465a49c5d7d00d6474a37665fb230 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -135,7 +135,7 @@ /* Link Down indication in WoWL mode: */ #define BRCMF_WOWL_LINKDOWN (1 << 31) -#define BRCMF_WOWL_MAXPATTERNS 8 +#define BRCMF_WOWL_MAXPATTERNS 16 #define BRCMF_WOWL_MAXPATTERNSIZE 128 #define BRCMF_COUNTRY_BUF_SZ 4 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index d58525ebe618ebd76912f0d5cf245cccac10d4a3..36af81975855c52534ed03b4c80a87c404533b61 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -688,7 +688,7 @@ static void brcmf_fws_macdesc_set_name(struct brcmf_fws_info *fws, struct brcmf_fws_mac_descriptor *desc) { if (desc == &fws->desc.other) - strlcpy(desc->name, "MAC-OTHER", sizeof(desc->name)); + strscpy(desc->name, "MAC-OTHER", sizeof(desc->name)); else if (desc->mac_handle) scnprintf(desc->name, sizeof(desc->name), "MAC-%d:%d", desc->mac_handle, desc->interface_id); @@ -2475,7 +2475,8 @@ bool brcmf_fws_fc_active(struct brcmf_fws_info *fws) return fws->fcmode != BRCMF_FWS_FCMODE_NONE; } -void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb) +void brcmf_fws_bustxcomplete(struct brcmf_fws_info *fws, struct sk_buff *skb, + bool success) { u32 hslot; @@ -2483,11 +2484,14 @@ void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb) brcmu_pkt_buf_free_skb(skb); return; } - brcmf_fws_lock(fws); - hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); - brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0, - 1); - brcmf_fws_unlock(fws); + + if (!success) { + brcmf_fws_lock(fws); + hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); + brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, + 0, 0, 1); + brcmf_fws_unlock(fws); + } } void brcmf_fws_bus_blocked(struct brcmf_pub *drvr, bool flow_blocked) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index b16a9d1c0508e27bd4c3e473c4802071e96934e4..f9c36cd8f1de101b7faf4db4e8866678b2e46648 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -40,7 +40,8 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_fws_reset_interface(struct brcmf_if *ifp); void brcmf_fws_add_interface(struct brcmf_if *ifp); void brcmf_fws_del_interface(struct brcmf_if *ifp); -void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb); +void brcmf_fws_bustxcomplete(struct brcmf_fws_info *fws, struct sk_buff *skb, + bool success); void brcmf_fws_bus_blocked(struct brcmf_pub *drvr, bool flow_blocked); void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index b2d0f7570aa97a37ba48393e100c01ffb979b5a6..cec53f934940a618bb91a892e33a7f0eacaacc16 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -71,6 +71,7 @@ #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS 32 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS 48 +#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES 10 struct msgbuf_common_hdr { u8 msgtype; @@ -806,8 +807,12 @@ static int brcmf_msgbuf_tx_queue_data(struct brcmf_pub *drvr, int ifidx, flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx); if (flowid == BRCMF_FLOWRING_INVALID_ID) { flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb); - if (flowid == BRCMF_FLOWRING_INVALID_ID) + if (flowid == BRCMF_FLOWRING_INVALID_ID) { return -ENOMEM; + } else { + brcmf_flowring_enqueue(flow, flowid, skb); + return 0; + } } queue_count = brcmf_flowring_enqueue(flow, flowid, skb); force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0); @@ -1395,9 +1400,27 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; + struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid]; + struct brcmf_flowring *flow = msgbuf->flow; void *ret_ptr; u8 ifidx; int err; + int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; + + /* make sure it is not in txflow */ + brcmf_commonring_lock(commonring_del); + flow->rings[flowid]->status = RING_CLOSING; + brcmf_commonring_unlock(commonring_del); + + /* wait for commonring txflow finished */ + while (retry && atomic_read(&commonring_del->outstanding_tx)) { + usleep_range(5000, 10000); + retry--; + } + if (!retry) { + brcmf_err("timed out waiting for txstatus\n"); + atomic_set(&commonring_del->outstanding_tx, 0); + } /* no need to submit if firmware can not be reached */ if (drvr->bus_if->state != BRCMF_BUS_UP) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 2e322edbb90700b4e339cd9d1356746e85d89456..6a849f4a94dd7f9e8fedc0c7235fe7957ee0cd06 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -8,10 +8,10 @@ #ifdef CONFIG_BRCMFMAC_PROTO_MSGBUF #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_MAX_ITEM 64 -#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 512 +#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 1024 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_MAX_ITEM 64 #define BRCMF_D2H_MSGRING_TX_COMPLETE_MAX_ITEM 1024 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 512 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 1024 #define BRCMF_H2D_TXFLOWRING_MAX_ITEM 512 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index 79388d49c256023f690259028e237230018cdab2..a83699de01ec3c4ba5802241624be8ef6c706073 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -70,14 +70,24 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, { struct brcmfmac_sdio_pd *sdio = &settings->bus.sdio; struct device_node *root, *np = dev->of_node; + const char *prop; int irq; int err; u32 irqf; u32 val; + /* Apple ARM64 platforms have their own idea of board type, passed in + * via the device tree. They also have an antenna SKU parameter + */ + if (!of_property_read_string(np, "brcm,board-type", &prop)) + settings->board_type = prop; + + if (!of_property_read_string(np, "apple,antenna-sku", &prop)) + settings->antenna_sku = prop; + /* Set board-type to the first string of the machine compatible prop */ root = of_find_node_by_path("/"); - if (root) { + if (root && !settings->board_type) { char *board_type; const char *tmp; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 479041f070f98b3ca646577ffa8d2a7a3da62eee..10d9d9c63b2811106e5a7dd967fac22fee57b166 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1128,7 +1128,7 @@ static void brcmf_p2p_afx_handler(struct work_struct *work) if (afx_hdl->is_listen && afx_hdl->my_listen_chan) /* 100ms ~ 300ms */ err = brcmf_p2p_discover_listen(p2p, afx_hdl->my_listen_chan, - 100 * (1 + prandom_u32() % 3)); + 100 * (1 + prandom_u32_max(3))); else err = brcmf_p2p_act_frm_search(p2p, afx_hdl->peer_listen_chan); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 97f0f13dfe505bec40d08784b191439516f6d560..80083f9ea311621b0501fe3faffed8fb55eeafdb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -59,6 +59,8 @@ BRCMF_FW_DEF(4365C, "brcmfmac4365c-pcie"); BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie"); BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); +BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); +BRCMF_FW_DEF(4355, "brcmfmac89459-pcie"); /* firmware config files */ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.txt"); @@ -66,6 +68,7 @@ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt"); /* per-board firmware binaries */ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.bin"); +MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.clm_blob"); static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -87,6 +90,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_43666_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), + BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFFF, 4378B1), /* revision ID 3 */ + BRCMF_FW_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355), }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -118,6 +123,12 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0 0x140 #define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1 0x144 +#define BRCMF_PCIE_64_PCIE2REG_INTMASK 0xC14 +#define BRCMF_PCIE_64_PCIE2REG_MAILBOXINT 0xC30 +#define BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK 0xC34 +#define BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0 0xA20 +#define BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1 0xA24 + #define BRCMF_PCIE2_INTA 0x01 #define BRCMF_PCIE2_INTB 0x02 @@ -137,6 +148,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_MB_INT_D2H3_DB0 0x400000 #define BRCMF_PCIE_MB_INT_D2H3_DB1 0x800000 +#define BRCMF_PCIE_MB_INT_FN0 (BRCMF_PCIE_MB_INT_FN0_0 | \ + BRCMF_PCIE_MB_INT_FN0_1) #define BRCMF_PCIE_MB_INT_D2H_DB (BRCMF_PCIE_MB_INT_D2H0_DB0 | \ BRCMF_PCIE_MB_INT_D2H0_DB1 | \ BRCMF_PCIE_MB_INT_D2H1_DB0 | \ @@ -146,6 +159,40 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_PCIE_MB_INT_D2H3_DB0 | \ BRCMF_PCIE_MB_INT_D2H3_DB1) +#define BRCMF_PCIE_64_MB_INT_D2H0_DB0 0x1 +#define BRCMF_PCIE_64_MB_INT_D2H0_DB1 0x2 +#define BRCMF_PCIE_64_MB_INT_D2H1_DB0 0x4 +#define BRCMF_PCIE_64_MB_INT_D2H1_DB1 0x8 +#define BRCMF_PCIE_64_MB_INT_D2H2_DB0 0x10 +#define BRCMF_PCIE_64_MB_INT_D2H2_DB1 0x20 +#define BRCMF_PCIE_64_MB_INT_D2H3_DB0 0x40 +#define BRCMF_PCIE_64_MB_INT_D2H3_DB1 0x80 +#define BRCMF_PCIE_64_MB_INT_D2H4_DB0 0x100 +#define BRCMF_PCIE_64_MB_INT_D2H4_DB1 0x200 +#define BRCMF_PCIE_64_MB_INT_D2H5_DB0 0x400 +#define BRCMF_PCIE_64_MB_INT_D2H5_DB1 0x800 +#define BRCMF_PCIE_64_MB_INT_D2H6_DB0 0x1000 +#define BRCMF_PCIE_64_MB_INT_D2H6_DB1 0x2000 +#define BRCMF_PCIE_64_MB_INT_D2H7_DB0 0x4000 +#define BRCMF_PCIE_64_MB_INT_D2H7_DB1 0x8000 + +#define BRCMF_PCIE_64_MB_INT_D2H_DB (BRCMF_PCIE_64_MB_INT_D2H0_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H0_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H1_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H1_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H2_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H2_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H3_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H3_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H4_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H4_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H5_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H5_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H6_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H6_DB1 | \ + BRCMF_PCIE_64_MB_INT_D2H7_DB0 | \ + BRCMF_PCIE_64_MB_INT_D2H7_DB1) + #define BRCMF_PCIE_SHARED_VERSION_7 7 #define BRCMF_PCIE_MIN_SHARED_VERSION 5 #define BRCMF_PCIE_MAX_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_7 @@ -255,12 +302,24 @@ struct brcmf_pcie_core_info { u32 wrapbase; }; +#define BRCMF_OTP_MAX_PARAM_LEN 16 + +struct brcmf_otp_params { + char module[BRCMF_OTP_MAX_PARAM_LEN]; + char vendor[BRCMF_OTP_MAX_PARAM_LEN]; + char version[BRCMF_OTP_MAX_PARAM_LEN]; + bool valid; +}; + struct brcmf_pciedev_info { enum brcmf_pcie_state state; bool in_irq; struct pci_dev *pdev; char fw_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; + char clm_name[BRCMF_FW_NAME_LEN]; + const struct firmware *clm_fw; + const struct brcmf_pcie_reginfo *reginfo; void __iomem *regs; void __iomem *tcm; u32 ram_base; @@ -280,6 +339,7 @@ struct brcmf_pciedev_info { void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value); struct brcmf_mp_device *settings; + struct brcmf_otp_params otp; }; struct brcmf_pcie_ringbuf { @@ -346,11 +406,49 @@ static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE }; +struct brcmf_pcie_reginfo { + u32 intmask; + u32 mailboxint; + u32 mailboxmask; + u32 h2d_mailbox_0; + u32 h2d_mailbox_1; + u32 int_d2h_db; + u32 int_fn0; +}; + +static const struct brcmf_pcie_reginfo brcmf_reginfo_default = { + .intmask = BRCMF_PCIE_PCIE2REG_INTMASK, + .mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT, + .mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK, + .h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, + .h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, + .int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB, + .int_fn0 = BRCMF_PCIE_MB_INT_FN0, +}; + +static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = { + .intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK, + .mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT, + .mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK, + .h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, + .h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, + .int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB, + .int_fn0 = 0, +}; + static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq); static struct brcmf_fw_request * brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo); +static u16 +brcmf_pcie_read_reg16(struct brcmf_pciedev_info *devinfo, u32 reg_offset) +{ + void __iomem *address = devinfo->regs + reg_offset; + + return ioread16(address); +} + static u32 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) { @@ -496,6 +594,8 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, } +#define READCC32(devinfo, reg) brcmf_pcie_read_reg32(devinfo, \ + CHIPCREGOFFS(reg)) #define WRITECC32(devinfo, reg, value) brcmf_pcie_write_reg32(devinfo, \ CHIPCREGOFFS(reg), value) @@ -779,30 +879,29 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo, static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXMASK, 0); + brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0); } static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXMASK, - BRCMF_PCIE_MB_INT_D2H_DB | - BRCMF_PCIE_MB_INT_FN0_0 | - BRCMF_PCIE_MB_INT_FN0_1); + brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, + devinfo->reginfo->int_d2h_db | + devinfo->reginfo->int_fn0); } static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) { if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) brcmf_pcie_write_reg32(devinfo, - BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); + devinfo->reginfo->h2d_mailbox_1, 1); } static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)arg; - if (brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT)) { + if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint)) { brcmf_pcie_intr_disable(devinfo); brcmf_dbg(PCIE, "Enter\n"); return IRQ_WAKE_THREAD; @@ -817,15 +916,14 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) u32 status; devinfo->in_irq = true; - status = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT); + status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); brcmf_dbg(PCIE, "Enter %x\n", status); if (status) { - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT, + brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); - if (status & (BRCMF_PCIE_MB_INT_FN0_0 | - BRCMF_PCIE_MB_INT_FN0_1)) + if (status & devinfo->reginfo->int_fn0) brcmf_pcie_handle_mb_data(devinfo); - if (status & BRCMF_PCIE_MB_INT_D2H_DB) { + if (status & devinfo->reginfo->int_d2h_db) { if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) brcmf_proto_msgbuf_rx_trigger( &devinfo->pdev->dev); @@ -884,8 +982,8 @@ static void brcmf_pcie_release_irq(struct brcmf_pciedev_info *devinfo) if (devinfo->in_irq) brcmf_err(bus, "Still in IRQ (processing) !!!\n"); - status = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT, status); + status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); + brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); devinfo->irq_allocated = false; } @@ -937,7 +1035,7 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); + brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1); return 0; } @@ -1382,23 +1480,25 @@ static int brcmf_pcie_get_memdump(struct device *dev, void *data, size_t len) return 0; } -static -int brcmf_pcie_get_fwname(struct device *dev, const char *ext, u8 *fw_name) +static int brcmf_pcie_get_blob(struct device *dev, const struct firmware **fw, + enum brcmf_blob_type type) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; - fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev, - brcmf_pcie_fwnames, - ARRAY_SIZE(brcmf_pcie_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; + switch (type) { + case BRCMF_BLOB_CLM: + *fw = devinfo->clm_fw; + devinfo->clm_fw = NULL; + break; + default: + return -ENOENT; + } + + if (!*fw) + return -ENOENT; - kfree(fwreq); return 0; } @@ -1445,7 +1545,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .wowl_config = brcmf_pcie_wowl_config, .get_ramsize = brcmf_pcie_get_ramsize, .get_memdump = brcmf_pcie_get_memdump, - .get_fwname = brcmf_pcie_get_fwname, + .get_blob = brcmf_pcie_get_blob, .reset = brcmf_pcie_reset, }; @@ -1698,15 +1798,22 @@ static int brcmf_pcie_buscoreprep(void *ctx) static int brcmf_pcie_buscore_reset(void *ctx, struct brcmf_chip *chip) { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)ctx; - u32 val; + struct brcmf_core *core; + u32 val, reg; devinfo->ci = chip; brcmf_pcie_reset_device(devinfo); - val = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT); + /* reginfo is not ready yet */ + core = brcmf_chip_get_core(chip, BCMA_CORE_PCIE2); + if (core->rev >= 64) + reg = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT; + else + reg = BRCMF_PCIE_PCIE2REG_MAILBOXINT; + + val = brcmf_pcie_read_reg32(devinfo, reg); if (val != 0xffffffff) - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT, - val); + brcmf_pcie_write_reg32(devinfo, reg, val); return 0; } @@ -1729,8 +1836,206 @@ static const struct brcmf_buscore_ops brcmf_pcie_buscore_ops = { .write32 = brcmf_pcie_buscore_write32, }; +#define BRCMF_OTP_SYS_VENDOR 0x15 +#define BRCMF_OTP_BRCM_CIS 0x80 + +#define BRCMF_OTP_VENDOR_HDR 0x00000008 + +static int +brcmf_pcie_parse_otp_sys_vendor(struct brcmf_pciedev_info *devinfo, + u8 *data, size_t size) +{ + int idx = 4; + const char *chip_params; + const char *board_params; + const char *p; + + /* 4-byte header and two empty strings */ + if (size < 6) + return -EINVAL; + + if (get_unaligned_le32(data) != BRCMF_OTP_VENDOR_HDR) + return -EINVAL; + + chip_params = &data[idx]; + + /* Skip first string, including terminator */ + idx += strnlen(chip_params, size - idx) + 1; + if (idx >= size) + return -EINVAL; + + board_params = &data[idx]; + + /* Skip to terminator of second string */ + idx += strnlen(board_params, size - idx); + if (idx >= size) + return -EINVAL; + + /* At this point both strings are guaranteed NUL-terminated */ + brcmf_dbg(PCIE, "OTP: chip_params='%s' board_params='%s'\n", + chip_params, board_params); + + p = skip_spaces(board_params); + while (*p) { + char tag = *p++; + const char *end; + size_t len; + + if (*p++ != '=') /* implicit NUL check */ + return -EINVAL; + + /* *p might be NUL here, if so end == p and len == 0 */ + end = strchrnul(p, ' '); + len = end - p; + + /* leave 1 byte for NUL in destination string */ + if (len > (BRCMF_OTP_MAX_PARAM_LEN - 1)) + return -EINVAL; + + /* Copy len characters plus a NUL terminator */ + switch (tag) { + case 'M': + strscpy(devinfo->otp.module, p, len + 1); + break; + case 'V': + strscpy(devinfo->otp.vendor, p, len + 1); + break; + case 'm': + strscpy(devinfo->otp.version, p, len + 1); + break; + } + + /* Skip to next arg, if any */ + p = skip_spaces(end); + } + + brcmf_dbg(PCIE, "OTP: module=%s vendor=%s version=%s\n", + devinfo->otp.module, devinfo->otp.vendor, + devinfo->otp.version); + + if (!devinfo->otp.module[0] || + !devinfo->otp.vendor[0] || + !devinfo->otp.version[0]) + return -EINVAL; + + devinfo->otp.valid = true; + return 0; +} + +static int +brcmf_pcie_parse_otp(struct brcmf_pciedev_info *devinfo, u8 *otp, size_t size) +{ + int p = 0; + int ret = -EINVAL; + + brcmf_dbg(PCIE, "parse_otp size=%zd\n", size); + + while (p < (size - 1)) { + u8 type = otp[p]; + u8 length = otp[p + 1]; + + if (type == 0) + break; + + if ((p + 2 + length) > size) + break; + + switch (type) { + case BRCMF_OTP_SYS_VENDOR: + brcmf_dbg(PCIE, "OTP @ 0x%x (%d): SYS_VENDOR\n", + p, length); + ret = brcmf_pcie_parse_otp_sys_vendor(devinfo, + &otp[p + 2], + length); + break; + case BRCMF_OTP_BRCM_CIS: + brcmf_dbg(PCIE, "OTP @ 0x%x (%d): BRCM_CIS\n", + p, length); + break; + default: + brcmf_dbg(PCIE, "OTP @ 0x%x (%d): Unknown type 0x%x\n", + p, length, type); + break; + } + + p += 2 + length; + } + + return ret; +} + +static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) +{ + const struct pci_dev *pdev = devinfo->pdev; + struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev); + u32 coreid, base, words, idx, sromctl; + u16 *otp; + struct brcmf_core *core; + int ret; + + switch (devinfo->ci->chip) { + case BRCM_CC_4378_CHIP_ID: + coreid = BCMA_CORE_GCI; + base = 0x1120; + words = 0x170; + break; + default: + /* OTP not supported on this chip */ + return 0; + } + + core = brcmf_chip_get_core(devinfo->ci, coreid); + if (!core) { + brcmf_err(bus, "No OTP core\n"); + return -ENODEV; + } + + if (coreid == BCMA_CORE_CHIPCOMMON) { + /* Chips with OTP accessed via ChipCommon need additional + * handling to access the OTP + */ + brcmf_pcie_select_core(devinfo, coreid); + sromctl = READCC32(devinfo, sromcontrol); + + if (!(sromctl & BCMA_CC_SROM_CONTROL_OTP_PRESENT)) { + /* Chip lacks OTP, try without it... */ + brcmf_err(bus, + "OTP unavailable, using default firmware\n"); + return 0; + } + + /* Map OTP to shadow area */ + WRITECC32(devinfo, sromcontrol, + sromctl | BCMA_CC_SROM_CONTROL_OTPSEL); + } + + otp = kcalloc(words, sizeof(u16), GFP_KERNEL); + if (!otp) + return -ENOMEM; + + /* Map bus window to SROM/OTP shadow area in core */ + base = brcmf_pcie_buscore_prep_addr(devinfo->pdev, base + core->base); + + brcmf_dbg(PCIE, "OTP data:\n"); + for (idx = 0; idx < words; idx++) { + otp[idx] = brcmf_pcie_read_reg16(devinfo, base + 2 * idx); + brcmf_dbg(PCIE, "[%8x] 0x%04x\n", base + 2 * idx, otp[idx]); + } + + if (coreid == BCMA_CORE_CHIPCOMMON) { + brcmf_pcie_select_core(devinfo, coreid); + WRITECC32(devinfo, sromcontrol, sromctl); + } + + ret = brcmf_pcie_parse_otp(devinfo, (u8 *)otp, 2 * words); + kfree(otp); + + return ret; +} + #define BRCMF_PCIE_FW_CODE 0 #define BRCMF_PCIE_FW_NVRAM 1 +#define BRCMF_PCIE_FW_CLM 2 static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq) @@ -1755,6 +2060,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary; nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; + devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary; kfree(fwreq); ret = brcmf_chip_get_raminfo(devinfo->ci); @@ -1830,6 +2136,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) struct brcmf_fw_name fwnames[] = { { ".bin", devinfo->fw_name }, { ".txt", devinfo->nvram_name }, + { ".clm_blob", devinfo->clm_name }, }; fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, @@ -1842,11 +2149,51 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY; fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; - fwreq->board_type = devinfo->settings->board_type; + fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL; /* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */ fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1; fwreq->bus_nr = devinfo->pdev->bus->number; + /* Apple platforms with fancy firmware/NVRAM selection */ + if (devinfo->settings->board_type && + devinfo->settings->antenna_sku && + devinfo->otp.valid) { + const struct brcmf_otp_params *otp = &devinfo->otp; + struct device *dev = &devinfo->pdev->dev; + const char **bt = fwreq->board_types; + + brcmf_dbg(PCIE, "Apple board: %s\n", + devinfo->settings->board_type); + + /* Example: apple,shikoku-RASP-m-6.11-X3 */ + bt[0] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s-%s-%s", + devinfo->settings->board_type, + otp->module, otp->vendor, otp->version, + devinfo->settings->antenna_sku); + bt[1] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s-%s", + devinfo->settings->board_type, + otp->module, otp->vendor, otp->version); + bt[2] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s", + devinfo->settings->board_type, + otp->module, otp->vendor); + bt[3] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s", + devinfo->settings->board_type, + otp->module); + bt[4] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s", + devinfo->settings->board_type, + devinfo->settings->antenna_sku); + bt[5] = devinfo->settings->board_type; + + if (!bt[0] || !bt[1] || !bt[2] || !bt[3] || !bt[4]) { + kfree(fwreq); + return NULL; + } + } else { + brcmf_dbg(PCIE, "Board: %s\n", devinfo->settings->board_type); + fwreq->board_types[0] = devinfo->settings->board_type; + } + return fwreq; } @@ -1857,6 +2204,7 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) struct brcmf_fw_request *fwreq; struct brcmf_pciedev_info *devinfo; struct brcmf_pciedev *pcie_bus_dev; + struct brcmf_core *core; struct brcmf_bus *bus; brcmf_dbg(PCIE, "Enter %x:%x\n", pdev->vendor, pdev->device); @@ -1876,6 +2224,12 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto fail; } + core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); + if (core->rev >= 64) + devinfo->reginfo = &brcmf_reginfo_64; + else + devinfo->reginfo = &brcmf_reginfo_default; + pcie_bus_dev = kzalloc(sizeof(*pcie_bus_dev), GFP_KERNEL); if (pcie_bus_dev == NULL) { ret = -ENOMEM; @@ -1918,6 +2272,12 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto fail_bus; + ret = brcmf_pcie_read_otp(devinfo); + if (ret) { + brcmf_err(bus, "failed to parse OTP\n"); + goto fail_brcmf; + } + fwreq = brcmf_pcie_prepare_fw_request(devinfo); if (!fwreq) { ret = -ENOMEM; @@ -1981,6 +2341,7 @@ brcmf_pcie_remove(struct pci_dev *pdev) brcmf_pcie_release_ringbuffers(devinfo); brcmf_pcie_reset_device(devinfo); brcmf_pcie_release_resource(devinfo); + release_firmware(devinfo->clm_fw); if (devinfo->ci) brcmf_chip_detach(devinfo->ci); @@ -2038,7 +2399,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); /* Check if device is still up and running, if so we are ready */ - if (brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_INTMASK) != 0) { + if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM)) goto cleanup; @@ -2105,6 +2466,9 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index fabfbb0b40b0ca09aafd74433bc42efedc516e85..170c61c8136cc1c0b97afe42d7adfff878e3e926 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -158,12 +158,12 @@ static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) struct brcmf_pno_macaddr_le pfn_mac; u8 *mac_addr = NULL; u8 *mac_mask = NULL; - int err, i; + int err, i, ri; - for (i = 0; i < pi->n_reqs; i++) - if (pi->reqs[i]->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) { - mac_addr = pi->reqs[i]->mac_addr; - mac_mask = pi->reqs[i]->mac_addr_mask; + for (ri = 0; ri < pi->n_reqs; ri++) + if (pi->reqs[ri]->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) { + mac_addr = pi->reqs[ri]->mac_addr; + mac_mask = pi->reqs[ri]->mac_addr_mask; break; } @@ -177,7 +177,7 @@ static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) memcpy(pfn_mac.mac, mac_addr, ETH_ALEN); for (i = 0; i < ETH_ALEN; i++) { pfn_mac.mac[i] &= mac_mask[i]; - pfn_mac.mac[i] |= get_random_int() & ~(mac_mask[i]); + pfn_mac.mac[i] |= get_random_u8() & ~(mac_mask[i]); } /* Clear multi bit */ pfn_mac.mac[0] &= 0xFE; @@ -185,7 +185,7 @@ static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) pfn_mac.mac[0] |= 0x02; brcmf_dbg(SCAN, "enabling random mac: reqid=%llu mac=%pM\n", - pi->reqs[i]->reqid, pfn_mac.mac); + pi->reqs[ri]->reqid, pfn_mac.mac); err = brcmf_fil_iovar_data_set(ifp, "pfn_macaddr", &pfn_mac, sizeof(pfn_mac)); if (err) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8968809399c7a4392cfec5e7a70371799f9c8e03..465d95d837592f9c15b0effa12737ab0dcc66db8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -618,6 +618,7 @@ BRCMF_FW_DEF(43430A0, "brcmfmac43430a0-sdio"); /* Note the names are not postfixed with a1 for backward compatibility */ BRCMF_FW_CLM_DEF(43430A1, "brcmfmac43430-sdio"); BRCMF_FW_DEF(43430B0, "brcmfmac43430b0-sdio"); +BRCMF_FW_CLM_DEF(43439, "brcmfmac43439-sdio"); BRCMF_FW_CLM_DEF(43455, "brcmfmac43455-sdio"); BRCMF_FW_DEF(43456, "brcmfmac43456-sdio"); BRCMF_FW_CLM_DEF(4354, "brcmfmac4354-sdio"); @@ -657,6 +658,7 @@ static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373), BRCMF_FW_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012), + BRCMF_FW_ENTRY(CY_CC_43439_CHIP_ID, 0xFFFFFFFF, 43439), BRCMF_FW_ENTRY(CY_CC_43752_CHIP_ID, 0xFFFFFFFF, 43752) }; @@ -4129,23 +4131,24 @@ brcmf_sdio_watchdog(struct timer_list *t) } } -static -int brcmf_sdio_get_fwname(struct device *dev, const char *ext, u8 *fw_name) +static int brcmf_sdio_get_blob(struct device *dev, const struct firmware **fw, + enum brcmf_blob_type type) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; - fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev, - brcmf_sdio_fwnames, - ARRAY_SIZE(brcmf_sdio_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; + switch (type) { + case BRCMF_BLOB_CLM: + *fw = sdiodev->clm_fw; + sdiodev->clm_fw = NULL; + break; + default: + return -ENOENT; + } + + if (!*fw) + return -ENOENT; - kfree(fwreq); return 0; } @@ -4180,13 +4183,14 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .wowl_config = brcmf_sdio_wowl_config, .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, - .get_fwname = brcmf_sdio_get_fwname, + .get_blob = brcmf_sdio_get_blob, .debugfs_create = brcmf_sdio_debugfs_create, .reset = brcmf_sdio_bus_reset }; #define BRCMF_SDIO_FW_CODE 0 #define BRCMF_SDIO_FW_NVRAM 1 +#define BRCMF_SDIO_FW_CLM 2 static void brcmf_sdio_firmware_callback(struct device *dev, int err, struct brcmf_fw_request *fwreq) @@ -4209,6 +4213,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, code = fwreq->items[BRCMF_SDIO_FW_CODE].binary; nvram = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.len; + sdiod->clm_fw = fwreq->items[BRCMF_SDIO_FW_CLM].binary; kfree(fwreq); /* try to download image and nvram to the dongle */ @@ -4407,6 +4412,7 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) struct brcmf_fw_name fwnames[] = { { ".bin", bus->sdiodev->fw_name }, { ".txt", bus->sdiodev->nvram_name }, + { ".clm_blob", bus->sdiodev->clm_name }, }; fwreq = brcmf_fw_alloc_request(bus->ci->chip, bus->ci->chiprev, @@ -4418,7 +4424,9 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) fwreq->items[BRCMF_SDIO_FW_CODE].type = BRCMF_FW_TYPE_BINARY; fwreq->items[BRCMF_SDIO_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; - fwreq->board_type = bus->sdiodev->settings->board_type; + fwreq->items[BRCMF_SDIO_FW_CLM].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_SDIO_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL; + fwreq->board_types[0] = bus->sdiodev->settings->board_type; return fwreq; } @@ -4574,6 +4582,8 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) if (bus->sdiodev->settings) brcmf_release_module_param(bus->sdiodev->settings); + release_firmware(bus->sdiodev->clm_fw); + bus->sdiodev->clm_fw = NULL; kfree(bus->rxbuf); kfree(bus->hdrbuf); kfree(bus); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 47351ff458cab234652b891843de07d0ce0e9119..b76d34d36bde6435f11390a3c10e3f5ccb243acd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -186,9 +186,11 @@ struct brcmf_sdio_dev { struct sg_table sgtable; char fw_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; + char clm_name[BRCMF_FW_NAME_LEN]; bool wowl_enabled; enum brcmf_sdiod_state state; struct brcmf_sdiod_freezer *freezer; + const struct firmware *clm_fw; }; /* sdio core registers */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 9fb68c2dc7e39cec6d1ef547115b5d89aa4dfddd..85e18fb9c497a0ebced73f25bb89735838a4d5f8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1154,24 +1154,11 @@ error: return NULL; } -static -int brcmf_usb_get_fwname(struct device *dev, const char *ext, u8 *fw_name) +static int brcmf_usb_get_blob(struct device *dev, const struct firmware **fw, + enum brcmf_blob_type type) { - struct brcmf_bus *bus = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; - - fwreq = brcmf_fw_alloc_request(bus->chip, bus->chiprev, - brcmf_usb_fwnames, - ARRAY_SIZE(brcmf_usb_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; - - kfree(fwreq); - return 0; + /* No blobs for USB devices... */ + return -ENOENT; } static const struct brcmf_bus_ops brcmf_usb_bus_ops = { @@ -1180,7 +1167,7 @@ static const struct brcmf_bus_ops brcmf_usb_bus_ops = { .txdata = brcmf_usb_tx, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, - .get_fwname = brcmf_usb_get_fwname, + .get_blob = brcmf_usb_get_blob, }; #define BRCMF_USB_FW_CODE 0 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h index ae1f3ad40d459f7722f57a0c40744e1ea58f6f18..2b0df07ced744df01534a78cb3a960238394151f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h @@ -123,7 +123,7 @@ */ /******************************************************************** - * Phy/Core Configuration. Defines macros to to check core phy/rev * + * Phy/Core Configuration. Defines macros to check core phy/rev * * compile-time configuration. Defines default core support. * * ****************************************************************** */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index ed0b707f0cdfc4ac5e00fc40a5950db745160e1a..f4939cf62767205389b4b39be86cf7cf09b30376 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -51,9 +51,12 @@ #define BRCM_CC_43664_CHIP_ID 43664 #define BRCM_CC_43666_CHIP_ID 43666 #define BRCM_CC_4371_CHIP_ID 0x4371 +#define BRCM_CC_4378_CHIP_ID 0x4378 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 +#define CY_CC_43439_CHIP_ID 43439 #define CY_CC_43752_CHIP_ID 43752 +#define CY_CC_89459_CHIP_ID 0x4355 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e @@ -87,7 +90,9 @@ #define BRCM_PCIE_4366_2G_DEVICE_ID 0x43c4 #define BRCM_PCIE_4366_5G_DEVICE_ID 0x43c5 #define BRCM_PCIE_4371_DEVICE_ID 0x440d - +#define BRCM_PCIE_4378_DEVICE_ID 0x4425 +#define CY_PCIE_89459_DEVICE_ID 0x4415 +#define CY_PCIE_89459_RAW_DEVICE_ID 0x4355 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c index 5234511dac78a8f985b8fc834e2dfe63febb4fd4..b0f23cf1a621b670d458d3e56b3572e3f6d01cca 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c @@ -5907,8 +5907,8 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev, struct ipw2100_priv *priv = libipw_priv(dev); char fw_ver[64], ucode_ver[64]; - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); ipw2100_get_fwversion(priv, fw_ver, sizeof(fw_ver)); ipw2100_get_ucodeversion(priv, ucode_ver, sizeof(ucode_ver)); @@ -5916,7 +5916,7 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev, snprintf(info->fw_version, sizeof(info->fw_version), "%s:%d:%s", fw_ver, priv->eeprom_version, ucode_ver); - strlcpy(info->bus_info, pci_name(priv->pci_dev), + strscpy(info->bus_info, pci_name(priv->pci_dev), sizeof(info->bus_info)); } @@ -6529,7 +6529,7 @@ static struct pci_driver ipw2100_pci_driver = { .shutdown = ipw2100_shutdown, }; -/** +/* * Initialize the ipw2100 driver/module * * @returns 0 if ok, < 0 errno node con error. @@ -6561,7 +6561,7 @@ out: return ret; } -/** +/* * Cleanup ipw2100 driver registration */ static void __exit ipw2100_exit(void) diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c index 029dacebe751b151c68f7f24615b631798e0350a..5b483de18c81f5e74c6c68620f1a6e89e167b981 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c @@ -10424,8 +10424,8 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev, char date[32]; u32 len; - strlcpy(info->driver, DRV_NAME, sizeof(info->driver)); - strlcpy(info->version, DRV_VERSION, sizeof(info->version)); + strscpy(info->driver, DRV_NAME, sizeof(info->driver)); + strscpy(info->version, DRV_VERSION, sizeof(info->version)); len = sizeof(vers); ipw_get_ordinal(p, IPW_ORD_STAT_FW_VERSION, vers, &len); @@ -10434,7 +10434,7 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev, snprintf(info->fw_version, sizeof(info->fw_version), "%s (%s)", vers, date); - strlcpy(info->bus_info, pci_name(p->pci_dev), + strscpy(info->bus_info, pci_name(p->pci_dev), sizeof(info->bus_info)); } diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.h b/drivers/net/wireless/intel/ipw2x00/ipw2200.h index 55cac934f4ee92e0b982e8c2956136a01eb0b39c..09ddd21608d44d52ae4366ca33e5e580c7d7d309 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.h +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.h @@ -651,7 +651,7 @@ struct ipw_rx_notification { struct notif_link_deterioration link_deterioration; struct notif_calibration calibration; struct notif_noise noise; - u8 raw[0]; + DECLARE_FLEX_ARRAY(u8, raw); } u; } __packed; diff --git a/drivers/net/wireless/intel/ipw2x00/libipw.h b/drivers/net/wireless/intel/ipw2x00/libipw.h index 7964ef7d15f02010f0e18289d4a2b20d254fecaf..bec7bc27374885e639e0ca64dd445e68a28f08bd 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw.h +++ b/drivers/net/wireless/intel/ipw2x00/libipw.h @@ -405,7 +405,7 @@ struct libipw_auth { __le16 transaction; __le16 status; /* challenge */ - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; struct libipw_channel_switch { @@ -423,7 +423,6 @@ struct libipw_action { union { struct libipw_action_exchange { u8 token; - struct libipw_info_element info_element[0]; } exchange; struct libipw_channel_switch channel_switch; @@ -441,7 +440,7 @@ struct libipw_disassoc { struct libipw_probe_request { struct libipw_hdr_3addr header; /* SSID, supported rates */ - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; struct libipw_probe_response { @@ -451,7 +450,7 @@ struct libipw_probe_response { __le16 capability; /* SSID, supported rates, FH params, DS params, * CF params, IBSS params, TIM (if beacon), RSN */ - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; /* Alias beacon for probe_response */ @@ -462,7 +461,7 @@ struct libipw_assoc_request { __le16 capability; __le16 listen_interval; /* SSID, supported rates, RSN */ - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; struct libipw_reassoc_request { @@ -470,7 +469,7 @@ struct libipw_reassoc_request { __le16 capability; __le16 listen_interval; u8 current_ap[ETH_ALEN]; - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; struct libipw_assoc_response { @@ -479,7 +478,7 @@ struct libipw_assoc_response { __le16 status; __le16 aid; /* supported rates */ - struct libipw_info_element info_element[]; + u8 variable[]; } __packed; struct libipw_txb { diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c index 7a684b76f39b7d5f7e930823b4023147c38b2c87..48d6870bbf4e255444ca83d2c89d5b9a62726c3b 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c @@ -1329,8 +1329,8 @@ static int libipw_handle_assoc_resp(struct libipw_device *ieee, struct libipw_as network->wpa_ie_len = 0; network->rsn_ie_len = 0; - if (libipw_parse_info_param - (frame->info_element, stats->len - sizeof(*frame), network)) + if (libipw_parse_info_param((void *)frame->variable, + stats->len - sizeof(*frame), network)) return 1; network->mode = 0; @@ -1389,8 +1389,8 @@ static int libipw_network_init(struct libipw_device *ieee, struct libipw_probe_r network->wpa_ie_len = 0; network->rsn_ie_len = 0; - if (libipw_parse_info_param - (beacon->info_element, stats->len - sizeof(*beacon), network)) + if (libipw_parse_info_param((void *)beacon->variable, + stats->len - sizeof(*beacon), network)) return 1; network->mode = 0; @@ -1510,7 +1510,7 @@ static void libipw_process_probe_response(struct libipw_device struct libipw_network *target; struct libipw_network *oldest = NULL; #ifdef CONFIG_LIBIPW_DEBUG - struct libipw_info_element *info_element = beacon->info_element; + struct libipw_info_element *info_element = (void *)beacon->variable; #endif unsigned long flags; diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c index 846138d6e33df174cb9605e65b10df97fa5896e1..7352d5b2095f48c3f39a22cb44e2c7f86bd56433 100644 --- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c +++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c @@ -3254,7 +3254,7 @@ il3945_store_measurement(struct device *d, struct device_attribute *attr, if (count) { char *p = buffer; - strlcpy(buffer, buf, sizeof(buffer)); + strscpy(buffer, buf, sizeof(buffer)); channel = simple_strtoul(p, NULL, 0); if (channel) params.channel = channel; diff --git a/drivers/net/wireless/intel/iwlegacy/4965-rs.c b/drivers/net/wireless/intel/iwlegacy/4965-rs.c index c62f299b9e0a84cd38062e3966361ab0e7d8d005..718efb1aa1b0ca4f0e0c5adf8c3f29475768242f 100644 --- a/drivers/net/wireless/intel/iwlegacy/4965-rs.c +++ b/drivers/net/wireless/intel/iwlegacy/4965-rs.c @@ -1167,7 +1167,7 @@ il4965_rs_switch_to_mimo2(struct il_priv *il, struct il_lq_sta *lq_sta, if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported) return -1; - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) return -1; /* Need both Tx chains/antennas to support MIMO */ @@ -2403,7 +2403,7 @@ il4965_rs_fill_link_cmd(struct il_priv *il, struct il_lq_sta *lq_sta, /* Repeat initial/next rate. * For legacy IL_NUMBER_TRY == 1, this loop will not execute. * For HT IL_HT_NUMBER_TRY == 3, this executes twice. */ - while (repeat_rate > 0) { + while (repeat_rate > 0 && idx < (LINK_QUAL_MAX_RETRY_NUM - 1)) { if (is_legacy(tbl_type.lq_type)) { if (ant_toggle_cnt < NUM_TRY_BEFORE_ANT_TOGGLE) ant_toggle_cnt++; @@ -2422,8 +2422,6 @@ il4965_rs_fill_link_cmd(struct il_priv *il, struct il_lq_sta *lq_sta, cpu_to_le32(new_rate); repeat_rate--; idx++; - if (idx >= LINK_QUAL_MAX_RETRY_NUM) - goto out; } il4965_rs_get_tbl_info_from_mcs(new_rate, lq_sta->band, @@ -2468,7 +2466,6 @@ il4965_rs_fill_link_cmd(struct il_priv *il, struct il_lq_sta *lq_sta, repeat_rate--; } -out: lq_cmd->agg_params.agg_frame_cnt_limit = LINK_QUAL_AGG_FRAME_LIMIT_DEF; lq_cmd->agg_params.agg_dis_start_th = LINK_QUAL_AGG_DISABLE_START_DEF; diff --git a/drivers/net/wireless/intel/iwlegacy/commands.h b/drivers/net/wireless/intel/iwlegacy/commands.h index 4a97310f8feebfdcd450598282960fdbd015459b..28cf4e832152cdd0b960c56fdaeb198a94f5349d 100644 --- a/drivers/net/wireless/intel/iwlegacy/commands.h +++ b/drivers/net/wireless/intel/iwlegacy/commands.h @@ -1710,7 +1710,7 @@ struct il4965_tx_resp { */ union { __le32 status; - struct agg_tx_status agg_status[0]; /* for each agg frame */ + DECLARE_FLEX_ARRAY(struct agg_tx_status, agg_status); /* for each agg frame */ } u; } __packed; @@ -3365,7 +3365,7 @@ struct il_rx_pkt { struct il_compressed_ba_resp compressed_ba; struct il_missed_beacon_notif missed_beacon; __le32 status; - u8 raw[0]; + DECLARE_FLEX_ARRAY(u8, raw); } u; } __packed; diff --git a/drivers/net/wireless/intel/iwlegacy/common.c b/drivers/net/wireless/intel/iwlegacy/common.c index 04d27a26260b05b778efed7d54455f792d957924..341c17fe2af4da6ff00b15da399c9d6b07de0905 100644 --- a/drivers/net/wireless/intel/iwlegacy/common.c +++ b/drivers/net/wireless/intel/iwlegacy/common.c @@ -1870,15 +1870,15 @@ il_set_ht_add_station(struct il_priv *il, u8 idx, struct ieee80211_sta *sta) goto done; D_ASSOC("spatial multiplexing power save mode: %s\n", - (sta->smps_mode == IEEE80211_SMPS_STATIC) ? "static" : - (sta->smps_mode == IEEE80211_SMPS_DYNAMIC) ? "dynamic" : + (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) ? "static" : + (sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) ? "dynamic" : "disabled"); sta_flags = il->stations[idx].sta.station_flags; sta_flags &= ~(STA_FLG_RTS_MIMO_PROT_MSK | STA_FLG_MIMO_DIS_MSK); - switch (sta->smps_mode) { + switch (sta->deflink.smps_mode) { case IEEE80211_SMPS_STATIC: sta_flags |= STA_FLG_MIMO_DIS_MSK; break; @@ -1888,7 +1888,7 @@ il_set_ht_add_station(struct il_priv *il, u8 idx, struct ieee80211_sta *sta) case IEEE80211_SMPS_OFF: break; default: - IL_WARN("Invalid MIMO PS mode %d\n", sta->smps_mode); + IL_WARN("Invalid MIMO PS mode %d\n", sta->deflink.smps_mode); break; } diff --git a/drivers/net/wireless/intel/iwlwifi/Kconfig b/drivers/net/wireless/intel/iwlwifi/Kconfig index a647a406b87bea854d73e1de1d851b6376b95e03..b20409f8c13ab554d84106212940a9b5e356cd2c 100644 --- a/drivers/net/wireless/intel/iwlwifi/Kconfig +++ b/drivers/net/wireless/intel/iwlwifi/Kconfig @@ -140,6 +140,7 @@ config IWLMEI depends on INTEL_MEI depends on PM depends on CFG80211 + depends on BROKEN help Enables the iwlmei kernel module. diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c index 8ff967edc8f0a8c4cf632646346bf953f0c24dfc..110fda65bd212fee2220c847216acd0cfd5d408a 100644 --- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c +++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c @@ -56,13 +56,16 @@ #define IWL_BZ_A_GF4_A_FW_PRE "iwlwifi-bz-a0-gf4-a0-" #define IWL_BZ_A_MR_A_FW_PRE "iwlwifi-bz-a0-mr-a0-" #define IWL_BZ_A_FM_A_FW_PRE "iwlwifi-bz-a0-fm-a0-" +#define IWL_BZ_A_FM4_A_FW_PRE "iwlwifi-bz-a0-fm4-a0-" #define IWL_GL_A_FM_A_FW_PRE "iwlwifi-gl-a0-fm-a0-" +#define IWL_GL_B_FM_B_FW_PRE "iwlwifi-gl-b0-fm-b0-" #define IWL_BZ_Z_GF_A_FW_PRE "iwlwifi-bz-z0-gf-a0-" #define IWL_BNJ_A_FM_A_FW_PRE "iwlwifi-BzBnj-a0-fm-a0-" #define IWL_BNJ_A_FM4_A_FW_PRE "iwlwifi-BzBnj-a0-fm4-a0-" #define IWL_BNJ_A_GF_A_FW_PRE "iwlwifi-BzBnj-a0-gf-a0-" #define IWL_BNJ_A_GF4_A_FW_PRE "iwlwifi-BzBnj-a0-gf4-a0-" #define IWL_BNJ_A_HR_B_FW_PRE "iwlwifi-BzBnj-a0-hr-b0-" +#define IWL_BNJ_B_FM_B_FW_PRE "iwlwifi-BzBnj-b0-fm-b0-" #define IWL_QU_B_HR_B_MODULE_FIRMWARE(api) \ @@ -119,8 +122,12 @@ IWL_BZ_A_MR_A_FW_PRE __stringify(api) ".ucode" #define IWL_BZ_A_FM_A_MODULE_FIRMWARE(api) \ IWL_BZ_A_FM_A_FW_PRE __stringify(api) ".ucode" +#define IWL_BZ_A_FM4_A_MODULE_FIRMWARE(api) \ + IWL_BZ_A_FM4_A_FW_PRE __stringify(api) ".ucode" #define IWL_GL_A_FM_A_MODULE_FIRMWARE(api) \ IWL_GL_A_FM_A_FW_PRE __stringify(api) ".ucode" +#define IWL_GL_B_FM_B_MODULE_FIRMWARE(api) \ + IWL_GL_B_FM_B_FW_PRE __stringify(api) ".ucode" #define IWL_BNJ_A_FM_A_MODULE_FIRMWARE(api) \ IWL_BNJ_A_FM_A_FW_PRE __stringify(api) ".ucode" #define IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(api) \ @@ -131,6 +138,8 @@ IWL_BNJ_A_GF4_A_FW_PRE __stringify(api) ".ucode" #define IWL_BNJ_A_HR_B_MODULE_FIRMWARE(api) \ IWL_BNJ_A_HR_B_FW_PRE __stringify(api) ".ucode" +#define IWL_BNJ_B_FM_B_MODULE_FIRMWARE(api) \ + IWL_BNJ_B_FM_B_FW_PRE __stringify(api) ".ucode" static const struct iwl_base_params iwl_22000_base_params = { .eeprom_size = OTP_LOW_IMAGE_SIZE_32K, @@ -240,7 +249,7 @@ static const struct iwl_ht_params iwl_22000_ht_params = { }, \ } -#define IWL_DEVICE_BZ_COMMON \ +#define IWL_DEVICE_BZ \ .ucode_api_max = IWL_22000_UCODE_API_MAX, \ .ucode_api_min = IWL_22000_UCODE_API_MIN, \ .led_mode = IWL_LED_RF_STATE, \ @@ -276,16 +285,13 @@ static const struct iwl_ht_params iwl_22000_ht_params = { .addr = LDBG_M2S_BUF_WRAP_CNT, \ .mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK, \ }, \ - } - -#define IWL_DEVICE_BZ \ - IWL_DEVICE_BZ_COMMON, \ + }, \ .trans.umac_prph_offset = 0x300000, \ .trans.device_family = IWL_DEVICE_FAMILY_BZ, \ .trans.base_params = &iwl_ax210_base_params, \ .min_txq_size = 128, \ .gp2_reg_addr = 0xd02c68, \ - .min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT, \ + .min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT, \ .mon_dram_regs = { \ .write_ptr = { \ .addr = DBGC_CUR_DBGBUF_STATUS, \ @@ -926,6 +932,13 @@ const struct iwl_cfg iwl_cfg_bz_a0_fm_a0 = { .num_rbds = IWL_NUM_RBDS_AX210_HE, }; +const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0 = { + .fw_name_pre = IWL_BZ_A_FM4_A_FW_PRE, + .uhb_supported = true, + IWL_DEVICE_BZ, + .num_rbds = IWL_NUM_RBDS_AX210_HE, +}; + const struct iwl_cfg iwl_cfg_gl_a0_fm_a0 = { .fw_name_pre = IWL_GL_A_FM_A_FW_PRE, .uhb_supported = true, @@ -933,6 +946,13 @@ const struct iwl_cfg iwl_cfg_gl_a0_fm_a0 = { .num_rbds = IWL_NUM_RBDS_AX210_HE, }; +const struct iwl_cfg iwl_cfg_gl_b0_fm_b0 = { + .fw_name_pre = IWL_GL_B_FM_B_FW_PRE, + .uhb_supported = true, + IWL_DEVICE_BZ, + .num_rbds = IWL_NUM_RBDS_AX210_HE, +}; + const struct iwl_cfg iwl_cfg_bz_z0_gf_a0 = { .fw_name_pre = IWL_BZ_Z_GF_A_FW_PRE, .uhb_supported = true, @@ -974,6 +994,13 @@ const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0 = { IWL_DEVICE_BZ, .num_rbds = IWL_NUM_RBDS_AX210_HE, }; + +const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0 = { + .fw_name_pre = IWL_BNJ_B_FM_B_FW_PRE, + .uhb_supported = true, + IWL_DEVICE_BZ, + .num_rbds = IWL_NUM_RBDS_AX210_HE, +}; MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QNJ_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); @@ -1007,3 +1034,6 @@ MODULE_FIRMWARE(IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_BNJ_A_GF_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_BNJ_A_GF4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); MODULE_FIRMWARE(IWL_BNJ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); +MODULE_FIRMWARE(IWL_BZ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); +MODULE_FIRMWARE(IWL_GL_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); +MODULE_FIRMWARE(IWL_BNJ_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX)); diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/agn.h b/drivers/net/wireless/intel/iwlwifi/dvm/agn.h index 411a6f6638b43e281811d26231da980f1de13fc4..fefaa414272b3a75ef0bffa1a36a7a4f2724b37f 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/agn.h +++ b/drivers/net/wireless/intel/iwlwifi/dvm/agn.h @@ -112,7 +112,7 @@ int iwl_load_ucode_wait_alive(struct iwl_priv *priv, enum iwl_ucode_type ucode_type); int iwl_send_calib_results(struct iwl_priv *priv); int iwl_calib_set(struct iwl_priv *priv, - const struct iwl_calib_hdr *cmd, int len); + const struct iwl_calib_cmd *cmd, size_t len); void iwl_calib_free_results(struct iwl_priv *priv); int iwl_dump_nic_event_log(struct iwl_priv *priv, bool full_log, char **buf); diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c index a11884fa254b8b42229be495b59d7e1adad808dd..f488620d2844dca4c1e39688f50d549844f83dcf 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c @@ -19,8 +19,7 @@ struct iwl_calib_result { struct list_head list; size_t cmd_len; - struct iwl_calib_hdr hdr; - /* data follows */ + struct iwl_calib_cmd cmd; }; struct statistics_general_data { @@ -43,12 +42,12 @@ int iwl_send_calib_results(struct iwl_priv *priv) int ret; hcmd.len[0] = res->cmd_len; - hcmd.data[0] = &res->hdr; + hcmd.data[0] = &res->cmd; hcmd.dataflags[0] = IWL_HCMD_DFL_NOCOPY; ret = iwl_dvm_send_cmd(priv, &hcmd); if (ret) { IWL_ERR(priv, "Error %d on calib cmd %d\n", - ret, res->hdr.op_code); + ret, res->cmd.hdr.op_code); return ret; } } @@ -57,19 +56,22 @@ int iwl_send_calib_results(struct iwl_priv *priv) } int iwl_calib_set(struct iwl_priv *priv, - const struct iwl_calib_hdr *cmd, int len) + const struct iwl_calib_cmd *cmd, size_t len) { struct iwl_calib_result *res, *tmp; - res = kmalloc(sizeof(*res) + len - sizeof(struct iwl_calib_hdr), - GFP_ATOMIC); + if (check_sub_overflow(len, sizeof(*cmd), &len)) + return -ENOMEM; + + res = kmalloc(struct_size(res, cmd.data, len), GFP_ATOMIC); if (!res) return -ENOMEM; - memcpy(&res->hdr, cmd, len); - res->cmd_len = len; + res->cmd = *cmd; + memcpy(res->cmd.data, cmd->data, len); + res->cmd_len = struct_size(cmd, data, len); list_for_each_entry(tmp, &priv->calib_results, list) { - if (tmp->hdr.op_code == res->hdr.op_code) { + if (tmp->cmd.hdr.op_code == res->cmd.hdr.op_code) { list_replace(&tmp->list, &res->list); kfree(tmp); return 0; diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/dev.h b/drivers/net/wireless/intel/iwlwifi/dvm/dev.h index bbd57409120125ce71c68513e7192df15ebd1c90..1a9eadace188a553a880a69b92449ac2ed3f29af 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/dev.h +++ b/drivers/net/wireless/intel/iwlwifi/dvm/dev.h @@ -696,6 +696,7 @@ struct iwl_priv { /* Scan related variables */ unsigned long scan_start; unsigned long scan_start_tsf; + size_t scan_cmd_size; void *scan_cmd; enum nl80211_band scan_band; struct cfg80211_scan_request *scan_request; diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c index baffa1cbe8fc4014821a06acd7994cb51c63ba46..687c906a9d727cf5b126944e0dd8c59a07054c9f 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c @@ -2,7 +2,7 @@ /****************************************************************************** * * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved. - * Copyright (C) 2019 - 2020 Intel Corporation + * Copyright (C) 2019 - 2020, 2022 Intel Corporation *****************************************************************************/ #include #include @@ -1242,7 +1242,7 @@ static int rs_switch_to_mimo2(struct iwl_priv *priv, if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported) return -1; - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) return -1; /* Need both Tx chains/antennas to support MIMO */ @@ -1297,7 +1297,7 @@ static int rs_switch_to_mimo3(struct iwl_priv *priv, if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported) return -1; - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) return -1; /* Need both Tx chains/antennas to support MIMO */ diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/scan.c b/drivers/net/wireless/intel/iwlwifi/dvm/scan.c index 2d38227dfdd20b8936cefca3d8567c3d9b49bb99..a7e85c5c8c7267497b4715166b8212bd9786478f 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/scan.c @@ -626,7 +626,7 @@ static int iwlagn_request_scan(struct iwl_priv *priv, struct ieee80211_vif *vif) u8 active_chains; u8 scan_tx_antennas = priv->nvm_data->valid_tx_ant; int ret; - int scan_cmd_size = sizeof(struct iwl_scan_cmd) + + size_t scan_cmd_size = sizeof(struct iwl_scan_cmd) + MAX_SCAN_CHANNEL * sizeof(struct iwl_scan_channel) + priv->fw->ucode_capa.max_probe_length; const u8 *ssid = NULL; @@ -649,9 +649,15 @@ static int iwlagn_request_scan(struct iwl_priv *priv, struct ieee80211_vif *vif) "fail to allocate memory for scan\n"); return -ENOMEM; } + priv->scan_cmd_size = scan_cmd_size; + } + if (priv->scan_cmd_size < scan_cmd_size) { + IWL_DEBUG_SCAN(priv, + "memory needed for scan grew unexpectedly\n"); + return -ENOMEM; } scan = priv->scan_cmd; - memset(scan, 0, scan_cmd_size); + memset(scan, 0, priv->scan_cmd_size); scan->quiet_plcp_th = IWL_PLCP_QUIET_THRESH; scan->quiet_time = IWL_ACTIVE_QUIET_TIME; diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c index 476068c0abb7fd7e5790d0c6d4c354315b522311..cef43cf80620a370a497ed903a8b566df64c3fda 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /****************************************************************************** * - * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved. + * Copyright(c) 2003 - 2014, 2022 Intel Corporation. All rights reserved. * * Portions of this file are derived from the ipw3945 project, as well * as portions of the ieee80211 subsystem header files. @@ -161,12 +161,12 @@ static void iwl_sta_calc_ht_flags(struct iwl_priv *priv, IWL_DEBUG_INFO(priv, "STA %pM SM PS mode: %s\n", sta->addr, - (sta->smps_mode == IEEE80211_SMPS_STATIC) ? + (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) ? "static" : - (sta->smps_mode == IEEE80211_SMPS_DYNAMIC) ? + (sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) ? "dynamic" : "disabled"); - switch (sta->smps_mode) { + switch (sta->deflink.smps_mode) { case IEEE80211_SMPS_STATIC: *flags |= STA_FLG_MIMO_DIS_MSK; break; @@ -176,7 +176,7 @@ static void iwl_sta_calc_ht_flags(struct iwl_priv *priv, case IEEE80211_SMPS_OFF: break; default: - IWL_WARN(priv, "Invalid MIMO PS mode %d\n", sta->smps_mode); + IWL_WARN(priv, "Invalid MIMO PS mode %d\n", sta->deflink.smps_mode); break; } diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c index 4b27a53d0bb4aa9206b6bcec94fce89388bebd16..bb13ca5d666cbbd8d4a270a6bc289a804aab454c 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c @@ -356,18 +356,18 @@ static bool iwlagn_wait_calib(struct iwl_notif_wait_data *notif_wait, struct iwl_rx_packet *pkt, void *data) { struct iwl_priv *priv = data; - struct iwl_calib_hdr *hdr; + struct iwl_calib_cmd *cmd; if (pkt->hdr.cmd != CALIBRATION_RES_NOTIFICATION) { WARN_ON(pkt->hdr.cmd != CALIBRATION_COMPLETE_NOTIFICATION); return true; } - hdr = (struct iwl_calib_hdr *)pkt->data; + cmd = (struct iwl_calib_cmd *)pkt->data; - if (iwl_calib_set(priv, hdr, iwl_rx_packet_payload_len(pkt))) + if (iwl_calib_set(priv, cmd, iwl_rx_packet_payload_len(pkt))) IWL_ERR(priv, "Failed to record calibration data %d\n", - hdr->op_code); + cmd->hdr.op_code); return false; } diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h index c78d2f1c722cb34e9fa32a833a5182585a8225ea..0b052c2e563a84828d1b41754703b3910c7572ac 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2020 Intel Corporation + * Copyright (C) 2018-2022 Intel Corporation */ #ifndef __iwl_fw_api_commands_h__ #define __iwl_fw_api_commands_h__ @@ -20,6 +20,8 @@ * &enum iwl_phy_ops_subcmd_ids * @DATA_PATH_GROUP: data path group, uses command IDs from * &enum iwl_data_path_subcmd_ids + * @SCAN_GROUP: scan group, uses command IDs from + * &enum iwl_scan_subcmd_ids * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids * @LOCATION_GROUP: location group, uses command IDs from * &enum iwl_location_subcmd_ids @@ -36,6 +38,7 @@ enum iwl_mvm_command_groups { MAC_CONF_GROUP = 0x3, PHY_OPS_GROUP = 0x4, DATA_PATH_GROUP = 0x5, + SCAN_GROUP = 0x6, NAN_GROUP = 0x7, LOCATION_GROUP = 0x8, PROT_OFFLOAD_GROUP = 0xb, diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h index 4cd9ab23954edd4562de8ba63c8319d78b7b1371..df0833890e55aae68b4df0384922c0a1f0718fe9 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2021 Intel Corporation + * Copyright (C) 2012-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015-2017 Intel Deutschland GmbH */ @@ -766,6 +766,65 @@ struct iwl_wowlan_status_v12 { u8 wake_packet[]; /* can be truncated from _length to _bufsize */ } __packed; /* WOWLAN_STATUSES_RSP_API_S_VER_12 */ +/** + * struct iwl_wowlan_info_notif - WoWLAN information notification + * @gtk: GTK data + * @igtk: IGTK data + * @replay_ctr: GTK rekey replay counter + * @pattern_number: number of the matched patterns + * @reserved1: reserved + * @qos_seq_ctr: QoS sequence counters to use next + * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason + * @num_of_gtk_rekeys: number of GTK rekeys + * @transmitted_ndps: number of transmitted neighbor discovery packets + * @received_beacons: number of received beacons + * @wake_packet_length: wakeup packet length + * @wake_packet_bufsize: wakeup packet buffer size + * @tid_tear_down: bit mask of tids whose BA sessions were closed + * in suspend state + * @station_id: station id + * @reserved2: reserved + */ +struct iwl_wowlan_info_notif { + struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM]; + struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM]; + __le64 replay_ctr; + __le16 pattern_number; + __le16 reserved1; + __le16 qos_seq_ctr[8]; + __le32 wakeup_reasons; + __le32 num_of_gtk_rekeys; + __le32 transmitted_ndps; + __le32 received_beacons; + __le32 wake_packet_length; + __le32 wake_packet_bufsize; + u8 tid_tear_down; + u8 station_id; + u8 reserved2[2]; +} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_1 */ + +/** + * struct iwl_wowlan_wake_pkt_notif - WoWLAN wake packet notification + * @wake_packet_length: wakeup packet length + * @station_id: station id + * @reserved: unused + * @wake_packet: wakeup packet + */ +struct iwl_wowlan_wake_pkt_notif { + __le32 wake_packet_length; + u8 station_id; + u8 reserved[3]; + u8 wake_packet[1]; +} __packed; /* WOWLAN_WAKE_PKT_NTFY_API_S_VER_1 */ + +/** + * struct iwl_mvm_d3_end_notif - d3 end notification + * @flags: See &enum iwl_d0i3_flags + */ +struct iwl_mvm_d3_end_notif { + __le32 flags; +} __packed; + /* TODO: NetDetect API */ #endif /* __iwl_fw_api_d3_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h index 5204aa94e72ac7fe959e96a5bbec477dc3cfeb50..a0123f81f5d8a301243c545bfcebb4799cfd6a34 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h @@ -3,7 +3,7 @@ * Copyright (C) 2012-2014 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH - * Copyright (C) 2021 Intel Corporation + * Copyright (C) 2021-2022 Intel Corporation */ #ifndef __iwl_fw_api_offload_h__ #define __iwl_fw_api_offload_h__ @@ -12,6 +12,21 @@ * enum iwl_prot_offload_subcmd_ids - protocol offload commands */ enum iwl_prot_offload_subcmd_ids { + /** + * @WOWLAN_WAKE_PKT_NOTIFICATION: Notification in &struct iwl_wowlan_wake_pkt_notif + */ + WOWLAN_WAKE_PKT_NOTIFICATION = 0xFC, + + /** + * @WOWLAN_INFO_NOTIFICATION: Notification in &struct iwl_wowlan_info_notif + */ + WOWLAN_INFO_NOTIFICATION = 0xFD, + + /** + * @D3_END_NOTIFICATION: End D3 state notification + */ + D3_END_NOTIFICATION = 0xFE, + /** * @STORED_BEACON_NTF: &struct iwl_stored_beacon_notif */ diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h index 1989b270862b03b976f257dd32d6d9402753ebb7..74a01888715b144af519c20c26764bd4e018975c 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2021 Intel Corporation + * Copyright (C) 2012-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2015-2017 Intel Deutschland GmbH */ @@ -668,7 +668,7 @@ struct iwl_rx_no_data { __le32 phy_info[2]; __le32 rx_vec[2]; } __packed; /* RX_NO_DATA_NTFY_API_S_VER_1, - TX_NO_DATA_NTFY_API_S_VER_2 */ + RX_NO_DATA_NTFY_API_S_VER_2 */ struct iwl_frame_release { u8 baid; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h index 5543d9cb74c81e2b3833091d9910706c2182dafe..7ba0e3409199bab76bfe89f03431be45e0543684 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2021 Intel Corporation + * Copyright (C) 2012-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -9,6 +9,16 @@ /* Scan Commands, Responses, Notifications */ +/** + * enum iwl_scan_subcmd_ids - scan commands + */ +enum iwl_scan_subcmd_ids { + /** + * @OFFLOAD_MATCH_INFO_NOTIF: &struct iwl_scan_offload_match_info + */ + OFFLOAD_MATCH_INFO_NOTIF = 0xFC, +}; + /* Max number of IEs for direct SSID scans in a command */ #define PROBE_OPTION_MAX 20 @@ -1188,7 +1198,7 @@ struct iwl_scan_offload_profile_match { } __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_2 */ /** - * struct iwl_scan_offload_profiles_query - match results query response + * struct iwl_scan_offload_match_info - match results information * @matched_profiles: bitmap of matched profiles, referencing the * matches passed in the scan offload request * @last_scan_age: age of the last offloaded scan @@ -1200,7 +1210,7 @@ struct iwl_scan_offload_profile_match { * @reserved: reserved * @matches: array of match information, one for each match */ -struct iwl_scan_offload_profiles_query { +struct iwl_scan_offload_match_info { __le32 matched_profiles; __le32 last_scan_age; __le32 n_scans_done; @@ -1210,7 +1220,9 @@ struct iwl_scan_offload_profiles_query { u8 self_recovery; __le16 reserved; struct iwl_scan_offload_profile_match matches[]; -} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 */ +} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 and + * SCAN_OFFLOAD_MATCH_INFO_NOTIFICATION_S_VER_1 + */ /** * struct iwl_umac_scan_iter_complete_notif - notifies end of scanning iteration diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h index f5b556a103e8ce01c0000afd606357c3fe3f6412..cfa5e1b3c3f689abda73370a821a98665d2d68f0 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h @@ -649,13 +649,16 @@ extern const struct iwl_cfg iwl_cfg_bz_a0_gf_a0; extern const struct iwl_cfg iwl_cfg_bz_a0_gf4_a0; extern const struct iwl_cfg iwl_cfg_bz_a0_mr_a0; extern const struct iwl_cfg iwl_cfg_bz_a0_fm_a0; +extern const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0; extern const struct iwl_cfg iwl_cfg_gl_a0_fm_a0; +extern const struct iwl_cfg iwl_cfg_gl_b0_fm_b0; extern const struct iwl_cfg iwl_cfg_bz_z0_gf_a0; extern const struct iwl_cfg iwl_cfg_bnj_a0_fm_a0; extern const struct iwl_cfg iwl_cfg_bnj_a0_fm4_a0; extern const struct iwl_cfg iwl_cfg_bnj_a0_gf_a0; extern const struct iwl_cfg iwl_cfg_bnj_a0_gf4_a0; extern const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0; +extern const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0; #endif /* CONFIG_IWLMVM */ #endif /* __IWL_CONFIG_H__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c index aeb0015b73d2b3ea1a7e82cf7f64a1c8c756557d..919b1f478b4ce4e6e733b1556b9753fa4bc14d5f 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c @@ -1427,7 +1427,7 @@ struct iwl_wowlan_status_data { u8 flags; } igtk; - u8 wake_packet[]; + u8 *wake_packet; }; static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm, @@ -1480,7 +1480,7 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm, if (reasons & IWL_WOWLAN_WAKEUP_BY_REM_WAKE_WAKEUP_PACKET) wakeup.tcp_match = true; - if (status->wake_packet_bufsize) { + if (status->wake_packet) { int pktsize = status->wake_packet_bufsize; int pktlen = status->wake_packet_length; const u8 *pktdata = status->wake_packet; @@ -1944,57 +1944,6 @@ out: return true; } -/* Occasionally, templates would be nice. This is one of those times ... */ -#define iwl_mvm_parse_wowlan_status_common(_ver) \ -static struct iwl_wowlan_status_data * \ -iwl_mvm_parse_wowlan_status_common_ ## _ver(struct iwl_mvm *mvm, \ - struct iwl_wowlan_status_ ##_ver *data,\ - int len) \ -{ \ - struct iwl_wowlan_status_data *status; \ - int data_size, i; \ - \ - if (len < sizeof(*data)) { \ - IWL_ERR(mvm, "Invalid WoWLAN status response!\n"); \ - return NULL; \ - } \ - \ - data_size = ALIGN(le32_to_cpu(data->wake_packet_bufsize), 4); \ - if (len != sizeof(*data) + data_size) { \ - IWL_ERR(mvm, "Invalid WoWLAN status response!\n"); \ - return NULL; \ - } \ - \ - status = kzalloc(sizeof(*status) + data_size, GFP_KERNEL); \ - if (!status) \ - return NULL; \ - \ - /* copy all the common fields */ \ - status->replay_ctr = le64_to_cpu(data->replay_ctr); \ - status->pattern_number = le16_to_cpu(data->pattern_number); \ - status->non_qos_seq_ctr = le16_to_cpu(data->non_qos_seq_ctr); \ - for (i = 0; i < 8; i++) \ - status->qos_seq_ctr[i] = \ - le16_to_cpu(data->qos_seq_ctr[i]); \ - status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons); \ - status->num_of_gtk_rekeys = \ - le32_to_cpu(data->num_of_gtk_rekeys); \ - status->received_beacons = le32_to_cpu(data->received_beacons); \ - status->wake_packet_length = \ - le32_to_cpu(data->wake_packet_length); \ - status->wake_packet_bufsize = \ - le32_to_cpu(data->wake_packet_bufsize); \ - memcpy(status->wake_packet, data->wake_packet, \ - status->wake_packet_bufsize); \ - \ - return status; \ -} - -iwl_mvm_parse_wowlan_status_common(v6) -iwl_mvm_parse_wowlan_status_common(v7) -iwl_mvm_parse_wowlan_status_common(v9) -iwl_mvm_parse_wowlan_status_common(v12) - static void iwl_mvm_convert_gtk_v2(struct iwl_wowlan_status_data *status, struct iwl_wowlan_gtk_status_v2 *data) { @@ -2054,6 +2003,96 @@ static void iwl_mvm_convert_igtk(struct iwl_wowlan_status_data *status, ((u64)ipn[0] << 40); } +static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm, + struct iwl_wowlan_info_notif *data, + struct iwl_wowlan_status_data *status, + u32 len) +{ + u32 i; + + if (len < sizeof(*data)) { + IWL_ERR(mvm, "Invalid WoWLAN info notification!\n"); + status = NULL; + return; + } + + iwl_mvm_convert_key_counters_v5(status, &data->gtk[0].sc); + iwl_mvm_convert_gtk_v3(status, &data->gtk[0]); + iwl_mvm_convert_igtk(status, &data->igtk[0]); + + status->replay_ctr = le64_to_cpu(data->replay_ctr); + status->pattern_number = le16_to_cpu(data->pattern_number); + for (i = 0; i < IWL_MAX_TID_COUNT; i++) + status->qos_seq_ctr[i] = + le16_to_cpu(data->qos_seq_ctr[i]); + status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons); + status->num_of_gtk_rekeys = + le32_to_cpu(data->num_of_gtk_rekeys); + status->received_beacons = le32_to_cpu(data->received_beacons); + status->tid_tear_down = data->tid_tear_down; +} + +/* Occasionally, templates would be nice. This is one of those times ... */ +#define iwl_mvm_parse_wowlan_status_common(_ver) \ +static struct iwl_wowlan_status_data * \ +iwl_mvm_parse_wowlan_status_common_ ## _ver(struct iwl_mvm *mvm, \ + struct iwl_wowlan_status_ ##_ver *data,\ + int len) \ +{ \ + struct iwl_wowlan_status_data *status; \ + int data_size, i; \ + \ + if (len < sizeof(*data)) { \ + IWL_ERR(mvm, "Invalid WoWLAN status response!\n"); \ + return NULL; \ + } \ + \ + data_size = ALIGN(le32_to_cpu(data->wake_packet_bufsize), 4); \ + if (len != sizeof(*data) + data_size) { \ + IWL_ERR(mvm, "Invalid WoWLAN status response!\n"); \ + return NULL; \ + } \ + \ + status = kzalloc(sizeof(*status), GFP_KERNEL); \ + if (!status) \ + return NULL; \ + \ + /* copy all the common fields */ \ + status->replay_ctr = le64_to_cpu(data->replay_ctr); \ + status->pattern_number = le16_to_cpu(data->pattern_number); \ + status->non_qos_seq_ctr = le16_to_cpu(data->non_qos_seq_ctr); \ + for (i = 0; i < 8; i++) \ + status->qos_seq_ctr[i] = \ + le16_to_cpu(data->qos_seq_ctr[i]); \ + status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons); \ + status->num_of_gtk_rekeys = \ + le32_to_cpu(data->num_of_gtk_rekeys); \ + status->received_beacons = le32_to_cpu(data->received_beacons); \ + status->wake_packet_length = \ + le32_to_cpu(data->wake_packet_length); \ + status->wake_packet_bufsize = \ + le32_to_cpu(data->wake_packet_bufsize); \ + if (status->wake_packet_bufsize) { \ + status->wake_packet = \ + kmemdup(data->wake_packet, \ + status->wake_packet_bufsize, \ + GFP_KERNEL); \ + if (!status->wake_packet) { \ + kfree(status); \ + return NULL; \ + } \ + } else { \ + status->wake_packet = NULL; \ + } \ + \ + return status; \ +} + +iwl_mvm_parse_wowlan_status_common(v6) +iwl_mvm_parse_wowlan_status_common(v7) +iwl_mvm_parse_wowlan_status_common(v9) +iwl_mvm_parse_wowlan_status_common(v12) + static struct iwl_wowlan_status_data * iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id) { @@ -2173,36 +2212,15 @@ out_free_resp: return status; } -static struct iwl_wowlan_status_data * -iwl_mvm_get_wakeup_status(struct iwl_mvm *mvm, u8 sta_id) -{ - u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, OFFLOADS_QUERY_CMD, - IWL_FW_CMD_VER_UNKNOWN); - __le32 station_id = cpu_to_le32(sta_id); - u32 cmd_size = cmd_ver != IWL_FW_CMD_VER_UNKNOWN ? sizeof(station_id) : 0; - - if (!mvm->net_detect) { - /* only for tracing for now */ - int ret = iwl_mvm_send_cmd_pdu(mvm, OFFLOADS_QUERY_CMD, 0, - cmd_size, &station_id); - if (ret) - IWL_ERR(mvm, "failed to query offload statistics (%d)\n", ret); - } - - return iwl_mvm_send_wowlan_get_status(mvm, sta_id); -} - /* releases the MVM mutex */ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm, - struct ieee80211_vif *vif) + struct ieee80211_vif *vif, + struct iwl_wowlan_status_data *status) { - struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); - struct iwl_wowlan_status_data *status; int i; bool keep; struct iwl_mvm_sta *mvm_ap_sta; - status = iwl_mvm_get_wakeup_status(mvm, mvmvif->ap_sta_id); if (!status) goto out_unlock; @@ -2212,7 +2230,7 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm, /* still at hard-coded place 0 for D3 image */ mvm_ap_sta = iwl_mvm_sta_from_staid_protected(mvm, 0); if (!mvm_ap_sta) - goto out_free; + goto out_unlock; for (i = 0; i < IWL_MAX_TID_COUNT; i++) { u16 seq = status->qos_seq_ctr[i]; @@ -2235,11 +2253,8 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm, keep = iwl_mvm_setup_connection_keep(mvm, vif, status); - kfree(status); return keep; -out_free: - kfree(status); out_unlock: mutex_unlock(&mvm->mutex); return false; @@ -2248,16 +2263,16 @@ out_unlock: #define ND_QUERY_BUF_LEN (sizeof(struct iwl_scan_offload_profile_match) * \ IWL_SCAN_MAX_PROFILES) -struct iwl_mvm_nd_query_results { +struct iwl_mvm_nd_results { u32 matched_profiles; u8 matches[ND_QUERY_BUF_LEN]; }; static int iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm, - struct iwl_mvm_nd_query_results *results) + struct iwl_mvm_nd_results *results) { - struct iwl_scan_offload_profiles_query *query; + struct iwl_scan_offload_match_info *query; struct iwl_host_cmd cmd = { .id = SCAN_OFFLOAD_PROFILES_QUERY_CMD, .flags = CMD_WANT_SKB, @@ -2274,7 +2289,7 @@ iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm, if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) { - query_len = sizeof(struct iwl_scan_offload_profiles_query); + query_len = sizeof(struct iwl_scan_offload_match_info); matches_len = sizeof(struct iwl_scan_offload_profile_match) * max_profiles; } else { @@ -2305,7 +2320,7 @@ out_free_resp: } static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm, - struct iwl_mvm_nd_query_results *query, + struct iwl_mvm_nd_results *results, int idx) { int n_chans = 0, i; @@ -2313,13 +2328,13 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm, if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) { struct iwl_scan_offload_profile_match *matches = - (struct iwl_scan_offload_profile_match *)query->matches; + (void *)results->matches; for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN; i++) n_chans += hweight8(matches[idx].matching_channels[i]); } else { struct iwl_scan_offload_profile_match_v1 *matches = - (struct iwl_scan_offload_profile_match_v1 *)query->matches; + (void *)results->matches; for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1; i++) n_chans += hweight8(matches[idx].matching_channels[i]); @@ -2329,7 +2344,7 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm, } static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm, - struct iwl_mvm_nd_query_results *query, + struct iwl_mvm_nd_results *results, struct cfg80211_wowlan_nd_match *match, int idx) { @@ -2338,7 +2353,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm, if (fw_has_api(&mvm->fw->ucode_capa, IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) { struct iwl_scan_offload_profile_match *matches = - (struct iwl_scan_offload_profile_match *)query->matches; + (void *)results->matches; for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN * 8; i++) if (matches[idx].matching_channels[i / 8] & (BIT(i % 8))) @@ -2346,7 +2361,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm, mvm->nd_channels[i]->center_freq; } else { struct iwl_scan_offload_profile_match_v1 *matches = - (struct iwl_scan_offload_profile_match_v1 *)query->matches; + (void *)results->matches; for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1 * 8; i++) if (matches[idx].matching_channels[i / 8] & (BIT(i % 8))) @@ -2355,25 +2370,50 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm, } } +/** + * enum iwl_d3_notif - d3 notifications + * @IWL_D3_NOTIF_WOWLAN_INFO: WOWLAN_INFO_NOTIF was received + * @IWL_D3_NOTIF_WOWLAN_WAKE_PKT: WOWLAN_WAKE_PKT_NOTIF was received + * @IWL_D3_NOTIF_PROT_OFFLOAD: PROT_OFFLOAD_NOTIF was received + * @IWL_D3_ND_MATCH_INFO: OFFLOAD_MATCH_INFO_NOTIF was received + * @IWL_D3_NOTIF_D3_END_NOTIF: D3_END_NOTIF was received + */ +enum iwl_d3_notif { + IWL_D3_NOTIF_WOWLAN_INFO = BIT(0), + IWL_D3_NOTIF_WOWLAN_WAKE_PKT = BIT(1), + IWL_D3_NOTIF_PROT_OFFLOAD = BIT(2), + IWL_D3_ND_MATCH_INFO = BIT(3), + IWL_D3_NOTIF_D3_END_NOTIF = BIT(4) +}; + +/* manage d3 resume data */ +struct iwl_d3_data { + struct iwl_wowlan_status_data *status; + bool test; + u32 d3_end_flags; + u32 notif_expected; /* bitmap - see &enum iwl_d3_notif */ + u32 notif_received; /* bitmap - see &enum iwl_d3_notif */ + struct iwl_mvm_nd_results *nd_results; + bool nd_results_valid; +}; + static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm, - struct ieee80211_vif *vif) + struct ieee80211_vif *vif, + struct iwl_d3_data *d3_data) { struct cfg80211_wowlan_nd_info *net_detect = NULL; struct cfg80211_wowlan_wakeup wakeup = { .pattern_idx = -1, }; struct cfg80211_wowlan_wakeup *wakeup_report = &wakeup; - struct iwl_wowlan_status_data *status; - struct iwl_mvm_nd_query_results query; unsigned long matched_profiles; u32 reasons = 0; int i, n_matches, ret; - status = iwl_mvm_get_wakeup_status(mvm, IWL_MVM_INVALID_STA); - if (status) { - reasons = status->wakeup_reasons; - kfree(status); - } + if (WARN_ON(!d3_data || !d3_data->status)) + goto out; + + reasons = d3_data->status->wakeup_reasons; if (reasons & IWL_WOWLAN_WAKEUP_BY_RFKILL_DEASSERTED) wakeup.rfkill_release = true; @@ -2381,13 +2421,22 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm, if (reasons != IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS) goto out; - ret = iwl_mvm_netdetect_query_results(mvm, &query); - if (ret || !query.matched_profiles) { + if (!iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP, + WOWLAN_INFO_NOTIFICATION, 0)) { + IWL_INFO(mvm, "Query FW for ND results\n"); + ret = iwl_mvm_netdetect_query_results(mvm, d3_data->nd_results); + + } else { + IWL_INFO(mvm, "Notification based ND results\n"); + ret = d3_data->nd_results_valid ? 0 : -1; + } + + if (ret || !d3_data->nd_results->matched_profiles) { wakeup_report = NULL; goto out; } - matched_profiles = query.matched_profiles; + matched_profiles = d3_data->nd_results->matched_profiles; if (mvm->n_nd_match_sets) { n_matches = hweight_long(matched_profiles); } else { @@ -2404,7 +2453,9 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm, struct cfg80211_wowlan_nd_match *match; int idx, n_channels = 0; - n_channels = iwl_mvm_query_num_match_chans(mvm, &query, i); + n_channels = iwl_mvm_query_num_match_chans(mvm, + d3_data->nd_results, + i); match = kzalloc(struct_size(match, channels, n_channels), GFP_KERNEL); @@ -2424,7 +2475,7 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm, if (mvm->n_nd_channels < n_channels) continue; - iwl_mvm_query_set_freqs(mvm, &query, match, i); + iwl_mvm_query_set_freqs(mvm, d3_data->nd_results, match, i); } out_report_nd: @@ -2504,16 +2555,317 @@ static bool iwl_mvm_check_rt_status(struct iwl_mvm *mvm, return false; } +/* + * This function assumes: + * 1. The mutex is already held. + * 2. The callee functions unlock the mutex. + */ +static bool +iwl_mvm_choose_query_wakeup_reasons(struct iwl_mvm *mvm, + struct ieee80211_vif *vif, + struct iwl_d3_data *d3_data) +{ + lockdep_assert_held(&mvm->mutex); + + /* if FW uses status notification, status shouldn't be NULL here */ + if (!d3_data->status) { + struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); + u8 sta_id = mvm->net_detect ? IWL_MVM_INVALID_STA : mvmvif->ap_sta_id; + + d3_data->status = iwl_mvm_send_wowlan_get_status(mvm, sta_id); + } + + if (mvm->net_detect) { + iwl_mvm_query_netdetect_reasons(mvm, vif, d3_data); + } else { + bool keep = iwl_mvm_query_wakeup_reasons(mvm, vif, + d3_data->status); + +#ifdef CONFIG_IWLWIFI_DEBUGFS + if (keep) + mvm->keep_vif = vif; +#endif + + return keep; + } + return false; +} + +#define IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT (IWL_WOWLAN_WAKEUP_BY_MAGIC_PACKET | \ + IWL_WOWLAN_WAKEUP_BY_PATTERN | \ + IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN |\ + IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN_WILDCARD |\ + IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN |\ + IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN_WILDCARD) + +static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm, + struct iwl_wowlan_wake_pkt_notif *notif, + struct iwl_wowlan_status_data *status, + u32 len) +{ + u32 data_size, packet_len = le32_to_cpu(notif->wake_packet_length); + + if (len < sizeof(*notif)) { + IWL_ERR(mvm, "Invalid WoWLAN wake packet notification!\n"); + return -EIO; + } + + if (WARN_ON(!status)) { + IWL_ERR(mvm, "Got wake packet notification but wowlan status data is NULL\n"); + return -EIO; + } + + if (WARN_ON(!(status->wakeup_reasons & + IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT))) { + IWL_ERR(mvm, "Got wakeup packet but wakeup reason is %x\n", + status->wakeup_reasons); + return -EIO; + } + + data_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif, wake_packet); + + /* data_size got the padding from the notification, remove it. */ + if (packet_len < data_size) + data_size = packet_len; + + status->wake_packet = kmemdup(notif->wake_packet, data_size, + GFP_ATOMIC); + + if (!status->wake_packet) + return -ENOMEM; + + status->wake_packet_length = packet_len; + status->wake_packet_bufsize = data_size; + + return 0; +} + +static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm, + struct iwl_d3_data *d3_data, + struct iwl_scan_offload_match_info *notif, + u32 len) +{ + struct iwl_wowlan_status_data *status = d3_data->status; + struct ieee80211_vif *vif = iwl_mvm_get_bss_vif(mvm); + struct iwl_mvm_nd_results *results = d3_data->nd_results; + size_t i, matches_len = sizeof(struct iwl_scan_offload_profile_match) * + iwl_umac_scan_get_max_profiles(mvm->fw); + + if (IS_ERR_OR_NULL(vif)) + return; + + if (len < sizeof(struct iwl_scan_offload_match_info)) { + IWL_ERR(mvm, "Invalid scan match info notification\n"); + return; + } + + if (!mvm->net_detect) { + IWL_ERR(mvm, "Unexpected scan match info notification\n"); + return; + } + + if (!status || status->wakeup_reasons != IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS) { + IWL_ERR(mvm, + "Ignore scan match info notification: no reason\n"); + return; + } + +#ifdef CONFIG_IWLWIFI_DEBUGFS + mvm->last_netdetect_scans = le32_to_cpu(notif->n_scans_done); +#endif + + results->matched_profiles = le32_to_cpu(notif->matched_profiles); + IWL_INFO(mvm, "number of matched profiles=%u\n", + results->matched_profiles); + + if (results->matched_profiles) { + memcpy(results->matches, notif->matches, matches_len); + d3_data->nd_results_valid = TRUE; + } + + /* no scan should be active at this point */ + mvm->scan_status = 0; + for (i = 0; i < mvm->max_scans; i++) + mvm->scan_uid_status[i] = 0; +} + +static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait, + struct iwl_rx_packet *pkt, void *data) +{ + struct iwl_mvm *mvm = + container_of(notif_wait, struct iwl_mvm, notif_wait); + struct iwl_d3_data *d3_data = data; + u32 len; + int ret; + + switch (WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)) { + case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION): { + struct iwl_wowlan_info_notif *notif = (void *)pkt->data; + + if (d3_data->notif_received & IWL_D3_NOTIF_WOWLAN_INFO) { + /* We might get two notifications due to dual bss */ + IWL_DEBUG_WOWLAN(mvm, + "Got additional wowlan info notification\n"); + break; + } + + d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_INFO; + len = iwl_rx_packet_payload_len(pkt); + iwl_mvm_parse_wowlan_info_notif(mvm, notif, d3_data->status, + len); + if (d3_data->status && + d3_data->status->wakeup_reasons & IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT) + /* We are supposed to get also wake packet notif */ + d3_data->notif_expected |= IWL_D3_NOTIF_WOWLAN_WAKE_PKT; + + break; + } + case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION): { + struct iwl_wowlan_wake_pkt_notif *notif = (void *)pkt->data; + + if (d3_data->notif_received & IWL_D3_NOTIF_WOWLAN_WAKE_PKT) { + /* We shouldn't get two wake packet notifications */ + IWL_ERR(mvm, + "Got additional wowlan wake packet notification\n"); + } else { + d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_WAKE_PKT; + len = iwl_rx_packet_payload_len(pkt); + ret = iwl_mvm_wowlan_store_wake_pkt(mvm, notif, + d3_data->status, + len); + if (ret) + IWL_ERR(mvm, + "Can't parse WOWLAN_WAKE_PKT_NOTIFICATION\n"); + } + + break; + } + case WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF): { + struct iwl_scan_offload_match_info *notif = (void *)pkt->data; + + if (d3_data->notif_received & IWL_D3_ND_MATCH_INFO) { + IWL_ERR(mvm, + "Got additional netdetect match info\n"); + break; + } + + d3_data->notif_received |= IWL_D3_ND_MATCH_INFO; + + /* explicitly set this in the 'expected' as well */ + d3_data->notif_expected |= IWL_D3_ND_MATCH_INFO; + + len = iwl_rx_packet_payload_len(pkt); + iwl_mvm_nd_match_info_handler(mvm, d3_data, notif, len); + break; + } + case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): { + struct iwl_mvm_d3_end_notif *notif = (void *)pkt->data; + + d3_data->d3_end_flags = __le32_to_cpu(notif->flags); + d3_data->notif_received |= IWL_D3_NOTIF_D3_END_NOTIF; + + break; + } + default: + WARN_ON(1); + } + + return d3_data->notif_received == d3_data->notif_expected; +} + +static int iwl_mvm_resume_firmware(struct iwl_mvm *mvm, bool test) +{ + int ret; + enum iwl_d3_status d3_status; + struct iwl_host_cmd cmd = { + .id = D0I3_END_CMD, + .flags = CMD_WANT_SKB | CMD_SEND_IN_D3, + }; + bool reset = fw_has_capa(&mvm->fw->ucode_capa, + IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG); + + ret = iwl_trans_d3_resume(mvm->trans, &d3_status, test, !reset); + if (ret) + return ret; + + if (d3_status != IWL_D3_STATUS_ALIVE) { + IWL_INFO(mvm, "Device was reset during suspend\n"); + return -ENOENT; + } + + /* + * We should trigger resume flow using command only for 22000 family + * AX210 and above don't need the command since they have + * the doorbell interrupt. + */ + if (mvm->trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_22000 && + fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_D0I3_END_FIRST)) { + ret = iwl_mvm_send_cmd(mvm, &cmd); + if (ret < 0) + IWL_ERR(mvm, "Failed to send D0I3_END_CMD first (%d)\n", + ret); + } + + return ret; +} + +#define IWL_MVM_D3_NOTIF_TIMEOUT (HZ / 5) + +static int iwl_mvm_d3_notif_wait(struct iwl_mvm *mvm, + struct iwl_d3_data *d3_data) +{ + static const u16 d3_resume_notif[] = { + WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION), + WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION), + WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF), + WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION) + }; + struct iwl_notification_wait wait_d3_notif; + int ret; + + iwl_init_notification_wait(&mvm->notif_wait, &wait_d3_notif, + d3_resume_notif, ARRAY_SIZE(d3_resume_notif), + iwl_mvm_wait_d3_notif, d3_data); + + ret = iwl_mvm_resume_firmware(mvm, d3_data->test); + if (ret) { + iwl_remove_notification(&mvm->notif_wait, &wait_d3_notif); + return ret; + } + + return iwl_wait_notification(&mvm->notif_wait, &wait_d3_notif, + IWL_MVM_D3_NOTIF_TIMEOUT); +} + +static inline bool iwl_mvm_d3_resume_notif_based(struct iwl_mvm *mvm) +{ + return iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP, + WOWLAN_INFO_NOTIFICATION, 0) && + iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP, + WOWLAN_WAKE_PKT_NOTIFICATION, 0) && + iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP, + D3_END_NOTIFICATION, 0); +} + static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test) { struct ieee80211_vif *vif = NULL; int ret = 1; - enum iwl_d3_status d3_status; - bool keep = false; + struct iwl_mvm_nd_results results = {}; + struct iwl_d3_data d3_data = { + .test = test, + .notif_expected = + IWL_D3_NOTIF_WOWLAN_INFO | + IWL_D3_NOTIF_D3_END_NOTIF, + .nd_results_valid = false, + .nd_results = &results, + }; bool unified_image = fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG); bool d0i3_first = fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_D0I3_END_FIRST); + bool resume_notif_based = iwl_mvm_d3_resume_notif_based(mvm); + bool keep = false; mutex_lock(&mvm->mutex); @@ -2537,54 +2889,30 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test) goto err; } - ret = iwl_trans_d3_resume(mvm->trans, &d3_status, test, !unified_image); - if (ret) - goto err; - - if (d3_status != IWL_D3_STATUS_ALIVE) { - IWL_INFO(mvm, "Device was reset during suspend\n"); - goto err; - } - - if (d0i3_first) { - struct iwl_host_cmd cmd = { - .id = D0I3_END_CMD, - .flags = CMD_WANT_SKB | CMD_SEND_IN_D3, - }; - int len; - - ret = iwl_mvm_send_cmd(mvm, &cmd); - if (ret < 0) { - IWL_ERR(mvm, "Failed to send D0I3_END_CMD first (%d)\n", - ret); + if (resume_notif_based) { + d3_data.status = kzalloc(sizeof(*d3_data.status), GFP_KERNEL); + if (!d3_data.status) { + IWL_ERR(mvm, "Failed to allocate wowlan status\n"); + ret = -ENOMEM; goto err; } - switch (mvm->cmd_ver.d0i3_resp) { - case 0: - break; - case 1: - len = iwl_rx_packet_payload_len(cmd.resp_pkt); - if (len != sizeof(u32)) { - IWL_ERR(mvm, - "Error with D0I3_END_CMD response size (%d)\n", - len); - goto err; - } - if (IWL_D0I3_RESET_REQUIRE & - le32_to_cpu(*(__le32 *)cmd.resp_pkt->data)) { - iwl_write32(mvm->trans, CSR_RESET, - CSR_RESET_REG_FLAG_FORCE_NMI); - iwl_free_resp(&cmd); - } - break; - default: - WARN_ON(1); - } + + ret = iwl_mvm_d3_notif_wait(mvm, &d3_data); + if (ret) + goto err; + } else { + ret = iwl_mvm_resume_firmware(mvm, test); + if (ret < 0) + goto err; } /* after the successful handshake, we're out of D3 */ mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED; + /* when reset is required we can't send these following commands */ + if (d3_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE) + goto query_wakeup_reasons; + /* * Query the current location and source from the D3 firmware so we * can play it back when we re-intiailize the D0 firmware @@ -2598,41 +2926,36 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test) /* Re-configure default SAR profile */ iwl_mvm_sar_select_profile(mvm, 1, 1); - if (mvm->net_detect) { + if (mvm->net_detect && unified_image) { /* If this is a non-unified image, we restart the FW, * so no need to stop the netdetect scan. If that * fails, continue and try to get the wake-up reasons, * but trigger a HW restart by keeping a failure code * in ret. */ - if (unified_image) - ret = iwl_mvm_scan_stop(mvm, IWL_MVM_SCAN_NETDETECT, - false); - - iwl_mvm_query_netdetect_reasons(mvm, vif); - /* has unlocked the mutex, so skip that */ - goto out; - } else { - keep = iwl_mvm_query_wakeup_reasons(mvm, vif); -#ifdef CONFIG_IWLWIFI_DEBUGFS - if (keep) - mvm->keep_vif = vif; -#endif - /* has unlocked the mutex, so skip that */ - goto out_iterate; + ret = iwl_mvm_scan_stop(mvm, IWL_MVM_SCAN_NETDETECT, + false); } +query_wakeup_reasons: + keep = iwl_mvm_choose_query_wakeup_reasons(mvm, vif, &d3_data); + /* has unlocked the mutex, so skip that */ + goto out; + err: - iwl_mvm_free_nd(mvm); mutex_unlock(&mvm->mutex); +out: + if (d3_data.status) + kfree(d3_data.status->wake_packet); + kfree(d3_data.status); + iwl_mvm_free_nd(mvm); -out_iterate: - if (!test) + if (!d3_data.test && !mvm->net_detect) ieee80211_iterate_active_interfaces_mtx(mvm->hw, - IEEE80211_IFACE_ITER_NORMAL, - iwl_mvm_d3_disconnect_iter, keep ? vif : NULL); + IEEE80211_IFACE_ITER_NORMAL, + iwl_mvm_d3_disconnect_iter, + keep ? vif : NULL); -out: clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status); /* no need to reset the device in unified images, if successful */ @@ -2641,9 +2964,14 @@ out: if (d0i3_first) return 0; - ret = iwl_mvm_send_cmd_pdu(mvm, D0I3_END_CMD, 0, 0, NULL); - if (!ret) + if (!iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP, + D3_END_NOTIFICATION, 0)) { + ret = iwl_mvm_send_cmd_pdu(mvm, D0I3_END_CMD, 0, 0, NULL); + if (!ret) + return 0; + } else if (!(d3_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE)) { return 0; + } } /* diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c index c0bd697b080a0909ff6d8c10cbda320ea1559f35..1e8123140973efa0f258cb05f16f81388c88ecba 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c @@ -430,14 +430,16 @@ static ssize_t iwl_dbgfs_amsdu_len_write(struct ieee80211_sta *sta, return -EBUSY; if (amsdu_len) { - mvmsta->orig_amsdu_len = sta->max_amsdu_len; - sta->max_amsdu_len = amsdu_len; - for (i = 0; i < ARRAY_SIZE(sta->max_tid_amsdu_len); i++) - sta->max_tid_amsdu_len[i] = amsdu_len; + mvmsta->orig_amsdu_len = sta->cur->max_amsdu_len; + sta->deflink.agg.max_amsdu_len = amsdu_len; + sta->deflink.agg.max_amsdu_len = amsdu_len; + for (i = 0; i < ARRAY_SIZE(sta->deflink.agg.max_tid_amsdu_len); i++) + sta->deflink.agg.max_tid_amsdu_len[i] = amsdu_len; } else { - sta->max_amsdu_len = mvmsta->orig_amsdu_len; + sta->deflink.agg.max_amsdu_len = mvmsta->orig_amsdu_len; mvmsta->orig_amsdu_len = 0; } + return count; } @@ -451,7 +453,7 @@ static ssize_t iwl_dbgfs_amsdu_len_read(struct file *file, char buf[32]; int pos; - pos = scnprintf(buf, sizeof(buf), "current %d ", sta->max_amsdu_len); + pos = scnprintf(buf, sizeof(buf), "current %d ", sta->cur->max_amsdu_len); pos += scnprintf(buf + pos, sizeof(buf) - pos, "stored %d\n", mvmsta->orig_amsdu_len); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c index ed586e6d7d64b2bf5dfc158f085c2c6596e760af..de0c545d50fd5935eb0a0e6faca315791b67e2d3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c @@ -1099,7 +1099,7 @@ static void iwl_mvm_mac_ctxt_cmd_fill_ap(struct iwl_mvm *mvm, iwl_mvm_mac_ap_iterator, &data); if (data.beacon_device_ts) { - u32 rand = (prandom_u32() % (64 - 36)) + 36; + u32 rand = prandom_u32_max(64 - 36) + 36; mvmvif->ap_beacon_time = data.beacon_device_ts + ieee80211_tu_to_usec(data.beacon_int * rand / 100); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c index 5eb28f8ee87e524560605a14613f2ad81551374e..8464c9b7baf1fe875aa714309909f96018c53f15 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c @@ -1833,8 +1833,8 @@ static void iwl_mvm_parse_ppe(struct iwl_mvm *mvm, * If nss < MAX: we can set zeros in other streams */ if (nss > MAX_HE_SUPP_NSS) { - IWL_INFO(mvm, "Got NSS = %d - trimming to %d\n", nss, - MAX_HE_SUPP_NSS); + IWL_DEBUG_INFO(mvm, "Got NSS = %d - trimming to %d\n", nss, + MAX_HE_SUPP_NSS); nss = MAX_HE_SUPP_NSS; } @@ -3193,7 +3193,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw, NL80211_TDLS_SETUP); } - sta->max_rc_amsdu_len = 1; + sta->deflink.agg.max_rc_amsdu_len = 1; } else if (old_state == IEEE80211_STA_NONE && new_state == IEEE80211_STA_AUTH) { /* @@ -4949,6 +4949,7 @@ static int iwl_mvm_mac_get_survey(struct ieee80211_hw *hw, int idx, static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo) { u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK; + u32 gi_ltf; switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) { case RATE_MCS_CHAN_WIDTH_20: @@ -5019,9 +5020,12 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo) RATE_HT_MCS_INDEX(rate_n_flags) : u32_get_bits(rate_n_flags, RATE_MCS_CODE_MSK); - if (format == RATE_MCS_HE_MSK) { - u32 gi_ltf = u32_get_bits(rate_n_flags, - RATE_MCS_HE_GI_LTF_MSK); + if (rate_n_flags & RATE_MCS_SGI_MSK) + rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI; + + switch (format) { + case RATE_MCS_HE_MSK: + gi_ltf = u32_get_bits(rate_n_flags, RATE_MCS_HE_GI_LTF_MSK); rinfo->flags |= RATE_INFO_FLAGS_HE_MCS; @@ -5060,19 +5064,14 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo) if (rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK) rinfo->he_dcm = 1; - return; - } - - if (rate_n_flags & RATE_MCS_SGI_MSK) - rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI; - - if (format == RATE_MCS_HT_MSK) { + break; + case RATE_MCS_HT_MSK: rinfo->flags |= RATE_INFO_FLAGS_MCS; - - } else if (format == RATE_MCS_VHT_MSK) { + break; + case RATE_MCS_VHT_MSK: rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS; + break; } - } static void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index bf35e130c87652c8f92889200343f8eb7765f7b1..97cba526e4651c589b4877c406af7dfd670362e1 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -860,6 +860,7 @@ struct iwl_mvm { /* Scan status, cmd (pre-allocated) and auxiliary station */ unsigned int scan_status; + size_t scan_cmd_size; void *scan_cmd; struct iwl_mcast_filter_cmd *mcast_filter_cmd; /* For CDB this is low band scan type, for non-CDB - type. */ @@ -1079,7 +1080,6 @@ struct iwl_mvm { struct list_head resp_pasn_list; struct { - u8 d0i3_resp; u8 range_resp; } cmd_ver; @@ -1705,7 +1705,7 @@ int iwl_mvm_update_quotas(struct iwl_mvm *mvm, bool force_upload, int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif, struct cfg80211_scan_request *req, struct ieee80211_scan_ies *ies); -int iwl_mvm_scan_size(struct iwl_mvm *mvm); +size_t iwl_mvm_scan_size(struct iwl_mvm *mvm); int iwl_mvm_scan_stop(struct iwl_mvm *mvm, int type, bool notify); int iwl_mvm_max_scan_ie_len(struct iwl_mvm *mvm); void iwl_mvm_report_scan_aborted(struct iwl_mvm *mvm); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index db43c8a83a319a9822a08fe91290779bbc2b0bae..d2d42cd48af22eda16fc8a63641d462a63a4e5fc 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -554,6 +554,13 @@ static const struct iwl_hcmd_names iwl_mvm_data_path_names[] = { HCMD_NAME(RX_QUEUES_NOTIFICATION), }; +/* Please keep this array *SORTED* by hex value. + * Access is done through binary search + */ +static const struct iwl_hcmd_names iwl_mvm_scan_names[] = { + HCMD_NAME(OFFLOAD_MATCH_INFO_NOTIF), +}; + /* Please keep this array *SORTED* by hex value. * Access is done through binary search */ @@ -574,6 +581,9 @@ static const struct iwl_hcmd_names iwl_mvm_location_names[] = { * Access is done through binary search */ static const struct iwl_hcmd_names iwl_mvm_prot_offload_names[] = { + HCMD_NAME(WOWLAN_WAKE_PKT_NOTIFICATION), + HCMD_NAME(WOWLAN_INFO_NOTIFICATION), + HCMD_NAME(D3_END_NOTIFICATION), HCMD_NAME(STORED_BEACON_NTF), }; @@ -593,6 +603,7 @@ static const struct iwl_hcmd_arr iwl_mvm_groups[] = { [MAC_CONF_GROUP] = HCMD_ARR(iwl_mvm_mac_conf_names), [PHY_OPS_GROUP] = HCMD_ARR(iwl_mvm_phy_names), [DATA_PATH_GROUP] = HCMD_ARR(iwl_mvm_data_path_names), + [SCAN_GROUP] = HCMD_ARR(iwl_mvm_scan_names), [LOCATION_GROUP] = HCMD_ARR(iwl_mvm_location_names), [PROT_OFFLOAD_GROUP] = HCMD_ARR(iwl_mvm_prot_offload_names), [REGULATORY_AND_NVM_GROUP] = @@ -1065,7 +1076,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, static const u8 no_reclaim_cmds[] = { TX_CMD, }; - int scan_size; + size_t scan_size; u32 min_backoff; struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused; @@ -1188,13 +1199,6 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, INIT_DELAYED_WORK(&mvm->cs_tx_unblock_dwork, iwl_mvm_tx_unblock_dwork); - mvm->cmd_ver.d0i3_resp = - iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP, D0I3_END_CMD, - 0); - /* we only support version 1 */ - if (WARN_ON_ONCE(mvm->cmd_ver.d0i3_resp > 1)) - goto out_free; - mvm->cmd_ver.range_resp = iwl_fw_lookup_notif_ver(mvm->fw, LOCATION_GROUP, TOF_RANGE_RESPONSE_NOTIF, 5); @@ -1299,6 +1303,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, mvm->scan_cmd = kmalloc(scan_size, GFP_KERNEL); if (!mvm->scan_cmd) goto out_free; + mvm->scan_cmd_size = scan_size; /* invalidate ids to prevent accidental removal of sta_id 0 */ mvm->aux_sta.sta_id = IWL_MVM_INVALID_STA; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c index d8c3d7ff4f44b8d282842c1c789741af009f1978..2e9081cb6627c9451ac757fb4b1dbb6c3e858c32 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c @@ -143,7 +143,7 @@ rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta, }; /* the station support only a single receive chain */ - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) max_nss = 1; for (i = 0; i < max_nss && i < IWL_TLC_NSS_MAX; i++) { @@ -205,7 +205,7 @@ rs_fw_he_set_enabled_rates(const struct ieee80211_sta *sta, u8 nss = sta->deflink.rx_nss; /* the station support only a single receive chain */ - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) nss = 1; for (i = 0; i < nss && i < IWL_TLC_NSS_MAX; i++) { @@ -270,7 +270,7 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta, cpu_to_le16(ht_cap->mcs.rx_mask[0]); /* the station support only a single receive chain */ - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] = 0; else @@ -340,9 +340,9 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, u16 size = le32_to_cpu(notif->amsdu_size); int i; - if (sta->max_amsdu_len < size) { + if (sta->deflink.agg.max_amsdu_len < size) { /* - * In debug sta->max_amsdu_len < size + * In debug sta->deflink.agg.max_amsdu_len < size * so also check with orig_amsdu_len which holds the * original data before debugfs changed the value */ @@ -352,18 +352,18 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm, mvmsta->amsdu_enabled = le32_to_cpu(notif->amsdu_enabled); mvmsta->max_amsdu_len = size; - sta->max_rc_amsdu_len = mvmsta->max_amsdu_len; + sta->deflink.agg.max_rc_amsdu_len = mvmsta->max_amsdu_len; for (i = 0; i < IWL_MAX_TID_COUNT; i++) { if (mvmsta->amsdu_enabled & BIT(i)) - sta->max_tid_amsdu_len[i] = + sta->deflink.agg.max_tid_amsdu_len[i] = iwl_mvm_max_amsdu_size(mvm, sta, i); else /* * Not so elegant, but this will effectively * prevent AMSDU on this TID */ - sta->max_tid_amsdu_len[i] = 1; + sta->deflink.agg.max_tid_amsdu_len[i] = 1; } IWL_DEBUG_RATE(mvm, @@ -450,7 +450,7 @@ void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta, * since TLC offload works with one mode we can assume * that only vht/ht is used and also set it as station max amsdu */ - sta->max_amsdu_len = max_amsdu_len; + sta->deflink.agg.max_amsdu_len = max_amsdu_len; cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, WIDE_ID(DATA_PATH_GROUP, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c index a79043f307751a1afa2e551153e9ef5254b6e393..0b50b816684a0201e72ab6029215088e2c4adf8c 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c @@ -138,7 +138,7 @@ static bool rs_mimo_allow(struct iwl_mvm *mvm, struct ieee80211_sta *sta, if (!sta->deflink.ht_cap.ht_supported) return false; - if (sta->smps_mode == IEEE80211_SMPS_STATIC) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) return false; if (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) < 2) @@ -1491,7 +1491,7 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta, struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta); int i; - sta->max_amsdu_len = rs_fw_get_max_amsdu_len(sta); + sta->deflink.agg.max_amsdu_len = rs_fw_get_max_amsdu_len(sta); /* * In case TLC offload is not active amsdu_enabled is either 0xFFFF @@ -1506,22 +1506,23 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta, if (mvmsta->vif->bss_conf.he_support && !iwlwifi_mod_params.disable_11ax) - mvmsta->max_amsdu_len = sta->max_amsdu_len; + mvmsta->max_amsdu_len = sta->deflink.agg.max_amsdu_len; else - mvmsta->max_amsdu_len = min_t(int, sta->max_amsdu_len, 8500); + mvmsta->max_amsdu_len = + min_t(int, sta->deflink.agg.max_amsdu_len, 8500); - sta->max_rc_amsdu_len = mvmsta->max_amsdu_len; + sta->deflink.agg.max_rc_amsdu_len = mvmsta->max_amsdu_len; for (i = 0; i < IWL_MAX_TID_COUNT; i++) { if (mvmsta->amsdu_enabled) - sta->max_tid_amsdu_len[i] = + sta->deflink.agg.max_tid_amsdu_len[i] = iwl_mvm_max_amsdu_size(mvm, sta, i); else /* * Not so elegant, but this will effectively * prevent AMSDU on this TID */ - sta->max_tid_amsdu_len[i] = 1; + sta->deflink.agg.max_tid_amsdu_len[i] = 1; } } @@ -2933,7 +2934,7 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta, lq_sta->lq.sta_id = mvmsta->sta_id; mvmsta->amsdu_enabled = 0; - mvmsta->max_amsdu_len = sta->max_amsdu_len; + mvmsta->max_amsdu_len = sta->cur->max_amsdu_len; for (j = 0; j < LQ_SIZE; j++) rs_rate_scale_clear_tbl_windows(mvm, &lq_sta->lq_info[j]); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c index 2c43a9989783cbe2311d48e1d87bda87bb885f0b..1aadccd8841fd2c536b7934384c2f256ac819c59 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2021 Intel Corporation + * Copyright (C) 2012-2014, 2018-2022 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2015-2017 Intel Deutschland GmbH */ @@ -1191,16 +1191,22 @@ struct iwl_mvm_rx_phy_data { enum iwl_rx_phy_info_type info_type; __le32 d0, d1, d2, d3; __le16 d4; + + u32 rate_n_flags; + u32 gp2_on_air_rise; + u16 phy_info; + u8 energy_a, energy_b; + u8 channel; }; static void iwl_mvm_decode_he_mu_ext(struct iwl_mvm *mvm, struct iwl_mvm_rx_phy_data *phy_data, - u32 rate_n_flags, struct ieee80211_radiotap_he_mu *he_mu) { u32 phy_data2 = le32_to_cpu(phy_data->d2); u32 phy_data3 = le32_to_cpu(phy_data->d3); u16 phy_data4 = le16_to_cpu(phy_data->d4); + u32 rate_n_flags = phy_data->rate_n_flags; if (FIELD_GET(IWL_RX_PHY_DATA4_HE_MU_EXT_CH1_CRC_OK, phy_data4)) { he_mu->flags1 |= @@ -1246,7 +1252,6 @@ static void iwl_mvm_decode_he_mu_ext(struct iwl_mvm *mvm, static void iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data, - u32 rate_n_flags, struct ieee80211_radiotap_he *he, struct ieee80211_radiotap_he_mu *he_mu, struct ieee80211_rx_status *rx_status) @@ -1260,6 +1265,7 @@ iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data, * the TSF/timers are not be transmitted in HE-MU. */ u8 ru = le32_get_bits(phy_data->d1, IWL_RX_PHY_DATA1_HE_RU_ALLOC_MASK); + u32 rate_n_flags = phy_data->rate_n_flags; u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK_V1; u8 offs = 0; @@ -1331,7 +1337,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm, struct ieee80211_radiotap_he *he, struct ieee80211_radiotap_he_mu *he_mu, struct ieee80211_rx_status *rx_status, - u32 rate_n_flags, int queue) + int queue) { switch (phy_data->info_type) { case IWL_RX_PHY_INFO_TYPE_NONE: @@ -1430,7 +1436,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm, le16_encode_bits(le16_get_bits(phy_data->d4, IWL_RX_PHY_DATA4_HE_MU_EXT_PREAMBLE_PUNC_TYPE_MASK), IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW); - iwl_mvm_decode_he_mu_ext(mvm, phy_data, rate_n_flags, he_mu); + iwl_mvm_decode_he_mu_ext(mvm, phy_data, he_mu); fallthrough; case IWL_RX_PHY_INFO_TYPE_HE_MU: he_mu->flags2 |= @@ -1444,8 +1450,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm, fallthrough; case IWL_RX_PHY_INFO_TYPE_HE_TB: case IWL_RX_PHY_INFO_TYPE_HE_TB_EXT: - iwl_mvm_decode_he_phy_ru_alloc(phy_data, rate_n_flags, - he, he_mu, rx_status); + iwl_mvm_decode_he_phy_ru_alloc(phy_data, he, he_mu, rx_status); break; case IWL_RX_PHY_INFO_TYPE_HE_SU: he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BEAM_CHANGE_KNOWN); @@ -1461,13 +1466,14 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm, static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb, struct iwl_mvm_rx_phy_data *phy_data, - u32 rate_n_flags, u16 phy_info, int queue) + int queue) { struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); struct ieee80211_radiotap_he *he = NULL; struct ieee80211_radiotap_he_mu *he_mu = NULL; + u32 rate_n_flags = phy_data->rate_n_flags; u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK; - u8 stbc, ltf; + u8 ltf; static const struct ieee80211_radiotap_he known = { .data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN | IEEE80211_RADIOTAP_HE_DATA1_DATA_DCM_KNOWN | @@ -1484,6 +1490,7 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb, .flags2 = cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW_KNOWN | IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW_KNOWN), }; + u16 phy_info = phy_data->phy_info; he = skb_put_data(skb, &known, sizeof(known)); rx_status->flag |= RX_FLAG_RADIOTAP_HE; @@ -1504,7 +1511,7 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb, if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) iwl_mvm_decode_he_phy_data(mvm, phy_data, he, he_mu, rx_status, - rate_n_flags, queue); + queue); /* update aggregation data for monitor sake on default queue */ if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) && @@ -1531,19 +1538,6 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb, he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN); - stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> RATE_MCS_STBC_POS; - rx_status->nss = - ((rate_n_flags & RATE_MCS_NSS_MSK) >> - RATE_MCS_NSS_POS) + 1; - rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK; - rx_status->encoding = RX_ENC_HE; - rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; - if (rate_n_flags & RATE_MCS_BF_MSK) - rx_status->enc_flags |= RX_ENC_FLAG_BF; - - rx_status->he_dcm = - !!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK); - #define CHECK_TYPE(F) \ BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA1_FORMAT_ ## F != \ (RATE_MCS_HE_TYPE_ ## F >> RATE_MCS_HE_TYPE_POS)) @@ -1661,6 +1655,107 @@ static void iwl_mvm_rx_get_sta_block_tx(void *data, struct ieee80211_sta *sta) rx_sta_csa->all_sta_unblocked = false; } +/* + * Note: requires also rx_status->band to be prefilled, as well + * as phy_data (apart from phy_data->info_type) + */ +static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm, + struct sk_buff *skb, + struct iwl_mvm_rx_phy_data *phy_data, + int queue) +{ + struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); + u32 rate_n_flags = phy_data->rate_n_flags; + u8 stbc = u32_get_bits(rate_n_flags, RATE_MCS_STBC_MSK); + u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK; + bool is_sgi; + + phy_data->info_type = IWL_RX_PHY_INFO_TYPE_NONE; + + if (phy_data->phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) + phy_data->info_type = + le32_get_bits(phy_data->d1, + IWL_RX_PHY_DATA1_INFO_TYPE_MASK); + + /* This may be overridden by iwl_mvm_rx_he() to HE_RU */ + switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) { + case RATE_MCS_CHAN_WIDTH_20: + break; + case RATE_MCS_CHAN_WIDTH_40: + rx_status->bw = RATE_INFO_BW_40; + break; + case RATE_MCS_CHAN_WIDTH_80: + rx_status->bw = RATE_INFO_BW_80; + break; + case RATE_MCS_CHAN_WIDTH_160: + rx_status->bw = RATE_INFO_BW_160; + break; + } + + /* must be before L-SIG data */ + if (format == RATE_MCS_HE_MSK) + iwl_mvm_rx_he(mvm, skb, phy_data, queue); + + iwl_mvm_decode_lsig(skb, phy_data); + + rx_status->device_timestamp = phy_data->gp2_on_air_rise; + rx_status->freq = ieee80211_channel_to_frequency(phy_data->channel, + rx_status->band); + iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags, + phy_data->energy_a, phy_data->energy_b); + + if (unlikely(mvm->monitor_on)) + iwl_mvm_add_rtap_sniffer_config(mvm, skb); + + is_sgi = format == RATE_MCS_HE_MSK ? + iwl_he_is_sgi(rate_n_flags) : + rate_n_flags & RATE_MCS_SGI_MSK; + + if (!(format == RATE_MCS_CCK_MSK) && is_sgi) + rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; + + if (rate_n_flags & RATE_MCS_LDPC_MSK) + rx_status->enc_flags |= RX_ENC_FLAG_LDPC; + + switch (format) { + case RATE_MCS_VHT_MSK: + rx_status->encoding = RX_ENC_VHT; + break; + case RATE_MCS_HE_MSK: + rx_status->encoding = RX_ENC_HE; + rx_status->he_dcm = + !!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK); + break; + } + + switch (format) { + case RATE_MCS_HT_MSK: + rx_status->encoding = RX_ENC_HT; + rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags); + rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; + break; + case RATE_MCS_VHT_MSK: + case RATE_MCS_HE_MSK: + rx_status->nss = + u32_get_bits(rate_n_flags, RATE_MCS_NSS_MSK) + 1; + rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK; + rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; + break; + default: { + int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags, + rx_status->band); + + rx_status->rate_idx = rate; + + if (WARN_ONCE(rate < 0 || rate > 0xFF, + "Invalid rate flags 0x%x, band %d,\n", + rate_n_flags, rx_status->band)) + rx_status->rate_idx = 0; + break; + } + } +} + void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, struct iwl_rx_cmd_buffer *rxb, int queue) { @@ -1670,17 +1765,12 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, struct ieee80211_hdr *hdr; u32 len; u32 pkt_len = iwl_rx_packet_payload_len(pkt); - u32 rate_n_flags, gp2_on_air_rise; - u16 phy_info; struct ieee80211_sta *sta = NULL; struct sk_buff *skb; - u8 crypt_len = 0, channel, energy_a, energy_b; + u8 crypt_len = 0; size_t desc_size; - struct iwl_mvm_rx_phy_data phy_data = { - .info_type = IWL_RX_PHY_INFO_TYPE_NONE, - }; + struct iwl_mvm_rx_phy_data phy_data = {}; u32 format; - bool is_sgi; if (unlikely(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))) return; @@ -1696,35 +1786,37 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, } if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) { - rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags); - channel = desc->v3.channel; - gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise); - energy_a = desc->v3.energy_a; - energy_b = desc->v3.energy_b; + phy_data.rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags); + phy_data.channel = desc->v3.channel; + phy_data.gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise); + phy_data.energy_a = desc->v3.energy_a; + phy_data.energy_b = desc->v3.energy_b; phy_data.d0 = desc->v3.phy_data0; phy_data.d1 = desc->v3.phy_data1; phy_data.d2 = desc->v3.phy_data2; phy_data.d3 = desc->v3.phy_data3; } else { - rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags); - channel = desc->v1.channel; - gp2_on_air_rise = le32_to_cpu(desc->v1.gp2_on_air_rise); - energy_a = desc->v1.energy_a; - energy_b = desc->v1.energy_b; + phy_data.rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags); + phy_data.channel = desc->v1.channel; + phy_data.gp2_on_air_rise = le32_to_cpu(desc->v1.gp2_on_air_rise); + phy_data.energy_a = desc->v1.energy_a; + phy_data.energy_b = desc->v1.energy_b; phy_data.d0 = desc->v1.phy_data0; phy_data.d1 = desc->v1.phy_data1; phy_data.d2 = desc->v1.phy_data2; phy_data.d3 = desc->v1.phy_data3; } + if (iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP, REPLY_RX_MPDU_CMD, 0) < 4) { - rate_n_flags = iwl_new_rate_from_v1(rate_n_flags); + phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags); IWL_DEBUG_DROP(mvm, "Got old format rate, converting. New rate: 0x%x\n", - rate_n_flags); + phy_data.rate_n_flags); } - format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK; + + format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK; len = le16_to_cpu(desc->mpdu_len); @@ -1733,14 +1825,9 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, return; } - phy_info = le16_to_cpu(desc->phy_info); + phy_data.phy_info = le16_to_cpu(desc->phy_info); phy_data.d4 = desc->phy_data4; - if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) - phy_data.info_type = - le32_get_bits(phy_data.d1, - IWL_RX_PHY_DATA1_INFO_TYPE_MASK); - hdr = (void *)(pkt->data + desc_size); /* Dont use dev_alloc_skb(), we'll have enough headroom once * ieee80211_hdr pulled. @@ -1763,27 +1850,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, rx_status = IEEE80211_SKB_RXCB(skb); - /* This may be overridden by iwl_mvm_rx_he() to HE_RU */ - switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) { - case RATE_MCS_CHAN_WIDTH_20: - break; - case RATE_MCS_CHAN_WIDTH_40: - rx_status->bw = RATE_INFO_BW_40; - break; - case RATE_MCS_CHAN_WIDTH_80: - rx_status->bw = RATE_INFO_BW_80; - break; - case RATE_MCS_CHAN_WIDTH_160: - rx_status->bw = RATE_INFO_BW_160; - break; - } - - if (format == RATE_MCS_HE_MSK) - iwl_mvm_rx_he(mvm, skb, &phy_data, rate_n_flags, - phy_info, queue); - - iwl_mvm_decode_lsig(skb, &phy_data); - /* * Keep packets with CRC errors (and with overrun) for monitor mode * (otherwise the firmware discards them) but mark them as bad. @@ -1794,12 +1860,13 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, le32_to_cpu(desc->status)); rx_status->flag |= RX_FLAG_FAILED_FCS_CRC; } + /* set the preamble flag if appropriate */ if (format == RATE_MCS_CCK_MSK && - phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE) + phy_data.phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE) rx_status->enc_flags |= RX_ENC_FLAG_SHORTPRE; - if (likely(!(phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) { + if (likely(!(phy_data.phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) { u64 tsf_on_air_rise; if (mvm->trans->trans_cfg->device_family >= @@ -1813,24 +1880,20 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, rx_status->flag |= RX_FLAG_MACTIME_PLCP_START; } - rx_status->device_timestamp = gp2_on_air_rise; if (iwl_mvm_is_band_in_rx_supported(mvm)) { u8 band = BAND_IN_RX_STATUS(desc->mac_phy_idx); rx_status->band = iwl_mvm_nl80211_band_from_rx_msdu(band); } else { - rx_status->band = channel > 14 ? NL80211_BAND_5GHZ : + rx_status->band = phy_data.channel > 14 ? NL80211_BAND_5GHZ : NL80211_BAND_2GHZ; } - rx_status->freq = ieee80211_channel_to_frequency(channel, - rx_status->band); - iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags, energy_a, - energy_b); /* update aggregation data for monitor sake on default queue */ - if (!queue && (phy_info & IWL_RX_MPDU_PHY_AMPDU)) { - bool toggle_bit = phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE; + if (!queue && (phy_data.phy_info & IWL_RX_MPDU_PHY_AMPDU)) { + bool toggle_bit; + toggle_bit = phy_data.phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE; rx_status->flag |= RX_FLAG_AMPDU_DETAILS; /* * Toggle is switched whenever new aggregation starts. Make @@ -1846,9 +1909,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, rx_status->ampdu_reference = mvm->ampdu_ref; } - if (unlikely(mvm->monitor_on)) - iwl_mvm_add_rtap_sniffer_config(mvm, skb); - rcu_read_lock(); if (desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_SRC_STA_FOUND)) { @@ -1867,13 +1927,15 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, sta = ieee80211_find_sta_by_ifaddr(mvm->hw, hdr->addr2, NULL); } - if (iwl_mvm_rx_crypto(mvm, sta, hdr, rx_status, phy_info, desc, + if (iwl_mvm_rx_crypto(mvm, sta, hdr, rx_status, phy_data.phy_info, desc, le32_to_cpu(pkt->len_n_flags), queue, &crypt_len)) { kfree_skb(skb); goto out; } + iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue); + if (sta) { struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta); struct ieee80211_vif *tx_blocked_vif = @@ -1971,43 +2033,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi, } } - is_sgi = format == RATE_MCS_HE_MSK ? - iwl_he_is_sgi(rate_n_flags) : - rate_n_flags & RATE_MCS_SGI_MSK; - - if (!(format == RATE_MCS_CCK_MSK) && is_sgi) - rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; - if (rate_n_flags & RATE_MCS_LDPC_MSK) - rx_status->enc_flags |= RX_ENC_FLAG_LDPC; - if (format == RATE_MCS_HT_MSK) { - u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> - RATE_MCS_STBC_POS; - rx_status->encoding = RX_ENC_HT; - rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags); - rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; - } else if (format == RATE_MCS_VHT_MSK) { - u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> - RATE_MCS_STBC_POS; - rx_status->nss = ((rate_n_flags & RATE_MCS_NSS_MSK) >> - RATE_MCS_NSS_POS) + 1; - rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK; - rx_status->encoding = RX_ENC_VHT; - rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; - if (rate_n_flags & RATE_MCS_BF_MSK) - rx_status->enc_flags |= RX_ENC_FLAG_BF; - } else if (!(format == RATE_MCS_HE_MSK)) { - int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags, - rx_status->band); - - if (WARN(rate < 0 || rate > 0xFF, - "Invalid rate flags 0x%x, band %d,\n", - rate_n_flags, rx_status->band)) { - kfree_skb(skb); - goto out; - } - rx_status->rate_idx = rate; - } - /* management stuff on default queue */ if (!queue) { if (unlikely((ieee80211_is_beacon(hdr->frame_control) || @@ -2039,32 +2064,32 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi, struct ieee80211_rx_status *rx_status; struct iwl_rx_packet *pkt = rxb_addr(rxb); struct iwl_rx_no_data *desc = (void *)pkt->data; - u32 rate_n_flags = le32_to_cpu(desc->rate); - u32 gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time); u32 rssi = le32_to_cpu(desc->rssi); u32 info_type = le32_to_cpu(desc->info) & RX_NO_DATA_INFO_TYPE_MSK; - u16 phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD; struct ieee80211_sta *sta = NULL; struct sk_buff *skb; - u8 channel, energy_a, energy_b; - u32 format; struct iwl_mvm_rx_phy_data phy_data = { - .info_type = le32_get_bits(desc->phy_info[1], - IWL_RX_PHY_DATA1_INFO_TYPE_MASK), .d0 = desc->phy_info[0], .d1 = desc->phy_info[1], + .phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD, + .gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time), + .rate_n_flags = le32_to_cpu(desc->rate), + .energy_a = u32_get_bits(rssi, RX_NO_DATA_CHAIN_A_MSK), + .energy_b = u32_get_bits(rssi, RX_NO_DATA_CHAIN_B_MSK), + .channel = u32_get_bits(rssi, RX_NO_DATA_CHANNEL_MSK), }; - bool is_sgi; + u32 format; if (iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP, RX_NO_DATA_NOTIF, 0) < 2) { IWL_DEBUG_DROP(mvm, "Got an old rate format. Old rate: 0x%x\n", - rate_n_flags); - rate_n_flags = iwl_new_rate_from_v1(rate_n_flags); + phy_data.rate_n_flags); + phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags); IWL_DEBUG_DROP(mvm, " Rate after conversion to the new format: 0x%x\n", - rate_n_flags); + phy_data.rate_n_flags); } - format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK; + + format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK; if (unlikely(iwl_rx_packet_payload_len(pkt) < sizeof(*desc))) return; @@ -2072,10 +2097,6 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi, if (unlikely(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status))) return; - energy_a = (rssi & RX_NO_DATA_CHAIN_A_MSK) >> RX_NO_DATA_CHAIN_A_POS; - energy_b = (rssi & RX_NO_DATA_CHAIN_B_MSK) >> RX_NO_DATA_CHAIN_B_POS; - channel = (rssi & RX_NO_DATA_CHANNEL_MSK) >> RX_NO_DATA_CHANNEL_POS; - /* Dont use dev_alloc_skb(), we'll have enough headroom once * ieee80211_hdr pulled. */ @@ -2106,86 +2127,31 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi, break; } - /* This may be overridden by iwl_mvm_rx_he() to HE_RU */ - switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) { - case RATE_MCS_CHAN_WIDTH_20: - break; - case RATE_MCS_CHAN_WIDTH_40: - rx_status->bw = RATE_INFO_BW_40; - break; - case RATE_MCS_CHAN_WIDTH_80: - rx_status->bw = RATE_INFO_BW_80; - break; - case RATE_MCS_CHAN_WIDTH_160: - rx_status->bw = RATE_INFO_BW_160; - break; - } - - if (format == RATE_MCS_HE_MSK) - iwl_mvm_rx_he(mvm, skb, &phy_data, rate_n_flags, - phy_info, queue); - - iwl_mvm_decode_lsig(skb, &phy_data); - - rx_status->device_timestamp = gp2_on_air_rise; - rx_status->band = channel > 14 ? NL80211_BAND_5GHZ : + rx_status->band = phy_data.channel > 14 ? NL80211_BAND_5GHZ : NL80211_BAND_2GHZ; - rx_status->freq = ieee80211_channel_to_frequency(channel, - rx_status->band); - iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags, energy_a, - energy_b); - rcu_read_lock(); + iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue); - is_sgi = format == RATE_MCS_HE_MSK ? - iwl_he_is_sgi(rate_n_flags) : - rate_n_flags & RATE_MCS_SGI_MSK; - - if (!(format == RATE_MCS_CCK_MSK) && is_sgi) - rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; - if (rate_n_flags & RATE_MCS_LDPC_MSK) - rx_status->enc_flags |= RX_ENC_FLAG_LDPC; - if (format == RATE_MCS_HT_MSK) { - u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> - RATE_MCS_STBC_POS; - rx_status->encoding = RX_ENC_HT; - rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags); - rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; - } else if (format == RATE_MCS_VHT_MSK) { - u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> - RATE_MCS_STBC_POS; - rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK; - rx_status->encoding = RX_ENC_VHT; - rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT; - if (rate_n_flags & RATE_MCS_BF_MSK) - rx_status->enc_flags |= RX_ENC_FLAG_BF; - /* - * take the nss from the rx_vec since the rate_n_flags has - * only 2 bits for the nss which gives a max of 4 ss but - * there may be up to 8 spatial streams - */ + /* + * Override the nss from the rx_vec since the rate_n_flags has + * only 2 bits for the nss which gives a max of 4 ss but there + * may be up to 8 spatial streams. + */ + switch (format) { + case RATE_MCS_VHT_MSK: rx_status->nss = le32_get_bits(desc->rx_vec[0], RX_NO_DATA_RX_VEC0_VHT_NSTS_MSK) + 1; - } else if (format == RATE_MCS_HE_MSK) { + break; + case RATE_MCS_HE_MSK: rx_status->nss = le32_get_bits(desc->rx_vec[0], RX_NO_DATA_RX_VEC0_HE_NSTS_MSK) + 1; - } else { - int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags, - rx_status->band); - - if (WARN(rate < 0 || rate > 0xFF, - "Invalid rate flags 0x%x, band %d,\n", - rate_n_flags, rx_status->band)) { - kfree_skb(skb); - goto out; - } - rx_status->rate_idx = rate; + break; } + rcu_read_lock(); ieee80211_rx_napi(mvm->hw, sta, skb, napi); -out: rcu_read_unlock(); } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c index 582a95ffc7ab4adc77ab41e94fcc7025b3cc40e2..acd8803dbcdd6866aec87b6aecc4d6426a5c1665 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c @@ -2626,7 +2626,7 @@ static int iwl_mvm_build_scan_cmd(struct iwl_mvm *mvm, u8 scan_ver; lockdep_assert_held(&mvm->mutex); - memset(mvm->scan_cmd, 0, ksize(mvm->scan_cmd)); + memset(mvm->scan_cmd, 0, mvm->scan_cmd_size); if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN)) { hcmd->id = SCAN_OFFLOAD_REQUEST_CMD; @@ -3091,7 +3091,7 @@ static int iwl_mvm_scan_stop_wait(struct iwl_mvm *mvm, int type) 1 * HZ); } -static int iwl_scan_req_umac_get_size(u8 scan_ver) +static size_t iwl_scan_req_umac_get_size(u8 scan_ver) { switch (scan_ver) { case 12: @@ -3104,7 +3104,7 @@ static int iwl_scan_req_umac_get_size(u8 scan_ver) return 0; } -int iwl_mvm_scan_size(struct iwl_mvm *mvm) +size_t iwl_mvm_scan_size(struct iwl_mvm *mvm) { int base_size, tail_size; u8 scan_ver = iwl_fw_lookup_cmd_ver(mvm->fw, SCAN_REQ_UMAC, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c index ff0d3b3df1400e2661d8feb8c2992f4bf3e2cef2..cbd8053a9e35af57c11c1b3437f0ee86a84da571 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c @@ -116,7 +116,7 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta, break; } - switch (sta->smps_mode) { + switch (sta->deflink.smps_mode) { case IEEE80211_SMPS_AUTOMATIC: case IEEE80211_SMPS_NUM_MODES: WARN_ON(1); @@ -384,6 +384,7 @@ static int iwl_mvm_disable_txq(struct iwl_mvm *mvm, struct ieee80211_sta *sta, iwl_mvm_txq_from_tid(sta, tid); mvmtxq->txq_id = IWL_MVM_INVALID_QUEUE; + list_del_init(&mvmtxq->list); } /* Regardless if this is a reserved TXQ for a STA - mark it as false */ @@ -478,6 +479,7 @@ static int iwl_mvm_remove_sta_queue_marking(struct iwl_mvm *mvm, int queue) mvmsta->tid_data[tid].txq_id = IWL_MVM_INVALID_QUEUE; mvmtxq->txq_id = IWL_MVM_INVALID_QUEUE; + list_del_init(&mvmtxq->list); } mvmsta->tfd_queue_msk &= ~BIT(queue); /* Don't use this queue anymore */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c index f9e08b339e0c4ee1a84060ef2e7f317c02f70f08..86d20e13bf47aad0b29ccf7b4ea44c01072af14f 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c @@ -926,7 +926,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb, * Take the min of ieee80211 station and mvm station */ max_amsdu_len = - min_t(unsigned int, sta->max_amsdu_len, + min_t(unsigned int, sta->cur->max_amsdu_len, iwl_mvm_max_amsdu_size(mvm, sta, tid)); /* diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index b16d4ae182d10b179094ae7d155c124129a649c4..4f699862e7f730d753bab41124b877d6d00a9ef5 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -1155,10 +1155,20 @@ static const struct iwl_dev_info iwl_dev_info_table[] = { IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY, iwl_cfg_bz_a0_fm_a0, iwl_bz_name), _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, - IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY, + IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY, + IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, + IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_NO_JACKET, + iwl_cfg_bz_a0_fm4_a0, iwl_bz_name), + _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, + IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP, IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET, iwl_cfg_gl_a0_fm_a0, iwl_bz_name), + _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, + IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP, + IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, + IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET, + iwl_cfg_gl_b0_fm_b0, iwl_bz_name), /* BZ Z step */ _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, @@ -1169,10 +1179,15 @@ static const struct iwl_dev_info iwl_dev_info_table[] = { /* BNJ */ _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, - IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY, + IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP, IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET, iwl_cfg_bnj_a0_fm_a0, iwl_bz_name), + _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, + IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP, + IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, + IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET, + iwl_cfg_bnj_b0_fm_b0, iwl_bz_name), _IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY, IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY, diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c index 68a4572cee5346ba097c56b22084c83b0b19e9c1..9c9f87fe8377701c9e7eb5bb9de9620153cdf1b3 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c @@ -1110,7 +1110,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans) poll = iwl_pcie_napi_poll_msix; netif_napi_add(&trans_pcie->napi_dev, &rxq->napi, - poll, NAPI_POLL_WEIGHT); + poll); napi_enable(&rxq->napi); } diff --git a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c index 0a376f112db9fc1b0412a32125308154561bd6bc..4e0a0c88169791bd5ff02cc909b3362cc13e257c 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c +++ b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c @@ -3848,7 +3848,7 @@ static void prism2_get_drvinfo(struct net_device *dev, iface = netdev_priv(dev); local = iface->local; - strlcpy(info->driver, "hostap", sizeof(info->driver)); + strscpy(info->driver, "hostap", sizeof(info->driver)); snprintf(info->fw_version, sizeof(info->fw_version), "%d.%d.%d", (local->sta_fw_ver >> 16) & 0xff, (local->sta_fw_ver >> 8) & 0xff, diff --git a/drivers/net/wireless/intersil/p54/main.c b/drivers/net/wireless/intersil/p54/main.c index b925e327e091e9523e96155901280989f77b1d70..e127453ab51ad40d46e4a82e0b751194229e1d9f 100644 --- a/drivers/net/wireless/intersil/p54/main.c +++ b/drivers/net/wireless/intersil/p54/main.c @@ -635,7 +635,7 @@ static int p54_get_survey(struct ieee80211_hw *dev, int idx, /* * hw/fw has not accumulated enough sample sets. * Wait for 100ms, this ought to be enough to - * to get at least one non-null set of channel + * get at least one non-null set of channel * usage statistics. */ msleep(100); diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index 6e55f153ff26e95a635adcaa75caa04dffc63732..a40636c90ec365975284bdc4ace553d97f22a857 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -229,6 +229,7 @@ static inline void hwsim_clear_magic(struct ieee80211_vif *vif) struct hwsim_sta_priv { u32 magic; unsigned int last_link; + u16 active_links_rx; }; #define HWSIM_STA_MAGIC 0x6d537749 @@ -652,7 +653,6 @@ struct mac80211_hwsim_data { u32 ciphers[ARRAY_SIZE(hwsim_ciphers)]; struct mac_address addresses[2]; - struct ieee80211_chanctx_conf *chanctx; int channels, idx; bool use_chanctx; bool destroy_on_close; @@ -1299,6 +1299,8 @@ static void mac80211_hwsim_config_mac_nl(struct ieee80211_hw *hw, struct sk_buff *skb; void *msg_head; + WARN_ON(!is_valid_ether_addr(addr)); + if (!_portid && !hwsim_virtio_enabled) return; @@ -1561,6 +1563,42 @@ static void mac80211_hwsim_add_vendor_rtap(struct sk_buff *skb) #endif } +static void mac80211_hwsim_rx(struct mac80211_hwsim_data *data, + struct ieee80211_rx_status *rx_status, + struct sk_buff *skb) +{ + struct ieee80211_hdr *hdr = (void *)skb->data; + + if (!ieee80211_has_morefrags(hdr->frame_control) && + !is_multicast_ether_addr(hdr->addr1) && + (ieee80211_is_mgmt(hdr->frame_control) || + ieee80211_is_data(hdr->frame_control))) { + struct ieee80211_sta *sta; + unsigned int link_id; + + rcu_read_lock(); + sta = ieee80211_find_sta_by_link_addrs(data->hw, hdr->addr2, + hdr->addr1, &link_id); + if (sta) { + struct hwsim_sta_priv *sp = (void *)sta->drv_priv; + + if (ieee80211_has_pm(hdr->frame_control)) + sp->active_links_rx &= ~BIT(link_id); + else + sp->active_links_rx |= BIT(link_id); + } + rcu_read_unlock(); + } + + memcpy(IEEE80211_SKB_RXCB(skb), rx_status, sizeof(*rx_status)); + + mac80211_hwsim_add_vendor_rtap(skb); + + data->rx_pkts++; + data->rx_bytes += skb->len; + ieee80211_rx_irqsafe(data->hw, skb); +} + static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw, struct sk_buff *skb, struct ieee80211_channel *chan) @@ -1688,13 +1726,7 @@ static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw, rx_status.mactime = now + data2->tsf_offset; - memcpy(IEEE80211_SKB_RXCB(nskb), &rx_status, sizeof(rx_status)); - - mac80211_hwsim_add_vendor_rtap(nskb); - - data2->rx_pkts++; - data2->rx_bytes += nskb->len; - ieee80211_rx_irqsafe(data2->hw, nskb); + mac80211_hwsim_rx(data2, &rx_status, nskb); } spin_unlock(&hwsim_radio_lock); @@ -1714,12 +1746,7 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data, if (!vif->valid_links) return &vif->bss_conf; - /* FIXME: handle multicast TX properly */ - if (is_multicast_ether_addr(hdr->addr1) || WARN_ON_ONCE(!sta)) { - unsigned int first_link = ffs(vif->valid_links) - 1; - - return rcu_dereference(vif->link_conf[first_link]); - } + WARN_ON(is_multicast_ether_addr(hdr->addr1)); if (WARN_ON_ONCE(!sta->valid_links)) return &vif->bss_conf; @@ -1731,6 +1758,12 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data, /* round-robin the available link IDs */ link_id = (sp->last_link + i + 1) % ARRAY_SIZE(vif->link_conf); + if (!(vif->active_links & BIT(link_id))) + continue; + + if (!(sp->active_links_rx & BIT(link_id))) + continue; + *link_sta = rcu_dereference(sta->link[link_id]); if (!*link_sta) continue; @@ -1739,6 +1772,10 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data, if (WARN_ON_ONCE(!bss_conf)) continue; + /* can happen while switching links */ + if (!rcu_access_pointer(bss_conf->chanctx_conf)) + continue; + sp->last_link = link_id; return bss_conf; } @@ -2401,10 +2438,19 @@ static int mac80211_hwsim_sta_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta) { + struct hwsim_sta_priv *sp = (void *)sta->drv_priv; + hwsim_check_magic(vif); hwsim_set_sta_magic(sta); mac80211_hwsim_sta_rc_update(hw, vif, sta, 0); + if (sta->valid_links) { + WARN(hweight16(sta->valid_links) > 1, + "expect to add STA with single link, have 0x%x\n", + sta->valid_links); + sp->active_links_rx = sta->valid_links; + } + return 0; } @@ -2430,6 +2476,14 @@ static int mac80211_hwsim_sta_state(struct ieee80211_hw *hw, if (old_state == IEEE80211_STA_NOTEXIST) return mac80211_hwsim_sta_add(hw, vif, sta); + /* + * when client is authorized (AP station marked as such), + * enable all links + */ + if (vif->type == NL80211_IFTYPE_STATION && + new_state == IEEE80211_STA_AUTHORIZED && !sta->tdls) + ieee80211_set_active_links_async(vif, vif->valid_links); + return 0; } @@ -2866,11 +2920,6 @@ static int mac80211_hwsim_croc(struct ieee80211_hw *hw, static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { - struct mac80211_hwsim_data *hwsim = hw->priv; - - mutex_lock(&hwsim->mutex); - hwsim->chanctx = ctx; - mutex_unlock(&hwsim->mutex); hwsim_set_chanctx_magic(ctx); wiphy_dbg(hw->wiphy, "add channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n", @@ -2882,11 +2931,6 @@ static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw, static void mac80211_hwsim_remove_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx) { - struct mac80211_hwsim_data *hwsim = hw->priv; - - mutex_lock(&hwsim->mutex); - hwsim->chanctx = NULL; - mutex_unlock(&hwsim->mutex); wiphy_dbg(hw->wiphy, "remove channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n", ctx->def.chan->center_freq, ctx->def.width, @@ -2899,11 +2943,6 @@ static void mac80211_hwsim_change_chanctx(struct ieee80211_hw *hw, struct ieee80211_chanctx_conf *ctx, u32 changed) { - struct mac80211_hwsim_data *hwsim = hw->priv; - - mutex_lock(&hwsim->mutex); - hwsim->chanctx = ctx; - mutex_unlock(&hwsim->mutex); hwsim_check_chanctx_magic(ctx); wiphy_dbg(hw->wiphy, "change channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n", @@ -2919,6 +2958,18 @@ static int mac80211_hwsim_assign_vif_chanctx(struct ieee80211_hw *hw, hwsim_check_magic(vif); hwsim_check_chanctx_magic(ctx); + /* if we activate a link while already associated wake it up */ + if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) { + struct sk_buff *skb; + + skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true); + if (skb) { + local_bh_disable(); + mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan); + local_bh_enable(); + } + } + return 0; } @@ -2929,6 +2980,22 @@ static void mac80211_hwsim_unassign_vif_chanctx(struct ieee80211_hw *hw, { hwsim_check_magic(vif); hwsim_check_chanctx_magic(ctx); + + /* if we deactivate a link while associated suspend it first */ + if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) { + struct sk_buff *skb; + + skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true); + if (skb) { + struct ieee80211_hdr *hdr = (void *)skb->data; + + hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM); + + local_bh_disable(); + mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan); + local_bh_enable(); + } + } } static const char mac80211_hwsim_gstrings_stats[][ETH_GSTRING_LEN] = { @@ -2995,18 +3062,22 @@ static int mac80211_hwsim_change_vif_links(struct ieee80211_hw *hw, u16 old_links, u16 new_links, struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]) { - unsigned long rem = old_links & ~new_links ?: BIT(0); + unsigned long rem = old_links & ~new_links; unsigned long add = new_links & ~old_links; int i; + if (!old_links) + rem |= BIT(0); + if (!new_links) + add |= BIT(0); + for_each_set_bit(i, &rem, IEEE80211_MLD_MAX_NUM_LINKS) mac80211_hwsim_config_mac_nl(hw, old[i]->addr, false); for_each_set_bit(i, &add, IEEE80211_MLD_MAX_NUM_LINKS) { struct ieee80211_bss_conf *link_conf; - /* FIXME: figure out how to get the locking here */ - link_conf = rcu_dereference_protected(vif->link_conf[i], 1); + link_conf = link_conf_dereference_protected(vif, i); if (WARN_ON(!link_conf)) continue; @@ -3021,6 +3092,13 @@ static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw, struct ieee80211_sta *sta, u16 old_links, u16 new_links) { + struct hwsim_sta_priv *sp = (void *)sta->drv_priv; + + hwsim_check_sta_magic(sta); + + if (vif->type == NL80211_IFTYPE_STATION) + sp->active_links_rx = new_links; + return 0; } @@ -3208,8 +3286,112 @@ out_err: static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = { { - .types_mask = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP), + .types_mask = BIT(NL80211_IFTYPE_STATION), + .he_cap = { + .has_he = true, + .he_cap_elem = { + .mac_cap_info[0] = + IEEE80211_HE_MAC_CAP0_HTC_HE, + .mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US | + IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8, + .mac_cap_info[2] = + IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_MU_CASCADING | + IEEE80211_HE_MAC_CAP2_ACK_EN, + .mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3, + .mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU, + .phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK | + IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD | + IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS, + .phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO, + + /* Leave all the other PHY capability bytes + * unset, as DCM, beam forming, RU and PPE + * threshold information are not supported + */ + }, + .he_mcs_nss_supp = { + .rx_mcs_80 = cpu_to_le16(0xfffa), + .tx_mcs_80 = cpu_to_le16(0xfffa), + .rx_mcs_160 = cpu_to_le16(0xffff), + .tx_mcs_160 = cpu_to_le16(0xffff), + .rx_mcs_80p80 = cpu_to_le16(0xffff), + .tx_mcs_80p80 = cpu_to_le16(0xffff), + }, + }, + .eht_cap = { + .has_eht = true, + .eht_cap_elem = { + .mac_cap_info[0] = + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_OM_CONTROL | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, + .phy_cap_info[0] = + IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ | + IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI | + IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE, + .phy_cap_info[3] = + IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK, + .phy_cap_info[4] = + IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | + IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP | + IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP | + IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI | + IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK, + .phy_cap_info[5] = + IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK | + IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT | + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK | + IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK, + .phy_cap_info[6] = + IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK, + .phy_cap_info[7] = + IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW, + }, + + /* For all MCS and bandwidth, set 8 NSS for both Tx and + * Rx + */ + .eht_mcs_nss_supp = { + /* + * Since B0, B1, B2 and B3 are not set in + * the supported channel width set field in the + * HE PHY capabilities information field the + * device is a 20MHz only device on 2.4GHz band. + */ + .only_20mhz = { + .rx_tx_mcs7_max_nss = 0x88, + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + }, + /* PPE threshold information is not supported */ + }, + }, + { + .types_mask = BIT(NL80211_IFTYPE_AP), .he_cap = { .has_he = true, .he_cap_elem = { @@ -3356,9 +3538,132 @@ static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = { static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = { { - /* TODO: should we support other types, e.g., P2P?*/ - .types_mask = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP), + /* TODO: should we support other types, e.g., P2P? */ + .types_mask = BIT(NL80211_IFTYPE_STATION), + .he_cap = { + .has_he = true, + .he_cap_elem = { + .mac_cap_info[0] = + IEEE80211_HE_MAC_CAP0_HTC_HE, + .mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US | + IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8, + .mac_cap_info[2] = + IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_MU_CASCADING | + IEEE80211_HE_MAC_CAP2_ACK_EN, + .mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3, + .mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU, + .phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G, + .phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK | + IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD | + IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS, + .phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO, + + /* Leave all the other PHY capability bytes + * unset, as DCM, beam forming, RU and PPE + * threshold information are not supported + */ + }, + .he_mcs_nss_supp = { + .rx_mcs_80 = cpu_to_le16(0xfffa), + .tx_mcs_80 = cpu_to_le16(0xfffa), + .rx_mcs_160 = cpu_to_le16(0xfffa), + .tx_mcs_160 = cpu_to_le16(0xfffa), + .rx_mcs_80p80 = cpu_to_le16(0xfffa), + .tx_mcs_80p80 = cpu_to_le16(0xfffa), + }, + }, + .eht_cap = { + .has_eht = true, + .eht_cap_elem = { + .mac_cap_info[0] = + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_OM_CONTROL | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, + .phy_cap_info[0] = + IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ | + IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI | + IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE | + IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK, + .phy_cap_info[1] = + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK | + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK, + .phy_cap_info[2] = + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK | + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK, + .phy_cap_info[3] = + IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK, + .phy_cap_info[4] = + IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | + IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP | + IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP | + IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI | + IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK, + .phy_cap_info[5] = + IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK | + IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT | + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK | + IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK, + .phy_cap_info[6] = + IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK, + .phy_cap_info[7] = + IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW | + IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ | + IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ | + IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ | + IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ, + }, + + /* For all MCS and bandwidth, set 8 NSS for both Tx and + * Rx + */ + .eht_mcs_nss_supp = { + /* + * As B1 and B2 are set in the supported + * channel width set field in the HE PHY + * capabilities information field include all + * the following MCS/NSS. + */ + .bw._80 = { + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + .bw._160 = { + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + }, + /* PPE threshold information is not supported */ + }, + }, + { + .types_mask = BIT(NL80211_IFTYPE_AP), .he_cap = { .has_he = true, .he_cap_elem = { @@ -3529,9 +3834,153 @@ static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = { static const struct ieee80211_sband_iftype_data sband_capa_6ghz[] = { { - /* TODO: should we support other types, e.g., P2P?*/ - .types_mask = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP), + /* TODO: should we support other types, e.g., P2P? */ + .types_mask = BIT(NL80211_IFTYPE_STATION), + .he_6ghz_capa = { + .capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START | + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP | + IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN | + IEEE80211_HE_6GHZ_CAP_SM_PS | + IEEE80211_HE_6GHZ_CAP_RD_RESPONDER | + IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS | + IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS), + }, + .he_cap = { + .has_he = true, + .he_cap_elem = { + .mac_cap_info[0] = + IEEE80211_HE_MAC_CAP0_HTC_HE, + .mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US | + IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8, + .mac_cap_info[2] = + IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_MU_CASCADING | + IEEE80211_HE_MAC_CAP2_ACK_EN, + .mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3, + .mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU, + .phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G, + .phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK | + IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A | + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD | + IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS, + .phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO, + + /* Leave all the other PHY capability bytes + * unset, as DCM, beam forming, RU and PPE + * threshold information are not supported + */ + }, + .he_mcs_nss_supp = { + .rx_mcs_80 = cpu_to_le16(0xfffa), + .tx_mcs_80 = cpu_to_le16(0xfffa), + .rx_mcs_160 = cpu_to_le16(0xfffa), + .tx_mcs_160 = cpu_to_le16(0xfffa), + .rx_mcs_80p80 = cpu_to_le16(0xfffa), + .tx_mcs_80p80 = cpu_to_le16(0xfffa), + }, + }, + .eht_cap = { + .has_eht = true, + .eht_cap_elem = { + .mac_cap_info[0] = + IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS | + IEEE80211_EHT_MAC_CAP0_OM_CONTROL | + IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1, + .phy_cap_info[0] = + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ | + IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ | + IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI | + IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | + IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE | + IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK, + .phy_cap_info[1] = + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK | + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK | + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK, + .phy_cap_info[2] = + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK | + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK | + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK, + .phy_cap_info[3] = + IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK | + IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK | + IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK, + .phy_cap_info[4] = + IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO | + IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP | + IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP | + IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI | + IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK, + .phy_cap_info[5] = + IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK | + IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP | + IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT | + IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK | + IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK, + .phy_cap_info[6] = + IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK | + IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP, + .phy_cap_info[7] = + IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW | + IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ | + IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ | + IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ | + IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ | + IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ | + IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ, + }, + + /* For all MCS and bandwidth, set 8 NSS for both Tx and + * Rx + */ + .eht_mcs_nss_supp = { + /* + * As B1 and B2 are set in the supported + * channel width set field in the HE PHY + * capabilities information field and 320MHz in + * 6GHz is supported include all the following + * MCS/NSS. + */ + .bw._80 = { + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + .bw._160 = { + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + .bw._320 = { + .rx_tx_mcs9_max_nss = 0x88, + .rx_tx_mcs11_max_nss = 0x88, + .rx_tx_mcs13_max_nss = 0x88, + }, + }, + /* PPE threshold information is not supported */ + }, + }, + { + .types_mask = BIT(NL80211_IFTYPE_AP), .he_6ghz_capa = { .capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START | IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP | @@ -3896,7 +4345,6 @@ static int mac80211_hwsim_new_radio(struct genl_info *info, hw->wiphy->max_remain_on_channel_duration = 1000; data->if_combination.radar_detect_widths = 0; data->if_combination.num_different_channels = data->channels; - data->chanctx = NULL; } else { data->if_combination.num_different_channels = 1; data->if_combination.radar_detect_widths = @@ -4471,13 +4919,9 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, if (data2->use_chanctx) { if (data2->tmp_chan) channel = data2->tmp_chan; - else if (data2->chanctx) - channel = data2->chanctx->def.chan; } else { channel = data2->channel; } - if (!channel) - goto out; if (!hwsim_virtio_enabled) { if (hwsim_net_get_netgroup(genl_info_net(info)) != @@ -4508,6 +4952,7 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, rx_status.freq); if (!iter_data.channel) goto out; + rx_status.band = iter_data.channel->band; mutex_lock(&data2->mutex); if (!hwsim_chans_compat(iter_data.channel, channel)) { @@ -4520,12 +4965,16 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, } } mutex_unlock(&data2->mutex); + } else if (!channel) { + goto out; } else { rx_status.freq = channel->center_freq; + rx_status.band = channel->band; } - rx_status.band = channel->band; rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]); + if (rx_status.rate_idx >= data2->hw->wiphy->bands[rx_status.band]->n_bitrates) + goto out; rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]); hdr = (void *)skb->data; @@ -4534,10 +4983,7 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2, ieee80211_is_probe_resp(hdr->frame_control)) rx_status.boottime_ns = ktime_get_boottime_ns(); - memcpy(IEEE80211_SKB_RXCB(skb), &rx_status, sizeof(rx_status)); - data2->rx_pkts++; - data2->rx_bytes += skb->len; - ieee80211_rx_irqsafe(data2->hw, skb); + mac80211_hwsim_rx(data2, &rx_status, skb); return 0; err: @@ -4912,6 +5358,7 @@ static struct genl_family hwsim_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = hwsim_ops, .n_small_ops = ARRAY_SIZE(hwsim_ops), + .resv_start_op = HWSIM_CMD_DEL_MAC_ADDR + 1, .mcgrps = hwsim_mcgrps, .n_mcgrps = ARRAY_SIZE(hwsim_mcgrps), }; @@ -5060,6 +5507,10 @@ static int hwsim_virtio_handle_cmd(struct sk_buff *skb) nlh = nlmsg_hdr(skb); gnlh = nlmsg_data(nlh); + + if (skb->len < nlh->nlmsg_len) + return -EINVAL; + err = genlmsg_parse(nlh, &hwsim_genl_family, tb, HWSIM_ATTR_MAX, hwsim_genl_policy, NULL); if (err) { @@ -5102,7 +5553,8 @@ static void hwsim_virtio_rx_work(struct work_struct *work) spin_unlock_irqrestore(&hwsim_virtio_lock, flags); skb->data = skb->head; - skb_set_tail_pointer(skb, len); + skb_reset_tail_pointer(skb); + skb_put(skb, len); hwsim_virtio_handle_cmd(skb); spin_lock_irqsave(&hwsim_virtio_lock, flags); diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c index b0b3f59dabc6fd4c9563bc83f6a440e4e098b90f..3e065cbb0af91dc8ce00d0e7f001dd5220c79766 100644 --- a/drivers/net/wireless/marvell/libertas/cfg.c +++ b/drivers/net/wireless/marvell/libertas/cfg.c @@ -546,7 +546,7 @@ static int lbs_ret_scan(struct lbs_private *priv, unsigned long dummy, pos = scanresp->bssdesc_and_tlvbuffer; lbs_deb_hex(LBS_DEB_SCAN, "SCAN_RSP", scanresp->bssdesc_and_tlvbuffer, - scanresp->bssdescriptsize); + bsssize); tsfdesc = pos + bsssize; tsfsize = 4 + 8 * scanresp->nr_sets; @@ -1435,7 +1435,7 @@ static int lbs_cfg_disconnect(struct wiphy *wiphy, struct net_device *dev, } static int lbs_cfg_set_default_key(struct wiphy *wiphy, - struct net_device *netdev, + struct net_device *netdev, int link_id, u8 key_index, bool unicast, bool multicast) { @@ -1455,8 +1455,8 @@ static int lbs_cfg_set_default_key(struct wiphy *wiphy, static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev, - u8 idx, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 idx, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct lbs_private *priv = wiphy_priv(wiphy); u16 key_info; @@ -1516,7 +1516,8 @@ static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev, static int lbs_cfg_del_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { lbs_deb_assoc("del_key: key_idx %d, mac_addr %pM\n", diff --git a/drivers/net/wireless/marvell/libertas/ethtool.c b/drivers/net/wireless/marvell/libertas/ethtool.c index d8e4f29b690d16f0648c9d14f934e350676e4a8b..9f53308a9935712aec898c35996351da0c0fb75f 100644 --- a/drivers/net/wireless/marvell/libertas/ethtool.c +++ b/drivers/net/wireless/marvell/libertas/ethtool.c @@ -20,8 +20,8 @@ static void lbs_ethtool_get_drvinfo(struct net_device *dev, priv->fwrelease >> 16 & 0xff, priv->fwrelease >> 8 & 0xff, priv->fwrelease & 0xff); - strlcpy(info->driver, "libertas", sizeof(info->driver)); - strlcpy(info->version, lbs_driver_version, sizeof(info->version)); + strscpy(info->driver, "libertas", sizeof(info->driver)); + strscpy(info->version, lbs_driver_version, sizeof(info->version)); } /* diff --git a/drivers/net/wireless/marvell/libertas/main.c b/drivers/net/wireless/marvell/libertas/main.c index 5c9f295536ead9f59ed8ce0923dceb6434249b0e..8f5220cee11235820263f47c99aabe5d9a5f1e1f 100644 --- a/drivers/net/wireless/marvell/libertas/main.c +++ b/drivers/net/wireless/marvell/libertas/main.c @@ -39,8 +39,7 @@ unsigned int lbs_debug; EXPORT_SYMBOL_GPL(lbs_debug); module_param_named(libertas_debug, lbs_debug, int, 0644); -unsigned int lbs_disablemesh; -EXPORT_SYMBOL_GPL(lbs_disablemesh); +static unsigned int lbs_disablemesh; module_param_named(libertas_disablemesh, lbs_disablemesh, int, 0644); diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c index bd835288ce5751c424750e55ae5c0ed2c7925f71..a04b66284af4a1e756f7e01158ca8d3b23913fc5 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c @@ -335,7 +335,7 @@ mwifiex_11n_create_rx_reorder_tbl(struct mwifiex_private *priv, u8 *ta, struct mwifiex_sta_node *node; /* - * If we get a TID, ta pair which is already present dispatch all the + * If we get a TID, ta pair which is already present dispatch all * the packets and move the window size until the ssn */ tbl = mwifiex_11n_get_rx_reorder_tbl(priv, tid, ta); diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c index 134114ac1ac00ab9acf86e1e617f3c909df367c6..bcd564dc3554a80c8a37ab18abd31d24c0cbdbe0 100644 --- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c +++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c @@ -142,7 +142,8 @@ static void *mwifiex_cfg80211_get_adapter(struct wiphy *wiphy) */ static int mwifiex_cfg80211_del_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev); static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; @@ -238,7 +239,7 @@ mwifiex_cfg80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev, tx_info->pkt_len = pkt_len; mwifiex_form_mgmt_frame(skb, buf, len); - *cookie = prandom_u32() | 1; + *cookie = get_random_u32() | 1; if (ieee80211_is_action(mgmt->frame_control)) skb = mwifiex_clone_skb_for_tx_status(priv, @@ -302,7 +303,7 @@ mwifiex_cfg80211_remain_on_channel(struct wiphy *wiphy, duration); if (!ret) { - *cookie = prandom_u32() | 1; + *cookie = get_random_u32() | 1; priv->roc_cfg.cookie = *cookie; priv->roc_cfg.chan = *chan; @@ -431,7 +432,7 @@ mwifiex_cfg80211_set_power_mgmt(struct wiphy *wiphy, */ static int mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool unicast, + int link_id, u8 key_index, bool unicast, bool multicast) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev); @@ -456,8 +457,8 @@ mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev, */ static int mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev); struct mwifiex_wep_key *wep_key; @@ -494,6 +495,7 @@ mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev, static int mwifiex_cfg80211_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *netdev, + int link_id, u8 key_index) { struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev); diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h index 26a48d8f49be98f0efe75b522ec3fe7e5b55354b..b4f945a549f71ce7711601d166431cf75fe57994 100644 --- a/drivers/net/wireless/marvell/mwifiex/fw.h +++ b/drivers/net/wireless/marvell/mwifiex/fw.h @@ -2104,7 +2104,7 @@ struct mwifiex_fw_mef_entry { struct host_cmd_ds_mef_cfg { __le32 criteria; __le16 num_entries; - struct mwifiex_fw_mef_entry mef_entry[]; + u8 mef_entry_data[]; } __packed; #define CONNECTION_TYPE_INFRA 0 @@ -2254,7 +2254,7 @@ struct coalesce_receive_filt_rule { struct host_cmd_ds_coalesce_cfg { __le16 action; __le16 num_of_rules; - struct coalesce_receive_filt_rule rule[]; + u8 rule_data[]; } __packed; struct host_cmd_ds_multi_chan_policy { diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c index fc77489cc511939de478df2d67b467e1b93220cf..7dddb4b5dea1dd9dadbb9c97b436660af8166832 100644 --- a/drivers/net/wireless/marvell/mwifiex/init.c +++ b/drivers/net/wireless/marvell/mwifiex/init.c @@ -51,9 +51,10 @@ static void wakeup_timer_fn(struct timer_list *t) adapter->if_ops.card_reset(adapter); } -static void fw_dump_timer_fn(struct timer_list *t) +static void fw_dump_work(struct work_struct *work) { - struct mwifiex_adapter *adapter = from_timer(adapter, t, devdump_timer); + struct mwifiex_adapter *adapter = + container_of(work, struct mwifiex_adapter, devdump_work.work); mwifiex_upload_device_dump(adapter); } @@ -309,7 +310,7 @@ static void mwifiex_init_adapter(struct mwifiex_adapter *adapter) adapter->active_scan_triggered = false; timer_setup(&adapter->wakeup_timer, wakeup_timer_fn, 0); adapter->devdump_len = 0; - timer_setup(&adapter->devdump_timer, fw_dump_timer_fn, 0); + INIT_DELAYED_WORK(&adapter->devdump_work, fw_dump_work); } /* @@ -388,7 +389,7 @@ static void mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter) { del_timer(&adapter->wakeup_timer); - del_timer_sync(&adapter->devdump_timer); + cancel_delayed_work_sync(&adapter->devdump_work); mwifiex_cancel_all_pending_cmd(adapter); wake_up_interruptible(&adapter->cmd_wait_q.wait); wake_up_interruptible(&adapter->hs_activate_wait_q); diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h index 87729d251fedc654742bc2bf079c31477a1d38de..63f861e6b28af3a4b808333104b8255ca40ee352 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.h +++ b/drivers/net/wireless/marvell/mwifiex/main.h @@ -37,6 +37,7 @@ #include #include #include +#include #include "decl.h" #include "ioctl.h" @@ -1043,7 +1044,7 @@ struct mwifiex_adapter { /* Device dump data/length */ void *devdump_data; int devdump_len; - struct timer_list devdump_timer; + struct delayed_work devdump_work; bool ignore_btcoex_events; }; diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c index f7f9277602a5e0ce7049e30a086f42b18ded231c..5dcf61761a165ae0d1b9c19cd84f4e740c47a09a 100644 --- a/drivers/net/wireless/marvell/mwifiex/pcie.c +++ b/drivers/net/wireless/marvell/mwifiex/pcie.c @@ -644,7 +644,7 @@ static int mwifiex_pm_wakeup_card(struct mwifiex_adapter *adapter) { struct pcie_service_card *card = adapter->card; const struct mwifiex_pcie_card_reg *reg = card->pcie.reg; - int retval; + int retval __maybe_unused; mwifiex_dbg(adapter, EVENT, "event: Wakeup device...\n"); diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c index 512b5bb9cf6f57021c2cf031116f88889ba5d539..e2800a831c8edd1646c82f3ee63f11a277588710 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c @@ -1435,7 +1435,7 @@ mwifiex_cmd_mef_cfg(struct mwifiex_private *priv, mef_entry = (struct mwifiex_fw_mef_entry *)pos; mef_entry->mode = mef->mef_entry[i].mode; mef_entry->action = mef->mef_entry[i].action; - pos += sizeof(*mef_cfg->mef_entry); + pos += sizeof(*mef_entry); if (mwifiex_cmd_append_rpn_expression(priv, &mef->mef_entry[i], &pos)) @@ -1631,7 +1631,7 @@ mwifiex_cmd_coalesce_cfg(struct mwifiex_private *priv, coalesce_cfg->action = cpu_to_le16(cmd_action); coalesce_cfg->num_of_rules = cpu_to_le16(cfg->num_of_rules); - rule = coalesce_cfg->rule; + rule = (void *)coalesce_cfg->rule_data; for (cnt = 0; cnt < cfg->num_of_rules; cnt++) { rule->header.type = cpu_to_le16(TLV_TYPE_COALESCE_RULE); diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c index b95e90a7d124a6d0a3df299d352659405f9ed216..df9cdd10a494f394deb45f3c980ba898e8f19823 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_event.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c @@ -611,8 +611,8 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv, * transmission event get lost, in this cornel case, * user would still get partial of the dump. */ - mod_timer(&adapter->devdump_timer, - jiffies + msecs_to_jiffies(MWIFIEX_TIMER_10S)); + schedule_delayed_work(&adapter->devdump_work, + msecs_to_jiffies(MWIFIEX_TIMER_10S)); } /* Overflow check */ @@ -623,7 +623,7 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv, adapter->event_skb->data, event_skb->len); adapter->devdump_len += event_skb->len; - if (le16_to_cpu(fw_dump_hdr->type == FW_DUMP_INFO_ENDED)) { + if (le16_to_cpu(fw_dump_hdr->type) == FW_DUMP_INFO_ENDED) { mwifiex_dbg(adapter, MSG, "receive end of transmission flag event!\n"); goto upload_dump; @@ -631,7 +631,7 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv, return; upload_dump: - del_timer_sync(&adapter->devdump_timer); + cancel_delayed_work_sync(&adapter->devdump_work); mwifiex_upload_device_dump(adapter); } diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c index c2f2ce2a3f95b6da8875108f0cba2df939d85e29..d3ab9572e71157f4436c7c7f429617070ef4b674 100644 --- a/drivers/net/wireless/marvell/mwifiex/usb.c +++ b/drivers/net/wireless/marvell/mwifiex/usb.c @@ -911,14 +911,14 @@ static int mwifiex_usb_prepare_tx_aggr_skb(struct mwifiex_adapter *adapter, memcpy(payload, skb_tmp->data, skb_tmp->len); if (skb_queue_empty(&port->tx_aggr.aggr_list)) { /* do not padding for last packet*/ - *(u16 *)payload = cpu_to_le16(skb_tmp->len); - *(u16 *)&payload[2] = + *(__le16 *)payload = cpu_to_le16(skb_tmp->len); + *(__le16 *)&payload[2] = cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2 | 0x80); skb_trim(skb_aggr, skb_aggr->len - pad); } else { /* add aggregation interface header */ - *(u16 *)payload = cpu_to_le16(skb_tmp->len + pad); - *(u16 *)&payload[2] = + *(__le16 *)payload = cpu_to_le16(skb_tmp->len + pad); + *(__le16 *)&payload[2] = cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2); } @@ -1097,9 +1097,9 @@ send_aggr_buf: } payload = skb->data; - *(u16 *)&payload[2] = + *(__le16 *)&payload[2] = cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2 | 0x80); - *(u16 *)payload = cpu_to_le16(skb->len); + *(__le16 *)payload = cpu_to_le16(skb->len); skb_send = skb; context = &port->tx_data_list[port->tx_data_ix++]; return mwifiex_usb_construct_send_urb(adapter, port, ep, diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index 40cb91097b2e73a2f30dcfffaeb54c95fcb16e25..7378c4d1e1567e06379c9e5497a2d236aecdeb8c 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -696,10 +696,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) skb_reserve(skb, q->buf_offset); - if (q == &dev->q_rx[MT_RXQ_MCU]) { - u32 *rxfce = (u32 *)skb->cb; - *rxfce = info; - } + *(u32 *)skb->cb = info; __skb_put(skb, len); done++; @@ -758,7 +755,7 @@ mt76_dma_init(struct mt76_dev *dev, dev->napi_dev.threaded = 1; mt76_for_each_q_rx(dev, i) { - netif_napi_add(&dev->napi_dev, &dev->napi[i], poll, 64); + netif_napi_add(&dev->napi_dev, &dev->napi[i], poll); mt76_dma_rx_fill(dev, &dev->q_rx[i]); napi_enable(&dev->napi[i]); } diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 253cbc1956d19953ca16dd10b154cf5d5b64c34e..6de13d6414389b5d9873363418cc1d3cfedfeff4 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -267,7 +267,8 @@ static void mt76_init_stream_cap(struct mt76_phy *phy, } vht_cap->vht_mcs.rx_mcs_map = cpu_to_le16(mcs_map); vht_cap->vht_mcs.tx_mcs_map = cpu_to_le16(mcs_map); - vht_cap->vht_mcs.tx_highest |= + if (ieee80211_hw_check(phy->hw, SUPPORTS_VHT_EXT_NSS_BW)) + vht_cap->vht_mcs.tx_highest |= cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE); } diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 4da77d47b0a677c4c6336e0682a1c03584a496ee..87db9498dea44a517ad9cfd0c17f7c90303427fb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -252,6 +252,30 @@ struct mt76_queue_ops { void (*reset_q)(struct mt76_dev *dev, struct mt76_queue *q); }; +enum mt76_phy_type { + MT_PHY_TYPE_CCK, + MT_PHY_TYPE_OFDM, + MT_PHY_TYPE_HT, + MT_PHY_TYPE_HT_GF, + MT_PHY_TYPE_VHT, + MT_PHY_TYPE_HE_SU = 8, + MT_PHY_TYPE_HE_EXT_SU, + MT_PHY_TYPE_HE_TB, + MT_PHY_TYPE_HE_MU, + __MT_PHY_TYPE_HE_MAX, +}; + +struct mt76_sta_stats { + u64 tx_mode[__MT_PHY_TYPE_HE_MAX]; + u64 tx_bw[4]; /* 20, 40, 80, 160 */ + u64 tx_nss[4]; /* 1, 2, 3, 4 */ + u64 tx_mcs[16]; /* mcs idx */ + u64 tx_bytes; + u32 tx_packets; + u32 tx_retries; + u32 tx_failed; +}; + enum mt76_wcid_flags { MT_WCID_FLAG_CHECK_PS, MT_WCID_FLAG_PS, @@ -299,6 +323,8 @@ struct mt76_wcid { struct list_head list; struct idr pktid; + + struct mt76_sta_stats stats; }; struct mt76_txq { @@ -342,7 +368,8 @@ struct mt76_rx_tid { #define MT_PACKET_ID_MASK GENMASK(6, 0) #define MT_PACKET_ID_NO_ACK 0 #define MT_PACKET_ID_NO_SKB 1 -#define MT_PACKET_ID_FIRST 2 +#define MT_PACKET_ID_WED 2 +#define MT_PACKET_ID_FIRST 3 #define MT_PACKET_ID_HAS_RATE BIT(7) /* This is timer for when to give up when waiting for TXS callback, * with starting time being the time at which the DMA_DONE callback @@ -527,7 +554,6 @@ struct mt76_usb { struct mt76_reg_pair *rp; int rp_len; u32 base; - bool burst; } mcu; }; @@ -815,26 +841,6 @@ struct mt76_power_limits { s8 ru[7][12]; }; -enum mt76_phy_type { - MT_PHY_TYPE_CCK, - MT_PHY_TYPE_OFDM, - MT_PHY_TYPE_HT, - MT_PHY_TYPE_HT_GF, - MT_PHY_TYPE_VHT, - MT_PHY_TYPE_HE_SU = 8, - MT_PHY_TYPE_HE_EXT_SU, - MT_PHY_TYPE_HE_TB, - MT_PHY_TYPE_HE_MU, - __MT_PHY_TYPE_HE_MAX, -}; - -struct mt76_sta_stats { - u64 tx_mode[__MT_PHY_TYPE_HE_MAX]; - u64 tx_bw[4]; /* 20, 40, 80, 160 */ - u64 tx_nss[4]; /* 1, 2, 3, 4 */ - u64 tx_mcs[16]; /* mcs idx */ -}; - struct mt76_ethtool_worker_info { u64 *data; int idx; diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/main.c b/drivers/net/wireless/mediatek/mt76/mt7603/main.c index 051715ed90ddd203874020207d83773cb73ed34f..ca50feb0b3a9d31fb768df9d1c1d8791f69f2d38 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/main.c @@ -658,7 +658,7 @@ mt7603_sta_rate_tbl_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mt7603_wtbl_set_rates(dev, msta, NULL, msta->rates); msta->rate_probe = false; mt7603_wtbl_set_smps(dev, msta, - sta->smps_mode == IEEE80211_SMPS_DYNAMIC); + sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC); spin_unlock_bh(&dev->mt76.lock); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index ad6c7d632eed45bdcf6c3e0abc43dcae9aab6507..2ce1705c0f4338607a54b8500d50e960ce9da374 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -345,6 +345,7 @@ static int mt7615_mac_fill_rx(struct mt7615_dev *dev, struct sk_buff *skb) u32 rxd1 = le32_to_cpu(rxd[1]); u32 rxd2 = le32_to_cpu(rxd[2]); u32 csum_mask = MT_RXD0_NORMAL_IP_SUM | MT_RXD0_NORMAL_UDP_TCP_SUM; + u32 csum_status = *(u32 *)skb->cb; bool unicast, hdr_trans, remove_pad, insert_ccmp_hdr = false; u16 hdr_gap; int phy_idx; @@ -394,7 +395,8 @@ static int mt7615_mac_fill_rx(struct mt7615_dev *dev, struct sk_buff *skb) spin_unlock_bh(&dev->sta_poll_lock); } - if ((rxd0 & csum_mask) == csum_mask) + if (mt76_is_mmio(&dev->mt76) && (rxd0 & csum_mask) == csum_mask && + !(csum_status & (BIT(0) | BIT(2) | BIT(3)))) skb->ip_summed = CHECKSUM_UNNECESSARY; if (rxd2 & MT_RXD2_NORMAL_FCS_ERR) @@ -610,14 +612,14 @@ static int mt7615_mac_fill_rx(struct mt7615_dev *dev, struct sk_buff *skb) * When header translation failure is indicated, * the hardware will insert an extra 2-byte field * containing the data length after the protocol - * type field. + * type field. This happens either when the LLC-SNAP + * pattern did not match, or if a VLAN header was + * detected. */ pad_start = 12; if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q) pad_start += 4; - - if (get_unaligned_be16(skb->data + pad_start) != - skb->len - pad_start - 2) + else pad_start = 0; } @@ -1088,7 +1090,7 @@ u32 mt7615_mac_get_sta_tid_sn(struct mt7615_dev *dev, int wcid, u8 tid) offset %= 32; val = mt76_rr(dev, addr); - val >>= (tid % 32); + val >>= offset; if (offset > 20) { addr += 4; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index 9bf8545c8c17c0b6f9935532ad420dc07b794136..8d4733f87cda9637a321cf32cbc551138ddf285b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -1195,12 +1195,16 @@ static void mt7615_sta_set_decap_offload(struct ieee80211_hw *hw, struct mt7615_dev *dev = mt7615_hw_dev(hw); struct mt7615_sta *msta = (struct mt7615_sta *)sta->drv_priv; + mt7615_mutex_acquire(dev); + if (enabled) set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); else clear_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); mt7615_mcu_set_sta_decap_offload(dev, vif, sta); + + mt7615_mutex_release(dev); } #ifdef CONFIG_PM diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c index 49ab3a1f3b9bb730b4005fe3df6798b9447ca414..304212f5f8da7a65864b0100d1a072c8d14557fd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c @@ -83,6 +83,7 @@ static int mt7663s_probe(struct sdio_func *func, .tx_complete_skb = mt7663_usb_sdio_tx_complete_skb, .tx_status_data = mt7663_usb_sdio_tx_status_data, .rx_skb = mt7615_queue_rx_skb, + .rx_check = mt7615_rx_check, .sta_ps = mt7615_sta_ps, .sta_add = mt7615_mac_sta_add, .sta_remove = mt7615_mac_sta_remove, @@ -180,7 +181,6 @@ static void mt7663s_remove(struct sdio_func *func) mt76_free_device(&dev->mt76); } -#ifdef CONFIG_PM static int mt7663s_suspend(struct device *dev) { struct sdio_func *func = dev_to_sdio_func(dev); @@ -235,28 +235,20 @@ static int mt7663s_resume(struct device *dev) return err; } -static const struct dev_pm_ops mt7663s_pm_ops = { - .suspend = mt7663s_suspend, - .resume = mt7663s_resume, -}; -#endif - MODULE_DEVICE_TABLE(sdio, mt7663s_table); MODULE_FIRMWARE(MT7663_OFFLOAD_FIRMWARE_N9); MODULE_FIRMWARE(MT7663_OFFLOAD_ROM_PATCH); MODULE_FIRMWARE(MT7663_FIRMWARE_N9); MODULE_FIRMWARE(MT7663_ROM_PATCH); +static DEFINE_SIMPLE_DEV_PM_OPS(mt7663s_pm_ops, mt7663s_suspend, mt7663s_resume); + static struct sdio_driver mt7663s_driver = { .name = KBUILD_MODNAME, .probe = mt7663s_probe, .remove = mt7663s_remove, .id_table = mt7663s_table, -#ifdef CONFIG_PM - .drv = { - .pm = &mt7663s_pm_ops, - } -#endif + .drv.pm = pm_sleep_ptr(&mt7663s_pm_ops), }; module_sdio_driver(mt7663s_driver); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c index 967641aebf5f00b23effcffd2e46760a2d5db1a5..f2d651d7adff46d8b6c4031b39d957a8ca52d1a7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c @@ -119,6 +119,7 @@ static int mt7663u_probe(struct usb_interface *usb_intf, .tx_complete_skb = mt7663_usb_sdio_tx_complete_skb, .tx_status_data = mt7663_usb_sdio_tx_status_data, .rx_skb = mt7615_queue_rx_skb, + .rx_check = mt7615_rx_check, .sta_ps = mt7615_sta_ps, .sta_add = mt7615_mac_sta_add, .sta_remove = mt7615_mac_sta_remove, diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 75afcb469d3cf98116bcc2e2ad79dda593d4e8bb..635192c878cb4b9422d42af13145a21f16ce65b7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -63,6 +63,12 @@ enum { REPEATER_BSSID_MAX = 0x3f, }; +struct mt76_connac_reg_map { + u32 phys; + u32 maps; + u32 size; +}; + struct mt76_connac_pm { bool enable:1; bool enable_user:1; @@ -348,9 +354,10 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, enum mt76_txq_id qid, u32 changed); +bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid, + __le32 *txs_data); bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, - int pid, __le32 *txs_data, - struct mt76_sta_stats *stats); + int pid, __le32 *txs_data); void mt76_connac2_mac_decode_he_radiotap(struct mt76_dev *dev, struct sk_buff *skb, __le32 *rxv, u32 mode); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h index 67ce216fb564920a5ce7cfe15693c2be50d3057c..f33171bcd343222e31997d7f19e1a851c6196154 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h @@ -158,6 +158,14 @@ enum { #define MT_TXS4_TIMESTAMP GENMASK(31, 0) +/* PPDU based TXS */ +#define MT_TXS5_MPDU_TX_BYTE GENMASK(22, 0) +#define MT_TXS5_MPDU_TX_CNT GENMASK(31, 23) + +#define MT_TXS6_MPDU_FAIL_CNT GENMASK(31, 23) + +#define MT_TXS7_MPDU_RETRY_CNT GENMASK(31, 23) + /* RXD DW1 */ #define MT_RXD1_NORMAL_WLAN_IDX GENMASK(9, 0) #define MT_RXD1_NORMAL_GROUP_1 BIT(11) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index 18dea8e1fb20b30896ea8aba4e697703d2de0df6..34ac3d81a510256f055cff164393afe32b808c3a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -490,6 +490,10 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, p_fmt = mt76_is_mmio(dev) ? MT_TX_TYPE_CT : MT_TX_TYPE_SF; q_idx = wmm_idx * MT76_CONNAC_MAX_WMM_SETS + mt76_connac_lmac_mapping(skb_get_queue_mapping(skb)); + + /* counting non-offloading skbs */ + wcid->stats.tx_bytes += skb->len; + wcid->stats.tx_packets++; } val = FIELD_PREP(MT_TXD0_TX_BYTES, skb->len + sz_txd) | @@ -550,35 +554,29 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, } EXPORT_SYMBOL_GPL(mt76_connac2_mac_write_txwi); -bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, - int pid, __le32 *txs_data, - struct mt76_sta_stats *stats) +bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid, + __le32 *txs_data) { + struct mt76_sta_stats *stats = &wcid->stats; struct ieee80211_supported_band *sband; struct mt76_phy *mphy; - struct ieee80211_tx_info *info; - struct sk_buff_head list; struct rate_info rate = {}; - struct sk_buff *skb; bool cck = false; u32 txrate, txs, mode; - mt76_tx_status_lock(dev, &list); - skb = mt76_tx_status_skb_get(dev, wcid, pid, &list); - if (!skb) - goto out; - txs = le32_to_cpu(txs_data[0]); - info = IEEE80211_SKB_CB(skb); - if (!(txs & MT_TXS0_ACK_ERROR_MASK)) - info->flags |= IEEE80211_TX_STAT_ACK; - - info->status.ampdu_len = 1; - info->status.ampdu_ack_len = !!(info->flags & - IEEE80211_TX_STAT_ACK); - - info->status.rates[0].idx = -1; + /* PPDU based reporting */ + if (FIELD_GET(MT_TXS0_TXS_FORMAT, txs) > 1) { + stats->tx_bytes += + le32_get_bits(txs_data[5], MT_TXS5_MPDU_TX_BYTE); + stats->tx_packets += + le32_get_bits(txs_data[5], MT_TXS5_MPDU_TX_CNT); + stats->tx_failed += + le32_get_bits(txs_data[6], MT_TXS6_MPDU_FAIL_CNT); + stats->tx_retries += + le32_get_bits(txs_data[7], MT_TXS7_MPDU_RETRY_CNT); + } txrate = FIELD_GET(MT_TXS0_TX_RATE, txs); @@ -613,7 +611,7 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, case MT_PHY_TYPE_HT: case MT_PHY_TYPE_HT_GF: if (rate.mcs > 31) - goto out; + return false; rate.flags = RATE_INFO_FLAGS_MCS; if (wcid->rate.flags & RATE_INFO_FLAGS_SHORT_GI) @@ -621,7 +619,7 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, break; case MT_PHY_TYPE_VHT: if (rate.mcs > 9) - goto out; + return false; rate.flags = RATE_INFO_FLAGS_VHT_MCS; break; @@ -630,14 +628,14 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, case MT_PHY_TYPE_HE_TB: case MT_PHY_TYPE_HE_MU: if (rate.mcs > 11) - goto out; + return false; rate.he_gi = wcid->rate.he_gi; rate.he_dcm = FIELD_GET(MT_TX_RATE_DCM, txrate); rate.flags = RATE_INFO_FLAGS_HE_MCS; break; default: - goto out; + return false; } stats->tx_mode[mode]++; @@ -662,10 +660,34 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, } wcid->rate = rate; -out: - if (skb) - mt76_tx_status_skb_done(dev, skb, &list); + return true; +} +EXPORT_SYMBOL_GPL(mt76_connac2_mac_fill_txs); + +bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid, + int pid, __le32 *txs_data) +{ + struct sk_buff_head list; + struct sk_buff *skb; + + mt76_tx_status_lock(dev, &list); + skb = mt76_tx_status_skb_get(dev, wcid, pid, &list); + if (skb) { + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + bool noacked = !(info->flags & IEEE80211_TX_STAT_ACK); + + if (!(le32_to_cpu(txs_data[0]) & MT_TXS0_ACK_ERROR_MASK)) + info->flags |= IEEE80211_TX_STAT_ACK; + + info->status.ampdu_len = 1; + info->status.ampdu_ack_len = !noacked; + info->status.rates[0].idx = -1; + wcid->stats.tx_failed += noacked; + + mt76_connac2_mac_fill_txs(dev, wcid, txs_data); + mt76_tx_status_skb_done(dev, skb, &list); + } mt76_tx_status_unlock(dev, &list); return !!skb; diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 9b17bd97ec0945a94dac93b61d6d4c38678bd06f..011fc9729b38c21eff1d3ae63ea56e8662e91263 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -260,8 +260,10 @@ mt76_connac_mcu_add_nested_tlv(struct sk_buff *skb, int tag, int len, ntlv = le16_to_cpu(ntlv_hdr->tlv_num); ntlv_hdr->tlv_num = cpu_to_le16(ntlv + 1); - if (sta_hdr) - le16_add_cpu(&sta_hdr->len, len); + if (sta_hdr) { + len += le16_to_cpu(sta_hdr->len); + sta_hdr->len = cpu_to_le16(len); + } return ptlv; } @@ -594,14 +596,14 @@ mt76_connac_mcu_sta_amsdu_tlv(struct sk_buff *skb, struct ieee80211_sta *sta, vif->type != NL80211_IFTYPE_STATION) return; - if (!sta->max_amsdu_len) + if (!sta->deflink.agg.max_amsdu_len) return; tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu)); amsdu = (struct sta_rec_amsdu *)tlv; amsdu->max_amsdu_num = 8; amsdu->amsdu_en = true; - amsdu->max_mpdu_size = sta->max_amsdu_len >= + amsdu->max_mpdu_size = sta->deflink.agg.max_amsdu_len >= IEEE80211_MAX_MPDU_LEN_VHT_7991; wcid->amsdu = true; @@ -896,7 +898,7 @@ void mt76_connac_mcu_wtbl_smps_tlv(struct sk_buff *skb, tlv = mt76_connac_mcu_add_nested_tlv(skb, WTBL_SMPS, sizeof(*smps), wtbl_tlv, sta_wtbl); smps = (struct wtbl_smps *)tlv; - smps->smps = (sta->smps_mode == IEEE80211_SMPS_DYNAMIC); + smps->smps = (sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC); } EXPORT_SYMBOL_GPL(mt76_connac_mcu_wtbl_smps_tlv); @@ -2648,7 +2650,7 @@ int mt76_connac_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, } EXPORT_SYMBOL_GPL(mt76_connac_mcu_add_key); -/* SIFS 20us + 512 byte beacon tranmitted by 1Mbps (3906us) */ +/* SIFS 20us + 512 byte beacon transmitted by 1Mbps (3906us) */ #define BCN_TX_ESTIMATE_TIME (4096 + 20) void mt76_connac_mcu_bss_ext_tlv(struct sk_buff *skb, struct mt76_vif *mvif) { @@ -2886,6 +2888,10 @@ int mt76_connac2_load_ram(struct mt76_dev *dev, const char *fw_wm, goto out; } + snprintf(dev->hw->wiphy->fw_version, + sizeof(dev->hw->wiphy->fw_version), + "%.10s-%.15s", hdr->fw_ver, hdr->build_date); + release_firmware(fw); if (!fw_wa) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index f1d7c05bd794d10c323b82440f19bb13fce0d7fd..718f427d8f6b260702bbffc86cba46aaaae8f059 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -10,6 +10,7 @@ #define FW_FEATURE_SET_KEY_IDX GENMASK(2, 1) #define FW_FEATURE_ENCRY_MODE BIT(4) #define FW_FEATURE_OVERRIDE_ADDR BIT(5) +#define FW_FEATURE_NON_DL BIT(6) #define DL_MODE_ENCRYPT BIT(0) #define DL_MODE_KEY_IDX GENMASK(2, 1) @@ -33,6 +34,12 @@ #define PATCH_SEC_ENC_SCRAMBLE_INFO_MASK GENMASK(15, 0) #define PATCH_SEC_ENC_AES_KEY_MASK GENMASK(7, 0) +enum { + FW_TYPE_DEFAULT = 0, + FW_TYPE_CLC = 2, + FW_TYPE_MAX_NUM = 255 +}; + #define MCU_PQ_ID(p, q) (((p) << 15) | ((q) << 10)) #define MCU_PKT_ID 0xa0 @@ -174,7 +181,8 @@ struct mt76_connac2_fw_region { __le32 addr; __le32 len; u8 feature_set; - u8 rsv1[15]; + u8 type; + u8 rsv1[14]; } __packed; struct tlv { @@ -1172,6 +1180,7 @@ enum { MCU_CE_CMD_SET_ROC = 0x1c, MCU_CE_CMD_SET_EDCA_PARMS = 0x1d, MCU_CE_CMD_SET_P2P_OPPPS = 0x33, + MCU_CE_CMD_SET_CLC = 0x5c, MCU_CE_CMD_SET_RATE_TX_POWER = 0x5d, MCU_CE_CMD_SCHED_SCAN_ENABLE = 0x61, MCU_CE_CMD_SCHED_SCAN_REQ = 0x62, diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c index de30cf5e2d2f9c3039ae46c1d811baf4aa11cc70..93d96739f802cf403dc1bd152f22ab1f78b5bc34 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c @@ -404,7 +404,7 @@ void mt76x02_mac_write_txwi(struct mt76x02_dev *dev, struct mt76x02_txwi *txwi, txwi->rate |= cpu_to_le16(MT_RXWI_RATE_LDPC); if ((info->flags & IEEE80211_TX_CTL_STBC) && nss == 1) txwi->rate |= cpu_to_le16(MT_RXWI_RATE_STBC); - if (nss > 1 && sta && sta->smps_mode == IEEE80211_SMPS_DYNAMIC) + if (nss > 1 && sta && sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) txwi_flags |= MT_TXWI_FLAGS_MMPS; if (!(info->flags & IEEE80211_TX_CTL_NO_ACK)) txwi->ack_ctl |= MT_TXWI_ACK_CTL_REQ; diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c index c6c16fe8ee859dd3ae433d42ab3f7ef1fc3ff693..02da543dfc5cf381f6edac1753c0d627504e3e48 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c @@ -21,29 +21,16 @@ static void mt76x02u_multiple_mcu_reads(struct mt76_dev *dev, u8 *data, int len) { struct mt76_usb *usb = &dev->usb; - u32 reg, val; int i; - if (usb->mcu.burst) { - WARN_ON_ONCE(len / 4 != usb->mcu.rp_len); - - reg = usb->mcu.rp[0].reg - usb->mcu.base; - for (i = 0; i < usb->mcu.rp_len; i++) { - val = get_unaligned_le32(data + 4 * i); - usb->mcu.rp[i].reg = reg++; - usb->mcu.rp[i].value = val; - } - } else { - WARN_ON_ONCE(len / 8 != usb->mcu.rp_len); - - for (i = 0; i < usb->mcu.rp_len; i++) { - reg = get_unaligned_le32(data + 8 * i) - - usb->mcu.base; - val = get_unaligned_le32(data + 8 * i + 4); - - WARN_ON_ONCE(usb->mcu.rp[i].reg != reg); - usb->mcu.rp[i].value = val; - } + WARN_ON_ONCE(len / 8 != usb->mcu.rp_len); + + for (i = 0; i < usb->mcu.rp_len; i++) { + u32 reg = get_unaligned_le32(data + 8 * i) - usb->mcu.base; + u32 val = get_unaligned_le32(data + 8 * i + 4); + + WARN_ON_ONCE(usb->mcu.rp[i].reg != reg); + usb->mcu.rp[i].value = val; } } @@ -207,7 +194,6 @@ mt76x02u_mcu_rd_rp(struct mt76_dev *dev, u32 base, usb->mcu.rp = data; usb->mcu.rp_len = n; usb->mcu.base = base; - usb->mcu.burst = false; ret = __mt76x02u_mcu_send_msg(dev, skb, CMD_RANDOM_READ, true); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c index fd76db8f5269c401a222c08700aab4ed60d63742..6ef3431cad6483332f2b463f70198d92f5d3db90 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c @@ -23,9 +23,9 @@ mt7915_implicit_txbf_set(void *data, u64 val) { struct mt7915_dev *dev = data; - if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state)) - return -EBUSY; - + /* The existing connected stations shall reconnect to apply + * new implicit txbf configuration. + */ dev->ibf = !!val; return mt7915_mcu_set_txbf(dev, MT_BF_TYPE_UPDATE); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 60ae834d95a6daad435952367c0fafd7daf425e5..a4bcc617c1a34c497092f010f8896b7ca40556dd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -176,7 +176,7 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) /* * We don't support reading GI info from txs packets. * For accurate tx status reporting and AQL improvement, - we need to make sure that flags match so polling GI + * we need to make sure that flags match so polling GI * from per-sta counters directly. */ rate = &msta->wcid.rate; @@ -232,7 +232,8 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) bool unicast, insert_ccmp_hdr = false; u8 remove_pad, amsdu_info; u8 mode = 0, qos_ctl = 0; - struct mt7915_sta *msta; + struct mt7915_sta *msta = NULL; + u32 csum_status = *(u32 *)skb->cb; bool hdr_trans; u16 hdr_gap; u16 seq_ctrl = 0; @@ -288,7 +289,8 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) if (!sband->channels) return -EINVAL; - if ((rxd0 & csum_mask) == csum_mask) + if ((rxd0 & csum_mask) == csum_mask && + !(csum_status & (BIT(0) | BIT(2) | BIT(3)))) skb->ip_summed = CHECKSUM_UNNECESSARY; if (rxd1 & MT_RXD1_NORMAL_FCS_ERR) @@ -446,14 +448,14 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb) * When header translation failure is indicated, * the hardware will insert an extra 2-byte field * containing the data length after the protocol - * type field. + * type field. This happens either when the LLC-SNAP + * pattern did not match, or if a VLAN header was + * detected. */ pad_start = 12; if (get_unaligned_be16(skb->data + pad_start) == ETH_P_8021Q) pad_start += 4; - - if (get_unaligned_be16(skb->data + pad_start) != - skb->len - pad_start - 2) + else pad_start = 0; } @@ -1001,7 +1003,7 @@ static void mt7915_mac_add_txs(struct mt7915_dev *dev, void *data) wcidx = le32_get_bits(txs_data[2], MT_TXS2_WCID); pid = le32_get_bits(txs_data[3], MT_TXS3_PID); - if (pid < MT_PACKET_ID_FIRST) + if (pid < MT_PACKET_ID_WED) return; if (wcidx >= mt7915_wtbl_size(dev)) @@ -1015,8 +1017,11 @@ static void mt7915_mac_add_txs(struct mt7915_dev *dev, void *data) msta = container_of(wcid, struct mt7915_sta, wcid); - mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data, - &msta->stats); + if (pid == MT_PACKET_ID_WED) + mt76_connac2_mac_fill_txs(&dev->mt76, wcid, txs_data); + else + mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data); + if (!wcid->sta) goto out; @@ -1047,7 +1052,7 @@ bool mt7915_rx_check(struct mt76_dev *mdev, void *data, int len) return false; case PKT_TYPE_TXS: for (rxd += 2; rxd + 8 <= end; rxd += 8) - mt7915_mac_add_txs(dev, rxd); + mt7915_mac_add_txs(dev, rxd); return false; case PKT_TYPE_RX_FW_MONITOR: mt7915_debugfs_rx_fw_monitor(dev, data, len); @@ -1084,7 +1089,7 @@ void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, break; case PKT_TYPE_TXS: for (rxd += 2; rxd + 8 <= end; rxd += 8) - mt7915_mac_add_txs(dev, rxd); + mt7915_mac_add_txs(dev, rxd); dev_kfree_skb(skb); break; case PKT_TYPE_RX_FW_MONITOR: @@ -2071,8 +2076,9 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw, } flowid = ffs(~msta->twt.flowid_mask) - 1; - le16p_replace_bits(&twt_agrt->req_type, flowid, - IEEE80211_TWT_REQTYPE_FLOWID); + twt_agrt->req_type &= ~cpu_to_le16(IEEE80211_TWT_REQTYPE_FLOWID); + twt_agrt->req_type |= le16_encode_bits(flowid, + IEEE80211_TWT_REQTYPE_FLOWID); table_id = ffs(~dev->twt.table_mask) - 1; exp = FIELD_GET(IEEE80211_TWT_REQTYPE_WAKE_INT_EXP, req_type); @@ -2122,8 +2128,9 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw, unlock: mutex_unlock(&dev->mt76.mutex); out: - le16p_replace_bits(&twt_agrt->req_type, setup_cmd, - IEEE80211_TWT_REQTYPE_SETUP_CMD); + twt_agrt->req_type &= ~cpu_to_le16(IEEE80211_TWT_REQTYPE_SETUP_CMD); + twt_agrt->req_type |= + le16_encode_bits(setup_cmd, IEEE80211_TWT_REQTYPE_SETUP_CMD); twt->control = (twt->control & IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT) | (twt->control & IEEE80211_TWT_CONTROL_RX_DISABLED); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index bd3386bf0f8a07ffe76d57ac0ba91b2630234a13..89b519cfd14c3bb272c8f977ba00542a0975c2f3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -1010,6 +1010,23 @@ static void mt7915_sta_statistics(struct ieee80211_hw *hw, } sinfo->txrate.flags = txrate->flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); + + /* offloading flows bypass networking stack, so driver counts and + * reports sta statistics via NL80211_STA_INFO when WED is active. + */ + if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) { + sinfo->tx_bytes = msta->wcid.stats.tx_bytes; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64); + + sinfo->tx_packets = msta->wcid.stats.tx_packets; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + + sinfo->tx_failed = msta->wcid.stats.tx_failed; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); + + sinfo->tx_retries = msta->wcid.stats.tx_retries; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES); + } } static void mt7915_sta_rc_work(void *data, struct ieee80211_sta *sta) @@ -1224,7 +1241,7 @@ static void mt7915_ethtool_worker(void *wi_data, struct ieee80211_sta *sta) if (msta->vif->mt76.idx != wi->idx) return; - mt76_ethtool_worker(wi, &msta->stats); + mt76_ethtool_worker(wi, &msta->wcid.stats); } static diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index f8306796194515a2b01bfccc073883c94c5ae7d4..8d297e4aa7d430c2ee42b052ccff0282f766d70d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -925,7 +925,7 @@ mt7915_mcu_sta_amsdu_tlv(struct mt7915_dev *dev, struct sk_buff *skb, vif->type != NL80211_IFTYPE_AP) return; - if (!sta->max_amsdu_len) + if (!sta->deflink.agg.max_amsdu_len) return; tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu)); @@ -934,7 +934,7 @@ mt7915_mcu_sta_amsdu_tlv(struct mt7915_dev *dev, struct sk_buff *skb, amsdu->amsdu_en = true; msta->wcid.amsdu = true; - switch (sta->max_amsdu_len) { + switch (sta->deflink.agg.max_amsdu_len) { case IEEE80211_MAX_MPDU_LEN_VHT_11454: if (!is_mt7915(&dev->mt76)) { amsdu->max_mpdu_size = @@ -1304,7 +1304,7 @@ int mt7915_mcu_set_fixed_rate_ctrl(struct mt7915_dev *dev, ra->phy = *phy; break; case RATE_PARAM_MMPS_UPDATE: - ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->smps_mode); + ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->deflink.smps_mode); break; default: break; @@ -1360,7 +1360,7 @@ mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev, struct sta_phy phy = {}; int ret, nrates = 0; -#define __sta_phy_bitrate_mask_check(_mcs, _gi, _he) \ +#define __sta_phy_bitrate_mask_check(_mcs, _gi, _ht, _he) \ do { \ u8 i, gi = mask->control[band]._gi; \ gi = (_he) ? gi : gi == NL80211_TXRATE_FORCE_SGI; \ @@ -1373,15 +1373,17 @@ mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev, continue; \ nrates += hweight16(mask->control[band]._mcs[i]); \ phy.mcs = ffs(mask->control[band]._mcs[i]) - 1; \ + if (_ht) \ + phy.mcs += 8 * i; \ } \ } while (0) if (sta->deflink.he_cap.has_he) { - __sta_phy_bitrate_mask_check(he_mcs, he_gi, 1); + __sta_phy_bitrate_mask_check(he_mcs, he_gi, 0, 1); } else if (sta->deflink.vht_cap.vht_supported) { - __sta_phy_bitrate_mask_check(vht_mcs, gi, 0); + __sta_phy_bitrate_mask_check(vht_mcs, gi, 0, 0); } else if (sta->deflink.ht_cap.ht_supported) { - __sta_phy_bitrate_mask_check(ht_mcs, gi, 0); + __sta_phy_bitrate_mask_check(ht_mcs, gi, 1, 0); } else { nrates = hweight32(mask->control[band].legacy); phy.mcs = ffs(mask->control[band].legacy) - 1; @@ -1459,7 +1461,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev, ra->channel = chandef->chan->hw_value; ra->bw = sta->deflink.bandwidth; ra->phy.bw = sta->deflink.bandwidth; - ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->smps_mode); + ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->deflink.smps_mode); if (supp_rate) { supp_rate &= mask->control[band].legacy; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c index 4499a630e8f150508ec283e68586479870617aa8..7bd5f6725d7b7b6de9874b1c6f53acd08d4e8592 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c @@ -75,6 +75,7 @@ static const u32 mt7915_offs[] = { [AGG_AWSCR0] = 0x05c, [AGG_PCR0] = 0x06c, [AGG_ACR0] = 0x084, + [AGG_ACR4] = 0x08c, [AGG_MRCR] = 0x098, [AGG_ATCR1] = 0x0f0, [AGG_ATCR3] = 0x0f4, @@ -148,6 +149,7 @@ static const u32 mt7916_offs[] = { [AGG_AWSCR0] = 0x030, [AGG_PCR0] = 0x040, [AGG_ACR0] = 0x054, + [AGG_ACR4] = 0x05c, [AGG_MRCR] = 0x068, [AGG_ATCR1] = 0x1a8, [AGG_ATCR3] = 0x080, @@ -204,147 +206,147 @@ static const u32 mt7916_offs[] = { [ETBF_PAR_RPT0] = 0x100, }; -static const struct __map mt7915_reg_map[] = { +static const struct mt76_connac_reg_map mt7915_reg_map[] = { { 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */ { 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure regs) */ { 0x40000000, 0x70000, 0x10000 }, /* WF_UMAC_SYSRAM */ - { 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */ - { 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */ - { 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */ - { 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */ + { 0x54000000, 0x02000, 0x01000 }, /* WFDMA PCIE0 MCU DMA0 */ + { 0x55000000, 0x03000, 0x01000 }, /* WFDMA PCIE0 MCU DMA1 */ + { 0x58000000, 0x06000, 0x01000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */ + { 0x59000000, 0x07000, 0x01000 }, /* WFDMA PCIE1 MCU DMA1 */ { 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */ { 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */ { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */ { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */ - { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */ - { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */ - { 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */ - { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */ - { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */ + { 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */ + { 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */ + { 0x820cc000, 0x0e000, 0x02000 }, /* WF_UMAC_TOP (PP) */ + { 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */ + { 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */ { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */ - { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ - { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ - { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ - { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ - { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ - { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ - { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ - { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ - { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ - { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ - { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ - { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ - { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ - { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ - { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ - { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ - { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ - { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ - { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ - { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ - { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ - { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ - { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ - { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ + { 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ + { 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ + { 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ + { 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ + { 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ + { 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ + { 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ + { 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ + { 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ + { 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ + { 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ + { 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ + { 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ + { 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ + { 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ + { 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ + { 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ + { 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ + { 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ + { 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ + { 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ + { 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ + { 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ + { 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ { 0x0, 0x0, 0x0 }, /* imply end of search */ }; -static const struct __map mt7916_reg_map[] = { - { 0x54000000, 0x02000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ - { 0x55000000, 0x03000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ - { 0x56000000, 0x04000, 0x1000 }, /* WFDMA_2 (Reserved) */ - { 0x57000000, 0x05000, 0x1000 }, /* WFDMA_3 (MCU wrap CR) */ - { 0x58000000, 0x06000, 0x1000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */ - { 0x59000000, 0x07000, 0x1000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */ - { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */ - { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */ - { 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */ - { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ - { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ - { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ - { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ - { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ - { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ - { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */ - { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ - { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */ - { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ - { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ - { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ - { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ - { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ - { 0x820ca000, 0x26000, 0x2000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */ - { 0x820d0000, 0x30000, 0x10000}, /* WF_LMAC_TOP (WF_WTBLON) */ - { 0x00400000, 0x80000, 0x10000}, /* WF_MCU_SYSRAM */ - { 0x00410000, 0x90000, 0x10000}, /* WF_MCU_SYSRAM (configure cr) */ - { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ - { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ - { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ - { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ - { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ - { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ - { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ - { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ - { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ - { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ - { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ - { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ - { 0x820c4000, 0xa8000, 0x1000 }, /* WF_LMAC_TOP (WF_UWTBL ) */ - { 0x820b0000, 0xae000, 0x1000 }, /* [APB2] WFSYS_ON */ - { 0x80020000, 0xb0000, 0x10000}, /* WF_TOP_MISC_OFF */ - { 0x81020000, 0xc0000, 0x10000}, /* WF_TOP_MISC_ON */ +static const struct mt76_connac_reg_map mt7916_reg_map[] = { + { 0x54000000, 0x02000, 0x01000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ + { 0x55000000, 0x03000, 0x01000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ + { 0x56000000, 0x04000, 0x01000 }, /* WFDMA_2 (Reserved) */ + { 0x57000000, 0x05000, 0x01000 }, /* WFDMA_3 (MCU wrap CR) */ + { 0x58000000, 0x06000, 0x01000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */ + { 0x59000000, 0x07000, 0x01000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */ + { 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */ + { 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */ + { 0x820cc000, 0x0e000, 0x02000 }, /* WF_UMAC_TOP (PP) */ + { 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ + { 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ + { 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ + { 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ + { 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ + { 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ + { 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */ + { 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ + { 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */ + { 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ + { 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ + { 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ + { 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ + { 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ + { 0x820ca000, 0x26000, 0x02000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */ + { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */ + { 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */ + { 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure cr) */ + { 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ + { 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ + { 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ + { 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ + { 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ + { 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ + { 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ + { 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ + { 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ + { 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ + { 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ + { 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ + { 0x820c4000, 0xa8000, 0x01000 }, /* WF_LMAC_TOP (WF_UWTBL ) */ + { 0x820b0000, 0xae000, 0x01000 }, /* [APB2] WFSYS_ON */ + { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */ + { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */ { 0x0, 0x0, 0x0 }, /* imply end of search */ }; -static const struct __map mt7986_reg_map[] = { - { 0x54000000, 0x402000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ - { 0x55000000, 0x403000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ - { 0x56000000, 0x404000, 0x1000 }, /* WFDMA_2 (Reserved) */ - { 0x57000000, 0x405000, 0x1000 }, /* WFDMA_3 (MCU wrap CR) */ - { 0x58000000, 0x406000, 0x1000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */ - { 0x59000000, 0x407000, 0x1000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */ - { 0x820c0000, 0x408000, 0x4000 }, /* WF_UMAC_TOP (PLE) */ - { 0x820c8000, 0x40c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */ - { 0x820cc000, 0x40e000, 0x2000 }, /* WF_UMAC_TOP (PP) */ - { 0x820e0000, 0x420000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ - { 0x820e1000, 0x420400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ - { 0x820e2000, 0x420800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ - { 0x820e3000, 0x420c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ - { 0x820e4000, 0x421000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ - { 0x820e5000, 0x421400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ - { 0x820ce000, 0x421c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */ - { 0x820e7000, 0x421e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ - { 0x820cf000, 0x422000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */ - { 0x820e9000, 0x423400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ - { 0x820ea000, 0x424000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ - { 0x820eb000, 0x424200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ - { 0x820ec000, 0x424600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ - { 0x820ed000, 0x424800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ - { 0x820ca000, 0x426000, 0x2000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */ - { 0x820d0000, 0x430000, 0x10000}, /* WF_LMAC_TOP (WF_WTBLON) */ - { 0x00400000, 0x480000, 0x10000}, /* WF_MCU_SYSRAM */ - { 0x00410000, 0x490000, 0x10000}, /* WF_MCU_SYSRAM */ - { 0x820f0000, 0x4a0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ - { 0x820f1000, 0x4a0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ - { 0x820f2000, 0x4a0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ - { 0x820f3000, 0x4a0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ - { 0x820f4000, 0x4a1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ - { 0x820f5000, 0x4a1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ - { 0x820f7000, 0x4a1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ - { 0x820f9000, 0x4a3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ - { 0x820fa000, 0x4a4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ - { 0x820fb000, 0x4a4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ - { 0x820fc000, 0x4a4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ - { 0x820fd000, 0x4a4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ - { 0x820c4000, 0x4a8000, 0x1000 }, /* WF_LMAC_TOP (WF_UWTBL ) */ - { 0x820b0000, 0x4ae000, 0x1000 }, /* [APB2] WFSYS_ON */ - { 0x80020000, 0x4b0000, 0x10000}, /* WF_TOP_MISC_OFF */ - { 0x81020000, 0x4c0000, 0x10000}, /* WF_TOP_MISC_ON */ - { 0x89000000, 0x4d0000, 0x1000 }, /* WF_MCU_CFG_ON */ - { 0x89010000, 0x4d1000, 0x1000 }, /* WF_MCU_CIRQ */ - { 0x89020000, 0x4d2000, 0x1000 }, /* WF_MCU_GPT */ - { 0x89030000, 0x4d3000, 0x1000 }, /* WF_MCU_WDT */ - { 0x80010000, 0x4d4000, 0x1000 }, /* WF_AXIDMA */ +static const struct mt76_connac_reg_map mt7986_reg_map[] = { + { 0x54000000, 0x402000, 0x01000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */ + { 0x55000000, 0x403000, 0x01000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */ + { 0x56000000, 0x404000, 0x01000 }, /* WFDMA_2 (Reserved) */ + { 0x57000000, 0x405000, 0x01000 }, /* WFDMA_3 (MCU wrap CR) */ + { 0x58000000, 0x406000, 0x01000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */ + { 0x59000000, 0x407000, 0x01000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */ + { 0x820c0000, 0x408000, 0x04000 }, /* WF_UMAC_TOP (PLE) */ + { 0x820c8000, 0x40c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */ + { 0x820cc000, 0x40e000, 0x02000 }, /* WF_UMAC_TOP (PP) */ + { 0x820e0000, 0x420000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ + { 0x820e1000, 0x420400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ + { 0x820e2000, 0x420800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ + { 0x820e3000, 0x420c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ + { 0x820e4000, 0x421000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ + { 0x820e5000, 0x421400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ + { 0x820ce000, 0x421c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */ + { 0x820e7000, 0x421e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ + { 0x820cf000, 0x422000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */ + { 0x820e9000, 0x423400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ + { 0x820ea000, 0x424000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ + { 0x820eb000, 0x424200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ + { 0x820ec000, 0x424600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ + { 0x820ed000, 0x424800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ + { 0x820ca000, 0x426000, 0x02000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */ + { 0x820d0000, 0x430000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */ + { 0x00400000, 0x480000, 0x10000 }, /* WF_MCU_SYSRAM */ + { 0x00410000, 0x490000, 0x10000 }, /* WF_MCU_SYSRAM */ + { 0x820f0000, 0x4a0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ + { 0x820f1000, 0x4a0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ + { 0x820f2000, 0x4a0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ + { 0x820f3000, 0x4a0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ + { 0x820f4000, 0x4a1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ + { 0x820f5000, 0x4a1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ + { 0x820f7000, 0x4a1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ + { 0x820f9000, 0x4a3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ + { 0x820fa000, 0x4a4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ + { 0x820fb000, 0x4a4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ + { 0x820fc000, 0x4a4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ + { 0x820fd000, 0x4a4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ + { 0x820c4000, 0x4a8000, 0x01000 }, /* WF_LMAC_TOP (WF_UWTBL ) */ + { 0x820b0000, 0x4ae000, 0x01000 }, /* [APB2] WFSYS_ON */ + { 0x80020000, 0x4b0000, 0x10000 }, /* WF_TOP_MISC_OFF */ + { 0x81020000, 0x4c0000, 0x10000 }, /* WF_TOP_MISC_ON */ + { 0x89000000, 0x4d0000, 0x01000 }, /* WF_MCU_CFG_ON */ + { 0x89010000, 0x4d1000, 0x01000 }, /* WF_MCU_CIRQ */ + { 0x89020000, 0x4d2000, 0x01000 }, /* WF_MCU_GPT */ + { 0x89030000, 0x4d3000, 0x01000 }, /* WF_MCU_WDT */ + { 0x80010000, 0x4d4000, 0x01000 }, /* WF_AXIDMA */ { 0x0, 0x0, 0x0 }, /* imply end of search */ }; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index 54ef2a12a4437a52e6bb375542464e1146c4cd93..1eb11617a62553ab22ad82b1a76ddc7f56d482d8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -127,8 +127,6 @@ struct mt7915_sta { unsigned long jiffies; unsigned long ampdu_state; - struct mt76_sta_stats stats; - struct mt76_connac_sta_key_conf bip; struct { diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c index d74f609775d3bbb23eecb8d04b294bf5eb1d323e..728a879c3b008fd6e56500e56efeb054d7b40052 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c @@ -99,6 +99,7 @@ static int mt7915_pci_hif2_probe(struct pci_dev *pdev) static int mt7915_wed_offload_enable(struct mtk_wed_device *wed) { struct mt7915_dev *dev; + struct mt7915_phy *phy; int ret; dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); @@ -112,18 +113,38 @@ static int mt7915_wed_offload_enable(struct mtk_wed_device *wed) if (!ret) return -EAGAIN; + phy = &dev->phy; + mt76_set(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H); + + phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; + if (phy) + mt76_set(dev, MT_AGG_ACR4(phy->band_idx), + MT_AGG_ACR_PPDU_TXS2H); + return 0; } static void mt7915_wed_offload_disable(struct mtk_wed_device *wed) { struct mt7915_dev *dev; + struct mt7915_phy *phy; dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed); spin_lock_bh(&dev->mt76.token_lock); dev->mt76.token_size = MT7915_TOKEN_SIZE; spin_unlock_bh(&dev->mt76.token_lock); + + /* MT_TXD5_TX_STATUS_HOST (MPDU format) has higher priority than + * MT_AGG_ACR_PPDU_TXS2H (PPDU format) even though ACR bit is set. + */ + phy = &dev->phy; + mt76_clear(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H); + + phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL; + if (phy) + mt76_clear(dev, MT_AGG_ACR4(phy->band_idx), + MT_AGG_ACR_PPDU_TXS2H); } #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h index 2493c3ad3c567c465e36767b4fdada41660e04f1..5920e705835a7b4ba0b9d399adf31518ffdec5e1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h @@ -4,17 +4,11 @@ #ifndef __MT7915_REGS_H #define __MT7915_REGS_H -struct __map { - u32 phys; - u32 maps; - u32 size; -}; - /* used to differentiate between generations */ struct mt7915_reg_desc { const u32 *reg_rev; const u32 *offs_rev; - const struct __map *map; + const struct mt76_connac_reg_map *map; u32 map_size; }; @@ -52,6 +46,7 @@ enum offs_rev { AGG_AWSCR0, AGG_PCR0, AGG_ACR0, + AGG_ACR4, AGG_MRCR, AGG_ATCR1, AGG_ATCR3, @@ -471,6 +466,9 @@ enum offs_rev { #define MT_AGG_ACR_CFEND_RATE GENMASK(13, 0) #define MT_AGG_ACR_BAR_RATE GENMASK(29, 16) +#define MT_AGG_ACR4(_band) MT_WF_AGG(_band, __OFFS(AGG_ACR4)) +#define MT_AGG_ACR_PPDU_TXS2H BIT(1) + #define MT_AGG_MRCR(_band) MT_WF_AGG(_band, __OFFS(AGG_MRCR)) #define MT_AGG_MRCR_BAR_CNT_LIMIT GENMASK(15, 12) #define MT_AGG_MRCR_LAST_RTS_CTS_RN BIT(6) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c index be4f07ad3af91bf28f776bdddbddd1b9911d7ea7..47e034a9b0037a35641fb1b4d2a20c400c873a88 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c @@ -13,6 +13,7 @@ mt7921_acpi_read(struct mt7921_dev *dev, u8 *method, u8 **tbl, u32 *len) acpi_handle root, handle; acpi_status status; u32 i = 0; + int ret; root = ACPI_HANDLE(mdev->dev); if (!root) @@ -52,9 +53,11 @@ mt7921_acpi_read(struct mt7921_dev *dev, u8 *method, u8 **tbl, u32 *len) *(*tbl + i) = (u8)sar_unit->integer.value; } free: + ret = (i == sar_root->package.count) ? 0 : -EINVAL; + kfree(sar_root); - return (i == sar_root->package.count) ? 0 : -EINVAL; + return ret; } /* MTCL : Country List Table for 6G band */ diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h index 54f30401343c85c373639dd00793a692e14542b9..4b647278eb305fe3e528e5880c0badc0f1014073 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h @@ -11,12 +11,15 @@ enum mt7921_eeprom_field { MT_EE_VERSION = 0x002, MT_EE_MAC_ADDR = 0x004, MT_EE_WIFI_CONF = 0x07c, - __MT_EE_MAX = 0x3bf + MT_EE_HW_TYPE = 0x55b, + __MT_EE_MAX = 0x9ff }; #define MT_EE_WIFI_CONF_TX_MASK BIT(0) #define MT_EE_WIFI_CONF_BAND_SEL GENMASK(3, 2) +#define MT_EE_HW_TYPE_ENCAP BIT(0) + enum mt7921_eeprom_band { MT_EE_NA, MT_EE_5GHZ, diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c index cd960e23770ff711c7cb5590c82b0e8d2aea7053..dcdb3cf04ac1ba1908f9237a2bf503baaed14095 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c @@ -39,6 +39,7 @@ mt7921_regd_notifier(struct wiphy *wiphy, dev->mt76.region = request->dfs_region; mt7921_mutex_acquire(dev); + mt7921_mcu_set_clc(dev, request->alpha2, request->country_ie_env); mt76_connac_mcu_set_channel_domain(hw->priv); mt7921_set_tx_sar_pwr(hw, NULL); mt7921_mutex_release(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index 47f0aa81ab0280d323a84ec773e210ecde8f2b79..650ab97ae052436bbd43cd69d66217991c491453 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -230,12 +230,13 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) struct mt76_phy *mphy = &dev->mt76.phy; struct mt7921_phy *phy = &dev->phy; struct ieee80211_supported_band *sband; + u32 csum_status = *(u32 *)skb->cb; u32 rxd0 = le32_to_cpu(rxd[0]); u32 rxd1 = le32_to_cpu(rxd[1]); u32 rxd2 = le32_to_cpu(rxd[2]); u32 rxd3 = le32_to_cpu(rxd[3]); u32 rxd4 = le32_to_cpu(rxd[4]); - struct mt7921_sta *msta; + struct mt7921_sta *msta = NULL; u16 seq_ctrl = 0; __le16 fc = 0; u8 mode = 0; @@ -290,7 +291,8 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) if (!sband->channels) return -EINVAL; - if ((rxd0 & csum_mask) == csum_mask) + if (mt76_is_mmio(&dev->mt76) && (rxd0 & csum_mask) == csum_mask && + !(csum_status & (BIT(0) | BIT(2) | BIT(3)))) skb->ip_summed = CHECKSUM_UNNECESSARY; if (rxd1 & MT_RXD1_NORMAL_FCS_ERR) @@ -486,7 +488,7 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb) return 0; } -void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi) +static void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi) { struct mt7921_sta *msta; u16 fc, tid; @@ -509,7 +511,6 @@ void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi) if (!test_and_set_bit(tid, &msta->ampdu_state)) ieee80211_start_tx_ba_session(sta, tid, 0); } -EXPORT_SYMBOL_GPL(mt7921_tx_check_aggr); void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data) { @@ -539,8 +540,7 @@ void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data) msta = container_of(wcid, struct mt7921_sta, wcid); - mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data, - &msta->stats); + mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data); if (!wcid->sta) goto out; @@ -552,7 +552,134 @@ void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data) out: rcu_read_unlock(); } -EXPORT_SYMBOL_GPL(mt7921_mac_add_txs); + +void mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t, + struct ieee80211_sta *sta, bool clear_status, + struct list_head *free_list) +{ + struct mt76_dev *mdev = &dev->mt76; + __le32 *txwi; + u16 wcid_idx; + + mt76_connac_txp_skb_unmap(mdev, t); + if (!t->skb) + goto out; + + txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t); + if (sta) { + struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; + + if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE))) + mt7921_tx_check_aggr(sta, txwi); + + wcid_idx = wcid->idx; + } else { + wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX); + } + + __mt76_tx_complete_skb(mdev, wcid_idx, t->skb, free_list); +out: + t->skb = NULL; + mt76_put_txwi(mdev, t); +} +EXPORT_SYMBOL_GPL(mt7921_txwi_free); + +static void mt7921_mac_tx_free(struct mt7921_dev *dev, void *data, int len) +{ + struct mt76_connac_tx_free *free = data; + __le32 *tx_info = (__le32 *)(data + sizeof(*free)); + struct mt76_dev *mdev = &dev->mt76; + struct mt76_txwi_cache *txwi; + struct ieee80211_sta *sta = NULL; + struct sk_buff *skb, *tmp; + void *end = data + len; + LIST_HEAD(free_list); + bool wake = false; + u8 i, count; + + /* clean DMA queues and unmap buffers first */ + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false); + mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_BE], false); + + count = le16_get_bits(free->ctrl, MT_TX_FREE_MSDU_CNT); + if (WARN_ON_ONCE((void *)&tx_info[count] > end)) + return; + + for (i = 0; i < count; i++) { + u32 msdu, info = le32_to_cpu(tx_info[i]); + u8 stat; + + /* 1'b1: new wcid pair. + * 1'b0: msdu_id with the same 'wcid pair' as above. + */ + if (info & MT_TX_FREE_PAIR) { + struct mt7921_sta *msta; + struct mt76_wcid *wcid; + u16 idx; + + count++; + idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info); + wcid = rcu_dereference(dev->mt76.wcid[idx]); + sta = wcid_to_sta(wcid); + if (!sta) + continue; + + msta = container_of(wcid, struct mt7921_sta, wcid); + spin_lock_bh(&dev->sta_poll_lock); + if (list_empty(&msta->poll_list)) + list_add_tail(&msta->poll_list, &dev->sta_poll_list); + spin_unlock_bh(&dev->sta_poll_lock); + continue; + } + + msdu = FIELD_GET(MT_TX_FREE_MSDU_ID, info); + stat = FIELD_GET(MT_TX_FREE_STATUS, info); + + txwi = mt76_token_release(mdev, msdu, &wake); + if (!txwi) + continue; + + mt7921_txwi_free(dev, txwi, sta, stat, &free_list); + } + + if (wake) + mt76_set_tx_blocked(&dev->mt76, false); + + list_for_each_entry_safe(skb, tmp, &free_list, list) { + skb_list_del_init(skb); + napi_consume_skb(skb, 1); + } + + rcu_read_lock(); + mt7921_mac_sta_poll(dev); + rcu_read_unlock(); + + mt76_worker_schedule(&dev->mt76.tx_worker); +} + +bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len) +{ + struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); + __le32 *rxd = (__le32 *)data; + __le32 *end = (__le32 *)&rxd[len / 4]; + enum rx_pkt_type type; + + type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE); + + switch (type) { + case PKT_TYPE_TXRX_NOTIFY: + /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ + mt7921_mac_tx_free(dev, data, len); /* mmio */ + return false; + case PKT_TYPE_TXS: + for (rxd += 2; rxd + 8 <= end; rxd += 8) + mt7921_mac_add_txs(dev, rxd); + return false; + default: + return true; + } +} +EXPORT_SYMBOL_GPL(mt7921_rx_check); void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, struct sk_buff *skb) @@ -570,6 +697,11 @@ void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, type = PKT_TYPE_NORMAL_MCU; switch (type) { + case PKT_TYPE_TXRX_NOTIFY: + /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ + mt7921_mac_tx_free(dev, skb->data, skb->len); + napi_consume_skb(skb, 1); + break; case PKT_TYPE_RX_EVENT: mt7921_mcu_rx_event(dev, skb); break; @@ -780,6 +912,7 @@ void mt7921_mac_reset_work(struct work_struct *work) void mt7921_reset(struct mt76_dev *mdev) { struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); + struct mt76_connac_pm *pm = &dev->pm; if (!dev->hw_init_done) return; @@ -787,8 +920,12 @@ void mt7921_reset(struct mt76_dev *mdev) if (dev->hw_full_reset) return; + if (pm->suspended) + return; + queue_work(dev->mt76.wq, &dev->reset_work); } +EXPORT_SYMBOL_GPL(mt7921_reset); void mt7921_mac_update_mib_stats(struct mt7921_phy *phy) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 1438a9f8d1fd97ec78c4328eb701dee87ff6ed90..7e409ac7d9a820f71090c78d49d9c4f4f8cda6cb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -752,6 +752,7 @@ void mt7921_mac_sta_assoc(struct mt76_dev *mdev, struct ieee80211_vif *vif, mt7921_mac_wtbl_update(dev, msta->wcid.idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); + memset(msta->airtime_ac, 0, sizeof(msta->airtime_ac)); mt7921_mcu_sta_update(dev, sta, vif, true, MT76_STA_INFO_STATE_ASSOC); @@ -1045,7 +1046,7 @@ mt7921_ethtool_worker(void *wi_data, struct ieee80211_sta *sta) if (msta->vif->mt76.idx != wi->idx) return; - mt76_ethtool_worker(wi, &msta->stats); + mt76_ethtool_worker(wi, &msta->wcid.stats); } static @@ -1404,6 +1405,8 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw, struct mt7921_sta *msta = (struct mt7921_sta *)sta->drv_priv; struct mt7921_dev *dev = mt7921_hw_dev(hw); + mt7921_mutex_acquire(dev); + if (enabled) set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags); else @@ -1411,6 +1414,8 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw, mt76_connac_mcu_sta_update_hdr_trans(&dev->mt76, vif, &msta->wcid, MCU_UNI_CMD(STA_REC_UPDATE)); + + mt7921_mutex_release(dev); } #if IS_ENABLED(CONFIG_IPV6) @@ -1526,17 +1531,23 @@ mt7921_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct mt7921_dev *dev = mt7921_hw_dev(hw); int err; + mt7921_mutex_acquire(dev); + err = mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, true); if (err) - return err; + goto out; err = mt7921_mcu_set_bss_pm(dev, vif, true); if (err) - return err; + goto out; + + err = mt7921_mcu_sta_update(dev, NULL, vif, true, + MT76_STA_INFO_STATE_NONE); +out: + mt7921_mutex_release(dev); - return mt7921_mcu_sta_update(dev, NULL, vif, true, - MT76_STA_INFO_STATE_NONE); + return err; } static void @@ -1548,11 +1559,16 @@ mt7921_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct mt7921_dev *dev = mt7921_hw_dev(hw); int err; + mt7921_mutex_acquire(dev); + err = mt7921_mcu_set_bss_pm(dev, vif, false); if (err) - return; + goto out; mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, false); + +out: + mt7921_mutex_release(dev); } const struct ieee80211_ops mt7921_ops = { diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index da12d0ae083542a2c3801dc4b69bbeaade78dc87..67bf92969a7b788df2e84bf500509edbd8cecba6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -2,14 +2,20 @@ /* Copyright (C) 2020 MediaTek Inc. */ #include +#include #include "mt7921.h" #include "mt7921_trace.h" +#include "eeprom.h" #include "mcu.h" #include "mac.h" #define MT_STA_BFER BIT(0) #define MT_STA_BFEE BIT(1) +static bool mt7921_disable_clc; +module_param_named(disable_clc, mt7921_disable_clc, bool, 0644); +MODULE_PARM_DESC(disable_clc, "disable CLC support"); + static int mt7921_mcu_parse_eeprom(struct mt76_dev *dev, struct sk_buff *skb) { @@ -84,6 +90,27 @@ int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd, } EXPORT_SYMBOL_GPL(mt7921_mcu_parse_response); +static int mt7921_mcu_read_eeprom(struct mt7921_dev *dev, u32 offset, u8 *val) +{ + struct mt7921_mcu_eeprom_info *res, req = { + .addr = cpu_to_le32(round_down(offset, + MT7921_EEPROM_BLOCK_SIZE)), + }; + struct sk_buff *skb; + int ret; + + ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_EXT_QUERY(EFUSE_ACCESS), + &req, sizeof(req), true, &skb); + if (ret) + return ret; + + res = (struct mt7921_mcu_eeprom_info *)skb->data; + *val = res->data[offset % MT7921_EEPROM_BLOCK_SIZE]; + dev_kfree_skb(skb); + + return 0; +} + #ifdef CONFIG_PM static int @@ -354,6 +381,90 @@ static char *mt7921_ram_name(struct mt7921_dev *dev) return ret; } +static int mt7921_load_clc(struct mt7921_dev *dev, const char *fw_name) +{ + const struct mt76_connac2_fw_trailer *hdr; + const struct mt76_connac2_fw_region *region; + const struct mt7921_clc *clc; + struct mt76_dev *mdev = &dev->mt76; + struct mt7921_phy *phy = &dev->phy; + const struct firmware *fw; + int ret, i, len, offset = 0; + u8 *clc_base = NULL, hw_encap = 0; + + if (mt7921_disable_clc || + mt76_is_usb(&dev->mt76)) + return 0; + + if (mt76_is_mmio(&dev->mt76)) { + ret = mt7921_mcu_read_eeprom(dev, MT_EE_HW_TYPE, &hw_encap); + if (ret) + return ret; + hw_encap = u8_get_bits(hw_encap, MT_EE_HW_TYPE_ENCAP); + } + + ret = request_firmware(&fw, fw_name, mdev->dev); + if (ret) + return ret; + + if (!fw || !fw->data || fw->size < sizeof(*hdr)) { + dev_err(mdev->dev, "Invalid firmware\n"); + ret = -EINVAL; + goto out; + } + + hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); + for (i = 0; i < hdr->n_region; i++) { + region = (const void *)((const u8 *)hdr - + (hdr->n_region - i) * sizeof(*region)); + len = le32_to_cpu(region->len); + + /* check if we have valid buffer size */ + if (offset + len > fw->size) { + dev_err(mdev->dev, "Invalid firmware region\n"); + ret = -EINVAL; + goto out; + } + + if ((region->feature_set & FW_FEATURE_NON_DL) && + region->type == FW_TYPE_CLC) { + clc_base = (u8 *)(fw->data + offset); + break; + } + offset += len; + } + + if (!clc_base) + goto out; + + for (offset = 0; offset < len; offset += le32_to_cpu(clc->len)) { + clc = (const struct mt7921_clc *)(clc_base + offset); + + /* do not init buf again if chip reset triggered */ + if (phy->clc[clc->idx]) + continue; + + /* header content sanity */ + if (clc->idx == MT7921_CLC_POWER && + u8_get_bits(clc->type, MT_EE_HW_TYPE_ENCAP) != hw_encap) + continue; + + phy->clc[clc->idx] = devm_kmemdup(mdev->dev, clc, + le32_to_cpu(clc->len), + GFP_KERNEL); + + if (!phy->clc[clc->idx]) { + ret = -ENOMEM; + goto out; + } + } + ret = mt7921_mcu_set_clc(dev, "00", ENVIRON_INDOOR); +out: + release_firmware(fw); + + return ret; +} + static int mt7921_load_firmware(struct mt7921_dev *dev) { int ret; @@ -423,6 +534,10 @@ int mt7921_run_firmware(struct mt7921_dev *dev) return err; set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state); + err = mt7921_load_clc(dev, mt7921_ram_name(dev)); + if (err) + return err; + return mt7921_mcu_fw_log_2_host(dev, 1); } EXPORT_SYMBOL_GPL(mt7921_run_firmware); @@ -930,3 +1045,86 @@ mt7921_mcu_uni_add_beacon_offload(struct mt7921_dev *dev, return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(BSS_INFO_UPDATE), &req, sizeof(req), true); } + +static +int __mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2, + enum environment_cap env_cap, + struct mt7921_clc *clc, + u8 idx) +{ + struct sk_buff *skb; + struct { + u8 ver; + u8 pad0; + __le16 len; + u8 idx; + u8 env; + u8 pad1[2]; + u8 alpha2[2]; + u8 type[2]; + u8 rsvd[64]; + } __packed req = { + .idx = idx, + .env = env_cap, + }; + int ret, valid_cnt = 0; + u8 i, *pos; + + if (!clc) + return 0; + + pos = clc->data; + for (i = 0; i < clc->nr_country; i++) { + struct mt7921_clc_rule *rule = (struct mt7921_clc_rule *)pos; + u16 len = le16_to_cpu(rule->len); + + pos += len + sizeof(*rule); + if (rule->alpha2[0] != alpha2[0] || + rule->alpha2[1] != alpha2[1]) + continue; + + memcpy(req.alpha2, rule->alpha2, 2); + memcpy(req.type, rule->type, 2); + + req.len = cpu_to_le16(sizeof(req) + len); + skb = __mt76_mcu_msg_alloc(&dev->mt76, &req, + le16_to_cpu(req.len), + sizeof(req), GFP_KERNEL); + if (!skb) + return -ENOMEM; + skb_put_data(skb, rule->data, len); + + ret = mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_CE_CMD(SET_CLC), false); + if (ret < 0) + return ret; + valid_cnt++; + } + + if (!valid_cnt) + return -ENOENT; + + return 0; +} + +int mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2, + enum environment_cap env_cap) +{ + struct mt7921_phy *phy = (struct mt7921_phy *)&dev->phy; + int i, ret; + + /* submit all clc config */ + for (i = 0; i < ARRAY_SIZE(phy->clc); i++) { + ret = __mt7921_mcu_set_clc(dev, alpha2, env_cap, + phy->clc[i], i); + + /* If no country found, set "00" as default */ + if (ret == -ENOENT) + ret = __mt7921_mcu_set_clc(dev, "00", + ENVIRON_INDOOR, + phy->clc[i], i); + if (ret < 0) + return ret; + } + return 0; +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h index 0d20f7d8d4746e5d245d67c1e766fc830acafaff..96dc870fd35e84a25145d240958b5bb2321f36aa 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h @@ -41,7 +41,7 @@ enum { struct mt7921_mcu_eeprom_info { __le32 addr; __le32 valid; - u8 data[16]; + u8 data[MT7921_EEPROM_BLOCK_SIZE]; } __packed; #define MT_RA_RATE_NSS GENMASK(8, 6) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h index c161031ac62aac89887aa5930b87d7536a10d3fb..eaba114a9c7e40e266beeaa014856a80f79147ea 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h @@ -41,6 +41,8 @@ #define MT7921_EEPROM_SIZE 3584 #define MT7921_TOKEN_SIZE 8192 +#define MT7921_EEPROM_BLOCK_SIZE 16 + #define MT7921_CFEND_RATE_DEFAULT 0x49 /* OFDM 24M */ #define MT7921_CFEND_RATE_11B 0x03 /* 11B LP, 11M */ @@ -100,7 +102,6 @@ struct mt7921_sta { unsigned long last_txs; unsigned long ampdu_state; - struct mt76_sta_stats stats; struct mt76_connac_sta_key_conf bip; }; @@ -149,6 +150,29 @@ struct mib_stats { u32 tx_amsdu_cnt; }; +enum { + MT7921_CLC_POWER, + MT7921_CLC_CHAN, + MT7921_CLC_MAX_NUM, +}; + +struct mt7921_clc_rule { + u8 alpha2[2]; + u8 type[2]; + __le16 len; + u8 data[]; +} __packed; + +struct mt7921_clc { + __le32 len; + u8 idx; + u8 ver; + u8 nr_country; + u8 type; + u8 rsv[8]; + u8 data[]; +}; + struct mt7921_phy { struct mt76_phy *mt76; struct mt7921_dev *dev; @@ -174,6 +198,8 @@ struct mt7921_phy { #ifdef CONFIG_ACPI struct mt7921_acpi_sar *acpisar; #endif + + struct mt7921_clc *clc[MT7921_CLC_MAX_NUM]; }; #define mt7921_init_reset(dev) ((dev)->hif_ops->init_reset(dev)) @@ -380,6 +406,7 @@ int mt7921e_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, void mt7921_tx_worker(struct mt76_worker *w); void mt7921_tx_token_put(struct mt7921_dev *dev); +bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len); void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, struct sk_buff *skb); void mt7921_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps); @@ -410,14 +437,13 @@ int mt7921_testmode_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif, void *data, int len); int mt7921_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *msg, struct netlink_callback *cb, void *data, int len); -void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi); +void mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t, + struct ieee80211_sta *sta, bool clear_status, + struct list_head *free_list); void mt7921_mac_sta_poll(struct mt7921_dev *dev); int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd, struct sk_buff *skb, int seq); -bool mt7921e_rx_check(struct mt76_dev *mdev, void *data, int len); -void mt7921e_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb); int mt7921e_driver_own(struct mt7921_dev *dev); int mt7921e_mac_reset(struct mt7921_dev *dev); int mt7921e_mcu_init(struct mt7921_dev *dev); @@ -479,4 +505,7 @@ mt7921_init_acpi_sar_power(struct mt7921_phy *phy, bool set_default) #endif int mt7921_set_tx_sar_pwr(struct ieee80211_hw *hw, const struct cfg80211_sar_specs *sar); + +int mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2, + enum environment_cap env_cap); #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index ea3069d18c35f3c4efe062004414e5aa2f57dc39..8a53d8f286dbefd72f247916f6872390c39b583a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -123,54 +123,51 @@ static void mt7921e_unregister_device(struct mt7921_dev *dev) static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr) { - static const struct { - u32 phys; - u32 mapped; - u32 size; - } fixed_map[] = { + static const struct mt76_connac_reg_map fixed_map[] = { { 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */ - { 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ - { 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ - { 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ - { 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ - { 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ - { 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ - { 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ + { 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */ + { 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */ + { 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */ + { 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */ + { 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */ + { 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */ + { 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */ { 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */ { 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure register) */ { 0x40000000, 0x70000, 0x10000 }, /* WF_UMAC_SYSRAM */ - { 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */ - { 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */ - { 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */ - { 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */ + { 0x54000000, 0x02000, 0x01000 }, /* WFDMA PCIE0 MCU DMA0 */ + { 0x55000000, 0x03000, 0x01000 }, /* WFDMA PCIE0 MCU DMA1 */ + { 0x58000000, 0x06000, 0x01000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */ + { 0x59000000, 0x07000, 0x01000 }, /* WFDMA PCIE1 MCU DMA1 */ { 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */ { 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */ { 0x7c060000, 0xe0000, 0x10000 }, /* CONN_INFRA, conn_host_csr_top */ { 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */ { 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */ - { 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */ - { 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */ - { 0x820cc000, 0x0e000, 0x1000 }, /* WF_UMAC_TOP (PP) */ - { 0x820cd000, 0x0f000, 0x1000 }, /* WF_MDP_TOP */ - { 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */ - { 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */ - { 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ - { 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ - { 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ - { 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ - { 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ - { 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ - { 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ - { 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ - { 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ - { 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ - { 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ - { 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ - { 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ - { 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ - { 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ - { 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ - { 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ + { 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */ + { 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */ + { 0x820cc000, 0x0e000, 0x01000 }, /* WF_UMAC_TOP (PP) */ + { 0x820cd000, 0x0f000, 0x01000 }, /* WF_MDP_TOP */ + { 0x74030000, 0x10000, 0x10000 }, /* PCIE_MAC_IREG */ + { 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */ + { 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */ + { 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */ + { 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */ + { 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */ + { 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */ + { 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */ + { 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */ + { 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */ + { 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */ + { 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */ + { 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */ + { 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */ + { 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */ + { 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */ + { 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */ + { 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */ + { 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */ + { 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */ }; int i; @@ -187,7 +184,7 @@ static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr) if (ofs > fixed_map[i].size) continue; - return fixed_map[i].mapped + ofs; + return fixed_map[i].maps + ofs; } if ((addr >= 0x18000000 && addr < 0x18c00000) || @@ -238,8 +235,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev, .token_size = MT7921_TOKEN_SIZE, .tx_prepare_skb = mt7921e_tx_prepare_skb, .tx_complete_skb = mt76_connac_tx_complete_skb, - .rx_check = mt7921e_rx_check, - .rx_skb = mt7921e_queue_rx_skb, + .rx_check = mt7921_rx_check, + .rx_skb = mt7921_queue_rx_skb, .rx_poll_complete = mt7921_rx_poll_complete, .sta_ps = mt7921_sta_ps, .sta_add = mt7921_mac_sta_add, @@ -288,6 +285,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev, goto err_free_pci_vec; } + pci_set_drvdata(pdev, mdev); + dev = container_of(mdev, struct mt7921_dev, mt76); dev->hif_ops = &mt7921_pcie_ops; @@ -367,6 +366,7 @@ static int mt7921_pci_suspend(struct device *device) int i, err; pm->suspended = true; + flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); cancel_work_sync(&pm->wake_work); @@ -409,9 +409,6 @@ static int mt7921_pci_suspend(struct device *device) if (err) goto restore_napi; - if (err) - goto restore_napi; - return 0; restore_napi: @@ -428,6 +425,9 @@ restore_napi: restore_suspend: pm->suspended = false; + if (err < 0) + mt7921_reset(&dev->mt76); + return err; } @@ -441,7 +441,7 @@ static int mt7921_pci_resume(struct device *device) err = mt7921_mcu_drv_pmctrl(dev); if (err < 0) - return err; + goto failed; mt7921_wpdma_reinit_cond(dev); @@ -471,11 +471,12 @@ static int mt7921_pci_resume(struct device *device) mt76_connac_mcu_set_deep_sleep(&dev->mt76, false); err = mt76_connac_mcu_set_hif_suspend(mdev, false); - if (err) - return err; - +failed: pm->suspended = false; + if (err < 0) + mt7921_reset(&dev->mt76); + return err; } diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c index e1800674089ac9ddb388aa281f4209dfc6da7f7f..8dd60408b117e298182026545c1ade2d6214316f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c @@ -53,154 +53,6 @@ int mt7921e_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, return 0; } -static void -mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t, - struct ieee80211_sta *sta, bool clear_status, - struct list_head *free_list) -{ - struct mt76_dev *mdev = &dev->mt76; - __le32 *txwi; - u16 wcid_idx; - - mt76_connac_txp_skb_unmap(mdev, t); - if (!t->skb) - goto out; - - txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t); - if (sta) { - struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; - - if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE))) - mt7921_tx_check_aggr(sta, txwi); - - wcid_idx = wcid->idx; - } else { - wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX); - } - - __mt76_tx_complete_skb(mdev, wcid_idx, t->skb, free_list); - -out: - t->skb = NULL; - mt76_put_txwi(mdev, t); -} - -static void -mt7921e_mac_tx_free(struct mt7921_dev *dev, void *data, int len) -{ - struct mt76_connac_tx_free *free = data; - __le32 *tx_info = (__le32 *)(data + sizeof(*free)); - struct mt76_dev *mdev = &dev->mt76; - struct mt76_txwi_cache *txwi; - struct ieee80211_sta *sta = NULL; - struct sk_buff *skb, *tmp; - void *end = data + len; - LIST_HEAD(free_list); - bool wake = false; - u8 i, count; - - /* clean DMA queues and unmap buffers first */ - mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false); - mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_BE], false); - - count = le16_get_bits(free->ctrl, MT_TX_FREE_MSDU_CNT); - if (WARN_ON_ONCE((void *)&tx_info[count] > end)) - return; - - for (i = 0; i < count; i++) { - u32 msdu, info = le32_to_cpu(tx_info[i]); - u8 stat; - - /* 1'b1: new wcid pair. - * 1'b0: msdu_id with the same 'wcid pair' as above. - */ - if (info & MT_TX_FREE_PAIR) { - struct mt7921_sta *msta; - struct mt76_wcid *wcid; - u16 idx; - - count++; - idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info); - wcid = rcu_dereference(dev->mt76.wcid[idx]); - sta = wcid_to_sta(wcid); - if (!sta) - continue; - - msta = container_of(wcid, struct mt7921_sta, wcid); - spin_lock_bh(&dev->sta_poll_lock); - if (list_empty(&msta->poll_list)) - list_add_tail(&msta->poll_list, &dev->sta_poll_list); - spin_unlock_bh(&dev->sta_poll_lock); - continue; - } - - msdu = FIELD_GET(MT_TX_FREE_MSDU_ID, info); - stat = FIELD_GET(MT_TX_FREE_STATUS, info); - - txwi = mt76_token_release(mdev, msdu, &wake); - if (!txwi) - continue; - - mt7921_txwi_free(dev, txwi, sta, stat, &free_list); - } - - if (wake) - mt76_set_tx_blocked(&dev->mt76, false); - - list_for_each_entry_safe(skb, tmp, &free_list, list) { - skb_list_del_init(skb); - napi_consume_skb(skb, 1); - } - - rcu_read_lock(); - mt7921_mac_sta_poll(dev); - rcu_read_unlock(); - - mt76_worker_schedule(&dev->mt76.tx_worker); -} - -bool mt7921e_rx_check(struct mt76_dev *mdev, void *data, int len) -{ - struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); - __le32 *rxd = (__le32 *)data; - __le32 *end = (__le32 *)&rxd[len / 4]; - enum rx_pkt_type type; - - type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE); - - switch (type) { - case PKT_TYPE_TXRX_NOTIFY: - mt7921e_mac_tx_free(dev, data, len); - return false; - case PKT_TYPE_TXS: - for (rxd += 2; rxd + 8 <= end; rxd += 8) - mt7921_mac_add_txs(dev, rxd); - return false; - default: - return true; - } -} - -void mt7921e_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, - struct sk_buff *skb) -{ - struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76); - __le32 *rxd = (__le32 *)skb->data; - enum rx_pkt_type type; - - type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE); - - switch (type) { - case PKT_TYPE_TXRX_NOTIFY: - mt7921e_mac_tx_free(dev, skb->data, skb->len); - napi_consume_skb(skb, 1); - break; - default: - mt7921_queue_rx_skb(mdev, q, skb); - break; - } -} - void mt7921_tx_token_put(struct mt7921_dev *dev) { struct mt76_txwi_cache *txwi; @@ -261,7 +113,7 @@ int mt7921e_mac_reset(struct mt7921_dev *dev) err = mt7921e_driver_own(dev); if (err) - return err; + goto out; err = mt7921_run_firmware(dev); if (err) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c index 5efda694fb9d56c338d6da47e922d3bb1199295c..86340d3205c55a01b6dce9264b2414c53d3889af 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c @@ -30,12 +30,7 @@ mt7921_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, if (ret) return ret; - if (cmd == MCU_UNI_CMD(HIF_CTRL) || - cmd == MCU_UNI_CMD(SUSPEND) || - cmd == MCU_UNI_CMD(OFFLOAD)) - mdev->mcu.timeout = HZ; - else - mdev->mcu.timeout = 3 * HZ; + mdev->mcu.timeout = 3 * HZ; if (cmd == MCU_CMD(FW_SCATTER)) txq = MT_MCUQ_FWDL; @@ -59,6 +54,8 @@ int mt7921e_mcu_init(struct mt7921_dev *dev) if (err) return err; + mt76_rmw_field(dev, MT_PCIE_MAC_PM, MT_PCIE_MAC_PM_L0S_DIS, 1); + err = mt7921_run_firmware(dev); mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_FWDL], false); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h index ea643260ceb6666209fec4cb9f4a0b178999d43f..c65582acfa55de50231af74e4540b980af4ac1fe 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h @@ -440,6 +440,8 @@ #define MT_PCIE_MAC_BASE 0x10000 #define MT_PCIE_MAC(ofs) (MT_PCIE_MAC_BASE + (ofs)) #define MT_PCIE_MAC_INT_ENABLE MT_PCIE_MAC(0x188) +#define MT_PCIE_MAC_PM MT_PCIE_MAC(0x194) +#define MT_PCIE_MAC_PM_L0S_DIS BIT(8) #define MT_DMA_SHDL(ofs) (0x7c026000 + (ofs)) #define MT_DMASHDL_SW_CONTROL MT_DMA_SHDL(0x004) diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c index 487acd6e2be8fd031e6d697b178b73f17deba94e..3b25a06fd9466b06f318fbf82b7763dcd89b2904 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c @@ -96,6 +96,7 @@ static int mt7921s_probe(struct sdio_func *func, .tx_complete_skb = mt7921_usb_sdio_tx_complete_skb, .tx_status_data = mt7921_usb_sdio_tx_status_data, .rx_skb = mt7921_queue_rx_skb, + .rx_check = mt7921_rx_check, .sta_ps = mt7921_sta_ps, .sta_add = mt7921_mac_sta_add, .sta_assoc = mt7921_mac_sta_assoc, @@ -194,7 +195,6 @@ static void mt7921s_remove(struct sdio_func *func) mt7921s_unregister_device(dev); } -#ifdef CONFIG_PM static int mt7921s_suspend(struct device *__dev) { struct sdio_func *func = dev_to_sdio_func(__dev); @@ -206,6 +206,7 @@ static int mt7921s_suspend(struct device *__dev) pm->suspended = true; set_bit(MT76_STATE_SUSPEND, &mdev->phy.state); + flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); cancel_work_sync(&pm->wake_work); @@ -261,6 +262,9 @@ restore_suspend: clear_bit(MT76_STATE_SUSPEND, &mdev->phy.state); pm->suspended = false; + if (err < 0) + mt7921_reset(&dev->mt76); + return err; } @@ -276,7 +280,7 @@ static int mt7921s_resume(struct device *__dev) err = mt7921_mcu_drv_pmctrl(dev); if (err < 0) - return err; + goto failed; mt76_worker_enable(&mdev->tx_worker); mt76_worker_enable(&mdev->sdio.txrx_worker); @@ -288,34 +292,27 @@ static int mt7921s_resume(struct device *__dev) mt76_connac_mcu_set_deep_sleep(mdev, false); err = mt76_connac_mcu_set_hif_suspend(mdev, false); - if (err) - return err; - +failed: pm->suspended = false; + if (err < 0) + mt7921_reset(&dev->mt76); + return err; } -static const struct dev_pm_ops mt7921s_pm_ops = { - .suspend = mt7921s_suspend, - .resume = mt7921s_resume, -}; -#endif - MODULE_DEVICE_TABLE(sdio, mt7921s_table); MODULE_FIRMWARE(MT7921_FIRMWARE_WM); MODULE_FIRMWARE(MT7921_ROM_PATCH); +static DEFINE_SIMPLE_DEV_PM_OPS(mt7921s_pm_ops, mt7921s_suspend, mt7921s_resume); + static struct sdio_driver mt7921s_driver = { .name = KBUILD_MODNAME, .probe = mt7921s_probe, .remove = mt7921s_remove, .id_table = mt7921s_table, -#ifdef CONFIG_PM - .drv = { - .pm = &mt7921s_pm_ops, - } -#endif + .drv.pm = pm_sleep_ptr(&mt7921s_pm_ops), }; module_sdio_driver(mt7921s_driver); MODULE_AUTHOR("Sean Wang "); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c index e038d7404323fb851ee9cc402b19ebe421ad920b..5c1489766d9fd72a3e3e9742c52fb43be6bef886 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c @@ -33,12 +33,7 @@ mt7921s_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, if (ret) return ret; - if (cmd == MCU_UNI_CMD(HIF_CTRL) || - cmd == MCU_UNI_CMD(SUSPEND) || - cmd == MCU_UNI_CMD(OFFLOAD)) - mdev->mcu.timeout = HZ; - else - mdev->mcu.timeout = 3 * HZ; + mdev->mcu.timeout = 3 * HZ; if (cmd == MCU_CMD(FW_SCATTER)) type = MT7921_SDIO_FWDL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c index dd3b8884e162037834bf2da2af1d7c8c9490cc95..29c0ee330dbeda6333a976ef1f6653eee9b9ea45 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c @@ -106,12 +106,7 @@ mt7921u_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb, if (ret) return ret; - if (cmd == MCU_UNI_CMD(HIF_CTRL) || - cmd == MCU_UNI_CMD(SUSPEND) || - cmd == MCU_UNI_CMD(OFFLOAD)) - mdev->mcu.timeout = HZ; - else - mdev->mcu.timeout = 3 * HZ; + mdev->mcu.timeout = 3 * HZ; if (cmd != MCU_CMD(FW_SCATTER)) ep = MT_EP_OUT_INBAND_CMD; @@ -183,6 +178,7 @@ static int mt7921u_probe(struct usb_interface *usb_intf, .tx_complete_skb = mt7921_usb_sdio_tx_complete_skb, .tx_status_data = mt7921_usb_sdio_tx_status_data, .rx_skb = mt7921_queue_rx_skb, + .rx_check = mt7921_rx_check, .sta_ps = mt7921_sta_ps, .sta_add = mt7921_mac_sta_add, .sta_assoc = mt7921_mac_sta_assoc, @@ -300,23 +296,34 @@ static void mt7921u_disconnect(struct usb_interface *usb_intf) static int mt7921u_suspend(struct usb_interface *intf, pm_message_t state) { struct mt7921_dev *dev = usb_get_intfdata(intf); + struct mt76_connac_pm *pm = &dev->pm; int err; + pm->suspended = true; + flush_work(&dev->reset_work); + err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, true); if (err) - return err; + goto failed; mt76u_stop_rx(&dev->mt76); mt76u_stop_tx(&dev->mt76); - set_bit(MT76_STATE_SUSPEND, &dev->mphy.state); - return 0; + +failed: + pm->suspended = false; + + if (err < 0) + mt7921_reset(&dev->mt76); + + return err; } static int mt7921u_resume(struct usb_interface *intf) { struct mt7921_dev *dev = usb_get_intfdata(intf); + struct mt76_connac_pm *pm = &dev->pm; bool reinit = true; int err, i; @@ -338,16 +345,21 @@ static int mt7921u_resume(struct usb_interface *intf) if (reinit || mt7921_dma_need_reinit(dev)) { err = mt7921u_dma_init(dev, true); if (err) - return err; + goto failed; } - clear_bit(MT76_STATE_SUSPEND, &dev->mphy.state); - err = mt76u_resume_rx(&dev->mt76); if (err < 0) - return err; + goto failed; + + err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, false); +failed: + pm->suspended = false; + + if (err < 0) + mt7921_reset(&dev->mt76); - return mt76_connac_mcu_set_hif_suspend(&dev->mt76, false); + return err; } #endif /* CONFIG_PM */ diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index aba2a98658214843f79b58d928ece91a315b439d..0ec308f99af5a52e72c0ea000482725d225ba1ea 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -478,14 +478,14 @@ static void mt76s_status_worker(struct mt76_worker *w) if (ndata_frames > 0) resched = true; - if (dev->drv->tx_status_data && + if (dev->drv->tx_status_data && ndata_frames > 0 && !test_and_set_bit(MT76_READING_STATS, &dev->phy.state) && !test_bit(MT76_STATE_SUSPEND, &dev->phy.state)) - queue_work(dev->wq, &dev->sdio.stat_work); + ieee80211_queue_work(dev->hw, &dev->sdio.stat_work); } while (nframes > 0); if (resched) - mt76_worker_schedule(&dev->sdio.txrx_worker); + mt76_worker_schedule(&dev->tx_worker); } static void mt76s_tx_status_data(struct work_struct *work) @@ -508,7 +508,7 @@ static void mt76s_tx_status_data(struct work_struct *work) } if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state)) - queue_work(dev->wq, &sdio->stat_work); + ieee80211_queue_work(dev->hw, &sdio->stat_work); else clear_bit(MT76_READING_STATS, &dev->phy.state); } diff --git a/drivers/net/wireless/mediatek/mt76/sdio_txrx.c b/drivers/net/wireless/mediatek/mt76/sdio_txrx.c index a2601aa9e7b1ceb1352640219fa781dbf63659ec..bfc4de50a4d23226cc75166b6b06ad571ea94097 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio_txrx.c +++ b/drivers/net/wireless/mediatek/mt76/sdio_txrx.c @@ -85,7 +85,7 @@ mt76s_rx_run_queue(struct mt76_dev *dev, enum mt76_rxq_id qid, struct mt76_sdio *sdio = &dev->sdio; int len = 0, err, i; struct page *page; - u8 *buf; + u8 *buf, *end; for (i = 0; i < intr->rx.num[qid]; i++) len += round_up(intr->rx.len[qid][i] + 4, 4); @@ -112,20 +112,29 @@ mt76s_rx_run_queue(struct mt76_dev *dev, enum mt76_rxq_id qid, return err; } - for (i = 0; i < intr->rx.num[qid]; i++) { + end = buf + len; + i = 0; + + while (i < intr->rx.num[qid] && buf < end) { int index = (q->head + i) % q->ndesc; struct mt76_queue_entry *e = &q->entry[index]; __le32 *rxd = (__le32 *)buf; /* parse rxd to get the actual packet length */ len = le32_get_bits(rxd[0], GENMASK(15, 0)); - e->skb = mt76s_build_rx_skb(buf, len, round_up(len + 4, 4)); - if (!e->skb) - break; + /* Optimized path for TXS */ + if (!dev->drv->rx_check || dev->drv->rx_check(dev, buf, len)) { + e->skb = mt76s_build_rx_skb(buf, len, + round_up(len + 4, 4)); + if (!e->skb) + break; + + if (q->queued + i + 1 == q->ndesc) + break; + i++; + } buf += round_up(len + 4, 4); - if (q->queued + i + 1 == q->ndesc) - break; } put_page(page); diff --git a/drivers/net/wireless/mediatek/mt76/testmode.c b/drivers/net/wireless/mediatek/mt76/testmode.c index 71fd3fbfa7d222942e0f8cda6d694c97867ab91d..0accc71a91c9a09ba305f4083c1932345b59b95c 100644 --- a/drivers/net/wireless/mediatek/mt76/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/testmode.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: ISC /* Copyright (C) 2020 Felix Fietkau */ + +#include #include "mt76.h" const struct nla_policy mt76_tm_policy[NUM_MT76_TM_ATTRS] = { @@ -123,12 +125,14 @@ int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len) if (!head) return -ENOMEM; - hdr = __skb_put_zero(head, head_len); + hdr = __skb_put_zero(head, sizeof(*hdr)); hdr->frame_control = cpu_to_le16(fc); memcpy(hdr->addr1, td->addr[0], ETH_ALEN); memcpy(hdr->addr2, td->addr[1], ETH_ALEN); memcpy(hdr->addr3, td->addr[2], ETH_ALEN); skb_set_queue_mapping(head, IEEE80211_AC_BE); + get_random_bytes(__skb_put(head, head_len - sizeof(*hdr)), + head_len - sizeof(*hdr)); info = IEEE80211_SKB_CB(head); info->flags = IEEE80211_TX_CTL_INJECTED | @@ -154,7 +158,7 @@ int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len) return -ENOMEM; } - __skb_put_zero(frag, frag_len); + get_random_bytes(__skb_put(frag, frag_len), frag_len); head->len += frag->len; head->data_len += frag->len; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index e67cc7909bce38d16d4df699ddb400c6e475e0d2..6c054850363f63c83b2fdc446ef06addaa1c86e2 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -60,14 +60,20 @@ mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) .skb = skb, .info = IEEE80211_SKB_CB(skb), }; + struct ieee80211_rate_status rs = {}; struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); struct mt76_wcid *wcid; wcid = rcu_dereference(dev->wcid[cb->wcid]); if (wcid) { status.sta = wcid_to_sta(wcid); - status.rates = NULL; - status.n_rates = 0; + if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { + rs.rate_idx = wcid->rate; + status.rates = &rs; + status.n_rates = 1; + } else { + status.n_rates = 0; + } } hw = mt76_tx_status_get_hw(dev, skb); diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c index 6b8964c19f50d4294ee86211edf722bd53fab612..4c4033bb1bb35ef5c95842d5281a642df0920398 100644 --- a/drivers/net/wireless/mediatek/mt76/usb.c +++ b/drivers/net/wireless/mediatek/mt76/usb.c @@ -528,6 +528,11 @@ mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb, head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN; data_len = min_t(int, len, data_len - head_room); + + if (len == data_len && + dev->drv->rx_check && !dev->drv->rx_check(dev, data, data_len)) + return 0; + skb = mt76u_build_rx_skb(dev, data, data_len, buf_size); if (!skb) return 0; diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c index 3ac373d29d9368e08bdf4f0b07edb03b9f560c1d..9bbfff803357859de5004c797e4ef83e073c50fc 100644 --- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c +++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c @@ -540,8 +540,9 @@ static int wilc_wfi_cfg_copy_wpa_info(struct wilc_wfi_key *key_info, return 0; } -static int add_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr, struct key_params *params) +static int add_key(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr, + struct key_params *params) { int ret = 0, keylen = params->key_len; @@ -644,7 +645,7 @@ static int add_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, return ret; } -static int del_key(struct wiphy *wiphy, struct net_device *netdev, +static int del_key(struct wiphy *wiphy, struct net_device *netdev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr) @@ -685,8 +686,9 @@ static int del_key(struct wiphy *wiphy, struct net_device *netdev, return 0; } -static int get_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr, void *cookie, +static int get_key(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr, + void *cookie, void (*callback)(void *cookie, struct key_params *)) { struct wilc_vif *vif = netdev_priv(netdev); @@ -723,13 +725,14 @@ static int get_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, /* wiphy_new_nm() will WARNON if not present */ static int set_default_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool unicast, bool multicast) + int link_id, u8 key_index, bool unicast, + bool multicast) { return 0; } static int set_default_mgmt_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index) + int link_id, u8 key_index) { struct wilc_vif *vif = netdev_priv(netdev); @@ -994,12 +997,11 @@ bool wilc_wfi_mgmt_frame_rx(struct wilc_vif *vif, u8 *buff, u32 size) { struct wilc *wl = vif->wilc; struct wilc_priv *priv = &vif->priv; - int freq, ret; + int freq; freq = ieee80211_channel_to_frequency(wl->op_ch, NL80211_BAND_2GHZ); - ret = cfg80211_rx_mgmt(&priv->wdev, freq, 0, buff, size, 0); - return ret; + return cfg80211_rx_mgmt(&priv->wdev, freq, 0, buff, size, 0); } void wilc_wfi_p2p_rx(struct wilc_vif *vif, u8 *buff, u32 size) @@ -1159,7 +1161,7 @@ static int mgmt_tx(struct wiphy *wiphy, const u8 *vendor_ie; int ret = 0; - *cookie = prandom_u32(); + *cookie = get_random_u32(); priv->tx_cookie = *cookie; mgmt = (const struct ieee80211_mgmt *)buf; diff --git a/drivers/net/wireless/microchip/wilc1000/mon.c b/drivers/net/wireless/microchip/wilc1000/mon.c index b5a1b65c087cab98e190145d56a569b43a92bfd1..03b7229a0ff5aab90e1b1e860e67142478f3f61c 100644 --- a/drivers/net/wireless/microchip/wilc1000/mon.c +++ b/drivers/net/wireless/microchip/wilc1000/mon.c @@ -229,7 +229,7 @@ struct net_device *wilc_wfi_init_mon_interface(struct wilc *wl, return NULL; wl->monitor_dev->type = ARPHRD_IEEE80211_RADIOTAP; - strlcpy(wl->monitor_dev->name, name, IFNAMSIZ); + strscpy(wl->monitor_dev->name, name, IFNAMSIZ); wl->monitor_dev->netdev_ops = &wilc_wfi_netdev_ops; wl->monitor_dev->needs_free_netdev = true; diff --git a/drivers/net/wireless/microchip/wilc1000/netdev.h b/drivers/net/wireless/microchip/wilc1000/netdev.h index 43c085c74b7a5f5e5aee209074e5611b23bc2866..bb1a315a7b7ead76bbdf252286e637b5a1f1b7bc 100644 --- a/drivers/net/wireless/microchip/wilc1000/netdev.h +++ b/drivers/net/wireless/microchip/wilc1000/netdev.h @@ -245,6 +245,7 @@ struct wilc { u8 *rx_buffer; u32 rx_buffer_offset; u8 *tx_buffer; + u32 *vmm_table; struct txq_handle txq[NQUEUES]; int txq_entries; diff --git a/drivers/net/wireless/microchip/wilc1000/sdio.c b/drivers/net/wireless/microchip/wilc1000/sdio.c index 600cc57e9da2031ef0e80754029cf75fb1a1fcf1..7390f94cd4ca228368fc0b24cc567ffb28b73772 100644 --- a/drivers/net/wireless/microchip/wilc1000/sdio.c +++ b/drivers/net/wireless/microchip/wilc1000/sdio.c @@ -28,6 +28,7 @@ struct wilc_sdio { u32 block_size; bool isinit; int has_thrpt_enh3; + u8 *cmd53_buf; }; struct sdio_cmd52 { @@ -47,6 +48,7 @@ struct sdio_cmd53 { u32 count: 9; u8 *buffer; u32 block_size; + bool use_global_buf; }; static const struct wilc_hif_func wilc_hif_sdio; @@ -91,6 +93,8 @@ static int wilc_sdio_cmd53(struct wilc *wilc, struct sdio_cmd53 *cmd) { struct sdio_func *func = container_of(wilc->dev, struct sdio_func, dev); int size, ret; + struct wilc_sdio *sdio_priv = wilc->bus_data; + u8 *buf = cmd->buffer; sdio_claim_host(func); @@ -101,12 +105,23 @@ static int wilc_sdio_cmd53(struct wilc *wilc, struct sdio_cmd53 *cmd) else size = cmd->count; + if (cmd->use_global_buf) { + if (size > sizeof(u32)) + return -EINVAL; + + buf = sdio_priv->cmd53_buf; + } + if (cmd->read_write) { /* write */ - ret = sdio_memcpy_toio(func, cmd->address, - (void *)cmd->buffer, size); + if (cmd->use_global_buf) + memcpy(buf, cmd->buffer, size); + + ret = sdio_memcpy_toio(func, cmd->address, buf, size); } else { /* read */ - ret = sdio_memcpy_fromio(func, (void *)cmd->buffer, - cmd->address, size); + ret = sdio_memcpy_fromio(func, buf, cmd->address, size); + + if (cmd->use_global_buf) + memcpy(cmd->buffer, buf, size); } sdio_release_host(func); @@ -128,6 +143,12 @@ static int wilc_sdio_probe(struct sdio_func *func, if (!sdio_priv) return -ENOMEM; + sdio_priv->cmd53_buf = kzalloc(sizeof(u32), GFP_KERNEL); + if (!sdio_priv->cmd53_buf) { + ret = -ENOMEM; + goto free; + } + ret = wilc_cfg80211_init(&wilc, &func->dev, WILC_HIF_SDIO, &wilc_hif_sdio); if (ret) @@ -161,6 +182,7 @@ dispose_irq: irq_dispose_mapping(wilc->dev_irq_num); wilc_netdev_cleanup(wilc); free: + kfree(sdio_priv->cmd53_buf); kfree(sdio_priv); return ret; } @@ -172,6 +194,7 @@ static void wilc_sdio_remove(struct sdio_func *func) clk_disable_unprepare(wilc->rtc_clk); wilc_netdev_cleanup(wilc); + kfree(sdio_priv->cmd53_buf); kfree(sdio_priv); } @@ -375,8 +398,9 @@ static int wilc_sdio_write_reg(struct wilc *wilc, u32 addr, u32 data) cmd.address = WILC_SDIO_FBR_DATA_REG; cmd.block_mode = 0; cmd.increment = 1; - cmd.count = 4; + cmd.count = sizeof(u32); cmd.buffer = (u8 *)&data; + cmd.use_global_buf = true; cmd.block_size = sdio_priv->block_size; ret = wilc_sdio_cmd53(wilc, &cmd); if (ret) @@ -414,6 +438,7 @@ static int wilc_sdio_write(struct wilc *wilc, u32 addr, u8 *buf, u32 size) nblk = size / block_size; nleft = size % block_size; + cmd.use_global_buf = false; if (nblk > 0) { cmd.block_mode = 1; cmd.increment = 1; @@ -492,8 +517,9 @@ static int wilc_sdio_read_reg(struct wilc *wilc, u32 addr, u32 *data) cmd.address = WILC_SDIO_FBR_DATA_REG; cmd.block_mode = 0; cmd.increment = 1; - cmd.count = 4; + cmd.count = sizeof(u32); cmd.buffer = (u8 *)data; + cmd.use_global_buf = true; cmd.block_size = sdio_priv->block_size; ret = wilc_sdio_cmd53(wilc, &cmd); @@ -535,6 +561,7 @@ static int wilc_sdio_read(struct wilc *wilc, u32 addr, u8 *buf, u32 size) nblk = size / block_size; nleft = size % block_size; + cmd.use_global_buf = false; if (nblk > 0) { cmd.block_mode = 1; cmd.increment = 1; diff --git a/drivers/net/wireless/microchip/wilc1000/wlan.c b/drivers/net/wireless/microchip/wilc1000/wlan.c index 947d9a0a494ef4c30e9924f52b6947a9c067cf99..58bbf50081e4747450416417a4325f47ac4f623c 100644 --- a/drivers/net/wireless/microchip/wilc1000/wlan.c +++ b/drivers/net/wireless/microchip/wilc1000/wlan.c @@ -714,7 +714,7 @@ int wilc_wlan_handle_txq(struct wilc *wilc, u32 *txq_count) int ret = 0; int counter; int timeout; - u32 vmm_table[WILC_VMM_TBL_SIZE]; + u32 *vmm_table = wilc->vmm_table; u8 ac_pkt_num_to_chip[NQUEUES] = {0, 0, 0, 0}; const struct wilc_hif_func *func; int srcu_idx; @@ -1252,6 +1252,8 @@ void wilc_wlan_cleanup(struct net_device *dev) while ((rqe = wilc_wlan_rxq_remove(wilc))) kfree(rqe); + kfree(wilc->vmm_table); + wilc->vmm_table = NULL; kfree(wilc->rx_buffer); wilc->rx_buffer = NULL; kfree(wilc->tx_buffer); @@ -1489,6 +1491,14 @@ int wilc_wlan_init(struct net_device *dev) goto fail; } + if (!wilc->vmm_table) + wilc->vmm_table = kzalloc(WILC_VMM_TBL_SIZE, GFP_KERNEL); + + if (!wilc->vmm_table) { + ret = -ENOBUFS; + goto fail; + } + if (!wilc->tx_buffer) wilc->tx_buffer = kmalloc(WILC_TX_BUFF_SIZE, GFP_KERNEL); @@ -1513,7 +1523,8 @@ int wilc_wlan_init(struct net_device *dev) return 0; fail: - + kfree(wilc->vmm_table); + wilc->vmm_table = NULL; kfree(wilc->rx_buffer); wilc->rx_buffer = NULL; kfree(wilc->tx_buffer); diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c index 1593e810b3ca431e176a003536ffe8ab6d50fc18..73e6f9408b515091e8f78329daec665070abc27a 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c +++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c @@ -449,7 +449,7 @@ qtnf_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev, { struct qtnf_vif *vif = qtnf_netdev_get_priv(wdev->netdev); const struct ieee80211_mgmt *mgmt_frame = (void *)params->buf; - u32 short_cookie = prandom_u32(); + u32 short_cookie = get_random_u32(); u16 flags = 0; u16 freq; @@ -532,8 +532,8 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev, } static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct qtnf_vif *vif = qtnf_netdev_get_priv(dev); int ret; @@ -548,7 +548,8 @@ static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev, } static int qtnf_del_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { struct qtnf_vif *vif = qtnf_netdev_get_priv(dev); int ret; @@ -569,7 +570,8 @@ static int qtnf_del_key(struct wiphy *wiphy, struct net_device *dev, } static int qtnf_set_default_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool unicast, bool multicast) + int link_id, u8 key_index, bool unicast, + bool multicast) { struct qtnf_vif *vif = qtnf_netdev_get_priv(dev); int ret; @@ -585,7 +587,7 @@ static int qtnf_set_default_key(struct wiphy *wiphy, struct net_device *dev, static int qtnf_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index) + int link_id, u8 key_index) { struct qtnf_vif *vif = qtnf_netdev_get_priv(dev); int ret; @@ -721,9 +723,8 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev, return -EFAULT; } - if (vif->wdev.iftype != NL80211_IFTYPE_STATION) { + if (vif->wdev.iftype != NL80211_IFTYPE_STATION) return -EOPNOTSUPP; - } ret = qtnf_cmd_send_disconnect(vif, reason_code); if (ret) @@ -750,7 +751,6 @@ qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev, struct ieee80211_channel *chan; int ret; - sband = wiphy->bands[NL80211_BAND_2GHZ]; if (sband && idx >= sband->n_channels) { idx -= sband->n_channels; @@ -1223,7 +1223,7 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac) mac->macinfo.extended_capabilities_len; } - strlcpy(wiphy->fw_version, hw_info->fw_version, + strscpy(wiphy->fw_version, hw_info->fw_version, sizeof(wiphy->fw_version)); wiphy->hw_version = hw_info->hw_version; diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c index 0fad53693292d7324485c2bc614ced12103ddd80..b1b73478d89b510d763f83d9570494c2aefb8efe 100644 --- a/drivers/net/wireless/quantenna/qtnfmac/commands.c +++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c @@ -967,7 +967,7 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus, hwinfo->total_rx_chain, hwinfo->total_tx_chain, hwinfo->fw_ver); - strlcpy(hwinfo->fw_version, bld_label, sizeof(hwinfo->fw_version)); + strscpy(hwinfo->fw_version, bld_label, sizeof(hwinfo->fw_version)); hwinfo->hw_version = hw_ver; return 0; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800.h b/drivers/net/wireless/ralink/rt2x00/rt2800.h index d758e887445783f2d0bf9e1365ef37edfcb1596a..de2ee5ffc34e7bd335f7c67fde912afa66dbae8c 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2800.h @@ -1016,6 +1016,8 @@ */ #define MAC_STATUS_CFG 0x1200 #define MAC_STATUS_CFG_BBP_RF_BUSY FIELD32(0x00000003) +#define MAC_STATUS_CFG_BBP_RF_BUSY_TX FIELD32(0x00000001) +#define MAC_STATUS_CFG_BBP_RF_BUSY_RX FIELD32(0x00000002) /* * PWR_PIN_CFG: @@ -2739,6 +2741,7 @@ enum rt2800_eeprom_word { #define EEPROM_NIC_CONF2_RX_STREAM FIELD16(0x000f) #define EEPROM_NIC_CONF2_TX_STREAM FIELD16(0x00f0) #define EEPROM_NIC_CONF2_CRYSTAL FIELD16(0x0600) +#define EEPROM_NIC_CONF2_EXTERNAL_PA FIELD16(0x8000) /* * EEPROM LNA diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c index 18102fbe36d6cc507bb69508154f14d2fc760b54..cbbb1a4849cff046a767fa002d379718435cebf1 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c @@ -198,6 +198,26 @@ static void rt2800_rfcsr_write_dccal(struct rt2x00_dev *rt2x00dev, rt2800_rfcsr_write_bank(rt2x00dev, 7, reg, value); } +static void rt2800_bbp_dcoc_write(struct rt2x00_dev *rt2x00dev, + const u8 reg, const u8 value) +{ + rt2800_bbp_write(rt2x00dev, 158, reg); + rt2800_bbp_write(rt2x00dev, 159, value); +} + +static u8 rt2800_bbp_dcoc_read(struct rt2x00_dev *rt2x00dev, const u8 reg) +{ + rt2800_bbp_write(rt2x00dev, 158, reg); + return rt2800_bbp_read(rt2x00dev, 159); +} + +static void rt2800_bbp_glrt_write(struct rt2x00_dev *rt2x00dev, + const u8 reg, const u8 value) +{ + rt2800_bbp_write(rt2x00dev, 195, reg); + rt2800_bbp_write(rt2x00dev, 196, value); +} + static u8 rt2800_rfcsr_read(struct rt2x00_dev *rt2x00dev, const unsigned int word) { @@ -2143,6 +2163,48 @@ void rt2800_config_erp(struct rt2x00_dev *rt2x00dev, struct rt2x00lib_erp *erp, } EXPORT_SYMBOL_GPL(rt2800_config_erp); +static int rt2800_wait_bbp_rf_ready(struct rt2x00_dev *rt2x00dev, + const struct rt2x00_field32 mask) +{ + unsigned int i; + u32 reg; + + for (i = 0; i < REGISTER_BUSY_COUNT; i++) { + reg = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG); + if (!rt2x00_get_field32(reg, mask)) + return 0; + + udelay(REGISTER_BUSY_DELAY); + } + + rt2x00_err(rt2x00dev, "BBP/RF register access failed, aborting\n"); + return -EACCES; +} + +static int rt2800_wait_bbp_ready(struct rt2x00_dev *rt2x00dev) +{ + unsigned int i; + u8 value; + + /* + * BBP was enabled after firmware was loaded, + * but we need to reactivate it now. + */ + rt2800_register_write(rt2x00dev, H2M_BBP_AGENT, 0); + rt2800_register_write(rt2x00dev, H2M_MAILBOX_CSR, 0); + msleep(1); + + for (i = 0; i < REGISTER_BUSY_COUNT; i++) { + value = rt2800_bbp_read(rt2x00dev, 0); + if ((value != 0xff) && (value != 0x00)) + return 0; + udelay(REGISTER_BUSY_DELAY); + } + + rt2x00_err(rt2x00dev, "BBP register access failed, aborting\n"); + return -EACCES; +} + static void rt2800_config_3572bt_ant(struct rt2x00_dev *rt2x00dev) { u32 reg; @@ -3793,16 +3855,23 @@ static void rt2800_config_channel_rf7620(struct rt2x00_dev *rt2x00dev, rfcsr |= tx_agc_fc; rt2800_rfcsr_write_bank(rt2x00dev, 7, 59, rfcsr); } + + if (conf_is_ht40(conf)) { + rt2800_bbp_glrt_write(rt2x00dev, 141, 0x10); + rt2800_bbp_glrt_write(rt2x00dev, 157, 0x2f); + } else { + rt2800_bbp_glrt_write(rt2x00dev, 141, 0x1a); + rt2800_bbp_glrt_write(rt2x00dev, 157, 0x40); + } } static void rt2800_config_alc(struct rt2x00_dev *rt2x00dev, struct ieee80211_channel *chan, int power_level) { u16 eeprom, target_power, max_power; - u32 mac_sys_ctrl, mac_status; + u32 mac_sys_ctrl; u32 reg; u8 bbp; - int i; /* hardware unit is 0.5dBm, limited to 23.5dBm */ power_level *= 2; @@ -3838,16 +3907,8 @@ static void rt2800_config_alc(struct rt2x00_dev *rt2x00dev, /* Disable Tx/Rx */ rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0); /* Check MAC Tx/Rx idle */ - for (i = 0; i < 10000; i++) { - mac_status = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG); - if (mac_status & 0x3) - usleep_range(50, 200); - else - break; - } - - if (i == 10000) - rt2x00_warn(rt2x00dev, "Wait MAC Status to MAX !!!\n"); + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY))) + rt2x00_warn(rt2x00dev, "RF busy while configuring ALC\n"); if (chan->center_freq > 2457) { bbp = rt2800_bbp_read(rt2x00dev, 30); @@ -4164,7 +4225,10 @@ static void rt2800_config_channel(struct rt2x00_dev *rt2x00dev, rt2800_bbp_write(rt2x00dev, 62, 0x37 - rt2x00dev->lna_gain); rt2800_bbp_write(rt2x00dev, 63, 0x37 - rt2x00dev->lna_gain); rt2800_bbp_write(rt2x00dev, 64, 0x37 - rt2x00dev->lna_gain); - rt2800_bbp_write(rt2x00dev, 86, 0); + if (rt2x00_rt(rt2x00dev, RT6352)) + rt2800_bbp_write(rt2x00dev, 86, 0x38); + else + rt2800_bbp_write(rt2x00dev, 86, 0); } if (rf->channel <= 14) { @@ -4365,7 +4429,45 @@ static void rt2800_config_channel(struct rt2x00_dev *rt2x00dev, reg = (rf->channel <= 14 ? 0x1c : 0x24) + 2*rt2x00dev->lna_gain; rt2800_bbp_write_with_rx_chain(rt2x00dev, 66, reg); - rt2800_iq_calibrate(rt2x00dev, rf->channel); + if (rt2x00_rt(rt2x00dev, RT5592)) + rt2800_iq_calibrate(rt2x00dev, rf->channel); + } + + if (rt2x00_rt(rt2x00dev, RT6352)) { + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, + &rt2x00dev->cap_flags)) { + reg = rt2800_register_read(rt2x00dev, RF_CONTROL3); + reg |= 0x00000101; + rt2800_register_write(rt2x00dev, RF_CONTROL3, reg); + + reg = rt2800_register_read(rt2x00dev, RF_BYPASS3); + reg |= 0x00000101; + rt2800_register_write(rt2x00dev, RF_BYPASS3, reg); + + rt2800_rfcsr_write_chanreg(rt2x00dev, 43, 0x73); + rt2800_rfcsr_write_chanreg(rt2x00dev, 44, 0x73); + rt2800_rfcsr_write_chanreg(rt2x00dev, 45, 0x73); + rt2800_rfcsr_write_chanreg(rt2x00dev, 46, 0x27); + rt2800_rfcsr_write_chanreg(rt2x00dev, 47, 0xC8); + rt2800_rfcsr_write_chanreg(rt2x00dev, 48, 0xA4); + rt2800_rfcsr_write_chanreg(rt2x00dev, 49, 0x05); + rt2800_rfcsr_write_chanreg(rt2x00dev, 54, 0x27); + rt2800_rfcsr_write_chanreg(rt2x00dev, 55, 0xC8); + rt2800_rfcsr_write_chanreg(rt2x00dev, 56, 0xA4); + rt2800_rfcsr_write_chanreg(rt2x00dev, 57, 0x05); + rt2800_rfcsr_write_chanreg(rt2x00dev, 58, 0x27); + rt2800_rfcsr_write_chanreg(rt2x00dev, 59, 0xC8); + rt2800_rfcsr_write_chanreg(rt2x00dev, 60, 0xA4); + rt2800_rfcsr_write_chanreg(rt2x00dev, 61, 0x05); + rt2800_rfcsr_write_dccal(rt2x00dev, 05, 0x00); + + rt2800_register_write(rt2x00dev, TX0_RF_GAIN_CORRECT, + 0x36303636); + rt2800_register_write(rt2x00dev, TX0_RF_GAIN_ATTEN, + 0x6C6C6B6C); + rt2800_register_write(rt2x00dev, TX1_RF_GAIN_ATTEN, + 0x6C6C6B6C); + } } bbp = rt2800_bbp_read(rt2x00dev, 4); @@ -5644,7 +5746,8 @@ static inline void rt2800_set_vgc(struct rt2x00_dev *rt2x00dev, if (qual->vgc_level != vgc_level) { if (rt2x00_rt(rt2x00dev, RT3572) || rt2x00_rt(rt2x00dev, RT3593) || - rt2x00_rt(rt2x00dev, RT3883)) { + rt2x00_rt(rt2x00dev, RT3883) || + rt2x00_rt(rt2x00dev, RT6352)) { rt2800_bbp_write_with_rx_chain(rt2x00dev, 66, vgc_level); } else if (rt2x00_rt(rt2x00dev, RT5592)) { @@ -5867,7 +5970,7 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev) rt2800_register_write(rt2x00dev, TX_SW_CFG0, 0x00000404); } else if (rt2x00_rt(rt2x00dev, RT6352)) { rt2800_register_write(rt2x00dev, TX_SW_CFG0, 0x00000401); - rt2800_register_write(rt2x00dev, TX_SW_CFG1, 0x000C0000); + rt2800_register_write(rt2x00dev, TX_SW_CFG1, 0x000C0001); rt2800_register_write(rt2x00dev, TX_SW_CFG2, 0x00000000); rt2800_register_write(rt2x00dev, TX_ALC_VGA3, 0x00000000); rt2800_register_write(rt2x00dev, TX0_BB_GAIN_ATTEN, 0x0); @@ -6129,6 +6232,27 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev) reg = rt2800_register_read(rt2x00dev, US_CYC_CNT); rt2x00_set_field32(®, US_CYC_CNT_CLOCK_CYCLE, 125); rt2800_register_write(rt2x00dev, US_CYC_CNT, reg); + } else if (rt2x00_is_soc(rt2x00dev)) { + struct clk *clk = clk_get_sys("bus", NULL); + int rate; + + if (IS_ERR(clk)) { + clk = clk_get_sys("cpu", NULL); + + if (IS_ERR(clk)) { + rate = 125; + } else { + rate = clk_get_rate(clk) / 3000000; + clk_put(clk); + } + } else { + rate = clk_get_rate(clk) / 1000000; + clk_put(clk); + } + + reg = rt2800_register_read(rt2x00dev, US_CYC_CNT); + rt2x00_set_field32(®, US_CYC_CNT_CLOCK_CYCLE, rate); + rt2800_register_write(rt2x00dev, US_CYC_CNT, reg); } reg = rt2800_register_read(rt2x00dev, HT_FBK_CFG0); @@ -6212,46 +6336,6 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev) return 0; } -static int rt2800_wait_bbp_rf_ready(struct rt2x00_dev *rt2x00dev) -{ - unsigned int i; - u32 reg; - - for (i = 0; i < REGISTER_BUSY_COUNT; i++) { - reg = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG); - if (!rt2x00_get_field32(reg, MAC_STATUS_CFG_BBP_RF_BUSY)) - return 0; - - udelay(REGISTER_BUSY_DELAY); - } - - rt2x00_err(rt2x00dev, "BBP/RF register access failed, aborting\n"); - return -EACCES; -} - -static int rt2800_wait_bbp_ready(struct rt2x00_dev *rt2x00dev) -{ - unsigned int i; - u8 value; - - /* - * BBP was enabled after firmware was loaded, - * but we need to reactivate it now. - */ - rt2800_register_write(rt2x00dev, H2M_BBP_AGENT, 0); - rt2800_register_write(rt2x00dev, H2M_MAILBOX_CSR, 0); - msleep(1); - - for (i = 0; i < REGISTER_BUSY_COUNT; i++) { - value = rt2800_bbp_read(rt2x00dev, 0); - if ((value != 0xff) && (value != 0x00)) - return 0; - udelay(REGISTER_BUSY_DELAY); - } - - rt2x00_err(rt2x00dev, "BBP register access failed, aborting\n"); - return -EACCES; -} static void rt2800_bbp4_mac_if_ctrl(struct rt2x00_dev *rt2x00dev) { @@ -6916,26 +7000,6 @@ static void rt2800_init_bbp_5592(struct rt2x00_dev *rt2x00dev) rt2800_bbp_write(rt2x00dev, 103, 0xc0); } -static void rt2800_bbp_glrt_write(struct rt2x00_dev *rt2x00dev, - const u8 reg, const u8 value) -{ - rt2800_bbp_write(rt2x00dev, 195, reg); - rt2800_bbp_write(rt2x00dev, 196, value); -} - -static void rt2800_bbp_dcoc_write(struct rt2x00_dev *rt2x00dev, - const u8 reg, const u8 value) -{ - rt2800_bbp_write(rt2x00dev, 158, reg); - rt2800_bbp_write(rt2x00dev, 159, value); -} - -static u8 rt2800_bbp_dcoc_read(struct rt2x00_dev *rt2x00dev, const u8 reg) -{ - rt2800_bbp_write(rt2x00dev, 158, reg); - return rt2800_bbp_read(rt2x00dev, 159); -} - static void rt2800_init_bbp_6352(struct rt2x00_dev *rt2x00dev) { u8 bbp; @@ -8398,6 +8462,1519 @@ static void rt2800_init_rfcsr_5592(struct rt2x00_dev *rt2x00dev) rt2800_led_open_drain_enable(rt2x00dev); } +static void rt2800_rf_self_txdc_cal(struct rt2x00_dev *rt2x00dev) +{ + u8 rfb5r1_org, rfb7r1_org, rfvalue; + u32 mac0518, mac051c, mac0528, mac052c; + u8 i; + + mac0518 = rt2800_register_read(rt2x00dev, RF_CONTROL0); + mac051c = rt2800_register_read(rt2x00dev, RF_BYPASS0); + mac0528 = rt2800_register_read(rt2x00dev, RF_CONTROL2); + mac052c = rt2800_register_read(rt2x00dev, RF_BYPASS2); + + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x0); + rt2800_register_write(rt2x00dev, RF_BYPASS2, 0x0); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0xC); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x3306); + rt2800_register_write(rt2x00dev, RF_CONTROL2, 0x3330); + rt2800_register_write(rt2x00dev, RF_BYPASS2, 0xfffff); + rfb5r1_org = rt2800_rfcsr_read_bank(rt2x00dev, 5, 1); + rfb7r1_org = rt2800_rfcsr_read_bank(rt2x00dev, 7, 1); + + rt2800_rfcsr_write_bank(rt2x00dev, 5, 1, 0x4); + for (i = 0; i < 100; ++i) { + usleep_range(50, 100); + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 1); + if ((rfvalue & 0x04) != 0x4) + break; + } + rt2800_rfcsr_write_bank(rt2x00dev, 5, 1, rfb5r1_org); + + rt2800_rfcsr_write_bank(rt2x00dev, 7, 1, 0x4); + for (i = 0; i < 100; ++i) { + usleep_range(50, 100); + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 1); + if ((rfvalue & 0x04) != 0x4) + break; + } + rt2800_rfcsr_write_bank(rt2x00dev, 7, 1, rfb7r1_org); + + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x0); + rt2800_register_write(rt2x00dev, RF_BYPASS2, 0x0); + rt2800_register_write(rt2x00dev, RF_CONTROL0, mac0518); + rt2800_register_write(rt2x00dev, RF_BYPASS0, mac051c); + rt2800_register_write(rt2x00dev, RF_CONTROL2, mac0528); + rt2800_register_write(rt2x00dev, RF_BYPASS2, mac052c); +} + +static int rt2800_calcrcalibrationcode(struct rt2x00_dev *rt2x00dev, int d1, int d2) +{ + int calcode = ((d2 - d1) * 1000) / 43; + + if ((calcode % 10) >= 5) + calcode += 10; + calcode = (calcode / 10); + + return calcode; +} + +static void rt2800_r_calibration(struct rt2x00_dev *rt2x00dev) +{ + u32 savemacsysctrl; + u8 saverfb0r1, saverfb0r34, saverfb0r35; + u8 saverfb5r4, saverfb5r17, saverfb5r18; + u8 saverfb5r19, saverfb5r20; + u8 savebbpr22, savebbpr47, savebbpr49; + u8 bytevalue = 0; + int rcalcode; + u8 r_cal_code = 0; + char d1 = 0, d2 = 0; + u8 rfvalue; + u32 MAC_RF_BYPASS0, MAC_RF_CONTROL0, MAC_PWR_PIN_CFG; + u32 maccfg; + + saverfb0r1 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1); + saverfb0r34 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 34); + saverfb0r35 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35); + saverfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4); + saverfb5r17 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17); + saverfb5r18 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18); + saverfb5r19 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19); + saverfb5r20 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20); + + savebbpr22 = rt2800_bbp_read(rt2x00dev, 22); + savebbpr47 = rt2800_bbp_read(rt2x00dev, 47); + savebbpr49 = rt2800_bbp_read(rt2x00dev, 49); + + savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + MAC_RF_BYPASS0 = rt2800_register_read(rt2x00dev, RF_BYPASS0); + MAC_RF_CONTROL0 = rt2800_register_read(rt2x00dev, RF_CONTROL0); + MAC_PWR_PIN_CFG = rt2800_register_read(rt2x00dev, PWR_PIN_CFG); + + maccfg = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + maccfg &= (~0x04); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, maccfg); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX))) + rt2x00_warn(rt2x00dev, "Wait MAC Tx Status to MAX !!!\n"); + + maccfg = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + maccfg &= (~0x04); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, maccfg); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX))) + rt2x00_warn(rt2x00dev, "Wait MAC Rx Status to MAX !!!\n"); + + rfvalue = (MAC_RF_BYPASS0 | 0x3004); + rt2800_register_write(rt2x00dev, RF_BYPASS0, rfvalue); + rfvalue = (MAC_RF_CONTROL0 | (~0x3002)); + rt2800_register_write(rt2x00dev, RF_CONTROL0, rfvalue); + + rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, 0x27); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, 0x80); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, 0x83); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, 0x00); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, 0x20); + + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x00); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 34, 0x13); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00); + + rt2800_register_write(rt2x00dev, PWR_PIN_CFG, 0x1); + + rt2800_bbp_write(rt2x00dev, 47, 0x04); + rt2800_bbp_write(rt2x00dev, 22, 0x80); + usleep_range(100, 200); + bytevalue = rt2800_bbp_read(rt2x00dev, 49); + if (bytevalue > 128) + d1 = bytevalue - 256; + else + d1 = (char)bytevalue; + rt2800_bbp_write(rt2x00dev, 22, 0x0); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x01); + + rt2800_bbp_write(rt2x00dev, 22, 0x80); + usleep_range(100, 200); + bytevalue = rt2800_bbp_read(rt2x00dev, 49); + if (bytevalue > 128) + d2 = bytevalue - 256; + else + d2 = (char)bytevalue; + rt2800_bbp_write(rt2x00dev, 22, 0x0); + + rcalcode = rt2800_calcrcalibrationcode(rt2x00dev, d1, d2); + if (rcalcode < 0) + r_cal_code = 256 + rcalcode; + else + r_cal_code = (u8)rcalcode; + + rt2800_rfcsr_write_bank(rt2x00dev, 0, 7, r_cal_code); + + rt2800_bbp_write(rt2x00dev, 22, 0x0); + + bytevalue = rt2800_bbp_read(rt2x00dev, 21); + bytevalue |= 0x1; + rt2800_bbp_write(rt2x00dev, 21, bytevalue); + bytevalue = rt2800_bbp_read(rt2x00dev, 21); + bytevalue &= (~0x1); + rt2800_bbp_write(rt2x00dev, 21, bytevalue); + + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, saverfb0r1); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 34, saverfb0r34); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, saverfb0r35); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, saverfb5r4); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, saverfb5r17); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, saverfb5r18); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, saverfb5r19); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, saverfb5r20); + + rt2800_bbp_write(rt2x00dev, 22, savebbpr22); + rt2800_bbp_write(rt2x00dev, 47, savebbpr47); + rt2800_bbp_write(rt2x00dev, 49, savebbpr49); + + rt2800_register_write(rt2x00dev, RF_BYPASS0, MAC_RF_BYPASS0); + rt2800_register_write(rt2x00dev, RF_CONTROL0, MAC_RF_CONTROL0); + + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl); + rt2800_register_write(rt2x00dev, PWR_PIN_CFG, MAC_PWR_PIN_CFG); +} + +static void rt2800_rxdcoc_calibration(struct rt2x00_dev *rt2x00dev) +{ + u8 bbpreg = 0; + u32 macvalue = 0; + u8 saverfb0r2, saverfb5r4, saverfb7r4, rfvalue; + int i; + + saverfb0r2 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2); + rfvalue = saverfb0r2; + rfvalue |= 0x03; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfvalue); + + rt2800_bbp_write(rt2x00dev, 158, 141); + bbpreg = rt2800_bbp_read(rt2x00dev, 159); + bbpreg |= 0x10; + rt2800_bbp_write(rt2x00dev, 159, bbpreg); + + macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x8); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX))) + rt2x00_warn(rt2x00dev, "RF TX busy in RX RXDCOC calibration\n"); + + saverfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4); + saverfb7r4 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4); + saverfb5r4 = saverfb5r4 & (~0x40); + saverfb7r4 = saverfb7r4 & (~0x40); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x64); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, saverfb5r4); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, saverfb7r4); + + rt2800_bbp_write(rt2x00dev, 158, 141); + bbpreg = rt2800_bbp_read(rt2x00dev, 159); + bbpreg = bbpreg & (~0x40); + rt2800_bbp_write(rt2x00dev, 159, bbpreg); + bbpreg |= 0x48; + rt2800_bbp_write(rt2x00dev, 159, bbpreg); + + for (i = 0; i < 10000; i++) { + bbpreg = rt2800_bbp_read(rt2x00dev, 159); + if ((bbpreg & 0x40) == 0) + break; + usleep_range(50, 100); + } + + bbpreg = rt2800_bbp_read(rt2x00dev, 159); + bbpreg = bbpreg & (~0x40); + rt2800_bbp_write(rt2x00dev, 159, bbpreg); + + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue); + + rt2800_bbp_write(rt2x00dev, 158, 141); + bbpreg = rt2800_bbp_read(rt2x00dev, 159); + bbpreg &= (~0x10); + rt2800_bbp_write(rt2x00dev, 159, bbpreg); + + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, saverfb0r2); +} + +static u32 rt2800_do_sqrt_accumulation(u32 si) +{ + u32 root, root_pre, bit; + char i; + + bit = 1 << 15; + root = 0; + for (i = 15; i >= 0; i = i - 1) { + root_pre = root + bit; + if ((root_pre * root_pre) <= si) + root = root_pre; + bit = bit >> 1; + } + + return root; +} + +static void rt2800_rxiq_calibration(struct rt2x00_dev *rt2x00dev) +{ + u8 rfb0r1, rfb0r2, rfb0r42; + u8 rfb4r0, rfb4r19; + u8 rfb5r3, rfb5r4, rfb5r17, rfb5r18, rfb5r19, rfb5r20; + u8 rfb6r0, rfb6r19; + u8 rfb7r3, rfb7r4, rfb7r17, rfb7r18, rfb7r19, rfb7r20; + + u8 bbp1, bbp4; + u8 bbpr241, bbpr242; + u32 i; + u8 ch_idx; + u8 bbpval; + u8 rfval, vga_idx = 0; + int mi = 0, mq = 0, si = 0, sq = 0, riq = 0; + int sigma_i, sigma_q, r_iq, g_rx; + int g_imb; + int ph_rx; + u32 savemacsysctrl = 0; + u32 orig_RF_CONTROL0 = 0; + u32 orig_RF_BYPASS0 = 0; + u32 orig_RF_CONTROL1 = 0; + u32 orig_RF_BYPASS1 = 0; + u32 orig_RF_CONTROL3 = 0; + u32 orig_RF_BYPASS3 = 0; + u32 bbpval1 = 0; + static const u8 rf_vga_table[] = {0x20, 0x21, 0x22, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f}; + + savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + orig_RF_CONTROL0 = rt2800_register_read(rt2x00dev, RF_CONTROL0); + orig_RF_BYPASS0 = rt2800_register_read(rt2x00dev, RF_BYPASS0); + orig_RF_CONTROL1 = rt2800_register_read(rt2x00dev, RF_CONTROL1); + orig_RF_BYPASS1 = rt2800_register_read(rt2x00dev, RF_BYPASS1); + orig_RF_CONTROL3 = rt2800_register_read(rt2x00dev, RF_CONTROL3); + orig_RF_BYPASS3 = rt2800_register_read(rt2x00dev, RF_BYPASS3); + + bbp1 = rt2800_bbp_read(rt2x00dev, 1); + bbp4 = rt2800_bbp_read(rt2x00dev, 4); + + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x0); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY))) + rt2x00_warn(rt2x00dev, "Timeout waiting for MAC status in RXIQ calibration\n"); + + bbpval = bbp4 & (~0x18); + bbpval = bbp4 | 0x00; + rt2800_bbp_write(rt2x00dev, 4, bbpval); + + bbpval = rt2800_bbp_read(rt2x00dev, 21); + bbpval = bbpval | 1; + rt2800_bbp_write(rt2x00dev, 21, bbpval); + bbpval = bbpval & 0xfe; + rt2800_bbp_write(rt2x00dev, 21, bbpval); + + rt2800_register_write(rt2x00dev, RF_CONTROL1, 0x00000202); + rt2800_register_write(rt2x00dev, RF_BYPASS1, 0x00000303); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) + rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x0101); + else + rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x0000); + + rt2800_register_write(rt2x00dev, RF_BYPASS3, 0xf1f1); + + rfb0r1 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1); + rfb0r2 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2); + rfb0r42 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42); + rfb4r0 = rt2800_rfcsr_read_bank(rt2x00dev, 4, 0); + rfb4r19 = rt2800_rfcsr_read_bank(rt2x00dev, 4, 19); + rfb5r3 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 3); + rfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4); + rfb5r17 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17); + rfb5r18 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18); + rfb5r19 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19); + rfb5r20 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20); + + rfb6r0 = rt2800_rfcsr_read_bank(rt2x00dev, 6, 0); + rfb6r19 = rt2800_rfcsr_read_bank(rt2x00dev, 6, 19); + rfb7r3 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 3); + rfb7r4 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4); + rfb7r17 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 17); + rfb7r18 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 18); + rfb7r19 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 19); + rfb7r20 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 20); + + rt2800_rfcsr_write_chanreg(rt2x00dev, 0, 0x87); + rt2800_rfcsr_write_chanreg(rt2x00dev, 19, 0x27); + rt2800_rfcsr_write_dccal(rt2x00dev, 3, 0x38); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x38); + rt2800_rfcsr_write_dccal(rt2x00dev, 17, 0x80); + rt2800_rfcsr_write_dccal(rt2x00dev, 18, 0xC1); + rt2800_rfcsr_write_dccal(rt2x00dev, 19, 0x60); + rt2800_rfcsr_write_dccal(rt2x00dev, 20, 0x00); + + rt2800_bbp_write(rt2x00dev, 23, 0x0); + rt2800_bbp_write(rt2x00dev, 24, 0x0); + + rt2800_bbp_dcoc_write(rt2x00dev, 5, 0x0); + + bbpr241 = rt2800_bbp_read(rt2x00dev, 241); + bbpr242 = rt2800_bbp_read(rt2x00dev, 242); + + rt2800_bbp_write(rt2x00dev, 241, 0x10); + rt2800_bbp_write(rt2x00dev, 242, 0x84); + rt2800_bbp_write(rt2x00dev, 244, 0x31); + + bbpval = rt2800_bbp_dcoc_read(rt2x00dev, 3); + bbpval = bbpval & (~0x7); + rt2800_bbp_dcoc_write(rt2x00dev, 3, bbpval); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004); + udelay(1); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000006); + usleep_range(1, 200); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003376); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001006); + udelay(1); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_bbp_write(rt2x00dev, 23, 0x06); + rt2800_bbp_write(rt2x00dev, 24, 0x06); + } else { + rt2800_bbp_write(rt2x00dev, 23, 0x02); + rt2800_bbp_write(rt2x00dev, 24, 0x02); + } + + for (ch_idx = 0; ch_idx < 2; ch_idx = ch_idx + 1) { + if (ch_idx == 0) { + rfval = rfb0r1 & (~0x3); + rfval = rfb0r1 | 0x1; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfval); + rfval = rfb0r2 & (~0x33); + rfval = rfb0r2 | 0x11; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfval); + rfval = rfb0r42 & (~0x50); + rfval = rfb0r42 | 0x10; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfval); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001006); + udelay(1); + + bbpval = bbp1 & (~0x18); + bbpval = bbpval | 0x00; + rt2800_bbp_write(rt2x00dev, 1, bbpval); + + rt2800_bbp_dcoc_write(rt2x00dev, 1, 0x00); + } else { + rfval = rfb0r1 & (~0x3); + rfval = rfb0r1 | 0x2; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfval); + rfval = rfb0r2 & (~0x33); + rfval = rfb0r2 | 0x22; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfval); + rfval = rfb0r42 & (~0x50); + rfval = rfb0r42 | 0x40; + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfval); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002006); + udelay(1); + + bbpval = bbp1 & (~0x18); + bbpval = bbpval | 0x08; + rt2800_bbp_write(rt2x00dev, 1, bbpval); + + rt2800_bbp_dcoc_write(rt2x00dev, 1, 0x01); + } + usleep_range(500, 1500); + + vga_idx = 0; + while (vga_idx < 11) { + rt2800_rfcsr_write_dccal(rt2x00dev, 3, rf_vga_table[vga_idx]); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, rf_vga_table[vga_idx]); + + rt2800_bbp_dcoc_write(rt2x00dev, 0, 0x93); + + for (i = 0; i < 10000; i++) { + bbpval = rt2800_bbp_read(rt2x00dev, 159); + if ((bbpval & 0xff) == 0x93) + usleep_range(50, 100); + else + break; + } + + if ((bbpval & 0xff) == 0x93) { + rt2x00_warn(rt2x00dev, "Fatal Error: Calibration doesn't finish"); + goto restore_value; + } + for (i = 0; i < 5; i++) { + u32 bbptemp = 0; + u8 value = 0; + int result = 0; + + rt2800_bbp_write(rt2x00dev, 158, 0x1e); + rt2800_bbp_write(rt2x00dev, 159, i); + rt2800_bbp_write(rt2x00dev, 158, 0x22); + value = rt2800_bbp_read(rt2x00dev, 159); + bbptemp = bbptemp + (value << 24); + rt2800_bbp_write(rt2x00dev, 158, 0x21); + value = rt2800_bbp_read(rt2x00dev, 159); + bbptemp = bbptemp + (value << 16); + rt2800_bbp_write(rt2x00dev, 158, 0x20); + value = rt2800_bbp_read(rt2x00dev, 159); + bbptemp = bbptemp + (value << 8); + rt2800_bbp_write(rt2x00dev, 158, 0x1f); + value = rt2800_bbp_read(rt2x00dev, 159); + bbptemp = bbptemp + value; + + if (i < 2 && (bbptemp & 0x800000)) + result = (bbptemp & 0xffffff) - 0x1000000; + else if (i == 4) + result = bbptemp; + else + result = bbptemp; + + if (i == 0) + mi = result / 4096; + else if (i == 1) + mq = result / 4096; + else if (i == 2) + si = bbptemp / 4096; + else if (i == 3) + sq = bbptemp / 4096; + else + riq = result / 4096; + } + + bbpval1 = si - mi * mi; + rt2x00_dbg(rt2x00dev, + "RXIQ si=%d, sq=%d, riq=%d, bbpval %d, vga_idx %d", + si, sq, riq, bbpval1, vga_idx); + + if (bbpval1 >= (100 * 100)) + break; + + if (bbpval1 <= 100) + vga_idx = vga_idx + 9; + else if (bbpval1 <= 158) + vga_idx = vga_idx + 8; + else if (bbpval1 <= 251) + vga_idx = vga_idx + 7; + else if (bbpval1 <= 398) + vga_idx = vga_idx + 6; + else if (bbpval1 <= 630) + vga_idx = vga_idx + 5; + else if (bbpval1 <= 1000) + vga_idx = vga_idx + 4; + else if (bbpval1 <= 1584) + vga_idx = vga_idx + 3; + else if (bbpval1 <= 2511) + vga_idx = vga_idx + 2; + else + vga_idx = vga_idx + 1; + } + + sigma_i = rt2800_do_sqrt_accumulation(100 * (si - mi * mi)); + sigma_q = rt2800_do_sqrt_accumulation(100 * (sq - mq * mq)); + r_iq = 10 * (riq - (mi * mq)); + + rt2x00_dbg(rt2x00dev, "Sigma_i=%d, Sigma_q=%d, R_iq=%d", sigma_i, sigma_q, r_iq); + + if (sigma_i <= 1400 && sigma_i >= 1000 && + (sigma_i - sigma_q) <= 112 && + (sigma_i - sigma_q) >= -112 && + mi <= 32 && mi >= -32 && + mq <= 32 && mq >= -32) { + r_iq = 10 * (riq - (mi * mq)); + rt2x00_dbg(rt2x00dev, "RXIQ Sigma_i=%d, Sigma_q=%d, R_iq=%d\n", + sigma_i, sigma_q, r_iq); + + g_rx = (1000 * sigma_q) / sigma_i; + g_imb = ((-2) * 128 * (1000 - g_rx)) / (1000 + g_rx); + ph_rx = (r_iq * 2292) / (sigma_i * sigma_q); + + if (ph_rx > 20 || ph_rx < -20) { + ph_rx = 0; + rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL"); + } + + if (g_imb > 12 || g_imb < -12) { + g_imb = 0; + rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL"); + } + } else { + g_imb = 0; + ph_rx = 0; + rt2x00_dbg(rt2x00dev, "RXIQ Sigma_i=%d, Sigma_q=%d, R_iq=%d\n", + sigma_i, sigma_q, r_iq); + rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL"); + } + + if (ch_idx == 0) { + rt2800_bbp_write(rt2x00dev, 158, 0x37); + rt2800_bbp_write(rt2x00dev, 159, g_imb & 0x3f); + rt2800_bbp_write(rt2x00dev, 158, 0x35); + rt2800_bbp_write(rt2x00dev, 159, ph_rx & 0x3f); + } else { + rt2800_bbp_write(rt2x00dev, 158, 0x55); + rt2800_bbp_write(rt2x00dev, 159, g_imb & 0x3f); + rt2800_bbp_write(rt2x00dev, 158, 0x53); + rt2800_bbp_write(rt2x00dev, 159, ph_rx & 0x3f); + } + } + +restore_value: + rt2800_bbp_write(rt2x00dev, 158, 0x3); + bbpval = rt2800_bbp_read(rt2x00dev, 159); + rt2800_bbp_write(rt2x00dev, 159, (bbpval | 0x07)); + + rt2800_bbp_write(rt2x00dev, 158, 0x00); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + rt2800_bbp_write(rt2x00dev, 1, bbp1); + rt2800_bbp_write(rt2x00dev, 4, bbp4); + rt2800_bbp_write(rt2x00dev, 241, bbpr241); + rt2800_bbp_write(rt2x00dev, 242, bbpr242); + + rt2800_bbp_write(rt2x00dev, 244, 0x00); + bbpval = rt2800_bbp_read(rt2x00dev, 21); + bbpval |= 0x1; + rt2800_bbp_write(rt2x00dev, 21, bbpval); + usleep_range(10, 200); + bbpval &= 0xfe; + rt2800_bbp_write(rt2x00dev, 21, bbpval); + + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfb0r1); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfb0r2); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfb0r42); + + rt2800_rfcsr_write_bank(rt2x00dev, 4, 0, rfb4r0); + rt2800_rfcsr_write_bank(rt2x00dev, 4, 19, rfb4r19); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 3, rfb5r3); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, rfb5r4); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, rfb5r17); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, rfb5r18); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, rfb5r19); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, rfb5r20); + + rt2800_rfcsr_write_bank(rt2x00dev, 6, 0, rfb6r0); + rt2800_rfcsr_write_bank(rt2x00dev, 6, 19, rfb6r19); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 3, rfb7r3); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, rfb7r4); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 17, rfb7r17); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 18, rfb7r18); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 19, rfb7r19); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 20, rfb7r20); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000006); + udelay(1); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004); + udelay(1); + rt2800_register_write(rt2x00dev, RF_CONTROL0, orig_RF_CONTROL0); + udelay(1); + rt2800_register_write(rt2x00dev, RF_BYPASS0, orig_RF_BYPASS0); + rt2800_register_write(rt2x00dev, RF_CONTROL1, orig_RF_CONTROL1); + rt2800_register_write(rt2x00dev, RF_BYPASS1, orig_RF_BYPASS1); + rt2800_register_write(rt2x00dev, RF_CONTROL3, orig_RF_CONTROL3); + rt2800_register_write(rt2x00dev, RF_BYPASS3, orig_RF_BYPASS3); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl); +} + +static void rt2800_rf_configstore(struct rt2x00_dev *rt2x00dev, + struct rf_reg_pair rf_reg_record[][13], u8 chain) +{ + u8 rfvalue = 0; + + if (chain == CHAIN_0) { + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1); + rf_reg_record[CHAIN_0][0].bank = 0; + rf_reg_record[CHAIN_0][0].reg = 1; + rf_reg_record[CHAIN_0][0].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2); + rf_reg_record[CHAIN_0][1].bank = 0; + rf_reg_record[CHAIN_0][1].reg = 2; + rf_reg_record[CHAIN_0][1].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35); + rf_reg_record[CHAIN_0][2].bank = 0; + rf_reg_record[CHAIN_0][2].reg = 35; + rf_reg_record[CHAIN_0][2].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42); + rf_reg_record[CHAIN_0][3].bank = 0; + rf_reg_record[CHAIN_0][3].reg = 42; + rf_reg_record[CHAIN_0][3].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 0); + rf_reg_record[CHAIN_0][4].bank = 4; + rf_reg_record[CHAIN_0][4].reg = 0; + rf_reg_record[CHAIN_0][4].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 2); + rf_reg_record[CHAIN_0][5].bank = 4; + rf_reg_record[CHAIN_0][5].reg = 2; + rf_reg_record[CHAIN_0][5].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 34); + rf_reg_record[CHAIN_0][6].bank = 4; + rf_reg_record[CHAIN_0][6].reg = 34; + rf_reg_record[CHAIN_0][6].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 3); + rf_reg_record[CHAIN_0][7].bank = 5; + rf_reg_record[CHAIN_0][7].reg = 3; + rf_reg_record[CHAIN_0][7].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4); + rf_reg_record[CHAIN_0][8].bank = 5; + rf_reg_record[CHAIN_0][8].reg = 4; + rf_reg_record[CHAIN_0][8].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17); + rf_reg_record[CHAIN_0][9].bank = 5; + rf_reg_record[CHAIN_0][9].reg = 17; + rf_reg_record[CHAIN_0][9].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18); + rf_reg_record[CHAIN_0][10].bank = 5; + rf_reg_record[CHAIN_0][10].reg = 18; + rf_reg_record[CHAIN_0][10].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19); + rf_reg_record[CHAIN_0][11].bank = 5; + rf_reg_record[CHAIN_0][11].reg = 19; + rf_reg_record[CHAIN_0][11].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20); + rf_reg_record[CHAIN_0][12].bank = 5; + rf_reg_record[CHAIN_0][12].reg = 20; + rf_reg_record[CHAIN_0][12].value = rfvalue; + } else if (chain == CHAIN_1) { + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1); + rf_reg_record[CHAIN_1][0].bank = 0; + rf_reg_record[CHAIN_1][0].reg = 1; + rf_reg_record[CHAIN_1][0].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2); + rf_reg_record[CHAIN_1][1].bank = 0; + rf_reg_record[CHAIN_1][1].reg = 2; + rf_reg_record[CHAIN_1][1].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35); + rf_reg_record[CHAIN_1][2].bank = 0; + rf_reg_record[CHAIN_1][2].reg = 35; + rf_reg_record[CHAIN_1][2].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42); + rf_reg_record[CHAIN_1][3].bank = 0; + rf_reg_record[CHAIN_1][3].reg = 42; + rf_reg_record[CHAIN_1][3].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 0); + rf_reg_record[CHAIN_1][4].bank = 6; + rf_reg_record[CHAIN_1][4].reg = 0; + rf_reg_record[CHAIN_1][4].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 2); + rf_reg_record[CHAIN_1][5].bank = 6; + rf_reg_record[CHAIN_1][5].reg = 2; + rf_reg_record[CHAIN_1][5].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 34); + rf_reg_record[CHAIN_1][6].bank = 6; + rf_reg_record[CHAIN_1][6].reg = 34; + rf_reg_record[CHAIN_1][6].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 3); + rf_reg_record[CHAIN_1][7].bank = 7; + rf_reg_record[CHAIN_1][7].reg = 3; + rf_reg_record[CHAIN_1][7].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4); + rf_reg_record[CHAIN_1][8].bank = 7; + rf_reg_record[CHAIN_1][8].reg = 4; + rf_reg_record[CHAIN_1][8].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 17); + rf_reg_record[CHAIN_1][9].bank = 7; + rf_reg_record[CHAIN_1][9].reg = 17; + rf_reg_record[CHAIN_1][9].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 18); + rf_reg_record[CHAIN_1][10].bank = 7; + rf_reg_record[CHAIN_1][10].reg = 18; + rf_reg_record[CHAIN_1][10].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 19); + rf_reg_record[CHAIN_1][11].bank = 7; + rf_reg_record[CHAIN_1][11].reg = 19; + rf_reg_record[CHAIN_1][11].value = rfvalue; + rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 20); + rf_reg_record[CHAIN_1][12].bank = 7; + rf_reg_record[CHAIN_1][12].reg = 20; + rf_reg_record[CHAIN_1][12].value = rfvalue; + } else { + rt2x00_warn(rt2x00dev, "Unknown chain = %u\n", chain); + } +} + +static void rt2800_rf_configrecover(struct rt2x00_dev *rt2x00dev, + struct rf_reg_pair rf_record[][13]) +{ + u8 chain_index = 0, record_index = 0; + u8 bank = 0, rf_register = 0, value = 0; + + for (chain_index = 0; chain_index < 2; chain_index++) { + for (record_index = 0; record_index < 13; record_index++) { + bank = rf_record[chain_index][record_index].bank; + rf_register = rf_record[chain_index][record_index].reg; + value = rf_record[chain_index][record_index].value; + rt2800_rfcsr_write_bank(rt2x00dev, bank, rf_register, value); + rt2x00_dbg(rt2x00dev, "bank: %d, rf_register: %d, value: %x\n", + bank, rf_register, value); + } + } +} + +static void rt2800_setbbptonegenerator(struct rt2x00_dev *rt2x00dev) +{ + rt2800_bbp_write(rt2x00dev, 158, 0xAA); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + rt2800_bbp_write(rt2x00dev, 158, 0xAB); + rt2800_bbp_write(rt2x00dev, 159, 0x0A); + + rt2800_bbp_write(rt2x00dev, 158, 0xAC); + rt2800_bbp_write(rt2x00dev, 159, 0x3F); + + rt2800_bbp_write(rt2x00dev, 158, 0xAD); + rt2800_bbp_write(rt2x00dev, 159, 0x3F); + + rt2800_bbp_write(rt2x00dev, 244, 0x40); +} + +static u32 rt2800_do_fft_accumulation(struct rt2x00_dev *rt2x00dev, u8 tidx, u8 read_neg) +{ + u32 macvalue = 0; + int fftout_i = 0, fftout_q = 0; + u32 ptmp = 0, pint = 0; + u8 bbp = 0; + u8 tidxi; + + rt2800_bbp_write(rt2x00dev, 158, 0x00); + rt2800_bbp_write(rt2x00dev, 159, 0x9b); + + bbp = 0x9b; + + while (bbp == 0x9b) { + usleep_range(10, 50); + bbp = rt2800_bbp_read(rt2x00dev, 159); + bbp = bbp & 0xff; + } + + rt2800_bbp_write(rt2x00dev, 158, 0xba); + rt2800_bbp_write(rt2x00dev, 159, tidx); + rt2800_bbp_write(rt2x00dev, 159, tidx); + rt2800_bbp_write(rt2x00dev, 159, tidx); + + macvalue = rt2800_register_read(rt2x00dev, 0x057C); + + fftout_i = (macvalue >> 16); + fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i; + fftout_q = (macvalue & 0xffff); + fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q; + ptmp = (fftout_i * fftout_i); + ptmp = ptmp + (fftout_q * fftout_q); + pint = ptmp; + rt2x00_dbg(rt2x00dev, "I = %d, Q = %d, power = %x\n", fftout_i, fftout_q, pint); + if (read_neg) { + pint = pint >> 1; + tidxi = 0x40 - tidx; + tidxi = tidxi & 0x3f; + + rt2800_bbp_write(rt2x00dev, 158, 0xba); + rt2800_bbp_write(rt2x00dev, 159, tidxi); + rt2800_bbp_write(rt2x00dev, 159, tidxi); + rt2800_bbp_write(rt2x00dev, 159, tidxi); + + macvalue = rt2800_register_read(rt2x00dev, 0x057C); + + fftout_i = (macvalue >> 16); + fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i; + fftout_q = (macvalue & 0xffff); + fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q; + ptmp = (fftout_i * fftout_i); + ptmp = ptmp + (fftout_q * fftout_q); + ptmp = ptmp >> 1; + pint = pint + ptmp; + } + + return pint; +} + +static u32 rt2800_read_fft_accumulation(struct rt2x00_dev *rt2x00dev, u8 tidx) +{ + u32 macvalue = 0; + int fftout_i = 0, fftout_q = 0; + u32 ptmp = 0, pint = 0; + + rt2800_bbp_write(rt2x00dev, 158, 0xBA); + rt2800_bbp_write(rt2x00dev, 159, tidx); + rt2800_bbp_write(rt2x00dev, 159, tidx); + rt2800_bbp_write(rt2x00dev, 159, tidx); + + macvalue = rt2800_register_read(rt2x00dev, 0x057C); + + fftout_i = (macvalue >> 16); + fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i; + fftout_q = (macvalue & 0xffff); + fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q; + ptmp = (fftout_i * fftout_i); + ptmp = ptmp + (fftout_q * fftout_q); + pint = ptmp; + + return pint; +} + +static void rt2800_write_dc(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 alc, u8 iorq, u8 dc) +{ + u8 bbp = 0; + + rt2800_bbp_write(rt2x00dev, 158, 0xb0); + bbp = alc | 0x80; + rt2800_bbp_write(rt2x00dev, 159, bbp); + + if (ch_idx == 0) + bbp = (iorq == 0) ? 0xb1 : 0xb2; + else + bbp = (iorq == 0) ? 0xb8 : 0xb9; + + rt2800_bbp_write(rt2x00dev, 158, bbp); + bbp = dc; + rt2800_bbp_write(rt2x00dev, 159, bbp); +} + +static void rt2800_loft_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, + u8 alc_idx, u8 dc_result[][RF_ALC_NUM][2]) +{ + u32 p0 = 0, p1 = 0, pf = 0; + char idx0 = 0, idx1 = 0; + u8 idxf[] = {0x00, 0x00}; + u8 ibit = 0x20; + u8 iorq; + char bidx; + + rt2800_bbp_write(rt2x00dev, 158, 0xb0); + rt2800_bbp_write(rt2x00dev, 159, 0x80); + + for (bidx = 5; bidx >= 0; bidx--) { + for (iorq = 0; iorq <= 1; iorq++) { + if (idxf[iorq] == 0x20) { + idx0 = 0x20; + p0 = pf; + } else { + idx0 = idxf[iorq] - ibit; + idx0 = idx0 & 0x3F; + rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idx0); + p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0); + } + + idx1 = idxf[iorq] + (bidx == 5 ? 0 : ibit); + idx1 = idx1 & 0x3F; + rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idx1); + p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0); + + rt2x00_dbg(rt2x00dev, "alc=%u, IorQ=%u, idx_final=%2x\n", + alc_idx, iorq, idxf[iorq]); + rt2x00_dbg(rt2x00dev, "p0=%x, p1=%x, pf=%x, idx_0=%x, idx_1=%x, ibit=%x\n", + p0, p1, pf, idx0, idx1, ibit); + + if (bidx != 5 && pf <= p0 && pf < p1) { + idxf[iorq] = idxf[iorq]; + } else if (p0 < p1) { + pf = p0; + idxf[iorq] = idx0 & 0x3F; + } else { + pf = p1; + idxf[iorq] = idx1 & 0x3F; + } + rt2x00_dbg(rt2x00dev, "IorQ=%u, idx_final[%u]:%x, pf:%8x\n", + iorq, iorq, idxf[iorq], pf); + + rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idxf[iorq]); + } + ibit = ibit >> 1; + } + dc_result[ch_idx][alc_idx][0] = idxf[0]; + dc_result[ch_idx][alc_idx][1] = idxf[1]; +} + +static void rt2800_iq_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 *ges, u8 *pes) +{ + u32 p0 = 0, p1 = 0, pf = 0; + char perr = 0, gerr = 0, iq_err = 0; + char pef = 0, gef = 0; + char psta, pend; + char gsta, gend; + + u8 ibit = 0x20; + u8 first_search = 0x00, touch_neg_max = 0x00; + char idx0 = 0, idx1 = 0; + u8 gop; + u8 bbp = 0; + char bidx; + + for (bidx = 5; bidx >= 1; bidx--) { + for (gop = 0; gop < 2; gop++) { + if (gop == 1 || bidx < 4) { + if (gop == 0) + iq_err = gerr; + else + iq_err = perr; + + first_search = (gop == 0) ? (bidx == 3) : (bidx == 5); + touch_neg_max = (gop) ? ((iq_err & 0x0F) == 0x08) : + ((iq_err & 0x3F) == 0x20); + + if (touch_neg_max) { + p0 = pf; + idx0 = iq_err; + } else { + idx0 = iq_err - ibit; + bbp = (ch_idx == 0) ? ((gop == 0) ? 0x28 : 0x29) : + ((gop == 0) ? 0x46 : 0x47); + + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, idx0); + + p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1); + } + + idx1 = iq_err + (first_search ? 0 : ibit); + idx1 = (gop == 0) ? (idx1 & 0x0F) : (idx1 & 0x3F); + + bbp = (ch_idx == 0) ? (gop == 0) ? 0x28 : 0x29 : + (gop == 0) ? 0x46 : 0x47; + + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, idx1); + + p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1); + + rt2x00_dbg(rt2x00dev, + "p0=%x, p1=%x, pwer_final=%x, idx0=%x, idx1=%x, iq_err=%x, gop=%d, ibit=%x\n", + p0, p1, pf, idx0, idx1, iq_err, gop, ibit); + + if (!(!first_search && pf <= p0 && pf < p1)) { + if (p0 < p1) { + pf = p0; + iq_err = idx0; + } else { + pf = p1; + iq_err = idx1; + } + } + + bbp = (ch_idx == 0) ? (gop == 0) ? 0x28 : 0x29 : + (gop == 0) ? 0x46 : 0x47; + + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, iq_err); + + if (gop == 0) + gerr = iq_err; + else + perr = iq_err; + + rt2x00_dbg(rt2x00dev, "IQCalibration pf=%8x (%2x, %2x) !\n", + pf, gerr & 0x0F, perr & 0x3F); + } + } + + if (bidx > 0) + ibit = (ibit >> 1); + } + gerr = (gerr & 0x08) ? (gerr & 0x0F) - 0x10 : (gerr & 0x0F); + perr = (perr & 0x20) ? (perr & 0x3F) - 0x40 : (perr & 0x3F); + + gerr = (gerr < -0x07) ? -0x07 : (gerr > 0x05) ? 0x05 : gerr; + gsta = gerr - 1; + gend = gerr + 2; + + perr = (perr < -0x1f) ? -0x1f : (perr > 0x1d) ? 0x1d : perr; + psta = perr - 1; + pend = perr + 2; + + for (gef = gsta; gef <= gend; gef = gef + 1) + for (pef = psta; pef <= pend; pef = pef + 1) { + bbp = (ch_idx == 0) ? 0x28 : 0x46; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, gef & 0x0F); + + bbp = (ch_idx == 0) ? 0x29 : 0x47; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, pef & 0x3F); + + p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1); + if (gef == gsta && pef == psta) { + pf = p1; + gerr = gef; + perr = pef; + } else if (pf > p1) { + pf = p1; + gerr = gef; + perr = pef; + } + rt2x00_dbg(rt2x00dev, "Fine IQCalibration p1=%8x pf=%8x (%2x, %2x) !\n", + p1, pf, gef & 0x0F, pef & 0x3F); + } + + ges[ch_idx] = gerr & 0x0F; + pes[ch_idx] = perr & 0x3F; +} + +static void rt2800_rf_aux_tx0_loopback(struct rt2x00_dev *rt2x00dev) +{ + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x21); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, 0x10); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x1b); + rt2800_rfcsr_write_bank(rt2x00dev, 4, 0, 0x81); + rt2800_rfcsr_write_bank(rt2x00dev, 4, 2, 0x81); + rt2800_rfcsr_write_bank(rt2x00dev, 4, 34, 0xee); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 3, 0x2d); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, 0x2d); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, 0x80); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, 0xd7); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, 0xa2); + rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, 0x20); +} + +static void rt2800_rf_aux_tx1_loopback(struct rt2x00_dev *rt2x00dev) +{ + rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x22); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, 0x20); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x4b); + rt2800_rfcsr_write_bank(rt2x00dev, 6, 0, 0x81); + rt2800_rfcsr_write_bank(rt2x00dev, 6, 2, 0x81); + rt2800_rfcsr_write_bank(rt2x00dev, 6, 34, 0xee); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 3, 0x2d); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, 0x2d); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 17, 0x80); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 18, 0xd7); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 19, 0xa2); + rt2800_rfcsr_write_bank(rt2x00dev, 7, 20, 0x20); +} + +static void rt2800_loft_iq_calibration(struct rt2x00_dev *rt2x00dev) +{ + struct rf_reg_pair rf_store[CHAIN_NUM][13]; + u32 macorg1 = 0; + u32 macorg2 = 0; + u32 macorg3 = 0; + u32 macorg4 = 0; + u32 macorg5 = 0; + u32 orig528 = 0; + u32 orig52c = 0; + + u32 savemacsysctrl = 0; + u32 macvalue = 0; + u32 mac13b8 = 0; + u32 p0 = 0, p1 = 0; + u32 p0_idx10 = 0, p1_idx10 = 0; + + u8 rfvalue; + u8 loft_dc_search_result[CHAIN_NUM][RF_ALC_NUM][2]; + u8 ger[CHAIN_NUM], per[CHAIN_NUM]; + + u8 vga_gain[] = {14, 14}; + u8 bbp = 0, ch_idx = 0, rf_alc_idx = 0, idx = 0; + u8 bbpr30, rfb0r39, rfb0r42; + u8 bbpr1; + u8 bbpr4; + u8 bbpr241, bbpr242; + u8 count_step; + + static const u8 rf_gain[] = {0x00, 0x01, 0x02, 0x04, 0x08, 0x0c}; + static const u8 rfvga_gain_table[] = {0x24, 0x25, 0x26, 0x27, 0x28, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3F}; + static const u8 bbp_2324gain[] = {0x16, 0x14, 0x12, 0x10, 0x0c, 0x08}; + + savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macorg1 = rt2800_register_read(rt2x00dev, TX_PIN_CFG); + macorg2 = rt2800_register_read(rt2x00dev, RF_CONTROL0); + macorg3 = rt2800_register_read(rt2x00dev, RF_BYPASS0); + macorg4 = rt2800_register_read(rt2x00dev, RF_CONTROL3); + macorg5 = rt2800_register_read(rt2x00dev, RF_BYPASS3); + mac13b8 = rt2800_register_read(rt2x00dev, 0x13b8); + orig528 = rt2800_register_read(rt2x00dev, RF_CONTROL2); + orig52c = rt2800_register_read(rt2x00dev, RF_BYPASS2); + + macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macvalue &= (~0x04); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX))) + rt2x00_warn(rt2x00dev, "RF TX busy in LOFT IQ calibration\n"); + + macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macvalue &= (~0x08); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX))) + rt2x00_warn(rt2x00dev, "RF RX busy in LOFT IQ calibration\n"); + + for (ch_idx = 0; ch_idx < 2; ch_idx++) + rt2800_rf_configstore(rt2x00dev, rf_store, ch_idx); + + bbpr30 = rt2800_bbp_read(rt2x00dev, 30); + rfb0r39 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 39); + rfb0r42 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42); + + rt2800_bbp_write(rt2x00dev, 30, 0x1F); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 39, 0x80); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x5B); + + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + + rt2800_setbbptonegenerator(rt2x00dev); + + for (ch_idx = 0; ch_idx < 2; ch_idx++) { + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x00); + rt2800_register_write(rt2x00dev, TX_PIN_CFG, 0x0000000F); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003306); + rt2800_register_write(rt2x00dev, 0x13b8, 0x10); + udelay(1); + + if (ch_idx == 0) + rt2800_rf_aux_tx0_loopback(rt2x00dev); + else + rt2800_rf_aux_tx1_loopback(rt2x00dev); + + udelay(1); + + if (ch_idx == 0) + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001004); + else + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002004); + + rt2800_bbp_write(rt2x00dev, 158, 0x05); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + rt2800_bbp_write(rt2x00dev, 158, 0x01); + if (ch_idx == 0) + rt2800_bbp_write(rt2x00dev, 159, 0x00); + else + rt2800_bbp_write(rt2x00dev, 159, 0x01); + + vga_gain[ch_idx] = 18; + for (rf_alc_idx = 0; rf_alc_idx < 3; rf_alc_idx++) { + rt2800_bbp_write(rt2x00dev, 23, bbp_2324gain[rf_alc_idx]); + rt2800_bbp_write(rt2x00dev, 24, bbp_2324gain[rf_alc_idx]); + + macvalue = rt2800_register_read(rt2x00dev, RF_CONTROL3); + macvalue &= (~0x0000F1F1); + macvalue |= (rf_gain[rf_alc_idx] << 4); + macvalue |= (rf_gain[rf_alc_idx] << 12); + rt2800_register_write(rt2x00dev, RF_CONTROL3, macvalue); + macvalue = (0x0000F1F1); + rt2800_register_write(rt2x00dev, RF_BYPASS3, macvalue); + + if (rf_alc_idx == 0) { + rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x21); + for (; vga_gain[ch_idx] > 0; + vga_gain[ch_idx] = vga_gain[ch_idx] - 2) { + rfvalue = rfvga_gain_table[vga_gain[ch_idx]]; + rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue); + rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x00); + rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x00); + p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0); + rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x21); + p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0); + rt2x00_dbg(rt2x00dev, "LOFT AGC %d %d\n", p0, p1); + if ((p0 < 7000 * 7000) && (p1 < (7000 * 7000))) + break; + } + + rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x00); + rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x00); + + rt2x00_dbg(rt2x00dev, "Used VGA %d %x\n", vga_gain[ch_idx], + rfvga_gain_table[vga_gain[ch_idx]]); + + if (vga_gain[ch_idx] < 0) + vga_gain[ch_idx] = 0; + } + + rfvalue = rfvga_gain_table[vga_gain[ch_idx]]; + + rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue); + + rt2800_loft_search(rt2x00dev, ch_idx, rf_alc_idx, loft_dc_search_result); + } + } + + for (rf_alc_idx = 0; rf_alc_idx < 3; rf_alc_idx++) { + for (idx = 0; idx < 4; idx++) { + rt2800_bbp_write(rt2x00dev, 158, 0xB0); + bbp = (idx << 2) + rf_alc_idx; + rt2800_bbp_write(rt2x00dev, 159, bbp); + rt2x00_dbg(rt2x00dev, " ALC %2x,", bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0xb1); + bbp = loft_dc_search_result[CHAIN_0][rf_alc_idx][0x00]; + bbp = bbp & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + rt2x00_dbg(rt2x00dev, " I0 %2x,", bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0xb2); + bbp = loft_dc_search_result[CHAIN_0][rf_alc_idx][0x01]; + bbp = bbp & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + rt2x00_dbg(rt2x00dev, " Q0 %2x,", bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0xb8); + bbp = loft_dc_search_result[CHAIN_1][rf_alc_idx][0x00]; + bbp = bbp & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + rt2x00_dbg(rt2x00dev, " I1 %2x,", bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0xb9); + bbp = loft_dc_search_result[CHAIN_1][rf_alc_idx][0x01]; + bbp = bbp & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + rt2x00_dbg(rt2x00dev, " Q1 %2x\n", bbp); + } + } + + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04); + + rt2800_bbp_write(rt2x00dev, 158, 0x00); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + bbp = 0x00; + rt2800_bbp_write(rt2x00dev, 244, 0x00); + + rt2800_bbp_write(rt2x00dev, 21, 0x01); + udelay(1); + rt2800_bbp_write(rt2x00dev, 21, 0x00); + + rt2800_rf_configrecover(rt2x00dev, rf_store); + + rt2800_register_write(rt2x00dev, TX_PIN_CFG, macorg1); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00); + rt2800_register_write(rt2x00dev, RF_CONTROL0, macorg2); + udelay(1); + rt2800_register_write(rt2x00dev, RF_BYPASS0, macorg3); + rt2800_register_write(rt2x00dev, RF_CONTROL3, macorg4); + rt2800_register_write(rt2x00dev, RF_BYPASS3, macorg5); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl); + rt2800_register_write(rt2x00dev, RF_CONTROL2, orig528); + rt2800_register_write(rt2x00dev, RF_BYPASS2, orig52c); + rt2800_register_write(rt2x00dev, 0x13b8, mac13b8); + + savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macorg1 = rt2800_register_read(rt2x00dev, TX_PIN_CFG); + macorg2 = rt2800_register_read(rt2x00dev, RF_CONTROL0); + macorg3 = rt2800_register_read(rt2x00dev, RF_BYPASS0); + macorg4 = rt2800_register_read(rt2x00dev, RF_CONTROL3); + macorg5 = rt2800_register_read(rt2x00dev, RF_BYPASS3); + + bbpr1 = rt2800_bbp_read(rt2x00dev, 1); + bbpr4 = rt2800_bbp_read(rt2x00dev, 4); + bbpr241 = rt2800_bbp_read(rt2x00dev, 241); + bbpr242 = rt2800_bbp_read(rt2x00dev, 242); + mac13b8 = rt2800_register_read(rt2x00dev, 0x13b8); + + macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macvalue &= (~0x04); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX))) + rt2x00_warn(rt2x00dev, "RF TX busy in LOFT IQ calibration\n"); + + macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL); + macvalue &= (~0x08); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue); + + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX))) + rt2x00_warn(rt2x00dev, "RF RX busy in LOFT IQ calibration\n"); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x00000101); + rt2800_register_write(rt2x00dev, RF_BYPASS3, 0x0000F1F1); + } + + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_bbp_write(rt2x00dev, 4, bbpr4 & (~0x18)); + rt2800_bbp_write(rt2x00dev, 21, 0x01); + udelay(1); + rt2800_bbp_write(rt2x00dev, 21, 0x00); + + rt2800_bbp_write(rt2x00dev, 241, 0x14); + rt2800_bbp_write(rt2x00dev, 242, 0x80); + rt2800_bbp_write(rt2x00dev, 244, 0x31); + } else { + rt2800_setbbptonegenerator(rt2x00dev); + } + + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003306); + udelay(1); + + rt2800_register_write(rt2x00dev, TX_PIN_CFG, 0x0000000F); + + if (!test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x00000000); + rt2800_register_write(rt2x00dev, RF_BYPASS3, 0x0000F1F1); + } + + rt2800_register_write(rt2x00dev, 0x13b8, 0x00000010); + + for (ch_idx = 0; ch_idx < 2; ch_idx++) + rt2800_rf_configstore(rt2x00dev, rf_store, ch_idx); + + rt2800_rfcsr_write_dccal(rt2x00dev, 3, 0x3B); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x3B); + + rt2800_bbp_write(rt2x00dev, 158, 0x03); + rt2800_bbp_write(rt2x00dev, 159, 0x60); + rt2800_bbp_write(rt2x00dev, 158, 0xB0); + rt2800_bbp_write(rt2x00dev, 159, 0x80); + + for (ch_idx = 0; ch_idx < 2; ch_idx++) { + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + + if (ch_idx == 0) { + rt2800_bbp_write(rt2x00dev, 158, 0x01); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + bbp = bbpr1 & (~0x18); + bbp = bbp | 0x00; + rt2800_bbp_write(rt2x00dev, 1, bbp); + } + rt2800_rf_aux_tx0_loopback(rt2x00dev); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001004); + } else { + rt2800_bbp_write(rt2x00dev, 158, 0x01); + rt2800_bbp_write(rt2x00dev, 159, 0x01); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX1, &rt2x00dev->cap_flags)) { + bbp = bbpr1 & (~0x18); + bbp = bbp | 0x08; + rt2800_bbp_write(rt2x00dev, 1, bbp); + } + rt2800_rf_aux_tx1_loopback(rt2x00dev); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002004); + } + + rt2800_bbp_write(rt2x00dev, 158, 0x05); + rt2800_bbp_write(rt2x00dev, 159, 0x04); + + bbp = (ch_idx == 0) ? 0x28 : 0x46; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_bbp_write(rt2x00dev, 23, 0x06); + rt2800_bbp_write(rt2x00dev, 24, 0x06); + count_step = 1; + } else { + rt2800_bbp_write(rt2x00dev, 23, 0x1F); + rt2800_bbp_write(rt2x00dev, 24, 0x1F); + count_step = 2; + } + + for (; vga_gain[ch_idx] < 19; vga_gain[ch_idx] = (vga_gain[ch_idx] + count_step)) { + rfvalue = rfvga_gain_table[vga_gain[ch_idx]]; + rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue); + rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue); + + bbp = (ch_idx == 0) ? 0x29 : 0x47; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 0); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) + p0_idx10 = rt2800_read_fft_accumulation(rt2x00dev, 0x0A); + + bbp = (ch_idx == 0) ? 0x29 : 0x47; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, 0x21); + p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 0); + if (test_bit(CAPABILITY_EXTERNAL_PA_TX1, &rt2x00dev->cap_flags)) + p1_idx10 = rt2800_read_fft_accumulation(rt2x00dev, 0x0A); + + rt2x00_dbg(rt2x00dev, "IQ AGC %d %d\n", p0, p1); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2x00_dbg(rt2x00dev, "IQ AGC IDX 10 %d %d\n", p0_idx10, p1_idx10); + if ((p0_idx10 > 7000 * 7000) || (p1_idx10 > 7000 * 7000)) { + if (vga_gain[ch_idx] != 0) + vga_gain[ch_idx] = vga_gain[ch_idx] - 1; + break; + } + } + + if ((p0 > 2500 * 2500) || (p1 > 2500 * 2500)) + break; + } + + if (vga_gain[ch_idx] > 18) + vga_gain[ch_idx] = 18; + rt2x00_dbg(rt2x00dev, "Used VGA %d %x\n", vga_gain[ch_idx], + rfvga_gain_table[vga_gain[ch_idx]]); + + bbp = (ch_idx == 0) ? 0x29 : 0x47; + rt2800_bbp_write(rt2x00dev, 158, bbp); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + rt2800_iq_search(rt2x00dev, ch_idx, ger, per); + } + + rt2800_bbp_write(rt2x00dev, 23, 0x00); + rt2800_bbp_write(rt2x00dev, 24, 0x00); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04); + + rt2800_bbp_write(rt2x00dev, 158, 0x28); + bbp = ger[CHAIN_0] & 0x0F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0x29); + bbp = per[CHAIN_0] & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0x46); + bbp = ger[CHAIN_1] & 0x0F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + + rt2800_bbp_write(rt2x00dev, 158, 0x47); + bbp = per[CHAIN_1] & 0x3F; + rt2800_bbp_write(rt2x00dev, 159, bbp); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) { + rt2800_bbp_write(rt2x00dev, 1, bbpr1); + rt2800_bbp_write(rt2x00dev, 241, bbpr241); + rt2800_bbp_write(rt2x00dev, 242, bbpr242); + } + rt2800_bbp_write(rt2x00dev, 244, 0x00); + + rt2800_bbp_write(rt2x00dev, 158, 0x00); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + rt2800_bbp_write(rt2x00dev, 158, 0xB0); + rt2800_bbp_write(rt2x00dev, 159, 0x00); + + rt2800_bbp_write(rt2x00dev, 30, bbpr30); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 39, rfb0r39); + rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfb0r42); + + if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) + rt2800_bbp_write(rt2x00dev, 4, bbpr4); + + rt2800_bbp_write(rt2x00dev, 21, 0x01); + udelay(1); + rt2800_bbp_write(rt2x00dev, 21, 0x00); + + rt2800_rf_configrecover(rt2x00dev, rf_store); + + rt2800_register_write(rt2x00dev, TX_PIN_CFG, macorg1); + rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00); + rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00); + rt2800_register_write(rt2x00dev, RF_CONTROL0, macorg2); + udelay(1); + rt2800_register_write(rt2x00dev, RF_BYPASS0, macorg3); + rt2800_register_write(rt2x00dev, RF_CONTROL3, macorg4); + rt2800_register_write(rt2x00dev, RF_BYPASS3, macorg5); + rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl); + rt2800_register_write(rt2x00dev, 0x13b8, mac13b8); +} + static void rt2800_bbp_core_soft_reset(struct rt2x00_dev *rt2x00dev, bool set_bw, bool is_ht40) { @@ -9005,8 +10582,13 @@ static void rt2800_init_rfcsr_6352(struct rt2x00_dev *rt2x00dev) rt2800_rfcsr_write_dccal(rt2x00dev, 5, 0x00); rt2800_rfcsr_write_dccal(rt2x00dev, 17, 0x7C); + rt2800_r_calibration(rt2x00dev); + rt2800_rf_self_txdc_cal(rt2x00dev); + rt2800_rxdcoc_calibration(rt2x00dev); rt2800_bw_filter_calibration(rt2x00dev, true); rt2800_bw_filter_calibration(rt2x00dev, false); + rt2800_loft_iq_calibration(rt2x00dev); + rt2800_rxiq_calibration(rt2x00dev); } static void rt2800_init_rfcsr(struct rt2x00_dev *rt2x00dev) @@ -9073,7 +10655,7 @@ int rt2800_enable_radio(struct rt2x00_dev *rt2x00dev) /* * Wait BBP/RF to wake up. */ - if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev))) + if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY))) return -EIO; /* @@ -9435,6 +11017,8 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev) rf = RF3853; else if (rt2x00_rt(rt2x00dev, RT5350)) rf = RF5350; + else if (rt2x00_rt(rt2x00dev, RT5592)) + rf = RF5592; else rf = rt2x00_get_field16(eeprom, EEPROM_NIC_CONF0_RF_TYPE); @@ -9564,7 +11148,8 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev) */ eeprom = rt2800_eeprom_read(rt2x00dev, EEPROM_NIC_CONF1); - if (rt2x00_rt(rt2x00dev, RT3352)) { + if (rt2x00_rt(rt2x00dev, RT3352) || + rt2x00_rt(rt2x00dev, RT6352)) { if (rt2x00_get_field16(eeprom, EEPROM_NIC_CONF1_EXTERNAL_TX0_PA_3352)) __set_bit(CAPABILITY_EXTERNAL_PA_TX0, @@ -9575,6 +11160,18 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev) &rt2x00dev->cap_flags); } + eeprom = rt2800_eeprom_read(rt2x00dev, EEPROM_NIC_CONF2); + + if (rt2x00_rt(rt2x00dev, RT6352) && eeprom != 0 && eeprom != 0xffff) { + if (!rt2x00_get_field16(eeprom, + EEPROM_NIC_CONF2_EXTERNAL_PA)) { + __clear_bit(CAPABILITY_EXTERNAL_PA_TX0, + &rt2x00dev->cap_flags); + __clear_bit(CAPABILITY_EXTERNAL_PA_TX1, + &rt2x00dev->cap_flags); + } + } + return 0; } diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h index e1761f467b9465df799e06e67b23d4f7f14ad69c..3cbef77b4bd3062e1d34c6ecfba58c11b2a2fd0d 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h @@ -17,6 +17,16 @@ #define WCID_START 33 #define WCID_END 222 #define STA_IDS_SIZE (WCID_END - WCID_START + 2) +#define CHAIN_0 0x0 +#define CHAIN_1 0x1 +#define RF_ALC_NUM 6 +#define CHAIN_NUM 2 + +struct rf_reg_pair { + u8 bank; + u8 reg; + u8 value; +}; /* RT2800 driver data structure */ struct rt2800_drv_data { diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h index 8f5772b98f58c803165ec59b3af5758010ac787f..07a6a5a9ce13ae1795f6180893700187e04b3aba 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h @@ -1309,8 +1309,11 @@ void rt2x00queue_unmap_skb(struct queue_entry *entry); */ static inline struct data_queue * rt2x00queue_get_tx_queue(struct rt2x00_dev *rt2x00dev, - const enum data_queue_qid queue) + enum data_queue_qid queue) { + if (queue >= rt2x00dev->ops->tx_queues && queue < IEEE80211_NUM_ACS) + queue = rt2x00dev->ops->tx_queues - 1; + if (queue < rt2x00dev->ops->tx_queues && rt2x00dev->tx) return &rt2x00dev->tx[queue]; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c index e95c101c271113828b918f6896db6c9ed9bfca7e..3a035afcf7f99c6191fd8862531703714c659e8f 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c @@ -1093,6 +1093,19 @@ static void rt2x00lib_remove_hw(struct rt2x00_dev *rt2x00dev) kfree(rt2x00dev->spec.channels_info); } +static const struct ieee80211_tpt_blink rt2x00_tpt_blink[] = { + { .throughput = 0 * 1024, .blink_time = 334 }, + { .throughput = 1 * 1024, .blink_time = 260 }, + { .throughput = 2 * 1024, .blink_time = 220 }, + { .throughput = 5 * 1024, .blink_time = 190 }, + { .throughput = 10 * 1024, .blink_time = 170 }, + { .throughput = 25 * 1024, .blink_time = 150 }, + { .throughput = 54 * 1024, .blink_time = 130 }, + { .throughput = 120 * 1024, .blink_time = 110 }, + { .throughput = 265 * 1024, .blink_time = 80 }, + { .throughput = 586 * 1024, .blink_time = 50 }, +}; + static int rt2x00lib_probe_hw(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; @@ -1174,6 +1187,11 @@ static int rt2x00lib_probe_hw(struct rt2x00_dev *rt2x00dev) #undef RT2X00_TASKLET_INIT + ieee80211_create_tpt_led_trigger(rt2x00dev->hw, + IEEE80211_TPT_LEDTRIG_FL_RADIO, + rt2x00_tpt_blink, + ARRAY_SIZE(rt2x00_tpt_blink)); + /* * Register HW. */ diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c index 4d06038afd837f9904792dfe2e86e3189e8a1e01..98df0aef816850fe8ccb290eb7ed63ffb312b1d0 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c @@ -318,7 +318,7 @@ static void rt2x00queue_create_tx_descriptor_ht(struct rt2x00_dev *rt2x00dev, * when using more then one tx stream (>MCS7). */ if (sta && txdesc->u.ht.mcs > 7 && - sta->smps_mode == IEEE80211_SMPS_DYNAMIC) + sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) __set_bit(ENTRY_TXD_HT_MIMO_PS, &txdesc->flags); } else { txdesc->u.ht.mcs = rt2x00_get_rate_mcs(hwrate->mcs); diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c index 49421d10e22bc46a80f9b4d6b932cc1700ed1c77..f7d95c9624a014a02ca0a510c2e29a626bf6f9fe 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c @@ -143,7 +143,7 @@ static int rtl8187_register_led(struct ieee80211_hw *dev, led->dev = dev; led->ledpin = ledpin; led->is_radio = is_radio; - strlcpy(led->name, name, sizeof(led->name)); + strscpy(led->name, name, sizeof(led->name)); led->led_dev.name = led->name; led->led_dev.default_trigger = default_trigger; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h index 7ddce3c3f0c486c63467bdab27e95b7c4d1f2a6e..782b089a2e1ba9ad2710858be2f1b8a92fc9ff56 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h @@ -1425,7 +1425,7 @@ struct rtl8xxxu_fileops { void (*set_tx_power) (struct rtl8xxxu_priv *priv, int channel, bool ht40); void (*update_rate_mask) (struct rtl8xxxu_priv *priv, - u32 ramask, u8 rateid, int sgi); + u32 ramask, u8 rateid, int sgi, int txbw_40mhz); void (*report_connect) (struct rtl8xxxu_priv *priv, u8 macid, bool connect); void (*fill_txdesc) (struct ieee80211_hw *hw, struct ieee80211_hdr *hdr, @@ -1511,9 +1511,9 @@ void rtl8xxxu_gen2_config_channel(struct ieee80211_hw *hw); void rtl8xxxu_gen1_usb_quirks(struct rtl8xxxu_priv *priv); void rtl8xxxu_gen2_usb_quirks(struct rtl8xxxu_priv *priv); void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv, - u32 ramask, u8 rateid, int sgi); + u32 ramask, u8 rateid, int sgi, int txbw_40mhz); void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv, - u32 ramask, u8 rateid, int sgi); + u32 ramask, u8 rateid, int sgi, int txbw_40mhz); void rtl8xxxu_gen1_report_connect(struct rtl8xxxu_priv *priv, u8 macid, bool connect); void rtl8xxxu_gen2_report_connect(struct rtl8xxxu_priv *priv, diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index c66f0726b2535c6321c7815bc9eedecd6604b40a..ac641a56efb0988afe057e9d75aa61498fb387cf 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -1878,13 +1878,6 @@ static int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv) /* We have 8 bits to indicate validity */ map_addr = offset * 8; - if (map_addr >= EFUSE_MAP_LEN) { - dev_warn(dev, "%s: Illegal map_addr (%04x), " - "efuse corrupt!\n", - __func__, map_addr); - ret = -EINVAL; - goto exit; - } for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) { /* Check word enable condition in the section */ if (word_mask & BIT(i)) { @@ -1895,6 +1888,13 @@ static int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv) ret = rtl8xxxu_read_efuse8(priv, efuse_addr++, &val8); if (ret) goto exit; + if (map_addr >= EFUSE_MAP_LEN - 1) { + dev_warn(dev, "%s: Illegal map_addr (%04x), " + "efuse corrupt!\n", + __func__, map_addr); + ret = -EINVAL; + goto exit; + } priv->efuse_wifi.raw[map_addr++] = val8; ret = rtl8xxxu_read_efuse8(priv, efuse_addr++, &val8); @@ -2929,12 +2929,12 @@ bool rtl8xxxu_gen2_simularity_compare(struct rtl8xxxu_priv *priv, } if (!(simubitmap & 0x30) && priv->tx_paths > 1) { - /* path B RX OK */ + /* path B TX OK */ for (i = 4; i < 6; i++) result[3][i] = result[c1][i]; } - if (!(simubitmap & 0x30) && priv->tx_paths > 1) { + if (!(simubitmap & 0xc0) && priv->tx_paths > 1) { /* path B RX OK */ for (i = 6; i < 8; i++) result[3][i] = result[c1][i]; @@ -4320,7 +4320,7 @@ static void rtl8xxxu_sw_scan_complete(struct ieee80211_hw *hw, } void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv, - u32 ramask, u8 rateid, int sgi) + u32 ramask, u8 rateid, int sgi, int txbw_40mhz) { struct h2c_cmd h2c; @@ -4340,10 +4340,15 @@ void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv, } void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv, - u32 ramask, u8 rateid, int sgi) + u32 ramask, u8 rateid, int sgi, int txbw_40mhz) { struct h2c_cmd h2c; - u8 bw = RTL8XXXU_CHANNEL_WIDTH_20; + u8 bw; + + if (txbw_40mhz) + bw = RTL8XXXU_CHANNEL_WIDTH_40; + else + bw = RTL8XXXU_CHANNEL_WIDTH_20; memset(&h2c, 0, sizeof(struct h2c_cmd)); @@ -4353,15 +4358,14 @@ void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv, h2c.b_macid_cfg.ramask2 = (ramask >> 16) & 0xff; h2c.b_macid_cfg.ramask3 = (ramask >> 24) & 0xff; - h2c.ramask.arg = 0x80; h2c.b_macid_cfg.data1 = rateid; if (sgi) h2c.b_macid_cfg.data1 |= BIT(7); h2c.b_macid_cfg.data2 = bw; - dev_dbg(&priv->udev->dev, "%s: rate mask %08x, arg %02x, size %zi\n", - __func__, ramask, h2c.ramask.arg, sizeof(h2c.b_macid_cfg)); + dev_dbg(&priv->udev->dev, "%s: rate mask %08x, rateid %02x, sgi %d, size %zi\n", + __func__, ramask, rateid, sgi, sizeof(h2c.b_macid_cfg)); rtl8xxxu_gen2_h2c_cmd(priv, &h2c, sizeof(h2c.b_macid_cfg)); } @@ -4556,6 +4560,53 @@ rtl8xxxu_wireless_mode(struct ieee80211_hw *hw, struct ieee80211_sta *sta) return network_type; } +static void rtl8xxxu_set_aifs(struct rtl8xxxu_priv *priv, u8 slot_time) +{ + u32 reg_edca_param[IEEE80211_NUM_ACS] = { + [IEEE80211_AC_VO] = REG_EDCA_VO_PARAM, + [IEEE80211_AC_VI] = REG_EDCA_VI_PARAM, + [IEEE80211_AC_BE] = REG_EDCA_BE_PARAM, + [IEEE80211_AC_BK] = REG_EDCA_BK_PARAM, + }; + u32 val32; + u16 wireless_mode = 0; + u8 aifs, aifsn, sifs; + int i; + + if (priv->vif) { + struct ieee80211_sta *sta; + + rcu_read_lock(); + sta = ieee80211_find_sta(priv->vif, priv->vif->bss_conf.bssid); + if (sta) + wireless_mode = rtl8xxxu_wireless_mode(priv->hw, sta); + rcu_read_unlock(); + } + + if (priv->hw->conf.chandef.chan->band == NL80211_BAND_5GHZ || + (wireless_mode & WIRELESS_MODE_N_24G)) + sifs = 16; + else + sifs = 10; + + for (i = 0; i < IEEE80211_NUM_ACS; i++) { + val32 = rtl8xxxu_read32(priv, reg_edca_param[i]); + + /* It was set in conf_tx. */ + aifsn = val32 & 0xff; + + /* aifsn not set yet or already fixed */ + if (aifsn < 2 || aifsn > 15) + continue; + + aifs = aifsn * slot_time + sifs; + + val32 &= ~0xff; + val32 |= aifs; + rtl8xxxu_write32(priv, reg_edca_param[i], val32); + } +} + static void rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_bss_conf *bss_conf, u64 changed) @@ -4622,7 +4673,11 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, RATE_INFO_FLAGS_SHORT_GI; } - rarpt->txrate.bw |= RATE_INFO_BW_20; + if (rtl8xxxu_ht40_2g && + (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40)) + rarpt->txrate.bw = RATE_INFO_BW_40; + else + rarpt->txrate.bw = RATE_INFO_BW_20; } bit_rate = cfg80211_calculate_bitrate(&rarpt->txrate); rarpt->bit_rate = bit_rate; @@ -4631,7 +4686,7 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, priv->vif = vif; priv->rssi_level = RTL8XXXU_RATR_STA_INIT; - priv->fops->update_rate_mask(priv, ramask, 0, sgi); + priv->fops->update_rate_mask(priv, ramask, 0, sgi, rarpt->txrate.bw == RATE_INFO_BW_40); rtl8xxxu_write8(priv, REG_BCN_MAX_ERR, 0xff); @@ -4671,6 +4726,8 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif, else val8 = 20; rtl8xxxu_write8(priv, REG_SLOT, val8); + + rtl8xxxu_set_aifs(priv, val8); } if (changed & BSS_CHANGED_BSSID) { @@ -4710,9 +4767,8 @@ static u32 rtl8xxxu_80211_to_rtl_queue(u32 queue) return rtlqueue; } -static u32 rtl8xxxu_queue_select(struct ieee80211_hw *hw, struct sk_buff *skb) +static u32 rtl8xxxu_queue_select(struct ieee80211_hdr *hdr, struct sk_buff *skb) { - struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; u32 queue; if (ieee80211_is_mgmt(hdr->frame_control)) @@ -5062,6 +5118,8 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw, if (control && control->sta) sta = control->sta; + queue = rtl8xxxu_queue_select(hdr, skb); + tx_desc = skb_push(skb, tx_desc_size); memset(tx_desc, 0, tx_desc_size); @@ -5074,7 +5132,6 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw, is_broadcast_ether_addr(ieee80211_get_DA(hdr))) tx_desc->txdw0 |= TXDESC_BROADMULTICAST; - queue = rtl8xxxu_queue_select(hw, skb); tx_desc->txdw1 = cpu_to_le32(queue << TXDESC_QUEUE_SHIFT); if (tx_info->control.hw_key) { @@ -6344,7 +6401,7 @@ static void rtl8xxxu_refresh_rate_mask(struct rtl8xxxu_priv *priv, } priv->rssi_level = rssi_level; - priv->fops->update_rate_mask(priv, rate_bitmap, ratr_idx, sgi); + priv->fops->update_rate_mask(priv, rate_bitmap, ratr_idx, sgi, txbw_40mhz); } } @@ -6657,7 +6714,6 @@ static int rtl8xxxu_probe(struct usb_interface *interface, hw = ieee80211_alloc_hw(sizeof(struct rtl8xxxu_priv), &rtl8xxxu_ops); if (!hw) { ret = -ENOMEM; - priv = NULL; goto err_put_dev; } @@ -6768,11 +6824,9 @@ static int rtl8xxxu_probe(struct usb_interface *interface, err_set_intfdata: usb_set_intfdata(interface, NULL); - if (priv) { - kfree(priv->fw_data); - mutex_destroy(&priv->usb_buf_mutex); - mutex_destroy(&priv->h2c_mutex); - } + kfree(priv->fw_data); + mutex_destroy(&priv->usb_buf_mutex); + mutex_destroy(&priv->h2c_mutex); ieee80211_free_hw(hw); err_put_dev: diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c index 15e6a6aded319921f890a57edc7be6f5e790951d..d18c092b6142636471271a3cb2f9031cea18794b 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c @@ -2386,11 +2386,10 @@ void rtl92d_phy_reload_iqk_setting(struct ieee80211_hw *hw, u8 channel) rtl_dbg(rtlpriv, COMP_SCAN, DBG_LOUD, "Just Read IQK Matrix reg for channel:%d....\n", channel); - _rtl92d_phy_patha_fill_iqk_matrix(hw, true, - rtlphy->iqk_matrix[ - indexforchannel].value, 0, - (rtlphy->iqk_matrix[ - indexforchannel].value[0][2] == 0)); + if (rtlphy->iqk_matrix[indexforchannel].value[0][0] != 0) + _rtl92d_phy_patha_fill_iqk_matrix(hw, true, + rtlphy->iqk_matrix[indexforchannel].value, 0, + rtlphy->iqk_matrix[indexforchannel].value[0][2] == 0); if (IS_92D_SINGLEPHY(rtlhal->version)) { if ((rtlphy->iqk_matrix[ indexforchannel].value[0][4] != 0) diff --git a/drivers/net/wireless/realtek/rtw88/bf.c b/drivers/net/wireless/realtek/rtw88/bf.c index 76c7f3257dd36d412952d93049ae7cf8887a4428..038a30b170eff2254934638708ca331a2b268db8 100644 --- a/drivers/net/wireless/realtek/rtw88/bf.c +++ b/drivers/net/wireless/realtek/rtw88/bf.c @@ -30,11 +30,11 @@ void rtw_bf_disassoc(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, void rtw_bf_assoc(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_bss_conf *bss_conf) { + const struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_hw *hw = rtwdev->hw; struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv; struct rtw_bfee *bfee = &rtwvif->bfee; struct rtw_bf_info *bfinfo = &rtwdev->bf_info; - struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_sta *sta; struct ieee80211_sta_vht_cap *vht_cap; struct ieee80211_sta_vht_cap *ic_vht_cap; diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c index cac053f485c3b09974c766abdfce93a991f5e101..6276ad6242991ae15ece36b1b2cd42940b5193ad 100644 --- a/drivers/net/wireless/realtek/rtw88/coex.c +++ b/drivers/net/wireless/realtek/rtw88/coex.c @@ -13,7 +13,7 @@ static u8 rtw_coex_next_rssi_state(struct rtw_dev *rtwdev, u8 pre_state, u8 rssi, u8 rssi_thresh) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 tol = chip->rssi_tolerance; u8 next_state; @@ -36,7 +36,7 @@ static u8 rtw_coex_next_rssi_state(struct rtw_dev *rtwdev, u8 pre_state, static void rtw_coex_limited_tx(struct rtw_dev *rtwdev, bool tx_limit_en, bool ampdu_limit_en) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; u8 num_of_active_port = 1; @@ -365,7 +365,7 @@ static void rtw_coex_set_wl_pri_mask(struct rtw_dev *rtwdev, u8 bitmap, void rtw_coex_write_scbd(struct rtw_dev *rtwdev, u16 bitpos, bool set) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; u16 val = 0x2; @@ -400,7 +400,7 @@ EXPORT_SYMBOL(rtw_coex_write_scbd); static u16 rtw_coex_read_scbd(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (!chip->scbd_support) return 0; @@ -410,7 +410,7 @@ static u16 rtw_coex_read_scbd(struct rtw_dev *rtwdev) static void rtw_coex_check_rfk(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_rfe *coex_rfe = &coex->rfe; @@ -489,7 +489,7 @@ static void rtw_coex_monitor_bt_ctr(struct rtw_dev *rtwdev) static void rtw_coex_monitor_bt_enable(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; @@ -524,10 +524,10 @@ static void rtw_coex_monitor_bt_enable(struct rtw_dev *rtwdev) static void rtw_coex_update_wl_link_info(struct rtw_dev *rtwdev, u8 reason) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_traffic_stats *stats = &rtwdev->stats; bool is_5G = false; bool wl_busy = false; @@ -706,10 +706,10 @@ static const char *rtw_coex_get_bt_status_string(u8 bt_status) static void rtw_coex_update_bt_link_info(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; - struct rtw_chip_info *chip = rtwdev->chip; u8 i; u8 rssi_state; u8 rssi_step; @@ -806,7 +806,7 @@ static void rtw_coex_update_bt_link_info(struct rtw_dev *rtwdev) static void rtw_coex_update_wl_ch_info(struct rtw_dev *rtwdev, u8 type) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; struct rtw_coex_dm *coex_dm = &rtwdev->coex.dm; struct rtw_coex_stat *coex_stat = &rtwdev->coex.stat; @@ -933,7 +933,7 @@ EXPORT_SYMBOL(rtw_coex_write_indirect_reg); static void rtw_coex_coex_ctrl_owner(struct rtw_dev *rtwdev, bool wifi_control) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_hw_reg *btg_reg = chip->btg_reg; if (wifi_control) { @@ -981,7 +981,7 @@ static void rtw_coex_mimo_ps(struct rtw_dev *rtwdev, bool force, bool state) static void rtw_btc_wltoggle_table_a(struct rtw_dev *rtwdev, bool force, u8 table_case) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; u8 h2c_para[6] = {0}; u32 table_wl = 0x5a5a5a5a; @@ -1065,9 +1065,9 @@ static void rtw_coex_set_table(struct rtw_dev *rtwdev, bool force, u32 table0, static void rtw_coex_table(struct rtw_dev *rtwdev, bool force, u8 type) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_dm *coex_dm = &coex->dm; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; struct rtw_coex_stat *coex_stat = &coex->stat; @@ -1135,9 +1135,9 @@ static void rtw_coex_power_save_state(struct rtw_dev *rtwdev, u8 ps_type, static void rtw_coex_set_tdma(struct rtw_dev *rtwdev, u8 byte1, u8 byte2, u8 byte3, u8 byte4, u8 byte5) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_dm *coex_dm = &coex->dm; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_stat *coex_stat = &coex->stat; u8 ps_type = COEX_PS_WIFI_NATIVE; bool ap_enable = false; @@ -1193,10 +1193,10 @@ static void rtw_coex_set_tdma(struct rtw_dev *rtwdev, u8 byte1, u8 byte2, static void rtw_coex_tdma(struct rtw_dev *rtwdev, bool force, u32 tcase) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_coex_stat *coex_stat = &coex->stat; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; u8 n, type; bool turn_on; @@ -1526,8 +1526,8 @@ static u8 rtw_coex_algorithm(struct rtw_dev *rtwdev) static void rtw_coex_action_coex_all_off(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -1549,11 +1549,11 @@ static void rtw_coex_action_coex_all_off(struct rtw_dev *rtwdev) static void rtw_coex_action_freerun(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 level = 0; bool bt_afh_loss = true; @@ -1594,8 +1594,8 @@ static void rtw_coex_action_freerun(struct rtw_dev *rtwdev) static void rtw_coex_action_rf4ce(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -1619,8 +1619,8 @@ static void rtw_coex_action_rf4ce(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_whql_test(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -1644,10 +1644,10 @@ static void rtw_coex_action_bt_whql_test(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_relink(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; u32 slot_type = 0; @@ -1684,11 +1684,11 @@ static void rtw_coex_action_bt_relink(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_idle(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_rfe *coex_rfe = &coex->rfe; u8 table_case = 0xff, tdma_case = 0xff; @@ -1753,10 +1753,10 @@ exit: static void rtw_coex_action_bt_inquiry(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; bool wl_hi_pri = false; u8 table_case, tdma_case; u32 slot_type = 0; @@ -1853,11 +1853,11 @@ static void rtw_coex_action_bt_inquiry(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_game_hid(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; struct rtw_coex_dm *coex_dm = &coex->dm; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -1901,10 +1901,10 @@ static void rtw_coex_action_bt_game_hid(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_hfp(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -1932,10 +1932,10 @@ static void rtw_coex_action_bt_hfp(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_hid(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; u32 slot_type = 0; bool bt_multi_link_remain = false, is_toggle_table = false; @@ -2015,11 +2015,11 @@ static void rtw_coex_action_bt_hid(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_a2dp(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; u32 slot_type = 0; @@ -2057,10 +2057,10 @@ static void rtw_coex_action_bt_a2dp(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_a2dpsink(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; bool ap_enable = false; @@ -2096,10 +2096,10 @@ static void rtw_coex_action_bt_a2dpsink(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_pan(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -2133,11 +2133,11 @@ static void rtw_coex_action_bt_pan(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_a2dp_hid(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case, interval = 0; u32 slot_type = 0; bool is_toggle_table = false; @@ -2190,10 +2190,10 @@ static void rtw_coex_action_bt_a2dp_hid(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_a2dp_pan(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; bool wl_cpt_test = false, bt_cpt_test = false; @@ -2247,10 +2247,10 @@ static void rtw_coex_action_bt_a2dp_pan(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_pan_hid(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -2282,10 +2282,10 @@ static void rtw_coex_action_bt_pan_hid(struct rtw_dev *rtwdev) static void rtw_coex_action_bt_a2dp_pan_hid(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -2316,9 +2316,9 @@ static void rtw_coex_action_bt_a2dp_pan_hid(struct rtw_dev *rtwdev) static void rtw_coex_action_wl_under5g(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_stat *coex_stat = &coex->stat; u8 table_case, tdma_case; @@ -2348,8 +2348,8 @@ static void rtw_coex_action_wl_under5g(struct rtw_dev *rtwdev) static void rtw_coex_action_wl_only(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -2372,9 +2372,9 @@ static void rtw_coex_action_wl_only(struct rtw_dev *rtwdev) static void rtw_coex_action_wl_native_lps(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_stat *coex_stat = &coex->stat; u8 table_case, tdma_case; @@ -2411,10 +2411,10 @@ static void rtw_coex_action_wl_native_lps(struct rtw_dev *rtwdev) static void rtw_coex_action_wl_linkscan(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; u32 slot_type = 0; @@ -2451,8 +2451,8 @@ static void rtw_coex_action_wl_linkscan(struct rtw_dev *rtwdev) static void rtw_coex_action_wl_not_connected(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 table_case, tdma_case; rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__); @@ -2528,8 +2528,8 @@ static void rtw_coex_action_wl_connected(struct rtw_dev *rtwdev) static void rtw_coex_run_coex(struct rtw_dev *rtwdev, u8 reason) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_dm *coex_dm = &coex->dm; struct rtw_coex_stat *coex_stat = &coex->stat; bool rf4ce_en = false; @@ -3002,9 +3002,9 @@ void rtw_coex_media_status_notify(struct rtw_dev *rtwdev, u8 type) void rtw_coex_bt_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_dm *coex_dm = &coex->dm; u32 bt_relink_time; u8 i, rsp_source = 0, type; @@ -3270,8 +3270,8 @@ static const u8 coex_bt_hidinfo_xb[] = {0x58, 0x62, 0x6f}; void rtw_coex_bt_hid_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_hid *hidinfo; struct rtw_coex_hid_info_a *hida; @@ -3360,8 +3360,8 @@ void rtw_coex_bt_hid_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length) void rtw_coex_query_bt_hid_list(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex_stat *coex_stat = &coex->stat; struct rtw_coex_hid *hidinfo; u8 i, handle; @@ -3582,7 +3582,7 @@ static const char *rtw_coex_get_reason_string(u8 reason) static u8 rtw_coex_get_table_index(struct rtw_dev *rtwdev, u32 wl_reg_6c0, u32 wl_reg_6c4) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; u8 ans = 0xFF; u8 n, i; @@ -3618,8 +3618,8 @@ static u8 rtw_coex_get_table_index(struct rtw_dev *rtwdev, u32 wl_reg_6c0, static u8 rtw_coex_get_tdma_index(struct rtw_dev *rtwdev, u8 *tdma_para) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; u8 ans = 0xFF; u8 n, i, j; u8 load_cur_tab_val; @@ -3736,7 +3736,7 @@ static int rtw_coex_val_info(struct rtw_dev *rtwdev, static void rtw_coex_set_coexinfo_hw(struct rtw_dev *rtwdev, struct seq_file *m) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_reg_domain *reg; char addr_info[INFO_SIZE]; int n_addr = 0; @@ -3910,7 +3910,7 @@ static const char *rtw_coex_get_wl_coex_mode(u8 coex_wl_link_mode) void rtw_coex_display_coex_info(struct rtw_dev *rtwdev, struct seq_file *m) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dm_info *dm_info = &rtwdev->dm_info; struct rtw_coex *coex = &rtwdev->coex; struct rtw_coex_stat *coex_stat = &coex->stat; diff --git a/drivers/net/wireless/realtek/rtw88/coex.h b/drivers/net/wireless/realtek/rtw88/coex.h index 07fa7aa34d4bac59eb8b0d21c0f61cdb7edceb0b..57cf29da9ea4e34e777e98ca3f6f52885b173991 100644 --- a/drivers/net/wireless/realtek/rtw88/coex.h +++ b/drivers/net/wireless/realtek/rtw88/coex.h @@ -327,7 +327,7 @@ struct coex_rf_para { static inline void rtw_coex_set_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_init(rtwdev); } @@ -335,7 +335,7 @@ static inline void rtw_coex_set_init(struct rtw_dev *rtwdev) static inline void rtw_coex_set_ant_switch(struct rtw_dev *rtwdev, u8 ctrl_type, u8 pos_type) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (!chip->ops->coex_set_ant_switch) return; @@ -345,28 +345,28 @@ void rtw_coex_set_ant_switch(struct rtw_dev *rtwdev, u8 ctrl_type, u8 pos_type) static inline void rtw_coex_set_gnt_fix(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_gnt_fix(rtwdev); } static inline void rtw_coex_set_gnt_debug(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_gnt_debug(rtwdev); } static inline void rtw_coex_set_rfe_type(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_rfe_type(rtwdev); } static inline void rtw_coex_set_wl_tx_power(struct rtw_dev *rtwdev, u8 wl_pwr) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_wl_tx_power(rtwdev, wl_pwr); } @@ -374,7 +374,7 @@ static inline void rtw_coex_set_wl_tx_power(struct rtw_dev *rtwdev, u8 wl_pwr) static inline void rtw_coex_set_wl_rx_gain(struct rtw_dev *rtwdev, bool low_gain) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->coex_set_wl_rx_gain(rtwdev, low_gain); } diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c index 7cde6bcf253b49e3e78c00ecdf69ec9d11dbb14b..9ebe544e51d0d37a1ade8f02c56dea8fb6ddbfe3 100644 --- a/drivers/net/wireless/realtek/rtw88/debug.c +++ b/drivers/net/wireless/realtek/rtw88/debug.c @@ -621,11 +621,13 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v) struct rtw_debugfs_priv *debugfs_priv = m->private; struct rtw_dev *rtwdev = debugfs_priv->rtwdev; struct rtw_hal *hal = &rtwdev->hal; - u8 path, rate; + u8 path, rate, bw, ch, regd; struct rtw_power_params pwr_param = {0}; - u8 bw = hal->current_band_width; - u8 ch = hal->current_channel; - u8 regd = rtw_regd_get(rtwdev); + + mutex_lock(&rtwdev->mutex); + bw = hal->current_band_width; + ch = hal->current_channel; + regd = rtw_regd_get(rtwdev); seq_printf(m, "channel: %u\n", ch); seq_printf(m, "bandwidth: %u\n", bw); @@ -667,6 +669,7 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v) } mutex_unlock(&hal->tx_power_mutex); + mutex_unlock(&rtwdev->mutex); return 0; } diff --git a/drivers/net/wireless/realtek/rtw88/efuse.c b/drivers/net/wireless/realtek/rtw88/efuse.c index c266c84ef2337469343bac3974dd9a63267e9d2a..b85075cd68d0a433563be736b5c3f7f26ee5f958 100644 --- a/drivers/net/wireless/realtek/rtw88/efuse.c +++ b/drivers/net/wireless/realtek/rtw88/efuse.c @@ -86,7 +86,7 @@ static int rtw_dump_logical_efuse_map(struct rtw_dev *rtwdev, u8 *phy_map, static int rtw_dump_physical_efuse_map(struct rtw_dev *rtwdev, u8 *map) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u32 size = rtwdev->efuse.physical_size; u32 efuse_ctl; u32 addr; @@ -145,7 +145,7 @@ EXPORT_SYMBOL(rtw_read8_physical_efuse); int rtw_parse_efuse_map(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; u32 phy_size = efuse->physical_size; u32 log_size = efuse->logical_size; diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c index 4fdab03296956354730be01265840726b5d18710..0b5f903c0f36631d66e6d70837f6f008f76502ea 100644 --- a/drivers/net/wireless/realtek/rtw88/fw.c +++ b/drivers/net/wireless/realtek/rtw88/fw.c @@ -14,6 +14,8 @@ #include "util.h" #include "wow.h" #include "ps.h" +#include "phy.h" +#include "mac.h" static void rtw_fw_c2h_cmd_handle_ext(struct rtw_dev *rtwdev, struct sk_buff *skb) @@ -116,7 +118,7 @@ legacy: si->ra_report.desc_rate = rate; si->ra_report.bit_rate = bit_rate; - sta->max_rc_amsdu_len = get_max_amsdu_len(bit_rate); + sta->deflink.agg.max_rc_amsdu_len = get_max_amsdu_len(bit_rate); } static void rtw_fw_ra_report_handle(struct rtw_dev *rtwdev, u8 *payload, @@ -904,7 +906,7 @@ void rtw_send_rsvd_page_h2c(struct rtw_dev *rtwdev) static struct sk_buff *rtw_nlo_info_get(struct ieee80211_hw *hw) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pno_request *pno_req = &rtwdev->wow.pno_req; struct rtw_nlo_info_hdr *nlo_hdr; struct cfg80211_ssid *ssid; @@ -959,7 +961,7 @@ static struct sk_buff *rtw_nlo_info_get(struct ieee80211_hw *hw) static struct sk_buff *rtw_cs_channel_info_get(struct ieee80211_hw *hw) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pno_request *pno_req = &rtwdev->wow.pno_req; struct ieee80211_channel *channels = pno_req->channels; struct sk_buff *skb; @@ -993,7 +995,7 @@ static struct sk_buff *rtw_cs_channel_info_get(struct ieee80211_hw *hw) static struct sk_buff *rtw_lps_pg_dpk_get(struct ieee80211_hw *hw) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dpk_info *dpk_info = &rtwdev->dm_info.dpk_info; struct rtw_lps_pg_dpk_hdr *dpk_hdr; struct sk_buff *skb; @@ -1018,7 +1020,7 @@ static struct sk_buff *rtw_lps_pg_dpk_get(struct ieee80211_hw *hw) static struct sk_buff *rtw_lps_pg_info_get(struct ieee80211_hw *hw) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_lps_conf *conf = &rtwdev->lps_conf; struct rtw_lps_pg_info_hdr *pg_info_hdr; struct rtw_wow_param *rtw_wow = &rtwdev->wow; @@ -1080,10 +1082,10 @@ static struct sk_buff *rtw_get_rsvd_page_skb(struct ieee80211_hw *hw, skb_new = ieee80211_proberesp_get(hw, vif); break; case RSVD_NULL: - skb_new = ieee80211_nullfunc_get(hw, vif, false); + skb_new = ieee80211_nullfunc_get(hw, vif, -1, false); break; case RSVD_QOS_NULL: - skb_new = ieee80211_nullfunc_get(hw, vif, true); + skb_new = ieee80211_nullfunc_get(hw, vif, -1, true); break; case RSVD_LPS_PG_DPK: skb_new = rtw_lps_pg_dpk_get(hw); @@ -1122,7 +1124,7 @@ static void rtw_fill_rsvd_page_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, enum rtw_rsvd_packet_type type) { struct rtw_tx_pkt_info pkt_info = {0}; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 *pkt_desc; rtw_tx_rsvd_page_pkt_info_update(rtwdev, &pkt_info, skb, type); @@ -1433,7 +1435,7 @@ static int __rtw_build_rsvd_page_from_vifs(struct rtw_dev *rtwdev) static u8 *rtw_build_rsvd_page(struct rtw_dev *rtwdev, u32 *size) { struct ieee80211_hw *hw = rtwdev->hw; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct sk_buff *iter; struct rtw_rsvd_page *rsvd_pkt; u32 page = 0; @@ -1647,7 +1649,7 @@ out: static void rtw_fw_read_fifo(struct rtw_dev *rtwdev, enum rtw_fw_fifo_sel sel, u32 offset, u32 size, u32 *buf) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u32 start_pg, residue; if (sel >= RTW_FW_FIFO_MAX) { @@ -1706,7 +1708,7 @@ int rtw_fw_dump_fifo(struct rtw_dev *rtwdev, u8 fifo_sel, u32 addr, u32 size, static void __rtw_fw_update_pkt(struct rtw_dev *rtwdev, u8 pkt_id, u16 size, u8 location) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 h2c_pkt[H2C_PKT_SIZE] = {0}; u16 total_size = H2C_PKT_HDR_SIZE + H2C_PKT_UPDATE_PKT_LEN; @@ -1818,8 +1820,8 @@ static int rtw_append_probe_req_ie(struct rtw_dev *rtwdev, struct sk_buff *skb, struct sk_buff_head *list, u8 *bands, struct rtw_vif *rtwvif) { + const struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_scan_ies *ies = rtwvif->scan_ies; - struct rtw_chip_info *chip = rtwdev->chip; struct sk_buff *new; u8 idx; @@ -1841,16 +1843,23 @@ static int rtw_append_probe_req_ie(struct rtw_dev *rtwdev, struct sk_buff *skb, static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes, struct sk_buff_head *probe_req_list) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct sk_buff *skb, *tmp; u8 page_offset = 1, *buf, page_size = chip->page_size; - u8 pages = page_offset + num_probes * RTW_PROBE_PG_CNT; u16 pg_addr = rtwdev->fifo.rsvd_h2c_info_addr, loc; u16 buf_offset = page_size * page_offset; u8 tx_desc_sz = chip->tx_pkt_desc_sz; + u8 page_cnt, pages; unsigned int pkt_len; int ret; + if (rtw_fw_feature_ext_check(&rtwdev->fw, FW_FEATURE_EXT_OLD_PAGE_NUM)) + page_cnt = RTW_OLD_PROBE_PG_CNT; + else + page_cnt = RTW_PROBE_PG_CNT; + + pages = page_offset + num_probes * page_cnt; + buf = kzalloc(page_size * pages, GFP_KERNEL); if (!buf) return -ENOMEM; @@ -1859,7 +1868,7 @@ static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes, skb_queue_walk_safe(probe_req_list, skb, tmp) { skb_unlink(skb, probe_req_list); rtw_fill_rsvd_page_desc(rtwdev, skb, RSVD_PROBE_REQ); - if (skb->len > page_size * RTW_PROBE_PG_CNT) { + if (skb->len > page_size * page_cnt) { ret = -EINVAL; goto out; } @@ -1869,8 +1878,8 @@ static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes, loc = pg_addr - rtwdev->fifo.rsvd_boundary + page_offset; __rtw_fw_update_pkt(rtwdev, RTW_PACKET_PROBE_REQ, pkt_len, loc); - buf_offset += RTW_PROBE_PG_CNT * page_size; - page_offset += RTW_PROBE_PG_CNT; + buf_offset += page_cnt * page_size; + page_offset += page_cnt; kfree_skb(skb); } @@ -2048,6 +2057,9 @@ void rtw_hw_scan_start(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, rtwvif->scan_req = req; ieee80211_stop_queues(rtwdev->hw); + rtw_leave_lps_deep(rtwdev); + rtw_hci_flush_all_queues(rtwdev, false); + rtw_mac_flush_all_queues(rtwdev, false); if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) get_random_mask_addr(mac_addr, req->mac_addr, req->mac_addr_mask); @@ -2080,10 +2092,9 @@ void rtw_hw_scan_complete(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, rtw_core_scan_complete(rtwdev, vif, true); rtwvif = (struct rtw_vif *)vif->drv_priv; - if (rtwvif->net_type == RTW_NET_MGD_LINKED) { - hal->current_channel = chan; - hal->current_band_type = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; - } + if (chan) + rtw_store_op_chan(rtwdev, false); + rtw_phy_set_tx_power_level(rtwdev, hal->current_channel); ieee80211_wake_queues(rtwdev->hw); ieee80211_scan_completed(rtwdev->hw, &info); @@ -2124,6 +2135,7 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, bool enable) { struct rtw_vif *rtwvif = vif ? (struct rtw_vif *)vif->drv_priv : NULL; + struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info; struct rtw_ch_switch_option cs_option = {0}; struct rtw_chan_list chan_list = {0}; int ret = 0; @@ -2132,7 +2144,7 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, return -EINVAL; cs_option.switch_en = enable; - cs_option.back_op_en = rtwvif->net_type == RTW_NET_MGD_LINKED; + cs_option.back_op_en = scan_info->op_chan != 0; if (enable) { ret = rtw_hw_scan_prehandle(rtwdev, rtwvif, &chan_list); if (ret) @@ -2171,14 +2183,33 @@ void rtw_hw_scan_status_report(struct rtw_dev *rtwdev, struct sk_buff *skb) rtw_dbg(rtwdev, RTW_DBG_HW_SCAN, "HW scan aborted with code: %d\n", rc); } -void rtw_store_op_chan(struct rtw_dev *rtwdev) +void rtw_store_op_chan(struct rtw_dev *rtwdev, bool backup) { struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info; struct rtw_hal *hal = &rtwdev->hal; + u8 band; + + if (backup) { + scan_info->op_chan = hal->current_channel; + scan_info->op_bw = hal->current_band_width; + scan_info->op_pri_ch_idx = hal->current_primary_channel_index; + scan_info->op_pri_ch = hal->primary_channel; + } else { + band = scan_info->op_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; + rtw_update_channel(rtwdev, scan_info->op_chan, + scan_info->op_pri_ch, + band, scan_info->op_bw); + } +} - scan_info->op_chan = hal->current_channel; - scan_info->op_bw = hal->current_band_width; - scan_info->op_pri_ch_idx = hal->current_primary_channel_index; +void rtw_clear_op_chan(struct rtw_dev *rtwdev) +{ + struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info; + + scan_info->op_chan = 0; + scan_info->op_bw = 0; + scan_info->op_pri_ch_idx = 0; + scan_info->op_pri_ch = 0; } static bool rtw_is_op_chan(struct rtw_dev *rtwdev, u8 channel) @@ -2193,7 +2224,7 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb) struct rtw_hal *hal = &rtwdev->hal; struct rtw_c2h_cmd *c2h; enum rtw_scan_notify_id id; - u8 chan, status; + u8 chan, band, status; if (!test_bit(RTW_FLAG_SCANNING, rtwdev->flags)) return; @@ -2204,10 +2235,13 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb) status = GET_CHAN_SWITCH_STATUS(c2h->payload); if (id == RTW_SCAN_NOTIFY_ID_POSTSWITCH) { - if (rtw_is_op_chan(rtwdev, chan)) + band = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; + rtw_update_channel(rtwdev, chan, chan, band, + RTW_CHANNEL_WIDTH_20); + if (rtw_is_op_chan(rtwdev, chan)) { + rtw_store_op_chan(rtwdev, false); ieee80211_wake_queues(rtwdev->hw); - hal->current_channel = chan; - hal->current_band_type = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; + } } else if (id == RTW_SCAN_NOTIFY_ID_PRESWITCH) { if (IS_CH_5G_BAND(chan)) { rtw_coex_switchband_notify(rtwdev, COEX_SWITCH_TO_5G); @@ -2220,7 +2254,12 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb) chan_type = COEX_SWITCH_TO_24G_NOFORSCAN; rtw_coex_switchband_notify(rtwdev, chan_type); } - if (rtw_is_op_chan(rtwdev, chan)) + /* The channel of C2H RTW_SCAN_NOTIFY_ID_PRESWITCH is next + * channel that hardware will switch. We need to stop queue + * if next channel is non-op channel. + */ + if (!rtw_is_op_chan(rtwdev, chan) && + rtw_is_op_chan(rtwdev, hal->current_channel)) ieee80211_stop_queues(rtwdev->hw); } diff --git a/drivers/net/wireless/realtek/rtw88/fw.h b/drivers/net/wireless/realtek/rtw88/fw.h index 7a37675c61e894faae1cb6849475db58b021d197..a5a965803a3cc194122c2a107c08ccd665d7b728 100644 --- a/drivers/net/wireless/realtek/rtw88/fw.h +++ b/drivers/net/wireless/realtek/rtw88/fw.h @@ -41,7 +41,8 @@ #define RTW_EX_CH_INFO_HDR_SIZE 2 #define RTW_SCAN_WIDTH 0 #define RTW_PRI_CH_IDX 1 -#define RTW_PROBE_PG_CNT 2 +#define RTW_OLD_PROBE_PG_CNT 2 +#define RTW_PROBE_PG_CNT 4 enum rtw_c2h_cmd_id { C2H_CCX_TX_RPT = 0x03, @@ -120,6 +121,10 @@ enum rtw_fw_feature { FW_FEATURE_MAX = BIT(31), }; +enum rtw_fw_feature_ext { + FW_FEATURE_EXT_OLD_PAGE_NUM = BIT(0), +}; + enum rtw_beacon_filter_offload_mode { BCN_FILTER_OFFLOAD_MODE_0 = 0, BCN_FILTER_OFFLOAD_MODE_1, @@ -323,6 +328,11 @@ struct rtw_fw_hdr_legacy { __le32 rsvd5; } __packed; +#define RTW_FW_VER_CODE(ver, sub_ver, idx) \ + (((ver) << 16) | ((sub_ver) << 8) | (idx)) +#define RTW_FW_SUIT_VER_CODE(s) \ + RTW_FW_VER_CODE((s).version, (s).sub_version, (s).sub_index) + /* C2H */ #define GET_CCX_REPORT_SEQNUM_V0(c2h_payload) (c2h_payload[6] & 0xfc) #define GET_CCX_REPORT_STATUS_V0(c2h_payload) (c2h_payload[0] & 0xc0) @@ -770,6 +780,12 @@ static inline bool rtw_fw_feature_check(struct rtw_fw_state *fw, return !!(fw->feature & feature); } +static inline bool rtw_fw_feature_ext_check(struct rtw_fw_state *fw, + enum rtw_fw_feature_ext feature) +{ + return !!(fw->feature_ext & feature); +} + void rtw_fw_c2h_cmd_rx_irqsafe(struct rtw_dev *rtwdev, u32 pkt_offset, struct sk_buff *skb); void rtw_fw_c2h_cmd_handle(struct rtw_dev *rtwdev, struct sk_buff *skb); @@ -831,7 +847,8 @@ int rtw_fw_dump_fifo(struct rtw_dev *rtwdev, u8 fifo_sel, u32 addr, u32 size, u32 *buffer); void rtw_fw_scan_notify(struct rtw_dev *rtwdev, bool start); void rtw_fw_adaptivity(struct rtw_dev *rtwdev); -void rtw_store_op_chan(struct rtw_dev *rtwdev); +void rtw_store_op_chan(struct rtw_dev *rtwdev, bool backup); +void rtw_clear_op_chan(struct rtw_dev *rtwdev); void rtw_hw_scan_start(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_scan_request *req); void rtw_hw_scan_complete(struct rtw_dev *rtwdev, struct ieee80211_vif *vif, diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c index caf2603da2d6550642a6bf468d05de745b23b478..52076e89d59a33b0db5747eacfc697b79adc1473 100644 --- a/drivers/net/wireless/realtek/rtw88/mac.c +++ b/drivers/net/wireless/realtek/rtw88/mac.c @@ -243,7 +243,7 @@ static int rtw_pwr_seq_parser(struct rtw_dev *rtwdev, static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_pwr_seq_cmd **pwr_seq; u8 rpwm; bool cur_pwr; @@ -587,7 +587,7 @@ static int download_firmware_to_mem(struct rtw_dev *rtwdev, const u8 *data, u32 src, u32 dst, u32 size) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u32 desc_size = chip->tx_pkt_desc_sz; u8 first_part; u32 mem_offset; @@ -934,7 +934,7 @@ static u32 get_priority_queues(struct rtw_dev *rtwdev, u32 queues) static void __rtw_mac_flush_prio_queue(struct rtw_dev *rtwdev, u32 prio_queue, bool drop) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_prioq_addr *addr; bool wsize; u16 avail_page, rsvd_page; @@ -996,7 +996,7 @@ void rtw_mac_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) static int txdma_queue_mapping(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_rqpn *rqpn = NULL; u16 txdma_pq_map = 0; @@ -1037,8 +1037,8 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev) static int set_trx_fifo_info(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fifo_conf *fifo = &rtwdev->fifo; - struct rtw_chip_info *chip = rtwdev->chip; u16 cur_pg_addr; u8 csi_buf_pg_num = chip->csi_buf_pg_num; @@ -1092,8 +1092,8 @@ static int __priority_queue_cfg(struct rtw_dev *rtwdev, const struct rtw_page_table *pg_tbl, u16 pubq_num) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fifo_conf *fifo = &rtwdev->fifo; - struct rtw_chip_info *chip = rtwdev->chip; rtw_write16(rtwdev, REG_FIFOPAGE_INFO_1, pg_tbl->hq_num); rtw_write16(rtwdev, REG_FIFOPAGE_INFO_2, pg_tbl->lq_num); @@ -1123,8 +1123,8 @@ static int __priority_queue_cfg_legacy(struct rtw_dev *rtwdev, const struct rtw_page_table *pg_tbl, u16 pubq_num) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fifo_conf *fifo = &rtwdev->fifo; - struct rtw_chip_info *chip = rtwdev->chip; u32 val32; val32 = BIT_RQPN_NE(pg_tbl->nq_num, pg_tbl->exq_num); @@ -1149,8 +1149,8 @@ static int __priority_queue_cfg_legacy(struct rtw_dev *rtwdev, static int priority_queue_cfg(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fifo_conf *fifo = &rtwdev->fifo; - struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_page_table *pg_tbl = NULL; u16 pubq_num; int ret; @@ -1277,7 +1277,7 @@ static int rtw_drv_info_cfg(struct rtw_dev *rtwdev) int rtw_mac_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; int ret; ret = rtw_init_trx_cfg(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c index c7b98a0599d5f08d893a9f50fee16e0da70e08c7..07578ccc4bab3bd6e520c667e6b01787712d7f68 100644 --- a/drivers/net/wireless/realtek/rtw88/mac80211.c +++ b/drivers/net/wireless/realtek/rtw88/mac80211.c @@ -101,7 +101,8 @@ static int rtw_ops_config(struct ieee80211_hw *hw, u32 changed) rtw_set_channel(rtwdev); if ((changed & IEEE80211_CONF_CHANGE_IDLE) && - (hw->conf.flags & IEEE80211_CONF_IDLE)) + (hw->conf.flags & IEEE80211_CONF_IDLE) && + !test_bit(RTW_FLAG_SCANNING, rtwdev->flags)) rtw_enter_ips(rtwdev); out: @@ -377,7 +378,6 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw, rtw_coex_media_status_notify(rtwdev, vif->cfg.assoc); if (rtw_bf_support) rtw_bf_assoc(rtwdev, vif, conf); - rtw_store_op_chan(rtwdev); } else { rtw_leave_lps(rtwdev); rtw_bf_disassoc(rtwdev, vif, conf); @@ -395,6 +395,10 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_BSSID) { ether_addr_copy(rtwvif->bssid, conf->bssid); config |= PORT_SET_BSSID; + if (is_zero_ether_addr(rtwvif->bssid)) + rtw_clear_op_chan(rtwdev); + else + rtw_store_op_chan(rtwdev, true); } if (changed & BSS_CHANGED_BEACON_INT) { @@ -434,7 +438,7 @@ static int rtw_ops_start_ap(struct ieee80211_hw *hw, struct ieee80211_bss_conf *link_conf) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; mutex_lock(&rtwdev->mutex); chip->ops->phy_calibration(rtwdev); @@ -752,7 +756,7 @@ static int rtw_ops_set_antenna(struct ieee80211_hw *hw, u32 rx_antenna) { struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; int ret; if (!chip->ops->set_antenna) @@ -872,7 +876,9 @@ static int rtw_ops_set_sar_specs(struct ieee80211_hw *hw, { struct rtw_dev *rtwdev = hw->priv; + mutex_lock(&rtwdev->mutex); rtw_set_sar_specs(rtwdev, sar); + mutex_unlock(&rtwdev->mutex); return 0; } diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c index 76dc9da88f6c21d2ef91b393189864546ebc31b6..67151dbf83842a783922d203431eeb4f05df3420 100644 --- a/drivers/net/wireless/realtek/rtw88/main.c +++ b/drivers/net/wireless/realtek/rtw88/main.c @@ -353,7 +353,7 @@ struct rtw_fwcd_hdr { static int rtw_fwcd_prep(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fwcd_desc *desc = &rtwdev->fw.fwcd_desc; const struct rtw_fwcd_segs *segs = chip->fwcd_segs; u32 prep_size = chip->fw_rxff_size + sizeof(struct rtw_fwcd_hdr); @@ -675,67 +675,126 @@ void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period) rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period - 1); } +void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel, + u8 primary_channel, enum rtw_supported_band band, + enum rtw_bandwidth bandwidth) +{ + enum nl80211_band nl_band = rtw_hw_to_nl80211_band(band); + struct rtw_hal *hal = &rtwdev->hal; + u8 *cch_by_bw = hal->cch_by_bw; + u32 center_freq, primary_freq; + enum rtw_sar_bands sar_band; + u8 primary_channel_idx; + + center_freq = ieee80211_channel_to_frequency(center_channel, nl_band); + primary_freq = ieee80211_channel_to_frequency(primary_channel, nl_band); + + /* assign the center channel used while 20M bw is selected */ + cch_by_bw[RTW_CHANNEL_WIDTH_20] = primary_channel; + + /* assign the center channel used while current bw is selected */ + cch_by_bw[bandwidth] = center_channel; + + switch (bandwidth) { + case RTW_CHANNEL_WIDTH_20: + default: + primary_channel_idx = RTW_SC_DONT_CARE; + break; + case RTW_CHANNEL_WIDTH_40: + if (primary_freq > center_freq) + primary_channel_idx = RTW_SC_20_UPPER; + else + primary_channel_idx = RTW_SC_20_LOWER; + break; + case RTW_CHANNEL_WIDTH_80: + if (primary_freq > center_freq) { + if (primary_freq - center_freq == 10) + primary_channel_idx = RTW_SC_20_UPPER; + else + primary_channel_idx = RTW_SC_20_UPMOST; + + /* assign the center channel used + * while 40M bw is selected + */ + cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_channel + 4; + } else { + if (center_freq - primary_freq == 10) + primary_channel_idx = RTW_SC_20_LOWER; + else + primary_channel_idx = RTW_SC_20_LOWEST; + + /* assign the center channel used + * while 40M bw is selected + */ + cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_channel - 4; + } + break; + } + + switch (center_channel) { + case 1 ... 14: + sar_band = RTW_SAR_BAND_0; + break; + case 36 ... 64: + sar_band = RTW_SAR_BAND_1; + break; + case 100 ... 144: + sar_band = RTW_SAR_BAND_3; + break; + case 149 ... 177: + sar_band = RTW_SAR_BAND_4; + break; + default: + WARN(1, "unknown ch(%u) to SAR band\n", center_channel); + sar_band = RTW_SAR_BAND_0; + break; + } + + hal->current_primary_channel_index = primary_channel_idx; + hal->current_band_width = bandwidth; + hal->primary_channel = primary_channel; + hal->current_channel = center_channel; + hal->current_band_type = band; + hal->sar_band = sar_band; +} + void rtw_get_channel_params(struct cfg80211_chan_def *chandef, struct rtw_channel_params *chan_params) { struct ieee80211_channel *channel = chandef->chan; enum nl80211_chan_width width = chandef->width; - u8 *cch_by_bw = chan_params->cch_by_bw; u32 primary_freq, center_freq; u8 center_chan; u8 bandwidth = RTW_CHANNEL_WIDTH_20; - u8 primary_chan_idx = 0; - u8 i; center_chan = channel->hw_value; primary_freq = channel->center_freq; center_freq = chandef->center_freq1; - /* assign the center channel used while 20M bw is selected */ - cch_by_bw[RTW_CHANNEL_WIDTH_20] = channel->hw_value; - switch (width) { case NL80211_CHAN_WIDTH_20_NOHT: case NL80211_CHAN_WIDTH_20: bandwidth = RTW_CHANNEL_WIDTH_20; - primary_chan_idx = RTW_SC_DONT_CARE; break; case NL80211_CHAN_WIDTH_40: bandwidth = RTW_CHANNEL_WIDTH_40; - if (primary_freq > center_freq) { - primary_chan_idx = RTW_SC_20_UPPER; + if (primary_freq > center_freq) center_chan -= 2; - } else { - primary_chan_idx = RTW_SC_20_LOWER; + else center_chan += 2; - } break; case NL80211_CHAN_WIDTH_80: bandwidth = RTW_CHANNEL_WIDTH_80; if (primary_freq > center_freq) { - if (primary_freq - center_freq == 10) { - primary_chan_idx = RTW_SC_20_UPPER; + if (primary_freq - center_freq == 10) center_chan -= 2; - } else { - primary_chan_idx = RTW_SC_20_UPMOST; + else center_chan -= 6; - } - /* assign the center channel used - * while 40M bw is selected - */ - cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_chan + 4; } else { - if (center_freq - primary_freq == 10) { - primary_chan_idx = RTW_SC_20_LOWER; + if (center_freq - primary_freq == 10) center_chan += 2; - } else { - primary_chan_idx = RTW_SC_20_LOWEST; + else center_chan += 6; - } - /* assign the center channel used - * while 40M bw is selected - */ - cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_chan - 4; } break; default: @@ -745,60 +804,30 @@ void rtw_get_channel_params(struct cfg80211_chan_def *chandef, chan_params->center_chan = center_chan; chan_params->bandwidth = bandwidth; - chan_params->primary_chan_idx = primary_chan_idx; - - /* assign the center channel used while current bw is selected */ - cch_by_bw[bandwidth] = center_chan; - - for (i = bandwidth + 1; i <= RTW_MAX_CHANNEL_WIDTH; i++) - cch_by_bw[i] = 0; + chan_params->primary_chan = channel->hw_value; } void rtw_set_channel(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_hw *hw = rtwdev->hw; struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; struct rtw_channel_params ch_param; - u8 center_chan, bandwidth, primary_chan_idx; - u8 i; + u8 center_chan, primary_chan, bandwidth, band; rtw_get_channel_params(&hw->conf.chandef, &ch_param); if (WARN(ch_param.center_chan == 0, "Invalid channel\n")) return; center_chan = ch_param.center_chan; + primary_chan = ch_param.primary_chan; bandwidth = ch_param.bandwidth; - primary_chan_idx = ch_param.primary_chan_idx; - - hal->current_band_width = bandwidth; - hal->current_channel = center_chan; - hal->current_primary_channel_index = primary_chan_idx; - hal->current_band_type = center_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; - - switch (center_chan) { - case 1 ... 14: - hal->sar_band = RTW_SAR_BAND_0; - break; - case 36 ... 64: - hal->sar_band = RTW_SAR_BAND_1; - break; - case 100 ... 144: - hal->sar_band = RTW_SAR_BAND_3; - break; - case 149 ... 177: - hal->sar_band = RTW_SAR_BAND_4; - break; - default: - WARN(1, "unknown ch(%u) to SAR band\n", center_chan); - hal->sar_band = RTW_SAR_BAND_0; - break; - } + band = ch_param.center_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G; - for (i = RTW_CHANNEL_WIDTH_20; i <= RTW_MAX_CHANNEL_WIDTH; i++) - hal->cch_by_bw[i] = ch_param.cch_by_bw[i]; + rtw_update_channel(rtwdev, center_chan, primary_chan, band, bandwidth); - chip->ops->set_channel(rtwdev, center_chan, bandwidth, primary_chan_idx); + chip->ops->set_channel(rtwdev, center_chan, bandwidth, + hal->current_primary_channel_index); if (hal->current_band_type == RTW_BAND_5G) { rtw_coex_switchband_notify(rtwdev, COEX_SWITCH_TO_5G); @@ -821,7 +850,7 @@ void rtw_set_channel(struct rtw_dev *rtwdev) void rtw_chip_prepare_tx(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (rtwdev->need_rfk) { rtwdev->need_rfk = false; @@ -890,8 +919,8 @@ static u8 hw_bw_cap_to_bitamp(u8 bw_cap) static void rtw_hw_config_rf_ant_num(struct rtw_dev *rtwdev, u8 hw_ant_num) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; if (hw_ant_num == EFUSE_HW_CAP_IGNORE || hw_ant_num >= hal->rf_path_num) @@ -1240,7 +1269,7 @@ void rtw_update_sta_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si, static int rtw_wait_firmware_completion(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fw_state *fw; fw = &rtwdev->fw; @@ -1261,7 +1290,7 @@ static int rtw_wait_firmware_completion(struct rtw_dev *rtwdev) static enum rtw_lps_deep_mode rtw_update_lps_deep_mode(struct rtw_dev *rtwdev, struct rtw_fw_state *fw) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (rtw_disable_lps_deep_mode || !chip->lps_deep_mode_supported || !fw->feature) @@ -1280,7 +1309,7 @@ static enum rtw_lps_deep_mode rtw_update_lps_deep_mode(struct rtw_dev *rtwdev, static int rtw_power_on(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_fw_state *fw = &rtwdev->fw; bool wifi_only; int ret; @@ -1469,8 +1498,8 @@ void rtw_core_stop(struct rtw_dev *rtwdev) static void rtw_init_ht_cap(struct rtw_dev *rtwdev, struct ieee80211_sta_ht_cap *ht_cap) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; - struct rtw_chip_info *chip = rtwdev->chip; ht_cap->ht_supported = true; ht_cap->cap = 0; @@ -1552,8 +1581,23 @@ static void rtw_init_vht_cap(struct rtw_dev *rtwdev, vht_cap->vht_mcs.tx_highest = highest; } +static u16 rtw_get_max_scan_ie_len(struct rtw_dev *rtwdev) +{ + u16 len; + + len = rtwdev->chip->max_scan_ie_len; + + if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_SCAN_OFFLOAD) && + rtwdev->chip->id == RTW_CHIP_TYPE_8822C) + len = IEEE80211_MAX_DATA_LEN; + else if (rtw_fw_feature_ext_check(&rtwdev->fw, FW_FEATURE_EXT_OLD_PAGE_NUM)) + len -= RTW_OLD_PROBE_PG_CNT * TX_PAGE_SIZE; + + return len; +} + static void rtw_set_supported_band(struct ieee80211_hw *hw, - struct rtw_chip_info *chip) + const struct rtw_chip_info *chip) { struct rtw_dev *rtwdev = hw->priv; struct ieee80211_supported_band *sband; @@ -1585,7 +1629,7 @@ err_out: } static void rtw_unset_supported_band(struct ieee80211_hw *hw, - struct rtw_chip_info *chip) + const struct rtw_chip_info *chip) { kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]); kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]); @@ -1607,7 +1651,7 @@ static void rtw_vif_smps_iter(void *data, u8 *mac, void rtw_set_txrx_1ss(struct rtw_dev *rtwdev, bool txrx_1ss) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_hal *hal = &rtwdev->hal; if (!chip->ops->config_txrx_mode || rtwdev->hal.txrx_1ss == txrx_1ss) @@ -1631,6 +1675,10 @@ static void __update_firmware_feature(struct rtw_dev *rtwdev, feature = le32_to_cpu(fw_hdr->feature); fw->feature = feature & FW_FEATURE_SIG ? feature : 0; + + if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C && + RTW_FW_SUIT_VER_CODE(rtwdev->fw) < RTW_FW_VER_CODE(9, 9, 13)) + fw->feature_ext |= FW_FEATURE_EXT_OLD_PAGE_NUM; } static void __update_firmware_info(struct rtw_dev *rtwdev, @@ -1724,7 +1772,7 @@ static int rtw_load_firmware(struct rtw_dev *rtwdev, enum rtw_fw_type type) static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_hal *hal = &rtwdev->hal; struct rtw_efuse *efuse = &rtwdev->efuse; @@ -1982,7 +2030,7 @@ static void rtw_stats_init(struct rtw_dev *rtwdev) int rtw_core_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_coex *coex = &rtwdev->coex; int ret; @@ -2045,7 +2093,7 @@ int rtw_core_init(struct rtw_dev *rtwdev) ret = rtw_load_firmware(rtwdev, RTW_NORMAL_FW); if (ret) { rtw_warn(rtwdev, "no firmware loaded\n"); - return ret; + goto out; } if (chip->wow_fw_name) { @@ -2055,11 +2103,15 @@ int rtw_core_init(struct rtw_dev *rtwdev) wait_for_completion(&rtwdev->fw.completion); if (rtwdev->fw.firmware) release_firmware(rtwdev->fw.firmware); - return ret; + goto out; } } return 0; + +out: + destroy_workqueue(rtwdev->tx_wq); + return ret; } EXPORT_SYMBOL(rtw_core_init); @@ -2136,7 +2188,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; hw->wiphy->max_scan_ssids = RTW_SCAN_MAX_SSIDS; - hw->wiphy->max_scan_ie_len = RTW_SCAN_MAX_IE_LEN; + hw->wiphy->max_scan_ie_len = rtw_get_max_scan_ie_len(rtwdev); wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0); wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_SCAN_RANDOM_SN); @@ -2180,7 +2232,7 @@ EXPORT_SYMBOL(rtw_register_hw); void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; ieee80211_unregister_hw(hw); rtw_unset_supported_band(hw, chip); diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index 7db627fc26be9e9a358bb349b52277e413f5424d..bccd7b28f60c7b75ede6abc943ae22a6a4fda13f 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -22,7 +22,6 @@ #define MAX_PG_CAM_BACKUP_NUM 8 #define RTW_SCAN_MAX_SSIDS 4 -#define RTW_SCAN_MAX_IE_LEN 128 #define RTW_MAX_PATTERN_NUM 12 #define RTW_MAX_PATTERN_MASK_SIZE 16 @@ -33,6 +32,7 @@ #define RFREG_MASK 0xfffff #define INV_RF_DATA 0xffffffff #define TX_PAGE_SIZE_SHIFT 7 +#define TX_PAGE_SIZE (1 << TX_PAGE_SIZE_SHIFT) #define RTW_CHANNEL_WIDTH_MAX 3 #define RTW_RF_PATH_MAX 4 @@ -510,12 +510,8 @@ struct rtw_timer_list { struct rtw_channel_params { u8 center_chan; + u8 primary_chan; u8 bandwidth; - u8 primary_chan_idx; - /* center channel by different available bandwidth, - * val of (bw > current bandwidth) is invalid - */ - u8 cch_by_bw[RTW_MAX_CHANNEL_WIDTH + 1]; }; struct rtw_hw_reg { @@ -1232,6 +1228,7 @@ struct rtw_chip_info { const char *wow_fw_name; const struct wiphy_wowlan_support *wowlan_stub; const u8 max_sched_scan_ssids; + const u16 max_scan_ie_len; /* coex paras */ u32 coex_para_ver; @@ -1853,6 +1850,7 @@ struct rtw_fw_state { u8 sub_index; u16 h2c_version; u32 feature; + u32 feature_ext; }; enum rtw_sar_sources { @@ -1896,6 +1894,7 @@ struct rtw_hal { u8 current_primary_channel_index; u8 current_band_width; u8 current_band_type; + u8 primary_channel; /* center channel for different available bandwidth, * val of (bw > current_band_width) is invalid @@ -1967,6 +1966,7 @@ struct rtw_hw_scan_info { struct ieee80211_vif *scanning_vif; u8 probe_pg_size; u8 op_pri_ch_idx; + u8 op_pri_ch; u8 op_chan; u8 op_bw; }; @@ -1978,7 +1978,7 @@ struct rtw_dev { struct rtw_hci hci; struct rtw_hw_scan_info scan_info; - struct rtw_chip_info *chip; + const struct rtw_chip_info *chip; struct rtw_hal hal; struct rtw_fifo_conf fifo; struct rtw_fw_state fw; @@ -2132,6 +2132,20 @@ static inline int rtw_chip_dump_fw_crash(struct rtw_dev *rtwdev) return 0; } +static inline +enum nl80211_band rtw_hw_to_nl80211_band(enum rtw_supported_band hw_band) +{ + switch (hw_band) { + default: + case RTW_BAND_2G: + return NL80211_BAND_2GHZ; + case RTW_BAND_5G: + return NL80211_BAND_5GHZ; + case RTW_BAND_60G: + return NL80211_BAND_60GHZ; + } +} + void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel); void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period); void rtw_get_channel_params(struct cfg80211_chan_def *chandef, @@ -2173,4 +2187,7 @@ int rtw_dump_fw(struct rtw_dev *rtwdev, const u32 ocp_src, u32 size, u32 fwcd_item); int rtw_dump_reg(struct rtw_dev *rtwdev, const u32 addr, const u32 size); void rtw_set_txrx_1ss(struct rtw_dev *rtwdev, bool config_1ss); +void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel, + u8 primary_channel, enum rtw_supported_band band, + enum rtw_bandwidth bandwidth); #endif diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index 24d5695363d3fc995fa88daa0fc2bafac5f41be6..0975d27240e4667ca3796d3d2b964457a943d3bc 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -322,7 +322,7 @@ static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct rtw_pci_tx_ring *tx_ring; struct rtw_pci_rx_ring *rx_ring; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; int tx_desc_size, rx_desc_size; u32 len; @@ -721,7 +721,7 @@ static void rtw_pci_dma_check(struct rtw_dev *rtwdev, u32 idx) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci_rx_buffer_desc *buf_desc; u32 desc_sz = chip->rx_buf_desc_sz; u16 total_pkt_size; @@ -834,7 +834,7 @@ static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, struct sk_buff *skb, u8 queue) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci_tx_ring *ring; struct rtw_pci_tx_data *tx_data; dma_addr_t dma; @@ -1073,7 +1073,7 @@ static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, u8 hw_queue, u32 limit) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct napi_struct *napi = &rtwpci->napi; struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; struct rtw_rx_pkt_stat pkt_stat; @@ -1425,7 +1425,7 @@ static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; struct pci_dev *pdev = rtwpci->pdev; u16 link_ctrl; @@ -1467,7 +1467,7 @@ static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; switch (chip->id) { case RTW_CHIP_TYPE_8822C: @@ -1483,7 +1483,7 @@ static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) { struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct pci_dev *pdev = rtwpci->pdev; const struct rtw_intf_phy_para *para; u16 cut; @@ -1538,7 +1538,7 @@ static int __maybe_unused rtw_pci_suspend(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) @@ -1550,7 +1550,7 @@ static int __maybe_unused rtw_pci_resume(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw_dev *rtwdev = hw->priv; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) @@ -1717,8 +1717,7 @@ static void rtw_pci_napi_init(struct rtw_dev *rtwdev) struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; init_dummy_netdev(&rtwpci->netdev); - netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll); } static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) @@ -1848,7 +1847,7 @@ void rtw_pci_shutdown(struct pci_dev *pdev) { struct ieee80211_hw *hw = pci_get_drvdata(pdev); struct rtw_dev *rtwdev; - struct rtw_chip_info *chip; + const struct rtw_chip_info *chip; if (!hw) return; diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c index 8982e0c98dac9f788edaf0096c69cd5b97e35656..bd7d05e0808483d301f7e3cf7b81c1743b8ddf0d 100644 --- a/drivers/net/wireless/realtek/rtw88/phy.c +++ b/drivers/net/wireless/realtek/rtw88/phy.c @@ -138,7 +138,7 @@ EXPORT_SYMBOL(rtw_phy_set_edcca_th); void rtw_phy_adaptivity_set_mode(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dm_info *dm_info = &rtwdev->dm_info; /* turn off in debugfs for debug usage */ @@ -165,7 +165,7 @@ void rtw_phy_adaptivity_set_mode(struct rtw_dev *rtwdev) static void rtw_phy_adaptivity_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; rtw_phy_adaptivity_set_mode(rtwdev); if (chip->ops->adaptivity_init) @@ -180,7 +180,7 @@ static void rtw_phy_adaptivity(struct rtw_dev *rtwdev) static void rtw_phy_cfo_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (chip->ops->cfo_init) chip->ops->cfo_init(rtwdev); @@ -199,7 +199,7 @@ static void rtw_phy_tx_path_div_init(struct rtw_dev *rtwdev) void rtw_phy_init(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dm_info *dm_info = &rtwdev->dm_info; u32 addr, mask; @@ -226,7 +226,7 @@ EXPORT_SYMBOL(rtw_phy_init); void rtw_phy_dig_write(struct rtw_dev *rtwdev, u8 igi) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_hal *hal = &rtwdev->hal; u32 addr, mask; u8 path; @@ -245,7 +245,7 @@ void rtw_phy_dig_write(struct rtw_dev *rtwdev, u8 igi) static void rtw_phy_stat_false_alarm(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; chip->ops->false_alarm_statistics(rtwdev); } @@ -603,7 +603,7 @@ static void rtw_phy_rrsr_update(struct rtw_dev *rtwdev) static void rtw_phy_dpk_track(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (chip->ops->dpk_track) chip->ops->dpk_track(rtwdev); @@ -659,7 +659,7 @@ EXPORT_SYMBOL(rtw_phy_parsing_cfo); static void rtw_phy_cfo_track(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (chip->ops->cfo_track) chip->ops->cfo_track(rtwdev); @@ -720,8 +720,8 @@ static u8 rtw_phy_cck_pd_lv(struct rtw_dev *rtwdev) static void rtw_phy_cck_pd(struct rtw_dev *rtwdev) { + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dm_info *dm_info = &rtwdev->dm_info; - struct rtw_chip_info *chip = rtwdev->chip; u32 cck_fa = dm_info->cck_fa_cnt; u8 level; @@ -816,23 +816,18 @@ static u8 rtw_phy_linear_2_db(u64 linear) u8 j; u32 dB; - if (linear >= db_invert_table[11][7]) - return 96; /* maximum 96 dB */ - for (i = 0; i < 12; i++) { - if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][7]) - break; - else if (i > 2 && linear <= db_invert_table[i][7]) - break; + for (j = 0; j < 8; j++) { + if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][j]) + goto cnt; + else if (i > 2 && linear <= db_invert_table[i][j]) + goto cnt; + } } - for (j = 0; j < 8; j++) { - if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][j]) - break; - else if (i > 2 && linear <= db_invert_table[i][j]) - break; - } + return 96; /* maximum 96 dB */ +cnt: if (j == 0 && i == 0) goto end; @@ -900,7 +895,7 @@ u32 rtw_phy_read_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, u32 addr, u32 mask) { struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const u32 *base_addr = chip->rf_base_addr; u32 val, direct_addr; @@ -923,7 +918,7 @@ u32 rtw_phy_read_rf_sipi(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, u32 addr, u32 mask) { struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_rf_sipi_addr *rf_sipi_addr; const struct rtw_rf_sipi_addr *rf_sipi_addr_a; u32 val32; @@ -972,8 +967,8 @@ bool rtw_phy_write_rf_reg_sipi(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, u32 addr, u32 mask, u32 data) { struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; - u32 *sipi_addr = chip->rf_sipi_addr; + const struct rtw_chip_info *chip = rtwdev->chip; + const u32 *sipi_addr = chip->rf_sipi_addr; u32 data_and_addr; u32 old_data = 0; u32 shift; @@ -1012,7 +1007,7 @@ bool rtw_phy_write_rf_reg(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path, u32 addr, u32 mask, u32 data) { struct rtw_hal *hal = &rtwdev->hal; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const u32 *base_addr = chip->rf_base_addr; u32 direct_addr; @@ -1747,7 +1742,7 @@ EXPORT_SYMBOL(rtw_phy_cfg_rf); static void rtw_load_rfk_table(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_dpk_info *dpk_info = &rtwdev->dm_info.dpk_info; if (!chip->rfk_init_tbl) @@ -1766,7 +1761,7 @@ static void rtw_load_rfk_table(struct rtw_dev *rtwdev) void rtw_phy_load_tables(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 rf_path; rtw_load_table(rtwdev, chip->mac_tbl); @@ -1875,7 +1870,7 @@ static u8 rtw_get_channel_group(u8 channel, u8 rate) static s8 rtw_phy_get_dis_dpd_by_rate_diff(struct rtw_dev *rtwdev, u16 rate) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; s8 dpd_diff = 0; if (!chip->en_dis_dpd) @@ -1909,7 +1904,7 @@ static u8 rtw_phy_get_2g_tx_power_index(struct rtw_dev *rtwdev, enum rtw_bandwidth bandwidth, u8 rate, u8 group) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 tx_power; bool mcs_rate; bool above_2ss; @@ -1956,7 +1951,7 @@ static u8 rtw_phy_get_5g_tx_power_index(struct rtw_dev *rtwdev, enum rtw_bandwidth bandwidth, u8 rate, u8 group) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; u8 tx_power; u8 upper, lower; bool mcs_rate; @@ -2209,7 +2204,7 @@ static void rtw_phy_set_tx_power_level_by_path(struct rtw_dev *rtwdev, void rtw_phy_set_tx_power_level(struct rtw_dev *rtwdev, u8 channel) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_hal *hal = &rtwdev->hal; u8 path; @@ -2484,7 +2479,7 @@ static void rtw_phy_set_tx_path_by_reg(struct rtw_dev *rtwdev, { struct rtw_path_div *path_div = &rtwdev->dm_path_div; enum rtw_bb_path tx_path_sel_cck = tx_path_sel_1ss; - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (tx_path_sel_1ss == path_div->current_tx_path) return; @@ -2539,7 +2534,7 @@ static void rtw_phy_tx_path_diversity_2ss(struct rtw_dev *rtwdev) void rtw_phy_tx_path_diversity(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; if (!chip->path_div_supported) return; diff --git a/drivers/net/wireless/realtek/rtw88/phy.h b/drivers/net/wireless/realtek/rtw88/phy.h index b6c5ae60a46200442d9b402c77748e7679ef89f6..ccfcbd3ced03aae0131d16c14d79c0f2d604087a 100644 --- a/drivers/net/wireless/realtek/rtw88/phy.h +++ b/drivers/net/wireless/realtek/rtw88/phy.h @@ -114,7 +114,7 @@ const struct rtw_table name ## _tbl = { \ static inline const struct rtw_rfe_def *rtw_get_rfe_def(struct rtw_dev *rtwdev) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct rtw_efuse *efuse = &rtwdev->efuse; const struct rtw_rfe_def *rfe_def = NULL; diff --git a/drivers/net/wireless/realtek/rtw88/ps.c b/drivers/net/wireless/realtek/rtw88/ps.c index bfa64c038f5f0a35d6721eba7f4976ce414588aa..c93da743681fc51954f635f4c1455f70a6e77196 100644 --- a/drivers/net/wireless/realtek/rtw88/ps.c +++ b/drivers/net/wireless/realtek/rtw88/ps.c @@ -19,14 +19,14 @@ static int rtw_ips_pwr_up(struct rtw_dev *rtwdev) rtw_err(rtwdev, "leave idle state failed\n"); rtw_set_channel(rtwdev); - clear_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags); return ret; } int rtw_enter_ips(struct rtw_dev *rtwdev) { - set_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags); + if (test_and_set_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags)) + return 0; rtw_coex_ips_notify(rtwdev, COEX_IPS_ENTER); @@ -50,6 +50,9 @@ int rtw_leave_ips(struct rtw_dev *rtwdev) { int ret; + if (!test_and_clear_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags)) + return 0; + rtw_hci_link_ps(rtwdev, false); ret = rtw_ips_pwr_up(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw88/regd.c b/drivers/net/wireless/realtek/rtw88/regd.c index 315c2b193e92cbef9cd541de3b757bc8d4eaae5a..2f547cbcf6da5214f07d1c656ea7d1fae6755548 100644 --- a/drivers/net/wireless/realtek/rtw88/regd.c +++ b/drivers/net/wireless/realtek/rtw88/regd.c @@ -479,6 +479,7 @@ void rtw_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request) rtw_dbg(rtwdev, RTW_DBG_REGD, "regd state: %d -> %d\n", rtwdev->regd.state, next_regd.state); + mutex_lock(&rtwdev->mutex); rtwdev->regd = next_regd; rtw_dbg_regd_dump(rtwdev, "get alpha2 %c%c from initiator %d: ", request->alpha2[0], @@ -487,6 +488,7 @@ void rtw_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request) rtw_phy_adaptivity_set_mode(rtwdev); rtw_phy_set_tx_power_level(rtwdev, hal->current_channel); + mutex_unlock(&rtwdev->mutex); } u8 rtw_regd_get(struct rtw_dev *rtwdev) diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c index 993bd6b1d723ea0e4f801e48fb20cf52496bf1f0..0a4f770fcbb7e3456b6d47985b7b89b86b1803ee 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c @@ -2720,7 +2720,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = { .max_power_index = 0x3f, .csi_buf_pg_num = 0, .band = RTW_BAND_2G, - .page_size = 128, + .page_size = TX_PAGE_SIZE, .dig_min = 0x20, .ht_supported = true, .vht_supported = false, @@ -2748,6 +2748,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = { .pwr_track_tbl = &rtw8723d_rtw_pwr_track_tbl, .iqk_threshold = 8, .ampdu_density = IEEE80211_HT_MPDU_DENSITY_16, + .max_scan_ie_len = IEEE80211_MAX_DATA_LEN, .coex_para_ver = 0x2007022f, .bt_desired_ver = 0x2f, diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c index 025262a8970eb0c4603dde48fc1731faf691eee1..9afdc5ce86b435914b24372250c994eb6b6a0c06 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c @@ -1898,7 +1898,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = { .max_power_index = 0x3f, .csi_buf_pg_num = 0, .band = RTW_BAND_2G | RTW_BAND_5G, - .page_size = 128, + .page_size = TX_PAGE_SIZE, .dig_min = 0x1c, .ht_supported = true, .vht_supported = true, @@ -1926,6 +1926,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = { .bfer_su_max_num = 2, .bfer_mu_max_num = 1, .ampdu_density = IEEE80211_HT_MPDU_DENSITY_2, + .max_scan_ie_len = IEEE80211_MAX_DATA_LEN, .coex_para_ver = 0x19092746, .bt_desired_ver = 0x46, diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c index 321848870561eb3263fa41875a6926d95edadd11..690e35c98f6e53e1b1f0d8b350a0c08c653a0d1d 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c @@ -2517,7 +2517,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = { .max_power_index = 0x3f, .csi_buf_pg_num = 0, .band = RTW_BAND_2G | RTW_BAND_5G, - .page_size = 128, + .page_size = TX_PAGE_SIZE, .dig_min = 0x1c, .ht_supported = true, .vht_supported = true, @@ -2549,6 +2549,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = { .l2h_th_ini_cs = 10 + EDCCA_IGI_BASE, .l2h_th_ini_ad = -14 + EDCCA_IGI_BASE, .ampdu_density = IEEE80211_HT_MPDU_DENSITY_2, + .max_scan_ie_len = IEEE80211_MAX_DATA_LEN, .coex_para_ver = 0x20070206, .bt_desired_ver = 0x6, diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c index 09f9e4adcf349d1daa0b8ce2a658e53a49a2cc34..fccb15dfb95950c0f86557dca551e2c5624e5a46 100644 --- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c +++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c @@ -5330,7 +5330,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = { .max_power_index = 0x7f, .csi_buf_pg_num = 50, .band = RTW_BAND_2G | RTW_BAND_5G, - .page_size = 128, + .page_size = TX_PAGE_SIZE, .dig_min = 0x20, .default_1ss_tx_path = BB_PATH_A, .path_div_supported = true, @@ -5375,6 +5375,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = { .wowlan_stub = &rtw_wowlan_stub_8822c, .max_sched_scan_ssids = 4, #endif + .max_scan_ie_len = (RTW_PROBE_PG_CNT - 1) * TX_PAGE_SIZE, .coex_para_ver = 0x22020720, .bt_desired_ver = 0x20, .scbd_support = true, diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c index 60d40a5c2c6a48aa20c0f069ae124f13653c6944..ab39245e9c2fda27a048814b91c7fb300dc5a151 100644 --- a/drivers/net/wireless/realtek/rtw88/tx.c +++ b/drivers/net/wireless/realtek/rtw88/tx.c @@ -384,7 +384,7 @@ void rtw_tx_pkt_info_update(struct rtw_dev *rtwdev, struct ieee80211_sta *sta, struct sk_buff *skb) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; struct rtw_sta_info *si; @@ -424,7 +424,7 @@ void rtw_tx_rsvd_page_pkt_info_update(struct rtw_dev *rtwdev, struct sk_buff *skb, enum rtw_rsvd_packet_type type) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; bool bmc; @@ -475,7 +475,7 @@ rtw_tx_write_data_rsvd_page_get(struct rtw_dev *rtwdev, struct rtw_tx_pkt_info *pkt_info, u8 *buf, u32 size) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct sk_buff *skb; u32 tx_pkt_desc_sz; u32 length; @@ -501,7 +501,7 @@ rtw_tx_write_data_h2c_get(struct rtw_dev *rtwdev, struct rtw_tx_pkt_info *pkt_info, u8 *buf, u32 size) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; struct sk_buff *skb; u32 tx_pkt_desc_sz; u32 length; diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c index 2c515af214e76e5bfecaeb4515545515b048acbe..cdfd66a85075aca310fe2069d1896650e3336583 100644 --- a/drivers/net/wireless/realtek/rtw88/util.c +++ b/drivers/net/wireless/realtek/rtw88/util.c @@ -23,7 +23,7 @@ EXPORT_SYMBOL(check_hw_ready); bool ltecoex_read_reg(struct rtw_dev *rtwdev, u16 offset, u32 *val) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_ltecoex_addr *ltecoex = chip->ltecoex_addr; if (!check_hw_ready(rtwdev, ltecoex->ctrl, LTECOEX_READY, 1)) @@ -37,7 +37,7 @@ bool ltecoex_read_reg(struct rtw_dev *rtwdev, u16 offset, u32 *val) bool ltecoex_reg_write(struct rtw_dev *rtwdev, u16 offset, u32 value) { - struct rtw_chip_info *chip = rtwdev->chip; + const struct rtw_chip_info *chip = rtwdev->chip; const struct rtw_ltecoex_addr *ltecoex = chip->ltecoex_addr; if (!check_hw_ready(rtwdev, ltecoex->ctrl, LTECOEX_READY, 1)) diff --git a/drivers/net/wireless/realtek/rtw89/Makefile b/drivers/net/wireless/realtek/rtw89/Makefile index 3006482d25c7746b30d7c4e6ca79ff724a50d031..a87f2aff4def2fef672cecb551f9081cfc7f74e2 100644 --- a/drivers/net/wireless/realtek/rtw89/Makefile +++ b/drivers/net/wireless/realtek/rtw89/Makefile @@ -12,6 +12,7 @@ rtw89_core-y += core.o \ sar.o \ coex.o \ ps.o \ + chan.o \ ser.o obj-$(CONFIG_RTW89_8852A) += rtw89_8852a.o diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c new file mode 100644 index 0000000000000000000000000000000000000000..a4f61c2f651236ac91e23597e0d58acf173c9726 --- /dev/null +++ b/drivers/net/wireless/realtek/rtw89/chan.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright(c) 2020-2022 Realtek Corporation + */ + +#include "chan.h" +#include "debug.h" + +static enum rtw89_subband rtw89_get_subband_type(enum rtw89_band band, + u8 center_chan) +{ + switch (band) { + default: + case RTW89_BAND_2G: + switch (center_chan) { + default: + case 1 ... 14: + return RTW89_CH_2G; + } + case RTW89_BAND_5G: + switch (center_chan) { + default: + case 36 ... 64: + return RTW89_CH_5G_BAND_1; + case 100 ... 144: + return RTW89_CH_5G_BAND_3; + case 149 ... 177: + return RTW89_CH_5G_BAND_4; + } + case RTW89_BAND_6G: + switch (center_chan) { + default: + case 1 ... 29: + return RTW89_CH_6G_BAND_IDX0; + case 33 ... 61: + return RTW89_CH_6G_BAND_IDX1; + case 65 ... 93: + return RTW89_CH_6G_BAND_IDX2; + case 97 ... 125: + return RTW89_CH_6G_BAND_IDX3; + case 129 ... 157: + return RTW89_CH_6G_BAND_IDX4; + case 161 ... 189: + return RTW89_CH_6G_BAND_IDX5; + case 193 ... 221: + return RTW89_CH_6G_BAND_IDX6; + case 225 ... 253: + return RTW89_CH_6G_BAND_IDX7; + } + } +} + +static enum rtw89_sc_offset rtw89_get_primary_chan_idx(enum rtw89_bandwidth bw, + u32 center_freq, + u32 primary_freq) +{ + u8 primary_chan_idx; + u32 offset; + + switch (bw) { + default: + case RTW89_CHANNEL_WIDTH_20: + primary_chan_idx = RTW89_SC_DONT_CARE; + break; + case RTW89_CHANNEL_WIDTH_40: + if (primary_freq > center_freq) + primary_chan_idx = RTW89_SC_20_UPPER; + else + primary_chan_idx = RTW89_SC_20_LOWER; + break; + case RTW89_CHANNEL_WIDTH_80: + case RTW89_CHANNEL_WIDTH_160: + if (primary_freq > center_freq) { + offset = (primary_freq - center_freq - 10) / 20; + primary_chan_idx = RTW89_SC_20_UPPER + offset * 2; + } else { + offset = (center_freq - primary_freq - 10) / 20; + primary_chan_idx = RTW89_SC_20_LOWER + offset * 2; + } + break; + } + + return primary_chan_idx; +} + +void rtw89_chan_create(struct rtw89_chan *chan, u8 center_chan, u8 primary_chan, + enum rtw89_band band, enum rtw89_bandwidth bandwidth) +{ + enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band); + u32 center_freq, primary_freq; + + memset(chan, 0, sizeof(*chan)); + chan->channel = center_chan; + chan->primary_channel = primary_chan; + chan->band_type = band; + chan->band_width = bandwidth; + + center_freq = ieee80211_channel_to_frequency(center_chan, nl_band); + primary_freq = ieee80211_channel_to_frequency(primary_chan, nl_band); + + chan->freq = center_freq; + chan->subband_type = rtw89_get_subband_type(band, center_chan); + chan->pri_ch_idx = rtw89_get_primary_chan_idx(bandwidth, center_freq, + primary_freq); +} + +bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx, + const struct rtw89_chan *new) +{ + struct rtw89_hal *hal = &rtwdev->hal; + struct rtw89_chan *chan = &hal->chan[idx]; + struct rtw89_chan_rcd *rcd = &hal->chan_rcd[idx]; + bool band_changed; + + rcd->prev_primary_channel = chan->primary_channel; + rcd->prev_band_type = chan->band_type; + band_changed = new->band_type != chan->band_type; + + *chan = *new; + return band_changed; +} + +static void __rtw89_config_entity_chandef(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx, + const struct cfg80211_chan_def *chandef, + bool from_stack) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + hal->chandef[idx] = *chandef; + + if (from_stack) + set_bit(idx, hal->entity_map); +} + +void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx, + const struct cfg80211_chan_def *chandef) +{ + __rtw89_config_entity_chandef(rtwdev, idx, chandef, true); +} + +static void rtw89_config_default_chandef(struct rtw89_dev *rtwdev) +{ + struct cfg80211_chan_def chandef = {0}; + + rtw89_get_default_chandef(&chandef); + __rtw89_config_entity_chandef(rtwdev, RTW89_SUB_ENTITY_0, &chandef, false); +} + +void rtw89_entity_init(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + bitmap_zero(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY); + rtw89_config_default_chandef(rtwdev); +} + +enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + enum rtw89_entity_mode mode; + u8 weight; + + weight = bitmap_weight(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY); + switch (weight) { + default: + rtw89_warn(rtwdev, "unknown ent chan weight: %d\n", weight); + bitmap_zero(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY); + fallthrough; + case 0: + rtw89_config_default_chandef(rtwdev); + fallthrough; + case 1: + mode = RTW89_ENTITY_MODE_SCC; + break; + } + + rtw89_set_entity_mode(rtwdev, mode); + return mode; +} + +int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_hal *hal = &rtwdev->hal; + struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv; + const struct rtw89_chip_info *chip = rtwdev->chip; + u8 idx; + + idx = find_first_zero_bit(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY); + if (idx >= chip->support_chanctx_num) + return -ENOENT; + + rtw89_config_entity_chandef(rtwdev, idx, &ctx->def); + rtw89_set_channel(rtwdev); + cfg->idx = idx; + return 0; +} + +void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_hal *hal = &rtwdev->hal; + struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv; + + clear_bit(cfg->idx, hal->entity_map); + rtw89_set_channel(rtwdev); +} + +void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx, + u32 changed) +{ + struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv; + u8 idx = cfg->idx; + + if (changed & IEEE80211_CHANCTX_CHANGE_WIDTH) { + rtw89_config_entity_chandef(rtwdev, idx, &ctx->def); + rtw89_set_channel(rtwdev); + } +} + +int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, + struct ieee80211_chanctx_conf *ctx) +{ + return 0; +} + +void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, + struct ieee80211_chanctx_conf *ctx) +{ +} diff --git a/drivers/net/wireless/realtek/rtw89/chan.h b/drivers/net/wireless/realtek/rtw89/chan.h new file mode 100644 index 0000000000000000000000000000000000000000..ecbd4503bead9bbc8f205749ea2b714f51ed497b --- /dev/null +++ b/drivers/net/wireless/realtek/rtw89/chan.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + * Copyright(c) 2020-2022 Realtek Corporation + */ + +#ifndef __RTW89_CHAN_H__ +#define __RTW89_CHAN_H__ + +#include "core.h" + +static inline bool rtw89_get_entity_state(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + return READ_ONCE(hal->entity_active); +} + +static inline void rtw89_set_entity_state(struct rtw89_dev *rtwdev, bool active) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + WRITE_ONCE(hal->entity_active, active); +} + +static inline +enum rtw89_entity_mode rtw89_get_entity_mode(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + return READ_ONCE(hal->entity_mode); +} + +static inline void rtw89_set_entity_mode(struct rtw89_dev *rtwdev, + enum rtw89_entity_mode mode) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + WRITE_ONCE(hal->entity_mode, mode); +} + +void rtw89_chan_create(struct rtw89_chan *chan, u8 center_chan, u8 primary_chan, + enum rtw89_band band, enum rtw89_bandwidth bandwidth); +bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx, + const struct rtw89_chan *new); +void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx, + const struct cfg80211_chan_def *chandef); +void rtw89_entity_init(struct rtw89_dev *rtwdev); +enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev); +int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx); +void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx); +void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev, + struct ieee80211_chanctx_conf *ctx, + u32 changed); +int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, + struct ieee80211_chanctx_conf *ctx); +void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif, + struct ieee80211_chanctx_conf *ctx); + +#endif diff --git a/drivers/net/wireless/realtek/rtw89/coex.c b/drivers/net/wireless/realtek/rtw89/coex.c index 683854bba217b352691d4b8ce9fcf461283e851c..bbdfa9ac203ccdf6bbc75532016a97c1a1998d4e 100644 --- a/drivers/net/wireless/realtek/rtw89/coex.c +++ b/drivers/net/wireless/realtek/rtw89/coex.c @@ -9,6 +9,7 @@ #include "ps.h" #include "reg.h" +#define RTW89_COEX_VERSION 0x06030013 #define FCXDEF_STEP 50 /* MUST <= FCXMAX_STEP and match with wl fw*/ enum btc_fbtc_tdma_template { @@ -77,21 +78,21 @@ static const struct rtw89_btc_fbtc_tdma t_def[] = { static const struct rtw89_btc_fbtc_slot s_def[] = { [CXST_OFF] = __DEF_FBTC_SLOT(100, 0x55555555, SLOT_MIX), - [CXST_B2W] = __DEF_FBTC_SLOT(5, 0x5a5a5a5a, SLOT_ISO), - [CXST_W1] = __DEF_FBTC_SLOT(70, 0x5a5a5a5a, SLOT_ISO), - [CXST_W2] = __DEF_FBTC_SLOT(70, 0x5a5a5aaa, SLOT_ISO), - [CXST_W2B] = __DEF_FBTC_SLOT(15, 0x5a5a5a5a, SLOT_ISO), - [CXST_B1] = __DEF_FBTC_SLOT(100, 0x55555555, SLOT_MIX), - [CXST_B2] = __DEF_FBTC_SLOT(7, 0x6a5a5a5a, SLOT_MIX), - [CXST_B3] = __DEF_FBTC_SLOT(5, 0x55555555, SLOT_MIX), - [CXST_B4] = __DEF_FBTC_SLOT(50, 0x55555555, SLOT_MIX), - [CXST_LK] = __DEF_FBTC_SLOT(20, 0x5a5a5a5a, SLOT_ISO), + [CXST_B2W] = __DEF_FBTC_SLOT(5, 0xea5a5a5a, SLOT_ISO), + [CXST_W1] = __DEF_FBTC_SLOT(70, 0xea5a5a5a, SLOT_ISO), + [CXST_W2] = __DEF_FBTC_SLOT(70, 0xea5a5aaa, SLOT_ISO), + [CXST_W2B] = __DEF_FBTC_SLOT(15, 0xea5a5a5a, SLOT_ISO), + [CXST_B1] = __DEF_FBTC_SLOT(100, 0xe5555555, SLOT_MIX), + [CXST_B2] = __DEF_FBTC_SLOT(7, 0xea5a5a5a, SLOT_MIX), + [CXST_B3] = __DEF_FBTC_SLOT(5, 0xe5555555, SLOT_MIX), + [CXST_B4] = __DEF_FBTC_SLOT(50, 0xe5555555, SLOT_MIX), + [CXST_LK] = __DEF_FBTC_SLOT(20, 0xea5a5a5a, SLOT_ISO), [CXST_BLK] = __DEF_FBTC_SLOT(250, 0x55555555, SLOT_MIX), - [CXST_E2G] = __DEF_FBTC_SLOT(20, 0x6a5a5a5a, SLOT_MIX), + [CXST_E2G] = __DEF_FBTC_SLOT(20, 0xea5a5a5a, SLOT_MIX), [CXST_E5G] = __DEF_FBTC_SLOT(20, 0xffffffff, SLOT_MIX), - [CXST_EBT] = __DEF_FBTC_SLOT(20, 0x55555555, SLOT_MIX), + [CXST_EBT] = __DEF_FBTC_SLOT(20, 0xe5555555, SLOT_MIX), [CXST_ENULL] = __DEF_FBTC_SLOT(7, 0xaaaaaaaa, SLOT_ISO), - [CXST_WLK] = __DEF_FBTC_SLOT(250, 0x6a5a6a5a, SLOT_MIX), + [CXST_WLK] = __DEF_FBTC_SLOT(250, 0xea5a5a5a, SLOT_MIX), [CXST_W1FDD] = __DEF_FBTC_SLOT(35, 0xfafafafa, SLOT_ISO), [CXST_B1FDD] = __DEF_FBTC_SLOT(100, 0xffffffff, SLOT_MIX), }; @@ -99,13 +100,13 @@ static const struct rtw89_btc_fbtc_slot s_def[] = { static const u32 cxtbl[] = { 0xffffffff, /* 0 */ 0xaaaaaaaa, /* 1 */ - 0x55555555, /* 2 */ - 0x66555555, /* 3 */ - 0x66556655, /* 4 */ + 0xe5555555, /* 2 */ + 0xee555555, /* 3 */ + 0xd5555555, /* 4 */ 0x5a5a5a5a, /* 5 */ - 0x5a5a5aaa, /* 6 */ - 0xaa5a5a5a, /* 7 */ - 0x6a5a5a5a, /* 8 */ + 0xfa5a5a5a, /* 6 */ + 0xda5a5a5a, /* 7 */ + 0xea5a5a5a, /* 8 */ 0x6a5a5aaa, /* 9 */ 0x6a5a6a5a, /* 10 */ 0x6a5a6aaa, /* 11 */ @@ -261,6 +262,12 @@ enum btc_cx_poicy_type { /* TDMA off + pri: WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo */ BTC_CXP_OFF_BWB1 = (BTC_CXP_OFF << 8) | 7, + /* TDMA off + pri: WL_Hi-Tx > BT, BT_Hi > other-WL > BT_Lo */ + BTC_CXP_OFF_BWB2 = (BTC_CXP_OFF << 8) | 8, + + /* TDMA off + pri: WL_Hi-Tx = BT */ + BTC_CXP_OFF_BWB3 = (BTC_CXP_OFF << 8) | 9, + /* TDMA off+Bcn-Protect + pri: WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo*/ BTC_CXP_OFFB_BWB0 = (BTC_CXP_OFFB << 8) | 0, @@ -270,6 +277,21 @@ enum btc_cx_poicy_type { /* TDMA off + Ext-Ctrl + pri: E2G-slot block all BT */ BTC_CXP_OFFE_DEF2 = (BTC_CXP_OFFE << 8) | 1, + /* TDMA off + Ext-Ctrl + pri: default */ + BTC_CXP_OFFE_2GBWISOB = (BTC_CXP_OFFE << 8) | 2, + + /* TDMA off + Ext-Ctrl + pri: E2G-slot block all BT */ + BTC_CXP_OFFE_2GISOB = (BTC_CXP_OFFE << 8) | 3, + + /* TDMA off + Ext-Ctrl + pri: E2G-slot WL > BT */ + BTC_CXP_OFFE_2GBWMIXB = (BTC_CXP_OFFE << 8) | 4, + + /* TDMA off + Ext-Ctrl + pri: E2G/EBT-slot WL > BT */ + BTC_CXP_OFFE_WL = (BTC_CXP_OFFE << 8) | 5, + + /* TDMA off + Ext-Ctrl + pri: default */ + BTC_CXP_OFFE_2GBWMIXB2 = (BTC_CXP_OFFE << 8) | 6, + /* TDMA Fix slot-0: W1:B1 = 30:30 */ BTC_CXP_FIX_TD3030 = (BTC_CXP_FIX << 8) | 0, @@ -300,6 +322,9 @@ enum btc_cx_poicy_type { /* TDMA Fix slot-9: W1:B1 = 40:20 */ BTC_CXP_FIX_TD4020 = (BTC_CXP_FIX << 8) | 9, + /* TDMA Fix slot-9: W1:B1 = 40:10 */ + BTC_CXP_FIX_TD4010ISO = (BTC_CXP_FIX << 8) | 10, + /* PS-TDMA Fix slot-0: W1:B1 = 30:30 */ BTC_CXP_PFIX_TD3030 = (BTC_CXP_PFIX << 8) | 0, @@ -322,25 +347,25 @@ enum btc_cx_poicy_type { BTC_CXP_PFIX_TDW1B1 = (BTC_CXP_PFIX << 8) | 6, /* TDMA Auto slot-0: W1:B1 = 50:200 */ - BTC_CXP_AUTO_TD50200 = (BTC_CXP_AUTO << 8) | 0, + BTC_CXP_AUTO_TD50B1 = (BTC_CXP_AUTO << 8) | 0, /* TDMA Auto slot-1: W1:B1 = 60:200 */ - BTC_CXP_AUTO_TD60200 = (BTC_CXP_AUTO << 8) | 1, + BTC_CXP_AUTO_TD60B1 = (BTC_CXP_AUTO << 8) | 1, /* TDMA Auto slot-2: W1:B1 = 20:200 */ - BTC_CXP_AUTO_TD20200 = (BTC_CXP_AUTO << 8) | 2, + BTC_CXP_AUTO_TD20B1 = (BTC_CXP_AUTO << 8) | 2, /* TDMA Auto slot-3: W1:B1 = user-define */ BTC_CXP_AUTO_TDW1B1 = (BTC_CXP_AUTO << 8) | 3, /* PS-TDMA Auto slot-0: W1:B1 = 50:200 */ - BTC_CXP_PAUTO_TD50200 = (BTC_CXP_PAUTO << 8) | 0, + BTC_CXP_PAUTO_TD50B1 = (BTC_CXP_PAUTO << 8) | 0, /* PS-TDMA Auto slot-1: W1:B1 = 60:200 */ - BTC_CXP_PAUTO_TD60200 = (BTC_CXP_PAUTO << 8) | 1, + BTC_CXP_PAUTO_TD60B1 = (BTC_CXP_PAUTO << 8) | 1, /* PS-TDMA Auto slot-2: W1:B1 = 20:200 */ - BTC_CXP_PAUTO_TD20200 = (BTC_CXP_PAUTO << 8) | 2, + BTC_CXP_PAUTO_TD20B1 = (BTC_CXP_PAUTO << 8) | 2, /* PS-TDMA Auto slot-3: W1:B1 = user-define */ BTC_CXP_PAUTO_TDW1B1 = (BTC_CXP_PAUTO << 8) | 3, @@ -412,7 +437,7 @@ enum btc_w2b_scoreboard { BTC_WSCB_TDMA = BIT(9), BTC_WSCB_FIX2M = BIT(10), BTC_WSCB_WLRFK = BIT(11), - BTC_WSCB_BTRFK_GNT = BIT(12), /* not used, use mailbox to inform BT */ + BTC_WSCB_RXSCAN_PRI = BIT(12), BTC_WSCB_BT_HILNA = BIT(13), BTC_WSCB_BTLOG = BIT(14), BTC_WSCB_ALL = GENMASK(23, 0), @@ -434,6 +459,16 @@ enum btc_wl_link_mode { BTC_WLINK_MAX }; +enum btc_wl_mrole_type { + BTC_WLMROLE_NONE = 0x0, + BTC_WLMROLE_STA_GC, + BTC_WLMROLE_STA_GC_NOA, + BTC_WLMROLE_STA_GO, + BTC_WLMROLE_STA_GO_NOA, + BTC_WLMROLE_STA_STA, + BTC_WLMROLE_MAX +}; + enum btc_bt_hid_type { BTC_HID_218 = BIT(0), BTC_HID_418 = BIT(1), @@ -460,6 +495,11 @@ enum btc_gnt_state { BTC_GNT_MAX }; +enum btc_ctr_path { + BTC_CTRL_BY_BT = 0, + BTC_CTRL_BY_WL +}; + enum btc_wl_max_tx_time { BTC_MAX_TX_TIME_L1 = 500, BTC_MAX_TX_TIME_L2 = 1000, @@ -531,6 +571,7 @@ enum btc_reason_and_action { #define BTC_FREERUN_ANTISO_MIN 30 #define BTC_TDMA_BTHID_MAX 2 #define BTC_BLINK_NOCONNECT 0 +#define BTC_B1_MAX 250 /* unit ms */ static void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason); @@ -551,8 +592,10 @@ static void _send_fw_cmd(struct rtw89_dev *rtwdev, u8 h2c_class, u8 h2c_func, "[BTC], %s(): return by btc not init!!\n", __func__); pfwinfo->cnt_h2c_fail++; return; - } else if ((wl->status.map.rf_off_pre == 1 && wl->status.map.rf_off == 1) || - (wl->status.map.lps_pre == 1 && wl->status.map.lps == 1)) { + } else if ((wl->status.map.rf_off_pre == BTC_LPS_RF_OFF && + wl->status.map.rf_off == BTC_LPS_RF_OFF) || + (wl->status.map.lps_pre == BTC_LPS_RF_OFF && + wl->status.map.lps == BTC_LPS_RF_OFF)) { rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): return by wl off!!\n", __func__); pfwinfo->cnt_h2c_fail++; @@ -616,8 +659,6 @@ static void _reset_btc_var(struct rtw89_dev *rtwdev, u8 type) memset(&btc->mdinfo, 0, sizeof(btc->mdinfo)); } -#define BTC_FWINFO_BUF 1024 - #define BTC_RPT_HDR_SIZE 3 #define BTC_CHK_WLSLOT_DRIFT_MAX 15 #define BTC_CHK_HANG_MAX 3 @@ -869,18 +910,24 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, struct rtw89_btc_btf_fwinfo *pfwinfo, u8 *prptbuf, u32 index) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &btc->dm; struct rtw89_btc_rpt_cmn_info *pcinfo = NULL; struct rtw89_btc_wl_info *wl = &btc->cx.wl; - struct rtw89_btc_fbtc_rpt_ctrl *prpt = NULL; + struct rtw89_btc_bt_info *bt = &btc->cx.bt; + struct rtw89_btc_fbtc_rpt_ctrl *prpt; + struct rtw89_btc_fbtc_rpt_ctrl_v1 *prpt_v1; struct rtw89_btc_fbtc_cysta *pcysta_le32 = NULL; + struct rtw89_btc_fbtc_cysta_v1 *pcysta_v1 = NULL; struct rtw89_btc_fbtc_cysta_cpu pcysta[1]; struct rtw89_btc_prpt *btc_prpt = NULL; struct rtw89_btc_fbtc_slot *rtp_slot = NULL; - u8 rpt_type = 0, *rpt_content = NULL, *pfinfo = NULL; - u16 wl_slot_set = 0; + void *rpt_content = NULL, *pfinfo = NULL; + u8 rpt_type = 0; + u16 wl_slot_set = 0, wl_slot_real = 0; u32 trace_step = btc->ctrl.trace_step, rpt_len = 0, diff_t; + u32 cnt_leak_slot = 0, bt_slot_real = 0, cnt_rx_imr = 0; u8 i; rtw89_debug(rtwdev, RTW89_DBG_BTC, @@ -904,100 +951,129 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, switch (rpt_type) { case BTC_RPT_TYPE_CTRL: pcinfo = &pfwinfo->rpt_ctrl.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_ctrl.finfo); - pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo); - pcinfo->req_fver = BTCRPT_VER; + if (chip->chip_id == RTL8852A) { + pfinfo = &pfwinfo->rpt_ctrl.finfo; + pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo); + } else { + pfinfo = &pfwinfo->rpt_ctrl.finfo_v1; + pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo_v1); + } + pcinfo->req_fver = chip->fcxbtcrpt_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_TDMA: pcinfo = &pfwinfo->rpt_fbtc_tdma.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_tdma.finfo); - pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo); - pcinfo->req_fver = FCXTDMA_VER; + if (chip->chip_id == RTL8852A) { + pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo); + } else { + pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo_v1; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo_v1); + } + pcinfo->req_fver = chip->fcxtdma_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_SLOT: pcinfo = &pfwinfo->rpt_fbtc_slots.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_slots.finfo); + pfinfo = &pfwinfo->rpt_fbtc_slots.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_slots.finfo); - pcinfo->req_fver = FCXSLOTS_VER; + pcinfo->req_fver = chip->fcxslots_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_CYSTA: pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_cysta.finfo); - pcysta_le32 = &pfwinfo->rpt_fbtc_cysta.finfo; - rtw89_btc_fbtc_cysta_to_cpu(pcysta_le32, pcysta); - pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo); - pcinfo->req_fver = FCXCYSTA_VER; + if (chip->chip_id == RTL8852A) { + pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo; + pcysta_le32 = &pfwinfo->rpt_fbtc_cysta.finfo; + rtw89_btc_fbtc_cysta_to_cpu(pcysta_le32, pcysta); + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo); + } else { + pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo_v1; + pcysta_v1 = &pfwinfo->rpt_fbtc_cysta.finfo_v1; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo_v1); + } + pcinfo->req_fver = chip->fcxcysta_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_STEP: pcinfo = &pfwinfo->rpt_fbtc_step.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_step.finfo); - pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.step[0]) * - trace_step + 8; - pcinfo->req_fver = FCXSTEP_VER; + if (chip->chip_id == RTL8852A) { + pfinfo = &pfwinfo->rpt_fbtc_step.finfo; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.step[0]) * + trace_step + + offsetof(struct rtw89_btc_fbtc_steps, step); + } else { + pfinfo = &pfwinfo->rpt_fbtc_step.finfo_v1; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo_v1.step[0]) * + trace_step + + offsetof(struct rtw89_btc_fbtc_steps_v1, step); + } + pcinfo->req_fver = chip->fcxstep_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_NULLSTA: pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_nullsta.finfo); - pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo); - pcinfo->req_fver = FCXNULLSTA_VER; + if (chip->chip_id == RTL8852A) { + pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo); + } else { + pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo_v1; + pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo_v1); + } + pcinfo->req_fver = chip->fcxnullsta_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_MREG: pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_mregval.finfo); + pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo); - pcinfo->req_fver = FCXMREG_VER; + pcinfo->req_fver = chip->fcxmreg_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_GPIO_DBG: pcinfo = &pfwinfo->rpt_fbtc_gpio_dbg.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_gpio_dbg.finfo); + pfinfo = &pfwinfo->rpt_fbtc_gpio_dbg.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_gpio_dbg.finfo); - pcinfo->req_fver = FCXGPIODBG_VER; + pcinfo->req_fver = chip->fcxgpiodbg_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_BT_VER: pcinfo = &pfwinfo->rpt_fbtc_btver.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btver.finfo); + pfinfo = &pfwinfo->rpt_fbtc_btver.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btver.finfo); - pcinfo->req_fver = FCX_BTVER_VER; + pcinfo->req_fver = chip->fcxbtver_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_BT_SCAN: pcinfo = &pfwinfo->rpt_fbtc_btscan.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btscan.finfo); + pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo); - pcinfo->req_fver = FCX_BTSCAN_VER; + pcinfo->req_fver = chip->fcxbtscan_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_BT_AFH: pcinfo = &pfwinfo->rpt_fbtc_btafh.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btafh.finfo); + pfinfo = &pfwinfo->rpt_fbtc_btafh.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btafh.finfo); - pcinfo->req_fver = FCX_BTAFH_VER; + pcinfo->req_fver = chip->fcxbtafh_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; case BTC_RPT_TYPE_BT_DEVICE: pcinfo = &pfwinfo->rpt_fbtc_btdev.cinfo; - pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btdev.finfo); + pfinfo = &pfwinfo->rpt_fbtc_btdev.finfo; pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btdev.finfo); - pcinfo->req_fver = FCX_BTDEVINFO_VER; + pcinfo->req_fver = chip->fcxbtdevinfo_ver; pcinfo->rx_len = rpt_len; pcinfo->rx_cnt++; break; @@ -1026,7 +1102,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, memcpy(pfinfo, rpt_content, pcinfo->req_len); pcinfo->valid = 1; - if (rpt_type == BTC_RPT_TYPE_TDMA) { + if (rpt_type == BTC_RPT_TYPE_TDMA && chip->chip_id == RTL8852A) { rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): check %d %zu\n", __func__, BTC_DCNT_TDMA_NONSYNC, sizeof(dm->tdma_now)); @@ -1039,7 +1115,8 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, dm->tdma_now.type, dm->tdma_now.rxflctrl, dm->tdma_now.txpause, dm->tdma_now.wtgle_n, dm->tdma_now.leak_n, dm->tdma_now.ext_ctrl, - dm->tdma_now.rsvd0, dm->tdma_now.rsvd1); + dm->tdma_now.rxflctrl_role, + dm->tdma_now.option_ctrl); rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): %d rpt_fbtc_tdma %x %x %x %x %x %x %x %x\n", @@ -1050,14 +1127,46 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, pfwinfo->rpt_fbtc_tdma.finfo.wtgle_n, pfwinfo->rpt_fbtc_tdma.finfo.leak_n, pfwinfo->rpt_fbtc_tdma.finfo.ext_ctrl, - pfwinfo->rpt_fbtc_tdma.finfo.rsvd0, - pfwinfo->rpt_fbtc_tdma.finfo.rsvd1); + pfwinfo->rpt_fbtc_tdma.finfo.rxflctrl_role, + pfwinfo->rpt_fbtc_tdma.finfo.option_ctrl); } _chk_btc_err(rtwdev, BTC_DCNT_TDMA_NONSYNC, memcmp(&dm->tdma_now, &pfwinfo->rpt_fbtc_tdma.finfo, sizeof(dm->tdma_now))); + } else if (rpt_type == BTC_RPT_TYPE_TDMA) { + rtw89_debug(rtwdev, RTW89_DBG_BTC, + "[BTC], %s(): check %d %zu\n", __func__, + BTC_DCNT_TDMA_NONSYNC, sizeof(dm->tdma_now)); + + if (memcmp(&dm->tdma_now, &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma, + sizeof(dm->tdma_now)) != 0) { + rtw89_debug(rtwdev, RTW89_DBG_BTC, + "[BTC], %s(): %d tdma_now %x %x %x %x %x %x %x %x\n", + __func__, BTC_DCNT_TDMA_NONSYNC, + dm->tdma_now.type, dm->tdma_now.rxflctrl, + dm->tdma_now.txpause, dm->tdma_now.wtgle_n, + dm->tdma_now.leak_n, dm->tdma_now.ext_ctrl, + dm->tdma_now.rxflctrl_role, + dm->tdma_now.option_ctrl); + rtw89_debug(rtwdev, RTW89_DBG_BTC, + "[BTC], %s(): %d rpt_fbtc_tdma %x %x %x %x %x %x %x %x\n", + __func__, BTC_DCNT_TDMA_NONSYNC, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.type, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.rxflctrl, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.txpause, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.wtgle_n, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.leak_n, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.ext_ctrl, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.rxflctrl_role, + pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.option_ctrl); + } + + _chk_btc_err(rtwdev, BTC_DCNT_TDMA_NONSYNC, + memcmp(&dm->tdma_now, + &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma, + sizeof(dm->tdma_now))); } if (rpt_type == BTC_RPT_TYPE_SLOT) { @@ -1097,7 +1206,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, sizeof(dm->slot_now))); } - if (rpt_type == BTC_RPT_TYPE_CYSTA && + if (rpt_type == BTC_RPT_TYPE_CYSTA && chip->chip_id == RTL8852A && pcysta->cycles >= BTC_CYSTA_CHK_PERIOD) { /* Check Leak-AP */ if (pcysta->slot_cnt[CXST_LK] != 0 && @@ -1120,16 +1229,55 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, } _chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_W1]); - _chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_W1]); + _chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_B1]); _chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE, (u32)pcysta->cycles); + } else if (rpt_type == BTC_RPT_TYPE_CYSTA && pcysta_v1 && + le16_to_cpu(pcysta_v1->cycles) >= BTC_CYSTA_CHK_PERIOD) { + cnt_leak_slot = le32_to_cpu(pcysta_v1->slot_cnt[CXST_LK]); + cnt_rx_imr = le32_to_cpu(pcysta_v1->leak_slot.cnt_rximr); + /* Check Leak-AP */ + if (cnt_leak_slot != 0 && cnt_rx_imr != 0 && + dm->tdma_now.rxflctrl) { + if (cnt_leak_slot < BTC_LEAK_AP_TH * cnt_rx_imr) + dm->leak_ap = 1; + } + + /* Check diff time between real WL slot and W1 slot */ + if (dm->tdma_now.type == CXTDMA_OFF) { + wl_slot_set = le16_to_cpu(dm->slot_now[CXST_W1].dur); + wl_slot_real = le16_to_cpu(pcysta_v1->cycle_time.tavg[CXT_WL]); + if (wl_slot_real > wl_slot_set) { + diff_t = wl_slot_real - wl_slot_set; + _chk_btc_err(rtwdev, BTC_DCNT_WL_SLOT_DRIFT, diff_t); + } + } + + /* Check diff time between real BT slot and EBT/E5G slot */ + if (dm->tdma_now.type == CXTDMA_OFF && + dm->tdma_now.ext_ctrl == CXECTL_EXT && + btc->bt_req_len != 0) { + bt_slot_real = le16_to_cpu(pcysta_v1->cycle_time.tavg[CXT_BT]); + + if (btc->bt_req_len > bt_slot_real) { + diff_t = btc->bt_req_len - bt_slot_real; + _chk_btc_err(rtwdev, BTC_DCNT_BT_SLOT_DRIFT, diff_t); + } + } + + _chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, + le32_to_cpu(pcysta_v1->slot_cnt[CXST_W1])); + _chk_btc_err(rtwdev, BTC_DCNT_B1_FREEZE, + le32_to_cpu(pcysta_v1->slot_cnt[CXST_B1])); + _chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE, + (u32)le16_to_cpu(pcysta_v1->cycles)); } - if (rpt_type == BTC_RPT_TYPE_CTRL) { + if (rpt_type == BTC_RPT_TYPE_CTRL && chip->chip_id == RTL8852A) { prpt = &pfwinfo->rpt_ctrl.finfo; btc->fwinfo.rpt_en_map = prpt->rpt_enable; wl->ver_info.fw_coex = prpt->wl_fw_coex_ver; wl->ver_info.fw = prpt->wl_fw_ver; - dm->wl_fw_cx_offload = !!(prpt->wl_fw_cx_offload); + dm->wl_fw_cx_offload = !!prpt->wl_fw_cx_offload; _chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE, pfwinfo->event[BTF_EVNT_RPT]); @@ -1142,6 +1290,33 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev, btc->cx.cnt_bt[BTC_BCNT_POLUT] = rtw89_mac_get_plt_cnt(rtwdev, RTW89_MAC_0); } + } else if (rpt_type == BTC_RPT_TYPE_CTRL) { + prpt_v1 = &pfwinfo->rpt_ctrl.finfo_v1; + btc->fwinfo.rpt_en_map = le32_to_cpu(prpt_v1->rpt_info.en); + wl->ver_info.fw_coex = le32_to_cpu(prpt_v1->wl_fw_info.cx_ver); + wl->ver_info.fw = le32_to_cpu(prpt_v1->wl_fw_info.fw_ver); + dm->wl_fw_cx_offload = !!le32_to_cpu(prpt_v1->wl_fw_info.cx_offload); + + for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++) + memcpy(&dm->gnt.band[i], &prpt_v1->gnt_val[i], + sizeof(dm->gnt.band[i])); + + btc->cx.cnt_bt[BTC_BCNT_HIPRI_TX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_HI_TX]); + btc->cx.cnt_bt[BTC_BCNT_HIPRI_RX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_HI_RX]); + btc->cx.cnt_bt[BTC_BCNT_LOPRI_TX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_LO_TX]); + btc->cx.cnt_bt[BTC_BCNT_LOPRI_RX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_LO_RX]); + btc->cx.cnt_bt[BTC_BCNT_POLUT] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_POLLUTED]); + + _chk_btc_err(rtwdev, BTC_DCNT_BTCNT_FREEZE, 0); + _chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE, + pfwinfo->event[BTF_EVNT_RPT]); + + if (le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0) + bt->rfk_info.map.timeout = 1; + else + bt->rfk_info.map.timeout = 0; + + dm->error.map.bt_rfk_timeout = bt->rfk_info.map.timeout; } if (rpt_type >= BTC_RPT_TYPE_BT_VER && @@ -1155,6 +1330,7 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev, struct rtw89_btc_btf_fwinfo *pfwinfo, u8 *pbuf, u32 buf_len) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc_prpt *btc_prpt = NULL; u32 index = 0, rpt_len = 0; @@ -1164,7 +1340,7 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev, while (pbuf) { btc_prpt = (struct rtw89_btc_prpt *)&pbuf[index]; - if (index + 2 >= BTC_FWINFO_BUF) + if (index + 2 >= chip->btc_fwinfo_buf) break; /* At least 3 bytes: type(1) & len(2) */ rpt_len = le16_to_cpu(btc_prpt->len); @@ -1182,10 +1358,12 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev, static void _append_tdma(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &btc->dm; - struct rtw89_btc_btf_tlv *tlv = NULL; - struct rtw89_btc_fbtc_tdma *v = NULL; + struct rtw89_btc_btf_tlv *tlv; + struct rtw89_btc_fbtc_tdma *v; + struct rtw89_btc_fbtc_tdma_v1 *v1; u16 len = btc->policy_len; if (!btc->update_policy_force && @@ -1197,12 +1375,19 @@ static void _append_tdma(struct rtw89_dev *rtwdev) } tlv = (struct rtw89_btc_btf_tlv *)&btc->policy[len]; - v = (struct rtw89_btc_fbtc_tdma *)&tlv->val[0]; tlv->type = CXPOLICY_TDMA; - tlv->len = sizeof(*v); - - memcpy(v, &dm->tdma, sizeof(*v)); - btc->policy_len += BTC_TLV_HDR_LEN + sizeof(*v); + if (chip->chip_id == RTL8852A) { + v = (struct rtw89_btc_fbtc_tdma *)&tlv->val[0]; + tlv->len = sizeof(*v); + memcpy(v, &dm->tdma, sizeof(*v)); + btc->policy_len += BTC_TLV_HDR_LEN + sizeof(*v); + } else { + tlv->len = sizeof(*v1); + v1 = (struct rtw89_btc_fbtc_tdma_v1 *)&tlv->val[0]; + v1->fver = chip->fcxtdma_ver; + v1->tdma = dm->tdma; + btc->policy_len += BTC_TLV_HDR_LEN + sizeof(*v1); + } rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): type:%d, rxflctrl=%d, txpause=%d, wtgle_n=%d, leak_n=%d, ext_ctrl=%d\n", @@ -1408,12 +1593,17 @@ static void _fw_set_policy(struct rtw89_dev *rtwdev, u16 policy_type, static void _fw_set_drv_info(struct rtw89_dev *rtwdev, u8 type) { + const struct rtw89_chip_info *chip = rtwdev->chip; + switch (type) { case CXDRVINFO_INIT: rtw89_fw_h2c_cxdrv_init(rtwdev); break; case CXDRVINFO_ROLE: - rtw89_fw_h2c_cxdrv_role(rtwdev); + if (chip->chip_id == RTL8852A) + rtw89_fw_h2c_cxdrv_role(rtwdev); + else + rtw89_fw_h2c_cxdrv_role_v1(rtwdev); break; case CXDRVINFO_CTRL: rtw89_fw_h2c_cxdrv_ctrl(rtwdev); @@ -1448,7 +1638,7 @@ void btc_fw_event(struct rtw89_dev *rtwdev, u8 evt_id, void *data, u32 len) } } -static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state) +static void _set_gnt(struct rtw89_dev *rtwdev, u8 phy_map, u8 wl_state, u8 bt_state) { struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &btc->dm; @@ -1462,7 +1652,7 @@ static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state) if (!(phy_map & BIT(i))) continue; - switch (state) { + switch (wl_state) { case BTC_GNT_HW: g[i].gnt_wl_sw_en = 0; g[i].gnt_wl = 0; @@ -1476,6 +1666,21 @@ static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state) g[i].gnt_wl = 1; break; } + + switch (bt_state) { + case BTC_GNT_HW: + g[i].gnt_bt_sw_en = 0; + g[i].gnt_bt = 0; + break; + case BTC_GNT_SW_LO: + g[i].gnt_bt_sw_en = 1; + g[i].gnt_bt = 0; + break; + case BTC_GNT_SW_HI: + g[i].gnt_bt_sw_en = 1; + g[i].gnt_bt = 1; + break; + } } rtw89_chip_mac_cfg_gnt(rtwdev, &dm->gnt); @@ -1534,6 +1739,7 @@ static void _set_wl_tx_power(struct rtw89_dev *rtwdev, u32 level) static void _set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; @@ -1546,6 +1752,8 @@ static void _set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level) rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): level = %d\n", __func__, level); + + chip->ops->btc_set_wl_rx_gain(rtwdev, level); } static void _set_bt_tx_power(struct rtw89_dev *rtwdev, u8 level) @@ -1683,28 +1891,45 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev) struct rtw89_btc_bt_info *bt = &btc->cx.bt; struct rtw89_btc_bt_link_info *b = &bt->link_info; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; + struct rtw89_btc_wl_active_role *r; + struct rtw89_btc_wl_active_role_v1 *r1; u8 en = 0, i, ch = 0, bw = 0; + u8 mode, connect_cnt; if (btc->ctrl.manual || wl->status.map.scan) return; - /* TODO if include module->ant.type == BTC_ANT_SHARED */ + if (chip->chip_id == RTL8852A) { + mode = wl_rinfo->link_mode; + connect_cnt = wl_rinfo->connect_cnt; + } else { + mode = wl_rinfo_v1->link_mode; + connect_cnt = wl_rinfo_v1->connect_cnt; + } + if (wl->status.map.rf_off || bt->whql_test || - wl_rinfo->link_mode == BTC_WLINK_NOLINK || - wl_rinfo->link_mode == BTC_WLINK_5G || - wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX) { + mode == BTC_WLINK_NOLINK || mode == BTC_WLINK_5G || + connect_cnt > BTC_TDMA_WLROLE_MAX) { en = false; - } else if (wl_rinfo->link_mode == BTC_WLINK_2G_MCC || - wl_rinfo->link_mode == BTC_WLINK_2G_SCC) { + } else if (mode == BTC_WLINK_2G_MCC || mode == BTC_WLINK_2G_SCC) { en = true; /* get p2p channel */ for (i = 0; i < RTW89_PORT_NUM; i++) { - if (wl_rinfo->active_role[i].role == - RTW89_WIFI_ROLE_P2P_GO || - wl_rinfo->active_role[i].role == - RTW89_WIFI_ROLE_P2P_CLIENT) { - ch = wl_rinfo->active_role[i].ch; - bw = wl_rinfo->active_role[i].bw; + r = &wl_rinfo->active_role[i]; + r1 = &wl_rinfo_v1->active_role_v1[i]; + + if (chip->chip_id == RTL8852A && + (r->role == RTW89_WIFI_ROLE_P2P_GO || + r->role == RTW89_WIFI_ROLE_P2P_CLIENT)) { + ch = r->ch; + bw = r->bw; + break; + } else if (chip->chip_id != RTL8852A && + (r1->role == RTW89_WIFI_ROLE_P2P_GO || + r1->role == RTW89_WIFI_ROLE_P2P_CLIENT)) { + ch = r1->ch; + bw = r1->bw; break; } } @@ -1712,10 +1937,18 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev) en = true; /* get 2g channel */ for (i = 0; i < RTW89_PORT_NUM; i++) { - if (wl_rinfo->active_role[i].connected && - wl_rinfo->active_role[i].band == RTW89_BAND_2G) { - ch = wl_rinfo->active_role[i].ch; - bw = wl_rinfo->active_role[i].bw; + r = &wl_rinfo->active_role[i]; + r1 = &wl_rinfo_v1->active_role_v1[i]; + + if (chip->chip_id == RTL8852A && + r->connected && r->band == RTW89_BAND_2G) { + ch = r->ch; + bw = r->bw; + break; + } else if (chip->chip_id != RTL8852A && + r1->connected && r1->band == RTW89_BAND_2G) { + ch = r1->ch; + bw = r1->bw; break; } } @@ -1768,6 +2001,7 @@ static bool _check_freerun(struct rtw89_dev *rtwdev) struct rtw89_btc_wl_info *wl = &btc->cx.wl; struct rtw89_btc_bt_info *bt = &btc->cx.bt; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; struct rtw89_btc_bt_link_info *bt_linfo = &bt->link_info; struct rtw89_btc_bt_hid_desc *hid = &bt_linfo->hid_desc; @@ -1777,7 +2011,8 @@ static bool _check_freerun(struct rtw89_dev *rtwdev) } /* The below is dedicated antenna case */ - if (wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX) { + if (wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX || + wl_rinfo_v1->connect_cnt > BTC_TDMA_WLROLE_MAX) { btc->dm.trx_para_level = 5; return true; } @@ -1826,6 +2061,7 @@ static bool _check_freerun(struct rtw89_dev *rtwdev) } #define _tdma_set_flctrl(btc, flc) ({(btc)->dm.tdma.rxflctrl = flc; }) +#define _tdma_set_flctrl_role(btc, role) ({(btc)->dm.tdma.rxflctrl_role = role; }) #define _tdma_set_tog(btc, wtg) ({(btc)->dm.tdma.wtgle_n = wtg; }) #define _tdma_set_lek(btc, lek) ({(btc)->dm.tdma.leak_n = lek; }) @@ -1903,6 +2139,15 @@ union btc_btinfo { static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, enum btc_reason_and_action action) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + chip->ops->btc_set_policy(rtwdev, policy_type); + _fw_set_policy(rtwdev, policy_type, action); +} + +#define BTC_B1_MAX 250 /* unit ms */ +void rtw89_btc_set_policy(struct rtw89_dev *rtwdev, u16 policy_type) { struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &btc->dm; @@ -1964,6 +2209,9 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, case BTC_CXP_OFF_BWB1: _slot_set_tbl(btc, CXST_OFF, cxtbl[8]); break; + case BTC_CXP_OFF_BWB3: + _slot_set_tbl(btc, CXST_OFF, cxtbl[6]); + break; } break; case BTC_CXP_OFFB: /* TDMA off + beacon protect */ @@ -2080,17 +2328,17 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, _write_scbd(rtwdev, BTC_WSCB_TDMA, true); *t = t_def[CXTD_AUTO]; switch (policy_type) { - case BTC_CXP_AUTO_TD50200: - _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_AUTO_TD50B1: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; - case BTC_CXP_AUTO_TD60200: - _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_AUTO_TD60B1: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; - case BTC_CXP_AUTO_TD20200: - _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_AUTO_TD20B1: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO_TDW1B1: /* W1:B1 = user-define */ _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], @@ -2104,17 +2352,17 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, _write_scbd(rtwdev, BTC_WSCB_TDMA, true); *t = t_def[CXTD_PAUTO]; switch (policy_type) { - case BTC_CXP_PAUTO_TD50200: - _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_PAUTO_TD50B1: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; - case BTC_CXP_PAUTO_TD60200: - _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_PAUTO_TD60B1: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; - case BTC_CXP_PAUTO_TD20200: - _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); + case BTC_CXP_PAUTO_TD20B1: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO_TDW1B1: _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], @@ -2129,29 +2377,29 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, *t = t_def[CXTD_AUTO2]; switch (policy_type) { case BTC_CXP_AUTO2_TD3050: - _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO2_TD3070: - _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO2_TD5050: - _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO2_TD6060: - _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO2_TD2080: - _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_AUTO2_TDW1B4: /* W1:B1 = user-define */ _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], @@ -2166,29 +2414,29 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, *t = t_def[CXTD_PAUTO2]; switch (policy_type) { case BTC_CXP_PAUTO2_TD3050: - _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO2_TD3070: - _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO2_TD5050: - _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO2_TD6060: - _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO2_TD2080: - _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); - _slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX); - _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); break; case BTC_CXP_PAUTO2_TDW1B4: /* W1:B1 = user-define */ _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], @@ -2199,62 +2447,402 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type, } break; } - - _fw_set_policy(rtwdev, policy_type, action); } +EXPORT_SYMBOL(rtw89_btc_set_policy); -static void _set_gnt_bt(struct rtw89_dev *rtwdev, u8 phy_map, u8 state) +void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type) { struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &btc->dm; - struct rtw89_mac_ax_gnt *g = dm->gnt.band; - u8 i; + struct rtw89_btc_fbtc_tdma *t = &dm->tdma; + struct rtw89_btc_fbtc_slot *s = dm->slot; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &btc->cx.wl.role_info_v1; + struct rtw89_btc_bt_hid_desc *hid = &btc->cx.bt.link_info.hid_desc; + struct rtw89_btc_bt_hfp_desc *hfp = &btc->cx.bt.link_info.hfp_desc; + u8 type, null_role; + u32 tbl_w1, tbl_b1, tbl_b4; - if (phy_map > BTC_PHY_ALL) - return; + type = FIELD_GET(BTC_CXP_MASK, policy_type); - for (i = 0; i < RTW89_PHY_MAX; i++) { - if (!(phy_map & BIT(i))) - continue; + if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { + if (btc->cx.wl.status.map._4way) + tbl_w1 = cxtbl[1]; + else if (hid->exist && hid->type == BTC_HID_218) + tbl_w1 = cxtbl[7]; /* Ack/BA no break bt Hi-Pri-rx */ + else + tbl_w1 = cxtbl[8]; - switch (state) { - case BTC_GNT_HW: - g[i].gnt_bt_sw_en = 0; - g[i].gnt_bt = 0; - break; - case BTC_GNT_SW_LO: - g[i].gnt_bt_sw_en = 1; - g[i].gnt_bt = 0; - break; - case BTC_GNT_SW_HI: - g[i].gnt_bt_sw_en = 1; - g[i].gnt_bt = 1; - break; + if (dm->leak_ap && + (type == BTC_CXP_PFIX || type == BTC_CXP_PAUTO2)) { + tbl_b1 = cxtbl[3]; + tbl_b4 = cxtbl[3]; + } else if (hid->exist && hid->type == BTC_HID_218) { + tbl_b1 = cxtbl[4]; /* Ack/BA no break bt Hi-Pri-rx */ + tbl_b4 = cxtbl[4]; + } else { + tbl_b1 = cxtbl[2]; + tbl_b4 = cxtbl[2]; } + } else { + tbl_w1 = cxtbl[16]; + tbl_b1 = cxtbl[17]; + tbl_b4 = cxtbl[17]; } - rtw89_chip_mac_cfg_gnt(rtwdev, &dm->gnt); -} + btc->bt_req_en = false; -static void _set_bt_plut(struct rtw89_dev *rtwdev, u8 phy_map, - u8 tx_val, u8 rx_val) -{ - struct rtw89_mac_ax_plt plt; + switch (type) { + case BTC_CXP_USERDEF0: + btc->update_policy_force = true; + *t = t_def[CXTD_OFF]; + s[CXST_OFF] = s_def[CXST_OFF]; + _slot_set_tbl(btc, CXST_OFF, cxtbl[2]); + break; + case BTC_CXP_OFF: /* TDMA off */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, false); + *t = t_def[CXTD_OFF]; + s[CXST_OFF] = s_def[CXST_OFF]; - plt.band = RTW89_MAC_0; - plt.tx = tx_val; - plt.rx = rx_val; + switch (policy_type) { + case BTC_CXP_OFF_BT: + _slot_set_tbl(btc, CXST_OFF, cxtbl[2]); + break; + case BTC_CXP_OFF_WL: + _slot_set_tbl(btc, CXST_OFF, cxtbl[1]); + break; + case BTC_CXP_OFF_EQ0: + _slot_set_tbl(btc, CXST_OFF, cxtbl[0]); + break; + case BTC_CXP_OFF_EQ1: + _slot_set_tbl(btc, CXST_OFF, cxtbl[16]); + break; + case BTC_CXP_OFF_EQ2: + _slot_set_tbl(btc, CXST_OFF, cxtbl[17]); + break; + case BTC_CXP_OFF_EQ3: + _slot_set_tbl(btc, CXST_OFF, cxtbl[18]); + break; + case BTC_CXP_OFF_BWB0: + _slot_set_tbl(btc, CXST_OFF, cxtbl[5]); + break; + case BTC_CXP_OFF_BWB1: + _slot_set_tbl(btc, CXST_OFF, cxtbl[8]); + break; + case BTC_CXP_OFF_BWB2: + _slot_set_tbl(btc, CXST_OFF, cxtbl[7]); + break; + case BTC_CXP_OFF_BWB3: + _slot_set_tbl(btc, CXST_OFF, cxtbl[6]); + break; + default: + break; + } + break; + case BTC_CXP_OFFB: /* TDMA off + beacon protect */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, false); + *t = t_def[CXTD_OFF_B2]; + s[CXST_OFF] = s_def[CXST_OFF]; - if (phy_map & BTC_PHY_0) - rtw89_mac_cfg_plt(rtwdev, &plt); + switch (policy_type) { + case BTC_CXP_OFFB_BWB0: + _slot_set_tbl(btc, CXST_OFF, cxtbl[8]); + break; + default: + break; + } + break; + case BTC_CXP_OFFE: /* TDMA off + beacon protect + Ext_control */ + btc->bt_req_en = true; + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_OFF_EXT]; - if (!rtwdev->dbcc_en) - return; + /* To avoid wl-s0 tx break by hid/hfp tx */ + if (hid->exist || hfp->exist) + tbl_w1 = cxtbl[16]; - plt.band = RTW89_MAC_1; - if (phy_map & BTC_PHY_1) - rtw89_mac_cfg_plt(rtwdev, &plt); -} + switch (policy_type) { + case BTC_CXP_OFFE_DEF: + s[CXST_E2G] = s_def[CXST_E2G]; + s[CXST_E5G] = s_def[CXST_E5G]; + s[CXST_EBT] = s_def[CXST_EBT]; + s[CXST_ENULL] = s_def[CXST_ENULL]; + break; + case BTC_CXP_OFFE_DEF2: + _slot_set(btc, CXST_E2G, 20, cxtbl[1], SLOT_ISO); + s[CXST_E5G] = s_def[CXST_E5G]; + s[CXST_EBT] = s_def[CXST_EBT]; + s[CXST_ENULL] = s_def[CXST_ENULL]; + break; + default: + break; + } + break; + case BTC_CXP_FIX: /* TDMA Fix-Slot */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_FIX]; + + switch (policy_type) { + case BTC_CXP_FIX_TD3030: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD5050: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 50, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD2030: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD4010: + _slot_set(btc, CXST_W1, 40, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD4010ISO: + _slot_set(btc, CXST_W1, 40, cxtbl[1], SLOT_ISO); + _slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD7010: + _slot_set(btc, CXST_W1, 70, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD2060: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD3060: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TD2080: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 80, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_FIX_TDW1B1: /* W1:B1 = user-define */ + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + break; + default: + break; + } + break; + case BTC_CXP_PFIX: /* PS-TDMA Fix-Slot */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_PFIX]; + + switch (policy_type) { + case BTC_CXP_PFIX_TD3030: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TD5050: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 50, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TD2030: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TD2060: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TD3070: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TD2080: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, 80, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PFIX_TDW1B1: /* W1:B1 = user-define */ + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + break; + default: + break; + } + break; + case BTC_CXP_AUTO: /* TDMA Auto-Slot */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_AUTO]; + + switch (policy_type) { + case BTC_CXP_AUTO_TD50B1: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_AUTO_TD60B1: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_AUTO_TD20B1: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_AUTO_TDW1B1: /* W1:B1 = user-define */ + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + break; + default: + break; + } + break; + case BTC_CXP_PAUTO: /* PS-TDMA Auto-Slot */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_PAUTO]; + + switch (policy_type) { + case BTC_CXP_PAUTO_TD50B1: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PAUTO_TD60B1: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PAUTO_TD20B1: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + break; + case BTC_CXP_PAUTO_TDW1B1: + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + break; + default: + break; + } + break; + case BTC_CXP_AUTO2: /* TDMA Auto-Slot2 */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_AUTO2]; + + switch (policy_type) { + case BTC_CXP_AUTO2_TD3050: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_AUTO2_TD3070: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_AUTO2_TD5050: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_AUTO2_TD6060: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_AUTO2_TD2080: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_AUTO2_TDW1B4: /* W1:B1 = user-define */ + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4], + tbl_b4, SLOT_MIX); + break; + default: + break; + } + break; + case BTC_CXP_PAUTO2: /* PS-TDMA Auto-Slot2 */ + _write_scbd(rtwdev, BTC_WSCB_TDMA, true); + *t = t_def[CXTD_PAUTO2]; + + switch (policy_type) { + case BTC_CXP_PAUTO2_TD3050: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_PAUTO2_TD3070: + _slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_PAUTO2_TD5050: + _slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_PAUTO2_TD6060: + _slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_PAUTO2_TD2080: + _slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX); + break; + case BTC_CXP_PAUTO2_TDW1B4: /* W1:B1 = user-define */ + _slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1], + tbl_w1, SLOT_ISO); + _slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1], + tbl_b1, SLOT_MIX); + _slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4], + tbl_b4, SLOT_MIX); + break; + default: + break; + } + break; + } + + if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC && dm->tdma.rxflctrl) { + null_role = FIELD_PREP(0x0f, dm->wl_scc.null_role1) | + FIELD_PREP(0xf0, dm->wl_scc.null_role2); + _tdma_set_flctrl_role(btc, null_role); + } + + /* enter leak_slot after each null-1 */ + if (dm->leak_ap && dm->tdma.leak_n > 1) + _tdma_set_lek(btc, 1); + + if (dm->tdma_instant_excute) { + btc->dm.tdma.option_ctrl |= BIT(0); + btc->update_policy_force = true; + } +} +EXPORT_SYMBOL(rtw89_btc_set_policy_v1); + +static void _set_bt_plut(struct rtw89_dev *rtwdev, u8 phy_map, + u8 tx_val, u8 rx_val) +{ + struct rtw89_mac_ax_plt plt; + + plt.band = RTW89_MAC_0; + plt.tx = tx_val; + plt.rx = rx_val; + + if (phy_map & BTC_PHY_0) + rtw89_mac_cfg_plt(rtwdev, &plt); + + if (!rtwdev->dbcc_en) + return; + + plt.band = RTW89_MAC_1; + if (phy_map & BTC_PHY_1) + rtw89_mac_cfg_plt(rtwdev, &plt); +} static void _set_ant(struct rtw89_dev *rtwdev, bool force_exec, u8 phy_map, u8 type) @@ -2300,86 +2888,74 @@ static void _set_ant(struct rtw89_dev *rtwdev, bool force_exec, switch (type) { case BTC_ANT_WPOWERON: - rtw89_chip_cfg_ctrl_path(rtwdev, false); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT); break; case BTC_ANT_WINIT: - if (bt->enable.now) { - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_LO); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI); - } else { - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO); - } - rtw89_chip_cfg_ctrl_path(rtwdev, true); + if (bt->enable.now) + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI); + else + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO); + + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_BT, BTC_PLT_BT); break; case BTC_ANT_WONLY: - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO); - rtw89_chip_cfg_ctrl_path(rtwdev, true); + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE); break; case BTC_ANT_WOFF: - rtw89_chip_cfg_ctrl_path(rtwdev, false); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE); break; case BTC_ANT_W2G: - rtw89_chip_cfg_ctrl_path(rtwdev, true); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); if (rtwdev->dbcc_en) { for (i = 0; i < RTW89_PHY_MAX; i++) { b2g = (wl_dinfo->real_band[i] == RTW89_BAND_2G); gnt_wl_ctrl = b2g ? BTC_GNT_HW : BTC_GNT_SW_HI; - _set_gnt_wl(rtwdev, BIT(i), gnt_wl_ctrl); - gnt_bt_ctrl = b2g ? BTC_GNT_HW : BTC_GNT_SW_HI; /* BT should control by GNT_BT if WL_2G at S0 */ if (i == 1 && wl_dinfo->real_band[0] == RTW89_BAND_2G && wl_dinfo->real_band[1] == RTW89_BAND_5G) gnt_bt_ctrl = BTC_GNT_HW; - _set_gnt_bt(rtwdev, BIT(i), gnt_bt_ctrl); - + _set_gnt(rtwdev, BIT(i), gnt_wl_ctrl, gnt_bt_ctrl); plt_ctrl = b2g ? BTC_PLT_BT : BTC_PLT_NONE; _set_bt_plut(rtwdev, BIT(i), plt_ctrl, plt_ctrl); } } else { - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_HW); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW); + _set_gnt(rtwdev, phy_map, BTC_GNT_HW, BTC_GNT_HW); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_BT, BTC_PLT_BT); } break; case BTC_ANT_W5G: - rtw89_chip_cfg_ctrl_path(rtwdev, true); - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_HW); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE); break; case BTC_ANT_W25G: - rtw89_chip_cfg_ctrl_path(rtwdev, true); - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_HW); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); + _set_gnt(rtwdev, phy_map, BTC_GNT_HW, BTC_GNT_HW); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_GNT_WL, BTC_PLT_GNT_WL); break; case BTC_ANT_FREERUN: - rtw89_chip_cfg_ctrl_path(rtwdev, true); - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_HI); _set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE); break; case BTC_ANT_WRFK: - rtw89_chip_cfg_ctrl_path(rtwdev, true); - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL); + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO); _set_bt_plut(rtwdev, phy_map, BTC_PLT_NONE, BTC_PLT_NONE); break; case BTC_ANT_BRFK: - rtw89_chip_cfg_ctrl_path(rtwdev, false); - _set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_LO); - _set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI); + rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT); + _set_gnt(rtwdev, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI); _set_bt_plut(rtwdev, phy_map, BTC_PLT_NONE, BTC_PLT_NONE); break; default: @@ -2491,14 +3067,19 @@ static void _action_bt_idle(struct rtw89_dev *rtwdev) static void _action_bt_hfp(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; _set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G); if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { - if (btc->cx.wl.status.map._4way) + if (btc->cx.wl.status.map._4way) { _set_policy(rtwdev, BTC_CXP_OFF_WL, BTC_ACT_BT_HFP); - else - _set_policy(rtwdev, BTC_CXP_OFF_BWB0, BTC_ACT_BT_HFP); + } else if (wl->status.map.traffic_dir & BIT(RTW89_TFC_UL)) { + btc->cx.bt.scan_rx_low_pri = true; + _set_policy(rtwdev, BTC_CXP_OFF_BWB2, BTC_ACT_BT_HFP); + } else { + _set_policy(rtwdev, BTC_CXP_OFF_BWB1, BTC_ACT_BT_HFP); + } } else { _set_policy(rtwdev, BTC_CXP_OFF_EQ2, BTC_ACT_BT_HFP); } @@ -2506,17 +3087,37 @@ static void _action_bt_hfp(struct rtw89_dev *rtwdev) static void _action_bt_hid(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; + struct rtw89_btc_bt_info *bt = &btc->cx.bt; + struct rtw89_btc_bt_hid_desc *hid = &bt->link_info.hid_desc; + u16 policy_type = BTC_CXP_OFF_BT; _set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G); - if (btc->mdinfo.ant.type == BTC_ANT_SHARED) /* shared-antenna */ - if (btc->cx.wl.status.map._4way) - _set_policy(rtwdev, BTC_CXP_OFF_WL, BTC_ACT_BT_HID); - else - _set_policy(rtwdev, BTC_CXP_OFF_BWB0, BTC_ACT_BT_HID); - else /* dedicated-antenna */ - _set_policy(rtwdev, BTC_CXP_OFF_EQ3, BTC_ACT_BT_HID); + if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */ + if (wl->status.map._4way) { + policy_type = BTC_CXP_OFF_WL; + } else if (wl->status.map.traffic_dir & BIT(RTW89_TFC_UL)) { + btc->cx.bt.scan_rx_low_pri = true; + if (hid->type & BTC_HID_BLE) + policy_type = BTC_CXP_OFF_BWB0; + else + policy_type = BTC_CXP_OFF_BWB2; + } else if (hid->type == BTC_HID_218) { + bt->scan_rx_low_pri = true; + policy_type = BTC_CXP_OFF_BWB2; + } else if (chip->para_ver == 0x1) { + policy_type = BTC_CXP_OFF_BWB3; + } else { + policy_type = BTC_CXP_OFF_BWB1; + } + } else { /* dedicated-antenna */ + policy_type = BTC_CXP_OFF_EQ3; + } + + _set_policy(rtwdev, policy_type, BTC_ACT_BT_HID); } static void _action_bt_a2dp(struct rtw89_dev *rtwdev) @@ -2537,7 +3138,7 @@ static void _action_bt_a2dp(struct rtw89_dev *rtwdev) BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP); } else { _set_policy(rtwdev, - BTC_CXP_PAUTO_TD50200, BTC_ACT_BT_A2DP); + BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP); } break; case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP */ @@ -2554,12 +3155,12 @@ static void _action_bt_a2dp(struct rtw89_dev *rtwdev) _set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1, BTC_ACT_BT_A2DP); } else { - _set_policy(rtwdev, BTC_CXP_AUTO_TD50200, + _set_policy(rtwdev, BTC_CXP_AUTO_TD50B1, BTC_ACT_BT_A2DP); } break; case BTC_WIDLE: /* wl-idle + bt-A2DP */ - _set_policy(rtwdev, BTC_CXP_AUTO_TD20200, BTC_ACT_BT_A2DP); + _set_policy(rtwdev, BTC_CXP_AUTO_TD20B1, BTC_ACT_BT_A2DP); break; } } @@ -2639,7 +3240,7 @@ static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev) BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP_HID); } else { _set_policy(rtwdev, - BTC_CXP_PAUTO_TD50200, BTC_ACT_BT_A2DP_HID); + BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP_HID); } break; case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP+HID */ @@ -2657,7 +3258,7 @@ static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev) _set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1, BTC_ACT_BT_A2DP_HID); } else { - _set_policy(rtwdev, BTC_CXP_AUTO_TD50200, + _set_policy(rtwdev, BTC_CXP_AUTO_TD50B1, BTC_ACT_BT_A2DP_HID); } break; @@ -2792,19 +3393,27 @@ static void _action_wl_rfk(struct rtw89_dev *rtwdev) static void _set_btg_ctrl(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info; - bool is_btg = false; + bool is_btg; + u8 mode; if (btc->ctrl.manual) return; + if (chip->chip_id == RTL8852A) + mode = wl_rinfo->link_mode; + else + mode = wl_rinfo_v1->link_mode; + /* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */ - if (wl_rinfo->link_mode == BTC_WLINK_5G) /* always 0 if 5G */ + if (mode == BTC_WLINK_5G) /* always 0 if 5G */ is_btg = false; - else if (wl_rinfo->link_mode == BTC_WLINK_25G_DBCC && + else if (mode == BTC_WLINK_25G_DBCC && wl_dinfo->real_band[RTW89_PHY_1] != RTW89_BAND_2G) is_btg = false; else @@ -2816,7 +3425,7 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev) btc->dm.wl_btg_rx = is_btg; - if (wl_rinfo->link_mode == BTC_WLINK_25G_MCC) + if (mode == BTC_WLINK_25G_MCC) return; rtw89_ctrl_btg(rtwdev, is_btg); @@ -2889,6 +3498,7 @@ static void rtw89_tx_time_iter(void *data, struct ieee80211_sta *sta) static void _set_wl_tx_limit(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_cx *cx = &btc->cx; struct rtw89_btc_dm *dm = &btc->dm; @@ -2898,16 +3508,22 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev) struct rtw89_btc_bt_hfp_desc *hfp = &b->hfp_desc; struct rtw89_btc_bt_hid_desc *hid = &b->hid_desc; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; struct rtw89_txtime_data data = {.rtwdev = rtwdev}; - u8 mode = wl_rinfo->link_mode; - u8 tx_retry = 0; - u32 tx_time = 0; - u16 enable = 0; + u8 mode; + u8 tx_retry; + u32 tx_time; + u16 enable; bool reenable = false; if (btc->ctrl.manual) return; + if (chip->chip_id == RTL8852A) + mode = wl_rinfo->link_mode; + else + mode = wl_rinfo_v1->link_mode; + if (btc->dm.freerun || btc->ctrl.igno_bt || b->profile_cnt.now == 0 || mode == BTC_WLINK_5G || mode == BTC_WLINK_NOLINK) { enable = 0; @@ -2951,13 +3567,21 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev) static void _set_bt_rx_agc(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; struct rtw89_btc_bt_info *bt = &btc->cx.bt; bool bt_hi_lna_rx = false; + u8 mode; + + if (chip->chip_id == RTL8852A) + mode = wl_rinfo->link_mode; + else + mode = wl_rinfo_v1->link_mode; - if (wl_rinfo->link_mode != BTC_WLINK_NOLINK && btc->dm.wl_btg_rx) + if (mode != BTC_WLINK_NOLINK && btc->dm.wl_btg_rx) bt_hi_lna_rx = true; if (bt_hi_lna_rx == bt->hi_lna_rx) @@ -2966,14 +3590,34 @@ static void _set_bt_rx_agc(struct rtw89_dev *rtwdev) _write_scbd(rtwdev, BTC_WSCB_BT_HILNA, bt_hi_lna_rx); } +static void _set_bt_rx_scan_pri(struct rtw89_dev *rtwdev) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_bt_info *bt = &btc->cx.bt; + + _write_scbd(rtwdev, BTC_WSCB_RXSCAN_PRI, (bool)(!!bt->scan_rx_low_pri)); +} + /* TODO add these functions */ static void _action_common(struct rtw89_dev *rtwdev) { + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; + _set_btg_ctrl(rtwdev); _set_wl_tx_limit(rtwdev); _set_bt_afh_info(rtwdev); _set_bt_rx_agc(rtwdev); _set_rf_trx_para(rtwdev); + _set_bt_rx_scan_pri(rtwdev); + + if (wl->scbd_change) { + rtw89_mac_cfg_sb(rtwdev, wl->scbd); + rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], write scbd: 0x%08x\n", + wl->scbd); + wl->scbd_change = false; + btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++; + } } static void _action_by_bt(struct rtw89_dev *rtwdev) @@ -3145,6 +3789,68 @@ static void _action_wl_2g_scc(struct rtw89_dev *rtwdev) } } +static void _action_wl_2g_scc_v1(struct rtw89_dev *rtwdev) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; + struct rtw89_btc_bt_info *bt = &btc->cx.bt; + struct rtw89_btc_dm *dm = &btc->dm; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &wl->role_info_v1; + u16 policy_type = BTC_CXP_OFF_BT; + u32 dur; + + if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED) { + policy_type = BTC_CXP_OFF_EQ0; + } else { + /* shared-antenna */ + switch (wl_rinfo->mrole_type) { + case BTC_WLMROLE_STA_GC: + dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION; + dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_P2P_CLIENT; + dm->wl_scc.ebt_null = 0; /* no ext-slot-control */ + _action_by_bt(rtwdev); + return; + case BTC_WLMROLE_STA_STA: + dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION; + dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_STATION; + dm->wl_scc.ebt_null = 0; /* no ext-slot-control */ + _action_by_bt(rtwdev); + return; + case BTC_WLMROLE_STA_GC_NOA: + case BTC_WLMROLE_STA_GO: + case BTC_WLMROLE_STA_GO_NOA: + dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION; + dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_NONE; + dur = wl_rinfo->mrole_noa_duration; + + if (wl->status.map._4way) { + dm->wl_scc.ebt_null = 0; + policy_type = BTC_CXP_OFFE_WL; + } else if (bt->link_info.status.map.connect == 0) { + dm->wl_scc.ebt_null = 0; + policy_type = BTC_CXP_OFFE_2GISOB; + } else if (bt->link_info.a2dp_desc.exist && + dur < btc->bt_req_len) { + dm->wl_scc.ebt_null = 1; /* tx null at EBT */ + policy_type = BTC_CXP_OFFE_2GBWMIXB2; + } else if (bt->link_info.a2dp_desc.exist || + bt->link_info.pan_desc.exist) { + dm->wl_scc.ebt_null = 1; /* tx null at EBT */ + policy_type = BTC_CXP_OFFE_2GBWISOB; + } else { + dm->wl_scc.ebt_null = 0; + policy_type = BTC_CXP_OFFE_2GBWISOB; + } + break; + default: + break; + } + } + + _set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G); + _set_policy(rtwdev, policy_type, BTC_ACT_WL_2G_SCC); +} + static void _action_wl_2g_ap(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; @@ -3234,20 +3940,20 @@ static void _write_scbd(struct rtw89_dev *rtwdev, u32 val, bool state) struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; u32 scbd_val = 0; + u8 force_exec = false; if (!chip->scbd) return; scbd_val = state ? wl->scbd | val : wl->scbd & ~val; - if (scbd_val == wl->scbd) - return; - rtw89_mac_cfg_sb(rtwdev, scbd_val); - rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], write scbd: 0x%08x\n", - scbd_val); - wl->scbd = scbd_val; + if (val & BTC_WSCB_ACTIVE || val & BTC_WSCB_ON) + force_exec = true; - btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++; + if (scbd_val != wl->scbd || force_exec) { + wl->scbd = scbd_val; + wl->scbd_change = true; + } } static u8 @@ -3266,55 +3972,205 @@ _update_rssi_state(struct rtw89_dev *rtwdev, u8 pre_state, u8 rssi, u8 thresh) if (rssi < thresh) next_state = BTC_RSSI_ST_LOW; else - next_state = BTC_RSSI_ST_STAY_HIGH; + next_state = BTC_RSSI_ST_STAY_HIGH; + } + + return next_state; +} + +static +void _update_dbcc_band(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) +{ + struct rtw89_btc *btc = &rtwdev->btc; + + btc->cx.wl.dbcc_info.real_band[phy_idx] = + btc->cx.wl.scan_info.phy_map & BIT(phy_idx) ? + btc->cx.wl.dbcc_info.scan_band[phy_idx] : + btc->cx.wl.dbcc_info.op_band[phy_idx]; +} + +static void _update_wl_info(struct rtw89_dev *rtwdev) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; + struct rtw89_btc_wl_link_info *wl_linfo = wl->link_info; + struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info; + u8 i, cnt_connect = 0, cnt_connecting = 0, cnt_active = 0; + u8 cnt_2g = 0, cnt_5g = 0, phy; + u32 wl_2g_ch[2] = {0}, wl_5g_ch[2] = {0}; + bool b2g = false, b5g = false, client_joined = false; + + memset(wl_rinfo, 0, sizeof(*wl_rinfo)); + + for (i = 0; i < RTW89_PORT_NUM; i++) { + /* check if role active? */ + if (!wl_linfo[i].active) + continue; + + cnt_active++; + wl_rinfo->active_role[cnt_active - 1].role = wl_linfo[i].role; + wl_rinfo->active_role[cnt_active - 1].pid = wl_linfo[i].pid; + wl_rinfo->active_role[cnt_active - 1].phy = wl_linfo[i].phy; + wl_rinfo->active_role[cnt_active - 1].band = wl_linfo[i].band; + wl_rinfo->active_role[cnt_active - 1].noa = (u8)wl_linfo[i].noa; + wl_rinfo->active_role[cnt_active - 1].connected = 0; + + wl->port_id[wl_linfo[i].role] = wl_linfo[i].pid; + + phy = wl_linfo[i].phy; + + /* check dbcc role */ + if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) { + wl_dinfo->role[phy] = wl_linfo[i].role; + wl_dinfo->op_band[phy] = wl_linfo[i].band; + _update_dbcc_band(rtwdev, phy); + _fw_set_drv_info(rtwdev, CXDRVINFO_DBCC); + } + + if (wl_linfo[i].connected == MLME_NO_LINK) { + continue; + } else if (wl_linfo[i].connected == MLME_LINKING) { + cnt_connecting++; + } else { + cnt_connect++; + if ((wl_linfo[i].role == RTW89_WIFI_ROLE_P2P_GO || + wl_linfo[i].role == RTW89_WIFI_ROLE_AP) && + wl_linfo[i].client_cnt > 1) + client_joined = true; + } + + wl_rinfo->role_map.val |= BIT(wl_linfo[i].role); + wl_rinfo->active_role[cnt_active - 1].ch = wl_linfo[i].ch; + wl_rinfo->active_role[cnt_active - 1].bw = wl_linfo[i].bw; + wl_rinfo->active_role[cnt_active - 1].connected = 1; + + /* only care 2 roles + BT coex */ + if (wl_linfo[i].band != RTW89_BAND_2G) { + if (cnt_5g <= ARRAY_SIZE(wl_5g_ch) - 1) + wl_5g_ch[cnt_5g] = wl_linfo[i].ch; + cnt_5g++; + b5g = true; + } else { + if (cnt_2g <= ARRAY_SIZE(wl_2g_ch) - 1) + wl_2g_ch[cnt_2g] = wl_linfo[i].ch; + cnt_2g++; + b2g = true; + } + } + + wl_rinfo->connect_cnt = cnt_connect; + + /* Be careful to change the following sequence!! */ + if (cnt_connect == 0) { + wl_rinfo->link_mode = BTC_WLINK_NOLINK; + wl_rinfo->role_map.role.none = 1; + } else if (!b2g && b5g) { + wl_rinfo->link_mode = BTC_WLINK_5G; + } else if (wl_rinfo->role_map.role.nan) { + wl_rinfo->link_mode = BTC_WLINK_2G_NAN; + } else if (cnt_connect > BTC_TDMA_WLROLE_MAX) { + wl_rinfo->link_mode = BTC_WLINK_OTHER; + } else if (b2g && b5g && cnt_connect == 2) { + if (rtwdev->dbcc_en) { + switch (wl_dinfo->role[RTW89_PHY_0]) { + case RTW89_WIFI_ROLE_STATION: + wl_rinfo->link_mode = BTC_WLINK_2G_STA; + break; + case RTW89_WIFI_ROLE_P2P_GO: + wl_rinfo->link_mode = BTC_WLINK_2G_GO; + break; + case RTW89_WIFI_ROLE_P2P_CLIENT: + wl_rinfo->link_mode = BTC_WLINK_2G_GC; + break; + case RTW89_WIFI_ROLE_AP: + wl_rinfo->link_mode = BTC_WLINK_2G_AP; + break; + default: + wl_rinfo->link_mode = BTC_WLINK_OTHER; + break; + } + } else { + wl_rinfo->link_mode = BTC_WLINK_25G_MCC; + } + } else if (!b5g && cnt_connect == 2) { + if (wl_rinfo->role_map.role.station && + (wl_rinfo->role_map.role.p2p_go || + wl_rinfo->role_map.role.p2p_gc || + wl_rinfo->role_map.role.ap)) { + if (wl_2g_ch[0] == wl_2g_ch[1]) + wl_rinfo->link_mode = BTC_WLINK_2G_SCC; + else + wl_rinfo->link_mode = BTC_WLINK_2G_MCC; + } else { + wl_rinfo->link_mode = BTC_WLINK_2G_MCC; + } + } else if (!b5g && cnt_connect == 1) { + if (wl_rinfo->role_map.role.station) + wl_rinfo->link_mode = BTC_WLINK_2G_STA; + else if (wl_rinfo->role_map.role.ap) + wl_rinfo->link_mode = BTC_WLINK_2G_AP; + else if (wl_rinfo->role_map.role.p2p_go) + wl_rinfo->link_mode = BTC_WLINK_2G_GO; + else if (wl_rinfo->role_map.role.p2p_gc) + wl_rinfo->link_mode = BTC_WLINK_2G_GC; + else + wl_rinfo->link_mode = BTC_WLINK_OTHER; } - return next_state; -} + /* if no client_joined, don't care P2P-GO/AP role */ + if (wl_rinfo->role_map.role.p2p_go || wl_rinfo->role_map.role.ap) { + if (!client_joined) { + if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC || + wl_rinfo->link_mode == BTC_WLINK_2G_MCC) { + wl_rinfo->link_mode = BTC_WLINK_2G_STA; + wl_rinfo->connect_cnt = 1; + } else if (wl_rinfo->link_mode == BTC_WLINK_2G_GO || + wl_rinfo->link_mode == BTC_WLINK_2G_AP) { + wl_rinfo->link_mode = BTC_WLINK_NOLINK; + wl_rinfo->connect_cnt = 0; + } + } + } -static -void _update_dbcc_band(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) -{ - struct rtw89_btc *btc = &rtwdev->btc; + rtw89_debug(rtwdev, RTW89_DBG_BTC, + "[BTC], cnt_connect = %d, connecting = %d, link_mode = %d\n", + cnt_connect, cnt_connecting, wl_rinfo->link_mode); - btc->cx.wl.dbcc_info.real_band[phy_idx] = - btc->cx.wl.scan_info.phy_map & BIT(phy_idx) ? - btc->cx.wl.dbcc_info.scan_band[phy_idx] : - btc->cx.wl.dbcc_info.op_band[phy_idx]; + _fw_set_drv_info(rtwdev, CXDRVINFO_ROLE); } -static void _update_wl_info(struct rtw89_dev *rtwdev) +static void _update_wl_info_v1(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; struct rtw89_btc_wl_link_info *wl_linfo = wl->link_info; - struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &wl->role_info_v1; struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info; - u8 i, cnt_connect = 0, cnt_connecting = 0, cnt_active = 0; + u8 cnt_connect = 0, cnt_connecting = 0, cnt_active = 0; u8 cnt_2g = 0, cnt_5g = 0, phy; - u32 wl_2g_ch[2] = {0}, wl_5g_ch[2] = {0}; + u32 wl_2g_ch[2] = {}, wl_5g_ch[2] = {}; bool b2g = false, b5g = false, client_joined = false; + u8 i; memset(wl_rinfo, 0, sizeof(*wl_rinfo)); for (i = 0; i < RTW89_PORT_NUM; i++) { - /* check if role active? */ if (!wl_linfo[i].active) continue; cnt_active++; - wl_rinfo->active_role[cnt_active - 1].role = wl_linfo[i].role; - wl_rinfo->active_role[cnt_active - 1].pid = wl_linfo[i].pid; - wl_rinfo->active_role[cnt_active - 1].phy = wl_linfo[i].phy; - wl_rinfo->active_role[cnt_active - 1].band = wl_linfo[i].band; - wl_rinfo->active_role[cnt_active - 1].noa = (u8)wl_linfo[i].noa; - wl_rinfo->active_role[cnt_active - 1].connected = 0; + wl_rinfo->active_role_v1[cnt_active - 1].role = wl_linfo[i].role; + wl_rinfo->active_role_v1[cnt_active - 1].pid = wl_linfo[i].pid; + wl_rinfo->active_role_v1[cnt_active - 1].phy = wl_linfo[i].phy; + wl_rinfo->active_role_v1[cnt_active - 1].band = wl_linfo[i].band; + wl_rinfo->active_role_v1[cnt_active - 1].noa = (u8)wl_linfo[i].noa; + wl_rinfo->active_role_v1[cnt_active - 1].connected = 0; wl->port_id[wl_linfo[i].role] = wl_linfo[i].pid; phy = wl_linfo[i].phy; - /* check dbcc role */ if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) { wl_dinfo->role[phy] = wl_linfo[i].role; wl_dinfo->op_band[phy] = wl_linfo[i].band; @@ -3335,9 +4191,9 @@ static void _update_wl_info(struct rtw89_dev *rtwdev) } wl_rinfo->role_map.val |= BIT(wl_linfo[i].role); - wl_rinfo->active_role[cnt_active - 1].ch = wl_linfo[i].ch; - wl_rinfo->active_role[cnt_active - 1].bw = wl_linfo[i].bw; - wl_rinfo->active_role[cnt_active - 1].connected = 1; + wl_rinfo->active_role_v1[cnt_active - 1].ch = wl_linfo[i].ch; + wl_rinfo->active_role_v1[cnt_active - 1].bw = wl_linfo[i].bw; + wl_rinfo->active_role_v1[cnt_active - 1].connected = 1; /* only care 2 roles + BT coex */ if (wl_linfo[i].band != RTW89_BAND_2G) { @@ -3428,8 +4284,8 @@ static void _update_wl_info(struct rtw89_dev *rtwdev) } rtw89_debug(rtwdev, RTW89_DBG_BTC, - "[BTC], cnt_connect = %d, link_mode = %d\n", - cnt_connect, wl_rinfo->link_mode); + "[BTC], cnt_connect = %d, connecting = %d, link_mode = %d\n", + cnt_connect, cnt_connecting, wl_rinfo->link_mode); _fw_set_drv_info(rtwdev, CXDRVINFO_ROLE); } @@ -3584,23 +4440,32 @@ static bool _chk_wl_rfk_request(struct rtw89_dev *rtwdev) static void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_dm *dm = &rtwdev->btc.dm; struct rtw89_btc_cx *cx = &btc->cx; struct rtw89_btc_wl_info *wl = &btc->cx.wl; + struct rtw89_btc_bt_info *bt = &btc->cx.bt; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; - u8 mode = wl_rinfo->link_mode; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; + u8 mode; lockdep_assert_held(&rtwdev->mutex); - rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): reason=%d, mode=%d\n", - __func__, reason, mode); - rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): wl_only=%d, bt_only=%d\n", - __func__, dm->wl_only, dm->bt_only); dm->run_reason = reason; _update_dm_step(rtwdev, reason); _update_btc_state_map(rtwdev); + if (chip->chip_id == RTL8852A) + mode = wl_rinfo->link_mode; + else + mode = wl_rinfo_v1->link_mode; + + rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): reason=%d, mode=%d\n", + __func__, reason, mode); + rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): wl_only=%d, bt_only=%d\n", + __func__, dm->wl_only, dm->bt_only); + /* Be careful to change the following function sequence!! */ if (btc->ctrl.manual) { rtw89_debug(rtwdev, RTW89_DBG_BTC, @@ -3657,6 +4522,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason) btc->ctrl.igno_bt = false; dm->freerun = false; + bt->scan_rx_low_pri = false; if (reason == BTC_RSN_NTFY_INIT) { _action_wl_init(rtwdev); @@ -3699,21 +4565,30 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason) _action_wl_2g_sta(rtwdev); break; case BTC_WLINK_2G_AP: + bt->scan_rx_low_pri = true; _action_wl_2g_ap(rtwdev); break; case BTC_WLINK_2G_GO: + bt->scan_rx_low_pri = true; _action_wl_2g_go(rtwdev); break; case BTC_WLINK_2G_GC: + bt->scan_rx_low_pri = true; _action_wl_2g_gc(rtwdev); break; case BTC_WLINK_2G_SCC: - _action_wl_2g_scc(rtwdev); + bt->scan_rx_low_pri = true; + if (chip->chip_id == RTL8852A) + _action_wl_2g_scc(rtwdev); + else if (chip->chip_id == RTL8852C) + _action_wl_2g_scc_v1(rtwdev); break; case BTC_WLINK_2G_MCC: + bt->scan_rx_low_pri = true; _action_wl_2g_mcc(rtwdev); break; case BTC_WLINK_25G_MCC: + bt->scan_rx_low_pri = true; _action_wl_25g_mcc(rtwdev); break; case BTC_WLINK_5G: @@ -3743,11 +4618,14 @@ void rtw89_btc_ntfy_poweron(struct rtw89_dev *rtwdev) void rtw89_btc_ntfy_poweroff(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): !!\n", __func__); btc->dm.cnt_notify[BTC_NCNT_POWER_OFF]++; btc->cx.wl.status.map.rf_off = 1; + btc->cx.wl.status.map.busy = 0; + wl->status.map.lps = BTC_LPS_OFF; _write_scbd(rtwdev, BTC_WSCB_ALL, false); _run_coex(rtwdev, BTC_RSN_NTFY_POWEROFF); @@ -3807,7 +4685,7 @@ void rtw89_btc_ntfy_init(struct rtw89_dev *rtwdev, u8 mode) _write_scbd(rtwdev, BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG, true); _update_bt_scbd(rtwdev, true); - if (rtw89_mac_get_ctrl_path(rtwdev)) { + if (rtw89_mac_get_ctrl_path(rtwdev) && chip->chip_id == RTL8852A) { rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): PTA owner warning!!\n", __func__); @@ -4150,7 +5028,8 @@ enum btc_wl_mode { void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, struct rtw89_sta *rtwsta, enum btc_role_state state) { - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + const struct rtw89_chip_info *chip = rtwdev->chip; struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); struct ieee80211_sta *sta = rtwsta_to_sta(rtwsta); struct rtw89_btc *btc = &rtwdev->btc; @@ -4165,8 +5044,7 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif vif->type == NL80211_IFTYPE_STATION); rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], port=%d\n", rtwvif->port); rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], band=%d ch=%d bw=%d\n", - hal->current_band_type, hal->current_channel, - hal->current_band_width); + chan->band_type, chan->channel, chan->band_width); rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], associated=%d\n", state == BTC_ROLE_MSTS_STA_CONN_END); rtw89_debug(rtwdev, RTW89_DBG_BTC, @@ -4205,9 +5083,9 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif r.connected = MLME_LINKED; r.bcn_period = vif->bss_conf.beacon_int; r.dtim_period = vif->bss_conf.dtim_period; - r.band = hal->current_band_type; - r.ch = hal->current_channel; - r.bw = hal->current_band_width; + r.band = chan->band_type; + r.ch = chan->channel; + r.bw = chan->band_width; ether_addr_copy(r.mac_addr, rtwvif->mac_addr); if (rtwsta && vif->type == NL80211_IFTYPE_STATION) @@ -4218,7 +5096,10 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif wlinfo = &wl->link_info[r.pid]; memcpy(wlinfo, &r, sizeof(*wlinfo)); - _update_wl_info(rtwdev); + if (chip->chip_id == RTL8852A) + _update_wl_info(rtwdev); + else + _update_wl_info_v1(rtwdev); if (wlinfo->role == RTW89_WIFI_ROLE_STATION && wlinfo->connected == MLME_NO_LINK) @@ -4240,6 +5121,7 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_wl_info *wl = &btc->cx.wl; + u32 val; rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): rf_state = %d\n", __func__, rf_state); @@ -4249,10 +5131,12 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta case BTC_RFCTRL_WL_OFF: wl->status.map.rf_off = 1; wl->status.map.lps = BTC_LPS_OFF; + wl->status.map.busy = 0; break; case BTC_RFCTRL_FW_CTRL: wl->status.map.rf_off = 0; wl->status.map.lps = BTC_LPS_RF_OFF; + wl->status.map.busy = 0; break; case BTC_RFCTRL_WL_ON: default: @@ -4262,14 +5146,17 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta } if (rf_state == BTC_RFCTRL_WL_ON) { + btc->dm.cnt_dm[BTC_DCNT_BTCNT_FREEZE] = 0; rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_MREG | RPT_EN_BT_VER_INFO, true); - _write_scbd(rtwdev, BTC_WSCB_ACTIVE, true); + val = BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG; + _write_scbd(rtwdev, val, true); _update_bt_scbd(rtwdev, true); chip->ops->btc_init_cfg(rtwdev); } else { rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_ALL, false); - _write_scbd(rtwdev, BTC_WSCB_ACTIVE | BTC_WSCB_WLBUSY, false); + if (rf_state == BTC_RFCTRL_WL_OFF) + _write_scbd(rtwdev, BTC_WSCB_ALL, false); } _run_coex(rtwdev, BTC_RSN_NTFY_RADIO_STATE); @@ -4609,10 +5496,10 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m) seq_printf(m, "========== [BTC COEX INFO (%d)] ==========\n", chip->chip_id); - ver_main = FIELD_GET(GENMASK(31, 24), chip->para_ver); - ver_sub = FIELD_GET(GENMASK(23, 16), chip->para_ver); - ver_hotfix = FIELD_GET(GENMASK(15, 8), chip->para_ver); - id_branch = FIELD_GET(GENMASK(7, 0), chip->para_ver); + ver_main = FIELD_GET(GENMASK(31, 24), RTW89_COEX_VERSION); + ver_sub = FIELD_GET(GENMASK(23, 16), RTW89_COEX_VERSION); + ver_hotfix = FIELD_GET(GENMASK(15, 8), RTW89_COEX_VERSION); + id_branch = FIELD_GET(GENMASK(7, 0), RTW89_COEX_VERSION); seq_printf(m, " %-15s : Coex:%d.%d.%d(branch:%d), ", "[coex_version]", ver_main, ver_sub, ver_hotfix, id_branch); @@ -4726,23 +5613,29 @@ static void _show_wl_role_info(struct rtw89_dev *rtwdev, struct seq_file *m) static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_cx *cx = &btc->cx; struct rtw89_btc_wl_info *wl = &cx->wl; struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info; + struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1; + u8 mode; if (!(btc->dm.coex_info_map & BTC_COEX_INFO_WL)) return; seq_puts(m, "========== [WL Status] ==========\n"); - seq_printf(m, " %-15s : link_mode:%d, ", - "[status]", (u32)wl_rinfo->link_mode); + if (chip->chip_id == RTL8852A) + mode = wl_rinfo->link_mode; + else + mode = wl_rinfo_v1->link_mode; + + seq_printf(m, " %-15s : link_mode:%d, ", "[status]", mode); seq_printf(m, - "rf_off:%s, power_save:%s, scan:%s(band:%d/phy_map:0x%x), ", - wl->status.map.rf_off ? "Y" : "N", - wl->status.map.lps ? "Y" : "N", + "rf_off:%d, power_save:%d, scan:%s(band:%d/phy_map:0x%x), ", + wl->status.map.rf_off, wl->status.map.lps, wl->status.map.scan ? "Y" : "N", wl->scan_info.band[RTW89_PHY_0], wl->scan_info.phy_map); @@ -4908,6 +5801,7 @@ static void _show_bt_info(struct rtw89_dev *rtwdev, struct seq_file *m) #define CASE_BTC_ACT_STR(e) case BTC_ACT_ ## e | BTC_ACT_EXT_BIT: return #e #define CASE_BTC_POLICY_STR(e) \ case BTC_CXP_ ## e | BTC_POLICY_EXT_BIT: return #e +#define CASE_BTC_SLOT_STR(e) case CXST_ ## e: return #e static const char *steps_to_str(u16 step) { @@ -4969,9 +5863,16 @@ static const char *steps_to_str(u16 step) CASE_BTC_POLICY_STR(OFF_EQ3); CASE_BTC_POLICY_STR(OFF_BWB0); CASE_BTC_POLICY_STR(OFF_BWB1); + CASE_BTC_POLICY_STR(OFF_BWB2); + CASE_BTC_POLICY_STR(OFF_BWB3); CASE_BTC_POLICY_STR(OFFB_BWB0); CASE_BTC_POLICY_STR(OFFE_DEF); CASE_BTC_POLICY_STR(OFFE_DEF2); + CASE_BTC_POLICY_STR(OFFE_2GBWISOB); + CASE_BTC_POLICY_STR(OFFE_2GISOB); + CASE_BTC_POLICY_STR(OFFE_2GBWMIXB); + CASE_BTC_POLICY_STR(OFFE_WL); + CASE_BTC_POLICY_STR(OFFE_2GBWMIXB2); CASE_BTC_POLICY_STR(FIX_TD3030); CASE_BTC_POLICY_STR(FIX_TD5050); CASE_BTC_POLICY_STR(FIX_TD2030); @@ -4982,6 +5883,7 @@ static const char *steps_to_str(u16 step) CASE_BTC_POLICY_STR(FIX_TD2080); CASE_BTC_POLICY_STR(FIX_TDW1B1); CASE_BTC_POLICY_STR(FIX_TD4020); + CASE_BTC_POLICY_STR(FIX_TD4010ISO); CASE_BTC_POLICY_STR(PFIX_TD3030); CASE_BTC_POLICY_STR(PFIX_TD5050); CASE_BTC_POLICY_STR(PFIX_TD2030); @@ -4989,13 +5891,13 @@ static const char *steps_to_str(u16 step) CASE_BTC_POLICY_STR(PFIX_TD3070); CASE_BTC_POLICY_STR(PFIX_TD2080); CASE_BTC_POLICY_STR(PFIX_TDW1B1); - CASE_BTC_POLICY_STR(AUTO_TD50200); - CASE_BTC_POLICY_STR(AUTO_TD60200); - CASE_BTC_POLICY_STR(AUTO_TD20200); + CASE_BTC_POLICY_STR(AUTO_TD50B1); + CASE_BTC_POLICY_STR(AUTO_TD60B1); + CASE_BTC_POLICY_STR(AUTO_TD20B1); CASE_BTC_POLICY_STR(AUTO_TDW1B1); - CASE_BTC_POLICY_STR(PAUTO_TD50200); - CASE_BTC_POLICY_STR(PAUTO_TD60200); - CASE_BTC_POLICY_STR(PAUTO_TD20200); + CASE_BTC_POLICY_STR(PAUTO_TD50B1); + CASE_BTC_POLICY_STR(PAUTO_TD60B1); + CASE_BTC_POLICY_STR(PAUTO_TD20B1); CASE_BTC_POLICY_STR(PAUTO_TDW1B1); CASE_BTC_POLICY_STR(AUTO2_TD3050); CASE_BTC_POLICY_STR(AUTO2_TD3070); @@ -5014,6 +5916,32 @@ static const char *steps_to_str(u16 step) } } +static const char *id_to_slot(u32 id) +{ + switch (id) { + CASE_BTC_SLOT_STR(OFF); + CASE_BTC_SLOT_STR(B2W); + CASE_BTC_SLOT_STR(W1); + CASE_BTC_SLOT_STR(W2); + CASE_BTC_SLOT_STR(W2B); + CASE_BTC_SLOT_STR(B1); + CASE_BTC_SLOT_STR(B2); + CASE_BTC_SLOT_STR(B3); + CASE_BTC_SLOT_STR(B4); + CASE_BTC_SLOT_STR(LK); + CASE_BTC_SLOT_STR(BLK); + CASE_BTC_SLOT_STR(E2G); + CASE_BTC_SLOT_STR(E5G); + CASE_BTC_SLOT_STR(EBT); + CASE_BTC_SLOT_STR(ENULL); + CASE_BTC_SLOT_STR(WLK); + CASE_BTC_SLOT_STR(W1FDD); + CASE_BTC_SLOT_STR(B1FDD); + default: + return "unknown"; + } +} + static void seq_print_segment(struct seq_file *m, const char *prefix, u16 *data, u8 len, u8 seg_len, u8 start_idx, u8 ring_len) @@ -5105,21 +6033,31 @@ static void _show_dm_info(struct rtw89_dev *rtwdev, struct seq_file *m) (bt->hi_lna_rx ? "Hi" : "Ori"), dm->wl_btg_rx); seq_printf(m, - " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU\n", + " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU, bt_scan_rx_low_pri:%d\n", "[dm_ctrl]", dm->wl_tx_limit.enable, dm->wl_tx_limit.tx_time, - dm->wl_tx_limit.tx_retry, btc->bt_req_len); + dm->wl_tx_limit.tx_retry, btc->bt_req_len, bt->scan_rx_low_pri); } static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo; - struct rtw89_btc_fbtc_cysta *pcysta = NULL; - - pcysta = &pfwinfo->rpt_fbtc_cysta.finfo; + struct rtw89_btc_fbtc_cysta *pcysta; + struct rtw89_btc_fbtc_cysta_v1 *pcysta_v1; + u32 except_cnt, exception_map; + + if (chip->chip_id == RTL8852A) { + pcysta = &pfwinfo->rpt_fbtc_cysta.finfo; + except_cnt = le32_to_cpu(pcysta->except_cnt); + exception_map = le32_to_cpu(pcysta->exception); + } else { + pcysta_v1 = &pfwinfo->rpt_fbtc_cysta.finfo_v1; + except_cnt = le32_to_cpu(pcysta_v1->except_cnt); + exception_map = le32_to_cpu(pcysta_v1->except_map); + } - if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW] == 0 && - pcysta->except_cnt == 0 && + if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW] == 0 && except_cnt == 0 && !pfwinfo->len_mismch && !pfwinfo->fver_mismch) return; @@ -5144,16 +6082,17 @@ static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m) } /* cycle statistics exceptions */ - if (pcysta->exception || pcysta->except_cnt) { + if (exception_map || except_cnt) { seq_printf(m, "exception-type: 0x%x, exception-cnt = %d", - pcysta->exception, pcysta->except_cnt); + exception_map, except_cnt); } seq_puts(m, "\n"); } static void _show_fbtc_tdma(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo; struct rtw89_btc_rpt_cmn_info *pcinfo = NULL; @@ -5166,7 +6105,10 @@ static void _show_fbtc_tdma(struct rtw89_dev *rtwdev, struct seq_file *m) if (!pcinfo->valid) return; - t = &pfwinfo->rpt_fbtc_tdma.finfo; + if (chip->chip_id == RTL8852A) + t = &pfwinfo->rpt_fbtc_tdma.finfo; + else + t = &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma; seq_printf(m, " %-15s : ", "[tdma_policy]"); @@ -5369,12 +6311,145 @@ static void _show_fbtc_cysta(struct rtw89_dev *rtwdev, struct seq_file *m) } } +static void _show_fbtc_cysta_v1(struct rtw89_dev *rtwdev, struct seq_file *m) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc; + struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo; + struct rtw89_btc_dm *dm = &btc->dm; + struct rtw89_btc_fbtc_a2dp_trx_stat *a2dp_trx; + struct rtw89_btc_fbtc_cysta_v1 *pcysta; + struct rtw89_btc_rpt_cmn_info *pcinfo; + u8 i, cnt = 0, slot_pair, divide_cnt; + u16 cycle, c_begin, c_end, store_index; + + pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo; + if (!pcinfo->valid) + return; + + pcysta = &pfwinfo->rpt_fbtc_cysta.finfo_v1; + seq_printf(m, + " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]", + "[cycle_cnt]", + le16_to_cpu(pcysta->cycles), + le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]), + le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]), + le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]), + le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK])); + + for (i = 0; i < CXST_MAX; i++) { + if (!le32_to_cpu(pcysta->slot_cnt[i])) + continue; + + seq_printf(m, ", %s:%d", id_to_slot(i), + le32_to_cpu(pcysta->slot_cnt[i])); + } + + if (dm->tdma_now.rxflctrl) + seq_printf(m, ", leak_rx:%d", le32_to_cpu(pcysta->leak_slot.cnt_rximr)); + + if (le32_to_cpu(pcysta->collision_cnt)) + seq_printf(m, ", collision:%d", le32_to_cpu(pcysta->collision_cnt)); + + if (le32_to_cpu(pcysta->skip_cnt)) + seq_printf(m, ", skip:%d", le32_to_cpu(pcysta->skip_cnt)); + + seq_puts(m, "\n"); + + seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]", + "[cycle_time]", + le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]), + le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]), + le16_to_cpu(pcysta->leak_slot.tavg) / 1000, + le16_to_cpu(pcysta->leak_slot.tavg) % 1000); + seq_printf(m, + ", max_t[wl:%d/bt:%d/lk:%d.%03d]", + le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]), + le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]), + le16_to_cpu(pcysta->leak_slot.tmax) / 1000, + le16_to_cpu(pcysta->leak_slot.tmax) % 1000); + seq_printf(m, + ", maxdiff_t[wl:%d/bt:%d]\n", + le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]), + le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT])); + + cycle = le16_to_cpu(pcysta->cycles); + if (cycle == 0) + return; + + /* 1 cycle record 1 wl-slot and 1 bt-slot */ + slot_pair = BTC_CYCLE_SLOT_MAX / 2; + + if (cycle <= slot_pair) + c_begin = 1; + else + c_begin = cycle - slot_pair + 1; + + c_end = cycle; + + if (a2dp->exist) + divide_cnt = 3; + else + divide_cnt = BTC_CYCLE_SLOT_MAX / 4; + + for (cycle = c_begin; cycle <= c_end; cycle++) { + cnt++; + store_index = ((cycle - 1) % slot_pair) * 2; + + if (cnt % divide_cnt == 1) { + seq_printf(m, "\n\r %-15s : ", "[cycle_step]"); + } else { + seq_printf(m, "->b%02d", + le16_to_cpu(pcysta->slot_step_time[store_index])); + if (a2dp->exist) { + a2dp_trx = &pcysta->a2dp_trx[store_index]; + seq_printf(m, "(%d/%d/%dM/%d/%d/%d)", + a2dp_trx->empty_cnt, + a2dp_trx->retry_cnt, + a2dp_trx->tx_rate ? 3 : 2, + a2dp_trx->tx_cnt, + a2dp_trx->ack_cnt, + a2dp_trx->nack_cnt); + } + seq_printf(m, "->w%02d", + le16_to_cpu(pcysta->slot_step_time[store_index + 1])); + if (a2dp->exist) { + a2dp_trx = &pcysta->a2dp_trx[store_index + 1]; + seq_printf(m, "(%d/%d/%dM/%d/%d/%d)", + a2dp_trx->empty_cnt, + a2dp_trx->retry_cnt, + a2dp_trx->tx_rate ? 3 : 2, + a2dp_trx->tx_cnt, + a2dp_trx->ack_cnt, + a2dp_trx->nack_cnt); + } + } + if (cnt % (BTC_CYCLE_SLOT_MAX / 4) == 0 || cnt == c_end) + seq_puts(m, "\n"); + } + + if (a2dp->exist) { + seq_printf(m, "%-15s : a2dp_ept:%d, a2dp_late:%d", + "[a2dp_t_sta]", + le16_to_cpu(pcysta->a2dp_ept.cnt), + le16_to_cpu(pcysta->a2dp_ept.cnt_timeout)); + + seq_printf(m, ", avg_t:%d, max_t:%d", + le16_to_cpu(pcysta->a2dp_ept.tavg), + le16_to_cpu(pcysta->a2dp_ept.tmax)); + + seq_puts(m, "\n"); + } +} + static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo; - struct rtw89_btc_rpt_cmn_info *pcinfo = NULL; - struct rtw89_btc_fbtc_cynullsta *ns = NULL; + struct rtw89_btc_rpt_cmn_info *pcinfo; + struct rtw89_btc_fbtc_cynullsta *ns; + struct rtw89_btc_fbtc_cynullsta_v1 *ns_v1; u8 i = 0; if (!btc->dm.tdma_now.rxflctrl) @@ -5384,25 +6459,58 @@ static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m) if (!pcinfo->valid) return; - ns = &pfwinfo->rpt_fbtc_nullsta.finfo; + if (chip->chip_id == RTL8852A) { + ns = &pfwinfo->rpt_fbtc_nullsta.finfo; - seq_printf(m, " %-15s : ", "[null_sta]"); + seq_printf(m, " %-15s : ", "[null_sta]"); - for (i = 0; i < 2; i++) { - if (i != 0) - seq_printf(m, ", null-%d", i); - else - seq_printf(m, "null-%d", i); - seq_printf(m, "[ok:%d/", le32_to_cpu(ns->result[i][1])); - seq_printf(m, "fail:%d/", le32_to_cpu(ns->result[i][0])); - seq_printf(m, "on_time:%d/", le32_to_cpu(ns->result[i][2])); - seq_printf(m, "retry:%d/", le32_to_cpu(ns->result[i][3])); - seq_printf(m, "avg_t:%d.%03d/", - le32_to_cpu(ns->avg_t[i]) / 1000, - le32_to_cpu(ns->avg_t[i]) % 1000); - seq_printf(m, "max_t:%d.%03d]", - le32_to_cpu(ns->max_t[i]) / 1000, - le32_to_cpu(ns->max_t[i]) % 1000); + for (i = 0; i < 2; i++) { + if (i != 0) + seq_printf(m, ", null-%d", i); + else + seq_printf(m, "null-%d", i); + seq_printf(m, "[ok:%d/", + le32_to_cpu(ns->result[i][1])); + seq_printf(m, "fail:%d/", + le32_to_cpu(ns->result[i][0])); + seq_printf(m, "on_time:%d/", + le32_to_cpu(ns->result[i][2])); + seq_printf(m, "retry:%d/", + le32_to_cpu(ns->result[i][3])); + seq_printf(m, "avg_t:%d.%03d/", + le32_to_cpu(ns->avg_t[i]) / 1000, + le32_to_cpu(ns->avg_t[i]) % 1000); + seq_printf(m, "max_t:%d.%03d]", + le32_to_cpu(ns->max_t[i]) / 1000, + le32_to_cpu(ns->max_t[i]) % 1000); + } + } else { + ns_v1 = &pfwinfo->rpt_fbtc_nullsta.finfo_v1; + + seq_printf(m, " %-15s : ", "[null_sta]"); + + for (i = 0; i < 2; i++) { + if (i != 0) + seq_printf(m, ", null-%d", i); + else + seq_printf(m, "null-%d", i); + seq_printf(m, "[Tx:%d/", + le32_to_cpu(ns_v1->result[i][4])); + seq_printf(m, "[ok:%d/", + le32_to_cpu(ns_v1->result[i][1])); + seq_printf(m, "fail:%d/", + le32_to_cpu(ns_v1->result[i][0])); + seq_printf(m, "on_time:%d/", + le32_to_cpu(ns_v1->result[i][2])); + seq_printf(m, "retry:%d/", + le32_to_cpu(ns_v1->result[i][3])); + seq_printf(m, "avg_t:%d.%03d/", + le32_to_cpu(ns_v1->avg_t[i]) / 1000, + le32_to_cpu(ns_v1->avg_t[i]) % 1000); + seq_printf(m, "max_t:%d.%03d]", + le32_to_cpu(ns_v1->max_t[i]) / 1000, + le32_to_cpu(ns_v1->max_t[i]) % 1000); + } } seq_puts(m, "\n"); } @@ -5478,6 +6586,7 @@ static void _show_fbtc_step(struct rtw89_dev *rtwdev, struct seq_file *m) static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; if (!(btc->dm.coex_info_map & BTC_COEX_INFO_DM)) @@ -5486,11 +6595,57 @@ static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m) _show_error(rtwdev, m); _show_fbtc_tdma(rtwdev, m); _show_fbtc_slots(rtwdev, m); - _show_fbtc_cysta(rtwdev, m); + + if (chip->chip_id == RTL8852A) + _show_fbtc_cysta(rtwdev, m); + else + _show_fbtc_cysta_v1(rtwdev, m); + _show_fbtc_nullsta(rtwdev, m); _show_fbtc_step(rtwdev, m); } +static void _get_gnt(struct rtw89_dev *rtwdev, struct rtw89_mac_ax_coex_gnt *gnt_cfg) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_mac_ax_gnt *gnt; + u32 val, status; + + if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B) { + rtw89_mac_read_lte(rtwdev, R_AX_LTE_SW_CFG_1, &val); + rtw89_mac_read_lte(rtwdev, R_AX_GNT_VAL, &status); + + gnt = &gnt_cfg->band[0]; + gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S0_SW_CTRL); + gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S0_STA); + gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S0_SW_CTRL); + gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S0_STA); + + gnt = &gnt_cfg->band[1]; + gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S1_SW_CTRL); + gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S1_STA); + gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S1_SW_CTRL); + gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S1_STA); + } else if (chip->chip_id == RTL8852C) { + val = rtw89_read32(rtwdev, R_AX_GNT_SW_CTRL); + status = rtw89_read32(rtwdev, R_AX_GNT_VAL_V1); + + gnt = &gnt_cfg->band[0]; + gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S0_SWCTRL); + gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S0); + gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S0_SWCTRL); + gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S0); + + gnt = &gnt_cfg->band[1]; + gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S1_SWCTRL); + gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S1); + gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S1_SWCTRL); + gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S1); + } else { + return; + } +} + static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m) { const struct rtw89_chip_info *chip = rtwdev->chip; @@ -5502,7 +6657,8 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m) struct rtw89_btc_cx *cx = &btc->cx; struct rtw89_btc_wl_info *wl = &btc->cx.wl; struct rtw89_btc_bt_info *bt = &btc->cx.bt; - struct rtw89_mac_ax_gnt gnt[2] = {0}; + struct rtw89_mac_ax_coex_gnt gnt_cfg = {}; + struct rtw89_mac_ax_gnt gnt; u8 i = 0, type = 0, cnt = 0; u32 val, offset; @@ -5519,45 +6675,28 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m) /* To avoid I/O if WL LPS or power-off */ if (!wl->status.map.lps && !wl->status.map.rf_off) { - rtw89_mac_read_lte(rtwdev, R_AX_LTE_SW_CFG_1, &val); - if (val & (B_AX_GNT_BT_RFC_S0_SW_VAL | - B_AX_GNT_BT_BB_S0_SW_VAL)) - gnt[0].gnt_bt = true; - if (val & (B_AX_GNT_BT_RFC_S0_SW_CTRL | - B_AX_GNT_BT_BB_S0_SW_CTRL)) - gnt[0].gnt_bt_sw_en = true; - if (val & (B_AX_GNT_WL_RFC_S0_SW_VAL | - B_AX_GNT_WL_BB_S0_SW_VAL)) - gnt[0].gnt_wl = true; - if (val & (B_AX_GNT_WL_RFC_S0_SW_CTRL | - B_AX_GNT_WL_BB_S0_SW_CTRL)) - gnt[0].gnt_wl_sw_en = true; - - if (val & (B_AX_GNT_BT_RFC_S1_SW_VAL | - B_AX_GNT_BT_BB_S1_SW_VAL)) - gnt[1].gnt_bt = true; - if (val & (B_AX_GNT_BT_RFC_S1_SW_CTRL | - B_AX_GNT_BT_BB_S1_SW_CTRL)) - gnt[1].gnt_bt_sw_en = true; - if (val & (B_AX_GNT_WL_RFC_S1_SW_VAL | - B_AX_GNT_WL_BB_S1_SW_VAL)) - gnt[1].gnt_wl = true; - if (val & (B_AX_GNT_WL_RFC_S1_SW_CTRL | - B_AX_GNT_WL_BB_S1_SW_CTRL)) - gnt[1].gnt_wl_sw_en = true; + if (chip->chip_id == RTL8852A) + btc->dm.pta_owner = rtw89_mac_get_ctrl_path(rtwdev); + else if (chip->chip_id == RTL8852C) + btc->dm.pta_owner = 0; + _get_gnt(rtwdev, &gnt_cfg); + gnt = gnt_cfg.band[0]; seq_printf(m, " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], ", "[gnt_status]", - (rtw89_mac_get_ctrl_path(rtwdev) ? "WL" : "BT"), - (gnt[0].gnt_wl_sw_en ? "SW" : "HW"), gnt[0].gnt_wl, - (gnt[0].gnt_bt_sw_en ? "SW" : "HW"), gnt[0].gnt_bt); + chip->chip_id == RTL8852C ? "HW" : + btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT", + gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl, + gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt); + gnt = gnt_cfg.band[1]; seq_printf(m, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n", - (gnt[1].gnt_wl_sw_en ? "SW" : "HW"), gnt[1].gnt_wl, - (gnt[1].gnt_bt_sw_en ? "SW" : "HW"), gnt[1].gnt_bt); + gnt.gnt_wl_sw_en ? "SW" : "HW", + gnt.gnt_wl, + gnt.gnt_bt_sw_en ? "SW" : "HW", + gnt.gnt_bt); } - pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo; if (!pcinfo->valid) { rtw89_debug(rtwdev, RTW89_DBG_BTC, @@ -5714,8 +6853,121 @@ static void _show_summary(struct rtw89_dev *rtwdev, struct seq_file *m) cnt[BTC_NCNT_CUSTOMERIZE]); } +static void _show_summary_v1(struct rtw89_dev *rtwdev, struct seq_file *m) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo; + struct rtw89_btc_fbtc_rpt_ctrl_v1 *prptctrl; + struct rtw89_btc_rpt_cmn_info *pcinfo; + struct rtw89_btc_cx *cx = &btc->cx; + struct rtw89_btc_dm *dm = &btc->dm; + struct rtw89_btc_wl_info *wl = &cx->wl; + struct rtw89_btc_bt_info *bt = &cx->bt; + u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify; + u8 i; + + if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY)) + return; + + seq_puts(m, "========== [Statistics] ==========\n"); + + pcinfo = &pfwinfo->rpt_ctrl.cinfo; + if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) { + prptctrl = &pfwinfo->rpt_ctrl.finfo_v1; + + seq_printf(m, + " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ", + "[summary]", pfwinfo->cnt_h2c, + pfwinfo->cnt_h2c_fail, + le32_to_cpu(prptctrl->rpt_info.cnt_h2c), + pfwinfo->cnt_c2h, + le32_to_cpu(prptctrl->rpt_info.cnt_c2h)); + + seq_printf(m, + "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x", + pfwinfo->event[BTF_EVNT_RPT], + le32_to_cpu(prptctrl->rpt_info.cnt), + le32_to_cpu(prptctrl->rpt_info.en), + dm->error.val); + + if (dm->error.map.wl_fw_hang) + seq_puts(m, " (WL FW Hang!!)"); + seq_puts(m, "\n"); + seq_printf(m, + " %-15s : send_ok:%d, send_fail:%d, recv:%d, ", + "[mailbox]", + le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok), + le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail), + le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv)); + + seq_printf(m, + "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n", + le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty), + le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl), + le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx), + le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack), + le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack)); + + seq_printf(m, + " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]", + "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ], + cx->cnt_wl[BTC_WCNT_RFK_GO], + cx->cnt_wl[BTC_WCNT_RFK_REJECT], + cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]); + + seq_printf(m, + ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n", + le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]), + le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_GO]), + le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REJECT]), + le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]), + le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_FAIL])); + + if (le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0) + bt->rfk_info.map.timeout = 1; + else + bt->rfk_info.map.timeout = 0; + + dm->error.map.wl_rfk_timeout = bt->rfk_info.map.timeout; + } else { + seq_printf(m, + " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x", + "[summary]", pfwinfo->cnt_h2c, + pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h, + pfwinfo->event[BTF_EVNT_RPT], + btc->fwinfo.rpt_en_map); + seq_puts(m, " (WL FW report invalid!!)\n"); + } + + for (i = 0; i < BTC_NCNT_NUM; i++) + cnt_sum += dm->cnt_notify[i]; + + seq_printf(m, + " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ", + "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO], + cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]); + + seq_printf(m, + "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n", + cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE], + cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK], + cnt[BTC_NCNT_WL_STA]); + + seq_printf(m, + " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ", + "[notify_cnt]", cnt[BTC_NCNT_SCAN_START], + cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND], + cnt[BTC_NCNT_SPECIAL_PACKET]); + + seq_printf(m, + "timer=%d, control=%d, customerize=%d\n", + cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL], + cnt[BTC_NCNT_CUSTOMERIZE]); +} + void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_fw_suit *fw_suit = &rtwdev->fw.normal; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_cx *cx = &btc->cx; @@ -5746,5 +6998,8 @@ void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m) _show_dm_info(rtwdev, m); _show_fw_dm_msg(rtwdev, m); _show_mreg(rtwdev, m); - _show_summary(rtwdev, m); + if (chip->chip_id == RTL8852A) + _show_summary(rtwdev, m); + else + _show_summary_v1(rtwdev, m); } diff --git a/drivers/net/wireless/realtek/rtw89/coex.h b/drivers/net/wireless/realtek/rtw89/coex.h index c3a722d259d7fd62564ab1da8f41773428253a13..ca16afa97ec07e571f2824a8d842007737752088 100644 --- a/drivers/net/wireless/realtek/rtw89/coex.h +++ b/drivers/net/wireless/realtek/rtw89/coex.h @@ -162,17 +162,19 @@ void rtw89_coex_act1_work(struct work_struct *work); void rtw89_coex_bt_devinfo_work(struct work_struct *work); void rtw89_coex_rfk_chk_work(struct work_struct *work); void rtw89_coex_power_on(struct rtw89_dev *rtwdev); +void rtw89_btc_set_policy(struct rtw89_dev *rtwdev, u16 policy_type); +void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type); static inline u8 rtw89_btc_phymap(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx, enum rtw89_rf_path_bit paths) { - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 phy_map; phy_map = FIELD_PREP(BTC_RFK_PATH_MAP, paths) | FIELD_PREP(BTC_RFK_PHY_MAP, BIT(phy_idx)) | - FIELD_PREP(BTC_RFK_BAND_MAP, hal->current_band_type); + FIELD_PREP(BTC_RFK_BAND_MAP, chan->band_type); return phy_map; } diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index a5880a54812e75e09be66c2a23b2e6bbd0a74229..bc2994865372b86043e73d45584ba56f4b79d9a5 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -5,6 +5,7 @@ #include #include "cam.h" +#include "chan.h" #include "coex.h" #include "core.h" #include "efuse.h" @@ -224,18 +225,22 @@ static void rtw89_traffic_stats_accu(struct rtw89_dev *rtwdev, } } -static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef, - struct rtw89_channel_params *chan_param) +void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef) +{ + cfg80211_chandef_create(chandef, &rtw89_channels_2ghz[0], + NL80211_CHAN_NO_HT); +} + +static void rtw89_get_channel_params(const struct cfg80211_chan_def *chandef, + struct rtw89_chan *chan) { struct ieee80211_channel *channel = chandef->chan; enum nl80211_chan_width width = chandef->width; u32 primary_freq, center_freq; u8 center_chan; u8 bandwidth = RTW89_CHANNEL_WIDTH_20; - u8 primary_chan_idx = 0; u32 offset; u8 band; - u8 subband; center_chan = channel->hw_value; primary_freq = channel->center_freq; @@ -245,15 +250,12 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef, case NL80211_CHAN_WIDTH_20_NOHT: case NL80211_CHAN_WIDTH_20: bandwidth = RTW89_CHANNEL_WIDTH_20; - primary_chan_idx = RTW89_SC_DONT_CARE; break; case NL80211_CHAN_WIDTH_40: bandwidth = RTW89_CHANNEL_WIDTH_40; if (primary_freq > center_freq) { - primary_chan_idx = RTW89_SC_20_UPPER; center_chan -= 2; } else { - primary_chan_idx = RTW89_SC_20_LOWER; center_chan += 2; } break; @@ -262,11 +264,9 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef, bandwidth = nl_to_rtw89_bandwidth(width); if (primary_freq > center_freq) { offset = (primary_freq - center_freq - 10) / 20; - primary_chan_idx = RTW89_SC_20_UPPER + offset * 2; center_chan -= 2 + offset * 4; } else { offset = (center_freq - primary_freq - 10) / 20; - primary_chan_idx = RTW89_SC_20_LOWER + offset * 2; center_chan += 2 + offset * 4; } break; @@ -288,110 +288,76 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef, break; } - switch (band) { - default: - case RTW89_BAND_2G: - switch (center_chan) { - default: - case 1 ... 14: - subband = RTW89_CH_2G; - break; - } - break; - case RTW89_BAND_5G: - switch (center_chan) { - default: - case 36 ... 64: - subband = RTW89_CH_5G_BAND_1; - break; - case 100 ... 144: - subband = RTW89_CH_5G_BAND_3; - break; - case 149 ... 177: - subband = RTW89_CH_5G_BAND_4; - break; - } - break; - case RTW89_BAND_6G: - switch (center_chan) { - default: - case 1 ... 29: - subband = RTW89_CH_6G_BAND_IDX0; - break; - case 33 ... 61: - subband = RTW89_CH_6G_BAND_IDX1; - break; - case 65 ... 93: - subband = RTW89_CH_6G_BAND_IDX2; - break; - case 97 ... 125: - subband = RTW89_CH_6G_BAND_IDX3; - break; - case 129 ... 157: - subband = RTW89_CH_6G_BAND_IDX4; - break; - case 161 ... 189: - subband = RTW89_CH_6G_BAND_IDX5; - break; - case 193 ... 221: - subband = RTW89_CH_6G_BAND_IDX6; - break; - case 225 ... 253: - subband = RTW89_CH_6G_BAND_IDX7; - break; - } - break; - } + rtw89_chan_create(chan, center_chan, channel->hw_value, band, bandwidth); +} - chan_param->center_chan = center_chan; - chan_param->center_freq = center_freq; - chan_param->primary_chan = channel->hw_value; - chan_param->bandwidth = bandwidth; - chan_param->pri_ch_idx = primary_chan_idx; - chan_param->band_type = band; - chan_param->subband_type = subband; +void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + const struct rtw89_chan *chan; + enum rtw89_sub_entity_idx sub_entity_idx; + enum rtw89_phy_idx phy_idx; + enum rtw89_entity_mode mode; + bool entity_active; + + entity_active = rtw89_get_entity_state(rtwdev); + if (!entity_active) + return; + + mode = rtw89_get_entity_mode(rtwdev); + if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode)) + return; + + sub_entity_idx = RTW89_SUB_ENTITY_0; + phy_idx = RTW89_PHY_0; + chan = rtw89_chan_get(rtwdev, sub_entity_idx); + if (chip->ops->set_txpwr) + chip->ops->set_txpwr(rtwdev, chan, phy_idx); } void rtw89_set_channel(struct rtw89_dev *rtwdev) { - struct ieee80211_hw *hw = rtwdev->hw; const struct rtw89_chip_info *chip = rtwdev->chip; - struct rtw89_hal *hal = &rtwdev->hal; - struct rtw89_channel_params ch_param; + const struct cfg80211_chan_def *chandef; + enum rtw89_sub_entity_idx sub_entity_idx; + enum rtw89_mac_idx mac_idx; + enum rtw89_phy_idx phy_idx; + struct rtw89_chan chan; struct rtw89_channel_help_params bak; - u8 center_chan, bandwidth; + enum rtw89_entity_mode mode; bool band_changed; + bool entity_active; - rtw89_get_channel_params(&hw->conf.chandef, &ch_param); - if (WARN(ch_param.center_chan == 0, "Invalid channel\n")) + entity_active = rtw89_get_entity_state(rtwdev); + + mode = rtw89_entity_recalc(rtwdev); + if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode)) return; - center_chan = ch_param.center_chan; - bandwidth = ch_param.bandwidth; - band_changed = hal->current_band_type != ch_param.band_type || - hal->current_channel == 0; + sub_entity_idx = RTW89_SUB_ENTITY_0; + mac_idx = RTW89_MAC_0; + phy_idx = RTW89_PHY_0; + chandef = rtw89_chandef_get(rtwdev, sub_entity_idx); + rtw89_get_channel_params(chandef, &chan); + if (WARN(chan.channel == 0, "Invalid channel\n")) + return; - hal->current_band_width = bandwidth; - hal->current_channel = center_chan; - hal->current_freq = ch_param.center_freq; - hal->prev_primary_channel = hal->current_primary_channel; - hal->prev_band_type = hal->current_band_type; - hal->current_primary_channel = ch_param.primary_chan; - hal->current_band_type = ch_param.band_type; - hal->current_subband = ch_param.subband_type; + band_changed = rtw89_assign_entity_chan(rtwdev, sub_entity_idx, &chan); - rtw89_chip_set_channel_prepare(rtwdev, &bak); + rtw89_chip_set_channel_prepare(rtwdev, &bak, &chan, mac_idx, phy_idx); - chip->ops->set_channel(rtwdev, &ch_param); + chip->ops->set_channel(rtwdev, &chan, mac_idx, phy_idx); - rtw89_chip_set_txpwr(rtwdev); + rtw89_core_set_chip_txpwr(rtwdev); - rtw89_chip_set_channel_done(rtwdev, &bak); + rtw89_chip_set_channel_done(rtwdev, &bak, &chan, mac_idx, phy_idx); - if (band_changed) { - rtw89_btc_ntfy_switch_band(rtwdev, RTW89_PHY_0, hal->current_band_type); - rtw89_chip_rfk_band_changed(rtwdev); + if (!entity_active || band_changed) { + rtw89_btc_ntfy_switch_band(rtwdev, phy_idx, chan.band_type); + rtw89_chip_rfk_band_changed(rtwdev, phy_idx); } + + rtw89_set_entity_state(rtwdev, true); } static enum rtw89_core_tx_type @@ -529,9 +495,15 @@ static u16 rtw89_core_get_mgmt_rate(struct rtw89_dev *rtwdev, struct sk_buff *skb = tx_req->skb; struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb); struct ieee80211_vif *vif = tx_info->control.vif; - struct rtw89_hal *hal = &rtwdev->hal; - u16 lowest_rate = hal->current_band_type == RTW89_BAND_2G ? - RTW89_HW_RATE_CCK1 : RTW89_HW_RATE_OFDM6; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u16 lowest_rate; + + if (tx_info->flags & IEEE80211_TX_CTL_NO_CCK_RATE || vif->p2p) + lowest_rate = RTW89_HW_RATE_OFDM6; + else if (chan->band_type == RTW89_BAND_2G) + lowest_rate = RTW89_HW_RATE_CCK1; + else + lowest_rate = RTW89_HW_RATE_OFDM6; if (!vif || !vif->bss_conf.basic_rates || !tx_req->sta) return lowest_rate; @@ -546,6 +518,7 @@ rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 qsel, ch_dma; qsel = desc_info->hiq ? RTW89_TX_QSEL_B0_HI : RTW89_TX_QSEL_B0_MGMT; @@ -564,9 +537,9 @@ rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev, desc_info->data_rate = rtw89_core_get_mgmt_rate(rtwdev, tx_req); rtw89_debug(rtwdev, RTW89_DBG_TXRX, - "tx mgmt frame with rate 0x%x on channel %d (bw %d)\n", - desc_info->data_rate, rtwdev->hal.current_channel, - rtwdev->hal.current_band_width); + "tx mgmt frame with rate 0x%x on channel %d (band %d, bw %d)\n", + desc_info->data_rate, chan->channel, chan->band_type, + chan->band_width); } static void @@ -591,15 +564,16 @@ static void rtw89_core_get_no_ul_ofdma_htc(struct rtw89_dev *rtwdev, __le32 *htc }; const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 om_bandwidth; if (!chip->dis_2g_40m_ul_ofdma || - hal->current_band_type != RTW89_BAND_2G || - hal->current_band_width != RTW89_CHANNEL_WIDTH_40) + chan->band_type != RTW89_BAND_2G || + chan->band_width != RTW89_CHANNEL_WIDTH_40) return; - om_bandwidth = hal->current_band_width < ARRAY_SIZE(rtw89_bandwidth_to_om) ? - rtw89_bandwidth_to_om[hal->current_band_width] : 0; + om_bandwidth = chan->band_width < ARRAY_SIZE(rtw89_bandwidth_to_om) ? + rtw89_bandwidth_to_om[chan->band_width] : 0; *htc = le32_encode_bits(RTW89_HTC_VARIANT_HE, RTW89_HTC_MASK_VARIANT) | le32_encode_bits(RTW89_HTC_VARIANT_HE_CID_OM, RTW89_HTC_MASK_CTL_ID) | le32_encode_bits(hal->rx_nss - 1, RTW89_HTC_MASK_HTC_OM_RX_NSS) | @@ -617,6 +591,7 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev, enum btc_pkt_type pkt_type) { struct ieee80211_sta *sta = tx_req->sta; + struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta); struct sk_buff *skb = tx_req->skb; struct ieee80211_hdr *hdr = (void *)skb->data; __le16 fc = hdr->frame_control; @@ -634,6 +609,9 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev, if (skb_headroom(skb) < IEEE80211_HT_CTL_LEN) return false; + if (rtwsta && rtwsta->ra_report.might_fallback_legacy) + return false; + return true; } @@ -713,7 +691,7 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif = tx_req->vif; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern; - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; struct sk_buff *skb = tx_req->skb; u8 tid, tid_indicate; @@ -736,9 +714,11 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev, if (IEEE80211_SKB_CB(skb)->control.hw_key) rtw89_core_tx_update_sec_key(rtwdev, tx_req); - if (rate_pattern->enable) + if (vif->p2p) + desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6; + else if (rate_pattern->enable) desc_info->data_retry_lowest_rate = rate_pattern->rate; - else if (hal->current_band_type == RTW89_BAND_2G) + else if (chan->band_type == RTW89_BAND_2G) desc_info->data_retry_lowest_rate = RTW89_HW_RATE_CCK1; else desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6; @@ -796,13 +776,16 @@ static void rtw89_core_tx_wake(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { + const struct rtw89_chip_info *chip = rtwdev->chip; + if (!RTW89_CHK_FW_FEATURE(TX_WAKE, &rtwdev->fw)) return; if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags)) return; - if (tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT) + if (chip->chip_id != RTL8852C && + tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT) return; rtw89_mac_notify_wake(rtwdev); @@ -872,6 +855,7 @@ int rtw89_h2c_tx(struct rtw89_dev *rtwdev, rtw89_debug(rtwdev, RTW89_DBG_FW, "ignore h2c due to power is off with firmware state=%d\n", test_bit(RTW89_FLAG_FW_RDY, rtwdev->flags)); + dev_kfree_skb(skb); return 0; } @@ -1021,7 +1005,8 @@ static __le32 rtw89_build_txwd_info0(struct rtw89_tx_desc_info *desc_info) static __le32 rtw89_build_txwd_info0_v1(struct rtw89_tx_desc_info *desc_info) { - u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb); + u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb) | + FIELD_PREP(RTW89_TXWD_INFO0_MULTIPORT_ID, desc_info->port); return cpu_to_le32(dword); } @@ -1171,9 +1156,14 @@ static void rtw89_core_rx_process_phy_ppdu_iter(void *data, { struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct rtw89_rx_phy_ppdu *phy_ppdu = (struct rtw89_rx_phy_ppdu *)data; + struct rtw89_dev *rtwdev = rtwsta->rtwdev; + int i; - if (rtwsta->mac_id == phy_ppdu->mac_id && phy_ppdu->to_self) + if (rtwsta->mac_id == phy_ppdu->mac_id && phy_ppdu->to_self) { ewma_rssi_add(&rtwsta->avg_rssi, phy_ppdu->rssi_avg); + for (i = 0; i < rtwdev->chip->rf_path_num; i++) + ewma_rssi_add(&rtwsta->rssi[i], phy_ppdu->rssi[i]); + } } #define VAR_LEN 0xff @@ -1229,15 +1219,15 @@ static int rtw89_core_process_phy_status_ie(struct rtw89_dev *rtwdev, u8 *addr, static void rtw89_core_update_phy_ppdu(struct rtw89_rx_phy_ppdu *phy_ppdu) { - s8 *rssi = phy_ppdu->rssi; + u8 *rssi = phy_ppdu->rssi; u8 *buf = phy_ppdu->buf; phy_ppdu->ie = RTW89_GET_PHY_STS_IE_MAP(buf); phy_ppdu->rssi_avg = RTW89_GET_PHY_STS_RSSI_AVG(buf); - rssi[RF_PATH_A] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_A(buf)); - rssi[RF_PATH_B] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_B(buf)); - rssi[RF_PATH_C] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_C(buf)); - rssi[RF_PATH_D] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_D(buf)); + rssi[RF_PATH_A] = RTW89_GET_PHY_STS_RSSI_A(buf); + rssi[RF_PATH_B] = RTW89_GET_PHY_STS_RSSI_B(buf); + rssi[RF_PATH_C] = RTW89_GET_PHY_STS_RSSI_C(buf); + rssi[RF_PATH_D] = RTW89_GET_PHY_STS_RSSI_D(buf); } static int rtw89_core_rx_process_phy_ppdu(struct rtw89_dev *rtwdev, @@ -1448,8 +1438,11 @@ static void rtw89_core_rx_stats(struct rtw89_dev *rtwdev, static void rtw89_correct_cck_chan(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *status) { - u16 chan = rtwdev->hal.prev_primary_channel; - u8 band = chan <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; + const struct rtw89_chan_rcd *rcd = + rtw89_chan_rcd_get(rtwdev, RTW89_SUB_ENTITY_0); + u16 chan = rcd->prev_primary_channel; + u8 band = rcd->prev_band_type == RTW89_BAND_2G ? + NL80211_BAND_2GHZ : NL80211_BAND_5GHZ; if (status->band != NL80211_BAND_2GHZ && status->encoding == RX_ENC_LEGACY && @@ -1661,19 +1654,20 @@ static void rtw89_core_update_rx_status(struct rtw89_dev *rtwdev, struct rtw89_rx_desc_info *desc_info, struct ieee80211_rx_status *rx_status) { - struct ieee80211_hw *hw = rtwdev->hw; - struct rtw89_hal *hal = &rtwdev->hal; + const struct cfg80211_chan_def *chandef = + rtw89_chandef_get(rtwdev, RTW89_SUB_ENTITY_0); + const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u16 data_rate; u8 data_rate_mode; /* currently using single PHY */ - rx_status->freq = hw->conf.chandef.chan->center_freq; - rx_status->band = hw->conf.chandef.chan->band; + rx_status->freq = chandef->chan->center_freq; + rx_status->band = chandef->chan->band; if (rtwdev->scanning && RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) { - u8 chan = hal->current_primary_channel; - u8 band = hal->current_band_type; + u8 chan = cur->primary_channel; + u8 band = cur->band_type; enum nl80211_band nl_band; nl_band = rtw89_hw_to_nl80211_band(band); @@ -1727,7 +1721,8 @@ static enum rtw89_ps_mode rtw89_update_ps_mode(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; - if (rtw89_disable_ps_mode || !chip->ps_mode_supported) + if (rtw89_disable_ps_mode || !chip->ps_mode_supported || + RTW89_CHK_FW_FEATURE(NO_DEEP_PS, &rtwdev->fw)) return RTW89_PS_MODE_NONE; if (chip->ps_mode_supported & BIT(RTW89_PS_MODE_PWR_GATED)) @@ -1810,7 +1805,7 @@ void rtw89_core_napi_init(struct rtw89_dev *rtwdev) { init_dummy_netdev(&rtwdev->netdev); netif_napi_add(&rtwdev->netdev, &rtwdev->napi, - rtwdev->hci.ops->napi_poll, NAPI_POLL_WEIGHT); + rtwdev->hci.ops->napi_poll); } EXPORT_SYMBOL(rtw89_core_napi_init); @@ -1907,21 +1902,14 @@ static void rtw89_core_stop_tx_ba_session(struct rtw89_dev *rtwdev, return; spin_lock_bh(&rtwdev->ba_lock); - if (!list_empty(&rtwtxq->list)) { - list_del_init(&rtwtxq->list); - goto out; - } - - set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); + if (!test_and_set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags)) + list_add_tail(&rtwtxq->list, &rtwdev->forbid_ba_list); + spin_unlock_bh(&rtwdev->ba_lock); - list_add_tail(&rtwtxq->list, &rtwdev->forbid_ba_list); ieee80211_stop_tx_ba_session(sta, txq->tid); cancel_delayed_work(&rtwdev->forbid_ba_work); ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->forbid_ba_work, RTW89_FORBID_BA_TIMER); - -out: - spin_unlock_bh(&rtwdev->ba_lock); } static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev, @@ -1933,6 +1921,9 @@ static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta = txq->sta; struct rtw89_sta *rtwsta = sta ? (struct rtw89_sta *)sta->drv_priv : NULL; + if (test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags)) + return; + if (unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE))) { rtw89_core_stop_tx_ba_session(rtwdev, rtwtxq); return; @@ -1941,9 +1932,6 @@ static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev, if (unlikely(!sta)) return; - if (test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags)) - return; - if (unlikely(test_bit(RTW89_TXQ_F_BLOCK_BA, &rtwtxq->flags))) return; @@ -2179,12 +2167,13 @@ static bool rtw89_traffic_stats_track(struct rtw89_dev *rtwdev) static void rtw89_vif_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { - if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION) + if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION && + rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT) return; if (rtwvif->stats.tx_tfc_lv == RTW89_TFC_IDLE && rtwvif->stats.rx_tfc_lv == RTW89_TFC_IDLE) - rtw89_enter_lps(rtwdev, rtwvif->mac_id); + rtw89_enter_lps(rtwdev, rtwvif); } static void rtw89_enter_lps_track(struct rtw89_dev *rtwdev) @@ -2237,6 +2226,7 @@ static void rtw89_track_work(struct work_struct *work) rtw89_chip_rfk_track(rtwdev); rtw89_phy_ra_update(rtwdev); rtw89_phy_cfo_track(rtwdev); + rtw89_phy_tx_path_div_track(rtwdev); if (rtwdev->lps_enabled && !rtwdev->btc.lps) rtw89_enter_lps_track(rtwdev); @@ -2266,45 +2256,69 @@ void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits) bitmap_zero(addr, nbits); } -int rtw89_core_acquire_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) +int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) { - struct rtw89_ba_cam_entry *entry; + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_cam_info *cam_info = &rtwdev->cam_info; + struct rtw89_ba_cam_entry *entry = NULL, *tmp; u8 idx; + int i; - idx = rtw89_core_acquire_bit_map(rtwsta->ba_cam_map, RTW89_BA_CAM_NUM); - if (idx == RTW89_BA_CAM_NUM) { - /* allocate a static BA CAM to tid=0, so replace the existing + lockdep_assert_held(&rtwdev->mutex); + + idx = rtw89_core_acquire_bit_map(cam_info->ba_cam_map, chip->bacam_num); + if (idx == chip->bacam_num) { + /* allocate a static BA CAM to tid=0/5, so replace the existing * one if BA CAM is full. Hardware will process the original tid * automatically. */ - if (tid != 0) + if (tid != 0 && tid != 5) return -ENOSPC; - idx = 0; + for_each_set_bit(i, cam_info->ba_cam_map, chip->bacam_num) { + tmp = &cam_info->ba_cam_entry[i]; + if (tmp->tid == 0 || tmp->tid == 5) + continue; + + idx = i; + entry = tmp; + list_del(&entry->list); + break; + } + + if (!entry) + return -ENOSPC; + } else { + entry = &cam_info->ba_cam_entry[idx]; } - entry = &rtwsta->ba_cam_entry[idx]; entry->tid = tid; + list_add_tail(&entry->list, &rtwsta->ba_cam_list); + *cam_idx = idx; return 0; } -int rtw89_core_release_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) +int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx) { - struct rtw89_ba_cam_entry *entry; - int i; + struct rtw89_cam_info *cam_info = &rtwdev->cam_info; + struct rtw89_ba_cam_entry *entry = NULL, *tmp; + u8 idx; - for (i = 0; i < RTW89_BA_CAM_NUM; i++) { - if (!test_bit(i, rtwsta->ba_cam_map)) - continue; + lockdep_assert_held(&rtwdev->mutex); - entry = &rtwsta->ba_cam_entry[i]; + list_for_each_entry_safe(entry, tmp, &rtwsta->ba_cam_list, list) { if (entry->tid != tid) continue; - rtw89_core_release_bit_map(rtwsta->ba_cam_map, i); - *cam_idx = i; + idx = entry - cam_info->ba_cam_entry; + list_del(&entry->list); + + rtw89_core_release_bit_map(cam_info->ba_cam_map, idx); + *cam_idx = idx; return 0; } @@ -2320,9 +2334,19 @@ void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc) struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; switch (vif->type) { + case NL80211_IFTYPE_STATION: + if (vif->p2p) + rtwvif->wifi_role = RTW89_WIFI_ROLE_P2P_CLIENT; + else + rtwvif->wifi_role = RTW89_WIFI_ROLE_STATION; + break; + case NL80211_IFTYPE_AP: + if (vif->p2p) + rtwvif->wifi_role = RTW89_WIFI_ROLE_P2P_GO; + else + rtwvif->wifi_role = RTW89_WIFI_ROLE_AP; + break; RTW89_TYPE_MAPPING(ADHOC); - RTW89_TYPE_MAPPING(STATION); - RTW89_TYPE_MAPPING(AP); RTW89_TYPE_MAPPING(MONITOR); RTW89_TYPE_MAPPING(MESH_POINT); default: @@ -2365,13 +2389,17 @@ int rtw89_core_sta_add(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; int i; + rtwsta->rtwdev = rtwdev; rtwsta->rtwvif = rtwvif; rtwsta->prev_rssi = 0; + INIT_LIST_HEAD(&rtwsta->ba_cam_list); for (i = 0; i < ARRAY_SIZE(sta->txq); i++) rtw89_core_txq_init(rtwdev, sta->txq[i]); ewma_rssi_init(&rtwsta->avg_rssi); + for (i = 0; i < rtwdev->chip->rf_path_num; i++) + ewma_rssi_init(&rtwsta->rssi[i]); if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) { /* for station mode, assign the mac_id from itself */ @@ -2541,6 +2569,60 @@ int rtw89_core_sta_remove(struct rtw89_dev *rtwdev, return 0; } +static void _rtw89_core_set_tid_config(struct rtw89_dev *rtwdev, + struct ieee80211_sta *sta, + struct cfg80211_tid_cfg *tid_conf) +{ + struct ieee80211_txq *txq; + struct rtw89_txq *rtwtxq; + u32 mask = tid_conf->mask; + u8 tids = tid_conf->tids; + int tids_nbit = BITS_PER_BYTE; + int i; + + for (i = 0; i < tids_nbit; i++, tids >>= 1) { + if (!tids) + break; + + if (!(tids & BIT(0))) + continue; + + txq = sta->txq[i]; + rtwtxq = (struct rtw89_txq *)txq->drv_priv; + + if (mask & BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL)) { + if (tid_conf->ampdu == NL80211_TID_CONFIG_ENABLE) { + clear_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); + } else { + if (test_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags)) + ieee80211_stop_tx_ba_session(sta, txq->tid); + spin_lock_bh(&rtwdev->ba_lock); + list_del_init(&rtwtxq->list); + set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags); + spin_unlock_bh(&rtwdev->ba_lock); + } + } + + if (mask & BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL) && tids == 0xff) { + if (tid_conf->amsdu == NL80211_TID_CONFIG_ENABLE) + sta->max_amsdu_subframes = 0; + else + sta->max_amsdu_subframes = 1; + } + } +} + +void rtw89_core_set_tid_config(struct rtw89_dev *rtwdev, + struct ieee80211_sta *sta, + struct cfg80211_tid_config *tid_config) +{ + int i; + + for (i = 0; i < tid_config->n_tid_conf; i++) + _rtw89_core_set_tid_config(rtwdev, sta, + &tid_config->tid_conf[i]); +} + static void rtw89_init_ht_cap(struct rtw89_dev *rtwdev, struct ieee80211_sta_ht_cap *ht_cap) { @@ -2669,8 +2751,7 @@ static void rtw89_init_he_cap(struct rtw89_dev *rtwdev, phy_cap_info = he_cap->he_cap_elem.phy_cap_info; he_cap->has_he = true; - if (i == NL80211_IFTYPE_AP) - mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE; + mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE; if (i == NL80211_IFTYPE_STATION) mac_cap_info[1] = IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_ALL_ACK | @@ -2706,6 +2787,8 @@ static void rtw89_init_he_cap(struct rtw89_dev *rtwdev, phy_cap_info[3] |= IEEE80211_HE_PHY_CAP3_RX_PARTIAL_BW_SU_IN_20MHZ_MU; phy_cap_info[4] = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4; + if (chip->support_bw160) + phy_cap_info[4] |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4; phy_cap_info[5] = no_ng16 ? 0 : IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK | IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK; @@ -2866,7 +2949,9 @@ int rtw89_core_start(struct rtw89_dev *rtwdev) /* efuse process */ /* pre-config BB/RF, BB reset/RFC reset */ - rtw89_chip_disable_bb_rf(rtwdev); + ret = rtw89_chip_disable_bb_rf(rtwdev); + if (ret) + return ret; ret = rtw89_chip_enable_bb_rf(rtwdev); if (ret) return ret; @@ -2894,6 +2979,7 @@ int rtw89_core_start(struct rtw89_dev *rtwdev) rtw89_btc_ntfy_radio_state(rtwdev, BTC_RFCTRL_WL_ON); rtw89_fw_h2c_fw_log(rtwdev, rtwdev->fw.fw_log_enable); + rtw89_fw_h2c_init_ba_cam(rtwdev); return 0; } @@ -2987,6 +3073,7 @@ int rtw89_core_init(struct rtw89_dev *rtwdev) return ret; } rtw89_ser_init(rtwdev); + rtw89_entity_init(rtwdev); return 0; } @@ -3007,7 +3094,7 @@ EXPORT_SYMBOL(rtw89_core_deinit); void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, const u8 *mac_addr, bool hw_scan) { - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); rtwdev->scanning = true; rtw89_leave_lps(rtwdev); @@ -3015,7 +3102,7 @@ void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, rtw89_leave_ips(rtwdev); ether_addr_copy(rtwvif->mac_addr, mac_addr); - rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, hal->current_band_type); + rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, chan->band_type); rtw89_chip_rfk_scan(rtwdev, true); rtw89_hci_recalc_int_mit(rtwdev); @@ -3141,6 +3228,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev) hw->vif_data_size = sizeof(struct rtw89_vif); hw->sta_data_size = sizeof(struct rtw89_sta); hw->txq_data_size = sizeof(struct rtw89_txq); + hw->chanctx_data_size = sizeof(struct rtw89_chanctx_cfg); SET_IEEE80211_PERM_ADDR(hw, efuse->addr); @@ -3148,6 +3236,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev) hw->queues = IEEE80211_NUM_ACS; hw->max_rx_aggregation_subframes = RTW89_MAX_RX_AGG_NUM; hw->max_tx_aggregation_subframes = RTW89_MAX_TX_AGG_NUM; + hw->uapsd_max_sp_len = IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL; ieee80211_hw_set(hw, SIGNAL_DBM); ieee80211_hw_set(hw, HAS_RATE_CONTROL); @@ -3164,17 +3253,26 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev) ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) | - BIT(NL80211_IFTYPE_AP); + BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO); + hw->wiphy->available_antennas_tx = BIT(rtwdev->chip->rf_path_num) - 1; hw->wiphy->available_antennas_rx = BIT(rtwdev->chip->rf_path_num) - 1; hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS | - WIPHY_FLAG_TDLS_EXTERNAL_SETUP; + WIPHY_FLAG_TDLS_EXTERNAL_SETUP | + WIPHY_FLAG_AP_UAPSD; hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR; hw->wiphy->max_scan_ssids = RTW89_SCANOFLD_MAX_SSID; hw->wiphy->max_scan_ie_len = RTW89_SCANOFLD_MAX_IE_LEN; + hw->wiphy->tid_config_support.vif |= BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL); + hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL); + hw->wiphy->tid_config_support.vif |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL); + hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL); + wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0); ret = rtw89_core_set_supported_band(rtwdev); @@ -3234,6 +3332,63 @@ void rtw89_core_unregister(struct rtw89_dev *rtwdev) } EXPORT_SYMBOL(rtw89_core_unregister); +struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device, + u32 bus_data_size, + const struct rtw89_chip_info *chip) +{ + struct ieee80211_hw *hw; + struct rtw89_dev *rtwdev; + struct ieee80211_ops *ops; + u32 driver_data_size; + u32 early_feat_map = 0; + bool no_chanctx; + + rtw89_early_fw_feature_recognize(device, chip, &early_feat_map); + + ops = kmemdup(&rtw89_ops, sizeof(rtw89_ops), GFP_KERNEL); + if (!ops) + goto err; + + no_chanctx = chip->support_chanctx_num == 0 || + !(early_feat_map & BIT(RTW89_FW_FEATURE_SCAN_OFFLOAD)); + + if (no_chanctx) { + ops->add_chanctx = NULL; + ops->remove_chanctx = NULL; + ops->change_chanctx = NULL; + ops->assign_vif_chanctx = NULL; + ops->unassign_vif_chanctx = NULL; + } + + driver_data_size = sizeof(struct rtw89_dev) + bus_data_size; + hw = ieee80211_alloc_hw(driver_data_size, ops); + if (!hw) + goto err; + + rtwdev = hw->priv; + rtwdev->hw = hw; + rtwdev->dev = device; + rtwdev->ops = ops; + rtwdev->chip = chip; + + rtw89_debug(rtwdev, RTW89_DBG_FW, "probe driver %s chanctx\n", + no_chanctx ? "without" : "with"); + + return rtwdev; + +err: + kfree(ops); + return NULL; +} +EXPORT_SYMBOL(rtw89_alloc_ieee80211_hw); + +void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev) +{ + kfree(rtwdev->ops); + ieee80211_free_hw(rtwdev->hw); +} +EXPORT_SYMBOL(rtw89_free_ieee80211_hw); + MODULE_AUTHOR("Realtek Corporation"); MODULE_DESCRIPTION("Realtek 802.11ax wireless core module"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index 7a9d6f5d8a5136d7d86034700ead0aafc8033f58..db041b32a8c2cf5a7eb0236ca7e4ee09ead21529 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -34,6 +34,7 @@ extern const struct ieee80211_ops rtw89_ops; #define MAX_RSSI 110 #define RSSI_FACTOR 1 #define RTW89_RSSI_RAW_TO_DBM(rssi) ((s8)((rssi) >> RSSI_FACTOR) - MAX_RSSI) +#define RTW89_TX_DIV_RSSI_RAW_TH (2 << RSSI_FACTOR) #define RTW89_HTC_MASK_VARIANT GENMASK(1, 0) #define RTW89_HTC_VARIANT_HE 3 @@ -522,7 +523,7 @@ struct rtw89_rx_phy_ppdu { u8 *buf; u32 len; u8 rssi_avg; - s8 rssi[RF_PATH_MAX]; + u8 rssi[RF_PATH_MAX]; u8 mac_id; u8 chan_idx; u8 ie; @@ -542,6 +543,12 @@ enum rtw89_phy_idx { RTW89_PHY_MAX }; +enum rtw89_sub_entity_idx { + RTW89_SUB_ENTITY_0 = 0, + + NUM_OF_RTW89_SUB_ENTITY, +}; + enum rtw89_rf_path { RF_PATH_A = 0, RF_PATH_B = 1, @@ -624,14 +631,23 @@ enum rtw89_sc_offset { RTW89_SC_40_LOWER = 10, }; -struct rtw89_channel_params { - u8 center_chan; - u32 center_freq; - u8 primary_chan; - u8 bandwidth; - u8 pri_ch_idx; - u8 band_type; - u8 subband_type; +struct rtw89_chan { + u8 channel; + u8 primary_channel; + enum rtw89_band band_type; + enum rtw89_bandwidth band_width; + + /* The follow-up are derived from the above. We must ensure that it + * is assigned correctly in rtw89_chan_create() if new one is added. + */ + u32 freq; + enum rtw89_subband subband_type; + enum rtw89_sc_offset pri_ch_idx; +}; + +struct rtw89_chan_rcd { + u8 prev_primary_channel; + enum rtw89_band prev_band_type; }; struct rtw89_channel_help_params { @@ -793,7 +809,7 @@ struct rtw89_mac_ax_gnt { u8 gnt_bt; u8 gnt_wl_sw_en; u8 gnt_wl; -}; +} __packed; #define RTW89_MAC_AX_COEX_GNT_NR 2 struct rtw89_mac_ax_coex_gnt { @@ -848,6 +864,7 @@ enum rtw89_btc_dcnt { BTC_DCNT_SLOT_NONSYNC, BTC_DCNT_BTCNT_FREEZE, BTC_DCNT_WL_SLOT_DRIFT, + BTC_DCNT_BT_SLOT_DRIFT, BTC_DCNT_WL_STA_LAST, BTC_DCNT_NUM, }; @@ -920,12 +937,12 @@ struct rtw89_btc_wl_smap { u32 roaming: 1; u32 _4way: 1; u32 rf_off: 1; - u32 lps: 1; + u32 lps: 2; u32 ips: 1; u32 init_ok: 1; u32 traffic_dir : 2; u32 rf_off_pre: 1; - u32 lps_pre: 1; + u32 lps_pre: 2; }; enum rtw89_tfc_lv { @@ -1108,6 +1125,27 @@ struct rtw89_btc_wl_active_role { u16 rx_rate; }; +struct rtw89_btc_wl_active_role_v1 { + u8 connected: 1; + u8 pid: 3; + u8 phy: 1; + u8 noa: 1; + u8 band: 2; + + u8 client_ps: 1; + u8 bw: 7; + + u8 role; + u8 ch; + + u16 tx_lvl; + u16 rx_lvl; + u16 tx_rate; + u16 rx_rate; + + u32 noa_duration; /* ms */ +}; + struct rtw89_btc_wl_role_info_bpos { u16 none: 1; u16 station: 1; @@ -1123,6 +1161,12 @@ struct rtw89_btc_wl_role_info_bpos { u16 nan: 1; }; +struct rtw89_btc_wl_scc_ctrl { + u8 null_role1; + u8 null_role2; + u8 ebt_null; /* if tx null at EBT slot */ +}; + union rtw89_btc_wl_role_info_map { u16 val; struct rtw89_btc_wl_role_info_bpos role; @@ -1135,6 +1179,21 @@ struct rtw89_btc_wl_role_info { /* struct size must be n*4 bytes */ struct rtw89_btc_wl_active_role active_role[RTW89_PORT_NUM]; }; +struct rtw89_btc_wl_role_info_v1 { /* struct size must be n*4 bytes */ + u8 connect_cnt; + u8 link_mode; + union rtw89_btc_wl_role_info_map role_map; + struct rtw89_btc_wl_active_role_v1 active_role_v1[RTW89_PORT_NUM]; + u32 mrole_type; /* btc_wl_mrole_type */ + u32 mrole_noa_duration; /* ms */ + + u32 dbcc_en: 1; + u32 dbcc_chg: 1; + u32 dbcc_2g_phy: 2; /* which phy operate in 2G, HW_PHY_0 or HW_PHY_1 */ + u32 link_mode_chg: 1; + u32 rsvd: 27; +}; + struct rtw89_btc_wl_ver_info { u32 fw_coex; /* match with which coex_ver */ u32 fw; @@ -1240,6 +1299,7 @@ struct rtw89_btc_wl_info { struct rtw89_btc_wl_ver_info ver_info; struct rtw89_btc_wl_afh_info afh_info; struct rtw89_btc_wl_role_info role_info; + struct rtw89_btc_wl_role_info_v1 role_info_v1; struct rtw89_btc_wl_scan_info scan_info; struct rtw89_btc_wl_dbcc_info dbcc_info; struct rtw89_btc_rf_para rf_para; @@ -1248,6 +1308,7 @@ struct rtw89_btc_wl_info { u8 port_id[RTW89_WIFI_ROLE_MLME_MAX]; u8 rssi_level; + bool scbd_change; u32 scbd; }; @@ -1333,7 +1394,8 @@ struct rtw89_btc_bt_info { u32 pag: 1; u32 run_patch_code: 1; u32 hi_lna_rx: 1; - u32 rsvd: 22; + u32 scan_rx_low_pri: 1; + u32 rsvd: 21; }; struct rtw89_btc_cx { @@ -1346,32 +1408,43 @@ struct rtw89_btc_cx { }; struct rtw89_btc_fbtc_tdma { - u8 type; + u8 type; /* chip_info::fcxtdma_ver */ u8 rxflctrl; u8 txpause; u8 wtgle_n; u8 leak_n; u8 ext_ctrl; - u8 rsvd0; - u8 rsvd1; + u8 rxflctrl_role; + u8 option_ctrl; +} __packed; + +struct rtw89_btc_fbtc_tdma_v1 { + u8 fver; /* chip_info::fcxtdma_ver */ + u8 rsvd; + __le16 rsvd1; + struct rtw89_btc_fbtc_tdma tdma; } __packed; #define CXMREG_MAX 30 #define FCXMAX_STEP 255 /*STEP trace record cnt, Max:65535, default:255*/ -#define BTCRPT_VER 1 #define BTC_CYCLE_SLOT_MAX 48 /* must be even number, non-zero */ -enum rtw89_btc_bt_rfk_counter { +enum rtw89_btc_bt_sta_counter { BTC_BCNT_RFK_REQ = 0, BTC_BCNT_RFK_GO = 1, BTC_BCNT_RFK_REJECT = 2, BTC_BCNT_RFK_FAIL = 3, BTC_BCNT_RFK_TIMEOUT = 4, - BTC_BCNT_RFK_MAX + BTC_BCNT_HI_TX = 5, + BTC_BCNT_HI_RX = 6, + BTC_BCNT_LO_TX = 7, + BTC_BCNT_LO_RX = 8, + BTC_BCNT_POLLUTED = 9, + BTC_BCNT_STA_MAX }; struct rtw89_btc_fbtc_rpt_ctrl { - u16 fver; + u16 fver; /* chip_info::fcxbtcrpt_ver */ u16 rpt_cnt; /* tmr counters */ u32 wl_fw_coex_ver; /* match which driver's coex version */ u32 wl_fw_cx_offload; @@ -1384,11 +1457,56 @@ struct rtw89_btc_fbtc_rpt_ctrl { u32 mb_a2dp_empty_cnt; /* a2dp empty count */ u32 mb_a2dp_flct_cnt; /* a2dp empty flow control counter */ u32 mb_a2dp_full_cnt; /* a2dp empty full counter */ - u32 bt_rfk_cnt[BTC_BCNT_RFK_MAX]; + u32 bt_rfk_cnt[BTC_BCNT_HI_TX]; u32 c2h_cnt; /* fw send c2h counter */ u32 h2c_cnt; /* fw recv h2c counter */ } __packed; +struct rtw89_btc_fbtc_rpt_ctrl_info { + __le32 cnt; /* fw report counter */ + __le32 en; /* report map */ + __le32 para; /* not used */ + + __le32 cnt_c2h; /* fw send c2h counter */ + __le32 cnt_h2c; /* fw recv h2c counter */ + __le32 len_c2h; /* The total length of the last C2H */ + + __le32 cnt_aoac_rf_on; /* rf-on counter for aoac switch notify */ + __le32 cnt_aoac_rf_off; /* rf-off counter for aoac switch notify */ +} __packed; + +struct rtw89_btc_fbtc_rpt_ctrl_wl_fw_info { + __le32 cx_ver; /* match which driver's coex version */ + __le32 cx_offload; + __le32 fw_ver; +} __packed; + +struct rtw89_btc_fbtc_rpt_ctrl_a2dp_empty { + __le32 cnt_empty; /* a2dp empty count */ + __le32 cnt_flowctrl; /* a2dp empty flow control counter */ + __le32 cnt_tx; + __le32 cnt_ack; + __le32 cnt_nack; +} __packed; + +struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox { + __le32 cnt_send_ok; /* fw send mailbox ok counter */ + __le32 cnt_send_fail; /* fw send mailbox fail counter */ + __le32 cnt_recv; /* fw recv mailbox counter */ + struct rtw89_btc_fbtc_rpt_ctrl_a2dp_empty a2dp; +} __packed; + +struct rtw89_btc_fbtc_rpt_ctrl_v1 { + u8 fver; + u8 rsvd; + __le16 rsvd1; + struct rtw89_btc_fbtc_rpt_ctrl_info rpt_info; + struct rtw89_btc_fbtc_rpt_ctrl_wl_fw_info wl_fw_info; + struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox bt_mbx_info; + __le32 bt_cnt[BTC_BCNT_STA_MAX]; + struct rtw89_mac_ax_gnt gnt_val[RTW89_PHY_MAX]; +} __packed; + enum rtw89_fbtc_ext_ctrl_type { CXECTL_OFF = 0x0, /* tdma off */ CXECTL_B2 = 0x1, /* allow B2 (beacon-early) */ @@ -1457,10 +1575,9 @@ enum { /* STEP TYPE */ CXSTEP_MAX, }; -#define FCXGPIODBG_VER 1 #define BTC_DBG_MAX1 32 struct rtw89_btc_fbtc_gpio_dbg { - u8 fver; + u8 fver; /* chip_info::fcxgpiodbg_ver */ u8 rsvd; u16 rsvd2; u32 en_map; /* which debug signal (see btc_wl_gpio_debug) is enable */ @@ -1468,9 +1585,8 @@ struct rtw89_btc_fbtc_gpio_dbg { u8 gpio_map[BTC_DBG_MAX1]; /*the debug signals to GPIO-Position */ } __packed; -#define FCXMREG_VER 1 struct rtw89_btc_fbtc_mreg_val { - u8 fver; + u8 fver; /* chip_info::fcxmreg_ver */ u8 reg_num; __le16 rsvd; __le32 mreg_val[CXMREG_MAX]; @@ -1492,16 +1608,14 @@ struct rtw89_btc_fbtc_slot { __le16 cxtype; } __packed; -#define FCXSLOTS_VER 1 struct rtw89_btc_fbtc_slots { - u8 fver; + u8 fver; /* chip_info::fcxslots_ver */ u8 tbl_num; __le16 rsvd; __le32 update_map; struct rtw89_btc_fbtc_slot slot[CXST_MAX]; } __packed; -#define FCXSTEP_VER 2 struct rtw89_btc_fbtc_step { u8 type; u8 val; @@ -1509,7 +1623,7 @@ struct rtw89_btc_fbtc_step { } __packed; struct rtw89_btc_fbtc_steps { - u8 fver; + u8 fver; /* chip_info::fcxstep_ver */ u8 rsvd; __le16 cnt; __le16 pos_old; @@ -1517,9 +1631,16 @@ struct rtw89_btc_fbtc_steps { struct rtw89_btc_fbtc_step step[FCXMAX_STEP]; } __packed; -#define FCXCYSTA_VER 2 -struct rtw89_btc_fbtc_cysta { /* statistics for cycles */ +struct rtw89_btc_fbtc_steps_v1 { u8 fver; + u8 en; + __le16 rsvd; + __le32 cnt; + struct rtw89_btc_fbtc_step step[FCXMAX_STEP]; +} __packed; + +struct rtw89_btc_fbtc_cysta { /* statistics for cycles */ + u8 fver; /* chip_info::fcxcysta_ver */ u8 rsvd; __le16 cycles; /* total cycle number */ __le16 cycles_a2dp[CXT_FLCTRL_MAX]; @@ -1544,19 +1665,80 @@ struct rtw89_btc_fbtc_cysta { /* statistics for cycles */ __le16 tslot_cycle[BTC_CYCLE_SLOT_MAX]; } __packed; -#define FCXNULLSTA_VER 1 -struct rtw89_btc_fbtc_cynullsta { /* cycle null statistics */ +struct rtw89_btc_fbtc_fdd_try_info { + __le16 cycles[CXT_FLCTRL_MAX]; + __le16 tavg[CXT_FLCTRL_MAX]; /* avg try BT-Slot-TDD/BT-slot-FDD time */ + __le16 tmax[CXT_FLCTRL_MAX]; /* max try BT-Slot-TDD/BT-slot-FDD time */ +} __packed; + +struct rtw89_btc_fbtc_cycle_time_info { + __le16 tavg[CXT_MAX]; /* avg wl/bt cycle time */ + __le16 tmax[CXT_MAX]; /* max wl/bt cycle time */ + __le16 tmaxdiff[CXT_MAX]; /* max wl-wl bt-bt cycle diff time */ +} __packed; + +struct rtw89_btc_fbtc_a2dp_trx_stat { + u8 empty_cnt; + u8 retry_cnt; + u8 tx_rate; + u8 tx_cnt; + u8 ack_cnt; + u8 nack_cnt; + u8 rsvd1; + u8 rsvd2; +} __packed; + +struct rtw89_btc_fbtc_cycle_a2dp_empty_info { + __le16 cnt; /* a2dp empty cnt */ + __le16 cnt_timeout; /* a2dp empty timeout cnt*/ + __le16 tavg; /* avg a2dp empty time */ + __le16 tmax; /* max a2dp empty time */ +} __packed; + +struct rtw89_btc_fbtc_cycle_leak_info { + __le32 cnt_rximr; /* the rximr occur at leak slot */ + __le16 tavg; /* avg leak-slot time */ + __le16 tmax; /* max leak-slot time */ +} __packed; + +struct rtw89_btc_fbtc_cysta_v1 { /* statistics for cycles */ u8 fver; u8 rsvd; + __le16 cycles; /* total cycle number */ + __le16 slot_step_time[BTC_CYCLE_SLOT_MAX]; + struct rtw89_btc_fbtc_cycle_time_info cycle_time; + struct rtw89_btc_fbtc_fdd_try_info fdd_try; + struct rtw89_btc_fbtc_cycle_a2dp_empty_info a2dp_ept; + struct rtw89_btc_fbtc_a2dp_trx_stat a2dp_trx[BTC_CYCLE_SLOT_MAX]; + struct rtw89_btc_fbtc_cycle_leak_info leak_slot; + __le32 slot_cnt[CXST_MAX]; /* slot count */ + __le32 bcn_cnt[CXBCN_MAX]; + __le32 collision_cnt; /* counter for event/timer occur at the same time */ + __le32 skip_cnt; + __le32 except_cnt; + __le32 except_map; +} __packed; + +struct rtw89_btc_fbtc_cynullsta { /* cycle null statistics */ + u8 fver; /* chip_info::fcxnullsta_ver */ + u8 rsvd; __le16 rsvd2; __le32 max_t[2]; /* max_t for 0:null0/1:null1 */ __le32 avg_t[2]; /* avg_t for 0:null0/1:null1 */ __le32 result[2][4]; /* 0:fail, 1:ok, 2:on_time, 3:retry */ } __packed; -#define FCX_BTVER_VER 1 +struct rtw89_btc_fbtc_cynullsta_v1 { /* cycle null statistics */ + u8 fver; /* chip_info::fcxnullsta_ver */ + u8 rsvd; + __le16 rsvd2; + __le32 max_t[2]; /* max_t for 0:null0/1:null1 */ + __le32 avg_t[2]; /* avg_t for 0:null0/1:null1 */ + __le32 result[2][5]; /* 0:fail, 1:ok, 2:on_time, 3:retry, 4:tx */ +} __packed; + struct rtw89_btc_fbtc_btver { - u8 fver; + u8 fver; /* chip_info::fcxbtver_ver */ u8 rsvd; __le16 rsvd2; __le32 coex_ver; /*bit[15:8]->shared, bit[7:0]->non-shared */ @@ -1564,17 +1746,15 @@ struct rtw89_btc_fbtc_btver { __le32 feature; } __packed; -#define FCX_BTSCAN_VER 1 struct rtw89_btc_fbtc_btscan { - u8 fver; + u8 fver; /* chip_info::fcxbtscan_ver */ u8 rsvd; __le16 rsvd2; u8 scan[6]; } __packed; -#define FCX_BTAFH_VER 1 struct rtw89_btc_fbtc_btafh { - u8 fver; + u8 fver; /* chip_info::fcxbtafh_ver */ u8 rsvd; __le16 rsvd2; u8 afh_l[4]; /*bit0:2402, bit1: 2403.... bit31:2433 */ @@ -1582,9 +1762,8 @@ struct rtw89_btc_fbtc_btafh { u8 afh_h[4]; /*bit0:2466, bit1:2467......bit14:2480 */ } __packed; -#define FCX_BTDEVINFO_VER 1 struct rtw89_btc_fbtc_btdevinfo { - u8 fver; + u8 fver; /* chip_info::fcxbtdevinfo_ver */ u8 rsvd; __le16 vendor_id; __le32 dev_name; /* only 24 bits valid */ @@ -1609,6 +1788,7 @@ struct rtw89_btc_dm { struct rtw89_btc_rf_trx_para rf_trx_para; struct rtw89_btc_wl_tx_limit_para wl_tx_limit; struct rtw89_btc_dm_step dm_step; + struct rtw89_btc_wl_scc_ctrl wl_scc; union rtw89_btc_dm_error_map error; u32 cnt_dm[BTC_DCNT_NUM]; u32 cnt_notify[BTC_NCNT_NUM]; @@ -1628,7 +1808,9 @@ struct rtw89_btc_dm { u32 wl_btg_rx: 1; u32 trx_para_level: 8; u32 wl_stb_chg: 1; - u32 rsvd: 3; + u32 pta_owner: 1; + u32 tdma_instant_excute: 1; + u32 rsvd: 1; u16 slot_dur[CXST_MAX]; @@ -1650,8 +1832,6 @@ struct rtw89_btc_dbg { u32 rb_val; }; -#define FCXTDMA_VER 1 - enum rtw89_btc_btf_fw_event { BTF_EVNT_RPT = 0, BTF_EVNT_BT_INFO = 1, @@ -1704,12 +1884,18 @@ struct rtw89_btc_rpt_cmn_info { struct rtw89_btc_report_ctrl_state { struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */ - struct rtw89_btc_fbtc_rpt_ctrl finfo; /* info from fw */ + union { + struct rtw89_btc_fbtc_rpt_ctrl finfo; /* info from fw for 52A*/ + struct rtw89_btc_fbtc_rpt_ctrl_v1 finfo_v1; /* info from fw for 52C*/ + }; }; struct rtw89_btc_rpt_fbtc_tdma { struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */ - struct rtw89_btc_fbtc_tdma finfo; /* info from fw */ + union { + struct rtw89_btc_fbtc_tdma finfo; /* info from fw */ + struct rtw89_btc_fbtc_tdma_v1 finfo_v1; /* info from fw for 52C*/ + }; }; struct rtw89_btc_rpt_fbtc_slots { @@ -1719,17 +1905,26 @@ struct rtw89_btc_rpt_fbtc_slots { struct rtw89_btc_rpt_fbtc_cysta { struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */ - struct rtw89_btc_fbtc_cysta finfo; /* info from fw */ + union { + struct rtw89_btc_fbtc_cysta finfo; /* info from fw for 52A*/ + struct rtw89_btc_fbtc_cysta_v1 finfo_v1; /* info from fw for 52C*/ + }; }; struct rtw89_btc_rpt_fbtc_step { struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */ - struct rtw89_btc_fbtc_steps finfo; /* info from fw */ + union { + struct rtw89_btc_fbtc_steps finfo; /* info from fw */ + struct rtw89_btc_fbtc_steps_v1 finfo_v1; /* info from fw */ + }; }; struct rtw89_btc_rpt_fbtc_nullsta { struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */ - struct rtw89_btc_fbtc_cynullsta finfo; /* info from fw */ + union { + struct rtw89_btc_fbtc_cynullsta finfo; /* info from fw */ + struct rtw89_btc_fbtc_cynullsta_v1 finfo_v1; /* info from fw */ + }; }; struct rtw89_btc_rpt_fbtc_mreg { @@ -1887,7 +2082,9 @@ struct rtw89_ra_info { u8 ra_csi_rate_en:1; u8 fixed_csi_rate_en:1; u8 cr_tbl_sel:1; - u8 rsvd2:5; + u8 fix_giltf_en:1; + u8 fix_giltf:3; + u8 rsvd2:1; u8 csi_mcs_ss_idx; u8 csi_mode:2; u8 csi_gi_ltf:3; @@ -1911,19 +2108,20 @@ struct rtw89_ra_report { struct rate_info txrate; u32 bit_rate; u16 hw_rate; + bool might_fallback_legacy; }; DECLARE_EWMA(rssi, 10, 16); -#define RTW89_BA_CAM_NUM 2 - struct rtw89_ba_cam_entry { + struct list_head list; u8 tid; }; #define RTW89_MAX_ADDR_CAM_NUM 128 #define RTW89_MAX_BSSID_CAM_NUM 20 #define RTW89_MAX_SEC_CAM_NUM 128 +#define RTW89_MAX_BA_CAM_NUM 8 #define RTW89_SEC_CAM_IN_ADDR_CAM 7 struct rtw89_addr_cam_entry { @@ -1967,18 +2165,21 @@ struct rtw89_sec_cam_entry { struct rtw89_sta { u8 mac_id; bool disassoc; + struct rtw89_dev *rtwdev; struct rtw89_vif *rtwvif; struct rtw89_ra_info ra; struct rtw89_ra_report ra_report; int max_agg_wait; u8 prev_rssi; struct ewma_rssi avg_rssi; + struct ewma_rssi rssi[RF_PATH_MAX]; struct rtw89_ampdu_params ampdu_params[IEEE80211_NUM_TIDS]; struct ieee80211_rx_status rx_status; u16 rx_hw_rate; __le32 htc_template; struct rtw89_addr_cam_entry addr_cam; /* AP mode or TDLS peer only */ struct rtw89_bssid_cam_entry bssid_cam; /* TDLS peer only */ + struct list_head ba_cam_list; bool use_cfg_mask; struct cfg80211_bitrate_mask mask; @@ -1987,9 +2188,6 @@ struct rtw89_sta { u32 ampdu_max_time:4; bool cctl_tx_retry_limit; u32 data_tx_cnt_lmt:6; - - DECLARE_BITMAP(ba_cam_map, RTW89_BA_CAM_NUM); - struct rtw89_ba_cam_entry ba_cam_entry[RTW89_BA_CAM_NUM]; }; struct rtw89_efuse { @@ -2007,6 +2205,8 @@ struct rtw89_phy_rate_pattern { bool enable; }; +#define RTW89_P2P_MAX_NOA_NUM 2 + struct rtw89_vif { struct list_head list; struct rtw89_dev *rtwdev; @@ -2022,6 +2222,7 @@ struct rtw89_vif { u8 wmm; u8 bcn_hit_cond; u8 hit_rule; + u8 last_noa_nr; bool trigger; bool lsig_txop; u8 tgt_ind; @@ -2091,7 +2292,7 @@ struct rtw89_hci_info { struct rtw89_chip_ops { int (*enable_bb_rf)(struct rtw89_dev *rtwdev); - void (*disable_bb_rf)(struct rtw89_dev *rtwdev); + int (*disable_bb_rf)(struct rtw89_dev *rtwdev); void (*bb_reset)(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx); void (*bb_sethw)(struct rtw89_dev *rtwdev); @@ -2100,20 +2301,29 @@ struct rtw89_chip_ops { bool (*write_rf)(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path, u32 addr, u32 mask, u32 data); void (*set_channel)(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param); + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx); void (*set_channel_help)(struct rtw89_dev *rtwdev, bool enter, - struct rtw89_channel_help_params *p); + struct rtw89_channel_help_params *p, + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx); int (*read_efuse)(struct rtw89_dev *rtwdev, u8 *log_map); int (*read_phycap)(struct rtw89_dev *rtwdev, u8 *phycap_map); void (*fem_setup)(struct rtw89_dev *rtwdev); void (*rfk_init)(struct rtw89_dev *rtwdev); void (*rfk_channel)(struct rtw89_dev *rtwdev); - void (*rfk_band_changed)(struct rtw89_dev *rtwdev); + void (*rfk_band_changed)(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx); void (*rfk_scan)(struct rtw89_dev *rtwdev, bool start); void (*rfk_track)(struct rtw89_dev *rtwdev); void (*power_trim)(struct rtw89_dev *rtwdev); - void (*set_txpwr)(struct rtw89_dev *rtwdev); - void (*set_txpwr_ctrl)(struct rtw89_dev *rtwdev); + void (*set_txpwr)(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, + enum rtw89_phy_idx phy_idx); + void (*set_txpwr_ctrl)(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx); int (*init_txpwr_unit)(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx); u8 (*get_thermal)(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path); void (*ctrl_btg)(struct rtw89_dev *rtwdev, bool btg); @@ -2150,6 +2360,8 @@ struct rtw89_chip_ops { void (*btc_bt_aci_imp)(struct rtw89_dev *rtwdev); void (*btc_update_bt_cnt)(struct rtw89_dev *rtwdev); void (*btc_wl_s1_standby)(struct rtw89_dev *rtwdev, bool state); + void (*btc_set_policy)(struct rtw89_dev *rtwdev, u16 policy_type); + void (*btc_set_wl_rx_gain)(struct rtw89_dev *rtwdev, u32 level); }; enum rtw89_dma_ch { @@ -2351,6 +2563,7 @@ struct rtw89_imr_info { u32 cpu_disp_imr_set; u32 other_disp_imr_clr; u32 other_disp_imr_set; + u32 bbrpt_com_err_imr_reg; u32 bbrpt_chinfo_err_imr_reg; u32 bbrpt_err_imr_set; u32 bbrpt_dfs_err_imr_reg; @@ -2373,17 +2586,40 @@ struct rtw89_imr_info { u32 tmac_imr_set; }; +struct rtw89_rrsr_cfgs { + struct rtw89_reg3_def ref_rate; + struct rtw89_reg3_def rsc; +}; + +struct rtw89_dig_regs { + u32 seg0_pd_reg; + u32 pd_lower_bound_mask; + u32 pd_spatial_reuse_en; + struct rtw89_reg_def p0_lna_init; + struct rtw89_reg_def p1_lna_init; + struct rtw89_reg_def p0_tia_init; + struct rtw89_reg_def p1_tia_init; + struct rtw89_reg_def p0_rxb_init; + struct rtw89_reg_def p1_rxb_init; + struct rtw89_reg_def p0_p20_pagcugc_en; + struct rtw89_reg_def p0_s20_pagcugc_en; + struct rtw89_reg_def p1_p20_pagcugc_en; + struct rtw89_reg_def p1_s20_pagcugc_en; +}; + struct rtw89_chip_info { enum rtw89_core_chip_id chip_id; const struct rtw89_chip_ops *ops; const char *fw_name; u32 fifo_size; + u32 dle_scc_rsvd_size; u16 max_amsdu_limit; bool dis_2g_40m_ul_ofdma; u32 rsvd_ple_ofst; const struct rtw89_hfc_param_ini *hfc_param_ini; const struct rtw89_dle_mem *dle_mem; u32 rf_base_addr[2]; + u8 support_chanctx_num; u8 support_bands; bool support_bw160; bool hw_sec_hdr; @@ -2393,6 +2629,9 @@ struct rtw89_chip_info { u8 acam_num; u8 bcam_num; u8 scam_num; + u8 bacam_num; + u8 bacam_dynamic_num; + bool bacam_v1; u8 sec_ctrl_efuse_size; u32 physical_efuse_size; @@ -2411,6 +2650,7 @@ struct rtw89_chip_info { const struct rtw89_phy_table *nctl_table; const struct rtw89_txpwr_table *byr_table; const struct rtw89_phy_dig_gain_table *dig_table; + const struct rtw89_dig_regs *dig_regs; const struct rtw89_phy_tssi_dbw_table *tssi_dbw_table; const s8 (*txpwr_lmt_2g)[RTW89_2G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] @@ -2436,6 +2676,20 @@ struct rtw89_chip_info { u8 btcx_desired; u8 scbd; u8 mailbox; + u16 btc_fwinfo_buf; + + u8 fcxbtcrpt_ver; + u8 fcxtdma_ver; + u8 fcxslots_ver; + u8 fcxcysta_ver; + u8 fcxstep_ver; + u8 fcxnullsta_ver; + u8 fcxmreg_ver; + u8 fcxgpiodbg_ver; + u8 fcxbtver_ver; + u8 fcxbtscan_ver; + u8 fcxbtafh_ver; + u8 fcxbtdevinfo_ver; u8 afh_guard_ch; const u8 *wl_rssi_thres; @@ -2463,6 +2717,8 @@ struct rtw89_chip_info { const struct rtw89_reg_def *dcfo_comp; u8 dcfo_comp_sft; const struct rtw89_imr_info *imr_info; + const struct rtw89_rrsr_cfgs *rrsr_cfgs; + u32 dma_ch_mask; }; union rtw89_bus_info { @@ -2514,6 +2770,8 @@ enum rtw89_fw_feature { RTW89_FW_FEATURE_SCAN_OFFLOAD, RTW89_FW_FEATURE_TX_WAKE, RTW89_FW_FEATURE_CRASH_TRIGGER, + RTW89_FW_FEATURE_PACKET_DROP, + RTW89_FW_FEATURE_NO_DEEP_PS, }; struct rtw89_fw_suit { @@ -2536,6 +2794,18 @@ struct rtw89_fw_suit { #define RTW89_FW_SUIT_VER_CODE(s) \ RTW89_FW_VER_CODE((s)->major_ver, (s)->minor_ver, (s)->sub_ver, (s)->sub_idex) +#define RTW89_MFW_HDR_VER_CODE(mfw_hdr) \ + RTW89_FW_VER_CODE((mfw_hdr)->ver.major, \ + (mfw_hdr)->ver.minor, \ + (mfw_hdr)->ver.sub, \ + (mfw_hdr)->ver.idx) + +#define RTW89_FW_HDR_VER_CODE(fw_hdr) \ + RTW89_FW_VER_CODE(GET_FW_HDR_MAJOR_VERSION(fw_hdr), \ + GET_FW_HDR_MINOR_VERSION(fw_hdr), \ + GET_FW_HDR_SUBVERSION(fw_hdr), \ + GET_FW_HDR_SUBINDEX(fw_hdr)) + struct rtw89_fw_info { const struct firmware *firmware; struct rtw89_dev *rtwdev; @@ -2558,6 +2828,8 @@ struct rtw89_cam_info { DECLARE_BITMAP(addr_cam_map, RTW89_MAX_ADDR_CAM_NUM); DECLARE_BITMAP(bssid_cam_map, RTW89_MAX_BSSID_CAM_NUM); DECLARE_BITMAP(sec_cam_map, RTW89_MAX_SEC_CAM_NUM); + DECLARE_BITMAP(ba_cam_map, RTW89_MAX_BA_CAM_NUM); + struct rtw89_ba_cam_entry ba_cam_entry[RTW89_MAX_BA_CAM_NUM]; }; enum rtw89_sar_sources { @@ -2599,24 +2871,34 @@ struct rtw89_sar_info { }; }; +struct rtw89_chanctx_cfg { + enum rtw89_sub_entity_idx idx; +}; + +enum rtw89_entity_mode { + RTW89_ENTITY_MODE_SCC, +}; + struct rtw89_hal { u32 rx_fltr; u8 cv; - u8 current_channel; - u32 current_freq; - u8 prev_primary_channel; - u8 current_primary_channel; - enum rtw89_subband current_subband; - u8 current_band_width; - u8 prev_band_type; - u8 current_band_type; u32 sw_amsdu_max_size; u32 antenna_tx; u32 antenna_rx; u8 tx_nss; u8 rx_nss; + bool tx_path_diversity; bool support_cckpd; bool support_igi; + + DECLARE_BITMAP(entity_map, NUM_OF_RTW89_SUB_ENTITY); + struct cfg80211_chan_def chandef[NUM_OF_RTW89_SUB_ENTITY]; + + bool entity_active; + enum rtw89_entity_mode entity_mode; + + struct rtw89_chan chan[NUM_OF_RTW89_SUB_ENTITY]; + struct rtw89_chan_rcd chan_rcd[NUM_OF_RTW89_SUB_ENTITY]; }; #define RTW89_MAX_MAC_ID_NUM 128 @@ -2632,11 +2914,37 @@ enum rtw89_flags { RTW89_FLAG_LEISURE_PS, RTW89_FLAG_LOW_POWER_MODE, RTW89_FLAG_INACTIVE_PS, - RTW89_FLAG_RESTART_TRIGGER, + RTW89_FLAG_CRASH_SIMULATING, NUM_OF_RTW89_FLAGS, }; +enum rtw89_pkt_drop_sel { + RTW89_PKT_DROP_SEL_MACID_BE_ONCE, + RTW89_PKT_DROP_SEL_MACID_BK_ONCE, + RTW89_PKT_DROP_SEL_MACID_VI_ONCE, + RTW89_PKT_DROP_SEL_MACID_VO_ONCE, + RTW89_PKT_DROP_SEL_MACID_ALL, + RTW89_PKT_DROP_SEL_MG0_ONCE, + RTW89_PKT_DROP_SEL_HIQ_ONCE, + RTW89_PKT_DROP_SEL_HIQ_PORT, + RTW89_PKT_DROP_SEL_HIQ_MBSSID, + RTW89_PKT_DROP_SEL_BAND, + RTW89_PKT_DROP_SEL_BAND_ONCE, + RTW89_PKT_DROP_SEL_REL_MACID, + RTW89_PKT_DROP_SEL_REL_HIQ_PORT, + RTW89_PKT_DROP_SEL_REL_HIQ_MBSSID, +}; + +struct rtw89_pkt_drop_params { + enum rtw89_pkt_drop_sel sel; + enum rtw89_mac_idx mac_band; + u8 macid; + u8 port; + u8 mbssid; + bool tf_trs; +}; + struct rtw89_pkt_stat { u16 beacon_nr; u32 rx_rate_cnt[RTW89_HW_RATE_NR]; @@ -3073,6 +3381,7 @@ struct rtw89_hw_scan_info { u8 op_chan; u8 op_bw; u8 op_band; + u32 last_chan_idx; }; enum rtw89_phy_bb_gain_band { @@ -3119,6 +3428,7 @@ struct rtw89_phy_efuse_gain { struct rtw89_dev { struct ieee80211_hw *hw; struct device *dev; + const struct ieee80211_ops *ops; bool dbcc_en; struct rtw89_hw_scan_info scan_info; @@ -3498,6 +3808,16 @@ static inline struct ieee80211_vif *rtwvif_to_vif(struct rtw89_vif *rtwvif) return container_of(p, struct ieee80211_vif, drv_priv); } +static inline struct ieee80211_vif *rtwvif_to_vif_safe(struct rtw89_vif *rtwvif) +{ + return rtwvif ? rtwvif_to_vif(rtwvif) : NULL; +} + +static inline struct rtw89_vif *vif_to_rtwvif_safe(struct ieee80211_vif *vif) +{ + return vif ? (struct rtw89_vif *)vif->drv_priv : NULL; +} + static inline struct ieee80211_sta *rtwsta_to_sta(struct rtw89_sta *rtwsta) { void *p = rtwsta; @@ -3541,6 +3861,20 @@ enum nl80211_band rtw89_hw_to_nl80211_band(enum rtw89_band hw_band) } } +static inline +enum rtw89_band rtw89_nl80211_to_hw_band(enum nl80211_band nl_band) +{ + switch (nl_band) { + default: + case NL80211_BAND_2GHZ: + return RTW89_BAND_2G; + case NL80211_BAND_5GHZ: + return RTW89_BAND_5G; + case NL80211_BAND_6GHZ: + return RTW89_BAND_6G; + } +} + static inline enum rtw89_bandwidth nl_to_rtw89_bandwidth(enum nl80211_chan_width width) { @@ -3588,16 +3922,51 @@ struct rtw89_bssid_cam_entry *rtw89_get_bssid_cam_of(struct rtw89_vif *rtwvif, static inline void rtw89_chip_set_channel_prepare(struct rtw89_dev *rtwdev, - struct rtw89_channel_help_params *p) + struct rtw89_channel_help_params *p, + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) { - rtwdev->chip->ops->set_channel_help(rtwdev, true, p); + rtwdev->chip->ops->set_channel_help(rtwdev, true, p, chan, + mac_idx, phy_idx); } static inline void rtw89_chip_set_channel_done(struct rtw89_dev *rtwdev, - struct rtw89_channel_help_params *p) + struct rtw89_channel_help_params *p, + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) +{ + rtwdev->chip->ops->set_channel_help(rtwdev, false, p, chan, + mac_idx, phy_idx); +} + +static inline +const struct cfg80211_chan_def *rtw89_chandef_get(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx) { - rtwdev->chip->ops->set_channel_help(rtwdev, false, p); + struct rtw89_hal *hal = &rtwdev->hal; + + return &hal->chandef[idx]; +} + +static inline +const struct rtw89_chan *rtw89_chan_get(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + return &hal->chan[idx]; +} + +static inline +const struct rtw89_chan_rcd *rtw89_chan_rcd_get(struct rtw89_dev *rtwdev, + enum rtw89_sub_entity_idx idx) +{ + struct rtw89_hal *hal = &rtwdev->hal; + + return &hal->chan_rcd[idx]; } static inline void rtw89_chip_fem_setup(struct rtw89_dev *rtwdev) @@ -3632,12 +4001,13 @@ static inline void rtw89_chip_rfk_channel(struct rtw89_dev *rtwdev) chip->ops->rfk_channel(rtwdev); } -static inline void rtw89_chip_rfk_band_changed(struct rtw89_dev *rtwdev) +static inline void rtw89_chip_rfk_band_changed(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx) { const struct rtw89_chip_info *chip = rtwdev->chip; if (chip->ops->rfk_band_changed) - chip->ops->rfk_band_changed(rtwdev); + chip->ops->rfk_band_changed(rtwdev, phy_idx); } static inline void rtw89_chip_rfk_scan(struct rtw89_dev *rtwdev, bool start) @@ -3661,19 +4031,7 @@ static inline void rtw89_chip_set_txpwr_ctrl(struct rtw89_dev *rtwdev) const struct rtw89_chip_info *chip = rtwdev->chip; if (chip->ops->set_txpwr_ctrl) - chip->ops->set_txpwr_ctrl(rtwdev); -} - -static inline void rtw89_chip_set_txpwr(struct rtw89_dev *rtwdev) -{ - const struct rtw89_chip_info *chip = rtwdev->chip; - u8 ch = rtwdev->hal.current_channel; - - if (!ch) - return; - - if (chip->ops->set_txpwr) - chip->ops->set_txpwr(rtwdev); + chip->ops->set_txpwr_ctrl(rtwdev, RTW89_PHY_0); } static inline void rtw89_chip_power_trim(struct rtw89_dev *rtwdev) @@ -3902,16 +4260,27 @@ int rtw89_core_sta_disconnect(struct rtw89_dev *rtwdev, int rtw89_core_sta_remove(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta); +void rtw89_core_set_tid_config(struct rtw89_dev *rtwdev, + struct ieee80211_sta *sta, + struct cfg80211_tid_config *tid_config); int rtw89_core_init(struct rtw89_dev *rtwdev); void rtw89_core_deinit(struct rtw89_dev *rtwdev); int rtw89_core_register(struct rtw89_dev *rtwdev); void rtw89_core_unregister(struct rtw89_dev *rtwdev); +struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device, + u32 bus_data_size, + const struct rtw89_chip_info *chip); +void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev); +void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev); +void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef); void rtw89_set_channel(struct rtw89_dev *rtwdev); u8 rtw89_core_acquire_bit_map(unsigned long *addr, unsigned long size); void rtw89_core_release_bit_map(unsigned long *addr, u8 bit); void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits); -int rtw89_core_acquire_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx); -int rtw89_core_release_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx); +int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx); +int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx); void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc); int rtw89_chip_info_setup(struct rtw89_dev *rtwdev); bool rtw89_ra_report_to_bitrate(struct rtw89_dev *rtwdev, u8 rpt_rate, u16 *bitrate); diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 829c61da99bb90dfb0e76a23c227cc08b9ae5fe7..730e83d54257ff8c326c32c06da253bcd2e8c337 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -525,7 +525,8 @@ static int __print_txpwr_map(struct seq_file *m, struct rtw89_dev *rtwdev, static void __print_regd(struct seq_file *m, struct rtw89_dev *rtwdev) { - u8 band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 band = chan->band_type; u8 regd = rtw89_regd_get(rtwdev, band); switch (regd) { @@ -2189,6 +2190,37 @@ out: return count; } +static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev) +{ + struct rtw89_cpuio_ctrl ctrl_para = {0}; + u16 pkt_id; + + rtw89_leave_ps_mode(rtwdev); + + pkt_id = rtw89_mac_dle_buf_req(rtwdev, 0x20, true); + switch (pkt_id) { + case 0xffff: + return -ETIMEDOUT; + case 0xfff: + return -ENOMEM; + default: + break; + } + + /* intentionally, enqueue two pkt, but has only one pkt id */ + ctrl_para.cmd_type = CPUIO_OP_CMD_ENQ_TO_HEAD; + ctrl_para.start_pktid = pkt_id; + ctrl_para.end_pktid = pkt_id; + ctrl_para.pkt_num = 1; /* start from 0 */ + ctrl_para.dst_pid = WDE_DLE_PORT_ID_WDRLS; + ctrl_para.dst_qid = WDE_DLE_QUEID_NO_REPORT; + + if (rtw89_mac_set_cpuio(rtwdev, &ctrl_para, true)) + return -EFAULT; + + return 0; +} + static int rtw89_debug_priv_fw_crash_get(struct seq_file *m, void *v) { @@ -2196,10 +2228,15 @@ rtw89_debug_priv_fw_crash_get(struct seq_file *m, void *v) struct rtw89_dev *rtwdev = debugfs_priv->rtwdev; seq_printf(m, "%d\n", - test_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags)); + test_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags)); return 0; } +enum rtw89_dbg_crash_simulation_type { + RTW89_DBG_SIM_CPU_EXCEPTION = 1, + RTW89_DBG_SIM_CTRL_ERROR = 2, +}; + static ssize_t rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf, size_t count, loff_t *loff) @@ -2207,22 +2244,30 @@ rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf, struct seq_file *m = (struct seq_file *)filp->private_data; struct rtw89_debugfs_priv *debugfs_priv = m->private; struct rtw89_dev *rtwdev = debugfs_priv->rtwdev; - bool fw_crash; + int (*sim)(struct rtw89_dev *rtwdev); + u8 crash_type; int ret; - if (!RTW89_CHK_FW_FEATURE(CRASH_TRIGGER, &rtwdev->fw)) - return -EOPNOTSUPP; - - ret = kstrtobool_from_user(user_buf, count, &fw_crash); + ret = kstrtou8_from_user(user_buf, count, 0, &crash_type); if (ret) return -EINVAL; - if (!fw_crash) + switch (crash_type) { + case RTW89_DBG_SIM_CPU_EXCEPTION: + if (!RTW89_CHK_FW_FEATURE(CRASH_TRIGGER, &rtwdev->fw)) + return -EOPNOTSUPP; + sim = rtw89_fw_h2c_trigger_cpu_exception; + break; + case RTW89_DBG_SIM_CTRL_ERROR: + sim = rtw89_dbg_trigger_ctrl_error; + break; + default: return -EINVAL; + } mutex_lock(&rtwdev->mutex); - set_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags); - ret = rtw89_fw_h2c_trigger_cpu_exception(rtwdev); + set_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags); + ret = sim(rtwdev); mutex_unlock(&rtwdev->mutex); if (ret) @@ -2289,7 +2334,10 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta) struct rate_info *rate = &rtwsta->ra_report.txrate; struct ieee80211_rx_status *status = &rtwsta->rx_status; struct seq_file *m = (struct seq_file *)data; + struct rtw89_dev *rtwdev = rtwsta->rtwdev; + struct rtw89_hal *hal = &rtwdev->hal; u8 rssi; + int i; seq_printf(m, "TX rate [%d]: ", rtwsta->mac_id); @@ -2305,9 +2353,10 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta) he_gi_str[rate->he_gi] : "N/A"); else seq_printf(m, "Legacy %d", rate->legacy); + seq_printf(m, "%s", rtwsta->ra_report.might_fallback_legacy ? " FB_G" : ""); seq_printf(m, "\t(hw_rate=0x%x)", rtwsta->ra_report.hw_rate); seq_printf(m, "\t==> agg_wait=%d (%d)\n", rtwsta->max_agg_wait, - sta->max_rc_amsdu_len); + sta->deflink.agg.max_rc_amsdu_len); seq_printf(m, "RX rate [%d]: ", rtwsta->mac_id); @@ -2333,8 +2382,15 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta) seq_printf(m, "\t(hw_rate=0x%x)\n", rtwsta->rx_hw_rate); rssi = ewma_rssi_read(&rtwsta->avg_rssi); - seq_printf(m, "RSSI: %d dBm (raw=%d, prev=%d)\n", + seq_printf(m, "RSSI: %d dBm (raw=%d, prev=%d) [", RTW89_RSSI_RAW_TO_DBM(rssi), rssi, rtwsta->prev_rssi); + for (i = 0; i < rtwdev->chip->rf_path_num; i++) { + rssi = ewma_rssi_read(&rtwsta->rssi[i]); + seq_printf(m, "%d%s%s", RTW89_RSSI_RAW_TO_DBM(rssi), + hal->tx_path_diversity && (hal->antenna_tx & BIT(i)) ? "*" : "", + i + 1 == rtwdev->chip->rf_path_num ? "" : ", "); + } + seq_puts(m, "]\n"); } static void @@ -2433,6 +2489,26 @@ void rtw89_vif_ids_get_iter(void *data, u8 *mac, struct ieee80211_vif *vif) rtw89_dump_addr_cam(m, &rtwvif->addr_cam); } +static void rtw89_dump_ba_cam(struct seq_file *m, struct rtw89_sta *rtwsta) +{ + struct rtw89_vif *rtwvif = rtwsta->rtwvif; + struct rtw89_dev *rtwdev = rtwvif->rtwdev; + struct rtw89_ba_cam_entry *entry; + bool first = true; + + list_for_each_entry(entry, &rtwsta->ba_cam_list, list) { + if (first) { + seq_puts(m, "\tba_cam "); + first = false; + } else { + seq_puts(m, ", "); + } + seq_printf(m, "tid[%u]=%d", entry->tid, + (int)(entry - rtwdev->cam_info.ba_cam_entry)); + } + seq_puts(m, "\n"); +} + static void rtw89_sta_ids_get_iter(void *data, struct ieee80211_sta *sta) { struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; @@ -2441,6 +2517,7 @@ static void rtw89_sta_ids_get_iter(void *data, struct ieee80211_sta *sta) seq_printf(m, "STA [%d] %pM %s\n", rtwsta->mac_id, sta->addr, sta->tdls ? "(TDLS)" : ""); rtw89_dump_addr_cam(m, &rtwsta->addr_cam); + rtw89_dump_ba_cam(m, rtwsta); } static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v) @@ -2449,6 +2526,8 @@ static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v) struct rtw89_dev *rtwdev = debugfs_priv->rtwdev; struct rtw89_cam_info *cam_info = &rtwdev->cam_info; + mutex_lock(&rtwdev->mutex); + seq_puts(m, "map:\n"); seq_printf(m, "\tmac_id: %*ph\n", (int)sizeof(rtwdev->mac_id_map), rtwdev->mac_id_map); @@ -2458,12 +2537,16 @@ static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v) cam_info->bssid_cam_map); seq_printf(m, "\tsec_cam: %*ph\n", (int)sizeof(cam_info->sec_cam_map), cam_info->sec_cam_map); + seq_printf(m, "\tba_cam: %*ph\n", (int)sizeof(cam_info->ba_cam_map), + cam_info->ba_cam_map); ieee80211_iterate_active_interfaces_atomic(rtwdev->hw, IEEE80211_IFACE_ITER_NORMAL, rtw89_vif_ids_get_iter, m); ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_ids_get_iter, m); + mutex_unlock(&rtwdev->mutex); + return 0; } diff --git a/drivers/net/wireless/realtek/rtw89/debug.h b/drivers/net/wireless/realtek/rtw89/debug.h index 6176152dbf6bf03cbf62d0f310194a62288b97dc..ee243aadde87392be94d681e9c9a337eb62fc06f 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.h +++ b/drivers/net/wireless/realtek/rtw89/debug.h @@ -25,6 +25,7 @@ enum rtw89_debug_mask { RTW89_DBG_BF = BIT(14), RTW89_DBG_HW_SCAN = BIT(15), RTW89_DBG_SAR = BIT(16), + RTW89_DBG_STATE = BIT(17), RTW89_DBG_UNEXP = BIT(31), }; diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 6473015a6b2a1e4fe70ed4dc87ccdc2ed8720261..d57e3610fb88ec8735fa4cb97805ee59aff4b95b 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -3,6 +3,7 @@ */ #include "cam.h" +#include "chan.h" #include "coex.h" #include "debug.h" #include "fw.h" @@ -224,6 +225,12 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = { __CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, SCAN_OFFLOAD), __CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, TX_WAKE), __CFG_FW_FEAT(RTL8852A, ge, 0, 13, 36, 0, CRASH_TRIGGER), + __CFG_FW_FEAT(RTL8852A, ge, 0, 13, 38, 0, PACKET_DROP), + __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 20, 0, PACKET_DROP), + __CFG_FW_FEAT(RTL8852C, le, 0, 27, 33, 0, NO_DEEP_PS), + __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 34, 0, TX_WAKE), + __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 36, 0, SCAN_OFFLOAD), + __CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER), }; static void rtw89_fw_recognize_features(struct rtw89_dev *rtwdev) @@ -247,6 +254,46 @@ static void rtw89_fw_recognize_features(struct rtw89_dev *rtwdev) } } +void rtw89_early_fw_feature_recognize(struct device *device, + const struct rtw89_chip_info *chip, + u32 *early_feat_map) +{ + union { + struct rtw89_mfw_hdr mfw_hdr; + u8 fw_hdr[RTW89_FW_HDR_SIZE]; + } buf = {}; + const struct firmware *firmware; + u32 ver_code; + int ret; + int i; + + ret = request_partial_firmware_into_buf(&firmware, chip->fw_name, + device, &buf, sizeof(buf), 0); + if (ret) { + dev_err(device, "failed to early request firmware: %d\n", ret); + return; + } + + ver_code = buf.mfw_hdr.sig != RTW89_MFW_SIG ? + RTW89_FW_HDR_VER_CODE(&buf.fw_hdr) : + RTW89_MFW_HDR_VER_CODE(&buf.mfw_hdr); + if (!ver_code) + goto out; + + for (i = 0; i < ARRAY_SIZE(fw_feat_tbl); i++) { + const struct __fw_feat_cfg *ent = &fw_feat_tbl[i]; + + if (chip->chip_id != ent->chip_id) + continue; + + if (ent->cond(ver_code, ent->ver_code)) + *early_feat_map |= BIT(ent->feature); + } + +out: + release_firmware(firmware); +} + int rtw89_fw_recognize(struct rtw89_dev *rtwdev) { int ret; @@ -571,6 +618,7 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, struct rtw89_sta *rtwsta, const u8 *scan_mac_addr) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CAM_LEN); if (!skb) { @@ -587,7 +635,8 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, H2C_FUNC_MAC_ADDR_CAM_UPD, 0, 1, H2C_CAM_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -596,7 +645,7 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_DCTL_SEC_CAM_LEN 68 @@ -605,6 +654,7 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_DCTL_SEC_CAM_LEN); if (!skb) { @@ -621,7 +671,8 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev, H2C_FUNC_MAC_DCTLINFO_UD_V1, 0, 0, H2C_DCTL_SEC_CAM_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -630,7 +681,7 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } EXPORT_SYMBOL(rtw89_fw_h2c_dctl_sec_cam_v1); @@ -638,14 +689,16 @@ EXPORT_SYMBOL(rtw89_fw_h2c_dctl_sec_cam_v1); int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, bool valid, struct ieee80211_ampdu_params *params) { + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_vif *rtwvif = rtwsta->rtwvif; u8 macid = rtwsta->mac_id; struct sk_buff *skb; u8 entry_idx; int ret; ret = valid ? - rtw89_core_acquire_sta_ba_entry(rtwsta, params->tid, &entry_idx) : - rtw89_core_release_sta_ba_entry(rtwsta, params->tid, &entry_idx); + rtw89_core_acquire_sta_ba_entry(rtwdev, rtwsta, params->tid, &entry_idx) : + rtw89_core_release_sta_ba_entry(rtwdev, rtwsta, params->tid, &entry_idx); if (ret) { /* it still works even if we don't have static BA CAM, because * hardware can create dynamic BA CAM automatically. @@ -663,7 +716,10 @@ int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, } skb_put(skb, H2C_BA_CAM_LEN); SET_BA_CAM_MACID(skb->data, macid); - SET_BA_CAM_ENTRY_IDX(skb->data, entry_idx); + if (chip->bacam_v1) + SET_BA_CAM_ENTRY_IDX_V1(skb->data, entry_idx); + else + SET_BA_CAM_ENTRY_IDX(skb->data, entry_idx); if (!valid) goto end; SET_BA_CAM_VALID(skb->data, valid); @@ -676,6 +732,11 @@ int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, SET_BA_CAM_INIT_REQ(skb->data, 1); SET_BA_CAM_SSN(skb->data, params->ssn); + if (chip->bacam_v1) { + SET_BA_CAM_STD_EN(skb->data, 1); + SET_BA_CAM_BAND(skb->data, rtwvif->mac_idx); + } + end: rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, H2C_CAT_MAC, @@ -683,7 +744,8 @@ end: H2C_FUNC_MAC_BA_CAM, 0, 1, H2C_BA_CAM_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -692,7 +754,59 @@ end: fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; +} + +static int rtw89_fw_h2c_init_dynamic_ba_cam_v1(struct rtw89_dev *rtwdev, + u8 entry_idx, u8 uid) +{ + struct sk_buff *skb; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_BA_CAM_LEN); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for dynamic h2c ba cam\n"); + return -ENOMEM; + } + skb_put(skb, H2C_BA_CAM_LEN); + + SET_BA_CAM_VALID(skb->data, 1); + SET_BA_CAM_ENTRY_IDX_V1(skb->data, entry_idx); + SET_BA_CAM_UID(skb->data, uid); + SET_BA_CAM_BAND(skb->data, 0); + SET_BA_CAM_STD_EN(skb->data, 0); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_BA_CAM, + H2C_FUNC_MAC_BA_CAM, 0, 1, + H2C_BA_CAM_LEN); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; +} + +void rtw89_fw_h2c_init_ba_cam_v1(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + u8 entry_idx = chip->bacam_num; + u8 uid = 0; + int i; + + for (i = 0; i < chip->bacam_dynamic_num; i++) { + rtw89_fw_h2c_init_dynamic_ba_cam_v1(rtwdev, entry_idx, uid); + entry_idx++; + uid++; + } } #define H2C_LOG_CFG_LEN 12 @@ -701,6 +815,7 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable) struct sk_buff *skb; u32 comp = enable ? BIT(RTW89_FW_LOG_COMP_INIT) | BIT(RTW89_FW_LOG_COMP_TASK) | BIT(RTW89_FW_LOG_COMP_PS) | BIT(RTW89_FW_LOG_COMP_ERROR) : 0; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LOG_CFG_LEN); if (!skb) { @@ -720,7 +835,8 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable) H2C_FUNC_LOG_CFG, 0, 0, H2C_LOG_CFG_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -729,7 +845,7 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_GENERAL_PKT_LEN 6 @@ -737,6 +853,7 @@ fail: int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_GENERAL_PKT_LEN); if (!skb) { @@ -757,7 +874,8 @@ int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid) H2C_FUNC_MAC_GENERAL_PKT, 0, 1, H2C_GENERAL_PKT_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -766,7 +884,7 @@ int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_LPS_PARM_LEN 8 @@ -774,6 +892,7 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev, struct rtw89_lps_parm *lps_param) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LPS_PARM_LEN); if (!skb) { @@ -799,7 +918,8 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev, H2C_FUNC_MAC_LPS_PARM, 0, 1, H2C_LPS_PARM_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -808,7 +928,73 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; +} + +#define H2C_P2P_ACT_LEN 20 +int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, + struct ieee80211_p2p_noa_desc *desc, + u8 act, u8 noa_id) +{ + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + bool p2p_type_gc = rtwvif->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT; + u8 ctwindow_oppps = vif->bss_conf.p2p_noa_attr.oppps_ctwindow; + struct sk_buff *skb; + u8 *cmd; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_P2P_ACT_LEN); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for h2c p2p act\n"); + return -ENOMEM; + } + skb_put(skb, H2C_P2P_ACT_LEN); + cmd = skb->data; + + RTW89_SET_FWCMD_P2P_MACID(cmd, rtwvif->mac_id); + RTW89_SET_FWCMD_P2P_P2PID(cmd, 0); + RTW89_SET_FWCMD_P2P_NOAID(cmd, noa_id); + RTW89_SET_FWCMD_P2P_ACT(cmd, act); + RTW89_SET_FWCMD_P2P_TYPE(cmd, p2p_type_gc); + RTW89_SET_FWCMD_P2P_ALL_SLEP(cmd, 0); + if (desc) { + RTW89_SET_FWCMD_NOA_START_TIME(cmd, desc->start_time); + RTW89_SET_FWCMD_NOA_INTERVAL(cmd, desc->interval); + RTW89_SET_FWCMD_NOA_DURATION(cmd, desc->duration); + RTW89_SET_FWCMD_NOA_COUNT(cmd, desc->count); + RTW89_SET_FWCMD_NOA_CTWINDOW(cmd, ctwindow_oppps); + } + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, H2C_CL_MAC_PS, + H2C_FUNC_P2P_ACT, 0, 0, + H2C_P2P_ACT_LEN); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; +} + +static void __rtw89_fw_h2c_set_tx_path(struct rtw89_dev *rtwdev, + struct sk_buff *skb) +{ + struct rtw89_hal *hal = &rtwdev->hal; + u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B; + u8 map_b = hal->antenna_tx == RF_AB ? 1 : 0; + + SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path); + SET_CMC_TBL_PATH_MAP_A(skb->data, 0); + SET_CMC_TBL_PATH_MAP_B(skb->data, map_b); + SET_CMC_TBL_PATH_MAP_C(skb->data, 0); + SET_CMC_TBL_PATH_MAP_D(skb->data, 0); } #define H2C_CMC_TBL_LEN 68 @@ -816,11 +1002,9 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { const struct rtw89_chip_info *chip = rtwdev->chip; - struct rtw89_hal *hal = &rtwdev->hal; struct sk_buff *skb; - u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B; - u8 map_b = hal->antenna_tx == RF_AB ? 1 : 0; u8 macid = rtwvif->mac_id; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN); if (!skb) { @@ -832,11 +1016,7 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev, SET_CTRL_INFO_OPERATION(skb->data, 1); if (chip->h2c_cctl_func_id == H2C_FUNC_MAC_CCTLINFO_UD) { SET_CMC_TBL_TXPWR_MODE(skb->data, 0); - SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path); - SET_CMC_TBL_PATH_MAP_A(skb->data, 0); - SET_CMC_TBL_PATH_MAP_B(skb->data, map_b); - SET_CMC_TBL_PATH_MAP_C(skb->data, 0); - SET_CMC_TBL_PATH_MAP_D(skb->data, 0); + __rtw89_fw_h2c_set_tx_path(rtwdev, skb); SET_CMC_TBL_ANTSEL_A(skb->data, 0); SET_CMC_TBL_ANTSEL_B(skb->data, 0); SET_CMC_TBL_ANTSEL_C(skb->data, 0); @@ -852,7 +1032,8 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev, chip->h2c_cctl_func_id, 0, 1, H2C_CMC_TBL_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -861,7 +1042,7 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } static void __get_sta_he_pkt_padding(struct rtw89_dev *rtwdev, @@ -926,17 +1107,26 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta) { const struct rtw89_chip_info *chip = rtwdev->chip; - struct rtw89_hal *hal = &rtwdev->hal; struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta); struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct sk_buff *skb; u8 pads[RTW89_PPE_BW_NUM]; u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id; + u16 lowest_rate; + int ret; memset(pads, 0, sizeof(pads)); if (sta) __get_sta_he_pkt_padding(rtwdev, sta, pads); + if (vif->p2p) + lowest_rate = RTW89_HW_RATE_OFDM6; + else if (chan->band_type == RTW89_BAND_2G) + lowest_rate = RTW89_HW_RATE_CCK1; + else + lowest_rate = RTW89_HW_RATE_OFDM6; + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN); if (!skb) { rtw89_err(rtwdev, "failed to alloc skb for fw dl\n"); @@ -947,10 +1137,7 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev, SET_CTRL_INFO_OPERATION(skb->data, 1); SET_CMC_TBL_DISRTSFB(skb->data, 1); SET_CMC_TBL_DISDATAFB(skb->data, 1); - if (hal->current_band_type == RTW89_BAND_2G) - SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, RTW89_HW_RATE_CCK1); - else - SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, RTW89_HW_RATE_OFDM6); + SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, lowest_rate); SET_CMC_TBL_RTS_TXCNT_LMT_SEL(skb->data, 0); SET_CMC_TBL_DATA_TXCNT_LMT_SEL(skb->data, 0); if (vif->type == NL80211_IFTYPE_STATION) @@ -980,7 +1167,8 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev, chip->h2c_cctl_func_id, 0, 1, H2C_CMC_TBL_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -989,7 +1177,7 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev, @@ -997,6 +1185,7 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev, { const struct rtw89_chip_info *chip = rtwdev->chip; struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN); if (!skb) { @@ -1020,7 +1209,47 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev, chip->h2c_cctl_func_id, 0, 1, H2C_CMC_TBL_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; +} + +int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + struct sk_buff *skb; + int ret; + + if (chip->h2c_cctl_func_id != H2C_FUNC_MAC_CCTLINFO_UD) + return 0; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for fw dl\n"); + return -ENOMEM; + } + skb_put(skb, H2C_CMC_TBL_LEN); + SET_CTRL_INFO_MACID(skb->data, rtwsta->mac_id); + SET_CTRL_INFO_OPERATION(skb->data, 1); + + __rtw89_fw_h2c_set_tx_path(rtwdev, skb); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, H2C_CL_MAC_FR_EXCHG, + H2C_FUNC_MAC_CCTLINFO_UD, 0, 1, + H2C_CMC_TBL_LEN); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1029,19 +1258,28 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_BCN_BASE_LEN 12 int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { - struct rtw89_hal *hal = &rtwdev->hal; struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct sk_buff *skb; struct sk_buff *skb_beacon; u16 tim_offset; int bcn_total_len; + u16 beacon_rate; + int ret; + + if (vif->p2p) + beacon_rate = RTW89_HW_RATE_OFDM6; + else if (chan->band_type == RTW89_BAND_2G) + beacon_rate = RTW89_HW_RATE_CCK1; + else + beacon_rate = RTW89_HW_RATE_OFDM6; skb_beacon = ieee80211_beacon_get_tim(rtwdev->hw, vif, &tim_offset, NULL, 0); @@ -1066,8 +1304,7 @@ int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev, SET_BCN_UPD_MACID(skb->data, rtwvif->mac_id); SET_BCN_UPD_SSN_SEL(skb->data, RTW89_MGMT_HW_SSN_SEL); SET_BCN_UPD_SSN_MODE(skb->data, RTW89_MGMT_HW_SEQ_MODE); - SET_BCN_UPD_RATE(skb->data, hal->current_band_type == RTW89_BAND_2G ? - RTW89_HW_RATE_CCK1 : RTW89_HW_RATE_OFDM6); + SET_BCN_UPD_RATE(skb->data, beacon_rate); skb_put_data(skb, skb_beacon->data, skb_beacon->len); dev_kfree_skb_any(skb_beacon); @@ -1077,10 +1314,11 @@ int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev, H2C_FUNC_MAC_BCN_UPD, 0, 1, bcn_total_len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } return 0; @@ -1095,6 +1333,7 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev, struct sk_buff *skb; u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id; u8 self_role; + int ret; if (rtwvif->net_type == RTW89_NET_TYPE_AP_MODE) { if (rtwsta) @@ -1121,7 +1360,8 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev, H2C_FUNC_MAC_FWROLE_MAINTAIN, 0, 1, H2C_ROLE_MAINTAIN_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1130,7 +1370,7 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_JOIN_INFO_LEN 4 @@ -1141,6 +1381,7 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id; u8 self_role = rtwvif->self_role; u8 net_type = rtwvif->net_type; + int ret; if (net_type == RTW89_NET_TYPE_AP_MODE && rtwsta) { self_role = RTW89_SELF_ROLE_AP_CLIENT; @@ -1172,7 +1413,8 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, H2C_FUNC_MAC_JOININFO, 0, 1, H2C_JOIN_INFO_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1181,7 +1423,7 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, @@ -1190,6 +1432,7 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, struct rtw89_fw_macid_pause_grp h2c = {{0}}; u8 len = sizeof(struct rtw89_fw_macid_pause_grp); struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_JOIN_INFO_LEN); if (!skb) { @@ -1206,7 +1449,8 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, H2C_FUNC_MAC_MACID_PAUSE, 1, 0, len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1215,7 +1459,7 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_EDCA_LEN 12 @@ -1223,6 +1467,7 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, u8 ac, u32 val) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_EDCA_LEN); if (!skb) { @@ -1241,7 +1486,8 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, H2C_FUNC_USR_EDCA, 0, 1, H2C_EDCA_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1250,7 +1496,47 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; +} + +#define H2C_TSF32_TOGL_LEN 4 +int rtw89_fw_h2c_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, + bool en) +{ + struct sk_buff *skb; + u16 early_us = en ? 2000 : 0; + u8 *cmd; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_TSF32_TOGL_LEN); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for h2c p2p act\n"); + return -ENOMEM; + } + skb_put(skb, H2C_TSF32_TOGL_LEN); + cmd = skb->data; + + RTW89_SET_FWCMD_TSF32_TOGL_BAND(cmd, rtwvif->mac_idx); + RTW89_SET_FWCMD_TSF32_TOGL_EN(cmd, en); + RTW89_SET_FWCMD_TSF32_TOGL_PORT(cmd, rtwvif->port); + RTW89_SET_FWCMD_TSF32_TOGL_EARLY(cmd, early_us); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD, + H2C_FUNC_TSF32_TOGL, 0, 0, + H2C_TSF32_TOGL_LEN); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; } #define H2C_OFLD_CFG_LEN 8 @@ -1258,6 +1544,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev) { static const u8 cfg[] = {0x09, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00}; struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_OFLD_CFG_LEN); if (!skb) { @@ -1271,7 +1558,8 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev) H2C_FUNC_OFLD_CFG, 0, 1, H2C_OFLD_CFG_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1280,7 +1568,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_RA_LEN 16 @@ -1288,6 +1576,7 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi { struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_RA_LEN); if (!skb) { @@ -1318,6 +1607,8 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi RTW89_SET_FWCMD_RA_MASK_2(cmd, FIELD_GET(MASKBYTE2, ra->ra_mask)); RTW89_SET_FWCMD_RA_MASK_3(cmd, FIELD_GET(MASKBYTE3, ra->ra_mask)); RTW89_SET_FWCMD_RA_MASK_4(cmd, FIELD_GET(MASKBYTE4, ra->ra_mask)); + RTW89_SET_FWCMD_RA_FIX_GILTF_EN(cmd, ra->fix_giltf_en); + RTW89_SET_FWCMD_RA_FIX_GILTF(cmd, ra->fix_giltf); if (csi) { RTW89_SET_FWCMD_RA_BFEE_CSI_CTL(cmd, 1); @@ -1336,7 +1627,8 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi H2C_FUNC_OUTSRC_RA_MACIDCFG, 0, 0, H2C_RA_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1345,7 +1637,7 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_LEN_CXDRVHDR 2 @@ -1359,6 +1651,7 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev) struct rtw89_btc_ant_info *ant = &module->ant; struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_INIT); if (!skb) { @@ -1395,7 +1688,8 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev) SET_DRV_INFO, 0, 0, H2C_LEN_CXDRVINFO_INIT); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1404,10 +1698,15 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } +#define PORT_DATA_OFFSET 4 +#define H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN 12 #define H2C_LEN_CXDRVINFO_ROLE (4 + 12 * RTW89_PORT_NUM + H2C_LEN_CXDRVHDR) +#define H2C_LEN_CXDRVINFO_ROLE_V1 (4 + 16 * RTW89_PORT_NUM + \ + H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN + \ + H2C_LEN_CXDRVHDR) int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev) { struct rtw89_btc *btc = &rtwdev->btc; @@ -1416,7 +1715,9 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev) struct rtw89_btc_wl_role_info_bpos *bpos = &role_info->role_map.role; struct rtw89_btc_wl_active_role *active = role_info->active_role; struct sk_buff *skb; + u8 offset = 0; u8 *cmd; + int ret; int i; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_ROLE); @@ -1447,19 +1748,19 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev) RTW89_SET_FWCMD_CXROLE_ROLE_NAN(cmd, bpos->nan); for (i = 0; i < RTW89_PORT_NUM; i++, active++) { - RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i); - RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i); - RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i); - RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i); - RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i); - RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i); - RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i); - RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i); - RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i); - RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i); - RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i); - RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i); - RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i); + RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i, offset); } rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, @@ -1467,7 +1768,8 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev) SET_DRV_INFO, 0, 0, H2C_LEN_CXDRVINFO_ROLE); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1476,16 +1778,101 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; +} + +int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev) +{ + struct rtw89_btc *btc = &rtwdev->btc; + struct rtw89_btc_wl_info *wl = &btc->cx.wl; + struct rtw89_btc_wl_role_info_v1 *role_info = &wl->role_info_v1; + struct rtw89_btc_wl_role_info_bpos *bpos = &role_info->role_map.role; + struct rtw89_btc_wl_active_role_v1 *active = role_info->active_role_v1; + struct sk_buff *skb; + u8 *cmd, offset; + int ret; + int i; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_ROLE_V1); + if (!skb) { + rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_role\n"); + return -ENOMEM; + } + skb_put(skb, H2C_LEN_CXDRVINFO_ROLE_V1); + cmd = skb->data; + + RTW89_SET_FWCMD_CXHDR_TYPE(cmd, CXDRVINFO_ROLE); + RTW89_SET_FWCMD_CXHDR_LEN(cmd, H2C_LEN_CXDRVINFO_ROLE_V1 - H2C_LEN_CXDRVHDR); + + RTW89_SET_FWCMD_CXROLE_CONNECT_CNT(cmd, role_info->connect_cnt); + RTW89_SET_FWCMD_CXROLE_LINK_MODE(cmd, role_info->link_mode); + + RTW89_SET_FWCMD_CXROLE_ROLE_NONE(cmd, bpos->none); + RTW89_SET_FWCMD_CXROLE_ROLE_STA(cmd, bpos->station); + RTW89_SET_FWCMD_CXROLE_ROLE_AP(cmd, bpos->ap); + RTW89_SET_FWCMD_CXROLE_ROLE_VAP(cmd, bpos->vap); + RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC(cmd, bpos->adhoc); + RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC_MASTER(cmd, bpos->adhoc_master); + RTW89_SET_FWCMD_CXROLE_ROLE_MESH(cmd, bpos->mesh); + RTW89_SET_FWCMD_CXROLE_ROLE_MONITOR(cmd, bpos->moniter); + RTW89_SET_FWCMD_CXROLE_ROLE_P2P_DEV(cmd, bpos->p2p_device); + RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GC(cmd, bpos->p2p_gc); + RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GO(cmd, bpos->p2p_go); + RTW89_SET_FWCMD_CXROLE_ROLE_NAN(cmd, bpos->nan); + + offset = PORT_DATA_OFFSET; + for (i = 0; i < RTW89_PORT_NUM; i++, active++) { + RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i, offset); + RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR(cmd, active->noa_duration, i, offset); + } + + offset = H2C_LEN_CXDRVINFO_ROLE_V1 - H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN; + RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(cmd, role_info->mrole_type, offset); + RTW89_SET_FWCMD_CXROLE_MROLE_NOA(cmd, role_info->mrole_noa_duration, offset); + RTW89_SET_FWCMD_CXROLE_DBCC_EN(cmd, role_info->dbcc_en, offset); + RTW89_SET_FWCMD_CXROLE_DBCC_CHG(cmd, role_info->dbcc_chg, offset); + RTW89_SET_FWCMD_CXROLE_DBCC_2G_PHY(cmd, role_info->dbcc_2g_phy, offset); + RTW89_SET_FWCMD_CXROLE_LINK_MODE_CHG(cmd, role_info->link_mode_chg, offset); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_OUTSRC, BTFC_SET, + SET_DRV_INFO, 0, 0, + H2C_LEN_CXDRVINFO_ROLE_V1); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; +fail: + dev_kfree_skb_any(skb); + + return ret; } #define H2C_LEN_CXDRVINFO_CTRL (4 + H2C_LEN_CXDRVHDR) int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev) { + const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_btc *btc = &rtwdev->btc; struct rtw89_btc_ctrl *ctrl = &btc->ctrl; struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_CTRL); if (!skb) { @@ -1501,14 +1888,16 @@ int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev) RTW89_SET_FWCMD_CXCTRL_MANUAL(cmd, ctrl->manual); RTW89_SET_FWCMD_CXCTRL_IGNORE_BT(cmd, ctrl->igno_bt); RTW89_SET_FWCMD_CXCTRL_ALWAYS_FREERUN(cmd, ctrl->always_freerun); - RTW89_SET_FWCMD_CXCTRL_TRACE_STEP(cmd, ctrl->trace_step); + if (chip->chip_id == RTL8852A) + RTW89_SET_FWCMD_CXCTRL_TRACE_STEP(cmd, ctrl->trace_step); rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, H2C_CAT_OUTSRC, BTFC_SET, SET_DRV_INFO, 0, 0, H2C_LEN_CXDRVINFO_CTRL); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1517,7 +1906,7 @@ int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_LEN_CXDRVINFO_RFK (4 + H2C_LEN_CXDRVHDR) @@ -1528,6 +1917,7 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev) struct rtw89_btc_wl_rfk_info *rfk_info = &wl->rfk_info; struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_RFK); if (!skb) { @@ -1551,7 +1941,8 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev) SET_DRV_INFO, 0, 0, H2C_LEN_CXDRVINFO_RFK); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1560,7 +1951,7 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_LEN_PKT_OFLD 4 @@ -1568,6 +1959,7 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id) { struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_PKT_OFLD); if (!skb) { @@ -1585,7 +1977,8 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id) H2C_FUNC_PACKET_OFLD, 1, 1, H2C_LEN_PKT_OFLD); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1594,7 +1987,7 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id, @@ -1603,6 +1996,7 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id, struct sk_buff *skb; u8 *cmd; u8 alloc_id; + int ret; alloc_id = rtw89_core_acquire_bit_map(rtwdev->pkt_offload, RTW89_MAX_PKT_OFLD_NUM); @@ -1629,7 +2023,8 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id, H2C_FUNC_PACKET_OFLD, 1, 1, H2C_LEN_PKT_OFLD + skb_ofld->len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1638,7 +2033,7 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } #define H2C_LEN_SCAN_LIST_OFFLOAD 4 @@ -1649,6 +2044,7 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len, struct sk_buff *skb; int skb_len = H2C_LEN_SCAN_LIST_OFFLOAD + len * RTW89_MAC_CHINFO_SIZE; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, skb_len); if (!skb) { @@ -1693,7 +2089,8 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len, H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD, H2C_FUNC_ADD_SCANOFLD_CH, 1, 1, skb_len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1702,10 +2099,10 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } -#define H2C_LEN_SCAN_OFFLOAD 20 +#define H2C_LEN_SCAN_OFFLOAD 28 int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev, struct rtw89_scan_option *option, struct rtw89_vif *rtwvif) @@ -1713,6 +2110,7 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev, struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info; struct sk_buff *skb; u8 *cmd; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_SCAN_OFFLOAD); if (!skb) { @@ -1736,6 +2134,8 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev, scan_info->op_pri_ch); RTW89_SET_FWCMD_SCANOFLD_TARGET_CENTRAL_CH(cmd, scan_info->op_chan); + RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BAND(cmd, + scan_info->op_band); } rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, @@ -1743,7 +2143,8 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev, H2C_FUNC_SCANOFLD, 1, 1, H2C_LEN_SCAN_OFFLOAD); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1752,7 +2153,7 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev, @@ -1762,6 +2163,7 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev, struct sk_buff *skb; u8 class = info->rf_path == RF_PATH_A ? H2C_CL_OUTSRC_RF_REG_A : H2C_CL_OUTSRC_RF_REG_B; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len); if (!skb) { @@ -1774,7 +2176,8 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev, H2C_CAT_OUTSRC, class, page, 0, 0, len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1783,14 +2186,16 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; struct rtw89_fw_h2c_rf_get_mccch *mccch; struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, sizeof(*mccch)); if (!skb) { @@ -1804,15 +2209,16 @@ int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev) mccch->ch_1 = cpu_to_le32(mcc_info->ch[1]); mccch->band_0 = cpu_to_le32(mcc_info->band[0]); mccch->band_1 = cpu_to_le32(mcc_info->band[1]); - mccch->current_channel = cpu_to_le32(rtwdev->hal.current_channel); - mccch->current_band_type = cpu_to_le32(rtwdev->hal.current_band_type); + mccch->current_channel = cpu_to_le32(chan->channel); + mccch->current_band_type = cpu_to_le32(chan->band_type); rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, H2C_CAT_OUTSRC, H2C_CL_OUTSRC_RF_FW_NOTIFY, H2C_FUNC_OUTSRC_RF_GET_MCCCH, 0, 0, sizeof(*mccch)); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1821,7 +2227,7 @@ int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } EXPORT_SYMBOL(rtw89_fw_h2c_rf_ntfy_mcc); @@ -1830,6 +2236,7 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev, bool rack, bool dack) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len); if (!skb) { @@ -1842,7 +2249,8 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev, H2C_CAT_OUTSRC, h2c_class, h2c_func, rack, dack, len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1851,12 +2259,13 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev, fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_no_hdr(rtwdev, len); if (!skb) { @@ -1865,7 +2274,8 @@ int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len) } skb_put_data(skb, buf, len); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -1874,7 +2284,7 @@ int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } void rtw89_fw_send_all_early_h2c(struct rtw89_dev *rtwdev) @@ -2169,7 +2579,7 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type, if (ssid_num) { ch_info->num_pkt = ssid_num; - band = ch_info->ch_band; + band = rtw89_hw_to_nl80211_band(ch_info->ch_band); list_for_each_entry(info, &scan_info->pkt_list[band], list) { ch_info->probe_id = info->id; @@ -2211,13 +2621,16 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev, struct ieee80211_channel *channel; struct list_head chan_list; bool random_seq = req->flags & NL80211_SCAN_FLAG_RANDOM_SN; - int list_len = req->n_channels, off_chan_time = 0; + int list_len, off_chan_time = 0; enum rtw89_chan_type type; - int ret = 0, i; + int ret = 0; + u32 idx; INIT_LIST_HEAD(&chan_list); - for (i = 0; i < req->n_channels; i++) { - channel = req->channels[i]; + for (idx = rtwdev->scan_info.last_chan_idx, list_len = 0; + idx < req->n_channels && list_len < RTW89_SCAN_LIST_LIMIT; + idx++, list_len++) { + channel = req->channels[idx]; ch_info = kzalloc(sizeof(*ch_info), GFP_KERNEL); if (!ch_info) { ret = -ENOMEM; @@ -2226,7 +2639,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev, ch_info->period = req->duration_mandatory ? req->duration : RTW89_CHANNEL_TIME; - ch_info->ch_band = channel->band; + ch_info->ch_band = rtw89_nl80211_to_hw_band(channel->band); ch_info->central_ch = channel->hw_value; ch_info->pri_ch = channel->hw_value; ch_info->rand_seq_num = random_seq; @@ -2258,6 +2671,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev, list_add_tail(&ch_info->list, &chan_list); off_chan_time += ch_info->period; } + rtwdev->scan_info.last_chan_idx = idx; ret = rtw89_fw_h2c_scan_list_offload(rtwdev, list_len, &chan_list); out: @@ -2289,9 +2703,11 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, { struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct cfg80211_scan_request *req = &scan_req->req; + u32 rx_fltr = rtwdev->hal.rx_fltr; u8 mac_addr[ETH_ALEN]; rtwdev->scan_info.scanning_vif = vif; + rtwdev->scan_info.last_chan_idx = 0; rtwvif->scan_ies = &scan_req->ies; rtwvif->scan_req = req; ieee80211_stop_queues(rtwdev->hw); @@ -2303,13 +2719,13 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, ether_addr_copy(mac_addr, vif->addr); rtw89_core_scan_start(rtwdev, rtwvif, mac_addr, true); - rtwdev->hal.rx_fltr &= ~B_AX_A_BCN_CHK_EN; - rtwdev->hal.rx_fltr &= ~B_AX_A_BC; - rtwdev->hal.rx_fltr &= ~B_AX_A_A1_MATCH; + rx_fltr &= ~B_AX_A_BCN_CHK_EN; + rx_fltr &= ~B_AX_A_BC; + rx_fltr &= ~B_AX_A_A1_MATCH; rtw89_write32_mask(rtwdev, rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0), B_AX_RX_FLTR_CFG_MASK, - rtwdev->hal.rx_fltr); + rx_fltr); } void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, @@ -2323,9 +2739,6 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, if (!vif) return; - rtwdev->hal.rx_fltr |= B_AX_A_BCN_CHK_EN; - rtwdev->hal.rx_fltr |= B_AX_A_BC; - rtwdev->hal.rx_fltr |= B_AX_A_A1_MATCH; rtw89_write32_mask(rtwdev, rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0), B_AX_RX_FLTR_CFG_MASK, @@ -2339,6 +2752,7 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, rtwvif = (struct rtw89_vif *)vif->drv_priv; rtwvif->scan_req = NULL; rtwvif->scan_ies = NULL; + rtwdev->scan_info.last_chan_idx = 0; rtwdev->scan_info.scanning_vif = NULL; if (rtwvif->net_type != RTW89_NET_TYPE_NO_LINK) @@ -2377,18 +2791,18 @@ out: void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup) { struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info; - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + struct rtw89_chan new; if (backup) { - scan_info->op_pri_ch = hal->current_primary_channel; - scan_info->op_chan = hal->current_channel; - scan_info->op_bw = hal->current_band_width; - scan_info->op_band = hal->current_band_type; + scan_info->op_pri_ch = cur->primary_channel; + scan_info->op_chan = cur->channel; + scan_info->op_bw = cur->band_width; + scan_info->op_band = cur->band_type; } else { - hal->current_primary_channel = scan_info->op_pri_ch; - hal->current_channel = scan_info->op_chan; - hal->current_band_width = scan_info->op_bw; - hal->current_band_type = scan_info->op_band; + rtw89_chan_create(&new, scan_info->op_chan, scan_info->op_pri_ch, + scan_info->op_band, scan_info->op_bw); + rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new); } } @@ -2397,6 +2811,7 @@ void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup) int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev) { struct sk_buff *skb; + int ret; skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_FW_CPU_EXCEPTION_LEN); if (!skb) { @@ -2415,7 +2830,62 @@ int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev) H2C_FUNC_CPU_EXCEPTION, 0, 0, H2C_FW_CPU_EXCEPTION_LEN); - if (rtw89_h2c_tx(rtwdev, skb, false)) { + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { + rtw89_err(rtwdev, "failed to send h2c\n"); + goto fail; + } + + return 0; + +fail: + dev_kfree_skb_any(skb); + return ret; +} + +#define H2C_PKT_DROP_LEN 24 +int rtw89_fw_h2c_pkt_drop(struct rtw89_dev *rtwdev, + const struct rtw89_pkt_drop_params *params) +{ + struct sk_buff *skb; + int ret; + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_PKT_DROP_LEN); + if (!skb) { + rtw89_err(rtwdev, + "failed to alloc skb for packet drop\n"); + return -ENOMEM; + } + + switch (params->sel) { + case RTW89_PKT_DROP_SEL_MACID_BE_ONCE: + case RTW89_PKT_DROP_SEL_MACID_BK_ONCE: + case RTW89_PKT_DROP_SEL_MACID_VI_ONCE: + case RTW89_PKT_DROP_SEL_MACID_VO_ONCE: + break; + default: + rtw89_debug(rtwdev, RTW89_DBG_FW, + "H2C of pkt drop might not fully support sel: %d yet\n", + params->sel); + break; + } + + skb_put(skb, H2C_PKT_DROP_LEN); + RTW89_SET_FWCMD_PKT_DROP_SEL(skb->data, params->sel); + RTW89_SET_FWCMD_PKT_DROP_MACID(skb->data, params->macid); + RTW89_SET_FWCMD_PKT_DROP_BAND(skb->data, params->mac_band); + RTW89_SET_FWCMD_PKT_DROP_PORT(skb->data, params->port); + RTW89_SET_FWCMD_PKT_DROP_MBSSID(skb->data, params->mbssid); + RTW89_SET_FWCMD_PKT_DROP_ROLE_A_INFO_TF_TRS(skb->data, params->tf_trs); + + rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, + H2C_CAT_MAC, + H2C_CL_MAC_FW_OFLD, + H2C_FUNC_PKT_DROP, 0, 0, + H2C_PKT_DROP_LEN); + + ret = rtw89_h2c_tx(rtwdev, skb, false); + if (ret) { rtw89_err(rtwdev, "failed to send h2c\n"); goto fail; } @@ -2424,5 +2894,5 @@ int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev) fail: dev_kfree_skb_any(skb); - return -EBUSY; + return ret; } diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index e75ad22aa85df8141787e1e23ca3d9163e7dcee1..0047d5d0e9b193a90ddd727dc532cdb005799be0 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -63,21 +63,32 @@ enum rtw89_mac_c2h_type { RTW89_FWCMD_C2HREG_FUNC_NULL = 0xFF }; -struct rtw89_c2h_phy_cap { - u32 func:7; - u32 ack:1; - u32 len:4; - u32 seq:4; - u32 rx_nss:8; - u32 bw:8; - - u32 tx_nss:8; - u32 prot:8; - u32 nic:8; - u32 wl_func:8; - - u32 hw_type:8; -} __packed; +#define RTW89_GET_C2H_PHYCAP_FUNC(info) \ + u32_get_bits(*((const u32 *)(info)), GENMASK(6, 0)) +#define RTW89_GET_C2H_PHYCAP_ACK(info) \ + u32_get_bits(*((const u32 *)(info)), BIT(7)) +#define RTW89_GET_C2H_PHYCAP_LEN(info) \ + u32_get_bits(*((const u32 *)(info)), GENMASK(11, 8)) +#define RTW89_GET_C2H_PHYCAP_SEQ(info) \ + u32_get_bits(*((const u32 *)(info)), GENMASK(15, 12)) +#define RTW89_GET_C2H_PHYCAP_RX_NSS(info) \ + u32_get_bits(*((const u32 *)(info)), GENMASK(23, 16)) +#define RTW89_GET_C2H_PHYCAP_BW(info) \ + u32_get_bits(*((const u32 *)(info)), GENMASK(31, 24)) +#define RTW89_GET_C2H_PHYCAP_TX_NSS(info) \ + u32_get_bits(*((const u32 *)(info) + 1), GENMASK(7, 0)) +#define RTW89_GET_C2H_PHYCAP_PROT(info) \ + u32_get_bits(*((const u32 *)(info) + 1), GENMASK(15, 8)) +#define RTW89_GET_C2H_PHYCAP_NIC(info) \ + u32_get_bits(*((const u32 *)(info) + 1), GENMASK(23, 16)) +#define RTW89_GET_C2H_PHYCAP_WL_FUNC(info) \ + u32_get_bits(*((const u32 *)(info) + 1), GENMASK(31, 24)) +#define RTW89_GET_C2H_PHYCAP_HW_TYPE(info) \ + u32_get_bits(*((const u32 *)(info) + 2), GENMASK(7, 0)) +#define RTW89_GET_C2H_PHYCAP_ANT_TX_NUM(info) \ + u32_get_bits(*((const u32 *)(info) + 3), GENMASK(15, 8)) +#define RTW89_GET_C2H_PHYCAP_ANT_RX_NUM(info) \ + u32_get_bits(*((const u32 *)(info) + 3), GENMASK(23, 16)) enum rtw89_fw_c2h_category { RTW89_C2H_CAT_TEST, @@ -144,6 +155,13 @@ enum rtw89_chan_type { RTW89_CHAN_DFS, }; +enum rtw89_p2pps_action { + RTW89_P2P_ACT_INIT = 0, + RTW89_P2P_ACT_UPDATE = 1, + RTW89_P2P_ACT_REMOVE = 2, + RTW89_P2P_ACT_TERMINATE = 3, +}; + #define FWDL_SECTION_MAX_NUM 10 #define FWDL_SECTION_CHKSUM_LEN 8 #define FWDL_SECTION_PER_PKT_LEN 2020 @@ -177,6 +195,7 @@ struct rtw89_h2creg_sch_tx_en { u16 rsvd:15; } __packed; +#define RTW89_H2C_MAX_SIZE 2048 #define RTW89_CHANNEL_TIME 45 #define RTW89_DFS_CHAN_TIME 105 #define RTW89_OFF_CHAN_TIME 100 @@ -186,7 +205,10 @@ struct rtw89_h2creg_sch_tx_en { #define RTW89_SCANOFLD_MAX_IE_LEN 512 #define RTW89_SCANOFLD_PKT_NONE 0xFF #define RTW89_SCANOFLD_DEBUG_MASK 0x1F -#define RTW89_MAC_CHINFO_SIZE 20 +#define RTW89_MAC_CHINFO_SIZE 24 +#define RTW89_SCAN_LIST_GUARD 4 +#define RTW89_SCAN_LIST_LIMIT \ + ((RTW89_H2C_MAX_SIZE / RTW89_MAC_CHINFO_SIZE) - RTW89_SCAN_LIST_GUARD) struct rtw89_mac_chinfo { u8 period; @@ -346,6 +368,16 @@ static inline void RTW89_SET_FWCMD_RA_CR_TBL_SEL(void *cmd, u32 val) le32p_replace_bits((__le32 *)(cmd) + 0x03, val, BIT(10)); } +static inline void RTW89_SET_FWCMD_RA_FIX_GILTF_EN(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)(cmd) + 0x03, val, BIT(11)); +} + +static inline void RTW89_SET_FWCMD_RA_FIX_GILTF(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)(cmd) + 0x03, val, GENMASK(14, 12)); +} + static inline void RTW89_SET_FWCMD_RA_FIXED_CSI_MCS_SS_IDX(void *cmd, u32 val) { le32p_replace_bits((__le32 *)(cmd) + 0x03, val, GENMASK(23, 16)); @@ -1798,6 +1830,36 @@ static inline void RTW89_SET_FWCMD_CPU_EXCEPTION_TYPE(void *cmd, u32 val) le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 0)); } +static inline void RTW89_SET_FWCMD_PKT_DROP_SEL(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_PKT_DROP_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8)); +} + +static inline void RTW89_SET_FWCMD_PKT_DROP_BAND(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16)); +} + +static inline void RTW89_SET_FWCMD_PKT_DROP_PORT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24)); +} + +static inline void RTW89_SET_FWCMD_PKT_DROP_MBSSID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_PKT_DROP_ROLE_A_INFO_TF_TRS(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(15, 8)); +} + enum rtw89_btc_btf_h2c_class { BTFC_SET = 0x10, BTFC_GET = 0x11, @@ -2006,69 +2068,104 @@ static inline void RTW89_SET_FWCMD_CXROLE_ROLE_NAN(void *cmd, u16 val) le16p_replace_bits((__le16 *)((u8 *)(cmd) + 4), val, BIT(11)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(0)); + u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_PID(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_PID(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, GENMASK(3, 1)); + u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(3, 1)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_PHY(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_PHY(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(4)); + u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(4)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(5)); + u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(5)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_BAND(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_BAND(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, GENMASK(7, 6)); + u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(7, 6)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (7 + 12 * (n)), val, BIT(0)); + u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, BIT(0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_BW(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_BW(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (7 + 12 * (n)), val, GENMASK(7, 1)); + u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, GENMASK(7, 1)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_ROLE(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_ROLE(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (8 + 12 * (n)), val, GENMASK(7, 0)); + u8p_replace_bits((u8 *)cmd + (8 + (12 + offset) * n), val, GENMASK(7, 0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_CH(void *cmd, u8 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_CH(void *cmd, u8 val, int n, u8 offset) { - u8p_replace_bits((u8 *)(cmd) + (9 + 12 * (n)), val, GENMASK(7, 0)); + u8p_replace_bits((u8 *)cmd + (9 + (12 + offset) * n), val, GENMASK(7, 0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(void *cmd, u16 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(void *cmd, u16 val, int n, u8 offset) { - le16p_replace_bits((__le16 *)((u8 *)(cmd) + (10 + 12 * (n))), val, GENMASK(15, 0)); + le16p_replace_bits((__le16 *)((u8 *)cmd + (10 + (12 + offset) * n)), val, GENMASK(15, 0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(void *cmd, u16 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(void *cmd, u16 val, int n, u8 offset) { - le16p_replace_bits((__le16 *)((u8 *)(cmd) + (12 + 12 * (n))), val, GENMASK(15, 0)); + le16p_replace_bits((__le16 *)((u8 *)cmd + (12 + (12 + offset) * n)), val, GENMASK(15, 0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(void *cmd, u16 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(void *cmd, u16 val, int n, u8 offset) { - le16p_replace_bits((__le16 *)((u8 *)(cmd) + (14 + 12 * (n))), val, GENMASK(15, 0)); + le16p_replace_bits((__le16 *)((u8 *)cmd + (14 + (12 + offset) * n)), val, GENMASK(15, 0)); } -static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(void *cmd, u16 val, int n) +static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(void *cmd, u16 val, int n, u8 offset) { - le16p_replace_bits((__le16 *)((u8 *)(cmd) + (16 + 12 * (n))), val, GENMASK(15, 0)); + le16p_replace_bits((__le16 *)((u8 *)cmd + (16 + (12 + offset) * n)), val, GENMASK(15, 0)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR(void *cmd, u32 val, int n, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + (20 + (12 + offset) * n)), val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset), val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_MROLE_NOA(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 4), val, GENMASK(31, 0)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_DBCC_EN(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(0)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_DBCC_CHG(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(1)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_DBCC_2G_PHY(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, GENMASK(3, 2)); +} + +static inline void RTW89_SET_FWCMD_CXROLE_LINK_MODE_CHG(void *cmd, u32 val, u8 offset) +{ + le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(4)); } static inline void RTW89_SET_FWCMD_CXCTRL_MANUAL(void *cmd, u32 val) @@ -2352,6 +2449,86 @@ static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_SLOW(void *cmd, u32 val) le32p_replace_bits((__le32 *)((u8 *)(cmd) + 16), val, GENMASK(31, 0)); } +static inline void RTW89_SET_FWCMD_P2P_MACID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_P2P_P2PID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(11, 8)); +} + +static inline void RTW89_SET_FWCMD_P2P_NOAID(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 12)); +} + +static inline void RTW89_SET_FWCMD_P2P_ACT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(19, 16)); +} + +static inline void RTW89_SET_FWCMD_P2P_TYPE(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(20)); +} + +static inline void RTW89_SET_FWCMD_P2P_ALL_SLEP(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(21)); +} + +static inline void RTW89_SET_FWCMD_NOA_START_TIME(void *cmd, __le32 val) +{ + *((__le32 *)cmd + 1) = val; +} + +static inline void RTW89_SET_FWCMD_NOA_INTERVAL(void *cmd, __le32 val) +{ + *((__le32 *)cmd + 2) = val; +} + +static inline void RTW89_SET_FWCMD_NOA_DURATION(void *cmd, __le32 val) +{ + *((__le32 *)cmd + 3) = val; +} + +static inline void RTW89_SET_FWCMD_NOA_COUNT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)(cmd) + 4, val, GENMASK(7, 0)); +} + +static inline void RTW89_SET_FWCMD_NOA_CTWINDOW(void *cmd, u32 val) +{ + u8 ctwnd; + + if (!(val & IEEE80211_P2P_OPPPS_ENABLE_BIT)) + return; + ctwnd = FIELD_GET(IEEE80211_P2P_OPPPS_CTWINDOW_MASK, val); + le32p_replace_bits((__le32 *)(cmd) + 4, ctwnd, GENMASK(23, 8)); +} + +static inline void RTW89_SET_FWCMD_TSF32_TOGL_BAND(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(0)); +} + +static inline void RTW89_SET_FWCMD_TSF32_TOGL_EN(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, BIT(1)); +} + +static inline void RTW89_SET_FWCMD_TSF32_TOGL_PORT(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(4, 2)); +} + +static inline void RTW89_SET_FWCMD_TSF32_TOGL_EARLY(void *cmd, u32 val) +{ + le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 16)); +} + #define RTW89_C2H_HEADER_LEN 8 #define RTW89_GET_C2H_CATEGORY(c2h) \ @@ -2421,6 +2598,8 @@ static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_SLOW(void *cmd, u32 val) le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(19, 16)) #define RTW89_GET_MAC_C2H_SCANOFLD_STATUS(c2h) \ le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(23, 20)) +#define RTW89_GET_MAC_C2H_ACTUAL_PERIOD(c2h) \ + le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(31, 24)) #define RTW89_GET_MAC_C2H_SCANOFLD_TX_FAIL(c2h) \ le32_get_bits(*((const __le32 *)(c2h) + 5), GENMASK(3, 0)) #define RTW89_GET_MAC_C2H_SCANOFLD_AIR_DENSITY(c2h) \ @@ -2446,7 +2625,14 @@ struct rtw89_mfw_info { struct rtw89_mfw_hdr { u8 sig; /* RTW89_MFW_SIG */ u8 fw_nr; - u8 rsvd[14]; + u8 rsvd0[2]; + struct { + u8 major; + u8 minor; + u8 sub; + u8 idx; + } ver; + u8 rsvd1[8]; struct rtw89_mfw_info info[]; } __packed; @@ -2493,6 +2679,7 @@ struct rtw89_fw_h2c_rf_reg_info { /* CLASS 2 - PS */ #define H2C_CL_MAC_PS 0x2 #define H2C_FUNC_MAC_LPS_PARM 0x0 +#define H2C_FUNC_P2P_ACT 0x1 /* CLASS 3 - FW download */ #define H2C_CL_MAC_FWDL 0x3 @@ -2519,9 +2706,11 @@ struct rtw89_fw_h2c_rf_reg_info { #define H2C_FUNC_PACKET_OFLD 0x1 #define H2C_FUNC_MAC_MACID_PAUSE 0x8 #define H2C_FUNC_USR_EDCA 0xF +#define H2C_FUNC_TSF32_TOGL 0x10 #define H2C_FUNC_OFLD_CFG 0x14 #define H2C_FUNC_ADD_SCANOFLD_CH 0x16 #define H2C_FUNC_SCANOFLD 0x17 +#define H2C_FUNC_PKT_DROP 0x1b /* CLASS 10 - Security CAM */ #define H2C_CL_MAC_SEC_CAM 0xa @@ -2552,7 +2741,7 @@ struct rtw89_fw_h2c_rf_get_mccch { #define RTW89_FW_RSVD_PLE_SIZE 0x800 -#define RTW89_WCPU_BASE_ADDR 0xA0000000 +#define RTW89_WCPU_BASE_MASK GENMASK(27, 0) #define RTW89_FW_BACKTRACE_INFO_SIZE 8 #define RTW89_VALID_FW_BACKTRACE_SIZE(_size) \ @@ -2563,6 +2752,9 @@ struct rtw89_fw_h2c_rf_get_mccch { int rtw89_fw_check_rdy(struct rtw89_dev *rtwdev); int rtw89_fw_recognize(struct rtw89_dev *rtwdev); +void rtw89_early_fw_feature_recognize(struct device *device, + const struct rtw89_chip_info *chip, + u32 *early_feat_map); int rtw89_fw_download(struct rtw89_dev *rtwdev, enum rtw89_fw_type type); int rtw89_load_firmware(struct rtw89_dev *rtwdev); void rtw89_unload_firmware(struct rtw89_dev *rtwdev); @@ -2577,6 +2769,8 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta); int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta); +int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta); int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *vif, @@ -2600,6 +2794,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi); int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev); +int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id); @@ -2623,6 +2818,7 @@ void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid); int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, bool valid, struct ieee80211_ampdu_params *params); +void rtw89_fw_h2c_init_ba_cam_v1(struct rtw89_dev *rtwdev); int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev, struct rtw89_lps_parm *lps_param); @@ -2642,5 +2838,20 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, bool enable); void rtw89_hw_scan_abort(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif); int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev); +int rtw89_fw_h2c_pkt_drop(struct rtw89_dev *rtwdev, + const struct rtw89_pkt_drop_params *params); +int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, + struct ieee80211_p2p_noa_desc *desc, + u8 act, u8 noa_id); +int rtw89_fw_h2c_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, + bool en); + +static inline void rtw89_fw_h2c_init_ba_cam(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + if (chip->bacam_v1) + rtw89_fw_h2c_init_ba_cam_v1(rtwdev); +} #endif diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 93124b815825feb0c445bf032d51df64239120b6..0508dfca8edf731e25acb1a3c976c85060b2a495 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -3,6 +3,7 @@ */ #include "cam.h" +#include "chan.h" #include "debug.h" #include "fw.h" #include "mac.h" @@ -826,6 +827,8 @@ static void hfc_func_en(struct rtw89_dev *rtwdev, bool en, bool h2c_en) static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) { + const struct rtw89_chip_info *chip = rtwdev->chip; + u32 dma_ch_mask = chip->dma_ch_mask; u8 ch; u32 ret = 0; @@ -847,6 +850,8 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) } for (ch = RTW89_DMA_ACH0; ch < RTW89_DMA_H2C; ch++) { + if (dma_ch_mask & BIT(ch)) + continue; ret = hfc_ch_ctrl(rtwdev, ch); if (ret) return ret; @@ -862,6 +867,8 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en) udelay(10); } for (ch = RTW89_DMA_ACH0; ch < RTW89_DMA_H2C; ch++) { + if (dma_ch_mask & BIT(ch)) + continue; ret = hfc_upd_ch_info(rtwdev, ch); if (ret) return ret; @@ -1053,18 +1060,29 @@ void rtw89_mac_power_mode_change(struct rtw89_dev *rtwdev, bool enter) enum rtw89_rpwm_req_pwr_state state; unsigned long delay = enter ? 10 : 150; int ret; + int i; if (enter) state = rtw89_mac_get_req_pwr_state(rtwdev); else state = RTW89_MAC_RPWM_REQ_PWR_STATE_ACTIVE; - rtw89_mac_send_rpwm(rtwdev, state, false); - ret = read_poll_timeout_atomic(rtw89_mac_check_cpwm_state, ret, !ret, - delay, 15000, false, rtwdev, state); - if (ret) - rtw89_err(rtwdev, "firmware failed to ack for %s ps mode\n", - enter ? "entering" : "leaving"); + for (i = 0; i < RPWM_TRY_CNT; i++) { + rtw89_mac_send_rpwm(rtwdev, state, false); + ret = read_poll_timeout_atomic(rtw89_mac_check_cpwm_state, ret, + !ret, delay, 15000, false, + rtwdev, state); + if (!ret) + break; + + if (i == RPWM_TRY_CNT - 1) + rtw89_err(rtwdev, "firmware failed to ack for %s ps mode\n", + enter ? "entering" : "leaving"); + else + rtw89_debug(rtwdev, RTW89_DBG_UNEXP, + "%d time firmware failed to ack for %s ps mode\n", + i + 1, enter ? "entering" : "leaving"); + } } void rtw89_mac_notify_wake(struct rtw89_dev *rtwdev) @@ -1081,7 +1099,6 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on) const struct rtw89_chip_info *chip = rtwdev->chip; const struct rtw89_pwr_cfg * const *cfg_seq; int (*cfg_func)(struct rtw89_dev *rtwdev); - struct rtw89_hal *hal = &rtwdev->hal; int ret; u8 val; @@ -1113,7 +1130,7 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on) clear_bit(RTW89_FLAG_POWERON, rtwdev->flags); clear_bit(RTW89_FLAG_FW_RDY, rtwdev->flags); rtw89_write8(rtwdev, R_AX_SCOREBOARD + 3, MAC_AX_NOTIFY_PWR_MAJOR); - hal->current_channel = 0; + rtw89_set_entity_state(rtwdev, false); } return 0; @@ -1207,8 +1224,8 @@ static int chip_func_en(struct rtw89_dev *rtwdev) { enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - if (chip_id == RTL8852A) - rtw89_write32_set(rtwdev, R_AX_SPSLDO_ON_CTRL0, + if (chip_id == RTL8852A || chip_id == RTL8852B) + rtw89_write32_set(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_OCP_L1_MASK); return 0; @@ -1239,6 +1256,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .wde_size0 = {RTW89_WDE_PG_64, 4095, 1,}, /* DLFW */ .wde_size4 = {RTW89_WDE_PG_64, 0, 4096,}, + /* PCIE 64 */ + .wde_size6 = {RTW89_WDE_PG_64, 512, 0,}, + /* DLFW */ + .wde_size9 = {RTW89_WDE_PG_64, 0, 1024,}, /* 8852C DLFW */ .wde_size18 = {RTW89_WDE_PG_64, 0, 2048,}, /* 8852C PCIE SCC */ @@ -1247,6 +1268,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .ple_size0 = {RTW89_PLE_PG_128, 1520, 16,}, /* DLFW */ .ple_size4 = {RTW89_PLE_PG_128, 64, 1472,}, + /* PCIE 64 */ + .ple_size6 = {RTW89_PLE_PG_128, 496, 16,}, + /* DLFW */ + .ple_size8 = {RTW89_PLE_PG_128, 64, 960,}, /* 8852C DLFW */ .ple_size18 = {RTW89_PLE_PG_128, 2544, 16,}, /* 8852C PCIE SCC */ @@ -1255,6 +1280,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .wde_qt0 = {3792, 196, 0, 107,}, /* DLFW */ .wde_qt4 = {0, 0, 0, 0,}, + /* PCIE 64 */ + .wde_qt6 = {448, 48, 0, 16,}, /* 8852C DLFW */ .wde_qt17 = {0, 0, 0, 0,}, /* 8852C PCIE SCC */ @@ -1265,6 +1292,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .ple_qt5 = {264, 0, 32, 20, 64, 13, 1101, 0, 64, 128, 120,}, /* DLFW */ .ple_qt13 = {0, 0, 16, 48, 0, 0, 0, 0, 0, 0, 0,}, + /* PCIE 64 */ + .ple_qt18 = {147, 0, 16, 20, 17, 13, 89, 0, 32, 14, 8, 0,}, /* DLFW 52C */ .ple_qt44 = {0, 0, 16, 256, 0, 0, 0, 0, 0, 0, 0, 0,}, /* DLFW 52C */ @@ -1273,6 +1302,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = { .ple_qt46 = {525, 0, 16, 20, 13, 13, 178, 0, 32, 62, 8, 16,}, /* 8852C PCIE SCC */ .ple_qt47 = {525, 0, 32, 20, 1034, 13, 1199, 0, 1053, 62, 160, 1037,}, + /* PCIE 64 */ + .ple_qt58 = {147, 0, 16, 20, 157, 13, 229, 0, 172, 14, 24, 0,}, }; EXPORT_SYMBOL(rtw89_mac_size); @@ -1307,6 +1338,17 @@ static inline u32 dle_used_size(const struct rtw89_dle_size *wde, ple->pge_size * (ple->lnk_pge_num + ple->unlnk_pge_num); } +static u32 dle_expected_used_size(struct rtw89_dev *rtwdev, + enum rtw89_qta_mode mode) +{ + u32 size = rtwdev->chip->fifo_size; + + if (mode == RTW89_QTA_SCC) + size -= rtwdev->chip->dle_scc_rsvd_size; + + return size; +} + static void dle_func_en(struct rtw89_dev *rtwdev, bool enable) { if (enable) @@ -1474,7 +1516,8 @@ static int dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode, ext_wde_min_qt_wcpu = ext_cfg->wde_min_qt->wcpu; } - if (dle_used_size(cfg->wde_size, cfg->ple_size) != rtwdev->chip->fifo_size) { + if (dle_used_size(cfg->wde_size, cfg->ple_size) != + dle_expected_used_size(rtwdev, mode)) { rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n"); ret = -EINVAL; goto error; @@ -1734,7 +1777,7 @@ static int addr_cam_init(struct rtw89_dev *rtwdev, u8 mac_idx) rtw89_write32(rtwdev, reg, val); ret = read_poll_timeout(rtw89_read16, p_val, !(p_val & B_AX_ADDR_CAM_CLR), - 1, TRXCFG_WAIT_CNT, false, rtwdev, B_AX_ADDR_CAM_CLR); + 1, TRXCFG_WAIT_CNT, false, rtwdev, reg); if (ret) { rtw89_err(rtwdev, "[ERR]ADDR_CAM reset\n"); return ret; @@ -1747,13 +1790,19 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx) { u32 ret; u32 reg; + u32 val; ret = rtw89_mac_check_mac_en(rtwdev, mac_idx, RTW89_CMAC_SEL); if (ret) return ret; reg = rtw89_mac_reg_by_idx(R_AX_PREBKF_CFG_1, mac_idx); - rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK, SIFS_MACTXEN_T1); + if (rtwdev->chip->chip_id == RTL8852C) + rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK, + SIFS_MACTXEN_T1_V1); + else + rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK, + SIFS_MACTXEN_T1); if (rtwdev->chip->chip_id == RTL8852B) { reg = rtw89_mac_reg_by_idx(R_AX_SCH_EXT_CTRL, mac_idx); @@ -1764,7 +1813,16 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx) rtw89_write32_clr(rtwdev, reg, B_AX_BTCCA_EN); reg = rtw89_mac_reg_by_idx(R_AX_PREBKF_CFG_0, mac_idx); - rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK, SCH_PREBKF_24US); + if (rtwdev->chip->chip_id == RTL8852C) { + val = rtw89_read32_mask(rtwdev, R_AX_SEC_ENG_CTRL, + B_AX_TX_PARTIAL_MODE); + if (!val) + rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK, + SCH_PREBKF_24US); + } else { + rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK, + SCH_PREBKF_24US); + } return 0; } @@ -1910,7 +1968,7 @@ static int nav_ctrl_init(struct rtw89_dev *rtwdev) rtw89_write32_set(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_PLCP_UP_NAV_EN | B_AX_WMAC_TF_UP_NAV_EN | B_AX_WMAC_NAV_UPPER_EN); - rtw89_write32_mask(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_NAV_UPPER_MASK, NAV_12MS); + rtw89_write32_mask(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_NAV_UPPER_MASK, NAV_25MS); return 0; } @@ -1953,6 +2011,8 @@ static int tmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) static int trxptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) { + const struct rtw89_chip_info *chip = rtwdev->chip; + const struct rtw89_rrsr_cfgs *rrsr = chip->rrsr_cfgs; u32 reg, val, sifs; int ret; @@ -1983,6 +2043,11 @@ static int trxptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) reg = rtw89_mac_reg_by_idx(R_AX_RXTRIG_TEST_USER_2, mac_idx); rtw89_write32_set(rtwdev, reg, B_AX_RXTRIG_FCSCHK_EN); + reg = rtw89_mac_reg_by_idx(rrsr->ref_rate.addr, mac_idx); + rtw89_write32_mask(rtwdev, reg, rrsr->ref_rate.mask, rrsr->ref_rate.data); + reg = rtw89_mac_reg_by_idx(rrsr->rsc.addr, mac_idx); + rtw89_write32_mask(rtwdev, reg, rrsr->rsc.mask, rrsr->rsc.data); + return 0; } @@ -2061,6 +2126,7 @@ static int rmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx) { + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u32 val, reg; int ret; @@ -2075,6 +2141,11 @@ static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx) val = u32_replace_bits(val, 0, B_AX_TXSC_80M_MASK); rtw89_write32(rtwdev, reg, val); + if (chip_id == RTL8852A || chip_id == RTL8852B) { + reg = rtw89_mac_reg_by_idx(R_AX_PTCL_RRSR1, mac_idx); + rtw89_write32_mask(rtwdev, reg, B_AX_RRSR_RATE_EN_MASK, RRSR_OFDM_CCK_EN); + } + return 0; } @@ -2134,6 +2205,25 @@ static int ptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx) return 0; } +static int cmac_dma_init(struct rtw89_dev *rtwdev, u8 mac_idx) +{ + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + u32 reg; + int ret; + + if (chip_id != RTL8852A && chip_id != RTL8852B) + return 0; + + ret = rtw89_mac_check_mac_en(rtwdev, mac_idx, RTW89_CMAC_SEL); + if (ret) + return ret; + + reg = rtw89_mac_reg_by_idx(R_AX_RXDMA_CTRL_0, mac_idx); + rtw89_write8_clr(rtwdev, reg, RX_FULL_MODE); + + return 0; +} + static int cmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) { int ret; @@ -2209,6 +2299,12 @@ static int cmac_init(struct rtw89_dev *rtwdev, u8 mac_idx) return ret; } + ret = cmac_dma_init(rtwdev, mac_idx); + if (ret) { + rtw89_err(rtwdev, "[ERR]CMAC%d DMA init %d\n", mac_idx, ret); + return ret; + } + return ret; } @@ -2236,23 +2332,42 @@ int rtw89_mac_setup_phycap(struct rtw89_dev *rtwdev) struct rtw89_hal *hal = &rtwdev->hal; const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_mac_c2h_info c2h_info = {0}; - struct rtw89_c2h_phy_cap *cap = - (struct rtw89_c2h_phy_cap *)&c2h_info.c2hreg[0]; + u8 tx_nss; + u8 rx_nss; + u8 tx_ant; + u8 rx_ant; u32 ret; ret = rtw89_mac_read_phycap(rtwdev, &c2h_info); if (ret) return ret; - hal->tx_nss = cap->tx_nss ? - min_t(u8, cap->tx_nss, chip->tx_nss) : chip->tx_nss; - hal->rx_nss = cap->rx_nss ? - min_t(u8, cap->rx_nss, chip->rx_nss) : chip->rx_nss; + tx_nss = RTW89_GET_C2H_PHYCAP_TX_NSS(c2h_info.c2hreg); + rx_nss = RTW89_GET_C2H_PHYCAP_RX_NSS(c2h_info.c2hreg); + tx_ant = RTW89_GET_C2H_PHYCAP_ANT_TX_NUM(c2h_info.c2hreg); + rx_ant = RTW89_GET_C2H_PHYCAP_ANT_RX_NUM(c2h_info.c2hreg); + + hal->tx_nss = tx_nss ? min_t(u8, tx_nss, chip->tx_nss) : chip->tx_nss; + hal->rx_nss = rx_nss ? min_t(u8, rx_nss, chip->rx_nss) : chip->rx_nss; + + if (tx_ant == 1) + hal->antenna_tx = RF_B; + if (rx_ant == 1) + hal->antenna_rx = RF_B; + + if (tx_nss == 1 && tx_ant == 2 && rx_ant == 2) { + hal->antenna_tx = RF_B; + hal->tx_path_diversity = true; + } rtw89_debug(rtwdev, RTW89_DBG_FW, "phycap hal/phy/chip: tx_nss=0x%x/0x%x/0x%x rx_nss=0x%x/0x%x/0x%x\n", - hal->tx_nss, cap->tx_nss, chip->tx_nss, - hal->rx_nss, cap->rx_nss, chip->rx_nss); + hal->tx_nss, tx_nss, chip->tx_nss, + hal->rx_nss, rx_nss, chip->rx_nss); + rtw89_debug(rtwdev, RTW89_DBG_FW, + "ant num/bitmap: tx=%d/0x%x rx=%d/0x%x\n", + tx_ant, hal->antenna_tx, rx_ant, hal->antenna_rx); + rtw89_debug(rtwdev, RTW89_DBG_FW, "TX path diversity=%d\n", hal->tx_path_diversity); return 0; } @@ -2429,8 +2544,7 @@ int rtw89_mac_resume_sch_tx_v1(struct rtw89_dev *rtwdev, u8 mac_idx, u32 tx_en) } EXPORT_SYMBOL(rtw89_mac_resume_sch_tx_v1); -static u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, - bool wd) +u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd) { u32 val, reg; int ret; @@ -2450,9 +2564,8 @@ static u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, return FIELD_GET(B_AX_WD_BUF_STAT_PKTID_MASK, val); } -static int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, - struct rtw89_cpuio_ctrl *ctrl_para, - bool wd) +int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, + struct rtw89_cpuio_ctrl *ctrl_para, bool wd) { u32 val, cmd_type, reg; int ret; @@ -2517,7 +2630,8 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode) return -EINVAL; } - if (dle_used_size(cfg->wde_size, cfg->ple_size) != rtwdev->chip->fifo_size) { + if (dle_used_size(cfg->wde_size, cfg->ple_size) != + dle_expected_used_size(rtwdev, mode)) { rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n"); return -EINVAL; } @@ -2766,7 +2880,7 @@ static void rtw89_bbrpt_imr_enable(struct rtw89_dev *rtwdev) { const struct rtw89_imr_info *imr = rtwdev->chip->imr_info; - rtw89_write32_set(rtwdev, R_AX_BBRPT_COM_ERR_IMR, + rtw89_write32_set(rtwdev, imr->bbrpt_com_err_imr_reg, B_AX_BBRPT_COM_NULL_PLPKTID_ERR_INT_EN); rtw89_write32_clr(rtwdev, imr->bbrpt_chinfo_err_imr_reg, B_AX_BBRPT_CHINFO_IMR_CLR); @@ -3026,6 +3140,8 @@ static int rtw89_mac_enable_cpu(struct rtw89_dev *rtwdev, u8 boot_reason, rtw89_write32(rtwdev, R_AX_HALT_H2C_CTRL, 0); rtw89_write32(rtwdev, R_AX_HALT_C2H_CTRL, 0); + rtw89_write32(rtwdev, R_AX_HALT_H2C, 0); + rtw89_write32(rtwdev, R_AX_HALT_C2H, 0); rtw89_write32_set(rtwdev, R_AX_SYS_CLK_CTRL, B_AX_CPU_CLK_EN); @@ -3103,14 +3219,6 @@ dle: return ret; } -static void rtw89_mac_hci_func_en(struct rtw89_dev *rtwdev) -{ - const struct rtw89_chip_info *chip = rtwdev->chip; - - rtw89_write32_set(rtwdev, chip->hci_func_en_addr, - B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN); -} - int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev) { rtw89_write8_set(rtwdev, R_AX_SYS_FUNC_EN, @@ -3124,7 +3232,7 @@ int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev) } EXPORT_SYMBOL(rtw89_mac_enable_bb_rf); -void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev) +int rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev) { rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN, B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN); @@ -3132,6 +3240,8 @@ void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev) B_AX_WLRF1_CTRL_7 | B_AX_WLRF1_CTRL_1 | B_AX_WLRF_CTRL_7 | B_AX_WLRF_CTRL_1); rtw89_write8_clr(rtwdev, R_AX_PHYREG_SET, PHYREG_SET_ALL_CYCLE); + + return 0; } EXPORT_SYMBOL(rtw89_mac_disable_bb_rf); @@ -3147,7 +3257,7 @@ int rtw89_mac_partial_init(struct rtw89_dev *rtwdev) return ret; } - rtw89_mac_hci_func_en(rtwdev); + rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); ret = rtw89_mac_dmac_pre_init(rtwdev); if (ret) @@ -3524,6 +3634,26 @@ static void rtw89_mac_port_cfg_bcn_early(struct rtw89_dev *rtwdev, BCN_ERLY_DEF); } +static void rtw89_mac_port_cfg_tbtt_shift(struct rtw89_dev *rtwdev, + struct rtw89_vif *rtwvif) +{ + const struct rtw89_port_reg *p = &rtw_port_base; + u16 val; + + if (rtwdev->chip->chip_id != RTL8852C) + return; + + if (rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT && + rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION) + return; + + val = FIELD_PREP(B_AX_TBTT_SHIFT_OFST_MAG, 1) | + B_AX_TBTT_SHIFT_OFST_SIGN; + + rtw89_write16_port_mask(rtwdev, rtwvif, p->tbtt_shift, + B_AX_TBTT_SHIFT_OFST_MASK, val); +} + int rtw89_mac_vif_init(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { int ret; @@ -3598,6 +3728,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) rtw89_mac_port_cfg_bcn_hold_time(rtwdev, rtwvif); rtw89_mac_port_cfg_bcn_mask_area(rtwdev, rtwvif); rtw89_mac_port_cfg_tbtt_early(rtwdev, rtwvif); + rtw89_mac_port_cfg_tbtt_shift(rtwdev, rtwvif); rtw89_mac_port_cfg_bss_color(rtwdev, rtwvif); rtw89_mac_port_cfg_mbssid(rtwdev, rtwvif); rtw89_mac_port_cfg_func_en(rtwdev, rtwvif); @@ -3607,6 +3738,50 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) return 0; } +static void rtw89_mac_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy, + struct cfg80211_bss *bss, + void *data) +{ + const struct cfg80211_bss_ies *ies; + const struct element *elem; + bool *tolerated = data; + + rcu_read_lock(); + ies = rcu_dereference(bss->ies); + elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data, + ies->len); + + if (!elem || elem->datalen < 10 || + !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) + *tolerated = false; + rcu_read_unlock(); +} + +void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev, + struct ieee80211_vif *vif) +{ + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + struct ieee80211_hw *hw = rtwdev->hw; + bool tolerated = true; + u32 reg; + + if (!vif->bss_conf.he_support || vif->type != NL80211_IFTYPE_STATION) + return; + + if (!(vif->bss_conf.chandef.chan->flags & IEEE80211_CHAN_RADAR)) + return; + + cfg80211_bss_iter(hw->wiphy, &vif->bss_conf.chandef, + rtw89_mac_check_he_obss_narrow_bw_ru_iter, + &tolerated); + + reg = rtw89_mac_reg_by_idx(R_AX_RXTRIG_TEST_USER_2, rtwvif->mac_idx); + if (tolerated) + rtw89_write32_clr(rtwdev, reg, B_AX_RXTRIG_RU26_DIS); + else + rtw89_write32_set(rtwdev, reg, B_AX_RXTRIG_RU26_DIS); +} + int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { int ret; @@ -3655,22 +3830,26 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) { struct ieee80211_vif *vif = rtwdev->scan_info.scanning_vif; - struct rtw89_hal *hal = &rtwdev->hal; - u8 reason, status, tx_fail, band; + struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif); + struct rtw89_chan new; + u8 reason, status, tx_fail, band, actual_period; + u32 last_chan = rtwdev->scan_info.last_chan_idx; u16 chan; + int ret; tx_fail = RTW89_GET_MAC_C2H_SCANOFLD_TX_FAIL(c2h->data); status = RTW89_GET_MAC_C2H_SCANOFLD_STATUS(c2h->data); chan = RTW89_GET_MAC_C2H_SCANOFLD_PRI_CH(c2h->data); reason = RTW89_GET_MAC_C2H_SCANOFLD_RSP(c2h->data); band = RTW89_GET_MAC_C2H_SCANOFLD_BAND(c2h->data); + actual_period = RTW89_GET_MAC_C2H_ACTUAL_PERIOD(c2h->data); if (!(rtwdev->chip->support_bands & BIT(NL80211_BAND_6GHZ))) band = chan > 14 ? RTW89_BAND_5G : RTW89_BAND_2G; rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN, - "band: %d, chan: %d, reason: %d, status: %d, tx_fail: %d\n", - band, chan, reason, status, tx_fail); + "band: %d, chan: %d, reason: %d, status: %d, tx_fail: %d, actual: %d\n", + band, chan, reason, status, tx_fail, actual_period); switch (reason) { case RTW89_SCAN_LEAVE_CH_NOTIFY: @@ -3678,15 +3857,20 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h, ieee80211_stop_queues(rtwdev->hw); return; case RTW89_SCAN_END_SCAN_NOTIFY: - rtw89_hw_scan_complete(rtwdev, vif, false); + if (rtwvif && rtwvif->scan_req && + last_chan < rtwvif->scan_req->n_channels) { + ret = rtw89_hw_scan_offload(rtwdev, vif, true); + if (ret) { + rtw89_hw_scan_abort(rtwdev, vif); + rtw89_warn(rtwdev, "HW scan failed: %d\n", ret); + } + } else { + rtw89_hw_scan_complete(rtwdev, vif, false); + } break; case RTW89_SCAN_ENTER_CH_NOTIFY: - hal->prev_band_type = hal->current_band_type; - hal->current_band_type = band; - hal->prev_primary_channel = hal->current_primary_channel; - hal->current_primary_channel = chan; - hal->current_channel = chan; - hal->current_band_width = RTW89_CHANNEL_WIDTH_20; + rtw89_chan_create(&new, chan, chan, band, RTW89_CHANNEL_WIDTH_20); + rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new); if (rtw89_is_op_chan(rtwdev, band, chan)) { rtw89_store_op_chan(rtwdev, false); ieee80211_wake_queues(rtwdev->hw); @@ -3738,6 +3922,12 @@ rtw89_mac_c2h_pkt_ofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h, { } +static void +rtw89_mac_c2h_tsf32_toggle_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h, + u32 len) +{ +} + static void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev, struct sk_buff *c2h, u32 len) = { @@ -3747,6 +3937,7 @@ void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev, [RTW89_MAC_C2H_FUNC_BCN_RESEND] = NULL, [RTW89_MAC_C2H_FUNC_MACID_PAUSE] = rtw89_mac_c2h_macid_pause, [RTW89_MAC_C2H_FUNC_SCANOFLD_RSP] = rtw89_mac_c2h_scanofld_rsp, + [RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT] = rtw89_mac_c2h_tsf32_toggle_rpt, }; static @@ -4628,3 +4819,48 @@ int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val) return 0; } + +static +void rtw89_mac_pkt_drop_sta(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta) +{ + static const enum rtw89_pkt_drop_sel sels[] = { + RTW89_PKT_DROP_SEL_MACID_BE_ONCE, + RTW89_PKT_DROP_SEL_MACID_BK_ONCE, + RTW89_PKT_DROP_SEL_MACID_VI_ONCE, + RTW89_PKT_DROP_SEL_MACID_VO_ONCE, + }; + struct rtw89_vif *rtwvif = rtwsta->rtwvif; + struct rtw89_pkt_drop_params params = {0}; + int i; + + params.mac_band = RTW89_MAC_0; + params.macid = rtwsta->mac_id; + params.port = rtwvif->port; + params.mbssid = 0; + params.tf_trs = rtwvif->trigger; + + for (i = 0; i < ARRAY_SIZE(sels); i++) { + params.sel = sels[i]; + rtw89_fw_h2c_pkt_drop(rtwdev, ¶ms); + } +} + +static void rtw89_mac_pkt_drop_vif_iter(void *data, struct ieee80211_sta *sta) +{ + struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; + struct rtw89_vif *rtwvif = rtwsta->rtwvif; + struct rtw89_dev *rtwdev = rtwvif->rtwdev; + struct rtw89_vif *target = data; + + if (rtwvif != target) + return; + + rtw89_mac_pkt_drop_sta(rtwdev, rtwsta); +} + +void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) +{ + ieee80211_iterate_stations_atomic(rtwdev->hw, + rtw89_mac_pkt_drop_vif_iter, + rtwvif); +} diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index f66619354734d079144d2fe53cb80056bf1b18e2..6f4ada1869a17c5d3c09392bc121d8a8df834f3d 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -6,11 +6,13 @@ #define __RTW89_MAC_H__ #include "core.h" +#include "reg.h" #define MAC_MEM_DUMP_PAGE_SIZE 0x40000 #define ADDR_CAM_ENT_SIZE 0x40 #define BSSID_CAM_ENT_SIZE 0x08 #define HFC_PAGE_UNIT 64 +#define RPWM_TRY_CNT 3 enum rtw89_mac_hwmod_sel { RTW89_DMAC_SEL = 0, @@ -304,6 +306,7 @@ enum rtw89_mac_c2h_ofld_func { RTW89_MAC_C2H_FUNC_PKT_OFLD_RSP, RTW89_MAC_C2H_FUNC_BCN_RESEND, RTW89_MAC_C2H_FUNC_MACID_PAUSE, + RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT = 0x6, RTW89_MAC_C2H_FUNC_SCANOFLD_RSP = 0x9, RTW89_MAC_C2H_FUNC_OFLD_MAX, }; @@ -688,23 +691,30 @@ struct rtw89_mac_size_set { const struct rtw89_hfc_prec_cfg hfc_preccfg_pcie; const struct rtw89_dle_size wde_size0; const struct rtw89_dle_size wde_size4; + const struct rtw89_dle_size wde_size6; + const struct rtw89_dle_size wde_size9; const struct rtw89_dle_size wde_size18; const struct rtw89_dle_size wde_size19; const struct rtw89_dle_size ple_size0; const struct rtw89_dle_size ple_size4; + const struct rtw89_dle_size ple_size6; + const struct rtw89_dle_size ple_size8; const struct rtw89_dle_size ple_size18; const struct rtw89_dle_size ple_size19; const struct rtw89_wde_quota wde_qt0; const struct rtw89_wde_quota wde_qt4; + const struct rtw89_wde_quota wde_qt6; const struct rtw89_wde_quota wde_qt17; const struct rtw89_wde_quota wde_qt18; const struct rtw89_ple_quota ple_qt4; const struct rtw89_ple_quota ple_qt5; const struct rtw89_ple_quota ple_qt13; + const struct rtw89_ple_quota ple_qt18; const struct rtw89_ple_quota ple_qt44; const struct rtw89_ple_quota ple_qt45; const struct rtw89_ple_quota ple_qt46; const struct rtw89_ple_quota ple_qt47; + const struct rtw89_ple_quota ple_qt58; }; extern const struct rtw89_mac_size_set rtw89_mac_size; @@ -798,9 +808,11 @@ int rtw89_mac_write_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 val); int rtw89_mac_read_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 *val); int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif); int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); +void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev, + struct ieee80211_vif *vif); int rtw89_mac_remove_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif); int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev); -void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev); +int rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev); static inline int rtw89_chip_enable_bb_rf(struct rtw89_dev *rtwdev) { @@ -809,11 +821,11 @@ static inline int rtw89_chip_enable_bb_rf(struct rtw89_dev *rtwdev) return chip->ops->enable_bb_rf(rtwdev); } -static inline void rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev) +static inline int rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev) { const struct rtw89_chip_info *chip = rtwdev->chip; - chip->ops->disable_bb_rf(rtwdev); + return chip->ops->disable_bb_rf(rtwdev); } u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev); @@ -911,6 +923,45 @@ static inline int rtw89_mac_txpwr_write32_mask(struct rtw89_dev *rtwdev, return 0; } +static inline void rtw89_mac_ctrl_hci_dma_tx(struct rtw89_dev *rtwdev, + bool enable) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + if (enable) + rtw89_write32_set(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_TXDMA_EN); + else + rtw89_write32_clr(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_TXDMA_EN); +} + +static inline void rtw89_mac_ctrl_hci_dma_rx(struct rtw89_dev *rtwdev, + bool enable) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + if (enable) + rtw89_write32_set(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_RXDMA_EN); + else + rtw89_write32_clr(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_RXDMA_EN); +} + +static inline void rtw89_mac_ctrl_hci_dma_trx(struct rtw89_dev *rtwdev, + bool enable) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + if (enable) + rtw89_write32_set(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN); + else + rtw89_write32_clr(rtwdev, chip->hci_func_en_addr, + B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN); +} + int rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, bool resume, u32 tx_time); int rtw89_mac_get_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta, @@ -944,8 +995,10 @@ enum rtw89_mac_xtal_si_offset { #define XTAL_SI_HIGH_ADDR_MASK GENMASK(2, 0) XTAL_SI_READ_VAL = 0x7A, XTAL_SI_WL_RFC_S0 = 0x80, +#define XTAL_SI_RF00S_EN GENMASK(2, 0) #define XTAL_SI_RF00 BIT(0) XTAL_SI_WL_RFC_S1 = 0x81, +#define XTAL_SI_RF10S_EN GENMASK(2, 0) #define XTAL_SI_RF10 BIT(0) XTAL_SI_ANAPAR_WL = 0x90, #define XTAL_SI_SRAM2RFC BIT(7) @@ -962,5 +1015,9 @@ enum rtw89_mac_xtal_si_offset { int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask); int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val); +void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); +u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd); +int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev, + struct rtw89_cpuio_ctrl *ctrl_para, bool wd); #endif diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index cef27e781ae244802b5da3c4d43eead5a32742be..a296bfa8188f2e775118829fb11a20880d56120c 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -3,6 +3,7 @@ */ #include "cam.h" +#include "chan.h" #include "coex.h" #include "debug.h" #include "fw.h" @@ -12,6 +13,7 @@ #include "reg.h" #include "sar.h" #include "ser.h" +#include "util.h" static void rtw89_ops_tx(struct ieee80211_hw *hw, struct ieee80211_tx_control *control, @@ -85,8 +87,11 @@ static int rtw89_ops_config(struct ieee80211_hw *hw, u32 changed) } } - if (changed & IEEE80211_CONF_CHANGE_CHANNEL) + if (changed & IEEE80211_CONF_CHANGE_CHANNEL) { + rtw89_config_entity_chandef(rtwdev, RTW89_SUB_ENTITY_0, + &hw->conf.chandef); rtw89_set_channel(rtwdev); + } if ((changed & IEEE80211_CONF_CHANGE_IDLE) && (hw->conf.flags & IEEE80211_CONF_IDLE)) @@ -104,6 +109,9 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw, struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; int ret = 0; + rtw89_debug(rtwdev, RTW89_DBG_STATE, "add vif %pM type %d, p2p %d\n", + vif->addr, vif->type, vif->p2p); + mutex_lock(&rtwdev->mutex); rtwvif->rtwdev = rtwdev; list_add_tail(&rtwvif->list, &rtwdev->rtwvifs_list); @@ -146,6 +154,9 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw, struct rtw89_dev *rtwdev = hw->priv; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + rtw89_debug(rtwdev, RTW89_DBG_STATE, "remove vif %pM type %d p2p %d\n", + vif->addr, vif->type, vif->p2p); + cancel_work_sync(&rtwvif->update_beacon_work); mutex_lock(&rtwdev->mutex); @@ -157,6 +168,23 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw, mutex_unlock(&rtwdev->mutex); } +static int rtw89_ops_change_interface(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + enum nl80211_iftype type, bool p2p) +{ + struct rtw89_dev *rtwdev = hw->priv; + + rtw89_debug(rtwdev, RTW89_DBG_STATE, "change vif %pM (%d)->(%d), p2p (%d)->(%d)\n", + vif->addr, vif->type, type, vif->p2p, p2p); + + rtw89_ops_remove_interface(hw, vif); + + vif->type = type; + vif->p2p = p2p; + + return rtw89_ops_add_interface(hw, vif); +} + static void rtw89_ops_configure_filter(struct ieee80211_hw *hw, unsigned int changed_flags, unsigned int *new_flags, @@ -235,11 +263,12 @@ static u8 rtw89_aifsn_to_aifs(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, u8 aifsn) { struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 slot_time; u8 sifs; slot_time = vif->bss_conf.use_short_slot ? 9 : 20; - sifs = rtwdev->hal.current_band_type == RTW89_BAND_5G ? 16 : 10; + sifs = chan->band_type == RTW89_BAND_5G ? 16 : 10; return aifsn * slot_time + sifs; } @@ -350,6 +379,7 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw, rtw89_phy_set_bss_color(rtwdev, vif); rtw89_chip_cfg_txpwr_ul_tb_offset(rtwdev, vif); rtw89_mac_port_update(rtwdev, rtwvif); + rtw89_mac_set_he_obss_narrow_bw_ru(rtwdev, vif); rtw89_store_op_chan(rtwdev, true); } else { /* Abort ongoing scan if cancel_scan isn't issued @@ -378,6 +408,9 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_MU_GROUPS) rtw89_mac_bf_set_gid_table(rtwdev, vif, conf); + if (changed & BSS_CHANGED_P2P_PS) + rtw89_process_p2p_ps(rtwdev, vif); + mutex_unlock(&rtwdev->mutex); } @@ -605,6 +638,20 @@ static void rtw89_ops_sta_statistics(struct ieee80211_hw *hw, sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } +static +void __rtw89_drop_packets(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif) +{ + struct rtw89_vif *rtwvif; + + if (vif) { + rtwvif = (struct rtw89_vif *)vif->drv_priv; + rtw89_mac_pkt_drop_vif(rtwdev, rtwvif); + } else { + rtw89_for_each_rtwvif(rtwdev, rtwvif) + rtw89_mac_pkt_drop_vif(rtwdev, rtwvif); + } +} + static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u32 queues, bool drop) { @@ -613,7 +660,12 @@ static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mutex_lock(&rtwdev->mutex); rtw89_leave_lps(rtwdev); rtw89_hci_flush_queues(rtwdev, queues, drop); - rtw89_mac_flush_txq(rtwdev, queues, drop); + + if (drop && RTW89_CHK_FW_FEATURE(PACKET_DROP, &rtwdev->fw)) + __rtw89_drop_packets(rtwdev, vif); + else + rtw89_mac_flush_txq(rtwdev, queues, drop); + mutex_unlock(&rtwdev->mutex); } @@ -629,7 +681,7 @@ static void rtw89_ra_mask_info_update_iter(void *data, struct ieee80211_sta *sta struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; struct ieee80211_vif *vif = rtwvif_to_vif(rtwsta->rtwvif); - if (vif != br_data->vif) + if (vif != br_data->vif || vif->p2p) return; rtwsta->use_cfg_mask = true; @@ -669,12 +721,13 @@ int rtw89_ops_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant) struct rtw89_dev *rtwdev = hw->priv; struct rtw89_hal *hal = &rtwdev->hal; - if (rx_ant != hw->wiphy->available_antennas_rx) + if (rx_ant != hw->wiphy->available_antennas_rx && rx_ant != hal->antenna_rx) return -EINVAL; mutex_lock(&rtwdev->mutex); hal->antenna_tx = tx_ant; hal->antenna_rx = rx_ant; + hal->tx_path_diversity = false; mutex_unlock(&rtwdev->mutex); return 0; @@ -772,6 +825,97 @@ static void rtw89_ops_sta_rc_update(struct ieee80211_hw *hw, rtw89_phy_ra_updata_sta(rtwdev, sta, changed); } +static int rtw89_ops_add_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_dev *rtwdev = hw->priv; + int ret; + + mutex_lock(&rtwdev->mutex); + ret = rtw89_chanctx_ops_add(rtwdev, ctx); + mutex_unlock(&rtwdev->mutex); + + return ret; +} + +static void rtw89_ops_remove_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_dev *rtwdev = hw->priv; + + mutex_lock(&rtwdev->mutex); + rtw89_chanctx_ops_remove(rtwdev, ctx); + mutex_unlock(&rtwdev->mutex); +} + +static void rtw89_ops_change_chanctx(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *ctx, + u32 changed) +{ + struct rtw89_dev *rtwdev = hw->priv; + + mutex_lock(&rtwdev->mutex); + rtw89_chanctx_ops_change(rtwdev, ctx, changed); + mutex_unlock(&rtwdev->mutex); +} + +static int rtw89_ops_assign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_dev *rtwdev = hw->priv; + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + int ret; + + mutex_lock(&rtwdev->mutex); + ret = rtw89_chanctx_ops_assign_vif(rtwdev, rtwvif, ctx); + mutex_unlock(&rtwdev->mutex); + + return ret; +} + +static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx_conf *ctx) +{ + struct rtw89_dev *rtwdev = hw->priv; + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + + mutex_lock(&rtwdev->mutex); + rtw89_chanctx_ops_unassign_vif(rtwdev, rtwvif, ctx); + mutex_unlock(&rtwdev->mutex); +} + +static void rtw89_set_tid_config_iter(void *data, struct ieee80211_sta *sta) +{ + struct cfg80211_tid_config *tid_config = data; + struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; + struct rtw89_dev *rtwdev = rtwsta->rtwvif->rtwdev; + + rtw89_core_set_tid_config(rtwdev, sta, tid_config); +} + +static int rtw89_ops_set_tid_config(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct cfg80211_tid_config *tid_config) +{ + struct rtw89_dev *rtwdev = hw->priv; + + mutex_lock(&rtwdev->mutex); + if (sta) + rtw89_core_set_tid_config(rtwdev, sta, tid_config); + else + ieee80211_iterate_stations_atomic(rtwdev->hw, + rtw89_set_tid_config_iter, + tid_config); + mutex_unlock(&rtwdev->mutex); + + return 0; +} + const struct ieee80211_ops rtw89_ops = { .tx = rtw89_ops_tx, .wake_tx_queue = rtw89_ops_wake_tx_queue, @@ -779,6 +923,7 @@ const struct ieee80211_ops rtw89_ops = { .stop = rtw89_ops_stop, .config = rtw89_ops_config, .add_interface = rtw89_ops_add_interface, + .change_interface = rtw89_ops_change_interface, .remove_interface = rtw89_ops_remove_interface, .configure_filter = rtw89_ops_configure_filter, .bss_info_changed = rtw89_ops_bss_info_changed, @@ -800,7 +945,13 @@ const struct ieee80211_ops rtw89_ops = { .reconfig_complete = rtw89_ops_reconfig_complete, .hw_scan = rtw89_ops_hw_scan, .cancel_hw_scan = rtw89_ops_cancel_hw_scan, + .add_chanctx = rtw89_ops_add_chanctx, + .remove_chanctx = rtw89_ops_remove_chanctx, + .change_chanctx = rtw89_ops_change_chanctx, + .assign_vif_chanctx = rtw89_ops_assign_vif_chanctx, + .unassign_vif_chanctx = rtw89_ops_unassign_vif_chanctx, .set_sar_specs = rtw89_ops_set_sar_specs, .sta_rc_update = rtw89_ops_sta_rc_update, + .set_tid_config = rtw89_ops_set_tid_config, }; EXPORT_SYMBOL(rtw89_ops); diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index c68fec9eb5a643b3de561663e2bab3076a721900..5f8e19639362dd118a6d82b64d011d6e7c66b5cf 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -169,6 +169,23 @@ static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev, return 0; } +static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable) +{ + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; + const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2; + + if (enable) { + rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask); + if (dma_stop2->addr) + rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask); + } else { + rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask); + if (dma_stop2->addr) + rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask); + } +} + static bool rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls, struct sk_buff *new, @@ -760,7 +777,8 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev) enable_intr: spin_lock_irqsave(&rtwpci->irq_lock, flags); - rtw89_chip_enable_intr(rtwdev, rtwpci); + if (likely(rtwpci->running)) + rtw89_chip_enable_intr(rtwdev, rtwpci); spin_unlock_irqrestore(&rtwpci->irq_lock, flags); return IRQ_HANDLED; } @@ -925,10 +943,12 @@ u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev, { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; + struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; u32 cnt; spin_lock_bh(&rtwpci->trx_lock); cnt = rtw89_pci_get_avail_txbd_num(tx_ring); + cnt = min(cnt, wd_ring->curr_num); spin_unlock_bh(&rtwpci->trx_lock); return cnt; @@ -1073,12 +1093,15 @@ static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop) static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs, bool drop) { + const struct rtw89_pci_info *info = rtwdev->pci_info; u8 i; for (i = 0; i < RTW89_TXCH_NUM; i++) { /* It may be unnecessary to flush FWCMD queue. */ if (i == RTW89_TXCH_CH12) continue; + if (info->tx_dma_ch_mask & BIT(i)) + continue; if (txchs & BIT(i)) __pci_flush_txch(rtwdev, i, drop); @@ -1357,6 +1380,7 @@ static const struct rtw89_pci_bd_ram bd_ram_table[RTW89_TXCH_NUM] = { static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_pci_tx_ring *tx_ring; struct rtw89_pci_rx_ring *rx_ring; struct rtw89_pci_dma_ring *bd_ring; @@ -1368,6 +1392,9 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev) int i; for (i = 0; i < RTW89_TXCH_NUM; i++) { + if (info->tx_dma_ch_mask & BIT(i)) + continue; + tx_ring = &rtwpci->tx_rings[i]; bd_ring = &tx_ring->bd_ring; bd_ram = &bd_ram_table[i]; @@ -1411,12 +1438,15 @@ static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev, static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; int txch; rtw89_pci_reset_trx_rings(rtwdev); spin_lock_bh(&rtwpci->trx_lock); for (txch = 0; txch < RTW89_TXCH_NUM; txch++) { + if (info->tx_dma_ch_mask & BIT(txch)) + continue; if (txch == RTW89_TXCH_CH12) { rtw89_pci_release_fwcmd(rtwdev, rtwpci, skb_queue_len(&rtwpci->h2c_queue), true); @@ -1604,33 +1634,41 @@ static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data) writel(data, rtwpci->mmap + addr); } -static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable) +static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable) { - enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; const struct rtw89_pci_info *info = rtwdev->pci_info; - u32 txhci_en = info->txhci_en_bit; - u32 rxhci_en = info->rxhci_en_bit; - if (enable) { - if (chip_id != RTL8852C) - rtw89_write32_clr(rtwdev, info->dma_stop1_reg, - B_AX_STOP_PCIEIO); - rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, - txhci_en | rxhci_en); - if (chip_id == RTL8852C) - rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, - B_AX_STOP_AXI_MST); + if (enable) + rtw89_write32_set(rtwdev, info->init_cfg_reg, + info->rxhci_en_bit | info->txhci_en_bit); + else + rtw89_write32_clr(rtwdev, info->init_cfg_reg, + info->rxhci_en_bit | info->txhci_en_bit); +} + +static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable) +{ + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + u32 reg, mask; + + if (chip_id == RTL8852C) { + reg = R_AX_HAXI_INIT_CFG1; + mask = B_AX_STOP_AXI_MST; } else { - if (chip_id != RTL8852C) - rtw89_write32_set(rtwdev, info->dma_stop1_reg, - B_AX_STOP_PCIEIO); - else - rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, - B_AX_STOP_AXI_MST); - if (chip_id == RTL8852C) - rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, - B_AX_STOP_AXI_MST); + reg = R_AX_PCIE_DMA_STOP1; + mask = B_AX_STOP_PCIEIO; } + + if (enable) + rtw89_write32_clr(rtwdev, reg, mask); + else + rtw89_write32_set(rtwdev, reg, mask); +} + +static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable) +{ + rtw89_pci_ctrl_dma_io(rtwdev, enable); + rtw89_pci_ctrl_dma_trx(rtwdev, enable); } static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit) @@ -1836,6 +1874,18 @@ __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate return 0; } +static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev) +{ + int ret; + + if (rtwdev->chip->chip_id != RTL8852B) + return 0; + + ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK, + PCIE_AUTOK_4, PCIE_PHY_GEN1); + return ret; +} + static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en) { enum rtw89_pcie_phy phy_rate; @@ -2049,7 +2099,7 @@ static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev) static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev) { - if (rtwdev->chip->chip_id != RTL8852A) + if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B) return; rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN); @@ -2234,19 +2284,19 @@ static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; u32 ret, check, dma_busy; - u32 dma_busy1 = info->dma_busy1_reg; + u32 dma_busy1 = info->dma_busy1.addr; u32 dma_busy2 = info->dma_busy2_reg; - check = B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY | - B_AX_ACH3_BUSY | B_AX_ACH4_BUSY | B_AX_ACH5_BUSY | - B_AX_ACH6_BUSY | B_AX_ACH7_BUSY | B_AX_CH8_BUSY | - B_AX_CH9_BUSY | B_AX_CH12_BUSY; + check = info->dma_busy1.mask; ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0, 10, 100, false, rtwdev, dma_busy1); if (ret) return ret; + if (!dma_busy2) + return 0; + check = B_AX_CH10_BUSY | B_AX_CH11_BUSY; ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0, @@ -2414,6 +2464,12 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) rtw89_pci_hci_ldo(rtwdev); rtw89_pci_dphy_delay(rtwdev); + ret = rtw89_pci_autok_x(rtwdev); + if (ret) { + rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret); + return ret; + } + ret = rtw89_pci_auto_refclk_cal(rtwdev, false); if (ret) { rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret); @@ -2432,7 +2488,7 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) rtw89_pci_set_dbg(rtwdev); rtw89_pci_set_keep_reg(rtwdev); - rtw89_write32_set(rtwdev, info->dma_stop1_reg, B_AX_STOP_WPDMA); + rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA); /* stop DMA activities */ rtw89_pci_ctrl_dma_all(rtwdev, false); @@ -2455,10 +2511,9 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) return ret; } - /* enable FW CMD queue to download firmware */ - rtw89_write32_set(rtwdev, info->dma_stop1_reg, B_AX_TX_STOP1_ALL); - rtw89_write32_clr(rtwdev, info->dma_stop1_reg, B_AX_STOP_CH12); - rtw89_write32_set(rtwdev, info->dma_stop2_reg, B_AX_TX_STOP2_ALL); + /* disable all channels except to FW CMD channel to download firmware */ + rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false); + rtw89_write32_clr(rtwdev, info->dma_stop1.addr, B_AX_STOP_CH12); /* start DMA activities */ rtw89_pci_ctrl_dma_all(rtwdev, true); @@ -2486,15 +2541,15 @@ int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en) if (rtw89_pci_ltr_is_err_reg_val(val)) return -EINVAL; - rtw89_write32_clr(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN); - rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_EN); + rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN | + B_AX_LTR_WD_NOEMP_CHK); rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK, PCI_LTR_SPC_500US); rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK, - PCI_LTR_IDLE_TIMER_800US); + PCI_LTR_IDLE_TIMER_3_2MS); rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28); rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28); - rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x88e088e0); + rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003); rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b); return 0; @@ -2571,11 +2626,10 @@ static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev) } /* enable DMA for all queues */ - rtw89_write32_clr(rtwdev, info->dma_stop1_reg, B_AX_TX_STOP1_ALL); - rtw89_write32_clr(rtwdev, info->dma_stop2_reg, B_AX_TX_STOP2_ALL); + rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true); /* Release PCI IO */ - rtw89_write32_clr(rtwdev, info->dma_stop1_reg, + rtw89_write32_clr(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO); return 0; @@ -2696,10 +2750,13 @@ static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_pci_tx_ring *tx_ring; int i; for (i = 0; i < RTW89_TXCH_NUM; i++) { + if (info->tx_dma_ch_mask & BIT(i)) + continue; tx_ring = &rtwpci->tx_rings[i]; rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring); rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring); @@ -2887,6 +2944,7 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_pci_tx_ring *tx_ring; u32 desc_size; u32 len; @@ -2894,6 +2952,8 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev, int ret; for (i = 0; i < RTW89_TXCH_NUM; i++) { + if (info->tx_dma_ch_mask & BIT(i)) + continue; tx_ring = &rtwpci->tx_rings[i]; desc_size = sizeof(struct rtw89_pci_tx_bd_32); len = RTW89_PCI_TXBD_NUM_MAX; @@ -3219,8 +3279,79 @@ static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev, pci_free_irq_vectors(pdev); } +static u16 gray_code_to_bin(u16 gray_code, u32 bit_num) +{ + u16 bin = 0, gray_bit; + u32 bit_idx; + + for (bit_idx = 0; bit_idx < bit_num; bit_idx++) { + gray_bit = (gray_code >> bit_idx) & 0x1; + if (bit_num - bit_idx > 1) + gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1; + bin |= (gray_bit << bit_idx); + } + + return bin; +} + +static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + u16 val16, filter_out_val; + u32 val, phy_offset; + int ret; + + if (rtwdev->chip->chip_id != RTL8852C) + return 0; + + val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK); + if (val == B_AX_ASPM_CTRL_L1) + return 0; + + ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val); + if (ret) + return ret; + + val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val); + if (val == RTW89_PCIE_GEN1_SPEED) { + phy_offset = R_RAC_DIRECT_OFFSET_G1; + } else if (val == RTW89_PCIE_GEN2_SPEED) { + phy_offset = R_RAC_DIRECT_OFFSET_G2; + val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT); + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, + val16 | B_PCIE_BIT_PINOUT_DIS); + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, + val16 & ~B_PCIE_BIT_RD_SEL); + + val16 = rtw89_read16_mask(rtwdev, + phy_offset + RAC_ANA1F * RAC_MULT, + FILTER_OUT_EQ_MASK); + val16 = gray_code_to_bin(val16, hweight16(val16)); + filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 * + RAC_MULT); + filter_out_val &= ~REG_FILTER_OUT_MASK; + filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16); + + rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT, + filter_out_val); + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT, + B_BAC_EQ_SEL); + rtw89_write16_set(rtwdev, + R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT, + B_PCIE_BIT_PSAVE); + } else { + return -EOPNOTSUPP; + } + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT, + B_PCIE_BIT_PSAVE); + + return 0; +} + static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable) { + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; int ret; if (rtw89_pci_disable_clkreq) @@ -3231,19 +3362,33 @@ static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable) if (ret) rtw89_err(rtwdev, "failed to set CLKREQ Delay\n"); - if (enable) - ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, - RTW89_PCIE_BIT_CLK); - else - ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1_CTRL, - RTW89_PCIE_BIT_CLK); - if (ret) - rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d", - enable ? "set" : "unset", ret); + if (chip_id == RTL8852A) { + if (enable) + ret = rtw89_pci_config_byte_set(rtwdev, + RTW89_PCIE_L1_CTRL, + RTW89_PCIE_BIT_CLK); + else + ret = rtw89_pci_config_byte_clr(rtwdev, + RTW89_PCIE_L1_CTRL, + RTW89_PCIE_BIT_CLK); + if (ret) + rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d", + enable ? "set" : "unset", ret); + } else if (chip_id == RTL8852C) { + rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL, + B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL); + if (enable) + rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL, + B_AX_CLK_REQ_N); + else + rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL, + B_AX_CLK_REQ_N); + } } static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable) { + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; u8 value = 0; int ret; @@ -3262,12 +3407,23 @@ static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable) if (ret) rtw89_err(rtwdev, "failed to read ASPM Delay\n"); - if (enable) - ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, - RTW89_PCIE_BIT_L1); - else - ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1_CTRL, - RTW89_PCIE_BIT_L1); + if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (enable) + ret = rtw89_pci_config_byte_set(rtwdev, + RTW89_PCIE_L1_CTRL, + RTW89_PCIE_BIT_L1); + else + ret = rtw89_pci_config_byte_clr(rtwdev, + RTW89_PCIE_L1_CTRL, + RTW89_PCIE_BIT_L1); + } else if (chip_id == RTL8852C) { + if (enable) + rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1, + B_AX_ASPM_CTRL_L1); + else + rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, + B_AX_ASPM_CTRL_L1); + } if (ret) rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d", enable ? "set" : "unset", ret); @@ -3328,17 +3484,34 @@ static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev) static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable) { + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; int ret; - if (enable) - ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_TIMER_CTRL, - RTW89_PCIE_BIT_L1SUB); - else - ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_TIMER_CTRL, - RTW89_PCIE_BIT_L1SUB); - if (ret) - rtw89_err(rtwdev, "failed to %s L1SS, ret=%d", - enable ? "set" : "unset", ret); + if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (enable) + ret = rtw89_pci_config_byte_set(rtwdev, + RTW89_PCIE_TIMER_CTRL, + RTW89_PCIE_BIT_L1SUB); + else + ret = rtw89_pci_config_byte_clr(rtwdev, + RTW89_PCIE_TIMER_CTRL, + RTW89_PCIE_BIT_L1SUB); + if (ret) + rtw89_err(rtwdev, "failed to %s L1SS, ret=%d", + enable ? "set" : "unset", ret); + } else if (chip_id == RTL8852C) { + ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1, + RTW89_PCIE_BIT_ASPM_L11 | + RTW89_PCIE_BIT_PCI_L11); + if (ret) + rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret); + if (enable) + rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, + B_AX_L1SUB_DISABLE); + else + rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1, + B_AX_L1SUB_DISABLE); + } } static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev) @@ -3360,26 +3533,6 @@ static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev) rtw89_pci_l1ss_set(rtwdev, true); } -static void rtw89_pci_ctrl_dma_all_pcie(struct rtw89_dev *rtwdev, u8 en) -{ - const struct rtw89_pci_info *info = rtwdev->pci_info; - u32 val32; - - if (en == MAC_AX_FUNC_EN) { - val32 = B_AX_STOP_PCIEIO; - rtw89_write32_clr(rtwdev, info->dma_stop1_reg, val32); - - val32 = B_AX_TXHCI_EN | B_AX_RXHCI_EN; - rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32); - } else { - val32 = B_AX_STOP_PCIEIO; - rtw89_write32_set(rtwdev, info->dma_stop1_reg, val32); - - val32 = B_AX_TXHCI_EN | B_AX_RXHCI_EN; - rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, val32); - } -} - static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev) { int ret = 0; @@ -3399,10 +3552,13 @@ static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev) static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) { - u32 val, dma_rst = 0; + u32 val; int ret; - rtw89_pci_ctrl_dma_all_pcie(rtwdev, MAC_AX_FUNC_DIS); + if (rtwdev->chip->chip_id == RTL8852C) + return 0; + + rtw89_pci_ctrl_dma_all(rtwdev, false); ret = rtw89_pci_poll_io_idle(rtwdev); if (ret) { val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); @@ -3410,12 +3566,10 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n", R_AX_DBG_ERR_FLAG, val); if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0) - dma_rst |= B_AX_HCI_TXDMA_EN; + rtw89_mac_ctrl_hci_dma_tx(rtwdev, false); if (val & B_AX_RX_STUCK) - dma_rst |= B_AX_HCI_RXDMA_EN; - val = rtw89_read32(rtwdev, R_AX_HCI_FUNC_EN); - rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val & ~dma_rst); - rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val | dma_rst); + rtw89_mac_ctrl_hci_dma_rx(rtwdev, false); + rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); ret = rtw89_pci_poll_io_idle(rtwdev); val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); rtw89_debug(rtwdev, RTW89_DBG_HCI, @@ -3426,18 +3580,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) return ret; } -static void rtw89_pci_ctrl_hci_dma_en(struct rtw89_dev *rtwdev, u8 en) -{ - u32 val32; - if (en == MAC_AX_FUNC_EN) { - val32 = B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN; - rtw89_write32_set(rtwdev, R_AX_HCI_FUNC_EN, val32); - } else { - val32 = B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN; - rtw89_write32_clr(rtwdev, R_AX_HCI_FUNC_EN, val32); - } -} static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev) { @@ -3457,15 +3600,18 @@ static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev) { u32 ret; - rtw89_pci_ctrl_hci_dma_en(rtwdev, MAC_AX_FUNC_DIS); - rtw89_pci_ctrl_hci_dma_en(rtwdev, MAC_AX_FUNC_EN); + if (rtwdev->chip->chip_id == RTL8852C) + return 0; + + rtw89_mac_ctrl_hci_dma_trx(rtwdev, false); + rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); rtw89_pci_clr_idx_all(rtwdev); ret = rtw89_pci_rst_bdram(rtwdev); if (ret) return ret; - rtw89_pci_ctrl_dma_all_pcie(rtwdev, MAC_AX_FUNC_EN); + rtw89_pci_ctrl_dma_all(rtwdev, true); return ret; } @@ -3535,14 +3681,20 @@ static int __maybe_unused rtw89_pci_suspend(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw89_dev *rtwdev = hw->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, - B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); - rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, - B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); + if (chip_id == RTL8852A || chip_id == RTL8852B) { + rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, + B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); + rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, + B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); + } else { + rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, + B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN); + } return 0; } @@ -3563,15 +3715,24 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); struct rtw89_dev *rtwdev = hw->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, - B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); - rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, - B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); + if (chip_id == RTL8852A || chip_id == RTL8852B) { + rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, + B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); + rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, + B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG); + } else { + rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, + B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN); + rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, + B_AX_SEL_REQ_ENTR_L1); + } rtw89_pci_l2_hci_ldo(rtwdev); + rtw89_pci_filter_out(rtwdev); rtw89_pci_link_cfg(rtwdev); rtw89_pci_l1ss_cfg(rtwdev); @@ -3614,27 +3775,23 @@ static const struct rtw89_hci_ops rtw89_pci_ops = { int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - struct ieee80211_hw *hw; struct rtw89_dev *rtwdev; const struct rtw89_driver_info *info; const struct rtw89_pci_info *pci_info; - int driver_data_size; int ret; - driver_data_size = sizeof(struct rtw89_dev) + sizeof(struct rtw89_pci); - hw = ieee80211_alloc_hw(driver_data_size, &rtw89_ops); - if (!hw) { + info = (const struct rtw89_driver_info *)id->driver_data; + + rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev, + sizeof(struct rtw89_pci), + info->chip); + if (!rtwdev) { dev_err(&pdev->dev, "failed to allocate hw\n"); return -ENOMEM; } - info = (const struct rtw89_driver_info *)id->driver_data; pci_info = info->bus.pci; - rtwdev = hw->priv; - rtwdev->hw = hw; - rtwdev->dev = &pdev->dev; - rtwdev->chip = info->chip; rtwdev->pci_info = info->bus.pci; rtwdev->hci.ops = &rtw89_pci_ops; rtwdev->hci.type = RTW89_HCI_TYPE_PCIE; @@ -3667,6 +3824,7 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_clear_resource; } + rtw89_pci_filter_out(rtwdev); rtw89_pci_link_cfg(rtwdev); rtw89_pci_l1ss_cfg(rtwdev); @@ -3696,7 +3854,7 @@ err_declaim_pci: err_core_deinit: rtw89_core_deinit(rtwdev); err_release_hw: - ieee80211_free_hw(hw); + rtw89_free_ieee80211_hw(rtwdev); return ret; } @@ -3715,7 +3873,7 @@ void rtw89_pci_remove(struct pci_dev *pdev) rtw89_pci_clear_resource(rtwdev, pdev); rtw89_pci_declaim_device(rtwdev, pdev); rtw89_core_deinit(rtwdev); - ieee80211_free_hw(hw); + rtw89_free_ieee80211_hw(rtwdev); } EXPORT_SYMBOL(rtw89_pci_remove); diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h index a118647213e35322fad5e97f26fa2d1fb2f497e2..179740607778a54a63c1c1cc3e9762e8a8e912cc 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.h +++ b/drivers/net/wireless/realtek/rtw89/pci.h @@ -11,11 +11,21 @@ #define MDIO_PG1_G1 1 #define MDIO_PG0_G2 2 #define MDIO_PG1_G2 3 +#define RAC_CTRL_PPR 0x00 +#define RAC_ANA0A 0x0A +#define B_BAC_EQ_SEL BIT(5) +#define RAC_ANA0C 0x0C +#define B_PCIE_BIT_PSAVE BIT(15) #define RAC_ANA10 0x10 +#define B_PCIE_BIT_PINOUT_DIS BIT(3) #define RAC_REG_REV2 0x1B #define BAC_CMU_EN_DLY_MASK GENMASK(15, 12) #define PCIE_DPHY_DLY_25US 0x1 #define RAC_ANA19 0x19 +#define B_PCIE_BIT_RD_SEL BIT(2) +#define RAC_REG_FLD_0 0x1D +#define BAC_AUTOK_N_MASK GENMASK(3, 2) +#define PCIE_AUTOK_4 0x3 #define RAC_ANA1F 0x1F #define RAC_ANA24 0x24 #define B_AX_DEGLITCH GENMASK(11, 8) @@ -45,9 +55,26 @@ #define B_AX_SEL_REQ_ENTR_L1 BIT(2) #define B_AX_SEL_REQ_EXIT_L1 BIT(0) +#define R_AX_PCIE_MIX_CFG_V1 0x300C +#define B_AX_ASPM_CTRL_L1 BIT(17) +#define B_AX_ASPM_CTRL_L0 BIT(16) +#define B_AX_ASPM_CTRL_MASK GENMASK(17, 16) +#define B_AX_XFER_PENDING_FW BIT(11) +#define B_AX_XFER_PENDING BIT(10) +#define B_AX_REQ_EXIT_L1 BIT(9) +#define B_AX_REQ_ENTR_L1 BIT(8) +#define B_AX_L1SUB_DISABLE BIT(0) + +#define R_AX_L1_CLK_CTRL 0x3010 +#define B_AX_CLK_REQ_N BIT(1) + #define R_AX_PCIE_BG_CLR 0x303C #define B_AX_BG_CLR_ASYNC_M3 BIT(4) +#define R_AX_PCIE_LAT_CTRL 0x3044 +#define B_AX_CLK_REQ_SEL_OPT BIT(1) +#define B_AX_CLK_REQ_SEL BIT(0) + #define R_AX_PCIE_IO_RCY_M1 0x3100 #define B_AX_PCIE_IO_RCY_P_M1 BIT(5) #define B_AX_PCIE_IO_RCY_WDT_P_M1 BIT(4) @@ -88,7 +115,10 @@ #define B_AX_PCIE_WDT_TIMER_S1_MASK GENMASK(31, 0) #define R_RAC_DIRECT_OFFSET_G1 0x3800 +#define FILTER_OUT_EQ_MASK GENMASK(14, 10) #define R_RAC_DIRECT_OFFSET_G2 0x3880 +#define REG_FILTER_OUT_MASK GENMASK(6, 2) +#define RAC_MULT 2 #define RTW89_PCI_WR_RETRY_CNT 20 @@ -383,6 +413,16 @@ #define B_AX_STOP_RPQ BIT(1) #define B_AX_STOP_RXQ BIT(0) #define B_AX_TX_STOP1_ALL GENMASK(18, 8) +#define B_AX_TX_STOP1_MASK (B_AX_STOP_ACH0 | B_AX_STOP_ACH1 | \ + B_AX_STOP_ACH2 | B_AX_STOP_ACH3 | \ + B_AX_STOP_ACH4 | B_AX_STOP_ACH5 | \ + B_AX_STOP_ACH6 | B_AX_STOP_ACH7 | \ + B_AX_STOP_CH8 | B_AX_STOP_CH9 | \ + B_AX_STOP_CH12) +#define B_AX_TX_STOP1_MASK_V1 (B_AX_STOP_ACH0 | B_AX_STOP_ACH1 | \ + B_AX_STOP_ACH2 | B_AX_STOP_ACH3 | \ + B_AX_STOP_CH8 | B_AX_STOP_CH9 | \ + B_AX_STOP_CH12) #define R_AX_PCIE_DMA_STOP2 0x1310 #define B_AX_STOP_CH11 BIT(1) @@ -431,6 +471,13 @@ #define B_AX_ACH0_BUSY BIT(8) #define B_AX_RPQ_BUSY BIT(1) #define B_AX_RXQ_BUSY BIT(0) +#define DMA_BUSY1_CHECK (B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY | \ + B_AX_ACH3_BUSY | B_AX_ACH4_BUSY | B_AX_ACH5_BUSY | \ + B_AX_ACH6_BUSY | B_AX_ACH7_BUSY | B_AX_CH8_BUSY | \ + B_AX_CH9_BUSY | B_AX_CH12_BUSY) +#define DMA_BUSY1_CHECK_V1 (B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY | \ + B_AX_ACH3_BUSY | B_AX_CH8_BUSY | B_AX_CH9_BUSY | \ + B_AX_CH12_BUSY) #define R_AX_PCIE_DMA_BUSY2 0x131C #define B_AX_CH11_BUSY BIT(1) @@ -505,6 +552,17 @@ #define RTW89_PCI_MULTITAG 8 /* PCIE CFG register */ +#define RTW89_PCIE_L1_STS_V1 0x80 +#define RTW89_BCFG_LINK_SPEED_MASK GENMASK(19, 16) +#define RTW89_PCIE_GEN1_SPEED 0x01 +#define RTW89_PCIE_GEN2_SPEED 0x02 +#define RTW89_PCIE_PHY_RATE 0x82 +#define RTW89_PCIE_PHY_RATE_MASK GENMASK(1, 0) +#define RTW89_PCIE_L1SS_STS_V1 0x0168 +#define RTW89_PCIE_BIT_ASPM_L11 BIT(3) +#define RTW89_PCIE_BIT_ASPM_L12 BIT(2) +#define RTW89_PCIE_BIT_PCI_L11 BIT(1) +#define RTW89_PCIE_BIT_PCI_L12 BIT(0) #define RTW89_PCIE_ASPM_CTRL 0x070F #define RTW89_L1DLY_MASK GENMASK(5, 3) #define RTW89_L0DLY_MASK GENMASK(2, 0) @@ -516,8 +574,7 @@ #define RTW89_PCIE_CLK_CTRL 0x0725 #define RTW89_PCIE_RST_MSTATE 0x0B48 #define RTW89_PCIE_BIT_CFG_RST_MSTATE BIT(0) -#define RTW89_PCIE_PHY_RATE 0x82 -#define RTW89_PCIE_PHY_RATE_MASK GENMASK(1, 0) + #define INTF_INTGRA_MINREF_V1 90 #define INTF_INTGRA_HOSTREF_V1 100 @@ -527,11 +584,6 @@ enum rtw89_pcie_phy { PCIE_PHY_GEN1_UNDEFINE = 0x7F, }; -enum mac_ax_func_sw { - MAC_AX_FUNC_DIS, - MAC_AX_FUNC_EN, -}; - enum rtw89_pcie_l0sdly { PCIE_L0SDLY_1US = 0, PCIE_L0SDLY_2US = 1, @@ -710,14 +762,15 @@ struct rtw89_pci_info { u32 max_tag_num_mask; u32 rxbd_rwptr_clr_reg; u32 txbd_rwptr_clr2_reg; - u32 dma_stop1_reg; - u32 dma_stop2_reg; - u32 dma_busy1_reg; + struct rtw89_reg_def dma_stop1; + struct rtw89_reg_def dma_stop2; + struct rtw89_reg_def dma_busy1; u32 dma_busy2_reg; u32 dma_busy3_reg; u32 rpwm_addr; u32 cpwm_addr; + u32 tx_dma_ch_mask; const struct rtw89_pci_bd_idx_addr *bd_idx_addr_low_power; const struct rtw89_pci_ch_dma_addr_set *dma_addr_set; diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index 1532c0a6bbc44512139b59773db0e22dc381a106..6a6bdc652e09e5a037ca0264e89fc3e471b13f3c 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -14,23 +14,14 @@ static u16 get_max_amsdu_len(struct rtw89_dev *rtwdev, const struct rtw89_ra_report *report) { - const struct rate_info *txrate = &report->txrate; u32 bit_rate = report->bit_rate; - u8 mcs; /* lower than ofdm, do not aggregate */ if (bit_rate < 550) return 1; - /* prevent hardware rate fallback to G mode rate */ - if (txrate->flags & RATE_INFO_FLAGS_MCS) - mcs = txrate->mcs & 0x07; - else if (txrate->flags & (RATE_INFO_FLAGS_VHT_MCS | RATE_INFO_FLAGS_HE_MCS)) - mcs = txrate->mcs; - else - mcs = 0; - - if (mcs <= 2) + /* avoid AMSDU for legacy rate */ + if (report->might_fallback_legacy) return 1; /* lower than 20M vht 2ss mcs8, make it small */ @@ -142,8 +133,8 @@ static u64 rtw89_phy_ra_mask_recover(u64 ra_mask, u64 ra_mask_bak) static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta) { - struct rtw89_hal *hal = &rtwdev->hal; struct ieee80211_sta *sta = rtwsta_to_sta(rtwsta); + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct cfg80211_bitrate_mask *mask = &rtwsta->mask; enum nl80211_band band; u64 cfg_mask; @@ -151,7 +142,7 @@ static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtw if (!rtwsta->use_cfg_mask) return -1; - switch (hal->current_band_type) { + switch (chan->band_type) { case RTW89_BAND_2G: band = NL80211_BAND_2GHZ; cfg_mask = u64_encode_bits(mask->control[NL80211_BAND_2GHZ].legacy, @@ -168,7 +159,7 @@ static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtw RA_MASK_OFDM_RATES); break; default: - rtw89_warn(rtwdev, "unhandled band type %d\n", hal->current_band_type); + rtw89_warn(rtwdev, "unhandled band type %d\n", chan->band_type); return -1; } @@ -202,6 +193,40 @@ static const u64 rtw89_ra_mask_he_rates[4] = {RA_MASK_HE_1SS_RATES, RA_MASK_HE_2SS_RATES, RA_MASK_HE_3SS_RATES, RA_MASK_HE_4SS_RATES}; +static void rtw89_phy_ra_gi_ltf(struct rtw89_dev *rtwdev, + struct rtw89_sta *rtwsta, + bool *fix_giltf_en, u8 *fix_giltf) +{ + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + struct cfg80211_bitrate_mask *mask = &rtwsta->mask; + u8 band = chan->band_type; + enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band); + u8 he_gi = mask->control[nl_band].he_gi; + u8 he_ltf = mask->control[nl_band].he_ltf; + + if (!rtwsta->use_cfg_mask) + return; + + if (he_ltf == 2 && he_gi == 2) { + *fix_giltf = RTW89_GILTF_LGI_4XHE32; + } else if (he_ltf == 2 && he_gi == 0) { + *fix_giltf = RTW89_GILTF_SGI_4XHE08; + } else if (he_ltf == 1 && he_gi == 1) { + *fix_giltf = RTW89_GILTF_2XHE16; + } else if (he_ltf == 1 && he_gi == 0) { + *fix_giltf = RTW89_GILTF_2XHE08; + } else if (he_ltf == 0 && he_gi == 1) { + *fix_giltf = RTW89_GILTF_1XHE16; + } else if (he_ltf == 0 && he_gi == 0) { + *fix_giltf = RTW89_GILTF_1XHE08; + } else { + *fix_giltf_en = false; + return; + } + + *fix_giltf_en = true; +} + static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta, bool csi) { @@ -209,6 +234,8 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif = rtwsta->rtwvif; struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern; struct rtw89_ra_info *ra = &rtwsta->ra; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + struct ieee80211_vif *vif = rtwvif_to_vif(rtwsta->rtwvif); const u64 *high_rate_masks = rtw89_ra_mask_ht_rates; u8 rssi = ewma_rssi_read(&rtwsta->avg_rssi); u64 ra_mask = 0; @@ -218,8 +245,10 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, u8 bw_mode = 0; u8 stbc_en = 0; u8 ldpc_en = 0; + u8 fix_giltf = 0; u8 i; bool sgi = false; + bool fix_giltf_en = false; memset(ra, 0, sizeof(*ra)); /* Set the ra mask from sta's capability */ @@ -234,6 +263,7 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[1] & IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD) ldpc_en = 1; + rtw89_phy_ra_gi_ltf(rtwdev, rtwsta, &fix_giltf_en, &fix_giltf); } else if (sta->deflink.vht_cap.vht_supported) { u16 mcs_map = le16_to_cpu(sta->deflink.vht_cap.vht_mcs.rx_mcs_map); @@ -260,13 +290,13 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, ldpc_en = 1; } - switch (rtwdev->hal.current_band_type) { + switch (chan->band_type) { case RTW89_BAND_2G: ra_mask |= sta->deflink.supp_rates[NL80211_BAND_2GHZ]; - if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] <= 0xf) + if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] & 0xf) mode |= RTW89_RA_MODE_CCK; - else - mode |= RTW89_RA_MODE_CCK | RTW89_RA_MODE_OFDM; + if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] & 0xff0) + mode |= RTW89_RA_MODE_OFDM; break; case RTW89_BAND_5G: ra_mask |= (u64)sta->deflink.supp_rates[NL80211_BAND_5GHZ] << 4; @@ -329,7 +359,7 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM) ra->dcm_cap = 1; - if (rate_pattern->enable) { + if (rate_pattern->enable && !vif->p2p) { ra_mask = rtw89_phy_ra_mask_cfg(rtwdev, rtwsta); ra_mask &= rate_pattern->ra_mask; mode = rate_pattern->ra_mode; @@ -343,6 +373,8 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev, ra->ss_num = min(sta->deflink.rx_nss, rtwdev->hal.tx_nss) - 1; ra->en_sgi = sgi; ra->ra_mask = ra_mask; + ra->fix_giltf_en = fix_giltf_en; + ra->fix_giltf = fix_giltf; if (!csi) return; @@ -416,6 +448,7 @@ void rtw89_phy_rate_pattern_vif(struct rtw89_dev *rtwdev, struct ieee80211_supported_band *sband; struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; struct rtw89_phy_rate_pattern next_pattern = {0}; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); static const u16 hw_rate_he[] = {RTW89_HW_RATE_HE_NSS1_MCS0, RTW89_HW_RATE_HE_NSS2_MCS0, RTW89_HW_RATE_HE_NSS3_MCS0, @@ -428,7 +461,7 @@ void rtw89_phy_rate_pattern_vif(struct rtw89_dev *rtwdev, RTW89_HW_RATE_MCS8, RTW89_HW_RATE_MCS16, RTW89_HW_RATE_MCS24}; - u8 band = rtwdev->hal.current_band_type; + u8 band = chan->band_type; enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band); u8 tx_nss = rtwdev->hal.tx_nss; u8 i; @@ -542,12 +575,12 @@ void rtw89_phy_ra_assoc(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta) } u8 rtw89_phy_get_txsc(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_bandwidth dbw) { - enum rtw89_bandwidth cbw = param->bandwidth; - u8 pri_ch = param->primary_chan; - u8 central_ch = param->center_chan; + enum rtw89_bandwidth cbw = chan->band_width; + u8 pri_ch = chan->primary_channel; + u8 central_ch = chan->channel; u8 txsc_idx = 0; u8 tmp = 0; @@ -1468,10 +1501,9 @@ EXPORT_SYMBOL(rtw89_phy_load_txpwr_byrate); (txpwr_rf) >> (__c->txpwr_factor_rf - __c->txpwr_factor_mac); \ }) -s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, +s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, u8 band, const struct rtw89_rate_desc *rate_desc) { - enum rtw89_band band = rtwdev->hal.current_band_type; s8 *byr; u8 idx; @@ -1538,11 +1570,10 @@ static u8 rtw89_channel_to_idx(struct rtw89_dev *rtwdev, u8 band, u8 channel) } } -s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, +s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band, u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch) { const struct rtw89_chip_info *chip = rtwdev->chip; - u8 band = rtwdev->hal.current_band_type; u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch); u8 regd = rtw89_regd_get(rtwdev, band); s8 lmt = 0, sar; @@ -1578,11 +1609,12 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, } EXPORT_SYMBOL(rtw89_phy_read_txpwr_limit); -#define __fill_txpwr_limit_nonbf_bf(ptr, bw, ntx, rs, ch) \ +#define __fill_txpwr_limit_nonbf_bf(ptr, band, bw, ntx, rs, ch) \ do { \ u8 __i; \ for (__i = 0; __i < RTW89_BF_NUM; __i++) \ ptr[__i] = rtw89_phy_read_txpwr_limit(rtwdev, \ + band, \ bw, ntx, \ rs, __i, \ (ch)); \ @@ -1590,64 +1622,75 @@ EXPORT_SYMBOL(rtw89_phy_read_txpwr_limit); static void rtw89_phy_fill_txpwr_limit_20m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit *lmt, - u8 ntx, u8 ch) + u8 band, u8 ntx, u8 ch) { - __fill_txpwr_limit_nonbf_bf(lmt->cck_20m, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->cck_20m, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_CCK, ch); - __fill_txpwr_limit_nonbf_bf(lmt->cck_40m, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->cck_40m, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_CCK, ch); - __fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_OFDM, ch); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch); } static void rtw89_phy_fill_txpwr_limit_40m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit *lmt, - u8 ntx, u8 ch, u8 pri_ch) + u8 band, u8 ntx, u8 ch, u8 pri_ch) { - __fill_txpwr_limit_nonbf_bf(lmt->cck_20m, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->cck_20m, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_CCK, ch - 2); - __fill_txpwr_limit_nonbf_bf(lmt->cck_40m, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->cck_40m, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_CCK, ch); - __fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_OFDM, pri_ch); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch); } static void rtw89_phy_fill_txpwr_limit_80m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit *lmt, - u8 ntx, u8 ch, u8 pri_ch) + u8 band, u8 ntx, u8 ch, u8 pri_ch) { s8 val_0p5_n[RTW89_BF_NUM]; s8 val_0p5_p[RTW89_BF_NUM]; u8 i; - __fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_OFDM, pri_ch); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 6); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 6); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 4); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 4); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], RTW89_CHANNEL_WIDTH_80, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], band, + RTW89_CHANNEL_WIDTH_80, ntx, RTW89_RS_MCS, ch); - __fill_txpwr_limit_nonbf_bf(val_0p5_n, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_0p5_n, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 4); - __fill_txpwr_limit_nonbf_bf(val_0p5_p, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_0p5_p, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 4); for (i = 0; i < RTW89_BF_NUM; i++) @@ -1656,7 +1699,7 @@ static void rtw89_phy_fill_txpwr_limit_80m(struct rtw89_dev *rtwdev, static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit *lmt, - u8 ntx, u8 ch, u8 pri_ch) + u8 band, u8 ntx, u8 ch, u8 pri_ch) { s8 val_0p5_n[RTW89_BF_NUM]; s8 val_0p5_p[RTW89_BF_NUM]; @@ -1665,60 +1708,75 @@ static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev, u8 i; /* fill ofdm section */ - __fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_OFDM, pri_ch); /* fill mcs 20m section */ - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 14); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 10); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 6); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch - 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[4], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[4], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 2); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[5], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[5], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 6); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[6], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[6], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 10); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[7], RTW89_CHANNEL_WIDTH_20, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[7], band, + RTW89_CHANNEL_WIDTH_20, ntx, RTW89_RS_MCS, ch + 14); /* fill mcs 40m section */ - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 12); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 4); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[2], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[2], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 4); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[3], RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[3], band, + RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 12); /* fill mcs 80m section */ - __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], RTW89_CHANNEL_WIDTH_80, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], band, + RTW89_CHANNEL_WIDTH_80, ntx, RTW89_RS_MCS, ch - 8); - __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[1], RTW89_CHANNEL_WIDTH_80, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[1], band, + RTW89_CHANNEL_WIDTH_80, ntx, RTW89_RS_MCS, ch + 8); /* fill mcs 160m section */ - __fill_txpwr_limit_nonbf_bf(lmt->mcs_160m, RTW89_CHANNEL_WIDTH_160, + __fill_txpwr_limit_nonbf_bf(lmt->mcs_160m, band, + RTW89_CHANNEL_WIDTH_160, ntx, RTW89_RS_MCS, ch); /* fill mcs 40m 0p5 section */ - __fill_txpwr_limit_nonbf_bf(val_0p5_n, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_0p5_n, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 4); - __fill_txpwr_limit_nonbf_bf(val_0p5_p, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_0p5_p, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 4); for (i = 0; i < RTW89_BF_NUM; i++) lmt->mcs_40m_0p5[i] = min_t(s8, val_0p5_n[i], val_0p5_p[i]); /* fill mcs 40m 2p5 section */ - __fill_txpwr_limit_nonbf_bf(val_2p5_n, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_2p5_n, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch - 8); - __fill_txpwr_limit_nonbf_bf(val_2p5_p, RTW89_CHANNEL_WIDTH_40, + __fill_txpwr_limit_nonbf_bf(val_2p5_p, band, RTW89_CHANNEL_WIDTH_40, ntx, RTW89_RS_MCS, ch + 8); for (i = 0; i < RTW89_BF_NUM; i++) @@ -1726,37 +1784,41 @@ static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev, } void rtw89_phy_fill_txpwr_limit(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, struct rtw89_txpwr_limit *lmt, u8 ntx) { - u8 pri_ch = rtwdev->hal.current_primary_channel; - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 band = chan->band_type; + u8 pri_ch = chan->primary_channel; + u8 ch = chan->channel; + u8 bw = chan->band_width; memset(lmt, 0, sizeof(*lmt)); switch (bw) { case RTW89_CHANNEL_WIDTH_20: - rtw89_phy_fill_txpwr_limit_20m(rtwdev, lmt, ntx, ch); + rtw89_phy_fill_txpwr_limit_20m(rtwdev, lmt, band, ntx, ch); break; case RTW89_CHANNEL_WIDTH_40: - rtw89_phy_fill_txpwr_limit_40m(rtwdev, lmt, ntx, ch, pri_ch); + rtw89_phy_fill_txpwr_limit_40m(rtwdev, lmt, band, ntx, ch, + pri_ch); break; case RTW89_CHANNEL_WIDTH_80: - rtw89_phy_fill_txpwr_limit_80m(rtwdev, lmt, ntx, ch, pri_ch); + rtw89_phy_fill_txpwr_limit_80m(rtwdev, lmt, band, ntx, ch, + pri_ch); break; case RTW89_CHANNEL_WIDTH_160: - rtw89_phy_fill_txpwr_limit_160m(rtwdev, lmt, ntx, ch, pri_ch); + rtw89_phy_fill_txpwr_limit_160m(rtwdev, lmt, band, ntx, ch, + pri_ch); break; } } EXPORT_SYMBOL(rtw89_phy_fill_txpwr_limit); -static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, +static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band, u8 ru, u8 ntx, u8 ch) { const struct rtw89_chip_info *chip = rtwdev->chip; - u8 band = rtwdev->hal.current_band_type; u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch); u8 regd = rtw89_regd_get(rtwdev, band); s8 lmt_ru = 0, sar; @@ -1794,85 +1856,106 @@ static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, static void rtw89_phy_fill_txpwr_limit_ru_20m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit_ru *lmt_ru, - u8 ntx, u8 ch) + u8 band, u8 ntx, u8 ch) { - lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch); - lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch); - lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch); } static void rtw89_phy_fill_txpwr_limit_ru_40m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit_ru *lmt_ru, - u8 ntx, u8 ch) + u8 band, u8 ntx, u8 ch) { - lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch - 2); - lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch + 2); - lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch - 2); - lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch + 2); - lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch - 2); - lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch + 2); } static void rtw89_phy_fill_txpwr_limit_ru_80m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit_ru *lmt_ru, - u8 ntx, u8 ch) + u8 band, u8 ntx, u8 ch) { - lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch - 6); - lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch - 2); - lmt_ru->ru26[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch + 2); - lmt_ru->ru26[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26, + lmt_ru->ru26[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU26, ntx, ch + 6); - lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch - 6); - lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch - 2); - lmt_ru->ru52[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch + 2); - lmt_ru->ru52[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52, + lmt_ru->ru52[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU52, ntx, ch + 6); - lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch - 6); - lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch - 2); - lmt_ru->ru106[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch + 2); - lmt_ru->ru106[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106, + lmt_ru->ru106[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, + RTW89_RU106, ntx, ch + 6); } static void rtw89_phy_fill_txpwr_limit_ru_160m(struct rtw89_dev *rtwdev, struct rtw89_txpwr_limit_ru *lmt_ru, - u8 ntx, u8 ch) + u8 band, u8 ntx, u8 ch) { static const int ofst[] = { -14, -10, -6, -2, 2, 6, 10, 14 }; int i; static_assert(ARRAY_SIZE(ofst) == RTW89_RU_SEC_NUM); for (i = 0; i < RTW89_RU_SEC_NUM; i++) { - lmt_ru->ru26[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, + lmt_ru->ru26[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, RTW89_RU26, ntx, ch + ofst[i]); - lmt_ru->ru52[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, + lmt_ru->ru52[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, RTW89_RU52, ntx, ch + ofst[i]); - lmt_ru->ru106[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, + lmt_ru->ru106[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band, RTW89_RU106, ntx, ch + ofst[i]); @@ -1880,26 +1963,32 @@ rtw89_phy_fill_txpwr_limit_ru_160m(struct rtw89_dev *rtwdev, } void rtw89_phy_fill_txpwr_limit_ru(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, struct rtw89_txpwr_limit_ru *lmt_ru, u8 ntx) { - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 band = chan->band_type; + u8 ch = chan->channel; + u8 bw = chan->band_width; memset(lmt_ru, 0, sizeof(*lmt_ru)); switch (bw) { case RTW89_CHANNEL_WIDTH_20: - rtw89_phy_fill_txpwr_limit_ru_20m(rtwdev, lmt_ru, ntx, ch); + rtw89_phy_fill_txpwr_limit_ru_20m(rtwdev, lmt_ru, band, ntx, + ch); break; case RTW89_CHANNEL_WIDTH_40: - rtw89_phy_fill_txpwr_limit_ru_40m(rtwdev, lmt_ru, ntx, ch); + rtw89_phy_fill_txpwr_limit_ru_40m(rtwdev, lmt_ru, band, ntx, + ch); break; case RTW89_CHANNEL_WIDTH_80: - rtw89_phy_fill_txpwr_limit_ru_80m(rtwdev, lmt_ru, ntx, ch); + rtw89_phy_fill_txpwr_limit_ru_80m(rtwdev, lmt_ru, band, ntx, + ch); break; case RTW89_CHANNEL_WIDTH_160: - rtw89_phy_fill_txpwr_limit_ru_160m(rtwdev, lmt_ru, ntx, ch); + rtw89_phy_fill_txpwr_limit_ru_160m(rtwdev, lmt_ru, band, ntx, + ch); break; } } @@ -1920,6 +2009,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) u8 mode, rate, bw, giltf, mac_id; u16 legacy_bitrate; bool valid; + u8 mcs = 0; mac_id = RTW89_GET_PHY_C2H_RA_RPT_MACID(c2h->data); if (mac_id != rtwsta->mac_id) @@ -1936,7 +2026,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) return; } - memset(ra_report, 0, sizeof(*ra_report)); + memset(&ra_report->txrate, 0, sizeof(ra_report->txrate)); switch (mode) { case RTW89_RA_RPT_MODE_LEGACY: @@ -1952,6 +2042,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) ra_report->txrate.mcs = rate; if (giltf) ra_report->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + mcs = ra_report->txrate.mcs & 0x07; break; case RTW89_RA_RPT_MODE_VHT: ra_report->txrate.flags |= RATE_INFO_FLAGS_VHT_MCS; @@ -1959,6 +2050,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) ra_report->txrate.nss = FIELD_GET(RTW89_RA_RATE_MASK_NSS, rate) + 1; if (giltf) ra_report->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; + mcs = ra_report->txrate.mcs; break; case RTW89_RA_RPT_MODE_HE: ra_report->txrate.flags |= RATE_INFO_FLAGS_HE_MCS; @@ -1970,6 +2062,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) ra_report->txrate.he_gi = NL80211_RATE_INFO_HE_GI_1_6; else ra_report->txrate.he_gi = NL80211_RATE_INFO_HE_GI_3_2; + mcs = ra_report->txrate.mcs; break; } @@ -1977,8 +2070,9 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta) ra_report->bit_rate = cfg80211_calculate_bitrate(&ra_report->txrate); ra_report->hw_rate = FIELD_PREP(RTW89_HW_RATE_MASK_MOD, mode) | FIELD_PREP(RTW89_HW_RATE_MASK_VAL, rate); - sta->max_rc_amsdu_len = get_max_amsdu_len(rtwdev, ra_report); - rtwsta->max_agg_wait = sta->max_rc_amsdu_len / 1500 - 1; + ra_report->might_fallback_legacy = mcs <= 2; + sta->deflink.agg.max_rc_amsdu_len = get_max_amsdu_len(rtwdev, ra_report); + rtwsta->max_agg_wait = sta->deflink.agg.max_rc_amsdu_len / 1500 - 1; } static void @@ -3247,10 +3341,11 @@ static void rtw89_phy_dig_update_rssi_info(struct rtw89_dev *rtwdev) static void rtw89_phy_dig_update_para(struct rtw89_dev *rtwdev) { struct rtw89_dig_info *dig = &rtwdev->dig; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); bool is_linked = rtwdev->total_sta_assoc > 0; const u16 *fa_th_src = NULL; - switch (rtwdev->hal.current_band_type) { + switch (chan->band_type) { case RTW89_BAND_2G: dig->lna_gain = dig->lna_gain_g; dig->tia_gain = dig->tia_gain_g; @@ -3410,26 +3505,32 @@ static void rtw89_phy_dig_igi_offset_by_env(struct rtw89_dev *rtwdev) static void rtw89_phy_dig_set_lna_idx(struct rtw89_dev *rtwdev, u8 lna_idx) { - rtw89_phy_write32_mask(rtwdev, R_PATH0_LNA_INIT, - B_PATH0_LNA_INIT_IDX_MSK, lna_idx); - rtw89_phy_write32_mask(rtwdev, R_PATH1_LNA_INIT, - B_PATH1_LNA_INIT_IDX_MSK, lna_idx); + const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs; + + rtw89_phy_write32_mask(rtwdev, dig_regs->p0_lna_init.addr, + dig_regs->p0_lna_init.mask, lna_idx); + rtw89_phy_write32_mask(rtwdev, dig_regs->p1_lna_init.addr, + dig_regs->p1_lna_init.mask, lna_idx); } static void rtw89_phy_dig_set_tia_idx(struct rtw89_dev *rtwdev, u8 tia_idx) { - rtw89_phy_write32_mask(rtwdev, R_PATH0_TIA_INIT, - B_PATH0_TIA_INIT_IDX_MSK, tia_idx); - rtw89_phy_write32_mask(rtwdev, R_PATH1_TIA_INIT, - B_PATH1_TIA_INIT_IDX_MSK, tia_idx); + const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs; + + rtw89_phy_write32_mask(rtwdev, dig_regs->p0_tia_init.addr, + dig_regs->p0_tia_init.mask, tia_idx); + rtw89_phy_write32_mask(rtwdev, dig_regs->p1_tia_init.addr, + dig_regs->p1_tia_init.mask, tia_idx); } static void rtw89_phy_dig_set_rxb_idx(struct rtw89_dev *rtwdev, u8 rxb_idx) { - rtw89_phy_write32_mask(rtwdev, R_PATH0_RXB_INIT, - B_PATH0_RXB_INIT_IDX_MSK, rxb_idx); - rtw89_phy_write32_mask(rtwdev, R_PATH1_RXB_INIT, - B_PATH1_RXB_INIT_IDX_MSK, rxb_idx); + const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs; + + rtw89_phy_write32_mask(rtwdev, dig_regs->p0_rxb_init.addr, + dig_regs->p0_rxb_init.mask, rxb_idx); + rtw89_phy_write32_mask(rtwdev, dig_regs->p1_rxb_init.addr, + dig_regs->p1_rxb_init.mask, rxb_idx); } static void rtw89_phy_dig_set_igi_cr(struct rtw89_dev *rtwdev, @@ -3443,21 +3544,19 @@ static void rtw89_phy_dig_set_igi_cr(struct rtw89_dev *rtwdev, set.lna_idx, set.tia_idx, set.rxb_idx); } -static const struct rtw89_reg_def sdagc_config[4] = { - {R_PATH0_P20_FOLLOW_BY_PAGCUGC, B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, - {R_PATH0_S20_FOLLOW_BY_PAGCUGC, B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, - {R_PATH1_P20_FOLLOW_BY_PAGCUGC, B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, - {R_PATH1_S20_FOLLOW_BY_PAGCUGC, B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, -}; - static void rtw89_phy_dig_sdagc_follow_pagc_config(struct rtw89_dev *rtwdev, bool enable) { - u8 i = 0; + const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs; - for (i = 0; i < ARRAY_SIZE(sdagc_config); i++) - rtw89_phy_write32_mask(rtwdev, sdagc_config[i].addr, - sdagc_config[i].mask, enable); + rtw89_phy_write32_mask(rtwdev, dig_regs->p0_p20_pagcugc_en.addr, + dig_regs->p0_p20_pagcugc_en.mask, enable); + rtw89_phy_write32_mask(rtwdev, dig_regs->p0_s20_pagcugc_en.addr, + dig_regs->p0_s20_pagcugc_en.mask, enable); + rtw89_phy_write32_mask(rtwdev, dig_regs->p1_p20_pagcugc_en.addr, + dig_regs->p1_p20_pagcugc_en.mask, enable); + rtw89_phy_write32_mask(rtwdev, dig_regs->p1_s20_pagcugc_en.addr, + dig_regs->p1_s20_pagcugc_en.mask, enable); rtw89_debug(rtwdev, RTW89_DBG_DIG, "sdagc_follow_pagc=%d\n", enable); } @@ -3483,7 +3582,9 @@ static void rtw89_phy_dig_config_igi(struct rtw89_dev *rtwdev) static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi, bool enable) { - enum rtw89_bandwidth cbw = rtwdev->hal.current_band_width; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs; + enum rtw89_bandwidth cbw = chan->band_width; struct rtw89_dig_info *dig = &rtwdev->dig; u8 final_rssi = 0, under_region = dig->pd_low_th_ofst; u8 ofdm_cca_th; @@ -3525,10 +3626,10 @@ static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi, "Dynamic PD th disabled, Set PD_low_bd=0\n"); } - rtw89_phy_write32_mask(rtwdev, R_SEG0R_PD, B_SEG0R_PD_LOWER_BOUND_MSK, - pd_val); - rtw89_phy_write32_mask(rtwdev, R_SEG0R_PD, - B_SEG0R_PD_SPATIAL_REUSE_EN_MSK, enable); + rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg, + dig_regs->pd_lower_bound_mask, pd_val); + rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg, + dig_regs->pd_spatial_reuse_en, enable); if (!rtwdev->hal.support_cckpd) return; @@ -3604,6 +3705,62 @@ void rtw89_phy_dig(struct rtw89_dev *rtwdev) rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, false); } +static void rtw89_phy_tx_path_div_sta_iter(void *data, struct ieee80211_sta *sta) +{ + struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv; + struct rtw89_dev *rtwdev = rtwsta->rtwdev; + struct rtw89_vif *rtwvif = rtwsta->rtwvif; + struct rtw89_hal *hal = &rtwdev->hal; + bool *done = data; + u8 rssi_a, rssi_b; + u32 candidate; + + if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION || sta->tdls) + return; + + if (*done) + return; + + *done = true; + + rssi_a = ewma_rssi_read(&rtwsta->rssi[RF_PATH_A]); + rssi_b = ewma_rssi_read(&rtwsta->rssi[RF_PATH_B]); + + if (rssi_a > rssi_b + RTW89_TX_DIV_RSSI_RAW_TH) + candidate = RF_A; + else if (rssi_b > rssi_a + RTW89_TX_DIV_RSSI_RAW_TH) + candidate = RF_B; + else + return; + + if (hal->antenna_tx == candidate) + return; + + hal->antenna_tx = candidate; + rtw89_fw_h2c_txpath_cmac_tbl(rtwdev, rtwsta); + + if (hal->antenna_tx == RF_A) { + rtw89_phy_write32_mask(rtwdev, R_P0_RFMODE, B_P0_RFMODE_MUX, 0x12); + rtw89_phy_write32_mask(rtwdev, R_P1_RFMODE, B_P1_RFMODE_MUX, 0x11); + } else if (hal->antenna_tx == RF_B) { + rtw89_phy_write32_mask(rtwdev, R_P0_RFMODE, B_P0_RFMODE_MUX, 0x11); + rtw89_phy_write32_mask(rtwdev, R_P1_RFMODE, B_P1_RFMODE_MUX, 0x12); + } +} + +void rtw89_phy_tx_path_div_track(struct rtw89_dev *rtwdev) +{ + struct rtw89_hal *hal = &rtwdev->hal; + bool done = false; + + if (!hal->tx_path_diversity) + return; + + ieee80211_iterate_stations_atomic(rtwdev->hw, + rtw89_phy_tx_path_div_sta_iter, + &done); +} + static void rtw89_phy_env_monitor_init(struct rtw89_dev *rtwdev) { rtw89_phy_ccx_top_setting_init(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw89/phy.h b/drivers/net/wireless/realtek/rtw89/phy.h index e20636f54b553283e8892a738b348beebaa98cd0..ee3bc5e111e1636c91207588169dceaa4eb979df 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.h +++ b/drivers/net/wireless/realtek/rtw89/phy.h @@ -56,7 +56,7 @@ #define CFO_TRK_STOP_TH (2 << 2) #define CFO_SW_COMP_FINE_TUNE (2 << 2) #define CFO_PERIOD_CNT 15 -#define CFO_BOUND 32 +#define CFO_BOUND 64 #define CFO_TP_UPPER 100 #define CFO_TP_LOWER 50 #define CFO_COMP_PERIOD 250 @@ -439,7 +439,7 @@ rtw89_rfk_parser(struct rtw89_dev *rtwdev, const struct rtw89_rfk_tbl *tbl); void rtw89_phy_write_reg3_tbl(struct rtw89_dev *rtwdev, const struct rtw89_phy_reg3_tbl *tbl); u8 rtw89_phy_get_txsc(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_bandwidth dbw); u32 rtw89_phy_read_rf(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path, u32 addr, u32 mask); @@ -460,15 +460,17 @@ void rtw89_phy_write32_idx(struct rtw89_dev *rtwdev, u32 addr, u32 mask, u32 data, enum rtw89_phy_idx phy_idx); void rtw89_phy_load_txpwr_byrate(struct rtw89_dev *rtwdev, const struct rtw89_txpwr_table *tbl); -s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, +s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, u8 band, const struct rtw89_rate_desc *rate_desc); void rtw89_phy_fill_txpwr_limit(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, struct rtw89_txpwr_limit *lmt, u8 ntx); void rtw89_phy_fill_txpwr_limit_ru(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, struct rtw89_txpwr_limit_ru *lmt_ru, u8 ntx); -s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, +s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band, u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch); void rtw89_phy_ra_assoc(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta); void rtw89_phy_ra_update(struct rtw89_dev *rtwdev); @@ -489,6 +491,7 @@ void rtw89_phy_set_phy_regs(struct rtw89_dev *rtwdev, u32 addr, u32 mask, u32 val); void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev); void rtw89_phy_dig(struct rtw89_dev *rtwdev); +void rtw89_phy_tx_path_div_track(struct rtw89_dev *rtwdev); void rtw89_phy_set_bss_color(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif); void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev, enum rtw89_mac_idx mac_idx, diff --git a/drivers/net/wireless/realtek/rtw89/ps.c b/drivers/net/wireless/realtek/rtw89/ps.c index a90b337205885a316f249d7a90b04407ed44e781..bf41a1141679291748b275f7a9bcb13b7e87c545 100644 --- a/drivers/net/wireless/realtek/rtw89/ps.c +++ b/drivers/net/wireless/realtek/rtw89/ps.c @@ -59,8 +59,11 @@ static void rtw89_ps_power_mode_change(struct rtw89_dev *rtwdev, bool enter) rtw89_mac_power_mode_change(rtwdev, enter); } -static void __rtw89_enter_ps_mode(struct rtw89_dev *rtwdev) +static void __rtw89_enter_ps_mode(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { + if (rtwvif->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT) + return; + if (!rtwdev->ps_mode) return; @@ -111,23 +114,23 @@ void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev) __rtw89_leave_ps_mode(rtwdev); } -void rtw89_enter_lps(struct rtw89_dev *rtwdev, u8 mac_id) +void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { lockdep_assert_held(&rtwdev->mutex); if (test_and_set_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags)) return; - __rtw89_enter_lps(rtwdev, mac_id); - __rtw89_enter_ps_mode(rtwdev); + __rtw89_enter_lps(rtwdev, rtwvif->mac_id); + __rtw89_enter_ps_mode(rtwdev, rtwvif); } static void rtw89_leave_lps_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { - if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION) + if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION && + rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT) return; - __rtw89_leave_ps_mode(rtwdev); __rtw89_leave_lps(rtwdev, rtwvif->mac_id); } @@ -140,6 +143,8 @@ void rtw89_leave_lps(struct rtw89_dev *rtwdev) if (!test_and_clear_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags)) return; + __rtw89_leave_ps_mode(rtwdev); + rtw89_for_each_rtwvif(rtwdev, rtwvif) rtw89_leave_lps_vif(rtwdev, rtwvif); } @@ -178,3 +183,64 @@ void rtw89_set_coex_ctrl_lps(struct rtw89_dev *rtwdev, bool btc_ctrl) if (btc_ctrl) rtw89_leave_lps(rtwdev); } + +static void rtw89_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif, + enum rtw89_p2pps_action act) +{ + if (act == RTW89_P2P_ACT_UPDATE || act == RTW89_P2P_ACT_REMOVE) + return; + + if (act == RTW89_P2P_ACT_INIT) + rtw89_fw_h2c_tsf32_toggle(rtwdev, rtwvif, true); + else if (act == RTW89_P2P_ACT_TERMINATE) + rtw89_fw_h2c_tsf32_toggle(rtwdev, rtwvif, false); +} + +static void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev, + struct ieee80211_vif *vif) +{ + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + enum rtw89_p2pps_action act; + u8 noa_id; + + if (rtwvif->last_noa_nr == 0) + return; + + for (noa_id = 0; noa_id < rtwvif->last_noa_nr; noa_id++) { + if (noa_id == rtwvif->last_noa_nr - 1) + act = RTW89_P2P_ACT_TERMINATE; + else + act = RTW89_P2P_ACT_REMOVE; + rtw89_tsf32_toggle(rtwdev, rtwvif, act); + rtw89_fw_h2c_p2p_act(rtwdev, vif, NULL, act, noa_id); + } +} + +static void rtw89_p2p_update_noa(struct rtw89_dev *rtwdev, + struct ieee80211_vif *vif) +{ + struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv; + struct ieee80211_p2p_noa_desc *desc; + enum rtw89_p2pps_action act; + u8 noa_id; + + for (noa_id = 0; noa_id < RTW89_P2P_MAX_NOA_NUM; noa_id++) { + desc = &vif->bss_conf.p2p_noa_attr.desc[noa_id]; + if (!desc->count || !desc->duration) + break; + + if (noa_id == 0) + act = RTW89_P2P_ACT_INIT; + else + act = RTW89_P2P_ACT_UPDATE; + rtw89_tsf32_toggle(rtwdev, rtwvif, act); + rtw89_fw_h2c_p2p_act(rtwdev, vif, desc, act, noa_id); + } + rtwvif->last_noa_nr = noa_id; +} + +void rtw89_process_p2p_ps(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif) +{ + rtw89_p2p_disable_all_noa(rtwdev, vif); + rtw89_p2p_update_noa(rtwdev, vif); +} diff --git a/drivers/net/wireless/realtek/rtw89/ps.h b/drivers/net/wireless/realtek/rtw89/ps.h index a184b68994aa42420b0e02f1894d7a41f4a3c2df..0feae39916238a6b17202b6446f0b57022737664 100644 --- a/drivers/net/wireless/realtek/rtw89/ps.h +++ b/drivers/net/wireless/realtek/rtw89/ps.h @@ -5,12 +5,13 @@ #ifndef __RTW89_PS_H_ #define __RTW89_PS_H_ -void rtw89_enter_lps(struct rtw89_dev *rtwdev, u8 mac_id); +void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif); void rtw89_leave_lps(struct rtw89_dev *rtwdev); void __rtw89_leave_ps_mode(struct rtw89_dev *rtwdev); void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev); void rtw89_enter_ips(struct rtw89_dev *rtwdev); void rtw89_leave_ips(struct rtw89_dev *rtwdev); void rtw89_set_coex_ctrl_lps(struct rtw89_dev *rtwdev, bool btc_ctrl); +void rtw89_process_p2p_ps(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif); #endif diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h index ebf28719d935e9be65805059758c8ddada150a9e..ca20bb024b407a9fe6bcea7d62eca4958367ca02 100644 --- a/drivers/net/wireless/realtek/rtw89/reg.h +++ b/drivers/net/wireless/realtek/rtw89/reg.h @@ -51,9 +51,6 @@ #define B_AX_EF_POR BIT(10) #define B_AX_EF_CELL_SEL_MASK GENMASK(9, 8) -#define R_AX_SPSLDO_ON_CTRL0 0x0200 -#define B_AX_OCP_L1_MASK GENMASK(15, 13) - #define R_AX_EFUSE_CTRL 0x0030 #define B_AX_EF_MODE_SEL_MASK GENMASK(31, 30) #define B_AX_EF_RDY BIT(29) @@ -143,6 +140,18 @@ #define R_AX_PMC_DBG_CTRL2 0x00CC #define B_AX_SYSON_DIS_PMCR_AX_WRMSK BIT(2) +#define R_AX_PCIE_MIO_INTF 0x00E4 +#define B_AX_PCIE_MIO_ADDR_PAGE_V1_MASK GENMASK(20, 16) +#define B_AX_PCIE_MIO_BYIOREG BIT(13) +#define B_AX_PCIE_MIO_RE BIT(12) +#define B_AX_PCIE_MIO_WE_MASK GENMASK(11, 8) +#define MIO_WRITE_BYTE_ALL 0xF +#define B_AX_PCIE_MIO_ADDR_MASK GENMASK(7, 0) +#define MIO_ADDR_PAGE_MASK GENMASK(12, 8) + +#define R_AX_PCIE_MIO_INTD 0x00E8 +#define B_AX_PCIE_MIO_DATA_MASK GENMASK(31, 0) + #define R_AX_SYS_CFG1 0x00F0 #define B_AX_CHIP_VER_MASK GENMASK(15, 12) @@ -191,6 +200,12 @@ #define R_AX_UDM2 0x01F8 #define R_AX_UDM3 0x01FC +#define R_AX_SPS_DIG_ON_CTRL0 0x0200 +#define B_AX_VREFPFM_L_MASK GENMASK(25, 22) +#define B_AX_REG_ZCDC_H_MASK GENMASK(18, 17) +#define B_AX_OCP_L1_MASK GENMASK(15, 13) +#define B_AX_VOL_L1_MASK GENMASK(3, 0) + #define R_AX_LDO_AON_CTRL0 0x0218 #define B_AX_PD_REGU_L BIT(16) @@ -383,6 +398,7 @@ #define R_AX_PHYREG_SET 0x8040 #define PHYREG_SET_ALL_CYCLE 0x8 +#define PHYREG_SET_XYN_CYCLE 0xE #define R_AX_HD0IMR 0x8110 #define B_AX_WDT_PTFM_INT_EN BIT(5) @@ -467,6 +483,7 @@ #define R_AX_LTR_CTRL_0 0x8410 #define B_AX_LTR_SPACE_IDX_MASK GENMASK(13, 12) #define B_AX_LTR_IDLE_TIMER_IDX_MASK GENMASK(10, 8) +#define B_AX_LTR_WD_NOEMP_CHK BIT(6) #define B_AX_APP_LTR_ACT BIT(5) #define B_AX_APP_LTR_IDLE BIT(4) #define B_AX_LTR_EN BIT(1) @@ -1024,15 +1041,13 @@ B_AX_WDE_DATCHN_NULLPG_ERR_INT_EN | \ B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN) #define B_AX_WDE_IMR_SET (B_AX_WDE_BUFREQ_QTAID_ERR_INT_EN | \ - B_AX_WDE_BUFREQ_SIZE0_INT_EN | \ - B_AX_WDE_BUFREQ_SIZELMT_INT_EN | \ - B_AX_WDE_BUFREQ_UNAVAL_ERR_INT_EN_V1 | \ - B_AX_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN_V1 | \ - B_AX_WDE_BUFRTN_SIZE_ERR_INT_EN_V1 | \ - B_AX_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN_V1 | \ - B_AX_WDE_GETNPG_STRPG_ERR_INT_EN_V1 | \ - B_AX_WDE_GETNPG_PGOFST_ERR_INT_EN_V1 | \ - B_AX_WDE_BUFMGN_FRZTO_ERR_INT_EN_V1 | \ + B_AX_WDE_BUFREQ_UNAVAL_ERR_INT_EN | \ + B_AX_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \ + B_AX_WDE_BUFRTN_SIZE_ERR_INT_EN | \ + B_AX_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \ + B_AX_WDE_GETNPG_STRPG_ERR_INT_EN | \ + B_AX_WDE_GETNPG_PGOFST_ERR_INT_EN | \ + B_AX_WDE_BUFMGN_FRZTO_ERR_INT_EN | \ B_AX_WDE_QUE_CMDTYPE_ERR_INT_EN | \ B_AX_WDE_QUE_DSTQUEID_ERR_INT_EN | \ B_AX_WDE_QUE_SRCQUEID_ERR_INT_EN | \ @@ -1043,10 +1058,7 @@ B_AX_WDE_QUEMGN_FRZTO_ERR_INT_EN | \ B_AX_WDE_DATCHN_ARBT_ERR_INT_EN | \ B_AX_WDE_DATCHN_NULLPG_ERR_INT_EN | \ - B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN | \ - B_AX_WDE_DATCHN_RRDY_ERR_INT_EN | \ - B_AX_WDE_DATCHN_ADRERR_ERR_INT_EN | \ - B_AX_WDE_DATCHN_CAMREQ_ERR_INT_EN) + B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN) #define B_AX_WDE_DATCHN_CAMREQ_ERR_INT_EN BIT(29) #define B_AX_WDE_DATCHN_ADRERR_ERR_INT_EN BIT(28) @@ -1826,6 +1838,13 @@ #define B_AX_TXSC_40M_MASK GENMASK(7, 4) #define B_AX_TXSC_20M_MASK GENMASK(3, 0) +#define R_AX_PTCL_RRSR1 0xC090 +#define R_AX_PTCL_RRSR1_C1 0xE090 +#define B_AX_RRSR_RATE_EN_MASK GENMASK(11, 8) +#define RRSR_OFDM_CCK_EN 3 +#define B_AX_RSC_MASK GENMASK(7, 6) +#define B_AX_RRSR_CCK_MASK GENMASK(3, 0) + #define R_AX_CMAC_ERR_IMR 0xC160 #define R_AX_CMAC_ERR_IMR_C1 0xE160 #define B_AX_WMAC_TX_ERR_IND_EN BIT(7) @@ -1882,6 +1901,7 @@ #define B_AX_SIFS_TIMEOUT_T2_MASK GENMASK(14, 8) #define B_AX_SIFS_MACTXEN_T1_MASK GENMASK(6, 0) #define SIFS_MACTXEN_T1 0x47 +#define SIFS_MACTXEN_T1_V1 0x41 #define R_AX_CCA_CFG_0 0xC340 #define R_AX_CCA_CFG_0_C1 0xE340 @@ -2098,6 +2118,8 @@ #define R_AX_TBTT_SHIFT_P3 0xC4E8 #define R_AX_TBTT_SHIFT_P4 0xC528 #define B_AX_TBTT_SHIFT_OFST_MASK GENMASK(11, 0) +#define B_AX_TBTT_SHIFT_OFST_SIGN BIT(11) +#define B_AX_TBTT_SHIFT_OFST_MAG GENMASK(10, 0) #define R_AX_BCN_CNT_TMR_P0 0xC434 #define R_AX_BCN_CNT_TMR_P1 0xC474 @@ -2258,6 +2280,7 @@ #define B_AX_F2PCMDRPT_FULL_DROP_ERR_INT_EN BIT(8) #define B_AX_FSM1_TIMEOUT_ERR_INT_EN BIT(1) #define B_AX_FSM_TIMEOUT_ERR_INT_EN BIT(0) +#define B_AX_PTCL_IMR_CLR_ALL GENMASK(31, 0) #define B_AX_PTCL_IMR_CLR (B_AX_FSM_TIMEOUT_ERR_INT_EN | \ B_AX_F2PCMDRPT_FULL_DROP_ERR_INT_EN | \ B_AX_TXPRT_FULL_DROP_ERR_INT_EN | \ @@ -2315,6 +2338,28 @@ #define B_AX_DLE_IMR_SET (B_AX_RXSTS_FSM_HANG_ERROR_IMR | \ B_AX_RXDATA_FSM_HANG_ERROR_IMR) +#define R_AX_RXDMA_CTRL_0 0xC804 +#define R_AX_RXDMA_CTRL_0_C1 0xE804 +#define B_AX_RXDMA_DBGOUT_EN BIT(31) +#define B_AX_RXDMA_DBG_SEL_MASK GENMASK(30, 29) +#define B_AX_RXDMA_FIFO_DBG_SEL_MASK GENMASK(28, 25) +#define B_AX_RXDMA_DEFAULT_PAGE_MASK GENMASK(22, 21) +#define B_AX_RXDMA_BUFF_REQ_PRI_MASK GENMASK(20, 19) +#define B_AX_RXDMA_TGT_QUEID_MASK GENMASK(18, 13) +#define B_AX_RXDMA_TGT_PRID_MASK GENMASK(12, 10) +#define B_AX_RXDMA_DIS_CSI_RELEASE BIT(9) +#define B_AX_RXDMA_DIS_RXSTS_WAIT_PTR_CLR BIT(7) +#define B_AX_RXDMA_DIS_CSI_WAIT_PTR_CLR BIT(6) +#define B_AX_RXSTS_PTR_FULL_MODE BIT(5) +#define B_AX_CSI_PTR_FULL_MODE BIT(4) +#define B_AX_RU3_PTR_FULL_MODE BIT(3) +#define B_AX_RU2_PTR_FULL_MODE BIT(2) +#define B_AX_RU1_PTR_FULL_MODE BIT(1) +#define B_AX_RU0_PTR_FULL_MODE BIT(0) +#define RX_FULL_MODE (B_AX_RU0_PTR_FULL_MODE | B_AX_RU1_PTR_FULL_MODE | \ + B_AX_RU2_PTR_FULL_MODE | B_AX_RU3_PTR_FULL_MODE | \ + B_AX_CSI_PTR_FULL_MODE | B_AX_RXSTS_PTR_FULL_MODE) + #define R_AX_RXDMA_PKT_INFO_0 0xC814 #define R_AX_RXDMA_PKT_INFO_1 0xC818 #define R_AX_RXDMA_PKT_INFO_2 0xC81C @@ -2553,6 +2598,20 @@ #define WMAC_SPEC_SIFS_OFDM_52C 0x11 #define WMAC_SPEC_SIFS_CCK 0xA +#define R_AX_TRXPTCL_RRSR_CTL_0 0xCC08 +#define R_AX_TRXPTCL_RRSR_CTL_0_C1 0xEC08 +#define B_AX_RESP_TX_MACID_CCA_TH_EN BIT(31) +#define B_AX_RESP_TX_PWRMODE_MASK GENMASK(30, 28) +#define B_AX_FTM_RRSR_RATE_EN_MASK GENMASK(27, 24) +#define B_AX_NESS_MASK GENMASK(23, 22) +#define B_AX_WMAC_RESP_DOPPLEB_AX_EN BIT(21) +#define B_AX_WMAC_RESP_DCM_EN BIT(20) +#define B_AX_WMAC_RRSB_AX_CCK_MASK GENMASK(19, 16) +#define B_AX_WMAC_RESP_RATE_EN_MASK GENMASK(15, 12) +#define B_AX_WMAC_RESP_RSC_MASK GENMASK(11, 10) +#define B_AX_WMAC_RESP_REF_RATE_SEL BIT(9) +#define B_AX_WMAC_RESP_REF_RATE_MASK GENMASK(8, 0) + #define R_AX_MAC_LOOPBACK 0xCC20 #define R_AX_MAC_LOOPBACK_C1 0xEC20 #define B_AX_MACLBK_EN BIT(0) @@ -2565,6 +2624,7 @@ #define B_AX_WMAC_TF_UP_NAV_EN BIT(16) #define B_AX_WMAC_NAV_UPPER_MASK GENMASK(15, 8) #define NAV_12MS 0xBC +#define NAV_25MS 0xC4 #define B_AX_WMAC_RTS_RST_DUR_MASK GENMASK(7, 0) #define R_AX_RXTRIG_TEST_USER_2 0xCCB0 @@ -2968,18 +3028,18 @@ #define R_AX_PATH_COM0 0xD800 #define AX_PATH_COM0_DFVAL 0x00000000 -#define AX_PATH_COM0_PATHA 0x08888880 -#define AX_PATH_COM0_PATHB 0x11111100 +#define AX_PATH_COM0_PATHA 0x08889880 +#define AX_PATH_COM0_PATHB 0x11111900 #define AX_PATH_COM0_PATHAB 0x19999980 #define R_AX_PATH_COM1 0xD804 #define AX_PATH_COM1_DFVAL 0x00000000 -#define AX_PATH_COM1_PATHA 0x11111111 -#define AX_PATH_COM1_PATHB 0x22222222 +#define AX_PATH_COM1_PATHA 0x13111111 +#define AX_PATH_COM1_PATHB 0x23222222 #define AX_PATH_COM1_PATHAB 0x33333333 #define R_AX_PATH_COM2 0xD808 #define AX_PATH_COM2_DFVAL 0x00000000 -#define AX_PATH_COM2_PATHA 0x01209111 -#define AX_PATH_COM2_PATHB 0x01209222 +#define AX_PATH_COM2_PATHA 0x01209313 +#define AX_PATH_COM2_PATHB 0x01209323 #define AX_PATH_COM2_PATHAB 0x01209333 #define R_AX_PATH_COM3 0xD80C #define AX_PATH_COM3_DFVAL 0x49249249 @@ -3125,6 +3185,18 @@ #define B_AX_GNT_WL_BB_VAL BIT(1) #define B_AX_GNT_WL_BB_SWCTRL BIT(0) +#define R_AX_GNT_VAL 0x0054 +#define B_AX_GNT_BT_RFC_S1_STA BIT(5) +#define B_AX_GNT_WL_RFC_S1_STA BIT(4) +#define B_AX_GNT_BT_RFC_S0_STA BIT(3) +#define B_AX_GNT_WL_RFC_S0_STA BIT(2) + +#define R_AX_GNT_VAL_V1 0xDA4C +#define B_AX_GNT_BT_RFC_S1 BIT(4) +#define B_AX_GNT_BT_RFC_S0 BIT(3) +#define B_AX_GNT_WL_RFC_S1 BIT(2) +#define B_AX_GNT_WL_RFC_S0 BIT(1) + #define R_AX_TDMA_MODE 0xDA4C #define R_AX_TDMA_MODE_C1 0xFA4C #define B_AX_R_BT_CMD_RPT_MASK GENMASK(31, 16) @@ -3356,6 +3428,7 @@ #define RR_DCK_FINE BIT(1) #define RR_DCK_LV BIT(0) #define RR_DCK1 0x93 +#define RR_DCK1_DONE BIT(5) #define RR_DCK1_CLR GENMASK(3, 0) #define RR_DCK1_SEL BIT(3) #define RR_DCK2 0x94 @@ -3431,8 +3504,9 @@ #define R_MAC_PIN_SEL 0x0734 #define B_CH_IDX_SEG0 GENMASK(23, 16) #define R_PLCP_HISTOGRAM 0x0738 -#define B_STS_DIS_TRIG_BY_BRK BIT(2) +#define B_STS_PARSING_TIME GENMASK(19, 16) #define B_STS_DIS_TRIG_BY_FAIL BIT(3) +#define B_STS_DIS_TRIG_BY_BRK BIT(2) #define R_PHY_STS_BITMAP_ADDR_START R_PHY_STS_BITMAP_SEARCH_FAIL #define B_PHY_STS_BITMAP_ADDR_MASK GENMASK(6, 2) #define R_PHY_STS_BITMAP_SEARCH_FAIL 0x073C @@ -3542,6 +3616,9 @@ #define B_P0_RXCK_VAL GENMASK(18, 16) #define B_P0_TXCK_ON BIT(15) #define B_P0_TXCK_VAL GENMASK(14, 12) +#define R_P0_RFMODE 0x12AC +#define B_P0_RFMODE_ORI_TXRX_FTM_TX GENMASK(31, 4) +#define B_P0_RFMODE_MUX GENMASK(11, 4) #define R_P0_NRBW 0x12B8 #define B_P0_NRBW_DBG BIT(30) #define R_S0_RXDC 0x12D4 @@ -3648,6 +3725,9 @@ #define B_P1_EN_SOUND_WO_NDP BIT(1) #define R_S1_HW_SI_DIS 0x3200 #define B_S1_HW_SI_DIS_W_R_TRIG GENMASK(30, 28) +#define R_P1_RFMODE 0x32AC +#define B_P1_RFMODE_ORI_TXRX_FTM_TX GENMASK(31, 4) +#define B_P1_RFMODE_MUX GENMASK(11, 4) #define R_P1_DBGMOD 0x32B8 #define B_P1_DBGMOD_ON BIT(30) #define R_S1_RXDC 0x32D4 @@ -3663,6 +3743,8 @@ #define R_S1_ADDCK 0x3E00 #define B_S1_ADDCK_I GENMASK(9, 0) #define B_S1_ADDCK_Q GENMASK(19, 10) +#define R_MUIC 0x40F8 +#define B_MUIC_EN BIT(0) #define R_DCFO 0x4264 #define B_DCFO GENMASK(1, 0) #define R_SEG0CSI 0x42AC @@ -3745,15 +3827,22 @@ #define R_PATH0_RXB_INIT 0x4658 #define B_PATH0_RXB_INIT_IDX_MSK GENMASK(9, 5) #define R_PATH0_LNA_INIT 0x4668 +#define R_PATH0_LNA_INIT_V1 0x472C #define B_PATH0_LNA_INIT_IDX_MSK GENMASK(26, 24) #define R_PATH0_BTG 0x466C #define B_PATH0_BTG_SHEN GENMASK(18, 17) #define R_PATH0_TIA_INIT 0x4674 #define B_PATH0_TIA_INIT_IDX_MSK BIT(17) #define R_PATH0_P20_FOLLOW_BY_PAGCUGC 0x46A0 +#define R_PATH0_P20_FOLLOW_BY_PAGCUGC_V1 0x4C24 +#define R_PATH0_P20_FOLLOW_BY_PAGCUGC_V2 0x46E8 #define B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5) #define R_PATH0_S20_FOLLOW_BY_PAGCUGC 0x46A4 +#define R_PATH0_S20_FOLLOW_BY_PAGCUGC_V1 0x4C28 +#define R_PATH0_S20_FOLLOW_BY_PAGCUGC_V2 0x46EC #define B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5) +#define R_PATH0_RXB_INIT_V1 0x46A8 +#define B_PATH0_RXB_INIT_IDX_MSK_V1 GENMASK(14, 10) #define R_PATH0_G_LNA6_OP1DB_V1 0x4688 #define B_PATH0_G_LNA6_OP1DB_V1 GENMASK(31, 24) #define R_PATH0_G_TIA0_LNA6_OP1DB_V1 0x4694 @@ -3780,7 +3869,10 @@ #define R_P0_AGC_CTL 0x4730 #define B_P0_AGC_EN BIT(31) #define R_PATH1_LNA_INIT 0x473C +#define R_PATH1_LNA_INIT_V1 0x4A80 #define B_PATH1_LNA_INIT_IDX_MSK GENMASK(26, 24) +#define R_PATH0_TIA_INIT_V1 0x473C +#define B_PATH0_TIA_INIT_IDX_MSK_V1 BIT(9) #define R_PATH1_TIA_INIT 0x4748 #define B_PATH1_TIA_INIT_IDX_MSK BIT(17) #define R_PATH1_BTG 0x4740 @@ -3790,8 +3882,12 @@ #define R_PATH1_G_LNA6_OP1DB_V1 0x476C #define B_PATH1_G_LNA6_OP1DB_V1 GENMASK(31, 24) #define R_PATH1_P20_FOLLOW_BY_PAGCUGC 0x4774 +#define R_PATH1_P20_FOLLOW_BY_PAGCUGC_V1 0x4CE8 +#define R_PATH1_P20_FOLLOW_BY_PAGCUGC_V2 0x47A8 #define B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5) #define R_PATH1_S20_FOLLOW_BY_PAGCUGC 0x4778 +#define R_PATH1_S20_FOLLOW_BY_PAGCUGC_V1 0x4CEC +#define R_PATH1_S20_FOLLOW_BY_PAGCUGC_V2 0x47AC #define B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5) #define R_PATH1_G_TIA0_LNA6_OP1DB_V1 0x4778 #define B_PATH1_G_TIA0_LNA6_OP1DB_V1 GENMASK(7, 0) @@ -3807,6 +3903,8 @@ #define B_P1_NBIIDX_VAL GENMASK(11, 0) #define B_P1_NBIIDX_NOTCH_EN BIT(12) #define R_SEG0R_PD 0x481C +#define R_SEG0R_PD_V1 0x4860 +#define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK_V1 BIT(30) #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK BIT(29) #define B_SEG0R_PD_LOWER_BOUND_MSK GENMASK(10, 6) #define R_2P4G_BAND 0x4970 @@ -3830,8 +3928,12 @@ #define B_BK_FC0_INV_MSK_V1 GENMASK(18, 0) #define R_CCK_FC0_INV_V1 0x4A20 #define B_CCK_FC0_INV_MSK_V1 GENMASK(18, 0) +#define R_PATH1_RXB_INIT_V1 0x4A5C +#define B_PATH1_RXB_INIT_IDX_MSK_V1 GENMASK(14, 10) #define R_P1_AGC_CTL 0x4A9C #define B_P1_AGC_EN BIT(31) +#define R_PATH1_TIA_INIT_V1 0x4AA8 +#define B_PATH1_TIA_INIT_IDX_MSK_V1 BIT(9) #define R_PATH0_RXBB_V1 0x4AD4 #define B_PATH0_RXBB_MSK_V1 GENMASK(31, 0) #define R_PATH1_RXBB_V1 0x4AE0 diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index 20c7afd3e70fe57aa327a25f2f024c6c62d1f67f..6e5a740b128f027ca4dfeefcaf133e3cad3a6dca 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -346,7 +346,7 @@ void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request rtw89_debug_regd(rtwdev, rtwdev->regd, "get from initiator %d, alpha2", request->initiator); - rtw89_chip_set_txpwr(rtwdev); + rtw89_core_set_chip_txpwr(rtwdev); exit: mutex_unlock(&rtwdev->mutex); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 81bd0c4fe21bcd619814cff552f630f6604f9e10..78414768035356bde880c2dfd44414b6fe467382 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -431,6 +431,7 @@ static const struct rtw89_imr_info rtw8852a_imr_info = { .cpu_disp_imr_set = B_AX_CPU_DISP_IMR_SET, .other_disp_imr_clr = B_AX_OTHER_DISP_IMR_CLR, .other_disp_imr_set = 0, + .bbrpt_com_err_imr_reg = R_AX_BBRPT_COM_ERR_IMR_ISR, .bbrpt_chinfo_err_imr_reg = R_AX_BBRPT_CHINFO_ERR_IMR_ISR, .bbrpt_err_imr_set = 0, .bbrpt_dfs_err_imr_reg = R_AX_BBRPT_DFS_ERR_IMR_ISR, @@ -453,6 +454,31 @@ static const struct rtw89_imr_info rtw8852a_imr_info = { .tmac_imr_set = B_AX_TMAC_IMR_SET, }; +static const struct rtw89_rrsr_cfgs rtw8852a_rrsr_cfgs = { + .ref_rate = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_REF_RATE_SEL, 0}, + .rsc = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_RSC_MASK, 2}, +}; + +static const struct rtw89_dig_regs rtw8852a_dig_regs = { + .seg0_pd_reg = R_SEG0R_PD, + .pd_lower_bound_mask = B_SEG0R_PD_LOWER_BOUND_MSK, + .pd_spatial_reuse_en = B_SEG0R_PD_SPATIAL_REUSE_EN_MSK, + .p0_lna_init = {R_PATH0_LNA_INIT, B_PATH0_LNA_INIT_IDX_MSK}, + .p1_lna_init = {R_PATH1_LNA_INIT, B_PATH1_LNA_INIT_IDX_MSK}, + .p0_tia_init = {R_PATH0_TIA_INIT, B_PATH0_TIA_INIT_IDX_MSK}, + .p1_tia_init = {R_PATH1_TIA_INIT, B_PATH1_TIA_INIT_IDX_MSK}, + .p0_rxb_init = {R_PATH0_RXB_INIT, B_PATH0_RXB_INIT_IDX_MSK}, + .p1_rxb_init = {R_PATH1_RXB_INIT, B_PATH1_RXB_INIT_IDX_MSK}, + .p0_p20_pagcugc_en = {R_PATH0_P20_FOLLOW_BY_PAGCUGC, + B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p0_s20_pagcugc_en = {R_PATH0_S20_FOLLOW_BY_PAGCUGC, + B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p1_p20_pagcugc_en = {R_PATH1_P20_FOLLOW_BY_PAGCUGC, + B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p1_s20_pagcugc_en = {R_PATH1_S20_FOLLOW_BY_PAGCUGC, + B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, +}; + static void rtw8852ae_efuse_parsing(struct rtw89_efuse *efuse, struct rtw8852a_efuse *map) { @@ -660,7 +686,7 @@ static void rtw8852a_power_trim(struct rtw89_dev *rtwdev) } static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, u8 mac_idx) { u32 rf_mod = rtw89_mac_reg_by_idx(R_AX_WMAC_RFMOD, mac_idx); @@ -669,20 +695,20 @@ static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev, u32 chk_rate = rtw89_mac_reg_by_idx(R_AX_TXRATE_CHK, mac_idx); u8 txsc20 = 0, txsc40 = 0; - switch (param->bandwidth) { + switch (chan->band_width) { case RTW89_CHANNEL_WIDTH_80: - txsc40 = rtw89_phy_get_txsc(rtwdev, param, + txsc40 = rtw89_phy_get_txsc(rtwdev, chan, RTW89_CHANNEL_WIDTH_40); fallthrough; case RTW89_CHANNEL_WIDTH_40: - txsc20 = rtw89_phy_get_txsc(rtwdev, param, + txsc20 = rtw89_phy_get_txsc(rtwdev, chan, RTW89_CHANNEL_WIDTH_20); break; default: break; } - switch (param->bandwidth) { + switch (chan->band_width) { case RTW89_CHANNEL_WIDTH_80: rtw89_write8_mask(rtwdev, rf_mod, B_AX_WMAC_RFMOD_MASK, BIT(1)); rtw89_write32(rtwdev, sub_carr, txsc20 | (txsc40 << 4)); @@ -699,7 +725,7 @@ static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev, break; } - if (param->center_chan > 14) + if (chan->channel > 14) rtw89_write8_set(rtwdev, chk_rate, B_AX_CHECK_CCK_EN | B_AX_RTS_LIMIT_IN_OFDM6); else @@ -1102,11 +1128,12 @@ static void rtw8852a_bb_sethw(struct rtw89_dev *rtwdev) if (rtwdev->hal.cv <= CHIP_CCV) { rtw89_phy_write32_set(rtwdev, R_RSTB_WATCH_DOG, B_P0_RSTB_WATCH_DOG); rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_1, 0x864FA000); - rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_2, 0x3F); + rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_2, 0x43F); rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_3, 0x7FFF); rtw89_phy_write32_set(rtwdev, R_SPOOF_ASYNC_RST, B_SPOOF_ASYNC_RST); rtw89_phy_write32_set(rtwdev, R_P0_TXPW_RSTB, B_P0_TXPW_RSTB_MANON); rtw89_phy_write32_set(rtwdev, R_P1_TXPW_RSTB, B_P1_TXPW_RSTB_MANON); + rtw89_phy_write32_set(rtwdev, R_PLCP_HISTOGRAM, B_STS_PARSING_TIME); } rtw89_phy_write32_mask(rtwdev, R_CFO_TRK0, B_CFO_TRK_MSK, 0x1f); rtw89_phy_write32_mask(rtwdev, R_CFO_TRK1, B_CFO_TRK_MSK, 0x0c); @@ -1130,35 +1157,38 @@ static void rtw8852a_bbrst_for_rfk(struct rtw89_dev *rtwdev, } static void rtw8852a_set_channel_bb(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - bool cck_en = param->center_chan <= 14; - u8 pri_ch_idx = param->pri_ch_idx; + bool cck_en = chan->channel <= 14; + u8 pri_ch_idx = chan->pri_ch_idx; if (cck_en) - rtw8852a_ctrl_sco_cck(rtwdev, param->center_chan, - param->primary_chan, param->bandwidth); + rtw8852a_ctrl_sco_cck(rtwdev, chan->channel, + chan->primary_channel, + chan->band_width); - rtw8852a_ctrl_ch(rtwdev, param->center_chan, phy_idx); - rtw8852a_ctrl_bw(rtwdev, pri_ch_idx, param->bandwidth, phy_idx); + rtw8852a_ctrl_ch(rtwdev, chan->channel, phy_idx); + rtw8852a_ctrl_bw(rtwdev, pri_ch_idx, chan->band_width, phy_idx); if (cck_en) { rtw89_phy_write32_mask(rtwdev, R_RXCCA, B_RXCCA_DIS, 0); } else { rtw89_phy_write32_mask(rtwdev, R_RXCCA, B_RXCCA_DIS, 1); rtw8852a_bbrst_for_rfk(rtwdev, phy_idx); } - rtw8852a_spur_elimination(rtwdev, param->center_chan); + rtw8852a_spur_elimination(rtwdev, chan->channel); rtw89_phy_write32_mask(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, - param->primary_chan); + chan->primary_channel); rtw8852a_bb_reset_all(rtwdev, phy_idx); } static void rtw8852a_set_channel(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *params) + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) { - rtw8852a_set_channel_mac(rtwdev, params, RTW89_MAC_0); - rtw8852a_set_channel_bb(rtwdev, params, RTW89_PHY_0); + rtw8852a_set_channel_mac(rtwdev, chan, mac_idx); + rtw8852a_set_channel_bb(rtwdev, chan, phy_idx); } static void rtw8852a_dfs_en(struct rtw89_dev *rtwdev, bool en) @@ -1209,25 +1239,27 @@ static void rtw8852a_adc_en(struct rtw89_dev *rtwdev, bool en) } static void rtw8852a_set_channel_help(struct rtw89_dev *rtwdev, bool enter, - struct rtw89_channel_help_params *p) + struct rtw89_channel_help_params *p, + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) { - u8 phy_idx = RTW89_PHY_0; - if (enter) { - rtw89_chip_stop_sch_tx(rtwdev, RTW89_MAC_0, &p->tx_en, RTW89_SCH_TX_SEL_ALL); - rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, false); + rtw89_chip_stop_sch_tx(rtwdev, mac_idx, &p->tx_en, + RTW89_SCH_TX_SEL_ALL); + rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, false); rtw8852a_dfs_en(rtwdev, false); - rtw8852a_tssi_cont_en_phyidx(rtwdev, false, RTW89_PHY_0); + rtw8852a_tssi_cont_en_phyidx(rtwdev, false, phy_idx); rtw8852a_adc_en(rtwdev, false); fsleep(40); rtw8852a_bb_reset_en(rtwdev, phy_idx, false); } else { - rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, true); + rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, true); rtw8852a_adc_en(rtwdev, true); rtw8852a_dfs_en(rtwdev, true); - rtw8852a_tssi_cont_en_phyidx(rtwdev, true, RTW89_PHY_0); + rtw8852a_tssi_cont_en_phyidx(rtwdev, true, phy_idx); rtw8852a_bb_reset_en(rtwdev, phy_idx, true); - rtw89_chip_resume_sch_tx(rtwdev, RTW89_MAC_0, p->tx_en); + rtw89_chip_resume_sch_tx(rtwdev, mac_idx, p->tx_en); } } @@ -1277,9 +1309,10 @@ static void rtw8852a_rfk_channel(struct rtw89_dev *rtwdev) rtw8852a_dpk(rtwdev, phy_idx); } -static void rtw8852a_rfk_band_changed(struct rtw89_dev *rtwdev) +static void rtw8852a_rfk_band_changed(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx) { - rtw8852a_tssi_scan(rtwdev, RTW89_PHY_0); + rtw8852a_tssi_scan(rtwdev, phy_idx); } static void rtw8852a_rfk_scan(struct rtw89_dev *rtwdev, bool start) @@ -1378,9 +1411,11 @@ static void rtw8852a_set_txpwr_ref(struct rtw89_dev *rtwdev, } static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - u8 ch = rtwdev->hal.current_channel; + u8 band = chan->band_type; + u8 ch = chan->channel; static const u8 rs[] = { RTW89_RS_CCK, RTW89_RS_OFDM, @@ -1406,7 +1441,8 @@ static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev, for (j = 0; j < rtw89_rs_idx_max[rs[i]]; j++) { cur.idx = j; shf = (j % 4) * 8; - tmp = rtw89_phy_read_txpwr_byrate(rtwdev, &cur); + tmp = rtw89_phy_read_txpwr_byrate(rtwdev, band, + &cur); val |= (tmp << shf); if ((j + 1) % 4) @@ -1421,8 +1457,10 @@ static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev, } static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { + u8 band = chan->band_type; struct rtw89_rate_desc desc = { .nss = RTW89_NSS_1, .rs = RTW89_RS_OFFSET, @@ -1433,7 +1471,7 @@ static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev, rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr offset\n"); for (desc.idx = 0; desc.idx < RTW89_RATE_OFFSET_MAX; desc.idx++) { - v = rtw89_phy_read_txpwr_byrate(rtwdev, &desc); + v = rtw89_phy_read_txpwr_byrate(rtwdev, band, &desc); val |= ((v & 0xf) << (4 * desc.idx)); } @@ -1442,29 +1480,31 @@ static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev, } static void rtw8852a_set_txpwr_limit(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { #define __MAC_TXPWR_LMT_PAGE_SIZE 40 - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 ch = chan->channel; + u8 bw = chan->band_width; struct rtw89_txpwr_limit lmt[NTX_NUM_8852A]; u32 addr, val; const s8 *ptr; - u8 i, j, k; + u8 i, j; rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr limit with ch=%d bw=%d\n", ch, bw); for (i = 0; i < NTX_NUM_8852A; i++) { - rtw89_phy_fill_txpwr_limit(rtwdev, &lmt[i], i); + rtw89_phy_fill_txpwr_limit(rtwdev, chan, &lmt[i], i); for (j = 0; j < __MAC_TXPWR_LMT_PAGE_SIZE; j += 4) { addr = R_AX_PWR_LMT + j + __MAC_TXPWR_LMT_PAGE_SIZE * i; ptr = (s8 *)&lmt[i] + j; - val = 0; - for (k = 0; k < 4; k++) - val |= (ptr[k] << (8 * k)); + val = FIELD_PREP(GENMASK(7, 0), ptr[0]) | + FIELD_PREP(GENMASK(15, 8), ptr[1]) | + FIELD_PREP(GENMASK(23, 16), ptr[2]) | + FIELD_PREP(GENMASK(31, 24), ptr[3]); rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val); } @@ -1473,30 +1513,32 @@ static void rtw8852a_set_txpwr_limit(struct rtw89_dev *rtwdev, } static void rtw8852a_set_txpwr_limit_ru(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { #define __MAC_TXPWR_LMT_RU_PAGE_SIZE 24 - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 ch = chan->channel; + u8 bw = chan->band_width; struct rtw89_txpwr_limit_ru lmt_ru[NTX_NUM_8852A]; u32 addr, val; const s8 *ptr; - u8 i, j, k; + u8 i, j; rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr limit ru with ch=%d bw=%d\n", ch, bw); for (i = 0; i < NTX_NUM_8852A; i++) { - rtw89_phy_fill_txpwr_limit_ru(rtwdev, &lmt_ru[i], i); + rtw89_phy_fill_txpwr_limit_ru(rtwdev, chan, &lmt_ru[i], i); for (j = 0; j < __MAC_TXPWR_LMT_RU_PAGE_SIZE; j += 4) { addr = R_AX_PWR_RU_LMT + j + __MAC_TXPWR_LMT_RU_PAGE_SIZE * i; ptr = (s8 *)&lmt_ru[i] + j; - val = 0; - for (k = 0; k < 4; k++) - val |= (ptr[k] << (8 * k)); + val = FIELD_PREP(GENMASK(7, 0), ptr[0]) | + FIELD_PREP(GENMASK(15, 8), ptr[1]) | + FIELD_PREP(GENMASK(23, 16), ptr[2]) | + FIELD_PREP(GENMASK(31, 24), ptr[3]); rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val); } @@ -1505,17 +1547,20 @@ static void rtw8852a_set_txpwr_limit_ru(struct rtw89_dev *rtwdev, #undef __MAC_TXPWR_LMT_RU_PAGE_SIZE } -static void rtw8852a_set_txpwr(struct rtw89_dev *rtwdev) +static void rtw8852a_set_txpwr(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, + enum rtw89_phy_idx phy_idx) { - rtw8852a_set_txpwr_byrate(rtwdev, RTW89_PHY_0); - rtw8852a_set_txpwr_limit(rtwdev, RTW89_PHY_0); - rtw8852a_set_txpwr_limit_ru(rtwdev, RTW89_PHY_0); + rtw8852a_set_txpwr_byrate(rtwdev, chan, phy_idx); + rtw8852a_set_txpwr_offset(rtwdev, chan, phy_idx); + rtw8852a_set_txpwr_limit(rtwdev, chan, phy_idx); + rtw8852a_set_txpwr_limit_ru(rtwdev, chan, phy_idx); } -static void rtw8852a_set_txpwr_ctrl(struct rtw89_dev *rtwdev) +static void rtw8852a_set_txpwr_ctrl(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx) { - rtw8852a_set_txpwr_ref(rtwdev, RTW89_PHY_0); - rtw8852a_set_txpwr_offset(rtwdev, RTW89_PHY_0); + rtw8852a_set_txpwr_ref(rtwdev, phy_idx); } static int @@ -1592,10 +1637,12 @@ void rtw8852a_bb_set_pmac_tx(struct rtw89_dev *rtwdev, struct rtw8852a_bb_pmac_info *tx_info, enum rtw89_phy_idx idx) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + if (!tx_info->en_pmac_tx) { rtw8852a_stop_pmac_tx(rtwdev, tx_info, idx); rtw89_phy_write32_idx(rtwdev, R_PD_CTRL, B_PD_HIT_DIS, 0, idx); - if (rtwdev->hal.current_band_type == RTW89_BAND_2G) + if (chan->band_type == RTW89_BAND_2G) rtw89_phy_write32_clr(rtwdev, R_RXCCA, B_RXCCA_DIS); return; } @@ -1797,6 +1844,9 @@ static void rtw8852a_btc_init_cfg(struct rtw89_dev *rtwdev) RF_PATH_A, BTC_BT_SS_GROUP, 0x5ff); rtw8852a_set_trx_mask(rtwdev, RF_PATH_B, BTC_BT_SS_GROUP, 0x5ff); + /* set path-A(S0) Tx/Rx no-mask if GNT_WL=0 && BT_S1=tx group */ + rtw8852a_set_trx_mask(rtwdev, + RF_PATH_A, BTC_BT_TX_GROUP, 0x5ff); } else { /* set WL Tx stb if GNT_WL = 0 && BT_S1 = ss group for 3-ant */ rtw8852a_set_trx_mask(rtwdev, RF_PATH_A, BTC_BT_SS_GROUP, 0x5df); @@ -2010,6 +2060,51 @@ void rtw8852a_btc_wl_s1_standby(struct rtw89_dev *rtwdev, bool state) rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); } +static void rtw8852a_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level) +{ + /* level=0 Default: TIA 1/0= (LNA2,TIAN6) = (7,1)/(5,1) = 21dB/12dB + * level=1 Fix LNA2=5: TIA 1/0= (LNA2,TIAN6) = (5,0)/(5,1) = 18dB/12dB + * To improve BT ACI in co-rx + */ + + switch (level) { + case 0: /* default */ + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); + break; + case 1: /* Fix LNA2=5 */ + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); + break; + } +} + +static void rtw8852a_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level) +{ + switch (level) { + case 0: /* original */ + rtw8852a_bb_ctrl_btc_preagc(rtwdev, false); + rtw8852a_set_wl_lna2(rtwdev, 0); + break; + case 1: /* for FDD free-run */ + rtw8852a_bb_ctrl_btc_preagc(rtwdev, true); + rtw8852a_set_wl_lna2(rtwdev, 0); + break; + case 2: /* for BTG Co-Rx*/ + rtw8852a_bb_ctrl_btc_preagc(rtwdev, false); + rtw8852a_set_wl_lna2(rtwdev, 1); + break; + } +} + static void rtw8852a_fill_freq_with_ppdu(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct ieee80211_rx_status *status) @@ -2030,12 +2125,12 @@ static void rtw8852a_query_ppdu(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *status) { u8 path; - s8 *rx_power = phy_ppdu->rssi; + u8 *rx_power = phy_ppdu->rssi; - status->signal = max_t(s8, rx_power[RF_PATH_A], rx_power[RF_PATH_B]); + status->signal = RTW89_RSSI_RAW_TO_DBM(max(rx_power[RF_PATH_A], rx_power[RF_PATH_B])); for (path = 0; path < rtwdev->chip->rf_path_num; path++) { status->chains |= BIT(path); - status->chain_signal[path] = rx_power[path]; + status->chain_signal[path] = RTW89_RSSI_RAW_TO_DBM(rx_power[path]); } if (phy_ppdu->valid) rtw8852a_fill_freq_with_ppdu(rtwdev, phy_ppdu, status); @@ -2086,6 +2181,8 @@ static const struct rtw89_chip_ops rtw8852a_chip_ops = { .btc_bt_aci_imp = rtw8852a_btc_bt_aci_imp, .btc_update_bt_cnt = rtw8852a_btc_update_bt_cnt, .btc_wl_s1_standby = rtw8852a_btc_wl_s1_standby, + .btc_set_wl_rx_gain = rtw8852a_btc_set_wl_rx_gain, + .btc_set_policy = rtw89_btc_set_policy, }; const struct rtw89_chip_info rtw8852a_chip_info = { @@ -2093,6 +2190,7 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .ops = &rtw8852a_chip_ops, .fw_name = "rtw89/rtw8852a_fw.bin", .fifo_size = 458752, + .dle_scc_rsvd_size = 0, .max_amsdu_limit = 3500, .dis_2g_40m_ul_ofdma = true, .rsvd_ple_ofst = 0x6f800, @@ -2114,7 +2212,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .txpwr_factor_rf = 2, .txpwr_factor_mac = 1, .dig_table = &rtw89_8852a_phy_dig_table, + .dig_regs = &rtw8852a_dig_regs, .tssi_dbw_table = NULL, + .support_chanctx_num = 1, .support_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ), .support_bw160 = false, @@ -2125,6 +2225,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .acam_num = 128, .bcam_num = 10, .scam_num = 128, + .bacam_num = 2, + .bacam_dynamic_num = 4, + .bacam_v1 = false, .sec_ctrl_efuse_size = 4, .physical_efuse_size = 1216, .logical_efuse_size = 1536, @@ -2133,11 +2236,26 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .dav_log_efuse_size = 0, .phycap_addr = 0x580, .phycap_size = 128, - .para_ver = 0x05050864, - .wlcx_desired = 0x05050000, - .btcx_desired = 0x5, + .para_ver = 0x0, + .wlcx_desired = 0x06000000, + .btcx_desired = 0x7, .scbd = 0x1, .mailbox = 0x1, + .btc_fwinfo_buf = 1024, + + .fcxbtcrpt_ver = 1, + .fcxtdma_ver = 1, + .fcxslots_ver = 1, + .fcxcysta_ver = 2, + .fcxstep_ver = 2, + .fcxnullsta_ver = 1, + .fcxmreg_ver = 1, + .fcxgpiodbg_ver = 1, + .fcxbtver_ver = 1, + .fcxbtscan_ver = 1, + .fcxbtafh_ver = 1, + .fcxbtdevinfo_ver = 1, + .afh_guard_ch = 6, .wl_rssi_thres = rtw89_btc_8852a_wl_rssi_thres, .bt_rssi_thres = rtw89_btc_8852a_bt_rssi_thres, @@ -2163,7 +2281,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = { .page_regs = &rtw8852a_page_regs, .dcfo_comp = &rtw8852a_dcfo_comp, .dcfo_comp_sft = 3, - .imr_info = &rtw8852a_imr_info + .imr_info = &rtw8852a_imr_info, + .rrsr_cfgs = &rtw8852a_rrsr_cfgs, + .dma_ch_mask = 0, }; EXPORT_SYMBOL(rtw8852a_chip_info); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c index 3d60feb78312145e8524ccdef5762a73502fdee2..582ff0d3a9ea08655e3f147d301940e63accb911 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c @@ -1359,7 +1359,7 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, u8 path) { struct rtw89_iqk_info *iqk_info = &rtwdev->iqk; - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u32 reg_rf18 = 0x0, reg_35c = 0x0; u8 idx = 0; u8 get_empty_table = false; @@ -1380,9 +1380,9 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev, rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]cfg ch = %d\n", reg_rf18); reg_35c = rtw89_phy_read32_mask(rtwdev, 0x35c, 0x00000c00); - iqk_info->iqk_band[path] = hal->current_band_type; - iqk_info->iqk_bw[path] = hal->current_band_width; - iqk_info->iqk_ch[path] = hal->current_channel; + iqk_info->iqk_band[path] = chan->band_type; + iqk_info->iqk_bw[path] = chan->band_width; + iqk_info->iqk_ch[path] = chan->channel; rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]iqk_info->iqk_band[%x] = 0x%x\n", path, @@ -1879,13 +1879,12 @@ static void _dpk_information(struct rtw89_dev *rtwdev, enum rtw89_rf_path path) { struct rtw89_dpk_info *dpk = &rtwdev->dpk; - struct rtw89_hal *hal = &rtwdev->hal; - + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 kidx = dpk->cur_idx[path]; - dpk->bp[path][kidx].band = hal->current_band_type; - dpk->bp[path][kidx].ch = hal->current_channel; - dpk->bp[path][kidx].bw = hal->current_band_width; + dpk->bp[path][kidx].band = chan->band_type; + dpk->bp[path][kidx].ch = chan->channel; + dpk->bp[path][kidx].bw = chan->band_width; rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] S%d[%d] (PHY%d): TSSI %s/ DBCC %s/ %s/ CH%d/ %s\n", @@ -2358,6 +2357,7 @@ static u8 _dpk_agc(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, #define DPK_RXBB_UPPER 0x1f #define DPK_RXBB_LOWER 0 #define DPK_GL_CRIT 7 + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); u8 tmp_txagc, tmp_rxbb = 0, tmp_gl_idx = 0; u8 agc_cnt = 0; bool limited_rxbb = false; @@ -2404,7 +2404,7 @@ static u8 _dpk_agc(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, "[DPK] Adjust RXBB (%d) = 0x%x\n", offset, tmp_rxbb); if (offset != 0 || agc_cnt == 0) { - if (rtwdev->hal.current_band_width < RTW89_CHANNEL_WIDTH_80) + if (chan->band_width < RTW89_CHANNEL_WIDTH_80) _dpk_bypass_rxcfir(rtwdev, path, true); else _dpk_lbk_rxiqk(rtwdev, phy, path); @@ -2548,11 +2548,12 @@ static bool _dpk_reload_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { struct rtw89_dpk_info *dpk = &rtwdev->dpk; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); bool is_reload = false; u8 idx, cur_band, cur_ch; - cur_band = rtwdev->hal.current_band_type; - cur_ch = rtwdev->hal.current_channel; + cur_band = chan->band_type; + cur_ch = chan->channel; for (idx = 0; idx < RTW89_DPK_BKUP_NUM; idx++) { if (cur_band != dpk->bp[path][idx].band || @@ -2681,12 +2682,13 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force, static bool _dpk_bypass_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { struct rtw89_fem_info *fem = &rtwdev->fem; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); - if (fem->epa_2g && rtwdev->hal.current_band_type == RTW89_BAND_2G) { + if (fem->epa_2g && chan->band_type == RTW89_BAND_2G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 2G_ext_PA exist!!\n"); return true; - } else if (fem->epa_5g && rtwdev->hal.current_band_type == RTW89_BAND_5G) { + } else if (fem->epa_5g && chan->band_type == RTW89_BAND_5G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 5G_ext_PA exist!!\n"); return true; @@ -2842,7 +2844,8 @@ static void _dpk_track(struct rtw89_dev *rtwdev) static void _tssi_rf_setting(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; if (band == RTW89_BAND_2G) rtw89_write_rf(rtwdev, path, RR_TXPOW, RR_TXPOW_TXG, 0x1); @@ -2852,7 +2855,8 @@ static void _tssi_rf_setting(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; rtw89_rfk_parser(rtwdev, &rtw8852a_tssi_sys_defs_tbl); rtw89_rfk_parser_by_cond(rtwdev, band == RTW89_BAND_2G, @@ -2863,7 +2867,8 @@ static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) static void _tssi_ini_txpwr_ctrl_bb(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; rtw89_rfk_parser_by_cond(rtwdev, path == RF_PATH_A, &rtw8852a_tssi_txpwr_ctrl_bb_defs_a_tbl, @@ -2905,8 +2910,9 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph __val; \ }) struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel; - u8 subband = rtwdev->hal.current_subband; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; + u8 subband = chan->subband_type; const s8 *thm_up_a = NULL; const s8 *thm_down_a = NULL; const s8 *thm_up_b = NULL; @@ -3099,7 +3105,8 @@ static void _tssi_set_txagc_offset_mv_avg(struct rtw89_dev *rtwdev, static void _tssi_pak(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - u8 subband = rtwdev->hal.current_subband; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 subband = chan->subband_type; switch (subband) { default: @@ -3275,7 +3282,8 @@ static s8 _tssi_get_ofdm_de(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; u32 gidx, gidx_1st, gidx_2nd; s8 de_1st = 0; s8 de_2nd = 0; @@ -3312,7 +3320,8 @@ static s8 _tssi_get_ofdm_trim_de(struct rtw89_dev *rtwdev, enum rtw89_rf_path path) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; u32 tgidx, tgidx_1st, tgidx_2nd; s8 tde_1st = 0; s8 tde_2nd = 0; @@ -3350,6 +3359,7 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev, { #define __DE_MASK 0x003ff000 struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); static const u32 r_cck_long[RF_PATH_NUM_8852A] = {0x5858, 0x7858}; static const u32 r_cck_short[RF_PATH_NUM_8852A] = {0x5860, 0x7860}; static const u32 r_mcs_20m[RF_PATH_NUM_8852A] = {0x5838, 0x7838}; @@ -3358,7 +3368,7 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev, static const u32 r_mcs_80m_80m[RF_PATH_NUM_8852A] = {0x5850, 0x7850}; static const u32 r_mcs_5m[RF_PATH_NUM_8852A] = {0x5828, 0x7828}; static const u32 r_mcs_10m[RF_PATH_NUM_8852A] = {0x5830, 0x7830}; - u8 ch = rtwdev->hal.current_channel; + u8 ch = chan->channel; u8 i, gidx; s8 ofdm_de; s8 trim_de; @@ -3478,9 +3488,11 @@ static void _tssi_track(struct rtw89_dev *rtwdev) static void _tssi_high_power(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel, ch_tmp; - u8 bw = rtwdev->hal.current_band_width; - u8 subband = rtwdev->hal.current_subband; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel, ch_tmp; + u8 bw = chan->band_width; + u8 band = chan->band_type; + u8 subband = chan->subband_type; s8 power; s32 xdbm; @@ -3491,7 +3503,7 @@ static void _tssi_high_power(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) else ch_tmp = ch; - power = rtw89_phy_read_txpwr_limit(rtwdev, bw, RTW89_1TX, + power = rtw89_phy_read_txpwr_limit(rtwdev, band, bw, RTW89_1TX, RTW89_RS_MCS, RTW89_NONBF, ch_tmp); xdbm = power * 100 / 4; @@ -3523,9 +3535,11 @@ static void _tssi_hw_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, static void _tssi_pre_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); const struct rtw89_chip_info *mac_reg = rtwdev->chip; - u8 ch = rtwdev->hal.current_channel, ch_tmp; - u8 bw = rtwdev->hal.current_band_width; + u8 ch = chan->channel, ch_tmp; + u8 bw = chan->band_width; + u8 band = chan->band_type; u32 tx_en; u8 phy_map = rtw89_btc_phymap(rtwdev, phy, 0); s8 power; @@ -3539,8 +3553,9 @@ static void _tssi_pre_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) else ch_tmp = ch; - power = rtw89_phy_read_txpwr_limit(rtwdev, RTW89_CHANNEL_WIDTH_20, RTW89_1TX, - RTW89_RS_OFDM, RTW89_NONBF, ch_tmp); + power = rtw89_phy_read_txpwr_limit(rtwdev, band, RTW89_CHANNEL_WIDTH_20, + RTW89_1TX, RTW89_RS_OFDM, + RTW89_NONBF, ch_tmp); xdbm = (power * 100) >> mac_reg->txpwr_factor_mac; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c index 190c4aefb02e3c994c78958cb3b4da54bdc2ac08..0cd8c0c44d19d4cede52e3aef991542c69b5e8ea 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c @@ -33,14 +33,15 @@ static const struct rtw89_pci_info rtw8852a_pci_info = { .max_tag_num_mask = B_AX_MAX_TAG_NUM, .rxbd_rwptr_clr_reg = R_AX_RXBD_RWPTR_CLR, .txbd_rwptr_clr2_reg = R_AX_TXBD_RWPTR_CLR2, - .dma_stop1_reg = R_AX_PCIE_DMA_STOP1, - .dma_stop2_reg = R_AX_PCIE_DMA_STOP2, - .dma_busy1_reg = R_AX_PCIE_DMA_BUSY1, + .dma_stop1 = {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK}, + .dma_stop2 = {R_AX_PCIE_DMA_STOP2, B_AX_TX_STOP2_ALL}, + .dma_busy1 = {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK}, .dma_busy2_reg = R_AX_PCIE_DMA_BUSY2, .dma_busy3_reg = R_AX_PCIE_DMA_BUSY1, .rpwm_addr = R_AX_PCIE_HRPWM, .cpwm_addr = R_AX_CPWM, + .tx_dma_ch_mask = 0, .bd_idx_addr_low_power = NULL, .dma_addr_set = &rtw89_pci_ch_dma_addr_set, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c new file mode 100644 index 0000000000000000000000000000000000000000..9f9908418ee4e3e296d6e6f99c4401f49670e3b0 --- /dev/null +++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright(c) 2019-2022 Realtek Corporation + */ + +#include "core.h" +#include "mac.h" +#include "reg.h" + +static const struct rtw89_dle_mem rtw8852b_dle_mem_pcie[] = { + [RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size6, + &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt6, + &rtw89_mac_size.wde_qt6, &rtw89_mac_size.ple_qt18, + &rtw89_mac_size.ple_qt58}, + [RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9, + &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4, + &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13, + &rtw89_mac_size.ple_qt13}, + [RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL, + NULL}, +}; + +static int rtw8852b_mac_enable_bb_rf(struct rtw89_dev *rtwdev) +{ + int ret; + + rtw89_write8_set(rtwdev, R_AX_SYS_FUNC_EN, + B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN); + rtw89_write32_mask(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_REG_ZCDC_H_MASK, 0x1); + rtw89_write32_set(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG); + rtw89_write32_clr(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG); + rtw89_write32_set(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG); + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, 0xC7, + FULL_BIT_MASK); + if (ret) + return ret; + + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, 0xC7, + FULL_BIT_MASK); + if (ret) + return ret; + + rtw89_write8(rtwdev, R_AX_PHYREG_SET, PHYREG_SET_XYN_CYCLE); + + return 0; +} + +static int rtw8852b_mac_disable_bb_rf(struct rtw89_dev *rtwdev) +{ + u8 wl_rfc_s0; + u8 wl_rfc_s1; + int ret; + + rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN, + B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN); + + ret = rtw89_mac_read_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, &wl_rfc_s0); + if (ret) + return ret; + wl_rfc_s0 &= ~XTAL_SI_RF00S_EN; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, wl_rfc_s0, + FULL_BIT_MASK); + if (ret) + return ret; + + ret = rtw89_mac_read_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, &wl_rfc_s1); + if (ret) + return ret; + wl_rfc_s1 &= ~XTAL_SI_RF10S_EN; + ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, wl_rfc_s1, + FULL_BIT_MASK); + return ret; +} + +static const struct rtw89_chip_ops rtw8852b_chip_ops = { + .enable_bb_rf = rtw8852b_mac_enable_bb_rf, + .disable_bb_rf = rtw8852b_mac_disable_bb_rf, +}; + +const struct rtw89_chip_info rtw8852b_chip_info = { + .chip_id = RTL8852B, + .fifo_size = 196608, + .dle_scc_rsvd_size = 98304, + .dle_mem = rtw8852b_dle_mem_pcie, + .dma_ch_mask = BIT(RTW89_DMA_ACH4) | BIT(RTW89_DMA_ACH5) | + BIT(RTW89_DMA_ACH6) | BIT(RTW89_DMA_ACH7) | + BIT(RTW89_DMA_B1MG) | BIT(RTW89_DMA_B1HI), +}; +EXPORT_SYMBOL(rtw8852b_chip_info); + +MODULE_FIRMWARE("rtw89/rtw8852b_fw.bin"); +MODULE_AUTHOR("Realtek Corporation"); +MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852B driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852be.c b/drivers/net/wireless/realtek/rtw89/rtw8852be.c new file mode 100644 index 0000000000000000000000000000000000000000..7bf95c38d3eb2e9314a4b91cc8894988b73c77f5 --- /dev/null +++ b/drivers/net/wireless/realtek/rtw89/rtw8852be.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright(c) 2020-2022 Realtek Corporation + */ + +#include +#include + +#include "pci.h" +#include "reg.h" + +static const struct rtw89_pci_info rtw8852b_pci_info = { + .dma_stop1 = {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK_V1}, + .dma_stop2 = {0}, + .dma_busy1 = {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK_V1}, + .dma_busy2_reg = 0, + .dma_busy3_reg = R_AX_PCIE_DMA_BUSY1, + + .tx_dma_ch_mask = BIT(RTW89_TXCH_ACH4) | BIT(RTW89_TXCH_ACH5) | + BIT(RTW89_TXCH_ACH6) | BIT(RTW89_TXCH_ACH7) | + BIT(RTW89_TXCH_CH10) | BIT(RTW89_TXCH_CH11), +}; + +MODULE_AUTHOR("Realtek Corporation"); +MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852BE driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c index b697aef2faf2d488a4b801f283311983e5dc176a..67653b3e1a356f4e50b186dae3f87b2788670e42 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c @@ -109,6 +109,7 @@ static const struct rtw89_imr_info rtw8852c_imr_info = { .cpu_disp_imr_set = B_AX_CPU_DISP_IMR_SET_V1, .other_disp_imr_clr = B_AX_OTHER_DISP_IMR_CLR_V1, .other_disp_imr_set = B_AX_OTHER_DISP_IMR_SET_V1, + .bbrpt_com_err_imr_reg = R_AX_BBRPT_COM_ERR_IMR, .bbrpt_chinfo_err_imr_reg = R_AX_BBRPT_CHINFO_ERR_IMR, .bbrpt_err_imr_set = R_AX_BBRPT_CHINFO_IMR_SET_V1, .bbrpt_dfs_err_imr_reg = R_AX_BBRPT_DFS_ERR_IMR, @@ -131,7 +132,34 @@ static const struct rtw89_imr_info rtw8852c_imr_info = { .tmac_imr_set = B_AX_TMAC_IMR_SET_V1, }; +static const struct rtw89_rrsr_cfgs rtw8852c_rrsr_cfgs = { + .ref_rate = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_REF_RATE_SEL, 0}, + .rsc = {R_AX_PTCL_RRSR1, B_AX_RSC_MASK, 2}, +}; + +static const struct rtw89_dig_regs rtw8852c_dig_regs = { + .seg0_pd_reg = R_SEG0R_PD, + .pd_lower_bound_mask = B_SEG0R_PD_LOWER_BOUND_MSK, + .pd_spatial_reuse_en = B_SEG0R_PD_SPATIAL_REUSE_EN_MSK, + .p0_lna_init = {R_PATH0_LNA_INIT_V1, B_PATH0_LNA_INIT_IDX_MSK}, + .p1_lna_init = {R_PATH1_LNA_INIT_V1, B_PATH1_LNA_INIT_IDX_MSK}, + .p0_tia_init = {R_PATH0_TIA_INIT_V1, B_PATH0_TIA_INIT_IDX_MSK_V1}, + .p1_tia_init = {R_PATH1_TIA_INIT_V1, B_PATH1_TIA_INIT_IDX_MSK_V1}, + .p0_rxb_init = {R_PATH0_RXB_INIT_V1, B_PATH0_RXB_INIT_IDX_MSK_V1}, + .p1_rxb_init = {R_PATH1_RXB_INIT_V1, B_PATH1_RXB_INIT_IDX_MSK_V1}, + .p0_p20_pagcugc_en = {R_PATH0_P20_FOLLOW_BY_PAGCUGC_V1, + B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p0_s20_pagcugc_en = {R_PATH0_S20_FOLLOW_BY_PAGCUGC_V1, + B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p1_p20_pagcugc_en = {R_PATH1_P20_FOLLOW_BY_PAGCUGC_V1, + B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK}, + .p1_s20_pagcugc_en = {R_PATH1_S20_FOLLOW_BY_PAGCUGC_V1, + B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK}, +}; + static void rtw8852c_ctrl_btg(struct rtw89_dev *rtwdev, bool btg); +static void rtw8852c_ctrl_tx_path_tmac(struct rtw89_dev *rtwdev, u8 tx_path, + enum rtw89_mac_idx mac_idx); static int rtw8852c_pwr_on_func(struct rtw89_dev *rtwdev) { @@ -567,7 +595,7 @@ static void rtw8852c_power_trim(struct rtw89_dev *rtwdev) } static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, u8 mac_idx) { u32 rf_mod = rtw89_mac_reg_by_idx(R_AX_WMAC_RFMOD, mac_idx); @@ -578,24 +606,24 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev, u8 rf_mod_val = 0, chk_rate_mask = 0; u32 txsc; - switch (param->bandwidth) { + switch (chan->band_width) { case RTW89_CHANNEL_WIDTH_160: - txsc80 = rtw89_phy_get_txsc(rtwdev, param, + txsc80 = rtw89_phy_get_txsc(rtwdev, chan, RTW89_CHANNEL_WIDTH_80); fallthrough; case RTW89_CHANNEL_WIDTH_80: - txsc40 = rtw89_phy_get_txsc(rtwdev, param, + txsc40 = rtw89_phy_get_txsc(rtwdev, chan, RTW89_CHANNEL_WIDTH_40); fallthrough; case RTW89_CHANNEL_WIDTH_40: - txsc20 = rtw89_phy_get_txsc(rtwdev, param, + txsc20 = rtw89_phy_get_txsc(rtwdev, chan, RTW89_CHANNEL_WIDTH_20); break; default: break; } - switch (param->bandwidth) { + switch (chan->band_width) { case RTW89_CHANNEL_WIDTH_160: rf_mod_val = AX_WMAC_RFMOD_160M; txsc = FIELD_PREP(B_AX_TXSC_20M_MASK, txsc20) | @@ -620,7 +648,7 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev, rtw89_write8_mask(rtwdev, rf_mod, B_AX_WMAC_RFMOD_MASK, rf_mod_val); rtw89_write32(rtwdev, sub_carr, txsc); - switch (param->band_type) { + switch (chan->band_type) { case RTW89_BAND_2G: chk_rate_mask = B_AX_BAND_MODE; break; @@ -629,7 +657,7 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev, chk_rate_mask = B_AX_CHECK_CCK_EN | B_AX_RTS_LIMIT_IN_OFDM6; break; default: - rtw89_warn(rtwdev, "Invalid band_type:%d\n", param->band_type); + rtw89_warn(rtwdev, "Invalid band_type:%d\n", chan->band_type); return; } rtw89_write8_clr(rtwdev, chk_rate, B_AX_BAND_MODE | B_AX_CHECK_CCK_EN | @@ -920,7 +948,7 @@ static void rtw8852c_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx, } static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev, - const struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx, enum rtw89_rf_path path) { @@ -939,7 +967,7 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev, if (rtwdev->dbcc_en && path == RF_PATH_B) phy_idx = RTW89_PHY_1; - if (param->band_type == RTW89_BAND_2G) { + if (chan->band_type == RTW89_BAND_2G) { offset_q0 = efuse_gain->offset[path][RTW89_GAIN_OFFSET_2G_CCK]; offset_base_q4 = efuse_gain->offset_base[phy_idx]; @@ -948,7 +976,7 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev, rtw89_phy_write32_mask(rtwdev, R_RPL_OFST, B_RPL_OFST_MASK, tmp & 0x7f); } - switch (param->subband_type) { + switch (chan->subband_type) { default: case RTW89_CH_2G: gain_band = RTW89_GAIN_OFFSET_2G_OFDM; @@ -977,14 +1005,14 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev, } static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev, - const struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { u8 sco; - u16 central_freq = param->center_freq; - u8 central_ch = param->center_chan; - u8 band = param->band_type; - u8 subband = param->subband_type; + u16 central_freq = chan->freq; + u8 central_ch = chan->channel; + u8 band = chan->band_type; + u8 subband = chan->subband_type; bool is_2g = band == RTW89_BAND_2G; u8 chan_idx; @@ -996,7 +1024,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev, if (phy_idx == RTW89_PHY_0) { /* Path A */ rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_A); - rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_A); + rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_A); if (is_2g) rtw89_phy_write32_idx(rtwdev, R_PATH0_BAND_SEL_V1, @@ -1009,7 +1037,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev, /* Path B */ if (!rtwdev->dbcc_en) { rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_B); - rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_B); + rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_B); if (is_2g) rtw89_phy_write32_idx(rtwdev, @@ -1038,7 +1066,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev, } else { /* Path B */ rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_B); - rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_B); + rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_B); if (is_2g) rtw89_phy_write32_idx(rtwdev, R_PATH1_BAND_SEL_V1, @@ -1095,7 +1123,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev, } } - chan_idx = rtw8852c_encode_chan_idx(rtwdev, param->primary_chan, band); + chan_idx = rtw8852c_encode_chan_idx(rtwdev, chan->primary_channel, band); rtw89_phy_write32_idx(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, chan_idx, phy_idx); } @@ -1246,12 +1274,12 @@ rtw8852c_ctrl_bw(struct rtw89_dev *rtwdev, u8 pri_ch, u8 bw, } static u32 rtw8852c_spur_freq(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param) + const struct rtw89_chan *chan) { - u8 center_chan = param->center_chan; - u8 bw = param->bandwidth; + u8 center_chan = chan->channel; + u8 bw = chan->band_width; - switch (param->band_type) { + switch (chan->band_type) { case RTW89_BAND_2G: if (bw == RTW89_CHANNEL_WIDTH_20) { if (center_chan >= 5 && center_chan <= 8) @@ -1285,19 +1313,19 @@ static u32 rtw8852c_spur_freq(struct rtw89_dev *rtwdev, #define MAX_TONE_NUM 2048 static void rtw8852c_set_csi_tone_idx(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { u32 spur_freq; s32 freq_diff, csi_idx, csi_tone_idx; - spur_freq = rtw8852c_spur_freq(rtwdev, param); + spur_freq = rtw8852c_spur_freq(rtwdev, chan); if (spur_freq == 0) { rtw89_phy_write32_idx(rtwdev, R_SEG0CSI_EN, B_SEG0CSI_EN, 0, phy_idx); return; } - freq_diff = (spur_freq - param->center_freq) * 1000000; + freq_diff = (spur_freq - chan->freq) * 1000000; csi_idx = s32_div_u32_round_closest(freq_diff, CARRIER_SPACING_78_125); s32_div_u32_round_down(csi_idx, MAX_TONE_NUM, &csi_tone_idx); @@ -1325,7 +1353,7 @@ static const struct rtw89_nbi_reg_def rtw8852c_nbi_reg_def[] = { }; static void rtw8852c_set_nbi_tone_idx(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_rf_path path) { const struct rtw89_nbi_reg_def *nbi = &rtw8852c_nbi_reg_def[path]; @@ -1335,34 +1363,37 @@ static void rtw8852c_set_nbi_tone_idx(struct rtw89_dev *rtwdev, s32 nbi_frac_idx, nbi_frac_tone_idx; bool notch2_chk = false; - spur_freq = rtw8852c_spur_freq(rtwdev, param); + spur_freq = rtw8852c_spur_freq(rtwdev, chan); if (spur_freq == 0) { rtw89_phy_write32_mask(rtwdev, nbi->notch1_en.addr, nbi->notch1_en.mask, 0); rtw89_phy_write32_mask(rtwdev, nbi->notch1_en.addr, nbi->notch1_en.mask, 0); return; } - fc = param->center_freq; - if (param->bandwidth == RTW89_CHANNEL_WIDTH_160) { + fc = chan->freq; + if (chan->band_width == RTW89_CHANNEL_WIDTH_160) { fc = (spur_freq > fc) ? fc + 40 : fc - 40; - if ((fc > spur_freq && param->center_chan < param->primary_chan) || - (fc < spur_freq && param->center_chan > param->primary_chan)) + if ((fc > spur_freq && + chan->channel < chan->primary_channel) || + (fc < spur_freq && + chan->channel > chan->primary_channel)) notch2_chk = true; } freq_diff = (spur_freq - fc) * 1000000; nbi_idx = s32_div_u32_round_down(freq_diff, CARRIER_SPACING_312_5, &nbi_frac_idx); - if (param->bandwidth == RTW89_CHANNEL_WIDTH_20) { + if (chan->band_width == RTW89_CHANNEL_WIDTH_20) { s32_div_u32_round_down(nbi_idx + 32, 64, &nbi_tone_idx); } else { - u16 tone_para = (param->bandwidth == RTW89_CHANNEL_WIDTH_40) ? 128 : 256; + u16 tone_para = (chan->band_width == RTW89_CHANNEL_WIDTH_40) ? + 128 : 256; s32_div_u32_round_down(nbi_idx, tone_para, &nbi_tone_idx); } nbi_frac_tone_idx = s32_div_u32_round_closest(nbi_frac_idx, CARRIER_SPACING_78_125); - if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && notch2_chk) { + if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && notch2_chk) { rtw89_phy_write32_mask(rtwdev, nbi->notch2_idx.addr, nbi->notch2_idx.mask, nbi_tone_idx); rtw89_phy_write32_mask(rtwdev, nbi->notch2_frac_idx.addr, @@ -1404,42 +1435,42 @@ static void rtw8852c_spur_notch(struct rtw89_dev *rtwdev, u32 val, } static void rtw8852c_spur_elimination(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, u8 pri_ch_idx, enum rtw89_phy_idx phy_idx) { - rtw8852c_set_csi_tone_idx(rtwdev, param, phy_idx); + rtw8852c_set_csi_tone_idx(rtwdev, chan, phy_idx); if (phy_idx == RTW89_PHY_0) { - if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && + if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && (pri_ch_idx == RTW89_SC_20_LOWER || pri_ch_idx == RTW89_SC_20_UP3X)) { rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_0); if (!rtwdev->dbcc_en) rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_1); - } else if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && + } else if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && (pri_ch_idx == RTW89_SC_20_UPPER || pri_ch_idx == RTW89_SC_20_LOW3X)) { rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_0); if (!rtwdev->dbcc_en) rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_1); } else { - rtw8852c_set_nbi_tone_idx(rtwdev, param, RF_PATH_A); + rtw8852c_set_nbi_tone_idx(rtwdev, chan, RF_PATH_A); if (!rtwdev->dbcc_en) - rtw8852c_set_nbi_tone_idx(rtwdev, param, + rtw8852c_set_nbi_tone_idx(rtwdev, chan, RF_PATH_B); } } else { - if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && + if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && (pri_ch_idx == RTW89_SC_20_LOWER || pri_ch_idx == RTW89_SC_20_UP3X)) { rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_1); - } else if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && + } else if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && (pri_ch_idx == RTW89_SC_20_UPPER || pri_ch_idx == RTW89_SC_20_LOW3X)) { rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_1); } else { - rtw8852c_set_nbi_tone_idx(rtwdev, param, RF_PATH_B); + rtw8852c_set_nbi_tone_idx(rtwdev, chan, RF_PATH_B); } } @@ -1450,14 +1481,14 @@ static void rtw8852c_spur_elimination(struct rtw89_dev *rtwdev, } static void rtw8852c_5m_mask(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - u8 pri_ch = param->primary_chan; + u8 pri_ch = chan->primary_channel; bool mask_5m_low; bool mask_5m_en; - switch (param->bandwidth) { + switch (chan->band_width) { case RTW89_CHANNEL_WIDTH_40: mask_5m_en = true; mask_5m_low = pri_ch == 2; @@ -1526,11 +1557,9 @@ static void rtw8852c_bb_reset_all(struct rtw89_dev *rtwdev, phy_idx); } -static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev, +static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev, enum rtw89_band band, enum rtw89_phy_idx phy_idx, bool en) { - struct rtw89_hal *hal = &rtwdev->hal; - if (en) { rtw89_phy_write32_idx(rtwdev, R_S0_HW_SI_DIS, B_S0_HW_SI_DIS_W_R_TRIG, 0x0, phy_idx); @@ -1538,7 +1567,7 @@ static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev, B_S1_HW_SI_DIS_W_R_TRIG, 0x0, phy_idx); rtw89_phy_write32_idx(rtwdev, R_RSTB_ASYNC, B_RSTB_ASYNC_ALL, 1, phy_idx); - if (hal->current_band_type == RTW89_BAND_2G) + if (band == RTW89_BAND_2G) rtw89_phy_write32_mask(rtwdev, R_RXCCA_V1, B_RXCCA_DIS_V1, 0x0); rtw89_phy_write32_mask(rtwdev, R_PD_CTRL, B_PD_HIT_DIS, 0x0); } else { @@ -1690,21 +1719,24 @@ static void rtw8852c_bb_sethw(struct rtw89_dev *rtwdev) } static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - bool cck_en = param->band_type == RTW89_BAND_2G; - u8 pri_ch_idx = param->pri_ch_idx; + struct rtw89_hal *hal = &rtwdev->hal; + bool cck_en = chan->band_type == RTW89_BAND_2G; + u8 pri_ch_idx = chan->pri_ch_idx; u32 mask, reg; u32 ru_alloc_msk[2] = {B_P80_AT_HIGH_FREQ_RU_ALLOC_PHY0, B_P80_AT_HIGH_FREQ_RU_ALLOC_PHY1}; + u8 ntx_path; - if (param->band_type == RTW89_BAND_2G) - rtw8852c_ctrl_sco_cck(rtwdev, param->center_chan, - param->primary_chan, param->bandwidth); + if (chan->band_type == RTW89_BAND_2G) + rtw8852c_ctrl_sco_cck(rtwdev, chan->channel, + chan->primary_channel, + chan->band_width); - rtw8852c_ctrl_ch(rtwdev, param, phy_idx); - rtw8852c_ctrl_bw(rtwdev, pri_ch_idx, param->bandwidth, phy_idx); + rtw8852c_ctrl_ch(rtwdev, chan, phy_idx); + rtw8852c_ctrl_bw(rtwdev, pri_ch_idx, chan->band_width, phy_idx); if (cck_en) { rtw89_phy_write32_mask(rtwdev, R_UPD_CLK_ADC, B_ENABLE_CCK, 1); rtw89_phy_write32_mask(rtwdev, R_RXCCA_V1, B_RXCCA_DIS_V1, 0); @@ -1717,17 +1749,17 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev, B_PD_ARBITER_OFF, 0x1, phy_idx); } - rtw8852c_spur_elimination(rtwdev, param, pri_ch_idx, phy_idx); - rtw8852c_ctrl_btg(rtwdev, param->band_type == RTW89_BAND_2G); - rtw8852c_5m_mask(rtwdev, param, phy_idx); + rtw8852c_spur_elimination(rtwdev, chan, pri_ch_idx, phy_idx); + rtw8852c_ctrl_btg(rtwdev, chan->band_type == RTW89_BAND_2G); + rtw8852c_5m_mask(rtwdev, chan, phy_idx); - if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && + if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && rtwdev->hal.cv != CHIP_CAV) { rtw89_phy_write32_idx(rtwdev, R_P80_AT_HIGH_FREQ, B_P80_AT_HIGH_FREQ, 0x0, phy_idx); reg = rtw89_mac_reg_by_idx(R_P80_AT_HIGH_FREQ_BB_WRP, phy_idx); - if (param->primary_chan > param->center_chan) { + if (chan->primary_channel > chan->channel) { rtw89_phy_write32_mask(rtwdev, R_P80_AT_HIGH_FREQ_RU_ALLOC, ru_alloc_msk[phy_idx], 1); @@ -1742,8 +1774,8 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev, } } - if (param->band_type == RTW89_BAND_6G && - param->bandwidth == RTW89_CHANNEL_WIDTH_160) + if (chan->band_type == RTW89_BAND_6G && + chan->band_width == RTW89_CHANNEL_WIDTH_160) rtw89_phy_write32_idx(rtwdev, R_CDD_EVM_CHK_EN, B_CDD_EVM_CHK_EN, 0, phy_idx); else @@ -1769,15 +1801,29 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev, } } + if (chan->band_type == RTW89_BAND_6G) + rtw89_phy_write32_set(rtwdev, R_MUIC, B_MUIC_EN); + else + rtw89_phy_write32_clr(rtwdev, R_MUIC, B_MUIC_EN); + + if (hal->antenna_tx) + ntx_path = hal->antenna_tx; + else + ntx_path = chan->band_type == RTW89_BAND_6G ? RF_B : RF_AB; + + rtw8852c_ctrl_tx_path_tmac(rtwdev, ntx_path, (enum rtw89_mac_idx)phy_idx); + rtw8852c_bb_reset_all(rtwdev, phy_idx); } static void rtw8852c_set_channel(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *params) + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) { - rtw8852c_set_channel_mac(rtwdev, params, RTW89_MAC_0); - rtw8852c_set_channel_bb(rtwdev, params, RTW89_PHY_0); - rtw8852c_set_channel_rf(rtwdev, params, RTW89_PHY_0); + rtw8852c_set_channel_mac(rtwdev, chan, mac_idx); + rtw8852c_set_channel_bb(rtwdev, chan, phy_idx); + rtw8852c_set_channel_rf(rtwdev, chan, phy_idx); } static void rtw8852c_dfs_en(struct rtw89_dev *rtwdev, bool en) @@ -1799,25 +1845,27 @@ static void rtw8852c_adc_en(struct rtw89_dev *rtwdev, bool en) } static void rtw8852c_set_channel_help(struct rtw89_dev *rtwdev, bool enter, - struct rtw89_channel_help_params *p) + struct rtw89_channel_help_params *p, + const struct rtw89_chan *chan, + enum rtw89_mac_idx mac_idx, + enum rtw89_phy_idx phy_idx) { - u8 phy_idx = RTW89_PHY_0; - if (enter) { - rtw89_chip_stop_sch_tx(rtwdev, RTW89_MAC_0, &p->tx_en, RTW89_SCH_TX_SEL_ALL); - rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, false); + rtw89_chip_stop_sch_tx(rtwdev, mac_idx, &p->tx_en, + RTW89_SCH_TX_SEL_ALL); + rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, false); rtw8852c_dfs_en(rtwdev, false); - rtw8852c_tssi_cont_en_phyidx(rtwdev, false, RTW89_PHY_0); + rtw8852c_tssi_cont_en_phyidx(rtwdev, false, phy_idx); rtw8852c_adc_en(rtwdev, false); fsleep(40); - rtw8852c_bb_reset_en(rtwdev, phy_idx, false); + rtw8852c_bb_reset_en(rtwdev, chan->band_type, phy_idx, false); } else { - rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, true); + rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, true); rtw8852c_adc_en(rtwdev, true); rtw8852c_dfs_en(rtwdev, true); - rtw8852c_tssi_cont_en_phyidx(rtwdev, true, RTW89_PHY_0); - rtw8852c_bb_reset_en(rtwdev, phy_idx, true); - rtw89_chip_resume_sch_tx(rtwdev, RTW89_MAC_0, p->tx_en); + rtw8852c_tssi_cont_en_phyidx(rtwdev, true, phy_idx); + rtw8852c_bb_reset_en(rtwdev, chan->band_type, phy_idx, true); + rtw89_chip_resume_sch_tx(rtwdev, mac_idx, p->tx_en); } } @@ -1847,9 +1895,10 @@ static void rtw8852c_rfk_channel(struct rtw89_dev *rtwdev) rtw89_fw_h2c_rf_ntfy_mcc(rtwdev); } -static void rtw8852c_rfk_band_changed(struct rtw89_dev *rtwdev) +static void rtw8852c_rfk_band_changed(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx) { - rtw8852c_tssi_scan(rtwdev, RTW89_PHY_0); + rtw8852c_tssi_scan(rtwdev, phy_idx); } static void rtw8852c_rfk_scan(struct rtw89_dev *rtwdev, bool start) @@ -1958,9 +2007,11 @@ static void rtw8852c_set_txpwr_ref(struct rtw89_dev *rtwdev, } static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - u8 ch = rtwdev->hal.current_channel; + u8 band = chan->band_type; + u8 ch = chan->channel; static const u8 rs[] = { RTW89_RS_CCK, RTW89_RS_OFDM, @@ -1986,7 +2037,8 @@ static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev, for (j = 0; j < rtw89_rs_idx_max[rs[i]]; j++) { cur.idx = j; shf = (j % 4) * 8; - tmp = rtw89_phy_read_txpwr_byrate(rtwdev, &cur); + tmp = rtw89_phy_read_txpwr_byrate(rtwdev, band, + &cur); val |= (tmp << shf); if ((j + 1) % 4) @@ -2001,8 +2053,10 @@ static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev, } static void rtw8852c_set_txpwr_offset(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { + u8 band = chan->band_type; struct rtw89_rate_desc desc = { .nss = RTW89_NSS_1, .rs = RTW89_RS_OFFSET, @@ -2013,7 +2067,7 @@ static void rtw8852c_set_txpwr_offset(struct rtw89_dev *rtwdev, rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr offset\n"); for (desc.idx = 0; desc.idx < RTW89_RATE_OFFSET_MAX; desc.idx++) { - v = rtw89_phy_read_txpwr_byrate(rtwdev, &desc); + v = rtw89_phy_read_txpwr_byrate(rtwdev, band, &desc); val |= ((v & 0xf) << (4 * desc.idx)); } @@ -2045,7 +2099,8 @@ static void rtw8852c_bb_set_tx_shape_dfir(struct rtw89_dev *rtwdev, __DECL_DFIR_ADDR(filter, 0x45BC, 0x45CC, 0x45D0, 0x45D4, 0x45D8, 0x45C0, 0x45C4, 0x45C8); - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; const u32 *param; int i; @@ -2076,9 +2131,10 @@ static void rtw8852c_bb_set_tx_shape_dfir(struct rtw89_dev *rtwdev, } static void rtw8852c_set_tx_shape(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - u8 band = rtwdev->hal.current_band_type; + u8 band = chan->band_type; u8 regd = rtw89_regd_get(rtwdev, band); u8 tx_shape_cck = rtw89_8852c_tx_shape[band][RTW89_RS_CCK][regd]; u8 tx_shape_ofdm = rtw89_8852c_tx_shape[band][RTW89_RS_OFDM][regd]; @@ -2092,29 +2148,31 @@ static void rtw8852c_set_tx_shape(struct rtw89_dev *rtwdev, } static void rtw8852c_set_txpwr_limit(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { #define __MAC_TXPWR_LMT_PAGE_SIZE 40 - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 ch = chan->channel; + u8 bw = chan->band_width; struct rtw89_txpwr_limit lmt[NTX_NUM_8852C]; u32 addr, val; const s8 *ptr; - u8 i, j, k; + u8 i, j; rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr limit with ch=%d bw=%d\n", ch, bw); for (i = 0; i < NTX_NUM_8852C; i++) { - rtw89_phy_fill_txpwr_limit(rtwdev, &lmt[i], i); + rtw89_phy_fill_txpwr_limit(rtwdev, chan, &lmt[i], i); for (j = 0; j < __MAC_TXPWR_LMT_PAGE_SIZE; j += 4) { addr = R_AX_PWR_LMT + j + __MAC_TXPWR_LMT_PAGE_SIZE * i; ptr = (s8 *)&lmt[i] + j; - val = 0; - for (k = 0; k < 4; k++) - val |= (ptr[k] << (8 * k)); + val = FIELD_PREP(GENMASK(7, 0), ptr[0]) | + FIELD_PREP(GENMASK(15, 8), ptr[1]) | + FIELD_PREP(GENMASK(23, 16), ptr[2]) | + FIELD_PREP(GENMASK(31, 24), ptr[3]); rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val); } @@ -2123,30 +2181,32 @@ static void rtw8852c_set_txpwr_limit(struct rtw89_dev *rtwdev, } static void rtw8852c_set_txpwr_limit_ru(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { #define __MAC_TXPWR_LMT_RU_PAGE_SIZE 24 - u8 ch = rtwdev->hal.current_channel; - u8 bw = rtwdev->hal.current_band_width; + u8 ch = chan->channel; + u8 bw = chan->band_width; struct rtw89_txpwr_limit_ru lmt_ru[NTX_NUM_8852C]; u32 addr, val; const s8 *ptr; - u8 i, j, k; + u8 i, j; rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr limit ru with ch=%d bw=%d\n", ch, bw); for (i = 0; i < NTX_NUM_8852C; i++) { - rtw89_phy_fill_txpwr_limit_ru(rtwdev, &lmt_ru[i], i); + rtw89_phy_fill_txpwr_limit_ru(rtwdev, chan, &lmt_ru[i], i); for (j = 0; j < __MAC_TXPWR_LMT_RU_PAGE_SIZE; j += 4) { addr = R_AX_PWR_RU_LMT + j + __MAC_TXPWR_LMT_RU_PAGE_SIZE * i; ptr = (s8 *)&lmt_ru[i] + j; - val = 0; - for (k = 0; k < 4; k++) - val |= (ptr[k] << (8 * k)); + val = FIELD_PREP(GENMASK(7, 0), ptr[0]) | + FIELD_PREP(GENMASK(15, 8), ptr[1]) | + FIELD_PREP(GENMASK(23, 16), ptr[2]) | + FIELD_PREP(GENMASK(31, 24), ptr[3]); rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val); } @@ -2155,18 +2215,21 @@ static void rtw8852c_set_txpwr_limit_ru(struct rtw89_dev *rtwdev, #undef __MAC_TXPWR_LMT_RU_PAGE_SIZE } -static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev) +static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev, + const struct rtw89_chan *chan, + enum rtw89_phy_idx phy_idx) { - rtw8852c_set_txpwr_byrate(rtwdev, RTW89_PHY_0); - rtw8852c_set_txpwr_offset(rtwdev, RTW89_PHY_0); - rtw8852c_set_tx_shape(rtwdev, RTW89_PHY_0); - rtw8852c_set_txpwr_limit(rtwdev, RTW89_PHY_0); - rtw8852c_set_txpwr_limit_ru(rtwdev, RTW89_PHY_0); + rtw8852c_set_txpwr_byrate(rtwdev, chan, phy_idx); + rtw8852c_set_txpwr_offset(rtwdev, chan, phy_idx); + rtw8852c_set_tx_shape(rtwdev, chan, phy_idx); + rtw8852c_set_txpwr_limit(rtwdev, chan, phy_idx); + rtw8852c_set_txpwr_limit_ru(rtwdev, chan, phy_idx); } -static void rtw8852c_set_txpwr_ctrl(struct rtw89_dev *rtwdev) +static void rtw8852c_set_txpwr_ctrl(struct rtw89_dev *rtwdev, + enum rtw89_phy_idx phy_idx) { - rtw8852c_set_txpwr_ref(rtwdev, RTW89_PHY_0); + rtw8852c_set_txpwr_ref(rtwdev, phy_idx); } static void @@ -2222,7 +2285,8 @@ rtw8852c_init_txpwr_unit(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) static void rtw8852c_bb_cfg_rx_path(struct rtw89_dev *rtwdev, u8 rx_path) { - struct rtw89_hal *hal = &rtwdev->hal; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 band = chan->band_type; u32 rst_mask0 = B_P0_TXPW_RSTB_MANON | B_P0_TXPW_RSTB_TSSI; u32 rst_mask1 = B_P1_TXPW_RSTB_MANON | B_P1_TXPW_RSTB_TSSI; @@ -2316,7 +2380,7 @@ static void rtw8852c_bb_cfg_rx_path(struct rtw89_dev *rtwdev, u8 rx_path) 1); rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHETB_MAX_NSS, 1); - rtw8852c_ctrl_btg(rtwdev, hal->current_band_type == RTW89_BAND_2G); + rtw8852c_ctrl_btg(rtwdev, band == RTW89_BAND_2G); rtw89_phy_write32_mask(rtwdev, R_P0_TXPW_RSTB, rst_mask0, 1); rtw89_phy_write32_mask(rtwdev, R_P0_TXPW_RSTB, @@ -2458,7 +2522,6 @@ static void rtw8852c_bb_ctrl_btc_preagc(struct rtw89_dev *rtwdev, bool bt_en) static void rtw8852c_bb_cfg_txrx_path(struct rtw89_dev *rtwdev) { struct rtw89_hal *hal = &rtwdev->hal; - u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_AB; rtw8852c_bb_cfg_rx_path(rtwdev, RF_PATH_AB); @@ -2473,8 +2536,6 @@ static void rtw8852c_bb_cfg_txrx_path(struct rtw89_dev *rtwdev) rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHE_MAX_NSS, 1); rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHETB_MAX_NSS, 1); } - - rtw8852c_ctrl_tx_path_tmac(rtwdev, ntx_path, RTW89_MAC_0); } static u8 rtw8852c_get_thermal(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path) @@ -2773,23 +2834,7 @@ void rtw8852c_btc_bt_aci_imp(struct rtw89_dev *rtwdev) static void rtw8852c_btc_update_bt_cnt(struct rtw89_dev *rtwdev) { - struct rtw89_btc *btc = &rtwdev->btc; - struct rtw89_btc_cx *cx = &btc->cx; - u32 val; - - val = rtw89_read32(rtwdev, R_BTC_BT_CNT_HIGH); - cx->cnt_bt[BTC_BCNT_HIPRI_TX] = FIELD_GET(B_AX_STATIS_BT_HI_TX_MASK, val); - cx->cnt_bt[BTC_BCNT_HIPRI_RX] = FIELD_GET(B_AX_STATIS_BT_HI_RX_MASK, val); - - val = rtw89_read32(rtwdev, R_BTC_BT_CNT_LOW); - cx->cnt_bt[BTC_BCNT_LOPRI_TX] = FIELD_GET(B_AX_STATIS_BT_LO_TX_1_MASK, val); - cx->cnt_bt[BTC_BCNT_LOPRI_RX] = FIELD_GET(B_AX_STATIS_BT_LO_RX_1_MASK, val); - - /* clock-gate off before reset counter*/ - rtw89_write32_set(rtwdev, R_AX_BTC_CFG, B_AX_DIS_BTC_CLK_G); - rtw89_write32_clr(rtwdev, R_AX_BT_CNT_CFG, B_AX_BT_CNT_RST); - rtw89_write32_set(rtwdev, R_AX_BT_CNT_CFG, B_AX_BT_CNT_RST); - rtw89_write32_clr(rtwdev, R_AX_BTC_CFG, B_AX_DIS_BTC_CLK_G); + /* Feature move to firmware */ } static @@ -2810,6 +2855,59 @@ void rtw8852c_btc_wl_s1_standby(struct rtw89_dev *rtwdev, bool state) rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); } +static void rtw8852c_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level) +{ + /* level=0 Default: TIA 1/0= (LNA2,TIAN6) = (7,1)/(5,1) = 21dB/12dB + * level=1 Fix LNA2=5: TIA 1/0= (LNA2,TIAN6) = (5,0)/(5,1) = 18dB/12dB + * To improve BT ACI in co-rx + */ + + switch (level) { + case 0: /* default */ + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); + break; + case 1: /* Fix LNA2=5 */ + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5); + rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0); + break; + } +} + +static void rtw8852c_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level) +{ + switch (level) { + case 0: /* original */ + rtw8852c_bb_ctrl_btc_preagc(rtwdev, false); + rtw8852c_set_wl_lna2(rtwdev, 0); + break; + case 1: /* for FDD free-run */ + rtw8852c_bb_ctrl_btc_preagc(rtwdev, true); + rtw8852c_set_wl_lna2(rtwdev, 0); + break; + case 2: /* for BTG Co-Rx*/ + rtw8852c_bb_ctrl_btc_preagc(rtwdev, false); + rtw8852c_set_wl_lna2(rtwdev, 1); + break; + } +} + static void rtw8852c_fill_freq_with_ppdu(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct ieee80211_rx_status *status) @@ -2831,12 +2929,12 @@ static void rtw8852c_query_ppdu(struct rtw89_dev *rtwdev, struct ieee80211_rx_status *status) { u8 path; - s8 *rx_power = phy_ppdu->rssi; + u8 *rx_power = phy_ppdu->rssi; - status->signal = max_t(s8, rx_power[RF_PATH_A], rx_power[RF_PATH_B]); + status->signal = RTW89_RSSI_RAW_TO_DBM(max(rx_power[RF_PATH_A], rx_power[RF_PATH_B])); for (path = 0; path < rtwdev->chip->rf_path_num; path++) { status->chains |= BIT(path); - status->chain_signal[path] = rx_power[path]; + status->chain_signal[path] = RTW89_RSSI_RAW_TO_DBM(rx_power[path]); } if (phy_ppdu->valid) rtw8852c_fill_freq_with_ppdu(rtwdev, phy_ppdu, status); @@ -2879,10 +2977,12 @@ static int rtw8852c_mac_enable_bb_rf(struct rtw89_dev *rtwdev) return 0; } -static void rtw8852c_mac_disable_bb_rf(struct rtw89_dev *rtwdev) +static int rtw8852c_mac_disable_bb_rf(struct rtw89_dev *rtwdev) { rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN, B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN); + + return 0; } static const struct rtw89_chip_ops rtw8852c_chip_ops = { @@ -2930,6 +3030,8 @@ static const struct rtw89_chip_ops rtw8852c_chip_ops = { .btc_bt_aci_imp = rtw8852c_btc_bt_aci_imp, .btc_update_bt_cnt = rtw8852c_btc_update_bt_cnt, .btc_wl_s1_standby = rtw8852c_btc_wl_s1_standby, + .btc_set_wl_rx_gain = rtw8852c_btc_set_wl_rx_gain, + .btc_set_policy = rtw89_btc_set_policy_v1, }; const struct rtw89_chip_info rtw8852c_chip_info = { @@ -2937,6 +3039,7 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .ops = &rtw8852c_chip_ops, .fw_name = "rtw89/rtw8852c_fw.bin", .fifo_size = 458752, + .dle_scc_rsvd_size = 0, .max_amsdu_limit = 8000, .dis_2g_40m_ul_ofdma = false, .rsvd_ple_ofst = 0x6f800, @@ -2960,7 +3063,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .txpwr_factor_rf = 2, .txpwr_factor_mac = 1, .dig_table = NULL, + .dig_regs = &rtw8852c_dig_regs, .tssi_dbw_table = &rtw89_8852c_tssi_dbw_table, + .support_chanctx_num = 1, .support_bands = BIT(NL80211_BAND_2GHZ) | BIT(NL80211_BAND_5GHZ) | BIT(NL80211_BAND_6GHZ), @@ -2972,6 +3077,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .acam_num = 128, .bcam_num = 20, .scam_num = 128, + .bacam_num = 8, + .bacam_dynamic_num = 8, + .bacam_v1 = true, .sec_ctrl_efuse_size = 4, .physical_efuse_size = 1216, .logical_efuse_size = 2048, @@ -2980,11 +3088,26 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .dav_log_efuse_size = 16, .phycap_addr = 0x590, .phycap_size = 0x60, - .para_ver = 0x05050764, - .wlcx_desired = 0x05050000, - .btcx_desired = 0x5, + .para_ver = 0x1, + .wlcx_desired = 0x06000000, + .btcx_desired = 0x7, .scbd = 0x1, .mailbox = 0x1, + .btc_fwinfo_buf = 1280, + + .fcxbtcrpt_ver = 4, + .fcxtdma_ver = 3, + .fcxslots_ver = 1, + .fcxcysta_ver = 3, + .fcxstep_ver = 3, + .fcxnullsta_ver = 2, + .fcxmreg_ver = 1, + .fcxgpiodbg_ver = 1, + .fcxbtver_ver = 1, + .fcxbtscan_ver = 1, + .fcxbtafh_ver = 1, + .fcxbtdevinfo_ver = 1, + .afh_guard_ch = 6, .wl_rssi_thres = rtw89_btc_8852c_wl_rssi_thres, .bt_rssi_thres = rtw89_btc_8852c_bt_rssi_thres, @@ -2995,7 +3118,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .rf_para_ulink = rtw89_btc_8852c_rf_ul, .rf_para_dlink_num = ARRAY_SIZE(rtw89_btc_8852c_rf_dl), .rf_para_dlink = rtw89_btc_8852c_rf_dl, - .ps_mode_supported = 0, + .ps_mode_supported = BIT(RTW89_PS_MODE_RFOFF) | + BIT(RTW89_PS_MODE_CLK_GATED) | + BIT(RTW89_PS_MODE_PWR_GATED), .low_power_hci_modes = BIT(RTW89_PS_MODE_CLK_GATED) | BIT(RTW89_PS_MODE_PWR_GATED), .h2c_cctl_func_id = H2C_FUNC_MAC_CCTLINFO_UD_V1, @@ -3009,7 +3134,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = { .page_regs = &rtw8852c_page_regs, .dcfo_comp = &rtw8852c_dcfo_comp, .dcfo_comp_sft = 5, - .imr_info = &rtw8852c_imr_info + .imr_info = &rtw8852c_imr_info, + .rrsr_cfgs = &rtw8852c_rrsr_cfgs, + .dma_ch_mask = 0, }; EXPORT_SYMBOL(rtw8852c_chip_info); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c index 4186d825d19b74c471a4e60ed00efbb8cc67c9c9..006c2cf93111666abd567107fdce8238b6ace192 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c @@ -1294,14 +1294,14 @@ static void _iqk_by_path(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx, u static void _iqk_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, u8 path) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_iqk_info *iqk_info = &rtwdev->iqk; - struct rtw89_hal *hal = &rtwdev->hal; rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]===>%s\n", __func__); - iqk_info->iqk_band[path] = hal->current_band_type; - iqk_info->iqk_bw[path] = hal->current_band_width; - iqk_info->iqk_ch[path] = hal->current_channel; + iqk_info->iqk_band[path] = chan->band_type; + iqk_info->iqk_bw[path] = chan->band_width; + iqk_info->iqk_ch[path] = chan->channel; rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]iqk_info->iqk_band[%x] = 0x%x\n", path, @@ -1546,7 +1546,8 @@ static void _rx_dck_toggle(struct rtw89_dev *rtwdev, u8 path) rtw89_write_rf(rtwdev, path, RR_DCK, RR_DCK_LV, 0x1); ret = read_poll_timeout_atomic(rtw89_read_rf, val, val, - 2, 1000, false, rtwdev, path, 0x93, BIT(5)); + 2, 2000, false, rtwdev, path, + RR_DCK1, RR_DCK1_DONE); if (ret) rtw89_warn(rtwdev, "[RX_DCK] S%d RXDCK timeout\n", path); else @@ -1691,14 +1692,14 @@ static void _dpk_information(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_dpk_info *dpk = &rtwdev->dpk; - struct rtw89_hal *hal = &rtwdev->hal; u8 kidx = dpk->cur_idx[path]; - dpk->bp[path][kidx].band = hal->current_band_type; - dpk->bp[path][kidx].ch = hal->current_channel; - dpk->bp[path][kidx].bw = hal->current_band_width; + dpk->bp[path][kidx].band = chan->band_type; + dpk->bp[path][kidx].ch = chan->channel; + dpk->bp[path][kidx].bw = chan->band_width; rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] S%d[%d] (PHY%d): TSSI %s/ DBCC %s/ %s/ CH%d/ %s\n", @@ -2272,12 +2273,13 @@ static void _dpk_idl_mpa(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, static bool _dpk_reload_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_dpk_info *dpk = &rtwdev->dpk; bool is_reload = false; u8 idx, cur_band, cur_ch; - cur_band = rtwdev->hal.current_band_type; - cur_ch = rtwdev->hal.current_channel; + cur_band = chan->band_type; + cur_ch = chan->channel; for (idx = 0; idx < RTW89_DPK_BKUP_NUM; idx++) { if (cur_band != dpk->bp[path][idx].band || @@ -2530,17 +2532,19 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force, static bool _dpk_bypass_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { struct rtw89_fem_info *fem = &rtwdev->fem; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 band = chan->band_type; - if (rtwdev->hal.cv == CHIP_CAV && rtwdev->hal.current_band_type != RTW89_BAND_2G) { + if (rtwdev->hal.cv == CHIP_CAV && band != RTW89_BAND_2G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to CAV & not 2G!!\n"); return true; - } else if (fem->epa_2g && rtwdev->hal.current_band_type == RTW89_BAND_2G) { + } else if (fem->epa_2g && band == RTW89_BAND_2G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 2G_ext_PA exist!!\n"); return true; - } else if (fem->epa_5g && rtwdev->hal.current_band_type == RTW89_BAND_5G) { + } else if (fem->epa_5g && band == RTW89_BAND_5G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 5G_ext_PA exist!!\n"); return true; - } else if (fem->epa_6g && rtwdev->hal.current_band_type == RTW89_BAND_6G) { + } else if (fem->epa_6g && band == RTW89_BAND_6G) { rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 6G_ext_PA exist!!\n"); return true; } @@ -2663,7 +2667,8 @@ static void _dpk_track(struct rtw89_dev *rtwdev) static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; rtw89_rfk_parser(rtwdev, &rtw8852c_tssi_sys_defs_tbl); @@ -2697,7 +2702,8 @@ static void _tssi_ini_txpwr_ctrl_bb_he_tb(struct rtw89_dev *rtwdev, static void _tssi_set_dck(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; if (path == RF_PATH_A) { rtw89_rfk_parser(rtwdev, &rtw8852c_tssi_dck_defs_a_tbl); @@ -2735,8 +2741,9 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph __val; \ }) struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel; - u8 subband = rtwdev->hal.current_subband; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; + u8 subband = chan->subband_type; const s8 *thm_up_a = NULL; const s8 *thm_down_a = NULL; const s8 *thm_up_b = NULL; @@ -2908,7 +2915,8 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph static void _tssi_slope_cal_org(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; if (path == RF_PATH_A) { rtw89_rfk_parser_by_cond(rtwdev, band == RTW89_BAND_2G, @@ -2924,7 +2932,8 @@ static void _tssi_slope_cal_org(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy static void _tssi_set_aligk_default(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { - enum rtw89_band band = rtwdev->hal.current_band_type; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; const struct rtw89_rfk_tbl *tbl; if (path == RF_PATH_A) { @@ -3335,8 +3344,9 @@ static s8 _tssi_get_ofdm_de(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, enum rtw89_rf_path path) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - enum rtw89_band band = rtwdev->hal.current_band_type; - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; + u8 ch = chan->channel; u32 gidx, gidx_1st, gidx_2nd; s8 de_1st; s8 de_2nd; @@ -3398,8 +3408,9 @@ static s8 _tssi_get_ofdm_trim_de(struct rtw89_dev *rtwdev, enum rtw89_rf_path path) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - enum rtw89_band band = rtwdev->hal.current_band_type; - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; + u8 ch = chan->channel; u32 tgidx, tgidx_1st, tgidx_2nd; s8 tde_1st = 0; s8 tde_2nd = 0; @@ -3462,7 +3473,8 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy) { struct rtw89_tssi_info *tssi_info = &rtwdev->tssi; - u8 ch = rtwdev->hal.current_channel; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + u8 ch = chan->channel; u8 gidx; s8 ofdm_de; s8 trim_de; @@ -3802,15 +3814,17 @@ void rtw8852c_ctrl_bw_ch(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy, } void rtw8852c_set_channel_rf(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx) { - rtw8852c_ctrl_bw_ch(rtwdev, phy_idx, param->center_chan, param->band_type, - param->bandwidth); + rtw8852c_ctrl_bw_ch(rtwdev, phy_idx, chan->channel, + chan->band_type, + chan->band_width); } void rtw8852c_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx) { + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); struct rtw89_mcc_info *mcc_info = &rtwdev->mcc; u8 idx = mcc_info->table_idx; int i; @@ -3823,8 +3837,8 @@ void rtw8852c_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_i } mcc_info->table_idx = idx; - mcc_info->ch[idx] = rtwdev->hal.current_channel; - mcc_info->band[idx] = rtwdev->hal.current_band_type; + mcc_info->ch[idx] = chan->channel; + mcc_info->band[idx] = chan->band_type; } void rtw8852c_rck(struct rtw89_dev *rtwdev) diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h index 5118a49da8d3e30ad0e96fc6fa2b5e3ee794c7b2..928a587cdd0564fd4789bc732a24200750e21d58 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h @@ -21,7 +21,7 @@ void rtw8852c_tssi_cont_en_phyidx(struct rtw89_dev *rtwdev, bool en, u8 phy_idx) void rtw8852c_wifi_scan_notify(struct rtw89_dev *rtwdev, bool scan_start, enum rtw89_phy_idx phy_idx); void rtw8852c_set_channel_rf(struct rtw89_dev *rtwdev, - struct rtw89_channel_params *param, + const struct rtw89_chan *chan, enum rtw89_phy_idx phy_idx); void rtw8852c_lck_init(struct rtw89_dev *rtwdev); void rtw8852c_lck_track(struct rtw89_dev *rtwdev); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c index feaa83b16171da3d33a9f7512004e94367a87465..11f35e7a7f0e7a3dbb6dd18b0e03a370eecb6acd 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c @@ -1767,7 +1767,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_bb_reg_gain[] = { {0x3070103, 0x34343C3C}, }; -static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { +static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0xF0010000, 0x00000000}, {0xF0020000, 0x00000001}, {0xF0320000, 0x00000002}, @@ -1777,13 +1777,17 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0xF0360000, 0x00000006}, {0xF0010001, 0x00000007}, {0xF0020001, 0x00000008}, - {0xF0320001, 0x00000009}, - {0xF0330001, 0x0000000A}, - {0xF0340001, 0x0000000B}, - {0xF0350001, 0x0000000C}, - {0xF0360001, 0x0000000D}, - {0xF03F0001, 0x0000000E}, - {0xF0400001, 0x0000000F}, + {0xF0030001, 0x00000009}, + {0xF0040001, 0x0000000A}, + {0xF0050001, 0x0000000B}, + {0xF0070001, 0x0000000C}, + {0xF0320001, 0x0000000D}, + {0xF0330001, 0x0000000E}, + {0xF0340001, 0x0000000F}, + {0xF0350001, 0x00000010}, + {0xF0360001, 0x00000011}, + {0xF03F0001, 0x00000012}, + {0xF0400001, 0x00000013}, {0x005, 0x00000000}, {0x10005, 0x00000000}, {0x000, 0x00030001}, @@ -1795,7 +1799,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03E, 0x00000620}, {0x03F, 0x0000020C}, {0x0EF, 0x00000000}, - {0x05F, 0x00000032}, + {0x05F, 0x00000038}, {0x097, 0x00043200}, {0x0A6, 0x00066DB7}, {0x0EF, 0x00004000}, @@ -1821,8 +1825,8 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x000, 0x00033C01}, {0x10000, 0x00033C00}, {0x01A, 0x00040004}, - {0x0FE, 0x00000000}, {0x096, 0x00015200}, + {0x10055, 0x00080080}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0004D000}, {0x0DA, 0x000D4009}, @@ -1850,6 +1854,18 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0000D300}, {0x0DA, 0x000D4000}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0000D300}, {0x0DA, 0x000D4000}, @@ -1922,6 +1938,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000CC}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000CC}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000CC}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000CC}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000CC}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000CC}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000CC}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -1958,6 +1982,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000C4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000C4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000C4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000C4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000C4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000C4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000C4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -1994,6 +2026,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000BC}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000BC}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000BC}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2030,6 +2070,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000B4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000B4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000B4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2066,6 +2114,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000AC}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000AC}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000AC}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2102,6 +2158,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000A4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000A4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2138,6 +2202,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000009C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000009C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000009C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2174,6 +2246,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000094}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000094}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000094}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2210,6 +2290,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000008C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000008C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000008C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2246,6 +2334,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000084}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000084}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000084}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2282,6 +2378,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000BC}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000BC}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000BC}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000BC}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2318,6 +2422,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000B4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000B4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000B4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000B4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2354,6 +2466,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000AC}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000AC}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000AC}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000AC}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2390,6 +2510,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000000A4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000000A4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000000A4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2426,6 +2554,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000009C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000009C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000009C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2462,6 +2598,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000094}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000094}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000094}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000094}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2498,6 +2642,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000008C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000008C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000008C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000008C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2534,6 +2686,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000084}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000084}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000084}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000084}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2570,6 +2730,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000003C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000003C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000003C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000003C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000003C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000003C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000003C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2606,6 +2774,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000034}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000034}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000034}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000034}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000034}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000034}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000034}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2642,6 +2818,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000002C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000002C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000002C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000002C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000002C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000002C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000002C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2678,6 +2862,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000024}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000024}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000024}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000024}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000024}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000024}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000024}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2714,6 +2906,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000001C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000001C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000001C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000001C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000001C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000001C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000001C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2750,6 +2950,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000014}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000014}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000014}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000014}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000014}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000014}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000014}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2786,6 +2994,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000000C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000000C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000000C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000000C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000000C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000000C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000000C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2822,6 +3038,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000004}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000004}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000004}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000004}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000004}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000004}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000004}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2871,6 +3095,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x08F, 0x000D1352}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x08F, 0x000D1352}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x08F, 0x000D1352}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -2905,6 +3137,52 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000015}, {0x033, 0x00000001}, {0x03F, 0x00000017}, + {0x033, 0x00000002}, + {0x03F, 0x00000017}, + {0x033, 0x00000003}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000007}, + {0xB0000000, 0x00000000}, {0x0EF, 0x00000000}, {0x0EF, 0x00008000}, {0x033, 0x00000020}, @@ -3416,6 +3694,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000EFFF}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000EFFF}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000EFFF}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -3522,7 +3808,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x033, 0x00000005}, {0x03F, 0x00004344}, {0x033, 0x00000006}, - {0x03F, 0x00004324}, + {0x03F, 0x00004344}, {0x033, 0x00000007}, {0x03F, 0x00004344}, {0x033, 0x00000008}, @@ -3585,6 +3871,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000200}, {0x0EF, 0x00000000}, {0x0EF, 0x00000010}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x030, 0x000084DC}, {0x030, 0x000103C9}, {0x030, 0x00018399}, @@ -3597,98 +3884,359 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x030, 0x00050011}, {0x030, 0x00058000}, {0x030, 0x00060000}, - {0x030, 0x00068000}, - {0x030, 0x00070000}, - {0x0EF, 0x00000000}, - {0x0EF, 0x00000080}, - {0x033, 0x00000004}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x00000005}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x00000006}, - {0x03E, 0x00000014}, - {0x03F, 0x00021C58}, - {0x033, 0x00000007}, - {0x03E, 0x00000014}, - {0x03F, 0x00022B58}, - {0x033, 0x00000008}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x00000009}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x0000000A}, - {0x03E, 0x00000014}, - {0x03F, 0x00021C58}, - {0x033, 0x0000000B}, - {0x03E, 0x00000014}, - {0x03F, 0x00022B58}, - {0x033, 0x0000000C}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x0000000D}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x0000000E}, - {0x03E, 0x00000014}, - {0x03F, 0x00021C58}, - {0x033, 0x0000000F}, - {0x03E, 0x00000014}, - {0x03F, 0x00022B58}, - {0x033, 0x00000010}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x00000011}, - {0x03E, 0x0000001B}, - {0x03F, 0x00023C58}, - {0x033, 0x00000012}, - {0x03E, 0x00000014}, - {0x03F, 0x00021C58}, - {0x033, 0x00000013}, - {0x03E, 0x00000014}, - {0x03F, 0x00022B58}, - {0x033, 0x00000014}, - {0x03E, 0x00000013}, - {0x03F, 0x00023C58}, - {0x033, 0x00000015}, - {0x03E, 0x0000001B}, - {0x03F, 0x00025A58}, - {0x033, 0x00000016}, - {0x03E, 0x0000001C}, - {0x03F, 0x00021C58}, - {0x033, 0x00000017}, - {0x03E, 0x00000014}, - {0x03F, 0x00022A58}, - {0x033, 0x00000018}, - {0x03E, 0x00000013}, - {0x03F, 0x00025A58}, - {0x033, 0x00000019}, - {0x03E, 0x0000001B}, - {0x03F, 0x00025A58}, - {0x033, 0x0000001A}, - {0x03E, 0x00000014}, - {0x03F, 0x00022A58}, - {0x033, 0x0000001B}, - {0x03E, 0x00000014}, - {0x03F, 0x00022A58}, - {0x033, 0x0000001C}, - {0x03E, 0x00000014}, - {0x03F, 0x0002CD58}, - {0x033, 0x0000001D}, - {0x03E, 0x0000001B}, - {0x03F, 0x00025A58}, - {0x033, 0x0000001E}, - {0x03E, 0x00000013}, - {0x03F, 0x00021E58}, - {0x033, 0x0000001F}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000020}, - {0x03E, 0x00000014}, - {0x03F, 0x0002CD58}, - {0x033, 0x00000021}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0xA0000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0xB0000000, 0x00000000}, + {0x030, 0x00068000}, + {0x030, 0x00070000}, + {0x0EF, 0x00000000}, + {0x0EF, 0x00000080}, + {0x033, 0x00000004}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x00000005}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x00000006}, + {0x03E, 0x00000014}, + {0x03F, 0x00021C58}, + {0x033, 0x00000007}, + {0x03E, 0x00000014}, + {0x03F, 0x00022B58}, + {0x033, 0x00000008}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x00000009}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x0000000A}, + {0x03E, 0x00000014}, + {0x03F, 0x00021C58}, + {0x033, 0x0000000B}, + {0x03E, 0x00000014}, + {0x03F, 0x00022B58}, + {0x033, 0x0000000C}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x0000000D}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x0000000E}, + {0x03E, 0x00000014}, + {0x03F, 0x00021C58}, + {0x033, 0x0000000F}, + {0x03E, 0x00000014}, + {0x03F, 0x00022B58}, + {0x033, 0x00000010}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x00000011}, + {0x03E, 0x0000001B}, + {0x03F, 0x00023C58}, + {0x033, 0x00000012}, + {0x03E, 0x00000014}, + {0x03F, 0x00021C58}, + {0x033, 0x00000013}, + {0x03E, 0x00000014}, + {0x03F, 0x00022B58}, + {0x033, 0x00000014}, + {0x03E, 0x00000013}, + {0x03F, 0x00023C58}, + {0x033, 0x00000015}, + {0x03E, 0x0000001B}, + {0x03F, 0x00025A58}, + {0x033, 0x00000016}, + {0x03E, 0x0000001C}, + {0x03F, 0x00021C58}, + {0x033, 0x00000017}, + {0x03E, 0x00000014}, + {0x03F, 0x00022A58}, + {0x033, 0x00000018}, + {0x03E, 0x00000013}, + {0x03F, 0x00025A58}, + {0x033, 0x00000019}, + {0x03E, 0x0000001B}, + {0x03F, 0x00025A58}, + {0x033, 0x0000001A}, + {0x03E, 0x00000014}, + {0x03F, 0x00022A58}, + {0x033, 0x0000001B}, + {0x03E, 0x00000014}, + {0x03F, 0x00022A58}, + {0x033, 0x0000001C}, + {0x03E, 0x00000014}, + {0x03F, 0x0002CD58}, + {0x033, 0x0000001D}, + {0x03E, 0x0000001B}, + {0x03F, 0x00025A58}, + {0x033, 0x0000001E}, + {0x03E, 0x00000013}, + {0x03F, 0x00021E58}, + {0x033, 0x0000001F}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, + {0x033, 0x00000020}, + {0x03E, 0x00000014}, + {0x03F, 0x0002CD58}, + {0x033, 0x00000021}, {0x03E, 0x0000001C}, {0x03F, 0x0002CD58}, {0x033, 0x00000022}, @@ -3831,6 +4379,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x030, 0x000300FF}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x030, 0x000300FF}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000300FF}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000300FF}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000300FF}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000300FF}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x030, 0x000300FF}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -3901,6 +4457,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x095, 0x00000008}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x095, 0x00000008}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x095, 0x00000008}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -3920,109 +4484,13 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0xB0000000, 0x00000000}, {0x0EE, 0x00001000}, {0x033, 0x00000020}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000052}, - {0x033, 0x00000024}, - {0x03F, 0x0000005A}, - {0x033, 0x00000028}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002C}, - {0x03F, 0x0000019C}, - {0x033, 0x00000030}, - {0x03F, 0x000001A4}, - {0x033, 0x00000034}, - {0x03F, 0x000001E7}, - {0x033, 0x00000038}, - {0x03F, 0x000002E7}, - {0x033, 0x0000003C}, - {0x03F, 0x000003E7}, - {0x033, 0x00000021}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000052}, - {0x033, 0x00000025}, - {0x03F, 0x0000005A}, - {0x033, 0x00000029}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002D}, - {0x03F, 0x0000019C}, - {0x033, 0x00000031}, - {0x03F, 0x000001A4}, - {0x033, 0x00000035}, - {0x03F, 0x000001E6}, - {0x033, 0x00000039}, - {0x03F, 0x000002E6}, - {0x033, 0x0000003D}, - {0x03F, 0x000003E6}, - {0x033, 0x00000022}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000052}, - {0x033, 0x00000026}, - {0x03F, 0x0000005A}, - {0x033, 0x0000002A}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002E}, - {0x03F, 0x0000019C}, - {0x033, 0x00000032}, - {0x03F, 0x000001A4}, - {0x033, 0x00000036}, - {0x03F, 0x000001E6}, - {0x033, 0x0000003A}, - {0x03F, 0x000002E6}, - {0x033, 0x0000003E}, - {0x03F, 0x000003E6}, - {0x033, 0x00000060}, - {0x03F, 0x00000052}, - {0x033, 0x00000064}, - {0x03F, 0x0000005A}, - {0x033, 0x00000068}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006C}, - {0x03F, 0x0000019C}, - {0x033, 0x00000070}, - {0x03F, 0x000001A4}, - {0x033, 0x00000074}, - {0x03F, 0x000001E6}, - {0x033, 0x00000078}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007C}, - {0x03F, 0x000003E6}, - {0x033, 0x00000061}, - {0x03F, 0x00000052}, - {0x033, 0x00000065}, - {0x03F, 0x0000005A}, - {0x033, 0x00000069}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006D}, - {0x03F, 0x0000019C}, - {0x033, 0x00000071}, - {0x03F, 0x000001A4}, - {0x033, 0x00000075}, - {0x03F, 0x000001E6}, - {0x033, 0x00000079}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007D}, - {0x03F, 0x000003E6}, - {0x033, 0x00000062}, - {0x03F, 0x00000052}, - {0x033, 0x00000066}, - {0x03F, 0x0000005A}, - {0x033, 0x0000006A}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006E}, - {0x03F, 0x0000019C}, - {0x033, 0x00000072}, - {0x03F, 0x000001A4}, - {0x033, 0x00000076}, - {0x03F, 0x000001E6}, - {0x033, 0x0000007A}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007E}, - {0x03F, 0x000003E6}, - {0x033, 0x00000063}, - {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, - {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, - {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, - {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000052}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000052}, @@ -4034,24 +4502,32 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x00000152}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000152}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x03F, 0x00000052}, {0xA0000000, 0x00000000}, {0x03F, 0x00000052}, {0xB0000000, 0x00000000}, - {0x033, 0x00000067}, + {0x033, 0x00000024}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000005A}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, @@ -4070,24 +4546,32 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000015A}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000015A}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x0000005A}, {0xA0000000, 0x00000000}, {0x03F, 0x0000005A}, {0xB0000000, 0x00000000}, - {0x033, 0x0000006B}, + {0x033, 0x00000028}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000009C}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, @@ -4106,24 +4590,32 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x0000019C}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000019C}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x0000009C}, {0xA0000000, 0x00000000}, {0x03F, 0x0000009C}, {0xB0000000, 0x00000000}, - {0x033, 0x0000006F}, + {0x033, 0x0000002C}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x0000019C}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, @@ -4142,24 +4634,32 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000001A4}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000001A4}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x0000019C}, {0xA0000000, 0x00000000}, {0x03F, 0x0000019C}, {0xB0000000, 0x00000000}, - {0x033, 0x00000073}, + {0x033, 0x00000030}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000001A4}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, @@ -4178,2976 +4678,8760 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x03F, 0x000001E6}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000001E6}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001A4}, {0xA0000000, 0x00000000}, {0x03F, 0x000001A4}, {0xB0000000, 0x00000000}, - {0x033, 0x00000077}, + {0x033, 0x00000034}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000002E6}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x000002E6}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x000001E7}, {0xA0000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x000001E7}, {0xB0000000, 0x00000000}, - {0x033, 0x0000007B}, + {0x033, 0x00000038}, {0x03F, 0x000002E7}, - {0x033, 0x0000007F}, + {0x033, 0x0000003C}, {0x03F, 0x000003E7}, - {0x0EE, 0x00000000}, - {0x100EE, 0x00004000}, + {0x033, 0x00000021}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000152}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000152}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0xA0000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000052}, {0xB0000000, 0x00000000}, - {0x100EE, 0x00000000}, - {0x100EE, 0x00002000}, - {0x10030, 0x000000FC}, - {0x10030, 0x000004F9}, - {0x10030, 0x000008F6}, - {0x10030, 0x00000CF3}, - {0x10030, 0x000010F0}, - {0x10030, 0x000014ED}, - {0x10030, 0x000018AC}, - {0x10030, 0x00001CA9}, - {0x10030, 0x00002069}, - {0x10030, 0x00002466}, - {0x10030, 0x00002829}, - {0x10030, 0x00002C26}, - {0x10030, 0x00003023}, - {0x10030, 0x00003420}, - {0x10030, 0x0000381D}, - {0x10030, 0x00003C1A}, - {0x10030, 0x00004017}, - {0x100EE, 0x00000000}, - {0x100EE, 0x00002000}, - {0x10030, 0x000780F4}, - {0x10030, 0x000784F1}, - {0x10030, 0x000788EE}, - {0x10030, 0x00078CEB}, - {0x10030, 0x000790E8}, - {0x10030, 0x000794E5}, - {0x10030, 0x000798E2}, - {0x10030, 0x00079CDF}, - {0x10030, 0x0007A0DC}, - {0x10030, 0x0007A4D9}, - {0x10030, 0x0007A8D6}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B0D0}, - {0x10030, 0x0007B4CD}, - {0x10030, 0x0007B8CA}, - {0x10030, 0x0007BC07}, - {0x10030, 0x0007C004}, - {0x100EE, 0x00000000}, - {0x0EF, 0x00002000}, - {0x033, 0x00000008}, - {0x03F, 0x00000004}, - {0x033, 0x00000009}, - {0x03F, 0x00000003}, - {0x033, 0x0000000A}, - {0x03F, 0x00000003}, - {0x033, 0x0000000B}, - {0x03F, 0x00000002}, - {0x033, 0x0000000C}, - {0x03F, 0x00000002}, - {0x033, 0x0000000D}, - {0x03F, 0x00000002}, - {0x033, 0x0000000E}, - {0x03F, 0x00000002}, - {0x033, 0x0000000F}, - {0x03F, 0x00000002}, - {0x0EF, 0x00000000}, - {0x0EB, 0x00040000}, - {0x030, 0x000109B7}, - {0x0EB, 0x00000000}, - {0x0EF, 0x00008000}, - {0x033, 0x00000020}, - {0x03F, 0x00050002}, - {0x033, 0x00000021}, - {0x03F, 0x00060032}, - {0x033, 0x00000022}, - {0x03F, 0x00050042}, - {0x033, 0x00000023}, - {0x03F, 0x00040042}, - {0x033, 0x00000024}, - {0x03F, 0x00008001}, {0x033, 0x00000025}, - {0x03F, 0x00008002}, - {0x033, 0x00000026}, - {0x03F, 0x00000003}, - {0x033, 0x00000027}, - {0x03F, 0x00000003}, - {0x033, 0x00000028}, - {0x03F, 0x00050002}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, {0x033, 0x00000029}, - {0x03F, 0x00060032}, - {0x033, 0x0000002A}, - {0x03F, 0x00050042}, - {0x033, 0x0000002B}, - {0x03F, 0x00040042}, - {0x033, 0x0000002C}, - {0x03F, 0x00008001}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, {0x033, 0x0000002D}, - {0x03F, 0x00008002}, - {0x033, 0x0000002E}, - {0x03F, 0x00000003}, - {0x033, 0x0000002F}, - {0x03F, 0x00000003}, - {0x033, 0x00000030}, - {0x03F, 0x00050002}, - {0x033, 0x00000031}, - {0x03F, 0x00060032}, - {0x033, 0x00000032}, - {0x03F, 0x00050042}, - {0x033, 0x00000033}, - {0x03F, 0x00040042}, - {0x033, 0x00000034}, - {0x03F, 0x00008001}, - {0x033, 0x00000035}, - {0x03F, 0x00008002}, - {0x033, 0x00000036}, - {0x03F, 0x00000003}, - {0x033, 0x00000037}, - {0x03F, 0x00000003}, - {0x033, 0x00000060}, - {0x03F, 0x00050002}, - {0x033, 0x00000061}, - {0x03F, 0x00060032}, - {0x033, 0x00000062}, - {0x03F, 0x00050042}, - {0x033, 0x00000063}, - {0x03F, 0x00040042}, - {0x033, 0x00000064}, - {0x03F, 0x00008001}, - {0x033, 0x00000065}, - {0x03F, 0x00008002}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000031}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000035}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000039}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000003D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000022}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000026}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000002A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000002E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000032}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000036}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000003A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000003E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000060}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000064}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000068}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006C}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000070}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000074}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000078}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007C}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000061}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000065}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000069}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000071}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000075}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000079}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000062}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000066}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000072}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000076}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000063}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000067}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006B}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000073}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000077}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007B}, + {0x03F, 0x000002E7}, + {0x033, 0x0000007F}, + {0x03F, 0x000003E7}, + {0x0EE, 0x00000000}, + {0x100EE, 0x00004000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701EF}, + {0x10030, 0x000705E7}, + {0x10030, 0x000709A7}, + {0x10030, 0x00070D61}, + {0x10030, 0x0007115B}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071CE5}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728A1}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781EF}, + {0x10030, 0x000785E9}, + {0x10030, 0x000789E3}, + {0x10030, 0x00078DA1}, + {0x10030, 0x0007915F}, + {0x10030, 0x00079559}, + {0x10030, 0x00079921}, + {0x10030, 0x00079D1B}, + {0x10030, 0x0007A0E3}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B823}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0xA0000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0xB0000000, 0x00000000}, + {0x100EE, 0x00000000}, + {0x100EE, 0x00002000}, + {0x10030, 0x000000FC}, + {0x10030, 0x000004F9}, + {0x10030, 0x000008F6}, + {0x10030, 0x00000CF3}, + {0x10030, 0x000010F0}, + {0x10030, 0x000014ED}, + {0x10030, 0x000018AC}, + {0x10030, 0x00001CA9}, + {0x10030, 0x00002069}, + {0x10030, 0x00002466}, + {0x10030, 0x00002829}, + {0x10030, 0x00002C26}, + {0x10030, 0x00003023}, + {0x10030, 0x00003420}, + {0x10030, 0x0000381D}, + {0x10030, 0x00003C1A}, + {0x10030, 0x00004017}, + {0x100EE, 0x00000000}, + {0x100EE, 0x00002000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, + {0xA0000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0xB0000000, 0x00000000}, + {0x10030, 0x000600F6}, + {0x10030, 0x000604F3}, + {0x10030, 0x000608F0}, + {0x10030, 0x00060CED}, + {0x10030, 0x000610EA}, + {0x10030, 0x000614E7}, + {0x10030, 0x000618E4}, + {0x10030, 0x00061CE1}, + {0x10030, 0x000620DE}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628D8}, + {0x10030, 0x00062CD5}, + {0x10030, 0x000630D2}, + {0x10030, 0x000634CF}, + {0x10030, 0x000638CC}, + {0x10030, 0x00063C09}, + {0x10030, 0x00064006}, + {0x10030, 0x000680F5}, + {0x10030, 0x000684F2}, + {0x10030, 0x000688EF}, + {0x10030, 0x00068CEC}, + {0x10030, 0x000690E9}, + {0x10030, 0x000694E6}, + {0x10030, 0x000698E3}, + {0x10030, 0x00069CE0}, + {0x10030, 0x0006A0DD}, + {0x10030, 0x0006A4DA}, + {0x10030, 0x0006A8D7}, + {0x10030, 0x0006ACD4}, + {0x10030, 0x0006B0D1}, + {0x10030, 0x0006B4CE}, + {0x10030, 0x0006B8CB}, + {0x10030, 0x0006BC08}, + {0x10030, 0x0006C005}, + {0x10030, 0x000700F5}, + {0x10030, 0x000704F2}, + {0x10030, 0x000708EF}, + {0x10030, 0x00070CEC}, + {0x10030, 0x000710E9}, + {0x10030, 0x000714E6}, + {0x10030, 0x000718E3}, + {0x10030, 0x00071CE0}, + {0x10030, 0x000720DD}, + {0x10030, 0x000724DA}, + {0x10030, 0x000728D7}, + {0x10030, 0x00072CD4}, + {0x10030, 0x000730D1}, + {0x10030, 0x000734CE}, + {0x10030, 0x000738CB}, + {0x10030, 0x00073C08}, + {0x10030, 0x00074005}, + {0x10030, 0x000780F4}, + {0x10030, 0x000784F1}, + {0x10030, 0x000788EE}, + {0x10030, 0x00078CEB}, + {0x10030, 0x000790E8}, + {0x10030, 0x000794E5}, + {0x10030, 0x000798E2}, + {0x10030, 0x00079CDF}, + {0x10030, 0x0007A0DC}, + {0x10030, 0x0007A4D9}, + {0x10030, 0x0007A8D6}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B0D0}, + {0x10030, 0x0007B4CD}, + {0x10030, 0x0007B8CA}, + {0x10030, 0x0007BC07}, + {0x10030, 0x0007C004}, + {0x100EE, 0x00000000}, + {0x0EF, 0x00002000}, + {0x033, 0x00000008}, + {0x03F, 0x00000004}, + {0x033, 0x00000009}, + {0x03F, 0x00000003}, + {0x033, 0x0000000A}, + {0x03F, 0x00000003}, + {0x033, 0x0000000B}, + {0x03F, 0x00000002}, + {0x033, 0x0000000C}, + {0x03F, 0x00000002}, + {0x033, 0x0000000D}, + {0x03F, 0x00000002}, + {0x033, 0x0000000E}, + {0x03F, 0x00000002}, + {0x033, 0x0000000F}, + {0x03F, 0x00000002}, + {0x0EF, 0x00000000}, + {0x0EB, 0x00040000}, + {0x030, 0x000109B7}, + {0x0EB, 0x00000000}, + {0x0EF, 0x00008000}, + {0x033, 0x00000020}, + {0x03F, 0x00050002}, + {0x033, 0x00000021}, + {0x03F, 0x00060032}, + {0x033, 0x00000022}, + {0x03F, 0x00050042}, + {0x033, 0x00000023}, + {0x03F, 0x00040042}, + {0x033, 0x00000024}, + {0x03F, 0x00008001}, + {0x033, 0x00000025}, + {0x03F, 0x00008002}, + {0x033, 0x00000026}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000027}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000028}, + {0x03F, 0x00050002}, + {0x033, 0x00000029}, + {0x03F, 0x00060032}, + {0x033, 0x0000002A}, + {0x03F, 0x00050042}, + {0x033, 0x0000002B}, + {0x03F, 0x00040042}, + {0x033, 0x0000002C}, + {0x03F, 0x00008001}, + {0x033, 0x0000002D}, + {0x03F, 0x00008002}, + {0x033, 0x0000002E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000002F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000030}, + {0x03F, 0x00050002}, + {0x033, 0x00000031}, + {0x03F, 0x00060032}, + {0x033, 0x00000032}, + {0x03F, 0x00050042}, + {0x033, 0x00000033}, + {0x03F, 0x00040042}, + {0x033, 0x00000034}, + {0x03F, 0x00008001}, + {0x033, 0x00000035}, + {0x03F, 0x00008002}, + {0x033, 0x00000036}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000037}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000060}, + {0x03F, 0x00050002}, + {0x033, 0x00000061}, + {0x03F, 0x00060032}, + {0x033, 0x00000062}, + {0x03F, 0x00050042}, + {0x033, 0x00000063}, + {0x03F, 0x00040042}, + {0x033, 0x00000064}, + {0x03F, 0x00008001}, + {0x033, 0x00000065}, + {0x03F, 0x00008002}, {0x033, 0x00000066}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000067}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000068}, + {0x03F, 0x00050002}, + {0x033, 0x00000069}, + {0x03F, 0x00060032}, + {0x033, 0x0000006A}, + {0x03F, 0x00050042}, + {0x033, 0x0000006B}, + {0x03F, 0x00040042}, + {0x033, 0x0000006C}, + {0x03F, 0x00008001}, + {0x033, 0x0000006D}, + {0x03F, 0x00008002}, + {0x033, 0x0000006E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000070}, + {0x03F, 0x00050002}, + {0x033, 0x00000071}, + {0x03F, 0x00060032}, + {0x033, 0x00000072}, + {0x03F, 0x00050042}, + {0x033, 0x00000073}, + {0x03F, 0x00040042}, + {0x033, 0x00000074}, + {0x03F, 0x00008001}, + {0x033, 0x00000075}, + {0x03F, 0x00008002}, + {0x033, 0x00000076}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000077}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000078}, + {0x03F, 0x00050002}, + {0x033, 0x00000079}, + {0x03F, 0x00060032}, + {0x033, 0x0000007A}, + {0x03F, 0x00050042}, + {0x033, 0x0000007B}, + {0x03F, 0x00040042}, + {0x033, 0x0000007C}, + {0x03F, 0x00008001}, + {0x033, 0x0000007D}, + {0x03F, 0x00008002}, + {0x033, 0x0000007E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A0}, + {0x03F, 0x00050002}, + {0x033, 0x000000A1}, + {0x03F, 0x00060032}, + {0x033, 0x000000A2}, + {0x03F, 0x00050042}, + {0x033, 0x000000A3}, + {0x03F, 0x00040042}, + {0x033, 0x000000A4}, + {0x03F, 0x00008001}, + {0x033, 0x000000A5}, + {0x03F, 0x00008002}, + {0x033, 0x000000A6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A8}, + {0x03F, 0x00050002}, + {0x033, 0x000000A9}, + {0x03F, 0x00060032}, + {0x033, 0x000000AA}, + {0x03F, 0x00050042}, + {0x033, 0x000000AB}, + {0x03F, 0x00040042}, + {0x033, 0x000000AC}, + {0x03F, 0x00008001}, + {0x033, 0x000000AD}, + {0x03F, 0x00008002}, + {0x033, 0x000000AE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000AF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000B0}, + {0x03F, 0x00050002}, + {0x033, 0x000000B1}, + {0x03F, 0x00060032}, + {0x033, 0x000000B2}, + {0x03F, 0x00050042}, + {0x033, 0x000000B3}, + {0x03F, 0x00040042}, + {0x033, 0x000000B4}, + {0x03F, 0x00008001}, + {0x033, 0x000000B5}, + {0x03F, 0x00008002}, + {0x033, 0x000000B6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000B7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E0}, + {0x03F, 0x00050002}, + {0x033, 0x000000E1}, + {0x03F, 0x00060032}, + {0x033, 0x000000E2}, + {0x03F, 0x00050042}, + {0x033, 0x000000E3}, + {0x03F, 0x00040042}, + {0x033, 0x000000E4}, + {0x03F, 0x00008001}, + {0x033, 0x000000E5}, + {0x03F, 0x00008002}, + {0x033, 0x000000E6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E8}, + {0x03F, 0x00050002}, + {0x033, 0x000000E9}, + {0x03F, 0x00060032}, + {0x033, 0x000000EA}, + {0x03F, 0x00050042}, + {0x033, 0x000000EB}, + {0x03F, 0x00040042}, + {0x033, 0x000000EC}, + {0x03F, 0x00008001}, + {0x033, 0x000000ED}, + {0x03F, 0x00008002}, + {0x033, 0x000000EE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000EF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F0}, + {0x03F, 0x00050002}, + {0x033, 0x000000F1}, + {0x03F, 0x00060032}, + {0x033, 0x000000F2}, + {0x03F, 0x00050042}, + {0x033, 0x000000F3}, + {0x03F, 0x00040042}, + {0x033, 0x000000F4}, + {0x03F, 0x00008001}, + {0x033, 0x000000F5}, + {0x03F, 0x00008002}, + {0x033, 0x000000F6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F8}, + {0x03F, 0x00050002}, + {0x033, 0x000000F9}, + {0x03F, 0x00060032}, + {0x033, 0x000000FA}, + {0x03F, 0x00050042}, + {0x033, 0x000000FB}, + {0x03F, 0x00040042}, + {0x033, 0x000000FC}, + {0x03F, 0x00008001}, + {0x033, 0x000000FD}, + {0x03F, 0x00008002}, + {0x033, 0x000000FE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000FF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000120}, + {0x03F, 0x00050002}, + {0x033, 0x00000121}, + {0x03F, 0x00060032}, + {0x033, 0x00000122}, + {0x03F, 0x00050042}, + {0x033, 0x00000123}, + {0x03F, 0x00040042}, + {0x033, 0x00000124}, + {0x03F, 0x00008001}, + {0x033, 0x00000125}, + {0x03F, 0x00008002}, + {0x033, 0x00000126}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000127}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000128}, + {0x03F, 0x00050002}, + {0x033, 0x00000129}, + {0x03F, 0x00060032}, + {0x033, 0x0000012A}, + {0x03F, 0x00050042}, + {0x033, 0x0000012B}, + {0x03F, 0x00040042}, + {0x033, 0x0000012C}, + {0x03F, 0x00008001}, + {0x033, 0x0000012D}, + {0x03F, 0x00008002}, + {0x033, 0x0000012E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000012F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000130}, + {0x03F, 0x00050002}, + {0x033, 0x00000131}, + {0x03F, 0x00060032}, + {0x033, 0x00000132}, + {0x03F, 0x00050042}, + {0x033, 0x00000133}, + {0x03F, 0x00040042}, + {0x033, 0x00000134}, + {0x03F, 0x00008001}, + {0x033, 0x00000135}, + {0x03F, 0x00008002}, + {0x033, 0x00000136}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000137}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000160}, + {0x03F, 0x00050002}, + {0x033, 0x00000161}, + {0x03F, 0x00060032}, + {0x033, 0x00000162}, + {0x03F, 0x00050042}, + {0x033, 0x00000163}, + {0x03F, 0x00040042}, + {0x033, 0x00000164}, + {0x03F, 0x00008001}, + {0x033, 0x00000165}, + {0x03F, 0x00008002}, + {0x033, 0x00000166}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000167}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000168}, + {0x03F, 0x00050002}, + {0x033, 0x00000169}, + {0x03F, 0x00060032}, + {0x033, 0x0000016A}, + {0x03F, 0x00050042}, + {0x033, 0x0000016B}, + {0x03F, 0x00040042}, + {0x033, 0x0000016C}, + {0x03F, 0x00008001}, + {0x033, 0x0000016D}, + {0x03F, 0x00008002}, + {0x033, 0x0000016E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000016F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000170}, + {0x03F, 0x00050002}, + {0x033, 0x00000171}, + {0x03F, 0x00060032}, + {0x033, 0x00000172}, + {0x03F, 0x00050042}, + {0x033, 0x00000173}, + {0x03F, 0x00040042}, + {0x033, 0x00000174}, + {0x03F, 0x00008001}, + {0x033, 0x00000175}, + {0x03F, 0x00008002}, + {0x033, 0x00000176}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000177}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000067}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000068}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000178}, {0x03F, 0x00050002}, - {0x033, 0x00000069}, + {0x033, 0x00000179}, {0x03F, 0x00060032}, - {0x033, 0x0000006A}, + {0x033, 0x0000017A}, {0x03F, 0x00050042}, - {0x033, 0x0000006B}, + {0x033, 0x0000017B}, {0x03F, 0x00040042}, - {0x033, 0x0000006C}, + {0x033, 0x0000017C}, {0x03F, 0x00008001}, - {0x033, 0x0000006D}, + {0x033, 0x0000017D}, {0x03F, 0x00008002}, - {0x033, 0x0000006E}, + {0x033, 0x0000017E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000006F}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000070}, - {0x03F, 0x00050002}, - {0x033, 0x00000071}, - {0x03F, 0x00060032}, - {0x033, 0x00000072}, - {0x03F, 0x00050042}, - {0x033, 0x00000073}, - {0x03F, 0x00040042}, - {0x033, 0x00000074}, - {0x03F, 0x00008001}, - {0x033, 0x00000075}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000076}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000077}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000078}, - {0x03F, 0x00050002}, - {0x033, 0x00000079}, - {0x03F, 0x00060032}, - {0x033, 0x0000007A}, - {0x03F, 0x00050042}, - {0x033, 0x0000007B}, - {0x03F, 0x00040042}, - {0x033, 0x0000007C}, - {0x03F, 0x00008001}, - {0x033, 0x0000007D}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000007E}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000007F}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000017F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000A0}, - {0x03F, 0x00050002}, - {0x033, 0x000000A1}, - {0x03F, 0x00060032}, - {0x033, 0x000000A2}, - {0x03F, 0x00050042}, - {0x033, 0x000000A3}, - {0x03F, 0x00040042}, - {0x033, 0x000000A4}, - {0x03F, 0x00008001}, - {0x033, 0x000000A5}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000A6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000A7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000A8}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001A0}, {0x03F, 0x00050002}, - {0x033, 0x000000A9}, + {0x033, 0x000001A1}, {0x03F, 0x00060032}, - {0x033, 0x000000AA}, + {0x033, 0x000001A2}, {0x03F, 0x00050042}, - {0x033, 0x000000AB}, + {0x033, 0x000001A3}, {0x03F, 0x00040042}, - {0x033, 0x000000AC}, + {0x033, 0x000001A4}, {0x03F, 0x00008001}, - {0x033, 0x000000AD}, + {0x033, 0x000001A5}, {0x03F, 0x00008002}, - {0x033, 0x000000AE}, + {0x033, 0x000001A6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000AF}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000B0}, - {0x03F, 0x00050002}, - {0x033, 0x000000B1}, - {0x03F, 0x00060032}, - {0x033, 0x000000B2}, - {0x03F, 0x00050042}, - {0x033, 0x000000B3}, - {0x03F, 0x00040042}, - {0x033, 0x000000B4}, - {0x03F, 0x00008001}, - {0x033, 0x000000B5}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000B6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000B7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E0}, - {0x03F, 0x00050002}, - {0x033, 0x000000E1}, - {0x03F, 0x00060032}, - {0x033, 0x000000E2}, - {0x03F, 0x00050042}, - {0x033, 0x000000E3}, - {0x03F, 0x00040042}, - {0x033, 0x000000E4}, - {0x03F, 0x00008001}, - {0x033, 0x000000E5}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E7}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001A7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E8}, - {0x03F, 0x00050002}, - {0x033, 0x000000E9}, - {0x03F, 0x00060032}, - {0x033, 0x000000EA}, - {0x03F, 0x00050042}, - {0x033, 0x000000EB}, - {0x03F, 0x00040042}, - {0x033, 0x000000EC}, - {0x03F, 0x00008001}, - {0x033, 0x000000ED}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000EE}, - {0x03F, 0x00000003}, - {0x033, 0x000000EF}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F0}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001A8}, {0x03F, 0x00050002}, - {0x033, 0x000000F1}, + {0x033, 0x000001A9}, {0x03F, 0x00060032}, - {0x033, 0x000000F2}, + {0x033, 0x000001AA}, {0x03F, 0x00050042}, - {0x033, 0x000000F3}, + {0x033, 0x000001AB}, {0x03F, 0x00040042}, - {0x033, 0x000000F4}, + {0x033, 0x000001AC}, {0x03F, 0x00008001}, - {0x033, 0x000000F5}, + {0x033, 0x000001AD}, {0x03F, 0x00008002}, - {0x033, 0x000000F6}, + {0x033, 0x000001AE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F8}, - {0x03F, 0x00050002}, - {0x033, 0x000000F9}, - {0x03F, 0x00060032}, - {0x033, 0x000000FA}, - {0x03F, 0x00050042}, - {0x033, 0x000000FB}, - {0x03F, 0x00040042}, - {0x033, 0x000000FC}, - {0x03F, 0x00008001}, - {0x033, 0x000000FD}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000FE}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000FF}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000120}, - {0x03F, 0x00050002}, - {0x033, 0x00000121}, - {0x03F, 0x00060032}, - {0x033, 0x00000122}, - {0x03F, 0x00050042}, - {0x033, 0x00000123}, - {0x03F, 0x00040042}, - {0x033, 0x00000124}, - {0x03F, 0x00008001}, - {0x033, 0x00000125}, - {0x03F, 0x00008002}, - {0x033, 0x00000126}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000127}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000128}, - {0x03F, 0x00050002}, - {0x033, 0x00000129}, - {0x03F, 0x00060032}, - {0x033, 0x0000012A}, - {0x03F, 0x00050042}, - {0x033, 0x0000012B}, - {0x03F, 0x00040042}, - {0x033, 0x0000012C}, - {0x03F, 0x00008001}, - {0x033, 0x0000012D}, - {0x03F, 0x00008002}, - {0x033, 0x0000012E}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000012F}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000130}, - {0x03F, 0x00050002}, - {0x033, 0x00000131}, - {0x03F, 0x00060032}, - {0x033, 0x00000132}, - {0x03F, 0x00050042}, - {0x033, 0x00000133}, - {0x03F, 0x00040042}, - {0x033, 0x00000134}, - {0x03F, 0x00008001}, - {0x033, 0x00000135}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000136}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000137}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001AF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000160}, - {0x03F, 0x00050002}, - {0x033, 0x00000161}, - {0x03F, 0x00060032}, - {0x033, 0x00000162}, - {0x03F, 0x00050042}, - {0x033, 0x00000163}, - {0x03F, 0x00040042}, - {0x033, 0x00000164}, - {0x03F, 0x00008001}, - {0x033, 0x00000165}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000166}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000167}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000168}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001B0}, {0x03F, 0x00050002}, - {0x033, 0x00000169}, + {0x033, 0x000001B1}, {0x03F, 0x00060032}, - {0x033, 0x0000016A}, + {0x033, 0x000001B2}, {0x03F, 0x00050042}, - {0x033, 0x0000016B}, + {0x033, 0x000001B3}, {0x03F, 0x00040042}, - {0x033, 0x0000016C}, + {0x033, 0x000001B4}, {0x03F, 0x00008001}, - {0x033, 0x0000016D}, + {0x033, 0x000001B5}, {0x03F, 0x00008002}, - {0x033, 0x0000016E}, + {0x033, 0x000001B6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000016F}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000170}, - {0x03F, 0x00050002}, - {0x033, 0x00000171}, - {0x03F, 0x00060032}, - {0x033, 0x00000172}, - {0x03F, 0x00050042}, - {0x033, 0x00000173}, - {0x03F, 0x00040042}, - {0x033, 0x00000174}, - {0x03F, 0x00008001}, - {0x033, 0x00000175}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000176}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000177}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000178}, - {0x03F, 0x00050002}, - {0x033, 0x00000179}, - {0x03F, 0x00060032}, - {0x033, 0x0000017A}, - {0x03F, 0x00050042}, - {0x033, 0x0000017B}, - {0x03F, 0x00040042}, - {0x033, 0x0000017C}, - {0x03F, 0x00008001}, - {0x033, 0x0000017D}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000017E}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000017F}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001B7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A0}, - {0x03F, 0x00050002}, - {0x033, 0x000001A1}, - {0x03F, 0x00060032}, - {0x033, 0x000001A2}, - {0x03F, 0x00050042}, - {0x033, 0x000001A3}, - {0x03F, 0x00040042}, - {0x033, 0x000001A4}, - {0x03F, 0x00008001}, - {0x033, 0x000001A5}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001A6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A8}, - {0x03F, 0x00050002}, - {0x033, 0x000001A9}, - {0x03F, 0x00060032}, - {0x033, 0x000001AA}, - {0x03F, 0x00050042}, - {0x033, 0x000001AB}, - {0x03F, 0x00040042}, - {0x033, 0x000001AC}, - {0x03F, 0x00008001}, - {0x033, 0x000001AD}, - {0x03F, 0x00008002}, - {0x033, 0x000001AE}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001AF}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001B0}, - {0x03F, 0x00050002}, - {0x033, 0x000001B1}, - {0x03F, 0x00060032}, - {0x033, 0x000001B2}, - {0x03F, 0x00050042}, - {0x033, 0x000001B3}, - {0x03F, 0x00040042}, - {0x033, 0x000001B4}, - {0x03F, 0x00008001}, - {0x033, 0x000001B5}, - {0x03F, 0x00008002}, - {0x033, 0x000001B6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001B7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001E0}, {0x03F, 0x00050002}, {0x033, 0x000001E1}, @@ -7161,9 +13445,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x033, 0x000001E5}, {0x03F, 0x00008002}, {0x033, 0x000001E6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001E7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001E8}, {0x03F, 0x00050002}, {0x033, 0x000001E9}, @@ -7177,9 +13545,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x033, 0x000001ED}, {0x03F, 0x00008002}, {0x033, 0x000001EE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001EF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001F0}, {0x03F, 0x00050002}, {0x033, 0x000001F1}, @@ -7193,9 +13645,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x033, 0x000001F5}, {0x03F, 0x00008002}, {0x033, 0x000001F6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001F7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001F7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001F8}, {0x03F, 0x00050002}, {0x033, 0x000001F9}, @@ -7209,9 +13745,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x033, 0x000001FD}, {0x03F, 0x00008002}, {0x033, 0x000001FE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x033, 0x000001FF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x0EF, 0x00000000}, {0x005, 0x00000001}, {0x10005, 0x00000001}, @@ -7253,7 +13873,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x10030, 0x00022000}, {0x10030, 0x00023000}, {0x10030, 0x00024000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x10030, 0x00025000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0xA0000000, 0x00000000}, + {0x10030, 0x00025000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00026003}, {0x10030, 0x00027003}, {0x10030, 0x00028000}, @@ -7261,7 +13923,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x10030, 0x0002A000}, {0x10030, 0x0002B000}, {0x10030, 0x0002C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0002D000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0002E003}, {0x10030, 0x0002F003}, {0x10030, 0x00030000}, @@ -7269,7 +13973,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x10030, 0x00032000}, {0x10030, 0x00033000}, {0x10030, 0x00034000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00035000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00036003}, {0x10030, 0x00037003}, {0x10030, 0x00038000}, @@ -7277,7 +14023,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x10030, 0x0003A000}, {0x10030, 0x0003B000}, {0x10030, 0x0003C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0003D000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0003E003}, {0x10030, 0x0003F003}, {0x10030, 0x00060000}, @@ -7285,35 +14073,283 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = { {0x10030, 0x00062000}, {0x10030, 0x00063000}, {0x10030, 0x00064000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00065000}, {0x10030, 0x00066000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00067003}, {0x10030, 0x00068000}, {0x10030, 0x00069000}, {0x10030, 0x0006A000}, {0x10030, 0x0006B000}, {0x10030, 0x0006C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0006D000}, {0x10030, 0x0006E000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0006F003}, {0x10030, 0x00070000}, {0x10030, 0x00071000}, {0x10030, 0x00072000}, {0x10030, 0x00073000}, {0x10030, 0x00074000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00075000}, {0x10030, 0x00076000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00077003}, {0x10030, 0x00078000}, {0x10030, 0x00079000}, {0x10030, 0x0007A000}, {0x10030, 0x0007B000}, {0x10030, 0x0007C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0007D000}, {0x10030, 0x0007E000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0007F003}, {0x100EE, 0x00000000}, - {0x0FE, 0x00000031}, + {0x0FE, 0x00000048}, }; static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { @@ -7326,13 +14362,17 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0xF0360000, 0x00000006}, {0xF0010001, 0x00000007}, {0xF0020001, 0x00000008}, - {0xF0320001, 0x00000009}, - {0xF0330001, 0x0000000A}, - {0xF0340001, 0x0000000B}, - {0xF0350001, 0x0000000C}, - {0xF0360001, 0x0000000D}, - {0xF03F0001, 0x0000000E}, - {0xF0400001, 0x0000000F}, + {0xF0030001, 0x00000009}, + {0xF0040001, 0x0000000A}, + {0xF0050001, 0x0000000B}, + {0xF0070001, 0x0000000C}, + {0xF0320001, 0x0000000D}, + {0xF0330001, 0x0000000E}, + {0xF0340001, 0x0000000F}, + {0xF0350001, 0x00000010}, + {0xF0360001, 0x00000011}, + {0xF03F0001, 0x00000012}, + {0xF0400001, 0x00000013}, {0x005, 0x00000000}, {0x10005, 0x00000000}, {0x0B9, 0x00020440}, @@ -7340,42 +14380,69 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10000, 0x00030000}, {0x018, 0x00011124}, {0x10018, 0x00011124}, - {0x05F, 0x00000032}, + {0x05F, 0x00000038}, {0x097, 0x00043200}, {0x0A6, 0x00066DB7}, {0x0EF, 0x00004000}, {0x033, 0x00000005}, {0x03E, 0x00000000}, {0x03F, 0x00010500}, + {0x033, 0x00000004}, + {0x03E, 0x00000000}, + {0x03F, 0x00000400}, {0x033, 0x00000003}, {0x03E, 0x00000000}, {0x03F, 0x00028B00}, {0x033, 0x00000002}, {0x03E, 0x00000000}, {0x03F, 0x0009AB00}, + {0x033, 0x00000001}, + {0x03E, 0x00000000}, + {0x03F, 0x00001A00}, + {0x033, 0x00000000}, + {0x03E, 0x00000000}, + {0x03F, 0x00002900}, {0x033, 0x0000000D}, {0x03E, 0x00000000}, {0x03F, 0x00010500}, + {0x033, 0x0000000C}, + {0x03E, 0x00000000}, + {0x03F, 0x00000400}, {0x033, 0x0000000B}, {0x03E, 0x00000000}, {0x03F, 0x00028B00}, {0x033, 0x0000000A}, {0x03E, 0x00000000}, {0x03F, 0x0009AB00}, + {0x033, 0x00000009}, + {0x03E, 0x00000000}, + {0x03F, 0x00001A00}, + {0x033, 0x00000008}, + {0x03E, 0x00000000}, + {0x03F, 0x00002900}, {0x033, 0x00000015}, {0x03E, 0x00000000}, {0x03F, 0x00010500}, + {0x033, 0x00000014}, + {0x03E, 0x00000000}, + {0x03F, 0x00000400}, {0x033, 0x00000013}, {0x03E, 0x00000000}, {0x03F, 0x00028B00}, {0x033, 0x00000012}, {0x03E, 0x00000000}, {0x03F, 0x0009AB00}, + {0x033, 0x00000011}, + {0x03E, 0x00000000}, + {0x03F, 0x00001A00}, + {0x033, 0x00000010}, + {0x03E, 0x00000000}, + {0x03F, 0x00002900}, {0x0EF, 0x00000000}, + {0x10055, 0x00080080}, {0x000, 0x00033C01}, {0x10000, 0x00033C00}, {0x01A, 0x00040004}, - {0x0FE, 0x00000000}, {0x096, 0x00015200}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0004D000}, @@ -7404,6 +14471,18 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0000D300}, {0x0DA, 0x000D4000}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x067, 0x0000D300}, + {0x0DA, 0x000D4000}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x067, 0x0000D300}, {0x0DA, 0x000D4000}, @@ -7430,7 +14509,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x0DA, 0x000D4009}, {0xB0000000, 0x00000000}, {0x057, 0x0000D589}, - {0x05A, 0x0007FFFF}, + {0x05A, 0x0007F0F8}, {0x043, 0x00005000}, {0x018, 0x00001001}, {0x10018, 0x00001001}, @@ -7462,6 +14541,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x08F, 0x000D1352}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x08F, 0x000D1352}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x08F, 0x000D1352}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, {0x08F, 0x000D1352}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, @@ -7496,6 +14583,52 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x03F, 0x00000015}, {0x033, 0x00000001}, {0x03F, 0x00000017}, + {0x033, 0x00000004}, + {0x03F, 0x00000017}, + {0x033, 0x00000005}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000007}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000017}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000007}, + {0xB0000000, 0x00000000}, {0x0EF, 0x00000000}, {0x0EF, 0x00008000}, {0x033, 0x00000020}, @@ -7947,3828 +15080,10246 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x033, 0x000001FF}, {0x03F, 0x00000003}, {0x0EF, 0x00000000}, - {0x0EF, 0x00000100}, + {0x0EF, 0x00000100}, + {0x033, 0x00000001}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000002}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000003}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000004}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000005}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000006}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000007}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000008}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000009}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000000A}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000000B}, + {0x03F, 0x0000AFFF}, + {0x033, 0x0000000C}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000000D}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000000E}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000000F}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000010}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000011}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000012}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000013}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000014}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000015}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000EFFF}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000E3FF}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000016}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000017}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000018}, + {0x03F, 0x0000FBFF}, + {0x033, 0x00000019}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001A}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001B}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001C}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001D}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001E}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000001F}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000020}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000021}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000022}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000023}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000024}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000025}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000026}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000027}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000028}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000029}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002A}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002B}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002C}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002D}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002E}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000002F}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000030}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000031}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000032}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000033}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000034}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000035}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000036}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000037}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000038}, + {0x03F, 0x0000EFFF}, + {0x033, 0x00000039}, + {0x03F, 0x0000EFFF}, + {0x033, 0x0000003A}, + {0x03F, 0x0000EFFF}, + {0x0EF, 0x00000000}, + {0x0EF, 0x00000040}, + {0x033, 0x00000000}, + {0x03F, 0x00004344}, {0x033, 0x00000001}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000002}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000003}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000004}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000005}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000006}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000007}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000008}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x00000009}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, {0x033, 0x0000000A}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00004344}, + {0x033, 0x0000000B}, + {0x03F, 0x00004344}, + {0x033, 0x00000010}, + {0x03F, 0x00004344}, + {0x033, 0x00000011}, + {0x03F, 0x00004344}, + {0x033, 0x00000012}, + {0x03F, 0x00004344}, + {0x033, 0x00000013}, + {0x03F, 0x00004344}, + {0x033, 0x00000014}, + {0x03F, 0x00004344}, + {0x033, 0x00000015}, + {0x03F, 0x00004344}, + {0x033, 0x00000016}, + {0x03F, 0x00004344}, + {0x033, 0x00000017}, + {0x03F, 0x00004344}, + {0x033, 0x00000018}, + {0x03F, 0x00004344}, + {0x033, 0x00000019}, + {0x03F, 0x00004344}, + {0x033, 0x0000001A}, + {0x03F, 0x00004344}, + {0x033, 0x0000001B}, + {0x03F, 0x00004344}, + {0x033, 0x0000001C}, + {0x03F, 0x00004344}, + {0x033, 0x0000001D}, + {0x03F, 0x00004344}, + {0x033, 0x0000001E}, + {0x03F, 0x00004344}, + {0x033, 0x0000001F}, + {0x03F, 0x00004344}, + {0x0EF, 0x00000000}, + {0x0EF, 0x00000020}, + {0x033, 0x00000010}, + {0x03F, 0x00000200}, + {0x033, 0x00000011}, + {0x03F, 0x00000200}, + {0x033, 0x00000012}, + {0x03F, 0x00000200}, + {0x033, 0x00000013}, + {0x03F, 0x00000200}, + {0x033, 0x00000020}, + {0x03F, 0x00000200}, + {0x033, 0x00000021}, + {0x03F, 0x00000200}, + {0x033, 0x00000022}, + {0x03F, 0x00000200}, + {0x033, 0x00000023}, + {0x03F, 0x00000200}, + {0x0EF, 0x00000000}, + {0x0EF, 0x00000010}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x030, 0x000085ED}, + {0x030, 0x000105CC}, + {0x030, 0x000184AA}, + {0x030, 0x00020388}, + {0x030, 0x00028377}, + {0x030, 0x00030377}, + {0x030, 0x00038255}, + {0x030, 0x00040244}, + {0x030, 0x00048133}, + {0x030, 0x00050112}, + {0x030, 0x00058101}, + {0x030, 0x00060001}, + {0xA0000000, 0x00000000}, + {0x030, 0x000084DC}, + {0x030, 0x000103C9}, + {0x030, 0x00018399}, + {0x030, 0x00020287}, + {0x030, 0x00028277}, + {0x030, 0x00030165}, + {0x030, 0x00038144}, + {0x030, 0x00040044}, + {0x030, 0x00048022}, + {0x030, 0x00050011}, + {0x030, 0x00058000}, + {0x030, 0x00060000}, + {0xB0000000, 0x00000000}, + {0x030, 0x00068000}, + {0x030, 0x00070000}, + {0x0EF, 0x00000000}, + {0x0EF, 0x00000080}, + {0x033, 0x00000004}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, + {0x033, 0x00000005}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, + {0x033, 0x00000006}, + {0x03E, 0x00000014}, + {0x03F, 0x00023958}, + {0x033, 0x00000007}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000008}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, + {0x033, 0x00000009}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, + {0x033, 0x0000000A}, + {0x03E, 0x00000014}, + {0x03F, 0x00023958}, {0x033, 0x0000000B}, - {0x03F, 0x0000AFFF}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, {0x033, 0x0000000C}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, {0x033, 0x0000000D}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, {0x033, 0x0000000E}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000014}, + {0x03F, 0x00023958}, {0x033, 0x0000000F}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, {0x033, 0x00000010}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, {0x033, 0x00000011}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x0000001B}, + {0x03F, 0x00022A58}, {0x033, 0x00000012}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000014}, + {0x03F, 0x00023958}, {0x033, 0x00000013}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, {0x033, 0x00000014}, - {0x03F, 0x0000EFFF}, + {0x03E, 0x00000013}, + {0x03F, 0x00022A58}, {0x033, 0x00000015}, + {0x03E, 0x0000001B}, + {0x03F, 0x00029858}, + {0x033, 0x00000016}, + {0x03E, 0x0000001C}, + {0x03F, 0x00023958}, + {0x033, 0x00000017}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000018}, + {0x03E, 0x00000013}, + {0x03F, 0x00029858}, + {0x033, 0x00000019}, + {0x03E, 0x0000001B}, + {0x03F, 0x00029858}, + {0x033, 0x0000001A}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000001B}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000001C}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000001D}, + {0x03E, 0x0000001B}, + {0x03F, 0x00029858}, + {0x033, 0x0000001E}, + {0x03E, 0x00000013}, + {0x03F, 0x00023A58}, + {0x033, 0x0000001F}, + {0x03E, 0x00000013}, + {0x03F, 0x00023A58}, + {0x033, 0x00000020}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000021}, + {0x03E, 0x0000001C}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000022}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000023}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000024}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000025}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000026}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000027}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000028}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000029}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000002A}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000002B}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000002C}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000002D}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000002E}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000002F}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000030}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000031}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000032}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000033}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000034}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000035}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000036}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000037}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x00000038}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x00000039}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000003A}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000003B}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000003C}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000003D}, + {0x03E, 0x00000014}, + {0x03F, 0x0002AC58}, + {0x033, 0x0000003E}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x033, 0x0000003F}, + {0x03E, 0x00000014}, + {0x03F, 0x00023A58}, + {0x0EF, 0x00000000}, + {0x0EE, 0x00000800}, + {0x033, 0x00000000}, + {0x03F, 0x00000031}, + {0x033, 0x00000001}, + {0x03F, 0x00000023}, + {0x033, 0x00000002}, + {0x03F, 0x00000015}, + {0x033, 0x00000003}, + {0x03F, 0x00000007}, + {0x0EE, 0x00000000}, + {0x0EC, 0x00000400}, + {0x033, 0x00000003}, + {0x03F, 0x00000030}, + {0x033, 0x00000004}, + {0x03F, 0x00000021}, + {0x0EC, 0x00000000}, + {0x0DE, 0x00000000}, + {0x0EF, 0x00000000}, + {0x033, 0x00000000}, + {0x008, 0x00060280}, + {0x009, 0x00030400}, + {0x0EF, 0x00000000}, + {0x0A7, 0x00080308}, + {0x066, 0x00006000}, + {0x0EF, 0x00000400}, + {0x030, 0x000001FF}, + {0x030, 0x000081FF}, + {0x030, 0x000101FF}, + {0x030, 0x000181FF}, + {0x030, 0x000201FF}, + {0x030, 0x000281FF}, + {0x030, 0x0003017F}, + {0x030, 0x000380FB}, + {0x0EF, 0x00000000}, + {0x06E, 0x00077A18}, + {0x06D, 0x00000C31}, + {0x06A, 0x000E0F8A}, + {0x06B, 0x000018A0}, + {0x06F, 0x000F81FC}, + {0x05E, 0x0000001F}, + {0x0EF, 0x00000200}, + {0x030, 0x0003D407}, + {0x030, 0x00035A87}, + {0x030, 0x0002CF07}, + {0x030, 0x00024F07}, + {0x030, 0x0001CF07}, + {0x030, 0x00014F07}, + {0x030, 0x0000CF07}, + {0x030, 0x00004F07}, + {0x0EF, 0x00000000}, + {0x0EB, 0x00080000}, + {0x030, 0x00008038}, + {0x030, 0x00010038}, + {0x030, 0x00018038}, + {0x030, 0x00020038}, + {0x030, 0x00028038}, + {0x030, 0x00030038}, + {0x030, 0x0003803C}, + {0x030, 0x0004003C}, + {0x030, 0x0004803C}, + {0x030, 0x0005003C}, + {0x030, 0x0005803C}, + {0x030, 0x0006003C}, + {0x030, 0x0006803C}, + {0x030, 0x0007003C}, + {0x0EB, 0x00000000}, + {0x094, 0x000000FC}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x095, 0x00000008}, + {0xA0000000, 0x00000000}, + {0x095, 0x00000000}, + {0xB0000000, 0x00000000}, + {0x0EE, 0x00001000}, + {0x033, 0x00000020}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000152}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000152}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000EFFF}, + {0x03F, 0x00000052}, {0xA0000000, 0x00000000}, - {0x03F, 0x0000E3FF}, + {0x03F, 0x00000052}, {0xB0000000, 0x00000000}, - {0x033, 0x00000016}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000017}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000018}, - {0x03F, 0x0000FBFF}, - {0x033, 0x00000019}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001A}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001B}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001C}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001D}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001E}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000001F}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000020}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000021}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000022}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000023}, - {0x03F, 0x0000EFFF}, {0x033, 0x00000024}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000025}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000026}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000027}, - {0x03F, 0x0000EFFF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, {0x033, 0x00000028}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000029}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000002A}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000002B}, - {0x03F, 0x0000EFFF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, {0x033, 0x0000002C}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000002D}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000002E}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000002F}, - {0x03F, 0x0000EFFF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, {0x033, 0x00000030}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000031}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000032}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000033}, - {0x03F, 0x0000EFFF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, {0x033, 0x00000034}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000035}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000036}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000037}, - {0x03F, 0x0000EFFF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E7}, + {0xB0000000, 0x00000000}, {0x033, 0x00000038}, - {0x03F, 0x0000EFFF}, - {0x033, 0x00000039}, - {0x03F, 0x0000EFFF}, - {0x033, 0x0000003A}, - {0x03F, 0x0000EFFF}, - {0x0EF, 0x00000000}, - {0x0EF, 0x00000040}, - {0x033, 0x00000000}, - {0x03F, 0x00004344}, - {0x033, 0x00000001}, - {0x03F, 0x00004344}, - {0x033, 0x00000002}, - {0x03F, 0x00004344}, - {0x033, 0x00000003}, - {0x03F, 0x00004344}, - {0x033, 0x00000004}, - {0x03F, 0x00004344}, - {0x033, 0x00000005}, - {0x03F, 0x00004344}, - {0x033, 0x00000006}, - {0x03F, 0x00004324}, - {0x033, 0x00000007}, - {0x03F, 0x00004344}, - {0x033, 0x00000008}, - {0x03F, 0x00004344}, - {0x033, 0x00000009}, - {0x03F, 0x00004344}, - {0x033, 0x0000000A}, - {0x03F, 0x00004344}, - {0x033, 0x0000000B}, - {0x03F, 0x00004344}, - {0x033, 0x00000010}, - {0x03F, 0x00004344}, - {0x033, 0x00000011}, - {0x03F, 0x00004344}, - {0x033, 0x00000012}, - {0x03F, 0x00004344}, - {0x033, 0x00000013}, - {0x03F, 0x00004344}, - {0x033, 0x00000014}, - {0x03F, 0x00004344}, - {0x033, 0x00000015}, - {0x03F, 0x00004344}, - {0x033, 0x00000016}, - {0x03F, 0x00004344}, - {0x033, 0x00000017}, - {0x03F, 0x00004344}, - {0x033, 0x00000018}, - {0x03F, 0x00004344}, - {0x033, 0x00000019}, - {0x03F, 0x00004344}, - {0x033, 0x0000001A}, - {0x03F, 0x00004344}, - {0x033, 0x0000001B}, - {0x03F, 0x00004344}, - {0x033, 0x0000001C}, - {0x03F, 0x00004344}, - {0x033, 0x0000001D}, - {0x03F, 0x00004344}, - {0x033, 0x0000001E}, - {0x03F, 0x00004344}, - {0x033, 0x0000001F}, - {0x03F, 0x00004344}, - {0x0EF, 0x00000000}, - {0x0EF, 0x00000020}, - {0x033, 0x00000010}, - {0x03F, 0x00000200}, - {0x033, 0x00000011}, - {0x03F, 0x00000200}, - {0x033, 0x00000012}, - {0x03F, 0x00000200}, - {0x033, 0x00000013}, - {0x03F, 0x00000200}, - {0x033, 0x00000020}, - {0x03F, 0x00000200}, - {0x033, 0x00000021}, - {0x03F, 0x00000200}, - {0x033, 0x00000022}, - {0x03F, 0x00000200}, - {0x033, 0x00000023}, - {0x03F, 0x00000200}, - {0x0EF, 0x00000000}, - {0x0EF, 0x00000010}, - {0x030, 0x000084DC}, - {0x030, 0x000103C9}, - {0x030, 0x00018399}, - {0x030, 0x00020287}, - {0x030, 0x00028277}, - {0x030, 0x00030165}, - {0x030, 0x00038144}, - {0x030, 0x00040044}, - {0x030, 0x00048022}, - {0x030, 0x00050011}, - {0x030, 0x00058000}, - {0x030, 0x00060000}, - {0x030, 0x00068000}, - {0x030, 0x00070000}, - {0x0EF, 0x00000000}, - {0x0EF, 0x00000080}, - {0x033, 0x00000004}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000005}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000006}, - {0x03E, 0x00000014}, - {0x03F, 0x00023958}, - {0x033, 0x00000007}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000008}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000009}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x0000000A}, - {0x03E, 0x00000014}, - {0x03F, 0x00023958}, - {0x033, 0x0000000B}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000000C}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x0000000D}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x0000000E}, - {0x03E, 0x00000014}, - {0x03F, 0x00023958}, - {0x033, 0x0000000F}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000010}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000011}, - {0x03E, 0x0000001B}, - {0x03F, 0x00022A58}, - {0x033, 0x00000012}, - {0x03E, 0x00000014}, - {0x03F, 0x00023958}, - {0x033, 0x00000013}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000014}, - {0x03E, 0x00000013}, - {0x03F, 0x00022A58}, - {0x033, 0x00000015}, - {0x03E, 0x0000001B}, - {0x03F, 0x00029858}, - {0x033, 0x00000016}, - {0x03E, 0x0000001C}, - {0x03F, 0x00023958}, - {0x033, 0x00000017}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000018}, - {0x03E, 0x00000013}, - {0x03F, 0x00029858}, - {0x033, 0x00000019}, - {0x03E, 0x0000001B}, - {0x03F, 0x00029858}, - {0x033, 0x0000001A}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000001B}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000001C}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x0000001D}, - {0x03E, 0x0000001B}, - {0x03F, 0x00029858}, - {0x033, 0x0000001E}, - {0x03E, 0x00000013}, - {0x03F, 0x00023A58}, - {0x033, 0x0000001F}, - {0x03E, 0x00000013}, - {0x03F, 0x00023A58}, - {0x033, 0x00000020}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000003C}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0xB0000000, 0x00000000}, {0x033, 0x00000021}, - {0x03E, 0x0000001C}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000025}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000029}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000002D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000031}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000035}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000039}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000003D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, {0x033, 0x00000022}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000023}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000024}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x00000025}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, {0x033, 0x00000026}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000027}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000028}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x00000029}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, {0x033, 0x0000002A}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000002B}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000002C}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x0000002D}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, {0x033, 0x0000002E}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000002F}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000030}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x00000031}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, {0x033, 0x00000032}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000033}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000034}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x00000035}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, {0x033, 0x00000036}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000037}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x00000038}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x00000039}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, {0x033, 0x0000003A}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000003B}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000003C}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, - {0x033, 0x0000003D}, - {0x03E, 0x00000014}, - {0x03F, 0x0002AC58}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, {0x033, 0x0000003E}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x033, 0x0000003F}, - {0x03E, 0x00000014}, - {0x03F, 0x00023A58}, - {0x0EF, 0x00000000}, - {0x0EE, 0x00000800}, - {0x033, 0x00000000}, - {0x03F, 0x00000031}, - {0x033, 0x00000001}, - {0x03F, 0x00000023}, - {0x033, 0x00000002}, - {0x03F, 0x00000015}, - {0x033, 0x00000003}, - {0x03F, 0x00000007}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000060}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000064}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000068}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006C}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000070}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000074}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000078}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007C}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000061}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000065}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000069}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000071}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000075}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000079}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007D}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000062}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000066}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000072}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000076}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007A}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E7}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000003E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000063}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000152}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000052}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000067}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000015A}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000005A}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006B}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000009C}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xA0000000, 0x00000000}, + {0x03F, 0x0000019C}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000073}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001A4}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000077}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000002E6}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xA0000000, 0x00000000}, + {0x03F, 0x000001E6}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007B}, + {0x03F, 0x000002E7}, + {0x033, 0x0000007F}, + {0x03F, 0x000003E7}, {0x0EE, 0x00000000}, - {0x0EC, 0x00000400}, - {0x033, 0x00000003}, - {0x03F, 0x00000030}, - {0x033, 0x00000004}, - {0x03F, 0x00000021}, - {0x0EC, 0x00000000}, - {0x0DE, 0x00000000}, - {0x0EF, 0x00000000}, - {0x033, 0x00000000}, - {0x008, 0x00060280}, - {0x009, 0x00030400}, - {0x0EF, 0x00000000}, - {0x0A7, 0x00080308}, - {0x066, 0x00006000}, - {0x0EF, 0x00000400}, - {0x030, 0x000001FF}, - {0x030, 0x000081FF}, - {0x030, 0x000101FF}, - {0x030, 0x000181FF}, - {0x030, 0x000201FF}, - {0x030, 0x000281FF}, - {0x030, 0x0003017F}, - {0x030, 0x000380FB}, - {0x0EF, 0x00000000}, - {0x06E, 0x00077A18}, - {0x06D, 0x00000C31}, - {0x06A, 0x000E0F8A}, - {0x06B, 0x000018A0}, - {0x06F, 0x000F81FC}, - {0x05E, 0x0000001F}, - {0x0EF, 0x00000200}, - {0x030, 0x0003D407}, - {0x030, 0x00035A87}, - {0x030, 0x0002CF07}, - {0x030, 0x00024F07}, - {0x030, 0x0001CF07}, - {0x030, 0x00014F07}, - {0x030, 0x0000CF07}, - {0x030, 0x00004F07}, - {0x0EF, 0x00000000}, - {0x0EB, 0x00080000}, - {0x030, 0x00008038}, - {0x030, 0x00010038}, - {0x030, 0x00018038}, - {0x030, 0x00020038}, - {0x030, 0x00028038}, - {0x030, 0x00030038}, - {0x030, 0x0003803C}, - {0x030, 0x0004003C}, - {0x030, 0x0004803C}, - {0x030, 0x0005003C}, - {0x030, 0x0005803C}, - {0x030, 0x0006003C}, - {0x030, 0x0006803C}, - {0x030, 0x0007003C}, - {0x0EB, 0x00000000}, - {0x094, 0x000000FC}, + {0x100EE, 0x00004000}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000000}, + {0x10030, 0x000201DF}, + {0x10030, 0x000205D9}, + {0x10030, 0x000209D3}, + {0x10030, 0x00020D99}, + {0x10030, 0x00021193}, + {0x10030, 0x0002155F}, + {0x10030, 0x00021959}, + {0x10030, 0x00021D21}, + {0x10030, 0x00022119}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228D9}, + {0x10030, 0x00022C9F}, + {0x10030, 0x00023099}, + {0x10030, 0x0002345F}, + {0x10030, 0x00023859}, + {0x10030, 0x00023C1F}, + {0x10030, 0x00024019}, + {0x10030, 0x00024413}, + {0x10030, 0x000281CD}, + {0x10030, 0x000285DB}, + {0x10030, 0x000289D5}, + {0x10030, 0x00028D9B}, + {0x10030, 0x0002918D}, + {0x10030, 0x00029555}, + {0x10030, 0x00029957}, + {0x10030, 0x00029D1F}, + {0x10030, 0x0002A119}, + {0x10030, 0x0002A4DF}, + {0x10030, 0x0002A8D9}, + {0x10030, 0x0002AC9F}, + {0x10030, 0x0002B099}, + {0x10030, 0x0002B45F}, + {0x10030, 0x0002B859}, + {0x10030, 0x0002BC1F}, + {0x10030, 0x0002C019}, + {0x10030, 0x0002C413}, + {0x10030, 0x000301D9}, + {0x10030, 0x000305DB}, + {0x10030, 0x000309D5}, + {0x10030, 0x00030D9B}, + {0x10030, 0x00031195}, + {0x10030, 0x0003155D}, + {0x10030, 0x00031955}, + {0x10030, 0x00031D1D}, + {0x10030, 0x00032119}, + {0x10030, 0x000324DF}, + {0x10030, 0x000328D9}, + {0x10030, 0x00032C9F}, + {0x10030, 0x00033099}, + {0x10030, 0x0003345F}, + {0x10030, 0x00033859}, + {0x10030, 0x00033C1F}, + {0x10030, 0x00034019}, + {0x10030, 0x00034413}, + {0x10030, 0x000601E1}, + {0x10030, 0x000605DB}, + {0x10030, 0x000609D5}, + {0x10030, 0x00060D9B}, + {0x10030, 0x00061195}, + {0x10030, 0x0006155B}, + {0x10030, 0x00061957}, + {0x10030, 0x00061D1F}, + {0x10030, 0x00062119}, + {0x10030, 0x000624DF}, + {0x10030, 0x000628D9}, + {0x10030, 0x00062C9F}, + {0x10030, 0x00063099}, + {0x10030, 0x0006345F}, + {0x10030, 0x00063859}, + {0x10030, 0x00063C1F}, + {0x10030, 0x00064019}, + {0x10030, 0x00064413}, + {0x10030, 0x000681E1}, + {0x10030, 0x000685DB}, + {0x10030, 0x000689D5}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955B}, + {0x10030, 0x00069957}, + {0x10030, 0x00069D1F}, + {0x10030, 0x0006A119}, + {0x10030, 0x0006A4DF}, + {0x10030, 0x0006A8D9}, + {0x10030, 0x0006AC9F}, + {0x10030, 0x0006B099}, + {0x10030, 0x0006B45F}, + {0x10030, 0x0006B859}, + {0x10030, 0x0006BC1F}, + {0x10030, 0x0006C019}, + {0x10030, 0x0006C413}, + {0x10030, 0x000701E1}, + {0x10030, 0x000705DB}, + {0x10030, 0x000709D5}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071957}, + {0x10030, 0x00071D1F}, + {0x10030, 0x00072119}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072C9F}, + {0x10030, 0x00073099}, + {0x10030, 0x0007345F}, + {0x10030, 0x00073859}, + {0x10030, 0x00073C1F}, + {0x10030, 0x00074019}, + {0x10030, 0x00074413}, + {0x10030, 0x000781DF}, + {0x10030, 0x000785D9}, + {0x10030, 0x000789D3}, + {0x10030, 0x00078D99}, + {0x10030, 0x00079193}, + {0x10030, 0x0007955F}, + {0x10030, 0x00079959}, + {0x10030, 0x00079D21}, + {0x10030, 0x0007A115}, + {0x10030, 0x0007A4DF}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007AC9F}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B45F}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC1F}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0x10030, 0x00000000}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201EF}, + {0x10030, 0x000205E9}, + {0x10030, 0x000209E3}, + {0x10030, 0x00020DA3}, + {0x10030, 0x00021161}, + {0x10030, 0x0002155B}, + {0x10030, 0x0002191F}, + {0x10030, 0x00021D19}, + {0x10030, 0x000220E1}, + {0x10030, 0x000224DB}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1B}, + {0x10030, 0x00024015}, + {0x10030, 0x0002440F}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285E7}, + {0x10030, 0x000289A7}, + {0x10030, 0x00028D65}, + {0x10030, 0x0002915F}, + {0x10030, 0x00029523}, + {0x10030, 0x0002991D}, + {0x10030, 0x00029CE5}, + {0x10030, 0x0002A0DF}, + {0x10030, 0x0002A4A7}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC67}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B427}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC19}, + {0x10030, 0x0002C013}, + {0x10030, 0x0002C40D}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305E7}, + {0x10030, 0x000309A7}, + {0x10030, 0x00030D65}, + {0x10030, 0x0003115F}, + {0x10030, 0x00031525}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031CE7}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324A9}, + {0x10030, 0x000328A3}, + {0x10030, 0x00032C69}, + {0x10030, 0x00033063}, + {0x10030, 0x00033429}, + {0x10030, 0x00033823}, + {0x10030, 0x00033C1D}, + {0x10030, 0x00034013}, + {0x10030, 0x0003440D}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x00072111}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x095, 0x00000008}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201A7}, + {0x10030, 0x000205A1}, + {0x10030, 0x0002099B}, + {0x10030, 0x00020D95}, + {0x10030, 0x0002115B}, + {0x10030, 0x00021555}, + {0x10030, 0x00021921}, + {0x10030, 0x00021D1B}, + {0x10030, 0x000220E3}, + {0x10030, 0x000224DD}, + {0x10030, 0x000228A3}, + {0x10030, 0x00022C9D}, + {0x10030, 0x00023063}, + {0x10030, 0x0002345D}, + {0x10030, 0x00023823}, + {0x10030, 0x00023C1D}, + {0x10030, 0x00024017}, + {0x10030, 0x00024411}, + {0x10030, 0x000281A9}, + {0x10030, 0x000285A3}, + {0x10030, 0x0002899D}, + {0x10030, 0x00028D97}, + {0x10030, 0x0002915D}, + {0x10030, 0x00029557}, + {0x10030, 0x0002991F}, + {0x10030, 0x00029D19}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DB}, + {0x10030, 0x0002A8A1}, + {0x10030, 0x0002AC9B}, + {0x10030, 0x0002B061}, + {0x10030, 0x0002B45B}, + {0x10030, 0x0002B821}, + {0x10030, 0x0002BC1B}, + {0x10030, 0x0002C015}, + {0x10030, 0x0002C40F}, + {0x10030, 0x000301A9}, + {0x10030, 0x000305A3}, + {0x10030, 0x0003099D}, + {0x10030, 0x00030D97}, + {0x10030, 0x0003115D}, + {0x10030, 0x00031557}, + {0x10030, 0x0003191F}, + {0x10030, 0x00031D19}, + {0x10030, 0x000320E1}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328A1}, + {0x10030, 0x00032C9B}, + {0x10030, 0x00033061}, + {0x10030, 0x0003345B}, + {0x10030, 0x00033821}, + {0x10030, 0x00033C1B}, + {0x10030, 0x00034015}, + {0x10030, 0x0003440F}, + {0x10030, 0x000601F1}, + {0x10030, 0x000605E9}, + {0x10030, 0x000609A9}, + {0x10030, 0x00060D65}, + {0x10030, 0x0006115F}, + {0x10030, 0x00061525}, + {0x10030, 0x0006191F}, + {0x10030, 0x00061CE7}, + {0x10030, 0x000620E1}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628A3}, + {0x10030, 0x00062C69}, + {0x10030, 0x00063063}, + {0x10030, 0x00063429}, + {0x10030, 0x00063823}, + {0x10030, 0x00063C1D}, + {0x10030, 0x00064013}, + {0x10030, 0x0006440D}, + {0x10030, 0x000681EF}, + {0x10030, 0x000685E7}, + {0x10030, 0x000689A7}, + {0x10030, 0x00068D61}, + {0x10030, 0x0006915B}, + {0x10030, 0x00069523}, + {0x10030, 0x0006991D}, + {0x10030, 0x00069CE5}, + {0x10030, 0x0006A0DF}, + {0x10030, 0x0006A4A7}, + {0x10030, 0x0006A8A1}, + {0x10030, 0x0006AC67}, + {0x10030, 0x0006B061}, + {0x10030, 0x0006B429}, + {0x10030, 0x0006B823}, + {0x10030, 0x0006BC1D}, + {0x10030, 0x0006C017}, + {0x10030, 0x0006C40D}, + {0x10030, 0x000701F1}, + {0x10030, 0x000705E9}, + {0x10030, 0x000709A9}, + {0x10030, 0x00070D63}, + {0x10030, 0x0007115D}, + {0x10030, 0x00071523}, + {0x10030, 0x0007191D}, + {0x10030, 0x00071D17}, + {0x10030, 0x000720DF}, + {0x10030, 0x000724D9}, + {0x10030, 0x000728D3}, + {0x10030, 0x00072C67}, + {0x10030, 0x00073061}, + {0x10030, 0x00073427}, + {0x10030, 0x00073821}, + {0x10030, 0x00073C1B}, + {0x10030, 0x00074015}, + {0x10030, 0x0007440D}, + {0x10030, 0x000781F1}, + {0x10030, 0x000785EB}, + {0x10030, 0x000789E5}, + {0x10030, 0x00078DA3}, + {0x10030, 0x00079161}, + {0x10030, 0x0007955B}, + {0x10030, 0x00079923}, + {0x10030, 0x00079D1D}, + {0x10030, 0x0007A117}, + {0x10030, 0x0007A4DD}, + {0x10030, 0x0007A8D7}, + {0x10030, 0x0007AC9D}, + {0x10030, 0x0007B063}, + {0x10030, 0x0007B45D}, + {0x10030, 0x0007B857}, + {0x10030, 0x0007BC1D}, + {0x10030, 0x0007C017}, + {0x10030, 0x0007C40F}, {0xA0000000, 0x00000000}, - {0x095, 0x00000000}, - {0xB0000000, 0x00000000}, - {0x0EE, 0x00001000}, - {0x033, 0x00000020}, - {0x03F, 0x00000052}, - {0x033, 0x00000024}, - {0x03F, 0x0000005A}, - {0x033, 0x00000028}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002C}, - {0x03F, 0x0000019C}, - {0x033, 0x00000030}, - {0x03F, 0x000001A4}, - {0x033, 0x00000034}, - {0x03F, 0x000001E7}, - {0x033, 0x00000038}, - {0x03F, 0x000002E7}, - {0x033, 0x0000003C}, - {0x03F, 0x000003E7}, - {0x033, 0x00000021}, - {0x03F, 0x00000052}, - {0x033, 0x00000025}, - {0x03F, 0x0000005A}, - {0x033, 0x00000029}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002D}, - {0x03F, 0x0000019C}, - {0x033, 0x00000031}, - {0x03F, 0x000001A4}, - {0x033, 0x00000035}, - {0x03F, 0x000001E6}, - {0x033, 0x00000039}, - {0x03F, 0x000002E6}, - {0x033, 0x0000003D}, - {0x03F, 0x000003E6}, - {0x033, 0x00000022}, - {0x03F, 0x00000052}, - {0x033, 0x00000026}, - {0x03F, 0x0000005A}, - {0x033, 0x0000002A}, - {0x03F, 0x0000009C}, - {0x033, 0x0000002E}, - {0x03F, 0x0000019C}, - {0x033, 0x00000032}, - {0x03F, 0x000001A4}, - {0x033, 0x00000036}, - {0x03F, 0x000001E6}, - {0x033, 0x0000003A}, - {0x03F, 0x000002E6}, - {0x033, 0x0000003E}, - {0x03F, 0x000003E6}, - {0x033, 0x00000060}, - {0x03F, 0x00000052}, - {0x033, 0x00000064}, - {0x03F, 0x0000005A}, - {0x033, 0x00000068}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006C}, - {0x03F, 0x0000019C}, - {0x033, 0x00000070}, - {0x03F, 0x000001A4}, - {0x033, 0x00000074}, - {0x03F, 0x000001E6}, - {0x033, 0x00000078}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007C}, - {0x03F, 0x000003E6}, - {0x033, 0x00000061}, - {0x03F, 0x00000052}, - {0x033, 0x00000065}, - {0x03F, 0x0000005A}, - {0x033, 0x00000069}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006D}, - {0x03F, 0x0000019C}, - {0x033, 0x00000071}, - {0x03F, 0x000001A4}, - {0x033, 0x00000075}, - {0x03F, 0x000001E6}, - {0x033, 0x00000079}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007D}, - {0x03F, 0x000003E6}, - {0x033, 0x00000062}, - {0x03F, 0x00000052}, - {0x033, 0x00000066}, - {0x03F, 0x0000005A}, - {0x033, 0x0000006A}, - {0x03F, 0x0000009C}, - {0x033, 0x0000006E}, - {0x03F, 0x0000019C}, - {0x033, 0x00000072}, - {0x03F, 0x000001A4}, - {0x033, 0x00000076}, - {0x03F, 0x000001E6}, - {0x033, 0x0000007A}, - {0x03F, 0x000002E6}, - {0x033, 0x0000007E}, - {0x03F, 0x000003E6}, - {0x033, 0x00000063}, + {0x10030, 0x000001EF}, + {0x10030, 0x000005E9}, + {0x10030, 0x000009E3}, + {0x10030, 0x00000DDD}, + {0x10030, 0x000011D7}, + {0x10030, 0x0000159F}, + {0x10030, 0x00001999}, + {0x10030, 0x00001D5F}, + {0x10030, 0x00002159}, + {0x10030, 0x0000251F}, + {0x10030, 0x00002919}, + {0x10030, 0x00002CDF}, + {0x10030, 0x000030D9}, + {0x10030, 0x0000349F}, + {0x10030, 0x00003899}, + {0x10030, 0x00003C5F}, + {0x10030, 0x00004059}, + {0x10030, 0x00004453}, + {0x10030, 0x000201ED}, + {0x10030, 0x000205AD}, + {0x10030, 0x000209A7}, + {0x10030, 0x00020DA1}, + {0x10030, 0x0002119B}, + {0x10030, 0x00021561}, + {0x10030, 0x0002195B}, + {0x10030, 0x00021D27}, + {0x10030, 0x00022121}, + {0x10030, 0x000224E9}, + {0x10030, 0x000228E3}, + {0x10030, 0x00022CA9}, + {0x10030, 0x000230A3}, + {0x10030, 0x00023469}, + {0x10030, 0x00023863}, + {0x10030, 0x00023C29}, + {0x10030, 0x00024023}, + {0x10030, 0x0002441D}, + {0x10030, 0x000281EF}, + {0x10030, 0x000285AF}, + {0x10030, 0x000289A9}, + {0x10030, 0x00028DA3}, + {0x10030, 0x0002919D}, + {0x10030, 0x00029563}, + {0x10030, 0x0002995D}, + {0x10030, 0x00029D25}, + {0x10030, 0x0002A11F}, + {0x10030, 0x0002A4E7}, + {0x10030, 0x0002A8E1}, + {0x10030, 0x0002ACA7}, + {0x10030, 0x0002B0A1}, + {0x10030, 0x0002B467}, + {0x10030, 0x0002B861}, + {0x10030, 0x0002BC27}, + {0x10030, 0x0002C021}, + {0x10030, 0x0002C41B}, + {0x10030, 0x000301EF}, + {0x10030, 0x000305AF}, + {0x10030, 0x000309A9}, + {0x10030, 0x00030DA3}, + {0x10030, 0x0003119D}, + {0x10030, 0x00031563}, + {0x10030, 0x0003195D}, + {0x10030, 0x00031D25}, + {0x10030, 0x0003211F}, + {0x10030, 0x000324E7}, + {0x10030, 0x000328E1}, + {0x10030, 0x00032CA7}, + {0x10030, 0x000330A1}, + {0x10030, 0x00033467}, + {0x10030, 0x00033861}, + {0x10030, 0x00033C27}, + {0x10030, 0x00034021}, + {0x10030, 0x0003441B}, + {0x10030, 0x000601EB}, + {0x10030, 0x000605AB}, + {0x10030, 0x000609A5}, + {0x10030, 0x00060D9F}, + {0x10030, 0x00061199}, + {0x10030, 0x00061593}, + {0x10030, 0x00061959}, + {0x10030, 0x00061D53}, + {0x10030, 0x0006211B}, + {0x10030, 0x00062515}, + {0x10030, 0x000628DD}, + {0x10030, 0x00062CD7}, + {0x10030, 0x0006309D}, + {0x10030, 0x00063497}, + {0x10030, 0x0006385D}, + {0x10030, 0x00063C57}, + {0x10030, 0x0006401D}, + {0x10030, 0x00064417}, + {0x10030, 0x000681E7}, + {0x10030, 0x000685A7}, + {0x10030, 0x000689A1}, + {0x10030, 0x00068D9B}, + {0x10030, 0x00069195}, + {0x10030, 0x0006955F}, + {0x10030, 0x00069959}, + {0x10030, 0x00069D21}, + {0x10030, 0x0006A11B}, + {0x10030, 0x0006A4E3}, + {0x10030, 0x0006A8DD}, + {0x10030, 0x0006ACA5}, + {0x10030, 0x0006B09F}, + {0x10030, 0x0006B465}, + {0x10030, 0x0006B85F}, + {0x10030, 0x0006BC25}, + {0x10030, 0x0006C01F}, + {0x10030, 0x0006C419}, + {0x10030, 0x000701E7}, + {0x10030, 0x000705A7}, + {0x10030, 0x000709A1}, + {0x10030, 0x00070D9B}, + {0x10030, 0x00071195}, + {0x10030, 0x0007155B}, + {0x10030, 0x00071955}, + {0x10030, 0x00071D1D}, + {0x10030, 0x00072117}, + {0x10030, 0x000724DF}, + {0x10030, 0x000728D9}, + {0x10030, 0x00072CA1}, + {0x10030, 0x0007309B}, + {0x10030, 0x00073461}, + {0x10030, 0x0007385B}, + {0x10030, 0x00073C21}, + {0x10030, 0x0007401B}, + {0x10030, 0x0007441B}, + {0x10030, 0x000781E9}, + {0x10030, 0x000785A9}, + {0x10030, 0x000789A3}, + {0x10030, 0x00078D9D}, + {0x10030, 0x00079197}, + {0x10030, 0x00079591}, + {0x10030, 0x00079957}, + {0x10030, 0x00079D51}, + {0x10030, 0x0007A119}, + {0x10030, 0x0007A513}, + {0x10030, 0x0007A8D9}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B099}, + {0x10030, 0x0007B493}, + {0x10030, 0x0007B859}, + {0x10030, 0x0007BC53}, + {0x10030, 0x0007C019}, + {0x10030, 0x0007C413}, + {0xB0000000, 0x00000000}, + {0x100EE, 0x00000000}, + {0x100EE, 0x00002000}, + {0x10030, 0x000000FC}, + {0x10030, 0x000004F9}, + {0x10030, 0x000008F6}, + {0x10030, 0x00000CF3}, + {0x10030, 0x000010F0}, + {0x10030, 0x000014ED}, + {0x10030, 0x000018AC}, + {0x10030, 0x00001CA9}, + {0x10030, 0x00002069}, + {0x10030, 0x00002466}, + {0x10030, 0x00002829}, + {0x10030, 0x00002C26}, + {0x10030, 0x00003023}, + {0x10030, 0x00003420}, + {0x10030, 0x0000381D}, + {0x10030, 0x00003C1A}, + {0x10030, 0x00004017}, + {0x100EE, 0x00000000}, + {0x100EE, 0x00002000}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x00000152}, + {0x10030, 0x000200E8}, + {0x10030, 0x000204E5}, + {0x10030, 0x000208E2}, + {0x10030, 0x00020CDF}, + {0x10030, 0x000210DC}, + {0x10030, 0x000214D9}, + {0x10030, 0x000218D6}, + {0x10030, 0x00021CD3}, + {0x10030, 0x000220D0}, + {0x10030, 0x000224CD}, + {0x10030, 0x000228CD}, + {0x10030, 0x00022CCD}, + {0x10030, 0x000230CD}, + {0x10030, 0x000234CD}, + {0x10030, 0x000238CD}, + {0x10030, 0x00023CCD}, + {0x10030, 0x000240CD}, + {0x10030, 0x000280E8}, + {0x10030, 0x000284E5}, + {0x10030, 0x000288E2}, + {0x10030, 0x00028CDF}, + {0x10030, 0x000290DC}, + {0x10030, 0x000294D9}, + {0x10030, 0x000298D6}, + {0x10030, 0x00029CD3}, + {0x10030, 0x0002A0D0}, + {0x10030, 0x0002A4CD}, + {0x10030, 0x0002A8CD}, + {0x10030, 0x0002ACCD}, + {0x10030, 0x0002B0CD}, + {0x10030, 0x0002B4CD}, + {0x10030, 0x0002B8CD}, + {0x10030, 0x0002BCCD}, + {0x10030, 0x0002C0CD}, + {0x10030, 0x000300E8}, + {0x10030, 0x000304E5}, + {0x10030, 0x000308E2}, + {0x10030, 0x00030CDF}, + {0x10030, 0x000310DC}, + {0x10030, 0x000314D9}, + {0x10030, 0x000318D6}, + {0x10030, 0x00031CD3}, + {0x10030, 0x000320D0}, + {0x10030, 0x000324CD}, + {0x10030, 0x000328CD}, + {0x10030, 0x00032CCD}, + {0x10030, 0x000330CD}, + {0x10030, 0x000334CD}, + {0x10030, 0x000338CD}, + {0x10030, 0x00033CCD}, + {0x10030, 0x000340CD}, {0xA0000000, 0x00000000}, - {0x03F, 0x00000052}, + {0x10030, 0x000200FA}, + {0x10030, 0x000204F7}, + {0x10030, 0x000208F4}, + {0x10030, 0x00020CF1}, + {0x10030, 0x000210EE}, + {0x10030, 0x000214EB}, + {0x10030, 0x000218E8}, + {0x10030, 0x00021CE5}, + {0x10030, 0x000220E2}, + {0x10030, 0x000224DF}, + {0x10030, 0x000228DC}, + {0x10030, 0x00022CD9}, + {0x10030, 0x000230D6}, + {0x10030, 0x000234D3}, + {0x10030, 0x000238D0}, + {0x10030, 0x00023C0D}, + {0x10030, 0x0002400A}, + {0x10030, 0x000280F9}, + {0x10030, 0x000284F6}, + {0x10030, 0x000288F3}, + {0x10030, 0x00028CF0}, + {0x10030, 0x000290ED}, + {0x10030, 0x000294EA}, + {0x10030, 0x000298E7}, + {0x10030, 0x00029CE4}, + {0x10030, 0x0002A0E1}, + {0x10030, 0x0002A4DE}, + {0x10030, 0x0002A8DB}, + {0x10030, 0x0002ACD8}, + {0x10030, 0x0002B0D5}, + {0x10030, 0x0002B4D2}, + {0x10030, 0x0002B8CF}, + {0x10030, 0x0002BC0C}, + {0x10030, 0x0002C009}, + {0x10030, 0x000300F6}, + {0x10030, 0x000304F3}, + {0x10030, 0x000308F0}, + {0x10030, 0x00030CED}, + {0x10030, 0x000310EA}, + {0x10030, 0x000314E7}, + {0x10030, 0x000318E4}, + {0x10030, 0x00031CE1}, + {0x10030, 0x000320DE}, + {0x10030, 0x000324DB}, + {0x10030, 0x000328D8}, + {0x10030, 0x00032CD5}, + {0x10030, 0x000330D2}, + {0x10030, 0x000334CF}, + {0x10030, 0x000338CC}, + {0x10030, 0x00033C09}, + {0x10030, 0x00034006}, {0xB0000000, 0x00000000}, - {0x033, 0x00000067}, + {0x10030, 0x000600F6}, + {0x10030, 0x000604F3}, + {0x10030, 0x000608F0}, + {0x10030, 0x00060CED}, + {0x10030, 0x000610EA}, + {0x10030, 0x000614E7}, + {0x10030, 0x000618E4}, + {0x10030, 0x00061CE1}, + {0x10030, 0x000620DE}, + {0x10030, 0x000624DB}, + {0x10030, 0x000628D8}, + {0x10030, 0x00062CD5}, + {0x10030, 0x000630D2}, + {0x10030, 0x000634CF}, + {0x10030, 0x000638CC}, + {0x10030, 0x00063C09}, + {0x10030, 0x00064006}, + {0x10030, 0x000680F5}, + {0x10030, 0x000684F2}, + {0x10030, 0x000688EF}, + {0x10030, 0x00068CEC}, + {0x10030, 0x000690E9}, + {0x10030, 0x000694E6}, + {0x10030, 0x000698E3}, + {0x10030, 0x00069CE0}, + {0x10030, 0x0006A0DD}, + {0x10030, 0x0006A4DA}, + {0x10030, 0x0006A8D7}, + {0x10030, 0x0006ACD4}, + {0x10030, 0x0006B0D1}, + {0x10030, 0x0006B4CE}, + {0x10030, 0x0006B8CB}, + {0x10030, 0x0006BC08}, + {0x10030, 0x0006C005}, + {0x10030, 0x000700F5}, + {0x10030, 0x000704F2}, + {0x10030, 0x000708EF}, + {0x10030, 0x00070CEC}, + {0x10030, 0x000710E9}, + {0x10030, 0x000714E6}, + {0x10030, 0x000718E3}, + {0x10030, 0x00071CE0}, + {0x10030, 0x000720DD}, + {0x10030, 0x000724DA}, + {0x10030, 0x000728D7}, + {0x10030, 0x00072CD4}, + {0x10030, 0x000730D1}, + {0x10030, 0x000734CE}, + {0x10030, 0x000738CB}, + {0x10030, 0x00073C08}, + {0x10030, 0x00074005}, + {0x10030, 0x000780F4}, + {0x10030, 0x000784F1}, + {0x10030, 0x000788EE}, + {0x10030, 0x00078CEB}, + {0x10030, 0x000790E8}, + {0x10030, 0x000794E5}, + {0x10030, 0x000798E2}, + {0x10030, 0x00079CDF}, + {0x10030, 0x0007A0DC}, + {0x10030, 0x0007A4D9}, + {0x10030, 0x0007A8D6}, + {0x10030, 0x0007ACD3}, + {0x10030, 0x0007B0D0}, + {0x10030, 0x0007B4CD}, + {0x10030, 0x0007B8CA}, + {0x10030, 0x0007BC07}, + {0x10030, 0x0007C004}, + {0x100EE, 0x00000000}, + {0x0EF, 0x00002000}, + {0x033, 0x00000008}, + {0x03F, 0x00000004}, + {0x033, 0x00000009}, + {0x03F, 0x00000003}, + {0x033, 0x0000000A}, + {0x03F, 0x00000003}, + {0x033, 0x0000000B}, + {0x03F, 0x00000002}, + {0x033, 0x0000000C}, + {0x03F, 0x00000002}, + {0x033, 0x0000000D}, + {0x03F, 0x00000002}, + {0x033, 0x0000000E}, + {0x03F, 0x00000002}, + {0x033, 0x0000000F}, + {0x03F, 0x00000002}, + {0x0EF, 0x00000000}, + {0x0EB, 0x00040000}, + {0x030, 0x000109B7}, + {0x0EB, 0x00000000}, + {0x0EF, 0x00008000}, + {0x033, 0x00000020}, + {0x03F, 0x00050002}, + {0x033, 0x00000021}, + {0x03F, 0x00060032}, + {0x033, 0x00000022}, + {0x03F, 0x00050042}, + {0x033, 0x00000023}, + {0x03F, 0x00040042}, + {0x033, 0x00000024}, + {0x03F, 0x00008001}, + {0x033, 0x00000025}, + {0x03F, 0x00008002}, + {0x033, 0x00000026}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000015A}, + {0x03F, 0x00008002}, {0xA0000000, 0x00000000}, - {0x03F, 0x0000005A}, + {0x03F, 0x00000003}, {0xB0000000, 0x00000000}, - {0x033, 0x0000006B}, + {0x033, 0x00000027}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0xA0000000, 0x00000000}, - {0x03F, 0x0000009C}, + {0x03F, 0x00000003}, {0xB0000000, 0x00000000}, - {0x033, 0x0000006F}, + {0x033, 0x00000028}, + {0x03F, 0x00050002}, + {0x033, 0x00000029}, + {0x03F, 0x00060032}, + {0x033, 0x0000002A}, + {0x03F, 0x00050042}, + {0x033, 0x0000002B}, + {0x03F, 0x00040042}, + {0x033, 0x0000002C}, + {0x03F, 0x00008001}, + {0x033, 0x0000002D}, + {0x03F, 0x00008002}, + {0x033, 0x0000002E}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0xA0000000, 0x00000000}, - {0x03F, 0x0000019C}, + {0x03F, 0x00000003}, {0xB0000000, 0x00000000}, - {0x033, 0x00000073}, + {0x033, 0x0000002F}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0xA0000000, 0x00000000}, - {0x03F, 0x000001A4}, + {0x03F, 0x00000003}, {0xB0000000, 0x00000000}, - {0x033, 0x00000077}, + {0x033, 0x00000030}, + {0x03F, 0x00050002}, + {0x033, 0x00000031}, + {0x03F, 0x00060032}, + {0x033, 0x00000032}, + {0x03F, 0x00050042}, + {0x033, 0x00000033}, + {0x03F, 0x00040042}, + {0x033, 0x00000034}, + {0x03F, 0x00008001}, + {0x033, 0x00000035}, + {0x03F, 0x00008002}, + {0x033, 0x00000036}, {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000001E6}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, - {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, - {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, - {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x03F, 0x000002E6}, - {0xA0000000, 0x00000000}, - {0x03F, 0x000001E6}, - {0xB0000000, 0x00000000}, - {0x033, 0x0000007B}, - {0x03F, 0x000002E7}, - {0x033, 0x0000007F}, - {0x03F, 0x000003E7}, - {0x0EE, 0x00000000}, - {0x100EE, 0x00004000}, - {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000037}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000060}, + {0x03F, 0x00050002}, + {0x033, 0x00000061}, + {0x03F, 0x00060032}, + {0x033, 0x00000062}, + {0x03F, 0x00050042}, + {0x033, 0x00000063}, + {0x03F, 0x00040042}, + {0x033, 0x00000064}, + {0x03F, 0x00008001}, + {0x033, 0x00000065}, + {0x03F, 0x00008002}, + {0x033, 0x00000066}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000067}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000068}, + {0x03F, 0x00050002}, + {0x033, 0x00000069}, + {0x03F, 0x00060032}, + {0x033, 0x0000006A}, + {0x03F, 0x00050042}, + {0x033, 0x0000006B}, + {0x03F, 0x00040042}, + {0x033, 0x0000006C}, + {0x03F, 0x00008001}, + {0x033, 0x0000006D}, + {0x03F, 0x00008002}, + {0x033, 0x0000006E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000006F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000070}, + {0x03F, 0x00050002}, + {0x033, 0x00000071}, + {0x03F, 0x00060032}, + {0x033, 0x00000072}, + {0x03F, 0x00050042}, + {0x033, 0x00000073}, + {0x03F, 0x00040042}, + {0x033, 0x00000074}, + {0x03F, 0x00008001}, + {0x033, 0x00000075}, + {0x03F, 0x00008002}, + {0x033, 0x00000076}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000077}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000078}, + {0x03F, 0x00050002}, + {0x033, 0x00000079}, + {0x03F, 0x00060032}, + {0x033, 0x0000007A}, + {0x03F, 0x00050042}, + {0x033, 0x0000007B}, + {0x03F, 0x00040042}, + {0x033, 0x0000007C}, + {0x03F, 0x00008001}, + {0x033, 0x0000007D}, + {0x03F, 0x00008002}, + {0x033, 0x0000007E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000007F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A0}, + {0x03F, 0x00050002}, + {0x033, 0x000000A1}, + {0x03F, 0x00060032}, + {0x033, 0x000000A2}, + {0x03F, 0x00050042}, + {0x033, 0x000000A3}, + {0x03F, 0x00040042}, + {0x033, 0x000000A4}, + {0x03F, 0x00008001}, + {0x033, 0x000000A5}, + {0x03F, 0x00008002}, + {0x033, 0x000000A6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000A8}, + {0x03F, 0x00050002}, + {0x033, 0x000000A9}, + {0x03F, 0x00060032}, + {0x033, 0x000000AA}, + {0x03F, 0x00050042}, + {0x033, 0x000000AB}, + {0x03F, 0x00040042}, + {0x033, 0x000000AC}, + {0x03F, 0x00008001}, + {0x033, 0x000000AD}, + {0x03F, 0x00008002}, + {0x033, 0x000000AE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000AF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000B0}, + {0x03F, 0x00050002}, + {0x033, 0x000000B1}, + {0x03F, 0x00060032}, + {0x033, 0x000000B2}, + {0x03F, 0x00050042}, + {0x033, 0x000000B3}, + {0x03F, 0x00040042}, + {0x033, 0x000000B4}, + {0x03F, 0x00008001}, + {0x033, 0x000000B5}, + {0x03F, 0x00008002}, + {0x033, 0x000000B6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000B7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E0}, + {0x03F, 0x00050002}, + {0x033, 0x000000E1}, + {0x03F, 0x00060032}, + {0x033, 0x000000E2}, + {0x03F, 0x00050042}, + {0x033, 0x000000E3}, + {0x03F, 0x00040042}, + {0x033, 0x000000E4}, + {0x03F, 0x00008001}, + {0x033, 0x000000E5}, + {0x03F, 0x00008002}, + {0x033, 0x000000E6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000E8}, + {0x03F, 0x00050002}, + {0x033, 0x000000E9}, + {0x03F, 0x00060032}, + {0x033, 0x000000EA}, + {0x03F, 0x00050042}, + {0x033, 0x000000EB}, + {0x03F, 0x00040042}, + {0x033, 0x000000EC}, + {0x03F, 0x00008001}, + {0x033, 0x000000ED}, + {0x03F, 0x00008002}, + {0x033, 0x000000EE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000EF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F0}, + {0x03F, 0x00050002}, + {0x033, 0x000000F1}, + {0x03F, 0x00060032}, + {0x033, 0x000000F2}, + {0x03F, 0x00050042}, + {0x033, 0x000000F3}, + {0x03F, 0x00040042}, + {0x033, 0x000000F4}, + {0x03F, 0x00008001}, + {0x033, 0x000000F5}, + {0x03F, 0x00008002}, + {0x033, 0x000000F6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781EF}, - {0x10030, 0x000785E9}, - {0x10030, 0x000789E3}, - {0x10030, 0x00078DA3}, - {0x10030, 0x00079161}, - {0x10030, 0x0007955B}, - {0x10030, 0x00079921}, - {0x10030, 0x00079D1B}, - {0x10030, 0x0007A0E1}, - {0x10030, 0x0007A4DB}, - {0x10030, 0x0007A8A1}, - {0x10030, 0x0007AC9B}, - {0x10030, 0x0007B061}, - {0x10030, 0x0007B45B}, - {0x10030, 0x0007B821}, - {0x10030, 0x0007BC1B}, - {0x10030, 0x0007C015}, - {0x10030, 0x0007C40F}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000F8}, + {0x03F, 0x00050002}, + {0x033, 0x000000F9}, + {0x03F, 0x00060032}, + {0x033, 0x000000FA}, + {0x03F, 0x00050042}, + {0x033, 0x000000FB}, + {0x03F, 0x00040042}, + {0x033, 0x000000FC}, + {0x03F, 0x00008001}, + {0x033, 0x000000FD}, + {0x03F, 0x00008002}, + {0x033, 0x000000FE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000000FF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000120}, + {0x03F, 0x00050002}, + {0x033, 0x00000121}, + {0x03F, 0x00060032}, + {0x033, 0x00000122}, + {0x03F, 0x00050042}, + {0x033, 0x00000123}, + {0x03F, 0x00040042}, + {0x033, 0x00000124}, + {0x03F, 0x00008001}, + {0x033, 0x00000125}, + {0x03F, 0x00008002}, + {0x033, 0x00000126}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000127}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000128}, + {0x03F, 0x00050002}, + {0x033, 0x00000129}, + {0x03F, 0x00060032}, + {0x033, 0x0000012A}, + {0x03F, 0x00050042}, + {0x033, 0x0000012B}, + {0x03F, 0x00040042}, + {0x033, 0x0000012C}, + {0x03F, 0x00008001}, + {0x033, 0x0000012D}, + {0x03F, 0x00008002}, + {0x033, 0x0000012E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000012F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000130}, + {0x03F, 0x00050002}, + {0x033, 0x00000131}, + {0x03F, 0x00060032}, + {0x033, 0x00000132}, + {0x03F, 0x00050042}, + {0x033, 0x00000133}, + {0x03F, 0x00040042}, + {0x033, 0x00000134}, + {0x03F, 0x00008001}, + {0x033, 0x00000135}, + {0x03F, 0x00008002}, + {0x033, 0x00000136}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000137}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000160}, + {0x03F, 0x00050002}, + {0x033, 0x00000161}, + {0x03F, 0x00060032}, + {0x033, 0x00000162}, + {0x03F, 0x00050042}, + {0x033, 0x00000163}, + {0x03F, 0x00040042}, + {0x033, 0x00000164}, + {0x03F, 0x00008001}, + {0x033, 0x00000165}, + {0x03F, 0x00008002}, + {0x033, 0x00000166}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000167}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000168}, + {0x03F, 0x00050002}, + {0x033, 0x00000169}, + {0x03F, 0x00060032}, + {0x033, 0x0000016A}, + {0x03F, 0x00050042}, + {0x033, 0x0000016B}, + {0x03F, 0x00040042}, + {0x033, 0x0000016C}, + {0x03F, 0x00008001}, + {0x033, 0x0000016D}, + {0x03F, 0x00008002}, + {0x033, 0x0000016E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000016F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000170}, + {0x03F, 0x00050002}, + {0x033, 0x00000171}, + {0x03F, 0x00060032}, + {0x033, 0x00000172}, + {0x03F, 0x00050042}, + {0x033, 0x00000173}, + {0x03F, 0x00040042}, + {0x033, 0x00000174}, + {0x03F, 0x00008001}, + {0x033, 0x00000175}, + {0x03F, 0x00008002}, + {0x033, 0x00000176}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000177}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, - {0x10030, 0x000201DF}, - {0x10030, 0x000205D9}, - {0x10030, 0x000209D3}, - {0x10030, 0x00020D99}, - {0x10030, 0x00021193}, - {0x10030, 0x0002155F}, - {0x10030, 0x00021959}, - {0x10030, 0x00021D21}, - {0x10030, 0x00022119}, - {0x10030, 0x000224DF}, - {0x10030, 0x000228D9}, - {0x10030, 0x00022C9F}, - {0x10030, 0x00023099}, - {0x10030, 0x0002345F}, - {0x10030, 0x00023859}, - {0x10030, 0x00023C1F}, - {0x10030, 0x00024019}, - {0x10030, 0x00024413}, - {0x10030, 0x000281CD}, - {0x10030, 0x000285DB}, - {0x10030, 0x000289D5}, - {0x10030, 0x00028D9B}, - {0x10030, 0x0002918D}, - {0x10030, 0x00029555}, - {0x10030, 0x00029957}, - {0x10030, 0x00029D1F}, - {0x10030, 0x0002A119}, - {0x10030, 0x0002A4DF}, - {0x10030, 0x0002A8D9}, - {0x10030, 0x0002AC9F}, - {0x10030, 0x0002B099}, - {0x10030, 0x0002B45F}, - {0x10030, 0x0002B859}, - {0x10030, 0x0002BC1F}, - {0x10030, 0x0002C019}, - {0x10030, 0x0002C413}, - {0x10030, 0x000301D9}, - {0x10030, 0x000305DB}, - {0x10030, 0x000309D5}, - {0x10030, 0x00030D9B}, - {0x10030, 0x00031195}, - {0x10030, 0x0003155D}, - {0x10030, 0x00031955}, - {0x10030, 0x00031D1D}, - {0x10030, 0x00032119}, - {0x10030, 0x000324DF}, - {0x10030, 0x000328D9}, - {0x10030, 0x00032C9F}, - {0x10030, 0x00033099}, - {0x10030, 0x0003345F}, - {0x10030, 0x00033859}, - {0x10030, 0x00033C1F}, - {0x10030, 0x00034019}, - {0x10030, 0x00034413}, - {0x10030, 0x000601E1}, - {0x10030, 0x000605DB}, - {0x10030, 0x000609D5}, - {0x10030, 0x00060D9B}, - {0x10030, 0x00061195}, - {0x10030, 0x0006155B}, - {0x10030, 0x00061957}, - {0x10030, 0x00061D1F}, - {0x10030, 0x00062119}, - {0x10030, 0x000624DF}, - {0x10030, 0x000628D9}, - {0x10030, 0x00062C9F}, - {0x10030, 0x00063099}, - {0x10030, 0x0006345F}, - {0x10030, 0x00063859}, - {0x10030, 0x00063C1F}, - {0x10030, 0x00064019}, - {0x10030, 0x00064413}, - {0x10030, 0x000681E1}, - {0x10030, 0x000685DB}, - {0x10030, 0x000689D5}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955B}, - {0x10030, 0x00069957}, - {0x10030, 0x00069D1F}, - {0x10030, 0x0006A119}, - {0x10030, 0x0006A4DF}, - {0x10030, 0x0006A8D9}, - {0x10030, 0x0006AC9F}, - {0x10030, 0x0006B099}, - {0x10030, 0x0006B45F}, - {0x10030, 0x0006B859}, - {0x10030, 0x0006BC1F}, - {0x10030, 0x0006C019}, - {0x10030, 0x0006C413}, - {0x10030, 0x000701E1}, - {0x10030, 0x000705DB}, - {0x10030, 0x000709D5}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071957}, - {0x10030, 0x00071D1F}, - {0x10030, 0x00072119}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072C9F}, - {0x10030, 0x00073099}, - {0x10030, 0x0007345F}, - {0x10030, 0x00073859}, - {0x10030, 0x00073C1F}, - {0x10030, 0x00074019}, - {0x10030, 0x00074413}, - {0x10030, 0x000781DF}, - {0x10030, 0x000785D9}, - {0x10030, 0x000789D3}, - {0x10030, 0x00078D99}, - {0x10030, 0x00079193}, - {0x10030, 0x0007955F}, - {0x10030, 0x00079959}, - {0x10030, 0x00079D21}, - {0x10030, 0x0007A115}, - {0x10030, 0x0007A4DF}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007AC9F}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B45F}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC1F}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, - {0x10030, 0x00000000}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x00000178}, + {0x03F, 0x00050002}, + {0x033, 0x00000179}, + {0x03F, 0x00060032}, + {0x033, 0x0000017A}, + {0x03F, 0x00050042}, + {0x033, 0x0000017B}, + {0x03F, 0x00040042}, + {0x033, 0x0000017C}, + {0x03F, 0x00008001}, + {0x033, 0x0000017D}, + {0x03F, 0x00008002}, + {0x033, 0x0000017E}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x0000017F}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001A0}, + {0x03F, 0x00050002}, + {0x033, 0x000001A1}, + {0x03F, 0x00060032}, + {0x033, 0x000001A2}, + {0x03F, 0x00050042}, + {0x033, 0x000001A3}, + {0x03F, 0x00040042}, + {0x033, 0x000001A4}, + {0x03F, 0x00008001}, + {0x033, 0x000001A5}, + {0x03F, 0x00008002}, + {0x033, 0x000001A6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, {0xA0000000, 0x00000000}, - {0x10030, 0x000001EF}, - {0x10030, 0x000005E9}, - {0x10030, 0x000009E3}, - {0x10030, 0x00000DDD}, - {0x10030, 0x000011D7}, - {0x10030, 0x0000159F}, - {0x10030, 0x00001999}, - {0x10030, 0x00001D5F}, - {0x10030, 0x00002159}, - {0x10030, 0x0000251F}, - {0x10030, 0x00002919}, - {0x10030, 0x00002CDF}, - {0x10030, 0x000030D9}, - {0x10030, 0x0000349F}, - {0x10030, 0x00003899}, - {0x10030, 0x00003C5F}, - {0x10030, 0x00004059}, - {0x10030, 0x00004453}, - {0x10030, 0x000201ED}, - {0x10030, 0x000205AD}, - {0x10030, 0x000209A7}, - {0x10030, 0x00020DA1}, - {0x10030, 0x0002119B}, - {0x10030, 0x00021561}, - {0x10030, 0x0002195B}, - {0x10030, 0x00021D27}, - {0x10030, 0x00022121}, - {0x10030, 0x000224E9}, - {0x10030, 0x000228E3}, - {0x10030, 0x00022CA9}, - {0x10030, 0x000230A3}, - {0x10030, 0x00023469}, - {0x10030, 0x00023863}, - {0x10030, 0x00023C29}, - {0x10030, 0x00024023}, - {0x10030, 0x0002441D}, - {0x10030, 0x000281EF}, - {0x10030, 0x000285AF}, - {0x10030, 0x000289A9}, - {0x10030, 0x00028DA3}, - {0x10030, 0x0002919D}, - {0x10030, 0x00029563}, - {0x10030, 0x0002995D}, - {0x10030, 0x00029D25}, - {0x10030, 0x0002A11F}, - {0x10030, 0x0002A4E7}, - {0x10030, 0x0002A8E1}, - {0x10030, 0x0002ACA7}, - {0x10030, 0x0002B0A1}, - {0x10030, 0x0002B467}, - {0x10030, 0x0002B861}, - {0x10030, 0x0002BC27}, - {0x10030, 0x0002C021}, - {0x10030, 0x0002C41B}, - {0x10030, 0x000301EF}, - {0x10030, 0x000305AF}, - {0x10030, 0x000309A9}, - {0x10030, 0x00030DA3}, - {0x10030, 0x0003119D}, - {0x10030, 0x00031563}, - {0x10030, 0x0003195D}, - {0x10030, 0x00031D25}, - {0x10030, 0x0003211F}, - {0x10030, 0x000324E7}, - {0x10030, 0x000328E1}, - {0x10030, 0x00032CA7}, - {0x10030, 0x000330A1}, - {0x10030, 0x00033467}, - {0x10030, 0x00033861}, - {0x10030, 0x00033C27}, - {0x10030, 0x00034021}, - {0x10030, 0x0003441B}, - {0x10030, 0x000601EB}, - {0x10030, 0x000605AB}, - {0x10030, 0x000609A5}, - {0x10030, 0x00060D9F}, - {0x10030, 0x00061199}, - {0x10030, 0x00061593}, - {0x10030, 0x00061959}, - {0x10030, 0x00061D53}, - {0x10030, 0x0006211B}, - {0x10030, 0x00062515}, - {0x10030, 0x000628DD}, - {0x10030, 0x00062CD7}, - {0x10030, 0x0006309D}, - {0x10030, 0x00063497}, - {0x10030, 0x0006385D}, - {0x10030, 0x00063C57}, - {0x10030, 0x0006401D}, - {0x10030, 0x00064417}, - {0x10030, 0x000681E7}, - {0x10030, 0x000685A7}, - {0x10030, 0x000689A1}, - {0x10030, 0x00068D9B}, - {0x10030, 0x00069195}, - {0x10030, 0x0006955F}, - {0x10030, 0x00069959}, - {0x10030, 0x00069D21}, - {0x10030, 0x0006A11B}, - {0x10030, 0x0006A4E3}, - {0x10030, 0x0006A8DD}, - {0x10030, 0x0006ACA5}, - {0x10030, 0x0006B09F}, - {0x10030, 0x0006B465}, - {0x10030, 0x0006B85F}, - {0x10030, 0x0006BC25}, - {0x10030, 0x0006C01F}, - {0x10030, 0x0006C419}, - {0x10030, 0x000701E7}, - {0x10030, 0x000705A7}, - {0x10030, 0x000709A1}, - {0x10030, 0x00070D9B}, - {0x10030, 0x00071195}, - {0x10030, 0x0007155B}, - {0x10030, 0x00071955}, - {0x10030, 0x00071D1D}, - {0x10030, 0x00072117}, - {0x10030, 0x000724DF}, - {0x10030, 0x000728D9}, - {0x10030, 0x00072CA1}, - {0x10030, 0x0007309B}, - {0x10030, 0x00073461}, - {0x10030, 0x0007385B}, - {0x10030, 0x00073C21}, - {0x10030, 0x0007401B}, - {0x10030, 0x0007441B}, - {0x10030, 0x000781E9}, - {0x10030, 0x000785A9}, - {0x10030, 0x000789A3}, - {0x10030, 0x00078D9D}, - {0x10030, 0x00079197}, - {0x10030, 0x00079591}, - {0x10030, 0x00079957}, - {0x10030, 0x00079D51}, - {0x10030, 0x0007A119}, - {0x10030, 0x0007A513}, - {0x10030, 0x0007A8D9}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B099}, - {0x10030, 0x0007B493}, - {0x10030, 0x0007B859}, - {0x10030, 0x0007BC53}, - {0x10030, 0x0007C019}, - {0x10030, 0x0007C413}, + {0x03F, 0x00000003}, {0xB0000000, 0x00000000}, - {0x100EE, 0x00000000}, - {0x100EE, 0x00002000}, - {0x10030, 0x000000FC}, - {0x10030, 0x000004F9}, - {0x10030, 0x000008F6}, - {0x10030, 0x00000CF3}, - {0x10030, 0x000010F0}, - {0x10030, 0x000014ED}, - {0x10030, 0x000018AC}, - {0x10030, 0x00001CA9}, - {0x10030, 0x00002069}, - {0x10030, 0x00002466}, - {0x10030, 0x00002829}, - {0x10030, 0x00002C26}, - {0x10030, 0x00003023}, - {0x10030, 0x00003420}, - {0x10030, 0x0000381D}, - {0x10030, 0x00003C1A}, - {0x10030, 0x00004017}, - {0x100EE, 0x00000000}, - {0x100EE, 0x00002000}, - {0x10030, 0x000780F4}, - {0x10030, 0x000784F1}, - {0x10030, 0x000788EE}, - {0x10030, 0x00078CEB}, - {0x10030, 0x000790E8}, - {0x10030, 0x000794E5}, - {0x10030, 0x000798E2}, - {0x10030, 0x00079CDF}, - {0x10030, 0x0007A0DC}, - {0x10030, 0x0007A4D9}, - {0x10030, 0x0007A8D6}, - {0x10030, 0x0007ACD3}, - {0x10030, 0x0007B0D0}, - {0x10030, 0x0007B4CD}, - {0x10030, 0x0007B8CA}, - {0x10030, 0x0007BC07}, - {0x10030, 0x0007C004}, - {0x100EE, 0x00000000}, - {0x0EF, 0x00002000}, - {0x033, 0x00000008}, - {0x03F, 0x00000004}, - {0x033, 0x00000009}, + {0x033, 0x000001A7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000000A}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000000B}, - {0x03F, 0x00000002}, - {0x033, 0x0000000C}, - {0x03F, 0x00000002}, - {0x033, 0x0000000D}, - {0x03F, 0x00000002}, - {0x033, 0x0000000E}, - {0x03F, 0x00000002}, - {0x033, 0x0000000F}, - {0x03F, 0x00000002}, - {0x0EF, 0x00000000}, - {0x0EB, 0x00040000}, - {0x030, 0x000109B7}, - {0x0EB, 0x00000000}, - {0x0EF, 0x00008000}, - {0x033, 0x00000020}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001A8}, {0x03F, 0x00050002}, - {0x033, 0x00000021}, + {0x033, 0x000001A9}, {0x03F, 0x00060032}, - {0x033, 0x00000022}, + {0x033, 0x000001AA}, {0x03F, 0x00050042}, - {0x033, 0x00000023}, + {0x033, 0x000001AB}, {0x03F, 0x00040042}, - {0x033, 0x00000024}, + {0x033, 0x000001AC}, {0x03F, 0x00008001}, - {0x033, 0x00000025}, + {0x033, 0x000001AD}, {0x03F, 0x00008002}, - {0x033, 0x00000026}, + {0x033, 0x000001AE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000027}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000028}, - {0x03F, 0x00050002}, - {0x033, 0x00000029}, - {0x03F, 0x00060032}, - {0x033, 0x0000002A}, - {0x03F, 0x00050042}, - {0x033, 0x0000002B}, - {0x03F, 0x00040042}, - {0x033, 0x0000002C}, - {0x03F, 0x00008001}, - {0x033, 0x0000002D}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000002E}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000002F}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000030}, - {0x03F, 0x00050002}, - {0x033, 0x00000031}, - {0x03F, 0x00060032}, - {0x033, 0x00000032}, - {0x03F, 0x00050042}, - {0x033, 0x00000033}, - {0x03F, 0x00040042}, - {0x033, 0x00000034}, - {0x03F, 0x00008001}, - {0x033, 0x00000035}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000036}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000037}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001AF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000060}, - {0x03F, 0x00050002}, - {0x033, 0x00000061}, - {0x03F, 0x00060032}, - {0x033, 0x00000062}, - {0x03F, 0x00050042}, - {0x033, 0x00000063}, - {0x03F, 0x00040042}, - {0x033, 0x00000064}, - {0x03F, 0x00008001}, - {0x033, 0x00000065}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000066}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000067}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000068}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001B0}, {0x03F, 0x00050002}, - {0x033, 0x00000069}, + {0x033, 0x000001B1}, {0x03F, 0x00060032}, - {0x033, 0x0000006A}, + {0x033, 0x000001B2}, {0x03F, 0x00050042}, - {0x033, 0x0000006B}, + {0x033, 0x000001B3}, {0x03F, 0x00040042}, - {0x033, 0x0000006C}, + {0x033, 0x000001B4}, {0x03F, 0x00008001}, - {0x033, 0x0000006D}, + {0x033, 0x000001B5}, {0x03F, 0x00008002}, - {0x033, 0x0000006E}, + {0x033, 0x000001B6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000006F}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000070}, - {0x03F, 0x00050002}, - {0x033, 0x00000071}, - {0x03F, 0x00060032}, - {0x033, 0x00000072}, - {0x03F, 0x00050042}, - {0x033, 0x00000073}, - {0x03F, 0x00040042}, - {0x033, 0x00000074}, - {0x03F, 0x00008001}, - {0x033, 0x00000075}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000076}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000077}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000078}, - {0x03F, 0x00050002}, - {0x033, 0x00000079}, - {0x03F, 0x00060032}, - {0x033, 0x0000007A}, - {0x03F, 0x00050042}, - {0x033, 0x0000007B}, - {0x03F, 0x00040042}, - {0x033, 0x0000007C}, - {0x03F, 0x00008001}, - {0x033, 0x0000007D}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000007E}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000007F}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001B7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000A0}, - {0x03F, 0x00050002}, - {0x033, 0x000000A1}, - {0x03F, 0x00060032}, - {0x033, 0x000000A2}, - {0x03F, 0x00050042}, - {0x033, 0x000000A3}, - {0x03F, 0x00040042}, - {0x033, 0x000000A4}, - {0x03F, 0x00008001}, - {0x033, 0x000000A5}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000A6}, - {0x03F, 0x00000003}, - {0x033, 0x000000A7}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000A8}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001E0}, {0x03F, 0x00050002}, - {0x033, 0x000000A9}, + {0x033, 0x000001E1}, {0x03F, 0x00060032}, - {0x033, 0x000000AA}, + {0x033, 0x000001E2}, {0x03F, 0x00050042}, - {0x033, 0x000000AB}, + {0x033, 0x000001E3}, {0x03F, 0x00040042}, - {0x033, 0x000000AC}, + {0x033, 0x000001E4}, {0x03F, 0x00008001}, - {0x033, 0x000000AD}, + {0x033, 0x000001E5}, {0x03F, 0x00008002}, - {0x033, 0x000000AE}, + {0x033, 0x000001E6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000AF}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000B0}, - {0x03F, 0x00050002}, - {0x033, 0x000000B1}, - {0x03F, 0x00060032}, - {0x033, 0x000000B2}, - {0x03F, 0x00050042}, - {0x033, 0x000000B3}, - {0x03F, 0x00040042}, - {0x033, 0x000000B4}, - {0x03F, 0x00008001}, - {0x033, 0x000000B5}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000B6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000B7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E0}, - {0x03F, 0x00050002}, - {0x033, 0x000000E1}, - {0x03F, 0x00060032}, - {0x033, 0x000000E2}, - {0x03F, 0x00050042}, - {0x033, 0x000000E3}, - {0x03F, 0x00040042}, - {0x033, 0x000000E4}, - {0x03F, 0x00008001}, - {0x033, 0x000000E5}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000E6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E7}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001E7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000E8}, - {0x03F, 0x00050002}, - {0x033, 0x000000E9}, - {0x03F, 0x00060032}, - {0x033, 0x000000EA}, - {0x03F, 0x00050042}, - {0x033, 0x000000EB}, - {0x03F, 0x00040042}, - {0x033, 0x000000EC}, - {0x03F, 0x00008001}, - {0x033, 0x000000ED}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000EE}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000EF}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F0}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001E8}, {0x03F, 0x00050002}, - {0x033, 0x000000F1}, + {0x033, 0x000001E9}, {0x03F, 0x00060032}, - {0x033, 0x000000F2}, + {0x033, 0x000001EA}, {0x03F, 0x00050042}, - {0x033, 0x000000F3}, + {0x033, 0x000001EB}, {0x03F, 0x00040042}, - {0x033, 0x000000F4}, + {0x033, 0x000001EC}, {0x03F, 0x00008001}, - {0x033, 0x000000F5}, + {0x033, 0x000001ED}, {0x03F, 0x00008002}, - {0x033, 0x000000F6}, + {0x033, 0x000001EE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000F8}, - {0x03F, 0x00050002}, - {0x033, 0x000000F9}, - {0x03F, 0x00060032}, - {0x033, 0x000000FA}, - {0x03F, 0x00050042}, - {0x033, 0x000000FB}, - {0x03F, 0x00040042}, - {0x033, 0x000000FC}, - {0x03F, 0x00008001}, - {0x033, 0x000000FD}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000000FE}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000000FF}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000120}, - {0x03F, 0x00050002}, - {0x033, 0x00000121}, - {0x03F, 0x00060032}, - {0x033, 0x00000122}, - {0x03F, 0x00050042}, - {0x033, 0x00000123}, - {0x03F, 0x00040042}, - {0x033, 0x00000124}, - {0x03F, 0x00008001}, - {0x033, 0x00000125}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000126}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000127}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001EF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000128}, - {0x03F, 0x00050002}, - {0x033, 0x00000129}, - {0x03F, 0x00060032}, - {0x033, 0x0000012A}, - {0x03F, 0x00050042}, - {0x033, 0x0000012B}, - {0x03F, 0x00040042}, - {0x033, 0x0000012C}, - {0x03F, 0x00008001}, - {0x033, 0x0000012D}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000012E}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000012F}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000130}, - {0x03F, 0x00050002}, - {0x033, 0x00000131}, - {0x03F, 0x00060032}, - {0x033, 0x00000132}, - {0x03F, 0x00050042}, - {0x033, 0x00000133}, - {0x03F, 0x00040042}, - {0x033, 0x00000134}, - {0x03F, 0x00008001}, - {0x033, 0x00000135}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x00000136}, - {0x03F, 0x00000003}, - {0x033, 0x00000137}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000160}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001F0}, {0x03F, 0x00050002}, - {0x033, 0x00000161}, + {0x033, 0x000001F1}, {0x03F, 0x00060032}, - {0x033, 0x00000162}, + {0x033, 0x000001F2}, {0x03F, 0x00050042}, - {0x033, 0x00000163}, + {0x033, 0x000001F3}, {0x03F, 0x00040042}, - {0x033, 0x00000164}, + {0x033, 0x000001F4}, {0x03F, 0x00008001}, - {0x033, 0x00000165}, + {0x033, 0x000001F5}, {0x03F, 0x00008002}, - {0x033, 0x00000166}, + {0x033, 0x000001F6}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000167}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000168}, - {0x03F, 0x00050002}, - {0x033, 0x00000169}, - {0x03F, 0x00060032}, - {0x033, 0x0000016A}, - {0x03F, 0x00050042}, - {0x033, 0x0000016B}, - {0x03F, 0x00040042}, - {0x033, 0x0000016C}, - {0x03F, 0x00008001}, - {0x033, 0x0000016D}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x0000016E}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000016F}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000170}, - {0x03F, 0x00050002}, - {0x033, 0x00000171}, - {0x03F, 0x00060032}, - {0x033, 0x00000172}, - {0x03F, 0x00050042}, - {0x033, 0x00000173}, - {0x03F, 0x00040042}, - {0x033, 0x00000174}, - {0x03F, 0x00008001}, - {0x033, 0x00000175}, - {0x03F, 0x00008002}, - {0x033, 0x00000176}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000177}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x00000178}, - {0x03F, 0x00050002}, - {0x033, 0x00000179}, - {0x03F, 0x00060032}, - {0x033, 0x0000017A}, - {0x03F, 0x00050042}, - {0x033, 0x0000017B}, - {0x03F, 0x00040042}, - {0x033, 0x0000017C}, - {0x03F, 0x00008001}, - {0x033, 0x0000017D}, - {0x03F, 0x00008002}, - {0x033, 0x0000017E}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x0000017F}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A0}, - {0x03F, 0x00050002}, - {0x033, 0x000001A1}, - {0x03F, 0x00060032}, - {0x033, 0x000001A2}, - {0x03F, 0x00050042}, - {0x033, 0x000001A3}, - {0x03F, 0x00040042}, - {0x033, 0x000001A4}, - {0x03F, 0x00008001}, - {0x033, 0x000001A5}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001A6}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A7}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001F7}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001A8}, - {0x03F, 0x00050002}, - {0x033, 0x000001A9}, - {0x03F, 0x00060032}, - {0x033, 0x000001AA}, - {0x03F, 0x00050042}, - {0x033, 0x000001AB}, - {0x03F, 0x00040042}, - {0x033, 0x000001AC}, - {0x03F, 0x00008001}, - {0x033, 0x000001AD}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001AE}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001AF}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001B0}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, + {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001F8}, {0x03F, 0x00050002}, - {0x033, 0x000001B1}, + {0x033, 0x000001F9}, {0x03F, 0x00060032}, - {0x033, 0x000001B2}, + {0x033, 0x000001FA}, {0x03F, 0x00050042}, - {0x033, 0x000001B3}, + {0x033, 0x000001FB}, {0x03F, 0x00040042}, - {0x033, 0x000001B4}, + {0x033, 0x000001FC}, {0x03F, 0x00008001}, - {0x033, 0x000001B5}, + {0x033, 0x000001FD}, {0x03F, 0x00008002}, - {0x033, 0x000001B6}, + {0x033, 0x000001FE}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001B7}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001E0}, - {0x03F, 0x00050002}, - {0x033, 0x000001E1}, - {0x03F, 0x00060032}, - {0x033, 0x000001E2}, - {0x03F, 0x00050042}, - {0x033, 0x000001E3}, - {0x03F, 0x00040042}, - {0x033, 0x000001E4}, - {0x03F, 0x00008001}, - {0x033, 0x000001E5}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001E6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001E7}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001E8}, - {0x03F, 0x00050002}, - {0x033, 0x000001E9}, - {0x03F, 0x00060032}, - {0x033, 0x000001EA}, - {0x03F, 0x00050042}, - {0x033, 0x000001EB}, - {0x03F, 0x00040042}, - {0x033, 0x000001EC}, - {0x03F, 0x00008001}, - {0x033, 0x000001ED}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001EE}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001EF}, + {0xB0000000, 0x00000000}, + {0x033, 0x000001FF}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001F0}, - {0x03F, 0x00050002}, - {0x033, 0x000001F1}, - {0x03F, 0x00060032}, - {0x033, 0x000001F2}, - {0x03F, 0x00050042}, - {0x033, 0x000001F3}, - {0x03F, 0x00040042}, - {0x033, 0x000001F4}, - {0x03F, 0x00008001}, - {0x033, 0x000001F5}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00008002}, - {0x033, 0x000001F6}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001F7}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001F8}, - {0x03F, 0x00050002}, - {0x033, 0x000001F9}, - {0x03F, 0x00060032}, - {0x033, 0x000001FA}, - {0x03F, 0x00050042}, - {0x033, 0x000001FB}, - {0x03F, 0x00040042}, - {0x033, 0x000001FC}, - {0x03F, 0x00008001}, - {0x033, 0x000001FD}, - {0x03F, 0x00008002}, - {0x033, 0x000001FE}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, {0x03F, 0x00000003}, - {0x033, 0x000001FF}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00000003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x03F, 0x00008002}, + {0xA0000000, 0x00000000}, {0x03F, 0x00000003}, + {0xB0000000, 0x00000000}, {0x0EF, 0x00000000}, {0x005, 0x00000001}, {0x10005, 0x00000001}, @@ -11810,7 +25361,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10030, 0x00022000}, {0x10030, 0x00023000}, {0x10030, 0x00024000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00025003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00025000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00026003}, {0x10030, 0x00027003}, {0x10030, 0x00028000}, @@ -11818,7 +25411,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10030, 0x0002A000}, {0x10030, 0x0002B000}, {0x10030, 0x0002C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0002D003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0002D000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0002E003}, {0x10030, 0x0002F003}, {0x10030, 0x00030000}, @@ -11826,7 +25461,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10030, 0x00032000}, {0x10030, 0x00033000}, {0x10030, 0x00034000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, {0x10030, 0x00035000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00035003}, + {0xA0000000, 0x00000000}, + {0x10030, 0x00035000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00036003}, {0x10030, 0x00037003}, {0x10030, 0x00038000}, @@ -11834,7 +25511,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10030, 0x0003A000}, {0x10030, 0x0003B000}, {0x10030, 0x0003C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0003D003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0003D000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0003E003}, {0x10030, 0x0003F003}, {0x10030, 0x00060000}, @@ -11842,32 +25561,280 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x10030, 0x00062000}, {0x10030, 0x00063000}, {0x10030, 0x00064000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065000}, + {0x10030, 0x00066000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00065003}, + {0x10030, 0x00066003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00065000}, {0x10030, 0x00066000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00067003}, {0x10030, 0x00068000}, {0x10030, 0x00069000}, {0x10030, 0x0006A000}, {0x10030, 0x0006B000}, {0x10030, 0x0006C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D000}, + {0x10030, 0x0006E000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0006D003}, + {0x10030, 0x0006E003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0006D000}, {0x10030, 0x0006E000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0006F003}, {0x10030, 0x00070000}, {0x10030, 0x00071000}, {0x10030, 0x00072000}, {0x10030, 0x00073000}, {0x10030, 0x00074000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075000}, + {0x10030, 0x00076000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x00075003}, + {0x10030, 0x00076003}, + {0xA0000000, 0x00000000}, {0x10030, 0x00075000}, {0x10030, 0x00076000}, + {0xB0000000, 0x00000000}, {0x10030, 0x00077003}, {0x10030, 0x00078000}, {0x10030, 0x00079000}, {0x10030, 0x0007A000}, {0x10030, 0x0007B000}, {0x10030, 0x0007C000}, + {0x80010000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90020000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90320000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90330000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90340000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90350000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90360000, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D000}, + {0x10030, 0x0007E000}, + {0x90010001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90020001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90030001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90040001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90050001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90070001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90320001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90330001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90340001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90350001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90360001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x903f0001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0x90400001, 0x00000000}, {0x40000000, 0x00000000}, + {0x10030, 0x0007D003}, + {0x10030, 0x0007E003}, + {0xA0000000, 0x00000000}, {0x10030, 0x0007D000}, {0x10030, 0x0007E000}, + {0xB0000000, 0x00000000}, {0x10030, 0x0007F003}, {0x0ED, 0x00000010}, {0x033, 0x00000001}, @@ -11884,7 +25851,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = { {0x03F, 0x0000000A}, {0x0ED, 0x00000000}, {0x100EE, 0x00000000}, - {0x0FE, 0x00000031}, + {0x0FE, 0x00000048}, }; static const struct rtw89_reg2_def rtw89_8852c_phy_nctl_regs[] = { @@ -13825,1207 +27792,1722 @@ static const s8 _txpwr_track_delta_swingidx_2g_cck_a_p[] = { const u8 rtw89_8852c_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM] [RTW89_REGD_NUM] = { [0][0][RTW89_ACMA] = 0, + [0][0][RTW89_CN] = 0, [0][0][RTW89_ETSI] = 0, [0][0][RTW89_FCC] = 1, [0][0][RTW89_IC] = 1, + [0][0][RTW89_KCC] = 0, [0][0][RTW89_MKK] = 0, + [0][0][RTW89_UK] = 0, [0][1][RTW89_ACMA] = 0, + [0][1][RTW89_CN] = 0, [0][1][RTW89_ETSI] = 0, [0][1][RTW89_FCC] = 3, [0][1][RTW89_IC] = 3, + [0][1][RTW89_KCC] = 0, [0][1][RTW89_MKK] = 0, + [0][1][RTW89_UK] = 0, [1][1][RTW89_ACMA] = 0, + [1][1][RTW89_CN] = 0, [1][1][RTW89_ETSI] = 0, [1][1][RTW89_FCC] = 3, [1][1][RTW89_IC] = 3, + [1][1][RTW89_KCC] = 0, [1][1][RTW89_MKK] = 0, - [2][1][RTW89_FCC] = 1, + [1][1][RTW89_UK] = 0, + [2][1][RTW89_ETSI] = 0, + [2][1][RTW89_FCC] = 0, + [2][1][RTW89_KCC] = 0, }; const s8 rtw89_8852c_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = { - [0][0][0][0][RTW89_WW][0] = 60, - [0][0][0][0][RTW89_WW][1] = 60, - [0][0][0][0][RTW89_WW][2] = 60, - [0][0][0][0][RTW89_WW][3] = 60, - [0][0][0][0][RTW89_WW][4] = 60, - [0][0][0][0][RTW89_WW][5] = 60, - [0][0][0][0][RTW89_WW][6] = 60, - [0][0][0][0][RTW89_WW][7] = 60, - [0][0][0][0][RTW89_WW][8] = 60, - [0][0][0][0][RTW89_WW][9] = 60, - [0][0][0][0][RTW89_WW][10] = 60, - [0][0][0][0][RTW89_WW][11] = 60, - [0][0][0][0][RTW89_WW][12] = 48, + [0][0][0][0][RTW89_WW][0] = 58, + [0][0][0][0][RTW89_WW][1] = 58, + [0][0][0][0][RTW89_WW][2] = 58, + [0][0][0][0][RTW89_WW][3] = 58, + [0][0][0][0][RTW89_WW][4] = 58, + [0][0][0][0][RTW89_WW][5] = 58, + [0][0][0][0][RTW89_WW][6] = 58, + [0][0][0][0][RTW89_WW][7] = 58, + [0][0][0][0][RTW89_WW][8] = 58, + [0][0][0][0][RTW89_WW][9] = 58, + [0][0][0][0][RTW89_WW][10] = 58, + [0][0][0][0][RTW89_WW][11] = 58, + [0][0][0][0][RTW89_WW][12] = 46, [0][0][0][0][RTW89_WW][13] = 72, - [0][1][0][0][RTW89_WW][0] = 48, - [0][1][0][0][RTW89_WW][1] = 48, - [0][1][0][0][RTW89_WW][2] = 48, - [0][1][0][0][RTW89_WW][3] = 48, - [0][1][0][0][RTW89_WW][4] = 48, - [0][1][0][0][RTW89_WW][5] = 48, - [0][1][0][0][RTW89_WW][6] = 48, - [0][1][0][0][RTW89_WW][7] = 48, - [0][1][0][0][RTW89_WW][8] = 48, - [0][1][0][0][RTW89_WW][9] = 48, - [0][1][0][0][RTW89_WW][10] = 48, - [0][1][0][0][RTW89_WW][11] = 46, - [0][1][0][0][RTW89_WW][12] = 34, + [0][1][0][0][RTW89_WW][0] = 42, + [0][1][0][0][RTW89_WW][1] = 42, + [0][1][0][0][RTW89_WW][2] = 42, + [0][1][0][0][RTW89_WW][3] = 42, + [0][1][0][0][RTW89_WW][4] = 42, + [0][1][0][0][RTW89_WW][5] = 42, + [0][1][0][0][RTW89_WW][6] = 42, + [0][1][0][0][RTW89_WW][7] = 42, + [0][1][0][0][RTW89_WW][8] = 42, + [0][1][0][0][RTW89_WW][9] = 42, + [0][1][0][0][RTW89_WW][10] = 42, + [0][1][0][0][RTW89_WW][11] = 42, + [0][1][0][0][RTW89_WW][12] = 18, [0][1][0][0][RTW89_WW][13] = 60, [1][0][0][0][RTW89_WW][0] = 0, [1][0][0][0][RTW89_WW][1] = 0, - [1][0][0][0][RTW89_WW][2] = 42, - [1][0][0][0][RTW89_WW][3] = 42, - [1][0][0][0][RTW89_WW][4] = 42, + [1][0][0][0][RTW89_WW][2] = 44, + [1][0][0][0][RTW89_WW][3] = 58, + [1][0][0][0][RTW89_WW][4] = 58, [1][0][0][0][RTW89_WW][5] = 58, - [1][0][0][0][RTW89_WW][6] = 42, - [1][0][0][0][RTW89_WW][7] = 42, - [1][0][0][0][RTW89_WW][8] = 42, - [1][0][0][0][RTW89_WW][9] = 34, - [1][0][0][0][RTW89_WW][10] = 22, + [1][0][0][0][RTW89_WW][6] = 46, + [1][0][0][0][RTW89_WW][7] = 46, + [1][0][0][0][RTW89_WW][8] = 28, + [1][0][0][0][RTW89_WW][9] = 26, + [1][0][0][0][RTW89_WW][10] = 26, [1][0][0][0][RTW89_WW][11] = 0, [1][0][0][0][RTW89_WW][12] = 0, [1][0][0][0][RTW89_WW][13] = 0, [1][1][0][0][RTW89_WW][0] = 0, [1][1][0][0][RTW89_WW][1] = 0, - [1][1][0][0][RTW89_WW][2] = 38, - [1][1][0][0][RTW89_WW][3] = 38, - [1][1][0][0][RTW89_WW][4] = 38, - [1][1][0][0][RTW89_WW][5] = 48, - [1][1][0][0][RTW89_WW][6] = 26, - [1][1][0][0][RTW89_WW][7] = 26, - [1][1][0][0][RTW89_WW][8] = 26, - [1][1][0][0][RTW89_WW][9] = 22, - [1][1][0][0][RTW89_WW][10] = 22, + [1][1][0][0][RTW89_WW][2] = 46, + [1][1][0][0][RTW89_WW][3] = 46, + [1][1][0][0][RTW89_WW][4] = 46, + [1][1][0][0][RTW89_WW][5] = 46, + [1][1][0][0][RTW89_WW][6] = 40, + [1][1][0][0][RTW89_WW][7] = 40, + [1][1][0][0][RTW89_WW][8] = 14, + [1][1][0][0][RTW89_WW][9] = 14, + [1][1][0][0][RTW89_WW][10] = 12, [1][1][0][0][RTW89_WW][11] = 0, [1][1][0][0][RTW89_WW][12] = 0, [1][1][0][0][RTW89_WW][13] = 0, - [0][0][1][0][RTW89_WW][0] = 60, - [0][0][1][0][RTW89_WW][1] = 60, - [0][0][1][0][RTW89_WW][2] = 60, - [0][0][1][0][RTW89_WW][3] = 60, - [0][0][1][0][RTW89_WW][4] = 60, - [0][0][1][0][RTW89_WW][5] = 60, - [0][0][1][0][RTW89_WW][6] = 60, - [0][0][1][0][RTW89_WW][7] = 60, - [0][0][1][0][RTW89_WW][8] = 60, - [0][0][1][0][RTW89_WW][9] = 60, - [0][0][1][0][RTW89_WW][10] = 60, - [0][0][1][0][RTW89_WW][11] = 46, - [0][0][1][0][RTW89_WW][12] = 42, + [0][0][1][0][RTW89_WW][0] = 58, + [0][0][1][0][RTW89_WW][1] = 58, + [0][0][1][0][RTW89_WW][2] = 58, + [0][0][1][0][RTW89_WW][3] = 58, + [0][0][1][0][RTW89_WW][4] = 58, + [0][0][1][0][RTW89_WW][5] = 58, + [0][0][1][0][RTW89_WW][6] = 58, + [0][0][1][0][RTW89_WW][7] = 58, + [0][0][1][0][RTW89_WW][8] = 58, + [0][0][1][0][RTW89_WW][9] = 58, + [0][0][1][0][RTW89_WW][10] = 58, + [0][0][1][0][RTW89_WW][11] = 58, + [0][0][1][0][RTW89_WW][12] = 58, [0][0][1][0][RTW89_WW][13] = 0, - [0][1][1][0][RTW89_WW][0] = 48, - [0][1][1][0][RTW89_WW][1] = 48, - [0][1][1][0][RTW89_WW][2] = 48, - [0][1][1][0][RTW89_WW][3] = 48, - [0][1][1][0][RTW89_WW][4] = 48, - [0][1][1][0][RTW89_WW][5] = 48, - [0][1][1][0][RTW89_WW][6] = 48, - [0][1][1][0][RTW89_WW][7] = 48, - [0][1][1][0][RTW89_WW][8] = 48, - [0][1][1][0][RTW89_WW][9] = 48, - [0][1][1][0][RTW89_WW][10] = 48, - [0][1][1][0][RTW89_WW][11] = 38, - [0][1][1][0][RTW89_WW][12] = 34, + [0][1][1][0][RTW89_WW][0] = 46, + [0][1][1][0][RTW89_WW][1] = 46, + [0][1][1][0][RTW89_WW][2] = 46, + [0][1][1][0][RTW89_WW][3] = 46, + [0][1][1][0][RTW89_WW][4] = 46, + [0][1][1][0][RTW89_WW][5] = 46, + [0][1][1][0][RTW89_WW][6] = 46, + [0][1][1][0][RTW89_WW][7] = 46, + [0][1][1][0][RTW89_WW][8] = 46, + [0][1][1][0][RTW89_WW][9] = 46, + [0][1][1][0][RTW89_WW][10] = 46, + [0][1][1][0][RTW89_WW][11] = 46, + [0][1][1][0][RTW89_WW][12] = 36, [0][1][1][0][RTW89_WW][13] = 0, - [0][0][2][0][RTW89_WW][0] = 60, - [0][0][2][0][RTW89_WW][1] = 60, - [0][0][2][0][RTW89_WW][2] = 60, - [0][0][2][0][RTW89_WW][3] = 60, - [0][0][2][0][RTW89_WW][4] = 60, - [0][0][2][0][RTW89_WW][5] = 60, - [0][0][2][0][RTW89_WW][6] = 60, - [0][0][2][0][RTW89_WW][7] = 60, - [0][0][2][0][RTW89_WW][8] = 60, - [0][0][2][0][RTW89_WW][9] = 60, - [0][0][2][0][RTW89_WW][10] = 60, - [0][0][2][0][RTW89_WW][11] = 46, - [0][0][2][0][RTW89_WW][12] = 42, + [0][0][2][0][RTW89_WW][0] = 58, + [0][0][2][0][RTW89_WW][1] = 58, + [0][0][2][0][RTW89_WW][2] = 58, + [0][0][2][0][RTW89_WW][3] = 58, + [0][0][2][0][RTW89_WW][4] = 58, + [0][0][2][0][RTW89_WW][5] = 58, + [0][0][2][0][RTW89_WW][6] = 58, + [0][0][2][0][RTW89_WW][7] = 58, + [0][0][2][0][RTW89_WW][8] = 58, + [0][0][2][0][RTW89_WW][9] = 58, + [0][0][2][0][RTW89_WW][10] = 58, + [0][0][2][0][RTW89_WW][11] = 58, + [0][0][2][0][RTW89_WW][12] = 38, [0][0][2][0][RTW89_WW][13] = 0, - [0][1][2][0][RTW89_WW][0] = 48, - [0][1][2][0][RTW89_WW][1] = 48, - [0][1][2][0][RTW89_WW][2] = 48, - [0][1][2][0][RTW89_WW][3] = 48, - [0][1][2][0][RTW89_WW][4] = 48, - [0][1][2][0][RTW89_WW][5] = 48, - [0][1][2][0][RTW89_WW][6] = 48, - [0][1][2][0][RTW89_WW][7] = 48, - [0][1][2][0][RTW89_WW][8] = 48, - [0][1][2][0][RTW89_WW][9] = 48, - [0][1][2][0][RTW89_WW][10] = 48, - [0][1][2][0][RTW89_WW][11] = 38, - [0][1][2][0][RTW89_WW][12] = 34, + [0][1][2][0][RTW89_WW][0] = 46, + [0][1][2][0][RTW89_WW][1] = 46, + [0][1][2][0][RTW89_WW][2] = 46, + [0][1][2][0][RTW89_WW][3] = 46, + [0][1][2][0][RTW89_WW][4] = 46, + [0][1][2][0][RTW89_WW][5] = 46, + [0][1][2][0][RTW89_WW][6] = 46, + [0][1][2][0][RTW89_WW][7] = 46, + [0][1][2][0][RTW89_WW][8] = 46, + [0][1][2][0][RTW89_WW][9] = 46, + [0][1][2][0][RTW89_WW][10] = 46, + [0][1][2][0][RTW89_WW][11] = 46, + [0][1][2][0][RTW89_WW][12] = 16, [0][1][2][0][RTW89_WW][13] = 0, [0][1][2][1][RTW89_WW][0] = 36, - [0][1][2][1][RTW89_WW][1] = 36, - [0][1][2][1][RTW89_WW][2] = 36, - [0][1][2][1][RTW89_WW][3] = 36, - [0][1][2][1][RTW89_WW][4] = 36, - [0][1][2][1][RTW89_WW][5] = 36, - [0][1][2][1][RTW89_WW][6] = 36, - [0][1][2][1][RTW89_WW][7] = 36, - [0][1][2][1][RTW89_WW][8] = 36, - [0][1][2][1][RTW89_WW][9] = 36, - [0][1][2][1][RTW89_WW][10] = 36, - [0][1][2][1][RTW89_WW][11] = 36, - [0][1][2][1][RTW89_WW][12] = 34, + [0][1][2][1][RTW89_WW][1] = 34, + [0][1][2][1][RTW89_WW][2] = 34, + [0][1][2][1][RTW89_WW][3] = 34, + [0][1][2][1][RTW89_WW][4] = 34, + [0][1][2][1][RTW89_WW][5] = 34, + [0][1][2][1][RTW89_WW][6] = 34, + [0][1][2][1][RTW89_WW][7] = 34, + [0][1][2][1][RTW89_WW][8] = 34, + [0][1][2][1][RTW89_WW][9] = 34, + [0][1][2][1][RTW89_WW][10] = 34, + [0][1][2][1][RTW89_WW][11] = 34, + [0][1][2][1][RTW89_WW][12] = 16, [0][1][2][1][RTW89_WW][13] = 0, [1][0][2][0][RTW89_WW][0] = 0, [1][0][2][0][RTW89_WW][1] = 0, - [1][0][2][0][RTW89_WW][2] = 60, - [1][0][2][0][RTW89_WW][3] = 60, - [1][0][2][0][RTW89_WW][4] = 60, - [1][0][2][0][RTW89_WW][5] = 60, - [1][0][2][0][RTW89_WW][6] = 60, - [1][0][2][0][RTW89_WW][7] = 60, - [1][0][2][0][RTW89_WW][8] = 60, - [1][0][2][0][RTW89_WW][9] = 60, - [1][0][2][0][RTW89_WW][10] = 58, + [1][0][2][0][RTW89_WW][2] = 58, + [1][0][2][0][RTW89_WW][3] = 58, + [1][0][2][0][RTW89_WW][4] = 58, + [1][0][2][0][RTW89_WW][5] = 58, + [1][0][2][0][RTW89_WW][6] = 58, + [1][0][2][0][RTW89_WW][7] = 58, + [1][0][2][0][RTW89_WW][8] = 58, + [1][0][2][0][RTW89_WW][9] = 58, + [1][0][2][0][RTW89_WW][10] = 56, [1][0][2][0][RTW89_WW][11] = 0, [1][0][2][0][RTW89_WW][12] = 0, [1][0][2][0][RTW89_WW][13] = 0, [1][1][2][0][RTW89_WW][0] = 0, [1][1][2][0][RTW89_WW][1] = 0, - [1][1][2][0][RTW89_WW][2] = 46, - [1][1][2][0][RTW89_WW][3] = 46, - [1][1][2][0][RTW89_WW][4] = 48, - [1][1][2][0][RTW89_WW][5] = 48, - [1][1][2][0][RTW89_WW][6] = 48, - [1][1][2][0][RTW89_WW][7] = 46, - [1][1][2][0][RTW89_WW][8] = 46, + [1][1][2][0][RTW89_WW][2] = 34, + [1][1][2][0][RTW89_WW][3] = 34, + [1][1][2][0][RTW89_WW][4] = 34, + [1][1][2][0][RTW89_WW][5] = 34, + [1][1][2][0][RTW89_WW][6] = 34, + [1][1][2][0][RTW89_WW][7] = 34, + [1][1][2][0][RTW89_WW][8] = 34, [1][1][2][0][RTW89_WW][9] = 34, - [1][1][2][0][RTW89_WW][10] = 30, + [1][1][2][0][RTW89_WW][10] = 34, [1][1][2][0][RTW89_WW][11] = 0, [1][1][2][0][RTW89_WW][12] = 0, [1][1][2][0][RTW89_WW][13] = 0, [1][1][2][1][RTW89_WW][0] = 0, [1][1][2][1][RTW89_WW][1] = 0, - [1][1][2][1][RTW89_WW][2] = 36, - [1][1][2][1][RTW89_WW][3] = 36, - [1][1][2][1][RTW89_WW][4] = 36, - [1][1][2][1][RTW89_WW][5] = 36, - [1][1][2][1][RTW89_WW][6] = 36, - [1][1][2][1][RTW89_WW][7] = 36, - [1][1][2][1][RTW89_WW][8] = 36, + [1][1][2][1][RTW89_WW][2] = 34, + [1][1][2][1][RTW89_WW][3] = 34, + [1][1][2][1][RTW89_WW][4] = 34, + [1][1][2][1][RTW89_WW][5] = 34, + [1][1][2][1][RTW89_WW][6] = 34, + [1][1][2][1][RTW89_WW][7] = 34, + [1][1][2][1][RTW89_WW][8] = 34, [1][1][2][1][RTW89_WW][9] = 34, - [1][1][2][1][RTW89_WW][10] = 30, + [1][1][2][1][RTW89_WW][10] = 36, [1][1][2][1][RTW89_WW][11] = 0, [1][1][2][1][RTW89_WW][12] = 0, [1][1][2][1][RTW89_WW][13] = 0, - [0][0][0][0][RTW89_FCC][0] = 70, + [0][0][0][0][RTW89_FCC][0] = 76, [0][0][0][0][RTW89_ETSI][0] = 60, [0][0][0][0][RTW89_MKK][0] = 68, - [0][0][0][0][RTW89_IC][0] = 74, + [0][0][0][0][RTW89_IC][0] = 76, + [0][0][0][0][RTW89_KCC][0] = 68, [0][0][0][0][RTW89_ACMA][0] = 60, - [0][0][0][0][RTW89_FCC][1] = 70, + [0][0][0][0][RTW89_CN][0] = 58, + [0][0][0][0][RTW89_UK][0] = 60, + [0][0][0][0][RTW89_FCC][1] = 76, [0][0][0][0][RTW89_ETSI][1] = 60, [0][0][0][0][RTW89_MKK][1] = 68, - [0][0][0][0][RTW89_IC][1] = 74, + [0][0][0][0][RTW89_IC][1] = 76, + [0][0][0][0][RTW89_KCC][1] = 68, [0][0][0][0][RTW89_ACMA][1] = 60, - [0][0][0][0][RTW89_FCC][2] = 70, + [0][0][0][0][RTW89_CN][1] = 58, + [0][0][0][0][RTW89_UK][1] = 60, + [0][0][0][0][RTW89_FCC][2] = 76, [0][0][0][0][RTW89_ETSI][2] = 60, [0][0][0][0][RTW89_MKK][2] = 68, - [0][0][0][0][RTW89_IC][2] = 74, + [0][0][0][0][RTW89_IC][2] = 76, + [0][0][0][0][RTW89_KCC][2] = 68, [0][0][0][0][RTW89_ACMA][2] = 60, - [0][0][0][0][RTW89_FCC][3] = 70, + [0][0][0][0][RTW89_CN][2] = 58, + [0][0][0][0][RTW89_UK][2] = 60, + [0][0][0][0][RTW89_FCC][3] = 76, [0][0][0][0][RTW89_ETSI][3] = 60, [0][0][0][0][RTW89_MKK][3] = 68, - [0][0][0][0][RTW89_IC][3] = 74, + [0][0][0][0][RTW89_IC][3] = 76, + [0][0][0][0][RTW89_KCC][3] = 68, [0][0][0][0][RTW89_ACMA][3] = 60, - [0][0][0][0][RTW89_FCC][4] = 70, + [0][0][0][0][RTW89_CN][3] = 58, + [0][0][0][0][RTW89_UK][3] = 60, + [0][0][0][0][RTW89_FCC][4] = 76, [0][0][0][0][RTW89_ETSI][4] = 60, [0][0][0][0][RTW89_MKK][4] = 68, - [0][0][0][0][RTW89_IC][4] = 74, + [0][0][0][0][RTW89_IC][4] = 76, + [0][0][0][0][RTW89_KCC][4] = 68, [0][0][0][0][RTW89_ACMA][4] = 60, - [0][0][0][0][RTW89_FCC][5] = 70, + [0][0][0][0][RTW89_CN][4] = 58, + [0][0][0][0][RTW89_UK][4] = 60, + [0][0][0][0][RTW89_FCC][5] = 76, [0][0][0][0][RTW89_ETSI][5] = 60, [0][0][0][0][RTW89_MKK][5] = 68, - [0][0][0][0][RTW89_IC][5] = 74, + [0][0][0][0][RTW89_IC][5] = 76, + [0][0][0][0][RTW89_KCC][5] = 68, [0][0][0][0][RTW89_ACMA][5] = 60, - [0][0][0][0][RTW89_FCC][6] = 70, + [0][0][0][0][RTW89_CN][5] = 58, + [0][0][0][0][RTW89_UK][5] = 60, + [0][0][0][0][RTW89_FCC][6] = 76, [0][0][0][0][RTW89_ETSI][6] = 60, [0][0][0][0][RTW89_MKK][6] = 68, - [0][0][0][0][RTW89_IC][6] = 74, + [0][0][0][0][RTW89_IC][6] = 76, + [0][0][0][0][RTW89_KCC][6] = 68, [0][0][0][0][RTW89_ACMA][6] = 60, - [0][0][0][0][RTW89_FCC][7] = 70, + [0][0][0][0][RTW89_CN][6] = 58, + [0][0][0][0][RTW89_UK][6] = 60, + [0][0][0][0][RTW89_FCC][7] = 76, [0][0][0][0][RTW89_ETSI][7] = 60, [0][0][0][0][RTW89_MKK][7] = 68, - [0][0][0][0][RTW89_IC][7] = 74, + [0][0][0][0][RTW89_IC][7] = 76, + [0][0][0][0][RTW89_KCC][7] = 68, [0][0][0][0][RTW89_ACMA][7] = 60, - [0][0][0][0][RTW89_FCC][8] = 70, + [0][0][0][0][RTW89_CN][7] = 58, + [0][0][0][0][RTW89_UK][7] = 60, + [0][0][0][0][RTW89_FCC][8] = 76, [0][0][0][0][RTW89_ETSI][8] = 60, [0][0][0][0][RTW89_MKK][8] = 68, - [0][0][0][0][RTW89_IC][8] = 74, + [0][0][0][0][RTW89_IC][8] = 76, + [0][0][0][0][RTW89_KCC][8] = 68, [0][0][0][0][RTW89_ACMA][8] = 60, - [0][0][0][0][RTW89_FCC][9] = 70, + [0][0][0][0][RTW89_CN][8] = 58, + [0][0][0][0][RTW89_UK][8] = 60, + [0][0][0][0][RTW89_FCC][9] = 76, [0][0][0][0][RTW89_ETSI][9] = 60, [0][0][0][0][RTW89_MKK][9] = 68, - [0][0][0][0][RTW89_IC][9] = 74, + [0][0][0][0][RTW89_IC][9] = 76, + [0][0][0][0][RTW89_KCC][9] = 70, [0][0][0][0][RTW89_ACMA][9] = 60, - [0][0][0][0][RTW89_FCC][10] = 70, + [0][0][0][0][RTW89_CN][9] = 58, + [0][0][0][0][RTW89_UK][9] = 60, + [0][0][0][0][RTW89_FCC][10] = 76, [0][0][0][0][RTW89_ETSI][10] = 60, [0][0][0][0][RTW89_MKK][10] = 68, - [0][0][0][0][RTW89_IC][10] = 74, + [0][0][0][0][RTW89_IC][10] = 76, + [0][0][0][0][RTW89_KCC][10] = 70, [0][0][0][0][RTW89_ACMA][10] = 60, - [0][0][0][0][RTW89_FCC][11] = 62, + [0][0][0][0][RTW89_CN][10] = 58, + [0][0][0][0][RTW89_UK][10] = 60, + [0][0][0][0][RTW89_FCC][11] = 58, [0][0][0][0][RTW89_ETSI][11] = 60, [0][0][0][0][RTW89_MKK][11] = 68, - [0][0][0][0][RTW89_IC][11] = 72, + [0][0][0][0][RTW89_IC][11] = 58, + [0][0][0][0][RTW89_KCC][11] = 70, [0][0][0][0][RTW89_ACMA][11] = 60, - [0][0][0][0][RTW89_FCC][12] = 48, + [0][0][0][0][RTW89_CN][11] = 58, + [0][0][0][0][RTW89_UK][11] = 60, + [0][0][0][0][RTW89_FCC][12] = 46, [0][0][0][0][RTW89_ETSI][12] = 60, [0][0][0][0][RTW89_MKK][12] = 68, - [0][0][0][0][RTW89_IC][12] = 58, + [0][0][0][0][RTW89_IC][12] = 46, + [0][0][0][0][RTW89_KCC][12] = 70, [0][0][0][0][RTW89_ACMA][12] = 60, + [0][0][0][0][RTW89_CN][12] = 58, + [0][0][0][0][RTW89_UK][12] = 60, [0][0][0][0][RTW89_FCC][13] = 127, [0][0][0][0][RTW89_ETSI][13] = 127, [0][0][0][0][RTW89_MKK][13] = 72, [0][0][0][0][RTW89_IC][13] = 127, + [0][0][0][0][RTW89_KCC][13] = 127, [0][0][0][0][RTW89_ACMA][13] = 127, - [0][1][0][0][RTW89_FCC][0] = 66, + [0][0][0][0][RTW89_CN][13] = 127, + [0][0][0][0][RTW89_UK][13] = 127, + [0][1][0][0][RTW89_FCC][0] = 76, [0][1][0][0][RTW89_ETSI][0] = 48, [0][1][0][0][RTW89_MKK][0] = 58, - [0][1][0][0][RTW89_IC][0] = 74, + [0][1][0][0][RTW89_IC][0] = 76, + [0][1][0][0][RTW89_KCC][0] = 56, [0][1][0][0][RTW89_ACMA][0] = 48, - [0][1][0][0][RTW89_FCC][1] = 66, + [0][1][0][0][RTW89_CN][0] = 42, + [0][1][0][0][RTW89_UK][0] = 48, + [0][1][0][0][RTW89_FCC][1] = 76, [0][1][0][0][RTW89_ETSI][1] = 48, [0][1][0][0][RTW89_MKK][1] = 58, - [0][1][0][0][RTW89_IC][1] = 74, + [0][1][0][0][RTW89_IC][1] = 76, + [0][1][0][0][RTW89_KCC][1] = 56, [0][1][0][0][RTW89_ACMA][1] = 48, - [0][1][0][0][RTW89_FCC][2] = 66, + [0][1][0][0][RTW89_CN][1] = 42, + [0][1][0][0][RTW89_UK][1] = 48, + [0][1][0][0][RTW89_FCC][2] = 76, [0][1][0][0][RTW89_ETSI][2] = 48, [0][1][0][0][RTW89_MKK][2] = 58, - [0][1][0][0][RTW89_IC][2] = 74, + [0][1][0][0][RTW89_IC][2] = 76, + [0][1][0][0][RTW89_KCC][2] = 56, [0][1][0][0][RTW89_ACMA][2] = 48, - [0][1][0][0][RTW89_FCC][3] = 66, + [0][1][0][0][RTW89_CN][2] = 42, + [0][1][0][0][RTW89_UK][2] = 48, + [0][1][0][0][RTW89_FCC][3] = 76, [0][1][0][0][RTW89_ETSI][3] = 48, [0][1][0][0][RTW89_MKK][3] = 58, - [0][1][0][0][RTW89_IC][3] = 74, + [0][1][0][0][RTW89_IC][3] = 76, + [0][1][0][0][RTW89_KCC][3] = 56, [0][1][0][0][RTW89_ACMA][3] = 48, - [0][1][0][0][RTW89_FCC][4] = 66, + [0][1][0][0][RTW89_CN][3] = 42, + [0][1][0][0][RTW89_UK][3] = 48, + [0][1][0][0][RTW89_FCC][4] = 76, [0][1][0][0][RTW89_ETSI][4] = 48, [0][1][0][0][RTW89_MKK][4] = 58, - [0][1][0][0][RTW89_IC][4] = 74, + [0][1][0][0][RTW89_IC][4] = 76, + [0][1][0][0][RTW89_KCC][4] = 56, [0][1][0][0][RTW89_ACMA][4] = 48, - [0][1][0][0][RTW89_FCC][5] = 66, + [0][1][0][0][RTW89_CN][4] = 42, + [0][1][0][0][RTW89_UK][4] = 48, + [0][1][0][0][RTW89_FCC][5] = 76, [0][1][0][0][RTW89_ETSI][5] = 48, [0][1][0][0][RTW89_MKK][5] = 58, - [0][1][0][0][RTW89_IC][5] = 74, + [0][1][0][0][RTW89_IC][5] = 76, + [0][1][0][0][RTW89_KCC][5] = 56, [0][1][0][0][RTW89_ACMA][5] = 48, - [0][1][0][0][RTW89_FCC][6] = 66, + [0][1][0][0][RTW89_CN][5] = 42, + [0][1][0][0][RTW89_UK][5] = 48, + [0][1][0][0][RTW89_FCC][6] = 76, [0][1][0][0][RTW89_ETSI][6] = 48, [0][1][0][0][RTW89_MKK][6] = 58, - [0][1][0][0][RTW89_IC][6] = 74, + [0][1][0][0][RTW89_IC][6] = 76, + [0][1][0][0][RTW89_KCC][6] = 56, [0][1][0][0][RTW89_ACMA][6] = 48, - [0][1][0][0][RTW89_FCC][7] = 66, + [0][1][0][0][RTW89_CN][6] = 42, + [0][1][0][0][RTW89_UK][6] = 48, + [0][1][0][0][RTW89_FCC][7] = 76, [0][1][0][0][RTW89_ETSI][7] = 48, [0][1][0][0][RTW89_MKK][7] = 58, - [0][1][0][0][RTW89_IC][7] = 74, + [0][1][0][0][RTW89_IC][7] = 76, + [0][1][0][0][RTW89_KCC][7] = 56, [0][1][0][0][RTW89_ACMA][7] = 48, - [0][1][0][0][RTW89_FCC][8] = 66, + [0][1][0][0][RTW89_CN][7] = 42, + [0][1][0][0][RTW89_UK][7] = 48, + [0][1][0][0][RTW89_FCC][8] = 76, [0][1][0][0][RTW89_ETSI][8] = 48, [0][1][0][0][RTW89_MKK][8] = 58, - [0][1][0][0][RTW89_IC][8] = 74, + [0][1][0][0][RTW89_IC][8] = 76, + [0][1][0][0][RTW89_KCC][8] = 56, [0][1][0][0][RTW89_ACMA][8] = 48, - [0][1][0][0][RTW89_FCC][9] = 66, + [0][1][0][0][RTW89_CN][8] = 42, + [0][1][0][0][RTW89_UK][8] = 48, + [0][1][0][0][RTW89_FCC][9] = 70, [0][1][0][0][RTW89_ETSI][9] = 48, [0][1][0][0][RTW89_MKK][9] = 58, - [0][1][0][0][RTW89_IC][9] = 74, + [0][1][0][0][RTW89_IC][9] = 70, + [0][1][0][0][RTW89_KCC][9] = 56, [0][1][0][0][RTW89_ACMA][9] = 48, - [0][1][0][0][RTW89_FCC][10] = 66, + [0][1][0][0][RTW89_CN][9] = 42, + [0][1][0][0][RTW89_UK][9] = 48, + [0][1][0][0][RTW89_FCC][10] = 72, [0][1][0][0][RTW89_ETSI][10] = 48, [0][1][0][0][RTW89_MKK][10] = 58, - [0][1][0][0][RTW89_IC][10] = 74, + [0][1][0][0][RTW89_IC][10] = 72, + [0][1][0][0][RTW89_KCC][10] = 56, [0][1][0][0][RTW89_ACMA][10] = 48, - [0][1][0][0][RTW89_FCC][11] = 46, + [0][1][0][0][RTW89_CN][10] = 42, + [0][1][0][0][RTW89_UK][10] = 48, + [0][1][0][0][RTW89_FCC][11] = 44, [0][1][0][0][RTW89_ETSI][11] = 48, [0][1][0][0][RTW89_MKK][11] = 58, - [0][1][0][0][RTW89_IC][11] = 56, + [0][1][0][0][RTW89_IC][11] = 44, + [0][1][0][0][RTW89_KCC][11] = 56, [0][1][0][0][RTW89_ACMA][11] = 48, - [0][1][0][0][RTW89_FCC][12] = 34, + [0][1][0][0][RTW89_CN][11] = 42, + [0][1][0][0][RTW89_UK][11] = 48, + [0][1][0][0][RTW89_FCC][12] = 18, [0][1][0][0][RTW89_ETSI][12] = 48, [0][1][0][0][RTW89_MKK][12] = 58, - [0][1][0][0][RTW89_IC][12] = 44, + [0][1][0][0][RTW89_IC][12] = 18, + [0][1][0][0][RTW89_KCC][12] = 56, [0][1][0][0][RTW89_ACMA][12] = 48, + [0][1][0][0][RTW89_CN][12] = 42, + [0][1][0][0][RTW89_UK][12] = 48, [0][1][0][0][RTW89_FCC][13] = 127, [0][1][0][0][RTW89_ETSI][13] = 127, [0][1][0][0][RTW89_MKK][13] = 60, [0][1][0][0][RTW89_IC][13] = 127, + [0][1][0][0][RTW89_KCC][13] = 127, [0][1][0][0][RTW89_ACMA][13] = 127, + [0][1][0][0][RTW89_CN][13] = 127, + [0][1][0][0][RTW89_UK][13] = 127, [1][0][0][0][RTW89_FCC][0] = 127, [1][0][0][0][RTW89_ETSI][0] = 127, [1][0][0][0][RTW89_MKK][0] = 127, [1][0][0][0][RTW89_IC][0] = 127, + [1][0][0][0][RTW89_KCC][0] = 127, [1][0][0][0][RTW89_ACMA][0] = 127, + [1][0][0][0][RTW89_CN][0] = 127, + [1][0][0][0][RTW89_UK][0] = 127, [1][0][0][0][RTW89_FCC][1] = 127, [1][0][0][0][RTW89_ETSI][1] = 127, [1][0][0][0][RTW89_MKK][1] = 127, [1][0][0][0][RTW89_IC][1] = 127, + [1][0][0][0][RTW89_KCC][1] = 127, [1][0][0][0][RTW89_ACMA][1] = 127, - [1][0][0][0][RTW89_FCC][2] = 42, + [1][0][0][0][RTW89_CN][1] = 127, + [1][0][0][0][RTW89_UK][1] = 127, + [1][0][0][0][RTW89_FCC][2] = 44, [1][0][0][0][RTW89_ETSI][2] = 60, [1][0][0][0][RTW89_MKK][2] = 66, - [1][0][0][0][RTW89_IC][2] = 52, + [1][0][0][0][RTW89_IC][2] = 44, + [1][0][0][0][RTW89_KCC][2] = 68, [1][0][0][0][RTW89_ACMA][2] = 60, - [1][0][0][0][RTW89_FCC][3] = 42, + [1][0][0][0][RTW89_CN][2] = 58, + [1][0][0][0][RTW89_UK][2] = 60, + [1][0][0][0][RTW89_FCC][3] = 60, [1][0][0][0][RTW89_ETSI][3] = 60, [1][0][0][0][RTW89_MKK][3] = 66, - [1][0][0][0][RTW89_IC][3] = 52, + [1][0][0][0][RTW89_IC][3] = 60, + [1][0][0][0][RTW89_KCC][3] = 68, [1][0][0][0][RTW89_ACMA][3] = 60, - [1][0][0][0][RTW89_FCC][4] = 42, + [1][0][0][0][RTW89_CN][3] = 58, + [1][0][0][0][RTW89_UK][3] = 60, + [1][0][0][0][RTW89_FCC][4] = 60, [1][0][0][0][RTW89_ETSI][4] = 60, [1][0][0][0][RTW89_MKK][4] = 66, - [1][0][0][0][RTW89_IC][4] = 52, + [1][0][0][0][RTW89_IC][4] = 60, + [1][0][0][0][RTW89_KCC][4] = 68, [1][0][0][0][RTW89_ACMA][4] = 60, - [1][0][0][0][RTW89_FCC][5] = 58, + [1][0][0][0][RTW89_CN][4] = 58, + [1][0][0][0][RTW89_UK][4] = 60, + [1][0][0][0][RTW89_FCC][5] = 62, [1][0][0][0][RTW89_ETSI][5] = 60, [1][0][0][0][RTW89_MKK][5] = 66, - [1][0][0][0][RTW89_IC][5] = 68, + [1][0][0][0][RTW89_IC][5] = 62, + [1][0][0][0][RTW89_KCC][5] = 68, [1][0][0][0][RTW89_ACMA][5] = 60, - [1][0][0][0][RTW89_FCC][6] = 42, + [1][0][0][0][RTW89_CN][5] = 58, + [1][0][0][0][RTW89_UK][5] = 60, + [1][0][0][0][RTW89_FCC][6] = 46, [1][0][0][0][RTW89_ETSI][6] = 60, [1][0][0][0][RTW89_MKK][6] = 66, - [1][0][0][0][RTW89_IC][6] = 52, + [1][0][0][0][RTW89_IC][6] = 46, + [1][0][0][0][RTW89_KCC][6] = 68, [1][0][0][0][RTW89_ACMA][6] = 60, - [1][0][0][0][RTW89_FCC][7] = 42, + [1][0][0][0][RTW89_CN][6] = 58, + [1][0][0][0][RTW89_UK][6] = 60, + [1][0][0][0][RTW89_FCC][7] = 46, [1][0][0][0][RTW89_ETSI][7] = 60, [1][0][0][0][RTW89_MKK][7] = 66, - [1][0][0][0][RTW89_IC][7] = 52, + [1][0][0][0][RTW89_IC][7] = 46, + [1][0][0][0][RTW89_KCC][7] = 68, [1][0][0][0][RTW89_ACMA][7] = 60, - [1][0][0][0][RTW89_FCC][8] = 42, + [1][0][0][0][RTW89_CN][7] = 58, + [1][0][0][0][RTW89_UK][7] = 60, + [1][0][0][0][RTW89_FCC][8] = 28, [1][0][0][0][RTW89_ETSI][8] = 60, [1][0][0][0][RTW89_MKK][8] = 66, - [1][0][0][0][RTW89_IC][8] = 52, + [1][0][0][0][RTW89_IC][8] = 28, + [1][0][0][0][RTW89_KCC][8] = 70, [1][0][0][0][RTW89_ACMA][8] = 60, - [1][0][0][0][RTW89_FCC][9] = 34, + [1][0][0][0][RTW89_CN][8] = 58, + [1][0][0][0][RTW89_UK][8] = 60, + [1][0][0][0][RTW89_FCC][9] = 26, [1][0][0][0][RTW89_ETSI][9] = 60, [1][0][0][0][RTW89_MKK][9] = 66, - [1][0][0][0][RTW89_IC][9] = 44, + [1][0][0][0][RTW89_IC][9] = 26, + [1][0][0][0][RTW89_KCC][9] = 70, [1][0][0][0][RTW89_ACMA][9] = 60, - [1][0][0][0][RTW89_FCC][10] = 22, + [1][0][0][0][RTW89_CN][9] = 58, + [1][0][0][0][RTW89_UK][9] = 60, + [1][0][0][0][RTW89_FCC][10] = 26, [1][0][0][0][RTW89_ETSI][10] = 60, [1][0][0][0][RTW89_MKK][10] = 66, - [1][0][0][0][RTW89_IC][10] = 32, + [1][0][0][0][RTW89_IC][10] = 26, + [1][0][0][0][RTW89_KCC][10] = 70, [1][0][0][0][RTW89_ACMA][10] = 60, + [1][0][0][0][RTW89_CN][10] = 58, + [1][0][0][0][RTW89_UK][10] = 60, [1][0][0][0][RTW89_FCC][11] = 127, [1][0][0][0][RTW89_ETSI][11] = 127, [1][0][0][0][RTW89_MKK][11] = 127, [1][0][0][0][RTW89_IC][11] = 127, + [1][0][0][0][RTW89_KCC][11] = 127, [1][0][0][0][RTW89_ACMA][11] = 127, + [1][0][0][0][RTW89_CN][11] = 127, + [1][0][0][0][RTW89_UK][11] = 127, [1][0][0][0][RTW89_FCC][12] = 127, [1][0][0][0][RTW89_ETSI][12] = 127, [1][0][0][0][RTW89_MKK][12] = 127, [1][0][0][0][RTW89_IC][12] = 127, + [1][0][0][0][RTW89_KCC][12] = 127, [1][0][0][0][RTW89_ACMA][12] = 127, + [1][0][0][0][RTW89_CN][12] = 127, + [1][0][0][0][RTW89_UK][12] = 127, [1][0][0][0][RTW89_FCC][13] = 127, [1][0][0][0][RTW89_ETSI][13] = 127, [1][0][0][0][RTW89_MKK][13] = 127, [1][0][0][0][RTW89_IC][13] = 127, + [1][0][0][0][RTW89_KCC][13] = 127, [1][0][0][0][RTW89_ACMA][13] = 127, + [1][0][0][0][RTW89_CN][13] = 127, + [1][0][0][0][RTW89_UK][13] = 127, [1][1][0][0][RTW89_FCC][0] = 127, [1][1][0][0][RTW89_ETSI][0] = 127, [1][1][0][0][RTW89_MKK][0] = 127, [1][1][0][0][RTW89_IC][0] = 127, + [1][1][0][0][RTW89_KCC][0] = 127, [1][1][0][0][RTW89_ACMA][0] = 127, + [1][1][0][0][RTW89_CN][0] = 127, + [1][1][0][0][RTW89_UK][0] = 127, [1][1][0][0][RTW89_FCC][1] = 127, [1][1][0][0][RTW89_ETSI][1] = 127, [1][1][0][0][RTW89_MKK][1] = 127, [1][1][0][0][RTW89_IC][1] = 127, + [1][1][0][0][RTW89_KCC][1] = 127, [1][1][0][0][RTW89_ACMA][1] = 127, - [1][1][0][0][RTW89_FCC][2] = 38, + [1][1][0][0][RTW89_CN][1] = 127, + [1][1][0][0][RTW89_UK][1] = 127, + [1][1][0][0][RTW89_FCC][2] = 46, [1][1][0][0][RTW89_ETSI][2] = 48, [1][1][0][0][RTW89_MKK][2] = 58, - [1][1][0][0][RTW89_IC][2] = 48, + [1][1][0][0][RTW89_IC][2] = 46, + [1][1][0][0][RTW89_KCC][2] = 56, [1][1][0][0][RTW89_ACMA][2] = 48, - [1][1][0][0][RTW89_FCC][3] = 38, + [1][1][0][0][RTW89_CN][2] = 46, + [1][1][0][0][RTW89_UK][2] = 48, + [1][1][0][0][RTW89_FCC][3] = 46, [1][1][0][0][RTW89_ETSI][3] = 48, [1][1][0][0][RTW89_MKK][3] = 58, - [1][1][0][0][RTW89_IC][3] = 48, + [1][1][0][0][RTW89_IC][3] = 46, + [1][1][0][0][RTW89_KCC][3] = 56, [1][1][0][0][RTW89_ACMA][3] = 48, - [1][1][0][0][RTW89_FCC][4] = 38, + [1][1][0][0][RTW89_CN][3] = 46, + [1][1][0][0][RTW89_UK][3] = 48, + [1][1][0][0][RTW89_FCC][4] = 46, [1][1][0][0][RTW89_ETSI][4] = 48, [1][1][0][0][RTW89_MKK][4] = 58, - [1][1][0][0][RTW89_IC][4] = 48, + [1][1][0][0][RTW89_IC][4] = 46, + [1][1][0][0][RTW89_KCC][4] = 56, [1][1][0][0][RTW89_ACMA][4] = 48, - [1][1][0][0][RTW89_FCC][5] = 54, + [1][1][0][0][RTW89_CN][4] = 46, + [1][1][0][0][RTW89_UK][4] = 48, + [1][1][0][0][RTW89_FCC][5] = 48, [1][1][0][0][RTW89_ETSI][5] = 48, [1][1][0][0][RTW89_MKK][5] = 58, - [1][1][0][0][RTW89_IC][5] = 64, + [1][1][0][0][RTW89_IC][5] = 48, + [1][1][0][0][RTW89_KCC][5] = 56, [1][1][0][0][RTW89_ACMA][5] = 48, - [1][1][0][0][RTW89_FCC][6] = 26, + [1][1][0][0][RTW89_CN][5] = 46, + [1][1][0][0][RTW89_UK][5] = 48, + [1][1][0][0][RTW89_FCC][6] = 40, [1][1][0][0][RTW89_ETSI][6] = 48, [1][1][0][0][RTW89_MKK][6] = 58, - [1][1][0][0][RTW89_IC][6] = 36, + [1][1][0][0][RTW89_IC][6] = 40, + [1][1][0][0][RTW89_KCC][6] = 56, [1][1][0][0][RTW89_ACMA][6] = 48, - [1][1][0][0][RTW89_FCC][7] = 26, + [1][1][0][0][RTW89_CN][6] = 46, + [1][1][0][0][RTW89_UK][6] = 48, + [1][1][0][0][RTW89_FCC][7] = 40, [1][1][0][0][RTW89_ETSI][7] = 48, [1][1][0][0][RTW89_MKK][7] = 58, - [1][1][0][0][RTW89_IC][7] = 36, + [1][1][0][0][RTW89_IC][7] = 40, + [1][1][0][0][RTW89_KCC][7] = 56, [1][1][0][0][RTW89_ACMA][7] = 48, - [1][1][0][0][RTW89_FCC][8] = 26, + [1][1][0][0][RTW89_CN][7] = 46, + [1][1][0][0][RTW89_UK][7] = 48, + [1][1][0][0][RTW89_FCC][8] = 14, [1][1][0][0][RTW89_ETSI][8] = 48, [1][1][0][0][RTW89_MKK][8] = 58, - [1][1][0][0][RTW89_IC][8] = 36, + [1][1][0][0][RTW89_IC][8] = 14, + [1][1][0][0][RTW89_KCC][8] = 58, [1][1][0][0][RTW89_ACMA][8] = 48, - [1][1][0][0][RTW89_FCC][9] = 22, + [1][1][0][0][RTW89_CN][8] = 46, + [1][1][0][0][RTW89_UK][8] = 48, + [1][1][0][0][RTW89_FCC][9] = 14, [1][1][0][0][RTW89_ETSI][9] = 48, [1][1][0][0][RTW89_MKK][9] = 58, - [1][1][0][0][RTW89_IC][9] = 32, + [1][1][0][0][RTW89_IC][9] = 14, + [1][1][0][0][RTW89_KCC][9] = 58, [1][1][0][0][RTW89_ACMA][9] = 48, - [1][1][0][0][RTW89_FCC][10] = 22, + [1][1][0][0][RTW89_CN][9] = 46, + [1][1][0][0][RTW89_UK][9] = 48, + [1][1][0][0][RTW89_FCC][10] = 12, [1][1][0][0][RTW89_ETSI][10] = 48, [1][1][0][0][RTW89_MKK][10] = 56, - [1][1][0][0][RTW89_IC][10] = 32, + [1][1][0][0][RTW89_IC][10] = 12, + [1][1][0][0][RTW89_KCC][10] = 58, [1][1][0][0][RTW89_ACMA][10] = 48, + [1][1][0][0][RTW89_CN][10] = 46, + [1][1][0][0][RTW89_UK][10] = 48, [1][1][0][0][RTW89_FCC][11] = 127, [1][1][0][0][RTW89_ETSI][11] = 127, [1][1][0][0][RTW89_MKK][11] = 127, [1][1][0][0][RTW89_IC][11] = 127, + [1][1][0][0][RTW89_KCC][11] = 127, [1][1][0][0][RTW89_ACMA][11] = 127, + [1][1][0][0][RTW89_CN][11] = 127, + [1][1][0][0][RTW89_UK][11] = 127, [1][1][0][0][RTW89_FCC][12] = 127, [1][1][0][0][RTW89_ETSI][12] = 127, [1][1][0][0][RTW89_MKK][12] = 127, [1][1][0][0][RTW89_IC][12] = 127, + [1][1][0][0][RTW89_KCC][12] = 127, [1][1][0][0][RTW89_ACMA][12] = 127, + [1][1][0][0][RTW89_CN][12] = 127, + [1][1][0][0][RTW89_UK][12] = 127, [1][1][0][0][RTW89_FCC][13] = 127, [1][1][0][0][RTW89_ETSI][13] = 127, [1][1][0][0][RTW89_MKK][13] = 127, [1][1][0][0][RTW89_IC][13] = 127, + [1][1][0][0][RTW89_KCC][13] = 127, [1][1][0][0][RTW89_ACMA][13] = 127, - [0][0][1][0][RTW89_FCC][0] = 68, + [1][1][0][0][RTW89_CN][13] = 127, + [1][1][0][0][RTW89_UK][13] = 127, + [0][0][1][0][RTW89_FCC][0] = 66, [0][0][1][0][RTW89_ETSI][0] = 60, [0][0][1][0][RTW89_MKK][0] = 76, - [0][0][1][0][RTW89_IC][0] = 78, + [0][0][1][0][RTW89_IC][0] = 66, + [0][0][1][0][RTW89_KCC][0] = 68, [0][0][1][0][RTW89_ACMA][0] = 60, + [0][0][1][0][RTW89_CN][0] = 58, + [0][0][1][0][RTW89_UK][0] = 60, [0][0][1][0][RTW89_FCC][1] = 68, [0][0][1][0][RTW89_ETSI][1] = 60, [0][0][1][0][RTW89_MKK][1] = 78, - [0][0][1][0][RTW89_IC][1] = 78, + [0][0][1][0][RTW89_IC][1] = 68, + [0][0][1][0][RTW89_KCC][1] = 68, [0][0][1][0][RTW89_ACMA][1] = 60, - [0][0][1][0][RTW89_FCC][2] = 70, + [0][0][1][0][RTW89_CN][1] = 58, + [0][0][1][0][RTW89_UK][1] = 60, + [0][0][1][0][RTW89_FCC][2] = 72, [0][0][1][0][RTW89_ETSI][2] = 60, [0][0][1][0][RTW89_MKK][2] = 78, - [0][0][1][0][RTW89_IC][2] = 78, + [0][0][1][0][RTW89_IC][2] = 72, + [0][0][1][0][RTW89_KCC][2] = 68, [0][0][1][0][RTW89_ACMA][2] = 60, - [0][0][1][0][RTW89_FCC][3] = 70, + [0][0][1][0][RTW89_CN][2] = 58, + [0][0][1][0][RTW89_UK][2] = 60, + [0][0][1][0][RTW89_FCC][3] = 76, [0][0][1][0][RTW89_ETSI][3] = 60, [0][0][1][0][RTW89_MKK][3] = 78, - [0][0][1][0][RTW89_IC][3] = 78, + [0][0][1][0][RTW89_IC][3] = 76, + [0][0][1][0][RTW89_KCC][3] = 68, [0][0][1][0][RTW89_ACMA][3] = 60, - [0][0][1][0][RTW89_FCC][4] = 70, + [0][0][1][0][RTW89_CN][3] = 58, + [0][0][1][0][RTW89_UK][3] = 60, + [0][0][1][0][RTW89_FCC][4] = 80, [0][0][1][0][RTW89_ETSI][4] = 60, [0][0][1][0][RTW89_MKK][4] = 78, - [0][0][1][0][RTW89_IC][4] = 78, + [0][0][1][0][RTW89_IC][4] = 80, + [0][0][1][0][RTW89_KCC][4] = 76, [0][0][1][0][RTW89_ACMA][4] = 60, - [0][0][1][0][RTW89_FCC][5] = 70, + [0][0][1][0][RTW89_CN][4] = 58, + [0][0][1][0][RTW89_UK][4] = 60, + [0][0][1][0][RTW89_FCC][5] = 80, [0][0][1][0][RTW89_ETSI][5] = 60, [0][0][1][0][RTW89_MKK][5] = 78, - [0][0][1][0][RTW89_IC][5] = 78, + [0][0][1][0][RTW89_IC][5] = 80, + [0][0][1][0][RTW89_KCC][5] = 76, [0][0][1][0][RTW89_ACMA][5] = 60, - [0][0][1][0][RTW89_FCC][6] = 70, + [0][0][1][0][RTW89_CN][5] = 58, + [0][0][1][0][RTW89_UK][5] = 60, + [0][0][1][0][RTW89_FCC][6] = 80, [0][0][1][0][RTW89_ETSI][6] = 60, [0][0][1][0][RTW89_MKK][6] = 76, - [0][0][1][0][RTW89_IC][6] = 78, + [0][0][1][0][RTW89_IC][6] = 80, + [0][0][1][0][RTW89_KCC][6] = 76, [0][0][1][0][RTW89_ACMA][6] = 60, - [0][0][1][0][RTW89_FCC][7] = 70, + [0][0][1][0][RTW89_CN][6] = 58, + [0][0][1][0][RTW89_UK][6] = 60, + [0][0][1][0][RTW89_FCC][7] = 80, [0][0][1][0][RTW89_ETSI][7] = 60, [0][0][1][0][RTW89_MKK][7] = 78, - [0][0][1][0][RTW89_IC][7] = 78, + [0][0][1][0][RTW89_IC][7] = 80, + [0][0][1][0][RTW89_KCC][7] = 76, [0][0][1][0][RTW89_ACMA][7] = 60, - [0][0][1][0][RTW89_FCC][8] = 70, + [0][0][1][0][RTW89_CN][7] = 58, + [0][0][1][0][RTW89_UK][7] = 60, + [0][0][1][0][RTW89_FCC][8] = 80, [0][0][1][0][RTW89_ETSI][8] = 60, [0][0][1][0][RTW89_MKK][8] = 78, - [0][0][1][0][RTW89_IC][8] = 78, + [0][0][1][0][RTW89_IC][8] = 80, + [0][0][1][0][RTW89_KCC][8] = 76, [0][0][1][0][RTW89_ACMA][8] = 60, - [0][0][1][0][RTW89_FCC][9] = 66, + [0][0][1][0][RTW89_CN][8] = 58, + [0][0][1][0][RTW89_UK][8] = 60, + [0][0][1][0][RTW89_FCC][9] = 76, [0][0][1][0][RTW89_ETSI][9] = 60, [0][0][1][0][RTW89_MKK][9] = 78, [0][0][1][0][RTW89_IC][9] = 76, + [0][0][1][0][RTW89_KCC][9] = 70, [0][0][1][0][RTW89_ACMA][9] = 60, + [0][0][1][0][RTW89_CN][9] = 58, + [0][0][1][0][RTW89_UK][9] = 60, [0][0][1][0][RTW89_FCC][10] = 66, [0][0][1][0][RTW89_ETSI][10] = 60, [0][0][1][0][RTW89_MKK][10] = 78, - [0][0][1][0][RTW89_IC][10] = 76, + [0][0][1][0][RTW89_IC][10] = 66, + [0][0][1][0][RTW89_KCC][10] = 70, [0][0][1][0][RTW89_ACMA][10] = 60, - [0][0][1][0][RTW89_FCC][11] = 46, + [0][0][1][0][RTW89_CN][10] = 58, + [0][0][1][0][RTW89_UK][10] = 60, + [0][0][1][0][RTW89_FCC][11] = 62, [0][0][1][0][RTW89_ETSI][11] = 60, [0][0][1][0][RTW89_MKK][11] = 78, - [0][0][1][0][RTW89_IC][11] = 56, + [0][0][1][0][RTW89_IC][11] = 62, + [0][0][1][0][RTW89_KCC][11] = 70, [0][0][1][0][RTW89_ACMA][11] = 60, - [0][0][1][0][RTW89_FCC][12] = 42, + [0][0][1][0][RTW89_CN][11] = 58, + [0][0][1][0][RTW89_UK][11] = 60, + [0][0][1][0][RTW89_FCC][12] = 60, [0][0][1][0][RTW89_ETSI][12] = 60, [0][0][1][0][RTW89_MKK][12] = 78, - [0][0][1][0][RTW89_IC][12] = 52, + [0][0][1][0][RTW89_IC][12] = 60, + [0][0][1][0][RTW89_KCC][12] = 70, [0][0][1][0][RTW89_ACMA][12] = 60, + [0][0][1][0][RTW89_CN][12] = 58, + [0][0][1][0][RTW89_UK][12] = 60, [0][0][1][0][RTW89_FCC][13] = 127, [0][0][1][0][RTW89_ETSI][13] = 127, [0][0][1][0][RTW89_MKK][13] = 127, [0][0][1][0][RTW89_IC][13] = 127, + [0][0][1][0][RTW89_KCC][13] = 127, [0][0][1][0][RTW89_ACMA][13] = 127, - [0][1][1][0][RTW89_FCC][0] = 54, + [0][0][1][0][RTW89_CN][13] = 127, + [0][0][1][0][RTW89_UK][13] = 127, + [0][1][1][0][RTW89_FCC][0] = 66, [0][1][1][0][RTW89_ETSI][0] = 48, [0][1][1][0][RTW89_MKK][0] = 66, - [0][1][1][0][RTW89_IC][0] = 64, + [0][1][1][0][RTW89_IC][0] = 66, + [0][1][1][0][RTW89_KCC][0] = 64, [0][1][1][0][RTW89_ACMA][0] = 48, - [0][1][1][0][RTW89_FCC][1] = 54, + [0][1][1][0][RTW89_CN][0] = 46, + [0][1][1][0][RTW89_UK][0] = 48, + [0][1][1][0][RTW89_FCC][1] = 68, [0][1][1][0][RTW89_ETSI][1] = 48, [0][1][1][0][RTW89_MKK][1] = 66, - [0][1][1][0][RTW89_IC][1] = 64, + [0][1][1][0][RTW89_IC][1] = 68, + [0][1][1][0][RTW89_KCC][1] = 64, [0][1][1][0][RTW89_ACMA][1] = 48, - [0][1][1][0][RTW89_FCC][2] = 58, + [0][1][1][0][RTW89_CN][1] = 46, + [0][1][1][0][RTW89_UK][1] = 48, + [0][1][1][0][RTW89_FCC][2] = 72, [0][1][1][0][RTW89_ETSI][2] = 48, [0][1][1][0][RTW89_MKK][2] = 66, - [0][1][1][0][RTW89_IC][2] = 68, + [0][1][1][0][RTW89_IC][2] = 72, + [0][1][1][0][RTW89_KCC][2] = 64, [0][1][1][0][RTW89_ACMA][2] = 48, - [0][1][1][0][RTW89_FCC][3] = 62, + [0][1][1][0][RTW89_CN][2] = 46, + [0][1][1][0][RTW89_UK][2] = 48, + [0][1][1][0][RTW89_FCC][3] = 76, [0][1][1][0][RTW89_ETSI][3] = 48, [0][1][1][0][RTW89_MKK][3] = 66, - [0][1][1][0][RTW89_IC][3] = 72, + [0][1][1][0][RTW89_IC][3] = 76, + [0][1][1][0][RTW89_KCC][3] = 64, [0][1][1][0][RTW89_ACMA][3] = 48, - [0][1][1][0][RTW89_FCC][4] = 70, + [0][1][1][0][RTW89_CN][3] = 46, + [0][1][1][0][RTW89_UK][3] = 48, + [0][1][1][0][RTW89_FCC][4] = 80, [0][1][1][0][RTW89_ETSI][4] = 48, [0][1][1][0][RTW89_MKK][4] = 66, - [0][1][1][0][RTW89_IC][4] = 78, + [0][1][1][0][RTW89_IC][4] = 80, + [0][1][1][0][RTW89_KCC][4] = 66, [0][1][1][0][RTW89_ACMA][4] = 48, - [0][1][1][0][RTW89_FCC][5] = 70, + [0][1][1][0][RTW89_CN][4] = 46, + [0][1][1][0][RTW89_UK][4] = 48, + [0][1][1][0][RTW89_FCC][5] = 80, [0][1][1][0][RTW89_ETSI][5] = 48, [0][1][1][0][RTW89_MKK][5] = 66, - [0][1][1][0][RTW89_IC][5] = 78, + [0][1][1][0][RTW89_IC][5] = 80, + [0][1][1][0][RTW89_KCC][5] = 66, [0][1][1][0][RTW89_ACMA][5] = 48, - [0][1][1][0][RTW89_FCC][6] = 70, + [0][1][1][0][RTW89_CN][5] = 46, + [0][1][1][0][RTW89_UK][5] = 48, + [0][1][1][0][RTW89_FCC][6] = 80, [0][1][1][0][RTW89_ETSI][6] = 48, [0][1][1][0][RTW89_MKK][6] = 66, - [0][1][1][0][RTW89_IC][6] = 78, + [0][1][1][0][RTW89_IC][6] = 80, + [0][1][1][0][RTW89_KCC][6] = 66, [0][1][1][0][RTW89_ACMA][6] = 48, - [0][1][1][0][RTW89_FCC][7] = 62, + [0][1][1][0][RTW89_CN][6] = 46, + [0][1][1][0][RTW89_UK][6] = 48, + [0][1][1][0][RTW89_FCC][7] = 78, [0][1][1][0][RTW89_ETSI][7] = 48, [0][1][1][0][RTW89_MKK][7] = 66, - [0][1][1][0][RTW89_IC][7] = 72, + [0][1][1][0][RTW89_IC][7] = 78, + [0][1][1][0][RTW89_KCC][7] = 66, [0][1][1][0][RTW89_ACMA][7] = 48, - [0][1][1][0][RTW89_FCC][8] = 58, + [0][1][1][0][RTW89_CN][7] = 46, + [0][1][1][0][RTW89_UK][7] = 48, + [0][1][1][0][RTW89_FCC][8] = 74, [0][1][1][0][RTW89_ETSI][8] = 48, [0][1][1][0][RTW89_MKK][8] = 66, - [0][1][1][0][RTW89_IC][8] = 68, + [0][1][1][0][RTW89_IC][8] = 74, + [0][1][1][0][RTW89_KCC][8] = 66, [0][1][1][0][RTW89_ACMA][8] = 48, - [0][1][1][0][RTW89_FCC][9] = 54, + [0][1][1][0][RTW89_CN][8] = 46, + [0][1][1][0][RTW89_UK][8] = 48, + [0][1][1][0][RTW89_FCC][9] = 70, [0][1][1][0][RTW89_ETSI][9] = 48, [0][1][1][0][RTW89_MKK][9] = 66, - [0][1][1][0][RTW89_IC][9] = 64, + [0][1][1][0][RTW89_IC][9] = 70, + [0][1][1][0][RTW89_KCC][9] = 64, [0][1][1][0][RTW89_ACMA][9] = 48, - [0][1][1][0][RTW89_FCC][10] = 54, + [0][1][1][0][RTW89_CN][9] = 46, + [0][1][1][0][RTW89_UK][9] = 48, + [0][1][1][0][RTW89_FCC][10] = 62, [0][1][1][0][RTW89_ETSI][10] = 48, [0][1][1][0][RTW89_MKK][10] = 66, - [0][1][1][0][RTW89_IC][10] = 64, + [0][1][1][0][RTW89_IC][10] = 62, + [0][1][1][0][RTW89_KCC][10] = 64, [0][1][1][0][RTW89_ACMA][10] = 48, - [0][1][1][0][RTW89_FCC][11] = 38, + [0][1][1][0][RTW89_CN][10] = 46, + [0][1][1][0][RTW89_UK][10] = 48, + [0][1][1][0][RTW89_FCC][11] = 60, [0][1][1][0][RTW89_ETSI][11] = 48, [0][1][1][0][RTW89_MKK][11] = 66, - [0][1][1][0][RTW89_IC][11] = 48, + [0][1][1][0][RTW89_IC][11] = 60, + [0][1][1][0][RTW89_KCC][11] = 64, [0][1][1][0][RTW89_ACMA][11] = 48, - [0][1][1][0][RTW89_FCC][12] = 34, + [0][1][1][0][RTW89_CN][11] = 46, + [0][1][1][0][RTW89_UK][11] = 48, + [0][1][1][0][RTW89_FCC][12] = 36, [0][1][1][0][RTW89_ETSI][12] = 48, [0][1][1][0][RTW89_MKK][12] = 66, - [0][1][1][0][RTW89_IC][12] = 44, + [0][1][1][0][RTW89_IC][12] = 36, + [0][1][1][0][RTW89_KCC][12] = 64, [0][1][1][0][RTW89_ACMA][12] = 48, + [0][1][1][0][RTW89_CN][12] = 46, + [0][1][1][0][RTW89_UK][12] = 48, [0][1][1][0][RTW89_FCC][13] = 127, [0][1][1][0][RTW89_ETSI][13] = 127, [0][1][1][0][RTW89_MKK][13] = 127, [0][1][1][0][RTW89_IC][13] = 127, + [0][1][1][0][RTW89_KCC][13] = 127, [0][1][1][0][RTW89_ACMA][13] = 127, - [0][0][2][0][RTW89_FCC][0] = 68, + [0][1][1][0][RTW89_CN][13] = 127, + [0][1][1][0][RTW89_UK][13] = 127, + [0][0][2][0][RTW89_FCC][0] = 66, [0][0][2][0][RTW89_ETSI][0] = 60, [0][0][2][0][RTW89_MKK][0] = 78, - [0][0][2][0][RTW89_IC][0] = 78, + [0][0][2][0][RTW89_IC][0] = 66, + [0][0][2][0][RTW89_KCC][0] = 70, [0][0][2][0][RTW89_ACMA][0] = 60, - [0][0][2][0][RTW89_FCC][1] = 68, + [0][0][2][0][RTW89_CN][0] = 58, + [0][0][2][0][RTW89_UK][0] = 60, + [0][0][2][0][RTW89_FCC][1] = 70, [0][0][2][0][RTW89_ETSI][1] = 60, [0][0][2][0][RTW89_MKK][1] = 78, - [0][0][2][0][RTW89_IC][1] = 78, + [0][0][2][0][RTW89_IC][1] = 70, + [0][0][2][0][RTW89_KCC][1] = 70, [0][0][2][0][RTW89_ACMA][1] = 60, - [0][0][2][0][RTW89_FCC][2] = 70, + [0][0][2][0][RTW89_CN][1] = 58, + [0][0][2][0][RTW89_UK][1] = 60, + [0][0][2][0][RTW89_FCC][2] = 74, [0][0][2][0][RTW89_ETSI][2] = 60, [0][0][2][0][RTW89_MKK][2] = 78, - [0][0][2][0][RTW89_IC][2] = 78, + [0][0][2][0][RTW89_IC][2] = 74, + [0][0][2][0][RTW89_KCC][2] = 70, [0][0][2][0][RTW89_ACMA][2] = 60, - [0][0][2][0][RTW89_FCC][3] = 70, + [0][0][2][0][RTW89_CN][2] = 58, + [0][0][2][0][RTW89_UK][2] = 60, + [0][0][2][0][RTW89_FCC][3] = 78, [0][0][2][0][RTW89_ETSI][3] = 60, [0][0][2][0][RTW89_MKK][3] = 78, [0][0][2][0][RTW89_IC][3] = 78, + [0][0][2][0][RTW89_KCC][3] = 70, [0][0][2][0][RTW89_ACMA][3] = 60, - [0][0][2][0][RTW89_FCC][4] = 70, + [0][0][2][0][RTW89_CN][3] = 58, + [0][0][2][0][RTW89_UK][3] = 60, + [0][0][2][0][RTW89_FCC][4] = 80, [0][0][2][0][RTW89_ETSI][4] = 60, [0][0][2][0][RTW89_MKK][4] = 78, - [0][0][2][0][RTW89_IC][4] = 78, + [0][0][2][0][RTW89_IC][4] = 80, + [0][0][2][0][RTW89_KCC][4] = 78, [0][0][2][0][RTW89_ACMA][4] = 60, - [0][0][2][0][RTW89_FCC][5] = 70, + [0][0][2][0][RTW89_CN][4] = 58, + [0][0][2][0][RTW89_UK][4] = 60, + [0][0][2][0][RTW89_FCC][5] = 80, [0][0][2][0][RTW89_ETSI][5] = 60, [0][0][2][0][RTW89_MKK][5] = 78, - [0][0][2][0][RTW89_IC][5] = 78, + [0][0][2][0][RTW89_IC][5] = 80, + [0][0][2][0][RTW89_KCC][5] = 78, [0][0][2][0][RTW89_ACMA][5] = 60, - [0][0][2][0][RTW89_FCC][6] = 70, + [0][0][2][0][RTW89_CN][5] = 58, + [0][0][2][0][RTW89_UK][5] = 60, + [0][0][2][0][RTW89_FCC][6] = 80, [0][0][2][0][RTW89_ETSI][6] = 60, [0][0][2][0][RTW89_MKK][6] = 78, - [0][0][2][0][RTW89_IC][6] = 78, + [0][0][2][0][RTW89_IC][6] = 80, + [0][0][2][0][RTW89_KCC][6] = 78, [0][0][2][0][RTW89_ACMA][6] = 60, - [0][0][2][0][RTW89_FCC][7] = 70, + [0][0][2][0][RTW89_CN][6] = 58, + [0][0][2][0][RTW89_UK][6] = 60, + [0][0][2][0][RTW89_FCC][7] = 80, [0][0][2][0][RTW89_ETSI][7] = 60, [0][0][2][0][RTW89_MKK][7] = 78, - [0][0][2][0][RTW89_IC][7] = 78, + [0][0][2][0][RTW89_IC][7] = 80, + [0][0][2][0][RTW89_KCC][7] = 78, [0][0][2][0][RTW89_ACMA][7] = 60, - [0][0][2][0][RTW89_FCC][8] = 68, + [0][0][2][0][RTW89_CN][7] = 58, + [0][0][2][0][RTW89_UK][7] = 60, + [0][0][2][0][RTW89_FCC][8] = 78, [0][0][2][0][RTW89_ETSI][8] = 60, [0][0][2][0][RTW89_MKK][8] = 78, [0][0][2][0][RTW89_IC][8] = 78, + [0][0][2][0][RTW89_KCC][8] = 78, [0][0][2][0][RTW89_ACMA][8] = 60, - [0][0][2][0][RTW89_FCC][9] = 64, + [0][0][2][0][RTW89_CN][8] = 58, + [0][0][2][0][RTW89_UK][8] = 60, + [0][0][2][0][RTW89_FCC][9] = 74, [0][0][2][0][RTW89_ETSI][9] = 60, [0][0][2][0][RTW89_MKK][9] = 78, [0][0][2][0][RTW89_IC][9] = 74, + [0][0][2][0][RTW89_KCC][9] = 66, [0][0][2][0][RTW89_ACMA][9] = 60, - [0][0][2][0][RTW89_FCC][10] = 64, + [0][0][2][0][RTW89_CN][9] = 58, + [0][0][2][0][RTW89_UK][9] = 60, + [0][0][2][0][RTW89_FCC][10] = 62, [0][0][2][0][RTW89_ETSI][10] = 60, [0][0][2][0][RTW89_MKK][10] = 78, - [0][0][2][0][RTW89_IC][10] = 74, + [0][0][2][0][RTW89_IC][10] = 62, + [0][0][2][0][RTW89_KCC][10] = 66, [0][0][2][0][RTW89_ACMA][10] = 60, - [0][0][2][0][RTW89_FCC][11] = 46, + [0][0][2][0][RTW89_CN][10] = 58, + [0][0][2][0][RTW89_UK][10] = 60, + [0][0][2][0][RTW89_FCC][11] = 60, [0][0][2][0][RTW89_ETSI][11] = 60, [0][0][2][0][RTW89_MKK][11] = 78, - [0][0][2][0][RTW89_IC][11] = 56, + [0][0][2][0][RTW89_IC][11] = 60, + [0][0][2][0][RTW89_KCC][11] = 66, [0][0][2][0][RTW89_ACMA][11] = 60, - [0][0][2][0][RTW89_FCC][12] = 42, + [0][0][2][0][RTW89_CN][11] = 58, + [0][0][2][0][RTW89_UK][11] = 60, + [0][0][2][0][RTW89_FCC][12] = 38, [0][0][2][0][RTW89_ETSI][12] = 60, [0][0][2][0][RTW89_MKK][12] = 78, - [0][0][2][0][RTW89_IC][12] = 52, + [0][0][2][0][RTW89_IC][12] = 38, + [0][0][2][0][RTW89_KCC][12] = 66, [0][0][2][0][RTW89_ACMA][12] = 60, + [0][0][2][0][RTW89_CN][12] = 58, + [0][0][2][0][RTW89_UK][12] = 60, [0][0][2][0][RTW89_FCC][13] = 127, [0][0][2][0][RTW89_ETSI][13] = 127, [0][0][2][0][RTW89_MKK][13] = 127, [0][0][2][0][RTW89_IC][13] = 127, + [0][0][2][0][RTW89_KCC][13] = 127, [0][0][2][0][RTW89_ACMA][13] = 127, - [0][1][2][0][RTW89_FCC][0] = 50, + [0][0][2][0][RTW89_CN][13] = 127, + [0][0][2][0][RTW89_UK][13] = 127, + [0][1][2][0][RTW89_FCC][0] = 64, [0][1][2][0][RTW89_ETSI][0] = 48, [0][1][2][0][RTW89_MKK][0] = 68, - [0][1][2][0][RTW89_IC][0] = 60, + [0][1][2][0][RTW89_IC][0] = 64, + [0][1][2][0][RTW89_KCC][0] = 66, [0][1][2][0][RTW89_ACMA][0] = 48, - [0][1][2][0][RTW89_FCC][1] = 50, + [0][1][2][0][RTW89_CN][0] = 46, + [0][1][2][0][RTW89_UK][0] = 48, + [0][1][2][0][RTW89_FCC][1] = 70, [0][1][2][0][RTW89_ETSI][1] = 48, [0][1][2][0][RTW89_MKK][1] = 68, - [0][1][2][0][RTW89_IC][1] = 60, + [0][1][2][0][RTW89_IC][1] = 70, + [0][1][2][0][RTW89_KCC][1] = 66, [0][1][2][0][RTW89_ACMA][1] = 48, - [0][1][2][0][RTW89_FCC][2] = 54, + [0][1][2][0][RTW89_CN][1] = 46, + [0][1][2][0][RTW89_UK][1] = 48, + [0][1][2][0][RTW89_FCC][2] = 74, [0][1][2][0][RTW89_ETSI][2] = 48, [0][1][2][0][RTW89_MKK][2] = 68, - [0][1][2][0][RTW89_IC][2] = 64, + [0][1][2][0][RTW89_IC][2] = 74, + [0][1][2][0][RTW89_KCC][2] = 66, [0][1][2][0][RTW89_ACMA][2] = 48, - [0][1][2][0][RTW89_FCC][3] = 58, + [0][1][2][0][RTW89_CN][2] = 46, + [0][1][2][0][RTW89_UK][2] = 48, + [0][1][2][0][RTW89_FCC][3] = 78, [0][1][2][0][RTW89_ETSI][3] = 48, [0][1][2][0][RTW89_MKK][3] = 68, - [0][1][2][0][RTW89_IC][3] = 68, + [0][1][2][0][RTW89_IC][3] = 78, + [0][1][2][0][RTW89_KCC][3] = 66, [0][1][2][0][RTW89_ACMA][3] = 48, - [0][1][2][0][RTW89_FCC][4] = 64, + [0][1][2][0][RTW89_CN][3] = 46, + [0][1][2][0][RTW89_UK][3] = 48, + [0][1][2][0][RTW89_FCC][4] = 80, [0][1][2][0][RTW89_ETSI][4] = 48, [0][1][2][0][RTW89_MKK][4] = 68, - [0][1][2][0][RTW89_IC][4] = 74, + [0][1][2][0][RTW89_IC][4] = 80, + [0][1][2][0][RTW89_KCC][4] = 66, [0][1][2][0][RTW89_ACMA][4] = 48, - [0][1][2][0][RTW89_FCC][5] = 70, + [0][1][2][0][RTW89_CN][4] = 46, + [0][1][2][0][RTW89_UK][4] = 48, + [0][1][2][0][RTW89_FCC][5] = 80, [0][1][2][0][RTW89_ETSI][5] = 48, [0][1][2][0][RTW89_MKK][5] = 68, - [0][1][2][0][RTW89_IC][5] = 78, + [0][1][2][0][RTW89_IC][5] = 80, + [0][1][2][0][RTW89_KCC][5] = 66, [0][1][2][0][RTW89_ACMA][5] = 48, - [0][1][2][0][RTW89_FCC][6] = 66, + [0][1][2][0][RTW89_CN][5] = 46, + [0][1][2][0][RTW89_UK][5] = 48, + [0][1][2][0][RTW89_FCC][6] = 80, [0][1][2][0][RTW89_ETSI][6] = 48, [0][1][2][0][RTW89_MKK][6] = 68, - [0][1][2][0][RTW89_IC][6] = 76, + [0][1][2][0][RTW89_IC][6] = 80, + [0][1][2][0][RTW89_KCC][6] = 66, [0][1][2][0][RTW89_ACMA][6] = 48, - [0][1][2][0][RTW89_FCC][7] = 58, + [0][1][2][0][RTW89_CN][6] = 46, + [0][1][2][0][RTW89_UK][6] = 48, + [0][1][2][0][RTW89_FCC][7] = 74, [0][1][2][0][RTW89_ETSI][7] = 48, [0][1][2][0][RTW89_MKK][7] = 68, - [0][1][2][0][RTW89_IC][7] = 68, + [0][1][2][0][RTW89_IC][7] = 74, + [0][1][2][0][RTW89_KCC][7] = 66, [0][1][2][0][RTW89_ACMA][7] = 48, - [0][1][2][0][RTW89_FCC][8] = 54, + [0][1][2][0][RTW89_CN][7] = 46, + [0][1][2][0][RTW89_UK][7] = 48, + [0][1][2][0][RTW89_FCC][8] = 70, [0][1][2][0][RTW89_ETSI][8] = 48, [0][1][2][0][RTW89_MKK][8] = 68, - [0][1][2][0][RTW89_IC][8] = 64, + [0][1][2][0][RTW89_IC][8] = 70, + [0][1][2][0][RTW89_KCC][8] = 66, [0][1][2][0][RTW89_ACMA][8] = 48, - [0][1][2][0][RTW89_FCC][9] = 50, + [0][1][2][0][RTW89_CN][8] = 46, + [0][1][2][0][RTW89_UK][8] = 48, + [0][1][2][0][RTW89_FCC][9] = 66, [0][1][2][0][RTW89_ETSI][9] = 48, [0][1][2][0][RTW89_MKK][9] = 68, - [0][1][2][0][RTW89_IC][9] = 60, + [0][1][2][0][RTW89_IC][9] = 66, + [0][1][2][0][RTW89_KCC][9] = 64, [0][1][2][0][RTW89_ACMA][9] = 48, - [0][1][2][0][RTW89_FCC][10] = 50, + [0][1][2][0][RTW89_CN][9] = 46, + [0][1][2][0][RTW89_UK][9] = 48, + [0][1][2][0][RTW89_FCC][10] = 58, [0][1][2][0][RTW89_ETSI][10] = 48, [0][1][2][0][RTW89_MKK][10] = 68, - [0][1][2][0][RTW89_IC][10] = 60, + [0][1][2][0][RTW89_IC][10] = 58, + [0][1][2][0][RTW89_KCC][10] = 64, [0][1][2][0][RTW89_ACMA][10] = 48, - [0][1][2][0][RTW89_FCC][11] = 38, + [0][1][2][0][RTW89_CN][10] = 46, + [0][1][2][0][RTW89_UK][10] = 48, + [0][1][2][0][RTW89_FCC][11] = 58, [0][1][2][0][RTW89_ETSI][11] = 48, [0][1][2][0][RTW89_MKK][11] = 68, - [0][1][2][0][RTW89_IC][11] = 48, + [0][1][2][0][RTW89_IC][11] = 58, + [0][1][2][0][RTW89_KCC][11] = 64, [0][1][2][0][RTW89_ACMA][11] = 48, - [0][1][2][0][RTW89_FCC][12] = 34, + [0][1][2][0][RTW89_CN][11] = 46, + [0][1][2][0][RTW89_UK][11] = 48, + [0][1][2][0][RTW89_FCC][12] = 16, [0][1][2][0][RTW89_ETSI][12] = 48, [0][1][2][0][RTW89_MKK][12] = 68, - [0][1][2][0][RTW89_IC][12] = 44, + [0][1][2][0][RTW89_IC][12] = 16, + [0][1][2][0][RTW89_KCC][12] = 64, [0][1][2][0][RTW89_ACMA][12] = 48, + [0][1][2][0][RTW89_CN][12] = 46, + [0][1][2][0][RTW89_UK][12] = 48, [0][1][2][0][RTW89_FCC][13] = 127, [0][1][2][0][RTW89_ETSI][13] = 127, [0][1][2][0][RTW89_MKK][13] = 127, [0][1][2][0][RTW89_IC][13] = 127, + [0][1][2][0][RTW89_KCC][13] = 127, [0][1][2][0][RTW89_ACMA][13] = 127, - [0][1][2][1][RTW89_FCC][0] = 50, + [0][1][2][0][RTW89_CN][13] = 127, + [0][1][2][0][RTW89_UK][13] = 127, + [0][1][2][1][RTW89_FCC][0] = 64, [0][1][2][1][RTW89_ETSI][0] = 36, [0][1][2][1][RTW89_MKK][0] = 68, - [0][1][2][1][RTW89_IC][0] = 60, + [0][1][2][1][RTW89_IC][0] = 64, + [0][1][2][1][RTW89_KCC][0] = 66, [0][1][2][1][RTW89_ACMA][0] = 36, - [0][1][2][1][RTW89_FCC][1] = 50, + [0][1][2][1][RTW89_CN][0] = 36, + [0][1][2][1][RTW89_UK][0] = 36, + [0][1][2][1][RTW89_FCC][1] = 70, [0][1][2][1][RTW89_ETSI][1] = 36, [0][1][2][1][RTW89_MKK][1] = 68, - [0][1][2][1][RTW89_IC][1] = 60, + [0][1][2][1][RTW89_IC][1] = 70, + [0][1][2][1][RTW89_KCC][1] = 66, [0][1][2][1][RTW89_ACMA][1] = 36, - [0][1][2][1][RTW89_FCC][2] = 54, + [0][1][2][1][RTW89_CN][1] = 34, + [0][1][2][1][RTW89_UK][1] = 36, + [0][1][2][1][RTW89_FCC][2] = 74, [0][1][2][1][RTW89_ETSI][2] = 36, [0][1][2][1][RTW89_MKK][2] = 68, - [0][1][2][1][RTW89_IC][2] = 64, + [0][1][2][1][RTW89_IC][2] = 74, + [0][1][2][1][RTW89_KCC][2] = 66, [0][1][2][1][RTW89_ACMA][2] = 36, - [0][1][2][1][RTW89_FCC][3] = 58, + [0][1][2][1][RTW89_CN][2] = 34, + [0][1][2][1][RTW89_UK][2] = 36, + [0][1][2][1][RTW89_FCC][3] = 78, [0][1][2][1][RTW89_ETSI][3] = 36, [0][1][2][1][RTW89_MKK][3] = 68, - [0][1][2][1][RTW89_IC][3] = 68, + [0][1][2][1][RTW89_IC][3] = 78, + [0][1][2][1][RTW89_KCC][3] = 66, [0][1][2][1][RTW89_ACMA][3] = 36, - [0][1][2][1][RTW89_FCC][4] = 64, + [0][1][2][1][RTW89_CN][3] = 34, + [0][1][2][1][RTW89_UK][3] = 36, + [0][1][2][1][RTW89_FCC][4] = 80, [0][1][2][1][RTW89_ETSI][4] = 36, [0][1][2][1][RTW89_MKK][4] = 68, - [0][1][2][1][RTW89_IC][4] = 74, + [0][1][2][1][RTW89_IC][4] = 80, + [0][1][2][1][RTW89_KCC][4] = 66, [0][1][2][1][RTW89_ACMA][4] = 36, - [0][1][2][1][RTW89_FCC][5] = 70, + [0][1][2][1][RTW89_CN][4] = 34, + [0][1][2][1][RTW89_UK][4] = 36, + [0][1][2][1][RTW89_FCC][5] = 80, [0][1][2][1][RTW89_ETSI][5] = 36, [0][1][2][1][RTW89_MKK][5] = 68, - [0][1][2][1][RTW89_IC][5] = 78, + [0][1][2][1][RTW89_IC][5] = 80, + [0][1][2][1][RTW89_KCC][5] = 66, [0][1][2][1][RTW89_ACMA][5] = 36, - [0][1][2][1][RTW89_FCC][6] = 66, + [0][1][2][1][RTW89_CN][5] = 34, + [0][1][2][1][RTW89_UK][5] = 36, + [0][1][2][1][RTW89_FCC][6] = 80, [0][1][2][1][RTW89_ETSI][6] = 36, [0][1][2][1][RTW89_MKK][6] = 68, - [0][1][2][1][RTW89_IC][6] = 76, + [0][1][2][1][RTW89_IC][6] = 80, + [0][1][2][1][RTW89_KCC][6] = 66, [0][1][2][1][RTW89_ACMA][6] = 36, - [0][1][2][1][RTW89_FCC][7] = 58, + [0][1][2][1][RTW89_CN][6] = 34, + [0][1][2][1][RTW89_UK][6] = 36, + [0][1][2][1][RTW89_FCC][7] = 74, [0][1][2][1][RTW89_ETSI][7] = 36, [0][1][2][1][RTW89_MKK][7] = 68, - [0][1][2][1][RTW89_IC][7] = 68, + [0][1][2][1][RTW89_IC][7] = 74, + [0][1][2][1][RTW89_KCC][7] = 66, [0][1][2][1][RTW89_ACMA][7] = 36, - [0][1][2][1][RTW89_FCC][8] = 54, + [0][1][2][1][RTW89_CN][7] = 34, + [0][1][2][1][RTW89_UK][7] = 36, + [0][1][2][1][RTW89_FCC][8] = 70, [0][1][2][1][RTW89_ETSI][8] = 36, [0][1][2][1][RTW89_MKK][8] = 68, - [0][1][2][1][RTW89_IC][8] = 64, + [0][1][2][1][RTW89_IC][8] = 70, + [0][1][2][1][RTW89_KCC][8] = 66, [0][1][2][1][RTW89_ACMA][8] = 36, - [0][1][2][1][RTW89_FCC][9] = 50, + [0][1][2][1][RTW89_CN][8] = 34, + [0][1][2][1][RTW89_UK][8] = 36, + [0][1][2][1][RTW89_FCC][9] = 66, [0][1][2][1][RTW89_ETSI][9] = 36, [0][1][2][1][RTW89_MKK][9] = 68, - [0][1][2][1][RTW89_IC][9] = 60, + [0][1][2][1][RTW89_IC][9] = 66, + [0][1][2][1][RTW89_KCC][9] = 64, [0][1][2][1][RTW89_ACMA][9] = 36, - [0][1][2][1][RTW89_FCC][10] = 50, + [0][1][2][1][RTW89_CN][9] = 34, + [0][1][2][1][RTW89_UK][9] = 36, + [0][1][2][1][RTW89_FCC][10] = 58, [0][1][2][1][RTW89_ETSI][10] = 36, [0][1][2][1][RTW89_MKK][10] = 68, - [0][1][2][1][RTW89_IC][10] = 60, + [0][1][2][1][RTW89_IC][10] = 58, + [0][1][2][1][RTW89_KCC][10] = 64, [0][1][2][1][RTW89_ACMA][10] = 36, - [0][1][2][1][RTW89_FCC][11] = 38, + [0][1][2][1][RTW89_CN][10] = 34, + [0][1][2][1][RTW89_UK][10] = 36, + [0][1][2][1][RTW89_FCC][11] = 58, [0][1][2][1][RTW89_ETSI][11] = 36, [0][1][2][1][RTW89_MKK][11] = 68, - [0][1][2][1][RTW89_IC][11] = 48, + [0][1][2][1][RTW89_IC][11] = 58, + [0][1][2][1][RTW89_KCC][11] = 64, [0][1][2][1][RTW89_ACMA][11] = 36, - [0][1][2][1][RTW89_FCC][12] = 34, + [0][1][2][1][RTW89_CN][11] = 34, + [0][1][2][1][RTW89_UK][11] = 36, + [0][1][2][1][RTW89_FCC][12] = 16, [0][1][2][1][RTW89_ETSI][12] = 36, [0][1][2][1][RTW89_MKK][12] = 68, - [0][1][2][1][RTW89_IC][12] = 44, + [0][1][2][1][RTW89_IC][12] = 16, + [0][1][2][1][RTW89_KCC][12] = 64, [0][1][2][1][RTW89_ACMA][12] = 36, + [0][1][2][1][RTW89_CN][12] = 34, + [0][1][2][1][RTW89_UK][12] = 36, [0][1][2][1][RTW89_FCC][13] = 127, [0][1][2][1][RTW89_ETSI][13] = 127, [0][1][2][1][RTW89_MKK][13] = 127, [0][1][2][1][RTW89_IC][13] = 127, + [0][1][2][1][RTW89_KCC][13] = 127, [0][1][2][1][RTW89_ACMA][13] = 127, + [0][1][2][1][RTW89_CN][13] = 127, + [0][1][2][1][RTW89_UK][13] = 127, [1][0][2][0][RTW89_FCC][0] = 127, [1][0][2][0][RTW89_ETSI][0] = 127, [1][0][2][0][RTW89_MKK][0] = 127, [1][0][2][0][RTW89_IC][0] = 127, + [1][0][2][0][RTW89_KCC][0] = 127, [1][0][2][0][RTW89_ACMA][0] = 127, + [1][0][2][0][RTW89_CN][0] = 127, + [1][0][2][0][RTW89_UK][0] = 127, [1][0][2][0][RTW89_FCC][1] = 127, [1][0][2][0][RTW89_ETSI][1] = 127, [1][0][2][0][RTW89_MKK][1] = 127, [1][0][2][0][RTW89_IC][1] = 127, + [1][0][2][0][RTW89_KCC][1] = 127, [1][0][2][0][RTW89_ACMA][1] = 127, - [1][0][2][0][RTW89_FCC][2] = 62, + [1][0][2][0][RTW89_CN][1] = 127, + [1][0][2][0][RTW89_UK][1] = 127, + [1][0][2][0][RTW89_FCC][2] = 64, [1][0][2][0][RTW89_ETSI][2] = 60, [1][0][2][0][RTW89_MKK][2] = 74, - [1][0][2][0][RTW89_IC][2] = 72, + [1][0][2][0][RTW89_IC][2] = 64, + [1][0][2][0][RTW89_KCC][2] = 68, [1][0][2][0][RTW89_ACMA][2] = 60, - [1][0][2][0][RTW89_FCC][3] = 62, + [1][0][2][0][RTW89_CN][2] = 58, + [1][0][2][0][RTW89_UK][2] = 60, + [1][0][2][0][RTW89_FCC][3] = 64, [1][0][2][0][RTW89_ETSI][3] = 60, [1][0][2][0][RTW89_MKK][3] = 74, - [1][0][2][0][RTW89_IC][3] = 72, + [1][0][2][0][RTW89_IC][3] = 64, + [1][0][2][0][RTW89_KCC][3] = 68, [1][0][2][0][RTW89_ACMA][3] = 60, - [1][0][2][0][RTW89_FCC][4] = 64, + [1][0][2][0][RTW89_CN][3] = 58, + [1][0][2][0][RTW89_UK][3] = 60, + [1][0][2][0][RTW89_FCC][4] = 68, [1][0][2][0][RTW89_ETSI][4] = 60, [1][0][2][0][RTW89_MKK][4] = 74, - [1][0][2][0][RTW89_IC][4] = 74, + [1][0][2][0][RTW89_IC][4] = 68, + [1][0][2][0][RTW89_KCC][4] = 68, [1][0][2][0][RTW89_ACMA][4] = 60, - [1][0][2][0][RTW89_FCC][5] = 64, + [1][0][2][0][RTW89_CN][4] = 58, + [1][0][2][0][RTW89_UK][4] = 60, + [1][0][2][0][RTW89_FCC][5] = 68, [1][0][2][0][RTW89_ETSI][5] = 60, [1][0][2][0][RTW89_MKK][5] = 74, - [1][0][2][0][RTW89_IC][5] = 74, + [1][0][2][0][RTW89_IC][5] = 68, + [1][0][2][0][RTW89_KCC][5] = 74, [1][0][2][0][RTW89_ACMA][5] = 60, - [1][0][2][0][RTW89_FCC][6] = 64, + [1][0][2][0][RTW89_CN][5] = 58, + [1][0][2][0][RTW89_UK][5] = 60, + [1][0][2][0][RTW89_FCC][6] = 66, [1][0][2][0][RTW89_ETSI][6] = 60, [1][0][2][0][RTW89_MKK][6] = 74, - [1][0][2][0][RTW89_IC][6] = 74, + [1][0][2][0][RTW89_IC][6] = 66, + [1][0][2][0][RTW89_KCC][6] = 74, [1][0][2][0][RTW89_ACMA][6] = 60, - [1][0][2][0][RTW89_FCC][7] = 60, + [1][0][2][0][RTW89_CN][6] = 58, + [1][0][2][0][RTW89_UK][6] = 60, + [1][0][2][0][RTW89_FCC][7] = 62, [1][0][2][0][RTW89_ETSI][7] = 60, [1][0][2][0][RTW89_MKK][7] = 74, - [1][0][2][0][RTW89_IC][7] = 70, + [1][0][2][0][RTW89_IC][7] = 62, + [1][0][2][0][RTW89_KCC][7] = 74, [1][0][2][0][RTW89_ACMA][7] = 60, - [1][0][2][0][RTW89_FCC][8] = 60, + [1][0][2][0][RTW89_CN][7] = 58, + [1][0][2][0][RTW89_UK][7] = 60, + [1][0][2][0][RTW89_FCC][8] = 62, [1][0][2][0][RTW89_ETSI][8] = 60, [1][0][2][0][RTW89_MKK][8] = 74, - [1][0][2][0][RTW89_IC][8] = 70, + [1][0][2][0][RTW89_IC][8] = 62, + [1][0][2][0][RTW89_KCC][8] = 68, [1][0][2][0][RTW89_ACMA][8] = 60, + [1][0][2][0][RTW89_CN][8] = 58, + [1][0][2][0][RTW89_UK][8] = 60, [1][0][2][0][RTW89_FCC][9] = 60, [1][0][2][0][RTW89_ETSI][9] = 60, [1][0][2][0][RTW89_MKK][9] = 74, - [1][0][2][0][RTW89_IC][9] = 70, + [1][0][2][0][RTW89_IC][9] = 60, + [1][0][2][0][RTW89_KCC][9] = 68, [1][0][2][0][RTW89_ACMA][9] = 60, - [1][0][2][0][RTW89_FCC][10] = 58, + [1][0][2][0][RTW89_CN][9] = 58, + [1][0][2][0][RTW89_UK][9] = 60, + [1][0][2][0][RTW89_FCC][10] = 56, [1][0][2][0][RTW89_ETSI][10] = 60, [1][0][2][0][RTW89_MKK][10] = 74, - [1][0][2][0][RTW89_IC][10] = 68, + [1][0][2][0][RTW89_IC][10] = 56, + [1][0][2][0][RTW89_KCC][10] = 68, [1][0][2][0][RTW89_ACMA][10] = 60, + [1][0][2][0][RTW89_CN][10] = 58, + [1][0][2][0][RTW89_UK][10] = 60, [1][0][2][0][RTW89_FCC][11] = 127, [1][0][2][0][RTW89_ETSI][11] = 127, [1][0][2][0][RTW89_MKK][11] = 127, [1][0][2][0][RTW89_IC][11] = 127, + [1][0][2][0][RTW89_KCC][11] = 127, [1][0][2][0][RTW89_ACMA][11] = 127, + [1][0][2][0][RTW89_CN][11] = 127, + [1][0][2][0][RTW89_UK][11] = 127, [1][0][2][0][RTW89_FCC][12] = 127, [1][0][2][0][RTW89_ETSI][12] = 127, [1][0][2][0][RTW89_MKK][12] = 127, [1][0][2][0][RTW89_IC][12] = 127, + [1][0][2][0][RTW89_KCC][12] = 127, [1][0][2][0][RTW89_ACMA][12] = 127, + [1][0][2][0][RTW89_CN][12] = 127, + [1][0][2][0][RTW89_UK][12] = 127, [1][0][2][0][RTW89_FCC][13] = 127, [1][0][2][0][RTW89_ETSI][13] = 127, [1][0][2][0][RTW89_MKK][13] = 127, [1][0][2][0][RTW89_IC][13] = 127, + [1][0][2][0][RTW89_KCC][13] = 127, [1][0][2][0][RTW89_ACMA][13] = 127, + [1][0][2][0][RTW89_CN][13] = 127, + [1][0][2][0][RTW89_UK][13] = 127, [1][1][2][0][RTW89_FCC][0] = 127, [1][1][2][0][RTW89_ETSI][0] = 127, [1][1][2][0][RTW89_MKK][0] = 127, [1][1][2][0][RTW89_IC][0] = 127, + [1][1][2][0][RTW89_KCC][0] = 127, [1][1][2][0][RTW89_ACMA][0] = 127, + [1][1][2][0][RTW89_CN][0] = 127, + [1][1][2][0][RTW89_UK][0] = 127, [1][1][2][0][RTW89_FCC][1] = 127, [1][1][2][0][RTW89_ETSI][1] = 127, [1][1][2][0][RTW89_MKK][1] = 127, [1][1][2][0][RTW89_IC][1] = 127, + [1][1][2][0][RTW89_KCC][1] = 127, [1][1][2][0][RTW89_ACMA][1] = 127, - [1][1][2][0][RTW89_FCC][2] = 46, + [1][1][2][0][RTW89_CN][1] = 127, + [1][1][2][0][RTW89_UK][1] = 127, + [1][1][2][0][RTW89_FCC][2] = 60, [1][1][2][0][RTW89_ETSI][2] = 48, [1][1][2][0][RTW89_MKK][2] = 68, - [1][1][2][0][RTW89_IC][2] = 56, + [1][1][2][0][RTW89_IC][2] = 60, + [1][1][2][0][RTW89_KCC][2] = 64, [1][1][2][0][RTW89_ACMA][2] = 48, - [1][1][2][0][RTW89_FCC][3] = 46, + [1][1][2][0][RTW89_CN][2] = 34, + [1][1][2][0][RTW89_UK][2] = 48, + [1][1][2][0][RTW89_FCC][3] = 60, [1][1][2][0][RTW89_ETSI][3] = 48, [1][1][2][0][RTW89_MKK][3] = 68, - [1][1][2][0][RTW89_IC][3] = 56, + [1][1][2][0][RTW89_IC][3] = 60, + [1][1][2][0][RTW89_KCC][3] = 64, [1][1][2][0][RTW89_ACMA][3] = 48, - [1][1][2][0][RTW89_FCC][4] = 50, + [1][1][2][0][RTW89_CN][3] = 34, + [1][1][2][0][RTW89_UK][3] = 48, + [1][1][2][0][RTW89_FCC][4] = 60, [1][1][2][0][RTW89_ETSI][4] = 48, [1][1][2][0][RTW89_MKK][4] = 68, [1][1][2][0][RTW89_IC][4] = 60, + [1][1][2][0][RTW89_KCC][4] = 64, [1][1][2][0][RTW89_ACMA][4] = 48, - [1][1][2][0][RTW89_FCC][5] = 58, + [1][1][2][0][RTW89_CN][4] = 34, + [1][1][2][0][RTW89_UK][4] = 48, + [1][1][2][0][RTW89_FCC][5] = 60, [1][1][2][0][RTW89_ETSI][5] = 48, [1][1][2][0][RTW89_MKK][5] = 68, - [1][1][2][0][RTW89_IC][5] = 68, + [1][1][2][0][RTW89_IC][5] = 60, + [1][1][2][0][RTW89_KCC][5] = 66, [1][1][2][0][RTW89_ACMA][5] = 48, - [1][1][2][0][RTW89_FCC][6] = 50, + [1][1][2][0][RTW89_CN][5] = 34, + [1][1][2][0][RTW89_UK][5] = 48, + [1][1][2][0][RTW89_FCC][6] = 58, [1][1][2][0][RTW89_ETSI][6] = 48, [1][1][2][0][RTW89_MKK][6] = 68, - [1][1][2][0][RTW89_IC][6] = 60, + [1][1][2][0][RTW89_IC][6] = 58, + [1][1][2][0][RTW89_KCC][6] = 66, [1][1][2][0][RTW89_ACMA][6] = 48, - [1][1][2][0][RTW89_FCC][7] = 46, + [1][1][2][0][RTW89_CN][6] = 34, + [1][1][2][0][RTW89_UK][6] = 48, + [1][1][2][0][RTW89_FCC][7] = 54, [1][1][2][0][RTW89_ETSI][7] = 48, [1][1][2][0][RTW89_MKK][7] = 68, - [1][1][2][0][RTW89_IC][7] = 56, + [1][1][2][0][RTW89_IC][7] = 54, + [1][1][2][0][RTW89_KCC][7] = 66, [1][1][2][0][RTW89_ACMA][7] = 48, - [1][1][2][0][RTW89_FCC][8] = 46, + [1][1][2][0][RTW89_CN][7] = 34, + [1][1][2][0][RTW89_UK][7] = 48, + [1][1][2][0][RTW89_FCC][8] = 54, [1][1][2][0][RTW89_ETSI][8] = 48, [1][1][2][0][RTW89_MKK][8] = 68, - [1][1][2][0][RTW89_IC][8] = 56, + [1][1][2][0][RTW89_IC][8] = 54, + [1][1][2][0][RTW89_KCC][8] = 64, [1][1][2][0][RTW89_ACMA][8] = 48, - [1][1][2][0][RTW89_FCC][9] = 34, + [1][1][2][0][RTW89_CN][8] = 34, + [1][1][2][0][RTW89_UK][8] = 48, + [1][1][2][0][RTW89_FCC][9] = 54, [1][1][2][0][RTW89_ETSI][9] = 48, [1][1][2][0][RTW89_MKK][9] = 68, - [1][1][2][0][RTW89_IC][9] = 44, + [1][1][2][0][RTW89_IC][9] = 54, + [1][1][2][0][RTW89_KCC][9] = 64, [1][1][2][0][RTW89_ACMA][9] = 48, - [1][1][2][0][RTW89_FCC][10] = 30, + [1][1][2][0][RTW89_CN][9] = 34, + [1][1][2][0][RTW89_UK][9] = 48, + [1][1][2][0][RTW89_FCC][10] = 46, [1][1][2][0][RTW89_ETSI][10] = 48, [1][1][2][0][RTW89_MKK][10] = 68, - [1][1][2][0][RTW89_IC][10] = 40, + [1][1][2][0][RTW89_IC][10] = 46, + [1][1][2][0][RTW89_KCC][10] = 64, [1][1][2][0][RTW89_ACMA][10] = 48, + [1][1][2][0][RTW89_CN][10] = 34, + [1][1][2][0][RTW89_UK][10] = 48, [1][1][2][0][RTW89_FCC][11] = 127, [1][1][2][0][RTW89_ETSI][11] = 127, [1][1][2][0][RTW89_MKK][11] = 127, [1][1][2][0][RTW89_IC][11] = 127, + [1][1][2][0][RTW89_KCC][11] = 127, [1][1][2][0][RTW89_ACMA][11] = 127, + [1][1][2][0][RTW89_CN][11] = 127, + [1][1][2][0][RTW89_UK][11] = 127, [1][1][2][0][RTW89_FCC][12] = 127, [1][1][2][0][RTW89_ETSI][12] = 127, [1][1][2][0][RTW89_MKK][12] = 127, [1][1][2][0][RTW89_IC][12] = 127, + [1][1][2][0][RTW89_KCC][12] = 127, [1][1][2][0][RTW89_ACMA][12] = 127, + [1][1][2][0][RTW89_CN][12] = 127, + [1][1][2][0][RTW89_UK][12] = 127, [1][1][2][0][RTW89_FCC][13] = 127, [1][1][2][0][RTW89_ETSI][13] = 127, [1][1][2][0][RTW89_MKK][13] = 127, [1][1][2][0][RTW89_IC][13] = 127, + [1][1][2][0][RTW89_KCC][13] = 127, [1][1][2][0][RTW89_ACMA][13] = 127, + [1][1][2][0][RTW89_CN][13] = 127, + [1][1][2][0][RTW89_UK][13] = 127, [1][1][2][1][RTW89_FCC][0] = 127, [1][1][2][1][RTW89_ETSI][0] = 127, [1][1][2][1][RTW89_MKK][0] = 127, [1][1][2][1][RTW89_IC][0] = 127, + [1][1][2][1][RTW89_KCC][0] = 127, [1][1][2][1][RTW89_ACMA][0] = 127, + [1][1][2][1][RTW89_CN][0] = 127, + [1][1][2][1][RTW89_UK][0] = 127, [1][1][2][1][RTW89_FCC][1] = 127, [1][1][2][1][RTW89_ETSI][1] = 127, [1][1][2][1][RTW89_MKK][1] = 127, [1][1][2][1][RTW89_IC][1] = 127, + [1][1][2][1][RTW89_KCC][1] = 127, [1][1][2][1][RTW89_ACMA][1] = 127, - [1][1][2][1][RTW89_FCC][2] = 46, + [1][1][2][1][RTW89_CN][1] = 127, + [1][1][2][1][RTW89_UK][1] = 127, + [1][1][2][1][RTW89_FCC][2] = 60, [1][1][2][1][RTW89_ETSI][2] = 36, [1][1][2][1][RTW89_MKK][2] = 68, - [1][1][2][1][RTW89_IC][2] = 56, + [1][1][2][1][RTW89_IC][2] = 60, + [1][1][2][1][RTW89_KCC][2] = 64, [1][1][2][1][RTW89_ACMA][2] = 36, - [1][1][2][1][RTW89_FCC][3] = 46, + [1][1][2][1][RTW89_CN][2] = 34, + [1][1][2][1][RTW89_UK][2] = 36, + [1][1][2][1][RTW89_FCC][3] = 60, [1][1][2][1][RTW89_ETSI][3] = 36, [1][1][2][1][RTW89_MKK][3] = 68, - [1][1][2][1][RTW89_IC][3] = 56, + [1][1][2][1][RTW89_IC][3] = 60, + [1][1][2][1][RTW89_KCC][3] = 64, [1][1][2][1][RTW89_ACMA][3] = 36, - [1][1][2][1][RTW89_FCC][4] = 50, + [1][1][2][1][RTW89_CN][3] = 34, + [1][1][2][1][RTW89_UK][3] = 36, + [1][1][2][1][RTW89_FCC][4] = 60, [1][1][2][1][RTW89_ETSI][4] = 36, [1][1][2][1][RTW89_MKK][4] = 68, [1][1][2][1][RTW89_IC][4] = 60, + [1][1][2][1][RTW89_KCC][4] = 64, [1][1][2][1][RTW89_ACMA][4] = 36, - [1][1][2][1][RTW89_FCC][5] = 58, + [1][1][2][1][RTW89_CN][4] = 34, + [1][1][2][1][RTW89_UK][4] = 36, + [1][1][2][1][RTW89_FCC][5] = 60, [1][1][2][1][RTW89_ETSI][5] = 36, [1][1][2][1][RTW89_MKK][5] = 68, - [1][1][2][1][RTW89_IC][5] = 68, + [1][1][2][1][RTW89_IC][5] = 60, + [1][1][2][1][RTW89_KCC][5] = 66, [1][1][2][1][RTW89_ACMA][5] = 36, - [1][1][2][1][RTW89_FCC][6] = 50, + [1][1][2][1][RTW89_CN][5] = 34, + [1][1][2][1][RTW89_UK][5] = 36, + [1][1][2][1][RTW89_FCC][6] = 58, [1][1][2][1][RTW89_ETSI][6] = 36, [1][1][2][1][RTW89_MKK][6] = 68, - [1][1][2][1][RTW89_IC][6] = 60, + [1][1][2][1][RTW89_IC][6] = 58, + [1][1][2][1][RTW89_KCC][6] = 66, [1][1][2][1][RTW89_ACMA][6] = 36, - [1][1][2][1][RTW89_FCC][7] = 46, + [1][1][2][1][RTW89_CN][6] = 34, + [1][1][2][1][RTW89_UK][6] = 36, + [1][1][2][1][RTW89_FCC][7] = 54, [1][1][2][1][RTW89_ETSI][7] = 36, [1][1][2][1][RTW89_MKK][7] = 68, - [1][1][2][1][RTW89_IC][7] = 56, + [1][1][2][1][RTW89_IC][7] = 54, + [1][1][2][1][RTW89_KCC][7] = 66, [1][1][2][1][RTW89_ACMA][7] = 36, - [1][1][2][1][RTW89_FCC][8] = 46, + [1][1][2][1][RTW89_CN][7] = 34, + [1][1][2][1][RTW89_UK][7] = 36, + [1][1][2][1][RTW89_FCC][8] = 54, [1][1][2][1][RTW89_ETSI][8] = 36, [1][1][2][1][RTW89_MKK][8] = 68, - [1][1][2][1][RTW89_IC][8] = 56, + [1][1][2][1][RTW89_IC][8] = 54, + [1][1][2][1][RTW89_KCC][8] = 64, [1][1][2][1][RTW89_ACMA][8] = 36, - [1][1][2][1][RTW89_FCC][9] = 34, + [1][1][2][1][RTW89_CN][8] = 34, + [1][1][2][1][RTW89_UK][8] = 36, + [1][1][2][1][RTW89_FCC][9] = 54, [1][1][2][1][RTW89_ETSI][9] = 36, [1][1][2][1][RTW89_MKK][9] = 68, - [1][1][2][1][RTW89_IC][9] = 44, + [1][1][2][1][RTW89_IC][9] = 54, + [1][1][2][1][RTW89_KCC][9] = 64, [1][1][2][1][RTW89_ACMA][9] = 36, - [1][1][2][1][RTW89_FCC][10] = 30, + [1][1][2][1][RTW89_CN][9] = 34, + [1][1][2][1][RTW89_UK][9] = 36, + [1][1][2][1][RTW89_FCC][10] = 46, [1][1][2][1][RTW89_ETSI][10] = 36, [1][1][2][1][RTW89_MKK][10] = 68, - [1][1][2][1][RTW89_IC][10] = 40, + [1][1][2][1][RTW89_IC][10] = 46, + [1][1][2][1][RTW89_KCC][10] = 64, [1][1][2][1][RTW89_ACMA][10] = 36, + [1][1][2][1][RTW89_CN][10] = 36, + [1][1][2][1][RTW89_UK][10] = 36, [1][1][2][1][RTW89_FCC][11] = 127, [1][1][2][1][RTW89_ETSI][11] = 127, [1][1][2][1][RTW89_MKK][11] = 127, [1][1][2][1][RTW89_IC][11] = 127, + [1][1][2][1][RTW89_KCC][11] = 127, [1][1][2][1][RTW89_ACMA][11] = 127, + [1][1][2][1][RTW89_CN][11] = 127, + [1][1][2][1][RTW89_UK][11] = 127, [1][1][2][1][RTW89_FCC][12] = 127, [1][1][2][1][RTW89_ETSI][12] = 127, [1][1][2][1][RTW89_MKK][12] = 127, [1][1][2][1][RTW89_IC][12] = 127, + [1][1][2][1][RTW89_KCC][12] = 127, [1][1][2][1][RTW89_ACMA][12] = 127, + [1][1][2][1][RTW89_CN][12] = 127, + [1][1][2][1][RTW89_UK][12] = 127, [1][1][2][1][RTW89_FCC][13] = 127, [1][1][2][1][RTW89_ETSI][13] = 127, [1][1][2][1][RTW89_MKK][13] = 127, [1][1][2][1][RTW89_IC][13] = 127, + [1][1][2][1][RTW89_KCC][13] = 127, [1][1][2][1][RTW89_ACMA][13] = 127, + [1][1][2][1][RTW89_CN][13] = 127, + [1][1][2][1][RTW89_UK][13] = 127, }; const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = { - [0][0][1][0][RTW89_WW][0] = 60, - [0][0][1][0][RTW89_WW][2] = 60, - [0][0][1][0][RTW89_WW][4] = 60, - [0][0][1][0][RTW89_WW][6] = 60, - [0][0][1][0][RTW89_WW][8] = 60, - [0][0][1][0][RTW89_WW][10] = 60, - [0][0][1][0][RTW89_WW][12] = 60, - [0][0][1][0][RTW89_WW][14] = 60, - [0][0][1][0][RTW89_WW][15] = 60, - [0][0][1][0][RTW89_WW][17] = 60, - [0][0][1][0][RTW89_WW][19] = 60, - [0][0][1][0][RTW89_WW][21] = 60, - [0][0][1][0][RTW89_WW][23] = 60, + [0][0][1][0][RTW89_WW][0] = 50, + [0][0][1][0][RTW89_WW][2] = 50, + [0][0][1][0][RTW89_WW][4] = 50, + [0][0][1][0][RTW89_WW][6] = 50, + [0][0][1][0][RTW89_WW][8] = 50, + [0][0][1][0][RTW89_WW][10] = 50, + [0][0][1][0][RTW89_WW][12] = 50, + [0][0][1][0][RTW89_WW][14] = 50, + [0][0][1][0][RTW89_WW][15] = 66, + [0][0][1][0][RTW89_WW][17] = 66, + [0][0][1][0][RTW89_WW][19] = 66, + [0][0][1][0][RTW89_WW][21] = 66, + [0][0][1][0][RTW89_WW][23] = 66, [0][0][1][0][RTW89_WW][25] = 66, [0][0][1][0][RTW89_WW][27] = 66, [0][0][1][0][RTW89_WW][29] = 66, - [0][0][1][0][RTW89_WW][31] = 60, - [0][0][1][0][RTW89_WW][33] = 60, + [0][0][1][0][RTW89_WW][31] = 66, + [0][0][1][0][RTW89_WW][33] = 66, [0][0][1][0][RTW89_WW][35] = 60, - [0][0][1][0][RTW89_WW][37] = 70, + [0][0][1][0][RTW89_WW][37] = 64, [0][0][1][0][RTW89_WW][38] = 30, [0][0][1][0][RTW89_WW][40] = 30, [0][0][1][0][RTW89_WW][42] = 30, [0][0][1][0][RTW89_WW][44] = 30, [0][0][1][0][RTW89_WW][46] = 30, - [0][0][1][0][RTW89_WW][48] = 70, - [0][0][1][0][RTW89_WW][50] = 70, - [0][0][1][0][RTW89_WW][52] = 70, - [0][1][1][0][RTW89_WW][0] = 42, - [0][1][1][0][RTW89_WW][2] = 42, - [0][1][1][0][RTW89_WW][4] = 42, - [0][1][1][0][RTW89_WW][6] = 42, - [0][1][1][0][RTW89_WW][8] = 48, - [0][1][1][0][RTW89_WW][10] = 48, - [0][1][1][0][RTW89_WW][12] = 48, - [0][1][1][0][RTW89_WW][14] = 48, - [0][1][1][0][RTW89_WW][15] = 48, - [0][1][1][0][RTW89_WW][17] = 48, - [0][1][1][0][RTW89_WW][19] = 48, - [0][1][1][0][RTW89_WW][21] = 48, - [0][1][1][0][RTW89_WW][23] = 48, + [0][0][1][0][RTW89_WW][48] = 72, + [0][0][1][0][RTW89_WW][50] = 72, + [0][0][1][0][RTW89_WW][52] = 72, + [0][1][1][0][RTW89_WW][0] = 34, + [0][1][1][0][RTW89_WW][2] = 34, + [0][1][1][0][RTW89_WW][4] = 34, + [0][1][1][0][RTW89_WW][6] = 36, + [0][1][1][0][RTW89_WW][8] = 46, + [0][1][1][0][RTW89_WW][10] = 46, + [0][1][1][0][RTW89_WW][12] = 46, + [0][1][1][0][RTW89_WW][14] = 46, + [0][1][1][0][RTW89_WW][15] = 54, + [0][1][1][0][RTW89_WW][17] = 54, + [0][1][1][0][RTW89_WW][19] = 54, + [0][1][1][0][RTW89_WW][21] = 54, + [0][1][1][0][RTW89_WW][23] = 54, [0][1][1][0][RTW89_WW][25] = 54, [0][1][1][0][RTW89_WW][27] = 54, [0][1][1][0][RTW89_WW][29] = 54, - [0][1][1][0][RTW89_WW][31] = 48, - [0][1][1][0][RTW89_WW][33] = 48, - [0][1][1][0][RTW89_WW][35] = 48, - [0][1][1][0][RTW89_WW][37] = 60, + [0][1][1][0][RTW89_WW][31] = 54, + [0][1][1][0][RTW89_WW][33] = 54, + [0][1][1][0][RTW89_WW][35] = 52, + [0][1][1][0][RTW89_WW][37] = 52, [0][1][1][0][RTW89_WW][38] = 18, - [0][1][1][0][RTW89_WW][40] = 16, + [0][1][1][0][RTW89_WW][40] = 18, [0][1][1][0][RTW89_WW][42] = 18, - [0][1][1][0][RTW89_WW][44] = 16, + [0][1][1][0][RTW89_WW][44] = 18, [0][1][1][0][RTW89_WW][46] = 18, [0][1][1][0][RTW89_WW][48] = 48, [0][1][1][0][RTW89_WW][50] = 48, [0][1][1][0][RTW89_WW][52] = 48, - [0][0][2][0][RTW89_WW][0] = 62, - [0][0][2][0][RTW89_WW][2] = 62, - [0][0][2][0][RTW89_WW][4] = 62, - [0][0][2][0][RTW89_WW][6] = 60, - [0][0][2][0][RTW89_WW][8] = 58, - [0][0][2][0][RTW89_WW][10] = 62, - [0][0][2][0][RTW89_WW][12] = 62, - [0][0][2][0][RTW89_WW][14] = 62, - [0][0][2][0][RTW89_WW][15] = 62, - [0][0][2][0][RTW89_WW][17] = 62, - [0][0][2][0][RTW89_WW][19] = 62, - [0][0][2][0][RTW89_WW][21] = 62, - [0][0][2][0][RTW89_WW][23] = 62, + [0][0][2][0][RTW89_WW][0] = 52, + [0][0][2][0][RTW89_WW][2] = 52, + [0][0][2][0][RTW89_WW][4] = 52, + [0][0][2][0][RTW89_WW][6] = 52, + [0][0][2][0][RTW89_WW][8] = 52, + [0][0][2][0][RTW89_WW][10] = 52, + [0][0][2][0][RTW89_WW][12] = 52, + [0][0][2][0][RTW89_WW][14] = 52, + [0][0][2][0][RTW89_WW][15] = 66, + [0][0][2][0][RTW89_WW][17] = 66, + [0][0][2][0][RTW89_WW][19] = 66, + [0][0][2][0][RTW89_WW][21] = 66, + [0][0][2][0][RTW89_WW][23] = 66, [0][0][2][0][RTW89_WW][25] = 66, [0][0][2][0][RTW89_WW][27] = 66, [0][0][2][0][RTW89_WW][29] = 66, - [0][0][2][0][RTW89_WW][31] = 62, - [0][0][2][0][RTW89_WW][33] = 62, - [0][0][2][0][RTW89_WW][35] = 62, - [0][0][2][0][RTW89_WW][37] = 70, + [0][0][2][0][RTW89_WW][31] = 66, + [0][0][2][0][RTW89_WW][33] = 66, + [0][0][2][0][RTW89_WW][35] = 56, + [0][0][2][0][RTW89_WW][37] = 64, [0][0][2][0][RTW89_WW][38] = 30, [0][0][2][0][RTW89_WW][40] = 30, [0][0][2][0][RTW89_WW][42] = 30, [0][0][2][0][RTW89_WW][44] = 30, [0][0][2][0][RTW89_WW][46] = 30, - [0][0][2][0][RTW89_WW][48] = 70, - [0][0][2][0][RTW89_WW][50] = 70, - [0][0][2][0][RTW89_WW][52] = 70, - [0][1][2][0][RTW89_WW][0] = 44, - [0][1][2][0][RTW89_WW][2] = 44, - [0][1][2][0][RTW89_WW][4] = 44, - [0][1][2][0][RTW89_WW][6] = 44, - [0][1][2][0][RTW89_WW][8] = 42, - [0][1][2][0][RTW89_WW][10] = 50, - [0][1][2][0][RTW89_WW][12] = 50, - [0][1][2][0][RTW89_WW][14] = 50, - [0][1][2][0][RTW89_WW][15] = 50, - [0][1][2][0][RTW89_WW][17] = 50, - [0][1][2][0][RTW89_WW][19] = 50, - [0][1][2][0][RTW89_WW][21] = 50, - [0][1][2][0][RTW89_WW][23] = 50, + [0][0][2][0][RTW89_WW][48] = 72, + [0][0][2][0][RTW89_WW][50] = 72, + [0][0][2][0][RTW89_WW][52] = 72, + [0][1][2][0][RTW89_WW][0] = 36, + [0][1][2][0][RTW89_WW][2] = 36, + [0][1][2][0][RTW89_WW][4] = 36, + [0][1][2][0][RTW89_WW][6] = 38, + [0][1][2][0][RTW89_WW][8] = 40, + [0][1][2][0][RTW89_WW][10] = 40, + [0][1][2][0][RTW89_WW][12] = 40, + [0][1][2][0][RTW89_WW][14] = 40, + [0][1][2][0][RTW89_WW][15] = 54, + [0][1][2][0][RTW89_WW][17] = 54, + [0][1][2][0][RTW89_WW][19] = 54, + [0][1][2][0][RTW89_WW][21] = 54, + [0][1][2][0][RTW89_WW][23] = 54, [0][1][2][0][RTW89_WW][25] = 54, [0][1][2][0][RTW89_WW][27] = 54, [0][1][2][0][RTW89_WW][29] = 54, - [0][1][2][0][RTW89_WW][31] = 50, - [0][1][2][0][RTW89_WW][33] = 50, - [0][1][2][0][RTW89_WW][35] = 50, - [0][1][2][0][RTW89_WW][37] = 62, + [0][1][2][0][RTW89_WW][31] = 54, + [0][1][2][0][RTW89_WW][33] = 54, + [0][1][2][0][RTW89_WW][35] = 46, + [0][1][2][0][RTW89_WW][37] = 52, [0][1][2][0][RTW89_WW][38] = 18, [0][1][2][0][RTW89_WW][40] = 18, [0][1][2][0][RTW89_WW][42] = 18, [0][1][2][0][RTW89_WW][44] = 18, [0][1][2][0][RTW89_WW][46] = 18, - [0][1][2][0][RTW89_WW][48] = 50, + [0][1][2][0][RTW89_WW][48] = 48, [0][1][2][0][RTW89_WW][50] = 50, - [0][1][2][0][RTW89_WW][52] = 50, - [0][1][2][1][RTW89_WW][0] = 38, - [0][1][2][1][RTW89_WW][2] = 38, - [0][1][2][1][RTW89_WW][4] = 38, - [0][1][2][1][RTW89_WW][6] = 38, - [0][1][2][1][RTW89_WW][8] = 38, - [0][1][2][1][RTW89_WW][10] = 38, - [0][1][2][1][RTW89_WW][12] = 38, - [0][1][2][1][RTW89_WW][14] = 38, - [0][1][2][1][RTW89_WW][15] = 38, - [0][1][2][1][RTW89_WW][17] = 38, - [0][1][2][1][RTW89_WW][19] = 38, - [0][1][2][1][RTW89_WW][21] = 38, - [0][1][2][1][RTW89_WW][23] = 38, + [0][1][2][0][RTW89_WW][52] = 48, + [0][1][2][1][RTW89_WW][0] = 36, + [0][1][2][1][RTW89_WW][2] = 36, + [0][1][2][1][RTW89_WW][4] = 36, + [0][1][2][1][RTW89_WW][6] = 36, + [0][1][2][1][RTW89_WW][8] = 36, + [0][1][2][1][RTW89_WW][10] = 36, + [0][1][2][1][RTW89_WW][12] = 36, + [0][1][2][1][RTW89_WW][14] = 36, + [0][1][2][1][RTW89_WW][15] = 40, + [0][1][2][1][RTW89_WW][17] = 40, + [0][1][2][1][RTW89_WW][19] = 40, + [0][1][2][1][RTW89_WW][21] = 40, + [0][1][2][1][RTW89_WW][23] = 40, [0][1][2][1][RTW89_WW][25] = 40, [0][1][2][1][RTW89_WW][27] = 40, [0][1][2][1][RTW89_WW][29] = 40, - [0][1][2][1][RTW89_WW][31] = 38, - [0][1][2][1][RTW89_WW][33] = 38, - [0][1][2][1][RTW89_WW][35] = 38, - [0][1][2][1][RTW89_WW][37] = 60, + [0][1][2][1][RTW89_WW][31] = 40, + [0][1][2][1][RTW89_WW][33] = 40, + [0][1][2][1][RTW89_WW][35] = 40, + [0][1][2][1][RTW89_WW][37] = 40, [0][1][2][1][RTW89_WW][38] = 6, [0][1][2][1][RTW89_WW][40] = 6, [0][1][2][1][RTW89_WW][42] = 6, [0][1][2][1][RTW89_WW][44] = 6, [0][1][2][1][RTW89_WW][46] = 6, - [0][1][2][1][RTW89_WW][48] = 50, + [0][1][2][1][RTW89_WW][48] = 48, [0][1][2][1][RTW89_WW][50] = 50, - [0][1][2][1][RTW89_WW][52] = 50, - [1][0][2][0][RTW89_WW][1] = 58, - [1][0][2][0][RTW89_WW][5] = 66, - [1][0][2][0][RTW89_WW][9] = 66, - [1][0][2][0][RTW89_WW][13] = 58, + [0][1][2][1][RTW89_WW][52] = 48, + [1][0][2][0][RTW89_WW][1] = 54, + [1][0][2][0][RTW89_WW][5] = 54, + [1][0][2][0][RTW89_WW][9] = 54, + [1][0][2][0][RTW89_WW][13] = 52, [1][0][2][0][RTW89_WW][16] = 56, - [1][0][2][0][RTW89_WW][20] = 66, - [1][0][2][0][RTW89_WW][24] = 66, + [1][0][2][0][RTW89_WW][20] = 56, + [1][0][2][0][RTW89_WW][24] = 56, [1][0][2][0][RTW89_WW][28] = 66, - [1][0][2][0][RTW89_WW][32] = 66, - [1][0][2][0][RTW89_WW][36] = 66, + [1][0][2][0][RTW89_WW][32] = 62, + [1][0][2][0][RTW89_WW][36] = 64, [1][0][2][0][RTW89_WW][39] = 30, [1][0][2][0][RTW89_WW][43] = 30, [1][0][2][0][RTW89_WW][47] = 68, [1][0][2][0][RTW89_WW][51] = 68, - [1][1][2][0][RTW89_WW][1] = 48, - [1][1][2][0][RTW89_WW][5] = 52, - [1][1][2][0][RTW89_WW][9] = 52, - [1][1][2][0][RTW89_WW][13] = 52, - [1][1][2][0][RTW89_WW][16] = 48, + [1][1][2][0][RTW89_WW][1] = 42, + [1][1][2][0][RTW89_WW][5] = 42, + [1][1][2][0][RTW89_WW][9] = 42, + [1][1][2][0][RTW89_WW][13] = 42, + [1][1][2][0][RTW89_WW][16] = 54, [1][1][2][0][RTW89_WW][20] = 54, [1][1][2][0][RTW89_WW][24] = 54, [1][1][2][0][RTW89_WW][28] = 54, [1][1][2][0][RTW89_WW][32] = 54, - [1][1][2][0][RTW89_WW][36] = 66, + [1][1][2][0][RTW89_WW][36] = 52, [1][1][2][0][RTW89_WW][39] = 18, [1][1][2][0][RTW89_WW][43] = 18, - [1][1][2][0][RTW89_WW][47] = 60, - [1][1][2][0][RTW89_WW][51] = 58, + [1][1][2][0][RTW89_WW][47] = 62, + [1][1][2][0][RTW89_WW][51] = 60, [1][1][2][1][RTW89_WW][1] = 40, [1][1][2][1][RTW89_WW][5] = 40, [1][1][2][1][RTW89_WW][9] = 40, @@ -15035,2082 +29517,3694 @@ const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM] [1][1][2][1][RTW89_WW][24] = 40, [1][1][2][1][RTW89_WW][28] = 40, [1][1][2][1][RTW89_WW][32] = 40, - [1][1][2][1][RTW89_WW][36] = 60, + [1][1][2][1][RTW89_WW][36] = 40, [1][1][2][1][RTW89_WW][39] = 6, [1][1][2][1][RTW89_WW][43] = 6, - [1][1][2][1][RTW89_WW][47] = 60, - [1][1][2][1][RTW89_WW][51] = 58, - [2][0][2][0][RTW89_WW][3] = 56, - [2][0][2][0][RTW89_WW][11] = 58, - [2][0][2][0][RTW89_WW][18] = 54, + [1][1][2][1][RTW89_WW][47] = 62, + [1][1][2][1][RTW89_WW][51] = 60, + [2][0][2][0][RTW89_WW][3] = 54, + [2][0][2][0][RTW89_WW][11] = 50, + [2][0][2][0][RTW89_WW][18] = 56, [2][0][2][0][RTW89_WW][26] = 60, [2][0][2][0][RTW89_WW][34] = 60, [2][0][2][0][RTW89_WW][41] = 30, - [2][0][2][0][RTW89_WW][49] = 56, - [2][1][2][0][RTW89_WW][3] = 48, - [2][1][2][0][RTW89_WW][11] = 52, - [2][1][2][0][RTW89_WW][18] = 48, - [2][1][2][0][RTW89_WW][26] = 54, - [2][1][2][0][RTW89_WW][34] = 60, + [2][0][2][0][RTW89_WW][49] = 62, + [2][1][2][0][RTW89_WW][3] = 46, + [2][1][2][0][RTW89_WW][11] = 38, + [2][1][2][0][RTW89_WW][18] = 50, + [2][1][2][0][RTW89_WW][26] = 52, + [2][1][2][0][RTW89_WW][34] = 52, [2][1][2][0][RTW89_WW][41] = 18, - [2][1][2][0][RTW89_WW][49] = 50, + [2][1][2][0][RTW89_WW][49] = 62, [2][1][2][1][RTW89_WW][3] = 40, - [2][1][2][1][RTW89_WW][11] = 40, + [2][1][2][1][RTW89_WW][11] = 38, [2][1][2][1][RTW89_WW][18] = 40, [2][1][2][1][RTW89_WW][26] = 42, - [2][1][2][1][RTW89_WW][34] = 60, + [2][1][2][1][RTW89_WW][34] = 40, [2][1][2][1][RTW89_WW][41] = 6, - [2][1][2][1][RTW89_WW][49] = 50, - [3][0][2][0][RTW89_WW][7] = 38, - [3][0][2][0][RTW89_WW][22] = 50, - [3][0][2][0][RTW89_WW][45] = 0, - [3][1][2][0][RTW89_WW][7] = 26, - [3][1][2][0][RTW89_WW][22] = 42, - [3][1][2][0][RTW89_WW][45] = 0, - [3][1][2][1][RTW89_WW][7] = 14, - [3][1][2][1][RTW89_WW][22] = 30, - [3][1][2][1][RTW89_WW][45] = 0, - [0][0][1][0][RTW89_FCC][0] = 70, + [2][1][2][1][RTW89_WW][49] = 62, + [3][0][2][0][RTW89_WW][7] = 40, + [3][0][2][0][RTW89_WW][22] = 42, + [3][0][2][0][RTW89_WW][45] = 52, + [3][1][2][0][RTW89_WW][7] = 32, + [3][1][2][0][RTW89_WW][22] = 36, + [3][1][2][0][RTW89_WW][45] = 46, + [3][1][2][1][RTW89_WW][7] = 32, + [3][1][2][1][RTW89_WW][22] = 36, + [3][1][2][1][RTW89_WW][45] = 46, + [0][0][1][0][RTW89_FCC][0] = 72, [0][0][1][0][RTW89_ETSI][0] = 66, [0][0][1][0][RTW89_MKK][0] = 66, - [0][0][1][0][RTW89_IC][0] = 62, - [0][0][1][0][RTW89_ACMA][0] = 60, - [0][0][1][0][RTW89_FCC][2] = 70, + [0][0][1][0][RTW89_IC][0] = 60, + [0][0][1][0][RTW89_KCC][0] = 52, + [0][0][1][0][RTW89_ACMA][0] = 66, + [0][0][1][0][RTW89_CN][0] = 50, + [0][0][1][0][RTW89_UK][0] = 66, + [0][0][1][0][RTW89_FCC][2] = 72, [0][0][1][0][RTW89_ETSI][2] = 66, [0][0][1][0][RTW89_MKK][2] = 66, - [0][0][1][0][RTW89_IC][2] = 62, - [0][0][1][0][RTW89_ACMA][2] = 60, - [0][0][1][0][RTW89_FCC][4] = 70, + [0][0][1][0][RTW89_IC][2] = 60, + [0][0][1][0][RTW89_KCC][2] = 52, + [0][0][1][0][RTW89_ACMA][2] = 66, + [0][0][1][0][RTW89_CN][2] = 50, + [0][0][1][0][RTW89_UK][2] = 66, + [0][0][1][0][RTW89_FCC][4] = 72, [0][0][1][0][RTW89_ETSI][4] = 66, [0][0][1][0][RTW89_MKK][4] = 66, - [0][0][1][0][RTW89_IC][4] = 62, - [0][0][1][0][RTW89_ACMA][4] = 60, - [0][0][1][0][RTW89_FCC][6] = 70, + [0][0][1][0][RTW89_IC][4] = 60, + [0][0][1][0][RTW89_KCC][4] = 52, + [0][0][1][0][RTW89_ACMA][4] = 66, + [0][0][1][0][RTW89_CN][4] = 50, + [0][0][1][0][RTW89_UK][4] = 66, + [0][0][1][0][RTW89_FCC][6] = 72, [0][0][1][0][RTW89_ETSI][6] = 66, [0][0][1][0][RTW89_MKK][6] = 66, - [0][0][1][0][RTW89_IC][6] = 62, - [0][0][1][0][RTW89_ACMA][6] = 60, - [0][0][1][0][RTW89_FCC][8] = 70, + [0][0][1][0][RTW89_IC][6] = 58, + [0][0][1][0][RTW89_KCC][6] = 62, + [0][0][1][0][RTW89_ACMA][6] = 66, + [0][0][1][0][RTW89_CN][6] = 50, + [0][0][1][0][RTW89_UK][6] = 66, + [0][0][1][0][RTW89_FCC][8] = 72, [0][0][1][0][RTW89_ETSI][8] = 66, [0][0][1][0][RTW89_MKK][8] = 66, - [0][0][1][0][RTW89_IC][8] = 66, - [0][0][1][0][RTW89_ACMA][8] = 60, - [0][0][1][0][RTW89_FCC][10] = 70, + [0][0][1][0][RTW89_IC][8] = 64, + [0][0][1][0][RTW89_KCC][8] = 70, + [0][0][1][0][RTW89_ACMA][8] = 66, + [0][0][1][0][RTW89_CN][8] = 50, + [0][0][1][0][RTW89_UK][8] = 66, + [0][0][1][0][RTW89_FCC][10] = 72, [0][0][1][0][RTW89_ETSI][10] = 66, [0][0][1][0][RTW89_MKK][10] = 66, - [0][0][1][0][RTW89_IC][10] = 66, - [0][0][1][0][RTW89_ACMA][10] = 60, - [0][0][1][0][RTW89_FCC][12] = 70, + [0][0][1][0][RTW89_IC][10] = 64, + [0][0][1][0][RTW89_KCC][10] = 70, + [0][0][1][0][RTW89_ACMA][10] = 66, + [0][0][1][0][RTW89_CN][10] = 50, + [0][0][1][0][RTW89_UK][10] = 66, + [0][0][1][0][RTW89_FCC][12] = 72, [0][0][1][0][RTW89_ETSI][12] = 66, [0][0][1][0][RTW89_MKK][12] = 66, - [0][0][1][0][RTW89_IC][12] = 66, - [0][0][1][0][RTW89_ACMA][12] = 60, + [0][0][1][0][RTW89_IC][12] = 64, + [0][0][1][0][RTW89_KCC][12] = 66, + [0][0][1][0][RTW89_ACMA][12] = 66, + [0][0][1][0][RTW89_CN][12] = 50, + [0][0][1][0][RTW89_UK][12] = 66, [0][0][1][0][RTW89_FCC][14] = 70, [0][0][1][0][RTW89_ETSI][14] = 66, [0][0][1][0][RTW89_MKK][14] = 66, - [0][0][1][0][RTW89_IC][14] = 66, - [0][0][1][0][RTW89_ACMA][14] = 60, - [0][0][1][0][RTW89_FCC][15] = 68, + [0][0][1][0][RTW89_IC][14] = 64, + [0][0][1][0][RTW89_KCC][14] = 66, + [0][0][1][0][RTW89_ACMA][14] = 66, + [0][0][1][0][RTW89_CN][14] = 50, + [0][0][1][0][RTW89_UK][14] = 66, + [0][0][1][0][RTW89_FCC][15] = 72, [0][0][1][0][RTW89_ETSI][15] = 66, [0][0][1][0][RTW89_MKK][15] = 70, - [0][0][1][0][RTW89_IC][15] = 70, - [0][0][1][0][RTW89_ACMA][15] = 60, - [0][0][1][0][RTW89_FCC][17] = 70, + [0][0][1][0][RTW89_IC][15] = 72, + [0][0][1][0][RTW89_KCC][15] = 70, + [0][0][1][0][RTW89_ACMA][15] = 66, + [0][0][1][0][RTW89_CN][15] = 127, + [0][0][1][0][RTW89_UK][15] = 66, + [0][0][1][0][RTW89_FCC][17] = 72, [0][0][1][0][RTW89_ETSI][17] = 66, [0][0][1][0][RTW89_MKK][17] = 70, - [0][0][1][0][RTW89_IC][17] = 70, - [0][0][1][0][RTW89_ACMA][17] = 60, - [0][0][1][0][RTW89_FCC][19] = 70, + [0][0][1][0][RTW89_IC][17] = 72, + [0][0][1][0][RTW89_KCC][17] = 70, + [0][0][1][0][RTW89_ACMA][17] = 66, + [0][0][1][0][RTW89_CN][17] = 127, + [0][0][1][0][RTW89_UK][17] = 66, + [0][0][1][0][RTW89_FCC][19] = 72, [0][0][1][0][RTW89_ETSI][19] = 66, [0][0][1][0][RTW89_MKK][19] = 70, - [0][0][1][0][RTW89_IC][19] = 70, - [0][0][1][0][RTW89_ACMA][19] = 60, - [0][0][1][0][RTW89_FCC][21] = 70, + [0][0][1][0][RTW89_IC][19] = 72, + [0][0][1][0][RTW89_KCC][19] = 70, + [0][0][1][0][RTW89_ACMA][19] = 66, + [0][0][1][0][RTW89_CN][19] = 127, + [0][0][1][0][RTW89_UK][19] = 66, + [0][0][1][0][RTW89_FCC][21] = 72, [0][0][1][0][RTW89_ETSI][21] = 66, [0][0][1][0][RTW89_MKK][21] = 70, - [0][0][1][0][RTW89_IC][21] = 70, - [0][0][1][0][RTW89_ACMA][21] = 60, - [0][0][1][0][RTW89_FCC][23] = 70, + [0][0][1][0][RTW89_IC][21] = 72, + [0][0][1][0][RTW89_KCC][21] = 70, + [0][0][1][0][RTW89_ACMA][21] = 66, + [0][0][1][0][RTW89_CN][21] = 127, + [0][0][1][0][RTW89_UK][21] = 66, + [0][0][1][0][RTW89_FCC][23] = 72, [0][0][1][0][RTW89_ETSI][23] = 66, [0][0][1][0][RTW89_MKK][23] = 70, - [0][0][1][0][RTW89_IC][23] = 70, - [0][0][1][0][RTW89_ACMA][23] = 60, - [0][0][1][0][RTW89_FCC][25] = 70, + [0][0][1][0][RTW89_IC][23] = 72, + [0][0][1][0][RTW89_KCC][23] = 70, + [0][0][1][0][RTW89_ACMA][23] = 66, + [0][0][1][0][RTW89_CN][23] = 127, + [0][0][1][0][RTW89_UK][23] = 66, + [0][0][1][0][RTW89_FCC][25] = 72, [0][0][1][0][RTW89_ETSI][25] = 66, [0][0][1][0][RTW89_MKK][25] = 70, [0][0][1][0][RTW89_IC][25] = 127, + [0][0][1][0][RTW89_KCC][25] = 70, [0][0][1][0][RTW89_ACMA][25] = 127, - [0][0][1][0][RTW89_FCC][27] = 70, + [0][0][1][0][RTW89_CN][25] = 127, + [0][0][1][0][RTW89_UK][25] = 66, + [0][0][1][0][RTW89_FCC][27] = 72, [0][0][1][0][RTW89_ETSI][27] = 66, [0][0][1][0][RTW89_MKK][27] = 70, [0][0][1][0][RTW89_IC][27] = 127, + [0][0][1][0][RTW89_KCC][27] = 70, [0][0][1][0][RTW89_ACMA][27] = 127, - [0][0][1][0][RTW89_FCC][29] = 70, + [0][0][1][0][RTW89_CN][27] = 127, + [0][0][1][0][RTW89_UK][27] = 66, + [0][0][1][0][RTW89_FCC][29] = 72, [0][0][1][0][RTW89_ETSI][29] = 66, [0][0][1][0][RTW89_MKK][29] = 70, [0][0][1][0][RTW89_IC][29] = 127, + [0][0][1][0][RTW89_KCC][29] = 70, [0][0][1][0][RTW89_ACMA][29] = 127, - [0][0][1][0][RTW89_FCC][31] = 70, + [0][0][1][0][RTW89_CN][29] = 127, + [0][0][1][0][RTW89_UK][29] = 66, + [0][0][1][0][RTW89_FCC][31] = 72, [0][0][1][0][RTW89_ETSI][31] = 66, [0][0][1][0][RTW89_MKK][31] = 70, - [0][0][1][0][RTW89_IC][31] = 70, - [0][0][1][0][RTW89_ACMA][31] = 60, - [0][0][1][0][RTW89_FCC][33] = 70, + [0][0][1][0][RTW89_IC][31] = 72, + [0][0][1][0][RTW89_KCC][31] = 70, + [0][0][1][0][RTW89_ACMA][31] = 66, + [0][0][1][0][RTW89_CN][31] = 127, + [0][0][1][0][RTW89_UK][31] = 66, + [0][0][1][0][RTW89_FCC][33] = 72, [0][0][1][0][RTW89_ETSI][33] = 66, [0][0][1][0][RTW89_MKK][33] = 70, - [0][0][1][0][RTW89_IC][33] = 70, - [0][0][1][0][RTW89_ACMA][33] = 60, - [0][0][1][0][RTW89_FCC][35] = 62, + [0][0][1][0][RTW89_IC][33] = 72, + [0][0][1][0][RTW89_KCC][33] = 70, + [0][0][1][0][RTW89_ACMA][33] = 66, + [0][0][1][0][RTW89_CN][33] = 127, + [0][0][1][0][RTW89_UK][33] = 66, + [0][0][1][0][RTW89_FCC][35] = 60, [0][0][1][0][RTW89_ETSI][35] = 66, [0][0][1][0][RTW89_MKK][35] = 70, - [0][0][1][0][RTW89_IC][35] = 70, - [0][0][1][0][RTW89_ACMA][35] = 60, - [0][0][1][0][RTW89_FCC][37] = 70, + [0][0][1][0][RTW89_IC][35] = 60, + [0][0][1][0][RTW89_KCC][35] = 70, + [0][0][1][0][RTW89_ACMA][35] = 66, + [0][0][1][0][RTW89_CN][35] = 127, + [0][0][1][0][RTW89_UK][35] = 66, + [0][0][1][0][RTW89_FCC][37] = 72, [0][0][1][0][RTW89_ETSI][37] = 127, [0][0][1][0][RTW89_MKK][37] = 70, - [0][0][1][0][RTW89_IC][37] = 70, + [0][0][1][0][RTW89_IC][37] = 72, + [0][0][1][0][RTW89_KCC][37] = 70, [0][0][1][0][RTW89_ACMA][37] = 70, - [0][0][1][0][RTW89_FCC][38] = 70, + [0][0][1][0][RTW89_CN][37] = 127, + [0][0][1][0][RTW89_UK][37] = 64, + [0][0][1][0][RTW89_FCC][38] = 72, [0][0][1][0][RTW89_ETSI][38] = 30, [0][0][1][0][RTW89_MKK][38] = 127, - [0][0][1][0][RTW89_IC][38] = 70, + [0][0][1][0][RTW89_IC][38] = 72, + [0][0][1][0][RTW89_KCC][38] = 62, [0][0][1][0][RTW89_ACMA][38] = 70, - [0][0][1][0][RTW89_FCC][40] = 70, + [0][0][1][0][RTW89_CN][38] = 68, + [0][0][1][0][RTW89_UK][38] = 64, + [0][0][1][0][RTW89_FCC][40] = 72, [0][0][1][0][RTW89_ETSI][40] = 30, [0][0][1][0][RTW89_MKK][40] = 127, - [0][0][1][0][RTW89_IC][40] = 70, + [0][0][1][0][RTW89_IC][40] = 72, + [0][0][1][0][RTW89_KCC][40] = 62, [0][0][1][0][RTW89_ACMA][40] = 70, - [0][0][1][0][RTW89_FCC][42] = 70, + [0][0][1][0][RTW89_CN][40] = 68, + [0][0][1][0][RTW89_UK][40] = 64, + [0][0][1][0][RTW89_FCC][42] = 72, [0][0][1][0][RTW89_ETSI][42] = 30, [0][0][1][0][RTW89_MKK][42] = 127, - [0][0][1][0][RTW89_IC][42] = 70, + [0][0][1][0][RTW89_IC][42] = 72, + [0][0][1][0][RTW89_KCC][42] = 62, [0][0][1][0][RTW89_ACMA][42] = 70, - [0][0][1][0][RTW89_FCC][44] = 70, + [0][0][1][0][RTW89_CN][42] = 68, + [0][0][1][0][RTW89_UK][42] = 64, + [0][0][1][0][RTW89_FCC][44] = 72, [0][0][1][0][RTW89_ETSI][44] = 30, [0][0][1][0][RTW89_MKK][44] = 127, - [0][0][1][0][RTW89_IC][44] = 70, + [0][0][1][0][RTW89_IC][44] = 72, + [0][0][1][0][RTW89_KCC][44] = 62, [0][0][1][0][RTW89_ACMA][44] = 70, - [0][0][1][0][RTW89_FCC][46] = 70, + [0][0][1][0][RTW89_CN][44] = 68, + [0][0][1][0][RTW89_UK][44] = 64, + [0][0][1][0][RTW89_FCC][46] = 72, [0][0][1][0][RTW89_ETSI][46] = 30, [0][0][1][0][RTW89_MKK][46] = 127, - [0][0][1][0][RTW89_IC][46] = 70, + [0][0][1][0][RTW89_IC][46] = 72, + [0][0][1][0][RTW89_KCC][46] = 62, [0][0][1][0][RTW89_ACMA][46] = 70, - [0][0][1][0][RTW89_FCC][48] = 70, + [0][0][1][0][RTW89_CN][46] = 68, + [0][0][1][0][RTW89_UK][46] = 64, + [0][0][1][0][RTW89_FCC][48] = 72, [0][0][1][0][RTW89_ETSI][48] = 127, [0][0][1][0][RTW89_MKK][48] = 127, [0][0][1][0][RTW89_IC][48] = 127, + [0][0][1][0][RTW89_KCC][48] = 127, [0][0][1][0][RTW89_ACMA][48] = 127, - [0][0][1][0][RTW89_FCC][50] = 70, + [0][0][1][0][RTW89_CN][48] = 127, + [0][0][1][0][RTW89_UK][48] = 127, + [0][0][1][0][RTW89_FCC][50] = 72, [0][0][1][0][RTW89_ETSI][50] = 127, [0][0][1][0][RTW89_MKK][50] = 127, [0][0][1][0][RTW89_IC][50] = 127, + [0][0][1][0][RTW89_KCC][50] = 127, [0][0][1][0][RTW89_ACMA][50] = 127, - [0][0][1][0][RTW89_FCC][52] = 70, + [0][0][1][0][RTW89_CN][50] = 127, + [0][0][1][0][RTW89_UK][50] = 127, + [0][0][1][0][RTW89_FCC][52] = 72, [0][0][1][0][RTW89_ETSI][52] = 127, [0][0][1][0][RTW89_MKK][52] = 127, [0][0][1][0][RTW89_IC][52] = 127, + [0][0][1][0][RTW89_KCC][52] = 127, [0][0][1][0][RTW89_ACMA][52] = 127, + [0][0][1][0][RTW89_CN][52] = 127, + [0][0][1][0][RTW89_UK][52] = 127, [0][1][1][0][RTW89_FCC][0] = 60, [0][1][1][0][RTW89_ETSI][0] = 54, [0][1][1][0][RTW89_MKK][0] = 54, - [0][1][1][0][RTW89_IC][0] = 42, - [0][1][1][0][RTW89_ACMA][0] = 48, + [0][1][1][0][RTW89_IC][0] = 34, + [0][1][1][0][RTW89_KCC][0] = 40, + [0][1][1][0][RTW89_ACMA][0] = 54, + [0][1][1][0][RTW89_CN][0] = 46, + [0][1][1][0][RTW89_UK][0] = 54, [0][1][1][0][RTW89_FCC][2] = 60, [0][1][1][0][RTW89_ETSI][2] = 54, [0][1][1][0][RTW89_MKK][2] = 54, - [0][1][1][0][RTW89_IC][2] = 42, - [0][1][1][0][RTW89_ACMA][2] = 48, + [0][1][1][0][RTW89_IC][2] = 34, + [0][1][1][0][RTW89_KCC][2] = 40, + [0][1][1][0][RTW89_ACMA][2] = 54, + [0][1][1][0][RTW89_CN][2] = 46, + [0][1][1][0][RTW89_UK][2] = 54, [0][1][1][0][RTW89_FCC][4] = 60, [0][1][1][0][RTW89_ETSI][4] = 54, [0][1][1][0][RTW89_MKK][4] = 54, - [0][1][1][0][RTW89_IC][4] = 42, - [0][1][1][0][RTW89_ACMA][4] = 48, + [0][1][1][0][RTW89_IC][4] = 34, + [0][1][1][0][RTW89_KCC][4] = 40, + [0][1][1][0][RTW89_ACMA][4] = 54, + [0][1][1][0][RTW89_CN][4] = 46, + [0][1][1][0][RTW89_UK][4] = 54, [0][1][1][0][RTW89_FCC][6] = 60, [0][1][1][0][RTW89_ETSI][6] = 54, [0][1][1][0][RTW89_MKK][6] = 54, - [0][1][1][0][RTW89_IC][6] = 42, - [0][1][1][0][RTW89_ACMA][6] = 48, - [0][1][1][0][RTW89_FCC][8] = 60, + [0][1][1][0][RTW89_IC][6] = 36, + [0][1][1][0][RTW89_KCC][6] = 60, + [0][1][1][0][RTW89_ACMA][6] = 54, + [0][1][1][0][RTW89_CN][6] = 46, + [0][1][1][0][RTW89_UK][6] = 54, + [0][1][1][0][RTW89_FCC][8] = 62, [0][1][1][0][RTW89_ETSI][8] = 54, [0][1][1][0][RTW89_MKK][8] = 52, - [0][1][1][0][RTW89_IC][8] = 54, - [0][1][1][0][RTW89_ACMA][8] = 48, - [0][1][1][0][RTW89_FCC][10] = 60, + [0][1][1][0][RTW89_IC][8] = 52, + [0][1][1][0][RTW89_KCC][8] = 60, + [0][1][1][0][RTW89_ACMA][8] = 54, + [0][1][1][0][RTW89_CN][8] = 46, + [0][1][1][0][RTW89_UK][8] = 54, + [0][1][1][0][RTW89_FCC][10] = 62, [0][1][1][0][RTW89_ETSI][10] = 54, [0][1][1][0][RTW89_MKK][10] = 54, - [0][1][1][0][RTW89_IC][10] = 54, - [0][1][1][0][RTW89_ACMA][10] = 48, - [0][1][1][0][RTW89_FCC][12] = 60, + [0][1][1][0][RTW89_IC][10] = 52, + [0][1][1][0][RTW89_KCC][10] = 60, + [0][1][1][0][RTW89_ACMA][10] = 54, + [0][1][1][0][RTW89_CN][10] = 46, + [0][1][1][0][RTW89_UK][10] = 54, + [0][1][1][0][RTW89_FCC][12] = 62, [0][1][1][0][RTW89_ETSI][12] = 54, [0][1][1][0][RTW89_MKK][12] = 54, - [0][1][1][0][RTW89_IC][12] = 54, - [0][1][1][0][RTW89_ACMA][12] = 48, + [0][1][1][0][RTW89_IC][12] = 52, + [0][1][1][0][RTW89_KCC][12] = 60, + [0][1][1][0][RTW89_ACMA][12] = 54, + [0][1][1][0][RTW89_CN][12] = 46, + [0][1][1][0][RTW89_UK][12] = 54, [0][1][1][0][RTW89_FCC][14] = 60, [0][1][1][0][RTW89_ETSI][14] = 54, [0][1][1][0][RTW89_MKK][14] = 54, - [0][1][1][0][RTW89_IC][14] = 54, - [0][1][1][0][RTW89_ACMA][14] = 48, - [0][1][1][0][RTW89_FCC][15] = 58, + [0][1][1][0][RTW89_IC][14] = 52, + [0][1][1][0][RTW89_KCC][14] = 60, + [0][1][1][0][RTW89_ACMA][14] = 54, + [0][1][1][0][RTW89_CN][14] = 46, + [0][1][1][0][RTW89_UK][14] = 54, + [0][1][1][0][RTW89_FCC][15] = 60, [0][1][1][0][RTW89_ETSI][15] = 54, [0][1][1][0][RTW89_MKK][15] = 70, - [0][1][1][0][RTW89_IC][15] = 68, - [0][1][1][0][RTW89_ACMA][15] = 48, + [0][1][1][0][RTW89_IC][15] = 60, + [0][1][1][0][RTW89_KCC][15] = 60, + [0][1][1][0][RTW89_ACMA][15] = 54, + [0][1][1][0][RTW89_CN][15] = 127, + [0][1][1][0][RTW89_UK][15] = 54, [0][1][1][0][RTW89_FCC][17] = 60, [0][1][1][0][RTW89_ETSI][17] = 54, [0][1][1][0][RTW89_MKK][17] = 70, - [0][1][1][0][RTW89_IC][17] = 70, - [0][1][1][0][RTW89_ACMA][17] = 48, + [0][1][1][0][RTW89_IC][17] = 60, + [0][1][1][0][RTW89_KCC][17] = 60, + [0][1][1][0][RTW89_ACMA][17] = 54, + [0][1][1][0][RTW89_CN][17] = 127, + [0][1][1][0][RTW89_UK][17] = 54, [0][1][1][0][RTW89_FCC][19] = 60, [0][1][1][0][RTW89_ETSI][19] = 54, [0][1][1][0][RTW89_MKK][19] = 70, - [0][1][1][0][RTW89_IC][19] = 70, - [0][1][1][0][RTW89_ACMA][19] = 48, + [0][1][1][0][RTW89_IC][19] = 60, + [0][1][1][0][RTW89_KCC][19] = 60, + [0][1][1][0][RTW89_ACMA][19] = 54, + [0][1][1][0][RTW89_CN][19] = 127, + [0][1][1][0][RTW89_UK][19] = 54, [0][1][1][0][RTW89_FCC][21] = 60, [0][1][1][0][RTW89_ETSI][21] = 54, [0][1][1][0][RTW89_MKK][21] = 70, - [0][1][1][0][RTW89_IC][21] = 70, - [0][1][1][0][RTW89_ACMA][21] = 48, + [0][1][1][0][RTW89_IC][21] = 60, + [0][1][1][0][RTW89_KCC][21] = 60, + [0][1][1][0][RTW89_ACMA][21] = 54, + [0][1][1][0][RTW89_CN][21] = 127, + [0][1][1][0][RTW89_UK][21] = 54, [0][1][1][0][RTW89_FCC][23] = 60, [0][1][1][0][RTW89_ETSI][23] = 54, [0][1][1][0][RTW89_MKK][23] = 70, - [0][1][1][0][RTW89_IC][23] = 70, - [0][1][1][0][RTW89_ACMA][23] = 48, + [0][1][1][0][RTW89_IC][23] = 60, + [0][1][1][0][RTW89_KCC][23] = 60, + [0][1][1][0][RTW89_ACMA][23] = 54, + [0][1][1][0][RTW89_CN][23] = 127, + [0][1][1][0][RTW89_UK][23] = 54, [0][1][1][0][RTW89_FCC][25] = 60, [0][1][1][0][RTW89_ETSI][25] = 54, [0][1][1][0][RTW89_MKK][25] = 70, [0][1][1][0][RTW89_IC][25] = 127, + [0][1][1][0][RTW89_KCC][25] = 60, [0][1][1][0][RTW89_ACMA][25] = 127, + [0][1][1][0][RTW89_CN][25] = 127, + [0][1][1][0][RTW89_UK][25] = 54, [0][1][1][0][RTW89_FCC][27] = 60, [0][1][1][0][RTW89_ETSI][27] = 54, [0][1][1][0][RTW89_MKK][27] = 70, [0][1][1][0][RTW89_IC][27] = 127, + [0][1][1][0][RTW89_KCC][27] = 60, [0][1][1][0][RTW89_ACMA][27] = 127, + [0][1][1][0][RTW89_CN][27] = 127, + [0][1][1][0][RTW89_UK][27] = 54, [0][1][1][0][RTW89_FCC][29] = 60, [0][1][1][0][RTW89_ETSI][29] = 54, [0][1][1][0][RTW89_MKK][29] = 70, [0][1][1][0][RTW89_IC][29] = 127, + [0][1][1][0][RTW89_KCC][29] = 60, [0][1][1][0][RTW89_ACMA][29] = 127, + [0][1][1][0][RTW89_CN][29] = 127, + [0][1][1][0][RTW89_UK][29] = 54, [0][1][1][0][RTW89_FCC][31] = 60, [0][1][1][0][RTW89_ETSI][31] = 54, [0][1][1][0][RTW89_MKK][31] = 70, - [0][1][1][0][RTW89_IC][31] = 70, - [0][1][1][0][RTW89_ACMA][31] = 48, + [0][1][1][0][RTW89_IC][31] = 60, + [0][1][1][0][RTW89_KCC][31] = 58, + [0][1][1][0][RTW89_ACMA][31] = 54, + [0][1][1][0][RTW89_CN][31] = 127, + [0][1][1][0][RTW89_UK][31] = 54, [0][1][1][0][RTW89_FCC][33] = 60, [0][1][1][0][RTW89_ETSI][33] = 54, [0][1][1][0][RTW89_MKK][33] = 70, - [0][1][1][0][RTW89_IC][33] = 70, - [0][1][1][0][RTW89_ACMA][33] = 48, - [0][1][1][0][RTW89_FCC][35] = 58, + [0][1][1][0][RTW89_IC][33] = 60, + [0][1][1][0][RTW89_KCC][33] = 58, + [0][1][1][0][RTW89_ACMA][33] = 54, + [0][1][1][0][RTW89_CN][33] = 127, + [0][1][1][0][RTW89_UK][33] = 54, + [0][1][1][0][RTW89_FCC][35] = 52, [0][1][1][0][RTW89_ETSI][35] = 54, [0][1][1][0][RTW89_MKK][35] = 70, - [0][1][1][0][RTW89_IC][35] = 68, - [0][1][1][0][RTW89_ACMA][35] = 48, - [0][1][1][0][RTW89_FCC][37] = 60, + [0][1][1][0][RTW89_IC][35] = 52, + [0][1][1][0][RTW89_KCC][35] = 58, + [0][1][1][0][RTW89_ACMA][35] = 54, + [0][1][1][0][RTW89_CN][35] = 127, + [0][1][1][0][RTW89_UK][35] = 54, + [0][1][1][0][RTW89_FCC][37] = 62, [0][1][1][0][RTW89_ETSI][37] = 127, [0][1][1][0][RTW89_MKK][37] = 70, - [0][1][1][0][RTW89_IC][37] = 70, - [0][1][1][0][RTW89_ACMA][37] = 70, - [0][1][1][0][RTW89_FCC][38] = 70, + [0][1][1][0][RTW89_IC][37] = 62, + [0][1][1][0][RTW89_KCC][37] = 58, + [0][1][1][0][RTW89_ACMA][37] = 64, + [0][1][1][0][RTW89_CN][37] = 127, + [0][1][1][0][RTW89_UK][37] = 52, + [0][1][1][0][RTW89_FCC][38] = 72, [0][1][1][0][RTW89_ETSI][38] = 18, [0][1][1][0][RTW89_MKK][38] = 127, - [0][1][1][0][RTW89_IC][38] = 70, + [0][1][1][0][RTW89_IC][38] = 72, + [0][1][1][0][RTW89_KCC][38] = 60, [0][1][1][0][RTW89_ACMA][38] = 70, - [0][1][1][0][RTW89_FCC][40] = 70, + [0][1][1][0][RTW89_CN][38] = 64, + [0][1][1][0][RTW89_UK][38] = 52, + [0][1][1][0][RTW89_FCC][40] = 72, [0][1][1][0][RTW89_ETSI][40] = 18, [0][1][1][0][RTW89_MKK][40] = 127, - [0][1][1][0][RTW89_IC][40] = 70, - [0][1][1][0][RTW89_ACMA][40] = 16, - [0][1][1][0][RTW89_FCC][42] = 70, + [0][1][1][0][RTW89_IC][40] = 72, + [0][1][1][0][RTW89_KCC][40] = 60, + [0][1][1][0][RTW89_ACMA][40] = 70, + [0][1][1][0][RTW89_CN][40] = 64, + [0][1][1][0][RTW89_UK][40] = 52, + [0][1][1][0][RTW89_FCC][42] = 72, [0][1][1][0][RTW89_ETSI][42] = 18, [0][1][1][0][RTW89_MKK][42] = 127, - [0][1][1][0][RTW89_IC][42] = 70, + [0][1][1][0][RTW89_IC][42] = 72, + [0][1][1][0][RTW89_KCC][42] = 60, [0][1][1][0][RTW89_ACMA][42] = 70, - [0][1][1][0][RTW89_FCC][44] = 70, + [0][1][1][0][RTW89_CN][42] = 64, + [0][1][1][0][RTW89_UK][42] = 52, + [0][1][1][0][RTW89_FCC][44] = 72, [0][1][1][0][RTW89_ETSI][44] = 18, [0][1][1][0][RTW89_MKK][44] = 127, - [0][1][1][0][RTW89_IC][44] = 70, - [0][1][1][0][RTW89_ACMA][44] = 16, - [0][1][1][0][RTW89_FCC][46] = 70, + [0][1][1][0][RTW89_IC][44] = 72, + [0][1][1][0][RTW89_KCC][44] = 60, + [0][1][1][0][RTW89_ACMA][44] = 70, + [0][1][1][0][RTW89_CN][44] = 60, + [0][1][1][0][RTW89_UK][44] = 52, + [0][1][1][0][RTW89_FCC][46] = 72, [0][1][1][0][RTW89_ETSI][46] = 18, [0][1][1][0][RTW89_MKK][46] = 127, - [0][1][1][0][RTW89_IC][46] = 70, + [0][1][1][0][RTW89_IC][46] = 72, + [0][1][1][0][RTW89_KCC][46] = 60, [0][1][1][0][RTW89_ACMA][46] = 70, + [0][1][1][0][RTW89_CN][46] = 60, + [0][1][1][0][RTW89_UK][46] = 52, [0][1][1][0][RTW89_FCC][48] = 48, [0][1][1][0][RTW89_ETSI][48] = 127, [0][1][1][0][RTW89_MKK][48] = 127, [0][1][1][0][RTW89_IC][48] = 127, + [0][1][1][0][RTW89_KCC][48] = 127, [0][1][1][0][RTW89_ACMA][48] = 127, + [0][1][1][0][RTW89_CN][48] = 127, + [0][1][1][0][RTW89_UK][48] = 127, [0][1][1][0][RTW89_FCC][50] = 48, [0][1][1][0][RTW89_ETSI][50] = 127, [0][1][1][0][RTW89_MKK][50] = 127, [0][1][1][0][RTW89_IC][50] = 127, + [0][1][1][0][RTW89_KCC][50] = 127, [0][1][1][0][RTW89_ACMA][50] = 127, + [0][1][1][0][RTW89_CN][50] = 127, + [0][1][1][0][RTW89_UK][50] = 127, [0][1][1][0][RTW89_FCC][52] = 48, [0][1][1][0][RTW89_ETSI][52] = 127, [0][1][1][0][RTW89_MKK][52] = 127, [0][1][1][0][RTW89_IC][52] = 127, + [0][1][1][0][RTW89_KCC][52] = 127, [0][1][1][0][RTW89_ACMA][52] = 127, + [0][1][1][0][RTW89_CN][52] = 127, + [0][1][1][0][RTW89_UK][52] = 127, [0][0][2][0][RTW89_FCC][0] = 70, [0][0][2][0][RTW89_ETSI][0] = 66, [0][0][2][0][RTW89_MKK][0] = 68, - [0][0][2][0][RTW89_IC][0] = 66, - [0][0][2][0][RTW89_ACMA][0] = 62, - [0][0][2][0][RTW89_FCC][2] = 70, + [0][0][2][0][RTW89_IC][0] = 60, + [0][0][2][0][RTW89_KCC][0] = 54, + [0][0][2][0][RTW89_ACMA][0] = 66, + [0][0][2][0][RTW89_CN][0] = 52, + [0][0][2][0][RTW89_UK][0] = 66, + [0][0][2][0][RTW89_FCC][2] = 72, [0][0][2][0][RTW89_ETSI][2] = 66, [0][0][2][0][RTW89_MKK][2] = 68, - [0][0][2][0][RTW89_IC][2] = 66, - [0][0][2][0][RTW89_ACMA][2] = 62, - [0][0][2][0][RTW89_FCC][4] = 70, + [0][0][2][0][RTW89_IC][2] = 60, + [0][0][2][0][RTW89_KCC][2] = 54, + [0][0][2][0][RTW89_ACMA][2] = 66, + [0][0][2][0][RTW89_CN][2] = 52, + [0][0][2][0][RTW89_UK][2] = 66, + [0][0][2][0][RTW89_FCC][4] = 72, [0][0][2][0][RTW89_ETSI][4] = 66, [0][0][2][0][RTW89_MKK][4] = 68, - [0][0][2][0][RTW89_IC][4] = 66, - [0][0][2][0][RTW89_ACMA][4] = 62, - [0][0][2][0][RTW89_FCC][6] = 70, + [0][0][2][0][RTW89_IC][4] = 60, + [0][0][2][0][RTW89_KCC][4] = 54, + [0][0][2][0][RTW89_ACMA][4] = 66, + [0][0][2][0][RTW89_CN][4] = 52, + [0][0][2][0][RTW89_UK][4] = 66, + [0][0][2][0][RTW89_FCC][6] = 72, [0][0][2][0][RTW89_ETSI][6] = 66, [0][0][2][0][RTW89_MKK][6] = 60, - [0][0][2][0][RTW89_IC][6] = 66, - [0][0][2][0][RTW89_ACMA][6] = 62, - [0][0][2][0][RTW89_FCC][8] = 70, + [0][0][2][0][RTW89_IC][6] = 60, + [0][0][2][0][RTW89_KCC][6] = 68, + [0][0][2][0][RTW89_ACMA][6] = 66, + [0][0][2][0][RTW89_CN][6] = 52, + [0][0][2][0][RTW89_UK][6] = 66, + [0][0][2][0][RTW89_FCC][8] = 72, [0][0][2][0][RTW89_ETSI][8] = 66, [0][0][2][0][RTW89_MKK][8] = 58, - [0][0][2][0][RTW89_IC][8] = 66, - [0][0][2][0][RTW89_ACMA][8] = 62, - [0][0][2][0][RTW89_FCC][10] = 70, + [0][0][2][0][RTW89_IC][8] = 64, + [0][0][2][0][RTW89_KCC][8] = 70, + [0][0][2][0][RTW89_ACMA][8] = 66, + [0][0][2][0][RTW89_CN][8] = 52, + [0][0][2][0][RTW89_UK][8] = 66, + [0][0][2][0][RTW89_FCC][10] = 72, [0][0][2][0][RTW89_ETSI][10] = 66, [0][0][2][0][RTW89_MKK][10] = 70, - [0][0][2][0][RTW89_IC][10] = 66, - [0][0][2][0][RTW89_ACMA][10] = 62, - [0][0][2][0][RTW89_FCC][12] = 70, + [0][0][2][0][RTW89_IC][10] = 64, + [0][0][2][0][RTW89_KCC][10] = 70, + [0][0][2][0][RTW89_ACMA][10] = 66, + [0][0][2][0][RTW89_CN][10] = 52, + [0][0][2][0][RTW89_UK][10] = 66, + [0][0][2][0][RTW89_FCC][12] = 72, [0][0][2][0][RTW89_ETSI][12] = 66, [0][0][2][0][RTW89_MKK][12] = 70, - [0][0][2][0][RTW89_IC][12] = 66, - [0][0][2][0][RTW89_ACMA][12] = 62, - [0][0][2][0][RTW89_FCC][14] = 70, + [0][0][2][0][RTW89_IC][12] = 64, + [0][0][2][0][RTW89_KCC][12] = 66, + [0][0][2][0][RTW89_ACMA][12] = 66, + [0][0][2][0][RTW89_CN][12] = 52, + [0][0][2][0][RTW89_UK][12] = 66, + [0][0][2][0][RTW89_FCC][14] = 68, [0][0][2][0][RTW89_ETSI][14] = 66, [0][0][2][0][RTW89_MKK][14] = 70, - [0][0][2][0][RTW89_IC][14] = 66, - [0][0][2][0][RTW89_ACMA][14] = 62, - [0][0][2][0][RTW89_FCC][15] = 66, + [0][0][2][0][RTW89_IC][14] = 64, + [0][0][2][0][RTW89_KCC][14] = 66, + [0][0][2][0][RTW89_ACMA][14] = 66, + [0][0][2][0][RTW89_CN][14] = 52, + [0][0][2][0][RTW89_UK][14] = 66, + [0][0][2][0][RTW89_FCC][15] = 70, [0][0][2][0][RTW89_ETSI][15] = 66, [0][0][2][0][RTW89_MKK][15] = 70, [0][0][2][0][RTW89_IC][15] = 70, - [0][0][2][0][RTW89_ACMA][15] = 62, - [0][0][2][0][RTW89_FCC][17] = 70, + [0][0][2][0][RTW89_KCC][15] = 70, + [0][0][2][0][RTW89_ACMA][15] = 66, + [0][0][2][0][RTW89_CN][15] = 127, + [0][0][2][0][RTW89_UK][15] = 66, + [0][0][2][0][RTW89_FCC][17] = 72, [0][0][2][0][RTW89_ETSI][17] = 66, [0][0][2][0][RTW89_MKK][17] = 70, - [0][0][2][0][RTW89_IC][17] = 70, - [0][0][2][0][RTW89_ACMA][17] = 62, - [0][0][2][0][RTW89_FCC][19] = 70, + [0][0][2][0][RTW89_IC][17] = 72, + [0][0][2][0][RTW89_KCC][17] = 70, + [0][0][2][0][RTW89_ACMA][17] = 66, + [0][0][2][0][RTW89_CN][17] = 127, + [0][0][2][0][RTW89_UK][17] = 66, + [0][0][2][0][RTW89_FCC][19] = 72, [0][0][2][0][RTW89_ETSI][19] = 66, [0][0][2][0][RTW89_MKK][19] = 70, - [0][0][2][0][RTW89_IC][19] = 70, - [0][0][2][0][RTW89_ACMA][19] = 62, - [0][0][2][0][RTW89_FCC][21] = 70, + [0][0][2][0][RTW89_IC][19] = 72, + [0][0][2][0][RTW89_KCC][19] = 70, + [0][0][2][0][RTW89_ACMA][19] = 66, + [0][0][2][0][RTW89_CN][19] = 127, + [0][0][2][0][RTW89_UK][19] = 66, + [0][0][2][0][RTW89_FCC][21] = 72, [0][0][2][0][RTW89_ETSI][21] = 66, [0][0][2][0][RTW89_MKK][21] = 70, - [0][0][2][0][RTW89_IC][21] = 70, - [0][0][2][0][RTW89_ACMA][21] = 62, - [0][0][2][0][RTW89_FCC][23] = 70, + [0][0][2][0][RTW89_IC][21] = 72, + [0][0][2][0][RTW89_KCC][21] = 70, + [0][0][2][0][RTW89_ACMA][21] = 66, + [0][0][2][0][RTW89_CN][21] = 127, + [0][0][2][0][RTW89_UK][21] = 66, + [0][0][2][0][RTW89_FCC][23] = 72, [0][0][2][0][RTW89_ETSI][23] = 66, [0][0][2][0][RTW89_MKK][23] = 70, - [0][0][2][0][RTW89_IC][23] = 70, - [0][0][2][0][RTW89_ACMA][23] = 62, - [0][0][2][0][RTW89_FCC][25] = 70, + [0][0][2][0][RTW89_IC][23] = 72, + [0][0][2][0][RTW89_KCC][23] = 70, + [0][0][2][0][RTW89_ACMA][23] = 66, + [0][0][2][0][RTW89_CN][23] = 127, + [0][0][2][0][RTW89_UK][23] = 66, + [0][0][2][0][RTW89_FCC][25] = 72, [0][0][2][0][RTW89_ETSI][25] = 66, [0][0][2][0][RTW89_MKK][25] = 70, [0][0][2][0][RTW89_IC][25] = 127, + [0][0][2][0][RTW89_KCC][25] = 70, [0][0][2][0][RTW89_ACMA][25] = 127, - [0][0][2][0][RTW89_FCC][27] = 70, + [0][0][2][0][RTW89_CN][25] = 127, + [0][0][2][0][RTW89_UK][25] = 66, + [0][0][2][0][RTW89_FCC][27] = 72, [0][0][2][0][RTW89_ETSI][27] = 66, [0][0][2][0][RTW89_MKK][27] = 70, [0][0][2][0][RTW89_IC][27] = 127, + [0][0][2][0][RTW89_KCC][27] = 70, [0][0][2][0][RTW89_ACMA][27] = 127, - [0][0][2][0][RTW89_FCC][29] = 70, + [0][0][2][0][RTW89_CN][27] = 127, + [0][0][2][0][RTW89_UK][27] = 66, + [0][0][2][0][RTW89_FCC][29] = 72, [0][0][2][0][RTW89_ETSI][29] = 66, [0][0][2][0][RTW89_MKK][29] = 70, [0][0][2][0][RTW89_IC][29] = 127, + [0][0][2][0][RTW89_KCC][29] = 70, [0][0][2][0][RTW89_ACMA][29] = 127, - [0][0][2][0][RTW89_FCC][31] = 70, + [0][0][2][0][RTW89_CN][29] = 127, + [0][0][2][0][RTW89_UK][29] = 66, + [0][0][2][0][RTW89_FCC][31] = 72, [0][0][2][0][RTW89_ETSI][31] = 66, [0][0][2][0][RTW89_MKK][31] = 70, - [0][0][2][0][RTW89_IC][31] = 70, - [0][0][2][0][RTW89_ACMA][31] = 62, - [0][0][2][0][RTW89_FCC][33] = 70, + [0][0][2][0][RTW89_IC][31] = 72, + [0][0][2][0][RTW89_KCC][31] = 70, + [0][0][2][0][RTW89_ACMA][31] = 66, + [0][0][2][0][RTW89_CN][31] = 127, + [0][0][2][0][RTW89_UK][31] = 66, + [0][0][2][0][RTW89_FCC][33] = 72, [0][0][2][0][RTW89_ETSI][33] = 66, [0][0][2][0][RTW89_MKK][33] = 70, - [0][0][2][0][RTW89_IC][33] = 70, - [0][0][2][0][RTW89_ACMA][33] = 62, - [0][0][2][0][RTW89_FCC][35] = 62, + [0][0][2][0][RTW89_IC][33] = 72, + [0][0][2][0][RTW89_KCC][33] = 70, + [0][0][2][0][RTW89_ACMA][33] = 66, + [0][0][2][0][RTW89_CN][33] = 127, + [0][0][2][0][RTW89_UK][33] = 66, + [0][0][2][0][RTW89_FCC][35] = 56, [0][0][2][0][RTW89_ETSI][35] = 66, [0][0][2][0][RTW89_MKK][35] = 70, - [0][0][2][0][RTW89_IC][35] = 70, - [0][0][2][0][RTW89_ACMA][35] = 62, - [0][0][2][0][RTW89_FCC][37] = 70, + [0][0][2][0][RTW89_IC][35] = 56, + [0][0][2][0][RTW89_KCC][35] = 70, + [0][0][2][0][RTW89_ACMA][35] = 66, + [0][0][2][0][RTW89_CN][35] = 127, + [0][0][2][0][RTW89_UK][35] = 66, + [0][0][2][0][RTW89_FCC][37] = 72, [0][0][2][0][RTW89_ETSI][37] = 127, [0][0][2][0][RTW89_MKK][37] = 70, - [0][0][2][0][RTW89_IC][37] = 70, + [0][0][2][0][RTW89_IC][37] = 72, + [0][0][2][0][RTW89_KCC][37] = 70, [0][0][2][0][RTW89_ACMA][37] = 70, - [0][0][2][0][RTW89_FCC][38] = 70, + [0][0][2][0][RTW89_CN][37] = 127, + [0][0][2][0][RTW89_UK][37] = 64, + [0][0][2][0][RTW89_FCC][38] = 72, [0][0][2][0][RTW89_ETSI][38] = 30, [0][0][2][0][RTW89_MKK][38] = 127, - [0][0][2][0][RTW89_IC][38] = 70, + [0][0][2][0][RTW89_IC][38] = 72, + [0][0][2][0][RTW89_KCC][38] = 58, [0][0][2][0][RTW89_ACMA][38] = 70, - [0][0][2][0][RTW89_FCC][40] = 70, + [0][0][2][0][RTW89_CN][38] = 68, + [0][0][2][0][RTW89_UK][38] = 64, + [0][0][2][0][RTW89_FCC][40] = 72, [0][0][2][0][RTW89_ETSI][40] = 30, [0][0][2][0][RTW89_MKK][40] = 127, - [0][0][2][0][RTW89_IC][40] = 70, + [0][0][2][0][RTW89_IC][40] = 72, + [0][0][2][0][RTW89_KCC][40] = 58, [0][0][2][0][RTW89_ACMA][40] = 70, - [0][0][2][0][RTW89_FCC][42] = 70, + [0][0][2][0][RTW89_CN][40] = 68, + [0][0][2][0][RTW89_UK][40] = 64, + [0][0][2][0][RTW89_FCC][42] = 72, [0][0][2][0][RTW89_ETSI][42] = 30, [0][0][2][0][RTW89_MKK][42] = 127, - [0][0][2][0][RTW89_IC][42] = 70, + [0][0][2][0][RTW89_IC][42] = 72, + [0][0][2][0][RTW89_KCC][42] = 58, [0][0][2][0][RTW89_ACMA][42] = 70, - [0][0][2][0][RTW89_FCC][44] = 70, + [0][0][2][0][RTW89_CN][42] = 68, + [0][0][2][0][RTW89_UK][42] = 64, + [0][0][2][0][RTW89_FCC][44] = 72, [0][0][2][0][RTW89_ETSI][44] = 30, [0][0][2][0][RTW89_MKK][44] = 127, - [0][0][2][0][RTW89_IC][44] = 70, + [0][0][2][0][RTW89_IC][44] = 72, + [0][0][2][0][RTW89_KCC][44] = 58, [0][0][2][0][RTW89_ACMA][44] = 70, - [0][0][2][0][RTW89_FCC][46] = 70, + [0][0][2][0][RTW89_CN][44] = 68, + [0][0][2][0][RTW89_UK][44] = 64, + [0][0][2][0][RTW89_FCC][46] = 72, [0][0][2][0][RTW89_ETSI][46] = 30, [0][0][2][0][RTW89_MKK][46] = 127, - [0][0][2][0][RTW89_IC][46] = 70, + [0][0][2][0][RTW89_IC][46] = 72, + [0][0][2][0][RTW89_KCC][46] = 58, [0][0][2][0][RTW89_ACMA][46] = 70, - [0][0][2][0][RTW89_FCC][48] = 70, + [0][0][2][0][RTW89_CN][46] = 68, + [0][0][2][0][RTW89_UK][46] = 64, + [0][0][2][0][RTW89_FCC][48] = 72, [0][0][2][0][RTW89_ETSI][48] = 127, [0][0][2][0][RTW89_MKK][48] = 127, [0][0][2][0][RTW89_IC][48] = 127, + [0][0][2][0][RTW89_KCC][48] = 127, [0][0][2][0][RTW89_ACMA][48] = 127, - [0][0][2][0][RTW89_FCC][50] = 70, + [0][0][2][0][RTW89_CN][48] = 127, + [0][0][2][0][RTW89_UK][48] = 127, + [0][0][2][0][RTW89_FCC][50] = 72, [0][0][2][0][RTW89_ETSI][50] = 127, [0][0][2][0][RTW89_MKK][50] = 127, [0][0][2][0][RTW89_IC][50] = 127, + [0][0][2][0][RTW89_KCC][50] = 127, [0][0][2][0][RTW89_ACMA][50] = 127, - [0][0][2][0][RTW89_FCC][52] = 70, + [0][0][2][0][RTW89_CN][50] = 127, + [0][0][2][0][RTW89_UK][50] = 127, + [0][0][2][0][RTW89_FCC][52] = 72, [0][0][2][0][RTW89_ETSI][52] = 127, [0][0][2][0][RTW89_MKK][52] = 127, [0][0][2][0][RTW89_IC][52] = 127, + [0][0][2][0][RTW89_KCC][52] = 127, [0][0][2][0][RTW89_ACMA][52] = 127, - [0][1][2][0][RTW89_FCC][0] = 62, + [0][0][2][0][RTW89_CN][52] = 127, + [0][0][2][0][RTW89_UK][52] = 127, + [0][1][2][0][RTW89_FCC][0] = 60, [0][1][2][0][RTW89_ETSI][0] = 54, [0][1][2][0][RTW89_MKK][0] = 54, - [0][1][2][0][RTW89_IC][0] = 44, - [0][1][2][0][RTW89_ACMA][0] = 50, + [0][1][2][0][RTW89_IC][0] = 36, + [0][1][2][0][RTW89_KCC][0] = 40, + [0][1][2][0][RTW89_ACMA][0] = 54, + [0][1][2][0][RTW89_CN][0] = 40, + [0][1][2][0][RTW89_UK][0] = 54, [0][1][2][0][RTW89_FCC][2] = 62, [0][1][2][0][RTW89_ETSI][2] = 54, [0][1][2][0][RTW89_MKK][2] = 54, - [0][1][2][0][RTW89_IC][2] = 44, - [0][1][2][0][RTW89_ACMA][2] = 50, + [0][1][2][0][RTW89_IC][2] = 36, + [0][1][2][0][RTW89_KCC][2] = 40, + [0][1][2][0][RTW89_ACMA][2] = 54, + [0][1][2][0][RTW89_CN][2] = 40, + [0][1][2][0][RTW89_UK][2] = 54, [0][1][2][0][RTW89_FCC][4] = 62, [0][1][2][0][RTW89_ETSI][4] = 54, [0][1][2][0][RTW89_MKK][4] = 54, - [0][1][2][0][RTW89_IC][4] = 44, - [0][1][2][0][RTW89_ACMA][4] = 50, + [0][1][2][0][RTW89_IC][4] = 36, + [0][1][2][0][RTW89_KCC][4] = 40, + [0][1][2][0][RTW89_ACMA][4] = 54, + [0][1][2][0][RTW89_CN][4] = 40, + [0][1][2][0][RTW89_UK][4] = 54, [0][1][2][0][RTW89_FCC][6] = 62, [0][1][2][0][RTW89_ETSI][6] = 54, [0][1][2][0][RTW89_MKK][6] = 50, - [0][1][2][0][RTW89_IC][6] = 44, - [0][1][2][0][RTW89_ACMA][6] = 50, + [0][1][2][0][RTW89_IC][6] = 38, + [0][1][2][0][RTW89_KCC][6] = 64, + [0][1][2][0][RTW89_ACMA][6] = 54, + [0][1][2][0][RTW89_CN][6] = 40, + [0][1][2][0][RTW89_UK][6] = 54, [0][1][2][0][RTW89_FCC][8] = 62, [0][1][2][0][RTW89_ETSI][8] = 54, [0][1][2][0][RTW89_MKK][8] = 42, - [0][1][2][0][RTW89_IC][8] = 54, - [0][1][2][0][RTW89_ACMA][8] = 50, + [0][1][2][0][RTW89_IC][8] = 52, + [0][1][2][0][RTW89_KCC][8] = 62, + [0][1][2][0][RTW89_ACMA][8] = 54, + [0][1][2][0][RTW89_CN][8] = 40, + [0][1][2][0][RTW89_UK][8] = 54, [0][1][2][0][RTW89_FCC][10] = 62, [0][1][2][0][RTW89_ETSI][10] = 54, [0][1][2][0][RTW89_MKK][10] = 54, - [0][1][2][0][RTW89_IC][10] = 54, - [0][1][2][0][RTW89_ACMA][10] = 50, + [0][1][2][0][RTW89_IC][10] = 52, + [0][1][2][0][RTW89_KCC][10] = 62, + [0][1][2][0][RTW89_ACMA][10] = 54, + [0][1][2][0][RTW89_CN][10] = 40, + [0][1][2][0][RTW89_UK][10] = 54, [0][1][2][0][RTW89_FCC][12] = 62, [0][1][2][0][RTW89_ETSI][12] = 54, [0][1][2][0][RTW89_MKK][12] = 54, - [0][1][2][0][RTW89_IC][12] = 54, - [0][1][2][0][RTW89_ACMA][12] = 50, + [0][1][2][0][RTW89_IC][12] = 52, + [0][1][2][0][RTW89_KCC][12] = 62, + [0][1][2][0][RTW89_ACMA][12] = 54, + [0][1][2][0][RTW89_CN][12] = 40, + [0][1][2][0][RTW89_UK][12] = 54, [0][1][2][0][RTW89_FCC][14] = 62, [0][1][2][0][RTW89_ETSI][14] = 54, [0][1][2][0][RTW89_MKK][14] = 54, - [0][1][2][0][RTW89_IC][14] = 54, - [0][1][2][0][RTW89_ACMA][14] = 50, + [0][1][2][0][RTW89_IC][14] = 52, + [0][1][2][0][RTW89_KCC][14] = 62, + [0][1][2][0][RTW89_ACMA][14] = 54, + [0][1][2][0][RTW89_CN][14] = 40, + [0][1][2][0][RTW89_UK][14] = 54, [0][1][2][0][RTW89_FCC][15] = 60, [0][1][2][0][RTW89_ETSI][15] = 54, [0][1][2][0][RTW89_MKK][15] = 68, - [0][1][2][0][RTW89_IC][15] = 70, - [0][1][2][0][RTW89_ACMA][15] = 50, + [0][1][2][0][RTW89_IC][15] = 60, + [0][1][2][0][RTW89_KCC][15] = 64, + [0][1][2][0][RTW89_ACMA][15] = 54, + [0][1][2][0][RTW89_CN][15] = 127, + [0][1][2][0][RTW89_UK][15] = 54, [0][1][2][0][RTW89_FCC][17] = 62, [0][1][2][0][RTW89_ETSI][17] = 54, [0][1][2][0][RTW89_MKK][17] = 68, - [0][1][2][0][RTW89_IC][17] = 70, - [0][1][2][0][RTW89_ACMA][17] = 50, + [0][1][2][0][RTW89_IC][17] = 62, + [0][1][2][0][RTW89_KCC][17] = 64, + [0][1][2][0][RTW89_ACMA][17] = 54, + [0][1][2][0][RTW89_CN][17] = 127, + [0][1][2][0][RTW89_UK][17] = 54, [0][1][2][0][RTW89_FCC][19] = 62, [0][1][2][0][RTW89_ETSI][19] = 54, [0][1][2][0][RTW89_MKK][19] = 68, - [0][1][2][0][RTW89_IC][19] = 70, - [0][1][2][0][RTW89_ACMA][19] = 50, + [0][1][2][0][RTW89_IC][19] = 62, + [0][1][2][0][RTW89_KCC][19] = 64, + [0][1][2][0][RTW89_ACMA][19] = 54, + [0][1][2][0][RTW89_CN][19] = 127, + [0][1][2][0][RTW89_UK][19] = 54, [0][1][2][0][RTW89_FCC][21] = 62, [0][1][2][0][RTW89_ETSI][21] = 54, [0][1][2][0][RTW89_MKK][21] = 68, - [0][1][2][0][RTW89_IC][21] = 70, - [0][1][2][0][RTW89_ACMA][21] = 50, + [0][1][2][0][RTW89_IC][21] = 62, + [0][1][2][0][RTW89_KCC][21] = 64, + [0][1][2][0][RTW89_ACMA][21] = 54, + [0][1][2][0][RTW89_CN][21] = 127, + [0][1][2][0][RTW89_UK][21] = 54, [0][1][2][0][RTW89_FCC][23] = 62, [0][1][2][0][RTW89_ETSI][23] = 54, [0][1][2][0][RTW89_MKK][23] = 68, - [0][1][2][0][RTW89_IC][23] = 70, - [0][1][2][0][RTW89_ACMA][23] = 50, + [0][1][2][0][RTW89_IC][23] = 62, + [0][1][2][0][RTW89_KCC][23] = 64, + [0][1][2][0][RTW89_ACMA][23] = 54, + [0][1][2][0][RTW89_CN][23] = 127, + [0][1][2][0][RTW89_UK][23] = 54, [0][1][2][0][RTW89_FCC][25] = 62, [0][1][2][0][RTW89_ETSI][25] = 54, [0][1][2][0][RTW89_MKK][25] = 68, [0][1][2][0][RTW89_IC][25] = 127, + [0][1][2][0][RTW89_KCC][25] = 64, [0][1][2][0][RTW89_ACMA][25] = 127, + [0][1][2][0][RTW89_CN][25] = 127, + [0][1][2][0][RTW89_UK][25] = 54, [0][1][2][0][RTW89_FCC][27] = 62, [0][1][2][0][RTW89_ETSI][27] = 54, [0][1][2][0][RTW89_MKK][27] = 68, [0][1][2][0][RTW89_IC][27] = 127, + [0][1][2][0][RTW89_KCC][27] = 64, [0][1][2][0][RTW89_ACMA][27] = 127, + [0][1][2][0][RTW89_CN][27] = 127, + [0][1][2][0][RTW89_UK][27] = 54, [0][1][2][0][RTW89_FCC][29] = 62, [0][1][2][0][RTW89_ETSI][29] = 54, [0][1][2][0][RTW89_MKK][29] = 68, [0][1][2][0][RTW89_IC][29] = 127, + [0][1][2][0][RTW89_KCC][29] = 64, [0][1][2][0][RTW89_ACMA][29] = 127, + [0][1][2][0][RTW89_CN][29] = 127, + [0][1][2][0][RTW89_UK][29] = 54, [0][1][2][0][RTW89_FCC][31] = 62, [0][1][2][0][RTW89_ETSI][31] = 54, [0][1][2][0][RTW89_MKK][31] = 68, - [0][1][2][0][RTW89_IC][31] = 70, - [0][1][2][0][RTW89_ACMA][31] = 50, + [0][1][2][0][RTW89_IC][31] = 62, + [0][1][2][0][RTW89_KCC][31] = 62, + [0][1][2][0][RTW89_ACMA][31] = 54, + [0][1][2][0][RTW89_CN][31] = 127, + [0][1][2][0][RTW89_UK][31] = 54, [0][1][2][0][RTW89_FCC][33] = 62, [0][1][2][0][RTW89_ETSI][33] = 54, [0][1][2][0][RTW89_MKK][33] = 68, - [0][1][2][0][RTW89_IC][33] = 70, - [0][1][2][0][RTW89_ACMA][33] = 50, - [0][1][2][0][RTW89_FCC][35] = 58, + [0][1][2][0][RTW89_IC][33] = 62, + [0][1][2][0][RTW89_KCC][33] = 62, + [0][1][2][0][RTW89_ACMA][33] = 54, + [0][1][2][0][RTW89_CN][33] = 127, + [0][1][2][0][RTW89_UK][33] = 54, + [0][1][2][0][RTW89_FCC][35] = 46, [0][1][2][0][RTW89_ETSI][35] = 54, [0][1][2][0][RTW89_MKK][35] = 68, - [0][1][2][0][RTW89_IC][35] = 68, - [0][1][2][0][RTW89_ACMA][35] = 50, - [0][1][2][0][RTW89_FCC][37] = 62, + [0][1][2][0][RTW89_IC][35] = 46, + [0][1][2][0][RTW89_KCC][35] = 62, + [0][1][2][0][RTW89_ACMA][35] = 54, + [0][1][2][0][RTW89_CN][35] = 127, + [0][1][2][0][RTW89_UK][35] = 54, + [0][1][2][0][RTW89_FCC][37] = 64, [0][1][2][0][RTW89_ETSI][37] = 127, [0][1][2][0][RTW89_MKK][37] = 68, - [0][1][2][0][RTW89_IC][37] = 70, - [0][1][2][0][RTW89_ACMA][37] = 70, - [0][1][2][0][RTW89_FCC][38] = 70, + [0][1][2][0][RTW89_IC][37] = 64, + [0][1][2][0][RTW89_KCC][37] = 62, + [0][1][2][0][RTW89_ACMA][37] = 64, + [0][1][2][0][RTW89_CN][37] = 127, + [0][1][2][0][RTW89_UK][37] = 52, + [0][1][2][0][RTW89_FCC][38] = 72, [0][1][2][0][RTW89_ETSI][38] = 18, [0][1][2][0][RTW89_MKK][38] = 127, - [0][1][2][0][RTW89_IC][38] = 70, + [0][1][2][0][RTW89_IC][38] = 72, + [0][1][2][0][RTW89_KCC][38] = 56, [0][1][2][0][RTW89_ACMA][38] = 70, - [0][1][2][0][RTW89_FCC][40] = 70, + [0][1][2][0][RTW89_CN][38] = 68, + [0][1][2][0][RTW89_UK][38] = 52, + [0][1][2][0][RTW89_FCC][40] = 72, [0][1][2][0][RTW89_ETSI][40] = 18, [0][1][2][0][RTW89_MKK][40] = 127, - [0][1][2][0][RTW89_IC][40] = 70, + [0][1][2][0][RTW89_IC][40] = 72, + [0][1][2][0][RTW89_KCC][40] = 56, [0][1][2][0][RTW89_ACMA][40] = 70, - [0][1][2][0][RTW89_FCC][42] = 70, + [0][1][2][0][RTW89_CN][40] = 68, + [0][1][2][0][RTW89_UK][40] = 52, + [0][1][2][0][RTW89_FCC][42] = 72, [0][1][2][0][RTW89_ETSI][42] = 18, [0][1][2][0][RTW89_MKK][42] = 127, - [0][1][2][0][RTW89_IC][42] = 70, + [0][1][2][0][RTW89_IC][42] = 72, + [0][1][2][0][RTW89_KCC][42] = 56, [0][1][2][0][RTW89_ACMA][42] = 70, - [0][1][2][0][RTW89_FCC][44] = 70, + [0][1][2][0][RTW89_CN][42] = 68, + [0][1][2][0][RTW89_UK][42] = 52, + [0][1][2][0][RTW89_FCC][44] = 72, [0][1][2][0][RTW89_ETSI][44] = 18, [0][1][2][0][RTW89_MKK][44] = 127, - [0][1][2][0][RTW89_IC][44] = 70, + [0][1][2][0][RTW89_IC][44] = 72, + [0][1][2][0][RTW89_KCC][44] = 56, [0][1][2][0][RTW89_ACMA][44] = 70, - [0][1][2][0][RTW89_FCC][46] = 70, + [0][1][2][0][RTW89_CN][44] = 68, + [0][1][2][0][RTW89_UK][44] = 52, + [0][1][2][0][RTW89_FCC][46] = 72, [0][1][2][0][RTW89_ETSI][46] = 18, [0][1][2][0][RTW89_MKK][46] = 127, - [0][1][2][0][RTW89_IC][46] = 70, + [0][1][2][0][RTW89_IC][46] = 72, + [0][1][2][0][RTW89_KCC][46] = 56, [0][1][2][0][RTW89_ACMA][46] = 70, - [0][1][2][0][RTW89_FCC][48] = 50, + [0][1][2][0][RTW89_CN][46] = 68, + [0][1][2][0][RTW89_UK][46] = 52, + [0][1][2][0][RTW89_FCC][48] = 48, [0][1][2][0][RTW89_ETSI][48] = 127, [0][1][2][0][RTW89_MKK][48] = 127, [0][1][2][0][RTW89_IC][48] = 127, + [0][1][2][0][RTW89_KCC][48] = 127, [0][1][2][0][RTW89_ACMA][48] = 127, + [0][1][2][0][RTW89_CN][48] = 127, + [0][1][2][0][RTW89_UK][48] = 127, [0][1][2][0][RTW89_FCC][50] = 50, [0][1][2][0][RTW89_ETSI][50] = 127, [0][1][2][0][RTW89_MKK][50] = 127, [0][1][2][0][RTW89_IC][50] = 127, + [0][1][2][0][RTW89_KCC][50] = 127, [0][1][2][0][RTW89_ACMA][50] = 127, - [0][1][2][0][RTW89_FCC][52] = 50, + [0][1][2][0][RTW89_CN][50] = 127, + [0][1][2][0][RTW89_UK][50] = 127, + [0][1][2][0][RTW89_FCC][52] = 48, [0][1][2][0][RTW89_ETSI][52] = 127, [0][1][2][0][RTW89_MKK][52] = 127, [0][1][2][0][RTW89_IC][52] = 127, + [0][1][2][0][RTW89_KCC][52] = 127, [0][1][2][0][RTW89_ACMA][52] = 127, + [0][1][2][0][RTW89_CN][52] = 127, + [0][1][2][0][RTW89_UK][52] = 127, [0][1][2][1][RTW89_FCC][0] = 60, [0][1][2][1][RTW89_ETSI][0] = 40, [0][1][2][1][RTW89_MKK][0] = 54, - [0][1][2][1][RTW89_IC][0] = 42, - [0][1][2][1][RTW89_ACMA][0] = 38, - [0][1][2][1][RTW89_FCC][2] = 60, + [0][1][2][1][RTW89_IC][0] = 40, + [0][1][2][1][RTW89_KCC][0] = 40, + [0][1][2][1][RTW89_ACMA][0] = 40, + [0][1][2][1][RTW89_CN][0] = 36, + [0][1][2][1][RTW89_UK][0] = 40, + [0][1][2][1][RTW89_FCC][2] = 62, [0][1][2][1][RTW89_ETSI][2] = 40, [0][1][2][1][RTW89_MKK][2] = 54, - [0][1][2][1][RTW89_IC][2] = 42, - [0][1][2][1][RTW89_ACMA][2] = 38, - [0][1][2][1][RTW89_FCC][4] = 60, + [0][1][2][1][RTW89_IC][2] = 40, + [0][1][2][1][RTW89_KCC][2] = 40, + [0][1][2][1][RTW89_ACMA][2] = 40, + [0][1][2][1][RTW89_CN][2] = 36, + [0][1][2][1][RTW89_UK][2] = 40, + [0][1][2][1][RTW89_FCC][4] = 62, [0][1][2][1][RTW89_ETSI][4] = 40, [0][1][2][1][RTW89_MKK][4] = 54, - [0][1][2][1][RTW89_IC][4] = 42, - [0][1][2][1][RTW89_ACMA][4] = 38, - [0][1][2][1][RTW89_FCC][6] = 60, + [0][1][2][1][RTW89_IC][4] = 40, + [0][1][2][1][RTW89_KCC][4] = 40, + [0][1][2][1][RTW89_ACMA][4] = 40, + [0][1][2][1][RTW89_CN][4] = 36, + [0][1][2][1][RTW89_UK][4] = 40, + [0][1][2][1][RTW89_FCC][6] = 62, [0][1][2][1][RTW89_ETSI][6] = 40, [0][1][2][1][RTW89_MKK][6] = 50, - [0][1][2][1][RTW89_IC][6] = 42, - [0][1][2][1][RTW89_ACMA][6] = 38, - [0][1][2][1][RTW89_FCC][8] = 60, + [0][1][2][1][RTW89_IC][6] = 40, + [0][1][2][1][RTW89_KCC][6] = 64, + [0][1][2][1][RTW89_ACMA][6] = 40, + [0][1][2][1][RTW89_CN][6] = 36, + [0][1][2][1][RTW89_UK][6] = 40, + [0][1][2][1][RTW89_FCC][8] = 62, [0][1][2][1][RTW89_ETSI][8] = 40, [0][1][2][1][RTW89_MKK][8] = 42, - [0][1][2][1][RTW89_IC][8] = 42, - [0][1][2][1][RTW89_ACMA][8] = 38, - [0][1][2][1][RTW89_FCC][10] = 60, + [0][1][2][1][RTW89_IC][8] = 40, + [0][1][2][1][RTW89_KCC][8] = 62, + [0][1][2][1][RTW89_ACMA][8] = 40, + [0][1][2][1][RTW89_CN][8] = 36, + [0][1][2][1][RTW89_UK][8] = 40, + [0][1][2][1][RTW89_FCC][10] = 62, [0][1][2][1][RTW89_ETSI][10] = 40, - [0][1][2][1][RTW89_MKK][10] = 66, - [0][1][2][1][RTW89_IC][10] = 42, - [0][1][2][1][RTW89_ACMA][10] = 38, - [0][1][2][1][RTW89_FCC][12] = 60, + [0][1][2][1][RTW89_MKK][10] = 54, + [0][1][2][1][RTW89_IC][10] = 40, + [0][1][2][1][RTW89_KCC][10] = 62, + [0][1][2][1][RTW89_ACMA][10] = 40, + [0][1][2][1][RTW89_CN][10] = 36, + [0][1][2][1][RTW89_UK][10] = 40, + [0][1][2][1][RTW89_FCC][12] = 62, [0][1][2][1][RTW89_ETSI][12] = 40, - [0][1][2][1][RTW89_MKK][12] = 66, - [0][1][2][1][RTW89_IC][12] = 42, - [0][1][2][1][RTW89_ACMA][12] = 38, - [0][1][2][1][RTW89_FCC][14] = 60, + [0][1][2][1][RTW89_MKK][12] = 54, + [0][1][2][1][RTW89_IC][12] = 40, + [0][1][2][1][RTW89_KCC][12] = 62, + [0][1][2][1][RTW89_ACMA][12] = 40, + [0][1][2][1][RTW89_CN][12] = 36, + [0][1][2][1][RTW89_UK][12] = 40, + [0][1][2][1][RTW89_FCC][14] = 62, [0][1][2][1][RTW89_ETSI][14] = 40, - [0][1][2][1][RTW89_MKK][14] = 66, - [0][1][2][1][RTW89_IC][14] = 42, - [0][1][2][1][RTW89_ACMA][14] = 38, + [0][1][2][1][RTW89_MKK][14] = 54, + [0][1][2][1][RTW89_IC][14] = 40, + [0][1][2][1][RTW89_KCC][14] = 62, + [0][1][2][1][RTW89_ACMA][14] = 40, + [0][1][2][1][RTW89_CN][14] = 36, + [0][1][2][1][RTW89_UK][14] = 40, [0][1][2][1][RTW89_FCC][15] = 60, [0][1][2][1][RTW89_ETSI][15] = 40, [0][1][2][1][RTW89_MKK][15] = 68, - [0][1][2][1][RTW89_IC][15] = 70, - [0][1][2][1][RTW89_ACMA][15] = 38, - [0][1][2][1][RTW89_FCC][17] = 60, + [0][1][2][1][RTW89_IC][15] = 60, + [0][1][2][1][RTW89_KCC][15] = 64, + [0][1][2][1][RTW89_ACMA][15] = 40, + [0][1][2][1][RTW89_CN][15] = 127, + [0][1][2][1][RTW89_UK][15] = 40, + [0][1][2][1][RTW89_FCC][17] = 62, [0][1][2][1][RTW89_ETSI][17] = 40, [0][1][2][1][RTW89_MKK][17] = 68, - [0][1][2][1][RTW89_IC][17] = 70, - [0][1][2][1][RTW89_ACMA][17] = 38, - [0][1][2][1][RTW89_FCC][19] = 60, + [0][1][2][1][RTW89_IC][17] = 62, + [0][1][2][1][RTW89_KCC][17] = 64, + [0][1][2][1][RTW89_ACMA][17] = 40, + [0][1][2][1][RTW89_CN][17] = 127, + [0][1][2][1][RTW89_UK][17] = 40, + [0][1][2][1][RTW89_FCC][19] = 62, [0][1][2][1][RTW89_ETSI][19] = 40, [0][1][2][1][RTW89_MKK][19] = 68, - [0][1][2][1][RTW89_IC][19] = 70, - [0][1][2][1][RTW89_ACMA][19] = 38, - [0][1][2][1][RTW89_FCC][21] = 60, + [0][1][2][1][RTW89_IC][19] = 62, + [0][1][2][1][RTW89_KCC][19] = 64, + [0][1][2][1][RTW89_ACMA][19] = 40, + [0][1][2][1][RTW89_CN][19] = 127, + [0][1][2][1][RTW89_UK][19] = 40, + [0][1][2][1][RTW89_FCC][21] = 62, [0][1][2][1][RTW89_ETSI][21] = 40, [0][1][2][1][RTW89_MKK][21] = 68, - [0][1][2][1][RTW89_IC][21] = 70, - [0][1][2][1][RTW89_ACMA][21] = 38, - [0][1][2][1][RTW89_FCC][23] = 60, + [0][1][2][1][RTW89_IC][21] = 62, + [0][1][2][1][RTW89_KCC][21] = 64, + [0][1][2][1][RTW89_ACMA][21] = 40, + [0][1][2][1][RTW89_CN][21] = 127, + [0][1][2][1][RTW89_UK][21] = 40, + [0][1][2][1][RTW89_FCC][23] = 62, [0][1][2][1][RTW89_ETSI][23] = 40, [0][1][2][1][RTW89_MKK][23] = 68, - [0][1][2][1][RTW89_IC][23] = 70, - [0][1][2][1][RTW89_ACMA][23] = 38, - [0][1][2][1][RTW89_FCC][25] = 58, + [0][1][2][1][RTW89_IC][23] = 62, + [0][1][2][1][RTW89_KCC][23] = 64, + [0][1][2][1][RTW89_ACMA][23] = 40, + [0][1][2][1][RTW89_CN][23] = 127, + [0][1][2][1][RTW89_UK][23] = 40, + [0][1][2][1][RTW89_FCC][25] = 46, [0][1][2][1][RTW89_ETSI][25] = 40, [0][1][2][1][RTW89_MKK][25] = 68, [0][1][2][1][RTW89_IC][25] = 127, + [0][1][2][1][RTW89_KCC][25] = 64, [0][1][2][1][RTW89_ACMA][25] = 127, - [0][1][2][1][RTW89_FCC][27] = 58, + [0][1][2][1][RTW89_CN][25] = 127, + [0][1][2][1][RTW89_UK][25] = 40, + [0][1][2][1][RTW89_FCC][27] = 46, [0][1][2][1][RTW89_ETSI][27] = 40, [0][1][2][1][RTW89_MKK][27] = 68, [0][1][2][1][RTW89_IC][27] = 127, + [0][1][2][1][RTW89_KCC][27] = 64, [0][1][2][1][RTW89_ACMA][27] = 127, - [0][1][2][1][RTW89_FCC][29] = 58, + [0][1][2][1][RTW89_CN][27] = 127, + [0][1][2][1][RTW89_UK][27] = 40, + [0][1][2][1][RTW89_FCC][29] = 46, [0][1][2][1][RTW89_ETSI][29] = 40, [0][1][2][1][RTW89_MKK][29] = 68, [0][1][2][1][RTW89_IC][29] = 127, + [0][1][2][1][RTW89_KCC][29] = 64, [0][1][2][1][RTW89_ACMA][29] = 127, - [0][1][2][1][RTW89_FCC][31] = 58, + [0][1][2][1][RTW89_CN][29] = 127, + [0][1][2][1][RTW89_UK][29] = 40, + [0][1][2][1][RTW89_FCC][31] = 46, [0][1][2][1][RTW89_ETSI][31] = 40, [0][1][2][1][RTW89_MKK][31] = 68, - [0][1][2][1][RTW89_IC][31] = 68, - [0][1][2][1][RTW89_ACMA][31] = 38, - [0][1][2][1][RTW89_FCC][33] = 58, + [0][1][2][1][RTW89_IC][31] = 46, + [0][1][2][1][RTW89_KCC][31] = 62, + [0][1][2][1][RTW89_ACMA][31] = 40, + [0][1][2][1][RTW89_CN][31] = 127, + [0][1][2][1][RTW89_UK][31] = 40, + [0][1][2][1][RTW89_FCC][33] = 46, [0][1][2][1][RTW89_ETSI][33] = 40, [0][1][2][1][RTW89_MKK][33] = 68, - [0][1][2][1][RTW89_IC][33] = 68, - [0][1][2][1][RTW89_ACMA][33] = 38, - [0][1][2][1][RTW89_FCC][35] = 58, + [0][1][2][1][RTW89_IC][33] = 46, + [0][1][2][1][RTW89_KCC][33] = 62, + [0][1][2][1][RTW89_ACMA][33] = 40, + [0][1][2][1][RTW89_CN][33] = 127, + [0][1][2][1][RTW89_UK][33] = 40, + [0][1][2][1][RTW89_FCC][35] = 46, [0][1][2][1][RTW89_ETSI][35] = 40, [0][1][2][1][RTW89_MKK][35] = 68, - [0][1][2][1][RTW89_IC][35] = 68, - [0][1][2][1][RTW89_ACMA][35] = 38, - [0][1][2][1][RTW89_FCC][37] = 60, + [0][1][2][1][RTW89_IC][35] = 46, + [0][1][2][1][RTW89_KCC][35] = 62, + [0][1][2][1][RTW89_ACMA][35] = 40, + [0][1][2][1][RTW89_CN][35] = 127, + [0][1][2][1][RTW89_UK][35] = 40, + [0][1][2][1][RTW89_FCC][37] = 64, [0][1][2][1][RTW89_ETSI][37] = 127, [0][1][2][1][RTW89_MKK][37] = 68, - [0][1][2][1][RTW89_IC][37] = 70, - [0][1][2][1][RTW89_ACMA][37] = 70, - [0][1][2][1][RTW89_FCC][38] = 70, + [0][1][2][1][RTW89_IC][37] = 64, + [0][1][2][1][RTW89_KCC][37] = 62, + [0][1][2][1][RTW89_ACMA][37] = 64, + [0][1][2][1][RTW89_CN][37] = 127, + [0][1][2][1][RTW89_UK][37] = 40, + [0][1][2][1][RTW89_FCC][38] = 72, [0][1][2][1][RTW89_ETSI][38] = 6, [0][1][2][1][RTW89_MKK][38] = 127, - [0][1][2][1][RTW89_IC][38] = 70, + [0][1][2][1][RTW89_IC][38] = 72, + [0][1][2][1][RTW89_KCC][38] = 56, [0][1][2][1][RTW89_ACMA][38] = 70, - [0][1][2][1][RTW89_FCC][40] = 70, + [0][1][2][1][RTW89_CN][38] = 60, + [0][1][2][1][RTW89_UK][38] = 40, + [0][1][2][1][RTW89_FCC][40] = 72, [0][1][2][1][RTW89_ETSI][40] = 6, [0][1][2][1][RTW89_MKK][40] = 127, - [0][1][2][1][RTW89_IC][40] = 70, + [0][1][2][1][RTW89_IC][40] = 72, + [0][1][2][1][RTW89_KCC][40] = 56, [0][1][2][1][RTW89_ACMA][40] = 70, - [0][1][2][1][RTW89_FCC][42] = 70, + [0][1][2][1][RTW89_CN][40] = 60, + [0][1][2][1][RTW89_UK][40] = 40, + [0][1][2][1][RTW89_FCC][42] = 72, [0][1][2][1][RTW89_ETSI][42] = 6, [0][1][2][1][RTW89_MKK][42] = 127, - [0][1][2][1][RTW89_IC][42] = 70, + [0][1][2][1][RTW89_IC][42] = 72, + [0][1][2][1][RTW89_KCC][42] = 56, [0][1][2][1][RTW89_ACMA][42] = 70, - [0][1][2][1][RTW89_FCC][44] = 70, + [0][1][2][1][RTW89_CN][42] = 60, + [0][1][2][1][RTW89_UK][42] = 40, + [0][1][2][1][RTW89_FCC][44] = 72, [0][1][2][1][RTW89_ETSI][44] = 6, [0][1][2][1][RTW89_MKK][44] = 127, - [0][1][2][1][RTW89_IC][44] = 70, + [0][1][2][1][RTW89_IC][44] = 72, + [0][1][2][1][RTW89_KCC][44] = 56, [0][1][2][1][RTW89_ACMA][44] = 70, - [0][1][2][1][RTW89_FCC][46] = 70, + [0][1][2][1][RTW89_CN][44] = 54, + [0][1][2][1][RTW89_UK][44] = 40, + [0][1][2][1][RTW89_FCC][46] = 72, [0][1][2][1][RTW89_ETSI][46] = 6, [0][1][2][1][RTW89_MKK][46] = 127, - [0][1][2][1][RTW89_IC][46] = 70, + [0][1][2][1][RTW89_IC][46] = 72, + [0][1][2][1][RTW89_KCC][46] = 56, [0][1][2][1][RTW89_ACMA][46] = 70, - [0][1][2][1][RTW89_FCC][48] = 50, + [0][1][2][1][RTW89_CN][46] = 54, + [0][1][2][1][RTW89_UK][46] = 40, + [0][1][2][1][RTW89_FCC][48] = 48, [0][1][2][1][RTW89_ETSI][48] = 127, [0][1][2][1][RTW89_MKK][48] = 127, [0][1][2][1][RTW89_IC][48] = 127, + [0][1][2][1][RTW89_KCC][48] = 127, [0][1][2][1][RTW89_ACMA][48] = 127, + [0][1][2][1][RTW89_CN][48] = 127, + [0][1][2][1][RTW89_UK][48] = 127, [0][1][2][1][RTW89_FCC][50] = 50, [0][1][2][1][RTW89_ETSI][50] = 127, [0][1][2][1][RTW89_MKK][50] = 127, [0][1][2][1][RTW89_IC][50] = 127, + [0][1][2][1][RTW89_KCC][50] = 127, [0][1][2][1][RTW89_ACMA][50] = 127, - [0][1][2][1][RTW89_FCC][52] = 50, + [0][1][2][1][RTW89_CN][50] = 127, + [0][1][2][1][RTW89_UK][50] = 127, + [0][1][2][1][RTW89_FCC][52] = 48, [0][1][2][1][RTW89_ETSI][52] = 127, [0][1][2][1][RTW89_MKK][52] = 127, [0][1][2][1][RTW89_IC][52] = 127, + [0][1][2][1][RTW89_KCC][52] = 127, [0][1][2][1][RTW89_ACMA][52] = 127, - [1][0][2][0][RTW89_FCC][1] = 58, + [0][1][2][1][RTW89_CN][52] = 127, + [0][1][2][1][RTW89_UK][52] = 127, + [1][0][2][0][RTW89_FCC][1] = 64, [1][0][2][0][RTW89_ETSI][1] = 66, [1][0][2][0][RTW89_MKK][1] = 66, - [1][0][2][0][RTW89_IC][1] = 66, + [1][0][2][0][RTW89_IC][1] = 62, + [1][0][2][0][RTW89_KCC][1] = 66, [1][0][2][0][RTW89_ACMA][1] = 66, + [1][0][2][0][RTW89_CN][1] = 54, + [1][0][2][0][RTW89_UK][1] = 66, [1][0][2][0][RTW89_FCC][5] = 68, [1][0][2][0][RTW89_ETSI][5] = 66, [1][0][2][0][RTW89_MKK][5] = 66, - [1][0][2][0][RTW89_IC][5] = 66, + [1][0][2][0][RTW89_IC][5] = 64, + [1][0][2][0][RTW89_KCC][5] = 54, [1][0][2][0][RTW89_ACMA][5] = 66, + [1][0][2][0][RTW89_CN][5] = 54, + [1][0][2][0][RTW89_UK][5] = 66, [1][0][2][0][RTW89_FCC][9] = 68, [1][0][2][0][RTW89_ETSI][9] = 66, [1][0][2][0][RTW89_MKK][9] = 66, - [1][0][2][0][RTW89_IC][9] = 66, + [1][0][2][0][RTW89_IC][9] = 64, + [1][0][2][0][RTW89_KCC][9] = 66, [1][0][2][0][RTW89_ACMA][9] = 66, - [1][0][2][0][RTW89_FCC][13] = 58, + [1][0][2][0][RTW89_CN][9] = 54, + [1][0][2][0][RTW89_UK][9] = 66, + [1][0][2][0][RTW89_FCC][13] = 60, [1][0][2][0][RTW89_ETSI][13] = 66, [1][0][2][0][RTW89_MKK][13] = 66, - [1][0][2][0][RTW89_IC][13] = 66, + [1][0][2][0][RTW89_IC][13] = 60, + [1][0][2][0][RTW89_KCC][13] = 52, [1][0][2][0][RTW89_ACMA][13] = 66, - [1][0][2][0][RTW89_FCC][16] = 56, + [1][0][2][0][RTW89_CN][13] = 54, + [1][0][2][0][RTW89_UK][13] = 66, + [1][0][2][0][RTW89_FCC][16] = 64, [1][0][2][0][RTW89_ETSI][16] = 66, [1][0][2][0][RTW89_MKK][16] = 66, - [1][0][2][0][RTW89_IC][16] = 66, + [1][0][2][0][RTW89_IC][16] = 64, + [1][0][2][0][RTW89_KCC][16] = 56, [1][0][2][0][RTW89_ACMA][16] = 66, + [1][0][2][0][RTW89_CN][16] = 127, + [1][0][2][0][RTW89_UK][16] = 66, [1][0][2][0][RTW89_FCC][20] = 68, [1][0][2][0][RTW89_ETSI][20] = 66, [1][0][2][0][RTW89_MKK][20] = 66, - [1][0][2][0][RTW89_IC][20] = 66, + [1][0][2][0][RTW89_IC][20] = 68, + [1][0][2][0][RTW89_KCC][20] = 56, [1][0][2][0][RTW89_ACMA][20] = 66, + [1][0][2][0][RTW89_CN][20] = 127, + [1][0][2][0][RTW89_UK][20] = 66, [1][0][2][0][RTW89_FCC][24] = 68, [1][0][2][0][RTW89_ETSI][24] = 66, [1][0][2][0][RTW89_MKK][24] = 66, [1][0][2][0][RTW89_IC][24] = 127, + [1][0][2][0][RTW89_KCC][24] = 56, [1][0][2][0][RTW89_ACMA][24] = 127, + [1][0][2][0][RTW89_CN][24] = 127, + [1][0][2][0][RTW89_UK][24] = 66, [1][0][2][0][RTW89_FCC][28] = 68, [1][0][2][0][RTW89_ETSI][28] = 66, [1][0][2][0][RTW89_MKK][28] = 66, [1][0][2][0][RTW89_IC][28] = 127, + [1][0][2][0][RTW89_KCC][28] = 66, [1][0][2][0][RTW89_ACMA][28] = 127, - [1][0][2][0][RTW89_FCC][32] = 68, + [1][0][2][0][RTW89_CN][28] = 127, + [1][0][2][0][RTW89_UK][28] = 66, + [1][0][2][0][RTW89_FCC][32] = 62, [1][0][2][0][RTW89_ETSI][32] = 66, [1][0][2][0][RTW89_MKK][32] = 66, - [1][0][2][0][RTW89_IC][32] = 66, + [1][0][2][0][RTW89_IC][32] = 62, + [1][0][2][0][RTW89_KCC][32] = 66, [1][0][2][0][RTW89_ACMA][32] = 66, + [1][0][2][0][RTW89_CN][32] = 127, + [1][0][2][0][RTW89_UK][32] = 66, [1][0][2][0][RTW89_FCC][36] = 68, [1][0][2][0][RTW89_ETSI][36] = 127, [1][0][2][0][RTW89_MKK][36] = 66, - [1][0][2][0][RTW89_IC][36] = 66, + [1][0][2][0][RTW89_IC][36] = 68, + [1][0][2][0][RTW89_KCC][36] = 66, [1][0][2][0][RTW89_ACMA][36] = 66, + [1][0][2][0][RTW89_CN][36] = 127, + [1][0][2][0][RTW89_UK][36] = 64, [1][0][2][0][RTW89_FCC][39] = 68, [1][0][2][0][RTW89_ETSI][39] = 30, [1][0][2][0][RTW89_MKK][39] = 127, - [1][0][2][0][RTW89_IC][39] = 66, + [1][0][2][0][RTW89_IC][39] = 68, + [1][0][2][0][RTW89_KCC][39] = 66, [1][0][2][0][RTW89_ACMA][39] = 66, + [1][0][2][0][RTW89_CN][39] = 62, + [1][0][2][0][RTW89_UK][39] = 64, [1][0][2][0][RTW89_FCC][43] = 68, [1][0][2][0][RTW89_ETSI][43] = 30, [1][0][2][0][RTW89_MKK][43] = 127, - [1][0][2][0][RTW89_IC][43] = 66, + [1][0][2][0][RTW89_IC][43] = 68, + [1][0][2][0][RTW89_KCC][43] = 66, [1][0][2][0][RTW89_ACMA][43] = 66, + [1][0][2][0][RTW89_CN][43] = 66, + [1][0][2][0][RTW89_UK][43] = 64, [1][0][2][0][RTW89_FCC][47] = 68, [1][0][2][0][RTW89_ETSI][47] = 127, [1][0][2][0][RTW89_MKK][47] = 127, [1][0][2][0][RTW89_IC][47] = 127, + [1][0][2][0][RTW89_KCC][47] = 127, [1][0][2][0][RTW89_ACMA][47] = 127, + [1][0][2][0][RTW89_CN][47] = 127, + [1][0][2][0][RTW89_UK][47] = 127, [1][0][2][0][RTW89_FCC][51] = 68, [1][0][2][0][RTW89_ETSI][51] = 127, [1][0][2][0][RTW89_MKK][51] = 127, [1][0][2][0][RTW89_IC][51] = 127, + [1][0][2][0][RTW89_KCC][51] = 127, [1][0][2][0][RTW89_ACMA][51] = 127, + [1][0][2][0][RTW89_CN][51] = 127, + [1][0][2][0][RTW89_UK][51] = 127, [1][1][2][0][RTW89_FCC][1] = 54, [1][1][2][0][RTW89_ETSI][1] = 54, [1][1][2][0][RTW89_MKK][1] = 48, - [1][1][2][0][RTW89_IC][1] = 60, - [1][1][2][0][RTW89_ACMA][1] = 60, + [1][1][2][0][RTW89_IC][1] = 48, + [1][1][2][0][RTW89_KCC][1] = 54, + [1][1][2][0][RTW89_ACMA][1] = 54, + [1][1][2][0][RTW89_CN][1] = 42, + [1][1][2][0][RTW89_UK][1] = 54, [1][1][2][0][RTW89_FCC][5] = 68, [1][1][2][0][RTW89_ETSI][5] = 54, [1][1][2][0][RTW89_MKK][5] = 52, - [1][1][2][0][RTW89_IC][5] = 60, - [1][1][2][0][RTW89_ACMA][5] = 60, + [1][1][2][0][RTW89_IC][5] = 48, + [1][1][2][0][RTW89_KCC][5] = 54, + [1][1][2][0][RTW89_ACMA][5] = 54, + [1][1][2][0][RTW89_CN][5] = 42, + [1][1][2][0][RTW89_UK][5] = 54, [1][1][2][0][RTW89_FCC][9] = 68, [1][1][2][0][RTW89_ETSI][9] = 54, [1][1][2][0][RTW89_MKK][9] = 52, - [1][1][2][0][RTW89_IC][9] = 60, - [1][1][2][0][RTW89_ACMA][9] = 60, + [1][1][2][0][RTW89_IC][9] = 52, + [1][1][2][0][RTW89_KCC][9] = 64, + [1][1][2][0][RTW89_ACMA][9] = 54, + [1][1][2][0][RTW89_CN][9] = 42, + [1][1][2][0][RTW89_UK][9] = 54, [1][1][2][0][RTW89_FCC][13] = 54, [1][1][2][0][RTW89_ETSI][13] = 54, [1][1][2][0][RTW89_MKK][13] = 52, - [1][1][2][0][RTW89_IC][13] = 60, - [1][1][2][0][RTW89_ACMA][13] = 60, - [1][1][2][0][RTW89_FCC][16] = 48, + [1][1][2][0][RTW89_IC][13] = 52, + [1][1][2][0][RTW89_KCC][13] = 52, + [1][1][2][0][RTW89_ACMA][13] = 54, + [1][1][2][0][RTW89_CN][13] = 42, + [1][1][2][0][RTW89_UK][13] = 54, + [1][1][2][0][RTW89_FCC][16] = 56, [1][1][2][0][RTW89_ETSI][16] = 54, [1][1][2][0][RTW89_MKK][16] = 66, - [1][1][2][0][RTW89_IC][16] = 58, - [1][1][2][0][RTW89_ACMA][16] = 60, + [1][1][2][0][RTW89_IC][16] = 56, + [1][1][2][0][RTW89_KCC][16] = 54, + [1][1][2][0][RTW89_ACMA][16] = 54, + [1][1][2][0][RTW89_CN][16] = 127, + [1][1][2][0][RTW89_UK][16] = 54, [1][1][2][0][RTW89_FCC][20] = 68, [1][1][2][0][RTW89_ETSI][20] = 54, [1][1][2][0][RTW89_MKK][20] = 66, - [1][1][2][0][RTW89_IC][20] = 66, - [1][1][2][0][RTW89_ACMA][20] = 60, + [1][1][2][0][RTW89_IC][20] = 68, + [1][1][2][0][RTW89_KCC][20] = 54, + [1][1][2][0][RTW89_ACMA][20] = 54, + [1][1][2][0][RTW89_CN][20] = 127, + [1][1][2][0][RTW89_UK][20] = 54, [1][1][2][0][RTW89_FCC][24] = 68, [1][1][2][0][RTW89_ETSI][24] = 54, [1][1][2][0][RTW89_MKK][24] = 66, [1][1][2][0][RTW89_IC][24] = 127, + [1][1][2][0][RTW89_KCC][24] = 54, [1][1][2][0][RTW89_ACMA][24] = 127, + [1][1][2][0][RTW89_CN][24] = 127, + [1][1][2][0][RTW89_UK][24] = 54, [1][1][2][0][RTW89_FCC][28] = 68, [1][1][2][0][RTW89_ETSI][28] = 54, [1][1][2][0][RTW89_MKK][28] = 66, [1][1][2][0][RTW89_IC][28] = 127, + [1][1][2][0][RTW89_KCC][28] = 66, [1][1][2][0][RTW89_ACMA][28] = 127, - [1][1][2][0][RTW89_FCC][32] = 60, + [1][1][2][0][RTW89_CN][28] = 127, + [1][1][2][0][RTW89_UK][28] = 54, + [1][1][2][0][RTW89_FCC][32] = 56, [1][1][2][0][RTW89_ETSI][32] = 54, [1][1][2][0][RTW89_MKK][32] = 66, - [1][1][2][0][RTW89_IC][32] = 66, + [1][1][2][0][RTW89_IC][32] = 56, + [1][1][2][0][RTW89_KCC][32] = 66, [1][1][2][0][RTW89_ACMA][32] = 54, + [1][1][2][0][RTW89_CN][32] = 127, + [1][1][2][0][RTW89_UK][32] = 54, [1][1][2][0][RTW89_FCC][36] = 68, [1][1][2][0][RTW89_ETSI][36] = 127, [1][1][2][0][RTW89_MKK][36] = 66, - [1][1][2][0][RTW89_IC][36] = 66, + [1][1][2][0][RTW89_IC][36] = 68, + [1][1][2][0][RTW89_KCC][36] = 66, [1][1][2][0][RTW89_ACMA][36] = 66, + [1][1][2][0][RTW89_CN][36] = 127, + [1][1][2][0][RTW89_UK][36] = 52, [1][1][2][0][RTW89_FCC][39] = 68, [1][1][2][0][RTW89_ETSI][39] = 18, [1][1][2][0][RTW89_MKK][39] = 127, - [1][1][2][0][RTW89_IC][39] = 66, + [1][1][2][0][RTW89_IC][39] = 68, + [1][1][2][0][RTW89_KCC][39] = 56, [1][1][2][0][RTW89_ACMA][39] = 66, + [1][1][2][0][RTW89_CN][39] = 62, + [1][1][2][0][RTW89_UK][39] = 52, [1][1][2][0][RTW89_FCC][43] = 68, [1][1][2][0][RTW89_ETSI][43] = 18, [1][1][2][0][RTW89_MKK][43] = 127, - [1][1][2][0][RTW89_IC][43] = 66, + [1][1][2][0][RTW89_IC][43] = 68, + [1][1][2][0][RTW89_KCC][43] = 56, [1][1][2][0][RTW89_ACMA][43] = 66, - [1][1][2][0][RTW89_FCC][47] = 60, + [1][1][2][0][RTW89_CN][43] = 66, + [1][1][2][0][RTW89_UK][43] = 52, + [1][1][2][0][RTW89_FCC][47] = 62, [1][1][2][0][RTW89_ETSI][47] = 127, [1][1][2][0][RTW89_MKK][47] = 127, [1][1][2][0][RTW89_IC][47] = 127, + [1][1][2][0][RTW89_KCC][47] = 127, [1][1][2][0][RTW89_ACMA][47] = 127, - [1][1][2][0][RTW89_FCC][51] = 58, + [1][1][2][0][RTW89_CN][47] = 127, + [1][1][2][0][RTW89_UK][47] = 127, + [1][1][2][0][RTW89_FCC][51] = 60, [1][1][2][0][RTW89_ETSI][51] = 127, [1][1][2][0][RTW89_MKK][51] = 127, [1][1][2][0][RTW89_IC][51] = 127, + [1][1][2][0][RTW89_KCC][51] = 127, [1][1][2][0][RTW89_ACMA][51] = 127, + [1][1][2][0][RTW89_CN][51] = 127, + [1][1][2][0][RTW89_UK][51] = 127, [1][1][2][1][RTW89_FCC][1] = 54, [1][1][2][1][RTW89_ETSI][1] = 40, [1][1][2][1][RTW89_MKK][1] = 48, - [1][1][2][1][RTW89_IC][1] = 48, - [1][1][2][1][RTW89_ACMA][1] = 48, - [1][1][2][1][RTW89_FCC][5] = 60, + [1][1][2][1][RTW89_IC][1] = 40, + [1][1][2][1][RTW89_KCC][1] = 54, + [1][1][2][1][RTW89_ACMA][1] = 40, + [1][1][2][1][RTW89_CN][1] = 42, + [1][1][2][1][RTW89_UK][1] = 40, + [1][1][2][1][RTW89_FCC][5] = 68, [1][1][2][1][RTW89_ETSI][5] = 40, [1][1][2][1][RTW89_MKK][5] = 52, - [1][1][2][1][RTW89_IC][5] = 48, - [1][1][2][1][RTW89_ACMA][5] = 48, - [1][1][2][1][RTW89_FCC][9] = 60, + [1][1][2][1][RTW89_IC][5] = 40, + [1][1][2][1][RTW89_KCC][5] = 54, + [1][1][2][1][RTW89_ACMA][5] = 40, + [1][1][2][1][RTW89_CN][5] = 42, + [1][1][2][1][RTW89_UK][5] = 40, + [1][1][2][1][RTW89_FCC][9] = 68, [1][1][2][1][RTW89_ETSI][9] = 40, [1][1][2][1][RTW89_MKK][9] = 52, - [1][1][2][1][RTW89_IC][9] = 48, - [1][1][2][1][RTW89_ACMA][9] = 48, + [1][1][2][1][RTW89_IC][9] = 40, + [1][1][2][1][RTW89_KCC][9] = 64, + [1][1][2][1][RTW89_ACMA][9] = 40, + [1][1][2][1][RTW89_CN][9] = 42, + [1][1][2][1][RTW89_UK][9] = 40, [1][1][2][1][RTW89_FCC][13] = 54, [1][1][2][1][RTW89_ETSI][13] = 40, [1][1][2][1][RTW89_MKK][13] = 52, - [1][1][2][1][RTW89_IC][13] = 48, - [1][1][2][1][RTW89_ACMA][13] = 48, - [1][1][2][1][RTW89_FCC][16] = 48, + [1][1][2][1][RTW89_IC][13] = 40, + [1][1][2][1][RTW89_KCC][13] = 52, + [1][1][2][1][RTW89_ACMA][13] = 40, + [1][1][2][1][RTW89_CN][13] = 42, + [1][1][2][1][RTW89_UK][13] = 40, + [1][1][2][1][RTW89_FCC][16] = 56, [1][1][2][1][RTW89_ETSI][16] = 40, [1][1][2][1][RTW89_MKK][16] = 66, - [1][1][2][1][RTW89_IC][16] = 58, - [1][1][2][1][RTW89_ACMA][16] = 48, - [1][1][2][1][RTW89_FCC][20] = 60, + [1][1][2][1][RTW89_IC][16] = 56, + [1][1][2][1][RTW89_KCC][16] = 54, + [1][1][2][1][RTW89_ACMA][16] = 40, + [1][1][2][1][RTW89_CN][16] = 127, + [1][1][2][1][RTW89_UK][16] = 40, + [1][1][2][1][RTW89_FCC][20] = 68, [1][1][2][1][RTW89_ETSI][20] = 40, [1][1][2][1][RTW89_MKK][20] = 66, - [1][1][2][1][RTW89_IC][20] = 66, - [1][1][2][1][RTW89_ACMA][20] = 48, - [1][1][2][1][RTW89_FCC][24] = 60, + [1][1][2][1][RTW89_IC][20] = 68, + [1][1][2][1][RTW89_KCC][20] = 54, + [1][1][2][1][RTW89_ACMA][20] = 40, + [1][1][2][1][RTW89_CN][20] = 127, + [1][1][2][1][RTW89_UK][20] = 40, + [1][1][2][1][RTW89_FCC][24] = 68, [1][1][2][1][RTW89_ETSI][24] = 40, [1][1][2][1][RTW89_MKK][24] = 66, [1][1][2][1][RTW89_IC][24] = 127, + [1][1][2][1][RTW89_KCC][24] = 54, [1][1][2][1][RTW89_ACMA][24] = 127, - [1][1][2][1][RTW89_FCC][28] = 60, + [1][1][2][1][RTW89_CN][24] = 127, + [1][1][2][1][RTW89_UK][24] = 40, + [1][1][2][1][RTW89_FCC][28] = 68, [1][1][2][1][RTW89_ETSI][28] = 40, [1][1][2][1][RTW89_MKK][28] = 66, [1][1][2][1][RTW89_IC][28] = 127, + [1][1][2][1][RTW89_KCC][28] = 66, [1][1][2][1][RTW89_ACMA][28] = 127, - [1][1][2][1][RTW89_FCC][32] = 60, + [1][1][2][1][RTW89_CN][28] = 127, + [1][1][2][1][RTW89_UK][28] = 40, + [1][1][2][1][RTW89_FCC][32] = 56, [1][1][2][1][RTW89_ETSI][32] = 40, [1][1][2][1][RTW89_MKK][32] = 66, - [1][1][2][1][RTW89_IC][32] = 66, - [1][1][2][1][RTW89_ACMA][32] = 42, - [1][1][2][1][RTW89_FCC][36] = 60, + [1][1][2][1][RTW89_IC][32] = 56, + [1][1][2][1][RTW89_KCC][32] = 66, + [1][1][2][1][RTW89_ACMA][32] = 40, + [1][1][2][1][RTW89_CN][32] = 127, + [1][1][2][1][RTW89_UK][32] = 40, + [1][1][2][1][RTW89_FCC][36] = 68, [1][1][2][1][RTW89_ETSI][36] = 127, [1][1][2][1][RTW89_MKK][36] = 66, - [1][1][2][1][RTW89_IC][36] = 66, + [1][1][2][1][RTW89_IC][36] = 68, + [1][1][2][1][RTW89_KCC][36] = 66, [1][1][2][1][RTW89_ACMA][36] = 66, + [1][1][2][1][RTW89_CN][36] = 127, + [1][1][2][1][RTW89_UK][36] = 40, [1][1][2][1][RTW89_FCC][39] = 68, [1][1][2][1][RTW89_ETSI][39] = 6, [1][1][2][1][RTW89_MKK][39] = 127, - [1][1][2][1][RTW89_IC][39] = 66, + [1][1][2][1][RTW89_IC][39] = 68, + [1][1][2][1][RTW89_KCC][39] = 56, [1][1][2][1][RTW89_ACMA][39] = 66, + [1][1][2][1][RTW89_CN][39] = 60, + [1][1][2][1][RTW89_UK][39] = 40, [1][1][2][1][RTW89_FCC][43] = 68, [1][1][2][1][RTW89_ETSI][43] = 6, [1][1][2][1][RTW89_MKK][43] = 127, - [1][1][2][1][RTW89_IC][43] = 66, + [1][1][2][1][RTW89_IC][43] = 68, + [1][1][2][1][RTW89_KCC][43] = 56, [1][1][2][1][RTW89_ACMA][43] = 66, - [1][1][2][1][RTW89_FCC][47] = 60, + [1][1][2][1][RTW89_CN][43] = 52, + [1][1][2][1][RTW89_UK][43] = 40, + [1][1][2][1][RTW89_FCC][47] = 62, [1][1][2][1][RTW89_ETSI][47] = 127, [1][1][2][1][RTW89_MKK][47] = 127, [1][1][2][1][RTW89_IC][47] = 127, + [1][1][2][1][RTW89_KCC][47] = 127, [1][1][2][1][RTW89_ACMA][47] = 127, - [1][1][2][1][RTW89_FCC][51] = 58, + [1][1][2][1][RTW89_CN][47] = 127, + [1][1][2][1][RTW89_UK][47] = 127, + [1][1][2][1][RTW89_FCC][51] = 60, [1][1][2][1][RTW89_ETSI][51] = 127, [1][1][2][1][RTW89_MKK][51] = 127, [1][1][2][1][RTW89_IC][51] = 127, + [1][1][2][1][RTW89_KCC][51] = 127, [1][1][2][1][RTW89_ACMA][51] = 127, - [2][0][2][0][RTW89_FCC][3] = 56, + [1][1][2][1][RTW89_CN][51] = 127, + [1][1][2][1][RTW89_UK][51] = 127, + [2][0][2][0][RTW89_FCC][3] = 58, [2][0][2][0][RTW89_ETSI][3] = 60, [2][0][2][0][RTW89_MKK][3] = 60, - [2][0][2][0][RTW89_IC][3] = 60, + [2][0][2][0][RTW89_IC][3] = 56, + [2][0][2][0][RTW89_KCC][3] = 60, [2][0][2][0][RTW89_ACMA][3] = 60, - [2][0][2][0][RTW89_FCC][11] = 58, + [2][0][2][0][RTW89_CN][3] = 54, + [2][0][2][0][RTW89_UK][3] = 60, + [2][0][2][0][RTW89_FCC][11] = 50, [2][0][2][0][RTW89_ETSI][11] = 60, [2][0][2][0][RTW89_MKK][11] = 60, - [2][0][2][0][RTW89_IC][11] = 60, + [2][0][2][0][RTW89_IC][11] = 50, + [2][0][2][0][RTW89_KCC][11] = 58, [2][0][2][0][RTW89_ACMA][11] = 60, - [2][0][2][0][RTW89_FCC][18] = 54, + [2][0][2][0][RTW89_CN][11] = 54, + [2][0][2][0][RTW89_UK][11] = 60, + [2][0][2][0][RTW89_FCC][18] = 60, [2][0][2][0][RTW89_ETSI][18] = 60, [2][0][2][0][RTW89_MKK][18] = 60, [2][0][2][0][RTW89_IC][18] = 60, + [2][0][2][0][RTW89_KCC][18] = 56, [2][0][2][0][RTW89_ACMA][18] = 60, + [2][0][2][0][RTW89_CN][18] = 127, + [2][0][2][0][RTW89_UK][18] = 60, [2][0][2][0][RTW89_FCC][26] = 62, [2][0][2][0][RTW89_ETSI][26] = 60, [2][0][2][0][RTW89_MKK][26] = 60, [2][0][2][0][RTW89_IC][26] = 127, + [2][0][2][0][RTW89_KCC][26] = 60, [2][0][2][0][RTW89_ACMA][26] = 127, + [2][0][2][0][RTW89_CN][26] = 127, + [2][0][2][0][RTW89_UK][26] = 60, [2][0][2][0][RTW89_FCC][34] = 62, [2][0][2][0][RTW89_ETSI][34] = 127, [2][0][2][0][RTW89_MKK][34] = 60, - [2][0][2][0][RTW89_IC][34] = 60, + [2][0][2][0][RTW89_IC][34] = 62, + [2][0][2][0][RTW89_KCC][34] = 60, [2][0][2][0][RTW89_ACMA][34] = 60, + [2][0][2][0][RTW89_CN][34] = 127, + [2][0][2][0][RTW89_UK][34] = 60, [2][0][2][0][RTW89_FCC][41] = 62, [2][0][2][0][RTW89_ETSI][41] = 30, [2][0][2][0][RTW89_MKK][41] = 127, - [2][0][2][0][RTW89_IC][41] = 60, + [2][0][2][0][RTW89_IC][41] = 62, + [2][0][2][0][RTW89_KCC][41] = 58, [2][0][2][0][RTW89_ACMA][41] = 60, - [2][0][2][0][RTW89_FCC][49] = 56, + [2][0][2][0][RTW89_CN][41] = 62, + [2][0][2][0][RTW89_UK][41] = 60, + [2][0][2][0][RTW89_FCC][49] = 62, [2][0][2][0][RTW89_ETSI][49] = 127, [2][0][2][0][RTW89_MKK][49] = 127, [2][0][2][0][RTW89_IC][49] = 127, + [2][0][2][0][RTW89_KCC][49] = 127, [2][0][2][0][RTW89_ACMA][49] = 127, + [2][0][2][0][RTW89_CN][49] = 127, + [2][0][2][0][RTW89_UK][49] = 127, [2][1][2][0][RTW89_FCC][3] = 48, [2][1][2][0][RTW89_ETSI][3] = 54, [2][1][2][0][RTW89_MKK][3] = 56, - [2][1][2][0][RTW89_IC][3] = 52, - [2][1][2][0][RTW89_ACMA][3] = 52, - [2][1][2][0][RTW89_FCC][11] = 54, + [2][1][2][0][RTW89_IC][3] = 46, + [2][1][2][0][RTW89_KCC][3] = 56, + [2][1][2][0][RTW89_ACMA][3] = 54, + [2][1][2][0][RTW89_CN][3] = 52, + [2][1][2][0][RTW89_UK][3] = 54, + [2][1][2][0][RTW89_FCC][11] = 38, [2][1][2][0][RTW89_ETSI][11] = 54, [2][1][2][0][RTW89_MKK][11] = 54, - [2][1][2][0][RTW89_IC][11] = 52, - [2][1][2][0][RTW89_ACMA][11] = 52, - [2][1][2][0][RTW89_FCC][18] = 48, + [2][1][2][0][RTW89_IC][11] = 38, + [2][1][2][0][RTW89_KCC][11] = 52, + [2][1][2][0][RTW89_ACMA][11] = 54, + [2][1][2][0][RTW89_CN][11] = 52, + [2][1][2][0][RTW89_UK][11] = 54, + [2][1][2][0][RTW89_FCC][18] = 50, [2][1][2][0][RTW89_ETSI][18] = 54, [2][1][2][0][RTW89_MKK][18] = 60, - [2][1][2][0][RTW89_IC][18] = 58, - [2][1][2][0][RTW89_ACMA][18] = 52, - [2][1][2][0][RTW89_FCC][26] = 62, + [2][1][2][0][RTW89_IC][18] = 50, + [2][1][2][0][RTW89_KCC][18] = 54, + [2][1][2][0][RTW89_ACMA][18] = 54, + [2][1][2][0][RTW89_CN][18] = 127, + [2][1][2][0][RTW89_UK][18] = 54, + [2][1][2][0][RTW89_FCC][26] = 52, [2][1][2][0][RTW89_ETSI][26] = 54, [2][1][2][0][RTW89_MKK][26] = 56, [2][1][2][0][RTW89_IC][26] = 127, + [2][1][2][0][RTW89_KCC][26] = 60, [2][1][2][0][RTW89_ACMA][26] = 127, + [2][1][2][0][RTW89_CN][26] = 127, + [2][1][2][0][RTW89_UK][26] = 54, [2][1][2][0][RTW89_FCC][34] = 62, [2][1][2][0][RTW89_ETSI][34] = 127, [2][1][2][0][RTW89_MKK][34] = 60, - [2][1][2][0][RTW89_IC][34] = 60, + [2][1][2][0][RTW89_IC][34] = 62, + [2][1][2][0][RTW89_KCC][34] = 60, [2][1][2][0][RTW89_ACMA][34] = 60, - [2][1][2][0][RTW89_FCC][41] = 62, + [2][1][2][0][RTW89_CN][34] = 127, + [2][1][2][0][RTW89_UK][34] = 52, + [2][1][2][0][RTW89_FCC][41] = 60, [2][1][2][0][RTW89_ETSI][41] = 18, [2][1][2][0][RTW89_MKK][41] = 127, [2][1][2][0][RTW89_IC][41] = 60, - [2][1][2][0][RTW89_ACMA][41] = 60, - [2][1][2][0][RTW89_FCC][49] = 50, + [2][1][2][0][RTW89_KCC][41] = 50, + [2][1][2][0][RTW89_ACMA][41] = 58, + [2][1][2][0][RTW89_CN][41] = 62, + [2][1][2][0][RTW89_UK][41] = 52, + [2][1][2][0][RTW89_FCC][49] = 62, [2][1][2][0][RTW89_ETSI][49] = 127, [2][1][2][0][RTW89_MKK][49] = 127, [2][1][2][0][RTW89_IC][49] = 127, + [2][1][2][0][RTW89_KCC][49] = 127, [2][1][2][0][RTW89_ACMA][49] = 127, + [2][1][2][0][RTW89_CN][49] = 127, + [2][1][2][0][RTW89_UK][49] = 127, [2][1][2][1][RTW89_FCC][3] = 48, [2][1][2][1][RTW89_ETSI][3] = 40, [2][1][2][1][RTW89_MKK][3] = 56, [2][1][2][1][RTW89_IC][3] = 40, + [2][1][2][1][RTW89_KCC][3] = 56, [2][1][2][1][RTW89_ACMA][3] = 40, - [2][1][2][1][RTW89_FCC][11] = 54, + [2][1][2][1][RTW89_CN][3] = 42, + [2][1][2][1][RTW89_UK][3] = 40, + [2][1][2][1][RTW89_FCC][11] = 38, [2][1][2][1][RTW89_ETSI][11] = 40, [2][1][2][1][RTW89_MKK][11] = 54, - [2][1][2][1][RTW89_IC][11] = 40, + [2][1][2][1][RTW89_IC][11] = 38, + [2][1][2][1][RTW89_KCC][11] = 52, [2][1][2][1][RTW89_ACMA][11] = 40, - [2][1][2][1][RTW89_FCC][18] = 48, + [2][1][2][1][RTW89_CN][11] = 42, + [2][1][2][1][RTW89_UK][11] = 40, + [2][1][2][1][RTW89_FCC][18] = 50, [2][1][2][1][RTW89_ETSI][18] = 40, [2][1][2][1][RTW89_MKK][18] = 60, - [2][1][2][1][RTW89_IC][18] = 58, + [2][1][2][1][RTW89_IC][18] = 50, + [2][1][2][1][RTW89_KCC][18] = 54, [2][1][2][1][RTW89_ACMA][18] = 40, - [2][1][2][1][RTW89_FCC][26] = 60, + [2][1][2][1][RTW89_CN][18] = 127, + [2][1][2][1][RTW89_UK][18] = 40, + [2][1][2][1][RTW89_FCC][26] = 52, [2][1][2][1][RTW89_ETSI][26] = 42, [2][1][2][1][RTW89_MKK][26] = 56, [2][1][2][1][RTW89_IC][26] = 127, + [2][1][2][1][RTW89_KCC][26] = 60, [2][1][2][1][RTW89_ACMA][26] = 127, - [2][1][2][1][RTW89_FCC][34] = 60, + [2][1][2][1][RTW89_CN][26] = 127, + [2][1][2][1][RTW89_UK][26] = 42, + [2][1][2][1][RTW89_FCC][34] = 62, [2][1][2][1][RTW89_ETSI][34] = 127, [2][1][2][1][RTW89_MKK][34] = 60, - [2][1][2][1][RTW89_IC][34] = 60, + [2][1][2][1][RTW89_IC][34] = 62, + [2][1][2][1][RTW89_KCC][34] = 60, [2][1][2][1][RTW89_ACMA][34] = 60, - [2][1][2][1][RTW89_FCC][41] = 62, + [2][1][2][1][RTW89_CN][34] = 127, + [2][1][2][1][RTW89_UK][34] = 40, + [2][1][2][1][RTW89_FCC][41] = 60, [2][1][2][1][RTW89_ETSI][41] = 6, [2][1][2][1][RTW89_MKK][41] = 127, [2][1][2][1][RTW89_IC][41] = 60, - [2][1][2][1][RTW89_ACMA][41] = 60, - [2][1][2][1][RTW89_FCC][49] = 50, + [2][1][2][1][RTW89_KCC][41] = 50, + [2][1][2][1][RTW89_ACMA][41] = 58, + [2][1][2][1][RTW89_CN][41] = 40, + [2][1][2][1][RTW89_UK][41] = 40, + [2][1][2][1][RTW89_FCC][49] = 62, [2][1][2][1][RTW89_ETSI][49] = 127, [2][1][2][1][RTW89_MKK][49] = 127, [2][1][2][1][RTW89_IC][49] = 127, + [2][1][2][1][RTW89_KCC][49] = 127, [2][1][2][1][RTW89_ACMA][49] = 127, - [3][0][2][0][RTW89_FCC][7] = 38, + [2][1][2][1][RTW89_CN][49] = 127, + [2][1][2][1][RTW89_UK][49] = 127, + [3][0][2][0][RTW89_FCC][7] = 40, [3][0][2][0][RTW89_ETSI][7] = 50, [3][0][2][0][RTW89_MKK][7] = 50, - [3][0][2][0][RTW89_IC][7] = 50, - [3][0][2][0][RTW89_ACMA][7] = 50, - [3][0][2][0][RTW89_FCC][22] = 52, + [3][0][2][0][RTW89_IC][7] = 40, + [3][0][2][0][RTW89_KCC][7] = 44, + [3][0][2][0][RTW89_ACMA][7] = 127, + [3][0][2][0][RTW89_CN][7] = 66, + [3][0][2][0][RTW89_UK][7] = 127, + [3][0][2][0][RTW89_FCC][22] = 42, [3][0][2][0][RTW89_ETSI][22] = 50, [3][0][2][0][RTW89_MKK][22] = 50, - [3][0][2][0][RTW89_IC][22] = 50, - [3][0][2][0][RTW89_ACMA][22] = 50, - [3][0][2][0][RTW89_FCC][45] = 127, + [3][0][2][0][RTW89_IC][22] = 127, + [3][0][2][0][RTW89_KCC][22] = 50, + [3][0][2][0][RTW89_ACMA][22] = 127, + [3][0][2][0][RTW89_CN][22] = 66, + [3][0][2][0][RTW89_UK][22] = 127, + [3][0][2][0][RTW89_FCC][45] = 52, [3][0][2][0][RTW89_ETSI][45] = 127, [3][0][2][0][RTW89_MKK][45] = 127, [3][0][2][0][RTW89_IC][45] = 127, + [3][0][2][0][RTW89_KCC][45] = 127, [3][0][2][0][RTW89_ACMA][45] = 127, - [3][1][2][0][RTW89_FCC][7] = 26, + [3][0][2][0][RTW89_CN][45] = 127, + [3][0][2][0][RTW89_UK][45] = 127, + [3][1][2][0][RTW89_FCC][7] = 32, [3][1][2][0][RTW89_ETSI][7] = 50, [3][1][2][0][RTW89_MKK][7] = 36, [3][1][2][0][RTW89_IC][7] = 44, - [3][1][2][0][RTW89_ACMA][7] = 44, - [3][1][2][0][RTW89_FCC][22] = 42, + [3][1][2][0][RTW89_KCC][7] = 50, + [3][1][2][0][RTW89_ACMA][7] = 127, + [3][1][2][0][RTW89_CN][7] = 54, + [3][1][2][0][RTW89_UK][7] = 127, + [3][1][2][0][RTW89_FCC][22] = 36, [3][1][2][0][RTW89_ETSI][22] = 50, [3][1][2][0][RTW89_MKK][22] = 48, - [3][1][2][0][RTW89_IC][22] = 44, - [3][1][2][0][RTW89_ACMA][22] = 44, - [3][1][2][0][RTW89_FCC][45] = 127, + [3][1][2][0][RTW89_IC][22] = 127, + [3][1][2][0][RTW89_KCC][22] = 50, + [3][1][2][0][RTW89_ACMA][22] = 127, + [3][1][2][0][RTW89_CN][22] = 54, + [3][1][2][0][RTW89_UK][22] = 127, + [3][1][2][0][RTW89_FCC][45] = 46, [3][1][2][0][RTW89_ETSI][45] = 127, [3][1][2][0][RTW89_MKK][45] = 127, [3][1][2][0][RTW89_IC][45] = 127, + [3][1][2][0][RTW89_KCC][45] = 127, [3][1][2][0][RTW89_ACMA][45] = 127, - [3][1][2][1][RTW89_FCC][7] = 14, + [3][1][2][0][RTW89_CN][45] = 127, + [3][1][2][0][RTW89_UK][45] = 127, + [3][1][2][1][RTW89_FCC][7] = 32, [3][1][2][1][RTW89_ETSI][7] = 42, [3][1][2][1][RTW89_MKK][7] = 36, - [3][1][2][1][RTW89_IC][7] = 32, - [3][1][2][1][RTW89_ACMA][7] = 32, - [3][1][2][1][RTW89_FCC][22] = 30, + [3][1][2][1][RTW89_IC][7] = 44, + [3][1][2][1][RTW89_KCC][7] = 50, + [3][1][2][1][RTW89_ACMA][7] = 127, + [3][1][2][1][RTW89_CN][7] = 42, + [3][1][2][1][RTW89_UK][7] = 127, + [3][1][2][1][RTW89_FCC][22] = 36, [3][1][2][1][RTW89_ETSI][22] = 42, [3][1][2][1][RTW89_MKK][22] = 48, - [3][1][2][1][RTW89_IC][22] = 32, - [3][1][2][1][RTW89_ACMA][22] = 32, - [3][1][2][1][RTW89_FCC][45] = 127, + [3][1][2][1][RTW89_IC][22] = 127, + [3][1][2][1][RTW89_KCC][22] = 50, + [3][1][2][1][RTW89_ACMA][22] = 127, + [3][1][2][1][RTW89_CN][22] = 42, + [3][1][2][1][RTW89_UK][22] = 127, + [3][1][2][1][RTW89_FCC][45] = 46, [3][1][2][1][RTW89_ETSI][45] = 127, [3][1][2][1][RTW89_MKK][45] = 127, [3][1][2][1][RTW89_IC][45] = 127, + [3][1][2][1][RTW89_KCC][45] = 127, [3][1][2][1][RTW89_ACMA][45] = 127, + [3][1][2][1][RTW89_CN][45] = 127, + [3][1][2][1][RTW89_UK][45] = 127, }; const s8 rtw89_8852c_txpwr_lmt_6g[RTW89_6G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = { - [0][0][1][0][RTW89_WW][0] = 72, - [0][0][1][0][RTW89_WW][2] = 72, - [0][0][1][0][RTW89_WW][4] = 72, - [0][0][1][0][RTW89_WW][6] = 72, - [0][0][1][0][RTW89_WW][8] = 72, - [0][0][1][0][RTW89_WW][10] = 72, - [0][0][1][0][RTW89_WW][12] = 72, - [0][0][1][0][RTW89_WW][14] = 72, - [0][0][1][0][RTW89_WW][15] = 72, - [0][0][1][0][RTW89_WW][17] = 72, - [0][0][1][0][RTW89_WW][19] = 72, - [0][0][1][0][RTW89_WW][21] = 72, - [0][0][1][0][RTW89_WW][23] = 72, - [0][0][1][0][RTW89_WW][25] = 72, - [0][0][1][0][RTW89_WW][27] = 72, - [0][0][1][0][RTW89_WW][29] = 72, - [0][0][1][0][RTW89_WW][30] = 72, - [0][0][1][0][RTW89_WW][32] = 72, - [0][0][1][0][RTW89_WW][34] = 72, - [0][0][1][0][RTW89_WW][36] = 72, - [0][0][1][0][RTW89_WW][38] = 72, - [0][0][1][0][RTW89_WW][40] = 72, - [0][0][1][0][RTW89_WW][42] = 72, - [0][0][1][0][RTW89_WW][44] = 72, - [0][0][1][0][RTW89_WW][45] = 72, - [0][0][1][0][RTW89_WW][47] = 72, - [0][0][1][0][RTW89_WW][49] = 72, - [0][0][1][0][RTW89_WW][51] = 72, - [0][0][1][0][RTW89_WW][53] = 72, - [0][0][1][0][RTW89_WW][55] = 72, - [0][0][1][0][RTW89_WW][57] = 72, - [0][0][1][0][RTW89_WW][59] = 72, - [0][0][1][0][RTW89_WW][60] = 72, - [0][0][1][0][RTW89_WW][62] = 72, - [0][0][1][0][RTW89_WW][64] = 72, - [0][0][1][0][RTW89_WW][66] = 72, - [0][0][1][0][RTW89_WW][68] = 72, - [0][0][1][0][RTW89_WW][70] = 72, - [0][0][1][0][RTW89_WW][72] = 72, - [0][0][1][0][RTW89_WW][74] = 72, - [0][0][1][0][RTW89_WW][75] = 72, - [0][0][1][0][RTW89_WW][77] = 72, - [0][0][1][0][RTW89_WW][79] = 72, - [0][0][1][0][RTW89_WW][81] = 72, - [0][0][1][0][RTW89_WW][83] = 72, - [0][0][1][0][RTW89_WW][85] = 72, - [0][0][1][0][RTW89_WW][87] = 72, - [0][0][1][0][RTW89_WW][89] = 72, - [0][0][1][0][RTW89_WW][90] = 72, - [0][0][1][0][RTW89_WW][92] = 72, - [0][0][1][0][RTW89_WW][94] = 72, - [0][0][1][0][RTW89_WW][96] = 72, - [0][0][1][0][RTW89_WW][98] = 72, - [0][0][1][0][RTW89_WW][100] = 72, - [0][0][1][0][RTW89_WW][102] = 72, - [0][0][1][0][RTW89_WW][104] = 72, - [0][0][1][0][RTW89_WW][105] = 72, - [0][0][1][0][RTW89_WW][107] = 72, - [0][0][1][0][RTW89_WW][109] = 72, + [0][0][1][0][RTW89_WW][0] = 24, + [0][0][1][0][RTW89_WW][2] = 22, + [0][0][1][0][RTW89_WW][4] = 22, + [0][0][1][0][RTW89_WW][6] = 22, + [0][0][1][0][RTW89_WW][8] = 22, + [0][0][1][0][RTW89_WW][10] = 22, + [0][0][1][0][RTW89_WW][12] = 22, + [0][0][1][0][RTW89_WW][14] = 22, + [0][0][1][0][RTW89_WW][15] = 22, + [0][0][1][0][RTW89_WW][17] = 22, + [0][0][1][0][RTW89_WW][19] = 22, + [0][0][1][0][RTW89_WW][21] = 22, + [0][0][1][0][RTW89_WW][23] = 22, + [0][0][1][0][RTW89_WW][25] = 22, + [0][0][1][0][RTW89_WW][27] = 22, + [0][0][1][0][RTW89_WW][29] = 22, + [0][0][1][0][RTW89_WW][30] = 22, + [0][0][1][0][RTW89_WW][32] = 22, + [0][0][1][0][RTW89_WW][34] = 22, + [0][0][1][0][RTW89_WW][36] = 22, + [0][0][1][0][RTW89_WW][38] = 22, + [0][0][1][0][RTW89_WW][40] = 22, + [0][0][1][0][RTW89_WW][42] = 22, + [0][0][1][0][RTW89_WW][44] = 22, + [0][0][1][0][RTW89_WW][45] = 22, + [0][0][1][0][RTW89_WW][47] = 22, + [0][0][1][0][RTW89_WW][49] = 24, + [0][0][1][0][RTW89_WW][51] = 22, + [0][0][1][0][RTW89_WW][53] = 22, + [0][0][1][0][RTW89_WW][55] = 22, + [0][0][1][0][RTW89_WW][57] = 22, + [0][0][1][0][RTW89_WW][59] = 22, + [0][0][1][0][RTW89_WW][60] = 22, + [0][0][1][0][RTW89_WW][62] = 22, + [0][0][1][0][RTW89_WW][64] = 22, + [0][0][1][0][RTW89_WW][66] = 22, + [0][0][1][0][RTW89_WW][68] = 22, + [0][0][1][0][RTW89_WW][70] = 24, + [0][0][1][0][RTW89_WW][72] = 22, + [0][0][1][0][RTW89_WW][74] = 22, + [0][0][1][0][RTW89_WW][75] = 22, + [0][0][1][0][RTW89_WW][77] = 22, + [0][0][1][0][RTW89_WW][79] = 22, + [0][0][1][0][RTW89_WW][81] = 22, + [0][0][1][0][RTW89_WW][83] = 22, + [0][0][1][0][RTW89_WW][85] = 22, + [0][0][1][0][RTW89_WW][87] = 22, + [0][0][1][0][RTW89_WW][89] = 22, + [0][0][1][0][RTW89_WW][90] = 22, + [0][0][1][0][RTW89_WW][92] = 22, + [0][0][1][0][RTW89_WW][94] = 22, + [0][0][1][0][RTW89_WW][96] = 22, + [0][0][1][0][RTW89_WW][98] = 22, + [0][0][1][0][RTW89_WW][100] = 22, + [0][0][1][0][RTW89_WW][102] = 22, + [0][0][1][0][RTW89_WW][104] = 22, + [0][0][1][0][RTW89_WW][105] = 22, + [0][0][1][0][RTW89_WW][107] = 24, + [0][0][1][0][RTW89_WW][109] = 24, [0][0][1][0][RTW89_WW][111] = 0, [0][0][1][0][RTW89_WW][113] = 0, [0][0][1][0][RTW89_WW][115] = 0, [0][0][1][0][RTW89_WW][117] = 0, [0][0][1][0][RTW89_WW][119] = 0, - [0][1][1][0][RTW89_WW][0] = 60, - [0][1][1][0][RTW89_WW][2] = 60, - [0][1][1][0][RTW89_WW][4] = 60, - [0][1][1][0][RTW89_WW][6] = 60, - [0][1][1][0][RTW89_WW][8] = 60, - [0][1][1][0][RTW89_WW][10] = 60, - [0][1][1][0][RTW89_WW][12] = 60, - [0][1][1][0][RTW89_WW][14] = 60, - [0][1][1][0][RTW89_WW][15] = 60, - [0][1][1][0][RTW89_WW][17] = 60, - [0][1][1][0][RTW89_WW][19] = 60, - [0][1][1][0][RTW89_WW][21] = 60, - [0][1][1][0][RTW89_WW][23] = 60, - [0][1][1][0][RTW89_WW][25] = 60, - [0][1][1][0][RTW89_WW][27] = 60, - [0][1][1][0][RTW89_WW][29] = 60, - [0][1][1][0][RTW89_WW][30] = 60, - [0][1][1][0][RTW89_WW][32] = 60, - [0][1][1][0][RTW89_WW][34] = 60, - [0][1][1][0][RTW89_WW][36] = 60, - [0][1][1][0][RTW89_WW][38] = 60, - [0][1][1][0][RTW89_WW][40] = 60, - [0][1][1][0][RTW89_WW][42] = 60, - [0][1][1][0][RTW89_WW][44] = 60, - [0][1][1][0][RTW89_WW][45] = 60, - [0][1][1][0][RTW89_WW][47] = 60, - [0][1][1][0][RTW89_WW][49] = 60, - [0][1][1][0][RTW89_WW][51] = 60, - [0][1][1][0][RTW89_WW][53] = 60, - [0][1][1][0][RTW89_WW][55] = 60, - [0][1][1][0][RTW89_WW][57] = 60, - [0][1][1][0][RTW89_WW][59] = 60, - [0][1][1][0][RTW89_WW][60] = 60, - [0][1][1][0][RTW89_WW][62] = 60, - [0][1][1][0][RTW89_WW][64] = 60, - [0][1][1][0][RTW89_WW][66] = 60, - [0][1][1][0][RTW89_WW][68] = 60, - [0][1][1][0][RTW89_WW][70] = 60, - [0][1][1][0][RTW89_WW][72] = 60, - [0][1][1][0][RTW89_WW][74] = 60, - [0][1][1][0][RTW89_WW][75] = 60, - [0][1][1][0][RTW89_WW][77] = 60, - [0][1][1][0][RTW89_WW][79] = 60, - [0][1][1][0][RTW89_WW][81] = 60, - [0][1][1][0][RTW89_WW][83] = 60, - [0][1][1][0][RTW89_WW][85] = 60, - [0][1][1][0][RTW89_WW][87] = 60, - [0][1][1][0][RTW89_WW][89] = 60, - [0][1][1][0][RTW89_WW][90] = 60, - [0][1][1][0][RTW89_WW][92] = 60, - [0][1][1][0][RTW89_WW][94] = 60, - [0][1][1][0][RTW89_WW][96] = 60, - [0][1][1][0][RTW89_WW][98] = 60, - [0][1][1][0][RTW89_WW][100] = 60, - [0][1][1][0][RTW89_WW][102] = 60, - [0][1][1][0][RTW89_WW][104] = 60, - [0][1][1][0][RTW89_WW][105] = 60, - [0][1][1][0][RTW89_WW][107] = 60, - [0][1][1][0][RTW89_WW][109] = 60, + [0][1][1][0][RTW89_WW][0] = -2, + [0][1][1][0][RTW89_WW][2] = -4, + [0][1][1][0][RTW89_WW][4] = -4, + [0][1][1][0][RTW89_WW][6] = -4, + [0][1][1][0][RTW89_WW][8] = -4, + [0][1][1][0][RTW89_WW][10] = -4, + [0][1][1][0][RTW89_WW][12] = -4, + [0][1][1][0][RTW89_WW][14] = -4, + [0][1][1][0][RTW89_WW][15] = -4, + [0][1][1][0][RTW89_WW][17] = -4, + [0][1][1][0][RTW89_WW][19] = -4, + [0][1][1][0][RTW89_WW][21] = -4, + [0][1][1][0][RTW89_WW][23] = -4, + [0][1][1][0][RTW89_WW][25] = -4, + [0][1][1][0][RTW89_WW][27] = -4, + [0][1][1][0][RTW89_WW][29] = -4, + [0][1][1][0][RTW89_WW][30] = -4, + [0][1][1][0][RTW89_WW][32] = -4, + [0][1][1][0][RTW89_WW][34] = -4, + [0][1][1][0][RTW89_WW][36] = -4, + [0][1][1][0][RTW89_WW][38] = -4, + [0][1][1][0][RTW89_WW][40] = -4, + [0][1][1][0][RTW89_WW][42] = -4, + [0][1][1][0][RTW89_WW][44] = -2, + [0][1][1][0][RTW89_WW][45] = -2, + [0][1][1][0][RTW89_WW][47] = -2, + [0][1][1][0][RTW89_WW][49] = -2, + [0][1][1][0][RTW89_WW][51] = -2, + [0][1][1][0][RTW89_WW][53] = -2, + [0][1][1][0][RTW89_WW][55] = -2, + [0][1][1][0][RTW89_WW][57] = -2, + [0][1][1][0][RTW89_WW][59] = -2, + [0][1][1][0][RTW89_WW][60] = -2, + [0][1][1][0][RTW89_WW][62] = -2, + [0][1][1][0][RTW89_WW][64] = -2, + [0][1][1][0][RTW89_WW][66] = -2, + [0][1][1][0][RTW89_WW][68] = -2, + [0][1][1][0][RTW89_WW][70] = -2, + [0][1][1][0][RTW89_WW][72] = -2, + [0][1][1][0][RTW89_WW][74] = -2, + [0][1][1][0][RTW89_WW][75] = -2, + [0][1][1][0][RTW89_WW][77] = -2, + [0][1][1][0][RTW89_WW][79] = -2, + [0][1][1][0][RTW89_WW][81] = -2, + [0][1][1][0][RTW89_WW][83] = -2, + [0][1][1][0][RTW89_WW][85] = -2, + [0][1][1][0][RTW89_WW][87] = -2, + [0][1][1][0][RTW89_WW][89] = -2, + [0][1][1][0][RTW89_WW][90] = -2, + [0][1][1][0][RTW89_WW][92] = -2, + [0][1][1][0][RTW89_WW][94] = -2, + [0][1][1][0][RTW89_WW][96] = -2, + [0][1][1][0][RTW89_WW][98] = -2, + [0][1][1][0][RTW89_WW][100] = -2, + [0][1][1][0][RTW89_WW][102] = -2, + [0][1][1][0][RTW89_WW][104] = -2, + [0][1][1][0][RTW89_WW][105] = -2, + [0][1][1][0][RTW89_WW][107] = 1, + [0][1][1][0][RTW89_WW][109] = 1, [0][1][1][0][RTW89_WW][111] = 0, [0][1][1][0][RTW89_WW][113] = 0, [0][1][1][0][RTW89_WW][115] = 0, [0][1][1][0][RTW89_WW][117] = 0, [0][1][1][0][RTW89_WW][119] = 0, - [0][0][2][0][RTW89_WW][0] = 72, - [0][0][2][0][RTW89_WW][2] = 72, - [0][0][2][0][RTW89_WW][4] = 72, - [0][0][2][0][RTW89_WW][6] = 72, - [0][0][2][0][RTW89_WW][8] = 72, - [0][0][2][0][RTW89_WW][10] = 72, - [0][0][2][0][RTW89_WW][12] = 72, - [0][0][2][0][RTW89_WW][14] = 72, - [0][0][2][0][RTW89_WW][15] = 72, - [0][0][2][0][RTW89_WW][17] = 72, - [0][0][2][0][RTW89_WW][19] = 72, - [0][0][2][0][RTW89_WW][21] = 72, - [0][0][2][0][RTW89_WW][23] = 72, - [0][0][2][0][RTW89_WW][25] = 72, - [0][0][2][0][RTW89_WW][27] = 72, - [0][0][2][0][RTW89_WW][29] = 72, - [0][0][2][0][RTW89_WW][30] = 72, - [0][0][2][0][RTW89_WW][32] = 72, - [0][0][2][0][RTW89_WW][34] = 72, - [0][0][2][0][RTW89_WW][36] = 72, - [0][0][2][0][RTW89_WW][38] = 72, - [0][0][2][0][RTW89_WW][40] = 72, - [0][0][2][0][RTW89_WW][42] = 72, - [0][0][2][0][RTW89_WW][44] = 72, - [0][0][2][0][RTW89_WW][45] = 72, - [0][0][2][0][RTW89_WW][47] = 72, - [0][0][2][0][RTW89_WW][49] = 72, - [0][0][2][0][RTW89_WW][51] = 72, - [0][0][2][0][RTW89_WW][53] = 72, - [0][0][2][0][RTW89_WW][55] = 72, - [0][0][2][0][RTW89_WW][57] = 72, - [0][0][2][0][RTW89_WW][59] = 72, - [0][0][2][0][RTW89_WW][60] = 72, - [0][0][2][0][RTW89_WW][62] = 72, - [0][0][2][0][RTW89_WW][64] = 72, - [0][0][2][0][RTW89_WW][66] = 72, - [0][0][2][0][RTW89_WW][68] = 72, - [0][0][2][0][RTW89_WW][70] = 72, - [0][0][2][0][RTW89_WW][72] = 72, - [0][0][2][0][RTW89_WW][74] = 72, - [0][0][2][0][RTW89_WW][75] = 72, - [0][0][2][0][RTW89_WW][77] = 72, - [0][0][2][0][RTW89_WW][79] = 72, - [0][0][2][0][RTW89_WW][81] = 72, - [0][0][2][0][RTW89_WW][83] = 72, - [0][0][2][0][RTW89_WW][85] = 72, - [0][0][2][0][RTW89_WW][87] = 72, - [0][0][2][0][RTW89_WW][89] = 72, - [0][0][2][0][RTW89_WW][90] = 72, - [0][0][2][0][RTW89_WW][92] = 72, - [0][0][2][0][RTW89_WW][94] = 72, - [0][0][2][0][RTW89_WW][96] = 72, - [0][0][2][0][RTW89_WW][98] = 72, - [0][0][2][0][RTW89_WW][100] = 72, - [0][0][2][0][RTW89_WW][102] = 72, - [0][0][2][0][RTW89_WW][104] = 72, - [0][0][2][0][RTW89_WW][105] = 72, - [0][0][2][0][RTW89_WW][107] = 72, - [0][0][2][0][RTW89_WW][109] = 72, + [0][0][2][0][RTW89_WW][0] = 24, + [0][0][2][0][RTW89_WW][2] = 22, + [0][0][2][0][RTW89_WW][4] = 22, + [0][0][2][0][RTW89_WW][6] = 22, + [0][0][2][0][RTW89_WW][8] = 22, + [0][0][2][0][RTW89_WW][10] = 22, + [0][0][2][0][RTW89_WW][12] = 22, + [0][0][2][0][RTW89_WW][14] = 22, + [0][0][2][0][RTW89_WW][15] = 22, + [0][0][2][0][RTW89_WW][17] = 22, + [0][0][2][0][RTW89_WW][19] = 22, + [0][0][2][0][RTW89_WW][21] = 22, + [0][0][2][0][RTW89_WW][23] = 22, + [0][0][2][0][RTW89_WW][25] = 22, + [0][0][2][0][RTW89_WW][27] = 22, + [0][0][2][0][RTW89_WW][29] = 22, + [0][0][2][0][RTW89_WW][30] = 22, + [0][0][2][0][RTW89_WW][32] = 22, + [0][0][2][0][RTW89_WW][34] = 22, + [0][0][2][0][RTW89_WW][36] = 22, + [0][0][2][0][RTW89_WW][38] = 22, + [0][0][2][0][RTW89_WW][40] = 22, + [0][0][2][0][RTW89_WW][42] = 22, + [0][0][2][0][RTW89_WW][44] = 22, + [0][0][2][0][RTW89_WW][45] = 22, + [0][0][2][0][RTW89_WW][47] = 22, + [0][0][2][0][RTW89_WW][49] = 24, + [0][0][2][0][RTW89_WW][51] = 22, + [0][0][2][0][RTW89_WW][53] = 22, + [0][0][2][0][RTW89_WW][55] = 22, + [0][0][2][0][RTW89_WW][57] = 22, + [0][0][2][0][RTW89_WW][59] = 22, + [0][0][2][0][RTW89_WW][60] = 22, + [0][0][2][0][RTW89_WW][62] = 22, + [0][0][2][0][RTW89_WW][64] = 22, + [0][0][2][0][RTW89_WW][66] = 22, + [0][0][2][0][RTW89_WW][68] = 22, + [0][0][2][0][RTW89_WW][70] = 24, + [0][0][2][0][RTW89_WW][72] = 22, + [0][0][2][0][RTW89_WW][74] = 22, + [0][0][2][0][RTW89_WW][75] = 22, + [0][0][2][0][RTW89_WW][77] = 22, + [0][0][2][0][RTW89_WW][79] = 22, + [0][0][2][0][RTW89_WW][81] = 22, + [0][0][2][0][RTW89_WW][83] = 22, + [0][0][2][0][RTW89_WW][85] = 22, + [0][0][2][0][RTW89_WW][87] = 22, + [0][0][2][0][RTW89_WW][89] = 22, + [0][0][2][0][RTW89_WW][90] = 22, + [0][0][2][0][RTW89_WW][92] = 22, + [0][0][2][0][RTW89_WW][94] = 22, + [0][0][2][0][RTW89_WW][96] = 22, + [0][0][2][0][RTW89_WW][98] = 22, + [0][0][2][0][RTW89_WW][100] = 22, + [0][0][2][0][RTW89_WW][102] = 22, + [0][0][2][0][RTW89_WW][104] = 22, + [0][0][2][0][RTW89_WW][105] = 22, + [0][0][2][0][RTW89_WW][107] = 24, + [0][0][2][0][RTW89_WW][109] = 24, [0][0][2][0][RTW89_WW][111] = 0, [0][0][2][0][RTW89_WW][113] = 0, [0][0][2][0][RTW89_WW][115] = 0, [0][0][2][0][RTW89_WW][117] = 0, [0][0][2][0][RTW89_WW][119] = 0, - [0][1][2][0][RTW89_WW][0] = 60, - [0][1][2][0][RTW89_WW][2] = 60, - [0][1][2][0][RTW89_WW][4] = 60, - [0][1][2][0][RTW89_WW][6] = 60, - [0][1][2][0][RTW89_WW][8] = 60, - [0][1][2][0][RTW89_WW][10] = 60, - [0][1][2][0][RTW89_WW][12] = 60, - [0][1][2][0][RTW89_WW][14] = 60, - [0][1][2][0][RTW89_WW][15] = 60, - [0][1][2][0][RTW89_WW][17] = 60, - [0][1][2][0][RTW89_WW][19] = 60, - [0][1][2][0][RTW89_WW][21] = 60, - [0][1][2][0][RTW89_WW][23] = 60, - [0][1][2][0][RTW89_WW][25] = 60, - [0][1][2][0][RTW89_WW][27] = 60, - [0][1][2][0][RTW89_WW][29] = 60, - [0][1][2][0][RTW89_WW][30] = 60, - [0][1][2][0][RTW89_WW][32] = 60, - [0][1][2][0][RTW89_WW][34] = 60, - [0][1][2][0][RTW89_WW][36] = 60, - [0][1][2][0][RTW89_WW][38] = 60, - [0][1][2][0][RTW89_WW][40] = 60, - [0][1][2][0][RTW89_WW][42] = 60, - [0][1][2][0][RTW89_WW][44] = 60, - [0][1][2][0][RTW89_WW][45] = 60, - [0][1][2][0][RTW89_WW][47] = 60, - [0][1][2][0][RTW89_WW][49] = 60, - [0][1][2][0][RTW89_WW][51] = 60, - [0][1][2][0][RTW89_WW][53] = 60, - [0][1][2][0][RTW89_WW][55] = 60, - [0][1][2][0][RTW89_WW][57] = 60, - [0][1][2][0][RTW89_WW][59] = 60, - [0][1][2][0][RTW89_WW][60] = 60, - [0][1][2][0][RTW89_WW][62] = 60, - [0][1][2][0][RTW89_WW][64] = 60, - [0][1][2][0][RTW89_WW][66] = 60, - [0][1][2][0][RTW89_WW][68] = 60, - [0][1][2][0][RTW89_WW][70] = 60, - [0][1][2][0][RTW89_WW][72] = 60, - [0][1][2][0][RTW89_WW][74] = 60, - [0][1][2][0][RTW89_WW][75] = 60, - [0][1][2][0][RTW89_WW][77] = 60, - [0][1][2][0][RTW89_WW][79] = 60, - [0][1][2][0][RTW89_WW][81] = 60, - [0][1][2][0][RTW89_WW][83] = 60, - [0][1][2][0][RTW89_WW][85] = 60, - [0][1][2][0][RTW89_WW][87] = 60, - [0][1][2][0][RTW89_WW][89] = 60, - [0][1][2][0][RTW89_WW][90] = 60, - [0][1][2][0][RTW89_WW][92] = 60, - [0][1][2][0][RTW89_WW][94] = 60, - [0][1][2][0][RTW89_WW][96] = 60, - [0][1][2][0][RTW89_WW][98] = 60, - [0][1][2][0][RTW89_WW][100] = 60, - [0][1][2][0][RTW89_WW][102] = 60, - [0][1][2][0][RTW89_WW][104] = 60, - [0][1][2][0][RTW89_WW][105] = 60, - [0][1][2][0][RTW89_WW][107] = 60, - [0][1][2][0][RTW89_WW][109] = 60, + [0][1][2][0][RTW89_WW][0] = -2, + [0][1][2][0][RTW89_WW][2] = -4, + [0][1][2][0][RTW89_WW][4] = -4, + [0][1][2][0][RTW89_WW][6] = -4, + [0][1][2][0][RTW89_WW][8] = -4, + [0][1][2][0][RTW89_WW][10] = -4, + [0][1][2][0][RTW89_WW][12] = -4, + [0][1][2][0][RTW89_WW][14] = -4, + [0][1][2][0][RTW89_WW][15] = -4, + [0][1][2][0][RTW89_WW][17] = -4, + [0][1][2][0][RTW89_WW][19] = -4, + [0][1][2][0][RTW89_WW][21] = -4, + [0][1][2][0][RTW89_WW][23] = -4, + [0][1][2][0][RTW89_WW][25] = -4, + [0][1][2][0][RTW89_WW][27] = -4, + [0][1][2][0][RTW89_WW][29] = -4, + [0][1][2][0][RTW89_WW][30] = -4, + [0][1][2][0][RTW89_WW][32] = -4, + [0][1][2][0][RTW89_WW][34] = -4, + [0][1][2][0][RTW89_WW][36] = -4, + [0][1][2][0][RTW89_WW][38] = -4, + [0][1][2][0][RTW89_WW][40] = -4, + [0][1][2][0][RTW89_WW][42] = -4, + [0][1][2][0][RTW89_WW][44] = -2, + [0][1][2][0][RTW89_WW][45] = -2, + [0][1][2][0][RTW89_WW][47] = -2, + [0][1][2][0][RTW89_WW][49] = -2, + [0][1][2][0][RTW89_WW][51] = -2, + [0][1][2][0][RTW89_WW][53] = -2, + [0][1][2][0][RTW89_WW][55] = -2, + [0][1][2][0][RTW89_WW][57] = -2, + [0][1][2][0][RTW89_WW][59] = -2, + [0][1][2][0][RTW89_WW][60] = -2, + [0][1][2][0][RTW89_WW][62] = -2, + [0][1][2][0][RTW89_WW][64] = -2, + [0][1][2][0][RTW89_WW][66] = -2, + [0][1][2][0][RTW89_WW][68] = -2, + [0][1][2][0][RTW89_WW][70] = -2, + [0][1][2][0][RTW89_WW][72] = -2, + [0][1][2][0][RTW89_WW][74] = -2, + [0][1][2][0][RTW89_WW][75] = -2, + [0][1][2][0][RTW89_WW][77] = -2, + [0][1][2][0][RTW89_WW][79] = -2, + [0][1][2][0][RTW89_WW][81] = -2, + [0][1][2][0][RTW89_WW][83] = -2, + [0][1][2][0][RTW89_WW][85] = -2, + [0][1][2][0][RTW89_WW][87] = -2, + [0][1][2][0][RTW89_WW][89] = -2, + [0][1][2][0][RTW89_WW][90] = -2, + [0][1][2][0][RTW89_WW][92] = -2, + [0][1][2][0][RTW89_WW][94] = -2, + [0][1][2][0][RTW89_WW][96] = -2, + [0][1][2][0][RTW89_WW][98] = -2, + [0][1][2][0][RTW89_WW][100] = -2, + [0][1][2][0][RTW89_WW][102] = -2, + [0][1][2][0][RTW89_WW][104] = -2, + [0][1][2][0][RTW89_WW][105] = -2, + [0][1][2][0][RTW89_WW][107] = 1, + [0][1][2][0][RTW89_WW][109] = 1, [0][1][2][0][RTW89_WW][111] = 0, [0][1][2][0][RTW89_WW][113] = 0, [0][1][2][0][RTW89_WW][115] = 0, [0][1][2][0][RTW89_WW][117] = 0, [0][1][2][0][RTW89_WW][119] = 0, - [0][1][2][1][RTW89_WW][0] = 48, - [0][1][2][1][RTW89_WW][2] = 48, - [0][1][2][1][RTW89_WW][4] = 48, - [0][1][2][1][RTW89_WW][6] = 48, - [0][1][2][1][RTW89_WW][8] = 48, - [0][1][2][1][RTW89_WW][10] = 48, - [0][1][2][1][RTW89_WW][12] = 48, - [0][1][2][1][RTW89_WW][14] = 48, - [0][1][2][1][RTW89_WW][15] = 48, - [0][1][2][1][RTW89_WW][17] = 48, - [0][1][2][1][RTW89_WW][19] = 48, - [0][1][2][1][RTW89_WW][21] = 48, - [0][1][2][1][RTW89_WW][23] = 48, - [0][1][2][1][RTW89_WW][25] = 48, - [0][1][2][1][RTW89_WW][27] = 48, - [0][1][2][1][RTW89_WW][29] = 48, - [0][1][2][1][RTW89_WW][30] = 48, - [0][1][2][1][RTW89_WW][32] = 48, - [0][1][2][1][RTW89_WW][34] = 48, - [0][1][2][1][RTW89_WW][36] = 48, - [0][1][2][1][RTW89_WW][38] = 48, - [0][1][2][1][RTW89_WW][40] = 48, - [0][1][2][1][RTW89_WW][42] = 48, - [0][1][2][1][RTW89_WW][44] = 48, - [0][1][2][1][RTW89_WW][45] = 48, - [0][1][2][1][RTW89_WW][47] = 48, - [0][1][2][1][RTW89_WW][49] = 48, - [0][1][2][1][RTW89_WW][51] = 48, - [0][1][2][1][RTW89_WW][53] = 48, - [0][1][2][1][RTW89_WW][55] = 48, - [0][1][2][1][RTW89_WW][57] = 48, - [0][1][2][1][RTW89_WW][59] = 48, - [0][1][2][1][RTW89_WW][60] = 48, - [0][1][2][1][RTW89_WW][62] = 48, - [0][1][2][1][RTW89_WW][64] = 48, - [0][1][2][1][RTW89_WW][66] = 48, - [0][1][2][1][RTW89_WW][68] = 48, - [0][1][2][1][RTW89_WW][70] = 48, - [0][1][2][1][RTW89_WW][72] = 48, - [0][1][2][1][RTW89_WW][74] = 48, - [0][1][2][1][RTW89_WW][75] = 48, - [0][1][2][1][RTW89_WW][77] = 48, - [0][1][2][1][RTW89_WW][79] = 48, - [0][1][2][1][RTW89_WW][81] = 48, - [0][1][2][1][RTW89_WW][83] = 48, - [0][1][2][1][RTW89_WW][85] = 48, - [0][1][2][1][RTW89_WW][87] = 48, - [0][1][2][1][RTW89_WW][89] = 48, - [0][1][2][1][RTW89_WW][90] = 48, - [0][1][2][1][RTW89_WW][92] = 48, - [0][1][2][1][RTW89_WW][94] = 48, - [0][1][2][1][RTW89_WW][96] = 48, - [0][1][2][1][RTW89_WW][98] = 48, - [0][1][2][1][RTW89_WW][100] = 48, - [0][1][2][1][RTW89_WW][102] = 48, - [0][1][2][1][RTW89_WW][104] = 48, - [0][1][2][1][RTW89_WW][105] = 48, - [0][1][2][1][RTW89_WW][107] = 48, - [0][1][2][1][RTW89_WW][109] = 48, + [0][1][2][1][RTW89_WW][0] = -2, + [0][1][2][1][RTW89_WW][2] = -4, + [0][1][2][1][RTW89_WW][4] = -4, + [0][1][2][1][RTW89_WW][6] = -4, + [0][1][2][1][RTW89_WW][8] = -4, + [0][1][2][1][RTW89_WW][10] = -4, + [0][1][2][1][RTW89_WW][12] = -4, + [0][1][2][1][RTW89_WW][14] = -4, + [0][1][2][1][RTW89_WW][15] = -4, + [0][1][2][1][RTW89_WW][17] = -4, + [0][1][2][1][RTW89_WW][19] = -4, + [0][1][2][1][RTW89_WW][21] = -4, + [0][1][2][1][RTW89_WW][23] = -4, + [0][1][2][1][RTW89_WW][25] = -4, + [0][1][2][1][RTW89_WW][27] = -4, + [0][1][2][1][RTW89_WW][29] = -4, + [0][1][2][1][RTW89_WW][30] = -4, + [0][1][2][1][RTW89_WW][32] = -4, + [0][1][2][1][RTW89_WW][34] = -4, + [0][1][2][1][RTW89_WW][36] = -4, + [0][1][2][1][RTW89_WW][38] = -4, + [0][1][2][1][RTW89_WW][40] = -4, + [0][1][2][1][RTW89_WW][42] = -4, + [0][1][2][1][RTW89_WW][44] = -2, + [0][1][2][1][RTW89_WW][45] = -2, + [0][1][2][1][RTW89_WW][47] = -2, + [0][1][2][1][RTW89_WW][49] = -2, + [0][1][2][1][RTW89_WW][51] = -2, + [0][1][2][1][RTW89_WW][53] = -2, + [0][1][2][1][RTW89_WW][55] = -2, + [0][1][2][1][RTW89_WW][57] = -2, + [0][1][2][1][RTW89_WW][59] = -2, + [0][1][2][1][RTW89_WW][60] = -2, + [0][1][2][1][RTW89_WW][62] = -2, + [0][1][2][1][RTW89_WW][64] = -2, + [0][1][2][1][RTW89_WW][66] = -2, + [0][1][2][1][RTW89_WW][68] = -2, + [0][1][2][1][RTW89_WW][70] = -2, + [0][1][2][1][RTW89_WW][72] = -2, + [0][1][2][1][RTW89_WW][74] = -2, + [0][1][2][1][RTW89_WW][75] = -2, + [0][1][2][1][RTW89_WW][77] = -2, + [0][1][2][1][RTW89_WW][79] = -2, + [0][1][2][1][RTW89_WW][81] = -2, + [0][1][2][1][RTW89_WW][83] = -2, + [0][1][2][1][RTW89_WW][85] = -2, + [0][1][2][1][RTW89_WW][87] = -2, + [0][1][2][1][RTW89_WW][89] = -2, + [0][1][2][1][RTW89_WW][90] = -2, + [0][1][2][1][RTW89_WW][92] = -2, + [0][1][2][1][RTW89_WW][94] = -2, + [0][1][2][1][RTW89_WW][96] = -2, + [0][1][2][1][RTW89_WW][98] = -2, + [0][1][2][1][RTW89_WW][100] = -2, + [0][1][2][1][RTW89_WW][102] = -2, + [0][1][2][1][RTW89_WW][104] = -2, + [0][1][2][1][RTW89_WW][105] = -2, + [0][1][2][1][RTW89_WW][107] = 1, + [0][1][2][1][RTW89_WW][109] = 1, [0][1][2][1][RTW89_WW][111] = 0, [0][1][2][1][RTW89_WW][113] = 0, [0][1][2][1][RTW89_WW][115] = 0, [0][1][2][1][RTW89_WW][117] = 0, [0][1][2][1][RTW89_WW][119] = 0, - [1][0][2][0][RTW89_WW][1] = 72, - [1][0][2][0][RTW89_WW][5] = 72, - [1][0][2][0][RTW89_WW][9] = 72, - [1][0][2][0][RTW89_WW][13] = 72, - [1][0][2][0][RTW89_WW][16] = 72, - [1][0][2][0][RTW89_WW][20] = 72, - [1][0][2][0][RTW89_WW][24] = 72, - [1][0][2][0][RTW89_WW][28] = 72, - [1][0][2][0][RTW89_WW][31] = 72, - [1][0][2][0][RTW89_WW][35] = 72, - [1][0][2][0][RTW89_WW][39] = 72, - [1][0][2][0][RTW89_WW][43] = 72, - [1][0][2][0][RTW89_WW][46] = 72, - [1][0][2][0][RTW89_WW][50] = 72, - [1][0][2][0][RTW89_WW][54] = 72, - [1][0][2][0][RTW89_WW][58] = 72, - [1][0][2][0][RTW89_WW][61] = 72, - [1][0][2][0][RTW89_WW][65] = 72, - [1][0][2][0][RTW89_WW][69] = 72, - [1][0][2][0][RTW89_WW][73] = 72, - [1][0][2][0][RTW89_WW][76] = 72, - [1][0][2][0][RTW89_WW][80] = 72, - [1][0][2][0][RTW89_WW][84] = 72, - [1][0][2][0][RTW89_WW][88] = 72, - [1][0][2][0][RTW89_WW][91] = 72, - [1][0][2][0][RTW89_WW][95] = 72, - [1][0][2][0][RTW89_WW][99] = 72, - [1][0][2][0][RTW89_WW][103] = 72, - [1][0][2][0][RTW89_WW][106] = 72, + [1][0][2][0][RTW89_WW][1] = 34, + [1][0][2][0][RTW89_WW][5] = 34, + [1][0][2][0][RTW89_WW][9] = 34, + [1][0][2][0][RTW89_WW][13] = 34, + [1][0][2][0][RTW89_WW][16] = 34, + [1][0][2][0][RTW89_WW][20] = 34, + [1][0][2][0][RTW89_WW][24] = 36, + [1][0][2][0][RTW89_WW][28] = 34, + [1][0][2][0][RTW89_WW][31] = 34, + [1][0][2][0][RTW89_WW][35] = 34, + [1][0][2][0][RTW89_WW][39] = 34, + [1][0][2][0][RTW89_WW][43] = 34, + [1][0][2][0][RTW89_WW][46] = 34, + [1][0][2][0][RTW89_WW][50] = 34, + [1][0][2][0][RTW89_WW][54] = 36, + [1][0][2][0][RTW89_WW][58] = 36, + [1][0][2][0][RTW89_WW][61] = 34, + [1][0][2][0][RTW89_WW][65] = 34, + [1][0][2][0][RTW89_WW][69] = 34, + [1][0][2][0][RTW89_WW][73] = 34, + [1][0][2][0][RTW89_WW][76] = 34, + [1][0][2][0][RTW89_WW][80] = 34, + [1][0][2][0][RTW89_WW][84] = 34, + [1][0][2][0][RTW89_WW][88] = 34, + [1][0][2][0][RTW89_WW][91] = 36, + [1][0][2][0][RTW89_WW][95] = 34, + [1][0][2][0][RTW89_WW][99] = 34, + [1][0][2][0][RTW89_WW][103] = 34, + [1][0][2][0][RTW89_WW][106] = 36, [1][0][2][0][RTW89_WW][110] = 0, [1][0][2][0][RTW89_WW][114] = 0, [1][0][2][0][RTW89_WW][118] = 0, - [1][1][2][0][RTW89_WW][1] = 60, - [1][1][2][0][RTW89_WW][5] = 60, - [1][1][2][0][RTW89_WW][9] = 60, - [1][1][2][0][RTW89_WW][13] = 60, - [1][1][2][0][RTW89_WW][16] = 60, - [1][1][2][0][RTW89_WW][20] = 60, - [1][1][2][0][RTW89_WW][24] = 60, - [1][1][2][0][RTW89_WW][28] = 60, - [1][1][2][0][RTW89_WW][31] = 60, - [1][1][2][0][RTW89_WW][35] = 60, - [1][1][2][0][RTW89_WW][39] = 60, - [1][1][2][0][RTW89_WW][43] = 60, - [1][1][2][0][RTW89_WW][46] = 60, - [1][1][2][0][RTW89_WW][50] = 60, - [1][1][2][0][RTW89_WW][54] = 60, - [1][1][2][0][RTW89_WW][58] = 60, - [1][1][2][0][RTW89_WW][61] = 60, - [1][1][2][0][RTW89_WW][65] = 60, - [1][1][2][0][RTW89_WW][69] = 60, - [1][1][2][0][RTW89_WW][73] = 60, - [1][1][2][0][RTW89_WW][76] = 60, - [1][1][2][0][RTW89_WW][80] = 60, - [1][1][2][0][RTW89_WW][84] = 60, - [1][1][2][0][RTW89_WW][88] = 60, - [1][1][2][0][RTW89_WW][91] = 60, - [1][1][2][0][RTW89_WW][95] = 60, - [1][1][2][0][RTW89_WW][99] = 60, - [1][1][2][0][RTW89_WW][103] = 60, - [1][1][2][0][RTW89_WW][106] = 60, + [1][1][2][0][RTW89_WW][1] = 10, + [1][1][2][0][RTW89_WW][5] = 10, + [1][1][2][0][RTW89_WW][9] = 10, + [1][1][2][0][RTW89_WW][13] = 10, + [1][1][2][0][RTW89_WW][16] = 10, + [1][1][2][0][RTW89_WW][20] = 10, + [1][1][2][0][RTW89_WW][24] = 10, + [1][1][2][0][RTW89_WW][28] = 10, + [1][1][2][0][RTW89_WW][31] = 10, + [1][1][2][0][RTW89_WW][35] = 10, + [1][1][2][0][RTW89_WW][39] = 10, + [1][1][2][0][RTW89_WW][43] = 10, + [1][1][2][0][RTW89_WW][46] = 12, + [1][1][2][0][RTW89_WW][50] = 12, + [1][1][2][0][RTW89_WW][54] = 10, + [1][1][2][0][RTW89_WW][58] = 10, + [1][1][2][0][RTW89_WW][61] = 10, + [1][1][2][0][RTW89_WW][65] = 10, + [1][1][2][0][RTW89_WW][69] = 10, + [1][1][2][0][RTW89_WW][73] = 10, + [1][1][2][0][RTW89_WW][76] = 10, + [1][1][2][0][RTW89_WW][80] = 10, + [1][1][2][0][RTW89_WW][84] = 10, + [1][1][2][0][RTW89_WW][88] = 10, + [1][1][2][0][RTW89_WW][91] = 12, + [1][1][2][0][RTW89_WW][95] = 10, + [1][1][2][0][RTW89_WW][99] = 10, + [1][1][2][0][RTW89_WW][103] = 10, + [1][1][2][0][RTW89_WW][106] = 12, [1][1][2][0][RTW89_WW][110] = 0, [1][1][2][0][RTW89_WW][114] = 0, [1][1][2][0][RTW89_WW][118] = 0, - [1][1][2][1][RTW89_WW][1] = 48, - [1][1][2][1][RTW89_WW][5] = 48, - [1][1][2][1][RTW89_WW][9] = 48, - [1][1][2][1][RTW89_WW][13] = 48, - [1][1][2][1][RTW89_WW][16] = 48, - [1][1][2][1][RTW89_WW][20] = 48, - [1][1][2][1][RTW89_WW][24] = 48, - [1][1][2][1][RTW89_WW][28] = 48, - [1][1][2][1][RTW89_WW][31] = 48, - [1][1][2][1][RTW89_WW][35] = 48, - [1][1][2][1][RTW89_WW][39] = 48, - [1][1][2][1][RTW89_WW][43] = 48, - [1][1][2][1][RTW89_WW][46] = 48, - [1][1][2][1][RTW89_WW][50] = 48, - [1][1][2][1][RTW89_WW][54] = 48, - [1][1][2][1][RTW89_WW][58] = 48, - [1][1][2][1][RTW89_WW][61] = 48, - [1][1][2][1][RTW89_WW][65] = 48, - [1][1][2][1][RTW89_WW][69] = 48, - [1][1][2][1][RTW89_WW][73] = 48, - [1][1][2][1][RTW89_WW][76] = 48, - [1][1][2][1][RTW89_WW][80] = 48, - [1][1][2][1][RTW89_WW][84] = 48, - [1][1][2][1][RTW89_WW][88] = 48, - [1][1][2][1][RTW89_WW][91] = 48, - [1][1][2][1][RTW89_WW][95] = 48, - [1][1][2][1][RTW89_WW][99] = 48, - [1][1][2][1][RTW89_WW][103] = 48, - [1][1][2][1][RTW89_WW][106] = 48, + [1][1][2][1][RTW89_WW][1] = 10, + [1][1][2][1][RTW89_WW][5] = 10, + [1][1][2][1][RTW89_WW][9] = 10, + [1][1][2][1][RTW89_WW][13] = 10, + [1][1][2][1][RTW89_WW][16] = 10, + [1][1][2][1][RTW89_WW][20] = 10, + [1][1][2][1][RTW89_WW][24] = 10, + [1][1][2][1][RTW89_WW][28] = 10, + [1][1][2][1][RTW89_WW][31] = 10, + [1][1][2][1][RTW89_WW][35] = 10, + [1][1][2][1][RTW89_WW][39] = 10, + [1][1][2][1][RTW89_WW][43] = 10, + [1][1][2][1][RTW89_WW][46] = 12, + [1][1][2][1][RTW89_WW][50] = 12, + [1][1][2][1][RTW89_WW][54] = 10, + [1][1][2][1][RTW89_WW][58] = 10, + [1][1][2][1][RTW89_WW][61] = 10, + [1][1][2][1][RTW89_WW][65] = 10, + [1][1][2][1][RTW89_WW][69] = 10, + [1][1][2][1][RTW89_WW][73] = 10, + [1][1][2][1][RTW89_WW][76] = 10, + [1][1][2][1][RTW89_WW][80] = 10, + [1][1][2][1][RTW89_WW][84] = 10, + [1][1][2][1][RTW89_WW][88] = 10, + [1][1][2][1][RTW89_WW][91] = 12, + [1][1][2][1][RTW89_WW][95] = 10, + [1][1][2][1][RTW89_WW][99] = 10, + [1][1][2][1][RTW89_WW][103] = 10, + [1][1][2][1][RTW89_WW][106] = 12, [1][1][2][1][RTW89_WW][110] = 0, [1][1][2][1][RTW89_WW][114] = 0, [1][1][2][1][RTW89_WW][118] = 0, - [2][0][2][0][RTW89_WW][3] = 64, - [2][0][2][0][RTW89_WW][11] = 64, - [2][0][2][0][RTW89_WW][18] = 64, - [2][0][2][0][RTW89_WW][26] = 64, - [2][0][2][0][RTW89_WW][33] = 64, - [2][0][2][0][RTW89_WW][41] = 64, - [2][0][2][0][RTW89_WW][48] = 64, - [2][0][2][0][RTW89_WW][56] = 64, - [2][0][2][0][RTW89_WW][63] = 64, - [2][0][2][0][RTW89_WW][71] = 64, - [2][0][2][0][RTW89_WW][78] = 64, - [2][0][2][0][RTW89_WW][86] = 64, - [2][0][2][0][RTW89_WW][93] = 64, - [2][0][2][0][RTW89_WW][101] = 64, + [2][0][2][0][RTW89_WW][3] = 46, + [2][0][2][0][RTW89_WW][11] = 46, + [2][0][2][0][RTW89_WW][18] = 46, + [2][0][2][0][RTW89_WW][26] = 46, + [2][0][2][0][RTW89_WW][33] = 46, + [2][0][2][0][RTW89_WW][41] = 46, + [2][0][2][0][RTW89_WW][48] = 46, + [2][0][2][0][RTW89_WW][56] = 46, + [2][0][2][0][RTW89_WW][63] = 46, + [2][0][2][0][RTW89_WW][71] = 46, + [2][0][2][0][RTW89_WW][78] = 46, + [2][0][2][0][RTW89_WW][86] = 46, + [2][0][2][0][RTW89_WW][93] = 46, + [2][0][2][0][RTW89_WW][101] = 44, [2][0][2][0][RTW89_WW][108] = 0, [2][0][2][0][RTW89_WW][116] = 0, - [2][1][2][0][RTW89_WW][3] = 52, - [2][1][2][0][RTW89_WW][11] = 52, - [2][1][2][0][RTW89_WW][18] = 52, - [2][1][2][0][RTW89_WW][26] = 52, - [2][1][2][0][RTW89_WW][33] = 52, - [2][1][2][0][RTW89_WW][41] = 52, - [2][1][2][0][RTW89_WW][48] = 52, - [2][1][2][0][RTW89_WW][56] = 52, - [2][1][2][0][RTW89_WW][63] = 52, - [2][1][2][0][RTW89_WW][71] = 52, - [2][1][2][0][RTW89_WW][78] = 52, - [2][1][2][0][RTW89_WW][86] = 52, - [2][1][2][0][RTW89_WW][93] = 52, - [2][1][2][0][RTW89_WW][101] = 52, + [2][1][2][0][RTW89_WW][3] = 22, + [2][1][2][0][RTW89_WW][11] = 20, + [2][1][2][0][RTW89_WW][18] = 20, + [2][1][2][0][RTW89_WW][26] = 20, + [2][1][2][0][RTW89_WW][33] = 20, + [2][1][2][0][RTW89_WW][41] = 22, + [2][1][2][0][RTW89_WW][48] = 22, + [2][1][2][0][RTW89_WW][56] = 20, + [2][1][2][0][RTW89_WW][63] = 22, + [2][1][2][0][RTW89_WW][71] = 20, + [2][1][2][0][RTW89_WW][78] = 20, + [2][1][2][0][RTW89_WW][86] = 20, + [2][1][2][0][RTW89_WW][93] = 22, + [2][1][2][0][RTW89_WW][101] = 22, [2][1][2][0][RTW89_WW][108] = 0, [2][1][2][0][RTW89_WW][116] = 0, - [2][1][2][1][RTW89_WW][3] = 40, - [2][1][2][1][RTW89_WW][11] = 40, - [2][1][2][1][RTW89_WW][18] = 40, - [2][1][2][1][RTW89_WW][26] = 40, - [2][1][2][1][RTW89_WW][33] = 40, - [2][1][2][1][RTW89_WW][41] = 40, - [2][1][2][1][RTW89_WW][48] = 40, - [2][1][2][1][RTW89_WW][56] = 40, - [2][1][2][1][RTW89_WW][63] = 40, - [2][1][2][1][RTW89_WW][71] = 40, - [2][1][2][1][RTW89_WW][78] = 40, - [2][1][2][1][RTW89_WW][86] = 40, - [2][1][2][1][RTW89_WW][93] = 40, - [2][1][2][1][RTW89_WW][101] = 40, + [2][1][2][1][RTW89_WW][3] = 22, + [2][1][2][1][RTW89_WW][11] = 20, + [2][1][2][1][RTW89_WW][18] = 20, + [2][1][2][1][RTW89_WW][26] = 20, + [2][1][2][1][RTW89_WW][33] = 20, + [2][1][2][1][RTW89_WW][41] = 22, + [2][1][2][1][RTW89_WW][48] = 22, + [2][1][2][1][RTW89_WW][56] = 20, + [2][1][2][1][RTW89_WW][63] = 22, + [2][1][2][1][RTW89_WW][71] = 20, + [2][1][2][1][RTW89_WW][78] = 20, + [2][1][2][1][RTW89_WW][86] = 20, + [2][1][2][1][RTW89_WW][93] = 22, + [2][1][2][1][RTW89_WW][101] = 22, [2][1][2][1][RTW89_WW][108] = 0, [2][1][2][1][RTW89_WW][116] = 0, - [3][0][2][0][RTW89_WW][7] = 56, - [3][0][2][0][RTW89_WW][22] = 56, - [3][0][2][0][RTW89_WW][37] = 56, - [3][0][2][0][RTW89_WW][52] = 56, - [3][0][2][0][RTW89_WW][67] = 56, - [3][0][2][0][RTW89_WW][82] = 56, - [3][0][2][0][RTW89_WW][97] = 56, + [3][0][2][0][RTW89_WW][7] = 38, + [3][0][2][0][RTW89_WW][22] = 38, + [3][0][2][0][RTW89_WW][37] = 38, + [3][0][2][0][RTW89_WW][52] = 54, + [3][0][2][0][RTW89_WW][67] = 54, + [3][0][2][0][RTW89_WW][82] = 26, + [3][0][2][0][RTW89_WW][97] = 26, [3][0][2][0][RTW89_WW][112] = 0, - [3][1][2][0][RTW89_WW][7] = 44, - [3][1][2][0][RTW89_WW][22] = 44, - [3][1][2][0][RTW89_WW][37] = 44, - [3][1][2][0][RTW89_WW][52] = 44, - [3][1][2][0][RTW89_WW][67] = 44, - [3][1][2][0][RTW89_WW][82] = 44, - [3][1][2][0][RTW89_WW][97] = 44, + [3][1][2][0][RTW89_WW][7] = 32, + [3][1][2][0][RTW89_WW][22] = 30, + [3][1][2][0][RTW89_WW][37] = 30, + [3][1][2][0][RTW89_WW][52] = 30, + [3][1][2][0][RTW89_WW][67] = 32, + [3][1][2][0][RTW89_WW][82] = 24, + [3][1][2][0][RTW89_WW][97] = 14, [3][1][2][0][RTW89_WW][112] = 0, [3][1][2][1][RTW89_WW][7] = 32, - [3][1][2][1][RTW89_WW][22] = 32, - [3][1][2][1][RTW89_WW][37] = 32, - [3][1][2][1][RTW89_WW][52] = 32, + [3][1][2][1][RTW89_WW][22] = 30, + [3][1][2][1][RTW89_WW][37] = 30, + [3][1][2][1][RTW89_WW][52] = 30, [3][1][2][1][RTW89_WW][67] = 32, - [3][1][2][1][RTW89_WW][82] = 32, - [3][1][2][1][RTW89_WW][97] = 32, + [3][1][2][1][RTW89_WW][82] = 24, + [3][1][2][1][RTW89_WW][97] = 14, [3][1][2][1][RTW89_WW][112] = 0, - [0][0][1][0][RTW89_FCC][0] = 72, - [0][0][1][0][RTW89_FCC][2] = 72, - [0][0][1][0][RTW89_FCC][4] = 72, - [0][0][1][0][RTW89_FCC][6] = 72, - [0][0][1][0][RTW89_FCC][8] = 72, - [0][0][1][0][RTW89_FCC][10] = 72, - [0][0][1][0][RTW89_FCC][12] = 72, - [0][0][1][0][RTW89_FCC][14] = 72, - [0][0][1][0][RTW89_FCC][15] = 72, - [0][0][1][0][RTW89_FCC][17] = 72, - [0][0][1][0][RTW89_FCC][19] = 72, - [0][0][1][0][RTW89_FCC][21] = 72, - [0][0][1][0][RTW89_FCC][23] = 72, - [0][0][1][0][RTW89_FCC][25] = 72, - [0][0][1][0][RTW89_FCC][27] = 72, - [0][0][1][0][RTW89_FCC][29] = 72, - [0][0][1][0][RTW89_FCC][30] = 72, - [0][0][1][0][RTW89_FCC][32] = 72, - [0][0][1][0][RTW89_FCC][34] = 72, - [0][0][1][0][RTW89_FCC][36] = 72, - [0][0][1][0][RTW89_FCC][38] = 72, - [0][0][1][0][RTW89_FCC][40] = 72, - [0][0][1][0][RTW89_FCC][42] = 72, - [0][0][1][0][RTW89_FCC][44] = 72, - [0][0][1][0][RTW89_FCC][45] = 72, - [0][0][1][0][RTW89_FCC][47] = 72, - [0][0][1][0][RTW89_FCC][49] = 72, - [0][0][1][0][RTW89_FCC][51] = 72, - [0][0][1][0][RTW89_FCC][53] = 72, - [0][0][1][0][RTW89_FCC][55] = 72, - [0][0][1][0][RTW89_FCC][57] = 72, - [0][0][1][0][RTW89_FCC][59] = 72, - [0][0][1][0][RTW89_FCC][60] = 72, - [0][0][1][0][RTW89_FCC][62] = 72, - [0][0][1][0][RTW89_FCC][64] = 72, - [0][0][1][0][RTW89_FCC][66] = 72, - [0][0][1][0][RTW89_FCC][68] = 72, - [0][0][1][0][RTW89_FCC][70] = 72, - [0][0][1][0][RTW89_FCC][72] = 72, - [0][0][1][0][RTW89_FCC][74] = 72, - [0][0][1][0][RTW89_FCC][75] = 72, - [0][0][1][0][RTW89_FCC][77] = 72, - [0][0][1][0][RTW89_FCC][79] = 72, - [0][0][1][0][RTW89_FCC][81] = 72, - [0][0][1][0][RTW89_FCC][83] = 72, - [0][0][1][0][RTW89_FCC][85] = 72, - [0][0][1][0][RTW89_FCC][87] = 72, - [0][0][1][0][RTW89_FCC][89] = 72, - [0][0][1][0][RTW89_FCC][90] = 72, - [0][0][1][0][RTW89_FCC][92] = 72, - [0][0][1][0][RTW89_FCC][94] = 72, - [0][0][1][0][RTW89_FCC][96] = 72, - [0][0][1][0][RTW89_FCC][98] = 72, - [0][0][1][0][RTW89_FCC][100] = 72, - [0][0][1][0][RTW89_FCC][102] = 72, - [0][0][1][0][RTW89_FCC][104] = 72, - [0][0][1][0][RTW89_FCC][105] = 72, - [0][0][1][0][RTW89_FCC][107] = 72, - [0][0][1][0][RTW89_FCC][109] = 72, + [0][0][1][0][RTW89_FCC][0] = 24, + [0][0][1][0][RTW89_ETSI][0] = 66, + [0][0][1][0][RTW89_KCC][0] = 24, + [0][0][1][0][RTW89_FCC][2] = 22, + [0][0][1][0][RTW89_ETSI][2] = 66, + [0][0][1][0][RTW89_KCC][2] = 24, + [0][0][1][0][RTW89_FCC][4] = 22, + [0][0][1][0][RTW89_ETSI][4] = 66, + [0][0][1][0][RTW89_KCC][4] = 24, + [0][0][1][0][RTW89_FCC][6] = 22, + [0][0][1][0][RTW89_ETSI][6] = 66, + [0][0][1][0][RTW89_KCC][6] = 24, + [0][0][1][0][RTW89_FCC][8] = 22, + [0][0][1][0][RTW89_ETSI][8] = 66, + [0][0][1][0][RTW89_KCC][8] = 24, + [0][0][1][0][RTW89_FCC][10] = 22, + [0][0][1][0][RTW89_ETSI][10] = 66, + [0][0][1][0][RTW89_KCC][10] = 24, + [0][0][1][0][RTW89_FCC][12] = 22, + [0][0][1][0][RTW89_ETSI][12] = 66, + [0][0][1][0][RTW89_KCC][12] = 24, + [0][0][1][0][RTW89_FCC][14] = 22, + [0][0][1][0][RTW89_ETSI][14] = 66, + [0][0][1][0][RTW89_KCC][14] = 24, + [0][0][1][0][RTW89_FCC][15] = 22, + [0][0][1][0][RTW89_ETSI][15] = 66, + [0][0][1][0][RTW89_KCC][15] = 24, + [0][0][1][0][RTW89_FCC][17] = 22, + [0][0][1][0][RTW89_ETSI][17] = 66, + [0][0][1][0][RTW89_KCC][17] = 24, + [0][0][1][0][RTW89_FCC][19] = 22, + [0][0][1][0][RTW89_ETSI][19] = 66, + [0][0][1][0][RTW89_KCC][19] = 24, + [0][0][1][0][RTW89_FCC][21] = 22, + [0][0][1][0][RTW89_ETSI][21] = 66, + [0][0][1][0][RTW89_KCC][21] = 24, + [0][0][1][0][RTW89_FCC][23] = 22, + [0][0][1][0][RTW89_ETSI][23] = 66, + [0][0][1][0][RTW89_KCC][23] = 24, + [0][0][1][0][RTW89_FCC][25] = 22, + [0][0][1][0][RTW89_ETSI][25] = 66, + [0][0][1][0][RTW89_KCC][25] = 24, + [0][0][1][0][RTW89_FCC][27] = 22, + [0][0][1][0][RTW89_ETSI][27] = 66, + [0][0][1][0][RTW89_KCC][27] = 24, + [0][0][1][0][RTW89_FCC][29] = 22, + [0][0][1][0][RTW89_ETSI][29] = 66, + [0][0][1][0][RTW89_KCC][29] = 24, + [0][0][1][0][RTW89_FCC][30] = 22, + [0][0][1][0][RTW89_ETSI][30] = 66, + [0][0][1][0][RTW89_KCC][30] = 24, + [0][0][1][0][RTW89_FCC][32] = 22, + [0][0][1][0][RTW89_ETSI][32] = 66, + [0][0][1][0][RTW89_KCC][32] = 24, + [0][0][1][0][RTW89_FCC][34] = 22, + [0][0][1][0][RTW89_ETSI][34] = 66, + [0][0][1][0][RTW89_KCC][34] = 24, + [0][0][1][0][RTW89_FCC][36] = 22, + [0][0][1][0][RTW89_ETSI][36] = 66, + [0][0][1][0][RTW89_KCC][36] = 24, + [0][0][1][0][RTW89_FCC][38] = 22, + [0][0][1][0][RTW89_ETSI][38] = 66, + [0][0][1][0][RTW89_KCC][38] = 24, + [0][0][1][0][RTW89_FCC][40] = 22, + [0][0][1][0][RTW89_ETSI][40] = 66, + [0][0][1][0][RTW89_KCC][40] = 24, + [0][0][1][0][RTW89_FCC][42] = 22, + [0][0][1][0][RTW89_ETSI][42] = 66, + [0][0][1][0][RTW89_KCC][42] = 24, + [0][0][1][0][RTW89_FCC][44] = 22, + [0][0][1][0][RTW89_ETSI][44] = 66, + [0][0][1][0][RTW89_KCC][44] = 24, + [0][0][1][0][RTW89_FCC][45] = 22, + [0][0][1][0][RTW89_ETSI][45] = 127, + [0][0][1][0][RTW89_KCC][45] = 24, + [0][0][1][0][RTW89_FCC][47] = 22, + [0][0][1][0][RTW89_ETSI][47] = 127, + [0][0][1][0][RTW89_KCC][47] = 24, + [0][0][1][0][RTW89_FCC][49] = 24, + [0][0][1][0][RTW89_ETSI][49] = 127, + [0][0][1][0][RTW89_KCC][49] = 24, + [0][0][1][0][RTW89_FCC][51] = 22, + [0][0][1][0][RTW89_ETSI][51] = 127, + [0][0][1][0][RTW89_KCC][51] = 24, + [0][0][1][0][RTW89_FCC][53] = 22, + [0][0][1][0][RTW89_ETSI][53] = 127, + [0][0][1][0][RTW89_KCC][53] = 24, + [0][0][1][0][RTW89_FCC][55] = 22, + [0][0][1][0][RTW89_ETSI][55] = 127, + [0][0][1][0][RTW89_KCC][55] = 26, + [0][0][1][0][RTW89_FCC][57] = 22, + [0][0][1][0][RTW89_ETSI][57] = 127, + [0][0][1][0][RTW89_KCC][57] = 26, + [0][0][1][0][RTW89_FCC][59] = 22, + [0][0][1][0][RTW89_ETSI][59] = 127, + [0][0][1][0][RTW89_KCC][59] = 26, + [0][0][1][0][RTW89_FCC][60] = 22, + [0][0][1][0][RTW89_ETSI][60] = 127, + [0][0][1][0][RTW89_KCC][60] = 26, + [0][0][1][0][RTW89_FCC][62] = 22, + [0][0][1][0][RTW89_ETSI][62] = 127, + [0][0][1][0][RTW89_KCC][62] = 26, + [0][0][1][0][RTW89_FCC][64] = 22, + [0][0][1][0][RTW89_ETSI][64] = 127, + [0][0][1][0][RTW89_KCC][64] = 26, + [0][0][1][0][RTW89_FCC][66] = 22, + [0][0][1][0][RTW89_ETSI][66] = 127, + [0][0][1][0][RTW89_KCC][66] = 26, + [0][0][1][0][RTW89_FCC][68] = 22, + [0][0][1][0][RTW89_ETSI][68] = 127, + [0][0][1][0][RTW89_KCC][68] = 26, + [0][0][1][0][RTW89_FCC][70] = 24, + [0][0][1][0][RTW89_ETSI][70] = 127, + [0][0][1][0][RTW89_KCC][70] = 26, + [0][0][1][0][RTW89_FCC][72] = 22, + [0][0][1][0][RTW89_ETSI][72] = 127, + [0][0][1][0][RTW89_KCC][72] = 26, + [0][0][1][0][RTW89_FCC][74] = 22, + [0][0][1][0][RTW89_ETSI][74] = 127, + [0][0][1][0][RTW89_KCC][74] = 26, + [0][0][1][0][RTW89_FCC][75] = 22, + [0][0][1][0][RTW89_ETSI][75] = 127, + [0][0][1][0][RTW89_KCC][75] = 26, + [0][0][1][0][RTW89_FCC][77] = 22, + [0][0][1][0][RTW89_ETSI][77] = 127, + [0][0][1][0][RTW89_KCC][77] = 26, + [0][0][1][0][RTW89_FCC][79] = 22, + [0][0][1][0][RTW89_ETSI][79] = 127, + [0][0][1][0][RTW89_KCC][79] = 26, + [0][0][1][0][RTW89_FCC][81] = 22, + [0][0][1][0][RTW89_ETSI][81] = 127, + [0][0][1][0][RTW89_KCC][81] = 26, + [0][0][1][0][RTW89_FCC][83] = 22, + [0][0][1][0][RTW89_ETSI][83] = 127, + [0][0][1][0][RTW89_KCC][83] = 32, + [0][0][1][0][RTW89_FCC][85] = 22, + [0][0][1][0][RTW89_ETSI][85] = 127, + [0][0][1][0][RTW89_KCC][85] = 32, + [0][0][1][0][RTW89_FCC][87] = 22, + [0][0][1][0][RTW89_ETSI][87] = 127, + [0][0][1][0][RTW89_KCC][87] = 32, + [0][0][1][0][RTW89_FCC][89] = 22, + [0][0][1][0][RTW89_ETSI][89] = 127, + [0][0][1][0][RTW89_KCC][89] = 32, + [0][0][1][0][RTW89_FCC][90] = 22, + [0][0][1][0][RTW89_ETSI][90] = 127, + [0][0][1][0][RTW89_KCC][90] = 32, + [0][0][1][0][RTW89_FCC][92] = 22, + [0][0][1][0][RTW89_ETSI][92] = 127, + [0][0][1][0][RTW89_KCC][92] = 32, + [0][0][1][0][RTW89_FCC][94] = 22, + [0][0][1][0][RTW89_ETSI][94] = 127, + [0][0][1][0][RTW89_KCC][94] = 32, + [0][0][1][0][RTW89_FCC][96] = 22, + [0][0][1][0][RTW89_ETSI][96] = 127, + [0][0][1][0][RTW89_KCC][96] = 32, + [0][0][1][0][RTW89_FCC][98] = 22, + [0][0][1][0][RTW89_ETSI][98] = 127, + [0][0][1][0][RTW89_KCC][98] = 32, + [0][0][1][0][RTW89_FCC][100] = 22, + [0][0][1][0][RTW89_ETSI][100] = 127, + [0][0][1][0][RTW89_KCC][100] = 32, + [0][0][1][0][RTW89_FCC][102] = 22, + [0][0][1][0][RTW89_ETSI][102] = 127, + [0][0][1][0][RTW89_KCC][102] = 32, + [0][0][1][0][RTW89_FCC][104] = 22, + [0][0][1][0][RTW89_ETSI][104] = 127, + [0][0][1][0][RTW89_KCC][104] = 32, + [0][0][1][0][RTW89_FCC][105] = 22, + [0][0][1][0][RTW89_ETSI][105] = 127, + [0][0][1][0][RTW89_KCC][105] = 32, + [0][0][1][0][RTW89_FCC][107] = 24, + [0][0][1][0][RTW89_ETSI][107] = 127, + [0][0][1][0][RTW89_KCC][107] = 32, + [0][0][1][0][RTW89_FCC][109] = 24, + [0][0][1][0][RTW89_ETSI][109] = 127, + [0][0][1][0][RTW89_KCC][109] = 32, [0][0][1][0][RTW89_FCC][111] = 127, + [0][0][1][0][RTW89_ETSI][111] = 127, + [0][0][1][0][RTW89_KCC][111] = 127, [0][0][1][0][RTW89_FCC][113] = 127, + [0][0][1][0][RTW89_ETSI][113] = 127, + [0][0][1][0][RTW89_KCC][113] = 127, [0][0][1][0][RTW89_FCC][115] = 127, + [0][0][1][0][RTW89_ETSI][115] = 127, + [0][0][1][0][RTW89_KCC][115] = 127, [0][0][1][0][RTW89_FCC][117] = 127, + [0][0][1][0][RTW89_ETSI][117] = 127, + [0][0][1][0][RTW89_KCC][117] = 127, [0][0][1][0][RTW89_FCC][119] = 127, - [0][1][1][0][RTW89_FCC][0] = 60, - [0][1][1][0][RTW89_FCC][2] = 60, - [0][1][1][0][RTW89_FCC][4] = 60, - [0][1][1][0][RTW89_FCC][6] = 60, - [0][1][1][0][RTW89_FCC][8] = 60, - [0][1][1][0][RTW89_FCC][10] = 60, - [0][1][1][0][RTW89_FCC][12] = 60, - [0][1][1][0][RTW89_FCC][14] = 60, - [0][1][1][0][RTW89_FCC][15] = 60, - [0][1][1][0][RTW89_FCC][17] = 60, - [0][1][1][0][RTW89_FCC][19] = 60, - [0][1][1][0][RTW89_FCC][21] = 60, - [0][1][1][0][RTW89_FCC][23] = 60, - [0][1][1][0][RTW89_FCC][25] = 60, - [0][1][1][0][RTW89_FCC][27] = 60, - [0][1][1][0][RTW89_FCC][29] = 60, - [0][1][1][0][RTW89_FCC][30] = 60, - [0][1][1][0][RTW89_FCC][32] = 60, - [0][1][1][0][RTW89_FCC][34] = 60, - [0][1][1][0][RTW89_FCC][36] = 60, - [0][1][1][0][RTW89_FCC][38] = 60, - [0][1][1][0][RTW89_FCC][40] = 60, - [0][1][1][0][RTW89_FCC][42] = 60, - [0][1][1][0][RTW89_FCC][44] = 60, - [0][1][1][0][RTW89_FCC][45] = 60, - [0][1][1][0][RTW89_FCC][47] = 60, - [0][1][1][0][RTW89_FCC][49] = 60, - [0][1][1][0][RTW89_FCC][51] = 60, - [0][1][1][0][RTW89_FCC][53] = 60, - [0][1][1][0][RTW89_FCC][55] = 60, - [0][1][1][0][RTW89_FCC][57] = 60, - [0][1][1][0][RTW89_FCC][59] = 60, - [0][1][1][0][RTW89_FCC][60] = 60, - [0][1][1][0][RTW89_FCC][62] = 60, - [0][1][1][0][RTW89_FCC][64] = 60, - [0][1][1][0][RTW89_FCC][66] = 60, - [0][1][1][0][RTW89_FCC][68] = 60, - [0][1][1][0][RTW89_FCC][70] = 60, - [0][1][1][0][RTW89_FCC][72] = 60, - [0][1][1][0][RTW89_FCC][74] = 60, - [0][1][1][0][RTW89_FCC][75] = 60, - [0][1][1][0][RTW89_FCC][77] = 60, - [0][1][1][0][RTW89_FCC][79] = 60, - [0][1][1][0][RTW89_FCC][81] = 60, - [0][1][1][0][RTW89_FCC][83] = 60, - [0][1][1][0][RTW89_FCC][85] = 60, - [0][1][1][0][RTW89_FCC][87] = 60, - [0][1][1][0][RTW89_FCC][89] = 60, - [0][1][1][0][RTW89_FCC][90] = 60, - [0][1][1][0][RTW89_FCC][92] = 60, - [0][1][1][0][RTW89_FCC][94] = 60, - [0][1][1][0][RTW89_FCC][96] = 60, - [0][1][1][0][RTW89_FCC][98] = 60, - [0][1][1][0][RTW89_FCC][100] = 60, - [0][1][1][0][RTW89_FCC][102] = 60, - [0][1][1][0][RTW89_FCC][104] = 60, - [0][1][1][0][RTW89_FCC][105] = 60, - [0][1][1][0][RTW89_FCC][107] = 60, - [0][1][1][0][RTW89_FCC][109] = 60, + [0][0][1][0][RTW89_ETSI][119] = 127, + [0][0][1][0][RTW89_KCC][119] = 127, + [0][1][1][0][RTW89_FCC][0] = -2, + [0][1][1][0][RTW89_ETSI][0] = 54, + [0][1][1][0][RTW89_KCC][0] = 12, + [0][1][1][0][RTW89_FCC][2] = -4, + [0][1][1][0][RTW89_ETSI][2] = 54, + [0][1][1][0][RTW89_KCC][2] = 12, + [0][1][1][0][RTW89_FCC][4] = -4, + [0][1][1][0][RTW89_ETSI][4] = 54, + [0][1][1][0][RTW89_KCC][4] = 12, + [0][1][1][0][RTW89_FCC][6] = -4, + [0][1][1][0][RTW89_ETSI][6] = 54, + [0][1][1][0][RTW89_KCC][6] = 12, + [0][1][1][0][RTW89_FCC][8] = -4, + [0][1][1][0][RTW89_ETSI][8] = 54, + [0][1][1][0][RTW89_KCC][8] = 12, + [0][1][1][0][RTW89_FCC][10] = -4, + [0][1][1][0][RTW89_ETSI][10] = 54, + [0][1][1][0][RTW89_KCC][10] = 12, + [0][1][1][0][RTW89_FCC][12] = -4, + [0][1][1][0][RTW89_ETSI][12] = 54, + [0][1][1][0][RTW89_KCC][12] = 12, + [0][1][1][0][RTW89_FCC][14] = -4, + [0][1][1][0][RTW89_ETSI][14] = 54, + [0][1][1][0][RTW89_KCC][14] = 12, + [0][1][1][0][RTW89_FCC][15] = -4, + [0][1][1][0][RTW89_ETSI][15] = 54, + [0][1][1][0][RTW89_KCC][15] = 12, + [0][1][1][0][RTW89_FCC][17] = -4, + [0][1][1][0][RTW89_ETSI][17] = 54, + [0][1][1][0][RTW89_KCC][17] = 12, + [0][1][1][0][RTW89_FCC][19] = -4, + [0][1][1][0][RTW89_ETSI][19] = 54, + [0][1][1][0][RTW89_KCC][19] = 12, + [0][1][1][0][RTW89_FCC][21] = -4, + [0][1][1][0][RTW89_ETSI][21] = 54, + [0][1][1][0][RTW89_KCC][21] = 12, + [0][1][1][0][RTW89_FCC][23] = -4, + [0][1][1][0][RTW89_ETSI][23] = 54, + [0][1][1][0][RTW89_KCC][23] = 12, + [0][1][1][0][RTW89_FCC][25] = -4, + [0][1][1][0][RTW89_ETSI][25] = 54, + [0][1][1][0][RTW89_KCC][25] = 12, + [0][1][1][0][RTW89_FCC][27] = -4, + [0][1][1][0][RTW89_ETSI][27] = 54, + [0][1][1][0][RTW89_KCC][27] = 12, + [0][1][1][0][RTW89_FCC][29] = -4, + [0][1][1][0][RTW89_ETSI][29] = 54, + [0][1][1][0][RTW89_KCC][29] = 12, + [0][1][1][0][RTW89_FCC][30] = -4, + [0][1][1][0][RTW89_ETSI][30] = 54, + [0][1][1][0][RTW89_KCC][30] = 12, + [0][1][1][0][RTW89_FCC][32] = -4, + [0][1][1][0][RTW89_ETSI][32] = 54, + [0][1][1][0][RTW89_KCC][32] = 12, + [0][1][1][0][RTW89_FCC][34] = -4, + [0][1][1][0][RTW89_ETSI][34] = 54, + [0][1][1][0][RTW89_KCC][34] = 12, + [0][1][1][0][RTW89_FCC][36] = -4, + [0][1][1][0][RTW89_ETSI][36] = 54, + [0][1][1][0][RTW89_KCC][36] = 12, + [0][1][1][0][RTW89_FCC][38] = -4, + [0][1][1][0][RTW89_ETSI][38] = 54, + [0][1][1][0][RTW89_KCC][38] = 12, + [0][1][1][0][RTW89_FCC][40] = -4, + [0][1][1][0][RTW89_ETSI][40] = 54, + [0][1][1][0][RTW89_KCC][40] = 12, + [0][1][1][0][RTW89_FCC][42] = -4, + [0][1][1][0][RTW89_ETSI][42] = 54, + [0][1][1][0][RTW89_KCC][42] = 12, + [0][1][1][0][RTW89_FCC][44] = -2, + [0][1][1][0][RTW89_ETSI][44] = 54, + [0][1][1][0][RTW89_KCC][44] = 12, + [0][1][1][0][RTW89_FCC][45] = -2, + [0][1][1][0][RTW89_ETSI][45] = 127, + [0][1][1][0][RTW89_KCC][45] = 12, + [0][1][1][0][RTW89_FCC][47] = -2, + [0][1][1][0][RTW89_ETSI][47] = 127, + [0][1][1][0][RTW89_KCC][47] = 12, + [0][1][1][0][RTW89_FCC][49] = -2, + [0][1][1][0][RTW89_ETSI][49] = 127, + [0][1][1][0][RTW89_KCC][49] = 12, + [0][1][1][0][RTW89_FCC][51] = -2, + [0][1][1][0][RTW89_ETSI][51] = 127, + [0][1][1][0][RTW89_KCC][51] = 12, + [0][1][1][0][RTW89_FCC][53] = -2, + [0][1][1][0][RTW89_ETSI][53] = 127, + [0][1][1][0][RTW89_KCC][53] = 12, + [0][1][1][0][RTW89_FCC][55] = -2, + [0][1][1][0][RTW89_ETSI][55] = 127, + [0][1][1][0][RTW89_KCC][55] = 12, + [0][1][1][0][RTW89_FCC][57] = -2, + [0][1][1][0][RTW89_ETSI][57] = 127, + [0][1][1][0][RTW89_KCC][57] = 12, + [0][1][1][0][RTW89_FCC][59] = -2, + [0][1][1][0][RTW89_ETSI][59] = 127, + [0][1][1][0][RTW89_KCC][59] = 12, + [0][1][1][0][RTW89_FCC][60] = -2, + [0][1][1][0][RTW89_ETSI][60] = 127, + [0][1][1][0][RTW89_KCC][60] = 12, + [0][1][1][0][RTW89_FCC][62] = -2, + [0][1][1][0][RTW89_ETSI][62] = 127, + [0][1][1][0][RTW89_KCC][62] = 12, + [0][1][1][0][RTW89_FCC][64] = -2, + [0][1][1][0][RTW89_ETSI][64] = 127, + [0][1][1][0][RTW89_KCC][64] = 12, + [0][1][1][0][RTW89_FCC][66] = -2, + [0][1][1][0][RTW89_ETSI][66] = 127, + [0][1][1][0][RTW89_KCC][66] = 12, + [0][1][1][0][RTW89_FCC][68] = -2, + [0][1][1][0][RTW89_ETSI][68] = 127, + [0][1][1][0][RTW89_KCC][68] = 12, + [0][1][1][0][RTW89_FCC][70] = -2, + [0][1][1][0][RTW89_ETSI][70] = 127, + [0][1][1][0][RTW89_KCC][70] = 12, + [0][1][1][0][RTW89_FCC][72] = -2, + [0][1][1][0][RTW89_ETSI][72] = 127, + [0][1][1][0][RTW89_KCC][72] = 12, + [0][1][1][0][RTW89_FCC][74] = -2, + [0][1][1][0][RTW89_ETSI][74] = 127, + [0][1][1][0][RTW89_KCC][74] = 12, + [0][1][1][0][RTW89_FCC][75] = -2, + [0][1][1][0][RTW89_ETSI][75] = 127, + [0][1][1][0][RTW89_KCC][75] = 12, + [0][1][1][0][RTW89_FCC][77] = -2, + [0][1][1][0][RTW89_ETSI][77] = 127, + [0][1][1][0][RTW89_KCC][77] = 12, + [0][1][1][0][RTW89_FCC][79] = -2, + [0][1][1][0][RTW89_ETSI][79] = 127, + [0][1][1][0][RTW89_KCC][79] = 12, + [0][1][1][0][RTW89_FCC][81] = -2, + [0][1][1][0][RTW89_ETSI][81] = 127, + [0][1][1][0][RTW89_KCC][81] = 12, + [0][1][1][0][RTW89_FCC][83] = -2, + [0][1][1][0][RTW89_ETSI][83] = 127, + [0][1][1][0][RTW89_KCC][83] = 20, + [0][1][1][0][RTW89_FCC][85] = -2, + [0][1][1][0][RTW89_ETSI][85] = 127, + [0][1][1][0][RTW89_KCC][85] = 20, + [0][1][1][0][RTW89_FCC][87] = -2, + [0][1][1][0][RTW89_ETSI][87] = 127, + [0][1][1][0][RTW89_KCC][87] = 20, + [0][1][1][0][RTW89_FCC][89] = -2, + [0][1][1][0][RTW89_ETSI][89] = 127, + [0][1][1][0][RTW89_KCC][89] = 20, + [0][1][1][0][RTW89_FCC][90] = -2, + [0][1][1][0][RTW89_ETSI][90] = 127, + [0][1][1][0][RTW89_KCC][90] = 20, + [0][1][1][0][RTW89_FCC][92] = -2, + [0][1][1][0][RTW89_ETSI][92] = 127, + [0][1][1][0][RTW89_KCC][92] = 20, + [0][1][1][0][RTW89_FCC][94] = -2, + [0][1][1][0][RTW89_ETSI][94] = 127, + [0][1][1][0][RTW89_KCC][94] = 20, + [0][1][1][0][RTW89_FCC][96] = -2, + [0][1][1][0][RTW89_ETSI][96] = 127, + [0][1][1][0][RTW89_KCC][96] = 20, + [0][1][1][0][RTW89_FCC][98] = -2, + [0][1][1][0][RTW89_ETSI][98] = 127, + [0][1][1][0][RTW89_KCC][98] = 20, + [0][1][1][0][RTW89_FCC][100] = -2, + [0][1][1][0][RTW89_ETSI][100] = 127, + [0][1][1][0][RTW89_KCC][100] = 20, + [0][1][1][0][RTW89_FCC][102] = -2, + [0][1][1][0][RTW89_ETSI][102] = 127, + [0][1][1][0][RTW89_KCC][102] = 20, + [0][1][1][0][RTW89_FCC][104] = -2, + [0][1][1][0][RTW89_ETSI][104] = 127, + [0][1][1][0][RTW89_KCC][104] = 20, + [0][1][1][0][RTW89_FCC][105] = -2, + [0][1][1][0][RTW89_ETSI][105] = 127, + [0][1][1][0][RTW89_KCC][105] = 20, + [0][1][1][0][RTW89_FCC][107] = 0, + [0][1][1][0][RTW89_ETSI][107] = 127, + [0][1][1][0][RTW89_KCC][107] = 20, + [0][1][1][0][RTW89_FCC][109] = 0, + [0][1][1][0][RTW89_ETSI][109] = 127, + [0][1][1][0][RTW89_KCC][109] = 20, [0][1][1][0][RTW89_FCC][111] = 127, + [0][1][1][0][RTW89_ETSI][111] = 127, + [0][1][1][0][RTW89_KCC][111] = 127, [0][1][1][0][RTW89_FCC][113] = 127, + [0][1][1][0][RTW89_ETSI][113] = 127, + [0][1][1][0][RTW89_KCC][113] = 127, [0][1][1][0][RTW89_FCC][115] = 127, + [0][1][1][0][RTW89_ETSI][115] = 127, + [0][1][1][0][RTW89_KCC][115] = 127, [0][1][1][0][RTW89_FCC][117] = 127, + [0][1][1][0][RTW89_ETSI][117] = 127, + [0][1][1][0][RTW89_KCC][117] = 127, [0][1][1][0][RTW89_FCC][119] = 127, - [0][0][2][0][RTW89_FCC][0] = 72, - [0][0][2][0][RTW89_FCC][2] = 72, - [0][0][2][0][RTW89_FCC][4] = 72, - [0][0][2][0][RTW89_FCC][6] = 72, - [0][0][2][0][RTW89_FCC][8] = 72, - [0][0][2][0][RTW89_FCC][10] = 72, - [0][0][2][0][RTW89_FCC][12] = 72, - [0][0][2][0][RTW89_FCC][14] = 72, - [0][0][2][0][RTW89_FCC][15] = 72, - [0][0][2][0][RTW89_FCC][17] = 72, - [0][0][2][0][RTW89_FCC][19] = 72, - [0][0][2][0][RTW89_FCC][21] = 72, - [0][0][2][0][RTW89_FCC][23] = 72, - [0][0][2][0][RTW89_FCC][25] = 72, - [0][0][2][0][RTW89_FCC][27] = 72, - [0][0][2][0][RTW89_FCC][29] = 72, - [0][0][2][0][RTW89_FCC][30] = 72, - [0][0][2][0][RTW89_FCC][32] = 72, - [0][0][2][0][RTW89_FCC][34] = 72, - [0][0][2][0][RTW89_FCC][36] = 72, - [0][0][2][0][RTW89_FCC][38] = 72, - [0][0][2][0][RTW89_FCC][40] = 72, - [0][0][2][0][RTW89_FCC][42] = 72, - [0][0][2][0][RTW89_FCC][44] = 72, - [0][0][2][0][RTW89_FCC][45] = 72, - [0][0][2][0][RTW89_FCC][47] = 72, - [0][0][2][0][RTW89_FCC][49] = 72, - [0][0][2][0][RTW89_FCC][51] = 72, - [0][0][2][0][RTW89_FCC][53] = 72, - [0][0][2][0][RTW89_FCC][55] = 72, - [0][0][2][0][RTW89_FCC][57] = 72, - [0][0][2][0][RTW89_FCC][59] = 72, - [0][0][2][0][RTW89_FCC][60] = 72, - [0][0][2][0][RTW89_FCC][62] = 72, - [0][0][2][0][RTW89_FCC][64] = 72, - [0][0][2][0][RTW89_FCC][66] = 72, - [0][0][2][0][RTW89_FCC][68] = 72, - [0][0][2][0][RTW89_FCC][70] = 72, - [0][0][2][0][RTW89_FCC][72] = 72, - [0][0][2][0][RTW89_FCC][74] = 72, - [0][0][2][0][RTW89_FCC][75] = 72, - [0][0][2][0][RTW89_FCC][77] = 72, - [0][0][2][0][RTW89_FCC][79] = 72, - [0][0][2][0][RTW89_FCC][81] = 72, - [0][0][2][0][RTW89_FCC][83] = 72, - [0][0][2][0][RTW89_FCC][85] = 72, - [0][0][2][0][RTW89_FCC][87] = 72, - [0][0][2][0][RTW89_FCC][89] = 72, - [0][0][2][0][RTW89_FCC][90] = 72, - [0][0][2][0][RTW89_FCC][92] = 72, - [0][0][2][0][RTW89_FCC][94] = 72, - [0][0][2][0][RTW89_FCC][96] = 72, - [0][0][2][0][RTW89_FCC][98] = 72, - [0][0][2][0][RTW89_FCC][100] = 72, - [0][0][2][0][RTW89_FCC][102] = 72, - [0][0][2][0][RTW89_FCC][104] = 72, - [0][0][2][0][RTW89_FCC][105] = 72, - [0][0][2][0][RTW89_FCC][107] = 72, - [0][0][2][0][RTW89_FCC][109] = 72, + [0][1][1][0][RTW89_ETSI][119] = 127, + [0][1][1][0][RTW89_KCC][119] = 127, + [0][0][2][0][RTW89_FCC][0] = 24, + [0][0][2][0][RTW89_ETSI][0] = 66, + [0][0][2][0][RTW89_KCC][0] = 24, + [0][0][2][0][RTW89_FCC][2] = 22, + [0][0][2][0][RTW89_ETSI][2] = 66, + [0][0][2][0][RTW89_KCC][2] = 24, + [0][0][2][0][RTW89_FCC][4] = 22, + [0][0][2][0][RTW89_ETSI][4] = 66, + [0][0][2][0][RTW89_KCC][4] = 24, + [0][0][2][0][RTW89_FCC][6] = 22, + [0][0][2][0][RTW89_ETSI][6] = 66, + [0][0][2][0][RTW89_KCC][6] = 24, + [0][0][2][0][RTW89_FCC][8] = 22, + [0][0][2][0][RTW89_ETSI][8] = 66, + [0][0][2][0][RTW89_KCC][8] = 24, + [0][0][2][0][RTW89_FCC][10] = 22, + [0][0][2][0][RTW89_ETSI][10] = 66, + [0][0][2][0][RTW89_KCC][10] = 24, + [0][0][2][0][RTW89_FCC][12] = 22, + [0][0][2][0][RTW89_ETSI][12] = 66, + [0][0][2][0][RTW89_KCC][12] = 24, + [0][0][2][0][RTW89_FCC][14] = 22, + [0][0][2][0][RTW89_ETSI][14] = 66, + [0][0][2][0][RTW89_KCC][14] = 24, + [0][0][2][0][RTW89_FCC][15] = 22, + [0][0][2][0][RTW89_ETSI][15] = 66, + [0][0][2][0][RTW89_KCC][15] = 24, + [0][0][2][0][RTW89_FCC][17] = 22, + [0][0][2][0][RTW89_ETSI][17] = 66, + [0][0][2][0][RTW89_KCC][17] = 24, + [0][0][2][0][RTW89_FCC][19] = 22, + [0][0][2][0][RTW89_ETSI][19] = 66, + [0][0][2][0][RTW89_KCC][19] = 24, + [0][0][2][0][RTW89_FCC][21] = 22, + [0][0][2][0][RTW89_ETSI][21] = 66, + [0][0][2][0][RTW89_KCC][21] = 24, + [0][0][2][0][RTW89_FCC][23] = 22, + [0][0][2][0][RTW89_ETSI][23] = 66, + [0][0][2][0][RTW89_KCC][23] = 24, + [0][0][2][0][RTW89_FCC][25] = 22, + [0][0][2][0][RTW89_ETSI][25] = 66, + [0][0][2][0][RTW89_KCC][25] = 24, + [0][0][2][0][RTW89_FCC][27] = 22, + [0][0][2][0][RTW89_ETSI][27] = 66, + [0][0][2][0][RTW89_KCC][27] = 24, + [0][0][2][0][RTW89_FCC][29] = 22, + [0][0][2][0][RTW89_ETSI][29] = 66, + [0][0][2][0][RTW89_KCC][29] = 24, + [0][0][2][0][RTW89_FCC][30] = 22, + [0][0][2][0][RTW89_ETSI][30] = 66, + [0][0][2][0][RTW89_KCC][30] = 24, + [0][0][2][0][RTW89_FCC][32] = 22, + [0][0][2][0][RTW89_ETSI][32] = 66, + [0][0][2][0][RTW89_KCC][32] = 24, + [0][0][2][0][RTW89_FCC][34] = 22, + [0][0][2][0][RTW89_ETSI][34] = 66, + [0][0][2][0][RTW89_KCC][34] = 24, + [0][0][2][0][RTW89_FCC][36] = 22, + [0][0][2][0][RTW89_ETSI][36] = 66, + [0][0][2][0][RTW89_KCC][36] = 24, + [0][0][2][0][RTW89_FCC][38] = 22, + [0][0][2][0][RTW89_ETSI][38] = 66, + [0][0][2][0][RTW89_KCC][38] = 24, + [0][0][2][0][RTW89_FCC][40] = 22, + [0][0][2][0][RTW89_ETSI][40] = 66, + [0][0][2][0][RTW89_KCC][40] = 24, + [0][0][2][0][RTW89_FCC][42] = 22, + [0][0][2][0][RTW89_ETSI][42] = 66, + [0][0][2][0][RTW89_KCC][42] = 24, + [0][0][2][0][RTW89_FCC][44] = 22, + [0][0][2][0][RTW89_ETSI][44] = 66, + [0][0][2][0][RTW89_KCC][44] = 24, + [0][0][2][0][RTW89_FCC][45] = 22, + [0][0][2][0][RTW89_ETSI][45] = 127, + [0][0][2][0][RTW89_KCC][45] = 24, + [0][0][2][0][RTW89_FCC][47] = 22, + [0][0][2][0][RTW89_ETSI][47] = 127, + [0][0][2][0][RTW89_KCC][47] = 24, + [0][0][2][0][RTW89_FCC][49] = 24, + [0][0][2][0][RTW89_ETSI][49] = 127, + [0][0][2][0][RTW89_KCC][49] = 24, + [0][0][2][0][RTW89_FCC][51] = 22, + [0][0][2][0][RTW89_ETSI][51] = 127, + [0][0][2][0][RTW89_KCC][51] = 24, + [0][0][2][0][RTW89_FCC][53] = 22, + [0][0][2][0][RTW89_ETSI][53] = 127, + [0][0][2][0][RTW89_KCC][53] = 24, + [0][0][2][0][RTW89_FCC][55] = 22, + [0][0][2][0][RTW89_ETSI][55] = 127, + [0][0][2][0][RTW89_KCC][55] = 26, + [0][0][2][0][RTW89_FCC][57] = 22, + [0][0][2][0][RTW89_ETSI][57] = 127, + [0][0][2][0][RTW89_KCC][57] = 26, + [0][0][2][0][RTW89_FCC][59] = 22, + [0][0][2][0][RTW89_ETSI][59] = 127, + [0][0][2][0][RTW89_KCC][59] = 26, + [0][0][2][0][RTW89_FCC][60] = 22, + [0][0][2][0][RTW89_ETSI][60] = 127, + [0][0][2][0][RTW89_KCC][60] = 26, + [0][0][2][0][RTW89_FCC][62] = 22, + [0][0][2][0][RTW89_ETSI][62] = 127, + [0][0][2][0][RTW89_KCC][62] = 26, + [0][0][2][0][RTW89_FCC][64] = 22, + [0][0][2][0][RTW89_ETSI][64] = 127, + [0][0][2][0][RTW89_KCC][64] = 26, + [0][0][2][0][RTW89_FCC][66] = 22, + [0][0][2][0][RTW89_ETSI][66] = 127, + [0][0][2][0][RTW89_KCC][66] = 26, + [0][0][2][0][RTW89_FCC][68] = 22, + [0][0][2][0][RTW89_ETSI][68] = 127, + [0][0][2][0][RTW89_KCC][68] = 26, + [0][0][2][0][RTW89_FCC][70] = 24, + [0][0][2][0][RTW89_ETSI][70] = 127, + [0][0][2][0][RTW89_KCC][70] = 26, + [0][0][2][0][RTW89_FCC][72] = 22, + [0][0][2][0][RTW89_ETSI][72] = 127, + [0][0][2][0][RTW89_KCC][72] = 26, + [0][0][2][0][RTW89_FCC][74] = 22, + [0][0][2][0][RTW89_ETSI][74] = 127, + [0][0][2][0][RTW89_KCC][74] = 26, + [0][0][2][0][RTW89_FCC][75] = 22, + [0][0][2][0][RTW89_ETSI][75] = 127, + [0][0][2][0][RTW89_KCC][75] = 26, + [0][0][2][0][RTW89_FCC][77] = 22, + [0][0][2][0][RTW89_ETSI][77] = 127, + [0][0][2][0][RTW89_KCC][77] = 26, + [0][0][2][0][RTW89_FCC][79] = 22, + [0][0][2][0][RTW89_ETSI][79] = 127, + [0][0][2][0][RTW89_KCC][79] = 26, + [0][0][2][0][RTW89_FCC][81] = 22, + [0][0][2][0][RTW89_ETSI][81] = 127, + [0][0][2][0][RTW89_KCC][81] = 26, + [0][0][2][0][RTW89_FCC][83] = 22, + [0][0][2][0][RTW89_ETSI][83] = 127, + [0][0][2][0][RTW89_KCC][83] = 32, + [0][0][2][0][RTW89_FCC][85] = 22, + [0][0][2][0][RTW89_ETSI][85] = 127, + [0][0][2][0][RTW89_KCC][85] = 32, + [0][0][2][0][RTW89_FCC][87] = 22, + [0][0][2][0][RTW89_ETSI][87] = 127, + [0][0][2][0][RTW89_KCC][87] = 32, + [0][0][2][0][RTW89_FCC][89] = 22, + [0][0][2][0][RTW89_ETSI][89] = 127, + [0][0][2][0][RTW89_KCC][89] = 32, + [0][0][2][0][RTW89_FCC][90] = 22, + [0][0][2][0][RTW89_ETSI][90] = 127, + [0][0][2][0][RTW89_KCC][90] = 32, + [0][0][2][0][RTW89_FCC][92] = 22, + [0][0][2][0][RTW89_ETSI][92] = 127, + [0][0][2][0][RTW89_KCC][92] = 32, + [0][0][2][0][RTW89_FCC][94] = 22, + [0][0][2][0][RTW89_ETSI][94] = 127, + [0][0][2][0][RTW89_KCC][94] = 32, + [0][0][2][0][RTW89_FCC][96] = 22, + [0][0][2][0][RTW89_ETSI][96] = 127, + [0][0][2][0][RTW89_KCC][96] = 32, + [0][0][2][0][RTW89_FCC][98] = 22, + [0][0][2][0][RTW89_ETSI][98] = 127, + [0][0][2][0][RTW89_KCC][98] = 32, + [0][0][2][0][RTW89_FCC][100] = 22, + [0][0][2][0][RTW89_ETSI][100] = 127, + [0][0][2][0][RTW89_KCC][100] = 32, + [0][0][2][0][RTW89_FCC][102] = 22, + [0][0][2][0][RTW89_ETSI][102] = 127, + [0][0][2][0][RTW89_KCC][102] = 32, + [0][0][2][0][RTW89_FCC][104] = 22, + [0][0][2][0][RTW89_ETSI][104] = 127, + [0][0][2][0][RTW89_KCC][104] = 32, + [0][0][2][0][RTW89_FCC][105] = 22, + [0][0][2][0][RTW89_ETSI][105] = 127, + [0][0][2][0][RTW89_KCC][105] = 32, + [0][0][2][0][RTW89_FCC][107] = 24, + [0][0][2][0][RTW89_ETSI][107] = 127, + [0][0][2][0][RTW89_KCC][107] = 32, + [0][0][2][0][RTW89_FCC][109] = 24, + [0][0][2][0][RTW89_ETSI][109] = 127, + [0][0][2][0][RTW89_KCC][109] = 32, [0][0][2][0][RTW89_FCC][111] = 127, + [0][0][2][0][RTW89_ETSI][111] = 127, + [0][0][2][0][RTW89_KCC][111] = 127, [0][0][2][0][RTW89_FCC][113] = 127, + [0][0][2][0][RTW89_ETSI][113] = 127, + [0][0][2][0][RTW89_KCC][113] = 127, [0][0][2][0][RTW89_FCC][115] = 127, + [0][0][2][0][RTW89_ETSI][115] = 127, + [0][0][2][0][RTW89_KCC][115] = 127, [0][0][2][0][RTW89_FCC][117] = 127, + [0][0][2][0][RTW89_ETSI][117] = 127, + [0][0][2][0][RTW89_KCC][117] = 127, [0][0][2][0][RTW89_FCC][119] = 127, - [0][1][2][0][RTW89_FCC][0] = 60, - [0][1][2][0][RTW89_FCC][2] = 60, - [0][1][2][0][RTW89_FCC][4] = 60, - [0][1][2][0][RTW89_FCC][6] = 60, - [0][1][2][0][RTW89_FCC][8] = 60, - [0][1][2][0][RTW89_FCC][10] = 60, - [0][1][2][0][RTW89_FCC][12] = 60, - [0][1][2][0][RTW89_FCC][14] = 60, - [0][1][2][0][RTW89_FCC][15] = 60, - [0][1][2][0][RTW89_FCC][17] = 60, - [0][1][2][0][RTW89_FCC][19] = 60, - [0][1][2][0][RTW89_FCC][21] = 60, - [0][1][2][0][RTW89_FCC][23] = 60, - [0][1][2][0][RTW89_FCC][25] = 60, - [0][1][2][0][RTW89_FCC][27] = 60, - [0][1][2][0][RTW89_FCC][29] = 60, - [0][1][2][0][RTW89_FCC][30] = 60, - [0][1][2][0][RTW89_FCC][32] = 60, - [0][1][2][0][RTW89_FCC][34] = 60, - [0][1][2][0][RTW89_FCC][36] = 60, - [0][1][2][0][RTW89_FCC][38] = 60, - [0][1][2][0][RTW89_FCC][40] = 60, - [0][1][2][0][RTW89_FCC][42] = 60, - [0][1][2][0][RTW89_FCC][44] = 60, - [0][1][2][0][RTW89_FCC][45] = 60, - [0][1][2][0][RTW89_FCC][47] = 60, - [0][1][2][0][RTW89_FCC][49] = 60, - [0][1][2][0][RTW89_FCC][51] = 60, - [0][1][2][0][RTW89_FCC][53] = 60, - [0][1][2][0][RTW89_FCC][55] = 60, - [0][1][2][0][RTW89_FCC][57] = 60, - [0][1][2][0][RTW89_FCC][59] = 60, - [0][1][2][0][RTW89_FCC][60] = 60, - [0][1][2][0][RTW89_FCC][62] = 60, - [0][1][2][0][RTW89_FCC][64] = 60, - [0][1][2][0][RTW89_FCC][66] = 60, - [0][1][2][0][RTW89_FCC][68] = 60, - [0][1][2][0][RTW89_FCC][70] = 60, - [0][1][2][0][RTW89_FCC][72] = 60, - [0][1][2][0][RTW89_FCC][74] = 60, - [0][1][2][0][RTW89_FCC][75] = 60, - [0][1][2][0][RTW89_FCC][77] = 60, - [0][1][2][0][RTW89_FCC][79] = 60, - [0][1][2][0][RTW89_FCC][81] = 60, - [0][1][2][0][RTW89_FCC][83] = 60, - [0][1][2][0][RTW89_FCC][85] = 60, - [0][1][2][0][RTW89_FCC][87] = 60, - [0][1][2][0][RTW89_FCC][89] = 60, - [0][1][2][0][RTW89_FCC][90] = 60, - [0][1][2][0][RTW89_FCC][92] = 60, - [0][1][2][0][RTW89_FCC][94] = 60, - [0][1][2][0][RTW89_FCC][96] = 60, - [0][1][2][0][RTW89_FCC][98] = 60, - [0][1][2][0][RTW89_FCC][100] = 60, - [0][1][2][0][RTW89_FCC][102] = 60, - [0][1][2][0][RTW89_FCC][104] = 60, - [0][1][2][0][RTW89_FCC][105] = 60, - [0][1][2][0][RTW89_FCC][107] = 60, - [0][1][2][0][RTW89_FCC][109] = 60, + [0][0][2][0][RTW89_ETSI][119] = 127, + [0][0][2][0][RTW89_KCC][119] = 127, + [0][1][2][0][RTW89_FCC][0] = -2, + [0][1][2][0][RTW89_ETSI][0] = 54, + [0][1][2][0][RTW89_KCC][0] = 12, + [0][1][2][0][RTW89_FCC][2] = -4, + [0][1][2][0][RTW89_ETSI][2] = 54, + [0][1][2][0][RTW89_KCC][2] = 12, + [0][1][2][0][RTW89_FCC][4] = -4, + [0][1][2][0][RTW89_ETSI][4] = 54, + [0][1][2][0][RTW89_KCC][4] = 12, + [0][1][2][0][RTW89_FCC][6] = -4, + [0][1][2][0][RTW89_ETSI][6] = 54, + [0][1][2][0][RTW89_KCC][6] = 12, + [0][1][2][0][RTW89_FCC][8] = -4, + [0][1][2][0][RTW89_ETSI][8] = 54, + [0][1][2][0][RTW89_KCC][8] = 12, + [0][1][2][0][RTW89_FCC][10] = -4, + [0][1][2][0][RTW89_ETSI][10] = 54, + [0][1][2][0][RTW89_KCC][10] = 12, + [0][1][2][0][RTW89_FCC][12] = -4, + [0][1][2][0][RTW89_ETSI][12] = 54, + [0][1][2][0][RTW89_KCC][12] = 12, + [0][1][2][0][RTW89_FCC][14] = -4, + [0][1][2][0][RTW89_ETSI][14] = 54, + [0][1][2][0][RTW89_KCC][14] = 12, + [0][1][2][0][RTW89_FCC][15] = -4, + [0][1][2][0][RTW89_ETSI][15] = 54, + [0][1][2][0][RTW89_KCC][15] = 12, + [0][1][2][0][RTW89_FCC][17] = -4, + [0][1][2][0][RTW89_ETSI][17] = 54, + [0][1][2][0][RTW89_KCC][17] = 12, + [0][1][2][0][RTW89_FCC][19] = -4, + [0][1][2][0][RTW89_ETSI][19] = 54, + [0][1][2][0][RTW89_KCC][19] = 12, + [0][1][2][0][RTW89_FCC][21] = -4, + [0][1][2][0][RTW89_ETSI][21] = 54, + [0][1][2][0][RTW89_KCC][21] = 12, + [0][1][2][0][RTW89_FCC][23] = -4, + [0][1][2][0][RTW89_ETSI][23] = 54, + [0][1][2][0][RTW89_KCC][23] = 12, + [0][1][2][0][RTW89_FCC][25] = -4, + [0][1][2][0][RTW89_ETSI][25] = 54, + [0][1][2][0][RTW89_KCC][25] = 12, + [0][1][2][0][RTW89_FCC][27] = -4, + [0][1][2][0][RTW89_ETSI][27] = 54, + [0][1][2][0][RTW89_KCC][27] = 12, + [0][1][2][0][RTW89_FCC][29] = -4, + [0][1][2][0][RTW89_ETSI][29] = 54, + [0][1][2][0][RTW89_KCC][29] = 12, + [0][1][2][0][RTW89_FCC][30] = -4, + [0][1][2][0][RTW89_ETSI][30] = 54, + [0][1][2][0][RTW89_KCC][30] = 12, + [0][1][2][0][RTW89_FCC][32] = -4, + [0][1][2][0][RTW89_ETSI][32] = 54, + [0][1][2][0][RTW89_KCC][32] = 12, + [0][1][2][0][RTW89_FCC][34] = -4, + [0][1][2][0][RTW89_ETSI][34] = 54, + [0][1][2][0][RTW89_KCC][34] = 12, + [0][1][2][0][RTW89_FCC][36] = -4, + [0][1][2][0][RTW89_ETSI][36] = 54, + [0][1][2][0][RTW89_KCC][36] = 12, + [0][1][2][0][RTW89_FCC][38] = -4, + [0][1][2][0][RTW89_ETSI][38] = 54, + [0][1][2][0][RTW89_KCC][38] = 12, + [0][1][2][0][RTW89_FCC][40] = -4, + [0][1][2][0][RTW89_ETSI][40] = 54, + [0][1][2][0][RTW89_KCC][40] = 12, + [0][1][2][0][RTW89_FCC][42] = -4, + [0][1][2][0][RTW89_ETSI][42] = 54, + [0][1][2][0][RTW89_KCC][42] = 12, + [0][1][2][0][RTW89_FCC][44] = -2, + [0][1][2][0][RTW89_ETSI][44] = 54, + [0][1][2][0][RTW89_KCC][44] = 12, + [0][1][2][0][RTW89_FCC][45] = -2, + [0][1][2][0][RTW89_ETSI][45] = 127, + [0][1][2][0][RTW89_KCC][45] = 12, + [0][1][2][0][RTW89_FCC][47] = -2, + [0][1][2][0][RTW89_ETSI][47] = 127, + [0][1][2][0][RTW89_KCC][47] = 12, + [0][1][2][0][RTW89_FCC][49] = -2, + [0][1][2][0][RTW89_ETSI][49] = 127, + [0][1][2][0][RTW89_KCC][49] = 12, + [0][1][2][0][RTW89_FCC][51] = -2, + [0][1][2][0][RTW89_ETSI][51] = 127, + [0][1][2][0][RTW89_KCC][51] = 12, + [0][1][2][0][RTW89_FCC][53] = -2, + [0][1][2][0][RTW89_ETSI][53] = 127, + [0][1][2][0][RTW89_KCC][53] = 12, + [0][1][2][0][RTW89_FCC][55] = -2, + [0][1][2][0][RTW89_ETSI][55] = 127, + [0][1][2][0][RTW89_KCC][55] = 12, + [0][1][2][0][RTW89_FCC][57] = -2, + [0][1][2][0][RTW89_ETSI][57] = 127, + [0][1][2][0][RTW89_KCC][57] = 12, + [0][1][2][0][RTW89_FCC][59] = -2, + [0][1][2][0][RTW89_ETSI][59] = 127, + [0][1][2][0][RTW89_KCC][59] = 12, + [0][1][2][0][RTW89_FCC][60] = -2, + [0][1][2][0][RTW89_ETSI][60] = 127, + [0][1][2][0][RTW89_KCC][60] = 12, + [0][1][2][0][RTW89_FCC][62] = -2, + [0][1][2][0][RTW89_ETSI][62] = 127, + [0][1][2][0][RTW89_KCC][62] = 12, + [0][1][2][0][RTW89_FCC][64] = -2, + [0][1][2][0][RTW89_ETSI][64] = 127, + [0][1][2][0][RTW89_KCC][64] = 12, + [0][1][2][0][RTW89_FCC][66] = -2, + [0][1][2][0][RTW89_ETSI][66] = 127, + [0][1][2][0][RTW89_KCC][66] = 12, + [0][1][2][0][RTW89_FCC][68] = -2, + [0][1][2][0][RTW89_ETSI][68] = 127, + [0][1][2][0][RTW89_KCC][68] = 12, + [0][1][2][0][RTW89_FCC][70] = -2, + [0][1][2][0][RTW89_ETSI][70] = 127, + [0][1][2][0][RTW89_KCC][70] = 12, + [0][1][2][0][RTW89_FCC][72] = -2, + [0][1][2][0][RTW89_ETSI][72] = 127, + [0][1][2][0][RTW89_KCC][72] = 12, + [0][1][2][0][RTW89_FCC][74] = -2, + [0][1][2][0][RTW89_ETSI][74] = 127, + [0][1][2][0][RTW89_KCC][74] = 12, + [0][1][2][0][RTW89_FCC][75] = -2, + [0][1][2][0][RTW89_ETSI][75] = 127, + [0][1][2][0][RTW89_KCC][75] = 12, + [0][1][2][0][RTW89_FCC][77] = -2, + [0][1][2][0][RTW89_ETSI][77] = 127, + [0][1][2][0][RTW89_KCC][77] = 12, + [0][1][2][0][RTW89_FCC][79] = -2, + [0][1][2][0][RTW89_ETSI][79] = 127, + [0][1][2][0][RTW89_KCC][79] = 12, + [0][1][2][0][RTW89_FCC][81] = -2, + [0][1][2][0][RTW89_ETSI][81] = 127, + [0][1][2][0][RTW89_KCC][81] = 12, + [0][1][2][0][RTW89_FCC][83] = -2, + [0][1][2][0][RTW89_ETSI][83] = 127, + [0][1][2][0][RTW89_KCC][83] = 20, + [0][1][2][0][RTW89_FCC][85] = -2, + [0][1][2][0][RTW89_ETSI][85] = 127, + [0][1][2][0][RTW89_KCC][85] = 20, + [0][1][2][0][RTW89_FCC][87] = -2, + [0][1][2][0][RTW89_ETSI][87] = 127, + [0][1][2][0][RTW89_KCC][87] = 20, + [0][1][2][0][RTW89_FCC][89] = -2, + [0][1][2][0][RTW89_ETSI][89] = 127, + [0][1][2][0][RTW89_KCC][89] = 20, + [0][1][2][0][RTW89_FCC][90] = -2, + [0][1][2][0][RTW89_ETSI][90] = 127, + [0][1][2][0][RTW89_KCC][90] = 20, + [0][1][2][0][RTW89_FCC][92] = -2, + [0][1][2][0][RTW89_ETSI][92] = 127, + [0][1][2][0][RTW89_KCC][92] = 20, + [0][1][2][0][RTW89_FCC][94] = -2, + [0][1][2][0][RTW89_ETSI][94] = 127, + [0][1][2][0][RTW89_KCC][94] = 20, + [0][1][2][0][RTW89_FCC][96] = -2, + [0][1][2][0][RTW89_ETSI][96] = 127, + [0][1][2][0][RTW89_KCC][96] = 20, + [0][1][2][0][RTW89_FCC][98] = -2, + [0][1][2][0][RTW89_ETSI][98] = 127, + [0][1][2][0][RTW89_KCC][98] = 20, + [0][1][2][0][RTW89_FCC][100] = -2, + [0][1][2][0][RTW89_ETSI][100] = 127, + [0][1][2][0][RTW89_KCC][100] = 20, + [0][1][2][0][RTW89_FCC][102] = -2, + [0][1][2][0][RTW89_ETSI][102] = 127, + [0][1][2][0][RTW89_KCC][102] = 20, + [0][1][2][0][RTW89_FCC][104] = -2, + [0][1][2][0][RTW89_ETSI][104] = 127, + [0][1][2][0][RTW89_KCC][104] = 20, + [0][1][2][0][RTW89_FCC][105] = -2, + [0][1][2][0][RTW89_ETSI][105] = 127, + [0][1][2][0][RTW89_KCC][105] = 20, + [0][1][2][0][RTW89_FCC][107] = 0, + [0][1][2][0][RTW89_ETSI][107] = 127, + [0][1][2][0][RTW89_KCC][107] = 20, + [0][1][2][0][RTW89_FCC][109] = 0, + [0][1][2][0][RTW89_ETSI][109] = 127, + [0][1][2][0][RTW89_KCC][109] = 20, [0][1][2][0][RTW89_FCC][111] = 127, + [0][1][2][0][RTW89_ETSI][111] = 127, + [0][1][2][0][RTW89_KCC][111] = 127, [0][1][2][0][RTW89_FCC][113] = 127, + [0][1][2][0][RTW89_ETSI][113] = 127, + [0][1][2][0][RTW89_KCC][113] = 127, [0][1][2][0][RTW89_FCC][115] = 127, + [0][1][2][0][RTW89_ETSI][115] = 127, + [0][1][2][0][RTW89_KCC][115] = 127, [0][1][2][0][RTW89_FCC][117] = 127, + [0][1][2][0][RTW89_ETSI][117] = 127, + [0][1][2][0][RTW89_KCC][117] = 127, [0][1][2][0][RTW89_FCC][119] = 127, - [0][1][2][1][RTW89_FCC][0] = 48, - [0][1][2][1][RTW89_FCC][2] = 48, - [0][1][2][1][RTW89_FCC][4] = 48, - [0][1][2][1][RTW89_FCC][6] = 48, - [0][1][2][1][RTW89_FCC][8] = 48, - [0][1][2][1][RTW89_FCC][10] = 48, - [0][1][2][1][RTW89_FCC][12] = 48, - [0][1][2][1][RTW89_FCC][14] = 48, - [0][1][2][1][RTW89_FCC][15] = 48, - [0][1][2][1][RTW89_FCC][17] = 48, - [0][1][2][1][RTW89_FCC][19] = 48, - [0][1][2][1][RTW89_FCC][21] = 48, - [0][1][2][1][RTW89_FCC][23] = 48, - [0][1][2][1][RTW89_FCC][25] = 48, - [0][1][2][1][RTW89_FCC][27] = 48, - [0][1][2][1][RTW89_FCC][29] = 48, - [0][1][2][1][RTW89_FCC][30] = 48, - [0][1][2][1][RTW89_FCC][32] = 48, - [0][1][2][1][RTW89_FCC][34] = 48, - [0][1][2][1][RTW89_FCC][36] = 48, - [0][1][2][1][RTW89_FCC][38] = 48, - [0][1][2][1][RTW89_FCC][40] = 48, - [0][1][2][1][RTW89_FCC][42] = 48, - [0][1][2][1][RTW89_FCC][44] = 48, - [0][1][2][1][RTW89_FCC][45] = 48, - [0][1][2][1][RTW89_FCC][47] = 48, - [0][1][2][1][RTW89_FCC][49] = 48, - [0][1][2][1][RTW89_FCC][51] = 48, - [0][1][2][1][RTW89_FCC][53] = 48, - [0][1][2][1][RTW89_FCC][55] = 48, - [0][1][2][1][RTW89_FCC][57] = 48, - [0][1][2][1][RTW89_FCC][59] = 48, - [0][1][2][1][RTW89_FCC][60] = 48, - [0][1][2][1][RTW89_FCC][62] = 48, - [0][1][2][1][RTW89_FCC][64] = 48, - [0][1][2][1][RTW89_FCC][66] = 48, - [0][1][2][1][RTW89_FCC][68] = 48, - [0][1][2][1][RTW89_FCC][70] = 48, - [0][1][2][1][RTW89_FCC][72] = 48, - [0][1][2][1][RTW89_FCC][74] = 48, - [0][1][2][1][RTW89_FCC][75] = 48, - [0][1][2][1][RTW89_FCC][77] = 48, - [0][1][2][1][RTW89_FCC][79] = 48, - [0][1][2][1][RTW89_FCC][81] = 48, - [0][1][2][1][RTW89_FCC][83] = 48, - [0][1][2][1][RTW89_FCC][85] = 48, - [0][1][2][1][RTW89_FCC][87] = 48, - [0][1][2][1][RTW89_FCC][89] = 48, - [0][1][2][1][RTW89_FCC][90] = 48, - [0][1][2][1][RTW89_FCC][92] = 48, - [0][1][2][1][RTW89_FCC][94] = 48, - [0][1][2][1][RTW89_FCC][96] = 48, - [0][1][2][1][RTW89_FCC][98] = 48, - [0][1][2][1][RTW89_FCC][100] = 48, - [0][1][2][1][RTW89_FCC][102] = 48, - [0][1][2][1][RTW89_FCC][104] = 48, - [0][1][2][1][RTW89_FCC][105] = 48, - [0][1][2][1][RTW89_FCC][107] = 48, - [0][1][2][1][RTW89_FCC][109] = 48, + [0][1][2][0][RTW89_ETSI][119] = 127, + [0][1][2][0][RTW89_KCC][119] = 127, + [0][1][2][1][RTW89_FCC][0] = -2, + [0][1][2][1][RTW89_ETSI][0] = 42, + [0][1][2][1][RTW89_KCC][0] = 12, + [0][1][2][1][RTW89_FCC][2] = -4, + [0][1][2][1][RTW89_ETSI][2] = 42, + [0][1][2][1][RTW89_KCC][2] = 12, + [0][1][2][1][RTW89_FCC][4] = -4, + [0][1][2][1][RTW89_ETSI][4] = 42, + [0][1][2][1][RTW89_KCC][4] = 12, + [0][1][2][1][RTW89_FCC][6] = -4, + [0][1][2][1][RTW89_ETSI][6] = 42, + [0][1][2][1][RTW89_KCC][6] = 12, + [0][1][2][1][RTW89_FCC][8] = -4, + [0][1][2][1][RTW89_ETSI][8] = 42, + [0][1][2][1][RTW89_KCC][8] = 12, + [0][1][2][1][RTW89_FCC][10] = -4, + [0][1][2][1][RTW89_ETSI][10] = 42, + [0][1][2][1][RTW89_KCC][10] = 12, + [0][1][2][1][RTW89_FCC][12] = -4, + [0][1][2][1][RTW89_ETSI][12] = 42, + [0][1][2][1][RTW89_KCC][12] = 12, + [0][1][2][1][RTW89_FCC][14] = -4, + [0][1][2][1][RTW89_ETSI][14] = 42, + [0][1][2][1][RTW89_KCC][14] = 12, + [0][1][2][1][RTW89_FCC][15] = -4, + [0][1][2][1][RTW89_ETSI][15] = 42, + [0][1][2][1][RTW89_KCC][15] = 12, + [0][1][2][1][RTW89_FCC][17] = -4, + [0][1][2][1][RTW89_ETSI][17] = 42, + [0][1][2][1][RTW89_KCC][17] = 12, + [0][1][2][1][RTW89_FCC][19] = -4, + [0][1][2][1][RTW89_ETSI][19] = 42, + [0][1][2][1][RTW89_KCC][19] = 12, + [0][1][2][1][RTW89_FCC][21] = -4, + [0][1][2][1][RTW89_ETSI][21] = 42, + [0][1][2][1][RTW89_KCC][21] = 12, + [0][1][2][1][RTW89_FCC][23] = -4, + [0][1][2][1][RTW89_ETSI][23] = 42, + [0][1][2][1][RTW89_KCC][23] = 12, + [0][1][2][1][RTW89_FCC][25] = -4, + [0][1][2][1][RTW89_ETSI][25] = 42, + [0][1][2][1][RTW89_KCC][25] = 12, + [0][1][2][1][RTW89_FCC][27] = -4, + [0][1][2][1][RTW89_ETSI][27] = 42, + [0][1][2][1][RTW89_KCC][27] = 12, + [0][1][2][1][RTW89_FCC][29] = -4, + [0][1][2][1][RTW89_ETSI][29] = 42, + [0][1][2][1][RTW89_KCC][29] = 12, + [0][1][2][1][RTW89_FCC][30] = -4, + [0][1][2][1][RTW89_ETSI][30] = 42, + [0][1][2][1][RTW89_KCC][30] = 12, + [0][1][2][1][RTW89_FCC][32] = -4, + [0][1][2][1][RTW89_ETSI][32] = 42, + [0][1][2][1][RTW89_KCC][32] = 12, + [0][1][2][1][RTW89_FCC][34] = -4, + [0][1][2][1][RTW89_ETSI][34] = 42, + [0][1][2][1][RTW89_KCC][34] = 12, + [0][1][2][1][RTW89_FCC][36] = -4, + [0][1][2][1][RTW89_ETSI][36] = 42, + [0][1][2][1][RTW89_KCC][36] = 12, + [0][1][2][1][RTW89_FCC][38] = -4, + [0][1][2][1][RTW89_ETSI][38] = 42, + [0][1][2][1][RTW89_KCC][38] = 12, + [0][1][2][1][RTW89_FCC][40] = -4, + [0][1][2][1][RTW89_ETSI][40] = 42, + [0][1][2][1][RTW89_KCC][40] = 12, + [0][1][2][1][RTW89_FCC][42] = -4, + [0][1][2][1][RTW89_ETSI][42] = 42, + [0][1][2][1][RTW89_KCC][42] = 12, + [0][1][2][1][RTW89_FCC][44] = -2, + [0][1][2][1][RTW89_ETSI][44] = 42, + [0][1][2][1][RTW89_KCC][44] = 12, + [0][1][2][1][RTW89_FCC][45] = -2, + [0][1][2][1][RTW89_ETSI][45] = 127, + [0][1][2][1][RTW89_KCC][45] = 12, + [0][1][2][1][RTW89_FCC][47] = -2, + [0][1][2][1][RTW89_ETSI][47] = 127, + [0][1][2][1][RTW89_KCC][47] = 12, + [0][1][2][1][RTW89_FCC][49] = -2, + [0][1][2][1][RTW89_ETSI][49] = 127, + [0][1][2][1][RTW89_KCC][49] = 12, + [0][1][2][1][RTW89_FCC][51] = -2, + [0][1][2][1][RTW89_ETSI][51] = 127, + [0][1][2][1][RTW89_KCC][51] = 12, + [0][1][2][1][RTW89_FCC][53] = -2, + [0][1][2][1][RTW89_ETSI][53] = 127, + [0][1][2][1][RTW89_KCC][53] = 12, + [0][1][2][1][RTW89_FCC][55] = -2, + [0][1][2][1][RTW89_ETSI][55] = 127, + [0][1][2][1][RTW89_KCC][55] = 12, + [0][1][2][1][RTW89_FCC][57] = -2, + [0][1][2][1][RTW89_ETSI][57] = 127, + [0][1][2][1][RTW89_KCC][57] = 12, + [0][1][2][1][RTW89_FCC][59] = -2, + [0][1][2][1][RTW89_ETSI][59] = 127, + [0][1][2][1][RTW89_KCC][59] = 12, + [0][1][2][1][RTW89_FCC][60] = -2, + [0][1][2][1][RTW89_ETSI][60] = 127, + [0][1][2][1][RTW89_KCC][60] = 12, + [0][1][2][1][RTW89_FCC][62] = -2, + [0][1][2][1][RTW89_ETSI][62] = 127, + [0][1][2][1][RTW89_KCC][62] = 12, + [0][1][2][1][RTW89_FCC][64] = -2, + [0][1][2][1][RTW89_ETSI][64] = 127, + [0][1][2][1][RTW89_KCC][64] = 12, + [0][1][2][1][RTW89_FCC][66] = -2, + [0][1][2][1][RTW89_ETSI][66] = 127, + [0][1][2][1][RTW89_KCC][66] = 12, + [0][1][2][1][RTW89_FCC][68] = -2, + [0][1][2][1][RTW89_ETSI][68] = 127, + [0][1][2][1][RTW89_KCC][68] = 12, + [0][1][2][1][RTW89_FCC][70] = -2, + [0][1][2][1][RTW89_ETSI][70] = 127, + [0][1][2][1][RTW89_KCC][70] = 12, + [0][1][2][1][RTW89_FCC][72] = -2, + [0][1][2][1][RTW89_ETSI][72] = 127, + [0][1][2][1][RTW89_KCC][72] = 12, + [0][1][2][1][RTW89_FCC][74] = -2, + [0][1][2][1][RTW89_ETSI][74] = 127, + [0][1][2][1][RTW89_KCC][74] = 12, + [0][1][2][1][RTW89_FCC][75] = -2, + [0][1][2][1][RTW89_ETSI][75] = 127, + [0][1][2][1][RTW89_KCC][75] = 12, + [0][1][2][1][RTW89_FCC][77] = -2, + [0][1][2][1][RTW89_ETSI][77] = 127, + [0][1][2][1][RTW89_KCC][77] = 12, + [0][1][2][1][RTW89_FCC][79] = -2, + [0][1][2][1][RTW89_ETSI][79] = 127, + [0][1][2][1][RTW89_KCC][79] = 12, + [0][1][2][1][RTW89_FCC][81] = -2, + [0][1][2][1][RTW89_ETSI][81] = 127, + [0][1][2][1][RTW89_KCC][81] = 12, + [0][1][2][1][RTW89_FCC][83] = -2, + [0][1][2][1][RTW89_ETSI][83] = 127, + [0][1][2][1][RTW89_KCC][83] = 20, + [0][1][2][1][RTW89_FCC][85] = -2, + [0][1][2][1][RTW89_ETSI][85] = 127, + [0][1][2][1][RTW89_KCC][85] = 20, + [0][1][2][1][RTW89_FCC][87] = -2, + [0][1][2][1][RTW89_ETSI][87] = 127, + [0][1][2][1][RTW89_KCC][87] = 20, + [0][1][2][1][RTW89_FCC][89] = -2, + [0][1][2][1][RTW89_ETSI][89] = 127, + [0][1][2][1][RTW89_KCC][89] = 20, + [0][1][2][1][RTW89_FCC][90] = -2, + [0][1][2][1][RTW89_ETSI][90] = 127, + [0][1][2][1][RTW89_KCC][90] = 20, + [0][1][2][1][RTW89_FCC][92] = -2, + [0][1][2][1][RTW89_ETSI][92] = 127, + [0][1][2][1][RTW89_KCC][92] = 20, + [0][1][2][1][RTW89_FCC][94] = -2, + [0][1][2][1][RTW89_ETSI][94] = 127, + [0][1][2][1][RTW89_KCC][94] = 20, + [0][1][2][1][RTW89_FCC][96] = -2, + [0][1][2][1][RTW89_ETSI][96] = 127, + [0][1][2][1][RTW89_KCC][96] = 20, + [0][1][2][1][RTW89_FCC][98] = -2, + [0][1][2][1][RTW89_ETSI][98] = 127, + [0][1][2][1][RTW89_KCC][98] = 20, + [0][1][2][1][RTW89_FCC][100] = -2, + [0][1][2][1][RTW89_ETSI][100] = 127, + [0][1][2][1][RTW89_KCC][100] = 20, + [0][1][2][1][RTW89_FCC][102] = -2, + [0][1][2][1][RTW89_ETSI][102] = 127, + [0][1][2][1][RTW89_KCC][102] = 20, + [0][1][2][1][RTW89_FCC][104] = -2, + [0][1][2][1][RTW89_ETSI][104] = 127, + [0][1][2][1][RTW89_KCC][104] = 20, + [0][1][2][1][RTW89_FCC][105] = -2, + [0][1][2][1][RTW89_ETSI][105] = 127, + [0][1][2][1][RTW89_KCC][105] = 20, + [0][1][2][1][RTW89_FCC][107] = 0, + [0][1][2][1][RTW89_ETSI][107] = 127, + [0][1][2][1][RTW89_KCC][107] = 20, + [0][1][2][1][RTW89_FCC][109] = 0, + [0][1][2][1][RTW89_ETSI][109] = 127, + [0][1][2][1][RTW89_KCC][109] = 20, [0][1][2][1][RTW89_FCC][111] = 127, + [0][1][2][1][RTW89_ETSI][111] = 127, + [0][1][2][1][RTW89_KCC][111] = 127, [0][1][2][1][RTW89_FCC][113] = 127, + [0][1][2][1][RTW89_ETSI][113] = 127, + [0][1][2][1][RTW89_KCC][113] = 127, [0][1][2][1][RTW89_FCC][115] = 127, + [0][1][2][1][RTW89_ETSI][115] = 127, + [0][1][2][1][RTW89_KCC][115] = 127, [0][1][2][1][RTW89_FCC][117] = 127, + [0][1][2][1][RTW89_ETSI][117] = 127, + [0][1][2][1][RTW89_KCC][117] = 127, [0][1][2][1][RTW89_FCC][119] = 127, - [1][0][2][0][RTW89_FCC][1] = 72, - [1][0][2][0][RTW89_FCC][5] = 72, - [1][0][2][0][RTW89_FCC][9] = 72, - [1][0][2][0][RTW89_FCC][13] = 72, - [1][0][2][0][RTW89_FCC][16] = 72, - [1][0][2][0][RTW89_FCC][20] = 72, - [1][0][2][0][RTW89_FCC][24] = 72, - [1][0][2][0][RTW89_FCC][28] = 72, - [1][0][2][0][RTW89_FCC][31] = 72, - [1][0][2][0][RTW89_FCC][35] = 72, - [1][0][2][0][RTW89_FCC][39] = 72, - [1][0][2][0][RTW89_FCC][43] = 72, - [1][0][2][0][RTW89_FCC][46] = 72, - [1][0][2][0][RTW89_FCC][50] = 72, - [1][0][2][0][RTW89_FCC][54] = 72, - [1][0][2][0][RTW89_FCC][58] = 72, - [1][0][2][0][RTW89_FCC][61] = 72, - [1][0][2][0][RTW89_FCC][65] = 72, - [1][0][2][0][RTW89_FCC][69] = 72, - [1][0][2][0][RTW89_FCC][73] = 72, - [1][0][2][0][RTW89_FCC][76] = 72, - [1][0][2][0][RTW89_FCC][80] = 72, - [1][0][2][0][RTW89_FCC][84] = 72, - [1][0][2][0][RTW89_FCC][88] = 72, - [1][0][2][0][RTW89_FCC][91] = 72, - [1][0][2][0][RTW89_FCC][95] = 72, - [1][0][2][0][RTW89_FCC][99] = 72, - [1][0][2][0][RTW89_FCC][103] = 72, - [1][0][2][0][RTW89_FCC][106] = 72, + [0][1][2][1][RTW89_ETSI][119] = 127, + [0][1][2][1][RTW89_KCC][119] = 127, + [1][0][2][0][RTW89_FCC][1] = 34, + [1][0][2][0][RTW89_ETSI][1] = 66, + [1][0][2][0][RTW89_KCC][1] = 40, + [1][0][2][0][RTW89_FCC][5] = 34, + [1][0][2][0][RTW89_ETSI][5] = 66, + [1][0][2][0][RTW89_KCC][5] = 40, + [1][0][2][0][RTW89_FCC][9] = 34, + [1][0][2][0][RTW89_ETSI][9] = 66, + [1][0][2][0][RTW89_KCC][9] = 40, + [1][0][2][0][RTW89_FCC][13] = 34, + [1][0][2][0][RTW89_ETSI][13] = 66, + [1][0][2][0][RTW89_KCC][13] = 40, + [1][0][2][0][RTW89_FCC][16] = 34, + [1][0][2][0][RTW89_ETSI][16] = 66, + [1][0][2][0][RTW89_KCC][16] = 40, + [1][0][2][0][RTW89_FCC][20] = 34, + [1][0][2][0][RTW89_ETSI][20] = 66, + [1][0][2][0][RTW89_KCC][20] = 40, + [1][0][2][0][RTW89_FCC][24] = 36, + [1][0][2][0][RTW89_ETSI][24] = 66, + [1][0][2][0][RTW89_KCC][24] = 40, + [1][0][2][0][RTW89_FCC][28] = 34, + [1][0][2][0][RTW89_ETSI][28] = 66, + [1][0][2][0][RTW89_KCC][28] = 40, + [1][0][2][0][RTW89_FCC][31] = 34, + [1][0][2][0][RTW89_ETSI][31] = 66, + [1][0][2][0][RTW89_KCC][31] = 40, + [1][0][2][0][RTW89_FCC][35] = 34, + [1][0][2][0][RTW89_ETSI][35] = 66, + [1][0][2][0][RTW89_KCC][35] = 40, + [1][0][2][0][RTW89_FCC][39] = 34, + [1][0][2][0][RTW89_ETSI][39] = 66, + [1][0][2][0][RTW89_KCC][39] = 40, + [1][0][2][0][RTW89_FCC][43] = 34, + [1][0][2][0][RTW89_ETSI][43] = 66, + [1][0][2][0][RTW89_KCC][43] = 40, + [1][0][2][0][RTW89_FCC][46] = 34, + [1][0][2][0][RTW89_ETSI][46] = 127, + [1][0][2][0][RTW89_KCC][46] = 40, + [1][0][2][0][RTW89_FCC][50] = 34, + [1][0][2][0][RTW89_ETSI][50] = 127, + [1][0][2][0][RTW89_KCC][50] = 40, + [1][0][2][0][RTW89_FCC][54] = 36, + [1][0][2][0][RTW89_ETSI][54] = 127, + [1][0][2][0][RTW89_KCC][54] = 40, + [1][0][2][0][RTW89_FCC][58] = 36, + [1][0][2][0][RTW89_ETSI][58] = 127, + [1][0][2][0][RTW89_KCC][58] = 40, + [1][0][2][0][RTW89_FCC][61] = 34, + [1][0][2][0][RTW89_ETSI][61] = 127, + [1][0][2][0][RTW89_KCC][61] = 40, + [1][0][2][0][RTW89_FCC][65] = 34, + [1][0][2][0][RTW89_ETSI][65] = 127, + [1][0][2][0][RTW89_KCC][65] = 40, + [1][0][2][0][RTW89_FCC][69] = 34, + [1][0][2][0][RTW89_ETSI][69] = 127, + [1][0][2][0][RTW89_KCC][69] = 40, + [1][0][2][0][RTW89_FCC][73] = 34, + [1][0][2][0][RTW89_ETSI][73] = 127, + [1][0][2][0][RTW89_KCC][73] = 40, + [1][0][2][0][RTW89_FCC][76] = 34, + [1][0][2][0][RTW89_ETSI][76] = 127, + [1][0][2][0][RTW89_KCC][76] = 40, + [1][0][2][0][RTW89_FCC][80] = 34, + [1][0][2][0][RTW89_ETSI][80] = 127, + [1][0][2][0][RTW89_KCC][80] = 42, + [1][0][2][0][RTW89_FCC][84] = 34, + [1][0][2][0][RTW89_ETSI][84] = 127, + [1][0][2][0][RTW89_KCC][84] = 42, + [1][0][2][0][RTW89_FCC][88] = 34, + [1][0][2][0][RTW89_ETSI][88] = 127, + [1][0][2][0][RTW89_KCC][88] = 42, + [1][0][2][0][RTW89_FCC][91] = 36, + [1][0][2][0][RTW89_ETSI][91] = 127, + [1][0][2][0][RTW89_KCC][91] = 42, + [1][0][2][0][RTW89_FCC][95] = 34, + [1][0][2][0][RTW89_ETSI][95] = 127, + [1][0][2][0][RTW89_KCC][95] = 42, + [1][0][2][0][RTW89_FCC][99] = 34, + [1][0][2][0][RTW89_ETSI][99] = 127, + [1][0][2][0][RTW89_KCC][99] = 42, + [1][0][2][0][RTW89_FCC][103] = 34, + [1][0][2][0][RTW89_ETSI][103] = 127, + [1][0][2][0][RTW89_KCC][103] = 42, + [1][0][2][0][RTW89_FCC][106] = 36, + [1][0][2][0][RTW89_ETSI][106] = 127, + [1][0][2][0][RTW89_KCC][106] = 42, [1][0][2][0][RTW89_FCC][110] = 127, + [1][0][2][0][RTW89_ETSI][110] = 127, + [1][0][2][0][RTW89_KCC][110] = 127, [1][0][2][0][RTW89_FCC][114] = 127, + [1][0][2][0][RTW89_ETSI][114] = 127, + [1][0][2][0][RTW89_KCC][114] = 127, [1][0][2][0][RTW89_FCC][118] = 127, - [1][1][2][0][RTW89_FCC][1] = 60, - [1][1][2][0][RTW89_FCC][5] = 60, - [1][1][2][0][RTW89_FCC][9] = 60, - [1][1][2][0][RTW89_FCC][13] = 60, - [1][1][2][0][RTW89_FCC][16] = 60, - [1][1][2][0][RTW89_FCC][20] = 60, - [1][1][2][0][RTW89_FCC][24] = 60, - [1][1][2][0][RTW89_FCC][28] = 60, - [1][1][2][0][RTW89_FCC][31] = 60, - [1][1][2][0][RTW89_FCC][35] = 60, - [1][1][2][0][RTW89_FCC][39] = 60, - [1][1][2][0][RTW89_FCC][43] = 60, - [1][1][2][0][RTW89_FCC][46] = 60, - [1][1][2][0][RTW89_FCC][50] = 60, - [1][1][2][0][RTW89_FCC][54] = 60, - [1][1][2][0][RTW89_FCC][58] = 60, - [1][1][2][0][RTW89_FCC][61] = 60, - [1][1][2][0][RTW89_FCC][65] = 60, - [1][1][2][0][RTW89_FCC][69] = 60, - [1][1][2][0][RTW89_FCC][73] = 60, - [1][1][2][0][RTW89_FCC][76] = 60, - [1][1][2][0][RTW89_FCC][80] = 60, - [1][1][2][0][RTW89_FCC][84] = 60, - [1][1][2][0][RTW89_FCC][88] = 60, - [1][1][2][0][RTW89_FCC][91] = 60, - [1][1][2][0][RTW89_FCC][95] = 60, - [1][1][2][0][RTW89_FCC][99] = 60, - [1][1][2][0][RTW89_FCC][103] = 60, - [1][1][2][0][RTW89_FCC][106] = 60, + [1][0][2][0][RTW89_ETSI][118] = 127, + [1][0][2][0][RTW89_KCC][118] = 127, + [1][1][2][0][RTW89_FCC][1] = 10, + [1][1][2][0][RTW89_ETSI][1] = 54, + [1][1][2][0][RTW89_KCC][1] = 28, + [1][1][2][0][RTW89_FCC][5] = 10, + [1][1][2][0][RTW89_ETSI][5] = 54, + [1][1][2][0][RTW89_KCC][5] = 28, + [1][1][2][0][RTW89_FCC][9] = 10, + [1][1][2][0][RTW89_ETSI][9] = 54, + [1][1][2][0][RTW89_KCC][9] = 28, + [1][1][2][0][RTW89_FCC][13] = 10, + [1][1][2][0][RTW89_ETSI][13] = 54, + [1][1][2][0][RTW89_KCC][13] = 28, + [1][1][2][0][RTW89_FCC][16] = 10, + [1][1][2][0][RTW89_ETSI][16] = 54, + [1][1][2][0][RTW89_KCC][16] = 28, + [1][1][2][0][RTW89_FCC][20] = 10, + [1][1][2][0][RTW89_ETSI][20] = 54, + [1][1][2][0][RTW89_KCC][20] = 28, + [1][1][2][0][RTW89_FCC][24] = 10, + [1][1][2][0][RTW89_ETSI][24] = 54, + [1][1][2][0][RTW89_KCC][24] = 28, + [1][1][2][0][RTW89_FCC][28] = 10, + [1][1][2][0][RTW89_ETSI][28] = 54, + [1][1][2][0][RTW89_KCC][28] = 28, + [1][1][2][0][RTW89_FCC][31] = 10, + [1][1][2][0][RTW89_ETSI][31] = 54, + [1][1][2][0][RTW89_KCC][31] = 28, + [1][1][2][0][RTW89_FCC][35] = 10, + [1][1][2][0][RTW89_ETSI][35] = 54, + [1][1][2][0][RTW89_KCC][35] = 28, + [1][1][2][0][RTW89_FCC][39] = 10, + [1][1][2][0][RTW89_ETSI][39] = 54, + [1][1][2][0][RTW89_KCC][39] = 28, + [1][1][2][0][RTW89_FCC][43] = 10, + [1][1][2][0][RTW89_ETSI][43] = 54, + [1][1][2][0][RTW89_KCC][43] = 28, + [1][1][2][0][RTW89_FCC][46] = 12, + [1][1][2][0][RTW89_ETSI][46] = 127, + [1][1][2][0][RTW89_KCC][46] = 28, + [1][1][2][0][RTW89_FCC][50] = 12, + [1][1][2][0][RTW89_ETSI][50] = 127, + [1][1][2][0][RTW89_KCC][50] = 28, + [1][1][2][0][RTW89_FCC][54] = 10, + [1][1][2][0][RTW89_ETSI][54] = 127, + [1][1][2][0][RTW89_KCC][54] = 28, + [1][1][2][0][RTW89_FCC][58] = 10, + [1][1][2][0][RTW89_ETSI][58] = 127, + [1][1][2][0][RTW89_KCC][58] = 28, + [1][1][2][0][RTW89_FCC][61] = 10, + [1][1][2][0][RTW89_ETSI][61] = 127, + [1][1][2][0][RTW89_KCC][61] = 28, + [1][1][2][0][RTW89_FCC][65] = 10, + [1][1][2][0][RTW89_ETSI][65] = 127, + [1][1][2][0][RTW89_KCC][65] = 28, + [1][1][2][0][RTW89_FCC][69] = 10, + [1][1][2][0][RTW89_ETSI][69] = 127, + [1][1][2][0][RTW89_KCC][69] = 28, + [1][1][2][0][RTW89_FCC][73] = 10, + [1][1][2][0][RTW89_ETSI][73] = 127, + [1][1][2][0][RTW89_KCC][73] = 28, + [1][1][2][0][RTW89_FCC][76] = 10, + [1][1][2][0][RTW89_ETSI][76] = 127, + [1][1][2][0][RTW89_KCC][76] = 28, + [1][1][2][0][RTW89_FCC][80] = 10, + [1][1][2][0][RTW89_ETSI][80] = 127, + [1][1][2][0][RTW89_KCC][80] = 32, + [1][1][2][0][RTW89_FCC][84] = 10, + [1][1][2][0][RTW89_ETSI][84] = 127, + [1][1][2][0][RTW89_KCC][84] = 32, + [1][1][2][0][RTW89_FCC][88] = 10, + [1][1][2][0][RTW89_ETSI][88] = 127, + [1][1][2][0][RTW89_KCC][88] = 32, + [1][1][2][0][RTW89_FCC][91] = 12, + [1][1][2][0][RTW89_ETSI][91] = 127, + [1][1][2][0][RTW89_KCC][91] = 32, + [1][1][2][0][RTW89_FCC][95] = 10, + [1][1][2][0][RTW89_ETSI][95] = 127, + [1][1][2][0][RTW89_KCC][95] = 32, + [1][1][2][0][RTW89_FCC][99] = 10, + [1][1][2][0][RTW89_ETSI][99] = 127, + [1][1][2][0][RTW89_KCC][99] = 32, + [1][1][2][0][RTW89_FCC][103] = 10, + [1][1][2][0][RTW89_ETSI][103] = 127, + [1][1][2][0][RTW89_KCC][103] = 32, + [1][1][2][0][RTW89_FCC][106] = 12, + [1][1][2][0][RTW89_ETSI][106] = 127, + [1][1][2][0][RTW89_KCC][106] = 32, [1][1][2][0][RTW89_FCC][110] = 127, + [1][1][2][0][RTW89_ETSI][110] = 127, + [1][1][2][0][RTW89_KCC][110] = 127, [1][1][2][0][RTW89_FCC][114] = 127, + [1][1][2][0][RTW89_ETSI][114] = 127, + [1][1][2][0][RTW89_KCC][114] = 127, [1][1][2][0][RTW89_FCC][118] = 127, - [1][1][2][1][RTW89_FCC][1] = 48, - [1][1][2][1][RTW89_FCC][5] = 48, - [1][1][2][1][RTW89_FCC][9] = 48, - [1][1][2][1][RTW89_FCC][13] = 48, - [1][1][2][1][RTW89_FCC][16] = 48, - [1][1][2][1][RTW89_FCC][20] = 48, - [1][1][2][1][RTW89_FCC][24] = 48, - [1][1][2][1][RTW89_FCC][28] = 48, - [1][1][2][1][RTW89_FCC][31] = 48, - [1][1][2][1][RTW89_FCC][35] = 48, - [1][1][2][1][RTW89_FCC][39] = 48, - [1][1][2][1][RTW89_FCC][43] = 48, - [1][1][2][1][RTW89_FCC][46] = 48, - [1][1][2][1][RTW89_FCC][50] = 48, - [1][1][2][1][RTW89_FCC][54] = 48, - [1][1][2][1][RTW89_FCC][58] = 48, - [1][1][2][1][RTW89_FCC][61] = 48, - [1][1][2][1][RTW89_FCC][65] = 48, - [1][1][2][1][RTW89_FCC][69] = 48, - [1][1][2][1][RTW89_FCC][73] = 48, - [1][1][2][1][RTW89_FCC][76] = 48, - [1][1][2][1][RTW89_FCC][80] = 48, - [1][1][2][1][RTW89_FCC][84] = 48, - [1][1][2][1][RTW89_FCC][88] = 48, - [1][1][2][1][RTW89_FCC][91] = 48, - [1][1][2][1][RTW89_FCC][95] = 48, - [1][1][2][1][RTW89_FCC][99] = 48, - [1][1][2][1][RTW89_FCC][103] = 48, - [1][1][2][1][RTW89_FCC][106] = 48, + [1][1][2][0][RTW89_ETSI][118] = 127, + [1][1][2][0][RTW89_KCC][118] = 127, + [1][1][2][1][RTW89_FCC][1] = 10, + [1][1][2][1][RTW89_ETSI][1] = 42, + [1][1][2][1][RTW89_KCC][1] = 28, + [1][1][2][1][RTW89_FCC][5] = 10, + [1][1][2][1][RTW89_ETSI][5] = 42, + [1][1][2][1][RTW89_KCC][5] = 28, + [1][1][2][1][RTW89_FCC][9] = 10, + [1][1][2][1][RTW89_ETSI][9] = 42, + [1][1][2][1][RTW89_KCC][9] = 28, + [1][1][2][1][RTW89_FCC][13] = 10, + [1][1][2][1][RTW89_ETSI][13] = 42, + [1][1][2][1][RTW89_KCC][13] = 28, + [1][1][2][1][RTW89_FCC][16] = 10, + [1][1][2][1][RTW89_ETSI][16] = 42, + [1][1][2][1][RTW89_KCC][16] = 28, + [1][1][2][1][RTW89_FCC][20] = 10, + [1][1][2][1][RTW89_ETSI][20] = 42, + [1][1][2][1][RTW89_KCC][20] = 28, + [1][1][2][1][RTW89_FCC][24] = 10, + [1][1][2][1][RTW89_ETSI][24] = 42, + [1][1][2][1][RTW89_KCC][24] = 28, + [1][1][2][1][RTW89_FCC][28] = 10, + [1][1][2][1][RTW89_ETSI][28] = 42, + [1][1][2][1][RTW89_KCC][28] = 28, + [1][1][2][1][RTW89_FCC][31] = 10, + [1][1][2][1][RTW89_ETSI][31] = 42, + [1][1][2][1][RTW89_KCC][31] = 28, + [1][1][2][1][RTW89_FCC][35] = 10, + [1][1][2][1][RTW89_ETSI][35] = 42, + [1][1][2][1][RTW89_KCC][35] = 28, + [1][1][2][1][RTW89_FCC][39] = 10, + [1][1][2][1][RTW89_ETSI][39] = 42, + [1][1][2][1][RTW89_KCC][39] = 28, + [1][1][2][1][RTW89_FCC][43] = 10, + [1][1][2][1][RTW89_ETSI][43] = 42, + [1][1][2][1][RTW89_KCC][43] = 28, + [1][1][2][1][RTW89_FCC][46] = 12, + [1][1][2][1][RTW89_ETSI][46] = 127, + [1][1][2][1][RTW89_KCC][46] = 28, + [1][1][2][1][RTW89_FCC][50] = 12, + [1][1][2][1][RTW89_ETSI][50] = 127, + [1][1][2][1][RTW89_KCC][50] = 28, + [1][1][2][1][RTW89_FCC][54] = 10, + [1][1][2][1][RTW89_ETSI][54] = 127, + [1][1][2][1][RTW89_KCC][54] = 28, + [1][1][2][1][RTW89_FCC][58] = 10, + [1][1][2][1][RTW89_ETSI][58] = 127, + [1][1][2][1][RTW89_KCC][58] = 28, + [1][1][2][1][RTW89_FCC][61] = 10, + [1][1][2][1][RTW89_ETSI][61] = 127, + [1][1][2][1][RTW89_KCC][61] = 28, + [1][1][2][1][RTW89_FCC][65] = 10, + [1][1][2][1][RTW89_ETSI][65] = 127, + [1][1][2][1][RTW89_KCC][65] = 28, + [1][1][2][1][RTW89_FCC][69] = 10, + [1][1][2][1][RTW89_ETSI][69] = 127, + [1][1][2][1][RTW89_KCC][69] = 28, + [1][1][2][1][RTW89_FCC][73] = 10, + [1][1][2][1][RTW89_ETSI][73] = 127, + [1][1][2][1][RTW89_KCC][73] = 28, + [1][1][2][1][RTW89_FCC][76] = 10, + [1][1][2][1][RTW89_ETSI][76] = 127, + [1][1][2][1][RTW89_KCC][76] = 28, + [1][1][2][1][RTW89_FCC][80] = 10, + [1][1][2][1][RTW89_ETSI][80] = 127, + [1][1][2][1][RTW89_KCC][80] = 32, + [1][1][2][1][RTW89_FCC][84] = 10, + [1][1][2][1][RTW89_ETSI][84] = 127, + [1][1][2][1][RTW89_KCC][84] = 32, + [1][1][2][1][RTW89_FCC][88] = 10, + [1][1][2][1][RTW89_ETSI][88] = 127, + [1][1][2][1][RTW89_KCC][88] = 32, + [1][1][2][1][RTW89_FCC][91] = 12, + [1][1][2][1][RTW89_ETSI][91] = 127, + [1][1][2][1][RTW89_KCC][91] = 32, + [1][1][2][1][RTW89_FCC][95] = 10, + [1][1][2][1][RTW89_ETSI][95] = 127, + [1][1][2][1][RTW89_KCC][95] = 32, + [1][1][2][1][RTW89_FCC][99] = 10, + [1][1][2][1][RTW89_ETSI][99] = 127, + [1][1][2][1][RTW89_KCC][99] = 32, + [1][1][2][1][RTW89_FCC][103] = 10, + [1][1][2][1][RTW89_ETSI][103] = 127, + [1][1][2][1][RTW89_KCC][103] = 32, + [1][1][2][1][RTW89_FCC][106] = 12, + [1][1][2][1][RTW89_ETSI][106] = 127, + [1][1][2][1][RTW89_KCC][106] = 32, [1][1][2][1][RTW89_FCC][110] = 127, + [1][1][2][1][RTW89_ETSI][110] = 127, + [1][1][2][1][RTW89_KCC][110] = 127, [1][1][2][1][RTW89_FCC][114] = 127, + [1][1][2][1][RTW89_ETSI][114] = 127, + [1][1][2][1][RTW89_KCC][114] = 127, [1][1][2][1][RTW89_FCC][118] = 127, - [2][0][2][0][RTW89_FCC][3] = 64, - [2][0][2][0][RTW89_FCC][11] = 64, - [2][0][2][0][RTW89_FCC][18] = 64, - [2][0][2][0][RTW89_FCC][26] = 64, - [2][0][2][0][RTW89_FCC][33] = 64, - [2][0][2][0][RTW89_FCC][41] = 64, - [2][0][2][0][RTW89_FCC][48] = 64, - [2][0][2][0][RTW89_FCC][56] = 64, - [2][0][2][0][RTW89_FCC][63] = 64, - [2][0][2][0][RTW89_FCC][71] = 64, - [2][0][2][0][RTW89_FCC][78] = 64, - [2][0][2][0][RTW89_FCC][86] = 64, - [2][0][2][0][RTW89_FCC][93] = 64, - [2][0][2][0][RTW89_FCC][101] = 64, + [1][1][2][1][RTW89_ETSI][118] = 127, + [1][1][2][1][RTW89_KCC][118] = 127, + [2][0][2][0][RTW89_FCC][3] = 46, + [2][0][2][0][RTW89_ETSI][3] = 48, + [2][0][2][0][RTW89_KCC][3] = 50, + [2][0][2][0][RTW89_FCC][11] = 46, + [2][0][2][0][RTW89_ETSI][11] = 48, + [2][0][2][0][RTW89_KCC][11] = 50, + [2][0][2][0][RTW89_FCC][18] = 46, + [2][0][2][0][RTW89_ETSI][18] = 48, + [2][0][2][0][RTW89_KCC][18] = 50, + [2][0][2][0][RTW89_FCC][26] = 46, + [2][0][2][0][RTW89_ETSI][26] = 48, + [2][0][2][0][RTW89_KCC][26] = 50, + [2][0][2][0][RTW89_FCC][33] = 46, + [2][0][2][0][RTW89_ETSI][33] = 48, + [2][0][2][0][RTW89_KCC][33] = 50, + [2][0][2][0][RTW89_FCC][41] = 46, + [2][0][2][0][RTW89_ETSI][41] = 48, + [2][0][2][0][RTW89_KCC][41] = 50, + [2][0][2][0][RTW89_FCC][48] = 46, + [2][0][2][0][RTW89_ETSI][48] = 127, + [2][0][2][0][RTW89_KCC][48] = 48, + [2][0][2][0][RTW89_FCC][56] = 46, + [2][0][2][0][RTW89_ETSI][56] = 127, + [2][0][2][0][RTW89_KCC][56] = 48, + [2][0][2][0][RTW89_FCC][63] = 46, + [2][0][2][0][RTW89_ETSI][63] = 127, + [2][0][2][0][RTW89_KCC][63] = 48, + [2][0][2][0][RTW89_FCC][71] = 46, + [2][0][2][0][RTW89_ETSI][71] = 127, + [2][0][2][0][RTW89_KCC][71] = 48, + [2][0][2][0][RTW89_FCC][78] = 46, + [2][0][2][0][RTW89_ETSI][78] = 127, + [2][0][2][0][RTW89_KCC][78] = 52, + [2][0][2][0][RTW89_FCC][86] = 46, + [2][0][2][0][RTW89_ETSI][86] = 127, + [2][0][2][0][RTW89_KCC][86] = 52, + [2][0][2][0][RTW89_FCC][93] = 46, + [2][0][2][0][RTW89_ETSI][93] = 127, + [2][0][2][0][RTW89_KCC][93] = 50, + [2][0][2][0][RTW89_FCC][101] = 44, + [2][0][2][0][RTW89_ETSI][101] = 127, + [2][0][2][0][RTW89_KCC][101] = 50, [2][0][2][0][RTW89_FCC][108] = 127, + [2][0][2][0][RTW89_ETSI][108] = 127, + [2][0][2][0][RTW89_KCC][108] = 127, [2][0][2][0][RTW89_FCC][116] = 127, - [2][1][2][0][RTW89_FCC][3] = 52, - [2][1][2][0][RTW89_FCC][11] = 52, - [2][1][2][0][RTW89_FCC][18] = 52, - [2][1][2][0][RTW89_FCC][26] = 52, - [2][1][2][0][RTW89_FCC][33] = 52, - [2][1][2][0][RTW89_FCC][41] = 52, - [2][1][2][0][RTW89_FCC][48] = 52, - [2][1][2][0][RTW89_FCC][56] = 52, - [2][1][2][0][RTW89_FCC][63] = 52, - [2][1][2][0][RTW89_FCC][71] = 52, - [2][1][2][0][RTW89_FCC][78] = 52, - [2][1][2][0][RTW89_FCC][86] = 52, - [2][1][2][0][RTW89_FCC][93] = 52, - [2][1][2][0][RTW89_FCC][101] = 52, + [2][0][2][0][RTW89_ETSI][116] = 127, + [2][0][2][0][RTW89_KCC][116] = 127, + [2][1][2][0][RTW89_FCC][3] = 22, + [2][1][2][0][RTW89_ETSI][3] = 48, + [2][1][2][0][RTW89_KCC][3] = 38, + [2][1][2][0][RTW89_FCC][11] = 20, + [2][1][2][0][RTW89_ETSI][11] = 48, + [2][1][2][0][RTW89_KCC][11] = 38, + [2][1][2][0][RTW89_FCC][18] = 20, + [2][1][2][0][RTW89_ETSI][18] = 48, + [2][1][2][0][RTW89_KCC][18] = 38, + [2][1][2][0][RTW89_FCC][26] = 20, + [2][1][2][0][RTW89_ETSI][26] = 48, + [2][1][2][0][RTW89_KCC][26] = 38, + [2][1][2][0][RTW89_FCC][33] = 20, + [2][1][2][0][RTW89_ETSI][33] = 48, + [2][1][2][0][RTW89_KCC][33] = 38, + [2][1][2][0][RTW89_FCC][41] = 22, + [2][1][2][0][RTW89_ETSI][41] = 48, + [2][1][2][0][RTW89_KCC][41] = 38, + [2][1][2][0][RTW89_FCC][48] = 22, + [2][1][2][0][RTW89_ETSI][48] = 127, + [2][1][2][0][RTW89_KCC][48] = 38, + [2][1][2][0][RTW89_FCC][56] = 20, + [2][1][2][0][RTW89_ETSI][56] = 127, + [2][1][2][0][RTW89_KCC][56] = 38, + [2][1][2][0][RTW89_FCC][63] = 22, + [2][1][2][0][RTW89_ETSI][63] = 127, + [2][1][2][0][RTW89_KCC][63] = 38, + [2][1][2][0][RTW89_FCC][71] = 20, + [2][1][2][0][RTW89_ETSI][71] = 127, + [2][1][2][0][RTW89_KCC][71] = 38, + [2][1][2][0][RTW89_FCC][78] = 20, + [2][1][2][0][RTW89_ETSI][78] = 127, + [2][1][2][0][RTW89_KCC][78] = 38, + [2][1][2][0][RTW89_FCC][86] = 20, + [2][1][2][0][RTW89_ETSI][86] = 127, + [2][1][2][0][RTW89_KCC][86] = 38, + [2][1][2][0][RTW89_FCC][93] = 22, + [2][1][2][0][RTW89_ETSI][93] = 127, + [2][1][2][0][RTW89_KCC][93] = 38, + [2][1][2][0][RTW89_FCC][101] = 22, + [2][1][2][0][RTW89_ETSI][101] = 127, + [2][1][2][0][RTW89_KCC][101] = 38, [2][1][2][0][RTW89_FCC][108] = 127, + [2][1][2][0][RTW89_ETSI][108] = 127, + [2][1][2][0][RTW89_KCC][108] = 127, [2][1][2][0][RTW89_FCC][116] = 127, - [2][1][2][1][RTW89_FCC][3] = 40, - [2][1][2][1][RTW89_FCC][11] = 40, - [2][1][2][1][RTW89_FCC][18] = 40, - [2][1][2][1][RTW89_FCC][26] = 40, - [2][1][2][1][RTW89_FCC][33] = 40, - [2][1][2][1][RTW89_FCC][41] = 40, - [2][1][2][1][RTW89_FCC][48] = 40, - [2][1][2][1][RTW89_FCC][56] = 40, - [2][1][2][1][RTW89_FCC][63] = 40, - [2][1][2][1][RTW89_FCC][71] = 40, - [2][1][2][1][RTW89_FCC][78] = 40, - [2][1][2][1][RTW89_FCC][86] = 40, - [2][1][2][1][RTW89_FCC][93] = 40, - [2][1][2][1][RTW89_FCC][101] = 40, + [2][1][2][0][RTW89_ETSI][116] = 127, + [2][1][2][0][RTW89_KCC][116] = 127, + [2][1][2][1][RTW89_FCC][3] = 22, + [2][1][2][1][RTW89_ETSI][3] = 42, + [2][1][2][1][RTW89_KCC][3] = 38, + [2][1][2][1][RTW89_FCC][11] = 20, + [2][1][2][1][RTW89_ETSI][11] = 42, + [2][1][2][1][RTW89_KCC][11] = 38, + [2][1][2][1][RTW89_FCC][18] = 20, + [2][1][2][1][RTW89_ETSI][18] = 42, + [2][1][2][1][RTW89_KCC][18] = 38, + [2][1][2][1][RTW89_FCC][26] = 20, + [2][1][2][1][RTW89_ETSI][26] = 42, + [2][1][2][1][RTW89_KCC][26] = 38, + [2][1][2][1][RTW89_FCC][33] = 20, + [2][1][2][1][RTW89_ETSI][33] = 42, + [2][1][2][1][RTW89_KCC][33] = 38, + [2][1][2][1][RTW89_FCC][41] = 22, + [2][1][2][1][RTW89_ETSI][41] = 42, + [2][1][2][1][RTW89_KCC][41] = 38, + [2][1][2][1][RTW89_FCC][48] = 22, + [2][1][2][1][RTW89_ETSI][48] = 127, + [2][1][2][1][RTW89_KCC][48] = 38, + [2][1][2][1][RTW89_FCC][56] = 20, + [2][1][2][1][RTW89_ETSI][56] = 127, + [2][1][2][1][RTW89_KCC][56] = 38, + [2][1][2][1][RTW89_FCC][63] = 22, + [2][1][2][1][RTW89_ETSI][63] = 127, + [2][1][2][1][RTW89_KCC][63] = 38, + [2][1][2][1][RTW89_FCC][71] = 20, + [2][1][2][1][RTW89_ETSI][71] = 127, + [2][1][2][1][RTW89_KCC][71] = 38, + [2][1][2][1][RTW89_FCC][78] = 20, + [2][1][2][1][RTW89_ETSI][78] = 127, + [2][1][2][1][RTW89_KCC][78] = 38, + [2][1][2][1][RTW89_FCC][86] = 20, + [2][1][2][1][RTW89_ETSI][86] = 127, + [2][1][2][1][RTW89_KCC][86] = 38, + [2][1][2][1][RTW89_FCC][93] = 22, + [2][1][2][1][RTW89_ETSI][93] = 127, + [2][1][2][1][RTW89_KCC][93] = 38, + [2][1][2][1][RTW89_FCC][101] = 22, + [2][1][2][1][RTW89_ETSI][101] = 127, + [2][1][2][1][RTW89_KCC][101] = 38, [2][1][2][1][RTW89_FCC][108] = 127, + [2][1][2][1][RTW89_ETSI][108] = 127, + [2][1][2][1][RTW89_KCC][108] = 127, [2][1][2][1][RTW89_FCC][116] = 127, - [3][0][2][0][RTW89_FCC][7] = 56, - [3][0][2][0][RTW89_FCC][22] = 56, - [3][0][2][0][RTW89_FCC][37] = 56, - [3][0][2][0][RTW89_FCC][52] = 56, - [3][0][2][0][RTW89_FCC][67] = 56, - [3][0][2][0][RTW89_FCC][82] = 56, - [3][0][2][0][RTW89_FCC][97] = 56, + [2][1][2][1][RTW89_ETSI][116] = 127, + [2][1][2][1][RTW89_KCC][116] = 127, + [3][0][2][0][RTW89_FCC][7] = 52, + [3][0][2][0][RTW89_ETSI][7] = 38, + [3][0][2][0][RTW89_KCC][7] = 42, + [3][0][2][0][RTW89_FCC][22] = 52, + [3][0][2][0][RTW89_ETSI][22] = 38, + [3][0][2][0][RTW89_KCC][22] = 42, + [3][0][2][0][RTW89_FCC][37] = 52, + [3][0][2][0][RTW89_ETSI][37] = 38, + [3][0][2][0][RTW89_KCC][37] = 42, + [3][0][2][0][RTW89_FCC][52] = 54, + [3][0][2][0][RTW89_ETSI][52] = 127, + [3][0][2][0][RTW89_KCC][52] = 56, + [3][0][2][0][RTW89_FCC][67] = 54, + [3][0][2][0][RTW89_ETSI][67] = 127, + [3][0][2][0][RTW89_KCC][67] = 54, + [3][0][2][0][RTW89_FCC][82] = 54, + [3][0][2][0][RTW89_ETSI][82] = 127, + [3][0][2][0][RTW89_KCC][82] = 26, + [3][0][2][0][RTW89_FCC][97] = 40, + [3][0][2][0][RTW89_ETSI][97] = 127, + [3][0][2][0][RTW89_KCC][97] = 26, [3][0][2][0][RTW89_FCC][112] = 127, - [3][1][2][0][RTW89_FCC][7] = 44, - [3][1][2][0][RTW89_FCC][22] = 44, - [3][1][2][0][RTW89_FCC][37] = 44, - [3][1][2][0][RTW89_FCC][52] = 44, - [3][1][2][0][RTW89_FCC][67] = 44, - [3][1][2][0][RTW89_FCC][82] = 44, - [3][1][2][0][RTW89_FCC][97] = 44, + [3][0][2][0][RTW89_ETSI][112] = 127, + [3][0][2][0][RTW89_KCC][112] = 127, + [3][1][2][0][RTW89_FCC][7] = 32, + [3][1][2][0][RTW89_ETSI][7] = 38, + [3][1][2][0][RTW89_KCC][7] = 40, + [3][1][2][0][RTW89_FCC][22] = 30, + [3][1][2][0][RTW89_ETSI][22] = 38, + [3][1][2][0][RTW89_KCC][22] = 40, + [3][1][2][0][RTW89_FCC][37] = 30, + [3][1][2][0][RTW89_ETSI][37] = 38, + [3][1][2][0][RTW89_KCC][37] = 40, + [3][1][2][0][RTW89_FCC][52] = 30, + [3][1][2][0][RTW89_ETSI][52] = 127, + [3][1][2][0][RTW89_KCC][52] = 48, + [3][1][2][0][RTW89_FCC][67] = 32, + [3][1][2][0][RTW89_ETSI][67] = 127, + [3][1][2][0][RTW89_KCC][67] = 48, + [3][1][2][0][RTW89_FCC][82] = 32, + [3][1][2][0][RTW89_ETSI][82] = 127, + [3][1][2][0][RTW89_KCC][82] = 24, + [3][1][2][0][RTW89_FCC][97] = 14, + [3][1][2][0][RTW89_ETSI][97] = 127, + [3][1][2][0][RTW89_KCC][97] = 24, [3][1][2][0][RTW89_FCC][112] = 127, + [3][1][2][0][RTW89_ETSI][112] = 127, + [3][1][2][0][RTW89_KCC][112] = 127, [3][1][2][1][RTW89_FCC][7] = 32, - [3][1][2][1][RTW89_FCC][22] = 32, - [3][1][2][1][RTW89_FCC][37] = 32, - [3][1][2][1][RTW89_FCC][52] = 32, + [3][1][2][1][RTW89_ETSI][7] = 38, + [3][1][2][1][RTW89_KCC][7] = 40, + [3][1][2][1][RTW89_FCC][22] = 30, + [3][1][2][1][RTW89_ETSI][22] = 38, + [3][1][2][1][RTW89_KCC][22] = 40, + [3][1][2][1][RTW89_FCC][37] = 30, + [3][1][2][1][RTW89_ETSI][37] = 38, + [3][1][2][1][RTW89_KCC][37] = 40, + [3][1][2][1][RTW89_FCC][52] = 30, + [3][1][2][1][RTW89_ETSI][52] = 127, + [3][1][2][1][RTW89_KCC][52] = 48, [3][1][2][1][RTW89_FCC][67] = 32, + [3][1][2][1][RTW89_ETSI][67] = 127, + [3][1][2][1][RTW89_KCC][67] = 48, [3][1][2][1][RTW89_FCC][82] = 32, - [3][1][2][1][RTW89_FCC][97] = 32, + [3][1][2][1][RTW89_ETSI][82] = 127, + [3][1][2][1][RTW89_KCC][82] = 24, + [3][1][2][1][RTW89_FCC][97] = 14, + [3][1][2][1][RTW89_ETSI][97] = 127, + [3][1][2][1][RTW89_KCC][97] = 24, [3][1][2][1][RTW89_FCC][112] = 127, + [3][1][2][1][RTW89_ETSI][112] = 127, + [3][1][2][1][RTW89_KCC][112] = 127, }; const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] @@ -17126,8 +33220,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [0][0][RTW89_WW][8] = 32, [0][0][RTW89_WW][9] = 32, [0][0][RTW89_WW][10] = 32, - [0][0][RTW89_WW][11] = 32, - [0][0][RTW89_WW][12] = 24, + [0][0][RTW89_WW][11] = 26, + [0][0][RTW89_WW][12] = -20, [0][0][RTW89_WW][13] = 0, [0][1][RTW89_WW][0] = 20, [0][1][RTW89_WW][1] = 22, @@ -17141,7 +33235,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [0][1][RTW89_WW][9] = 22, [0][1][RTW89_WW][10] = 22, [0][1][RTW89_WW][11] = 22, - [0][1][RTW89_WW][12] = 20, + [0][1][RTW89_WW][12] = -30, [0][1][RTW89_WW][13] = 0, [1][0][RTW89_WW][0] = 42, [1][0][RTW89_WW][1] = 44, @@ -17154,8 +33248,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [1][0][RTW89_WW][8] = 44, [1][0][RTW89_WW][9] = 44, [1][0][RTW89_WW][10] = 44, - [1][0][RTW89_WW][11] = 42, - [1][0][RTW89_WW][12] = 30, + [1][0][RTW89_WW][11] = 36, + [1][0][RTW89_WW][12] = 4, [1][0][RTW89_WW][13] = 0, [1][1][RTW89_WW][0] = 32, [1][1][RTW89_WW][1] = 32, @@ -17169,7 +33263,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [1][1][RTW89_WW][9] = 32, [1][1][RTW89_WW][10] = 32, [1][1][RTW89_WW][11] = 30, - [1][1][RTW89_WW][12] = 24, + [1][1][RTW89_WW][12] = -6, [1][1][RTW89_WW][13] = 0, [2][0][RTW89_WW][0] = 56, [2][0][RTW89_WW][1] = 56, @@ -17182,8 +33276,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [2][0][RTW89_WW][8] = 56, [2][0][RTW89_WW][9] = 56, [2][0][RTW89_WW][10] = 56, - [2][0][RTW89_WW][11] = 42, - [2][0][RTW89_WW][12] = 38, + [2][0][RTW89_WW][11] = 48, + [2][0][RTW89_WW][12] = 16, [2][0][RTW89_WW][13] = 0, [2][1][RTW89_WW][0] = 44, [2][1][RTW89_WW][1] = 44, @@ -17196,2213 +33290,3353 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [2][1][RTW89_WW][8] = 44, [2][1][RTW89_WW][9] = 44, [2][1][RTW89_WW][10] = 44, - [2][1][RTW89_WW][11] = 30, - [2][1][RTW89_WW][12] = 26, + [2][1][RTW89_WW][11] = 44, + [2][1][RTW89_WW][12] = 6, [2][1][RTW89_WW][13] = 0, [0][0][RTW89_FCC][0] = 60, [0][0][RTW89_ETSI][0] = 34, [0][0][RTW89_MKK][0] = 36, - [0][0][RTW89_IC][0] = 68, - [0][0][RTW89_ACMA][0] = 32, + [0][0][RTW89_IC][0] = 60, + [0][0][RTW89_KCC][0] = 42, + [0][0][RTW89_ACMA][0] = 34, + [0][0][RTW89_CN][0] = 32, + [0][0][RTW89_UK][0] = 34, [0][0][RTW89_FCC][1] = 60, [0][0][RTW89_ETSI][1] = 38, [0][0][RTW89_MKK][1] = 40, - [0][0][RTW89_IC][1] = 68, - [0][0][RTW89_ACMA][1] = 32, + [0][0][RTW89_IC][1] = 60, + [0][0][RTW89_KCC][1] = 42, + [0][0][RTW89_ACMA][1] = 38, + [0][0][RTW89_CN][1] = 32, + [0][0][RTW89_UK][1] = 38, [0][0][RTW89_FCC][2] = 64, [0][0][RTW89_ETSI][2] = 38, [0][0][RTW89_MKK][2] = 40, - [0][0][RTW89_IC][2] = 72, - [0][0][RTW89_ACMA][2] = 32, + [0][0][RTW89_IC][2] = 64, + [0][0][RTW89_KCC][2] = 42, + [0][0][RTW89_ACMA][2] = 38, + [0][0][RTW89_CN][2] = 32, + [0][0][RTW89_UK][2] = 38, [0][0][RTW89_FCC][3] = 68, [0][0][RTW89_ETSI][3] = 38, [0][0][RTW89_MKK][3] = 40, - [0][0][RTW89_IC][3] = 76, - [0][0][RTW89_ACMA][3] = 32, + [0][0][RTW89_IC][3] = 68, + [0][0][RTW89_KCC][3] = 42, + [0][0][RTW89_ACMA][3] = 38, + [0][0][RTW89_CN][3] = 32, + [0][0][RTW89_UK][3] = 38, [0][0][RTW89_FCC][4] = 68, [0][0][RTW89_ETSI][4] = 38, [0][0][RTW89_MKK][4] = 40, - [0][0][RTW89_IC][4] = 76, - [0][0][RTW89_ACMA][4] = 32, - [0][0][RTW89_FCC][5] = 76, + [0][0][RTW89_IC][4] = 68, + [0][0][RTW89_KCC][4] = 42, + [0][0][RTW89_ACMA][4] = 38, + [0][0][RTW89_CN][4] = 32, + [0][0][RTW89_UK][4] = 38, + [0][0][RTW89_FCC][5] = 78, [0][0][RTW89_ETSI][5] = 38, [0][0][RTW89_MKK][5] = 40, - [0][0][RTW89_IC][5] = 84, - [0][0][RTW89_ACMA][5] = 32, - [0][0][RTW89_FCC][6] = 66, + [0][0][RTW89_IC][5] = 78, + [0][0][RTW89_KCC][5] = 42, + [0][0][RTW89_ACMA][5] = 38, + [0][0][RTW89_CN][5] = 32, + [0][0][RTW89_UK][5] = 38, + [0][0][RTW89_FCC][6] = 54, [0][0][RTW89_ETSI][6] = 38, [0][0][RTW89_MKK][6] = 40, - [0][0][RTW89_IC][6] = 74, - [0][0][RTW89_ACMA][6] = 32, - [0][0][RTW89_FCC][7] = 66, + [0][0][RTW89_IC][6] = 54, + [0][0][RTW89_KCC][6] = 42, + [0][0][RTW89_ACMA][6] = 38, + [0][0][RTW89_CN][6] = 32, + [0][0][RTW89_UK][6] = 38, + [0][0][RTW89_FCC][7] = 54, [0][0][RTW89_ETSI][7] = 38, [0][0][RTW89_MKK][7] = 40, - [0][0][RTW89_IC][7] = 74, - [0][0][RTW89_ACMA][7] = 32, - [0][0][RTW89_FCC][8] = 62, + [0][0][RTW89_IC][7] = 54, + [0][0][RTW89_KCC][7] = 42, + [0][0][RTW89_ACMA][7] = 38, + [0][0][RTW89_CN][7] = 32, + [0][0][RTW89_UK][7] = 38, + [0][0][RTW89_FCC][8] = 50, [0][0][RTW89_ETSI][8] = 38, [0][0][RTW89_MKK][8] = 40, - [0][0][RTW89_IC][8] = 70, - [0][0][RTW89_ACMA][8] = 32, - [0][0][RTW89_FCC][9] = 58, + [0][0][RTW89_IC][8] = 50, + [0][0][RTW89_KCC][8] = 42, + [0][0][RTW89_ACMA][8] = 38, + [0][0][RTW89_CN][8] = 32, + [0][0][RTW89_UK][8] = 38, + [0][0][RTW89_FCC][9] = 46, [0][0][RTW89_ETSI][9] = 38, [0][0][RTW89_MKK][9] = 40, - [0][0][RTW89_IC][9] = 66, - [0][0][RTW89_ACMA][9] = 32, - [0][0][RTW89_FCC][10] = 58, + [0][0][RTW89_IC][9] = 46, + [0][0][RTW89_KCC][9] = 40, + [0][0][RTW89_ACMA][9] = 38, + [0][0][RTW89_CN][9] = 32, + [0][0][RTW89_UK][9] = 38, + [0][0][RTW89_FCC][10] = 46, [0][0][RTW89_ETSI][10] = 38, [0][0][RTW89_MKK][10] = 40, - [0][0][RTW89_IC][10] = 66, - [0][0][RTW89_ACMA][10] = 32, - [0][0][RTW89_FCC][11] = 42, + [0][0][RTW89_IC][10] = 46, + [0][0][RTW89_KCC][10] = 40, + [0][0][RTW89_ACMA][10] = 38, + [0][0][RTW89_CN][10] = 32, + [0][0][RTW89_UK][10] = 38, + [0][0][RTW89_FCC][11] = 26, [0][0][RTW89_ETSI][11] = 38, [0][0][RTW89_MKK][11] = 40, - [0][0][RTW89_IC][11] = 56, - [0][0][RTW89_ACMA][11] = 32, - [0][0][RTW89_FCC][12] = 24, + [0][0][RTW89_IC][11] = 26, + [0][0][RTW89_KCC][11] = 40, + [0][0][RTW89_ACMA][11] = 38, + [0][0][RTW89_CN][11] = 32, + [0][0][RTW89_UK][11] = 38, + [0][0][RTW89_FCC][12] = -20, [0][0][RTW89_ETSI][12] = 34, [0][0][RTW89_MKK][12] = 36, - [0][0][RTW89_IC][12] = 32, - [0][0][RTW89_ACMA][12] = 32, + [0][0][RTW89_IC][12] = -20, + [0][0][RTW89_KCC][12] = 40, + [0][0][RTW89_ACMA][12] = 34, + [0][0][RTW89_CN][12] = 32, + [0][0][RTW89_UK][12] = 34, [0][0][RTW89_FCC][13] = 127, [0][0][RTW89_ETSI][13] = 127, [0][0][RTW89_MKK][13] = 127, [0][0][RTW89_IC][13] = 127, + [0][0][RTW89_KCC][13] = 127, [0][0][RTW89_ACMA][13] = 127, - [0][1][RTW89_FCC][0] = 46, + [0][0][RTW89_CN][13] = 127, + [0][0][RTW89_UK][13] = 127, + [0][1][RTW89_FCC][0] = 56, [0][1][RTW89_ETSI][0] = 22, [0][1][RTW89_MKK][0] = 24, - [0][1][RTW89_IC][0] = 62, - [0][1][RTW89_ACMA][0] = 20, - [0][1][RTW89_FCC][1] = 46, + [0][1][RTW89_IC][0] = 56, + [0][1][RTW89_KCC][0] = 30, + [0][1][RTW89_ACMA][0] = 22, + [0][1][RTW89_CN][0] = 20, + [0][1][RTW89_UK][0] = 22, + [0][1][RTW89_FCC][1] = 56, [0][1][RTW89_ETSI][1] = 24, [0][1][RTW89_MKK][1] = 30, - [0][1][RTW89_IC][1] = 62, - [0][1][RTW89_ACMA][1] = 22, - [0][1][RTW89_FCC][2] = 50, + [0][1][RTW89_IC][1] = 56, + [0][1][RTW89_KCC][1] = 30, + [0][1][RTW89_ACMA][1] = 24, + [0][1][RTW89_CN][1] = 22, + [0][1][RTW89_UK][1] = 24, + [0][1][RTW89_FCC][2] = 60, [0][1][RTW89_ETSI][2] = 24, [0][1][RTW89_MKK][2] = 30, - [0][1][RTW89_IC][2] = 66, - [0][1][RTW89_ACMA][2] = 22, - [0][1][RTW89_FCC][3] = 54, + [0][1][RTW89_IC][2] = 60, + [0][1][RTW89_KCC][2] = 30, + [0][1][RTW89_ACMA][2] = 24, + [0][1][RTW89_CN][2] = 22, + [0][1][RTW89_UK][2] = 24, + [0][1][RTW89_FCC][3] = 64, [0][1][RTW89_ETSI][3] = 24, [0][1][RTW89_MKK][3] = 30, - [0][1][RTW89_IC][3] = 70, - [0][1][RTW89_ACMA][3] = 22, - [0][1][RTW89_FCC][4] = 58, + [0][1][RTW89_IC][3] = 64, + [0][1][RTW89_KCC][3] = 30, + [0][1][RTW89_ACMA][3] = 24, + [0][1][RTW89_CN][3] = 22, + [0][1][RTW89_UK][3] = 24, + [0][1][RTW89_FCC][4] = 68, [0][1][RTW89_ETSI][4] = 24, [0][1][RTW89_MKK][4] = 30, - [0][1][RTW89_IC][4] = 74, - [0][1][RTW89_ACMA][4] = 22, - [0][1][RTW89_FCC][5] = 66, + [0][1][RTW89_IC][4] = 68, + [0][1][RTW89_KCC][4] = 28, + [0][1][RTW89_ACMA][4] = 24, + [0][1][RTW89_CN][4] = 22, + [0][1][RTW89_UK][4] = 24, + [0][1][RTW89_FCC][5] = 76, [0][1][RTW89_ETSI][5] = 24, [0][1][RTW89_MKK][5] = 30, - [0][1][RTW89_IC][5] = 74, - [0][1][RTW89_ACMA][5] = 22, - [0][1][RTW89_FCC][6] = 58, + [0][1][RTW89_IC][5] = 76, + [0][1][RTW89_KCC][5] = 28, + [0][1][RTW89_ACMA][5] = 24, + [0][1][RTW89_CN][5] = 22, + [0][1][RTW89_UK][5] = 24, + [0][1][RTW89_FCC][6] = 54, [0][1][RTW89_ETSI][6] = 24, [0][1][RTW89_MKK][6] = 30, - [0][1][RTW89_IC][6] = 72, - [0][1][RTW89_ACMA][6] = 22, - [0][1][RTW89_FCC][7] = 54, + [0][1][RTW89_IC][6] = 54, + [0][1][RTW89_KCC][6] = 28, + [0][1][RTW89_ACMA][6] = 24, + [0][1][RTW89_CN][6] = 22, + [0][1][RTW89_UK][6] = 24, + [0][1][RTW89_FCC][7] = 50, [0][1][RTW89_ETSI][7] = 24, [0][1][RTW89_MKK][7] = 30, - [0][1][RTW89_IC][7] = 68, - [0][1][RTW89_ACMA][7] = 22, - [0][1][RTW89_FCC][8] = 50, + [0][1][RTW89_IC][7] = 50, + [0][1][RTW89_KCC][7] = 28, + [0][1][RTW89_ACMA][7] = 24, + [0][1][RTW89_CN][7] = 22, + [0][1][RTW89_UK][7] = 24, + [0][1][RTW89_FCC][8] = 46, [0][1][RTW89_ETSI][8] = 24, [0][1][RTW89_MKK][8] = 30, - [0][1][RTW89_IC][8] = 64, - [0][1][RTW89_ACMA][8] = 22, - [0][1][RTW89_FCC][9] = 46, + [0][1][RTW89_IC][8] = 46, + [0][1][RTW89_KCC][8] = 28, + [0][1][RTW89_ACMA][8] = 24, + [0][1][RTW89_CN][8] = 22, + [0][1][RTW89_UK][8] = 24, + [0][1][RTW89_FCC][9] = 42, [0][1][RTW89_ETSI][9] = 24, [0][1][RTW89_MKK][9] = 30, - [0][1][RTW89_IC][9] = 60, - [0][1][RTW89_ACMA][9] = 22, - [0][1][RTW89_FCC][10] = 46, + [0][1][RTW89_IC][9] = 42, + [0][1][RTW89_KCC][9] = 28, + [0][1][RTW89_ACMA][9] = 24, + [0][1][RTW89_CN][9] = 22, + [0][1][RTW89_UK][9] = 24, + [0][1][RTW89_FCC][10] = 42, [0][1][RTW89_ETSI][10] = 24, [0][1][RTW89_MKK][10] = 30, - [0][1][RTW89_IC][10] = 60, - [0][1][RTW89_ACMA][10] = 22, - [0][1][RTW89_FCC][11] = 30, + [0][1][RTW89_IC][10] = 42, + [0][1][RTW89_KCC][10] = 28, + [0][1][RTW89_ACMA][10] = 24, + [0][1][RTW89_CN][10] = 22, + [0][1][RTW89_UK][10] = 24, + [0][1][RTW89_FCC][11] = 22, [0][1][RTW89_ETSI][11] = 24, [0][1][RTW89_MKK][11] = 30, - [0][1][RTW89_IC][11] = 52, - [0][1][RTW89_ACMA][11] = 22, - [0][1][RTW89_FCC][12] = 22, + [0][1][RTW89_IC][11] = 22, + [0][1][RTW89_KCC][11] = 28, + [0][1][RTW89_ACMA][11] = 24, + [0][1][RTW89_CN][11] = 22, + [0][1][RTW89_UK][11] = 24, + [0][1][RTW89_FCC][12] = -30, [0][1][RTW89_ETSI][12] = 20, [0][1][RTW89_MKK][12] = 24, - [0][1][RTW89_IC][12] = 30, + [0][1][RTW89_IC][12] = -30, + [0][1][RTW89_KCC][12] = 28, [0][1][RTW89_ACMA][12] = 20, + [0][1][RTW89_CN][12] = 20, + [0][1][RTW89_UK][12] = 20, [0][1][RTW89_FCC][13] = 127, [0][1][RTW89_ETSI][13] = 127, [0][1][RTW89_MKK][13] = 127, [0][1][RTW89_IC][13] = 127, + [0][1][RTW89_KCC][13] = 127, [0][1][RTW89_ACMA][13] = 127, - [1][0][RTW89_FCC][0] = 64, + [0][1][RTW89_CN][13] = 127, + [0][1][RTW89_UK][13] = 127, + [1][0][RTW89_FCC][0] = 66, [1][0][RTW89_ETSI][0] = 46, [1][0][RTW89_MKK][0] = 48, - [1][0][RTW89_IC][0] = 78, - [1][0][RTW89_ACMA][0] = 42, - [1][0][RTW89_FCC][1] = 64, + [1][0][RTW89_IC][0] = 66, + [1][0][RTW89_KCC][0] = 50, + [1][0][RTW89_ACMA][0] = 46, + [1][0][RTW89_CN][0] = 42, + [1][0][RTW89_UK][0] = 46, + [1][0][RTW89_FCC][1] = 66, [1][0][RTW89_ETSI][1] = 46, [1][0][RTW89_MKK][1] = 48, - [1][0][RTW89_IC][1] = 78, - [1][0][RTW89_ACMA][1] = 44, - [1][0][RTW89_FCC][2] = 68, + [1][0][RTW89_IC][1] = 66, + [1][0][RTW89_KCC][1] = 50, + [1][0][RTW89_ACMA][1] = 46, + [1][0][RTW89_CN][1] = 44, + [1][0][RTW89_UK][1] = 46, + [1][0][RTW89_FCC][2] = 70, [1][0][RTW89_ETSI][2] = 46, [1][0][RTW89_MKK][2] = 48, - [1][0][RTW89_IC][2] = 82, - [1][0][RTW89_ACMA][2] = 44, - [1][0][RTW89_FCC][3] = 70, + [1][0][RTW89_IC][2] = 70, + [1][0][RTW89_KCC][2] = 50, + [1][0][RTW89_ACMA][2] = 46, + [1][0][RTW89_CN][2] = 44, + [1][0][RTW89_UK][2] = 46, + [1][0][RTW89_FCC][3] = 72, [1][0][RTW89_ETSI][3] = 46, [1][0][RTW89_MKK][3] = 48, - [1][0][RTW89_IC][3] = 84, - [1][0][RTW89_ACMA][3] = 44, - [1][0][RTW89_FCC][4] = 70, + [1][0][RTW89_IC][3] = 72, + [1][0][RTW89_KCC][3] = 50, + [1][0][RTW89_ACMA][3] = 46, + [1][0][RTW89_CN][3] = 44, + [1][0][RTW89_UK][3] = 46, + [1][0][RTW89_FCC][4] = 72, [1][0][RTW89_ETSI][4] = 46, [1][0][RTW89_MKK][4] = 48, - [1][0][RTW89_IC][4] = 84, - [1][0][RTW89_ACMA][4] = 44, - [1][0][RTW89_FCC][5] = 76, + [1][0][RTW89_IC][4] = 72, + [1][0][RTW89_KCC][4] = 50, + [1][0][RTW89_ACMA][4] = 46, + [1][0][RTW89_CN][4] = 44, + [1][0][RTW89_UK][4] = 46, + [1][0][RTW89_FCC][5] = 82, [1][0][RTW89_ETSI][5] = 46, [1][0][RTW89_MKK][5] = 48, - [1][0][RTW89_IC][5] = 84, - [1][0][RTW89_ACMA][5] = 44, - [1][0][RTW89_FCC][6] = 64, + [1][0][RTW89_IC][5] = 82, + [1][0][RTW89_KCC][5] = 50, + [1][0][RTW89_ACMA][5] = 46, + [1][0][RTW89_CN][5] = 44, + [1][0][RTW89_UK][5] = 46, + [1][0][RTW89_FCC][6] = 58, [1][0][RTW89_ETSI][6] = 44, [1][0][RTW89_MKK][6] = 48, - [1][0][RTW89_IC][6] = 78, + [1][0][RTW89_IC][6] = 58, + [1][0][RTW89_KCC][6] = 50, [1][0][RTW89_ACMA][6] = 44, - [1][0][RTW89_FCC][7] = 64, + [1][0][RTW89_CN][6] = 44, + [1][0][RTW89_UK][6] = 44, + [1][0][RTW89_FCC][7] = 58, [1][0][RTW89_ETSI][7] = 46, [1][0][RTW89_MKK][7] = 48, - [1][0][RTW89_IC][7] = 78, - [1][0][RTW89_ACMA][7] = 44, - [1][0][RTW89_FCC][8] = 64, + [1][0][RTW89_IC][7] = 58, + [1][0][RTW89_KCC][7] = 50, + [1][0][RTW89_ACMA][7] = 46, + [1][0][RTW89_CN][7] = 44, + [1][0][RTW89_UK][7] = 46, + [1][0][RTW89_FCC][8] = 58, [1][0][RTW89_ETSI][8] = 46, [1][0][RTW89_MKK][8] = 48, - [1][0][RTW89_IC][8] = 78, - [1][0][RTW89_ACMA][8] = 44, - [1][0][RTW89_FCC][9] = 60, + [1][0][RTW89_IC][8] = 58, + [1][0][RTW89_KCC][8] = 50, + [1][0][RTW89_ACMA][8] = 46, + [1][0][RTW89_CN][8] = 44, + [1][0][RTW89_UK][8] = 46, + [1][0][RTW89_FCC][9] = 54, [1][0][RTW89_ETSI][9] = 46, [1][0][RTW89_MKK][9] = 48, - [1][0][RTW89_IC][9] = 74, - [1][0][RTW89_ACMA][9] = 44, - [1][0][RTW89_FCC][10] = 60, + [1][0][RTW89_IC][9] = 54, + [1][0][RTW89_KCC][9] = 50, + [1][0][RTW89_ACMA][9] = 46, + [1][0][RTW89_CN][9] = 44, + [1][0][RTW89_UK][9] = 46, + [1][0][RTW89_FCC][10] = 54, [1][0][RTW89_ETSI][10] = 46, [1][0][RTW89_MKK][10] = 48, - [1][0][RTW89_IC][10] = 74, - [1][0][RTW89_ACMA][10] = 44, - [1][0][RTW89_FCC][11] = 42, + [1][0][RTW89_IC][10] = 54, + [1][0][RTW89_KCC][10] = 50, + [1][0][RTW89_ACMA][10] = 46, + [1][0][RTW89_CN][10] = 44, + [1][0][RTW89_UK][10] = 46, + [1][0][RTW89_FCC][11] = 36, [1][0][RTW89_ETSI][11] = 46, [1][0][RTW89_MKK][11] = 48, - [1][0][RTW89_IC][11] = 72, - [1][0][RTW89_ACMA][11] = 44, - [1][0][RTW89_FCC][12] = 30, + [1][0][RTW89_IC][11] = 36, + [1][0][RTW89_KCC][11] = 50, + [1][0][RTW89_ACMA][11] = 46, + [1][0][RTW89_CN][11] = 44, + [1][0][RTW89_UK][11] = 46, + [1][0][RTW89_FCC][12] = 4, [1][0][RTW89_ETSI][12] = 46, [1][0][RTW89_MKK][12] = 46, - [1][0][RTW89_IC][12] = 38, - [1][0][RTW89_ACMA][12] = 42, + [1][0][RTW89_IC][12] = 4, + [1][0][RTW89_KCC][12] = 50, + [1][0][RTW89_ACMA][12] = 46, + [1][0][RTW89_CN][12] = 42, + [1][0][RTW89_UK][12] = 46, [1][0][RTW89_FCC][13] = 127, [1][0][RTW89_ETSI][13] = 127, [1][0][RTW89_MKK][13] = 127, [1][0][RTW89_IC][13] = 127, + [1][0][RTW89_KCC][13] = 127, [1][0][RTW89_ACMA][13] = 127, - [1][1][RTW89_FCC][0] = 46, + [1][0][RTW89_CN][13] = 127, + [1][0][RTW89_UK][13] = 127, + [1][1][RTW89_FCC][0] = 58, [1][1][RTW89_ETSI][0] = 32, [1][1][RTW89_MKK][0] = 34, - [1][1][RTW89_IC][0] = 66, + [1][1][RTW89_IC][0] = 58, + [1][1][RTW89_KCC][0] = 38, [1][1][RTW89_ACMA][0] = 32, - [1][1][RTW89_FCC][1] = 46, + [1][1][RTW89_CN][0] = 32, + [1][1][RTW89_UK][0] = 32, + [1][1][RTW89_FCC][1] = 58, [1][1][RTW89_ETSI][1] = 34, [1][1][RTW89_MKK][1] = 34, - [1][1][RTW89_IC][1] = 66, - [1][1][RTW89_ACMA][1] = 32, - [1][1][RTW89_FCC][2] = 50, + [1][1][RTW89_IC][1] = 58, + [1][1][RTW89_KCC][1] = 38, + [1][1][RTW89_ACMA][1] = 34, + [1][1][RTW89_CN][1] = 32, + [1][1][RTW89_UK][1] = 34, + [1][1][RTW89_FCC][2] = 62, [1][1][RTW89_ETSI][2] = 34, [1][1][RTW89_MKK][2] = 34, - [1][1][RTW89_IC][2] = 70, - [1][1][RTW89_ACMA][2] = 32, - [1][1][RTW89_FCC][3] = 54, + [1][1][RTW89_IC][2] = 62, + [1][1][RTW89_KCC][2] = 38, + [1][1][RTW89_ACMA][2] = 34, + [1][1][RTW89_CN][2] = 32, + [1][1][RTW89_UK][2] = 34, + [1][1][RTW89_FCC][3] = 66, [1][1][RTW89_ETSI][3] = 34, [1][1][RTW89_MKK][3] = 34, - [1][1][RTW89_IC][3] = 74, - [1][1][RTW89_ACMA][3] = 32, - [1][1][RTW89_FCC][4] = 58, + [1][1][RTW89_IC][3] = 66, + [1][1][RTW89_KCC][3] = 38, + [1][1][RTW89_ACMA][3] = 34, + [1][1][RTW89_CN][3] = 32, + [1][1][RTW89_UK][3] = 34, + [1][1][RTW89_FCC][4] = 70, [1][1][RTW89_ETSI][4] = 34, [1][1][RTW89_MKK][4] = 34, - [1][1][RTW89_IC][4] = 74, - [1][1][RTW89_ACMA][4] = 32, - [1][1][RTW89_FCC][5] = 66, + [1][1][RTW89_IC][4] = 70, + [1][1][RTW89_KCC][4] = 38, + [1][1][RTW89_ACMA][4] = 34, + [1][1][RTW89_CN][4] = 32, + [1][1][RTW89_UK][4] = 34, + [1][1][RTW89_FCC][5] = 82, [1][1][RTW89_ETSI][5] = 34, [1][1][RTW89_MKK][5] = 34, - [1][1][RTW89_IC][5] = 74, - [1][1][RTW89_ACMA][5] = 32, - [1][1][RTW89_FCC][6] = 58, + [1][1][RTW89_IC][5] = 82, + [1][1][RTW89_KCC][5] = 38, + [1][1][RTW89_ACMA][5] = 34, + [1][1][RTW89_CN][5] = 32, + [1][1][RTW89_UK][5] = 34, + [1][1][RTW89_FCC][6] = 60, [1][1][RTW89_ETSI][6] = 34, [1][1][RTW89_MKK][6] = 34, - [1][1][RTW89_IC][6] = 74, - [1][1][RTW89_ACMA][6] = 32, - [1][1][RTW89_FCC][7] = 54, + [1][1][RTW89_IC][6] = 60, + [1][1][RTW89_KCC][6] = 38, + [1][1][RTW89_ACMA][6] = 34, + [1][1][RTW89_CN][6] = 32, + [1][1][RTW89_UK][6] = 34, + [1][1][RTW89_FCC][7] = 56, [1][1][RTW89_ETSI][7] = 34, [1][1][RTW89_MKK][7] = 34, - [1][1][RTW89_IC][7] = 74, - [1][1][RTW89_ACMA][7] = 32, - [1][1][RTW89_FCC][8] = 50, + [1][1][RTW89_IC][7] = 56, + [1][1][RTW89_KCC][7] = 38, + [1][1][RTW89_ACMA][7] = 34, + [1][1][RTW89_CN][7] = 32, + [1][1][RTW89_UK][7] = 34, + [1][1][RTW89_FCC][8] = 52, [1][1][RTW89_ETSI][8] = 34, [1][1][RTW89_MKK][8] = 34, - [1][1][RTW89_IC][8] = 70, - [1][1][RTW89_ACMA][8] = 32, - [1][1][RTW89_FCC][9] = 46, + [1][1][RTW89_IC][8] = 52, + [1][1][RTW89_KCC][8] = 38, + [1][1][RTW89_ACMA][8] = 34, + [1][1][RTW89_CN][8] = 32, + [1][1][RTW89_UK][8] = 34, + [1][1][RTW89_FCC][9] = 48, [1][1][RTW89_ETSI][9] = 34, [1][1][RTW89_MKK][9] = 34, - [1][1][RTW89_IC][9] = 66, - [1][1][RTW89_ACMA][9] = 32, - [1][1][RTW89_FCC][10] = 46, + [1][1][RTW89_IC][9] = 48, + [1][1][RTW89_KCC][9] = 38, + [1][1][RTW89_ACMA][9] = 34, + [1][1][RTW89_CN][9] = 32, + [1][1][RTW89_UK][9] = 34, + [1][1][RTW89_FCC][10] = 48, [1][1][RTW89_ETSI][10] = 34, [1][1][RTW89_MKK][10] = 34, - [1][1][RTW89_IC][10] = 66, - [1][1][RTW89_ACMA][10] = 32, + [1][1][RTW89_IC][10] = 48, + [1][1][RTW89_KCC][10] = 38, + [1][1][RTW89_ACMA][10] = 34, + [1][1][RTW89_CN][10] = 32, + [1][1][RTW89_UK][10] = 34, [1][1][RTW89_FCC][11] = 30, [1][1][RTW89_ETSI][11] = 34, [1][1][RTW89_MKK][11] = 34, - [1][1][RTW89_IC][11] = 48, - [1][1][RTW89_ACMA][11] = 32, - [1][1][RTW89_FCC][12] = 24, + [1][1][RTW89_IC][11] = 30, + [1][1][RTW89_KCC][11] = 38, + [1][1][RTW89_ACMA][11] = 34, + [1][1][RTW89_CN][11] = 32, + [1][1][RTW89_UK][11] = 34, + [1][1][RTW89_FCC][12] = -6, [1][1][RTW89_ETSI][12] = 34, [1][1][RTW89_MKK][12] = 34, - [1][1][RTW89_IC][12] = 32, - [1][1][RTW89_ACMA][12] = 32, + [1][1][RTW89_IC][12] = -6, + [1][1][RTW89_KCC][12] = 38, + [1][1][RTW89_ACMA][12] = 34, + [1][1][RTW89_CN][12] = 32, + [1][1][RTW89_UK][12] = 34, [1][1][RTW89_FCC][13] = 127, [1][1][RTW89_ETSI][13] = 127, [1][1][RTW89_MKK][13] = 127, [1][1][RTW89_IC][13] = 127, + [1][1][RTW89_KCC][13] = 127, [1][1][RTW89_ACMA][13] = 127, - [2][0][RTW89_FCC][0] = 64, + [1][1][RTW89_CN][13] = 127, + [1][1][RTW89_UK][13] = 127, + [2][0][RTW89_FCC][0] = 70, [2][0][RTW89_ETSI][0] = 58, [2][0][RTW89_MKK][0] = 58, - [2][0][RTW89_IC][0] = 78, - [2][0][RTW89_ACMA][0] = 56, - [2][0][RTW89_FCC][1] = 64, + [2][0][RTW89_IC][0] = 70, + [2][0][RTW89_KCC][0] = 64, + [2][0][RTW89_ACMA][0] = 58, + [2][0][RTW89_CN][0] = 56, + [2][0][RTW89_UK][0] = 58, + [2][0][RTW89_FCC][1] = 70, [2][0][RTW89_ETSI][1] = 58, [2][0][RTW89_MKK][1] = 58, - [2][0][RTW89_IC][1] = 78, - [2][0][RTW89_ACMA][1] = 56, - [2][0][RTW89_FCC][2] = 66, + [2][0][RTW89_IC][1] = 70, + [2][0][RTW89_KCC][1] = 64, + [2][0][RTW89_ACMA][1] = 58, + [2][0][RTW89_CN][1] = 56, + [2][0][RTW89_UK][1] = 58, + [2][0][RTW89_FCC][2] = 72, [2][0][RTW89_ETSI][2] = 58, [2][0][RTW89_MKK][2] = 58, - [2][0][RTW89_IC][2] = 80, - [2][0][RTW89_ACMA][2] = 56, - [2][0][RTW89_FCC][3] = 66, + [2][0][RTW89_IC][2] = 72, + [2][0][RTW89_KCC][2] = 64, + [2][0][RTW89_ACMA][2] = 58, + [2][0][RTW89_CN][2] = 56, + [2][0][RTW89_UK][2] = 58, + [2][0][RTW89_FCC][3] = 72, [2][0][RTW89_ETSI][3] = 58, [2][0][RTW89_MKK][3] = 58, - [2][0][RTW89_IC][3] = 80, - [2][0][RTW89_ACMA][3] = 56, - [2][0][RTW89_FCC][4] = 66, + [2][0][RTW89_IC][3] = 72, + [2][0][RTW89_KCC][3] = 64, + [2][0][RTW89_ACMA][3] = 58, + [2][0][RTW89_CN][3] = 56, + [2][0][RTW89_UK][3] = 58, + [2][0][RTW89_FCC][4] = 72, [2][0][RTW89_ETSI][4] = 58, [2][0][RTW89_MKK][4] = 58, - [2][0][RTW89_IC][4] = 80, - [2][0][RTW89_ACMA][4] = 56, - [2][0][RTW89_FCC][5] = 76, + [2][0][RTW89_IC][4] = 72, + [2][0][RTW89_KCC][4] = 64, + [2][0][RTW89_ACMA][4] = 58, + [2][0][RTW89_CN][4] = 56, + [2][0][RTW89_UK][4] = 58, + [2][0][RTW89_FCC][5] = 82, [2][0][RTW89_ETSI][5] = 58, [2][0][RTW89_MKK][5] = 58, - [2][0][RTW89_IC][5] = 84, - [2][0][RTW89_ACMA][5] = 56, - [2][0][RTW89_FCC][6] = 62, + [2][0][RTW89_IC][5] = 82, + [2][0][RTW89_KCC][5] = 64, + [2][0][RTW89_ACMA][5] = 58, + [2][0][RTW89_CN][5] = 56, + [2][0][RTW89_UK][5] = 58, + [2][0][RTW89_FCC][6] = 66, [2][0][RTW89_ETSI][6] = 56, [2][0][RTW89_MKK][6] = 58, - [2][0][RTW89_IC][6] = 76, + [2][0][RTW89_IC][6] = 66, + [2][0][RTW89_KCC][6] = 64, [2][0][RTW89_ACMA][6] = 56, - [2][0][RTW89_FCC][7] = 62, + [2][0][RTW89_CN][6] = 56, + [2][0][RTW89_UK][6] = 56, + [2][0][RTW89_FCC][7] = 66, [2][0][RTW89_ETSI][7] = 58, [2][0][RTW89_MKK][7] = 58, - [2][0][RTW89_IC][7] = 76, - [2][0][RTW89_ACMA][7] = 56, - [2][0][RTW89_FCC][8] = 62, + [2][0][RTW89_IC][7] = 66, + [2][0][RTW89_KCC][7] = 64, + [2][0][RTW89_ACMA][7] = 58, + [2][0][RTW89_CN][7] = 56, + [2][0][RTW89_UK][7] = 58, + [2][0][RTW89_FCC][8] = 66, [2][0][RTW89_ETSI][8] = 58, [2][0][RTW89_MKK][8] = 58, - [2][0][RTW89_IC][8] = 76, - [2][0][RTW89_ACMA][8] = 56, - [2][0][RTW89_FCC][9] = 60, + [2][0][RTW89_IC][8] = 66, + [2][0][RTW89_KCC][8] = 64, + [2][0][RTW89_ACMA][8] = 58, + [2][0][RTW89_CN][8] = 56, + [2][0][RTW89_UK][8] = 58, + [2][0][RTW89_FCC][9] = 64, [2][0][RTW89_ETSI][9] = 58, [2][0][RTW89_MKK][9] = 58, - [2][0][RTW89_IC][9] = 74, - [2][0][RTW89_ACMA][9] = 56, - [2][0][RTW89_FCC][10] = 60, + [2][0][RTW89_IC][9] = 64, + [2][0][RTW89_KCC][9] = 64, + [2][0][RTW89_ACMA][9] = 58, + [2][0][RTW89_CN][9] = 56, + [2][0][RTW89_UK][9] = 58, + [2][0][RTW89_FCC][10] = 64, [2][0][RTW89_ETSI][10] = 58, [2][0][RTW89_MKK][10] = 58, - [2][0][RTW89_IC][10] = 74, - [2][0][RTW89_ACMA][10] = 56, - [2][0][RTW89_FCC][11] = 42, + [2][0][RTW89_IC][10] = 64, + [2][0][RTW89_KCC][10] = 64, + [2][0][RTW89_ACMA][10] = 58, + [2][0][RTW89_CN][10] = 56, + [2][0][RTW89_UK][10] = 58, + [2][0][RTW89_FCC][11] = 48, [2][0][RTW89_ETSI][11] = 58, [2][0][RTW89_MKK][11] = 58, - [2][0][RTW89_IC][11] = 66, - [2][0][RTW89_ACMA][11] = 56, - [2][0][RTW89_FCC][12] = 38, + [2][0][RTW89_IC][11] = 48, + [2][0][RTW89_KCC][11] = 64, + [2][0][RTW89_ACMA][11] = 58, + [2][0][RTW89_CN][11] = 56, + [2][0][RTW89_UK][11] = 58, + [2][0][RTW89_FCC][12] = 16, [2][0][RTW89_ETSI][12] = 58, [2][0][RTW89_MKK][12] = 58, - [2][0][RTW89_IC][12] = 56, - [2][0][RTW89_ACMA][12] = 56, + [2][0][RTW89_IC][12] = 16, + [2][0][RTW89_KCC][12] = 64, + [2][0][RTW89_ACMA][12] = 58, + [2][0][RTW89_CN][12] = 56, + [2][0][RTW89_UK][12] = 58, [2][0][RTW89_FCC][13] = 127, [2][0][RTW89_ETSI][13] = 127, [2][0][RTW89_MKK][13] = 127, [2][0][RTW89_IC][13] = 127, + [2][0][RTW89_KCC][13] = 127, [2][0][RTW89_ACMA][13] = 127, - [2][1][RTW89_FCC][0] = 46, + [2][0][RTW89_CN][13] = 127, + [2][0][RTW89_UK][13] = 127, + [2][1][RTW89_FCC][0] = 64, [2][1][RTW89_ETSI][0] = 46, [2][1][RTW89_MKK][0] = 46, - [2][1][RTW89_IC][0] = 70, - [2][1][RTW89_ACMA][0] = 44, - [2][1][RTW89_FCC][1] = 46, + [2][1][RTW89_IC][0] = 64, + [2][1][RTW89_KCC][0] = 52, + [2][1][RTW89_ACMA][0] = 46, + [2][1][RTW89_CN][0] = 44, + [2][1][RTW89_UK][0] = 46, + [2][1][RTW89_FCC][1] = 64, [2][1][RTW89_ETSI][1] = 46, [2][1][RTW89_MKK][1] = 46, - [2][1][RTW89_IC][1] = 70, - [2][1][RTW89_ACMA][1] = 44, - [2][1][RTW89_FCC][2] = 50, + [2][1][RTW89_IC][1] = 64, + [2][1][RTW89_KCC][1] = 52, + [2][1][RTW89_ACMA][1] = 46, + [2][1][RTW89_CN][1] = 44, + [2][1][RTW89_UK][1] = 46, + [2][1][RTW89_FCC][2] = 68, [2][1][RTW89_ETSI][2] = 46, [2][1][RTW89_MKK][2] = 46, - [2][1][RTW89_IC][2] = 74, - [2][1][RTW89_ACMA][2] = 44, - [2][1][RTW89_FCC][3] = 54, + [2][1][RTW89_IC][2] = 68, + [2][1][RTW89_KCC][2] = 52, + [2][1][RTW89_ACMA][2] = 46, + [2][1][RTW89_CN][2] = 44, + [2][1][RTW89_UK][2] = 46, + [2][1][RTW89_FCC][3] = 72, [2][1][RTW89_ETSI][3] = 46, [2][1][RTW89_MKK][3] = 46, - [2][1][RTW89_IC][3] = 78, - [2][1][RTW89_ACMA][3] = 44, - [2][1][RTW89_FCC][4] = 56, + [2][1][RTW89_IC][3] = 72, + [2][1][RTW89_KCC][3] = 52, + [2][1][RTW89_ACMA][3] = 46, + [2][1][RTW89_CN][3] = 44, + [2][1][RTW89_UK][3] = 46, + [2][1][RTW89_FCC][4] = 74, [2][1][RTW89_ETSI][4] = 46, [2][1][RTW89_MKK][4] = 46, - [2][1][RTW89_IC][4] = 80, - [2][1][RTW89_ACMA][4] = 44, - [2][1][RTW89_FCC][5] = 72, + [2][1][RTW89_IC][4] = 74, + [2][1][RTW89_KCC][4] = 50, + [2][1][RTW89_ACMA][4] = 46, + [2][1][RTW89_CN][4] = 44, + [2][1][RTW89_UK][4] = 46, + [2][1][RTW89_FCC][5] = 82, [2][1][RTW89_ETSI][5] = 46, [2][1][RTW89_MKK][5] = 46, - [2][1][RTW89_IC][5] = 80, - [2][1][RTW89_ACMA][5] = 44, - [2][1][RTW89_FCC][6] = 54, + [2][1][RTW89_IC][5] = 82, + [2][1][RTW89_KCC][5] = 50, + [2][1][RTW89_ACMA][5] = 46, + [2][1][RTW89_CN][5] = 44, + [2][1][RTW89_UK][5] = 46, + [2][1][RTW89_FCC][6] = 72, [2][1][RTW89_ETSI][6] = 44, [2][1][RTW89_MKK][6] = 46, - [2][1][RTW89_IC][6] = 78, + [2][1][RTW89_IC][6] = 72, + [2][1][RTW89_KCC][6] = 50, [2][1][RTW89_ACMA][6] = 44, - [2][1][RTW89_FCC][7] = 54, + [2][1][RTW89_CN][6] = 44, + [2][1][RTW89_UK][6] = 44, + [2][1][RTW89_FCC][7] = 72, [2][1][RTW89_ETSI][7] = 46, [2][1][RTW89_MKK][7] = 46, - [2][1][RTW89_IC][7] = 78, - [2][1][RTW89_ACMA][7] = 44, - [2][1][RTW89_FCC][8] = 50, + [2][1][RTW89_IC][7] = 72, + [2][1][RTW89_KCC][7] = 50, + [2][1][RTW89_ACMA][7] = 46, + [2][1][RTW89_CN][7] = 44, + [2][1][RTW89_UK][7] = 46, + [2][1][RTW89_FCC][8] = 68, [2][1][RTW89_ETSI][8] = 46, [2][1][RTW89_MKK][8] = 46, - [2][1][RTW89_IC][8] = 74, - [2][1][RTW89_ACMA][8] = 44, - [2][1][RTW89_FCC][9] = 46, + [2][1][RTW89_IC][8] = 68, + [2][1][RTW89_KCC][8] = 50, + [2][1][RTW89_ACMA][8] = 46, + [2][1][RTW89_CN][8] = 44, + [2][1][RTW89_UK][8] = 46, + [2][1][RTW89_FCC][9] = 64, [2][1][RTW89_ETSI][9] = 46, [2][1][RTW89_MKK][9] = 46, - [2][1][RTW89_IC][9] = 70, - [2][1][RTW89_ACMA][9] = 44, - [2][1][RTW89_FCC][10] = 46, + [2][1][RTW89_IC][9] = 64, + [2][1][RTW89_KCC][9] = 52, + [2][1][RTW89_ACMA][9] = 46, + [2][1][RTW89_CN][9] = 44, + [2][1][RTW89_UK][9] = 46, + [2][1][RTW89_FCC][10] = 64, [2][1][RTW89_ETSI][10] = 46, [2][1][RTW89_MKK][10] = 46, - [2][1][RTW89_IC][10] = 70, - [2][1][RTW89_ACMA][10] = 44, - [2][1][RTW89_FCC][11] = 30, + [2][1][RTW89_IC][10] = 64, + [2][1][RTW89_KCC][10] = 52, + [2][1][RTW89_ACMA][10] = 46, + [2][1][RTW89_CN][10] = 44, + [2][1][RTW89_UK][10] = 46, + [2][1][RTW89_FCC][11] = 46, [2][1][RTW89_ETSI][11] = 46, [2][1][RTW89_MKK][11] = 46, - [2][1][RTW89_IC][11] = 60, - [2][1][RTW89_ACMA][11] = 44, - [2][1][RTW89_FCC][12] = 26, + [2][1][RTW89_IC][11] = 46, + [2][1][RTW89_KCC][11] = 52, + [2][1][RTW89_ACMA][11] = 46, + [2][1][RTW89_CN][11] = 44, + [2][1][RTW89_UK][11] = 46, + [2][1][RTW89_FCC][12] = 6, [2][1][RTW89_ETSI][12] = 44, [2][1][RTW89_MKK][12] = 46, - [2][1][RTW89_IC][12] = 44, - [2][1][RTW89_ACMA][12] = 42, + [2][1][RTW89_IC][12] = 6, + [2][1][RTW89_KCC][12] = 52, + [2][1][RTW89_ACMA][12] = 44, + [2][1][RTW89_CN][12] = 42, + [2][1][RTW89_UK][12] = 44, [2][1][RTW89_FCC][13] = 127, [2][1][RTW89_ETSI][13] = 127, [2][1][RTW89_MKK][13] = 127, [2][1][RTW89_IC][13] = 127, + [2][1][RTW89_KCC][13] = 127, [2][1][RTW89_ACMA][13] = 127, + [2][1][RTW89_CN][13] = 127, + [2][1][RTW89_UK][13] = 127, }; const s8 rtw89_8852c_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM] [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = { - [0][0][RTW89_WW][0] = 24, - [0][0][RTW89_WW][2] = 24, - [0][0][RTW89_WW][4] = 22, - [0][0][RTW89_WW][6] = 22, - [0][0][RTW89_WW][8] = 18, - [0][0][RTW89_WW][10] = 18, - [0][0][RTW89_WW][12] = 24, - [0][0][RTW89_WW][14] = 24, - [0][0][RTW89_WW][15] = 24, - [0][0][RTW89_WW][17] = 24, - [0][0][RTW89_WW][19] = 24, - [0][0][RTW89_WW][21] = 24, - [0][0][RTW89_WW][23] = 24, + [0][0][RTW89_WW][0] = 16, + [0][0][RTW89_WW][2] = 16, + [0][0][RTW89_WW][4] = 16, + [0][0][RTW89_WW][6] = 10, + [0][0][RTW89_WW][8] = 16, + [0][0][RTW89_WW][10] = 16, + [0][0][RTW89_WW][12] = 16, + [0][0][RTW89_WW][14] = 16, + [0][0][RTW89_WW][15] = 30, + [0][0][RTW89_WW][17] = 30, + [0][0][RTW89_WW][19] = 30, + [0][0][RTW89_WW][21] = 30, + [0][0][RTW89_WW][23] = 30, [0][0][RTW89_WW][25] = 30, [0][0][RTW89_WW][27] = 30, [0][0][RTW89_WW][29] = 30, - [0][0][RTW89_WW][31] = 24, - [0][0][RTW89_WW][33] = 24, - [0][0][RTW89_WW][35] = 24, - [0][0][RTW89_WW][37] = 44, + [0][0][RTW89_WW][31] = 30, + [0][0][RTW89_WW][33] = 30, + [0][0][RTW89_WW][35] = 30, + [0][0][RTW89_WW][37] = 30, [0][0][RTW89_WW][38] = 28, [0][0][RTW89_WW][40] = 28, [0][0][RTW89_WW][42] = 28, [0][0][RTW89_WW][44] = 28, [0][0][RTW89_WW][46] = 28, - [0][0][RTW89_WW][48] = 24, - [0][0][RTW89_WW][50] = 24, - [0][0][RTW89_WW][52] = 24, - [0][1][RTW89_WW][0] = 0, + [0][0][RTW89_WW][48] = 46, + [0][0][RTW89_WW][50] = 44, + [0][0][RTW89_WW][52] = 34, + [0][1][RTW89_WW][0] = 4, [0][1][RTW89_WW][2] = 4, - [0][1][RTW89_WW][4] = 0, - [0][1][RTW89_WW][6] = 0, - [0][1][RTW89_WW][8] = 12, - [0][1][RTW89_WW][10] = 12, - [0][1][RTW89_WW][12] = 12, - [0][1][RTW89_WW][14] = 12, - [0][1][RTW89_WW][15] = 12, - [0][1][RTW89_WW][17] = 12, - [0][1][RTW89_WW][19] = 12, - [0][1][RTW89_WW][21] = 12, - [0][1][RTW89_WW][23] = 12, + [0][1][RTW89_WW][4] = 4, + [0][1][RTW89_WW][6] = 1, + [0][1][RTW89_WW][8] = 4, + [0][1][RTW89_WW][10] = 4, + [0][1][RTW89_WW][12] = 4, + [0][1][RTW89_WW][14] = 4, + [0][1][RTW89_WW][15] = 18, + [0][1][RTW89_WW][17] = 18, + [0][1][RTW89_WW][19] = 18, + [0][1][RTW89_WW][21] = 18, + [0][1][RTW89_WW][23] = 18, [0][1][RTW89_WW][25] = 18, [0][1][RTW89_WW][27] = 16, [0][1][RTW89_WW][29] = 16, - [0][1][RTW89_WW][31] = 12, - [0][1][RTW89_WW][33] = 12, - [0][1][RTW89_WW][35] = 12, - [0][1][RTW89_WW][37] = 30, + [0][1][RTW89_WW][31] = 16, + [0][1][RTW89_WW][33] = 16, + [0][1][RTW89_WW][35] = 16, + [0][1][RTW89_WW][37] = 18, [0][1][RTW89_WW][38] = 16, [0][1][RTW89_WW][40] = 16, [0][1][RTW89_WW][42] = 16, [0][1][RTW89_WW][44] = 16, [0][1][RTW89_WW][46] = 16, - [0][1][RTW89_WW][48] = 12, - [0][1][RTW89_WW][50] = 12, - [0][1][RTW89_WW][52] = 12, - [1][0][RTW89_WW][0] = 34, - [1][0][RTW89_WW][2] = 34, - [1][0][RTW89_WW][4] = 34, - [1][0][RTW89_WW][6] = 34, - [1][0][RTW89_WW][8] = 34, - [1][0][RTW89_WW][10] = 34, - [1][0][RTW89_WW][12] = 34, - [1][0][RTW89_WW][14] = 34, - [1][0][RTW89_WW][15] = 34, - [1][0][RTW89_WW][17] = 34, - [1][0][RTW89_WW][19] = 34, - [1][0][RTW89_WW][21] = 34, - [1][0][RTW89_WW][23] = 34, + [0][1][RTW89_WW][48] = 20, + [0][1][RTW89_WW][50] = 20, + [0][1][RTW89_WW][52] = 8, + [1][0][RTW89_WW][0] = 26, + [1][0][RTW89_WW][2] = 26, + [1][0][RTW89_WW][4] = 26, + [1][0][RTW89_WW][6] = 24, + [1][0][RTW89_WW][8] = 26, + [1][0][RTW89_WW][10] = 26, + [1][0][RTW89_WW][12] = 26, + [1][0][RTW89_WW][14] = 26, + [1][0][RTW89_WW][15] = 40, + [1][0][RTW89_WW][17] = 40, + [1][0][RTW89_WW][19] = 40, + [1][0][RTW89_WW][21] = 40, + [1][0][RTW89_WW][23] = 40, [1][0][RTW89_WW][25] = 40, [1][0][RTW89_WW][27] = 42, [1][0][RTW89_WW][29] = 42, - [1][0][RTW89_WW][31] = 34, - [1][0][RTW89_WW][33] = 34, - [1][0][RTW89_WW][35] = 34, - [1][0][RTW89_WW][37] = 56, + [1][0][RTW89_WW][31] = 42, + [1][0][RTW89_WW][33] = 42, + [1][0][RTW89_WW][35] = 42, + [1][0][RTW89_WW][37] = 42, [1][0][RTW89_WW][38] = 28, [1][0][RTW89_WW][40] = 28, [1][0][RTW89_WW][42] = 28, [1][0][RTW89_WW][44] = 28, [1][0][RTW89_WW][46] = 28, - [1][0][RTW89_WW][48] = 36, - [1][0][RTW89_WW][50] = 36, - [1][0][RTW89_WW][52] = 36, - [1][1][RTW89_WW][0] = 10, + [1][0][RTW89_WW][48] = 56, + [1][0][RTW89_WW][50] = 58, + [1][0][RTW89_WW][52] = 56, + [1][1][RTW89_WW][0] = 14, [1][1][RTW89_WW][2] = 14, - [1][1][RTW89_WW][4] = 10, - [1][1][RTW89_WW][6] = 10, - [1][1][RTW89_WW][8] = 20, - [1][1][RTW89_WW][10] = 20, - [1][1][RTW89_WW][12] = 22, - [1][1][RTW89_WW][14] = 22, - [1][1][RTW89_WW][15] = 22, - [1][1][RTW89_WW][17] = 22, - [1][1][RTW89_WW][19] = 22, - [1][1][RTW89_WW][21] = 22, - [1][1][RTW89_WW][23] = 22, + [1][1][RTW89_WW][4] = 14, + [1][1][RTW89_WW][6] = 8, + [1][1][RTW89_WW][8] = 14, + [1][1][RTW89_WW][10] = 14, + [1][1][RTW89_WW][12] = 14, + [1][1][RTW89_WW][14] = 14, + [1][1][RTW89_WW][15] = 28, + [1][1][RTW89_WW][17] = 28, + [1][1][RTW89_WW][19] = 28, + [1][1][RTW89_WW][21] = 28, + [1][1][RTW89_WW][23] = 28, [1][1][RTW89_WW][25] = 28, [1][1][RTW89_WW][27] = 30, [1][1][RTW89_WW][29] = 30, - [1][1][RTW89_WW][31] = 22, - [1][1][RTW89_WW][33] = 22, - [1][1][RTW89_WW][35] = 22, - [1][1][RTW89_WW][37] = 40, + [1][1][RTW89_WW][31] = 30, + [1][1][RTW89_WW][33] = 30, + [1][1][RTW89_WW][35] = 30, + [1][1][RTW89_WW][37] = 32, [1][1][RTW89_WW][38] = 16, [1][1][RTW89_WW][40] = 16, [1][1][RTW89_WW][42] = 16, [1][1][RTW89_WW][44] = 16, [1][1][RTW89_WW][46] = 16, - [1][1][RTW89_WW][48] = 24, - [1][1][RTW89_WW][50] = 24, - [1][1][RTW89_WW][52] = 24, - [2][0][RTW89_WW][0] = 46, - [2][0][RTW89_WW][2] = 46, - [2][0][RTW89_WW][4] = 46, - [2][0][RTW89_WW][6] = 46, - [2][0][RTW89_WW][8] = 44, - [2][0][RTW89_WW][10] = 44, - [2][0][RTW89_WW][12] = 48, - [2][0][RTW89_WW][14] = 48, - [2][0][RTW89_WW][15] = 48, - [2][0][RTW89_WW][17] = 48, - [2][0][RTW89_WW][19] = 48, - [2][0][RTW89_WW][21] = 48, - [2][0][RTW89_WW][23] = 48, + [1][1][RTW89_WW][48] = 34, + [1][1][RTW89_WW][50] = 34, + [1][1][RTW89_WW][52] = 30, + [2][0][RTW89_WW][0] = 40, + [2][0][RTW89_WW][2] = 40, + [2][0][RTW89_WW][4] = 40, + [2][0][RTW89_WW][6] = 36, + [2][0][RTW89_WW][8] = 40, + [2][0][RTW89_WW][10] = 40, + [2][0][RTW89_WW][12] = 40, + [2][0][RTW89_WW][14] = 40, + [2][0][RTW89_WW][15] = 52, + [2][0][RTW89_WW][17] = 52, + [2][0][RTW89_WW][19] = 52, + [2][0][RTW89_WW][21] = 52, + [2][0][RTW89_WW][23] = 52, [2][0][RTW89_WW][25] = 52, [2][0][RTW89_WW][27] = 52, [2][0][RTW89_WW][29] = 52, - [2][0][RTW89_WW][31] = 48, - [2][0][RTW89_WW][33] = 48, - [2][0][RTW89_WW][35] = 48, - [2][0][RTW89_WW][37] = 62, + [2][0][RTW89_WW][31] = 52, + [2][0][RTW89_WW][33] = 52, + [2][0][RTW89_WW][35] = 52, + [2][0][RTW89_WW][37] = 52, [2][0][RTW89_WW][38] = 28, [2][0][RTW89_WW][40] = 28, [2][0][RTW89_WW][42] = 28, [2][0][RTW89_WW][44] = 28, [2][0][RTW89_WW][46] = 28, - [2][0][RTW89_WW][48] = 48, - [2][0][RTW89_WW][50] = 48, - [2][0][RTW89_WW][52] = 48, - [2][1][RTW89_WW][0] = 20, - [2][1][RTW89_WW][2] = 18, - [2][1][RTW89_WW][4] = 22, - [2][1][RTW89_WW][6] = 22, - [2][1][RTW89_WW][8] = 32, - [2][1][RTW89_WW][10] = 32, - [2][1][RTW89_WW][12] = 36, - [2][1][RTW89_WW][14] = 36, - [2][1][RTW89_WW][15] = 36, - [2][1][RTW89_WW][17] = 36, - [2][1][RTW89_WW][19] = 36, - [2][1][RTW89_WW][21] = 36, - [2][1][RTW89_WW][23] = 36, + [2][0][RTW89_WW][48] = 64, + [2][0][RTW89_WW][50] = 64, + [2][0][RTW89_WW][52] = 64, + [2][1][RTW89_WW][0] = 26, + [2][1][RTW89_WW][2] = 26, + [2][1][RTW89_WW][4] = 26, + [2][1][RTW89_WW][6] = 20, + [2][1][RTW89_WW][8] = 28, + [2][1][RTW89_WW][10] = 28, + [2][1][RTW89_WW][12] = 28, + [2][1][RTW89_WW][14] = 28, + [2][1][RTW89_WW][15] = 40, + [2][1][RTW89_WW][17] = 40, + [2][1][RTW89_WW][19] = 40, + [2][1][RTW89_WW][21] = 40, + [2][1][RTW89_WW][23] = 40, [2][1][RTW89_WW][25] = 40, [2][1][RTW89_WW][27] = 40, [2][1][RTW89_WW][29] = 40, - [2][1][RTW89_WW][31] = 36, - [2][1][RTW89_WW][33] = 36, - [2][1][RTW89_WW][35] = 36, + [2][1][RTW89_WW][31] = 40, + [2][1][RTW89_WW][33] = 40, + [2][1][RTW89_WW][35] = 40, [2][1][RTW89_WW][37] = 42, [2][1][RTW89_WW][38] = 16, [2][1][RTW89_WW][40] = 16, [2][1][RTW89_WW][42] = 16, [2][1][RTW89_WW][44] = 16, [2][1][RTW89_WW][46] = 16, - [2][1][RTW89_WW][48] = 36, - [2][1][RTW89_WW][50] = 36, - [2][1][RTW89_WW][52] = 36, - [0][0][RTW89_FCC][0] = 44, + [2][1][RTW89_WW][48] = 40, + [2][1][RTW89_WW][50] = 40, + [2][1][RTW89_WW][52] = 40, + [0][0][RTW89_FCC][0] = 50, [0][0][RTW89_ETSI][0] = 30, [0][0][RTW89_MKK][0] = 36, - [0][0][RTW89_IC][0] = 24, - [0][0][RTW89_ACMA][0] = 24, - [0][0][RTW89_FCC][2] = 44, + [0][0][RTW89_IC][0] = 32, + [0][0][RTW89_KCC][0] = 42, + [0][0][RTW89_ACMA][0] = 30, + [0][0][RTW89_CN][0] = 16, + [0][0][RTW89_UK][0] = 30, + [0][0][RTW89_FCC][2] = 50, [0][0][RTW89_ETSI][2] = 30, [0][0][RTW89_MKK][2] = 36, - [0][0][RTW89_IC][2] = 24, - [0][0][RTW89_ACMA][2] = 24, - [0][0][RTW89_FCC][4] = 44, + [0][0][RTW89_IC][2] = 32, + [0][0][RTW89_KCC][2] = 42, + [0][0][RTW89_ACMA][2] = 30, + [0][0][RTW89_CN][2] = 16, + [0][0][RTW89_UK][2] = 30, + [0][0][RTW89_FCC][4] = 50, [0][0][RTW89_ETSI][4] = 30, [0][0][RTW89_MKK][4] = 22, - [0][0][RTW89_IC][4] = 24, - [0][0][RTW89_ACMA][4] = 24, - [0][0][RTW89_FCC][6] = 44, + [0][0][RTW89_IC][4] = 32, + [0][0][RTW89_KCC][4] = 42, + [0][0][RTW89_ACMA][4] = 30, + [0][0][RTW89_CN][4] = 16, + [0][0][RTW89_UK][4] = 30, + [0][0][RTW89_FCC][6] = 50, [0][0][RTW89_ETSI][6] = 30, [0][0][RTW89_MKK][6] = 22, - [0][0][RTW89_IC][6] = 24, - [0][0][RTW89_ACMA][6] = 24, - [0][0][RTW89_FCC][8] = 44, + [0][0][RTW89_IC][6] = 32, + [0][0][RTW89_KCC][6] = 10, + [0][0][RTW89_ACMA][6] = 30, + [0][0][RTW89_CN][6] = 16, + [0][0][RTW89_UK][6] = 30, + [0][0][RTW89_FCC][8] = 52, [0][0][RTW89_ETSI][8] = 28, [0][0][RTW89_MKK][8] = 18, [0][0][RTW89_IC][8] = 52, - [0][0][RTW89_ACMA][8] = 24, - [0][0][RTW89_FCC][10] = 44, + [0][0][RTW89_KCC][8] = 44, + [0][0][RTW89_ACMA][8] = 28, + [0][0][RTW89_CN][8] = 16, + [0][0][RTW89_UK][8] = 28, + [0][0][RTW89_FCC][10] = 52, [0][0][RTW89_ETSI][10] = 28, [0][0][RTW89_MKK][10] = 18, [0][0][RTW89_IC][10] = 52, - [0][0][RTW89_ACMA][10] = 24, - [0][0][RTW89_FCC][12] = 44, + [0][0][RTW89_KCC][10] = 44, + [0][0][RTW89_ACMA][10] = 28, + [0][0][RTW89_CN][10] = 16, + [0][0][RTW89_UK][10] = 28, + [0][0][RTW89_FCC][12] = 52, [0][0][RTW89_ETSI][12] = 28, [0][0][RTW89_MKK][12] = 34, [0][0][RTW89_IC][12] = 52, - [0][0][RTW89_ACMA][12] = 24, - [0][0][RTW89_FCC][14] = 44, + [0][0][RTW89_KCC][12] = 40, + [0][0][RTW89_ACMA][12] = 28, + [0][0][RTW89_CN][12] = 16, + [0][0][RTW89_UK][12] = 28, + [0][0][RTW89_FCC][14] = 52, [0][0][RTW89_ETSI][14] = 28, [0][0][RTW89_MKK][14] = 34, [0][0][RTW89_IC][14] = 52, - [0][0][RTW89_ACMA][14] = 24, - [0][0][RTW89_FCC][15] = 44, + [0][0][RTW89_KCC][14] = 40, + [0][0][RTW89_ACMA][14] = 28, + [0][0][RTW89_CN][14] = 16, + [0][0][RTW89_UK][14] = 28, + [0][0][RTW89_FCC][15] = 52, [0][0][RTW89_ETSI][15] = 30, [0][0][RTW89_MKK][15] = 56, [0][0][RTW89_IC][15] = 52, - [0][0][RTW89_ACMA][15] = 24, - [0][0][RTW89_FCC][17] = 44, + [0][0][RTW89_KCC][15] = 42, + [0][0][RTW89_ACMA][15] = 30, + [0][0][RTW89_CN][15] = 127, + [0][0][RTW89_UK][15] = 30, + [0][0][RTW89_FCC][17] = 52, [0][0][RTW89_ETSI][17] = 30, [0][0][RTW89_MKK][17] = 58, [0][0][RTW89_IC][17] = 52, - [0][0][RTW89_ACMA][17] = 24, - [0][0][RTW89_FCC][19] = 44, + [0][0][RTW89_KCC][17] = 42, + [0][0][RTW89_ACMA][17] = 30, + [0][0][RTW89_CN][17] = 127, + [0][0][RTW89_UK][17] = 30, + [0][0][RTW89_FCC][19] = 52, [0][0][RTW89_ETSI][19] = 30, [0][0][RTW89_MKK][19] = 58, [0][0][RTW89_IC][19] = 52, - [0][0][RTW89_ACMA][19] = 24, - [0][0][RTW89_FCC][21] = 44, + [0][0][RTW89_KCC][19] = 42, + [0][0][RTW89_ACMA][19] = 30, + [0][0][RTW89_CN][19] = 127, + [0][0][RTW89_UK][19] = 30, + [0][0][RTW89_FCC][21] = 52, [0][0][RTW89_ETSI][21] = 30, [0][0][RTW89_MKK][21] = 58, [0][0][RTW89_IC][21] = 52, - [0][0][RTW89_ACMA][21] = 24, - [0][0][RTW89_FCC][23] = 44, + [0][0][RTW89_KCC][21] = 42, + [0][0][RTW89_ACMA][21] = 30, + [0][0][RTW89_CN][21] = 127, + [0][0][RTW89_UK][21] = 30, + [0][0][RTW89_FCC][23] = 52, [0][0][RTW89_ETSI][23] = 30, [0][0][RTW89_MKK][23] = 58, [0][0][RTW89_IC][23] = 52, - [0][0][RTW89_ACMA][23] = 24, - [0][0][RTW89_FCC][25] = 44, + [0][0][RTW89_KCC][23] = 42, + [0][0][RTW89_ACMA][23] = 30, + [0][0][RTW89_CN][23] = 127, + [0][0][RTW89_UK][23] = 30, + [0][0][RTW89_FCC][25] = 52, [0][0][RTW89_ETSI][25] = 30, [0][0][RTW89_MKK][25] = 58, [0][0][RTW89_IC][25] = 127, + [0][0][RTW89_KCC][25] = 42, [0][0][RTW89_ACMA][25] = 127, - [0][0][RTW89_FCC][27] = 44, + [0][0][RTW89_CN][25] = 127, + [0][0][RTW89_UK][25] = 30, + [0][0][RTW89_FCC][27] = 52, [0][0][RTW89_ETSI][27] = 30, [0][0][RTW89_MKK][27] = 58, [0][0][RTW89_IC][27] = 127, + [0][0][RTW89_KCC][27] = 42, [0][0][RTW89_ACMA][27] = 127, - [0][0][RTW89_FCC][29] = 44, + [0][0][RTW89_CN][27] = 127, + [0][0][RTW89_UK][27] = 30, + [0][0][RTW89_FCC][29] = 52, [0][0][RTW89_ETSI][29] = 30, [0][0][RTW89_MKK][29] = 58, [0][0][RTW89_IC][29] = 127, + [0][0][RTW89_KCC][29] = 42, [0][0][RTW89_ACMA][29] = 127, - [0][0][RTW89_FCC][31] = 44, + [0][0][RTW89_CN][29] = 127, + [0][0][RTW89_UK][29] = 30, + [0][0][RTW89_FCC][31] = 52, [0][0][RTW89_ETSI][31] = 30, [0][0][RTW89_MKK][31] = 58, - [0][0][RTW89_IC][31] = 52, - [0][0][RTW89_ACMA][31] = 24, + [0][0][RTW89_IC][31] = 44, + [0][0][RTW89_KCC][31] = 42, + [0][0][RTW89_ACMA][31] = 30, + [0][0][RTW89_CN][31] = 127, + [0][0][RTW89_UK][31] = 30, [0][0][RTW89_FCC][33] = 44, [0][0][RTW89_ETSI][33] = 30, [0][0][RTW89_MKK][33] = 58, - [0][0][RTW89_IC][33] = 52, - [0][0][RTW89_ACMA][33] = 24, + [0][0][RTW89_IC][33] = 44, + [0][0][RTW89_KCC][33] = 42, + [0][0][RTW89_ACMA][33] = 30, + [0][0][RTW89_CN][33] = 127, + [0][0][RTW89_UK][33] = 30, [0][0][RTW89_FCC][35] = 44, [0][0][RTW89_ETSI][35] = 30, [0][0][RTW89_MKK][35] = 58, - [0][0][RTW89_IC][35] = 52, - [0][0][RTW89_ACMA][35] = 24, - [0][0][RTW89_FCC][37] = 44, + [0][0][RTW89_IC][35] = 44, + [0][0][RTW89_KCC][35] = 42, + [0][0][RTW89_ACMA][35] = 30, + [0][0][RTW89_CN][35] = 127, + [0][0][RTW89_UK][35] = 30, + [0][0][RTW89_FCC][37] = 52, [0][0][RTW89_ETSI][37] = 127, [0][0][RTW89_MKK][37] = 58, [0][0][RTW89_IC][37] = 52, + [0][0][RTW89_KCC][37] = 42, [0][0][RTW89_ACMA][37] = 52, - [0][0][RTW89_FCC][38] = 76, + [0][0][RTW89_CN][37] = 127, + [0][0][RTW89_UK][37] = 30, + [0][0][RTW89_FCC][38] = 64, [0][0][RTW89_ETSI][38] = 28, [0][0][RTW89_MKK][38] = 127, - [0][0][RTW89_IC][38] = 84, - [0][0][RTW89_ACMA][38] = 84, - [0][0][RTW89_FCC][40] = 76, + [0][0][RTW89_IC][38] = 64, + [0][0][RTW89_KCC][38] = 42, + [0][0][RTW89_ACMA][38] = 64, + [0][0][RTW89_CN][38] = 54, + [0][0][RTW89_UK][38] = 30, + [0][0][RTW89_FCC][40] = 64, [0][0][RTW89_ETSI][40] = 28, [0][0][RTW89_MKK][40] = 127, - [0][0][RTW89_IC][40] = 84, - [0][0][RTW89_ACMA][40] = 84, - [0][0][RTW89_FCC][42] = 76, + [0][0][RTW89_IC][40] = 64, + [0][0][RTW89_KCC][40] = 42, + [0][0][RTW89_ACMA][40] = 64, + [0][0][RTW89_CN][40] = 54, + [0][0][RTW89_UK][40] = 30, + [0][0][RTW89_FCC][42] = 60, [0][0][RTW89_ETSI][42] = 28, [0][0][RTW89_MKK][42] = 127, - [0][0][RTW89_IC][42] = 84, - [0][0][RTW89_ACMA][42] = 84, - [0][0][RTW89_FCC][44] = 76, + [0][0][RTW89_IC][42] = 60, + [0][0][RTW89_KCC][42] = 42, + [0][0][RTW89_ACMA][42] = 60, + [0][0][RTW89_CN][42] = 54, + [0][0][RTW89_UK][42] = 30, + [0][0][RTW89_FCC][44] = 60, [0][0][RTW89_ETSI][44] = 28, [0][0][RTW89_MKK][44] = 127, - [0][0][RTW89_IC][44] = 84, - [0][0][RTW89_ACMA][44] = 84, - [0][0][RTW89_FCC][46] = 76, + [0][0][RTW89_IC][44] = 60, + [0][0][RTW89_KCC][44] = 42, + [0][0][RTW89_ACMA][44] = 60, + [0][0][RTW89_CN][44] = 54, + [0][0][RTW89_UK][44] = 30, + [0][0][RTW89_FCC][46] = 60, [0][0][RTW89_ETSI][46] = 28, [0][0][RTW89_MKK][46] = 127, - [0][0][RTW89_IC][46] = 84, - [0][0][RTW89_ACMA][46] = 84, - [0][0][RTW89_FCC][48] = 24, + [0][0][RTW89_IC][46] = 60, + [0][0][RTW89_KCC][46] = 42, + [0][0][RTW89_ACMA][46] = 60, + [0][0][RTW89_CN][46] = 54, + [0][0][RTW89_UK][46] = 30, + [0][0][RTW89_FCC][48] = 46, [0][0][RTW89_ETSI][48] = 127, [0][0][RTW89_MKK][48] = 127, [0][0][RTW89_IC][48] = 127, + [0][0][RTW89_KCC][48] = 127, [0][0][RTW89_ACMA][48] = 127, - [0][0][RTW89_FCC][50] = 24, + [0][0][RTW89_CN][48] = 127, + [0][0][RTW89_UK][48] = 127, + [0][0][RTW89_FCC][50] = 44, [0][0][RTW89_ETSI][50] = 127, [0][0][RTW89_MKK][50] = 127, [0][0][RTW89_IC][50] = 127, + [0][0][RTW89_KCC][50] = 127, [0][0][RTW89_ACMA][50] = 127, - [0][0][RTW89_FCC][52] = 24, + [0][0][RTW89_CN][50] = 127, + [0][0][RTW89_UK][50] = 127, + [0][0][RTW89_FCC][52] = 34, [0][0][RTW89_ETSI][52] = 127, [0][0][RTW89_MKK][52] = 127, [0][0][RTW89_IC][52] = 127, + [0][0][RTW89_KCC][52] = 127, [0][0][RTW89_ACMA][52] = 127, - [0][1][RTW89_FCC][0] = 26, + [0][0][RTW89_CN][52] = 127, + [0][0][RTW89_UK][52] = 127, + [0][1][RTW89_FCC][0] = 30, [0][1][RTW89_ETSI][0] = 18, [0][1][RTW89_MKK][0] = 20, - [0][1][RTW89_IC][0] = 0, - [0][1][RTW89_ACMA][0] = 12, - [0][1][RTW89_FCC][2] = 30, + [0][1][RTW89_IC][0] = 8, + [0][1][RTW89_KCC][0] = 26, + [0][1][RTW89_ACMA][0] = 18, + [0][1][RTW89_CN][0] = 4, + [0][1][RTW89_UK][0] = 18, + [0][1][RTW89_FCC][2] = 32, [0][1][RTW89_ETSI][2] = 18, [0][1][RTW89_MKK][2] = 20, - [0][1][RTW89_IC][2] = 4, - [0][1][RTW89_ACMA][2] = 12, - [0][1][RTW89_FCC][4] = 26, + [0][1][RTW89_IC][2] = 8, + [0][1][RTW89_KCC][2] = 26, + [0][1][RTW89_ACMA][2] = 18, + [0][1][RTW89_CN][2] = 4, + [0][1][RTW89_UK][2] = 18, + [0][1][RTW89_FCC][4] = 30, [0][1][RTW89_ETSI][4] = 18, [0][1][RTW89_MKK][4] = 8, - [0][1][RTW89_IC][4] = 0, - [0][1][RTW89_ACMA][4] = 12, - [0][1][RTW89_FCC][6] = 26, + [0][1][RTW89_IC][4] = 8, + [0][1][RTW89_KCC][4] = 26, + [0][1][RTW89_ACMA][4] = 18, + [0][1][RTW89_CN][4] = 4, + [0][1][RTW89_UK][4] = 18, + [0][1][RTW89_FCC][6] = 30, [0][1][RTW89_ETSI][6] = 18, [0][1][RTW89_MKK][6] = 8, - [0][1][RTW89_IC][6] = 0, - [0][1][RTW89_ACMA][6] = 12, - [0][1][RTW89_FCC][8] = 26, + [0][1][RTW89_IC][6] = 8, + [0][1][RTW89_KCC][6] = 0, + [0][1][RTW89_ACMA][6] = 18, + [0][1][RTW89_CN][6] = 4, + [0][1][RTW89_UK][6] = 18, + [0][1][RTW89_FCC][8] = 30, [0][1][RTW89_ETSI][8] = 16, [0][1][RTW89_MKK][8] = 20, - [0][1][RTW89_IC][8] = 34, - [0][1][RTW89_ACMA][8] = 12, - [0][1][RTW89_FCC][10] = 26, + [0][1][RTW89_IC][8] = 30, + [0][1][RTW89_KCC][8] = 28, + [0][1][RTW89_ACMA][8] = 16, + [0][1][RTW89_CN][8] = 4, + [0][1][RTW89_UK][8] = 16, + [0][1][RTW89_FCC][10] = 30, [0][1][RTW89_ETSI][10] = 16, [0][1][RTW89_MKK][10] = 20, - [0][1][RTW89_IC][10] = 34, - [0][1][RTW89_ACMA][10] = 12, + [0][1][RTW89_IC][10] = 30, + [0][1][RTW89_KCC][10] = 28, + [0][1][RTW89_ACMA][10] = 16, + [0][1][RTW89_CN][10] = 4, + [0][1][RTW89_UK][10] = 16, [0][1][RTW89_FCC][12] = 30, [0][1][RTW89_ETSI][12] = 16, [0][1][RTW89_MKK][12] = 34, - [0][1][RTW89_IC][12] = 38, - [0][1][RTW89_ACMA][12] = 12, - [0][1][RTW89_FCC][14] = 26, + [0][1][RTW89_IC][12] = 30, + [0][1][RTW89_KCC][12] = 28, + [0][1][RTW89_ACMA][12] = 16, + [0][1][RTW89_CN][12] = 4, + [0][1][RTW89_UK][12] = 16, + [0][1][RTW89_FCC][14] = 30, [0][1][RTW89_ETSI][14] = 16, [0][1][RTW89_MKK][14] = 34, - [0][1][RTW89_IC][14] = 34, - [0][1][RTW89_ACMA][14] = 12, - [0][1][RTW89_FCC][15] = 26, + [0][1][RTW89_IC][14] = 30, + [0][1][RTW89_KCC][14] = 28, + [0][1][RTW89_ACMA][14] = 16, + [0][1][RTW89_CN][14] = 4, + [0][1][RTW89_UK][14] = 16, + [0][1][RTW89_FCC][15] = 32, [0][1][RTW89_ETSI][15] = 18, [0][1][RTW89_MKK][15] = 44, - [0][1][RTW89_IC][15] = 34, - [0][1][RTW89_ACMA][15] = 12, - [0][1][RTW89_FCC][17] = 26, + [0][1][RTW89_IC][15] = 32, + [0][1][RTW89_KCC][15] = 28, + [0][1][RTW89_ACMA][15] = 18, + [0][1][RTW89_CN][15] = 127, + [0][1][RTW89_UK][15] = 18, + [0][1][RTW89_FCC][17] = 32, [0][1][RTW89_ETSI][17] = 18, [0][1][RTW89_MKK][17] = 44, - [0][1][RTW89_IC][17] = 34, - [0][1][RTW89_ACMA][17] = 12, - [0][1][RTW89_FCC][19] = 30, + [0][1][RTW89_IC][17] = 32, + [0][1][RTW89_KCC][17] = 28, + [0][1][RTW89_ACMA][17] = 18, + [0][1][RTW89_CN][17] = 127, + [0][1][RTW89_UK][17] = 18, + [0][1][RTW89_FCC][19] = 32, [0][1][RTW89_ETSI][19] = 18, [0][1][RTW89_MKK][19] = 44, - [0][1][RTW89_IC][19] = 38, - [0][1][RTW89_ACMA][19] = 12, - [0][1][RTW89_FCC][21] = 30, + [0][1][RTW89_IC][19] = 32, + [0][1][RTW89_KCC][19] = 28, + [0][1][RTW89_ACMA][19] = 18, + [0][1][RTW89_CN][19] = 127, + [0][1][RTW89_UK][19] = 18, + [0][1][RTW89_FCC][21] = 32, [0][1][RTW89_ETSI][21] = 18, [0][1][RTW89_MKK][21] = 44, - [0][1][RTW89_IC][21] = 38, - [0][1][RTW89_ACMA][21] = 12, - [0][1][RTW89_FCC][23] = 30, + [0][1][RTW89_IC][21] = 32, + [0][1][RTW89_KCC][21] = 28, + [0][1][RTW89_ACMA][21] = 18, + [0][1][RTW89_CN][21] = 127, + [0][1][RTW89_UK][21] = 18, + [0][1][RTW89_FCC][23] = 32, [0][1][RTW89_ETSI][23] = 18, [0][1][RTW89_MKK][23] = 44, - [0][1][RTW89_IC][23] = 38, - [0][1][RTW89_ACMA][23] = 12, - [0][1][RTW89_FCC][25] = 30, + [0][1][RTW89_IC][23] = 32, + [0][1][RTW89_KCC][23] = 28, + [0][1][RTW89_ACMA][23] = 18, + [0][1][RTW89_CN][23] = 127, + [0][1][RTW89_UK][23] = 18, + [0][1][RTW89_FCC][25] = 32, [0][1][RTW89_ETSI][25] = 18, [0][1][RTW89_MKK][25] = 44, [0][1][RTW89_IC][25] = 127, + [0][1][RTW89_KCC][25] = 28, [0][1][RTW89_ACMA][25] = 127, - [0][1][RTW89_FCC][27] = 30, + [0][1][RTW89_CN][25] = 127, + [0][1][RTW89_UK][25] = 18, + [0][1][RTW89_FCC][27] = 32, [0][1][RTW89_ETSI][27] = 16, [0][1][RTW89_MKK][27] = 44, [0][1][RTW89_IC][27] = 127, + [0][1][RTW89_KCC][27] = 28, [0][1][RTW89_ACMA][27] = 127, - [0][1][RTW89_FCC][29] = 30, + [0][1][RTW89_CN][27] = 127, + [0][1][RTW89_UK][27] = 16, + [0][1][RTW89_FCC][29] = 32, [0][1][RTW89_ETSI][29] = 16, [0][1][RTW89_MKK][29] = 44, [0][1][RTW89_IC][29] = 127, + [0][1][RTW89_KCC][29] = 28, [0][1][RTW89_ACMA][29] = 127, - [0][1][RTW89_FCC][31] = 30, + [0][1][RTW89_CN][29] = 127, + [0][1][RTW89_UK][29] = 16, + [0][1][RTW89_FCC][31] = 32, [0][1][RTW89_ETSI][31] = 16, [0][1][RTW89_MKK][31] = 44, - [0][1][RTW89_IC][31] = 34, - [0][1][RTW89_ACMA][31] = 12, - [0][1][RTW89_FCC][33] = 26, + [0][1][RTW89_IC][31] = 30, + [0][1][RTW89_KCC][31] = 28, + [0][1][RTW89_ACMA][31] = 16, + [0][1][RTW89_CN][31] = 127, + [0][1][RTW89_UK][31] = 16, + [0][1][RTW89_FCC][33] = 30, [0][1][RTW89_ETSI][33] = 16, [0][1][RTW89_MKK][33] = 44, - [0][1][RTW89_IC][33] = 34, - [0][1][RTW89_ACMA][33] = 12, - [0][1][RTW89_FCC][35] = 26, + [0][1][RTW89_IC][33] = 30, + [0][1][RTW89_KCC][33] = 28, + [0][1][RTW89_ACMA][33] = 16, + [0][1][RTW89_CN][33] = 127, + [0][1][RTW89_UK][33] = 16, + [0][1][RTW89_FCC][35] = 30, [0][1][RTW89_ETSI][35] = 16, [0][1][RTW89_MKK][35] = 44, - [0][1][RTW89_IC][35] = 34, - [0][1][RTW89_ACMA][35] = 12, - [0][1][RTW89_FCC][37] = 30, + [0][1][RTW89_IC][35] = 30, + [0][1][RTW89_KCC][35] = 28, + [0][1][RTW89_ACMA][35] = 16, + [0][1][RTW89_CN][35] = 127, + [0][1][RTW89_UK][35] = 16, + [0][1][RTW89_FCC][37] = 34, [0][1][RTW89_ETSI][37] = 127, [0][1][RTW89_MKK][37] = 44, - [0][1][RTW89_IC][37] = 38, - [0][1][RTW89_ACMA][37] = 38, - [0][1][RTW89_FCC][38] = 74, + [0][1][RTW89_IC][37] = 34, + [0][1][RTW89_KCC][37] = 28, + [0][1][RTW89_ACMA][37] = 34, + [0][1][RTW89_CN][37] = 127, + [0][1][RTW89_UK][37] = 18, + [0][1][RTW89_FCC][38] = 62, [0][1][RTW89_ETSI][38] = 16, [0][1][RTW89_MKK][38] = 127, - [0][1][RTW89_IC][38] = 82, - [0][1][RTW89_ACMA][38] = 84, - [0][1][RTW89_FCC][40] = 74, + [0][1][RTW89_IC][38] = 62, + [0][1][RTW89_KCC][38] = 28, + [0][1][RTW89_ACMA][38] = 62, + [0][1][RTW89_CN][38] = 42, + [0][1][RTW89_UK][38] = 18, + [0][1][RTW89_FCC][40] = 62, [0][1][RTW89_ETSI][40] = 16, [0][1][RTW89_MKK][40] = 127, - [0][1][RTW89_IC][40] = 82, - [0][1][RTW89_ACMA][40] = 84, - [0][1][RTW89_FCC][42] = 74, + [0][1][RTW89_IC][40] = 62, + [0][1][RTW89_KCC][40] = 28, + [0][1][RTW89_ACMA][40] = 62, + [0][1][RTW89_CN][40] = 42, + [0][1][RTW89_UK][40] = 18, + [0][1][RTW89_FCC][42] = 58, [0][1][RTW89_ETSI][42] = 16, [0][1][RTW89_MKK][42] = 127, - [0][1][RTW89_IC][42] = 82, - [0][1][RTW89_ACMA][42] = 84, - [0][1][RTW89_FCC][44] = 74, + [0][1][RTW89_IC][42] = 58, + [0][1][RTW89_KCC][42] = 28, + [0][1][RTW89_ACMA][42] = 58, + [0][1][RTW89_CN][42] = 42, + [0][1][RTW89_UK][42] = 18, + [0][1][RTW89_FCC][44] = 56, [0][1][RTW89_ETSI][44] = 16, [0][1][RTW89_MKK][44] = 127, - [0][1][RTW89_IC][44] = 82, - [0][1][RTW89_ACMA][44] = 84, - [0][1][RTW89_FCC][46] = 74, + [0][1][RTW89_IC][44] = 56, + [0][1][RTW89_KCC][44] = 28, + [0][1][RTW89_ACMA][44] = 56, + [0][1][RTW89_CN][44] = 42, + [0][1][RTW89_UK][44] = 18, + [0][1][RTW89_FCC][46] = 56, [0][1][RTW89_ETSI][46] = 16, [0][1][RTW89_MKK][46] = 127, - [0][1][RTW89_IC][46] = 82, - [0][1][RTW89_ACMA][46] = 84, - [0][1][RTW89_FCC][48] = 12, + [0][1][RTW89_IC][46] = 56, + [0][1][RTW89_KCC][46] = 28, + [0][1][RTW89_ACMA][46] = 56, + [0][1][RTW89_CN][46] = 42, + [0][1][RTW89_UK][46] = 18, + [0][1][RTW89_FCC][48] = 20, [0][1][RTW89_ETSI][48] = 127, [0][1][RTW89_MKK][48] = 127, [0][1][RTW89_IC][48] = 127, + [0][1][RTW89_KCC][48] = 127, [0][1][RTW89_ACMA][48] = 127, - [0][1][RTW89_FCC][50] = 12, + [0][1][RTW89_CN][48] = 127, + [0][1][RTW89_UK][48] = 127, + [0][1][RTW89_FCC][50] = 20, [0][1][RTW89_ETSI][50] = 127, [0][1][RTW89_MKK][50] = 127, [0][1][RTW89_IC][50] = 127, + [0][1][RTW89_KCC][50] = 127, [0][1][RTW89_ACMA][50] = 127, - [0][1][RTW89_FCC][52] = 12, + [0][1][RTW89_CN][50] = 127, + [0][1][RTW89_UK][50] = 127, + [0][1][RTW89_FCC][52] = 8, [0][1][RTW89_ETSI][52] = 127, [0][1][RTW89_MKK][52] = 127, [0][1][RTW89_IC][52] = 127, + [0][1][RTW89_KCC][52] = 127, [0][1][RTW89_ACMA][52] = 127, - [1][0][RTW89_FCC][0] = 54, + [0][1][RTW89_CN][52] = 127, + [0][1][RTW89_UK][52] = 127, + [1][0][RTW89_FCC][0] = 62, [1][0][RTW89_ETSI][0] = 40, [1][0][RTW89_MKK][0] = 48, - [1][0][RTW89_IC][0] = 36, - [1][0][RTW89_ACMA][0] = 34, - [1][0][RTW89_FCC][2] = 54, + [1][0][RTW89_IC][0] = 42, + [1][0][RTW89_KCC][0] = 50, + [1][0][RTW89_ACMA][0] = 40, + [1][0][RTW89_CN][0] = 26, + [1][0][RTW89_UK][0] = 40, + [1][0][RTW89_FCC][2] = 62, [1][0][RTW89_ETSI][2] = 40, [1][0][RTW89_MKK][2] = 48, - [1][0][RTW89_IC][2] = 36, - [1][0][RTW89_ACMA][2] = 34, - [1][0][RTW89_FCC][4] = 54, + [1][0][RTW89_IC][2] = 42, + [1][0][RTW89_KCC][2] = 50, + [1][0][RTW89_ACMA][2] = 40, + [1][0][RTW89_CN][2] = 26, + [1][0][RTW89_UK][2] = 40, + [1][0][RTW89_FCC][4] = 64, [1][0][RTW89_ETSI][4] = 40, [1][0][RTW89_MKK][4] = 40, - [1][0][RTW89_IC][4] = 36, - [1][0][RTW89_ACMA][4] = 34, - [1][0][RTW89_FCC][6] = 54, + [1][0][RTW89_IC][4] = 42, + [1][0][RTW89_KCC][4] = 50, + [1][0][RTW89_ACMA][4] = 40, + [1][0][RTW89_CN][4] = 26, + [1][0][RTW89_UK][4] = 40, + [1][0][RTW89_FCC][6] = 64, [1][0][RTW89_ETSI][6] = 40, [1][0][RTW89_MKK][6] = 40, - [1][0][RTW89_IC][6] = 36, - [1][0][RTW89_ACMA][6] = 34, - [1][0][RTW89_FCC][8] = 54, + [1][0][RTW89_IC][6] = 42, + [1][0][RTW89_KCC][6] = 24, + [1][0][RTW89_ACMA][6] = 40, + [1][0][RTW89_CN][6] = 26, + [1][0][RTW89_UK][6] = 40, + [1][0][RTW89_FCC][8] = 62, [1][0][RTW89_ETSI][8] = 40, [1][0][RTW89_MKK][8] = 34, [1][0][RTW89_IC][8] = 62, - [1][0][RTW89_ACMA][8] = 34, - [1][0][RTW89_FCC][10] = 54, + [1][0][RTW89_KCC][8] = 52, + [1][0][RTW89_ACMA][8] = 40, + [1][0][RTW89_CN][8] = 26, + [1][0][RTW89_UK][8] = 40, + [1][0][RTW89_FCC][10] = 62, [1][0][RTW89_ETSI][10] = 40, [1][0][RTW89_MKK][10] = 34, [1][0][RTW89_IC][10] = 62, - [1][0][RTW89_ACMA][10] = 34, - [1][0][RTW89_FCC][12] = 56, + [1][0][RTW89_KCC][10] = 52, + [1][0][RTW89_ACMA][10] = 40, + [1][0][RTW89_CN][10] = 26, + [1][0][RTW89_UK][10] = 40, + [1][0][RTW89_FCC][12] = 62, [1][0][RTW89_ETSI][12] = 40, [1][0][RTW89_MKK][12] = 46, - [1][0][RTW89_IC][12] = 64, - [1][0][RTW89_ACMA][12] = 34, - [1][0][RTW89_FCC][14] = 54, + [1][0][RTW89_IC][12] = 62, + [1][0][RTW89_KCC][12] = 52, + [1][0][RTW89_ACMA][12] = 40, + [1][0][RTW89_CN][12] = 26, + [1][0][RTW89_UK][12] = 40, + [1][0][RTW89_FCC][14] = 62, [1][0][RTW89_ETSI][14] = 40, [1][0][RTW89_MKK][14] = 46, [1][0][RTW89_IC][14] = 62, - [1][0][RTW89_ACMA][14] = 34, - [1][0][RTW89_FCC][15] = 54, + [1][0][RTW89_KCC][14] = 52, + [1][0][RTW89_ACMA][14] = 40, + [1][0][RTW89_CN][14] = 26, + [1][0][RTW89_UK][14] = 40, + [1][0][RTW89_FCC][15] = 62, [1][0][RTW89_ETSI][15] = 40, [1][0][RTW89_MKK][15] = 62, [1][0][RTW89_IC][15] = 62, - [1][0][RTW89_ACMA][15] = 34, - [1][0][RTW89_FCC][17] = 54, + [1][0][RTW89_KCC][15] = 52, + [1][0][RTW89_ACMA][15] = 40, + [1][0][RTW89_CN][15] = 127, + [1][0][RTW89_UK][15] = 40, + [1][0][RTW89_FCC][17] = 62, [1][0][RTW89_ETSI][17] = 40, [1][0][RTW89_MKK][17] = 68, [1][0][RTW89_IC][17] = 62, - [1][0][RTW89_ACMA][17] = 34, - [1][0][RTW89_FCC][19] = 54, + [1][0][RTW89_KCC][17] = 52, + [1][0][RTW89_ACMA][17] = 40, + [1][0][RTW89_CN][17] = 127, + [1][0][RTW89_UK][17] = 40, + [1][0][RTW89_FCC][19] = 64, [1][0][RTW89_ETSI][19] = 40, [1][0][RTW89_MKK][19] = 68, - [1][0][RTW89_IC][19] = 62, - [1][0][RTW89_ACMA][19] = 34, - [1][0][RTW89_FCC][21] = 54, + [1][0][RTW89_IC][19] = 64, + [1][0][RTW89_KCC][19] = 52, + [1][0][RTW89_ACMA][19] = 40, + [1][0][RTW89_CN][19] = 127, + [1][0][RTW89_UK][19] = 40, + [1][0][RTW89_FCC][21] = 64, [1][0][RTW89_ETSI][21] = 40, [1][0][RTW89_MKK][21] = 68, - [1][0][RTW89_IC][21] = 62, - [1][0][RTW89_ACMA][21] = 34, - [1][0][RTW89_FCC][23] = 54, + [1][0][RTW89_IC][21] = 64, + [1][0][RTW89_KCC][21] = 52, + [1][0][RTW89_ACMA][21] = 40, + [1][0][RTW89_CN][21] = 127, + [1][0][RTW89_UK][21] = 40, + [1][0][RTW89_FCC][23] = 64, [1][0][RTW89_ETSI][23] = 40, [1][0][RTW89_MKK][23] = 68, - [1][0][RTW89_IC][23] = 62, - [1][0][RTW89_ACMA][23] = 34, - [1][0][RTW89_FCC][25] = 54, + [1][0][RTW89_IC][23] = 64, + [1][0][RTW89_KCC][23] = 52, + [1][0][RTW89_ACMA][23] = 40, + [1][0][RTW89_CN][23] = 127, + [1][0][RTW89_UK][23] = 40, + [1][0][RTW89_FCC][25] = 64, [1][0][RTW89_ETSI][25] = 40, [1][0][RTW89_MKK][25] = 68, [1][0][RTW89_IC][25] = 127, + [1][0][RTW89_KCC][25] = 52, [1][0][RTW89_ACMA][25] = 127, - [1][0][RTW89_FCC][27] = 54, + [1][0][RTW89_CN][25] = 127, + [1][0][RTW89_UK][25] = 40, + [1][0][RTW89_FCC][27] = 64, [1][0][RTW89_ETSI][27] = 42, [1][0][RTW89_MKK][27] = 68, [1][0][RTW89_IC][27] = 127, + [1][0][RTW89_KCC][27] = 52, [1][0][RTW89_ACMA][27] = 127, - [1][0][RTW89_FCC][29] = 54, + [1][0][RTW89_CN][27] = 127, + [1][0][RTW89_UK][27] = 42, + [1][0][RTW89_FCC][29] = 64, [1][0][RTW89_ETSI][29] = 42, [1][0][RTW89_MKK][29] = 68, [1][0][RTW89_IC][29] = 127, + [1][0][RTW89_KCC][29] = 52, [1][0][RTW89_ACMA][29] = 127, - [1][0][RTW89_FCC][31] = 54, + [1][0][RTW89_CN][29] = 127, + [1][0][RTW89_UK][29] = 42, + [1][0][RTW89_FCC][31] = 64, [1][0][RTW89_ETSI][31] = 42, [1][0][RTW89_MKK][31] = 68, - [1][0][RTW89_IC][31] = 62, - [1][0][RTW89_ACMA][31] = 34, - [1][0][RTW89_FCC][33] = 54, + [1][0][RTW89_IC][31] = 56, + [1][0][RTW89_KCC][31] = 52, + [1][0][RTW89_ACMA][31] = 42, + [1][0][RTW89_CN][31] = 127, + [1][0][RTW89_UK][31] = 42, + [1][0][RTW89_FCC][33] = 56, [1][0][RTW89_ETSI][33] = 42, [1][0][RTW89_MKK][33] = 68, - [1][0][RTW89_IC][33] = 62, - [1][0][RTW89_ACMA][33] = 34, - [1][0][RTW89_FCC][35] = 54, + [1][0][RTW89_IC][33] = 56, + [1][0][RTW89_KCC][33] = 52, + [1][0][RTW89_ACMA][33] = 42, + [1][0][RTW89_CN][33] = 127, + [1][0][RTW89_UK][33] = 42, + [1][0][RTW89_FCC][35] = 56, [1][0][RTW89_ETSI][35] = 42, [1][0][RTW89_MKK][35] = 68, - [1][0][RTW89_IC][35] = 62, - [1][0][RTW89_ACMA][35] = 34, - [1][0][RTW89_FCC][37] = 56, + [1][0][RTW89_IC][35] = 56, + [1][0][RTW89_KCC][35] = 52, + [1][0][RTW89_ACMA][35] = 42, + [1][0][RTW89_CN][35] = 127, + [1][0][RTW89_UK][35] = 42, + [1][0][RTW89_FCC][37] = 66, [1][0][RTW89_ETSI][37] = 127, [1][0][RTW89_MKK][37] = 68, - [1][0][RTW89_IC][37] = 64, - [1][0][RTW89_ACMA][37] = 64, + [1][0][RTW89_IC][37] = 66, + [1][0][RTW89_KCC][37] = 52, + [1][0][RTW89_ACMA][37] = 66, + [1][0][RTW89_CN][37] = 127, + [1][0][RTW89_UK][37] = 42, [1][0][RTW89_FCC][38] = 76, [1][0][RTW89_ETSI][38] = 28, [1][0][RTW89_MKK][38] = 127, - [1][0][RTW89_IC][38] = 84, - [1][0][RTW89_ACMA][38] = 84, + [1][0][RTW89_IC][38] = 76, + [1][0][RTW89_KCC][38] = 54, + [1][0][RTW89_ACMA][38] = 76, + [1][0][RTW89_CN][38] = 66, + [1][0][RTW89_UK][38] = 44, [1][0][RTW89_FCC][40] = 76, [1][0][RTW89_ETSI][40] = 28, [1][0][RTW89_MKK][40] = 127, - [1][0][RTW89_IC][40] = 84, - [1][0][RTW89_ACMA][40] = 84, - [1][0][RTW89_FCC][42] = 76, + [1][0][RTW89_IC][40] = 76, + [1][0][RTW89_KCC][40] = 54, + [1][0][RTW89_ACMA][40] = 76, + [1][0][RTW89_CN][40] = 66, + [1][0][RTW89_UK][40] = 44, + [1][0][RTW89_FCC][42] = 68, [1][0][RTW89_ETSI][42] = 28, [1][0][RTW89_MKK][42] = 127, - [1][0][RTW89_IC][42] = 84, - [1][0][RTW89_ACMA][42] = 84, - [1][0][RTW89_FCC][44] = 76, + [1][0][RTW89_IC][42] = 68, + [1][0][RTW89_KCC][42] = 54, + [1][0][RTW89_ACMA][42] = 68, + [1][0][RTW89_CN][42] = 66, + [1][0][RTW89_UK][42] = 44, + [1][0][RTW89_FCC][44] = 70, [1][0][RTW89_ETSI][44] = 28, [1][0][RTW89_MKK][44] = 127, - [1][0][RTW89_IC][44] = 84, - [1][0][RTW89_ACMA][44] = 84, - [1][0][RTW89_FCC][46] = 76, + [1][0][RTW89_IC][44] = 70, + [1][0][RTW89_KCC][44] = 54, + [1][0][RTW89_ACMA][44] = 70, + [1][0][RTW89_CN][44] = 66, + [1][0][RTW89_UK][44] = 42, + [1][0][RTW89_FCC][46] = 70, [1][0][RTW89_ETSI][46] = 28, [1][0][RTW89_MKK][46] = 127, - [1][0][RTW89_IC][46] = 84, - [1][0][RTW89_ACMA][46] = 84, - [1][0][RTW89_FCC][48] = 36, + [1][0][RTW89_IC][46] = 70, + [1][0][RTW89_KCC][46] = 54, + [1][0][RTW89_ACMA][46] = 70, + [1][0][RTW89_CN][46] = 66, + [1][0][RTW89_UK][46] = 42, + [1][0][RTW89_FCC][48] = 56, [1][0][RTW89_ETSI][48] = 127, [1][0][RTW89_MKK][48] = 127, [1][0][RTW89_IC][48] = 127, + [1][0][RTW89_KCC][48] = 127, [1][0][RTW89_ACMA][48] = 127, - [1][0][RTW89_FCC][50] = 36, + [1][0][RTW89_CN][48] = 127, + [1][0][RTW89_UK][48] = 127, + [1][0][RTW89_FCC][50] = 58, [1][0][RTW89_ETSI][50] = 127, [1][0][RTW89_MKK][50] = 127, [1][0][RTW89_IC][50] = 127, + [1][0][RTW89_KCC][50] = 127, [1][0][RTW89_ACMA][50] = 127, - [1][0][RTW89_FCC][52] = 36, + [1][0][RTW89_CN][50] = 127, + [1][0][RTW89_UK][50] = 127, + [1][0][RTW89_FCC][52] = 56, [1][0][RTW89_ETSI][52] = 127, [1][0][RTW89_MKK][52] = 127, [1][0][RTW89_IC][52] = 127, + [1][0][RTW89_KCC][52] = 127, [1][0][RTW89_ACMA][52] = 127, - [1][1][RTW89_FCC][0] = 34, + [1][0][RTW89_CN][52] = 127, + [1][0][RTW89_UK][52] = 127, + [1][1][RTW89_FCC][0] = 44, [1][1][RTW89_ETSI][0] = 30, [1][1][RTW89_MKK][0] = 34, - [1][1][RTW89_IC][0] = 10, - [1][1][RTW89_ACMA][0] = 22, - [1][1][RTW89_FCC][2] = 36, + [1][1][RTW89_IC][0] = 20, + [1][1][RTW89_KCC][0] = 34, + [1][1][RTW89_ACMA][0] = 30, + [1][1][RTW89_CN][0] = 14, + [1][1][RTW89_UK][0] = 30, + [1][1][RTW89_FCC][2] = 44, [1][1][RTW89_ETSI][2] = 30, [1][1][RTW89_MKK][2] = 34, - [1][1][RTW89_IC][2] = 14, - [1][1][RTW89_ACMA][2] = 22, - [1][1][RTW89_FCC][4] = 34, + [1][1][RTW89_IC][2] = 18, + [1][1][RTW89_KCC][2] = 34, + [1][1][RTW89_ACMA][2] = 30, + [1][1][RTW89_CN][2] = 14, + [1][1][RTW89_UK][2] = 30, + [1][1][RTW89_FCC][4] = 46, [1][1][RTW89_ETSI][4] = 30, [1][1][RTW89_MKK][4] = 26, - [1][1][RTW89_IC][4] = 10, - [1][1][RTW89_ACMA][4] = 22, - [1][1][RTW89_FCC][6] = 34, + [1][1][RTW89_IC][4] = 20, + [1][1][RTW89_KCC][4] = 34, + [1][1][RTW89_ACMA][4] = 30, + [1][1][RTW89_CN][4] = 14, + [1][1][RTW89_UK][4] = 30, + [1][1][RTW89_FCC][6] = 46, [1][1][RTW89_ETSI][6] = 30, [1][1][RTW89_MKK][6] = 26, - [1][1][RTW89_IC][6] = 10, - [1][1][RTW89_ACMA][6] = 22, - [1][1][RTW89_FCC][8] = 36, + [1][1][RTW89_IC][6] = 20, + [1][1][RTW89_KCC][6] = 8, + [1][1][RTW89_ACMA][6] = 30, + [1][1][RTW89_CN][6] = 14, + [1][1][RTW89_UK][6] = 30, + [1][1][RTW89_FCC][8] = 44, [1][1][RTW89_ETSI][8] = 30, [1][1][RTW89_MKK][8] = 20, [1][1][RTW89_IC][8] = 44, - [1][1][RTW89_ACMA][8] = 22, - [1][1][RTW89_FCC][10] = 36, + [1][1][RTW89_KCC][8] = 34, + [1][1][RTW89_ACMA][8] = 30, + [1][1][RTW89_CN][8] = 14, + [1][1][RTW89_UK][8] = 30, + [1][1][RTW89_FCC][10] = 44, [1][1][RTW89_ETSI][10] = 30, [1][1][RTW89_MKK][10] = 20, [1][1][RTW89_IC][10] = 44, - [1][1][RTW89_ACMA][10] = 22, - [1][1][RTW89_FCC][12] = 38, + [1][1][RTW89_KCC][10] = 34, + [1][1][RTW89_ACMA][10] = 30, + [1][1][RTW89_CN][10] = 14, + [1][1][RTW89_UK][10] = 30, + [1][1][RTW89_FCC][12] = 44, [1][1][RTW89_ETSI][12] = 30, [1][1][RTW89_MKK][12] = 34, - [1][1][RTW89_IC][12] = 46, - [1][1][RTW89_ACMA][12] = 22, - [1][1][RTW89_FCC][14] = 34, + [1][1][RTW89_IC][12] = 44, + [1][1][RTW89_KCC][12] = 38, + [1][1][RTW89_ACMA][12] = 30, + [1][1][RTW89_CN][12] = 14, + [1][1][RTW89_UK][12] = 30, + [1][1][RTW89_FCC][14] = 44, [1][1][RTW89_ETSI][14] = 30, [1][1][RTW89_MKK][14] = 34, - [1][1][RTW89_IC][14] = 40, - [1][1][RTW89_ACMA][14] = 22, - [1][1][RTW89_FCC][15] = 34, + [1][1][RTW89_IC][14] = 44, + [1][1][RTW89_KCC][14] = 38, + [1][1][RTW89_ACMA][14] = 30, + [1][1][RTW89_CN][14] = 14, + [1][1][RTW89_UK][14] = 30, + [1][1][RTW89_FCC][15] = 44, [1][1][RTW89_ETSI][15] = 28, [1][1][RTW89_MKK][15] = 56, - [1][1][RTW89_IC][15] = 42, - [1][1][RTW89_ACMA][15] = 22, - [1][1][RTW89_FCC][17] = 34, + [1][1][RTW89_IC][15] = 44, + [1][1][RTW89_KCC][15] = 36, + [1][1][RTW89_ACMA][15] = 28, + [1][1][RTW89_CN][15] = 127, + [1][1][RTW89_UK][15] = 28, + [1][1][RTW89_FCC][17] = 44, [1][1][RTW89_ETSI][17] = 28, [1][1][RTW89_MKK][17] = 58, - [1][1][RTW89_IC][17] = 42, - [1][1][RTW89_ACMA][17] = 22, - [1][1][RTW89_FCC][19] = 34, + [1][1][RTW89_IC][17] = 44, + [1][1][RTW89_KCC][17] = 36, + [1][1][RTW89_ACMA][17] = 28, + [1][1][RTW89_CN][17] = 127, + [1][1][RTW89_UK][17] = 28, + [1][1][RTW89_FCC][19] = 44, [1][1][RTW89_ETSI][19] = 28, [1][1][RTW89_MKK][19] = 58, - [1][1][RTW89_IC][19] = 42, - [1][1][RTW89_ACMA][19] = 22, - [1][1][RTW89_FCC][21] = 34, + [1][1][RTW89_IC][19] = 44, + [1][1][RTW89_KCC][19] = 36, + [1][1][RTW89_ACMA][19] = 28, + [1][1][RTW89_CN][19] = 127, + [1][1][RTW89_UK][19] = 28, + [1][1][RTW89_FCC][21] = 44, [1][1][RTW89_ETSI][21] = 28, [1][1][RTW89_MKK][21] = 58, - [1][1][RTW89_IC][21] = 42, - [1][1][RTW89_ACMA][21] = 22, - [1][1][RTW89_FCC][23] = 34, + [1][1][RTW89_IC][21] = 44, + [1][1][RTW89_KCC][21] = 36, + [1][1][RTW89_ACMA][21] = 28, + [1][1][RTW89_CN][21] = 127, + [1][1][RTW89_UK][21] = 28, + [1][1][RTW89_FCC][23] = 44, [1][1][RTW89_ETSI][23] = 28, [1][1][RTW89_MKK][23] = 58, - [1][1][RTW89_IC][23] = 42, - [1][1][RTW89_ACMA][23] = 22, - [1][1][RTW89_FCC][25] = 34, + [1][1][RTW89_IC][23] = 44, + [1][1][RTW89_KCC][23] = 36, + [1][1][RTW89_ACMA][23] = 28, + [1][1][RTW89_CN][23] = 127, + [1][1][RTW89_UK][23] = 28, + [1][1][RTW89_FCC][25] = 44, [1][1][RTW89_ETSI][25] = 28, [1][1][RTW89_MKK][25] = 58, [1][1][RTW89_IC][25] = 127, + [1][1][RTW89_KCC][25] = 36, [1][1][RTW89_ACMA][25] = 127, - [1][1][RTW89_FCC][27] = 34, + [1][1][RTW89_CN][25] = 127, + [1][1][RTW89_UK][25] = 28, + [1][1][RTW89_FCC][27] = 44, [1][1][RTW89_ETSI][27] = 30, [1][1][RTW89_MKK][27] = 58, [1][1][RTW89_IC][27] = 127, + [1][1][RTW89_KCC][27] = 36, [1][1][RTW89_ACMA][27] = 127, - [1][1][RTW89_FCC][29] = 34, + [1][1][RTW89_CN][27] = 127, + [1][1][RTW89_UK][27] = 30, + [1][1][RTW89_FCC][29] = 44, [1][1][RTW89_ETSI][29] = 30, [1][1][RTW89_MKK][29] = 58, [1][1][RTW89_IC][29] = 127, + [1][1][RTW89_KCC][29] = 36, [1][1][RTW89_ACMA][29] = 127, - [1][1][RTW89_FCC][31] = 34, + [1][1][RTW89_CN][29] = 127, + [1][1][RTW89_UK][29] = 30, + [1][1][RTW89_FCC][31] = 44, [1][1][RTW89_ETSI][31] = 30, [1][1][RTW89_MKK][31] = 58, [1][1][RTW89_IC][31] = 38, - [1][1][RTW89_ACMA][31] = 22, - [1][1][RTW89_FCC][33] = 32, + [1][1][RTW89_KCC][31] = 36, + [1][1][RTW89_ACMA][31] = 30, + [1][1][RTW89_CN][31] = 127, + [1][1][RTW89_UK][31] = 30, + [1][1][RTW89_FCC][33] = 38, [1][1][RTW89_ETSI][33] = 30, [1][1][RTW89_MKK][33] = 58, [1][1][RTW89_IC][33] = 38, - [1][1][RTW89_ACMA][33] = 22, - [1][1][RTW89_FCC][35] = 32, + [1][1][RTW89_KCC][33] = 36, + [1][1][RTW89_ACMA][33] = 30, + [1][1][RTW89_CN][33] = 127, + [1][1][RTW89_UK][33] = 30, + [1][1][RTW89_FCC][35] = 38, [1][1][RTW89_ETSI][35] = 30, [1][1][RTW89_MKK][35] = 58, [1][1][RTW89_IC][35] = 38, - [1][1][RTW89_ACMA][35] = 22, - [1][1][RTW89_FCC][37] = 40, + [1][1][RTW89_KCC][35] = 36, + [1][1][RTW89_ACMA][35] = 30, + [1][1][RTW89_CN][35] = 127, + [1][1][RTW89_UK][35] = 30, + [1][1][RTW89_FCC][37] = 46, [1][1][RTW89_ETSI][37] = 127, [1][1][RTW89_MKK][37] = 58, - [1][1][RTW89_IC][37] = 48, - [1][1][RTW89_ACMA][37] = 48, - [1][1][RTW89_FCC][38] = 76, + [1][1][RTW89_IC][37] = 46, + [1][1][RTW89_KCC][37] = 36, + [1][1][RTW89_ACMA][37] = 46, + [1][1][RTW89_CN][37] = 127, + [1][1][RTW89_UK][37] = 32, + [1][1][RTW89_FCC][38] = 74, [1][1][RTW89_ETSI][38] = 16, [1][1][RTW89_MKK][38] = 127, - [1][1][RTW89_IC][38] = 84, - [1][1][RTW89_ACMA][38] = 82, - [1][1][RTW89_FCC][40] = 76, + [1][1][RTW89_IC][38] = 74, + [1][1][RTW89_KCC][38] = 36, + [1][1][RTW89_ACMA][38] = 74, + [1][1][RTW89_CN][38] = 54, + [1][1][RTW89_UK][38] = 30, + [1][1][RTW89_FCC][40] = 74, [1][1][RTW89_ETSI][40] = 16, [1][1][RTW89_MKK][40] = 127, - [1][1][RTW89_IC][40] = 84, - [1][1][RTW89_ACMA][40] = 82, - [1][1][RTW89_FCC][42] = 76, + [1][1][RTW89_IC][40] = 74, + [1][1][RTW89_KCC][40] = 36, + [1][1][RTW89_ACMA][40] = 74, + [1][1][RTW89_CN][40] = 54, + [1][1][RTW89_UK][40] = 30, + [1][1][RTW89_FCC][42] = 74, [1][1][RTW89_ETSI][42] = 16, [1][1][RTW89_MKK][42] = 127, - [1][1][RTW89_IC][42] = 84, - [1][1][RTW89_ACMA][42] = 84, - [1][1][RTW89_FCC][44] = 76, + [1][1][RTW89_IC][42] = 74, + [1][1][RTW89_KCC][42] = 36, + [1][1][RTW89_ACMA][42] = 74, + [1][1][RTW89_CN][42] = 54, + [1][1][RTW89_UK][42] = 30, + [1][1][RTW89_FCC][44] = 74, [1][1][RTW89_ETSI][44] = 16, [1][1][RTW89_MKK][44] = 127, - [1][1][RTW89_IC][44] = 84, - [1][1][RTW89_ACMA][44] = 84, - [1][1][RTW89_FCC][46] = 76, + [1][1][RTW89_IC][44] = 74, + [1][1][RTW89_KCC][44] = 36, + [1][1][RTW89_ACMA][44] = 74, + [1][1][RTW89_CN][44] = 54, + [1][1][RTW89_UK][44] = 30, + [1][1][RTW89_FCC][46] = 74, [1][1][RTW89_ETSI][46] = 16, [1][1][RTW89_MKK][46] = 127, - [1][1][RTW89_IC][46] = 84, - [1][1][RTW89_ACMA][46] = 84, - [1][1][RTW89_FCC][48] = 24, + [1][1][RTW89_IC][46] = 74, + [1][1][RTW89_KCC][46] = 36, + [1][1][RTW89_ACMA][46] = 74, + [1][1][RTW89_CN][46] = 54, + [1][1][RTW89_UK][46] = 30, + [1][1][RTW89_FCC][48] = 34, [1][1][RTW89_ETSI][48] = 127, [1][1][RTW89_MKK][48] = 127, [1][1][RTW89_IC][48] = 127, + [1][1][RTW89_KCC][48] = 127, [1][1][RTW89_ACMA][48] = 127, - [1][1][RTW89_FCC][50] = 24, + [1][1][RTW89_CN][48] = 127, + [1][1][RTW89_UK][48] = 127, + [1][1][RTW89_FCC][50] = 34, [1][1][RTW89_ETSI][50] = 127, [1][1][RTW89_MKK][50] = 127, [1][1][RTW89_IC][50] = 127, + [1][1][RTW89_KCC][50] = 127, [1][1][RTW89_ACMA][50] = 127, - [1][1][RTW89_FCC][52] = 24, + [1][1][RTW89_CN][50] = 127, + [1][1][RTW89_UK][50] = 127, + [1][1][RTW89_FCC][52] = 30, [1][1][RTW89_ETSI][52] = 127, [1][1][RTW89_MKK][52] = 127, [1][1][RTW89_IC][52] = 127, + [1][1][RTW89_KCC][52] = 127, [1][1][RTW89_ACMA][52] = 127, - [2][0][RTW89_FCC][0] = 62, + [1][1][RTW89_CN][52] = 127, + [1][1][RTW89_UK][52] = 127, + [2][0][RTW89_FCC][0] = 68, [2][0][RTW89_ETSI][0] = 52, [2][0][RTW89_MKK][0] = 60, - [2][0][RTW89_IC][0] = 46, - [2][0][RTW89_ACMA][0] = 48, - [2][0][RTW89_FCC][2] = 62, + [2][0][RTW89_IC][0] = 52, + [2][0][RTW89_KCC][0] = 64, + [2][0][RTW89_ACMA][0] = 52, + [2][0][RTW89_CN][0] = 40, + [2][0][RTW89_UK][0] = 52, + [2][0][RTW89_FCC][2] = 64, [2][0][RTW89_ETSI][2] = 52, [2][0][RTW89_MKK][2] = 60, - [2][0][RTW89_IC][2] = 46, - [2][0][RTW89_ACMA][2] = 48, - [2][0][RTW89_FCC][4] = 62, + [2][0][RTW89_IC][2] = 50, + [2][0][RTW89_KCC][2] = 64, + [2][0][RTW89_ACMA][2] = 52, + [2][0][RTW89_CN][2] = 40, + [2][0][RTW89_UK][2] = 52, + [2][0][RTW89_FCC][4] = 68, [2][0][RTW89_ETSI][4] = 52, [2][0][RTW89_MKK][4] = 50, - [2][0][RTW89_IC][4] = 46, - [2][0][RTW89_ACMA][4] = 48, - [2][0][RTW89_FCC][6] = 62, + [2][0][RTW89_IC][4] = 50, + [2][0][RTW89_KCC][4] = 64, + [2][0][RTW89_ACMA][4] = 52, + [2][0][RTW89_CN][4] = 40, + [2][0][RTW89_UK][4] = 52, + [2][0][RTW89_FCC][6] = 68, [2][0][RTW89_ETSI][6] = 52, [2][0][RTW89_MKK][6] = 50, - [2][0][RTW89_IC][6] = 46, - [2][0][RTW89_ACMA][6] = 48, - [2][0][RTW89_FCC][8] = 62, + [2][0][RTW89_IC][6] = 50, + [2][0][RTW89_KCC][6] = 36, + [2][0][RTW89_ACMA][6] = 52, + [2][0][RTW89_CN][6] = 40, + [2][0][RTW89_UK][6] = 52, + [2][0][RTW89_FCC][8] = 68, [2][0][RTW89_ETSI][8] = 52, [2][0][RTW89_MKK][8] = 44, - [2][0][RTW89_IC][8] = 66, - [2][0][RTW89_ACMA][8] = 48, - [2][0][RTW89_FCC][10] = 62, + [2][0][RTW89_IC][8] = 64, + [2][0][RTW89_KCC][8] = 62, + [2][0][RTW89_ACMA][8] = 52, + [2][0][RTW89_CN][8] = 40, + [2][0][RTW89_UK][8] = 52, + [2][0][RTW89_FCC][10] = 68, [2][0][RTW89_ETSI][10] = 52, [2][0][RTW89_MKK][10] = 44, - [2][0][RTW89_IC][10] = 66, - [2][0][RTW89_ACMA][10] = 48, - [2][0][RTW89_FCC][12] = 62, + [2][0][RTW89_IC][10] = 64, + [2][0][RTW89_KCC][10] = 62, + [2][0][RTW89_ACMA][10] = 52, + [2][0][RTW89_CN][10] = 40, + [2][0][RTW89_UK][10] = 52, + [2][0][RTW89_FCC][12] = 68, [2][0][RTW89_ETSI][12] = 52, [2][0][RTW89_MKK][12] = 58, - [2][0][RTW89_IC][12] = 66, - [2][0][RTW89_ACMA][12] = 48, - [2][0][RTW89_FCC][14] = 62, + [2][0][RTW89_IC][12] = 64, + [2][0][RTW89_KCC][12] = 62, + [2][0][RTW89_ACMA][12] = 52, + [2][0][RTW89_CN][12] = 40, + [2][0][RTW89_UK][12] = 52, + [2][0][RTW89_FCC][14] = 68, [2][0][RTW89_ETSI][14] = 52, [2][0][RTW89_MKK][14] = 58, - [2][0][RTW89_IC][14] = 66, - [2][0][RTW89_ACMA][14] = 48, - [2][0][RTW89_FCC][15] = 62, + [2][0][RTW89_IC][14] = 64, + [2][0][RTW89_KCC][14] = 62, + [2][0][RTW89_ACMA][14] = 52, + [2][0][RTW89_CN][14] = 40, + [2][0][RTW89_UK][14] = 52, + [2][0][RTW89_FCC][15] = 68, [2][0][RTW89_ETSI][15] = 52, [2][0][RTW89_MKK][15] = 68, - [2][0][RTW89_IC][15] = 70, - [2][0][RTW89_ACMA][15] = 48, - [2][0][RTW89_FCC][17] = 62, + [2][0][RTW89_IC][15] = 68, + [2][0][RTW89_KCC][15] = 62, + [2][0][RTW89_ACMA][15] = 52, + [2][0][RTW89_CN][15] = 127, + [2][0][RTW89_UK][15] = 52, + [2][0][RTW89_FCC][17] = 68, [2][0][RTW89_ETSI][17] = 52, [2][0][RTW89_MKK][17] = 74, - [2][0][RTW89_IC][17] = 70, - [2][0][RTW89_ACMA][17] = 48, - [2][0][RTW89_FCC][19] = 62, + [2][0][RTW89_IC][17] = 68, + [2][0][RTW89_KCC][17] = 62, + [2][0][RTW89_ACMA][17] = 52, + [2][0][RTW89_CN][17] = 127, + [2][0][RTW89_UK][17] = 52, + [2][0][RTW89_FCC][19] = 70, [2][0][RTW89_ETSI][19] = 52, [2][0][RTW89_MKK][19] = 74, [2][0][RTW89_IC][19] = 70, - [2][0][RTW89_ACMA][19] = 48, - [2][0][RTW89_FCC][21] = 62, + [2][0][RTW89_KCC][19] = 62, + [2][0][RTW89_ACMA][19] = 52, + [2][0][RTW89_CN][19] = 127, + [2][0][RTW89_UK][19] = 52, + [2][0][RTW89_FCC][21] = 70, [2][0][RTW89_ETSI][21] = 52, [2][0][RTW89_MKK][21] = 74, [2][0][RTW89_IC][21] = 70, - [2][0][RTW89_ACMA][21] = 48, - [2][0][RTW89_FCC][23] = 62, + [2][0][RTW89_KCC][21] = 62, + [2][0][RTW89_ACMA][21] = 52, + [2][0][RTW89_CN][21] = 127, + [2][0][RTW89_UK][21] = 52, + [2][0][RTW89_FCC][23] = 70, [2][0][RTW89_ETSI][23] = 52, [2][0][RTW89_MKK][23] = 74, [2][0][RTW89_IC][23] = 70, - [2][0][RTW89_ACMA][23] = 48, - [2][0][RTW89_FCC][25] = 62, + [2][0][RTW89_KCC][23] = 62, + [2][0][RTW89_ACMA][23] = 52, + [2][0][RTW89_CN][23] = 127, + [2][0][RTW89_UK][23] = 52, + [2][0][RTW89_FCC][25] = 70, [2][0][RTW89_ETSI][25] = 52, [2][0][RTW89_MKK][25] = 74, [2][0][RTW89_IC][25] = 127, + [2][0][RTW89_KCC][25] = 62, [2][0][RTW89_ACMA][25] = 127, - [2][0][RTW89_FCC][27] = 62, + [2][0][RTW89_CN][25] = 127, + [2][0][RTW89_UK][25] = 52, + [2][0][RTW89_FCC][27] = 70, [2][0][RTW89_ETSI][27] = 52, [2][0][RTW89_MKK][27] = 74, [2][0][RTW89_IC][27] = 127, + [2][0][RTW89_KCC][27] = 62, [2][0][RTW89_ACMA][27] = 127, - [2][0][RTW89_FCC][29] = 62, + [2][0][RTW89_CN][27] = 127, + [2][0][RTW89_UK][27] = 52, + [2][0][RTW89_FCC][29] = 70, [2][0][RTW89_ETSI][29] = 52, [2][0][RTW89_MKK][29] = 74, [2][0][RTW89_IC][29] = 127, + [2][0][RTW89_KCC][29] = 62, [2][0][RTW89_ACMA][29] = 127, - [2][0][RTW89_FCC][31] = 62, + [2][0][RTW89_CN][29] = 127, + [2][0][RTW89_UK][29] = 52, + [2][0][RTW89_FCC][31] = 70, [2][0][RTW89_ETSI][31] = 52, [2][0][RTW89_MKK][31] = 74, - [2][0][RTW89_IC][31] = 72, - [2][0][RTW89_ACMA][31] = 48, - [2][0][RTW89_FCC][33] = 64, + [2][0][RTW89_IC][31] = 62, + [2][0][RTW89_KCC][31] = 62, + [2][0][RTW89_ACMA][31] = 52, + [2][0][RTW89_CN][31] = 127, + [2][0][RTW89_UK][31] = 52, + [2][0][RTW89_FCC][33] = 62, [2][0][RTW89_ETSI][33] = 52, [2][0][RTW89_MKK][33] = 74, - [2][0][RTW89_IC][33] = 72, - [2][0][RTW89_ACMA][33] = 48, - [2][0][RTW89_FCC][35] = 64, + [2][0][RTW89_IC][33] = 62, + [2][0][RTW89_KCC][33] = 62, + [2][0][RTW89_ACMA][33] = 52, + [2][0][RTW89_CN][33] = 127, + [2][0][RTW89_UK][33] = 52, + [2][0][RTW89_FCC][35] = 62, [2][0][RTW89_ETSI][35] = 52, [2][0][RTW89_MKK][35] = 74, - [2][0][RTW89_IC][35] = 72, - [2][0][RTW89_ACMA][35] = 48, - [2][0][RTW89_FCC][37] = 62, + [2][0][RTW89_IC][35] = 62, + [2][0][RTW89_KCC][35] = 62, + [2][0][RTW89_ACMA][35] = 52, + [2][0][RTW89_CN][35] = 127, + [2][0][RTW89_UK][35] = 52, + [2][0][RTW89_FCC][37] = 70, [2][0][RTW89_ETSI][37] = 127, [2][0][RTW89_MKK][37] = 74, [2][0][RTW89_IC][37] = 70, - [2][0][RTW89_ACMA][37] = 76, - [2][0][RTW89_FCC][38] = 76, + [2][0][RTW89_KCC][37] = 62, + [2][0][RTW89_ACMA][37] = 70, + [2][0][RTW89_CN][37] = 127, + [2][0][RTW89_UK][37] = 52, + [2][0][RTW89_FCC][38] = 82, [2][0][RTW89_ETSI][38] = 28, [2][0][RTW89_MKK][38] = 127, - [2][0][RTW89_IC][38] = 84, - [2][0][RTW89_ACMA][38] = 84, - [2][0][RTW89_FCC][40] = 76, + [2][0][RTW89_IC][38] = 82, + [2][0][RTW89_KCC][38] = 64, + [2][0][RTW89_ACMA][38] = 82, + [2][0][RTW89_CN][38] = 68, + [2][0][RTW89_UK][38] = 54, + [2][0][RTW89_FCC][40] = 82, [2][0][RTW89_ETSI][40] = 28, [2][0][RTW89_MKK][40] = 127, - [2][0][RTW89_IC][40] = 84, - [2][0][RTW89_ACMA][40] = 84, + [2][0][RTW89_IC][40] = 82, + [2][0][RTW89_KCC][40] = 64, + [2][0][RTW89_ACMA][40] = 82, + [2][0][RTW89_CN][40] = 68, + [2][0][RTW89_UK][40] = 54, [2][0][RTW89_FCC][42] = 76, [2][0][RTW89_ETSI][42] = 28, [2][0][RTW89_MKK][42] = 127, - [2][0][RTW89_IC][42] = 84, - [2][0][RTW89_ACMA][42] = 84, - [2][0][RTW89_FCC][44] = 76, + [2][0][RTW89_IC][42] = 76, + [2][0][RTW89_KCC][42] = 64, + [2][0][RTW89_ACMA][42] = 76, + [2][0][RTW89_CN][42] = 68, + [2][0][RTW89_UK][42] = 54, + [2][0][RTW89_FCC][44] = 80, [2][0][RTW89_ETSI][44] = 28, [2][0][RTW89_MKK][44] = 127, - [2][0][RTW89_IC][44] = 84, - [2][0][RTW89_ACMA][44] = 84, - [2][0][RTW89_FCC][46] = 76, + [2][0][RTW89_IC][44] = 80, + [2][0][RTW89_KCC][44] = 64, + [2][0][RTW89_ACMA][44] = 80, + [2][0][RTW89_CN][44] = 68, + [2][0][RTW89_UK][44] = 54, + [2][0][RTW89_FCC][46] = 80, [2][0][RTW89_ETSI][46] = 28, [2][0][RTW89_MKK][46] = 127, - [2][0][RTW89_IC][46] = 84, - [2][0][RTW89_ACMA][46] = 84, - [2][0][RTW89_FCC][48] = 48, + [2][0][RTW89_IC][46] = 80, + [2][0][RTW89_KCC][46] = 64, + [2][0][RTW89_ACMA][46] = 80, + [2][0][RTW89_CN][46] = 68, + [2][0][RTW89_UK][46] = 54, + [2][0][RTW89_FCC][48] = 64, [2][0][RTW89_ETSI][48] = 127, [2][0][RTW89_MKK][48] = 127, [2][0][RTW89_IC][48] = 127, + [2][0][RTW89_KCC][48] = 127, [2][0][RTW89_ACMA][48] = 127, - [2][0][RTW89_FCC][50] = 48, + [2][0][RTW89_CN][48] = 127, + [2][0][RTW89_UK][48] = 127, + [2][0][RTW89_FCC][50] = 64, [2][0][RTW89_ETSI][50] = 127, [2][0][RTW89_MKK][50] = 127, [2][0][RTW89_IC][50] = 127, + [2][0][RTW89_KCC][50] = 127, [2][0][RTW89_ACMA][50] = 127, - [2][0][RTW89_FCC][52] = 48, + [2][0][RTW89_CN][50] = 127, + [2][0][RTW89_UK][50] = 127, + [2][0][RTW89_FCC][52] = 64, [2][0][RTW89_ETSI][52] = 127, [2][0][RTW89_MKK][52] = 127, [2][0][RTW89_IC][52] = 127, + [2][0][RTW89_KCC][52] = 127, [2][0][RTW89_ACMA][52] = 127, - [2][1][RTW89_FCC][0] = 42, + [2][0][RTW89_CN][52] = 127, + [2][0][RTW89_UK][52] = 127, + [2][1][RTW89_FCC][0] = 50, [2][1][RTW89_ETSI][0] = 40, [2][1][RTW89_MKK][0] = 44, - [2][1][RTW89_IC][0] = 20, - [2][1][RTW89_ACMA][0] = 36, - [2][1][RTW89_FCC][2] = 42, + [2][1][RTW89_IC][0] = 26, + [2][1][RTW89_KCC][0] = 44, + [2][1][RTW89_ACMA][0] = 40, + [2][1][RTW89_CN][0] = 28, + [2][1][RTW89_UK][0] = 40, + [2][1][RTW89_FCC][2] = 50, [2][1][RTW89_ETSI][2] = 40, [2][1][RTW89_MKK][2] = 44, - [2][1][RTW89_IC][2] = 18, - [2][1][RTW89_ACMA][2] = 36, - [2][1][RTW89_FCC][4] = 42, + [2][1][RTW89_IC][2] = 26, + [2][1][RTW89_KCC][2] = 44, + [2][1][RTW89_ACMA][2] = 40, + [2][1][RTW89_CN][2] = 28, + [2][1][RTW89_UK][2] = 40, + [2][1][RTW89_FCC][4] = 50, [2][1][RTW89_ETSI][4] = 40, [2][1][RTW89_MKK][4] = 36, - [2][1][RTW89_IC][4] = 22, - [2][1][RTW89_ACMA][4] = 36, - [2][1][RTW89_FCC][6] = 42, + [2][1][RTW89_IC][4] = 26, + [2][1][RTW89_KCC][4] = 44, + [2][1][RTW89_ACMA][4] = 40, + [2][1][RTW89_CN][4] = 28, + [2][1][RTW89_UK][4] = 40, + [2][1][RTW89_FCC][6] = 50, [2][1][RTW89_ETSI][6] = 40, [2][1][RTW89_MKK][6] = 36, - [2][1][RTW89_IC][6] = 22, - [2][1][RTW89_ACMA][6] = 36, - [2][1][RTW89_FCC][8] = 42, + [2][1][RTW89_IC][6] = 26, + [2][1][RTW89_KCC][6] = 20, + [2][1][RTW89_ACMA][6] = 40, + [2][1][RTW89_CN][6] = 28, + [2][1][RTW89_UK][6] = 40, + [2][1][RTW89_FCC][8] = 50, [2][1][RTW89_ETSI][8] = 40, [2][1][RTW89_MKK][8] = 32, [2][1][RTW89_IC][8] = 50, - [2][1][RTW89_ACMA][8] = 36, - [2][1][RTW89_FCC][10] = 42, + [2][1][RTW89_KCC][8] = 46, + [2][1][RTW89_ACMA][8] = 40, + [2][1][RTW89_CN][8] = 28, + [2][1][RTW89_UK][8] = 40, + [2][1][RTW89_FCC][10] = 50, [2][1][RTW89_ETSI][10] = 40, [2][1][RTW89_MKK][10] = 32, [2][1][RTW89_IC][10] = 50, - [2][1][RTW89_ACMA][10] = 36, - [2][1][RTW89_FCC][12] = 44, + [2][1][RTW89_KCC][10] = 46, + [2][1][RTW89_ACMA][10] = 40, + [2][1][RTW89_CN][10] = 28, + [2][1][RTW89_UK][10] = 40, + [2][1][RTW89_FCC][12] = 48, [2][1][RTW89_ETSI][12] = 40, [2][1][RTW89_MKK][12] = 44, - [2][1][RTW89_IC][12] = 52, - [2][1][RTW89_ACMA][12] = 36, - [2][1][RTW89_FCC][14] = 44, + [2][1][RTW89_IC][12] = 48, + [2][1][RTW89_KCC][12] = 46, + [2][1][RTW89_ACMA][12] = 40, + [2][1][RTW89_CN][12] = 28, + [2][1][RTW89_UK][12] = 40, + [2][1][RTW89_FCC][14] = 48, [2][1][RTW89_ETSI][14] = 40, [2][1][RTW89_MKK][14] = 44, - [2][1][RTW89_IC][14] = 52, - [2][1][RTW89_ACMA][14] = 36, - [2][1][RTW89_FCC][15] = 42, + [2][1][RTW89_IC][14] = 48, + [2][1][RTW89_KCC][14] = 46, + [2][1][RTW89_ACMA][14] = 40, + [2][1][RTW89_CN][14] = 28, + [2][1][RTW89_UK][14] = 40, + [2][1][RTW89_FCC][15] = 50, [2][1][RTW89_ETSI][15] = 40, [2][1][RTW89_MKK][15] = 66, [2][1][RTW89_IC][15] = 50, - [2][1][RTW89_ACMA][15] = 36, - [2][1][RTW89_FCC][17] = 42, + [2][1][RTW89_KCC][15] = 46, + [2][1][RTW89_ACMA][15] = 40, + [2][1][RTW89_CN][15] = 127, + [2][1][RTW89_UK][15] = 40, + [2][1][RTW89_FCC][17] = 50, [2][1][RTW89_ETSI][17] = 40, [2][1][RTW89_MKK][17] = 66, [2][1][RTW89_IC][17] = 50, - [2][1][RTW89_ACMA][17] = 36, - [2][1][RTW89_FCC][19] = 42, + [2][1][RTW89_KCC][17] = 46, + [2][1][RTW89_ACMA][17] = 40, + [2][1][RTW89_CN][17] = 127, + [2][1][RTW89_UK][17] = 40, + [2][1][RTW89_FCC][19] = 50, [2][1][RTW89_ETSI][19] = 40, [2][1][RTW89_MKK][19] = 66, [2][1][RTW89_IC][19] = 50, - [2][1][RTW89_ACMA][19] = 36, - [2][1][RTW89_FCC][21] = 42, + [2][1][RTW89_KCC][19] = 46, + [2][1][RTW89_ACMA][19] = 40, + [2][1][RTW89_CN][19] = 127, + [2][1][RTW89_UK][19] = 40, + [2][1][RTW89_FCC][21] = 50, [2][1][RTW89_ETSI][21] = 40, [2][1][RTW89_MKK][21] = 66, [2][1][RTW89_IC][21] = 50, - [2][1][RTW89_ACMA][21] = 36, - [2][1][RTW89_FCC][23] = 42, + [2][1][RTW89_KCC][21] = 46, + [2][1][RTW89_ACMA][21] = 40, + [2][1][RTW89_CN][21] = 127, + [2][1][RTW89_UK][21] = 40, + [2][1][RTW89_FCC][23] = 50, [2][1][RTW89_ETSI][23] = 40, [2][1][RTW89_MKK][23] = 66, [2][1][RTW89_IC][23] = 50, - [2][1][RTW89_ACMA][23] = 36, - [2][1][RTW89_FCC][25] = 42, + [2][1][RTW89_KCC][23] = 46, + [2][1][RTW89_ACMA][23] = 40, + [2][1][RTW89_CN][23] = 127, + [2][1][RTW89_UK][23] = 40, + [2][1][RTW89_FCC][25] = 50, [2][1][RTW89_ETSI][25] = 40, [2][1][RTW89_MKK][25] = 66, [2][1][RTW89_IC][25] = 127, + [2][1][RTW89_KCC][25] = 46, [2][1][RTW89_ACMA][25] = 127, - [2][1][RTW89_FCC][27] = 42, + [2][1][RTW89_CN][25] = 127, + [2][1][RTW89_UK][25] = 40, + [2][1][RTW89_FCC][27] = 50, [2][1][RTW89_ETSI][27] = 40, [2][1][RTW89_MKK][27] = 66, [2][1][RTW89_IC][27] = 127, + [2][1][RTW89_KCC][27] = 46, [2][1][RTW89_ACMA][27] = 127, - [2][1][RTW89_FCC][29] = 42, + [2][1][RTW89_CN][27] = 127, + [2][1][RTW89_UK][27] = 40, + [2][1][RTW89_FCC][29] = 50, [2][1][RTW89_ETSI][29] = 40, [2][1][RTW89_MKK][29] = 66, [2][1][RTW89_IC][29] = 127, + [2][1][RTW89_KCC][29] = 46, [2][1][RTW89_ACMA][29] = 127, - [2][1][RTW89_FCC][31] = 42, + [2][1][RTW89_CN][29] = 127, + [2][1][RTW89_UK][29] = 40, + [2][1][RTW89_FCC][31] = 50, [2][1][RTW89_ETSI][31] = 40, [2][1][RTW89_MKK][31] = 66, - [2][1][RTW89_IC][31] = 50, - [2][1][RTW89_ACMA][31] = 36, - [2][1][RTW89_FCC][33] = 42, + [2][1][RTW89_IC][31] = 48, + [2][1][RTW89_KCC][31] = 46, + [2][1][RTW89_ACMA][31] = 40, + [2][1][RTW89_CN][31] = 127, + [2][1][RTW89_UK][31] = 40, + [2][1][RTW89_FCC][33] = 48, [2][1][RTW89_ETSI][33] = 40, [2][1][RTW89_MKK][33] = 66, - [2][1][RTW89_IC][33] = 50, - [2][1][RTW89_ACMA][33] = 36, - [2][1][RTW89_FCC][35] = 42, + [2][1][RTW89_IC][33] = 48, + [2][1][RTW89_KCC][33] = 46, + [2][1][RTW89_ACMA][33] = 40, + [2][1][RTW89_CN][33] = 127, + [2][1][RTW89_UK][33] = 40, + [2][1][RTW89_FCC][35] = 48, [2][1][RTW89_ETSI][35] = 40, [2][1][RTW89_MKK][35] = 66, - [2][1][RTW89_IC][35] = 50, - [2][1][RTW89_ACMA][35] = 36, - [2][1][RTW89_FCC][37] = 42, + [2][1][RTW89_IC][35] = 48, + [2][1][RTW89_KCC][35] = 46, + [2][1][RTW89_ACMA][35] = 40, + [2][1][RTW89_CN][35] = 127, + [2][1][RTW89_UK][35] = 40, + [2][1][RTW89_FCC][37] = 52, [2][1][RTW89_ETSI][37] = 127, [2][1][RTW89_MKK][37] = 66, - [2][1][RTW89_IC][37] = 50, - [2][1][RTW89_ACMA][37] = 60, - [2][1][RTW89_FCC][38] = 76, + [2][1][RTW89_IC][37] = 52, + [2][1][RTW89_KCC][37] = 46, + [2][1][RTW89_ACMA][37] = 52, + [2][1][RTW89_CN][37] = 127, + [2][1][RTW89_UK][37] = 42, + [2][1][RTW89_FCC][38] = 78, [2][1][RTW89_ETSI][38] = 16, [2][1][RTW89_MKK][38] = 127, - [2][1][RTW89_IC][38] = 84, - [2][1][RTW89_ACMA][38] = 84, - [2][1][RTW89_FCC][40] = 76, + [2][1][RTW89_IC][38] = 78, + [2][1][RTW89_KCC][38] = 46, + [2][1][RTW89_ACMA][38] = 78, + [2][1][RTW89_CN][38] = 56, + [2][1][RTW89_UK][38] = 42, + [2][1][RTW89_FCC][40] = 78, [2][1][RTW89_ETSI][40] = 16, [2][1][RTW89_MKK][40] = 127, - [2][1][RTW89_IC][40] = 84, - [2][1][RTW89_ACMA][40] = 84, - [2][1][RTW89_FCC][42] = 76, + [2][1][RTW89_IC][40] = 78, + [2][1][RTW89_KCC][40] = 46, + [2][1][RTW89_ACMA][40] = 78, + [2][1][RTW89_CN][40] = 56, + [2][1][RTW89_UK][40] = 42, + [2][1][RTW89_FCC][42] = 78, [2][1][RTW89_ETSI][42] = 16, [2][1][RTW89_MKK][42] = 127, - [2][1][RTW89_IC][42] = 84, - [2][1][RTW89_ACMA][42] = 84, - [2][1][RTW89_FCC][44] = 76, + [2][1][RTW89_IC][42] = 78, + [2][1][RTW89_KCC][42] = 46, + [2][1][RTW89_ACMA][42] = 78, + [2][1][RTW89_CN][42] = 56, + [2][1][RTW89_UK][42] = 42, + [2][1][RTW89_FCC][44] = 74, [2][1][RTW89_ETSI][44] = 16, [2][1][RTW89_MKK][44] = 127, - [2][1][RTW89_IC][44] = 84, - [2][1][RTW89_ACMA][44] = 84, - [2][1][RTW89_FCC][46] = 76, + [2][1][RTW89_IC][44] = 74, + [2][1][RTW89_KCC][44] = 46, + [2][1][RTW89_ACMA][44] = 74, + [2][1][RTW89_CN][44] = 56, + [2][1][RTW89_UK][44] = 42, + [2][1][RTW89_FCC][46] = 74, [2][1][RTW89_ETSI][46] = 16, [2][1][RTW89_MKK][46] = 127, - [2][1][RTW89_IC][46] = 84, - [2][1][RTW89_ACMA][46] = 84, - [2][1][RTW89_FCC][48] = 36, + [2][1][RTW89_IC][46] = 74, + [2][1][RTW89_KCC][46] = 46, + [2][1][RTW89_ACMA][46] = 74, + [2][1][RTW89_CN][46] = 56, + [2][1][RTW89_UK][46] = 42, + [2][1][RTW89_FCC][48] = 40, [2][1][RTW89_ETSI][48] = 127, [2][1][RTW89_MKK][48] = 127, [2][1][RTW89_IC][48] = 127, + [2][1][RTW89_KCC][48] = 127, [2][1][RTW89_ACMA][48] = 127, - [2][1][RTW89_FCC][50] = 36, + [2][1][RTW89_CN][48] = 127, + [2][1][RTW89_UK][48] = 127, + [2][1][RTW89_FCC][50] = 40, [2][1][RTW89_ETSI][50] = 127, [2][1][RTW89_MKK][50] = 127, [2][1][RTW89_IC][50] = 127, + [2][1][RTW89_KCC][50] = 127, [2][1][RTW89_ACMA][50] = 127, - [2][1][RTW89_FCC][52] = 36, + [2][1][RTW89_CN][50] = 127, + [2][1][RTW89_UK][50] = 127, + [2][1][RTW89_FCC][52] = 40, [2][1][RTW89_ETSI][52] = 127, [2][1][RTW89_MKK][52] = 127, [2][1][RTW89_IC][52] = 127, + [2][1][RTW89_KCC][52] = 127, [2][1][RTW89_ACMA][52] = 127, + [2][1][RTW89_CN][52] = 127, + [2][1][RTW89_UK][52] = 127, }; const s8 rtw89_8852c_txpwr_lmt_ru_6g[RTW89_RU_NUM][RTW89_NTX_NUM] [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = { - [0][0][RTW89_WW][0] = 76, - [0][0][RTW89_WW][2] = 76, - [0][0][RTW89_WW][4] = 76, - [0][0][RTW89_WW][6] = 76, - [0][0][RTW89_WW][8] = 76, - [0][0][RTW89_WW][10] = 76, - [0][0][RTW89_WW][12] = 76, - [0][0][RTW89_WW][14] = 76, - [0][0][RTW89_WW][15] = 76, - [0][0][RTW89_WW][17] = 76, - [0][0][RTW89_WW][19] = 76, - [0][0][RTW89_WW][21] = 76, - [0][0][RTW89_WW][23] = 76, - [0][0][RTW89_WW][25] = 76, - [0][0][RTW89_WW][27] = 76, - [0][0][RTW89_WW][29] = 76, - [0][0][RTW89_WW][30] = 76, - [0][0][RTW89_WW][32] = 76, - [0][0][RTW89_WW][34] = 76, - [0][0][RTW89_WW][36] = 76, - [0][0][RTW89_WW][38] = 76, - [0][0][RTW89_WW][40] = 76, - [0][0][RTW89_WW][42] = 76, - [0][0][RTW89_WW][44] = 76, - [0][0][RTW89_WW][45] = 76, - [0][0][RTW89_WW][47] = 76, - [0][0][RTW89_WW][49] = 76, - [0][0][RTW89_WW][51] = 76, - [0][0][RTW89_WW][53] = 76, - [0][0][RTW89_WW][55] = 76, - [0][0][RTW89_WW][57] = 76, - [0][0][RTW89_WW][59] = 76, - [0][0][RTW89_WW][60] = 76, - [0][0][RTW89_WW][62] = 76, - [0][0][RTW89_WW][64] = 76, - [0][0][RTW89_WW][66] = 76, - [0][0][RTW89_WW][68] = 76, - [0][0][RTW89_WW][70] = 76, - [0][0][RTW89_WW][72] = 76, - [0][0][RTW89_WW][74] = 76, - [0][0][RTW89_WW][75] = 76, - [0][0][RTW89_WW][77] = 76, - [0][0][RTW89_WW][79] = 76, - [0][0][RTW89_WW][81] = 76, - [0][0][RTW89_WW][83] = 76, - [0][0][RTW89_WW][85] = 76, - [0][0][RTW89_WW][87] = 76, - [0][0][RTW89_WW][89] = 76, - [0][0][RTW89_WW][90] = 76, - [0][0][RTW89_WW][92] = 76, - [0][0][RTW89_WW][94] = 76, - [0][0][RTW89_WW][96] = 76, - [0][0][RTW89_WW][98] = 76, - [0][0][RTW89_WW][100] = 76, - [0][0][RTW89_WW][102] = 76, - [0][0][RTW89_WW][104] = 76, - [0][0][RTW89_WW][105] = 76, - [0][0][RTW89_WW][107] = 76, - [0][0][RTW89_WW][109] = 76, + [0][0][RTW89_WW][0] = -16, + [0][0][RTW89_WW][2] = -18, + [0][0][RTW89_WW][4] = -18, + [0][0][RTW89_WW][6] = -18, + [0][0][RTW89_WW][8] = -18, + [0][0][RTW89_WW][10] = -18, + [0][0][RTW89_WW][12] = -18, + [0][0][RTW89_WW][14] = -18, + [0][0][RTW89_WW][15] = -18, + [0][0][RTW89_WW][17] = -18, + [0][0][RTW89_WW][19] = -18, + [0][0][RTW89_WW][21] = -18, + [0][0][RTW89_WW][23] = -18, + [0][0][RTW89_WW][25] = -18, + [0][0][RTW89_WW][27] = -18, + [0][0][RTW89_WW][29] = -18, + [0][0][RTW89_WW][30] = -18, + [0][0][RTW89_WW][32] = -18, + [0][0][RTW89_WW][34] = -18, + [0][0][RTW89_WW][36] = -18, + [0][0][RTW89_WW][38] = -18, + [0][0][RTW89_WW][40] = -18, + [0][0][RTW89_WW][42] = -18, + [0][0][RTW89_WW][44] = -16, + [0][0][RTW89_WW][45] = -16, + [0][0][RTW89_WW][47] = -18, + [0][0][RTW89_WW][49] = -18, + [0][0][RTW89_WW][51] = -18, + [0][0][RTW89_WW][53] = -16, + [0][0][RTW89_WW][55] = -18, + [0][0][RTW89_WW][57] = -18, + [0][0][RTW89_WW][59] = -18, + [0][0][RTW89_WW][60] = -18, + [0][0][RTW89_WW][62] = -18, + [0][0][RTW89_WW][64] = -18, + [0][0][RTW89_WW][66] = -18, + [0][0][RTW89_WW][68] = -18, + [0][0][RTW89_WW][70] = -16, + [0][0][RTW89_WW][72] = -18, + [0][0][RTW89_WW][74] = -18, + [0][0][RTW89_WW][75] = -18, + [0][0][RTW89_WW][77] = -18, + [0][0][RTW89_WW][79] = -18, + [0][0][RTW89_WW][81] = -18, + [0][0][RTW89_WW][83] = -18, + [0][0][RTW89_WW][85] = -18, + [0][0][RTW89_WW][87] = -16, + [0][0][RTW89_WW][89] = -16, + [0][0][RTW89_WW][90] = -16, + [0][0][RTW89_WW][92] = -16, + [0][0][RTW89_WW][94] = -16, + [0][0][RTW89_WW][96] = -16, + [0][0][RTW89_WW][98] = -16, + [0][0][RTW89_WW][100] = -16, + [0][0][RTW89_WW][102] = -16, + [0][0][RTW89_WW][104] = -16, + [0][0][RTW89_WW][105] = -16, + [0][0][RTW89_WW][107] = -12, + [0][0][RTW89_WW][109] = -12, [0][0][RTW89_WW][111] = 0, [0][0][RTW89_WW][113] = 0, [0][0][RTW89_WW][115] = 0, [0][0][RTW89_WW][117] = 0, [0][0][RTW89_WW][119] = 0, - [0][1][RTW89_WW][0] = 76, - [0][1][RTW89_WW][2] = 76, - [0][1][RTW89_WW][4] = 76, - [0][1][RTW89_WW][6] = 76, - [0][1][RTW89_WW][8] = 76, - [0][1][RTW89_WW][10] = 76, - [0][1][RTW89_WW][12] = 76, - [0][1][RTW89_WW][14] = 76, - [0][1][RTW89_WW][15] = 76, - [0][1][RTW89_WW][17] = 76, - [0][1][RTW89_WW][19] = 76, - [0][1][RTW89_WW][21] = 76, - [0][1][RTW89_WW][23] = 76, - [0][1][RTW89_WW][25] = 76, - [0][1][RTW89_WW][27] = 76, - [0][1][RTW89_WW][29] = 76, - [0][1][RTW89_WW][30] = 76, - [0][1][RTW89_WW][32] = 76, - [0][1][RTW89_WW][34] = 76, - [0][1][RTW89_WW][36] = 76, - [0][1][RTW89_WW][38] = 76, - [0][1][RTW89_WW][40] = 76, - [0][1][RTW89_WW][42] = 76, - [0][1][RTW89_WW][44] = 76, - [0][1][RTW89_WW][45] = 76, - [0][1][RTW89_WW][47] = 76, - [0][1][RTW89_WW][49] = 76, - [0][1][RTW89_WW][51] = 76, - [0][1][RTW89_WW][53] = 76, - [0][1][RTW89_WW][55] = 76, - [0][1][RTW89_WW][57] = 76, - [0][1][RTW89_WW][59] = 76, - [0][1][RTW89_WW][60] = 76, - [0][1][RTW89_WW][62] = 76, - [0][1][RTW89_WW][64] = 76, - [0][1][RTW89_WW][66] = 76, - [0][1][RTW89_WW][68] = 76, - [0][1][RTW89_WW][70] = 76, - [0][1][RTW89_WW][72] = 76, - [0][1][RTW89_WW][74] = 76, - [0][1][RTW89_WW][75] = 76, - [0][1][RTW89_WW][77] = 76, - [0][1][RTW89_WW][79] = 76, - [0][1][RTW89_WW][81] = 76, - [0][1][RTW89_WW][83] = 76, - [0][1][RTW89_WW][85] = 76, - [0][1][RTW89_WW][87] = 76, - [0][1][RTW89_WW][89] = 76, - [0][1][RTW89_WW][90] = 76, - [0][1][RTW89_WW][92] = 76, - [0][1][RTW89_WW][94] = 76, - [0][1][RTW89_WW][96] = 76, - [0][1][RTW89_WW][98] = 76, - [0][1][RTW89_WW][100] = 76, - [0][1][RTW89_WW][102] = 76, - [0][1][RTW89_WW][104] = 76, - [0][1][RTW89_WW][105] = 76, - [0][1][RTW89_WW][107] = 76, - [0][1][RTW89_WW][109] = 76, + [0][1][RTW89_WW][0] = -40, + [0][1][RTW89_WW][2] = -40, + [0][1][RTW89_WW][4] = -40, + [0][1][RTW89_WW][6] = -40, + [0][1][RTW89_WW][8] = -40, + [0][1][RTW89_WW][10] = -40, + [0][1][RTW89_WW][12] = -40, + [0][1][RTW89_WW][14] = -40, + [0][1][RTW89_WW][15] = -40, + [0][1][RTW89_WW][17] = -40, + [0][1][RTW89_WW][19] = -40, + [0][1][RTW89_WW][21] = -40, + [0][1][RTW89_WW][23] = -40, + [0][1][RTW89_WW][25] = -40, + [0][1][RTW89_WW][27] = -40, + [0][1][RTW89_WW][29] = -40, + [0][1][RTW89_WW][30] = -40, + [0][1][RTW89_WW][32] = -40, + [0][1][RTW89_WW][34] = -40, + [0][1][RTW89_WW][36] = -40, + [0][1][RTW89_WW][38] = -40, + [0][1][RTW89_WW][40] = -40, + [0][1][RTW89_WW][42] = -40, + [0][1][RTW89_WW][44] = -40, + [0][1][RTW89_WW][45] = -40, + [0][1][RTW89_WW][47] = -40, + [0][1][RTW89_WW][49] = -40, + [0][1][RTW89_WW][51] = -40, + [0][1][RTW89_WW][53] = -40, + [0][1][RTW89_WW][55] = -40, + [0][1][RTW89_WW][57] = -40, + [0][1][RTW89_WW][59] = -40, + [0][1][RTW89_WW][60] = -40, + [0][1][RTW89_WW][62] = -40, + [0][1][RTW89_WW][64] = -40, + [0][1][RTW89_WW][66] = -40, + [0][1][RTW89_WW][68] = -40, + [0][1][RTW89_WW][70] = -38, + [0][1][RTW89_WW][72] = -38, + [0][1][RTW89_WW][74] = -38, + [0][1][RTW89_WW][75] = -38, + [0][1][RTW89_WW][77] = -38, + [0][1][RTW89_WW][79] = -38, + [0][1][RTW89_WW][81] = -38, + [0][1][RTW89_WW][83] = -38, + [0][1][RTW89_WW][85] = -38, + [0][1][RTW89_WW][87] = -40, + [0][1][RTW89_WW][89] = -38, + [0][1][RTW89_WW][90] = -38, + [0][1][RTW89_WW][92] = -38, + [0][1][RTW89_WW][94] = -38, + [0][1][RTW89_WW][96] = -38, + [0][1][RTW89_WW][98] = -38, + [0][1][RTW89_WW][100] = -38, + [0][1][RTW89_WW][102] = -38, + [0][1][RTW89_WW][104] = -38, + [0][1][RTW89_WW][105] = -38, + [0][1][RTW89_WW][107] = -34, + [0][1][RTW89_WW][109] = -34, [0][1][RTW89_WW][111] = 0, [0][1][RTW89_WW][113] = 0, [0][1][RTW89_WW][115] = 0, [0][1][RTW89_WW][117] = 0, [0][1][RTW89_WW][119] = 0, - [1][0][RTW89_WW][0] = 76, - [1][0][RTW89_WW][2] = 76, - [1][0][RTW89_WW][4] = 76, - [1][0][RTW89_WW][6] = 76, - [1][0][RTW89_WW][8] = 76, - [1][0][RTW89_WW][10] = 76, - [1][0][RTW89_WW][12] = 76, - [1][0][RTW89_WW][14] = 76, - [1][0][RTW89_WW][15] = 76, - [1][0][RTW89_WW][17] = 76, - [1][0][RTW89_WW][19] = 76, - [1][0][RTW89_WW][21] = 76, - [1][0][RTW89_WW][23] = 76, - [1][0][RTW89_WW][25] = 76, - [1][0][RTW89_WW][27] = 76, - [1][0][RTW89_WW][29] = 76, - [1][0][RTW89_WW][30] = 76, - [1][0][RTW89_WW][32] = 76, - [1][0][RTW89_WW][34] = 76, - [1][0][RTW89_WW][36] = 76, - [1][0][RTW89_WW][38] = 76, - [1][0][RTW89_WW][40] = 76, - [1][0][RTW89_WW][42] = 76, - [1][0][RTW89_WW][44] = 76, - [1][0][RTW89_WW][45] = 76, - [1][0][RTW89_WW][47] = 76, - [1][0][RTW89_WW][49] = 76, - [1][0][RTW89_WW][51] = 76, - [1][0][RTW89_WW][53] = 76, - [1][0][RTW89_WW][55] = 76, - [1][0][RTW89_WW][57] = 76, - [1][0][RTW89_WW][59] = 76, - [1][0][RTW89_WW][60] = 76, - [1][0][RTW89_WW][62] = 76, - [1][0][RTW89_WW][64] = 76, - [1][0][RTW89_WW][66] = 76, - [1][0][RTW89_WW][68] = 76, - [1][0][RTW89_WW][70] = 76, - [1][0][RTW89_WW][72] = 76, - [1][0][RTW89_WW][74] = 76, - [1][0][RTW89_WW][75] = 76, - [1][0][RTW89_WW][77] = 76, - [1][0][RTW89_WW][79] = 76, - [1][0][RTW89_WW][81] = 76, - [1][0][RTW89_WW][83] = 76, - [1][0][RTW89_WW][85] = 76, - [1][0][RTW89_WW][87] = 76, - [1][0][RTW89_WW][89] = 76, - [1][0][RTW89_WW][90] = 76, - [1][0][RTW89_WW][92] = 76, - [1][0][RTW89_WW][94] = 76, - [1][0][RTW89_WW][96] = 76, - [1][0][RTW89_WW][98] = 76, - [1][0][RTW89_WW][100] = 76, - [1][0][RTW89_WW][102] = 76, - [1][0][RTW89_WW][104] = 76, - [1][0][RTW89_WW][105] = 76, - [1][0][RTW89_WW][107] = 76, - [1][0][RTW89_WW][109] = 76, + [1][0][RTW89_WW][0] = -4, + [1][0][RTW89_WW][2] = -4, + [1][0][RTW89_WW][4] = -4, + [1][0][RTW89_WW][6] = -4, + [1][0][RTW89_WW][8] = -4, + [1][0][RTW89_WW][10] = -4, + [1][0][RTW89_WW][12] = -4, + [1][0][RTW89_WW][14] = -4, + [1][0][RTW89_WW][15] = -4, + [1][0][RTW89_WW][17] = -4, + [1][0][RTW89_WW][19] = -4, + [1][0][RTW89_WW][21] = -4, + [1][0][RTW89_WW][23] = -4, + [1][0][RTW89_WW][25] = -4, + [1][0][RTW89_WW][27] = -4, + [1][0][RTW89_WW][29] = -4, + [1][0][RTW89_WW][30] = -4, + [1][0][RTW89_WW][32] = -4, + [1][0][RTW89_WW][34] = -4, + [1][0][RTW89_WW][36] = -4, + [1][0][RTW89_WW][38] = -4, + [1][0][RTW89_WW][40] = -4, + [1][0][RTW89_WW][42] = -4, + [1][0][RTW89_WW][44] = -4, + [1][0][RTW89_WW][45] = -4, + [1][0][RTW89_WW][47] = -4, + [1][0][RTW89_WW][49] = -4, + [1][0][RTW89_WW][51] = -4, + [1][0][RTW89_WW][53] = -4, + [1][0][RTW89_WW][55] = -4, + [1][0][RTW89_WW][57] = -4, + [1][0][RTW89_WW][59] = -4, + [1][0][RTW89_WW][60] = -4, + [1][0][RTW89_WW][62] = -4, + [1][0][RTW89_WW][64] = -4, + [1][0][RTW89_WW][66] = -4, + [1][0][RTW89_WW][68] = -4, + [1][0][RTW89_WW][70] = -4, + [1][0][RTW89_WW][72] = -4, + [1][0][RTW89_WW][74] = -4, + [1][0][RTW89_WW][75] = -4, + [1][0][RTW89_WW][77] = -4, + [1][0][RTW89_WW][79] = -4, + [1][0][RTW89_WW][81] = -4, + [1][0][RTW89_WW][83] = -4, + [1][0][RTW89_WW][85] = -4, + [1][0][RTW89_WW][87] = -4, + [1][0][RTW89_WW][89] = -4, + [1][0][RTW89_WW][90] = -4, + [1][0][RTW89_WW][92] = -4, + [1][0][RTW89_WW][94] = -4, + [1][0][RTW89_WW][96] = -4, + [1][0][RTW89_WW][98] = -4, + [1][0][RTW89_WW][100] = -4, + [1][0][RTW89_WW][102] = -4, + [1][0][RTW89_WW][104] = -4, + [1][0][RTW89_WW][105] = -4, + [1][0][RTW89_WW][107] = 1, + [1][0][RTW89_WW][109] = 2, [1][0][RTW89_WW][111] = 0, [1][0][RTW89_WW][113] = 0, [1][0][RTW89_WW][115] = 0, [1][0][RTW89_WW][117] = 0, [1][0][RTW89_WW][119] = 0, - [1][1][RTW89_WW][0] = 76, - [1][1][RTW89_WW][2] = 76, - [1][1][RTW89_WW][4] = 76, - [1][1][RTW89_WW][6] = 76, - [1][1][RTW89_WW][8] = 76, - [1][1][RTW89_WW][10] = 76, - [1][1][RTW89_WW][12] = 76, - [1][1][RTW89_WW][14] = 76, - [1][1][RTW89_WW][15] = 76, - [1][1][RTW89_WW][17] = 76, - [1][1][RTW89_WW][19] = 76, - [1][1][RTW89_WW][21] = 76, - [1][1][RTW89_WW][23] = 76, - [1][1][RTW89_WW][25] = 76, - [1][1][RTW89_WW][27] = 76, - [1][1][RTW89_WW][29] = 76, - [1][1][RTW89_WW][30] = 76, - [1][1][RTW89_WW][32] = 76, - [1][1][RTW89_WW][34] = 76, - [1][1][RTW89_WW][36] = 76, - [1][1][RTW89_WW][38] = 76, - [1][1][RTW89_WW][40] = 76, - [1][1][RTW89_WW][42] = 76, - [1][1][RTW89_WW][44] = 76, - [1][1][RTW89_WW][45] = 76, - [1][1][RTW89_WW][47] = 76, - [1][1][RTW89_WW][49] = 76, - [1][1][RTW89_WW][51] = 76, - [1][1][RTW89_WW][53] = 76, - [1][1][RTW89_WW][55] = 76, - [1][1][RTW89_WW][57] = 76, - [1][1][RTW89_WW][59] = 76, - [1][1][RTW89_WW][60] = 76, - [1][1][RTW89_WW][62] = 76, - [1][1][RTW89_WW][64] = 76, - [1][1][RTW89_WW][66] = 76, - [1][1][RTW89_WW][68] = 76, - [1][1][RTW89_WW][70] = 76, - [1][1][RTW89_WW][72] = 76, - [1][1][RTW89_WW][74] = 76, - [1][1][RTW89_WW][75] = 76, - [1][1][RTW89_WW][77] = 76, - [1][1][RTW89_WW][79] = 76, - [1][1][RTW89_WW][81] = 76, - [1][1][RTW89_WW][83] = 76, - [1][1][RTW89_WW][85] = 76, - [1][1][RTW89_WW][87] = 76, - [1][1][RTW89_WW][89] = 76, - [1][1][RTW89_WW][90] = 76, - [1][1][RTW89_WW][92] = 76, - [1][1][RTW89_WW][94] = 76, - [1][1][RTW89_WW][96] = 76, - [1][1][RTW89_WW][98] = 76, - [1][1][RTW89_WW][100] = 76, - [1][1][RTW89_WW][102] = 76, - [1][1][RTW89_WW][104] = 76, - [1][1][RTW89_WW][105] = 76, - [1][1][RTW89_WW][107] = 76, - [1][1][RTW89_WW][109] = 76, + [1][1][RTW89_WW][0] = -26, + [1][1][RTW89_WW][2] = -28, + [1][1][RTW89_WW][4] = -28, + [1][1][RTW89_WW][6] = -28, + [1][1][RTW89_WW][8] = -28, + [1][1][RTW89_WW][10] = -28, + [1][1][RTW89_WW][12] = -28, + [1][1][RTW89_WW][14] = -28, + [1][1][RTW89_WW][15] = -28, + [1][1][RTW89_WW][17] = -28, + [1][1][RTW89_WW][19] = -28, + [1][1][RTW89_WW][21] = -28, + [1][1][RTW89_WW][23] = -28, + [1][1][RTW89_WW][25] = -28, + [1][1][RTW89_WW][27] = -28, + [1][1][RTW89_WW][29] = -28, + [1][1][RTW89_WW][30] = -28, + [1][1][RTW89_WW][32] = -28, + [1][1][RTW89_WW][34] = -28, + [1][1][RTW89_WW][36] = -28, + [1][1][RTW89_WW][38] = -28, + [1][1][RTW89_WW][40] = -28, + [1][1][RTW89_WW][42] = -28, + [1][1][RTW89_WW][44] = -28, + [1][1][RTW89_WW][45] = -26, + [1][1][RTW89_WW][47] = -28, + [1][1][RTW89_WW][49] = -28, + [1][1][RTW89_WW][51] = -28, + [1][1][RTW89_WW][53] = -26, + [1][1][RTW89_WW][55] = -28, + [1][1][RTW89_WW][57] = -28, + [1][1][RTW89_WW][59] = -28, + [1][1][RTW89_WW][60] = -28, + [1][1][RTW89_WW][62] = -28, + [1][1][RTW89_WW][64] = -28, + [1][1][RTW89_WW][66] = -28, + [1][1][RTW89_WW][68] = -28, + [1][1][RTW89_WW][70] = -26, + [1][1][RTW89_WW][72] = -28, + [1][1][RTW89_WW][74] = -28, + [1][1][RTW89_WW][75] = -28, + [1][1][RTW89_WW][77] = -28, + [1][1][RTW89_WW][79] = -28, + [1][1][RTW89_WW][81] = -28, + [1][1][RTW89_WW][83] = -28, + [1][1][RTW89_WW][85] = -28, + [1][1][RTW89_WW][87] = -28, + [1][1][RTW89_WW][89] = -26, + [1][1][RTW89_WW][90] = -26, + [1][1][RTW89_WW][92] = -26, + [1][1][RTW89_WW][94] = -26, + [1][1][RTW89_WW][96] = -26, + [1][1][RTW89_WW][98] = -26, + [1][1][RTW89_WW][100] = -26, + [1][1][RTW89_WW][102] = -26, + [1][1][RTW89_WW][104] = -26, + [1][1][RTW89_WW][105] = -26, + [1][1][RTW89_WW][107] = -22, + [1][1][RTW89_WW][109] = -22, [1][1][RTW89_WW][111] = 0, [1][1][RTW89_WW][113] = 0, [1][1][RTW89_WW][115] = 0, [1][1][RTW89_WW][117] = 0, [1][1][RTW89_WW][119] = 0, - [2][0][RTW89_WW][0] = 76, - [2][0][RTW89_WW][2] = 76, - [2][0][RTW89_WW][4] = 76, - [2][0][RTW89_WW][6] = 76, - [2][0][RTW89_WW][8] = 76, - [2][0][RTW89_WW][10] = 76, - [2][0][RTW89_WW][12] = 76, - [2][0][RTW89_WW][14] = 76, - [2][0][RTW89_WW][15] = 76, - [2][0][RTW89_WW][17] = 76, - [2][0][RTW89_WW][19] = 76, - [2][0][RTW89_WW][21] = 76, - [2][0][RTW89_WW][23] = 76, - [2][0][RTW89_WW][25] = 76, - [2][0][RTW89_WW][27] = 76, - [2][0][RTW89_WW][29] = 76, - [2][0][RTW89_WW][30] = 76, - [2][0][RTW89_WW][32] = 76, - [2][0][RTW89_WW][34] = 76, - [2][0][RTW89_WW][36] = 76, - [2][0][RTW89_WW][38] = 76, - [2][0][RTW89_WW][40] = 76, - [2][0][RTW89_WW][42] = 76, - [2][0][RTW89_WW][44] = 76, - [2][0][RTW89_WW][45] = 76, - [2][0][RTW89_WW][47] = 76, - [2][0][RTW89_WW][49] = 76, - [2][0][RTW89_WW][51] = 76, - [2][0][RTW89_WW][53] = 76, - [2][0][RTW89_WW][55] = 76, - [2][0][RTW89_WW][57] = 76, - [2][0][RTW89_WW][59] = 76, - [2][0][RTW89_WW][60] = 76, - [2][0][RTW89_WW][62] = 76, - [2][0][RTW89_WW][64] = 76, - [2][0][RTW89_WW][66] = 76, - [2][0][RTW89_WW][68] = 76, - [2][0][RTW89_WW][70] = 76, - [2][0][RTW89_WW][72] = 76, - [2][0][RTW89_WW][74] = 76, - [2][0][RTW89_WW][75] = 76, - [2][0][RTW89_WW][77] = 76, - [2][0][RTW89_WW][79] = 76, - [2][0][RTW89_WW][81] = 76, - [2][0][RTW89_WW][83] = 76, - [2][0][RTW89_WW][85] = 76, - [2][0][RTW89_WW][87] = 76, - [2][0][RTW89_WW][89] = 76, - [2][0][RTW89_WW][90] = 76, - [2][0][RTW89_WW][92] = 76, - [2][0][RTW89_WW][94] = 76, - [2][0][RTW89_WW][96] = 76, - [2][0][RTW89_WW][98] = 76, - [2][0][RTW89_WW][100] = 76, - [2][0][RTW89_WW][102] = 76, - [2][0][RTW89_WW][104] = 76, - [2][0][RTW89_WW][105] = 76, - [2][0][RTW89_WW][107] = 76, - [2][0][RTW89_WW][109] = 76, + [2][0][RTW89_WW][0] = 8, + [2][0][RTW89_WW][2] = 8, + [2][0][RTW89_WW][4] = 8, + [2][0][RTW89_WW][6] = 8, + [2][0][RTW89_WW][8] = 8, + [2][0][RTW89_WW][10] = 8, + [2][0][RTW89_WW][12] = 8, + [2][0][RTW89_WW][14] = 8, + [2][0][RTW89_WW][15] = 8, + [2][0][RTW89_WW][17] = 8, + [2][0][RTW89_WW][19] = 8, + [2][0][RTW89_WW][21] = 8, + [2][0][RTW89_WW][23] = 8, + [2][0][RTW89_WW][25] = 8, + [2][0][RTW89_WW][27] = 8, + [2][0][RTW89_WW][29] = 8, + [2][0][RTW89_WW][30] = 8, + [2][0][RTW89_WW][32] = 8, + [2][0][RTW89_WW][34] = 8, + [2][0][RTW89_WW][36] = 8, + [2][0][RTW89_WW][38] = 8, + [2][0][RTW89_WW][40] = 8, + [2][0][RTW89_WW][42] = 8, + [2][0][RTW89_WW][44] = 8, + [2][0][RTW89_WW][45] = 8, + [2][0][RTW89_WW][47] = 8, + [2][0][RTW89_WW][49] = 8, + [2][0][RTW89_WW][51] = 8, + [2][0][RTW89_WW][53] = 8, + [2][0][RTW89_WW][55] = 8, + [2][0][RTW89_WW][57] = 8, + [2][0][RTW89_WW][59] = 8, + [2][0][RTW89_WW][60] = 8, + [2][0][RTW89_WW][62] = 8, + [2][0][RTW89_WW][64] = 8, + [2][0][RTW89_WW][66] = 8, + [2][0][RTW89_WW][68] = 8, + [2][0][RTW89_WW][70] = 8, + [2][0][RTW89_WW][72] = 8, + [2][0][RTW89_WW][74] = 8, + [2][0][RTW89_WW][75] = 8, + [2][0][RTW89_WW][77] = 8, + [2][0][RTW89_WW][79] = 8, + [2][0][RTW89_WW][81] = 8, + [2][0][RTW89_WW][83] = 8, + [2][0][RTW89_WW][85] = 8, + [2][0][RTW89_WW][87] = 8, + [2][0][RTW89_WW][89] = 8, + [2][0][RTW89_WW][90] = 8, + [2][0][RTW89_WW][92] = 8, + [2][0][RTW89_WW][94] = 8, + [2][0][RTW89_WW][96] = 8, + [2][0][RTW89_WW][98] = 8, + [2][0][RTW89_WW][100] = 8, + [2][0][RTW89_WW][102] = 8, + [2][0][RTW89_WW][104] = 8, + [2][0][RTW89_WW][105] = 8, + [2][0][RTW89_WW][107] = 10, + [2][0][RTW89_WW][109] = 12, [2][0][RTW89_WW][111] = 0, [2][0][RTW89_WW][113] = 0, [2][0][RTW89_WW][115] = 0, [2][0][RTW89_WW][117] = 0, [2][0][RTW89_WW][119] = 0, - [2][1][RTW89_WW][0] = 76, - [2][1][RTW89_WW][2] = 76, - [2][1][RTW89_WW][4] = 76, - [2][1][RTW89_WW][6] = 76, - [2][1][RTW89_WW][8] = 76, - [2][1][RTW89_WW][10] = 76, - [2][1][RTW89_WW][12] = 76, - [2][1][RTW89_WW][14] = 76, - [2][1][RTW89_WW][15] = 76, - [2][1][RTW89_WW][17] = 76, - [2][1][RTW89_WW][19] = 76, - [2][1][RTW89_WW][21] = 76, - [2][1][RTW89_WW][23] = 76, - [2][1][RTW89_WW][25] = 76, - [2][1][RTW89_WW][27] = 76, - [2][1][RTW89_WW][29] = 76, - [2][1][RTW89_WW][30] = 76, - [2][1][RTW89_WW][32] = 76, - [2][1][RTW89_WW][34] = 76, - [2][1][RTW89_WW][36] = 76, - [2][1][RTW89_WW][38] = 76, - [2][1][RTW89_WW][40] = 76, - [2][1][RTW89_WW][42] = 76, - [2][1][RTW89_WW][44] = 76, - [2][1][RTW89_WW][45] = 76, - [2][1][RTW89_WW][47] = 76, - [2][1][RTW89_WW][49] = 76, - [2][1][RTW89_WW][51] = 76, - [2][1][RTW89_WW][53] = 76, - [2][1][RTW89_WW][55] = 76, - [2][1][RTW89_WW][57] = 76, - [2][1][RTW89_WW][59] = 76, - [2][1][RTW89_WW][60] = 76, - [2][1][RTW89_WW][62] = 76, - [2][1][RTW89_WW][64] = 76, - [2][1][RTW89_WW][66] = 76, - [2][1][RTW89_WW][68] = 76, - [2][1][RTW89_WW][70] = 76, - [2][1][RTW89_WW][72] = 76, - [2][1][RTW89_WW][74] = 76, - [2][1][RTW89_WW][75] = 76, - [2][1][RTW89_WW][77] = 76, - [2][1][RTW89_WW][79] = 76, - [2][1][RTW89_WW][81] = 76, - [2][1][RTW89_WW][83] = 76, - [2][1][RTW89_WW][85] = 76, - [2][1][RTW89_WW][87] = 76, - [2][1][RTW89_WW][89] = 76, - [2][1][RTW89_WW][90] = 76, - [2][1][RTW89_WW][92] = 76, - [2][1][RTW89_WW][94] = 76, - [2][1][RTW89_WW][96] = 76, - [2][1][RTW89_WW][98] = 76, - [2][1][RTW89_WW][100] = 76, - [2][1][RTW89_WW][102] = 76, - [2][1][RTW89_WW][104] = 76, - [2][1][RTW89_WW][105] = 76, - [2][1][RTW89_WW][107] = 76, - [2][1][RTW89_WW][109] = 76, + [2][1][RTW89_WW][0] = -16, + [2][1][RTW89_WW][2] = -16, + [2][1][RTW89_WW][4] = -16, + [2][1][RTW89_WW][6] = -16, + [2][1][RTW89_WW][8] = -16, + [2][1][RTW89_WW][10] = -16, + [2][1][RTW89_WW][12] = -16, + [2][1][RTW89_WW][14] = -16, + [2][1][RTW89_WW][15] = -16, + [2][1][RTW89_WW][17] = -16, + [2][1][RTW89_WW][19] = -16, + [2][1][RTW89_WW][21] = -16, + [2][1][RTW89_WW][23] = -16, + [2][1][RTW89_WW][25] = -16, + [2][1][RTW89_WW][27] = -16, + [2][1][RTW89_WW][29] = -16, + [2][1][RTW89_WW][30] = -16, + [2][1][RTW89_WW][32] = -16, + [2][1][RTW89_WW][34] = -16, + [2][1][RTW89_WW][36] = -16, + [2][1][RTW89_WW][38] = -16, + [2][1][RTW89_WW][40] = -16, + [2][1][RTW89_WW][42] = -16, + [2][1][RTW89_WW][44] = -16, + [2][1][RTW89_WW][45] = -16, + [2][1][RTW89_WW][47] = -16, + [2][1][RTW89_WW][49] = -16, + [2][1][RTW89_WW][51] = -16, + [2][1][RTW89_WW][53] = -16, + [2][1][RTW89_WW][55] = -16, + [2][1][RTW89_WW][57] = -16, + [2][1][RTW89_WW][59] = -16, + [2][1][RTW89_WW][60] = -16, + [2][1][RTW89_WW][62] = -16, + [2][1][RTW89_WW][64] = -16, + [2][1][RTW89_WW][66] = -16, + [2][1][RTW89_WW][68] = -16, + [2][1][RTW89_WW][70] = -16, + [2][1][RTW89_WW][72] = -16, + [2][1][RTW89_WW][74] = -16, + [2][1][RTW89_WW][75] = -16, + [2][1][RTW89_WW][77] = -16, + [2][1][RTW89_WW][79] = -16, + [2][1][RTW89_WW][81] = -16, + [2][1][RTW89_WW][83] = -16, + [2][1][RTW89_WW][85] = -18, + [2][1][RTW89_WW][87] = -16, + [2][1][RTW89_WW][89] = -16, + [2][1][RTW89_WW][90] = -16, + [2][1][RTW89_WW][92] = -16, + [2][1][RTW89_WW][94] = -16, + [2][1][RTW89_WW][96] = -16, + [2][1][RTW89_WW][98] = -16, + [2][1][RTW89_WW][100] = -16, + [2][1][RTW89_WW][102] = -16, + [2][1][RTW89_WW][104] = -16, + [2][1][RTW89_WW][105] = -16, + [2][1][RTW89_WW][107] = -12, + [2][1][RTW89_WW][109] = -10, [2][1][RTW89_WW][111] = 0, [2][1][RTW89_WW][113] = 0, [2][1][RTW89_WW][115] = 0, [2][1][RTW89_WW][117] = 0, [2][1][RTW89_WW][119] = 0, - [0][0][RTW89_FCC][0] = 76, - [0][0][RTW89_FCC][2] = 76, - [0][0][RTW89_FCC][4] = 76, - [0][0][RTW89_FCC][6] = 76, - [0][0][RTW89_FCC][8] = 76, - [0][0][RTW89_FCC][10] = 76, - [0][0][RTW89_FCC][12] = 76, - [0][0][RTW89_FCC][14] = 76, - [0][0][RTW89_FCC][15] = 76, - [0][0][RTW89_FCC][17] = 76, - [0][0][RTW89_FCC][19] = 76, - [0][0][RTW89_FCC][21] = 76, - [0][0][RTW89_FCC][23] = 76, - [0][0][RTW89_FCC][25] = 76, - [0][0][RTW89_FCC][27] = 76, - [0][0][RTW89_FCC][29] = 76, - [0][0][RTW89_FCC][30] = 76, - [0][0][RTW89_FCC][32] = 76, - [0][0][RTW89_FCC][34] = 76, - [0][0][RTW89_FCC][36] = 76, - [0][0][RTW89_FCC][38] = 76, - [0][0][RTW89_FCC][40] = 76, - [0][0][RTW89_FCC][42] = 76, - [0][0][RTW89_FCC][44] = 76, - [0][0][RTW89_FCC][45] = 76, - [0][0][RTW89_FCC][47] = 76, - [0][0][RTW89_FCC][49] = 76, - [0][0][RTW89_FCC][51] = 76, - [0][0][RTW89_FCC][53] = 76, - [0][0][RTW89_FCC][55] = 76, - [0][0][RTW89_FCC][57] = 76, - [0][0][RTW89_FCC][59] = 76, - [0][0][RTW89_FCC][60] = 76, - [0][0][RTW89_FCC][62] = 76, - [0][0][RTW89_FCC][64] = 76, - [0][0][RTW89_FCC][66] = 76, - [0][0][RTW89_FCC][68] = 76, - [0][0][RTW89_FCC][70] = 76, - [0][0][RTW89_FCC][72] = 76, - [0][0][RTW89_FCC][74] = 76, - [0][0][RTW89_FCC][75] = 76, - [0][0][RTW89_FCC][77] = 76, - [0][0][RTW89_FCC][79] = 76, - [0][0][RTW89_FCC][81] = 76, - [0][0][RTW89_FCC][83] = 76, - [0][0][RTW89_FCC][85] = 76, - [0][0][RTW89_FCC][87] = 76, - [0][0][RTW89_FCC][89] = 76, - [0][0][RTW89_FCC][90] = 76, - [0][0][RTW89_FCC][92] = 76, - [0][0][RTW89_FCC][94] = 76, - [0][0][RTW89_FCC][96] = 76, - [0][0][RTW89_FCC][98] = 76, - [0][0][RTW89_FCC][100] = 76, - [0][0][RTW89_FCC][102] = 76, - [0][0][RTW89_FCC][104] = 76, - [0][0][RTW89_FCC][105] = 76, - [0][0][RTW89_FCC][107] = 76, - [0][0][RTW89_FCC][109] = 76, + [0][0][RTW89_FCC][0] = -16, + [0][0][RTW89_ETSI][0] = 32, + [0][0][RTW89_FCC][2] = -18, + [0][0][RTW89_ETSI][2] = 32, + [0][0][RTW89_FCC][4] = -18, + [0][0][RTW89_ETSI][4] = 32, + [0][0][RTW89_FCC][6] = -18, + [0][0][RTW89_ETSI][6] = 32, + [0][0][RTW89_FCC][8] = -18, + [0][0][RTW89_ETSI][8] = 32, + [0][0][RTW89_FCC][10] = -18, + [0][0][RTW89_ETSI][10] = 32, + [0][0][RTW89_FCC][12] = -18, + [0][0][RTW89_ETSI][12] = 32, + [0][0][RTW89_FCC][14] = -18, + [0][0][RTW89_ETSI][14] = 32, + [0][0][RTW89_FCC][15] = -18, + [0][0][RTW89_ETSI][15] = 32, + [0][0][RTW89_FCC][17] = -18, + [0][0][RTW89_ETSI][17] = 32, + [0][0][RTW89_FCC][19] = -18, + [0][0][RTW89_ETSI][19] = 32, + [0][0][RTW89_FCC][21] = -18, + [0][0][RTW89_ETSI][21] = 32, + [0][0][RTW89_FCC][23] = -18, + [0][0][RTW89_ETSI][23] = 32, + [0][0][RTW89_FCC][25] = -18, + [0][0][RTW89_ETSI][25] = 32, + [0][0][RTW89_FCC][27] = -18, + [0][0][RTW89_ETSI][27] = 32, + [0][0][RTW89_FCC][29] = -18, + [0][0][RTW89_ETSI][29] = 32, + [0][0][RTW89_FCC][30] = -18, + [0][0][RTW89_ETSI][30] = 32, + [0][0][RTW89_FCC][32] = -18, + [0][0][RTW89_ETSI][32] = 32, + [0][0][RTW89_FCC][34] = -18, + [0][0][RTW89_ETSI][34] = 32, + [0][0][RTW89_FCC][36] = -18, + [0][0][RTW89_ETSI][36] = 32, + [0][0][RTW89_FCC][38] = -18, + [0][0][RTW89_ETSI][38] = 32, + [0][0][RTW89_FCC][40] = -18, + [0][0][RTW89_ETSI][40] = 32, + [0][0][RTW89_FCC][42] = -18, + [0][0][RTW89_ETSI][42] = 32, + [0][0][RTW89_FCC][44] = -16, + [0][0][RTW89_ETSI][44] = 32, + [0][0][RTW89_FCC][45] = -16, + [0][0][RTW89_ETSI][45] = 127, + [0][0][RTW89_FCC][47] = -18, + [0][0][RTW89_ETSI][47] = 127, + [0][0][RTW89_FCC][49] = -18, + [0][0][RTW89_ETSI][49] = 127, + [0][0][RTW89_FCC][51] = -18, + [0][0][RTW89_ETSI][51] = 127, + [0][0][RTW89_FCC][53] = -16, + [0][0][RTW89_ETSI][53] = 127, + [0][0][RTW89_FCC][55] = -18, + [0][0][RTW89_ETSI][55] = 127, + [0][0][RTW89_FCC][57] = -18, + [0][0][RTW89_ETSI][57] = 127, + [0][0][RTW89_FCC][59] = -18, + [0][0][RTW89_ETSI][59] = 127, + [0][0][RTW89_FCC][60] = -18, + [0][0][RTW89_ETSI][60] = 127, + [0][0][RTW89_FCC][62] = -18, + [0][0][RTW89_ETSI][62] = 127, + [0][0][RTW89_FCC][64] = -18, + [0][0][RTW89_ETSI][64] = 127, + [0][0][RTW89_FCC][66] = -18, + [0][0][RTW89_ETSI][66] = 127, + [0][0][RTW89_FCC][68] = -18, + [0][0][RTW89_ETSI][68] = 127, + [0][0][RTW89_FCC][70] = -16, + [0][0][RTW89_ETSI][70] = 127, + [0][0][RTW89_FCC][72] = -18, + [0][0][RTW89_ETSI][72] = 127, + [0][0][RTW89_FCC][74] = -18, + [0][0][RTW89_ETSI][74] = 127, + [0][0][RTW89_FCC][75] = -18, + [0][0][RTW89_ETSI][75] = 127, + [0][0][RTW89_FCC][77] = -18, + [0][0][RTW89_ETSI][77] = 127, + [0][0][RTW89_FCC][79] = -18, + [0][0][RTW89_ETSI][79] = 127, + [0][0][RTW89_FCC][81] = -18, + [0][0][RTW89_ETSI][81] = 127, + [0][0][RTW89_FCC][83] = -18, + [0][0][RTW89_ETSI][83] = 127, + [0][0][RTW89_FCC][85] = -18, + [0][0][RTW89_ETSI][85] = 127, + [0][0][RTW89_FCC][87] = -16, + [0][0][RTW89_ETSI][87] = 127, + [0][0][RTW89_FCC][89] = -16, + [0][0][RTW89_ETSI][89] = 127, + [0][0][RTW89_FCC][90] = -16, + [0][0][RTW89_ETSI][90] = 127, + [0][0][RTW89_FCC][92] = -16, + [0][0][RTW89_ETSI][92] = 127, + [0][0][RTW89_FCC][94] = -16, + [0][0][RTW89_ETSI][94] = 127, + [0][0][RTW89_FCC][96] = -16, + [0][0][RTW89_ETSI][96] = 127, + [0][0][RTW89_FCC][98] = -16, + [0][0][RTW89_ETSI][98] = 127, + [0][0][RTW89_FCC][100] = -16, + [0][0][RTW89_ETSI][100] = 127, + [0][0][RTW89_FCC][102] = -16, + [0][0][RTW89_ETSI][102] = 127, + [0][0][RTW89_FCC][104] = -16, + [0][0][RTW89_ETSI][104] = 127, + [0][0][RTW89_FCC][105] = -16, + [0][0][RTW89_ETSI][105] = 127, + [0][0][RTW89_FCC][107] = -12, + [0][0][RTW89_ETSI][107] = 127, + [0][0][RTW89_FCC][109] = -12, + [0][0][RTW89_ETSI][109] = 127, [0][0][RTW89_FCC][111] = 127, + [0][0][RTW89_ETSI][111] = 127, [0][0][RTW89_FCC][113] = 127, + [0][0][RTW89_ETSI][113] = 127, [0][0][RTW89_FCC][115] = 127, + [0][0][RTW89_ETSI][115] = 127, [0][0][RTW89_FCC][117] = 127, + [0][0][RTW89_ETSI][117] = 127, [0][0][RTW89_FCC][119] = 127, - [0][1][RTW89_FCC][0] = 76, - [0][1][RTW89_FCC][2] = 76, - [0][1][RTW89_FCC][4] = 76, - [0][1][RTW89_FCC][6] = 76, - [0][1][RTW89_FCC][8] = 76, - [0][1][RTW89_FCC][10] = 76, - [0][1][RTW89_FCC][12] = 76, - [0][1][RTW89_FCC][14] = 76, - [0][1][RTW89_FCC][15] = 76, - [0][1][RTW89_FCC][17] = 76, - [0][1][RTW89_FCC][19] = 76, - [0][1][RTW89_FCC][21] = 76, - [0][1][RTW89_FCC][23] = 76, - [0][1][RTW89_FCC][25] = 76, - [0][1][RTW89_FCC][27] = 76, - [0][1][RTW89_FCC][29] = 76, - [0][1][RTW89_FCC][30] = 76, - [0][1][RTW89_FCC][32] = 76, - [0][1][RTW89_FCC][34] = 76, - [0][1][RTW89_FCC][36] = 76, - [0][1][RTW89_FCC][38] = 76, - [0][1][RTW89_FCC][40] = 76, - [0][1][RTW89_FCC][42] = 76, - [0][1][RTW89_FCC][44] = 76, - [0][1][RTW89_FCC][45] = 76, - [0][1][RTW89_FCC][47] = 76, - [0][1][RTW89_FCC][49] = 76, - [0][1][RTW89_FCC][51] = 76, - [0][1][RTW89_FCC][53] = 76, - [0][1][RTW89_FCC][55] = 76, - [0][1][RTW89_FCC][57] = 76, - [0][1][RTW89_FCC][59] = 76, - [0][1][RTW89_FCC][60] = 76, - [0][1][RTW89_FCC][62] = 76, - [0][1][RTW89_FCC][64] = 76, - [0][1][RTW89_FCC][66] = 76, - [0][1][RTW89_FCC][68] = 76, - [0][1][RTW89_FCC][70] = 76, - [0][1][RTW89_FCC][72] = 76, - [0][1][RTW89_FCC][74] = 76, - [0][1][RTW89_FCC][75] = 76, - [0][1][RTW89_FCC][77] = 76, - [0][1][RTW89_FCC][79] = 76, - [0][1][RTW89_FCC][81] = 76, - [0][1][RTW89_FCC][83] = 76, - [0][1][RTW89_FCC][85] = 76, - [0][1][RTW89_FCC][87] = 76, - [0][1][RTW89_FCC][89] = 76, - [0][1][RTW89_FCC][90] = 76, - [0][1][RTW89_FCC][92] = 76, - [0][1][RTW89_FCC][94] = 76, - [0][1][RTW89_FCC][96] = 76, - [0][1][RTW89_FCC][98] = 76, - [0][1][RTW89_FCC][100] = 76, - [0][1][RTW89_FCC][102] = 76, - [0][1][RTW89_FCC][104] = 76, - [0][1][RTW89_FCC][105] = 76, - [0][1][RTW89_FCC][107] = 76, - [0][1][RTW89_FCC][109] = 76, + [0][0][RTW89_ETSI][119] = 127, + [0][1][RTW89_FCC][0] = -40, + [0][1][RTW89_ETSI][0] = 20, + [0][1][RTW89_FCC][2] = -40, + [0][1][RTW89_ETSI][2] = 20, + [0][1][RTW89_FCC][4] = -40, + [0][1][RTW89_ETSI][4] = 20, + [0][1][RTW89_FCC][6] = -40, + [0][1][RTW89_ETSI][6] = 20, + [0][1][RTW89_FCC][8] = -40, + [0][1][RTW89_ETSI][8] = 20, + [0][1][RTW89_FCC][10] = -40, + [0][1][RTW89_ETSI][10] = 20, + [0][1][RTW89_FCC][12] = -40, + [0][1][RTW89_ETSI][12] = 20, + [0][1][RTW89_FCC][14] = -40, + [0][1][RTW89_ETSI][14] = 20, + [0][1][RTW89_FCC][15] = -40, + [0][1][RTW89_ETSI][15] = 20, + [0][1][RTW89_FCC][17] = -40, + [0][1][RTW89_ETSI][17] = 20, + [0][1][RTW89_FCC][19] = -40, + [0][1][RTW89_ETSI][19] = 20, + [0][1][RTW89_FCC][21] = -40, + [0][1][RTW89_ETSI][21] = 20, + [0][1][RTW89_FCC][23] = -40, + [0][1][RTW89_ETSI][23] = 20, + [0][1][RTW89_FCC][25] = -40, + [0][1][RTW89_ETSI][25] = 20, + [0][1][RTW89_FCC][27] = -40, + [0][1][RTW89_ETSI][27] = 20, + [0][1][RTW89_FCC][29] = -40, + [0][1][RTW89_ETSI][29] = 20, + [0][1][RTW89_FCC][30] = -40, + [0][1][RTW89_ETSI][30] = 20, + [0][1][RTW89_FCC][32] = -40, + [0][1][RTW89_ETSI][32] = 20, + [0][1][RTW89_FCC][34] = -40, + [0][1][RTW89_ETSI][34] = 20, + [0][1][RTW89_FCC][36] = -40, + [0][1][RTW89_ETSI][36] = 20, + [0][1][RTW89_FCC][38] = -40, + [0][1][RTW89_ETSI][38] = 20, + [0][1][RTW89_FCC][40] = -40, + [0][1][RTW89_ETSI][40] = 20, + [0][1][RTW89_FCC][42] = -40, + [0][1][RTW89_ETSI][42] = 20, + [0][1][RTW89_FCC][44] = -40, + [0][1][RTW89_ETSI][44] = 20, + [0][1][RTW89_FCC][45] = -40, + [0][1][RTW89_ETSI][45] = 127, + [0][1][RTW89_FCC][47] = -40, + [0][1][RTW89_ETSI][47] = 127, + [0][1][RTW89_FCC][49] = -40, + [0][1][RTW89_ETSI][49] = 127, + [0][1][RTW89_FCC][51] = -40, + [0][1][RTW89_ETSI][51] = 127, + [0][1][RTW89_FCC][53] = -40, + [0][1][RTW89_ETSI][53] = 127, + [0][1][RTW89_FCC][55] = -40, + [0][1][RTW89_ETSI][55] = 127, + [0][1][RTW89_FCC][57] = -40, + [0][1][RTW89_ETSI][57] = 127, + [0][1][RTW89_FCC][59] = -40, + [0][1][RTW89_ETSI][59] = 127, + [0][1][RTW89_FCC][60] = -40, + [0][1][RTW89_ETSI][60] = 127, + [0][1][RTW89_FCC][62] = -40, + [0][1][RTW89_ETSI][62] = 127, + [0][1][RTW89_FCC][64] = -40, + [0][1][RTW89_ETSI][64] = 127, + [0][1][RTW89_FCC][66] = -40, + [0][1][RTW89_ETSI][66] = 127, + [0][1][RTW89_FCC][68] = -40, + [0][1][RTW89_ETSI][68] = 127, + [0][1][RTW89_FCC][70] = -38, + [0][1][RTW89_ETSI][70] = 127, + [0][1][RTW89_FCC][72] = -38, + [0][1][RTW89_ETSI][72] = 127, + [0][1][RTW89_FCC][74] = -38, + [0][1][RTW89_ETSI][74] = 127, + [0][1][RTW89_FCC][75] = -38, + [0][1][RTW89_ETSI][75] = 127, + [0][1][RTW89_FCC][77] = -38, + [0][1][RTW89_ETSI][77] = 127, + [0][1][RTW89_FCC][79] = -38, + [0][1][RTW89_ETSI][79] = 127, + [0][1][RTW89_FCC][81] = -38, + [0][1][RTW89_ETSI][81] = 127, + [0][1][RTW89_FCC][83] = -38, + [0][1][RTW89_ETSI][83] = 127, + [0][1][RTW89_FCC][85] = -38, + [0][1][RTW89_ETSI][85] = 127, + [0][1][RTW89_FCC][87] = -40, + [0][1][RTW89_ETSI][87] = 127, + [0][1][RTW89_FCC][89] = -38, + [0][1][RTW89_ETSI][89] = 127, + [0][1][RTW89_FCC][90] = -38, + [0][1][RTW89_ETSI][90] = 127, + [0][1][RTW89_FCC][92] = -38, + [0][1][RTW89_ETSI][92] = 127, + [0][1][RTW89_FCC][94] = -38, + [0][1][RTW89_ETSI][94] = 127, + [0][1][RTW89_FCC][96] = -38, + [0][1][RTW89_ETSI][96] = 127, + [0][1][RTW89_FCC][98] = -38, + [0][1][RTW89_ETSI][98] = 127, + [0][1][RTW89_FCC][100] = -38, + [0][1][RTW89_ETSI][100] = 127, + [0][1][RTW89_FCC][102] = -38, + [0][1][RTW89_ETSI][102] = 127, + [0][1][RTW89_FCC][104] = -38, + [0][1][RTW89_ETSI][104] = 127, + [0][1][RTW89_FCC][105] = -38, + [0][1][RTW89_ETSI][105] = 127, + [0][1][RTW89_FCC][107] = -34, + [0][1][RTW89_ETSI][107] = 127, + [0][1][RTW89_FCC][109] = -34, + [0][1][RTW89_ETSI][109] = 127, [0][1][RTW89_FCC][111] = 127, + [0][1][RTW89_ETSI][111] = 127, [0][1][RTW89_FCC][113] = 127, + [0][1][RTW89_ETSI][113] = 127, [0][1][RTW89_FCC][115] = 127, + [0][1][RTW89_ETSI][115] = 127, [0][1][RTW89_FCC][117] = 127, + [0][1][RTW89_ETSI][117] = 127, [0][1][RTW89_FCC][119] = 127, - [1][0][RTW89_FCC][0] = 76, - [1][0][RTW89_FCC][2] = 76, - [1][0][RTW89_FCC][4] = 76, - [1][0][RTW89_FCC][6] = 76, - [1][0][RTW89_FCC][8] = 76, - [1][0][RTW89_FCC][10] = 76, - [1][0][RTW89_FCC][12] = 76, - [1][0][RTW89_FCC][14] = 76, - [1][0][RTW89_FCC][15] = 76, - [1][0][RTW89_FCC][17] = 76, - [1][0][RTW89_FCC][19] = 76, - [1][0][RTW89_FCC][21] = 76, - [1][0][RTW89_FCC][23] = 76, - [1][0][RTW89_FCC][25] = 76, - [1][0][RTW89_FCC][27] = 76, - [1][0][RTW89_FCC][29] = 76, - [1][0][RTW89_FCC][30] = 76, - [1][0][RTW89_FCC][32] = 76, - [1][0][RTW89_FCC][34] = 76, - [1][0][RTW89_FCC][36] = 76, - [1][0][RTW89_FCC][38] = 76, - [1][0][RTW89_FCC][40] = 76, - [1][0][RTW89_FCC][42] = 76, - [1][0][RTW89_FCC][44] = 76, - [1][0][RTW89_FCC][45] = 76, - [1][0][RTW89_FCC][47] = 76, - [1][0][RTW89_FCC][49] = 76, - [1][0][RTW89_FCC][51] = 76, - [1][0][RTW89_FCC][53] = 76, - [1][0][RTW89_FCC][55] = 76, - [1][0][RTW89_FCC][57] = 76, - [1][0][RTW89_FCC][59] = 76, - [1][0][RTW89_FCC][60] = 76, - [1][0][RTW89_FCC][62] = 76, - [1][0][RTW89_FCC][64] = 76, - [1][0][RTW89_FCC][66] = 76, - [1][0][RTW89_FCC][68] = 76, - [1][0][RTW89_FCC][70] = 76, - [1][0][RTW89_FCC][72] = 76, - [1][0][RTW89_FCC][74] = 76, - [1][0][RTW89_FCC][75] = 76, - [1][0][RTW89_FCC][77] = 76, - [1][0][RTW89_FCC][79] = 76, - [1][0][RTW89_FCC][81] = 76, - [1][0][RTW89_FCC][83] = 76, - [1][0][RTW89_FCC][85] = 76, - [1][0][RTW89_FCC][87] = 76, - [1][0][RTW89_FCC][89] = 76, - [1][0][RTW89_FCC][90] = 76, - [1][0][RTW89_FCC][92] = 76, - [1][0][RTW89_FCC][94] = 76, - [1][0][RTW89_FCC][96] = 76, - [1][0][RTW89_FCC][98] = 76, - [1][0][RTW89_FCC][100] = 76, - [1][0][RTW89_FCC][102] = 76, - [1][0][RTW89_FCC][104] = 76, - [1][0][RTW89_FCC][105] = 76, - [1][0][RTW89_FCC][107] = 76, - [1][0][RTW89_FCC][109] = 76, + [0][1][RTW89_ETSI][119] = 127, + [1][0][RTW89_FCC][0] = -4, + [1][0][RTW89_ETSI][0] = 46, + [1][0][RTW89_FCC][2] = -4, + [1][0][RTW89_ETSI][2] = 46, + [1][0][RTW89_FCC][4] = -4, + [1][0][RTW89_ETSI][4] = 46, + [1][0][RTW89_FCC][6] = -4, + [1][0][RTW89_ETSI][6] = 46, + [1][0][RTW89_FCC][8] = -4, + [1][0][RTW89_ETSI][8] = 46, + [1][0][RTW89_FCC][10] = -4, + [1][0][RTW89_ETSI][10] = 46, + [1][0][RTW89_FCC][12] = -4, + [1][0][RTW89_ETSI][12] = 46, + [1][0][RTW89_FCC][14] = -4, + [1][0][RTW89_ETSI][14] = 46, + [1][0][RTW89_FCC][15] = -4, + [1][0][RTW89_ETSI][15] = 46, + [1][0][RTW89_FCC][17] = -4, + [1][0][RTW89_ETSI][17] = 46, + [1][0][RTW89_FCC][19] = -4, + [1][0][RTW89_ETSI][19] = 46, + [1][0][RTW89_FCC][21] = -4, + [1][0][RTW89_ETSI][21] = 46, + [1][0][RTW89_FCC][23] = -4, + [1][0][RTW89_ETSI][23] = 46, + [1][0][RTW89_FCC][25] = -4, + [1][0][RTW89_ETSI][25] = 46, + [1][0][RTW89_FCC][27] = -4, + [1][0][RTW89_ETSI][27] = 46, + [1][0][RTW89_FCC][29] = -4, + [1][0][RTW89_ETSI][29] = 46, + [1][0][RTW89_FCC][30] = -4, + [1][0][RTW89_ETSI][30] = 46, + [1][0][RTW89_FCC][32] = -4, + [1][0][RTW89_ETSI][32] = 46, + [1][0][RTW89_FCC][34] = -4, + [1][0][RTW89_ETSI][34] = 46, + [1][0][RTW89_FCC][36] = -4, + [1][0][RTW89_ETSI][36] = 46, + [1][0][RTW89_FCC][38] = -4, + [1][0][RTW89_ETSI][38] = 46, + [1][0][RTW89_FCC][40] = -4, + [1][0][RTW89_ETSI][40] = 46, + [1][0][RTW89_FCC][42] = -4, + [1][0][RTW89_ETSI][42] = 46, + [1][0][RTW89_FCC][44] = -4, + [1][0][RTW89_ETSI][44] = 46, + [1][0][RTW89_FCC][45] = -4, + [1][0][RTW89_ETSI][45] = 127, + [1][0][RTW89_FCC][47] = -4, + [1][0][RTW89_ETSI][47] = 127, + [1][0][RTW89_FCC][49] = -4, + [1][0][RTW89_ETSI][49] = 127, + [1][0][RTW89_FCC][51] = -4, + [1][0][RTW89_ETSI][51] = 127, + [1][0][RTW89_FCC][53] = -4, + [1][0][RTW89_ETSI][53] = 127, + [1][0][RTW89_FCC][55] = -4, + [1][0][RTW89_ETSI][55] = 127, + [1][0][RTW89_FCC][57] = -4, + [1][0][RTW89_ETSI][57] = 127, + [1][0][RTW89_FCC][59] = -4, + [1][0][RTW89_ETSI][59] = 127, + [1][0][RTW89_FCC][60] = -4, + [1][0][RTW89_ETSI][60] = 127, + [1][0][RTW89_FCC][62] = -4, + [1][0][RTW89_ETSI][62] = 127, + [1][0][RTW89_FCC][64] = -4, + [1][0][RTW89_ETSI][64] = 127, + [1][0][RTW89_FCC][66] = -4, + [1][0][RTW89_ETSI][66] = 127, + [1][0][RTW89_FCC][68] = -4, + [1][0][RTW89_ETSI][68] = 127, + [1][0][RTW89_FCC][70] = -4, + [1][0][RTW89_ETSI][70] = 127, + [1][0][RTW89_FCC][72] = -4, + [1][0][RTW89_ETSI][72] = 127, + [1][0][RTW89_FCC][74] = -4, + [1][0][RTW89_ETSI][74] = 127, + [1][0][RTW89_FCC][75] = -4, + [1][0][RTW89_ETSI][75] = 127, + [1][0][RTW89_FCC][77] = -4, + [1][0][RTW89_ETSI][77] = 127, + [1][0][RTW89_FCC][79] = -4, + [1][0][RTW89_ETSI][79] = 127, + [1][0][RTW89_FCC][81] = -4, + [1][0][RTW89_ETSI][81] = 127, + [1][0][RTW89_FCC][83] = -4, + [1][0][RTW89_ETSI][83] = 127, + [1][0][RTW89_FCC][85] = -4, + [1][0][RTW89_ETSI][85] = 127, + [1][0][RTW89_FCC][87] = -4, + [1][0][RTW89_ETSI][87] = 127, + [1][0][RTW89_FCC][89] = -4, + [1][0][RTW89_ETSI][89] = 127, + [1][0][RTW89_FCC][90] = -4, + [1][0][RTW89_ETSI][90] = 127, + [1][0][RTW89_FCC][92] = -4, + [1][0][RTW89_ETSI][92] = 127, + [1][0][RTW89_FCC][94] = -4, + [1][0][RTW89_ETSI][94] = 127, + [1][0][RTW89_FCC][96] = -4, + [1][0][RTW89_ETSI][96] = 127, + [1][0][RTW89_FCC][98] = -4, + [1][0][RTW89_ETSI][98] = 127, + [1][0][RTW89_FCC][100] = -4, + [1][0][RTW89_ETSI][100] = 127, + [1][0][RTW89_FCC][102] = -4, + [1][0][RTW89_ETSI][102] = 127, + [1][0][RTW89_FCC][104] = -4, + [1][0][RTW89_ETSI][104] = 127, + [1][0][RTW89_FCC][105] = -4, + [1][0][RTW89_ETSI][105] = 127, + [1][0][RTW89_FCC][107] = 0, + [1][0][RTW89_ETSI][107] = 127, + [1][0][RTW89_FCC][109] = 2, + [1][0][RTW89_ETSI][109] = 127, [1][0][RTW89_FCC][111] = 127, + [1][0][RTW89_ETSI][111] = 127, [1][0][RTW89_FCC][113] = 127, + [1][0][RTW89_ETSI][113] = 127, [1][0][RTW89_FCC][115] = 127, + [1][0][RTW89_ETSI][115] = 127, [1][0][RTW89_FCC][117] = 127, + [1][0][RTW89_ETSI][117] = 127, [1][0][RTW89_FCC][119] = 127, - [1][1][RTW89_FCC][0] = 76, - [1][1][RTW89_FCC][2] = 76, - [1][1][RTW89_FCC][4] = 76, - [1][1][RTW89_FCC][6] = 76, - [1][1][RTW89_FCC][8] = 76, - [1][1][RTW89_FCC][10] = 76, - [1][1][RTW89_FCC][12] = 76, - [1][1][RTW89_FCC][14] = 76, - [1][1][RTW89_FCC][15] = 76, - [1][1][RTW89_FCC][17] = 76, - [1][1][RTW89_FCC][19] = 76, - [1][1][RTW89_FCC][21] = 76, - [1][1][RTW89_FCC][23] = 76, - [1][1][RTW89_FCC][25] = 76, - [1][1][RTW89_FCC][27] = 76, - [1][1][RTW89_FCC][29] = 76, - [1][1][RTW89_FCC][30] = 76, - [1][1][RTW89_FCC][32] = 76, - [1][1][RTW89_FCC][34] = 76, - [1][1][RTW89_FCC][36] = 76, - [1][1][RTW89_FCC][38] = 76, - [1][1][RTW89_FCC][40] = 76, - [1][1][RTW89_FCC][42] = 76, - [1][1][RTW89_FCC][44] = 76, - [1][1][RTW89_FCC][45] = 76, - [1][1][RTW89_FCC][47] = 76, - [1][1][RTW89_FCC][49] = 76, - [1][1][RTW89_FCC][51] = 76, - [1][1][RTW89_FCC][53] = 76, - [1][1][RTW89_FCC][55] = 76, - [1][1][RTW89_FCC][57] = 76, - [1][1][RTW89_FCC][59] = 76, - [1][1][RTW89_FCC][60] = 76, - [1][1][RTW89_FCC][62] = 76, - [1][1][RTW89_FCC][64] = 76, - [1][1][RTW89_FCC][66] = 76, - [1][1][RTW89_FCC][68] = 76, - [1][1][RTW89_FCC][70] = 76, - [1][1][RTW89_FCC][72] = 76, - [1][1][RTW89_FCC][74] = 76, - [1][1][RTW89_FCC][75] = 76, - [1][1][RTW89_FCC][77] = 76, - [1][1][RTW89_FCC][79] = 76, - [1][1][RTW89_FCC][81] = 76, - [1][1][RTW89_FCC][83] = 76, - [1][1][RTW89_FCC][85] = 76, - [1][1][RTW89_FCC][87] = 76, - [1][1][RTW89_FCC][89] = 76, - [1][1][RTW89_FCC][90] = 76, - [1][1][RTW89_FCC][92] = 76, - [1][1][RTW89_FCC][94] = 76, - [1][1][RTW89_FCC][96] = 76, - [1][1][RTW89_FCC][98] = 76, - [1][1][RTW89_FCC][100] = 76, - [1][1][RTW89_FCC][102] = 76, - [1][1][RTW89_FCC][104] = 76, - [1][1][RTW89_FCC][105] = 76, - [1][1][RTW89_FCC][107] = 76, - [1][1][RTW89_FCC][109] = 76, + [1][0][RTW89_ETSI][119] = 127, + [1][1][RTW89_FCC][0] = -26, + [1][1][RTW89_ETSI][0] = 32, + [1][1][RTW89_FCC][2] = -28, + [1][1][RTW89_ETSI][2] = 32, + [1][1][RTW89_FCC][4] = -28, + [1][1][RTW89_ETSI][4] = 32, + [1][1][RTW89_FCC][6] = -28, + [1][1][RTW89_ETSI][6] = 32, + [1][1][RTW89_FCC][8] = -28, + [1][1][RTW89_ETSI][8] = 32, + [1][1][RTW89_FCC][10] = -28, + [1][1][RTW89_ETSI][10] = 32, + [1][1][RTW89_FCC][12] = -28, + [1][1][RTW89_ETSI][12] = 32, + [1][1][RTW89_FCC][14] = -28, + [1][1][RTW89_ETSI][14] = 32, + [1][1][RTW89_FCC][15] = -28, + [1][1][RTW89_ETSI][15] = 32, + [1][1][RTW89_FCC][17] = -28, + [1][1][RTW89_ETSI][17] = 32, + [1][1][RTW89_FCC][19] = -28, + [1][1][RTW89_ETSI][19] = 32, + [1][1][RTW89_FCC][21] = -28, + [1][1][RTW89_ETSI][21] = 32, + [1][1][RTW89_FCC][23] = -28, + [1][1][RTW89_ETSI][23] = 32, + [1][1][RTW89_FCC][25] = -28, + [1][1][RTW89_ETSI][25] = 32, + [1][1][RTW89_FCC][27] = -28, + [1][1][RTW89_ETSI][27] = 32, + [1][1][RTW89_FCC][29] = -28, + [1][1][RTW89_ETSI][29] = 32, + [1][1][RTW89_FCC][30] = -28, + [1][1][RTW89_ETSI][30] = 32, + [1][1][RTW89_FCC][32] = -28, + [1][1][RTW89_ETSI][32] = 32, + [1][1][RTW89_FCC][34] = -28, + [1][1][RTW89_ETSI][34] = 32, + [1][1][RTW89_FCC][36] = -28, + [1][1][RTW89_ETSI][36] = 32, + [1][1][RTW89_FCC][38] = -28, + [1][1][RTW89_ETSI][38] = 32, + [1][1][RTW89_FCC][40] = -28, + [1][1][RTW89_ETSI][40] = 32, + [1][1][RTW89_FCC][42] = -28, + [1][1][RTW89_ETSI][42] = 32, + [1][1][RTW89_FCC][44] = -28, + [1][1][RTW89_ETSI][44] = 34, + [1][1][RTW89_FCC][45] = -26, + [1][1][RTW89_ETSI][45] = 127, + [1][1][RTW89_FCC][47] = -28, + [1][1][RTW89_ETSI][47] = 127, + [1][1][RTW89_FCC][49] = -28, + [1][1][RTW89_ETSI][49] = 127, + [1][1][RTW89_FCC][51] = -28, + [1][1][RTW89_ETSI][51] = 127, + [1][1][RTW89_FCC][53] = -26, + [1][1][RTW89_ETSI][53] = 127, + [1][1][RTW89_FCC][55] = -28, + [1][1][RTW89_ETSI][55] = 127, + [1][1][RTW89_FCC][57] = -28, + [1][1][RTW89_ETSI][57] = 127, + [1][1][RTW89_FCC][59] = -28, + [1][1][RTW89_ETSI][59] = 127, + [1][1][RTW89_FCC][60] = -28, + [1][1][RTW89_ETSI][60] = 127, + [1][1][RTW89_FCC][62] = -28, + [1][1][RTW89_ETSI][62] = 127, + [1][1][RTW89_FCC][64] = -28, + [1][1][RTW89_ETSI][64] = 127, + [1][1][RTW89_FCC][66] = -28, + [1][1][RTW89_ETSI][66] = 127, + [1][1][RTW89_FCC][68] = -28, + [1][1][RTW89_ETSI][68] = 127, + [1][1][RTW89_FCC][70] = -26, + [1][1][RTW89_ETSI][70] = 127, + [1][1][RTW89_FCC][72] = -28, + [1][1][RTW89_ETSI][72] = 127, + [1][1][RTW89_FCC][74] = -28, + [1][1][RTW89_ETSI][74] = 127, + [1][1][RTW89_FCC][75] = -28, + [1][1][RTW89_ETSI][75] = 127, + [1][1][RTW89_FCC][77] = -28, + [1][1][RTW89_ETSI][77] = 127, + [1][1][RTW89_FCC][79] = -28, + [1][1][RTW89_ETSI][79] = 127, + [1][1][RTW89_FCC][81] = -28, + [1][1][RTW89_ETSI][81] = 127, + [1][1][RTW89_FCC][83] = -28, + [1][1][RTW89_ETSI][83] = 127, + [1][1][RTW89_FCC][85] = -28, + [1][1][RTW89_ETSI][85] = 127, + [1][1][RTW89_FCC][87] = -28, + [1][1][RTW89_ETSI][87] = 127, + [1][1][RTW89_FCC][89] = -26, + [1][1][RTW89_ETSI][89] = 127, + [1][1][RTW89_FCC][90] = -26, + [1][1][RTW89_ETSI][90] = 127, + [1][1][RTW89_FCC][92] = -26, + [1][1][RTW89_ETSI][92] = 127, + [1][1][RTW89_FCC][94] = -26, + [1][1][RTW89_ETSI][94] = 127, + [1][1][RTW89_FCC][96] = -26, + [1][1][RTW89_ETSI][96] = 127, + [1][1][RTW89_FCC][98] = -26, + [1][1][RTW89_ETSI][98] = 127, + [1][1][RTW89_FCC][100] = -26, + [1][1][RTW89_ETSI][100] = 127, + [1][1][RTW89_FCC][102] = -26, + [1][1][RTW89_ETSI][102] = 127, + [1][1][RTW89_FCC][104] = -26, + [1][1][RTW89_ETSI][104] = 127, + [1][1][RTW89_FCC][105] = -26, + [1][1][RTW89_ETSI][105] = 127, + [1][1][RTW89_FCC][107] = -22, + [1][1][RTW89_ETSI][107] = 127, + [1][1][RTW89_FCC][109] = -22, + [1][1][RTW89_ETSI][109] = 127, [1][1][RTW89_FCC][111] = 127, + [1][1][RTW89_ETSI][111] = 127, [1][1][RTW89_FCC][113] = 127, + [1][1][RTW89_ETSI][113] = 127, [1][1][RTW89_FCC][115] = 127, + [1][1][RTW89_ETSI][115] = 127, [1][1][RTW89_FCC][117] = 127, + [1][1][RTW89_ETSI][117] = 127, [1][1][RTW89_FCC][119] = 127, - [2][0][RTW89_FCC][0] = 76, - [2][0][RTW89_FCC][2] = 76, - [2][0][RTW89_FCC][4] = 76, - [2][0][RTW89_FCC][6] = 76, - [2][0][RTW89_FCC][8] = 76, - [2][0][RTW89_FCC][10] = 76, - [2][0][RTW89_FCC][12] = 76, - [2][0][RTW89_FCC][14] = 76, - [2][0][RTW89_FCC][15] = 76, - [2][0][RTW89_FCC][17] = 76, - [2][0][RTW89_FCC][19] = 76, - [2][0][RTW89_FCC][21] = 76, - [2][0][RTW89_FCC][23] = 76, - [2][0][RTW89_FCC][25] = 76, - [2][0][RTW89_FCC][27] = 76, - [2][0][RTW89_FCC][29] = 76, - [2][0][RTW89_FCC][30] = 76, - [2][0][RTW89_FCC][32] = 76, - [2][0][RTW89_FCC][34] = 76, - [2][0][RTW89_FCC][36] = 76, - [2][0][RTW89_FCC][38] = 76, - [2][0][RTW89_FCC][40] = 76, - [2][0][RTW89_FCC][42] = 76, - [2][0][RTW89_FCC][44] = 76, - [2][0][RTW89_FCC][45] = 76, - [2][0][RTW89_FCC][47] = 76, - [2][0][RTW89_FCC][49] = 76, - [2][0][RTW89_FCC][51] = 76, - [2][0][RTW89_FCC][53] = 76, - [2][0][RTW89_FCC][55] = 76, - [2][0][RTW89_FCC][57] = 76, - [2][0][RTW89_FCC][59] = 76, - [2][0][RTW89_FCC][60] = 76, - [2][0][RTW89_FCC][62] = 76, - [2][0][RTW89_FCC][64] = 76, - [2][0][RTW89_FCC][66] = 76, - [2][0][RTW89_FCC][68] = 76, - [2][0][RTW89_FCC][70] = 76, - [2][0][RTW89_FCC][72] = 76, - [2][0][RTW89_FCC][74] = 76, - [2][0][RTW89_FCC][75] = 76, - [2][0][RTW89_FCC][77] = 76, - [2][0][RTW89_FCC][79] = 76, - [2][0][RTW89_FCC][81] = 76, - [2][0][RTW89_FCC][83] = 76, - [2][0][RTW89_FCC][85] = 76, - [2][0][RTW89_FCC][87] = 76, - [2][0][RTW89_FCC][89] = 76, - [2][0][RTW89_FCC][90] = 76, - [2][0][RTW89_FCC][92] = 76, - [2][0][RTW89_FCC][94] = 76, - [2][0][RTW89_FCC][96] = 76, - [2][0][RTW89_FCC][98] = 76, - [2][0][RTW89_FCC][100] = 76, - [2][0][RTW89_FCC][102] = 76, - [2][0][RTW89_FCC][104] = 76, - [2][0][RTW89_FCC][105] = 76, - [2][0][RTW89_FCC][107] = 76, - [2][0][RTW89_FCC][109] = 76, + [1][1][RTW89_ETSI][119] = 127, + [2][0][RTW89_FCC][0] = 8, + [2][0][RTW89_ETSI][0] = 56, + [2][0][RTW89_FCC][2] = 8, + [2][0][RTW89_ETSI][2] = 56, + [2][0][RTW89_FCC][4] = 8, + [2][0][RTW89_ETSI][4] = 56, + [2][0][RTW89_FCC][6] = 8, + [2][0][RTW89_ETSI][6] = 56, + [2][0][RTW89_FCC][8] = 8, + [2][0][RTW89_ETSI][8] = 56, + [2][0][RTW89_FCC][10] = 8, + [2][0][RTW89_ETSI][10] = 56, + [2][0][RTW89_FCC][12] = 8, + [2][0][RTW89_ETSI][12] = 56, + [2][0][RTW89_FCC][14] = 8, + [2][0][RTW89_ETSI][14] = 56, + [2][0][RTW89_FCC][15] = 8, + [2][0][RTW89_ETSI][15] = 56, + [2][0][RTW89_FCC][17] = 8, + [2][0][RTW89_ETSI][17] = 56, + [2][0][RTW89_FCC][19] = 8, + [2][0][RTW89_ETSI][19] = 56, + [2][0][RTW89_FCC][21] = 8, + [2][0][RTW89_ETSI][21] = 56, + [2][0][RTW89_FCC][23] = 8, + [2][0][RTW89_ETSI][23] = 56, + [2][0][RTW89_FCC][25] = 8, + [2][0][RTW89_ETSI][25] = 56, + [2][0][RTW89_FCC][27] = 8, + [2][0][RTW89_ETSI][27] = 56, + [2][0][RTW89_FCC][29] = 8, + [2][0][RTW89_ETSI][29] = 56, + [2][0][RTW89_FCC][30] = 8, + [2][0][RTW89_ETSI][30] = 56, + [2][0][RTW89_FCC][32] = 8, + [2][0][RTW89_ETSI][32] = 56, + [2][0][RTW89_FCC][34] = 8, + [2][0][RTW89_ETSI][34] = 56, + [2][0][RTW89_FCC][36] = 8, + [2][0][RTW89_ETSI][36] = 56, + [2][0][RTW89_FCC][38] = 8, + [2][0][RTW89_ETSI][38] = 56, + [2][0][RTW89_FCC][40] = 8, + [2][0][RTW89_ETSI][40] = 56, + [2][0][RTW89_FCC][42] = 8, + [2][0][RTW89_ETSI][42] = 56, + [2][0][RTW89_FCC][44] = 8, + [2][0][RTW89_ETSI][44] = 56, + [2][0][RTW89_FCC][45] = 8, + [2][0][RTW89_ETSI][45] = 127, + [2][0][RTW89_FCC][47] = 8, + [2][0][RTW89_ETSI][47] = 127, + [2][0][RTW89_FCC][49] = 8, + [2][0][RTW89_ETSI][49] = 127, + [2][0][RTW89_FCC][51] = 8, + [2][0][RTW89_ETSI][51] = 127, + [2][0][RTW89_FCC][53] = 8, + [2][0][RTW89_ETSI][53] = 127, + [2][0][RTW89_FCC][55] = 8, + [2][0][RTW89_ETSI][55] = 127, + [2][0][RTW89_FCC][57] = 8, + [2][0][RTW89_ETSI][57] = 127, + [2][0][RTW89_FCC][59] = 8, + [2][0][RTW89_ETSI][59] = 127, + [2][0][RTW89_FCC][60] = 8, + [2][0][RTW89_ETSI][60] = 127, + [2][0][RTW89_FCC][62] = 8, + [2][0][RTW89_ETSI][62] = 127, + [2][0][RTW89_FCC][64] = 8, + [2][0][RTW89_ETSI][64] = 127, + [2][0][RTW89_FCC][66] = 8, + [2][0][RTW89_ETSI][66] = 127, + [2][0][RTW89_FCC][68] = 8, + [2][0][RTW89_ETSI][68] = 127, + [2][0][RTW89_FCC][70] = 8, + [2][0][RTW89_ETSI][70] = 127, + [2][0][RTW89_FCC][72] = 8, + [2][0][RTW89_ETSI][72] = 127, + [2][0][RTW89_FCC][74] = 8, + [2][0][RTW89_ETSI][74] = 127, + [2][0][RTW89_FCC][75] = 8, + [2][0][RTW89_ETSI][75] = 127, + [2][0][RTW89_FCC][77] = 8, + [2][0][RTW89_ETSI][77] = 127, + [2][0][RTW89_FCC][79] = 8, + [2][0][RTW89_ETSI][79] = 127, + [2][0][RTW89_FCC][81] = 8, + [2][0][RTW89_ETSI][81] = 127, + [2][0][RTW89_FCC][83] = 8, + [2][0][RTW89_ETSI][83] = 127, + [2][0][RTW89_FCC][85] = 8, + [2][0][RTW89_ETSI][85] = 127, + [2][0][RTW89_FCC][87] = 8, + [2][0][RTW89_ETSI][87] = 127, + [2][0][RTW89_FCC][89] = 8, + [2][0][RTW89_ETSI][89] = 127, + [2][0][RTW89_FCC][90] = 8, + [2][0][RTW89_ETSI][90] = 127, + [2][0][RTW89_FCC][92] = 8, + [2][0][RTW89_ETSI][92] = 127, + [2][0][RTW89_FCC][94] = 8, + [2][0][RTW89_ETSI][94] = 127, + [2][0][RTW89_FCC][96] = 8, + [2][0][RTW89_ETSI][96] = 127, + [2][0][RTW89_FCC][98] = 8, + [2][0][RTW89_ETSI][98] = 127, + [2][0][RTW89_FCC][100] = 8, + [2][0][RTW89_ETSI][100] = 127, + [2][0][RTW89_FCC][102] = 8, + [2][0][RTW89_ETSI][102] = 127, + [2][0][RTW89_FCC][104] = 8, + [2][0][RTW89_ETSI][104] = 127, + [2][0][RTW89_FCC][105] = 8, + [2][0][RTW89_ETSI][105] = 127, + [2][0][RTW89_FCC][107] = 10, + [2][0][RTW89_ETSI][107] = 127, + [2][0][RTW89_FCC][109] = 12, + [2][0][RTW89_ETSI][109] = 127, [2][0][RTW89_FCC][111] = 127, + [2][0][RTW89_ETSI][111] = 127, [2][0][RTW89_FCC][113] = 127, + [2][0][RTW89_ETSI][113] = 127, [2][0][RTW89_FCC][115] = 127, + [2][0][RTW89_ETSI][115] = 127, [2][0][RTW89_FCC][117] = 127, + [2][0][RTW89_ETSI][117] = 127, [2][0][RTW89_FCC][119] = 127, - [2][1][RTW89_FCC][0] = 76, - [2][1][RTW89_FCC][2] = 76, - [2][1][RTW89_FCC][4] = 76, - [2][1][RTW89_FCC][6] = 76, - [2][1][RTW89_FCC][8] = 76, - [2][1][RTW89_FCC][10] = 76, - [2][1][RTW89_FCC][12] = 76, - [2][1][RTW89_FCC][14] = 76, - [2][1][RTW89_FCC][15] = 76, - [2][1][RTW89_FCC][17] = 76, - [2][1][RTW89_FCC][19] = 76, - [2][1][RTW89_FCC][21] = 76, - [2][1][RTW89_FCC][23] = 76, - [2][1][RTW89_FCC][25] = 76, - [2][1][RTW89_FCC][27] = 76, - [2][1][RTW89_FCC][29] = 76, - [2][1][RTW89_FCC][30] = 76, - [2][1][RTW89_FCC][32] = 76, - [2][1][RTW89_FCC][34] = 76, - [2][1][RTW89_FCC][36] = 76, - [2][1][RTW89_FCC][38] = 76, - [2][1][RTW89_FCC][40] = 76, - [2][1][RTW89_FCC][42] = 76, - [2][1][RTW89_FCC][44] = 76, - [2][1][RTW89_FCC][45] = 76, - [2][1][RTW89_FCC][47] = 76, - [2][1][RTW89_FCC][49] = 76, - [2][1][RTW89_FCC][51] = 76, - [2][1][RTW89_FCC][53] = 76, - [2][1][RTW89_FCC][55] = 76, - [2][1][RTW89_FCC][57] = 76, - [2][1][RTW89_FCC][59] = 76, - [2][1][RTW89_FCC][60] = 76, - [2][1][RTW89_FCC][62] = 76, - [2][1][RTW89_FCC][64] = 76, - [2][1][RTW89_FCC][66] = 76, - [2][1][RTW89_FCC][68] = 76, - [2][1][RTW89_FCC][70] = 76, - [2][1][RTW89_FCC][72] = 76, - [2][1][RTW89_FCC][74] = 76, - [2][1][RTW89_FCC][75] = 76, - [2][1][RTW89_FCC][77] = 76, - [2][1][RTW89_FCC][79] = 76, - [2][1][RTW89_FCC][81] = 76, - [2][1][RTW89_FCC][83] = 76, - [2][1][RTW89_FCC][85] = 76, - [2][1][RTW89_FCC][87] = 76, - [2][1][RTW89_FCC][89] = 76, - [2][1][RTW89_FCC][90] = 76, - [2][1][RTW89_FCC][92] = 76, - [2][1][RTW89_FCC][94] = 76, - [2][1][RTW89_FCC][96] = 76, - [2][1][RTW89_FCC][98] = 76, - [2][1][RTW89_FCC][100] = 76, - [2][1][RTW89_FCC][102] = 76, - [2][1][RTW89_FCC][104] = 76, - [2][1][RTW89_FCC][105] = 76, - [2][1][RTW89_FCC][107] = 76, - [2][1][RTW89_FCC][109] = 76, + [2][0][RTW89_ETSI][119] = 127, + [2][1][RTW89_FCC][0] = -16, + [2][1][RTW89_ETSI][0] = 44, + [2][1][RTW89_FCC][2] = -16, + [2][1][RTW89_ETSI][2] = 44, + [2][1][RTW89_FCC][4] = -16, + [2][1][RTW89_ETSI][4] = 44, + [2][1][RTW89_FCC][6] = -16, + [2][1][RTW89_ETSI][6] = 44, + [2][1][RTW89_FCC][8] = -16, + [2][1][RTW89_ETSI][8] = 44, + [2][1][RTW89_FCC][10] = -16, + [2][1][RTW89_ETSI][10] = 44, + [2][1][RTW89_FCC][12] = -16, + [2][1][RTW89_ETSI][12] = 44, + [2][1][RTW89_FCC][14] = -16, + [2][1][RTW89_ETSI][14] = 44, + [2][1][RTW89_FCC][15] = -16, + [2][1][RTW89_ETSI][15] = 44, + [2][1][RTW89_FCC][17] = -16, + [2][1][RTW89_ETSI][17] = 44, + [2][1][RTW89_FCC][19] = -16, + [2][1][RTW89_ETSI][19] = 44, + [2][1][RTW89_FCC][21] = -16, + [2][1][RTW89_ETSI][21] = 44, + [2][1][RTW89_FCC][23] = -16, + [2][1][RTW89_ETSI][23] = 44, + [2][1][RTW89_FCC][25] = -16, + [2][1][RTW89_ETSI][25] = 44, + [2][1][RTW89_FCC][27] = -16, + [2][1][RTW89_ETSI][27] = 44, + [2][1][RTW89_FCC][29] = -16, + [2][1][RTW89_ETSI][29] = 44, + [2][1][RTW89_FCC][30] = -16, + [2][1][RTW89_ETSI][30] = 44, + [2][1][RTW89_FCC][32] = -16, + [2][1][RTW89_ETSI][32] = 44, + [2][1][RTW89_FCC][34] = -16, + [2][1][RTW89_ETSI][34] = 44, + [2][1][RTW89_FCC][36] = -16, + [2][1][RTW89_ETSI][36] = 44, + [2][1][RTW89_FCC][38] = -16, + [2][1][RTW89_ETSI][38] = 44, + [2][1][RTW89_FCC][40] = -16, + [2][1][RTW89_ETSI][40] = 44, + [2][1][RTW89_FCC][42] = -16, + [2][1][RTW89_ETSI][42] = 44, + [2][1][RTW89_FCC][44] = -16, + [2][1][RTW89_ETSI][44] = 44, + [2][1][RTW89_FCC][45] = -16, + [2][1][RTW89_ETSI][45] = 127, + [2][1][RTW89_FCC][47] = -16, + [2][1][RTW89_ETSI][47] = 127, + [2][1][RTW89_FCC][49] = -16, + [2][1][RTW89_ETSI][49] = 127, + [2][1][RTW89_FCC][51] = -16, + [2][1][RTW89_ETSI][51] = 127, + [2][1][RTW89_FCC][53] = -16, + [2][1][RTW89_ETSI][53] = 127, + [2][1][RTW89_FCC][55] = -16, + [2][1][RTW89_ETSI][55] = 127, + [2][1][RTW89_FCC][57] = -16, + [2][1][RTW89_ETSI][57] = 127, + [2][1][RTW89_FCC][59] = -16, + [2][1][RTW89_ETSI][59] = 127, + [2][1][RTW89_FCC][60] = -16, + [2][1][RTW89_ETSI][60] = 127, + [2][1][RTW89_FCC][62] = -16, + [2][1][RTW89_ETSI][62] = 127, + [2][1][RTW89_FCC][64] = -16, + [2][1][RTW89_ETSI][64] = 127, + [2][1][RTW89_FCC][66] = -16, + [2][1][RTW89_ETSI][66] = 127, + [2][1][RTW89_FCC][68] = -16, + [2][1][RTW89_ETSI][68] = 127, + [2][1][RTW89_FCC][70] = -16, + [2][1][RTW89_ETSI][70] = 127, + [2][1][RTW89_FCC][72] = -16, + [2][1][RTW89_ETSI][72] = 127, + [2][1][RTW89_FCC][74] = -16, + [2][1][RTW89_ETSI][74] = 127, + [2][1][RTW89_FCC][75] = -16, + [2][1][RTW89_ETSI][75] = 127, + [2][1][RTW89_FCC][77] = -16, + [2][1][RTW89_ETSI][77] = 127, + [2][1][RTW89_FCC][79] = -16, + [2][1][RTW89_ETSI][79] = 127, + [2][1][RTW89_FCC][81] = -16, + [2][1][RTW89_ETSI][81] = 127, + [2][1][RTW89_FCC][83] = -16, + [2][1][RTW89_ETSI][83] = 127, + [2][1][RTW89_FCC][85] = -18, + [2][1][RTW89_ETSI][85] = 127, + [2][1][RTW89_FCC][87] = -16, + [2][1][RTW89_ETSI][87] = 127, + [2][1][RTW89_FCC][89] = -16, + [2][1][RTW89_ETSI][89] = 127, + [2][1][RTW89_FCC][90] = -16, + [2][1][RTW89_ETSI][90] = 127, + [2][1][RTW89_FCC][92] = -16, + [2][1][RTW89_ETSI][92] = 127, + [2][1][RTW89_FCC][94] = -16, + [2][1][RTW89_ETSI][94] = 127, + [2][1][RTW89_FCC][96] = -16, + [2][1][RTW89_ETSI][96] = 127, + [2][1][RTW89_FCC][98] = -16, + [2][1][RTW89_ETSI][98] = 127, + [2][1][RTW89_FCC][100] = -16, + [2][1][RTW89_ETSI][100] = 127, + [2][1][RTW89_FCC][102] = -16, + [2][1][RTW89_ETSI][102] = 127, + [2][1][RTW89_FCC][104] = -16, + [2][1][RTW89_ETSI][104] = 127, + [2][1][RTW89_FCC][105] = -16, + [2][1][RTW89_ETSI][105] = 127, + [2][1][RTW89_FCC][107] = -12, + [2][1][RTW89_ETSI][107] = 127, + [2][1][RTW89_FCC][109] = -10, + [2][1][RTW89_ETSI][109] = 127, [2][1][RTW89_FCC][111] = 127, + [2][1][RTW89_ETSI][111] = 127, [2][1][RTW89_FCC][113] = 127, + [2][1][RTW89_ETSI][113] = 127, [2][1][RTW89_FCC][115] = 127, + [2][1][RTW89_ETSI][115] = 127, [2][1][RTW89_FCC][117] = 127, + [2][1][RTW89_ETSI][117] = 127, [2][1][RTW89_FCC][119] = 127, + [2][1][RTW89_ETSI][119] = 127, }; const struct rtw89_phy_table rtw89_8852c_phy_bb_table = { diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c index fc039449401306e28cf0180c0dacb5982575d0c1..35901f64d17de2174a10b5ae4759101aefc8e0a5 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c @@ -42,14 +42,15 @@ static const struct rtw89_pci_info rtw8852c_pci_info = { .max_tag_num_mask = B_AX_MAX_TAG_NUM_V1_MASK, .rxbd_rwptr_clr_reg = R_AX_RXBD_RWPTR_CLR_V1, .txbd_rwptr_clr2_reg = R_AX_TXBD_RWPTR_CLR2_V1, - .dma_stop1_reg = R_AX_HAXI_DMA_STOP1, - .dma_stop2_reg = R_AX_HAXI_DMA_STOP2, - .dma_busy1_reg = R_AX_HAXI_DMA_BUSY1, + .dma_stop1 = {R_AX_HAXI_DMA_STOP1, B_AX_TX_STOP1_MASK}, + .dma_stop2 = {R_AX_HAXI_DMA_STOP2, B_AX_TX_STOP2_ALL}, + .dma_busy1 = {R_AX_HAXI_DMA_BUSY1, DMA_BUSY1_CHECK}, .dma_busy2_reg = R_AX_HAXI_DMA_BUSY2, .dma_busy3_reg = R_AX_HAXI_DMA_BUSY3, .rpwm_addr = R_AX_PCIE_HRPWM_V1, .cpwm_addr = R_AX_PCIE_CRPWM, + .tx_dma_ch_mask = 0, .bd_idx_addr_low_power = &rtw8852c_bd_idx_addr_low_power, .dma_addr_set = &rtw89_pci_ch_dma_addr_set_v1, diff --git a/drivers/net/wireless/realtek/rtw89/sar.c b/drivers/net/wireless/realtek/rtw89/sar.c index eb2d3ec28775f37f19b9a7e48ad670acb0d50578..dfccae81c3804e7c31ac0ed7ef15df808215ce85 100644 --- a/drivers/net/wireless/realtek/rtw89/sar.c +++ b/drivers/net/wireless/realtek/rtw89/sar.c @@ -81,9 +81,9 @@ static const struct rtw89_sar_span rtw89_sar_overlapping_6ghz[] = { static int rtw89_query_sar_config_common(struct rtw89_dev *rtwdev, s32 *cfg) { struct rtw89_sar_cfg_common *rtwsar = &rtwdev->sar.cfg_common; - struct rtw89_hal *hal = &rtwdev->hal; - enum rtw89_band band = hal->current_band_type; - u32 center_freq = hal->current_freq; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0); + enum rtw89_band band = chan->band_type; + u32 center_freq = chan->freq; const struct rtw89_sar_span *span = NULL; enum rtw89_sar_subband subband_l, subband_h; int idx; @@ -228,7 +228,7 @@ static int rtw89_apply_sar_common(struct rtw89_dev *rtwdev, } rtw89_sar_set_src(rtwdev, RTW89_SAR_SOURCE_COMMON, cfg_common, sar); - rtw89_chip_set_txpwr(rtwdev); + rtw89_core_set_chip_txpwr(rtwdev); exit: mutex_unlock(&rtwdev->mutex); diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c index 726223f25dc69cebd9a27d3ed419681df833b944..c1a4bc1c64d16fba26f642c0a43118a66182ab12 100644 --- a/drivers/net/wireless/realtek/rtw89/ser.c +++ b/drivers/net/wireless/realtek/rtw89/ser.c @@ -5,6 +5,7 @@ #include #include "cam.h" +#include "chan.h" #include "debug.h" #include "fw.h" #include "mac.h" @@ -152,7 +153,10 @@ static void ser_state_run(struct rtw89_ser *ser, u8 evt) rtw89_debug(rtwdev, RTW89_DBG_SER, "ser: %s receive %s\n", ser_st_name(ser), ser_ev_name(ser, evt)); + mutex_lock(&rtwdev->mutex); rtw89_leave_lps(rtwdev); + mutex_unlock(&rtwdev->mutex); + ser->st_tbl[ser->state].st_func(ser, evt); } @@ -298,7 +302,7 @@ static void ser_reset_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) rtwvif->trigger = false; } -static void ser_sta_deinit_addr_cam_iter(void *data, struct ieee80211_sta *sta) +static void ser_sta_deinit_cam_iter(void *data, struct ieee80211_sta *sta) { struct rtw89_vif *rtwvif = (struct rtw89_vif *)data; struct rtw89_dev *rtwdev = rtwvif->rtwdev; @@ -308,15 +312,19 @@ static void ser_sta_deinit_addr_cam_iter(void *data, struct ieee80211_sta *sta) rtw89_cam_deinit_addr_cam(rtwdev, &rtwsta->addr_cam); if (sta->tdls) rtw89_cam_deinit_bssid_cam(rtwdev, &rtwsta->bssid_cam); + + INIT_LIST_HEAD(&rtwsta->ba_cam_list); } static void ser_deinit_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { ieee80211_iterate_stations_atomic(rtwdev->hw, - ser_sta_deinit_addr_cam_iter, + ser_sta_deinit_cam_iter, rtwvif); rtw89_cam_deinit(rtwdev, rtwvif); + + bitmap_zero(rtwdev->cam_info.ba_cam_map, RTW89_MAX_BA_CAM_NUM); } static void ser_reset_mac_binding(struct rtw89_dev *rtwdev) @@ -388,6 +396,7 @@ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt) switch (evt) { case SER_EV_STATE_IN: rtw89_hci_recovery_complete(rtwdev); + clear_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags); break; case SER_EV_L1_RESET: ser_state_goto(ser, SER_RESET_TRX_ST); @@ -531,7 +540,7 @@ static int rtw89_ser_fw_backtrace_dump(struct rtw89_dev *rtwdev, u8 *buf, const struct __fw_backtrace_entry *ent) { struct __fw_backtrace_info *ptr = (struct __fw_backtrace_info *)buf; - u32 fwbt_addr = ent->wcpu_addr - RTW89_WCPU_BASE_ADDR; + u32 fwbt_addr = ent->wcpu_addr & RTW89_WCPU_BASE_MASK; u32 fwbt_size = ent->size; u32 fwbt_key = ent->key; u32 i; @@ -601,6 +610,7 @@ bottom: ser_reset_mac_binding(rtwdev); rtw89_core_stop(rtwdev); + rtw89_entity_init(rtwdev); INIT_LIST_HEAD(&rtwdev->rtwvifs_list); } @@ -623,7 +633,6 @@ static void ser_l2_reset_st_hdl(struct rtw89_ser *ser, u8 evt) fallthrough; case SER_EV_L2_RECFG_DONE: ser_state_goto(ser, SER_IDLE_ST); - clear_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags); break; case SER_EV_STATE_OUT: diff --git a/drivers/net/wireless/rndis_wlan.c b/drivers/net/wireless/rndis_wlan.c index 05524291d60c0bd5274b6748ecae253d2de0329c..82a7458e01aec063643ec83da31cb3a50ae96cd4 100644 --- a/drivers/net/wireless/rndis_wlan.c +++ b/drivers/net/wireless/rndis_wlan.c @@ -251,7 +251,7 @@ struct ndis_80211_bssid_ex { struct ndis_80211_bssid_list_ex { __le32 num_items; - struct ndis_80211_bssid_ex bssid[]; + u8 bssid_data[]; } __packed; struct ndis_80211_fixed_ies { @@ -489,14 +489,16 @@ static int rndis_join_ibss(struct wiphy *wiphy, struct net_device *dev, static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev); static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params); + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params); static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr); + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr); static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool unicast, bool multicast); + int link_id, u8 key_index, bool unicast, + bool multicast); static int rndis_get_station(struct wiphy *wiphy, struct net_device *dev, const u8 *mac, struct station_info *sinfo); @@ -2082,7 +2084,8 @@ resize_buf: netdev_dbg(usbdev->net, "%s(): buflen: %d\n", __func__, len); bssid_len = 0; - bssid = next_bssid_list_item(bssid_list->bssid, &bssid_len, buf, len); + bssid = next_bssid_list_item((void *)bssid_list->bssid_data, + &bssid_len, buf, len); /* Device returns incorrect 'num_items'. Workaround by ignoring the * received 'num_items' and walking through full bssid buffer instead. @@ -2377,8 +2380,8 @@ static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev) } static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct rndis_wlan_private *priv = wiphy_priv(wiphy); struct usbnet *usbdev = priv->usbdev; @@ -2413,7 +2416,8 @@ static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev, } static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { struct rndis_wlan_private *priv = wiphy_priv(wiphy); struct usbnet *usbdev = priv->usbdev; @@ -2424,7 +2428,8 @@ static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev, } static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool unicast, bool multicast) + int link_id, u8 key_index, bool unicast, + bool multicast) { struct rndis_wlan_private *priv = wiphy_priv(wiphy); struct usbnet *usbdev = priv->usbdev; diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c index bf39c4bda26f82aff8fd1addfca6690643b8b46a..2fbec51c8f944bf9c5f87b728e3bec17acd2890c 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c +++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c @@ -889,6 +889,7 @@ static void rsi_mac80211_conf_filter(struct ieee80211_hw *hw, * for a hardware TX queue. * @hw: Pointer to the ieee80211_hw structure * @vif: Pointer to the ieee80211_vif structure. + * @link_id: the link ID if MLO is used, otherwise 0 * @queue: Queue number. * @params: Pointer to ieee80211_tx_queue_params structure. * diff --git a/drivers/net/wireless/silabs/wfx/main.c b/drivers/net/wireless/silabs/wfx/main.c index e015bfb8d221f33ef91b9c2decc5ad55cb5c778c..84d82ddded567ebbd70953ea0d0857c88d1f64f6 100644 --- a/drivers/net/wireless/silabs/wfx/main.c +++ b/drivers/net/wireless/silabs/wfx/main.c @@ -181,7 +181,7 @@ int wfx_send_pds(struct wfx_dev *wdev, u8 *buf, size_t len) while (len > 0) { chunk_type = get_unaligned_le16(buf + 0); chunk_len = get_unaligned_le16(buf + 2); - if (chunk_len > len) { + if (chunk_len < 4 || chunk_len > len) { dev_err(wdev->dev, "PDS:%d: corrupted file\n", chunk_num); return -EINVAL; } diff --git a/drivers/net/wireless/st/cw1200/queue.c b/drivers/net/wireless/st/cw1200/queue.c index e06da4b3b0d46051de9557a9dc29bfc8788f9b5c..805a3c1bf8fe24b5c50c882740a0f715f5d60b2b 100644 --- a/drivers/net/wireless/st/cw1200/queue.c +++ b/drivers/net/wireless/st/cw1200/queue.c @@ -91,23 +91,25 @@ static void __cw1200_queue_gc(struct cw1200_queue *queue, bool unlock) { struct cw1200_queue_stats *stats = queue->stats; - struct cw1200_queue_item *item = NULL, *tmp; + struct cw1200_queue_item *item = NULL, *iter, *tmp; bool wakeup_stats = false; - list_for_each_entry_safe(item, tmp, &queue->queue, head) { - if (time_is_after_jiffies(item->queue_timestamp + queue->ttl)) + list_for_each_entry_safe(iter, tmp, &queue->queue, head) { + if (time_is_after_jiffies(iter->queue_timestamp + queue->ttl)) { + item = iter; break; + } --queue->num_queued; - --queue->link_map_cache[item->txpriv.link_id]; + --queue->link_map_cache[iter->txpriv.link_id]; spin_lock_bh(&stats->lock); --stats->num_queued; - if (!--stats->link_map_cache[item->txpriv.link_id]) + if (!--stats->link_map_cache[iter->txpriv.link_id]) wakeup_stats = true; spin_unlock_bh(&stats->lock); cw1200_debug_tx_ttl(stats->priv); - cw1200_queue_register_post_gc(head, item); - item->skb = NULL; - list_move_tail(&item->head, &queue->free_pool); + cw1200_queue_register_post_gc(head, iter); + iter->skb = NULL; + list_move_tail(&iter->head, &queue->free_pool); } if (wakeup_stats) diff --git a/drivers/net/wireless/st/cw1200/sta.c b/drivers/net/wireless/st/cw1200/sta.c index 26d3614519b1f3c1ed892d49902266ef8bab7f03..8ef1d06b9bbddb1808fb6965482fbefdefab4bb2 100644 --- a/drivers/net/wireless/st/cw1200/sta.c +++ b/drivers/net/wireless/st/cw1200/sta.c @@ -195,7 +195,7 @@ void __cw1200_cqm_bssloss_sm(struct cw1200_common *priv, priv->bss_loss_state++; - skb = ieee80211_nullfunc_get(priv->hw, priv->vif, false); + skb = ieee80211_nullfunc_get(priv->hw, priv->vif, -1, false); WARN_ON(!skb); if (skb) cw1200_tx(priv->hw, NULL, skb); @@ -2263,7 +2263,7 @@ static int cw1200_upload_null(struct cw1200_common *priv) .rate = 0xFF, }; - frame.skb = ieee80211_nullfunc_get(priv->hw, priv->vif, false); + frame.skb = ieee80211_nullfunc_get(priv->hw, priv->vif,-1, false); if (!frame.skb) return -ENOMEM; diff --git a/drivers/net/wireless/st/cw1200/txrx.c b/drivers/net/wireless/st/cw1200/txrx.c index fde21fca6c5eae684010e02269bf3ff6c99b0f24..6894b919ff94b563fc437b3c2b23b51987c6fde1 100644 --- a/drivers/net/wireless/st/cw1200/txrx.c +++ b/drivers/net/wireless/st/cw1200/txrx.c @@ -762,8 +762,7 @@ void cw1200_tx(struct ieee80211_hw *dev, if (ret) goto drop; - rcu_read_lock(); - sta = rcu_dereference(t.sta); + sta = t.sta; spin_lock_bh(&priv->ps_state_lock); { @@ -776,8 +775,6 @@ void cw1200_tx(struct ieee80211_hw *dev, if (tid_update && sta) ieee80211_sta_set_buffered(sta, t.txpriv.tid, true); - rcu_read_unlock(); - cw1200_bh_wakeup(priv); return; @@ -1145,8 +1142,7 @@ void cw1200_rx_cb(struct cw1200_common *priv, /* Remove TSF from the end of frame */ if (arg->flags & WSM_RX_STATUS_TSF_INCLUDED) { - memcpy(&hdr->mactime, skb->data + skb->len - 8, 8); - hdr->mactime = le64_to_cpu(hdr->mactime); + hdr->mactime = get_unaligned_le64(skb->data + skb->len - 8); if (skb->len >= 8) skb_trim(skb, skb->len - 8); } else { diff --git a/drivers/net/wireless/st/cw1200/wsm.c b/drivers/net/wireless/st/cw1200/wsm.c index 5a3e7a626702d82670e53530e4f12777cd907dec..4a9e4b5d3547a2436c8cc7d80cb4ce7265aafda6 100644 --- a/drivers/net/wireless/st/cw1200/wsm.c +++ b/drivers/net/wireless/st/cw1200/wsm.c @@ -1594,7 +1594,7 @@ static int cw1200_get_prio_queue(struct cw1200_common *priv, edca = &priv->edca.params[i]; score = ((edca->aifns + edca->cwmin) << 16) + ((edca->cwmax - edca->cwmin) * - (get_random_int() & 0xFFFF)); + get_random_u16()); if (score < best && (winner < 0 || i != 3)) { best = score; winner = i; diff --git a/drivers/net/wireless/ti/wl1251/main.c b/drivers/net/wireless/ti/wl1251/main.c index 9144ef5538a8e249a42c2b5074f88c50a5285658..289371689a8deb728039bcfe205374df8829280b 100644 --- a/drivers/net/wireless/ti/wl1251/main.c +++ b/drivers/net/wireless/ti/wl1251/main.c @@ -546,7 +546,7 @@ static int wl1251_build_null_data(struct wl1251 *wl) size = sizeof(struct wl12xx_null_data_template); ptr = NULL; } else { - skb = ieee80211_nullfunc_get(wl->hw, wl->vif, false); + skb = ieee80211_nullfunc_get(wl->hw, wl->vif, -1, false); if (!skb) goto out; size = skb->len; diff --git a/drivers/net/wireless/ti/wl18xx/event.c b/drivers/net/wireless/ti/wl18xx/event.c index 13d78ada4bb69ab3153beec51a0dbd3a3d10de76..34d95f458e1a92f2e53b00e1493270ff48a096aa 100644 --- a/drivers/net/wireless/ti/wl18xx/event.c +++ b/drivers/net/wireless/ti/wl18xx/event.c @@ -131,10 +131,10 @@ int wl18xx_process_mailbox_events(struct wl1271 *wl) if (vector & TIME_SYNC_EVENT_ID) wlcore_event_time_sync(wl, - mbox->time_sync_tsf_high_msb, - mbox->time_sync_tsf_high_lsb, - mbox->time_sync_tsf_low_msb, - mbox->time_sync_tsf_low_lsb); + le16_to_cpu(mbox->time_sync_tsf_high_msb), + le16_to_cpu(mbox->time_sync_tsf_high_lsb), + le16_to_cpu(mbox->time_sync_tsf_low_msb), + le16_to_cpu(mbox->time_sync_tsf_low_lsb)); if (vector & RADAR_DETECTED_EVENT_ID) { wl1271_info("radar event: channel %d type %s", diff --git a/drivers/net/wireless/ti/wlcore/cmd.c b/drivers/net/wireless/ti/wlcore/cmd.c index 138edd28b0de045238cc6beacb480c8f742210d8..a939fd89a7f5e4f5dbaf07dcb6fc3bcc48c7c55a 100644 --- a/drivers/net/wireless/ti/wlcore/cmd.c +++ b/drivers/net/wireless/ti/wlcore/cmd.c @@ -1065,7 +1065,7 @@ int wl12xx_cmd_build_null_data(struct wl1271 *wl, struct wl12xx_vif *wlvif) } else { skb = ieee80211_nullfunc_get(wl->hw, wl12xx_wlvif_to_vif(wlvif), - false); + -1, false); if (!skb) goto out; size = skb->len; @@ -1092,7 +1092,7 @@ int wl12xx_cmd_build_klv_null_data(struct wl1271 *wl, struct sk_buff *skb = NULL; int ret = -ENOMEM; - skb = ieee80211_nullfunc_get(wl->hw, vif, false); + skb = ieee80211_nullfunc_get(wl->hw, vif,-1, false); if (!skb) goto out; diff --git a/drivers/net/wireless/ti/wlcore/main.c b/drivers/net/wireless/ti/wlcore/main.c index 3e3922d4c78801d6b50fd2ea538791e0cf0df398..28c0f06e311f75fb1739f0c81b6aa8aae978ef62 100644 --- a/drivers/net/wireless/ti/wlcore/main.c +++ b/drivers/net/wireless/ti/wlcore/main.c @@ -6100,7 +6100,7 @@ static int wl1271_register_hw(struct wl1271 *wl) wl1271_warning("Fuse mac address is zero. using random mac"); /* Use TI oui and a random nic */ oui_addr = WLCORE_TI_OUI_ADDRESS; - nic_addr = get_random_int(); + nic_addr = get_random_u32(); } else { oui_addr = wl->fuse_oui_addr; /* fuse has the BD_ADDR, the WLAN addresses are the next two */ diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c index dad38fc0424334372f3bbe248e2844ff8d997d4e..1b532e00a56fb6a6490dd048cdd2810e32d1cd94 100644 --- a/drivers/net/wireless/wl3501_cs.c +++ b/drivers/net/wireless/wl3501_cs.c @@ -1441,7 +1441,7 @@ static void wl3501_detach(struct pcmcia_device *link) static int wl3501_get_name(struct net_device *dev, struct iw_request_info *info, union iwreq_data *wrqu, char *extra) { - strlcpy(wrqu->name, "IEEE 802.11-DS", sizeof(wrqu->name)); + strscpy(wrqu->name, "IEEE 802.11-DS", sizeof(wrqu->name)); return 0; } @@ -1652,7 +1652,7 @@ static int wl3501_set_nick(struct net_device *dev, struct iw_request_info *info, if (wrqu->data.length > sizeof(this->nick)) return -E2BIG; - strlcpy(this->nick, extra, wrqu->data.length); + strscpy(this->nick, extra, wrqu->data.length); return 0; } @@ -1661,7 +1661,7 @@ static int wl3501_get_nick(struct net_device *dev, struct iw_request_info *info, { struct wl3501_card *this = netdev_priv(dev); - strlcpy(extra, this->nick, 32); + strscpy(extra, this->nick, 32); wrqu->data.length = strlen(extra); return 0; } @@ -1965,7 +1965,7 @@ static int wl3501_config(struct pcmcia_device *link) this->firmware_date[0] = '\0'; this->rssi = 255; this->chan = iw_default_channel(this->reg_domain); - strlcpy(this->nick, "Planet WL3501", sizeof(this->nick)); + strscpy(this->nick, "Planet WL3501", sizeof(this->nick)); spin_lock_init(&this->lock); init_waitqueue_head(&this->wait); netif_start_queue(dev); diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c index 57304a5adf68e07e11893c364a00b51163e22827..b7f9237dedf76835687a9bb45676b72df866453d 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c +++ b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c @@ -590,7 +590,7 @@ int ipc_imem_sys_devlink_write(struct iosm_devlink *ipc_devlink, goto out; } - memcpy(skb_put(skb, count), buf, count); + skb_put_data(skb, buf, count); IPC_CB(skb)->op_type = UL_USR_OP_BLOCKED; diff --git a/drivers/net/wwan/iosm/iosm_ipc_wwan.c b/drivers/net/wwan/iosm/iosm_ipc_wwan.c index 27151148c782ccfcfd47c2405b3d7147735aa820..2f1f8b5d5b59532baa30b148a760635e89c2dc68 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_wwan.c +++ b/drivers/net/wwan/iosm/iosm_ipc_wwan.c @@ -103,8 +103,8 @@ static int ipc_wwan_link_stop(struct net_device *netdev) } /* Transmit a packet */ -static int ipc_wwan_link_transmit(struct sk_buff *skb, - struct net_device *netdev) +static netdev_tx_t ipc_wwan_link_transmit(struct sk_buff *skb, + struct net_device *netdev) { struct iosm_netdev_priv *priv = wwan_netdev_drvpriv(netdev); struct iosm_wwan *ipc_wwan = priv->ipc_wwan; @@ -323,15 +323,16 @@ struct iosm_wwan *ipc_wwan_init(struct iosm_imem *ipc_imem, struct device *dev) ipc_wwan->dev = dev; ipc_wwan->ipc_imem = ipc_imem; + mutex_init(&ipc_wwan->if_mutex); + /* WWAN core will create a netdev for the default IP MUX channel */ if (wwan_register_ops(ipc_wwan->dev, &iosm_wwan_ops, ipc_wwan, IP_MUX_SESSION_DEFAULT)) { + mutex_destroy(&ipc_wwan->if_mutex); kfree(ipc_wwan); return NULL; } - mutex_init(&ipc_wwan->if_mutex); - return ipc_wwan; } diff --git a/drivers/net/wwan/mhi_wwan_ctrl.c b/drivers/net/wwan/mhi_wwan_ctrl.c index e4d0f696687f209560094e413ffe379283dcefa5..f7ca52353f401231a4b6ce5c138fb0202e3c91b2 100644 --- a/drivers/net/wwan/mhi_wwan_ctrl.c +++ b/drivers/net/wwan/mhi_wwan_ctrl.c @@ -258,6 +258,7 @@ static void mhi_wwan_ctrl_remove(struct mhi_device *mhi_dev) static const struct mhi_device_id mhi_wwan_ctrl_match_table[] = { { .chan = "DUN", .driver_data = WWAN_PORT_AT }, + { .chan = "DUN2", .driver_data = WWAN_PORT_AT }, { .chan = "MBIM", .driver_data = WWAN_PORT_MBIM }, { .chan = "QMI", .driver_data = WWAN_PORT_QMI }, { .chan = "DIAG", .driver_data = WWAN_PORT_QCDM }, diff --git a/drivers/net/wwan/t7xx/t7xx_netdev.c b/drivers/net/wwan/t7xx/t7xx_netdev.c index c6b6547f2c6f62c2124ef3feaf49d96cadf463b4..f71d3bc3b237c53924bfb7f60ba2c0958a27b2ba 100644 --- a/drivers/net/wwan/t7xx/t7xx_netdev.c +++ b/drivers/net/wwan/t7xx/t7xx_netdev.c @@ -74,7 +74,7 @@ static int t7xx_ccmni_send_packet(struct t7xx_ccmni *ccmni, struct sk_buff *skb, return 0; } -static int t7xx_ccmni_start_xmit(struct sk_buff *skb, struct net_device *dev) +static netdev_tx_t t7xx_ccmni_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct t7xx_ccmni *ccmni = wwan_netdev_drvpriv(dev); int skb_len = skb->len; diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index b8c7843730edfcd2eec0f5a5afa4dc2f0c684a1a..62e9f7d6c9fe97d2d5bc87038a2753a8317cd965 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/net/wwan/wwan_hwsim.c b/drivers/net/wwan/wwan_hwsim.c index fad642f9ffd8d886e589542034de062adb636b83..ff09a8cedf93896f7682b92ce7c6eb4bd27febd1 100644 --- a/drivers/net/wwan/wwan_hwsim.c +++ b/drivers/net/wwan/wwan_hwsim.c @@ -157,8 +157,8 @@ static int wwan_hwsim_port_tx(struct wwan_port *wport, struct sk_buff *in) if ((i + 1) < in->len && in->data[i + 1] == '\n') i++; n = i - s + 1; - memcpy(skb_put(out, n), &in->data[s], n);/* Echo */ - memcpy(skb_put(out, 6), "\r\nOK\r\n", 6); + skb_put_data(out, &in->data[s], n);/* Echo */ + skb_put_data(out, "\r\nOK\r\n", 6); s = i + 1; port->pstate = AT_PARSER_WAIT_A; } else if (port->pstate == AT_PARSER_SKIP_LINE) { @@ -171,7 +171,7 @@ static int wwan_hwsim_port_tx(struct wwan_port *wport, struct sk_buff *in) if (i > s) { /* Echo the processed portion of a not yet completed command */ n = i - s; - memcpy(skb_put(out, n), &in->data[s], n); + skb_put_data(out, &in->data[s], n); } consume_skb(in); diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h index 8174d7b2966c0a10ae520e470bcd2dda720f4693..1545cbee77a46c37c4cdbafc63c1ae3c9f7cc06c 100644 --- a/drivers/net/xen-netback/common.h +++ b/drivers/net/xen-netback/common.h @@ -62,7 +62,7 @@ struct pending_tx_info { * ubuf_to_vif is a helper which finds the struct xenvif from a pointer * to this field. */ - struct ubuf_info callback_struct; + struct ubuf_info_msgzc callback_struct; }; #define XEN_NETIF_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE) diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index fb32ae82d9b04afae68032ace0b1792648d2e108..650fa180220fd6d1fb75f529a19680c7aab0173d 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c @@ -591,8 +591,8 @@ int xenvif_init_queue(struct xenvif_queue *queue) } for (i = 0; i < MAX_PENDING_REQS; i++) { - queue->pending_tx_info[i].callback_struct = (struct ubuf_info) - { .callback = xenvif_zerocopy_callback, + queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc) + { { .callback = xenvif_zerocopy_callback }, { { .ctx = NULL, .desc = i } } }; queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE; @@ -723,8 +723,7 @@ int xenvif_connect_data(struct xenvif_queue *queue, init_waitqueue_head(&queue->dealloc_wq); atomic_set(&queue->inflight_packets, 0); - netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll); queue->stalled = true; diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index a256695fc89ec3a2d98aa07e59b906df0cedefef..3d2081bbbc8609ad018b00b4d1379e775c6ffdbb 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c @@ -133,7 +133,7 @@ static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue, /* Find the containing VIF's structure from a pointer in pending_tx_info array */ -static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf) +static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info_msgzc *ubuf) { u16 pending_idx = ubuf->desc; struct pending_tx_info *temp = @@ -1228,11 +1228,12 @@ static int xenvif_tx_submit(struct xenvif_queue *queue) return work_done; } -void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, +void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base, bool zerocopy_success) { unsigned long flags; pending_ring_idx_t index; + struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); struct xenvif_queue *queue = ubuf_to_queue(ubuf); /* This is the only place where we grab this lock, to protect callbacks @@ -1241,7 +1242,7 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf, spin_lock_irqsave(&queue->callback_lock, flags); do { u16 pending_idx = ubuf->desc; - ubuf = (struct ubuf_info *) ubuf->ctx; + ubuf = (struct ubuf_info_msgzc *) ubuf->ctx; BUG_ON(queue->dealloc_prod - queue->dealloc_cons >= MAX_PENDING_REQS); index = pending_index(queue->dealloc_prod); diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c index 990360d75cb6481936d2a7c5e1e5e63ee25a9db6..c1ba4294f364793f16397bca14ba6b75f5c8bd59 100644 --- a/drivers/net/xen-netback/xenbus.c +++ b/drivers/net/xen-netback/xenbus.c @@ -256,7 +256,6 @@ static void backend_disconnect(struct backend_info *be) unsigned int queue_index; xen_unregister_watchers(vif); - xenbus_rm(XBT_NIL, be->dev->nodename, "hotplug-status"); #ifdef CONFIG_DEBUG_FS xenvif_debugfs_delif(vif); #endif /* CONFIG_DEBUG_FS */ @@ -866,13 +865,12 @@ static int connect_data_rings(struct backend_info *be, * queue-N. */ if (num_queues == 1) { - xspath = kzalloc(strlen(dev->otherend) + 1, GFP_KERNEL); + xspath = kstrdup(dev->otherend, GFP_KERNEL); if (!xspath) { xenbus_dev_fatal(dev, -ENOMEM, "reading ring references"); return -ENOMEM; } - strcpy(xspath, dev->otherend); } else { xspathsize = strlen(dev->otherend) + xenstore_path_ext_size; xspath = kzalloc(xspathsize, GFP_KERNEL); @@ -984,6 +982,7 @@ static int netback_remove(struct xenbus_device *dev) struct backend_info *be = dev_get_drvdata(&dev->dev); unregister_hotplug_status_watch(be); + xenbus_rm(XBT_NIL, dev->nodename, "hotplug-status"); if (be->vif) { kobject_uevent(&dev->dev.kobj, KOBJ_OFFLINE); backend_disconnect(be); diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 27a11cc08c61ea1498e50dc9eff2d3d249284a19..9af2b027c19c6899c6b5a3efc8f2d3a54f92d71f 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -673,7 +673,7 @@ static int xennet_xdp_xmit(struct net_device *dev, int n, return nxmit; } -struct sk_buff *bounce_skb(const struct sk_buff *skb) +static struct sk_buff *bounce_skb(const struct sk_buff *skb) { unsigned int headerlen = skb_headroom(skb); /* Align size to allocate full pages and avoid contiguous data leaks */ @@ -2224,8 +2224,7 @@ static int xennet_create_queues(struct netfront_info *info, return ret; } - netif_napi_add(queue->info->netdev, &queue->napi, - xennet_poll, 64); + netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll); if (netif_running(info->netdev)) napi_enable(&queue->napi); } diff --git a/drivers/nfc/fdp/i2c.c b/drivers/nfc/fdp/i2c.c index 28a9e1eb9bcf49d8aec2cb0f5ef5a268d4fbeedc..2d53e0f88d2f95a6cf0f781285ba499279f9f33c 100644 --- a/drivers/nfc/fdp/i2c.c +++ b/drivers/nfc/fdp/i2c.c @@ -336,14 +336,12 @@ static int fdp_nci_i2c_probe(struct i2c_client *client) return 0; } -static int fdp_nci_i2c_remove(struct i2c_client *client) +static void fdp_nci_i2c_remove(struct i2c_client *client) { struct fdp_i2c_phy *phy = i2c_get_clientdata(client); fdp_nci_remove(phy->ndev); fdp_nci_i2c_disable(phy); - - return 0; } static const struct acpi_device_id fdp_nci_i2c_acpi_match[] = { diff --git a/drivers/nfc/microread/i2c.c b/drivers/nfc/microread/i2c.c index 067295124eb9ea7ce560a3a2a1407c634a49de9a..5eaa18f8135592e87a3ea7b329bf60fd4c33b0b0 100644 --- a/drivers/nfc/microread/i2c.c +++ b/drivers/nfc/microread/i2c.c @@ -268,15 +268,13 @@ err_irq: return r; } -static int microread_i2c_remove(struct i2c_client *client) +static void microread_i2c_remove(struct i2c_client *client) { struct microread_i2c_phy *phy = i2c_get_clientdata(client); microread_remove(phy->hdev); free_irq(client->irq, phy); - - return 0; } static const struct i2c_device_id microread_i2c_id[] = { diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c index 01329b91d59d547577680a058bced7d001c6c364..acef0cfd76af35406f1b245924e94745e80a5623 100644 --- a/drivers/nfc/nfcmrvl/i2c.c +++ b/drivers/nfc/nfcmrvl/i2c.c @@ -231,13 +231,11 @@ static int nfcmrvl_i2c_probe(struct i2c_client *client, return 0; } -static int nfcmrvl_i2c_remove(struct i2c_client *client) +static void nfcmrvl_i2c_remove(struct i2c_client *client) { struct nfcmrvl_i2c_drv_data *drv_data = i2c_get_clientdata(client); nfcmrvl_nci_unregister_dev(drv_data->priv); - - return 0; } diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c index ae2ba08d8ac3f3b7b4fa9db0b8ec8093d2258723..ec6446511984dea7b50343df66ff2bd8a8c76e75 100644 --- a/drivers/nfc/nxp-nci/i2c.c +++ b/drivers/nfc/nxp-nci/i2c.c @@ -314,14 +314,12 @@ static int nxp_nci_i2c_probe(struct i2c_client *client, return r; } -static int nxp_nci_i2c_remove(struct i2c_client *client) +static void nxp_nci_i2c_remove(struct i2c_client *client) { struct nxp_nci_i2c_phy *phy = i2c_get_clientdata(client); nxp_nci_remove(phy->ndev); free_irq(client->irq, phy); - - return 0; } static const struct i2c_device_id nxp_nci_i2c_id_table[] = { diff --git a/drivers/nfc/pn533/i2c.c b/drivers/nfc/pn533/i2c.c index 673eb5e9b8876f408ac882793be6743dcd3d5414..ddf3db286bada65a1b2ac7d969980ae98f2cddd2 100644 --- a/drivers/nfc/pn533/i2c.c +++ b/drivers/nfc/pn533/i2c.c @@ -227,7 +227,7 @@ nfc_alloc_err: return r; } -static int pn533_i2c_remove(struct i2c_client *client) +static void pn533_i2c_remove(struct i2c_client *client) { struct pn533_i2c_phy *phy = i2c_get_clientdata(client); @@ -235,8 +235,6 @@ static int pn533_i2c_remove(struct i2c_client *client) pn53x_unregister_nfc(phy->priv); pn53x_common_clean(phy->priv); - - return 0; } static const struct of_device_id of_pn533_i2c_match[] __maybe_unused = { diff --git a/drivers/nfc/pn533/uart.c b/drivers/nfc/pn533/uart.c index 2caf997f9bc94a2456b1d70c4847f8c4320ee435..07596bf5f7d6d6adda6481bb2e58824c056ce82e 100644 --- a/drivers/nfc/pn533/uart.c +++ b/drivers/nfc/pn533/uart.c @@ -310,6 +310,7 @@ static void pn532_uart_remove(struct serdev_device *serdev) pn53x_unregister_nfc(pn532->priv); serdev_device_close(serdev); pn53x_common_clean(pn532->priv); + del_timer_sync(&pn532->cmd_timeout); kfree_skb(pn532->recv_skb); kfree(pn532); } diff --git a/drivers/nfc/pn544/i2c.c b/drivers/nfc/pn544/i2c.c index 62a0f1a010cbfe6a0d18efc1969d0d32afec42e6..9e754abcfa2a122c4bae0eca53e75b4710c7944e 100644 --- a/drivers/nfc/pn544/i2c.c +++ b/drivers/nfc/pn544/i2c.c @@ -928,7 +928,7 @@ static int pn544_hci_i2c_probe(struct i2c_client *client, return 0; } -static int pn544_hci_i2c_remove(struct i2c_client *client) +static void pn544_hci_i2c_remove(struct i2c_client *client) { struct pn544_i2c_phy *phy = i2c_get_clientdata(client); @@ -940,8 +940,6 @@ static int pn544_hci_i2c_remove(struct i2c_client *client) if (phy->powered) pn544_hci_i2c_disable(phy); - - return 0; } static const struct of_device_id of_pn544_i2c_match[] __maybe_unused = { diff --git a/drivers/nfc/s3fwrn5/i2c.c b/drivers/nfc/s3fwrn5/i2c.c index 4d1cf1bb55b08cf60afb09a78d01dd5589681692..f824dc7099ce735ee3e2d6102a37cb6d138ccc2c 100644 --- a/drivers/nfc/s3fwrn5/i2c.c +++ b/drivers/nfc/s3fwrn5/i2c.c @@ -246,14 +246,12 @@ disable_clk: return ret; } -static int s3fwrn5_i2c_remove(struct i2c_client *client) +static void s3fwrn5_i2c_remove(struct i2c_client *client) { struct s3fwrn5_i2c_phy *phy = i2c_get_clientdata(client); s3fwrn5_remove(phy->common.ndev); clk_disable_unprepare(phy->clk); - - return 0; } static const struct i2c_device_id s3fwrn5_i2c_id_table[] = { diff --git a/drivers/nfc/st-nci/i2c.c b/drivers/nfc/st-nci/i2c.c index cbd968f013c70db9bc60ec9ed766a2ea9fa22a54..89fa24d71bef9f3b1e8f9d994110761c71d72d05 100644 --- a/drivers/nfc/st-nci/i2c.c +++ b/drivers/nfc/st-nci/i2c.c @@ -250,13 +250,11 @@ static int st_nci_i2c_probe(struct i2c_client *client, return r; } -static int st_nci_i2c_remove(struct i2c_client *client) +static void st_nci_i2c_remove(struct i2c_client *client) { struct st_nci_i2c_phy *phy = i2c_get_clientdata(client); ndlc_remove(phy->ndlc); - - return 0; } static const struct i2c_device_id st_nci_i2c_id_table[] = { diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c index 42dc0e5eb1610a4d940f85b9304ea8de0e4ac369..76b55986bcf85b1afd9822ce7be0593a6dd1dc99 100644 --- a/drivers/nfc/st21nfca/i2c.c +++ b/drivers/nfc/st21nfca/i2c.c @@ -562,7 +562,7 @@ out_free: return r; } -static int st21nfca_hci_i2c_remove(struct i2c_client *client) +static void st21nfca_hci_i2c_remove(struct i2c_client *client) { struct st21nfca_i2c_phy *phy = i2c_get_clientdata(client); @@ -571,8 +571,6 @@ static int st21nfca_hci_i2c_remove(struct i2c_client *client) if (phy->powered) st21nfca_hci_i2c_disable(phy); kfree_skb(phy->pending_skb); - - return 0; } static const struct i2c_device_id st21nfca_hci_i2c_id_table[] = { diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index bf4f5c09d9b1b3685eb359b64733335d729061fc..c60ec0b373c51595ac2cc940834a2a6ef82d746e 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -170,15 +170,12 @@ EXPORT_SYMBOL(nvdimm_namespace_disk_name); const uuid_t *nd_dev_to_uuid(struct device *dev) { - if (!dev) - return &uuid_null; - - if (is_namespace_pmem(dev)) { + if (dev && is_namespace_pmem(dev)) { struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev); return nspm->uuid; - } else - return &uuid_null; + } + return &uuid_null; } EXPORT_SYMBOL(nd_dev_to_uuid); @@ -388,7 +385,7 @@ static resource_size_t init_dpa_allocation(struct nd_label_id *label_id, * * BLK-space is valid as long as it does not precede a PMEM * allocation in a given region. PMEM-space must be contiguous - * and adjacent to an existing existing allocation (if one + * and adjacent to an existing allocation (if one * exists). If reserving PMEM any space is valid. */ static void space_valid(struct nd_region *nd_region, struct nvdimm_drvdata *ndd, @@ -839,7 +836,6 @@ static ssize_t size_store(struct device *dev, { struct nd_region *nd_region = to_nd_region(dev->parent); unsigned long long val; - uuid_t **uuid = NULL; int rc; rc = kstrtoull(buf, 0, &val); @@ -853,16 +849,12 @@ static ssize_t size_store(struct device *dev, if (rc >= 0) rc = nd_namespace_label_update(nd_region, dev); - if (is_namespace_pmem(dev)) { + /* setting size zero == 'delete namespace' */ + if (rc == 0 && val == 0 && is_namespace_pmem(dev)) { struct nd_namespace_pmem *nspm = to_nd_namespace_pmem(dev); - uuid = &nspm->uuid; - } - - if (rc == 0 && val == 0 && uuid) { - /* setting size zero == 'delete namespace' */ - kfree(*uuid); - *uuid = NULL; + kfree(nspm->uuid); + nspm->uuid = NULL; } dev_dbg(dev, "%llx %s (%d)\n", val, rc < 0 ? "fail" : "success", rc); @@ -1712,8 +1704,6 @@ static struct device *create_namespace_pmem(struct nd_region *nd_region, res->flags = IORESOURCE_MEM; for (i = 0; i < nd_region->ndr_mappings; i++) { - uuid_t uuid; - nsl_get_uuid(ndd, nd_label, &uuid); if (has_uuid_at_pos(nd_region, &uuid, cookie, i)) continue; diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index ec5219680092d7abcf4ed4bc6911f361a43419ea..85ca5b4da3cf3b71e51b7580273a6419f2a05ce6 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -652,7 +652,7 @@ void devm_namespace_disable(struct device *dev, struct nd_namespace_common *ndns); #if IS_ENABLED(CONFIG_ND_CLAIM) /* max struct page size independent of kernel config */ -#define MAX_STRUCT_PAGE_SIZE 64 +#define MAX_STRUCT_PAGE_SIZE 128 int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap); #else static inline int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, diff --git a/drivers/nvdimm/pfn_devs.c b/drivers/nvdimm/pfn_devs.c index 0e92ab4b32833a45ab3a60f8ef3cc3607f5a40e1..61af072ac98f99d33b813b4851dc21b87e82d48c 100644 --- a/drivers/nvdimm/pfn_devs.c +++ b/drivers/nvdimm/pfn_devs.c @@ -787,7 +787,7 @@ static int nd_pfn_init(struct nd_pfn *nd_pfn) * when populating the vmemmap. This *should* be equal to * PMD_SIZE for most architectures. * - * Also make sure size of struct page is less than 64. We + * Also make sure size of struct page is less than 128. We * want to make sure we use large enough size here so that * we don't have a dynamic reserve space depending on * struct page size. But we also want to make sure we notice diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 7e88cd24238051d5414aa5b513239657cd4904a4..96e6e9a5f235d82684c8005b4b0d317e7293ad3f 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -45,7 +45,7 @@ static struct nd_region *to_region(struct pmem_device *pmem) return to_nd_region(to_dev(pmem)->parent); } -static phys_addr_t to_phys(struct pmem_device *pmem, phys_addr_t offset) +static phys_addr_t pmem_to_phys(struct pmem_device *pmem, phys_addr_t offset) { return pmem->phys_addr + offset; } @@ -63,7 +63,7 @@ static phys_addr_t to_offset(struct pmem_device *pmem, sector_t sector) static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { - phys_addr_t phys = to_phys(pmem, offset); + phys_addr_t phys = pmem_to_phys(pmem, offset); unsigned long pfn_start, pfn_end, pfn; /* only pmem in the linear map supports HWPoison */ @@ -97,7 +97,7 @@ static void pmem_clear_bb(struct pmem_device *pmem, sector_t sector, long blks) static long __pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { - phys_addr_t phys = to_phys(pmem, offset); + phys_addr_t phys = pmem_to_phys(pmem, offset); long cleared = nvdimm_clear_poison(to_dev(pmem), phys, len); if (cleared > 0) { diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 473a71bbd9c9e68176adaaa8364a0213ab90c8b9..e0875d3697624cf8ae0199cc9084f517210cd675 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -509,16 +509,13 @@ static ssize_t align_store(struct device *dev, { struct nd_region *nd_region = to_nd_region(dev); unsigned long val, dpa; - u32 remainder; + u32 mappings, remainder; int rc; rc = kstrtoul(buf, 0, &val); if (rc) return rc; - if (!nd_region->ndr_mappings) - return -ENXIO; - /* * Ensure space-align is evenly divisible by the region * interleave-width because the kernel typically has no facility @@ -526,7 +523,8 @@ static ssize_t align_store(struct device *dev, * contribute to the tail capacity in system-physical-address * space for the namespace. */ - dpa = div_u64_rem(val, nd_region->ndr_mappings, &remainder); + mappings = max_t(u32, 1, nd_region->ndr_mappings); + dpa = div_u64_rem(val, mappings, &remainder); if (!is_power_of_2(dpa) || dpa < PAGE_SIZE || val > region_size(nd_region) || remainder) return -EINVAL; @@ -1096,7 +1094,7 @@ int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) return rc; } /** - * nvdimm_flush - flush any posted write queues between the cpu and pmem media + * generic_nvdimm_flush() - flush any posted write queues between the cpu and pmem media * @nd_region: interleaved pmem region */ int generic_nvdimm_flush(struct nd_region *nd_region) diff --git a/drivers/nvdimm/security.c b/drivers/nvdimm/security.c index b5aa55c61461621138fd110873ce94cb889f70ee..8aefb60c42fff328468d9dc49fc3d170c5dc8b0d 100644 --- a/drivers/nvdimm/security.c +++ b/drivers/nvdimm/security.c @@ -408,7 +408,7 @@ static int security_overwrite(struct nvdimm *nvdimm, unsigned int keyid) return rc; } -void __nvdimm_security_overwrite_query(struct nvdimm *nvdimm) +static void __nvdimm_security_overwrite_query(struct nvdimm *nvdimm) { struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nvdimm->dev); int rc; diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index 04bd28f17dccea86d7e41ce4613beee1c268a191..d90e4f0c08b7b90201e745663d658afd61b21d57 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -23,7 +23,7 @@ u32 nvme_auth_get_seqnum(void) mutex_lock(&nvme_dhchap_mutex); if (!nvme_dhchap_seqnum) - nvme_dhchap_seqnum = prandom_u32(); + nvme_dhchap_seqnum = get_random_u32(); else { nvme_dhchap_seqnum++; if (!nvme_dhchap_seqnum) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index af367b22871b1f71ae803165d3978d1f87f1f800..059737c1a2c19c3a9d73f0b0c4ec2d5a878017e1 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1111,8 +1111,8 @@ static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, return effects; } -static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects, - struct nvme_command *cmd, int status) +void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects, + struct nvme_command *cmd, int status) { if (effects & NVME_CMD_EFFECTS_CSE_MASK) { nvme_unfreeze(ctrl); @@ -1148,21 +1148,16 @@ static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects, break; } } +EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU); -int nvme_execute_passthru_rq(struct request *rq) +int nvme_execute_passthru_rq(struct request *rq, u32 *effects) { struct nvme_command *cmd = nvme_req(rq)->cmd; struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl; struct nvme_ns *ns = rq->q->queuedata; - u32 effects; - int ret; - effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); - ret = nvme_execute_rq(rq, false); - if (effects) /* nothing to be done for zero cmd effects */ - nvme_passthru_end(ctrl, effects, cmd, ret); - - return ret; + *effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode); + return nvme_execute_rq(rq, false); } EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU); @@ -1177,7 +1172,8 @@ static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl) queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2); } -static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status) +static enum rq_end_io_ret nvme_keep_alive_end_io(struct request *rq, + blk_status_t status) { struct nvme_ctrl *ctrl = rq->end_io_data; unsigned long flags; @@ -1189,7 +1185,7 @@ static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status) dev_err(ctrl->device, "failed nvme_keep_alive_end_io error=%d\n", status); - return; + return RQ_END_IO_NONE; } ctrl->comp_seen = false; @@ -1200,6 +1196,7 @@ static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status) spin_unlock_irqrestore(&ctrl->lock, flags); if (startka) nvme_queue_keep_alive_work(ctrl); + return RQ_END_IO_NONE; } static void nvme_keep_alive_work(struct work_struct *work) @@ -2162,14 +2159,14 @@ static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new, static int nvme_pr_clear(struct block_device *bdev, u64 key) { - u32 cdw10 = 1 | (key ? 1 << 3 : 0); + u32 cdw10 = 1 | (key ? 0 : 1 << 3); - return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register); + return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); } static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type) { - u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0); + u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3); return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release); } @@ -2696,7 +2693,7 @@ static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ct if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) { nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE); if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) { - strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); + strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE); return; } @@ -2704,7 +2701,11 @@ static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ct dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n"); } - /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */ + /* + * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe + * Base Specification 2.0. It is slightly different from the format + * specified there due to historic reasons, and we can't change it now. + */ off = snprintf(subsys->subnqn, NVMF_NQN_SIZE, "nqn.2014.08.org.nvmexpress:%04x%04x", le16_to_cpu(id->vid), le16_to_cpu(id->ssvid)); @@ -2894,7 +2895,6 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) nvme_init_subnqn(subsys, ctrl, id); memcpy(subsys->serial, id->sn, sizeof(subsys->serial)); memcpy(subsys->model, id->mn, sizeof(subsys->model)); - memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev)); subsys->vendor_id = le16_to_cpu(id->vid); subsys->cmic = id->cmic; @@ -3113,6 +3113,8 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl) ctrl->quirks |= core_quirks[i].quirks; } } + memcpy(ctrl->subsys->firmware_rev, id->fr, + sizeof(ctrl->subsys->firmware_rev)); if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) { dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n"); @@ -3976,6 +3978,7 @@ static const struct file_operations nvme_ns_chr_fops = { .unlocked_ioctl = nvme_ns_chr_ioctl, .compat_ioctl = compat_ptr_ioctl, .uring_cmd = nvme_ns_chr_uring_cmd, + .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, }; static int nvme_add_ns_cdev(struct nvme_ns *ns) @@ -4703,6 +4706,8 @@ static void nvme_fw_act_work(struct work_struct *work) nvme_start_queues(ctrl); /* read FW slot information to clear the AER */ nvme_get_fw_slot_info(ctrl); + + queue_work(nvme_wq, &ctrl->async_event_work); } static u32 nvme_aer_type(u32 result) @@ -4715,9 +4720,10 @@ static u32 nvme_aer_subtype(u32 result) return (result & 0xff00) >> 8; } -static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) +static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) { u32 aer_notice_type = nvme_aer_subtype(result); + bool requeue = true; trace_nvme_async_event(ctrl, aer_notice_type); @@ -4734,6 +4740,7 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) */ if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) { nvme_auth_stop(ctrl); + requeue = false; queue_work(nvme_wq, &ctrl->fw_act_work); } break; @@ -4750,6 +4757,7 @@ static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result) default: dev_warn(ctrl->device, "async event result %08x\n", result); } + return requeue; } static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl) @@ -4765,13 +4773,14 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, u32 result = le32_to_cpu(res->u32); u32 aer_type = nvme_aer_type(result); u32 aer_subtype = nvme_aer_subtype(result); + bool requeue = true; if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS) return; switch (aer_type) { case NVME_AER_NOTICE: - nvme_handle_aen_notice(ctrl, result); + requeue = nvme_handle_aen_notice(ctrl, result); break; case NVME_AER_ERROR: /* @@ -4792,10 +4801,114 @@ void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, default: break; } - queue_work(nvme_wq, &ctrl->async_event_work); + + if (requeue) + queue_work(nvme_wq, &ctrl->async_event_work); } EXPORT_SYMBOL_GPL(nvme_complete_async_event); +int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, unsigned int flags, + unsigned int cmd_size) +{ + int ret; + + memset(set, 0, sizeof(*set)); + set->ops = ops; + set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; + if (ctrl->ops->flags & NVME_F_FABRICS) + set->reserved_tags = NVMF_RESERVED_TAGS; + set->numa_node = ctrl->numa_node; + set->flags = flags; + set->cmd_size = cmd_size; + set->driver_data = ctrl; + set->nr_hw_queues = 1; + set->timeout = NVME_ADMIN_TIMEOUT; + ret = blk_mq_alloc_tag_set(set); + if (ret) + return ret; + + ctrl->admin_q = blk_mq_init_queue(set); + if (IS_ERR(ctrl->admin_q)) { + ret = PTR_ERR(ctrl->admin_q); + goto out_free_tagset; + } + + if (ctrl->ops->flags & NVME_F_FABRICS) { + ctrl->fabrics_q = blk_mq_init_queue(set); + if (IS_ERR(ctrl->fabrics_q)) { + ret = PTR_ERR(ctrl->fabrics_q); + goto out_cleanup_admin_q; + } + } + + ctrl->admin_tagset = set; + return 0; + +out_cleanup_admin_q: + blk_mq_destroy_queue(ctrl->fabrics_q); +out_free_tagset: + blk_mq_free_tag_set(ctrl->admin_tagset); + return ret; +} +EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set); + +void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl) +{ + blk_mq_destroy_queue(ctrl->admin_q); + if (ctrl->ops->flags & NVME_F_FABRICS) + blk_mq_destroy_queue(ctrl->fabrics_q); + blk_mq_free_tag_set(ctrl->admin_tagset); +} +EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set); + +int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, unsigned int flags, + unsigned int cmd_size) +{ + int ret; + + memset(set, 0, sizeof(*set)); + set->ops = ops; + set->queue_depth = ctrl->sqsize + 1; + set->reserved_tags = NVMF_RESERVED_TAGS; + set->numa_node = ctrl->numa_node; + set->flags = flags; + set->cmd_size = cmd_size, + set->driver_data = ctrl; + set->nr_hw_queues = ctrl->queue_count - 1; + set->timeout = NVME_IO_TIMEOUT; + if (ops->map_queues) + set->nr_maps = ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; + ret = blk_mq_alloc_tag_set(set); + if (ret) + return ret; + + if (ctrl->ops->flags & NVME_F_FABRICS) { + ctrl->connect_q = blk_mq_init_queue(set); + if (IS_ERR(ctrl->connect_q)) { + ret = PTR_ERR(ctrl->connect_q); + goto out_free_tag_set; + } + } + + ctrl->tagset = set; + return 0; + +out_free_tag_set: + blk_mq_free_tag_set(set); + return ret; +} +EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set); + +void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl) +{ + if (ctrl->ops->flags & NVME_F_FABRICS) + blk_mq_destroy_queue(ctrl->connect_q); + blk_mq_free_tag_set(ctrl->tagset); +} +EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set); + void nvme_stop_ctrl(struct nvme_ctrl *ctrl) { nvme_mpath_stop(ctrl); @@ -4815,6 +4928,16 @@ void nvme_start_ctrl(struct nvme_ctrl *ctrl) nvme_enable_aen(ctrl); + /* + * persistent discovery controllers need to send indication to userspace + * to re-read the discovery log page to learn about possible changes + * that were missed. We identify persistent discovery controllers by + * checking that they started once before, hence are reconnecting back. + */ + if (test_and_set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) && + nvme_discovery_ctrl(ctrl)) + nvme_change_uevent(ctrl, "NVME_EVENT=rediscover"); + if (ctrl->queue_count > 1) { nvme_queue_scan(ctrl); nvme_start_queues(ctrl); diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 10cc4a8146027933a7d8a9b7b88f25eeda65049b..ce27276f552dada471ea5306bc43450a04484909 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -49,7 +49,7 @@ static struct nvmf_host *nvmf_host_add(const char *hostnqn) goto out_unlock; kref_init(&host->ref); - strlcpy(host->nqn, hostnqn, NVMF_NQN_SIZE); + strscpy(host->nqn, hostnqn, NVMF_NQN_SIZE); list_add_tail(&host->list, &nvmf_hosts); out_unlock: @@ -971,13 +971,17 @@ bool nvmf_ip_options_match(struct nvme_ctrl *ctrl, return false; /* - * Checking the local address is rough. In most cases, none is specified - * and the host port is selected by the stack. + * Checking the local address or host interfaces is rough. + * + * In most cases, none is specified and the host port or + * host interface is selected by the stack. * * Assume no match if: - * - local address is specified and address is not the same - * - local address is not specified but remote is, or vice versa - * (admin using specific host_traddr when it matters). + * - local address or host interface is specified and address + * or host interface is not the same + * - local address or host interface is not specified but + * remote is, or vice versa (admin using specific + * host_traddr/host_iface when it matters). */ if ((opts->mask & NVMF_OPT_HOST_TRADDR) && (ctrl->opts->mask & NVMF_OPT_HOST_TRADDR)) { @@ -988,6 +992,15 @@ bool nvmf_ip_options_match(struct nvme_ctrl *ctrl, return false; } + if ((opts->mask & NVMF_OPT_HOST_IFACE) && + (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) { + if (strcmp(opts->host_iface, ctrl->opts->host_iface)) + return false; + } else if ((opts->mask & NVMF_OPT_HOST_IFACE) || + (ctrl->opts->mask & NVMF_OPT_HOST_IFACE)) { + return false; + } + return true; } EXPORT_SYMBOL_GPL(nvmf_ip_options_match); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 127abaf9ba5d694c4c5fd92d9510dcdd789b1e2a..5d57a042dbcade8170facfddb205080a06ffc088 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1829,7 +1829,7 @@ nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq, { struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq); - return __nvme_fc_exit_request(set->driver_data, op); + return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op); } static int @@ -2135,7 +2135,7 @@ static int nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) { - struct nvme_fc_ctrl *ctrl = set->driver_data; + struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data); struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq); int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; struct nvme_fc_queue *queue = &ctrl->queues[queue_idx]; @@ -2206,36 +2206,28 @@ nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl) } } -static inline void -__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, struct nvme_fc_ctrl *ctrl, - unsigned int qidx) +static inline int +__nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx) { + struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data); struct nvme_fc_queue *queue = &ctrl->queues[qidx]; hctx->driver_data = queue; queue->hctx = hctx; + return 0; } static int -nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, - unsigned int hctx_idx) +nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_fc_ctrl *ctrl = data; - - __nvme_fc_init_hctx(hctx, ctrl, hctx_idx + 1); - - return 0; + return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1); } static int nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_fc_ctrl *ctrl = data; - - __nvme_fc_init_hctx(hctx, ctrl, hctx_idx); - - return 0; + return __nvme_fc_init_hctx(hctx, data, hctx_idx); } static void @@ -2391,10 +2383,8 @@ nvme_fc_ctrl_free(struct kref *ref) container_of(ref, struct nvme_fc_ctrl, ref); unsigned long flags; - if (ctrl->ctrl.tagset) { - blk_mq_destroy_queue(ctrl->ctrl.connect_q); - blk_mq_free_tag_set(&ctrl->tag_set); - } + if (ctrl->ctrl.tagset) + nvme_remove_io_tag_set(&ctrl->ctrl); /* remove from rport list */ spin_lock_irqsave(&ctrl->rport->lock, flags); @@ -2402,9 +2392,7 @@ nvme_fc_ctrl_free(struct kref *ref) spin_unlock_irqrestore(&ctrl->rport->lock, flags); nvme_start_admin_queue(&ctrl->ctrl); - blk_mq_destroy_queue(ctrl->ctrl.admin_q); - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); - blk_mq_free_tag_set(&ctrl->admin_tag_set); + nvme_remove_admin_tag_set(&ctrl->ctrl); kfree(ctrl->queues); @@ -2860,9 +2848,9 @@ nvme_fc_complete_rq(struct request *rq) nvme_fc_ctrl_put(ctrl); } -static int nvme_fc_map_queues(struct blk_mq_tag_set *set) +static void nvme_fc_map_queues(struct blk_mq_tag_set *set) { - struct nvme_fc_ctrl *ctrl = set->driver_data; + struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data); int i; for (i = 0; i < set->nr_maps; i++) { @@ -2880,7 +2868,6 @@ static int nvme_fc_map_queues(struct blk_mq_tag_set *set) else blk_mq_map_queues(map); } - return 0; } static const struct blk_mq_ops nvme_fc_mq_ops = { @@ -2915,32 +2902,16 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) nvme_fc_init_io_queues(ctrl); - memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set)); - ctrl->tag_set.ops = &nvme_fc_mq_ops; - ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size; - ctrl->tag_set.reserved_tags = NVMF_RESERVED_TAGS; - ctrl->tag_set.numa_node = ctrl->ctrl.numa_node; - ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; - ctrl->tag_set.cmd_size = - struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, - ctrl->lport->ops->fcprqst_priv_sz); - ctrl->tag_set.driver_data = ctrl; - ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1; - ctrl->tag_set.timeout = NVME_IO_TIMEOUT; - - ret = blk_mq_alloc_tag_set(&ctrl->tag_set); + ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set, + &nvme_fc_mq_ops, BLK_MQ_F_SHOULD_MERGE, + struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, + ctrl->lport->ops->fcprqst_priv_sz)); if (ret) return ret; - ctrl->ctrl.tagset = &ctrl->tag_set; - - ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl)); - if (ret) - goto out_free_tag_set; - ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1); if (ret) - goto out_cleanup_blk_queue; + goto out_cleanup_tagset; ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1); if (ret) @@ -2952,10 +2923,8 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) out_delete_hw_queues: nvme_fc_delete_hw_io_queues(ctrl); -out_cleanup_blk_queue: - blk_mq_destroy_queue(ctrl->ctrl.connect_q); -out_free_tag_set: - blk_mq_free_tag_set(&ctrl->tag_set); +out_cleanup_tagset: + nvme_remove_io_tag_set(&ctrl->ctrl); nvme_fc_free_io_queues(ctrl); /* force put free routine to ignore io queues */ @@ -3166,15 +3135,7 @@ nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) "to maxcmd\n", opts->queue_size, ctrl->ctrl.maxcmd); opts->queue_size = ctrl->ctrl.maxcmd; - } - - if (opts->queue_size > ctrl->ctrl.sqsize + 1) { - /* warn if sqsize is lower than queue_size */ - dev_warn(ctrl->ctrl.device, - "queue_size %zu > ctrl sqsize %u, reducing " - "to sqsize\n", - opts->queue_size, ctrl->ctrl.sqsize + 1); - opts->queue_size = ctrl->ctrl.sqsize + 1; + ctrl->ctrl.sqsize = opts->queue_size - 1; } ret = nvme_fc_init_aen_ops(ctrl); @@ -3547,35 +3508,12 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, nvme_fc_init_queue(ctrl, 0); - memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); - ctrl->admin_tag_set.ops = &nvme_fc_admin_mq_ops; - ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; - ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS; - ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node; - ctrl->admin_tag_set.cmd_size = - struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, - ctrl->lport->ops->fcprqst_priv_sz); - ctrl->admin_tag_set.driver_data = ctrl; - ctrl->admin_tag_set.nr_hw_queues = 1; - ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT; - ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED; - - ret = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); + ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set, + &nvme_fc_admin_mq_ops, BLK_MQ_F_NO_SCHED, + struct_size((struct nvme_fcp_op_w_sgl *)NULL, priv, + ctrl->lport->ops->fcprqst_priv_sz)); if (ret) goto out_free_queues; - ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set; - - ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.fabrics_q)) { - ret = PTR_ERR(ctrl->ctrl.fabrics_q); - goto out_free_admin_tag_set; - } - - ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.admin_q)) { - ret = PTR_ERR(ctrl->ctrl.admin_q); - goto out_cleanup_fabrics_q; - } /* * Would have been nice to init io queues tag set as well. @@ -3586,7 +3524,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0); if (ret) - goto out_cleanup_admin_q; + goto out_cleanup_tagset; /* at this point, teardown path changes to ref counting on nvme ctrl */ @@ -3641,12 +3579,8 @@ fail_ctrl: return ERR_PTR(-EIO); -out_cleanup_admin_q: - blk_mq_destroy_queue(ctrl->ctrl.admin_q); -out_cleanup_fabrics_q: - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); -out_free_admin_tag_set: - blk_mq_free_tag_set(&ctrl->admin_tag_set); +out_cleanup_tagset: + nvme_remove_admin_tag_set(&ctrl->ctrl); out_free_queues: kfree(ctrl->queues); out_free_ida: diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 27614bee73806cb2b0b777e583ff39d28667ed48..81f5550b670da05c55d961fed84368c858e0325d 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -20,19 +20,20 @@ static void __user *nvme_to_user_ptr(uintptr_t ptrval) return (void __user *)ptrval; } -static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, - unsigned len, u32 seed, bool write) +static void *nvme_add_user_metadata(struct request *req, void __user *ubuf, + unsigned len, u32 seed) { struct bio_integrity_payload *bip; int ret = -ENOMEM; void *buf; + struct bio *bio = req->bio; buf = kmalloc(len, GFP_KERNEL); if (!buf) goto out; ret = -EFAULT; - if (write && copy_from_user(buf, ubuf, len)) + if ((req_op(req) == REQ_OP_DRV_OUT) && copy_from_user(buf, ubuf, len)) goto out_free_meta; bip = bio_integrity_alloc(bio, GFP_KERNEL, 1); @@ -45,9 +46,13 @@ static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf, bip->bip_iter.bi_sector = seed; ret = bio_integrity_add_page(bio, virt_to_page(buf), len, offset_in_page(buf)); - if (ret == len) - return buf; - ret = -ENOMEM; + if (ret != len) { + ret = -ENOMEM; + goto out_free_meta; + } + + req->cmd_flags |= REQ_INTEGRITY; + return buf; out_free_meta: kfree(buf); out: @@ -65,90 +70,102 @@ static int nvme_finish_user_metadata(struct request *req, void __user *ubuf, } static struct request *nvme_alloc_user_request(struct request_queue *q, - struct nvme_command *cmd, void __user *ubuffer, - unsigned bufflen, void __user *meta_buffer, unsigned meta_len, - u32 meta_seed, void **metap, unsigned timeout, bool vec, - blk_opf_t rq_flags, blk_mq_req_flags_t blk_flags) + struct nvme_command *cmd, blk_opf_t rq_flags, + blk_mq_req_flags_t blk_flags) { - bool write = nvme_is_write(cmd); - struct nvme_ns *ns = q->queuedata; - struct block_device *bdev = ns ? ns->disk->part0 : NULL; struct request *req; - struct bio *bio = NULL; - void *meta = NULL; - int ret; req = blk_mq_alloc_request(q, nvme_req_op(cmd) | rq_flags, blk_flags); if (IS_ERR(req)) return req; nvme_init_request(req, cmd); - - if (timeout) - req->timeout = timeout; nvme_req(req)->flags |= NVME_REQ_USERCMD; + return req; +} - if (ubuffer && bufflen) { - if (!vec) - ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, - GFP_KERNEL); - else { - struct iovec fast_iov[UIO_FASTIOV]; - struct iovec *iov = fast_iov; - struct iov_iter iter; - - ret = import_iovec(rq_data_dir(req), ubuffer, bufflen, - UIO_FASTIOV, &iov, &iter); - if (ret < 0) - goto out; - ret = blk_rq_map_user_iov(q, req, NULL, &iter, - GFP_KERNEL); - kfree(iov); - } - if (ret) +static int nvme_map_user_request(struct request *req, u64 ubuffer, + unsigned bufflen, void __user *meta_buffer, unsigned meta_len, + u32 meta_seed, void **metap, struct io_uring_cmd *ioucmd, + bool vec) +{ + struct request_queue *q = req->q; + struct nvme_ns *ns = q->queuedata; + struct block_device *bdev = ns ? ns->disk->part0 : NULL; + struct bio *bio = NULL; + void *meta = NULL; + int ret; + + if (ioucmd && (ioucmd->flags & IORING_URING_CMD_FIXED)) { + struct iov_iter iter; + + /* fixedbufs is only for non-vectored io */ + if (WARN_ON_ONCE(vec)) + return -EINVAL; + ret = io_uring_cmd_import_fixed(ubuffer, bufflen, + rq_data_dir(req), &iter, ioucmd); + if (ret < 0) goto out; - bio = req->bio; - if (bdev) - bio_set_dev(bio, bdev); - if (bdev && meta_buffer && meta_len) { - meta = nvme_add_user_metadata(bio, meta_buffer, meta_len, - meta_seed, write); - if (IS_ERR(meta)) { - ret = PTR_ERR(meta); - goto out_unmap; - } - req->cmd_flags |= REQ_INTEGRITY; - *metap = meta; + ret = blk_rq_map_user_iov(q, req, NULL, &iter, GFP_KERNEL); + } else { + ret = blk_rq_map_user_io(req, NULL, nvme_to_user_ptr(ubuffer), + bufflen, GFP_KERNEL, vec, 0, 0, + rq_data_dir(req)); + } + + if (ret) + goto out; + bio = req->bio; + if (bdev) + bio_set_dev(bio, bdev); + + if (bdev && meta_buffer && meta_len) { + meta = nvme_add_user_metadata(req, meta_buffer, meta_len, + meta_seed); + if (IS_ERR(meta)) { + ret = PTR_ERR(meta); + goto out_unmap; } + *metap = meta; } - return req; + return ret; out_unmap: if (bio) blk_rq_unmap_user(bio); out: blk_mq_free_request(req); - return ERR_PTR(ret); + return ret; } static int nvme_submit_user_cmd(struct request_queue *q, - struct nvme_command *cmd, void __user *ubuffer, + struct nvme_command *cmd, u64 ubuffer, unsigned bufflen, void __user *meta_buffer, unsigned meta_len, u32 meta_seed, u64 *result, unsigned timeout, bool vec) { + struct nvme_ctrl *ctrl; struct request *req; void *meta = NULL; struct bio *bio; + u32 effects; int ret; - req = nvme_alloc_user_request(q, cmd, ubuffer, bufflen, meta_buffer, - meta_len, meta_seed, &meta, timeout, vec, 0, 0); + req = nvme_alloc_user_request(q, cmd, 0, 0); if (IS_ERR(req)) return PTR_ERR(req); + req->timeout = timeout; + if (ubuffer && bufflen) { + ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer, + meta_len, meta_seed, &meta, NULL, vec); + if (ret) + return ret; + } + bio = req->bio; + ctrl = nvme_req(req)->ctrl; - ret = nvme_execute_passthru_rq(req); + ret = nvme_execute_passthru_rq(req, &effects); if (result) *result = le64_to_cpu(nvme_req(req)->result.u64); @@ -158,6 +175,10 @@ static int nvme_submit_user_cmd(struct request_queue *q, if (bio) blk_rq_unmap_user(bio); blk_mq_free_request(req); + + if (effects) + nvme_passthru_end(ctrl, effects, cmd, ret); + return ret; } @@ -220,7 +241,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.appmask = cpu_to_le16(io.appmask); return nvme_submit_user_cmd(ns->queue, &c, - nvme_to_user_ptr(io.addr), length, + io.addr, length, metadata, meta_len, lower_32_bits(io.slba), NULL, 0, false); } @@ -274,7 +295,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, timeout = msecs_to_jiffies(cmd.timeout_ms); status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, - nvme_to_user_ptr(cmd.addr), cmd.data_len, + cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 0, &result, timeout, false); @@ -320,7 +341,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, timeout = msecs_to_jiffies(cmd.timeout_ms); status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c, - nvme_to_user_ptr(cmd.addr), cmd.data_len, + cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata), cmd.metadata_len, 0, &cmd.result, timeout, vec); @@ -349,9 +370,15 @@ struct nvme_uring_cmd_pdu { struct bio *bio; struct request *req; }; - void *meta; /* kernel-resident buffer */ - void __user *meta_buffer; u32 meta_len; + u32 nvme_status; + union { + struct { + void *meta; /* kernel-resident buffer */ + void __user *meta_buffer; + }; + u64 result; + } u; }; static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( @@ -360,11 +387,10 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( return (struct nvme_uring_cmd_pdu *)&ioucmd->pdu; } -static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd) +static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd) { struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); struct request *req = pdu->req; - struct bio *bio = req->bio; int status; u64 result; @@ -375,27 +401,72 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd) result = le64_to_cpu(nvme_req(req)->result.u64); - if (pdu->meta) - status = nvme_finish_user_metadata(req, pdu->meta_buffer, - pdu->meta, pdu->meta_len, status); - if (bio) - blk_rq_unmap_user(bio); + if (pdu->meta_len) + status = nvme_finish_user_metadata(req, pdu->u.meta_buffer, + pdu->u.meta, pdu->meta_len, status); + if (req->bio) + blk_rq_unmap_user(req->bio); blk_mq_free_request(req); io_uring_cmd_done(ioucmd, status, result); } -static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) +static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd) +{ + struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); + + if (pdu->bio) + blk_rq_unmap_user(pdu->bio); + + io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result); +} + +static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, + blk_status_t err) { struct io_uring_cmd *ioucmd = req->end_io_data; struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); - /* extract bio before reusing the same field for request */ - struct bio *bio = pdu->bio; + void *cookie = READ_ONCE(ioucmd->cookie); + req->bio = pdu->bio; + if (nvme_req(req)->flags & NVME_REQ_CANCELLED) + pdu->nvme_status = -EINTR; + else + pdu->nvme_status = nvme_req(req)->status; + pdu->u.result = le64_to_cpu(nvme_req(req)->result.u64); + + /* + * For iopoll, complete it directly. + * Otherwise, move the completion to task work. + */ + if (cookie != NULL && blk_rq_is_poll(req)) + nvme_uring_task_cb(ioucmd); + else + io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); + + return RQ_END_IO_FREE; +} + +static enum rq_end_io_ret nvme_uring_cmd_end_io_meta(struct request *req, + blk_status_t err) +{ + struct io_uring_cmd *ioucmd = req->end_io_data; + struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); + void *cookie = READ_ONCE(ioucmd->cookie); + + req->bio = pdu->bio; pdu->req = req; - req->bio = bio; - /* this takes care of moving rest of completion-work to task context */ - io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); + + /* + * For iopoll, complete it directly. + * Otherwise, move the completion to task work. + */ + if (cookie != NULL && blk_rq_is_poll(req)) + nvme_uring_task_meta_cb(ioucmd); + else + io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_meta_cb); + + return RQ_END_IO_NONE; } static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, @@ -410,6 +481,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, blk_opf_t rq_flags = 0; blk_mq_req_flags_t blk_flags = 0; void *meta = NULL; + int ret; if (!capable(CAP_SYS_ADMIN)) return -EACCES; @@ -445,23 +517,45 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, rq_flags = REQ_NOWAIT; blk_flags = BLK_MQ_REQ_NOWAIT; } + if (issue_flags & IO_URING_F_IOPOLL) + rq_flags |= REQ_POLLED; - req = nvme_alloc_user_request(q, &c, nvme_to_user_ptr(d.addr), - d.data_len, nvme_to_user_ptr(d.metadata), - d.metadata_len, 0, &meta, d.timeout_ms ? - msecs_to_jiffies(d.timeout_ms) : 0, vec, rq_flags, - blk_flags); +retry: + req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags); if (IS_ERR(req)) return PTR_ERR(req); - req->end_io = nvme_uring_cmd_end_io; - req->end_io_data = ioucmd; + req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0; + + if (d.addr && d.data_len) { + ret = nvme_map_user_request(req, d.addr, + d.data_len, nvme_to_user_ptr(d.metadata), + d.metadata_len, 0, &meta, ioucmd, vec); + if (ret) + return ret; + } + if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) { + if (unlikely(!req->bio)) { + /* we can't poll this, so alloc regular req instead */ + blk_mq_free_request(req); + rq_flags &= ~REQ_POLLED; + goto retry; + } else { + WRITE_ONCE(ioucmd->cookie, req->bio); + req->bio->bi_opf |= REQ_POLLED; + } + } /* to free bio on completion, as req->bio will be null at that time */ pdu->bio = req->bio; - pdu->meta = meta; - pdu->meta_buffer = nvme_to_user_ptr(d.metadata); pdu->meta_len = d.metadata_len; - + req->end_io_data = ioucmd; + if (pdu->meta_len) { + pdu->u.meta = meta; + pdu->u.meta_buffer = nvme_to_user_ptr(d.metadata); + req->end_io = nvme_uring_cmd_end_io_meta; + } else { + req->end_io = nvme_uring_cmd_end_io; + } blk_execute_rq_nowait(req, false); return -EIOCBQUEUED; } @@ -559,9 +653,6 @@ long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) static int nvme_uring_cmd_checks(unsigned int issue_flags) { - /* IOPOLL not supported yet */ - if (issue_flags & IO_URING_F_IOPOLL) - return -EOPNOTSUPP; /* NVMe passthrough requires big SQE/CQE support */ if ((issue_flags & (IO_URING_F_SQE128|IO_URING_F_CQE32)) != @@ -604,6 +695,25 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) return nvme_ns_uring_cmd(ns, ioucmd, issue_flags); } +int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, + struct io_comp_batch *iob, + unsigned int poll_flags) +{ + struct bio *bio; + int ret = 0; + struct nvme_ns *ns; + struct request_queue *q; + + rcu_read_lock(); + bio = READ_ONCE(ioucmd->cookie); + ns = container_of(file_inode(ioucmd->file)->i_cdev, + struct nvme_ns, cdev); + q = ns->queue; + if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev) + ret = bio_poll(bio, iob, poll_flags); + rcu_read_unlock(); + return ret; +} #ifdef CONFIG_NVME_MULTIPATH static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *argp, struct nvme_ns_head *head, int srcu_idx) @@ -685,6 +795,31 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, srcu_read_unlock(&head->srcu, srcu_idx); return ret; } + +int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, + struct io_comp_batch *iob, + unsigned int poll_flags) +{ + struct cdev *cdev = file_inode(ioucmd->file)->i_cdev; + struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev); + int srcu_idx = srcu_read_lock(&head->srcu); + struct nvme_ns *ns = nvme_find_path(head); + struct bio *bio; + int ret = 0; + struct request_queue *q; + + if (ns) { + rcu_read_lock(); + bio = READ_ONCE(ioucmd->cookie); + q = ns->queue; + if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio + && bio->bi_bdev) + ret = bio_poll(bio, iob, poll_flags); + rcu_read_unlock(); + } + srcu_read_unlock(&head->srcu, srcu_idx); + return ret; +} #endif /* CONFIG_NVME_MULTIPATH */ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) @@ -692,6 +827,10 @@ int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags) struct nvme_ctrl *ctrl = ioucmd->file->private_data; int ret; + /* IOPOLL not supported yet */ + if (issue_flags & IO_URING_F_IOPOLL) + return -EOPNOTSUPP; + ret = nvme_uring_cmd_checks(issue_flags); if (ret) return ret; @@ -757,11 +896,17 @@ long nvme_dev_ioctl(struct file *file, unsigned int cmd, case NVME_IOCTL_IO_CMD: return nvme_dev_user_cmd(ctrl, argp); case NVME_IOCTL_RESET: + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; dev_warn(ctrl->device, "resetting controller\n"); return nvme_reset_ctrl_sync(ctrl); case NVME_IOCTL_SUBSYS_RESET: + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; return nvme_reset_subsystem(ctrl); case NVME_IOCTL_RESCAN: + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; nvme_queue_scan(ctrl); return 0; default: diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 6ef497c75a1655529ef7010bb5fcf07beb6127a5..0ea7e441e080f2863dcc7eec7102560572fb3cb7 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -182,6 +182,7 @@ void nvme_mpath_revalidate_paths(struct nvme_ns *ns) for_each_node(node) rcu_assign_pointer(head->current_path[node], NULL); + kblockd_schedule_work(&head->requeue_work); } static bool nvme_path_is_disabled(struct nvme_ns *ns) @@ -439,6 +440,7 @@ static const struct file_operations nvme_ns_head_chr_fops = { .unlocked_ioctl = nvme_ns_head_chr_ioctl, .compat_ioctl = compat_ptr_ioctl, .uring_cmd = nvme_ns_head_chr_uring_cmd, + .uring_cmd_iopoll = nvme_ns_head_chr_uring_cmd_iopoll, }; static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 1bdf714dcd9e410c1b2415d660b98055dfa78208..a29877217ee65ca007d696b13f2c20dadd0d14ab 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -233,6 +233,12 @@ struct nvme_fault_inject { #endif }; +enum nvme_ctrl_flags { + NVME_CTRL_FAILFAST_EXPIRED = 0, + NVME_CTRL_ADMIN_Q_STOPPED = 1, + NVME_CTRL_STARTED_ONCE = 2, +}; + struct nvme_ctrl { bool comp_seen; enum nvme_ctrl_state state; @@ -354,8 +360,6 @@ struct nvme_ctrl { u16 maxcmd; int nr_reconnects; unsigned long flags; -#define NVME_CTRL_FAILFAST_EXPIRED 0 -#define NVME_CTRL_ADMIN_Q_STOPPED 1 struct nvmf_ctrl_options *opts; struct page *discard_page; @@ -602,11 +606,23 @@ static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj) static inline void nvme_should_fail(struct request *req) {} #endif +bool nvme_wait_reset(struct nvme_ctrl *ctrl); +int nvme_try_sched_reset(struct nvme_ctrl *ctrl); + static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) { + int ret; + if (!ctrl->subsystem) return -ENOTTY; - return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); + if (!nvme_wait_reset(ctrl)) + return -EBUSY; + + ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65); + if (ret) + return ret; + + return nvme_try_sched_reset(ctrl); } /* @@ -712,7 +728,6 @@ void nvme_cancel_tagset(struct nvme_ctrl *ctrl); void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl); bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, enum nvme_ctrl_state new_state); -bool nvme_wait_reset(struct nvme_ctrl *ctrl); int nvme_disable_ctrl(struct nvme_ctrl *ctrl); int nvme_enable_ctrl(struct nvme_ctrl *ctrl); int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); @@ -722,6 +737,14 @@ void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); void nvme_start_ctrl(struct nvme_ctrl *ctrl); void nvme_stop_ctrl(struct nvme_ctrl *ctrl); int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl); +int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, unsigned int flags, + unsigned int cmd_size); +void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl); +int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, + const struct blk_mq_ops *ops, unsigned int flags, + unsigned int cmd_size); +void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl); void nvme_remove_namespaces(struct nvme_ctrl *ctrl); @@ -802,7 +825,6 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); int nvme_reset_ctrl(struct nvme_ctrl *ctrl); int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl); -int nvme_try_sched_reset(struct nvme_ctrl *ctrl); int nvme_delete_ctrl(struct nvme_ctrl *ctrl); void nvme_queue_scan(struct nvme_ctrl *ctrl); int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, @@ -821,6 +843,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg); long nvme_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, + struct io_comp_batch *iob, unsigned int poll_flags); +int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, + struct io_comp_batch *iob, unsigned int poll_flags); int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags); int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, @@ -968,14 +994,6 @@ static inline int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf) } #endif -static inline int nvme_ctrl_init_connect_q(struct nvme_ctrl *ctrl) -{ - ctrl->connect_q = blk_mq_init_queue(ctrl->tagset); - if (IS_ERR(ctrl->connect_q)) - return PTR_ERR(ctrl->connect_q); - return 0; -} - static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev) { return dev_to_disk(dev)->private_data; @@ -1023,7 +1041,9 @@ static inline void nvme_auth_free(struct nvme_ctrl *ctrl) {}; u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); -int nvme_execute_passthru_rq(struct request *rq); +int nvme_execute_passthru_rq(struct request *rq, u32 *effects); +void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects, + struct nvme_command *cmd, int status); struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); void nvme_put_ns(struct nvme_ns *ns); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 3a1c37f32f30d93992556937e7ed356bd617cbfa..bcbef6bc5672f09e43f7e939c8bcc25d0b26b4c7 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -226,12 +226,12 @@ struct nvme_queue { struct nvme_iod { struct nvme_request req; struct nvme_command cmd; - struct nvme_queue *nvmeq; bool use_sgl; - int aborted; - int npages; /* In the PRP list. 0 means small pool in use */ - dma_addr_t first_dma; + bool aborted; + s8 nr_allocations; /* PRP list pool allocations. 0 means small + pool in use */ unsigned int dma_len; /* length of single DMA segment mapping */ + dma_addr_t first_dma; dma_addr_t meta_dma; struct sg_table sgt; }; @@ -430,11 +430,6 @@ static int nvme_pci_init_request(struct blk_mq_tag_set *set, { struct nvme_dev *dev = set->driver_data; struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0; - struct nvme_queue *nvmeq = &dev->queues[queue_idx]; - - BUG_ON(!nvmeq); - iod->nvmeq = nvmeq; nvme_req(req)->ctrl = &dev->ctrl; nvme_req(req)->cmd = &iod->cmd; @@ -450,7 +445,7 @@ static int queue_irq_offset(struct nvme_dev *dev) return 0; } -static int nvme_pci_map_queues(struct blk_mq_tag_set *set) +static void nvme_pci_map_queues(struct blk_mq_tag_set *set) { struct nvme_dev *dev = set->driver_data; int i, qoff, offset; @@ -477,8 +472,6 @@ static int nvme_pci_map_queues(struct blk_mq_tag_set *set) qoff += map->nr_queues; offset += map->nr_queues; } - - return 0; } /* @@ -528,7 +521,7 @@ static void **nvme_pci_iod_list(struct request *req) static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) { - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; int nseg = blk_rq_nr_phys_segments(req); unsigned int avg_seg_size; @@ -536,7 +529,7 @@ static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req) if (!nvme_ctrl_sgl_supported(&dev->ctrl)) return false; - if (!iod->nvmeq->qid) + if (!nvmeq->qid) return false; if (!sgl_threshold || avg_seg_size < sgl_threshold) return false; @@ -550,7 +543,7 @@ static void nvme_free_prps(struct nvme_dev *dev, struct request *req) dma_addr_t dma_addr = iod->first_dma; int i; - for (i = 0; i < iod->npages; i++) { + for (i = 0; i < iod->nr_allocations; i++) { __le64 *prp_list = nvme_pci_iod_list(req)[i]; dma_addr_t next_dma_addr = le64_to_cpu(prp_list[last_prp]); @@ -566,7 +559,7 @@ static void nvme_free_sgls(struct nvme_dev *dev, struct request *req) dma_addr_t dma_addr = iod->first_dma; int i; - for (i = 0; i < iod->npages; i++) { + for (i = 0; i < iod->nr_allocations; i++) { struct nvme_sgl_desc *sg_list = nvme_pci_iod_list(req)[i]; dma_addr_t next_dma_addr = le64_to_cpu((sg_list[last_sg]).addr); @@ -589,7 +582,7 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req) dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0); - if (iod->npages == 0) + if (iod->nr_allocations == 0) dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0], iod->first_dma); else if (iod->use_sgl) @@ -651,15 +644,15 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE); if (nprps <= (256 / 8)) { pool = dev->prp_small_pool; - iod->npages = 0; + iod->nr_allocations = 0; } else { pool = dev->prp_page_pool; - iod->npages = 1; + iod->nr_allocations = 1; } prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); if (!prp_list) { - iod->npages = -1; + iod->nr_allocations = -1; return BLK_STS_RESOURCE; } list[0] = prp_list; @@ -671,7 +664,7 @@ static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev, prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma); if (!prp_list) goto free_prps; - list[iod->npages++] = prp_list; + list[iod->nr_allocations++] = prp_list; prp_list[0] = old_prp_list[i - 1]; old_prp_list[i - 1] = cpu_to_le64(prp_dma); i = 1; @@ -746,15 +739,15 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, if (entries <= (256 / sizeof(struct nvme_sgl_desc))) { pool = dev->prp_small_pool; - iod->npages = 0; + iod->nr_allocations = 0; } else { pool = dev->prp_page_pool; - iod->npages = 1; + iod->nr_allocations = 1; } sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma); if (!sg_list) { - iod->npages = -1; + iod->nr_allocations = -1; return BLK_STS_RESOURCE; } @@ -773,7 +766,7 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev, goto free_sgls; i = 0; - nvme_pci_iod_list(req)[iod->npages++] = sg_list; + nvme_pci_iod_list(req)[iod->nr_allocations++] = sg_list; sg_list[i++] = *link; nvme_pci_sgl_set_seg(link, sgl_dma, entries); } @@ -833,6 +826,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, int rc; if (blk_rq_nr_phys_segments(req) == 1) { + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; struct bio_vec bv = req_bvec(req); if (!is_pci_p2pdma_page(bv.bv_page)) { @@ -840,7 +834,7 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, return nvme_setup_prp_simple(dev, req, &cmnd->rw, &bv); - if (iod->nvmeq->qid && sgl_threshold && + if (nvmeq->qid && sgl_threshold && nvme_ctrl_sgl_supported(&dev->ctrl)) return nvme_setup_sgl_simple(dev, req, &cmnd->rw, &bv); @@ -898,8 +892,8 @@ static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req) struct nvme_iod *iod = blk_mq_rq_to_pdu(req); blk_status_t ret; - iod->aborted = 0; - iod->npages = -1; + iod->aborted = false; + iod->nr_allocations = -1; iod->sgt.nents = 0; ret = nvme_setup_cmd(req->q->queuedata, req); @@ -1019,12 +1013,16 @@ static void nvme_queue_rqs(struct request **rqlist) static __always_inline void nvme_pci_unmap_rq(struct request *req) { - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - struct nvme_dev *dev = iod->nvmeq->dev; + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + struct nvme_dev *dev = nvmeq->dev; + + if (blk_integrity_rq(req)) { + struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - if (blk_integrity_rq(req)) dma_unmap_page(dev->dev, iod->meta_dma, rq_integrity_vec(req)->bv_len, rq_data_dir(req)); + } + if (blk_rq_nr_phys_segments(req)) nvme_unmap_data(dev, req); } @@ -1270,15 +1268,15 @@ static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid) return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid); } -static void abort_endio(struct request *req, blk_status_t error) +static enum rq_end_io_ret abort_endio(struct request *req, blk_status_t error) { - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - struct nvme_queue *nvmeq = iod->nvmeq; + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; dev_warn(nvmeq->dev->ctrl.device, "Abort status: 0x%x", nvme_req(req)->status); atomic_inc(&nvmeq->dev->ctrl.abort_limit); blk_mq_free_request(req); + return RQ_END_IO_NONE; } static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) @@ -1335,7 +1333,7 @@ static void nvme_warn_reset(struct nvme_dev *dev, u32 csts) static enum blk_eh_timer_return nvme_timeout(struct request *req) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - struct nvme_queue *nvmeq = iod->nvmeq; + struct nvme_queue *nvmeq = req->mq_hctx->driver_data; struct nvme_dev *dev = nvmeq->dev; struct request *abort_req; struct nvme_command cmd = { }; @@ -1416,7 +1414,7 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) atomic_inc(&dev->ctrl.abort_limit); return BLK_EH_RESET_TIMER; } - iod->aborted = 1; + iod->aborted = true; cmd.abort.opcode = nvme_admin_abort_cmd; cmd.abort.cid = nvme_cid(req); @@ -2450,22 +2448,25 @@ out_unlock: return result; } -static void nvme_del_queue_end(struct request *req, blk_status_t error) +static enum rq_end_io_ret nvme_del_queue_end(struct request *req, + blk_status_t error) { struct nvme_queue *nvmeq = req->end_io_data; blk_mq_free_request(req); complete(&nvmeq->delete_done); + return RQ_END_IO_NONE; } -static void nvme_del_cq_end(struct request *req, blk_status_t error) +static enum rq_end_io_ret nvme_del_cq_end(struct request *req, + blk_status_t error) { struct nvme_queue *nvmeq = req->end_io_data; if (error) set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags); - nvme_del_queue_end(req, error); + return nvme_del_queue_end(req, error); } static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) @@ -2529,9 +2530,11 @@ static void nvme_pci_alloc_tag_set(struct nvme_dev *dev) set->ops = &nvme_mq_ops; set->nr_hw_queues = dev->online_queues - 1; - set->nr_maps = 2; /* default + read */ + set->nr_maps = 1; + if (dev->io_queues[HCTX_TYPE_READ]) + set->nr_maps = 2; if (dev->io_queues[HCTX_TYPE_POLL]) - set->nr_maps++; + set->nr_maps = 3; set->timeout = NVME_IO_TIMEOUT; set->numa_node = dev->ctrl.numa_node; set->queue_depth = min_t(unsigned, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1; @@ -2834,6 +2837,8 @@ static void nvme_reset_work(struct work_struct *work) nvme_start_admin_queue(&dev->ctrl); } + dma_set_min_align_mask(dev->dev, NVME_CTRL_PAGE_SIZE - 1); + /* * Limit the max command size to prevent iod->sg allocations going * over a single page. @@ -2846,7 +2851,6 @@ static void nvme_reset_work(struct work_struct *work) * Don't limit the IOMMU merged segment size. */ dma_set_max_seg_size(dev->dev, 0xffffffff); - dma_set_min_align_mask(dev->dev, NVME_CTRL_PAGE_SIZE - 1); mutex_unlock(&dev->shutdown_lock); @@ -3470,6 +3474,10 @@ static const struct pci_device_id nvme_id_table[] = { { PCI_DEVICE(0x1987, 0x5016), /* Phison E16 */ .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN | NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1987, 0x5019), /* phison E19 */ + .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, + { PCI_DEVICE(0x1987, 0x5021), /* Phison E21 */ + .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1b4b, 0x1092), /* Lexar 256 GB SSD */ .driver_data = NVME_QUIRK_NO_NS_DESC_LIST | NVME_QUIRK_IGNORE_DEV_SUBNQN, }, @@ -3513,10 +3521,16 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1e49, 0x0021), /* ZHITAI TiPro5000 NVMe SSD */ + .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0x1e49, 0x0041), /* ZHITAI TiPro7000 NVMe SSD */ .driver_data = NVME_QUIRK_NO_DEEPEST_PS, }, { PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */ + .driver_data = NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */ + .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0061), .driver_data = NVME_QUIRK_DMA_ADDRESS_BITS_48, }, { PCI_DEVICE(PCI_VENDOR_ID_AMAZON, 0x0065), @@ -3563,6 +3577,8 @@ static int __init nvme_init(void) BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64); BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); + BUILD_BUG_ON(DIV_ROUND_UP(nvme_pci_npages_prp(), NVME_CTRL_PAGE_SIZE) > + S8_MAX); return pci_register_driver(&nvme_driver); } diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 3100643be2993c9ff07820daee9ca82635f8dd69..6e079abb22ee97d11bd29f38811ab152fe6ae2c8 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -295,7 +295,7 @@ static int nvme_rdma_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) { - struct nvme_rdma_ctrl *ctrl = set->driver_data; + struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(set->driver_data); struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx]; @@ -320,7 +320,7 @@ static int nvme_rdma_init_request(struct blk_mq_tag_set *set, static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_rdma_ctrl *ctrl = data; + struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(data); struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); @@ -332,7 +332,7 @@ static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_rdma_ctrl *ctrl = data; + struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(data); struct nvme_rdma_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); @@ -696,11 +696,12 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) return ret; } -static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl) +static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl, + int first, int last) { int i, ret = 0; - for (i = 1; i < ctrl->ctrl.queue_count; i++) { + for (i = first; i < last; i++) { ret = nvme_rdma_start_queue(ctrl, i); if (ret) goto out_stop_queues; @@ -709,7 +710,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl) return 0; out_stop_queues: - for (i--; i >= 1; i--) + for (i--; i >= first; i--) nvme_rdma_stop_queue(&ctrl->queues[i]); return ret; } @@ -787,64 +788,21 @@ out_free_queues: return ret; } -static int nvme_rdma_alloc_admin_tag_set(struct nvme_ctrl *nctrl) +static int nvme_rdma_alloc_tag_set(struct nvme_ctrl *ctrl) { - struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl); - struct blk_mq_tag_set *set = &ctrl->admin_tag_set; - int ret; + unsigned int cmd_size = sizeof(struct nvme_rdma_request) + + NVME_RDMA_DATA_SGL_SIZE; - memset(set, 0, sizeof(*set)); - set->ops = &nvme_rdma_admin_mq_ops; - set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; - set->reserved_tags = NVMF_RESERVED_TAGS; - set->numa_node = nctrl->numa_node; - set->cmd_size = sizeof(struct nvme_rdma_request) + - NVME_RDMA_DATA_SGL_SIZE; - set->driver_data = ctrl; - set->nr_hw_queues = 1; - set->timeout = NVME_ADMIN_TIMEOUT; - set->flags = BLK_MQ_F_NO_SCHED; - ret = blk_mq_alloc_tag_set(set); - if (!ret) - ctrl->ctrl.admin_tagset = set; - return ret; -} - -static int nvme_rdma_alloc_tag_set(struct nvme_ctrl *nctrl) -{ - struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl); - struct blk_mq_tag_set *set = &ctrl->tag_set; - int ret; + if (ctrl->max_integrity_segments) + cmd_size += sizeof(struct nvme_rdma_sgl) + + NVME_RDMA_METADATA_SGL_SIZE; - memset(set, 0, sizeof(*set)); - set->ops = &nvme_rdma_mq_ops; - set->queue_depth = nctrl->sqsize + 1; - set->reserved_tags = NVMF_RESERVED_TAGS; - set->numa_node = nctrl->numa_node; - set->flags = BLK_MQ_F_SHOULD_MERGE; - set->cmd_size = sizeof(struct nvme_rdma_request) + - NVME_RDMA_DATA_SGL_SIZE; - if (nctrl->max_integrity_segments) - set->cmd_size += sizeof(struct nvme_rdma_sgl) + - NVME_RDMA_METADATA_SGL_SIZE; - set->driver_data = ctrl; - set->nr_hw_queues = nctrl->queue_count - 1; - set->timeout = NVME_IO_TIMEOUT; - set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; - ret = blk_mq_alloc_tag_set(set); - if (!ret) - ctrl->ctrl.tagset = set; - return ret; + return nvme_alloc_io_tag_set(ctrl, &to_rdma_ctrl(ctrl)->tag_set, + &nvme_rdma_mq_ops, BLK_MQ_F_SHOULD_MERGE, cmd_size); } -static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl, - bool remove) +static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl) { - if (remove) { - blk_mq_destroy_queue(ctrl->ctrl.admin_q); - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); - blk_mq_free_tag_set(ctrl->ctrl.admin_tagset); - } if (ctrl->async_event_sqe.data) { cancel_work_sync(&ctrl->ctrl.async_event_work); nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe, @@ -886,26 +844,19 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, goto out_free_queue; if (new) { - error = nvme_rdma_alloc_admin_tag_set(&ctrl->ctrl); + error = nvme_alloc_admin_tag_set(&ctrl->ctrl, + &ctrl->admin_tag_set, &nvme_rdma_admin_mq_ops, + BLK_MQ_F_NO_SCHED, + sizeof(struct nvme_rdma_request) + + NVME_RDMA_DATA_SGL_SIZE); if (error) goto out_free_async_qe; - ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.fabrics_q)) { - error = PTR_ERR(ctrl->ctrl.fabrics_q); - goto out_free_tagset; - } - - ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.admin_q)) { - error = PTR_ERR(ctrl->ctrl.admin_q); - goto out_cleanup_fabrics_q; - } } error = nvme_rdma_start_queue(ctrl, 0); if (error) - goto out_cleanup_queue; + goto out_remove_admin_tag_set; error = nvme_enable_ctrl(&ctrl->ctrl); if (error) @@ -932,15 +883,9 @@ out_quiesce_queue: out_stop_queue: nvme_rdma_stop_queue(&ctrl->queues[0]); nvme_cancel_admin_tagset(&ctrl->ctrl); -out_cleanup_queue: - if (new) - blk_mq_destroy_queue(ctrl->ctrl.admin_q); -out_cleanup_fabrics_q: +out_remove_admin_tag_set: if (new) - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); -out_free_tagset: - if (new) - blk_mq_free_tag_set(ctrl->ctrl.admin_tagset); + nvme_remove_admin_tag_set(&ctrl->ctrl); out_free_async_qe: if (ctrl->async_event_sqe.data) { nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe, @@ -952,19 +897,9 @@ out_free_queue: return error; } -static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl, - bool remove) -{ - if (remove) { - blk_mq_destroy_queue(ctrl->ctrl.connect_q); - blk_mq_free_tag_set(ctrl->ctrl.tagset); - } - nvme_rdma_free_io_queues(ctrl); -} - static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) { - int ret; + int ret, nr_queues; ret = nvme_rdma_alloc_io_queues(ctrl); if (ret) @@ -974,15 +909,17 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) ret = nvme_rdma_alloc_tag_set(&ctrl->ctrl); if (ret) goto out_free_io_queues; - - ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl)); - if (ret) - goto out_free_tag_set; } - ret = nvme_rdma_start_io_queues(ctrl); + /* + * Only start IO queues for which we have allocated the tagset + * and limitted it to the available queues. On reconnects, the + * queue number might have changed. + */ + nr_queues = min(ctrl->tag_set.nr_hw_queues + 1, ctrl->ctrl.queue_count); + ret = nvme_rdma_start_io_queues(ctrl, 1, nr_queues); if (ret) - goto out_cleanup_connect_q; + goto out_cleanup_tagset; if (!new) { nvme_start_queues(&ctrl->ctrl); @@ -1000,19 +937,25 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) nvme_unfreeze(&ctrl->ctrl); } + /* + * If the number of queues has increased (reconnect case) + * start all new queues now. + */ + ret = nvme_rdma_start_io_queues(ctrl, nr_queues, + ctrl->tag_set.nr_hw_queues + 1); + if (ret) + goto out_wait_freeze_timed_out; + return 0; out_wait_freeze_timed_out: nvme_stop_queues(&ctrl->ctrl); nvme_sync_io_queues(&ctrl->ctrl); nvme_rdma_stop_io_queues(ctrl); -out_cleanup_connect_q: +out_cleanup_tagset: nvme_cancel_tagset(&ctrl->ctrl); if (new) - blk_mq_destroy_queue(ctrl->ctrl.connect_q); -out_free_tag_set: - if (new) - blk_mq_free_tag_set(ctrl->ctrl.tagset); + nvme_remove_io_tag_set(&ctrl->ctrl); out_free_io_queues: nvme_rdma_free_io_queues(ctrl); return ret; @@ -1025,9 +968,11 @@ static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl, blk_sync_queue(ctrl->ctrl.admin_q); nvme_rdma_stop_queue(&ctrl->queues[0]); nvme_cancel_admin_tagset(&ctrl->ctrl); - if (remove) + if (remove) { nvme_start_admin_queue(&ctrl->ctrl); - nvme_rdma_destroy_admin_queue(ctrl, remove); + nvme_remove_admin_tag_set(&ctrl->ctrl); + } + nvme_rdma_destroy_admin_queue(ctrl); } static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl, @@ -1039,9 +984,11 @@ static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl, nvme_sync_io_queues(&ctrl->ctrl); nvme_rdma_stop_io_queues(ctrl); nvme_cancel_tagset(&ctrl->ctrl); - if (remove) + if (remove) { nvme_start_queues(&ctrl->ctrl); - nvme_rdma_destroy_io_queues(ctrl, remove); + nvme_remove_io_tag_set(&ctrl->ctrl); + } + nvme_rdma_free_io_queues(ctrl); } } @@ -1049,7 +996,7 @@ static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl) { struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl); - cancel_work_sync(&ctrl->err_work); + flush_work(&ctrl->err_work); cancel_delayed_work_sync(&ctrl->reconnect_work); } @@ -1163,14 +1110,18 @@ destroy_io: nvme_sync_io_queues(&ctrl->ctrl); nvme_rdma_stop_io_queues(ctrl); nvme_cancel_tagset(&ctrl->ctrl); - nvme_rdma_destroy_io_queues(ctrl, new); + if (new) + nvme_remove_io_tag_set(&ctrl->ctrl); + nvme_rdma_free_io_queues(ctrl); } destroy_admin: nvme_stop_admin_queue(&ctrl->ctrl); blk_sync_queue(ctrl->ctrl.admin_q); nvme_rdma_stop_queue(&ctrl->queues[0]); nvme_cancel_admin_tagset(&ctrl->ctrl); - nvme_rdma_destroy_admin_queue(ctrl, new); + if (new) + nvme_remove_admin_tag_set(&ctrl->ctrl); + nvme_rdma_destroy_admin_queue(ctrl); return ret; } @@ -2188,9 +2139,9 @@ static void nvme_rdma_complete_rq(struct request *rq) nvme_complete_rq(rq); } -static int nvme_rdma_map_queues(struct blk_mq_tag_set *set) +static void nvme_rdma_map_queues(struct blk_mq_tag_set *set) { - struct nvme_rdma_ctrl *ctrl = set->driver_data; + struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(set->driver_data); struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { @@ -2231,8 +2182,6 @@ static int nvme_rdma_map_queues(struct blk_mq_tag_set *set) ctrl->io_queues[HCTX_TYPE_DEFAULT], ctrl->io_queues[HCTX_TYPE_READ], ctrl->io_queues[HCTX_TYPE_POLL]); - - return 0; } static const struct blk_mq_ops nvme_rdma_mq_ops = { diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 044da18c06f51249bb917ccb32aae23b2c87d1c8..1eed0fc26b3aee054616c0fc92ddd2eb680262a8 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -121,7 +121,6 @@ struct nvme_tcp_queue { struct mutex send_mutex; struct llist_head req_list; struct list_head send_list; - bool more_requests; /* recv state */ void *pdu; @@ -134,7 +133,6 @@ struct nvme_tcp_queue { /* send state */ struct nvme_tcp_request *request; - int queue_size; u32 maxh2cdata; size_t cmnd_capsule_len; struct nvme_tcp_ctrl *ctrl; @@ -320,7 +318,7 @@ static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) { return !list_empty(&queue->send_list) || - !llist_empty(&queue->req_list) || queue->more_requests; + !llist_empty(&queue->req_list); } static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, @@ -339,9 +337,7 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, */ if (queue->io_cpu == raw_smp_processor_id() && sync && empty && mutex_trylock(&queue->send_mutex)) { - queue->more_requests = !last; nvme_tcp_send_all(queue); - queue->more_requests = false; mutex_unlock(&queue->send_mutex); } @@ -466,7 +462,7 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) { - struct nvme_tcp_ctrl *ctrl = set->driver_data; + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); struct nvme_tcp_cmd_pdu *pdu; int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; @@ -490,7 +486,7 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_tcp_ctrl *ctrl = data; + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; hctx->driver_data = queue; @@ -500,7 +496,7 @@ static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_tcp_ctrl *ctrl = data; + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); struct nvme_tcp_queue *queue = &ctrl->queues[0]; hctx->driver_data = queue; @@ -1229,7 +1225,7 @@ static void nvme_tcp_io_work(struct work_struct *w) else if (unlikely(result < 0)) return; - if (!pending) + if (!pending || !queue->rd_enabled) return; } while (!time_after(jiffies, deadline)); /* quota is exhausted */ @@ -1479,8 +1475,7 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); } -static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, - int qid, size_t queue_size) +static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; @@ -1492,7 +1487,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, INIT_LIST_HEAD(&queue->send_list); mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); - queue->queue_size = queue_size; if (qid > 0) queue->cmnd_capsule_len = nctrl->ioccsz * 16; @@ -1690,51 +1684,6 @@ static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) return ret; } -static int nvme_tcp_alloc_admin_tag_set(struct nvme_ctrl *nctrl) -{ - struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); - struct blk_mq_tag_set *set = &ctrl->admin_tag_set; - int ret; - - memset(set, 0, sizeof(*set)); - set->ops = &nvme_tcp_admin_mq_ops; - set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; - set->reserved_tags = NVMF_RESERVED_TAGS; - set->numa_node = nctrl->numa_node; - set->flags = BLK_MQ_F_BLOCKING; - set->cmd_size = sizeof(struct nvme_tcp_request); - set->driver_data = ctrl; - set->nr_hw_queues = 1; - set->timeout = NVME_ADMIN_TIMEOUT; - ret = blk_mq_alloc_tag_set(set); - if (!ret) - nctrl->admin_tagset = set; - return ret; -} - -static int nvme_tcp_alloc_tag_set(struct nvme_ctrl *nctrl) -{ - struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); - struct blk_mq_tag_set *set = &ctrl->tag_set; - int ret; - - memset(set, 0, sizeof(*set)); - set->ops = &nvme_tcp_mq_ops; - set->queue_depth = nctrl->sqsize + 1; - set->reserved_tags = NVMF_RESERVED_TAGS; - set->numa_node = nctrl->numa_node; - set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; - set->cmd_size = sizeof(struct nvme_tcp_request); - set->driver_data = ctrl; - set->nr_hw_queues = nctrl->queue_count - 1; - set->timeout = NVME_IO_TIMEOUT; - set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; - ret = blk_mq_alloc_tag_set(set); - if (!ret) - nctrl->tagset = set; - return ret; -} - static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) { if (to_tcp_ctrl(ctrl)->async_req.pdu) { @@ -1762,11 +1711,12 @@ static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) nvme_tcp_stop_queue(ctrl, i); } -static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) +static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, + int first, int last) { int i, ret; - for (i = 1; i < ctrl->queue_count; i++) { + for (i = first; i < last; i++) { ret = nvme_tcp_start_queue(ctrl, i); if (ret) goto out_stop_queues; @@ -1775,7 +1725,7 @@ static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) return 0; out_stop_queues: - for (i--; i >= 1; i--) + for (i--; i >= first; i--) nvme_tcp_stop_queue(ctrl, i); return ret; } @@ -1784,7 +1734,7 @@ static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) { int ret; - ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); + ret = nvme_tcp_alloc_queue(ctrl, 0); if (ret) return ret; @@ -1804,7 +1754,7 @@ static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) int i, ret; for (i = 1; i < ctrl->queue_count; i++) { - ret = nvme_tcp_alloc_queue(ctrl, i, ctrl->sqsize + 1); + ret = nvme_tcp_alloc_queue(ctrl, i); if (ret) goto out_free_queues; } @@ -1892,32 +1842,35 @@ static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) { nvme_tcp_stop_io_queues(ctrl); - if (remove) { - blk_mq_destroy_queue(ctrl->connect_q); - blk_mq_free_tag_set(ctrl->tagset); - } + if (remove) + nvme_remove_io_tag_set(ctrl); nvme_tcp_free_io_queues(ctrl); } static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) { - int ret; + int ret, nr_queues; ret = nvme_tcp_alloc_io_queues(ctrl); if (ret) return ret; if (new) { - ret = nvme_tcp_alloc_tag_set(ctrl); + ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set, + &nvme_tcp_mq_ops, + BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING, + sizeof(struct nvme_tcp_request)); if (ret) goto out_free_io_queues; - - ret = nvme_ctrl_init_connect_q(ctrl); - if (ret) - goto out_free_tag_set; } - ret = nvme_tcp_start_io_queues(ctrl); + /* + * Only start IO queues for which we have allocated the tagset + * and limitted it to the available queues. On reconnects, the + * queue number might have changed. + */ + nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); + ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues); if (ret) goto out_cleanup_connect_q; @@ -1937,6 +1890,15 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) nvme_unfreeze(ctrl); } + /* + * If the number of queues has increased (reconnect case) + * start all new queues now. + */ + ret = nvme_tcp_start_io_queues(ctrl, nr_queues, + ctrl->tagset->nr_hw_queues + 1); + if (ret) + goto out_wait_freeze_timed_out; + return 0; out_wait_freeze_timed_out: @@ -1946,10 +1908,7 @@ out_wait_freeze_timed_out: out_cleanup_connect_q: nvme_cancel_tagset(ctrl); if (new) - blk_mq_destroy_queue(ctrl->connect_q); -out_free_tag_set: - if (new) - blk_mq_free_tag_set(ctrl->tagset); + nvme_remove_io_tag_set(ctrl); out_free_io_queues: nvme_tcp_free_io_queues(ctrl); return ret; @@ -1958,11 +1917,8 @@ out_free_io_queues: static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) { nvme_tcp_stop_queue(ctrl, 0); - if (remove) { - blk_mq_destroy_queue(ctrl->admin_q); - blk_mq_destroy_queue(ctrl->fabrics_q); - blk_mq_free_tag_set(ctrl->admin_tagset); - } + if (remove) + nvme_remove_admin_tag_set(ctrl); nvme_tcp_free_admin_queue(ctrl); } @@ -1975,26 +1931,17 @@ static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) return error; if (new) { - error = nvme_tcp_alloc_admin_tag_set(ctrl); + error = nvme_alloc_admin_tag_set(ctrl, + &to_tcp_ctrl(ctrl)->admin_tag_set, + &nvme_tcp_admin_mq_ops, BLK_MQ_F_BLOCKING, + sizeof(struct nvme_tcp_request)); if (error) goto out_free_queue; - - ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); - if (IS_ERR(ctrl->fabrics_q)) { - error = PTR_ERR(ctrl->fabrics_q); - goto out_free_tagset; - } - - ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); - if (IS_ERR(ctrl->admin_q)) { - error = PTR_ERR(ctrl->admin_q); - goto out_cleanup_fabrics_q; - } } error = nvme_tcp_start_queue(ctrl, 0); if (error) - goto out_cleanup_queue; + goto out_cleanup_tagset; error = nvme_enable_ctrl(ctrl); if (error) @@ -2014,15 +1961,9 @@ out_quiesce_queue: out_stop_queue: nvme_tcp_stop_queue(ctrl, 0); nvme_cancel_admin_tagset(ctrl); -out_cleanup_queue: - if (new) - blk_mq_destroy_queue(ctrl->admin_q); -out_cleanup_fabrics_q: +out_cleanup_tagset: if (new) - blk_mq_destroy_queue(ctrl->fabrics_q); -out_free_tagset: - if (new) - blk_mq_free_tag_set(ctrl->admin_tagset); + nvme_remove_admin_tag_set(ctrl); out_free_queue: nvme_tcp_free_admin_queue(ctrl); return error; @@ -2240,7 +2181,7 @@ out_fail: static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) { - cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); + flush_work(&to_tcp_ctrl(ctrl)->err_work); cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); } @@ -2471,9 +2412,9 @@ static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_OK; } -static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) +static void nvme_tcp_map_queues(struct blk_mq_tag_set *set) { - struct nvme_tcp_ctrl *ctrl = set->driver_data; + struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { @@ -2512,8 +2453,6 @@ static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) ctrl->io_queues[HCTX_TYPE_DEFAULT], ctrl->io_queues[HCTX_TYPE_READ], ctrl->io_queues[HCTX_TYPE_POLL]); - - return 0; } static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) @@ -2532,6 +2471,25 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) return queue->nr_cqe; } +static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) +{ + struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0]; + struct sockaddr_storage src_addr; + int ret, len; + + len = nvmf_get_address(ctrl, buf, size); + + ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr); + if (ret > 0) { + if (len > 0) + len--; /* strip trailing newline */ + len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n", + (len) ? "," : "", &src_addr); + } + + return len; +} + static const struct blk_mq_ops nvme_tcp_mq_ops = { .queue_rq = nvme_tcp_queue_rq, .commit_rqs = nvme_tcp_commit_rqs, @@ -2563,7 +2521,7 @@ static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { .free_ctrl = nvme_tcp_free_ctrl, .submit_async_event = nvme_tcp_submit_async_event, .delete_ctrl = nvme_tcp_delete_ctrl, - .get_address = nvmf_get_address, + .get_address = nvme_tcp_get_address, .stop_ctrl = nvme_tcp_stop_ctrl, }; diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index fc8a957fad0ac06ad3a4a6ea1a1b167a0d4662b7..c8a061ce3ee566f95f7665ba6ad03d56edb58c8b 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -449,7 +449,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) if (req->port->inline_data_size) id->sgls |= cpu_to_le32(1 << 20); - strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); + strscpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); /* * Max command capsule size is sqe + in-capsule data size. diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index cf690df347758d00710379dbf998318b0ec67358..c4113b43dbfeebabec926333d485954191e993c6 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -196,6 +196,7 @@ int nvmet_setup_auth(struct nvmet_ctrl *ctrl) if (IS_ERR(ctrl->ctrl_key)) { ret = PTR_ERR(ctrl->ctrl_key); ctrl->ctrl_key = NULL; + goto out_free_hash; } pr_debug("%s: using ctrl hash %s key %*ph\n", __func__, ctrl->ctrl_key->hash > 0 ? diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 2bcd60758919b433fa3eb14c5fb24df101965116..e34a2896fedb294d8e2269414ed5d0dae8eaafc8 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1281,6 +1281,34 @@ static ssize_t nvmet_subsys_attr_pi_enable_store(struct config_item *item, CONFIGFS_ATTR(nvmet_subsys_, attr_pi_enable); #endif +static ssize_t nvmet_subsys_attr_qid_max_show(struct config_item *item, + char *page) +{ + return snprintf(page, PAGE_SIZE, "%u\n", to_subsys(item)->max_qid); +} + +static ssize_t nvmet_subsys_attr_qid_max_store(struct config_item *item, + const char *page, size_t cnt) +{ + struct nvmet_port *port = to_nvmet_port(item); + u16 qid_max; + + if (nvmet_is_port_enabled(port, __func__)) + return -EACCES; + + if (sscanf(page, "%hu\n", &qid_max) != 1) + return -EINVAL; + + if (qid_max < 1 || qid_max > NVMET_NR_QUEUES) + return -EINVAL; + + down_write(&nvmet_config_sem); + to_subsys(item)->max_qid = qid_max; + up_write(&nvmet_config_sem); + return cnt; +} +CONFIGFS_ATTR(nvmet_subsys_, attr_qid_max); + static struct configfs_attribute *nvmet_subsys_attrs[] = { &nvmet_subsys_attr_attr_allow_any_host, &nvmet_subsys_attr_attr_version, @@ -1288,6 +1316,7 @@ static struct configfs_attribute *nvmet_subsys_attrs[] = { &nvmet_subsys_attr_attr_cntlid_min, &nvmet_subsys_attr_attr_cntlid_max, &nvmet_subsys_attr_attr_model, + &nvmet_subsys_attr_attr_qid_max, #ifdef CONFIG_BLK_DEV_INTEGRITY &nvmet_subsys_attr_attr_pi_enable, #endif diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a1345790005f428ce84a718469178157975736cc..14677145bbba098772d1338fc871d258acfbdb9e 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -735,6 +735,8 @@ static void nvmet_set_error(struct nvmet_req *req, u16 status) static void __nvmet_req_complete(struct nvmet_req *req, u16 status) { + struct nvmet_ns *ns = req->ns; + if (!req->sq->sqhd_disabled) nvmet_update_sq_head(req); req->cqe->sq_id = cpu_to_le16(req->sq->qid); @@ -745,9 +747,9 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status) trace_nvmet_req_complete(req); - if (req->ns) - nvmet_put_namespace(req->ns); req->ops->queue_response(req); + if (ns) + nvmet_put_namespace(ns); } void nvmet_req_complete(struct nvmet_req *req, u16 status) @@ -830,6 +832,7 @@ int nvmet_sq_init(struct nvmet_sq *sq) } init_completion(&sq->free_done); init_completion(&sq->confirm_done); + nvmet_auth_sq_init(sq); return 0; } diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index c2162eef8ce1a204bc9f8e993d8cafef86350085..668d257fa98636dc1785e7b5f6bb6b35e8188ab9 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -292,7 +292,7 @@ static void nvmet_execute_disc_identify(struct nvmet_req *req) id->oaes = cpu_to_le32(NVMET_DISC_AEN_CFG_OPTIONAL); - strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); + strscpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); status = nvmet_copy_to_sgl(req, 0, id, sizeof(*id)); diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index ebdf9aa8104199c8b296ba302af9115c31b22a73..7970a7640e5851a045d1e27706e76491e81339ad 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -23,17 +23,12 @@ static void nvmet_auth_expired_work(struct work_struct *work) sq->dhchap_tid = -1; } -void nvmet_init_auth(struct nvmet_ctrl *ctrl, struct nvmet_req *req) +void nvmet_auth_sq_init(struct nvmet_sq *sq) { - u32 result = le32_to_cpu(req->cqe->result.u32); - /* Initialize in-band authentication */ - INIT_DELAYED_WORK(&req->sq->auth_expired_work, - nvmet_auth_expired_work); - req->sq->authenticated = false; - req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; - result |= (u32)NVME_CONNECT_AUTHREQ_ATR << 16; - req->cqe->result.u32 = cpu_to_le32(result); + INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work); + sq->authenticated = false; + sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } static u16 nvmet_auth_negotiate(struct nvmet_req *req, void *d) @@ -177,7 +172,7 @@ static u16 nvmet_auth_reply(struct nvmet_req *req, void *d) return 0; } -static u16 nvmet_auth_failure2(struct nvmet_req *req, void *d) +static u16 nvmet_auth_failure2(void *d) { struct nvmf_auth_dhchap_failure_data *data = d; @@ -229,10 +224,8 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } status = nvmet_copy_from_sgl(req, 0, d, tl); - if (status) { - kfree(d); - goto done; - } + if (status) + goto done_kfree; data = d; pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__, @@ -310,7 +303,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) goto done_kfree; break; case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2: - status = nvmet_auth_failure2(req, d); + status = nvmet_auth_failure2(d); if (status) { pr_warn("ctrl %d qid %d: authentication failed (%d)\n", ctrl->cntlid, req->sq->qid, status); diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index f91a56180d3dd68fd7986c938f03aab5724298b0..43b5bd8bb6a52dc807a62cc29b2d17eb194c0ed9 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -198,6 +198,12 @@ err: return ret; } +static u32 nvmet_connect_result(struct nvmet_ctrl *ctrl) +{ + return (u32)ctrl->cntlid | + (nvmet_has_auth(ctrl) ? NVME_CONNECT_AUTHREQ_ATR : 0); +} + static void nvmet_execute_admin_connect(struct nvmet_req *req) { struct nvmf_connect_command *c = &req->cmd->connect; @@ -269,10 +275,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) ctrl->cntlid, ctrl->subsys->subsysnqn, ctrl->hostnqn, ctrl->pi_support ? " T10-PI is enabled" : "", nvmet_has_auth(ctrl) ? " with DH-HMAC-CHAP" : ""); - req->cqe->result.u16 = cpu_to_le16(ctrl->cntlid); - - if (nvmet_has_auth(ctrl)) - nvmet_init_auth(ctrl, req); + req->cqe->result.u32 = cpu_to_le32(nvmet_connect_result(ctrl)); out: kfree(d); complete: @@ -328,14 +331,8 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) if (status) goto out_ctrl_put; - /* pass back cntlid for successful completion */ - req->cqe->result.u16 = cpu_to_le16(ctrl->cntlid); - pr_debug("adding queue %d to ctrl %d.\n", qid, ctrl->cntlid); - req->cqe->result.u16 = cpu_to_le16(ctrl->cntlid); - if (nvmet_has_auth(ctrl)) - nvmet_init_auth(ctrl, req); - + req->cqe->result.u32 = cpu_to_le32(nvmet_connect_result(ctrl)); out: kfree(d); complete: diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 2dc1c1035626b2330b799106527a3c323e7956f1..c2d6cea0236b0a70210a6f9b13f19b57da6c1214 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -12,11 +12,9 @@ void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id) { - const struct queue_limits *ql = &bdev_get_queue(bdev)->limits; - /* Number of logical blocks per physical block. */ - const u32 lpp = ql->physical_block_size / ql->logical_block_size; /* Logical blocks per physical block, 0's based. */ - const __le16 lpp0b = to0based(lpp); + const __le16 lpp0b = to0based(bdev_physical_block_size(bdev) / + bdev_logical_block_size(bdev)); /* * For NVMe 1.2 and later, bit 1 indicates that the fields NAWUN, @@ -42,11 +40,12 @@ void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id) /* NPWA = Namespace Preferred Write Alignment. 0's based */ id->npwa = id->npwg; /* NPDG = Namespace Preferred Deallocate Granularity. 0's based */ - id->npdg = to0based(ql->discard_granularity / ql->logical_block_size); + id->npdg = to0based(bdev_discard_granularity(bdev) / + bdev_logical_block_size(bdev)); /* NPDG = Namespace Preferred Deallocate Alignment */ id->npda = id->npdg; /* NOWS = Namespace Optimal Write Size */ - id->nows = to0based(ql->io_opt / ql->logical_block_size); + id->nows = to0based(bdev_io_opt(bdev) / bdev_logical_block_size(bdev)); } void nvmet_bdev_ns_disable(struct nvmet_ns *ns) @@ -334,6 +333,11 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) { struct bio *bio = &req->b.inline_bio; + if (!bdev_write_cache(req->ns->bdev)) { + nvmet_req_complete(req, NVME_SC_SUCCESS); + return; + } + if (!nvmet_check_transfer_len(req, 0)) return; @@ -347,6 +351,9 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) u16 nvmet_bdev_flush(struct nvmet_req *req) { + if (!bdev_write_cache(req->ns->bdev)) + return 0; + if (blkdev_issue_flush(req->ns->bdev)) return NVME_SC_INTERNAL | NVME_SC_DNR; return 0; diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index 9750a7fca26887cb4dc6339bd2d2ff52a42dd973..b45fe3adf015fd1327ee73a6452ad32feafbff81 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -204,7 +204,7 @@ static int nvme_loop_init_request(struct blk_mq_tag_set *set, struct request *req, unsigned int hctx_idx, unsigned int numa_node) { - struct nvme_loop_ctrl *ctrl = set->driver_data; + struct nvme_loop_ctrl *ctrl = to_loop_ctrl(set->driver_data); struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); nvme_req(req)->ctrl = &ctrl->ctrl; @@ -218,7 +218,7 @@ static struct lock_class_key loop_hctx_fq_lock_key; static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_loop_ctrl *ctrl = data; + struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data); struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); @@ -238,7 +238,7 @@ static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx) { - struct nvme_loop_ctrl *ctrl = data; + struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data); struct nvme_loop_queue *queue = &ctrl->queues[0]; BUG_ON(hctx_idx != 0); @@ -266,9 +266,7 @@ static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags)) return; nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); - blk_mq_destroy_queue(ctrl->ctrl.admin_q); - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); - blk_mq_free_tag_set(&ctrl->admin_tag_set); + nvme_remove_admin_tag_set(&ctrl->ctrl); } static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) @@ -282,10 +280,8 @@ static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) list_del(&ctrl->list); mutex_unlock(&nvme_loop_ctrl_mutex); - if (nctrl->tagset) { - blk_mq_destroy_queue(ctrl->ctrl.connect_q); - blk_mq_free_tag_set(&ctrl->tag_set); - } + if (nctrl->tagset) + nvme_remove_io_tag_set(nctrl); kfree(ctrl->queues); nvmf_free_options(nctrl->opts); free_ctrl: @@ -350,52 +346,31 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) { int error; - memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); - ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; - ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH; - ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS; - ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node; - ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + - NVME_INLINE_SG_CNT * sizeof(struct scatterlist); - ctrl->admin_tag_set.driver_data = ctrl; - ctrl->admin_tag_set.nr_hw_queues = 1; - ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT; - ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED; - ctrl->queues[0].ctrl = ctrl; error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); if (error) return error; ctrl->ctrl.queue_count = 1; - error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); + error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set, + &nvme_loop_admin_mq_ops, BLK_MQ_F_NO_SCHED, + sizeof(struct nvme_loop_iod) + + NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); if (error) goto out_free_sq; - ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set; - ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.fabrics_q)) { - error = PTR_ERR(ctrl->ctrl.fabrics_q); - goto out_free_tagset; - } - - ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); - if (IS_ERR(ctrl->ctrl.admin_q)) { - error = PTR_ERR(ctrl->ctrl.admin_q); - goto out_cleanup_fabrics_q; - } /* reset stopped state for the fresh admin queue */ clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags); error = nvmf_connect_admin_queue(&ctrl->ctrl); if (error) - goto out_cleanup_queue; + goto out_cleanup_tagset; set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); error = nvme_enable_ctrl(&ctrl->ctrl); if (error) - goto out_cleanup_queue; + goto out_cleanup_tagset; ctrl->ctrl.max_hw_sectors = (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); @@ -404,17 +379,13 @@ static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) error = nvme_init_ctrl_finish(&ctrl->ctrl); if (error) - goto out_cleanup_queue; + goto out_cleanup_tagset; return 0; -out_cleanup_queue: +out_cleanup_tagset: clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags); - blk_mq_destroy_queue(ctrl->ctrl.admin_q); -out_cleanup_fabrics_q: - blk_mq_destroy_queue(ctrl->ctrl.fabrics_q); -out_free_tagset: - blk_mq_free_tag_set(&ctrl->admin_tag_set); + nvme_remove_admin_tag_set(&ctrl->ctrl); out_free_sq: nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); return error; @@ -522,37 +493,21 @@ static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) if (ret) return ret; - memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set)); - ctrl->tag_set.ops = &nvme_loop_mq_ops; - ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size; - ctrl->tag_set.reserved_tags = NVMF_RESERVED_TAGS; - ctrl->tag_set.numa_node = ctrl->ctrl.numa_node; - ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; - ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) + - NVME_INLINE_SG_CNT * sizeof(struct scatterlist); - ctrl->tag_set.driver_data = ctrl; - ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1; - ctrl->tag_set.timeout = NVME_IO_TIMEOUT; - ctrl->ctrl.tagset = &ctrl->tag_set; - - ret = blk_mq_alloc_tag_set(&ctrl->tag_set); + ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set, + &nvme_loop_mq_ops, BLK_MQ_F_SHOULD_MERGE, + sizeof(struct nvme_loop_iod) + + NVME_INLINE_SG_CNT * sizeof(struct scatterlist)); if (ret) goto out_destroy_queues; - ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl)); - if (ret) - goto out_free_tagset; - ret = nvme_loop_connect_io_queues(ctrl); if (ret) - goto out_cleanup_connect_q; + goto out_cleanup_tagset; return 0; -out_cleanup_connect_q: - blk_mq_destroy_queue(ctrl->ctrl.connect_q); -out_free_tagset: - blk_mq_free_tag_set(&ctrl->tag_set); +out_cleanup_tagset: + nvme_remove_io_tag_set(&ctrl->ctrl); out_destroy_queues: nvme_loop_destroy_io_queues(ctrl); return ret; @@ -601,7 +556,6 @@ static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, ret = -ENOMEM; - ctrl->ctrl.sqsize = opts->queue_size - 1; ctrl->ctrl.kato = opts->kato; ctrl->port = nvme_loop_find_port(&ctrl->ctrl); @@ -621,6 +575,7 @@ static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, opts->queue_size, ctrl->ctrl.maxcmd); opts->queue_size = ctrl->ctrl.maxcmd; } + ctrl->ctrl.sqsize = opts->queue_size - 1; if (opts->nr_io_queues) { ret = nvme_loop_create_io_queues(ctrl); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 6ffeeb0a1c49e3feeee56cb708219a34b043981c..dfe3894205aa77f71e49617e32f824bc6dda976e 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -704,7 +704,7 @@ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret, bool set_ctrl); int nvmet_auth_set_host_hash(struct nvmet_host *host, const char *hash); int nvmet_setup_auth(struct nvmet_ctrl *ctrl); -void nvmet_init_auth(struct nvmet_ctrl *ctrl, struct nvmet_req *req); +void nvmet_auth_sq_init(struct nvmet_sq *sq); void nvmet_destroy_auth(struct nvmet_ctrl *ctrl); void nvmet_auth_sq_free(struct nvmet_sq *sq); int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id); @@ -726,8 +726,9 @@ static inline int nvmet_setup_auth(struct nvmet_ctrl *ctrl) { return 0; } -static inline void nvmet_init_auth(struct nvmet_ctrl *ctrl, - struct nvmet_req *req) {}; +static inline void nvmet_auth_sq_init(struct nvmet_sq *sq) +{ +} static inline void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) {}; static inline void nvmet_auth_sq_free(struct nvmet_sq *sq) {}; static inline bool nvmet_check_auth_status(struct nvmet_req *req) diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index 6f39a29828b1228463649e0523b129a022c0463f..79af5140af8bfe5cd10c0a3a7bc32fdaeae631ca 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -215,9 +215,11 @@ static void nvmet_passthru_execute_cmd_work(struct work_struct *w) { struct nvmet_req *req = container_of(w, struct nvmet_req, p.work); struct request *rq = req->p.rq; + struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl; + u32 effects; int status; - status = nvme_execute_passthru_rq(rq); + status = nvme_execute_passthru_rq(rq, &effects); if (status == NVME_SC_SUCCESS && req->cmd->common.opcode == nvme_admin_identify) { @@ -238,16 +240,20 @@ static void nvmet_passthru_execute_cmd_work(struct work_struct *w) req->cqe->result = nvme_req(rq)->result; nvmet_req_complete(req, status); blk_mq_free_request(rq); + + if (effects) + nvme_passthru_end(ctrl, effects, req->cmd, status); } -static void nvmet_passthru_req_done(struct request *rq, - blk_status_t blk_status) +static enum rq_end_io_ret nvmet_passthru_req_done(struct request *rq, + blk_status_t blk_status) { struct nvmet_req *req = rq->end_io_data; req->cqe->result = nvme_req(rq)->result; nvmet_req_complete(req, nvme_req(rq)->status); blk_mq_free_request(rq); + return RQ_END_IO_NONE; } static int nvmet_passthru_map_sg(struct nvmet_req *req, struct request *rq) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index dc3b4dc8fe08b9d04e44a2aadf77d547a7520cc5..6c1476e086ef4cc5d0c10c64e82285c11bebc124 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -77,9 +77,8 @@ struct nvmet_tcp_cmd { u32 pdu_len; u32 pdu_recv; int sg_idx; - int nr_mapped; struct msghdr recv_msg; - struct kvec *iov; + struct bio_vec *iov; u32 flags; struct list_head entry; @@ -165,9 +164,7 @@ static DEFINE_MUTEX(nvmet_tcp_queue_mutex); static struct workqueue_struct *nvmet_tcp_wq; static const struct nvmet_fabrics_ops nvmet_tcp_ops; static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c); -static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd); static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd); -static void nvmet_tcp_unmap_pdu_iovec(struct nvmet_tcp_cmd *cmd); static inline u16 nvmet_tcp_cmd_tag(struct nvmet_tcp_queue *queue, struct nvmet_tcp_cmd *cmd) @@ -301,35 +298,21 @@ static int nvmet_tcp_check_ddgst(struct nvmet_tcp_queue *queue, void *pdu) static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd) { - WARN_ON(unlikely(cmd->nr_mapped > 0)); - kfree(cmd->iov); sgl_free(cmd->req.sg); cmd->iov = NULL; cmd->req.sg = NULL; } -static void nvmet_tcp_unmap_pdu_iovec(struct nvmet_tcp_cmd *cmd) -{ - struct scatterlist *sg; - int i; - - sg = &cmd->req.sg[cmd->sg_idx]; - - for (i = 0; i < cmd->nr_mapped; i++) - kunmap(sg_page(&sg[i])); - - cmd->nr_mapped = 0; -} - -static void nvmet_tcp_map_pdu_iovec(struct nvmet_tcp_cmd *cmd) +static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) { - struct kvec *iov = cmd->iov; + struct bio_vec *iov = cmd->iov; struct scatterlist *sg; u32 length, offset, sg_offset; + int nr_pages; length = cmd->pdu_len; - cmd->nr_mapped = DIV_ROUND_UP(length, PAGE_SIZE); + nr_pages = DIV_ROUND_UP(length, PAGE_SIZE); offset = cmd->rbytes_done; cmd->sg_idx = offset / PAGE_SIZE; sg_offset = offset % PAGE_SIZE; @@ -338,8 +321,9 @@ static void nvmet_tcp_map_pdu_iovec(struct nvmet_tcp_cmd *cmd) while (length) { u32 iov_len = min_t(u32, length, sg->length - sg_offset); - iov->iov_base = kmap(sg_page(sg)) + sg->offset + sg_offset; - iov->iov_len = iov_len; + iov->bv_page = sg_page(sg); + iov->bv_len = sg->length; + iov->bv_offset = sg->offset + sg_offset; length -= iov_len; sg = sg_next(sg); @@ -347,8 +331,8 @@ static void nvmet_tcp_map_pdu_iovec(struct nvmet_tcp_cmd *cmd) sg_offset = 0; } - iov_iter_kvec(&cmd->recv_msg.msg_iter, READ, cmd->iov, - cmd->nr_mapped, cmd->pdu_len); + iov_iter_bvec(&cmd->recv_msg.msg_iter, READ, cmd->iov, + nr_pages, cmd->pdu_len); } static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue) @@ -926,7 +910,7 @@ static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, } queue->rcv_state = NVMET_TCP_RECV_DATA; - nvmet_tcp_map_pdu_iovec(cmd); + nvmet_tcp_build_pdu_iovec(cmd); cmd->flags |= NVMET_TCP_F_INIT_FAILED; } @@ -935,10 +919,17 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) struct nvme_tcp_data_pdu *data = &queue->pdu.data; struct nvmet_tcp_cmd *cmd; - if (likely(queue->nr_cmds)) + if (likely(queue->nr_cmds)) { + if (unlikely(data->ttag >= queue->nr_cmds)) { + pr_err("queue %d: received out of bound ttag %u, nr_cmds %u\n", + queue->idx, data->ttag, queue->nr_cmds); + nvmet_tcp_fatal_error(queue); + return -EPROTO; + } cmd = &queue->cmds[data->ttag]; - else + } else { cmd = &queue->connect; + } if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) { pr_err("ttag %u unexpected data offset %u (expected %u)\n", @@ -952,7 +943,7 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) cmd->pdu_len = le32_to_cpu(data->data_length); cmd->pdu_recv = 0; - nvmet_tcp_map_pdu_iovec(cmd); + nvmet_tcp_build_pdu_iovec(cmd); queue->cmd = cmd; queue->rcv_state = NVMET_TCP_RECV_DATA; @@ -976,6 +967,13 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) return nvmet_tcp_handle_icreq(queue); } + if (unlikely(hdr->type == nvme_tcp_icreq)) { + pr_err("queue %d: received icreq pdu in state %d\n", + queue->idx, queue->state); + nvmet_tcp_fatal_error(queue); + return -EPROTO; + } + if (hdr->type == nvme_tcp_h2c_data) { ret = nvmet_tcp_handle_h2c_data_pdu(queue); if (unlikely(ret)) @@ -1021,7 +1019,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) if (nvmet_tcp_need_data_in(queue->cmd)) { if (nvmet_tcp_has_inline_data(queue->cmd)) { queue->rcv_state = NVMET_TCP_RECV_DATA; - nvmet_tcp_map_pdu_iovec(queue->cmd); + nvmet_tcp_build_pdu_iovec(queue->cmd); return 0; } /* send back R2T */ @@ -1141,7 +1139,6 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) cmd->rbytes_done += ret; } - nvmet_tcp_unmap_pdu_iovec(cmd); if (queue->data_digest) { nvmet_tcp_prep_recv_ddgst(cmd); return 0; @@ -1179,7 +1176,8 @@ static int nvmet_tcp_try_recv_ddgst(struct nvmet_tcp_queue *queue) queue->idx, cmd->req.cmd->common.command_id, queue->pdu.cmd.hdr.type, le32_to_cpu(cmd->recv_ddgst), le32_to_cpu(cmd->exp_ddgst)); - nvmet_tcp_finish_cmd(cmd); + nvmet_req_uninit(&cmd->req); + nvmet_tcp_free_cmd_buffers(cmd); nvmet_tcp_fatal_error(queue); ret = -EPROTO; goto out; @@ -1408,13 +1406,6 @@ static void nvmet_tcp_restore_socket_callbacks(struct nvmet_tcp_queue *queue) write_unlock_bh(&sock->sk->sk_callback_lock); } -static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd) -{ - nvmet_req_uninit(&cmd->req); - nvmet_tcp_unmap_pdu_iovec(cmd); - nvmet_tcp_free_cmd_buffers(cmd); -} - static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue) { struct nvmet_tcp_cmd *cmd = queue->cmds; @@ -1423,17 +1414,28 @@ static void nvmet_tcp_uninit_data_in_cmds(struct nvmet_tcp_queue *queue) for (i = 0; i < queue->nr_cmds; i++, cmd++) { if (nvmet_tcp_need_data_in(cmd)) nvmet_req_uninit(&cmd->req); - - nvmet_tcp_unmap_pdu_iovec(cmd); - nvmet_tcp_free_cmd_buffers(cmd); } if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) { /* failed in connect */ - nvmet_tcp_finish_cmd(&queue->connect); + nvmet_req_uninit(&queue->connect.req); } } +static void nvmet_tcp_free_cmd_data_in_buffers(struct nvmet_tcp_queue *queue) +{ + struct nvmet_tcp_cmd *cmd = queue->cmds; + int i; + + for (i = 0; i < queue->nr_cmds; i++, cmd++) { + if (nvmet_tcp_need_data_in(cmd)) + nvmet_tcp_free_cmd_buffers(cmd); + } + + if (!queue->nr_cmds && nvmet_tcp_need_data_in(&queue->connect)) + nvmet_tcp_free_cmd_buffers(&queue->connect); +} + static void nvmet_tcp_release_queue_work(struct work_struct *w) { struct page *page; @@ -1452,6 +1454,7 @@ static void nvmet_tcp_release_queue_work(struct work_struct *w) nvmet_tcp_uninit_data_in_cmds(queue); nvmet_sq_destroy(&queue->nvme_sq); cancel_work_sync(&queue->io_work); + nvmet_tcp_free_cmd_data_in_buffers(queue); sock_release(queue->sock); nvmet_tcp_free_cmds(queue); if (queue->hdr_digest || queue->data_digest) @@ -1506,6 +1509,9 @@ static void nvmet_tcp_state_change(struct sock *sk) goto done; switch (sk->sk_state) { + case TCP_FIN_WAIT2: + case TCP_LAST_ACK: + break; case TCP_FIN_WAIT1: case TCP_CLOSE_WAIT: case TCP_CLOSE: diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index c7ef69f29fe4e70cd4102f5b503112c2c801ade1..1254cf57e008ddbb0da7988defba901d84c85e77 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -100,6 +100,7 @@ void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req) struct nvme_id_ns_zns *id_zns; u64 zsze; u16 status; + u32 mar, mor; if (le32_to_cpu(req->cmd->identify.nsid) == NVME_NSID_ALL) { req->error_loc = offsetof(struct nvme_identify, nsid); @@ -130,8 +131,20 @@ void nvmet_execute_identify_cns_cs_ns(struct nvmet_req *req) zsze = (bdev_zone_sectors(req->ns->bdev) << 9) >> req->ns->blksize_shift; id_zns->lbafe[0].zsze = cpu_to_le64(zsze); - id_zns->mor = cpu_to_le32(bdev_max_open_zones(req->ns->bdev)); - id_zns->mar = cpu_to_le32(bdev_max_active_zones(req->ns->bdev)); + + mor = bdev_max_open_zones(req->ns->bdev); + if (!mor) + mor = U32_MAX; + else + mor--; + id_zns->mor = cpu_to_le32(mor); + + mar = bdev_max_active_zones(req->ns->bdev); + if (!mar) + mar = U32_MAX; + else + mar--; + id_zns->mar = cpu_to_le32(mar); done: status = nvmet_copy_to_sgl(req, 0, id_zns, sizeof(*id_zns)); @@ -387,7 +400,6 @@ static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) { struct block_device *bdev = req->ns->bdev; unsigned int nr_zones = bdev_nr_zones(bdev); - struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = NULL; sector_t sector = 0; int ret; @@ -396,7 +408,7 @@ static u16 nvmet_bdev_zone_mgmt_emulate_all(struct nvmet_req *req) }; d.zbitmap = kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(*(d.zbitmap)), - GFP_NOIO, q->node); + GFP_NOIO, bdev->bd_disk->node_id); if (!d.zbitmap) { ret = -ENOMEM; goto out; diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig index d72d879a6d342f3a1fda5c4a251348b903777d40..ec8a49c040031205dbe69a2b4f7ae9a6a639f63d 100644 --- a/drivers/nvmem/Kconfig +++ b/drivers/nvmem/Kconfig @@ -21,6 +21,40 @@ config NVMEM_SYSFS This interface is mostly used by userspace applications to read/write directly into nvmem. +# Devices + +config NVMEM_APPLE_EFUSES + tristate "Apple eFuse support" + depends on ARCH_APPLE || COMPILE_TEST + default ARCH_APPLE + help + Say y here to enable support for reading eFuses on Apple SoCs + such as the M1. These are e.g. used to store factory programmed + calibration data required for the PCIe or the USB-C PHY. + + This driver can also be built as a module. If so, the module will + be called nvmem-apple-efuses. + +config NVMEM_BCM_OCOTP + tristate "Broadcom On-Chip OTP Controller support" + depends on ARCH_BCM_IPROC || COMPILE_TEST + depends on HAS_IOMEM + default ARCH_BCM_IPROC + help + Say y here to enable read/write access to the Broadcom OTP + controller. + + This driver can also be built as a module. If so, the module + will be called nvmem-bcm-ocotp. + +config NVMEM_BRCM_NVRAM + tristate "Broadcom's NVRAM support" + depends on ARCH_BCM_5301X || COMPILE_TEST + depends on HAS_IOMEM + help + This driver provides support for Broadcom's NVRAM that can be accessed + using I/O mapping. + config NVMEM_IMX_IIM tristate "i.MX IC Identification Module support" depends on ARCH_MXC || COMPILE_TEST @@ -52,7 +86,7 @@ config NVMEM_IMX_OCOTP_SCU This is a driver for the SCU On-Chip OTP Controller (OCOTP) available on i.MX8 SoCs. -config JZ4780_EFUSE +config NVMEM_JZ4780_EFUSE tristate "JZ4780 EFUSE Memory Support" depends on MACH_INGENIC || COMPILE_TEST depends on HAS_IOMEM @@ -64,6 +98,27 @@ config JZ4780_EFUSE To compile this driver as a module, choose M here: the module will be called nvmem_jz4780_efuse. +config NVMEM_LAN9662_OTPC + tristate "Microchip LAN9662 OTP controller support" + depends on SOC_LAN966 || COMPILE_TEST + depends on HAS_IOMEM + help + This driver enables the OTP controller available on Microchip LAN9662 + SoCs. It controls the access to the OTP memory connected to it. + +config NVMEM_LAYERSCAPE_SFP + tristate "Layerscape SFP (Security Fuse Processor) support" + depends on ARCH_LAYERSCAPE || COMPILE_TEST + depends on HAS_IOMEM + select REGMAP_MMIO + help + This driver provides support to read the eFuses on Freescale + Layerscape SoC's. For example, the vendor provides a per part + unique ID there. + + This driver can also be built as a module. If so, the module + will be called layerscape-sfp. + config NVMEM_LPC18XX_EEPROM tristate "NXP LPC18XX EEPROM Memory Support" depends on ARCH_LPC18XX || COMPILE_TEST @@ -84,19 +139,34 @@ config NVMEM_LPC18XX_OTP To compile this driver as a module, choose M here: the module will be called nvmem_lpc18xx_otp. -config NVMEM_MXS_OCOTP - tristate "Freescale MXS On-Chip OTP Memory Support" - depends on ARCH_MXS || COMPILE_TEST - depends on HAS_IOMEM +config NVMEM_MESON_EFUSE + tristate "Amlogic Meson GX eFuse Support" + depends on (ARCH_MESON || COMPILE_TEST) && MESON_SM help - If you say Y here, you will get readonly access to the - One Time Programmable memory pages that are stored - on the Freescale i.MX23/i.MX28 processor. + This is a driver to retrieve specific values from the eFuse found on + the Amlogic Meson GX SoCs. This driver can also be built as a module. If so, the module - will be called nvmem-mxs-ocotp. + will be called nvmem_meson_efuse. + +config NVMEM_MESON_MX_EFUSE + tristate "Amlogic Meson6/Meson8/Meson8b eFuse Support" + depends on ARCH_MESON || COMPILE_TEST + help + This is a driver to retrieve specific values from the eFuse found on + the Amlogic Meson6, Meson8 and Meson8b SoCs. -config MTK_EFUSE + This driver can also be built as a module. If so, the module + will be called nvmem_meson_mx_efuse. + +config NVMEM_MICROCHIP_OTPC + tristate "Microchip OTPC support" + depends on ARCH_AT91 || COMPILE_TEST + help + This driver enable the OTP controller available on Microchip SAMA7G5 + SoCs. It controlls the access to the OTP memory connected to it. + +config NVMEM_MTK_EFUSE tristate "Mediatek SoCs EFUSE support" depends on ARCH_MEDIATEK || COMPILE_TEST depends on HAS_IOMEM @@ -107,12 +177,17 @@ config MTK_EFUSE This driver can also be built as a module. If so, the module will be called efuse-mtk. -config MICROCHIP_OTPC - tristate "Microchip OTPC support" - depends on ARCH_AT91 || COMPILE_TEST +config NVMEM_MXS_OCOTP + tristate "Freescale MXS On-Chip OTP Memory Support" + depends on ARCH_MXS || COMPILE_TEST + depends on HAS_IOMEM help - This driver enable the OTP controller available on Microchip SAMA7G5 - SoCs. It controlls the access to the OTP memory connected to it. + If you say Y here, you will get readonly access to the + One Time Programmable memory pages that are stored + on the Freescale i.MX23/i.MX28 processor. + + This driver can also be built as a module. If so, the module + will be called nvmem-mxs-ocotp. config NVMEM_NINTENDO_OTP tristate "Nintendo Wii and Wii U OTP Support" @@ -126,7 +201,7 @@ config NVMEM_NINTENDO_OTP This driver can also be built as a module. If so, the module will be called nvmem-nintendo-otp. -config QCOM_QFPROM +config NVMEM_QCOM_QFPROM tristate "QCOM QFPROM Support" depends on ARCH_QCOM || COMPILE_TEST depends on HAS_IOMEM @@ -137,15 +212,23 @@ config QCOM_QFPROM This driver can also be built as a module. If so, the module will be called nvmem_qfprom. -config NVMEM_SPMI_SDAM - tristate "SPMI SDAM Support" - depends on SPMI +config NVMEM_RAVE_SP_EEPROM + tristate "Rave SP EEPROM Support" + depends on RAVE_SP_CORE help - This driver supports the Shared Direct Access Memory Module on - Qualcomm Technologies, Inc. PMICs. It provides the clients - an interface to read/write to the SDAM module's shared memory. + Say y here to enable Rave SP EEPROM support. + +config NVMEM_RMEM + tristate "Reserved Memory Based Driver Support" + depends on HAS_IOMEM + help + This driver maps reserved memory into an nvmem device. It might be + useful to expose information left by firmware in memory. -config ROCKCHIP_EFUSE + This driver can also be built as a module. If so, the module + will be called nvmem-rmem. + +config NVMEM_ROCKCHIP_EFUSE tristate "Rockchip eFuse Support" depends on ARCH_ROCKCHIP || COMPILE_TEST depends on HAS_IOMEM @@ -156,7 +239,7 @@ config ROCKCHIP_EFUSE This driver can also be built as a module. If so, the module will be called nvmem_rockchip_efuse. -config ROCKCHIP_OTP +config NVMEM_ROCKCHIP_OTP tristate "Rockchip OTP controller support" depends on ARCH_ROCKCHIP || COMPILE_TEST depends on HAS_IOMEM @@ -167,17 +250,45 @@ config ROCKCHIP_OTP This driver can also be built as a module. If so, the module will be called nvmem_rockchip_otp. -config NVMEM_BCM_OCOTP - tristate "Broadcom On-Chip OTP Controller support" - depends on ARCH_BCM_IPROC || COMPILE_TEST +config NVMEM_SC27XX_EFUSE + tristate "Spreadtrum SC27XX eFuse Support" + depends on MFD_SC27XX_PMIC || COMPILE_TEST depends on HAS_IOMEM - default ARCH_BCM_IPROC help - Say y here to enable read/write access to the Broadcom OTP - controller. + This is a simple driver to dump specified values of Spreadtrum + SC27XX PMICs from eFuse. This driver can also be built as a module. If so, the module - will be called nvmem-bcm-ocotp. + will be called nvmem-sc27xx-efuse. + +config NVMEM_SNVS_LPGPR + tristate "Support for Low Power General Purpose Register" + depends on ARCH_MXC || COMPILE_TEST + help + This is a driver for Low Power General Purpose Register (LPGPR) available on + i.MX6 and i.MX7 SoCs in Secure Non-Volatile Storage (SNVS) of this chip. + + This driver can also be built as a module. If so, the module + will be called nvmem-snvs-lpgpr. + +config NVMEM_SPMI_SDAM + tristate "SPMI SDAM Support" + depends on SPMI + help + This driver supports the Shared Direct Access Memory Module on + Qualcomm Technologies, Inc. PMICs. It provides the clients + an interface to read/write to the SDAM module's shared memory. + +config NVMEM_SPRD_EFUSE + tristate "Spreadtrum SoC eFuse Support" + depends on ARCH_SPRD || COMPILE_TEST + depends on HAS_IOMEM + help + This is a simple driver to dump specified values of Spreadtrum + SoCs from eFuse. + + This driver can also be built as a module. If so, the module + will be called nvmem-sprd-efuse. config NVMEM_STM32_ROMEM tristate "STMicroelectronics STM32 factory-programmed memory support" @@ -189,6 +300,18 @@ config NVMEM_STM32_ROMEM This driver can also be built as a module. If so, the module will be called nvmem-stm32-romem. +config NVMEM_SUNPLUS_OCOTP + tristate "Sunplus SoC OTP support" + depends on SOC_SP7021 || COMPILE_TEST + depends on HAS_IOMEM + help + This is a driver for the On-chip OTP controller (OCOTP) available + on Sunplus SoCs. It provides access to 128 bytes of one-time + programmable eFuse. + + This driver can also be built as a module. If so, the module + will be called nvmem-sunplus-ocotp. + config NVMEM_SUNXI_SID tristate "Allwinner SoCs SID support" depends on ARCH_SUNXI @@ -199,7 +322,20 @@ config NVMEM_SUNXI_SID This driver can also be built as a module. If so, the module will be called nvmem_sunxi_sid. -config UNIPHIER_EFUSE +config NVMEM_U_BOOT_ENV + tristate "U-Boot environment variables support" + depends on OF && MTD + select CRC32 + help + U-Boot stores its setup as environment variables. This driver adds + support for verifying & exporting such data. It also exposes variables + as NVMEM cells so they can be referenced by other drivers. + + Currently this drivers works only with env variables on top of MTD. + + If compiled as module it will be called nvmem_u-boot-env. + +config NVMEM_UNIPHIER_EFUSE tristate "UniPhier SoCs eFuse support" depends on ARCH_UNIPHIER || COMPILE_TEST depends on HAS_IOMEM @@ -221,53 +357,6 @@ config NVMEM_VF610_OCOTP This driver can also be build as a module. If so, the module will be called nvmem-vf610-ocotp. -config MESON_EFUSE - tristate "Amlogic Meson GX eFuse Support" - depends on (ARCH_MESON || COMPILE_TEST) && MESON_SM - help - This is a driver to retrieve specific values from the eFuse found on - the Amlogic Meson GX SoCs. - - This driver can also be built as a module. If so, the module - will be called nvmem_meson_efuse. - -config MESON_MX_EFUSE - tristate "Amlogic Meson6/Meson8/Meson8b eFuse Support" - depends on ARCH_MESON || COMPILE_TEST - help - This is a driver to retrieve specific values from the eFuse found on - the Amlogic Meson6, Meson8 and Meson8b SoCs. - - This driver can also be built as a module. If so, the module - will be called nvmem_meson_mx_efuse. - -config NVMEM_SNVS_LPGPR - tristate "Support for Low Power General Purpose Register" - depends on ARCH_MXC || COMPILE_TEST - help - This is a driver for Low Power General Purpose Register (LPGPR) available on - i.MX6 and i.MX7 SoCs in Secure Non-Volatile Storage (SNVS) of this chip. - - This driver can also be built as a module. If so, the module - will be called nvmem-snvs-lpgpr. - -config RAVE_SP_EEPROM - tristate "Rave SP EEPROM Support" - depends on RAVE_SP_CORE - help - Say y here to enable Rave SP EEPROM support. - -config SC27XX_EFUSE - tristate "Spreadtrum SC27XX eFuse Support" - depends on MFD_SC27XX_PMIC || COMPILE_TEST - depends on HAS_IOMEM - help - This is a simple driver to dump specified values of Spreadtrum - SC27XX PMICs from eFuse. - - This driver can also be built as a module. If so, the module - will be called nvmem-sc27xx-efuse. - config NVMEM_ZYNQMP bool "Xilinx ZYNQMP SoC nvmem firmware support" depends on ARCH_ZYNQMP @@ -278,70 +367,4 @@ config NVMEM_ZYNQMP If sure, say yes. If unsure, say no. -config SPRD_EFUSE - tristate "Spreadtrum SoC eFuse Support" - depends on ARCH_SPRD || COMPILE_TEST - depends on HAS_IOMEM - help - This is a simple driver to dump specified values of Spreadtrum - SoCs from eFuse. - - This driver can also be built as a module. If so, the module - will be called nvmem-sprd-efuse. - -config NVMEM_RMEM - tristate "Reserved Memory Based Driver Support" - depends on HAS_IOMEM - help - This driver maps reserved memory into an nvmem device. It might be - useful to expose information left by firmware in memory. - - This driver can also be built as a module. If so, the module - will be called nvmem-rmem. - -config NVMEM_BRCM_NVRAM - tristate "Broadcom's NVRAM support" - depends on ARCH_BCM_5301X || COMPILE_TEST - depends on HAS_IOMEM - help - This driver provides support for Broadcom's NVRAM that can be accessed - using I/O mapping. - -config NVMEM_LAYERSCAPE_SFP - tristate "Layerscape SFP (Security Fuse Processor) support" - depends on ARCH_LAYERSCAPE || COMPILE_TEST - depends on HAS_IOMEM - select REGMAP_MMIO - help - This driver provides support to read the eFuses on Freescale - Layerscape SoC's. For example, the vendor provides a per part - unique ID there. - - This driver can also be built as a module. If so, the module - will be called layerscape-sfp. - -config NVMEM_SUNPLUS_OCOTP - tristate "Sunplus SoC OTP support" - depends on SOC_SP7021 || COMPILE_TEST - depends on HAS_IOMEM - help - This is a driver for the On-chip OTP controller (OCOTP) available - on Sunplus SoCs. It provides access to 128 bytes of one-time - programmable eFuse. - - This driver can also be built as a module. If so, the module - will be called nvmem-sunplus-ocotp. - -config NVMEM_APPLE_EFUSES - tristate "Apple eFuse support" - depends on ARCH_APPLE || COMPILE_TEST - default ARCH_APPLE - help - Say y here to enable support for reading eFuses on Apple SoCs - such as the M1. These are e.g. used to store factory programmed - calibration data required for the PCIe or the USB-C PHY. - - This driver can also be built as a module. If so, the module will - be called nvmem-apple-efuses. - endif diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile index c710b64f9fe41cfedcca8b43544d6119ef939607..fa80fe17e567e0c5a551e4c8146268990436c4e4 100644 --- a/drivers/nvmem/Makefile +++ b/drivers/nvmem/Makefile @@ -7,65 +7,69 @@ obj-$(CONFIG_NVMEM) += nvmem_core.o nvmem_core-y := core.o # Devices -obj-$(CONFIG_NVMEM_BCM_OCOTP) += nvmem-bcm-ocotp.o -nvmem-bcm-ocotp-y := bcm-ocotp.o -obj-$(CONFIG_NVMEM_IMX_IIM) += nvmem-imx-iim.o -nvmem-imx-iim-y := imx-iim.o -obj-$(CONFIG_NVMEM_IMX_OCOTP) += nvmem-imx-ocotp.o -nvmem-imx-ocotp-y := imx-ocotp.o +obj-$(CONFIG_NVMEM_APPLE_EFUSES) += nvmem-apple-efuses.o +nvmem-apple-efuses-y := apple-efuses.o +obj-$(CONFIG_NVMEM_BCM_OCOTP) += nvmem-bcm-ocotp.o +nvmem-bcm-ocotp-y := bcm-ocotp.o +obj-$(CONFIG_NVMEM_BRCM_NVRAM) += nvmem_brcm_nvram.o +nvmem_brcm_nvram-y := brcm_nvram.o +obj-$(CONFIG_NVMEM_IMX_IIM) += nvmem-imx-iim.o +nvmem-imx-iim-y := imx-iim.o +obj-$(CONFIG_NVMEM_IMX_OCOTP) += nvmem-imx-ocotp.o +nvmem-imx-ocotp-y := imx-ocotp.o obj-$(CONFIG_NVMEM_IMX_OCOTP_SCU) += nvmem-imx-ocotp-scu.o -nvmem-imx-ocotp-scu-y := imx-ocotp-scu.o -obj-$(CONFIG_JZ4780_EFUSE) += nvmem_jz4780_efuse.o -nvmem_jz4780_efuse-y := jz4780-efuse.o +nvmem-imx-ocotp-scu-y := imx-ocotp-scu.o +obj-$(CONFIG_NVMEM_JZ4780_EFUSE) += nvmem_jz4780_efuse.o +nvmem_jz4780_efuse-y := jz4780-efuse.o +obj-$(CONFIG_NVMEM_LAN9662_OTPC) += nvmem-lan9662-otpc.o +nvmem-lan9662-otpc-y := lan9662-otpc.o +obj-$(CONFIG_NVMEM_LAYERSCAPE_SFP) += nvmem-layerscape-sfp.o +nvmem-layerscape-sfp-y := layerscape-sfp.o obj-$(CONFIG_NVMEM_LPC18XX_EEPROM) += nvmem_lpc18xx_eeprom.o -nvmem_lpc18xx_eeprom-y := lpc18xx_eeprom.o -obj-$(CONFIG_NVMEM_LPC18XX_OTP) += nvmem_lpc18xx_otp.o -nvmem_lpc18xx_otp-y := lpc18xx_otp.o -obj-$(CONFIG_NVMEM_MXS_OCOTP) += nvmem-mxs-ocotp.o -nvmem-mxs-ocotp-y := mxs-ocotp.o +nvmem_lpc18xx_eeprom-y := lpc18xx_eeprom.o +obj-$(CONFIG_NVMEM_LPC18XX_OTP) += nvmem_lpc18xx_otp.o +nvmem_lpc18xx_otp-y := lpc18xx_otp.o +obj-$(CONFIG_NVMEM_MESON_EFUSE) += nvmem_meson_efuse.o +nvmem_meson_efuse-y := meson-efuse.o +obj-$(CONFIG_NVMEM_MESON_MX_EFUSE) += nvmem_meson_mx_efuse.o +nvmem_meson_mx_efuse-y := meson-mx-efuse.o +obj-$(CONFIG_NVMEM_MICROCHIP_OTPC) += nvmem-microchip-otpc.o +nvmem-microchip-otpc-y := microchip-otpc.o +obj-$(CONFIG_NVMEM_MTK_EFUSE) += nvmem_mtk-efuse.o +nvmem_mtk-efuse-y := mtk-efuse.o +obj-$(CONFIG_NVMEM_MXS_OCOTP) += nvmem-mxs-ocotp.o +nvmem-mxs-ocotp-y := mxs-ocotp.o obj-$(CONFIG_NVMEM_NINTENDO_OTP) += nvmem-nintendo-otp.o -nvmem-nintendo-otp-y := nintendo-otp.o -obj-$(CONFIG_MTK_EFUSE) += nvmem_mtk-efuse.o -nvmem_mtk-efuse-y := mtk-efuse.o -obj-$(CONFIG_QCOM_QFPROM) += nvmem_qfprom.o -nvmem_qfprom-y := qfprom.o -obj-$(CONFIG_NVMEM_SPMI_SDAM) += nvmem_qcom-spmi-sdam.o -nvmem_qcom-spmi-sdam-y += qcom-spmi-sdam.o -obj-$(CONFIG_ROCKCHIP_EFUSE) += nvmem_rockchip_efuse.o -nvmem_rockchip_efuse-y := rockchip-efuse.o -obj-$(CONFIG_ROCKCHIP_OTP) += nvmem-rockchip-otp.o -nvmem-rockchip-otp-y := rockchip-otp.o -obj-$(CONFIG_NVMEM_SUNXI_SID) += nvmem_sunxi_sid.o -nvmem_stm32_romem-y := stm32-romem.o -obj-$(CONFIG_NVMEM_STM32_ROMEM) += nvmem_stm32_romem.o -nvmem_sunxi_sid-y := sunxi_sid.o -obj-$(CONFIG_UNIPHIER_EFUSE) += nvmem-uniphier-efuse.o -nvmem-uniphier-efuse-y := uniphier-efuse.o -obj-$(CONFIG_NVMEM_VF610_OCOTP) += nvmem-vf610-ocotp.o -nvmem-vf610-ocotp-y := vf610-ocotp.o -obj-$(CONFIG_MESON_EFUSE) += nvmem_meson_efuse.o -nvmem_meson_efuse-y := meson-efuse.o -obj-$(CONFIG_MESON_MX_EFUSE) += nvmem_meson_mx_efuse.o -nvmem_meson_mx_efuse-y := meson-mx-efuse.o -obj-$(CONFIG_NVMEM_SNVS_LPGPR) += nvmem_snvs_lpgpr.o -nvmem_snvs_lpgpr-y := snvs_lpgpr.o -obj-$(CONFIG_RAVE_SP_EEPROM) += nvmem-rave-sp-eeprom.o -nvmem-rave-sp-eeprom-y := rave-sp-eeprom.o -obj-$(CONFIG_SC27XX_EFUSE) += nvmem-sc27xx-efuse.o -nvmem-sc27xx-efuse-y := sc27xx-efuse.o -obj-$(CONFIG_NVMEM_ZYNQMP) += nvmem_zynqmp_nvmem.o -nvmem_zynqmp_nvmem-y := zynqmp_nvmem.o -obj-$(CONFIG_SPRD_EFUSE) += nvmem_sprd_efuse.o -nvmem_sprd_efuse-y := sprd-efuse.o -obj-$(CONFIG_NVMEM_RMEM) += nvmem-rmem.o -nvmem-rmem-y := rmem.o -obj-$(CONFIG_NVMEM_BRCM_NVRAM) += nvmem_brcm_nvram.o -nvmem_brcm_nvram-y := brcm_nvram.o -obj-$(CONFIG_NVMEM_LAYERSCAPE_SFP) += nvmem-layerscape-sfp.o -nvmem-layerscape-sfp-y := layerscape-sfp.o +nvmem-nintendo-otp-y := nintendo-otp.o +obj-$(CONFIG_NVMEM_QCOM_QFPROM) += nvmem_qfprom.o +nvmem_qfprom-y := qfprom.o +obj-$(CONFIG_NVMEM_RAVE_SP_EEPROM) += nvmem-rave-sp-eeprom.o +nvmem-rave-sp-eeprom-y := rave-sp-eeprom.o +obj-$(CONFIG_NVMEM_RMEM) += nvmem-rmem.o +nvmem-rmem-y := rmem.o +obj-$(CONFIG_NVMEM_ROCKCHIP_EFUSE) += nvmem_rockchip_efuse.o +nvmem_rockchip_efuse-y := rockchip-efuse.o +obj-$(CONFIG_NVMEM_ROCKCHIP_OTP) += nvmem-rockchip-otp.o +nvmem-rockchip-otp-y := rockchip-otp.o +obj-$(CONFIG_NVMEM_SC27XX_EFUSE) += nvmem-sc27xx-efuse.o +nvmem-sc27xx-efuse-y := sc27xx-efuse.o +obj-$(CONFIG_NVMEM_SNVS_LPGPR) += nvmem_snvs_lpgpr.o +nvmem_snvs_lpgpr-y := snvs_lpgpr.o +obj-$(CONFIG_NVMEM_SPMI_SDAM) += nvmem_qcom-spmi-sdam.o +nvmem_qcom-spmi-sdam-y += qcom-spmi-sdam.o +obj-$(CONFIG_NVMEM_SPRD_EFUSE) += nvmem_sprd_efuse.o +nvmem_sprd_efuse-y := sprd-efuse.o +obj-$(CONFIG_NVMEM_STM32_ROMEM) += nvmem_stm32_romem.o +nvmem_stm32_romem-y := stm32-romem.o obj-$(CONFIG_NVMEM_SUNPLUS_OCOTP) += nvmem_sunplus_ocotp.o -nvmem_sunplus_ocotp-y := sunplus-ocotp.o -obj-$(CONFIG_NVMEM_APPLE_EFUSES) += nvmem-apple-efuses.o -nvmem-apple-efuses-y := apple-efuses.o -obj-$(CONFIG_MICROCHIP_OTPC) += nvmem-microchip-otpc.o -nvmem-microchip-otpc-y := microchip-otpc.o +nvmem_sunplus_ocotp-y := sunplus-ocotp.o +obj-$(CONFIG_NVMEM_SUNXI_SID) += nvmem_sunxi_sid.o +nvmem_sunxi_sid-y := sunxi_sid.o +obj-$(CONFIG_NVMEM_U_BOOT_ENV) += nvmem_u-boot-env.o +nvmem_u-boot-env-y := u-boot-env.o +obj-$(CONFIG_NVMEM_UNIPHIER_EFUSE) += nvmem-uniphier-efuse.o +nvmem-uniphier-efuse-y := uniphier-efuse.o +obj-$(CONFIG_NVMEM_VF610_OCOTP) += nvmem-vf610-ocotp.o +nvmem-vf610-ocotp-y := vf610-ocotp.o +obj-$(CONFIG_NVMEM_ZYNQMP) += nvmem_zynqmp_nvmem.o +nvmem_zynqmp_nvmem-y := zynqmp_nvmem.o diff --git a/drivers/nvmem/brcm_nvram.c b/drivers/nvmem/brcm_nvram.c index 450b927691c3625b7e5a4ee4608085e426b03e87..4441daa2096517a6d2bb421871e1ef0eb1250057 100644 --- a/drivers/nvmem/brcm_nvram.c +++ b/drivers/nvmem/brcm_nvram.c @@ -96,7 +96,7 @@ static int brcm_nvram_parse(struct brcm_nvram *priv) len = le32_to_cpu(header.len); - data = kcalloc(1, len, GFP_KERNEL); + data = kzalloc(len, GFP_KERNEL); memcpy_fromio(data, priv->base, len); data[len - 1] = '\0'; diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 1e3c754efd0d8dae59d2b40d3876b03bfa11e44b..321d7d63e0683c96ae8bd35fbd7c7e989fa7d075 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -810,18 +810,24 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) switch (config->id) { case NVMEM_DEVID_NONE: - dev_set_name(&nvmem->dev, "%s", config->name); + rval = dev_set_name(&nvmem->dev, "%s", config->name); break; case NVMEM_DEVID_AUTO: - dev_set_name(&nvmem->dev, "%s%d", config->name, nvmem->id); + rval = dev_set_name(&nvmem->dev, "%s%d", config->name, nvmem->id); break; default: - dev_set_name(&nvmem->dev, "%s%d", + rval = dev_set_name(&nvmem->dev, "%s%d", config->name ? : "nvmem", config->name ? config->id : nvmem->id); break; } + if (rval) { + ida_free(&nvmem_ida, nvmem->id); + kfree(nvmem); + return ERR_PTR(rval); + } + nvmem->read_only = device_property_present(config->dev, "read-only") || config->read_only || !nvmem->reg_write; @@ -829,21 +835,18 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) nvmem->dev.groups = nvmem_dev_groups; #endif - if (nvmem->nkeepout) { - rval = nvmem_validate_keepouts(nvmem); - if (rval) { - ida_free(&nvmem_ida, nvmem->id); - kfree(nvmem); - return ERR_PTR(rval); - } - } - dev_dbg(&nvmem->dev, "Registering nvmem device %s\n", config->name); rval = device_register(&nvmem->dev); if (rval) goto err_put_device; + if (nvmem->nkeepout) { + rval = nvmem_validate_keepouts(nvmem); + if (rval) + goto err_device_del; + } + if (config->compat) { rval = nvmem_sysfs_setup_compat(nvmem, config); if (rval) diff --git a/drivers/nvmem/lan9662-otpc.c b/drivers/nvmem/lan9662-otpc.c new file mode 100644 index 0000000000000000000000000000000000000000..f6732fd216d801400f15504989ebd18b078ce572 --- /dev/null +++ b/drivers/nvmem/lan9662-otpc.c @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include + +#define OTP_OTP_PWR_DN(t) (t + 0x00) +#define OTP_OTP_PWR_DN_OTP_PWRDN_N BIT(0) +#define OTP_OTP_ADDR_HI(t) (t + 0x04) +#define OTP_OTP_ADDR_LO(t) (t + 0x08) +#define OTP_OTP_PRGM_DATA(t) (t + 0x10) +#define OTP_OTP_PRGM_MODE(t) (t + 0x14) +#define OTP_OTP_PRGM_MODE_OTP_PGM_MODE_BYTE BIT(0) +#define OTP_OTP_RD_DATA(t) (t + 0x18) +#define OTP_OTP_FUNC_CMD(t) (t + 0x20) +#define OTP_OTP_FUNC_CMD_OTP_PROGRAM BIT(1) +#define OTP_OTP_FUNC_CMD_OTP_READ BIT(0) +#define OTP_OTP_CMD_GO(t) (t + 0x28) +#define OTP_OTP_CMD_GO_OTP_GO BIT(0) +#define OTP_OTP_PASS_FAIL(t) (t + 0x2c) +#define OTP_OTP_PASS_FAIL_OTP_READ_PROHIBITED BIT(3) +#define OTP_OTP_PASS_FAIL_OTP_WRITE_PROHIBITED BIT(2) +#define OTP_OTP_PASS_FAIL_OTP_FAIL BIT(0) +#define OTP_OTP_STATUS(t) (t + 0x30) +#define OTP_OTP_STATUS_OTP_CPUMPEN BIT(1) +#define OTP_OTP_STATUS_OTP_BUSY BIT(0) + +#define OTP_MEM_SIZE 8192 +#define OTP_SLEEP_US 10 +#define OTP_TIMEOUT_US 500000 + +struct lan9662_otp { + struct device *dev; + void __iomem *base; +}; + +static bool lan9662_otp_wait_flag_clear(void __iomem *reg, u32 flag) +{ + u32 val; + + return readl_poll_timeout(reg, val, !(val & flag), + OTP_SLEEP_US, OTP_TIMEOUT_US); +} + +static int lan9662_otp_power(struct lan9662_otp *otp, bool up) +{ + void __iomem *pwrdn = OTP_OTP_PWR_DN(otp->base); + + if (up) { + writel(readl(pwrdn) & ~OTP_OTP_PWR_DN_OTP_PWRDN_N, pwrdn); + if (lan9662_otp_wait_flag_clear(OTP_OTP_STATUS(otp->base), + OTP_OTP_STATUS_OTP_CPUMPEN)) + return -ETIMEDOUT; + } else { + writel(readl(pwrdn) | OTP_OTP_PWR_DN_OTP_PWRDN_N, pwrdn); + } + + return 0; +} + +static int lan9662_otp_execute(struct lan9662_otp *otp) +{ + if (lan9662_otp_wait_flag_clear(OTP_OTP_CMD_GO(otp->base), + OTP_OTP_CMD_GO_OTP_GO)) + return -ETIMEDOUT; + + if (lan9662_otp_wait_flag_clear(OTP_OTP_STATUS(otp->base), + OTP_OTP_STATUS_OTP_BUSY)) + return -ETIMEDOUT; + + return 0; +} + +static void lan9662_otp_set_address(struct lan9662_otp *otp, u32 offset) +{ + writel(0xff & (offset >> 8), OTP_OTP_ADDR_HI(otp->base)); + writel(0xff & offset, OTP_OTP_ADDR_LO(otp->base)); +} + +static int lan9662_otp_read_byte(struct lan9662_otp *otp, u32 offset, u8 *dst) +{ + u32 pass; + int rc; + + lan9662_otp_set_address(otp, offset); + writel(OTP_OTP_FUNC_CMD_OTP_READ, OTP_OTP_FUNC_CMD(otp->base)); + writel(OTP_OTP_CMD_GO_OTP_GO, OTP_OTP_CMD_GO(otp->base)); + rc = lan9662_otp_execute(otp); + if (!rc) { + pass = readl(OTP_OTP_PASS_FAIL(otp->base)); + if (pass & OTP_OTP_PASS_FAIL_OTP_READ_PROHIBITED) + return -EACCES; + *dst = (u8) readl(OTP_OTP_RD_DATA(otp->base)); + } + return rc; +} + +static int lan9662_otp_write_byte(struct lan9662_otp *otp, u32 offset, u8 data) +{ + u32 pass; + int rc; + + lan9662_otp_set_address(otp, offset); + writel(OTP_OTP_PRGM_MODE_OTP_PGM_MODE_BYTE, OTP_OTP_PRGM_MODE(otp->base)); + writel(data, OTP_OTP_PRGM_DATA(otp->base)); + writel(OTP_OTP_FUNC_CMD_OTP_PROGRAM, OTP_OTP_FUNC_CMD(otp->base)); + writel(OTP_OTP_CMD_GO_OTP_GO, OTP_OTP_CMD_GO(otp->base)); + + rc = lan9662_otp_execute(otp); + if (!rc) { + pass = readl(OTP_OTP_PASS_FAIL(otp->base)); + if (pass & OTP_OTP_PASS_FAIL_OTP_WRITE_PROHIBITED) + return -EACCES; + if (pass & OTP_OTP_PASS_FAIL_OTP_FAIL) + return -EIO; + } + return rc; +} + +static int lan9662_otp_read(void *context, unsigned int offset, + void *_val, size_t bytes) +{ + struct lan9662_otp *otp = context; + u8 *val = _val; + uint8_t data; + int i, rc = 0; + + lan9662_otp_power(otp, true); + for (i = 0; i < bytes; i++) { + rc = lan9662_otp_read_byte(otp, offset + i, &data); + if (rc < 0) + break; + *val++ = data; + } + lan9662_otp_power(otp, false); + + return rc; +} + +static int lan9662_otp_write(void *context, unsigned int offset, + void *_val, size_t bytes) +{ + struct lan9662_otp *otp = context; + u8 *val = _val; + u8 data, newdata; + int i, rc = 0; + + lan9662_otp_power(otp, true); + for (i = 0; i < bytes; i++) { + /* Skip zero bytes */ + if (val[i]) { + rc = lan9662_otp_read_byte(otp, offset + i, &data); + if (rc < 0) + break; + + newdata = data | val[i]; + if (newdata == data) + continue; + + rc = lan9662_otp_write_byte(otp, offset + i, + newdata); + if (rc < 0) + break; + } + } + lan9662_otp_power(otp, false); + + return rc; +} + +static struct nvmem_config otp_config = { + .name = "lan9662-otp", + .stride = 1, + .word_size = 1, + .reg_read = lan9662_otp_read, + .reg_write = lan9662_otp_write, + .size = OTP_MEM_SIZE, +}; + +static int lan9662_otp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct nvmem_device *nvmem; + struct lan9662_otp *otp; + + otp = devm_kzalloc(&pdev->dev, sizeof(*otp), GFP_KERNEL); + if (!otp) + return -ENOMEM; + + otp->dev = dev; + otp->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(otp->base)) + return PTR_ERR(otp->base); + + otp_config.priv = otp; + otp_config.dev = dev; + + nvmem = devm_nvmem_register(dev, &otp_config); + + return PTR_ERR_OR_ZERO(nvmem); +} + +static const struct of_device_id lan9662_otp_match[] = { + { .compatible = "microchip,lan9662-otp", }, + { }, +}; +MODULE_DEVICE_TABLE(of, lan9662_otp_match); + +static struct platform_driver lan9662_otp_driver = { + .probe = lan9662_otp_probe, + .driver = { + .name = "lan9662-otp", + .of_match_table = lan9662_otp_match, + }, +}; +module_platform_driver(lan9662_otp_driver); + +MODULE_AUTHOR("Horatiu Vultur "); +MODULE_DESCRIPTION("lan9662 OTP driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/nvmem/u-boot-env.c b/drivers/nvmem/u-boot-env.c new file mode 100644 index 0000000000000000000000000000000000000000..8e72d1bbd6490d003502ba94cf9a0124e4783956 --- /dev/null +++ b/drivers/nvmem/u-boot-env.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Rafał Miłecki + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum u_boot_env_format { + U_BOOT_FORMAT_SINGLE, + U_BOOT_FORMAT_REDUNDANT, +}; + +struct u_boot_env { + struct device *dev; + enum u_boot_env_format format; + + struct mtd_info *mtd; + + /* Cells */ + struct nvmem_cell_info *cells; + int ncells; +}; + +struct u_boot_env_image_single { + __le32 crc32; + uint8_t data[]; +} __packed; + +struct u_boot_env_image_redundant { + __le32 crc32; + u8 mark; + uint8_t data[]; +} __packed; + +static int u_boot_env_read(void *context, unsigned int offset, void *val, + size_t bytes) +{ + struct u_boot_env *priv = context; + struct device *dev = priv->dev; + size_t bytes_read; + int err; + + err = mtd_read(priv->mtd, offset, bytes, &bytes_read, val); + if (err && !mtd_is_bitflip(err)) { + dev_err(dev, "Failed to read from mtd: %d\n", err); + return err; + } + + if (bytes_read != bytes) { + dev_err(dev, "Failed to read %zu bytes\n", bytes); + return -EIO; + } + + return 0; +} + +static int u_boot_env_add_cells(struct u_boot_env *priv, uint8_t *buf, + size_t data_offset, size_t data_len) +{ + struct device *dev = priv->dev; + char *data = buf + data_offset; + char *var, *value, *eq; + int idx; + + priv->ncells = 0; + for (var = data; var < data + data_len && *var; var += strlen(var) + 1) + priv->ncells++; + + priv->cells = devm_kcalloc(dev, priv->ncells, sizeof(*priv->cells), GFP_KERNEL); + if (!priv->cells) + return -ENOMEM; + + for (var = data, idx = 0; + var < data + data_len && *var; + var = value + strlen(value) + 1, idx++) { + eq = strchr(var, '='); + if (!eq) + break; + *eq = '\0'; + value = eq + 1; + + priv->cells[idx].name = devm_kstrdup(dev, var, GFP_KERNEL); + if (!priv->cells[idx].name) + return -ENOMEM; + priv->cells[idx].offset = data_offset + value - data; + priv->cells[idx].bytes = strlen(value); + priv->cells[idx].np = of_get_child_by_name(dev->of_node, priv->cells[idx].name); + } + + if (WARN_ON(idx != priv->ncells)) + priv->ncells = idx; + + return 0; +} + +static int u_boot_env_parse(struct u_boot_env *priv) +{ + struct device *dev = priv->dev; + size_t crc32_data_offset; + size_t crc32_data_len; + size_t crc32_offset; + size_t data_offset; + size_t data_len; + uint32_t crc32; + uint32_t calc; + size_t bytes; + uint8_t *buf; + int err; + + buf = kcalloc(1, priv->mtd->size, GFP_KERNEL); + if (!buf) { + err = -ENOMEM; + goto err_out; + } + + err = mtd_read(priv->mtd, 0, priv->mtd->size, &bytes, buf); + if ((err && !mtd_is_bitflip(err)) || bytes != priv->mtd->size) { + dev_err(dev, "Failed to read from mtd: %d\n", err); + goto err_kfree; + } + + switch (priv->format) { + case U_BOOT_FORMAT_SINGLE: + crc32_offset = offsetof(struct u_boot_env_image_single, crc32); + crc32_data_offset = offsetof(struct u_boot_env_image_single, data); + data_offset = offsetof(struct u_boot_env_image_single, data); + break; + case U_BOOT_FORMAT_REDUNDANT: + crc32_offset = offsetof(struct u_boot_env_image_redundant, crc32); + crc32_data_offset = offsetof(struct u_boot_env_image_redundant, mark); + data_offset = offsetof(struct u_boot_env_image_redundant, data); + break; + } + crc32 = le32_to_cpu(*(__le32 *)(buf + crc32_offset)); + crc32_data_len = priv->mtd->size - crc32_data_offset; + data_len = priv->mtd->size - data_offset; + + calc = crc32(~0, buf + crc32_data_offset, crc32_data_len) ^ ~0L; + if (calc != crc32) { + dev_err(dev, "Invalid calculated CRC32: 0x%08x (expected: 0x%08x)\n", calc, crc32); + err = -EINVAL; + goto err_kfree; + } + + buf[priv->mtd->size - 1] = '\0'; + err = u_boot_env_add_cells(priv, buf, data_offset, data_len); + if (err) + dev_err(dev, "Failed to add cells: %d\n", err); + +err_kfree: + kfree(buf); +err_out: + return err; +} + +static int u_boot_env_probe(struct platform_device *pdev) +{ + struct nvmem_config config = { + .name = "u-boot-env", + .reg_read = u_boot_env_read, + }; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct u_boot_env *priv; + int err; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + + priv->format = (uintptr_t)of_device_get_match_data(dev); + + priv->mtd = of_get_mtd_device_by_node(np); + if (IS_ERR(priv->mtd)) { + dev_err_probe(dev, PTR_ERR(priv->mtd), "Failed to get %pOF MTD\n", np); + return PTR_ERR(priv->mtd); + } + + err = u_boot_env_parse(priv); + if (err) + return err; + + config.dev = dev; + config.cells = priv->cells; + config.ncells = priv->ncells; + config.priv = priv; + config.size = priv->mtd->size; + + return PTR_ERR_OR_ZERO(devm_nvmem_register(dev, &config)); +} + +static const struct of_device_id u_boot_env_of_match_table[] = { + { .compatible = "u-boot,env", .data = (void *)U_BOOT_FORMAT_SINGLE, }, + { .compatible = "u-boot,env-redundant-bool", .data = (void *)U_BOOT_FORMAT_REDUNDANT, }, + { .compatible = "u-boot,env-redundant-count", .data = (void *)U_BOOT_FORMAT_REDUNDANT, }, + {}, +}; + +static struct platform_driver u_boot_env_driver = { + .probe = u_boot_env_probe, + .driver = { + .name = "u_boot_env", + .of_match_table = u_boot_env_of_match_table, + }, +}; +module_platform_driver(u_boot_env_driver); + +MODULE_AUTHOR("Rafał Miłecki"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, u_boot_env_of_match_table); diff --git a/drivers/of/address.c b/drivers/of/address.c index 96f0a12e507cd12e8dad8f7703f3e094f7ddb00e..c34ac33b7338a991051e75adec81770fe8a35292 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -579,7 +579,8 @@ u64 of_translate_address(struct device_node *dev, const __be32 *in_addr) } EXPORT_SYMBOL(of_translate_address); -static struct device_node *__of_get_dma_parent(const struct device_node *np) +#ifdef CONFIG_HAS_DMA +struct device_node *__of_get_dma_parent(const struct device_node *np) { struct of_phandle_args args; int ret, index; @@ -596,6 +597,7 @@ static struct device_node *__of_get_dma_parent(const struct device_node *np) return of_node_get(args.np); } +#endif static struct device_node *of_get_next_dma_parent(struct device_node *np) { diff --git a/drivers/of/base.c b/drivers/of/base.c index 7fa960bd3df16addf7e026cc0741d60e6ddbc892..d5a5c35eba72ac62d3a06be7956bf97297adb8a1 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -561,7 +561,7 @@ EXPORT_SYMBOL(of_device_is_compatible); * a NULL terminated array of strings. Returns the best match * score or 0. */ -int of_device_compatible_match(struct device_node *device, +int of_device_compatible_match(const struct device_node *device, const char *const *compat) { unsigned int tmp, score = 0; @@ -578,6 +578,7 @@ int of_device_compatible_match(struct device_node *device, return score; } +EXPORT_SYMBOL_GPL(of_device_compatible_match); /** * of_machine_is_compatible - Test root of device tree for a given compatible value @@ -1228,7 +1229,7 @@ int of_modalias_node(struct device_node *node, char *modalias, int len) if (!compatible || strlen(compatible) > cplen) return -ENODEV; p = strchr(compatible, ','); - strlcpy(modalias, p ? p + 1 : compatible, len); + strscpy(modalias, p ? p + 1 : compatible, len); return 0; } EXPORT_SYMBOL_GPL(of_modalias_node); @@ -2088,12 +2089,13 @@ int of_find_last_cache_level(unsigned int cpu) struct device_node *prev = NULL, *np = of_cpu_device_node_get(cpu); while (np) { + of_node_put(prev); prev = np; - of_node_put(np); np = of_find_next_cache_node(np); } of_property_read_u32(prev, "cache-level", &cache_level); + of_node_put(prev); return cache_level; } diff --git a/drivers/of/device.c b/drivers/of/device.c index 75b6cbffa7558bf91cf4a545dc9119dc0dfb5551..8cefe5a7d04e2e8143975562395ced6c0ec9fd62 100644 --- a/drivers/of/device.c +++ b/drivers/of/device.c @@ -116,12 +116,19 @@ int of_dma_configure_id(struct device *dev, struct device_node *np, { const struct iommu_ops *iommu; const struct bus_dma_region *map = NULL; + struct device_node *bus_np; u64 dma_start = 0; u64 mask, end, size = 0; bool coherent; int ret; - ret = of_dma_get_range(np, &map); + if (np == dev->of_node) + bus_np = __of_get_dma_parent(np); + else + bus_np = of_node_get(np); + + ret = of_dma_get_range(bus_np, &map); + of_node_put(bus_np); if (ret < 0) { /* * For legacy reasons, we have to assume some devices need diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index 7bc92923104c1e21017c81d03bfc796f2f8d4d83..7b571a631639702817cd550ad599896fc3bfb2ae 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -314,7 +314,7 @@ static int unflatten_dt_nodes(const void *blob, for (offset = 0; offset >= 0 && depth >= initial_depth; offset = fdt_next_node(blob, offset, &depth)) { - if (WARN_ON_ONCE(depth >= FDT_MAX_DEPTH)) + if (WARN_ON_ONCE(depth >= FDT_MAX_DEPTH - 1)) continue; if (!IS_ENABLED(CONFIG_OF_KOBJ) && @@ -828,15 +828,6 @@ uint32_t __init of_get_flat_dt_phandle(unsigned long node) return fdt_get_phandle(initial_boot_params, node); } -struct fdt_scan_status { - const char *name; - int namelen; - int depth; - int found; - int (*iterator)(unsigned long node, const char *uname, int depth, void *data); - void *data; -}; - const char * __init of_flat_dt_get_machine_name(void) { const char *name; @@ -936,6 +927,8 @@ static void __init early_init_dt_check_for_initrd(unsigned long node) if (!prop) return; end = of_read_number(prop, len/4); + if (start > end) + return; __early_init_dt_declare_initrd(start, end); phys_initrd_start = start; @@ -1178,7 +1171,7 @@ int __init early_init_dt_scan_chosen(char *cmdline) /* Retrieve command line */ p = of_get_flat_dt_prop(node, "bootargs", &l); if (p != NULL && l > 0) - strlcpy(cmdline, p, min(l, COMMAND_LINE_SIZE)); + strscpy(cmdline, p, min(l, COMMAND_LINE_SIZE)); /* * CONFIG_CMDLINE is meant to be a default in case nothing else @@ -1190,11 +1183,11 @@ int __init early_init_dt_scan_chosen(char *cmdline) strlcat(cmdline, " ", COMMAND_LINE_SIZE); strlcat(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #elif defined(CONFIG_CMDLINE_FORCE) - strlcpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + strscpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #else /* No arguments from boot loader, use kernel's cmdl*/ if (!((char *)cmdline)[0]) - strlcpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); + strscpy(cmdline, CONFIG_CMDLINE, COMMAND_LINE_SIZE); #endif #endif /* CONFIG_CMDLINE */ diff --git a/drivers/of/irq.c b/drivers/of/irq.c index d22f605fa7eece6da24e83136765be6fb91c3b59..2bac44f09554b146fdc5a20435f2fcd60a585ca7 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -592,6 +592,9 @@ void __init of_irq_init(const struct of_device_id *matches) ret = desc->irq_init_cb(desc->dev, desc->interrupt_parent); if (ret) { + pr_err("%s: Failed to init %pOF (%p), parent %p\n", + __func__, desc->dev, desc->dev, + desc->interrupt_parent); of_node_clear_flag(desc->dev, OF_POPULATED); kfree(desc); continue; diff --git a/drivers/of/of_private.h b/drivers/of/of_private.h index 9324483397f67eafee92cf62b896cff0e02862d5..fb6792d381a6b6c16c415cc6a3cec2362b29ab85 100644 --- a/drivers/of/of_private.h +++ b/drivers/of/of_private.h @@ -155,12 +155,17 @@ struct bus_dma_region; #if defined(CONFIG_OF_ADDRESS) && defined(CONFIG_HAS_DMA) int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map); +struct device_node *__of_get_dma_parent(const struct device_node *np); #else static inline int of_dma_get_range(struct device_node *np, const struct bus_dma_region **map) { return -ENODEV; } +static inline struct device_node *__of_get_dma_parent(const struct device_node *np) +{ + return of_get_parent(np); +} #endif void fdt_init_reserved_mem(void); diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index eafa8ffefbd0ad4c2102b47d74900045bf60e8a5..b89ab5d9fea55415ba3682c2a4542974e392d651 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -2465,7 +2465,7 @@ static int unittest_i2c_bus_probe(struct platform_device *pdev) adap = &std->adap; i2c_set_adapdata(adap, std); adap->nr = -1; - strlcpy(adap->name, pdev->name, sizeof(adap->name)); + strscpy(adap->name, pdev->name, sizeof(adap->name)); adap->class = I2C_CLASS_DEPRECATED; adap->algo = &unittest_i2c_algo; adap->dev.parent = dev; @@ -2524,13 +2524,12 @@ static int unittest_i2c_dev_probe(struct i2c_client *client, return 0; }; -static int unittest_i2c_dev_remove(struct i2c_client *client) +static void unittest_i2c_dev_remove(struct i2c_client *client) { struct device *dev = &client->dev; struct device_node *np = client->dev.of_node; dev_dbg(dev, "%s for node @%pOF\n", __func__, np); - return 0; } static const struct i2c_device_id unittest_i2c_dev_id[] = { @@ -2601,7 +2600,7 @@ static int unittest_i2c_mux_probe(struct i2c_client *client, return 0; }; -static int unittest_i2c_mux_remove(struct i2c_client *client) +static void unittest_i2c_mux_remove(struct i2c_client *client) { struct device *dev = &client->dev; struct device_node *np = client->dev.of_node; @@ -2609,7 +2608,6 @@ static int unittest_i2c_mux_remove(struct i2c_client *client) dev_dbg(dev, "%s for node @%pOF\n", __func__, np); i2c_mux_del_adapters(muxc); - return 0; } static const struct i2c_device_id unittest_i2c_mux_id[] = { @@ -3467,6 +3465,9 @@ static int __init of_unittest(void) pr_info("start of unittest - you will see error messages\n"); + /* Taint the kernel so we know we've run tests. */ + add_taint(TAINT_TEST, LOCKDEP_STILL_OK); + /* adding data for unittest */ if (IS_ENABLED(CONFIG_UML)) diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 77d1ba3a4154f93ca7a1fe46de791d9e54dd0cce..e87567dbe99f27d09c9de29fc40e9e1108f4a5da 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -873,7 +873,7 @@ int dev_pm_opp_config_clks_simple(struct device *dev, } } - return ret; + return 0; } EXPORT_SYMBOL_GPL(dev_pm_opp_config_clks_simple); diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index 9be007c9420f9e87a675a611b62f4f4f62faa661..a66386043aa6643d87216005c426da904a0408c7 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -268,7 +268,7 @@ static int ioc_count; * Each bit can represent a number of pages. * LSbs represent lower addresses (IOVA's). * -* This was was copied from sba_iommu.c. Don't try to unify +* This was copied from sba_iommu.c. Don't try to unify * the two resource managers unless a way to have different * allocation policies is also adjusted. We'd like to avoid * I/O TLB thrashing by having resource allocation policy @@ -1380,15 +1380,17 @@ ccio_init_resource(struct resource *res, char *name, void __iomem *ioaddr) } } -static void __init ccio_init_resources(struct ioc *ioc) +static int __init ccio_init_resources(struct ioc *ioc) { struct resource *res = ioc->mmio_region; char *name = kmalloc(14, GFP_KERNEL); - + if (unlikely(!name)) + return -ENOMEM; snprintf(name, 14, "GSC Bus [%d/]", ioc->hw_path); ccio_init_resource(res, name, &ioc->ioc_regs->io_io_low); ccio_init_resource(res + 1, name, &ioc->ioc_regs->io_io_low_hv); + return 0; } static int new_ioc_area(struct resource *res, unsigned long size, @@ -1543,7 +1545,11 @@ static int __init ccio_probe(struct parisc_device *dev) return -ENOMEM; } ccio_ioc_init(ioc); - ccio_init_resources(ioc); + if (ccio_init_resources(ioc)) { + iounmap(ioc->ioc_regs); + kfree(ioc); + return -ENOMEM; + } hppa_dma_ops = &ccio_ops; hba = kzalloc(sizeof(*hba), GFP_KERNEL); diff --git a/drivers/parisc/eisa_enumerator.c b/drivers/parisc/eisa_enumerator.c index f54a6f45039158958d58f8ff3ce21a0a12c6dc3d..f0cb31198a8f0345b8b453dd281ed5b0092050c0 100644 --- a/drivers/parisc/eisa_enumerator.c +++ b/drivers/parisc/eisa_enumerator.c @@ -393,7 +393,7 @@ static int parse_slot_config(int slot, } if (p0 + function_len < pos) { - printk(KERN_ERR "eisa_enumerator: function %d length mis-match " + printk(KERN_ERR "eisa_enumerator: function %d length mismatch " "got %d, expected %d\n", num_func, pos-p0, function_len); res=-1; @@ -407,13 +407,13 @@ static int parse_slot_config(int slot, } if (pos != es->config_data_length) { - printk(KERN_ERR "eisa_enumerator: config data length mis-match got %d, expected %d\n", + printk(KERN_ERR "eisa_enumerator: config data length mismatch got %d, expected %d\n", pos, es->config_data_length); res=-1; } if (num_func != es->num_functions) { - printk(KERN_ERR "eisa_enumerator: number of functions mis-match got %d, expected %d\n", + printk(KERN_ERR "eisa_enumerator: number of functions mismatch got %d, expected %d\n", num_func, es->num_functions); res=-2; } @@ -451,7 +451,7 @@ static int init_slot(int slot, struct eeprom_eisa_slot_info *es) } if (es->eisa_slot_id != id) { print_eisa_id(id_string, id); - printk(KERN_ERR "EISA slot %d id mis-match: got %s", + printk(KERN_ERR "EISA slot %d id mismatch: got %s", slot, id_string); print_eisa_id(id_string, es->eisa_slot_id); diff --git a/drivers/parisc/iosapic.c b/drivers/parisc/iosapic.c index 3a8c986156348465b5890317906ecb8dad4bcd5b..bdef7a8d6ab8e89ed60573c88581fcd14ff75d9c 100644 --- a/drivers/parisc/iosapic.c +++ b/drivers/parisc/iosapic.c @@ -221,16 +221,7 @@ static size_t irt_num_entry; static struct irt_entry *iosapic_alloc_irt(int num_entries) { - unsigned long a; - - /* The IRT needs to be 8-byte aligned for the PDC call. - * Normally kmalloc would guarantee larger alignment, but - * if CONFIG_DEBUG_SLAB is enabled, then we can get only - * 4-byte alignment on 32-bit kernels - */ - a = (unsigned long)kmalloc(sizeof(struct irt_entry) * num_entries + 8, GFP_KERNEL); - a = (a + 7UL) & ~7UL; - return (struct irt_entry *)a; + return kcalloc(num_entries, sizeof(struct irt_entry), GFP_KERNEL); } /** diff --git a/drivers/parisc/led.c b/drivers/parisc/led.c index 1e4a5663d01122bffa88f0ecae49fdf3d5a6f3de..d4be9d2ee74d9254b7d9a57797d2460c646060ad 100644 --- a/drivers/parisc/led.c +++ b/drivers/parisc/led.c @@ -646,7 +646,7 @@ int lcd_print( const char *str ) cancel_delayed_work_sync(&led_task); /* copy display string to buffer for procfs */ - strlcpy(lcd_text, str, sizeof(lcd_text)); + strscpy(lcd_text, str, sizeof(lcd_text)); /* Set LCD Cursor to 1st character */ gsc_writeb(lcd_info.reset_cmd1, LCD_CMD_REG); diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index 374b9199878d4121da8d78849c62707b8635e151..ecd870087a3db23cbf2dd680487cde77245188bd 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -28,6 +28,12 @@ #include #include #include +/* + * The semantics of 64 register access on 32bit systems can't be guaranteed + * by the C standard, we hope the _lo_hi() macros defining readq and writeq + * here will behave as expected. + */ +#include #include #include diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c index eda4ded4d5e52a11f357e249a6826d893d227ceb..7c45927e2131c6dcd0b39ef93f9ff49c4c84e0f9 100644 --- a/drivers/parport/parport_pc.c +++ b/drivers/parport/parport_pc.c @@ -2604,6 +2604,7 @@ enum parport_pc_pci_cards { oxsemi_pcie_pport, aks_0100, mobility_pp, + netmos_9900, netmos_9705, netmos_9715, netmos_9755, @@ -2665,6 +2666,7 @@ static struct parport_pc_pci { /* oxsemi_pcie_pport */ { 1, { { 0, 1 }, } }, /* aks_0100 */ { 1, { { 0, -1 }, } }, /* mobility_pp */ { 1, { { 0, 1 }, } }, + /* netmos_9900 */ { 1, { { 0, -1 }, } }, /* The netmos entries below are untested */ /* netmos_9705 */ { 1, { { 0, -1 }, } }, @@ -2746,6 +2748,8 @@ static const struct pci_device_id parport_pc_pci_tbl[] = { PCI_ANY_ID, PCI_ANY_ID, 0, 0, aks_0100 }, { 0x14f2, 0x0121, PCI_ANY_ID, PCI_ANY_ID, 0, 0, mobility_pp }, /* NetMos communication controllers */ + { PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900, + 0xA000, 0x2000, 0, 0, netmos_9900 }, { PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9705, PCI_ANY_ID, PCI_ANY_ID, 0, 0, netmos_9705 }, { PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9715, diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index d1c5fcf00a8a2b0834624434edcbb491df06fc89..bfd9bac37e24f14e628c9621054691b6b1bb41fe 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -274,7 +274,7 @@ config VMD config PCIE_BRCMSTB tristate "Broadcom Brcmstb PCIe host controller" - depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || \ + depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCMBCA || \ BMIPS_GENERIC || COMPILE_TEST depends on OF depends on PCI_MSI_IRQ_DOMAIN diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index 6e5debdbc55b9d9ca446687e3f83180e2d34bad5..2616585ca5f8aa0982775c06eeebd1062c918976 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -51,6 +51,7 @@ enum imx6_pcie_variants { IMX7D, IMX8MQ, IMX8MM, + IMX8MP, }; #define IMX6_PCIE_FLAG_IMX6_PHY BIT(0) @@ -61,6 +62,7 @@ struct imx6_pcie_drvdata { enum imx6_pcie_variants variant; u32 flags; int dbi_length; + const char *gpr; }; struct imx6_pcie { @@ -150,7 +152,8 @@ struct imx6_pcie { static unsigned int imx6_pcie_grp_offset(const struct imx6_pcie *imx6_pcie) { WARN_ON(imx6_pcie->drvdata->variant != IMX8MQ && - imx6_pcie->drvdata->variant != IMX8MM); + imx6_pcie->drvdata->variant != IMX8MM && + imx6_pcie->drvdata->variant != IMX8MP); return imx6_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14; } @@ -301,6 +304,7 @@ static void imx6_pcie_init_phy(struct imx6_pcie *imx6_pcie) { switch (imx6_pcie->drvdata->variant) { case IMX8MM: + case IMX8MP: /* * The PHY initialization had been done in the PHY * driver, break here directly. @@ -558,6 +562,7 @@ static int imx6_pcie_enable_ref_clk(struct imx6_pcie *imx6_pcie) break; case IMX8MM: case IMX8MQ: + case IMX8MP: ret = clk_prepare_enable(imx6_pcie->pcie_aux); if (ret) { dev_err(dev, "unable to enable pcie_aux clock\n"); @@ -602,6 +607,7 @@ static void imx6_pcie_disable_ref_clk(struct imx6_pcie *imx6_pcie) break; case IMX8MM: case IMX8MQ: + case IMX8MP: clk_disable_unprepare(imx6_pcie->pcie_aux); break; default: @@ -669,6 +675,7 @@ static void imx6_pcie_assert_core_reset(struct imx6_pcie *imx6_pcie) reset_control_assert(imx6_pcie->pciephy_reset); fallthrough; case IMX8MM: + case IMX8MP: reset_control_assert(imx6_pcie->apps_reset); break; case IMX6SX: @@ -744,6 +751,7 @@ static int imx6_pcie_deassert_core_reset(struct imx6_pcie *imx6_pcie) break; case IMX6Q: /* Nothing to do */ case IMX8MM: + case IMX8MP: break; } @@ -793,6 +801,7 @@ static void imx6_pcie_ltssm_enable(struct device *dev) case IMX7D: case IMX8MQ: case IMX8MM: + case IMX8MP: reset_control_deassert(imx6_pcie->apps_reset); break; } @@ -812,6 +821,7 @@ static void imx6_pcie_ltssm_disable(struct device *dev) case IMX7D: case IMX8MQ: case IMX8MM: + case IMX8MP: reset_control_assert(imx6_pcie->apps_reset); break; } @@ -935,7 +945,7 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) } if (imx6_pcie->phy) { - ret = phy_power_on(imx6_pcie->phy); + ret = phy_init(imx6_pcie->phy); if (ret) { dev_err(dev, "pcie PHY power up failed\n"); goto err_clk_disable; @@ -949,7 +959,7 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) } if (imx6_pcie->phy) { - ret = phy_init(imx6_pcie->phy); + ret = phy_power_on(imx6_pcie->phy); if (ret) { dev_err(dev, "waiting for PHY ready timeout!\n"); goto err_phy_off; @@ -961,7 +971,7 @@ static int imx6_pcie_host_init(struct dw_pcie_rp *pp) err_phy_off: if (imx6_pcie->phy) - phy_power_off(imx6_pcie->phy); + phy_exit(imx6_pcie->phy); err_clk_disable: imx6_pcie_clk_disable(imx6_pcie); err_reg_disable: @@ -1179,6 +1189,7 @@ static int imx6_pcie_probe(struct platform_device *pdev) } break; case IMX8MM: + case IMX8MP: imx6_pcie->pcie_aux = devm_clk_get(dev, "pcie_aux"); if (IS_ERR(imx6_pcie->pcie_aux)) return dev_err_probe(dev, PTR_ERR(imx6_pcie->pcie_aux), @@ -1216,7 +1227,7 @@ static int imx6_pcie_probe(struct platform_device *pdev) /* Grab GPR config register range */ imx6_pcie->iomuxc_gpr = - syscon_regmap_lookup_by_compatible("fsl,imx6q-iomuxc-gpr"); + syscon_regmap_lookup_by_compatible(imx6_pcie->drvdata->gpr); if (IS_ERR(imx6_pcie->iomuxc_gpr)) { dev_err(dev, "unable to find iomuxc registers\n"); return PTR_ERR(imx6_pcie->iomuxc_gpr); @@ -1295,12 +1306,14 @@ static const struct imx6_pcie_drvdata drvdata[] = { .flags = IMX6_PCIE_FLAG_IMX6_PHY | IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE, .dbi_length = 0x200, + .gpr = "fsl,imx6q-iomuxc-gpr", }, [IMX6SX] = { .variant = IMX6SX, .flags = IMX6_PCIE_FLAG_IMX6_PHY | IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE | IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .gpr = "fsl,imx6q-iomuxc-gpr", }, [IMX6QP] = { .variant = IMX6QP, @@ -1308,17 +1321,26 @@ static const struct imx6_pcie_drvdata drvdata[] = { IMX6_PCIE_FLAG_IMX6_SPEED_CHANGE | IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, .dbi_length = 0x200, + .gpr = "fsl,imx6q-iomuxc-gpr", }, [IMX7D] = { .variant = IMX7D, .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .gpr = "fsl,imx7d-iomuxc-gpr", }, [IMX8MQ] = { .variant = IMX8MQ, + .gpr = "fsl,imx8mq-iomuxc-gpr", }, [IMX8MM] = { .variant = IMX8MM, .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .gpr = "fsl,imx8mm-iomuxc-gpr", + }, + [IMX8MP] = { + .variant = IMX8MP, + .flags = IMX6_PCIE_FLAG_SUPPORTS_SUSPEND, + .gpr = "fsl,imx8mp-iomuxc-gpr", }, }; @@ -1329,6 +1351,7 @@ static const struct of_device_id imx6_pcie_of_match[] = { { .compatible = "fsl,imx7d-pcie", .data = &drvdata[IMX7D], }, { .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], }, { .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], }, + { .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], }, {}, }; diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 7746f94a715f54b384d21fe47ae46cec94fdd968..39f3b37d4033ceb17471cbb6a5e76833deacc8db 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -267,15 +267,6 @@ static void dw_pcie_free_msi(struct dw_pcie_rp *pp) irq_domain_remove(pp->msi_domain); irq_domain_remove(pp->irq_domain); - - if (pp->msi_data) { - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct device *dev = pci->dev; - - dma_unmap_page(dev, pp->msi_data, PAGE_SIZE, DMA_FROM_DEVICE); - if (pp->msi_page) - __free_page(pp->msi_page); - } } static void dw_pcie_msi_init(struct dw_pcie_rp *pp) @@ -336,6 +327,7 @@ static int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct device *dev = pci->dev; struct platform_device *pdev = to_platform_device(dev); + u64 *msi_vaddr; int ret; u32 ctrl, num_ctrls; @@ -375,22 +367,16 @@ static int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) dw_chained_msi_isr, pp); } - ret = dma_set_mask(dev, DMA_BIT_MASK(32)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (ret) dev_warn(dev, "Failed to set DMA mask to 32-bit. Devices with only 32-bit MSI support may not work properly\n"); - pp->msi_page = alloc_page(GFP_DMA32); - pp->msi_data = dma_map_page(dev, pp->msi_page, 0, - PAGE_SIZE, DMA_FROM_DEVICE); - ret = dma_mapping_error(dev, pp->msi_data); - if (ret) { - dev_err(pci->dev, "Failed to map MSI data\n"); - __free_page(pp->msi_page); - pp->msi_page = NULL; - pp->msi_data = 0; + msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, + GFP_KERNEL); + if (!msi_vaddr) { + dev_err(dev, "Failed to alloc and map MSI data\n"); dw_pcie_free_msi(pp); - - return ret; + return -ENOMEM; } return 0; diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index 09b887093a84fe60b11e3959869029ed18b0a2b6..a871ae7eb59eca4cf2ded8ab4368e4227f3c5755 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -243,7 +243,6 @@ struct dw_pcie_rp { struct irq_domain *irq_domain; struct irq_domain *msi_domain; dma_addr_t msi_data; - struct page *msi_page; struct irq_chip *msi_irq_chip; u32 num_vectors; u32 irq_mask[MAX_MSI_CTRLS]; diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c index 7f67aad71df4e0dd041565a1504948c7b51bf405..d09507f822a7d79c37e501f5005ceaae5579704f 100644 --- a/drivers/pci/controller/dwc/pcie-kirin.c +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -366,12 +367,11 @@ static int kirin_pcie_get_gpio_enable(struct kirin_pcie *pcie, struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; char name[32]; int ret, i; /* This is an optional property */ - ret = of_gpio_named_count(np, "hisilicon,clken-gpios"); + ret = gpiod_count(dev, "hisilicon,clken"); if (ret < 0) return 0; diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c index ec99116ad05c89f8d7da49baf628ca4723fdafbf..6d0d1b759ca24a8a85f042090a25e6f1951aa36b 100644 --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -26,6 +27,7 @@ #define PARF_SYS_CTRL 0x00 #define PARF_DB_CTRL 0x10 #define PARF_PM_CTRL 0x20 +#define PARF_MHI_CLOCK_RESET_CTRL 0x174 #define PARF_MHI_BASE_ADDR_LOWER 0x178 #define PARF_MHI_BASE_ADDR_UPPER 0x17c #define PARF_DEBUG_INT_EN 0x190 @@ -45,6 +47,11 @@ #define PARF_ATU_BASE_ADDR 0x634 #define PARF_ATU_BASE_ADDR_HI 0x638 #define PARF_SRIS_MODE 0x644 +#define PARF_DEBUG_CNT_PM_LINKST_IN_L2 0xc04 +#define PARF_DEBUG_CNT_PM_LINKST_IN_L1 0xc0c +#define PARF_DEBUG_CNT_PM_LINKST_IN_L0S 0xc10 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1 0xc84 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2 0xc88 #define PARF_DEVICE_TYPE 0x1000 #define PARF_BDF_TO_SID_CFG 0x2c00 @@ -83,6 +90,9 @@ #define PARF_PM_CTRL_READY_ENTR_L23 BIT(2) #define PARF_PM_CTRL_REQ_NOT_ENTR_L1 BIT(5) +/* PARF_MHI_CLOCK_RESET_CTRL fields */ +#define PARF_MSTR_AXI_CLK_EN BIT(1) + /* PARF_AXI_MSTR_RD_HALT_NO_WRITES register fields */ #define PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN BIT(0) @@ -95,6 +105,7 @@ /* PARF_SYS_CTRL register fields */ #define PARF_SYS_CTRL_AUX_PWR_DET BIT(4) #define PARF_SYS_CTRL_CORE_CLK_CGC_DIS BIT(6) +#define PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS BIT(10) #define PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE BIT(11) /* PARF_DB_CTRL register fields */ @@ -130,21 +141,33 @@ enum qcom_pcie_ep_link_status { QCOM_PCIE_EP_LINK_DOWN, }; -static struct clk_bulk_data qcom_pcie_ep_clks[] = { - { .id = "cfg" }, - { .id = "aux" }, - { .id = "bus_master" }, - { .id = "bus_slave" }, - { .id = "ref" }, - { .id = "sleep" }, - { .id = "slave_q2a" }, -}; - +/** + * struct qcom_pcie_ep - Qualcomm PCIe Endpoint Controller + * @pci: Designware PCIe controller struct + * @parf: Qualcomm PCIe specific PARF register base + * @elbi: Designware PCIe specific ELBI register base + * @mmio: MMIO register base + * @perst_map: PERST regmap + * @mmio_res: MMIO region resource + * @core_reset: PCIe Endpoint core reset + * @reset: PERST# GPIO + * @wake: WAKE# GPIO + * @phy: PHY controller block + * @debugfs: PCIe Endpoint Debugfs directory + * @clks: PCIe clocks + * @num_clks: PCIe clocks count + * @perst_en: Flag for PERST enable + * @perst_sep_en: Flag for PERST separation enable + * @link_status: PCIe Link status + * @global_irq: Qualcomm PCIe specific Global IRQ + * @perst_irq: PERST# IRQ + */ struct qcom_pcie_ep { struct dw_pcie pci; void __iomem *parf; void __iomem *elbi; + void __iomem *mmio; struct regmap *perst_map; struct resource *mmio_res; @@ -152,6 +175,10 @@ struct qcom_pcie_ep { struct gpio_desc *reset; struct gpio_desc *wake; struct phy *phy; + struct dentry *debugfs; + + struct clk_bulk_data *clks; + int num_clks; u32 perst_en; u32 perst_sep_en; @@ -193,8 +220,10 @@ static int qcom_pcie_ep_core_reset(struct qcom_pcie_ep *pcie_ep) */ static void qcom_pcie_ep_configure_tcsr(struct qcom_pcie_ep *pcie_ep) { - regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0); - regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0); + if (pcie_ep->perst_map) { + regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0); + regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0); + } } static int qcom_pcie_dw_link_up(struct dw_pcie *pci) @@ -227,8 +256,7 @@ static int qcom_pcie_enable_resources(struct qcom_pcie_ep *pcie_ep) { int ret; - ret = clk_bulk_prepare_enable(ARRAY_SIZE(qcom_pcie_ep_clks), - qcom_pcie_ep_clks); + ret = clk_bulk_prepare_enable(pcie_ep->num_clks, pcie_ep->clks); if (ret) return ret; @@ -249,8 +277,7 @@ static int qcom_pcie_enable_resources(struct qcom_pcie_ep *pcie_ep) err_phy_exit: phy_exit(pcie_ep->phy); err_disable_clk: - clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), - qcom_pcie_ep_clks); + clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks); return ret; } @@ -259,8 +286,7 @@ static void qcom_pcie_disable_resources(struct qcom_pcie_ep *pcie_ep) { phy_power_off(pcie_ep->phy); phy_exit(pcie_ep->phy); - clk_bulk_disable_unprepare(ARRAY_SIZE(qcom_pcie_ep_clks), - qcom_pcie_ep_clks); + clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks); } static int qcom_pcie_perst_deassert(struct dw_pcie *pci) @@ -318,8 +344,14 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) val &= ~PARF_Q2A_FLUSH_EN; writel_relaxed(val, pcie_ep->parf + PARF_Q2A_FLUSH); - /* Disable DBI Wakeup, core clock CGC and enable AUX power */ + /* + * Disable Master AXI clock during idle. Do not allow DBI access + * to take the core out of L1. Disable core clock gating that + * gates PIPE clock from propagating to core clock. Report to the + * host that Vaux is present. + */ val = readl_relaxed(pcie_ep->parf + PARF_SYS_CTRL); + val &= ~PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS; val |= PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE | PARF_SYS_CTRL_CORE_CLK_CGC_DIS | PARF_SYS_CTRL_AUX_PWR_DET; @@ -375,6 +407,11 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) pcie_ep->parf + PARF_MHI_BASE_ADDR_LOWER); writel_relaxed(0, pcie_ep->parf + PARF_MHI_BASE_ADDR_UPPER); + /* Gate Master AXI clock to MHI bus during L1SS */ + val = readl_relaxed(pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL); + val &= ~PARF_MSTR_AXI_CLK_EN; + val = readl_relaxed(pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL); + dw_pcie_ep_init_notify(&pcie_ep->pci.ep); /* Enable LTSSM */ @@ -437,11 +474,19 @@ static int qcom_pcie_ep_get_io_resources(struct platform_device *pdev, pcie_ep->mmio_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mmio"); + if (!pcie_ep->mmio_res) { + dev_err(dev, "Failed to get mmio resource\n"); + return -EINVAL; + } + + pcie_ep->mmio = devm_pci_remap_cfg_resource(dev, pcie_ep->mmio_res); + if (IS_ERR(pcie_ep->mmio)) + return PTR_ERR(pcie_ep->mmio); syscon = of_parse_phandle(dev->of_node, "qcom,perst-regs", 0); if (!syscon) { - dev_err(dev, "Failed to parse qcom,perst-regs\n"); - return -EINVAL; + dev_dbg(dev, "PERST separation not available\n"); + return 0; } pcie_ep->perst_map = syscon_node_to_regmap(syscon); @@ -474,14 +519,15 @@ static int qcom_pcie_ep_get_resources(struct platform_device *pdev, ret = qcom_pcie_ep_get_io_resources(pdev, pcie_ep); if (ret) { - dev_err(&pdev->dev, "Failed to get io resources %d\n", ret); + dev_err(dev, "Failed to get io resources %d\n", ret); return ret; } - ret = devm_clk_bulk_get(dev, ARRAY_SIZE(qcom_pcie_ep_clks), - qcom_pcie_ep_clks); - if (ret) - return ret; + pcie_ep->num_clks = devm_clk_bulk_get_all(dev, &pcie_ep->clks); + if (pcie_ep->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return pcie_ep->num_clks; + } pcie_ep->core_reset = devm_reset_control_get_exclusive(dev, "core"); if (IS_ERR(pcie_ep->core_reset)) @@ -495,7 +541,7 @@ static int qcom_pcie_ep_get_resources(struct platform_device *pdev, if (IS_ERR(pcie_ep->wake)) return PTR_ERR(pcie_ep->wake); - pcie_ep->phy = devm_phy_optional_get(&pdev->dev, "pciephy"); + pcie_ep->phy = devm_phy_optional_get(dev, "pciephy"); if (IS_ERR(pcie_ep->phy)) ret = PTR_ERR(pcie_ep->phy); @@ -571,13 +617,13 @@ static irqreturn_t qcom_pcie_ep_perst_irq_thread(int irq, void *data) static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, struct qcom_pcie_ep *pcie_ep) { - int irq, ret; + int ret; - irq = platform_get_irq_byname(pdev, "global"); - if (irq < 0) - return irq; + pcie_ep->global_irq = platform_get_irq_byname(pdev, "global"); + if (pcie_ep->global_irq < 0) + return pcie_ep->global_irq; - ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->global_irq, NULL, qcom_pcie_ep_global_irq_thread, IRQF_ONESHOT, "global_irq", pcie_ep); @@ -594,7 +640,7 @@ static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, "perst_irq", pcie_ep); if (ret) { dev_err(&pdev->dev, "Failed to request PERST IRQ\n"); - disable_irq(irq); + disable_irq(pcie_ep->global_irq); return ret; } @@ -617,6 +663,37 @@ static int qcom_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } } +static int qcom_pcie_ep_link_transition_count(struct seq_file *s, void *data) +{ + struct qcom_pcie_ep *pcie_ep = (struct qcom_pcie_ep *) + dev_get_drvdata(s->private); + + seq_printf(s, "L0s transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L0S)); + + seq_printf(s, "L1 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L1)); + + seq_printf(s, "L1.1 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1)); + + seq_printf(s, "L1.2 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2)); + + seq_printf(s, "L2 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L2)); + + return 0; +} + +static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + + debugfs_create_devm_seqfile(pci->dev, "link_transition_count", pcie_ep->debugfs, + qcom_pcie_ep_link_transition_count); +} + static const struct pci_epc_features qcom_pcie_epc_features = { .linkup_notifier = true, .core_init_notifier = true, @@ -649,6 +726,7 @@ static int qcom_pcie_ep_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct qcom_pcie_ep *pcie_ep; + char *name; int ret; pcie_ep = devm_kzalloc(dev, sizeof(*pcie_ep), GFP_KERNEL); @@ -680,8 +758,21 @@ static int qcom_pcie_ep_probe(struct platform_device *pdev) if (ret) goto err_disable_resources; + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) { + ret = -ENOMEM; + goto err_disable_irqs; + } + + pcie_ep->debugfs = debugfs_create_dir(name, NULL); + qcom_pcie_ep_init_debugfs(pcie_ep); + return 0; +err_disable_irqs: + disable_irq(pcie_ep->global_irq); + disable_irq(pcie_ep->perst_irq); + err_disable_resources: qcom_pcie_disable_resources(pcie_ep); @@ -692,6 +783,11 @@ static int qcom_pcie_ep_remove(struct platform_device *pdev) { struct qcom_pcie_ep *pcie_ep = platform_get_drvdata(pdev); + disable_irq(pcie_ep->global_irq); + disable_irq(pcie_ep->perst_irq); + + debugfs_remove_recursive(pcie_ep->debugfs); + if (pcie_ep->link_status == QCOM_PCIE_EP_LINK_DISABLED) return 0; @@ -702,8 +798,10 @@ static int qcom_pcie_ep_remove(struct platform_device *pdev) static const struct of_device_id qcom_pcie_ep_match[] = { { .compatible = "qcom,sdx55-pcie-ep", }, + { .compatible = "qcom,sm8450-pcie-ep", }, { } }; +MODULE_DEVICE_TABLE(of, qcom_pcie_ep_match); static struct platform_driver qcom_pcie_ep_driver = { .probe = qcom_pcie_ep_probe, diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 66886dc6e777f975a0ceb3305bd64c97e7162892..f711acacaeaf8f163a399a8922a72685863b223c 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -180,7 +180,7 @@ struct qcom_pcie_resources_2_3_3 { /* 6 clocks typically, 7 for sm8250 */ struct qcom_pcie_resources_2_7_0 { - struct clk_bulk_data clks[9]; + struct clk_bulk_data clks[12]; int num_clks; struct regulator_bulk_data supplies[2]; struct reset_control *pci_reset; @@ -208,17 +208,12 @@ struct qcom_pcie_ops { int (*init)(struct qcom_pcie *pcie); int (*post_init)(struct qcom_pcie *pcie); void (*deinit)(struct qcom_pcie *pcie); - void (*post_deinit)(struct qcom_pcie *pcie); void (*ltssm_enable)(struct qcom_pcie *pcie); int (*config_sid)(struct qcom_pcie *pcie); }; struct qcom_pcie_cfg { const struct qcom_pcie_ops *ops; - unsigned int has_tbu_clk:1; - unsigned int has_ddrss_sf_tbu_clk:1; - unsigned int has_aggre0_clk:1; - unsigned int has_aggre1_clk:1; }; struct qcom_pcie { @@ -1175,6 +1170,7 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; struct dw_pcie *pci = pcie->pci; struct device *dev = pci->dev; + unsigned int num_clks, num_opt_clks; unsigned int idx; int ret; @@ -1195,18 +1191,25 @@ static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) res->clks[idx++].id = "bus_master"; res->clks[idx++].id = "bus_slave"; res->clks[idx++].id = "slave_q2a"; - if (pcie->cfg->has_tbu_clk) - res->clks[idx++].id = "tbu"; - if (pcie->cfg->has_ddrss_sf_tbu_clk) - res->clks[idx++].id = "ddrss_sf_tbu"; - if (pcie->cfg->has_aggre0_clk) - res->clks[idx++].id = "aggre0"; - if (pcie->cfg->has_aggre1_clk) - res->clks[idx++].id = "aggre1"; + num_clks = idx; + + ret = devm_clk_bulk_get(dev, num_clks, res->clks); + if (ret < 0) + return ret; + + res->clks[idx++].id = "tbu"; + res->clks[idx++].id = "ddrss_sf_tbu"; + res->clks[idx++].id = "aggre0"; + res->clks[idx++].id = "aggre1"; + res->clks[idx++].id = "noc_aggr_4"; + res->clks[idx++].id = "noc_aggr_south_sf"; + res->clks[idx++].id = "cnoc_qx"; + + num_opt_clks = idx - num_clks; res->num_clks = idx; - ret = devm_clk_bulk_get(dev, res->num_clks, res->clks); + ret = devm_clk_bulk_get_optional(dev, num_opt_clks, res->clks + num_clks); if (ret < 0) return ret; @@ -1509,15 +1512,13 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp) if (pcie->cfg->ops->config_sid) { ret = pcie->cfg->ops->config_sid(pcie); if (ret) - goto err; + goto err_assert_reset; } return 0; -err: +err_assert_reset: qcom_ep_reset_assert(pcie); - if (pcie->cfg->ops->post_deinit) - pcie->cfg->ops->post_deinit(pcie); err_disable_phy: phy_power_off(pcie->phy); err_deinit: @@ -1601,68 +1602,35 @@ static const struct qcom_pcie_ops ops_2_9_0 = { .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, }; -static const struct qcom_pcie_cfg apq8084_cfg = { +static const struct qcom_pcie_cfg cfg_1_0_0 = { .ops = &ops_1_0_0, }; -static const struct qcom_pcie_cfg ipq8064_cfg = { +static const struct qcom_pcie_cfg cfg_1_9_0 = { + .ops = &ops_1_9_0, +}; + +static const struct qcom_pcie_cfg cfg_2_1_0 = { .ops = &ops_2_1_0, }; -static const struct qcom_pcie_cfg msm8996_cfg = { +static const struct qcom_pcie_cfg cfg_2_3_2 = { .ops = &ops_2_3_2, }; -static const struct qcom_pcie_cfg ipq8074_cfg = { +static const struct qcom_pcie_cfg cfg_2_3_3 = { .ops = &ops_2_3_3, }; -static const struct qcom_pcie_cfg ipq4019_cfg = { +static const struct qcom_pcie_cfg cfg_2_4_0 = { .ops = &ops_2_4_0, }; -static const struct qcom_pcie_cfg sdm845_cfg = { +static const struct qcom_pcie_cfg cfg_2_7_0 = { .ops = &ops_2_7_0, - .has_tbu_clk = true, -}; - -static const struct qcom_pcie_cfg sm8150_cfg = { - /* sm8150 has qcom IP rev 1.5.0. However 1.5.0 ops are same as - * 1.9.0, so reuse the same. - */ - .ops = &ops_1_9_0, -}; - -static const struct qcom_pcie_cfg sm8250_cfg = { - .ops = &ops_1_9_0, - .has_tbu_clk = true, - .has_ddrss_sf_tbu_clk = true, -}; - -static const struct qcom_pcie_cfg sm8450_pcie0_cfg = { - .ops = &ops_1_9_0, - .has_ddrss_sf_tbu_clk = true, - .has_aggre0_clk = true, - .has_aggre1_clk = true, -}; - -static const struct qcom_pcie_cfg sm8450_pcie1_cfg = { - .ops = &ops_1_9_0, - .has_ddrss_sf_tbu_clk = true, - .has_aggre1_clk = true, -}; - -static const struct qcom_pcie_cfg sc7280_cfg = { - .ops = &ops_1_9_0, - .has_tbu_clk = true, -}; - -static const struct qcom_pcie_cfg sc8180x_cfg = { - .ops = &ops_1_9_0, - .has_tbu_clk = true, }; -static const struct qcom_pcie_cfg ipq6018_cfg = { +static const struct qcom_pcie_cfg cfg_2_9_0 = { .ops = &ops_2_9_0, }; @@ -1761,22 +1729,24 @@ err_pm_runtime_put: } static const struct of_device_id qcom_pcie_match[] = { - { .compatible = "qcom,pcie-apq8084", .data = &apq8084_cfg }, - { .compatible = "qcom,pcie-ipq8064", .data = &ipq8064_cfg }, - { .compatible = "qcom,pcie-ipq8064-v2", .data = &ipq8064_cfg }, - { .compatible = "qcom,pcie-apq8064", .data = &ipq8064_cfg }, - { .compatible = "qcom,pcie-msm8996", .data = &msm8996_cfg }, - { .compatible = "qcom,pcie-ipq8074", .data = &ipq8074_cfg }, - { .compatible = "qcom,pcie-ipq4019", .data = &ipq4019_cfg }, - { .compatible = "qcom,pcie-qcs404", .data = &ipq4019_cfg }, - { .compatible = "qcom,pcie-sdm845", .data = &sdm845_cfg }, - { .compatible = "qcom,pcie-sm8150", .data = &sm8150_cfg }, - { .compatible = "qcom,pcie-sm8250", .data = &sm8250_cfg }, - { .compatible = "qcom,pcie-sc8180x", .data = &sc8180x_cfg }, - { .compatible = "qcom,pcie-sm8450-pcie0", .data = &sm8450_pcie0_cfg }, - { .compatible = "qcom,pcie-sm8450-pcie1", .data = &sm8450_pcie1_cfg }, - { .compatible = "qcom,pcie-sc7280", .data = &sc7280_cfg }, - { .compatible = "qcom,pcie-ipq6018", .data = &ipq6018_cfg }, + { .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 }, + { .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 }, + { .compatible = "qcom,pcie-ipq6018", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-ipq8064", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 }, + { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 }, + { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 }, + { .compatible = "qcom,pcie-sa8540p", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sc7280", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sc8280xp", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 }, + { .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 }, { } }; diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c index 966c8b48bd96937de4000f59bc512e1aa0905ea2..ba36bbc5897d0023fcbb901ed08bba049602451f 100644 --- a/drivers/pci/controller/pci-aardvark.c +++ b/drivers/pci/controller/pci-aardvark.c @@ -33,6 +33,7 @@ #define PCIE_CORE_DEV_ID_REG 0x0 #define PCIE_CORE_CMD_STATUS_REG 0x4 #define PCIE_CORE_DEV_REV_REG 0x8 +#define PCIE_CORE_SSDEV_ID_REG 0x2c #define PCIE_CORE_PCIEXP_CAP 0xc0 #define PCIE_CORE_PCIERR_CAP 0x100 #define PCIE_CORE_ERR_CAPCTL_REG 0x118 @@ -1077,7 +1078,10 @@ static int advk_sw_pci_bridge_init(struct advk_pcie *pcie) /* Indicates supports for Completion Retry Status */ bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_CRSVIS); + bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff; + bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16; bridge->has_pcie = true; + bridge->pcie_start = PCIE_CORE_PCIEXP_CAP; bridge->data = pcie; bridge->ops = &advk_pci_bridge_emul_ops; diff --git a/drivers/pci/controller/pci-ftpci100.c b/drivers/pci/controller/pci-ftpci100.c index 88980a44461df681e2284007c165fbdee2bfe83a..0cfd9d5a497c9d7c8cbbef2200f9d495b835a1fd 100644 --- a/drivers/pci/controller/pci-ftpci100.c +++ b/drivers/pci/controller/pci-ftpci100.c @@ -103,13 +103,6 @@ #define FARADAY_PCI_DMA_MEM2_BASE 0x00000000 #define FARADAY_PCI_DMA_MEM3_BASE 0x00000000 -/* Defines for PCI configuration command register */ -#define PCI_CONF_ENABLE BIT(31) -#define PCI_CONF_WHERE(r) ((r) & 0xFC) -#define PCI_CONF_BUS(b) (((b) & 0xFF) << 16) -#define PCI_CONF_DEVICE(d) (((d) & 0x1F) << 11) -#define PCI_CONF_FUNCTION(f) (((f) & 0x07) << 8) - /** * struct faraday_pci_variant - encodes IP block differences * @cascaded_irq: this host has cascaded IRQs from an interrupt controller @@ -190,11 +183,8 @@ static int faraday_raw_pci_read_config(struct faraday_pci *p, int bus_number, unsigned int fn, int config, int size, u32 *value) { - writel(PCI_CONF_BUS(bus_number) | - PCI_CONF_DEVICE(PCI_SLOT(fn)) | - PCI_CONF_FUNCTION(PCI_FUNC(fn)) | - PCI_CONF_WHERE(config) | - PCI_CONF_ENABLE, + writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn), + PCI_FUNC(fn), config), p->base + FTPCI_CONFIG); *value = readl(p->base + FTPCI_DATA); @@ -225,11 +215,8 @@ static int faraday_raw_pci_write_config(struct faraday_pci *p, int bus_number, { int ret = PCIBIOS_SUCCESSFUL; - writel(PCI_CONF_BUS(bus_number) | - PCI_CONF_DEVICE(PCI_SLOT(fn)) | - PCI_CONF_FUNCTION(PCI_FUNC(fn)) | - PCI_CONF_WHERE(config) | - PCI_CONF_ENABLE, + writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn), + PCI_FUNC(fn), config), p->base + FTPCI_CONFIG); switch (size) { diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c index af915c951f0660d1eef83bd09c3ef42124569d23..1ced73726a2671a4ae8bf851871c86bc52044180 100644 --- a/drivers/pci/controller/pci-mvebu.c +++ b/drivers/pci/controller/pci-mvebu.c @@ -523,7 +523,7 @@ static int mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) /* Are the new iobase/iolimit values invalid? */ if (conf->iolimit < conf->iobase || - conf->iolimitupper < conf->iobaseupper) + le16_to_cpu(conf->iolimitupper) < le16_to_cpu(conf->iobaseupper)) return mvebu_pcie_set_window(port, port->io_target, port->io_attr, &desired, &port->iowin); @@ -535,10 +535,10 @@ static int mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) * is the CPU address. */ desired.remap = ((conf->iobase & 0xF0) << 8) | - (conf->iobaseupper << 16); + (le16_to_cpu(conf->iobaseupper) << 16); desired.base = port->pcie->io.start + desired.remap; desired.size = ((0xFFF | ((conf->iolimit & 0xF0) << 8) | - (conf->iolimitupper << 16)) - + (le16_to_cpu(conf->iolimitupper) << 16)) - desired.remap) + 1; @@ -552,7 +552,7 @@ static int mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) struct pci_bridge_emul_conf *conf = &port->bridge.conf; /* Are the new membase/memlimit values invalid? */ - if (conf->memlimit < conf->membase) + if (le16_to_cpu(conf->memlimit) < le16_to_cpu(conf->membase)) return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, &desired, &port->memwin); @@ -562,8 +562,8 @@ static int mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) * window to setup, according to the PCI-to-PCI bridge * specifications. */ - desired.base = ((conf->membase & 0xFFF0) << 16); - desired.size = (((conf->memlimit & 0xFFF0) << 16) | 0xFFFFF) - + desired.base = ((le16_to_cpu(conf->membase) & 0xFFF0) << 16); + desired.size = (((le16_to_cpu(conf->memlimit) & 0xFFF0) << 16) | 0xFFFFF) - desired.base + 1; return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, &desired, @@ -946,6 +946,7 @@ static int mvebu_pci_bridge_emul_init(struct mvebu_pcie_port *port) bridge->subsystem_vendor_id = ssdev_id & 0xffff; bridge->subsystem_id = ssdev_id >> 16; bridge->has_pcie = true; + bridge->pcie_start = PCIE_CAP_PCIEXP; bridge->data = port; bridge->ops = &mvebu_pci_bridge_emul_ops; diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c index 8e323e93be91576b3bb988c57696e37d858c4cbd..24478ae5a345d1fc3145e736f55dfa7c5b479ab7 100644 --- a/drivers/pci/controller/pci-tegra.c +++ b/drivers/pci/controller/pci-tegra.c @@ -415,13 +415,6 @@ static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset) * address (access to which generates correct config transaction) falls in * this 4 KiB region. */ -static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn, - unsigned int where) -{ - return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) | - (PCI_FUNC(devfn) << 8) | (where & 0xff); -} - static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, int where) @@ -443,7 +436,9 @@ static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus, unsigned int offset; u32 base; - offset = tegra_pcie_conf_offset(bus->number, devfn, where); + offset = PCI_CONF1_EXT_ADDRESS(bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), where) & + ~PCI_CONF1_ENABLE; /* move 4 KiB window to offset within the FPCI region */ base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index a2c3c207a04b70e01a46a9f77705c56f6c3b7944..66f37e403a09c31dd9d3549c728cf48e47c2b1e4 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -516,8 +516,8 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, u32 stat, idx; int ret, i; - reset = gpiod_get_from_of_node(np, "reset-gpios", 0, - GPIOD_OUT_LOW, "PERST#"); + reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_LOW, "PERST#"); if (IS_ERR(reset)) return PTR_ERR(reset); diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index 11cdb9b6f1094769fa568b0cfebf386fbaad78d6..b8612ce5f4d0cf82c23198bf5b802e09d1412145 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -1071,7 +1071,7 @@ static struct platform_driver mtk_pcie_driver = { .probe = mtk_pcie_probe, .remove = mtk_pcie_remove, .driver = { - .name = "mtk-pcie", + .name = "mtk-pcie-gen3", .of_match_table = mtk_pcie_of_match, .pm = &mtk_pcie_pm_ops, }, diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c index 33eb37a2225c14c7e9be5f53c45b9ca7cb96bef1..4bd1abf26008f7205f9a19b4772de97a1ea16d40 100644 --- a/drivers/pci/controller/pcie-mt7621.c +++ b/drivers/pci/controller/pcie-mt7621.c @@ -30,6 +30,8 @@ #include #include +#include "../pci.h" + /* MediaTek-specific configuration registers */ #define PCIE_FTS_NUM 0x70c #define PCIE_FTS_NUM_MASK GENMASK(15, 8) @@ -120,19 +122,12 @@ static inline void pcie_port_write(struct mt7621_pcie_port *port, writel_relaxed(val, port->base + reg); } -static inline u32 mt7621_pcie_get_cfgaddr(unsigned int bus, unsigned int slot, - unsigned int func, unsigned int where) -{ - return (((where & 0xf00) >> 8) << 24) | (bus << 16) | (slot << 11) | - (func << 8) | (where & 0xfc) | 0x80000000; -} - static void __iomem *mt7621_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, int where) { struct mt7621_pcie *pcie = bus->sysdata; - u32 address = mt7621_pcie_get_cfgaddr(bus->number, PCI_SLOT(devfn), - PCI_FUNC(devfn), where); + u32 address = PCI_CONF1_EXT_ADDRESS(bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), where); writel_relaxed(address, pcie->base + RALINK_PCI_CONFIG_ADDR); @@ -147,7 +142,7 @@ static struct pci_ops mt7621_pcie_ops = { static u32 read_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg) { - u32 address = mt7621_pcie_get_cfgaddr(0, dev, 0, reg); + u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg); pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); return pcie_read(pcie, RALINK_PCI_CONFIG_DATA); @@ -156,7 +151,7 @@ static u32 read_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg) static void write_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg, u32 val) { - u32 address = mt7621_pcie_get_cfgaddr(0, dev, 0, reg); + u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg); pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); pcie_write(pcie, val, RALINK_PCI_CONFIG_DATA); diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 9037a7827eca75918a6376bc0ab38e3905fa7fe2..fdd2ec09651e96f8eb54d39965918c7b8bdb63c3 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -526,7 +526,7 @@ static int msix_setup_msi_descs(struct pci_dev *dev, void __iomem *base, desc.pci.msi_attrib.can_mask = !pci_msi_ignore_mask && !desc.pci.msi_attrib.is_virtual; - if (!desc.pci.msi_attrib.can_mask) { + if (desc.pci.msi_attrib.can_mask) { addr = pci_msix_desc_addr(&desc); desc.pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 4496a7c5c4785377f9ce2487324d2a44091866f5..88dc66ee1c467ea5ec04d317e738cf571aaf3b91 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -649,7 +649,7 @@ struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients) if (!closest_pdevs) return NULL; - while ((pdev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) { + for_each_pci_dev(pdev) { if (!pci_has_p2pmem(pdev)) continue; diff --git a/drivers/pci/pci-bridge-emul.c b/drivers/pci/pci-bridge-emul.c index 9c2ca28e3ecf0c8d970752e762537ee9362a7e5d..9334b2dd47641ba5c03814b6eb328778174bbdbf 100644 --- a/drivers/pci/pci-bridge-emul.c +++ b/drivers/pci/pci-bridge-emul.c @@ -22,11 +22,7 @@ #define PCI_BRIDGE_CONF_END PCI_STD_HEADER_SIZEOF #define PCI_CAP_SSID_SIZEOF (PCI_SSVID_DEVICE_ID + 2) -#define PCI_CAP_SSID_START PCI_BRIDGE_CONF_END -#define PCI_CAP_SSID_END (PCI_CAP_SSID_START + PCI_CAP_SSID_SIZEOF) #define PCI_CAP_PCIE_SIZEOF (PCI_EXP_SLTSTA2 + 2) -#define PCI_CAP_PCIE_START PCI_CAP_SSID_END -#define PCI_CAP_PCIE_END (PCI_CAP_PCIE_START + PCI_CAP_PCIE_SIZEOF) /** * struct pci_bridge_reg_behavior - register bits behaviors @@ -324,7 +320,7 @@ pci_bridge_emul_read_ssid(struct pci_bridge_emul *bridge, int reg, u32 *value) switch (reg) { case PCI_CAP_LIST_ID: *value = PCI_CAP_ID_SSVID | - (bridge->has_pcie ? (PCI_CAP_PCIE_START << 8) : 0); + ((bridge->pcie_start > bridge->ssid_start) ? (bridge->pcie_start << 8) : 0); return PCI_BRIDGE_EMUL_HANDLED; case PCI_SSVID_VENDOR_ID: @@ -365,18 +361,33 @@ int pci_bridge_emul_init(struct pci_bridge_emul *bridge, if (!bridge->pci_regs_behavior) return -ENOMEM; - if (bridge->subsystem_vendor_id) - bridge->conf.capabilities_pointer = PCI_CAP_SSID_START; - else if (bridge->has_pcie) - bridge->conf.capabilities_pointer = PCI_CAP_PCIE_START; - else - bridge->conf.capabilities_pointer = 0; + /* If ssid_start and pcie_start were not specified then choose the lowest possible value. */ + if (!bridge->ssid_start && !bridge->pcie_start) { + if (bridge->subsystem_vendor_id) + bridge->ssid_start = PCI_BRIDGE_CONF_END; + if (bridge->has_pcie) + bridge->pcie_start = bridge->ssid_start + PCI_CAP_SSID_SIZEOF; + } else if (!bridge->ssid_start && bridge->subsystem_vendor_id) { + if (bridge->pcie_start - PCI_BRIDGE_CONF_END >= PCI_CAP_SSID_SIZEOF) + bridge->ssid_start = PCI_BRIDGE_CONF_END; + else + bridge->ssid_start = bridge->pcie_start + PCI_CAP_PCIE_SIZEOF; + } else if (!bridge->pcie_start && bridge->has_pcie) { + if (bridge->ssid_start - PCI_BRIDGE_CONF_END >= PCI_CAP_PCIE_SIZEOF) + bridge->pcie_start = PCI_BRIDGE_CONF_END; + else + bridge->pcie_start = bridge->ssid_start + PCI_CAP_SSID_SIZEOF; + } + + bridge->conf.capabilities_pointer = min(bridge->ssid_start, bridge->pcie_start); if (bridge->conf.capabilities_pointer) bridge->conf.status |= cpu_to_le16(PCI_STATUS_CAP_LIST); if (bridge->has_pcie) { bridge->pcie_conf.cap_id = PCI_CAP_ID_EXP; + bridge->pcie_conf.next = (bridge->ssid_start > bridge->pcie_start) ? + bridge->ssid_start : 0; bridge->pcie_conf.cap |= cpu_to_le16(PCI_EXP_TYPE_ROOT_PORT << 4); bridge->pcie_cap_regs_behavior = kmemdup(pcie_cap_regs_behavior, @@ -459,15 +470,17 @@ int pci_bridge_emul_conf_read(struct pci_bridge_emul *bridge, int where, read_op = bridge->ops->read_base; cfgspace = (__le32 *) &bridge->conf; behavior = bridge->pci_regs_behavior; - } else if (reg >= PCI_CAP_SSID_START && reg < PCI_CAP_SSID_END && bridge->subsystem_vendor_id) { + } else if (reg >= bridge->ssid_start && reg < bridge->ssid_start + PCI_CAP_SSID_SIZEOF && + bridge->subsystem_vendor_id) { /* Emulated PCI Bridge Subsystem Vendor ID capability */ - reg -= PCI_CAP_SSID_START; + reg -= bridge->ssid_start; read_op = pci_bridge_emul_read_ssid; cfgspace = NULL; behavior = NULL; - } else if (reg >= PCI_CAP_PCIE_START && reg < PCI_CAP_PCIE_END && bridge->has_pcie) { + } else if (reg >= bridge->pcie_start && reg < bridge->pcie_start + PCI_CAP_PCIE_SIZEOF && + bridge->has_pcie) { /* Our emulated PCIe capability */ - reg -= PCI_CAP_PCIE_START; + reg -= bridge->pcie_start; read_op = bridge->ops->read_pcie; cfgspace = (__le32 *) &bridge->pcie_conf; behavior = bridge->pcie_cap_regs_behavior; @@ -538,9 +551,10 @@ int pci_bridge_emul_conf_write(struct pci_bridge_emul *bridge, int where, write_op = bridge->ops->write_base; cfgspace = (__le32 *) &bridge->conf; behavior = bridge->pci_regs_behavior; - } else if (reg >= PCI_CAP_PCIE_START && reg < PCI_CAP_PCIE_END && bridge->has_pcie) { + } else if (reg >= bridge->pcie_start && reg < bridge->pcie_start + PCI_CAP_PCIE_SIZEOF && + bridge->has_pcie) { /* Our emulated PCIe capability */ - reg -= PCI_CAP_PCIE_START; + reg -= bridge->pcie_start; write_op = bridge->ops->write_pcie; cfgspace = (__le32 *) &bridge->pcie_conf; behavior = bridge->pcie_cap_regs_behavior; diff --git a/drivers/pci/pci-bridge-emul.h b/drivers/pci/pci-bridge-emul.h index 71392b67471da45a5234f77ea9cafc5a3cd7995b..2a0e59c7f0d90e16798bffc52508a943ad96a05f 100644 --- a/drivers/pci/pci-bridge-emul.h +++ b/drivers/pci/pci-bridge-emul.h @@ -131,6 +131,8 @@ struct pci_bridge_emul { struct pci_bridge_reg_behavior *pci_regs_behavior; struct pci_bridge_reg_behavior *pcie_cap_regs_behavior; void *data; + u8 pcie_start; + u8 ssid_start; bool has_pcie; u16 subsystem_vendor_id; u16 subsystem_id; diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 49238ddd39eec54a86873f2859963a80ab9568b9..107d77f3c846708f0eac6c857b757502e2cc6de0 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -774,6 +774,12 @@ static int pci_pm_suspend(struct device *dev) pci_dev->skip_bus_pm = false; + /* + * Disabling PTM allows some systems, e.g., Intel mobile chips + * since Coffee Lake, to enter a lower-power PM state. + */ + pci_suspend_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_suspend(dev, PMSG_SUSPEND); @@ -867,20 +873,15 @@ static int pci_pm_suspend_noirq(struct device *dev) } } - if (pci_dev->skip_bus_pm) { + if (!pci_dev->state_saved) { + pci_save_state(pci_dev); + /* - * Either the device is a bridge with a child in D0 below it, or - * the function is running for the second time in a row without - * going through full resume, which is possible only during - * suspend-to-idle in a spurious wakeup case. The device should - * be in D0 at this point, but if it is a bridge, it may be - * necessary to save its state. + * If the device is a bridge with a child in D0 below it, + * it needs to stay in D0, so check skip_bus_pm to avoid + * putting it into a low-power state in that case. */ - if (!pci_dev->state_saved) - pci_save_state(pci_dev); - } else if (!pci_dev->state_saved) { - pci_save_state(pci_dev); - if (pci_power_manageable(pci_dev)) + if (!pci_dev->skip_bus_pm && pci_power_manageable(pci_dev)) pci_prepare_to_sleep(pci_dev); } @@ -987,6 +988,8 @@ static int pci_pm_resume(struct device *dev) if (pci_dev->state_saved) pci_restore_standard_config(pci_dev); + pci_resume_ptm(pci_dev); + if (pci_has_legacy_pm_support(pci_dev)) return pci_legacy_resume(dev); @@ -1274,6 +1277,8 @@ static int pci_pm_runtime_suspend(struct device *dev) pci_power_t prev = pci_dev->current_state; int error; + pci_suspend_ptm(pci_dev); + /* * If pci_dev->driver is not set (unbound), we leave the device in D0, * but it may go to D3cold when the bridge above it runtime suspends. @@ -1335,6 +1340,7 @@ static int pci_pm_runtime_resume(struct device *dev) * D3cold when the bridge above it runtime suspended. */ pci_pm_default_resume_early(pci_dev); + pci_resume_ptm(pci_dev); if (!pci_dev->driver) return 0; diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index fc804e08e3cb59672b7bc851cac016a864dfff0a..0a2eeb82cebde8149c9c9b46bf2d72dfbae132ca 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "pci.h" static int sysfs_initialized; /* = 0 */ @@ -1373,6 +1374,112 @@ static const struct attribute_group pci_dev_reset_attr_group = { .is_visible = pci_dev_reset_attr_is_visible, }; +#define pci_dev_resource_resize_attr(n) \ +static ssize_t resource##n##_resize_show(struct device *dev, \ + struct device_attribute *attr, \ + char * buf) \ +{ \ + struct pci_dev *pdev = to_pci_dev(dev); \ + ssize_t ret; \ + \ + pci_config_pm_runtime_get(pdev); \ + \ + ret = sysfs_emit(buf, "%016llx\n", \ + (u64)pci_rebar_get_possible_sizes(pdev, n)); \ + \ + pci_config_pm_runtime_put(pdev); \ + \ + return ret; \ +} \ + \ +static ssize_t resource##n##_resize_store(struct device *dev, \ + struct device_attribute *attr,\ + const char *buf, size_t count)\ +{ \ + struct pci_dev *pdev = to_pci_dev(dev); \ + unsigned long size, flags; \ + int ret, i; \ + u16 cmd; \ + \ + if (kstrtoul(buf, 0, &size) < 0) \ + return -EINVAL; \ + \ + device_lock(dev); \ + if (dev->driver) { \ + ret = -EBUSY; \ + goto unlock; \ + } \ + \ + pci_config_pm_runtime_get(pdev); \ + \ + if ((pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) { \ + ret = aperture_remove_conflicting_pci_devices(pdev, \ + "resourceN_resize"); \ + if (ret) \ + goto pm_put; \ + } \ + \ + pci_read_config_word(pdev, PCI_COMMAND, &cmd); \ + pci_write_config_word(pdev, PCI_COMMAND, \ + cmd & ~PCI_COMMAND_MEMORY); \ + \ + flags = pci_resource_flags(pdev, n); \ + \ + pci_remove_resource_files(pdev); \ + \ + for (i = 0; i < PCI_STD_NUM_BARS; i++) { \ + if (pci_resource_len(pdev, i) && \ + pci_resource_flags(pdev, i) == flags) \ + pci_release_resource(pdev, i); \ + } \ + \ + ret = pci_resize_resource(pdev, n, size); \ + \ + pci_assign_unassigned_bus_resources(pdev->bus); \ + \ + if (pci_create_resource_files(pdev)) \ + pci_warn(pdev, "Failed to recreate resource files after BAR resizing\n");\ + \ + pci_write_config_word(pdev, PCI_COMMAND, cmd); \ +pm_put: \ + pci_config_pm_runtime_put(pdev); \ +unlock: \ + device_unlock(dev); \ + \ + return ret ? ret : count; \ +} \ +static DEVICE_ATTR_RW(resource##n##_resize) + +pci_dev_resource_resize_attr(0); +pci_dev_resource_resize_attr(1); +pci_dev_resource_resize_attr(2); +pci_dev_resource_resize_attr(3); +pci_dev_resource_resize_attr(4); +pci_dev_resource_resize_attr(5); + +static struct attribute *resource_resize_attrs[] = { + &dev_attr_resource0_resize.attr, + &dev_attr_resource1_resize.attr, + &dev_attr_resource2_resize.attr, + &dev_attr_resource3_resize.attr, + &dev_attr_resource4_resize.attr, + &dev_attr_resource5_resize.attr, + NULL, +}; + +static umode_t resource_resize_is_visible(struct kobject *kobj, + struct attribute *a, int n) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + + return pci_rebar_get_current_size(pdev, n) < 0 ? 0 : a->mode; +} + +static const struct attribute_group pci_dev_resource_resize_group = { + .attrs = resource_resize_attrs, + .is_visible = resource_resize_is_visible, +}; + int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev) { if (!sysfs_initialized) @@ -1494,6 +1601,7 @@ const struct attribute_group *pci_dev_groups[] = { #ifdef CONFIG_ACPI &pci_dev_acpi_attr_group, #endif + &pci_dev_resource_resize_group, NULL, }; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 95bc329e74c0e0df073c1e6ce053ffaae9240708..2127aba3550b5d611970bc8d3dccd7d5a60fa301 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -66,13 +66,15 @@ struct pci_pme_device { static void pci_dev_d3_sleep(struct pci_dev *dev) { - unsigned int delay = dev->d3hot_delay; - - if (delay < pci_pm_d3hot_delay) - delay = pci_pm_d3hot_delay; - - if (delay) - msleep(delay); + unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay); + unsigned int upper; + + if (delay_ms) { + /* Use a 20% upper bound, 1ms minimum */ + upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U); + usleep_range(delay_ms * USEC_PER_MSEC, + (delay_ms + upper) * USEC_PER_MSEC); + } } bool pci_reset_supported(struct pci_dev *dev) @@ -1663,6 +1665,7 @@ int pci_save_state(struct pci_dev *dev) return i; pci_save_ltr_state(dev); + pci_save_aspm_l1ss_state(dev); pci_save_dpc_state(dev); pci_save_aer_state(dev); pci_save_ptm_state(dev); @@ -1769,6 +1772,7 @@ void pci_restore_state(struct pci_dev *dev) * LTR itself (in the PCIe capability). */ pci_restore_ltr_state(dev); + pci_restore_aspm_l1ss_state(dev); pci_restore_pcie_state(dev); pci_restore_pasid_state(dev); @@ -2706,24 +2710,12 @@ int pci_prepare_to_sleep(struct pci_dev *dev) if (target_state == PCI_POWER_ERROR) return -EIO; - /* - * There are systems (for example, Intel mobile chips since Coffee - * Lake) where the power drawn while suspended can be significantly - * reduced by disabling PTM on PCIe root ports as this allows the - * port to enter a lower-power PM state and the SoC to reach a - * lower-power idle state as a whole. - */ - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) - pci_disable_ptm(dev); - pci_enable_wake(dev, target_state, wakeup); error = pci_set_power_state(dev, target_state); - if (error) { + if (error) pci_enable_wake(dev, target_state, false); - pci_restore_ptm_state(dev); - } return error; } @@ -2764,24 +2756,12 @@ int pci_finish_runtime_suspend(struct pci_dev *dev) if (target_state == PCI_POWER_ERROR) return -EIO; - /* - * There are systems (for example, Intel mobile chips since Coffee - * Lake) where the power drawn while suspended can be significantly - * reduced by disabling PTM on PCIe root ports as this allows the - * port to enter a lower-power PM state and the SoC to reach a - * lower-power idle state as a whole. - */ - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) - pci_disable_ptm(dev); - __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); error = pci_set_power_state(dev, target_state); - if (error) { + if (error) pci_enable_wake(dev, target_state, false); - pci_restore_ptm_state(dev); - } return error; } @@ -3485,6 +3465,11 @@ void pci_allocate_cap_save_buffers(struct pci_dev *dev) if (error) pci_err(dev, "unable to allocate suspend buffer for LTR\n"); + error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS, + 2 * sizeof(u32)); + if (error) + pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n"); + pci_allocate_vc_save_buffers(dev); } diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 785f31086313ab8c28515e377dafa18ac94732ef..b1ebb7ab8805170d224b15b292be52b092090457 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -505,13 +505,17 @@ static inline int pci_iov_bus_range(struct pci_bus *bus) #endif /* CONFIG_PCI_IOV */ #ifdef CONFIG_PCIE_PTM +void pci_ptm_init(struct pci_dev *dev); void pci_save_ptm_state(struct pci_dev *dev); void pci_restore_ptm_state(struct pci_dev *dev); -void pci_disable_ptm(struct pci_dev *dev); +void pci_suspend_ptm(struct pci_dev *dev); +void pci_resume_ptm(struct pci_dev *dev); #else +static inline void pci_ptm_init(struct pci_dev *dev) { } static inline void pci_save_ptm_state(struct pci_dev *dev) { } static inline void pci_restore_ptm_state(struct pci_dev *dev) { } -static inline void pci_disable_ptm(struct pci_dev *dev) { } +static inline void pci_suspend_ptm(struct pci_dev *dev) { } +static inline void pci_resume_ptm(struct pci_dev *dev) { } #endif unsigned long pci_cardbus_resource_alignment(struct resource *); @@ -561,10 +565,14 @@ bool pcie_wait_for_link(struct pci_dev *pdev, bool active); void pcie_aspm_init_link_state(struct pci_dev *pdev); void pcie_aspm_exit_link_state(struct pci_dev *pdev); void pcie_aspm_powersave_config_link(struct pci_dev *pdev); +void pci_save_aspm_l1ss_state(struct pci_dev *dev); +void pci_restore_aspm_l1ss_state(struct pci_dev *dev); #else static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { } static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { } static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { } +static inline void pci_save_aspm_l1ss_state(struct pci_dev *dev) { } +static inline void pci_restore_aspm_l1ss_state(struct pci_dev *dev) { } #endif #ifdef CONFIG_PCIE_ECRC @@ -575,12 +583,6 @@ static inline void pcie_set_ecrc_checking(struct pci_dev *dev) { } static inline void pcie_ecrc_get_policy(char *str) { } #endif -#ifdef CONFIG_PCIE_PTM -void pci_ptm_init(struct pci_dev *dev); -#else -static inline void pci_ptm_init(struct pci_dev *dev) { } -#endif - struct pci_dev_reset_methods { u16 vendor; u16 device; @@ -774,4 +776,49 @@ static inline pci_power_t mid_pci_get_power_state(struct pci_dev *pdev) } #endif +/* + * Config Address for PCI Configuration Mechanism #1 + * + * See PCI Local Bus Specification, Revision 3.0, + * Section 3.2.2.3.2, Figure 3-2, p. 50. + */ + +#define PCI_CONF1_BUS_SHIFT 16 /* Bus number */ +#define PCI_CONF1_DEV_SHIFT 11 /* Device number */ +#define PCI_CONF1_FUNC_SHIFT 8 /* Function number */ + +#define PCI_CONF1_BUS_MASK 0xff +#define PCI_CONF1_DEV_MASK 0x1f +#define PCI_CONF1_FUNC_MASK 0x7 +#define PCI_CONF1_REG_MASK 0xfc /* Limit aligned offset to a maximum of 256B */ + +#define PCI_CONF1_ENABLE BIT(31) +#define PCI_CONF1_BUS(x) (((x) & PCI_CONF1_BUS_MASK) << PCI_CONF1_BUS_SHIFT) +#define PCI_CONF1_DEV(x) (((x) & PCI_CONF1_DEV_MASK) << PCI_CONF1_DEV_SHIFT) +#define PCI_CONF1_FUNC(x) (((x) & PCI_CONF1_FUNC_MASK) << PCI_CONF1_FUNC_SHIFT) +#define PCI_CONF1_REG(x) ((x) & PCI_CONF1_REG_MASK) + +#define PCI_CONF1_ADDRESS(bus, dev, func, reg) \ + (PCI_CONF1_ENABLE | \ + PCI_CONF1_BUS(bus) | \ + PCI_CONF1_DEV(dev) | \ + PCI_CONF1_FUNC(func) | \ + PCI_CONF1_REG(reg)) + +/* + * Extension of PCI Config Address for accessing extended PCIe registers + * + * No standardized specification, but used on lot of non-ECAM-compliant ARM SoCs + * or on AMD Barcelona and new CPUs. Reserved bits [27:24] of PCI Config Address + * are used for specifying additional 4 high bits of PCI Express register. + */ + +#define PCI_CONF1_EXT_REG_SHIFT 16 +#define PCI_CONF1_EXT_REG_MASK 0xf00 +#define PCI_CONF1_EXT_REG(x) (((x) & PCI_CONF1_EXT_REG_MASK) << PCI_CONF1_EXT_REG_SHIFT) + +#define PCI_CONF1_EXT_ADDRESS(bus, dev, func, reg) \ + (PCI_CONF1_ADDRESS(bus, dev, func, reg) | \ + PCI_CONF1_EXT_REG(reg)) + #endif /* DRIVERS_PCI_H */ diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index a8aec190986c0a5e3bc02c6fe92911d4f7426de3..53a1fa306e1ee7b70f20581dbca928792dbe6384 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -350,29 +351,43 @@ static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) return 0; } +/* + * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1 + * register. Ports enter L1.2 when the most recent LTR value is greater + * than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we + * don't enter L1.2 too aggressively. + * + * See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3. + */ static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value) { - u32 threshold_ns = threshold_us * 1000; + u64 threshold_ns = (u64) threshold_us * 1000; - /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */ - if (threshold_ns < 32) { - *scale = 0; + /* + * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max + * value of 0x3ff. + */ + if (threshold_ns <= 0x3ff * 1) { + *scale = 0; /* Value times 1ns */ *value = threshold_ns; - } else if (threshold_ns < 1024) { - *scale = 1; - *value = threshold_ns >> 5; - } else if (threshold_ns < 32768) { - *scale = 2; - *value = threshold_ns >> 10; - } else if (threshold_ns < 1048576) { - *scale = 3; - *value = threshold_ns >> 15; - } else if (threshold_ns < 33554432) { - *scale = 4; - *value = threshold_ns >> 20; + } else if (threshold_ns <= 0x3ff * 32) { + *scale = 1; /* Value times 32ns */ + *value = roundup(threshold_ns, 32) / 32; + } else if (threshold_ns <= 0x3ff * 1024) { + *scale = 2; /* Value times 1024ns */ + *value = roundup(threshold_ns, 1024) / 1024; + } else if (threshold_ns <= 0x3ff * 32768) { + *scale = 3; /* Value times 32768ns */ + *value = roundup(threshold_ns, 32768) / 32768; + } else if (threshold_ns <= 0x3ff * 1048576) { + *scale = 4; /* Value times 1048576ns */ + *value = roundup(threshold_ns, 1048576) / 1048576; + } else if (threshold_ns <= 0x3ff * (u64) 33554432) { + *scale = 5; /* Value times 33554432ns */ + *value = roundup(threshold_ns, 33554432) / 33554432; } else { *scale = 5; - *value = threshold_ns >> 25; + *value = 0x3ff; /* Max representable value */ } } @@ -455,6 +470,31 @@ static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, pci_write_config_dword(pdev, pos, val); } +static void aspm_program_l1ss(struct pci_dev *dev, u32 ctl1, u32 ctl2) +{ + u16 l1ss = dev->l1ss; + u32 l1_2_enable; + + /* + * Per PCIe r6.0, sec 5.5.4, T_POWER_ON in PCI_L1SS_CTL2 must be + * programmed prior to setting the L1.2 enable bits in PCI_L1SS_CTL1. + */ + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL2, ctl2); + + /* + * In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD in + * PCI_L1SS_CTL1 must be programmed *before* setting the L1.2 + * enable bits, even though they're all in PCI_L1SS_CTL1. + */ + l1_2_enable = ctl1 & PCI_L1SS_CTL1_L1_2_MASK; + ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK; + + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, ctl1); + if (l1_2_enable) + pci_write_config_dword(dev, l1ss + PCI_L1SS_CTL1, + ctl1 | l1_2_enable); +} + /* Calculate L1.2 PM substate timing parameters */ static void aspm_calc_l1ss_info(struct pcie_link_state *link, u32 parent_l1ss_cap, u32 child_l1ss_cap) @@ -464,7 +504,6 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, u32 t_common_mode, t_power_on, l1_2_threshold, scale, value; u32 ctl1 = 0, ctl2 = 0; u32 pctl1, pctl2, cctl1, cctl2; - u32 pl1_2_enables, cl1_2_enables; if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) return; @@ -513,39 +552,78 @@ static void aspm_calc_l1ss_info(struct pcie_link_state *link, ctl2 == pctl2 && ctl2 == cctl2) return; - /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */ - pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK; - cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK; + pctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE); + pctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); + aspm_program_l1ss(parent, pctl1, ctl2); + + cctl1 &= ~(PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE); + cctl1 |= (ctl1 & (PCI_L1SS_CTL1_CM_RESTORE_TIME | + PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); + aspm_program_l1ss(child, cctl1, ctl2); +} - if (pl1_2_enables || cl1_2_enables) { - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_L1_2_MASK, 0); - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_L1_2_MASK, 0); - } +static void aspm_l1ss_init(struct pcie_link_state *link) +{ + struct pci_dev *child = link->downstream, *parent = link->pdev; + u32 parent_l1ss_cap, child_l1ss_cap; + u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; - /* Program T_POWER_ON times in both ports */ - pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); - pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); + if (!parent->l1ss || !child->l1ss) + return; - /* Program Common_Mode_Restore_Time in upstream device */ - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1); + /* Setup L1 substate */ + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, + &parent_l1ss_cap); + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, + &child_l1ss_cap); - /* Program LTR_L1.2_THRESHOLD time in both ports */ - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_LTR_L12_TH_VALUE | - PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1); - - if (pl1_2_enables || cl1_2_enables) { - pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, - pl1_2_enables); - pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, - cl1_2_enables); - } + if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + parent_l1ss_cap = 0; + if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) + child_l1ss_cap = 0; + + /* + * If we don't have LTR for the entire path from the Root Complex + * to this device, we can't use ASPM L1.2 because it relies on the + * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. + */ + if (!child->ltr_path) + child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; + + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) + link->aspm_support |= ASPM_STATE_L1_1_PCIPM; + if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) + link->aspm_support |= ASPM_STATE_L1_2_PCIPM; + + if (parent_l1ss_cap) + pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, + &parent_l1ss_ctl1); + if (child_l1ss_cap) + pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, + &child_l1ss_ctl1); + + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) + link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; + if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) + link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; + + if (link->aspm_support & ASPM_STATE_L1SS) + aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); } static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) @@ -553,8 +631,6 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) struct pci_dev *child = link->downstream, *parent = link->pdev; u32 parent_lnkcap, child_lnkcap; u16 parent_lnkctl, child_lnkctl; - u32 parent_l1ss_cap, child_l1ss_cap; - u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0; struct pci_bus *linkbus = parent->subordinate; if (blacklist) { @@ -609,52 +685,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1) link->aspm_enabled |= ASPM_STATE_L1; - /* Setup L1 substate */ - pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, - &parent_l1ss_cap); - pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, - &child_l1ss_cap); - - if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) - parent_l1ss_cap = 0; - if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) - child_l1ss_cap = 0; - - /* - * If we don't have LTR for the entire path from the Root Complex - * to this device, we can't use ASPM L1.2 because it relies on the - * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18. - */ - if (!child->ltr_path) - child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2; - - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) - link->aspm_support |= ASPM_STATE_L1_1_PCIPM; - if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) - link->aspm_support |= ASPM_STATE_L1_2_PCIPM; - - if (parent_l1ss_cap) - pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - &parent_l1ss_ctl1); - if (child_l1ss_cap) - pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, - &child_l1ss_ctl1); - - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) - link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; - if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) - link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; - - if (link->aspm_support & ASPM_STATE_L1SS) - aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap); + aspm_l1ss_init(link); /* Save default state */ link->aspm_default = link->aspm_enabled; @@ -726,6 +757,43 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) PCI_L1SS_CTL1_L1SS_MASK, val); } +void pci_save_aspm_l1ss_state(struct pci_dev *dev) +{ + struct pci_cap_saved_state *save_state; + u16 l1ss = dev->l1ss; + u32 *cap; + + if (!l1ss) + return; + + save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS); + if (!save_state) + return; + + cap = (u32 *)&save_state->cap.data[0]; + pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL2, cap++); + pci_read_config_dword(dev, l1ss + PCI_L1SS_CTL1, cap++); +} + +void pci_restore_aspm_l1ss_state(struct pci_dev *dev) +{ + struct pci_cap_saved_state *save_state; + u32 *cap, ctl1, ctl2; + u16 l1ss = dev->l1ss; + + if (!l1ss) + return; + + save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_L1SS); + if (!save_state) + return; + + cap = (u32 *)&save_state->cap.data[0]; + ctl2 = *cap++; + ctl1 = *cap; + aspm_program_l1ss(dev, ctl1, ctl2); +} + static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) { pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index 3e9afee02e8d1616bd4ab5278bc2e10e95f42a6b..f5ffea17c7f87207e1c413a430d94c3fce876942 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -335,11 +335,16 @@ void pci_dpc_init(struct pci_dev *pdev) return; pdev->dpc_rp_extensions = true; - pdev->dpc_rp_log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8; - if (pdev->dpc_rp_log_size < 4 || pdev->dpc_rp_log_size > 9) { - pci_err(pdev, "RP PIO log size %u is invalid\n", - pdev->dpc_rp_log_size); - pdev->dpc_rp_log_size = 0; + + /* Quirks may set dpc_rp_log_size if device or firmware is buggy */ + if (!pdev->dpc_rp_log_size) { + pdev->dpc_rp_log_size = + (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8; + if (pdev->dpc_rp_log_size < 4 || pdev->dpc_rp_log_size > 9) { + pci_err(pdev, "RP PIO log size %u is invalid\n", + pdev->dpc_rp_log_size); + pdev->dpc_rp_log_size = 0; + } } } diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c index 368a254e31242fd0a3163fc7cb93a227eabb6227..b4e5f553467c3b6fb3eb44a8a1749c946b0ba244 100644 --- a/drivers/pci/pcie/ptm.c +++ b/drivers/pci/pcie/ptm.c @@ -9,30 +9,38 @@ #include #include "../pci.h" -static void pci_ptm_info(struct pci_dev *dev) +/* + * If the next upstream device supports PTM, return it; otherwise return + * NULL. PTM Messages are local, so both link partners must support it. + */ +static struct pci_dev *pci_upstream_ptm(struct pci_dev *dev) { - char clock_desc[8]; + struct pci_dev *ups = pci_upstream_bridge(dev); - switch (dev->ptm_granularity) { - case 0: - snprintf(clock_desc, sizeof(clock_desc), "unknown"); - break; - case 255: - snprintf(clock_desc, sizeof(clock_desc), ">254ns"); - break; - default: - snprintf(clock_desc, sizeof(clock_desc), "%uns", - dev->ptm_granularity); - break; - } - pci_info(dev, "PTM enabled%s, %s granularity\n", - dev->ptm_root ? " (root)" : "", clock_desc); + /* + * Switch Downstream Ports are not permitted to have a PTM + * capability; their PTM behavior is controlled by the Upstream + * Port (PCIe r5.0, sec 7.9.16), so if the upstream bridge is a + * Switch Downstream Port, look up one more level. + */ + if (ups && pci_pcie_type(ups) == PCI_EXP_TYPE_DOWNSTREAM) + ups = pci_upstream_bridge(ups); + + if (ups && ups->ptm_cap) + return ups; + + return NULL; } -void pci_disable_ptm(struct pci_dev *dev) +/* + * Find the PTM Capability (if present) and extract the information we need + * to use it. + */ +void pci_ptm_init(struct pci_dev *dev) { - int ptm; - u16 ctrl; + u16 ptm; + u32 cap; + struct pci_dev *ups; if (!pci_is_pcie(dev)) return; @@ -41,21 +49,47 @@ void pci_disable_ptm(struct pci_dev *dev) if (!ptm) return; - pci_read_config_word(dev, ptm + PCI_PTM_CTRL, &ctrl); - ctrl &= ~(PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT); - pci_write_config_word(dev, ptm + PCI_PTM_CTRL, ctrl); + dev->ptm_cap = ptm; + pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_PTM, sizeof(u32)); + + pci_read_config_dword(dev, ptm + PCI_PTM_CAP, &cap); + dev->ptm_granularity = (cap & PCI_PTM_GRANULARITY_MASK) >> 8; + + /* + * Per the spec recommendation (PCIe r6.0, sec 7.9.15.3), select the + * furthest upstream Time Source as the PTM Root. For Endpoints, + * "the Effective Granularity is the maximum Local Clock Granularity + * reported by the PTM Root and all intervening PTM Time Sources." + */ + ups = pci_upstream_ptm(dev); + if (ups) { + if (ups->ptm_granularity == 0) + dev->ptm_granularity = 0; + else if (ups->ptm_granularity > dev->ptm_granularity) + dev->ptm_granularity = ups->ptm_granularity; + } else if (cap & PCI_PTM_CAP_ROOT) { + dev->ptm_root = 1; + } else if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) { + + /* + * Per sec 7.9.15.3, this should be the Local Clock + * Granularity of the associated Time Source. But it + * doesn't say how to find that Time Source. + */ + dev->ptm_granularity = 0; + } + + if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(dev) == PCI_EXP_TYPE_UPSTREAM) + pci_enable_ptm(dev, NULL); } void pci_save_ptm_state(struct pci_dev *dev) { - int ptm; + u16 ptm = dev->ptm_cap; struct pci_cap_saved_state *save_state; - u16 *cap; + u32 *cap; - if (!pci_is_pcie(dev)) - return; - - ptm = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM); if (!ptm) return; @@ -63,146 +97,152 @@ void pci_save_ptm_state(struct pci_dev *dev) if (!save_state) return; - cap = (u16 *)&save_state->cap.data[0]; - pci_read_config_word(dev, ptm + PCI_PTM_CTRL, cap); + cap = (u32 *)&save_state->cap.data[0]; + pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, cap); } void pci_restore_ptm_state(struct pci_dev *dev) { + u16 ptm = dev->ptm_cap; struct pci_cap_saved_state *save_state; - int ptm; - u16 *cap; + u32 *cap; - if (!pci_is_pcie(dev)) + if (!ptm) return; save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_PTM); - ptm = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM); - if (!save_state || !ptm) + if (!save_state) return; - cap = (u16 *)&save_state->cap.data[0]; - pci_write_config_word(dev, ptm + PCI_PTM_CTRL, *cap); + cap = (u32 *)&save_state->cap.data[0]; + pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, *cap); } -void pci_ptm_init(struct pci_dev *dev) +/* Enable PTM in the Control register if possible */ +static int __pci_enable_ptm(struct pci_dev *dev) { - int pos; - u32 cap, ctrl; - u8 local_clock; + u16 ptm = dev->ptm_cap; struct pci_dev *ups; + u32 ctrl; - if (!pci_is_pcie(dev)) - return; - - /* - * Enable PTM only on interior devices (root ports, switch ports, - * etc.) on the assumption that it causes no link traffic until an - * endpoint enables it. - */ - if ((pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT || - pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END)) - return; + if (!ptm) + return -EINVAL; /* - * Switch Downstream Ports are not permitted to have a PTM - * capability; their PTM behavior is controlled by the Upstream - * Port (PCIe r5.0, sec 7.9.16). + * A device uses local PTM Messages to request time information + * from a PTM Root that's farther upstream. Every device along the + * path must support PTM and have it enabled so it can handle the + * messages. Therefore, if this device is not a PTM Root, the + * upstream link partner must have PTM enabled before we can enable + * PTM. */ - ups = pci_upstream_bridge(dev); - if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM && - ups && ups->ptm_enabled) { - dev->ptm_granularity = ups->ptm_granularity; - dev->ptm_enabled = 1; - return; + if (!dev->ptm_root) { + ups = pci_upstream_ptm(dev); + if (!ups || !ups->ptm_enabled) + return -EINVAL; } - pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM); - if (!pos) - return; - - pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_PTM, sizeof(u16)); - - pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap); - local_clock = (cap & PCI_PTM_GRANULARITY_MASK) >> 8; - - /* - * There's no point in enabling PTM unless it's enabled in the - * upstream device or this device can be a PTM Root itself. Per - * the spec recommendation (PCIe r3.1, sec 7.32.3), select the - * furthest upstream Time Source as the PTM Root. - */ - if (ups && ups->ptm_enabled) { - ctrl = PCI_PTM_CTRL_ENABLE; - if (ups->ptm_granularity == 0) - dev->ptm_granularity = 0; - else if (ups->ptm_granularity > local_clock) - dev->ptm_granularity = ups->ptm_granularity; - } else { - if (cap & PCI_PTM_CAP_ROOT) { - ctrl = PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT; - dev->ptm_root = 1; - dev->ptm_granularity = local_clock; - } else - return; - } + pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, &ctrl); + ctrl |= PCI_PTM_CTRL_ENABLE; + ctrl &= ~PCI_PTM_GRANULARITY_MASK; ctrl |= dev->ptm_granularity << 8; - pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl); - dev->ptm_enabled = 1; + if (dev->ptm_root) + ctrl |= PCI_PTM_CTRL_ROOT; - pci_ptm_info(dev); + pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, ctrl); + return 0; } +/** + * pci_enable_ptm() - Enable Precision Time Measurement + * @dev: PCI device + * @granularity: pointer to return granularity + * + * Enable Precision Time Measurement for @dev. If successful and + * @granularity is non-NULL, return the Effective Granularity. + * + * Return: zero if successful, or -EINVAL if @dev lacks a PTM Capability or + * is not a PTM Root and lacks an upstream path of PTM-enabled devices. + */ int pci_enable_ptm(struct pci_dev *dev, u8 *granularity) { - int pos; - u32 cap, ctrl; - struct pci_dev *ups; - - if (!pci_is_pcie(dev)) - return -EINVAL; - - pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_PTM); - if (!pos) - return -EINVAL; - - pci_read_config_dword(dev, pos + PCI_PTM_CAP, &cap); - if (!(cap & PCI_PTM_CAP_REQ)) - return -EINVAL; - - /* - * For a PCIe Endpoint, PTM is only useful if the endpoint can - * issue PTM requests to upstream devices that have PTM enabled. - * - * For Root Complex Integrated Endpoints, there is no upstream - * device, so there must be some implementation-specific way to - * associate the endpoint with a time source. - */ - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT) { - ups = pci_upstream_bridge(dev); - if (!ups || !ups->ptm_enabled) - return -EINVAL; + int rc; + char clock_desc[8]; - dev->ptm_granularity = ups->ptm_granularity; - } else if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) { - dev->ptm_granularity = 0; - } else - return -EINVAL; + rc = __pci_enable_ptm(dev); + if (rc) + return rc; - ctrl = PCI_PTM_CTRL_ENABLE; - ctrl |= dev->ptm_granularity << 8; - pci_write_config_dword(dev, pos + PCI_PTM_CTRL, ctrl); dev->ptm_enabled = 1; - pci_ptm_info(dev); - if (granularity) *granularity = dev->ptm_granularity; + + switch (dev->ptm_granularity) { + case 0: + snprintf(clock_desc, sizeof(clock_desc), "unknown"); + break; + case 255: + snprintf(clock_desc, sizeof(clock_desc), ">254ns"); + break; + default: + snprintf(clock_desc, sizeof(clock_desc), "%uns", + dev->ptm_granularity); + break; + } + pci_info(dev, "PTM enabled%s, %s granularity\n", + dev->ptm_root ? " (root)" : "", clock_desc); + return 0; } EXPORT_SYMBOL(pci_enable_ptm); +static void __pci_disable_ptm(struct pci_dev *dev) +{ + u16 ptm = dev->ptm_cap; + u32 ctrl; + + if (!ptm) + return; + + pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, &ctrl); + ctrl &= ~(PCI_PTM_CTRL_ENABLE | PCI_PTM_CTRL_ROOT); + pci_write_config_dword(dev, ptm + PCI_PTM_CTRL, ctrl); +} + +/** + * pci_disable_ptm() - Disable Precision Time Measurement + * @dev: PCI device + * + * Disable Precision Time Measurement for @dev. + */ +void pci_disable_ptm(struct pci_dev *dev) +{ + if (dev->ptm_enabled) { + __pci_disable_ptm(dev); + dev->ptm_enabled = 0; + } +} +EXPORT_SYMBOL(pci_disable_ptm); + +/* + * Disable PTM, but preserve dev->ptm_enabled so we silently re-enable it on + * resume if necessary. + */ +void pci_suspend_ptm(struct pci_dev *dev) +{ + if (dev->ptm_enabled) + __pci_disable_ptm(dev); +} + +/* If PTM was enabled before suspend, re-enable it when resuming */ +void pci_resume_ptm(struct pci_dev *dev) +{ + if (dev->ptm_enabled) + __pci_enable_ptm(dev); +} + bool pcie_ptm_enabled(struct pci_dev *dev) { if (!dev) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index c5286b027f00df50f07a03667f55bc51f9a6fceb..b66fa42c4b1fa21546f60514fcc64107528b4e82 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1297,7 +1297,7 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, if ((secondary || subordinate) && !pcibios_assign_all_busses() && !is_cardbus && !broken) { - unsigned int cmax; + unsigned int cmax, buses; /* * Bus already configured by firmware, process it in the @@ -1322,7 +1322,8 @@ static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev, child->bridge_ctl = bctl; } - cmax = pci_scan_child_bus(child); + buses = subordinate - secondary; + cmax = pci_scan_child_bus_extend(child, buses); if (cmax > subordinate) pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n", subordinate, cmax); @@ -2920,8 +2921,8 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, * hotplug bridges too much during the second scan below. */ used_buses++; - if (cmax - max > 1) - used_buses += cmax - max - 1; + if (max - cmax > 1) + used_buses += max - cmax - 1; } /* Scan bridges that need to be reconfigured */ @@ -2929,7 +2930,6 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, unsigned int buses = 0; if (!hotplug_bridges && normal_bridges == 1) { - /* * There is only one bridge on the bus (upstream * port) so it gets all available buses which it @@ -2938,7 +2938,6 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, */ buses = available_buses; } else if (dev->is_hotplug_bridge) { - /* * Distribute the extra buses between hotplug * bridges if any. @@ -2957,7 +2956,7 @@ static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus, /* * Make sure a hotplug bridge has at least the minimum requested * number of buses but allow it to grow up to the maximum available - * bus number of there is room. + * bus number if there is room. */ if (bus->self && bus->self->is_hotplug_bridge) { used_buses = max_t(unsigned int, available_buses, diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 4944798e75b5a2fd58a31f6815046cd1bdd79a27..285acc4aaccc1e3f9cae1190ab09a19fb6eb5853 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -5956,3 +5956,39 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x56b1, aspm_l1_acceptable_latency DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x56c0, aspm_l1_acceptable_latency); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x56c1, aspm_l1_acceptable_latency); #endif + +#ifdef CONFIG_PCIE_DPC +/* + * Intel Tiger Lake and Alder Lake BIOS has a bug that clears the DPC + * RP PIO Log Size of the integrated Thunderbolt PCIe Root Ports. + */ +static void dpc_log_size(struct pci_dev *dev) +{ + u16 dpc, val; + + dpc = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DPC); + if (!dpc) + return; + + pci_read_config_word(dev, dpc + PCI_EXP_DPC_CAP, &val); + if (!(val & PCI_EXP_DPC_CAP_RP_EXT)) + return; + + if (!((val & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8)) { + pci_info(dev, "Overriding RP PIO Log Size to 4\n"); + dev->dpc_rp_log_size = 4; + } +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x461f, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x462f, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x463f, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x466e, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a23, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a25, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a27, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a29, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2b, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2d, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a2f, dpc_log_size); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x9a31, dpc_log_size); +#endif diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 8cb68e6f6ef93948a58e9db4a4285722127ec1aa..b4096598dbcbb9e3aee7079258fd6bc7e06a9450 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -1745,119 +1745,6 @@ static enum enable_type pci_realloc_detect(struct pci_bus *bus, } #endif -/* - * First try will not touch PCI bridge res. - * Second and later try will clear small leaf bridge res. - * Will stop till to the max depth if can not find good one. - */ -void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus) -{ - LIST_HEAD(realloc_head); - /* List of resources that want additional resources */ - struct list_head *add_list = NULL; - int tried_times = 0; - enum release_type rel_type = leaf_only; - LIST_HEAD(fail_head); - struct pci_dev_resource *fail_res; - int pci_try_num = 1; - enum enable_type enable_local; - - /* Don't realloc if asked to do so */ - enable_local = pci_realloc_detect(bus, pci_realloc_enable); - if (pci_realloc_enabled(enable_local)) { - int max_depth = pci_bus_get_depth(bus); - - pci_try_num = max_depth + 1; - dev_info(&bus->dev, "max bus depth: %d pci_try_num: %d\n", - max_depth, pci_try_num); - } - -again: - /* - * Last try will use add_list, otherwise will try good to have as must - * have, so can realloc parent bridge resource - */ - if (tried_times + 1 == pci_try_num) - add_list = &realloc_head; - /* - * Depth first, calculate sizes and alignments of all subordinate buses. - */ - __pci_bus_size_bridges(bus, add_list); - - /* Depth last, allocate resources and update the hardware. */ - __pci_bus_assign_resources(bus, add_list, &fail_head); - if (add_list) - BUG_ON(!list_empty(add_list)); - tried_times++; - - /* Any device complain? */ - if (list_empty(&fail_head)) - goto dump; - - if (tried_times >= pci_try_num) { - if (enable_local == undefined) - dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n"); - else if (enable_local == auto_enabled) - dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n"); - - free_list(&fail_head); - goto dump; - } - - dev_info(&bus->dev, "No. %d try to assign unassigned res\n", - tried_times + 1); - - /* Third times and later will not check if it is leaf */ - if ((tried_times + 1) > 2) - rel_type = whole_subtree; - - /* - * Try to release leaf bridge's resources that doesn't fit resource of - * child device under that bridge. - */ - list_for_each_entry(fail_res, &fail_head, list) - pci_bus_release_bridge_resources(fail_res->dev->bus, - fail_res->flags & PCI_RES_TYPE_MASK, - rel_type); - - /* Restore size and flags */ - list_for_each_entry(fail_res, &fail_head, list) { - struct resource *res = fail_res->res; - int idx; - - res->start = fail_res->start; - res->end = fail_res->end; - res->flags = fail_res->flags; - - if (pci_is_bridge(fail_res->dev)) { - idx = res - &fail_res->dev->resource[0]; - if (idx >= PCI_BRIDGE_RESOURCES && - idx <= PCI_BRIDGE_RESOURCE_END) - res->flags = 0; - } - } - free_list(&fail_head); - - goto again; - -dump: - /* Dump the resource on buses */ - pci_bus_dump_resources(bus); -} - -void __init pci_assign_unassigned_resources(void) -{ - struct pci_bus *root_bus; - - list_for_each_entry(root_bus, &pci_root_buses, node) { - pci_assign_unassigned_root_bus_resources(root_bus); - - /* Make sure the root bridge has a companion ACPI device */ - if (ACPI_HANDLE(root_bus->bridge)) - acpi_ioapic_add(ACPI_HANDLE(root_bus->bridge)); - } -} - static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res, struct list_head *add_list, resource_size_t new_size) @@ -2029,7 +1916,7 @@ static void pci_bus_distribute_available_resources(struct pci_bus *bus, } static void pci_bridge_distribute_available_resources(struct pci_dev *bridge, - struct list_head *add_list) + struct list_head *add_list) { struct resource available_io, available_mmio, available_mmio_pref; @@ -2047,6 +1934,119 @@ static void pci_bridge_distribute_available_resources(struct pci_dev *bridge, available_mmio_pref); } +/* + * First try will not touch PCI bridge res. + * Second and later try will clear small leaf bridge res. + * Will stop till to the max depth if can not find good one. + */ +void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus) +{ + LIST_HEAD(realloc_head); + /* List of resources that want additional resources */ + struct list_head *add_list = NULL; + int tried_times = 0; + enum release_type rel_type = leaf_only; + LIST_HEAD(fail_head); + struct pci_dev_resource *fail_res; + int pci_try_num = 1; + enum enable_type enable_local; + + /* Don't realloc if asked to do so */ + enable_local = pci_realloc_detect(bus, pci_realloc_enable); + if (pci_realloc_enabled(enable_local)) { + int max_depth = pci_bus_get_depth(bus); + + pci_try_num = max_depth + 1; + dev_info(&bus->dev, "max bus depth: %d pci_try_num: %d\n", + max_depth, pci_try_num); + } + +again: + /* + * Last try will use add_list, otherwise will try good to have as must + * have, so can realloc parent bridge resource + */ + if (tried_times + 1 == pci_try_num) + add_list = &realloc_head; + /* + * Depth first, calculate sizes and alignments of all subordinate buses. + */ + __pci_bus_size_bridges(bus, add_list); + + /* Depth last, allocate resources and update the hardware. */ + __pci_bus_assign_resources(bus, add_list, &fail_head); + if (add_list) + BUG_ON(!list_empty(add_list)); + tried_times++; + + /* Any device complain? */ + if (list_empty(&fail_head)) + goto dump; + + if (tried_times >= pci_try_num) { + if (enable_local == undefined) + dev_info(&bus->dev, "Some PCI device resources are unassigned, try booting with pci=realloc\n"); + else if (enable_local == auto_enabled) + dev_info(&bus->dev, "Automatically enabled pci realloc, if you have problem, try booting with pci=realloc=off\n"); + + free_list(&fail_head); + goto dump; + } + + dev_info(&bus->dev, "No. %d try to assign unassigned res\n", + tried_times + 1); + + /* Third times and later will not check if it is leaf */ + if ((tried_times + 1) > 2) + rel_type = whole_subtree; + + /* + * Try to release leaf bridge's resources that doesn't fit resource of + * child device under that bridge. + */ + list_for_each_entry(fail_res, &fail_head, list) + pci_bus_release_bridge_resources(fail_res->dev->bus, + fail_res->flags & PCI_RES_TYPE_MASK, + rel_type); + + /* Restore size and flags */ + list_for_each_entry(fail_res, &fail_head, list) { + struct resource *res = fail_res->res; + int idx; + + res->start = fail_res->start; + res->end = fail_res->end; + res->flags = fail_res->flags; + + if (pci_is_bridge(fail_res->dev)) { + idx = res - &fail_res->dev->resource[0]; + if (idx >= PCI_BRIDGE_RESOURCES && + idx <= PCI_BRIDGE_RESOURCE_END) + res->flags = 0; + } + } + free_list(&fail_head); + + goto again; + +dump: + /* Dump the resource on buses */ + pci_bus_dump_resources(bus); +} + +void __init pci_assign_unassigned_resources(void) +{ + struct pci_bus *root_bus; + + list_for_each_entry(root_bus, &pci_root_buses, node) { + pci_assign_unassigned_root_bus_resources(root_bus); + + /* Make sure the root bridge has a companion ACPI device */ + if (ACPI_HANDLE(root_bus->bridge)) + acpi_ioapic_add(ACPI_HANDLE(root_bus->bridge)); + } +} + void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge) { struct pci_bus *parent = bridge->subordinate; diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index 439ac5f5907a6ee5778025b3180ff268e3c3166b..b492e67c3d871747ec73d69dbb1dc1d1fc774dca 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -214,6 +214,17 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev, root = pci_find_parent_resource(dev, res); if (!root) { + /* + * If dev is behind a bridge, accesses will only reach it + * if res is inside the relevant bridge window. + */ + if (pci_upstream_bridge(dev)) + return -ENXIO; + + /* + * On the root bus, assume the host bridge will forward + * everything. + */ if (res->flags & IORESOURCE_IO) root = &ioport_resource; else diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index 689271c4245c2ef99529e62c8d435cdcbf36ad1a..7378e2f3e525fd1290cc482e8329a74d635685b5 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -521,24 +521,14 @@ static int pcifront_rescan_root(struct pcifront_device *pdev, int err; struct pci_bus *b; -#ifndef CONFIG_PCI_DOMAINS - if (domain != 0) { - dev_err(&pdev->xdev->dev, - "PCI Root in non-zero PCI Domain! domain=%d\n", domain); - dev_err(&pdev->xdev->dev, - "Please compile with CONFIG_PCI_DOMAINS\n"); - return -EINVAL; - } -#endif - - dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n", - domain, bus); - b = pci_find_bus(domain, bus); if (!b) /* If the bus is unknown, create it. */ return pcifront_scan_root(pdev, domain, bus); + dev_info(&pdev->xdev->dev, "Rescanning PCI Frontend Bus %04x:%02x\n", + domain, bus); + err = pcifront_scan_bus(pdev, domain, bus, b); /* Claim resources before going "live" with our devices */ @@ -819,76 +809,73 @@ out: return err; } -static int pcifront_try_connect(struct pcifront_device *pdev) +static void pcifront_connect(struct pcifront_device *pdev) { - int err = -EFAULT; + int err; int i, num_roots, len; char str[64]; unsigned int domain, bus; - - /* Only connect once */ - if (xenbus_read_driver_state(pdev->xdev->nodename) != - XenbusStateInitialised) - goto out; - - err = pcifront_connect_and_init_dma(pdev); - if (err && err != -EEXIST) { - xenbus_dev_fatal(pdev->xdev, err, - "Error setting up PCI Frontend"); - goto out; - } - err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, "root_num", "%d", &num_roots); if (err == -ENOENT) { xenbus_dev_error(pdev->xdev, err, "No PCI Roots found, trying 0000:00"); - err = pcifront_scan_root(pdev, 0, 0); + err = pcifront_rescan_root(pdev, 0, 0); if (err) { xenbus_dev_fatal(pdev->xdev, err, "Error scanning PCI root 0000:00"); - goto out; + return; } num_roots = 0; } else if (err != 1) { - if (err == 0) - err = -EINVAL; - xenbus_dev_fatal(pdev->xdev, err, + xenbus_dev_fatal(pdev->xdev, err >= 0 ? -EINVAL : err, "Error reading number of PCI roots"); - goto out; + return; } for (i = 0; i < num_roots; i++) { len = snprintf(str, sizeof(str), "root-%d", i); - if (unlikely(len >= (sizeof(str) - 1))) { - err = -ENOMEM; - goto out; - } + if (unlikely(len >= (sizeof(str) - 1))) + return; err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str, "%x:%x", &domain, &bus); if (err != 2) { - if (err >= 0) - err = -EINVAL; - xenbus_dev_fatal(pdev->xdev, err, + xenbus_dev_fatal(pdev->xdev, err >= 0 ? -EINVAL : err, "Error reading PCI root %d", i); - goto out; + return; } - err = pcifront_scan_root(pdev, domain, bus); + err = pcifront_rescan_root(pdev, domain, bus); if (err) { xenbus_dev_fatal(pdev->xdev, err, "Error scanning PCI root %04x:%02x", domain, bus); - goto out; + return; } } - err = xenbus_switch_state(pdev->xdev, XenbusStateConnected); + xenbus_switch_state(pdev->xdev, XenbusStateConnected); +} -out: - return err; +static void pcifront_try_connect(struct pcifront_device *pdev) +{ + int err; + + /* Only connect once */ + if (xenbus_read_driver_state(pdev->xdev->nodename) != + XenbusStateInitialised) + return; + + err = pcifront_connect_and_init_dma(pdev); + if (err && err != -EEXIST) { + xenbus_dev_fatal(pdev->xdev, err, + "Error setting up PCI Frontend"); + return; + } + + pcifront_connect(pdev); } static int pcifront_try_disconnect(struct pcifront_device *pdev) @@ -914,80 +901,37 @@ out: return err; } -static int pcifront_attach_devices(struct pcifront_device *pdev) +static void pcifront_attach_devices(struct pcifront_device *pdev) { - int err = -EFAULT; - int i, num_roots, len; - unsigned int domain, bus; - char str[64]; - - if (xenbus_read_driver_state(pdev->xdev->nodename) != + if (xenbus_read_driver_state(pdev->xdev->nodename) == XenbusStateReconfiguring) - goto out; - - err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, - "root_num", "%d", &num_roots); - if (err == -ENOENT) { - xenbus_dev_error(pdev->xdev, err, - "No PCI Roots found, trying 0000:00"); - err = pcifront_rescan_root(pdev, 0, 0); - if (err) { - xenbus_dev_fatal(pdev->xdev, err, - "Error scanning PCI root 0000:00"); - goto out; - } - num_roots = 0; - } else if (err != 1) { - if (err == 0) - err = -EINVAL; - xenbus_dev_fatal(pdev->xdev, err, - "Error reading number of PCI roots"); - goto out; - } - - for (i = 0; i < num_roots; i++) { - len = snprintf(str, sizeof(str), "root-%d", i); - if (unlikely(len >= (sizeof(str) - 1))) { - err = -ENOMEM; - goto out; - } - - err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, str, - "%x:%x", &domain, &bus); - if (err != 2) { - if (err >= 0) - err = -EINVAL; - xenbus_dev_fatal(pdev->xdev, err, - "Error reading PCI root %d", i); - goto out; - } - - err = pcifront_rescan_root(pdev, domain, bus); - if (err) { - xenbus_dev_fatal(pdev->xdev, err, - "Error scanning PCI root %04x:%02x", - domain, bus); - goto out; - } - } - - xenbus_switch_state(pdev->xdev, XenbusStateConnected); - -out: - return err; + pcifront_connect(pdev); } static int pcifront_detach_devices(struct pcifront_device *pdev) { int err = 0; int i, num_devs; + enum xenbus_state state; unsigned int domain, bus, slot, func; struct pci_dev *pci_dev; char str[64]; - if (xenbus_read_driver_state(pdev->xdev->nodename) != - XenbusStateConnected) + state = xenbus_read_driver_state(pdev->xdev->nodename); + if (state == XenbusStateInitialised) { + dev_dbg(&pdev->xdev->dev, "Handle skipped connect.\n"); + /* We missed Connected and need to initialize. */ + err = pcifront_connect_and_init_dma(pdev); + if (err && err != -EEXIST) { + xenbus_dev_fatal(pdev->xdev, err, + "Error setting up PCI Frontend"); + goto out; + } + + goto out_switch_state; + } else if (state != XenbusStateConnected) { goto out; + } err = xenbus_scanf(XBT_NIL, pdev->xdev->otherend, "num_devs", "%d", &num_devs); @@ -1048,6 +992,7 @@ static int pcifront_detach_devices(struct pcifront_device *pdev) domain, bus, slot, func); } + out_switch_state: err = xenbus_switch_state(pdev->xdev, XenbusStateReconfiguring); out: diff --git a/drivers/pcmcia/Kconfig b/drivers/pcmcia/Kconfig index bf495bf0f48a2e0e787d043323c3306a83b24474..1525023e49b65b29747b363ca38ea152c91aa05d 100644 --- a/drivers/pcmcia/Kconfig +++ b/drivers/pcmcia/Kconfig @@ -240,10 +240,6 @@ config PCMCIA_PROBE bool default y if ISA && !ARCH_SA1100 && !PARISC -config PCMCIA_VRC4171 - tristate "NEC VRC4171 Card Controllers support" - depends on CPU_VR41XX && ISA && PCMCIA - config OMAP_CF tristate "OMAP CompactFlash Controller" depends on PCMCIA @@ -252,15 +248,6 @@ config OMAP_CF Say Y here to support the CompactFlash controller on OMAP. Note that this doesn't support "True IDE" mode. -config AT91_CF - tristate "AT91 CompactFlash Controller" - depends on PCI - depends on OF - depends on PCMCIA && ARCH_AT91 - help - Say Y here to support the CompactFlash controller on AT91 chips. - Or choose M to compile the driver as a module named "at91_cf". - config ELECTRA_CF tristate "Electra CompactFlash Controller" depends on PCMCIA && PPC_PASEMI diff --git a/drivers/pcmcia/Makefile b/drivers/pcmcia/Makefile index c59ddde42007d8de5e53eecc2a53eba453023ae2..b3a2accf47af1e361de846f6a1ae24d3724b407a 100644 --- a/drivers/pcmcia/Makefile +++ b/drivers/pcmcia/Makefile @@ -29,9 +29,7 @@ obj-$(CONFIG_PCMCIA_SA11XX_BASE) += sa11xx_base.o obj-$(CONFIG_PCMCIA_SA1100) += sa1100_cs.o obj-$(CONFIG_PCMCIA_SA1111) += sa1111_cs.o obj-$(CONFIG_PCMCIA_BCM63XX) += bcm63xx_pcmcia.o -obj-$(CONFIG_PCMCIA_VRC4171) += vrc4171_card.o obj-$(CONFIG_OMAP_CF) += omap_cf.o -obj-$(CONFIG_AT91_CF) += at91_cf.o obj-$(CONFIG_ELECTRA_CF) += electra_cf.o obj-$(CONFIG_PCMCIA_ALCHEMY_DEVBOARD) += db1xxx_ss.o obj-$(CONFIG_PCMCIA_MAX1600) += max1600.o diff --git a/drivers/pcmcia/at91_cf.c b/drivers/pcmcia/at91_cf.c deleted file mode 100644 index 92df2c2c5d0762fd1aba3788552257be9901315a..0000000000000000000000000000000000000000 --- a/drivers/pcmcia/at91_cf.c +++ /dev/null @@ -1,407 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * at91_cf.c -- AT91 CompactFlash controller driver - * - * Copyright (C) 2005 David Brownell - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -/* - * A0..A10 work in each range; A23 indicates I/O space; A25 is CFRNW; - * some other bit in {A24,A22..A11} is nREG to flag memory access - * (vs attributes). So more than 2KB/region would just be waste. - * Note: These are offsets from the physical base address. - */ -#define CF_ATTR_PHYS (0) -#define CF_IO_PHYS (1 << 23) -#define CF_MEM_PHYS (0x017ff800) - -struct at91_cf_data { - int irq_pin; /* I/O IRQ */ - int det_pin; /* Card detect */ - int vcc_pin; /* power switching */ - int rst_pin; /* card reset */ - u8 chipselect; /* EBI Chip Select number */ - u8 flags; -#define AT91_CF_TRUE_IDE 0x01 -#define AT91_IDE_SWAP_A0_A2 0x02 -}; - -struct regmap *mc; - -/*--------------------------------------------------------------------------*/ - -struct at91_cf_socket { - struct pcmcia_socket socket; - - unsigned present:1; - - struct platform_device *pdev; - struct at91_cf_data *board; - - unsigned long phys_baseaddr; -}; - -static inline int at91_cf_present(struct at91_cf_socket *cf) -{ - return !gpio_get_value(cf->board->det_pin); -} - -/*--------------------------------------------------------------------------*/ - -static int at91_cf_ss_init(struct pcmcia_socket *s) -{ - return 0; -} - -static irqreturn_t at91_cf_irq(int irq, void *_cf) -{ - struct at91_cf_socket *cf = _cf; - - if (irq == gpio_to_irq(cf->board->det_pin)) { - unsigned present = at91_cf_present(cf); - - /* kick pccard as needed */ - if (present != cf->present) { - cf->present = present; - dev_dbg(&cf->pdev->dev, "card %s\n", - present ? "present" : "gone"); - pcmcia_parse_events(&cf->socket, SS_DETECT); - } - } - - return IRQ_HANDLED; -} - -static int at91_cf_get_status(struct pcmcia_socket *s, u_int *sp) -{ - struct at91_cf_socket *cf; - - if (!sp) - return -EINVAL; - - cf = container_of(s, struct at91_cf_socket, socket); - - /* NOTE: CF is always 3VCARD */ - if (at91_cf_present(cf)) { - int rdy = gpio_is_valid(cf->board->irq_pin); /* RDY/nIRQ */ - int vcc = gpio_is_valid(cf->board->vcc_pin); - - *sp = SS_DETECT | SS_3VCARD; - if (!rdy || gpio_get_value(cf->board->irq_pin)) - *sp |= SS_READY; - if (!vcc || gpio_get_value(cf->board->vcc_pin)) - *sp |= SS_POWERON; - } else - *sp = 0; - - return 0; -} - -static int -at91_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s) -{ - struct at91_cf_socket *cf; - - cf = container_of(sock, struct at91_cf_socket, socket); - - /* switch Vcc if needed and possible */ - if (gpio_is_valid(cf->board->vcc_pin)) { - switch (s->Vcc) { - case 0: - gpio_set_value(cf->board->vcc_pin, 0); - break; - case 33: - gpio_set_value(cf->board->vcc_pin, 1); - break; - default: - return -EINVAL; - } - } - - /* toggle reset if needed */ - gpio_set_value(cf->board->rst_pin, s->flags & SS_RESET); - - dev_dbg(&cf->pdev->dev, "Vcc %d, io_irq %d, flags %04x csc %04x\n", - s->Vcc, s->io_irq, s->flags, s->csc_mask); - - return 0; -} - -static int at91_cf_ss_suspend(struct pcmcia_socket *s) -{ - return at91_cf_set_socket(s, &dead_socket); -} - -/* we already mapped the I/O region */ -static int at91_cf_set_io_map(struct pcmcia_socket *s, struct pccard_io_map *io) -{ - struct at91_cf_socket *cf; - u32 csr; - - cf = container_of(s, struct at91_cf_socket, socket); - io->flags &= (MAP_ACTIVE | MAP_16BIT | MAP_AUTOSZ); - - /* - * Use 16 bit accesses unless/until we need 8-bit i/o space. - * - * NOTE: this CF controller ignores IOIS16, so we can't really do - * MAP_AUTOSZ. The 16bit mode allows single byte access on either - * D0-D7 (even addr) or D8-D15 (odd), so it's close enough for many - * purposes (and handles ide-cs). - * - * The 8bit mode is needed for odd byte access on D0-D7. It seems - * some cards only like that way to get at the odd byte, despite - * CF 3.0 spec table 35 also giving the D8-D15 option. - */ - if (!(io->flags & (MAP_16BIT | MAP_AUTOSZ))) { - csr = AT91_MC_SMC_DBW_8; - dev_dbg(&cf->pdev->dev, "8bit i/o bus\n"); - } else { - csr = AT91_MC_SMC_DBW_16; - dev_dbg(&cf->pdev->dev, "16bit i/o bus\n"); - } - regmap_update_bits(mc, AT91_MC_SMC_CSR(cf->board->chipselect), - AT91_MC_SMC_DBW, csr); - - io->start = cf->socket.io_offset; - io->stop = io->start + SZ_2K - 1; - - return 0; -} - -/* pcmcia layer maps/unmaps mem regions */ -static int -at91_cf_set_mem_map(struct pcmcia_socket *s, struct pccard_mem_map *map) -{ - struct at91_cf_socket *cf; - - if (map->card_start) - return -EINVAL; - - cf = container_of(s, struct at91_cf_socket, socket); - - map->flags &= (MAP_ACTIVE | MAP_ATTRIB | MAP_16BIT); - if (map->flags & MAP_ATTRIB) - map->static_start = cf->phys_baseaddr + CF_ATTR_PHYS; - else - map->static_start = cf->phys_baseaddr + CF_MEM_PHYS; - - return 0; -} - -static struct pccard_operations at91_cf_ops = { - .init = at91_cf_ss_init, - .suspend = at91_cf_ss_suspend, - .get_status = at91_cf_get_status, - .set_socket = at91_cf_set_socket, - .set_io_map = at91_cf_set_io_map, - .set_mem_map = at91_cf_set_mem_map, -}; - -/*--------------------------------------------------------------------------*/ - -static const struct of_device_id at91_cf_dt_ids[] = { - { .compatible = "atmel,at91rm9200-cf" }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, at91_cf_dt_ids); - -static int at91_cf_probe(struct platform_device *pdev) -{ - struct at91_cf_socket *cf; - struct at91_cf_data *board; - struct resource *io; - struct resource realio; - int status; - - board = devm_kzalloc(&pdev->dev, sizeof(*board), GFP_KERNEL); - if (!board) - return -ENOMEM; - - board->irq_pin = of_get_gpio(pdev->dev.of_node, 0); - board->det_pin = of_get_gpio(pdev->dev.of_node, 1); - board->vcc_pin = of_get_gpio(pdev->dev.of_node, 2); - board->rst_pin = of_get_gpio(pdev->dev.of_node, 3); - - mc = syscon_regmap_lookup_by_compatible("atmel,at91rm9200-sdramc"); - if (IS_ERR(mc)) - return PTR_ERR(mc); - - if (!gpio_is_valid(board->det_pin) || !gpio_is_valid(board->rst_pin)) - return -ENODEV; - - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -ENODEV; - - cf = devm_kzalloc(&pdev->dev, sizeof(*cf), GFP_KERNEL); - if (!cf) - return -ENOMEM; - - cf->board = board; - cf->pdev = pdev; - cf->phys_baseaddr = io->start; - platform_set_drvdata(pdev, cf); - - /* must be a GPIO; ergo must trigger on both edges */ - status = devm_gpio_request(&pdev->dev, board->det_pin, "cf_det"); - if (status < 0) - return status; - - status = devm_request_irq(&pdev->dev, gpio_to_irq(board->det_pin), - at91_cf_irq, 0, "at91_cf detect", cf); - if (status < 0) - return status; - - device_init_wakeup(&pdev->dev, 1); - - status = devm_gpio_request(&pdev->dev, board->rst_pin, "cf_rst"); - if (status < 0) - goto fail0a; - - if (gpio_is_valid(board->vcc_pin)) { - status = devm_gpio_request(&pdev->dev, board->vcc_pin, "cf_vcc"); - if (status < 0) - goto fail0a; - } - - /* - * The card driver will request this irq later as needed. - * but it causes lots of "irqNN: nobody cared" messages - * unless we report that we handle everything (sigh). - * (Note: DK board doesn't wire the IRQ pin...) - */ - if (gpio_is_valid(board->irq_pin)) { - status = devm_gpio_request(&pdev->dev, board->irq_pin, "cf_irq"); - if (status < 0) - goto fail0a; - - status = devm_request_irq(&pdev->dev, gpio_to_irq(board->irq_pin), - at91_cf_irq, IRQF_SHARED, "at91_cf", cf); - if (status < 0) - goto fail0a; - cf->socket.pci_irq = gpio_to_irq(board->irq_pin); - } else - cf->socket.pci_irq = nr_irqs + 1; - - /* - * pcmcia layer only remaps "real" memory not iospace - * io_offset is set to 0x10000 to avoid the check in static_find_io(). - * */ - cf->socket.io_offset = 0x10000; - realio.start = cf->socket.io_offset; - realio.end = realio.start + SZ_64K - 1; - status = pci_remap_iospace(&realio, cf->phys_baseaddr + CF_IO_PHYS); - if (status) - goto fail0a; - - /* reserve chip-select regions */ - if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io), "at91_cf")) { - status = -ENXIO; - goto fail0a; - } - - dev_info(&pdev->dev, "irqs det #%d, io #%d\n", - gpio_to_irq(board->det_pin), gpio_to_irq(board->irq_pin)); - - cf->socket.owner = THIS_MODULE; - cf->socket.dev.parent = &pdev->dev; - cf->socket.ops = &at91_cf_ops; - cf->socket.resource_ops = &pccard_static_ops; - cf->socket.features = SS_CAP_PCCARD | SS_CAP_STATIC_MAP - | SS_CAP_MEM_ALIGN; - cf->socket.map_size = SZ_2K; - cf->socket.io[0].res = io; - - status = pcmcia_register_socket(&cf->socket); - if (status < 0) - goto fail0a; - - return 0; - -fail0a: - device_init_wakeup(&pdev->dev, 0); - return status; -} - -static int at91_cf_remove(struct platform_device *pdev) -{ - struct at91_cf_socket *cf = platform_get_drvdata(pdev); - - pcmcia_unregister_socket(&cf->socket); - device_init_wakeup(&pdev->dev, 0); - - return 0; -} - -#ifdef CONFIG_PM - -static int at91_cf_suspend(struct platform_device *pdev, pm_message_t mesg) -{ - struct at91_cf_socket *cf = platform_get_drvdata(pdev); - struct at91_cf_data *board = cf->board; - - if (device_may_wakeup(&pdev->dev)) { - enable_irq_wake(gpio_to_irq(board->det_pin)); - if (gpio_is_valid(board->irq_pin)) - enable_irq_wake(gpio_to_irq(board->irq_pin)); - } - return 0; -} - -static int at91_cf_resume(struct platform_device *pdev) -{ - struct at91_cf_socket *cf = platform_get_drvdata(pdev); - struct at91_cf_data *board = cf->board; - - if (device_may_wakeup(&pdev->dev)) { - disable_irq_wake(gpio_to_irq(board->det_pin)); - if (gpio_is_valid(board->irq_pin)) - disable_irq_wake(gpio_to_irq(board->irq_pin)); - } - - return 0; -} - -#else -#define at91_cf_suspend NULL -#define at91_cf_resume NULL -#endif - -static struct platform_driver at91_cf_driver = { - .driver = { - .name = "at91_cf", - .of_match_table = at91_cf_dt_ids, - }, - .probe = at91_cf_probe, - .remove = at91_cf_remove, - .suspend = at91_cf_suspend, - .resume = at91_cf_resume, -}; - -module_platform_driver(at91_cf_driver); - -MODULE_DESCRIPTION("AT91 Compact Flash Driver"); -MODULE_AUTHOR("David Brownell"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:at91_cf"); diff --git a/drivers/pcmcia/i82092.c b/drivers/pcmcia/i82092.c index 192c9049d654fdd15275d4e7e3a465618564dbba..a335748bdef5a3e3a0d8df6809ff30535de7dbae 100644 --- a/drivers/pcmcia/i82092.c +++ b/drivers/pcmcia/i82092.c @@ -661,12 +661,12 @@ static int i82092aa_set_mem_map(struct pcmcia_socket *socket, return 0; } -static int i82092aa_module_init(void) +static int __init i82092aa_module_init(void) { return pci_register_driver(&i82092aa_pci_driver); } -static void i82092aa_module_exit(void) +static void __exit i82092aa_module_exit(void) { pci_unregister_driver(&i82092aa_pci_driver); if (sockets[0].io_base > 0) diff --git a/drivers/pcmcia/omap_cf.c b/drivers/pcmcia/omap_cf.c index 1972a8f6fa8e893ce193aa525c789812771b89ce..d3f827d4224a35461b6e1ea116af838ff34ad90a 100644 --- a/drivers/pcmcia/omap_cf.c +++ b/drivers/pcmcia/omap_cf.c @@ -124,8 +124,6 @@ static int omap_cf_get_status(struct pcmcia_socket *s, u_int *sp) static int omap_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s) { - u16 control; - /* REVISIT some non-OSK boards may support power switching */ switch (s->Vcc) { case 0: @@ -135,7 +133,7 @@ omap_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s) return -EINVAL; } - control = omap_readw(CF_CONTROL); + omap_readw(CF_CONTROL); if (s->flags & SS_RESET) omap_writew(CF_CONTROL_RESET, CF_CONTROL); else diff --git a/drivers/pcmcia/sa1100_generic.c b/drivers/pcmcia/sa1100_generic.c index 47b060c5741861a3f76100d56b850c146721d119..c2b6e828c2c6ed2f9ea736880beb21ab0b1c6c55 100644 --- a/drivers/pcmcia/sa1100_generic.c +++ b/drivers/pcmcia/sa1100_generic.c @@ -125,7 +125,7 @@ static int sa11x0_drv_pcmcia_legacy_probe(struct platform_device *dev) return ret; } -static int sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev) +static void sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev) { struct skt_dev_info *sinfo = platform_get_drvdata(dev); int i; @@ -134,8 +134,6 @@ static int sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev) for (i = 0; i < sinfo->nskt; i++) soc_pcmcia_remove_one(&sinfo->skt[i]); - - return 0; } static int sa11x0_drv_pcmcia_probe(struct platform_device *pdev) @@ -167,8 +165,10 @@ static int sa11x0_drv_pcmcia_remove(struct platform_device *dev) { struct soc_pcmcia_socket *skt; - if (dev->id == -1) - return sa11x0_drv_pcmcia_legacy_remove(dev); + if (dev->id == -1) { + sa11x0_drv_pcmcia_legacy_remove(dev); + return 0; + } skt = platform_get_drvdata(dev); diff --git a/drivers/pcmcia/vrc4171_card.c b/drivers/pcmcia/vrc4171_card.c deleted file mode 100644 index 177d778921440ff9d82d167c7e5bc6b1de2b064b..0000000000000000000000000000000000000000 --- a/drivers/pcmcia/vrc4171_card.c +++ /dev/null @@ -1,745 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * vrc4171_card.c, NEC VRC4171 Card Controller driver for Socket Services. - * - * Copyright (C) 2003-2005 Yoichi Yuasa - */ -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include "i82365.h" - -MODULE_DESCRIPTION("NEC VRC4171 Card Controllers driver for Socket Services"); -MODULE_AUTHOR("Yoichi Yuasa "); -MODULE_LICENSE("GPL"); - -#define CARD_MAX_SLOTS 2 -#define CARD_SLOTA 0 -#define CARD_SLOTB 1 -#define CARD_SLOTB_OFFSET 0x40 - -#define CARD_MEM_START 0x10000000 -#define CARD_MEM_END 0x13ffffff -#define CARD_MAX_MEM_OFFSET 0x3ffffff -#define CARD_MAX_MEM_SPEED 1000 - -#define CARD_CONTROLLER_INDEX 0x03e0 -#define CARD_CONTROLLER_DATA 0x03e1 - /* Power register */ - #define VPP_GET_VCC 0x01 - #define POWER_ENABLE 0x10 - #define CARD_VOLTAGE_SENSE 0x1f - #define VCC_3VORXV_CAPABLE 0x00 - #define VCC_XV_ONLY 0x01 - #define VCC_3V_CAPABLE 0x02 - #define VCC_5V_ONLY 0x03 - #define CARD_VOLTAGE_SELECT 0x2f - #define VCC_3V 0x01 - #define VCC_5V 0x00 - #define VCC_XV 0x02 - #define VCC_STATUS_3V 0x02 - #define VCC_STATUS_5V 0x01 - #define VCC_STATUS_XV 0x03 - #define GLOBAL_CONTROL 0x1e - #define EXWRBK 0x04 - #define IRQPM_EN 0x08 - #define CLRPMIRQ 0x10 - -#define INTERRUPT_STATUS 0x05fa - #define IRQ_A 0x02 - #define IRQ_B 0x04 - -#define CONFIGURATION1 0x05fe - #define SLOTB_CONFIG 0xc000 - #define SLOTB_NONE 0x0000 - #define SLOTB_PCCARD 0x4000 - #define SLOTB_CF 0x8000 - #define SLOTB_FLASHROM 0xc000 - -#define CARD_CONTROLLER_START CARD_CONTROLLER_INDEX -#define CARD_CONTROLLER_END CARD_CONTROLLER_DATA - -#define IO_MAX_MAPS 2 -#define MEM_MAX_MAPS 5 - -enum vrc4171_slot { - SLOT_PROBE = 0, - SLOT_NOPROBE_IO, - SLOT_NOPROBE_MEM, - SLOT_NOPROBE_ALL, - SLOT_INITIALIZED, -}; - -enum vrc4171_slotb { - SLOTB_IS_NONE, - SLOTB_IS_PCCARD, - SLOTB_IS_CF, - SLOTB_IS_FLASHROM, -}; - -struct vrc4171_socket { - enum vrc4171_slot slot; - struct pcmcia_socket pcmcia_socket; - char name[24]; - int csc_irq; - int io_irq; - spinlock_t lock; -}; - -static struct vrc4171_socket vrc4171_sockets[CARD_MAX_SLOTS]; -static enum vrc4171_slotb vrc4171_slotb = SLOTB_IS_NONE; -static char vrc4171_card_name[] = "NEC VRC4171 Card Controller"; -static unsigned int vrc4171_irq; -static uint16_t vrc4171_irq_mask = 0xdeb8; - -static struct resource vrc4171_card_resource[3] = { - { .name = vrc4171_card_name, - .start = CARD_CONTROLLER_START, - .end = CARD_CONTROLLER_END, - .flags = IORESOURCE_IO, }, - { .name = vrc4171_card_name, - .start = INTERRUPT_STATUS, - .end = INTERRUPT_STATUS, - .flags = IORESOURCE_IO, }, - { .name = vrc4171_card_name, - .start = CONFIGURATION1, - .end = CONFIGURATION1, - .flags = IORESOURCE_IO, }, -}; - -static struct platform_device vrc4171_card_device = { - .name = vrc4171_card_name, - .id = 0, - .num_resources = 3, - .resource = vrc4171_card_resource, -}; - -static inline uint16_t vrc4171_get_irq_status(void) -{ - return inw(INTERRUPT_STATUS); -} - -static inline void vrc4171_set_multifunction_pin(enum vrc4171_slotb config) -{ - uint16_t config1; - - config1 = inw(CONFIGURATION1); - config1 &= ~SLOTB_CONFIG; - - switch (config) { - case SLOTB_IS_NONE: - config1 |= SLOTB_NONE; - break; - case SLOTB_IS_PCCARD: - config1 |= SLOTB_PCCARD; - break; - case SLOTB_IS_CF: - config1 |= SLOTB_CF; - break; - case SLOTB_IS_FLASHROM: - config1 |= SLOTB_FLASHROM; - break; - default: - break; - } - - outw(config1, CONFIGURATION1); -} - -static inline uint8_t exca_read_byte(int slot, uint8_t index) -{ - if (slot == CARD_SLOTB) - index += CARD_SLOTB_OFFSET; - - outb(index, CARD_CONTROLLER_INDEX); - return inb(CARD_CONTROLLER_DATA); -} - -static inline uint16_t exca_read_word(int slot, uint8_t index) -{ - uint16_t data; - - if (slot == CARD_SLOTB) - index += CARD_SLOTB_OFFSET; - - outb(index++, CARD_CONTROLLER_INDEX); - data = inb(CARD_CONTROLLER_DATA); - - outb(index, CARD_CONTROLLER_INDEX); - data |= ((uint16_t)inb(CARD_CONTROLLER_DATA)) << 8; - - return data; -} - -static inline uint8_t exca_write_byte(int slot, uint8_t index, uint8_t data) -{ - if (slot == CARD_SLOTB) - index += CARD_SLOTB_OFFSET; - - outb(index, CARD_CONTROLLER_INDEX); - outb(data, CARD_CONTROLLER_DATA); - - return data; -} - -static inline uint16_t exca_write_word(int slot, uint8_t index, uint16_t data) -{ - if (slot == CARD_SLOTB) - index += CARD_SLOTB_OFFSET; - - outb(index++, CARD_CONTROLLER_INDEX); - outb(data, CARD_CONTROLLER_DATA); - - outb(index, CARD_CONTROLLER_INDEX); - outb((uint8_t)(data >> 8), CARD_CONTROLLER_DATA); - - return data; -} - -static inline int search_nonuse_irq(void) -{ - int i; - - for (i = 0; i < 16; i++) { - if (vrc4171_irq_mask & (1 << i)) { - vrc4171_irq_mask &= ~(1 << i); - return i; - } - } - - return -1; -} - -static int pccard_init(struct pcmcia_socket *sock) -{ - struct vrc4171_socket *socket; - unsigned int slot; - - sock->features |= SS_CAP_PCCARD | SS_CAP_PAGE_REGS; - sock->irq_mask = 0; - sock->map_size = 0x1000; - sock->pci_irq = vrc4171_irq; - - slot = sock->sock; - socket = &vrc4171_sockets[slot]; - socket->csc_irq = search_nonuse_irq(); - socket->io_irq = search_nonuse_irq(); - spin_lock_init(&socket->lock); - - return 0; -} - -static int pccard_get_status(struct pcmcia_socket *sock, u_int *value) -{ - unsigned int slot; - uint8_t status, sense; - u_int val = 0; - - if (sock == NULL || sock->sock >= CARD_MAX_SLOTS || value == NULL) - return -EINVAL; - - slot = sock->sock; - - status = exca_read_byte(slot, I365_STATUS); - if (exca_read_byte(slot, I365_INTCTL) & I365_PC_IOCARD) { - if (status & I365_CS_STSCHG) - val |= SS_STSCHG; - } else { - if (!(status & I365_CS_BVD1)) - val |= SS_BATDEAD; - else if ((status & (I365_CS_BVD1 | I365_CS_BVD2)) == I365_CS_BVD1) - val |= SS_BATWARN; - } - if ((status & I365_CS_DETECT) == I365_CS_DETECT) - val |= SS_DETECT; - if (status & I365_CS_WRPROT) - val |= SS_WRPROT; - if (status & I365_CS_READY) - val |= SS_READY; - if (status & I365_CS_POWERON) - val |= SS_POWERON; - - sense = exca_read_byte(slot, CARD_VOLTAGE_SENSE); - switch (sense) { - case VCC_3VORXV_CAPABLE: - val |= SS_3VCARD | SS_XVCARD; - break; - case VCC_XV_ONLY: - val |= SS_XVCARD; - break; - case VCC_3V_CAPABLE: - val |= SS_3VCARD; - break; - default: - /* 5V only */ - break; - } - - *value = val; - - return 0; -} - -static inline uint8_t set_Vcc_value(u_char Vcc) -{ - switch (Vcc) { - case 33: - return VCC_3V; - case 50: - return VCC_5V; - } - - /* Small voltage is chosen for safety. */ - return VCC_3V; -} - -static int pccard_set_socket(struct pcmcia_socket *sock, socket_state_t *state) -{ - struct vrc4171_socket *socket; - unsigned int slot; - uint8_t voltage, power, control, cscint; - - if (sock == NULL || sock->sock >= CARD_MAX_SLOTS || - (state->Vpp != state->Vcc && state->Vpp != 0) || - (state->Vcc != 50 && state->Vcc != 33 && state->Vcc != 0)) - return -EINVAL; - - slot = sock->sock; - socket = &vrc4171_sockets[slot]; - - spin_lock_irq(&socket->lock); - - voltage = set_Vcc_value(state->Vcc); - exca_write_byte(slot, CARD_VOLTAGE_SELECT, voltage); - - power = POWER_ENABLE; - if (state->Vpp == state->Vcc) - power |= VPP_GET_VCC; - if (state->flags & SS_OUTPUT_ENA) - power |= I365_PWR_OUT; - exca_write_byte(slot, I365_POWER, power); - - control = 0; - if (state->io_irq != 0) - control |= socket->io_irq; - if (state->flags & SS_IOCARD) - control |= I365_PC_IOCARD; - if (state->flags & SS_RESET) - control &= ~I365_PC_RESET; - else - control |= I365_PC_RESET; - exca_write_byte(slot, I365_INTCTL, control); - - cscint = 0; - exca_write_byte(slot, I365_CSCINT, cscint); - exca_read_byte(slot, I365_CSC); /* clear CardStatus change */ - if (state->csc_mask != 0) - cscint |= socket->csc_irq << 8; - if (state->flags & SS_IOCARD) { - if (state->csc_mask & SS_STSCHG) - cscint |= I365_CSC_STSCHG; - } else { - if (state->csc_mask & SS_BATDEAD) - cscint |= I365_CSC_BVD1; - if (state->csc_mask & SS_BATWARN) - cscint |= I365_CSC_BVD2; - } - if (state->csc_mask & SS_READY) - cscint |= I365_CSC_READY; - if (state->csc_mask & SS_DETECT) - cscint |= I365_CSC_DETECT; - exca_write_byte(slot, I365_CSCINT, cscint); - - spin_unlock_irq(&socket->lock); - - return 0; -} - -static int pccard_set_io_map(struct pcmcia_socket *sock, struct pccard_io_map *io) -{ - unsigned int slot; - uint8_t ioctl, addrwin; - u_char map; - - if (sock == NULL || sock->sock >= CARD_MAX_SLOTS || - io == NULL || io->map >= IO_MAX_MAPS || - io->start > 0xffff || io->stop > 0xffff || io->start > io->stop) - return -EINVAL; - - slot = sock->sock; - map = io->map; - - addrwin = exca_read_byte(slot, I365_ADDRWIN); - if (addrwin & I365_ENA_IO(map)) { - addrwin &= ~I365_ENA_IO(map); - exca_write_byte(slot, I365_ADDRWIN, addrwin); - } - - exca_write_word(slot, I365_IO(map)+I365_W_START, io->start); - exca_write_word(slot, I365_IO(map)+I365_W_STOP, io->stop); - - ioctl = 0; - if (io->speed > 0) - ioctl |= I365_IOCTL_WAIT(map); - if (io->flags & MAP_16BIT) - ioctl |= I365_IOCTL_16BIT(map); - if (io->flags & MAP_AUTOSZ) - ioctl |= I365_IOCTL_IOCS16(map); - if (io->flags & MAP_0WS) - ioctl |= I365_IOCTL_0WS(map); - exca_write_byte(slot, I365_IOCTL, ioctl); - - if (io->flags & MAP_ACTIVE) { - addrwin |= I365_ENA_IO(map); - exca_write_byte(slot, I365_ADDRWIN, addrwin); - } - - return 0; -} - -static int pccard_set_mem_map(struct pcmcia_socket *sock, struct pccard_mem_map *mem) -{ - unsigned int slot; - uint16_t start, stop, offset; - uint8_t addrwin; - u_char map; - - if (sock == NULL || sock->sock >= CARD_MAX_SLOTS || - mem == NULL || mem->map >= MEM_MAX_MAPS || - mem->res->start < CARD_MEM_START || mem->res->start > CARD_MEM_END || - mem->res->end < CARD_MEM_START || mem->res->end > CARD_MEM_END || - mem->res->start > mem->res->end || - mem->card_start > CARD_MAX_MEM_OFFSET || - mem->speed > CARD_MAX_MEM_SPEED) - return -EINVAL; - - slot = sock->sock; - map = mem->map; - - addrwin = exca_read_byte(slot, I365_ADDRWIN); - if (addrwin & I365_ENA_MEM(map)) { - addrwin &= ~I365_ENA_MEM(map); - exca_write_byte(slot, I365_ADDRWIN, addrwin); - } - - start = (mem->res->start >> 12) & 0x3fff; - if (mem->flags & MAP_16BIT) - start |= I365_MEM_16BIT; - exca_write_word(slot, I365_MEM(map)+I365_W_START, start); - - stop = (mem->res->end >> 12) & 0x3fff; - switch (mem->speed) { - case 0: - break; - case 1: - stop |= I365_MEM_WS0; - break; - case 2: - stop |= I365_MEM_WS1; - break; - default: - stop |= I365_MEM_WS0 | I365_MEM_WS1; - break; - } - exca_write_word(slot, I365_MEM(map)+I365_W_STOP, stop); - - offset = (mem->card_start >> 12) & 0x3fff; - if (mem->flags & MAP_ATTRIB) - offset |= I365_MEM_REG; - if (mem->flags & MAP_WRPROT) - offset |= I365_MEM_WRPROT; - exca_write_word(slot, I365_MEM(map)+I365_W_OFF, offset); - - if (mem->flags & MAP_ACTIVE) { - addrwin |= I365_ENA_MEM(map); - exca_write_byte(slot, I365_ADDRWIN, addrwin); - } - - return 0; -} - -static struct pccard_operations vrc4171_pccard_operations = { - .init = pccard_init, - .get_status = pccard_get_status, - .set_socket = pccard_set_socket, - .set_io_map = pccard_set_io_map, - .set_mem_map = pccard_set_mem_map, -}; - -static inline unsigned int get_events(int slot) -{ - unsigned int events = 0; - uint8_t status, csc; - - status = exca_read_byte(slot, I365_STATUS); - csc = exca_read_byte(slot, I365_CSC); - - if (exca_read_byte(slot, I365_INTCTL) & I365_PC_IOCARD) { - if ((csc & I365_CSC_STSCHG) && (status & I365_CS_STSCHG)) - events |= SS_STSCHG; - } else { - if (csc & (I365_CSC_BVD1 | I365_CSC_BVD2)) { - if (!(status & I365_CS_BVD1)) - events |= SS_BATDEAD; - else if ((status & (I365_CS_BVD1 | I365_CS_BVD2)) == I365_CS_BVD1) - events |= SS_BATWARN; - } - } - if ((csc & I365_CSC_READY) && (status & I365_CS_READY)) - events |= SS_READY; - if ((csc & I365_CSC_DETECT) && ((status & I365_CS_DETECT) == I365_CS_DETECT)) - events |= SS_DETECT; - - return events; -} - -static irqreturn_t pccard_interrupt(int irq, void *dev_id) -{ - struct vrc4171_socket *socket; - unsigned int events; - irqreturn_t retval = IRQ_NONE; - uint16_t status; - - status = vrc4171_get_irq_status(); - if (status & IRQ_A) { - socket = &vrc4171_sockets[CARD_SLOTA]; - if (socket->slot == SLOT_INITIALIZED) { - if (status & (1 << socket->csc_irq)) { - events = get_events(CARD_SLOTA); - if (events != 0) { - pcmcia_parse_events(&socket->pcmcia_socket, events); - retval = IRQ_HANDLED; - } - } - } - } - - if (status & IRQ_B) { - socket = &vrc4171_sockets[CARD_SLOTB]; - if (socket->slot == SLOT_INITIALIZED) { - if (status & (1 << socket->csc_irq)) { - events = get_events(CARD_SLOTB); - if (events != 0) { - pcmcia_parse_events(&socket->pcmcia_socket, events); - retval = IRQ_HANDLED; - } - } - } - } - - return retval; -} - -static inline void reserve_using_irq(int slot) -{ - unsigned int irq; - - irq = exca_read_byte(slot, I365_INTCTL); - irq &= 0x0f; - vrc4171_irq_mask &= ~(1 << irq); - - irq = exca_read_byte(slot, I365_CSCINT); - irq = (irq & 0xf0) >> 4; - vrc4171_irq_mask &= ~(1 << irq); -} - -static int vrc4171_add_sockets(void) -{ - struct vrc4171_socket *socket; - int slot, retval; - - for (slot = 0; slot < CARD_MAX_SLOTS; slot++) { - if (slot == CARD_SLOTB && vrc4171_slotb == SLOTB_IS_NONE) - continue; - - socket = &vrc4171_sockets[slot]; - if (socket->slot != SLOT_PROBE) { - uint8_t addrwin; - - switch (socket->slot) { - case SLOT_NOPROBE_MEM: - addrwin = exca_read_byte(slot, I365_ADDRWIN); - addrwin &= 0x1f; - exca_write_byte(slot, I365_ADDRWIN, addrwin); - break; - case SLOT_NOPROBE_IO: - addrwin = exca_read_byte(slot, I365_ADDRWIN); - addrwin &= 0xc0; - exca_write_byte(slot, I365_ADDRWIN, addrwin); - break; - default: - break; - } - - reserve_using_irq(slot); - continue; - } - - sprintf(socket->name, "NEC VRC4171 Card Slot %1c", 'A' + slot); - socket->pcmcia_socket.dev.parent = &vrc4171_card_device.dev; - socket->pcmcia_socket.ops = &vrc4171_pccard_operations; - socket->pcmcia_socket.owner = THIS_MODULE; - - retval = pcmcia_register_socket(&socket->pcmcia_socket); - if (retval < 0) - return retval; - - exca_write_byte(slot, I365_ADDRWIN, 0); - exca_write_byte(slot, GLOBAL_CONTROL, 0); - - socket->slot = SLOT_INITIALIZED; - } - - return 0; -} - -static void vrc4171_remove_sockets(void) -{ - struct vrc4171_socket *socket; - int slot; - - for (slot = 0; slot < CARD_MAX_SLOTS; slot++) { - if (slot == CARD_SLOTB && vrc4171_slotb == SLOTB_IS_NONE) - continue; - - socket = &vrc4171_sockets[slot]; - if (socket->slot == SLOT_INITIALIZED) - pcmcia_unregister_socket(&socket->pcmcia_socket); - - socket->slot = SLOT_PROBE; - } -} - -static int vrc4171_card_setup(char *options) -{ - if (options == NULL || *options == '\0') - return 1; - - if (strncmp(options, "irq:", 4) == 0) { - int irq; - options += 4; - irq = simple_strtoul(options, &options, 0); - if (irq >= 0 && irq < nr_irqs) - vrc4171_irq = irq; - - if (*options != ',') - return 1; - options++; - } - - if (strncmp(options, "slota:", 6) == 0) { - options += 6; - if (*options != '\0') { - if (strncmp(options, "memnoprobe", 10) == 0) { - vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_MEM; - options += 10; - } else if (strncmp(options, "ionoprobe", 9) == 0) { - vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_IO; - options += 9; - } else if ( strncmp(options, "noprobe", 7) == 0) { - vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_ALL; - options += 7; - } - - if (*options != ',') - return 1; - options++; - } else - return 1; - - } - - if (strncmp(options, "slotb:", 6) == 0) { - options += 6; - if (*options != '\0') { - if (strncmp(options, "pccard", 6) == 0) { - vrc4171_slotb = SLOTB_IS_PCCARD; - options += 6; - } else if (strncmp(options, "cf", 2) == 0) { - vrc4171_slotb = SLOTB_IS_CF; - options += 2; - } else if (strncmp(options, "flashrom", 8) == 0) { - vrc4171_slotb = SLOTB_IS_FLASHROM; - options += 8; - } else if (strncmp(options, "none", 4) == 0) { - vrc4171_slotb = SLOTB_IS_NONE; - options += 4; - } - - if (*options != ',') - return 1; - options++; - - if (strncmp(options, "memnoprobe", 10) == 0) - vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_MEM; - if (strncmp(options, "ionoprobe", 9) == 0) - vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_IO; - if (strncmp(options, "noprobe", 7) == 0) - vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_ALL; - } - } - - return 1; -} - -__setup("vrc4171_card=", vrc4171_card_setup); - -static struct platform_driver vrc4171_card_driver = { - .driver = { - .name = vrc4171_card_name, - }, -}; - -static int vrc4171_card_init(void) -{ - int retval; - - retval = platform_driver_register(&vrc4171_card_driver); - if (retval < 0) - return retval; - - retval = platform_device_register(&vrc4171_card_device); - if (retval < 0) { - platform_driver_unregister(&vrc4171_card_driver); - return retval; - } - - vrc4171_set_multifunction_pin(vrc4171_slotb); - - retval = vrc4171_add_sockets(); - if (retval == 0) - retval = request_irq(vrc4171_irq, pccard_interrupt, IRQF_SHARED, - vrc4171_card_name, vrc4171_sockets); - - if (retval < 0) { - vrc4171_remove_sockets(); - platform_device_unregister(&vrc4171_card_device); - platform_driver_unregister(&vrc4171_card_driver); - return retval; - } - - printk(KERN_INFO "%s, connected to IRQ %d\n", - vrc4171_card_driver.driver.name, vrc4171_irq); - - return 0; -} - -static void vrc4171_card_exit(void) -{ - free_irq(vrc4171_irq, vrc4171_sockets); - vrc4171_remove_sockets(); - platform_device_unregister(&vrc4171_card_device); - platform_driver_unregister(&vrc4171_card_driver); -} - -module_init(vrc4171_card_init); -module_exit(vrc4171_card_exit); diff --git a/drivers/peci/controller/peci-aspeed.c b/drivers/peci/controller/peci-aspeed.c index 1925ddc13f002badd78274e698c4afd2ef5e2675..731c5d8f75c66a3a4865f48e640dcd6d85a3e69f 100644 --- a/drivers/peci/controller/peci-aspeed.c +++ b/drivers/peci/controller/peci-aspeed.c @@ -523,7 +523,7 @@ static int aspeed_peci_probe(struct platform_device *pdev) return PTR_ERR(priv->base); priv->irq = platform_get_irq(pdev, 0); - if (!priv->irq) + if (priv->irq < 0) return priv->irq; ret = devm_request_irq(&pdev->dev, priv->irq, aspeed_peci_irq_handler, diff --git a/drivers/peci/cpu.c b/drivers/peci/cpu.c index 68eb61c65d345f20a6d3fac0d5723a7dd0b9269e..de4a7b3e5966ede8ddeb8bf2262f6f1024510023 100644 --- a/drivers/peci/cpu.c +++ b/drivers/peci/cpu.c @@ -188,8 +188,6 @@ static void adev_release(struct device *dev) { struct auxiliary_device *adev = to_auxiliary_dev(dev); - auxiliary_device_uninit(adev); - kfree(adev->name); kfree(adev); } @@ -234,6 +232,7 @@ static void unregister_adev(void *_adev) struct auxiliary_device *adev = _adev; auxiliary_device_delete(adev); + auxiliary_device_uninit(adev); } static int devm_adev_add(struct device *dev, int idx) diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index 1e2d69453771d4aca267b0a64d4d0419c0dc820f..341010f20b777905f67d25dbb072fe4c515b487f 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -183,6 +183,13 @@ config APPLE_M1_CPU_PMU Provides support for the non-architectural CPU PMUs present on the Apple M1 SoCs and derivatives. +config ALIBABA_UNCORE_DRW_PMU + tristate "Alibaba T-Head Yitian 710 DDR Sub-system Driveway PMU driver" + depends on (ARM64 && ACPI) || COMPILE_TEST + help + Support for Driveway PMU events monitoring on Yitian 710 DDR + Sub-system. + source "drivers/perf/hisilicon/Kconfig" config MARVELL_CN10K_DDR_PMU diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile index 57a279c61df55285bb2a4c24452f8164e6ca52c8..050d04ee19dd78a33be71331b5a8994ce3fc22d9 100644 --- a/drivers/perf/Makefile +++ b/drivers/perf/Makefile @@ -20,3 +20,4 @@ obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o obj-$(CONFIG_APPLE_M1_CPU_PMU) += apple_m1_cpu_pmu.o +obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c new file mode 100644 index 0000000000000000000000000000000000000000..a7689fecb49d9611eb2b6f335f1212180e638660 --- /dev/null +++ b/drivers/perf/alibaba_uncore_drw_pmu.c @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Alibaba DDR Sub-System Driveway PMU driver + * + * Copyright (C) 2022 Alibaba Inc + */ + +#define ALI_DRW_PMUNAME "ali_drw" +#define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu" +#define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16 +#define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19 + +#define ALI_DRW_PMU_PA_SHIFT 12 +#define ALI_DRW_PMU_CNT_INIT 0x00000000 +#define ALI_DRW_CNT_MAX_PERIOD 0xffffffff +#define ALI_DRW_PMU_CYCLE_EVT_ID 0x80 + +#define ALI_DRW_PMU_CNT_CTRL 0xC00 +#define ALI_DRW_PMU_CNT_RST BIT(2) +#define ALI_DRW_PMU_CNT_STOP BIT(1) +#define ALI_DRW_PMU_CNT_START BIT(0) + +#define ALI_DRW_PMU_CNT_STATE 0xC04 +#define ALI_DRW_PMU_TEST_CTRL 0xC08 +#define ALI_DRW_PMU_CNT_PRELOAD 0xC0C + +#define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0) +#define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0) +#define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10 +#define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14 + +/* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */ +#define ALI_DRW_PMU_EVENT_SEL0 0xC68 +/* counter 0-3 use sel0, counter 4-7 use sel1...*/ +#define ALI_DRW_PMU_EVENT_SELn(n) \ + (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4) +#define ALI_DRW_PMCOM_CNT_EN BIT(7) +#define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0) +#define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \ + (8 * (n % 4)) + +/* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */ +#define ALI_DRW_PMU_COMMON_COUNTER0 0xC78 +#define ALI_DRW_PMU_COMMON_COUNTERn(n) \ + (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n)) + +#define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8 +#define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC +#define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0 +#define ALI_DRW_PMU_OV_INTR_CLR 0xCC4 +#define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8 +#define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8) +#define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0) +#define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0) + +static int ali_drw_cpuhp_state_num; + +static LIST_HEAD(ali_drw_pmu_irqs); +static DEFINE_MUTEX(ali_drw_pmu_irqs_lock); + +struct ali_drw_pmu_irq { + struct hlist_node node; + struct list_head irqs_node; + struct list_head pmus_node; + int irq_num; + int cpu; + refcount_t refcount; +}; + +struct ali_drw_pmu { + void __iomem *cfg_base; + struct device *dev; + + struct list_head pmus_node; + struct ali_drw_pmu_irq *irq; + int irq_num; + int cpu; + DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS); + struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; + int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; + + struct pmu pmu; +}; + +#define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu)) + +#define DRW_CONFIG_EVENTID GENMASK(7, 0) +#define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config) + +static ssize_t ali_drw_pmu_format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + + return sprintf(buf, "%s\n", (char *)eattr->var); +} + +/* + * PMU event attributes + */ +static ssize_t ali_drw_pmu_event_show(struct device *dev, + struct device_attribute *attr, char *page) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + + return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var); +} + +#define ALI_DRW_PMU_ATTR(_name, _func, _config) \ + (&((struct dev_ext_attribute[]) { \ + { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ + })[0].attr.attr) + +#define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \ + ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config) +#define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \ + ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config) + +static struct attribute *ali_drw_pmu_events_attrs[] = { + ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0), + ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1), + ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2), + ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3), + ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4), + ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7), + ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8), + ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9), + ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA), + ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB), + ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC), + ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD), + ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE), + ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF), + ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10), + ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11), + ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12), + ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13), + ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14), + ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15), + ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16), + ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17), + ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18), + ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20), + ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21), + ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26), + ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27), + ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28), + ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29), + ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A), + ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B), + ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D), + ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E), + ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30), + ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31), + ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34), + ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35), + ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36), + ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37), + ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38), + ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39), + ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A), + ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B), + ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C), + ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D), + ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80), + NULL, +}; + +static struct attribute_group ali_drw_pmu_events_attr_group = { + .name = "events", + .attrs = ali_drw_pmu_events_attrs, +}; + +static struct attribute *ali_drw_pmu_format_attr[] = { + ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"), + NULL, +}; + +static const struct attribute_group ali_drw_pmu_format_group = { + .name = "format", + .attrs = ali_drw_pmu_format_attr, +}; + +static ssize_t ali_drw_pmu_cpumask_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev)); + + return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu)); +} + +static struct device_attribute ali_drw_pmu_cpumask_attr = + __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL); + +static struct attribute *ali_drw_pmu_cpumask_attrs[] = { + &ali_drw_pmu_cpumask_attr.attr, + NULL, +}; + +static const struct attribute_group ali_drw_pmu_cpumask_attr_group = { + .attrs = ali_drw_pmu_cpumask_attrs, +}; + +static const struct attribute_group *ali_drw_pmu_attr_groups[] = { + &ali_drw_pmu_events_attr_group, + &ali_drw_pmu_cpumask_attr_group, + &ali_drw_pmu_format_group, + NULL, +}; + +/* find a counter for event, then in add func, hw.idx will equal to counter */ +static int ali_drw_get_counter_idx(struct perf_event *event) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + int idx; + + for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) { + if (!test_and_set_bit(idx, drw_pmu->used_mask)) + return idx; + } + + /* The counters are all in use. */ + return -EBUSY; +} + +static u64 ali_drw_pmu_read_counter(struct perf_event *event) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + u64 cycle_high, cycle_low; + + if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { + cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH); + cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK; + cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW); + cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK; + return (cycle_high << 32 | cycle_low); + } + + return readl(drw_pmu->cfg_base + + ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx)); +} + +static void ali_drw_pmu_event_update(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + u64 delta, prev, now; + + do { + prev = local64_read(&hwc->prev_count); + now = ali_drw_pmu_read_counter(event); + } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); + + /* handle overflow. */ + delta = now - prev; + if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) + delta &= ALI_DRW_PMU_OV_INTR_MASK; + else + delta &= ALI_DRW_CNT_MAX_PERIOD; + local64_add(delta, &event->count); +} + +static void ali_drw_pmu_event_set_period(struct perf_event *event) +{ + u64 pre_val; + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + + /* set a preload counter for test purpose */ + writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx, + drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); + + /* set conunter initial value */ + pre_val = ALI_DRW_PMU_CNT_INIT; + writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); + local64_set(&event->hw.prev_count, pre_val); + + /* set sel mode to zero to start test */ + writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); +} + +static void ali_drw_pmu_enable_counter(struct perf_event *event) +{ + u32 val, subval, reg, shift; + int counter = event->hw.idx; + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + + reg = ALI_DRW_PMU_EVENT_SELn(counter); + val = readl(drw_pmu->cfg_base + reg); + subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) | + FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]); + + shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); + val &= ~(GENMASK(7, 0) << shift); + val |= subval << shift; + + writel(val, drw_pmu->cfg_base + reg); +} + +static void ali_drw_pmu_disable_counter(struct perf_event *event) +{ + u32 val, reg, subval, shift; + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + int counter = event->hw.idx; + + reg = ALI_DRW_PMU_EVENT_SELn(counter); + val = readl(drw_pmu->cfg_base + reg); + subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) | + FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0); + + shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); + val &= ~(GENMASK(7, 0) << shift); + val |= subval << shift; + + writel(val, drw_pmu->cfg_base + reg); +} + +static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data) +{ + struct ali_drw_pmu_irq *irq = data; + struct ali_drw_pmu *drw_pmu; + irqreturn_t ret = IRQ_NONE; + + rcu_read_lock(); + list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) { + unsigned long status, clr_status; + struct perf_event *event; + unsigned int idx; + + for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { + event = drw_pmu->events[idx]; + if (!event) + continue; + ali_drw_pmu_disable_counter(event); + } + + /* common counter intr status */ + status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS); + status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status); + if (status) { + for_each_set_bit(idx, &status, + ALI_DRW_PMU_COMMON_MAX_COUNTERS) { + event = drw_pmu->events[idx]; + if (WARN_ON_ONCE(!event)) + continue; + ali_drw_pmu_event_update(event); + ali_drw_pmu_event_set_period(event); + } + + /* clear common counter intr status */ + clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1); + writel(clr_status, + drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); + } + + for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { + event = drw_pmu->events[idx]; + if (!event) + continue; + if (!(event->hw.state & PERF_HES_STOPPED)) + ali_drw_pmu_enable_counter(event); + } + if (status) + ret = IRQ_HANDLED; + } + rcu_read_unlock(); + return ret; +} + +static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device + *pdev, int irq_num) +{ + int ret; + struct ali_drw_pmu_irq *irq; + + list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) { + if (irq->irq_num == irq_num + && refcount_inc_not_zero(&irq->refcount)) + return irq; + } + + irq = kzalloc(sizeof(*irq), GFP_KERNEL); + if (!irq) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&irq->pmus_node); + + /* Pick one CPU to be the preferred one to use */ + irq->cpu = smp_processor_id(); + refcount_set(&irq->refcount, 1); + + /* + * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same + * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers + * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not + * share with other component. + */ + ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, + IRQF_SHARED, dev_name(&pdev->dev), irq); + if (ret < 0) { + dev_err(&pdev->dev, + "Fail to request IRQ:%d ret:%d\n", irq_num, ret); + goto out_free; + } + + ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); + if (ret) + goto out_free; + + ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num, + &irq->node); + if (ret) + goto out_free; + + irq->irq_num = irq_num; + list_add(&irq->irqs_node, &ali_drw_pmu_irqs); + + return irq; + +out_free: + kfree(irq); + return ERR_PTR(ret); +} + +static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu, + struct platform_device *pdev) +{ + int irq_num; + struct ali_drw_pmu_irq *irq; + + /* Read and init IRQ */ + irq_num = platform_get_irq(pdev, 0); + if (irq_num < 0) + return irq_num; + + mutex_lock(&ali_drw_pmu_irqs_lock); + irq = __ali_drw_pmu_init_irq(pdev, irq_num); + mutex_unlock(&ali_drw_pmu_irqs_lock); + + if (IS_ERR(irq)) + return PTR_ERR(irq); + + drw_pmu->irq = irq; + + mutex_lock(&ali_drw_pmu_irqs_lock); + list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node); + mutex_unlock(&ali_drw_pmu_irqs_lock); + + return 0; +} + +static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu) +{ + struct ali_drw_pmu_irq *irq = drw_pmu->irq; + + mutex_lock(&ali_drw_pmu_irqs_lock); + list_del_rcu(&drw_pmu->pmus_node); + + if (!refcount_dec_and_test(&irq->refcount)) { + mutex_unlock(&ali_drw_pmu_irqs_lock); + return; + } + + list_del(&irq->irqs_node); + mutex_unlock(&ali_drw_pmu_irqs_lock); + + WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); + cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num, + &irq->node); + kfree(irq); +} + +static int ali_drw_pmu_event_init(struct perf_event *event) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + struct perf_event *sibling; + struct device *dev = drw_pmu->pmu.dev; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + if (is_sampling_event(event)) { + dev_err(dev, "Sampling not supported!\n"); + return -EOPNOTSUPP; + } + + if (event->attach_state & PERF_ATTACH_TASK) { + dev_err(dev, "Per-task counter cannot allocate!\n"); + return -EOPNOTSUPP; + } + + event->cpu = drw_pmu->cpu; + if (event->cpu < 0) { + dev_err(dev, "Per-task mode not supported!\n"); + return -EOPNOTSUPP; + } + + if (event->group_leader != event && + !is_software_event(event->group_leader)) { + dev_err(dev, "driveway only allow one event!\n"); + return -EINVAL; + } + + for_each_sibling_event(sibling, event->group_leader) { + if (sibling != event && !is_software_event(sibling)) { + dev_err(dev, "driveway event not allowed!\n"); + return -EINVAL; + } + } + + /* reset all the pmu counters */ + writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); + + hwc->idx = -1; + + return 0; +} + +static void ali_drw_pmu_start(struct perf_event *event, int flags) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + + event->hw.state = 0; + + if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { + writel(ALI_DRW_PMU_CNT_START, + drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); + return; + } + + ali_drw_pmu_event_set_period(event); + if (flags & PERF_EF_RELOAD) { + unsigned long prev_raw_count = + local64_read(&event->hw.prev_count); + writel(prev_raw_count, + drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); + } + + ali_drw_pmu_enable_counter(event); + + writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); +} + +static void ali_drw_pmu_stop(struct perf_event *event, int flags) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + + if (event->hw.state & PERF_HES_STOPPED) + return; + + if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID) + ali_drw_pmu_disable_counter(event); + + writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); + + ali_drw_pmu_event_update(event); + event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; +} + +static int ali_drw_pmu_add(struct perf_event *event, int flags) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = -1; + int evtid; + + evtid = GET_DRW_EVENTID(event); + + if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) { + idx = ali_drw_get_counter_idx(event); + if (idx < 0) + return idx; + drw_pmu->events[idx] = event; + drw_pmu->evtids[idx] = evtid; + } + hwc->idx = idx; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + + if (flags & PERF_EF_START) + ali_drw_pmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + + return 0; +} + +static void ali_drw_pmu_del(struct perf_event *event, int flags) +{ + struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + ali_drw_pmu_stop(event, PERF_EF_UPDATE); + + if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) { + drw_pmu->events[idx] = NULL; + drw_pmu->evtids[idx] = 0; + clear_bit(idx, drw_pmu->used_mask); + } + + perf_event_update_userpage(event); +} + +static void ali_drw_pmu_read(struct perf_event *event) +{ + ali_drw_pmu_event_update(event); +} + +static int ali_drw_pmu_probe(struct platform_device *pdev) +{ + struct ali_drw_pmu *drw_pmu; + struct resource *res; + char *name; + int ret; + + drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL); + if (!drw_pmu) + return -ENOMEM; + + drw_pmu->dev = &pdev->dev; + platform_set_drvdata(pdev, drw_pmu); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(drw_pmu->cfg_base)) + return PTR_ERR(drw_pmu->cfg_base); + + name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx", + (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT)); + if (!name) + return -ENOMEM; + + writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); + + /* enable the generation of interrupt by all common counters */ + writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, + drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL); + + /* clearing interrupt status */ + writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); + + drw_pmu->cpu = smp_processor_id(); + + ret = ali_drw_pmu_init_irq(drw_pmu, pdev); + if (ret) + return ret; + + drw_pmu->pmu = (struct pmu) { + .module = THIS_MODULE, + .task_ctx_nr = perf_invalid_context, + .event_init = ali_drw_pmu_event_init, + .add = ali_drw_pmu_add, + .del = ali_drw_pmu_del, + .start = ali_drw_pmu_start, + .stop = ali_drw_pmu_stop, + .read = ali_drw_pmu_read, + .attr_groups = ali_drw_pmu_attr_groups, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + }; + + ret = perf_pmu_register(&drw_pmu->pmu, name, -1); + if (ret) { + dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n"); + ali_drw_pmu_uninit_irq(drw_pmu); + } + + return ret; +} + +static int ali_drw_pmu_remove(struct platform_device *pdev) +{ + struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); + + /* disable the generation of interrupt by all common counters */ + writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, + drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL); + + ali_drw_pmu_uninit_irq(drw_pmu); + perf_pmu_unregister(&drw_pmu->pmu); + + return 0; +} + +static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct ali_drw_pmu_irq *irq; + struct ali_drw_pmu *drw_pmu; + unsigned int target; + int ret; + cpumask_t node_online_cpus; + + irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); + if (cpu != irq->cpu) + return 0; + + ret = cpumask_and(&node_online_cpus, + cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); + if (ret) + target = cpumask_any_but(&node_online_cpus, cpu); + else + target = cpumask_any_but(cpu_online_mask, cpu); + + if (target >= nr_cpu_ids) + return 0; + + /* We're only reading, but this isn't the place to be involving RCU */ + mutex_lock(&ali_drw_pmu_irqs_lock); + list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node) + perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target); + mutex_unlock(&ali_drw_pmu_irqs_lock); + + WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); + irq->cpu = target; + + return 0; +} + +/* + * Due to historical reasons, the HID used in the production environment is + * ARMHD700, so we leave ARMHD700 as Compatible ID. + */ +static const struct acpi_device_id ali_drw_acpi_match[] = { + {"BABA5000", 0}, + {"ARMHD700", 0}, + {} +}; + +MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match); + +static struct platform_driver ali_drw_pmu_driver = { + .driver = { + .name = "ali_drw_pmu", + .acpi_match_table = ali_drw_acpi_match, + }, + .probe = ali_drw_pmu_probe, + .remove = ali_drw_pmu_remove, +}; + +static int __init ali_drw_pmu_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "ali_drw_pmu:online", + NULL, ali_drw_pmu_offline_cpu); + + if (ret < 0) { + pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n", + ret); + return ret; + } + ali_drw_cpuhp_state_num = ret; + + ret = platform_driver_register(&ali_drw_pmu_driver); + if (ret) + cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); + + return ret; +} + +static void __exit ali_drw_pmu_exit(void) +{ + platform_driver_unregister(&ali_drw_pmu_driver); + cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); +} + +module_init(ali_drw_pmu_init); +module_exit(ali_drw_pmu_exit); + +MODULE_AUTHOR("Hongbo Yao "); +MODULE_AUTHOR("Neng Chen "); +MODULE_AUTHOR("Shuai Xue "); +MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 80d8309652a4dafcc8dac6c8bf3eff592b2d5254..b80a9b74662b1d86fa931d2f5e49dc32da580cc2 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -36,7 +36,7 @@ #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0) #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) -#define CMN_CHILD_NODE_ADDR GENMASK(27, 0) +#define CMN_CHILD_NODE_ADDR GENMASK(29, 0) #define CMN_CHILD_NODE_EXTERNAL BIT(31) #define CMN_MAX_DIMENSION 12 diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c index a36698a90d2f2640014426edfff60c452ad5e96d..4a15c86f45efbf1a09b46915eb768d7258d07e1a 100644 --- a/drivers/perf/arm_dsu_pmu.c +++ b/drivers/perf/arm_dsu_pmu.c @@ -639,6 +639,7 @@ static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask) static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask) { #ifdef CONFIG_ACPI + struct acpi_device *parent_adev = acpi_dev_parent(ACPI_COMPANION(dev)); int cpu; /* @@ -653,8 +654,7 @@ static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask) continue; acpi_dev = ACPI_COMPANION(cpu_dev); - if (acpi_dev && - acpi_dev->parent == ACPI_COMPANION(dev)->parent) + if (acpi_dev && acpi_dev_parent(acpi_dev) == parent_adev) cpumask_set_cpu(cpu, mask); } #endif diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index 59d3980b8ca2a201d3e1510468364104ec75da63..3f07df5a7e950af47545b45718303460a28438ca 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -894,7 +894,7 @@ static struct arm_pmu *__armpmu_alloc(gfp_t flags) * pmu::filter_match callback and pmu::event_init group * validation). */ - .capabilities = PERF_PMU_CAP_HETEROGENEOUS_CPUS, + .capabilities = PERF_PMU_CAP_HETEROGENEOUS_CPUS | PERF_PMU_CAP_EXTENDED_REGS, }; pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] = diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c index 513de1f54e2d7371ceca11b9427a265394aeadc9..933b96e243b84b1354b9d2f4eae6cbea0838cb04 100644 --- a/drivers/perf/arm_pmu_platform.c +++ b/drivers/perf/arm_pmu_platform.c @@ -117,7 +117,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) if (num_irqs == 1) { int irq = platform_get_irq(pdev, 0); - if (irq && irq_is_percpu_devid(irq)) + if ((irq > 0) && irq_is_percpu_devid(irq)) return pmu_parse_percpu_irq(pmu, irq); } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index b65a7d9640e1549ea476418a52f0884748ea2550..00e3a637f7b6320222a9b3a8ef431613ca1dae15 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -44,7 +44,9 @@ * This allows us to perform the check, i.e, perfmon_capable(), * in the context of the event owner, once, during the event_init(). */ -#define SPE_PMU_HW_FLAGS_CX BIT(0) +#define SPE_PMU_HW_FLAGS_CX 0x00001 + +static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX); static void set_spe_event_has_cx(struct perf_event *event) { @@ -674,9 +676,9 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) static u64 arm_spe_pmsevfr_res0(u16 pmsver) { switch (pmsver) { - case ID_AA64DFR0_PMSVER_8_2: + case ID_AA64DFR0_EL1_PMSVer_IMP: return SYS_PMSEVFR_EL1_RES0_8_2; - case ID_AA64DFR0_PMSVER_8_3: + case ID_AA64DFR0_EL1_PMSVer_V1P1: /* Return the highest version we support in default */ default: return SYS_PMSEVFR_EL1_RES0_8_3; @@ -958,7 +960,7 @@ static void __arm_spe_pmu_dev_probe(void *info) struct device *dev = &spe_pmu->pdev->dev; fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), - ID_AA64DFR0_PMSVER_SHIFT); + ID_AA64DFR0_EL1_PMSVer_SHIFT); if (!fld) { dev_err(dev, "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index 30234c261b05c33b3616a4c98114dee478f64bbc..aaca6db7d8f6c47f24a43a4831031e4afca1a2c9 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -840,16 +840,16 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) { struct platform_device *pdev = to_platform_device(dev->parent); struct platform_device *sdev = to_platform_device(dev); - struct acpi_device *adev = ACPI_COMPANION(dev); struct l2cache_pmu *l2cache_pmu = data; struct cluster_pmu *cluster; - unsigned long fw_cluster_id; + u64 fw_cluster_id; int err; int irq; - if (!adev || kstrtoul(adev->pnp.unique_id, 10, &fw_cluster_id) < 0) { + err = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &fw_cluster_id); + if (err) { dev_err(&pdev->dev, "unable to read ACPI uid\n"); - return -ENODEV; + return err; } cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL); @@ -879,7 +879,7 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) } dev_info(&pdev->dev, - "Registered L2 cache PMU cluster %ld\n", fw_cluster_id); + "Registered L2 cache PMU cluster %lld\n", fw_cluster_id); spin_lock_init(&cluster->pmu_lock); diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c index 1ff2ff6582bf86808a107876c2b2473b64df24a9..346311a05460b240ef33228eb5cb067eab00bb25 100644 --- a/drivers/perf/qcom_l3_pmu.c +++ b/drivers/perf/qcom_l3_pmu.c @@ -742,7 +742,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) l3pmu = devm_kzalloc(&pdev->dev, sizeof(*l3pmu), GFP_KERNEL); name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "l3cache_%s_%s", - acpi_dev->parent->pnp.unique_id, acpi_dev->pnp.unique_id); + acpi_dev_parent(acpi_dev)->pnp.unique_id, + acpi_dev->pnp.unique_id); if (!l3pmu || !name) return -ENOMEM; diff --git a/drivers/perf/riscv_pmu_legacy.c b/drivers/perf/riscv_pmu_legacy.c index 2c20b0de8cb094aff055cc8fa915db0c4ff490b0..ca9e20bfc7acae44381b730bc5a24476f3239d2c 100644 --- a/drivers/perf/riscv_pmu_legacy.c +++ b/drivers/perf/riscv_pmu_legacy.c @@ -14,7 +14,6 @@ #define RISCV_PMU_LEGACY_CYCLE 0 #define RISCV_PMU_LEGACY_INSTRET 1 -#define RISCV_PMU_LEGACY_NUM_CTR 2 static bool pmu_init_done; @@ -83,7 +82,8 @@ static void pmu_legacy_init(struct riscv_pmu *pmu) { pr_info("Legacy PMU implementation is available\n"); - pmu->num_counters = RISCV_PMU_LEGACY_NUM_CTR; + pmu->cmask = BIT(RISCV_PMU_LEGACY_CYCLE) | + BIT(RISCV_PMU_LEGACY_INSTRET); pmu->ctr_start = pmu_legacy_ctr_start; pmu->ctr_stop = NULL; pmu->event_map = pmu_legacy_event_map; diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 6f6681bbfd36de515b52bfc58ed6e04a1d34518f..3852c18362f53ec2630a7f10c5a6d6e6baf59a9c 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -271,7 +272,6 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) struct sbiret ret; int idx; uint64_t cbase = 0; - uint64_t cmask = GENMASK_ULL(rvpmu->num_counters - 1, 0); unsigned long cflags = 0; if (event->attr.exclude_kernel) @@ -281,11 +281,12 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) /* retrieve the available counter index */ #if defined(CONFIG_32BIT) - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, cmask, - cflags, hwc->event_base, hwc->config, hwc->config >> 32); + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + rvpmu->cmask, cflags, hwc->event_base, hwc->config, + hwc->config >> 32); #else - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, cmask, - cflags, hwc->event_base, hwc->config, 0); + ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + rvpmu->cmask, cflags, hwc->event_base, hwc->config, 0); #endif if (ret.error) { pr_debug("Not able to find a counter for event %lx config %llx\n", @@ -294,7 +295,7 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) } idx = ret.value; - if (idx >= rvpmu->num_counters || !pmu_ctr_list[idx].value) + if (!test_bit(idx, &rvpmu->cmask) || !pmu_ctr_list[idx].value) return -ENOENT; /* Additional sanity check for the counter id */ @@ -463,7 +464,7 @@ static int pmu_sbi_find_num_ctrs(void) return sbi_err_map_linux_errno(ret.error); } -static int pmu_sbi_get_ctrinfo(int nctr) +static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) { struct sbiret ret; int i, num_hw_ctr = 0, num_fw_ctr = 0; @@ -473,11 +474,14 @@ static int pmu_sbi_get_ctrinfo(int nctr) if (!pmu_ctr_list) return -ENOMEM; - for (i = 0; i <= nctr; i++) { + for (i = 0; i < nctr; i++) { ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_GET_INFO, i, 0, 0, 0, 0, 0); if (ret.error) /* The logical counter ids are not expected to be contiguous */ continue; + + *mask |= BIT(i); + cinfo.value = ret.value; if (cinfo.type == SBI_PMU_CTR_TYPE_FW) num_fw_ctr++; @@ -498,7 +502,7 @@ static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) * which may include counters that are not enabled yet. */ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - 0, GENMASK_ULL(pmu->num_counters - 1, 0), 0, 0, 0, 0); + 0, pmu->cmask, 0, 0, 0, 0); } static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) @@ -567,6 +571,7 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) unsigned long overflow; unsigned long overflowed_ctrs = 0; struct cpu_hw_events *cpu_hw_evt = dev; + u64 start_clock = sched_clock(); if (WARN_ON_ONCE(!cpu_hw_evt)) return IRQ_NONE; @@ -635,7 +640,9 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) perf_event_overflow(event, &data, regs); } } + pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs); + perf_sample_event_took(sched_clock() - start_clock); return IRQ_HANDLED; } @@ -645,8 +652,11 @@ static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); - /* Enable the access for TIME csr only from the user mode now */ - csr_write(CSR_SCOUNTEREN, 0x2); + /* + * Enable the access for CYCLE, TIME, and INSTRET CSRs from userspace, + * as is necessary to maintain uABI compatibility. + */ + csr_write(CSR_SCOUNTEREN, 0x7); /* Stop all the counters so that they can be enabled from perf */ pmu_sbi_stop_all(pmu); @@ -788,8 +798,9 @@ static void riscv_pmu_destroy(struct riscv_pmu *pmu) static int pmu_sbi_device_probe(struct platform_device *pdev) { struct riscv_pmu *pmu = NULL; - int num_counters; + unsigned long cmask = 0; int ret = -ENODEV; + int num_counters; pr_info("SBI PMU extension is available\n"); pmu = riscv_pmu_alloc(); @@ -803,7 +814,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } /* cache all the information about counters now */ - if (pmu_sbi_get_ctrinfo(num_counters)) + if (pmu_sbi_get_ctrinfo(num_counters, &cmask)) goto out_free; ret = pmu_sbi_setup_irqs(pmu, pdev); @@ -812,8 +823,9 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; } + pmu->pmu.attr_groups = riscv_pmu_attr_groups; - pmu->num_counters = num_counters; + pmu->cmask = cmask; pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; pmu->event_map = pmu_sbi_event_map; diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig index 300b0f2b5f8421e587f05ea1449ec358e013ad0f..7bd00a11d074b4c2b04ee1756119d89c3d83cb21 100644 --- a/drivers/phy/Kconfig +++ b/drivers/phy/Kconfig @@ -91,6 +91,7 @@ source "drivers/phy/rockchip/Kconfig" source "drivers/phy/samsung/Kconfig" source "drivers/phy/socionext/Kconfig" source "drivers/phy/st/Kconfig" +source "drivers/phy/sunplus/Kconfig" source "drivers/phy/tegra/Kconfig" source "drivers/phy/ti/Kconfig" source "drivers/phy/intel/Kconfig" diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile index 01e9efffc726fbd1d625b5593480f1fc48ba2abb..54f312c10a408aa81fc4251fe75dd90e54d0ae73 100644 --- a/drivers/phy/Makefile +++ b/drivers/phy/Makefile @@ -31,6 +31,7 @@ obj-y += allwinner/ \ samsung/ \ socionext/ \ st/ \ + sunplus/ \ tegra/ \ ti/ \ xilinx/ diff --git a/drivers/phy/allwinner/phy-sun4i-usb.c b/drivers/phy/allwinner/phy-sun4i-usb.c index d5f3b42eb8ce69e2407cd9b9d863c51365ace970..3a3831f6059a3f5e9733902a16ad947f34697ed6 100644 --- a/drivers/phy/allwinner/phy-sun4i-usb.c +++ b/drivers/phy/allwinner/phy-sun4i-usb.c @@ -768,7 +768,7 @@ static int sun4i_usb_phy_probe(struct platform_device *pdev) if (data->cfg->dedicated_clocks) snprintf(name, sizeof(name), "usb%d_phy", i); else - strlcpy(name, "usb_phy", sizeof(name)); + strscpy(name, "usb_phy", sizeof(name)); phy->clk = devm_clk_get(dev, name); if (IS_ERR(phy->clk)) { diff --git a/drivers/phy/amlogic/phy-meson-axg-mipi-pcie-analog.c b/drivers/phy/amlogic/phy-meson-axg-mipi-pcie-analog.c index 1027ece6ca123fc81b6a10600c327fa7088edca7..a3e1108b736d6540be073b0d98aa000002a744a7 100644 --- a/drivers/phy/amlogic/phy-meson-axg-mipi-pcie-analog.c +++ b/drivers/phy/amlogic/phy-meson-axg-mipi-pcie-analog.c @@ -197,7 +197,7 @@ static int phy_axg_mipi_pcie_analog_probe(struct platform_device *pdev) struct phy_provider *phy; struct device *dev = &pdev->dev; struct phy_axg_mipi_pcie_analog_priv *priv; - struct device_node *np = dev->of_node; + struct device_node *np = dev->of_node, *parent_np; struct regmap *map; int ret; @@ -206,7 +206,9 @@ static int phy_axg_mipi_pcie_analog_probe(struct platform_device *pdev) return -ENOMEM; /* Get the hhi system controller node */ - map = syscon_node_to_regmap(of_get_parent(dev->of_node)); + parent_np = of_get_parent(dev->of_node); + map = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(map)) { dev_err(dev, "failed to get HHI regmap\n"); diff --git a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c index 54d65a6f0fccfdfea01918be8403218d8ee4d393..d2a1da8d9e58880186df69eb02bd281f34dd6111 100644 --- a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c +++ b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c @@ -388,7 +388,6 @@ static int phy_g12a_usb3_pcie_probe(struct platform_device *pdev) struct phy_g12a_usb3_pcie_priv *priv; struct phy_provider *phy_provider; void __iomem *base; - int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -408,43 +407,24 @@ static int phy_g12a_usb3_pcie_probe(struct platform_device *pdev) if (IS_ERR(priv->regmap_cr)) return PTR_ERR(priv->regmap_cr); - priv->clk_ref = devm_clk_get(dev, "ref_clk"); + priv->clk_ref = devm_clk_get_enabled(dev, "ref_clk"); if (IS_ERR(priv->clk_ref)) return PTR_ERR(priv->clk_ref); - ret = clk_prepare_enable(priv->clk_ref); - if (ret) - return ret; - priv->reset = devm_reset_control_array_get_exclusive(dev); - if (IS_ERR(priv->reset)) { - ret = PTR_ERR(priv->reset); - goto err_disable_clk_ref; - } + if (IS_ERR(priv->reset)) + return PTR_ERR(priv->reset); priv->phy = devm_phy_create(dev, np, &phy_g12a_usb3_pcie_ops); - if (IS_ERR(priv->phy)) { - ret = PTR_ERR(priv->phy); - dev_err_probe(dev, ret, "failed to create PHY\n"); - goto err_disable_clk_ref; - } + if (IS_ERR(priv->phy)) + return dev_err_probe(dev, PTR_ERR(priv->phy), "failed to create PHY\n"); phy_set_drvdata(priv->phy, priv); dev_set_drvdata(dev, priv); phy_provider = devm_of_phy_provider_register(dev, phy_g12a_usb3_pcie_xlate); - if (IS_ERR(phy_provider)) { - ret = PTR_ERR(phy_provider); - goto err_disable_clk_ref; - } - - return 0; - -err_disable_clk_ref: - clk_disable_unprepare(priv->clk_ref); - - return ret; + return PTR_ERR_OR_ZERO(phy_provider); } static const struct of_device_id phy_g12a_usb3_pcie_of_match[] = { diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig index 93a6a8ee47167313251b3823a55aa67090e8bab5..1d89a2fd9b79b328ec5299fffaf97a12f6b34243 100644 --- a/drivers/phy/broadcom/Kconfig +++ b/drivers/phy/broadcom/Kconfig @@ -93,11 +93,11 @@ config PHY_BRCM_SATA config PHY_BRCM_USB tristate "Broadcom STB USB PHY driver" - depends on ARCH_BCM4908 || ARCH_BRCMSTB || COMPILE_TEST + depends on ARCH_BCMBCA || ARCH_BRCMSTB || COMPILE_TEST depends on OF select GENERIC_PHY select SOC_BRCMSTB if ARCH_BRCMSTB - default ARCH_BCM4908 || ARCH_BRCMSTB + default ARCH_BCMBCA || ARCH_BRCMSTB help Enable this to support the Broadcom STB USB PHY. This driver is required by the USB XHCI, EHCI and OHCI diff --git a/drivers/phy/freescale/phy-fsl-imx8m-pcie.c b/drivers/phy/freescale/phy-fsl-imx8m-pcie.c index ad7d2edfc41469ec66f9d829d1ae8c34f5eeeee5..c93286483b4259178a5d46ef0d73f708af564825 100644 --- a/drivers/phy/freescale/phy-fsl-imx8m-pcie.c +++ b/drivers/phy/freescale/phy-fsl-imx8m-pcie.c @@ -59,7 +59,7 @@ struct imx8_pcie_phy { bool clkreq_unused; }; -static int imx8_pcie_phy_init(struct phy *phy) +static int imx8_pcie_phy_power_on(struct phy *phy) { int ret; u32 val, pad_mode; @@ -137,14 +137,14 @@ static int imx8_pcie_phy_init(struct phy *phy) return ret; } -static int imx8_pcie_phy_power_on(struct phy *phy) +static int imx8_pcie_phy_init(struct phy *phy) { struct imx8_pcie_phy *imx8_phy = phy_get_drvdata(phy); return clk_prepare_enable(imx8_phy->clk); } -static int imx8_pcie_phy_power_off(struct phy *phy) +static int imx8_pcie_phy_exit(struct phy *phy) { struct imx8_pcie_phy *imx8_phy = phy_get_drvdata(phy); @@ -155,8 +155,8 @@ static int imx8_pcie_phy_power_off(struct phy *phy) static const struct phy_ops imx8_pcie_phy_ops = { .init = imx8_pcie_phy_init, + .exit = imx8_pcie_phy_exit, .power_on = imx8_pcie_phy_power_on, - .power_off = imx8_pcie_phy_power_off, .owner = THIS_MODULE, }; diff --git a/drivers/phy/intel/phy-intel-lgm-combo.c b/drivers/phy/intel/phy-intel-lgm-combo.c index 157683d10367014e472446b3402032fd9d7dc986..6010e246d52e70689dcb4e3c7c537909c7e68e09 100644 --- a/drivers/phy/intel/phy-intel-lgm-combo.c +++ b/drivers/phy/intel/phy-intel-lgm-combo.c @@ -413,44 +413,29 @@ static int intel_cbphy_fwnode_parse(struct intel_combo_phy *cbphy) u32 val; cbphy->core_clk = devm_clk_get(dev, NULL); - if (IS_ERR(cbphy->core_clk)) { - ret = PTR_ERR(cbphy->core_clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Get clk failed:%d!\n", ret); - return ret; - } + if (IS_ERR(cbphy->core_clk)) + return dev_err_probe(dev, PTR_ERR(cbphy->core_clk), + "Get clk failed!\n"); cbphy->core_rst = devm_reset_control_get_optional(dev, "core"); - if (IS_ERR(cbphy->core_rst)) { - ret = PTR_ERR(cbphy->core_rst); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Get core reset control err: %d!\n", ret); - return ret; - } + if (IS_ERR(cbphy->core_rst)) + return dev_err_probe(dev, PTR_ERR(cbphy->core_rst), + "Get core reset control err!\n"); cbphy->phy_rst = devm_reset_control_get_optional(dev, "phy"); - if (IS_ERR(cbphy->phy_rst)) { - ret = PTR_ERR(cbphy->phy_rst); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Get PHY reset control err: %d!\n", ret); - return ret; - } + if (IS_ERR(cbphy->phy_rst)) + return dev_err_probe(dev, PTR_ERR(cbphy->phy_rst), + "Get PHY reset control err!\n"); cbphy->iphy[0].app_rst = devm_reset_control_get_optional(dev, "iphy0"); - if (IS_ERR(cbphy->iphy[0].app_rst)) { - ret = PTR_ERR(cbphy->iphy[0].app_rst); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Get phy0 reset control err: %d!\n", ret); - return ret; - } + if (IS_ERR(cbphy->iphy[0].app_rst)) + return dev_err_probe(dev, PTR_ERR(cbphy->iphy[0].app_rst), + "Get phy0 reset control err!\n"); cbphy->iphy[1].app_rst = devm_reset_control_get_optional(dev, "iphy1"); - if (IS_ERR(cbphy->iphy[1].app_rst)) { - ret = PTR_ERR(cbphy->iphy[1].app_rst); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Get phy1 reset control err: %d!\n", ret); - return ret; - } + if (IS_ERR(cbphy->iphy[1].app_rst)) + return dev_err_probe(dev, PTR_ERR(cbphy->iphy[1].app_rst), + "Get phy1 reset control err!\n"); cbphy->app_base = devm_platform_ioremap_resource_byname(pdev, "app"); if (IS_ERR(cbphy->app_base)) diff --git a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c index a4d7d9bd100d31bfc915d14f9f211daae8768938..67712c77d806f2658e87ebd80fbe701c2463989a 100644 --- a/drivers/phy/marvell/phy-mvebu-a3700-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-a3700-comphy.c @@ -274,7 +274,6 @@ struct mvebu_a3700_comphy_lane { int submode; bool invert_tx; bool invert_rx; - bool needs_reset; }; struct gbe_phy_init_data_fix { @@ -1097,40 +1096,12 @@ mvebu_a3700_comphy_pcie_power_off(struct mvebu_a3700_comphy_lane *lane) 0x0, PU_PLL_BIT | PU_RX_BIT | PU_TX_BIT); } -static int mvebu_a3700_comphy_reset(struct phy *phy) +static void mvebu_a3700_comphy_usb3_power_off(struct mvebu_a3700_comphy_lane *lane) { - struct mvebu_a3700_comphy_lane *lane = phy_get_drvdata(phy); - u16 mask, data; - - dev_dbg(lane->dev, "resetting lane %d\n", lane->id); - - /* COMPHY reset for internal logic */ - comphy_lane_reg_set(lane, COMPHY_SFT_RESET, - SFT_RST_NO_REG, SFT_RST_NO_REG); - - /* COMPHY register reset (cleared automatically) */ - comphy_lane_reg_set(lane, COMPHY_SFT_RESET, SFT_RST, SFT_RST); - - /* PIPE soft and register reset */ - data = PIPE_SOFT_RESET | PIPE_REG_RESET; - mask = data; - comphy_lane_reg_set(lane, COMPHY_PIPE_RST_CLK_CTRL, data, mask); - - /* Release PIPE register reset */ - comphy_lane_reg_set(lane, COMPHY_PIPE_RST_CLK_CTRL, - 0x0, PIPE_REG_RESET); - - /* Reset SB configuration register (only for lanes 0 and 1) */ - if (lane->id == 0 || lane->id == 1) { - u32 mask, data; - - data = PIN_RESET_CORE_BIT | PIN_RESET_COMPHY_BIT | - PIN_PU_PLL_BIT | PIN_PU_RX_BIT | PIN_PU_TX_BIT; - mask = data | PIN_PU_IVREF_BIT | PIN_TX_IDLE_BIT; - comphy_periph_reg_set(lane, COMPHY_PHY_CFG1, data, mask); - } - - return 0; + /* + * The USB3 MAC sets the USB3 PHY to low state, so we do not + * need to power off USB3 PHY again. + */ } static bool mvebu_a3700_comphy_check_mode(int lane, @@ -1171,10 +1142,6 @@ static int mvebu_a3700_comphy_set_mode(struct phy *phy, enum phy_mode mode, (lane->mode != mode || lane->submode != submode)) return -EBUSY; - /* If changing mode, ensure reset is called */ - if (lane->mode != PHY_MODE_INVALID && lane->mode != mode) - lane->needs_reset = true; - /* Just remember the mode, ->power_on() will do the real setup */ lane->mode = mode; lane->submode = submode; @@ -1185,7 +1152,6 @@ static int mvebu_a3700_comphy_set_mode(struct phy *phy, enum phy_mode mode, static int mvebu_a3700_comphy_power_on(struct phy *phy) { struct mvebu_a3700_comphy_lane *lane = phy_get_drvdata(phy); - int ret; if (!mvebu_a3700_comphy_check_mode(lane->id, lane->mode, lane->submode)) { @@ -1193,14 +1159,6 @@ static int mvebu_a3700_comphy_power_on(struct phy *phy) return -EINVAL; } - if (lane->needs_reset) { - ret = mvebu_a3700_comphy_reset(phy); - if (ret) - return ret; - - lane->needs_reset = false; - } - switch (lane->mode) { case PHY_MODE_USB_HOST_SS: dev_dbg(lane->dev, "set lane %d to USB3 host mode\n", lane->id); @@ -1224,38 +1182,28 @@ static int mvebu_a3700_comphy_power_off(struct phy *phy) { struct mvebu_a3700_comphy_lane *lane = phy_get_drvdata(phy); - switch (lane->mode) { - case PHY_MODE_USB_HOST_SS: - /* - * The USB3 MAC sets the USB3 PHY to low state, so we do not - * need to power off USB3 PHY again. - */ - break; - - case PHY_MODE_SATA: - mvebu_a3700_comphy_sata_power_off(lane); - break; - - case PHY_MODE_ETHERNET: + switch (lane->id) { + case 0: + mvebu_a3700_comphy_usb3_power_off(lane); mvebu_a3700_comphy_ethernet_power_off(lane); - break; - - case PHY_MODE_PCIE: + return 0; + case 1: mvebu_a3700_comphy_pcie_power_off(lane); - break; - + mvebu_a3700_comphy_ethernet_power_off(lane); + return 0; + case 2: + mvebu_a3700_comphy_usb3_power_off(lane); + mvebu_a3700_comphy_sata_power_off(lane); + return 0; default: dev_err(lane->dev, "invalid COMPHY mode\n"); return -EINVAL; } - - return 0; } static const struct phy_ops mvebu_a3700_comphy_ops = { .power_on = mvebu_a3700_comphy_power_on, .power_off = mvebu_a3700_comphy_power_off, - .reset = mvebu_a3700_comphy_reset, .set_mode = mvebu_a3700_comphy_set_mode, .owner = THIS_MODULE, }; @@ -1393,8 +1341,7 @@ static int mvebu_a3700_comphy_probe(struct platform_device *pdev) * To avoid relying on the bootloader/firmware configuration, * power off all comphys. */ - mvebu_a3700_comphy_reset(phy); - lane->needs_reset = false; + mvebu_a3700_comphy_power_off(phy); } provider = devm_of_phy_provider_register(&pdev->dev, diff --git a/drivers/phy/mediatek/phy-mtk-dp.c b/drivers/phy/mediatek/phy-mtk-dp.c index 31266e7ca324479690218656f9fea541e1b49921..232fd3f1ff1b1a109e165a460914ad22b19a42cf 100644 --- a/drivers/phy/mediatek/phy-mtk-dp.c +++ b/drivers/phy/mediatek/phy-mtk-dp.c @@ -85,7 +85,7 @@ struct mtk_dp_phy { static int mtk_dp_phy_init(struct phy *phy) { struct mtk_dp_phy *dp_phy = phy_get_drvdata(phy); - u32 driving_params[] = { + static const u32 driving_params[] = { DRIVING_PARAM_3_DEFAULT, DRIVING_PARAM_4_DEFAULT, DRIVING_PARAM_5_DEFAULT, diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt2701.c b/drivers/phy/mediatek/phy-mtk-hdmi-mt2701.c index b74c65a1762c72d1a1f5f3f1b838c73f16210048..e51b2d13eab473dddace48c75c2a8d73c8c65635 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi-mt2701.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt2701.c @@ -5,83 +5,66 @@ */ #include "phy-mtk-hdmi.h" +#include "phy-mtk-io.h" #define HDMI_CON0 0x00 -#define RG_HDMITX_DRV_IBIAS 0 -#define RG_HDMITX_DRV_IBIAS_MASK (0x3f << 0) -#define RG_HDMITX_EN_SER 12 -#define RG_HDMITX_EN_SER_MASK (0x0f << 12) -#define RG_HDMITX_EN_SLDO 16 -#define RG_HDMITX_EN_SLDO_MASK (0x0f << 16) -#define RG_HDMITX_EN_PRED 20 -#define RG_HDMITX_EN_PRED_MASK (0x0f << 20) -#define RG_HDMITX_EN_IMP 24 -#define RG_HDMITX_EN_IMP_MASK (0x0f << 24) -#define RG_HDMITX_EN_DRV 28 -#define RG_HDMITX_EN_DRV_MASK (0x0f << 28) +#define RG_HDMITX_DRV_IBIAS_MASK GENMASK(5, 0) +#define RG_HDMITX_EN_SER_MASK GENMASK(15, 12) +#define RG_HDMITX_EN_SLDO_MASK GENMASK(19, 16) +#define RG_HDMITX_EN_PRED_MASK GENMASK(23, 20) +#define RG_HDMITX_EN_IMP_MASK GENMASK(27, 24) +#define RG_HDMITX_EN_DRV_MASK GENMASK(31, 28) #define HDMI_CON1 0x04 -#define RG_HDMITX_PRED_IBIAS 18 -#define RG_HDMITX_PRED_IBIAS_MASK (0x0f << 18) -#define RG_HDMITX_PRED_IMP (0x01 << 22) -#define RG_HDMITX_DRV_IMP 26 -#define RG_HDMITX_DRV_IMP_MASK (0x3f << 26) +#define RG_HDMITX_PRED_IBIAS_MASK GENMASK(21, 18) +#define RG_HDMITX_PRED_IMP BIT(22) +#define RG_HDMITX_DRV_IMP_MASK GENMASK(31, 26) #define HDMI_CON2 0x08 -#define RG_HDMITX_EN_TX_CKLDO (0x01 << 0) -#define RG_HDMITX_EN_TX_POSDIV (0x01 << 1) -#define RG_HDMITX_TX_POSDIV 3 -#define RG_HDMITX_TX_POSDIV_MASK (0x03 << 3) -#define RG_HDMITX_EN_MBIAS (0x01 << 6) -#define RG_HDMITX_MBIAS_LPF_EN (0x01 << 7) +#define RG_HDMITX_EN_TX_CKLDO BIT(0) +#define RG_HDMITX_EN_TX_POSDIV BIT(1) +#define RG_HDMITX_TX_POSDIV_MASK GENMASK(4, 3) +#define RG_HDMITX_EN_MBIAS BIT(6) +#define RG_HDMITX_MBIAS_LPF_EN BIT(7) #define HDMI_CON4 0x10 -#define RG_HDMITX_RESERVE_MASK (0xffffffff << 0) +#define RG_HDMITX_RESERVE_MASK GENMASK(31, 0) #define HDMI_CON6 0x18 -#define RG_HTPLL_BR 0 -#define RG_HTPLL_BR_MASK (0x03 << 0) -#define RG_HTPLL_BC 2 -#define RG_HTPLL_BC_MASK (0x03 << 2) -#define RG_HTPLL_BP 4 -#define RG_HTPLL_BP_MASK (0x0f << 4) -#define RG_HTPLL_IR 8 -#define RG_HTPLL_IR_MASK (0x0f << 8) -#define RG_HTPLL_IC 12 -#define RG_HTPLL_IC_MASK (0x0f << 12) -#define RG_HTPLL_POSDIV 16 -#define RG_HTPLL_POSDIV_MASK (0x03 << 16) -#define RG_HTPLL_PREDIV 18 -#define RG_HTPLL_PREDIV_MASK (0x03 << 18) -#define RG_HTPLL_FBKSEL 20 -#define RG_HTPLL_FBKSEL_MASK (0x03 << 20) -#define RG_HTPLL_RLH_EN (0x01 << 22) -#define RG_HTPLL_FBKDIV 24 -#define RG_HTPLL_FBKDIV_MASK (0x7f << 24) -#define RG_HTPLL_EN (0x01 << 31) +#define RG_HTPLL_BR_MASK GENMASK(1, 0) +#define RG_HTPLL_BC_MASK GENMASK(3, 2) +#define RG_HTPLL_BP_MASK GENMASK(7, 4) +#define RG_HTPLL_IR_MASK GENMASK(11, 8) +#define RG_HTPLL_IC_MASK GENMASK(15, 12) +#define RG_HTPLL_POSDIV_MASK GENMASK(17, 16) +#define RG_HTPLL_PREDIV_MASK GENMASK(19, 18) +#define RG_HTPLL_FBKSEL_MASK GENMASK(21, 20) +#define RG_HTPLL_RLH_EN BIT(22) +#define RG_HTPLL_FBKDIV_MASK GENMASK(30, 24) +#define RG_HTPLL_EN BIT(31) #define HDMI_CON7 0x1c -#define RG_HTPLL_AUTOK_EN (0x01 << 23) -#define RG_HTPLL_DIVEN 28 -#define RG_HTPLL_DIVEN_MASK (0x07 << 28) +#define RG_HTPLL_AUTOK_EN BIT(23) +#define RG_HTPLL_DIVEN_MASK GENMASK(30, 28) static int mtk_hdmi_pll_prepare(struct clk_hw *hw) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON7, RG_HTPLL_AUTOK_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_RLH_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_POSDIV_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_MBIAS); + mtk_phy_set_bits(base + HDMI_CON7, RG_HTPLL_AUTOK_EN); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_RLH_EN); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_POSDIV_MASK); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_EN_MBIAS); usleep_range(80, 100); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_EN); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); usleep_range(80, 100); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SER_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_PRED_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_DRV_MASK); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_SER_MASK); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_PRED_MASK); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_DRV_MASK); usleep_range(80, 100); return 0; } @@ -89,20 +72,21 @@ static int mtk_hdmi_pll_prepare(struct clk_hw *hw) static void mtk_hdmi_pll_unprepare(struct clk_hw *hw) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_DRV_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_PRED_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SER_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_DRV_MASK); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_PRED_MASK); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_SER_MASK); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); usleep_range(80, 100); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_EN); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_EN); usleep_range(80, 100); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_MBIAS); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_POSDIV_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_RLH_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON7, RG_HTPLL_AUTOK_EN); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_EN_MBIAS); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_POSDIV_MASK); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_RLH_EN); + mtk_phy_clear_bits(base + HDMI_CON7, RG_HTPLL_AUTOK_EN); usleep_range(80, 100); } @@ -116,6 +100,7 @@ static int mtk_hdmi_pll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; u32 pos_div; if (rate <= 64000000) @@ -125,37 +110,25 @@ static int mtk_hdmi_pll_set_rate(struct clk_hw *hw, unsigned long rate, else pos_div = 1; - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_PREDIV_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_POSDIV_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_TX_POSDIV); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (0x1 << RG_HTPLL_IC), - RG_HTPLL_IC_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (0x1 << RG_HTPLL_IR), - RG_HTPLL_IR_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON2, (pos_div << RG_HDMITX_TX_POSDIV), - RG_HDMITX_TX_POSDIV_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (1 << RG_HTPLL_FBKSEL), - RG_HTPLL_FBKSEL_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (19 << RG_HTPLL_FBKDIV), - RG_HTPLL_FBKDIV_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON7, (0x2 << RG_HTPLL_DIVEN), - RG_HTPLL_DIVEN_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (0xc << RG_HTPLL_BP), - RG_HTPLL_BP_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (0x2 << RG_HTPLL_BC), - RG_HTPLL_BC_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, (0x1 << RG_HTPLL_BR), - RG_HTPLL_BR_MASK); - - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PRED_IMP); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1, (0x3 << RG_HDMITX_PRED_IBIAS), - RG_HDMITX_PRED_IBIAS_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_IMP_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1, (0x28 << RG_HDMITX_DRV_IMP), - RG_HDMITX_DRV_IMP_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON4, 0x28, RG_HDMITX_RESERVE_MASK); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0, (0xa << RG_HDMITX_DRV_IBIAS), - RG_HDMITX_DRV_IBIAS_MASK); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_PREDIV_MASK); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_POSDIV_MASK); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_EN_TX_POSDIV); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_IC_MASK, 0x1); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_IR_MASK, 0x1); + mtk_phy_update_field(base + HDMI_CON2, RG_HDMITX_TX_POSDIV_MASK, pos_div); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_FBKSEL_MASK, 1); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_FBKDIV_MASK, 19); + mtk_phy_update_field(base + HDMI_CON7, RG_HTPLL_DIVEN_MASK, 0x2); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_BP_MASK, 0xc); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_BC_MASK, 0x2); + mtk_phy_update_field(base + HDMI_CON6, RG_HTPLL_BR_MASK, 0x1); + + mtk_phy_clear_bits(base + HDMI_CON1, RG_HDMITX_PRED_IMP); + mtk_phy_update_field(base + HDMI_CON1, RG_HDMITX_PRED_IBIAS_MASK, 0x3); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_IMP_MASK); + mtk_phy_update_field(base + HDMI_CON1, RG_HDMITX_DRV_IMP_MASK, 0x28); + mtk_phy_update_field(base + HDMI_CON4, RG_HDMITX_RESERVE_MASK, 0x28); + mtk_phy_update_field(base + HDMI_CON0, RG_HDMITX_DRV_IBIAS_MASK, 0xa); return 0; } @@ -164,9 +137,10 @@ static unsigned long mtk_hdmi_pll_recalc_rate(struct clk_hw *hw, { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); unsigned long out_rate, val; + u32 tmp; - val = (readl(hdmi_phy->regs + HDMI_CON6) - & RG_HTPLL_PREDIV_MASK) >> RG_HTPLL_PREDIV; + tmp = readl(hdmi_phy->regs + HDMI_CON6); + val = FIELD_GET(RG_HTPLL_PREDIV_MASK, tmp); switch (val) { case 0x00: out_rate = parent_rate; @@ -179,14 +153,14 @@ static unsigned long mtk_hdmi_pll_recalc_rate(struct clk_hw *hw, break; } - val = (readl(hdmi_phy->regs + HDMI_CON6) - & RG_HTPLL_FBKDIV_MASK) >> RG_HTPLL_FBKDIV; + val = FIELD_GET(RG_HTPLL_FBKDIV_MASK, tmp); out_rate *= (val + 1) * 2; - val = (readl(hdmi_phy->regs + HDMI_CON2) - & RG_HDMITX_TX_POSDIV_MASK); - out_rate >>= (val >> RG_HDMITX_TX_POSDIV); - if (readl(hdmi_phy->regs + HDMI_CON2) & RG_HDMITX_EN_TX_POSDIV) + tmp = readl(hdmi_phy->regs + HDMI_CON2); + val = FIELD_GET(RG_HDMITX_TX_POSDIV_MASK, tmp); + out_rate >>= val; + + if (tmp & RG_HDMITX_EN_TX_POSDIV) out_rate /= 5; return out_rate; @@ -202,37 +176,41 @@ static const struct clk_ops mtk_hdmi_phy_pll_ops = { static void mtk_hdmi_phy_enable_tmds(struct mtk_hdmi_phy *hdmi_phy) { - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON7, RG_HTPLL_AUTOK_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_RLH_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_POSDIV_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_MBIAS); + void __iomem *base = hdmi_phy->regs; + + mtk_phy_set_bits(base + HDMI_CON7, RG_HTPLL_AUTOK_EN); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_RLH_EN); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_POSDIV_MASK); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_EN_MBIAS); usleep_range(80, 100); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); + mtk_phy_set_bits(base + HDMI_CON6, RG_HTPLL_EN); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); usleep_range(80, 100); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SER_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_PRED_MASK); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_DRV_MASK); + mtk_phy_set_bits(base + HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_SER_MASK); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_PRED_MASK); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_EN_DRV_MASK); usleep_range(80, 100); } static void mtk_hdmi_phy_disable_tmds(struct mtk_hdmi_phy *hdmi_phy) { - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_DRV_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_PRED_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SER_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); + void __iomem *base = hdmi_phy->regs; + + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_DRV_MASK); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_PRED_MASK); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_SER_MASK); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_MBIAS_LPF_EN); usleep_range(80, 100); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_EN); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_EN_SLDO_MASK); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_EN_TX_CKLDO); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_EN); usleep_range(80, 100); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON2, RG_HDMITX_EN_MBIAS); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_POSDIV_MASK); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON6, RG_HTPLL_RLH_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON7, RG_HTPLL_AUTOK_EN); + mtk_phy_clear_bits(base + HDMI_CON2, RG_HDMITX_EN_MBIAS); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_POSDIV_MASK); + mtk_phy_clear_bits(base + HDMI_CON6, RG_HTPLL_RLH_EN); + mtk_phy_clear_bits(base + HDMI_CON7, RG_HTPLL_AUTOK_EN); usleep_range(80, 100); } diff --git a/drivers/phy/mediatek/phy-mtk-hdmi-mt8173.c b/drivers/phy/mediatek/phy-mtk-hdmi-mt8173.c index 6cdfdf5a698a4425fa95b2764753cc410e2f20ca..d04758396046363ab9edc584ae10bf31e9c5fb0f 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi-mt8173.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi-mt8173.c @@ -5,121 +5,99 @@ */ #include "phy-mtk-hdmi.h" +#include "phy-mtk-io.h" #define HDMI_CON0 0x00 #define RG_HDMITX_PLL_EN BIT(31) -#define RG_HDMITX_PLL_FBKDIV (0x7f << 24) -#define PLL_FBKDIV_SHIFT 24 -#define RG_HDMITX_PLL_FBKSEL (0x3 << 22) -#define PLL_FBKSEL_SHIFT 22 -#define RG_HDMITX_PLL_PREDIV (0x3 << 20) -#define PREDIV_SHIFT 20 -#define RG_HDMITX_PLL_POSDIV (0x3 << 18) -#define POSDIV_SHIFT 18 -#define RG_HDMITX_PLL_RST_DLY (0x3 << 16) -#define RG_HDMITX_PLL_IR (0xf << 12) -#define PLL_IR_SHIFT 12 -#define RG_HDMITX_PLL_IC (0xf << 8) -#define PLL_IC_SHIFT 8 -#define RG_HDMITX_PLL_BP (0xf << 4) -#define PLL_BP_SHIFT 4 -#define RG_HDMITX_PLL_BR (0x3 << 2) -#define PLL_BR_SHIFT 2 -#define RG_HDMITX_PLL_BC (0x3 << 0) -#define PLL_BC_SHIFT 0 +#define RG_HDMITX_PLL_FBKDIV GENMASK(30, 24) +#define RG_HDMITX_PLL_FBKSEL GENMASK(23, 22) +#define RG_HDMITX_PLL_PREDIV GENMASK(21, 20) +#define RG_HDMITX_PLL_POSDIV GENMASK(19, 18) +#define RG_HDMITX_PLL_RST_DLY GENMASK(17, 16) +#define RG_HDMITX_PLL_IR GENMASK(15, 12) +#define RG_HDMITX_PLL_IC GENMASK(11, 8) +#define RG_HDMITX_PLL_BP GENMASK(7, 4) +#define RG_HDMITX_PLL_BR GENMASK(3, 2) +#define RG_HDMITX_PLL_BC GENMASK(1, 0) #define HDMI_CON1 0x04 -#define RG_HDMITX_PLL_DIVEN (0x7 << 29) -#define PLL_DIVEN_SHIFT 29 +#define RG_HDMITX_PLL_DIVEN GENMASK(31, 29) #define RG_HDMITX_PLL_AUTOK_EN BIT(28) -#define RG_HDMITX_PLL_AUTOK_KF (0x3 << 26) -#define RG_HDMITX_PLL_AUTOK_KS (0x3 << 24) +#define RG_HDMITX_PLL_AUTOK_KF GENMASK(27, 26) +#define RG_HDMITX_PLL_AUTOK_KS GENMASK(25, 24) #define RG_HDMITX_PLL_AUTOK_LOAD BIT(23) -#define RG_HDMITX_PLL_BAND (0x3f << 16) +#define RG_HDMITX_PLL_BAND GENMASK(21, 16) #define RG_HDMITX_PLL_REF_SEL BIT(15) #define RG_HDMITX_PLL_BIAS_EN BIT(14) #define RG_HDMITX_PLL_BIAS_LPF_EN BIT(13) #define RG_HDMITX_PLL_TXDIV_EN BIT(12) -#define RG_HDMITX_PLL_TXDIV (0x3 << 10) -#define PLL_TXDIV_SHIFT 10 +#define RG_HDMITX_PLL_TXDIV GENMASK(11, 10) #define RG_HDMITX_PLL_LVROD_EN BIT(9) #define RG_HDMITX_PLL_MONVC_EN BIT(8) #define RG_HDMITX_PLL_MONCK_EN BIT(7) #define RG_HDMITX_PLL_MONREF_EN BIT(6) #define RG_HDMITX_PLL_TST_EN BIT(5) #define RG_HDMITX_PLL_TST_CK_EN BIT(4) -#define RG_HDMITX_PLL_TST_SEL (0xf << 0) +#define RG_HDMITX_PLL_TST_SEL GENMASK(3, 0) #define HDMI_CON2 0x08 -#define RGS_HDMITX_PLL_AUTOK_BAND (0x7f << 8) +#define RGS_HDMITX_PLL_AUTOK_BAND GENMASK(14, 8) #define RGS_HDMITX_PLL_AUTOK_FAIL BIT(1) #define RG_HDMITX_EN_TX_CKLDO BIT(0) #define HDMI_CON3 0x0c -#define RG_HDMITX_SER_EN (0xf << 28) -#define RG_HDMITX_PRD_EN (0xf << 24) -#define RG_HDMITX_PRD_IMP_EN (0xf << 20) -#define RG_HDMITX_DRV_EN (0xf << 16) -#define RG_HDMITX_DRV_IMP_EN (0xf << 12) -#define DRV_IMP_EN_SHIFT 12 +#define RG_HDMITX_SER_EN GENMASK(31, 28) +#define RG_HDMITX_PRD_EN GENMASK(27, 24) +#define RG_HDMITX_PRD_IMP_EN GENMASK(23, 20) +#define RG_HDMITX_DRV_EN GENMASK(19, 16) +#define RG_HDMITX_DRV_IMP_EN GENMASK(15, 12) #define RG_HDMITX_MHLCK_FORCE BIT(10) #define RG_HDMITX_MHLCK_PPIX_EN BIT(9) #define RG_HDMITX_MHLCK_EN BIT(8) -#define RG_HDMITX_SER_DIN_SEL (0xf << 4) +#define RG_HDMITX_SER_DIN_SEL GENMASK(7, 4) #define RG_HDMITX_SER_5T1_BIST_EN BIT(3) #define RG_HDMITX_SER_BIST_TOG BIT(2) #define RG_HDMITX_SER_DIN_TOG BIT(1) #define RG_HDMITX_SER_CLKDIG_INV BIT(0) #define HDMI_CON4 0x10 -#define RG_HDMITX_PRD_IBIAS_CLK (0xf << 24) -#define RG_HDMITX_PRD_IBIAS_D2 (0xf << 16) -#define RG_HDMITX_PRD_IBIAS_D1 (0xf << 8) -#define RG_HDMITX_PRD_IBIAS_D0 (0xf << 0) -#define PRD_IBIAS_CLK_SHIFT 24 -#define PRD_IBIAS_D2_SHIFT 16 -#define PRD_IBIAS_D1_SHIFT 8 -#define PRD_IBIAS_D0_SHIFT 0 +#define RG_HDMITX_PRD_IBIAS_CLK GENMASK(27, 24) +#define RG_HDMITX_PRD_IBIAS_D2 GENMASK(19, 16) +#define RG_HDMITX_PRD_IBIAS_D1 GENMASK(11, 8) +#define RG_HDMITX_PRD_IBIAS_D0 GENMASK(3, 0) #define HDMI_CON5 0x14 -#define RG_HDMITX_DRV_IBIAS_CLK (0x3f << 24) -#define RG_HDMITX_DRV_IBIAS_D2 (0x3f << 16) -#define RG_HDMITX_DRV_IBIAS_D1 (0x3f << 8) -#define RG_HDMITX_DRV_IBIAS_D0 (0x3f << 0) -#define DRV_IBIAS_CLK_SHIFT 24 -#define DRV_IBIAS_D2_SHIFT 16 -#define DRV_IBIAS_D1_SHIFT 8 -#define DRV_IBIAS_D0_SHIFT 0 +#define RG_HDMITX_DRV_IBIAS_CLK GENMASK(29, 24) +#define RG_HDMITX_DRV_IBIAS_D2 GENMASK(21, 16) +#define RG_HDMITX_DRV_IBIAS_D1 GENMASK(13, 8) +#define RG_HDMITX_DRV_IBIAS_D0 GENMASK(5, 0) #define HDMI_CON6 0x18 -#define RG_HDMITX_DRV_IMP_CLK (0x3f << 24) -#define RG_HDMITX_DRV_IMP_D2 (0x3f << 16) -#define RG_HDMITX_DRV_IMP_D1 (0x3f << 8) -#define RG_HDMITX_DRV_IMP_D0 (0x3f << 0) -#define DRV_IMP_CLK_SHIFT 24 -#define DRV_IMP_D2_SHIFT 16 -#define DRV_IMP_D1_SHIFT 8 -#define DRV_IMP_D0_SHIFT 0 +#define RG_HDMITX_DRV_IMP_CLK GENMASK(29, 24) +#define RG_HDMITX_DRV_IMP_D2 GENMASK(21, 16) +#define RG_HDMITX_DRV_IMP_D1 GENMASK(13, 8) +#define RG_HDMITX_DRV_IMP_D0 GENMASK(5, 0) #define HDMI_CON7 0x1c -#define RG_HDMITX_MHLCK_DRV_IBIAS (0x1f << 27) -#define RG_HDMITX_SER_DIN (0x3ff << 16) -#define RG_HDMITX_CHLDC_TST (0xf << 12) -#define RG_HDMITX_CHLCK_TST (0xf << 8) -#define RG_HDMITX_RESERVE (0xff << 0) +#define RG_HDMITX_MHLCK_DRV_IBIAS GENMASK(31, 27) +#define RG_HDMITX_SER_DIN GENMASK(25, 16) +#define RG_HDMITX_CHLDC_TST GENMASK(15, 12) +#define RG_HDMITX_CHLCK_TST GENMASK(11, 8) +#define RG_HDMITX_RESERVE GENMASK(7, 0) #define HDMI_CON8 0x20 -#define RGS_HDMITX_2T1_LEV (0xf << 16) -#define RGS_HDMITX_2T1_EDG (0xf << 12) -#define RGS_HDMITX_5T1_LEV (0xf << 8) -#define RGS_HDMITX_5T1_EDG (0xf << 4) +#define RGS_HDMITX_2T1_LEV GENMASK(19, 16) +#define RGS_HDMITX_2T1_EDG GENMASK(15, 12) +#define RGS_HDMITX_5T1_LEV GENMASK(11, 8) +#define RGS_HDMITX_5T1_EDG GENMASK(7, 4) #define RGS_HDMITX_PLUG_TST BIT(0) static int mtk_hdmi_pll_prepare(struct clk_hw *hw) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3, RG_HDMITX_MHLCK_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_EN); + mtk_phy_set_bits(base + HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_PLL_POSDIV); + mtk_phy_clear_bits(base + HDMI_CON3, RG_HDMITX_MHLCK_EN); + mtk_phy_set_bits(base + HDMI_CON1, RG_HDMITX_PLL_BIAS_EN); usleep_range(100, 150); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_EN); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_PLL_EN); usleep_range(100, 150); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN); + mtk_phy_set_bits(base + HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN); + mtk_phy_set_bits(base + HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN); return 0; } @@ -127,15 +105,16 @@ static int mtk_hdmi_pll_prepare(struct clk_hw *hw) static void mtk_hdmi_pll_unprepare(struct clk_hw *hw) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN); + mtk_phy_clear_bits(base + HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN); + mtk_phy_clear_bits(base + HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN); usleep_range(100, 150); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_EN); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_PLL_EN); usleep_range(100, 150); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_EN); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV); - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN); + mtk_phy_clear_bits(base + HDMI_CON1, RG_HDMITX_PLL_BIAS_EN); + mtk_phy_clear_bits(base + HDMI_CON0, RG_HDMITX_PLL_POSDIV); + mtk_phy_clear_bits(base + HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN); usleep_range(100, 150); } @@ -157,6 +136,7 @@ static int mtk_hdmi_pll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw); + void __iomem *base = hdmi_phy->regs; unsigned int pre_div; unsigned int div; unsigned int pre_ibias; @@ -177,65 +157,57 @@ static int mtk_hdmi_pll_set_rate(struct clk_hw *hw, unsigned long rate, div = 1; } - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0, - (pre_div << PREDIV_SHIFT), RG_HDMITX_PLL_PREDIV); - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0, - (0x1 << PLL_IC_SHIFT) | (0x1 << PLL_IR_SHIFT), - RG_HDMITX_PLL_IC | RG_HDMITX_PLL_IR); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1, - (div << PLL_TXDIV_SHIFT), RG_HDMITX_PLL_TXDIV); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0, - (0x1 << PLL_FBKSEL_SHIFT) | (19 << PLL_FBKDIV_SHIFT), - RG_HDMITX_PLL_FBKSEL | RG_HDMITX_PLL_FBKDIV); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1, - (0x2 << PLL_DIVEN_SHIFT), RG_HDMITX_PLL_DIVEN); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0, - (0xc << PLL_BP_SHIFT) | (0x2 << PLL_BC_SHIFT) | - (0x1 << PLL_BR_SHIFT), - RG_HDMITX_PLL_BP | RG_HDMITX_PLL_BC | - RG_HDMITX_PLL_BR); + mtk_phy_update_field(base + HDMI_CON0, RG_HDMITX_PLL_PREDIV, pre_div); + mtk_phy_set_bits(base + HDMI_CON0, RG_HDMITX_PLL_POSDIV); + mtk_phy_update_bits(base + HDMI_CON0, + RG_HDMITX_PLL_IC | RG_HDMITX_PLL_IR, + FIELD_PREP(RG_HDMITX_PLL_IC, 0x1) | + FIELD_PREP(RG_HDMITX_PLL_IR, 0x1)); + mtk_phy_update_field(base + HDMI_CON1, RG_HDMITX_PLL_TXDIV, div); + mtk_phy_update_bits(base + HDMI_CON0, + RG_HDMITX_PLL_FBKSEL | RG_HDMITX_PLL_FBKDIV, + FIELD_PREP(RG_HDMITX_PLL_FBKSEL, 0x1) | + FIELD_PREP(RG_HDMITX_PLL_FBKDIV, 19)); + mtk_phy_update_field(base + HDMI_CON1, RG_HDMITX_PLL_DIVEN, 0x2); + mtk_phy_update_bits(base + HDMI_CON0, + RG_HDMITX_PLL_BP | RG_HDMITX_PLL_BC | + RG_HDMITX_PLL_BR, + FIELD_PREP(RG_HDMITX_PLL_BP, 0xc) | + FIELD_PREP(RG_HDMITX_PLL_BC, 0x2) | + FIELD_PREP(RG_HDMITX_PLL_BR, 0x1)); if (rate < 165000000) { - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3, - RG_HDMITX_PRD_IMP_EN); + mtk_phy_clear_bits(base + HDMI_CON3, RG_HDMITX_PRD_IMP_EN); pre_ibias = 0x3; imp_en = 0x0; hdmi_ibias = hdmi_phy->ibias; } else { - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON3, - RG_HDMITX_PRD_IMP_EN); + mtk_phy_set_bits(base + HDMI_CON3, RG_HDMITX_PRD_IMP_EN); pre_ibias = 0x6; imp_en = 0xf; hdmi_ibias = hdmi_phy->ibias_up; } - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON4, - (pre_ibias << PRD_IBIAS_CLK_SHIFT) | - (pre_ibias << PRD_IBIAS_D2_SHIFT) | - (pre_ibias << PRD_IBIAS_D1_SHIFT) | - (pre_ibias << PRD_IBIAS_D0_SHIFT), - RG_HDMITX_PRD_IBIAS_CLK | - RG_HDMITX_PRD_IBIAS_D2 | - RG_HDMITX_PRD_IBIAS_D1 | - RG_HDMITX_PRD_IBIAS_D0); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON3, - (imp_en << DRV_IMP_EN_SHIFT), - RG_HDMITX_DRV_IMP_EN); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6, - (hdmi_phy->drv_imp_clk << DRV_IMP_CLK_SHIFT) | - (hdmi_phy->drv_imp_d2 << DRV_IMP_D2_SHIFT) | - (hdmi_phy->drv_imp_d1 << DRV_IMP_D1_SHIFT) | - (hdmi_phy->drv_imp_d0 << DRV_IMP_D0_SHIFT), - RG_HDMITX_DRV_IMP_CLK | RG_HDMITX_DRV_IMP_D2 | - RG_HDMITX_DRV_IMP_D1 | RG_HDMITX_DRV_IMP_D0); - mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON5, - (hdmi_ibias << DRV_IBIAS_CLK_SHIFT) | - (hdmi_ibias << DRV_IBIAS_D2_SHIFT) | - (hdmi_ibias << DRV_IBIAS_D1_SHIFT) | - (hdmi_ibias << DRV_IBIAS_D0_SHIFT), - RG_HDMITX_DRV_IBIAS_CLK | - RG_HDMITX_DRV_IBIAS_D2 | - RG_HDMITX_DRV_IBIAS_D1 | - RG_HDMITX_DRV_IBIAS_D0); + mtk_phy_update_bits(base + HDMI_CON4, + RG_HDMITX_PRD_IBIAS_CLK | RG_HDMITX_PRD_IBIAS_D2 | + RG_HDMITX_PRD_IBIAS_D1 | RG_HDMITX_PRD_IBIAS_D0, + FIELD_PREP(RG_HDMITX_PRD_IBIAS_CLK, pre_ibias) | + FIELD_PREP(RG_HDMITX_PRD_IBIAS_D2, pre_ibias) | + FIELD_PREP(RG_HDMITX_PRD_IBIAS_D1, pre_ibias) | + FIELD_PREP(RG_HDMITX_PRD_IBIAS_D0, pre_ibias)); + mtk_phy_update_field(base + HDMI_CON3, RG_HDMITX_DRV_IMP_EN, imp_en); + mtk_phy_update_bits(base + HDMI_CON6, + RG_HDMITX_DRV_IMP_CLK | RG_HDMITX_DRV_IMP_D2 | + RG_HDMITX_DRV_IMP_D1 | RG_HDMITX_DRV_IMP_D0, + FIELD_PREP(RG_HDMITX_DRV_IMP_CLK, hdmi_phy->drv_imp_clk) | + FIELD_PREP(RG_HDMITX_DRV_IMP_D2, hdmi_phy->drv_imp_d2) | + FIELD_PREP(RG_HDMITX_DRV_IMP_D1, hdmi_phy->drv_imp_d1) | + FIELD_PREP(RG_HDMITX_DRV_IMP_D0, hdmi_phy->drv_imp_d0)); + mtk_phy_update_bits(base + HDMI_CON5, + RG_HDMITX_DRV_IBIAS_CLK | RG_HDMITX_DRV_IBIAS_D2 | + RG_HDMITX_DRV_IBIAS_D1 | RG_HDMITX_DRV_IBIAS_D0, + FIELD_PREP(RG_HDMITX_DRV_IBIAS_CLK, hdmi_ibias) | + FIELD_PREP(RG_HDMITX_DRV_IBIAS_D2, hdmi_ibias) | + FIELD_PREP(RG_HDMITX_DRV_IBIAS_D1, hdmi_ibias) | + FIELD_PREP(RG_HDMITX_DRV_IBIAS_D0, hdmi_ibias)); return 0; } @@ -257,17 +229,17 @@ static const struct clk_ops mtk_hdmi_phy_pll_ops = { static void mtk_hdmi_phy_enable_tmds(struct mtk_hdmi_phy *hdmi_phy) { - mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON3, - RG_HDMITX_SER_EN | RG_HDMITX_PRD_EN | - RG_HDMITX_DRV_EN); + mtk_phy_set_bits(hdmi_phy->regs + HDMI_CON3, + RG_HDMITX_SER_EN | RG_HDMITX_PRD_EN | + RG_HDMITX_DRV_EN); usleep_range(100, 150); } static void mtk_hdmi_phy_disable_tmds(struct mtk_hdmi_phy *hdmi_phy) { - mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3, - RG_HDMITX_DRV_EN | RG_HDMITX_PRD_EN | - RG_HDMITX_SER_EN); + mtk_phy_clear_bits(hdmi_phy->regs + HDMI_CON3, + RG_HDMITX_DRV_EN | RG_HDMITX_PRD_EN | + RG_HDMITX_SER_EN); } struct mtk_hdmi_phy_conf mtk_hdmi_phy_8173_conf = { diff --git a/drivers/phy/mediatek/phy-mtk-hdmi.c b/drivers/phy/mediatek/phy-mtk-hdmi.c index d4bd419abc3c4d5d16f1520d0de3e0e13c89ec57..b16d437d67219c322414557e26e9eeea57793cbd 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi.c +++ b/drivers/phy/mediatek/phy-mtk-hdmi.c @@ -15,39 +15,6 @@ static const struct phy_ops mtk_hdmi_phy_dev_ops = { .owner = THIS_MODULE, }; -void mtk_hdmi_phy_clear_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 bits) -{ - void __iomem *reg = hdmi_phy->regs + offset; - u32 tmp; - - tmp = readl(reg); - tmp &= ~bits; - writel(tmp, reg); -} - -void mtk_hdmi_phy_set_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 bits) -{ - void __iomem *reg = hdmi_phy->regs + offset; - u32 tmp; - - tmp = readl(reg); - tmp |= bits; - writel(tmp, reg); -} - -void mtk_hdmi_phy_mask(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 val, u32 mask) -{ - void __iomem *reg = hdmi_phy->regs + offset; - u32 tmp; - - tmp = readl(reg); - tmp = (tmp & ~mask) | (val & mask); - writel(tmp, reg); -} - inline struct mtk_hdmi_phy *to_mtk_hdmi_phy(struct clk_hw *hw) { return container_of(hw, struct mtk_hdmi_phy, pll_hw); diff --git a/drivers/phy/mediatek/phy-mtk-hdmi.h b/drivers/phy/mediatek/phy-mtk-hdmi.h index dcf9bb13699b3ba0df3e62d3b7eb1f5a69df979d..c7fa65cff989151507c613d3789f351eee47f578 100644 --- a/drivers/phy/mediatek/phy-mtk-hdmi.h +++ b/drivers/phy/mediatek/phy-mtk-hdmi.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -42,12 +41,6 @@ struct mtk_hdmi_phy { unsigned int ibias_up; }; -void mtk_hdmi_phy_clear_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 bits); -void mtk_hdmi_phy_set_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 bits); -void mtk_hdmi_phy_mask(struct mtk_hdmi_phy *hdmi_phy, u32 offset, - u32 val, u32 mask); struct mtk_hdmi_phy *to_mtk_hdmi_phy(struct clk_hw *hw); extern struct mtk_hdmi_phy_conf mtk_hdmi_phy_8173_conf; diff --git a/drivers/phy/mediatek/phy-mtk-io.h b/drivers/phy/mediatek/phy-mtk-io.h index 500fcdab165d2b708ce6bef3bc415d96a95ba8e1..d20ad5e5be8141fc534b35334b2c34c39cbf74d6 100644 --- a/drivers/phy/mediatek/phy-mtk-io.h +++ b/drivers/phy/mediatek/phy-mtk-io.h @@ -8,6 +8,7 @@ #ifndef __PHY_MTK_H__ #define __PHY_MTK_H__ +#include #include static inline void mtk_phy_clear_bits(void __iomem *reg, u32 bits) @@ -35,4 +36,11 @@ static inline void mtk_phy_update_bits(void __iomem *reg, u32 mask, u32 val) writel(tmp, reg); } +/* field @mask shall be constant and continuous */ +#define mtk_phy_update_field(reg, mask, val) \ +({ \ + typeof(mask) mask_ = (mask); \ + mtk_phy_update_bits(reg, mask_, FIELD_PREP(mask_, val)); \ +}) + #endif diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8173.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8173.c index 7a847954594f188bf763d91d544a37ef69736ebf..673cb0f08959e0e4f2b1ade2ebaf796e8b76f8bc 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8173.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8173.c @@ -4,14 +4,15 @@ * Author: jitao.shi */ +#include "phy-mtk-io.h" #include "phy-mtk-mipi-dsi.h" #define MIPITX_DSI_CON 0x00 #define RG_DSI_LDOCORE_EN BIT(0) #define RG_DSI_CKG_LDOOUT_EN BIT(1) -#define RG_DSI_BCLK_SEL (3 << 2) -#define RG_DSI_LD_IDX_SEL (7 << 4) -#define RG_DSI_PHYCLK_SEL (2 << 8) +#define RG_DSI_BCLK_SEL GENMASK(3, 2) +#define RG_DSI_LD_IDX_SEL GENMASK(6, 4) +#define RG_DSI_PHYCLK_SEL GENMASK(9, 8) #define RG_DSI_DSICLK_FREQ_SEL BIT(10) #define RG_DSI_LPTX_CLMP_EN BIT(11) @@ -27,41 +28,46 @@ #define RG_DSI_LNTx_LPTX_IMINUS BIT(4) #define RG_DSI_LNTx_LPCD_IPLUS BIT(5) #define RG_DSI_LNTx_LPCD_IMINUS BIT(6) -#define RG_DSI_LNTx_RT_CODE (0xf << 8) +#define RG_DSI_LNTx_RT_CODE GENMASK(11, 8) #define MIPITX_DSI_TOP_CON 0x40 #define RG_DSI_LNT_INTR_EN BIT(0) #define RG_DSI_LNT_HS_BIAS_EN BIT(1) #define RG_DSI_LNT_IMP_CAL_EN BIT(2) #define RG_DSI_LNT_TESTMODE_EN BIT(3) -#define RG_DSI_LNT_IMP_CAL_CODE (0xf << 4) -#define RG_DSI_LNT_AIO_SEL (7 << 8) +#define RG_DSI_LNT_IMP_CAL_CODE GENMASK(7, 4) +#define RG_DSI_LNT_AIO_SEL GENMASK(10, 8) #define RG_DSI_PAD_TIE_LOW_EN BIT(11) #define RG_DSI_DEBUG_INPUT_EN BIT(12) -#define RG_DSI_PRESERVE (7 << 13) +#define RG_DSI_PRESERVE GENMASK(15, 13) #define MIPITX_DSI_BG_CON 0x44 #define RG_DSI_BG_CORE_EN BIT(0) #define RG_DSI_BG_CKEN BIT(1) -#define RG_DSI_BG_DIV (0x3 << 2) +#define RG_DSI_BG_DIV GENMASK(3, 2) #define RG_DSI_BG_FAST_CHARGE BIT(4) -#define RG_DSI_VOUT_MSK (0x3ffff << 5) -#define RG_DSI_V12_SEL (7 << 5) -#define RG_DSI_V10_SEL (7 << 8) -#define RG_DSI_V072_SEL (7 << 11) -#define RG_DSI_V04_SEL (7 << 14) -#define RG_DSI_V032_SEL (7 << 17) -#define RG_DSI_V02_SEL (7 << 20) -#define RG_DSI_BG_R1_TRIM (0xf << 24) -#define RG_DSI_BG_R2_TRIM (0xf << 28) + +#define RG_DSI_V12_SEL GENMASK(7, 5) +#define RG_DSI_V10_SEL GENMASK(10, 8) +#define RG_DSI_V072_SEL GENMASK(13, 11) +#define RG_DSI_V04_SEL GENMASK(16, 14) +#define RG_DSI_V032_SEL GENMASK(19, 17) +#define RG_DSI_V02_SEL GENMASK(22, 20) +#define RG_DSI_VOUT_MSK \ + (RG_DSI_V12_SEL | RG_DSI_V10_SEL | RG_DSI_V072_SEL | \ + RG_DSI_V04_SEL | RG_DSI_V032_SEL | RG_DSI_V02_SEL) +#define RG_DSI_BG_R1_TRIM GENMASK(27, 24) +#define RG_DSI_BG_R2_TRIM GENMASK(31, 28) #define MIPITX_DSI_PLL_CON0 0x50 #define RG_DSI_MPPLL_PLL_EN BIT(0) -#define RG_DSI_MPPLL_DIV_MSK (0x1ff << 1) -#define RG_DSI_MPPLL_PREDIV (3 << 1) -#define RG_DSI_MPPLL_TXDIV0 (3 << 3) -#define RG_DSI_MPPLL_TXDIV1 (3 << 5) -#define RG_DSI_MPPLL_POSDIV (7 << 7) +#define RG_DSI_MPPLL_PREDIV GENMASK(2, 1) +#define RG_DSI_MPPLL_TXDIV0 GENMASK(4, 3) +#define RG_DSI_MPPLL_TXDIV1 GENMASK(6, 5) +#define RG_DSI_MPPLL_POSDIV GENMASK(9, 7) +#define RG_DSI_MPPLL_DIV_MSK \ + (RG_DSI_MPPLL_PREDIV | RG_DSI_MPPLL_TXDIV0 | \ + RG_DSI_MPPLL_TXDIV1 | RG_DSI_MPPLL_POSDIV) #define RG_DSI_MPPLL_MONVC_EN BIT(10) #define RG_DSI_MPPLL_MONREF_EN BIT(11) #define RG_DSI_MPPLL_VOD_EN BIT(12) @@ -70,12 +76,12 @@ #define RG_DSI_MPPLL_SDM_FRA_EN BIT(0) #define RG_DSI_MPPLL_SDM_SSC_PH_INIT BIT(1) #define RG_DSI_MPPLL_SDM_SSC_EN BIT(2) -#define RG_DSI_MPPLL_SDM_SSC_PRD (0xffff << 16) +#define RG_DSI_MPPLL_SDM_SSC_PRD GENMASK(31, 16) #define MIPITX_DSI_PLL_CON2 0x58 #define MIPITX_DSI_PLL_TOP 0x64 -#define RG_DSI_MPPLL_PRESERVE (0xff << 8) +#define RG_DSI_MPPLL_PRESERVE GENMASK(15, 8) #define MIPITX_DSI_PLL_PWR 0x68 #define RG_DSI_MPPLL_SDM_PWR_ON BIT(0) @@ -116,6 +122,7 @@ static int mtk_mipi_tx_pll_prepare(struct clk_hw *hw) { struct mtk_mipi_tx *mipi_tx = mtk_mipi_tx_from_clk_hw(hw); + void __iomem *base = mipi_tx->regs; u8 txdiv, txdiv0, txdiv1; u64 pcw; @@ -145,34 +152,38 @@ static int mtk_mipi_tx_pll_prepare(struct clk_hw *hw) return -EINVAL; } - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_BG_CON, - RG_DSI_VOUT_MSK | - RG_DSI_BG_CKEN | RG_DSI_BG_CORE_EN, - (4 << 20) | (4 << 17) | (4 << 14) | - (4 << 11) | (4 << 8) | (4 << 5) | - RG_DSI_BG_CKEN | RG_DSI_BG_CORE_EN); + mtk_phy_update_bits(base + MIPITX_DSI_BG_CON, + RG_DSI_VOUT_MSK | RG_DSI_BG_CKEN | + RG_DSI_BG_CORE_EN, + FIELD_PREP(RG_DSI_V02_SEL, 4) | + FIELD_PREP(RG_DSI_V032_SEL, 4) | + FIELD_PREP(RG_DSI_V04_SEL, 4) | + FIELD_PREP(RG_DSI_V072_SEL, 4) | + FIELD_PREP(RG_DSI_V10_SEL, 4) | + FIELD_PREP(RG_DSI_V12_SEL, 4) | + RG_DSI_BG_CKEN | RG_DSI_BG_CORE_EN); usleep_range(30, 100); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_TOP_CON, - RG_DSI_LNT_IMP_CAL_CODE | RG_DSI_LNT_HS_BIAS_EN, - (8 << 4) | RG_DSI_LNT_HS_BIAS_EN); + mtk_phy_update_bits(base + MIPITX_DSI_TOP_CON, + RG_DSI_LNT_IMP_CAL_CODE | RG_DSI_LNT_HS_BIAS_EN, + FIELD_PREP(RG_DSI_LNT_IMP_CAL_CODE, 8) | + RG_DSI_LNT_HS_BIAS_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_DSI_CON, - RG_DSI_CKG_LDOOUT_EN | RG_DSI_LDOCORE_EN); + mtk_phy_set_bits(base + MIPITX_DSI_CON, + RG_DSI_CKG_LDOOUT_EN | RG_DSI_LDOCORE_EN); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_PLL_PWR, - RG_DSI_MPPLL_SDM_PWR_ON | - RG_DSI_MPPLL_SDM_ISO_EN, - RG_DSI_MPPLL_SDM_PWR_ON); + mtk_phy_update_bits(base + MIPITX_DSI_PLL_PWR, + RG_DSI_MPPLL_SDM_PWR_ON | RG_DSI_MPPLL_SDM_ISO_EN, + RG_DSI_MPPLL_SDM_PWR_ON); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_PLL_CON0, - RG_DSI_MPPLL_PLL_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_PLL_CON0, RG_DSI_MPPLL_PLL_EN); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_PLL_CON0, - RG_DSI_MPPLL_TXDIV0 | RG_DSI_MPPLL_TXDIV1 | - RG_DSI_MPPLL_PREDIV, - (txdiv0 << 3) | (txdiv1 << 5)); + mtk_phy_update_bits(base + MIPITX_DSI_PLL_CON0, + RG_DSI_MPPLL_TXDIV0 | RG_DSI_MPPLL_TXDIV1 | + RG_DSI_MPPLL_PREDIV, + FIELD_PREP(RG_DSI_MPPLL_TXDIV0, txdiv0) | + FIELD_PREP(RG_DSI_MPPLL_TXDIV1, txdiv1)); /* * PLL PCW config @@ -182,23 +193,20 @@ static int mtk_mipi_tx_pll_prepare(struct clk_hw *hw) * Post DIV =4, so need data_Rate*4 * Ref_clk is 26MHz */ - pcw = div_u64(((u64)mipi_tx->data_rate * 2 * txdiv) << 24, - 26000000); - writel(pcw, mipi_tx->regs + MIPITX_DSI_PLL_CON2); + pcw = div_u64(((u64)mipi_tx->data_rate * 2 * txdiv) << 24, 26000000); + writel(pcw, base + MIPITX_DSI_PLL_CON2); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_DSI_PLL_CON1, - RG_DSI_MPPLL_SDM_FRA_EN); + mtk_phy_set_bits(base + MIPITX_DSI_PLL_CON1, RG_DSI_MPPLL_SDM_FRA_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_DSI_PLL_CON0, RG_DSI_MPPLL_PLL_EN); + mtk_phy_set_bits(base + MIPITX_DSI_PLL_CON0, RG_DSI_MPPLL_PLL_EN); usleep_range(20, 100); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_PLL_CON1, - RG_DSI_MPPLL_SDM_SSC_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_PLL_CON1, RG_DSI_MPPLL_SDM_SSC_EN); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_PLL_TOP, - RG_DSI_MPPLL_PRESERVE, - mipi_tx->driver_data->mppll_preserve); + mtk_phy_update_field(base + MIPITX_DSI_PLL_TOP, + RG_DSI_MPPLL_PRESERVE, + mipi_tx->driver_data->mppll_preserve); return 0; } @@ -206,31 +214,27 @@ static int mtk_mipi_tx_pll_prepare(struct clk_hw *hw) static void mtk_mipi_tx_pll_unprepare(struct clk_hw *hw) { struct mtk_mipi_tx *mipi_tx = mtk_mipi_tx_from_clk_hw(hw); + void __iomem *base = mipi_tx->regs; dev_dbg(mipi_tx->dev, "unprepare\n"); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_PLL_CON0, - RG_DSI_MPPLL_PLL_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_PLL_CON0, RG_DSI_MPPLL_PLL_EN); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_PLL_TOP, - RG_DSI_MPPLL_PRESERVE, 0); + mtk_phy_clear_bits(base + MIPITX_DSI_PLL_TOP, RG_DSI_MPPLL_PRESERVE); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_DSI_PLL_PWR, - RG_DSI_MPPLL_SDM_ISO_EN | - RG_DSI_MPPLL_SDM_PWR_ON, - RG_DSI_MPPLL_SDM_ISO_EN); + mtk_phy_update_bits(base + MIPITX_DSI_PLL_PWR, + RG_DSI_MPPLL_SDM_ISO_EN | RG_DSI_MPPLL_SDM_PWR_ON, + RG_DSI_MPPLL_SDM_ISO_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_TOP_CON, - RG_DSI_LNT_HS_BIAS_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_TOP_CON, RG_DSI_LNT_HS_BIAS_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_CON, - RG_DSI_CKG_LDOOUT_EN | RG_DSI_LDOCORE_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_CON, + RG_DSI_CKG_LDOOUT_EN | RG_DSI_LDOCORE_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_BG_CON, - RG_DSI_BG_CKEN | RG_DSI_BG_CORE_EN); + mtk_phy_clear_bits(base + MIPITX_DSI_BG_CON, + RG_DSI_BG_CKEN | RG_DSI_BG_CORE_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_PLL_CON0, - RG_DSI_MPPLL_DIV_MSK); + mtk_phy_clear_bits(base + MIPITX_DSI_PLL_CON0, RG_DSI_MPPLL_DIV_MSK); } static long mtk_mipi_tx_pll_round_rate(struct clk_hw *hw, unsigned long rate, @@ -254,10 +258,10 @@ static void mtk_mipi_tx_power_on_signal(struct phy *phy) for (reg = MIPITX_DSI_CLOCK_LANE; reg <= MIPITX_DSI_DATA_LANE3; reg += 4) - mtk_mipi_tx_set_bits(mipi_tx, reg, RG_DSI_LNTx_LDOOUT_EN); + mtk_phy_set_bits(mipi_tx->regs + reg, RG_DSI_LNTx_LDOOUT_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_DSI_TOP_CON, - RG_DSI_PAD_TIE_LOW_EN); + mtk_phy_clear_bits(mipi_tx->regs + MIPITX_DSI_TOP_CON, + RG_DSI_PAD_TIE_LOW_EN); } static void mtk_mipi_tx_power_off_signal(struct phy *phy) @@ -265,23 +269,23 @@ static void mtk_mipi_tx_power_off_signal(struct phy *phy) struct mtk_mipi_tx *mipi_tx = phy_get_drvdata(phy); u32 reg; - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_DSI_TOP_CON, - RG_DSI_PAD_TIE_LOW_EN); + mtk_phy_set_bits(mipi_tx->regs + MIPITX_DSI_TOP_CON, + RG_DSI_PAD_TIE_LOW_EN); for (reg = MIPITX_DSI_CLOCK_LANE; reg <= MIPITX_DSI_DATA_LANE3; reg += 4) - mtk_mipi_tx_clear_bits(mipi_tx, reg, RG_DSI_LNTx_LDOOUT_EN); + mtk_phy_clear_bits(mipi_tx->regs + reg, RG_DSI_LNTx_LDOOUT_EN); } const struct mtk_mipitx_data mt2701_mipitx_data = { - .mppll_preserve = (3 << 8), + .mppll_preserve = 3, .mipi_tx_clk_ops = &mtk_mipi_tx_pll_ops, .mipi_tx_enable_signal = mtk_mipi_tx_power_on_signal, .mipi_tx_disable_signal = mtk_mipi_tx_power_off_signal, }; const struct mtk_mipitx_data mt8173_mipitx_data = { - .mppll_preserve = (0 << 8), + .mppll_preserve = 0, .mipi_tx_clk_ops = &mtk_mipi_tx_pll_ops, .mipi_tx_enable_signal = mtk_mipi_tx_power_on_signal, .mipi_tx_disable_signal = mtk_mipi_tx_power_off_signal, diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8183.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8183.c index 99108426d57c1efa25be9179090cf80a51045913..f021ec5a70e5c31082dfd2ce965b474cd6cb5fcb 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8183.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi-mt8183.c @@ -4,6 +4,7 @@ * Author: jitao.shi */ +#include "phy-mtk-io.h" #include "phy-mtk-mipi-dsi.h" #define MIPITX_LANE_CON 0x000c @@ -18,7 +19,7 @@ #define RG_DSI_PAD_TIEL_SEL BIT(8) #define MIPITX_VOLTAGE_SEL 0x0010 -#define RG_DSI_HSTX_LDO_REF_SEL (0xf << 6) +#define RG_DSI_HSTX_LDO_REF_SEL GENMASK(9, 6) #define MIPITX_PLL_PWR 0x0028 #define MIPITX_PLL_CON0 0x002c @@ -26,7 +27,7 @@ #define MIPITX_PLL_CON2 0x0034 #define MIPITX_PLL_CON3 0x0038 #define MIPITX_PLL_CON4 0x003c -#define RG_DSI_PLL_IBIAS (3 << 10) +#define RG_DSI_PLL_IBIAS GENMASK(11, 10) #define MIPITX_D2P_RTCODE 0x0100 #define MIPITX_D2_SW_CTL_EN 0x0144 @@ -41,11 +42,12 @@ #define AD_DSI_PLL_SDM_ISO_EN BIT(1) #define RG_DSI_PLL_EN BIT(4) -#define RG_DSI_PLL_POSDIV (0x7 << 8) +#define RG_DSI_PLL_POSDIV GENMASK(10, 8) static int mtk_mipi_tx_pll_enable(struct clk_hw *hw) { struct mtk_mipi_tx *mipi_tx = mtk_mipi_tx_from_clk_hw(hw); + void __iomem *base = mipi_tx->regs; unsigned int txdiv, txdiv0; u64 pcw; @@ -70,17 +72,16 @@ static int mtk_mipi_tx_pll_enable(struct clk_hw *hw) return -EINVAL; } - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_PLL_CON4, RG_DSI_PLL_IBIAS); + mtk_phy_clear_bits(base + MIPITX_PLL_CON4, RG_DSI_PLL_IBIAS); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_PLL_PWR, AD_DSI_PLL_SDM_PWR_ON); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_PLL_CON1, RG_DSI_PLL_EN); + mtk_phy_set_bits(base + MIPITX_PLL_PWR, AD_DSI_PLL_SDM_PWR_ON); + mtk_phy_clear_bits(base + MIPITX_PLL_CON1, RG_DSI_PLL_EN); udelay(1); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_PLL_PWR, AD_DSI_PLL_SDM_ISO_EN); + mtk_phy_clear_bits(base + MIPITX_PLL_PWR, AD_DSI_PLL_SDM_ISO_EN); pcw = div_u64(((u64)mipi_tx->data_rate * txdiv) << 24, 26000000); - writel(pcw, mipi_tx->regs + MIPITX_PLL_CON0); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_PLL_CON1, RG_DSI_PLL_POSDIV, - txdiv0 << 8); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_PLL_CON1, RG_DSI_PLL_EN); + writel(pcw, base + MIPITX_PLL_CON0); + mtk_phy_update_field(base + MIPITX_PLL_CON1, RG_DSI_PLL_POSDIV, txdiv0); + mtk_phy_set_bits(base + MIPITX_PLL_CON1, RG_DSI_PLL_EN); return 0; } @@ -88,11 +89,12 @@ static int mtk_mipi_tx_pll_enable(struct clk_hw *hw) static void mtk_mipi_tx_pll_disable(struct clk_hw *hw) { struct mtk_mipi_tx *mipi_tx = mtk_mipi_tx_from_clk_hw(hw); + void __iomem *base = mipi_tx->regs; - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_PLL_CON1, RG_DSI_PLL_EN); + mtk_phy_clear_bits(base + MIPITX_PLL_CON1, RG_DSI_PLL_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_PLL_PWR, AD_DSI_PLL_SDM_ISO_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_PLL_PWR, AD_DSI_PLL_SDM_PWR_ON); + mtk_phy_set_bits(base + MIPITX_PLL_PWR, AD_DSI_PLL_SDM_ISO_EN); + mtk_phy_clear_bits(base + MIPITX_PLL_PWR, AD_DSI_PLL_SDM_PWR_ON); } static long mtk_mipi_tx_pll_round_rate(struct clk_hw *hw, unsigned long rate, @@ -121,7 +123,7 @@ static void mtk_mipi_tx_config_calibration_data(struct mtk_mipi_tx *mipi_tx) mipi_tx->rt_code[i] |= 0x10 << 5; for (j = 0; j < 10; j++) - mtk_mipi_tx_update_bits(mipi_tx, + mtk_phy_update_bits(mipi_tx->regs + MIPITX_D2P_RTCODE * (i + 1) + j * 4, 1, mipi_tx->rt_code[i] >> j & 1); } @@ -130,44 +132,42 @@ static void mtk_mipi_tx_config_calibration_data(struct mtk_mipi_tx *mipi_tx) static void mtk_mipi_tx_power_on_signal(struct phy *phy) { struct mtk_mipi_tx *mipi_tx = phy_get_drvdata(phy); + void __iomem *base = mipi_tx->regs; /* BG_LPF_EN / BG_CORE_EN */ - writel(RG_DSI_PAD_TIEL_SEL | RG_DSI_BG_CORE_EN, - mipi_tx->regs + MIPITX_LANE_CON); + writel(RG_DSI_PAD_TIEL_SEL | RG_DSI_BG_CORE_EN, base + MIPITX_LANE_CON); usleep_range(30, 100); - writel(RG_DSI_BG_CORE_EN | RG_DSI_BG_LPF_EN, - mipi_tx->regs + MIPITX_LANE_CON); + writel(RG_DSI_BG_CORE_EN | RG_DSI_BG_LPF_EN, base + MIPITX_LANE_CON); /* Switch OFF each Lane */ - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_D0_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_D1_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_D2_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_D3_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_clear_bits(mipi_tx, MIPITX_CK_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_clear_bits(base + MIPITX_D0_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_clear_bits(base + MIPITX_D1_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_clear_bits(base + MIPITX_D2_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_clear_bits(base + MIPITX_D3_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_clear_bits(base + MIPITX_CK_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_update_bits(mipi_tx, MIPITX_VOLTAGE_SEL, - RG_DSI_HSTX_LDO_REF_SEL, - (mipi_tx->mipitx_drive - 3000) / 200 << 6); + mtk_phy_update_field(base + MIPITX_VOLTAGE_SEL, RG_DSI_HSTX_LDO_REF_SEL, + (mipi_tx->mipitx_drive - 3000) / 200); mtk_mipi_tx_config_calibration_data(mipi_tx); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_CK_CKMODE_EN, DSI_CK_CKMODE_EN); + mtk_phy_set_bits(base + MIPITX_CK_CKMODE_EN, DSI_CK_CKMODE_EN); } static void mtk_mipi_tx_power_off_signal(struct phy *phy) { struct mtk_mipi_tx *mipi_tx = phy_get_drvdata(phy); + void __iomem *base = mipi_tx->regs; /* Switch ON each Lane */ - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_D0_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_D1_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_D2_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_D3_SW_CTL_EN, DSI_SW_CTL_EN); - mtk_mipi_tx_set_bits(mipi_tx, MIPITX_CK_SW_CTL_EN, DSI_SW_CTL_EN); - - writel(RG_DSI_PAD_TIEL_SEL | RG_DSI_BG_CORE_EN, - mipi_tx->regs + MIPITX_LANE_CON); - writel(RG_DSI_PAD_TIEL_SEL, mipi_tx->regs + MIPITX_LANE_CON); + mtk_phy_set_bits(base + MIPITX_D0_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_set_bits(base + MIPITX_D1_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_set_bits(base + MIPITX_D2_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_set_bits(base + MIPITX_D3_SW_CTL_EN, DSI_SW_CTL_EN); + mtk_phy_set_bits(base + MIPITX_CK_SW_CTL_EN, DSI_SW_CTL_EN); + + writel(RG_DSI_PAD_TIEL_SEL | RG_DSI_BG_CORE_EN, base + MIPITX_LANE_CON); + writel(RG_DSI_PAD_TIEL_SEL, base + MIPITX_LANE_CON); } const struct mtk_mipitx_data mt8183_mipitx_data = { diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c index 28506932bd9199e7a12503a61de0d37e7287a94b..cf9c386385bbb2d9a9c13cb1e23bc4d021974fd5 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.c +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.c @@ -10,30 +10,6 @@ inline struct mtk_mipi_tx *mtk_mipi_tx_from_clk_hw(struct clk_hw *hw) return container_of(hw, struct mtk_mipi_tx, pll_hw); } -void mtk_mipi_tx_clear_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, - u32 bits) -{ - u32 temp = readl(mipi_tx->regs + offset); - - writel(temp & ~bits, mipi_tx->regs + offset); -} - -void mtk_mipi_tx_set_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, - u32 bits) -{ - u32 temp = readl(mipi_tx->regs + offset); - - writel(temp | bits, mipi_tx->regs + offset); -} - -void mtk_mipi_tx_update_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, - u32 mask, u32 data) -{ - u32 temp = readl(mipi_tx->regs + offset); - - writel((temp & ~mask) | (data & mask), mipi_tx->regs + offset); -} - int mtk_mipi_tx_pll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate) { diff --git a/drivers/phy/mediatek/phy-mtk-mipi-dsi.h b/drivers/phy/mediatek/phy-mtk-mipi-dsi.h index c76f07c3fdeb10132fb9125cb3c598c50f4131fc..47b60b1a72261752d3b22a1144e374b7c639605d 100644 --- a/drivers/phy/mediatek/phy-mtk-mipi-dsi.h +++ b/drivers/phy/mediatek/phy-mtk-mipi-dsi.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -37,10 +36,6 @@ struct mtk_mipi_tx { }; struct mtk_mipi_tx *mtk_mipi_tx_from_clk_hw(struct clk_hw *hw); -void mtk_mipi_tx_clear_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, u32 bits); -void mtk_mipi_tx_set_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, u32 bits); -void mtk_mipi_tx_update_bits(struct mtk_mipi_tx *mipi_tx, u32 offset, u32 mask, - u32 data); int mtk_mipi_tx_pll_set_rate(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate); unsigned long mtk_mipi_tx_pll_recalc_rate(struct clk_hw *hw, diff --git a/drivers/phy/mediatek/phy-mtk-pcie.c b/drivers/phy/mediatek/phy-mtk-pcie.c index 7f29d43442bfc71a9b62c2b4e99b9c422b5438a9..25dbd6e35722239a245ac338d26ad9a5ac95d4d8 100644 --- a/drivers/phy/mediatek/phy-mtk-pcie.c +++ b/drivers/phy/mediatek/phy-mtk-pcie.c @@ -89,14 +89,14 @@ static void mtk_pcie_efuse_set_lane(struct mtk_pcie_phy *pcie_phy, addr = pcie_phy->sif_base + PEXTP_ANA_LN0_TRX_REG + lane * PEXTP_ANA_LANE_OFFSET; - mtk_phy_update_bits(addr + PEXTP_ANA_TX_REG, EFUSE_LN_TX_PMOS_SEL, - FIELD_PREP(EFUSE_LN_TX_PMOS_SEL, data->tx_pmos)); + mtk_phy_update_field(addr + PEXTP_ANA_TX_REG, EFUSE_LN_TX_PMOS_SEL, + data->tx_pmos); - mtk_phy_update_bits(addr + PEXTP_ANA_TX_REG, EFUSE_LN_TX_NMOS_SEL, - FIELD_PREP(EFUSE_LN_TX_NMOS_SEL, data->tx_nmos)); + mtk_phy_update_field(addr + PEXTP_ANA_TX_REG, EFUSE_LN_TX_NMOS_SEL, + data->tx_nmos); - mtk_phy_update_bits(addr + PEXTP_ANA_RX_REG, EFUSE_LN_RX_SEL, - FIELD_PREP(EFUSE_LN_RX_SEL, data->rx_data)); + mtk_phy_update_field(addr + PEXTP_ANA_RX_REG, EFUSE_LN_RX_SEL, + data->rx_data); } /** @@ -116,9 +116,8 @@ static int mtk_pcie_phy_init(struct phy *phy) return 0; /* Set global data */ - mtk_phy_update_bits(pcie_phy->sif_base + PEXTP_ANA_GLB_00_REG, - EFUSE_GLB_INTR_SEL, - FIELD_PREP(EFUSE_GLB_INTR_SEL, pcie_phy->efuse_glb_intr)); + mtk_phy_update_field(pcie_phy->sif_base + PEXTP_ANA_GLB_00_REG, + EFUSE_GLB_INTR_SEL, pcie_phy->efuse_glb_intr); for (i = 0; i < pcie_phy->data->num_lanes; i++) mtk_pcie_efuse_set_lane(pcie_phy, i); diff --git a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c index 8ee7682b8e93eb86346d1e55e1f6963a5f8f77d0..e906a82791bdaad9e5ac0ce1aecc278d576c03e4 100644 --- a/drivers/phy/mediatek/phy-mtk-tphy.c +++ b/drivers/phy/mediatek/phy-mtk-tphy.c @@ -49,35 +49,28 @@ #define U3P_USBPHYACR0 0x000 #define PA0_RG_U2PLL_FORCE_ON BIT(15) #define PA0_USB20_PLL_PREDIV GENMASK(7, 6) -#define PA0_USB20_PLL_PREDIV_VAL(x) ((0x3 & (x)) << 6) #define PA0_RG_USB20_INTR_EN BIT(5) #define U3P_USBPHYACR1 0x004 #define PA1_RG_INTR_CAL GENMASK(23, 19) -#define PA1_RG_INTR_CAL_VAL(x) ((0x1f & (x)) << 19) #define PA1_RG_VRT_SEL GENMASK(14, 12) -#define PA1_RG_VRT_SEL_VAL(x) ((0x7 & (x)) << 12) #define PA1_RG_TERM_SEL GENMASK(10, 8) -#define PA1_RG_TERM_SEL_VAL(x) ((0x7 & (x)) << 8) #define U3P_USBPHYACR2 0x008 #define PA2_RG_U2PLL_BW GENMASK(21, 19) -#define PA2_RG_U2PLL_BW_VAL(x) ((0x7 & (x)) << 19) #define PA2_RG_SIF_U2PLL_FORCE_EN BIT(18) #define U3P_USBPHYACR5 0x014 #define PA5_RG_U2_HSTX_SRCAL_EN BIT(15) #define PA5_RG_U2_HSTX_SRCTRL GENMASK(14, 12) -#define PA5_RG_U2_HSTX_SRCTRL_VAL(x) ((0x7 & (x)) << 12) #define PA5_RG_U2_HS_100U_U3_EN BIT(11) #define U3P_USBPHYACR6 0x018 +#define PA6_RG_U2_PRE_EMP GENMASK(31, 30) #define PA6_RG_U2_BC11_SW_EN BIT(23) #define PA6_RG_U2_OTG_VBUSCMP_EN BIT(20) #define PA6_RG_U2_DISCTH GENMASK(7, 4) -#define PA6_RG_U2_DISCTH_VAL(x) ((0xf & (x)) << 4) #define PA6_RG_U2_SQTH GENMASK(3, 0) -#define PA6_RG_U2_SQTH_VAL(x) (0xf & (x)) #define U3P_U2PHYACR4 0x020 #define P2C_RG_USB20_GPIO_CTL BIT(9) @@ -104,11 +97,9 @@ #define P2C_FORCE_SUSPENDM BIT(18) #define P2C_FORCE_TERMSEL BIT(17) #define P2C_RG_DATAIN GENMASK(13, 10) -#define P2C_RG_DATAIN_VAL(x) ((0xf & (x)) << 10) #define P2C_RG_DMPULLDOWN BIT(7) #define P2C_RG_DPPULLDOWN BIT(6) #define P2C_RG_XCVRSEL GENMASK(5, 4) -#define P2C_RG_XCVRSEL_VAL(x) ((0x3 & (x)) << 4) #define P2C_RG_SUSPENDM BIT(3) #define P2C_RG_TERMSEL BIT(2) #define P2C_DTM0_PART_MASK \ @@ -139,87 +130,65 @@ #define U3P_U3_PHYA_REG0 0x000 #define P3A_RG_IEXT_INTR GENMASK(15, 10) -#define P3A_RG_IEXT_INTR_VAL(x) ((0x3f & (x)) << 10) #define P3A_RG_CLKDRV_OFF GENMASK(3, 2) -#define P3A_RG_CLKDRV_OFF_VAL(x) ((0x3 & (x)) << 2) #define U3P_U3_PHYA_REG1 0x004 #define P3A_RG_CLKDRV_AMP GENMASK(31, 29) -#define P3A_RG_CLKDRV_AMP_VAL(x) ((0x7 & (x)) << 29) #define U3P_U3_PHYA_REG6 0x018 #define P3A_RG_TX_EIDLE_CM GENMASK(31, 28) -#define P3A_RG_TX_EIDLE_CM_VAL(x) ((0xf & (x)) << 28) #define U3P_U3_PHYA_REG9 0x024 #define P3A_RG_RX_DAC_MUX GENMASK(5, 1) -#define P3A_RG_RX_DAC_MUX_VAL(x) ((0x1f & (x)) << 1) #define U3P_U3_PHYA_DA_REG0 0x100 #define P3A_RG_XTAL_EXT_PE2H GENMASK(17, 16) -#define P3A_RG_XTAL_EXT_PE2H_VAL(x) ((0x3 & (x)) << 16) #define P3A_RG_XTAL_EXT_PE1H GENMASK(13, 12) -#define P3A_RG_XTAL_EXT_PE1H_VAL(x) ((0x3 & (x)) << 12) #define P3A_RG_XTAL_EXT_EN_U3 GENMASK(11, 10) -#define P3A_RG_XTAL_EXT_EN_U3_VAL(x) ((0x3 & (x)) << 10) #define U3P_U3_PHYA_DA_REG4 0x108 #define P3A_RG_PLL_DIVEN_PE2H GENMASK(21, 19) #define P3A_RG_PLL_BC_PE2H GENMASK(7, 6) -#define P3A_RG_PLL_BC_PE2H_VAL(x) ((0x3 & (x)) << 6) #define U3P_U3_PHYA_DA_REG5 0x10c #define P3A_RG_PLL_BR_PE2H GENMASK(29, 28) -#define P3A_RG_PLL_BR_PE2H_VAL(x) ((0x3 & (x)) << 28) #define P3A_RG_PLL_IC_PE2H GENMASK(15, 12) -#define P3A_RG_PLL_IC_PE2H_VAL(x) ((0xf & (x)) << 12) #define U3P_U3_PHYA_DA_REG6 0x110 #define P3A_RG_PLL_IR_PE2H GENMASK(19, 16) -#define P3A_RG_PLL_IR_PE2H_VAL(x) ((0xf & (x)) << 16) #define U3P_U3_PHYA_DA_REG7 0x114 #define P3A_RG_PLL_BP_PE2H GENMASK(19, 16) -#define P3A_RG_PLL_BP_PE2H_VAL(x) ((0xf & (x)) << 16) #define U3P_U3_PHYA_DA_REG20 0x13c #define P3A_RG_PLL_DELTA1_PE2H GENMASK(31, 16) -#define P3A_RG_PLL_DELTA1_PE2H_VAL(x) ((0xffff & (x)) << 16) #define U3P_U3_PHYA_DA_REG25 0x148 #define P3A_RG_PLL_DELTA_PE2H GENMASK(15, 0) -#define P3A_RG_PLL_DELTA_PE2H_VAL(x) (0xffff & (x)) #define U3P_U3_PHYD_LFPS1 0x00c #define P3D_RG_FWAKE_TH GENMASK(21, 16) -#define P3D_RG_FWAKE_TH_VAL(x) ((0x3f & (x)) << 16) #define U3P_U3_PHYD_IMPCAL0 0x010 #define P3D_RG_FORCE_TX_IMPEL BIT(31) #define P3D_RG_TX_IMPEL GENMASK(28, 24) -#define P3D_RG_TX_IMPEL_VAL(x) ((0x1f & (x)) << 24) #define U3P_U3_PHYD_IMPCAL1 0x014 #define P3D_RG_FORCE_RX_IMPEL BIT(31) #define P3D_RG_RX_IMPEL GENMASK(28, 24) -#define P3D_RG_RX_IMPEL_VAL(x) ((0x1f & (x)) << 24) #define U3P_U3_PHYD_RSV 0x054 #define P3D_RG_EFUSE_AUTO_LOAD_DIS BIT(12) #define U3P_U3_PHYD_CDR1 0x05c #define P3D_RG_CDR_BIR_LTD1 GENMASK(28, 24) -#define P3D_RG_CDR_BIR_LTD1_VAL(x) ((0x1f & (x)) << 24) #define P3D_RG_CDR_BIR_LTD0 GENMASK(12, 8) -#define P3D_RG_CDR_BIR_LTD0_VAL(x) ((0x1f & (x)) << 8) #define U3P_U3_PHYD_RXDET1 0x128 #define P3D_RG_RXDET_STB2_SET GENMASK(17, 9) -#define P3D_RG_RXDET_STB2_SET_VAL(x) ((0x1ff & (x)) << 9) #define U3P_U3_PHYD_RXDET2 0x12c #define P3D_RG_RXDET_STB2_SET_P3 GENMASK(8, 0) -#define P3D_RG_RXDET_STB2_SET_P3_VAL(x) (0x1ff & (x)) #define U3P_SPLLC_XTALCTL3 0x018 #define XC3_RG_U3_XTAL_RX_PWD BIT(9) @@ -227,10 +196,8 @@ #define U3P_U2FREQ_FMCR0 0x00 #define P2F_RG_MONCLK_SEL GENMASK(27, 26) -#define P2F_RG_MONCLK_SEL_VAL(x) ((0x3 & (x)) << 26) #define P2F_RG_FREQDET_EN BIT(24) #define P2F_RG_CYCLECNT GENMASK(23, 0) -#define P2F_RG_CYCLECNT_VAL(x) ((P2F_RG_CYCLECNT) & (x)) #define U3P_U2FREQ_VALUE 0x0c @@ -247,60 +214,45 @@ #define PHYD_CTRL_SIGNAL_MODE4 0x1c /* CDR Charge Pump P-path current adjustment */ #define RG_CDR_BICLTD1_GEN1_MSK GENMASK(23, 20) -#define RG_CDR_BICLTD1_GEN1_VAL(x) ((0xf & (x)) << 20) #define RG_CDR_BICLTD0_GEN1_MSK GENMASK(11, 8) -#define RG_CDR_BICLTD0_GEN1_VAL(x) ((0xf & (x)) << 8) #define PHYD_DESIGN_OPTION2 0x24 /* Symbol lock count selection */ #define RG_LOCK_CNT_SEL_MSK GENMASK(5, 4) -#define RG_LOCK_CNT_SEL_VAL(x) ((0x3 & (x)) << 4) #define PHYD_DESIGN_OPTION9 0x40 /* COMWAK GAP width window */ #define RG_TG_MAX_MSK GENMASK(20, 16) -#define RG_TG_MAX_VAL(x) ((0x1f & (x)) << 16) /* COMINIT GAP width window */ #define RG_T2_MAX_MSK GENMASK(13, 8) -#define RG_T2_MAX_VAL(x) ((0x3f & (x)) << 8) /* COMWAK GAP width window */ #define RG_TG_MIN_MSK GENMASK(7, 5) -#define RG_TG_MIN_VAL(x) ((0x7 & (x)) << 5) /* COMINIT GAP width window */ #define RG_T2_MIN_MSK GENMASK(4, 0) -#define RG_T2_MIN_VAL(x) (0x1f & (x)) #define ANA_RG_CTRL_SIGNAL1 0x4c /* TX driver tail current control for 0dB de-empahsis mdoe for Gen1 speed */ #define RG_IDRV_0DB_GEN1_MSK GENMASK(13, 8) -#define RG_IDRV_0DB_GEN1_VAL(x) ((0x3f & (x)) << 8) #define ANA_RG_CTRL_SIGNAL4 0x58 #define RG_CDR_BICLTR_GEN1_MSK GENMASK(23, 20) -#define RG_CDR_BICLTR_GEN1_VAL(x) ((0xf & (x)) << 20) /* Loop filter R1 resistance adjustment for Gen1 speed */ #define RG_CDR_BR_GEN2_MSK GENMASK(10, 8) -#define RG_CDR_BR_GEN2_VAL(x) ((0x7 & (x)) << 8) #define ANA_RG_CTRL_SIGNAL6 0x60 /* I-path capacitance adjustment for Gen1 */ #define RG_CDR_BC_GEN1_MSK GENMASK(28, 24) -#define RG_CDR_BC_GEN1_VAL(x) ((0x1f & (x)) << 24) #define RG_CDR_BIRLTR_GEN1_MSK GENMASK(4, 0) -#define RG_CDR_BIRLTR_GEN1_VAL(x) (0x1f & (x)) #define ANA_EQ_EYE_CTRL_SIGNAL1 0x6c /* RX Gen1 LEQ tuning step */ #define RG_EQ_DLEQ_LFI_GEN1_MSK GENMASK(11, 8) -#define RG_EQ_DLEQ_LFI_GEN1_VAL(x) ((0xf & (x)) << 8) #define ANA_EQ_EYE_CTRL_SIGNAL4 0xd8 #define RG_CDR_BIRLTD0_GEN1_MSK GENMASK(20, 16) -#define RG_CDR_BIRLTD0_GEN1_VAL(x) ((0x1f & (x)) << 16) #define ANA_EQ_EYE_CTRL_SIGNAL5 0xdc #define RG_CDR_BIRLTD0_GEN3_MSK GENMASK(4, 0) -#define RG_CDR_BIRLTD0_GEN3_VAL(x) (0x1f & (x)) /* PHY switch between pcie/usb3/sgmii/sata */ #define USB_PHY_SWITCH_CTRL 0x0 @@ -370,6 +322,7 @@ struct mtk_phy_instance { int eye_term; int intr; int discth; + int pre_emphasis; bool bc12_en; }; @@ -411,9 +364,9 @@ static void hs_slew_rate_calibrate(struct mtk_tphy *tphy, /* set cycle count as 1024, and select u2 channel */ tmp = readl(fmreg + U3P_U2FREQ_FMCR0); tmp &= ~(P2F_RG_CYCLECNT | P2F_RG_MONCLK_SEL); - tmp |= P2F_RG_CYCLECNT_VAL(U3P_FM_DET_CYCLE_CNT); + tmp |= FIELD_PREP(P2F_RG_CYCLECNT, U3P_FM_DET_CYCLE_CNT); if (tphy->pdata->version == MTK_PHY_V1) - tmp |= P2F_RG_MONCLK_SEL_VAL(instance->index >> 1); + tmp |= FIELD_PREP(P2F_RG_MONCLK_SEL, instance->index >> 1); writel(tmp, fmreg + U3P_U2FREQ_FMCR0); @@ -446,8 +399,8 @@ static void hs_slew_rate_calibrate(struct mtk_tphy *tphy, tphy->src_ref_clk, tphy->src_coef); /* set HS slew rate */ - mtk_phy_update_bits(com + U3P_USBPHYACR5, PA5_RG_U2_HSTX_SRCTRL, - PA5_RG_U2_HSTX_SRCTRL_VAL(calibration_val)); + mtk_phy_update_field(com + U3P_USBPHYACR5, PA5_RG_U2_HSTX_SRCTRL, + calibration_val); /* disable USB ring oscillator */ mtk_phy_clear_bits(com + U3P_USBPHYACR5, PA5_RG_U2_HSTX_SRCAL_EN); @@ -457,33 +410,30 @@ static void u3_phy_instance_init(struct mtk_tphy *tphy, struct mtk_phy_instance *instance) { struct u3phy_banks *u3_banks = &instance->u3_banks; + void __iomem *phya = u3_banks->phya; + void __iomem *phyd = u3_banks->phyd; /* gating PCIe Analog XTAL clock */ mtk_phy_set_bits(u3_banks->spllc + U3P_SPLLC_XTALCTL3, XC3_RG_U3_XTAL_RX_PWD | XC3_RG_U3_FRC_XTAL_RX_PWD); /* gating XSQ */ - mtk_phy_update_bits(u3_banks->phya + U3P_U3_PHYA_DA_REG0, - P3A_RG_XTAL_EXT_EN_U3, P3A_RG_XTAL_EXT_EN_U3_VAL(2)); + mtk_phy_update_field(phya + U3P_U3_PHYA_DA_REG0, P3A_RG_XTAL_EXT_EN_U3, 2); - mtk_phy_update_bits(u3_banks->phya + U3P_U3_PHYA_REG9, - P3A_RG_RX_DAC_MUX, P3A_RG_RX_DAC_MUX_VAL(4)); + mtk_phy_update_field(phya + U3P_U3_PHYA_REG9, P3A_RG_RX_DAC_MUX, 4); - mtk_phy_update_bits(u3_banks->phya + U3P_U3_PHYA_REG6, - P3A_RG_TX_EIDLE_CM, P3A_RG_TX_EIDLE_CM_VAL(0xe)); + mtk_phy_update_field(phya + U3P_U3_PHYA_REG6, P3A_RG_TX_EIDLE_CM, 0xe); mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_CDR1, P3D_RG_CDR_BIR_LTD0 | P3D_RG_CDR_BIR_LTD1, - P3D_RG_CDR_BIR_LTD0_VAL(0xc) | P3D_RG_CDR_BIR_LTD1_VAL(0x3)); + FIELD_PREP(P3D_RG_CDR_BIR_LTD0, 0xc) | + FIELD_PREP(P3D_RG_CDR_BIR_LTD1, 0x3)); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_LFPS1, - P3D_RG_FWAKE_TH, P3D_RG_FWAKE_TH_VAL(0x34)); + mtk_phy_update_field(phyd + U3P_U3_PHYD_LFPS1, P3D_RG_FWAKE_TH, 0x34); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_RXDET1, - P3D_RG_RXDET_STB2_SET, P3D_RG_RXDET_STB2_SET_VAL(0x10)); + mtk_phy_update_field(phyd + U3P_U3_PHYD_RXDET1, P3D_RG_RXDET_STB2_SET, 0x10); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_RXDET2, - P3D_RG_RXDET_STB2_SET_P3, P3D_RG_RXDET_STB2_SET_P3_VAL(0x10)); + mtk_phy_update_field(phyd + U3P_U3_PHYD_RXDET2, P3D_RG_RXDET_STB2_SET_P3, 0x10); dev_dbg(tphy->dev, "%s(%d)\n", __func__, instance->index); } @@ -497,11 +447,9 @@ static void u2_phy_pll_26m_set(struct mtk_tphy *tphy, if (!tphy->pdata->sw_pll_48m_to_26m) return; - mtk_phy_update_bits(com + U3P_USBPHYACR0, PA0_USB20_PLL_PREDIV, - PA0_USB20_PLL_PREDIV_VAL(0)); + mtk_phy_update_field(com + U3P_USBPHYACR0, PA0_USB20_PLL_PREDIV, 0); - mtk_phy_update_bits(com + U3P_USBPHYACR2, PA2_RG_U2PLL_BW, - PA2_RG_U2PLL_BW_VAL(3)); + mtk_phy_update_field(com + U3P_USBPHYACR2, PA2_RG_U2PLL_BW, 3); writel(P2R_RG_U2PLL_FBDIV_26M, com + U3P_U2PHYA_RESV); @@ -519,8 +467,8 @@ static void u2_phy_instance_init(struct mtk_tphy *tphy, /* switch to USB function, and enable usb pll */ mtk_phy_clear_bits(com + U3P_U2PHYDTM0, P2C_FORCE_UART_EN | P2C_FORCE_SUSPENDM); - mtk_phy_update_bits(com + U3P_U2PHYDTM0, P2C_RG_XCVRSEL | P2C_RG_DATAIN, - P2C_RG_XCVRSEL_VAL(1) | P2C_RG_DATAIN_VAL(0)); + mtk_phy_clear_bits(com + U3P_U2PHYDTM0, + P2C_RG_XCVRSEL | P2C_RG_DATAIN | P2C_DTM0_PART_MASK); mtk_phy_clear_bits(com + U3P_U2PHYDTM1, P2C_RG_UART_EN); @@ -529,8 +477,7 @@ static void u2_phy_instance_init(struct mtk_tphy *tphy, /* disable switch 100uA current to SSUSB */ mtk_phy_clear_bits(com + U3P_USBPHYACR5, PA5_RG_U2_HS_100U_U3_EN); - if (!index) - mtk_phy_clear_bits(com + U3P_U2PHYACR4, P2C_U2_GPIO_CTR_MSK); + mtk_phy_clear_bits(com + U3P_U2PHYACR4, P2C_U2_GPIO_CTR_MSK); if (tphy->pdata->avoid_rx_sen_degradation) { if (!index) { @@ -548,7 +495,7 @@ static void u2_phy_instance_init(struct mtk_tphy *tphy, /* DP/DM BC1.1 path Disable */ mtk_phy_clear_bits(com + U3P_USBPHYACR6, PA6_RG_U2_BC11_SW_EN); - mtk_phy_update_bits(com + U3P_USBPHYACR6, PA6_RG_U2_SQTH, PA6_RG_U2_SQTH_VAL(2)); + mtk_phy_update_field(com + U3P_USBPHYACR6, PA6_RG_U2_SQTH, 2); /* Workaround only for mt8195, HW fix it for others (V3) */ u2_phy_pll_26m_set(tphy, instance); @@ -563,9 +510,6 @@ static void u2_phy_instance_power_on(struct mtk_tphy *tphy, void __iomem *com = u2_banks->com; u32 index = instance->index; - mtk_phy_clear_bits(com + U3P_U2PHYDTM0, - P2C_RG_XCVRSEL | P2C_RG_DATAIN | P2C_DTM0_PART_MASK); - /* OTG Enable */ mtk_phy_set_bits(com + U3P_USBPHYACR6, PA6_RG_U2_OTG_VBUSCMP_EN); @@ -588,8 +532,6 @@ static void u2_phy_instance_power_off(struct mtk_tphy *tphy, void __iomem *com = u2_banks->com; u32 index = instance->index; - mtk_phy_clear_bits(com + U3P_U2PHYDTM0, P2C_RG_XCVRSEL | P2C_RG_DATAIN); - /* OTG Disable */ mtk_phy_clear_bits(com + U3P_USBPHYACR6, PA6_RG_U2_OTG_VBUSCMP_EN); @@ -656,43 +598,39 @@ static void pcie_phy_instance_init(struct mtk_tphy *tphy, mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG0, P3A_RG_XTAL_EXT_PE1H | P3A_RG_XTAL_EXT_PE2H, - P3A_RG_XTAL_EXT_PE1H_VAL(0x2) | P3A_RG_XTAL_EXT_PE2H_VAL(0x2)); + FIELD_PREP(P3A_RG_XTAL_EXT_PE1H, 0x2) | + FIELD_PREP(P3A_RG_XTAL_EXT_PE2H, 0x2)); /* ref clk drive */ - mtk_phy_update_bits(phya + U3P_U3_PHYA_REG1, P3A_RG_CLKDRV_AMP, - P3A_RG_CLKDRV_AMP_VAL(0x4)); + mtk_phy_update_field(phya + U3P_U3_PHYA_REG1, P3A_RG_CLKDRV_AMP, 0x4); - mtk_phy_update_bits(phya + U3P_U3_PHYA_REG0, P3A_RG_CLKDRV_OFF, - P3A_RG_CLKDRV_OFF_VAL(0x1)); + mtk_phy_update_field(phya + U3P_U3_PHYA_REG0, P3A_RG_CLKDRV_OFF, 0x1); /* SSC delta -5000ppm */ - mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG20, P3A_RG_PLL_DELTA1_PE2H, - P3A_RG_PLL_DELTA1_PE2H_VAL(0x3c)); + mtk_phy_update_field(phya + U3P_U3_PHYA_DA_REG20, P3A_RG_PLL_DELTA1_PE2H, 0x3c); - mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG25, P3A_RG_PLL_DELTA_PE2H, - P3A_RG_PLL_DELTA_PE2H_VAL(0x36)); + mtk_phy_update_field(phya + U3P_U3_PHYA_DA_REG25, P3A_RG_PLL_DELTA_PE2H, 0x36); /* change pll BW 0.6M */ mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG5, P3A_RG_PLL_BR_PE2H | P3A_RG_PLL_IC_PE2H, - P3A_RG_PLL_BR_PE2H_VAL(0x1) | P3A_RG_PLL_IC_PE2H_VAL(0x1)); + FIELD_PREP(P3A_RG_PLL_BR_PE2H, 0x1) | + FIELD_PREP(P3A_RG_PLL_IC_PE2H, 0x1)); mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG4, P3A_RG_PLL_DIVEN_PE2H | P3A_RG_PLL_BC_PE2H, - P3A_RG_PLL_BC_PE2H_VAL(0x3)); + FIELD_PREP(P3A_RG_PLL_BC_PE2H, 0x3)); - mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG6, P3A_RG_PLL_IR_PE2H, - P3A_RG_PLL_IR_PE2H_VAL(0x2)); + mtk_phy_update_field(phya + U3P_U3_PHYA_DA_REG6, P3A_RG_PLL_IR_PE2H, 0x2); - mtk_phy_update_bits(phya + U3P_U3_PHYA_DA_REG7, P3A_RG_PLL_BP_PE2H, - P3A_RG_PLL_BP_PE2H_VAL(0xa)); + mtk_phy_update_field(phya + U3P_U3_PHYA_DA_REG7, P3A_RG_PLL_BP_PE2H, 0xa); /* Tx Detect Rx Timing: 10us -> 5us */ - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_RXDET1, - P3D_RG_RXDET_STB2_SET, P3D_RG_RXDET_STB2_SET_VAL(0x10)); + mtk_phy_update_field(u3_banks->phyd + U3P_U3_PHYD_RXDET1, + P3D_RG_RXDET_STB2_SET, 0x10); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_RXDET2, - P3D_RG_RXDET_STB2_SET_P3, P3D_RG_RXDET_STB2_SET_P3_VAL(0x10)); + mtk_phy_update_field(u3_banks->phyd + U3P_U3_PHYD_RXDET2, + P3D_RG_RXDET_STB2_SET_P3, 0x10); /* wait for PCIe subsys register to active */ usleep_range(2500, 3000); @@ -733,38 +671,38 @@ static void sata_phy_instance_init(struct mtk_tphy *tphy, /* charge current adjustment */ mtk_phy_update_bits(phyd + ANA_RG_CTRL_SIGNAL6, RG_CDR_BIRLTR_GEN1_MSK | RG_CDR_BC_GEN1_MSK, - RG_CDR_BIRLTR_GEN1_VAL(0x6) | RG_CDR_BC_GEN1_VAL(0x1a)); + FIELD_PREP(RG_CDR_BIRLTR_GEN1_MSK, 0x6) | + FIELD_PREP(RG_CDR_BC_GEN1_MSK, 0x1a)); - mtk_phy_update_bits(phyd + ANA_EQ_EYE_CTRL_SIGNAL4, RG_CDR_BIRLTD0_GEN1_MSK, - RG_CDR_BIRLTD0_GEN1_VAL(0x18)); + mtk_phy_update_field(phyd + ANA_EQ_EYE_CTRL_SIGNAL4, RG_CDR_BIRLTD0_GEN1_MSK, 0x18); - mtk_phy_update_bits(phyd + ANA_EQ_EYE_CTRL_SIGNAL5, RG_CDR_BIRLTD0_GEN3_MSK, - RG_CDR_BIRLTD0_GEN3_VAL(0x06)); + mtk_phy_update_field(phyd + ANA_EQ_EYE_CTRL_SIGNAL5, RG_CDR_BIRLTD0_GEN3_MSK, 0x06); mtk_phy_update_bits(phyd + ANA_RG_CTRL_SIGNAL4, RG_CDR_BICLTR_GEN1_MSK | RG_CDR_BR_GEN2_MSK, - RG_CDR_BICLTR_GEN1_VAL(0x0c) | RG_CDR_BR_GEN2_VAL(0x07)); + FIELD_PREP(RG_CDR_BICLTR_GEN1_MSK, 0x0c) | + FIELD_PREP(RG_CDR_BR_GEN2_MSK, 0x07)); mtk_phy_update_bits(phyd + PHYD_CTRL_SIGNAL_MODE4, RG_CDR_BICLTD0_GEN1_MSK | RG_CDR_BICLTD1_GEN1_MSK, - RG_CDR_BICLTD0_GEN1_VAL(0x08) | RG_CDR_BICLTD1_GEN1_VAL(0x02)); + FIELD_PREP(RG_CDR_BICLTD0_GEN1_MSK, 0x08) | + FIELD_PREP(RG_CDR_BICLTD1_GEN1_MSK, 0x02)); - mtk_phy_update_bits(phyd + PHYD_DESIGN_OPTION2, RG_LOCK_CNT_SEL_MSK, - RG_LOCK_CNT_SEL_VAL(0x02)); + mtk_phy_update_field(phyd + PHYD_DESIGN_OPTION2, RG_LOCK_CNT_SEL_MSK, 0x02); mtk_phy_update_bits(phyd + PHYD_DESIGN_OPTION9, RG_T2_MIN_MSK | RG_TG_MIN_MSK, - RG_T2_MIN_VAL(0x12) | RG_TG_MIN_VAL(0x04)); + FIELD_PREP(RG_T2_MIN_MSK, 0x12) | + FIELD_PREP(RG_TG_MIN_MSK, 0x04)); mtk_phy_update_bits(phyd + PHYD_DESIGN_OPTION9, RG_T2_MAX_MSK | RG_TG_MAX_MSK, - RG_T2_MAX_VAL(0x31) | RG_TG_MAX_VAL(0x0e)); + FIELD_PREP(RG_T2_MAX_MSK, 0x31) | + FIELD_PREP(RG_TG_MAX_MSK, 0x0e)); - mtk_phy_update_bits(phyd + ANA_RG_CTRL_SIGNAL1, RG_IDRV_0DB_GEN1_MSK, - RG_IDRV_0DB_GEN1_VAL(0x20)); + mtk_phy_update_field(phyd + ANA_RG_CTRL_SIGNAL1, RG_IDRV_0DB_GEN1_MSK, 0x20); - mtk_phy_update_bits(phyd + ANA_EQ_EYE_CTRL_SIGNAL1, RG_EQ_DLEQ_LFI_GEN1_MSK, - RG_EQ_DLEQ_LFI_GEN1_VAL(0x03)); + mtk_phy_update_field(phyd + ANA_EQ_EYE_CTRL_SIGNAL1, RG_EQ_DLEQ_LFI_GEN1_MSK, 0x03); dev_dbg(tphy->dev, "%s(%d)\n", __func__, instance->index); } @@ -841,10 +779,13 @@ static void phy_parse_property(struct mtk_tphy *tphy, &instance->intr); device_property_read_u32(dev, "mediatek,discth", &instance->discth); + device_property_read_u32(dev, "mediatek,pre-emphasis", + &instance->pre_emphasis); dev_dbg(dev, "bc12:%d, src:%d, vrt:%d, term:%d, intr:%d, disc:%d\n", instance->bc12_en, instance->eye_src, instance->eye_vrt, instance->eye_term, instance->intr, instance->discth); + dev_dbg(dev, "pre-emp:%d\n", instance->pre_emphasis); } static void u2_phy_props_set(struct mtk_tphy *tphy, @@ -857,24 +798,33 @@ static void u2_phy_props_set(struct mtk_tphy *tphy, mtk_phy_set_bits(com + U3P_U2PHYBC12C, P2C_RG_CHGDT_EN); if (tphy->pdata->version < MTK_PHY_V3 && instance->eye_src) - mtk_phy_update_bits(com + U3P_USBPHYACR5, PA5_RG_U2_HSTX_SRCTRL, - PA5_RG_U2_HSTX_SRCTRL_VAL(instance->eye_src)); + mtk_phy_update_field(com + U3P_USBPHYACR5, PA5_RG_U2_HSTX_SRCTRL, + instance->eye_src); if (instance->eye_vrt) - mtk_phy_update_bits(com + U3P_USBPHYACR1, PA1_RG_VRT_SEL, - PA1_RG_VRT_SEL_VAL(instance->eye_vrt)); + mtk_phy_update_field(com + U3P_USBPHYACR1, PA1_RG_VRT_SEL, + instance->eye_vrt); if (instance->eye_term) - mtk_phy_update_bits(com + U3P_USBPHYACR1, PA1_RG_TERM_SEL, - PA1_RG_TERM_SEL_VAL(instance->eye_term)); + mtk_phy_update_field(com + U3P_USBPHYACR1, PA1_RG_TERM_SEL, + instance->eye_term); - if (instance->intr) - mtk_phy_update_bits(com + U3P_USBPHYACR1, PA1_RG_INTR_CAL, - PA1_RG_INTR_CAL_VAL(instance->intr)); + if (instance->intr) { + if (u2_banks->misc) + mtk_phy_set_bits(u2_banks->misc + U3P_MISC_REG1, + MR1_EFUSE_AUTO_LOAD_DIS); + + mtk_phy_update_field(com + U3P_USBPHYACR1, PA1_RG_INTR_CAL, + instance->intr); + } if (instance->discth) - mtk_phy_update_bits(com + U3P_USBPHYACR6, PA6_RG_U2_DISCTH, - PA6_RG_U2_DISCTH_VAL(instance->discth)); + mtk_phy_update_field(com + U3P_USBPHYACR6, PA6_RG_U2_DISCTH, + instance->discth); + + if (instance->pre_emphasis) + mtk_phy_update_field(com + U3P_USBPHYACR6, PA6_RG_U2_PRE_EMP, + instance->pre_emphasis); } /* type switch for usb3/pcie/sgmii/sata */ @@ -906,7 +856,7 @@ static int phy_type_syscon_get(struct mtk_phy_instance *instance, static int phy_type_set(struct mtk_phy_instance *instance) { int type; - u32 mask; + u32 offset; if (!instance->type_sw) return 0; @@ -929,8 +879,9 @@ static int phy_type_set(struct mtk_phy_instance *instance) return 0; } - mask = RG_PHY_SW_TYPE << (instance->type_sw_index * BITS_PER_BYTE); - regmap_update_bits(instance->type_sw, instance->type_sw_reg, mask, type); + offset = instance->type_sw_index * BITS_PER_BYTE; + regmap_update_bits(instance->type_sw, instance->type_sw_reg, + RG_PHY_SW_TYPE << offset, type << offset); return 0; } @@ -1022,23 +973,23 @@ static void phy_efuse_set(struct mtk_phy_instance *instance) case PHY_TYPE_USB2: mtk_phy_set_bits(u2_banks->misc + U3P_MISC_REG1, MR1_EFUSE_AUTO_LOAD_DIS); - mtk_phy_update_bits(u2_banks->com + U3P_USBPHYACR1, PA1_RG_INTR_CAL, - PA1_RG_INTR_CAL_VAL(instance->efuse_intr)); + mtk_phy_update_field(u2_banks->com + U3P_USBPHYACR1, PA1_RG_INTR_CAL, + instance->efuse_intr); break; case PHY_TYPE_USB3: case PHY_TYPE_PCIE: mtk_phy_set_bits(u3_banks->phyd + U3P_U3_PHYD_RSV, P3D_RG_EFUSE_AUTO_LOAD_DIS); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_IMPCAL0, P3D_RG_TX_IMPEL, - P3D_RG_TX_IMPEL_VAL(instance->efuse_tx_imp)); + mtk_phy_update_field(u3_banks->phyd + U3P_U3_PHYD_IMPCAL0, P3D_RG_TX_IMPEL, + instance->efuse_tx_imp); mtk_phy_set_bits(u3_banks->phyd + U3P_U3_PHYD_IMPCAL0, P3D_RG_FORCE_TX_IMPEL); - mtk_phy_update_bits(u3_banks->phyd + U3P_U3_PHYD_IMPCAL1, P3D_RG_RX_IMPEL, - P3D_RG_RX_IMPEL_VAL(instance->efuse_rx_imp)); + mtk_phy_update_field(u3_banks->phyd + U3P_U3_PHYD_IMPCAL1, P3D_RG_RX_IMPEL, + instance->efuse_rx_imp); mtk_phy_set_bits(u3_banks->phyd + U3P_U3_PHYD_IMPCAL1, P3D_RG_FORCE_RX_IMPEL); - mtk_phy_update_bits(u3_banks->phya + U3P_U3_PHYA_REG0, P3A_RG_IEXT_INTR, - P3A_RG_IEXT_INTR_VAL(instance->efuse_intr)); + mtk_phy_update_field(u3_banks->phya + U3P_U3_PHYA_REG0, P3A_RG_IEXT_INTR, + instance->efuse_intr); break; default: dev_warn(dev, "no sw efuse for type %d\n", instance->type); diff --git a/drivers/phy/mediatek/phy-mtk-ufs.c b/drivers/phy/mediatek/phy-mtk-ufs.c index a6af069412038e49739aa3e01f148f090392b14d..fc19e0fa8ed551bec19efeb64ce65d1a94addabd 100644 --- a/drivers/phy/mediatek/phy-mtk-ufs.c +++ b/drivers/phy/mediatek/phy-mtk-ufs.c @@ -11,6 +11,8 @@ #include #include +#include "phy-mtk-io.h" + /* mphy register and offsets */ #define MP_GLB_DIG_8C 0x008C #define FRC_PLL_ISO_EN BIT(8) @@ -39,34 +41,6 @@ struct ufs_mtk_phy { struct clk_bulk_data clks[UFSPHY_CLKS_CNT]; }; -static inline u32 mphy_readl(struct ufs_mtk_phy *phy, u32 reg) -{ - return readl(phy->mmio + reg); -} - -static inline void mphy_writel(struct ufs_mtk_phy *phy, u32 val, u32 reg) -{ - writel(val, phy->mmio + reg); -} - -static void mphy_set_bit(struct ufs_mtk_phy *phy, u32 reg, u32 bit) -{ - u32 val; - - val = mphy_readl(phy, reg); - val |= bit; - mphy_writel(phy, val, reg); -} - -static void mphy_clr_bit(struct ufs_mtk_phy *phy, u32 reg, u32 bit) -{ - u32 val; - - val = mphy_readl(phy, reg); - val &= ~bit; - mphy_writel(phy, val, reg); -} - static struct ufs_mtk_phy *get_ufs_mtk_phy(struct phy *generic_phy) { return (struct ufs_mtk_phy *)phy_get_drvdata(generic_phy); @@ -84,57 +58,61 @@ static int ufs_mtk_phy_clk_init(struct ufs_mtk_phy *phy) static void ufs_mtk_phy_set_active(struct ufs_mtk_phy *phy) { + void __iomem *mmio = phy->mmio; + /* release DA_MP_PLL_PWR_ON */ - mphy_set_bit(phy, MP_GLB_DIG_8C, PLL_PWR_ON); - mphy_clr_bit(phy, MP_GLB_DIG_8C, FRC_FRC_PWR_ON); + mtk_phy_set_bits(mmio + MP_GLB_DIG_8C, PLL_PWR_ON); + mtk_phy_clear_bits(mmio + MP_GLB_DIG_8C, FRC_FRC_PWR_ON); /* release DA_MP_PLL_ISO_EN */ - mphy_clr_bit(phy, MP_GLB_DIG_8C, PLL_ISO_EN); - mphy_clr_bit(phy, MP_GLB_DIG_8C, FRC_PLL_ISO_EN); + mtk_phy_clear_bits(mmio + MP_GLB_DIG_8C, PLL_ISO_EN); + mtk_phy_clear_bits(mmio + MP_GLB_DIG_8C, FRC_PLL_ISO_EN); /* release DA_MP_CDR_PWR_ON */ - mphy_set_bit(phy, MP_LN_RX_44, CDR_PWR_ON); - mphy_clr_bit(phy, MP_LN_RX_44, FRC_CDR_PWR_ON); + mtk_phy_set_bits(mmio + MP_LN_RX_44, CDR_PWR_ON); + mtk_phy_clear_bits(mmio + MP_LN_RX_44, FRC_CDR_PWR_ON); /* release DA_MP_CDR_ISO_EN */ - mphy_clr_bit(phy, MP_LN_RX_44, CDR_ISO_EN); - mphy_clr_bit(phy, MP_LN_RX_44, FRC_CDR_ISO_EN); + mtk_phy_clear_bits(mmio + MP_LN_RX_44, CDR_ISO_EN); + mtk_phy_clear_bits(mmio + MP_LN_RX_44, FRC_CDR_ISO_EN); /* release DA_MP_RX0_SQ_EN */ - mphy_set_bit(phy, MP_LN_DIG_RX_AC, RX_SQ_EN); - mphy_clr_bit(phy, MP_LN_DIG_RX_AC, FRC_RX_SQ_EN); + mtk_phy_set_bits(mmio + MP_LN_DIG_RX_AC, RX_SQ_EN); + mtk_phy_clear_bits(mmio + MP_LN_DIG_RX_AC, FRC_RX_SQ_EN); /* delay 1us to wait DIFZ stable */ udelay(1); /* release DIFZ */ - mphy_clr_bit(phy, MP_LN_DIG_RX_9C, FSM_DIFZ_FRC); + mtk_phy_clear_bits(mmio + MP_LN_DIG_RX_9C, FSM_DIFZ_FRC); } static void ufs_mtk_phy_set_deep_hibern(struct ufs_mtk_phy *phy) { + void __iomem *mmio = phy->mmio; + /* force DIFZ */ - mphy_set_bit(phy, MP_LN_DIG_RX_9C, FSM_DIFZ_FRC); + mtk_phy_set_bits(mmio + MP_LN_DIG_RX_9C, FSM_DIFZ_FRC); /* force DA_MP_RX0_SQ_EN */ - mphy_set_bit(phy, MP_LN_DIG_RX_AC, FRC_RX_SQ_EN); - mphy_clr_bit(phy, MP_LN_DIG_RX_AC, RX_SQ_EN); + mtk_phy_set_bits(mmio + MP_LN_DIG_RX_AC, FRC_RX_SQ_EN); + mtk_phy_clear_bits(mmio + MP_LN_DIG_RX_AC, RX_SQ_EN); /* force DA_MP_CDR_ISO_EN */ - mphy_set_bit(phy, MP_LN_RX_44, FRC_CDR_ISO_EN); - mphy_set_bit(phy, MP_LN_RX_44, CDR_ISO_EN); + mtk_phy_set_bits(mmio + MP_LN_RX_44, FRC_CDR_ISO_EN); + mtk_phy_set_bits(mmio + MP_LN_RX_44, CDR_ISO_EN); /* force DA_MP_CDR_PWR_ON */ - mphy_set_bit(phy, MP_LN_RX_44, FRC_CDR_PWR_ON); - mphy_clr_bit(phy, MP_LN_RX_44, CDR_PWR_ON); + mtk_phy_set_bits(mmio + MP_LN_RX_44, FRC_CDR_PWR_ON); + mtk_phy_clear_bits(mmio + MP_LN_RX_44, CDR_PWR_ON); /* force DA_MP_PLL_ISO_EN */ - mphy_set_bit(phy, MP_GLB_DIG_8C, FRC_PLL_ISO_EN); - mphy_set_bit(phy, MP_GLB_DIG_8C, PLL_ISO_EN); + mtk_phy_set_bits(mmio + MP_GLB_DIG_8C, FRC_PLL_ISO_EN); + mtk_phy_set_bits(mmio + MP_GLB_DIG_8C, PLL_ISO_EN); /* force DA_MP_PLL_PWR_ON */ - mphy_set_bit(phy, MP_GLB_DIG_8C, FRC_FRC_PWR_ON); - mphy_clr_bit(phy, MP_GLB_DIG_8C, PLL_PWR_ON); + mtk_phy_set_bits(mmio + MP_GLB_DIG_8C, FRC_FRC_PWR_ON); + mtk_phy_clear_bits(mmio + MP_GLB_DIG_8C, PLL_PWR_ON); } static int ufs_mtk_phy_power_on(struct phy *generic_phy) diff --git a/drivers/phy/mediatek/phy-mtk-xsphy.c b/drivers/phy/mediatek/phy-mtk-xsphy.c index c0cdb78f77fabac1d12fc9cacb662496d4ca2657..b222fbbd71d18642cddbb4029cb1823510087853 100644 --- a/drivers/phy/mediatek/phy-mtk-xsphy.c +++ b/drivers/phy/mediatek/phy-mtk-xsphy.c @@ -37,7 +37,6 @@ #define XSP_U2FREQ_FMCR0 ((SSUSB_SIFSLV_U2FREQ) + 0x00) #define P2F_RG_FREQDET_EN BIT(24) #define P2F_RG_CYCLECNT GENMASK(23, 0) -#define P2F_RG_CYCLECNT_VAL(x) ((P2F_RG_CYCLECNT) & (x)) #define XSP_U2FREQ_MMONR0 ((SSUSB_SIFSLV_U2FREQ) + 0x0c) @@ -50,16 +49,12 @@ #define XSP_USBPHYACR1 ((SSUSB_SIFSLV_U2PHY_COM) + 0x04) #define P2A1_RG_INTR_CAL GENMASK(23, 19) -#define P2A1_RG_INTR_CAL_VAL(x) ((0x1f & (x)) << 19) #define P2A1_RG_VRT_SEL GENMASK(14, 12) -#define P2A1_RG_VRT_SEL_VAL(x) ((0x7 & (x)) << 12) #define P2A1_RG_TERM_SEL GENMASK(10, 8) -#define P2A1_RG_TERM_SEL_VAL(x) ((0x7 & (x)) << 8) #define XSP_USBPHYACR5 ((SSUSB_SIFSLV_U2PHY_COM) + 0x014) #define P2A5_RG_HSTX_SRCAL_EN BIT(15) #define P2A5_RG_HSTX_SRCTRL GENMASK(14, 12) -#define P2A5_RG_HSTX_SRCTRL_VAL(x) ((0x7 & (x)) << 12) #define XSP_USBPHYACR6 ((SSUSB_SIFSLV_U2PHY_COM) + 0x018) #define P2A6_RG_BC11_SW_EN BIT(23) @@ -74,15 +69,12 @@ #define SSPXTP_PHYA_GLB_00 ((SSPXTP_SIFSLV_PHYA_GLB) + 0x00) #define RG_XTP_GLB_BIAS_INTR_CTRL GENMASK(21, 16) -#define RG_XTP_GLB_BIAS_INTR_CTRL_VAL(x) ((0x3f & (x)) << 16) #define SSPXTP_PHYA_LN_04 ((SSPXTP_SIFSLV_PHYA_LN) + 0x04) #define RG_XTP_LN0_TX_IMPSEL GENMASK(4, 0) -#define RG_XTP_LN0_TX_IMPSEL_VAL(x) (0x1f & (x)) #define SSPXTP_PHYA_LN_14 ((SSPXTP_SIFSLV_PHYA_LN) + 0x014) #define RG_XTP_LN0_RX_IMPSEL GENMASK(4, 0) -#define RG_XTP_LN0_RX_IMPSEL_VAL(x) (0x1f & (x)) #define XSP_REF_CLK 26 /* MHZ */ #define XSP_SLEW_RATE_COEF 17 @@ -134,8 +126,8 @@ static void u2_phy_slew_rate_calibrate(struct mtk_xsphy *xsphy, mtk_phy_set_bits(pbase + XSP_U2FREQ_FMMONR1, P2F_RG_FRCK_EN); /* set cycle count as 1024 */ - mtk_phy_update_bits(pbase + XSP_U2FREQ_FMCR0, P2F_RG_CYCLECNT, - P2F_RG_CYCLECNT_VAL(XSP_FM_DET_CYCLE_CNT)); + mtk_phy_update_field(pbase + XSP_U2FREQ_FMCR0, P2F_RG_CYCLECNT, + XSP_FM_DET_CYCLE_CNT); /* enable frequency meter */ mtk_phy_set_bits(pbase + XSP_U2FREQ_FMCR0, P2F_RG_FREQDET_EN); @@ -166,8 +158,7 @@ static void u2_phy_slew_rate_calibrate(struct mtk_xsphy *xsphy, xsphy->src_ref_clk, xsphy->src_coef); /* set HS slew rate */ - mtk_phy_update_bits(pbase + XSP_USBPHYACR5, P2A5_RG_HSTX_SRCTRL, - P2A5_RG_HSTX_SRCTRL_VAL(calib_val)); + mtk_phy_update_field(pbase + XSP_USBPHYACR5, P2A5_RG_HSTX_SRCTRL, calib_val); /* disable USB ring oscillator */ mtk_phy_clear_bits(pbase + XSP_USBPHYACR5, P2A5_RG_HSTX_SRCAL_EN); @@ -280,20 +271,20 @@ static void u2_phy_props_set(struct mtk_xsphy *xsphy, void __iomem *pbase = inst->port_base; if (inst->efuse_intr) - mtk_phy_update_bits(pbase + XSP_USBPHYACR1, P2A1_RG_INTR_CAL, - P2A1_RG_INTR_CAL_VAL(inst->efuse_intr)); + mtk_phy_update_field(pbase + XSP_USBPHYACR1, P2A1_RG_INTR_CAL, + inst->efuse_intr); if (inst->eye_src) - mtk_phy_update_bits(pbase + XSP_USBPHYACR5, P2A5_RG_HSTX_SRCTRL, - P2A5_RG_HSTX_SRCTRL_VAL(inst->eye_src)); + mtk_phy_update_field(pbase + XSP_USBPHYACR5, P2A5_RG_HSTX_SRCTRL, + inst->eye_src); if (inst->eye_vrt) - mtk_phy_update_bits(pbase + XSP_USBPHYACR1, P2A1_RG_VRT_SEL, - P2A1_RG_VRT_SEL_VAL(inst->eye_vrt)); + mtk_phy_update_field(pbase + XSP_USBPHYACR1, P2A1_RG_VRT_SEL, + inst->eye_vrt); if (inst->eye_term) - mtk_phy_update_bits(pbase + XSP_USBPHYACR1, P2A1_RG_TERM_SEL, - P2A1_RG_TERM_SEL_VAL(inst->eye_term)); + mtk_phy_update_field(pbase + XSP_USBPHYACR1, P2A1_RG_TERM_SEL, + inst->eye_term); } static void u3_phy_props_set(struct mtk_xsphy *xsphy, @@ -302,19 +293,16 @@ static void u3_phy_props_set(struct mtk_xsphy *xsphy, void __iomem *pbase = inst->port_base; if (inst->efuse_intr) - mtk_phy_update_bits(xsphy->glb_base + SSPXTP_PHYA_GLB_00, - RG_XTP_GLB_BIAS_INTR_CTRL, - RG_XTP_GLB_BIAS_INTR_CTRL_VAL(inst->efuse_intr)); + mtk_phy_update_field(xsphy->glb_base + SSPXTP_PHYA_GLB_00, + RG_XTP_GLB_BIAS_INTR_CTRL, inst->efuse_intr); if (inst->efuse_tx_imp) - mtk_phy_update_bits(pbase + SSPXTP_PHYA_LN_04, - RG_XTP_LN0_TX_IMPSEL, - RG_XTP_LN0_TX_IMPSEL_VAL(inst->efuse_tx_imp)); + mtk_phy_update_field(pbase + SSPXTP_PHYA_LN_04, + RG_XTP_LN0_TX_IMPSEL, inst->efuse_tx_imp); if (inst->efuse_rx_imp) - mtk_phy_update_bits(pbase + SSPXTP_PHYA_LN_14, - RG_XTP_LN0_RX_IMPSEL, - RG_XTP_LN0_RX_IMPSEL_VAL(inst->efuse_rx_imp)); + mtk_phy_update_field(pbase + SSPXTP_PHYA_LN_14, + RG_XTP_LN0_RX_IMPSEL, inst->efuse_rx_imp); } static int mtk_phy_init(struct phy *phy) diff --git a/drivers/phy/microchip/lan966x_serdes.c b/drivers/phy/microchip/lan966x_serdes.c index e86a879b92b54cbebeb4b0391236f19ccdcf79d3..c1a41b6cd29b1d8f785134627547d68e38079747 100644 --- a/drivers/phy/microchip/lan966x_serdes.c +++ b/drivers/phy/microchip/lan966x_serdes.c @@ -42,7 +42,10 @@ #define SERDES_MUX_QSGMII(i, p, m, c) \ SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_QSGMII, m, c) #define SERDES_MUX_RGMII(i, p, m, c) \ - SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_RGMII, m, c) + SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_RGMII, m, c), \ + SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_RGMII_TXID, m, c), \ + SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_RGMII_RXID, m, c), \ + SERDES_MUX(i, p, PHY_MODE_ETHERNET, PHY_INTERFACE_MODE_RGMII_ID, m, c) static void lan_rmw_(u32 val, u32 mask, void __iomem *mem, u32 offset) { @@ -94,21 +97,29 @@ static const struct serdes_mux lan966x_serdes_muxes[] = { HSIO_HW_CFG_SD6G_1_CFG_SET(1)), SERDES_MUX_RGMII(RGMII(0), 2, HSIO_HW_CFG_RGMII_0_CFG | - HSIO_HW_CFG_RGMII_ENA, - HSIO_HW_CFG_RGMII_0_CFG_SET(BIT(0)) | - HSIO_HW_CFG_RGMII_ENA_SET(BIT(0))), + HSIO_HW_CFG_RGMII_ENA | + HSIO_HW_CFG_GMII_ENA, + HSIO_HW_CFG_RGMII_0_CFG_SET(0) | + HSIO_HW_CFG_RGMII_ENA_SET(BIT(0)) | + HSIO_HW_CFG_GMII_ENA_SET(BIT(2))), SERDES_MUX_RGMII(RGMII(1), 3, HSIO_HW_CFG_RGMII_1_CFG | - HSIO_HW_CFG_RGMII_ENA, - HSIO_HW_CFG_RGMII_1_CFG_SET(BIT(0)) | - HSIO_HW_CFG_RGMII_ENA_SET(BIT(1))), + HSIO_HW_CFG_RGMII_ENA | + HSIO_HW_CFG_GMII_ENA, + HSIO_HW_CFG_RGMII_1_CFG_SET(0) | + HSIO_HW_CFG_RGMII_ENA_SET(BIT(1)) | + HSIO_HW_CFG_GMII_ENA_SET(BIT(3))), SERDES_MUX_RGMII(RGMII(0), 5, HSIO_HW_CFG_RGMII_0_CFG | - HSIO_HW_CFG_RGMII_ENA, + HSIO_HW_CFG_RGMII_ENA | + HSIO_HW_CFG_GMII_ENA, HSIO_HW_CFG_RGMII_0_CFG_SET(BIT(0)) | - HSIO_HW_CFG_RGMII_ENA_SET(BIT(0))), + HSIO_HW_CFG_RGMII_ENA_SET(BIT(0)) | + HSIO_HW_CFG_GMII_ENA_SET(BIT(5))), SERDES_MUX_RGMII(RGMII(1), 6, HSIO_HW_CFG_RGMII_1_CFG | - HSIO_HW_CFG_RGMII_ENA, + HSIO_HW_CFG_RGMII_ENA | + HSIO_HW_CFG_GMII_ENA, HSIO_HW_CFG_RGMII_1_CFG_SET(BIT(0)) | - HSIO_HW_CFG_RGMII_ENA_SET(BIT(1))), + HSIO_HW_CFG_RGMII_ENA_SET(BIT(1)) | + HSIO_HW_CFG_GMII_ENA_SET(BIT(6))), }; struct serdes_ctrl { @@ -382,6 +393,67 @@ static int lan966x_sd6g40_setup(struct serdes_macro *macro, u32 idx, int mode) return lan966x_sd6g40_setup_lane(macro, conf, idx); } +static int lan966x_rgmii_setup(struct serdes_macro *macro, u32 idx, int mode) +{ + bool tx_delay = false; + bool rx_delay = false; + + /* Configure RGMII */ + lan_rmw(HSIO_RGMII_CFG_RGMII_RX_RST_SET(0) | + HSIO_RGMII_CFG_RGMII_TX_RST_SET(0) | + HSIO_RGMII_CFG_TX_CLK_CFG_SET(macro->speed == SPEED_1000 ? 1 : + macro->speed == SPEED_100 ? 2 : + macro->speed == SPEED_10 ? 3 : 0), + HSIO_RGMII_CFG_RGMII_RX_RST | + HSIO_RGMII_CFG_RGMII_TX_RST | + HSIO_RGMII_CFG_TX_CLK_CFG, + macro->ctrl->regs, HSIO_RGMII_CFG(idx)); + + if (mode == PHY_INTERFACE_MODE_RGMII || + mode == PHY_INTERFACE_MODE_RGMII_TXID) + rx_delay = true; + + if (mode == PHY_INTERFACE_MODE_RGMII || + mode == PHY_INTERFACE_MODE_RGMII_RXID) + tx_delay = true; + + /* Setup DLL configuration */ + lan_rmw(HSIO_DLL_CFG_DLL_RST_SET(0) | + HSIO_DLL_CFG_DLL_ENA_SET(rx_delay), + HSIO_DLL_CFG_DLL_RST | + HSIO_DLL_CFG_DLL_ENA, + macro->ctrl->regs, HSIO_DLL_CFG(idx == 0 ? 0x0 : 0x2)); + + lan_rmw(HSIO_DLL_CFG_DELAY_ENA_SET(rx_delay), + HSIO_DLL_CFG_DELAY_ENA, + macro->ctrl->regs, HSIO_DLL_CFG(idx == 0 ? 0x0 : 0x2)); + + lan_rmw(HSIO_DLL_CFG_DLL_RST_SET(0) | + HSIO_DLL_CFG_DLL_ENA_SET(tx_delay), + HSIO_DLL_CFG_DLL_RST | + HSIO_DLL_CFG_DLL_ENA, + macro->ctrl->regs, HSIO_DLL_CFG(idx == 0 ? 0x1 : 0x3)); + + lan_rmw(HSIO_DLL_CFG_DELAY_ENA_SET(tx_delay), + HSIO_DLL_CFG_DELAY_ENA, + macro->ctrl->regs, HSIO_DLL_CFG(idx == 0 ? 0x1 : 0x3)); + + return 0; +} + +static int serdes_set_speed(struct phy *phy, int speed) +{ + struct serdes_macro *macro = phy_get_drvdata(phy); + + if (!phy_interface_mode_is_rgmii(macro->mode)) + return 0; + + macro->speed = speed; + lan966x_rgmii_setup(macro, macro->idx - (SERDES6G_MAX + 1), macro->mode); + + return 0; +} + static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct serdes_macro *macro = phy_get_drvdata(phy); @@ -401,6 +473,9 @@ static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode) submode == PHY_INTERFACE_MODE_2500BASEX) submode = PHY_INTERFACE_MODE_SGMII; + if (submode == PHY_INTERFACE_MODE_QUSGMII) + submode = PHY_INTERFACE_MODE_QSGMII; + for (i = 0; i < ARRAY_SIZE(lan966x_serdes_muxes); i++) { if (macro->idx != lan966x_serdes_muxes[i].idx || mode != lan966x_serdes_muxes[i].mode || @@ -424,7 +499,9 @@ static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode) macro->mode); if (macro->idx < RGMII_MAX) - return 0; + return lan966x_rgmii_setup(macro, + macro->idx - (SERDES6G_MAX + 1), + macro->mode); return -EOPNOTSUPP; } @@ -434,6 +511,7 @@ static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode) static const struct phy_ops serdes_ops = { .set_mode = serdes_set_mode, + .set_speed = serdes_set_speed, .owner = THIS_MODULE, }; diff --git a/drivers/phy/microchip/lan966x_serdes_regs.h b/drivers/phy/microchip/lan966x_serdes_regs.h index ea30f64ffd5c8ef458bdf0de96721a45c816d167..ac54cd01fea658eee386d55671171d724fd89509 100644 --- a/drivers/phy/microchip/lan966x_serdes_regs.h +++ b/drivers/phy/microchip/lan966x_serdes_regs.h @@ -206,4 +206,46 @@ enum lan966x_target { #define HSIO_HW_CFG_QSGMII_ENA_GET(x)\ FIELD_GET(HSIO_HW_CFG_QSGMII_ENA, x) +/* HSIO:HW_CFGSTAT:RGMII_CFG */ +#define HSIO_RGMII_CFG(r) __REG(TARGET_HSIO, 0, 1, 104, 0, 1, 52, 20, r, 2, 4) + +#define HSIO_RGMII_CFG_TX_CLK_CFG GENMASK(4, 2) +#define HSIO_RGMII_CFG_TX_CLK_CFG_SET(x)\ + FIELD_PREP(HSIO_RGMII_CFG_TX_CLK_CFG, x) +#define HSIO_RGMII_CFG_TX_CLK_CFG_GET(x)\ + FIELD_GET(HSIO_RGMII_CFG_TX_CLK_CFG, x) + +#define HSIO_RGMII_CFG_RGMII_TX_RST BIT(1) +#define HSIO_RGMII_CFG_RGMII_TX_RST_SET(x)\ + FIELD_PREP(HSIO_RGMII_CFG_RGMII_TX_RST, x) +#define HSIO_RGMII_CFG_RGMII_TX_RST_GET(x)\ + FIELD_GET(HSIO_RGMII_CFG_RGMII_TX_RST, x) + +#define HSIO_RGMII_CFG_RGMII_RX_RST BIT(0) +#define HSIO_RGMII_CFG_RGMII_RX_RST_SET(x)\ + FIELD_PREP(HSIO_RGMII_CFG_RGMII_RX_RST, x) +#define HSIO_RGMII_CFG_RGMII_RX_RST_GET(x)\ + FIELD_GET(HSIO_RGMII_CFG_RGMII_RX_RST, x) + +/* HSIO:HW_CFGSTAT:DLL_CFG */ +#define HSIO_DLL_CFG(r) __REG(TARGET_HSIO, 0, 1, 104, 0, 1, 52, 36, r, 4, 4) + +#define HSIO_DLL_CFG_DELAY_ENA BIT(2) +#define HSIO_DLL_CFG_DELAY_ENA_SET(x)\ + FIELD_PREP(HSIO_DLL_CFG_DELAY_ENA, x) +#define HSIO_DLL_CFG_DELAY_ENA_GET(x)\ + FIELD_GET(HSIO_DLL_CFG_DELAY_ENA, x) + +#define HSIO_DLL_CFG_DLL_ENA BIT(1) +#define HSIO_DLL_CFG_DLL_ENA_SET(x)\ + FIELD_PREP(HSIO_DLL_CFG_DLL_ENA, x) +#define HSIO_DLL_CFG_DLL_ENA_GET(x)\ + FIELD_GET(HSIO_DLL_CFG_DLL_ENA, x) + +#define HSIO_DLL_CFG_DLL_RST BIT(0) +#define HSIO_DLL_CFG_DLL_RST_SET(x)\ + FIELD_PREP(HSIO_DLL_CFG_DLL_RST, x) +#define HSIO_DLL_CFG_DLL_RST_GET(x)\ + FIELD_GET(HSIO_DLL_CFG_DLL_RST, x) + #endif /* _LAN966X_HSIO_REGS_H_ */ diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c index 7e35707898456f97016940cc10c155b09e3b866e..fc8ca0f3018df560d9332bba0bb01925612b707b 100644 --- a/drivers/phy/qualcomm/phy-qcom-edp.c +++ b/drivers/phy/qualcomm/phy-qcom-edp.c @@ -70,8 +70,19 @@ #define TXn_TRAN_DRVR_EMP_EN 0x0078 +struct qcom_edp_cfg { + bool is_dp; + + /* DP PHY swing and pre_emphasis tables */ + const u8 (*swing_hbr_rbr)[4][4]; + const u8 (*swing_hbr3_hbr2)[4][4]; + const u8 (*pre_emphasis_hbr_rbr)[4][4]; + const u8 (*pre_emphasis_hbr3_hbr2)[4][4]; +}; + struct qcom_edp { struct device *dev; + const struct qcom_edp_cfg *cfg; struct phy *phy; @@ -89,10 +100,84 @@ struct qcom_edp { struct regulator_bulk_data supplies[2]; }; +static const u8 dp_swing_hbr_rbr[4][4] = { + { 0x08, 0x0f, 0x16, 0x1f }, + { 0x11, 0x1e, 0x1f, 0xff }, + { 0x16, 0x1f, 0xff, 0xff }, + { 0x1f, 0xff, 0xff, 0xff } +}; + +static const u8 dp_pre_emp_hbr_rbr[4][4] = { + { 0x00, 0x0d, 0x14, 0x1a }, + { 0x00, 0x0e, 0x15, 0xff }, + { 0x00, 0x0e, 0xff, 0xff }, + { 0x03, 0xff, 0xff, 0xff } +}; + +static const u8 dp_swing_hbr2_hbr3[4][4] = { + { 0x02, 0x12, 0x16, 0x1a }, + { 0x09, 0x19, 0x1f, 0xff }, + { 0x10, 0x1f, 0xff, 0xff }, + { 0x1f, 0xff, 0xff, 0xff } +}; + +static const u8 dp_pre_emp_hbr2_hbr3[4][4] = { + { 0x00, 0x0c, 0x15, 0x1b }, + { 0x02, 0x0e, 0x16, 0xff }, + { 0x02, 0x11, 0xff, 0xff }, + { 0x04, 0xff, 0xff, 0xff } +}; + +static const struct qcom_edp_cfg dp_phy_cfg = { + .is_dp = true, + .swing_hbr_rbr = &dp_swing_hbr_rbr, + .swing_hbr3_hbr2 = &dp_swing_hbr2_hbr3, + .pre_emphasis_hbr_rbr = &dp_pre_emp_hbr_rbr, + .pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3, +}; + +static const u8 edp_swing_hbr_rbr[4][4] = { + { 0x07, 0x0f, 0x16, 0x1f }, + { 0x0d, 0x16, 0x1e, 0xff }, + { 0x11, 0x1b, 0xff, 0xff }, + { 0x16, 0xff, 0xff, 0xff } +}; + +static const u8 edp_pre_emp_hbr_rbr[4][4] = { + { 0x05, 0x12, 0x17, 0x1d }, + { 0x05, 0x11, 0x18, 0xff }, + { 0x06, 0x11, 0xff, 0xff }, + { 0x00, 0xff, 0xff, 0xff } +}; + +static const u8 edp_swing_hbr2_hbr3[4][4] = { + { 0x0b, 0x11, 0x17, 0x1c }, + { 0x10, 0x19, 0x1f, 0xff }, + { 0x19, 0x1f, 0xff, 0xff }, + { 0x1f, 0xff, 0xff, 0xff } +}; + +static const u8 edp_pre_emp_hbr2_hbr3[4][4] = { + { 0x08, 0x11, 0x17, 0x1b }, + { 0x00, 0x0c, 0x13, 0xff }, + { 0x05, 0x10, 0xff, 0xff }, + { 0x00, 0xff, 0xff, 0xff } +}; + +static const struct qcom_edp_cfg edp_phy_cfg = { + .is_dp = false, + .swing_hbr_rbr = &edp_swing_hbr_rbr, + .swing_hbr3_hbr2 = &edp_swing_hbr2_hbr3, + .pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr, + .pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3, +}; + static int qcom_edp_phy_init(struct phy *phy) { struct qcom_edp *edp = phy_get_drvdata(phy); + const struct qcom_edp_cfg *cfg = edp->cfg; int ret; + u8 cfg8; ret = regulator_bulk_enable(ARRAY_SIZE(edp->supplies), edp->supplies); if (ret) @@ -117,6 +202,13 @@ static int qcom_edp_phy_init(struct phy *phy) DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, edp->edp + DP_PHY_PD_CTL); + if (cfg && cfg->is_dp) + cfg8 = 0xb7; + else + cfg8 = 0x37; + + writel(0xfc, edp->edp + DP_PHY_MODE); + writel(0x00, edp->edp + DP_PHY_AUX_CFG0); writel(0x13, edp->edp + DP_PHY_AUX_CFG1); writel(0x24, edp->edp + DP_PHY_AUX_CFG2); @@ -125,7 +217,7 @@ static int qcom_edp_phy_init(struct phy *phy) writel(0x26, edp->edp + DP_PHY_AUX_CFG5); writel(0x0a, edp->edp + DP_PHY_AUX_CFG6); writel(0x03, edp->edp + DP_PHY_AUX_CFG7); - writel(0x37, edp->edp + DP_PHY_AUX_CFG8); + writel(cfg8, edp->edp + DP_PHY_AUX_CFG8); writel(0x03, edp->edp + DP_PHY_AUX_CFG9); writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | @@ -142,14 +234,60 @@ out_disable_supplies: return ret; } +static int qcom_edp_set_voltages(struct qcom_edp *edp, const struct phy_configure_opts_dp *dp_opts) +{ + const struct qcom_edp_cfg *cfg = edp->cfg; + unsigned int v_level = 0; + unsigned int p_level = 0; + u8 ldo_config; + u8 swing; + u8 emph; + int i; + + if (!cfg) + return 0; + + for (i = 0; i < dp_opts->lanes; i++) { + v_level = max(v_level, dp_opts->voltage[i]); + p_level = max(p_level, dp_opts->pre[i]); + } + + if (dp_opts->link_rate <= 2700) { + swing = (*cfg->swing_hbr_rbr)[v_level][p_level]; + emph = (*cfg->pre_emphasis_hbr_rbr)[v_level][p_level]; + } else { + swing = (*cfg->swing_hbr3_hbr2)[v_level][p_level]; + emph = (*cfg->pre_emphasis_hbr3_hbr2)[v_level][p_level]; + } + + if (swing == 0xff || emph == 0xff) + return -EINVAL; + + ldo_config = (cfg && cfg->is_dp) ? 0x1 : 0x0; + + writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG); + writel(swing, edp->tx0 + TXn_TX_DRV_LVL); + writel(emph, edp->tx0 + TXn_TX_EMP_POST1_LVL); + + writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG); + writel(swing, edp->tx1 + TXn_TX_DRV_LVL); + writel(emph, edp->tx1 + TXn_TX_EMP_POST1_LVL); + + return 0; +} + static int qcom_edp_phy_configure(struct phy *phy, union phy_configure_opts *opts) { const struct phy_configure_opts_dp *dp_opts = &opts->dp; struct qcom_edp *edp = phy_get_drvdata(phy); + int ret = 0; memcpy(&edp->dp_opts, dp_opts, sizeof(*dp_opts)); - return 0; + if (dp_opts->set_voltages) + ret = qcom_edp_set_voltages(edp, dp_opts); + + return ret; } static int qcom_edp_configure_ssc(const struct qcom_edp *edp) @@ -272,31 +410,30 @@ static int qcom_edp_configure_pll(const struct qcom_edp *edp) return 0; } -static int qcom_edp_set_vco_div(const struct qcom_edp *edp) +static int qcom_edp_set_vco_div(const struct qcom_edp *edp, unsigned long *pixel_freq) { const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts; - unsigned long pixel_freq; u32 vco_div; switch (dp_opts->link_rate) { case 1620: vco_div = 0x1; - pixel_freq = 1620000000UL / 2; + *pixel_freq = 1620000000UL / 2; break; case 2700: vco_div = 0x1; - pixel_freq = 2700000000UL / 2; + *pixel_freq = 2700000000UL / 2; break; case 5400: vco_div = 0x2; - pixel_freq = 5400000000UL / 4; + *pixel_freq = 5400000000UL / 4; break; case 8100: vco_div = 0x0; - pixel_freq = 8100000000UL / 6; + *pixel_freq = 8100000000UL / 6; break; default: @@ -306,18 +443,20 @@ static int qcom_edp_set_vco_div(const struct qcom_edp *edp) writel(vco_div, edp->edp + DP_PHY_VCO_DIV); - clk_set_rate(edp->dp_link_hw.clk, dp_opts->link_rate * 100000); - clk_set_rate(edp->dp_pixel_hw.clk, pixel_freq); - return 0; } static int qcom_edp_phy_power_on(struct phy *phy) { const struct qcom_edp *edp = phy_get_drvdata(phy); + const struct qcom_edp_cfg *cfg = edp->cfg; + u32 bias0_en, drvr0_en, bias1_en, drvr1_en; + unsigned long pixel_freq; + u8 ldo_config; int timeout; int ret; u32 val; + u8 cfg1; writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN | @@ -330,8 +469,11 @@ static int qcom_edp_phy_power_on(struct phy *phy) if (timeout) return timeout; - writel(0x01, edp->tx0 + TXn_LDO_CONFIG); - writel(0x01, edp->tx1 + TXn_LDO_CONFIG); + + ldo_config = (cfg && cfg->is_dp) ? 0x1 : 0x0; + + writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG); + writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG); writel(0x00, edp->tx0 + TXn_LANE_MODE_1); writel(0x00, edp->tx1 + TXn_LANE_MODE_1); @@ -363,7 +505,7 @@ static int qcom_edp_phy_power_on(struct phy *phy) writel(0x01, edp->tx1 + TXn_TRAN_DRVR_EMP_EN); writel(0x04, edp->tx1 + TXn_TX_BAND); - ret = qcom_edp_set_vco_div(edp); + ret = qcom_edp_set_vco_div(edp, &pixel_freq); if (ret) return ret; @@ -398,19 +540,46 @@ static int qcom_edp_phy_power_on(struct phy *phy) writel(0x1f, edp->tx0 + TXn_TX_DRV_LVL); writel(0x1f, edp->tx1 + TXn_TX_DRV_LVL); - writel(0x4, edp->tx0 + TXn_HIGHZ_DRVR_EN); - writel(0x3, edp->tx0 + TXn_TRANSCEIVER_BIAS_EN); - writel(0x4, edp->tx1 + TXn_HIGHZ_DRVR_EN); - writel(0x0, edp->tx1 + TXn_TRANSCEIVER_BIAS_EN); - writel(0x3, edp->edp + DP_PHY_CFG_1); + if (edp->dp_opts.lanes == 1) { + bias0_en = 0x01; + bias1_en = 0x00; + drvr0_en = 0x06; + drvr1_en = 0x07; + cfg1 = 0x1; + } else if (edp->dp_opts.lanes == 2) { + bias0_en = 0x03; + bias1_en = 0x00; + drvr0_en = 0x04; + drvr1_en = 0x07; + cfg1 = 0x3; + } else { + bias0_en = 0x03; + bias1_en = 0x03; + drvr0_en = 0x04; + drvr1_en = 0x04; + cfg1 = 0xf; + } + + writel(drvr0_en, edp->tx0 + TXn_HIGHZ_DRVR_EN); + writel(bias0_en, edp->tx0 + TXn_TRANSCEIVER_BIAS_EN); + writel(drvr1_en, edp->tx1 + TXn_HIGHZ_DRVR_EN); + writel(bias1_en, edp->tx1 + TXn_TRANSCEIVER_BIAS_EN); + writel(cfg1, edp->edp + DP_PHY_CFG_1); writel(0x18, edp->edp + DP_PHY_CFG); usleep_range(100, 1000); writel(0x19, edp->edp + DP_PHY_CFG); - return readl_poll_timeout(edp->edp + DP_PHY_STATUS, - val, val & BIT(1), 500, 10000); + ret = readl_poll_timeout(edp->edp + DP_PHY_STATUS, + val, val & BIT(1), 500, 10000); + if (ret) + return ret; + + clk_set_rate(edp->dp_link_hw.clk, edp->dp_opts.link_rate * 100000); + clk_set_rate(edp->dp_pixel_hw.clk, pixel_freq); + + return 0; } static int qcom_edp_phy_power_off(struct phy *phy) @@ -571,21 +740,24 @@ static int qcom_edp_clks_register(struct qcom_edp *edp, struct device_node *np) { struct clk_hw_onecell_data *data; struct clk_init_data init = { }; + char name[64]; int ret; data = devm_kzalloc(edp->dev, struct_size(data, hws, 2), GFP_KERNEL); if (!data) return -ENOMEM; + snprintf(name, sizeof(name), "%s::link_clk", dev_name(edp->dev)); init.ops = &qcom_edp_dp_link_clk_ops; - init.name = "edp_phy_pll_link_clk"; + init.name = name; edp->dp_link_hw.init = &init; ret = devm_clk_hw_register(edp->dev, &edp->dp_link_hw); if (ret) return ret; + snprintf(name, sizeof(name), "%s::vco_div_clk", dev_name(edp->dev)); init.ops = &qcom_edp_dp_pixel_clk_ops; - init.name = "edp_phy_pll_vco_div_clk"; + init.name = name; edp->dp_pixel_hw.init = &init; ret = devm_clk_hw_register(edp->dev, &edp->dp_pixel_hw); if (ret) @@ -610,6 +782,7 @@ static int qcom_edp_phy_probe(struct platform_device *pdev) return -ENOMEM; edp->dev = dev; + edp->cfg = of_device_get_match_data(&pdev->dev); edp->edp = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(edp->edp)) @@ -670,6 +843,8 @@ static int qcom_edp_phy_probe(struct platform_device *pdev) static const struct of_device_id qcom_edp_phy_match_table[] = { { .compatible = "qcom,sc7280-edp-phy" }, { .compatible = "qcom,sc8180x-edp-phy" }, + { .compatible = "qcom,sc8280xp-dp-phy", .data = &dp_phy_cfg }, + { .compatible = "qcom,sc8280xp-edp-phy", .data = &edp_phy_cfg }, { } }; MODULE_DEVICE_TABLE(of, qcom_edp_phy_match_table); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index 4b182897610446ef3adaa619c538e14f06c7c13f..9807c4d935cdb4a482a8213d1196b58ce620890c 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -28,16 +28,11 @@ #define SW_RESET BIT(0) /* QPHY_POWER_DOWN_CONTROL */ #define SW_PWRDN BIT(0) -#define REFCLK_DRV_DSBL BIT(1) /* QPHY_START_CONTROL bits */ #define SERDES_START BIT(0) #define PCS_START BIT(1) -#define PLL_READY_GATE_EN BIT(3) /* QPHY_PCS_STATUS bit */ #define PHYSTATUS BIT(6) -#define PHYSTATUS_4_20 BIT(7) -/* QPHY_PCS_READY_STATUS & QPHY_COM_PCS_READY_STATUS bit */ -#define PCS_READY BIT(0) /* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ /* DP PHY soft reset */ @@ -71,11 +66,6 @@ #define POWER_DOWN_DELAY_US_MIN 10 #define POWER_DOWN_DELAY_US_MAX 11 -#define MAX_PROP_NAME 32 - -/* Define the assumed distance between lanes for underspecified device trees. */ -#define QMP_PHY_LEGACY_LANE_STRIDE 0x400 - struct qmp_phy_init_tbl { unsigned int offset; unsigned int val; @@ -115,22 +105,14 @@ struct qmp_phy_init_tbl { /* set of registers with offsets different per-PHY */ enum qphy_reg_layout { - /* Common block control registers */ - QPHY_COM_SW_RESET, - QPHY_COM_POWER_DOWN_CONTROL, - QPHY_COM_START_CONTROL, - QPHY_COM_PCS_READY_STATUS, /* PCS registers */ QPHY_SW_RESET, QPHY_START_CTRL, - QPHY_PCS_READY_STATUS, QPHY_PCS_STATUS, QPHY_PCS_AUTONOMOUS_MODE_CTRL, QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, QPHY_PCS_LFPS_RXTERM_IRQ_STATUS, QPHY_PCS_POWER_DOWN_CONTROL, - /* PCS_MISC registers */ - QPHY_PCS_MISC_TYPEC_CTRL, /* Keep last to ensure regs_layout arrays are properly initialized */ QPHY_LAYOUT_SIZE }; @@ -606,6 +588,160 @@ static const struct qmp_phy_init_tbl qmp_v4_dp_tx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V4_TX_TX_EMP_POST1_LVL, 0x20), }; +static const struct qmp_phy_init_tbl qmp_v5_dp_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V4_COM_SVS_MODE_CLK_SEL, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_SYSCLK_EN_SEL, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_SYS_CLK_CTRL, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CLK_ENABLE1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_SYSCLK_BUF_ENABLE, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CLK_SELECT, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_CCTRL_MODE1, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_PLL_RCTRL_MODE1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CP_CTRL_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CP_CTRL_MODE1, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CMN_CONFIG, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_INTEGLOOP_GAIN0_MODE0, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_INTEGLOOP_GAIN1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE_MAP, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_BG_TIMER, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CORECLK_DIV_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_BIAS_EN_CLKBUFLR_EN, 0x17), + QMP_PHY_INIT_CFG(QSERDES_V4_COM_CORE_CLK_EN, 0x1f), +}; + +static const struct qmp_phy_init_tbl qmp_v5_5nm_dp_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_LANE_MODE_3, 0x51), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_TRANSCEIVER_BIAS_EN, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_VMODE_CTRL1, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_PRE_STALL_LDO_BOOST_EN, 0x0), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_INTERFACE_SELECT, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_CLKBUF_ENABLE, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_RESET_TSYNC_EN, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_TRAN_DRVR_EMP_EN, 0xf), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_PARRATE_REC_DETECT_IDLE_EN, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_TX, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_RX, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_TX_BAND, 0x01), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb43dp_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_EN_CENTER, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_PER1, 0x31), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_PER2, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE1_MODE0, 0xfd), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE0, 0x0d), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE1_MODE1, 0xfd), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE1, 0x0d), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_BUF_ENABLE, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE1, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_EN_SEL, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP_EN, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE0, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE0, 0x34), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE1, 0x34), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE1, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE0, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MSB_MODE0, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MSB_MODE1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START1_MODE0, 0x55), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START2_MODE0, 0xd5), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START3_MODE0, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START1_MODE1, 0x55), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START2_MODE1, 0xd5), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START3_MODE1, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE1_MODE0, 0xd4), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE1_MODE1, 0xd4), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE2_MODE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_SEL, 0x13), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_HS_SWITCH_SEL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CORECLK_DIV_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CORECLK_DIV_MODE1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CORE_CLK_EN, 0x60), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CMN_CONFIG, 0x76), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_IVCO, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_INTEGLOOP_GAIN0_MODE0, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_INTEGLOOP_GAIN0_MODE1, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_INITVAL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAXVAL2, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SVS_MODE_CLK_SEL, 0x0a), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb43dp_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_LANE_MODE_1, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_LANE_MODE_2, 0xc2), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_LANE_MODE_3, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_TX, 0x1f), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_RX, 0x0a), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb43dp_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_SIGDET_ENABLES, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B0, 0xd2), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B1, 0xd2), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B2, 0xdb), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B3, 0x21), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B4, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B5, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B6, 0x45), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B7, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B0, 0x6b), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B1, 0x63), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B2, 0xb6), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B3, 0x23), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B4, 0x35), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B5, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B6, 0x8e), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_MODE_RATE2_B7, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_IVCM_CAL_CODE_OVERRIDE, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_IVCM_CAL_CTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_SUMMER_CAL_SPD_MODE, 0x1b), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_UCDR_PI_CONTROLS, 0x15), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_UCDR_SB2_GAIN2_RATE2, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_RX_IVCM_POSTCAL_OFFSET, 0x7c), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_VGA_CAL_CNTRL1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_VGA_CAL_MAN_VAL, 0x0d), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_DFE_DAC_ENABLE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_DFE_3, 0x45), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_GM_CAL, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_UCDR_FO_GAIN_RATE2, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_UCDR_SO_GAIN_RATE2, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_5NM_RX_Q_PI_INTRINSIC_BIAS_RATE32, 0x3f), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb43dp_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_REFGEN_REQ_CONFIG1, 0x21), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RX_CONFIG, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07), +}; /* list of regulators */ struct qmp_regulator_data { @@ -618,14 +754,69 @@ static struct qmp_regulator_data qmp_phy_vreg_l[] = { { .name = "vdda-pll", .enable_load = 36000 }, }; +static const u8 qmp_dp_v3_pre_emphasis_hbr3_hbr2[4][4] = { + { 0x00, 0x0c, 0x15, 0x1a }, + { 0x02, 0x0e, 0x16, 0xff }, + { 0x02, 0x11, 0xff, 0xff }, + { 0x04, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v3_voltage_swing_hbr3_hbr2[4][4] = { + { 0x02, 0x12, 0x16, 0x1a }, + { 0x09, 0x19, 0x1f, 0xff }, + { 0x10, 0x1f, 0xff, 0xff }, + { 0x1f, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v3_pre_emphasis_hbr_rbr[4][4] = { + { 0x00, 0x0c, 0x14, 0x19 }, + { 0x00, 0x0b, 0x12, 0xff }, + { 0x00, 0x0b, 0xff, 0xff }, + { 0x04, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v3_voltage_swing_hbr_rbr[4][4] = { + { 0x08, 0x0f, 0x16, 0x1f }, + { 0x11, 0x1e, 0x1f, 0xff }, + { 0x19, 0x1f, 0xff, 0xff }, + { 0x1f, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v5_pre_emphasis_hbr3_hbr2[4][4] = { + { 0x20, 0x2c, 0x35, 0x3b }, + { 0x22, 0x2e, 0x36, 0xff }, + { 0x22, 0x31, 0xff, 0xff }, + { 0x24, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v5_voltage_swing_hbr3_hbr2[4][4] = { + { 0x22, 0x32, 0x36, 0x3a }, + { 0x29, 0x39, 0x3f, 0xff }, + { 0x30, 0x3f, 0xff, 0xff }, + { 0x3f, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v5_pre_emphasis_hbr_rbr[4][4] = { + { 0x20, 0x2d, 0x34, 0x3a }, + { 0x20, 0x2e, 0x35, 0xff }, + { 0x20, 0x2e, 0xff, 0xff }, + { 0x24, 0xff, 0xff, 0xff } +}; + +static const u8 qmp_dp_v5_voltage_swing_hbr_rbr[4][4] = { + { 0x28, 0x2f, 0x36, 0x3f }, + { 0x31, 0x3e, 0x3f, 0xff }, + { 0x36, 0x3f, 0xff, 0xff }, + { 0x3f, 0xff, 0xff, 0xff } +}; + struct qmp_phy; /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { /* phy-type - PCIE/UFS/USB */ unsigned int type; - /* number of lanes provided by phy */ - int nlanes; + int lanes; /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; @@ -649,6 +840,12 @@ struct qmp_phy_cfg { const struct qmp_phy_init_tbl *serdes_tbl_hbr3; int serdes_tbl_hbr3_num; + /* DP PHY swing and pre_emphasis tables */ + const u8 (*swing_hbr_rbr)[4][4]; + const u8 (*swing_hbr3_hbr2)[4][4]; + const u8 (*pre_emphasis_hbr_rbr)[4][4]; + const u8 (*pre_emphasis_hbr3_hbr2)[4][4]; + /* DP PHY callbacks */ int (*configure_dp_phy)(struct qmp_phy *qphy); void (*configure_dp_tx)(struct qmp_phy *qphy); @@ -679,11 +876,6 @@ struct qmp_phy_cfg { int pwrdn_delay_min; int pwrdn_delay_max; - /* true, if PHY has a separate DP_COM control block */ - bool has_phy_dp_com_ctrl; - /* true, if PHY has secondary tx/rx lanes to be configured */ - bool is_dual_lane_phy; - /* Offset from PCS to PCS_USB region */ unsigned int pcs_usb_offset; @@ -708,9 +900,7 @@ struct qmp_phy_combo_cfg { * @pcs_misc: iomapped memory space for lane's pcs_misc * @pcs_usb: iomapped memory space for lane's pcs_usb * @pipe_clk: pipe clock - * @index: lane index * @qmp: QMP phy to which this lane belongs - * @lane_rst: lane's reset controller * @mode: current PHY mode * @dp_aux_cfg: Display port aux config * @dp_opts: Display port optional config @@ -728,9 +918,7 @@ struct qmp_phy { void __iomem *pcs_misc; void __iomem *pcs_usb; struct clk *pipe_clk; - unsigned int index; struct qcom_qmp *qmp; - struct reset_control *lane_rst; enum phy_mode mode; unsigned int dp_aux_cfg; struct phy_configure_opts_dp dp_opts; @@ -784,6 +972,8 @@ static void qcom_qmp_v4_phy_configure_dp_tx(struct qmp_phy *qphy); static int qcom_qmp_v4_phy_configure_dp_phy(struct qmp_phy *qphy); static int qcom_qmp_v4_dp_phy_calibrate(struct qmp_phy *qphy); +static int qcom_qmp_v5_phy_configure_dp_phy(struct qmp_phy *qphy); + static inline void qphy_setbits(void __iomem *base, u32 offset, u32 val) { u32 reg; @@ -833,7 +1023,7 @@ static const char * const sc7180_usb3phy_reset_l[] = { static const struct qmp_phy_cfg sc7180_usb3phy_cfg = { .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v3_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v3_usb3_serdes_tbl), @@ -858,14 +1048,11 @@ static const struct qmp_phy_cfg sc7180_usb3phy_cfg = { .has_pwrdn_delay = true, .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, - - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sc7180_dpphy_cfg = { .type = PHY_TYPE_DP, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v3_dp_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v3_dp_serdes_tbl), @@ -881,6 +1068,11 @@ static const struct qmp_phy_cfg sc7180_dpphy_cfg = { .serdes_tbl_hbr3 = qmp_v3_dp_serdes_tbl_hbr3, .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v3_dp_serdes_tbl_hbr3), + .swing_hbr_rbr = &qmp_dp_v3_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v3_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v3_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v3_pre_emphasis_hbr3_hbr2, + .clk_list = qmp_v3_phy_clk_l, .num_clks = ARRAY_SIZE(qmp_v3_phy_clk_l), .reset_list = sc7180_usb3phy_reset_l, @@ -889,9 +1081,6 @@ static const struct qmp_phy_cfg sc7180_dpphy_cfg = { .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), .regs = qmp_v3_usb3phy_regs_layout, - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, - .dp_aux_init = qcom_qmp_v3_phy_dp_aux_init, .configure_dp_tx = qcom_qmp_v3_phy_configure_dp_tx, .configure_dp_phy = qcom_qmp_v3_phy_configure_dp_phy, @@ -903,9 +1092,43 @@ static const struct qmp_phy_combo_cfg sc7180_usb3dpphy_cfg = { .dp_cfg = &sc7180_dpphy_cfg, }; +static const struct qmp_phy_cfg sdm845_usb3phy_cfg = { + .type = PHY_TYPE_USB3, + .lanes = 2, + + .serdes_tbl = qmp_v3_usb3_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(qmp_v3_usb3_serdes_tbl), + .tx_tbl = qmp_v3_usb3_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(qmp_v3_usb3_tx_tbl), + .rx_tbl = qmp_v3_usb3_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(qmp_v3_usb3_rx_tbl), + .pcs_tbl = qmp_v3_usb3_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(qmp_v3_usb3_pcs_tbl), + .clk_list = qmp_v3_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v3_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v3_usb3phy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + .phy_status = PHYSTATUS, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + +static const struct qmp_phy_combo_cfg sdm845_usb3dpphy_cfg = { + .usb_cfg = &sdm845_usb3phy_cfg, + .dp_cfg = &sc7180_dpphy_cfg, +}; + static const struct qmp_phy_cfg sm8150_usb3phy_cfg = { .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = sm8150_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), @@ -934,14 +1157,11 @@ static const struct qmp_phy_cfg sm8150_usb3phy_cfg = { .has_pwrdn_delay = true, .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, - - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sc8180x_dpphy_cfg = { .type = PHY_TYPE_DP, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v4_dp_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl), @@ -957,6 +1177,11 @@ static const struct qmp_phy_cfg sc8180x_dpphy_cfg = { .serdes_tbl_hbr3 = qmp_v4_dp_serdes_tbl_hbr3, .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_hbr3), + .swing_hbr_rbr = &qmp_dp_v3_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v3_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v3_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v3_pre_emphasis_hbr3_hbr2, + .clk_list = qmp_v3_phy_clk_l, .num_clks = ARRAY_SIZE(qmp_v3_phy_clk_l), .reset_list = sc7180_usb3phy_reset_l, @@ -965,9 +1190,6 @@ static const struct qmp_phy_cfg sc8180x_dpphy_cfg = { .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), .regs = qmp_v3_usb3phy_regs_layout, - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, - .dp_aux_init = qcom_qmp_v4_phy_dp_aux_init, .configure_dp_tx = qcom_qmp_v4_phy_configure_dp_tx, .configure_dp_phy = qcom_qmp_v4_phy_configure_dp_phy, @@ -979,9 +1201,81 @@ static const struct qmp_phy_combo_cfg sc8180x_usb3dpphy_cfg = { .dp_cfg = &sc8180x_dpphy_cfg, }; +static const struct qmp_phy_cfg sc8280xp_usb43dp_usb_cfg = { + .type = PHY_TYPE_USB3, + .lanes = 2, + + .serdes_tbl = sc8280xp_usb43dp_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sc8280xp_usb43dp_serdes_tbl), + .tx_tbl = sc8280xp_usb43dp_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sc8280xp_usb43dp_tx_tbl), + .rx_tbl = sc8280xp_usb43dp_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sc8280xp_usb43dp_rx_tbl), + .pcs_tbl = sc8280xp_usb43dp_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sc8280xp_usb43dp_pcs_tbl), + .clk_list = qmp_v4_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + .pcs_usb_offset = 0x300, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + .phy_status = PHYSTATUS, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, +}; + +static const struct qmp_phy_cfg sc8280xp_usb43dp_dp_cfg = { + .type = PHY_TYPE_DP, + .lanes = 2, + + .serdes_tbl = qmp_v5_dp_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(qmp_v5_dp_serdes_tbl), + .tx_tbl = qmp_v5_5nm_dp_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(qmp_v5_5nm_dp_tx_tbl), + + .serdes_tbl_rbr = qmp_v4_dp_serdes_tbl_rbr, + .serdes_tbl_rbr_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_rbr), + .serdes_tbl_hbr = qmp_v4_dp_serdes_tbl_hbr, + .serdes_tbl_hbr_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_hbr), + .serdes_tbl_hbr2 = qmp_v4_dp_serdes_tbl_hbr2, + .serdes_tbl_hbr2_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_hbr2), + .serdes_tbl_hbr3 = qmp_v4_dp_serdes_tbl_hbr3, + .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_hbr3), + + .swing_hbr_rbr = &qmp_dp_v5_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v5_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v5_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v5_pre_emphasis_hbr3_hbr2, + + .clk_list = qmp_v4_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + + .dp_aux_init = qcom_qmp_v4_phy_dp_aux_init, + .configure_dp_tx = qcom_qmp_v4_phy_configure_dp_tx, + .configure_dp_phy = qcom_qmp_v5_phy_configure_dp_phy, + .calibrate_dp_phy = qcom_qmp_v4_dp_phy_calibrate, +}; + +static const struct qmp_phy_combo_cfg sc8280xp_usb43dpphy_combo_cfg = { + .usb_cfg = &sc8280xp_usb43dp_usb_cfg, + .dp_cfg = &sc8280xp_usb43dp_dp_cfg, +}; + static const struct qmp_phy_cfg sm8250_usb3phy_cfg = { .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = sm8150_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), @@ -1009,14 +1303,11 @@ static const struct qmp_phy_cfg sm8250_usb3phy_cfg = { .has_pwrdn_delay = true, .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, - - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8250_dpphy_cfg = { .type = PHY_TYPE_DP, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v4_dp_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl), @@ -1032,6 +1323,11 @@ static const struct qmp_phy_cfg sm8250_dpphy_cfg = { .serdes_tbl_hbr3 = qmp_v4_dp_serdes_tbl_hbr3, .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v4_dp_serdes_tbl_hbr3), + .swing_hbr_rbr = &qmp_dp_v3_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v3_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v3_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v3_pre_emphasis_hbr3_hbr2, + .clk_list = qmp_v4_phy_clk_l, .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), .reset_list = msm8996_usb3phy_reset_l, @@ -1040,9 +1336,6 @@ static const struct qmp_phy_cfg sm8250_dpphy_cfg = { .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), .regs = qmp_v4_usb3phy_regs_layout, - .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, - .dp_aux_init = qcom_qmp_v4_phy_dp_aux_init, .configure_dp_tx = qcom_qmp_v4_phy_configure_dp_tx, .configure_dp_phy = qcom_qmp_v4_phy_configure_dp_phy, @@ -1054,7 +1347,7 @@ static const struct qmp_phy_combo_cfg sm8250_usb3dpphy_cfg = { .dp_cfg = &sm8250_dpphy_cfg, }; -static void qcom_qmp_phy_combo_configure_lane(void __iomem *base, +static void qmp_combo_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num, @@ -1077,15 +1370,15 @@ static void qcom_qmp_phy_combo_configure_lane(void __iomem *base, } } -static void qcom_qmp_phy_combo_configure(void __iomem *base, +static void qmp_combo_configure(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num) { - qcom_qmp_phy_combo_configure_lane(base, regs, tbl, num, 0xff); + qmp_combo_configure_lane(base, regs, tbl, num, 0xff); } -static int qcom_qmp_phy_combo_serdes_init(struct qmp_phy *qphy) +static int qmp_combo_serdes_init(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *serdes = qphy->serdes; @@ -1093,27 +1386,27 @@ static int qcom_qmp_phy_combo_serdes_init(struct qmp_phy *qphy) const struct qmp_phy_init_tbl *serdes_tbl = cfg->serdes_tbl; int serdes_tbl_num = cfg->serdes_tbl_num; - qcom_qmp_phy_combo_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); + qmp_combo_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); if (cfg->type == PHY_TYPE_DP) { switch (dp_opts->link_rate) { case 1620: - qcom_qmp_phy_combo_configure(serdes, cfg->regs, + qmp_combo_configure(serdes, cfg->regs, cfg->serdes_tbl_rbr, cfg->serdes_tbl_rbr_num); break; case 2700: - qcom_qmp_phy_combo_configure(serdes, cfg->regs, + qmp_combo_configure(serdes, cfg->regs, cfg->serdes_tbl_hbr, cfg->serdes_tbl_hbr_num); break; case 5400: - qcom_qmp_phy_combo_configure(serdes, cfg->regs, + qmp_combo_configure(serdes, cfg->regs, cfg->serdes_tbl_hbr2, cfg->serdes_tbl_hbr2_num); break; case 8100: - qcom_qmp_phy_combo_configure(serdes, cfg->regs, + qmp_combo_configure(serdes, cfg->regs, cfg->serdes_tbl_hbr3, cfg->serdes_tbl_hbr3_num); break; @@ -1169,38 +1462,11 @@ static void qcom_qmp_v3_phy_dp_aux_init(struct qmp_phy *qphy) qphy->pcs + QSERDES_V3_DP_PHY_AUX_INTERRUPT_MASK); } -static const u8 qmp_dp_v3_pre_emphasis_hbr3_hbr2[4][4] = { - { 0x00, 0x0c, 0x15, 0x1a }, - { 0x02, 0x0e, 0x16, 0xff }, - { 0x02, 0x11, 0xff, 0xff }, - { 0x04, 0xff, 0xff, 0xff } -}; - -static const u8 qmp_dp_v3_voltage_swing_hbr3_hbr2[4][4] = { - { 0x02, 0x12, 0x16, 0x1a }, - { 0x09, 0x19, 0x1f, 0xff }, - { 0x10, 0x1f, 0xff, 0xff }, - { 0x1f, 0xff, 0xff, 0xff } -}; - -static const u8 qmp_dp_v3_pre_emphasis_hbr_rbr[4][4] = { - { 0x00, 0x0c, 0x14, 0x19 }, - { 0x00, 0x0b, 0x12, 0xff }, - { 0x00, 0x0b, 0xff, 0xff }, - { 0x04, 0xff, 0xff, 0xff } -}; - -static const u8 qmp_dp_v3_voltage_swing_hbr_rbr[4][4] = { - { 0x08, 0x0f, 0x16, 0x1f }, - { 0x11, 0x1e, 0x1f, 0xff }, - { 0x19, 0x1f, 0xff, 0xff }, - { 0x1f, 0xff, 0xff, 0xff } -}; - -static int qcom_qmp_phy_combo_configure_dp_swing(struct qmp_phy *qphy, +static int qmp_combo_configure_dp_swing(struct qmp_phy *qphy, unsigned int drv_lvl_reg, unsigned int emp_post_reg) { const struct phy_configure_opts_dp *dp_opts = &qphy->dp_opts; + const struct qmp_phy_cfg *cfg = qphy->cfg; unsigned int v_level = 0, p_level = 0; u8 voltage_swing_cfg, pre_emphasis_cfg; int i; @@ -1211,11 +1477,11 @@ static int qcom_qmp_phy_combo_configure_dp_swing(struct qmp_phy *qphy, } if (dp_opts->link_rate <= 2700) { - voltage_swing_cfg = qmp_dp_v3_voltage_swing_hbr_rbr[v_level][p_level]; - pre_emphasis_cfg = qmp_dp_v3_pre_emphasis_hbr_rbr[v_level][p_level]; + voltage_swing_cfg = (*cfg->swing_hbr_rbr)[v_level][p_level]; + pre_emphasis_cfg = (*cfg->pre_emphasis_hbr_rbr)[v_level][p_level]; } else { - voltage_swing_cfg = qmp_dp_v3_voltage_swing_hbr3_hbr2[v_level][p_level]; - pre_emphasis_cfg = qmp_dp_v3_pre_emphasis_hbr3_hbr2[v_level][p_level]; + voltage_swing_cfg = (*cfg->swing_hbr3_hbr2)[v_level][p_level]; + pre_emphasis_cfg = (*cfg->pre_emphasis_hbr3_hbr2)[v_level][p_level]; } /* TODO: Move check to config check */ @@ -1239,8 +1505,7 @@ static void qcom_qmp_v3_phy_configure_dp_tx(struct qmp_phy *qphy) const struct phy_configure_opts_dp *dp_opts = &qphy->dp_opts; u32 bias_en, drvr_en; - if (qcom_qmp_phy_combo_configure_dp_swing(qphy, - QSERDES_V3_TX_TX_DRV_LVL, + if (qmp_combo_configure_dp_swing(qphy, QSERDES_V3_TX_TX_DRV_LVL, QSERDES_V3_TX_TX_EMP_POST1_LVL) < 0) return; @@ -1258,7 +1523,7 @@ static void qcom_qmp_v3_phy_configure_dp_tx(struct qmp_phy *qphy) writel(bias_en, qphy->tx2 + QSERDES_V3_TX_TRANSCEIVER_BIAS_EN); } -static bool qcom_qmp_phy_combo_configure_dp_mode(struct qmp_phy *qphy) +static bool qmp_combo_configure_dp_mode(struct qmp_phy *qphy) { u32 val; bool reverse = false; @@ -1295,7 +1560,7 @@ static int qcom_qmp_v3_phy_configure_dp_phy(struct qmp_phy *qphy) u32 phy_vco_div, status; unsigned long pixel_freq; - qcom_qmp_phy_combo_configure_dp_mode(qphy); + qmp_combo_configure_dp_mode(qphy); writel(0x05, qphy->pcs + QSERDES_V3_DP_PHY_TX0_TX1_LANE_CTL); writel(0x05, qphy->pcs + QSERDES_V3_DP_PHY_TX2_TX3_LANE_CTL); @@ -1415,23 +1680,20 @@ static void qcom_qmp_v4_phy_configure_dp_tx(struct qmp_phy *qphy) writel(0x20, qphy->tx + QSERDES_V4_TX_TX_EMP_POST1_LVL); writel(0x20, qphy->tx2 + QSERDES_V4_TX_TX_EMP_POST1_LVL); - qcom_qmp_phy_combo_configure_dp_swing(qphy, - QSERDES_V4_TX_TX_DRV_LVL, + qmp_combo_configure_dp_swing(qphy, QSERDES_V4_TX_TX_DRV_LVL, QSERDES_V4_TX_TX_EMP_POST1_LVL); } -static int qcom_qmp_v4_phy_configure_dp_phy(struct qmp_phy *qphy) +static int qcom_qmp_v45_phy_configure_dp_phy(struct qmp_phy *qphy) { const struct qmp_phy_dp_clks *dp_clks = qphy->dp_clks; const struct phy_configure_opts_dp *dp_opts = &qphy->dp_opts; u32 phy_vco_div, status; unsigned long pixel_freq; - u32 bias0_en, drvr0_en, bias1_en, drvr1_en; - bool reverse; writel(0x0f, qphy->pcs + QSERDES_V4_DP_PHY_CFG_1); - reverse = qcom_qmp_phy_combo_configure_dp_mode(qphy); + qmp_combo_configure_dp_mode(qphy); writel(0x13, qphy->pcs + QSERDES_DP_PHY_AUX_CFG1); writel(0xa4, qphy->pcs + QSERDES_DP_PHY_AUX_CFG2); @@ -1509,6 +1771,21 @@ static int qcom_qmp_v4_phy_configure_dp_phy(struct qmp_phy *qphy) 10000)) return -ETIMEDOUT; + return 0; +} + +static int qcom_qmp_v4_phy_configure_dp_phy(struct qmp_phy *qphy) +{ + const struct phy_configure_opts_dp *dp_opts = &qphy->dp_opts; + u32 bias0_en, drvr0_en, bias1_en, drvr1_en; + bool reverse = false; + u32 status; + int ret; + + ret = qcom_qmp_v45_phy_configure_dp_phy(qphy); + if (ret < 0) + return ret; + /* * At least for 7nm DP PHY this has to be done after enabling link * clock. @@ -1559,6 +1836,63 @@ static int qcom_qmp_v4_phy_configure_dp_phy(struct qmp_phy *qphy) return 0; } +static int qcom_qmp_v5_phy_configure_dp_phy(struct qmp_phy *qphy) +{ + const struct phy_configure_opts_dp *dp_opts = &qphy->dp_opts; + u32 bias0_en, drvr0_en, bias1_en, drvr1_en; + bool reverse = false; + u32 status; + int ret; + + ret = qcom_qmp_v45_phy_configure_dp_phy(qphy); + if (ret < 0) + return ret; + + if (dp_opts->lanes == 1) { + bias0_en = reverse ? 0x3e : 0x1a; + drvr0_en = reverse ? 0x13 : 0x10; + bias1_en = reverse ? 0x15 : 0x3e; + drvr1_en = reverse ? 0x10 : 0x13; + } else if (dp_opts->lanes == 2) { + bias0_en = reverse ? 0x3f : 0x15; + drvr0_en = 0x10; + bias1_en = reverse ? 0x15 : 0x3f; + drvr1_en = 0x10; + } else { + bias0_en = 0x3f; + bias1_en = 0x3f; + drvr0_en = 0x10; + drvr1_en = 0x10; + } + + writel(drvr0_en, qphy->tx + QSERDES_V5_5NM_TX_HIGHZ_DRVR_EN); + writel(bias0_en, qphy->tx + QSERDES_V5_5NM_TX_TRANSCEIVER_BIAS_EN); + writel(drvr1_en, qphy->tx2 + QSERDES_V5_5NM_TX_HIGHZ_DRVR_EN); + writel(bias1_en, qphy->tx2 + QSERDES_V5_5NM_TX_TRANSCEIVER_BIAS_EN); + + writel(0x18, qphy->pcs + QSERDES_DP_PHY_CFG); + udelay(2000); + writel(0x19, qphy->pcs + QSERDES_DP_PHY_CFG); + + if (readl_poll_timeout(qphy->pcs + QSERDES_V4_DP_PHY_STATUS, + status, + ((status & BIT(1)) > 0), + 500, + 10000)) + return -ETIMEDOUT; + + writel(0x0a, qphy->tx + QSERDES_V5_5NM_TX_TX_POL_INV); + writel(0x0a, qphy->tx2 + QSERDES_V5_5NM_TX_TX_POL_INV); + + writel(0x27, qphy->tx + QSERDES_V5_5NM_TX_TX_DRV_LVL); + writel(0x27, qphy->tx2 + QSERDES_V5_5NM_TX_TX_DRV_LVL); + + writel(0x20, qphy->tx + QSERDES_V5_5NM_TX_TX_EMP_POST1_LVL); + writel(0x20, qphy->tx2 + QSERDES_V5_5NM_TX_TX_EMP_POST1_LVL); + + return 0; +} + /* * We need to calibrate the aux setting here as many times * as the caller tries @@ -1603,7 +1937,7 @@ static int qcom_qmp_dp_phy_calibrate(struct phy *phy) return 0; } -static int qcom_qmp_phy_combo_com_init(struct qmp_phy *qphy) +static int qmp_combo_com_init(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -1640,28 +1974,25 @@ static int qcom_qmp_phy_combo_com_init(struct qmp_phy *qphy) if (ret) goto err_assert_reset; - if (cfg->has_phy_dp_com_ctrl) { - qphy_setbits(dp_com, QPHY_V3_DP_COM_POWER_DOWN_CTRL, - SW_PWRDN); - /* override hardware control for reset of qmp phy */ - qphy_setbits(dp_com, QPHY_V3_DP_COM_RESET_OVRD_CTRL, - SW_DPPHY_RESET_MUX | SW_DPPHY_RESET | - SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET); + qphy_setbits(dp_com, QPHY_V3_DP_COM_POWER_DOWN_CTRL, SW_PWRDN); - /* Default type-c orientation, i.e CC1 */ - qphy_setbits(dp_com, QPHY_V3_DP_COM_TYPEC_CTRL, 0x02); + /* override hardware control for reset of qmp phy */ + qphy_setbits(dp_com, QPHY_V3_DP_COM_RESET_OVRD_CTRL, + SW_DPPHY_RESET_MUX | SW_DPPHY_RESET | + SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET); - qphy_setbits(dp_com, QPHY_V3_DP_COM_PHY_MODE_CTRL, - USB3_MODE | DP_MODE); + /* Default type-c orientation, i.e CC1 */ + qphy_setbits(dp_com, QPHY_V3_DP_COM_TYPEC_CTRL, 0x02); - /* bring both QMP USB and QMP DP PHYs PCS block out of reset */ - qphy_clrbits(dp_com, QPHY_V3_DP_COM_RESET_OVRD_CTRL, - SW_DPPHY_RESET_MUX | SW_DPPHY_RESET | - SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET); + qphy_setbits(dp_com, QPHY_V3_DP_COM_PHY_MODE_CTRL, USB3_MODE | DP_MODE); - qphy_clrbits(dp_com, QPHY_V3_DP_COM_SWI_CTRL, 0x03); - qphy_clrbits(dp_com, QPHY_V3_DP_COM_SW_RESET, SW_RESET); - } + /* bring both QMP USB and QMP DP PHYs PCS block out of reset */ + qphy_clrbits(dp_com, QPHY_V3_DP_COM_RESET_OVRD_CTRL, + SW_DPPHY_RESET_MUX | SW_DPPHY_RESET | + SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET); + + qphy_clrbits(dp_com, QPHY_V3_DP_COM_SWI_CTRL, 0x03); + qphy_clrbits(dp_com, QPHY_V3_DP_COM_SW_RESET, SW_RESET); if (cfg->regs[QPHY_PCS_POWER_DOWN_CONTROL]) qphy_setbits(pcs, @@ -1685,7 +2016,7 @@ err_unlock: return ret; } -static int qcom_qmp_phy_combo_com_exit(struct qmp_phy *qphy) +static int qmp_combo_com_exit(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -1709,7 +2040,7 @@ static int qcom_qmp_phy_combo_com_exit(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_combo_init(struct phy *phy) +static int qmp_combo_init(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -1717,7 +2048,7 @@ static int qcom_qmp_phy_combo_init(struct phy *phy) int ret; dev_vdbg(qmp->dev, "Initializing QMP phy\n"); - ret = qcom_qmp_phy_combo_com_init(qphy); + ret = qmp_combo_com_init(qphy); if (ret) return ret; @@ -1727,7 +2058,7 @@ static int qcom_qmp_phy_combo_init(struct phy *phy) return 0; } -static int qcom_qmp_phy_combo_power_on(struct phy *phy) +static int qmp_combo_power_on(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -1739,7 +2070,7 @@ static int qcom_qmp_phy_combo_power_on(struct phy *phy) unsigned int mask, val, ready; int ret; - qcom_qmp_phy_combo_serdes_init(qphy); + qmp_combo_serdes_init(qphy); ret = clk_prepare_enable(qphy->pipe_clk); if (ret) { @@ -1748,33 +2079,29 @@ static int qcom_qmp_phy_combo_power_on(struct phy *phy) } /* Tx, Rx, and PCS configurations */ - qcom_qmp_phy_combo_configure_lane(tx, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 1); + qmp_combo_configure_lane(tx, cfg->regs, cfg->tx_tbl, cfg->tx_tbl_num, 1); - /* Configuration for other LANE for USB-DP combo PHY */ - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_combo_configure_lane(qphy->tx2, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_combo_configure_lane(qphy->tx2, cfg->regs, cfg->tx_tbl, + cfg->tx_tbl_num, 2); } /* Configure special DP tx tunings */ if (cfg->type == PHY_TYPE_DP) cfg->configure_dp_tx(qphy); - qcom_qmp_phy_combo_configure_lane(rx, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 1); + qmp_combo_configure_lane(rx, cfg->regs, cfg->rx_tbl, cfg->rx_tbl_num, 1); - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_combo_configure_lane(qphy->rx2, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_combo_configure_lane(qphy->rx2, cfg->regs, cfg->rx_tbl, + cfg->rx_tbl_num, 2); } /* Configure link rate, swing, etc. */ - if (cfg->type == PHY_TYPE_DP) { + if (cfg->type == PHY_TYPE_DP) cfg->configure_dp_phy(qphy); - } else { - qcom_qmp_phy_combo_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); - } + else + qmp_combo_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); ret = reset_control_deassert(qmp->ufs_reset); if (ret) @@ -1808,7 +2135,7 @@ err_disable_pipe_clk: return ret; } -static int qcom_qmp_phy_combo_power_off(struct phy *phy) +static int qmp_combo_power_off(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -1838,42 +2165,41 @@ static int qcom_qmp_phy_combo_power_off(struct phy *phy) return 0; } -static int qcom_qmp_phy_combo_exit(struct phy *phy) +static int qmp_combo_exit(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); - qcom_qmp_phy_combo_com_exit(qphy); + qmp_combo_com_exit(qphy); return 0; } -static int qcom_qmp_phy_combo_enable(struct phy *phy) +static int qmp_combo_enable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_combo_init(phy); + ret = qmp_combo_init(phy); if (ret) return ret; - ret = qcom_qmp_phy_combo_power_on(phy); + ret = qmp_combo_power_on(phy); if (ret) - qcom_qmp_phy_combo_exit(phy); + qmp_combo_exit(phy); return ret; } -static int qcom_qmp_phy_combo_disable(struct phy *phy) +static int qmp_combo_disable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_combo_power_off(phy); + ret = qmp_combo_power_off(phy); if (ret) return ret; - return qcom_qmp_phy_combo_exit(phy); + return qmp_combo_exit(phy); } -static int qcom_qmp_phy_combo_set_mode(struct phy *phy, - enum phy_mode mode, int submode) +static int qmp_combo_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct qmp_phy *qphy = phy_get_drvdata(phy); @@ -1882,7 +2208,7 @@ static int qcom_qmp_phy_combo_set_mode(struct phy *phy, return 0; } -static void qcom_qmp_phy_combo_enable_autonomous_mode(struct qmp_phy *qphy) +static void qmp_combo_enable_autonomous_mode(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs; @@ -1911,7 +2237,7 @@ static void qcom_qmp_phy_combo_enable_autonomous_mode(struct qmp_phy *qphy) qphy_clrbits(pcs_misc, QPHY_V3_PCS_MISC_CLAMP_ENABLE, CLAMP_EN); } -static void qcom_qmp_phy_combo_disable_autonomous_mode(struct qmp_phy *qphy) +static void qmp_combo_disable_autonomous_mode(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs_usb; @@ -1929,7 +2255,7 @@ static void qcom_qmp_phy_combo_disable_autonomous_mode(struct qmp_phy *qphy) qphy_clrbits(pcs_usb, cfg->regs[QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR], IRQ_CLEAR); } -static int __maybe_unused qcom_qmp_phy_combo_runtime_suspend(struct device *dev) +static int __maybe_unused qmp_combo_runtime_suspend(struct device *dev) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct qmp_phy *qphy = qmp->phys[0]; @@ -1946,7 +2272,7 @@ static int __maybe_unused qcom_qmp_phy_combo_runtime_suspend(struct device *dev) return 0; } - qcom_qmp_phy_combo_enable_autonomous_mode(qphy); + qmp_combo_enable_autonomous_mode(qphy); clk_disable_unprepare(qphy->pipe_clk); clk_bulk_disable_unprepare(cfg->num_clks, qmp->clks); @@ -1954,7 +2280,7 @@ static int __maybe_unused qcom_qmp_phy_combo_runtime_suspend(struct device *dev) return 0; } -static int __maybe_unused qcom_qmp_phy_combo_runtime_resume(struct device *dev) +static int __maybe_unused qmp_combo_runtime_resume(struct device *dev) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct qmp_phy *qphy = qmp->phys[0]; @@ -1983,12 +2309,12 @@ static int __maybe_unused qcom_qmp_phy_combo_runtime_resume(struct device *dev) return ret; } - qcom_qmp_phy_combo_disable_autonomous_mode(qphy); + qmp_combo_disable_autonomous_mode(qphy); return 0; } -static int qcom_qmp_phy_combo_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_combo_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_vregs; @@ -2020,7 +2346,7 @@ static int qcom_qmp_phy_combo_vreg_init(struct device *dev, const struct qmp_phy return 0; } -static int qcom_qmp_phy_combo_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_combo_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int i; @@ -2041,7 +2367,7 @@ static int qcom_qmp_phy_combo_reset_init(struct device *dev, const struct qmp_ph return 0; } -static int qcom_qmp_phy_combo_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_combo_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_clks; @@ -2309,33 +2635,31 @@ static int phy_dp_clks_register(struct qcom_qmp *qmp, struct qmp_phy *qphy, return devm_add_action_or_reset(qmp->dev, phy_clk_release_provider, np); } -static const struct phy_ops qcom_qmp_phy_combo_usb_ops = { - .init = qcom_qmp_phy_combo_enable, - .exit = qcom_qmp_phy_combo_disable, - .set_mode = qcom_qmp_phy_combo_set_mode, +static const struct phy_ops qmp_combo_usb_ops = { + .init = qmp_combo_enable, + .exit = qmp_combo_disable, + .set_mode = qmp_combo_set_mode, .owner = THIS_MODULE, }; -static const struct phy_ops qcom_qmp_phy_combo_dp_ops = { - .init = qcom_qmp_phy_combo_init, +static const struct phy_ops qmp_combo_dp_ops = { + .init = qmp_combo_init, .configure = qcom_qmp_dp_phy_configure, - .power_on = qcom_qmp_phy_combo_power_on, + .power_on = qmp_combo_power_on, .calibrate = qcom_qmp_dp_phy_calibrate, - .power_off = qcom_qmp_phy_combo_power_off, - .exit = qcom_qmp_phy_combo_exit, - .set_mode = qcom_qmp_phy_combo_set_mode, + .power_off = qmp_combo_power_off, + .exit = qmp_combo_exit, + .set_mode = qmp_combo_set_mode, .owner = THIS_MODULE, }; -static -int qcom_qmp_phy_combo_create(struct device *dev, struct device_node *np, int id, +static int qmp_combo_create(struct device *dev, struct device_node *np, int id, void __iomem *serdes, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct phy *generic_phy; struct qmp_phy *qphy; const struct phy_ops *ops; - char prop_name[MAX_PROP_NAME]; int ret; qphy = devm_kzalloc(dev, sizeof(*qphy), GFP_KERNEL); @@ -2350,49 +2674,39 @@ int qcom_qmp_phy_combo_create(struct device *dev, struct device_node *np, int id * For dual lane PHYs: tx2 -> 3, rx2 -> 4, pcs_misc (optional) -> 5 * For single lane PHYs: pcs_misc (optional) -> 3. */ - qphy->tx = of_iomap(np, 0); - if (!qphy->tx) - return -ENOMEM; + qphy->tx = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(qphy->tx)) + return PTR_ERR(qphy->tx); - qphy->rx = of_iomap(np, 1); - if (!qphy->rx) - return -ENOMEM; + qphy->rx = devm_of_iomap(dev, np, 1, NULL); + if (IS_ERR(qphy->rx)) + return PTR_ERR(qphy->rx); - qphy->pcs = of_iomap(np, 2); - if (!qphy->pcs) - return -ENOMEM; + qphy->pcs = devm_of_iomap(dev, np, 2, NULL); + if (IS_ERR(qphy->pcs)) + return PTR_ERR(qphy->pcs); if (cfg->pcs_usb_offset) qphy->pcs_usb = qphy->pcs + cfg->pcs_usb_offset; - /* - * If this is a dual-lane PHY, then there should be registers for the - * second lane. Some old device trees did not specify this, so fall - * back to old legacy behavior of assuming they can be reached at an - * offset from the first lane. - */ - if (cfg->is_dual_lane_phy) { - qphy->tx2 = of_iomap(np, 3); - qphy->rx2 = of_iomap(np, 4); - if (!qphy->tx2 || !qphy->rx2) { - dev_warn(dev, - "Underspecified device tree, falling back to legacy register regions\n"); - - /* In the old version, pcs_misc is at index 3. */ - qphy->pcs_misc = qphy->tx2; - qphy->tx2 = qphy->tx + QMP_PHY_LEGACY_LANE_STRIDE; - qphy->rx2 = qphy->rx + QMP_PHY_LEGACY_LANE_STRIDE; + if (cfg->lanes >= 2) { + qphy->tx2 = devm_of_iomap(dev, np, 3, NULL); + if (IS_ERR(qphy->tx2)) + return PTR_ERR(qphy->tx2); - } else { - qphy->pcs_misc = of_iomap(np, 5); - } + qphy->rx2 = devm_of_iomap(dev, np, 4, NULL); + if (IS_ERR(qphy->rx2)) + return PTR_ERR(qphy->rx2); + qphy->pcs_misc = devm_of_iomap(dev, np, 5, NULL); } else { - qphy->pcs_misc = of_iomap(np, 3); + qphy->pcs_misc = devm_of_iomap(dev, np, 3, NULL); } - if (!qphy->pcs_misc) + if (IS_ERR(qphy->pcs_misc)) { dev_vdbg(dev, "PHY pcs_misc-reg not used\n"); + qphy->pcs_misc = NULL; + } /* * Get PHY's Pipe clock, if any. USB3 and PCIe are PIPE3 @@ -2401,24 +2715,19 @@ int qcom_qmp_phy_combo_create(struct device *dev, struct device_node *np, int id * Otherwise, we initialize pipe clock to NULL for * all phys that don't need this. */ - snprintf(prop_name, sizeof(prop_name), "pipe%d", id); - qphy->pipe_clk = devm_get_clk_from_child(dev, np, prop_name); + qphy->pipe_clk = devm_get_clk_from_child(dev, np, NULL); if (IS_ERR(qphy->pipe_clk)) { - if (cfg->type == PHY_TYPE_USB3) { - ret = PTR_ERR(qphy->pipe_clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, - "failed to get lane%d pipe_clk, %d\n", - id, ret); - return ret; - } + if (cfg->type == PHY_TYPE_USB3) + return dev_err_probe(dev, PTR_ERR(qphy->pipe_clk), + "failed to get lane%d pipe_clk\n", + id); qphy->pipe_clk = NULL; } if (cfg->type == PHY_TYPE_DP) - ops = &qcom_qmp_phy_combo_dp_ops; + ops = &qmp_combo_dp_ops; else - ops = &qcom_qmp_phy_combo_usb_ops; + ops = &qmp_combo_usb_ops; generic_phy = devm_phy_create(dev, np, ops); if (IS_ERR(generic_phy)) { @@ -2428,7 +2737,6 @@ int qcom_qmp_phy_combo_create(struct device *dev, struct device_node *np, int id } qphy->phy = generic_phy; - qphy->index = id; qphy->qmp = qmp; qmp->phys[id] = qphy; phy_set_drvdata(generic_phy, qphy); @@ -2436,11 +2744,15 @@ int qcom_qmp_phy_combo_create(struct device *dev, struct device_node *np, int id return 0; } -static const struct of_device_id qcom_qmp_combo_phy_of_match_table[] = { +static const struct of_device_id qmp_combo_of_match_table[] = { { .compatible = "qcom,sc7180-qmp-usb3-dp-phy", .data = &sc7180_usb3dpphy_cfg, }, + { + .compatible = "qcom,sdm845-qmp-usb3-dp-phy", + .data = &sdm845_usb3dpphy_cfg, + }, { .compatible = "qcom,sm8250-qmp-usb3-dp-phy", .data = &sm8250_usb3dpphy_cfg, @@ -2449,16 +2761,20 @@ static const struct of_device_id qcom_qmp_combo_phy_of_match_table[] = { .compatible = "qcom,sc8180x-qmp-usb3-dp-phy", .data = &sc8180x_usb3dpphy_cfg, }, + { + .compatible = "qcom,sc8280xp-qmp-usb43dp-phy", + .data = &sc8280xp_usb43dpphy_combo_cfg, + }, { } }; -MODULE_DEVICE_TABLE(of, qcom_qmp_combo_phy_of_match_table); +MODULE_DEVICE_TABLE(of, qmp_combo_of_match_table); -static const struct dev_pm_ops qcom_qmp_phy_combo_pm_ops = { - SET_RUNTIME_PM_OPS(qcom_qmp_phy_combo_runtime_suspend, - qcom_qmp_phy_combo_runtime_resume, NULL) +static const struct dev_pm_ops qmp_combo_pm_ops = { + SET_RUNTIME_PM_OPS(qmp_combo_runtime_suspend, + qmp_combo_runtime_resume, NULL) }; -static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) +static int qmp_combo_probe(struct platform_device *pdev) { struct qcom_qmp *qmp; struct device *dev = &pdev->dev; @@ -2494,12 +2810,9 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) if (IS_ERR(serdes)) return PTR_ERR(serdes); - /* per PHY dp_com; if PHY has dp_com control block */ - if (cfg->has_phy_dp_com_ctrl) { - qmp->dp_com = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(qmp->dp_com)) - return PTR_ERR(qmp->dp_com); - } + qmp->dp_com = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(qmp->dp_com)) + return PTR_ERR(qmp->dp_com); /* Only two serdes for combo PHY */ dp_serdes = devm_platform_ioremap_resource(pdev, 2); @@ -2511,21 +2824,18 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) mutex_init(&qmp->phy_mutex); - ret = qcom_qmp_phy_combo_clk_init(dev, cfg); + ret = qmp_combo_clk_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_combo_reset_init(dev, cfg); + ret = qmp_combo_reset_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_combo_vreg_init(dev, cfg); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + ret = qmp_combo_vreg_init(dev, cfg); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); num = of_get_available_child_count(dev->of_node); /* do we have a rogue child node ? */ @@ -2537,7 +2847,9 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) return -ENOMEM; pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. @@ -2551,7 +2863,7 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) serdes = dp_serdes; /* Create per-lane phy */ - ret = qcom_qmp_phy_combo_create(dev, child, id, serdes, cfg); + ret = qmp_combo_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -2569,7 +2881,7 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) serdes = usb_serdes; /* Create per-lane phy */ - ret = qcom_qmp_phy_combo_create(dev, child, id, serdes, cfg); + ret = qmp_combo_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -2592,29 +2904,24 @@ static int qcom_qmp_phy_combo_probe(struct platform_device *pdev) } phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_info(dev, "Registered Qcom-QMP phy\n"); - else - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); err_node_put: - pm_runtime_disable(dev); of_node_put(child); return ret; } -static struct platform_driver qcom_qmp_phy_combo_driver = { - .probe = qcom_qmp_phy_combo_probe, +static struct platform_driver qmp_combo_driver = { + .probe = qmp_combo_probe, .driver = { .name = "qcom-qmp-combo-phy", - .pm = &qcom_qmp_phy_combo_pm_ops, - .of_match_table = qcom_qmp_combo_phy_of_match_table, + .pm = &qmp_combo_pm_ops, + .of_match_table = qmp_combo_of_match_table, }, }; -module_platform_driver(qcom_qmp_phy_combo_driver); +module_platform_driver(qmp_combo_driver); MODULE_AUTHOR("Vivek Gautam "); MODULE_DESCRIPTION("Qualcomm QMP USB+DP combo PHY driver"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c index be6a94439b6c53eb4f94b19fa2b00369b26301a4..461f0b5d464a8b788449111dda43bab95938a419 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c @@ -36,46 +36,13 @@ /* QPHY_PCS_STATUS bit */ #define PHYSTATUS BIT(6) #define PHYSTATUS_4_20 BIT(7) -/* QPHY_PCS_READY_STATUS & QPHY_COM_PCS_READY_STATUS bit */ +/* QPHY_COM_PCS_READY_STATUS bit */ #define PCS_READY BIT(0) -/* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ -/* DP PHY soft reset */ -#define SW_DPPHY_RESET BIT(0) -/* mux to select DP PHY reset control, 0:HW control, 1: software reset */ -#define SW_DPPHY_RESET_MUX BIT(1) -/* USB3 PHY soft reset */ -#define SW_USB3PHY_RESET BIT(2) -/* mux to select USB3 PHY reset control, 0:HW control, 1: software reset */ -#define SW_USB3PHY_RESET_MUX BIT(3) - -/* QPHY_V3_DP_COM_PHY_MODE_CTRL register bits */ -#define USB3_MODE BIT(0) /* enables USB3 mode */ -#define DP_MODE BIT(1) /* enables DP mode */ - -/* QPHY_PCS_AUTONOMOUS_MODE_CTRL register bits */ -#define ARCVR_DTCT_EN BIT(0) -#define ALFPS_DTCT_EN BIT(1) -#define ARCVR_DTCT_EVENT_SEL BIT(4) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR register bits */ -#define IRQ_CLEAR BIT(0) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_STATUS register bits */ -#define RCVR_DETECT BIT(0) - -/* QPHY_V3_PCS_MISC_CLAMP_ENABLE register bits */ -#define CLAMP_EN BIT(0) /* enables i/o clamp_n */ - #define PHY_INIT_COMPLETE_TIMEOUT 10000 #define POWER_DOWN_DELAY_US_MIN 10 #define POWER_DOWN_DELAY_US_MAX 11 -#define MAX_PROP_NAME 32 - -/* Define the assumed distance between lanes for underspecified device trees. */ -#define QMP_PHY_LEGACY_LANE_STRIDE 0x400 - struct qmp_phy_init_tbl { unsigned int offset; unsigned int val; @@ -123,14 +90,8 @@ enum qphy_reg_layout { /* PCS registers */ QPHY_SW_RESET, QPHY_START_CTRL, - QPHY_PCS_READY_STATUS, QPHY_PCS_STATUS, - QPHY_PCS_AUTONOMOUS_MODE_CTRL, - QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, - QPHY_PCS_LFPS_RXTERM_IRQ_STATUS, QPHY_PCS_POWER_DOWN_CONTROL, - /* PCS_MISC registers */ - QPHY_PCS_MISC_TYPEC_CTRL, /* Keep last to ensure regs_layout arrays are properly initialized */ QPHY_LAYOUT_SIZE }; @@ -223,36 +184,20 @@ static const struct qmp_phy_init_tbl msm8996_pcie_pcs_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V2_PCS_TXDEEMPH_M3P5DB_V0, 0x0e), }; -struct qmp_phy; - /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { - /* phy-type - PCIE/UFS/USB */ - unsigned int type; - /* number of lanes provided by phy */ - int nlanes; + /* number of PHYs provided by this block */ + int num_phys; /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; int serdes_tbl_num; - const struct qmp_phy_init_tbl *serdes_tbl_sec; - int serdes_tbl_num_sec; const struct qmp_phy_init_tbl *tx_tbl; int tx_tbl_num; - const struct qmp_phy_init_tbl *tx_tbl_sec; - int tx_tbl_num_sec; const struct qmp_phy_init_tbl *rx_tbl; int rx_tbl_num; - const struct qmp_phy_init_tbl *rx_tbl_sec; - int rx_tbl_num_sec; const struct qmp_phy_init_tbl *pcs_tbl; int pcs_tbl_num; - const struct qmp_phy_init_tbl *pcs_tbl_sec; - int pcs_tbl_num_sec; - const struct qmp_phy_init_tbl *pcs_misc_tbl; - int pcs_misc_tbl_num; - const struct qmp_phy_init_tbl *pcs_misc_tbl_sec; - int pcs_misc_tbl_num_sec; /* clock ids to be requested */ const char * const *clk_list; @@ -289,12 +234,10 @@ struct qmp_phy_cfg { * @tx: iomapped memory space for lane's tx * @rx: iomapped memory space for lane's rx * @pcs: iomapped memory space for lane's pcs - * @pcs_misc: iomapped memory space for lane's pcs_misc * @pipe_clk: pipe clock * @index: lane index * @qmp: QMP phy to which this lane belongs * @lane_rst: lane's reset controller - * @mode: current PHY mode */ struct qmp_phy { struct phy *phy; @@ -303,12 +246,10 @@ struct qmp_phy { void __iomem *tx; void __iomem *rx; void __iomem *pcs; - void __iomem *pcs_misc; struct clk *pipe_clk; unsigned int index; struct qcom_qmp *qmp; struct reset_control *lane_rst; - enum phy_mode mode; }; /** @@ -377,8 +318,7 @@ static const char * const qmp_phy_vreg_l[] = { }; static const struct qmp_phy_cfg msm8996_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 3, + .num_phys = 3, .serdes_tbl = msm8996_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(msm8996_pcie_serdes_tbl), @@ -406,7 +346,7 @@ static const struct qmp_phy_cfg msm8996_pciephy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; -static void qcom_qmp_phy_pcie_msm8996_configure_lane(void __iomem *base, +static void qmp_pcie_msm8996_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num, @@ -429,15 +369,15 @@ static void qcom_qmp_phy_pcie_msm8996_configure_lane(void __iomem *base, } } -static void qcom_qmp_phy_pcie_msm8996_configure(void __iomem *base, +static void qmp_pcie_msm8996_configure(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num) { - qcom_qmp_phy_pcie_msm8996_configure_lane(base, regs, tbl, num, 0xff); + qmp_pcie_msm8996_configure_lane(base, regs, tbl, num, 0xff); } -static int qcom_qmp_phy_pcie_msm8996_serdes_init(struct qmp_phy *qphy) +static int qmp_pcie_msm8996_serdes_init(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -448,11 +388,7 @@ static int qcom_qmp_phy_pcie_msm8996_serdes_init(struct qmp_phy *qphy) unsigned int mask, val; int ret; - qcom_qmp_phy_pcie_msm8996_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); - if (cfg->serdes_tbl_sec) - qcom_qmp_phy_pcie_msm8996_configure(serdes, cfg->regs, cfg->serdes_tbl_sec, - cfg->serdes_tbl_num_sec); - + qmp_pcie_msm8996_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); qphy_clrbits(serdes, cfg->regs[QPHY_COM_SW_RESET], SW_RESET); qphy_setbits(serdes, cfg->regs[QPHY_COM_START_CONTROL], @@ -472,7 +408,7 @@ static int qcom_qmp_phy_pcie_msm8996_serdes_init(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_pcie_msm8996_com_init(struct qmp_phy *qphy) +static int qmp_pcie_msm8996_com_init(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -525,7 +461,7 @@ err_unlock: return ret; } -static int qcom_qmp_phy_pcie_msm8996_com_exit(struct qmp_phy *qphy) +static int qmp_pcie_msm8996_com_exit(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -555,21 +491,21 @@ static int qcom_qmp_phy_pcie_msm8996_com_exit(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_pcie_msm8996_init(struct phy *phy) +static int qmp_pcie_msm8996_init(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; int ret; dev_vdbg(qmp->dev, "Initializing QMP phy\n"); - ret = qcom_qmp_phy_pcie_msm8996_com_init(qphy); + ret = qmp_pcie_msm8996_com_init(qphy); if (ret) return ret; return 0; } -static int qcom_qmp_phy_pcie_msm8996_power_on(struct phy *phy) +static int qmp_pcie_msm8996_power_on(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -577,12 +513,11 @@ static int qcom_qmp_phy_pcie_msm8996_power_on(struct phy *phy) void __iomem *tx = qphy->tx; void __iomem *rx = qphy->rx; void __iomem *pcs = qphy->pcs; - void __iomem *pcs_misc = qphy->pcs_misc; void __iomem *status; unsigned int mask, val, ready; int ret; - qcom_qmp_phy_pcie_msm8996_serdes_init(qphy); + qmp_pcie_msm8996_serdes_init(qphy); ret = reset_control_deassert(qphy->lane_rst); if (ret) { @@ -598,28 +533,13 @@ static int qcom_qmp_phy_pcie_msm8996_power_on(struct phy *phy) } /* Tx, Rx, and PCS configurations */ - qcom_qmp_phy_pcie_msm8996_configure_lane(tx, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 1); - if (cfg->tx_tbl_sec) - qcom_qmp_phy_pcie_msm8996_configure_lane(tx, cfg->regs, cfg->tx_tbl_sec, - cfg->tx_tbl_num_sec, 1); - - qcom_qmp_phy_pcie_msm8996_configure_lane(rx, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 1); - if (cfg->rx_tbl_sec) - qcom_qmp_phy_pcie_msm8996_configure_lane(rx, cfg->regs, - cfg->rx_tbl_sec, cfg->rx_tbl_num_sec, 1); - - qcom_qmp_phy_pcie_msm8996_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); - if (cfg->pcs_tbl_sec) - qcom_qmp_phy_pcie_msm8996_configure(pcs, cfg->regs, cfg->pcs_tbl_sec, - cfg->pcs_tbl_num_sec); - - qcom_qmp_phy_pcie_msm8996_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl, - cfg->pcs_misc_tbl_num); - if (cfg->pcs_misc_tbl_sec) - qcom_qmp_phy_pcie_msm8996_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl_sec, - cfg->pcs_misc_tbl_num_sec); + qmp_pcie_msm8996_configure_lane(tx, cfg->regs, cfg->tx_tbl, + cfg->tx_tbl_num, 1); + + qmp_pcie_msm8996_configure_lane(rx, cfg->regs, cfg->rx_tbl, + cfg->rx_tbl_num, 1); + + qmp_pcie_msm8996_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); /* * Pull out PHY from POWER DOWN state. @@ -657,7 +577,7 @@ err_reset_lane: return ret; } -static int qcom_qmp_phy_pcie_msm8996_power_off(struct phy *phy) +static int qmp_pcie_msm8996_power_off(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -682,53 +602,43 @@ static int qcom_qmp_phy_pcie_msm8996_power_off(struct phy *phy) return 0; } -static int qcom_qmp_phy_pcie_msm8996_exit(struct phy *phy) +static int qmp_pcie_msm8996_exit(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); reset_control_assert(qphy->lane_rst); - qcom_qmp_phy_pcie_msm8996_com_exit(qphy); + qmp_pcie_msm8996_com_exit(qphy); return 0; } -static int qcom_qmp_phy_pcie_msm8996_enable(struct phy *phy) +static int qmp_pcie_msm8996_enable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_pcie_msm8996_init(phy); + ret = qmp_pcie_msm8996_init(phy); if (ret) return ret; - ret = qcom_qmp_phy_pcie_msm8996_power_on(phy); + ret = qmp_pcie_msm8996_power_on(phy); if (ret) - qcom_qmp_phy_pcie_msm8996_exit(phy); + qmp_pcie_msm8996_exit(phy); return ret; } -static int qcom_qmp_phy_pcie_msm8996_disable(struct phy *phy) +static int qmp_pcie_msm8996_disable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_pcie_msm8996_power_off(phy); + ret = qmp_pcie_msm8996_power_off(phy); if (ret) return ret; - return qcom_qmp_phy_pcie_msm8996_exit(phy); + return qmp_pcie_msm8996_exit(phy); } -static int qcom_qmp_phy_pcie_msm8996_set_mode(struct phy *phy, - enum phy_mode mode, int submode) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - - qphy->mode = mode; - - return 0; -} - -static int qcom_qmp_phy_pcie_msm8996_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_msm8996_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_vregs; @@ -744,7 +654,7 @@ static int qcom_qmp_phy_pcie_msm8996_vreg_init(struct device *dev, const struct return devm_regulator_bulk_get(dev, num, qmp->vregs); } -static int qcom_qmp_phy_pcie_msm8996_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_msm8996_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int i; @@ -765,7 +675,7 @@ static int qcom_qmp_phy_pcie_msm8996_reset_init(struct device *dev, const struct return 0; } -static int qcom_qmp_phy_pcie_msm8996_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_msm8996_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_clks; @@ -841,10 +751,9 @@ static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np) return devm_add_action_or_reset(qmp->dev, phy_clk_release_provider, np); } -static const struct phy_ops qcom_qmp_phy_pcie_msm8996_ops = { - .power_on = qcom_qmp_phy_pcie_msm8996_enable, - .power_off = qcom_qmp_phy_pcie_msm8996_disable, - .set_mode = qcom_qmp_phy_pcie_msm8996_set_mode, +static const struct phy_ops qmp_pcie_msm8996_ops = { + .power_on = qmp_pcie_msm8996_enable, + .power_off = qmp_pcie_msm8996_disable, .owner = THIS_MODULE, }; @@ -853,14 +762,12 @@ static void qcom_qmp_reset_control_put(void *data) reset_control_put(data); } -static -int qcom_qmp_phy_pcie_msm8996_create(struct device *dev, struct device_node *np, int id, +static int qmp_pcie_msm8996_create(struct device *dev, struct device_node *np, int id, void __iomem *serdes, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct phy *generic_phy; struct qmp_phy *qphy; - char prop_name[MAX_PROP_NAME]; int ret; qphy = devm_kzalloc(dev, sizeof(*qphy), GFP_KERNEL); @@ -872,36 +779,26 @@ int qcom_qmp_phy_pcie_msm8996_create(struct device *dev, struct device_node *np, /* * Get memory resources for each phy lane: * Resources are indexed as: tx -> 0; rx -> 1; pcs -> 2. - * For dual lane PHYs: tx2 -> 3, rx2 -> 4, pcs_misc (optional) -> 5 - * For single lane PHYs: pcs_misc (optional) -> 3. */ - qphy->tx = of_iomap(np, 0); - if (!qphy->tx) - return -ENOMEM; - - qphy->rx = of_iomap(np, 1); - if (!qphy->rx) - return -ENOMEM; - - qphy->pcs = of_iomap(np, 2); - if (!qphy->pcs) - return -ENOMEM; + qphy->tx = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(qphy->tx)) + return PTR_ERR(qphy->tx); - qphy->pcs_misc = of_iomap(np, 3); + qphy->rx = devm_of_iomap(dev, np, 1, NULL); + if (IS_ERR(qphy->rx)) + return PTR_ERR(qphy->rx); - if (!qphy->pcs_misc) - dev_vdbg(dev, "PHY pcs_misc-reg not used\n"); + qphy->pcs = devm_of_iomap(dev, np, 2, NULL); + if (IS_ERR(qphy->pcs)) + return PTR_ERR(qphy->pcs); - snprintf(prop_name, sizeof(prop_name), "pipe%d", id); - qphy->pipe_clk = devm_get_clk_from_child(dev, np, prop_name); + qphy->pipe_clk = devm_get_clk_from_child(dev, np, NULL); if (IS_ERR(qphy->pipe_clk)) { return dev_err_probe(dev, PTR_ERR(qphy->pipe_clk), "failed to get lane%d pipe clock\n", id); } - /* Get lane reset, if any */ - snprintf(prop_name, sizeof(prop_name), "lane%d", id); - qphy->lane_rst = of_reset_control_get_exclusive(np, prop_name); + qphy->lane_rst = of_reset_control_get_exclusive_by_index(np, 0); if (IS_ERR(qphy->lane_rst)) { dev_err(dev, "failed to get lane%d reset\n", id); return PTR_ERR(qphy->lane_rst); @@ -911,7 +808,7 @@ int qcom_qmp_phy_pcie_msm8996_create(struct device *dev, struct device_node *np, if (ret) return ret; - generic_phy = devm_phy_create(dev, np, &qcom_qmp_phy_pcie_msm8996_ops); + generic_phy = devm_phy_create(dev, np, &qmp_pcie_msm8996_ops); if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create qphy %d\n", ret); @@ -927,16 +824,16 @@ int qcom_qmp_phy_pcie_msm8996_create(struct device *dev, struct device_node *np, return 0; } -static const struct of_device_id qcom_qmp_phy_pcie_msm8996_of_match_table[] = { +static const struct of_device_id qmp_pcie_msm8996_of_match_table[] = { { .compatible = "qcom,msm8996-qmp-pcie-phy", .data = &msm8996_pciephy_cfg, }, { }, }; -MODULE_DEVICE_TABLE(of, qcom_qmp_phy_pcie_msm8996_of_match_table); +MODULE_DEVICE_TABLE(of, qmp_pcie_msm8996_of_match_table); -static int qcom_qmp_phy_pcie_msm8996_probe(struct platform_device *pdev) +static int qmp_pcie_msm8996_probe(struct platform_device *pdev) { struct qcom_qmp *qmp; struct device *dev = &pdev->dev; @@ -964,25 +861,22 @@ static int qcom_qmp_phy_pcie_msm8996_probe(struct platform_device *pdev) if (IS_ERR(serdes)) return PTR_ERR(serdes); - expected_phys = cfg->nlanes; + expected_phys = cfg->num_phys; mutex_init(&qmp->phy_mutex); - ret = qcom_qmp_phy_pcie_msm8996_clk_init(dev, cfg); + ret = qmp_pcie_msm8996_clk_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_pcie_msm8996_reset_init(dev, cfg); + ret = qmp_pcie_msm8996_reset_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_pcie_msm8996_vreg_init(dev, cfg); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + ret = qmp_pcie_msm8996_vreg_init(dev, cfg); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); num = of_get_available_child_count(dev->of_node); /* do we have a rogue child node ? */ @@ -993,18 +887,10 @@ static int qcom_qmp_phy_pcie_msm8996_probe(struct platform_device *pdev) if (!qmp->phys) return -ENOMEM; - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - /* - * Prevent runtime pm from being ON by default. Users can enable - * it using power/control in sysfs. - */ - pm_runtime_forbid(dev); - id = 0; for_each_available_child_of_node(dev->of_node, child) { /* Create per-lane phy */ - ret = qcom_qmp_phy_pcie_msm8996_create(dev, child, id, serdes, cfg); + ret = qmp_pcie_msm8996_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -1026,28 +912,23 @@ static int qcom_qmp_phy_pcie_msm8996_probe(struct platform_device *pdev) } phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_info(dev, "Registered Qcom-QMP phy\n"); - else - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); err_node_put: - pm_runtime_disable(dev); of_node_put(child); return ret; } -static struct platform_driver qcom_qmp_phy_pcie_msm8996_driver = { - .probe = qcom_qmp_phy_pcie_msm8996_probe, +static struct platform_driver qmp_pcie_msm8996_driver = { + .probe = qmp_pcie_msm8996_probe, .driver = { .name = "qcom-qmp-msm8996-pcie-phy", - .of_match_table = qcom_qmp_phy_pcie_msm8996_of_match_table, + .of_match_table = qmp_pcie_msm8996_of_match_table, }, }; -module_platform_driver(qcom_qmp_phy_pcie_msm8996_driver); +module_platform_driver(qmp_pcie_msm8996_driver); MODULE_AUTHOR("Vivek Gautam "); MODULE_DESCRIPTION("Qualcomm QMP MSM8996 PCIe PHY driver"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index 2d65e1f56bfc34f7f43984b5aff5100f2826c16c..5be5348fbb26b023039e254451045c6ce0af6b8e 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -32,49 +32,11 @@ /* QPHY_START_CONTROL bits */ #define SERDES_START BIT(0) #define PCS_START BIT(1) -#define PLL_READY_GATE_EN BIT(3) /* QPHY_PCS_STATUS bit */ #define PHYSTATUS BIT(6) #define PHYSTATUS_4_20 BIT(7) -/* QPHY_PCS_READY_STATUS & QPHY_COM_PCS_READY_STATUS bit */ -#define PCS_READY BIT(0) - -/* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ -/* DP PHY soft reset */ -#define SW_DPPHY_RESET BIT(0) -/* mux to select DP PHY reset control, 0:HW control, 1: software reset */ -#define SW_DPPHY_RESET_MUX BIT(1) -/* USB3 PHY soft reset */ -#define SW_USB3PHY_RESET BIT(2) -/* mux to select USB3 PHY reset control, 0:HW control, 1: software reset */ -#define SW_USB3PHY_RESET_MUX BIT(3) - -/* QPHY_V3_DP_COM_PHY_MODE_CTRL register bits */ -#define USB3_MODE BIT(0) /* enables USB3 mode */ -#define DP_MODE BIT(1) /* enables DP mode */ - -/* QPHY_PCS_AUTONOMOUS_MODE_CTRL register bits */ -#define ARCVR_DTCT_EN BIT(0) -#define ALFPS_DTCT_EN BIT(1) -#define ARCVR_DTCT_EVENT_SEL BIT(4) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR register bits */ -#define IRQ_CLEAR BIT(0) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_STATUS register bits */ -#define RCVR_DETECT BIT(0) - -/* QPHY_V3_PCS_MISC_CLAMP_ENABLE register bits */ -#define CLAMP_EN BIT(0) /* enables i/o clamp_n */ #define PHY_INIT_COMPLETE_TIMEOUT 10000 -#define POWER_DOWN_DELAY_US_MIN 10 -#define POWER_DOWN_DELAY_US_MAX 11 - -#define MAX_PROP_NAME 32 - -/* Define the assumed distance between lanes for underspecified device trees. */ -#define QMP_PHY_LEGACY_LANE_STRIDE 0x400 struct qmp_phy_init_tbl { unsigned int offset; @@ -123,14 +85,8 @@ enum qphy_reg_layout { /* PCS registers */ QPHY_SW_RESET, QPHY_START_CTRL, - QPHY_PCS_READY_STATUS, QPHY_PCS_STATUS, - QPHY_PCS_AUTONOMOUS_MODE_CTRL, - QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, - QPHY_PCS_LFPS_RXTERM_IRQ_STATUS, QPHY_PCS_POWER_DOWN_CONTROL, - /* PCS_MISC registers */ - QPHY_PCS_MISC_TYPEC_CTRL, /* Keep last to ensure regs_layout arrays are properly initialized */ QPHY_LAYOUT_SIZE }; @@ -1344,14 +1300,9 @@ static const struct qmp_phy_init_tbl sm8450_qmp_gen4x2_pcie_pcs_misc_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V5_20_PCS_PCIE_G4_PRE_GAIN, 0x2e), }; -struct qmp_phy; - /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { - /* phy-type - PCIE/UFS/USB */ - unsigned int type; - /* number of lanes provided by phy */ - int nlanes; + int lanes; /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; @@ -1390,7 +1341,6 @@ struct qmp_phy_cfg { unsigned int start_ctrl; unsigned int pwrdn_ctrl; - unsigned int mask_com_pcs_ready; /* bit offset of PHYSTATUS in QPHY_PCS_STATUS register */ unsigned int phy_status; @@ -1400,9 +1350,6 @@ struct qmp_phy_cfg { int pwrdn_delay_min; int pwrdn_delay_max; - /* true, if PHY has secondary tx/rx lanes to be configured */ - bool is_dual_lane_phy; - /* QMP PHY pipe clock interface rate */ unsigned long pipe_clock_rate; }; @@ -1420,9 +1367,7 @@ struct qmp_phy_cfg { * @rx2: iomapped memory space for second lane's rx (in dual lane PHYs) * @pcs_misc: iomapped memory space for lane's pcs_misc * @pipe_clk: pipe clock - * @index: lane index * @qmp: QMP phy to which this lane belongs - * @mode: current PHY mode */ struct qmp_phy { struct phy *phy; @@ -1435,9 +1380,7 @@ struct qmp_phy { void __iomem *rx2; void __iomem *pcs_misc; struct clk *pipe_clk; - unsigned int index; struct qcom_qmp *qmp; - enum phy_mode mode; }; /** @@ -1514,8 +1457,7 @@ static const char * const sdm845_pciephy_reset_l[] = { }; static const struct qmp_phy_cfg ipq8074_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = ipq8074_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(ipq8074_pcie_serdes_tbl), @@ -1543,8 +1485,7 @@ static const struct qmp_phy_cfg ipq8074_pciephy_cfg = { }; static const struct qmp_phy_cfg ipq8074_pciephy_gen3_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = ipq8074_pcie_gen3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(ipq8074_pcie_gen3_serdes_tbl), @@ -1573,8 +1514,7 @@ static const struct qmp_phy_cfg ipq8074_pciephy_gen3_cfg = { }; static const struct qmp_phy_cfg ipq6018_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = ipq6018_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(ipq6018_pcie_serdes_tbl), @@ -1603,8 +1543,7 @@ static const struct qmp_phy_cfg ipq6018_pciephy_cfg = { }; static const struct qmp_phy_cfg sdm845_qmp_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sdm845_qmp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sdm845_qmp_pcie_serdes_tbl), @@ -1634,8 +1573,7 @@ static const struct qmp_phy_cfg sdm845_qmp_pciephy_cfg = { }; static const struct qmp_phy_cfg sdm845_qhp_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sdm845_qhp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sdm845_qhp_pcie_serdes_tbl), @@ -1663,8 +1601,7 @@ static const struct qmp_phy_cfg sdm845_qhp_pciephy_cfg = { }; static const struct qmp_phy_cfg sm8250_qmp_gen3x1_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8250_qmp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8250_qmp_pcie_serdes_tbl), @@ -1702,8 +1639,7 @@ static const struct qmp_phy_cfg sm8250_qmp_gen3x1_pciephy_cfg = { }; static const struct qmp_phy_cfg sm8250_qmp_gen3x2_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sm8250_qmp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8250_qmp_pcie_serdes_tbl), @@ -1735,15 +1671,13 @@ static const struct qmp_phy_cfg sm8250_qmp_gen3x2_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, .phy_status = PHYSTATUS, - .is_dual_lane_phy = true, .has_pwrdn_delay = true, .pwrdn_delay_min = 995, /* us */ .pwrdn_delay_max = 1005, /* us */ }; static const struct qmp_phy_cfg msm8998_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = msm8998_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(msm8998_pcie_serdes_tbl), @@ -1767,8 +1701,7 @@ static const struct qmp_phy_cfg msm8998_pciephy_cfg = { }; static const struct qmp_phy_cfg sc8180x_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sc8180x_qmp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sc8180x_qmp_pcie_serdes_tbl), @@ -1797,8 +1730,7 @@ static const struct qmp_phy_cfg sc8180x_pciephy_cfg = { }; static const struct qmp_phy_cfg sdx55_qmp_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sdx55_qmp_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sdx55_qmp_pcie_serdes_tbl), @@ -1822,15 +1754,13 @@ static const struct qmp_phy_cfg sdx55_qmp_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS_4_20, - .is_dual_lane_phy = true, .has_pwrdn_delay = true, .pwrdn_delay_min = 995, /* us */ .pwrdn_delay_max = 1005, /* us */ }; static const struct qmp_phy_cfg sm8450_qmp_gen3x1_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8450_qmp_gen3x1_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8450_qmp_gen3x1_pcie_serdes_tbl), @@ -1860,8 +1790,7 @@ static const struct qmp_phy_cfg sm8450_qmp_gen3x1_pciephy_cfg = { }; static const struct qmp_phy_cfg sm8450_qmp_gen4x2_pciephy_cfg = { - .type = PHY_TYPE_PCIE, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sm8450_qmp_gen4x2_pcie_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8450_qmp_gen4x2_pcie_serdes_tbl), @@ -1885,13 +1814,12 @@ static const struct qmp_phy_cfg sm8450_qmp_gen4x2_pciephy_cfg = { .pwrdn_ctrl = SW_PWRDN | REFCLK_DRV_DSBL, .phy_status = PHYSTATUS_4_20, - .is_dual_lane_phy = true, .has_pwrdn_delay = true, .pwrdn_delay_min = 995, /* us */ .pwrdn_delay_max = 1005, /* us */ }; -static void qcom_qmp_phy_pcie_configure_lane(void __iomem *base, +static void qmp_pcie_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num, @@ -1914,31 +1842,30 @@ static void qcom_qmp_phy_pcie_configure_lane(void __iomem *base, } } -static void qcom_qmp_phy_pcie_configure(void __iomem *base, - const unsigned int *regs, - const struct qmp_phy_init_tbl tbl[], - int num) +static void qmp_pcie_configure(void __iomem *base, + const unsigned int *regs, + const struct qmp_phy_init_tbl tbl[], + int num) { - qcom_qmp_phy_pcie_configure_lane(base, regs, tbl, num, 0xff); + qmp_pcie_configure_lane(base, regs, tbl, num, 0xff); } -static int qcom_qmp_phy_pcie_serdes_init(struct qmp_phy *qphy) +static int qmp_pcie_serdes_init(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *serdes = qphy->serdes; const struct qmp_phy_init_tbl *serdes_tbl = cfg->serdes_tbl; int serdes_tbl_num = cfg->serdes_tbl_num; - qcom_qmp_phy_pcie_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); - if (cfg->serdes_tbl_sec) - qcom_qmp_phy_pcie_configure(serdes, cfg->regs, cfg->serdes_tbl_sec, - cfg->serdes_tbl_num_sec); + qmp_pcie_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); + qmp_pcie_configure(serdes, cfg->regs, cfg->serdes_tbl_sec, cfg->serdes_tbl_num_sec); return 0; } -static int qcom_qmp_phy_pcie_com_init(struct qmp_phy *qphy) +static int qmp_pcie_init(struct phy *phy) { + struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs = qphy->pcs; @@ -1985,8 +1912,9 @@ err_disable_regulators: return ret; } -static int qcom_qmp_phy_pcie_com_exit(struct qmp_phy *qphy) +static int qmp_pcie_exit(struct phy *phy) { + struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -1999,21 +1927,7 @@ static int qcom_qmp_phy_pcie_com_exit(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_pcie_init(struct phy *phy) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - struct qcom_qmp *qmp = qphy->qmp; - int ret; - dev_vdbg(qmp->dev, "Initializing QMP phy\n"); - - ret = qcom_qmp_phy_pcie_com_init(qphy); - if (ret) - return ret; - - return 0; -} - -static int qcom_qmp_phy_pcie_power_on(struct phy *phy) +static int qmp_pcie_power_on(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -2026,7 +1940,7 @@ static int qcom_qmp_phy_pcie_power_on(struct phy *phy) unsigned int mask, val, ready; int ret; - qcom_qmp_phy_pcie_serdes_init(qphy); + qmp_pcie_serdes_init(qphy); ret = clk_prepare_enable(qphy->pipe_clk); if (ret) { @@ -2035,47 +1949,31 @@ static int qcom_qmp_phy_pcie_power_on(struct phy *phy) } /* Tx, Rx, and PCS configurations */ - qcom_qmp_phy_pcie_configure_lane(tx, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 1); - if (cfg->tx_tbl_sec) - qcom_qmp_phy_pcie_configure_lane(tx, cfg->regs, cfg->tx_tbl_sec, - cfg->tx_tbl_num_sec, 1); - - /* Configuration for other LANE for USB-DP combo PHY */ - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_pcie_configure_lane(qphy->tx2, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 2); - if (cfg->tx_tbl_sec) - qcom_qmp_phy_pcie_configure_lane(qphy->tx2, cfg->regs, - cfg->tx_tbl_sec, - cfg->tx_tbl_num_sec, 2); + qmp_pcie_configure_lane(tx, cfg->regs, cfg->tx_tbl, cfg->tx_tbl_num, 1); + qmp_pcie_configure_lane(tx, cfg->regs, cfg->tx_tbl_sec, cfg->tx_tbl_num_sec, 1); + + if (cfg->lanes >= 2) { + qmp_pcie_configure_lane(qphy->tx2, cfg->regs, cfg->tx_tbl, + cfg->tx_tbl_num, 2); + qmp_pcie_configure_lane(qphy->tx2, cfg->regs, cfg->tx_tbl_sec, + cfg->tx_tbl_num_sec, 2); } - qcom_qmp_phy_pcie_configure_lane(rx, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 1); - if (cfg->rx_tbl_sec) - qcom_qmp_phy_pcie_configure_lane(rx, cfg->regs, - cfg->rx_tbl_sec, cfg->rx_tbl_num_sec, 1); - - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_pcie_configure_lane(qphy->rx2, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 2); - if (cfg->rx_tbl_sec) - qcom_qmp_phy_pcie_configure_lane(qphy->rx2, cfg->regs, - cfg->rx_tbl_sec, - cfg->rx_tbl_num_sec, 2); + qmp_pcie_configure_lane(rx, cfg->regs, cfg->rx_tbl, cfg->rx_tbl_num, 1); + qmp_pcie_configure_lane(rx, cfg->regs, cfg->rx_tbl_sec, cfg->rx_tbl_num_sec, 1); + + if (cfg->lanes >= 2) { + qmp_pcie_configure_lane(qphy->rx2, cfg->regs, cfg->rx_tbl, + cfg->rx_tbl_num, 2); + qmp_pcie_configure_lane(qphy->rx2, cfg->regs, cfg->rx_tbl_sec, + cfg->rx_tbl_num_sec, 2); } - qcom_qmp_phy_pcie_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); - if (cfg->pcs_tbl_sec) - qcom_qmp_phy_pcie_configure(pcs, cfg->regs, cfg->pcs_tbl_sec, - cfg->pcs_tbl_num_sec); + qmp_pcie_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); + qmp_pcie_configure(pcs, cfg->regs, cfg->pcs_tbl_sec, cfg->pcs_tbl_num_sec); - qcom_qmp_phy_pcie_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl, - cfg->pcs_misc_tbl_num); - if (cfg->pcs_misc_tbl_sec) - qcom_qmp_phy_pcie_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl_sec, - cfg->pcs_misc_tbl_num_sec); + qmp_pcie_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl, cfg->pcs_misc_tbl_num); + qmp_pcie_configure(pcs_misc, cfg->regs, cfg->pcs_misc_tbl_sec, cfg->pcs_misc_tbl_num_sec); /* * Pull out PHY from POWER DOWN state. @@ -2111,7 +2009,7 @@ err_disable_pipe_clk: return ret; } -static int qcom_qmp_phy_pcie_power_off(struct phy *phy) +static int qmp_pcie_power_off(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -2136,51 +2034,33 @@ static int qcom_qmp_phy_pcie_power_off(struct phy *phy) return 0; } -static int qcom_qmp_phy_pcie_exit(struct phy *phy) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - - qcom_qmp_phy_pcie_com_exit(qphy); - - return 0; -} - -static int qcom_qmp_phy_pcie_enable(struct phy *phy) +static int qmp_pcie_enable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_pcie_init(phy); + ret = qmp_pcie_init(phy); if (ret) return ret; - ret = qcom_qmp_phy_pcie_power_on(phy); + ret = qmp_pcie_power_on(phy); if (ret) - qcom_qmp_phy_pcie_exit(phy); + qmp_pcie_exit(phy); return ret; } -static int qcom_qmp_phy_pcie_disable(struct phy *phy) +static int qmp_pcie_disable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_pcie_power_off(phy); + ret = qmp_pcie_power_off(phy); if (ret) return ret; - return qcom_qmp_phy_pcie_exit(phy); -} - -static int qcom_qmp_phy_pcie_set_mode(struct phy *phy, - enum phy_mode mode, int submode) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - qphy->mode = mode; - - return 0; + return qmp_pcie_exit(phy); } -static int qcom_qmp_phy_pcie_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_vregs; @@ -2196,7 +2076,7 @@ static int qcom_qmp_phy_pcie_vreg_init(struct device *dev, const struct qmp_phy_ return devm_regulator_bulk_get(dev, num, qmp->vregs); } -static int qcom_qmp_phy_pcie_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int i; @@ -2217,7 +2097,7 @@ static int qcom_qmp_phy_pcie_reset_init(struct device *dev, const struct qmp_phy return 0; } -static int qcom_qmp_phy_pcie_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_pcie_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_clks; @@ -2300,21 +2180,18 @@ static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np) return devm_add_action_or_reset(qmp->dev, phy_clk_release_provider, np); } -static const struct phy_ops qcom_qmp_phy_pcie_ops = { - .power_on = qcom_qmp_phy_pcie_enable, - .power_off = qcom_qmp_phy_pcie_disable, - .set_mode = qcom_qmp_phy_pcie_set_mode, +static const struct phy_ops qmp_pcie_ops = { + .power_on = qmp_pcie_enable, + .power_off = qmp_pcie_disable, .owner = THIS_MODULE, }; -static -int qcom_qmp_phy_pcie_create(struct device *dev, struct device_node *np, int id, +static int qmp_pcie_create(struct device *dev, struct device_node *np, int id, void __iomem *serdes, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct phy *generic_phy; struct qmp_phy *qphy; - char prop_name[MAX_PROP_NAME]; int ret; qphy = devm_kzalloc(dev, sizeof(*qphy), GFP_KERNEL); @@ -2329,59 +2206,51 @@ int qcom_qmp_phy_pcie_create(struct device *dev, struct device_node *np, int id, * For dual lane PHYs: tx2 -> 3, rx2 -> 4, pcs_misc (optional) -> 5 * For single lane PHYs: pcs_misc (optional) -> 3. */ - qphy->tx = of_iomap(np, 0); - if (!qphy->tx) - return -ENOMEM; + qphy->tx = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(qphy->tx)) + return PTR_ERR(qphy->tx); - qphy->rx = of_iomap(np, 1); - if (!qphy->rx) - return -ENOMEM; + if (of_device_is_compatible(dev->of_node, "qcom,sdm845-qhp-pcie-phy")) + qphy->rx = qphy->tx; + else + qphy->rx = devm_of_iomap(dev, np, 1, NULL); + if (IS_ERR(qphy->rx)) + return PTR_ERR(qphy->rx); - qphy->pcs = of_iomap(np, 2); - if (!qphy->pcs) - return -ENOMEM; + qphy->pcs = devm_of_iomap(dev, np, 2, NULL); + if (IS_ERR(qphy->pcs)) + return PTR_ERR(qphy->pcs); - /* - * If this is a dual-lane PHY, then there should be registers for the - * second lane. Some old device trees did not specify this, so fall - * back to old legacy behavior of assuming they can be reached at an - * offset from the first lane. - */ - if (cfg->is_dual_lane_phy) { - qphy->tx2 = of_iomap(np, 3); - qphy->rx2 = of_iomap(np, 4); - if (!qphy->tx2 || !qphy->rx2) { - dev_warn(dev, - "Underspecified device tree, falling back to legacy register regions\n"); - - /* In the old version, pcs_misc is at index 3. */ - qphy->pcs_misc = qphy->tx2; - qphy->tx2 = qphy->tx + QMP_PHY_LEGACY_LANE_STRIDE; - qphy->rx2 = qphy->rx + QMP_PHY_LEGACY_LANE_STRIDE; - - } else { - qphy->pcs_misc = of_iomap(np, 5); - } + if (cfg->lanes >= 2) { + qphy->tx2 = devm_of_iomap(dev, np, 3, NULL); + if (IS_ERR(qphy->tx2)) + return PTR_ERR(qphy->tx2); + qphy->rx2 = devm_of_iomap(dev, np, 4, NULL); + if (IS_ERR(qphy->rx2)) + return PTR_ERR(qphy->rx2); + + qphy->pcs_misc = devm_of_iomap(dev, np, 5, NULL); } else { - qphy->pcs_misc = of_iomap(np, 3); + qphy->pcs_misc = devm_of_iomap(dev, np, 3, NULL); } - if (!qphy->pcs_misc && + if (IS_ERR(qphy->pcs_misc) && of_device_is_compatible(dev->of_node, "qcom,ipq6018-qmp-pcie-phy")) qphy->pcs_misc = qphy->pcs + 0x400; - if (!qphy->pcs_misc) - dev_vdbg(dev, "PHY pcs_misc-reg not used\n"); + if (IS_ERR(qphy->pcs_misc)) { + if (cfg->pcs_misc_tbl || cfg->pcs_misc_tbl_sec) + return PTR_ERR(qphy->pcs_misc); + } - snprintf(prop_name, sizeof(prop_name), "pipe%d", id); - qphy->pipe_clk = devm_get_clk_from_child(dev, np, prop_name); + qphy->pipe_clk = devm_get_clk_from_child(dev, np, NULL); if (IS_ERR(qphy->pipe_clk)) { return dev_err_probe(dev, PTR_ERR(qphy->pipe_clk), "failed to get lane%d pipe clock\n", id); } - generic_phy = devm_phy_create(dev, np, &qcom_qmp_phy_pcie_ops); + generic_phy = devm_phy_create(dev, np, &qmp_pcie_ops); if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create qphy %d\n", ret); @@ -2389,7 +2258,6 @@ int qcom_qmp_phy_pcie_create(struct device *dev, struct device_node *np, int id, } qphy->phy = generic_phy; - qphy->index = id; qphy->qmp = qmp; qmp->phys[id] = qphy; phy_set_drvdata(generic_phy, qphy); @@ -2397,7 +2265,7 @@ int qcom_qmp_phy_pcie_create(struct device *dev, struct device_node *np, int id, return 0; } -static const struct of_device_id qcom_qmp_phy_pcie_of_match_table[] = { +static const struct of_device_id qmp_pcie_of_match_table[] = { { .compatible = "qcom,msm8998-qmp-pcie-phy", .data = &msm8998_pciephy_cfg, @@ -2440,9 +2308,9 @@ static const struct of_device_id qcom_qmp_phy_pcie_of_match_table[] = { }, { }, }; -MODULE_DEVICE_TABLE(of, qcom_qmp_phy_pcie_of_match_table); +MODULE_DEVICE_TABLE(of, qmp_pcie_of_match_table); -static int qcom_qmp_phy_pcie_probe(struct platform_device *pdev) +static int qmp_pcie_probe(struct platform_device *pdev) { struct qcom_qmp *qmp; struct device *dev = &pdev->dev; @@ -2470,21 +2338,18 @@ static int qcom_qmp_phy_pcie_probe(struct platform_device *pdev) if (IS_ERR(serdes)) return PTR_ERR(serdes); - ret = qcom_qmp_phy_pcie_clk_init(dev, cfg); + ret = qmp_pcie_clk_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_pcie_reset_init(dev, cfg); + ret = qmp_pcie_reset_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_pcie_vreg_init(dev, cfg); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + ret = qmp_pcie_vreg_init(dev, cfg); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); num = of_get_available_child_count(dev->of_node); /* do we have a rogue child node ? */ @@ -2495,18 +2360,10 @@ static int qcom_qmp_phy_pcie_probe(struct platform_device *pdev) if (!qmp->phys) return -ENOMEM; - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - /* - * Prevent runtime pm from being ON by default. Users can enable - * it using power/control in sysfs. - */ - pm_runtime_forbid(dev); - id = 0; for_each_available_child_of_node(dev->of_node, child) { /* Create per-lane phy */ - ret = qcom_qmp_phy_pcie_create(dev, child, id, serdes, cfg); + ret = qmp_pcie_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -2528,28 +2385,23 @@ static int qcom_qmp_phy_pcie_probe(struct platform_device *pdev) } phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_info(dev, "Registered Qcom-QMP phy\n"); - else - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); err_node_put: - pm_runtime_disable(dev); of_node_put(child); return ret; } -static struct platform_driver qcom_qmp_phy_pcie_driver = { - .probe = qcom_qmp_phy_pcie_probe, +static struct platform_driver qmp_pcie_driver = { + .probe = qmp_pcie_probe, .driver = { .name = "qcom-qmp-pcie-phy", - .of_match_table = qcom_qmp_phy_pcie_of_match_table, + .of_match_table = qmp_pcie_of_match_table, }, }; -module_platform_driver(qcom_qmp_phy_pcie_driver); +module_platform_driver(qmp_pcie_driver); MODULE_AUTHOR("Vivek Gautam "); MODULE_DESCRIPTION("Qualcomm QMP PCIe PHY driver"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v5.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v5.h index 61a44519f969c27e260631d5823ffef9bbb1e85e..04f260711ea1cef1c52ee4c1b1f51e40c3a038c8 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v5.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-v5.h @@ -7,11 +7,24 @@ #define QCOM_PHY_QMP_PCS_V5_H_ /* Only for QMP V5 PHY - USB/PCIe PCS registers */ +#define QPHY_V5_PCS_LOCK_DETECT_CONFIG1 0x0c4 +#define QPHY_V5_PCS_LOCK_DETECT_CONFIG2 0x0c8 +#define QPHY_V5_PCS_LOCK_DETECT_CONFIG3 0x0cc +#define QPHY_V5_PCS_LOCK_DETECT_CONFIG6 0x0d8 #define QPHY_V5_PCS_REFGEN_REQ_CONFIG1 0x0dc #define QPHY_V5_PCS_G3S2_PRE_GAIN 0x170 #define QPHY_V5_PCS_RX_SIGDET_LVL 0x188 +#define QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_L 0x190 +#define QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_H 0x194 #define QPHY_V5_PCS_RATE_SLEW_CNTRL1 0x198 +#define QPHY_V5_PCS_CDR_RESET_TIME 0x1b0 +#define QPHY_V5_PCS_RX_CONFIG 0x1b0 +#define QPHY_V5_PCS_ALIGN_DETECT_CONFIG1 0x1c0 +#define QPHY_V5_PCS_ALIGN_DETECT_CONFIG2 0x1c4 +#define QPHY_V5_PCS_PCS_TX_RX_CONFIG 0x1d0 +#define QPHY_V5_PCS_EQ_CONFIG1 0x1dc #define QPHY_V5_PCS_EQ_CONFIG2 0x1e0 #define QPHY_V5_PCS_EQ_CONFIG3 0x1e4 +#define QPHY_V5_PCS_EQ_CONFIG5 0x1ec #endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v5_5nm.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v5_5nm.h new file mode 100644 index 0000000000000000000000000000000000000000..a1c088bd51581146dab74b81610b88382f41ac90 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v5_5nm.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_QSERDES_TXRX_V5_5NM_H_ +#define QCOM_PHY_QMP_QSERDES_TXRX_V5_5NM_H_ + +/* Only for QMP V5 5NM PHY - TX registers */ +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_TX 0x30 +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_RX 0x34 +#define QSERDES_V5_5NM_TX_LANE_MODE_1 0x78 +#define QSERDES_V5_5NM_TX_LANE_MODE_2 0x7c +#define QSERDES_V5_5NM_TX_LANE_MODE_3 0x80 +#define QSERDES_V5_5NM_TX_BIST_MODE_LANENO 0x00 +#define QSERDES_V5_5NM_TX_BIST_INVERT 0x04 +#define QSERDES_V5_5NM_TX_CLKBUF_ENABLE 0x08 +#define QSERDES_V5_5NM_TX_TX_EMP_POST1_LVL 0x0c +#define QSERDES_V5_5NM_TX_TX_IDLE_LVL_LARGE_AMP 0x10 +#define QSERDES_V5_5NM_TX_TX_DRV_LVL 0x14 +#define QSERDES_V5_5NM_TX_TX_DRV_LVL_OFFSET 0x18 +#define QSERDES_V5_5NM_TX_RESET_TSYNC_EN 0x1c +#define QSERDES_V5_5NM_TX_PRE_STALL_LDO_BOOST_EN 0x20 +#define QSERDES_V5_5NM_TX_LPB_EN 0x24 +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_TX 0x28 +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_RX 0x2c +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_TX 0x30 +#define QSERDES_V5_5NM_TX_RES_CODE_LANE_OFFSET_RX 0x34 +#define QSERDES_V5_5NM_TX_PERL_LENGTH1 0x38 +#define QSERDES_V5_5NM_TX_PERL_LENGTH2 0x3c +#define QSERDES_V5_5NM_TX_SERDES_BYP_EN_OUT 0x40 +#define QSERDES_V5_5NM_TX_DEBUG_BUS_SEL 0x44 +#define QSERDES_V5_5NM_TX_TRANSCEIVER_BIAS_EN 0x48 +#define QSERDES_V5_5NM_TX_HIGHZ_DRVR_EN 0x4c +#define QSERDES_V5_5NM_TX_TX_POL_INV 0x50 +#define QSERDES_V5_5NM_TX_PARRATE_REC_DETECT_IDLE_EN 0x54 +#define QSERDES_V5_5NM_TX_BIST_PATTERN1 0x58 +#define QSERDES_V5_5NM_TX_BIST_PATTERN2 0x5c +#define QSERDES_V5_5NM_TX_BIST_PATTERN3 0x60 +#define QSERDES_V5_5NM_TX_BIST_PATTERN4 0x64 +#define QSERDES_V5_5NM_TX_BIST_PATTERN5 0x68 +#define QSERDES_V5_5NM_TX_BIST_PATTERN6 0x6c +#define QSERDES_V5_5NM_TX_BIST_PATTERN7 0x70 +#define QSERDES_V5_5NM_TX_BIST_PATTERN8 0x74 +#define QSERDES_V5_5NM_TX_LANE_MODE_1 0x78 +#define QSERDES_V5_5NM_TX_LANE_MODE_2 0x7c +#define QSERDES_V5_5NM_TX_LANE_MODE_3 0x80 +#define QSERDES_V5_5NM_TX_ATB_SEL1 0x84 +#define QSERDES_V5_5NM_TX_ATB_SEL2 0x88 +#define QSERDES_V5_5NM_TX_RCV_DETECT_LVL 0x8c +#define QSERDES_V5_5NM_TX_RCV_DETECT_LVL_2 0x90 +#define QSERDES_V5_5NM_TX_PRBS_SEED1 0x94 +#define QSERDES_V5_5NM_TX_PRBS_SEED2 0x98 +#define QSERDES_V5_5NM_TX_PRBS_SEED3 0x9c +#define QSERDES_V5_5NM_TX_PRBS_SEED4 0xa0 +#define QSERDES_V5_5NM_TX_RESET_GEN 0xa4 +#define QSERDES_V5_5NM_TX_RESET_GEN_MUXES 0xa8 +#define QSERDES_V5_5NM_TX_TRAN_DRVR_EMP_EN 0xac +#define QSERDES_V5_5NM_TX_VMODE_CTRL1 0xb0 +#define QSERDES_V5_5NM_TX_ALOG_OBSV_BUS_CTRL_1 0xb4 +#define QSERDES_V5_5NM_TX_BIST_STATUS 0xb8 +#define QSERDES_V5_5NM_TX_BIST_ERROR_COUNT1 0xbc +#define QSERDES_V5_5NM_TX_BIST_ERROR_COUNT2 0xc0 +#define QSERDES_V5_5NM_TX_ALOG_OBSV_BUS_STATUS_1 0xc4 +#define QSERDES_V5_5NM_TX_LANE_DIG_CONFIG 0xc8 +#define QSERDES_V5_5NM_TX_PI_QEC_CTRL 0xcc +#define QSERDES_V5_5NM_TX_PRE_EMPH 0xd0 +#define QSERDES_V5_5NM_TX_SW_RESET 0xd4 +#define QSERDES_V5_5NM_TX_TX_BAND 0xd8 +#define QSERDES_V5_5NM_TX_SLEW_CNTL0 0xdc +#define QSERDES_V5_5NM_TX_SLEW_CNTL1 0xe0 +#define QSERDES_V5_5NM_TX_INTERFACE_SELECT 0xe4 +#define QSERDES_V5_5NM_TX_DIG_BKUP_CTRL 0xe8 +#define QSERDES_V5_5NM_TX_DEBUG_BUS0 0xec +#define QSERDES_V5_5NM_TX_DEBUG_BUS1 0xf0 +#define QSERDES_V5_5NM_TX_DEBUG_BUS2 0xf4 +#define QSERDES_V5_5NM_TX_DEBUG_BUS3 0xf8 +#define QSERDES_V5_5NM_TX_TX_BKUP_RO_BUS 0xfc + +/* Only for QMP V5 5NM PHY - RX registers */ +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_FO_GAIN_RATE0 0x000 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_FO_GAIN_RATE1 0x004 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_FO_GAIN_RATE2 0x008 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_FO_GAIN_RATE3 0x00c +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_SO_GAIN_RATE0 0x010 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_SO_GAIN_RATE1 0x014 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_SO_GAIN_RATE2 0x018 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_SO_GAIN_RATE3 0x01c +#define QSERDES_V5_5NM_RX_UCDR_SO_SATURATION 0x020 +#define QSERDES_V5_5NM_RX_UCDR_FO_TO_SO_DELAY 0x024 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_LOW_RATE0 0x028 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_HIGH_RATE0 0x02c +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_LOW_RATE1 0x030 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_HIGH_RATE1 0x034 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_LOW_RATE2 0x038 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_HIGH_RATE2 0x03c +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_LOW_RATE3 0x040 +#define QSERDES_V5_5NM_RX_UCDR_FASTLOCK_COUNT_HIGH_RATE3 0x044 +#define QSERDES_V5_5NM_RX_UCDR_PI_CTRL1 0x048 +#define QSERDES_V5_5NM_RX_UCDR_PI_CTRL2 0x04c +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH1_RATE0 0x050 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH1_RATE1 0x054 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH1_RATE2 0x058 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH1_RATE3 0x05c +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH2_RATE0 0x060 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH2_RATE1 0x064 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH2_RATE2 0x068 +#define QSERDES_V5_5NM_RX_UCDR_SB2_THRESH2_RATE3 0x06c +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN1_RATE0 0x070 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN1_RATE1 0x074 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN1_RATE2 0x078 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN1_RATE3 0x07c +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN2_RATE0 0x080 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN2_RATE1 0x084 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN2_RATE2 0x088 +#define QSERDES_V5_5NM_RX_UCDR_SB2_GAIN2_RATE3 0x08c +#define QSERDES_V5_5NM_RX_RXCLK_DIV2_CTRL 0x090 +#define QSERDES_V5_5NM_RX_RX_BAND 0x094 +#define QSERDES_V5_5NM_RX_RX_TERM_BW 0x098 +#define QSERDES_V5_5NM_RX_UCDR_FO_GAIN_RATE0 0x09c +#define QSERDES_V5_5NM_RX_UCDR_FO_GAIN_RATE1 0x0a0 +#define QSERDES_V5_5NM_RX_UCDR_FO_GAIN_RATE2 0x0a4 +#define QSERDES_V5_5NM_RX_UCDR_FO_GAIN_RATE3 0x0a8 +#define QSERDES_V5_5NM_RX_UCDR_SO_GAIN_RATE0 0x0ac +#define QSERDES_V5_5NM_RX_UCDR_SO_GAIN_RATE1 0x0b0 +#define QSERDES_V5_5NM_RX_UCDR_SO_GAIN_RATE2 0x0b4 +#define QSERDES_V5_5NM_RX_UCDR_SO_GAIN_RATE3 0x0b8 +#define QSERDES_V5_5NM_RX_UCDR_PI_CONTROLS 0x0bc +#define QSERDES_V5_5NM_RX_UCDR_PD_DATA_FILTER_ENABLES 0x0c0 +#define QSERDES_V5_5NM_RX_UCDR_SO_ACC_DEFAULT_VAL_RATE0 0x0c4 +#define QSERDES_V5_5NM_RX_UCDR_SO_ACC_DEFAULT_VAL_RATE1 0x0c8 +#define QSERDES_V5_5NM_RX_UCDR_SO_ACC_DEFAULT_VAL_RATE2 0x0cc +#define QSERDES_V5_5NM_RX_UCDR_SO_ACC_DEFAULT_VAL_RATE3 0x0d0 +#define QSERDES_V5_5NM_RX_AUX_CONTROL 0x0d4 +#define QSERDES_V5_5NM_RX_AUXDATA_TB 0x0d8 +#define QSERDES_V5_5NM_RX_RCLK_AUXDATA_SEL 0x0dc +#define QSERDES_V5_5NM_RX_EOM_CTRL 0x0e0 +#define QSERDES_V5_5NM_RX_AC_JTAG_ENABLE 0x0e4 +#define QSERDES_V5_5NM_RX_AC_JTAG_INITP 0x0e8 +#define QSERDES_V5_5NM_RX_AC_JTAG_INITN 0x0ec +#define QSERDES_V5_5NM_RX_AC_JTAG_LVL 0x0f0 +#define QSERDES_V5_5NM_RX_AC_JTAG_MODE 0x0f4 +#define QSERDES_V5_5NM_RX_AC_JTAG_RESET 0x0f8 +#define QSERDES_V5_5NM_RX_RX_RCVR_IQ_EN 0x0fc +#define QSERDES_V5_5NM_RX_RX_Q_EN_RATES 0x100 +#define QSERDES_V5_5NM_RX_RX_IDAC_I0_DC_OFFSETS 0x104 +#define QSERDES_V5_5NM_RX_RX_IDAC_I0BAR_DC_OFFSETS 0x108 +#define QSERDES_V5_5NM_RX_RX_IDAC_I1_DC_OFFSETS 0x10c +#define QSERDES_V5_5NM_RX_RX_IDAC_I1BAR_DC_OFFSETS 0x110 +#define QSERDES_V5_5NM_RX_RX_IDAC_Q_DC_OFFSETS 0x114 +#define QSERDES_V5_5NM_RX_RX_IDAC_QBAR_DC_OFFSETS 0x118 +#define QSERDES_V5_5NM_RX_RX_IDAC_A_DC_OFFSETS 0x11c +#define QSERDES_V5_5NM_RX_RX_IDAC_ABAR_DC_OFFSETS 0x120 +#define QSERDES_V5_5NM_RX_RX_IDAC_EN 0x124 +#define QSERDES_V5_5NM_RX_RX_IDAC_ENABLES 0x128 +#define QSERDES_V5_5NM_RX_RX_IDAC_SIGN 0x12c +#define QSERDES_V5_5NM_RX_RX_IVCM_CAL_CODE_OVERRIDE 0x130 +#define QSERDES_V5_5NM_RX_RX_IVCM_CAL_CTRL1 0x134 +#define QSERDES_V5_5NM_RX_RX_IVCM_CAL_CTRL2 0x138 +#define QSERDES_V5_5NM_RX_RX_IVCM_POSTCAL_OFFSET 0x13c +#define QSERDES_V5_5NM_RX_RX_SUMMER_CAL_SPD_MODE 0x140 +#define QSERDES_V5_5NM_RX_RX_HIGHZ_PARRATE 0x144 +#define QSERDES_V5_5NM_RX_RX_TERM_AC_BYPASS_DC_COUPLE_OFFSET 0x148 +#define QSERDES_V5_5NM_RX_DFE_1 0x14c +#define QSERDES_V5_5NM_RX_DFE_2 0x150 +#define QSERDES_V5_5NM_RX_DFE_3 0x154 +#define QSERDES_V5_5NM_RX_DFE_4 0x158 +#define QSERDES_V5_5NM_RX_DFE_TAP3_CTRL 0x15c +#define QSERDES_V5_5NM_RX_DFE_TAP3_MANVAL_KTAP 0x160 +#define QSERDES_V5_5NM_RX_DFE_TAP4_CTRL 0x164 +#define QSERDES_V5_5NM_RX_DFE_TAP4_MANVAL_KTAP 0x168 +#define QSERDES_V5_5NM_RX_DFE_TAP5_CTRL 0x16c +#define QSERDES_V5_5NM_RX_DFE_TAP5_MANVAL_KTAP 0x170 +#define QSERDES_V5_5NM_RX_TX_ADPT_CTRL 0x174 +#define QSERDES_V5_5NM_RX_DFE_DAC_ENABLE1 0x178 +#define QSERDES_V5_5NM_RX_DFE_DAC_ENABLE2 0x17c +#define QSERDES_V5_5NM_RX_TX_ADAPT_PRE_THRESH1 0x180 +#define QSERDES_V5_5NM_RX_TX_ADAPT_PRE_THRESH2 0x184 +#define QSERDES_V5_5NM_RX_TX_ADAPT_POST_THRESH1 0x188 +#define QSERDES_V5_5NM_RX_TX_ADAPT_POST_THRESH2 0x18c +#define QSERDES_V5_5NM_RX_TX_ADAPT_MAIN_THRESH1 0x190 +#define QSERDES_V5_5NM_RX_TX_ADAPT_MAIN_THRESH2 0x194 +#define QSERDES_V5_5NM_RX_VGA_CAL_CNTRL1 0x198 +#define QSERDES_V5_5NM_RX_VGA_CAL_CNTRL2 0x19c +#define QSERDES_V5_5NM_RX_VGA_CAL_MAN_VAL 0x1a0 +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_CNTRL1 0x1a4 +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_CNTRL2 0x1a8 +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_MAN_VAL_RATE0 0x1ac +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_MAN_VAL_RATE1 0x1b0 +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_MAN_VAL_RATE2 0x1b4 +#define QSERDES_V5_5NM_RX_VTHRESH_CAL_MAN_VAL_RATE3 0x1b8 +#define QSERDES_V5_5NM_RX_GM_CAL 0x1bc +#define QSERDES_V5_5NM_RX_RX_VGA_GAIN2_BLK1 0x1c0 +#define QSERDES_V5_5NM_RX_RX_VGA_GAIN2_BLK2 0x1c4 +#define QSERDES_V5_5NM_RX_RX_EQU_ADAPTOR_CNTRL2 0x1c8 +#define QSERDES_V5_5NM_RX_RX_EQU_ADAPTOR_CNTRL3 0x1cc +#define QSERDES_V5_5NM_RX_RX_EQU_ADAPTOR_CNTRL4 0x1d0 +#define QSERDES_V5_5NM_RX_RX_IDAC_TSETTLE_LOW 0x1d4 +#define QSERDES_V5_5NM_RX_RX_EQ_OFFSET_LSB 0x1d8 +#define QSERDES_V5_5NM_RX_RX_EQ_OFFSET_MSB 0x1dc +#define QSERDES_V5_5NM_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x1e0 +#define QSERDES_V5_5NM_RX_RX_OFFSET_ADAPTOR_CNTRL2 0x1e4 +#define QSERDES_V5_5NM_RX_SIGDET_ENABLES 0x1e8 +#define QSERDES_V5_5NM_RX_SIGDET_CNTRL 0x1ec +#define QSERDES_V5_5NM_RX_SIGDET_LVL 0x1f0 +#define QSERDES_V5_5NM_RX_SIGDET_DEGLITCH_CNTRL 0x1f4 +#define QSERDES_V5_5NM_RX_CDR_FREEZE_UP_DN 0x1f8 +#define QSERDES_V5_5NM_RX_CDR_RESET_OVERRIDE 0x1fc +#define QSERDES_V5_5NM_RX_RX_INTERFACE_MODE 0x200 +#define QSERDES_V5_5NM_RX_JITTER_GEN_MODE 0x204 +#define QSERDES_V5_5NM_RX_SJ_AMP1 0x208 +#define QSERDES_V5_5NM_RX_SJ_AMP2 0x20c +#define QSERDES_V5_5NM_RX_SJ_PER1 0x210 +#define QSERDES_V5_5NM_RX_SJ_PER2 0x214 +#define QSERDES_V5_5NM_RX_PPM_OFFSET1 0x218 +#define QSERDES_V5_5NM_RX_PPM_OFFSET2 0x21c +#define QSERDES_V5_5NM_RX_SIGN_PPM_PERIOD1 0x220 +#define QSERDES_V5_5NM_RX_SIGN_PPM_PERIOD2 0x224 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B0 0x228 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B1 0x22c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B2 0x230 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B3 0x234 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B4 0x238 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B5 0x23c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B6 0x240 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE_0_1_B7 0x244 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B0 0x248 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B1 0x24c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B2 0x250 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B3 0x254 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B4 0x258 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B5 0x25c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B6 0x260 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE2_B7 0x264 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B0 0x268 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B1 0x26c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B2 0x270 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B3 0x274 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B4 0x278 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B5 0x27c +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B6 0x280 +#define QSERDES_V5_5NM_RX_RX_MODE_RATE3_B7 0x284 +#define QSERDES_V5_5NM_RX_PHPRE_CTRL 0x288 +#define QSERDES_V5_5NM_RX_PHPRE_INITVAL 0x28c +#define QSERDES_V5_5NM_RX_DFE_EN_TIMER 0x290 +#define QSERDES_V5_5NM_RX_DFE_CTLE_POST_CAL_OFFSET 0x294 +#define QSERDES_V5_5NM_RX_DCC_CTRL1 0x298 +#define QSERDES_V5_5NM_RX_DCC_CTRL2 0x29c +#define QSERDES_V5_5NM_RX_DCC_OFFSET 0x2a0 +#define QSERDES_V5_5NM_RX_DCC_CMUX_POSTCAL_OFFSET 0x2a4 +#define QSERDES_V5_5NM_RX_DCC_CMUX_CAL_CTRL1 0x2a8 +#define QSERDES_V5_5NM_RX_DCC_CMUX_CAL_CTRL2 0x2ac +#define QSERDES_V5_5NM_RX_ALOG_OBSV_BUS_CTRL_1 0x2b0 +#define QSERDES_V5_5NM_RX_RX_MARG_CTRL1 0x2b4 +#define QSERDES_V5_5NM_RX_RX_MARG_CTRL2 0x2b8 +#define QSERDES_V5_5NM_RX_RX_MARG_CTRL3 0x2bc +#define QSERDES_V5_5NM_RX_RX_MARG_CTRL_4 0x2c0 +#define QSERDES_V5_5NM_RX_RX_MARG_CFG_RATE_0_1 0x2c4 +#define QSERDES_V5_5NM_RX_RX_MARG_CFG_RATE_2_3 0x2c8 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_CTRL1 0x2cc +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_CTRL2 0x2d0 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH1_RATE210 0x2d4 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH1_RATE3 0x2d8 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH2_RATE210 0x2dc +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH2_RATE3 0x2e0 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH3_RATE210 0x2e4 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH3_RATE3 0x2e8 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH4_RATE210 0x2ec +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH4_RATE3 0x2f0 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH5_RATE210 0x2f4 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH5_RATE3 0x2f8 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH6_RATE210 0x2fc +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH6_RATE3 0x300 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH7_RATE210 0x304 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_THRESH7_RATE3 0x308 +#define QSERDES_V5_5NM_RX_Q_PI_INTRINSIC_BIAS_RATE10 0x30c +#define QSERDES_V5_5NM_RX_Q_PI_INTRINSIC_BIAS_RATE32 0x310 +#define QSERDES_V5_5NM_RX_RX_MARG_VERTICAL_CTRL 0x314 +#define QSERDES_V5_5NM_RX_RX_MARG_VERTICAL_CODE 0x318 +#define QSERDES_V5_5NM_RX_RES_CODE_THRESH_HIGH_AND_BYP 0x31c +#define QSERDES_V5_5NM_RX_RES_CODE_THRESH_LOW 0x320 +#define QSERDES_V5_5NM_RX_RX_BKUP_CTRL1 0x324 +#define QSERDES_V5_5NM_RX_RX_BKUP_CTRL2 0x328 +#define QSERDES_V5_5NM_RX_RX_BKUP_CTRL3 0x32c +#define QSERDES_V5_5NM_RX_PI_CTRL1 0x330 +#define QSERDES_V5_5NM_RX_PI_CTRL2 0x334 +#define QSERDES_V5_5NM_RX_PI_QUAD 0x338 +#define QSERDES_V5_5NM_RX_QPI_CTRL1 0x33c +#define QSERDES_V5_5NM_RX_QPI_CTRL2 0x340 +#define QSERDES_V5_5NM_RX_QPI_QUAD 0x344 +#define QSERDES_V5_5NM_RX_IDATA1 0x348 +#define QSERDES_V5_5NM_RX_IDATA2 0x34c +#define QSERDES_V5_5NM_RX_IDATA3 0x350 +#define QSERDES_V5_5NM_RX_AC_JTAG_OUTP 0x354 +#define QSERDES_V5_5NM_RX_AC_JTAG_OUTN 0x358 +#define QSERDES_V5_5NM_RX_RX_SIGDET 0x35c +#define QSERDES_V5_5NM_RX_ALOG_OBSV_BUS_STATUS_1 0x360 +#define QSERDES_V5_5NM_RX_READ_EQCODE 0x364 +#define QSERDES_V5_5NM_RX_READ_OFFSETCODE 0x368 +#define QSERDES_V5_5NM_RX_IA_ERROR_COUNTER_LOW 0x36c +#define QSERDES_V5_5NM_RX_IA_ERROR_COUNTER_HIGH 0x370 +#define QSERDES_V5_5NM_RX_VGA_READ_CODE 0x374 +#define QSERDES_V5_5NM_RX_VTHRESH_READ_CODE 0x378 +#define QSERDES_V5_5NM_RX_DFE_TAP1_READ_CODE 0x37c +#define QSERDES_V5_5NM_RX_DFE_TAP2_READ_CODE 0x380 +#define QSERDES_V5_5NM_RX_DFE_TAP3_READ_CODE 0x384 +#define QSERDES_V5_5NM_RX_DFE_TAP4_READ_CODE 0x388 +#define QSERDES_V5_5NM_RX_DFE_TAP5_READ_CODE 0x38c +#define QSERDES_V5_5NM_RX_IDAC_STATUS_I0 0x390 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_I0BAR 0x394 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_I1 0x398 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_I1BAR 0x39c +#define QSERDES_V5_5NM_RX_IDAC_STATUS_Q 0x3a0 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_QBAR 0x3a4 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_A 0x3a8 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_ABAR 0x3ac +#define QSERDES_V5_5NM_RX_IDAC_STATUS_SM_ON 0x3b0 +#define QSERDES_V5_5NM_RX_IDAC_STATUS_SIGNERROR 0x3b4 +#define QSERDES_V5_5NM_RX_IVCM_CAL_STATUS 0x3b8 +#define QSERDES_V5_5NM_RX_IVCM_CAL_DEBUG_STATUS 0x3bc +#define QSERDES_V5_5NM_RX_DCC_CAL_STATUS 0x3c0 +#define QSERDES_V5_5NM_RX_DCC_READ_CODE_STATUS 0x3c4 +#define QSERDES_V5_5NM_RX_RX_MARG_DEBUG1_STATUS 0x3c8 +#define QSERDES_V5_5NM_RX_RX_MARG_DEBUG2_STATUS 0x3cc +#define QSERDES_V5_5NM_RX_RX_MARG_READ_CODE_STATUS 0x3d0 +#define QSERDES_V5_5NM_RX_EOM_ERR_CNT_LSB_STATUS 0x3d4 +#define QSERDES_V5_5NM_RX_EOM_ERR_CNT_MSB_STATUS 0x3d8 +#define QSERDES_V5_5NM_RX_RX_MARG_COARSE_TUNE_STATUS 0x3dc +#define QSERDES_V5_5NM_RX_RX_BKUP_READ_BUS1_STATUS 0x3e0 +#define QSERDES_V5_5NM_RX_RX_BKUP_READ_BUS2_STATUS 0x3e4 +#define QSERDES_V5_5NM_RX_RX_BKUP_READ_BUS3_STATUS 0x3e8 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c index c8583f5a54bdbce9ead6afae52e5e5b2692515ab..c08d34ad1313e77db98773debe32d97f1875404d 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c @@ -28,53 +28,15 @@ #define SW_RESET BIT(0) /* QPHY_POWER_DOWN_CONTROL */ #define SW_PWRDN BIT(0) -#define REFCLK_DRV_DSBL BIT(1) /* QPHY_START_CONTROL bits */ #define SERDES_START BIT(0) #define PCS_START BIT(1) -#define PLL_READY_GATE_EN BIT(3) /* QPHY_PCS_STATUS bit */ #define PHYSTATUS BIT(6) -#define PHYSTATUS_4_20 BIT(7) -/* QPHY_PCS_READY_STATUS & QPHY_COM_PCS_READY_STATUS bit */ +/* QPHY_PCS_READY_STATUS bit */ #define PCS_READY BIT(0) -/* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ -/* DP PHY soft reset */ -#define SW_DPPHY_RESET BIT(0) -/* mux to select DP PHY reset control, 0:HW control, 1: software reset */ -#define SW_DPPHY_RESET_MUX BIT(1) -/* USB3 PHY soft reset */ -#define SW_USB3PHY_RESET BIT(2) -/* mux to select USB3 PHY reset control, 0:HW control, 1: software reset */ -#define SW_USB3PHY_RESET_MUX BIT(3) - -/* QPHY_V3_DP_COM_PHY_MODE_CTRL register bits */ -#define USB3_MODE BIT(0) /* enables USB3 mode */ -#define DP_MODE BIT(1) /* enables DP mode */ - -/* QPHY_PCS_AUTONOMOUS_MODE_CTRL register bits */ -#define ARCVR_DTCT_EN BIT(0) -#define ALFPS_DTCT_EN BIT(1) -#define ARCVR_DTCT_EVENT_SEL BIT(4) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR register bits */ -#define IRQ_CLEAR BIT(0) - -/* QPHY_PCS_LFPS_RXTERM_IRQ_STATUS register bits */ -#define RCVR_DETECT BIT(0) - -/* QPHY_V3_PCS_MISC_CLAMP_ENABLE register bits */ -#define CLAMP_EN BIT(0) /* enables i/o clamp_n */ - #define PHY_INIT_COMPLETE_TIMEOUT 10000 -#define POWER_DOWN_DELAY_US_MIN 10 -#define POWER_DOWN_DELAY_US_MAX 11 - -#define MAX_PROP_NAME 32 - -/* Define the assumed distance between lanes for underspecified device trees. */ -#define QMP_PHY_LEGACY_LANE_STRIDE 0x400 struct qmp_phy_init_tbl { unsigned int offset; @@ -115,22 +77,11 @@ struct qmp_phy_init_tbl { /* set of registers with offsets different per-PHY */ enum qphy_reg_layout { - /* Common block control registers */ - QPHY_COM_SW_RESET, - QPHY_COM_POWER_DOWN_CONTROL, - QPHY_COM_START_CONTROL, - QPHY_COM_PCS_READY_STATUS, /* PCS registers */ QPHY_SW_RESET, QPHY_START_CTRL, QPHY_PCS_READY_STATUS, - QPHY_PCS_STATUS, - QPHY_PCS_AUTONOMOUS_MODE_CTRL, - QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, - QPHY_PCS_LFPS_RXTERM_IRQ_STATUS, QPHY_PCS_POWER_DOWN_CONTROL, - /* PCS_MISC registers */ - QPHY_PCS_MISC_TYPEC_CTRL, /* Keep last to ensure regs_layout arrays are properly initialized */ QPHY_LAYOUT_SIZE }; @@ -580,14 +531,9 @@ static const struct qmp_phy_init_tbl sm8350_ufsphy_pcs_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V5_PCS_UFS_MULTI_LANE_CTRL1, 0x02), }; -struct qmp_phy; - /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { - /* phy-type - PCIE/UFS/USB */ - unsigned int type; - /* number of lanes provided by phy */ - int nlanes; + int lanes; /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; @@ -614,9 +560,6 @@ struct qmp_phy_cfg { /* bit offset of PHYSTATUS in QPHY_PCS_STATUS register */ unsigned int phy_status; - /* true, if PHY has secondary tx/rx lanes to be configured */ - bool is_dual_lane_phy; - /* true, if PCS block has no separate SW_RESET register */ bool no_pcs_sw_reset; }; @@ -633,9 +576,7 @@ struct qmp_phy_cfg { * @tx2: iomapped memory space for second lane's tx (in dual lane PHYs) * @rx2: iomapped memory space for second lane's rx (in dual lane PHYs) * @pcs_misc: iomapped memory space for lane's pcs_misc - * @index: lane index * @qmp: QMP phy to which this lane belongs - * @mode: current PHY mode */ struct qmp_phy { struct phy *phy; @@ -647,9 +588,7 @@ struct qmp_phy { void __iomem *tx2; void __iomem *rx2; void __iomem *pcs_misc; - unsigned int index; struct qcom_qmp *qmp; - enum phy_mode mode; }; /** @@ -719,8 +658,7 @@ static const char * const qmp_phy_vreg_l[] = { }; static const struct qmp_phy_cfg msm8996_ufs_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 1, + .lanes = 1, .serdes_tbl = msm8996_ufs_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(msm8996_ufs_serdes_tbl), @@ -745,8 +683,7 @@ static const struct qmp_phy_cfg msm8996_ufs_cfg = { }; static const struct qmp_phy_cfg sdm845_ufsphy_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sdm845_ufsphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sdm845_ufsphy_serdes_tbl), @@ -766,13 +703,11 @@ static const struct qmp_phy_cfg sdm845_ufsphy_cfg = { .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - .is_dual_lane_phy = true, .no_pcs_sw_reset = true, }; static const struct qmp_phy_cfg sm6115_ufsphy_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm6115_ufsphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm6115_ufsphy_serdes_tbl), @@ -795,8 +730,7 @@ static const struct qmp_phy_cfg sm6115_ufsphy_cfg = { }; static const struct qmp_phy_cfg sm8150_ufsphy_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sm8150_ufsphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_ufsphy_serdes_tbl), @@ -815,13 +749,10 @@ static const struct qmp_phy_cfg sm8150_ufsphy_cfg = { .start_ctrl = SERDES_START, .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8350_ufsphy_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sm8350_ufsphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8350_ufsphy_serdes_tbl), @@ -840,13 +771,10 @@ static const struct qmp_phy_cfg sm8350_ufsphy_cfg = { .start_ctrl = SERDES_START, .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8450_ufsphy_cfg = { - .type = PHY_TYPE_UFS, - .nlanes = 2, + .lanes = 2, .serdes_tbl = sm8350_ufsphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8350_ufsphy_serdes_tbl), @@ -865,11 +793,9 @@ static const struct qmp_phy_cfg sm8450_ufsphy_cfg = { .start_ctrl = SERDES_START, .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - - .is_dual_lane_phy = true, }; -static void qcom_qmp_phy_ufs_configure_lane(void __iomem *base, +static void qmp_ufs_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num, @@ -892,27 +818,27 @@ static void qcom_qmp_phy_ufs_configure_lane(void __iomem *base, } } -static void qcom_qmp_phy_ufs_configure(void __iomem *base, +static void qmp_ufs_configure(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num) { - qcom_qmp_phy_ufs_configure_lane(base, regs, tbl, num, 0xff); + qmp_ufs_configure_lane(base, regs, tbl, num, 0xff); } -static int qcom_qmp_phy_ufs_serdes_init(struct qmp_phy *qphy) +static int qmp_ufs_serdes_init(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *serdes = qphy->serdes; const struct qmp_phy_init_tbl *serdes_tbl = cfg->serdes_tbl; int serdes_tbl_num = cfg->serdes_tbl_num; - qcom_qmp_phy_ufs_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); + qmp_ufs_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); return 0; } -static int qcom_qmp_phy_ufs_com_init(struct qmp_phy *qphy) +static int qmp_ufs_com_init(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -946,7 +872,7 @@ err_disable_regulators: return ret; } -static int qcom_qmp_phy_ufs_com_exit(struct qmp_phy *qphy) +static int qmp_ufs_com_exit(struct qmp_phy *qphy) { struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -960,7 +886,7 @@ static int qcom_qmp_phy_ufs_com_exit(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_ufs_init(struct phy *phy) +static int qmp_ufs_init(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -995,14 +921,14 @@ static int qcom_qmp_phy_ufs_init(struct phy *phy) return ret; } - ret = qcom_qmp_phy_ufs_com_init(qphy); + ret = qmp_ufs_com_init(qphy); if (ret) return ret; return 0; } -static int qcom_qmp_phy_ufs_power_on(struct phy *phy) +static int qmp_ufs_power_on(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -1014,27 +940,24 @@ static int qcom_qmp_phy_ufs_power_on(struct phy *phy) unsigned int mask, val, ready; int ret; - qcom_qmp_phy_ufs_serdes_init(qphy); + qmp_ufs_serdes_init(qphy); /* Tx, Rx, and PCS configurations */ - qcom_qmp_phy_ufs_configure_lane(tx, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 1); + qmp_ufs_configure_lane(tx, cfg->regs, cfg->tx_tbl, cfg->tx_tbl_num, 1); - /* Configuration for other LANE for USB-DP combo PHY */ - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_ufs_configure_lane(qphy->tx2, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_ufs_configure_lane(qphy->tx2, cfg->regs, + cfg->tx_tbl, cfg->tx_tbl_num, 2); } - qcom_qmp_phy_ufs_configure_lane(rx, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 1); + qmp_ufs_configure_lane(rx, cfg->regs, cfg->rx_tbl, cfg->rx_tbl_num, 1); - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_ufs_configure_lane(qphy->rx2, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_ufs_configure_lane(qphy->rx2, cfg->regs, + cfg->rx_tbl, cfg->rx_tbl_num, 2); } - qcom_qmp_phy_ufs_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); + qmp_ufs_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); ret = reset_control_deassert(qmp->ufs_reset); if (ret) @@ -1060,7 +983,7 @@ static int qcom_qmp_phy_ufs_power_on(struct phy *phy) return 0; } -static int qcom_qmp_phy_ufs_power_off(struct phy *phy) +static int qmp_ufs_power_off(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -1084,51 +1007,41 @@ static int qcom_qmp_phy_ufs_power_off(struct phy *phy) return 0; } -static int qcom_qmp_phy_ufs_exit(struct phy *phy) +static int qmp_ufs_exit(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); - qcom_qmp_phy_ufs_com_exit(qphy); + qmp_ufs_com_exit(qphy); return 0; } -static int qcom_qmp_phy_ufs_enable(struct phy *phy) +static int qmp_ufs_enable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_ufs_init(phy); + ret = qmp_ufs_init(phy); if (ret) return ret; - ret = qcom_qmp_phy_ufs_power_on(phy); + ret = qmp_ufs_power_on(phy); if (ret) - qcom_qmp_phy_ufs_exit(phy); + qmp_ufs_exit(phy); return ret; } -static int qcom_qmp_phy_ufs_disable(struct phy *phy) +static int qmp_ufs_disable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_ufs_power_off(phy); + ret = qmp_ufs_power_off(phy); if (ret) return ret; - return qcom_qmp_phy_ufs_exit(phy); + return qmp_ufs_exit(phy); } -static int qcom_qmp_phy_ufs_set_mode(struct phy *phy, - enum phy_mode mode, int submode) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - - qphy->mode = mode; - - return 0; -} - -static int qcom_qmp_phy_ufs_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_ufs_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_vregs; @@ -1144,7 +1057,7 @@ static int qcom_qmp_phy_ufs_vreg_init(struct device *dev, const struct qmp_phy_c return devm_regulator_bulk_get(dev, num, qmp->vregs); } -static int qcom_qmp_phy_ufs_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_ufs_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_clks; @@ -1161,14 +1074,12 @@ static int qcom_qmp_phy_ufs_clk_init(struct device *dev, const struct qmp_phy_cf } static const struct phy_ops qcom_qmp_ufs_ops = { - .power_on = qcom_qmp_phy_ufs_enable, - .power_off = qcom_qmp_phy_ufs_disable, - .set_mode = qcom_qmp_phy_ufs_set_mode, + .power_on = qmp_ufs_enable, + .power_off = qmp_ufs_disable, .owner = THIS_MODULE, }; -static -int qcom_qmp_phy_ufs_create(struct device *dev, struct device_node *np, int id, +static int qmp_ufs_create(struct device *dev, struct device_node *np, int id, void __iomem *serdes, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); @@ -1188,45 +1099,33 @@ int qcom_qmp_phy_ufs_create(struct device *dev, struct device_node *np, int id, * For dual lane PHYs: tx2 -> 3, rx2 -> 4, pcs_misc (optional) -> 5 * For single lane PHYs: pcs_misc (optional) -> 3. */ - qphy->tx = of_iomap(np, 0); - if (!qphy->tx) - return -ENOMEM; + qphy->tx = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(qphy->tx)) + return PTR_ERR(qphy->tx); - qphy->rx = of_iomap(np, 1); - if (!qphy->rx) - return -ENOMEM; + qphy->rx = devm_of_iomap(dev, np, 1, NULL); + if (IS_ERR(qphy->rx)) + return PTR_ERR(qphy->rx); - qphy->pcs = of_iomap(np, 2); - if (!qphy->pcs) - return -ENOMEM; + qphy->pcs = devm_of_iomap(dev, np, 2, NULL); + if (IS_ERR(qphy->pcs)) + return PTR_ERR(qphy->pcs); - /* - * If this is a dual-lane PHY, then there should be registers for the - * second lane. Some old device trees did not specify this, so fall - * back to old legacy behavior of assuming they can be reached at an - * offset from the first lane. - */ - if (cfg->is_dual_lane_phy) { - qphy->tx2 = of_iomap(np, 3); - qphy->rx2 = of_iomap(np, 4); - if (!qphy->tx2 || !qphy->rx2) { - dev_warn(dev, - "Underspecified device tree, falling back to legacy register regions\n"); - - /* In the old version, pcs_misc is at index 3. */ - qphy->pcs_misc = qphy->tx2; - qphy->tx2 = qphy->tx + QMP_PHY_LEGACY_LANE_STRIDE; - qphy->rx2 = qphy->rx + QMP_PHY_LEGACY_LANE_STRIDE; - - } else { - qphy->pcs_misc = of_iomap(np, 5); - } + if (cfg->lanes >= 2) { + qphy->tx2 = devm_of_iomap(dev, np, 3, NULL); + if (IS_ERR(qphy->tx2)) + return PTR_ERR(qphy->tx2); + + qphy->rx2 = devm_of_iomap(dev, np, 4, NULL); + if (IS_ERR(qphy->rx2)) + return PTR_ERR(qphy->rx2); + qphy->pcs_misc = devm_of_iomap(dev, np, 5, NULL); } else { - qphy->pcs_misc = of_iomap(np, 3); + qphy->pcs_misc = devm_of_iomap(dev, np, 3, NULL); } - if (!qphy->pcs_misc) + if (IS_ERR(qphy->pcs_misc)) dev_vdbg(dev, "PHY pcs_misc-reg not used\n"); generic_phy = devm_phy_create(dev, np, &qcom_qmp_ufs_ops); @@ -1237,7 +1136,6 @@ int qcom_qmp_phy_ufs_create(struct device *dev, struct device_node *np, int id, } qphy->phy = generic_phy; - qphy->index = id; qphy->qmp = qmp; qmp->phys[id] = qphy; phy_set_drvdata(generic_phy, qphy); @@ -1245,7 +1143,7 @@ int qcom_qmp_phy_ufs_create(struct device *dev, struct device_node *np, int id, return 0; } -static const struct of_device_id qcom_qmp_phy_ufs_of_match_table[] = { +static const struct of_device_id qmp_ufs_of_match_table[] = { { .compatible = "qcom,msm8996-qmp-ufs-phy", .data = &msm8996_ufs_cfg, @@ -1282,9 +1180,9 @@ static const struct of_device_id qcom_qmp_phy_ufs_of_match_table[] = { }, { }, }; -MODULE_DEVICE_TABLE(of, qcom_qmp_phy_ufs_of_match_table); +MODULE_DEVICE_TABLE(of, qmp_ufs_of_match_table); -static int qcom_qmp_phy_ufs_probe(struct platform_device *pdev) +static int qmp_ufs_probe(struct platform_device *pdev) { struct qcom_qmp *qmp; struct device *dev = &pdev->dev; @@ -1312,17 +1210,14 @@ static int qcom_qmp_phy_ufs_probe(struct platform_device *pdev) if (IS_ERR(serdes)) return PTR_ERR(serdes); - ret = qcom_qmp_phy_ufs_clk_init(dev, cfg); + ret = qmp_ufs_clk_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_ufs_vreg_init(dev, cfg); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + ret = qmp_ufs_vreg_init(dev, cfg); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); num = of_get_available_child_count(dev->of_node); /* do we have a rogue child node ? */ @@ -1333,18 +1228,10 @@ static int qcom_qmp_phy_ufs_probe(struct platform_device *pdev) if (!qmp->phys) return -ENOMEM; - pm_runtime_set_active(dev); - pm_runtime_enable(dev); - /* - * Prevent runtime pm from being ON by default. Users can enable - * it using power/control in sysfs. - */ - pm_runtime_forbid(dev); - id = 0; for_each_available_child_of_node(dev->of_node, child) { /* Create per-lane phy */ - ret = qcom_qmp_phy_ufs_create(dev, child, id, serdes, cfg); + ret = qmp_ufs_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -1355,28 +1242,23 @@ static int qcom_qmp_phy_ufs_probe(struct platform_device *pdev) } phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_info(dev, "Registered Qcom-QMP phy\n"); - else - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); err_node_put: - pm_runtime_disable(dev); of_node_put(child); return ret; } -static struct platform_driver qcom_qmp_phy_ufs_driver = { - .probe = qcom_qmp_phy_ufs_probe, +static struct platform_driver qmp_ufs_driver = { + .probe = qmp_ufs_probe, .driver = { .name = "qcom-qmp-ufs-phy", - .of_match_table = qcom_qmp_phy_ufs_of_match_table, + .of_match_table = qmp_ufs_of_match_table, }, }; -module_platform_driver(qcom_qmp_phy_ufs_driver); +module_platform_driver(qmp_ufs_driver); MODULE_AUTHOR("Vivek Gautam "); MODULE_DESCRIPTION("Qualcomm QMP UFS PHY driver"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index 1d270356a97f6720af2835345f0e623cf58dc8cb..b84c0d4b57541644e96dc0b567f724a1775ff725 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -28,16 +28,11 @@ #define SW_RESET BIT(0) /* QPHY_POWER_DOWN_CONTROL */ #define SW_PWRDN BIT(0) -#define REFCLK_DRV_DSBL BIT(1) /* QPHY_START_CONTROL bits */ #define SERDES_START BIT(0) #define PCS_START BIT(1) -#define PLL_READY_GATE_EN BIT(3) /* QPHY_PCS_STATUS bit */ #define PHYSTATUS BIT(6) -#define PHYSTATUS_4_20 BIT(7) -/* QPHY_PCS_READY_STATUS & QPHY_COM_PCS_READY_STATUS bit */ -#define PCS_READY BIT(0) /* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ /* DP PHY soft reset */ @@ -71,11 +66,6 @@ #define POWER_DOWN_DELAY_US_MIN 10 #define POWER_DOWN_DELAY_US_MAX 11 -#define MAX_PROP_NAME 32 - -/* Define the assumed distance between lanes for underspecified device trees. */ -#define QMP_PHY_LEGACY_LANE_STRIDE 0x400 - struct qmp_phy_init_tbl { unsigned int offset; unsigned int val; @@ -115,15 +105,9 @@ struct qmp_phy_init_tbl { /* set of registers with offsets different per-PHY */ enum qphy_reg_layout { - /* Common block control registers */ - QPHY_COM_SW_RESET, - QPHY_COM_POWER_DOWN_CONTROL, - QPHY_COM_START_CONTROL, - QPHY_COM_PCS_READY_STATUS, /* PCS registers */ QPHY_SW_RESET, QPHY_START_CTRL, - QPHY_PCS_READY_STATUS, QPHY_PCS_STATUS, QPHY_PCS_AUTONOMOUS_MODE_CTRL, QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, @@ -1338,14 +1322,114 @@ static const struct qmp_phy_init_tbl qcm2290_usb3_pcs_tbl[] = { QMP_PHY_INIT_CFG(QPHY_V3_PCS_RX_SIGDET_LVL, 0x88), }; -struct qmp_phy; +static const struct qmp_phy_init_tbl sc8280xp_usb3_uniphy_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_EN_SEL, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_HSCLK_SEL, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_HSCLK_SEL, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE0, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START1_MODE0, 0xab), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START2_MODE0, 0xea), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START3_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0xca), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE1_MODE0, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE0, 0x34), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE0, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP_EN, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SYSCLK_BUF_ENABLE, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE2_MODE1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE1_MODE1, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CORECLK_DIV_MODE1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DEC_START_MODE1, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START1_MODE1, 0xab), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START2_MODE1, 0xea), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_DIV_FRAC_START3_MODE1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP2_MODE1, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_LOCK_CMP1_MODE1, 0x34), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_CP_CTRL_MODE1, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_RCTRL_MODE1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_PLL_CCTRL_MODE1, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE1_MODE1, 0xca), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_BIN_VCOCAL_CMP_CODE2_MODE1, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_EN_CENTER, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_PER1, 0x31), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_PER2, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE1_MODE1, 0xde), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE1, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE1_MODE0, 0xde), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_STEP_SIZE2_MODE0, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V5_COM_VCO_TUNE_MAP, 0x02), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb3_uniphy_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_1, 0xa5), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_2, 0x82), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_3, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_LANE_MODE_4, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_PI_QEC_CTRL, 0x21), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_TX, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V5_TX_RES_CODE_LANE_OFFSET_RX, 0x0e), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb3_uniphy_rx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH4, 0xdc), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH3, 0xbd), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH2, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_HIGH, 0x7f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_00_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH4, 0xa9), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH3, 0x7b), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH2, 0xe4), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_HIGH, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_MODE_01_LOW, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_PI_CONTROLS, 0x99), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH1, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_THRESH2, 0x08), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SB2_GAIN2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_FO_GAIN, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_LOW, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_FASTLOCK_COUNT_HIGH, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL1, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_VGA_CAL_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQU_ADAPTOR_CNTRL4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x47), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_CNTRL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_DFE_CTLE_POST_CAL_OFFSET, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_UCDR_SO_GAIN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_GM_CAL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V5_RX_SIGDET_ENABLES, 0x00), +}; + +static const struct qmp_phy_init_tbl sc8280xp_usb3_uniphy_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG1, 0xd0), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_RX_SIGDET_LVL, 0xaa), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_RXEQTRAINING_DFE_TIME_S2, 0x07), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_USB3_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_CDR_RESET_TIME, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_ALIGN_DETECT_CONFIG1, 0x88), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_ALIGN_DETECT_CONFIG2, 0x13), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_EQ_CONFIG5, 0x10), + QMP_PHY_INIT_CFG(QPHY_V5_PCS_REFGEN_REQ_CONFIG1, 0x21), +}; /* struct qmp_phy_cfg - per-PHY initialization config */ struct qmp_phy_cfg { - /* phy-type - PCIE/UFS/USB */ - unsigned int type; - /* number of lanes provided by phy */ - int nlanes; + int lanes; /* Init sequence for PHY blocks - serdes, tx, rx, pcs */ const struct qmp_phy_init_tbl *serdes_tbl; @@ -1385,8 +1469,6 @@ struct qmp_phy_cfg { /* true, if PHY has a separate DP_COM control block */ bool has_phy_dp_com_ctrl; - /* true, if PHY has secondary tx/rx lanes to be configured */ - bool is_dual_lane_phy; /* Offset from PCS to PCS_USB region */ unsigned int pcs_usb_offset; @@ -1406,7 +1488,6 @@ struct qmp_phy_cfg { * @pcs_misc: iomapped memory space for lane's pcs_misc * @pcs_usb: iomapped memory space for lane's pcs_usb * @pipe_clk: pipe clock - * @index: lane index * @qmp: QMP phy to which this lane belongs * @mode: current PHY mode */ @@ -1422,7 +1503,6 @@ struct qmp_phy { void __iomem *pcs_misc; void __iomem *pcs_usb; struct clk *pipe_clk; - unsigned int index; struct qcom_qmp *qmp; enum phy_mode mode; }; @@ -1520,8 +1600,7 @@ static const char * const qmp_phy_vreg_l[] = { }; static const struct qmp_phy_cfg ipq8074_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = ipq8074_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(ipq8074_usb3_serdes_tbl), @@ -1545,8 +1624,7 @@ static const struct qmp_phy_cfg ipq8074_usb3phy_cfg = { }; static const struct qmp_phy_cfg msm8996_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = msm8996_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(msm8996_usb3_serdes_tbl), @@ -1570,8 +1648,7 @@ static const struct qmp_phy_cfg msm8996_usb3phy_cfg = { }; static const struct qmp_phy_cfg qmp_v3_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v3_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v3_usb3_serdes_tbl), @@ -1598,12 +1675,10 @@ static const struct qmp_phy_cfg qmp_v3_usb3phy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sc7180_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qmp_v3_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v3_usb3_serdes_tbl), @@ -1630,12 +1705,38 @@ static const struct qmp_phy_cfg sc7180_usb3phy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, +}; + +static const struct qmp_phy_cfg sc8280xp_usb3_uniphy_cfg = { + .lanes = 1, + + .serdes_tbl = sc8280xp_usb3_uniphy_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(sc8280xp_usb3_uniphy_serdes_tbl), + .tx_tbl = sc8280xp_usb3_uniphy_tx_tbl, + .tx_tbl_num = ARRAY_SIZE(sc8280xp_usb3_uniphy_tx_tbl), + .rx_tbl = sc8280xp_usb3_uniphy_rx_tbl, + .rx_tbl_num = ARRAY_SIZE(sc8280xp_usb3_uniphy_rx_tbl), + .pcs_tbl = sc8280xp_usb3_uniphy_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(sc8280xp_usb3_uniphy_pcs_tbl), + .clk_list = qmp_v4_phy_clk_l, + .num_clks = ARRAY_SIZE(qmp_v4_phy_clk_l), + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_l, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), + .regs = qmp_v4_usb3phy_regs_layout, + + .start_ctrl = SERDES_START | PCS_START, + .pwrdn_ctrl = SW_PWRDN, + .phy_status = PHYSTATUS, + + .has_pwrdn_delay = true, + .pwrdn_delay_min = POWER_DOWN_DELAY_US_MIN, + .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, }; static const struct qmp_phy_cfg qmp_v3_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = qmp_v3_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qmp_v3_usb3_uniphy_serdes_tbl), @@ -1663,8 +1764,7 @@ static const struct qmp_phy_cfg qmp_v3_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg msm8998_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = msm8998_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(msm8998_usb3_serdes_tbl), @@ -1685,13 +1785,10 @@ static const struct qmp_phy_cfg msm8998_usb3phy_cfg = { .start_ctrl = SERDES_START | PCS_START, .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8150_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = sm8150_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), @@ -1722,12 +1819,10 @@ static const struct qmp_phy_cfg sm8150_usb3phy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8150_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), @@ -1758,8 +1853,7 @@ static const struct qmp_phy_cfg sm8150_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg sm8250_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = sm8150_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), @@ -1789,12 +1883,10 @@ static const struct qmp_phy_cfg sm8250_usb3phy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8250_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), @@ -1825,8 +1917,7 @@ static const struct qmp_phy_cfg sm8250_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), @@ -1857,8 +1948,7 @@ static const struct qmp_phy_cfg sdx55_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg sdx65_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), @@ -1889,8 +1979,7 @@ static const struct qmp_phy_cfg sdx65_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg sm8350_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = sm8150_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_serdes_tbl), @@ -1920,12 +2009,10 @@ static const struct qmp_phy_cfg sm8350_usb3phy_cfg = { .pwrdn_delay_max = POWER_DOWN_DELAY_US_MAX, .has_phy_dp_com_ctrl = true, - .is_dual_lane_phy = true, }; static const struct qmp_phy_cfg sm8350_usb3_uniphy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 1, .serdes_tbl = sm8150_usb3_uniphy_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(sm8150_usb3_uniphy_serdes_tbl), @@ -1956,8 +2043,7 @@ static const struct qmp_phy_cfg sm8350_usb3_uniphy_cfg = { }; static const struct qmp_phy_cfg qcm2290_usb3phy_cfg = { - .type = PHY_TYPE_USB3, - .nlanes = 1, + .lanes = 2, .serdes_tbl = qcm2290_usb3_serdes_tbl, .serdes_tbl_num = ARRAY_SIZE(qcm2290_usb3_serdes_tbl), @@ -1978,11 +2064,9 @@ static const struct qmp_phy_cfg qcm2290_usb3phy_cfg = { .start_ctrl = SERDES_START | PCS_START, .pwrdn_ctrl = SW_PWRDN, .phy_status = PHYSTATUS, - - .is_dual_lane_phy = true, }; -static void qcom_qmp_phy_usb_configure_lane(void __iomem *base, +static void qmp_usb_configure_lane(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num, @@ -2005,28 +2089,29 @@ static void qcom_qmp_phy_usb_configure_lane(void __iomem *base, } } -static void qcom_qmp_phy_usb_configure(void __iomem *base, +static void qmp_usb_configure(void __iomem *base, const unsigned int *regs, const struct qmp_phy_init_tbl tbl[], int num) { - qcom_qmp_phy_usb_configure_lane(base, regs, tbl, num, 0xff); + qmp_usb_configure_lane(base, regs, tbl, num, 0xff); } -static int qcom_qmp_phy_usb_serdes_init(struct qmp_phy *qphy) +static int qmp_usb_serdes_init(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *serdes = qphy->serdes; const struct qmp_phy_init_tbl *serdes_tbl = cfg->serdes_tbl; int serdes_tbl_num = cfg->serdes_tbl_num; - qcom_qmp_phy_usb_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); + qmp_usb_configure(serdes, cfg->regs, serdes_tbl, serdes_tbl_num); return 0; } -static int qcom_qmp_phy_usb_com_init(struct qmp_phy *qphy) +static int qmp_usb_init(struct phy *phy) { + struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs = qphy->pcs; @@ -2097,8 +2182,9 @@ err_disable_regulators: return ret; } -static int qcom_qmp_phy_usb_com_exit(struct qmp_phy *qphy) +static int qmp_usb_exit(struct phy *phy) { + struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -2111,21 +2197,7 @@ static int qcom_qmp_phy_usb_com_exit(struct qmp_phy *qphy) return 0; } -static int qcom_qmp_phy_usb_init(struct phy *phy) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - struct qcom_qmp *qmp = qphy->qmp; - int ret; - dev_vdbg(qmp->dev, "Initializing QMP phy\n"); - - ret = qcom_qmp_phy_usb_com_init(qphy); - if (ret) - return ret; - - return 0; -} - -static int qcom_qmp_phy_usb_power_on(struct phy *phy) +static int qmp_usb_power_on(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); struct qcom_qmp *qmp = qphy->qmp; @@ -2137,7 +2209,7 @@ static int qcom_qmp_phy_usb_power_on(struct phy *phy) unsigned int mask, val, ready; int ret; - qcom_qmp_phy_usb_serdes_init(qphy); + qmp_usb_serdes_init(qphy); ret = clk_prepare_enable(qphy->pipe_clk); if (ret) { @@ -2146,25 +2218,22 @@ static int qcom_qmp_phy_usb_power_on(struct phy *phy) } /* Tx, Rx, and PCS configurations */ - qcom_qmp_phy_usb_configure_lane(tx, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 1); + qmp_usb_configure_lane(tx, cfg->regs, cfg->tx_tbl, cfg->tx_tbl_num, 1); - /* Configuration for other LANE for USB-DP combo PHY */ - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_usb_configure_lane(qphy->tx2, cfg->regs, - cfg->tx_tbl, cfg->tx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_usb_configure_lane(qphy->tx2, cfg->regs, + cfg->tx_tbl, cfg->tx_tbl_num, 2); } - qcom_qmp_phy_usb_configure_lane(rx, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 1); + qmp_usb_configure_lane(rx, cfg->regs, cfg->rx_tbl, cfg->rx_tbl_num, 1); - if (cfg->is_dual_lane_phy) { - qcom_qmp_phy_usb_configure_lane(qphy->rx2, cfg->regs, - cfg->rx_tbl, cfg->rx_tbl_num, 2); + if (cfg->lanes >= 2) { + qmp_usb_configure_lane(qphy->rx2, cfg->regs, + cfg->rx_tbl, cfg->rx_tbl_num, 2); } /* Configure link rate, swing, etc. */ - qcom_qmp_phy_usb_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); + qmp_usb_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num); if (cfg->has_pwrdn_delay) usleep_range(cfg->pwrdn_delay_min, cfg->pwrdn_delay_max); @@ -2194,7 +2263,7 @@ err_disable_pipe_clk: return ret; } -static int qcom_qmp_phy_usb_power_off(struct phy *phy) +static int qmp_usb_power_off(struct phy *phy) { struct qmp_phy *qphy = phy_get_drvdata(phy); const struct qmp_phy_cfg *cfg = qphy->cfg; @@ -2219,42 +2288,32 @@ static int qcom_qmp_phy_usb_power_off(struct phy *phy) return 0; } -static int qcom_qmp_phy_usb_exit(struct phy *phy) -{ - struct qmp_phy *qphy = phy_get_drvdata(phy); - - qcom_qmp_phy_usb_com_exit(qphy); - - return 0; -} - -static int qcom_qmp_phy_usb_enable(struct phy *phy) +static int qmp_usb_enable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_usb_init(phy); + ret = qmp_usb_init(phy); if (ret) return ret; - ret = qcom_qmp_phy_usb_power_on(phy); + ret = qmp_usb_power_on(phy); if (ret) - qcom_qmp_phy_usb_exit(phy); + qmp_usb_exit(phy); return ret; } -static int qcom_qmp_phy_usb_disable(struct phy *phy) +static int qmp_usb_disable(struct phy *phy) { int ret; - ret = qcom_qmp_phy_usb_power_off(phy); + ret = qmp_usb_power_off(phy); if (ret) return ret; - return qcom_qmp_phy_usb_exit(phy); + return qmp_usb_exit(phy); } -static int qcom_qmp_phy_usb_set_mode(struct phy *phy, - enum phy_mode mode, int submode) +static int qmp_usb_set_mode(struct phy *phy, enum phy_mode mode, int submode) { struct qmp_phy *qphy = phy_get_drvdata(phy); @@ -2263,7 +2322,7 @@ static int qcom_qmp_phy_usb_set_mode(struct phy *phy, return 0; } -static void qcom_qmp_phy_usb_enable_autonomous_mode(struct qmp_phy *qphy) +static void qmp_usb_enable_autonomous_mode(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs; @@ -2292,7 +2351,7 @@ static void qcom_qmp_phy_usb_enable_autonomous_mode(struct qmp_phy *qphy) qphy_clrbits(pcs_misc, QPHY_V3_PCS_MISC_CLAMP_ENABLE, CLAMP_EN); } -static void qcom_qmp_phy_usb_disable_autonomous_mode(struct qmp_phy *qphy) +static void qmp_usb_disable_autonomous_mode(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs; @@ -2310,7 +2369,7 @@ static void qcom_qmp_phy_usb_disable_autonomous_mode(struct qmp_phy *qphy) qphy_clrbits(pcs_usb, cfg->regs[QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR], IRQ_CLEAR); } -static int __maybe_unused qcom_qmp_phy_usb_runtime_suspend(struct device *dev) +static int __maybe_unused qmp_usb_runtime_suspend(struct device *dev) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct qmp_phy *qphy = qmp->phys[0]; @@ -2318,16 +2377,12 @@ static int __maybe_unused qcom_qmp_phy_usb_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qphy->mode); - /* Supported only for USB3 PHY and luckily USB3 is the first phy */ - if (cfg->type != PHY_TYPE_USB3) - return 0; - if (!qphy->phy->init_count) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } - qcom_qmp_phy_usb_enable_autonomous_mode(qphy); + qmp_usb_enable_autonomous_mode(qphy); clk_disable_unprepare(qphy->pipe_clk); clk_bulk_disable_unprepare(cfg->num_clks, qmp->clks); @@ -2335,7 +2390,7 @@ static int __maybe_unused qcom_qmp_phy_usb_runtime_suspend(struct device *dev) return 0; } -static int __maybe_unused qcom_qmp_phy_usb_runtime_resume(struct device *dev) +static int __maybe_unused qmp_usb_runtime_resume(struct device *dev) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct qmp_phy *qphy = qmp->phys[0]; @@ -2344,10 +2399,6 @@ static int __maybe_unused qcom_qmp_phy_usb_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qphy->mode); - /* Supported only for USB3 PHY and luckily USB3 is the first phy */ - if (cfg->type != PHY_TYPE_USB3) - return 0; - if (!qphy->phy->init_count) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; @@ -2364,12 +2415,12 @@ static int __maybe_unused qcom_qmp_phy_usb_runtime_resume(struct device *dev) return ret; } - qcom_qmp_phy_usb_disable_autonomous_mode(qphy); + qmp_usb_disable_autonomous_mode(qphy); return 0; } -static int qcom_qmp_phy_usb_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_usb_vreg_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_vregs; @@ -2385,7 +2436,7 @@ static int qcom_qmp_phy_usb_vreg_init(struct device *dev, const struct qmp_phy_c return devm_regulator_bulk_get(dev, num, qmp->vregs); } -static int qcom_qmp_phy_usb_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_usb_reset_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int i; @@ -2406,7 +2457,7 @@ static int qcom_qmp_phy_usb_reset_init(struct device *dev, const struct qmp_phy_ return 0; } -static int qcom_qmp_phy_usb_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) +static int qmp_usb_clk_init(struct device *dev, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); int num = cfg->num_clks; @@ -2482,23 +2533,47 @@ static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np) return devm_add_action_or_reset(qmp->dev, phy_clk_release_provider, np); } -static const struct phy_ops qcom_qmp_phy_usb_ops = { - .init = qcom_qmp_phy_usb_enable, - .exit = qcom_qmp_phy_usb_disable, - .set_mode = qcom_qmp_phy_usb_set_mode, +static const struct phy_ops qmp_usb_ops = { + .init = qmp_usb_enable, + .exit = qmp_usb_disable, + .set_mode = qmp_usb_set_mode, .owner = THIS_MODULE, }; +static void __iomem *qmp_usb_iomap(struct device *dev, struct device_node *np, + int index, bool exclusive) +{ + struct resource res; + + if (!exclusive) { + if (of_address_to_resource(np, index, &res)) + return IOMEM_ERR_PTR(-EINVAL); + + return devm_ioremap(dev, res.start, resource_size(&res)); + } + + return devm_of_iomap(dev, np, index, NULL); +} + static -int qcom_qmp_phy_usb_create(struct device *dev, struct device_node *np, int id, +int qmp_usb_create(struct device *dev, struct device_node *np, int id, void __iomem *serdes, const struct qmp_phy_cfg *cfg) { struct qcom_qmp *qmp = dev_get_drvdata(dev); struct phy *generic_phy; struct qmp_phy *qphy; - char prop_name[MAX_PROP_NAME]; + bool exclusive = true; int ret; + /* + * FIXME: These bindings should be fixed to not rely on overlapping + * mappings for PCS. + */ + if (of_device_is_compatible(dev->of_node, "qcom,sdx65-qmp-usb3-uni-phy")) + exclusive = false; + if (of_device_is_compatible(dev->of_node, "qcom,sm8350-qmp-usb3-uni-phy")) + exclusive = false; + qphy = devm_kzalloc(dev, sizeof(*qphy), GFP_KERNEL); if (!qphy) return -ENOMEM; @@ -2511,58 +2586,47 @@ int qcom_qmp_phy_usb_create(struct device *dev, struct device_node *np, int id, * For dual lane PHYs: tx2 -> 3, rx2 -> 4, pcs_misc (optional) -> 5 * For single lane PHYs: pcs_misc (optional) -> 3. */ - qphy->tx = of_iomap(np, 0); - if (!qphy->tx) - return -ENOMEM; + qphy->tx = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(qphy->tx)) + return PTR_ERR(qphy->tx); - qphy->rx = of_iomap(np, 1); - if (!qphy->rx) - return -ENOMEM; + qphy->rx = devm_of_iomap(dev, np, 1, NULL); + if (IS_ERR(qphy->rx)) + return PTR_ERR(qphy->rx); - qphy->pcs = of_iomap(np, 2); - if (!qphy->pcs) - return -ENOMEM; + qphy->pcs = qmp_usb_iomap(dev, np, 2, exclusive); + if (IS_ERR(qphy->pcs)) + return PTR_ERR(qphy->pcs); if (cfg->pcs_usb_offset) qphy->pcs_usb = qphy->pcs + cfg->pcs_usb_offset; - /* - * If this is a dual-lane PHY, then there should be registers for the - * second lane. Some old device trees did not specify this, so fall - * back to old legacy behavior of assuming they can be reached at an - * offset from the first lane. - */ - if (cfg->is_dual_lane_phy) { - qphy->tx2 = of_iomap(np, 3); - qphy->rx2 = of_iomap(np, 4); - if (!qphy->tx2 || !qphy->rx2) { - dev_warn(dev, - "Underspecified device tree, falling back to legacy register regions\n"); - - /* In the old version, pcs_misc is at index 3. */ - qphy->pcs_misc = qphy->tx2; - qphy->tx2 = qphy->tx + QMP_PHY_LEGACY_LANE_STRIDE; - qphy->rx2 = qphy->rx + QMP_PHY_LEGACY_LANE_STRIDE; - - } else { - qphy->pcs_misc = of_iomap(np, 5); - } + if (cfg->lanes >= 2) { + qphy->tx2 = devm_of_iomap(dev, np, 3, NULL); + if (IS_ERR(qphy->tx2)) + return PTR_ERR(qphy->tx2); + qphy->rx2 = devm_of_iomap(dev, np, 4, NULL); + if (IS_ERR(qphy->rx2)) + return PTR_ERR(qphy->rx2); + + qphy->pcs_misc = devm_of_iomap(dev, np, 5, NULL); } else { - qphy->pcs_misc = of_iomap(np, 3); + qphy->pcs_misc = devm_of_iomap(dev, np, 3, NULL); } - if (!qphy->pcs_misc) + if (IS_ERR(qphy->pcs_misc)) { dev_vdbg(dev, "PHY pcs_misc-reg not used\n"); + qphy->pcs_misc = NULL; + } - snprintf(prop_name, sizeof(prop_name), "pipe%d", id); - qphy->pipe_clk = devm_get_clk_from_child(dev, np, prop_name); + qphy->pipe_clk = devm_get_clk_from_child(dev, np, NULL); if (IS_ERR(qphy->pipe_clk)) { return dev_err_probe(dev, PTR_ERR(qphy->pipe_clk), "failed to get lane%d pipe clock\n", id); } - generic_phy = devm_phy_create(dev, np, &qcom_qmp_phy_usb_ops); + generic_phy = devm_phy_create(dev, np, &qmp_usb_ops); if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create qphy %d\n", ret); @@ -2570,7 +2634,6 @@ int qcom_qmp_phy_usb_create(struct device *dev, struct device_node *np, int id, } qphy->phy = generic_phy; - qphy->index = id; qphy->qmp = qmp; qmp->phys[id] = qphy; phy_set_drvdata(generic_phy, qphy); @@ -2578,7 +2641,7 @@ int qcom_qmp_phy_usb_create(struct device *dev, struct device_node *np, int id, return 0; } -static const struct of_device_id qcom_qmp_phy_usb_of_match_table[] = { +static const struct of_device_id qmp_usb_of_match_table[] = { { .compatible = "qcom,ipq8074-qmp-usb3-phy", .data = &ipq8074_usb3phy_cfg, @@ -2594,6 +2657,9 @@ static const struct of_device_id qcom_qmp_phy_usb_of_match_table[] = { }, { .compatible = "qcom,sc8180x-qmp-usb3-phy", .data = &sm8150_usb3phy_cfg, + }, { + .compatible = "qcom,sc8280xp-qmp-usb3-uni-phy", + .data = &sc8280xp_usb3_uniphy_cfg, }, { .compatible = "qcom,sdm845-qmp-usb3-phy", .data = &qmp_v3_usb3phy_cfg, @@ -2636,14 +2702,14 @@ static const struct of_device_id qcom_qmp_phy_usb_of_match_table[] = { }, { }, }; -MODULE_DEVICE_TABLE(of, qcom_qmp_phy_usb_of_match_table); +MODULE_DEVICE_TABLE(of, qmp_usb_of_match_table); -static const struct dev_pm_ops qcom_qmp_phy_usb_pm_ops = { - SET_RUNTIME_PM_OPS(qcom_qmp_phy_usb_runtime_suspend, - qcom_qmp_phy_usb_runtime_resume, NULL) +static const struct dev_pm_ops qmp_usb_pm_ops = { + SET_RUNTIME_PM_OPS(qmp_usb_runtime_suspend, + qmp_usb_runtime_resume, NULL) }; -static int qcom_qmp_phy_usb_probe(struct platform_device *pdev) +static int qmp_usb_probe(struct platform_device *pdev) { struct qcom_qmp *qmp; struct device *dev = &pdev->dev; @@ -2678,21 +2744,18 @@ static int qcom_qmp_phy_usb_probe(struct platform_device *pdev) return PTR_ERR(qmp->dp_com); } - ret = qcom_qmp_phy_usb_clk_init(dev, cfg); + ret = qmp_usb_clk_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_usb_reset_init(dev, cfg); + ret = qmp_usb_reset_init(dev, cfg); if (ret) return ret; - ret = qcom_qmp_phy_usb_vreg_init(dev, cfg); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + ret = qmp_usb_vreg_init(dev, cfg); + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); num = of_get_available_child_count(dev->of_node); /* do we have a rogue child node ? */ @@ -2704,7 +2767,9 @@ static int qcom_qmp_phy_usb_probe(struct platform_device *pdev) return -ENOMEM; pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. @@ -2714,7 +2779,7 @@ static int qcom_qmp_phy_usb_probe(struct platform_device *pdev) id = 0; for_each_available_child_of_node(dev->of_node, child) { /* Create per-lane phy */ - ret = qcom_qmp_phy_usb_create(dev, child, id, serdes, cfg); + ret = qmp_usb_create(dev, child, id, serdes, cfg); if (ret) { dev_err(dev, "failed to create lane%d phy, %d\n", id, ret); @@ -2736,29 +2801,24 @@ static int qcom_qmp_phy_usb_probe(struct platform_device *pdev) } phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_info(dev, "Registered Qcom-QMP phy\n"); - else - pm_runtime_disable(dev); return PTR_ERR_OR_ZERO(phy_provider); err_node_put: - pm_runtime_disable(dev); of_node_put(child); return ret; } -static struct platform_driver qcom_qmp_phy_usb_driver = { - .probe = qcom_qmp_phy_usb_probe, +static struct platform_driver qmp_usb_driver = { + .probe = qmp_usb_probe, .driver = { .name = "qcom-qmp-usb-phy", - .pm = &qcom_qmp_phy_usb_pm_ops, - .of_match_table = qcom_qmp_phy_usb_of_match_table, + .pm = &qmp_usb_pm_ops, + .of_match_table = qmp_usb_of_match_table, }, }; -module_platform_driver(qcom_qmp_phy_usb_driver); +module_platform_driver(qmp_usb_driver); MODULE_AUTHOR("Vivek Gautam "); MODULE_DESCRIPTION("Qualcomm QMP USB PHY driver"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index b139c8af5e8b1b8e4aee5ef1c4e3e65964419f2d..26274e3c0cf9512615f71b217ecce49147602342 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -19,6 +19,7 @@ #include "phy-qcom-qmp-qserdes-com-v5.h" #include "phy-qcom-qmp-qserdes-txrx-v5.h" #include "phy-qcom-qmp-qserdes-txrx-v5_20.h" +#include "phy-qcom-qmp-qserdes-txrx-v5_5nm.h" #include "phy-qcom-qmp-qserdes-pll.h" diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c index 7529a7e6e5df74f99e3c3c0ab01a30c343756306..2ef638b32e8f08bc655adb504fe20a8a0723940a 100644 --- a/drivers/phy/qualcomm/phy-qcom-qusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c @@ -973,20 +973,14 @@ static int qusb2_phy_probe(struct platform_device *pdev) return PTR_ERR(qphy->base); qphy->cfg_ahb_clk = devm_clk_get(dev, "cfg_ahb"); - if (IS_ERR(qphy->cfg_ahb_clk)) { - ret = PTR_ERR(qphy->cfg_ahb_clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get cfg ahb clk, %d\n", ret); - return ret; - } + if (IS_ERR(qphy->cfg_ahb_clk)) + return dev_err_probe(dev, PTR_ERR(qphy->cfg_ahb_clk), + "failed to get cfg ahb clk\n"); qphy->ref_clk = devm_clk_get(dev, "ref"); - if (IS_ERR(qphy->ref_clk)) { - ret = PTR_ERR(qphy->ref_clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get ref clk, %d\n", ret); - return ret; - } + if (IS_ERR(qphy->ref_clk)) + return dev_err_probe(dev, PTR_ERR(qphy->ref_clk), + "failed to get ref clk\n"); qphy->iface_clk = devm_clk_get_optional(dev, "iface"); if (IS_ERR(qphy->iface_clk)) @@ -1003,12 +997,9 @@ static int qusb2_phy_probe(struct platform_device *pdev) qphy->vregs[i].supply = qusb2_phy_vreg_names[i]; ret = devm_regulator_bulk_get(dev, num, qphy->vregs); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); /* Get the specific init parameters of QMP phy */ qphy->cfg = of_device_get_match_data(dev); diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c index 5d203784f75d2d78a856a98850795c28557a51d0..a590635962140f817e0ee3287d576bbd043ffa3c 100644 --- a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c +++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c @@ -52,6 +52,12 @@ #define USB2_SUSPEND_N BIT(2) #define USB2_SUSPEND_N_SEL BIT(3) +#define USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X0 (0x6c) +#define USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X1 (0x70) +#define USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X2 (0x74) +#define USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X3 (0x78) +#define PARAM_OVRD_MASK 0xFF + #define USB2_PHY_USB_PHY_CFG0 (0x94) #define UTMI_PHY_DATAPATH_CTRL_OVERRIDE_EN BIT(0) #define UTMI_PHY_CMN_CTRL_OVERRIDE_EN BIT(1) @@ -60,12 +66,47 @@ #define REFCLK_SEL_MASK GENMASK(1, 0) #define REFCLK_SEL_DEFAULT (0x2 << 0) +#define HS_DISCONNECT_MASK GENMASK(2, 0) +#define SQUELCH_DETECTOR_MASK GENMASK(7, 5) + +#define HS_AMPLITUDE_MASK GENMASK(3, 0) +#define PREEMPHASIS_DURATION_MASK BIT(5) +#define PREEMPHASIS_AMPLITUDE_MASK GENMASK(7, 6) + +#define HS_RISE_FALL_MASK GENMASK(1, 0) +#define HS_CROSSOVER_VOLTAGE_MASK GENMASK(3, 2) +#define HS_OUTPUT_IMPEDANCE_MASK GENMASK(5, 4) + +#define LS_FS_OUTPUT_IMPEDANCE_MASK GENMASK(3, 0) + static const char * const qcom_snps_hsphy_vreg_names[] = { "vdda-pll", "vdda33", "vdda18", }; #define SNPS_HS_NUM_VREGS ARRAY_SIZE(qcom_snps_hsphy_vreg_names) +struct override_param { + s32 value; + u8 reg_val; +}; + +struct override_param_map { + const char *prop_name; + const struct override_param *param_table; + u8 table_size; + u8 reg_offset; + u8 param_mask; +}; + +struct phy_override_seq { + bool need_update; + u8 offset; + u8 value; + u8 mask; +}; + +#define NUM_HSPHY_TUNING_PARAMS (9) + /** * struct qcom_snps_hsphy - snps hs phy attributes * @@ -91,6 +132,7 @@ struct qcom_snps_hsphy { bool phy_initialized; enum phy_mode mode; + struct phy_override_seq update_seq_cfg[NUM_HSPHY_TUNING_PARAMS]; }; static inline void qcom_snps_hsphy_write_mask(void __iomem *base, u32 offset, @@ -173,10 +215,158 @@ static int qcom_snps_hsphy_set_mode(struct phy *phy, enum phy_mode mode, return 0; } +static const struct override_param hs_disconnect_sc7280[] = { + { -272, 0 }, + { 0, 1 }, + { 317, 2 }, + { 630, 3 }, + { 973, 4 }, + { 1332, 5 }, + { 1743, 6 }, + { 2156, 7 }, +}; + +static const struct override_param squelch_det_threshold_sc7280[] = { + { -2090, 7 }, + { -1560, 6 }, + { -1030, 5 }, + { -530, 4 }, + { 0, 3 }, + { 530, 2 }, + { 1060, 1 }, + { 1590, 0 }, +}; + +static const struct override_param hs_amplitude_sc7280[] = { + { -660, 0 }, + { -440, 1 }, + { -220, 2 }, + { 0, 3 }, + { 230, 4 }, + { 440, 5 }, + { 650, 6 }, + { 890, 7 }, + { 1110, 8 }, + { 1330, 9 }, + { 1560, 10 }, + { 1780, 11 }, + { 2000, 12 }, + { 2220, 13 }, + { 2430, 14 }, + { 2670, 15 }, +}; + +static const struct override_param preemphasis_duration_sc7280[] = { + { 10000, 1 }, + { 20000, 0 }, +}; + +static const struct override_param preemphasis_amplitude_sc7280[] = { + { 10000, 1 }, + { 20000, 2 }, + { 30000, 3 }, + { 40000, 0 }, +}; + +static const struct override_param hs_rise_fall_time_sc7280[] = { + { -4100, 3 }, + { 0, 2 }, + { 2810, 1 }, + { 5430, 0 }, +}; + +static const struct override_param hs_crossover_voltage_sc7280[] = { + { -31000, 1 }, + { 0, 3 }, + { 28000, 2 }, +}; + +static const struct override_param hs_output_impedance_sc7280[] = { + { -2300000, 3 }, + { 0, 2 }, + { 2600000, 1 }, + { 6100000, 0 }, +}; + +static const struct override_param ls_fs_output_impedance_sc7280[] = { + { -1053, 15 }, + { -557, 7 }, + { 0, 3 }, + { 612, 1 }, + { 1310, 0 }, +}; + +static const struct override_param_map sc7280_snps_7nm_phy[] = { + { + "qcom,hs-disconnect-bp", + hs_disconnect_sc7280, + ARRAY_SIZE(hs_disconnect_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X0, + HS_DISCONNECT_MASK + }, + { + "qcom,squelch-detector-bp", + squelch_det_threshold_sc7280, + ARRAY_SIZE(squelch_det_threshold_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X0, + SQUELCH_DETECTOR_MASK + }, + { + "qcom,hs-amplitude-bp", + hs_amplitude_sc7280, + ARRAY_SIZE(hs_amplitude_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X1, + HS_AMPLITUDE_MASK + }, + { + "qcom,pre-emphasis-duration-bp", + preemphasis_duration_sc7280, + ARRAY_SIZE(preemphasis_duration_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X1, + PREEMPHASIS_DURATION_MASK, + }, + { + "qcom,pre-emphasis-amplitude-bp", + preemphasis_amplitude_sc7280, + ARRAY_SIZE(preemphasis_amplitude_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X1, + PREEMPHASIS_AMPLITUDE_MASK, + }, + { + "qcom,hs-rise-fall-time-bp", + hs_rise_fall_time_sc7280, + ARRAY_SIZE(hs_rise_fall_time_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X2, + HS_RISE_FALL_MASK + }, + { + "qcom,hs-crossover-voltage-microvolt", + hs_crossover_voltage_sc7280, + ARRAY_SIZE(hs_crossover_voltage_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X2, + HS_CROSSOVER_VOLTAGE_MASK + }, + { + "qcom,hs-output-impedance-micro-ohms", + hs_output_impedance_sc7280, + ARRAY_SIZE(hs_output_impedance_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X2, + HS_OUTPUT_IMPEDANCE_MASK, + }, + { + "qcom,ls-fs-output-impedance-bp", + ls_fs_output_impedance_sc7280, + ARRAY_SIZE(ls_fs_output_impedance_sc7280), + USB2_PHY_USB_PHY_HS_PHY_OVERRIDE_X3, + LS_FS_OUTPUT_IMPEDANCE_MASK, + }, + {}, +}; + static int qcom_snps_hsphy_init(struct phy *phy) { struct qcom_snps_hsphy *hsphy = phy_get_drvdata(phy); - int ret; + int ret, i; dev_vdbg(&phy->dev, "%s(): Initializing SNPS HS phy\n", __func__); @@ -223,6 +413,14 @@ static int qcom_snps_hsphy_init(struct phy *phy) qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_HS_PHY_CTRL1, VBUSVLDEXT0, VBUSVLDEXT0); + for (i = 0; i < ARRAY_SIZE(hsphy->update_seq_cfg); i++) { + if (hsphy->update_seq_cfg[i].need_update) + qcom_snps_hsphy_write_mask(hsphy->base, + hsphy->update_seq_cfg[i].offset, + hsphy->update_seq_cfg[i].mask, + hsphy->update_seq_cfg[i].value); + } + qcom_snps_hsphy_write_mask(hsphy->base, USB2_PHY_USB_PHY_HS_PHY_CTRL_COMMON2, VREGBYPASS, VREGBYPASS); @@ -280,7 +478,10 @@ static const struct phy_ops qcom_snps_hsphy_gen_ops = { static const struct of_device_id qcom_snps_hsphy_of_match_table[] = { { .compatible = "qcom,sm8150-usb-hs-phy", }, { .compatible = "qcom,usb-snps-hs-5nm-phy", }, - { .compatible = "qcom,usb-snps-hs-7nm-phy", }, + { + .compatible = "qcom,usb-snps-hs-7nm-phy", + .data = &sc7280_snps_7nm_phy, + }, { .compatible = "qcom,usb-snps-femto-v2-phy", }, { } }; @@ -291,6 +492,55 @@ static const struct dev_pm_ops qcom_snps_hsphy_pm_ops = { qcom_snps_hsphy_runtime_resume, NULL) }; +static void qcom_snps_hsphy_override_param_update_val( + const struct override_param_map map, + s32 dt_val, struct phy_override_seq *seq_entry) +{ + int i; + + /* + * Param table for each param is in increasing order + * of dt values. We need to iterate over the list to + * select the entry that matches the dt value and pick + * up the corresponding register value. + */ + for (i = 0; i < map.table_size - 1; i++) { + if (map.param_table[i].value == dt_val) + break; + } + + seq_entry->need_update = true; + seq_entry->offset = map.reg_offset; + seq_entry->mask = map.param_mask; + seq_entry->value = map.param_table[i].reg_val << __ffs(map.param_mask); +} + +static void qcom_snps_hsphy_read_override_param_seq(struct device *dev) +{ + struct device_node *node = dev->of_node; + s32 val; + int ret, i; + struct qcom_snps_hsphy *hsphy; + const struct override_param_map *cfg = of_device_get_match_data(dev); + + if (!cfg) + return; + + hsphy = dev_get_drvdata(dev); + + for (i = 0; cfg[i].prop_name != NULL; i++) { + ret = of_property_read_s32(node, cfg[i].prop_name, &val); + if (ret) + continue; + + qcom_snps_hsphy_override_param_update_val(cfg[i], val, + &hsphy->update_seq_cfg[i]); + dev_dbg(&hsphy->phy->dev, "Read param: %s dt_val: %d reg_val: 0x%x\n", + cfg[i].prop_name, val, hsphy->update_seq_cfg[i].value); + + } +} + static int qcom_snps_hsphy_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -309,12 +559,9 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) return PTR_ERR(hsphy->base); hsphy->ref_clk = devm_clk_get(dev, "ref"); - if (IS_ERR(hsphy->ref_clk)) { - ret = PTR_ERR(hsphy->ref_clk); - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get ref clk, %d\n", ret); - return ret; - } + if (IS_ERR(hsphy->ref_clk)) + return dev_err_probe(dev, PTR_ERR(hsphy->ref_clk), + "failed to get ref clk\n"); hsphy->phy_reset = devm_reset_control_get_exclusive(&pdev->dev, NULL); if (IS_ERR(hsphy->phy_reset)) { @@ -327,12 +574,9 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) hsphy->vregs[i].supply = qcom_snps_hsphy_vreg_names[i]; ret = devm_regulator_bulk_get(dev, num, hsphy->vregs); - if (ret) { - if (ret != -EPROBE_DEFER) - dev_err(dev, "failed to get regulator supplies: %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, + "failed to get regulator supplies\n"); pm_runtime_set_active(dev); pm_runtime_enable(dev); @@ -352,6 +596,7 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) dev_set_drvdata(dev, hsphy); phy_set_drvdata(generic_phy, hsphy); + qcom_snps_hsphy_read_override_param_seq(dev); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); if (!IS_ERR(phy_provider)) diff --git a/drivers/phy/qualcomm/phy-qcom-usb-hsic.c b/drivers/phy/qualcomm/phy-qcom-usb-hsic.c index 716a77748ed83684af9375ba0cc5b40e3390de5c..20f6dd37c7c106c8520bbef5aea15ef52cf026d2 100644 --- a/drivers/phy/qualcomm/phy-qcom-usb-hsic.c +++ b/drivers/phy/qualcomm/phy-qcom-usb-hsic.c @@ -54,8 +54,10 @@ static int qcom_usb_hsic_phy_power_on(struct phy *phy) /* Configure pins for HSIC functionality */ pins_default = pinctrl_lookup_state(uphy->pctl, PINCTRL_STATE_DEFAULT); - if (IS_ERR(pins_default)) - return PTR_ERR(pins_default); + if (IS_ERR(pins_default)) { + ret = PTR_ERR(pins_default); + goto err_ulpi; + } ret = pinctrl_select_state(uphy->pctl, pins_default); if (ret) diff --git a/drivers/phy/rockchip/Kconfig b/drivers/phy/rockchip/Kconfig index 9022e395c056a576fa44e68ab3859b2f4b2ef616..94360fc96a6fb2e16cce53e44c0222d719e6e1ab 100644 --- a/drivers/phy/rockchip/Kconfig +++ b/drivers/phy/rockchip/Kconfig @@ -83,6 +83,15 @@ config PHY_ROCKCHIP_PCIE help Enable this to support the Rockchip PCIe PHY. +config PHY_ROCKCHIP_SNPS_PCIE3 + tristate "Rockchip Snps PCIe3 PHY Driver" + depends on (ARCH_ROCKCHIP && OF) || COMPILE_TEST + depends on HAS_IOMEM + select GENERIC_PHY + select MFD_SYSCON + help + Enable this to support the Rockchip snps PCIe3 PHY. + config PHY_ROCKCHIP_TYPEC tristate "Rockchip TYPEC PHY Driver" depends on OF && (ARCH_ROCKCHIP || COMPILE_TEST) diff --git a/drivers/phy/rockchip/Makefile b/drivers/phy/rockchip/Makefile index a5041efb5b8fdd06b22aeb0729b702f6b073ba5b..7eab129230d175bca9d439336ca56e2e5fe1cb35 100644 --- a/drivers/phy/rockchip/Makefile +++ b/drivers/phy/rockchip/Makefile @@ -8,5 +8,6 @@ obj-$(CONFIG_PHY_ROCKCHIP_INNO_HDMI) += phy-rockchip-inno-hdmi.o obj-$(CONFIG_PHY_ROCKCHIP_INNO_USB2) += phy-rockchip-inno-usb2.o obj-$(CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY) += phy-rockchip-naneng-combphy.o obj-$(CONFIG_PHY_ROCKCHIP_PCIE) += phy-rockchip-pcie.o +obj-$(CONFIG_PHY_ROCKCHIP_SNPS_PCIE3) += phy-rockchip-snps-pcie3.o obj-$(CONFIG_PHY_ROCKCHIP_TYPEC) += phy-rockchip-typec.o obj-$(CONFIG_PHY_ROCKCHIP_USB) += phy-rockchip-usb.o diff --git a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c index ca13a604ab4ff295504c2889828574b04ac2ef7e..75f948bdea6a10ddd9cc0075408959c7ef26d1f1 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c @@ -27,6 +27,9 @@ #define RK3368_GRF_SOC_CON6_OFFSET 0x0418 +#define RK3568_GRF_VI_CON0 0x0340 +#define RK3568_GRF_VI_CON1 0x0344 + /* PHY */ #define CSIDPHY_CTRL_LANE_ENABLE 0x00 #define CSIDPHY_CTRL_LANE_ENABLE_CK BIT(6) @@ -58,9 +61,11 @@ #define RK1808_CSIDPHY_CLK_WR_THS_SETTLE 0x160 #define RK3326_CSIDPHY_CLK_WR_THS_SETTLE 0x100 #define RK3368_CSIDPHY_CLK_WR_THS_SETTLE 0x100 +#define RK3568_CSIDPHY_CLK_WR_THS_SETTLE 0x160 /* Calibration reception enable */ #define RK1808_CSIDPHY_CLK_CALIB_EN 0x168 +#define RK3568_CSIDPHY_CLK_CALIB_EN 0x168 /* * The higher 16-bit of this register is used for write protection @@ -103,6 +108,12 @@ static const struct dphy_reg rk3368_grf_dphy_regs[] = { [GRF_DPHY_CSIPHY_FORCERXMODE] = PHY_REG(RK3368_GRF_SOC_CON6_OFFSET, 4, 8), }; +static const struct dphy_reg rk3568_grf_dphy_regs[] = { + [GRF_DPHY_CSIPHY_FORCERXMODE] = PHY_REG(RK3568_GRF_VI_CON0, 4, 0), + [GRF_DPHY_CSIPHY_DATALANE_EN] = PHY_REG(RK3568_GRF_VI_CON0, 4, 4), + [GRF_DPHY_CSIPHY_CLKLANE_EN] = PHY_REG(RK3568_GRF_VI_CON0, 1, 8), +}; + struct hsfreq_range { u32 range_h; u8 cfg_bit; @@ -352,6 +363,15 @@ static const struct dphy_drv_data rk3368_mipidphy_drv_data = { .grf_regs = rk3368_grf_dphy_regs, }; +static const struct dphy_drv_data rk3568_mipidphy_drv_data = { + .pwrctl_offset = -1, + .ths_settle_offset = RK3568_CSIDPHY_CLK_WR_THS_SETTLE, + .calib_offset = RK3568_CSIDPHY_CLK_CALIB_EN, + .hsfreq_ranges = rk1808_mipidphy_hsfreq_ranges, + .num_hsfreq_ranges = ARRAY_SIZE(rk1808_mipidphy_hsfreq_ranges), + .grf_regs = rk3568_grf_dphy_regs, +}; + static const struct of_device_id rockchip_inno_csidphy_match_id[] = { { .compatible = "rockchip,px30-csi-dphy", @@ -369,6 +389,10 @@ static const struct of_device_id rockchip_inno_csidphy_match_id[] = { .compatible = "rockchip,rk3368-csi-dphy", .data = &rk3368_mipidphy_drv_data, }, + { + .compatible = "rockchip,rk3568-csi-dphy", + .data = &rk3568_mipidphy_drv_data, + }, {} }; MODULE_DEVICE_TABLE(of, rockchip_inno_csidphy_match_id); diff --git a/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c index 630e01b5c19b9543d1f9afb8af4216dfad4b40c3..2c5847faff6363435ebb4981cdb389e358df36a7 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-dsidphy.c @@ -84,9 +84,25 @@ #define DATA_LANE_0_SKEW_PHASE_MASK GENMASK(2, 0) #define DATA_LANE_0_SKEW_PHASE(x) UPDATE(x, 2, 0) /* Analog Register Part: reg08 */ +#define PLL_POST_DIV_ENABLE_MASK BIT(5) +#define PLL_POST_DIV_ENABLE BIT(5) #define SAMPLE_CLOCK_DIRECTION_MASK BIT(4) #define SAMPLE_CLOCK_DIRECTION_REVERSE BIT(4) #define SAMPLE_CLOCK_DIRECTION_FORWARD 0 +#define LOWFRE_EN_MASK BIT(5) +#define PLL_OUTPUT_FREQUENCY_DIV_BY_1 0 +#define PLL_OUTPUT_FREQUENCY_DIV_BY_2 1 +/* Analog Register Part: reg0b */ +#define CLOCK_LANE_VOD_RANGE_SET_MASK GENMASK(3, 0) +#define CLOCK_LANE_VOD_RANGE_SET(x) UPDATE(x, 3, 0) +#define VOD_MIN_RANGE 0x1 +#define VOD_MID_RANGE 0x3 +#define VOD_BIG_RANGE 0x7 +#define VOD_MAX_RANGE 0xf +/* Analog Register Part: reg1E */ +#define PLL_MODE_SEL_MASK GENMASK(6, 5) +#define PLL_MODE_SEL_LVDS_MODE 0 +#define PLL_MODE_SEL_MIPI_MODE BIT(5) /* Digital Register Part: reg00 */ #define REG_DIG_RSTN_MASK BIT(0) #define REG_DIG_RSTN_NORMAL BIT(0) @@ -102,20 +118,22 @@ #define T_LPX_CNT_MASK GENMASK(5, 0) #define T_LPX_CNT(x) UPDATE(x, 5, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg06 */ +#define T_HS_ZERO_CNT_HI_MASK BIT(7) +#define T_HS_ZERO_CNT_HI(x) UPDATE(x, 7, 7) #define T_HS_PREPARE_CNT_MASK GENMASK(6, 0) #define T_HS_PREPARE_CNT(x) UPDATE(x, 6, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg07 */ -#define T_HS_ZERO_CNT_MASK GENMASK(5, 0) -#define T_HS_ZERO_CNT(x) UPDATE(x, 5, 0) +#define T_HS_ZERO_CNT_LO_MASK GENMASK(5, 0) +#define T_HS_ZERO_CNT_LO(x) UPDATE(x, 5, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg08 */ #define T_HS_TRAIL_CNT_MASK GENMASK(6, 0) #define T_HS_TRAIL_CNT(x) UPDATE(x, 6, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg09 */ -#define T_HS_EXIT_CNT_MASK GENMASK(4, 0) -#define T_HS_EXIT_CNT(x) UPDATE(x, 4, 0) +#define T_HS_EXIT_CNT_LO_MASK GENMASK(4, 0) +#define T_HS_EXIT_CNT_LO(x) UPDATE(x, 4, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg0a */ -#define T_CLK_POST_CNT_MASK GENMASK(3, 0) -#define T_CLK_POST_CNT(x) UPDATE(x, 3, 0) +#define T_CLK_POST_CNT_LO_MASK GENMASK(3, 0) +#define T_CLK_POST_CNT_LO(x) UPDATE(x, 3, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg0c */ #define LPDT_TX_PPI_SYNC_MASK BIT(2) #define LPDT_TX_PPI_SYNC_ENABLE BIT(2) @@ -129,9 +147,13 @@ #define T_CLK_PRE_CNT_MASK GENMASK(3, 0) #define T_CLK_PRE_CNT(x) UPDATE(x, 3, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg10 */ +#define T_CLK_POST_CNT_HI_MASK GENMASK(7, 6) +#define T_CLK_POST_CNT_HI(x) UPDATE(x, 7, 6) #define T_TA_GO_CNT_MASK GENMASK(5, 0) #define T_TA_GO_CNT(x) UPDATE(x, 5, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg11 */ +#define T_HS_EXIT_CNT_HI_MASK BIT(6) +#define T_HS_EXIT_CNT_HI(x) UPDATE(x, 6, 6) #define T_TA_SURE_CNT_MASK GENMASK(5, 0) #define T_TA_SURE_CNT(x) UPDATE(x, 5, 0) /* Clock/Data0/Data1/Data2/Data3 Lane Register Part: reg12 */ @@ -169,11 +191,23 @@ #define DSI_PHY_STATUS 0xb0 #define PHY_LOCK BIT(0) +enum phy_max_rate { + MAX_1GHZ, + MAX_2_5GHZ, +}; + +struct inno_video_phy_plat_data { + const struct inno_mipi_dphy_timing *inno_mipi_dphy_timing_table; + const unsigned int num_timings; + enum phy_max_rate max_rate; +}; + struct inno_dsidphy { struct device *dev; struct clk *ref_clk; struct clk *pclk_phy; struct clk *pclk_host; + const struct inno_video_phy_plat_data *pdata; void __iomem *phy_base; void __iomem *host_base; struct reset_control *rst; @@ -200,6 +234,53 @@ enum { REGISTER_PART_LVDS, }; +struct inno_mipi_dphy_timing { + unsigned long rate; + u8 lpx; + u8 hs_prepare; + u8 clk_lane_hs_zero; + u8 data_lane_hs_zero; + u8 hs_trail; +}; + +static const +struct inno_mipi_dphy_timing inno_mipi_dphy_timing_table_max_1ghz[] = { + { 110000000, 0x0, 0x20, 0x16, 0x02, 0x22}, + { 150000000, 0x0, 0x06, 0x16, 0x03, 0x45}, + { 200000000, 0x0, 0x18, 0x17, 0x04, 0x0b}, + { 250000000, 0x0, 0x05, 0x17, 0x05, 0x16}, + { 300000000, 0x0, 0x51, 0x18, 0x06, 0x2c}, + { 400000000, 0x0, 0x64, 0x19, 0x07, 0x33}, + { 500000000, 0x0, 0x20, 0x1b, 0x07, 0x4e}, + { 600000000, 0x0, 0x6a, 0x1d, 0x08, 0x3a}, + { 700000000, 0x0, 0x3e, 0x1e, 0x08, 0x6a}, + { 800000000, 0x0, 0x21, 0x1f, 0x09, 0x29}, + {1000000000, 0x0, 0x09, 0x20, 0x09, 0x27}, +}; + +static const +struct inno_mipi_dphy_timing inno_mipi_dphy_timing_table_max_2_5ghz[] = { + { 110000000, 0x02, 0x7f, 0x16, 0x02, 0x02}, + { 150000000, 0x02, 0x7f, 0x16, 0x03, 0x02}, + { 200000000, 0x02, 0x7f, 0x17, 0x04, 0x02}, + { 250000000, 0x02, 0x7f, 0x17, 0x05, 0x04}, + { 300000000, 0x02, 0x7f, 0x18, 0x06, 0x04}, + { 400000000, 0x03, 0x7e, 0x19, 0x07, 0x04}, + { 500000000, 0x03, 0x7c, 0x1b, 0x07, 0x08}, + { 600000000, 0x03, 0x70, 0x1d, 0x08, 0x10}, + { 700000000, 0x05, 0x40, 0x1e, 0x08, 0x30}, + { 800000000, 0x05, 0x02, 0x1f, 0x09, 0x30}, + {1000000000, 0x05, 0x08, 0x20, 0x09, 0x30}, + {1200000000, 0x06, 0x03, 0x32, 0x14, 0x0f}, + {1400000000, 0x09, 0x03, 0x32, 0x14, 0x0f}, + {1600000000, 0x0d, 0x42, 0x36, 0x0e, 0x0f}, + {1800000000, 0x0e, 0x47, 0x7a, 0x0e, 0x0f}, + {2000000000, 0x11, 0x64, 0x7a, 0x0e, 0x0b}, + {2200000000, 0x13, 0x64, 0x7e, 0x15, 0x0b}, + {2400000000, 0x13, 0x33, 0x7f, 0x15, 0x6a}, + {2500000000, 0x15, 0x54, 0x7f, 0x15, 0x6a}, +}; + static inline struct inno_dsidphy *hw_to_inno(struct clk_hw *hw) { return container_of(hw, struct inno_dsidphy, pll.hw); @@ -290,31 +371,15 @@ static unsigned long inno_dsidphy_pll_calc_rate(struct inno_dsidphy *inno, static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno) { struct phy_configure_opts_mipi_dphy *cfg = &inno->dphy_cfg; - const struct { - unsigned long rate; - u8 hs_prepare; - u8 clk_lane_hs_zero; - u8 data_lane_hs_zero; - u8 hs_trail; - } timings[] = { - { 110000000, 0x20, 0x16, 0x02, 0x22}, - { 150000000, 0x06, 0x16, 0x03, 0x45}, - { 200000000, 0x18, 0x17, 0x04, 0x0b}, - { 250000000, 0x05, 0x17, 0x05, 0x16}, - { 300000000, 0x51, 0x18, 0x06, 0x2c}, - { 400000000, 0x64, 0x19, 0x07, 0x33}, - { 500000000, 0x20, 0x1b, 0x07, 0x4e}, - { 600000000, 0x6a, 0x1d, 0x08, 0x3a}, - { 700000000, 0x3e, 0x1e, 0x08, 0x6a}, - { 800000000, 0x21, 0x1f, 0x09, 0x29}, - {1000000000, 0x09, 0x20, 0x09, 0x27}, - }; + const struct inno_mipi_dphy_timing *timings; u32 t_txbyteclkhs, t_txclkesc; u32 txbyteclkhs, txclkesc, esc_clk_div; u32 hs_exit, clk_post, clk_pre, wakeup, lpx, ta_go, ta_sure, ta_wait; u32 hs_prepare, hs_trail, hs_zero, clk_lane_hs_zero, data_lane_hs_zero; unsigned int i; + timings = inno->pdata->inno_mipi_dphy_timing_table; + inno_dsidphy_pll_calc_rate(inno, cfg->hs_clk_rate); /* Select MIPI mode */ @@ -327,6 +392,13 @@ static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno) REG_FBDIV_HI_MASK, REG_FBDIV_HI(inno->pll.fbdiv)); phy_update_bits(inno, REGISTER_PART_ANALOG, 0x04, REG_FBDIV_LO_MASK, REG_FBDIV_LO(inno->pll.fbdiv)); + if (inno->pdata->max_rate == MAX_2_5GHZ) { + phy_update_bits(inno, REGISTER_PART_ANALOG, 0x08, + PLL_POST_DIV_ENABLE_MASK, PLL_POST_DIV_ENABLE); + phy_update_bits(inno, REGISTER_PART_ANALOG, 0x0b, + CLOCK_LANE_VOD_RANGE_SET_MASK, + CLOCK_LANE_VOD_RANGE_SET(VOD_MAX_RANGE)); + } /* Enable PLL and LDO */ phy_update_bits(inno, REGISTER_PART_ANALOG, 0x01, REG_LDOPD_MASK | REG_PLLPD_MASK, @@ -367,14 +439,6 @@ static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno) */ clk_pre = DIV_ROUND_UP(cfg->clk_pre, BITS_PER_BYTE); - /* - * The value of counter for HS Tlpx Time - * Tlpx = Tpin_txbyteclkhs * (2 + value) - */ - lpx = DIV_ROUND_UP(cfg->lpx, t_txbyteclkhs); - if (lpx >= 2) - lpx -= 2; - /* * The value of counter for HS Tta-go * Tta-go for turnaround @@ -394,13 +458,24 @@ static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno) */ ta_wait = DIV_ROUND_UP(cfg->ta_get, t_txclkesc); - for (i = 0; i < ARRAY_SIZE(timings); i++) + for (i = 0; i < inno->pdata->num_timings; i++) if (inno->pll.rate <= timings[i].rate) break; - if (i == ARRAY_SIZE(timings)) + if (i == inno->pdata->num_timings) --i; + /* + * The value of counter for HS Tlpx Time + * Tlpx = Tpin_txbyteclkhs * (2 + value) + */ + if (inno->pdata->max_rate == MAX_1GHZ) { + lpx = DIV_ROUND_UP(cfg->lpx, t_txbyteclkhs); + if (lpx >= 2) + lpx -= 2; + } else + lpx = timings[i].lpx; + hs_prepare = timings[i].hs_prepare; hs_trail = timings[i].hs_trail; clk_lane_hs_zero = timings[i].clk_lane_hs_zero; @@ -417,14 +492,23 @@ static void inno_dsidphy_mipi_mode_enable(struct inno_dsidphy *inno) T_LPX_CNT(lpx)); phy_update_bits(inno, i, 0x06, T_HS_PREPARE_CNT_MASK, T_HS_PREPARE_CNT(hs_prepare)); - phy_update_bits(inno, i, 0x07, T_HS_ZERO_CNT_MASK, - T_HS_ZERO_CNT(hs_zero)); + if (inno->pdata->max_rate == MAX_2_5GHZ) + phy_update_bits(inno, i, 0x06, T_HS_ZERO_CNT_HI_MASK, + T_HS_ZERO_CNT_HI(hs_zero >> 6)); + phy_update_bits(inno, i, 0x07, T_HS_ZERO_CNT_LO_MASK, + T_HS_ZERO_CNT_LO(hs_zero)); phy_update_bits(inno, i, 0x08, T_HS_TRAIL_CNT_MASK, T_HS_TRAIL_CNT(hs_trail)); - phy_update_bits(inno, i, 0x09, T_HS_EXIT_CNT_MASK, - T_HS_EXIT_CNT(hs_exit)); - phy_update_bits(inno, i, 0x0a, T_CLK_POST_CNT_MASK, - T_CLK_POST_CNT(clk_post)); + if (inno->pdata->max_rate == MAX_2_5GHZ) + phy_update_bits(inno, i, 0x11, T_HS_EXIT_CNT_HI_MASK, + T_HS_EXIT_CNT_HI(hs_exit >> 5)); + phy_update_bits(inno, i, 0x09, T_HS_EXIT_CNT_LO_MASK, + T_HS_EXIT_CNT_LO(hs_exit)); + if (inno->pdata->max_rate == MAX_2_5GHZ) + phy_update_bits(inno, i, 0x10, T_CLK_POST_CNT_HI_MASK, + T_CLK_POST_CNT_HI(clk_post >> 4)); + phy_update_bits(inno, i, 0x0a, T_CLK_POST_CNT_LO_MASK, + T_CLK_POST_CNT_LO(clk_post)); phy_update_bits(inno, i, 0x0e, T_CLK_PRE_CNT_MASK, T_CLK_PRE_CNT(clk_pre)); phy_update_bits(inno, i, 0x0c, T_WAKEUP_CNT_HI_MASK, @@ -452,8 +536,9 @@ static void inno_dsidphy_lvds_mode_enable(struct inno_dsidphy *inno) /* Sample clock reverse direction */ phy_update_bits(inno, REGISTER_PART_ANALOG, 0x08, - SAMPLE_CLOCK_DIRECTION_MASK, - SAMPLE_CLOCK_DIRECTION_REVERSE); + SAMPLE_CLOCK_DIRECTION_MASK | LOWFRE_EN_MASK, + SAMPLE_CLOCK_DIRECTION_REVERSE | + PLL_OUTPUT_FREQUENCY_DIV_BY_1); /* Select LVDS mode */ phy_update_bits(inno, REGISTER_PART_LVDS, 0x03, @@ -473,6 +558,10 @@ static void inno_dsidphy_lvds_mode_enable(struct inno_dsidphy *inno) msleep(20); + /* Select PLL mode */ + phy_update_bits(inno, REGISTER_PART_ANALOG, 0x1e, + PLL_MODE_SEL_MASK, PLL_MODE_SEL_LVDS_MODE); + /* Reset LVDS digital logic */ phy_update_bits(inno, REGISTER_PART_LVDS, 0x00, LVDS_DIGITAL_INTERNAL_RESET_MASK, @@ -592,6 +681,18 @@ static const struct phy_ops inno_dsidphy_ops = { .owner = THIS_MODULE, }; +static const struct inno_video_phy_plat_data max_1ghz_video_phy_plat_data = { + .inno_mipi_dphy_timing_table = inno_mipi_dphy_timing_table_max_1ghz, + .num_timings = ARRAY_SIZE(inno_mipi_dphy_timing_table_max_1ghz), + .max_rate = MAX_1GHZ, +}; + +static const struct inno_video_phy_plat_data max_2_5ghz_video_phy_plat_data = { + .inno_mipi_dphy_timing_table = inno_mipi_dphy_timing_table_max_2_5ghz, + .num_timings = ARRAY_SIZE(inno_mipi_dphy_timing_table_max_2_5ghz), + .max_rate = MAX_2_5GHZ, +}; + static int inno_dsidphy_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -605,6 +706,7 @@ static int inno_dsidphy_probe(struct platform_device *pdev) return -ENOMEM; inno->dev = dev; + inno->pdata = of_device_get_match_data(inno->dev); platform_set_drvdata(pdev, inno); inno->phy_base = devm_platform_ioremap_resource(pdev, 0); @@ -663,9 +765,19 @@ static int inno_dsidphy_remove(struct platform_device *pdev) } static const struct of_device_id inno_dsidphy_of_match[] = { - { .compatible = "rockchip,px30-dsi-dphy", }, - { .compatible = "rockchip,rk3128-dsi-dphy", }, - { .compatible = "rockchip,rk3368-dsi-dphy", }, + { + .compatible = "rockchip,px30-dsi-dphy", + .data = &max_1ghz_video_phy_plat_data, + }, { + .compatible = "rockchip,rk3128-dsi-dphy", + .data = &max_1ghz_video_phy_plat_data, + }, { + .compatible = "rockchip,rk3368-dsi-dphy", + .data = &max_1ghz_video_phy_plat_data, + }, { + .compatible = "rockchip,rk3568-dsi-dphy", + .data = &max_2_5ghz_video_phy_plat_data, + }, {} }; MODULE_DEVICE_TABLE(of, inno_dsidphy_of_match); diff --git a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c index 0b1e9337ee8e2b169989cdb34724d073c42fc4a7..e6ededc515239086a7b2e38ffd0a453f6ec88ba6 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c @@ -1124,7 +1124,7 @@ static int rockchip_usb2phy_otg_port_init(struct rockchip_usb2phy *rphy, struct rockchip_usb2phy_port *rport, struct device_node *child_np) { - int ret; + int ret, id; rport->port_id = USB2PHY_PORT_OTG; rport->port_cfg = &rphy->phy_cfg->port_cfgs[USB2PHY_PORT_OTG]; @@ -1162,13 +1162,15 @@ static int rockchip_usb2phy_otg_port_init(struct rockchip_usb2phy *rphy, ret = devm_extcon_register_notifier(rphy->dev, rphy->edev, EXTCON_USB_HOST, &rport->event_nb); - if (ret) + if (ret) { dev_err(rphy->dev, "register USB HOST notifier failed\n"); + goto out; + } if (!of_property_read_bool(rphy->dev->of_node, "extcon")) { /* do initial sync of usb state */ - ret = property_enabled(rphy->grf, &rport->port_cfg->utmi_id); - extcon_set_state_sync(rphy->edev, EXTCON_USB_HOST, !ret); + id = property_enabled(rphy->grf, &rport->port_cfg->utmi_id); + extcon_set_state_sync(rphy->edev, EXTCON_USB_HOST, !id); } } diff --git a/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c b/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c new file mode 100644 index 0000000000000000000000000000000000000000..1d355b32ba5599394c090c8939592f49c0408a82 --- /dev/null +++ b/drivers/phy/rockchip/phy-rockchip-snps-pcie3.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rockchip PCIE3.0 phy driver + * + * Copyright (C) 2022 Rockchip Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Register for RK3568 */ +#define GRF_PCIE30PHY_CON1 0x4 +#define GRF_PCIE30PHY_CON6 0x18 +#define GRF_PCIE30PHY_CON9 0x24 +#define GRF_PCIE30PHY_DA_OCM (BIT(15) | BIT(31)) +#define GRF_PCIE30PHY_STATUS0 0x80 +#define GRF_PCIE30PHY_WR_EN (0xf << 16) +#define SRAM_INIT_DONE(reg) (reg & BIT(14)) + +#define RK3568_BIFURCATION_LANE_0_1 BIT(0) + +/* Register for RK3588 */ +#define PHP_GRF_PCIESEL_CON 0x100 +#define RK3588_PCIE3PHY_GRF_CMN_CON0 0x0 +#define RK3588_PCIE3PHY_GRF_PHY0_STATUS1 0x904 +#define RK3588_PCIE3PHY_GRF_PHY1_STATUS1 0xa04 +#define RK3588_SRAM_INIT_DONE(reg) (reg & BIT(0)) + +#define RK3588_BIFURCATION_LANE_0_1 BIT(0) +#define RK3588_BIFURCATION_LANE_2_3 BIT(1) +#define RK3588_LANE_AGGREGATION BIT(2) + +struct rockchip_p3phy_ops; + +struct rockchip_p3phy_priv { + const struct rockchip_p3phy_ops *ops; + void __iomem *mmio; + /* mode: RC, EP */ + int mode; + /* pcie30_phymode: Aggregation, Bifurcation */ + int pcie30_phymode; + struct regmap *phy_grf; + struct regmap *pipe_grf; + struct reset_control *p30phy; + struct phy *phy; + struct clk_bulk_data *clks; + int num_clks; + int num_lanes; + u32 lanes[4]; +}; + +struct rockchip_p3phy_ops { + int (*phy_init)(struct rockchip_p3phy_priv *priv); +}; + +static int rockchip_p3phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct rockchip_p3phy_priv *priv = phy_get_drvdata(phy); + + /* Actually We don't care EP/RC mode, but just record it */ + switch (submode) { + case PHY_MODE_PCIE_RC: + priv->mode = PHY_MODE_PCIE_RC; + break; + case PHY_MODE_PCIE_EP: + priv->mode = PHY_MODE_PCIE_EP; + break; + default: + dev_err(&phy->dev, "%s, invalid mode\n", __func__); + return -EINVAL; + } + + return 0; +} + +static int rockchip_p3phy_rk3568_init(struct rockchip_p3phy_priv *priv) +{ + struct phy *phy = priv->phy; + bool bifurcation = false; + int ret; + u32 reg; + + /* Deassert PCIe PMA output clamp mode */ + regmap_write(priv->phy_grf, GRF_PCIE30PHY_CON9, GRF_PCIE30PHY_DA_OCM); + + for (int i = 0; i < priv->num_lanes; i++) { + dev_info(&phy->dev, "lane number %d, val %d\n", i, priv->lanes[i]); + if (priv->lanes[i] > 1) + bifurcation = true; + } + + /* Set bifurcation if needed, and it doesn't care RC/EP */ + if (bifurcation) { + dev_info(&phy->dev, "bifurcation enabled\n"); + regmap_write(priv->phy_grf, GRF_PCIE30PHY_CON6, + GRF_PCIE30PHY_WR_EN | RK3568_BIFURCATION_LANE_0_1); + regmap_write(priv->phy_grf, GRF_PCIE30PHY_CON1, + GRF_PCIE30PHY_DA_OCM); + } else { + dev_dbg(&phy->dev, "bifurcation disabled\n"); + regmap_write(priv->phy_grf, GRF_PCIE30PHY_CON6, + GRF_PCIE30PHY_WR_EN & ~RK3568_BIFURCATION_LANE_0_1); + } + + reset_control_deassert(priv->p30phy); + + ret = regmap_read_poll_timeout(priv->phy_grf, + GRF_PCIE30PHY_STATUS0, + reg, SRAM_INIT_DONE(reg), + 0, 500); + if (ret) + dev_err(&priv->phy->dev, "%s: lock failed 0x%x, check input refclk and power supply\n", + __func__, reg); + return ret; +} + +static const struct rockchip_p3phy_ops rk3568_ops = { + .phy_init = rockchip_p3phy_rk3568_init, +}; + +static int rockchip_p3phy_rk3588_init(struct rockchip_p3phy_priv *priv) +{ + u32 reg = 0; + u8 mode = 0; + int ret; + + /* Deassert PCIe PMA output clamp mode */ + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_CMN_CON0, BIT(8) | BIT(24)); + + /* Set bifurcation if needed */ + for (int i = 0; i < priv->num_lanes; i++) { + if (!priv->lanes[i]) + mode |= (BIT(i) << 3); + + if (priv->lanes[i] > 1) + mode |= (BIT(i) >> 1); + } + + if (!mode) + reg = RK3588_LANE_AGGREGATION; + else { + if (mode & (BIT(0) | BIT(1))) + reg |= RK3588_BIFURCATION_LANE_0_1; + + if (mode & (BIT(2) | BIT(3))) + reg |= RK3588_BIFURCATION_LANE_2_3; + } + + regmap_write(priv->phy_grf, RK3588_PCIE3PHY_GRF_CMN_CON0, (0x7<<16) | reg); + + /* Set pcie1ln_sel in PHP_GRF_PCIESEL_CON */ + if (!IS_ERR(priv->pipe_grf)) { + reg = (mode & (BIT(6) | BIT(7))) >> 6; + if (reg) + regmap_write(priv->pipe_grf, PHP_GRF_PCIESEL_CON, + (reg << 16) | reg); + } + + reset_control_deassert(priv->p30phy); + + ret = regmap_read_poll_timeout(priv->phy_grf, + RK3588_PCIE3PHY_GRF_PHY0_STATUS1, + reg, RK3588_SRAM_INIT_DONE(reg), + 0, 500); + ret |= regmap_read_poll_timeout(priv->phy_grf, + RK3588_PCIE3PHY_GRF_PHY1_STATUS1, + reg, RK3588_SRAM_INIT_DONE(reg), + 0, 500); + if (ret) + dev_err(&priv->phy->dev, "lock failed 0x%x, check input refclk and power supply\n", + reg); + return ret; +} + +static const struct rockchip_p3phy_ops rk3588_ops = { + .phy_init = rockchip_p3phy_rk3588_init, +}; + +static int rochchip_p3phy_init(struct phy *phy) +{ + struct rockchip_p3phy_priv *priv = phy_get_drvdata(phy); + int ret; + + ret = clk_bulk_prepare_enable(priv->num_clks, priv->clks); + if (ret) { + dev_err(&priv->phy->dev, "failed to enable PCIe bulk clks %d\n", ret); + return ret; + } + + reset_control_assert(priv->p30phy); + udelay(1); + + if (priv->ops->phy_init) { + ret = priv->ops->phy_init(priv); + if (ret) + clk_bulk_disable_unprepare(priv->num_clks, priv->clks); + } + + return ret; +} + +static int rochchip_p3phy_exit(struct phy *phy) +{ + struct rockchip_p3phy_priv *priv = phy_get_drvdata(phy); + + clk_bulk_disable_unprepare(priv->num_clks, priv->clks); + reset_control_assert(priv->p30phy); + return 0; +} + +static const struct phy_ops rochchip_p3phy_ops = { + .init = rochchip_p3phy_init, + .exit = rochchip_p3phy_exit, + .set_mode = rockchip_p3phy_set_mode, + .owner = THIS_MODULE, +}; + +static int rockchip_p3phy_probe(struct platform_device *pdev) +{ + struct phy_provider *phy_provider; + struct device *dev = &pdev->dev; + struct rockchip_p3phy_priv *priv; + struct device_node *np = dev->of_node; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->mmio = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(priv->mmio)) { + ret = PTR_ERR(priv->mmio); + return ret; + } + + priv->ops = of_device_get_match_data(&pdev->dev); + if (!priv->ops) { + dev_err(dev, "no of match data provided\n"); + return -EINVAL; + } + + priv->phy_grf = syscon_regmap_lookup_by_phandle(np, "rockchip,phy-grf"); + if (IS_ERR(priv->phy_grf)) { + dev_err(dev, "failed to find rockchip,phy_grf regmap\n"); + return PTR_ERR(priv->phy_grf); + } + + if (of_device_is_compatible(np, "rockchip,rk3588-pcie3-phy")) { + priv->pipe_grf = + syscon_regmap_lookup_by_phandle(dev->of_node, + "rockchip,pipe-grf"); + if (IS_ERR(priv->pipe_grf)) + dev_info(dev, "failed to find rockchip,pipe_grf regmap\n"); + } else { + priv->pipe_grf = NULL; + } + + priv->num_lanes = of_property_read_variable_u32_array(dev->of_node, "data-lanes", + priv->lanes, 2, + ARRAY_SIZE(priv->lanes)); + + /* if no data-lanes assume aggregation */ + if (priv->num_lanes == -EINVAL) { + dev_dbg(dev, "no data-lanes property found\n"); + priv->num_lanes = 1; + priv->lanes[0] = 1; + } else if (priv->num_lanes < 0) { + dev_err(dev, "failed to read data-lanes property %d\n", priv->num_lanes); + return priv->num_lanes; + } + + priv->phy = devm_phy_create(dev, NULL, &rochchip_p3phy_ops); + if (IS_ERR(priv->phy)) { + dev_err(dev, "failed to create combphy\n"); + return PTR_ERR(priv->phy); + } + + priv->p30phy = devm_reset_control_get_optional_exclusive(dev, "phy"); + if (IS_ERR(priv->p30phy)) { + return dev_err_probe(dev, PTR_ERR(priv->p30phy), + "failed to get phy reset control\n"); + } + if (!priv->p30phy) + dev_info(dev, "no phy reset control specified\n"); + + priv->num_clks = devm_clk_bulk_get_all(dev, &priv->clks); + if (priv->num_clks < 1) + return -ENODEV; + + dev_set_drvdata(dev, priv); + phy_set_drvdata(priv->phy, priv); + phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + return PTR_ERR_OR_ZERO(phy_provider); +} + +static const struct of_device_id rockchip_p3phy_of_match[] = { + { .compatible = "rockchip,rk3568-pcie3-phy", .data = &rk3568_ops }, + { .compatible = "rockchip,rk3588-pcie3-phy", .data = &rk3588_ops }, + { }, +}; +MODULE_DEVICE_TABLE(of, rockchip_p3phy_of_match); + +static struct platform_driver rockchip_p3phy_driver = { + .probe = rockchip_p3phy_probe, + .driver = { + .name = "rockchip-snps-pcie3-phy", + .of_match_table = rockchip_p3phy_of_match, + }, +}; +module_platform_driver(rockchip_p3phy_driver); +MODULE_DESCRIPTION("Rockchip Synopsys PCIe 3.0 PHY driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/sunplus/Kconfig b/drivers/phy/sunplus/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..3bd3cfb53a63a6b320da1a10143384d0f3cbc27e --- /dev/null +++ b/drivers/phy/sunplus/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config PHY_SUNPLUS_USB + tristate "Sunplus SP7021 USB 2.0 PHY driver" + depends on OF && (SOC_SP7021 || COMPILE_TEST) + select GENERIC_PHY + help + Enable this to support the USB 2.0 PHY on Sunplus SP7021 + SoC. The USB 2.0 PHY controller supports battery charger + and synchronous signals, various power down modes including + operating, partial and suspend modes, and high-speed, + full-speed and low-speed data transfer. diff --git a/drivers/phy/sunplus/Makefile b/drivers/phy/sunplus/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..71754d5cb54577588f2c43f6698874620deff933 --- /dev/null +++ b/drivers/phy/sunplus/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_PHY_SUNPLUS_USB) += phy-sunplus-usb2.o diff --git a/drivers/phy/sunplus/phy-sunplus-usb2.c b/drivers/phy/sunplus/phy-sunplus-usb2.c new file mode 100644 index 0000000000000000000000000000000000000000..b932087c55b2943dde6519b684380da802e7f204 --- /dev/null +++ b/drivers/phy/sunplus/phy-sunplus-usb2.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Sunplus SP7021 USB 2.0 phy driver + * + * Copyright (C) 2022 Sunplus Technology Inc., All rights reserved. + * + * Note 1 : non-posted write command for the registers accesses of + * Sunplus SP7021. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HIGH_MASK_BITS GENMASK(31, 16) +#define LOW_MASK_BITS GENMASK(15, 0) +#define OTP_DISC_LEVEL_DEFAULT 0xd + +/* GROUP UPHY */ +#define CONFIG1 0x4 +#define J_HS_TX_PWRSAV BIT(5) +#define CONFIG3 0xc +#define J_FORCE_DISC_ON BIT(5) +#define J_DEBUG_CTRL_ADDR_MACRO BIT(0) +#define CONFIG7 0x1c +#define J_DISC 0X1f +#define CONFIG9 0x24 +#define J_ECO_PATH BIT(6) +#define CONFIG16 0x40 +#define J_TBCWAIT_MASK GENMASK(6, 5) +#define J_TBCWAIT_1P1_MS FIELD_PREP(J_TBCWAIT_MASK, 0) +#define J_TVDM_SRC_DIS_MASK GENMASK(4, 3) +#define J_TVDM_SRC_DIS_8P2_MS FIELD_PREP(J_TVDM_SRC_DIS_MASK, 3) +#define J_TVDM_SRC_EN_MASK GENMASK(2, 1) +#define J_TVDM_SRC_EN_1P6_MS FIELD_PREP(J_TVDM_SRC_EN_MASK, 0) +#define J_BC_EN BIT(0) +#define CONFIG17 0x44 +#define IBG_TRIM0_MASK GENMASK(7, 5) +#define IBG_TRIM0_SSLVHT FIELD_PREP(IBG_TRIM0_MASK, 4) +#define J_VDATREE_TRIM_MASK GENMASK(4, 1) +#define J_VDATREE_TRIM_DEFAULT FIELD_PREP(J_VDATREE_TRIM_MASK, 9) +#define CONFIG23 0x5c +#define PROB_MASK GENMASK(5, 3) +#define PROB FIELD_PREP(PROB_MASK, 7) + +/* GROUP MOON4 */ +#define UPHY_CONTROL0 0x0 +#define UPHY_CONTROL1 0x4 +#define UPHY_CONTROL2 0x8 +#define MO1_UPHY_RX_CLK_SEL BIT(6) +#define MASK_MO1_UPHY_RX_CLK_SEL BIT(6 + 16) +#define UPHY_CONTROL3 0xc +#define MO1_UPHY_PLL_POWER_OFF_SEL BIT(7) +#define MASK_MO1_UPHY_PLL_POWER_OFF_SEL BIT(7 + 16) +#define MO1_UPHY_PLL_POWER_OFF BIT(3) +#define MASK_UPHY_PLL_POWER_OFF BIT(3 + 16) + +struct sp_usbphy { + struct device *dev; + struct resource *phy_res_mem; + struct resource *moon4_res_mem; + struct reset_control *rstc; + struct clk *phy_clk; + void __iomem *phy_regs; + void __iomem *moon4_regs; + u32 disc_vol_addr_off; +}; + +static int update_disc_vol(struct sp_usbphy *usbphy) +{ + struct nvmem_cell *cell; + char *disc_name = "disc_vol"; + ssize_t otp_l = 0; + char *otp_v; + u32 val, set; + + cell = nvmem_cell_get(usbphy->dev, disc_name); + if (IS_ERR_OR_NULL(cell)) { + if (PTR_ERR(cell) == -EPROBE_DEFER) + return -EPROBE_DEFER; + } + + otp_v = nvmem_cell_read(cell, &otp_l); + nvmem_cell_put(cell); + + if (!IS_ERR(otp_v)) { + set = *(otp_v + 1); + set = (set << (sizeof(char) * 8)) | *otp_v; + set = (set >> usbphy->disc_vol_addr_off) & J_DISC; + } + + if (IS_ERR(otp_v) || set == 0) + set = OTP_DISC_LEVEL_DEFAULT; + + val = readl(usbphy->phy_regs + CONFIG7); + val = (val & ~J_DISC) | set; + writel(val, usbphy->phy_regs + CONFIG7); + + return 0; +} + +static int sp_uphy_init(struct phy *phy) +{ + struct sp_usbphy *usbphy = phy_get_drvdata(phy); + u32 val; + int ret; + + ret = clk_prepare_enable(usbphy->phy_clk); + if (ret) + goto err_clk; + + ret = reset_control_deassert(usbphy->rstc); + if (ret) + goto err_reset; + + /* Default value modification */ + writel(HIGH_MASK_BITS | 0x4002, usbphy->moon4_regs + UPHY_CONTROL0); + writel(HIGH_MASK_BITS | 0x8747, usbphy->moon4_regs + UPHY_CONTROL1); + + /* disconnect voltage */ + ret = update_disc_vol(usbphy); + if (ret < 0) + return ret; + + /* board uphy 0 internal register modification for tid certification */ + val = readl(usbphy->phy_regs + CONFIG9); + val &= ~(J_ECO_PATH); + writel(val, usbphy->phy_regs + CONFIG9); + + val = readl(usbphy->phy_regs + CONFIG1); + val &= ~(J_HS_TX_PWRSAV); + writel(val, usbphy->phy_regs + CONFIG1); + + val = readl(usbphy->phy_regs + CONFIG23); + val = (val & ~PROB) | PROB; + writel(val, usbphy->phy_regs + CONFIG23); + + /* port 0 uphy clk fix */ + writel(MASK_MO1_UPHY_RX_CLK_SEL | MO1_UPHY_RX_CLK_SEL, + usbphy->moon4_regs + UPHY_CONTROL2); + + /* battery charger */ + writel(J_TBCWAIT_1P1_MS | J_TVDM_SRC_DIS_8P2_MS | J_TVDM_SRC_EN_1P6_MS | J_BC_EN, + usbphy->phy_regs + CONFIG16); + writel(IBG_TRIM0_SSLVHT | J_VDATREE_TRIM_DEFAULT, usbphy->phy_regs + CONFIG17); + + /* chirp mode */ + writel(J_FORCE_DISC_ON | J_DEBUG_CTRL_ADDR_MACRO, usbphy->phy_regs + CONFIG3); + + return 0; + +err_reset: + reset_control_assert(usbphy->rstc); +err_clk: + clk_disable_unprepare(usbphy->phy_clk); + + return ret; +} + +static int sp_uphy_power_on(struct phy *phy) +{ + struct sp_usbphy *usbphy = phy_get_drvdata(phy); + u32 pll_pwr_on, pll_pwr_off; + + /* PLL power off/on twice */ + pll_pwr_off = (readl(usbphy->moon4_regs + UPHY_CONTROL3) & ~LOW_MASK_BITS) + | MO1_UPHY_PLL_POWER_OFF_SEL | MO1_UPHY_PLL_POWER_OFF; + pll_pwr_on = (readl(usbphy->moon4_regs + UPHY_CONTROL3) & ~LOW_MASK_BITS) + | MO1_UPHY_PLL_POWER_OFF_SEL; + + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | pll_pwr_off, + usbphy->moon4_regs + UPHY_CONTROL3); + mdelay(1); + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | pll_pwr_on, + usbphy->moon4_regs + UPHY_CONTROL3); + mdelay(1); + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | pll_pwr_off, + usbphy->moon4_regs + UPHY_CONTROL3); + mdelay(1); + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | pll_pwr_on, + usbphy->moon4_regs + UPHY_CONTROL3); + mdelay(1); + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | 0x0, + usbphy->moon4_regs + UPHY_CONTROL3); + + return 0; +} + +static int sp_uphy_power_off(struct phy *phy) +{ + struct sp_usbphy *usbphy = phy_get_drvdata(phy); + u32 pll_pwr_off; + + pll_pwr_off = (readl(usbphy->moon4_regs + UPHY_CONTROL3) & ~LOW_MASK_BITS) + | MO1_UPHY_PLL_POWER_OFF_SEL | MO1_UPHY_PLL_POWER_OFF; + + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | pll_pwr_off, + usbphy->moon4_regs + UPHY_CONTROL3); + mdelay(1); + writel(MASK_MO1_UPHY_PLL_POWER_OFF_SEL | MASK_UPHY_PLL_POWER_OFF | 0x0, + usbphy->moon4_regs + UPHY_CONTROL3); + + return 0; +} + +static int sp_uphy_exit(struct phy *phy) +{ + struct sp_usbphy *usbphy = phy_get_drvdata(phy); + + reset_control_assert(usbphy->rstc); + clk_disable_unprepare(usbphy->phy_clk); + + return 0; +} + +static const struct phy_ops sp_uphy_ops = { + .init = sp_uphy_init, + .power_on = sp_uphy_power_on, + .power_off = sp_uphy_power_off, + .exit = sp_uphy_exit, +}; + +static const struct of_device_id sp_uphy_dt_ids[] = { + {.compatible = "sunplus,sp7021-usb2-phy", }, + { } +}; +MODULE_DEVICE_TABLE(of, sp_uphy_dt_ids); + +static int sp_usb_phy_probe(struct platform_device *pdev) +{ + struct sp_usbphy *usbphy; + struct phy_provider *phy_provider; + struct phy *phy; + int ret; + + usbphy = devm_kzalloc(&pdev->dev, sizeof(*usbphy), GFP_KERNEL); + if (!usbphy) + return -ENOMEM; + + usbphy->dev = &pdev->dev; + + usbphy->phy_res_mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "phy"); + usbphy->phy_regs = devm_ioremap_resource(&pdev->dev, usbphy->phy_res_mem); + if (IS_ERR(usbphy->phy_regs)) + return PTR_ERR(usbphy->phy_regs); + + usbphy->moon4_res_mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "moon4"); + usbphy->moon4_regs = devm_ioremap(&pdev->dev, usbphy->moon4_res_mem->start, + resource_size(usbphy->moon4_res_mem)); + if (IS_ERR(usbphy->moon4_regs)) + return PTR_ERR(usbphy->moon4_regs); + + usbphy->phy_clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(usbphy->phy_clk)) + return PTR_ERR(usbphy->phy_clk); + + usbphy->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(usbphy->rstc)) + return PTR_ERR(usbphy->rstc); + + of_property_read_u32(pdev->dev.of_node, "sunplus,disc-vol-addr-off", + &usbphy->disc_vol_addr_off); + + phy = devm_phy_create(&pdev->dev, NULL, &sp_uphy_ops); + if (IS_ERR(phy)) { + ret = -PTR_ERR(phy); + return ret; + } + + phy_set_drvdata(phy, usbphy); + phy_provider = devm_of_phy_provider_register(&pdev->dev, of_phy_simple_xlate); + + return PTR_ERR_OR_ZERO(phy_provider); +} + +static struct platform_driver sunplus_usb_phy_driver = { + .probe = sp_usb_phy_probe, + .driver = { + .name = "sunplus-usb2-phy", + .of_match_table = sp_uphy_dt_ids, + }, +}; +module_platform_driver(sunplus_usb_phy_driver); + +MODULE_AUTHOR("Vincent Shih "); +MODULE_DESCRIPTION("Sunplus USB 2.0 phy driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c index ae3915ed9fefa869234d286f005c45f3273e8e53..0996ede63387a25a4760b82f70c3ec7c7e3873c1 100644 --- a/drivers/phy/tegra/xusb-tegra186.c +++ b/drivers/phy/tegra/xusb-tegra186.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved. */ #include @@ -638,7 +638,7 @@ static void tegra186_utmi_bias_pad_power_off(struct tegra_xusb_padctl *padctl) mutex_unlock(&padctl->lock); } -static void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy) +static void tegra186_utmi_pad_power_on(struct phy *phy) { struct tegra_xusb_lane *lane = phy_get_drvdata(phy); struct tegra_xusb_padctl *padctl = lane->pad->padctl; @@ -656,6 +656,8 @@ static void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy) return; } + dev_dbg(dev, "power on UTMI pad %u\n", index); + tegra186_utmi_bias_pad_power_on(padctl); udelay(2); @@ -669,7 +671,7 @@ static void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy) padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index)); } -static void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy) +static void tegra186_utmi_pad_power_down(struct phy *phy) { struct tegra_xusb_lane *lane = phy_get_drvdata(phy); struct tegra_xusb_padctl *padctl = lane->pad->padctl; @@ -679,6 +681,8 @@ static void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy) if (!phy) return; + dev_dbg(padctl->dev, "power down UTMI pad %u\n", index); + value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index)); value |= USB2_OTG_PD; padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index)); @@ -849,15 +853,14 @@ static int tegra186_utmi_phy_power_on(struct phy *phy) value |= RPD_CTRL(priv->calib.rpd_ctrl); padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index)); - /* TODO: pad power saving */ - tegra_phy_xusb_utmi_pad_power_on(phy); + tegra186_utmi_pad_power_on(phy); + return 0; } static int tegra186_utmi_phy_power_off(struct phy *phy) { - /* TODO: pad power saving */ - tegra_phy_xusb_utmi_pad_power_down(phy); + tegra186_utmi_pad_power_down(phy); return 0; } @@ -1381,12 +1384,9 @@ tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl) return -ENOMEM; err = tegra_fuse_readl(TEGRA_FUSE_SKU_CALIB_0, &value); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "failed to read calibration fuse: %d\n", - err); - return err; - } + if (err) + return dev_err_probe(dev, err, + "failed to read calibration fuse\n"); dev_dbg(dev, "FUSE_USB_CALIB_0 %#x\n", value); @@ -1486,6 +1486,8 @@ static const struct tegra_xusb_padctl_ops tegra186_xusb_padctl_ops = { .suspend_noirq = tegra186_xusb_padctl_suspend_noirq, .resume_noirq = tegra186_xusb_padctl_resume_noirq, .vbus_override = tegra186_xusb_padctl_vbus_override, + .utmi_pad_power_on = tegra186_utmi_pad_power_on, + .utmi_pad_power_down = tegra186_utmi_pad_power_down, }; #if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c index aa5237eacd29d74c9f62e3657ead50ada1e0ee8e..95091876c422230af448470035362725be9bf2c2 100644 --- a/drivers/phy/tegra/xusb.c +++ b/drivers/phy/tegra/xusb.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved. */ #include @@ -656,6 +656,7 @@ static int tegra_xusb_setup_usb_role_switch(struct tegra_xusb_port *port) struct usb_role_switch_desc role_sx_desc = { .fwnode = dev_fwnode(&port->dev), .set = tegra_xusb_role_sw_set, + .allow_userspace_control = true, }; int err = 0; @@ -1270,7 +1271,7 @@ static int tegra_xusb_padctl_remove(struct platform_device *pdev) padctl->soc->ops->remove(padctl); - return err; + return 0; } static __maybe_unused int tegra_xusb_padctl_suspend_noirq(struct device *dev) @@ -1458,6 +1459,26 @@ int tegra_phy_xusb_utmi_port_reset(struct phy *phy) } EXPORT_SYMBOL_GPL(tegra_phy_xusb_utmi_port_reset); +void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy) +{ + struct tegra_xusb_lane *lane = phy_get_drvdata(phy); + struct tegra_xusb_padctl *padctl = lane->pad->padctl; + + if (padctl->soc->ops->utmi_pad_power_on) + padctl->soc->ops->utmi_pad_power_on(phy); +} +EXPORT_SYMBOL_GPL(tegra_phy_xusb_utmi_pad_power_on); + +void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy) +{ + struct tegra_xusb_lane *lane = phy_get_drvdata(phy); + struct tegra_xusb_padctl *padctl = lane->pad->padctl; + + if (padctl->soc->ops->utmi_pad_power_down) + padctl->soc->ops->utmi_pad_power_down(phy); +} +EXPORT_SYMBOL_GPL(tegra_phy_xusb_utmi_pad_power_down); + int tegra_xusb_padctl_get_usb3_companion(struct tegra_xusb_padctl *padctl, unsigned int port) { diff --git a/drivers/phy/tegra/xusb.h b/drivers/phy/tegra/xusb.h index 034f7a2c28d6269a0acd11e36a3ff8362a4e27a8..8cfbbdbd6e0c0103b9086128e7889ab9afc0027f 100644 --- a/drivers/phy/tegra/xusb.h +++ b/drivers/phy/tegra/xusb.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved. * Copyright (c) 2015, Google Inc. */ @@ -412,6 +412,8 @@ struct tegra_xusb_padctl_ops { unsigned int index, bool enable); int (*vbus_override)(struct tegra_xusb_padctl *padctl, bool set); int (*utmi_port_reset)(struct phy *phy); + void (*utmi_pad_power_on)(struct phy *phy); + void (*utmi_pad_power_down)(struct phy *phy); }; struct tegra_xusb_padctl_soc { diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c index d0ab69750c6b44ab36c5148af2216c6f235951ac..0bcfd6d96b4d0a67d0d0cc5a7dc579610844c8e4 100644 --- a/drivers/phy/ti/phy-gmii-sel.c +++ b/drivers/phy/ti/phy-gmii-sel.c @@ -22,6 +22,12 @@ #define AM33XX_GMII_SEL_MODE_RMII 1 #define AM33XX_GMII_SEL_MODE_RGMII 2 +/* J72xx SoC specific definitions for the CONTROL port */ +#define J72XX_GMII_SEL_MODE_QSGMII 4 +#define J72XX_GMII_SEL_MODE_QSGMII_SUB 6 + +#define PHY_GMII_PORT(n) BIT((n) - 1) + enum { PHY_GMII_SEL_PORT_MODE = 0, PHY_GMII_SEL_RGMII_ID_MODE, @@ -43,6 +49,7 @@ struct phy_gmii_sel_soc_data { u32 features; const struct reg_field (*regfields)[PHY_GMII_SEL_LAST]; bool use_of_data; + u64 extra_modes; }; struct phy_gmii_sel_priv { @@ -53,6 +60,7 @@ struct phy_gmii_sel_priv { struct phy_gmii_sel_phy_priv *if_phys; u32 num_ports; u32 reg_offset; + u32 qsgmii_main_ports; }; static int phy_gmii_sel_mode(struct phy *phy, enum phy_mode mode, int submode) @@ -88,10 +96,17 @@ static int phy_gmii_sel_mode(struct phy *phy, enum phy_mode mode, int submode) gmii_sel_mode = AM33XX_GMII_SEL_MODE_MII; break; + case PHY_INTERFACE_MODE_QSGMII: + if (!(soc_data->extra_modes & BIT(PHY_INTERFACE_MODE_QSGMII))) + goto unsupported; + if (if_phy->priv->qsgmii_main_ports & BIT(if_phy->id - 1)) + gmii_sel_mode = J72XX_GMII_SEL_MODE_QSGMII; + else + gmii_sel_mode = J72XX_GMII_SEL_MODE_QSGMII_SUB; + break; + default: - dev_warn(dev, "port%u: unsupported mode: \"%s\"\n", - if_phy->id, phy_modes(submode)); - return -EINVAL; + goto unsupported; } if_phy->phy_if_mode = submode; @@ -123,6 +138,11 @@ static int phy_gmii_sel_mode(struct phy *phy, enum phy_mode mode, int submode) } return 0; + +unsupported: + dev_warn(dev, "port%u: unsupported mode: \"%s\"\n", + if_phy->id, phy_modes(submode)); + return -EINVAL; } static const @@ -188,6 +208,13 @@ struct phy_gmii_sel_soc_data phy_gmii_sel_soc_am654 = { .regfields = phy_gmii_sel_fields_am654, }; +static const +struct phy_gmii_sel_soc_data phy_gmii_sel_cpsw5g_soc_j7200 = { + .use_of_data = true, + .regfields = phy_gmii_sel_fields_am654, + .extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII), +}; + static const struct of_device_id phy_gmii_sel_id_table[] = { { .compatible = "ti,am3352-phy-gmii-sel", @@ -209,6 +236,10 @@ static const struct of_device_id phy_gmii_sel_id_table[] = { .compatible = "ti,am654-phy-gmii-sel", .data = &phy_gmii_sel_soc_am654, }, + { + .compatible = "ti,j7200-cpsw5g-phy-gmii-sel", + .data = &phy_gmii_sel_cpsw5g_soc_j7200, + }, {} }; MODULE_DEVICE_TABLE(of, phy_gmii_sel_id_table); @@ -350,6 +381,7 @@ static int phy_gmii_sel_probe(struct platform_device *pdev) struct device_node *node = dev->of_node; const struct of_device_id *of_id; struct phy_gmii_sel_priv *priv; + u32 main_ports = 1; int ret; of_id = of_match_node(phy_gmii_sel_id_table, pdev->dev.of_node); @@ -363,6 +395,15 @@ static int phy_gmii_sel_probe(struct platform_device *pdev) priv->dev = &pdev->dev; priv->soc_data = of_id->data; priv->num_ports = priv->soc_data->num_ports; + of_property_read_u32(node, "ti,qsgmii-main-ports", &main_ports); + /* + * Ensure that main_ports is within bounds. If the property + * ti,qsgmii-main-ports is not mentioned, or the value mentioned + * is out of bounds, default to 1. + */ + if (main_ports < 1 || main_ports > 4) + main_ports = 1; + priv->qsgmii_main_ports = PHY_GMII_PORT(main_ports); priv->regmap = syscon_node_to_regmap(node->parent); if (IS_ERR(priv->regmap)) { diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c index 70bac931f99ac24dc115304a8dae61ae51a523c7..41725c6bcdf6f30656127165c96838b0eed502c9 100644 --- a/drivers/phy/ti/phy-j721e-wiz.c +++ b/drivers/phy/ti/phy-j721e-wiz.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,15 @@ #include #include +#define REF_CLK_19_2MHZ 19200000 +#define REF_CLK_25MHZ 25000000 +#define REF_CLK_100MHZ 100000000 +#define REF_CLK_156_25MHZ 156250000 + +/* SCM offsets */ +#define SERDES_SUP_CTRL 0x4400 + +/* SERDES offsets */ #define WIZ_SERDES_CTRL 0x404 #define WIZ_SERDES_TOP_CTRL 0x408 #define WIZ_SERDES_RST 0x40c @@ -85,6 +95,18 @@ static const struct reg_field pma_cmn_refclk_dig_div = REG_FIELD(WIZ_SERDES_TOP_CTRL, 26, 27); static const struct reg_field pma_cmn_refclk1_dig_div = REG_FIELD(WIZ_SERDES_TOP_CTRL, 24, 25); + +static const struct reg_field sup_pll0_refclk_mux_sel = + REG_FIELD(SERDES_SUP_CTRL, 0, 1); +static const struct reg_field sup_pll1_refclk_mux_sel = + REG_FIELD(SERDES_SUP_CTRL, 2, 3); +static const struct reg_field sup_pma_cmn_refclk1_int_mode = + REG_FIELD(SERDES_SUP_CTRL, 4, 5); +static const struct reg_field sup_refclk_dig_sel_10g = + REG_FIELD(SERDES_SUP_CTRL, 6, 7); +static const struct reg_field sup_legacy_clk_override = + REG_FIELD(SERDES_SUP_CTRL, 8, 8); + static const char * const output_clk_names[] = { [TI_WIZ_PLL0_REFCLK] = "pll0-refclk", [TI_WIZ_PLL1_REFCLK] = "pll1-refclk", @@ -129,6 +151,26 @@ static const struct reg_field p0_fullrt_div[WIZ_MAX_LANES] = { REG_FIELD(WIZ_LANECTL(3), 22, 23), }; +static const struct reg_field p0_mac_src_sel[WIZ_MAX_LANES] = { + REG_FIELD(WIZ_LANECTL(0), 20, 21), + REG_FIELD(WIZ_LANECTL(1), 20, 21), + REG_FIELD(WIZ_LANECTL(2), 20, 21), + REG_FIELD(WIZ_LANECTL(3), 20, 21), +}; + +static const struct reg_field p0_rxfclk_sel[WIZ_MAX_LANES] = { + REG_FIELD(WIZ_LANECTL(0), 6, 7), + REG_FIELD(WIZ_LANECTL(1), 6, 7), + REG_FIELD(WIZ_LANECTL(2), 6, 7), + REG_FIELD(WIZ_LANECTL(3), 6, 7), +}; + +static const struct reg_field p0_refclk_sel[WIZ_MAX_LANES] = { + REG_FIELD(WIZ_LANECTL(0), 18, 19), + REG_FIELD(WIZ_LANECTL(1), 18, 19), + REG_FIELD(WIZ_LANECTL(2), 18, 19), + REG_FIELD(WIZ_LANECTL(3), 18, 19), +}; static const struct reg_field p_mac_div_sel0[WIZ_MAX_LANES] = { REG_FIELD(WIZ_LANEDIV(0), 16, 22), REG_FIELD(WIZ_LANEDIV(1), 16, 22), @@ -228,6 +270,27 @@ static const struct wiz_clk_mux_sel clk_mux_sel_10g[] = { }, }; +static const struct wiz_clk_mux_sel clk_mux_sel_10g_2_refclk[] = { + { + .num_parents = 3, + .parents = { WIZ_CORE_REFCLK, WIZ_CORE_REFCLK1, WIZ_EXT_REFCLK }, + .table = { 2, 3, 0 }, + .node_name = "pll0-refclk", + }, + { + .num_parents = 3, + .parents = { WIZ_CORE_REFCLK, WIZ_CORE_REFCLK1, WIZ_EXT_REFCLK }, + .table = { 2, 3, 0 }, + .node_name = "pll1-refclk", + }, + { + .num_parents = 3, + .parents = { WIZ_CORE_REFCLK, WIZ_CORE_REFCLK1, WIZ_EXT_REFCLK }, + .table = { 2, 3, 0 }, + .node_name = "refclk-dig", + }, +}; + static const struct clk_div_table clk_div_table[] = { { .val = 0, .div = 1, }, { .val = 1, .div = 2, }, @@ -249,14 +312,18 @@ static const struct wiz_clk_div_sel clk_div_sel[] = { enum wiz_type { J721E_WIZ_16G, - J721E_WIZ_10G, + J721E_WIZ_10G, /* Also for J7200 SR1.0 */ AM64_WIZ_10G, + J7200_WIZ_10G, /* J7200 SR2.0 */ }; struct wiz_data { enum wiz_type type; + const struct reg_field *pll0_refclk_mux_sel; + const struct reg_field *pll1_refclk_mux_sel; const struct reg_field *refclk_dig_sel; const struct reg_field *pma_cmn_refclk1_dig_div; + const struct reg_field *pma_cmn_refclk1_int_mode; const struct wiz_clk_mux_sel *clk_mux_sel; unsigned int clk_div_sel_num; }; @@ -266,6 +333,7 @@ struct wiz_data { struct wiz { struct regmap *regmap; + struct regmap *scm_regmap; enum wiz_type type; const struct wiz_clk_mux_sel *clk_mux_sel; const struct wiz_clk_div_sel *clk_div_sel; @@ -280,13 +348,18 @@ struct wiz { struct regmap_field *p_mac_div_sel0[WIZ_MAX_LANES]; struct regmap_field *p_mac_div_sel1[WIZ_MAX_LANES]; struct regmap_field *p0_fullrt_div[WIZ_MAX_LANES]; + struct regmap_field *p0_mac_src_sel[WIZ_MAX_LANES]; + struct regmap_field *p0_rxfclk_sel[WIZ_MAX_LANES]; + struct regmap_field *p0_refclk_sel[WIZ_MAX_LANES]; struct regmap_field *pma_cmn_refclk_int_mode; + struct regmap_field *pma_cmn_refclk1_int_mode; struct regmap_field *pma_cmn_refclk_mode; struct regmap_field *pma_cmn_refclk_dig_div; struct regmap_field *pma_cmn_refclk1_dig_div; struct regmap_field *mux_sel_field[WIZ_MUX_NUM_CLOCKS]; struct regmap_field *div_sel_field[WIZ_DIV_NUM_CLOCKS_16G]; struct regmap_field *typec_ln10_swap; + struct regmap_field *sup_legacy_clk_override; struct device *dev; u32 num_lanes; @@ -325,7 +398,9 @@ static int wiz_p_mac_div_sel(struct wiz *wiz) int i; for (i = 0; i < num_lanes; i++) { - if (wiz->lane_phy_type[i] == PHY_TYPE_QSGMII) { + if (wiz->lane_phy_type[i] == PHY_TYPE_SGMII || + wiz->lane_phy_type[i] == PHY_TYPE_QSGMII || + wiz->lane_phy_type[i] == PHY_TYPE_USXGMII) { ret = regmap_field_write(wiz->p_mac_div_sel0[i], 1); if (ret) return ret; @@ -354,6 +429,13 @@ static int wiz_mode_select(struct wiz *wiz) else continue; + if (wiz->lane_phy_type[i] == PHY_TYPE_USXGMII) { + ret = regmap_field_write(wiz->p0_mac_src_sel[i], 0x3); + ret = regmap_field_write(wiz->p0_rxfclk_sel[i], 0x3); + ret = regmap_field_write(wiz->p0_refclk_sel[i], 0x3); + mode = LANE_MODE_GEN1; + } + ret = regmap_field_write(wiz->p_standard_mode[i], mode); if (ret) return ret; @@ -416,6 +498,7 @@ static int wiz_init(struct wiz *wiz) static int wiz_regfield_init(struct wiz *wiz) { struct regmap *regmap = wiz->regmap; + struct regmap *scm_regmap = wiz->regmap; /* updated later to scm_regmap if applicable */ int num_lanes = wiz->num_lanes; struct device *dev = wiz->dev; const struct wiz_data *data = wiz->data; @@ -465,27 +548,46 @@ static int wiz_regfield_init(struct wiz *wiz) } } + if (wiz->scm_regmap) { + scm_regmap = wiz->scm_regmap; + wiz->sup_legacy_clk_override = + devm_regmap_field_alloc(dev, scm_regmap, sup_legacy_clk_override); + if (IS_ERR(wiz->sup_legacy_clk_override)) { + dev_err(dev, "SUP_LEGACY_CLK_OVERRIDE reg field init failed\n"); + return PTR_ERR(wiz->sup_legacy_clk_override); + } + } + wiz->mux_sel_field[PLL0_REFCLK] = - devm_regmap_field_alloc(dev, regmap, pll0_refclk_mux_sel); + devm_regmap_field_alloc(dev, scm_regmap, *data->pll0_refclk_mux_sel); if (IS_ERR(wiz->mux_sel_field[PLL0_REFCLK])) { dev_err(dev, "PLL0_REFCLK_SEL reg field init failed\n"); return PTR_ERR(wiz->mux_sel_field[PLL0_REFCLK]); } wiz->mux_sel_field[PLL1_REFCLK] = - devm_regmap_field_alloc(dev, regmap, pll1_refclk_mux_sel); + devm_regmap_field_alloc(dev, scm_regmap, *data->pll1_refclk_mux_sel); if (IS_ERR(wiz->mux_sel_field[PLL1_REFCLK])) { dev_err(dev, "PLL1_REFCLK_SEL reg field init failed\n"); return PTR_ERR(wiz->mux_sel_field[PLL1_REFCLK]); } - wiz->mux_sel_field[REFCLK_DIG] = devm_regmap_field_alloc(dev, regmap, + wiz->mux_sel_field[REFCLK_DIG] = devm_regmap_field_alloc(dev, scm_regmap, *data->refclk_dig_sel); if (IS_ERR(wiz->mux_sel_field[REFCLK_DIG])) { dev_err(dev, "REFCLK_DIG_SEL reg field init failed\n"); return PTR_ERR(wiz->mux_sel_field[REFCLK_DIG]); } + if (data->pma_cmn_refclk1_int_mode) { + wiz->pma_cmn_refclk1_int_mode = + devm_regmap_field_alloc(dev, scm_regmap, *data->pma_cmn_refclk1_int_mode); + if (IS_ERR(wiz->pma_cmn_refclk1_int_mode)) { + dev_err(dev, "PMA_CMN_REFCLK1_INT_MODE reg field init failed\n"); + return PTR_ERR(wiz->pma_cmn_refclk1_int_mode); + } + } + for (i = 0; i < num_lanes; i++) { wiz->p_enable[i] = devm_regmap_field_alloc(dev, regmap, p_enable[i]); @@ -523,6 +625,24 @@ static int wiz_regfield_init(struct wiz *wiz) return PTR_ERR(wiz->p0_fullrt_div[i]); } + wiz->p0_mac_src_sel[i] = devm_regmap_field_alloc(dev, regmap, p0_mac_src_sel[i]); + if (IS_ERR(wiz->p0_mac_src_sel[i])) { + dev_err(dev, "P%d_MAC_SRC_SEL reg field init failed\n", i); + return PTR_ERR(wiz->p0_mac_src_sel[i]); + } + + wiz->p0_rxfclk_sel[i] = devm_regmap_field_alloc(dev, regmap, p0_rxfclk_sel[i]); + if (IS_ERR(wiz->p0_rxfclk_sel[i])) { + dev_err(dev, "P%d_RXFCLK_SEL reg field init failed\n", i); + return PTR_ERR(wiz->p0_rxfclk_sel[i]); + } + + wiz->p0_refclk_sel[i] = devm_regmap_field_alloc(dev, regmap, p0_refclk_sel[i]); + if (IS_ERR(wiz->p0_refclk_sel[i])) { + dev_err(dev, "P%d_REFCLK_SEL reg field init failed\n", i); + return PTR_ERR(wiz->p0_refclk_sel[i]); + } + wiz->p_mac_div_sel0[i] = devm_regmap_field_alloc(dev, regmap, p_mac_div_sel0[i]); if (IS_ERR(wiz->p_mac_div_sel0[i])) { @@ -597,6 +717,8 @@ static int wiz_phy_en_refclk_register(struct wiz *wiz) struct device *dev = wiz->dev; struct clk_init_data *init; struct clk *clk; + char *clk_name; + unsigned int sz; wiz_phy_en_refclk = devm_kzalloc(dev, sizeof(*wiz_phy_en_refclk), GFP_KERNEL); if (!wiz_phy_en_refclk) @@ -606,12 +728,23 @@ static int wiz_phy_en_refclk_register(struct wiz *wiz) init->ops = &wiz_phy_en_refclk_ops; init->flags = 0; - init->name = output_clk_names[TI_WIZ_PHY_EN_REFCLK]; + + sz = strlen(dev_name(dev)) + strlen(output_clk_names[TI_WIZ_PHY_EN_REFCLK]) + 2; + + clk_name = kzalloc(sz, GFP_KERNEL); + if (!clk_name) + return -ENOMEM; + + snprintf(clk_name, sz, "%s_%s", dev_name(dev), output_clk_names[TI_WIZ_PHY_EN_REFCLK]); + init->name = clk_name; wiz_phy_en_refclk->phy_en_refclk = wiz->phy_en_refclk; wiz_phy_en_refclk->hw.init = init; clk = devm_clk_register(dev, &wiz_phy_en_refclk->hw); + + kfree(clk_name); + if (IS_ERR(clk)) return PTR_ERR(clk); @@ -856,9 +989,13 @@ static void wiz_clock_cleanup(struct wiz *wiz, struct device_node *node) struct device_node *clk_node; int i; - if (wiz->type == AM64_WIZ_10G) { + switch (wiz->type) { + case AM64_WIZ_10G: + case J7200_WIZ_10G: of_clk_del_provider(dev->of_node); return; + default: + break; } for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) { @@ -885,9 +1022,6 @@ static int wiz_clock_register(struct wiz *wiz) int ret; int i; - if (wiz->type != AM64_WIZ_10G) - return 0; - clk_index = TI_WIZ_PLL0_REFCLK; for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++, clk_index++) { ret = wiz_mux_clk_register(wiz, wiz->mux_sel_field[i], &clk_mux_sel[i], clk_index); @@ -937,6 +1071,41 @@ static int wiz_clock_init(struct wiz *wiz, struct device_node *node) else regmap_field_write(wiz->pma_cmn_refclk_int_mode, 0x3); + switch (wiz->type) { + case AM64_WIZ_10G: + case J7200_WIZ_10G: + switch (rate) { + case REF_CLK_100MHZ: + regmap_field_write(wiz->div_sel_field[CMN_REFCLK_DIG_DIV], 0x2); + break; + case REF_CLK_156_25MHZ: + regmap_field_write(wiz->div_sel_field[CMN_REFCLK_DIG_DIV], 0x3); + break; + default: + regmap_field_write(wiz->div_sel_field[CMN_REFCLK_DIG_DIV], 0); + break; + } + break; + default: + break; + } + + if (wiz->data->pma_cmn_refclk1_int_mode) { + clk = devm_clk_get(dev, "core_ref1_clk"); + if (IS_ERR(clk)) { + dev_err(dev, "core_ref1_clk clock not found\n"); + ret = PTR_ERR(clk); + return ret; + } + wiz->input_clks[WIZ_CORE_REFCLK1] = clk; + + rate = clk_get_rate(clk); + if (rate >= 100000000) + regmap_field_write(wiz->pma_cmn_refclk1_int_mode, 0x1); + else + regmap_field_write(wiz->pma_cmn_refclk1_int_mode, 0x3); + } + clk = devm_clk_get(dev, "ext_ref_clk"); if (IS_ERR(clk)) { dev_err(dev, "ext_ref_clk clock not found\n"); @@ -951,11 +1120,15 @@ static int wiz_clock_init(struct wiz *wiz, struct device_node *node) else regmap_field_write(wiz->pma_cmn_refclk_mode, 0x2); - if (wiz->type == AM64_WIZ_10G) { + switch (wiz->type) { + case AM64_WIZ_10G: + case J7200_WIZ_10G: ret = wiz_clock_register(wiz); if (ret) dev_err(dev, "Failed to register wiz clocks\n"); return ret; + default: + break; } for (i = 0; i < WIZ_MUX_NUM_CLOCKS; i++) { @@ -1025,12 +1198,19 @@ static int wiz_phy_reset_assert(struct reset_controller_dev *rcdev, static int wiz_phy_fullrt_div(struct wiz *wiz, int lane) { - if (wiz->type != AM64_WIZ_10G) + switch (wiz->type) { + case AM64_WIZ_10G: + if (wiz->lane_phy_type[lane] == PHY_TYPE_PCIE) + return regmap_field_write(wiz->p0_fullrt_div[lane], 0x1); + break; + case J721E_WIZ_10G: + case J7200_WIZ_10G: + if (wiz->lane_phy_type[lane] == PHY_TYPE_SGMII) + return regmap_field_write(wiz->p0_fullrt_div[lane], 0x2); + break; + default: return 0; - - if (wiz->lane_phy_type[lane] == PHY_TYPE_PCIE) - return regmap_field_write(wiz->p0_fullrt_div[lane], 0x1); - + } return 0; } @@ -1083,6 +1263,8 @@ static const struct regmap_config wiz_regmap_config = { static struct wiz_data j721e_16g_data = { .type = J721E_WIZ_16G, + .pll0_refclk_mux_sel = &pll0_refclk_mux_sel, + .pll1_refclk_mux_sel = &pll1_refclk_mux_sel, .refclk_dig_sel = &refclk_dig_sel_16g, .pma_cmn_refclk1_dig_div = &pma_cmn_refclk1_dig_div, .clk_mux_sel = clk_mux_sel_16g, @@ -1091,6 +1273,8 @@ static struct wiz_data j721e_16g_data = { static struct wiz_data j721e_10g_data = { .type = J721E_WIZ_10G, + .pll0_refclk_mux_sel = &pll0_refclk_mux_sel, + .pll1_refclk_mux_sel = &pll1_refclk_mux_sel, .refclk_dig_sel = &refclk_dig_sel_10g, .clk_mux_sel = clk_mux_sel_10g, .clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G, @@ -1098,11 +1282,23 @@ static struct wiz_data j721e_10g_data = { static struct wiz_data am64_10g_data = { .type = AM64_WIZ_10G, + .pll0_refclk_mux_sel = &pll0_refclk_mux_sel, + .pll1_refclk_mux_sel = &pll1_refclk_mux_sel, .refclk_dig_sel = &refclk_dig_sel_10g, .clk_mux_sel = clk_mux_sel_10g, .clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G, }; +static struct wiz_data j7200_pg2_10g_data = { + .type = J7200_WIZ_10G, + .pll0_refclk_mux_sel = &sup_pll0_refclk_mux_sel, + .pll1_refclk_mux_sel = &sup_pll1_refclk_mux_sel, + .refclk_dig_sel = &sup_refclk_dig_sel_10g, + .pma_cmn_refclk1_int_mode = &sup_pma_cmn_refclk1_int_mode, + .clk_mux_sel = clk_mux_sel_10g_2_refclk, + .clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G, +}; + static const struct of_device_id wiz_id_table[] = { { .compatible = "ti,j721e-wiz-16g", .data = &j721e_16g_data, @@ -1113,6 +1309,9 @@ static const struct of_device_id wiz_id_table[] = { { .compatible = "ti,am64-wiz-10g", .data = &am64_10g_data, }, + { + .compatible = "ti,j7200-wiz-10g", .data = &j7200_pg2_10g_data, + }, {} }; MODULE_DEVICE_TABLE(of, wiz_id_table); @@ -1210,6 +1409,17 @@ static int wiz_probe(struct platform_device *pdev) goto err_addr_to_resource; } + wiz->scm_regmap = syscon_regmap_lookup_by_phandle(node, "ti,scm"); + if (IS_ERR(wiz->scm_regmap)) { + if (wiz->type == J7200_WIZ_10G) { + dev_err(dev, "Couldn't get ti,scm regmap\n"); + ret = -ENODEV; + goto err_addr_to_resource; + } + + wiz->scm_regmap = NULL; + } + ret = of_property_read_u32(node, "num-lanes", &num_lanes); if (ret) { dev_err(dev, "Failed to read num-lanes property\n"); @@ -1254,7 +1464,7 @@ static int wiz_probe(struct platform_device *pdev) ret = wiz_get_lane_phy_types(dev, wiz); if (ret) - return ret; + goto err_addr_to_resource; wiz->dev = dev; wiz->regmap = regmap; @@ -1271,6 +1481,10 @@ static int wiz_probe(struct platform_device *pdev) goto err_addr_to_resource; } + /* Enable supplemental Control override if available */ + if (wiz->scm_regmap) + regmap_field_write(wiz->sup_legacy_clk_override, 1); + phy_reset_dev = &wiz->wiz_phy_reset_dev; phy_reset_dev->dev = dev; phy_reset_dev->ops = &wiz_phy_reset_ops, diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index 1cf74b0c42e56f19fb690774eb06eb3bd644b8c8..f71fefff400f5dd4101cc8ac1c2069035a2db9b4 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -135,6 +135,20 @@ config PINCTRL_BM1880 help Pinctrl driver for Bitmain BM1880 SoC. +config PINCTRL_CY8C95X0 + tristate "Cypress CY8C95X0 I2C pinctrl and GPIO driver" + depends on I2C + select GPIOLIB + select GPIOLIB_IRQCHIP + select PINMUX + select PINCONF + select GENERIC_PINCONF + select REGMAP_I2C + help + Support for 20/40/60 pin Cypress Cy8C95x0 pinctrl/gpio I2C expander. + This driver can also be built as a module. If so, the module will be + called pinctrl-cy8c95x0. + config PINCTRL_DA850_PUPD tristate "TI DA850/OMAP-L138/AM18XX pull-up and pull-down groups" depends on OF && (ARCH_DAVINCI_DA850 || COMPILE_TEST) @@ -292,7 +306,7 @@ config PINCTRL_MCP23S08 corresponding interrupt-controller. config PINCTRL_MICROCHIP_SGPIO - bool "Pinctrl driver for Microsemi/Microchip Serial GPIO" + tristate "Pinctrl driver for Microsemi/Microchip Serial GPIO" depends on OF depends on HAS_IOMEM select GPIOLIB @@ -310,6 +324,9 @@ config PINCTRL_MICROCHIP_SGPIO connect control signals from SFP modules and to act as an LED controller. + If compiled as a module, the module name will be + pinctrl-microchip-sgpio. + config PINCTRL_OCELOT tristate "Pinctrl driver for the Microsemi Ocelot and Jaguar2 SoCs" depends on OF @@ -321,6 +338,11 @@ config PINCTRL_OCELOT select GENERIC_PINMUX_FUNCTIONS select OF_GPIO select REGMAP_MMIO + help + Support for the internal GPIO interfaces on Microsemi Ocelot and + Jaguar2 SoCs. + + If conpiled as a module, the module name will be pinctrl-ocelot. config PINCTRL_OXNAS bool @@ -412,23 +434,6 @@ config PINCTRL_ST select PINCONF select GPIOLIB_IRQCHIP -config PINCTRL_STARFIVE - tristate "Pinctrl and GPIO driver for the StarFive JH7100 SoC" - depends on SOC_STARFIVE || COMPILE_TEST - depends on OF - default SOC_STARFIVE - select GENERIC_PINCTRL_GROUPS - select GENERIC_PINMUX_FUNCTIONS - select GENERIC_PINCONF - select GPIOLIB - select GPIOLIB_IRQCHIP - select OF_GPIO - help - Say yes here to support pin control on the StarFive JH7100 SoC. - This also provides an interface to the GPIO pins not used by other - peripherals supporting inputs, outputs, configuring pull-up/pull-down - and interrupts on input changes. - config PINCTRL_STMFX tristate "STMicroelectronics STMFX GPIO expander pinctrl driver" depends on I2C @@ -526,6 +531,7 @@ source "drivers/pinctrl/renesas/Kconfig" source "drivers/pinctrl/samsung/Kconfig" source "drivers/pinctrl/spear/Kconfig" source "drivers/pinctrl/sprd/Kconfig" +source "drivers/pinctrl/starfive/Kconfig" source "drivers/pinctrl/stm32/Kconfig" source "drivers/pinctrl/sunplus/Kconfig" source "drivers/pinctrl/sunxi/Kconfig" diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile index e76f5cdc64b0edd8992fa6d71a0928f0f018fc38..89bfa01b5231ad5a4f99861a7d094264d932c85f 100644 --- a/drivers/pinctrl/Makefile +++ b/drivers/pinctrl/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_PINCTRL_AT91) += pinctrl-at91.o obj-$(CONFIG_PINCTRL_AT91PIO4) += pinctrl-at91-pio4.o obj-$(CONFIG_PINCTRL_AXP209) += pinctrl-axp209.o obj-$(CONFIG_PINCTRL_BM1880) += pinctrl-bm1880.o +obj-$(CONFIG_PINCTRL_CY8C95X0) += pinctrl-cy8c95x0.o obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o obj-$(CONFIG_PINCTRL_DA9062) += pinctrl-da9062.o obj-$(CONFIG_PINCTRL_DIGICOLOR) += pinctrl-digicolor.o @@ -43,7 +44,6 @@ obj-$(CONFIG_PINCTRL_RK805) += pinctrl-rk805.o obj-$(CONFIG_PINCTRL_ROCKCHIP) += pinctrl-rockchip.o obj-$(CONFIG_PINCTRL_SINGLE) += pinctrl-single.o obj-$(CONFIG_PINCTRL_ST) += pinctrl-st.o -obj-$(CONFIG_PINCTRL_STARFIVE) += pinctrl-starfive.o obj-$(CONFIG_PINCTRL_STMFX) += pinctrl-stmfx.o obj-$(CONFIG_PINCTRL_SX150X) += pinctrl-sx150x.o obj-$(CONFIG_PINCTRL_TB10X) += pinctrl-tb10x.o @@ -70,6 +70,7 @@ obj-$(CONFIG_PINCTRL_RENESAS) += renesas/ obj-$(CONFIG_PINCTRL_SAMSUNG) += samsung/ obj-$(CONFIG_PINCTRL_SPEAR) += spear/ obj-y += sprd/ +obj-$(CONFIG_SOC_STARFIVE) += starfive/ obj-$(CONFIG_PINCTRL_STM32) += stm32/ obj-y += sunplus/ obj-$(CONFIG_PINCTRL_SUNXI) += sunxi/ diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed.c b/drivers/pinctrl/aspeed/pinctrl-aspeed.c index 83d47ff1cea8f2b1dd8f970a7f1e715b3db5d6cb..a30912a92f057dcefe283a6a762abf7bb15b5860 100644 --- a/drivers/pinctrl/aspeed/pinctrl-aspeed.c +++ b/drivers/pinctrl/aspeed/pinctrl-aspeed.c @@ -92,19 +92,10 @@ static int aspeed_sig_expr_enable(struct aspeed_pinmux_data *ctx, static int aspeed_sig_expr_disable(struct aspeed_pinmux_data *ctx, const struct aspeed_sig_expr *expr) { - int ret; - pr_debug("Disabling signal %s for %s\n", expr->signal, expr->function); - ret = aspeed_sig_expr_eval(ctx, expr, true); - if (ret < 0) - return ret; - - if (ret) - return aspeed_sig_expr_set(ctx, expr, false); - - return 0; + return aspeed_sig_expr_set(ctx, expr, false); } /** diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig index 8f4d89806fcba4ea61a83c71817dc764826a1523..35b51ce4298e2588d0ae9dbda95749858e75dba8 100644 --- a/drivers/pinctrl/bcm/Kconfig +++ b/drivers/pinctrl/bcm/Kconfig @@ -31,13 +31,13 @@ config PINCTRL_BCM2835 config PINCTRL_BCM4908 tristate "Broadcom BCM4908 pinmux driver" - depends on OF && (ARCH_BCM4908 || COMPILE_TEST) + depends on OF && (ARCH_BCMBCA || COMPILE_TEST) select PINMUX select PINCONF select GENERIC_PINCONF select GENERIC_PINCTRL_GROUPS select GENERIC_PINMUX_FUNCTIONS - default ARCH_BCM4908 + default ARCH_BCMBCA help Driver for BCM4908 family SoCs with integrated pin controller. diff --git a/drivers/pinctrl/bcm/pinctrl-bcm6318.c b/drivers/pinctrl/bcm/pinctrl-bcm6318.c index 9311220fb6cba7414d5bf89fd0f10dab647adaad..64073546310e6c737ee36193a831e7aed5f76f2b 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm6318.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm6318.c @@ -27,12 +27,6 @@ #define BCM6318_PAD_REG 0x54 #define BCM6328_PAD_MASK GENMASK(3, 0) -struct bcm6318_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; -}; - struct bcm6318_function { const char *name; const char * const *groups; @@ -146,64 +140,57 @@ static unsigned gpio47_pins[] = { 47 }; static unsigned gpio48_pins[] = { 48 }; static unsigned gpio49_pins[] = { 49 }; -#define BCM6318_GROUP(n) \ - { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ - } - -static struct bcm6318_pingroup bcm6318_groups[] = { - BCM6318_GROUP(gpio0), - BCM6318_GROUP(gpio1), - BCM6318_GROUP(gpio2), - BCM6318_GROUP(gpio3), - BCM6318_GROUP(gpio4), - BCM6318_GROUP(gpio5), - BCM6318_GROUP(gpio6), - BCM6318_GROUP(gpio7), - BCM6318_GROUP(gpio8), - BCM6318_GROUP(gpio9), - BCM6318_GROUP(gpio10), - BCM6318_GROUP(gpio11), - BCM6318_GROUP(gpio12), - BCM6318_GROUP(gpio13), - BCM6318_GROUP(gpio14), - BCM6318_GROUP(gpio15), - BCM6318_GROUP(gpio16), - BCM6318_GROUP(gpio17), - BCM6318_GROUP(gpio18), - BCM6318_GROUP(gpio19), - BCM6318_GROUP(gpio20), - BCM6318_GROUP(gpio21), - BCM6318_GROUP(gpio22), - BCM6318_GROUP(gpio23), - BCM6318_GROUP(gpio24), - BCM6318_GROUP(gpio25), - BCM6318_GROUP(gpio26), - BCM6318_GROUP(gpio27), - BCM6318_GROUP(gpio28), - BCM6318_GROUP(gpio29), - BCM6318_GROUP(gpio30), - BCM6318_GROUP(gpio31), - BCM6318_GROUP(gpio32), - BCM6318_GROUP(gpio33), - BCM6318_GROUP(gpio34), - BCM6318_GROUP(gpio35), - BCM6318_GROUP(gpio36), - BCM6318_GROUP(gpio37), - BCM6318_GROUP(gpio38), - BCM6318_GROUP(gpio39), - BCM6318_GROUP(gpio40), - BCM6318_GROUP(gpio41), - BCM6318_GROUP(gpio42), - BCM6318_GROUP(gpio43), - BCM6318_GROUP(gpio44), - BCM6318_GROUP(gpio45), - BCM6318_GROUP(gpio46), - BCM6318_GROUP(gpio47), - BCM6318_GROUP(gpio48), - BCM6318_GROUP(gpio49), +static struct pingroup bcm6318_groups[] = { + BCM_PIN_GROUP(gpio0), + BCM_PIN_GROUP(gpio1), + BCM_PIN_GROUP(gpio2), + BCM_PIN_GROUP(gpio3), + BCM_PIN_GROUP(gpio4), + BCM_PIN_GROUP(gpio5), + BCM_PIN_GROUP(gpio6), + BCM_PIN_GROUP(gpio7), + BCM_PIN_GROUP(gpio8), + BCM_PIN_GROUP(gpio9), + BCM_PIN_GROUP(gpio10), + BCM_PIN_GROUP(gpio11), + BCM_PIN_GROUP(gpio12), + BCM_PIN_GROUP(gpio13), + BCM_PIN_GROUP(gpio14), + BCM_PIN_GROUP(gpio15), + BCM_PIN_GROUP(gpio16), + BCM_PIN_GROUP(gpio17), + BCM_PIN_GROUP(gpio18), + BCM_PIN_GROUP(gpio19), + BCM_PIN_GROUP(gpio20), + BCM_PIN_GROUP(gpio21), + BCM_PIN_GROUP(gpio22), + BCM_PIN_GROUP(gpio23), + BCM_PIN_GROUP(gpio24), + BCM_PIN_GROUP(gpio25), + BCM_PIN_GROUP(gpio26), + BCM_PIN_GROUP(gpio27), + BCM_PIN_GROUP(gpio28), + BCM_PIN_GROUP(gpio29), + BCM_PIN_GROUP(gpio30), + BCM_PIN_GROUP(gpio31), + BCM_PIN_GROUP(gpio32), + BCM_PIN_GROUP(gpio33), + BCM_PIN_GROUP(gpio34), + BCM_PIN_GROUP(gpio35), + BCM_PIN_GROUP(gpio36), + BCM_PIN_GROUP(gpio37), + BCM_PIN_GROUP(gpio38), + BCM_PIN_GROUP(gpio39), + BCM_PIN_GROUP(gpio40), + BCM_PIN_GROUP(gpio41), + BCM_PIN_GROUP(gpio42), + BCM_PIN_GROUP(gpio43), + BCM_PIN_GROUP(gpio44), + BCM_PIN_GROUP(gpio45), + BCM_PIN_GROUP(gpio46), + BCM_PIN_GROUP(gpio47), + BCM_PIN_GROUP(gpio48), + BCM_PIN_GROUP(gpio49), }; /* GPIO_MODE */ @@ -368,10 +355,10 @@ static const char *bcm6318_pinctrl_get_group_name(struct pinctrl_dev *pctldev, static int bcm6318_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { *pins = bcm6318_groups[group].pins; - *num_pins = bcm6318_groups[group].num_pins; + *npins = bcm6318_groups[group].npins; return 0; } @@ -424,7 +411,7 @@ static int bcm6318_pinctrl_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - const struct bcm6318_pingroup *pg = &bcm6318_groups[group]; + const struct pingroup *pg = &bcm6318_groups[group]; const struct bcm6318_function *f = &bcm6318_funcs[selector]; bcm6318_rmw_mux(pc, pg->pins[0], f->mode_val, f->mux_val); diff --git a/drivers/pinctrl/bcm/pinctrl-bcm63268.c b/drivers/pinctrl/bcm/pinctrl-bcm63268.c index 1c1060a395970f648b5213beea50fb67c5e93695..80c2fc55ffa292980b59090ed9df1aad8373cb82 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm63268.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm63268.c @@ -40,12 +40,6 @@ enum bcm63268_pinctrl_reg { BCM63268_BASEMODE, }; -struct bcm63268_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; -}; - struct bcm63268_function { const char *name; const char * const *groups; @@ -185,74 +179,67 @@ static unsigned vdsl_phy1_grp_pins[] = { 12, 13 }; static unsigned vdsl_phy2_grp_pins[] = { 24, 25 }; static unsigned vdsl_phy3_grp_pins[] = { 26, 27 }; -#define BCM63268_GROUP(n) \ - { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ - } - -static struct bcm63268_pingroup bcm63268_groups[] = { - BCM63268_GROUP(gpio0), - BCM63268_GROUP(gpio1), - BCM63268_GROUP(gpio2), - BCM63268_GROUP(gpio3), - BCM63268_GROUP(gpio4), - BCM63268_GROUP(gpio5), - BCM63268_GROUP(gpio6), - BCM63268_GROUP(gpio7), - BCM63268_GROUP(gpio8), - BCM63268_GROUP(gpio9), - BCM63268_GROUP(gpio10), - BCM63268_GROUP(gpio11), - BCM63268_GROUP(gpio12), - BCM63268_GROUP(gpio13), - BCM63268_GROUP(gpio14), - BCM63268_GROUP(gpio15), - BCM63268_GROUP(gpio16), - BCM63268_GROUP(gpio17), - BCM63268_GROUP(gpio18), - BCM63268_GROUP(gpio19), - BCM63268_GROUP(gpio20), - BCM63268_GROUP(gpio21), - BCM63268_GROUP(gpio22), - BCM63268_GROUP(gpio23), - BCM63268_GROUP(gpio24), - BCM63268_GROUP(gpio25), - BCM63268_GROUP(gpio26), - BCM63268_GROUP(gpio27), - BCM63268_GROUP(gpio28), - BCM63268_GROUP(gpio29), - BCM63268_GROUP(gpio30), - BCM63268_GROUP(gpio31), - BCM63268_GROUP(gpio32), - BCM63268_GROUP(gpio33), - BCM63268_GROUP(gpio34), - BCM63268_GROUP(gpio35), - BCM63268_GROUP(gpio36), - BCM63268_GROUP(gpio37), - BCM63268_GROUP(gpio38), - BCM63268_GROUP(gpio39), - BCM63268_GROUP(gpio40), - BCM63268_GROUP(gpio41), - BCM63268_GROUP(gpio42), - BCM63268_GROUP(gpio43), - BCM63268_GROUP(gpio44), - BCM63268_GROUP(gpio45), - BCM63268_GROUP(gpio46), - BCM63268_GROUP(gpio47), - BCM63268_GROUP(gpio48), - BCM63268_GROUP(gpio49), - BCM63268_GROUP(gpio50), - BCM63268_GROUP(gpio51), +static struct pingroup bcm63268_groups[] = { + BCM_PIN_GROUP(gpio0), + BCM_PIN_GROUP(gpio1), + BCM_PIN_GROUP(gpio2), + BCM_PIN_GROUP(gpio3), + BCM_PIN_GROUP(gpio4), + BCM_PIN_GROUP(gpio5), + BCM_PIN_GROUP(gpio6), + BCM_PIN_GROUP(gpio7), + BCM_PIN_GROUP(gpio8), + BCM_PIN_GROUP(gpio9), + BCM_PIN_GROUP(gpio10), + BCM_PIN_GROUP(gpio11), + BCM_PIN_GROUP(gpio12), + BCM_PIN_GROUP(gpio13), + BCM_PIN_GROUP(gpio14), + BCM_PIN_GROUP(gpio15), + BCM_PIN_GROUP(gpio16), + BCM_PIN_GROUP(gpio17), + BCM_PIN_GROUP(gpio18), + BCM_PIN_GROUP(gpio19), + BCM_PIN_GROUP(gpio20), + BCM_PIN_GROUP(gpio21), + BCM_PIN_GROUP(gpio22), + BCM_PIN_GROUP(gpio23), + BCM_PIN_GROUP(gpio24), + BCM_PIN_GROUP(gpio25), + BCM_PIN_GROUP(gpio26), + BCM_PIN_GROUP(gpio27), + BCM_PIN_GROUP(gpio28), + BCM_PIN_GROUP(gpio29), + BCM_PIN_GROUP(gpio30), + BCM_PIN_GROUP(gpio31), + BCM_PIN_GROUP(gpio32), + BCM_PIN_GROUP(gpio33), + BCM_PIN_GROUP(gpio34), + BCM_PIN_GROUP(gpio35), + BCM_PIN_GROUP(gpio36), + BCM_PIN_GROUP(gpio37), + BCM_PIN_GROUP(gpio38), + BCM_PIN_GROUP(gpio39), + BCM_PIN_GROUP(gpio40), + BCM_PIN_GROUP(gpio41), + BCM_PIN_GROUP(gpio42), + BCM_PIN_GROUP(gpio43), + BCM_PIN_GROUP(gpio44), + BCM_PIN_GROUP(gpio45), + BCM_PIN_GROUP(gpio46), + BCM_PIN_GROUP(gpio47), + BCM_PIN_GROUP(gpio48), + BCM_PIN_GROUP(gpio49), + BCM_PIN_GROUP(gpio50), + BCM_PIN_GROUP(gpio51), /* multi pin groups */ - BCM63268_GROUP(nand_grp), - BCM63268_GROUP(dectpd_grp), - BCM63268_GROUP(vdsl_phy0_grp), - BCM63268_GROUP(vdsl_phy1_grp), - BCM63268_GROUP(vdsl_phy2_grp), - BCM63268_GROUP(vdsl_phy3_grp), + BCM_PIN_GROUP(nand_grp), + BCM_PIN_GROUP(dectpd_grp), + BCM_PIN_GROUP(vdsl_phy0_grp), + BCM_PIN_GROUP(vdsl_phy1_grp), + BCM_PIN_GROUP(vdsl_phy2_grp), + BCM_PIN_GROUP(vdsl_phy3_grp), }; static const char * const led_groups[] = { @@ -487,10 +474,10 @@ static const char *bcm63268_pinctrl_get_group_name(struct pinctrl_dev *pctldev, static int bcm63268_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { *pins = bcm63268_groups[group].pins; - *num_pins = bcm63268_groups[group].num_pins; + *npins = bcm63268_groups[group].npins; return 0; } @@ -545,13 +532,13 @@ static int bcm63268_pinctrl_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - const struct bcm63268_pingroup *pg = &bcm63268_groups[group]; + const struct pingroup *pg = &bcm63268_groups[group]; const struct bcm63268_function *f = &bcm63268_funcs[selector]; unsigned i; unsigned int reg; unsigned int val, mask; - for (i = 0; i < pg->num_pins; i++) + for (i = 0; i < pg->npins; i++) bcm63268_set_gpio(pc, pg->pins[i]); switch (f->reg) { diff --git a/drivers/pinctrl/bcm/pinctrl-bcm6328.c b/drivers/pinctrl/bcm/pinctrl-bcm6328.c index ffa8864abab6d57880cc0587519471985daafe38..1e8cc2c80c81f55f2dc804684fc619eefa734716 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm6328.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm6328.c @@ -26,12 +26,6 @@ #define BCM6328_MUX_OTHER_REG 0x24 #define BCM6328_MUX_MASK GENMASK(1, 0) -struct bcm6328_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; -}; - struct bcm6328_function { const char *name; const char * const *groups; @@ -125,49 +119,42 @@ static unsigned gpio31_pins[] = { 31 }; static unsigned hsspi_cs1_pins[] = { 36 }; static unsigned usb_port1_pins[] = { 38 }; -#define BCM6328_GROUP(n) \ - { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ - } - -static struct bcm6328_pingroup bcm6328_groups[] = { - BCM6328_GROUP(gpio0), - BCM6328_GROUP(gpio1), - BCM6328_GROUP(gpio2), - BCM6328_GROUP(gpio3), - BCM6328_GROUP(gpio4), - BCM6328_GROUP(gpio5), - BCM6328_GROUP(gpio6), - BCM6328_GROUP(gpio7), - BCM6328_GROUP(gpio8), - BCM6328_GROUP(gpio9), - BCM6328_GROUP(gpio10), - BCM6328_GROUP(gpio11), - BCM6328_GROUP(gpio12), - BCM6328_GROUP(gpio13), - BCM6328_GROUP(gpio14), - BCM6328_GROUP(gpio15), - BCM6328_GROUP(gpio16), - BCM6328_GROUP(gpio17), - BCM6328_GROUP(gpio18), - BCM6328_GROUP(gpio19), - BCM6328_GROUP(gpio20), - BCM6328_GROUP(gpio21), - BCM6328_GROUP(gpio22), - BCM6328_GROUP(gpio23), - BCM6328_GROUP(gpio24), - BCM6328_GROUP(gpio25), - BCM6328_GROUP(gpio26), - BCM6328_GROUP(gpio27), - BCM6328_GROUP(gpio28), - BCM6328_GROUP(gpio29), - BCM6328_GROUP(gpio30), - BCM6328_GROUP(gpio31), - - BCM6328_GROUP(hsspi_cs1), - BCM6328_GROUP(usb_port1), +static struct pingroup bcm6328_groups[] = { + BCM_PIN_GROUP(gpio0), + BCM_PIN_GROUP(gpio1), + BCM_PIN_GROUP(gpio2), + BCM_PIN_GROUP(gpio3), + BCM_PIN_GROUP(gpio4), + BCM_PIN_GROUP(gpio5), + BCM_PIN_GROUP(gpio6), + BCM_PIN_GROUP(gpio7), + BCM_PIN_GROUP(gpio8), + BCM_PIN_GROUP(gpio9), + BCM_PIN_GROUP(gpio10), + BCM_PIN_GROUP(gpio11), + BCM_PIN_GROUP(gpio12), + BCM_PIN_GROUP(gpio13), + BCM_PIN_GROUP(gpio14), + BCM_PIN_GROUP(gpio15), + BCM_PIN_GROUP(gpio16), + BCM_PIN_GROUP(gpio17), + BCM_PIN_GROUP(gpio18), + BCM_PIN_GROUP(gpio19), + BCM_PIN_GROUP(gpio20), + BCM_PIN_GROUP(gpio21), + BCM_PIN_GROUP(gpio22), + BCM_PIN_GROUP(gpio23), + BCM_PIN_GROUP(gpio24), + BCM_PIN_GROUP(gpio25), + BCM_PIN_GROUP(gpio26), + BCM_PIN_GROUP(gpio27), + BCM_PIN_GROUP(gpio28), + BCM_PIN_GROUP(gpio29), + BCM_PIN_GROUP(gpio30), + BCM_PIN_GROUP(gpio31), + + BCM_PIN_GROUP(hsspi_cs1), + BCM_PIN_GROUP(usb_port1), }; /* GPIO_MODE */ @@ -292,10 +279,10 @@ static const char *bcm6328_pinctrl_get_group_name(struct pinctrl_dev *pctldev, static int bcm6328_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { *pins = bcm6328_groups[group].pins; - *num_pins = bcm6328_groups[group].num_pins; + *npins = bcm6328_groups[group].npins; return 0; } @@ -338,7 +325,7 @@ static int bcm6328_pinctrl_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - const struct bcm6328_pingroup *pg = &bcm6328_groups[group]; + const struct pingroup *pg = &bcm6328_groups[group]; const struct bcm6328_function *f = &bcm6328_funcs[selector]; bcm6328_rmw_mux(pc, pg->pins[0], f->mode_val, f->mux_val); diff --git a/drivers/pinctrl/bcm/pinctrl-bcm6358.c b/drivers/pinctrl/bcm/pinctrl-bcm6358.c index 9f6cd7447887ff622726d591d2cc5770377957d6..891de49d76e7448efa7862ae045aa94a3e218886 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm6358.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm6358.c @@ -35,9 +35,7 @@ #define BCM6358_MODE_MUX_SYS_IRQ BIT(15) struct bcm6358_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; + struct pingroup grp; const uint16_t mode_val; @@ -131,9 +129,7 @@ static unsigned sys_irq_grp_pins[] = { 5 }; #define BCM6358_GPIO_MUX_GROUP(n, bit, dir) \ { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ + .grp = BCM_PIN_GROUP(n), \ .mode_val = BCM6358_MODE_MUX_##bit, \ .direction = dir, \ } @@ -219,15 +215,15 @@ static int bcm6358_pinctrl_get_group_count(struct pinctrl_dev *pctldev) static const char *bcm6358_pinctrl_get_group_name(struct pinctrl_dev *pctldev, unsigned group) { - return bcm6358_groups[group].name; + return bcm6358_groups[group].grp.name; } static int bcm6358_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { - *pins = bcm6358_groups[group].pins; - *num_pins = bcm6358_groups[group].num_pins; + *pins = bcm6358_groups[group].grp.pins; + *npins = bcm6358_groups[group].grp.npins; return 0; } @@ -264,12 +260,12 @@ static int bcm6358_pinctrl_set_mux(struct pinctrl_dev *pctldev, unsigned int mask = val; unsigned pin; - for (pin = 0; pin < pg->num_pins; pin++) + for (pin = 0; pin < pg->grp.npins; pin++) mask |= (unsigned long)bcm6358_pins[pin].drv_data; regmap_field_update_bits(priv->overlays, mask, val); - for (pin = 0; pin < pg->num_pins; pin++) { + for (pin = 0; pin < pg->grp.npins; pin++) { struct pinctrl_gpio_range *range; unsigned int hw_gpio = bcm6358_pins[pin].number; diff --git a/drivers/pinctrl/bcm/pinctrl-bcm6362.c b/drivers/pinctrl/bcm/pinctrl-bcm6362.c index 13c7230949b2bc042e250d0467ec3e09556f7146..d9ba1b6c2aebba511565d4506bf6f9a1eef2a18b 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm6362.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm6362.c @@ -35,12 +35,6 @@ enum bcm6362_pinctrl_reg { BCM6362_BASEMODE, }; -struct bcm6362_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; -}; - struct bcm6362_function { const char *name; const char * const *groups; @@ -162,63 +156,56 @@ static unsigned nand_grp_pins[] = { 18, 19, 20, 21, 22, 23, 27, }; -#define BCM6362_GROUP(n) \ - { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ - } - -static struct bcm6362_pingroup bcm6362_groups[] = { - BCM6362_GROUP(gpio0), - BCM6362_GROUP(gpio1), - BCM6362_GROUP(gpio2), - BCM6362_GROUP(gpio3), - BCM6362_GROUP(gpio4), - BCM6362_GROUP(gpio5), - BCM6362_GROUP(gpio6), - BCM6362_GROUP(gpio7), - BCM6362_GROUP(gpio8), - BCM6362_GROUP(gpio9), - BCM6362_GROUP(gpio10), - BCM6362_GROUP(gpio11), - BCM6362_GROUP(gpio12), - BCM6362_GROUP(gpio13), - BCM6362_GROUP(gpio14), - BCM6362_GROUP(gpio15), - BCM6362_GROUP(gpio16), - BCM6362_GROUP(gpio17), - BCM6362_GROUP(gpio18), - BCM6362_GROUP(gpio19), - BCM6362_GROUP(gpio20), - BCM6362_GROUP(gpio21), - BCM6362_GROUP(gpio22), - BCM6362_GROUP(gpio23), - BCM6362_GROUP(gpio24), - BCM6362_GROUP(gpio25), - BCM6362_GROUP(gpio26), - BCM6362_GROUP(gpio27), - BCM6362_GROUP(gpio28), - BCM6362_GROUP(gpio29), - BCM6362_GROUP(gpio30), - BCM6362_GROUP(gpio31), - BCM6362_GROUP(gpio32), - BCM6362_GROUP(gpio33), - BCM6362_GROUP(gpio34), - BCM6362_GROUP(gpio35), - BCM6362_GROUP(gpio36), - BCM6362_GROUP(gpio37), - BCM6362_GROUP(gpio38), - BCM6362_GROUP(gpio39), - BCM6362_GROUP(gpio40), - BCM6362_GROUP(gpio41), - BCM6362_GROUP(gpio42), - BCM6362_GROUP(gpio43), - BCM6362_GROUP(gpio44), - BCM6362_GROUP(gpio45), - BCM6362_GROUP(gpio46), - BCM6362_GROUP(gpio47), - BCM6362_GROUP(nand_grp), +static struct pingroup bcm6362_groups[] = { + BCM_PIN_GROUP(gpio0), + BCM_PIN_GROUP(gpio1), + BCM_PIN_GROUP(gpio2), + BCM_PIN_GROUP(gpio3), + BCM_PIN_GROUP(gpio4), + BCM_PIN_GROUP(gpio5), + BCM_PIN_GROUP(gpio6), + BCM_PIN_GROUP(gpio7), + BCM_PIN_GROUP(gpio8), + BCM_PIN_GROUP(gpio9), + BCM_PIN_GROUP(gpio10), + BCM_PIN_GROUP(gpio11), + BCM_PIN_GROUP(gpio12), + BCM_PIN_GROUP(gpio13), + BCM_PIN_GROUP(gpio14), + BCM_PIN_GROUP(gpio15), + BCM_PIN_GROUP(gpio16), + BCM_PIN_GROUP(gpio17), + BCM_PIN_GROUP(gpio18), + BCM_PIN_GROUP(gpio19), + BCM_PIN_GROUP(gpio20), + BCM_PIN_GROUP(gpio21), + BCM_PIN_GROUP(gpio22), + BCM_PIN_GROUP(gpio23), + BCM_PIN_GROUP(gpio24), + BCM_PIN_GROUP(gpio25), + BCM_PIN_GROUP(gpio26), + BCM_PIN_GROUP(gpio27), + BCM_PIN_GROUP(gpio28), + BCM_PIN_GROUP(gpio29), + BCM_PIN_GROUP(gpio30), + BCM_PIN_GROUP(gpio31), + BCM_PIN_GROUP(gpio32), + BCM_PIN_GROUP(gpio33), + BCM_PIN_GROUP(gpio34), + BCM_PIN_GROUP(gpio35), + BCM_PIN_GROUP(gpio36), + BCM_PIN_GROUP(gpio37), + BCM_PIN_GROUP(gpio38), + BCM_PIN_GROUP(gpio39), + BCM_PIN_GROUP(gpio40), + BCM_PIN_GROUP(gpio41), + BCM_PIN_GROUP(gpio42), + BCM_PIN_GROUP(gpio43), + BCM_PIN_GROUP(gpio44), + BCM_PIN_GROUP(gpio45), + BCM_PIN_GROUP(gpio46), + BCM_PIN_GROUP(gpio47), + BCM_PIN_GROUP(nand_grp), }; static const char * const led_groups[] = { @@ -463,10 +450,10 @@ static const char *bcm6362_pinctrl_get_group_name(struct pinctrl_dev *pctldev, static int bcm6362_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { *pins = bcm6362_groups[group].pins; - *num_pins = bcm6362_groups[group].num_pins; + *npins = bcm6362_groups[group].npins; return 0; } @@ -519,13 +506,13 @@ static int bcm6362_pinctrl_set_mux(struct pinctrl_dev *pctldev, unsigned selector, unsigned group) { struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); - const struct bcm6362_pingroup *pg = &bcm6362_groups[group]; + const struct pingroup *pg = &bcm6362_groups[group]; const struct bcm6362_function *f = &bcm6362_funcs[selector]; unsigned i; unsigned int reg; unsigned int val, mask; - for (i = 0; i < pg->num_pins; i++) + for (i = 0; i < pg->npins; i++) bcm6362_set_gpio(pc, pg->pins[i]); switch (f->reg) { diff --git a/drivers/pinctrl/bcm/pinctrl-bcm6368.c b/drivers/pinctrl/bcm/pinctrl-bcm6368.c index b33a74aec82ba6331d68d13ed91b19325bf2b43f..6208467ba6f9438179a233c42b6842b3cbb99563 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm6368.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm6368.c @@ -26,12 +26,6 @@ #define BCM6368_BASEMODE_GPIO 0x0 #define BCM6368_BASEMODE_UART1 0x1 -struct bcm6368_pingroup { - const char *name; - const unsigned * const pins; - const unsigned num_pins; -}; - struct bcm6368_function { const char *name; const char * const *groups; @@ -127,47 +121,40 @@ static unsigned gpio30_pins[] = { 30 }; static unsigned gpio31_pins[] = { 31 }; static unsigned uart1_grp_pins[] = { 30, 31, 32, 33 }; -#define BCM6368_GROUP(n) \ - { \ - .name = #n, \ - .pins = n##_pins, \ - .num_pins = ARRAY_SIZE(n##_pins), \ - } - -static struct bcm6368_pingroup bcm6368_groups[] = { - BCM6368_GROUP(gpio0), - BCM6368_GROUP(gpio1), - BCM6368_GROUP(gpio2), - BCM6368_GROUP(gpio3), - BCM6368_GROUP(gpio4), - BCM6368_GROUP(gpio5), - BCM6368_GROUP(gpio6), - BCM6368_GROUP(gpio7), - BCM6368_GROUP(gpio8), - BCM6368_GROUP(gpio9), - BCM6368_GROUP(gpio10), - BCM6368_GROUP(gpio11), - BCM6368_GROUP(gpio12), - BCM6368_GROUP(gpio13), - BCM6368_GROUP(gpio14), - BCM6368_GROUP(gpio15), - BCM6368_GROUP(gpio16), - BCM6368_GROUP(gpio17), - BCM6368_GROUP(gpio18), - BCM6368_GROUP(gpio19), - BCM6368_GROUP(gpio20), - BCM6368_GROUP(gpio21), - BCM6368_GROUP(gpio22), - BCM6368_GROUP(gpio23), - BCM6368_GROUP(gpio24), - BCM6368_GROUP(gpio25), - BCM6368_GROUP(gpio26), - BCM6368_GROUP(gpio27), - BCM6368_GROUP(gpio28), - BCM6368_GROUP(gpio29), - BCM6368_GROUP(gpio30), - BCM6368_GROUP(gpio31), - BCM6368_GROUP(uart1_grp), +static struct pingroup bcm6368_groups[] = { + BCM_PIN_GROUP(gpio0), + BCM_PIN_GROUP(gpio1), + BCM_PIN_GROUP(gpio2), + BCM_PIN_GROUP(gpio3), + BCM_PIN_GROUP(gpio4), + BCM_PIN_GROUP(gpio5), + BCM_PIN_GROUP(gpio6), + BCM_PIN_GROUP(gpio7), + BCM_PIN_GROUP(gpio8), + BCM_PIN_GROUP(gpio9), + BCM_PIN_GROUP(gpio10), + BCM_PIN_GROUP(gpio11), + BCM_PIN_GROUP(gpio12), + BCM_PIN_GROUP(gpio13), + BCM_PIN_GROUP(gpio14), + BCM_PIN_GROUP(gpio15), + BCM_PIN_GROUP(gpio16), + BCM_PIN_GROUP(gpio17), + BCM_PIN_GROUP(gpio18), + BCM_PIN_GROUP(gpio19), + BCM_PIN_GROUP(gpio20), + BCM_PIN_GROUP(gpio21), + BCM_PIN_GROUP(gpio22), + BCM_PIN_GROUP(gpio23), + BCM_PIN_GROUP(gpio24), + BCM_PIN_GROUP(gpio25), + BCM_PIN_GROUP(gpio26), + BCM_PIN_GROUP(gpio27), + BCM_PIN_GROUP(gpio28), + BCM_PIN_GROUP(gpio29), + BCM_PIN_GROUP(gpio30), + BCM_PIN_GROUP(gpio31), + BCM_PIN_GROUP(uart1_grp), }; static const char * const analog_afe_0_groups[] = { @@ -358,10 +345,10 @@ static const char *bcm6368_pinctrl_get_group_name(struct pinctrl_dev *pctldev, static int bcm6368_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, unsigned group, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { *pins = bcm6368_groups[group].pins; - *num_pins = bcm6368_groups[group].num_pins; + *npins = bcm6368_groups[group].npins; return 0; } @@ -393,14 +380,14 @@ static int bcm6368_pinctrl_set_mux(struct pinctrl_dev *pctldev, { struct bcm63xx_pinctrl *pc = pinctrl_dev_get_drvdata(pctldev); struct bcm6368_priv *priv = pc->driver_data; - const struct bcm6368_pingroup *pg = &bcm6368_groups[group]; + const struct pingroup *pg = &bcm6368_groups[group]; const struct bcm6368_function *fun = &bcm6368_funcs[selector]; int i, pin; if (fun->basemode) { unsigned int mask = 0; - for (i = 0; i < pg->num_pins; i++) { + for (i = 0; i < pg->npins; i++) { pin = pg->pins[i]; if (pin < BCM63XX_BANK_GPIOS) mask |= BIT(pin); @@ -419,7 +406,7 @@ static int bcm6368_pinctrl_set_mux(struct pinctrl_dev *pctldev, BIT(pin)); } - for (pin = 0; pin < pg->num_pins; pin++) { + for (pin = 0; pin < pg->npins; pin++) { struct pinctrl_gpio_range *range; int hw_gpio = bcm6368_pins[pin].number; diff --git a/drivers/pinctrl/bcm/pinctrl-bcm63xx.h b/drivers/pinctrl/bcm/pinctrl-bcm63xx.h index d58c8cd5b6b8e941cb5e469074f054c09b2fc66b..95243027ecd9ecc53afface8281702aebd67ec71 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm63xx.h +++ b/drivers/pinctrl/bcm/pinctrl-bcm63xx.h @@ -21,6 +21,8 @@ struct bcm63xx_pinctrl_soc { unsigned int ngpios; }; +#define BCM_PIN_GROUP(n) PINCTRL_PINGROUP(#n, n##_pins, ARRAY_SIZE(n##_pins)) + struct bcm63xx_pinctrl { struct device *dev; struct regmap *regs; diff --git a/drivers/pinctrl/bcm/pinctrl-ns.c b/drivers/pinctrl/bcm/pinctrl-ns.c index 65a86543c58cc0fc5f1d9b10c7412cc7d8797581..465cc96814a114e595ae9f0754f55a8eaff93011 100644 --- a/drivers/pinctrl/bcm/pinctrl-ns.c +++ b/drivers/pinctrl/bcm/pinctrl-ns.c @@ -233,10 +233,8 @@ static int ns_pinctrl_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cru_gpio_control"); ns_pinctrl->base = devm_ioremap_resource(dev, res); - if (IS_ERR(ns_pinctrl->base)) { - dev_err(dev, "Failed to map pinctrl regs\n"); + if (IS_ERR(ns_pinctrl->base)) return PTR_ERR(ns_pinctrl->base); - } memcpy(pctldesc, &ns_pinctrl_desc, sizeof(*pctldesc)); diff --git a/drivers/pinctrl/berlin/berlin.c b/drivers/pinctrl/berlin/berlin.c index a073eedd71aa5b3967ffff3d0a3d6d3b4c52e780..1e427ea4d31bcbfce660cd168e9aee6556607b5f 100644 --- a/drivers/pinctrl/berlin/berlin.c +++ b/drivers/pinctrl/berlin/berlin.c @@ -209,7 +209,7 @@ static int berlin_pinctrl_build_state(struct platform_device *pdev) for (i = 0; i < pctrl->desc->ngroups; i++) { desc_group = pctrl->desc->groups + i; - /* compute the maxiumum number of functions a group can have */ + /* compute the maximum number of functions a group can have */ max_functions += 1 << (desc_group->bit_width + 1); } diff --git a/drivers/pinctrl/freescale/Kconfig b/drivers/pinctrl/freescale/Kconfig index d96b1130efd398db65573220a998540549eb0822..7a32f77792d9a5236b0647a2716953422620e1d4 100644 --- a/drivers/pinctrl/freescale/Kconfig +++ b/drivers/pinctrl/freescale/Kconfig @@ -119,28 +119,32 @@ config PINCTRL_IMX7ULP config PINCTRL_IMX8MM tristate "IMX8MM pinctrl driver" - depends on ARCH_MXC + depends on OF + depends on SOC_IMX8M select PINCTRL_IMX help Say Y here to enable the imx8mm pinctrl driver config PINCTRL_IMX8MN tristate "IMX8MN pinctrl driver" - depends on ARCH_MXC + depends on OF + depends on SOC_IMX8M select PINCTRL_IMX help Say Y here to enable the imx8mn pinctrl driver config PINCTRL_IMX8MP tristate "IMX8MP pinctrl driver" - depends on ARCH_MXC + depends on OF + depends on SOC_IMX8M select PINCTRL_IMX help Say Y here to enable the imx8mp pinctrl driver config PINCTRL_IMX8MQ tristate "IMX8MQ pinctrl driver" - depends on ARCH_MXC + depends on OF + depends on SOC_IMX8M select PINCTRL_IMX help Say Y here to enable the imx8mq pinctrl driver diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig index 1600a2c18eeefadf0862d44ed429efe38a014817..fed02c6fea0629b1c971ce7aabd7ade31bf93f35 100644 --- a/drivers/pinctrl/mediatek/Kconfig +++ b/drivers/pinctrl/mediatek/Kconfig @@ -162,6 +162,18 @@ config PINCTRL_MT8186 default ARM64 && ARCH_MEDIATEK select PINCTRL_MTK_PARIS +config PINCTRL_MT8188 + bool "MediaTek MT8188 pin control" + depends on OF + depends on ARM64 || COMPILE_TEST + default ARM64 && ARCH_MEDIATEK + select PINCTRL_MTK_PARIS + help + Say yes here to support pin controller and gpio driver + on MediaTek MT8188 SoC. + In MTK platform, we support virtual gpio and use it to + map specific eint which doesn't have real gpio pin. + config PINCTRL_MT8192 bool "Mediatek MT8192 pin control" depends on OF diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile index c8f226ae36c943638f0afdeef1a8172a1756982f..53265404a39d8386dbc9f03e734b766313cc04c4 100644 --- a/drivers/pinctrl/mediatek/Makefile +++ b/drivers/pinctrl/mediatek/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_PINCTRL_MT8167) += pinctrl-mt8167.o obj-$(CONFIG_PINCTRL_MT8173) += pinctrl-mt8173.o obj-$(CONFIG_PINCTRL_MT8183) += pinctrl-mt8183.o obj-$(CONFIG_PINCTRL_MT8186) += pinctrl-mt8186.o +obj-$(CONFIG_PINCTRL_MT8188) += pinctrl-mt8188.o obj-$(CONFIG_PINCTRL_MT8192) += pinctrl-mt8192.o obj-$(CONFIG_PINCTRL_MT8195) += pinctrl-mt8195.o obj-$(CONFIG_PINCTRL_MT8365) += pinctrl-mt8365.o diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8188.c b/drivers/pinctrl/mediatek/pinctrl-mt8188.c new file mode 100644 index 0000000000000000000000000000000000000000..d0e75c1b4417ab00c87ba00b020c4f8ad3e7b36e --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mt8188.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 MediaTek Inc. + * Author: Hui Liu + * + */ + +#include +#include "pinctrl-mtk-mt8188.h" +#include "pinctrl-paris.h" + +/* MT8188 have multiple bases to program pin configuration listed as the below: + * iocfg[0]:0x10005000, iocfg[1]:0x11c00000, iocfg[2]:0x11e10000, + * iocfg[3]:0x11e20000, iocfg[4]:0x11ea0000 + * _i_based could be used to indicate what base the pin should be mapped into. + */ + +#define PIN_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, \ + 32, 0) + +#define PINS_FIELD_BASE(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits) \ + PIN_FIELD_CALC(s_pin, e_pin, i_base, s_addr, x_addrs, s_bit, x_bits, \ + 32, 1) + +static const struct mtk_pin_field_calc mt8188_pin_mode_range[] = { + PIN_FIELD(0, 177, 0x0300, 0x10, 0, 4), +}; + +static const struct mtk_pin_field_calc mt8188_pin_dir_range[] = { + PIN_FIELD(0, 177, 0x0000, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_di_range[] = { + PIN_FIELD(0, 177, 0x0200, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_do_range[] = { + PIN_FIELD(0, 177, 0x0100, 0x10, 0, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_smt_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x0170, 0x10, 8, 1), + PIN_FIELD_BASE(1, 1, 1, 0x0170, 0x10, 9, 1), + PIN_FIELD_BASE(2, 2, 1, 0x0170, 0x10, 10, 1), + PIN_FIELD_BASE(3, 3, 1, 0x0170, 0x10, 11, 1), + PIN_FIELD_BASE(4, 4, 1, 0x0170, 0x10, 18, 1), + PIN_FIELD_BASE(5, 5, 1, 0x0170, 0x10, 18, 1), + PIN_FIELD_BASE(6, 6, 1, 0x0170, 0x10, 18, 1), + PIN_FIELD_BASE(7, 7, 1, 0x0170, 0x10, 12, 1), + PIN_FIELD_BASE(8, 8, 1, 0x0170, 0x10, 13, 1), + PIN_FIELD_BASE(9, 9, 1, 0x0170, 0x10, 14, 1), + PIN_FIELD_BASE(10, 10, 1, 0x0170, 0x10, 15, 1), + PIN_FIELD_BASE(11, 11, 1, 0x0170, 0x10, 19, 1), + PIN_FIELD_BASE(12, 12, 2, 0x0160, 0x10, 12, 1), + PIN_FIELD_BASE(13, 13, 2, 0x0160, 0x10, 13, 1), + PIN_FIELD_BASE(14, 14, 2, 0x0160, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 2, 0x0160, 0x10, 15, 1), + PIN_FIELD_BASE(16, 16, 3, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(17, 17, 3, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(18, 18, 4, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(19, 19, 4, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(20, 20, 4, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(21, 21, 4, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(22, 22, 4, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(23, 23, 4, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(24, 24, 4, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0170, 0x10, 17, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0170, 0x10, 17, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0170, 0x10, 17, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0170, 0x10, 18, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0170, 0x10, 16, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0170, 0x10, 17, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0170, 0x10, 19, 1), + PIN_FIELD_BASE(32, 32, 1, 0x0170, 0x10, 19, 1), + PIN_FIELD_BASE(33, 33, 1, 0x0170, 0x10, 20, 1), + PIN_FIELD_BASE(34, 34, 1, 0x0170, 0x10, 20, 1), + PIN_FIELD_BASE(35, 35, 1, 0x0170, 0x10, 19, 1), + PIN_FIELD_BASE(36, 36, 1, 0x0170, 0x10, 20, 1), + PIN_FIELD_BASE(37, 37, 1, 0x0170, 0x10, 21, 1), + PIN_FIELD_BASE(38, 38, 1, 0x0170, 0x10, 20, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0170, 0x10, 21, 1), + PIN_FIELD_BASE(40, 40, 1, 0x0170, 0x10, 21, 1), + PIN_FIELD_BASE(41, 41, 1, 0x0170, 0x10, 21, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0160, 0x10, 21, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0160, 0x10, 22, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0160, 0x10, 21, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0160, 0x10, 22, 1), + PIN_FIELD_BASE(46, 46, 3, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0170, 0x10, 16, 1), + PIN_FIELD_BASE(48, 48, 1, 0x0170, 0x10, 16, 1), + PIN_FIELD_BASE(49, 49, 1, 0x0170, 0x10, 16, 1), + PIN_FIELD_BASE(50, 50, 3, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(51, 51, 3, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(52, 52, 3, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(53, 53, 3, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(54, 54, 3, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(55, 55, 1, 0x0170, 0x10, 25, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0170, 0x10, 28, 1), + PIN_FIELD_BASE(57, 57, 2, 0x0160, 0x10, 29, 1), + PIN_FIELD_BASE(58, 58, 2, 0x0160, 0x10, 31, 1), + PIN_FIELD_BASE(59, 59, 1, 0x0170, 0x10, 26, 1), + PIN_FIELD_BASE(60, 60, 1, 0x0170, 0x10, 29, 1), + PIN_FIELD_BASE(61, 61, 1, 0x0170, 0x10, 27, 1), + PIN_FIELD_BASE(62, 62, 1, 0x0170, 0x10, 30, 1), + PIN_FIELD_BASE(63, 63, 2, 0x0160, 0x10, 30, 1), + PIN_FIELD_BASE(64, 64, 2, 0x0170, 0x10, 0, 1), + PIN_FIELD_BASE(65, 65, 4, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(66, 66, 4, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(67, 67, 4, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(68, 68, 4, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(69, 69, 1, 0x0180, 0x10, 0, 1), + PIN_FIELD_BASE(70, 70, 1, 0x0170, 0x10, 31, 1), + PIN_FIELD_BASE(71, 71, 1, 0x0180, 0x10, 4, 1), + PIN_FIELD_BASE(72, 72, 1, 0x0180, 0x10, 3, 1), + PIN_FIELD_BASE(73, 73, 1, 0x0180, 0x10, 1, 1), + PIN_FIELD_BASE(74, 74, 1, 0x0180, 0x10, 2, 1), + PIN_FIELD_BASE(75, 75, 1, 0x0180, 0x10, 6, 1), + PIN_FIELD_BASE(76, 76, 1, 0x0180, 0x10, 5, 1), + PIN_FIELD_BASE(77, 77, 1, 0x0180, 0x10, 8, 1), + PIN_FIELD_BASE(78, 78, 1, 0x0180, 0x10, 7, 1), + PIN_FIELD_BASE(79, 79, 4, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(80, 80, 4, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(81, 81, 4, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(82, 82, 4, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(83, 83, 2, 0x0160, 0x10, 26, 1), + PIN_FIELD_BASE(84, 84, 2, 0x0160, 0x10, 26, 1), + PIN_FIELD_BASE(85, 85, 2, 0x0160, 0x10, 27, 1), + PIN_FIELD_BASE(86, 86, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(87, 87, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(88, 88, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(89, 89, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(90, 90, 2, 0x0160, 0x10, 27, 1), + PIN_FIELD_BASE(91, 91, 2, 0x0160, 0x10, 27, 1), + PIN_FIELD_BASE(92, 92, 2, 0x0160, 0x10, 18, 1), + PIN_FIELD_BASE(93, 93, 2, 0x0160, 0x10, 18, 1), + PIN_FIELD_BASE(94, 94, 2, 0x0160, 0x10, 18, 1), + PIN_FIELD_BASE(95, 95, 2, 0x0160, 0x10, 18, 1), + PIN_FIELD_BASE(96, 96, 2, 0x0160, 0x10, 22, 1), + PIN_FIELD_BASE(97, 97, 2, 0x0160, 0x10, 23, 1), + PIN_FIELD_BASE(98, 98, 2, 0x0160, 0x10, 24, 1), + PIN_FIELD_BASE(99, 99, 2, 0x0160, 0x10, 22, 1), + PIN_FIELD_BASE(100, 100, 2, 0x0160, 0x10, 16, 1), + PIN_FIELD_BASE(101, 101, 2, 0x0160, 0x10, 23, 1), + PIN_FIELD_BASE(102, 102, 2, 0x0160, 0x10, 23, 1), + PIN_FIELD_BASE(103, 103, 2, 0x0160, 0x10, 23, 1), + PIN_FIELD_BASE(104, 104, 2, 0x0160, 0x10, 24, 1), + PIN_FIELD_BASE(105, 105, 2, 0x0160, 0x10, 24, 1), + PIN_FIELD_BASE(106, 106, 2, 0x0160, 0x10, 24, 1), + PIN_FIELD_BASE(107, 107, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(108, 108, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(109, 109, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(110, 110, 2, 0x0160, 0x10, 17, 1), + PIN_FIELD_BASE(111, 111, 2, 0x0160, 0x10, 19, 1), + PIN_FIELD_BASE(112, 112, 2, 0x0160, 0x10, 19, 1), + PIN_FIELD_BASE(113, 113, 2, 0x0160, 0x10, 19, 1), + PIN_FIELD_BASE(114, 114, 2, 0x0160, 0x10, 19, 1), + PIN_FIELD_BASE(115, 115, 2, 0x0160, 0x10, 20, 1), + PIN_FIELD_BASE(116, 116, 2, 0x0160, 0x10, 20, 1), + PIN_FIELD_BASE(117, 117, 2, 0x0160, 0x10, 20, 1), + PIN_FIELD_BASE(118, 118, 2, 0x0160, 0x10, 20, 1), + PIN_FIELD_BASE(119, 119, 2, 0x0160, 0x10, 21, 1), + PIN_FIELD_BASE(120, 120, 2, 0x0160, 0x10, 21, 1), + PIN_FIELD_BASE(121, 121, 3, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(122, 122, 3, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(123, 123, 3, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(124, 124, 3, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(125, 125, 2, 0x0160, 0x10, 25, 1), + PIN_FIELD_BASE(126, 126, 2, 0x0160, 0x10, 25, 1), + PIN_FIELD_BASE(127, 127, 2, 0x0160, 0x10, 25, 1), + PIN_FIELD_BASE(128, 128, 2, 0x0160, 0x10, 25, 1), + PIN_FIELD_BASE(129, 129, 2, 0x0160, 0x10, 26, 1), + PIN_FIELD_BASE(130, 130, 2, 0x0160, 0x10, 26, 1), + PIN_FIELD_BASE(131, 131, 1, 0x0170, 0x10, 0, 1), + PIN_FIELD_BASE(132, 132, 1, 0x0170, 0x10, 1, 1), + PIN_FIELD_BASE(133, 133, 1, 0x0170, 0x10, 6, 1), + PIN_FIELD_BASE(134, 134, 1, 0x0170, 0x10, 7, 1), + PIN_FIELD_BASE(135, 135, 1, 0x0170, 0x10, 22, 1), + PIN_FIELD_BASE(136, 136, 1, 0x0170, 0x10, 22, 1), + PIN_FIELD_BASE(137, 137, 1, 0x0170, 0x10, 22, 1), + PIN_FIELD_BASE(138, 138, 1, 0x0170, 0x10, 22, 1), + PIN_FIELD_BASE(139, 139, 1, 0x0170, 0x10, 23, 1), + PIN_FIELD_BASE(140, 140, 1, 0x0170, 0x10, 23, 1), + PIN_FIELD_BASE(141, 141, 1, 0x0170, 0x10, 23, 1), + PIN_FIELD_BASE(142, 142, 1, 0x0170, 0x10, 23, 1), + PIN_FIELD_BASE(143, 143, 1, 0x0170, 0x10, 2, 1), + PIN_FIELD_BASE(144, 144, 1, 0x0170, 0x10, 3, 1), + PIN_FIELD_BASE(145, 145, 1, 0x0170, 0x10, 4, 1), + PIN_FIELD_BASE(146, 146, 1, 0x0170, 0x10, 5, 1), + PIN_FIELD_BASE(147, 147, 1, 0x0170, 0x10, 24, 1), + PIN_FIELD_BASE(148, 148, 1, 0x0170, 0x10, 24, 1), + PIN_FIELD_BASE(149, 149, 1, 0x0170, 0x10, 24, 1), + PIN_FIELD_BASE(150, 150, 1, 0x0170, 0x10, 24, 1), + PIN_FIELD_BASE(151, 151, 2, 0x0160, 0x10, 9, 1), + PIN_FIELD_BASE(152, 152, 2, 0x0160, 0x10, 8, 1), + PIN_FIELD_BASE(153, 153, 2, 0x0160, 0x10, 7, 1), + PIN_FIELD_BASE(154, 154, 2, 0x0160, 0x10, 6, 1), + PIN_FIELD_BASE(155, 155, 2, 0x0160, 0x10, 11, 1), + PIN_FIELD_BASE(156, 156, 2, 0x0160, 0x10, 1, 1), + PIN_FIELD_BASE(157, 157, 2, 0x0160, 0x10, 0, 1), + PIN_FIELD_BASE(158, 158, 2, 0x0160, 0x10, 5, 1), + PIN_FIELD_BASE(159, 159, 2, 0x0160, 0x10, 4, 1), + PIN_FIELD_BASE(160, 160, 2, 0x0160, 0x10, 3, 1), + PIN_FIELD_BASE(161, 161, 2, 0x0160, 0x10, 2, 1), + PIN_FIELD_BASE(162, 162, 2, 0x0160, 0x10, 10, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(164, 164, 4, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(165, 165, 4, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(166, 166, 4, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(167, 167, 4, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(168, 168, 4, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(169, 169, 3, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(170, 170, 3, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(171, 171, 3, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(172, 172, 3, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(173, 173, 3, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(174, 174, 3, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(175, 175, 2, 0x0160, 0x10, 28, 1), + PIN_FIELD_BASE(176, 176, 2, 0x0160, 0x10, 28, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_ies_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(1, 1, 1, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(2, 2, 1, 0x0080, 0x10, 28, 1), + PIN_FIELD_BASE(3, 3, 1, 0x0080, 0x10, 29, 1), + PIN_FIELD_BASE(4, 4, 1, 0x0080, 0x10, 30, 1), + PIN_FIELD_BASE(5, 5, 1, 0x0080, 0x10, 31, 1), + PIN_FIELD_BASE(6, 6, 1, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(7, 7, 1, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(8, 8, 1, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(9, 9, 1, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(10, 10, 1, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(11, 11, 1, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(12, 12, 2, 0x0070, 0x10, 24, 1), + PIN_FIELD_BASE(13, 13, 2, 0x0070, 0x10, 25, 1), + PIN_FIELD_BASE(14, 14, 2, 0x0070, 0x10, 26, 1), + PIN_FIELD_BASE(15, 15, 2, 0x0070, 0x10, 27, 1), + PIN_FIELD_BASE(16, 16, 3, 0x0040, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 3, 0x0040, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 4, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(19, 19, 4, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(20, 20, 4, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(21, 21, 4, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(22, 22, 4, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(23, 23, 4, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(24, 24, 4, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(25, 25, 1, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(26, 26, 1, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(27, 27, 1, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(28, 28, 1, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(29, 29, 1, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(30, 30, 1, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(31, 31, 1, 0x0090, 0x10, 31, 1), + PIN_FIELD_BASE(32, 32, 1, 0x0090, 0x10, 30, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(36, 36, 1, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(37, 37, 1, 0x0090, 0x10, 9, 1), + PIN_FIELD_BASE(38, 38, 1, 0x0090, 0x10, 6, 1), + PIN_FIELD_BASE(39, 39, 1, 0x0090, 0x10, 7, 1), + PIN_FIELD_BASE(40, 40, 1, 0x0090, 0x10, 8, 1), + PIN_FIELD_BASE(41, 41, 1, 0x0090, 0x10, 10, 1), + PIN_FIELD_BASE(42, 42, 2, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(46, 46, 3, 0x0040, 0x10, 0, 1), + PIN_FIELD_BASE(47, 47, 1, 0x0090, 0x10, 13, 1), + PIN_FIELD_BASE(48, 48, 1, 0x0090, 0x10, 12, 1), + PIN_FIELD_BASE(49, 49, 1, 0x0090, 0x10, 11, 1), + PIN_FIELD_BASE(50, 50, 3, 0x0040, 0x10, 5, 1), + PIN_FIELD_BASE(51, 51, 3, 0x0040, 0x10, 4, 1), + PIN_FIELD_BASE(52, 52, 3, 0x0040, 0x10, 3, 1), + PIN_FIELD_BASE(53, 53, 3, 0x0040, 0x10, 6, 1), + PIN_FIELD_BASE(54, 54, 3, 0x0040, 0x10, 7, 1), + PIN_FIELD_BASE(55, 55, 1, 0x0090, 0x10, 14, 1), + PIN_FIELD_BASE(56, 56, 1, 0x0090, 0x10, 17, 1), + PIN_FIELD_BASE(57, 57, 2, 0x0080, 0x10, 22, 1), + PIN_FIELD_BASE(58, 58, 2, 0x0080, 0x10, 25, 1), + PIN_FIELD_BASE(59, 59, 1, 0x0090, 0x10, 15, 1), + PIN_FIELD_BASE(60, 60, 1, 0x0090, 0x10, 18, 1), + PIN_FIELD_BASE(61, 61, 1, 0x0090, 0x10, 16, 1), + PIN_FIELD_BASE(62, 62, 1, 0x0090, 0x10, 19, 1), + PIN_FIELD_BASE(63, 63, 2, 0x0080, 0x10, 23, 1), + PIN_FIELD_BASE(64, 64, 2, 0x0080, 0x10, 26, 1), + PIN_FIELD_BASE(65, 65, 4, 0x0050, 0x10, 13, 1), + PIN_FIELD_BASE(66, 66, 4, 0x0050, 0x10, 15, 1), + PIN_FIELD_BASE(67, 67, 4, 0x0050, 0x10, 14, 1), + PIN_FIELD_BASE(68, 68, 4, 0x0050, 0x10, 16, 1), + PIN_FIELD_BASE(69, 69, 1, 0x0090, 0x10, 21, 1), + PIN_FIELD_BASE(70, 70, 1, 0x0090, 0x10, 20, 1), + PIN_FIELD_BASE(71, 71, 1, 0x0090, 0x10, 25, 1), + PIN_FIELD_BASE(72, 72, 1, 0x0090, 0x10, 24, 1), + PIN_FIELD_BASE(73, 73, 1, 0x0090, 0x10, 22, 1), + PIN_FIELD_BASE(74, 74, 1, 0x0090, 0x10, 23, 1), + PIN_FIELD_BASE(75, 75, 1, 0x0090, 0x10, 27, 1), + PIN_FIELD_BASE(76, 76, 1, 0x0090, 0x10, 26, 1), + PIN_FIELD_BASE(77, 77, 1, 0x0090, 0x10, 29, 1), + PIN_FIELD_BASE(78, 78, 1, 0x0090, 0x10, 28, 1), + PIN_FIELD_BASE(79, 79, 4, 0x0050, 0x10, 18, 1), + PIN_FIELD_BASE(80, 80, 4, 0x0050, 0x10, 17, 1), + PIN_FIELD_BASE(81, 81, 4, 0x0050, 0x10, 20, 1), + PIN_FIELD_BASE(82, 82, 4, 0x0050, 0x10, 19, 1), + PIN_FIELD_BASE(83, 83, 2, 0x0080, 0x10, 30, 1), + PIN_FIELD_BASE(84, 84, 2, 0x0080, 0x10, 29, 1), + PIN_FIELD_BASE(85, 85, 2, 0x0080, 0x10, 31, 1), + PIN_FIELD_BASE(86, 86, 2, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(87, 87, 2, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(88, 88, 2, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(89, 89, 2, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(90, 90, 2, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(91, 91, 2, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(92, 92, 2, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(93, 93, 2, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(94, 94, 2, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(95, 95, 2, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(96, 96, 2, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(97, 97, 2, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(98, 98, 2, 0x0080, 0x10, 24, 1), + PIN_FIELD_BASE(99, 99, 2, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(100, 100, 2, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(101, 101, 2, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(102, 102, 2, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(103, 103, 2, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(104, 104, 2, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(105, 105, 2, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(106, 106, 2, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(107, 107, 2, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(108, 108, 2, 0x0070, 0x10, 28, 1), + PIN_FIELD_BASE(109, 109, 2, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(110, 110, 2, 0x0070, 0x10, 29, 1), + PIN_FIELD_BASE(111, 111, 2, 0x0070, 0x10, 30, 1), + PIN_FIELD_BASE(112, 112, 2, 0x0070, 0x10, 31, 1), + PIN_FIELD_BASE(113, 113, 2, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(114, 114, 2, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(115, 115, 2, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(116, 116, 2, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(117, 117, 2, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(118, 118, 2, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(119, 119, 2, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(120, 120, 2, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(121, 121, 3, 0x0040, 0x10, 14, 1), + PIN_FIELD_BASE(122, 122, 3, 0x0040, 0x10, 17, 1), + PIN_FIELD_BASE(123, 123, 3, 0x0040, 0x10, 16, 1), + PIN_FIELD_BASE(124, 124, 3, 0x0040, 0x10, 15, 1), + PIN_FIELD_BASE(125, 125, 2, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(126, 126, 2, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(127, 127, 2, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(128, 128, 2, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(129, 129, 2, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(130, 130, 2, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(131, 131, 1, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(132, 132, 1, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(133, 133, 1, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(134, 134, 1, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(135, 135, 1, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(136, 136, 1, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(137, 137, 1, 0x0080, 0x10, 15, 1), + PIN_FIELD_BASE(138, 138, 1, 0x0080, 0x10, 16, 1), + PIN_FIELD_BASE(139, 139, 1, 0x0080, 0x10, 17, 1), + PIN_FIELD_BASE(140, 140, 1, 0x0080, 0x10, 18, 1), + PIN_FIELD_BASE(141, 141, 1, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(142, 142, 1, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(143, 143, 1, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(144, 144, 1, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(145, 145, 1, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(146, 146, 1, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(147, 147, 1, 0x0080, 0x10, 20, 1), + PIN_FIELD_BASE(148, 148, 1, 0x0080, 0x10, 21, 1), + PIN_FIELD_BASE(149, 149, 1, 0x0080, 0x10, 19, 1), + PIN_FIELD_BASE(150, 150, 1, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(151, 151, 2, 0x0070, 0x10, 21, 1), + PIN_FIELD_BASE(152, 152, 2, 0x0070, 0x10, 20, 1), + PIN_FIELD_BASE(153, 153, 2, 0x0070, 0x10, 19, 1), + PIN_FIELD_BASE(154, 154, 2, 0x0070, 0x10, 18, 1), + PIN_FIELD_BASE(155, 155, 2, 0x0070, 0x10, 23, 1), + PIN_FIELD_BASE(156, 156, 2, 0x0070, 0x10, 13, 1), + PIN_FIELD_BASE(157, 157, 2, 0x0070, 0x10, 12, 1), + PIN_FIELD_BASE(158, 158, 2, 0x0070, 0x10, 17, 1), + PIN_FIELD_BASE(159, 159, 2, 0x0070, 0x10, 16, 1), + PIN_FIELD_BASE(160, 160, 2, 0x0070, 0x10, 15, 1), + PIN_FIELD_BASE(161, 161, 2, 0x0070, 0x10, 14, 1), + PIN_FIELD_BASE(162, 162, 2, 0x0070, 0x10, 22, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(164, 164, 4, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(165, 165, 4, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(166, 166, 4, 0x0050, 0x10, 10, 1), + PIN_FIELD_BASE(167, 167, 4, 0x0050, 0x10, 11, 1), + PIN_FIELD_BASE(168, 168, 4, 0x0050, 0x10, 12, 1), + PIN_FIELD_BASE(169, 169, 3, 0x0040, 0x10, 9, 1), + PIN_FIELD_BASE(170, 170, 3, 0x0040, 0x10, 8, 1), + PIN_FIELD_BASE(171, 171, 3, 0x0040, 0x10, 10, 1), + PIN_FIELD_BASE(172, 172, 3, 0x0040, 0x10, 11, 1), + PIN_FIELD_BASE(173, 173, 3, 0x0040, 0x10, 12, 1), + PIN_FIELD_BASE(174, 174, 3, 0x0040, 0x10, 13, 1), + PIN_FIELD_BASE(175, 175, 2, 0x0080, 0x10, 27, 1), + PIN_FIELD_BASE(176, 176, 2, 0x0080, 0x10, 28, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_tdsel_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x01b0, 0x10, 0, 4), + PIN_FIELD_BASE(1, 1, 1, 0x01b0, 0x10, 4, 4), + PIN_FIELD_BASE(2, 2, 1, 0x01b0, 0x10, 8, 4), + PIN_FIELD_BASE(3, 3, 1, 0x01b0, 0x10, 12, 4), + PIN_FIELD_BASE(4, 4, 1, 0x01c0, 0x10, 16, 4), + PIN_FIELD_BASE(5, 5, 1, 0x01c0, 0x10, 20, 4), + PIN_FIELD_BASE(6, 6, 1, 0x01c0, 0x10, 20, 4), + PIN_FIELD_BASE(7, 7, 1, 0x01b0, 0x10, 16, 4), + PIN_FIELD_BASE(8, 8, 1, 0x01b0, 0x10, 20, 4), + PIN_FIELD_BASE(9, 9, 1, 0x01b0, 0x10, 24, 4), + PIN_FIELD_BASE(10, 10, 1, 0x01b0, 0x10, 28, 4), + PIN_FIELD_BASE(11, 11, 1, 0x01c0, 0x10, 20, 4), + PIN_FIELD_BASE(12, 12, 2, 0x0190, 0x10, 16, 4), + PIN_FIELD_BASE(13, 13, 2, 0x0190, 0x10, 20, 4), + PIN_FIELD_BASE(14, 14, 2, 0x0190, 0x10, 24, 4), + PIN_FIELD_BASE(15, 15, 2, 0x0190, 0x10, 28, 4), + PIN_FIELD_BASE(16, 16, 3, 0x0100, 0x10, 8, 4), + PIN_FIELD_BASE(17, 17, 3, 0x0100, 0x10, 8, 4), + PIN_FIELD_BASE(18, 18, 4, 0x0110, 0x10, 4, 4), + PIN_FIELD_BASE(19, 19, 4, 0x0110, 0x10, 8, 4), + PIN_FIELD_BASE(20, 20, 4, 0x0110, 0x10, 8, 4), + PIN_FIELD_BASE(21, 21, 4, 0x0110, 0x10, 8, 4), + PIN_FIELD_BASE(22, 22, 4, 0x0100, 0x10, 0, 4), + PIN_FIELD_BASE(23, 23, 4, 0x0100, 0x10, 4, 4), + PIN_FIELD_BASE(24, 24, 4, 0x0100, 0x10, 8, 4), + PIN_FIELD_BASE(25, 25, 1, 0x01c0, 0x10, 8, 4), + PIN_FIELD_BASE(26, 26, 1, 0x01c0, 0x10, 8, 4), + PIN_FIELD_BASE(27, 27, 1, 0x01c0, 0x10, 8, 4), + PIN_FIELD_BASE(28, 28, 1, 0x01c0, 0x10, 12, 4), + PIN_FIELD_BASE(29, 29, 1, 0x01c0, 0x10, 0, 4), + PIN_FIELD_BASE(30, 30, 1, 0x01c0, 0x10, 8, 4), + PIN_FIELD_BASE(31, 31, 1, 0x01c0, 0x10, 20, 4), + PIN_FIELD_BASE(32, 32, 1, 0x01c0, 0x10, 24, 4), + PIN_FIELD_BASE(33, 33, 1, 0x01c0, 0x10, 24, 4), + PIN_FIELD_BASE(34, 34, 1, 0x01c0, 0x10, 28, 4), + PIN_FIELD_BASE(35, 35, 1, 0x01c0, 0x10, 24, 4), + PIN_FIELD_BASE(36, 36, 1, 0x01c0, 0x10, 24, 4), + PIN_FIELD_BASE(37, 37, 1, 0x01c0, 0x10, 28, 4), + PIN_FIELD_BASE(38, 38, 1, 0x01c0, 0x10, 28, 4), + PIN_FIELD_BASE(39, 39, 1, 0x01c0, 0x10, 28, 4), + PIN_FIELD_BASE(40, 40, 1, 0x01d0, 0x10, 0, 4), + PIN_FIELD_BASE(41, 41, 1, 0x01d0, 0x10, 0, 4), + PIN_FIELD_BASE(42, 42, 2, 0x01a0, 0x10, 16, 4), + PIN_FIELD_BASE(43, 43, 2, 0x01a0, 0x10, 20, 4), + PIN_FIELD_BASE(44, 44, 2, 0x01a0, 0x10, 16, 4), + PIN_FIELD_BASE(45, 45, 2, 0x01a0, 0x10, 20, 4), + PIN_FIELD_BASE(46, 46, 3, 0x0100, 0x10, 8, 4), + PIN_FIELD_BASE(47, 47, 1, 0x01c0, 0x10, 0, 4), + PIN_FIELD_BASE(48, 48, 1, 0x01c0, 0x10, 0, 4), + PIN_FIELD_BASE(49, 49, 1, 0x01c0, 0x10, 0, 4), + PIN_FIELD_BASE(50, 50, 3, 0x0100, 0x10, 8, 4), + PIN_FIELD_BASE(51, 51, 3, 0x0100, 0x10, 12, 4), + PIN_FIELD_BASE(52, 52, 3, 0x0100, 0x10, 12, 4), + PIN_FIELD_BASE(53, 53, 3, 0x0100, 0x10, 12, 4), + PIN_FIELD_BASE(54, 54, 3, 0x0100, 0x10, 12, 4), + PIN_FIELD_BASE(55, 55, 1, 0x01c0, 0x10, 12, 4), + PIN_FIELD_BASE(56, 56, 1, 0x01c0, 0x10, 12, 4), + PIN_FIELD_BASE(57, 57, 2, 0x01a0, 0x10, 24, 4), + PIN_FIELD_BASE(58, 58, 2, 0x01a0, 0x10, 24, 4), + PIN_FIELD_BASE(59, 59, 1, 0x01c0, 0x10, 16, 4), + PIN_FIELD_BASE(60, 60, 1, 0x01c0, 0x10, 12, 4), + PIN_FIELD_BASE(61, 61, 1, 0x01c0, 0x10, 16, 4), + PIN_FIELD_BASE(62, 62, 1, 0x01c0, 0x10, 16, 4), + PIN_FIELD_BASE(63, 63, 2, 0x01a0, 0x10, 20, 4), + PIN_FIELD_BASE(64, 64, 2, 0x01a0, 0x10, 20, 4), + PIN_FIELD_BASE(65, 65, 4, 0x0110, 0x10, 12, 4), + PIN_FIELD_BASE(66, 66, 4, 0x0110, 0x10, 8, 4), + PIN_FIELD_BASE(67, 67, 4, 0x0110, 0x10, 12, 4), + PIN_FIELD_BASE(68, 68, 4, 0x0110, 0x10, 12, 4), + PIN_FIELD_BASE(69, 69, 1, 0x01d0, 0x10, 16, 4), + PIN_FIELD_BASE(70, 70, 1, 0x01d0, 0x10, 12, 4), + PIN_FIELD_BASE(71, 71, 1, 0x01e0, 0x10, 0, 4), + PIN_FIELD_BASE(72, 72, 1, 0x01d0, 0x10, 28, 4), + PIN_FIELD_BASE(73, 73, 1, 0x01d0, 0x10, 20, 4), + PIN_FIELD_BASE(74, 74, 1, 0x01d0, 0x10, 24, 4), + PIN_FIELD_BASE(75, 75, 1, 0x01e0, 0x10, 8, 4), + PIN_FIELD_BASE(76, 76, 1, 0x01e0, 0x10, 4, 4), + PIN_FIELD_BASE(77, 77, 1, 0x01e0, 0x10, 16, 4), + PIN_FIELD_BASE(78, 78, 1, 0x01e0, 0x10, 12, 4), + PIN_FIELD_BASE(79, 79, 4, 0x0110, 0x10, 20, 4), + PIN_FIELD_BASE(80, 80, 4, 0x0110, 0x10, 16, 4), + PIN_FIELD_BASE(81, 81, 4, 0x0110, 0x10, 28, 4), + PIN_FIELD_BASE(82, 82, 4, 0x0110, 0x10, 24, 4), + PIN_FIELD_BASE(83, 83, 2, 0x01b0, 0x10, 8, 4), + PIN_FIELD_BASE(84, 84, 2, 0x01b0, 0x10, 8, 4), + PIN_FIELD_BASE(85, 85, 2, 0x01b0, 0x10, 12, 4), + PIN_FIELD_BASE(86, 86, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(87, 87, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(88, 88, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(89, 89, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(90, 90, 2, 0x01b0, 0x10, 12, 4), + PIN_FIELD_BASE(91, 91, 2, 0x01b0, 0x10, 12, 4), + PIN_FIELD_BASE(92, 92, 2, 0x01a0, 0x10, 4, 4), + PIN_FIELD_BASE(93, 93, 2, 0x01a0, 0x10, 4, 4), + PIN_FIELD_BASE(94, 94, 2, 0x01a0, 0x10, 4, 4), + PIN_FIELD_BASE(95, 95, 2, 0x01a0, 0x10, 4, 4), + PIN_FIELD_BASE(96, 96, 2, 0x01a0, 0x10, 24, 4), + PIN_FIELD_BASE(97, 97, 2, 0x01a0, 0x10, 28, 4), + PIN_FIELD_BASE(98, 98, 2, 0x01b0, 0x10, 0, 4), + PIN_FIELD_BASE(99, 99, 2, 0x01a0, 0x10, 24, 4), + PIN_FIELD_BASE(100, 100, 2, 0x01b0, 0x10, 20, 4), + PIN_FIELD_BASE(101, 101, 2, 0x01a0, 0x10, 28, 4), + PIN_FIELD_BASE(102, 102, 2, 0x01a0, 0x10, 28, 4), + PIN_FIELD_BASE(103, 103, 2, 0x01a0, 0x10, 28, 4), + PIN_FIELD_BASE(104, 104, 2, 0x01b0, 0x10, 0, 4), + PIN_FIELD_BASE(105, 105, 2, 0x01b0, 0x10, 0, 4), + PIN_FIELD_BASE(106, 106, 2, 0x01b0, 0x10, 0, 4), + PIN_FIELD_BASE(107, 107, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(108, 108, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(109, 109, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(110, 110, 2, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(111, 111, 2, 0x01a0, 0x10, 8, 4), + PIN_FIELD_BASE(112, 112, 2, 0x01a0, 0x10, 8, 4), + PIN_FIELD_BASE(113, 113, 2, 0x01a0, 0x10, 8, 4), + PIN_FIELD_BASE(114, 114, 2, 0x01a0, 0x10, 8, 4), + PIN_FIELD_BASE(115, 115, 2, 0x01a0, 0x10, 12, 4), + PIN_FIELD_BASE(116, 116, 2, 0x01a0, 0x10, 12, 4), + PIN_FIELD_BASE(117, 117, 2, 0x01a0, 0x10, 12, 4), + PIN_FIELD_BASE(118, 118, 2, 0x01a0, 0x10, 12, 4), + PIN_FIELD_BASE(119, 119, 2, 0x01a0, 0x10, 16, 4), + PIN_FIELD_BASE(120, 120, 2, 0x01a0, 0x10, 16, 4), + PIN_FIELD_BASE(121, 121, 3, 0x00f0, 0x10, 24, 4), + PIN_FIELD_BASE(122, 122, 3, 0x0100, 0x10, 4, 4), + PIN_FIELD_BASE(123, 123, 3, 0x0100, 0x10, 0, 4), + PIN_FIELD_BASE(124, 124, 3, 0x00f0, 0x10, 28, 4), + PIN_FIELD_BASE(125, 125, 2, 0x01b0, 0x10, 4, 4), + PIN_FIELD_BASE(126, 126, 2, 0x01b0, 0x10, 4, 4), + PIN_FIELD_BASE(127, 127, 2, 0x01b0, 0x10, 4, 4), + PIN_FIELD_BASE(128, 128, 2, 0x01b0, 0x10, 4, 4), + PIN_FIELD_BASE(129, 129, 2, 0x01b0, 0x10, 8, 4), + PIN_FIELD_BASE(130, 130, 2, 0x01b0, 0x10, 8, 4), + PIN_FIELD_BASE(131, 131, 1, 0x01a0, 0x10, 0, 4), + PIN_FIELD_BASE(132, 132, 1, 0x01a0, 0x10, 20, 4), + PIN_FIELD_BASE(133, 133, 1, 0x01a0, 0x10, 24, 4), + PIN_FIELD_BASE(134, 134, 1, 0x01a0, 0x10, 28, 4), + PIN_FIELD_BASE(135, 135, 1, 0x01d0, 0x10, 0, 4), + PIN_FIELD_BASE(136, 136, 1, 0x01d0, 0x10, 0, 4), + PIN_FIELD_BASE(137, 137, 1, 0x01d0, 0x10, 4, 4), + PIN_FIELD_BASE(138, 138, 1, 0x01d0, 0x10, 4, 4), + PIN_FIELD_BASE(139, 139, 1, 0x01d0, 0x10, 4, 4), + PIN_FIELD_BASE(140, 140, 1, 0x01d0, 0x10, 4, 4), + PIN_FIELD_BASE(141, 141, 1, 0x01d0, 0x10, 8, 4), + PIN_FIELD_BASE(142, 142, 1, 0x01d0, 0x10, 8, 4), + PIN_FIELD_BASE(143, 143, 1, 0x01a0, 0x10, 4, 4), + PIN_FIELD_BASE(144, 144, 1, 0x01a0, 0x10, 8, 4), + PIN_FIELD_BASE(145, 145, 1, 0x01a0, 0x10, 12, 4), + PIN_FIELD_BASE(146, 146, 1, 0x01a0, 0x10, 16, 4), + PIN_FIELD_BASE(147, 147, 1, 0x01d0, 0x10, 8, 4), + PIN_FIELD_BASE(148, 148, 1, 0x01d0, 0x10, 8, 4), + PIN_FIELD_BASE(149, 149, 1, 0x01c0, 0x10, 4, 4), + PIN_FIELD_BASE(150, 150, 1, 0x01c0, 0x10, 4, 4), + PIN_FIELD_BASE(151, 151, 2, 0x0190, 0x10, 4, 4), + PIN_FIELD_BASE(152, 152, 2, 0x0190, 0x10, 0, 4), + PIN_FIELD_BASE(153, 153, 2, 0x0180, 0x10, 28, 4), + PIN_FIELD_BASE(154, 154, 2, 0x0180, 0x10, 24, 4), + PIN_FIELD_BASE(155, 155, 2, 0x0190, 0x10, 12, 4), + PIN_FIELD_BASE(156, 156, 2, 0x0180, 0x10, 4, 4), + PIN_FIELD_BASE(157, 157, 2, 0x0180, 0x10, 0, 4), + PIN_FIELD_BASE(158, 158, 2, 0x0180, 0x10, 20, 4), + PIN_FIELD_BASE(159, 159, 2, 0x0180, 0x10, 16, 4), + PIN_FIELD_BASE(160, 160, 2, 0x0180, 0x10, 12, 4), + PIN_FIELD_BASE(161, 161, 2, 0x0180, 0x10, 8, 4), + PIN_FIELD_BASE(162, 162, 2, 0x0190, 0x10, 8, 4), + PIN_FIELD_BASE(163, 163, 4, 0x0100, 0x10, 16, 4), + PIN_FIELD_BASE(164, 164, 4, 0x0100, 0x10, 12, 4), + PIN_FIELD_BASE(165, 165, 4, 0x0100, 0x10, 20, 4), + PIN_FIELD_BASE(166, 166, 4, 0x0100, 0x10, 24, 4), + PIN_FIELD_BASE(167, 167, 4, 0x0100, 0x10, 28, 4), + PIN_FIELD_BASE(168, 168, 4, 0x0110, 0x10, 0, 4), + PIN_FIELD_BASE(169, 169, 3, 0x00f0, 0x10, 4, 4), + PIN_FIELD_BASE(170, 170, 3, 0x00f0, 0x10, 0, 4), + PIN_FIELD_BASE(171, 171, 3, 0x00f0, 0x10, 8, 4), + PIN_FIELD_BASE(172, 172, 3, 0x00f0, 0x10, 12, 4), + PIN_FIELD_BASE(173, 173, 3, 0x00f0, 0x10, 16, 4), + PIN_FIELD_BASE(174, 174, 3, 0x00f0, 0x10, 20, 4), + PIN_FIELD_BASE(175, 175, 2, 0x01b0, 0x10, 16, 4), + PIN_FIELD_BASE(176, 176, 2, 0x01b0, 0x10, 16, 4), +}; + +static const struct mtk_pin_field_calc mt8188_pin_rdsel_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x0130, 0x10, 18, 2), + PIN_FIELD_BASE(1, 1, 1, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(2, 2, 1, 0x0130, 0x10, 22, 2), + PIN_FIELD_BASE(3, 3, 1, 0x0130, 0x10, 24, 2), + PIN_FIELD_BASE(4, 4, 1, 0x0140, 0x10, 14, 2), + PIN_FIELD_BASE(5, 5, 1, 0x0140, 0x10, 16, 2), + PIN_FIELD_BASE(6, 6, 1, 0x0140, 0x10, 16, 2), + PIN_FIELD_BASE(7, 7, 1, 0x0130, 0x10, 26, 2), + PIN_FIELD_BASE(8, 8, 1, 0x0130, 0x10, 28, 2), + PIN_FIELD_BASE(9, 9, 1, 0x0130, 0x10, 30, 2), + PIN_FIELD_BASE(10, 10, 1, 0x0140, 0x10, 0, 2), + PIN_FIELD_BASE(11, 11, 1, 0x0140, 0x10, 16, 2), + PIN_FIELD_BASE(12, 12, 2, 0x0130, 0x10, 12, 2), + PIN_FIELD_BASE(13, 13, 2, 0x0130, 0x10, 14, 2), + PIN_FIELD_BASE(14, 14, 2, 0x0130, 0x10, 16, 2), + PIN_FIELD_BASE(15, 15, 2, 0x0130, 0x10, 18, 2), + PIN_FIELD_BASE(16, 16, 3, 0x00b0, 0x10, 14, 2), + PIN_FIELD_BASE(17, 17, 3, 0x00b0, 0x10, 14, 2), + PIN_FIELD_BASE(18, 18, 4, 0x00c0, 0x10, 12, 2), + PIN_FIELD_BASE(19, 19, 4, 0x00c0, 0x10, 12, 2), + PIN_FIELD_BASE(20, 20, 4, 0x00c0, 0x10, 12, 2), + PIN_FIELD_BASE(21, 21, 4, 0x00c0, 0x10, 12, 2), + PIN_FIELD_BASE(22, 22, 4, 0x00b0, 0x10, 0, 2), + PIN_FIELD_BASE(23, 23, 4, 0x00b0, 0x10, 2, 2), + PIN_FIELD_BASE(24, 24, 4, 0x00b0, 0x10, 4, 2), + PIN_FIELD_BASE(25, 25, 1, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(26, 26, 1, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(27, 27, 1, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(28, 28, 1, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(29, 29, 1, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(30, 30, 1, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(31, 31, 1, 0x0140, 0x10, 16, 2), + PIN_FIELD_BASE(32, 32, 1, 0x0140, 0x10, 18, 2), + PIN_FIELD_BASE(33, 33, 1, 0x0140, 0x10, 18, 2), + PIN_FIELD_BASE(34, 34, 1, 0x0140, 0x10, 20, 2), + PIN_FIELD_BASE(35, 35, 1, 0x0140, 0x10, 18, 2), + PIN_FIELD_BASE(36, 36, 1, 0x0140, 0x10, 18, 2), + PIN_FIELD_BASE(37, 37, 1, 0x0140, 0x10, 20, 2), + PIN_FIELD_BASE(38, 38, 1, 0x0140, 0x10, 20, 2), + PIN_FIELD_BASE(39, 39, 1, 0x0140, 0x10, 20, 2), + PIN_FIELD_BASE(40, 40, 1, 0x0140, 0x10, 22, 2), + PIN_FIELD_BASE(41, 41, 1, 0x0140, 0x10, 22, 2), + PIN_FIELD_BASE(42, 42, 2, 0x0130, 0x10, 30, 2), + PIN_FIELD_BASE(43, 43, 2, 0x0140, 0x10, 0, 2), + PIN_FIELD_BASE(44, 44, 2, 0x0130, 0x10, 30, 2), + PIN_FIELD_BASE(45, 45, 2, 0x0140, 0x10, 0, 2), + PIN_FIELD_BASE(46, 46, 3, 0x00b0, 0x10, 14, 2), + PIN_FIELD_BASE(47, 47, 1, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(48, 48, 1, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(49, 49, 1, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(50, 50, 3, 0x00b0, 0x10, 14, 2), + PIN_FIELD_BASE(51, 51, 3, 0x00b0, 0x10, 16, 2), + PIN_FIELD_BASE(52, 52, 3, 0x00b0, 0x10, 16, 2), + PIN_FIELD_BASE(53, 53, 3, 0x00b0, 0x10, 16, 2), + PIN_FIELD_BASE(54, 54, 3, 0x00b0, 0x10, 16, 2), + PIN_FIELD_BASE(55, 55, 1, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(56, 56, 1, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(57, 57, 2, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(58, 58, 2, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(59, 59, 1, 0x0140, 0x10, 14, 2), + PIN_FIELD_BASE(60, 60, 1, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(61, 61, 1, 0x0140, 0x10, 14, 2), + PIN_FIELD_BASE(62, 62, 1, 0x0140, 0x10, 14, 2), + PIN_FIELD_BASE(63, 63, 2, 0x0140, 0x10, 0, 2), + PIN_FIELD_BASE(64, 64, 2, 0x0140, 0x10, 0, 2), + PIN_FIELD_BASE(65, 65, 4, 0x00c0, 0x10, 14, 2), + PIN_FIELD_BASE(66, 66, 4, 0x00c0, 0x10, 14, 2), + PIN_FIELD_BASE(67, 67, 4, 0x00c0, 0x10, 14, 2), + PIN_FIELD_BASE(68, 68, 4, 0x00c0, 0x10, 14, 2), + PIN_FIELD_BASE(69, 69, 1, 0x0150, 0x10, 14, 2), + PIN_FIELD_BASE(70, 70, 1, 0x0150, 0x10, 12, 2), + PIN_FIELD_BASE(71, 71, 1, 0x0150, 0x10, 22, 2), + PIN_FIELD_BASE(72, 72, 1, 0x0150, 0x10, 20, 2), + PIN_FIELD_BASE(73, 73, 1, 0x0150, 0x10, 16, 2), + PIN_FIELD_BASE(74, 74, 1, 0x0150, 0x10, 18, 2), + PIN_FIELD_BASE(75, 75, 1, 0x0150, 0x10, 26, 2), + PIN_FIELD_BASE(76, 76, 1, 0x0150, 0x10, 24, 2), + PIN_FIELD_BASE(77, 77, 1, 0x0150, 0x10, 30, 2), + PIN_FIELD_BASE(78, 78, 1, 0x0150, 0x10, 28, 2), + PIN_FIELD_BASE(79, 79, 4, 0x00c0, 0x10, 18, 2), + PIN_FIELD_BASE(80, 80, 4, 0x00c0, 0x10, 16, 2), + PIN_FIELD_BASE(81, 81, 4, 0x00c0, 0x10, 22, 2), + PIN_FIELD_BASE(82, 82, 4, 0x00c0, 0x10, 20, 2), + PIN_FIELD_BASE(83, 83, 2, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(84, 84, 2, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(85, 85, 2, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(86, 86, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(87, 87, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(88, 88, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(89, 89, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(90, 90, 2, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(91, 91, 2, 0x0140, 0x10, 12, 2), + PIN_FIELD_BASE(92, 92, 2, 0x0130, 0x10, 22, 2), + PIN_FIELD_BASE(93, 93, 2, 0x0130, 0x10, 22, 2), + PIN_FIELD_BASE(94, 94, 2, 0x0130, 0x10, 22, 2), + PIN_FIELD_BASE(95, 95, 2, 0x0130, 0x10, 22, 2), + PIN_FIELD_BASE(96, 96, 2, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(97, 97, 2, 0x0140, 0x10, 4, 2), + PIN_FIELD_BASE(98, 98, 2, 0x0140, 0x10, 6, 2), + PIN_FIELD_BASE(99, 99, 2, 0x0140, 0x10, 2, 2), + PIN_FIELD_BASE(100, 100, 2, 0x0140, 0x10, 16, 2), + PIN_FIELD_BASE(101, 101, 2, 0x0140, 0x10, 4, 2), + PIN_FIELD_BASE(102, 102, 2, 0x0140, 0x10, 4, 2), + PIN_FIELD_BASE(103, 103, 2, 0x0140, 0x10, 4, 2), + PIN_FIELD_BASE(104, 104, 2, 0x0140, 0x10, 6, 2), + PIN_FIELD_BASE(105, 105, 2, 0x0140, 0x10, 6, 2), + PIN_FIELD_BASE(106, 106, 2, 0x0140, 0x10, 6, 2), + PIN_FIELD_BASE(107, 107, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(108, 108, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(109, 109, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(110, 110, 2, 0x0130, 0x10, 20, 2), + PIN_FIELD_BASE(111, 111, 2, 0x0130, 0x10, 24, 2), + PIN_FIELD_BASE(112, 112, 2, 0x0130, 0x10, 24, 2), + PIN_FIELD_BASE(113, 113, 2, 0x0130, 0x10, 24, 2), + PIN_FIELD_BASE(114, 114, 2, 0x0130, 0x10, 24, 2), + PIN_FIELD_BASE(115, 115, 2, 0x0130, 0x10, 28, 2), + PIN_FIELD_BASE(116, 116, 2, 0x0130, 0x10, 28, 2), + PIN_FIELD_BASE(117, 117, 2, 0x0130, 0x10, 28, 2), + PIN_FIELD_BASE(118, 118, 2, 0x0130, 0x10, 28, 2), + PIN_FIELD_BASE(119, 119, 2, 0x0130, 0x10, 30, 2), + PIN_FIELD_BASE(120, 120, 2, 0x0130, 0x10, 30, 2), + PIN_FIELD_BASE(121, 121, 3, 0x00b0, 0x10, 6, 2), + PIN_FIELD_BASE(122, 122, 3, 0x00b0, 0x10, 12, 2), + PIN_FIELD_BASE(123, 123, 3, 0x00b0, 0x10, 10, 2), + PIN_FIELD_BASE(124, 124, 3, 0x00b0, 0x10, 8, 2), + PIN_FIELD_BASE(125, 125, 2, 0x0140, 0x10, 8, 2), + PIN_FIELD_BASE(126, 126, 2, 0x0140, 0x10, 8, 2), + PIN_FIELD_BASE(127, 127, 2, 0x0140, 0x10, 8, 2), + PIN_FIELD_BASE(128, 128, 2, 0x0140, 0x10, 8, 2), + PIN_FIELD_BASE(129, 129, 2, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(130, 130, 2, 0x0140, 0x10, 10, 2), + PIN_FIELD_BASE(131, 131, 1, 0x0120, 0x10, 0, 6), + PIN_FIELD_BASE(132, 132, 1, 0x0130, 0x10, 0, 6), + PIN_FIELD_BASE(133, 133, 1, 0x0130, 0x10, 6, 6), + PIN_FIELD_BASE(134, 134, 1, 0x0130, 0x10, 12, 6), + PIN_FIELD_BASE(135, 135, 1, 0x0140, 0x10, 24, 6), + PIN_FIELD_BASE(136, 136, 1, 0x0140, 0x10, 24, 6), + PIN_FIELD_BASE(137, 137, 1, 0x0150, 0x10, 0, 6), + PIN_FIELD_BASE(138, 138, 1, 0x0150, 0x10, 0, 6), + PIN_FIELD_BASE(139, 139, 1, 0x0150, 0x10, 0, 6), + PIN_FIELD_BASE(140, 140, 1, 0x0150, 0x10, 0, 6), + PIN_FIELD_BASE(141, 141, 1, 0x0150, 0x10, 6, 6), + PIN_FIELD_BASE(142, 142, 1, 0x0150, 0x10, 6, 6), + PIN_FIELD_BASE(143, 143, 1, 0x0120, 0x10, 6, 6), + PIN_FIELD_BASE(144, 144, 1, 0x0120, 0x10, 12, 6), + PIN_FIELD_BASE(145, 145, 1, 0x0120, 0x10, 18, 6), + PIN_FIELD_BASE(146, 146, 1, 0x0120, 0x10, 24, 6), + PIN_FIELD_BASE(147, 147, 1, 0x0150, 0x10, 6, 6), + PIN_FIELD_BASE(148, 148, 1, 0x0150, 0x10, 6, 6), + PIN_FIELD_BASE(149, 149, 1, 0x0140, 0x10, 4, 6), + PIN_FIELD_BASE(150, 150, 1, 0x0140, 0x10, 4, 6), + PIN_FIELD_BASE(151, 151, 2, 0x0120, 0x10, 24, 6), + PIN_FIELD_BASE(152, 152, 2, 0x0120, 0x10, 18, 6), + PIN_FIELD_BASE(153, 153, 2, 0x0120, 0x10, 12, 6), + PIN_FIELD_BASE(154, 154, 2, 0x0120, 0x10, 6, 6), + PIN_FIELD_BASE(155, 155, 2, 0x0130, 0x10, 6, 6), + PIN_FIELD_BASE(156, 156, 2, 0x0110, 0x10, 6, 6), + PIN_FIELD_BASE(157, 157, 2, 0x0110, 0x10, 0, 6), + PIN_FIELD_BASE(158, 158, 2, 0x0120, 0x10, 0, 6), + PIN_FIELD_BASE(159, 159, 2, 0x0110, 0x10, 24, 6), + PIN_FIELD_BASE(160, 160, 2, 0x0110, 0x10, 18, 6), + PIN_FIELD_BASE(161, 161, 2, 0x0110, 0x10, 12, 6), + PIN_FIELD_BASE(162, 162, 2, 0x0130, 0x10, 0, 6), + PIN_FIELD_BASE(163, 163, 4, 0x00b0, 0x10, 12, 6), + PIN_FIELD_BASE(164, 164, 4, 0x00b0, 0x10, 6, 6), + PIN_FIELD_BASE(165, 165, 4, 0x00b0, 0x10, 18, 6), + PIN_FIELD_BASE(166, 166, 4, 0x00b0, 0x10, 24, 6), + PIN_FIELD_BASE(167, 167, 4, 0x00c0, 0x10, 0, 6), + PIN_FIELD_BASE(168, 168, 4, 0x00c0, 0x10, 6, 6), + PIN_FIELD_BASE(169, 169, 3, 0x00a0, 0x10, 6, 6), + PIN_FIELD_BASE(170, 170, 3, 0x00a0, 0x10, 0, 6), + PIN_FIELD_BASE(171, 171, 3, 0x00a0, 0x10, 12, 6), + PIN_FIELD_BASE(172, 172, 3, 0x00a0, 0x10, 18, 6), + PIN_FIELD_BASE(173, 173, 3, 0x00a0, 0x10, 24, 6), + PIN_FIELD_BASE(174, 174, 3, 0x00b0, 0x10, 0, 6), + PIN_FIELD_BASE(175, 175, 2, 0x0140, 0x10, 14, 2), + PIN_FIELD_BASE(176, 176, 2, 0x0140, 0x10, 14, 2), +}; + +static const struct mtk_pin_field_calc mt8188_pin_pupd_range[] = { + PIN_FIELD_BASE(42, 42, 2, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(131, 131, 1, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(132, 132, 1, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(133, 133, 1, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(134, 134, 1, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(135, 135, 1, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(136, 136, 1, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(137, 137, 1, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(138, 138, 1, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(139, 139, 1, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(140, 140, 1, 0x00d0, 0x10, 16, 1), + PIN_FIELD_BASE(141, 141, 1, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 1, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 1, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 1, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 1, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 1, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 1, 0x00d0, 0x10, 18, 1), + PIN_FIELD_BASE(148, 148, 1, 0x00d0, 0x10, 19, 1), + PIN_FIELD_BASE(149, 149, 1, 0x00d0, 0x10, 17, 1), + PIN_FIELD_BASE(150, 150, 1, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(151, 151, 2, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(152, 152, 2, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(153, 153, 2, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(154, 154, 2, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(155, 155, 2, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(156, 156, 2, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(157, 157, 2, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(158, 158, 2, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(159, 159, 2, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(160, 160, 2, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(161, 161, 2, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(162, 162, 2, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(164, 164, 4, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(165, 165, 4, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 4, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(167, 167, 4, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(168, 168, 4, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(169, 169, 3, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(170, 170, 3, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(171, 171, 3, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(172, 172, 3, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(173, 173, 3, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(174, 174, 3, 0x0060, 0x10, 5, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_r0_range[] = { + PIN_FIELD_BASE(42, 42, 2, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(43, 43, 2, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(44, 44, 2, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(45, 45, 2, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(131, 131, 1, 0x0100, 0x10, 1, 1), + PIN_FIELD_BASE(132, 132, 1, 0x0100, 0x10, 2, 1), + PIN_FIELD_BASE(133, 133, 1, 0x0100, 0x10, 9, 1), + PIN_FIELD_BASE(134, 134, 1, 0x0100, 0x10, 10, 1), + PIN_FIELD_BASE(135, 135, 1, 0x0100, 0x10, 11, 1), + PIN_FIELD_BASE(136, 136, 1, 0x0100, 0x10, 12, 1), + PIN_FIELD_BASE(137, 137, 1, 0x0100, 0x10, 13, 1), + PIN_FIELD_BASE(138, 138, 1, 0x0100, 0x10, 14, 1), + PIN_FIELD_BASE(139, 139, 1, 0x0100, 0x10, 15, 1), + PIN_FIELD_BASE(140, 140, 1, 0x0100, 0x10, 16, 1), + PIN_FIELD_BASE(141, 141, 1, 0x0100, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 1, 0x0100, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 1, 0x0100, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 1, 0x0100, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 1, 0x0100, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 1, 0x0100, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 1, 0x0100, 0x10, 18, 1), + PIN_FIELD_BASE(148, 148, 1, 0x0100, 0x10, 19, 1), + PIN_FIELD_BASE(149, 149, 1, 0x0100, 0x10, 17, 1), + PIN_FIELD_BASE(150, 150, 1, 0x0100, 0x10, 0, 1), + PIN_FIELD_BASE(151, 151, 2, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(152, 152, 2, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(153, 153, 2, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(154, 154, 2, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(155, 155, 2, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(156, 156, 2, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(157, 157, 2, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(158, 158, 2, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(159, 159, 2, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(160, 160, 2, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(161, 161, 2, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(162, 162, 2, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(163, 163, 4, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(164, 164, 4, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(165, 165, 4, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 4, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(167, 167, 4, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(168, 168, 4, 0x0090, 0x10, 5, 1), + PIN_FIELD_BASE(169, 169, 3, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(170, 170, 3, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(171, 171, 3, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(172, 172, 3, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(173, 173, 3, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(174, 174, 3, 0x0080, 0x10, 5, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_r1_range[] = { + PIN_FIELD_BASE(42, 42, 2, 0x0100, 0x10, 12, 1), + PIN_FIELD_BASE(43, 43, 2, 0x0100, 0x10, 13, 1), + PIN_FIELD_BASE(44, 44, 2, 0x0100, 0x10, 14, 1), + PIN_FIELD_BASE(45, 45, 2, 0x0100, 0x10, 15, 1), + PIN_FIELD_BASE(131, 131, 1, 0x0110, 0x10, 1, 1), + PIN_FIELD_BASE(132, 132, 1, 0x0110, 0x10, 2, 1), + PIN_FIELD_BASE(133, 133, 1, 0x0110, 0x10, 9, 1), + PIN_FIELD_BASE(134, 134, 1, 0x0110, 0x10, 10, 1), + PIN_FIELD_BASE(135, 135, 1, 0x0110, 0x10, 11, 1), + PIN_FIELD_BASE(136, 136, 1, 0x0110, 0x10, 12, 1), + PIN_FIELD_BASE(137, 137, 1, 0x0110, 0x10, 13, 1), + PIN_FIELD_BASE(138, 138, 1, 0x0110, 0x10, 14, 1), + PIN_FIELD_BASE(139, 139, 1, 0x0110, 0x10, 15, 1), + PIN_FIELD_BASE(140, 140, 1, 0x0110, 0x10, 16, 1), + PIN_FIELD_BASE(141, 141, 1, 0x0110, 0x10, 3, 1), + PIN_FIELD_BASE(142, 142, 1, 0x0110, 0x10, 4, 1), + PIN_FIELD_BASE(143, 143, 1, 0x0110, 0x10, 5, 1), + PIN_FIELD_BASE(144, 144, 1, 0x0110, 0x10, 6, 1), + PIN_FIELD_BASE(145, 145, 1, 0x0110, 0x10, 7, 1), + PIN_FIELD_BASE(146, 146, 1, 0x0110, 0x10, 8, 1), + PIN_FIELD_BASE(147, 147, 1, 0x0110, 0x10, 18, 1), + PIN_FIELD_BASE(148, 148, 1, 0x0110, 0x10, 19, 1), + PIN_FIELD_BASE(149, 149, 1, 0x0110, 0x10, 17, 1), + PIN_FIELD_BASE(150, 150, 1, 0x0110, 0x10, 0, 1), + PIN_FIELD_BASE(151, 151, 2, 0x0100, 0x10, 9, 1), + PIN_FIELD_BASE(152, 152, 2, 0x0100, 0x10, 8, 1), + PIN_FIELD_BASE(153, 153, 2, 0x0100, 0x10, 7, 1), + PIN_FIELD_BASE(154, 154, 2, 0x0100, 0x10, 6, 1), + PIN_FIELD_BASE(155, 155, 2, 0x0100, 0x10, 11, 1), + PIN_FIELD_BASE(156, 156, 2, 0x0100, 0x10, 1, 1), + PIN_FIELD_BASE(157, 157, 2, 0x0100, 0x10, 0, 1), + PIN_FIELD_BASE(158, 158, 2, 0x0100, 0x10, 5, 1), + PIN_FIELD_BASE(159, 159, 2, 0x0100, 0x10, 4, 1), + PIN_FIELD_BASE(160, 160, 2, 0x0100, 0x10, 3, 1), + PIN_FIELD_BASE(161, 161, 2, 0x0100, 0x10, 2, 1), + PIN_FIELD_BASE(162, 162, 2, 0x0100, 0x10, 10, 1), + PIN_FIELD_BASE(163, 163, 4, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(164, 164, 4, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(165, 165, 4, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(166, 166, 4, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(167, 167, 4, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(168, 168, 4, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(169, 169, 3, 0x0090, 0x10, 1, 1), + PIN_FIELD_BASE(170, 170, 3, 0x0090, 0x10, 0, 1), + PIN_FIELD_BASE(171, 171, 3, 0x0090, 0x10, 2, 1), + PIN_FIELD_BASE(172, 172, 3, 0x0090, 0x10, 3, 1), + PIN_FIELD_BASE(173, 173, 3, 0x0090, 0x10, 4, 1), + PIN_FIELD_BASE(174, 174, 3, 0x0090, 0x10, 5, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_pu_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(1, 1, 1, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(2, 2, 1, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(3, 3, 1, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(4, 4, 1, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(5, 5, 1, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(6, 6, 1, 0x00e0, 0x10, 12, 1), + PIN_FIELD_BASE(7, 7, 1, 0x00e0, 0x10, 13, 1), + PIN_FIELD_BASE(8, 8, 1, 0x00e0, 0x10, 14, 1), + PIN_FIELD_BASE(9, 9, 1, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(10, 10, 1, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(11, 11, 1, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(12, 12, 2, 0x00d0, 0x10, 12, 1), + PIN_FIELD_BASE(13, 13, 2, 0x00d0, 0x10, 13, 1), + PIN_FIELD_BASE(14, 14, 2, 0x00d0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 2, 0x00d0, 0x10, 15, 1), + PIN_FIELD_BASE(16, 16, 3, 0x0070, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 3, 0x0070, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 4, 0x0080, 0x10, 3, 1), + PIN_FIELD_BASE(19, 19, 4, 0x0080, 0x10, 5, 1), + PIN_FIELD_BASE(20, 20, 4, 0x0080, 0x10, 4, 1), + PIN_FIELD_BASE(21, 21, 4, 0x0080, 0x10, 6, 1), + PIN_FIELD_BASE(22, 22, 4, 0x0080, 0x10, 0, 1), + PIN_FIELD_BASE(23, 23, 4, 0x0080, 0x10, 1, 1), + PIN_FIELD_BASE(24, 24, 4, 0x0080, 0x10, 2, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00f0, 0x10, 11, 1), + PIN_FIELD_BASE(32, 32, 1, 0x00f0, 0x10, 10, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00f0, 0x10, 13, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00f0, 0x10, 12, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00f0, 0x10, 15, 1), + PIN_FIELD_BASE(36, 36, 1, 0x00f0, 0x10, 14, 1), + PIN_FIELD_BASE(37, 37, 1, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(38, 38, 1, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(40, 40, 1, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(41, 41, 1, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(46, 46, 3, 0x0070, 0x10, 0, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00e0, 0x10, 25, 1), + PIN_FIELD_BASE(48, 48, 1, 0x00e0, 0x10, 24, 1), + PIN_FIELD_BASE(49, 49, 1, 0x00e0, 0x10, 23, 1), + PIN_FIELD_BASE(50, 50, 3, 0x0070, 0x10, 5, 1), + PIN_FIELD_BASE(51, 51, 3, 0x0070, 0x10, 4, 1), + PIN_FIELD_BASE(52, 52, 3, 0x0070, 0x10, 3, 1), + PIN_FIELD_BASE(53, 53, 3, 0x0070, 0x10, 6, 1), + PIN_FIELD_BASE(54, 54, 3, 0x0070, 0x10, 7, 1), + PIN_FIELD_BASE(55, 55, 1, 0x00e0, 0x10, 26, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00e0, 0x10, 29, 1), + PIN_FIELD_BASE(57, 57, 2, 0x00e0, 0x10, 6, 1), + PIN_FIELD_BASE(58, 58, 2, 0x00e0, 0x10, 9, 1), + PIN_FIELD_BASE(59, 59, 1, 0x00e0, 0x10, 27, 1), + PIN_FIELD_BASE(60, 60, 1, 0x00e0, 0x10, 30, 1), + PIN_FIELD_BASE(61, 61, 1, 0x00e0, 0x10, 28, 1), + PIN_FIELD_BASE(62, 62, 1, 0x00e0, 0x10, 31, 1), + PIN_FIELD_BASE(63, 63, 2, 0x00e0, 0x10, 7, 1), + PIN_FIELD_BASE(64, 64, 2, 0x00e0, 0x10, 10, 1), + PIN_FIELD_BASE(65, 65, 4, 0x0080, 0x10, 7, 1), + PIN_FIELD_BASE(66, 66, 4, 0x0080, 0x10, 9, 1), + PIN_FIELD_BASE(67, 67, 4, 0x0080, 0x10, 8, 1), + PIN_FIELD_BASE(68, 68, 4, 0x0080, 0x10, 10, 1), + PIN_FIELD_BASE(69, 69, 1, 0x00f0, 0x10, 1, 1), + PIN_FIELD_BASE(70, 70, 1, 0x00f0, 0x10, 0, 1), + PIN_FIELD_BASE(71, 71, 1, 0x00f0, 0x10, 5, 1), + PIN_FIELD_BASE(72, 72, 1, 0x00f0, 0x10, 4, 1), + PIN_FIELD_BASE(73, 73, 1, 0x00f0, 0x10, 2, 1), + PIN_FIELD_BASE(74, 74, 1, 0x00f0, 0x10, 3, 1), + PIN_FIELD_BASE(75, 75, 1, 0x00f0, 0x10, 7, 1), + PIN_FIELD_BASE(76, 76, 1, 0x00f0, 0x10, 6, 1), + PIN_FIELD_BASE(77, 77, 1, 0x00f0, 0x10, 9, 1), + PIN_FIELD_BASE(78, 78, 1, 0x00f0, 0x10, 8, 1), + PIN_FIELD_BASE(79, 79, 4, 0x0080, 0x10, 12, 1), + PIN_FIELD_BASE(80, 80, 4, 0x0080, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 4, 0x0080, 0x10, 14, 1), + PIN_FIELD_BASE(82, 82, 4, 0x0080, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 2, 0x00e0, 0x10, 16, 1), + PIN_FIELD_BASE(84, 84, 2, 0x00e0, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 2, 0x00e0, 0x10, 17, 1), + PIN_FIELD_BASE(86, 86, 2, 0x00e0, 0x10, 19, 1), + PIN_FIELD_BASE(87, 87, 2, 0x00e0, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 2, 0x00e0, 0x10, 20, 1), + PIN_FIELD_BASE(89, 89, 2, 0x00e0, 0x10, 22, 1), + PIN_FIELD_BASE(90, 90, 2, 0x00e0, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 2, 0x00e0, 0x10, 23, 1), + PIN_FIELD_BASE(92, 92, 2, 0x00e0, 0x10, 3, 1), + PIN_FIELD_BASE(93, 93, 2, 0x00e0, 0x10, 2, 1), + PIN_FIELD_BASE(94, 94, 2, 0x00e0, 0x10, 5, 1), + PIN_FIELD_BASE(95, 95, 2, 0x00e0, 0x10, 4, 1), + PIN_FIELD_BASE(96, 96, 2, 0x00d0, 0x10, 31, 1), + PIN_FIELD_BASE(97, 97, 2, 0x00e0, 0x10, 0, 1), + PIN_FIELD_BASE(98, 98, 2, 0x00e0, 0x10, 8, 1), + PIN_FIELD_BASE(99, 99, 2, 0x00d0, 0x10, 30, 1), + PIN_FIELD_BASE(100, 100, 2, 0x00e0, 0x10, 1, 1), + PIN_FIELD_BASE(101, 101, 2, 0x00d0, 0x10, 0, 1), + PIN_FIELD_BASE(102, 102, 2, 0x00d0, 0x10, 5, 1), + PIN_FIELD_BASE(103, 103, 2, 0x00d0, 0x10, 3, 1), + PIN_FIELD_BASE(104, 104, 2, 0x00d0, 0x10, 4, 1), + PIN_FIELD_BASE(105, 105, 2, 0x00d0, 0x10, 1, 1), + PIN_FIELD_BASE(106, 106, 2, 0x00d0, 0x10, 2, 1), + PIN_FIELD_BASE(107, 107, 2, 0x00d0, 0x10, 21, 1), + PIN_FIELD_BASE(108, 108, 2, 0x00d0, 0x10, 16, 1), + PIN_FIELD_BASE(109, 109, 2, 0x00d0, 0x10, 22, 1), + PIN_FIELD_BASE(110, 110, 2, 0x00d0, 0x10, 17, 1), + PIN_FIELD_BASE(111, 111, 2, 0x00d0, 0x10, 18, 1), + PIN_FIELD_BASE(112, 112, 2, 0x00d0, 0x10, 19, 1), + PIN_FIELD_BASE(113, 113, 2, 0x00d0, 0x10, 20, 1), + PIN_FIELD_BASE(114, 114, 2, 0x00d0, 0x10, 28, 1), + PIN_FIELD_BASE(115, 115, 2, 0x00d0, 0x10, 23, 1), + PIN_FIELD_BASE(116, 116, 2, 0x00d0, 0x10, 29, 1), + PIN_FIELD_BASE(117, 117, 2, 0x00d0, 0x10, 24, 1), + PIN_FIELD_BASE(118, 118, 2, 0x00d0, 0x10, 25, 1), + PIN_FIELD_BASE(119, 119, 2, 0x00d0, 0x10, 26, 1), + PIN_FIELD_BASE(120, 120, 2, 0x00d0, 0x10, 27, 1), + PIN_FIELD_BASE(121, 121, 3, 0x0070, 0x10, 8, 1), + PIN_FIELD_BASE(122, 122, 3, 0x0070, 0x10, 11, 1), + PIN_FIELD_BASE(123, 123, 3, 0x0070, 0x10, 10, 1), + PIN_FIELD_BASE(124, 124, 3, 0x0070, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 2, 0x00d0, 0x10, 6, 1), + PIN_FIELD_BASE(126, 126, 2, 0x00d0, 0x10, 7, 1), + PIN_FIELD_BASE(127, 127, 2, 0x00d0, 0x10, 8, 1), + PIN_FIELD_BASE(128, 128, 2, 0x00d0, 0x10, 9, 1), + PIN_FIELD_BASE(129, 129, 2, 0x00d0, 0x10, 10, 1), + PIN_FIELD_BASE(130, 130, 2, 0x00d0, 0x10, 11, 1), + PIN_FIELD_BASE(175, 175, 2, 0x00e0, 0x10, 11, 1), + PIN_FIELD_BASE(176, 176, 2, 0x00e0, 0x10, 12, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_pd_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(1, 1, 1, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(2, 2, 1, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(3, 3, 1, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(4, 4, 1, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(5, 5, 1, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(6, 6, 1, 0x00b0, 0x10, 12, 1), + PIN_FIELD_BASE(7, 7, 1, 0x00b0, 0x10, 13, 1), + PIN_FIELD_BASE(8, 8, 1, 0x00b0, 0x10, 14, 1), + PIN_FIELD_BASE(9, 9, 1, 0x00b0, 0x10, 15, 1), + PIN_FIELD_BASE(10, 10, 1, 0x00b0, 0x10, 16, 1), + PIN_FIELD_BASE(11, 11, 1, 0x00b0, 0x10, 17, 1), + PIN_FIELD_BASE(12, 12, 2, 0x00a0, 0x10, 12, 1), + PIN_FIELD_BASE(13, 13, 2, 0x00a0, 0x10, 13, 1), + PIN_FIELD_BASE(14, 14, 2, 0x00a0, 0x10, 14, 1), + PIN_FIELD_BASE(15, 15, 2, 0x00a0, 0x10, 15, 1), + PIN_FIELD_BASE(16, 16, 3, 0x0050, 0x10, 1, 1), + PIN_FIELD_BASE(17, 17, 3, 0x0050, 0x10, 2, 1), + PIN_FIELD_BASE(18, 18, 4, 0x0060, 0x10, 3, 1), + PIN_FIELD_BASE(19, 19, 4, 0x0060, 0x10, 5, 1), + PIN_FIELD_BASE(20, 20, 4, 0x0060, 0x10, 4, 1), + PIN_FIELD_BASE(21, 21, 4, 0x0060, 0x10, 6, 1), + PIN_FIELD_BASE(22, 22, 4, 0x0060, 0x10, 0, 1), + PIN_FIELD_BASE(23, 23, 4, 0x0060, 0x10, 1, 1), + PIN_FIELD_BASE(24, 24, 4, 0x0060, 0x10, 2, 1), + PIN_FIELD_BASE(25, 25, 1, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(26, 26, 1, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(27, 27, 1, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(28, 28, 1, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(29, 29, 1, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(30, 30, 1, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(31, 31, 1, 0x00c0, 0x10, 11, 1), + PIN_FIELD_BASE(32, 32, 1, 0x00c0, 0x10, 10, 1), + PIN_FIELD_BASE(33, 33, 1, 0x00c0, 0x10, 13, 1), + PIN_FIELD_BASE(34, 34, 1, 0x00c0, 0x10, 12, 1), + PIN_FIELD_BASE(35, 35, 1, 0x00c0, 0x10, 15, 1), + PIN_FIELD_BASE(36, 36, 1, 0x00c0, 0x10, 14, 1), + PIN_FIELD_BASE(37, 37, 1, 0x00b0, 0x10, 21, 1), + PIN_FIELD_BASE(38, 38, 1, 0x00b0, 0x10, 18, 1), + PIN_FIELD_BASE(39, 39, 1, 0x00b0, 0x10, 19, 1), + PIN_FIELD_BASE(40, 40, 1, 0x00b0, 0x10, 20, 1), + PIN_FIELD_BASE(41, 41, 1, 0x00b0, 0x10, 22, 1), + PIN_FIELD_BASE(46, 46, 3, 0x0050, 0x10, 0, 1), + PIN_FIELD_BASE(47, 47, 1, 0x00b0, 0x10, 25, 1), + PIN_FIELD_BASE(48, 48, 1, 0x00b0, 0x10, 24, 1), + PIN_FIELD_BASE(49, 49, 1, 0x00b0, 0x10, 23, 1), + PIN_FIELD_BASE(50, 50, 3, 0x0050, 0x10, 5, 1), + PIN_FIELD_BASE(51, 51, 3, 0x0050, 0x10, 4, 1), + PIN_FIELD_BASE(52, 52, 3, 0x0050, 0x10, 3, 1), + PIN_FIELD_BASE(53, 53, 3, 0x0050, 0x10, 6, 1), + PIN_FIELD_BASE(54, 54, 3, 0x0050, 0x10, 7, 1), + PIN_FIELD_BASE(55, 55, 1, 0x00b0, 0x10, 26, 1), + PIN_FIELD_BASE(56, 56, 1, 0x00b0, 0x10, 29, 1), + PIN_FIELD_BASE(57, 57, 2, 0x00b0, 0x10, 6, 1), + PIN_FIELD_BASE(58, 58, 2, 0x00b0, 0x10, 9, 1), + PIN_FIELD_BASE(59, 59, 1, 0x00b0, 0x10, 27, 1), + PIN_FIELD_BASE(60, 60, 1, 0x00b0, 0x10, 30, 1), + PIN_FIELD_BASE(61, 61, 1, 0x00b0, 0x10, 28, 1), + PIN_FIELD_BASE(62, 62, 1, 0x00b0, 0x10, 31, 1), + PIN_FIELD_BASE(63, 63, 2, 0x00b0, 0x10, 7, 1), + PIN_FIELD_BASE(64, 64, 2, 0x00b0, 0x10, 10, 1), + PIN_FIELD_BASE(65, 65, 4, 0x0060, 0x10, 7, 1), + PIN_FIELD_BASE(66, 66, 4, 0x0060, 0x10, 9, 1), + PIN_FIELD_BASE(67, 67, 4, 0x0060, 0x10, 8, 1), + PIN_FIELD_BASE(68, 68, 4, 0x0060, 0x10, 10, 1), + PIN_FIELD_BASE(69, 69, 1, 0x00c0, 0x10, 1, 1), + PIN_FIELD_BASE(70, 70, 1, 0x00c0, 0x10, 0, 1), + PIN_FIELD_BASE(71, 71, 1, 0x00c0, 0x10, 5, 1), + PIN_FIELD_BASE(72, 72, 1, 0x00c0, 0x10, 4, 1), + PIN_FIELD_BASE(73, 73, 1, 0x00c0, 0x10, 2, 1), + PIN_FIELD_BASE(74, 74, 1, 0x00c0, 0x10, 3, 1), + PIN_FIELD_BASE(75, 75, 1, 0x00c0, 0x10, 7, 1), + PIN_FIELD_BASE(76, 76, 1, 0x00c0, 0x10, 6, 1), + PIN_FIELD_BASE(77, 77, 1, 0x00c0, 0x10, 9, 1), + PIN_FIELD_BASE(78, 78, 1, 0x00c0, 0x10, 8, 1), + PIN_FIELD_BASE(79, 79, 4, 0x0060, 0x10, 12, 1), + PIN_FIELD_BASE(80, 80, 4, 0x0060, 0x10, 11, 1), + PIN_FIELD_BASE(81, 81, 4, 0x0060, 0x10, 14, 1), + PIN_FIELD_BASE(82, 82, 4, 0x0060, 0x10, 13, 1), + PIN_FIELD_BASE(83, 83, 2, 0x00b0, 0x10, 16, 1), + PIN_FIELD_BASE(84, 84, 2, 0x00b0, 0x10, 15, 1), + PIN_FIELD_BASE(85, 85, 2, 0x00b0, 0x10, 17, 1), + PIN_FIELD_BASE(86, 86, 2, 0x00b0, 0x10, 19, 1), + PIN_FIELD_BASE(87, 87, 2, 0x00b0, 0x10, 18, 1), + PIN_FIELD_BASE(88, 88, 2, 0x00b0, 0x10, 20, 1), + PIN_FIELD_BASE(89, 89, 2, 0x00b0, 0x10, 22, 1), + PIN_FIELD_BASE(90, 90, 2, 0x00b0, 0x10, 21, 1), + PIN_FIELD_BASE(91, 91, 2, 0x00b0, 0x10, 23, 1), + PIN_FIELD_BASE(92, 92, 2, 0x00b0, 0x10, 3, 1), + PIN_FIELD_BASE(93, 93, 2, 0x00b0, 0x10, 2, 1), + PIN_FIELD_BASE(94, 94, 2, 0x00b0, 0x10, 5, 1), + PIN_FIELD_BASE(95, 95, 2, 0x00b0, 0x10, 4, 1), + PIN_FIELD_BASE(96, 96, 2, 0x00a0, 0x10, 31, 1), + PIN_FIELD_BASE(97, 97, 2, 0x00b0, 0x10, 0, 1), + PIN_FIELD_BASE(98, 98, 2, 0x00b0, 0x10, 8, 1), + PIN_FIELD_BASE(99, 99, 2, 0x00a0, 0x10, 30, 1), + PIN_FIELD_BASE(100, 100, 2, 0x00b0, 0x10, 1, 1), + PIN_FIELD_BASE(101, 101, 2, 0x00a0, 0x10, 0, 1), + PIN_FIELD_BASE(102, 102, 2, 0x00a0, 0x10, 5, 1), + PIN_FIELD_BASE(103, 103, 2, 0x00a0, 0x10, 3, 1), + PIN_FIELD_BASE(104, 104, 2, 0x00a0, 0x10, 4, 1), + PIN_FIELD_BASE(105, 105, 2, 0x00a0, 0x10, 1, 1), + PIN_FIELD_BASE(106, 106, 2, 0x00a0, 0x10, 2, 1), + PIN_FIELD_BASE(107, 107, 2, 0x00a0, 0x10, 21, 1), + PIN_FIELD_BASE(108, 108, 2, 0x00a0, 0x10, 16, 1), + PIN_FIELD_BASE(109, 109, 2, 0x00a0, 0x10, 22, 1), + PIN_FIELD_BASE(110, 110, 2, 0x00a0, 0x10, 17, 1), + PIN_FIELD_BASE(111, 111, 2, 0x00a0, 0x10, 18, 1), + PIN_FIELD_BASE(112, 112, 2, 0x00a0, 0x10, 19, 1), + PIN_FIELD_BASE(113, 113, 2, 0x00a0, 0x10, 20, 1), + PIN_FIELD_BASE(114, 114, 2, 0x00a0, 0x10, 28, 1), + PIN_FIELD_BASE(115, 115, 2, 0x00a0, 0x10, 23, 1), + PIN_FIELD_BASE(116, 116, 2, 0x00a0, 0x10, 29, 1), + PIN_FIELD_BASE(117, 117, 2, 0x00a0, 0x10, 24, 1), + PIN_FIELD_BASE(118, 118, 2, 0x00a0, 0x10, 25, 1), + PIN_FIELD_BASE(119, 119, 2, 0x00a0, 0x10, 26, 1), + PIN_FIELD_BASE(120, 120, 2, 0x00a0, 0x10, 27, 1), + PIN_FIELD_BASE(121, 121, 3, 0x0050, 0x10, 8, 1), + PIN_FIELD_BASE(122, 122, 3, 0x0050, 0x10, 11, 1), + PIN_FIELD_BASE(123, 123, 3, 0x0050, 0x10, 10, 1), + PIN_FIELD_BASE(124, 124, 3, 0x0050, 0x10, 9, 1), + PIN_FIELD_BASE(125, 125, 2, 0x00a0, 0x10, 6, 1), + PIN_FIELD_BASE(126, 126, 2, 0x00a0, 0x10, 7, 1), + PIN_FIELD_BASE(127, 127, 2, 0x00a0, 0x10, 8, 1), + PIN_FIELD_BASE(128, 128, 2, 0x00a0, 0x10, 9, 1), + PIN_FIELD_BASE(129, 129, 2, 0x00a0, 0x10, 10, 1), + PIN_FIELD_BASE(130, 130, 2, 0x00a0, 0x10, 11, 1), + PIN_FIELD_BASE(175, 175, 2, 0x00b0, 0x10, 11, 1), + PIN_FIELD_BASE(176, 176, 2, 0x00b0, 0x10, 12, 1), +}; + +static const struct mtk_pin_field_calc mt8188_pin_drv_range[] = { + PIN_FIELD_BASE(0, 0, 1, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(1, 1, 1, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(2, 2, 1, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(3, 3, 1, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(4, 4, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(5, 5, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(6, 6, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(7, 7, 1, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(8, 8, 1, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(9, 9, 1, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(10, 10, 1, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(11, 11, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(12, 12, 2, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(13, 13, 2, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(14, 14, 2, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(15, 15, 2, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(16, 16, 3, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(17, 17, 3, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(18, 18, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(19, 19, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(20, 20, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(21, 21, 4, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(22, 22, 4, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(23, 23, 4, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(24, 24, 4, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(25, 25, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(26, 26, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(27, 27, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(28, 28, 1, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(29, 29, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(30, 30, 1, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(31, 31, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(32, 32, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(33, 33, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(34, 34, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(35, 35, 1, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(36, 36, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(37, 37, 1, 0x0010, 0x10, 27, 3), + PIN_FIELD_BASE(38, 38, 1, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(39, 39, 1, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(40, 40, 1, 0x0010, 0x10, 24, 3), + PIN_FIELD_BASE(41, 41, 1, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(42, 42, 2, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(43, 43, 2, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(44, 44, 2, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(45, 45, 2, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(46, 46, 3, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(47, 47, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(48, 48, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(49, 49, 1, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(50, 50, 3, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(51, 51, 3, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(52, 52, 3, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(53, 53, 3, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(54, 54, 3, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(55, 55, 1, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(57, 57, 2, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(58, 58, 2, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(59, 59, 1, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(60, 60, 1, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(61, 61, 1, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(62, 62, 1, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(63, 63, 2, 0x0030, 0x10, 12, 3), + PIN_FIELD_BASE(64, 64, 2, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(65, 65, 4, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(66, 66, 4, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(67, 67, 4, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(68, 68, 4, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(69, 69, 1, 0x0030, 0x10, 18, 3), + PIN_FIELD_BASE(70, 70, 1, 0x0030, 0x10, 15, 3), + PIN_FIELD_BASE(71, 71, 1, 0x0040, 0x10, 0, 3), + PIN_FIELD_BASE(72, 72, 1, 0x0030, 0x10, 27, 3), + PIN_FIELD_BASE(73, 73, 1, 0x0030, 0x10, 21, 3), + PIN_FIELD_BASE(74, 74, 1, 0x0030, 0x10, 24, 3), + PIN_FIELD_BASE(75, 75, 1, 0x0040, 0x10, 6, 3), + PIN_FIELD_BASE(76, 76, 1, 0x0040, 0x10, 3, 3), + PIN_FIELD_BASE(77, 77, 1, 0x0040, 0x10, 12, 3), + PIN_FIELD_BASE(78, 78, 1, 0x0040, 0x10, 9, 3), + PIN_FIELD_BASE(79, 79, 4, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(80, 80, 4, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(81, 81, 4, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(82, 82, 4, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(83, 83, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(84, 84, 2, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(85, 85, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(86, 86, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(87, 87, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(88, 88, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(89, 89, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(90, 90, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(91, 91, 2, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(92, 92, 2, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(93, 93, 2, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(94, 94, 2, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(95, 95, 2, 0x0020, 0x10, 9, 3), + PIN_FIELD_BASE(96, 96, 2, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(97, 97, 2, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(98, 98, 2, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(99, 99, 2, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(100, 100, 2, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(101, 101, 2, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(102, 102, 2, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(103, 103, 2, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(104, 104, 2, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(105, 105, 2, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(106, 106, 2, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(107, 107, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(108, 108, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(109, 109, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(110, 110, 2, 0x0020, 0x10, 6, 3), + PIN_FIELD_BASE(111, 111, 2, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(112, 112, 2, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(113, 113, 2, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(114, 114, 2, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(115, 115, 2, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(116, 116, 2, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(117, 117, 2, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(118, 118, 2, 0x0020, 0x10, 12, 3), + PIN_FIELD_BASE(119, 119, 2, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(120, 120, 2, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(121, 121, 3, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(122, 122, 3, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(123, 123, 3, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(124, 124, 3, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(125, 125, 2, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(126, 126, 2, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(127, 127, 2, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(128, 128, 2, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(129, 129, 2, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(130, 130, 2, 0x0020, 0x10, 27, 3), + PIN_FIELD_BASE(131, 131, 1, 0x0000, 0x10, 0, 3), + PIN_FIELD_BASE(132, 132, 1, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(133, 133, 1, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(134, 134, 1, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(135, 135, 1, 0x0020, 0x10, 15, 3), + PIN_FIELD_BASE(136, 136, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(137, 137, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(138, 138, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(139, 139, 1, 0x0020, 0x10, 18, 3), + PIN_FIELD_BASE(140, 140, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(141, 141, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(142, 142, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(143, 143, 1, 0x0000, 0x10, 3, 3), + PIN_FIELD_BASE(144, 144, 1, 0x0000, 0x10, 6, 3), + PIN_FIELD_BASE(145, 145, 1, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(146, 146, 1, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(147, 147, 1, 0x0020, 0x10, 21, 3), + PIN_FIELD_BASE(148, 148, 1, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(149, 149, 1, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(150, 150, 1, 0x0020, 0x10, 24, 3), + PIN_FIELD_BASE(151, 151, 2, 0x0010, 0x10, 15, 3), + PIN_FIELD_BASE(152, 152, 2, 0x0010, 0x10, 12, 3), + PIN_FIELD_BASE(153, 153, 2, 0x0010, 0x10, 9, 3), + PIN_FIELD_BASE(154, 154, 2, 0x0010, 0x10, 6, 3), + PIN_FIELD_BASE(155, 155, 2, 0x0010, 0x10, 21, 3), + PIN_FIELD_BASE(156, 156, 2, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(157, 157, 2, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(158, 158, 2, 0x0010, 0x10, 3, 3), + PIN_FIELD_BASE(159, 159, 2, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(160, 160, 2, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(161, 161, 2, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(162, 162, 2, 0x0010, 0x10, 18, 3), + PIN_FIELD_BASE(163, 163, 4, 0x0000, 0x10, 12, 3), + PIN_FIELD_BASE(164, 164, 4, 0x0000, 0x10, 9, 3), + PIN_FIELD_BASE(165, 165, 4, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(166, 166, 4, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(167, 167, 4, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(168, 168, 4, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(169, 169, 3, 0x0000, 0x10, 18, 3), + PIN_FIELD_BASE(170, 170, 3, 0x0000, 0x10, 15, 3), + PIN_FIELD_BASE(171, 171, 3, 0x0000, 0x10, 21, 3), + PIN_FIELD_BASE(172, 172, 3, 0x0000, 0x10, 24, 3), + PIN_FIELD_BASE(173, 173, 3, 0x0000, 0x10, 27, 3), + PIN_FIELD_BASE(174, 174, 3, 0x0010, 0x10, 0, 3), + PIN_FIELD_BASE(175, 175, 2, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(176, 176, 2, 0x0030, 0x10, 3, 3), +}; + +static const struct mtk_pin_field_calc mt8188_pin_drv_adv_range[] = { + PIN_FIELD_BASE(53, 53, 3, 0x0020, 0x10, 0, 3), + PIN_FIELD_BASE(54, 54, 3, 0x0020, 0x10, 3, 3), + PIN_FIELD_BASE(55, 55, 1, 0x0060, 0x10, 0, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0060, 0x10, 9, 3), + PIN_FIELD_BASE(57, 57, 2, 0x0050, 0x10, 0, 3), + PIN_FIELD_BASE(58, 58, 2, 0x0050, 0x10, 6, 3), + PIN_FIELD_BASE(59, 59, 1, 0x0060, 0x10, 3, 3), + PIN_FIELD_BASE(60, 60, 1, 0x0060, 0x10, 12, 3), + PIN_FIELD_BASE(61, 61, 1, 0x0060, 0x10, 6, 3), + PIN_FIELD_BASE(62, 62, 1, 0x0060, 0x10, 15, 3), + PIN_FIELD_BASE(63, 63, 2, 0x0050, 0x10, 3, 3), + PIN_FIELD_BASE(64, 64, 2, 0x0050, 0x10, 9, 3), + PIN_FIELD_BASE(65, 65, 4, 0x0030, 0x10, 0, 3), + PIN_FIELD_BASE(66, 66, 4, 0x0030, 0x10, 6, 3), + PIN_FIELD_BASE(67, 67, 4, 0x0030, 0x10, 3, 3), + PIN_FIELD_BASE(68, 68, 4, 0x0030, 0x10, 9, 3), + PIN_FIELD_BASE(175, 175, 2, 0x0050, 0x10, 12, 3), + PIN_FIELD_BASE(176, 176, 2, 0x0050, 0x10, 15, 3), +}; + +static const struct mtk_pin_field_calc mt8188_pin_rsel_range[] = { + PIN_FIELD_BASE(53, 53, 3, 0x00c0, 0x10, 0, 3), + PIN_FIELD_BASE(54, 54, 3, 0x00c0, 0x10, 3, 3), + PIN_FIELD_BASE(55, 55, 1, 0x0160, 0x10, 0, 3), + PIN_FIELD_BASE(56, 56, 1, 0x0160, 0x10, 9, 3), + PIN_FIELD_BASE(57, 57, 2, 0x0150, 0x10, 0, 3), + PIN_FIELD_BASE(58, 58, 2, 0x0150, 0x10, 6, 3), + PIN_FIELD_BASE(59, 59, 1, 0x0160, 0x10, 3, 3), + PIN_FIELD_BASE(60, 60, 1, 0x0160, 0x10, 12, 3), + PIN_FIELD_BASE(61, 61, 1, 0x0160, 0x10, 6, 3), + PIN_FIELD_BASE(62, 62, 1, 0x0160, 0x10, 15, 3), + PIN_FIELD_BASE(63, 63, 2, 0x0150, 0x10, 3, 3), + PIN_FIELD_BASE(64, 64, 2, 0x0150, 0x10, 9, 3), + PIN_FIELD_BASE(65, 65, 4, 0x00d0, 0x10, 0, 3), + PIN_FIELD_BASE(66, 66, 4, 0x00d0, 0x10, 6, 3), + PIN_FIELD_BASE(67, 67, 4, 0x00d0, 0x10, 3, 3), + PIN_FIELD_BASE(68, 68, 4, 0x00d0, 0x10, 9, 3), + PIN_FIELD_BASE(175, 175, 2, 0x0150, 0x10, 12, 3), + PIN_FIELD_BASE(176, 176, 2, 0x0150, 0x10, 15, 3), +}; + +static const struct mtk_pin_rsel mt8188_pin_rsel_val_range[] = { + PIN_RSEL(53, 68, 0x0, 75000, 75000), + PIN_RSEL(53, 68, 0x1, 10000, 5000), + PIN_RSEL(53, 68, 0x2, 5000, 75000), + PIN_RSEL(53, 68, 0x3, 4000, 5000), + PIN_RSEL(53, 68, 0x4, 3000, 75000), + PIN_RSEL(53, 68, 0x5, 2000, 5000), + PIN_RSEL(53, 68, 0x6, 1500, 75000), + PIN_RSEL(53, 68, 0x7, 1000, 5000), + PIN_RSEL(175, 176, 0x0, 75000, 75000), + PIN_RSEL(175, 176, 0x1, 10000, 5000), + PIN_RSEL(175, 176, 0x2, 5000, 75000), + PIN_RSEL(175, 176, 0x3, 4000, 5000), + PIN_RSEL(175, 176, 0x4, 3000, 75000), + PIN_RSEL(175, 176, 0x5, 2000, 5000), + PIN_RSEL(175, 176, 0x6, 1500, 75000), + PIN_RSEL(175, 176, 0x7, 1000, 5000), +}; + +static const unsigned int mt8188_pull_type[] = { + MTK_PULL_PU_PD_TYPE, /*0*/ + MTK_PULL_PU_PD_TYPE, /*1*/ + MTK_PULL_PU_PD_TYPE, /*2*/ + MTK_PULL_PU_PD_TYPE, /*3*/ + MTK_PULL_PU_PD_TYPE, /*4*/ + MTK_PULL_PU_PD_TYPE, /*5*/ + MTK_PULL_PU_PD_TYPE, /*6*/ + MTK_PULL_PU_PD_TYPE, /*7*/ + MTK_PULL_PU_PD_TYPE, /*8*/ + MTK_PULL_PU_PD_TYPE, /*9*/ + MTK_PULL_PU_PD_TYPE, /*10*/ + MTK_PULL_PU_PD_TYPE, /*11*/ + MTK_PULL_PU_PD_TYPE, /*12*/ + MTK_PULL_PU_PD_TYPE, /*13*/ + MTK_PULL_PU_PD_TYPE, /*14*/ + MTK_PULL_PU_PD_TYPE, /*15*/ + MTK_PULL_PU_PD_TYPE, /*16*/ + MTK_PULL_PU_PD_TYPE, /*17*/ + MTK_PULL_PU_PD_TYPE, /*18*/ + MTK_PULL_PU_PD_TYPE, /*19*/ + MTK_PULL_PU_PD_TYPE, /*20*/ + MTK_PULL_PU_PD_TYPE, /*21*/ + MTK_PULL_PU_PD_TYPE, /*22*/ + MTK_PULL_PU_PD_TYPE, /*23*/ + MTK_PULL_PU_PD_TYPE, /*24*/ + MTK_PULL_PU_PD_TYPE, /*25*/ + MTK_PULL_PU_PD_TYPE, /*26*/ + MTK_PULL_PU_PD_TYPE, /*27*/ + MTK_PULL_PU_PD_TYPE, /*28*/ + MTK_PULL_PU_PD_TYPE, /*29*/ + MTK_PULL_PU_PD_TYPE, /*30*/ + MTK_PULL_PU_PD_TYPE, /*31*/ + MTK_PULL_PU_PD_TYPE, /*32*/ + MTK_PULL_PU_PD_TYPE, /*33*/ + MTK_PULL_PU_PD_TYPE, /*34*/ + MTK_PULL_PU_PD_TYPE, /*35*/ + MTK_PULL_PU_PD_TYPE, /*36*/ + MTK_PULL_PU_PD_TYPE, /*37*/ + MTK_PULL_PU_PD_TYPE, /*38*/ + MTK_PULL_PU_PD_TYPE, /*39*/ + MTK_PULL_PU_PD_TYPE, /*40*/ + MTK_PULL_PU_PD_TYPE, /*41*/ + MTK_PULL_PUPD_R1R0_TYPE, /*42*/ + MTK_PULL_PUPD_R1R0_TYPE, /*43*/ + MTK_PULL_PUPD_R1R0_TYPE, /*44*/ + MTK_PULL_PUPD_R1R0_TYPE, /*45*/ + MTK_PULL_PU_PD_TYPE, /*46*/ + MTK_PULL_PU_PD_TYPE, /*47*/ + MTK_PULL_PU_PD_TYPE, /*48*/ + MTK_PULL_PU_PD_TYPE, /*49*/ + MTK_PULL_PU_PD_TYPE, /*50*/ + MTK_PULL_PU_PD_TYPE, /*51*/ + MTK_PULL_PU_PD_TYPE, /*52*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*53*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*54*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*55*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*56*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*57*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*58*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*59*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*60*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*61*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*62*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*63*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*64*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*65*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*66*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*67*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*68*/ + MTK_PULL_PU_PD_TYPE, /*69*/ + MTK_PULL_PU_PD_TYPE, /*70*/ + MTK_PULL_PU_PD_TYPE, /*71*/ + MTK_PULL_PU_PD_TYPE, /*72*/ + MTK_PULL_PU_PD_TYPE, /*73*/ + MTK_PULL_PU_PD_TYPE, /*74*/ + MTK_PULL_PU_PD_TYPE, /*75*/ + MTK_PULL_PU_PD_TYPE, /*76*/ + MTK_PULL_PU_PD_TYPE, /*77*/ + MTK_PULL_PU_PD_TYPE, /*78*/ + MTK_PULL_PU_PD_TYPE, /*79*/ + MTK_PULL_PU_PD_TYPE, /*80*/ + MTK_PULL_PU_PD_TYPE, /*81*/ + MTK_PULL_PU_PD_TYPE, /*82*/ + MTK_PULL_PU_PD_TYPE, /*83*/ + MTK_PULL_PU_PD_TYPE, /*84*/ + MTK_PULL_PU_PD_TYPE, /*85*/ + MTK_PULL_PU_PD_TYPE, /*86*/ + MTK_PULL_PU_PD_TYPE, /*87*/ + MTK_PULL_PU_PD_TYPE, /*88*/ + MTK_PULL_PU_PD_TYPE, /*89*/ + MTK_PULL_PU_PD_TYPE, /*90*/ + MTK_PULL_PU_PD_TYPE, /*91*/ + MTK_PULL_PU_PD_TYPE, /*92*/ + MTK_PULL_PU_PD_TYPE, /*93*/ + MTK_PULL_PU_PD_TYPE, /*94*/ + MTK_PULL_PU_PD_TYPE, /*95*/ + MTK_PULL_PU_PD_TYPE, /*96*/ + MTK_PULL_PU_PD_TYPE, /*97*/ + MTK_PULL_PU_PD_TYPE, /*98*/ + MTK_PULL_PU_PD_TYPE, /*99*/ + MTK_PULL_PU_PD_TYPE, /*100*/ + MTK_PULL_PU_PD_TYPE, /*101*/ + MTK_PULL_PU_PD_TYPE, /*102*/ + MTK_PULL_PU_PD_TYPE, /*103*/ + MTK_PULL_PU_PD_TYPE, /*104*/ + MTK_PULL_PU_PD_TYPE, /*105*/ + MTK_PULL_PU_PD_TYPE, /*106*/ + MTK_PULL_PU_PD_TYPE, /*107*/ + MTK_PULL_PU_PD_TYPE, /*108*/ + MTK_PULL_PU_PD_TYPE, /*109*/ + MTK_PULL_PU_PD_TYPE, /*110*/ + MTK_PULL_PU_PD_TYPE, /*111*/ + MTK_PULL_PU_PD_TYPE, /*112*/ + MTK_PULL_PU_PD_TYPE, /*113*/ + MTK_PULL_PU_PD_TYPE, /*114*/ + MTK_PULL_PU_PD_TYPE, /*115*/ + MTK_PULL_PU_PD_TYPE, /*116*/ + MTK_PULL_PU_PD_TYPE, /*117*/ + MTK_PULL_PU_PD_TYPE, /*118*/ + MTK_PULL_PU_PD_TYPE, /*119*/ + MTK_PULL_PU_PD_TYPE, /*120*/ + MTK_PULL_PU_PD_TYPE, /*121*/ + MTK_PULL_PU_PD_TYPE, /*122*/ + MTK_PULL_PU_PD_TYPE, /*123*/ + MTK_PULL_PU_PD_TYPE, /*124*/ + MTK_PULL_PU_PD_TYPE, /*125*/ + MTK_PULL_PU_PD_TYPE, /*126*/ + MTK_PULL_PU_PD_TYPE, /*127*/ + MTK_PULL_PU_PD_TYPE, /*128*/ + MTK_PULL_PU_PD_TYPE, /*129*/ + MTK_PULL_PU_PD_TYPE, /*130*/ + MTK_PULL_PUPD_R1R0_TYPE, /*131*/ + MTK_PULL_PUPD_R1R0_TYPE, /*132*/ + MTK_PULL_PUPD_R1R0_TYPE, /*133*/ + MTK_PULL_PUPD_R1R0_TYPE, /*134*/ + MTK_PULL_PUPD_R1R0_TYPE, /*135*/ + MTK_PULL_PUPD_R1R0_TYPE, /*136*/ + MTK_PULL_PUPD_R1R0_TYPE, /*137*/ + MTK_PULL_PUPD_R1R0_TYPE, /*138*/ + MTK_PULL_PUPD_R1R0_TYPE, /*139*/ + MTK_PULL_PUPD_R1R0_TYPE, /*140*/ + MTK_PULL_PUPD_R1R0_TYPE, /*141*/ + MTK_PULL_PUPD_R1R0_TYPE, /*142*/ + MTK_PULL_PUPD_R1R0_TYPE, /*143*/ + MTK_PULL_PUPD_R1R0_TYPE, /*144*/ + MTK_PULL_PUPD_R1R0_TYPE, /*145*/ + MTK_PULL_PUPD_R1R0_TYPE, /*146*/ + MTK_PULL_PUPD_R1R0_TYPE, /*147*/ + MTK_PULL_PUPD_R1R0_TYPE, /*148*/ + MTK_PULL_PUPD_R1R0_TYPE, /*149*/ + MTK_PULL_PUPD_R1R0_TYPE, /*150*/ + MTK_PULL_PUPD_R1R0_TYPE, /*151*/ + MTK_PULL_PUPD_R1R0_TYPE, /*152*/ + MTK_PULL_PUPD_R1R0_TYPE, /*153*/ + MTK_PULL_PUPD_R1R0_TYPE, /*154*/ + MTK_PULL_PUPD_R1R0_TYPE, /*155*/ + MTK_PULL_PUPD_R1R0_TYPE, /*156*/ + MTK_PULL_PUPD_R1R0_TYPE, /*157*/ + MTK_PULL_PUPD_R1R0_TYPE, /*158*/ + MTK_PULL_PUPD_R1R0_TYPE, /*159*/ + MTK_PULL_PUPD_R1R0_TYPE, /*160*/ + MTK_PULL_PUPD_R1R0_TYPE, /*161*/ + MTK_PULL_PUPD_R1R0_TYPE, /*162*/ + MTK_PULL_PUPD_R1R0_TYPE, /*163*/ + MTK_PULL_PUPD_R1R0_TYPE, /*164*/ + MTK_PULL_PUPD_R1R0_TYPE, /*165*/ + MTK_PULL_PUPD_R1R0_TYPE, /*166*/ + MTK_PULL_PUPD_R1R0_TYPE, /*167*/ + MTK_PULL_PUPD_R1R0_TYPE, /*168*/ + MTK_PULL_PUPD_R1R0_TYPE, /*169*/ + MTK_PULL_PUPD_R1R0_TYPE, /*170*/ + MTK_PULL_PUPD_R1R0_TYPE, /*171*/ + MTK_PULL_PUPD_R1R0_TYPE, /*172*/ + MTK_PULL_PUPD_R1R0_TYPE, /*173*/ + MTK_PULL_PUPD_R1R0_TYPE, /*174*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*175*/ + MTK_PULL_PU_PD_RSEL_TYPE, /*176*/ +}; + +static const struct mtk_pin_reg_calc mt8188_reg_cals[PINCTRL_PIN_REG_MAX] = { + [PINCTRL_PIN_REG_MODE] = MTK_RANGE(mt8188_pin_mode_range), + [PINCTRL_PIN_REG_DIR] = MTK_RANGE(mt8188_pin_dir_range), + [PINCTRL_PIN_REG_DI] = MTK_RANGE(mt8188_pin_di_range), + [PINCTRL_PIN_REG_DO] = MTK_RANGE(mt8188_pin_do_range), + [PINCTRL_PIN_REG_SMT] = MTK_RANGE(mt8188_pin_smt_range), + [PINCTRL_PIN_REG_IES] = MTK_RANGE(mt8188_pin_ies_range), + [PINCTRL_PIN_REG_TDSEL] = MTK_RANGE(mt8188_pin_tdsel_range), + [PINCTRL_PIN_REG_RDSEL] = MTK_RANGE(mt8188_pin_rdsel_range), + [PINCTRL_PIN_REG_PUPD] = MTK_RANGE(mt8188_pin_pupd_range), + [PINCTRL_PIN_REG_R0] = MTK_RANGE(mt8188_pin_r0_range), + [PINCTRL_PIN_REG_R1] = MTK_RANGE(mt8188_pin_r1_range), + [PINCTRL_PIN_REG_PU] = MTK_RANGE(mt8188_pin_pu_range), + [PINCTRL_PIN_REG_PD] = MTK_RANGE(mt8188_pin_pd_range), + [PINCTRL_PIN_REG_DRV] = MTK_RANGE(mt8188_pin_drv_range), + [PINCTRL_PIN_REG_DRV_ADV] = MTK_RANGE(mt8188_pin_drv_adv_range), + [PINCTRL_PIN_REG_RSEL] = MTK_RANGE(mt8188_pin_rsel_range), +}; + +static const char * const mt8188_pinctrl_register_base_name[] = { + "iocfg0", "iocfg_rm", "iocfg_lt", "iocfg_lm", "iocfg_rt", +}; + +static const struct mtk_eint_hw mt8188_eint_hw = { + .port_mask = 0xf, + .ports = 7, + .ap_num = 225, + .db_cnt = 32, +}; + +static const struct mtk_pin_soc mt8188_data = { + .reg_cal = mt8188_reg_cals, + .pins = mtk_pins_mt8188, + .npins = ARRAY_SIZE(mtk_pins_mt8188), + .ngrps = ARRAY_SIZE(mtk_pins_mt8188), + .eint_hw = &mt8188_eint_hw, + .nfuncs = 8, + .gpio_m = 0, + .base_names = mt8188_pinctrl_register_base_name, + .nbase_names = ARRAY_SIZE(mt8188_pinctrl_register_base_name), + .pull_type = mt8188_pull_type, + .pin_rsel = mt8188_pin_rsel_val_range, + .npin_rsel = ARRAY_SIZE(mt8188_pin_rsel_val_range), + .bias_set_combo = mtk_pinconf_bias_set_combo, + .bias_get_combo = mtk_pinconf_bias_get_combo, + .drive_set = mtk_pinconf_drive_set_rev1, + .drive_get = mtk_pinconf_drive_get_rev1, + .adv_drive_set = mtk_pinconf_adv_drive_set_raw, + .adv_drive_get = mtk_pinconf_adv_drive_get_raw, +}; + +static const struct of_device_id mt8188_pinctrl_of_match[] = { + { .compatible = "mediatek,mt8188-pinctrl", .data = &mt8188_data }, + { } +}; + +static struct platform_driver mt8188_pinctrl_driver = { + .driver = { + .name = "mt8188-pinctrl", + .of_match_table = mt8188_pinctrl_of_match, + .pm = &mtk_paris_pinctrl_pm_ops + }, + .probe = mtk_paris_pinctrl_probe, +}; + +static int __init mt8188_pinctrl_init(void) +{ + return platform_driver_register(&mt8188_pinctrl_driver); +} + +arch_initcall(mt8188_pinctrl_init); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MediaTek MT8188 Pinctrl Driver"); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8188.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8188.h new file mode 100644 index 0000000000000000000000000000000000000000..a487323748e271ee62046d3027bd57a8e9ce3ee3 --- /dev/null +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8188.h @@ -0,0 +1,2259 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 MediaTek Inc. + * Author: Hui Liu + * + */ + +#ifndef __PINCTRL_MTK_MT8188_H +#define __PINCTRL_MTK_MT8188_H + +#include "pinctrl-paris.h" + +static const struct mtk_pin_desc mtk_pins_mt8188[] = { + MTK_PIN( + 0, "GPIO0", + MTK_EINT_FUNCTION(0, 0), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO0"), + MTK_FUNCTION(1, "B0_TP_GPIO0_AO"), + MTK_FUNCTION(2, "O_SPIM5_CSB"), + MTK_FUNCTION(3, "O_UTXD1"), + MTK_FUNCTION(4, "O_DMIC3_CLK"), + MTK_FUNCTION(5, "B0_I2SIN_MCK"), + MTK_FUNCTION(6, "O_I2SO2_MCK"), + MTK_FUNCTION(7, "B0_DBG_MON_A0") + ), + + MTK_PIN( + 1, "GPIO1", + MTK_EINT_FUNCTION(0, 1), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO1"), + MTK_FUNCTION(1, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(2, "O_SPIM5_CLK"), + MTK_FUNCTION(3, "I1_URXD1"), + MTK_FUNCTION(4, "I0_DMIC3_DAT"), + MTK_FUNCTION(5, "B0_I2SIN_BCK"), + MTK_FUNCTION(6, "B0_I2SO2_BCK"), + MTK_FUNCTION(7, "B0_DBG_MON_A1") + ), + + MTK_PIN( + 2, "GPIO2", + MTK_EINT_FUNCTION(0, 2), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO2"), + MTK_FUNCTION(1, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(2, "B0_SPIM5_MOSI"), + MTK_FUNCTION(3, "O_URTS1"), + MTK_FUNCTION(4, "I0_DMIC3_DAT_R"), + MTK_FUNCTION(5, "B0_I2SIN_WS"), + MTK_FUNCTION(6, "B0_I2SO2_WS"), + MTK_FUNCTION(7, "B0_DBG_MON_A2") + ), + + MTK_PIN( + 3, "GPIO3", + MTK_EINT_FUNCTION(0, 3), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO3"), + MTK_FUNCTION(1, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(2, "B0_SPIM5_MISO"), + MTK_FUNCTION(3, "I1_UCTS1"), + MTK_FUNCTION(4, "O_DMIC4_CLK"), + MTK_FUNCTION(5, "I0_I2SIN_D0"), + MTK_FUNCTION(6, "O_I2SO2_D0"), + MTK_FUNCTION(7, "B0_DBG_MON_A3") + ), + + MTK_PIN( + 4, "GPIO4", + MTK_EINT_FUNCTION(0, 4), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO4"), + MTK_FUNCTION(1, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(2, "I0_SPDIF_IN2"), + MTK_FUNCTION(3, "O_I2SO1_MCK"), + MTK_FUNCTION(4, "I0_DMIC4_DAT"), + MTK_FUNCTION(5, "I0_I2SIN_D1"), + MTK_FUNCTION(6, "O_I2SO2_D1"), + MTK_FUNCTION(7, "B0_DBG_MON_A4") + ), + + MTK_PIN( + 5, "GPIO5", + MTK_EINT_FUNCTION(0, 5), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO5"), + MTK_FUNCTION(1, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(2, "I0_SPDIF_IN1"), + MTK_FUNCTION(3, "O_I2SO1_BCK"), + MTK_FUNCTION(4, "I0_DMIC4_DAT_R"), + MTK_FUNCTION(5, "I0_I2SIN_D2"), + MTK_FUNCTION(6, "O_I2SO2_D2"), + MTK_FUNCTION(7, "B0_DBG_MON_A5") + ), + + MTK_PIN( + 6, "GPIO6", + MTK_EINT_FUNCTION(0, 6), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO6"), + MTK_FUNCTION(1, "B0_TP_GPIO6_AO"), + MTK_FUNCTION(2, "I0_SPDIF_IN0"), + MTK_FUNCTION(3, "O_I2SO1_WS"), + MTK_FUNCTION(4, "O_DMIC1_CLK"), + MTK_FUNCTION(5, "I0_I2SIN_D3"), + MTK_FUNCTION(6, "O_I2SO2_D3"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO0") + ), + + MTK_PIN( + 7, "GPIO7", + MTK_EINT_FUNCTION(0, 7), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO7"), + MTK_FUNCTION(1, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(2, "O_SPIM3_CSB"), + MTK_FUNCTION(3, "B0_TDMIN_MCK"), + MTK_FUNCTION(4, "I0_DMIC1_DAT"), + MTK_FUNCTION(5, "O_CMVREF0"), + MTK_FUNCTION(6, "O_CLKM0"), + MTK_FUNCTION(7, "B0_DBG_MON_A6") + ), + + MTK_PIN( + 8, "GPIO8", + MTK_EINT_FUNCTION(0, 8), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO8"), + MTK_FUNCTION(1, "B0_TP_GPIO0_AO"), + MTK_FUNCTION(2, "O_SPIM3_CLK"), + MTK_FUNCTION(3, "B0_TDMIN_BCK"), + MTK_FUNCTION(4, "I0_DMIC1_DAT_R"), + MTK_FUNCTION(5, "O_CMVREF1"), + MTK_FUNCTION(6, "O_CLKM1"), + MTK_FUNCTION(7, "B0_DBG_MON_A7") + ), + + MTK_PIN( + 9, "GPIO9", + MTK_EINT_FUNCTION(0, 9), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO9"), + MTK_FUNCTION(1, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(2, "B0_SPIM3_MOSI"), + MTK_FUNCTION(3, "B0_TDMIN_LRCK"), + MTK_FUNCTION(4, "O_DMIC2_CLK"), + MTK_FUNCTION(5, "O_CMFLASH0"), + MTK_FUNCTION(6, "O_PWM_0"), + MTK_FUNCTION(7, "B0_DBG_MON_A8") + ), + + MTK_PIN( + 10, "GPIO10", + MTK_EINT_FUNCTION(0, 10), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO10"), + MTK_FUNCTION(1, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(2, "B0_SPIM3_MISO"), + MTK_FUNCTION(3, "I0_TDMIN_DI"), + MTK_FUNCTION(4, "I0_DMIC2_DAT"), + MTK_FUNCTION(5, "O_CMFLASH1"), + MTK_FUNCTION(6, "O_PWM_1"), + MTK_FUNCTION(7, "B0_DBG_MON_A9") + ), + + MTK_PIN( + 11, "GPIO11", + MTK_EINT_FUNCTION(0, 11), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO11"), + MTK_FUNCTION(1, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(2, "O_SPDIF_OUT"), + MTK_FUNCTION(3, "O_I2SO1_D0"), + MTK_FUNCTION(4, "I0_DMIC2_DAT_R"), + MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"), + MTK_FUNCTION(6, "O_CMVREF6"), + MTK_FUNCTION(7, "B0_DBG_MON_A10") + ), + + MTK_PIN( + 12, "GPIO12", + MTK_EINT_FUNCTION(0, 12), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO12"), + MTK_FUNCTION(1, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(2, "O_SPIM4_CSB"), + MTK_FUNCTION(3, "B1_JTMS_SEL3"), + MTK_FUNCTION(4, "B1_APU_JTAG_TMS"), + MTK_FUNCTION(5, "I0_VPU_UDI_TMS"), + MTK_FUNCTION(6, "I0_IPU_JTAG_TMS"), + MTK_FUNCTION(7, "I0_HDMITX20_HTPLG") + ), + + MTK_PIN( + 13, "GPIO13", + MTK_EINT_FUNCTION(0, 13), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO13"), + MTK_FUNCTION(1, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(2, "O_SPIM4_CLK"), + MTK_FUNCTION(3, "I0_JTCK_SEL3"), + MTK_FUNCTION(4, "I0_APU_JTAG_TCK"), + MTK_FUNCTION(5, "I0_VPU_UDI_TCK"), + MTK_FUNCTION(6, "I0_IPU_JTAG_TCK"), + MTK_FUNCTION(7, "B1_HDMITX20_CEC") + ), + + MTK_PIN( + 14, "GPIO14", + MTK_EINT_FUNCTION(0, 14), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO14"), + MTK_FUNCTION(1, "B0_TP_GPIO6_AO"), + MTK_FUNCTION(2, "B0_SPIM4_MOSI"), + MTK_FUNCTION(3, "I1_JTDI_SEL3"), + MTK_FUNCTION(4, "I1_APU_JTAG_TDI"), + MTK_FUNCTION(5, "I0_VPU_UDI_TDI"), + MTK_FUNCTION(6, "I0_IPU_JTAG_TDI"), + MTK_FUNCTION(7, "B1_HDMITX20_SCL") + ), + + MTK_PIN( + 15, "GPIO15", + MTK_EINT_FUNCTION(0, 15), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO15"), + MTK_FUNCTION(1, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(2, "B0_SPIM4_MISO"), + MTK_FUNCTION(3, "O_JTDO_SEL3"), + MTK_FUNCTION(4, "O_APU_JTAG_TDO"), + MTK_FUNCTION(5, "O_VPU_UDI_TDO"), + MTK_FUNCTION(6, "O_IPU_JTAG_TDO"), + MTK_FUNCTION(7, "B1_HDMITX20_SDA") + ), + + MTK_PIN( + 16, "GPIO16", + MTK_EINT_FUNCTION(0, 16), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO16"), + MTK_FUNCTION(1, "B0_TP_GPIO0_AO"), + MTK_FUNCTION(2, "O_UTXD3"), + MTK_FUNCTION(3, "I1_JTRSTn_SEL3"), + MTK_FUNCTION(4, "I0_APU_JTAG_TRST"), + MTK_FUNCTION(5, "I0_VPU_UDI_NTRST"), + MTK_FUNCTION(6, "I0_IPU_JTAG_TRST"), + MTK_FUNCTION(7, "O_HDMITX20_PWR5V") + ), + + MTK_PIN( + 17, "GPIO17", + MTK_EINT_FUNCTION(0, 17), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO17"), + MTK_FUNCTION(1, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(2, "I1_URXD3"), + MTK_FUNCTION(3, "O_CMFLASH2"), + MTK_FUNCTION(4, "I0_EDP_TX_HPD"), + MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"), + MTK_FUNCTION(6, "O_CMVREF7"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO1") + ), + + MTK_PIN( + 18, "GPIO18", + MTK_EINT_FUNCTION(0, 18), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO18"), + MTK_FUNCTION(1, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(2, "O_CMFLASH0"), + MTK_FUNCTION(3, "O_CMVREF4"), + MTK_FUNCTION(4, "B0_TDMIN_MCK"), + MTK_FUNCTION(5, "O_UTXD1"), + MTK_FUNCTION(6, "O_TP_UTXD1_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_A11") + ), + + MTK_PIN( + 19, "GPIO19", + MTK_EINT_FUNCTION(0, 19), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO19"), + MTK_FUNCTION(1, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(2, "O_CMFLASH1"), + MTK_FUNCTION(3, "O_CMVREF5"), + MTK_FUNCTION(4, "B0_TDMIN_BCK"), + MTK_FUNCTION(5, "I1_URXD1"), + MTK_FUNCTION(6, "I1_TP_URXD1_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_A12") + ), + + MTK_PIN( + 20, "GPIO20", + MTK_EINT_FUNCTION(0, 20), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO20"), + MTK_FUNCTION(1, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(2, "O_CMFLASH2"), + MTK_FUNCTION(3, "O_CLKM2"), + MTK_FUNCTION(4, "B0_TDMIN_LRCK"), + MTK_FUNCTION(5, "O_URTS1"), + MTK_FUNCTION(6, "O_TP_URTS1_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_A13") + ), + + MTK_PIN( + 21, "GPIO21", + MTK_EINT_FUNCTION(0, 21), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO21"), + MTK_FUNCTION(1, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(2, "O_CMFLASH3"), + MTK_FUNCTION(3, "O_CLKM3"), + MTK_FUNCTION(4, "I0_TDMIN_DI"), + MTK_FUNCTION(5, "I1_UCTS1"), + MTK_FUNCTION(6, "I1_TP_UCTS1_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_A14") + ), + + MTK_PIN( + 22, "GPIO22", + MTK_EINT_FUNCTION(0, 22), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO22"), + MTK_FUNCTION(1, "O_CMMCLK0"), + MTK_FUNCTION(5, "B0_TP_GPIO6_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_A15") + ), + + MTK_PIN( + 23, "GPIO23", + MTK_EINT_FUNCTION(0, 23), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO23"), + MTK_FUNCTION(1, "O_CMMCLK1"), + MTK_FUNCTION(3, "O_PWM_2"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL"), + MTK_FUNCTION(5, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(6, "I0_DP_TX_HPD"), + MTK_FUNCTION(7, "B0_DBG_MON_A16") + ), + + MTK_PIN( + 24, "GPIO24", + MTK_EINT_FUNCTION(0, 24), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO24"), + MTK_FUNCTION(1, "O_CMMCLK2"), + MTK_FUNCTION(3, "O_PWM_3"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA"), + MTK_FUNCTION(5, "I0_DVFSRC_EXT_REQ"), + MTK_FUNCTION(6, "I0_EDP_TX_HPD"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO2") + ), + + MTK_PIN( + 25, "GPIO25", + MTK_EINT_FUNCTION(0, 25), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO25"), + MTK_FUNCTION(1, "O_LCM_RST"), + MTK_FUNCTION(2, "O_LCM1_RST"), + MTK_FUNCTION(3, "I0_DP_TX_HPD") + ), + + MTK_PIN( + 26, "GPIO26", + MTK_EINT_FUNCTION(0, 26), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO26"), + MTK_FUNCTION(1, "I0_DSI_TE"), + MTK_FUNCTION(2, "I0_DSI1_TE"), + MTK_FUNCTION(3, "I0_EDP_TX_HPD") + ), + + MTK_PIN( + 27, "GPIO27", + MTK_EINT_FUNCTION(0, 27), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO27"), + MTK_FUNCTION(1, "O_LCM1_RST"), + MTK_FUNCTION(2, "O_LCM_RST"), + MTK_FUNCTION(3, "I0_DP_TX_HPD"), + MTK_FUNCTION(4, "O_CMVREF2"), + MTK_FUNCTION(5, "O_mbistwriteen_trigger"), + MTK_FUNCTION(6, "O_PWM_2"), + MTK_FUNCTION(7, "B0_DBG_MON_A17") + ), + + MTK_PIN( + 28, "GPIO28", + MTK_EINT_FUNCTION(0, 28), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO28"), + MTK_FUNCTION(1, "I0_DSI1_TE"), + MTK_FUNCTION(2, "I0_DSI_TE"), + MTK_FUNCTION(3, "I0_EDP_TX_HPD"), + MTK_FUNCTION(4, "O_CMVREF3"), + MTK_FUNCTION(5, "O_mbistreaden_trigger"), + MTK_FUNCTION(6, "O_PWM_3"), + MTK_FUNCTION(7, "B0_DBG_MON_A18") + ), + + MTK_PIN( + 29, "GPIO29", + MTK_EINT_FUNCTION(0, 29), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO29"), + MTK_FUNCTION(1, "O_DISP_PWM0"), + MTK_FUNCTION(2, "O_DISP_PWM1") + ), + + MTK_PIN( + 30, "GPIO30", + MTK_EINT_FUNCTION(0, 30), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO30"), + MTK_FUNCTION(1, "O_DISP_PWM1"), + MTK_FUNCTION(2, "O_DISP_PWM0"), + MTK_FUNCTION(3, "O_CMFLASH3"), + MTK_FUNCTION(4, "O_PWM_1"), + MTK_FUNCTION(7, "B0_DBG_MON_A19") + ), + + MTK_PIN( + 31, "GPIO31", + MTK_EINT_FUNCTION(0, 31), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO31"), + MTK_FUNCTION(1, "O_UTXD0"), + MTK_FUNCTION(2, "O_TP_UTXD1_AO"), + MTK_FUNCTION(3, "O_ADSP_UTXD0"), + MTK_FUNCTION(4, "O_TP_UTXD2_AO"), + MTK_FUNCTION(5, "O_MD32_0_TXD"), + MTK_FUNCTION(6, "O_MD32_1_TXD"), + MTK_FUNCTION(7, "O_SSPM_UTXD_AO") + ), + + MTK_PIN( + 32, "GPIO32", + MTK_EINT_FUNCTION(0, 32), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO32"), + MTK_FUNCTION(1, "I1_URXD0"), + MTK_FUNCTION(2, "I1_TP_URXD1_AO"), + MTK_FUNCTION(3, "I1_ADSP_URXD0"), + MTK_FUNCTION(4, "I1_TP_URXD2_AO"), + MTK_FUNCTION(5, "I1_MD32_0_RXD"), + MTK_FUNCTION(6, "I1_MD32_1_RXD"), + MTK_FUNCTION(7, "I1_SSPM_URXD_AO") + ), + + MTK_PIN( + 33, "GPIO33", + MTK_EINT_FUNCTION(0, 33), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO33"), + MTK_FUNCTION(1, "O_UTXD1"), + MTK_FUNCTION(2, "O_URTS2"), + MTK_FUNCTION(3, "O_ADSP_UTXD0"), + MTK_FUNCTION(4, "O_TP_UTXD1_AO"), + MTK_FUNCTION(5, "O_mbistwriteen_trigger"), + MTK_FUNCTION(6, "O_MD32_0_TXD"), + MTK_FUNCTION(7, "O_SSPM_UTXD_AO") + ), + + MTK_PIN( + 34, "GPIO34", + MTK_EINT_FUNCTION(0, 34), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO34"), + MTK_FUNCTION(1, "I1_URXD1"), + MTK_FUNCTION(2, "I1_UCTS2"), + MTK_FUNCTION(3, "I1_ADSP_URXD0"), + MTK_FUNCTION(4, "I1_TP_URXD1_AO"), + MTK_FUNCTION(5, "O_mbistreaden_trigger"), + MTK_FUNCTION(6, "I1_MD32_0_RXD"), + MTK_FUNCTION(7, "I1_SSPM_URXD_AO") + ), + + MTK_PIN( + 35, "GPIO35", + MTK_EINT_FUNCTION(0, 35), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO35"), + MTK_FUNCTION(1, "O_UTXD2"), + MTK_FUNCTION(2, "O_URTS1"), + MTK_FUNCTION(3, "O_ADSP_UTXD0"), + MTK_FUNCTION(4, "O_TP_URTS1_AO"), + MTK_FUNCTION(5, "O_TP_UTXD2_AO"), + MTK_FUNCTION(6, "O_MD32_1_TXD"), + MTK_FUNCTION(7, "B0_DBG_MON_A20") + ), + + MTK_PIN( + 36, "GPIO36", + MTK_EINT_FUNCTION(0, 36), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO36"), + MTK_FUNCTION(1, "I1_URXD2"), + MTK_FUNCTION(2, "I1_UCTS1"), + MTK_FUNCTION(3, "I1_ADSP_URXD0"), + MTK_FUNCTION(4, "I1_TP_UCTS1_AO"), + MTK_FUNCTION(5, "I1_TP_URXD2_AO"), + MTK_FUNCTION(6, "I1_MD32_1_RXD"), + MTK_FUNCTION(7, "B0_DBG_MON_A21") + ), + + MTK_PIN( + 37, "GPIO37", + MTK_EINT_FUNCTION(0, 37), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO37"), + MTK_FUNCTION(1, "B1_JTMS_SEL1"), + MTK_FUNCTION(2, "I0_UDI_TMS"), + MTK_FUNCTION(3, "I1_SPM_JTAG_TMS"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TMS"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TMS"), + MTK_FUNCTION(7, "I1_MCUPM_JTAG_TMS") + ), + + MTK_PIN( + 38, "GPIO38", + MTK_EINT_FUNCTION(0, 38), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO38"), + MTK_FUNCTION(1, "I0_JTCK_SEL1"), + MTK_FUNCTION(2, "I0_UDI_TCK"), + MTK_FUNCTION(3, "I1_SPM_JTAG_TCK"), + MTK_FUNCTION(4, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TCK"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TCK"), + MTK_FUNCTION(7, "I1_MCUPM_JTAG_TCK") + ), + + MTK_PIN( + 39, "GPIO39", + MTK_EINT_FUNCTION(0, 39), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO39"), + MTK_FUNCTION(1, "I1_JTDI_SEL1"), + MTK_FUNCTION(2, "I0_UDI_TDI"), + MTK_FUNCTION(3, "I1_SPM_JTAG_TDI"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TDI"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TDI"), + MTK_FUNCTION(7, "I1_MCUPM_JTAG_TDI") + ), + + MTK_PIN( + 40, "GPIO40", + MTK_EINT_FUNCTION(0, 40), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO40"), + MTK_FUNCTION(1, "O_JTDO_SEL1"), + MTK_FUNCTION(2, "O_UDI_TDO"), + MTK_FUNCTION(3, "O_SPM_JTAG_TDO"), + MTK_FUNCTION(4, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(5, "O_SCP_JTAG0_TDO"), + MTK_FUNCTION(6, "O_CCU0_JTAG_TDO"), + MTK_FUNCTION(7, "O_MCUPM_JTAG_TDO") + ), + + MTK_PIN( + 41, "GPIO41", + MTK_EINT_FUNCTION(0, 41), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO41"), + MTK_FUNCTION(1, "I1_JTRSTn_SEL1"), + MTK_FUNCTION(2, "I0_UDI_NTRST"), + MTK_FUNCTION(3, "I0_SPM_JTAG_TRSTN"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(5, "I0_SCP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TRST"), + MTK_FUNCTION(7, "I0_MCUPM_JTAG_TRSTN") + ), + + MTK_PIN( + 42, "GPIO42", + MTK_EINT_FUNCTION(0, 42), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO42"), + MTK_FUNCTION(1, "B1_KPCOL0") + ), + + MTK_PIN( + 43, "GPIO43", + MTK_EINT_FUNCTION(0, 43), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO43"), + MTK_FUNCTION(1, "B1_KPCOL1"), + MTK_FUNCTION(2, "I0_DP_TX_HPD"), + MTK_FUNCTION(3, "O_CMFLASH2"), + MTK_FUNCTION(4, "I0_DVFSRC_EXT_REQ"), + MTK_FUNCTION(7, "O_mbistwriteen_trigger") + ), + + MTK_PIN( + 44, "GPIO44", + MTK_EINT_FUNCTION(0, 44), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO44"), + MTK_FUNCTION(1, "B1_KPROW0") + ), + + MTK_PIN( + 45, "GPIO45", + MTK_EINT_FUNCTION(0, 45), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO45"), + MTK_FUNCTION(1, "B1_KPROW1"), + MTK_FUNCTION(2, "I0_EDP_TX_HPD"), + MTK_FUNCTION(3, "O_CMFLASH3"), + MTK_FUNCTION(4, "B0_I2SIN_MCK"), + MTK_FUNCTION(7, "O_mbistreaden_trigger") + ), + + MTK_PIN( + 46, "GPIO46", + MTK_EINT_FUNCTION(0, 46), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO46"), + MTK_FUNCTION(1, "I0_DP_TX_HPD"), + MTK_FUNCTION(2, "O_PWM_0"), + MTK_FUNCTION(3, "I0_VBUSVALID_2P"), + MTK_FUNCTION(7, "B0_DBG_MON_A22") + ), + + MTK_PIN( + 47, "GPIO47", + MTK_EINT_FUNCTION(0, 47), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO47"), + MTK_FUNCTION(1, "I1_WAKEN"), + MTK_FUNCTION(6, "O_GDU_TROOPS_DET0") + ), + + MTK_PIN( + 48, "GPIO48", + MTK_EINT_FUNCTION(0, 48), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO48"), + MTK_FUNCTION(1, "O_PERSTN"), + MTK_FUNCTION(6, "O_GDU_TROOPS_DET1") + ), + + MTK_PIN( + 49, "GPIO49", + MTK_EINT_FUNCTION(0, 49), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO49"), + MTK_FUNCTION(1, "B1_CLKREQN"), + MTK_FUNCTION(6, "O_GDU_TROOPS_DET2") + ), + + MTK_PIN( + 50, "GPIO50", + MTK_EINT_FUNCTION(0, 50), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO50"), + MTK_FUNCTION(1, "O_HDMITX20_PWR5V"), + MTK_FUNCTION(3, "I1_IDDIG_1P"), + MTK_FUNCTION(4, "I1_SCP_JTAG1_TMS"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TMS"), + MTK_FUNCTION(6, "I1_MD32_0_JTAG_TMS"), + MTK_FUNCTION(7, "I1_MD32_1_JTAG_TMS") + ), + + MTK_PIN( + 51, "GPIO51", + MTK_EINT_FUNCTION(0, 51), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO51"), + MTK_FUNCTION(1, "I0_HDMITX20_HTPLG"), + MTK_FUNCTION(2, "I0_EDP_TX_HPD"), + MTK_FUNCTION(3, "O_USB_DRVVBUS_1P"), + MTK_FUNCTION(4, "I1_SCP_JTAG1_TCK"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TCK"), + MTK_FUNCTION(6, "I1_MD32_0_JTAG_TCK"), + MTK_FUNCTION(7, "I1_MD32_1_JTAG_TCK") + ), + + MTK_PIN( + 52, "GPIO52", + MTK_EINT_FUNCTION(0, 52), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO52"), + MTK_FUNCTION(1, "B1_HDMITX20_CEC"), + MTK_FUNCTION(3, "I0_VBUSVALID_1P"), + MTK_FUNCTION(4, "I1_SCP_JTAG1_TDI"), + MTK_FUNCTION(5, "I1_SSPM_JTAG_TDI"), + MTK_FUNCTION(6, "I1_MD32_0_JTAG_TDI"), + MTK_FUNCTION(7, "I1_MD32_1_JTAG_TDI") + ), + + MTK_PIN( + 53, "GPIO53", + MTK_EINT_FUNCTION(0, 53), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO53"), + MTK_FUNCTION(1, "B1_HDMITX20_SCL"), + MTK_FUNCTION(3, "I1_IDDIG_2P"), + MTK_FUNCTION(4, "O_SCP_JTAG1_TDO"), + MTK_FUNCTION(5, "O_SSPM_JTAG_TDO"), + MTK_FUNCTION(6, "O_MD32_0_JTAG_TDO"), + MTK_FUNCTION(7, "O_MD32_1_JTAG_TDO") + ), + + MTK_PIN( + 54, "GPIO54", + MTK_EINT_FUNCTION(0, 54), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO54"), + MTK_FUNCTION(1, "B1_HDMITX20_SDA"), + MTK_FUNCTION(3, "O_USB_DRVVBUS_2P"), + MTK_FUNCTION(4, "I0_SCP_JTAG1_TRSTN"), + MTK_FUNCTION(5, "I0_SSPM_JTAG_TRSTN"), + MTK_FUNCTION(6, "I1_MD32_0_JTAG_TRST"), + MTK_FUNCTION(7, "I1_MD32_1_JTAG_TRST") + ), + + MTK_PIN( + 55, "GPIO55", + MTK_EINT_FUNCTION(0, 55), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO55"), + MTK_FUNCTION(1, "B1_SCL0"), + MTK_FUNCTION(2, "B1_SCP_SCL0"), + MTK_FUNCTION(3, "B1_SCP_SCL1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL") + ), + + MTK_PIN( + 56, "GPIO56", + MTK_EINT_FUNCTION(0, 56), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO56"), + MTK_FUNCTION(1, "B1_SDA0"), + MTK_FUNCTION(2, "B1_SCP_SDA0"), + MTK_FUNCTION(3, "B1_SCP_SDA1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA") + ), + + MTK_PIN( + 57, "GPIO57", + MTK_EINT_FUNCTION(0, 57), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO57"), + MTK_FUNCTION(1, "B1_SCL1") + ), + + MTK_PIN( + 58, "GPIO58", + MTK_EINT_FUNCTION(0, 58), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO58"), + MTK_FUNCTION(1, "B1_SDA1") + ), + + MTK_PIN( + 59, "GPIO59", + MTK_EINT_FUNCTION(0, 59), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO59"), + MTK_FUNCTION(1, "B1_SCL2"), + MTK_FUNCTION(2, "B1_SCP_SCL0"), + MTK_FUNCTION(3, "B1_SCP_SCL1") + ), + + MTK_PIN( + 60, "GPIO60", + MTK_EINT_FUNCTION(0, 60), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO60"), + MTK_FUNCTION(1, "B1_SDA2"), + MTK_FUNCTION(2, "B1_SCP_SDA0"), + MTK_FUNCTION(3, "B1_SCP_SDA1") + ), + + MTK_PIN( + 61, "GPIO61", + MTK_EINT_FUNCTION(0, 61), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO61"), + MTK_FUNCTION(1, "B1_SCL3"), + MTK_FUNCTION(2, "B1_SCP_SCL0"), + MTK_FUNCTION(3, "B1_SCP_SCL1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL") + ), + + MTK_PIN( + 62, "GPIO62", + MTK_EINT_FUNCTION(0, 62), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO62"), + MTK_FUNCTION(1, "B1_SDA3"), + MTK_FUNCTION(2, "B1_SCP_SDA0"), + MTK_FUNCTION(3, "B1_SCP_SDA1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA") + ), + + MTK_PIN( + 63, "GPIO63", + MTK_EINT_FUNCTION(0, 63), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO63"), + MTK_FUNCTION(1, "B1_SCL4") + ), + + MTK_PIN( + 64, "GPIO64", + MTK_EINT_FUNCTION(0, 64), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO64"), + MTK_FUNCTION(1, "B1_SDA4") + ), + + MTK_PIN( + 65, "GPIO65", + MTK_EINT_FUNCTION(0, 65), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO65"), + MTK_FUNCTION(1, "B1_SCL5"), + MTK_FUNCTION(2, "B1_SCP_SCL0"), + MTK_FUNCTION(3, "B1_SCP_SCL1") + ), + + MTK_PIN( + 66, "GPIO66", + MTK_EINT_FUNCTION(0, 66), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO66"), + MTK_FUNCTION(1, "B1_SDA5"), + MTK_FUNCTION(2, "B1_SCP_SDA0"), + MTK_FUNCTION(3, "B1_SCP_SDA1") + ), + + MTK_PIN( + 67, "GPIO67", + MTK_EINT_FUNCTION(0, 67), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO67"), + MTK_FUNCTION(1, "B1_SCL6"), + MTK_FUNCTION(2, "B1_SCP_SCL0"), + MTK_FUNCTION(3, "B1_SCP_SCL1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SCL") + ), + + MTK_PIN( + 68, "GPIO68", + MTK_EINT_FUNCTION(0, 68), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO68"), + MTK_FUNCTION(1, "B1_SDA6"), + MTK_FUNCTION(2, "B1_SCP_SDA0"), + MTK_FUNCTION(3, "B1_SCP_SDA1"), + MTK_FUNCTION(4, "B1_PCIE_PHY_I2C_SDA") + ), + + MTK_PIN( + 69, "GPIO69", + MTK_EINT_FUNCTION(0, 69), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO69"), + MTK_FUNCTION(1, "O_SPIM0_CSB"), + MTK_FUNCTION(2, "O_SCP_SPI0_CS"), + MTK_FUNCTION(3, "O_DMIC3_CLK"), + MTK_FUNCTION(4, "B0_MD32_1_GPIO0"), + MTK_FUNCTION(5, "O_CMVREF0"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_0"), + MTK_FUNCTION(7, "B0_DBG_MON_A23") + ), + + MTK_PIN( + 70, "GPIO70", + MTK_EINT_FUNCTION(0, 70), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO70"), + MTK_FUNCTION(1, "O_SPIM0_CLK"), + MTK_FUNCTION(2, "O_SCP_SPI0_CK"), + MTK_FUNCTION(3, "I0_DMIC3_DAT"), + MTK_FUNCTION(4, "B0_MD32_1_GPIO1"), + MTK_FUNCTION(5, "O_CMVREF1"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_1"), + MTK_FUNCTION(7, "B0_DBG_MON_A24") + ), + + MTK_PIN( + 71, "GPIO71", + MTK_EINT_FUNCTION(0, 71), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO71"), + MTK_FUNCTION(1, "B0_SPIM0_MOSI"), + MTK_FUNCTION(2, "O_SCP_SPI0_MO"), + MTK_FUNCTION(3, "I0_DMIC3_DAT_R"), + MTK_FUNCTION(4, "B0_MD32_1_GPIO2"), + MTK_FUNCTION(5, "O_CMVREF2"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP0_2"), + MTK_FUNCTION(7, "B0_DBG_MON_A25") + ), + + MTK_PIN( + 72, "GPIO72", + MTK_EINT_FUNCTION(0, 72), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO72"), + MTK_FUNCTION(1, "B0_SPIM0_MISO"), + MTK_FUNCTION(2, "I0_SCP_SPI0_MI"), + MTK_FUNCTION(3, "O_DMIC4_CLK"), + MTK_FUNCTION(5, "O_CMVREF3"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_0"), + MTK_FUNCTION(7, "B0_DBG_MON_A26") + ), + + MTK_PIN( + 73, "GPIO73", + MTK_EINT_FUNCTION(0, 73), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO73"), + MTK_FUNCTION(1, "B0_SPIM0_MIO2"), + MTK_FUNCTION(2, "O_UTXD3"), + MTK_FUNCTION(3, "I0_DMIC4_DAT"), + MTK_FUNCTION(4, "O_CLKM0"), + MTK_FUNCTION(5, "O_CMVREF4"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_1"), + MTK_FUNCTION(7, "B0_DBG_MON_A27") + ), + + MTK_PIN( + 74, "GPIO74", + MTK_EINT_FUNCTION(0, 74), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO74"), + MTK_FUNCTION(1, "B0_SPIM0_MIO3"), + MTK_FUNCTION(2, "I1_URXD3"), + MTK_FUNCTION(3, "I0_DMIC4_DAT_R"), + MTK_FUNCTION(4, "O_CLKM1"), + MTK_FUNCTION(5, "O_CMVREF5"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP1_2"), + MTK_FUNCTION(7, "B0_DBG_MON_A28") + ), + + MTK_PIN( + 75, "GPIO75", + MTK_EINT_FUNCTION(0, 75), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO75"), + MTK_FUNCTION(1, "O_SPIM1_CSB"), + MTK_FUNCTION(2, "O_SCP_SPI1_A_CS"), + MTK_FUNCTION(3, "B0_TDMIN_MCK"), + MTK_FUNCTION(4, "B1_SCP_SCL0"), + MTK_FUNCTION(5, "O_CMVREF6"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_0"), + MTK_FUNCTION(7, "B0_DBG_MON_A29") + ), + + MTK_PIN( + 76, "GPIO76", + MTK_EINT_FUNCTION(0, 76), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO76"), + MTK_FUNCTION(1, "O_SPIM1_CLK"), + MTK_FUNCTION(2, "O_SCP_SPI1_A_CK"), + MTK_FUNCTION(3, "B0_TDMIN_BCK"), + MTK_FUNCTION(4, "B1_SCP_SDA0"), + MTK_FUNCTION(5, "O_CMVREF7"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_1"), + MTK_FUNCTION(7, "B0_DBG_MON_A30") + ), + + MTK_PIN( + 77, "GPIO77", + MTK_EINT_FUNCTION(0, 77), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO77"), + MTK_FUNCTION(1, "B0_SPIM1_MOSI"), + MTK_FUNCTION(2, "O_SCP_SPI1_A_MO"), + MTK_FUNCTION(3, "B0_TDMIN_LRCK"), + MTK_FUNCTION(4, "B1_SCP_SCL1"), + MTK_FUNCTION(6, "O_GDU_SUM_TROOP2_2"), + MTK_FUNCTION(7, "B0_DBG_MON_A31") + ), + + MTK_PIN( + 78, "GPIO78", + MTK_EINT_FUNCTION(0, 78), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO78"), + MTK_FUNCTION(1, "B0_SPIM1_MISO"), + MTK_FUNCTION(2, "I0_SCP_SPI1_A_MI"), + MTK_FUNCTION(3, "I0_TDMIN_DI"), + MTK_FUNCTION(4, "B1_SCP_SDA1"), + MTK_FUNCTION(7, "B0_DBG_MON_A32") + ), + + MTK_PIN( + 79, "GPIO79", + MTK_EINT_FUNCTION(0, 79), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO79"), + MTK_FUNCTION(1, "O_SPIM2_CSB"), + MTK_FUNCTION(2, "O_SCP_SPI2_CS"), + MTK_FUNCTION(3, "O_I2SO1_MCK"), + MTK_FUNCTION(4, "O_UTXD2"), + MTK_FUNCTION(5, "O_TP_UTXD2_AO"), + MTK_FUNCTION(6, "B0_PCM_SYNC"), + MTK_FUNCTION(7, "B0_DBG_MON_B0") + ), + + MTK_PIN( + 80, "GPIO80", + MTK_EINT_FUNCTION(0, 80), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO80"), + MTK_FUNCTION(1, "O_SPIM2_CLK"), + MTK_FUNCTION(2, "O_SCP_SPI2_CK"), + MTK_FUNCTION(3, "O_I2SO1_BCK"), + MTK_FUNCTION(4, "I1_URXD2"), + MTK_FUNCTION(5, "I1_TP_URXD2_AO"), + MTK_FUNCTION(6, "B0_PCM_CLK"), + MTK_FUNCTION(7, "B0_DBG_MON_B1") + ), + + MTK_PIN( + 81, "GPIO81", + MTK_EINT_FUNCTION(0, 81), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO81"), + MTK_FUNCTION(1, "B0_SPIM2_MOSI"), + MTK_FUNCTION(2, "O_SCP_SPI2_MO"), + MTK_FUNCTION(3, "O_I2SO1_WS"), + MTK_FUNCTION(4, "O_URTS2"), + MTK_FUNCTION(5, "O_TP_URTS2_AO"), + MTK_FUNCTION(6, "O_PCM_DO"), + MTK_FUNCTION(7, "B0_DBG_MON_B2") + ), + + MTK_PIN( + 82, "GPIO82", + MTK_EINT_FUNCTION(0, 82), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO82"), + MTK_FUNCTION(1, "B0_SPIM2_MISO"), + MTK_FUNCTION(2, "I0_SCP_SPI2_MI"), + MTK_FUNCTION(3, "O_I2SO1_D0"), + MTK_FUNCTION(4, "I1_UCTS2"), + MTK_FUNCTION(5, "I1_TP_UCTS2_AO"), + MTK_FUNCTION(6, "I0_PCM_DI"), + MTK_FUNCTION(7, "B0_DBG_MON_B3") + ), + + MTK_PIN( + 83, "GPIO83", + MTK_EINT_FUNCTION(0, 83), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO83"), + MTK_FUNCTION(1, "I1_IDDIG") + ), + + MTK_PIN( + 84, "GPIO84", + MTK_EINT_FUNCTION(0, 84), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO84"), + MTK_FUNCTION(1, "O_USB_DRVVBUS") + ), + + MTK_PIN( + 85, "GPIO85", + MTK_EINT_FUNCTION(0, 85), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO85"), + MTK_FUNCTION(1, "I0_VBUSVALID") + ), + + MTK_PIN( + 86, "GPIO86", + MTK_EINT_FUNCTION(0, 86), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO86"), + MTK_FUNCTION(1, "I1_IDDIG_1P"), + MTK_FUNCTION(2, "O_UTXD1"), + MTK_FUNCTION(3, "O_URTS2"), + MTK_FUNCTION(4, "O_PWM_2"), + MTK_FUNCTION(5, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(6, "O_AUXIF_ST0"), + MTK_FUNCTION(7, "B0_DBG_MON_B4") + ), + + MTK_PIN( + 87, "GPIO87", + MTK_EINT_FUNCTION(0, 87), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO87"), + MTK_FUNCTION(1, "O_USB_DRVVBUS_1P"), + MTK_FUNCTION(2, "I1_URXD1"), + MTK_FUNCTION(3, "I1_UCTS2"), + MTK_FUNCTION(4, "O_PWM_3"), + MTK_FUNCTION(5, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(6, "O_AUXIF_CLK0"), + MTK_FUNCTION(7, "B0_DBG_MON_B5") + ), + + MTK_PIN( + 88, "GPIO88", + MTK_EINT_FUNCTION(0, 88), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO88"), + MTK_FUNCTION(1, "I0_VBUSVALID_1P"), + MTK_FUNCTION(2, "O_UTXD2"), + MTK_FUNCTION(3, "O_URTS1"), + MTK_FUNCTION(4, "O_CLKM2"), + MTK_FUNCTION(5, "B0_TP_GPIO6_AO"), + MTK_FUNCTION(6, "O_AUXIF_ST1"), + MTK_FUNCTION(7, "B0_DBG_MON_B6") + ), + + MTK_PIN( + 89, "GPIO89", + MTK_EINT_FUNCTION(0, 89), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO89"), + MTK_FUNCTION(1, "I1_IDDIG_2P"), + MTK_FUNCTION(2, "I1_URXD2"), + MTK_FUNCTION(3, "I1_UCTS1"), + MTK_FUNCTION(4, "O_CLKM3"), + MTK_FUNCTION(5, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(6, "O_AUXIF_CLK1"), + MTK_FUNCTION(7, "B0_DBG_MON_B7") + ), + + MTK_PIN( + 90, "GPIO90", + MTK_EINT_FUNCTION(0, 90), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO90"), + MTK_FUNCTION(1, "O_USB_DRVVBUS_2P"), + MTK_FUNCTION(2, "O_UTXD3"), + MTK_FUNCTION(3, "O_ADSP_UTXD0"), + MTK_FUNCTION(4, "O_SSPM_UTXD_AO"), + MTK_FUNCTION(5, "O_MD32_0_TXD"), + MTK_FUNCTION(6, "O_MD32_1_TXD"), + MTK_FUNCTION(7, "B0_DBG_MON_B8") + ), + + MTK_PIN( + 91, "GPIO91", + MTK_EINT_FUNCTION(0, 91), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO91"), + MTK_FUNCTION(1, "I0_VBUSVALID_2P"), + MTK_FUNCTION(2, "I1_URXD3"), + MTK_FUNCTION(3, "I1_ADSP_URXD0"), + MTK_FUNCTION(4, "I1_SSPM_URXD_AO"), + MTK_FUNCTION(5, "I1_MD32_0_RXD"), + MTK_FUNCTION(6, "I1_MD32_1_RXD"), + MTK_FUNCTION(7, "B0_DBG_MON_B9") + ), + + MTK_PIN( + 92, "GPIO92", + MTK_EINT_FUNCTION(0, 92), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO92"), + MTK_FUNCTION(1, "O_PWRAP_SPI0_CSN") + ), + + MTK_PIN( + 93, "GPIO93", + MTK_EINT_FUNCTION(0, 93), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO93"), + MTK_FUNCTION(1, "O_PWRAP_SPI0_CK") + ), + + MTK_PIN( + 94, "GPIO94", + MTK_EINT_FUNCTION(0, 94), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO94"), + MTK_FUNCTION(1, "B0_PWRAP_SPI0_MO"), + MTK_FUNCTION(2, "B0_PWRAP_SPI0_MI") + ), + + MTK_PIN( + 95, "GPIO95", + MTK_EINT_FUNCTION(0, 95), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO95"), + MTK_FUNCTION(1, "B0_PWRAP_SPI0_MI"), + MTK_FUNCTION(2, "B0_PWRAP_SPI0_MO") + ), + + MTK_PIN( + 96, "GPIO96", + MTK_EINT_FUNCTION(0, 96), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO96"), + MTK_FUNCTION(1, "O_SRCLKENA0") + ), + + MTK_PIN( + 97, "GPIO97", + MTK_EINT_FUNCTION(0, 97), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO97"), + MTK_FUNCTION(1, "O_SRCLKENA1") + ), + + MTK_PIN( + 98, "GPIO98", + MTK_EINT_FUNCTION(0, 98), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO98"), + MTK_FUNCTION(1, "O_SCP_VREQ_VAO"), + MTK_FUNCTION(2, "I0_DVFSRC_EXT_REQ") + ), + + MTK_PIN( + 99, "GPIO99", + MTK_EINT_FUNCTION(0, 99), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO99"), + MTK_FUNCTION(1, "I0_RTC32K_CK") + ), + + MTK_PIN( + 100, "GPIO100", + MTK_EINT_FUNCTION(0, 100), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO100"), + MTK_FUNCTION(1, "O_WATCHDOG") + ), + + MTK_PIN( + 101, "GPIO101", + MTK_EINT_FUNCTION(0, 101), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO101"), + MTK_FUNCTION(1, "O_AUD_CLK_MOSI"), + MTK_FUNCTION(2, "O_I2SO1_MCK"), + MTK_FUNCTION(3, "B0_I2SIN_BCK") + ), + + MTK_PIN( + 102, "GPIO102", + MTK_EINT_FUNCTION(0, 102), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO102"), + MTK_FUNCTION(1, "O_AUD_SYNC_MOSI"), + MTK_FUNCTION(2, "O_I2SO1_BCK"), + MTK_FUNCTION(3, "B0_I2SIN_WS") + ), + + MTK_PIN( + 103, "GPIO103", + MTK_EINT_FUNCTION(0, 103), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO103"), + MTK_FUNCTION(1, "O_AUD_DAT_MOSI0"), + MTK_FUNCTION(2, "O_I2SO1_WS"), + MTK_FUNCTION(3, "I0_I2SIN_D0") + ), + + MTK_PIN( + 104, "GPIO104", + MTK_EINT_FUNCTION(0, 104), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO104"), + MTK_FUNCTION(1, "O_AUD_DAT_MOSI1"), + MTK_FUNCTION(2, "O_I2SO1_D0"), + MTK_FUNCTION(3, "I0_I2SIN_D1") + ), + + MTK_PIN( + 105, "GPIO105", + MTK_EINT_FUNCTION(0, 105), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO105"), + MTK_FUNCTION(1, "I0_AUD_DAT_MISO0"), + MTK_FUNCTION(2, "I0_VOW_DAT_MISO"), + MTK_FUNCTION(3, "I0_I2SIN_D2") + ), + + MTK_PIN( + 106, "GPIO106", + MTK_EINT_FUNCTION(0, 106), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO106"), + MTK_FUNCTION(1, "I0_AUD_DAT_MISO1"), + MTK_FUNCTION(2, "I0_VOW_CLK_MISO"), + MTK_FUNCTION(3, "I0_I2SIN_D3") + ), + + MTK_PIN( + 107, "GPIO107", + MTK_EINT_FUNCTION(0, 107), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO107"), + MTK_FUNCTION(1, "B0_I2SIN_MCK"), + MTK_FUNCTION(2, "I0_SPLIN_MCK"), + MTK_FUNCTION(3, "I0_SPDIF_IN0"), + MTK_FUNCTION(4, "O_CMVREF4"), + MTK_FUNCTION(5, "O_AUXIF_ST0"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR0") + ), + + MTK_PIN( + 108, "GPIO108", + MTK_EINT_FUNCTION(0, 108), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO108"), + MTK_FUNCTION(1, "B0_I2SIN_BCK"), + MTK_FUNCTION(2, "I0_SPLIN_LRCK"), + MTK_FUNCTION(3, "O_DMIC4_CLK"), + MTK_FUNCTION(4, "O_CMVREF5"), + MTK_FUNCTION(5, "O_AUXIF_CLK0"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR1"), + MTK_FUNCTION(7, "B0_DBG_MON_B10") + ), + + MTK_PIN( + 109, "GPIO109", + MTK_EINT_FUNCTION(0, 109), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO109"), + MTK_FUNCTION(1, "B0_I2SIN_WS"), + MTK_FUNCTION(2, "I0_SPLIN_BCK"), + MTK_FUNCTION(3, "I0_DMIC4_DAT"), + MTK_FUNCTION(4, "O_CMVREF6"), + MTK_FUNCTION(5, "O_AUXIF_ST1"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR2"), + MTK_FUNCTION(7, "B0_DBG_MON_B11") + ), + + MTK_PIN( + 110, "GPIO110", + MTK_EINT_FUNCTION(0, 110), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO110"), + MTK_FUNCTION(1, "I0_I2SIN_D0"), + MTK_FUNCTION(2, "I0_SPLIN_D0"), + MTK_FUNCTION(3, "I0_DMIC4_DAT_R"), + MTK_FUNCTION(4, "O_CMVREF7"), + MTK_FUNCTION(5, "O_AUXIF_CLK1"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR3"), + MTK_FUNCTION(7, "B0_DBG_MON_B12") + ), + + MTK_PIN( + 111, "GPIO111", + MTK_EINT_FUNCTION(0, 111), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO111"), + MTK_FUNCTION(1, "I0_I2SIN_D1"), + MTK_FUNCTION(2, "I0_SPLIN_D1"), + MTK_FUNCTION(3, "O_DMIC3_CLK"), + MTK_FUNCTION(4, "O_SPDIF_OUT"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR4"), + MTK_FUNCTION(7, "B0_DBG_MON_B13") + ), + + MTK_PIN( + 112, "GPIO112", + MTK_EINT_FUNCTION(0, 112), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO112"), + MTK_FUNCTION(1, "I0_I2SIN_D2"), + MTK_FUNCTION(2, "I0_SPLIN_D2"), + MTK_FUNCTION(3, "I0_DMIC3_DAT"), + MTK_FUNCTION(4, "B0_TDMIN_MCK"), + MTK_FUNCTION(5, "O_I2SO1_WS"), + MTK_FUNCTION(6, "O_PGD_LV_LSC_PWR5"), + MTK_FUNCTION(7, "B0_DBG_MON_B14") + ), + + MTK_PIN( + 113, "GPIO113", + MTK_EINT_FUNCTION(0, 113), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO113"), + MTK_FUNCTION(1, "I0_I2SIN_D3"), + MTK_FUNCTION(2, "I0_SPLIN_D3"), + MTK_FUNCTION(3, "I0_DMIC3_DAT_R"), + MTK_FUNCTION(4, "B0_TDMIN_BCK"), + MTK_FUNCTION(5, "O_I2SO1_D0"), + MTK_FUNCTION(7, "B0_DBG_MON_B15") + ), + + MTK_PIN( + 114, "GPIO114", + MTK_EINT_FUNCTION(0, 114), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO114"), + MTK_FUNCTION(1, "O_I2SO2_MCK"), + MTK_FUNCTION(2, "B0_I2SIN_MCK"), + MTK_FUNCTION(3, "I1_MCUPM_JTAG_TMS"), + MTK_FUNCTION(4, "B1_APU_JTAG_TMS"), + MTK_FUNCTION(5, "I1_SCP_JTAG1_TMS"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TMS"), + MTK_FUNCTION(7, "B0_DBG_MON_B16") + ), + + MTK_PIN( + 115, "GPIO115", + MTK_EINT_FUNCTION(0, 115), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO115"), + MTK_FUNCTION(1, "B0_I2SO2_BCK"), + MTK_FUNCTION(2, "B0_I2SIN_BCK"), + MTK_FUNCTION(3, "I1_MCUPM_JTAG_TCK"), + MTK_FUNCTION(4, "I0_APU_JTAG_TCK"), + MTK_FUNCTION(5, "I1_SCP_JTAG1_TCK"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TCK"), + MTK_FUNCTION(7, "B0_DBG_MON_B17") + ), + + MTK_PIN( + 116, "GPIO116", + MTK_EINT_FUNCTION(0, 116), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO116"), + MTK_FUNCTION(1, "B0_I2SO2_WS"), + MTK_FUNCTION(2, "B0_I2SIN_WS"), + MTK_FUNCTION(3, "I1_MCUPM_JTAG_TDI"), + MTK_FUNCTION(4, "I1_APU_JTAG_TDI"), + MTK_FUNCTION(5, "I1_SCP_JTAG1_TDI"), + MTK_FUNCTION(6, "I1_SPM_JTAG_TDI"), + MTK_FUNCTION(7, "B0_DBG_MON_B18") + ), + + MTK_PIN( + 117, "GPIO117", + MTK_EINT_FUNCTION(0, 117), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO117"), + MTK_FUNCTION(1, "O_I2SO2_D0"), + MTK_FUNCTION(2, "I0_I2SIN_D0"), + MTK_FUNCTION(3, "O_MCUPM_JTAG_TDO"), + MTK_FUNCTION(4, "O_APU_JTAG_TDO"), + MTK_FUNCTION(5, "O_SCP_JTAG1_TDO"), + MTK_FUNCTION(6, "O_SPM_JTAG_TDO"), + MTK_FUNCTION(7, "B0_DBG_MON_B19") + ), + + MTK_PIN( + 118, "GPIO118", + MTK_EINT_FUNCTION(0, 118), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO118"), + MTK_FUNCTION(1, "O_I2SO2_D1"), + MTK_FUNCTION(2, "I0_I2SIN_D1"), + MTK_FUNCTION(3, "I0_MCUPM_JTAG_TRSTN"), + MTK_FUNCTION(4, "I0_APU_JTAG_TRST"), + MTK_FUNCTION(5, "I0_SCP_JTAG1_TRSTN"), + MTK_FUNCTION(6, "I0_SPM_JTAG_TRSTN"), + MTK_FUNCTION(7, "B0_DBG_MON_B20") + ), + + MTK_PIN( + 119, "GPIO119", + MTK_EINT_FUNCTION(0, 119), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO119"), + MTK_FUNCTION(1, "O_I2SO2_D2"), + MTK_FUNCTION(2, "I0_I2SIN_D2"), + MTK_FUNCTION(3, "O_UTXD3"), + MTK_FUNCTION(4, "B0_TDMIN_LRCK"), + MTK_FUNCTION(5, "O_I2SO1_MCK"), + MTK_FUNCTION(6, "O_SSPM_UTXD_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_B21") + ), + + MTK_PIN( + 120, "GPIO120", + MTK_EINT_FUNCTION(0, 120), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO120"), + MTK_FUNCTION(1, "O_I2SO2_D3"), + MTK_FUNCTION(2, "I0_I2SIN_D3"), + MTK_FUNCTION(3, "I1_URXD3"), + MTK_FUNCTION(4, "I0_TDMIN_DI"), + MTK_FUNCTION(5, "O_I2SO1_BCK"), + MTK_FUNCTION(6, "I1_SSPM_URXD_AO"), + MTK_FUNCTION(7, "B0_DBG_MON_B22") + ), + + MTK_PIN( + 121, "GPIO121", + MTK_EINT_FUNCTION(0, 121), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO121"), + MTK_FUNCTION(1, "B0_PCM_CLK"), + MTK_FUNCTION(2, "O_SPIM4_CSB"), + MTK_FUNCTION(3, "O_SCP_SPI1_B_CS"), + MTK_FUNCTION(4, "O_TP_UTXD2_AO"), + MTK_FUNCTION(5, "O_AUXIF_ST0"), + MTK_FUNCTION(6, "O_PGD_DA_EFUSE_RDY"), + MTK_FUNCTION(7, "B0_DBG_MON_B23") + ), + + MTK_PIN( + 122, "GPIO122", + MTK_EINT_FUNCTION(0, 122), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO122"), + MTK_FUNCTION(1, "B0_PCM_SYNC"), + MTK_FUNCTION(2, "O_SPIM4_CLK"), + MTK_FUNCTION(3, "O_SCP_SPI1_B_CK"), + MTK_FUNCTION(4, "I1_TP_URXD2_AO"), + MTK_FUNCTION(5, "O_AUXIF_CLK0"), + MTK_FUNCTION(6, "O_PGD_DA_EFUSE_RDY_PRE"), + MTK_FUNCTION(7, "B0_DBG_MON_B24") + ), + + MTK_PIN( + 123, "GPIO123", + MTK_EINT_FUNCTION(0, 123), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO123"), + MTK_FUNCTION(1, "O_PCM_DO"), + MTK_FUNCTION(2, "B0_SPIM4_MOSI"), + MTK_FUNCTION(3, "O_SCP_SPI1_B_MO"), + MTK_FUNCTION(4, "O_TP_URTS2_AO"), + MTK_FUNCTION(5, "O_AUXIF_ST1"), + MTK_FUNCTION(6, "O_PGD_DA_PWRGD_RESET"), + MTK_FUNCTION(7, "B0_DBG_MON_B25") + ), + + MTK_PIN( + 124, "GPIO124", + MTK_EINT_FUNCTION(0, 124), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO124"), + MTK_FUNCTION(1, "I0_PCM_DI"), + MTK_FUNCTION(2, "B0_SPIM4_MISO"), + MTK_FUNCTION(3, "I0_SCP_SPI1_B_MI"), + MTK_FUNCTION(4, "I1_TP_UCTS2_AO"), + MTK_FUNCTION(5, "O_AUXIF_CLK1"), + MTK_FUNCTION(6, "O_PGD_DA_PWRGD_ENB"), + MTK_FUNCTION(7, "B0_DBG_MON_B26") + ), + + MTK_PIN( + 125, "GPIO125", + MTK_EINT_FUNCTION(0, 125), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO125"), + MTK_FUNCTION(1, "O_DMIC1_CLK"), + MTK_FUNCTION(2, "O_SPINOR_CK"), + MTK_FUNCTION(3, "B0_TDMIN_MCK"), + MTK_FUNCTION(6, "O_LVTS_FOUT"), + MTK_FUNCTION(7, "B0_DBG_MON_B27") + ), + + MTK_PIN( + 126, "GPIO126", + MTK_EINT_FUNCTION(0, 126), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO126"), + MTK_FUNCTION(1, "I0_DMIC1_DAT"), + MTK_FUNCTION(2, "O_SPINOR_CS"), + MTK_FUNCTION(3, "B0_TDMIN_BCK"), + MTK_FUNCTION(6, "O_LVTS_SDO"), + MTK_FUNCTION(7, "B0_DBG_MON_B28") + ), + + MTK_PIN( + 127, "GPIO127", + MTK_EINT_FUNCTION(0, 127), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO127"), + MTK_FUNCTION(1, "I0_DMIC1_DAT_R"), + MTK_FUNCTION(2, "B0_SPINOR_IO0"), + MTK_FUNCTION(3, "B0_TDMIN_LRCK"), + MTK_FUNCTION(6, "I0_LVTS_26M"), + MTK_FUNCTION(7, "B0_DBG_MON_B29") + ), + + MTK_PIN( + 128, "GPIO128", + MTK_EINT_FUNCTION(0, 128), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO128"), + MTK_FUNCTION(1, "O_DMIC2_CLK"), + MTK_FUNCTION(2, "B0_SPINOR_IO1"), + MTK_FUNCTION(3, "I0_TDMIN_DI"), + MTK_FUNCTION(6, "I0_LVTS_SCF"), + MTK_FUNCTION(7, "B0_DBG_MON_B30") + ), + + MTK_PIN( + 129, "GPIO129", + MTK_EINT_FUNCTION(0, 129), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO129"), + MTK_FUNCTION(1, "I0_DMIC2_DAT"), + MTK_FUNCTION(2, "B0_SPINOR_IO2"), + MTK_FUNCTION(3, "I0_SPDIF_IN1"), + MTK_FUNCTION(6, "I0_LVTS_SCK"), + MTK_FUNCTION(7, "B0_DBG_MON_B31") + ), + + MTK_PIN( + 130, "GPIO130", + MTK_EINT_FUNCTION(0, 130), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO130"), + MTK_FUNCTION(1, "I0_DMIC2_DAT_R"), + MTK_FUNCTION(2, "B0_SPINOR_IO3"), + MTK_FUNCTION(3, "I0_SPDIF_IN2"), + MTK_FUNCTION(6, "I0_LVTS_SDI"), + MTK_FUNCTION(7, "B0_DBG_MON_B32") + ), + + MTK_PIN( + 131, "GPIO131", + MTK_EINT_FUNCTION(0, 131), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO131"), + MTK_FUNCTION(1, "O_DPI_D0"), + MTK_FUNCTION(2, "O_GBE_TXD3"), + MTK_FUNCTION(3, "O_DMIC1_CLK"), + MTK_FUNCTION(4, "O_I2SO2_MCK"), + MTK_FUNCTION(5, "B0_TP_GPIO0_AO"), + MTK_FUNCTION(6, "O_SPIM5_CSB"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR0") + ), + + MTK_PIN( + 132, "GPIO132", + MTK_EINT_FUNCTION(0, 132), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO132"), + MTK_FUNCTION(1, "O_DPI_D1"), + MTK_FUNCTION(2, "O_GBE_TXD2"), + MTK_FUNCTION(3, "I0_DMIC1_DAT"), + MTK_FUNCTION(4, "B0_I2SO2_BCK"), + MTK_FUNCTION(5, "B0_TP_GPIO1_AO"), + MTK_FUNCTION(6, "O_SPIM5_CLK"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR1") + ), + + MTK_PIN( + 133, "GPIO133", + MTK_EINT_FUNCTION(0, 133), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO133"), + MTK_FUNCTION(1, "O_DPI_D2"), + MTK_FUNCTION(2, "O_GBE_TXD1"), + MTK_FUNCTION(3, "I0_DMIC1_DAT_R"), + MTK_FUNCTION(4, "B0_I2SO2_WS"), + MTK_FUNCTION(5, "B0_TP_GPIO2_AO"), + MTK_FUNCTION(6, "B0_SPIM5_MOSI"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR2") + ), + + MTK_PIN( + 134, "GPIO134", + MTK_EINT_FUNCTION(0, 134), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO134"), + MTK_FUNCTION(1, "O_DPI_D3"), + MTK_FUNCTION(2, "O_GBE_TXD0"), + MTK_FUNCTION(3, "O_DMIC2_CLK"), + MTK_FUNCTION(4, "O_I2SO2_D0"), + MTK_FUNCTION(5, "B0_TP_GPIO3_AO"), + MTK_FUNCTION(6, "B0_SPIM5_MISO"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR3") + ), + + MTK_PIN( + 135, "GPIO135", + MTK_EINT_FUNCTION(0, 135), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO135"), + MTK_FUNCTION(1, "O_DPI_D4"), + MTK_FUNCTION(2, "I0_GBE_RXD3"), + MTK_FUNCTION(3, "I0_DMIC2_DAT"), + MTK_FUNCTION(4, "O_I2SO2_D1"), + MTK_FUNCTION(5, "B0_TP_GPIO4_AO"), + MTK_FUNCTION(6, "I1_WAKEN"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR4") + ), + + MTK_PIN( + 136, "GPIO136", + MTK_EINT_FUNCTION(0, 136), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO136"), + MTK_FUNCTION(1, "O_DPI_D5"), + MTK_FUNCTION(2, "I0_GBE_RXD2"), + MTK_FUNCTION(3, "I0_DMIC2_DAT_R"), + MTK_FUNCTION(4, "O_I2SO2_D2"), + MTK_FUNCTION(5, "B0_TP_GPIO5_AO"), + MTK_FUNCTION(6, "O_PERSTN"), + MTK_FUNCTION(7, "O_PGD_LV_HSC_PWR5") + ), + + MTK_PIN( + 137, "GPIO137", + MTK_EINT_FUNCTION(0, 137), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO137"), + MTK_FUNCTION(1, "O_DPI_D6"), + MTK_FUNCTION(2, "I0_GBE_RXD1"), + MTK_FUNCTION(3, "O_DMIC3_CLK"), + MTK_FUNCTION(4, "O_I2SO2_D3"), + MTK_FUNCTION(5, "B0_TP_GPIO6_AO"), + MTK_FUNCTION(6, "B1_CLKREQN"), + MTK_FUNCTION(7, "O_PWM_0") + ), + + MTK_PIN( + 138, "GPIO138", + MTK_EINT_FUNCTION(0, 138), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO138"), + MTK_FUNCTION(1, "O_DPI_D7"), + MTK_FUNCTION(2, "I0_GBE_RXD0"), + MTK_FUNCTION(3, "I0_DMIC3_DAT"), + MTK_FUNCTION(4, "O_CLKM2"), + MTK_FUNCTION(5, "B0_TP_GPIO7_AO"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO0") + ), + + MTK_PIN( + 139, "GPIO139", + MTK_EINT_FUNCTION(0, 139), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO139"), + MTK_FUNCTION(1, "O_DPI_D8"), + MTK_FUNCTION(2, "B0_GBE_TXC"), + MTK_FUNCTION(3, "I0_DMIC3_DAT_R"), + MTK_FUNCTION(4, "O_CLKM3"), + MTK_FUNCTION(5, "O_TP_UTXD2_AO"), + MTK_FUNCTION(6, "O_UTXD2"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO1") + ), + + MTK_PIN( + 140, "GPIO140", + MTK_EINT_FUNCTION(0, 140), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO140"), + MTK_FUNCTION(1, "O_DPI_D9"), + MTK_FUNCTION(2, "I0_GBE_RXC"), + MTK_FUNCTION(3, "O_DMIC4_CLK"), + MTK_FUNCTION(4, "O_PWM_2"), + MTK_FUNCTION(5, "I1_TP_URXD2_AO"), + MTK_FUNCTION(6, "I1_URXD2"), + MTK_FUNCTION(7, "B0_MD32_0_GPIO2") + ), + + MTK_PIN( + 141, "GPIO141", + MTK_EINT_FUNCTION(0, 141), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO141"), + MTK_FUNCTION(1, "O_DPI_D10"), + MTK_FUNCTION(2, "I0_GBE_RXDV"), + MTK_FUNCTION(3, "I0_DMIC4_DAT"), + MTK_FUNCTION(4, "O_PWM_3"), + MTK_FUNCTION(5, "O_TP_URTS2_AO"), + MTK_FUNCTION(6, "O_URTS2"), + MTK_FUNCTION(7, "B0_MD32_1_GPIO0") + ), + + MTK_PIN( + 142, "GPIO142", + MTK_EINT_FUNCTION(0, 142), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO142"), + MTK_FUNCTION(1, "O_DPI_D11"), + MTK_FUNCTION(2, "O_GBE_TXEN"), + MTK_FUNCTION(3, "I0_DMIC4_DAT_R"), + MTK_FUNCTION(4, "O_PWM_1"), + MTK_FUNCTION(5, "I1_TP_UCTS2_AO"), + MTK_FUNCTION(6, "I1_UCTS2"), + MTK_FUNCTION(7, "B0_MD32_1_GPIO1") + ), + + MTK_PIN( + 143, "GPIO143", + MTK_EINT_FUNCTION(0, 143), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO143"), + MTK_FUNCTION(1, "O_DPI_D12"), + MTK_FUNCTION(2, "O_GBE_MDC"), + MTK_FUNCTION(3, "B0_MD32_0_GPIO0"), + MTK_FUNCTION(4, "O_CLKM0"), + MTK_FUNCTION(5, "O_SPIM3_CSB"), + MTK_FUNCTION(6, "O_UTXD1"), + MTK_FUNCTION(7, "B0_MD32_1_GPIO2") + ), + + MTK_PIN( + 144, "GPIO144", + MTK_EINT_FUNCTION(0, 144), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO144"), + MTK_FUNCTION(1, "O_DPI_D13"), + MTK_FUNCTION(2, "B1_GBE_MDIO"), + MTK_FUNCTION(3, "B0_MD32_0_GPIO1"), + MTK_FUNCTION(4, "O_CLKM1"), + MTK_FUNCTION(5, "O_SPIM3_CLK"), + MTK_FUNCTION(6, "I1_URXD1"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR0") + ), + + MTK_PIN( + 145, "GPIO145", + MTK_EINT_FUNCTION(0, 145), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO145"), + MTK_FUNCTION(1, "O_DPI_D14"), + MTK_FUNCTION(2, "O_GBE_TXER"), + MTK_FUNCTION(3, "B0_MD32_1_GPIO0"), + MTK_FUNCTION(4, "O_CMFLASH0"), + MTK_FUNCTION(5, "B0_SPIM3_MOSI"), + MTK_FUNCTION(6, "B0_GBE_AUX_PPS2"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR1") + ), + + MTK_PIN( + 146, "GPIO146", + MTK_EINT_FUNCTION(0, 146), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO146"), + MTK_FUNCTION(1, "O_DPI_D15"), + MTK_FUNCTION(2, "I0_GBE_RXER"), + MTK_FUNCTION(3, "B0_MD32_1_GPIO1"), + MTK_FUNCTION(4, "O_CMFLASH1"), + MTK_FUNCTION(5, "B0_SPIM3_MISO"), + MTK_FUNCTION(6, "B0_GBE_AUX_PPS3"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR2") + ), + + MTK_PIN( + 147, "GPIO147", + MTK_EINT_FUNCTION(0, 147), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO147"), + MTK_FUNCTION(1, "O_DPI_HSYNC"), + MTK_FUNCTION(2, "I0_GBE_COL"), + MTK_FUNCTION(3, "O_I2SO1_MCK"), + MTK_FUNCTION(4, "O_CMVREF0"), + MTK_FUNCTION(5, "O_SPDIF_OUT"), + MTK_FUNCTION(6, "O_URTS1"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR3") + ), + + MTK_PIN( + 148, "GPIO148", + MTK_EINT_FUNCTION(0, 148), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO148"), + MTK_FUNCTION(1, "O_DPI_VSYNC"), + MTK_FUNCTION(2, "I0_GBE_INTR"), + MTK_FUNCTION(3, "O_I2SO1_BCK"), + MTK_FUNCTION(4, "O_CMVREF1"), + MTK_FUNCTION(5, "I0_SPDIF_IN0"), + MTK_FUNCTION(6, "I1_UCTS1"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR4") + ), + + MTK_PIN( + 149, "GPIO149", + MTK_EINT_FUNCTION(0, 149), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO149"), + MTK_FUNCTION(1, "O_DPI_DE"), + MTK_FUNCTION(2, "B0_GBE_AUX_PPS0"), + MTK_FUNCTION(3, "O_I2SO1_WS"), + MTK_FUNCTION(4, "O_CMVREF2"), + MTK_FUNCTION(5, "I0_SPDIF_IN1"), + MTK_FUNCTION(6, "O_UTXD3"), + MTK_FUNCTION(7, "O_PGD_HV_HSC_PWR5") + ), + + MTK_PIN( + 150, "GPIO150", + MTK_EINT_FUNCTION(0, 150), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO150"), + MTK_FUNCTION(1, "O_DPI_CK"), + MTK_FUNCTION(2, "B0_GBE_AUX_PPS1"), + MTK_FUNCTION(3, "O_I2SO1_D0"), + MTK_FUNCTION(4, "O_CMVREF3"), + MTK_FUNCTION(5, "I0_SPDIF_IN2"), + MTK_FUNCTION(6, "I1_URXD3") + ), + + MTK_PIN( + 151, "GPIO151", + MTK_EINT_FUNCTION(0, 151), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO151"), + MTK_FUNCTION(1, "B1_MSDC0_DAT7") + ), + + MTK_PIN( + 152, "GPIO152", + MTK_EINT_FUNCTION(0, 152), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO152"), + MTK_FUNCTION(1, "B1_MSDC0_DAT6") + ), + + MTK_PIN( + 153, "GPIO153", + MTK_EINT_FUNCTION(0, 153), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO153"), + MTK_FUNCTION(1, "B1_MSDC0_DAT5") + ), + + MTK_PIN( + 154, "GPIO154", + MTK_EINT_FUNCTION(0, 154), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO154"), + MTK_FUNCTION(1, "B1_MSDC0_DAT4") + ), + + MTK_PIN( + 155, "GPIO155", + MTK_EINT_FUNCTION(0, 155), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO155"), + MTK_FUNCTION(1, "O_MSDC0_RSTB") + ), + + MTK_PIN( + 156, "GPIO156", + MTK_EINT_FUNCTION(0, 156), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO156"), + MTK_FUNCTION(1, "B1_MSDC0_CMD") + ), + + MTK_PIN( + 157, "GPIO157", + MTK_EINT_FUNCTION(0, 157), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO157"), + MTK_FUNCTION(1, "B1_MSDC0_CLK") + ), + + MTK_PIN( + 158, "GPIO158", + MTK_EINT_FUNCTION(0, 158), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO158"), + MTK_FUNCTION(1, "B1_MSDC0_DAT3") + ), + + MTK_PIN( + 159, "GPIO159", + MTK_EINT_FUNCTION(0, 159), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO159"), + MTK_FUNCTION(1, "B1_MSDC0_DAT2") + ), + + MTK_PIN( + 160, "GPIO160", + MTK_EINT_FUNCTION(0, 160), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO160"), + MTK_FUNCTION(1, "B1_MSDC0_DAT1") + ), + + MTK_PIN( + 161, "GPIO161", + MTK_EINT_FUNCTION(0, 161), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO161"), + MTK_FUNCTION(1, "B1_MSDC0_DAT0") + ), + + MTK_PIN( + 162, "GPIO162", + MTK_EINT_FUNCTION(0, 162), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO162"), + MTK_FUNCTION(1, "B0_MSDC0_DSL") + ), + + MTK_PIN( + 163, "GPIO163", + MTK_EINT_FUNCTION(0, 163), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO163"), + MTK_FUNCTION(1, "B1_MSDC1_CMD"), + MTK_FUNCTION(2, "O_SPDIF_OUT"), + MTK_FUNCTION(3, "I1_MD32_0_JTAG_TMS"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TMS"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TMS"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TMS"), + MTK_FUNCTION(7, "I0_IPU_JTAG_TMS") + ), + + MTK_PIN( + 164, "GPIO164", + MTK_EINT_FUNCTION(0, 164), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO164"), + MTK_FUNCTION(1, "B1_MSDC1_CLK"), + MTK_FUNCTION(2, "I0_SPDIF_IN0"), + MTK_FUNCTION(3, "I1_MD32_0_JTAG_TCK"), + MTK_FUNCTION(4, "I0_ADSP_JTAG0_TCK"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TCK"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TCK"), + MTK_FUNCTION(7, "I0_IPU_JTAG_TCK") + ), + + MTK_PIN( + 165, "GPIO165", + MTK_EINT_FUNCTION(0, 165), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO165"), + MTK_FUNCTION(1, "B1_MSDC1_DAT0"), + MTK_FUNCTION(2, "I0_SPDIF_IN1"), + MTK_FUNCTION(3, "I1_MD32_0_JTAG_TDI"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TDI"), + MTK_FUNCTION(5, "I1_SCP_JTAG0_TDI"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TDI"), + MTK_FUNCTION(7, "I0_IPU_JTAG_TDI") + ), + + MTK_PIN( + 166, "GPIO166", + MTK_EINT_FUNCTION(0, 166), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO166"), + MTK_FUNCTION(1, "B1_MSDC1_DAT1"), + MTK_FUNCTION(2, "I0_SPDIF_IN2"), + MTK_FUNCTION(3, "O_MD32_0_JTAG_TDO"), + MTK_FUNCTION(4, "O_ADSP_JTAG0_TDO"), + MTK_FUNCTION(5, "O_SCP_JTAG0_TDO"), + MTK_FUNCTION(6, "O_CCU0_JTAG_TDO"), + MTK_FUNCTION(7, "O_IPU_JTAG_TDO") + ), + + MTK_PIN( + 167, "GPIO167", + MTK_EINT_FUNCTION(0, 167), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO167"), + MTK_FUNCTION(1, "B1_MSDC1_DAT2"), + MTK_FUNCTION(2, "O_PWM_0"), + MTK_FUNCTION(3, "I1_MD32_0_JTAG_TRST"), + MTK_FUNCTION(4, "I1_ADSP_JTAG0_TRSTN"), + MTK_FUNCTION(5, "I0_SCP_JTAG0_TRSTN"), + MTK_FUNCTION(6, "I1_CCU0_JTAG_TRST"), + MTK_FUNCTION(7, "I0_IPU_JTAG_TRST") + ), + + MTK_PIN( + 168, "GPIO168", + MTK_EINT_FUNCTION(0, 168), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO168"), + MTK_FUNCTION(1, "B1_MSDC1_DAT3"), + MTK_FUNCTION(2, "O_PWM_1"), + MTK_FUNCTION(3, "O_CLKM0") + ), + + MTK_PIN( + 169, "GPIO169", + MTK_EINT_FUNCTION(0, 169), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO169"), + MTK_FUNCTION(1, "B1_MSDC2_CMD"), + MTK_FUNCTION(2, "O_LVTS_FOUT"), + MTK_FUNCTION(3, "I1_MD32_1_JTAG_TMS"), + MTK_FUNCTION(4, "I0_UDI_TMS"), + MTK_FUNCTION(5, "I0_VPU_UDI_TMS"), + MTK_FUNCTION(6, "B0_TDMIN_MCK"), + MTK_FUNCTION(7, "I1_SSPM_JTAG_TMS") + ), + + MTK_PIN( + 170, "GPIO170", + MTK_EINT_FUNCTION(0, 170), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO170"), + MTK_FUNCTION(1, "B1_MSDC2_CLK"), + MTK_FUNCTION(2, "O_LVTS_SDO"), + MTK_FUNCTION(3, "I1_MD32_1_JTAG_TCK"), + MTK_FUNCTION(4, "I0_UDI_TCK"), + MTK_FUNCTION(5, "I0_VPU_UDI_TCK"), + MTK_FUNCTION(6, "B0_TDMIN_BCK"), + MTK_FUNCTION(7, "I1_SSPM_JTAG_TCK") + ), + + MTK_PIN( + 171, "GPIO171", + MTK_EINT_FUNCTION(0, 171), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO171"), + MTK_FUNCTION(1, "B1_MSDC2_DAT0"), + MTK_FUNCTION(2, "I0_LVTS_26M"), + MTK_FUNCTION(3, "I1_MD32_1_JTAG_TDI"), + MTK_FUNCTION(4, "I0_UDI_TDI"), + MTK_FUNCTION(5, "I0_VPU_UDI_TDI"), + MTK_FUNCTION(6, "B0_TDMIN_LRCK"), + MTK_FUNCTION(7, "I1_SSPM_JTAG_TDI") + ), + + MTK_PIN( + 172, "GPIO172", + MTK_EINT_FUNCTION(0, 172), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO172"), + MTK_FUNCTION(1, "B1_MSDC2_DAT1"), + MTK_FUNCTION(2, "I0_LVTS_SCF"), + MTK_FUNCTION(3, "O_MD32_1_JTAG_TDO"), + MTK_FUNCTION(4, "O_UDI_TDO"), + MTK_FUNCTION(5, "O_VPU_UDI_TDO"), + MTK_FUNCTION(6, "I0_TDMIN_DI"), + MTK_FUNCTION(7, "O_SSPM_JTAG_TDO") + ), + + MTK_PIN( + 173, "GPIO173", + MTK_EINT_FUNCTION(0, 173), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO173"), + MTK_FUNCTION(1, "B1_MSDC2_DAT2"), + MTK_FUNCTION(2, "I0_LVTS_SCK"), + MTK_FUNCTION(3, "I1_MD32_1_JTAG_TRST"), + MTK_FUNCTION(4, "I0_UDI_NTRST"), + MTK_FUNCTION(5, "I0_VPU_UDI_NTRST"), + MTK_FUNCTION(7, "I0_SSPM_JTAG_TRSTN") + ), + + MTK_PIN( + 174, "GPIO174", + MTK_EINT_FUNCTION(0, 174), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO174"), + MTK_FUNCTION(1, "B1_MSDC2_DAT3"), + MTK_FUNCTION(2, "I0_LVTS_SDI") + ), + + MTK_PIN( + 175, "GPIO175", + MTK_EINT_FUNCTION(0, 175), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO175"), + MTK_FUNCTION(1, "B0_SPMI_M_SCL") + ), + + MTK_PIN( + 176, "GPIO176", + MTK_EINT_FUNCTION(0, 176), + DRV_GRP4, + MTK_FUNCTION(0, "B_GPIO176"), + MTK_FUNCTION(1, "B0_SPMI_M_SDA") + ), + + MTK_PIN( + 177, "GPIO177", + MTK_EINT_FUNCTION(0, 212), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 178, "GPIO178", + MTK_EINT_FUNCTION(0, 213), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 179, "GPIO179", + MTK_EINT_FUNCTION(0, 214), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 180, "GPIO180", + MTK_EINT_FUNCTION(0, 215), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 181, "GPIO181", + MTK_EINT_FUNCTION(0, 216), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 182, "GPIO182", + MTK_EINT_FUNCTION(0, 217), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 183, "GPIO183", + MTK_EINT_FUNCTION(0, 218), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 184, "GPIO184", + MTK_EINT_FUNCTION(0, 219), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 185, "GPIO185", + MTK_EINT_FUNCTION(0, 220), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 186, "GPIO186", + MTK_EINT_FUNCTION(0, 221), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 187, "GPIO187", + MTK_EINT_FUNCTION(0, 222), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 188, "GPIO188", + MTK_EINT_FUNCTION(0, 223), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ), + + MTK_PIN( + 189, "GPIO189", + MTK_EINT_FUNCTION(0, 224), + DRV_FIXED, + MTK_FUNCTION(0, NULL) + ) +}; + +#endif /* __PINCTRL__MTK_MT8188_H */ diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index cc2cd73ff8f98229b3687e49c942164f1a1b3129..530f3f934e196bbda701d0b81d3ddf99d986f2d9 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -608,6 +608,7 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc) pc->chip.label = pc->data->name; pc->chip.parent = pc->dev; + pc->chip.fwnode = pc->fwnode; pc->chip.request = gpiochip_generic_request; pc->chip.free = gpiochip_generic_free; pc->chip.set_config = gpiochip_generic_config; @@ -619,8 +620,6 @@ static int meson_gpiolib_register(struct meson_pinctrl *pc) pc->chip.base = -1; pc->chip.ngpio = pc->data->num_pins; pc->chip.can_sleep = false; - pc->chip.of_node = pc->of_node; - pc->chip.of_gpio_n_cells = 2; ret = gpiochip_add_data(&pc->chip, pc); if (ret) { @@ -678,8 +677,8 @@ static int meson_pinctrl_parse_dt(struct meson_pinctrl *pc) return -EINVAL; } - gpio_np = to_of_node(gpiochip_node_get_first(pc->dev)); - pc->of_node = gpio_np; + pc->fwnode = gpiochip_node_get_first(pc->dev); + gpio_np = to_of_node(pc->fwnode); pc->reg_mux = meson_map_resource(pc, gpio_np, "mux"); if (IS_ERR_OR_NULL(pc->reg_mux)) { diff --git a/drivers/pinctrl/meson/pinctrl-meson.h b/drivers/pinctrl/meson/pinctrl-meson.h index b197827027bd0bddc22b1b357ebb23de16942ffe..34fc4e8612e47cb8cc26fdc82888c6dbb15c1f03 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.h +++ b/drivers/pinctrl/meson/pinctrl-meson.h @@ -12,6 +12,8 @@ #include #include +struct fwnode_handle; + struct meson_pinctrl; /** @@ -131,7 +133,7 @@ struct meson_pinctrl { struct regmap *reg_gpio; struct regmap *reg_ds; struct gpio_chip chip; - struct device_node *of_node; + struct fwnode_handle *fwnode; }; #define FUNCTION(fn) \ diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index bcde042d29dc3b6811a41b8d6b7c8676a4199cf1..261b46841b9f66c88c106f69ff82baefa4f1d06e 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -112,14 +112,14 @@ struct armada_37xx_pinctrl { struct armada_37xx_pm_state pm; }; -#define PIN_GRP(_name, _start, _nr, _mask, _func1, _func2) \ +#define PIN_GRP_GPIO_0(_name, _start, _nr) \ { \ .name = _name, \ .start_pin = _start, \ .npins = _nr, \ - .reg_mask = _mask, \ - .val = {0, _mask}, \ - .funcs = {_func1, _func2} \ + .reg_mask = 0, \ + .val = {0}, \ + .funcs = {"gpio"} \ } #define PIN_GRP_GPIO(_name, _start, _nr, _mask, _func1) \ @@ -179,6 +179,7 @@ static struct armada_37xx_pin_group armada_37xx_nb_groups[] = { "pwm", "led"), PIN_GRP_GPIO("pmic1", 7, 1, BIT(7), "pmic"), PIN_GRP_GPIO("pmic0", 6, 1, BIT(8), "pmic"), + PIN_GRP_GPIO_0("gpio1_5", 5, 1), PIN_GRP_GPIO("i2c2", 2, 2, BIT(9), "i2c"), PIN_GRP_GPIO("i2c1", 0, 2, BIT(10), "i2c"), PIN_GRP_GPIO("spi_cs1", 17, 1, BIT(12), "spi"), @@ -195,15 +196,18 @@ static struct armada_37xx_pin_group armada_37xx_nb_groups[] = { static struct armada_37xx_pin_group armada_37xx_sb_groups[] = { PIN_GRP_GPIO("usb32_drvvbus0", 0, 1, BIT(0), "drvbus"), PIN_GRP_GPIO("usb2_drvvbus1", 1, 1, BIT(1), "drvbus"), + PIN_GRP_GPIO_0("gpio2_2", 2, 1), PIN_GRP_GPIO("sdio_sb", 24, 6, BIT(2), "sdio"), PIN_GRP_GPIO("rgmii", 6, 12, BIT(3), "mii"), PIN_GRP_GPIO("smi", 18, 2, BIT(4), "smi"), PIN_GRP_GPIO("pcie1", 3, 1, BIT(5), "pcie"), /* this actually controls "pcie1_reset" */ PIN_GRP_GPIO("pcie1_clkreq", 4, 1, BIT(9), "pcie"), PIN_GRP_GPIO("pcie1_wakeup", 5, 1, BIT(10), "pcie"), - PIN_GRP_GPIO("ptp", 20, 3, BIT(11) | BIT(12) | BIT(13), "ptp"), - PIN_GRP("ptp_clk", 21, 1, BIT(6), "ptp", "mii"), - PIN_GRP("ptp_trig", 22, 1, BIT(7), "ptp", "mii"), + PIN_GRP_GPIO("ptp", 20, 1, BIT(11), "ptp"), + PIN_GRP_GPIO_3("ptp_clk", 21, 1, BIT(6) | BIT(12), 0, BIT(6), BIT(12), + "ptp", "mii"), + PIN_GRP_GPIO_3("ptp_trig", 22, 1, BIT(7) | BIT(13), 0, BIT(7), BIT(13), + "ptp", "mii"), PIN_GRP_GPIO_3("mii_col", 23, 1, BIT(8) | BIT(14), 0, BIT(8), BIT(14), "mii", "mii_err"), }; @@ -486,11 +490,15 @@ static int armada_37xx_gpio_request_enable(struct pinctrl_dev *pctldev, struct armada_37xx_pinctrl *info = pinctrl_dev_get_drvdata(pctldev); struct armada_37xx_pin_group *group; int grp = 0; + int ret; dev_dbg(info->dev, "requesting gpio %d\n", offset); - while ((group = armada_37xx_find_next_grp_by_pin(info, offset, &grp))) - armada_37xx_pmx_set_by_name(pctldev, "gpio", group); + while ((group = armada_37xx_find_next_grp_by_pin(info, offset, &grp))) { + ret = armada_37xx_pmx_set_by_name(pctldev, "gpio", group); + if (ret) + return ret; + } return 0; } diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c index ac3d4d91266d7ba7ae1ac6d7b399681db98b7697..758d21f0a8503a5d4c1548b2d06cae52d34acb06 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c @@ -674,163 +674,160 @@ static const unsigned hwobs_oc4_1_pins[] = { DB8500_PIN_D17, DB8500_PIN_D16, DB8500_PIN_D21, DB8500_PIN_D20, DB8500_PIN_C20, DB8500_PIN_B21, DB8500_PIN_C21, DB8500_PIN_A22, DB8500_PIN_B24, DB8500_PIN_C22 }; -#define DB8500_PIN_GROUP(a, b) { .name = #a, .pins = a##_pins, \ - .npins = ARRAY_SIZE(a##_pins), .altsetting = b } - static const struct nmk_pingroup nmk_db8500_groups[] = { /* Altfunction A column */ - DB8500_PIN_GROUP(u0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(u1rxtx_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(u1ctsrts_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ipi2c_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ipi2c_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp0txrx_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp0tfstck_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp0rfsrck_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc0_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc0_dat47_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc0dat31dir_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp1txrx_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcdb_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcdvsi0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcdvsi1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcd_d0_d7_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcd_d8_d11_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcd_d12_d15_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(lcd_d12_d23_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(kp_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(kpskaskb_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc2_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc2_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ssp1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ssp0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ipgpio0_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(ipgpio1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(modem_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(kp_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp2sck_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(msp2_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc4_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc1_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(mc1dir_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(hsir_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(hsit_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(hsit_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(clkout1_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(clkout1_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(clkout2_a_1, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(clkout2_a_2, NMK_GPIO_ALT_A), - DB8500_PIN_GROUP(usb_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u1rxtx_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u1ctsrts_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ipi2c_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ipi2c_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp0txrx_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp0tfstck_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp0rfsrck_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc0_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc0_dat47_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc0dat31dir_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp1txrx_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcdb_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcdvsi0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcdvsi1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcd_d0_d7_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcd_d8_d11_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcd_d12_d15_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(lcd_d12_d23_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(kp_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(kpskaskb_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc2_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc2_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ssp1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ssp0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ipgpio0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(ipgpio1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(modem_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(kp_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp2sck_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(msp2_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc4_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc1_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mc1dir_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(hsir_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(hsit_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(hsit_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(clkout1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(clkout1_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(clkout2_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(clkout2_a_2, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(usb_a_1, NMK_GPIO_ALT_A), /* Altfunction B column */ - DB8500_PIN_GROUP(trig_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(i2c4_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(i2c1_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(i2c2_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(i2c2_b_2, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(msp0txrx_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(i2c1_b_2, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(u2rxtx_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(uartmodtx_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(msp0sck_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(uartmodrx_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(stmmod_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(uartmodrx_b_2, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(spi3_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(msp1txrx_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(kp_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(kp_b_2, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(sm_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(smcs0_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(smcs1_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(ipgpio7_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(ipgpio2_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(ipgpio3_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(lcdaclk_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(lcda_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(lcd_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(lcd_d16_d23_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(ddrtrig_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(pwl_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(spi1_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(mc3_b_1, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(pwl_b_2, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(pwl_b_3, NMK_GPIO_ALT_B), - DB8500_PIN_GROUP(pwl_b_4, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(trig_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2c4_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2c1_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2c2_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2c2_b_2, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(msp0txrx_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2c1_b_2, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(u2rxtx_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(uartmodtx_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(msp0sck_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(uartmodrx_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(stmmod_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(uartmodrx_b_2, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(spi3_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(msp1txrx_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(kp_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(kp_b_2, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(sm_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(smcs0_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(smcs1_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(ipgpio7_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(ipgpio2_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(ipgpio3_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(lcdaclk_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(lcda_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(lcd_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(lcd_d16_d23_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(ddrtrig_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(pwl_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(spi1_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(mc3_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(pwl_b_2, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(pwl_b_3, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(pwl_b_4, NMK_GPIO_ALT_B), /* Altfunction C column */ - DB8500_PIN_GROUP(ipjtag_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio6_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio0_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio1_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio3_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio2_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(slim0_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ms_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(iptrigout_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(u2rxtx_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(u2ctsrts_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(u0_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio4_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio5_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio6_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio7_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(smcleale_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(stmape_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(u2rxtx_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio2_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio3_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio4_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(ipgpio5_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(mc5_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(mc2rstn_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(kp_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(smps0_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(smps1_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(u2rxtx_c_3, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(stmape_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(uartmodrx_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(uartmodtx_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(stmmod_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(usbsim_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(mc4rstn_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(clkout1_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(clkout2_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(i2c3_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(spi0_c_1, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(usbsim_c_2, NMK_GPIO_ALT_C), - DB8500_PIN_GROUP(i2c3_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipjtag_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio6_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio0_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio1_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio3_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio2_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(slim0_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ms_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(iptrigout_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u2rxtx_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u2ctsrts_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u0_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio4_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio5_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio6_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio7_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(smcleale_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(stmape_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u2rxtx_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio2_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio3_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio4_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(ipgpio5_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(mc5_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(mc2rstn_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(kp_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(smps0_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(smps1_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u2rxtx_c_3, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(stmape_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(uartmodrx_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(uartmodtx_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(stmmod_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(usbsim_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(mc4rstn_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(clkout1_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(clkout2_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(i2c3_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(spi0_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(usbsim_c_2, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(i2c3_c_2, NMK_GPIO_ALT_C), /* Other alt C1 column */ - DB8500_PIN_GROUP(u2rx_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(stmape_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(remap0_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(remap1_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(ptma9_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(kp_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(rf_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(hxclk_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(uartmodrx_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(uartmodtx_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(stmmod_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(hxgpio_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(rf_oc1_2, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(spi2_oc1_1, NMK_GPIO_ALT_C1), - DB8500_PIN_GROUP(spi2_oc1_2, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(u2rx_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(stmape_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(remap0_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(remap1_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(ptma9_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(kp_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(rf_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(hxclk_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(uartmodrx_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(uartmodtx_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(stmmod_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(hxgpio_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(rf_oc1_2, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(spi2_oc1_1, NMK_GPIO_ALT_C1), + NMK_PIN_GROUP(spi2_oc1_2, NMK_GPIO_ALT_C1), /* Other alt C2 column */ - DB8500_PIN_GROUP(sbag_oc2_1, NMK_GPIO_ALT_C2), - DB8500_PIN_GROUP(etmr4_oc2_1, NMK_GPIO_ALT_C2), - DB8500_PIN_GROUP(ptma9_oc2_1, NMK_GPIO_ALT_C2), + NMK_PIN_GROUP(sbag_oc2_1, NMK_GPIO_ALT_C2), + NMK_PIN_GROUP(etmr4_oc2_1, NMK_GPIO_ALT_C2), + NMK_PIN_GROUP(ptma9_oc2_1, NMK_GPIO_ALT_C2), /* Other alt C3 column */ - DB8500_PIN_GROUP(stmmod_oc3_1, NMK_GPIO_ALT_C3), - DB8500_PIN_GROUP(stmmod_oc3_2, NMK_GPIO_ALT_C3), - DB8500_PIN_GROUP(uartmodrx_oc3_1, NMK_GPIO_ALT_C3), - DB8500_PIN_GROUP(uartmodtx_oc3_1, NMK_GPIO_ALT_C3), - DB8500_PIN_GROUP(etmr4_oc3_1, NMK_GPIO_ALT_C3), + NMK_PIN_GROUP(stmmod_oc3_1, NMK_GPIO_ALT_C3), + NMK_PIN_GROUP(stmmod_oc3_2, NMK_GPIO_ALT_C3), + NMK_PIN_GROUP(uartmodrx_oc3_1, NMK_GPIO_ALT_C3), + NMK_PIN_GROUP(uartmodtx_oc3_1, NMK_GPIO_ALT_C3), + NMK_PIN_GROUP(etmr4_oc3_1, NMK_GPIO_ALT_C3), /* Other alt C4 column */ - DB8500_PIN_GROUP(sbag_oc4_1, NMK_GPIO_ALT_C4), - DB8500_PIN_GROUP(hwobs_oc4_1, NMK_GPIO_ALT_C4), + NMK_PIN_GROUP(sbag_oc4_1, NMK_GPIO_ALT_C4), + NMK_PIN_GROUP(hwobs_oc4_1, NMK_GPIO_ALT_C4), }; /* We use this macro to define the groups applicable to a function */ diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c b/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c index 8d944bb3a036c22fd8a8bb80170d8c63fcdb99a8..c0d7c86d09391035b6d3d9f8cde690af9f2afb4c 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c @@ -303,23 +303,20 @@ static const unsigned usbhs_c_1_pins[] = { STN8815_PIN_E21, STN8815_PIN_E20, STN8815_PIN_C16, STN8815_PIN_A15, STN8815_PIN_D17, STN8815_PIN_C17 }; -#define STN8815_PIN_GROUP(a, b) { .name = #a, .pins = a##_pins, \ - .npins = ARRAY_SIZE(a##_pins), .altsetting = b } - static const struct nmk_pingroup nmk_stn8815_groups[] = { - STN8815_PIN_GROUP(u0txrx_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(u0ctsrts_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(u0modem_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(mmcsd_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(mmcsd_b_1, NMK_GPIO_ALT_B), - STN8815_PIN_GROUP(u1_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(i2c1_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A), - STN8815_PIN_GROUP(u1_b_1, NMK_GPIO_ALT_B), - STN8815_PIN_GROUP(i2cusb_b_1, NMK_GPIO_ALT_B), - STN8815_PIN_GROUP(clcd_16_23_b_1, NMK_GPIO_ALT_B), - STN8815_PIN_GROUP(usbfs_b_1, NMK_GPIO_ALT_B), - STN8815_PIN_GROUP(usbhs_c_1, NMK_GPIO_ALT_C), + NMK_PIN_GROUP(u0txrx_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u0ctsrts_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u0modem_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mmcsd_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(mmcsd_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(u1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(i2c1_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(i2c0_a_1, NMK_GPIO_ALT_A), + NMK_PIN_GROUP(u1_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(i2cusb_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(clcd_16_23_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(usbfs_b_1, NMK_GPIO_ALT_B), + NMK_PIN_GROUP(usbhs_c_1, NMK_GPIO_ALT_C), }; /* We use this macro to define the groups applicable to a function */ diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.c b/drivers/pinctrl/nomadik/pinctrl-nomadik.c index f5014d09d81a222bcc39eff62a01ade384f08839..f7d02513d8cc103f98cdc5752dd0abb17df93020 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.c +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.c @@ -244,7 +244,6 @@ enum nmk_gpio_slpm { struct nmk_gpio_chip { struct gpio_chip chip; - struct irq_chip irqchip; void __iomem *addr; struct clk *clk; unsigned int bank; @@ -608,8 +607,8 @@ static int __maybe_unused nmk_prcm_gpiocr_get_mode(struct pinctrl_dev *pctldev, static void nmk_gpio_irq_ack(struct irq_data *d) { - struct gpio_chip *chip = irq_data_get_irq_chip_data(d); - struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(chip); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); clk_enable(nmk_chip->clk); writel(BIT(d->hwirq), nmk_chip->addr + NMK_GPIO_IC); @@ -675,15 +674,11 @@ static void __nmk_gpio_set_wake(struct nmk_gpio_chip *nmk_chip, __nmk_gpio_irq_modify(nmk_chip, offset, WAKE, on); } -static int nmk_gpio_irq_maskunmask(struct irq_data *d, bool enable) +static void nmk_gpio_irq_maskunmask(struct nmk_gpio_chip *nmk_chip, + struct irq_data *d, bool enable) { - struct nmk_gpio_chip *nmk_chip; unsigned long flags; - nmk_chip = irq_data_get_irq_chip_data(d); - if (!nmk_chip) - return -EINVAL; - clk_enable(nmk_chip->clk); spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); spin_lock(&nmk_chip->lock); @@ -696,29 +691,32 @@ static int nmk_gpio_irq_maskunmask(struct irq_data *d, bool enable) spin_unlock(&nmk_chip->lock); spin_unlock_irqrestore(&nmk_gpio_slpm_lock, flags); clk_disable(nmk_chip->clk); - - return 0; } static void nmk_gpio_irq_mask(struct irq_data *d) { - nmk_gpio_irq_maskunmask(d, false); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + nmk_gpio_irq_maskunmask(nmk_chip, d, false); + gpiochip_disable_irq(gc, irqd_to_hwirq(d)); } static void nmk_gpio_irq_unmask(struct irq_data *d) { - nmk_gpio_irq_maskunmask(d, true); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + gpiochip_enable_irq(gc, irqd_to_hwirq(d)); + nmk_gpio_irq_maskunmask(nmk_chip, d, true); } static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on) { - struct nmk_gpio_chip *nmk_chip; + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); unsigned long flags; - nmk_chip = irq_data_get_irq_chip_data(d); - if (!nmk_chip) - return -EINVAL; - clk_enable(nmk_chip->clk); spin_lock_irqsave(&nmk_gpio_slpm_lock, flags); spin_lock(&nmk_chip->lock); @@ -740,14 +738,12 @@ static int nmk_gpio_irq_set_wake(struct irq_data *d, unsigned int on) static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); bool enabled = !irqd_irq_disabled(d); bool wake = irqd_is_wakeup_set(d); - struct nmk_gpio_chip *nmk_chip; unsigned long flags; - nmk_chip = irq_data_get_irq_chip_data(d); - if (!nmk_chip) - return -EINVAL; if (type & IRQ_TYPE_LEVEL_HIGH) return -EINVAL; if (type & IRQ_TYPE_LEVEL_LOW) @@ -784,7 +780,8 @@ static int nmk_gpio_irq_set_type(struct irq_data *d, unsigned int type) static unsigned int nmk_gpio_irq_startup(struct irq_data *d) { - struct nmk_gpio_chip *nmk_chip = irq_data_get_irq_chip_data(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); clk_enable(nmk_chip->clk); nmk_gpio_irq_unmask(d); @@ -793,7 +790,8 @@ static unsigned int nmk_gpio_irq_startup(struct irq_data *d) static void nmk_gpio_irq_shutdown(struct irq_data *d) { - struct nmk_gpio_chip *nmk_chip = irq_data_get_irq_chip_data(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); nmk_gpio_irq_mask(d); clk_disable(nmk_chip->clk); @@ -1078,13 +1076,34 @@ static struct nmk_gpio_chip *nmk_gpio_populate_chip(struct device_node *np, return nmk_chip; } +static void nmk_gpio_irq_print_chip(struct irq_data *d, struct seq_file *p) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct nmk_gpio_chip *nmk_chip = gpiochip_get_data(gc); + + seq_printf(p, "nmk%u-%u-%u", nmk_chip->bank, + gc->base, gc->base + gc->ngpio - 1); +} + +static const struct irq_chip nmk_irq_chip = { + .irq_ack = nmk_gpio_irq_ack, + .irq_mask = nmk_gpio_irq_mask, + .irq_unmask = nmk_gpio_irq_unmask, + .irq_set_type = nmk_gpio_irq_set_type, + .irq_set_wake = nmk_gpio_irq_set_wake, + .irq_startup = nmk_gpio_irq_startup, + .irq_shutdown = nmk_gpio_irq_shutdown, + .irq_print_chip = nmk_gpio_irq_print_chip, + .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int nmk_gpio_probe(struct platform_device *dev) { struct device_node *np = dev->dev.of_node; struct nmk_gpio_chip *nmk_chip; struct gpio_chip *chip; struct gpio_irq_chip *girq; - struct irq_chip *irqchip; bool supports_sleepmode; int irq; int ret; @@ -1125,22 +1144,8 @@ static int nmk_gpio_probe(struct platform_device *dev) chip->can_sleep = false; chip->owner = THIS_MODULE; - irqchip = &nmk_chip->irqchip; - irqchip->irq_ack = nmk_gpio_irq_ack; - irqchip->irq_mask = nmk_gpio_irq_mask; - irqchip->irq_unmask = nmk_gpio_irq_unmask; - irqchip->irq_set_type = nmk_gpio_irq_set_type; - irqchip->irq_set_wake = nmk_gpio_irq_set_wake; - irqchip->irq_startup = nmk_gpio_irq_startup; - irqchip->irq_shutdown = nmk_gpio_irq_shutdown; - irqchip->flags = IRQCHIP_MASK_ON_SUSPEND; - irqchip->name = kasprintf(GFP_KERNEL, "nmk%u-%u-%u", - dev->id, - chip->base, - chip->base + chip->ngpio - 1); - girq = &chip->irq; - girq->chip = irqchip; + gpio_irq_chip_set_chip(girq, &nmk_irq_chip); girq->parent_handler = nmk_gpio_irq_handler; girq->num_parents = 1; girq->parents = devm_kcalloc(&dev->dev, 1, @@ -1179,17 +1184,17 @@ static const char *nmk_get_group_name(struct pinctrl_dev *pctldev, { struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev); - return npct->soc->groups[selector].name; + return npct->soc->groups[selector].grp.name; } static int nmk_get_group_pins(struct pinctrl_dev *pctldev, unsigned selector, const unsigned **pins, - unsigned *num_pins) + unsigned *npins) { struct nmk_pinctrl *npct = pinctrl_dev_get_drvdata(pctldev); - *pins = npct->soc->groups[selector].pins; - *num_pins = npct->soc->groups[selector].npins; + *pins = npct->soc->groups[selector].grp.pins; + *npins = npct->soc->groups[selector].grp.npins; return 0; } @@ -1531,7 +1536,7 @@ static int nmk_pmx_set(struct pinctrl_dev *pctldev, unsigned function, if (g->altsetting < 0) return -EINVAL; - dev_dbg(npct->dev, "enable group %s, %u pins\n", g->name, g->npins); + dev_dbg(npct->dev, "enable group %s, %u pins\n", g->grp.name, g->grp.npins); /* * If we're setting altfunc C by setting both AFSLA and AFSLB to 1, @@ -1566,26 +1571,26 @@ static int nmk_pmx_set(struct pinctrl_dev *pctldev, unsigned function, * Then mask the pins that need to be sleeping now when we're * switching to the ALT C function. */ - for (i = 0; i < g->npins; i++) - slpm[g->pins[i] / NMK_GPIO_PER_CHIP] &= ~BIT(g->pins[i]); + for (i = 0; i < g->grp.npins; i++) + slpm[g->grp.pins[i] / NMK_GPIO_PER_CHIP] &= ~BIT(g->grp.pins[i]); nmk_gpio_glitch_slpm_init(slpm); } - for (i = 0; i < g->npins; i++) { + for (i = 0; i < g->grp.npins; i++) { struct nmk_gpio_chip *nmk_chip; unsigned bit; - nmk_chip = find_nmk_gpio_from_pin(g->pins[i]); + nmk_chip = find_nmk_gpio_from_pin(g->grp.pins[i]); if (!nmk_chip) { dev_err(npct->dev, "invalid pin offset %d in group %s at index %d\n", - g->pins[i], g->name, i); + g->grp.pins[i], g->grp.name, i); goto out_glitch; } - dev_dbg(npct->dev, "setting pin %d to altsetting %d\n", g->pins[i], g->altsetting); + dev_dbg(npct->dev, "setting pin %d to altsetting %d\n", g->grp.pins[i], g->altsetting); clk_enable(nmk_chip->clk); - bit = g->pins[i] % NMK_GPIO_PER_CHIP; + bit = g->grp.pins[i] % NMK_GPIO_PER_CHIP; /* * If the pin is switching to altfunc, and there was an * interrupt installed on it which has been lazy disabled, @@ -1608,7 +1613,7 @@ static int nmk_pmx_set(struct pinctrl_dev *pctldev, unsigned function, * then some bits in PRCM GPIOCR registers must be cleared. */ if ((g->altsetting & NMK_GPIO_ALT_C) == NMK_GPIO_ALT_C) - nmk_prcm_altcx_set_mode(npct, g->pins[i], + nmk_prcm_altcx_set_mode(npct, g->grp.pins[i], g->altsetting >> NMK_GPIO_ALT_CX_SHIFT); } @@ -1802,10 +1807,6 @@ static const struct of_device_id nmk_pinctrl_match[] = { .compatible = "stericsson,db8500-pinctrl", .data = (void *)PINCTRL_NMK_DB8500, }, - { - .compatible = "stericsson,db8540-pinctrl", - .data = (void *)PINCTRL_NMK_DB8540, - }, {}, }; @@ -1856,8 +1857,6 @@ static int nmk_pinctrl_probe(struct platform_device *pdev) nmk_pinctrl_stn8815_init(&npct->soc); if (version == PINCTRL_NMK_DB8500) nmk_pinctrl_db8500_init(&npct->soc); - if (version == PINCTRL_NMK_DB8540) - nmk_pinctrl_db8540_init(&npct->soc); /* * Since we depend on the GPIO chips to provide clock and register base diff --git a/drivers/pinctrl/nomadik/pinctrl-nomadik.h b/drivers/pinctrl/nomadik/pinctrl-nomadik.h index ae0bac06639fe7ca621ea1c47523fcaaff4de328..84e297757335756a8a6e5efc298b397e1ee0a707 100644 --- a/drivers/pinctrl/nomadik/pinctrl-nomadik.h +++ b/drivers/pinctrl/nomadik/pinctrl-nomadik.h @@ -5,7 +5,6 @@ /* Package definitions */ #define PINCTRL_NMK_STN8815 0 #define PINCTRL_NMK_DB8500 1 -#define PINCTRL_NMK_DB8540 2 /* Alternate functions: function C is set in hw by setting both A and B */ #define NMK_GPIO_ALT_GPIO 0 @@ -105,21 +104,21 @@ struct nmk_function { /** * struct nmk_pingroup - describes a Nomadik pin group - * @name: the name of this specific pin group - * @pins: an array of discrete physical pins used in this group, taken - * from the driver-local pin enumeration space - * @num_pins: the number of pins in this group array, i.e. the number of - * elements in .pins so we can iterate over that array + * @grp: Generic data of the pin group (name and pins) * @altsetting: the altsetting to apply to all pins in this group to * configure them to be used by a function */ struct nmk_pingroup { - const char *name; - const unsigned int *pins; - const unsigned npins; + struct pingroup grp; int altsetting; }; +#define NMK_PIN_GROUP(a, b) \ + { \ + .grp = PINCTRL_PINGROUP(#a, a##_pins, ARRAY_SIZE(a##_pins)), \ + .altsetting = b, \ + } + /** * struct nmk_pinctrl_soc_data - Nomadik pin controller per-SoC configuration * @pins: An array describing all pins the pin controller affects. @@ -173,17 +172,4 @@ nmk_pinctrl_db8500_init(const struct nmk_pinctrl_soc_data **soc) #endif -#ifdef CONFIG_PINCTRL_DB8540 - -void nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc); - -#else - -static inline void -nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc) -{ -} - -#endif - #endif /* PINCTRL_PINCTRL_NOMADIK_H */ diff --git a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c index 64d8a568b3dbd3ec4e056565f0c8b1c3a3829696..1c4e89b046de191928f5cb3a9d66a614241fdc8b 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c +++ b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c @@ -81,11 +81,11 @@ struct npcm7xx_gpio { int irq; struct irq_chip irq_chip; u32 pinctrl_id; - int (*direction_input)(struct gpio_chip *chip, unsigned offset); - int (*direction_output)(struct gpio_chip *chip, unsigned offset, + int (*direction_input)(struct gpio_chip *chip, unsigned int offset); + int (*direction_output)(struct gpio_chip *chip, unsigned int offset, int value); - int (*request)(struct gpio_chip *chip, unsigned offset); - void (*free)(struct gpio_chip *chip, unsigned offset); + int (*request)(struct gpio_chip *chip, unsigned int offset); + void (*free)(struct gpio_chip *chip, unsigned int offset); }; struct npcm7xx_pinctrl { diff --git a/drivers/pinctrl/nuvoton/pinctrl-wpcm450.c b/drivers/pinctrl/nuvoton/pinctrl-wpcm450.c index 0dbeb91f0bf27db2098c548c71d3aaea6266ca78..8193b92da4031142c6962760105fb8526235cd26 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-wpcm450.c +++ b/drivers/pinctrl/nuvoton/pinctrl-wpcm450.c @@ -1081,10 +1081,13 @@ static int wpcm450_gpio_register(struct platform_device *pdev, girq->num_parents = 0; for (i = 0; i < WPCM450_NUM_GPIO_IRQS; i++) { - int irq = fwnode_irq_get(child, i); + int irq; + irq = fwnode_irq_get(child, i); if (irq < 0) break; + if (!irq) + continue; girq->parents[i] = irq; girq->num_parents++; diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index 4691a33bc374fc9e5dddc6bd250ce1610ff47686..6be896871718224213d2039f853abcbf005e0dd8 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -246,7 +246,7 @@ static void amd_gpio_dbg_show(struct seq_file *s, struct gpio_chip *gc) } seq_printf(s, "GPIO bank%d\n", bank); for (; i < pin_num; i++) { - seq_printf(s, "📌%d\t", i); + seq_printf(s, "#%d\t", i); raw_spin_lock_irqsave(&gpio_dev->lock, flags); pin_reg = readl(gpio_dev->base + i * 4); raw_spin_unlock_irqrestore(&gpio_dev->lock, flags); @@ -278,32 +278,32 @@ static void amd_gpio_dbg_show(struct seq_file *s, struct gpio_chip *gc) } if (pin_reg & BIT(INTERRUPT_MASK_OFF)) - interrupt_mask = "-"; + interrupt_mask = "😛"; else - interrupt_mask = "+"; - seq_printf(s, "int %s (🎭 %s)| active-%s| %s-🔫| ", + interrupt_mask = "😷"; + seq_printf(s, "int %s (%s)| active-%s| %s-⚡| ", interrupt_enable, interrupt_mask, active_level, level_trig); if (pin_reg & BIT(WAKE_CNTRL_OFF_S0I3)) - wake_cntrl0 = "+"; + wake_cntrl0 = "⏰"; else - wake_cntrl0 = "∅"; - seq_printf(s, "S0i3 🌅 %s| ", wake_cntrl0); + wake_cntrl0 = " ∅"; + seq_printf(s, "S0i3 %s| ", wake_cntrl0); if (pin_reg & BIT(WAKE_CNTRL_OFF_S3)) - wake_cntrl1 = "+"; + wake_cntrl1 = "⏰"; else - wake_cntrl1 = "∅"; - seq_printf(s, "S3 🌅 %s| ", wake_cntrl1); + wake_cntrl1 = " ∅"; + seq_printf(s, "S3 %s| ", wake_cntrl1); if (pin_reg & BIT(WAKE_CNTRL_OFF_S4)) - wake_cntrl2 = "+"; + wake_cntrl2 = "⏰"; else - wake_cntrl2 = "∅"; - seq_printf(s, "S4/S5 🌅 %s| ", wake_cntrl2); + wake_cntrl2 = " ∅"; + seq_printf(s, "S4/S5 %s| ", wake_cntrl2); if (pin_reg & BIT(PULL_UP_ENABLE_OFF)) { pull_up_enable = "+"; @@ -367,7 +367,7 @@ static void amd_gpio_dbg_show(struct seq_file *s, struct gpio_chip *gc) debounce_enable = " ∅"; } snprintf(debounce_value, sizeof(debounce_value), "%u", time * unit); - seq_printf(s, "debounce %s (⏰ %sus)| ", debounce_enable, debounce_value); + seq_printf(s, "debounce %s (🕑 %sus)| ", debounce_enable, debounce_value); seq_printf(s, " 0x%x\n", pin_reg); } } @@ -639,7 +639,7 @@ static bool do_amd_gpio_irq_handler(int irq, void *dev_id) if (!(regval & PIN_IRQ_PENDING) || !(regval & BIT(INTERRUPT_MASK_OFF))) continue; - generic_handle_domain_irq(gc->irq.domain, irqnr + i); + generic_handle_domain_irq_safe(gc->irq.domain, irqnr + i); /* Clear interrupt. * We must read the pin register again, in case the @@ -1051,13 +1051,13 @@ static void amd_get_iomux_res(struct amd_gpio *gpio_dev) index = device_property_match_string(dev, "pinctrl-resource-names", "iomux"); if (index < 0) { - dev_warn(dev, "failed to get iomux index\n"); + dev_dbg(dev, "iomux not supported\n"); goto out_no_pinmux; } gpio_dev->iomux_base = devm_platform_ioremap_resource(gpio_dev->pdev, index); if (IS_ERR(gpio_dev->iomux_base)) { - dev_warn(dev, "Failed to get iomux %d io resource\n", index); + dev_dbg(dev, "iomux not supported %d io resource\n", index); goto out_no_pinmux; } diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c index 5634fa063ebfe9eb72249f966fce0d0c0f7cb9b0..81dbffab621fb728d482d2f2b1be840dde9d589f 100644 --- a/drivers/pinctrl/pinctrl-at91.c +++ b/drivers/pinctrl/pinctrl-at91.c @@ -22,8 +22,7 @@ #include /* Since we request GPIOs from ourself */ #include - -#include +#include #include "pinctrl-at91.h" #include "core.h" @@ -33,16 +32,34 @@ struct at91_pinctrl_mux_ops; +/** + * struct at91_gpio_chip: at91 gpio chip + * @chip: gpio chip + * @range: gpio range + * @next: bank sharing same clock + * @pioc_hwirq: PIO bank interrupt identifier on AIC + * @pioc_virq: PIO bank Linux virtual interrupt + * @pioc_idx: PIO bank index + * @regbase: PIO bank virtual address + * @clock: associated clock + * @ops: at91 pinctrl mux ops + * @wakeups: wakeup interrupts + * @backups: interrupts disabled in suspend + * @id: gpio chip identifier + */ struct at91_gpio_chip { struct gpio_chip chip; struct pinctrl_gpio_range range; - struct at91_gpio_chip *next; /* Bank sharing same clock */ - int pioc_hwirq; /* PIO bank interrupt identifier on AIC */ - int pioc_virq; /* PIO bank Linux virtual interrupt */ - int pioc_idx; /* PIO bank index */ - void __iomem *regbase; /* PIO bank virtual address */ - struct clk *clock; /* associated clock */ - const struct at91_pinctrl_mux_ops *ops; /* ops */ + struct at91_gpio_chip *next; + int pioc_hwirq; + int pioc_virq; + int pioc_idx; + void __iomem *regbase; + struct clk *clock; + const struct at91_pinctrl_mux_ops *ops; + u32 wakeups; + u32 backups; + u32 id; }; static struct at91_gpio_chip *gpio_chips[MAX_GPIO_BANKS]; @@ -1615,70 +1632,51 @@ static void gpio_irq_ack(struct irq_data *d) /* the interrupt is already cleared before by reading ISR */ } -static u32 wakeups[MAX_GPIO_BANKS]; -static u32 backups[MAX_GPIO_BANKS]; - static int gpio_irq_set_wake(struct irq_data *d, unsigned state) { struct at91_gpio_chip *at91_gpio = irq_data_get_irq_chip_data(d); - unsigned bank = at91_gpio->pioc_idx; unsigned mask = 1 << d->hwirq; - if (unlikely(bank >= MAX_GPIO_BANKS)) - return -EINVAL; - if (state) - wakeups[bank] |= mask; + at91_gpio->wakeups |= mask; else - wakeups[bank] &= ~mask; + at91_gpio->wakeups &= ~mask; irq_set_irq_wake(at91_gpio->pioc_virq, state); return 0; } -void at91_pinctrl_gpio_suspend(void) +static int at91_gpio_suspend(struct device *dev) { - int i; - - for (i = 0; i < gpio_banks; i++) { - void __iomem *pio; + struct at91_gpio_chip *at91_chip = dev_get_drvdata(dev); + void __iomem *pio = at91_chip->regbase; - if (!gpio_chips[i]) - continue; - - pio = gpio_chips[i]->regbase; + at91_chip->backups = readl_relaxed(pio + PIO_IMR); + writel_relaxed(at91_chip->backups, pio + PIO_IDR); + writel_relaxed(at91_chip->wakeups, pio + PIO_IER); - backups[i] = readl_relaxed(pio + PIO_IMR); - writel_relaxed(backups[i], pio + PIO_IDR); - writel_relaxed(wakeups[i], pio + PIO_IER); + if (!at91_chip->wakeups) + clk_disable_unprepare(at91_chip->clock); + else + dev_dbg(dev, "GPIO-%c may wake for %08x\n", + 'A' + at91_chip->id, at91_chip->wakeups); - if (!wakeups[i]) - clk_disable_unprepare(gpio_chips[i]->clock); - else - printk(KERN_DEBUG "GPIO-%c may wake for %08x\n", - 'A'+i, wakeups[i]); - } + return 0; } -void at91_pinctrl_gpio_resume(void) +static int at91_gpio_resume(struct device *dev) { - int i; - - for (i = 0; i < gpio_banks; i++) { - void __iomem *pio; + struct at91_gpio_chip *at91_chip = dev_get_drvdata(dev); + void __iomem *pio = at91_chip->regbase; - if (!gpio_chips[i]) - continue; - - pio = gpio_chips[i]->regbase; + if (!at91_chip->wakeups) + clk_prepare_enable(at91_chip->clock); - if (!wakeups[i]) - clk_prepare_enable(gpio_chips[i]->clock); + writel_relaxed(at91_chip->wakeups, pio + PIO_IDR); + writel_relaxed(at91_chip->backups, pio + PIO_IER); - writel_relaxed(wakeups[i], pio + PIO_IDR); - writel_relaxed(backups[i], pio + PIO_IER); - } + return 0; } static void gpio_irq_handler(struct irq_desc *desc) @@ -1860,6 +1858,7 @@ static int at91_gpio_probe(struct platform_device *pdev) } at91_chip->chip = at91_gpio_template; + at91_chip->id = alias_idx; chip = &at91_chip->chip; chip->label = dev_name(&pdev->dev); @@ -1905,6 +1904,7 @@ static int at91_gpio_probe(struct platform_device *pdev) goto gpiochip_add_err; gpio_chips[alias_idx] = at91_chip; + platform_set_drvdata(pdev, at91_chip); gpio_banks = max(gpio_banks, alias_idx + 1); dev_info(&pdev->dev, "at address %p\n", at91_chip->regbase); @@ -1920,10 +1920,15 @@ err: return ret; } +static const struct dev_pm_ops at91_gpio_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(at91_gpio_suspend, at91_gpio_resume) +}; + static struct platform_driver at91_gpio_driver = { .driver = { .name = "gpio-at91", .of_match_table = at91_gpio_of_match, + .pm = pm_ptr(&at91_gpio_pm_ops), }, .probe = at91_gpio_probe, }; diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c new file mode 100644 index 0000000000000000000000000000000000000000..68509a2301b8fa252d26faa66c1a8384bef4d21f --- /dev/null +++ b/drivers/pinctrl/pinctrl-cy8c95x0.c @@ -0,0 +1,1419 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CY8C95X0 20/40/60 pin I2C GPIO port expander with interrupt support + * + * Copyright (C) 2022 9elements GmbH + * Authors: Patrick Rudolph + * Naresh Solanki + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/* Fast access registers */ +#define CY8C95X0_INPUT 0x00 +#define CY8C95X0_OUTPUT 0x08 +#define CY8C95X0_INTSTATUS 0x10 + +#define CY8C95X0_INPUT_(x) (CY8C95X0_INPUT + (x)) +#define CY8C95X0_OUTPUT_(x) (CY8C95X0_OUTPUT + (x)) +#define CY8C95X0_INTSTATUS_(x) (CY8C95X0_INTSTATUS + (x)) + +/* Port Select configures the port */ +#define CY8C95X0_PORTSEL 0x18 +/* Port settings, write PORTSEL first */ +#define CY8C95X0_INTMASK 0x19 +#define CY8C95X0_PWMSEL 0x1A +#define CY8C95X0_INVERT 0x1B +#define CY8C95X0_DIRECTION 0x1C +/* Drive mode register change state on writing '1' */ +#define CY8C95X0_DRV_PU 0x1D +#define CY8C95X0_DRV_PD 0x1E +#define CY8C95X0_DRV_ODH 0x1F +#define CY8C95X0_DRV_ODL 0x20 +#define CY8C95X0_DRV_PP_FAST 0x21 +#define CY8C95X0_DRV_PP_SLOW 0x22 +#define CY8C95X0_DRV_HIZ 0x23 +#define CY8C95X0_DEVID 0x2E +#define CY8C95X0_WATCHDOG 0x2F +#define CY8C95X0_COMMAND 0x30 + +#define CY8C95X0_PIN_TO_OFFSET(x) (((x) >= 20) ? ((x) + 4) : (x)) + +static const struct i2c_device_id cy8c95x0_id[] = { + { "cy8c9520", 20, }, + { "cy8c9540", 40, }, + { "cy8c9560", 60, }, + { } +}; +MODULE_DEVICE_TABLE(i2c, cy8c95x0_id); + +#define OF_CY8C95X(__nrgpio) ((void *)(__nrgpio)) + +static const struct of_device_id cy8c95x0_dt_ids[] = { + { .compatible = "cypress,cy8c9520", .data = OF_CY8C95X(20), }, + { .compatible = "cypress,cy8c9540", .data = OF_CY8C95X(40), }, + { .compatible = "cypress,cy8c9560", .data = OF_CY8C95X(60), }, + { } +}; +MODULE_DEVICE_TABLE(of, cy8c95x0_dt_ids); + +static const struct acpi_gpio_params cy8c95x0_irq_gpios = { 0, 0, true }; + +static const struct acpi_gpio_mapping cy8c95x0_acpi_irq_gpios[] = { + { "irq-gpios", &cy8c95x0_irq_gpios, 1, ACPI_GPIO_QUIRK_ABSOLUTE_NUMBER }, + { } +}; + +static int cy8c95x0_acpi_get_irq(struct device *dev) +{ + int ret; + + ret = devm_acpi_dev_add_driver_gpios(dev, cy8c95x0_acpi_irq_gpios); + if (ret) + dev_warn(dev, "can't add GPIO ACPI mapping\n"); + + ret = acpi_dev_gpio_irq_get_by(ACPI_COMPANION(dev), "irq-gpios", 0); + if (ret < 0) + return ret; + + dev_info(dev, "ACPI interrupt quirk (IRQ %d)\n", ret); + return ret; +} + +static const struct dmi_system_id cy8c95x0_dmi_acpi_irq_info[] = { + { + /* + * On Intel Galileo Gen 1 board the IRQ pin is provided + * as an absolute number instead of being relative. + * Since first controller (gpio-sch.c) and second + * (gpio-dwapb.c) are at the fixed bases, we may safely + * refer to the number in the global space to get an IRQ + * out of it. + */ + .matches = { + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Galileo"), + }, + }, + {} +}; + +#define MAX_BANK 8 +#define BANK_SZ 8 +#define MAX_LINE (MAX_BANK * BANK_SZ) + +#define CY8C95X0_GPIO_MASK GENMASK(7, 0) + +/** + * struct cy8c95x0_pinctrl - driver data + * @regmap: Device's regmap + * @irq_lock: IRQ bus lock + * @i2c_lock: Mutex for the device internal mux register + * @irq_mask: I/O bits affected by interrupts + * @irq_trig_raise: I/O bits affected by raising voltage level + * @irq_trig_fall: I/O bits affected by falling voltage level + * @irq_trig_low: I/O bits affected by a low voltage level + * @irq_trig_high: I/O bits affected by a high voltage level + * @push_pull: I/O bits configured as push pull driver + * @shiftmask: Mask used to compensate for Gport2 width + * @nport: Number of Gports in this chip + * @gpio_chip: gpiolib chip + * @driver_data: private driver data + * @regulator: Pointer to the regulator for the IC + * @dev: struct device + * @pctldev: pin controller device + * @pinctrl_desc: pin controller description + * @name: Chip controller name + * @tpin: Total number of pins + */ +struct cy8c95x0_pinctrl { + struct regmap *regmap; + struct mutex irq_lock; + struct mutex i2c_lock; + DECLARE_BITMAP(irq_mask, MAX_LINE); + DECLARE_BITMAP(irq_trig_raise, MAX_LINE); + DECLARE_BITMAP(irq_trig_fall, MAX_LINE); + DECLARE_BITMAP(irq_trig_low, MAX_LINE); + DECLARE_BITMAP(irq_trig_high, MAX_LINE); + DECLARE_BITMAP(push_pull, MAX_LINE); + DECLARE_BITMAP(shiftmask, MAX_LINE); + int nport; + struct gpio_chip gpio_chip; + unsigned long driver_data; + struct regulator *regulator; + struct device *dev; + struct pinctrl_dev *pctldev; + struct pinctrl_desc pinctrl_desc; + char name[32]; + unsigned int tpin; +}; + +static const struct pinctrl_pin_desc cy8c9560_pins[] = { + PINCTRL_PIN(0, "gp00"), + PINCTRL_PIN(1, "gp01"), + PINCTRL_PIN(2, "gp02"), + PINCTRL_PIN(3, "gp03"), + PINCTRL_PIN(4, "gp04"), + PINCTRL_PIN(5, "gp05"), + PINCTRL_PIN(6, "gp06"), + PINCTRL_PIN(7, "gp07"), + + PINCTRL_PIN(8, "gp10"), + PINCTRL_PIN(9, "gp11"), + PINCTRL_PIN(10, "gp12"), + PINCTRL_PIN(11, "gp13"), + PINCTRL_PIN(12, "gp14"), + PINCTRL_PIN(13, "gp15"), + PINCTRL_PIN(14, "gp16"), + PINCTRL_PIN(15, "gp17"), + + PINCTRL_PIN(16, "gp20"), + PINCTRL_PIN(17, "gp21"), + PINCTRL_PIN(18, "gp22"), + PINCTRL_PIN(19, "gp23"), + + PINCTRL_PIN(20, "gp30"), + PINCTRL_PIN(21, "gp31"), + PINCTRL_PIN(22, "gp32"), + PINCTRL_PIN(23, "gp33"), + PINCTRL_PIN(24, "gp34"), + PINCTRL_PIN(25, "gp35"), + PINCTRL_PIN(26, "gp36"), + PINCTRL_PIN(27, "gp37"), + + PINCTRL_PIN(28, "gp40"), + PINCTRL_PIN(29, "gp41"), + PINCTRL_PIN(30, "gp42"), + PINCTRL_PIN(31, "gp43"), + PINCTRL_PIN(32, "gp44"), + PINCTRL_PIN(33, "gp45"), + PINCTRL_PIN(34, "gp46"), + PINCTRL_PIN(35, "gp47"), + + PINCTRL_PIN(36, "gp50"), + PINCTRL_PIN(37, "gp51"), + PINCTRL_PIN(38, "gp52"), + PINCTRL_PIN(39, "gp53"), + PINCTRL_PIN(40, "gp54"), + PINCTRL_PIN(41, "gp55"), + PINCTRL_PIN(42, "gp56"), + PINCTRL_PIN(43, "gp57"), + + PINCTRL_PIN(44, "gp60"), + PINCTRL_PIN(45, "gp61"), + PINCTRL_PIN(46, "gp62"), + PINCTRL_PIN(47, "gp63"), + PINCTRL_PIN(48, "gp64"), + PINCTRL_PIN(49, "gp65"), + PINCTRL_PIN(50, "gp66"), + PINCTRL_PIN(51, "gp67"), + + PINCTRL_PIN(52, "gp70"), + PINCTRL_PIN(53, "gp71"), + PINCTRL_PIN(54, "gp72"), + PINCTRL_PIN(55, "gp73"), + PINCTRL_PIN(56, "gp74"), + PINCTRL_PIN(57, "gp75"), + PINCTRL_PIN(58, "gp76"), + PINCTRL_PIN(59, "gp77"), +}; + +static const char * const cy8c95x0_groups[] = { + "gp00", + "gp01", + "gp02", + "gp03", + "gp04", + "gp05", + "gp06", + "gp07", + + "gp10", + "gp11", + "gp12", + "gp13", + "gp14", + "gp15", + "gp16", + "gp17", + + "gp20", + "gp21", + "gp22", + "gp23", + + "gp30", + "gp31", + "gp32", + "gp33", + "gp34", + "gp35", + "gp36", + "gp37", + + "gp40", + "gp41", + "gp42", + "gp43", + "gp44", + "gp45", + "gp46", + "gp47", + + "gp50", + "gp51", + "gp52", + "gp53", + "gp54", + "gp55", + "gp56", + "gp57", + + "gp60", + "gp61", + "gp62", + "gp63", + "gp64", + "gp65", + "gp66", + "gp67", + + "gp70", + "gp71", + "gp72", + "gp73", + "gp74", + "gp75", + "gp76", + "gp77", +}; + +static inline u8 cypress_get_port(struct cy8c95x0_pinctrl *chip, unsigned int pin) +{ + /* Account for GPORT2 which only has 4 bits */ + return CY8C95X0_PIN_TO_OFFSET(pin) / BANK_SZ; +} + +static int cypress_get_pin_mask(struct cy8c95x0_pinctrl *chip, unsigned int pin) +{ + /* Account for GPORT2 which only has 4 bits */ + return BIT(CY8C95X0_PIN_TO_OFFSET(pin) % BANK_SZ); +} + +static bool cy8c95x0_readable_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case 0x24 ... 0x27: + return false; + default: + return true; + } +} + +static bool cy8c95x0_writeable_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case CY8C95X0_INPUT_(0) ... CY8C95X0_INPUT_(7): + return false; + case CY8C95X0_DEVID: + return false; + case 0x24 ... 0x27: + return false; + default: + return true; + } +} + +static bool cy8c95x0_volatile_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case CY8C95X0_INPUT_(0) ... CY8C95X0_INPUT_(7): + case CY8C95X0_INTSTATUS_(0) ... CY8C95X0_INTSTATUS_(7): + case CY8C95X0_INTMASK: + case CY8C95X0_INVERT: + case CY8C95X0_PWMSEL: + case CY8C95X0_DIRECTION: + case CY8C95X0_DRV_PU: + case CY8C95X0_DRV_PD: + case CY8C95X0_DRV_ODH: + case CY8C95X0_DRV_ODL: + case CY8C95X0_DRV_PP_FAST: + case CY8C95X0_DRV_PP_SLOW: + case CY8C95X0_DRV_HIZ: + return true; + default: + return false; + } +} + +static bool cy8c95x0_precious_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case CY8C95X0_INTSTATUS_(0) ... CY8C95X0_INTSTATUS_(7): + return true; + default: + return false; + } +} + +static const struct reg_default cy8c95x0_reg_defaults[] = { + { CY8C95X0_OUTPUT_(0), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(1), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(2), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(3), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(4), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(5), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(6), GENMASK(7, 0) }, + { CY8C95X0_OUTPUT_(7), GENMASK(7, 0) }, + { CY8C95X0_PORTSEL, 0 }, + { CY8C95X0_PWMSEL, 0 }, +}; + +static const struct regmap_config cy8c95x0_i2c_regmap = { + .reg_bits = 8, + .val_bits = 8, + + .reg_defaults = cy8c95x0_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(cy8c95x0_reg_defaults), + + .readable_reg = cy8c95x0_readable_register, + .writeable_reg = cy8c95x0_writeable_register, + .volatile_reg = cy8c95x0_volatile_register, + .precious_reg = cy8c95x0_precious_register, + + .cache_type = REGCACHE_FLAT, + .max_register = CY8C95X0_COMMAND, +}; + +static int cy8c95x0_write_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, + unsigned long *val, unsigned long *mask) +{ + DECLARE_BITMAP(tmask, MAX_LINE); + DECLARE_BITMAP(tval, MAX_LINE); + int write_val; + int ret = 0; + int i, off = 0; + u8 bits; + + /* Add the 4 bit gap of Gport2 */ + bitmap_andnot(tmask, mask, chip->shiftmask, MAX_LINE); + bitmap_shift_left(tmask, tmask, 4, MAX_LINE); + bitmap_replace(tmask, tmask, mask, chip->shiftmask, BANK_SZ * 3); + + bitmap_andnot(tval, val, chip->shiftmask, MAX_LINE); + bitmap_shift_left(tval, tval, 4, MAX_LINE); + bitmap_replace(tval, tval, val, chip->shiftmask, BANK_SZ * 3); + + mutex_lock(&chip->i2c_lock); + for (i = 0; i < chip->nport; i++) { + /* Skip over unused banks */ + bits = bitmap_get_value8(tmask, i * BANK_SZ); + if (!bits) + continue; + + switch (reg) { + /* Muxed registers */ + case CY8C95X0_INTMASK: + case CY8C95X0_PWMSEL: + case CY8C95X0_INVERT: + case CY8C95X0_DIRECTION: + case CY8C95X0_DRV_PU: + case CY8C95X0_DRV_PD: + case CY8C95X0_DRV_ODH: + case CY8C95X0_DRV_ODL: + case CY8C95X0_DRV_PP_FAST: + case CY8C95X0_DRV_PP_SLOW: + case CY8C95X0_DRV_HIZ: + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, i); + if (ret < 0) + goto out; + off = reg; + break; + /* Direct access registers */ + case CY8C95X0_INPUT: + case CY8C95X0_OUTPUT: + case CY8C95X0_INTSTATUS: + off = reg + i; + break; + default: + ret = -EINVAL; + goto out; + } + + write_val = bitmap_get_value8(tval, i * BANK_SZ); + + ret = regmap_update_bits(chip->regmap, off, bits, write_val); + if (ret < 0) + goto out; + } +out: + mutex_unlock(&chip->i2c_lock); + + if (ret < 0) + dev_err(chip->dev, "failed writing register %d: err %d\n", off, ret); + + return ret; +} + +static int cy8c95x0_read_regs_mask(struct cy8c95x0_pinctrl *chip, int reg, + unsigned long *val, unsigned long *mask) +{ + DECLARE_BITMAP(tmask, MAX_LINE); + DECLARE_BITMAP(tval, MAX_LINE); + DECLARE_BITMAP(tmp, MAX_LINE); + int read_val; + int ret = 0; + int i, off = 0; + u8 bits; + + /* Add the 4 bit gap of Gport2 */ + bitmap_andnot(tmask, mask, chip->shiftmask, MAX_LINE); + bitmap_shift_left(tmask, tmask, 4, MAX_LINE); + bitmap_replace(tmask, tmask, mask, chip->shiftmask, BANK_SZ * 3); + + bitmap_andnot(tval, val, chip->shiftmask, MAX_LINE); + bitmap_shift_left(tval, tval, 4, MAX_LINE); + bitmap_replace(tval, tval, val, chip->shiftmask, BANK_SZ * 3); + + mutex_lock(&chip->i2c_lock); + for (i = 0; i < chip->nport; i++) { + /* Skip over unused banks */ + bits = bitmap_get_value8(tmask, i * BANK_SZ); + if (!bits) + continue; + + switch (reg) { + /* Muxed registers */ + case CY8C95X0_INTMASK: + case CY8C95X0_PWMSEL: + case CY8C95X0_INVERT: + case CY8C95X0_DIRECTION: + case CY8C95X0_DRV_PU: + case CY8C95X0_DRV_PD: + case CY8C95X0_DRV_ODH: + case CY8C95X0_DRV_ODL: + case CY8C95X0_DRV_PP_FAST: + case CY8C95X0_DRV_PP_SLOW: + case CY8C95X0_DRV_HIZ: + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, i); + if (ret < 0) + goto out; + off = reg; + break; + /* Direct access registers */ + case CY8C95X0_INPUT: + case CY8C95X0_OUTPUT: + case CY8C95X0_INTSTATUS: + off = reg + i; + break; + default: + ret = -EINVAL; + goto out; + } + + ret = regmap_read(chip->regmap, off, &read_val); + if (ret < 0) + goto out; + + read_val &= bits; + read_val |= bitmap_get_value8(tval, i * BANK_SZ) & ~bits; + bitmap_set_value8(tval, read_val, i * BANK_SZ); + } + + /* Fill the 4 bit gap of Gport2 */ + bitmap_shift_right(tmp, tval, 4, MAX_LINE); + bitmap_replace(val, tmp, tval, chip->shiftmask, MAX_LINE); + +out: + mutex_unlock(&chip->i2c_lock); + + if (ret < 0) + dev_err(chip->dev, "failed reading register %d: err %d\n", off, ret); + + return ret; +} + +static int cy8c95x0_gpio_direction_input(struct gpio_chip *gc, unsigned int off) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + u8 port = cypress_get_port(chip, off); + u8 bit = cypress_get_pin_mask(chip, off); + int ret; + + mutex_lock(&chip->i2c_lock); + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret) + goto out; + + ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, bit); + if (ret) + goto out; + + if (test_bit(off, chip->push_pull)) { + /* + * Disable driving the pin by forcing it to HighZ. Only setting the + * direction register isn't sufficient in Push-Pull mode. + */ + ret = regmap_write_bits(chip->regmap, CY8C95X0_DRV_HIZ, bit, bit); + if (ret) + goto out; + + __clear_bit(off, chip->push_pull); + } + +out: + mutex_unlock(&chip->i2c_lock); + + return ret; +} + +static int cy8c95x0_gpio_direction_output(struct gpio_chip *gc, + unsigned int off, int val) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + u8 port = cypress_get_port(chip, off); + u8 outreg = CY8C95X0_OUTPUT_(port); + u8 bit = cypress_get_pin_mask(chip, off); + int ret; + + /* Set output level */ + ret = regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0); + if (ret) + return ret; + + mutex_lock(&chip->i2c_lock); + /* Select port... */ + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret) + goto out; + + /* ...then direction */ + ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, 0); + +out: + mutex_unlock(&chip->i2c_lock); + + return ret; +} + +static int cy8c95x0_gpio_get_value(struct gpio_chip *gc, unsigned int off) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + u8 inreg = CY8C95X0_INPUT_(cypress_get_port(chip, off)); + u8 bit = cypress_get_pin_mask(chip, off); + u32 reg_val; + int ret; + + ret = regmap_read(chip->regmap, inreg, ®_val); + if (ret < 0) { + /* + * NOTE: + * Diagnostic already emitted; that's all we should + * do unless gpio_*_value_cansleep() calls become different + * from their nonsleeping siblings (and report faults). + */ + return 0; + } + + return !!(reg_val & bit); +} + +static void cy8c95x0_gpio_set_value(struct gpio_chip *gc, unsigned int off, + int val) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + u8 outreg = CY8C95X0_OUTPUT_(cypress_get_port(chip, off)); + u8 bit = cypress_get_pin_mask(chip, off); + + regmap_write_bits(chip->regmap, outreg, bit, val ? bit : 0); +} + +static int cy8c95x0_gpio_get_direction(struct gpio_chip *gc, unsigned int off) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + u8 port = cypress_get_port(chip, off); + u8 bit = cypress_get_pin_mask(chip, off); + u32 reg_val; + int ret; + + mutex_lock(&chip->i2c_lock); + + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret < 0) + goto out; + + ret = regmap_read(chip->regmap, CY8C95X0_DIRECTION, ®_val); + if (ret < 0) + goto out; + + mutex_unlock(&chip->i2c_lock); + + if (reg_val & bit) + return GPIO_LINE_DIRECTION_IN; + + return GPIO_LINE_DIRECTION_OUT; +out: + mutex_unlock(&chip->i2c_lock); + return ret; +} + +static int cy8c95x0_gpio_get_pincfg(struct cy8c95x0_pinctrl *chip, + unsigned int off, + unsigned long *config) +{ + enum pin_config_param param = pinconf_to_config_param(*config); + u8 port = cypress_get_port(chip, off); + u8 bit = cypress_get_pin_mask(chip, off); + unsigned int reg; + u32 reg_val; + u16 arg = 0; + int ret; + + mutex_lock(&chip->i2c_lock); + + /* Select port */ + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret < 0) + goto out; + + switch (param) { + case PIN_CONFIG_BIAS_PULL_UP: + reg = CY8C95X0_DRV_PU; + break; + case PIN_CONFIG_BIAS_PULL_DOWN: + reg = CY8C95X0_DRV_PD; + break; + case PIN_CONFIG_BIAS_DISABLE: + reg = CY8C95X0_DRV_HIZ; + break; + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + reg = CY8C95X0_DRV_ODL; + break; + case PIN_CONFIG_DRIVE_OPEN_SOURCE: + reg = CY8C95X0_DRV_ODH; + break; + case PIN_CONFIG_DRIVE_PUSH_PULL: + reg = CY8C95X0_DRV_PP_FAST; + break; + case PIN_CONFIG_INPUT_ENABLE: + reg = CY8C95X0_DIRECTION; + break; + case PIN_CONFIG_MODE_PWM: + reg = CY8C95X0_PWMSEL; + break; + case PIN_CONFIG_OUTPUT: + reg = CY8C95X0_OUTPUT_(port); + break; + case PIN_CONFIG_OUTPUT_ENABLE: + reg = CY8C95X0_DIRECTION; + break; + + case PIN_CONFIG_BIAS_HIGH_IMPEDANCE: + case PIN_CONFIG_BIAS_BUS_HOLD: + case PIN_CONFIG_BIAS_PULL_PIN_DEFAULT: + case PIN_CONFIG_DRIVE_STRENGTH: + case PIN_CONFIG_DRIVE_STRENGTH_UA: + case PIN_CONFIG_INPUT_DEBOUNCE: + case PIN_CONFIG_INPUT_SCHMITT: + case PIN_CONFIG_INPUT_SCHMITT_ENABLE: + case PIN_CONFIG_MODE_LOW_POWER: + case PIN_CONFIG_PERSIST_STATE: + case PIN_CONFIG_POWER_SOURCE: + case PIN_CONFIG_SKEW_DELAY: + case PIN_CONFIG_SLEEP_HARDWARE_STATE: + case PIN_CONFIG_SLEW_RATE: + default: + ret = -ENOTSUPP; + goto out; + } + /* + * Writing 1 to one of the drive mode registers will automatically + * clear conflicting set bits in the other drive mode registers. + */ + ret = regmap_read(chip->regmap, reg, ®_val); + if (reg_val & bit) + arg = 1; + + *config = pinconf_to_config_packed(param, (u16)arg); +out: + mutex_unlock(&chip->i2c_lock); + + return ret; +} + +static int cy8c95x0_gpio_set_pincfg(struct cy8c95x0_pinctrl *chip, + unsigned int off, + unsigned long config) +{ + u8 port = cypress_get_port(chip, off); + u8 bit = cypress_get_pin_mask(chip, off); + unsigned long param = pinconf_to_config_param(config); + unsigned int reg; + int ret; + + mutex_lock(&chip->i2c_lock); + + /* Select port */ + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret < 0) + goto out; + + switch (param) { + case PIN_CONFIG_BIAS_PULL_UP: + __clear_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_PU; + break; + case PIN_CONFIG_BIAS_PULL_DOWN: + __clear_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_PD; + break; + case PIN_CONFIG_BIAS_DISABLE: + __clear_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_HIZ; + break; + case PIN_CONFIG_DRIVE_OPEN_DRAIN: + __clear_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_ODL; + break; + case PIN_CONFIG_DRIVE_OPEN_SOURCE: + __clear_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_ODH; + break; + case PIN_CONFIG_DRIVE_PUSH_PULL: + __set_bit(off, chip->push_pull); + reg = CY8C95X0_DRV_PP_FAST; + break; + case PIN_CONFIG_MODE_PWM: + reg = CY8C95X0_PWMSEL; + break; + default: + ret = -ENOTSUPP; + goto out; + } + /* + * Writing 1 to one of the drive mode registers will automatically + * clear conflicting set bits in the other drive mode registers. + */ + ret = regmap_write_bits(chip->regmap, reg, bit, bit); + +out: + mutex_unlock(&chip->i2c_lock); + return ret; +} + +static int cy8c95x0_gpio_get_multiple(struct gpio_chip *gc, + unsigned long *mask, unsigned long *bits) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + + return cy8c95x0_read_regs_mask(chip, CY8C95X0_INPUT, bits, mask); +} + +static void cy8c95x0_gpio_set_multiple(struct gpio_chip *gc, + unsigned long *mask, unsigned long *bits) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + + cy8c95x0_write_regs_mask(chip, CY8C95X0_OUTPUT, bits, mask); +} + +static int cy8c95x0_add_pin_ranges(struct gpio_chip *gc) +{ + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + struct device *dev = chip->dev; + int ret; + + ret = gpiochip_add_pin_range(gc, dev_name(dev), 0, 0, chip->tpin); + if (ret) + dev_err(dev, "failed to add GPIO pin range\n"); + + return ret; +} + +static int cy8c95x0_setup_gpiochip(struct cy8c95x0_pinctrl *chip) +{ + struct gpio_chip *gc = &chip->gpio_chip; + + gc->direction_input = cy8c95x0_gpio_direction_input; + gc->direction_output = cy8c95x0_gpio_direction_output; + gc->get = cy8c95x0_gpio_get_value; + gc->set = cy8c95x0_gpio_set_value; + gc->get_direction = cy8c95x0_gpio_get_direction; + gc->get_multiple = cy8c95x0_gpio_get_multiple; + gc->set_multiple = cy8c95x0_gpio_set_multiple; + gc->set_config = gpiochip_generic_config, + gc->can_sleep = true; + gc->add_pin_ranges = cy8c95x0_add_pin_ranges; + + gc->base = -1; + gc->ngpio = chip->tpin; + + gc->parent = chip->dev; + gc->owner = THIS_MODULE; + gc->names = NULL; + + gc->label = dev_name(chip->dev); + + return devm_gpiochip_add_data(chip->dev, gc, chip); +} + +static void cy8c95x0_irq_mask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + + set_bit(hwirq, chip->irq_mask); + gpiochip_disable_irq(gc, hwirq); +} + +static void cy8c95x0_irq_unmask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + + gpiochip_enable_irq(gc, hwirq); + clear_bit(hwirq, chip->irq_mask); +} + +static void cy8c95x0_irq_bus_lock(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + + mutex_lock(&chip->irq_lock); +} + +static void cy8c95x0_irq_bus_sync_unlock(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + DECLARE_BITMAP(ones, MAX_LINE); + DECLARE_BITMAP(irq_mask, MAX_LINE); + DECLARE_BITMAP(reg_direction, MAX_LINE); + + bitmap_fill(ones, MAX_LINE); + + cy8c95x0_write_regs_mask(chip, CY8C95X0_INTMASK, chip->irq_mask, ones); + + /* Switch direction to input if needed */ + cy8c95x0_read_regs_mask(chip, CY8C95X0_DIRECTION, reg_direction, chip->irq_mask); + bitmap_or(irq_mask, chip->irq_mask, reg_direction, MAX_LINE); + bitmap_complement(irq_mask, irq_mask, MAX_LINE); + + /* Look for any newly setup interrupt */ + cy8c95x0_write_regs_mask(chip, CY8C95X0_DIRECTION, ones, irq_mask); + + mutex_unlock(&chip->irq_lock); +} + +static int cy8c95x0_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned int trig_type; + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + case IRQ_TYPE_EDGE_FALLING: + case IRQ_TYPE_EDGE_BOTH: + trig_type = type; + break; + case IRQ_TYPE_LEVEL_HIGH: + trig_type = IRQ_TYPE_EDGE_RISING; + break; + case IRQ_TYPE_LEVEL_LOW: + trig_type = IRQ_TYPE_EDGE_FALLING; + break; + default: + dev_err(chip->dev, "irq %d: unsupported type %d\n", d->irq, type); + return -EINVAL; + } + + assign_bit(hwirq, chip->irq_trig_fall, trig_type & IRQ_TYPE_EDGE_FALLING); + assign_bit(hwirq, chip->irq_trig_raise, trig_type & IRQ_TYPE_EDGE_RISING); + assign_bit(hwirq, chip->irq_trig_low, type == IRQ_TYPE_LEVEL_LOW); + assign_bit(hwirq, chip->irq_trig_high, type == IRQ_TYPE_LEVEL_HIGH); + + return 0; +} + +static void cy8c95x0_irq_shutdown(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct cy8c95x0_pinctrl *chip = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + + clear_bit(hwirq, chip->irq_trig_raise); + clear_bit(hwirq, chip->irq_trig_fall); + clear_bit(hwirq, chip->irq_trig_low); + clear_bit(hwirq, chip->irq_trig_high); +} + +static const struct irq_chip cy8c95x0_irqchip = { + .name = "cy8c95x0-irq", + .irq_mask = cy8c95x0_irq_mask, + .irq_unmask = cy8c95x0_irq_unmask, + .irq_bus_lock = cy8c95x0_irq_bus_lock, + .irq_bus_sync_unlock = cy8c95x0_irq_bus_sync_unlock, + .irq_set_type = cy8c95x0_irq_set_type, + .irq_shutdown = cy8c95x0_irq_shutdown, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + +static bool cy8c95x0_irq_pending(struct cy8c95x0_pinctrl *chip, unsigned long *pending) +{ + DECLARE_BITMAP(ones, MAX_LINE); + DECLARE_BITMAP(cur_stat, MAX_LINE); + DECLARE_BITMAP(new_stat, MAX_LINE); + DECLARE_BITMAP(trigger, MAX_LINE); + + bitmap_fill(ones, MAX_LINE); + + /* Read the current interrupt status from the device */ + if (cy8c95x0_read_regs_mask(chip, CY8C95X0_INTSTATUS, trigger, ones)) + return false; + + /* Check latched inputs */ + if (cy8c95x0_read_regs_mask(chip, CY8C95X0_INPUT, cur_stat, trigger)) + return false; + + /* Apply filter for rising/falling edge selection */ + bitmap_replace(new_stat, chip->irq_trig_fall, chip->irq_trig_raise, + cur_stat, MAX_LINE); + + bitmap_and(pending, new_stat, trigger, MAX_LINE); + + return !bitmap_empty(pending, MAX_LINE); +} + +static irqreturn_t cy8c95x0_irq_handler(int irq, void *devid) +{ + struct cy8c95x0_pinctrl *chip = devid; + struct gpio_chip *gc = &chip->gpio_chip; + DECLARE_BITMAP(pending, MAX_LINE); + int nested_irq, level; + bool ret; + + ret = cy8c95x0_irq_pending(chip, pending); + if (!ret) + return IRQ_RETVAL(0); + + ret = 0; + for_each_set_bit(level, pending, MAX_LINE) { + /* Already accounted for 4bit gap in GPort2 */ + nested_irq = irq_find_mapping(gc->irq.domain, level); + + if (unlikely(nested_irq <= 0)) { + dev_warn_ratelimited(gc->parent, "unmapped interrupt %d\n", level); + continue; + } + + if (test_bit(level, chip->irq_trig_low)) + while (!cy8c95x0_gpio_get_value(gc, level)) + handle_nested_irq(nested_irq); + else if (test_bit(level, chip->irq_trig_high)) + while (cy8c95x0_gpio_get_value(gc, level)) + handle_nested_irq(nested_irq); + else + handle_nested_irq(nested_irq); + + ret = 1; + } + + return IRQ_RETVAL(ret); +} + +static int cy8c95x0_pinctrl_get_groups_count(struct pinctrl_dev *pctldev) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + + return chip->tpin; +} + +static const char *cy8c95x0_pinctrl_get_group_name(struct pinctrl_dev *pctldev, + unsigned int group) +{ + return cy8c95x0_groups[group]; +} + +static int cy8c95x0_pinctrl_get_group_pins(struct pinctrl_dev *pctldev, + unsigned int group, + const unsigned int **pins, + unsigned int *num_pins) +{ + *pins = &cy8c9560_pins[group].number; + *num_pins = 1; + return 0; +} + +static const char *cy8c95x0_get_fname(unsigned int selector) +{ + if (selector == 0) + return "gpio"; + else + return "pwm"; +} + +static void cy8c95x0_pin_dbg_show(struct pinctrl_dev *pctldev, struct seq_file *s, + unsigned int pin) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + DECLARE_BITMAP(mask, MAX_LINE); + DECLARE_BITMAP(pwm, MAX_LINE); + + bitmap_zero(mask, MAX_LINE); + __set_bit(pin, mask); + + if (cy8c95x0_read_regs_mask(chip, CY8C95X0_PWMSEL, pwm, mask)) { + seq_puts(s, "not available"); + return; + } + + seq_printf(s, "MODE:%s", cy8c95x0_get_fname(test_bit(pin, pwm))); +} + +static const struct pinctrl_ops cy8c95x0_pinctrl_ops = { + .get_groups_count = cy8c95x0_pinctrl_get_groups_count, + .get_group_name = cy8c95x0_pinctrl_get_group_name, + .get_group_pins = cy8c95x0_pinctrl_get_group_pins, +#ifdef CONFIG_OF + .dt_node_to_map = pinconf_generic_dt_node_to_map_pin, + .dt_free_map = pinconf_generic_dt_free_map, +#endif + .pin_dbg_show = cy8c95x0_pin_dbg_show, +}; + +static const char *cy8c95x0_get_function_name(struct pinctrl_dev *pctldev, unsigned int selector) +{ + return cy8c95x0_get_fname(selector); +} + +static int cy8c95x0_get_functions_count(struct pinctrl_dev *pctldev) +{ + return 2; +} + +static int cy8c95x0_get_function_groups(struct pinctrl_dev *pctldev, unsigned int selector, + const char * const **groups, + unsigned int * const num_groups) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + + *groups = cy8c95x0_groups; + *num_groups = chip->tpin; + return 0; +} + +static int cy8c95x0_pinmux_cfg(struct cy8c95x0_pinctrl *chip, + unsigned int val, + unsigned long off) +{ + u8 port = cypress_get_port(chip, off); + u8 bit = cypress_get_pin_mask(chip, off); + int ret; + + /* Select port */ + ret = regmap_write(chip->regmap, CY8C95X0_PORTSEL, port); + if (ret < 0) + return ret; + + ret = regmap_write_bits(chip->regmap, CY8C95X0_PWMSEL, bit, val ? bit : 0); + if (ret < 0) + return ret; + + /* Set direction to output & set output to 1 so that PWM can work */ + ret = regmap_write_bits(chip->regmap, CY8C95X0_DIRECTION, bit, bit); + if (ret < 0) + return ret; + + return regmap_write_bits(chip->regmap, CY8C95X0_OUTPUT_(port), bit, bit); +} + +static int cy8c95x0_set_mux(struct pinctrl_dev *pctldev, unsigned int selector, + unsigned int group) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + int ret; + + mutex_lock(&chip->i2c_lock); + ret = cy8c95x0_pinmux_cfg(chip, selector, group); + mutex_unlock(&chip->i2c_lock); + + return ret; +} + +static const struct pinmux_ops cy8c95x0_pmxops = { + .get_functions_count = cy8c95x0_get_functions_count, + .get_function_name = cy8c95x0_get_function_name, + .get_function_groups = cy8c95x0_get_function_groups, + .set_mux = cy8c95x0_set_mux, + .strict = true, +}; + +static int cy8c95x0_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin, + unsigned long *config) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + + return cy8c95x0_gpio_get_pincfg(chip, pin, config); +} + +static int cy8c95x0_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin, + unsigned long *configs, unsigned int num_configs) +{ + struct cy8c95x0_pinctrl *chip = pinctrl_dev_get_drvdata(pctldev); + int ret = 0; + int i; + + for (i = 0; i < num_configs; i++) { + ret = cy8c95x0_gpio_set_pincfg(chip, pin, configs[i]); + if (ret) + return ret; + } + + return ret; +} + +static const struct pinconf_ops cy8c95x0_pinconf_ops = { + .pin_config_get = cy8c95x0_pinconf_get, + .pin_config_set = cy8c95x0_pinconf_set, + .is_generic = true, +}; + +static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq) +{ + struct gpio_irq_chip *girq = &chip->gpio_chip.irq; + DECLARE_BITMAP(pending_irqs, MAX_LINE); + int ret; + + mutex_init(&chip->irq_lock); + + bitmap_zero(pending_irqs, MAX_LINE); + + /* Read IRQ status register to clear all pending interrupts */ + ret = cy8c95x0_irq_pending(chip, pending_irqs); + if (ret) { + dev_err(chip->dev, "failed to clear irq status register\n"); + return ret; + } + + /* Mask all interrupts */ + bitmap_fill(chip->irq_mask, MAX_LINE); + + gpio_irq_chip_set_chip(girq, &cy8c95x0_irqchip); + + /* This will let us handle the parent IRQ in the driver */ + girq->parent_handler = NULL; + girq->num_parents = 0; + girq->parents = NULL; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_simple_irq; + girq->threaded = true; + + ret = devm_request_threaded_irq(chip->dev, irq, + NULL, cy8c95x0_irq_handler, + IRQF_ONESHOT | IRQF_SHARED | IRQF_TRIGGER_HIGH, + dev_name(chip->dev), chip); + if (ret) { + dev_err(chip->dev, "failed to request irq %d\n", irq); + return ret; + } + dev_info(chip->dev, "Registered threaded IRQ\n"); + + return 0; +} + +static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip) +{ + struct pinctrl_desc *pd = &chip->pinctrl_desc; + + pd->pctlops = &cy8c95x0_pinctrl_ops; + pd->confops = &cy8c95x0_pinconf_ops; + pd->pmxops = &cy8c95x0_pmxops; + pd->name = dev_name(chip->dev); + pd->pins = cy8c9560_pins; + pd->npins = chip->tpin; + pd->owner = THIS_MODULE; + + chip->pctldev = devm_pinctrl_register(chip->dev, pd, chip); + if (IS_ERR(chip->pctldev)) + return dev_err_probe(chip->dev, PTR_ERR(chip->pctldev), + "can't register controller\n"); + + return 0; +} + +static int cy8c95x0_detect(struct i2c_client *client, + struct i2c_board_info *info) +{ + struct i2c_adapter *adapter = client->adapter; + int ret; + const char *name; + + if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA)) + return -ENODEV; + + ret = i2c_smbus_read_byte_data(client, CY8C95X0_DEVID); + if (ret < 0) + return ret; + switch (ret & GENMASK(7, 4)) { + case 0x20: + name = cy8c95x0_id[0].name; + break; + case 0x40: + name = cy8c95x0_id[1].name; + break; + case 0x60: + name = cy8c95x0_id[2].name; + break; + default: + return -ENODEV; + } + + dev_info(&client->dev, "Found a %s chip at 0x%02x.\n", name, client->addr); + strscpy(info->type, name, I2C_NAME_SIZE); + + return 0; +} + +static int cy8c95x0_probe(struct i2c_client *client) +{ + struct cy8c95x0_pinctrl *chip; + struct regulator *reg; + int ret; + + chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + chip->dev = &client->dev; + + /* Set the device type */ + chip->driver_data = (unsigned long)device_get_match_data(&client->dev); + if (!chip->driver_data) + chip->driver_data = i2c_match_id(cy8c95x0_id, client)->driver_data; + if (!chip->driver_data) + return -ENODEV; + + i2c_set_clientdata(client, chip); + + chip->tpin = chip->driver_data & CY8C95X0_GPIO_MASK; + chip->nport = DIV_ROUND_UP(CY8C95X0_PIN_TO_OFFSET(chip->tpin), BANK_SZ); + + switch (chip->tpin) { + case 20: + strscpy(chip->name, cy8c95x0_id[0].name, I2C_NAME_SIZE); + break; + case 40: + strscpy(chip->name, cy8c95x0_id[1].name, I2C_NAME_SIZE); + break; + case 60: + strscpy(chip->name, cy8c95x0_id[2].name, I2C_NAME_SIZE); + break; + default: + return -ENODEV; + } + + reg = devm_regulator_get(&client->dev, "vdd"); + if (IS_ERR(reg)) { + if (PTR_ERR(reg) == -EPROBE_DEFER) + return -EPROBE_DEFER; + } else { + ret = regulator_enable(reg); + if (ret) { + dev_err(&client->dev, "failed to enable regulator vdd: %d\n", ret); + return ret; + } + chip->regulator = reg; + } + + chip->regmap = devm_regmap_init_i2c(client, &cy8c95x0_i2c_regmap); + if (IS_ERR(chip->regmap)) { + ret = PTR_ERR(chip->regmap); + goto err_exit; + } + + bitmap_zero(chip->push_pull, MAX_LINE); + bitmap_zero(chip->shiftmask, MAX_LINE); + bitmap_set(chip->shiftmask, 0, 20); + mutex_init(&chip->i2c_lock); + + if (dmi_first_match(cy8c95x0_dmi_acpi_irq_info)) { + ret = cy8c95x0_acpi_get_irq(&client->dev); + if (ret > 0) + client->irq = ret; + } + + if (client->irq) { + ret = cy8c95x0_irq_setup(chip, client->irq); + if (ret) + goto err_exit; + } + + ret = cy8c95x0_setup_pinctrl(chip); + if (ret) + goto err_exit; + + ret = cy8c95x0_setup_gpiochip(chip); + if (ret) + goto err_exit; + + return 0; + +err_exit: + if (!IS_ERR_OR_NULL(chip->regulator)) + regulator_disable(chip->regulator); + return ret; +} + +static void cy8c95x0_remove(struct i2c_client *client) +{ + struct cy8c95x0_pinctrl *chip = i2c_get_clientdata(client); + + if (!IS_ERR_OR_NULL(chip->regulator)) + regulator_disable(chip->regulator); +} + +static const struct acpi_device_id cy8c95x0_acpi_ids[] = { + { "INT3490", 40, }, + { } +}; +MODULE_DEVICE_TABLE(acpi, cy8c95x0_acpi_ids); + +static struct i2c_driver cy8c95x0_driver = { + .driver = { + .name = "cy8c95x0-pinctrl", + .of_match_table = cy8c95x0_dt_ids, + .acpi_match_table = cy8c95x0_acpi_ids, + }, + .probe_new = cy8c95x0_probe, + .remove = cy8c95x0_remove, + .id_table = cy8c95x0_id, + .detect = cy8c95x0_detect, +}; +module_i2c_driver(cy8c95x0_driver); + +MODULE_AUTHOR("Patrick Rudolph "); +MODULE_AUTHOR("Naresh Solanki "); +MODULE_DESCRIPTION("Pinctrl driver for CY8C95X0"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c index 3a9ee9c8af11676aaaed388a3cc1ef43c9061e10..7e732076dedf0b478724ddad34782624a12cfc29 100644 --- a/drivers/pinctrl/pinctrl-ingenic.c +++ b/drivers/pinctrl/pinctrl-ingenic.c @@ -12,14 +12,14 @@ #include #include #include -#include -#include -#include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -4152,7 +4152,7 @@ static const struct of_device_id ingenic_gpio_of_matches[] __initconst = { }; static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc, - struct device_node *node) + struct fwnode_handle *fwnode) { struct ingenic_gpio_chip *jzgc; struct device *dev = jzpc->dev; @@ -4160,7 +4160,7 @@ static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc, unsigned int bank; int err; - err = of_property_read_u32(node, "reg", &bank); + err = fwnode_property_read_u32(fwnode, "reg", &bank); if (err) { dev_err(dev, "Cannot read \"reg\" property: %i\n", err); return err; @@ -4185,7 +4185,7 @@ static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc, jzgc->gc.ngpio = 32; jzgc->gc.parent = dev; - jzgc->gc.of_node = node; + jzgc->gc.fwnode = fwnode; jzgc->gc.owner = THIS_MODULE; jzgc->gc.set = ingenic_gpio_set; @@ -4196,9 +4196,12 @@ static int __init ingenic_gpio_probe(struct ingenic_pinctrl *jzpc, jzgc->gc.request = gpiochip_generic_request; jzgc->gc.free = gpiochip_generic_free; - jzgc->irq = irq_of_parse_and_map(node, 0); - if (!jzgc->irq) + err = fwnode_irq_get(fwnode, 0); + if (err < 0) + return err; + if (!err) return -EINVAL; + jzgc->irq = err; girq = &jzgc->gc.irq; gpio_irq_chip_set_chip(girq, &ingenic_gpio_irqchip); @@ -4227,12 +4230,12 @@ static int __init ingenic_pinctrl_probe(struct platform_device *pdev) struct pinctrl_desc *pctl_desc; void __iomem *base; const struct ingenic_chip_info *chip_info; - struct device_node *node; struct regmap_config regmap_config; + struct fwnode_handle *fwnode; unsigned int i; int err; - chip_info = of_device_get_match_data(dev); + chip_info = device_get_match_data(dev); if (!chip_info) { dev_err(dev, "Unsupported SoC\n"); return -EINVAL; @@ -4319,11 +4322,11 @@ static int __init ingenic_pinctrl_probe(struct platform_device *pdev) dev_set_drvdata(dev, jzpc->map); - for_each_child_of_node(dev->of_node, node) { - if (of_match_node(ingenic_gpio_of_matches, node)) { - err = ingenic_gpio_probe(jzpc, node); + device_for_each_child_node(dev, fwnode) { + if (of_match_node(ingenic_gpio_of_matches, to_of_node(fwnode))) { + err = ingenic_gpio_probe(jzpc, fwnode); if (err) { - of_node_put(node); + fwnode_handle_put(fwnode); return err; } } diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c index 695236636d05779a252368e8296e4df3ceff778d..5f356edfd0fd56d84ae8730881dc17322c6d13b1 100644 --- a/drivers/pinctrl/pinctrl-mcp23s08.c +++ b/drivers/pinctrl/pinctrl-mcp23s08.c @@ -549,9 +549,6 @@ int mcp23s08_probe_one(struct mcp23s08 *mcp, struct device *dev, mcp->chip.get = mcp23s08_get; mcp->chip.direction_output = mcp23s08_direction_output; mcp->chip.set = mcp23s08_set; -#ifdef CONFIG_OF_GPIO - mcp->chip.of_gpio_n_cells = 2; -#endif mcp->chip.base = base; mcp->chip.can_sleep = true; diff --git a/drivers/pinctrl/pinctrl-microchip-sgpio.c b/drivers/pinctrl/pinctrl-microchip-sgpio.c index 6f55bf7d5e054d73b4cbe19b95868caac57e9808..af27b72c8958685bfde1b654c57b95013a24af5c 100644 --- a/drivers/pinctrl/pinctrl-microchip-sgpio.c +++ b/drivers/pinctrl/pinctrl-microchip-sgpio.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -864,9 +865,10 @@ static int microchip_sgpio_register_bank(struct device *dev, gc->can_sleep = !bank->is_input; if (bank->is_input && priv->properties->flags & SGPIO_FLAGS_HAS_IRQ) { - int irq = fwnode_irq_get(fwnode, 0); + int irq; - if (irq) { + irq = fwnode_irq_get(fwnode, 0); + if (irq > 0) { struct gpio_irq_chip *girq = &gc->irq; gpio_irq_chip_set_chip(girq, µchip_sgpio_irqchip); @@ -904,7 +906,6 @@ static int microchip_sgpio_probe(struct platform_device *pdev) struct reset_control *reset; struct sgpio_priv *priv; struct clk *clk; - u32 __iomem *regs; u32 val; struct regmap_config regmap_config = { .reg_bits = 32, @@ -937,11 +938,7 @@ static int microchip_sgpio_probe(struct platform_device *pdev) return -EINVAL; } - regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(regs)) - return PTR_ERR(regs); - - priv->regs = devm_regmap_init_mmio(dev, regs, ®map_config); + priv->regs = ocelot_regmap_from_resource(pdev, 0, ®map_config); if (IS_ERR(priv->regs)) return PTR_ERR(priv->regs); @@ -999,6 +996,7 @@ static const struct of_device_id microchip_sgpio_gpio_of_match[] = { /* sentinel */ } }; +MODULE_DEVICE_TABLE(of, microchip_sgpio_gpio_of_match); static struct platform_driver microchip_sgpio_pinctrl_driver = { .driver = { @@ -1008,4 +1006,7 @@ static struct platform_driver microchip_sgpio_pinctrl_driver = { }, .probe = microchip_sgpio_probe, }; -builtin_platform_driver(microchip_sgpio_pinctrl_driver); +module_platform_driver(microchip_sgpio_pinctrl_driver); + +MODULE_DESCRIPTION("Microchip SGPIO Pinctrl Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/pinctrl-ocelot.c b/drivers/pinctrl/pinctrl-ocelot.c index c5fd154990c8b4b61bd52ca5b733ad9a930738a8..62ce3957abe4e554933a13a8ac87e6e7b01c06d5 100644 --- a/drivers/pinctrl/pinctrl-ocelot.c +++ b/drivers/pinctrl/pinctrl-ocelot.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -331,6 +332,7 @@ struct ocelot_pinctrl { const struct ocelot_pincfg_data *pincfg_data; struct ocelot_pmx_func func[FUNC_MAX]; u8 stride; + struct workqueue_struct *wq; }; struct ocelot_match_data { @@ -338,6 +340,11 @@ struct ocelot_match_data { struct ocelot_pincfg_data pincfg_data; }; +struct ocelot_irq_work { + struct work_struct irq_work; + struct irq_desc *irq_desc; +}; + #define LUTON_P(p, f0, f1) \ static struct ocelot_pin_caps luton_pin_##p = { \ .pin = p, \ @@ -1813,6 +1820,75 @@ static void ocelot_irq_mask(struct irq_data *data) gpiochip_disable_irq(chip, gpio); } +static void ocelot_irq_work(struct work_struct *work) +{ + struct ocelot_irq_work *w = container_of(work, struct ocelot_irq_work, irq_work); + struct irq_chip *parent_chip = irq_desc_get_chip(w->irq_desc); + struct gpio_chip *chip = irq_desc_get_chip_data(w->irq_desc); + struct irq_data *data = irq_desc_get_irq_data(w->irq_desc); + unsigned int gpio = irqd_to_hwirq(data); + + local_irq_disable(); + chained_irq_enter(parent_chip, w->irq_desc); + generic_handle_domain_irq(chip->irq.domain, gpio); + chained_irq_exit(parent_chip, w->irq_desc); + local_irq_enable(); + + kfree(w); +} + +static void ocelot_irq_unmask_level(struct irq_data *data) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(data); + struct ocelot_pinctrl *info = gpiochip_get_data(chip); + struct irq_desc *desc = irq_data_to_desc(data); + unsigned int gpio = irqd_to_hwirq(data); + unsigned int bit = BIT(gpio % 32); + bool ack = false, active = false; + u8 trigger_level; + int val; + + trigger_level = irqd_get_trigger_type(data); + + /* Check if the interrupt line is still active. */ + regmap_read(info->map, REG(OCELOT_GPIO_IN, info, gpio), &val); + if ((!(val & bit) && trigger_level == IRQ_TYPE_LEVEL_LOW) || + (val & bit && trigger_level == IRQ_TYPE_LEVEL_HIGH)) + active = true; + + /* + * Check if the interrupt controller has seen any changes in the + * interrupt line. + */ + regmap_read(info->map, REG(OCELOT_GPIO_INTR, info, gpio), &val); + if (val & bit) + ack = true; + + /* Enable the interrupt now */ + gpiochip_enable_irq(chip, gpio); + regmap_update_bits(info->map, REG(OCELOT_GPIO_INTR_ENA, info, gpio), + bit, bit); + + /* + * In case the interrupt line is still active and the interrupt + * controller has not seen any changes in the interrupt line, then it + * means that there happen another interrupt while the line was active. + * So we missed that one, so we need to kick the interrupt again + * handler. + */ + if (active && !ack) { + struct ocelot_irq_work *work; + + work = kmalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return; + + work->irq_desc = desc; + INIT_WORK(&work->irq_work, ocelot_irq_work); + queue_work(info->wq, &work->irq_work); + } +} + static void ocelot_irq_unmask(struct irq_data *data) { struct gpio_chip *chip = irq_data_get_irq_chip_data(data); @@ -1836,13 +1912,12 @@ static void ocelot_irq_ack(struct irq_data *data) static int ocelot_irq_set_type(struct irq_data *data, unsigned int type); -static struct irq_chip ocelot_eoi_irqchip = { +static struct irq_chip ocelot_level_irqchip = { .name = "gpio", .irq_mask = ocelot_irq_mask, - .irq_eoi = ocelot_irq_ack, - .irq_unmask = ocelot_irq_unmask, - .flags = IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED | - IRQCHIP_IMMUTABLE, + .irq_ack = ocelot_irq_ack, + .irq_unmask = ocelot_irq_unmask_level, + .flags = IRQCHIP_IMMUTABLE, .irq_set_type = ocelot_irq_set_type, GPIOCHIP_IRQ_RESOURCE_HELPERS }; @@ -1859,14 +1934,9 @@ static struct irq_chip ocelot_irqchip = { static int ocelot_irq_set_type(struct irq_data *data, unsigned int type) { - type &= IRQ_TYPE_SENSE_MASK; - - if (!(type & (IRQ_TYPE_EDGE_BOTH | IRQ_TYPE_LEVEL_HIGH))) - return -EINVAL; - - if (type & IRQ_TYPE_LEVEL_HIGH) - irq_set_chip_handler_name_locked(data, &ocelot_eoi_irqchip, - handle_fasteoi_irq, NULL); + if (type & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) + irq_set_chip_handler_name_locked(data, &ocelot_level_irqchip, + handle_level_irq, NULL); if (type & IRQ_TYPE_EDGE_BOTH) irq_set_chip_handler_name_locked(data, &ocelot_irqchip, handle_edge_irq, NULL); @@ -1975,7 +2045,6 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev) struct ocelot_pinctrl *info; struct reset_control *reset; struct regmap *pincfg; - void __iomem *base; int ret; struct regmap_config regmap_config = { .reg_bits = 32, @@ -1996,6 +2065,10 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev) if (!info->desc) return -ENOMEM; + info->wq = alloc_ordered_workqueue("ocelot_ordered", 0); + if (!info->wq) + return -ENOMEM; + info->pincfg_data = &data->pincfg_data; reset = devm_reset_control_get_optional_shared(dev, "switch"); @@ -2004,21 +2077,15 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev) "Failed to get reset\n"); reset_control_reset(reset); - base = devm_ioremap_resource(dev, - platform_get_resource(pdev, IORESOURCE_MEM, 0)); - if (IS_ERR(base)) - return PTR_ERR(base); - info->stride = 1 + (info->desc->npins - 1) / 32; regmap_config.max_register = OCELOT_GPIO_SD_MAP * info->stride + 15 * 4; - info->map = devm_regmap_init_mmio(dev, base, ®map_config); - if (IS_ERR(info->map)) { - dev_err(dev, "Failed to create regmap\n"); - return PTR_ERR(info->map); - } - dev_set_drvdata(dev, info->map); + info->map = ocelot_regmap_from_resource(pdev, 0, ®map_config); + if (IS_ERR(info->map)) + return dev_err_probe(dev, PTR_ERR(info->map), + "Failed to create regmap\n"); + dev_set_drvdata(dev, info); info->dev = dev; /* Pinconf registers */ @@ -2043,6 +2110,15 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev) return 0; } +static int ocelot_pinctrl_remove(struct platform_device *pdev) +{ + struct ocelot_pinctrl *info = platform_get_drvdata(pdev); + + destroy_workqueue(info->wq); + + return 0; +} + static struct platform_driver ocelot_pinctrl_driver = { .driver = { .name = "pinctrl-ocelot", @@ -2050,6 +2126,9 @@ static struct platform_driver ocelot_pinctrl_driver = { .suppress_bind_attrs = true, }, .probe = ocelot_pinctrl_probe, + .remove = ocelot_pinctrl_remove, }; module_platform_driver(ocelot_pinctrl_driver); + +MODULE_DESCRIPTION("Ocelot Chip Pinctrl Driver"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c index 5de691c630b4fc77e044eac5e874aa06ba3df402..7ca4ecb6eb8d7ec8261ecf58f4f40490c7d09369 100644 --- a/drivers/pinctrl/pinctrl-pistachio.c +++ b/drivers/pinctrl/pinctrl-pistachio.c @@ -10,13 +10,13 @@ #include #include #include -#include -#include +#include #include #include #include #include #include +#include #include #include @@ -1347,46 +1347,51 @@ static struct pistachio_gpio_bank pistachio_gpio_banks[] = { static int pistachio_gpio_register(struct pistachio_pinctrl *pctl) { - struct device_node *node = pctl->dev->of_node; struct pistachio_gpio_bank *bank; unsigned int i; int irq, ret = 0; for (i = 0; i < pctl->nbanks; i++) { char child_name[sizeof("gpioXX")]; - struct device_node *child; + struct fwnode_handle *child; struct gpio_irq_chip *girq; snprintf(child_name, sizeof(child_name), "gpio%d", i); - child = of_get_child_by_name(node, child_name); + child = device_get_named_child_node(pctl->dev, child_name); if (!child) { dev_err(pctl->dev, "No node for bank %u\n", i); ret = -ENODEV; goto err; } - if (!of_find_property(child, "gpio-controller", NULL)) { + if (!fwnode_property_present(child, "gpio-controller")) { + fwnode_handle_put(child); dev_err(pctl->dev, "No gpio-controller property for bank %u\n", i); - of_node_put(child); ret = -ENODEV; goto err; } - irq = irq_of_parse_and_map(child, 0); - if (!irq) { + ret = fwnode_irq_get(child, 0); + if (ret < 0) { + fwnode_handle_put(child); + dev_err(pctl->dev, "Failed to retrieve IRQ for bank %u\n", i); + goto err; + } + if (!ret) { + fwnode_handle_put(child); dev_err(pctl->dev, "No IRQ for bank %u\n", i); - of_node_put(child); ret = -EINVAL; goto err; } + irq = ret; bank = &pctl->gpio_banks[i]; bank->pctl = pctl; bank->base = pctl->base + GPIO_BANK_BASE(i); bank->gpio_chip.parent = pctl->dev; - bank->gpio_chip.of_node = child; + bank->gpio_chip.fwnode = child; girq = &bank->gpio_chip.irq; girq->chip = &bank->irq_chip; diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index 32e41395fc76875dd551acaef2442385c9297a08..53bdfc40f0558b9b5d96915c9e59e3770411ca3e 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -57,6 +57,7 @@ #define IOMUX_UNROUTED BIT(3) #define IOMUX_WIDTH_3BIT BIT(4) #define IOMUX_WIDTH_2BIT BIT(5) +#define IOMUX_L_SOURCE_PMU BIT(6) #define PIN_BANK(id, pins, label) \ { \ @@ -147,6 +148,21 @@ .pull_type[3] = pull3, \ } +#define PIN_BANK_IOMUX_FLAGS_OFFSET(id, pins, label, iom0, iom1, iom2, \ + iom3, offset0, offset1, offset2, \ + offset3) \ + { \ + .bank_num = id, \ + .nr_pins = pins, \ + .name = label, \ + .iomux = { \ + { .type = iom0, .offset = offset0 }, \ + { .type = iom1, .offset = offset1 }, \ + { .type = iom2, .offset = offset2 }, \ + { .type = iom3, .offset = offset3 }, \ + }, \ + } + #define PIN_BANK_IOMUX_DRV_FLAGS_OFFSET(id, pins, label, iom0, iom1, \ iom2, iom3, drv0, drv1, drv2, \ drv3, offset0, offset1, \ @@ -443,6 +459,37 @@ static struct rockchip_mux_recalced_data rv1108_mux_recalced_data[] = { }, }; +static struct rockchip_mux_recalced_data rv1126_mux_recalced_data[] = { + { + .num = 0, + .pin = 20, + .reg = 0x10000, + .bit = 0, + .mask = 0xf + }, + { + .num = 0, + .pin = 21, + .reg = 0x10000, + .bit = 4, + .mask = 0xf + }, + { + .num = 0, + .pin = 22, + .reg = 0x10000, + .bit = 8, + .mask = 0xf + }, + { + .num = 0, + .pin = 23, + .reg = 0x10000, + .bit = 12, + .mask = 0xf + }, +}; + static struct rockchip_mux_recalced_data rk3128_mux_recalced_data[] = { { .num = 2, @@ -642,6 +689,103 @@ static struct rockchip_mux_route_data px30_mux_route_data[] = { RK_MUXROUTE_SAME(1, RK_PB7, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-rxm1 */ }; +static struct rockchip_mux_route_data rv1126_mux_route_data[] = { + RK_MUXROUTE_GRF(3, RK_PD2, 1, 0x10260, WRITE_MASK_VAL(0, 0, 0)), /* I2S0_MCLK_M0 */ + RK_MUXROUTE_GRF(3, RK_PB0, 3, 0x10260, WRITE_MASK_VAL(0, 0, 1)), /* I2S0_MCLK_M1 */ + + RK_MUXROUTE_GRF(0, RK_PD4, 4, 0x10260, WRITE_MASK_VAL(3, 2, 0)), /* I2S1_MCLK_M0 */ + RK_MUXROUTE_GRF(1, RK_PD5, 2, 0x10260, WRITE_MASK_VAL(3, 2, 1)), /* I2S1_MCLK_M1 */ + RK_MUXROUTE_GRF(2, RK_PC7, 6, 0x10260, WRITE_MASK_VAL(3, 2, 2)), /* I2S1_MCLK_M2 */ + + RK_MUXROUTE_GRF(1, RK_PD0, 1, 0x10260, WRITE_MASK_VAL(4, 4, 0)), /* I2S2_MCLK_M0 */ + RK_MUXROUTE_GRF(2, RK_PB3, 2, 0x10260, WRITE_MASK_VAL(4, 4, 1)), /* I2S2_MCLK_M1 */ + + RK_MUXROUTE_GRF(3, RK_PD4, 2, 0x10260, WRITE_MASK_VAL(12, 12, 0)), /* PDM_CLK0_M0 */ + RK_MUXROUTE_GRF(3, RK_PC0, 3, 0x10260, WRITE_MASK_VAL(12, 12, 1)), /* PDM_CLK0_M1 */ + + RK_MUXROUTE_GRF(3, RK_PC6, 1, 0x10264, WRITE_MASK_VAL(0, 0, 0)), /* CIF_CLKOUT_M0 */ + RK_MUXROUTE_GRF(2, RK_PD1, 3, 0x10264, WRITE_MASK_VAL(0, 0, 1)), /* CIF_CLKOUT_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA4, 5, 0x10264, WRITE_MASK_VAL(5, 4, 0)), /* I2C3_SCL_M0 */ + RK_MUXROUTE_GRF(2, RK_PD4, 7, 0x10264, WRITE_MASK_VAL(5, 4, 1)), /* I2C3_SCL_M1 */ + RK_MUXROUTE_GRF(1, RK_PD6, 3, 0x10264, WRITE_MASK_VAL(5, 4, 2)), /* I2C3_SCL_M2 */ + + RK_MUXROUTE_GRF(3, RK_PA0, 7, 0x10264, WRITE_MASK_VAL(6, 6, 0)), /* I2C4_SCL_M0 */ + RK_MUXROUTE_GRF(4, RK_PA0, 4, 0x10264, WRITE_MASK_VAL(6, 6, 1)), /* I2C4_SCL_M1 */ + + RK_MUXROUTE_GRF(2, RK_PA5, 7, 0x10264, WRITE_MASK_VAL(9, 8, 0)), /* I2C5_SCL_M0 */ + RK_MUXROUTE_GRF(3, RK_PB0, 5, 0x10264, WRITE_MASK_VAL(9, 8, 1)), /* I2C5_SCL_M1 */ + RK_MUXROUTE_GRF(1, RK_PD0, 4, 0x10264, WRITE_MASK_VAL(9, 8, 2)), /* I2C5_SCL_M2 */ + + RK_MUXROUTE_GRF(3, RK_PC0, 5, 0x10264, WRITE_MASK_VAL(11, 10, 0)), /* SPI1_CLK_M0 */ + RK_MUXROUTE_GRF(1, RK_PC6, 3, 0x10264, WRITE_MASK_VAL(11, 10, 1)), /* SPI1_CLK_M1 */ + RK_MUXROUTE_GRF(2, RK_PD5, 6, 0x10264, WRITE_MASK_VAL(11, 10, 2)), /* SPI1_CLK_M2 */ + + RK_MUXROUTE_GRF(3, RK_PC0, 2, 0x10264, WRITE_MASK_VAL(12, 12, 0)), /* RGMII_CLK_M0 */ + RK_MUXROUTE_GRF(2, RK_PB7, 2, 0x10264, WRITE_MASK_VAL(12, 12, 1)), /* RGMII_CLK_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA1, 3, 0x10264, WRITE_MASK_VAL(13, 13, 0)), /* CAN_TXD_M0 */ + RK_MUXROUTE_GRF(3, RK_PA7, 5, 0x10264, WRITE_MASK_VAL(13, 13, 1)), /* CAN_TXD_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA4, 6, 0x10268, WRITE_MASK_VAL(0, 0, 0)), /* PWM8_M0 */ + RK_MUXROUTE_GRF(2, RK_PD7, 5, 0x10268, WRITE_MASK_VAL(0, 0, 1)), /* PWM8_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA5, 6, 0x10268, WRITE_MASK_VAL(2, 2, 0)), /* PWM9_M0 */ + RK_MUXROUTE_GRF(2, RK_PD6, 5, 0x10268, WRITE_MASK_VAL(2, 2, 1)), /* PWM9_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA6, 6, 0x10268, WRITE_MASK_VAL(4, 4, 0)), /* PWM10_M0 */ + RK_MUXROUTE_GRF(2, RK_PD5, 5, 0x10268, WRITE_MASK_VAL(4, 4, 1)), /* PWM10_M1 */ + + RK_MUXROUTE_GRF(3, RK_PA7, 6, 0x10268, WRITE_MASK_VAL(6, 6, 0)), /* PWM11_IR_M0 */ + RK_MUXROUTE_GRF(3, RK_PA1, 5, 0x10268, WRITE_MASK_VAL(6, 6, 1)), /* PWM11_IR_M1 */ + + RK_MUXROUTE_GRF(1, RK_PA5, 3, 0x10268, WRITE_MASK_VAL(8, 8, 0)), /* UART2_TX_M0 */ + RK_MUXROUTE_GRF(3, RK_PA2, 1, 0x10268, WRITE_MASK_VAL(8, 8, 1)), /* UART2_TX_M1 */ + + RK_MUXROUTE_GRF(3, RK_PC6, 3, 0x10268, WRITE_MASK_VAL(11, 10, 0)), /* UART3_TX_M0 */ + RK_MUXROUTE_GRF(1, RK_PA7, 2, 0x10268, WRITE_MASK_VAL(11, 10, 1)), /* UART3_TX_M1 */ + RK_MUXROUTE_GRF(3, RK_PA0, 4, 0x10268, WRITE_MASK_VAL(11, 10, 2)), /* UART3_TX_M2 */ + + RK_MUXROUTE_GRF(3, RK_PA4, 4, 0x10268, WRITE_MASK_VAL(13, 12, 0)), /* UART4_TX_M0 */ + RK_MUXROUTE_GRF(2, RK_PA6, 4, 0x10268, WRITE_MASK_VAL(13, 12, 1)), /* UART4_TX_M1 */ + RK_MUXROUTE_GRF(1, RK_PD5, 3, 0x10268, WRITE_MASK_VAL(13, 12, 2)), /* UART4_TX_M2 */ + + RK_MUXROUTE_GRF(3, RK_PA6, 4, 0x10268, WRITE_MASK_VAL(15, 14, 0)), /* UART5_TX_M0 */ + RK_MUXROUTE_GRF(2, RK_PB0, 4, 0x10268, WRITE_MASK_VAL(15, 14, 1)), /* UART5_TX_M1 */ + RK_MUXROUTE_GRF(2, RK_PA0, 3, 0x10268, WRITE_MASK_VAL(15, 14, 2)), /* UART5_TX_M2 */ + + RK_MUXROUTE_PMU(0, RK_PB6, 3, 0x0114, WRITE_MASK_VAL(0, 0, 0)), /* PWM0_M0 */ + RK_MUXROUTE_PMU(2, RK_PB3, 5, 0x0114, WRITE_MASK_VAL(0, 0, 1)), /* PWM0_M1 */ + + RK_MUXROUTE_PMU(0, RK_PB7, 3, 0x0114, WRITE_MASK_VAL(2, 2, 0)), /* PWM1_M0 */ + RK_MUXROUTE_PMU(2, RK_PB2, 5, 0x0114, WRITE_MASK_VAL(2, 2, 1)), /* PWM1_M1 */ + + RK_MUXROUTE_PMU(0, RK_PC0, 3, 0x0114, WRITE_MASK_VAL(4, 4, 0)), /* PWM2_M0 */ + RK_MUXROUTE_PMU(2, RK_PB1, 5, 0x0114, WRITE_MASK_VAL(4, 4, 1)), /* PWM2_M1 */ + + RK_MUXROUTE_PMU(0, RK_PC1, 3, 0x0114, WRITE_MASK_VAL(6, 6, 0)), /* PWM3_IR_M0 */ + RK_MUXROUTE_PMU(2, RK_PB0, 5, 0x0114, WRITE_MASK_VAL(6, 6, 1)), /* PWM3_IR_M1 */ + + RK_MUXROUTE_PMU(0, RK_PC2, 3, 0x0114, WRITE_MASK_VAL(8, 8, 0)), /* PWM4_M0 */ + RK_MUXROUTE_PMU(2, RK_PA7, 5, 0x0114, WRITE_MASK_VAL(8, 8, 1)), /* PWM4_M1 */ + + RK_MUXROUTE_PMU(0, RK_PC3, 3, 0x0114, WRITE_MASK_VAL(10, 10, 0)), /* PWM5_M0 */ + RK_MUXROUTE_PMU(2, RK_PA6, 5, 0x0114, WRITE_MASK_VAL(10, 10, 1)), /* PWM5_M1 */ + + RK_MUXROUTE_PMU(0, RK_PB2, 3, 0x0114, WRITE_MASK_VAL(12, 12, 0)), /* PWM6_M0 */ + RK_MUXROUTE_PMU(2, RK_PD4, 5, 0x0114, WRITE_MASK_VAL(12, 12, 1)), /* PWM6_M1 */ + + RK_MUXROUTE_PMU(0, RK_PB1, 3, 0x0114, WRITE_MASK_VAL(14, 14, 0)), /* PWM7_IR_M0 */ + RK_MUXROUTE_PMU(3, RK_PA0, 5, 0x0114, WRITE_MASK_VAL(14, 14, 1)), /* PWM7_IR_M1 */ + + RK_MUXROUTE_PMU(0, RK_PB0, 1, 0x0118, WRITE_MASK_VAL(1, 0, 0)), /* SPI0_CLK_M0 */ + RK_MUXROUTE_PMU(2, RK_PA1, 1, 0x0118, WRITE_MASK_VAL(1, 0, 1)), /* SPI0_CLK_M1 */ + RK_MUXROUTE_PMU(2, RK_PB2, 6, 0x0118, WRITE_MASK_VAL(1, 0, 2)), /* SPI0_CLK_M2 */ + + RK_MUXROUTE_PMU(0, RK_PB6, 2, 0x0118, WRITE_MASK_VAL(2, 2, 0)), /* UART1_TX_M0 */ + RK_MUXROUTE_PMU(1, RK_PD0, 5, 0x0118, WRITE_MASK_VAL(2, 2, 1)), /* UART1_TX_M1 */ +}; + static struct rockchip_mux_route_data rk3128_mux_route_data[] = { RK_MUXROUTE_SAME(1, RK_PB2, 1, 0x144, BIT(16 + 3) | BIT(16 + 4)), /* spi-0 */ RK_MUXROUTE_SAME(1, RK_PD3, 3, 0x144, BIT(16 + 3) | BIT(16 + 4) | BIT(3)), /* spi-1 */ @@ -877,8 +1021,12 @@ static int rockchip_get_mux(struct rockchip_pin_bank *bank, int pin) if (bank->iomux[iomux_num].type & IOMUX_GPIO_ONLY) return RK_FUNC_GPIO; - regmap = (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU) - ? info->regmap_pmu : info->regmap_base; + if (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU) + regmap = info->regmap_pmu; + else if (bank->iomux[iomux_num].type & IOMUX_L_SOURCE_PMU) + regmap = (pin % 8 < 4) ? info->regmap_pmu : info->regmap_base; + else + regmap = info->regmap_base; /* get basic quadrupel of mux registers and the correct reg inside */ mux_type = bank->iomux[iomux_num].type; @@ -987,8 +1135,12 @@ static int rockchip_set_mux(struct rockchip_pin_bank *bank, int pin, int mux) dev_dbg(dev, "setting mux of GPIO%d-%d to %d\n", bank->bank_num, pin, mux); - regmap = (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU) - ? info->regmap_pmu : info->regmap_base; + if (bank->iomux[iomux_num].type & IOMUX_SOURCE_PMU) + regmap = info->regmap_pmu; + else if (bank->iomux[iomux_num].type & IOMUX_L_SOURCE_PMU) + regmap = (pin % 8 < 4) ? info->regmap_pmu : info->regmap_base; + else + regmap = info->regmap_base; /* get basic quadrupel of mux registers and the correct reg inside */ mux_type = bank->iomux[iomux_num].type; @@ -1268,6 +1420,119 @@ static int rv1108_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank, return 0; } +#define RV1126_PULL_PMU_OFFSET 0x40 +#define RV1126_PULL_GRF_GPIO1A0_OFFSET 0x10108 +#define RV1126_PULL_PINS_PER_REG 8 +#define RV1126_PULL_BITS_PER_PIN 2 +#define RV1126_PULL_BANK_STRIDE 16 +#define RV1126_GPIO_C4_D7(p) (p >= 20 && p <= 31) /* GPIO0_C4 ~ GPIO0_D7 */ + +static int rv1126_calc_pull_reg_and_bit(struct rockchip_pin_bank *bank, + int pin_num, struct regmap **regmap, + int *reg, u8 *bit) +{ + struct rockchip_pinctrl *info = bank->drvdata; + + /* The first 24 pins of the first bank are located in PMU */ + if (bank->bank_num == 0) { + if (RV1126_GPIO_C4_D7(pin_num)) { + *regmap = info->regmap_base; + *reg = RV1126_PULL_GRF_GPIO1A0_OFFSET; + *reg -= (((31 - pin_num) / RV1126_PULL_PINS_PER_REG + 1) * 4); + *bit = pin_num % RV1126_PULL_PINS_PER_REG; + *bit *= RV1126_PULL_BITS_PER_PIN; + return 0; + } + *regmap = info->regmap_pmu; + *reg = RV1126_PULL_PMU_OFFSET; + } else { + *reg = RV1126_PULL_GRF_GPIO1A0_OFFSET; + *regmap = info->regmap_base; + *reg += (bank->bank_num - 1) * RV1126_PULL_BANK_STRIDE; + } + + *reg += ((pin_num / RV1126_PULL_PINS_PER_REG) * 4); + *bit = (pin_num % RV1126_PULL_PINS_PER_REG); + *bit *= RV1126_PULL_BITS_PER_PIN; + + return 0; +} + +#define RV1126_DRV_PMU_OFFSET 0x20 +#define RV1126_DRV_GRF_GPIO1A0_OFFSET 0x10090 +#define RV1126_DRV_BITS_PER_PIN 4 +#define RV1126_DRV_PINS_PER_REG 4 +#define RV1126_DRV_BANK_STRIDE 32 + +static int rv1126_calc_drv_reg_and_bit(struct rockchip_pin_bank *bank, + int pin_num, struct regmap **regmap, + int *reg, u8 *bit) +{ + struct rockchip_pinctrl *info = bank->drvdata; + + /* The first 24 pins of the first bank are located in PMU */ + if (bank->bank_num == 0) { + if (RV1126_GPIO_C4_D7(pin_num)) { + *regmap = info->regmap_base; + *reg = RV1126_DRV_GRF_GPIO1A0_OFFSET; + *reg -= (((31 - pin_num) / RV1126_DRV_PINS_PER_REG + 1) * 4); + *reg -= 0x4; + *bit = pin_num % RV1126_DRV_PINS_PER_REG; + *bit *= RV1126_DRV_BITS_PER_PIN; + return 0; + } + *regmap = info->regmap_pmu; + *reg = RV1126_DRV_PMU_OFFSET; + } else { + *regmap = info->regmap_base; + *reg = RV1126_DRV_GRF_GPIO1A0_OFFSET; + *reg += (bank->bank_num - 1) * RV1126_DRV_BANK_STRIDE; + } + + *reg += ((pin_num / RV1126_DRV_PINS_PER_REG) * 4); + *bit = pin_num % RV1126_DRV_PINS_PER_REG; + *bit *= RV1126_DRV_BITS_PER_PIN; + + return 0; +} + +#define RV1126_SCHMITT_PMU_OFFSET 0x60 +#define RV1126_SCHMITT_GRF_GPIO1A0_OFFSET 0x10188 +#define RV1126_SCHMITT_BANK_STRIDE 16 +#define RV1126_SCHMITT_PINS_PER_GRF_REG 8 +#define RV1126_SCHMITT_PINS_PER_PMU_REG 8 + +static int rv1126_calc_schmitt_reg_and_bit(struct rockchip_pin_bank *bank, + int pin_num, + struct regmap **regmap, + int *reg, u8 *bit) +{ + struct rockchip_pinctrl *info = bank->drvdata; + int pins_per_reg; + + if (bank->bank_num == 0) { + if (RV1126_GPIO_C4_D7(pin_num)) { + *regmap = info->regmap_base; + *reg = RV1126_SCHMITT_GRF_GPIO1A0_OFFSET; + *reg -= (((31 - pin_num) / RV1126_SCHMITT_PINS_PER_GRF_REG + 1) * 4); + *bit = pin_num % RV1126_SCHMITT_PINS_PER_GRF_REG; + return 0; + } + *regmap = info->regmap_pmu; + *reg = RV1126_SCHMITT_PMU_OFFSET; + pins_per_reg = RV1126_SCHMITT_PINS_PER_PMU_REG; + } else { + *regmap = info->regmap_base; + *reg = RV1126_SCHMITT_GRF_GPIO1A0_OFFSET; + pins_per_reg = RV1126_SCHMITT_PINS_PER_GRF_REG; + *reg += (bank->bank_num - 1) * RV1126_SCHMITT_BANK_STRIDE; + } + *reg += ((pin_num / pins_per_reg) * 4); + *bit = pin_num % pins_per_reg; + + return 0; +} + #define RK3308_SCHMITT_PINS_PER_REG 8 #define RK3308_SCHMITT_BANK_STRIDE 16 #define RK3308_SCHMITT_GRF_OFFSET 0x1a0 @@ -1998,6 +2263,12 @@ static int rockchip_set_drive_perpin(struct rockchip_pin_bank *bank, goto config; } + if (ctrl->type == RV1126) { + rmask_bits = RV1126_DRV_BITS_PER_PIN; + ret = strength; + goto config; + } + ret = -EINVAL; for (i = 0; i < ARRAY_SIZE(rockchip_perpin_drv_list[drv_type]); i++) { if (rockchip_perpin_drv_list[drv_type][i] == strength) { @@ -2168,6 +2439,7 @@ static int rockchip_set_pull(struct rockchip_pin_bank *bank, break; case PX30: case RV1108: + case RV1126: case RK3188: case RK3288: case RK3308: @@ -2393,11 +2665,24 @@ static int rockchip_pmx_set(struct pinctrl_dev *pctldev, unsigned selector, return 0; } +static int rockchip_pmx_gpio_set_direction(struct pinctrl_dev *pctldev, + struct pinctrl_gpio_range *range, + unsigned offset, + bool input) +{ + struct rockchip_pinctrl *info = pinctrl_dev_get_drvdata(pctldev); + struct rockchip_pin_bank *bank; + + bank = pin_to_bank(info, offset); + return rockchip_set_mux(bank, offset - bank->pin_base, RK_FUNC_GPIO); +} + static const struct pinmux_ops rockchip_pmx_ops = { .get_functions_count = rockchip_pmx_get_funcs_count, .get_function_name = rockchip_pmx_get_func_name, .get_function_groups = rockchip_pmx_get_groups, .set_mux = rockchip_pmx_set, + .gpio_set_direction = rockchip_pmx_gpio_set_direction, }; /* @@ -2416,6 +2701,7 @@ static bool rockchip_pinconf_pull_valid(struct rockchip_pin_ctrl *ctrl, return pull ? false : true; case PX30: case RV1108: + case RV1126: case RK3188: case RK3288: case RK3308: @@ -2889,12 +3175,14 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data( /* preset iomux offset value, set new start value */ if (iom->offset >= 0) { - if (iom->type & IOMUX_SOURCE_PMU) + if ((iom->type & IOMUX_SOURCE_PMU) || + (iom->type & IOMUX_L_SOURCE_PMU)) pmu_offs = iom->offset; else grf_offs = iom->offset; } else { /* set current iomux offset */ - iom->offset = (iom->type & IOMUX_SOURCE_PMU) ? + iom->offset = ((iom->type & IOMUX_SOURCE_PMU) || + (iom->type & IOMUX_L_SOURCE_PMU)) ? pmu_offs : grf_offs; } @@ -2919,7 +3207,7 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data( inc = (iom->type & (IOMUX_WIDTH_4BIT | IOMUX_WIDTH_3BIT | IOMUX_WIDTH_2BIT)) ? 8 : 4; - if (iom->type & IOMUX_SOURCE_PMU) + if ((iom->type & IOMUX_SOURCE_PMU) || (iom->type & IOMUX_L_SOURCE_PMU)) pmu_offs += inc; else grf_offs += inc; @@ -3178,6 +3466,48 @@ static struct rockchip_pin_ctrl rv1108_pin_ctrl = { .schmitt_calc_reg = rv1108_calc_schmitt_reg_and_bit, }; +static struct rockchip_pin_bank rv1126_pin_banks[] = { + PIN_BANK_IOMUX_FLAGS(0, 32, "gpio0", + IOMUX_WIDTH_4BIT | IOMUX_SOURCE_PMU, + IOMUX_WIDTH_4BIT | IOMUX_SOURCE_PMU, + IOMUX_WIDTH_4BIT | IOMUX_L_SOURCE_PMU, + IOMUX_WIDTH_4BIT), + PIN_BANK_IOMUX_FLAGS_OFFSET(1, 32, "gpio1", + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + 0x10010, 0x10018, 0x10020, 0x10028), + PIN_BANK_IOMUX_FLAGS(2, 32, "gpio2", + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT), + PIN_BANK_IOMUX_FLAGS(3, 32, "gpio3", + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT, + IOMUX_WIDTH_4BIT), + PIN_BANK_IOMUX_FLAGS(4, 2, "gpio4", + IOMUX_WIDTH_4BIT, 0, 0, 0), +}; + +static struct rockchip_pin_ctrl rv1126_pin_ctrl = { + .pin_banks = rv1126_pin_banks, + .nr_banks = ARRAY_SIZE(rv1126_pin_banks), + .label = "RV1126-GPIO", + .type = RV1126, + .grf_mux_offset = 0x10004, /* mux offset from GPIO0_D0 */ + .pmu_mux_offset = 0x0, + .iomux_routes = rv1126_mux_route_data, + .niomux_routes = ARRAY_SIZE(rv1126_mux_route_data), + .iomux_recalced = rv1126_mux_recalced_data, + .niomux_recalced = ARRAY_SIZE(rv1126_mux_recalced_data), + .pull_calc_reg = rv1126_calc_pull_reg_and_bit, + .drv_calc_reg = rv1126_calc_drv_reg_and_bit, + .schmitt_calc_reg = rv1126_calc_schmitt_reg_and_bit, +}; + static struct rockchip_pin_bank rk2928_pin_banks[] = { PIN_BANK(0, 32, "gpio0"), PIN_BANK(1, 32, "gpio1"), @@ -3568,6 +3898,8 @@ static const struct of_device_id rockchip_pinctrl_dt_match[] = { .data = &px30_pin_ctrl }, { .compatible = "rockchip,rv1108-pinctrl", .data = &rv1108_pin_ctrl }, + { .compatible = "rockchip,rv1126-pinctrl", + .data = &rv1126_pin_ctrl }, { .compatible = "rockchip,rk2928-pinctrl", .data = &rk2928_pin_ctrl }, { .compatible = "rockchip,rk3036-pinctrl", diff --git a/drivers/pinctrl/pinctrl-rockchip.h b/drivers/pinctrl/pinctrl-rockchip.h index ec46f8815ac9002820e7300404467b8cdd1fe824..4759f336941ef3017d1f36fb42ab33ae3d2c504b 100644 --- a/drivers/pinctrl/pinctrl-rockchip.h +++ b/drivers/pinctrl/pinctrl-rockchip.h @@ -186,6 +186,7 @@ enum rockchip_pinctrl_type { PX30, RV1108, + RV1126, RK2928, RK3066B, RK3128, diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c index 0fea71fd9a005a14d75db4ec3544963813602a70..cf7f9cbe604400e9f20889f432c6ddce27bbc890 100644 --- a/drivers/pinctrl/pinctrl-st.c +++ b/drivers/pinctrl/pinctrl-st.c @@ -12,7 +12,6 @@ #include #include #include -#include /* of_get_named_gpio() */ #include #include #include @@ -1162,6 +1161,31 @@ static void st_parse_syscfgs(struct st_pinctrl *info, int bank, return; } +static int st_pctl_dt_calculate_pin(struct st_pinctrl *info, + phandle bank, unsigned int offset) +{ + struct device_node *np; + struct gpio_chip *chip; + int retval = -EINVAL; + int i; + + np = of_find_node_by_phandle(bank); + if (!np) + return -EINVAL; + + for (i = 0; i < info->nbanks; i++) { + chip = &info->banks[i].gpio_chip; + if (chip->of_node == np) { + if (offset < chip->ngpio) + retval = chip->base + offset; + break; + } + } + + of_node_put(np); + return retval; +} + /* * Each pin is represented in of the below forms. * @@ -1175,6 +1199,8 @@ static int st_pctl_dt_parse_groups(struct device_node *np, struct device *dev = info->dev; struct st_pinconf *conf; struct device_node *pins; + phandle bank; + unsigned int offset; int i = 0, npins = 0, nr_props, ret = 0; pins = of_get_child_by_name(np, "st,pins"); @@ -1214,9 +1240,9 @@ static int st_pctl_dt_parse_groups(struct device_node *np, conf = &grp->pin_conf[i]; /* bank & offset */ - be32_to_cpup(list++); - be32_to_cpup(list++); - conf->pin = of_get_named_gpio(pins, pp->name, 0); + bank = be32_to_cpup(list++); + offset = be32_to_cpup(list++); + conf->pin = st_pctl_dt_calculate_pin(info, bank, offset); conf->name = pp->name; grp->pins[i] = conf->pin; /* mux */ diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig index f415c13caae044ad387b7c4c85ca7bfa3f1862cf..9dc2d803a58674cd9ba24ac39ad43d3490b81f50 100644 --- a/drivers/pinctrl/qcom/Kconfig +++ b/drivers/pinctrl/qcom/Kconfig @@ -15,6 +15,7 @@ config PINCTRL_MSM config PINCTRL_APQ8064 tristate "Qualcomm APQ8064 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -23,6 +24,7 @@ config PINCTRL_APQ8064 config PINCTRL_APQ8084 tristate "Qualcomm APQ8084 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -31,6 +33,7 @@ config PINCTRL_APQ8084 config PINCTRL_IPQ4019 tristate "Qualcomm IPQ4019 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -39,6 +42,7 @@ config PINCTRL_IPQ4019 config PINCTRL_IPQ8064 tristate "Qualcomm IPQ8064 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -47,6 +51,7 @@ config PINCTRL_IPQ8064 config PINCTRL_IPQ8074 tristate "Qualcomm Technologies, Inc. IPQ8074 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for @@ -57,6 +62,7 @@ config PINCTRL_IPQ8074 config PINCTRL_IPQ6018 tristate "Qualcomm Technologies, Inc. IPQ6018 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for @@ -67,6 +73,7 @@ config PINCTRL_IPQ6018 config PINCTRL_MSM8226 tristate "Qualcomm 8226 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -76,6 +83,7 @@ config PINCTRL_MSM8226 config PINCTRL_MSM8660 tristate "Qualcomm 8660 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -84,6 +92,7 @@ config PINCTRL_MSM8660 config PINCTRL_MSM8960 tristate "Qualcomm 8960 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -100,6 +109,7 @@ config PINCTRL_MDM9607 config PINCTRL_MDM9615 tristate "Qualcomm 9615 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -108,6 +118,7 @@ config PINCTRL_MDM9615 config PINCTRL_MSM8X74 tristate "Qualcomm 8x74 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -116,6 +127,7 @@ config PINCTRL_MSM8X74 config PINCTRL_MSM8909 tristate "Qualcomm 8909 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -132,6 +144,7 @@ config PINCTRL_MSM8916 config PINCTRL_MSM8953 tristate "Qualcomm 8953 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -142,6 +155,7 @@ config PINCTRL_MSM8953 config PINCTRL_MSM8976 tristate "Qualcomm 8976 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -152,6 +166,7 @@ config PINCTRL_MSM8976 config PINCTRL_MSM8994 tristate "Qualcomm 8994 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -161,6 +176,7 @@ config PINCTRL_MSM8994 config PINCTRL_MSM8996 tristate "Qualcomm MSM8996 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -169,6 +185,7 @@ config PINCTRL_MSM8996 config PINCTRL_MSM8998 tristate "Qualcomm MSM8998 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -177,6 +194,7 @@ config PINCTRL_MSM8998 config PINCTRL_QCM2290 tristate "Qualcomm QCM2290 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -185,6 +203,7 @@ config PINCTRL_QCM2290 config PINCTRL_QCS404 tristate "Qualcomm QCS404 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -232,6 +251,7 @@ config PINCTRL_QCOM_SSBI_PMIC config PINCTRL_SC7180 tristate "Qualcomm Technologies Inc SC7180 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -241,6 +261,7 @@ config PINCTRL_SC7180 config PINCTRL_SC7280 tristate "Qualcomm Technologies Inc SC7280 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -250,6 +271,7 @@ config PINCTRL_SC7280 config PINCTRL_SC7280_LPASS_LPI tristate "Qualcomm Technologies Inc SC7280 LPASS LPI pin controller driver" depends on GPIOLIB + depends on ARM64 || COMPILE_TEST depends on PINCTRL_LPASS_LPI help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -259,6 +281,7 @@ config PINCTRL_SC7280_LPASS_LPI config PINCTRL_SC8180X tristate "Qualcomm Technologies Inc SC8180x pin controller driver" depends on (OF || ACPI) + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -268,6 +291,7 @@ config PINCTRL_SC8180X config PINCTRL_SC8280XP tristate "Qualcomm Technologies Inc SC8280xp pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -277,6 +301,7 @@ config PINCTRL_SC8280XP config PINCTRL_SDM660 tristate "Qualcomm Technologies Inc SDM660 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -286,6 +311,7 @@ config PINCTRL_SDM660 config PINCTRL_SDM845 tristate "Qualcomm Technologies Inc SDM845 pin controller driver" depends on (OF || ACPI) + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -295,6 +321,7 @@ config PINCTRL_SDM845 config PINCTRL_SDX55 tristate "Qualcomm Technologies Inc SDX55 pin controller driver" depends on OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -304,6 +331,7 @@ config PINCTRL_SDX55 config PINCTRL_SM6115 tristate "Qualcomm Technologies Inc SM6115,SM4250 pin controller driver" depends on GPIOLIB && OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -313,6 +341,7 @@ config PINCTRL_SM6115 config PINCTRL_SM6125 tristate "Qualcomm Technologies Inc SM6125 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -322,6 +351,7 @@ config PINCTRL_SM6125 config PINCTRL_SM6350 tristate "Qualcomm Technologies Inc SM6350 pin controller driver" depends on GPIOLIB && OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -331,6 +361,7 @@ config PINCTRL_SM6350 config PINCTRL_SM6375 tristate "Qualcomm Technologies Inc SM6375 pin controller driver" depends on GPIOLIB && OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -340,6 +371,7 @@ config PINCTRL_SM6375 config PINCTRL_SDX65 tristate "Qualcomm Technologies Inc SDX65 pin controller driver" depends on GPIOLIB && OF + depends on ARM || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -349,6 +381,7 @@ config PINCTRL_SDX65 config PINCTRL_SM8150 tristate "Qualcomm Technologies Inc SM8150 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -358,6 +391,7 @@ config PINCTRL_SM8150 config PINCTRL_SM8250 tristate "Qualcomm Technologies Inc SM8250 pin controller driver" depends on OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -367,6 +401,7 @@ config PINCTRL_SM8250 config PINCTRL_SM8250_LPASS_LPI tristate "Qualcomm Technologies Inc SM8250 LPASS LPI pin controller driver" depends on GPIOLIB + depends on ARM64 || COMPILE_TEST depends on PINCTRL_LPASS_LPI help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -375,6 +410,7 @@ config PINCTRL_SM8250_LPASS_LPI config PINCTRL_SM8350 tristate "Qualcomm Technologies Inc SM8350 pin controller driver" + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the @@ -384,12 +420,33 @@ config PINCTRL_SM8350 config PINCTRL_SM8450 tristate "Qualcomm Technologies Inc SM8450 pin controller driver" depends on GPIOLIB && OF + depends on ARM64 || COMPILE_TEST depends on PINCTRL_MSM help This is the pinctrl, pinmux, pinconf and gpiolib driver for the Qualcomm Technologies Inc TLMM block found on the Qualcomm Technologies Inc SM8450 platform. +config PINCTRL_SM8450_LPASS_LPI + tristate "Qualcomm Technologies Inc SM8450 LPASS LPI pin controller driver" + depends on GPIOLIB + depends on ARM64 || COMPILE_TEST + depends on PINCTRL_LPASS_LPI + help + This is the pinctrl, pinmux, pinconf and gpiolib driver for the + Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI + (Low Power Island) found on the Qualcomm Technologies Inc SM8450 platform. + +config PINCTRL_SC8280XP_LPASS_LPI + tristate "Qualcomm Technologies Inc SC8280XP LPASS LPI pin controller driver" + depends on GPIOLIB + depends on ARM64 || COMPILE_TEST + depends on PINCTRL_LPASS_LPI + help + This is the pinctrl, pinmux, pinconf and gpiolib driver for the + Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI + (Low Power Island) found on the Qualcomm Technologies Inc SC8280XP platform. + config PINCTRL_LPASS_LPI tristate "Qualcomm Technologies Inc LPASS LPI pin controller driver" select PINMUX diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile index fbd64853a24db299974de85138a65318650cdc03..8269a1db8794a45583f8acf71eeafadd7b8410ea 100644 --- a/drivers/pinctrl/qcom/Makefile +++ b/drivers/pinctrl/qcom/Makefile @@ -45,4 +45,6 @@ obj-$(CONFIG_PINCTRL_SM8250) += pinctrl-sm8250.o obj-$(CONFIG_PINCTRL_SM8250_LPASS_LPI) += pinctrl-sm8250-lpass-lpi.o obj-$(CONFIG_PINCTRL_SM8350) += pinctrl-sm8350.o obj-$(CONFIG_PINCTRL_SM8450) += pinctrl-sm8450.o +obj-$(CONFIG_PINCTRL_SM8450_LPASS_LPI) += pinctrl-sm8450-lpass-lpi.o +obj-$(CONFIG_PINCTRL_SC8280XP_LPASS_LPI) += pinctrl-sc8280xp-lpass-lpi.o obj-$(CONFIG_PINCTRL_LPASS_LPI) += pinctrl-lpass-lpi.o diff --git a/drivers/pinctrl/qcom/pinctrl-sc8180x.c b/drivers/pinctrl/qcom/pinctrl-sc8180x.c index 6bec7f1431348d51480c756d5af5bc029a727973..704a99d2f93cebe91e21e14ac51655bde2de9a3b 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8180x.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8180x.c @@ -530,10 +530,10 @@ DECLARE_MSM_GPIO_PINS(187); DECLARE_MSM_GPIO_PINS(188); DECLARE_MSM_GPIO_PINS(189); -static const unsigned int sdc2_clk_pins[] = { 190 }; -static const unsigned int sdc2_cmd_pins[] = { 191 }; -static const unsigned int sdc2_data_pins[] = { 192 }; -static const unsigned int ufs_reset_pins[] = { 193 }; +static const unsigned int ufs_reset_pins[] = { 190 }; +static const unsigned int sdc2_clk_pins[] = { 191 }; +static const unsigned int sdc2_cmd_pins[] = { 192 }; +static const unsigned int sdc2_data_pins[] = { 193 }; enum sc8180x_functions { msm_mux_adsp_ext, @@ -1582,7 +1582,7 @@ static const int sc8180x_acpi_reserved_gpios[] = { static const struct msm_gpio_wakeirq_map sc8180x_pdc_map[] = { { 3, 31 }, { 5, 32 }, { 8, 33 }, { 9, 34 }, { 10, 100 }, { 12, 104 }, { 24, 37 }, { 26, 38 }, { 27, 41 }, { 28, 42 }, { 30, 39 }, { 36, 43 }, - { 37, 43 }, { 38, 45 }, { 39, 118 }, { 39, 125 }, { 41, 47 }, + { 37, 44 }, { 38, 45 }, { 39, 118 }, { 39, 125 }, { 41, 47 }, { 42, 48 }, { 46, 50 }, { 47, 49 }, { 48, 51 }, { 49, 53 }, { 50, 52 }, { 51, 116 }, { 51, 123 }, { 53, 54 }, { 54, 55 }, { 55, 56 }, { 56, 57 }, { 58, 58 }, { 60, 60 }, { 68, 62 }, { 70, 63 }, { 76, 86 }, diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c new file mode 100644 index 0000000000000000000000000000000000000000..4b9c0beac32efe9bc8ed7c89006c6411a893e22f --- /dev/null +++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp-lpass-lpi.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Linaro Ltd. + */ + +#include +#include +#include + +#include "pinctrl-lpass-lpi.h" + +enum lpass_lpi_functions { + LPI_MUX_dmic1_clk, + LPI_MUX_dmic1_data, + LPI_MUX_dmic2_clk, + LPI_MUX_dmic2_data, + LPI_MUX_dmic3_clk, + LPI_MUX_dmic3_data, + LPI_MUX_dmic4_clk, + LPI_MUX_dmic4_data, + LPI_MUX_i2s1_clk, + LPI_MUX_i2s1_data, + LPI_MUX_i2s1_ws, + LPI_MUX_i2s2_clk, + LPI_MUX_i2s2_data, + LPI_MUX_i2s2_ws, + LPI_MUX_i2s3_clk, + LPI_MUX_i2s3_data, + LPI_MUX_i2s3_ws, + LPI_MUX_qua_mi2s_data, + LPI_MUX_qua_mi2s_sclk, + LPI_MUX_qua_mi2s_ws, + LPI_MUX_swr_rx_clk, + LPI_MUX_swr_rx_data, + LPI_MUX_swr_tx_clk, + LPI_MUX_swr_tx_data, + LPI_MUX_wsa_swr_clk, + LPI_MUX_wsa_swr_data, + LPI_MUX_wsa2_swr_clk, + LPI_MUX_wsa2_swr_data, + LPI_MUX_ext_mclk1_a, + LPI_MUX_ext_mclk1_b, + LPI_MUX_ext_mclk1_c, + LPI_MUX_gpio, + LPI_MUX__, +}; + +static int gpio0_pins[] = { 0 }; +static int gpio1_pins[] = { 1 }; +static int gpio2_pins[] = { 2 }; +static int gpio3_pins[] = { 3 }; +static int gpio4_pins[] = { 4 }; +static int gpio5_pins[] = { 5 }; +static int gpio6_pins[] = { 6 }; +static int gpio7_pins[] = { 7 }; +static int gpio8_pins[] = { 8 }; +static int gpio9_pins[] = { 9 }; +static int gpio10_pins[] = { 10 }; +static int gpio11_pins[] = { 11 }; +static int gpio12_pins[] = { 12 }; +static int gpio13_pins[] = { 13 }; +static int gpio14_pins[] = { 14 }; +static int gpio15_pins[] = { 15 }; +static int gpio16_pins[] = { 16 }; +static int gpio17_pins[] = { 17 }; +static int gpio18_pins[] = { 18 }; + +static const struct pinctrl_pin_desc sc8280xp_lpi_pins[] = { + PINCTRL_PIN(0, "gpio0"), + PINCTRL_PIN(1, "gpio1"), + PINCTRL_PIN(2, "gpio2"), + PINCTRL_PIN(3, "gpio3"), + PINCTRL_PIN(4, "gpio4"), + PINCTRL_PIN(5, "gpio5"), + PINCTRL_PIN(6, "gpio6"), + PINCTRL_PIN(7, "gpio7"), + PINCTRL_PIN(8, "gpio8"), + PINCTRL_PIN(9, "gpio9"), + PINCTRL_PIN(10, "gpio10"), + PINCTRL_PIN(11, "gpio11"), + PINCTRL_PIN(12, "gpio12"), + PINCTRL_PIN(13, "gpio13"), + PINCTRL_PIN(14, "gpio14"), + PINCTRL_PIN(15, "gpio15"), + PINCTRL_PIN(16, "gpio16"), + PINCTRL_PIN(17, "gpio17"), + PINCTRL_PIN(18, "gpio18"), +}; + +static const char * const swr_tx_clk_groups[] = { "gpio0" }; +static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" }; +static const char * const swr_rx_clk_groups[] = { "gpio3" }; +static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" }; +static const char * const dmic1_clk_groups[] = { "gpio6" }; +static const char * const dmic1_data_groups[] = { "gpio7" }; +static const char * const dmic2_clk_groups[] = { "gpio8" }; +static const char * const dmic2_data_groups[] = { "gpio9" }; +static const char * const dmic4_clk_groups[] = { "gpio17" }; +static const char * const dmic4_data_groups[] = { "gpio18" }; +static const char * const i2s2_clk_groups[] = { "gpio10" }; +static const char * const i2s2_ws_groups[] = { "gpio11" }; +static const char * const dmic3_clk_groups[] = { "gpio12" }; +static const char * const dmic3_data_groups[] = { "gpio13" }; +static const char * const qua_mi2s_sclk_groups[] = { "gpio0" }; +static const char * const qua_mi2s_ws_groups[] = { "gpio1" }; +static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" }; +static const char * const i2s1_clk_groups[] = { "gpio6" }; +static const char * const i2s1_ws_groups[] = { "gpio7" }; +static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; +static const char * const wsa_swr_clk_groups[] = { "gpio10" }; +static const char * const wsa_swr_data_groups[] = { "gpio11" }; +static const char * const wsa2_swr_clk_groups[] = { "gpio15" }; +static const char * const wsa2_swr_data_groups[] = { "gpio16" }; +static const char * const i2s2_data_groups[] = { "gpio15", "gpio16" }; +static const char * const i2s3_clk_groups[] = { "gpio12"}; +static const char * const i2s3_ws_groups[] = { "gpio13"}; +static const char * const i2s3_data_groups[] = { "gpio17", "gpio18"}; +static const char * const ext_mclk1_c_groups[] = { "gpio5" }; +static const char * const ext_mclk1_b_groups[] = { "gpio9" }; +static const char * const ext_mclk1_a_groups[] = { "gpio13" }; + +static const struct lpi_pingroup sc8280xp_groups[] = { + LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), + LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _), + LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _), + LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _), + LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _), + LPI_PINGROUP(5, 12, swr_rx_data, ext_mclk1_c, qua_mi2s_data, _), + LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _), + LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _), + LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _), + LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _), + LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _), + LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _), + LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s3_clk, _, _), + LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s3_ws, ext_mclk1_a, _), + LPI_PINGROUP(14, 6, swr_tx_data, _, _, _), + LPI_PINGROUP(15, 20, i2s2_data, wsa2_swr_clk, _, _), + LPI_PINGROUP(16, 22, i2s2_data, wsa2_swr_data, _, _), + LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s3_data, _, _), + LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s3_data, _, _), +}; + +static const struct lpi_function sc8280xp_functions[] = { + LPI_FUNCTION(dmic1_clk), + LPI_FUNCTION(dmic1_data), + LPI_FUNCTION(dmic2_clk), + LPI_FUNCTION(dmic2_data), + LPI_FUNCTION(dmic3_clk), + LPI_FUNCTION(dmic3_data), + LPI_FUNCTION(dmic4_clk), + LPI_FUNCTION(dmic4_data), + LPI_FUNCTION(i2s1_clk), + LPI_FUNCTION(i2s1_data), + LPI_FUNCTION(i2s1_ws), + LPI_FUNCTION(i2s2_clk), + LPI_FUNCTION(i2s2_data), + LPI_FUNCTION(i2s2_ws), + LPI_FUNCTION(i2s3_clk), + LPI_FUNCTION(i2s3_data), + LPI_FUNCTION(i2s3_ws), + LPI_FUNCTION(qua_mi2s_data), + LPI_FUNCTION(qua_mi2s_sclk), + LPI_FUNCTION(qua_mi2s_ws), + LPI_FUNCTION(swr_rx_clk), + LPI_FUNCTION(swr_rx_data), + LPI_FUNCTION(swr_tx_clk), + LPI_FUNCTION(swr_tx_data), + LPI_FUNCTION(wsa_swr_clk), + LPI_FUNCTION(wsa_swr_data), + LPI_FUNCTION(wsa2_swr_clk), + LPI_FUNCTION(wsa2_swr_data), + LPI_FUNCTION(ext_mclk1_a), + LPI_FUNCTION(ext_mclk1_b), + LPI_FUNCTION(ext_mclk1_c), +}; + +static const struct lpi_pinctrl_variant_data sc8280xp_lpi_data = { + .pins = sc8280xp_lpi_pins, + .npins = ARRAY_SIZE(sc8280xp_lpi_pins), + .groups = sc8280xp_groups, + .ngroups = ARRAY_SIZE(sc8280xp_groups), + .functions = sc8280xp_functions, + .nfunctions = ARRAY_SIZE(sc8280xp_functions), +}; + +static const struct of_device_id lpi_pinctrl_of_match[] = { + { + .compatible = "qcom,sc8280xp-lpass-lpi-pinctrl", + .data = &sc8280xp_lpi_data, + }, + { } +}; +MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); + +static struct platform_driver lpi_pinctrl_driver = { + .driver = { + .name = "qcom-sc8280xp-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + }, + .probe = lpi_pinctrl_probe, + .remove = lpi_pinctrl_remove, +}; + +module_platform_driver(lpi_pinctrl_driver); +MODULE_DESCRIPTION("QTI SC8280XP LPI GPIO pin control driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c new file mode 100644 index 0000000000000000000000000000000000000000..c3c8c34148f11c28956d5a70d77742518d944e1e --- /dev/null +++ b/drivers/pinctrl/qcom/pinctrl-sm8450-lpass-lpi.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2022 Linaro Ltd. + */ + +#include +#include +#include + +#include "pinctrl-lpass-lpi.h" + +enum lpass_lpi_functions { + LPI_MUX_dmic1_clk, + LPI_MUX_dmic1_data, + LPI_MUX_dmic2_clk, + LPI_MUX_dmic2_data, + LPI_MUX_dmic3_clk, + LPI_MUX_dmic3_data, + LPI_MUX_dmic4_clk, + LPI_MUX_dmic4_data, + LPI_MUX_i2s1_clk, + LPI_MUX_i2s1_data, + LPI_MUX_i2s1_ws, + LPI_MUX_i2s2_clk, + LPI_MUX_i2s2_data, + LPI_MUX_i2s2_ws, + LPI_MUX_i2s3_clk, + LPI_MUX_i2s3_data, + LPI_MUX_i2s3_ws, + LPI_MUX_i2s4_clk, + LPI_MUX_i2s4_data, + LPI_MUX_i2s4_ws, + LPI_MUX_qua_mi2s_data, + LPI_MUX_qua_mi2s_sclk, + LPI_MUX_qua_mi2s_ws, + LPI_MUX_swr_rx_clk, + LPI_MUX_swr_rx_data, + LPI_MUX_swr_tx_clk, + LPI_MUX_swr_tx_data, + LPI_MUX_wsa_swr_clk, + LPI_MUX_wsa_swr_data, + LPI_MUX_wsa2_swr_clk, + LPI_MUX_wsa2_swr_data, + LPI_MUX_slimbus_clk, + LPI_MUX_slimbus_data, + LPI_MUX_ext_mclk1_a, + LPI_MUX_ext_mclk1_b, + LPI_MUX_ext_mclk1_c, + LPI_MUX_ext_mclk1_d, + LPI_MUX_ext_mclk1_e, + LPI_MUX_gpio, + LPI_MUX__, +}; + +static int gpio0_pins[] = { 0 }; +static int gpio1_pins[] = { 1 }; +static int gpio2_pins[] = { 2 }; +static int gpio3_pins[] = { 3 }; +static int gpio4_pins[] = { 4 }; +static int gpio5_pins[] = { 5 }; +static int gpio6_pins[] = { 6 }; +static int gpio7_pins[] = { 7 }; +static int gpio8_pins[] = { 8 }; +static int gpio9_pins[] = { 9 }; +static int gpio10_pins[] = { 10 }; +static int gpio11_pins[] = { 11 }; +static int gpio12_pins[] = { 12 }; +static int gpio13_pins[] = { 13 }; +static int gpio14_pins[] = { 14 }; +static int gpio15_pins[] = { 15 }; +static int gpio16_pins[] = { 16 }; +static int gpio17_pins[] = { 17 }; +static int gpio18_pins[] = { 18 }; +static int gpio19_pins[] = { 19 }; +static int gpio20_pins[] = { 20 }; +static int gpio21_pins[] = { 21 }; +static int gpio22_pins[] = { 22 }; + +static const struct pinctrl_pin_desc sm8450_lpi_pins[] = { + PINCTRL_PIN(0, "gpio0"), + PINCTRL_PIN(1, "gpio1"), + PINCTRL_PIN(2, "gpio2"), + PINCTRL_PIN(3, "gpio3"), + PINCTRL_PIN(4, "gpio4"), + PINCTRL_PIN(5, "gpio5"), + PINCTRL_PIN(6, "gpio6"), + PINCTRL_PIN(7, "gpio7"), + PINCTRL_PIN(8, "gpio8"), + PINCTRL_PIN(9, "gpio9"), + PINCTRL_PIN(10, "gpio10"), + PINCTRL_PIN(11, "gpio11"), + PINCTRL_PIN(12, "gpio12"), + PINCTRL_PIN(13, "gpio13"), + PINCTRL_PIN(14, "gpio14"), + PINCTRL_PIN(15, "gpio15"), + PINCTRL_PIN(16, "gpio16"), + PINCTRL_PIN(17, "gpio17"), + PINCTRL_PIN(18, "gpio18"), + PINCTRL_PIN(19, "gpio19"), + PINCTRL_PIN(20, "gpio20"), + PINCTRL_PIN(21, "gpio21"), + PINCTRL_PIN(22, "gpio22"), +}; + +static const char * const swr_tx_clk_groups[] = { "gpio0" }; +static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" }; +static const char * const swr_rx_clk_groups[] = { "gpio3" }; +static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5", "gpio15" }; +static const char * const dmic1_clk_groups[] = { "gpio6" }; +static const char * const dmic1_data_groups[] = { "gpio7" }; +static const char * const dmic2_clk_groups[] = { "gpio8" }; +static const char * const dmic2_data_groups[] = { "gpio9" }; +static const char * const dmic4_clk_groups[] = { "gpio17" }; +static const char * const dmic4_data_groups[] = { "gpio18" }; +static const char * const i2s2_clk_groups[] = { "gpio10" }; +static const char * const i2s2_ws_groups[] = { "gpio11" }; +static const char * const dmic3_clk_groups[] = { "gpio12" }; +static const char * const dmic3_data_groups[] = { "gpio13" }; +static const char * const qua_mi2s_sclk_groups[] = { "gpio0" }; +static const char * const qua_mi2s_ws_groups[] = { "gpio1" }; +static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" }; +static const char * const i2s1_clk_groups[] = { "gpio6" }; +static const char * const i2s1_ws_groups[] = { "gpio7" }; +static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" }; +static const char * const wsa_swr_clk_groups[] = { "gpio10" }; +static const char * const wsa_swr_data_groups[] = { "gpio11" }; +static const char * const wsa2_swr_clk_groups[] = { "gpio15" }; +static const char * const wsa2_swr_data_groups[] = { "gpio16" }; +static const char * const i2s2_data_groups[] = { "gpio15", "gpio16" }; +static const char * const i2s4_ws_groups[] = { "gpio13" }; +static const char * const i2s4_clk_groups[] = { "gpio12" }; +static const char * const i2s4_data_groups[] = { "gpio17", "gpio18" }; +static const char * const slimbus_clk_groups[] = { "gpio19"}; +static const char * const i2s3_clk_groups[] = { "gpio19"}; +static const char * const i2s3_ws_groups[] = { "gpio20"}; +static const char * const i2s3_data_groups[] = { "gpio21", "gpio22"}; +static const char * const slimbus_data_groups[] = { "gpio20"}; +static const char * const ext_mclk1_c_groups[] = { "gpio5" }; +static const char * const ext_mclk1_b_groups[] = { "gpio9" }; +static const char * const ext_mclk1_a_groups[] = { "gpio13" }; +static const char * const ext_mclk1_d_groups[] = { "gpio14" }; +static const char * const ext_mclk1_e_groups[] = { "gpio22" }; + +static const struct lpi_pingroup sm8450_groups[] = { + LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _), + LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _), + LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _), + LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _), + LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _), + LPI_PINGROUP(5, 12, swr_rx_data, ext_mclk1_c, qua_mi2s_data, _), + LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _, _), + LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _), + LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _), + LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, ext_mclk1_b, _), + LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _), + LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _), + LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s4_clk, _, _), + LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s4_ws, ext_mclk1_a, _), + LPI_PINGROUP(14, 6, swr_tx_data, ext_mclk1_d, _, _), + LPI_PINGROUP(15, 20, i2s2_data, wsa2_swr_clk, _, _), + LPI_PINGROUP(16, 22, i2s2_data, wsa2_swr_data, _, _), + LPI_PINGROUP(17, LPI_NO_SLEW, dmic4_clk, i2s4_data, _, _), + LPI_PINGROUP(18, LPI_NO_SLEW, dmic4_data, i2s4_data, _, _), + LPI_PINGROUP(19, LPI_NO_SLEW, i2s3_clk, slimbus_clk, _, _), + LPI_PINGROUP(20, LPI_NO_SLEW, i2s3_ws, slimbus_data, _, _), + LPI_PINGROUP(21, LPI_NO_SLEW, i2s3_data, _, _, _), + LPI_PINGROUP(22, LPI_NO_SLEW, i2s3_data, ext_mclk1_e, _, _), +}; + +static const struct lpi_function sm8450_functions[] = { + LPI_FUNCTION(dmic1_clk), + LPI_FUNCTION(dmic1_data), + LPI_FUNCTION(dmic2_clk), + LPI_FUNCTION(dmic2_data), + LPI_FUNCTION(dmic3_clk), + LPI_FUNCTION(dmic3_data), + LPI_FUNCTION(dmic4_clk), + LPI_FUNCTION(dmic4_data), + LPI_FUNCTION(i2s1_clk), + LPI_FUNCTION(i2s1_data), + LPI_FUNCTION(i2s1_ws), + LPI_FUNCTION(i2s2_clk), + LPI_FUNCTION(i2s2_data), + LPI_FUNCTION(i2s2_ws), + LPI_FUNCTION(i2s3_clk), + LPI_FUNCTION(i2s3_data), + LPI_FUNCTION(i2s3_ws), + LPI_FUNCTION(i2s4_clk), + LPI_FUNCTION(i2s4_data), + LPI_FUNCTION(i2s4_ws), + LPI_FUNCTION(qua_mi2s_data), + LPI_FUNCTION(qua_mi2s_sclk), + LPI_FUNCTION(qua_mi2s_ws), + LPI_FUNCTION(swr_rx_clk), + LPI_FUNCTION(swr_rx_data), + LPI_FUNCTION(swr_tx_clk), + LPI_FUNCTION(swr_tx_data), + LPI_FUNCTION(slimbus_clk), + LPI_FUNCTION(slimbus_data), + LPI_FUNCTION(wsa_swr_clk), + LPI_FUNCTION(wsa_swr_data), + LPI_FUNCTION(wsa2_swr_clk), + LPI_FUNCTION(wsa2_swr_data), + LPI_FUNCTION(ext_mclk1_a), + LPI_FUNCTION(ext_mclk1_b), + LPI_FUNCTION(ext_mclk1_c), + LPI_FUNCTION(ext_mclk1_d), + LPI_FUNCTION(ext_mclk1_e), +}; + +static const struct lpi_pinctrl_variant_data sm8450_lpi_data = { + .pins = sm8450_lpi_pins, + .npins = ARRAY_SIZE(sm8450_lpi_pins), + .groups = sm8450_groups, + .ngroups = ARRAY_SIZE(sm8450_groups), + .functions = sm8450_functions, + .nfunctions = ARRAY_SIZE(sm8450_functions), +}; + +static const struct of_device_id lpi_pinctrl_of_match[] = { + { + .compatible = "qcom,sm8450-lpass-lpi-pinctrl", + .data = &sm8450_lpi_data, + }, + { } +}; +MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match); + +static struct platform_driver lpi_pinctrl_driver = { + .driver = { + .name = "qcom-sm8450-lpass-lpi-pinctrl", + .of_match_table = lpi_pinctrl_of_match, + }, + .probe = lpi_pinctrl_probe, + .remove = lpi_pinctrl_remove, +}; + +module_platform_driver(lpi_pinctrl_driver); +MODULE_DESCRIPTION("QTI SM8450 LPI GPIO pin control driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c index ccaf40a9c0e6bf237a5e473d343f5b0b780da4f7..8c31a8f6b7e4ef888c69f747d11f73dfd810b90d 100644 --- a/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c +++ b/drivers/pinctrl/qcom/pinctrl-spmi-gpio.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2012-2014, 2016-2021 The Linux Foundation. All rights reserved. + * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved. */ #include @@ -36,6 +37,8 @@ #define PMIC_GPIO_SUBTYPE_GPIOC_8CH 0xd #define PMIC_GPIO_SUBTYPE_GPIO_LV 0x10 #define PMIC_GPIO_SUBTYPE_GPIO_MV 0x11 +#define PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2 0x12 +#define PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3 0x13 #define PMIC_MPP_REG_RT_STS 0x10 #define PMIC_MPP_REG_RT_STS_VAL_MASK 0x1 @@ -98,6 +101,9 @@ #define PMIC_GPIO_OUT_BUF_OPEN_DRAIN_NMOS 1 #define PMIC_GPIO_OUT_BUF_OPEN_DRAIN_PMOS 2 +#define PMIC_GPIO_OUT_STRENGTH_LOW 1 +#define PMIC_GPIO_OUT_STRENGTH_HIGH 3 + /* PMIC_GPIO_REG_EN_CTL */ #define PMIC_GPIO_REG_MASTER_EN_SHIFT 7 @@ -171,7 +177,6 @@ struct pmic_gpio_state { struct regmap *map; struct pinctrl_dev *ctrl; struct gpio_chip chip; - struct irq_chip irq; u8 usid; u8 pid_base; }; @@ -437,7 +442,17 @@ static int pmic_gpio_config_get(struct pinctrl_dev *pctldev, arg = pad->pullup; break; case PMIC_GPIO_CONF_STRENGTH: - arg = pad->strength; + switch (pad->strength) { + case PMIC_GPIO_OUT_STRENGTH_HIGH: + arg = PMIC_GPIO_STRENGTH_HIGH; + break; + case PMIC_GPIO_OUT_STRENGTH_LOW: + arg = PMIC_GPIO_STRENGTH_LOW; + break; + default: + arg = pad->strength; + break; + } break; case PMIC_GPIO_CONF_ATEST: arg = pad->atest; @@ -524,7 +539,17 @@ static int pmic_gpio_config_set(struct pinctrl_dev *pctldev, unsigned int pin, case PMIC_GPIO_CONF_STRENGTH: if (arg > PMIC_GPIO_STRENGTH_LOW) return -EINVAL; - pad->strength = arg; + switch (arg) { + case PMIC_GPIO_STRENGTH_HIGH: + pad->strength = PMIC_GPIO_OUT_STRENGTH_HIGH; + break; + case PMIC_GPIO_STRENGTH_LOW: + pad->strength = PMIC_GPIO_OUT_STRENGTH_LOW; + break; + default: + pad->strength = arg; + break; + } break; case PMIC_GPIO_CONF_ATEST: if (!pad->lv_mv_type || arg > 4) @@ -823,6 +848,16 @@ static int pmic_gpio_populate(struct pmic_gpio_state *state, pad->have_buffer = true; pad->lv_mv_type = true; break; + case PMIC_GPIO_SUBTYPE_GPIO_LV_VIN2: + pad->num_sources = 2; + pad->have_buffer = true; + pad->lv_mv_type = true; + break; + case PMIC_GPIO_SUBTYPE_GPIO_MV_VIN3: + pad->num_sources = 3; + pad->have_buffer = true; + pad->lv_mv_type = true; + break; default: dev_err(state->dev, "unknown GPIO type 0x%x\n", subtype); return -ENODEV; @@ -985,6 +1020,33 @@ static int pmic_gpio_populate_parent_fwspec(struct gpio_chip *chip, return 0; } +static void pmic_gpio_irq_mask(struct irq_data *data) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + + irq_chip_mask_parent(data); + gpiochip_disable_irq(gc, data->hwirq); +} + +static void pmic_gpio_irq_unmask(struct irq_data *data) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + + gpiochip_enable_irq(gc, data->hwirq); + irq_chip_unmask_parent(data); +} + +static const struct irq_chip spmi_gpio_irq_chip = { + .name = "spmi-gpio", + .irq_ack = irq_chip_ack_parent, + .irq_mask = pmic_gpio_irq_mask, + .irq_unmask = pmic_gpio_irq_unmask, + .irq_set_type = irq_chip_set_type_parent, + .irq_set_wake = irq_chip_set_wake_parent, + .flags = IRQCHIP_IMMUTABLE | IRQCHIP_MASK_ON_SUSPEND, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int pmic_gpio_probe(struct platform_device *pdev) { struct irq_domain *parent_domain; @@ -1078,16 +1140,8 @@ static int pmic_gpio_probe(struct platform_device *pdev) if (!parent_domain) return -ENXIO; - state->irq.name = "spmi-gpio", - state->irq.irq_ack = irq_chip_ack_parent, - state->irq.irq_mask = irq_chip_mask_parent, - state->irq.irq_unmask = irq_chip_unmask_parent, - state->irq.irq_set_type = irq_chip_set_type_parent, - state->irq.irq_set_wake = irq_chip_set_wake_parent, - state->irq.flags = IRQCHIP_MASK_ON_SUSPEND, - girq = &state->chip.irq; - girq->chip = &state->irq; + gpio_irq_chip_set_chip(girq, &spmi_gpio_irq_chip); girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_level_irq; girq->fwnode = of_node_to_fwnode(state->dev->of_node); @@ -1147,6 +1201,7 @@ static const struct of_device_id pmic_gpio_of_match[] = { { .compatible = "qcom,pm6150-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm6150l-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm6350-gpio", .data = (void *) 9 }, + { .compatible = "qcom,pm7250b-gpio", .data = (void *) 12 }, { .compatible = "qcom,pm7325-gpio", .data = (void *) 10 }, { .compatible = "qcom,pm8005-gpio", .data = (void *) 4 }, { .compatible = "qcom,pm8008-gpio", .data = (void *) 2 }, diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index 4837bceb767b458edf3db99d5066afee22b8134f..bd13b5ef246d8cd339c7762e32ea96a7e7ed853e 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -1166,15 +1166,15 @@ static int samsung_pinctrl_probe(struct platform_device *pdev) if (ret) goto err_put_banks; - ret = samsung_gpiolib_register(pdev, drvdata); - if (ret) - goto err_unregister; - if (ctrl->eint_gpio_init) ctrl->eint_gpio_init(drvdata); if (ctrl->eint_wkup_init) ctrl->eint_wkup_init(drvdata); + ret = samsung_gpiolib_register(pdev, drvdata); + if (ret) + goto err_unregister; + platform_set_drvdata(pdev, drvdata); return 0; diff --git a/drivers/pinctrl/starfive/Kconfig b/drivers/pinctrl/starfive/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..55c514e622f91fd5001d3e27ec9e8502e102aaed --- /dev/null +++ b/drivers/pinctrl/starfive/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config PINCTRL_STARFIVE_JH7100 + tristate "Pinctrl and GPIO driver for the StarFive JH7100 SoC" + depends on SOC_STARFIVE || COMPILE_TEST + depends on OF + select GENERIC_PINCTRL_GROUPS + select GENERIC_PINMUX_FUNCTIONS + select GENERIC_PINCONF + select GPIOLIB + select GPIOLIB_IRQCHIP + select OF_GPIO + default SOC_STARFIVE + help + Say yes here to support pin control on the StarFive JH7100 SoC. + This also provides an interface to the GPIO pins not used by other + peripherals supporting inputs, outputs, configuring pull-up/pull-down + and interrupts on input changes. diff --git a/drivers/pinctrl/starfive/Makefile b/drivers/pinctrl/starfive/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0293f26a0a993bf5c275866e1b6c402920dcb0ef --- /dev/null +++ b/drivers/pinctrl/starfive/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_PINCTRL_STARFIVE_JH7100) += pinctrl-starfive-jh7100.o diff --git a/drivers/pinctrl/pinctrl-starfive.c b/drivers/pinctrl/starfive/pinctrl-starfive-jh7100.c similarity index 99% rename from drivers/pinctrl/pinctrl-starfive.c rename to drivers/pinctrl/starfive/pinctrl-starfive-jh7100.c index 3eb40e230d981238ae03952b449ebd5b40266f4c..5b544fb7f3d88240dc9decc023341e0228afd033 100644 --- a/drivers/pinctrl/pinctrl-starfive.c +++ b/drivers/pinctrl/starfive/pinctrl-starfive-jh7100.c @@ -20,12 +20,12 @@ #include #include -#include +#include -#include "core.h" -#include "pinctrl-utils.h" -#include "pinmux.h" -#include "pinconf.h" +#include "../core.h" +#include "../pinctrl-utils.h" +#include "../pinmux.h" +#include "../pinconf.h" #define DRIVER_NAME "pinctrl-starfive" diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index 14bcca73238aee98f42606bc84042e34aab8ea1e..e485506ea599c8e532a355c6c64964c33170e245 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -1603,10 +1603,9 @@ int stm32_pctl_probe(struct platform_device *pdev) bank->clk = of_clk_get_by_name(np, NULL); if (IS_ERR(bank->clk)) { - if (PTR_ERR(bank->clk) != -EPROBE_DEFER) - dev_err(dev, "failed to get clk (%ld)\n", PTR_ERR(bank->clk)); fwnode_handle_put(child); - return PTR_ERR(bank->clk); + return dev_err_probe(dev, PTR_ERR(bank->clk), + "failed to get clk\n"); } i++; } diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-a100-r.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-a100-r.c index afc1f5df75450bb5810f482966932bb4b0702aa1..b82ad135bf2aa6aeaaf36724ec586f179cfdb2ae 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun50i-a100-r.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-a100-r.c @@ -99,7 +99,7 @@ MODULE_DEVICE_TABLE(of, a100_r_pinctrl_match); static struct platform_driver a100_r_pinctrl_driver = { .probe = a100_r_pinctrl_probe, .driver = { - .name = "sun50iw10p1-r-pinctrl", + .name = "sun50i-a100-r-pinctrl", .of_match_table = a100_r_pinctrl_match, }, }; diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c index 31d62bbb7f43f001fe93409ae63fb1fb671a15ac..96a350e70668ac690dd25233f056b5fbf095b6db 100644 --- a/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c +++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-h5.c @@ -551,12 +551,9 @@ static int sun50i_h5_pinctrl_probe(struct platform_device *pdev) int ret; ret = platform_irq_count(pdev); - if (ret < 0) { - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Couldn't determine irq count: %pe\n", - ERR_PTR(ret)); - return ret; - } + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Couldn't determine irq count\n"); switch (ret) { case 2: diff --git a/drivers/platform/Kconfig b/drivers/platform/Kconfig index b437847b62377bdd68051638c4e0807be9be1ee2..dbd32771220524db46216c995e3dee364d60637f 100644 --- a/drivers/platform/Kconfig +++ b/drivers/platform/Kconfig @@ -3,6 +3,8 @@ if MIPS source "drivers/platform/mips/Kconfig" endif +source "drivers/platform/loongarch/Kconfig" + source "drivers/platform/goldfish/Kconfig" source "drivers/platform/chrome/Kconfig" diff --git a/drivers/platform/Makefile b/drivers/platform/Makefile index 4de08ef4ec9d087356dff9b1d5d4f701ec510ccf..41640172975a795df917a824d0608e356588c925 100644 --- a/drivers/platform/Makefile +++ b/drivers/platform/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_X86) += x86/ +obj-$(CONFIG_LOONGARCH) += loongarch/ obj-$(CONFIG_MELLANOX_PLATFORM) += mellanox/ obj-$(CONFIG_MIPS) += mips/ obj-$(CONFIG_OLPC_EC) += olpc/ diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index c45fb376d653d9bbc1adddd2d89dbe39949b45a4..6b954c5acadb1ea180465cb09397f40ee19104a1 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -265,6 +265,17 @@ config CHROMEOS_PRIVACY_SCREEN this should probably always be built into the kernel to avoid or minimize drm probe deferral. +config CROS_TYPEC_SWITCH + tristate "ChromeOS EC Type-C Switch Control" + depends on MFD_CROS_EC_DEV && TYPEC && ACPI + default MFD_CROS_EC_DEV + help + If you say Y here, you get support for configuring the ChromeOS EC Type-C + muxes and retimers. + + To compile this driver as a module, choose M here: the module will be + called cros_typec_switch. + source "drivers/platform/chrome/wilco_ec/Kconfig" # Kunit test cases diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index f7e74a845afc22db90a1825d0a96b0181d346f4f..2950610101f15f4a6f9b36bad8482c6ce2b90854 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_CHROMEOS_TBMC) += chromeos_tbmc.o obj-$(CONFIG_CROS_EC) += cros_ec.o obj-$(CONFIG_CROS_EC_I2C) += cros_ec_i2c.o obj-$(CONFIG_CROS_EC_ISHTP) += cros_ec_ishtp.o +obj-$(CONFIG_CROS_TYPEC_SWITCH) += cros_typec_switch.o obj-$(CONFIG_CROS_EC_RPMSG) += cros_ec_rpmsg.o obj-$(CONFIG_CROS_EC_SPI) += cros_ec_spi.o cros_ec_lpcs-objs := cros_ec_lpc.o cros_ec_lpc_mec.o diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c index 4e14b4d6635d765539d186b11738356471c60fa6..a2cdbfbaeae6b0ba7d2961308a0e3f76fd3a58d4 100644 --- a/drivers/platform/chrome/chromeos_laptop.c +++ b/drivers/platform/chrome/chromeos_laptop.c @@ -740,6 +740,7 @@ static int __init chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop, const struct chromeos_laptop *src) { + struct i2c_peripheral *i2c_peripherals; struct i2c_peripheral *i2c_dev; struct i2c_board_info *info; int i; @@ -748,17 +749,15 @@ chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop, if (!src->num_i2c_peripherals) return 0; - cros_laptop->i2c_peripherals = kmemdup(src->i2c_peripherals, - src->num_i2c_peripherals * - sizeof(*src->i2c_peripherals), - GFP_KERNEL); - if (!cros_laptop->i2c_peripherals) + i2c_peripherals = kmemdup(src->i2c_peripherals, + src->num_i2c_peripherals * + sizeof(*src->i2c_peripherals), + GFP_KERNEL); + if (!i2c_peripherals) return -ENOMEM; - cros_laptop->num_i2c_peripherals = src->num_i2c_peripherals; - - for (i = 0; i < cros_laptop->num_i2c_peripherals; i++) { - i2c_dev = &cros_laptop->i2c_peripherals[i]; + for (i = 0; i < src->num_i2c_peripherals; i++) { + i2c_dev = &i2c_peripherals[i]; info = &i2c_dev->board_info; error = chromeos_laptop_setup_irq(i2c_dev); @@ -775,16 +774,19 @@ chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop, } } + cros_laptop->i2c_peripherals = i2c_peripherals; + cros_laptop->num_i2c_peripherals = src->num_i2c_peripherals; + return 0; err_out: while (--i >= 0) { - i2c_dev = &cros_laptop->i2c_peripherals[i]; + i2c_dev = &i2c_peripherals[i]; info = &i2c_dev->board_info; if (!IS_ERR_OR_NULL(info->fwnode)) fwnode_remove_software_node(info->fwnode); } - kfree(cros_laptop->i2c_peripherals); + kfree(i2c_peripherals); return error; } diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 8aace50d446d65ac8d7cb5b89ee87efc3880bc7d..ec733f683f34f645838ff9a0b684b532fac71a6a 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -115,7 +115,7 @@ static int cros_ec_sleep_event(struct cros_ec_device *ec_dev, u8 sleep_event) if (ec_dev->host_sleep_v1) { buf.u.req1.sleep_event = sleep_event; buf.u.req1.suspend_params.sleep_timeout_ms = - EC_HOST_SLEEP_TIMEOUT_DEFAULT; + ec_dev->suspend_timeout_ms; buf.msg.outsize = sizeof(buf.u.req1); if ((sleep_event == HOST_SLEEP_EVENT_S3_RESUME) || @@ -188,6 +188,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev) ec_dev->max_passthru = 0; ec_dev->ec = NULL; ec_dev->pd = NULL; + ec_dev->suspend_timeout_ms = EC_HOST_SLEEP_TIMEOUT_DEFAULT; ec_dev->din = devm_kzalloc(dev, ec_dev->din_size, GFP_KERNEL); if (!ec_dev->din) @@ -349,10 +350,16 @@ EXPORT_SYMBOL(cros_ec_suspend); static void cros_ec_report_events_during_suspend(struct cros_ec_device *ec_dev) { + bool wake_event; + while (ec_dev->mkbp_event_supported && - cros_ec_get_next_event(ec_dev, NULL, NULL) > 0) + cros_ec_get_next_event(ec_dev, &wake_event, NULL) > 0) { blocking_notifier_call_chain(&ec_dev->event_notifier, 1, ec_dev); + + if (wake_event && device_may_wakeup(ec_dev->dev)) + pm_wakeup_event(ec_dev->dev, 0); + } } /** diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index fd33de546aee0bb5ccea837e6fa2214cffa90014..0de7c255254e0ba7f4e33c4da8a7b7573e815bb5 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -327,6 +327,9 @@ static long cros_ec_chardev_ioctl_readmem(struct cros_ec_dev *ec, if (copy_from_user(&s_mem, arg, sizeof(s_mem))) return -EFAULT; + if (s_mem.bytes > sizeof(s_mem.buffer)) + return -EINVAL; + num = ec_dev->cmd_readmem(ec_dev, s_mem.offset, s_mem.bytes, s_mem.buffer); if (num <= 0) diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index 0dbceee87a4b1a44d79e7cc9cd186972d9a19723..4e63adf083ea1f8fca9187fe4646281d14088143 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -470,6 +470,9 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) debugfs_create_x32("last_resume_result", 0444, debug_info->dir, &ec->ec_dev->last_resume_result); + debugfs_create_u16("suspend_timeout_ms", 0664, debug_info->dir, + &ec->ec_dev->suspend_timeout_ms); + ec->debug_info = debug_info; dev_set_drvdata(&pd->dev, ec); diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c index 9f5b95763173cca7b5926823c1c7f4745300db2e..b6823c654c3f975c2dc235e5672967d88c769344 100644 --- a/drivers/platform/chrome/cros_ec_i2c.c +++ b/drivers/platform/chrome/cros_ec_i2c.c @@ -317,13 +317,11 @@ static int cros_ec_i2c_probe(struct i2c_client *client, return 0; } -static int cros_ec_i2c_remove(struct i2c_client *client) +static void cros_ec_i2c_remove(struct i2c_client *client) { struct cros_ec_device *ec_dev = i2c_get_clientdata(client); cros_ec_unregister(ec_dev); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 05d2e8765a66bac7ec2436261da921db8a72d184..475a6dd72db6bb2fdad1a4ce1d6cee3231acef86 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -773,6 +773,7 @@ int cros_ec_get_next_event(struct cros_ec_device *ec_dev, u8 event_type; u32 host_event; int ret; + u32 ver_mask; /* * Default value for wake_event. @@ -794,6 +795,37 @@ int cros_ec_get_next_event(struct cros_ec_device *ec_dev, return get_keyboard_state_event(ec_dev); ret = get_next_event(ec_dev); + /* + * -ENOPROTOOPT is returned when EC returns EC_RES_INVALID_VERSION. + * This can occur when EC based device (e.g. Fingerprint MCU) jumps to + * the RO image which doesn't support newer version of the command. In + * this case we will attempt to update maximum supported version of the + * EC_CMD_GET_NEXT_EVENT. + */ + if (ret == -ENOPROTOOPT) { + dev_dbg(ec_dev->dev, + "GET_NEXT_EVENT returned invalid version error.\n"); + ret = cros_ec_get_host_command_version_mask(ec_dev, + EC_CMD_GET_NEXT_EVENT, + &ver_mask); + if (ret < 0 || ver_mask == 0) + /* + * Do not change the MKBP supported version if we can't + * obtain supported version correctly. Please note that + * calling EC_CMD_GET_NEXT_EVENT returned + * EC_RES_INVALID_VERSION which means that the command + * is present. + */ + return -ENOPROTOOPT; + + ec_dev->mkbp_event_supported = fls(ver_mask); + dev_dbg(ec_dev->dev, "MKBP support version changed to %u\n", + ec_dev->mkbp_event_supported - 1); + + /* Try to get next event with new MKBP support version set. */ + ret = get_next_event(ec_dev); + } + if (ret <= 0) return ret; diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index de6ee0f926a660c45e4188830de087315f814100..2a7ff14dc37e9cb4b834d8888eebcad96ed08645 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -20,12 +20,14 @@ #include #include #include +#include #include #include #define DRV_NAME "cros-ec-typec" -#define DP_PORT_VDO (BIT(DP_PIN_ASSIGN_C) | BIT(DP_PIN_ASSIGN_D) | DP_CAP_DFP_D) +#define DP_PORT_VDO (DP_CONF_SET_PIN_ASSIGN(BIT(DP_PIN_ASSIGN_C) | BIT(DP_PIN_ASSIGN_D)) | \ + DP_CAP_DFP_D) /* Supported alt modes. */ enum { @@ -55,6 +57,7 @@ struct cros_typec_port { struct usb_pd_identity c_identity; struct typec_switch *ori_sw; struct typec_mux *mux; + struct typec_retimer *retimer; struct usb_role_switch *role_sw; /* Variables keeping track of switch state. */ @@ -70,6 +73,11 @@ struct cros_typec_port { struct ec_response_typec_discovery *disc_data; struct list_head partner_mode_list; struct list_head plug_mode_list; + + /* PDO-related structs */ + struct usb_power_delivery *partner_pd; + struct usb_power_delivery_capabilities *partner_src_caps; + struct usb_power_delivery_capabilities *partner_sink_caps; }; /* Platform-specific data for the Chrome OS EC Type C controller. */ @@ -143,6 +151,12 @@ static int cros_typec_get_switch_handles(struct cros_typec_port *port, goto mux_err; } + port->retimer = fwnode_typec_retimer_get(fwnode); + if (IS_ERR(port->retimer)) { + dev_dbg(dev, "Retimer handle not found.\n"); + goto retimer_sw_err; + } + port->ori_sw = fwnode_typec_switch_get(fwnode); if (IS_ERR(port->ori_sw)) { dev_dbg(dev, "Orientation switch handle not found.\n"); @@ -158,12 +172,12 @@ static int cros_typec_get_switch_handles(struct cros_typec_port *port, return 0; role_sw_err: - usb_role_switch_put(port->role_sw); -ori_sw_err: typec_switch_put(port->ori_sw); -mux_err: +ori_sw_err: + typec_retimer_put(port->retimer); +retimer_sw_err: typec_mux_put(port->mux); - +mux_err: return -ENODEV; } @@ -206,6 +220,21 @@ static void cros_typec_unregister_altmodes(struct cros_typec_data *typec, int po } } +/* + * Map the Type-C Mux state to retimer state and call the retimer set function. We need this + * because we re-use the Type-C mux state for retimers. + */ +static int cros_typec_retimer_set(struct typec_retimer *retimer, struct typec_mux_state state) +{ + struct typec_retimer_state rstate = { + .alt = state.alt, + .mode = state.mode, + .data = state.data, + }; + + return typec_retimer_set(retimer, &rstate); +} + static int cros_typec_usb_disconnect_state(struct cros_typec_port *port) { port->state.alt = NULL; @@ -214,6 +243,7 @@ static int cros_typec_usb_disconnect_state(struct cros_typec_port *port) usb_role_switch_set_role(port->role_sw, USB_ROLE_NONE); typec_switch_set(port->ori_sw, TYPEC_ORIENTATION_NONE); + cros_typec_retimer_set(port->retimer, port->state); return typec_mux_set(port->mux, &port->state); } @@ -228,6 +258,14 @@ static void cros_typec_remove_partner(struct cros_typec_data *typec, cros_typec_unregister_altmodes(typec, port_num, true); + typec_partner_set_usb_power_delivery(port->partner, NULL); + usb_power_delivery_unregister_capabilities(port->partner_sink_caps); + port->partner_sink_caps = NULL; + usb_power_delivery_unregister_capabilities(port->partner_src_caps); + port->partner_src_caps = NULL; + usb_power_delivery_unregister(port->partner_pd); + port->partner_pd = NULL; + cros_typec_usb_disconnect_state(port); port->mux_flags = USB_PD_MUX_NONE; @@ -411,9 +449,14 @@ unregister_ports: static int cros_typec_usb_safe_state(struct cros_typec_port *port) { + int ret; port->state.mode = TYPEC_STATE_SAFE; - return typec_mux_set(port->mux, &port->state); + ret = cros_typec_retimer_set(port->retimer, port->state); + if (!ret) + ret = typec_mux_set(port->mux, &port->state); + + return ret; } /* @@ -510,7 +553,11 @@ static int cros_typec_enable_dp(struct cros_typec_data *typec, port->state.data = &dp_data; port->state.mode = TYPEC_MODAL_STATE(ffs(pd_ctrl->dp_mode)); - return typec_mux_set(port->mux, &port->state); + ret = cros_typec_retimer_set(port->retimer, port->state); + if (!ret) + ret = typec_mux_set(port->mux, &port->state); + + return ret; } static int cros_typec_enable_usb4(struct cros_typec_data *typec, @@ -599,7 +646,10 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num, } else if (port->mux_flags & USB_PD_MUX_USB_ENABLED) { port->state.alt = NULL; port->state.mode = TYPEC_STATE_USB; - ret = typec_mux_set(port->mux, &port->state); + + ret = cros_typec_retimer_set(port->retimer, port->state); + if (!ret) + ret = typec_mux_set(port->mux, &port->state); } else { dev_dbg(typec->dev, "Unrecognized mode requested, mux flags: %x\n", @@ -697,7 +747,7 @@ static int cros_typec_register_altmodes(struct cros_typec_data *typec, int port_ for (j = 0; j < sop_disc->svids[i].mode_count; j++) { memset(&desc, 0, sizeof(desc)); desc.svid = sop_disc->svids[i].svid; - desc.mode = j; + desc.mode = j + 1; desc.vdo = sop_disc->svids[i].mode_vdo[j]; if (is_partner) @@ -902,6 +952,46 @@ static int cros_typec_send_clear_event(struct cros_typec_data *typec, int port_n sizeof(req), NULL, 0); } +static void cros_typec_register_partner_pdos(struct cros_typec_data *typec, + struct ec_response_typec_status *resp, int port_num) +{ + struct usb_power_delivery_capabilities_desc caps_desc = {}; + struct usb_power_delivery_desc desc = { + .revision = (le16_to_cpu(resp->sop_revision) & 0xff00) >> 4, + }; + struct cros_typec_port *port = typec->ports[port_num]; + + if (!port->partner || port->partner_pd) + return; + + /* If no caps are available, don't bother creating a device. */ + if (!resp->source_cap_count && !resp->sink_cap_count) + return; + + port->partner_pd = usb_power_delivery_register(NULL, &desc); + if (IS_ERR(port->partner_pd)) { + dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num); + return; + } + + typec_partner_set_usb_power_delivery(port->partner, port->partner_pd); + + memcpy(caps_desc.pdo, resp->source_cap_pdos, sizeof(u32) * resp->source_cap_count); + caps_desc.role = TYPEC_SOURCE; + port->partner_src_caps = usb_power_delivery_register_capabilities(port->partner_pd, + &caps_desc); + if (IS_ERR(port->partner_src_caps)) + dev_warn(typec->dev, "Failed to register source caps, port: %d\n", port_num); + + memset(&caps_desc, 0, sizeof(caps_desc)); + memcpy(caps_desc.pdo, resp->sink_cap_pdos, sizeof(u32) * resp->sink_cap_count); + caps_desc.role = TYPEC_SINK; + port->partner_sink_caps = usb_power_delivery_register_capabilities(port->partner_pd, + &caps_desc); + if (IS_ERR(port->partner_sink_caps)) + dev_warn(typec->dev, "Failed to register sink caps, port: %d\n", port_num); +} + static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num) { struct ec_response_typec_status resp; @@ -949,6 +1039,8 @@ static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num } if (resp.sop_connected) typec_set_pwr_opmode(typec->ports[port_num]->port, TYPEC_PWR_MODE_PD); + + cros_typec_register_partner_pdos(typec, &resp, port_num); } if (resp.events & PD_STATUS_EVENT_SOP_PRIME_DISC_DONE && diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c new file mode 100644 index 0000000000000000000000000000000000000000..a26219e97c931f6510417df43cafa9762ff06f9f --- /dev/null +++ b/drivers/platform/chrome/cros_typec_switch.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2022 Google LLC + * + * This driver provides the ability to configure Type-C muxes and retimers which are controlled by + * the ChromeOS EC. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Handles and other relevant data required for each port's switches. */ +struct cros_typec_port { + int port_num; + struct typec_mux_dev *mode_switch; + struct typec_retimer *retimer; + struct cros_typec_switch_data *sdata; +}; + +/* Driver-specific data. */ +struct cros_typec_switch_data { + struct device *dev; + struct cros_ec_device *ec; + struct cros_typec_port *ports[EC_USB_PD_MAX_PORTS]; +}; + +static int cros_typec_cmd_mux_set(struct cros_typec_switch_data *sdata, int port_num, u8 index, + u8 state) +{ + struct ec_params_typec_control req = { + .port = port_num, + .command = TYPEC_CONTROL_COMMAND_USB_MUX_SET, + .mux_params = { + .mux_index = index, + .mux_flags = state, + }, + }; + + return cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_CONTROL, &req, sizeof(req), NULL, 0); +} + +static int cros_typec_get_mux_state(unsigned long mode, struct typec_altmode *alt) +{ + int ret = -EOPNOTSUPP; + + if (mode == TYPEC_STATE_SAFE) + ret = USB_PD_MUX_SAFE_MODE; + else if (mode == TYPEC_STATE_USB) + ret = USB_PD_MUX_USB_ENABLED; + else if (alt && alt->svid == USB_TYPEC_DP_SID) + ret = USB_PD_MUX_DP_ENABLED; + + return ret; +} + +static int cros_typec_send_clear_event(struct cros_typec_switch_data *sdata, int port_num, + u32 events_mask) +{ + struct ec_params_typec_control req = { + .port = port_num, + .command = TYPEC_CONTROL_COMMAND_CLEAR_EVENTS, + .clear_events_mask = events_mask, + }; + + return cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_CONTROL, &req, sizeof(req), NULL, 0); +} + +static bool cros_typec_check_event(struct cros_typec_switch_data *sdata, int port_num, u32 mask) +{ + struct ec_response_typec_status resp; + struct ec_params_typec_status req = { + .port = port_num, + }; + int ret; + + ret = cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_STATUS, &req, sizeof(req), + &resp, sizeof(resp)); + if (ret < 0) { + dev_warn(sdata->dev, "EC_CMD_TYPEC_STATUS failed for port: %d\n", port_num); + return false; + } + + if (resp.events & mask) + return true; + + return false; +} + +/* + * The ChromeOS EC treats both mode-switches and retimers as "muxes" for the purposes of the + * host command API. This common function configures and verifies the retimer/mode-switch + * according to the provided setting. + */ +static int cros_typec_configure_mux(struct cros_typec_switch_data *sdata, int port_num, int index, + unsigned long mode, struct typec_altmode *alt) +{ + unsigned long end; + u32 event_mask; + u8 mux_state; + int ret; + + ret = cros_typec_get_mux_state(mode, alt); + if (ret < 0) + return ret; + mux_state = (u8)ret; + + /* Clear any old mux set done event. */ + if (index == 0) + event_mask = PD_STATUS_EVENT_MUX_0_SET_DONE; + else + event_mask = PD_STATUS_EVENT_MUX_1_SET_DONE; + + ret = cros_typec_send_clear_event(sdata, port_num, event_mask); + if (ret < 0) + return ret; + + /* Send the set command. */ + ret = cros_typec_cmd_mux_set(sdata, port_num, index, mux_state); + if (ret < 0) + return ret; + + /* Check for the mux set done event. */ + end = jiffies + msecs_to_jiffies(1000); + do { + if (cros_typec_check_event(sdata, port_num, event_mask)) + return 0; + + usleep_range(500, 1000); + } while (time_before(jiffies, end)); + + dev_err(sdata->dev, "Timed out waiting for mux set done on index: %d, state: %d\n", + index, mux_state); + + return -ETIMEDOUT; +} + +static int cros_typec_mode_switch_set(struct typec_mux_dev *mode_switch, + struct typec_mux_state *state) +{ + struct cros_typec_port *port = typec_mux_get_drvdata(mode_switch); + + /* Mode switches have index 0. */ + return cros_typec_configure_mux(port->sdata, port->port_num, 0, state->mode, state->alt); +} + +static int cros_typec_retimer_set(struct typec_retimer *retimer, struct typec_retimer_state *state) +{ + struct cros_typec_port *port = typec_retimer_get_drvdata(retimer); + + /* Retimers have index 1. */ + return cros_typec_configure_mux(port->sdata, port->port_num, 1, state->mode, state->alt); +} + +static void cros_typec_unregister_switches(struct cros_typec_switch_data *sdata) +{ + int i; + + for (i = 0; i < EC_USB_PD_MAX_PORTS; i++) { + if (!sdata->ports[i]) + continue; + typec_retimer_unregister(sdata->ports[i]->retimer); + typec_mux_unregister(sdata->ports[i]->mode_switch); + } +} + +static int cros_typec_register_mode_switch(struct cros_typec_port *port, + struct fwnode_handle *fwnode) +{ + struct typec_mux_desc mode_switch_desc = { + .fwnode = fwnode, + .drvdata = port, + .name = fwnode_get_name(fwnode), + .set = cros_typec_mode_switch_set, + }; + + port->mode_switch = typec_mux_register(port->sdata->dev, &mode_switch_desc); + + return PTR_ERR_OR_ZERO(port->mode_switch); +} + +static int cros_typec_register_retimer(struct cros_typec_port *port, struct fwnode_handle *fwnode) +{ + struct typec_retimer_desc retimer_desc = { + .fwnode = fwnode, + .drvdata = port, + .name = fwnode_get_name(fwnode), + .set = cros_typec_retimer_set, + }; + + port->retimer = typec_retimer_register(port->sdata->dev, &retimer_desc); + + return PTR_ERR_OR_ZERO(port->retimer); +} + +static int cros_typec_register_switches(struct cros_typec_switch_data *sdata) +{ + struct cros_typec_port *port; + struct device *dev = sdata->dev; + struct fwnode_handle *fwnode; + struct acpi_device *adev; + unsigned long long index; + int nports, ret; + + nports = device_get_child_node_count(dev); + if (nports == 0) { + dev_err(dev, "No switch devices found.\n"); + return -ENODEV; + } + + device_for_each_child_node(dev, fwnode) { + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) { + ret = -ENOMEM; + goto err_switch; + } + + adev = to_acpi_device_node(fwnode); + if (!adev) { + dev_err(fwnode->dev, "Couldn't get ACPI device handle\n"); + ret = -ENODEV; + goto err_switch; + } + + ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index); + if (ACPI_FAILURE(ret)) { + dev_err(fwnode->dev, "_ADR wasn't evaluated\n"); + ret = -ENODATA; + goto err_switch; + } + + if (index >= EC_USB_PD_MAX_PORTS) { + dev_err(fwnode->dev, "Invalid port index number: %llu\n", index); + ret = -EINVAL; + goto err_switch; + } + port->sdata = sdata; + port->port_num = index; + sdata->ports[index] = port; + + ret = cros_typec_register_retimer(port, fwnode); + if (ret) { + dev_err(dev, "Retimer switch register failed\n"); + goto err_switch; + } + + dev_dbg(dev, "Retimer switch registered for index %llu\n", index); + + if (!device_property_present(fwnode->dev, "mode-switch")) + continue; + + ret = cros_typec_register_mode_switch(port, fwnode); + if (ret) { + dev_err(dev, "Mode switch register failed\n"); + goto err_switch; + } + + dev_dbg(dev, "Mode switch registered for index %llu\n", index); + } + + return 0; +err_switch: + cros_typec_unregister_switches(sdata); + return ret; +} + +static int cros_typec_switch_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cros_typec_switch_data *sdata; + + sdata = devm_kzalloc(dev, sizeof(*sdata), GFP_KERNEL); + if (!sdata) + return -ENOMEM; + + sdata->dev = dev; + sdata->ec = dev_get_drvdata(pdev->dev.parent); + + platform_set_drvdata(pdev, sdata); + + return cros_typec_register_switches(sdata); +} + +static int cros_typec_switch_remove(struct platform_device *pdev) +{ + struct cros_typec_switch_data *sdata = platform_get_drvdata(pdev); + + cros_typec_unregister_switches(sdata); + return 0; +} + +#ifdef CONFIG_ACPI +static const struct acpi_device_id cros_typec_switch_acpi_id[] = { + { "GOOG001A", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, cros_typec_switch_acpi_id); +#endif + +static struct platform_driver cros_typec_switch_driver = { + .driver = { + .name = "cros-typec-switch", + .acpi_match_table = ACPI_PTR(cros_typec_switch_acpi_id), + }, + .probe = cros_typec_switch_probe, + .remove = cros_typec_switch_remove, +}; + +module_platform_driver(cros_typec_switch_driver); + +MODULE_AUTHOR("Prashant Malani "); +MODULE_DESCRIPTION("ChromeOS EC Type-C Switch control"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/loongarch/Kconfig b/drivers/platform/loongarch/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..5633e4d73991a69e4386e2771386d7c9a0181134 --- /dev/null +++ b/drivers/platform/loongarch/Kconfig @@ -0,0 +1,31 @@ +# +# LoongArch Platform Specific Drivers +# + +menuconfig LOONGARCH_PLATFORM_DEVICES + bool "LoongArch Platform Specific Device Drivers" + default y + depends on LOONGARCH + help + Say Y here to get to see options for device drivers of various + LoongArch platforms, including vendor-specific laptop/desktop + extension and hardware monitor drivers. This option itself does + not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if LOONGARCH_PLATFORM_DEVICES + +config LOONGSON_LAPTOP + tristate "Generic Loongson-3 Laptop Driver" + depends on ACPI + depends on BACKLIGHT_CLASS_DEVICE + depends on INPUT + depends on MACH_LOONGSON64 + select ACPI_VIDEO + select INPUT_SPARSEKMAP + default y + help + ACPI-based Loongson-3 family laptops generic driver. + +endif # LOONGARCH_PLATFORM_DEVICES diff --git a/drivers/platform/loongarch/Makefile b/drivers/platform/loongarch/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f43ab03db1a2d5bde40f59cbd403c866ca7f1026 --- /dev/null +++ b/drivers/platform/loongarch/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_LOONGSON_LAPTOP) += loongson-laptop.o diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c new file mode 100644 index 0000000000000000000000000000000000000000..f0166ad5d2c2890635a40449dc66ba59e0080694 --- /dev/null +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -0,0 +1,624 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generic Loongson processor based LAPTOP/ALL-IN-ONE driver + * + * Jianmin Lv + * Huacai Chen + * + * Copyright (C) 2022 Loongson Technology Corporation Limited + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* 1. Driver-wide structs and misc. variables */ + +/* ACPI HIDs */ +#define LOONGSON_ACPI_EC_HID "PNP0C09" +#define LOONGSON_ACPI_HKEY_HID "LOON0000" + +#define ACPI_LAPTOP_NAME "loongson-laptop" +#define ACPI_LAPTOP_ACPI_EVENT_PREFIX "loongson" + +#define MAX_ACPI_ARGS 3 +#define GENERIC_HOTKEY_MAP_MAX 64 + +#define GENERIC_EVENT_TYPE_OFF 12 +#define GENERIC_EVENT_TYPE_MASK 0xF000 +#define GENERIC_EVENT_CODE_MASK 0x0FFF + +struct generic_sub_driver { + u32 type; + char *name; + acpi_handle *handle; + struct acpi_device *device; + struct platform_driver *driver; + int (*init)(struct generic_sub_driver *sub_driver); + void (*notify)(struct generic_sub_driver *sub_driver, u32 event); + u8 acpi_notify_installed; +}; + +static u32 input_device_registered; +static struct input_dev *generic_inputdev; + +static acpi_handle hotkey_handle; +static struct key_entry hotkey_keycode_map[GENERIC_HOTKEY_MAP_MAX]; + +int loongson_laptop_turn_on_backlight(void); +int loongson_laptop_turn_off_backlight(void); +static int loongson_laptop_backlight_update(struct backlight_device *bd); + +/* 2. ACPI Helpers and device model */ + +static int acpi_evalf(acpi_handle handle, int *res, char *method, char *fmt, ...) +{ + char res_type; + char *fmt0 = fmt; + va_list ap; + int success, quiet; + acpi_status status; + struct acpi_object_list params; + struct acpi_buffer result, *resultp; + union acpi_object in_objs[MAX_ACPI_ARGS], out_obj; + + if (!*fmt) { + pr_err("acpi_evalf() called with empty format\n"); + return 0; + } + + if (*fmt == 'q') { + quiet = 1; + fmt++; + } else + quiet = 0; + + res_type = *(fmt++); + + params.count = 0; + params.pointer = &in_objs[0]; + + va_start(ap, fmt); + while (*fmt) { + char c = *(fmt++); + switch (c) { + case 'd': /* int */ + in_objs[params.count].integer.value = va_arg(ap, int); + in_objs[params.count++].type = ACPI_TYPE_INTEGER; + break; + /* add more types as needed */ + default: + pr_err("acpi_evalf() called with invalid format character '%c'\n", c); + va_end(ap); + return 0; + } + } + va_end(ap); + + if (res_type != 'v') { + result.length = sizeof(out_obj); + result.pointer = &out_obj; + resultp = &result; + } else + resultp = NULL; + + status = acpi_evaluate_object(handle, method, ¶ms, resultp); + + switch (res_type) { + case 'd': /* int */ + success = (status == AE_OK && out_obj.type == ACPI_TYPE_INTEGER); + if (success && res) + *res = out_obj.integer.value; + break; + case 'v': /* void */ + success = status == AE_OK; + break; + /* add more types as needed */ + default: + pr_err("acpi_evalf() called with invalid format character '%c'\n", res_type); + return 0; + } + + if (!success && !quiet) + pr_err("acpi_evalf(%s, %s, ...) failed: %s\n", + method, fmt0, acpi_format_exception(status)); + + return success; +} + +static int hotkey_status_get(int *status) +{ + if (!acpi_evalf(hotkey_handle, status, "GSWS", "d")) + return -EIO; + + return 0; +} + +static void dispatch_acpi_notify(acpi_handle handle, u32 event, void *data) +{ + struct generic_sub_driver *sub_driver = data; + + if (!sub_driver || !sub_driver->notify) + return; + sub_driver->notify(sub_driver, event); +} + +static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver) +{ + acpi_status status; + + if (!*sub_driver->handle) + return 0; + + sub_driver->device = acpi_fetch_acpi_dev(*sub_driver->handle); + if (!sub_driver->device) { + pr_err("acpi_fetch_acpi_dev(%s) failed\n", sub_driver->name); + return -ENODEV; + } + + sub_driver->device->driver_data = sub_driver; + sprintf(acpi_device_class(sub_driver->device), "%s/%s", + ACPI_LAPTOP_ACPI_EVENT_PREFIX, sub_driver->name); + + status = acpi_install_notify_handler(*sub_driver->handle, + sub_driver->type, dispatch_acpi_notify, sub_driver); + if (ACPI_FAILURE(status)) { + if (status == AE_ALREADY_EXISTS) { + pr_notice("Another device driver is already " + "handling %s events\n", sub_driver->name); + } else { + pr_err("acpi_install_notify_handler(%s) failed: %s\n", + sub_driver->name, acpi_format_exception(status)); + } + return -ENODEV; + } + sub_driver->acpi_notify_installed = 1; + + return 0; +} + +static int loongson_hotkey_suspend(struct device *dev) +{ + return 0; +} + +static int loongson_hotkey_resume(struct device *dev) +{ + int status = 0; + struct key_entry ke; + struct backlight_device *bd; + + /* + * Only if the firmware supports SW_LID event model, we can handle the + * event. This is for the consideration of development board without EC. + */ + if (test_bit(SW_LID, generic_inputdev->swbit)) { + if (hotkey_status_get(&status) < 0) + return -EIO; + /* + * The input device sw element records the last lid status. + * When the system is awakened by other wake-up sources, + * the lid event will also be reported. The judgment of + * adding SW_LID bit which in sw element can avoid this + * case. + * + * Input system will drop lid event when current lid event + * value and last lid status in the same. So laptop driver + * doesn't report repeated events. + * + * Lid status is generally 0, but hardware exception is + * considered. So add lid status confirmation. + */ + if (test_bit(SW_LID, generic_inputdev->sw) && !(status & (1 << SW_LID))) { + ke.type = KE_SW; + ke.sw.value = (u8)status; + ke.sw.code = SW_LID; + sparse_keymap_report_entry(generic_inputdev, &ke, 1, true); + } + } + + bd = backlight_device_get_by_type(BACKLIGHT_PLATFORM); + if (bd) { + loongson_laptop_backlight_update(bd) ? + pr_warn("Loongson_backlight: resume brightness failed") : + pr_info("Loongson_backlight: resume brightness %d\n", bd->props.brightness); + } + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(loongson_hotkey_pm, + loongson_hotkey_suspend, loongson_hotkey_resume); + +static int loongson_hotkey_probe(struct platform_device *pdev) +{ + hotkey_handle = ACPI_HANDLE(&pdev->dev); + + if (!hotkey_handle) + return -ENODEV; + + return 0; +} + +static const struct acpi_device_id loongson_device_ids[] = { + {LOONGSON_ACPI_HKEY_HID, 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, loongson_device_ids); + +static struct platform_driver loongson_hotkey_driver = { + .probe = loongson_hotkey_probe, + .driver = { + .name = "loongson-hotkey", + .owner = THIS_MODULE, + .pm = pm_ptr(&loongson_hotkey_pm), + .acpi_match_table = loongson_device_ids, + }, +}; + +static int hotkey_map(void) +{ + u32 index; + acpi_status status; + struct acpi_buffer buf; + union acpi_object *pack; + + buf.length = ACPI_ALLOCATE_BUFFER; + status = acpi_evaluate_object_typed(hotkey_handle, "KMAP", NULL, &buf, ACPI_TYPE_PACKAGE); + if (status != AE_OK) { + pr_err("ACPI exception: %s\n", acpi_format_exception(status)); + return -1; + } + pack = buf.pointer; + for (index = 0; index < pack->package.count; index++) { + union acpi_object *element, *sub_pack; + + sub_pack = &pack->package.elements[index]; + + element = &sub_pack->package.elements[0]; + hotkey_keycode_map[index].type = element->integer.value; + element = &sub_pack->package.elements[1]; + hotkey_keycode_map[index].code = element->integer.value; + element = &sub_pack->package.elements[2]; + hotkey_keycode_map[index].keycode = element->integer.value; + } + + return 0; +} + +static int hotkey_backlight_set(bool enable) +{ + if (!acpi_evalf(hotkey_handle, NULL, "VCBL", "vd", enable ? 1 : 0)) + return -EIO; + + return 0; +} + +static int ec_get_brightness(void) +{ + int status = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, &status, "ECBG", "d")) + return -EIO; + + return status; +} + +static int ec_set_brightness(int level) +{ + + int ret = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, NULL, "ECBS", "vd", level)) + ret = -EIO; + + return ret; +} + +static int ec_backlight_level(u8 level) +{ + int status = 0; + + if (!hotkey_handle) + return -ENXIO; + + if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d")) + return -EIO; + + if ((status < 0) || (level > status)) + return status; + + if (!acpi_evalf(hotkey_handle, &status, "ECSL", "d")) + return -EIO; + + if ((status < 0) || (level < status)) + return status; + + return level; +} + +static int loongson_laptop_backlight_update(struct backlight_device *bd) +{ + int lvl = ec_backlight_level(bd->props.brightness); + + if (lvl < 0) + return -EIO; + if (ec_set_brightness(lvl)) + return -EIO; + + return 0; +} + +static int loongson_laptop_get_brightness(struct backlight_device *bd) +{ + int level; + + level = ec_get_brightness(); + if (level < 0) + return -EIO; + + return level; +} + +static const struct backlight_ops backlight_laptop_ops = { + .update_status = loongson_laptop_backlight_update, + .get_brightness = loongson_laptop_get_brightness, +}; + +static int laptop_backlight_register(void) +{ + int status = 0; + struct backlight_properties props; + + memset(&props, 0, sizeof(props)); + + if (!acpi_evalf(hotkey_handle, &status, "ECLL", "d")) + return -EIO; + + props.brightness = 1; + props.max_brightness = status; + props.type = BACKLIGHT_PLATFORM; + + backlight_device_register("loongson_laptop", + NULL, NULL, &backlight_laptop_ops, &props); + + return 0; +} + +int loongson_laptop_turn_on_backlight(void) +{ + int status; + union acpi_object arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list args = { 1, &arg0 }; + + arg0.integer.value = 1; + status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); + if (ACPI_FAILURE(status)) { + pr_info("Loongson lvds error: 0x%x\n", status); + return -ENODEV; + } + + return 0; +} + +int loongson_laptop_turn_off_backlight(void) +{ + int status; + union acpi_object arg0 = { ACPI_TYPE_INTEGER }; + struct acpi_object_list args = { 1, &arg0 }; + + arg0.integer.value = 0; + status = acpi_evaluate_object(NULL, "\\BLSW", &args, NULL); + if (ACPI_FAILURE(status)) { + pr_info("Loongson lvds error: 0x%x\n", status); + return -ENODEV; + } + + return 0; +} + +static int __init event_init(struct generic_sub_driver *sub_driver) +{ + int ret; + + ret = hotkey_map(); + if (ret < 0) { + pr_err("Failed to parse keymap from DSDT\n"); + return ret; + } + + ret = sparse_keymap_setup(generic_inputdev, hotkey_keycode_map, NULL); + if (ret < 0) { + pr_err("Failed to setup input device keymap\n"); + input_free_device(generic_inputdev); + + return ret; + } + + /* + * This hotkey driver handle backlight event when + * acpi_video_get_backlight_type() gets acpi_backlight_vendor + */ + if (acpi_video_get_backlight_type() == acpi_backlight_vendor) + hotkey_backlight_set(true); + else + hotkey_backlight_set(false); + + pr_info("ACPI: enabling firmware HKEY event interface...\n"); + + return ret; +} + +static void event_notify(struct generic_sub_driver *sub_driver, u32 event) +{ + int type, scan_code; + struct key_entry *ke = NULL; + + scan_code = event & GENERIC_EVENT_CODE_MASK; + type = (event & GENERIC_EVENT_TYPE_MASK) >> GENERIC_EVENT_TYPE_OFF; + ke = sparse_keymap_entry_from_scancode(generic_inputdev, scan_code); + if (ke) { + if (type == KE_SW) { + int status = 0; + + if (hotkey_status_get(&status) < 0) + return; + + ke->sw.value = !!(status & (1 << ke->sw.code)); + } + sparse_keymap_report_entry(generic_inputdev, ke, 1, true); + } +} + +/* 3. Infrastructure */ + +static void generic_subdriver_exit(struct generic_sub_driver *sub_driver); + +static int __init generic_subdriver_init(struct generic_sub_driver *sub_driver) +{ + int ret; + + if (!sub_driver || !sub_driver->driver) + return -EINVAL; + + ret = platform_driver_register(sub_driver->driver); + if (ret) + return -EINVAL; + + if (sub_driver->init) + sub_driver->init(sub_driver); + + if (sub_driver->notify) { + ret = setup_acpi_notify(sub_driver); + if (ret == -ENODEV) { + ret = 0; + goto err_out; + } + if (ret < 0) + goto err_out; + } + + return 0; + +err_out: + generic_subdriver_exit(sub_driver); + return (ret < 0) ? ret : 0; +} + +static void generic_subdriver_exit(struct generic_sub_driver *sub_driver) +{ + + if (sub_driver->acpi_notify_installed) { + acpi_remove_notify_handler(*sub_driver->handle, + sub_driver->type, dispatch_acpi_notify); + sub_driver->acpi_notify_installed = 0; + } + platform_driver_unregister(sub_driver->driver); +} + +static struct generic_sub_driver generic_sub_drivers[] __refdata = { + { + .name = "hotkey", + .init = event_init, + .notify = event_notify, + .handle = &hotkey_handle, + .type = ACPI_DEVICE_NOTIFY, + .driver = &loongson_hotkey_driver, + }, +}; + +static int __init generic_acpi_laptop_init(void) +{ + bool ec_found; + int i, ret, status; + + if (acpi_disabled) + return -ENODEV; + + /* The EC device is required */ + ec_found = acpi_dev_found(LOONGSON_ACPI_EC_HID); + if (!ec_found) + return -ENODEV; + + /* Enable SCI for EC */ + acpi_write_bit_register(ACPI_BITREG_SCI_ENABLE, 1); + + generic_inputdev = input_allocate_device(); + if (!generic_inputdev) { + pr_err("Unable to allocate input device\n"); + return -ENOMEM; + } + + /* Prepare input device, but don't register */ + generic_inputdev->name = + "Loongson Generic Laptop/All-in-One Extra Buttons"; + generic_inputdev->phys = ACPI_LAPTOP_NAME "/input0"; + generic_inputdev->id.bustype = BUS_HOST; + generic_inputdev->dev.parent = NULL; + + /* Init subdrivers */ + for (i = 0; i < ARRAY_SIZE(generic_sub_drivers); i++) { + ret = generic_subdriver_init(&generic_sub_drivers[i]); + if (ret < 0) { + input_free_device(generic_inputdev); + while (--i >= 0) + generic_subdriver_exit(&generic_sub_drivers[i]); + return ret; + } + } + + ret = input_register_device(generic_inputdev); + if (ret < 0) { + input_free_device(generic_inputdev); + while (--i >= 0) + generic_subdriver_exit(&generic_sub_drivers[i]); + pr_err("Unable to register input device\n"); + return ret; + } + + input_device_registered = 1; + + if (acpi_evalf(hotkey_handle, &status, "ECBG", "d")) { + pr_info("Loongson Laptop used, init brightness is 0x%x\n", status); + ret = laptop_backlight_register(); + if (ret < 0) + pr_err("Loongson Laptop: laptop-backlight device register failed\n"); + } + + return 0; +} + +static void __exit generic_acpi_laptop_exit(void) +{ + if (generic_inputdev) { + if (input_device_registered) + input_unregister_device(generic_inputdev); + else + input_free_device(generic_inputdev); + } +} + +module_init(generic_acpi_laptop_init); +module_exit(generic_acpi_laptop_exit); + +MODULE_AUTHOR("Jianmin Lv "); +MODULE_AUTHOR("Huacai Chen "); +MODULE_DESCRIPTION("Loongson Laptop/All-in-One ACPI Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/platform/mellanox/mlxreg-lc.c b/drivers/platform/mellanox/mlxreg-lc.c index 55834ccb4ac7c3275858adbdfc75721ef88dfc4e..8d833836a6d322c2a6331097cb27bb6349bc37cf 100644 --- a/drivers/platform/mellanox/mlxreg-lc.c +++ b/drivers/platform/mellanox/mlxreg-lc.c @@ -460,8 +460,6 @@ static int mlxreg_lc_power_on_off(struct mlxreg_lc *mlxreg_lc, u8 action) u32 regval; int err; - mutex_lock(&mlxreg_lc->lock); - err = regmap_read(mlxreg_lc->par_regmap, mlxreg_lc->data->reg_pwr, ®val); if (err) goto regmap_read_fail; @@ -474,7 +472,6 @@ static int mlxreg_lc_power_on_off(struct mlxreg_lc *mlxreg_lc, u8 action) err = regmap_write(mlxreg_lc->par_regmap, mlxreg_lc->data->reg_pwr, regval); regmap_read_fail: - mutex_unlock(&mlxreg_lc->lock); return err; } @@ -491,8 +488,6 @@ static int mlxreg_lc_enable_disable(struct mlxreg_lc *mlxreg_lc, bool action) * line card which is already has been enabled. Disabling does not affect the disabled line * card. */ - mutex_lock(&mlxreg_lc->lock); - err = regmap_read(mlxreg_lc->par_regmap, mlxreg_lc->data->reg_ena, ®val); if (err) goto regmap_read_fail; @@ -505,7 +500,6 @@ static int mlxreg_lc_enable_disable(struct mlxreg_lc *mlxreg_lc, bool action) err = regmap_write(mlxreg_lc->par_regmap, mlxreg_lc->data->reg_ena, regval); regmap_read_fail: - mutex_unlock(&mlxreg_lc->lock); return err; } @@ -537,6 +531,15 @@ mlxreg_lc_sn4800_c16_config_init(struct mlxreg_lc *mlxreg_lc, void *regmap, static void mlxreg_lc_state_update(struct mlxreg_lc *mlxreg_lc, enum mlxreg_lc_state state, u8 action) +{ + if (action) + mlxreg_lc->state |= state; + else + mlxreg_lc->state &= ~state; +} + +static void +mlxreg_lc_state_update_locked(struct mlxreg_lc *mlxreg_lc, enum mlxreg_lc_state state, u8 action) { mutex_lock(&mlxreg_lc->lock); @@ -560,8 +563,9 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind, dev_info(mlxreg_lc->dev, "linecard#%d state %d event kind %d action %d\n", mlxreg_lc->data->slot, mlxreg_lc->state, kind, action); + mutex_lock(&mlxreg_lc->lock); if (!(mlxreg_lc->state & MLXREG_LC_INITIALIZED)) - return 0; + goto mlxreg_lc_non_initialzed_exit; switch (kind) { case MLXREG_HOTPLUG_LC_SYNCED: @@ -574,7 +578,7 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind, if (!(mlxreg_lc->state & MLXREG_LC_POWERED) && action) { err = mlxreg_lc_power_on_off(mlxreg_lc, 1); if (err) - return err; + goto mlxreg_lc_power_on_off_fail; } /* In case line card is configured - enable it. */ if (mlxreg_lc->state & MLXREG_LC_CONFIGURED && action) @@ -588,12 +592,13 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind, /* In case line card is configured - enable it. */ if (mlxreg_lc->state & MLXREG_LC_CONFIGURED) err = mlxreg_lc_enable_disable(mlxreg_lc, 1); - return err; + + goto mlxreg_lc_enable_disable_exit; } err = mlxreg_lc_create_static_devices(mlxreg_lc, mlxreg_lc->main_devs, mlxreg_lc->main_devs_num); if (err) - return err; + goto mlxreg_lc_create_static_devices_fail; /* In case line card is already in ready state - enable it. */ if (mlxreg_lc->state & MLXREG_LC_CONFIGURED) @@ -620,6 +625,12 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind, break; } +mlxreg_lc_enable_disable_exit: +mlxreg_lc_power_on_off_fail: +mlxreg_lc_create_static_devices_fail: +mlxreg_lc_non_initialzed_exit: + mutex_unlock(&mlxreg_lc->lock); + return err; } @@ -665,7 +676,7 @@ static int mlxreg_lc_completion_notify(void *handle, struct i2c_adapter *parent, if (err) goto mlxreg_lc_create_static_devices_failed; - mlxreg_lc_state_update(mlxreg_lc, MLXREG_LC_POWERED, 1); + mlxreg_lc_state_update_locked(mlxreg_lc, MLXREG_LC_POWERED, 1); } /* Verify if line card is synchronized. */ @@ -676,7 +687,7 @@ static int mlxreg_lc_completion_notify(void *handle, struct i2c_adapter *parent, /* Power on line card if necessary. */ if (regval & mlxreg_lc->data->mask) { mlxreg_lc->state |= MLXREG_LC_SYNCED; - mlxreg_lc_state_update(mlxreg_lc, MLXREG_LC_SYNCED, 1); + mlxreg_lc_state_update_locked(mlxreg_lc, MLXREG_LC_SYNCED, 1); if (mlxreg_lc->state & ~MLXREG_LC_POWERED) { err = mlxreg_lc_power_on_off(mlxreg_lc, 1); if (err) @@ -684,7 +695,7 @@ static int mlxreg_lc_completion_notify(void *handle, struct i2c_adapter *parent, } } - mlxreg_lc_state_update(mlxreg_lc, MLXREG_LC_INITIALIZED, 1); + mlxreg_lc_state_update_locked(mlxreg_lc, MLXREG_LC_INITIALIZED, 1); return 0; @@ -814,10 +825,9 @@ static int mlxreg_lc_probe(struct platform_device *pdev) mutex_init(&mlxreg_lc->lock); /* Set event notification callback. */ - if (data->notifier) { - data->notifier->user_handler = mlxreg_lc_event_handler; - data->notifier->handle = mlxreg_lc; - } + data->notifier->user_handler = mlxreg_lc_event_handler; + data->notifier->handle = mlxreg_lc; + data->hpdev.adapter = i2c_get_adapter(data->hpdev.nr); if (!data->hpdev.adapter) { dev_err(&pdev->dev, "Failed to get adapter for bus %d\n", @@ -863,7 +873,6 @@ static int mlxreg_lc_probe(struct platform_device *pdev) if (err) { dev_err(&pdev->dev, "Failed to sync regmap for client %s at bus %d at addr 0x%02x\n", data->hpdev.brdinfo->type, data->hpdev.nr, data->hpdev.brdinfo->addr); - err = PTR_ERR(regmap); goto regcache_sync_fail; } @@ -878,16 +887,14 @@ static int mlxreg_lc_probe(struct platform_device *pdev) if (err) goto mlxreg_lc_config_init_fail; - return err; + return 0; mlxreg_lc_config_init_fail: regcache_sync_fail: regmap_write_fail: devm_regmap_init_i2c_fail: - if (data->hpdev.client) { - i2c_unregister_device(data->hpdev.client); - data->hpdev.client = NULL; - } + i2c_unregister_device(data->hpdev.client); + data->hpdev.client = NULL; i2c_new_device_fail: i2c_put_adapter(data->hpdev.adapter); data->hpdev.adapter = NULL; @@ -905,6 +912,8 @@ static int mlxreg_lc_remove(struct platform_device *pdev) struct mlxreg_core_data *data = dev_get_platdata(&pdev->dev); struct mlxreg_lc *mlxreg_lc = platform_get_drvdata(pdev); + mlxreg_lc_state_update_locked(mlxreg_lc, MLXREG_LC_INITIALIZED, 0); + /* * Probing and removing are invoked by hotplug events raised upon line card insertion and * removing. If probing procedure fails all data is cleared. However, hotplug event still diff --git a/drivers/platform/surface/surface3_power.c b/drivers/platform/surface/surface3_power.c index 444ec81ba02d78cc88666a392eaf8569a425be0b..73961a24c849f44c1d2c705fab5be29856e7f241 100644 --- a/drivers/platform/surface/surface3_power.c +++ b/drivers/platform/surface/surface3_power.c @@ -519,7 +519,7 @@ static int mshw0011_probe(struct i2c_client *client) i2c_set_clientdata(client, data); memset(&board_info, 0, sizeof(board_info)); - strlcpy(board_info.type, "MSHW0011-bat0", I2C_NAME_SIZE); + strscpy(board_info.type, "MSHW0011-bat0", I2C_NAME_SIZE); bat0 = i2c_acpi_new_device(dev, 1, &board_info); if (IS_ERR(bat0)) @@ -554,7 +554,7 @@ out_err: return error; } -static int mshw0011_remove(struct i2c_client *client) +static void mshw0011_remove(struct i2c_client *client) { struct mshw0011_data *cdata = i2c_get_clientdata(client); @@ -564,8 +564,6 @@ static int mshw0011_remove(struct i2c_client *client) kthread_stop(cdata->poll_task); i2c_unregister_device(cdata->bat0); - - return 0; } static const struct acpi_device_id mshw0011_acpi_match[] = { diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c index 44e31797055724d085ef5742dfbe47cc8fbba12b..50500e562963d6027553aabf6678c7e9fd23bf73 100644 --- a/drivers/platform/surface/surface_acpi_notify.c +++ b/drivers/platform/surface/surface_acpi_notify.c @@ -355,7 +355,8 @@ static u32 san_evt_bat_nf(struct ssam_event_notifier *nf, INIT_DELAYED_WORK(&work->work, san_evt_bat_workfn); work->dev = d->dev; - memcpy(&work->event, event, sizeof(struct ssam_event) + event->length); + work->event = *event; + memcpy(work->event.data, event->data, event->length); queue_delayed_work(san_wq, &work->work, delay); return SSAM_NOTIF_HANDLED; diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c index d5655f6a4a4162b245b2bc3c62bfe8cf6a37ab0f..585911020cea076229a853f49888a7047b851e98 100644 --- a/drivers/platform/surface/surface_aggregator_registry.c +++ b/drivers/platform/surface/surface_aggregator_registry.c @@ -86,38 +86,38 @@ static const struct software_node ssam_node_bas_dtx = { .parent = &ssam_node_root, }; -/* HID keyboard (TID1). */ -static const struct software_node ssam_node_hid_tid1_keyboard = { +/* HID keyboard (SAM, TID=1). */ +static const struct software_node ssam_node_hid_sam_keyboard = { .name = "ssam:01:15:01:01:00", .parent = &ssam_node_root, }; -/* HID pen stash (TID1; pen taken / stashed away evens). */ -static const struct software_node ssam_node_hid_tid1_penstash = { +/* HID pen stash (SAM, TID=1; pen taken / stashed away evens). */ +static const struct software_node ssam_node_hid_sam_penstash = { .name = "ssam:01:15:01:02:00", .parent = &ssam_node_root, }; -/* HID touchpad (TID1). */ -static const struct software_node ssam_node_hid_tid1_touchpad = { +/* HID touchpad (SAM, TID=1). */ +static const struct software_node ssam_node_hid_sam_touchpad = { .name = "ssam:01:15:01:03:00", .parent = &ssam_node_root, }; -/* HID device instance 6 (TID1, unknown HID device). */ -static const struct software_node ssam_node_hid_tid1_iid6 = { +/* HID device instance 6 (SAM, TID=1, HID sensor collection). */ +static const struct software_node ssam_node_hid_sam_sensors = { .name = "ssam:01:15:01:06:00", .parent = &ssam_node_root, }; -/* HID device instance 7 (TID1, unknown HID device). */ -static const struct software_node ssam_node_hid_tid1_iid7 = { +/* HID device instance 7 (SAM, TID=1, UCM UCSI HID client). */ +static const struct software_node ssam_node_hid_sam_ucm_ucsi = { .name = "ssam:01:15:01:07:00", .parent = &ssam_node_root, }; -/* HID system controls (TID1). */ -static const struct software_node ssam_node_hid_tid1_sysctrl = { +/* HID system controls (SAM, TID=1). */ +static const struct software_node ssam_node_hid_sam_sysctrl = { .name = "ssam:01:15:01:08:00", .parent = &ssam_node_root, }; @@ -182,8 +182,8 @@ static const struct software_node ssam_node_hid_kip_touchpad = { .parent = &ssam_node_hub_kip, }; -/* HID device instance 5 (KIP hub, unknown HID device). */ -static const struct software_node ssam_node_hid_kip_iid5 = { +/* HID device instance 5 (KIP hub, type-cover firmware update). */ +static const struct software_node ssam_node_hid_kip_fwupd = { .name = "ssam:01:15:02:05:00", .parent = &ssam_node_hub_kip, }; @@ -241,12 +241,12 @@ static const struct software_node *ssam_node_group_sls[] = { &ssam_node_bat_main, &ssam_node_tmp_pprof, &ssam_node_pos_tablet_switch, - &ssam_node_hid_tid1_keyboard, - &ssam_node_hid_tid1_penstash, - &ssam_node_hid_tid1_touchpad, - &ssam_node_hid_tid1_iid6, - &ssam_node_hid_tid1_iid7, - &ssam_node_hid_tid1_sysctrl, + &ssam_node_hid_sam_keyboard, + &ssam_node_hid_sam_penstash, + &ssam_node_hid_sam_touchpad, + &ssam_node_hid_sam_sensors, + &ssam_node_hid_sam_ucm_ucsi, + &ssam_node_hid_sam_sysctrl, NULL, }; @@ -278,7 +278,9 @@ static const struct software_node *ssam_node_group_sp8[] = { &ssam_node_hid_kip_keyboard, &ssam_node_hid_kip_penstash, &ssam_node_hid_kip_touchpad, - &ssam_node_hid_kip_iid5, + &ssam_node_hid_kip_fwupd, + &ssam_node_hid_sam_sensors, + &ssam_node_hid_sam_ucm_ucsi, NULL, }; @@ -325,6 +327,9 @@ static const struct acpi_device_id ssam_platform_hub_match[] = { /* Surface Laptop Go 1 */ { "MSHW0118", (unsigned long)ssam_node_group_slg1 }, + /* Surface Laptop Go 2 */ + { "MSHW0290", (unsigned long)ssam_node_group_slg1 }, + /* Surface Laptop Studio */ { "MSHW0123", (unsigned long)ssam_node_group_sls }, diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 0cc5ac35fc570b1b61df133357e3d21056b30452..f5312f51de19f9d3ff38ef15cf677d78bdee158f 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -791,6 +791,7 @@ config SAMSUNG_Q10 config ACPI_TOSHIBA tristate "Toshiba Laptop Extras" depends on ACPI + depends on ACPI_BATTERY depends on ACPI_WMI select LEDS_CLASS select NEW_LEDS @@ -798,6 +799,7 @@ config ACPI_TOSHIBA depends on INPUT depends on SERIO_I8042 || SERIO_I8042 = n depends on ACPI_VIDEO || ACPI_VIDEO = n + depends on HWMON || HWMON = n depends on RFKILL || RFKILL = n depends on IIO select INPUT_SPARSEKMAP diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c index b933a5165edbf2859c6b797f52f16c2d1bc32652..18224f9a5bc07604c73c1789eb3a288fce862547 100644 --- a/drivers/platform/x86/acer-wmi.c +++ b/drivers/platform/x86/acer-wmi.c @@ -99,6 +99,7 @@ static const struct key_entry acer_wmi_keymap[] __initconst = { {KE_KEY, 0x22, {KEY_PROG2} }, /* Arcade */ {KE_KEY, 0x23, {KEY_PROG3} }, /* P_Key */ {KE_KEY, 0x24, {KEY_PROG4} }, /* Social networking_Key */ + {KE_KEY, 0x27, {KEY_HELP} }, {KE_KEY, 0x29, {KEY_PROG3} }, /* P_Key for TM8372 */ {KE_IGNORE, 0x41, {KEY_MUTE} }, {KE_IGNORE, 0x42, {KEY_PREVIOUSSONG} }, @@ -112,7 +113,13 @@ static const struct key_entry acer_wmi_keymap[] __initconst = { {KE_IGNORE, 0x48, {KEY_VOLUMEUP} }, {KE_IGNORE, 0x49, {KEY_VOLUMEDOWN} }, {KE_IGNORE, 0x4a, {KEY_VOLUMEDOWN} }, - {KE_IGNORE, 0x61, {KEY_SWITCHVIDEOMODE} }, + /* + * 0x61 is KEY_SWITCHVIDEOMODE. Usually this is a duplicate input event + * with the "Video Bus" input device events. But sometimes it is not + * a dup. Map it to KEY_UNKNOWN instead of using KE_IGNORE so that + * udev/hwdb can override it on systems where it is not a dup. + */ + {KE_KEY, 0x61, {KEY_UNKNOWN} }, {KE_IGNORE, 0x62, {KEY_BRIGHTNESSUP} }, {KE_IGNORE, 0x63, {KEY_BRIGHTNESSDOWN} }, {KE_KEY, 0x64, {KEY_SWITCHVIDEOMODE} }, /* Display Switch */ @@ -2456,7 +2463,7 @@ static int __init acer_wmi_init(void) goto error_platform_register; } - acer_platform_device = platform_device_alloc("acer-wmi", -1); + acer_platform_device = platform_device_alloc("acer-wmi", PLATFORM_DEVID_NONE); if (!acer_platform_device) { err = -ENOMEM; goto error_device_alloc; diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c index 3463629f87640a2efcaf477a731e5a9058a4fe14..d2c0fc38c201f86bc791fcc5b0be7c8142be6fb9 100644 --- a/drivers/platform/x86/acerhdf.c +++ b/drivers/platform/x86/acerhdf.c @@ -676,7 +676,7 @@ static int __init acerhdf_register_platform(void) if (err) return err; - acerhdf_dev = platform_device_alloc("acerhdf", -1); + acerhdf_dev = platform_device_alloc("acerhdf", PLATFORM_DEVID_NONE); if (!acerhdf_dev) { err = -ENOMEM; goto err_device_alloc; diff --git a/drivers/platform/x86/amd/Kconfig b/drivers/platform/x86/amd/Kconfig index c0d0a3c5170c96e384e03abe4ed76d77ed5411a5..a825af8126c83625b44efd76ca478a7a3cade1b7 100644 --- a/drivers/platform/x86/amd/Kconfig +++ b/drivers/platform/x86/amd/Kconfig @@ -3,6 +3,8 @@ # AMD x86 Platform Specific Drivers # +source "drivers/platform/x86/amd/pmf/Kconfig" + config AMD_PMC tristate "AMD SoC PMC driver" depends on ACPI && PCI && RTC_CLASS diff --git a/drivers/platform/x86/amd/Makefile b/drivers/platform/x86/amd/Makefile index a03fbb08e808631f4bcfb3206ee9091f51a17e2a..2c229198e24c967e0f73015ee0bad82b70e0beb6 100644 --- a/drivers/platform/x86/amd/Makefile +++ b/drivers/platform/x86/amd/Makefile @@ -8,3 +8,4 @@ amd-pmc-y := pmc.o obj-$(CONFIG_AMD_PMC) += amd-pmc.o amd_hsmp-y := hsmp.o obj-$(CONFIG_AMD_HSMP) += amd_hsmp.o +obj-$(CONFIG_AMD_PMF) += pmf/ diff --git a/drivers/platform/x86/amd/hsmp.c b/drivers/platform/x86/amd/hsmp.c index a0c54b838c114fa19f22d64738335f31f22b2931..521c6a229362c3f60ec668d3ecaa02abb0c72e6e 100644 --- a/drivers/platform/x86/amd/hsmp.c +++ b/drivers/platform/x86/amd/hsmp.c @@ -392,7 +392,7 @@ static int __init hsmp_plt_init(void) if (ret) return ret; - amd_hsmp_platdev = platform_device_alloc(DRIVER_NAME, -1); + amd_hsmp_platdev = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE); if (!amd_hsmp_platdev) { ret = -ENOMEM; goto drv_unregister; diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c index 700eb19e84500ca729ae8d7eedcb6536c0cd75f6..ce859b300712be9fc1d43f97a93523df241c6afd 100644 --- a/drivers/platform/x86/amd/pmc.c +++ b/drivers/platform/x86/amd/pmc.c @@ -39,7 +39,9 @@ #define AMD_PMC_STB_INDEX_ADDRESS 0xF8 #define AMD_PMC_STB_INDEX_DATA 0xFC #define AMD_PMC_STB_PMI_0 0x03E30600 -#define AMD_PMC_STB_PREDEF 0xC6000001 +#define AMD_PMC_STB_S2IDLE_PREPARE 0xC6000001 +#define AMD_PMC_STB_S2IDLE_RESTORE 0xC6000002 +#define AMD_PMC_STB_S2IDLE_CHECK 0xC6000003 /* STB S2D(Spill to DRAM) has different message port offset */ #define STB_SPILL_TO_DRAM 0xBE @@ -151,9 +153,7 @@ struct amd_pmc_dev { struct device *dev; struct pci_dev *rdev; struct mutex lock; /* generic mutex lock */ -#if IS_ENABLED(CONFIG_DEBUG_FS) struct dentry *dbgfs_dir; -#endif /* CONFIG_DEBUG_FS */ }; static bool enable_stb; @@ -369,7 +369,64 @@ static void amd_pmc_validate_deepest(struct amd_pmc_dev *pdev) } #endif -#ifdef CONFIG_DEBUG_FS +static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev) +{ + int rc; + u32 val; + + rc = amd_pmc_send_cmd(dev, 0, &val, SMU_MSG_GETSMUVERSION, 1); + if (rc) + return rc; + + dev->smu_program = (val >> 24) & GENMASK(7, 0); + dev->major = (val >> 16) & GENMASK(7, 0); + dev->minor = (val >> 8) & GENMASK(7, 0); + dev->rev = (val >> 0) & GENMASK(7, 0); + + dev_dbg(dev->dev, "SMU program %u version is %u.%u.%u\n", + dev->smu_program, dev->major, dev->minor, dev->rev); + + return 0; +} + +static ssize_t smu_fw_version_show(struct device *d, struct device_attribute *attr, + char *buf) +{ + struct amd_pmc_dev *dev = dev_get_drvdata(d); + + if (!dev->major) { + int rc = amd_pmc_get_smu_version(dev); + + if (rc) + return rc; + } + return sysfs_emit(buf, "%u.%u.%u\n", dev->major, dev->minor, dev->rev); +} + +static ssize_t smu_program_show(struct device *d, struct device_attribute *attr, + char *buf) +{ + struct amd_pmc_dev *dev = dev_get_drvdata(d); + + if (!dev->major) { + int rc = amd_pmc_get_smu_version(dev); + + if (rc) + return rc; + } + return sysfs_emit(buf, "%u\n", dev->smu_program); +} + +static DEVICE_ATTR_RO(smu_fw_version); +static DEVICE_ATTR_RO(smu_program); + +static struct attribute *pmc_attrs[] = { + &dev_attr_smu_fw_version.attr, + &dev_attr_smu_program.attr, + NULL, +}; +ATTRIBUTE_GROUPS(pmc); + static int smu_fw_info_show(struct seq_file *s, void *unused) { struct amd_pmc_dev *dev = s->private; @@ -435,26 +492,6 @@ static int s0ix_stats_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(s0ix_stats); -static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev) -{ - int rc; - u32 val; - - rc = amd_pmc_send_cmd(dev, 0, &val, SMU_MSG_GETSMUVERSION, 1); - if (rc) - return rc; - - dev->smu_program = (val >> 24) & GENMASK(7, 0); - dev->major = (val >> 16) & GENMASK(7, 0); - dev->minor = (val >> 8) & GENMASK(7, 0); - dev->rev = (val >> 0) & GENMASK(7, 0); - - dev_dbg(dev->dev, "SMU program %u version is %u.%u.%u\n", - dev->smu_program, dev->major, dev->minor, dev->rev); - - return 0; -} - static int amd_pmc_idlemask_show(struct seq_file *s, void *unused) { struct amd_pmc_dev *dev = s->private; @@ -504,15 +541,6 @@ static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev) &amd_pmc_stb_debugfs_fops); } } -#else -static inline void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev) -{ -} - -static inline void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev) -{ -} -#endif /* CONFIG_DEBUG_FS */ static void amd_pmc_dump_registers(struct amd_pmc_dev *dev) { @@ -691,8 +719,6 @@ static void amd_pmc_s2idle_prepare(void) } } - /* Dump the IdleMask before we send hint to SMU */ - amd_pmc_idlemask_read(pdev, pdev->dev, NULL); msg = amd_pmc_get_os_hint(pdev); rc = amd_pmc_send_cmd(pdev, arg, NULL, msg, 0); if (rc) { @@ -700,11 +726,22 @@ static void amd_pmc_s2idle_prepare(void) return; } - if (enable_stb) { - rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_PREDEF); - if (rc) - dev_err(pdev->dev, "error writing to STB: %d\n", rc); - } + rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_PREPARE); + if (rc) + dev_err(pdev->dev, "error writing to STB: %d\n", rc); +} + +static void amd_pmc_s2idle_check(void) +{ + struct amd_pmc_dev *pdev = &pmc; + int rc; + + /* Dump the IdleMask before we add to the STB */ + amd_pmc_idlemask_read(pdev, pdev->dev, NULL); + + rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_CHECK); + if (rc) + dev_err(pdev->dev, "error writing to STB: %d\n", rc); } static void amd_pmc_s2idle_restore(void) @@ -721,15 +758,9 @@ static void amd_pmc_s2idle_restore(void) /* Let SMU know that we are looking for stats */ amd_pmc_send_cmd(pdev, 0, NULL, SMU_MSG_LOG_DUMP_DATA, 0); - /* Dump the IdleMask to see the blockers */ - amd_pmc_idlemask_read(pdev, pdev->dev, NULL); - - /* Write data incremented by 1 to distinguish in stb_read */ - if (enable_stb) { - rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_PREDEF + 1); - if (rc) - dev_err(pdev->dev, "error writing to STB: %d\n", rc); - } + rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_RESTORE); + if (rc) + dev_err(pdev->dev, "error writing to STB: %d\n", rc); /* Notify on failed entry */ amd_pmc_validate_deepest(pdev); @@ -737,6 +768,7 @@ static void amd_pmc_s2idle_restore(void) static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = { .prepare = amd_pmc_s2idle_prepare, + .check = amd_pmc_s2idle_check, .restore = amd_pmc_s2idle_restore, }; #endif @@ -935,6 +967,7 @@ static struct platform_driver amd_pmc_driver = { .driver = { .name = "amd_pmc", .acpi_match_table = amd_pmc_acpi_ids, + .dev_groups = pmc_groups, }, .probe = amd_pmc_probe, .remove = amd_pmc_remove, diff --git a/drivers/platform/x86/amd/pmf/Kconfig b/drivers/platform/x86/amd/pmf/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..c375498c40717f7c311d9790fdb083081ea46d3f --- /dev/null +++ b/drivers/platform/x86/amd/pmf/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# AMD PMF Driver +# + +config AMD_PMF + tristate "AMD Platform Management Framework" + depends on ACPI && PCI + select ACPI_PLATFORM_PROFILE + help + This driver provides support for the AMD Platform Management Framework. + The goal is to enhance end user experience by making AMD PCs smarter, + quiter, power efficient by adapting to user behavior and environment. + + To compile this driver as a module, choose M here: the module will + be called amd_pmf. diff --git a/drivers/platform/x86/amd/pmf/Makefile b/drivers/platform/x86/amd/pmf/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..fdededf5439200e4592ab7192fbf56bc22df01d2 --- /dev/null +++ b/drivers/platform/x86/amd/pmf/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for linux/drivers/platform/x86/amd/pmf +# AMD Platform Management Framework +# + +obj-$(CONFIG_AMD_PMF) += amd-pmf.o +amd-pmf-objs := core.o acpi.o sps.o \ + auto-mode.o cnqf.o diff --git a/drivers/platform/x86/amd/pmf/acpi.c b/drivers/platform/x86/amd/pmf/acpi.c new file mode 100644 index 0000000000000000000000000000000000000000..081e84e116e792a0aeb48922a68ed07680bcdd87 --- /dev/null +++ b/drivers/platform/x86/amd/pmf/acpi.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Platform Management Framework Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#include +#include "pmf.h" + +#define APMF_CQL_NOTIFICATION 2 +#define APMF_AMT_NOTIFICATION 3 + +static union acpi_object *apmf_if_call(struct amd_pmf_dev *pdev, int fn, struct acpi_buffer *param) +{ + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_handle ahandle = ACPI_HANDLE(pdev->dev); + struct acpi_object_list apmf_if_arg_list; + union acpi_object apmf_if_args[2]; + acpi_status status; + + apmf_if_arg_list.count = 2; + apmf_if_arg_list.pointer = &apmf_if_args[0]; + + apmf_if_args[0].type = ACPI_TYPE_INTEGER; + apmf_if_args[0].integer.value = fn; + + if (param) { + apmf_if_args[1].type = ACPI_TYPE_BUFFER; + apmf_if_args[1].buffer.length = param->length; + apmf_if_args[1].buffer.pointer = param->pointer; + } else { + apmf_if_args[1].type = ACPI_TYPE_INTEGER; + apmf_if_args[1].integer.value = 0; + } + + status = acpi_evaluate_object(ahandle, "APMF", &apmf_if_arg_list, &buffer); + if (ACPI_FAILURE(status)) { + dev_err(pdev->dev, "APMF method:%d call failed\n", fn); + kfree(buffer.pointer); + return NULL; + } + + return buffer.pointer; +} + +static int apmf_if_call_store_buffer(struct amd_pmf_dev *pdev, int fn, void *dest, size_t out_sz) +{ + union acpi_object *info; + size_t size; + int err = 0; + + info = apmf_if_call(pdev, fn, NULL); + if (!info) + return -EIO; + + if (info->type != ACPI_TYPE_BUFFER) { + dev_err(pdev->dev, "object is not a buffer\n"); + err = -EINVAL; + goto out; + } + + if (info->buffer.length < 2) { + dev_err(pdev->dev, "buffer too small\n"); + err = -EINVAL; + goto out; + } + + size = *(u16 *)info->buffer.pointer; + if (info->buffer.length < size) { + dev_err(pdev->dev, "buffer smaller then headersize %u < %zu\n", + info->buffer.length, size); + err = -EINVAL; + goto out; + } + + if (size < out_sz) { + dev_err(pdev->dev, "buffer too small %zu\n", size); + err = -EINVAL; + goto out; + } + + memcpy(dest, info->buffer.pointer, out_sz); + +out: + kfree(info); + return err; +} + +int is_apmf_func_supported(struct amd_pmf_dev *pdev, unsigned long index) +{ + /* If bit-n is set, that indicates function n+1 is supported */ + return !!(pdev->supported_func & BIT(index - 1)); +} + +int apmf_get_static_slider_granular(struct amd_pmf_dev *pdev, + struct apmf_static_slider_granular_output *data) +{ + if (!is_apmf_func_supported(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) + return -EINVAL; + + return apmf_if_call_store_buffer(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR, + data, sizeof(*data)); +} + +static void apmf_sbios_heartbeat_notify(struct work_struct *work) +{ + struct amd_pmf_dev *dev = container_of(work, struct amd_pmf_dev, heart_beat.work); + union acpi_object *info; + + dev_dbg(dev->dev, "Sending heartbeat to SBIOS\n"); + info = apmf_if_call(dev, APMF_FUNC_SBIOS_HEARTBEAT, NULL); + if (!info) + goto out; + + schedule_delayed_work(&dev->heart_beat, msecs_to_jiffies(dev->hb_interval * 1000)); + +out: + kfree(info); +} + +int apmf_update_fan_idx(struct amd_pmf_dev *pdev, bool manual, u32 idx) +{ + union acpi_object *info; + struct apmf_fan_idx args; + struct acpi_buffer params; + int err = 0; + + args.size = sizeof(args); + args.fan_ctl_mode = manual; + args.fan_ctl_idx = idx; + + params.length = sizeof(args); + params.pointer = (void *)&args; + + info = apmf_if_call(pdev, APMF_FUNC_SET_FAN_IDX, ¶ms); + if (!info) { + err = -EIO; + goto out; + } + +out: + kfree(info); + return err; +} + +int apmf_get_auto_mode_def(struct amd_pmf_dev *pdev, struct apmf_auto_mode *data) +{ + return apmf_if_call_store_buffer(pdev, APMF_FUNC_AUTO_MODE, data, sizeof(*data)); +} + +int apmf_get_sbios_requests(struct amd_pmf_dev *pdev, struct apmf_sbios_req *req) +{ + return apmf_if_call_store_buffer(pdev, APMF_FUNC_SBIOS_REQUESTS, + req, sizeof(*req)); +} + +static void apmf_event_handler(acpi_handle handle, u32 event, void *data) +{ + struct amd_pmf_dev *pmf_dev = data; + struct apmf_sbios_req req; + int ret; + + mutex_lock(&pmf_dev->update_mutex); + ret = apmf_get_sbios_requests(pmf_dev, &req); + if (ret) { + dev_err(pmf_dev->dev, "Failed to get SBIOS requests:%d\n", ret); + goto out; + } + + if (req.pending_req & BIT(APMF_AMT_NOTIFICATION)) { + dev_dbg(pmf_dev->dev, "AMT is supported and notifications %s\n", + req.amt_event ? "Enabled" : "Disabled"); + pmf_dev->amt_enabled = !!req.amt_event; + + if (pmf_dev->amt_enabled) + amd_pmf_handle_amt(pmf_dev); + else + amd_pmf_reset_amt(pmf_dev); + } + + if (req.pending_req & BIT(APMF_CQL_NOTIFICATION)) { + dev_dbg(pmf_dev->dev, "CQL is supported and notifications %s\n", + req.cql_event ? "Enabled" : "Disabled"); + + /* update the target mode information */ + if (pmf_dev->amt_enabled) + amd_pmf_update_2_cql(pmf_dev, req.cql_event); + } +out: + mutex_unlock(&pmf_dev->update_mutex); +} + +static int apmf_if_verify_interface(struct amd_pmf_dev *pdev) +{ + struct apmf_verify_interface output; + int err; + + err = apmf_if_call_store_buffer(pdev, APMF_FUNC_VERIFY_INTERFACE, &output, sizeof(output)); + if (err) + return err; + + pdev->supported_func = output.supported_functions; + dev_dbg(pdev->dev, "supported functions:0x%x notifications:0x%x\n", + output.supported_functions, output.notification_mask); + + return 0; +} + +static int apmf_get_system_params(struct amd_pmf_dev *dev) +{ + struct apmf_system_params params; + int err; + + if (!is_apmf_func_supported(dev, APMF_FUNC_GET_SYS_PARAMS)) + return -EINVAL; + + err = apmf_if_call_store_buffer(dev, APMF_FUNC_GET_SYS_PARAMS, ¶ms, sizeof(params)); + if (err) + return err; + + dev_dbg(dev->dev, "system params mask:0x%x flags:0x%x cmd_code:0x%x heartbeat:%d\n", + params.valid_mask, + params.flags, + params.command_code, + params.heartbeat_int); + params.flags = params.flags & params.valid_mask; + dev->hb_interval = params.heartbeat_int; + + return 0; +} + +int apmf_get_dyn_slider_def_ac(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data) +{ + return apmf_if_call_store_buffer(pdev, APMF_FUNC_DYN_SLIDER_AC, data, sizeof(*data)); +} + +int apmf_get_dyn_slider_def_dc(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data) +{ + return apmf_if_call_store_buffer(pdev, APMF_FUNC_DYN_SLIDER_DC, data, sizeof(*data)); +} + +int apmf_install_handler(struct amd_pmf_dev *pmf_dev) +{ + acpi_handle ahandle = ACPI_HANDLE(pmf_dev->dev); + acpi_status status; + + /* Install the APMF Notify handler */ + if (is_apmf_func_supported(pmf_dev, APMF_FUNC_AUTO_MODE) && + is_apmf_func_supported(pmf_dev, APMF_FUNC_SBIOS_REQUESTS)) { + status = acpi_install_notify_handler(ahandle, ACPI_ALL_NOTIFY, + apmf_event_handler, pmf_dev); + if (ACPI_FAILURE(status)) { + dev_err(pmf_dev->dev, "failed to install notify handler\n"); + return -ENODEV; + } + + /* Call the handler once manually to catch up with possibly missed notifies. */ + apmf_event_handler(ahandle, 0, pmf_dev); + } + + return 0; +} + +void apmf_acpi_deinit(struct amd_pmf_dev *pmf_dev) +{ + acpi_handle ahandle = ACPI_HANDLE(pmf_dev->dev); + + if (pmf_dev->hb_interval) + cancel_delayed_work_sync(&pmf_dev->heart_beat); + + if (is_apmf_func_supported(pmf_dev, APMF_FUNC_AUTO_MODE) && + is_apmf_func_supported(pmf_dev, APMF_FUNC_SBIOS_REQUESTS)) + acpi_remove_notify_handler(ahandle, ACPI_ALL_NOTIFY, apmf_event_handler); +} + +int apmf_acpi_init(struct amd_pmf_dev *pmf_dev) +{ + int ret; + + ret = apmf_if_verify_interface(pmf_dev); + if (ret) { + dev_err(pmf_dev->dev, "APMF verify interface failed :%d\n", ret); + goto out; + } + + ret = apmf_get_system_params(pmf_dev); + if (ret) { + dev_err(pmf_dev->dev, "APMF apmf_get_system_params failed :%d\n", ret); + goto out; + } + + if (pmf_dev->hb_interval) { + /* send heartbeats only if the interval is not zero */ + INIT_DELAYED_WORK(&pmf_dev->heart_beat, apmf_sbios_heartbeat_notify); + schedule_delayed_work(&pmf_dev->heart_beat, 0); + } + +out: + return ret; +} diff --git a/drivers/platform/x86/amd/pmf/auto-mode.c b/drivers/platform/x86/amd/pmf/auto-mode.c new file mode 100644 index 0000000000000000000000000000000000000000..644af42e07cf8168dee748042c5f92698fbc59a2 --- /dev/null +++ b/drivers/platform/x86/amd/pmf/auto-mode.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Platform Management Framework Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#include +#include +#include "pmf.h" + +static struct auto_mode_mode_config config_store; +static const char *state_as_str(unsigned int state); + +static void amd_pmf_set_automode(struct amd_pmf_dev *dev, int idx, + struct auto_mode_mode_config *table) +{ + struct power_table_control *pwr_ctrl = &config_store.mode_set[idx].power_control; + + amd_pmf_send_cmd(dev, SET_SPL, false, pwr_ctrl->spl, NULL); + amd_pmf_send_cmd(dev, SET_FPPT, false, pwr_ctrl->fppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT, false, pwr_ctrl->sppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false, pwr_ctrl->sppt_apu_only, NULL); + amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false, pwr_ctrl->stt_min, NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false, + pwr_ctrl->stt_skin_temp[STT_TEMP_APU], NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false, + pwr_ctrl->stt_skin_temp[STT_TEMP_HS2], NULL); + + if (is_apmf_func_supported(dev, APMF_FUNC_SET_FAN_IDX)) + apmf_update_fan_idx(dev, config_store.mode_set[idx].fan_control.manual, + config_store.mode_set[idx].fan_control.fan_id); +} + +static int amd_pmf_get_moving_avg(struct amd_pmf_dev *pdev, int socket_power) +{ + int i, total = 0; + + if (pdev->socket_power_history_idx == -1) { + for (i = 0; i < AVG_SAMPLE_SIZE; i++) + pdev->socket_power_history[i] = socket_power; + } + + pdev->socket_power_history_idx = (pdev->socket_power_history_idx + 1) % AVG_SAMPLE_SIZE; + pdev->socket_power_history[pdev->socket_power_history_idx] = socket_power; + + for (i = 0; i < AVG_SAMPLE_SIZE; i++) + total += pdev->socket_power_history[i]; + + return total / AVG_SAMPLE_SIZE; +} + +void amd_pmf_trans_automode(struct amd_pmf_dev *dev, int socket_power, ktime_t time_elapsed_ms) +{ + int avg_power = 0; + bool update = false; + int i, j; + + /* Get the average moving average computed by auto mode algorithm */ + avg_power = amd_pmf_get_moving_avg(dev, socket_power); + + for (i = 0; i < AUTO_TRANSITION_MAX; i++) { + if ((config_store.transition[i].shifting_up && avg_power >= + config_store.transition[i].power_threshold) || + (!config_store.transition[i].shifting_up && avg_power <= + config_store.transition[i].power_threshold)) { + if (config_store.transition[i].timer < + config_store.transition[i].time_constant) + config_store.transition[i].timer += time_elapsed_ms; + } else { + config_store.transition[i].timer = 0; + } + + if (config_store.transition[i].timer >= + config_store.transition[i].time_constant && + !config_store.transition[i].applied) { + config_store.transition[i].applied = true; + update = true; + } else if (config_store.transition[i].timer <= + config_store.transition[i].time_constant && + config_store.transition[i].applied) { + config_store.transition[i].applied = false; + update = true; + } + } + + dev_dbg(dev->dev, "[AUTO_MODE] avg power: %u mW mode: %s\n", avg_power, + state_as_str(config_store.current_mode)); + + if (update) { + for (j = 0; j < AUTO_TRANSITION_MAX; j++) { + /* Apply the mode with highest priority indentified */ + if (config_store.transition[j].applied) { + if (config_store.current_mode != + config_store.transition[j].target_mode) { + config_store.current_mode = + config_store.transition[j].target_mode; + dev_dbg(dev->dev, "[AUTO_MODE] moving to mode:%s\n", + state_as_str(config_store.current_mode)); + amd_pmf_set_automode(dev, config_store.current_mode, NULL); + } + break; + } + } + } +} + +void amd_pmf_update_2_cql(struct amd_pmf_dev *dev, bool is_cql_event) +{ + int mode = config_store.current_mode; + + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode = + is_cql_event ? AUTO_PERFORMANCE_ON_LAP : AUTO_PERFORMANCE; + + if ((mode == AUTO_PERFORMANCE || mode == AUTO_PERFORMANCE_ON_LAP) && + mode != config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode) { + mode = config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode; + amd_pmf_set_automode(dev, mode, NULL); + } + dev_dbg(dev->dev, "updated CQL thermals\n"); +} + +static void amd_pmf_get_power_threshold(void) +{ + config_store.transition[AUTO_TRANSITION_TO_QUIET].power_threshold = + config_store.mode_set[AUTO_BALANCE].power_floor - + config_store.transition[AUTO_TRANSITION_TO_QUIET].power_delta; + + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_threshold = + config_store.mode_set[AUTO_BALANCE].power_floor - + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_delta; + + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_threshold = + config_store.mode_set[AUTO_QUIET].power_floor - + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_delta; + + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_threshold = + config_store.mode_set[AUTO_PERFORMANCE].power_floor - + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_delta; +} + +static const char *state_as_str(unsigned int state) +{ + switch (state) { + case AUTO_QUIET: + return "QUIET"; + case AUTO_BALANCE: + return "BALANCED"; + case AUTO_PERFORMANCE_ON_LAP: + return "ON_LAP"; + case AUTO_PERFORMANCE: + return "PERFORMANCE"; + default: + return "Unknown Auto Mode State"; + } +} + +static void amd_pmf_load_defaults_auto_mode(struct amd_pmf_dev *dev) +{ + struct apmf_auto_mode output; + struct power_table_control *pwr_ctrl; + int i; + + apmf_get_auto_mode_def(dev, &output); + /* time constant */ + config_store.transition[AUTO_TRANSITION_TO_QUIET].time_constant = + output.balanced_to_quiet; + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].time_constant = + output.balanced_to_perf; + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].time_constant = + output.quiet_to_balanced; + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].time_constant = + output.perf_to_balanced; + + /* power floor */ + config_store.mode_set[AUTO_QUIET].power_floor = output.pfloor_quiet; + config_store.mode_set[AUTO_BALANCE].power_floor = output.pfloor_balanced; + config_store.mode_set[AUTO_PERFORMANCE].power_floor = output.pfloor_perf; + config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].power_floor = output.pfloor_perf; + + /* Power delta for mode change */ + config_store.transition[AUTO_TRANSITION_TO_QUIET].power_delta = + output.pd_balanced_to_quiet; + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_delta = + output.pd_balanced_to_perf; + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_delta = + output.pd_quiet_to_balanced; + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_delta = + output.pd_perf_to_balanced; + + /* Power threshold */ + amd_pmf_get_power_threshold(); + + /* skin temperature limits */ + pwr_ctrl = &config_store.mode_set[AUTO_QUIET].power_control; + pwr_ctrl->spl = output.spl_quiet; + pwr_ctrl->sppt = output.sppt_quiet; + pwr_ctrl->fppt = output.fppt_quiet; + pwr_ctrl->sppt_apu_only = output.sppt_apu_only_quiet; + pwr_ctrl->stt_min = output.stt_min_limit_quiet; + pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_quiet; + pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_quiet; + + pwr_ctrl = &config_store.mode_set[AUTO_BALANCE].power_control; + pwr_ctrl->spl = output.spl_balanced; + pwr_ctrl->sppt = output.sppt_balanced; + pwr_ctrl->fppt = output.fppt_balanced; + pwr_ctrl->sppt_apu_only = output.sppt_apu_only_balanced; + pwr_ctrl->stt_min = output.stt_min_limit_balanced; + pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_balanced; + pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_balanced; + + pwr_ctrl = &config_store.mode_set[AUTO_PERFORMANCE].power_control; + pwr_ctrl->spl = output.spl_perf; + pwr_ctrl->sppt = output.sppt_perf; + pwr_ctrl->fppt = output.fppt_perf; + pwr_ctrl->sppt_apu_only = output.sppt_apu_only_perf; + pwr_ctrl->stt_min = output.stt_min_limit_perf; + pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_perf; + pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_perf; + + pwr_ctrl = &config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].power_control; + pwr_ctrl->spl = output.spl_perf_on_lap; + pwr_ctrl->sppt = output.sppt_perf_on_lap; + pwr_ctrl->fppt = output.fppt_perf_on_lap; + pwr_ctrl->sppt_apu_only = output.sppt_apu_only_perf_on_lap; + pwr_ctrl->stt_min = output.stt_min_limit_perf_on_lap; + pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_perf_on_lap; + pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_perf_on_lap; + + /* Fan ID */ + config_store.mode_set[AUTO_QUIET].fan_control.fan_id = output.fan_id_quiet; + config_store.mode_set[AUTO_BALANCE].fan_control.fan_id = output.fan_id_balanced; + config_store.mode_set[AUTO_PERFORMANCE].fan_control.fan_id = output.fan_id_perf; + config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].fan_control.fan_id = + output.fan_id_perf; + + config_store.transition[AUTO_TRANSITION_TO_QUIET].target_mode = AUTO_QUIET; + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode = + AUTO_PERFORMANCE; + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].target_mode = + AUTO_BALANCE; + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].target_mode = + AUTO_BALANCE; + + config_store.transition[AUTO_TRANSITION_TO_QUIET].shifting_up = false; + config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].shifting_up = true; + config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].shifting_up = true; + config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].shifting_up = + false; + + for (i = 0 ; i < AUTO_MODE_MAX ; i++) { + if (config_store.mode_set[i].fan_control.fan_id == FAN_INDEX_AUTO) + config_store.mode_set[i].fan_control.manual = false; + else + config_store.mode_set[i].fan_control.manual = true; + } + + /* set to initial default values */ + config_store.current_mode = AUTO_BALANCE; + dev->socket_power_history_idx = -1; +} + +int amd_pmf_reset_amt(struct amd_pmf_dev *dev) +{ + /* + * OEM BIOS implementation guide says that if the auto mode is enabled + * the platform_profile registration shall be done by the OEM driver. + * There could be cases where both static slider and auto mode BIOS + * functions are enabled, in that case enable static slider updates + * only if it advertised as supported. + */ + + if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) { + int mode = amd_pmf_get_pprof_modes(dev); + + if (mode < 0) + return mode; + + dev_dbg(dev->dev, "resetting AMT thermals\n"); + amd_pmf_update_slider(dev, SLIDER_OP_SET, mode, NULL); + } + return 0; +} + +void amd_pmf_handle_amt(struct amd_pmf_dev *dev) +{ + amd_pmf_set_automode(dev, config_store.current_mode, NULL); +} + +void amd_pmf_deinit_auto_mode(struct amd_pmf_dev *dev) +{ + cancel_delayed_work_sync(&dev->work_buffer); +} + +void amd_pmf_init_auto_mode(struct amd_pmf_dev *dev) +{ + amd_pmf_load_defaults_auto_mode(dev); + /* update the thermal limits for Automode */ + amd_pmf_set_automode(dev, config_store.current_mode, NULL); + amd_pmf_init_metrics_table(dev); +} diff --git a/drivers/platform/x86/amd/pmf/cnqf.c b/drivers/platform/x86/amd/pmf/cnqf.c new file mode 100644 index 0000000000000000000000000000000000000000..668c7c0fea831cf1db6d902b3da228e24a9cfaef --- /dev/null +++ b/drivers/platform/x86/amd/pmf/cnqf.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Platform Management Framework Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#include +#include "pmf.h" + +static struct cnqf_config config_store; + +static int amd_pmf_set_cnqf(struct amd_pmf_dev *dev, int src, int idx, + struct cnqf_config *table) +{ + struct power_table_control *pc; + + pc = &config_store.mode_set[src][idx].power_control; + + amd_pmf_send_cmd(dev, SET_SPL, false, pc->spl, NULL); + amd_pmf_send_cmd(dev, SET_FPPT, false, pc->fppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT, false, pc->sppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false, pc->sppt_apu_only, NULL); + amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false, pc->stt_min, NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false, pc->stt_skin_temp[STT_TEMP_APU], + NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false, pc->stt_skin_temp[STT_TEMP_HS2], + NULL); + + if (is_apmf_func_supported(dev, APMF_FUNC_SET_FAN_IDX)) + apmf_update_fan_idx(dev, + config_store.mode_set[src][idx].fan_control.manual, + config_store.mode_set[src][idx].fan_control.fan_id); + + return 0; +} + +static void amd_pmf_update_power_threshold(int src) +{ + struct cnqf_mode_settings *ts; + struct cnqf_tran_params *tp; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_TO_QUIET]; + ts = &config_store.mode_set[src][CNQF_MODE_BALANCE]; + tp->power_threshold = ts->power_floor; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_TO_TURBO]; + ts = &config_store.mode_set[src][CNQF_MODE_PERFORMANCE]; + tp->power_threshold = ts->power_floor; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE]; + ts = &config_store.mode_set[src][CNQF_MODE_BALANCE]; + tp->power_threshold = ts->power_floor; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE]; + ts = &config_store.mode_set[src][CNQF_MODE_PERFORMANCE]; + tp->power_threshold = ts->power_floor; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_QUIET_TO_BALANCE]; + ts = &config_store.mode_set[src][CNQF_MODE_QUIET]; + tp->power_threshold = ts->power_floor; + + tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE]; + ts = &config_store.mode_set[src][CNQF_MODE_TURBO]; + tp->power_threshold = ts->power_floor; +} + +static const char *state_as_str(unsigned int state) +{ + switch (state) { + case CNQF_MODE_QUIET: + return "QUIET"; + case CNQF_MODE_BALANCE: + return "BALANCED"; + case CNQF_MODE_TURBO: + return "TURBO"; + case CNQF_MODE_PERFORMANCE: + return "PERFORMANCE"; + default: + return "Unknown CnQF mode"; + } +} + +static int amd_pmf_cnqf_get_power_source(struct amd_pmf_dev *dev) +{ + if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) && + is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) + return amd_pmf_get_power_source(); + else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) + return POWER_SOURCE_DC; + else + return POWER_SOURCE_AC; +} + +int amd_pmf_trans_cnqf(struct amd_pmf_dev *dev, int socket_power, ktime_t time_lapsed_ms) +{ + struct cnqf_tran_params *tp; + int src, i, j; + u32 avg_power = 0; + + src = amd_pmf_cnqf_get_power_source(dev); + + if (dev->current_profile == PLATFORM_PROFILE_BALANCED) { + amd_pmf_set_cnqf(dev, src, config_store.current_mode, NULL); + } else { + /* + * Return from here if the platform_profile is not balanced + * so that preference is given to user mode selection, rather + * than enforcing CnQF to run all the time (if enabled) + */ + return -EINVAL; + } + + for (i = 0; i < CNQF_TRANSITION_MAX; i++) { + config_store.trans_param[src][i].timer += time_lapsed_ms; + config_store.trans_param[src][i].total_power += socket_power; + config_store.trans_param[src][i].count++; + + tp = &config_store.trans_param[src][i]; + if (tp->timer >= tp->time_constant && tp->count) { + avg_power = tp->total_power / tp->count; + + /* Reset the indices */ + tp->timer = 0; + tp->total_power = 0; + tp->count = 0; + + if ((tp->shifting_up && avg_power >= tp->power_threshold) || + (!tp->shifting_up && avg_power <= tp->power_threshold)) { + tp->priority = true; + } else { + tp->priority = false; + } + } + } + + dev_dbg(dev->dev, "[CNQF] Avg power: %u mW socket power: %u mW mode:%s\n", + avg_power, socket_power, state_as_str(config_store.current_mode)); + + for (j = 0; j < CNQF_TRANSITION_MAX; j++) { + /* apply the highest priority */ + if (config_store.trans_param[src][j].priority) { + if (config_store.current_mode != + config_store.trans_param[src][j].target_mode) { + config_store.current_mode = + config_store.trans_param[src][j].target_mode; + dev_dbg(dev->dev, "Moving to Mode :%s\n", + state_as_str(config_store.current_mode)); + amd_pmf_set_cnqf(dev, src, + config_store.current_mode, NULL); + } + break; + } + } + return 0; +} + +static void amd_pmf_update_trans_data(int idx, struct apmf_dyn_slider_output out) +{ + struct cnqf_tran_params *tp; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_TO_QUIET]; + tp->time_constant = out.t_balanced_to_quiet; + tp->target_mode = CNQF_MODE_QUIET; + tp->shifting_up = false; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE]; + tp->time_constant = out.t_balanced_to_perf; + tp->target_mode = CNQF_MODE_PERFORMANCE; + tp->shifting_up = true; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_QUIET_TO_BALANCE]; + tp->time_constant = out.t_quiet_to_balanced; + tp->target_mode = CNQF_MODE_BALANCE; + tp->shifting_up = true; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE]; + tp->time_constant = out.t_perf_to_balanced; + tp->target_mode = CNQF_MODE_BALANCE; + tp->shifting_up = false; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE]; + tp->time_constant = out.t_turbo_to_perf; + tp->target_mode = CNQF_MODE_PERFORMANCE; + tp->shifting_up = false; + + tp = &config_store.trans_param[idx][CNQF_TRANSITION_TO_TURBO]; + tp->time_constant = out.t_perf_to_turbo; + tp->target_mode = CNQF_MODE_TURBO; + tp->shifting_up = true; +} + +static void amd_pmf_update_mode_set(int idx, struct apmf_dyn_slider_output out) +{ + struct cnqf_mode_settings *ms; + + /* Quiet Mode */ + ms = &config_store.mode_set[idx][CNQF_MODE_QUIET]; + ms->power_floor = out.ps[APMF_CNQF_QUIET].pfloor; + ms->power_control.fppt = out.ps[APMF_CNQF_QUIET].fppt; + ms->power_control.sppt = out.ps[APMF_CNQF_QUIET].sppt; + ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_QUIET].sppt_apu_only; + ms->power_control.spl = out.ps[APMF_CNQF_QUIET].spl; + ms->power_control.stt_min = out.ps[APMF_CNQF_QUIET].stt_min_limit; + ms->power_control.stt_skin_temp[STT_TEMP_APU] = + out.ps[APMF_CNQF_QUIET].stt_skintemp[STT_TEMP_APU]; + ms->power_control.stt_skin_temp[STT_TEMP_HS2] = + out.ps[APMF_CNQF_QUIET].stt_skintemp[STT_TEMP_HS2]; + ms->fan_control.fan_id = out.ps[APMF_CNQF_QUIET].fan_id; + + /* Balance Mode */ + ms = &config_store.mode_set[idx][CNQF_MODE_BALANCE]; + ms->power_floor = out.ps[APMF_CNQF_BALANCE].pfloor; + ms->power_control.fppt = out.ps[APMF_CNQF_BALANCE].fppt; + ms->power_control.sppt = out.ps[APMF_CNQF_BALANCE].sppt; + ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_BALANCE].sppt_apu_only; + ms->power_control.spl = out.ps[APMF_CNQF_BALANCE].spl; + ms->power_control.stt_min = out.ps[APMF_CNQF_BALANCE].stt_min_limit; + ms->power_control.stt_skin_temp[STT_TEMP_APU] = + out.ps[APMF_CNQF_BALANCE].stt_skintemp[STT_TEMP_APU]; + ms->power_control.stt_skin_temp[STT_TEMP_HS2] = + out.ps[APMF_CNQF_BALANCE].stt_skintemp[STT_TEMP_HS2]; + ms->fan_control.fan_id = out.ps[APMF_CNQF_BALANCE].fan_id; + + /* Performance Mode */ + ms = &config_store.mode_set[idx][CNQF_MODE_PERFORMANCE]; + ms->power_floor = out.ps[APMF_CNQF_PERFORMANCE].pfloor; + ms->power_control.fppt = out.ps[APMF_CNQF_PERFORMANCE].fppt; + ms->power_control.sppt = out.ps[APMF_CNQF_PERFORMANCE].sppt; + ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_PERFORMANCE].sppt_apu_only; + ms->power_control.spl = out.ps[APMF_CNQF_PERFORMANCE].spl; + ms->power_control.stt_min = out.ps[APMF_CNQF_PERFORMANCE].stt_min_limit; + ms->power_control.stt_skin_temp[STT_TEMP_APU] = + out.ps[APMF_CNQF_PERFORMANCE].stt_skintemp[STT_TEMP_APU]; + ms->power_control.stt_skin_temp[STT_TEMP_HS2] = + out.ps[APMF_CNQF_PERFORMANCE].stt_skintemp[STT_TEMP_HS2]; + ms->fan_control.fan_id = out.ps[APMF_CNQF_PERFORMANCE].fan_id; + + /* Turbo Mode */ + ms = &config_store.mode_set[idx][CNQF_MODE_TURBO]; + ms->power_floor = out.ps[APMF_CNQF_TURBO].pfloor; + ms->power_control.fppt = out.ps[APMF_CNQF_TURBO].fppt; + ms->power_control.sppt = out.ps[APMF_CNQF_TURBO].sppt; + ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_TURBO].sppt_apu_only; + ms->power_control.spl = out.ps[APMF_CNQF_TURBO].spl; + ms->power_control.stt_min = out.ps[APMF_CNQF_TURBO].stt_min_limit; + ms->power_control.stt_skin_temp[STT_TEMP_APU] = + out.ps[APMF_CNQF_TURBO].stt_skintemp[STT_TEMP_APU]; + ms->power_control.stt_skin_temp[STT_TEMP_HS2] = + out.ps[APMF_CNQF_TURBO].stt_skintemp[STT_TEMP_HS2]; + ms->fan_control.fan_id = out.ps[APMF_CNQF_TURBO].fan_id; +} + +static int amd_pmf_check_flags(struct amd_pmf_dev *dev) +{ + struct apmf_dyn_slider_output out = {}; + + if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC)) + apmf_get_dyn_slider_def_ac(dev, &out); + else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) + apmf_get_dyn_slider_def_dc(dev, &out); + + return out.flags; +} + +static int amd_pmf_load_defaults_cnqf(struct amd_pmf_dev *dev) +{ + struct apmf_dyn_slider_output out; + int i, j, ret; + + for (i = 0; i < POWER_SOURCE_MAX; i++) { + if (!is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC + i)) + continue; + + if (i == POWER_SOURCE_AC) + ret = apmf_get_dyn_slider_def_ac(dev, &out); + else + ret = apmf_get_dyn_slider_def_dc(dev, &out); + if (ret) { + dev_err(dev->dev, "APMF apmf_get_dyn_slider_def_dc failed :%d\n", ret); + return ret; + } + + amd_pmf_update_mode_set(i, out); + amd_pmf_update_trans_data(i, out); + amd_pmf_update_power_threshold(i); + + for (j = 0; j < CNQF_MODE_MAX; j++) { + if (config_store.mode_set[i][j].fan_control.fan_id == FAN_INDEX_AUTO) + config_store.mode_set[i][j].fan_control.manual = false; + else + config_store.mode_set[i][j].fan_control.manual = true; + } + } + + /* set to initial default values */ + config_store.current_mode = CNQF_MODE_BALANCE; + + return 0; +} + +static ssize_t cnqf_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + int mode, result, src; + bool input; + + mode = amd_pmf_get_pprof_modes(pdev); + if (mode < 0) + return mode; + + result = kstrtobool(buf, &input); + if (result) + return result; + + src = amd_pmf_cnqf_get_power_source(pdev); + pdev->cnqf_enabled = input; + + if (pdev->cnqf_enabled && pdev->current_profile == PLATFORM_PROFILE_BALANCED) { + amd_pmf_set_cnqf(pdev, src, config_store.current_mode, NULL); + } else { + if (is_apmf_func_supported(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) + amd_pmf_update_slider(pdev, SLIDER_OP_SET, mode, NULL); + } + + dev_dbg(pdev->dev, "Received CnQF %s\n", input ? "on" : "off"); + return count; +} + +static ssize_t cnqf_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%s\n", pdev->cnqf_enabled ? "on" : "off"); +} + +static DEVICE_ATTR_RW(cnqf_enable); + +static umode_t cnqf_feature_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct amd_pmf_dev *pdev = dev_get_drvdata(dev); + + return pdev->cnqf_supported ? attr->mode : 0; +} + +static struct attribute *cnqf_feature_attrs[] = { + &dev_attr_cnqf_enable.attr, + NULL +}; + +const struct attribute_group cnqf_feature_attribute_group = { + .is_visible = cnqf_feature_is_visible, + .attrs = cnqf_feature_attrs, +}; + +void amd_pmf_deinit_cnqf(struct amd_pmf_dev *dev) +{ + cancel_delayed_work_sync(&dev->work_buffer); +} + +int amd_pmf_init_cnqf(struct amd_pmf_dev *dev) +{ + int ret, src; + + /* + * Note the caller of this function has already checked that both + * APMF_FUNC_DYN_SLIDER_AC and APMF_FUNC_DYN_SLIDER_DC are supported. + */ + + ret = amd_pmf_load_defaults_cnqf(dev); + if (ret < 0) + return ret; + + amd_pmf_init_metrics_table(dev); + + dev->cnqf_supported = true; + dev->cnqf_enabled = amd_pmf_check_flags(dev); + + /* update the thermal for CnQF */ + if (dev->cnqf_enabled && dev->current_profile == PLATFORM_PROFILE_BALANCED) { + src = amd_pmf_cnqf_get_power_source(dev); + amd_pmf_set_cnqf(dev, src, config_store.current_mode, NULL); + } + + return 0; +} diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c new file mode 100644 index 0000000000000000000000000000000000000000..a5f5a4bcff6d9a50c8a21e30c96e623aa935f718 --- /dev/null +++ b/drivers/platform/x86/amd/pmf/core.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * AMD Platform Management Framework Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#include +#include +#include +#include +#include +#include +#include "pmf.h" + +/* PMF-SMU communication registers */ +#define AMD_PMF_REGISTER_MESSAGE 0xA18 +#define AMD_PMF_REGISTER_RESPONSE 0xA78 +#define AMD_PMF_REGISTER_ARGUMENT 0xA58 + +/* Base address of SMU for mapping physical address to virtual address */ +#define AMD_PMF_SMU_INDEX_ADDRESS 0xB8 +#define AMD_PMF_SMU_INDEX_DATA 0xBC +#define AMD_PMF_MAPPING_SIZE 0x01000 +#define AMD_PMF_BASE_ADDR_OFFSET 0x10000 +#define AMD_PMF_BASE_ADDR_LO 0x13B102E8 +#define AMD_PMF_BASE_ADDR_HI 0x13B102EC +#define AMD_PMF_BASE_ADDR_LO_MASK GENMASK(15, 0) +#define AMD_PMF_BASE_ADDR_HI_MASK GENMASK(31, 20) + +/* SMU Response Codes */ +#define AMD_PMF_RESULT_OK 0x01 +#define AMD_PMF_RESULT_CMD_REJECT_BUSY 0xFC +#define AMD_PMF_RESULT_CMD_REJECT_PREREQ 0xFD +#define AMD_PMF_RESULT_CMD_UNKNOWN 0xFE +#define AMD_PMF_RESULT_FAILED 0xFF + +/* List of supported CPU ids */ +#define AMD_CPU_ID_RMB 0x14b5 +#define AMD_CPU_ID_PS 0x14e8 + +#define PMF_MSG_DELAY_MIN_US 50 +#define RESPONSE_REGISTER_LOOP_MAX 20000 + +#define DELAY_MIN_US 2000 +#define DELAY_MAX_US 3000 + +/* override Metrics Table sample size time (in ms) */ +static int metrics_table_loop_ms = 1000; +module_param(metrics_table_loop_ms, int, 0644); +MODULE_PARM_DESC(metrics_table_loop_ms, "Metrics Table sample size time (default = 1000ms)"); + +/* Force load on supported older platforms */ +static bool force_load; +module_param(force_load, bool, 0444); +MODULE_PARM_DESC(force_load, "Force load this driver on supported older platforms (experimental)"); + +static int current_power_limits_show(struct seq_file *seq, void *unused) +{ + struct amd_pmf_dev *dev = seq->private; + struct amd_pmf_static_slider_granular table; + int mode, src = 0; + + mode = amd_pmf_get_pprof_modes(dev); + if (mode < 0) + return mode; + + src = amd_pmf_get_power_source(); + amd_pmf_update_slider(dev, SLIDER_OP_GET, mode, &table); + seq_printf(seq, "spl:%u fppt:%u sppt:%u sppt_apu_only:%u stt_min:%u stt[APU]:%u stt[HS2]: %u\n", + table.prop[src][mode].spl, + table.prop[src][mode].fppt, + table.prop[src][mode].sppt, + table.prop[src][mode].sppt_apu_only, + table.prop[src][mode].stt_min, + table.prop[src][mode].stt_skin_temp[STT_TEMP_APU], + table.prop[src][mode].stt_skin_temp[STT_TEMP_HS2]); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(current_power_limits); + +static void amd_pmf_dbgfs_unregister(struct amd_pmf_dev *dev) +{ + debugfs_remove_recursive(dev->dbgfs_dir); +} + +static void amd_pmf_dbgfs_register(struct amd_pmf_dev *dev) +{ + dev->dbgfs_dir = debugfs_create_dir("amd_pmf", NULL); + debugfs_create_file("current_power_limits", 0644, dev->dbgfs_dir, dev, + ¤t_power_limits_fops); +} + +int amd_pmf_get_power_source(void) +{ + if (power_supply_is_system_supplied() > 0) + return POWER_SOURCE_AC; + else + return POWER_SOURCE_DC; +} + +static void amd_pmf_get_metrics(struct work_struct *work) +{ + struct amd_pmf_dev *dev = container_of(work, struct amd_pmf_dev, work_buffer.work); + ktime_t time_elapsed_ms; + int socket_power; + + mutex_lock(&dev->update_mutex); + /* Transfer table contents */ + memset(dev->buf, 0, sizeof(dev->m_table)); + amd_pmf_send_cmd(dev, SET_TRANSFER_TABLE, 0, 7, NULL); + memcpy(&dev->m_table, dev->buf, sizeof(dev->m_table)); + + time_elapsed_ms = ktime_to_ms(ktime_get()) - dev->start_time; + /* Calculate the avg SoC power consumption */ + socket_power = dev->m_table.apu_power + dev->m_table.dgpu_power; + + if (dev->amt_enabled) { + /* Apply the Auto Mode transition */ + amd_pmf_trans_automode(dev, socket_power, time_elapsed_ms); + } + + if (dev->cnqf_enabled) { + /* Apply the CnQF transition */ + amd_pmf_trans_cnqf(dev, socket_power, time_elapsed_ms); + } + + dev->start_time = ktime_to_ms(ktime_get()); + schedule_delayed_work(&dev->work_buffer, msecs_to_jiffies(metrics_table_loop_ms)); + mutex_unlock(&dev->update_mutex); +} + +static inline u32 amd_pmf_reg_read(struct amd_pmf_dev *dev, int reg_offset) +{ + return ioread32(dev->regbase + reg_offset); +} + +static inline void amd_pmf_reg_write(struct amd_pmf_dev *dev, int reg_offset, u32 val) +{ + iowrite32(val, dev->regbase + reg_offset); +} + +static void __maybe_unused amd_pmf_dump_registers(struct amd_pmf_dev *dev) +{ + u32 value; + + value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_RESPONSE); + dev_dbg(dev->dev, "AMD_PMF_REGISTER_RESPONSE:%x\n", value); + + value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_ARGUMENT); + dev_dbg(dev->dev, "AMD_PMF_REGISTER_ARGUMENT:%d\n", value); + + value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_MESSAGE); + dev_dbg(dev->dev, "AMD_PMF_REGISTER_MESSAGE:%x\n", value); +} + +int amd_pmf_send_cmd(struct amd_pmf_dev *dev, u8 message, bool get, u32 arg, u32 *data) +{ + int rc; + u32 val; + + mutex_lock(&dev->lock); + + /* Wait until we get a valid response */ + rc = readx_poll_timeout(ioread32, dev->regbase + AMD_PMF_REGISTER_RESPONSE, + val, val != 0, PMF_MSG_DELAY_MIN_US, + PMF_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX); + if (rc) { + dev_err(dev->dev, "failed to talk to SMU\n"); + goto out_unlock; + } + + /* Write zero to response register */ + amd_pmf_reg_write(dev, AMD_PMF_REGISTER_RESPONSE, 0); + + /* Write argument into argument register */ + amd_pmf_reg_write(dev, AMD_PMF_REGISTER_ARGUMENT, arg); + + /* Write message ID to message ID register */ + amd_pmf_reg_write(dev, AMD_PMF_REGISTER_MESSAGE, message); + + /* Wait until we get a valid response */ + rc = readx_poll_timeout(ioread32, dev->regbase + AMD_PMF_REGISTER_RESPONSE, + val, val != 0, PMF_MSG_DELAY_MIN_US, + PMF_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX); + if (rc) { + dev_err(dev->dev, "SMU response timed out\n"); + goto out_unlock; + } + + switch (val) { + case AMD_PMF_RESULT_OK: + if (get) { + /* PMFW may take longer time to return back the data */ + usleep_range(DELAY_MIN_US, 10 * DELAY_MAX_US); + *data = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_ARGUMENT); + } + break; + case AMD_PMF_RESULT_CMD_REJECT_BUSY: + dev_err(dev->dev, "SMU not ready. err: 0x%x\n", val); + rc = -EBUSY; + goto out_unlock; + case AMD_PMF_RESULT_CMD_UNKNOWN: + dev_err(dev->dev, "SMU cmd unknown. err: 0x%x\n", val); + rc = -EINVAL; + goto out_unlock; + case AMD_PMF_RESULT_CMD_REJECT_PREREQ: + case AMD_PMF_RESULT_FAILED: + default: + dev_err(dev->dev, "SMU cmd failed. err: 0x%x\n", val); + rc = -EIO; + goto out_unlock; + } + +out_unlock: + mutex_unlock(&dev->lock); + amd_pmf_dump_registers(dev); + return rc; +} + +static const struct pci_device_id pmf_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RMB) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PS) }, + { } +}; + +int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev) +{ + u64 phys_addr; + u32 hi, low; + + INIT_DELAYED_WORK(&dev->work_buffer, amd_pmf_get_metrics); + + /* Get Metrics Table Address */ + dev->buf = kzalloc(sizeof(dev->m_table), GFP_KERNEL); + if (!dev->buf) + return -ENOMEM; + + phys_addr = virt_to_phys(dev->buf); + hi = phys_addr >> 32; + low = phys_addr & GENMASK(31, 0); + + amd_pmf_send_cmd(dev, SET_DRAM_ADDR_HIGH, 0, hi, NULL); + amd_pmf_send_cmd(dev, SET_DRAM_ADDR_LOW, 0, low, NULL); + + /* + * Start collecting the metrics data after a small delay + * or else, we might end up getting stale values from PMFW. + */ + schedule_delayed_work(&dev->work_buffer, msecs_to_jiffies(metrics_table_loop_ms * 3)); + + return 0; +} + +static void amd_pmf_init_features(struct amd_pmf_dev *dev) +{ + int ret; + + /* Enable Static Slider */ + if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) { + amd_pmf_init_sps(dev); + dev_dbg(dev->dev, "SPS enabled and Platform Profiles registered\n"); + } + + /* Enable Auto Mode */ + if (is_apmf_func_supported(dev, APMF_FUNC_AUTO_MODE)) { + amd_pmf_init_auto_mode(dev); + dev_dbg(dev->dev, "Auto Mode Init done\n"); + } else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) || + is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) { + /* Enable Cool n Quiet Framework (CnQF) */ + ret = amd_pmf_init_cnqf(dev); + if (ret) + dev_warn(dev->dev, "CnQF Init failed\n"); + } +} + +static void amd_pmf_deinit_features(struct amd_pmf_dev *dev) +{ + if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) + amd_pmf_deinit_sps(dev); + + if (is_apmf_func_supported(dev, APMF_FUNC_AUTO_MODE)) { + amd_pmf_deinit_auto_mode(dev); + } else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) || + is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) { + amd_pmf_deinit_cnqf(dev); + } +} + +static const struct acpi_device_id amd_pmf_acpi_ids[] = { + {"AMDI0100", 0x100}, + {"AMDI0102", 0}, + { } +}; +MODULE_DEVICE_TABLE(acpi, amd_pmf_acpi_ids); + +static int amd_pmf_probe(struct platform_device *pdev) +{ + const struct acpi_device_id *id; + struct amd_pmf_dev *dev; + struct pci_dev *rdev; + u32 base_addr_lo; + u32 base_addr_hi; + u64 base_addr; + u32 val; + int err; + + id = acpi_match_device(amd_pmf_acpi_ids, &pdev->dev); + if (!id) + return -ENODEV; + + if (id->driver_data == 0x100 && !force_load) + return -ENODEV; + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->dev = &pdev->dev; + + rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0)); + if (!rdev || !pci_match_id(pmf_pci_ids, rdev)) { + pci_dev_put(rdev); + return -ENODEV; + } + + dev->cpu_id = rdev->device; + err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_LO); + if (err) { + dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS); + pci_dev_put(rdev); + return pcibios_err_to_errno(err); + } + + err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val); + if (err) { + pci_dev_put(rdev); + return pcibios_err_to_errno(err); + } + + base_addr_lo = val & AMD_PMF_BASE_ADDR_HI_MASK; + + err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_HI); + if (err) { + dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS); + pci_dev_put(rdev); + return pcibios_err_to_errno(err); + } + + err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val); + if (err) { + pci_dev_put(rdev); + return pcibios_err_to_errno(err); + } + + base_addr_hi = val & AMD_PMF_BASE_ADDR_LO_MASK; + pci_dev_put(rdev); + base_addr = ((u64)base_addr_hi << 32 | base_addr_lo); + + dev->regbase = devm_ioremap(dev->dev, base_addr + AMD_PMF_BASE_ADDR_OFFSET, + AMD_PMF_MAPPING_SIZE); + if (!dev->regbase) + return -ENOMEM; + + apmf_acpi_init(dev); + platform_set_drvdata(pdev, dev); + amd_pmf_init_features(dev); + apmf_install_handler(dev); + amd_pmf_dbgfs_register(dev); + + mutex_init(&dev->lock); + mutex_init(&dev->update_mutex); + dev_info(dev->dev, "registered PMF device successfully\n"); + + return 0; +} + +static int amd_pmf_remove(struct platform_device *pdev) +{ + struct amd_pmf_dev *dev = platform_get_drvdata(pdev); + + mutex_destroy(&dev->lock); + mutex_destroy(&dev->update_mutex); + amd_pmf_deinit_features(dev); + apmf_acpi_deinit(dev); + amd_pmf_dbgfs_unregister(dev); + kfree(dev->buf); + return 0; +} + +static const struct attribute_group *amd_pmf_driver_groups[] = { + &cnqf_feature_attribute_group, + NULL, +}; + +static struct platform_driver amd_pmf_driver = { + .driver = { + .name = "amd-pmf", + .acpi_match_table = amd_pmf_acpi_ids, + .dev_groups = amd_pmf_driver_groups, + }, + .probe = amd_pmf_probe, + .remove = amd_pmf_remove, +}; +module_platform_driver(amd_pmf_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD Platform Management Framework Driver"); diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h new file mode 100644 index 0000000000000000000000000000000000000000..84bbe2c6ea612b88ab9aea13e3d139f46f98e670 --- /dev/null +++ b/drivers/platform/x86/amd/pmf/pmf.h @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD Platform Management Framework Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#ifndef PMF_H +#define PMF_H + +#include +#include + +/* APMF Functions */ +#define APMF_FUNC_VERIFY_INTERFACE 0 +#define APMF_FUNC_GET_SYS_PARAMS 1 +#define APMF_FUNC_SBIOS_REQUESTS 2 +#define APMF_FUNC_SBIOS_HEARTBEAT 4 +#define APMF_FUNC_AUTO_MODE 5 +#define APMF_FUNC_SET_FAN_IDX 7 +#define APMF_FUNC_STATIC_SLIDER_GRANULAR 9 +#define APMF_FUNC_DYN_SLIDER_AC 11 +#define APMF_FUNC_DYN_SLIDER_DC 12 + +/* Message Definitions */ +#define SET_SPL 0x03 /* SPL: Sustained Power Limit */ +#define SET_SPPT 0x05 /* SPPT: Slow Package Power Tracking */ +#define SET_FPPT 0x07 /* FPPT: Fast Package Power Tracking */ +#define GET_SPL 0x0B +#define GET_SPPT 0x0D +#define GET_FPPT 0x0F +#define SET_DRAM_ADDR_HIGH 0x14 +#define SET_DRAM_ADDR_LOW 0x15 +#define SET_TRANSFER_TABLE 0x16 +#define SET_STT_MIN_LIMIT 0x18 /* STT: Skin Temperature Tracking */ +#define SET_STT_LIMIT_APU 0x19 +#define SET_STT_LIMIT_HS2 0x1A +#define SET_SPPT_APU_ONLY 0x1D +#define GET_SPPT_APU_ONLY 0x1E +#define GET_STT_MIN_LIMIT 0x1F +#define GET_STT_LIMIT_APU 0x20 +#define GET_STT_LIMIT_HS2 0x21 + +/* Fan Index for Auto Mode */ +#define FAN_INDEX_AUTO 0xFFFFFFFF + +#define ARG_NONE 0 +#define AVG_SAMPLE_SIZE 3 + +/* AMD PMF BIOS interfaces */ +struct apmf_verify_interface { + u16 size; + u16 version; + u32 notification_mask; + u32 supported_functions; +} __packed; + +struct apmf_system_params { + u16 size; + u32 valid_mask; + u32 flags; + u8 command_code; + u32 heartbeat_int; +} __packed; + +struct apmf_sbios_req { + u16 size; + u32 pending_req; + u8 rsd; + u8 cql_event; + u8 amt_event; + u32 fppt; + u32 sppt; + u32 fppt_apu_only; + u32 spl; + u32 stt_min_limit; + u8 skin_temp_apu; + u8 skin_temp_hs2; +} __packed; + +struct apmf_fan_idx { + u16 size; + u8 fan_ctl_mode; + u32 fan_ctl_idx; +} __packed; + +struct smu_pmf_metrics { + u16 gfxclk_freq; /* in MHz */ + u16 socclk_freq; /* in MHz */ + u16 vclk_freq; /* in MHz */ + u16 dclk_freq; /* in MHz */ + u16 memclk_freq; /* in MHz */ + u16 spare; + u16 gfx_activity; /* in Centi */ + u16 uvd_activity; /* in Centi */ + u16 voltage[2]; /* in mV */ + u16 currents[2]; /* in mA */ + u16 power[2];/* in mW */ + u16 core_freq[8]; /* in MHz */ + u16 core_power[8]; /* in mW */ + u16 core_temp[8]; /* in centi-Celsius */ + u16 l3_freq; /* in MHz */ + u16 l3_temp; /* in centi-Celsius */ + u16 gfx_temp; /* in centi-Celsius */ + u16 soc_temp; /* in centi-Celsius */ + u16 throttler_status; + u16 current_socketpower; /* in mW */ + u16 stapm_orig_limit; /* in W */ + u16 stapm_cur_limit; /* in W */ + u32 apu_power; /* in mW */ + u32 dgpu_power; /* in mW */ + u16 vdd_tdc_val; /* in mA */ + u16 soc_tdc_val; /* in mA */ + u16 vdd_edc_val; /* in mA */ + u16 soc_edcv_al; /* in mA */ + u16 infra_cpu_maxfreq; /* in MHz */ + u16 infra_gfx_maxfreq; /* in MHz */ + u16 skin_temp; /* in centi-Celsius */ + u16 device_state; +} __packed; + +enum amd_stt_skin_temp { + STT_TEMP_APU, + STT_TEMP_HS2, + STT_TEMP_COUNT, +}; + +enum amd_slider_op { + SLIDER_OP_GET, + SLIDER_OP_SET, +}; + +enum power_source { + POWER_SOURCE_AC, + POWER_SOURCE_DC, + POWER_SOURCE_MAX, +}; + +enum power_modes { + POWER_MODE_PERFORMANCE, + POWER_MODE_BALANCED_POWER, + POWER_MODE_POWER_SAVER, + POWER_MODE_MAX, +}; + +struct amd_pmf_dev { + void __iomem *regbase; + void __iomem *smu_virt_addr; + void *buf; + u32 base_addr; + u32 cpu_id; + struct device *dev; + struct mutex lock; /* protects the PMF interface */ + u32 supported_func; + enum platform_profile_option current_profile; + struct platform_profile_handler pprof; + struct dentry *dbgfs_dir; + int hb_interval; /* SBIOS heartbeat interval */ + struct delayed_work heart_beat; + struct smu_pmf_metrics m_table; + struct delayed_work work_buffer; + ktime_t start_time; + int socket_power_history[AVG_SAMPLE_SIZE]; + int socket_power_history_idx; + bool amt_enabled; + struct mutex update_mutex; /* protects race between ACPI handler and metrics thread */ + bool cnqf_enabled; + bool cnqf_supported; +}; + +struct apmf_sps_prop_granular { + u32 fppt; + u32 sppt; + u32 sppt_apu_only; + u32 spl; + u32 stt_min; + u8 stt_skin_temp[STT_TEMP_COUNT]; + u32 fan_id; +} __packed; + +/* Static Slider */ +struct apmf_static_slider_granular_output { + u16 size; + struct apmf_sps_prop_granular prop[POWER_SOURCE_MAX * POWER_MODE_MAX]; +} __packed; + +struct amd_pmf_static_slider_granular { + u16 size; + struct apmf_sps_prop_granular prop[POWER_SOURCE_MAX][POWER_MODE_MAX]; +}; + +struct fan_table_control { + bool manual; + unsigned long fan_id; +}; + +struct power_table_control { + u32 spl; + u32 sppt; + u32 fppt; + u32 sppt_apu_only; + u32 stt_min; + u32 stt_skin_temp[STT_TEMP_COUNT]; + u32 reserved[16]; +}; + +/* Auto Mode Layer */ +enum auto_mode_transition_priority { + AUTO_TRANSITION_TO_PERFORMANCE, /* Any other mode to Performance Mode */ + AUTO_TRANSITION_FROM_QUIET_TO_BALANCE, /* Quiet Mode to Balance Mode */ + AUTO_TRANSITION_TO_QUIET, /* Any other mode to Quiet Mode */ + AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE, /* Performance Mode to Balance Mode */ + AUTO_TRANSITION_MAX, +}; + +enum auto_mode_mode { + AUTO_QUIET, + AUTO_BALANCE, + AUTO_PERFORMANCE_ON_LAP, + AUTO_PERFORMANCE, + AUTO_MODE_MAX, +}; + +struct auto_mode_trans_params { + u32 time_constant; /* minimum time required to switch to next mode */ + u32 power_delta; /* delta power to shift mode */ + u32 power_threshold; + u32 timer; /* elapsed time. if timer > TimeThreshold, it will move to next mode */ + u32 applied; + enum auto_mode_mode target_mode; + u32 shifting_up; +}; + +struct auto_mode_mode_settings { + struct power_table_control power_control; + struct fan_table_control fan_control; + u32 power_floor; +}; + +struct auto_mode_mode_config { + struct auto_mode_trans_params transition[AUTO_TRANSITION_MAX]; + struct auto_mode_mode_settings mode_set[AUTO_MODE_MAX]; + enum auto_mode_mode current_mode; +}; + +struct apmf_auto_mode { + u16 size; + /* time constant */ + u32 balanced_to_perf; + u32 perf_to_balanced; + u32 quiet_to_balanced; + u32 balanced_to_quiet; + /* power floor */ + u32 pfloor_perf; + u32 pfloor_balanced; + u32 pfloor_quiet; + /* Power delta for mode change */ + u32 pd_balanced_to_perf; + u32 pd_perf_to_balanced; + u32 pd_quiet_to_balanced; + u32 pd_balanced_to_quiet; + /* skin temperature limits */ + u8 stt_apu_perf_on_lap; /* CQL ON */ + u8 stt_hs2_perf_on_lap; /* CQL ON */ + u8 stt_apu_perf; + u8 stt_hs2_perf; + u8 stt_apu_balanced; + u8 stt_hs2_balanced; + u8 stt_apu_quiet; + u8 stt_hs2_quiet; + u32 stt_min_limit_perf_on_lap; /* CQL ON */ + u32 stt_min_limit_perf; + u32 stt_min_limit_balanced; + u32 stt_min_limit_quiet; + /* SPL based */ + u32 fppt_perf_on_lap; /* CQL ON */ + u32 sppt_perf_on_lap; /* CQL ON */ + u32 spl_perf_on_lap; /* CQL ON */ + u32 sppt_apu_only_perf_on_lap; /* CQL ON */ + u32 fppt_perf; + u32 sppt_perf; + u32 spl_perf; + u32 sppt_apu_only_perf; + u32 fppt_balanced; + u32 sppt_balanced; + u32 spl_balanced; + u32 sppt_apu_only_balanced; + u32 fppt_quiet; + u32 sppt_quiet; + u32 spl_quiet; + u32 sppt_apu_only_quiet; + /* Fan ID */ + u32 fan_id_perf; + u32 fan_id_balanced; + u32 fan_id_quiet; +} __packed; + +/* CnQF Layer */ +enum cnqf_trans_priority { + CNQF_TRANSITION_TO_TURBO, /* Any other mode to Turbo Mode */ + CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE, /* quiet/balance to Performance Mode */ + CNQF_TRANSITION_FROM_QUIET_TO_BALANCE, /* Quiet Mode to Balance Mode */ + CNQF_TRANSITION_TO_QUIET, /* Any other mode to Quiet Mode */ + CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE, /* Performance/Turbo to Balance Mode */ + CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE, /* Turbo mode to Performance Mode */ + CNQF_TRANSITION_MAX, +}; + +enum cnqf_mode { + CNQF_MODE_QUIET, + CNQF_MODE_BALANCE, + CNQF_MODE_PERFORMANCE, + CNQF_MODE_TURBO, + CNQF_MODE_MAX, +}; + +enum apmf_cnqf_pos { + APMF_CNQF_TURBO, + APMF_CNQF_PERFORMANCE, + APMF_CNQF_BALANCE, + APMF_CNQF_QUIET, + APMF_CNQF_MAX, +}; + +struct cnqf_mode_settings { + struct power_table_control power_control; + struct fan_table_control fan_control; + u32 power_floor; +}; + +struct cnqf_tran_params { + u32 time_constant; /* minimum time required to switch to next mode */ + u32 power_threshold; + u32 timer; /* elapsed time. if timer > timethreshold, it will move to next mode */ + u32 total_power; + u32 count; + bool priority; + bool shifting_up; + enum cnqf_mode target_mode; +}; + +struct cnqf_config { + struct cnqf_tran_params trans_param[POWER_SOURCE_MAX][CNQF_TRANSITION_MAX]; + struct cnqf_mode_settings mode_set[POWER_SOURCE_MAX][CNQF_MODE_MAX]; + struct power_table_control defaults; + enum cnqf_mode current_mode; + u32 power_src; + u32 avg_power; +}; + +struct apmf_cnqf_power_set { + u32 pfloor; + u32 fppt; + u32 sppt; + u32 sppt_apu_only; + u32 spl; + u32 stt_min_limit; + u8 stt_skintemp[STT_TEMP_COUNT]; + u32 fan_id; +} __packed; + +struct apmf_dyn_slider_output { + u16 size; + u16 flags; + u32 t_perf_to_turbo; + u32 t_balanced_to_perf; + u32 t_quiet_to_balanced; + u32 t_balanced_to_quiet; + u32 t_perf_to_balanced; + u32 t_turbo_to_perf; + struct apmf_cnqf_power_set ps[APMF_CNQF_MAX]; +} __packed; + +/* Core Layer */ +int apmf_acpi_init(struct amd_pmf_dev *pmf_dev); +void apmf_acpi_deinit(struct amd_pmf_dev *pmf_dev); +int is_apmf_func_supported(struct amd_pmf_dev *pdev, unsigned long index); +int amd_pmf_send_cmd(struct amd_pmf_dev *dev, u8 message, bool get, u32 arg, u32 *data); +int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev); +int amd_pmf_get_power_source(void); +int apmf_install_handler(struct amd_pmf_dev *pmf_dev); + +/* SPS Layer */ +int amd_pmf_get_pprof_modes(struct amd_pmf_dev *pmf); +void amd_pmf_update_slider(struct amd_pmf_dev *dev, bool op, int idx, + struct amd_pmf_static_slider_granular *table); +int amd_pmf_init_sps(struct amd_pmf_dev *dev); +void amd_pmf_deinit_sps(struct amd_pmf_dev *dev); +int apmf_get_static_slider_granular(struct amd_pmf_dev *pdev, + struct apmf_static_slider_granular_output *output); + + +int apmf_update_fan_idx(struct amd_pmf_dev *pdev, bool manual, u32 idx); + +/* Auto Mode Layer */ +int apmf_get_auto_mode_def(struct amd_pmf_dev *pdev, struct apmf_auto_mode *data); +void amd_pmf_init_auto_mode(struct amd_pmf_dev *dev); +void amd_pmf_deinit_auto_mode(struct amd_pmf_dev *dev); +void amd_pmf_trans_automode(struct amd_pmf_dev *dev, int socket_power, ktime_t time_elapsed_ms); +int apmf_get_sbios_requests(struct amd_pmf_dev *pdev, struct apmf_sbios_req *req); + +void amd_pmf_update_2_cql(struct amd_pmf_dev *dev, bool is_cql_event); +int amd_pmf_reset_amt(struct amd_pmf_dev *dev); +void amd_pmf_handle_amt(struct amd_pmf_dev *dev); + +/* CnQF Layer */ +int apmf_get_dyn_slider_def_ac(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data); +int apmf_get_dyn_slider_def_dc(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data); +int amd_pmf_init_cnqf(struct amd_pmf_dev *dev); +void amd_pmf_deinit_cnqf(struct amd_pmf_dev *dev); +int amd_pmf_trans_cnqf(struct amd_pmf_dev *dev, int socket_power, ktime_t time_lapsed_ms); +extern const struct attribute_group cnqf_feature_attribute_group; + +#endif /* PMF_H */ diff --git a/drivers/platform/x86/amd/pmf/sps.c b/drivers/platform/x86/amd/pmf/sps.c new file mode 100644 index 0000000000000000000000000000000000000000..dba7e36962dc1eadd034dcd8e61d9ebfe51fd93f --- /dev/null +++ b/drivers/platform/x86/amd/pmf/sps.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD Platform Management Framework (PMF) Driver + * + * Copyright (c) 2022, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Shyam Sundar S K + */ + +#include "pmf.h" + +static struct amd_pmf_static_slider_granular config_store; + +static void amd_pmf_load_defaults_sps(struct amd_pmf_dev *dev) +{ + struct apmf_static_slider_granular_output output; + int i, j, idx = 0; + + memset(&config_store, 0, sizeof(config_store)); + apmf_get_static_slider_granular(dev, &output); + + for (i = 0; i < POWER_SOURCE_MAX; i++) { + for (j = 0; j < POWER_MODE_MAX; j++) { + config_store.prop[i][j].spl = output.prop[idx].spl; + config_store.prop[i][j].sppt = output.prop[idx].sppt; + config_store.prop[i][j].sppt_apu_only = + output.prop[idx].sppt_apu_only; + config_store.prop[i][j].fppt = output.prop[idx].fppt; + config_store.prop[i][j].stt_min = output.prop[idx].stt_min; + config_store.prop[i][j].stt_skin_temp[STT_TEMP_APU] = + output.prop[idx].stt_skin_temp[STT_TEMP_APU]; + config_store.prop[i][j].stt_skin_temp[STT_TEMP_HS2] = + output.prop[idx].stt_skin_temp[STT_TEMP_HS2]; + config_store.prop[i][j].fan_id = output.prop[idx].fan_id; + idx++; + } + } +} + +void amd_pmf_update_slider(struct amd_pmf_dev *dev, bool op, int idx, + struct amd_pmf_static_slider_granular *table) +{ + int src = amd_pmf_get_power_source(); + + if (op == SLIDER_OP_SET) { + amd_pmf_send_cmd(dev, SET_SPL, false, config_store.prop[src][idx].spl, NULL); + amd_pmf_send_cmd(dev, SET_FPPT, false, config_store.prop[src][idx].fppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT, false, config_store.prop[src][idx].sppt, NULL); + amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false, + config_store.prop[src][idx].sppt_apu_only, NULL); + amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false, + config_store.prop[src][idx].stt_min, NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false, + config_store.prop[src][idx].stt_skin_temp[STT_TEMP_APU], NULL); + amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false, + config_store.prop[src][idx].stt_skin_temp[STT_TEMP_HS2], NULL); + } else if (op == SLIDER_OP_GET) { + amd_pmf_send_cmd(dev, GET_SPL, true, ARG_NONE, &table->prop[src][idx].spl); + amd_pmf_send_cmd(dev, GET_FPPT, true, ARG_NONE, &table->prop[src][idx].fppt); + amd_pmf_send_cmd(dev, GET_SPPT, true, ARG_NONE, &table->prop[src][idx].sppt); + amd_pmf_send_cmd(dev, GET_SPPT_APU_ONLY, true, ARG_NONE, + &table->prop[src][idx].sppt_apu_only); + amd_pmf_send_cmd(dev, GET_STT_MIN_LIMIT, true, ARG_NONE, + &table->prop[src][idx].stt_min); + amd_pmf_send_cmd(dev, GET_STT_LIMIT_APU, true, ARG_NONE, + (u32 *)&table->prop[src][idx].stt_skin_temp[STT_TEMP_APU]); + amd_pmf_send_cmd(dev, GET_STT_LIMIT_HS2, true, ARG_NONE, + (u32 *)&table->prop[src][idx].stt_skin_temp[STT_TEMP_HS2]); + } +} + +static int amd_pmf_profile_get(struct platform_profile_handler *pprof, + enum platform_profile_option *profile) +{ + struct amd_pmf_dev *pmf = container_of(pprof, struct amd_pmf_dev, pprof); + + *profile = pmf->current_profile; + return 0; +} + +int amd_pmf_get_pprof_modes(struct amd_pmf_dev *pmf) +{ + int mode; + + switch (pmf->current_profile) { + case PLATFORM_PROFILE_PERFORMANCE: + mode = POWER_MODE_PERFORMANCE; + break; + case PLATFORM_PROFILE_BALANCED: + mode = POWER_MODE_BALANCED_POWER; + break; + case PLATFORM_PROFILE_LOW_POWER: + mode = POWER_MODE_POWER_SAVER; + break; + default: + dev_err(pmf->dev, "Unknown Platform Profile.\n"); + return -EOPNOTSUPP; + } + + return mode; +} + +static int amd_pmf_profile_set(struct platform_profile_handler *pprof, + enum platform_profile_option profile) +{ + struct amd_pmf_dev *pmf = container_of(pprof, struct amd_pmf_dev, pprof); + int mode; + + pmf->current_profile = profile; + mode = amd_pmf_get_pprof_modes(pmf); + if (mode < 0) + return mode; + + amd_pmf_update_slider(pmf, SLIDER_OP_SET, mode, NULL); + return 0; +} + +int amd_pmf_init_sps(struct amd_pmf_dev *dev) +{ + int err; + + dev->current_profile = PLATFORM_PROFILE_BALANCED; + amd_pmf_load_defaults_sps(dev); + + dev->pprof.profile_get = amd_pmf_profile_get; + dev->pprof.profile_set = amd_pmf_profile_set; + + /* Setup supported modes */ + set_bit(PLATFORM_PROFILE_LOW_POWER, dev->pprof.choices); + set_bit(PLATFORM_PROFILE_BALANCED, dev->pprof.choices); + set_bit(PLATFORM_PROFILE_PERFORMANCE, dev->pprof.choices); + + /* Create platform_profile structure and register */ + err = platform_profile_register(&dev->pprof); + if (err) + dev_err(dev->dev, "Failed to register SPS support, this is most likely an SBIOS bug: %d\n", + err); + + return err; +} + +void amd_pmf_deinit_sps(struct amd_pmf_dev *dev) +{ + platform_profile_remove(); +} diff --git a/drivers/platform/x86/amilo-rfkill.c b/drivers/platform/x86/amilo-rfkill.c index 493e169c8f61510c1bf6c98810cd423f0bf2c2aa..3e313c4d538d3565f6af1f7041be439d8a923e2a 100644 --- a/drivers/platform/x86/amilo-rfkill.c +++ b/drivers/platform/x86/amilo-rfkill.c @@ -150,7 +150,8 @@ static int __init amilo_rfkill_init(void) if (rc) return rc; - amilo_rfkill_pdev = platform_device_register_simple(KBUILD_MODNAME, -1, + amilo_rfkill_pdev = platform_device_register_simple(KBUILD_MODNAME, + PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(amilo_rfkill_pdev)) { rc = PTR_ERR(amilo_rfkill_pdev); diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c index 4d2d32bfbe2a6da7bbff43e3e4047fea3b715c64..47b2f8bb6fb53f74875ada2afef4b91cb33ebf60 100644 --- a/drivers/platform/x86/asus-laptop.c +++ b/drivers/platform/x86/asus-laptop.c @@ -1633,7 +1633,7 @@ static int asus_platform_init(struct asus_laptop *asus) { int result; - asus->platform_device = platform_device_alloc(ASUS_LAPTOP_FILE, -1); + asus->platform_device = platform_device_alloc(ASUS_LAPTOP_FILE, PLATFORM_DEVID_NONE); if (!asus->platform_device) return -ENOMEM; platform_set_drvdata(asus->platform_device, asus); diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index bbfed85051ee75c4ff0bad54fcae537bdfc38260..613c45c9fbe3693b21a627431ca42c2a56e81e05 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -43,7 +43,7 @@ MODULE_PARM_DESC(wapf, "WAPF value"); static int tablet_mode_sw = -1; module_param(tablet_mode_sw, uint, 0444); -MODULE_PARM_DESC(tablet_mode_sw, "Tablet mode detect: -1:auto 0:disable 1:kbd-dock 2:lid-flip"); +MODULE_PARM_DESC(tablet_mode_sw, "Tablet mode detect: -1:auto 0:disable 1:kbd-dock 2:lid-flip 3:lid-flip-rog"); static struct quirk_entry *quirks; @@ -108,12 +108,17 @@ static struct quirk_entry quirk_asus_forceals = { }; static struct quirk_entry quirk_asus_use_kbd_dock_devid = { - .use_kbd_dock_devid = true, + .tablet_switch_mode = asus_wmi_kbd_dock_devid, }; static struct quirk_entry quirk_asus_use_lid_flip_devid = { .wmi_backlight_set_devstate = true, - .use_lid_flip_devid = true, + .tablet_switch_mode = asus_wmi_lid_flip_devid, +}; + +static struct quirk_entry quirk_asus_tablet_mode = { + .wmi_backlight_set_devstate = true, + .tablet_switch_mode = asus_wmi_lid_flip_rog_devid, }; static int dmi_matched(const struct dmi_system_id *dmi) @@ -450,6 +455,15 @@ static const struct dmi_system_id asus_quirks[] = { }, .driver_data = &quirk_asus_use_lid_flip_devid, }, + { + .callback = dmi_matched, + .ident = "ASUS ROG FLOW X13", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "GV301Q"), + }, + .driver_data = &quirk_asus_tablet_mode, + }, {}, }; @@ -469,20 +483,8 @@ static void asus_nb_wmi_quirks(struct asus_wmi_driver *driver) else wapf = quirks->wapf; - switch (tablet_mode_sw) { - case 0: - quirks->use_kbd_dock_devid = false; - quirks->use_lid_flip_devid = false; - break; - case 1: - quirks->use_kbd_dock_devid = true; - quirks->use_lid_flip_devid = false; - break; - case 2: - quirks->use_kbd_dock_devid = false; - quirks->use_lid_flip_devid = true; - break; - } + if (tablet_mode_sw != -1) + quirks->tablet_switch_mode = tablet_mode_sw; if (quirks->i8042_filter) { ret = i8042_install_filter(quirks->i8042_filter); @@ -554,12 +556,14 @@ static const struct key_entry asus_nb_wmi_keymap[] = { { KE_KEY, 0xA5, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + TV + HDMI */ { KE_KEY, 0xA6, { KEY_SWITCHVIDEOMODE } }, /* SDSP CRT + TV + HDMI */ { KE_KEY, 0xA7, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + CRT + TV + HDMI */ + { KE_KEY, 0xAE, { KEY_FN_F5 } }, /* Fn+F5 fan mode on 2020+ */ { KE_KEY, 0xB3, { KEY_PROG4 } }, /* AURA */ { KE_KEY, 0xB5, { KEY_CALC } }, { KE_KEY, 0xC4, { KEY_KBDILLUMUP } }, { KE_KEY, 0xC5, { KEY_KBDILLUMDOWN } }, { KE_IGNORE, 0xC6, }, /* Ambient Light Sensor notification */ { KE_KEY, 0xFA, { KEY_PROG2 } }, /* Lid flip action */ + { KE_KEY, 0xBD, { KEY_PROG2 } }, /* Lid flip action on ROG xflow laptops */ { KE_END, 0}, }; diff --git a/drivers/platform/x86/asus-tf103c-dock.c b/drivers/platform/x86/asus-tf103c-dock.c index 6fd0c9fea82da144f29b5566cfcc67893e4f06c5..62310e06282beace61d839a78293e93eed7df210 100644 --- a/drivers/platform/x86/asus-tf103c-dock.c +++ b/drivers/platform/x86/asus-tf103c-dock.c @@ -878,14 +878,12 @@ static int tf103c_dock_probe(struct i2c_client *client) return 0; } -static int tf103c_dock_remove(struct i2c_client *client) +static void tf103c_dock_remove(struct i2c_client *client) { struct tf103c_dock_data *dock = i2c_get_clientdata(client); tf103c_dock_stop_hpd(dock); tf103c_dock_disable(dock); - - return 0; } static int __maybe_unused tf103c_dock_suspend(struct device *dev) diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 434249ac47a5cc66850d26c999d88e303c1d3f2a..6e8e093f96b3a2fa60184f8353838386738837ce 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -68,9 +68,11 @@ module_param(fnlock_default, bool, 0444); #define NOTIFY_KBD_FBM 0x99 #define NOTIFY_KBD_TTP 0xae #define NOTIFY_LID_FLIP 0xfa +#define NOTIFY_LID_FLIP_ROG 0xbd #define ASUS_WMI_FNLOCK_BIOS_DISABLED BIT(0) +#define ASUS_GPU_FAN_DESC "gpu_fan" #define ASUS_FAN_DESC "cpu_fan" #define ASUS_FAN_MFUN 0x13 #define ASUS_FAN_SFUN_READ 0x06 @@ -107,7 +109,7 @@ module_param(fnlock_default, bool, 0444); #define WMI_EVENT_MASK 0xFFFF #define FAN_CURVE_POINTS 8 -#define FAN_CURVE_BUF_LEN (FAN_CURVE_POINTS * 2) +#define FAN_CURVE_BUF_LEN 32 #define FAN_CURVE_DEV_CPU 0x00 #define FAN_CURVE_DEV_GPU 0x01 /* Mask to determine if setting temperature or percentage */ @@ -221,19 +223,25 @@ struct asus_wmi { struct asus_rfkill gps; struct asus_rfkill uwb; + int tablet_switch_event_code; + u32 tablet_switch_dev_id; + enum fan_type fan_type; + enum fan_type gpu_fan_type; int fan_pwm_mode; + int gpu_fan_pwm_mode; int agfn_pwm; bool fan_boost_mode_available; u8 fan_boost_mode_mask; u8 fan_boost_mode; - bool egpu_enable_available; // 0 = enable - bool egpu_enable; - + bool egpu_enable_available; bool dgpu_disable_available; - bool dgpu_disable; + bool gpu_mux_mode_available; + + bool kbd_rgb_mode_available; + bool kbd_rgb_state_available; bool throttle_thermal_policy_available; u8 throttle_thermal_policy_mode; @@ -249,7 +257,6 @@ struct asus_wmi { bool battery_rsoc_available; bool panel_overdrive_available; - bool panel_overdrive; struct hotplug_slot hotplug_slot; struct mutex hotplug_lock; @@ -486,10 +493,28 @@ static bool asus_wmi_dev_is_present(struct asus_wmi *asus, u32 dev_id) } /* Input **********************************************************************/ +static void asus_wmi_tablet_sw_init(struct asus_wmi *asus, u32 dev_id, int event_code) +{ + struct device *dev = &asus->platform_device->dev; + int result; + + result = asus_wmi_get_devstate_simple(asus, dev_id); + if (result >= 0) { + input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE); + input_report_switch(asus->inputdev, SW_TABLET_MODE, result); + asus->tablet_switch_dev_id = dev_id; + asus->tablet_switch_event_code = event_code; + } else if (result == -ENODEV) { + dev_err(dev, "This device has tablet-mode-switch quirk but got ENODEV checking it. This is a bug."); + } else { + dev_err(dev, "Error checking for tablet-mode-switch: %d\n", result); + } +} static int asus_wmi_input_init(struct asus_wmi *asus) { - int err, result; + struct device *dev = &asus->platform_device->dev; + int err; asus->inputdev = input_allocate_device(); if (!asus->inputdev) @@ -498,35 +523,25 @@ static int asus_wmi_input_init(struct asus_wmi *asus) asus->inputdev->name = asus->driver->input_name; asus->inputdev->phys = asus->driver->input_phys; asus->inputdev->id.bustype = BUS_HOST; - asus->inputdev->dev.parent = &asus->platform_device->dev; + asus->inputdev->dev.parent = dev; set_bit(EV_REP, asus->inputdev->evbit); err = sparse_keymap_setup(asus->inputdev, asus->driver->keymap, NULL); if (err) goto err_free_dev; - if (asus->driver->quirks->use_kbd_dock_devid) { - result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_KBD_DOCK); - if (result >= 0) { - input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE); - input_report_switch(asus->inputdev, SW_TABLET_MODE, !result); - } else if (result != -ENODEV) { - pr_err("Error checking for keyboard-dock: %d\n", result); - } - } - - if (asus->driver->quirks->use_lid_flip_devid) { - result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_LID_FLIP); - if (result < 0) - asus->driver->quirks->use_lid_flip_devid = 0; - if (result >= 0) { - input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE); - input_report_switch(asus->inputdev, SW_TABLET_MODE, result); - } else if (result == -ENODEV) { - pr_err("This device has lid_flip quirk but got ENODEV checking it. This is a bug."); - } else { - pr_err("Error checking for lid-flip: %d\n", result); - } + switch (asus->driver->quirks->tablet_switch_mode) { + case asus_wmi_no_tablet_switch: + break; + case asus_wmi_kbd_dock_devid: + asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_KBD_DOCK, NOTIFY_KBD_DOCK_CHANGE); + break; + case asus_wmi_lid_flip_devid: + asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_LID_FLIP, NOTIFY_LID_FLIP); + break; + case asus_wmi_lid_flip_rog_devid: + asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_LID_FLIP_ROG, NOTIFY_LID_FLIP_ROG); + break; } err = input_register_device(asus->inputdev); @@ -550,10 +565,14 @@ static void asus_wmi_input_exit(struct asus_wmi *asus) /* Tablet mode ****************************************************************/ -static void lid_flip_tablet_mode_get_state(struct asus_wmi *asus) +static void asus_wmi_tablet_mode_get_state(struct asus_wmi *asus) { - int result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_LID_FLIP); + int result; + + if (!asus->tablet_switch_dev_id) + return; + result = asus_wmi_get_devstate_simple(asus, asus->tablet_switch_dev_id); if (result >= 0) { input_report_switch(asus->inputdev, SW_TABLET_MODE, result); input_sync(asus->inputdev); @@ -561,179 +580,267 @@ static void lid_flip_tablet_mode_get_state(struct asus_wmi *asus) } /* dGPU ********************************************************************/ -static int dgpu_disable_check_present(struct asus_wmi *asus) +static ssize_t dgpu_disable_show(struct device *dev, + struct device_attribute *attr, char *buf) { - u32 result; - int err; - - asus->dgpu_disable_available = false; - - err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_DGPU, &result); - if (err) { - if (err == -ENODEV) - return 0; - return err; - } + struct asus_wmi *asus = dev_get_drvdata(dev); + int result; - if (result & ASUS_WMI_DSTS_PRESENCE_BIT) { - asus->dgpu_disable_available = true; - asus->dgpu_disable = result & ASUS_WMI_DSTS_STATUS_BIT; - } + result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_DGPU); + if (result < 0) + return result; - return 0; + return sysfs_emit(buf, "%d\n", result); } -static int dgpu_disable_write(struct asus_wmi *asus) +/* + * A user may be required to store the value twice, typcial store first, then + * rescan PCI bus to activate power, then store a second time to save correctly. + * The reason for this is that an extra code path in the ACPI is enabled when + * the device and bus are powered. + */ +static ssize_t dgpu_disable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - u32 retval; - u8 value; - int err; + int result, err; + u32 disable; + + struct asus_wmi *asus = dev_get_drvdata(dev); - /* Don't rely on type conversion */ - value = asus->dgpu_disable ? 1 : 0; + result = kstrtou32(buf, 10, &disable); + if (result) + return result; + + if (disable > 1) + return -EINVAL; - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_DGPU, value, &retval); + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_DGPU, disable, &result); if (err) { pr_warn("Failed to set dgpu disable: %d\n", err); return err; } - if (retval > 1) { - pr_warn("Failed to set dgpu disable (retval): 0x%x\n", retval); + if (result > 1) { + pr_warn("Failed to set dgpu disable (result): 0x%x\n", result); return -EIO; } sysfs_notify(&asus->platform_device->dev.kobj, NULL, "dgpu_disable"); - return 0; + return count; } +static DEVICE_ATTR_RW(dgpu_disable); -static ssize_t dgpu_disable_show(struct device *dev, +/* eGPU ********************************************************************/ +static ssize_t egpu_enable_show(struct device *dev, struct device_attribute *attr, char *buf) { struct asus_wmi *asus = dev_get_drvdata(dev); - u8 mode = asus->dgpu_disable; + int result; - return sysfs_emit(buf, "%d\n", mode); + result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_EGPU); + if (result < 0) + return result; + + return sysfs_emit(buf, "%d\n", result); } -/* - * A user may be required to store the value twice, typcial store first, then - * rescan PCI bus to activate power, then store a second time to save correctly. - * The reason for this is that an extra code path in the ACPI is enabled when - * the device and bus are powered. - */ -static ssize_t dgpu_disable_store(struct device *dev, +/* The ACPI call to enable the eGPU also disables the internal dGPU */ +static ssize_t egpu_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - bool disable; - int result; + int result, err; + u32 enable; struct asus_wmi *asus = dev_get_drvdata(dev); - result = kstrtobool(buf, &disable); - if (result) - return result; + err = kstrtou32(buf, 10, &enable); + if (err) + return err; - asus->dgpu_disable = disable; + if (enable > 1) + return -EINVAL; - result = dgpu_disable_write(asus); - if (result) - return result; + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_EGPU, enable, &result); + if (err) { + pr_warn("Failed to set egpu disable: %d\n", err); + return err; + } + + if (result > 1) { + pr_warn("Failed to set egpu disable (retval): 0x%x\n", result); + return -EIO; + } + + sysfs_notify(&asus->platform_device->dev.kobj, NULL, "egpu_enable"); return count; } +static DEVICE_ATTR_RW(egpu_enable); -static DEVICE_ATTR_RW(dgpu_disable); +/* gpu mux switch *************************************************************/ +static ssize_t gpu_mux_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + int result; -/* eGPU ********************************************************************/ -static int egpu_enable_check_present(struct asus_wmi *asus) + result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_GPU_MUX); + if (result < 0) + return result; + + return sysfs_emit(buf, "%d\n", result); +} + +static ssize_t gpu_mux_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - u32 result; - int err; + struct asus_wmi *asus = dev_get_drvdata(dev); + int result, err; + u32 optimus; - asus->egpu_enable_available = false; + err = kstrtou32(buf, 10, &optimus); + if (err) + return err; + + if (optimus > 1) + return -EINVAL; - err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_EGPU, &result); + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_MUX, optimus, &result); if (err) { - if (err == -ENODEV) - return 0; + dev_err(dev, "Failed to set GPU MUX mode: %d\n", err); return err; } - - if (result & ASUS_WMI_DSTS_PRESENCE_BIT) { - asus->egpu_enable_available = true; - asus->egpu_enable = result & ASUS_WMI_DSTS_STATUS_BIT; + /* !1 is considered a fail by ASUS */ + if (result != 1) { + dev_warn(dev, "Failed to set GPU MUX mode (result): 0x%x\n", result); + return -EIO; } - return 0; + sysfs_notify(&asus->platform_device->dev.kobj, NULL, "gpu_mux_mode"); + + return count; } +static DEVICE_ATTR_RW(gpu_mux_mode); -static int egpu_enable_write(struct asus_wmi *asus) +/* TUF Laptop Keyboard RGB Modes **********************************************/ +static ssize_t kbd_rgb_mode_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - u32 retval; - u8 value; + u32 cmd, mode, r, g, b, speed; int err; - /* Don't rely on type conversion */ - value = asus->egpu_enable ? 1 : 0; + if (sscanf(buf, "%d %d %d %d %d %d", &cmd, &mode, &r, &g, &b, &speed) != 6) + return -EINVAL; - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_EGPU, value, &retval); + cmd = !!cmd; - if (err) { - pr_warn("Failed to set egpu disable: %d\n", err); - return err; - } + /* These are the known usable modes across all TUF/ROG */ + if (mode >= 12 || mode == 9) + mode = 10; - if (retval > 1) { - pr_warn("Failed to set egpu disable (retval): 0x%x\n", retval); - return -EIO; + switch (speed) { + case 0: + speed = 0xe1; + break; + case 1: + speed = 0xeb; + break; + case 2: + speed = 0xf5; + break; + default: + speed = 0xeb; } - sysfs_notify(&asus->platform_device->dev.kobj, NULL, "egpu_enable"); + err = asus_wmi_evaluate_method3(ASUS_WMI_METHODID_DEVS, ASUS_WMI_DEVID_TUF_RGB_MODE, + cmd | (mode << 8) | (r << 16) | (g << 24), b | (speed << 8), NULL); + if (err) + return err; - return 0; + return count; } +static DEVICE_ATTR_WO(kbd_rgb_mode); -static ssize_t egpu_enable_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t kbd_rgb_mode_index_show(struct device *device, + struct device_attribute *attr, + char *buf) { - struct asus_wmi *asus = dev_get_drvdata(dev); - bool mode = asus->egpu_enable; - - return sysfs_emit(buf, "%d\n", mode); + return sysfs_emit(buf, "%s\n", "cmd mode red green blue speed"); } +static DEVICE_ATTR_RO(kbd_rgb_mode_index); -/* The ACPI call to enable the eGPU also disables the internal dGPU */ -static ssize_t egpu_enable_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - bool enable; - int result; - - struct asus_wmi *asus = dev_get_drvdata(dev); +static struct attribute *kbd_rgb_mode_attrs[] = { + &dev_attr_kbd_rgb_mode.attr, + &dev_attr_kbd_rgb_mode_index.attr, + NULL, +}; - result = kstrtobool(buf, &enable); - if (result) - return result; +static const struct attribute_group kbd_rgb_mode_group = { + .attrs = kbd_rgb_mode_attrs, +}; - asus->egpu_enable = enable; +/* TUF Laptop Keyboard RGB State **********************************************/ +static ssize_t kbd_rgb_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + u32 flags, cmd, boot, awake, sleep, keyboard; + int err; - result = egpu_enable_write(asus); - if (result) - return result; + if (sscanf(buf, "%d %d %d %d %d", &cmd, &boot, &awake, &sleep, &keyboard) != 5) + return -EINVAL; - /* Ensure that the kernel status of dgpu is updated */ - result = dgpu_disable_check_present(asus); - if (result) - return result; + if (cmd) + cmd = BIT(2); + + flags = 0; + if (boot) + flags |= BIT(1); + if (awake) + flags |= BIT(3); + if (sleep) + flags |= BIT(5); + if (keyboard) + flags |= BIT(7); + + /* 0xbd is the required default arg0 for the method. Nothing happens otherwise */ + err = asus_wmi_evaluate_method3(ASUS_WMI_METHODID_DEVS, + ASUS_WMI_DEVID_TUF_RGB_STATE, 0xbd | cmd << 8 | (flags << 16), 0, NULL); + if (err) + return err; return count; } +static DEVICE_ATTR_WO(kbd_rgb_state); -static DEVICE_ATTR_RW(egpu_enable); +static ssize_t kbd_rgb_state_index_show(struct device *device, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", "cmd boot awake sleep keyboard"); +} +static DEVICE_ATTR_RO(kbd_rgb_state_index); + +static struct attribute *kbd_rgb_state_attrs[] = { + &dev_attr_kbd_rgb_state.attr, + &dev_attr_kbd_rgb_state_index.attr, + NULL, +}; + +static const struct attribute_group kbd_rgb_state_group = { + .attrs = kbd_rgb_state_attrs, +}; + +static const struct attribute_group *kbd_rgb_mode_groups[] = { + NULL, + NULL, + NULL, +}; /* Battery ********************************************************************/ @@ -771,7 +878,7 @@ static ssize_t charge_control_end_threshold_show(struct device *device, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%d\n", charge_end_threshold); + return sysfs_emit(buf, "%d\n", charge_end_threshold); } static DEVICE_ATTR_RW(charge_control_end_threshold); @@ -1053,7 +1160,12 @@ static void asus_wmi_led_exit(struct asus_wmi *asus) static int asus_wmi_led_init(struct asus_wmi *asus) { - int rv = 0, led_val; + int rv = 0, num_rgb_groups = 0, led_val; + + if (asus->kbd_rgb_mode_available) + kbd_rgb_mode_groups[num_rgb_groups++] = &kbd_rgb_mode_group; + if (asus->kbd_rgb_state_available) + kbd_rgb_mode_groups[num_rgb_groups++] = &kbd_rgb_state_group; asus->led_workqueue = create_singlethread_workqueue("led_workqueue"); if (!asus->led_workqueue) @@ -1081,6 +1193,9 @@ static int asus_wmi_led_init(struct asus_wmi *asus) asus->kbd_led.brightness_get = kbd_led_get; asus->kbd_led.max_brightness = 3; + if (num_rgb_groups != 0) + asus->kbd_led.groups = kbd_rgb_mode_groups; + rv = led_classdev_register(&asus->platform_device->dev, &asus->kbd_led); if (rv) @@ -1118,7 +1233,7 @@ static int asus_wmi_led_init(struct asus_wmi *asus) } if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_MICMUTE_LED)) { - asus->micmute_led.name = "asus::micmute"; + asus->micmute_led.name = "platform::micmute"; asus->micmute_led.max_brightness = 1; asus->micmute_led.brightness = ledtrig_audio_get(LED_AUDIO_MICMUTE); asus->micmute_led.brightness_set_blocking = micmute_led_set; @@ -1555,84 +1670,51 @@ exit: } /* Panel Overdrive ************************************************************/ -static int panel_od_check_present(struct asus_wmi *asus) -{ - u32 result; - int err; - - asus->panel_overdrive_available = false; - - err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_PANEL_OD, &result); - if (err) { - if (err == -ENODEV) - return 0; - return err; - } - - if (result & ASUS_WMI_DSTS_PRESENCE_BIT) { - asus->panel_overdrive_available = true; - asus->panel_overdrive = result & ASUS_WMI_DSTS_STATUS_BIT; - } - - return 0; -} - -static int panel_od_write(struct asus_wmi *asus) -{ - u32 retval; - u8 value; - int err; - - /* Don't rely on type conversion */ - value = asus->panel_overdrive ? 1 : 0; - - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_PANEL_OD, value, &retval); - - if (err) { - pr_warn("Failed to set panel overdrive: %d\n", err); - return err; - } - - if (retval > 1) { - pr_warn("Failed to set panel overdrive (retval): 0x%x\n", retval); - return -EIO; - } - - sysfs_notify(&asus->platform_device->dev.kobj, NULL, "panel_od"); - - return 0; -} - static ssize_t panel_od_show(struct device *dev, struct device_attribute *attr, char *buf) { struct asus_wmi *asus = dev_get_drvdata(dev); + int result; + + result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_PANEL_OD); + if (result < 0) + return result; - return sysfs_emit(buf, "%d\n", asus->panel_overdrive); + return sysfs_emit(buf, "%d\n", result); } static ssize_t panel_od_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - bool overdrive; - int result; + int result, err; + u32 overdrive; struct asus_wmi *asus = dev_get_drvdata(dev); - result = kstrtobool(buf, &overdrive); + result = kstrtou32(buf, 10, &overdrive); if (result) return result; - asus->panel_overdrive = overdrive; - result = panel_od_write(asus); + if (overdrive > 1) + return -EINVAL; - if (result) - return result; + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_PANEL_OD, overdrive, &result); + + if (err) { + pr_warn("Failed to set panel overdrive: %d\n", err); + return err; + } + + if (result > 1) { + pr_warn("Failed to set panel overdrive (result): 0x%x\n", result); + return -EIO; + } + + sysfs_notify(&asus->platform_device->dev.kobj, NULL, "panel_od"); return count; } - static DEVICE_ATTR_RW(panel_od); /* Quirks *********************************************************************/ @@ -1782,6 +1864,18 @@ static int asus_fan_set_auto(struct asus_wmi *asus) return -ENXIO; } + /* + * Modern models like the G713 also have GPU fan control (this is not AGFN) + */ + if (asus->gpu_fan_type == FAN_TYPE_SPEC83) { + status = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_FAN_CTRL, + 0, &retval); + if (status) + return status; + + if (retval != 1) + return -EIO; + } return 0; } @@ -1819,7 +1913,7 @@ static ssize_t pwm1_show(struct device *dev, value = -1; } - return sprintf(buf, "%d\n", value); + return sysfs_emit(buf, "%d\n", value); } static ssize_t pwm1_store(struct device *dev, @@ -1879,7 +1973,7 @@ static ssize_t fan1_input_show(struct device *dev, return -ENXIO; } - return sprintf(buf, "%d\n", value < 0 ? -1 : value*100); + return sysfs_emit(buf, "%d\n", value < 0 ? -1 : value * 100); } static ssize_t pwm1_enable_show(struct device *dev, @@ -1897,7 +1991,7 @@ static ssize_t pwm1_enable_show(struct device *dev, * in practice on X532FL at least (the bit is always 0) and there's * also nothing in the DSDT to indicate that this behaviour exists. */ - return sprintf(buf, "%d\n", asus->fan_pwm_mode); + return sysfs_emit(buf, "%d\n", asus->fan_pwm_mode); } static ssize_t pwm1_enable_store(struct device *dev, @@ -1965,7 +2059,7 @@ static ssize_t fan1_label_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%s\n", ASUS_FAN_DESC); + return sysfs_emit(buf, "%s\n", ASUS_FAN_DESC); } static ssize_t asus_hwmon_temp1(struct device *dev, @@ -1984,11 +2078,86 @@ static ssize_t asus_hwmon_temp1(struct device *dev, deci_kelvin_to_millicelsius(value & 0xFFFF)); } +/* GPU fan on modern ROG laptops */ +static ssize_t fan2_input_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + int value; + int ret; + + ret = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_GPU_FAN_CTRL, &value); + if (ret < 0) + return ret; + + value &= 0xffff; + + return sysfs_emit(buf, "%d\n", value * 100); +} + +static ssize_t fan2_label_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", ASUS_GPU_FAN_DESC); +} + +static ssize_t pwm2_enable_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", asus->gpu_fan_pwm_mode); +} + +static ssize_t pwm2_enable_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct asus_wmi *asus = dev_get_drvdata(dev); + int state; + int value; + int ret; + u32 retval; + + ret = kstrtouint(buf, 10, &state); + if (ret) + return ret; + + switch (state) { /* standard documented hwmon values */ + case ASUS_FAN_CTRL_FULLSPEED: + value = 1; + break; + case ASUS_FAN_CTRL_AUTO: + value = 0; + break; + default: + return -EINVAL; + } + + ret = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_FAN_CTRL, + value, &retval); + if (ret) + return ret; + + if (retval != 1) + return -EIO; + + asus->gpu_fan_pwm_mode = state; + return count; +} + /* Fan1 */ static DEVICE_ATTR_RW(pwm1); static DEVICE_ATTR_RW(pwm1_enable); static DEVICE_ATTR_RO(fan1_input); static DEVICE_ATTR_RO(fan1_label); +/* Fan2 - GPU fan */ +static DEVICE_ATTR_RW(pwm2_enable); +static DEVICE_ATTR_RO(fan2_input); +static DEVICE_ATTR_RO(fan2_label); /* Temperature */ static DEVICE_ATTR(temp1_input, S_IRUGO, asus_hwmon_temp1, NULL); @@ -1996,8 +2165,11 @@ static DEVICE_ATTR(temp1_input, S_IRUGO, asus_hwmon_temp1, NULL); static struct attribute *hwmon_attributes[] = { &dev_attr_pwm1.attr, &dev_attr_pwm1_enable.attr, + &dev_attr_pwm2_enable.attr, &dev_attr_fan1_input.attr, &dev_attr_fan1_label.attr, + &dev_attr_fan2_input.attr, + &dev_attr_fan2_label.attr, &dev_attr_temp1_input.attr, NULL @@ -2006,7 +2178,7 @@ static struct attribute *hwmon_attributes[] = { static umode_t asus_hwmon_sysfs_is_visible(struct kobject *kobj, struct attribute *attr, int idx) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct asus_wmi *asus = dev_get_drvdata(dev->parent); u32 value = ASUS_WMI_UNSUPPORTED_METHOD; @@ -2018,6 +2190,11 @@ static umode_t asus_hwmon_sysfs_is_visible(struct kobject *kobj, || attr == &dev_attr_pwm1_enable.attr) { if (asus->fan_type == FAN_TYPE_NONE) return 0; + } else if (attr == &dev_attr_fan2_input.attr + || attr == &dev_attr_fan2_label.attr + || attr == &dev_attr_pwm2_enable.attr) { + if (asus->gpu_fan_type == FAN_TYPE_NONE) + return 0; } else if (attr == &dev_attr_temp1_input.attr) { int err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_THERMAL_CTRL, @@ -2060,6 +2237,7 @@ static int asus_wmi_hwmon_init(struct asus_wmi *asus) static int asus_wmi_fan_init(struct asus_wmi *asus) { + asus->gpu_fan_type = FAN_TYPE_NONE; asus->fan_type = FAN_TYPE_NONE; asus->agfn_pwm = -1; @@ -2068,6 +2246,10 @@ static int asus_wmi_fan_init(struct asus_wmi *asus) else if (asus_wmi_has_agfn_fan(asus)) asus->fan_type = FAN_TYPE_AGFN; + /* Modern models like G713 also have GPU fan control */ + if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_FAN_CTRL)) + asus->gpu_fan_type = FAN_TYPE_SPEC83; + if (asus->fan_type == FAN_TYPE_NONE) return -ENODEV; @@ -2158,7 +2340,7 @@ static ssize_t fan_boost_mode_show(struct device *dev, { struct asus_wmi *asus = dev_get_drvdata(dev); - return scnprintf(buf, PAGE_SIZE, "%d\n", asus->fan_boost_mode); + return sysfs_emit(buf, "%d\n", asus->fan_boost_mode); } static ssize_t fan_boost_mode_store(struct device *dev, @@ -2233,8 +2415,10 @@ static int fan_curve_get_factory_default(struct asus_wmi *asus, u32 fan_dev) curves = &asus->custom_fan_curves[fan_idx]; err = asus_wmi_evaluate_method_buf(asus->dsts_id, fan_dev, mode, buf, FAN_CURVE_BUF_LEN); - if (err) + if (err) { + pr_warn("%s (0x%08x) failed: %d\n", __func__, fan_dev, err); return err; + } fan_curve_copy_from_buf(curves, buf); curves->device_id = fan_dev; @@ -2252,9 +2436,6 @@ static int fan_curve_check_present(struct asus_wmi *asus, bool *available, err = fan_curve_get_factory_default(asus, fan_dev); if (err) { - pr_debug("fan_curve_get_factory_default(0x%08x) failed: %d\n", - fan_dev, err); - /* Don't cause probe to fail on devices without fan-curves */ return 0; } @@ -2711,7 +2892,7 @@ static ssize_t throttle_thermal_policy_show(struct device *dev, struct asus_wmi *asus = dev_get_drvdata(dev); u8 mode = asus->throttle_thermal_policy_mode; - return scnprintf(buf, PAGE_SIZE, "%d\n", mode); + return sysfs_emit(buf, "%d\n", mode); } static ssize_t throttle_thermal_policy_store(struct device *dev, @@ -3063,9 +3244,7 @@ static void asus_wmi_handle_event_code(int code, struct asus_wmi *asus) { unsigned int key_value = 1; bool autorelease = 1; - int result, orig_code; - - orig_code = code; + int orig_code = code; if (asus->driver->key_filter) { asus->driver->key_filter(asus->driver, &code, &key_value, @@ -3108,30 +3287,18 @@ static void asus_wmi_handle_event_code(int code, struct asus_wmi *asus) return; } - if (asus->driver->quirks->use_kbd_dock_devid && code == NOTIFY_KBD_DOCK_CHANGE) { - result = asus_wmi_get_devstate_simple(asus, - ASUS_WMI_DEVID_KBD_DOCK); - if (result >= 0) { - input_report_switch(asus->inputdev, SW_TABLET_MODE, - !result); - input_sync(asus->inputdev); - } - return; - } - - if (asus->driver->quirks->use_lid_flip_devid && code == NOTIFY_LID_FLIP) { - lid_flip_tablet_mode_get_state(asus); + if (code == asus->tablet_switch_event_code) { + asus_wmi_tablet_mode_get_state(asus); return; } - if (asus->fan_boost_mode_available && code == NOTIFY_KBD_FBM) { - fan_boost_mode_switch_next(asus); + if (code == NOTIFY_KBD_FBM || code == NOTIFY_KBD_TTP) { + if (asus->fan_boost_mode_available) + fan_boost_mode_switch_next(asus); + if (asus->throttle_thermal_policy_available) + throttle_thermal_policy_switch_next(asus); return; - } - if (asus->throttle_thermal_policy_available && code == NOTIFY_KBD_TTP) { - throttle_thermal_policy_switch_next(asus); - return; } if (is_display_toggle(code) && asus->driver->quirks->no_display_toggle) @@ -3283,6 +3450,7 @@ static struct attribute *platform_attributes[] = { &dev_attr_touchpad.attr, &dev_attr_egpu_enable.attr, &dev_attr_dgpu_disable.attr, + &dev_attr_gpu_mux_mode.attr, &dev_attr_lid_resume.attr, &dev_attr_als_enable.attr, &dev_attr_fan_boost_mode.attr, @@ -3294,7 +3462,7 @@ static struct attribute *platform_attributes[] = { static umode_t asus_sysfs_is_visible(struct kobject *kobj, struct attribute *attr, int idx) { - struct device *dev = container_of(kobj, struct device, kobj); + struct device *dev = kobj_to_dev(kobj); struct asus_wmi *asus = dev_get_drvdata(dev); bool ok = true; int devid = -1; @@ -3313,6 +3481,8 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj, ok = asus->egpu_enable_available; else if (attr == &dev_attr_dgpu_disable.attr) ok = asus->dgpu_disable_available; + else if (attr == &dev_attr_gpu_mux_mode.attr) + ok = asus->gpu_mux_mode_available; else if (attr == &dev_attr_fan_boost_mode.attr) ok = asus->fan_boost_mode_available; else if (attr == &dev_attr_throttle_thermal_policy.attr) @@ -3573,13 +3743,12 @@ static int asus_wmi_add(struct platform_device *pdev) if (err) goto fail_platform; - err = egpu_enable_check_present(asus); - if (err) - goto fail_egpu_enable; - - err = dgpu_disable_check_present(asus); - if (err) - goto fail_dgpu_disable; + asus->egpu_enable_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_EGPU); + asus->dgpu_disable_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_DGPU); + asus->gpu_mux_mode_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_MUX); + asus->kbd_rgb_mode_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE); + asus->kbd_rgb_state_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_STATE); + asus->panel_overdrive_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_PANEL_OD); err = fan_boost_mode_check_present(asus); if (err) @@ -3595,10 +3764,6 @@ static int asus_wmi_add(struct platform_device *pdev) if (err) goto fail_platform_profile_setup; - err = panel_od_check_present(asus); - if (err) - goto fail_panel_od; - err = asus_wmi_sysfs_init(asus->platform_device); if (err) goto fail_sysfs; @@ -3681,10 +3846,7 @@ fail_platform_profile_setup: if (asus->platform_profile_support) platform_profile_remove(); fail_fan_boost_mode: -fail_egpu_enable: -fail_dgpu_disable: fail_platform: -fail_panel_od: kfree(asus); return err; } @@ -3743,9 +3905,7 @@ static int asus_hotk_resume(struct device *device) if (asus_wmi_has_fnlock_key(asus)) asus_wmi_fnlock_update(asus); - if (asus->driver->quirks->use_lid_flip_devid) - lid_flip_tablet_mode_get_state(asus); - + asus_wmi_tablet_mode_get_state(asus); return 0; } @@ -3785,9 +3945,7 @@ static int asus_hotk_restore(struct device *device) if (asus_wmi_has_fnlock_key(asus)) asus_wmi_fnlock_update(asus); - if (asus->driver->quirks->use_lid_flip_devid) - lid_flip_tablet_mode_get_state(asus); - + asus_wmi_tablet_mode_get_state(asus); return 0; } diff --git a/drivers/platform/x86/asus-wmi.h b/drivers/platform/x86/asus-wmi.h index f30252efe1db04badb934bcf155e37e154f66e68..65316998b898aaf7cbff2dbc3c1a969ac9e93de9 100644 --- a/drivers/platform/x86/asus-wmi.h +++ b/drivers/platform/x86/asus-wmi.h @@ -25,14 +25,20 @@ struct module; struct key_entry; struct asus_wmi; +enum asus_wmi_tablet_switch_mode { + asus_wmi_no_tablet_switch, + asus_wmi_kbd_dock_devid, + asus_wmi_lid_flip_devid, + asus_wmi_lid_flip_rog_devid, +}; + struct quirk_entry { bool hotplug_wireless; bool scalar_panel_brightness; bool store_backlight_power; bool wmi_backlight_set_devstate; bool wmi_force_als_set; - bool use_kbd_dock_devid; - bool use_lid_flip_devid; + enum asus_wmi_tablet_switch_mode tablet_switch_mode; int wapf; /* * For machines with AMD graphic chips, it will send out WMI event diff --git a/drivers/platform/x86/compal-laptop.c b/drivers/platform/x86/compal-laptop.c index 0942f50bd793c843ad92cdcf9210e03c3a5531c0..e10d2f64dfadd7904a4527198fff2f19b06d4747 100644 --- a/drivers/platform/x86/compal-laptop.c +++ b/drivers/platform/x86/compal-laptop.c @@ -721,16 +721,6 @@ static struct attribute *compal_hwmon_attrs[] = { }; ATTRIBUTE_GROUPS(compal_hwmon); -static int compal_probe(struct platform_device *); -static int compal_remove(struct platform_device *); -static struct platform_driver compal_driver = { - .driver = { - .name = DRIVER_NAME, - }, - .probe = compal_probe, - .remove = compal_remove, -}; - static enum power_supply_property compal_bat_properties[] = { POWER_SUPPLY_PROP_STATUS, POWER_SUPPLY_PROP_HEALTH, @@ -965,6 +955,80 @@ err_wifi: return ret; } +static int compal_probe(struct platform_device *pdev) +{ + int err; + struct compal_data *data; + struct device *hwmon_dev; + struct power_supply_config psy_cfg = {}; + + if (!extra_features) + return 0; + + /* Fan control */ + data = devm_kzalloc(&pdev->dev, sizeof(struct compal_data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + initialize_fan_control_data(data); + + err = sysfs_create_group(&pdev->dev.kobj, &compal_platform_attr_group); + if (err) + return err; + + hwmon_dev = devm_hwmon_device_register_with_groups(&pdev->dev, + "compal", data, + compal_hwmon_groups); + if (IS_ERR(hwmon_dev)) { + err = PTR_ERR(hwmon_dev); + goto remove; + } + + /* Power supply */ + initialize_power_supply_data(data); + psy_cfg.drv_data = data; + data->psy = power_supply_register(&compal_device->dev, &psy_bat_desc, + &psy_cfg); + if (IS_ERR(data->psy)) { + err = PTR_ERR(data->psy); + goto remove; + } + + platform_set_drvdata(pdev, data); + + return 0; + +remove: + sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group); + return err; +} + +static int compal_remove(struct platform_device *pdev) +{ + struct compal_data *data; + + if (!extra_features) + return 0; + + pr_info("Unloading: resetting fan control to motherboard\n"); + pwm_disable_control(); + + data = platform_get_drvdata(pdev); + power_supply_unregister(data->psy); + + sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group); + + return 0; +} + +static struct platform_driver compal_driver = { + .driver = { + .name = DRIVER_NAME, + }, + .probe = compal_probe, + .remove = compal_remove, +}; + static int __init compal_init(void) { int ret; @@ -996,7 +1060,7 @@ static int __init compal_init(void) if (ret) goto err_backlight; - compal_device = platform_device_alloc(DRIVER_NAME, -1); + compal_device = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE); if (!compal_device) { ret = -ENOMEM; goto err_platform_driver; @@ -1028,54 +1092,6 @@ err_backlight: return ret; } -static int compal_probe(struct platform_device *pdev) -{ - int err; - struct compal_data *data; - struct device *hwmon_dev; - struct power_supply_config psy_cfg = {}; - - if (!extra_features) - return 0; - - /* Fan control */ - data = devm_kzalloc(&pdev->dev, sizeof(struct compal_data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - initialize_fan_control_data(data); - - err = sysfs_create_group(&pdev->dev.kobj, &compal_platform_attr_group); - if (err) - return err; - - hwmon_dev = devm_hwmon_device_register_with_groups(&pdev->dev, - "compal", data, - compal_hwmon_groups); - if (IS_ERR(hwmon_dev)) { - err = PTR_ERR(hwmon_dev); - goto remove; - } - - /* Power supply */ - initialize_power_supply_data(data); - psy_cfg.drv_data = data; - data->psy = power_supply_register(&compal_device->dev, &psy_bat_desc, - &psy_cfg); - if (IS_ERR(data->psy)) { - err = PTR_ERR(data->psy); - goto remove; - } - - platform_set_drvdata(pdev, data); - - return 0; - -remove: - sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group); - return err; -} - static void __exit compal_cleanup(void) { platform_device_unregister(compal_device); @@ -1089,25 +1105,6 @@ static void __exit compal_cleanup(void) pr_info("Driver unloaded\n"); } -static int compal_remove(struct platform_device *pdev) -{ - struct compal_data *data; - - if (!extra_features) - return 0; - - pr_info("Unloading: resetting fan control to motherboard\n"); - pwm_disable_control(); - - data = platform_get_drvdata(pdev); - power_supply_unregister(data->psy); - - sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group); - - return 0; -} - - module_init(compal_init); module_exit(compal_cleanup); diff --git a/drivers/platform/x86/dell/alienware-wmi.c b/drivers/platform/x86/dell/alienware-wmi.c index f212482555298ba31d34d35c406068a56465b10b..a34e07ef2c79dd280252e07cb5128fd7a57c37b0 100644 --- a/drivers/platform/x86/dell/alienware-wmi.c +++ b/drivers/platform/x86/dell/alienware-wmi.c @@ -791,7 +791,7 @@ static int __init alienware_wmi_init(void) ret = platform_driver_register(&platform_driver); if (ret) goto fail_platform_driver; - platform_device = platform_device_alloc("alienware-wmi", -1); + platform_device = platform_device_alloc("alienware-wmi", PLATFORM_DEVID_NONE); if (!platform_device) { ret = -ENOMEM; goto fail_platform_device1; diff --git a/drivers/platform/x86/dell/dcdbas.c b/drivers/platform/x86/dell/dcdbas.c index 42beafbc54b2ac53b9cc2853dc5fc509c593d251..0ecb7b1647508b512faa72e9def42321f9dfbef1 100644 --- a/drivers/platform/x86/dell/dcdbas.c +++ b/drivers/platform/x86/dell/dcdbas.c @@ -716,7 +716,7 @@ static struct platform_driver dcdbas_driver = { static const struct platform_device_info dcdbas_dev_info __initconst = { .name = DRIVER_NAME, - .id = -1, + .id = PLATFORM_DEVID_NONE, .dma_mask = DMA_BIT_MASK(32), }; diff --git a/drivers/platform/x86/dell/dell-laptop.c b/drivers/platform/x86/dell/dell-laptop.c index 1321687d923ed5e7d81738e4bf7e7baf1264f7c1..e92c3ad06d6965499d2965fc9a114e86c49b9524 100644 --- a/drivers/platform/x86/dell/dell-laptop.c +++ b/drivers/platform/x86/dell/dell-laptop.c @@ -2193,7 +2193,7 @@ static int __init dell_init(void) ret = platform_driver_register(&platform_driver); if (ret) goto fail_platform_driver; - platform_device = platform_device_alloc("dell-laptop", -1); + platform_device = platform_device_alloc("dell-laptop", PLATFORM_DEVID_NONE); if (!platform_device) { ret = -ENOMEM; goto fail_platform_device1; diff --git a/drivers/platform/x86/dell/dell-smbios-base.c b/drivers/platform/x86/dell/dell-smbios-base.c index fc086b66f70b377374dcd71627f2efb0769c8e1d..e61bfaf8b5c48faf8fdf2df15cf0faa140b5869e 100644 --- a/drivers/platform/x86/dell/dell-smbios-base.c +++ b/drivers/platform/x86/dell/dell-smbios-base.c @@ -441,7 +441,7 @@ static ssize_t location_show(struct device *dev, i = match_attribute(dev, attr); if (i > 0) - return scnprintf(buf, PAGE_SIZE, "%08x", da_tokens[i].location); + return sysfs_emit(buf, "%08x", da_tokens[i].location); return 0; } @@ -455,7 +455,7 @@ static ssize_t value_show(struct device *dev, i = match_attribute(dev, attr); if (i > 0) - return scnprintf(buf, PAGE_SIZE, "%08x", da_tokens[i].value); + return sysfs_emit(buf, "%08x", da_tokens[i].value); return 0; } diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index e07d3ba85a3f4922a25dce51b7782e5aa79c062c..0a259a27459f6de978122967fb550524279f5375 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -344,6 +344,9 @@ static const struct key_entry dell_wmi_keymap_type_0011[] = { * They are events with extended data */ static const struct key_entry dell_wmi_keymap_type_0012[] = { + /* Backlight brightness change event */ + { KE_IGNORE, 0x0003, { KEY_RESERVED } }, + /* Ultra-performance mode switch request */ { KE_IGNORE, 0x000d, { KEY_RESERVED } }, diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c index 074b7e68c227c9b324e1699d5ebb2d6b64dc9a29..c82b3d6867c5b28869bf33c911a7613e9efec657 100644 --- a/drivers/platform/x86/dell/dell-wmi-privacy.c +++ b/drivers/platform/x86/dell/dell-wmi-privacy.c @@ -174,15 +174,12 @@ static ssize_t dell_privacy_current_state_show(struct device *dev, static DEVICE_ATTR_RO(dell_privacy_supported_type); static DEVICE_ATTR_RO(dell_privacy_current_state); -static struct attribute *privacy_attributes[] = { +static struct attribute *privacy_attrs[] = { &dev_attr_dell_privacy_supported_type.attr, &dev_attr_dell_privacy_current_state.attr, NULL, }; - -static const struct attribute_group privacy_attribute_group = { - .attrs = privacy_attributes -}; +ATTRIBUTE_GROUPS(privacy); /* * Describes the Device State class exposed by BIOS which can be consumed by @@ -342,10 +339,6 @@ static int dell_privacy_wmi_probe(struct wmi_device *wdev, const void *context) if (ret) return ret; - ret = devm_device_add_group(&wdev->dev, &privacy_attribute_group); - if (ret) - return ret; - if (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO)) { ret = dell_privacy_leds_setup(&priv->wdev->dev); if (ret) @@ -374,6 +367,7 @@ static const struct wmi_device_id dell_wmi_privacy_wmi_id_table[] = { static struct wmi_driver dell_privacy_wmi_driver = { .driver = { .name = "dell-privacy", + .dev_groups = privacy_groups, }, .probe = dell_privacy_wmi_probe, .remove = dell_privacy_wmi_remove, diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c index 636bdfa83284d50acc239f88b01c76359d75f920..0a6411a8a104c52973a5b9e17614751d86744b8f 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c @@ -270,7 +270,7 @@ void strlcpy_attr(char *dest, char *src) size_t len = strlen(src) + 1; if (len > 1 && len <= MAX_BUFF) - strlcpy(dest, src, len); + strscpy(dest, src, len); /*len can be zero because any property not-applicable to attribute can * be empty so check only for too long buffers and log error diff --git a/drivers/platform/x86/dell/dell_rbu.c b/drivers/platform/x86/dell/dell_rbu.c index e9f4b30dcafabffb58f618aec47a9c305c742e73..9f51e0fcab04e1c61ce68b90b754b6b43a741b58 100644 --- a/drivers/platform/x86/dell/dell_rbu.c +++ b/drivers/platform/x86/dell/dell_rbu.c @@ -645,7 +645,7 @@ static int __init dcdrbu_init(void) spin_lock_init(&rbu_data.lock); init_packet_head(); - rbu_device = platform_device_register_simple("dell_rbu", -1, NULL, 0); + rbu_device = platform_device_register_simple("dell_rbu", PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(rbu_device)) { pr_err("platform_device_register_simple failed\n"); return PTR_ERR(rbu_device); diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c index ba08c9235f769652fff9d7ee9a8ce73905aa15a2..a388a28b6f2a2188a3fa0c472b92a7ae96ae5808 100644 --- a/drivers/platform/x86/eeepc-laptop.c +++ b/drivers/platform/x86/eeepc-laptop.c @@ -444,7 +444,7 @@ static int eeepc_platform_init(struct eeepc_laptop *eeepc) { int result; - eeepc->platform_device = platform_device_alloc(EEEPC_LAPTOP_FILE, -1); + eeepc->platform_device = platform_device_alloc(EEEPC_LAPTOP_FILE, PLATFORM_DEVID_NONE); if (!eeepc->platform_device) return -ENOMEM; platform_set_drvdata(eeepc->platform_device, eeepc); diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c index 80929380ec7e33cf8af0a4a1ea1dcfea41ea2986..b543d117b12c7a5f664cc2db1a12cb684c6331fd 100644 --- a/drivers/platform/x86/fujitsu-laptop.c +++ b/drivers/platform/x86/fujitsu-laptop.c @@ -543,7 +543,7 @@ static int fujitsu_laptop_platform_add(struct acpi_device *device) struct fujitsu_laptop *priv = acpi_driver_data(device); int ret; - priv->pf_device = platform_device_alloc("fujitsu-laptop", -1); + priv->pf_device = platform_device_alloc("fujitsu-laptop", PLATFORM_DEVID_NONE); if (!priv->pf_device) return -ENOMEM; diff --git a/drivers/platform/x86/hdaps.c b/drivers/platform/x86/hdaps.c index 9996485f529590bb18af79a2aa209f92c16b37ad..f11f726d20629b7ddab4e9f0845a36ad2ae301ac 100644 --- a/drivers/platform/x86/hdaps.c +++ b/drivers/platform/x86/hdaps.c @@ -547,7 +547,7 @@ static int __init hdaps_init(void) if (ret) goto out_region; - pdev = platform_device_register_simple("hdaps", -1, NULL, 0); + pdev = platform_device_register_simple("hdaps", PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pdev)) { ret = PTR_ERR(pdev); goto out_driver; diff --git a/drivers/platform/x86/hp-wmi.c b/drivers/platform/x86/hp-wmi.c index bc7020e9df9e8d40820099d7f19b0bb0c683a934..627a6d0eaf831e91c4f88c502e54c4167cedf76f 100644 --- a/drivers/platform/x86/hp-wmi.c +++ b/drivers/platform/x86/hp-wmi.c @@ -177,7 +177,8 @@ enum hp_thermal_profile_omen_v1 { enum hp_thermal_profile { HP_THERMAL_PROFILE_PERFORMANCE = 0x00, HP_THERMAL_PROFILE_DEFAULT = 0x01, - HP_THERMAL_PROFILE_COOL = 0x02 + HP_THERMAL_PROFILE_COOL = 0x02, + HP_THERMAL_PROFILE_QUIET = 0x03, }; #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW) @@ -206,15 +207,17 @@ struct bios_rfkill2_state { }; static const struct key_entry hp_wmi_keymap[] = { - { KE_KEY, 0x02, { KEY_BRIGHTNESSUP } }, - { KE_KEY, 0x03, { KEY_BRIGHTNESSDOWN } }, - { KE_KEY, 0x20e6, { KEY_PROG1 } }, - { KE_KEY, 0x20e8, { KEY_MEDIA } }, - { KE_KEY, 0x2142, { KEY_MEDIA } }, - { KE_KEY, 0x213b, { KEY_INFO } }, - { KE_KEY, 0x2169, { KEY_ROTATE_DISPLAY } }, - { KE_KEY, 0x216a, { KEY_SETUP } }, - { KE_KEY, 0x231b, { KEY_HELP } }, + { KE_KEY, 0x02, { KEY_BRIGHTNESSUP } }, + { KE_KEY, 0x03, { KEY_BRIGHTNESSDOWN } }, + { KE_KEY, 0x20e6, { KEY_PROG1 } }, + { KE_KEY, 0x20e8, { KEY_MEDIA } }, + { KE_KEY, 0x2142, { KEY_MEDIA } }, + { KE_KEY, 0x213b, { KEY_INFO } }, + { KE_KEY, 0x2169, { KEY_ROTATE_DISPLAY } }, + { KE_KEY, 0x216a, { KEY_SETUP } }, + { KE_KEY, 0x21a9, { KEY_TOUCHPAD_OFF } }, + { KE_KEY, 0x121a9, { KEY_TOUCHPAD_ON } }, + { KE_KEY, 0x231b, { KEY_HELP } }, { KE_END, 0 } }; @@ -1194,6 +1197,9 @@ static int hp_wmi_platform_profile_get(struct platform_profile_handler *pprof, case HP_THERMAL_PROFILE_COOL: *profile = PLATFORM_PROFILE_COOL; break; + case HP_THERMAL_PROFILE_QUIET: + *profile = PLATFORM_PROFILE_QUIET; + break; default: return -EINVAL; } @@ -1216,6 +1222,9 @@ static int hp_wmi_platform_profile_set(struct platform_profile_handler *pprof, case PLATFORM_PROFILE_COOL: tp = HP_THERMAL_PROFILE_COOL; break; + case PLATFORM_PROFILE_QUIET: + tp = HP_THERMAL_PROFILE_QUIET; + break; default: return -EOPNOTSUPP; } @@ -1263,6 +1272,8 @@ static int thermal_profile_setup(void) platform_profile_handler.profile_get = hp_wmi_platform_profile_get; platform_profile_handler.profile_set = hp_wmi_platform_profile_set; + + set_bit(PLATFORM_PROFILE_QUIET, platform_profile_handler.choices); } set_bit(PLATFORM_PROFILE_COOL, platform_profile_handler.choices); @@ -1508,7 +1519,7 @@ static int __init hp_wmi_init(void) if (bios_capable) { hp_wmi_platform_dev = - platform_device_register_simple("hp-wmi", -1, NULL, 0); + platform_device_register_simple("hp-wmi", PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(hp_wmi_platform_dev)) { err = PTR_ERR(hp_wmi_platform_dev); goto err_destroy_input; diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c index eac3e6b4ea113064aa47f41bf587ab22a85ac68f..5873c2663a65b0704bd9912529a1928354353ed1 100644 --- a/drivers/platform/x86/huawei-wmi.c +++ b/drivers/platform/x86/huawei-wmi.c @@ -871,7 +871,7 @@ static __init int huawei_wmi_init(void) if (err) goto pdrv_err; - pdev = platform_device_register_simple("huawei-wmi", -1, NULL, 0); + pdev = platform_device_register_simple("huawei-wmi", PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pdev)) { err = PTR_ERR(pdev); goto pdev_err; diff --git a/drivers/platform/x86/intel/chtwc_int33fe.c b/drivers/platform/x86/intel/chtwc_int33fe.c index c52ac23e233157108dbbbe04cb864a780f96ccf1..2c9a7d52be0736d171b7791e9fac206ed9a8cb25 100644 --- a/drivers/platform/x86/intel/chtwc_int33fe.c +++ b/drivers/platform/x86/intel/chtwc_int33fe.c @@ -219,7 +219,7 @@ static int cht_int33fe_add_nodes(struct cht_int33fe_data *data) /* * Update node used in "usb-role-switch" property. Note that we - * rely on software_node_register_nodes() to use the original + * rely on software_node_register_node_group() to use the original * instance of properties instead of copying them. */ fusb302_mux_refs[0].node = mux_ref_node; @@ -270,7 +270,7 @@ cht_int33fe_register_max17047(struct device *dev, struct cht_int33fe_data *data) } memset(&board_info, 0, sizeof(board_info)); - strlcpy(board_info.type, "max17047", I2C_NAME_SIZE); + strscpy(board_info.type, "max17047", I2C_NAME_SIZE); board_info.dev_name = "max17047"; board_info.fwnode = fwnode; data->battery_fg = i2c_acpi_new_device(dev, 1, &board_info); @@ -361,7 +361,7 @@ static int cht_int33fe_typec_probe(struct platform_device *pdev) } memset(&board_info, 0, sizeof(board_info)); - strlcpy(board_info.type, "typec_fusb302", I2C_NAME_SIZE); + strscpy(board_info.type, "typec_fusb302", I2C_NAME_SIZE); board_info.dev_name = "fusb302"; board_info.fwnode = fwnode; board_info.irq = fusb302_irq; @@ -381,7 +381,7 @@ static int cht_int33fe_typec_probe(struct platform_device *pdev) memset(&board_info, 0, sizeof(board_info)); board_info.dev_name = "pi3usb30532"; board_info.fwnode = fwnode; - strlcpy(board_info.type, "pi3usb30532", I2C_NAME_SIZE); + strscpy(board_info.type, "pi3usb30532", I2C_NAME_SIZE); data->pi3usb30532 = i2c_acpi_new_device(dev, 3, &board_info); if (IS_ERR(data->pi3usb30532)) { diff --git a/drivers/platform/x86/intel/int0002_vgpio.c b/drivers/platform/x86/intel/int0002_vgpio.c index 617dbf98980ec56db1787c5084086410007049aa..97cfbc520a02cd1945dc98e74e004429347c23e9 100644 --- a/drivers/platform/x86/intel/int0002_vgpio.c +++ b/drivers/platform/x86/intel/int0002_vgpio.c @@ -125,8 +125,7 @@ static irqreturn_t int0002_irq(int irq, void *data) if (!(gpe_sts_reg & GPE0A_PME_B0_STS_BIT)) return IRQ_NONE; - generic_handle_irq(irq_find_mapping(chip->irq.domain, - GPE0A_PME_B0_VIRT_GPIO_PIN)); + generic_handle_domain_irq_safe(chip->irq.domain, GPE0A_PME_B0_VIRT_GPIO_PIN); pm_wakeup_hard_event(chip->parent); diff --git a/drivers/platform/x86/intel/int3472/common.c b/drivers/platform/x86/intel/int3472/common.c index 77cf058e4168f6894cf7e690203967cf9d426e8f..9db2bb0bbba4bd90c1d8d46348ad47e1b2626184 100644 --- a/drivers/platform/x86/intel/int3472/common.c +++ b/drivers/platform/x86/intel/int3472/common.c @@ -62,7 +62,7 @@ int skl_int3472_get_sensor_adev_and_name(struct device *dev, struct acpi_device *sensor; int ret = 0; - sensor = acpi_dev_get_first_consumer_dev(adev); + sensor = acpi_dev_get_next_consumer_dev(adev, NULL); if (!sensor) { dev_err(dev, "INT3472 seems to have no dependents.\n"); return -ENODEV; diff --git a/drivers/platform/x86/intel/int3472/discrete.c b/drivers/platform/x86/intel/int3472/discrete.c index ed4c9d760757449051b10d86a3d2194f26ae6950..974a132db651647ee10cb8a73fcc03b0002d54fc 100644 --- a/drivers/platform/x86/intel/int3472/discrete.c +++ b/drivers/platform/x86/intel/int3472/discrete.c @@ -331,7 +331,22 @@ static int skl_int3472_parse_crs(struct int3472_discrete_device *int3472) return 0; } -static int skl_int3472_discrete_remove(struct platform_device *pdev); +static int skl_int3472_discrete_remove(struct platform_device *pdev) +{ + struct int3472_discrete_device *int3472 = platform_get_drvdata(pdev); + + gpiod_remove_lookup_table(&int3472->gpios); + + if (int3472->clock.cl) + skl_int3472_unregister_clock(int3472); + + gpiod_put(int3472->clock.ena_gpio); + gpiod_put(int3472->clock.led_gpio); + + skl_int3472_unregister_regulator(int3472); + + return 0; +} static int skl_int3472_discrete_probe(struct platform_device *pdev) { @@ -383,23 +398,6 @@ static int skl_int3472_discrete_probe(struct platform_device *pdev) return 0; } -static int skl_int3472_discrete_remove(struct platform_device *pdev) -{ - struct int3472_discrete_device *int3472 = platform_get_drvdata(pdev); - - gpiod_remove_lookup_table(&int3472->gpios); - - if (int3472->clock.cl) - skl_int3472_unregister_clock(int3472); - - gpiod_put(int3472->clock.ena_gpio); - gpiod_put(int3472->clock.led_gpio); - - skl_int3472_unregister_regulator(int3472); - - return 0; -} - static const struct acpi_device_id int3472_device_id[] = { { "INT3472", 0 }, { } diff --git a/drivers/platform/x86/intel/int3472/tps68470.c b/drivers/platform/x86/intel/int3472/tps68470.c index 22f61b47f9e54f005e7898e74d5167bedd4bca6a..5b8d1a9620a5d06e0193fa16a9d832e17775dbc3 100644 --- a/drivers/platform/x86/intel/int3472/tps68470.c +++ b/drivers/platform/x86/intel/int3472/tps68470.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Author: Dan Scally */ +#include #include #include #include @@ -95,20 +96,65 @@ static int skl_int3472_tps68470_calc_type(struct acpi_device *adev) return DESIGNED_FOR_WINDOWS; } +/* + * Return the size of the flexible array member, because we'll need that later + * on to pass .pdata_size to cells. + */ +static int +skl_int3472_fill_clk_pdata(struct device *dev, struct tps68470_clk_platform_data **clk_pdata) +{ + struct acpi_device *adev = ACPI_COMPANION(dev); + struct acpi_device *consumer; + unsigned int n_consumers = 0; + const char *sensor_name; + unsigned int i = 0; + + for_each_acpi_consumer_dev(adev, consumer) + n_consumers++; + + if (!n_consumers) { + dev_err(dev, "INT3472 seems to have no dependents\n"); + return -ENODEV; + } + + *clk_pdata = devm_kzalloc(dev, struct_size(*clk_pdata, consumers, n_consumers), + GFP_KERNEL); + if (!*clk_pdata) + return -ENOMEM; + + (*clk_pdata)->n_consumers = n_consumers; + i = 0; + + for_each_acpi_consumer_dev(adev, consumer) { + sensor_name = devm_kasprintf(dev, GFP_KERNEL, I2C_DEV_NAME_FORMAT, + acpi_dev_name(consumer)); + if (!sensor_name) { + acpi_dev_put(consumer); + return -ENOMEM; + } + + (*clk_pdata)->consumers[i].consumer_dev_name = sensor_name; + i++; + } + + return n_consumers; +} + static int skl_int3472_tps68470_probe(struct i2c_client *client) { struct acpi_device *adev = ACPI_COMPANION(&client->dev); const struct int3472_tps68470_board_data *board_data; - struct tps68470_clk_platform_data clk_pdata = {}; + struct tps68470_clk_platform_data *clk_pdata; struct mfd_cell *cells; struct regmap *regmap; + int n_consumers; int device_type; int ret; + int i; - ret = skl_int3472_get_sensor_adev_and_name(&client->dev, NULL, - &clk_pdata.consumer_dev_name); - if (ret) - return ret; + n_consumers = skl_int3472_fill_clk_pdata(&client->dev, &clk_pdata); + if (n_consumers < 0) + return n_consumers; regmap = devm_regmap_init_i2c(client, &tps68470_regmap_config); if (IS_ERR(regmap)) { @@ -142,22 +188,25 @@ static int skl_int3472_tps68470_probe(struct i2c_client *client) * the clk + regulators must be ready when this happens. */ cells[0].name = "tps68470-clk"; - cells[0].platform_data = &clk_pdata; - cells[0].pdata_size = sizeof(clk_pdata); + cells[0].platform_data = clk_pdata; + cells[0].pdata_size = struct_size(clk_pdata, consumers, n_consumers); cells[1].name = "tps68470-regulator"; cells[1].platform_data = (void *)board_data->tps68470_regulator_pdata; cells[1].pdata_size = sizeof(struct tps68470_regulator_platform_data); cells[2].name = "tps68470-gpio"; - gpiod_add_lookup_table(board_data->tps68470_gpio_lookup_table); + for (i = 0; i < board_data->n_gpiod_lookups; i++) + gpiod_add_lookup_table(board_data->tps68470_gpio_lookup_tables[i]); ret = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_NONE, cells, TPS68470_WIN_MFD_CELL_COUNT, NULL, 0, NULL); kfree(cells); - if (ret) - gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_table); + if (ret) { + for (i = 0; i < board_data->n_gpiod_lookups; i++) + gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_tables[i]); + } break; case DESIGNED_FOR_CHROMEOS: @@ -178,15 +227,16 @@ static int skl_int3472_tps68470_probe(struct i2c_client *client) return ret; } -static int skl_int3472_tps68470_remove(struct i2c_client *client) +static void skl_int3472_tps68470_remove(struct i2c_client *client) { const struct int3472_tps68470_board_data *board_data; + int i; board_data = int3472_tps68470_get_board_data(dev_name(&client->dev)); - if (board_data) - gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_table); - - return 0; + if (board_data) { + for (i = 0; i < board_data->n_gpiod_lookups; i++) + gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_tables[i]); + } } static const struct acpi_device_id int3472_device_id[] = { diff --git a/drivers/platform/x86/intel/int3472/tps68470.h b/drivers/platform/x86/intel/int3472/tps68470.h index cfd33eb62740df8afe303ec60e03b1f503861f32..35915e70159330dbb245ad5851e03c9f506ea50f 100644 --- a/drivers/platform/x86/intel/int3472/tps68470.h +++ b/drivers/platform/x86/intel/int3472/tps68470.h @@ -16,8 +16,9 @@ struct tps68470_regulator_platform_data; struct int3472_tps68470_board_data { const char *dev_name; - struct gpiod_lookup_table *tps68470_gpio_lookup_table; const struct tps68470_regulator_platform_data *tps68470_regulator_pdata; + unsigned int n_gpiod_lookups; + struct gpiod_lookup_table *tps68470_gpio_lookup_tables[]; }; const struct int3472_tps68470_board_data *int3472_tps68470_get_board_data(const char *dev_name); diff --git a/drivers/platform/x86/intel/int3472/tps68470_board_data.c b/drivers/platform/x86/intel/int3472/tps68470_board_data.c index 525f09a3b5ff7427c76f6bf200c11cb2e35f38a6..309eab9c055888a7affde7ed898a131f072c5f9c 100644 --- a/drivers/platform/x86/intel/int3472/tps68470_board_data.c +++ b/drivers/platform/x86/intel/int3472/tps68470_board_data.c @@ -30,6 +30,15 @@ static struct regulator_consumer_supply int347a_vcm_consumer_supplies[] = { static struct regulator_consumer_supply int347a_vsio_consumer_supplies[] = { REGULATOR_SUPPLY("dovdd", "i2c-INT347A:00"), REGULATOR_SUPPLY("vsio", "i2c-INT347A:00-VCM"), + REGULATOR_SUPPLY("vddd", "i2c-INT347E:00"), +}; + +static struct regulator_consumer_supply int347a_aux1_consumer_supplies[] = { + REGULATOR_SUPPLY("vdda", "i2c-INT347E:00"), +}; + +static struct regulator_consumer_supply int347a_aux2_consumer_supplies[] = { + REGULATOR_SUPPLY("vdddo", "i2c-INT347E:00"), }; static const struct regulator_init_data surface_go_tps68470_core_reg_init_data = { @@ -86,6 +95,28 @@ static const struct regulator_init_data surface_go_tps68470_vsio_reg_init_data = .consumer_supplies = int347a_vsio_consumer_supplies, }; +static const struct regulator_init_data surface_go_tps68470_aux1_reg_init_data = { + .constraints = { + .min_uV = 2815200, + .max_uV = 2815200, + .apply_uV = 1, + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = ARRAY_SIZE(int347a_aux1_consumer_supplies), + .consumer_supplies = int347a_aux1_consumer_supplies, +}; + +static const struct regulator_init_data surface_go_tps68470_aux2_reg_init_data = { + .constraints = { + .min_uV = 1800600, + .max_uV = 1800600, + .apply_uV = 1, + .valid_ops_mask = REGULATOR_CHANGE_STATUS, + }, + .num_consumer_supplies = ARRAY_SIZE(int347a_aux2_consumer_supplies), + .consumer_supplies = int347a_aux2_consumer_supplies, +}; + static const struct tps68470_regulator_platform_data surface_go_tps68470_pdata = { .reg_init_data = { [TPS68470_CORE] = &surface_go_tps68470_core_reg_init_data, @@ -93,10 +124,12 @@ static const struct tps68470_regulator_platform_data surface_go_tps68470_pdata = [TPS68470_VCM] = &surface_go_tps68470_vcm_reg_init_data, [TPS68470_VIO] = &surface_go_tps68470_vio_reg_init_data, [TPS68470_VSIO] = &surface_go_tps68470_vsio_reg_init_data, + [TPS68470_AUX1] = &surface_go_tps68470_aux1_reg_init_data, + [TPS68470_AUX2] = &surface_go_tps68470_aux2_reg_init_data, }, }; -static struct gpiod_lookup_table surface_go_tps68470_gpios = { +static struct gpiod_lookup_table surface_go_int347a_gpios = { .dev_id = "i2c-INT347A:00", .table = { GPIO_LOOKUP("tps68470-gpio", 9, "reset", GPIO_ACTIVE_LOW), @@ -105,16 +138,31 @@ static struct gpiod_lookup_table surface_go_tps68470_gpios = { } }; +static struct gpiod_lookup_table surface_go_int347e_gpios = { + .dev_id = "i2c-INT347E:00", + .table = { + GPIO_LOOKUP("tps68470-gpio", 5, "enable", GPIO_ACTIVE_HIGH), + { } + } +}; + static const struct int3472_tps68470_board_data surface_go_tps68470_board_data = { .dev_name = "i2c-INT3472:05", - .tps68470_gpio_lookup_table = &surface_go_tps68470_gpios, .tps68470_regulator_pdata = &surface_go_tps68470_pdata, + .n_gpiod_lookups = 2, + .tps68470_gpio_lookup_tables = { + &surface_go_int347a_gpios, + &surface_go_int347e_gpios, + }, }; static const struct int3472_tps68470_board_data surface_go3_tps68470_board_data = { .dev_name = "i2c-INT3472:01", - .tps68470_gpio_lookup_table = &surface_go_tps68470_gpios, .tps68470_regulator_pdata = &surface_go_tps68470_pdata, + .n_gpiod_lookups = 1, + .tps68470_gpio_lookup_tables = { + &surface_go_int347a_gpios + }, }; static const struct dmi_system_id int3472_tps68470_board_data_table[] = { diff --git a/drivers/platform/x86/intel/oaktrail.c b/drivers/platform/x86/intel/oaktrail.c index 1a09a75bd16dfda146f65ef41fcc9cc7eece6612..7c5c623630c1447c96ac30f6944f536811f2d6a6 100644 --- a/drivers/platform/x86/intel/oaktrail.c +++ b/drivers/platform/x86/intel/oaktrail.c @@ -317,7 +317,7 @@ static int __init oaktrail_init(void) goto err_driver_reg; } - oaktrail_device = platform_device_alloc(DRIVER_NAME, -1); + oaktrail_device = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE); if (!oaktrail_device) { pr_warn("Unable to allocate platform device\n"); ret = -ENOMEM; diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c index 84eabd6156bbe73a64fb628f19c2d9146ff229c9..cb24de9e97dc5a1cca68770da5e2cc0a0a64b963 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c @@ -113,7 +113,7 @@ show_uncore_perf_status(current_freq_khz); struct uncore_data *data = container_of(attr, struct uncore_data,\ member_name##_dev_attr);\ \ - return scnprintf(buf, PAGE_SIZE, "%u\n", \ + return sysfs_emit(buf, "%u\n", \ data->member_name); \ } \ diff --git a/drivers/platform/x86/intel/wmi/thunderbolt.c b/drivers/platform/x86/intel/wmi/thunderbolt.c index 4ae87060d18b4c277de34609515f027ea116914d..fc333ff82d1ee910b7ffc327425d6a30714317e1 100644 --- a/drivers/platform/x86/intel/wmi/thunderbolt.c +++ b/drivers/platform/x86/intel/wmi/thunderbolt.c @@ -51,26 +51,7 @@ static struct attribute *tbt_attrs[] = { &dev_attr_force_power.attr, NULL }; - -static const struct attribute_group tbt_attribute_group = { - .attrs = tbt_attrs, -}; - -static int intel_wmi_thunderbolt_probe(struct wmi_device *wdev, - const void *context) -{ - int ret; - - ret = sysfs_create_group(&wdev->dev.kobj, &tbt_attribute_group); - kobject_uevent(&wdev->dev.kobj, KOBJ_CHANGE); - return ret; -} - -static void intel_wmi_thunderbolt_remove(struct wmi_device *wdev) -{ - sysfs_remove_group(&wdev->dev.kobj, &tbt_attribute_group); - kobject_uevent(&wdev->dev.kobj, KOBJ_CHANGE); -} +ATTRIBUTE_GROUPS(tbt); static const struct wmi_device_id intel_wmi_thunderbolt_id_table[] = { { .guid_string = INTEL_WMI_THUNDERBOLT_GUID }, @@ -80,9 +61,8 @@ static const struct wmi_device_id intel_wmi_thunderbolt_id_table[] = { static struct wmi_driver intel_wmi_thunderbolt_driver = { .driver = { .name = "intel-wmi-thunderbolt", + .dev_groups = tbt_groups, }, - .probe = intel_wmi_thunderbolt_probe, - .remove = intel_wmi_thunderbolt_remove, .id_table = intel_wmi_thunderbolt_id_table, }; diff --git a/drivers/platform/x86/mlx-platform.c b/drivers/platform/x86/mlx-platform.c index 5e072a0666f49d6118acf90e21e446ebb4c83fa3..2fac05a17a5c0d64bfea5d7ac6085936439bdbf5 100644 --- a/drivers/platform/x86/mlx-platform.c +++ b/drivers/platform/x86/mlx-platform.c @@ -5181,7 +5181,7 @@ static int __init mlxplat_init(void) if (!dmi_check_system(mlxplat_dmi_table)) return -ENODEV; - mlxplat_dev = platform_device_register_simple(MLX_PLAT_DEVICE_NAME, -1, + mlxplat_dev = platform_device_register_simple(MLX_PLAT_DEVICE_NAME, PLATFORM_DEVID_NONE, mlxplat_lpc_resources, ARRAY_SIZE(mlxplat_lpc_resources)); diff --git a/drivers/platform/x86/msi-laptop.c b/drivers/platform/x86/msi-laptop.c index 24ffc8e2d2d1eb2d8bcad2bffb28569edf75a1a4..6b18ec543ac3af2e89b5a093cd1bd8176b4b4ec6 100644 --- a/drivers/platform/x86/msi-laptop.c +++ b/drivers/platform/x86/msi-laptop.c @@ -53,8 +53,6 @@ #include #include -#define MSI_DRIVER_VERSION "0.5" - #define MSI_LCD_LEVEL_MAX 9 #define MSI_EC_COMMAND_WIRELESS 0x10 @@ -592,15 +590,22 @@ static int dmi_check_cb(const struct dmi_system_id *dmi) return 1; } +static unsigned long msi_work_delay(int msecs) +{ + if (quirks->ec_delay) + return msecs_to_jiffies(msecs); + + return 0; +} + static const struct dmi_system_id msi_dmi_table[] __initconst = { { .ident = "MSI S270", .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "MICRO-STAR INT'L CO.,LTD"), + DMI_MATCH(DMI_SYS_VENDOR, "MICRO-STAR INT"), DMI_MATCH(DMI_PRODUCT_NAME, "MS-1013"), DMI_MATCH(DMI_PRODUCT_VERSION, "0131"), - DMI_MATCH(DMI_CHASSIS_VENDOR, - "MICRO-STAR INT'L CO.,LTD") + DMI_MATCH(DMI_CHASSIS_VENDOR, "MICRO-STAR INT") }, .driver_data = &quirk_old_ec_model, .callback = dmi_check_cb @@ -633,8 +638,7 @@ static const struct dmi_system_id msi_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "NOTEBOOK"), DMI_MATCH(DMI_PRODUCT_NAME, "SAM2000"), DMI_MATCH(DMI_PRODUCT_VERSION, "0131"), - DMI_MATCH(DMI_CHASSIS_VENDOR, - "MICRO-STAR INT'L CO.,LTD") + DMI_MATCH(DMI_CHASSIS_VENDOR, "MICRO-STAR INT") }, .driver_data = &quirk_old_ec_model, .callback = dmi_check_cb @@ -705,6 +709,7 @@ static const struct dmi_system_id msi_dmi_table[] __initconst = { }, { } }; +MODULE_DEVICE_TABLE(dmi, msi_dmi_table); static int rfkill_bluetooth_set(void *data, bool blocked) { @@ -785,7 +790,6 @@ static void msi_update_rfkill(struct work_struct *ignored) msi_rfkill_set_state(rfk_threeg, !threeg_s); } static DECLARE_DELAYED_WORK(msi_rfkill_dwork, msi_update_rfkill); -static DECLARE_WORK(msi_rfkill_work, msi_update_rfkill); static void msi_send_touchpad_key(struct work_struct *ignored) { @@ -801,7 +805,6 @@ static void msi_send_touchpad_key(struct work_struct *ignored) KEY_TOUCHPAD_ON : KEY_TOUCHPAD_OFF, 1, true); } static DECLARE_DELAYED_WORK(msi_touchpad_dwork, msi_send_touchpad_key); -static DECLARE_WORK(msi_touchpad_work, msi_send_touchpad_key); static bool msi_laptop_i8042_filter(unsigned char data, unsigned char str, struct serio *port) @@ -819,20 +822,12 @@ static bool msi_laptop_i8042_filter(unsigned char data, unsigned char str, extended = false; switch (data) { case 0xE4: - if (quirks->ec_delay) { - schedule_delayed_work(&msi_touchpad_dwork, - round_jiffies_relative(0.5 * HZ)); - } else - schedule_work(&msi_touchpad_work); + schedule_delayed_work(&msi_touchpad_dwork, msi_work_delay(500)); break; case 0x54: case 0x62: case 0x76: - if (quirks->ec_delay) { - schedule_delayed_work(&msi_rfkill_dwork, - round_jiffies_relative(0.5 * HZ)); - } else - schedule_work(&msi_rfkill_work); + schedule_delayed_work(&msi_rfkill_dwork, msi_work_delay(500)); break; } } @@ -899,12 +894,7 @@ static int rfkill_init(struct platform_device *sdev) } /* schedule to run rfkill state initial */ - if (quirks->ec_delay) { - schedule_delayed_work(&msi_rfkill_init, - round_jiffies_relative(1 * HZ)); - } else - schedule_work(&msi_rfkill_work); - + schedule_delayed_work(&msi_rfkill_init, msi_work_delay(1000)); return 0; err_threeg: @@ -921,8 +911,7 @@ err_bluetooth: return retval; } -#ifdef CONFIG_PM_SLEEP -static int msi_laptop_resume(struct device *device) +static int msi_scm_disable_hw_fn_handling(void) { u8 data; int result; @@ -942,6 +931,12 @@ static int msi_laptop_resume(struct device *device) return 0; } + +#ifdef CONFIG_PM_SLEEP +static int msi_laptop_resume(struct device *device) +{ + return msi_scm_disable_hw_fn_handling(); +} #endif static int __init msi_laptop_input_setup(void) @@ -974,7 +969,6 @@ err_free_dev: static int __init load_scm_model_init(struct platform_device *sdev) { - u8 data; int result; if (!quirks->ec_read_only) { @@ -988,12 +982,7 @@ static int __init load_scm_model_init(struct platform_device *sdev) } /* disable hardware control by fn key */ - result = ec_read(MSI_STANDARD_EC_SCM_LOAD_ADDRESS, &data); - if (result < 0) - return result; - - result = ec_write(MSI_STANDARD_EC_SCM_LOAD_ADDRESS, - data | MSI_STANDARD_EC_SCM_LOAD_MASK); + result = msi_scm_disable_hw_fn_handling(); if (result < 0) return result; @@ -1022,9 +1011,19 @@ fail_input: rfkill_cleanup(); fail_rfkill: - return result; +} + +static void msi_scm_model_exit(void) +{ + if (!quirks->load_scm_model) + return; + i8042_remove_filter(msi_laptop_i8042_filter); + cancel_delayed_work_sync(&msi_touchpad_dwork); + input_unregister_device(msi_laptop_input_dev); + cancel_delayed_work_sync(&msi_rfkill_dwork); + rfkill_cleanup(); } static int __init msi_init(void) @@ -1048,8 +1047,7 @@ static int __init msi_init(void) return -EINVAL; /* Register backlight stuff */ - - if (quirks->old_ec_model || + if (quirks->old_ec_model && acpi_video_get_backlight_type() == acpi_backlight_vendor) { struct backlight_properties props; memset(&props, 0, sizeof(struct backlight_properties)); @@ -1068,7 +1066,7 @@ static int __init msi_init(void) /* Register platform stuff */ - msipf_device = platform_device_alloc("msi-laptop-pf", -1); + msipf_device = platform_device_alloc("msi-laptop-pf", PLATFORM_DEVID_NONE); if (!msipf_device) { ret = -ENOMEM; goto fail_platform_driver; @@ -1108,19 +1106,12 @@ static int __init msi_init(void) set_auto_brightness(auto_brightness); } - pr_info("driver " MSI_DRIVER_VERSION " successfully loaded\n"); - return 0; fail_create_attr: sysfs_remove_group(&msipf_device->dev.kobj, &msipf_attribute_group); fail_create_group: - if (quirks->load_scm_model) { - i8042_remove_filter(msi_laptop_i8042_filter); - cancel_delayed_work_sync(&msi_rfkill_dwork); - cancel_work_sync(&msi_rfkill_work); - rfkill_cleanup(); - } + msi_scm_model_exit(); fail_scm_model_init: platform_device_del(msipf_device); fail_device_add: @@ -1135,14 +1126,7 @@ fail_backlight: static void __exit msi_cleanup(void) { - if (quirks->load_scm_model) { - i8042_remove_filter(msi_laptop_i8042_filter); - input_unregister_device(msi_laptop_input_dev); - cancel_delayed_work_sync(&msi_rfkill_dwork); - cancel_work_sync(&msi_rfkill_work); - rfkill_cleanup(); - } - + msi_scm_model_exit(); sysfs_remove_group(&msipf_device->dev.kobj, &msipf_attribute_group); if (!quirks->old_ec_model && threeg_exists) device_remove_file(&msipf_device->dev, &dev_attr_threeg); @@ -1155,8 +1139,6 @@ static void __exit msi_cleanup(void) if (auto_brightness != 2) set_auto_brightness(1); } - - pr_info("driver unloaded\n"); } module_init(msi_init); @@ -1164,16 +1146,4 @@ module_exit(msi_cleanup); MODULE_AUTHOR("Lennart Poettering"); MODULE_DESCRIPTION("MSI Laptop Support"); -MODULE_VERSION(MSI_DRIVER_VERSION); MODULE_LICENSE("GPL"); - -MODULE_ALIAS("dmi:*:svnMICRO-STARINT'LCO.,LTD:pnMS-1013:pvr0131*:cvnMICRO-STARINT'LCO.,LTD:ct10:*"); -MODULE_ALIAS("dmi:*:svnMicro-StarInternational:pnMS-1058:pvr0581:rvnMSI:rnMS-1058:*:ct10:*"); -MODULE_ALIAS("dmi:*:svnMicro-StarInternational:pnMS-1412:*:rvnMSI:rnMS-1412:*:cvnMICRO-STARINT'LCO.,LTD:ct10:*"); -MODULE_ALIAS("dmi:*:svnNOTEBOOK:pnSAM2000:pvr0131*:cvnMICRO-STARINT'LCO.,LTD:ct10:*"); -MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N034:*"); -MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N051:*"); -MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N014:*"); -MODULE_ALIAS("dmi:*:svnMicro-StarInternational*:pnCR620:*"); -MODULE_ALIAS("dmi:*:svnMicro-StarInternational*:pnU270series:*"); -MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnU90/U100:*"); diff --git a/drivers/platform/x86/p2sb.c b/drivers/platform/x86/p2sb.c index fb2e141f3eb84bd58916103feb4c7e81f85618a7..384d0962ae93ab0d840d9e19f00ca85cdcd46bc3 100644 --- a/drivers/platform/x86/p2sb.c +++ b/drivers/platform/x86/p2sb.c @@ -42,10 +42,24 @@ static int p2sb_get_devfn(unsigned int *devfn) return 0; } +/* Copy resource from the first BAR of the device in question */ static int p2sb_read_bar0(struct pci_dev *pdev, struct resource *mem) { - /* Copy resource from the first BAR of the device in question */ - *mem = pdev->resource[0]; + struct resource *bar0 = &pdev->resource[0]; + + /* Make sure we have no dangling pointers in the output */ + memset(mem, 0, sizeof(*mem)); + + /* + * We copy only selected fields from the original resource. + * Because a PCI device will be removed soon, we may not use + * any allocated data, hence we may not copy any pointers. + */ + mem->start = bar0->start; + mem->end = bar0->end; + mem->flags = bar0->flags; + mem->desc = bar0->desc; + return 0; } diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c index d9a095d2c0eb039d15b4250c99b7c6eaaff69c40..ad3083f9946d4d8f9881a4db3b6c5db21ea02a3b 100644 --- a/drivers/platform/x86/panasonic-laptop.c +++ b/drivers/platform/x86/panasonic-laptop.c @@ -1034,7 +1034,7 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) /* optical drive initialization */ if (ACPI_SUCCESS(check_optd_present())) { pcc->platform = platform_device_register_simple("panasonic", - -1, NULL, 0); + PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pcc->platform)) { result = PTR_ERR(pcc->platform); goto out_backlight; diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c index 154317e9910d2a82d1f97689d4ac907ba85841fc..93a6414c661141d6078d6cb55a91a30f057240c3 100644 --- a/drivers/platform/x86/pmc_atom.c +++ b/drivers/platform/x86/pmc_atom.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Intel Atom SOC Power Management Controller Driver - * Copyright (c) 2014, Intel Corporation. + * Intel Atom SoC Power Management Controller Driver + * Copyright (c) 2014-2015,2017,2022 Intel Corporation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -60,7 +60,7 @@ static const struct pmc_clk byt_clks[] = { .freq = 19200000, .parent_name = "xtal", }, - {}, + {} }; static const struct pmc_clk cht_clks[] = { @@ -69,7 +69,7 @@ static const struct pmc_clk cht_clks[] = { .freq = 19200000, .parent_name = NULL, }, - {}, + {} }; static const struct pmc_bit_map d3_sts_0_map[] = { @@ -105,7 +105,7 @@ static const struct pmc_bit_map d3_sts_0_map[] = { {"LPSS2_F5_I2C5", BIT_LPSS2_F5_I2C5}, {"LPSS2_F6_I2C6", BIT_LPSS2_F6_I2C6}, {"LPSS2_F7_I2C7", BIT_LPSS2_F7_I2C7}, - {}, + {} }; static struct pmc_bit_map byt_d3_sts_1_map[] = { @@ -113,21 +113,21 @@ static struct pmc_bit_map byt_d3_sts_1_map[] = { {"OTG_SS_PHY", BIT_OTG_SS_PHY}, {"USH_SS_PHY", BIT_USH_SS_PHY}, {"DFX", BIT_DFX}, - {}, + {} }; static struct pmc_bit_map cht_d3_sts_1_map[] = { {"SMB", BIT_SMB}, {"GMM", BIT_STS_GMM}, {"ISH", BIT_STS_ISH}, - {}, + {} }; static struct pmc_bit_map cht_func_dis_2_map[] = { {"SMB", BIT_SMB}, {"GMM", BIT_FD_GMM}, {"ISH", BIT_FD_ISH}, - {}, + {} }; static const struct pmc_bit_map byt_pss_map[] = { @@ -149,7 +149,7 @@ static const struct pmc_bit_map byt_pss_map[] = { {"OTG_VCCA", PMC_PSS_BIT_OTG_VCCA}, {"USB", PMC_PSS_BIT_USB}, {"USB_SUS", PMC_PSS_BIT_USB_SUS}, - {}, + {} }; static const struct pmc_bit_map cht_pss_map[] = { @@ -172,7 +172,7 @@ static const struct pmc_bit_map cht_pss_map[] = { {"DFX_CLUSTER3", PMC_PSS_BIT_CHT_DFX_CLUSTER3}, {"DFX_CLUSTER4", PMC_PSS_BIT_CHT_DFX_CLUSTER4}, {"DFX_CLUSTER5", PMC_PSS_BIT_CHT_DFX_CLUSTER5}, - {}, + {} }; static const struct pmc_reg_map byt_reg_map = { @@ -232,7 +232,7 @@ static void pmc_power_off(void) pm1_cnt_port = acpi_base_addr + PM1_CNT; pm1_cnt_value = inl(pm1_cnt_port); - pm1_cnt_value &= SLEEP_TYPE_MASK; + pm1_cnt_value &= ~SLEEP_TYPE_MASK; pm1_cnt_value |= SLEEP_TYPE_S5; pm1_cnt_value |= SLEEP_ENABLE; @@ -354,7 +354,7 @@ static bool pmc_clk_is_critical = true; static int dmi_callback(const struct dmi_system_id *d) { - pr_info("%s critclks quirk enabled\n", d->ident); + pr_info("%s: PMC critical clocks quirk enabled\n", d->ident); return 1; } @@ -417,8 +417,7 @@ static const struct dmi_system_id critclk_systems[] = { DMI_MATCH(DMI_SYS_VENDOR, "SIEMENS AG"), }, }, - - { /*sentinel*/ } + {} }; static int pmc_setup_clks(struct pci_dev *pdev, void __iomem *pmc_regmap, @@ -490,15 +489,11 @@ static int pmc_setup_dev(struct pci_dev *pdev, const struct pci_device_id *ent) return ret; } -/* - * Data for PCI driver interface - * - * used by pci_match_id() call below. - */ +/* Data for PCI driver interface used by pci_match_id() call below */ static const struct pci_device_id pmc_pci_ids[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_VLV_PMC), (kernel_ulong_t)&byt_data }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_CHT_PMC), (kernel_ulong_t)&cht_data }, - { 0, }, + {} }; static int __init pmc_atom_init(void) @@ -506,8 +501,9 @@ static int __init pmc_atom_init(void) struct pci_dev *pdev = NULL; const struct pci_device_id *ent; - /* We look for our device - PCU PMC - * we assume that there is max. one device. + /* + * We look for our device - PCU PMC. + * We assume that there is maximum one device. * * We can't use plain pci_driver mechanism, * as the device is really a multiple function device, @@ -519,7 +515,7 @@ static int __init pmc_atom_init(void) if (ent) return pmc_setup_dev(pdev, ent); } - /* Device not found. */ + /* Device not found */ return -ENODEV; } @@ -527,6 +523,6 @@ device_initcall(pmc_atom_init); /* MODULE_AUTHOR("Aubrey Li "); -MODULE_DESCRIPTION("Intel Atom SOC Power Management Controller Interface"); +MODULE_DESCRIPTION("Intel Atom SoC Power Management Controller Interface"); MODULE_LICENSE("GPL v2"); */ diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c index cc30cf08f32da8a7cdb1900b935b773e3f882639..b4aa8ba35d2d7df1be52ab171f4facf61e1fedf7 100644 --- a/drivers/platform/x86/samsung-laptop.c +++ b/drivers/platform/x86/samsung-laptop.c @@ -1474,7 +1474,7 @@ static int __init samsung_platform_init(struct samsung_laptop *samsung) { struct platform_device *pdev; - pdev = platform_device_register_simple("samsung", -1, NULL, 0); + pdev = platform_device_register_simple("samsung", PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(pdev)) return PTR_ERR(pdev); diff --git a/drivers/platform/x86/simatic-ipc.c b/drivers/platform/x86/simatic-ipc.c index ca3647b751d5d613736008990d2475589568c7ea..ca76076fc706a62def16d7bdd0b69baca7a3adfd 100644 --- a/drivers/platform/x86/simatic-ipc.c +++ b/drivers/platform/x86/simatic-ipc.c @@ -41,10 +41,12 @@ static struct { {SIMATIC_IPC_IPC127E, SIMATIC_IPC_DEVICE_127E, SIMATIC_IPC_DEVICE_NONE}, {SIMATIC_IPC_IPC227D, SIMATIC_IPC_DEVICE_227D, SIMATIC_IPC_DEVICE_NONE}, {SIMATIC_IPC_IPC227E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_227E}, + {SIMATIC_IPC_IPC227G, SIMATIC_IPC_DEVICE_227G, SIMATIC_IPC_DEVICE_227G}, {SIMATIC_IPC_IPC277E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_227E}, {SIMATIC_IPC_IPC427D, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_NONE}, {SIMATIC_IPC_IPC427E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_427E}, {SIMATIC_IPC_IPC477E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_427E}, + {SIMATIC_IPC_IPC427G, SIMATIC_IPC_DEVICE_227G, SIMATIC_IPC_DEVICE_227G}, }; static int register_platform_devices(u32 station_id) @@ -65,7 +67,8 @@ static int register_platform_devices(u32 station_id) } if (ledmode != SIMATIC_IPC_DEVICE_NONE) { - if (ledmode == SIMATIC_IPC_DEVICE_127E) + if (ledmode == SIMATIC_IPC_DEVICE_127E || + ledmode == SIMATIC_IPC_DEVICE_227G) pdevname = KBUILD_MODNAME "_leds_gpio"; platform_data.devmode = ledmode; ipc_led_platform_device = @@ -80,6 +83,11 @@ static int register_platform_devices(u32 station_id) ipc_led_platform_device->name); } + if (wdtmode == SIMATIC_IPC_DEVICE_227G) { + request_module("w83627hf_wdt"); + return 0; + } + if (wdtmode != SIMATIC_IPC_DEVICE_NONE) { platform_data.devmode = wdtmode; ipc_wdt_platform_device = diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c index 07ef05f727a2c07b4d59314660cb2eadaf949f59..765fcaba4d121ea09752b7554437788b1743b0b0 100644 --- a/drivers/platform/x86/sony-laptop.c +++ b/drivers/platform/x86/sony-laptop.c @@ -584,7 +584,7 @@ static int sony_pf_add(void) if (ret) goto out; - sony_pf_device = platform_device_alloc("sony-laptop", -1); + sony_pf_device = platform_device_alloc("sony-laptop", PLATFORM_DEVID_NONE); if (!sony_pf_device) { ret = -ENOMEM; goto out_platform_registered; diff --git a/drivers/platform/x86/tc1100-wmi.c b/drivers/platform/x86/tc1100-wmi.c index 9072eb30261856d23e5fc8a1f49e99270d0c101f..ded26213c42025edcf6b26c04e54ab06eebdf0f0 100644 --- a/drivers/platform/x86/tc1100-wmi.c +++ b/drivers/platform/x86/tc1100-wmi.c @@ -233,7 +233,7 @@ static int __init tc1100_init(void) if (!wmi_has_guid(GUID)) return -ENODEV; - tc1100_device = platform_device_alloc("tc1100-wmi", -1); + tc1100_device = platform_device_alloc("tc1100-wmi", PLATFORM_DEVID_NONE); if (!tc1100_device) return -ENOMEM; diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index 22d4e8633e30e953cec9a558751bd218f17ce26c..6a823b850a7780dedd72ea9c66a24a886c4e3180 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -7623,9 +7623,9 @@ static int __init volume_create_alsa_mixer(void) data = card->private_data; data->card = card; - strlcpy(card->driver, TPACPI_ALSA_DRVNAME, + strscpy(card->driver, TPACPI_ALSA_DRVNAME, sizeof(card->driver)); - strlcpy(card->shortname, TPACPI_ALSA_SHRTNAME, + strscpy(card->shortname, TPACPI_ALSA_SHRTNAME, sizeof(card->shortname)); snprintf(card->mixername, sizeof(card->mixername), "ThinkPad EC %s", (thinkpad_id.ec_version_str) ? @@ -10592,10 +10592,9 @@ static int tpacpi_dytc_profile_init(struct ibm_init_struct *iibm) /* Ensure initial values are correct */ dytc_profile_refresh(); - /* Set AMT correctly now we know current profile */ - if ((dytc_capabilities & BIT(DYTC_FC_PSC)) && - (dytc_capabilities & BIT(DYTC_FC_AMT))) - dytc_control_amt(dytc_current_profile == PLATFORM_PROFILE_BALANCED); + /* Workaround for https://bugzilla.kernel.org/show_bug.cgi?id=216347 */ + if (dytc_capabilities & BIT(DYTC_FC_PSC)) + dytc_profile_set(NULL, PLATFORM_PROFILE_BALANCED); return 0; } @@ -11716,7 +11715,7 @@ static int __init thinkpad_acpi_module_init(void) tp_features.quirks = dmi_id->driver_data; /* Device initialization */ - tpacpi_pdev = platform_device_register_simple(TPACPI_DRVR_NAME, -1, + tpacpi_pdev = platform_device_register_simple(TPACPI_DRVR_NAME, PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(tpacpi_pdev)) { ret = PTR_ERR(tpacpi_pdev); @@ -11727,7 +11726,7 @@ static int __init thinkpad_acpi_module_init(void) } tpacpi_sensors_pdev = platform_device_register_simple( TPACPI_HWMON_DRVR_NAME, - -1, NULL, 0); + PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(tpacpi_sensors_pdev)) { ret = PTR_ERR(tpacpi_sensors_pdev); tpacpi_sensors_pdev = NULL; diff --git a/drivers/platform/x86/topstar-laptop.c b/drivers/platform/x86/topstar-laptop.c index f7761d98c0fd00aeffb19a9192bd8e1a2e45a941..6d18fbf8762b1bfcc03b171286c75f27ee6859e8 100644 --- a/drivers/platform/x86/topstar-laptop.c +++ b/drivers/platform/x86/topstar-laptop.c @@ -192,7 +192,7 @@ static int topstar_platform_init(struct topstar_laptop *topstar) { int err; - topstar->platform = platform_device_alloc(TOPSTAR_LAPTOP_CLASS, -1); + topstar->platform = platform_device_alloc(TOPSTAR_LAPTOP_CLASS, PLATFORM_DEVID_NONE); if (!topstar->platform) return -ENOMEM; diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c index 030dc37d50b82f1837b743ef3af1ba3448aa3e66..160abd3b3af8be37b2abf7018e53d552e20c7591 100644 --- a/drivers/platform/x86/toshiba_acpi.c +++ b/drivers/platform/x86/toshiba_acpi.c @@ -23,6 +23,7 @@ #define PROC_INTERFACE_VERSION 1 #include +#include #include #include #include @@ -38,18 +39,24 @@ #include #include #include -#include #include #include #include +#include #include #include +#include #include MODULE_AUTHOR("John Belmonte"); MODULE_DESCRIPTION("Toshiba Laptop ACPI Extras Driver"); MODULE_LICENSE("GPL"); +static int turn_on_panel_on_resume = -1; +module_param(turn_on_panel_on_resume, int, 0644); +MODULE_PARM_DESC(turn_on_panel_on_resume, + "Call HCI_PANEL_POWER_ON on resume (-1 = auto, 0 = no, 1 = yes"); + #define TOSHIBA_WMI_EVENT_GUID "59142400-C6A3-40FA-BADB-8A2652834100" /* Scan code for Fn key on TOS1900 models */ @@ -100,18 +107,21 @@ MODULE_LICENSE("GPL"); #define TOS_NOT_INSTALLED 0x8e00 /* Registers */ +#define HCI_PANEL_POWER_ON 0x0002 #define HCI_FAN 0x0004 #define HCI_TR_BACKLIGHT 0x0005 #define HCI_SYSTEM_EVENT 0x0016 #define HCI_VIDEO_OUT 0x001c #define HCI_HOTKEY_EVENT 0x001e #define HCI_LCD_BRIGHTNESS 0x002a +#define HCI_FAN_RPM 0x0045 #define HCI_WIRELESS 0x0056 #define HCI_ACCELEROMETER 0x006d #define HCI_COOLING_METHOD 0x007f #define HCI_KBD_ILLUMINATION 0x0095 #define HCI_ECO_MODE 0x0097 #define HCI_ACCELEROMETER2 0x00a6 +#define HCI_BATTERY_CHARGE_MODE 0x00ba #define HCI_SYSTEM_INFO 0xc000 #define SCI_PANEL_POWER_ON 0x010d #define SCI_ILLUMINATION 0x014e @@ -170,6 +180,9 @@ struct toshiba_acpi_dev { struct miscdevice miscdev; struct rfkill *wwan_rfk; struct iio_dev *indio_dev; +#if IS_ENABLED(CONFIG_HWMON) + struct device *hwmon_device; +#endif int force_fan; int last_key_event; @@ -185,6 +198,7 @@ struct toshiba_acpi_dev { unsigned int illumination_supported:1; unsigned int video_supported:1; unsigned int fan_supported:1; + unsigned int fan_rpm_supported:1; unsigned int system_event_supported:1; unsigned int ntfy_supported:1; unsigned int info_supported:1; @@ -201,6 +215,7 @@ struct toshiba_acpi_dev { unsigned int usb_three_supported:1; unsigned int wwan_supported:1; unsigned int cooling_method_supported:1; + unsigned int battery_charge_mode_supported:1; unsigned int sysfs_created:1; unsigned int special_functions; @@ -667,12 +682,15 @@ static void toshiba_eco_mode_available(struct toshiba_acpi_dev *dev) return; } - if (out[0] == TOS_INPUT_DATA_ERROR) { + if (out[0] == TOS_INPUT_DATA_ERROR || out[0] == TOS_NOT_SUPPORTED) { /* * If we receive 0x8300 (Input Data Error), it means that the * LED device is present, but that we just screwed the input * parameters. * + * On some laptops 0x8000 (Not supported) is also returned in + * this case, so we need to allow for that as well. + * * Let's query the status of the LED to see if we really have a * success response, indicating the actual presense of the LED, * bail out otherwise. @@ -1274,6 +1292,69 @@ static int toshiba_cooling_method_set(struct toshiba_acpi_dev *dev, u32 state) return (result == TOS_SUCCESS || result == TOS_SUCCESS2) ? 0 : -EIO; } +/* Battery charge control */ +static void toshiba_battery_charge_mode_available(struct toshiba_acpi_dev *dev) +{ + u32 in[TCI_WORDS] = { HCI_GET, HCI_BATTERY_CHARGE_MODE, 0, 0, 0, 0 }; + u32 out[TCI_WORDS]; + acpi_status status; + + dev->battery_charge_mode_supported = 0; + + status = tci_raw(dev, in, out); + if (ACPI_FAILURE(status)) { + pr_err("ACPI call to get Battery Charge Mode failed\n"); + return; + } + + if (out[0] != TOS_SUCCESS && out[0] != TOS_SUCCESS2) + return; + + dev->battery_charge_mode_supported = 1; +} + +static int toshiba_battery_charge_mode_get(struct toshiba_acpi_dev *dev, u32 *state) +{ + u32 in[TCI_WORDS] = { HCI_GET, HCI_BATTERY_CHARGE_MODE, 0, 0, 0, 0x1 }; + u32 out[TCI_WORDS]; + int retries = 3; + + do { + acpi_status status = tci_raw(dev, in, out); + + if (ACPI_FAILURE(status)) + pr_err("ACPI call to get Battery Charge Mode failed\n"); + switch (out[0]) { + case TOS_SUCCESS: + case TOS_SUCCESS2: + *state = out[2]; + return 0; + case TOS_NOT_SUPPORTED: + return -ENODEV; + case TOS_DATA_NOT_AVAILABLE: + retries--; + break; + default: + return -EIO; + } + } while (retries); + + return -EIO; +} + +static int toshiba_battery_charge_mode_set(struct toshiba_acpi_dev *dev, u32 state) +{ + u32 result = hci_write(dev, HCI_BATTERY_CHARGE_MODE, state); + + if (result == TOS_FAILURE) + pr_err("ACPI call to set Battery Charge Mode failed\n"); + + if (result == TOS_NOT_SUPPORTED) + return -ENODEV; + + return (result == TOS_SUCCESS || result == TOS_SUCCESS2) ? 0 : -EIO; +} + /* Transflective Backlight */ static int get_tr_backlight_status(struct toshiba_acpi_dev *dev, u32 *status) { @@ -1608,6 +1689,29 @@ static const struct proc_ops fan_proc_ops = { .proc_write = fan_proc_write, }; +/* Fan RPM */ +static int get_fan_rpm(struct toshiba_acpi_dev *dev, u32 *rpm) +{ + u32 in[TCI_WORDS] = { HCI_GET, HCI_FAN_RPM, 0, 1, 0, 0 }; + u32 out[TCI_WORDS]; + acpi_status status = tci_raw(dev, in, out); + + if (ACPI_FAILURE(status)) { + pr_err("ACPI call to get Fan speed failed\n"); + return -EIO; + } + + if (out[0] == TOS_NOT_SUPPORTED) + return -ENODEV; + + if (out[0] == TOS_SUCCESS) { + *rpm = out[2]; + return 0; + } + + return -EIO; +} + static int keys_proc_show(struct seq_file *m, void *v) { struct toshiba_acpi_dev *dev = m->private; @@ -2778,6 +2882,7 @@ static int toshiba_acpi_setup_keyboard(struct toshiba_acpi_dev *dev) dev->hotkey_dev->name = "Toshiba input device"; dev->hotkey_dev->phys = "toshiba_acpi/input0"; dev->hotkey_dev->id.bustype = BUS_HOST; + dev->hotkey_dev->dev.parent = &dev->acpi_dev->dev; if (dev->hotkey_event_type == HCI_SYSTEM_TYPE1 || !dev->kbd_function_keys_supported) @@ -2900,6 +3005,139 @@ static int toshiba_acpi_setup_backlight(struct toshiba_acpi_dev *dev) return 0; } +/* HWMON support for fan */ +#if IS_ENABLED(CONFIG_HWMON) +static umode_t toshiba_acpi_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + return 0444; +} + +static int toshiba_acpi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + /* + * There is only a single channel and single attribute (for the + * fan) at this point. + * This can be replaced with more advanced logic in the future, + * should the need arise. + */ + if (type == hwmon_fan && channel == 0 && attr == hwmon_fan_input) { + u32 value; + int ret; + + ret = get_fan_rpm(toshiba_acpi, &value); + if (ret) + return ret; + + *val = value; + return 0; + } + return -EOPNOTSUPP; +} + +static const struct hwmon_channel_info *toshiba_acpi_hwmon_info[] = { + HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT), + NULL +}; + +static const struct hwmon_ops toshiba_acpi_hwmon_ops = { + .is_visible = toshiba_acpi_hwmon_is_visible, + .read = toshiba_acpi_hwmon_read, +}; + +static const struct hwmon_chip_info toshiba_acpi_hwmon_chip_info = { + .ops = &toshiba_acpi_hwmon_ops, + .info = toshiba_acpi_hwmon_info, +}; +#endif + +/* ACPI battery hooking */ +static ssize_t charge_control_end_threshold_show(struct device *device, + struct device_attribute *attr, + char *buf) +{ + u32 state; + int status; + + if (toshiba_acpi == NULL) { + pr_err("Toshiba ACPI object invalid\n"); + return -ENODEV; + } + + status = toshiba_battery_charge_mode_get(toshiba_acpi, &state); + + if (status != 0) + return status; + + if (state == 1) + return sprintf(buf, "80\n"); + else + return sprintf(buf, "100\n"); +} + +static ssize_t charge_control_end_threshold_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + u32 value; + int rval; + + if (toshiba_acpi == NULL) { + pr_err("Toshiba ACPI object invalid\n"); + return -ENODEV; + } + + rval = kstrtou32(buf, 10, &value); + if (rval) + return rval; + + if (value < 1 || value > 100) + return -EINVAL; + rval = toshiba_battery_charge_mode_set(toshiba_acpi, + (value < 90) ? 1 : 0); + if (rval < 0) + return rval; + else + return count; +} + +static DEVICE_ATTR_RW(charge_control_end_threshold); + +static struct attribute *toshiba_acpi_battery_attrs[] = { + &dev_attr_charge_control_end_threshold.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(toshiba_acpi_battery); + +static int toshiba_acpi_battery_add(struct power_supply *battery) +{ + if (toshiba_acpi == NULL) { + pr_err("Init order issue\n"); + return -ENODEV; + } + if (!toshiba_acpi->battery_charge_mode_supported) + return -ENODEV; + if (device_add_groups(&battery->dev, toshiba_acpi_battery_groups)) + return -ENODEV; + return 0; +} + +static int toshiba_acpi_battery_remove(struct power_supply *battery) +{ + device_remove_groups(&battery->dev, toshiba_acpi_battery_groups); + return 0; +} + +static struct acpi_battery_hook battery_hook = { + .add_battery = toshiba_acpi_battery_add, + .remove_battery = toshiba_acpi_battery_remove, + .name = "Toshiba Battery Extension", +}; + static void print_supported_features(struct toshiba_acpi_dev *dev) { pr_info("Supported laptop features:"); @@ -2912,6 +3150,8 @@ static void print_supported_features(struct toshiba_acpi_dev *dev) pr_cont(" video-out"); if (dev->fan_supported) pr_cont(" fan"); + if (dev->fan_rpm_supported) + pr_cont(" fan-rpm"); if (dev->tr_backlight_supported) pr_cont(" transflective-backlight"); if (dev->illumination_supported) @@ -2940,6 +3180,8 @@ static void print_supported_features(struct toshiba_acpi_dev *dev) pr_cont(" wwan"); if (dev->cooling_method_supported) pr_cont(" cooling-method"); + if (dev->battery_charge_mode_supported) + pr_cont(" battery-charge-mode"); pr_cont("\n"); } @@ -2952,6 +3194,11 @@ static int toshiba_acpi_remove(struct acpi_device *acpi_dev) remove_toshiba_proc_entries(dev); +#if IS_ENABLED(CONFIG_HWMON) + if (dev->hwmon_device) + hwmon_device_unregister(dev->hwmon_device); +#endif + if (dev->accelerometer_supported && dev->indio_dev) { iio_device_unregister(dev->indio_dev); iio_device_free(dev->indio_dev); @@ -2980,6 +3227,9 @@ static int toshiba_acpi_remove(struct acpi_device *acpi_dev) rfkill_destroy(dev->wwan_rfk); } + if (dev->battery_charge_mode_supported) + battery_hook_unregister(&battery_hook); + if (toshiba_acpi) toshiba_acpi = NULL; @@ -2999,6 +3249,43 @@ static const char *find_hci_method(acpi_handle handle) return NULL; } +/* + * Some Toshibas have a broken acpi-video interface for brightness control, + * these are quirked in drivers/acpi/video_detect.c to use the GPU native + * (/sys/class/backlight/intel_backlight) instead. + * But these need a HCI_SET call to actually turn the panel back on at resume, + * without this call the screen stays black at resume. + * Either HCI_LCD_BRIGHTNESS (used by acpi_video's _BCM) or HCI_PANEL_POWER_ON + * works. toshiba_acpi_resume() uses HCI_PANEL_POWER_ON to avoid changing + * the configured brightness level. + */ +static const struct dmi_system_id turn_on_panel_on_resume_dmi_ids[] = { + { + /* Toshiba Portégé R700 */ + /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"), + }, + }, + { + /* Toshiba Satellite/Portégé R830 */ + /* Portégé: https://bugs.freedesktop.org/show_bug.cgi?id=82634 */ + /* Satellite: https://bugzilla.kernel.org/show_bug.cgi?id=21012 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "R830"), + }, + }, + { + /* Toshiba Satellite/Portégé Z830 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_MATCH(DMI_PRODUCT_NAME, "Z830"), + }, + }, +}; + static int toshiba_acpi_add(struct acpi_device *acpi_dev) { struct toshiba_acpi_dev *dev; @@ -3141,12 +3428,32 @@ iio_error: ret = get_fan_status(dev, &dummy); dev->fan_supported = !ret; + ret = get_fan_rpm(dev, &dummy); + dev->fan_rpm_supported = !ret; + +#if IS_ENABLED(CONFIG_HWMON) + if (dev->fan_rpm_supported) { + dev->hwmon_device = hwmon_device_register_with_info( + &dev->acpi_dev->dev, "toshiba_acpi_sensors", NULL, + &toshiba_acpi_hwmon_chip_info, NULL); + if (IS_ERR(dev->hwmon_device)) { + dev->hwmon_device = NULL; + pr_warn("unable to register hwmon device, skipping\n"); + } + } +#endif + + if (turn_on_panel_on_resume == -1) + turn_on_panel_on_resume = dmi_check_system(turn_on_panel_on_resume_dmi_ids); + toshiba_wwan_available(dev); if (dev->wwan_supported) toshiba_acpi_setup_wwan_rfkill(dev); toshiba_cooling_method_available(dev); + toshiba_battery_charge_mode_available(dev); + print_supported_features(dev); ret = sysfs_create_group(&dev->acpi_dev->dev.kobj, @@ -3161,6 +3468,13 @@ iio_error: toshiba_acpi = dev; + /* + * As the battery hook relies on the static variable toshiba_acpi being + * set, this must be done after toshiba_acpi is assigned. + */ + if (dev->battery_charge_mode_supported) + battery_hook_register(&battery_hook); + return 0; error: @@ -3257,6 +3571,9 @@ static int toshiba_acpi_resume(struct device *device) rfkill_set_hw_state(dev->wwan_rfk, !dev->killswitch); } + if (turn_on_panel_on_resume) + hci_write(dev, HCI_PANEL_POWER_ON, 1); + return 0; } #endif diff --git a/drivers/platform/x86/winmate-fm07-keys.c b/drivers/platform/x86/winmate-fm07-keys.c index 2c90c5c7eca264e685e0ed2397a335ec1114a4cf..465ffad81a65b1d4da05900c5d5b0bfcffc379e9 100644 --- a/drivers/platform/x86/winmate-fm07-keys.c +++ b/drivers/platform/x86/winmate-fm07-keys.c @@ -161,7 +161,7 @@ static int __init fm07keys_init(void) return ret; } - dev = platform_device_register_simple(DRV_NAME, -1, NULL, 0); + dev = platform_device_register_simple(DRV_NAME, PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(dev)) { ret = PTR_ERR(dev); pr_err("fm07keys: failed to allocate device, err = %d\n", ret); diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c index aed293b5af81b12b46e8dffc2e04139b6afa6582..223550a10d4ddc5a3fae20b2c7b39a62037f21bf 100644 --- a/drivers/platform/x86/wmi.c +++ b/drivers/platform/x86/wmi.c @@ -95,9 +95,6 @@ module_param(debug_dump_wdg, bool, 0444); MODULE_PARM_DESC(debug_dump_wdg, "Dump available WMI interfaces [0/1]"); -static int acpi_wmi_remove(struct platform_device *device); -static int acpi_wmi_probe(struct platform_device *device); - static const struct acpi_device_id wmi_device_ids[] = { {"PNP0C14", 0}, {"pnp0c14", 0}, @@ -105,13 +102,10 @@ static const struct acpi_device_id wmi_device_ids[] = { }; MODULE_DEVICE_TABLE(acpi, wmi_device_ids); -static struct platform_driver acpi_wmi_driver = { - .driver = { - .name = "acpi-wmi", - .acpi_match_table = wmi_device_ids, - }, - .probe = acpi_wmi_probe, - .remove = acpi_wmi_remove, +/* allow duplicate GUIDs as these device drivers use struct wmi_driver */ +static const char * const allow_duplicates[] = { + "05901221-D566-11D1-B2F0-00A0C9062910", /* wmi-bmof */ + NULL }; /* @@ -1073,6 +1067,23 @@ static const struct device_type wmi_type_data = { .release = wmi_dev_release, }; +/* + * _WDG is a static list that is only parsed at startup, + * so it's safe to count entries without extra protection. + */ +static int guid_count(const guid_t *guid) +{ + struct wmi_block *wblock; + int count = 0; + + list_for_each_entry(wblock, &wmi_block_list, list) { + if (guid_equal(&wblock->gblock.guid, guid)) + count++; + } + + return count; +} + static int wmi_create_device(struct device *wmi_bus_dev, struct wmi_block *wblock, struct acpi_device *device) @@ -1080,6 +1091,7 @@ static int wmi_create_device(struct device *wmi_bus_dev, struct acpi_device_info *info; char method[WMI_ACPI_METHOD_NAME_SIZE]; int result; + uint count; if (wblock->gblock.flags & ACPI_WMI_EVENT) { wblock->dev.dev.type = &wmi_type_event; @@ -1134,7 +1146,11 @@ static int wmi_create_device(struct device *wmi_bus_dev, wblock->dev.dev.bus = &wmi_bus_type; wblock->dev.dev.parent = wmi_bus_dev; - dev_set_name(&wblock->dev.dev, "%pUL", &wblock->gblock.guid); + count = guid_count(&wblock->gblock.guid); + if (count) + dev_set_name(&wblock->dev.dev, "%pUL-%d", &wblock->gblock.guid, count); + else + dev_set_name(&wblock->dev.dev, "%pUL", &wblock->gblock.guid); device_initialize(&wblock->dev.dev); @@ -1154,11 +1170,20 @@ static void wmi_free_devices(struct acpi_device *device) } } -static bool guid_already_parsed(struct acpi_device *device, const guid_t *guid) +static bool guid_already_parsed_for_legacy(struct acpi_device *device, const guid_t *guid) { struct wmi_block *wblock; list_for_each_entry(wblock, &wmi_block_list, list) { + /* skip warning and register if we know the driver will use struct wmi_driver */ + for (int i = 0; allow_duplicates[i] != NULL; i++) { + guid_t tmp; + + if (guid_parse(allow_duplicates[i], &tmp)) + continue; + if (guid_equal(&tmp, guid)) + return false; + } if (guid_equal(&wblock->gblock.guid, guid)) { /* * Because we historically didn't track the relationship @@ -1208,13 +1233,7 @@ static int parse_wdg(struct device *wmi_bus_dev, struct acpi_device *device) if (debug_dump_wdg) wmi_dump_wdg(&gblock[i]); - /* - * Some WMI devices, like those for nVidia hooks, have a - * duplicate GUID. It's not clear what we should do in this - * case yet, so for now, we'll just ignore the duplicate - * for device creation. - */ - if (guid_already_parsed(device, &gblock[i].guid)) + if (guid_already_parsed_for_legacy(device, &gblock[i].guid)) continue; wblock = kzalloc(sizeof(*wblock), GFP_KERNEL); @@ -1449,6 +1468,15 @@ void wmi_driver_unregister(struct wmi_driver *driver) } EXPORT_SYMBOL(wmi_driver_unregister); +static struct platform_driver acpi_wmi_driver = { + .driver = { + .name = "acpi-wmi", + .acpi_match_table = wmi_device_ids, + }, + .probe = acpi_wmi_probe, + .remove = acpi_wmi_remove, +}; + static int __init acpi_wmi_init(void) { int error; diff --git a/drivers/platform/x86/x86-android-tablets.c b/drivers/platform/x86/x86-android-tablets.c index 4803759774358b6d10e4db3fc5ce96d57ce95292..4acd6fa8d43b82b120043ae340f24900d4d3ce26 100644 --- a/drivers/platform/x86/x86-android-tablets.c +++ b/drivers/platform/x86/x86-android-tablets.c @@ -663,9 +663,23 @@ static const struct x86_i2c_client_info chuwi_hi8_i2c_clients[] __initconst = { }, }; +static int __init chuwi_hi8_init(void) +{ + /* + * Avoid the acpi_unregister_gsi() call in x86_acpi_irq_helper_get() + * breaking the touchscreen + logging various errors when the Windows + * BIOS is used. + */ + if (acpi_dev_present("MSSL0001", NULL, 1)) + return -ENODEV; + + return 0; +} + static const struct x86_dev_info chuwi_hi8_info __initconst = { .i2c_client_info = chuwi_hi8_i2c_clients, .i2c_client_count = ARRAY_SIZE(chuwi_hi8_i2c_clients), + .init = chuwi_hi8_init, }; #define CZC_EC_EXTRA_PORT 0x68 diff --git a/drivers/pnp/pnpacpi/rsparser.c b/drivers/pnp/pnpacpi/rsparser.c index da78dc77aed32e4fbd589d138f13fa3cdcbba4d8..4f05f610391b006c9415c0e2537b5e3ae9260c6c 100644 --- a/drivers/pnp/pnpacpi/rsparser.c +++ b/drivers/pnp/pnpacpi/rsparser.c @@ -206,7 +206,8 @@ static acpi_status pnpacpi_allocated_resource(struct acpi_resource *res, if (i >= 0) { flags = acpi_dev_irq_flags(gpio->triggering, gpio->polarity, - gpio->shareable); + gpio->shareable, + gpio->wake_capable); } else { flags = IORESOURCE_DISABLED; } @@ -315,7 +316,7 @@ static __init void pnpacpi_parse_irq_option(struct pnp_dev *dev, if (p->interrupts[i]) __set_bit(p->interrupts[i], map.bits); - flags = acpi_dev_irq_flags(p->triggering, p->polarity, p->shareable); + flags = acpi_dev_irq_flags(p->triggering, p->polarity, p->shareable, p->wake_capable); pnp_register_irq_resource(dev, option_flags, &map, flags); } @@ -339,7 +340,7 @@ static __init void pnpacpi_parse_ext_irq_option(struct pnp_dev *dev, } } - flags = acpi_dev_irq_flags(p->triggering, p->polarity, p->shareable); + flags = acpi_dev_irq_flags(p->triggering, p->polarity, p->shareable, p->wake_capable); pnp_register_irq_resource(dev, option_flags, &map, flags); } diff --git a/drivers/pnp/pnpbios/pnpbios.h b/drivers/pnp/pnpbios/pnpbios.h index 2ce739ff9c1a06d495dad573e0f554c23af88dd4..f3302006842e1f2c6c32693508d2cf94c6c8078c 100644 --- a/drivers/pnp/pnpbios/pnpbios.h +++ b/drivers/pnp/pnpbios/pnpbios.h @@ -153,7 +153,6 @@ extern int pnpbios_dont_use_current_config; extern int pnpbios_parse_data_stream(struct pnp_dev *dev, struct pnp_bios_node * node); extern int pnpbios_read_resources_from_node(struct pnp_dev *dev, struct pnp_bios_node *node); extern int pnpbios_write_resources_to_node(struct pnp_dev *dev, struct pnp_bios_node *node); -extern void pnpid32_to_pnpid(u32 id, char *str); extern void pnpbios_print_status(const char * module, u16 status); extern void pnpbios_calls_init(union pnp_bios_install_struct * header); diff --git a/drivers/power/reset/qcom-pon.c b/drivers/power/reset/qcom-pon.c index 4a688741a88a4d1ad4c1af2612515f6dfbfd3811..16bc01738be953a9cdafce1355913780557454c2 100644 --- a/drivers/power/reset/qcom-pon.c +++ b/drivers/power/reset/qcom-pon.c @@ -82,6 +82,7 @@ static const struct of_device_id pm8916_pon_id_table[] = { { .compatible = "qcom,pm8916-pon", .data = (void *)GEN1_REASON_SHIFT }, { .compatible = "qcom,pms405-pon", .data = (void *)GEN1_REASON_SHIFT }, { .compatible = "qcom,pm8998-pon", .data = (void *)GEN2_REASON_SHIFT }, + { .compatible = "qcom,pmk8350-pon", .data = (void *)GEN2_REASON_SHIFT }, { } }; MODULE_DEVICE_TABLE(of, pm8916_pon_id_table); diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 1aa8323ad9f6a03ebf2844580094ff929218f7a9..0bbfe6a7ce4d2b58d10e61226e206382b03acba2 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -619,6 +619,21 @@ config CHARGER_MT6360 Average Input Current Regulation, Battery Temperature Sensing, Over-Temperature Protection, DPDM Detection for BC1.2. +config CHARGER_MT6370 + tristate "MediaTek MT6370 Charger Driver" + depends on MFD_MT6370 + depends on REGULATOR + depends on IIO + select LINEAR_RANGES + help + Say Y here to enable MT6370 Charger Part. + The device supports High-Accuracy Voltage/Current Regulation, + Average Input Current Regulation, Battery Temperature Sensing, + Over-Temperature Protection, DPDM Detection for BC1.2. + + This driver can also be built as a module. If so, the module + will be called "mt6370-charger". + config CHARGER_QCOM_SMBB tristate "Qualcomm Switch-Mode Battery Charger and Boost" depends on MFD_SPMI_PMIC || COMPILE_TEST @@ -708,6 +723,12 @@ config CHARGER_BQ256XX charge management and system power path management devices for single cell Li-ion and Li-polymer batteries. +config CHARGER_RK817 + tristate "Rockchip RK817 PMIC Battery Charger" + depends on MFD_RK808 + help + Say Y to include support for Rockchip RK817 Battery Charger. + config CHARGER_SMB347 tristate "Summit Microelectronics SMB3XX Battery Charger" depends on I2C diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index 7f02f36aea55d8633e7a6218e6d55b8e20050841..0ee8653e882e7a0e5f3c621a358533f5cb90e280 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -82,6 +82,7 @@ obj-$(CONFIG_CHARGER_MAX8997) += max8997_charger.o obj-$(CONFIG_CHARGER_MAX8998) += max8998_charger.o obj-$(CONFIG_CHARGER_MP2629) += mp2629_charger.o obj-$(CONFIG_CHARGER_MT6360) += mt6360_charger.o +obj-$(CONFIG_CHARGER_MT6370) += mt6370-charger.o obj-$(CONFIG_CHARGER_QCOM_SMBB) += qcom_smbb.o obj-$(CONFIG_CHARGER_BQ2415X) += bq2415x_charger.o obj-$(CONFIG_CHARGER_BQ24190) += bq24190_charger.o @@ -91,6 +92,7 @@ obj-$(CONFIG_CHARGER_BQ2515X) += bq2515x_charger.o obj-$(CONFIG_CHARGER_BQ25890) += bq25890_charger.o obj-$(CONFIG_CHARGER_BQ25980) += bq25980_charger.o obj-$(CONFIG_CHARGER_BQ256XX) += bq256xx_charger.o +obj-$(CONFIG_CHARGER_RK817) += rk817_charger.o obj-$(CONFIG_CHARGER_SMB347) += smb347-charger.o obj-$(CONFIG_CHARGER_TPS65090) += tps65090-charger.o obj-$(CONFIG_CHARGER_TPS65217) += tps65217_charger.o diff --git a/drivers/power/supply/ab8500_chargalg.c b/drivers/power/supply/ab8500_chargalg.c index ae4be553f4248beb4e60369db0e4eb8a2cd0b7c2..ea4ad61d4c7e2ed66f386a29e367cefe5705a81a 100644 --- a/drivers/power/supply/ab8500_chargalg.c +++ b/drivers/power/supply/ab8500_chargalg.c @@ -252,12 +252,6 @@ static enum power_supply_property ab8500_chargalg_props[] = { POWER_SUPPLY_PROP_HEALTH, }; -struct ab8500_chargalg_sysfs_entry { - struct attribute attr; - ssize_t (*show)(struct ab8500_chargalg *di, char *buf); - ssize_t (*store)(struct ab8500_chargalg *di, const char *buf, size_t length); -}; - /** * ab8500_chargalg_safety_timer_expired() - Expiration of the safety timer * @timer: pointer to the hrtimer structure @@ -490,8 +484,6 @@ static int ab8500_chargalg_kick_watchdog(struct ab8500_chargalg *di) static int ab8500_chargalg_ac_en(struct ab8500_chargalg *di, int enable, int vset_uv, int iset_ua) { - static int ab8500_chargalg_ex_ac_enable_toggle; - if (!di->ac_chg || !di->ac_chg->ops.enable) return -ENXIO; diff --git a/drivers/power/supply/adp5061.c b/drivers/power/supply/adp5061.c index 003557043ab3afa5a35ee4babf3f797476af2e6c..fcf8ff0bc974b86c289580ed17a6d3230d042a98 100644 --- a/drivers/power/supply/adp5061.c +++ b/drivers/power/supply/adp5061.c @@ -427,11 +427,11 @@ static int adp5061_get_chg_type(struct adp5061_state *st, if (ret < 0) return ret; - chg_type = adp5061_chg_type[ADP5061_CHG_STATUS_1_CHG_STATUS(status1)]; - if (chg_type > ADP5061_CHG_FAST_CV) + chg_type = ADP5061_CHG_STATUS_1_CHG_STATUS(status1); + if (chg_type >= ARRAY_SIZE(adp5061_chg_type)) val->intval = POWER_SUPPLY_STATUS_UNKNOWN; else - val->intval = chg_type; + val->intval = adp5061_chg_type[chg_type]; return ret; } @@ -493,6 +493,9 @@ static int adp5061_get_battery_status(struct adp5061_state *st, case 0x4: /* VBAT_SNS > VWEAK */ val->intval = POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; break; + default: + val->intval = POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN; + break; } return ret; diff --git a/drivers/power/supply/bq2415x_charger.c b/drivers/power/supply/bq2415x_charger.c index 5724001e66b97dde85964161e6acd54c2532abbc..6b99e1c675b87cacdcff64104c83eec077872076 100644 --- a/drivers/power/supply/bq2415x_charger.c +++ b/drivers/power/supply/bq2415x_charger.c @@ -1696,7 +1696,7 @@ error_1: /* main bq2415x remove function */ -static int bq2415x_remove(struct i2c_client *client) +static void bq2415x_remove(struct i2c_client *client) { struct bq2415x_device *bq = i2c_get_clientdata(client); @@ -1715,8 +1715,6 @@ static int bq2415x_remove(struct i2c_client *client) dev_info(bq->dev, "driver unregistered\n"); kfree(bq->name); - - return 0; } static const struct i2c_device_id bq2415x_i2c_id_table[] = { diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c index 27f5c76486175c6ee1caf6e40955cf90c24c590b..2274679c5ddd215eac319a13cc982c6a7c531c76 100644 --- a/drivers/power/supply/bq24190_charger.c +++ b/drivers/power/supply/bq24190_charger.c @@ -1901,7 +1901,7 @@ out_pmrt: return ret; } -static int bq24190_remove(struct i2c_client *client) +static void bq24190_remove(struct i2c_client *client) { struct bq24190_dev_info *bdi = i2c_get_clientdata(client); int error; @@ -1918,8 +1918,6 @@ static int bq24190_remove(struct i2c_client *client) pm_runtime_put_sync(bdi->dev); pm_runtime_dont_use_autosuspend(bdi->dev); pm_runtime_disable(bdi->dev); - - return 0; } static void bq24190_shutdown(struct i2c_client *client) diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c index ecba9ab86fafb10bd210a90a183243dd24cc7dcc..a309bbedfe521956d63d33b3f9dc174b04d9547a 100644 --- a/drivers/power/supply/bq24257_charger.c +++ b/drivers/power/supply/bq24257_charger.c @@ -1077,7 +1077,7 @@ static int bq24257_probe(struct i2c_client *client, return 0; } -static int bq24257_remove(struct i2c_client *client) +static void bq24257_remove(struct i2c_client *client) { struct bq24257_device *bq = i2c_get_clientdata(client); @@ -1085,8 +1085,6 @@ static int bq24257_remove(struct i2c_client *client) cancel_delayed_work_sync(&bq->iilimit_setup_work); bq24257_field_write(bq, F_RESET, 1); /* reset to defaults */ - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c index 852a6fec4339b037df56aa28a389c057179f1853..6020b58c641d237de3cbc4da18d61bd64a353efd 100644 --- a/drivers/power/supply/bq25890_charger.c +++ b/drivers/power/supply/bq25890_charger.c @@ -613,6 +613,33 @@ static int bq25890_power_supply_get_property(struct power_supply *psy, return 0; } +static int bq25890_power_supply_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct bq25890_device *bq = power_supply_get_drvdata(psy); + u8 lval; + + switch (psp) { + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + lval = bq25890_find_idx(val->intval, TBL_IINLIM); + return bq25890_field_write(bq, F_IINLIM, lval); + default: + return -EINVAL; + } +} + +static int bq25890_power_supply_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + switch (psp) { + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + return true; + default: + return false; + } +} + /* On the BQ25892 try to get charger-type info from our supplier */ static void bq25890_charger_external_power_changed(struct power_supply *psy) { @@ -874,6 +901,8 @@ static const struct power_supply_desc bq25890_power_supply_desc = { .properties = bq25890_power_supply_props, .num_properties = ARRAY_SIZE(bq25890_power_supply_props), .get_property = bq25890_power_supply_get_property, + .set_property = bq25890_power_supply_set_property, + .property_is_writeable = bq25890_power_supply_property_is_writeable, .external_power_changed = bq25890_charger_external_power_changed, }; @@ -946,6 +975,7 @@ static void bq25890_pump_express_work(struct work_struct *data) return; error_print: + bq25890_field_write(bq, F_PUMPX_EN, 0); dev_err(bq->dev, "Failed to request hi-voltage charging\n"); } @@ -1258,7 +1288,7 @@ err_unregister_usb_notifier: return ret; } -static int bq25890_remove(struct i2c_client *client) +static void bq25890_remove(struct i2c_client *client) { struct bq25890_device *bq = i2c_get_clientdata(client); @@ -1269,8 +1299,6 @@ static int bq25890_remove(struct i2c_client *client) /* reset all registers to default values */ bq25890_chip_reset(bq); } - - return 0; } static void bq25890_shutdown(struct i2c_client *client) diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 35e6a394c0df346614614b8d6896339f205a77f0..8bf048fbd36a2f873e02ba430581b5d4795eac24 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -868,11 +868,11 @@ enum bq27xxx_dm_reg_id { BQ27XXX_DM_TERMINATE_VOLTAGE, }; -#define bq27000_dm_regs 0 -#define bq27010_dm_regs 0 -#define bq2750x_dm_regs 0 -#define bq2751x_dm_regs 0 -#define bq2752x_dm_regs 0 +#define bq27000_dm_regs NULL +#define bq27010_dm_regs NULL +#define bq2750x_dm_regs NULL +#define bq2751x_dm_regs NULL +#define bq2752x_dm_regs NULL #if 0 /* not yet tested */ static struct bq27xxx_dm_reg bq27500_dm_regs[] = { @@ -881,24 +881,24 @@ static struct bq27xxx_dm_reg bq27500_dm_regs[] = { [BQ27XXX_DM_TERMINATE_VOLTAGE] = { 80, 48, 2, 1000, 32767 }, }; #else -#define bq27500_dm_regs 0 +#define bq27500_dm_regs NULL #endif /* todo create data memory definitions from datasheets and test on chips */ -#define bq27510g1_dm_regs 0 -#define bq27510g2_dm_regs 0 -#define bq27510g3_dm_regs 0 -#define bq27520g1_dm_regs 0 -#define bq27520g2_dm_regs 0 -#define bq27520g3_dm_regs 0 -#define bq27520g4_dm_regs 0 -#define bq27521_dm_regs 0 -#define bq27530_dm_regs 0 -#define bq27531_dm_regs 0 -#define bq27541_dm_regs 0 -#define bq27542_dm_regs 0 -#define bq27546_dm_regs 0 -#define bq27742_dm_regs 0 +#define bq27510g1_dm_regs NULL +#define bq27510g2_dm_regs NULL +#define bq27510g3_dm_regs NULL +#define bq27520g1_dm_regs NULL +#define bq27520g2_dm_regs NULL +#define bq27520g3_dm_regs NULL +#define bq27520g4_dm_regs NULL +#define bq27521_dm_regs NULL +#define bq27530_dm_regs NULL +#define bq27531_dm_regs NULL +#define bq27541_dm_regs NULL +#define bq27542_dm_regs NULL +#define bq27546_dm_regs NULL +#define bq27742_dm_regs NULL #if 0 /* not yet tested */ static struct bq27xxx_dm_reg bq27545_dm_regs[] = { @@ -907,7 +907,7 @@ static struct bq27xxx_dm_reg bq27545_dm_regs[] = { [BQ27XXX_DM_TERMINATE_VOLTAGE] = { 80, 67, 2, 2800, 3700 }, }; #else -#define bq27545_dm_regs 0 +#define bq27545_dm_regs NULL #endif static struct bq27xxx_dm_reg bq27411_dm_regs[] = { @@ -937,7 +937,7 @@ static struct bq27xxx_dm_reg bq27426_dm_regs[] = { #if 0 /* not yet tested */ #define bq27441_dm_regs bq27421_dm_regs #else -#define bq27441_dm_regs 0 +#define bq27441_dm_regs NULL #endif #if 0 /* not yet tested */ @@ -947,13 +947,13 @@ static struct bq27xxx_dm_reg bq27621_dm_regs[] = { [BQ27XXX_DM_TERMINATE_VOLTAGE] = { 82, 9, 2, 2500, 3700 }, }; #else -#define bq27621_dm_regs 0 +#define bq27621_dm_regs NULL #endif -#define bq27z561_dm_regs 0 -#define bq28z610_dm_regs 0 -#define bq34z100_dm_regs 0 -#define bq78z100_dm_regs 0 +#define bq27z561_dm_regs NULL +#define bq28z610_dm_regs NULL +#define bq34z100_dm_regs NULL +#define bq78z100_dm_regs NULL #define BQ27XXX_O_ZERO BIT(0) #define BQ27XXX_O_OTDC BIT(1) /* has OTC/OTD overtemperature flags */ @@ -1044,12 +1044,12 @@ struct bq27xxx_dm_buf { .block = (di)->dm_regs[i].offset / BQ27XXX_DM_SZ, \ } -static inline u16 *bq27xxx_dm_reg_ptr(struct bq27xxx_dm_buf *buf, +static inline __be16 *bq27xxx_dm_reg_ptr(struct bq27xxx_dm_buf *buf, struct bq27xxx_dm_reg *reg) { if (buf->class == reg->subclass_id && buf->block == reg->offset / BQ27XXX_DM_SZ) - return (u16 *) (buf->data + reg->offset % BQ27XXX_DM_SZ); + return (__be16 *) (buf->data + reg->offset % BQ27XXX_DM_SZ); return NULL; } @@ -1275,7 +1275,7 @@ static void bq27xxx_battery_update_dm_block(struct bq27xxx_device_info *di, { struct bq27xxx_dm_reg *reg = &di->dm_regs[reg_id]; const char *str = bq27xxx_dm_reg_name[reg_id]; - u16 *prev = bq27xxx_dm_reg_ptr(buf, reg); + __be16 *prev = bq27xxx_dm_reg_ptr(buf, reg); if (prev == NULL) { dev_warn(di->dev, "buffer does not match %s dm spec\n", str); diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c index cf38cbfe13e9d16d6d3b499d7f604b6b9c03cbe3..94b00bb89c177fe80df9c7a8465a10419192e744 100644 --- a/drivers/power/supply/bq27xxx_battery_i2c.c +++ b/drivers/power/supply/bq27xxx_battery_i2c.c @@ -205,7 +205,7 @@ err_failed: return ret; } -static int bq27xxx_battery_i2c_remove(struct i2c_client *client) +static void bq27xxx_battery_i2c_remove(struct i2c_client *client) { struct bq27xxx_device_info *di = i2c_get_clientdata(client); @@ -214,8 +214,6 @@ static int bq27xxx_battery_i2c_remove(struct i2c_client *client) mutex_lock(&battery_mutex); idr_remove(&battery_id, di->id); mutex_unlock(&battery_mutex); - - return 0; } static const struct i2c_device_id bq27xxx_i2c_id_table[] = { diff --git a/drivers/power/supply/cpcap-charger.c b/drivers/power/supply/cpcap-charger.c index 60e0ce105a295428ff8f76b4696dbc8bd18968ed..be9764541d52a5e2338213b286c0d9e111df67e7 100644 --- a/drivers/power/supply/cpcap-charger.c +++ b/drivers/power/supply/cpcap-charger.c @@ -5,7 +5,7 @@ * Copyright (C) 2017 Tony Lindgren * * Rewritten for Linux power framework with some parts based on - * on earlier driver found in the Motorola Linux kernel: + * earlier driver found in the Motorola Linux kernel: * * Copyright (C) 2009-2010 Motorola, Inc. */ diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c index 728e2a6cc9c3108a7d35f1e71d14bc0d6b80d4b1..6d52641151d9ac4a1eb9dcade7ce1f1e32eb0d08 100644 --- a/drivers/power/supply/cw2015_battery.c +++ b/drivers/power/supply/cw2015_battery.c @@ -21,6 +21,7 @@ #include #include #include +#include #define CW2015_SIZE_BATINFO 64 @@ -698,7 +699,8 @@ static int cw_bat_probe(struct i2c_client *client) } cw_bat->battery_workqueue = create_singlethread_workqueue("rk_battery"); - INIT_DELAYED_WORK(&cw_bat->battery_delay_work, cw_bat_work); + devm_delayed_work_autocancel(&client->dev, + &cw_bat->battery_delay_work, cw_bat_work); queue_delayed_work(cw_bat->battery_workqueue, &cw_bat->battery_delay_work, msecs_to_jiffies(10)); return 0; @@ -725,15 +727,6 @@ static int __maybe_unused cw_bat_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(cw_bat_pm_ops, cw_bat_suspend, cw_bat_resume); -static int cw_bat_remove(struct i2c_client *client) -{ - struct cw_battery *cw_bat = i2c_get_clientdata(client); - - cancel_delayed_work_sync(&cw_bat->battery_delay_work); - power_supply_put_battery_info(cw_bat->rk_bat, cw_bat->battery); - return 0; -} - static const struct i2c_device_id cw_bat_id_table[] = { { "cw2015", 0 }, { } @@ -752,7 +745,6 @@ static struct i2c_driver cw_bat_driver = { .pm = &cw_bat_pm_ops, }, .probe_new = cw_bat_probe, - .remove = cw_bat_remove, .id_table = cw_bat_id_table, }; diff --git a/drivers/power/supply/ds2782_battery.c b/drivers/power/supply/ds2782_battery.c index 9ae273fde7a2c53cd9177011f556d86ab63a7635..d78cd05402f66cafe86fd6b5b6cd9e05ac614385 100644 --- a/drivers/power/supply/ds2782_battery.c +++ b/drivers/power/supply/ds2782_battery.c @@ -312,7 +312,7 @@ static void ds278x_power_supply_init(struct power_supply_desc *battery) battery->external_power_changed = NULL; } -static int ds278x_battery_remove(struct i2c_client *client) +static void ds278x_battery_remove(struct i2c_client *client) { struct ds278x_info *info = i2c_get_clientdata(client); int id = info->id; @@ -325,8 +325,6 @@ static int ds278x_battery_remove(struct i2c_client *client) mutex_lock(&battery_lock); idr_remove(&battery_id, id); mutex_unlock(&battery_lock); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/power/supply/lp8727_charger.c b/drivers/power/supply/lp8727_charger.c index 9ee54e397754bac6a9311cc2fd49b9f31ca84a64..384a374b52c1955d35a0e24d6206e93e66408ff0 100644 --- a/drivers/power/supply/lp8727_charger.c +++ b/drivers/power/supply/lp8727_charger.c @@ -590,13 +590,12 @@ static int lp8727_probe(struct i2c_client *cl, const struct i2c_device_id *id) return 0; } -static int lp8727_remove(struct i2c_client *cl) +static void lp8727_remove(struct i2c_client *cl) { struct lp8727_chg *pchg = i2c_get_clientdata(cl); lp8727_release_irq(pchg); lp8727_unregister_psy(pchg); - return 0; } static const struct of_device_id lp8727_dt_ids[] = { diff --git a/drivers/power/supply/max1721x_battery.c b/drivers/power/supply/max1721x_battery.c index 473e53cd2801054d1f17c56901f2017ca0b64fcd..d8d52e09da7bea4502427a580ff7f95d914dedb8 100644 --- a/drivers/power/supply/max1721x_battery.c +++ b/drivers/power/supply/max1721x_battery.c @@ -444,5 +444,5 @@ module_w1_family(w1_max1721x_family); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Alex A. Mihaylov "); -MODULE_DESCRIPTION("Maxim MAX17211/MAX17215 Fuel Gauage IC driver"); +MODULE_DESCRIPTION("Maxim MAX17211/MAX17215 Fuel Gauge IC driver"); MODULE_ALIAS("w1-family-" __stringify(W1_MAX1721X_FAMILY_ID)); diff --git a/drivers/power/supply/mt6370-charger.c b/drivers/power/supply/mt6370-charger.c new file mode 100644 index 0000000000000000000000000000000000000000..f27dae5043f5b41704b26a635db355eab5305ec7 --- /dev/null +++ b/drivers/power/supply/mt6370-charger.c @@ -0,0 +1,961 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * + * Author: ChiaEn Wu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MT6370_REG_CHG_CTRL1 0x111 +#define MT6370_REG_CHG_CTRL2 0x112 +#define MT6370_REG_CHG_CTRL3 0x113 +#define MT6370_REG_CHG_CTRL4 0x114 +#define MT6370_REG_CHG_CTRL5 0x115 +#define MT6370_REG_CHG_CTRL6 0x116 +#define MT6370_REG_CHG_CTRL7 0x117 +#define MT6370_REG_CHG_CTRL8 0x118 +#define MT6370_REG_CHG_CTRL9 0x119 +#define MT6370_REG_CHG_CTRL10 0x11A +#define MT6370_REG_DEVICE_TYPE 0x122 +#define MT6370_REG_USB_STATUS1 0x127 +#define MT6370_REG_CHG_STAT 0x14A +#define MT6370_REG_FLED_EN 0x17E +#define MT6370_REG_CHG_STAT1 0X1D0 +#define MT6370_REG_OVPCTRL_STAT 0x1D8 + +#define MT6370_VOBST_MASK GENMASK(7, 2) +#define MT6370_OTG_PIN_EN_MASK BIT(1) +#define MT6370_OPA_MODE_MASK BIT(0) +#define MT6370_OTG_OC_MASK GENMASK(2, 0) + +#define MT6370_MIVR_IBUS_TH_100_mA 100000 +#define MT6370_ADC_CHAN_IBUS 5 +#define MT6370_ADC_CHAN_MAX 9 + +enum mt6370_chg_reg_field { + /* MT6370_REG_CHG_CTRL2 */ + F_IINLMTSEL, F_CFO_EN, F_CHG_EN, + /* MT6370_REG_CHG_CTRL3 */ + F_IAICR, F_AICR_EN, F_ILIM_EN, + /* MT6370_REG_CHG_CTRL4 */ + F_VOREG, + /* MT6370_REG_CHG_CTRL6 */ + F_VMIVR, + /* MT6370_REG_CHG_CTRL7 */ + F_ICHG, + /* MT6370_REG_CHG_CTRL8 */ + F_IPREC, + /* MT6370_REG_CHG_CTRL9 */ + F_IEOC, + /* MT6370_REG_DEVICE_TYPE */ + F_USBCHGEN, + /* MT6370_REG_USB_STATUS1 */ + F_USB_STAT, F_CHGDET, + /* MT6370_REG_CHG_STAT */ + F_CHG_STAT, F_BOOST_STAT, F_VBAT_LVL, + /* MT6370_REG_FLED_EN */ + F_FL_STROBE, + /* MT6370_REG_CHG_STAT1 */ + F_CHG_MIVR_STAT, + /* MT6370_REG_OVPCTRL_STAT */ + F_UVP_D_STAT, + F_MAX +}; + +enum mt6370_irq { + MT6370_IRQ_ATTACH_I = 0, + MT6370_IRQ_UVP_D_EVT, + MT6370_IRQ_MIVR, + MT6370_IRQ_MAX +}; + +struct mt6370_priv { + struct device *dev; + struct iio_channel *iio_adcs; + struct mutex attach_lock; + struct power_supply *psy; + struct regmap *regmap; + struct regmap_field *rmap_fields[F_MAX]; + struct regulator_dev *rdev; + struct workqueue_struct *wq; + struct work_struct bc12_work; + struct delayed_work mivr_dwork; + unsigned int irq_nums[MT6370_IRQ_MAX]; + int attach; + int psy_usb_type; + bool pwr_rdy; +}; + +enum mt6370_usb_status { + MT6370_USB_STAT_NO_VBUS = 0, + MT6370_USB_STAT_VBUS_FLOW_IS_UNDER_GOING, + MT6370_USB_STAT_SDP, + MT6370_USB_STAT_SDP_NSTD, + MT6370_USB_STAT_DCP, + MT6370_USB_STAT_CDP, + MT6370_USB_STAT_MAX +}; + +struct mt6370_chg_field { + const char *name; + const struct linear_range *range; + struct reg_field field; +}; + +enum { + MT6370_RANGE_F_IAICR = 0, + MT6370_RANGE_F_VOREG, + MT6370_RANGE_F_VMIVR, + MT6370_RANGE_F_ICHG, + MT6370_RANGE_F_IPREC, + MT6370_RANGE_F_IEOC, + MT6370_RANGE_F_MAX +}; + +static const struct linear_range mt6370_chg_ranges[MT6370_RANGE_F_MAX] = { + LINEAR_RANGE_IDX(MT6370_RANGE_F_IAICR, 100000, 0x0, 0x3F, 50000), + LINEAR_RANGE_IDX(MT6370_RANGE_F_VOREG, 3900000, 0x0, 0x51, 10000), + LINEAR_RANGE_IDX(MT6370_RANGE_F_VMIVR, 3900000, 0x0, 0x5F, 100000), + LINEAR_RANGE_IDX(MT6370_RANGE_F_ICHG, 900000, 0x08, 0x31, 100000), + LINEAR_RANGE_IDX(MT6370_RANGE_F_IPREC, 100000, 0x0, 0x0F, 50000), + LINEAR_RANGE_IDX(MT6370_RANGE_F_IEOC, 100000, 0x0, 0x0F, 50000), +}; + +#define MT6370_CHG_FIELD(_fd, _reg, _lsb, _msb) \ +[_fd] = { \ + .name = #_fd, \ + .range = NULL, \ + .field = REG_FIELD(_reg, _lsb, _msb), \ +} + +#define MT6370_CHG_FIELD_RANGE(_fd, _reg, _lsb, _msb) \ +[_fd] = { \ + .name = #_fd, \ + .range = &mt6370_chg_ranges[MT6370_RANGE_##_fd], \ + .field = REG_FIELD(_reg, _lsb, _msb), \ +} + +static const struct mt6370_chg_field mt6370_chg_fields[F_MAX] = { + MT6370_CHG_FIELD(F_IINLMTSEL, MT6370_REG_CHG_CTRL2, 2, 3), + MT6370_CHG_FIELD(F_CFO_EN, MT6370_REG_CHG_CTRL2, 1, 1), + MT6370_CHG_FIELD(F_CHG_EN, MT6370_REG_CHG_CTRL2, 0, 0), + MT6370_CHG_FIELD_RANGE(F_IAICR, MT6370_REG_CHG_CTRL3, 2, 7), + MT6370_CHG_FIELD(F_AICR_EN, MT6370_REG_CHG_CTRL3, 1, 1), + MT6370_CHG_FIELD(F_ILIM_EN, MT6370_REG_CHG_CTRL3, 0, 0), + MT6370_CHG_FIELD_RANGE(F_VOREG, MT6370_REG_CHG_CTRL4, 1, 7), + MT6370_CHG_FIELD_RANGE(F_VMIVR, MT6370_REG_CHG_CTRL6, 1, 7), + MT6370_CHG_FIELD_RANGE(F_ICHG, MT6370_REG_CHG_CTRL7, 2, 7), + MT6370_CHG_FIELD_RANGE(F_IPREC, MT6370_REG_CHG_CTRL8, 0, 3), + MT6370_CHG_FIELD_RANGE(F_IEOC, MT6370_REG_CHG_CTRL9, 4, 7), + MT6370_CHG_FIELD(F_USBCHGEN, MT6370_REG_DEVICE_TYPE, 7, 7), + MT6370_CHG_FIELD(F_USB_STAT, MT6370_REG_USB_STATUS1, 4, 6), + MT6370_CHG_FIELD(F_CHGDET, MT6370_REG_USB_STATUS1, 3, 3), + MT6370_CHG_FIELD(F_CHG_STAT, MT6370_REG_CHG_STAT, 6, 7), + MT6370_CHG_FIELD(F_BOOST_STAT, MT6370_REG_CHG_STAT, 3, 3), + MT6370_CHG_FIELD(F_VBAT_LVL, MT6370_REG_CHG_STAT, 5, 5), + MT6370_CHG_FIELD(F_FL_STROBE, MT6370_REG_FLED_EN, 2, 2), + MT6370_CHG_FIELD(F_CHG_MIVR_STAT, MT6370_REG_CHG_STAT1, 6, 6), + MT6370_CHG_FIELD(F_UVP_D_STAT, MT6370_REG_OVPCTRL_STAT, 4, 4), +}; + +static inline int mt6370_chg_field_get(struct mt6370_priv *priv, + enum mt6370_chg_reg_field fd, + unsigned int *val) +{ + int ret; + unsigned int reg_val; + + ret = regmap_field_read(priv->rmap_fields[fd], ®_val); + if (ret) + return ret; + + if (mt6370_chg_fields[fd].range) + return linear_range_get_value(mt6370_chg_fields[fd].range, + reg_val, val); + + *val = reg_val; + return 0; +} + +static inline int mt6370_chg_field_set(struct mt6370_priv *priv, + enum mt6370_chg_reg_field fd, + unsigned int val) +{ + int ret; + bool f; + const struct linear_range *r; + + if (mt6370_chg_fields[fd].range) { + r = mt6370_chg_fields[fd].range; + + if (fd == F_VMIVR) { + ret = linear_range_get_selector_high(r, val, &val, &f); + if (ret) + val = r->max_sel; + } else { + linear_range_get_selector_within(r, val, &val); + } + } + + return regmap_field_write(priv->rmap_fields[fd], val); +} + +enum { + MT6370_CHG_STAT_READY = 0, + MT6370_CHG_STAT_CHARGE_IN_PROGRESS, + MT6370_CHG_STAT_DONE, + MT6370_CHG_STAT_FAULT, + MT6370_CHG_STAT_MAX +}; + +enum { + MT6370_ATTACH_STAT_DETACH = 0, + MT6370_ATTACH_STAT_ATTACH_WAIT_FOR_BC12, + MT6370_ATTACH_STAT_ATTACH_BC12_DONE, + MT6370_ATTACH_STAT_ATTACH_MAX +}; + +static int mt6370_chg_otg_of_parse_cb(struct device_node *of, + const struct regulator_desc *rdesc, + struct regulator_config *rcfg) +{ + struct mt6370_priv *priv = rcfg->driver_data; + + rcfg->ena_gpiod = fwnode_gpiod_get_index(of_fwnode_handle(of), + "enable", 0, GPIOD_OUT_LOW | + GPIOD_FLAGS_BIT_NONEXCLUSIVE, + rdesc->name); + if (IS_ERR(rcfg->ena_gpiod)) { + rcfg->ena_gpiod = NULL; + return 0; + } + + return regmap_update_bits(priv->regmap, MT6370_REG_CHG_CTRL1, + MT6370_OTG_PIN_EN_MASK, + MT6370_OTG_PIN_EN_MASK); +} + +static void mt6370_chg_bc12_work_func(struct work_struct *work) +{ + struct mt6370_priv *priv = container_of(work, struct mt6370_priv, + bc12_work); + int ret; + bool rpt_psy = false; + unsigned int attach, usb_stat; + + mutex_lock(&priv->attach_lock); + attach = priv->attach; + + switch (attach) { + case MT6370_ATTACH_STAT_DETACH: + usb_stat = 0; + break; + case MT6370_ATTACH_STAT_ATTACH_WAIT_FOR_BC12: + ret = mt6370_chg_field_set(priv, F_USBCHGEN, attach); + if (ret) + dev_err(priv->dev, "Failed to enable USB CHG EN\n"); + goto bc12_work_func_out; + case MT6370_ATTACH_STAT_ATTACH_BC12_DONE: + ret = mt6370_chg_field_get(priv, F_USB_STAT, &usb_stat); + if (ret) { + dev_err(priv->dev, "Failed to get USB status\n"); + goto bc12_work_func_out; + } + break; + default: + dev_err(priv->dev, "Invalid attach state\n"); + goto bc12_work_func_out; + } + + rpt_psy = true; + + switch (usb_stat) { + case MT6370_USB_STAT_SDP: + case MT6370_USB_STAT_SDP_NSTD: + priv->psy_usb_type = POWER_SUPPLY_USB_TYPE_SDP; + break; + case MT6370_USB_STAT_DCP: + priv->psy_usb_type = POWER_SUPPLY_USB_TYPE_DCP; + break; + case MT6370_USB_STAT_CDP: + priv->psy_usb_type = POWER_SUPPLY_USB_TYPE_CDP; + break; + case MT6370_USB_STAT_NO_VBUS: + case MT6370_USB_STAT_VBUS_FLOW_IS_UNDER_GOING: + default: + priv->psy_usb_type = POWER_SUPPLY_USB_TYPE_UNKNOWN; + break; + } + +bc12_work_func_out: + mutex_unlock(&priv->attach_lock); + + if (rpt_psy) + power_supply_changed(priv->psy); +} + +static int mt6370_chg_toggle_cfo(struct mt6370_priv *priv) +{ + int ret; + unsigned int fl_strobe; + + /* check if flash led in strobe mode */ + ret = mt6370_chg_field_get(priv, F_FL_STROBE, &fl_strobe); + if (ret) { + dev_err(priv->dev, "Failed to get FL_STROBE_EN\n"); + return ret; + } + + if (fl_strobe) { + dev_err(priv->dev, "Flash led is still in strobe mode\n"); + return ret; + } + + /* cfo off */ + ret = mt6370_chg_field_set(priv, F_CFO_EN, 0); + if (ret) { + dev_err(priv->dev, "Failed to disable CFO_EN\n"); + return ret; + } + + /* cfo on */ + ret = mt6370_chg_field_set(priv, F_CFO_EN, 1); + if (ret) + dev_err(priv->dev, "Failed to enable CFO_EN\n"); + + return ret; +} + +static int mt6370_chg_read_adc_chan(struct mt6370_priv *priv, unsigned int chan, + int *val) +{ + int ret; + + if (chan >= MT6370_ADC_CHAN_MAX) + return -EINVAL; + + ret = iio_read_channel_processed(&priv->iio_adcs[chan], val); + if (ret) + dev_err(priv->dev, "Failed to read ADC\n"); + + return ret; +} + +static void mt6370_chg_mivr_dwork_func(struct work_struct *work) +{ + struct mt6370_priv *priv = container_of(work, struct mt6370_priv, + mivr_dwork.work); + int ret; + unsigned int mivr_stat, ibus; + + ret = mt6370_chg_field_get(priv, F_CHG_MIVR_STAT, &mivr_stat); + if (ret) { + dev_err(priv->dev, "Failed to get mivr state\n"); + goto mivr_handler_out; + } + + if (!mivr_stat) + goto mivr_handler_out; + + ret = mt6370_chg_read_adc_chan(priv, MT6370_ADC_CHAN_IBUS, &ibus); + if (ret) { + dev_err(priv->dev, "Failed to get ibus\n"); + goto mivr_handler_out; + } + + if (ibus < MT6370_MIVR_IBUS_TH_100_mA) { + ret = mt6370_chg_toggle_cfo(priv); + if (ret) + dev_err(priv->dev, "Failed to toggle cfo\n"); + } + +mivr_handler_out: + enable_irq(priv->irq_nums[MT6370_IRQ_MIVR]); + pm_relax(priv->dev); +} + +static void mt6370_chg_pwr_rdy_check(struct mt6370_priv *priv) +{ + int ret; + unsigned int opposite_pwr_rdy, otg_en; + union power_supply_propval val; + + /* Check in OTG mode or not */ + ret = mt6370_chg_field_get(priv, F_BOOST_STAT, &otg_en); + if (ret) { + dev_err(priv->dev, "Failed to get OTG state\n"); + return; + } + + if (otg_en) + return; + + ret = mt6370_chg_field_get(priv, F_UVP_D_STAT, &opposite_pwr_rdy); + if (ret) { + dev_err(priv->dev, "Failed to get opposite power ready state\n"); + return; + } + + val.intval = opposite_pwr_rdy ? + MT6370_ATTACH_STAT_DETACH : + MT6370_ATTACH_STAT_ATTACH_WAIT_FOR_BC12; + + ret = power_supply_set_property(priv->psy, POWER_SUPPLY_PROP_ONLINE, + &val); + if (ret) + dev_err(priv->dev, "Failed to start attach/detach flow\n"); +} + +static int mt6370_chg_get_online(struct mt6370_priv *priv, + union power_supply_propval *val) +{ + mutex_lock(&priv->attach_lock); + val->intval = !!priv->attach; + mutex_unlock(&priv->attach_lock); + + return 0; +} + +static int mt6370_chg_get_status(struct mt6370_priv *priv, + union power_supply_propval *val) +{ + int ret; + unsigned int chg_stat; + union power_supply_propval online; + + ret = power_supply_get_property(priv->psy, POWER_SUPPLY_PROP_ONLINE, + &online); + if (ret) { + dev_err(priv->dev, "Failed to get online status\n"); + return ret; + } + + if (!online.intval) { + val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + return 0; + } + + ret = mt6370_chg_field_get(priv, F_CHG_STAT, &chg_stat); + if (ret) + return ret; + + switch (chg_stat) { + case MT6370_CHG_STAT_READY: + case MT6370_CHG_STAT_FAULT: + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + return ret; + case MT6370_CHG_STAT_CHARGE_IN_PROGRESS: + val->intval = POWER_SUPPLY_STATUS_CHARGING; + return ret; + case MT6370_CHG_STAT_DONE: + val->intval = POWER_SUPPLY_STATUS_FULL; + return ret; + default: + val->intval = POWER_SUPPLY_STATUS_UNKNOWN; + return ret; + } +} + +static int mt6370_chg_get_charge_type(struct mt6370_priv *priv, + union power_supply_propval *val) +{ + int type, ret; + unsigned int chg_stat, vbat_lvl; + + ret = mt6370_chg_field_get(priv, F_CHG_STAT, &chg_stat); + if (ret) + return ret; + + ret = mt6370_chg_field_get(priv, F_VBAT_LVL, &vbat_lvl); + if (ret) + return ret; + + switch (chg_stat) { + case MT6370_CHG_STAT_CHARGE_IN_PROGRESS: + if (vbat_lvl) + type = POWER_SUPPLY_CHARGE_TYPE_FAST; + else + type = POWER_SUPPLY_CHARGE_TYPE_TRICKLE; + break; + case MT6370_CHG_STAT_READY: + case MT6370_CHG_STAT_DONE: + case MT6370_CHG_STAT_FAULT: + default: + type = POWER_SUPPLY_CHARGE_TYPE_NONE; + break; + } + + val->intval = type; + + return 0; +} + +static int mt6370_chg_set_online(struct mt6370_priv *priv, + const union power_supply_propval *val) +{ + bool pwr_rdy = !!val->intval; + + mutex_lock(&priv->attach_lock); + if (pwr_rdy == !!priv->attach) { + dev_err(priv->dev, "pwr_rdy is same(%d)\n", pwr_rdy); + mutex_unlock(&priv->attach_lock); + return 0; + } + + priv->attach = pwr_rdy; + mutex_unlock(&priv->attach_lock); + + if (!queue_work(priv->wq, &priv->bc12_work)) + dev_err(priv->dev, "bc12 work has already queued\n"); + + return 0; +} + +static int mt6370_chg_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct mt6370_priv *priv = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + return mt6370_chg_get_online(priv, val); + case POWER_SUPPLY_PROP_STATUS: + return mt6370_chg_get_status(priv, val); + case POWER_SUPPLY_PROP_CHARGE_TYPE: + return mt6370_chg_get_charge_type(priv, val); + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + return mt6370_chg_field_get(priv, F_ICHG, &val->intval); + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + val->intval = linear_range_get_max_value(&mt6370_chg_ranges[MT6370_RANGE_F_ICHG]); + return 0; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + return mt6370_chg_field_get(priv, F_VOREG, &val->intval); + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX: + val->intval = linear_range_get_max_value(&mt6370_chg_ranges[MT6370_RANGE_F_VOREG]); + return 0; + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + return mt6370_chg_field_get(priv, F_IAICR, &val->intval); + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + return mt6370_chg_field_get(priv, F_VMIVR, &val->intval); + case POWER_SUPPLY_PROP_PRECHARGE_CURRENT: + return mt6370_chg_field_get(priv, F_IPREC, &val->intval); + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + return mt6370_chg_field_get(priv, F_IEOC, &val->intval); + case POWER_SUPPLY_PROP_USB_TYPE: + val->intval = priv->psy_usb_type; + return 0; + default: + return -EINVAL; + } +} + +static int mt6370_chg_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct mt6370_priv *priv = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + return mt6370_chg_set_online(priv, val); + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + return mt6370_chg_field_set(priv, F_ICHG, val->intval); + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + return mt6370_chg_field_set(priv, F_VOREG, val->intval); + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + return mt6370_chg_field_set(priv, F_IAICR, val->intval); + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + return mt6370_chg_field_set(priv, F_VMIVR, val->intval); + case POWER_SUPPLY_PROP_PRECHARGE_CURRENT: + return mt6370_chg_field_set(priv, F_IPREC, val->intval); + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + return mt6370_chg_field_set(priv, F_IEOC, val->intval); + default: + return -EINVAL; + } +} + +static int mt6370_chg_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT: + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + case POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT: + case POWER_SUPPLY_PROP_PRECHARGE_CURRENT: + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + return 1; + default: + return 0; + } +} + +static enum power_supply_property mt6370_chg_properties[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_CHARGE_TYPE, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, + POWER_SUPPLY_PROP_INPUT_VOLTAGE_LIMIT, + POWER_SUPPLY_PROP_PRECHARGE_CURRENT, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_USB_TYPE, +}; + +static enum power_supply_usb_type mt6370_chg_usb_types[] = { + POWER_SUPPLY_USB_TYPE_UNKNOWN, + POWER_SUPPLY_USB_TYPE_SDP, + POWER_SUPPLY_USB_TYPE_CDP, + POWER_SUPPLY_USB_TYPE_DCP, +}; + +static const struct power_supply_desc mt6370_chg_psy_desc = { + .name = "mt6370-charger", + .type = POWER_SUPPLY_TYPE_USB, + .properties = mt6370_chg_properties, + .num_properties = ARRAY_SIZE(mt6370_chg_properties), + .get_property = mt6370_chg_get_property, + .set_property = mt6370_chg_set_property, + .property_is_writeable = mt6370_chg_property_is_writeable, + .usb_types = mt6370_chg_usb_types, + .num_usb_types = ARRAY_SIZE(mt6370_chg_usb_types), +}; + +static const struct regulator_ops mt6370_chg_otg_ops = { + .list_voltage = regulator_list_voltage_linear, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_current_limit = regulator_set_current_limit_regmap, + .get_current_limit = regulator_get_current_limit_regmap, +}; + +static const u32 mt6370_chg_otg_oc_ma[] = { + 500000, 700000, 1100000, 1300000, 1800000, 2100000, 2400000, +}; + +static const struct regulator_desc mt6370_chg_otg_rdesc = { + .of_match = "usb-otg-vbus-regulator", + .of_parse_cb = mt6370_chg_otg_of_parse_cb, + .name = "mt6370-usb-otg-vbus", + .ops = &mt6370_chg_otg_ops, + .owner = THIS_MODULE, + .type = REGULATOR_VOLTAGE, + .min_uV = 4425000, + .uV_step = 25000, + .n_voltages = 57, + .vsel_reg = MT6370_REG_CHG_CTRL5, + .vsel_mask = MT6370_VOBST_MASK, + .enable_reg = MT6370_REG_CHG_CTRL1, + .enable_mask = MT6370_OPA_MODE_MASK, + .curr_table = mt6370_chg_otg_oc_ma, + .n_current_limits = ARRAY_SIZE(mt6370_chg_otg_oc_ma), + .csel_reg = MT6370_REG_CHG_CTRL10, + .csel_mask = MT6370_OTG_OC_MASK, +}; + +static int mt6370_chg_init_rmap_fields(struct mt6370_priv *priv) +{ + int i; + const struct mt6370_chg_field *fds = mt6370_chg_fields; + + for (i = 0; i < F_MAX; i++) { + priv->rmap_fields[i] = devm_regmap_field_alloc(priv->dev, + priv->regmap, + fds[i].field); + if (IS_ERR(priv->rmap_fields[i])) + return dev_err_probe(priv->dev, + PTR_ERR(priv->rmap_fields[i]), + "Failed to allocate regmapfield[%s]\n", + fds[i].name); + } + + return 0; +} + +static int mt6370_chg_init_setting(struct mt6370_priv *priv) +{ + int ret; + + /* Disable usb_chg_en */ + ret = mt6370_chg_field_set(priv, F_USBCHGEN, 0); + if (ret) { + dev_err(priv->dev, "Failed to disable usb_chg_en\n"); + return ret; + } + + /* Disable input current limit */ + ret = mt6370_chg_field_set(priv, F_ILIM_EN, 0); + if (ret) { + dev_err(priv->dev, "Failed to disable input current limit\n"); + return ret; + } + + /* ICHG/IEOC Workaround, ICHG can not be set less than 900mA */ + ret = mt6370_chg_field_set(priv, F_ICHG, 900000); + if (ret) { + dev_err(priv->dev, "Failed to set ICHG to 900mA"); + return ret; + } + + /* Change input current limit selection to using IAICR results */ + ret = mt6370_chg_field_set(priv, F_IINLMTSEL, 2); + if (ret) { + dev_err(priv->dev, "Failed to set IINLMTSEL\n"); + return ret; + } + + return 0; +} + +#define MT6370_CHG_DT_PROP_DECL(_name, _type, _field) \ +{ \ + .name = "mediatek,chg-" #_name, \ + .type = MT6370_PARSE_TYPE_##_type, \ + .fd = _field, \ +} + +static int mt6370_chg_init_otg_regulator(struct mt6370_priv *priv) +{ + struct regulator_config rcfg = { + .dev = priv->dev, + .regmap = priv->regmap, + .driver_data = priv, + }; + + priv->rdev = devm_regulator_register(priv->dev, &mt6370_chg_otg_rdesc, + &rcfg); + + return PTR_ERR_OR_ZERO(priv->rdev); +} + +static int mt6370_chg_init_psy(struct mt6370_priv *priv) +{ + struct power_supply_config cfg = { + .drv_data = priv, + .of_node = dev_of_node(priv->dev), + }; + + priv->psy = devm_power_supply_register(priv->dev, &mt6370_chg_psy_desc, + &cfg); + + return PTR_ERR_OR_ZERO(priv->psy); +} + +static void mt6370_chg_destroy_attach_lock(void *data) +{ + struct mutex *attach_lock = data; + + mutex_destroy(attach_lock); +} + +static void mt6370_chg_destroy_wq(void *data) +{ + struct workqueue_struct *wq = data; + + flush_workqueue(wq); + destroy_workqueue(wq); +} + +static irqreturn_t mt6370_attach_i_handler(int irq, void *data) +{ + struct mt6370_priv *priv = data; + unsigned int otg_en; + int ret; + + /* Check in OTG mode or not */ + ret = mt6370_chg_field_get(priv, F_BOOST_STAT, &otg_en); + if (ret) { + dev_err(priv->dev, "Failed to get OTG state\n"); + return IRQ_NONE; + } + + if (otg_en) + return IRQ_HANDLED; + + mutex_lock(&priv->attach_lock); + priv->attach = MT6370_ATTACH_STAT_ATTACH_BC12_DONE; + mutex_unlock(&priv->attach_lock); + + if (!queue_work(priv->wq, &priv->bc12_work)) + dev_err(priv->dev, "bc12 work has already queued\n"); + + return IRQ_HANDLED; +} + +static irqreturn_t mt6370_uvp_d_evt_handler(int irq, void *data) +{ + struct mt6370_priv *priv = data; + + mt6370_chg_pwr_rdy_check(priv); + + return IRQ_HANDLED; +} + +static irqreturn_t mt6370_mivr_handler(int irq, void *data) +{ + struct mt6370_priv *priv = data; + + pm_stay_awake(priv->dev); + disable_irq_nosync(priv->irq_nums[MT6370_IRQ_MIVR]); + schedule_delayed_work(&priv->mivr_dwork, msecs_to_jiffies(200)); + + return IRQ_HANDLED; +} + +#define MT6370_CHG_IRQ(_name) \ +{ \ + .name = #_name, \ + .handler = mt6370_##_name##_handler, \ +} + +static int mt6370_chg_init_irq(struct mt6370_priv *priv) +{ + int i, ret; + const struct { + char *name; + irq_handler_t handler; + } mt6370_chg_irqs[] = { + MT6370_CHG_IRQ(attach_i), + MT6370_CHG_IRQ(uvp_d_evt), + MT6370_CHG_IRQ(mivr), + }; + + for (i = 0; i < ARRAY_SIZE(mt6370_chg_irqs); i++) { + ret = platform_get_irq_byname(to_platform_device(priv->dev), + mt6370_chg_irqs[i].name); + if (ret < 0) + return dev_err_probe(priv->dev, ret, + "Failed to get irq %s\n", + mt6370_chg_irqs[i].name); + + priv->irq_nums[i] = ret; + ret = devm_request_threaded_irq(priv->dev, ret, NULL, + mt6370_chg_irqs[i].handler, + IRQF_TRIGGER_FALLING, + dev_name(priv->dev), priv); + if (ret) + return dev_err_probe(priv->dev, ret, + "Failed to request irq %s\n", + mt6370_chg_irqs[i].name); + } + + return 0; +} + +static int mt6370_chg_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mt6370_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = &pdev->dev; + + priv->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!priv->regmap) + return dev_err_probe(dev, -ENODEV, "Failed to get regmap\n"); + + ret = mt6370_chg_init_rmap_fields(priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to init regmap fields\n"); + + platform_set_drvdata(pdev, priv); + + priv->iio_adcs = devm_iio_channel_get_all(priv->dev); + if (IS_ERR(priv->iio_adcs)) + return dev_err_probe(dev, PTR_ERR(priv->iio_adcs), + "Failed to get iio adc\n"); + + ret = mt6370_chg_init_otg_regulator(priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to init OTG regulator\n"); + + ret = mt6370_chg_init_psy(priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to init psy\n"); + + mutex_init(&priv->attach_lock); + ret = devm_add_action_or_reset(dev, mt6370_chg_destroy_attach_lock, + &priv->attach_lock); + if (ret) + return dev_err_probe(dev, ret, "Failed to init attach lock\n"); + + priv->attach = MT6370_ATTACH_STAT_DETACH; + + priv->wq = create_singlethread_workqueue(dev_name(priv->dev)); + if (!priv->wq) + return dev_err_probe(dev, -ENOMEM, + "Failed to create workqueue\n"); + + ret = devm_add_action_or_reset(dev, mt6370_chg_destroy_wq, priv->wq); + if (ret) + return dev_err_probe(dev, ret, "Failed to init wq\n"); + + ret = devm_work_autocancel(dev, &priv->bc12_work, mt6370_chg_bc12_work_func); + if (ret) + return dev_err_probe(dev, ret, "Failed to init bc12 work\n"); + + ret = devm_delayed_work_autocancel(dev, &priv->mivr_dwork, mt6370_chg_mivr_dwork_func); + if (ret) + return dev_err_probe(dev, ret, "Failed to init mivr delayed work\n"); + + ret = mt6370_chg_init_setting(priv); + if (ret) + return dev_err_probe(dev, ret, + "Failed to init mt6370 charger setting\n"); + + ret = mt6370_chg_init_irq(priv); + if (ret) + return ret; + + mt6370_chg_pwr_rdy_check(priv); + + return 0; +} + +static const struct of_device_id mt6370_chg_of_match[] = { + { .compatible = "mediatek,mt6370-charger", }, + {} +}; +MODULE_DEVICE_TABLE(of, mt6370_chg_of_match); + +static struct platform_driver mt6370_chg_driver = { + .probe = mt6370_chg_probe, + .driver = { + .name = "mt6370-charger", + .of_match_table = mt6370_chg_of_match, + }, +}; +module_platform_driver(mt6370_chg_driver); + +MODULE_AUTHOR("ChiaEn Wu "); +MODULE_DESCRIPTION("MediaTek MT6370 Charger Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index 4239591e152216c8ec46bad67928199344ebf0ec..5369abaceb5ccb46072502fbce7a12d8ed5aa250 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -442,7 +442,7 @@ static int add_prop_uevent(struct device *dev, struct kobj_uevent_env *env, if (ret == -ENODEV || ret == -ENODATA) { /* * When a battery is absent, we expect -ENODEV. Don't abort; - * send the uevent with at least the the PRESENT=0 property + * send the uevent with at least the PRESENT=0 property */ return 0; } diff --git a/drivers/power/supply/rk817_charger.c b/drivers/power/supply/rk817_charger.c new file mode 100644 index 0000000000000000000000000000000000000000..635f051b08216c3907a243589e250a5e2f4f3a15 --- /dev/null +++ b/drivers/power/supply/rk817_charger.c @@ -0,0 +1,1211 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Charger Driver for Rockchip rk817 + * + * Copyright (c) 2021 Maya Matuszczyk + * + * Authors: Maya Matuszczyk + * Chris Morgan + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Charging statuses reported by hardware register */ +enum rk817_charge_status { + CHRG_OFF, + DEAD_CHRG, + TRICKLE_CHRG, + CC_OR_CV_CHRG, + CHARGE_FINISH, + USB_OVER_VOL, + BAT_TMP_ERR, + BAT_TIM_ERR, +}; + +/* + * Max charging current read to/written from hardware register. + * Note how highest value corresponding to 0x7 is the lowest + * current, this is per the datasheet. + */ +enum rk817_chg_cur { + CHG_1A, + CHG_1_5A, + CHG_2A, + CHG_2_5A, + CHG_2_75A, + CHG_3A, + CHG_3_5A, + CHG_0_5A, +}; + +struct rk817_charger { + struct device *dev; + struct rk808 *rk808; + + struct power_supply *bat_ps; + struct power_supply *chg_ps; + bool plugged_in; + bool battery_present; + + /* + * voltage_k and voltage_b values are used to calibrate the ADC + * voltage readings. While they are documented in the BSP kernel and + * datasheet as voltage_k and voltage_b, there is no further + * information explaining them in more detail. + */ + + uint32_t voltage_k; + uint32_t voltage_b; + + /* + * soc - state of charge - like the BSP this is stored as a percentage, + * to the thousandth. BSP has a display state of charge (dsoc) and a + * remaining state of charge (rsoc). This value will be used for both + * purposes here so we don't do any fancy math to try and "smooth" the + * charge and just report it as it is. Note for example an soc of 100 + * is stored as 100000, an soc of 50 is stored as 50000, etc. + */ + int soc; + + /* + * Capacity of battery when fully charged, equal or less than design + * capacity depending upon wear. BSP kernel saves to nvram in mAh, + * so this value is in mAh not the standard uAh. + */ + int fcc_mah; + + /* + * Calibrate the SOC on a fully charged battery, this way we can use + * the calibrated SOC value to correct for columb counter drift. + */ + bool soc_cal; + + /* Implementation specific immutable properties from device tree */ + int res_div; + int sleep_enter_current_ua; + int sleep_filter_current_ua; + int bat_charge_full_design_uah; + int bat_voltage_min_design_uv; + int bat_voltage_max_design_uv; + + /* Values updated periodically by driver for display. */ + int charge_now_uah; + int volt_avg_uv; + int cur_avg_ua; + int max_chg_cur_ua; + int max_chg_volt_uv; + int charge_status; + int charger_input_volt_avg_uv; + + /* Work queue to periodically update values. */ + struct delayed_work work; +}; + +/* ADC coefficients extracted from BSP kernel */ +#define ADC_TO_CURRENT(adc_value, res_div) \ + (adc_value * 172 / res_div) + +#define CURRENT_TO_ADC(current, samp_res) \ + (current * samp_res / 172) + +#define CHARGE_TO_ADC(capacity, res_div) \ + (capacity * res_div * 3600 / 172 * 1000) + +#define ADC_TO_CHARGE_UAH(adc_value, res_div) \ + (adc_value / 3600 * 172 / res_div) + +static u8 rk817_chg_cur_to_reg(u32 chg_cur_ma) +{ + if (chg_cur_ma >= 3500) + return CHG_3_5A; + else if (chg_cur_ma >= 3000) + return CHG_3A; + else if (chg_cur_ma >= 2750) + return CHG_2_75A; + else if (chg_cur_ma >= 2500) + return CHG_2_5A; + else if (chg_cur_ma >= 2000) + return CHG_2A; + else if (chg_cur_ma >= 1500) + return CHG_1_5A; + else if (chg_cur_ma >= 1000) + return CHG_1A; + else if (chg_cur_ma >= 500) + return CHG_0_5A; + else + return -EINVAL; +} + +static int rk817_chg_cur_from_reg(u8 reg) +{ + switch (reg) { + case CHG_0_5A: + return 500000; + case CHG_1A: + return 1000000; + case CHG_1_5A: + return 1500000; + case CHG_2A: + return 2000000; + case CHG_2_5A: + return 2500000; + case CHG_2_75A: + return 2750000; + case CHG_3A: + return 3000000; + case CHG_3_5A: + return 3500000; + default: + return -EINVAL; + } +} + +static void rk817_bat_calib_vol(struct rk817_charger *charger) +{ + uint32_t vcalib0 = 0; + uint32_t vcalib1 = 0; + u8 bulk_reg[2]; + + /* calibrate voltage */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_VCALIB0_H, + bulk_reg, 2); + vcalib0 = get_unaligned_be16(bulk_reg); + + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_VCALIB1_H, + bulk_reg, 2); + vcalib1 = get_unaligned_be16(bulk_reg); + + /* values were taken from BSP kernel */ + charger->voltage_k = (4025 - 2300) * 1000 / + ((vcalib1 - vcalib0) ? (vcalib1 - vcalib0) : 1); + charger->voltage_b = 4025 - (charger->voltage_k * vcalib1) / 1000; +} + +static void rk817_bat_calib_cur(struct rk817_charger *charger) +{ + u8 bulk_reg[2]; + + /* calibrate current */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_IOFFSET_H, + bulk_reg, 2); + regmap_bulk_write(charger->rk808->regmap, RK817_GAS_GAUGE_CAL_OFFSET_H, + bulk_reg, 2); +} + +/* + * note that only the fcc_mah is really used by this driver, the other values + * are to ensure we can remain backwards compatible with the BSP kernel. + */ +static int rk817_record_battery_nvram_values(struct rk817_charger *charger) +{ + u8 bulk_reg[3]; + int ret, rsoc; + + /* + * write the soc value to the nvram location used by the BSP kernel + * for the dsoc value. + */ + put_unaligned_le24(charger->soc, bulk_reg); + ret = regmap_bulk_write(charger->rk808->regmap, RK817_GAS_GAUGE_BAT_R1, + bulk_reg, 3); + if (ret < 0) + return ret; + /* + * write the remaining capacity in mah to the nvram location used by + * the BSP kernel for the rsoc value. + */ + rsoc = (charger->soc * charger->fcc_mah) / 100000; + put_unaligned_le24(rsoc, bulk_reg); + ret = regmap_bulk_write(charger->rk808->regmap, RK817_GAS_GAUGE_DATA0, + bulk_reg, 3); + if (ret < 0) + return ret; + /* write the fcc_mah in mAh, just as the BSP kernel does. */ + put_unaligned_le24(charger->fcc_mah, bulk_reg); + ret = regmap_bulk_write(charger->rk808->regmap, RK817_GAS_GAUGE_DATA3, + bulk_reg, 3); + if (ret < 0) + return ret; + + return 0; +} + +static int rk817_bat_calib_cap(struct rk817_charger *charger) +{ + struct rk808 *rk808 = charger->rk808; + int tmp, charge_now, charge_now_adc, volt_avg; + u8 bulk_reg[4]; + + /* Calibrate the soc and fcc on a fully charged battery */ + + if (charger->charge_status == CHARGE_FINISH && (!charger->soc_cal)) { + /* + * soc should be 100000 and columb counter should show the full + * charge capacity. Note that if the device is unplugged for a + * period of several days the columb counter will have a large + * margin of error, so setting it back to the full charge on + * a completed charge cycle should correct this (my device was + * showing 33% battery after 3 days unplugged when it should + * have been closer to 95% based on voltage and charge + * current). + */ + + charger->soc = 100000; + charge_now_adc = CHARGE_TO_ADC(charger->fcc_mah, + charger->res_div); + put_unaligned_be32(charge_now_adc, bulk_reg); + regmap_bulk_write(rk808->regmap, RK817_GAS_GAUGE_Q_INIT_H3, + bulk_reg, 4); + + charger->soc_cal = 1; + dev_dbg(charger->dev, + "Fully charged. SOC is %d, full capacity is %d\n", + charger->soc, charger->fcc_mah * 1000); + } + + /* + * The columb counter can drift up slightly, so we should correct for + * it. But don't correct it until we're at 100% soc. + */ + if (charger->charge_status == CHARGE_FINISH && charger->soc_cal) { + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_Q_PRES_H3, + bulk_reg, 4); + charge_now_adc = get_unaligned_be32(bulk_reg); + if (charge_now_adc < 0) + return charge_now_adc; + charge_now = ADC_TO_CHARGE_UAH(charge_now_adc, + charger->res_div); + + /* + * Re-init columb counter with updated values to correct drift. + */ + if (charge_now / 1000 > charger->fcc_mah) { + dev_dbg(charger->dev, + "Recalibrating columb counter to %d uah\n", + charge_now); + /* + * Order of operations matters here to ensure we keep + * enough precision until the last step to keep from + * making needless updates to columb counter. + */ + charge_now_adc = CHARGE_TO_ADC(charger->fcc_mah, + charger->res_div); + put_unaligned_be32(charge_now_adc, bulk_reg); + regmap_bulk_write(rk808->regmap, + RK817_GAS_GAUGE_Q_INIT_H3, + bulk_reg, 4); + } + } + + /* + * Calibrate the fully charged capacity when we previously had a full + * battery (soc_cal = 1) and are now empty (at or below minimum design + * voltage). If our columb counter is still positive, subtract that + * from our fcc value to get a calibrated fcc, and if our columb + * counter is negative add that to our fcc (but not to exceed our + * design capacity). + */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_BAT_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + volt_avg = (charger->voltage_k * tmp) + 1000 * charger->voltage_b; + if (volt_avg <= charger->bat_voltage_min_design_uv && + charger->soc_cal) { + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_Q_PRES_H3, + bulk_reg, 4); + charge_now_adc = get_unaligned_be32(bulk_reg); + charge_now = ADC_TO_CHARGE_UAH(charge_now_adc, + charger->res_div); + /* + * Note, if charge_now is negative this will add it (what we + * want) and if it's positive this will subtract (also what + * we want). + */ + charger->fcc_mah = charger->fcc_mah - (charge_now / 1000); + + dev_dbg(charger->dev, + "Recalibrating full charge capacity to %d uah\n", + charger->fcc_mah * 1000); + } + + rk817_record_battery_nvram_values(charger); + + return 0; +} + +static void rk817_read_props(struct rk817_charger *charger) +{ + int tmp, reg; + u8 bulk_reg[4]; + + /* + * Recalibrate voltage and current readings if we need to BSP does both + * on CUR_CALIB_UPD, ignoring VOL_CALIB_UPD. Curiously enough, both + * documentation and the BSP show that you perform an update if bit 7 + * is 1, but you clear the status by writing a 1 to bit 7. + */ + regmap_read(charger->rk808->regmap, RK817_GAS_GAUGE_ADC_CONFIG1, ®); + if (reg & RK817_VOL_CUR_CALIB_UPD) { + rk817_bat_calib_cur(charger); + rk817_bat_calib_vol(charger); + regmap_write_bits(charger->rk808->regmap, + RK817_GAS_GAUGE_ADC_CONFIG1, + RK817_VOL_CUR_CALIB_UPD, + RK817_VOL_CUR_CALIB_UPD); + } + + /* Update reported charge. */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_Q_PRES_H3, + bulk_reg, 4); + tmp = get_unaligned_be32(bulk_reg); + charger->charge_now_uah = ADC_TO_CHARGE_UAH(tmp, charger->res_div); + if (charger->charge_now_uah < 0) + charger->charge_now_uah = 0; + if (charger->charge_now_uah > charger->fcc_mah * 1000) + charger->charge_now_uah = charger->fcc_mah * 1000; + + /* Update soc based on reported charge. */ + charger->soc = charger->charge_now_uah * 100 / charger->fcc_mah; + + /* Update reported voltage. */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_BAT_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + charger->volt_avg_uv = (charger->voltage_k * tmp) + 1000 * + charger->voltage_b; + + /* + * Update reported current. Note value from registers is a signed 16 + * bit int. + */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_BAT_CUR_H, + bulk_reg, 2); + tmp = (short int)get_unaligned_be16(bulk_reg); + charger->cur_avg_ua = ADC_TO_CURRENT(tmp, charger->res_div); + + /* + * Update the max charge current. This value shouldn't change, but we + * can read it to report what the PMIC says it is instead of simply + * returning the default value. + */ + regmap_read(charger->rk808->regmap, RK817_PMIC_CHRG_OUT, ®); + charger->max_chg_cur_ua = + rk817_chg_cur_from_reg(reg & RK817_CHRG_CUR_SEL); + + /* + * Update max charge voltage. Like the max charge current this value + * shouldn't change, but we can report what the PMIC says. + */ + regmap_read(charger->rk808->regmap, RK817_PMIC_CHRG_OUT, ®); + charger->max_chg_volt_uv = ((((reg & RK817_CHRG_VOL_SEL) >> 4) * + 50000) + 4100000); + + /* Check if battery still present. */ + regmap_read(charger->rk808->regmap, RK817_PMIC_CHRG_STS, ®); + charger->battery_present = (reg & RK817_BAT_EXS); + + /* Get which type of charge we are using (if any). */ + regmap_read(charger->rk808->regmap, RK817_PMIC_CHRG_STS, ®); + charger->charge_status = (reg >> 4) & 0x07; + + /* + * Get charger input voltage. Note that on my example hardware (an + * Odroid Go Advance) the voltage of the power connector is measured + * on the register labelled USB in the datasheet; I don't know if this + * is how it is designed or just a quirk of the implementation. I + * believe this will also measure the voltage of the USB output when in + * OTG mode, if that is the case we may need to change this in the + * future to return 0 if the power supply status is offline (I can't + * test this with my current implementation. Also, when the voltage + * should be zero sometimes the ADC still shows a single bit (which + * would register as 20000uv). When this happens set it to 0. + */ + regmap_bulk_read(charger->rk808->regmap, RK817_GAS_GAUGE_USB_VOL_H, + bulk_reg, 2); + reg = get_unaligned_be16(bulk_reg); + if (reg > 1) { + tmp = ((charger->voltage_k * reg / 1000 + charger->voltage_b) * + 60 / 46); + charger->charger_input_volt_avg_uv = tmp * 1000; + } else { + charger->charger_input_volt_avg_uv = 0; + } + + /* Calibrate battery capacity and soc. */ + rk817_bat_calib_cap(charger); +} + +static int rk817_bat_get_prop(struct power_supply *ps, + enum power_supply_property prop, + union power_supply_propval *val) +{ + struct rk817_charger *charger = power_supply_get_drvdata(ps); + + switch (prop) { + case POWER_SUPPLY_PROP_PRESENT: + val->intval = charger->battery_present; + break; + case POWER_SUPPLY_PROP_STATUS: + if (charger->cur_avg_ua < 0) { + val->intval = POWER_SUPPLY_STATUS_DISCHARGING; + break; + } + switch (charger->charge_status) { + case CHRG_OFF: + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; + break; + /* + * Dead charge is documented, but not explained. I never + * observed it but assume it's a pre-charge for a dead + * battery. + */ + case DEAD_CHRG: + case TRICKLE_CHRG: + case CC_OR_CV_CHRG: + val->intval = POWER_SUPPLY_STATUS_CHARGING; + break; + case CHARGE_FINISH: + val->intval = POWER_SUPPLY_STATUS_FULL; + break; + default: + val->intval = POWER_SUPPLY_STATUS_UNKNOWN; + return -EINVAL; + + } + break; + case POWER_SUPPLY_PROP_CHARGE_TYPE: + switch (charger->charge_status) { + case CHRG_OFF: + case CHARGE_FINISH: + val->intval = POWER_SUPPLY_CHARGE_TYPE_NONE; + break; + case TRICKLE_CHRG: + val->intval = POWER_SUPPLY_CHARGE_TYPE_TRICKLE; + break; + case DEAD_CHRG: + case CC_OR_CV_CHRG: + val->intval = POWER_SUPPLY_CHARGE_TYPE_STANDARD; + break; + default: + val->intval = POWER_SUPPLY_CHARGE_TYPE_UNKNOWN; + break; + } + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + val->intval = charger->fcc_mah * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + val->intval = charger->bat_charge_full_design_uah; + break; + case POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN: + val->intval = 0; + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + val->intval = charger->charge_now_uah; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + val->intval = charger->bat_voltage_min_design_uv; + break; + case POWER_SUPPLY_PROP_CAPACITY: + /* Add 500 so that values like 99999 are 100% not 99%. */ + val->intval = (charger->soc + 500) / 1000; + if (val->intval > 100) + val->intval = 100; + if (val->intval < 0) + val->intval = 0; + break; + case POWER_SUPPLY_PROP_VOLTAGE_AVG: + val->intval = charger->volt_avg_uv; + break; + case POWER_SUPPLY_PROP_CURRENT_AVG: + val->intval = charger->cur_avg_ua; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + val->intval = charger->max_chg_cur_ua; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX: + val->intval = charger->max_chg_volt_uv; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + val->intval = charger->bat_voltage_max_design_uv; + break; + default: + return -EINVAL; + } + return 0; +} + +static int rk817_chg_get_prop(struct power_supply *ps, + enum power_supply_property prop, + union power_supply_propval *val) +{ + struct rk817_charger *charger = power_supply_get_drvdata(ps); + + switch (prop) { + case POWER_SUPPLY_PROP_ONLINE: + val->intval = charger->plugged_in; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + /* max voltage from datasheet at 5.5v (default 5.0v) */ + val->intval = 5500000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + /* min voltage from datasheet at 3.8v (default 5.0v) */ + val->intval = 3800000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_AVG: + val->intval = charger->charger_input_volt_avg_uv; + break; + /* + * While it's possible that other implementations could use different + * USB types, the current implementation for this PMIC (the Odroid Go + * Advance) only uses a dedicated charging port with no rx/tx lines. + */ + case POWER_SUPPLY_PROP_USB_TYPE: + val->intval = POWER_SUPPLY_USB_TYPE_DCP; + break; + default: + return -EINVAL; + } + return 0; + +} + +static irqreturn_t rk817_plug_in_isr(int irq, void *cg) +{ + struct rk817_charger *charger; + + charger = (struct rk817_charger *)cg; + charger->plugged_in = 1; + power_supply_changed(charger->chg_ps); + power_supply_changed(charger->bat_ps); + /* try to recalibrate capacity if we hit full charge. */ + charger->soc_cal = 0; + + rk817_read_props(charger); + + dev_dbg(charger->dev, "Power Cord Inserted\n"); + + return IRQ_HANDLED; +} + +static irqreturn_t rk817_plug_out_isr(int irq, void *cg) +{ + struct rk817_charger *charger; + struct rk808 *rk808; + + charger = (struct rk817_charger *)cg; + rk808 = charger->rk808; + charger->plugged_in = 0; + power_supply_changed(charger->bat_ps); + power_supply_changed(charger->chg_ps); + + /* + * For some reason the bits of RK817_PMIC_CHRG_IN reset whenever the + * power cord is unplugged. This was not documented in the BSP kernel + * or the datasheet and only discovered by trial and error. Set minimum + * USB input voltage to 4.5v and enable USB voltage input limit. + */ + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, + RK817_USB_VLIM_SEL, (0x05 << 4)); + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, RK817_USB_VLIM_EN, + (0x01 << 7)); + + /* + * Set average USB input current limit to 1.5A and enable USB current + * input limit. + */ + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, + RK817_USB_ILIM_SEL, 0x03); + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, RK817_USB_ILIM_EN, + (0x01 << 3)); + + rk817_read_props(charger); + + dev_dbg(charger->dev, "Power Cord Removed\n"); + + return IRQ_HANDLED; +} + +static enum power_supply_property rk817_bat_props[] = { + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_CHARGE_TYPE, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_EMPTY_DESIGN, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE_MAX, + POWER_SUPPLY_PROP_VOLTAGE_AVG, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CURRENT_AVG, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, +}; + +static enum power_supply_property rk817_chg_props[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_USB_TYPE, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_AVG, +}; + +static enum power_supply_usb_type rk817_usb_type[] = { + POWER_SUPPLY_USB_TYPE_DCP, + POWER_SUPPLY_USB_TYPE_UNKNOWN, +}; + +static const struct power_supply_desc rk817_bat_desc = { + .name = "rk817-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .properties = rk817_bat_props, + .num_properties = ARRAY_SIZE(rk817_bat_props), + .get_property = rk817_bat_get_prop, +}; + +static const struct power_supply_desc rk817_chg_desc = { + .name = "rk817-charger", + .type = POWER_SUPPLY_TYPE_USB, + .usb_types = rk817_usb_type, + .num_usb_types = ARRAY_SIZE(rk817_usb_type), + .properties = rk817_chg_props, + .num_properties = ARRAY_SIZE(rk817_chg_props), + .get_property = rk817_chg_get_prop, +}; + +static int rk817_read_battery_nvram_values(struct rk817_charger *charger) +{ + u8 bulk_reg[3]; + int ret; + + /* Read the nvram data for full charge capacity. */ + ret = regmap_bulk_read(charger->rk808->regmap, + RK817_GAS_GAUGE_DATA3, bulk_reg, 3); + if (ret < 0) + return ret; + charger->fcc_mah = get_unaligned_le24(bulk_reg); + + /* + * Sanity checking for values equal to zero or less than would be + * practical for this device (BSP Kernel assumes 500mAH or less) for + * practicality purposes. Also check if the value is too large and + * correct it. + */ + if ((charger->fcc_mah < 500) || + ((charger->fcc_mah * 1000) > charger->bat_charge_full_design_uah)) { + dev_info(charger->dev, + "Invalid NVRAM max charge, setting to %u uAH\n", + charger->bat_charge_full_design_uah); + charger->fcc_mah = charger->bat_charge_full_design_uah / 1000; + } + + /* + * Read the nvram for state of charge. Sanity check for values greater + * than 100 (10000). If the value is off it should get corrected + * automatically when the voltage drops to the min (soc is 0) or when + * the battery is full (soc is 100). + */ + ret = regmap_bulk_read(charger->rk808->regmap, + RK817_GAS_GAUGE_BAT_R1, bulk_reg, 3); + if (ret < 0) + return ret; + charger->soc = get_unaligned_le24(bulk_reg); + if (charger->soc > 10000) + charger->soc = 10000; + + return 0; +} + +static int +rk817_read_or_set_full_charge_on_boot(struct rk817_charger *charger, + struct power_supply_battery_info *bat_info) +{ + struct rk808 *rk808 = charger->rk808; + u8 bulk_reg[4]; + u32 boot_voltage, boot_charge_mah, tmp; + int ret, reg, off_time; + bool first_boot; + + /* + * Check if the battery is uninitalized. If it is, the columb counter + * needs to be set up. + */ + ret = regmap_read(rk808->regmap, RK817_GAS_GAUGE_GG_STS, ®); + if (ret < 0) + return ret; + first_boot = reg & RK817_BAT_CON; + /* + * If the battery is uninitialized, use the poweron voltage and an ocv + * lookup to guess our charge. The number won't be very accurate until + * we hit either our minimum voltage (0%) or full charge (100%). + */ + if (first_boot) { + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_PWRON_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + boot_voltage = (charger->voltage_k * tmp) + + 1000 * charger->voltage_b; + /* + * Since only implementation has no working thermistor, assume + * 20C for OCV lookup. If lookup fails, report error with OCV + * table. + */ + charger->soc = power_supply_batinfo_ocv2cap(bat_info, + boot_voltage, + 20) * 1000; + if (charger->soc < 0) + charger->soc = 0; + + /* Guess that full charge capacity is the design capacity */ + charger->fcc_mah = charger->bat_charge_full_design_uah / 1000; + /* + * Set battery as "set up". BSP driver uses this value even + * though datasheet claims it's a read-only value. + */ + regmap_write_bits(rk808->regmap, RK817_GAS_GAUGE_GG_STS, + RK817_BAT_CON, 0); + /* Save nvram values */ + ret = rk817_record_battery_nvram_values(charger); + if (ret < 0) + return ret; + } else { + ret = rk817_read_battery_nvram_values(charger); + if (ret < 0) + return ret; + + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_Q_PRES_H3, + bulk_reg, 4); + tmp = get_unaligned_be32(bulk_reg); + if (tmp < 0) + tmp = 0; + boot_charge_mah = ADC_TO_CHARGE_UAH(tmp, + charger->res_div) / 1000; + /* + * Check if the columb counter has been off for more than 300 + * minutes as it tends to drift downward. If so, re-init soc + * with the boot voltage instead. Note the unit values for the + * OFF_CNT register appear to be in decaminutes and stops + * counting at 2550 (0xFF) minutes. BSP kernel used OCV, but + * for me occasionally that would show invalid values. Boot + * voltage is only accurate for me on first poweron (not + * reboots), but we shouldn't ever encounter an OFF_CNT more + * than 0 on a reboot anyway. + */ + regmap_read(rk808->regmap, RK817_GAS_GAUGE_OFF_CNT, &off_time); + if (off_time >= 30) { + regmap_bulk_read(rk808->regmap, + RK817_GAS_GAUGE_PWRON_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + boot_voltage = (charger->voltage_k * tmp) + + 1000 * charger->voltage_b; + charger->soc = + power_supply_batinfo_ocv2cap(bat_info, + boot_voltage, + 20) * 1000; + } else { + charger->soc = (boot_charge_mah * 1000 * 100 / + charger->fcc_mah); + } + } + + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_PWRON_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + boot_voltage = (charger->voltage_k * tmp) + 1000 * charger->voltage_b; + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_Q_PRES_H3, + bulk_reg, 4); + tmp = get_unaligned_be32(bulk_reg); + if (tmp < 0) + tmp = 0; + boot_charge_mah = ADC_TO_CHARGE_UAH(tmp, charger->res_div) / 1000; + regmap_bulk_read(rk808->regmap, RK817_GAS_GAUGE_OCV_VOL_H, + bulk_reg, 2); + tmp = get_unaligned_be16(bulk_reg); + boot_voltage = (charger->voltage_k * tmp) + 1000 * charger->voltage_b; + + /* + * Now we have our full charge capacity and soc, init the columb + * counter. + */ + boot_charge_mah = charger->soc * charger->fcc_mah / 100 / 1000; + if (boot_charge_mah > charger->fcc_mah) + boot_charge_mah = charger->fcc_mah; + tmp = CHARGE_TO_ADC(boot_charge_mah, charger->res_div); + put_unaligned_be32(tmp, bulk_reg); + ret = regmap_bulk_write(rk808->regmap, RK817_GAS_GAUGE_Q_INIT_H3, + bulk_reg, 4); + if (ret < 0) + return ret; + + /* Set QMAX value to max design capacity. */ + tmp = CHARGE_TO_ADC((charger->bat_charge_full_design_uah / 1000), + charger->res_div); + put_unaligned_be32(tmp, bulk_reg); + ret = regmap_bulk_write(rk808->regmap, RK817_GAS_GAUGE_Q_MAX_H3, + bulk_reg, 4); + if (ret < 0) + return ret; + + return 0; +} + +static int rk817_battery_init(struct rk817_charger *charger, + struct power_supply_battery_info *bat_info) +{ + struct rk808 *rk808 = charger->rk808; + u32 tmp, max_chg_vol_mv, max_chg_cur_ma; + u8 max_chg_vol_reg, chg_term_i_reg, max_chg_cur_reg; + int ret, chg_term_ma; + u8 bulk_reg[2]; + + /* Get initial plug state */ + regmap_read(rk808->regmap, RK817_SYS_STS, &tmp); + charger->plugged_in = (tmp & RK817_PLUG_IN_STS); + + /* + * Turn on all ADC functions to measure battery, USB, and sys voltage, + * as well as batt temp. Note only tested implementation so far does + * not use a battery with a thermistor. + */ + regmap_write(rk808->regmap, RK817_GAS_GAUGE_ADC_CONFIG0, 0xfc); + + /* + * Set relax mode voltage sampling interval and ADC offset calibration + * interval to 8 minutes to mirror BSP kernel. Set voltage and current + * modes to average to mirror BSP kernel. + */ + regmap_write(rk808->regmap, RK817_GAS_GAUGE_GG_CON, 0x04); + + /* Calibrate voltage like the BSP does here. */ + rk817_bat_calib_vol(charger); + + /* Write relax threshold, derived from sleep enter current. */ + tmp = CURRENT_TO_ADC(charger->sleep_enter_current_ua, + charger->res_div); + put_unaligned_be16(tmp, bulk_reg); + regmap_bulk_write(rk808->regmap, RK817_GAS_GAUGE_RELAX_THRE_H, + bulk_reg, 2); + + /* Write sleep sample current, derived from sleep filter current. */ + tmp = CURRENT_TO_ADC(charger->sleep_filter_current_ua, + charger->res_div); + put_unaligned_be16(tmp, bulk_reg); + regmap_bulk_write(rk808->regmap, RK817_GAS_GAUGE_SLEEP_CON_SAMP_CUR_H, + bulk_reg, 2); + + /* Restart battery relax voltage */ + regmap_write_bits(rk808->regmap, RK817_GAS_GAUGE_GG_STS, + RK817_RELAX_VOL_UPD, (0x0 << 2)); + + /* + * Set OCV Threshold Voltage to 127.5mV. This was hard coded like this + * in the BSP. + */ + regmap_write(rk808->regmap, RK817_GAS_GAUGE_OCV_THRE_VOL, 0xff); + + /* + * Set maximum charging voltage to battery max voltage. Trying to be + * incredibly safe with these value, as setting them wrong could + * overcharge the battery, which would be very bad. + */ + max_chg_vol_mv = bat_info->constant_charge_voltage_max_uv / 1000; + max_chg_cur_ma = bat_info->constant_charge_current_max_ua / 1000; + + if (max_chg_vol_mv < 4100) { + return dev_err_probe(charger->dev, -EINVAL, + "invalid max charger voltage, value %u unsupported\n", + max_chg_vol_mv * 1000); + } + if (max_chg_vol_mv > 4450) { + dev_info(charger->dev, + "Setting max charge voltage to 4450000uv\n"); + max_chg_vol_mv = 4450; + } + + if (max_chg_cur_ma < 500) { + return dev_err_probe(charger->dev, -EINVAL, + "invalid max charger current, value %u unsupported\n", + max_chg_cur_ma * 1000); + } + if (max_chg_cur_ma > 3500) + dev_info(charger->dev, + "Setting max charge current to 3500000ua\n"); + + /* + * Now that the values are sanity checked, if we subtract 4100 from the + * max voltage and divide by 50, we conviently get the exact value for + * the registers, which are 4.1v, 4.15v, 4.2v, 4.25v, 4.3v, 4.35v, + * 4.4v, and 4.45v; these correspond to values 0x00 through 0x07. + */ + max_chg_vol_reg = (max_chg_vol_mv - 4100) / 50; + + max_chg_cur_reg = rk817_chg_cur_to_reg(max_chg_cur_ma); + + if (max_chg_vol_reg < 0 || max_chg_vol_reg > 7) { + return dev_err_probe(charger->dev, -EINVAL, + "invalid max charger voltage, value %u unsupported\n", + max_chg_vol_mv * 1000); + } + if (max_chg_cur_reg < 0 || max_chg_cur_reg > 7) { + return dev_err_probe(charger->dev, -EINVAL, + "invalid max charger current, value %u unsupported\n", + max_chg_cur_ma * 1000); + } + + /* + * Write the values to the registers, and deliver an emergency warning + * in the event they are not written correctly. + */ + ret = regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_OUT, + RK817_CHRG_VOL_SEL, (max_chg_vol_reg << 4)); + if (ret) { + dev_emerg(charger->dev, + "Danger, unable to set max charger voltage: %u\n", + ret); + } + + ret = regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_OUT, + RK817_CHRG_CUR_SEL, max_chg_cur_reg); + if (ret) { + dev_emerg(charger->dev, + "Danger, unable to set max charger current: %u\n", + ret); + } + + /* Set charge finishing mode to analog */ + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_TERM, + RK817_CHRG_TERM_ANA_DIG, (0x0 << 2)); + + /* + * Set charge finish current, warn if value not in range and keep + * default. + */ + chg_term_ma = bat_info->charge_term_current_ua / 1000; + if (chg_term_ma < 150 || chg_term_ma > 400) { + dev_warn(charger->dev, + "Invalid charge termination %u, keeping default\n", + chg_term_ma * 1000); + chg_term_ma = 200; + } + + /* + * Values of 150ma, 200ma, 300ma, and 400ma correspond to 00, 01, 10, + * and 11. + */ + chg_term_i_reg = (chg_term_ma - 100) / 100; + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_TERM, + RK817_CHRG_TERM_ANA_SEL, chg_term_i_reg); + + ret = rk817_read_or_set_full_charge_on_boot(charger, bat_info); + if (ret < 0) + return ret; + + /* + * Set minimum USB input voltage to 4.5v and enable USB voltage input + * limit. + */ + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, + RK817_USB_VLIM_SEL, (0x05 << 4)); + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, RK817_USB_VLIM_EN, + (0x01 << 7)); + + /* + * Set average USB input current limit to 1.5A and enable USB current + * input limit. + */ + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, + RK817_USB_ILIM_SEL, 0x03); + regmap_write_bits(rk808->regmap, RK817_PMIC_CHRG_IN, RK817_USB_ILIM_EN, + (0x01 << 3)); + + return 0; +} + +static void rk817_charging_monitor(struct work_struct *work) +{ + struct rk817_charger *charger; + + charger = container_of(work, struct rk817_charger, work.work); + + rk817_read_props(charger); + + /* Run every 8 seconds like the BSP driver did. */ + queue_delayed_work(system_wq, &charger->work, msecs_to_jiffies(8000)); +} + +static int rk817_charger_probe(struct platform_device *pdev) +{ + struct rk808 *rk808 = dev_get_drvdata(pdev->dev.parent); + struct rk817_charger *charger; + struct device_node *node; + struct power_supply_battery_info *bat_info; + struct device *dev = &pdev->dev; + struct power_supply_config pscfg = {}; + int plugin_irq, plugout_irq; + int of_value; + int ret; + + node = of_get_child_by_name(dev->parent->of_node, "charger"); + if (!node) + return -ENODEV; + + charger = devm_kzalloc(&pdev->dev, sizeof(*charger), GFP_KERNEL); + if (!charger) + return -ENOMEM; + + charger->rk808 = rk808; + + charger->dev = &pdev->dev; + platform_set_drvdata(pdev, charger); + + rk817_bat_calib_vol(charger); + + pscfg.drv_data = charger; + pscfg.of_node = node; + + /* + * Get sample resistor value. Note only values of 10000 or 20000 + * microohms are allowed. Schematic for my test implementation (an + * Odroid Go Advance) shows a 10 milliohm resistor for reference. + */ + ret = of_property_read_u32(node, "rockchip,resistor-sense-micro-ohms", + &of_value); + if (ret < 0) { + return dev_err_probe(dev, ret, + "Error reading sample resistor value\n"); + } + /* + * Store as a 1 or a 2, since all we really use the value for is as a + * divisor in some calculations. + */ + charger->res_div = (of_value == 20000) ? 2 : 1; + + /* + * Get sleep enter current value. Not sure what this value is for + * other than to help calibrate the relax threshold. + */ + ret = of_property_read_u32(node, + "rockchip,sleep-enter-current-microamp", + &of_value); + if (ret < 0) { + return dev_err_probe(dev, ret, + "Error reading sleep enter cur value\n"); + } + charger->sleep_enter_current_ua = of_value; + + /* Get sleep filter current value */ + ret = of_property_read_u32(node, + "rockchip,sleep-filter-current-microamp", + &of_value); + if (ret < 0) { + return dev_err_probe(dev, ret, + "Error reading sleep filter cur value\n"); + } + + charger->sleep_filter_current_ua = of_value; + + charger->bat_ps = devm_power_supply_register(&pdev->dev, + &rk817_bat_desc, &pscfg); + + charger->chg_ps = devm_power_supply_register(&pdev->dev, + &rk817_chg_desc, &pscfg); + + if (IS_ERR(charger->chg_ps)) + return dev_err_probe(dev, -EINVAL, + "Battery failed to probe\n"); + + if (IS_ERR(charger->chg_ps)) + return dev_err_probe(dev, -EINVAL, + "Charger failed to probe\n"); + + ret = power_supply_get_battery_info(charger->bat_ps, + &bat_info); + if (ret) { + return dev_err_probe(dev, ret, + "Unable to get battery info: %d\n", ret); + } + + if ((bat_info->charge_full_design_uah <= 0) || + (bat_info->voltage_min_design_uv <= 0) || + (bat_info->voltage_max_design_uv <= 0) || + (bat_info->constant_charge_voltage_max_uv <= 0) || + (bat_info->constant_charge_current_max_ua <= 0) || + (bat_info->charge_term_current_ua <= 0)) { + return dev_err_probe(dev, -EINVAL, + "Required bat info missing or invalid\n"); + } + + charger->bat_charge_full_design_uah = bat_info->charge_full_design_uah; + charger->bat_voltage_min_design_uv = bat_info->voltage_min_design_uv; + charger->bat_voltage_max_design_uv = bat_info->voltage_max_design_uv; + + /* + * Has to run after power_supply_get_battery_info as it depends on some + * values discovered from that routine. + */ + ret = rk817_battery_init(charger, bat_info); + if (ret) + return ret; + + power_supply_put_battery_info(charger->bat_ps, bat_info); + + plugin_irq = platform_get_irq(pdev, 0); + if (plugin_irq < 0) + return plugin_irq; + + plugout_irq = platform_get_irq(pdev, 1); + if (plugout_irq < 0) + return plugout_irq; + + ret = devm_request_threaded_irq(charger->dev, plugin_irq, NULL, + rk817_plug_in_isr, + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "rk817_plug_in", charger); + if (ret) { + return dev_err_probe(&pdev->dev, ret, + "plug_in_irq request failed!\n"); + } + + ret = devm_request_threaded_irq(charger->dev, plugout_irq, NULL, + rk817_plug_out_isr, + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "rk817_plug_out", charger); + if (ret) { + return dev_err_probe(&pdev->dev, ret, + "plug_out_irq request failed!\n"); + } + + ret = devm_delayed_work_autocancel(&pdev->dev, &charger->work, + rk817_charging_monitor); + if (ret) + return ret; + + /* Force the first update immediately. */ + mod_delayed_work(system_wq, &charger->work, 0); + + return 0; +} + + +static struct platform_driver rk817_charger_driver = { + .probe = rk817_charger_probe, + .driver = { + .name = "rk817-charger", + }, +}; +module_platform_driver(rk817_charger_driver); + +MODULE_DESCRIPTION("Battery power supply driver for RK817 PMIC"); +MODULE_AUTHOR("Maya Matuszczyk "); +MODULE_AUTHOR("Chris Morgan "); +MODULE_LICENSE("GPL"); diff --git a/drivers/power/supply/rt5033_battery.c b/drivers/power/supply/rt5033_battery.c index 7a23c70f48791b119048d3c76f358ce12f3edc4f..736dec608ff6d9e66c2cbd4b776a26faed46d5f4 100644 --- a/drivers/power/supply/rt5033_battery.c +++ b/drivers/power/supply/rt5033_battery.c @@ -149,13 +149,11 @@ static int rt5033_battery_probe(struct i2c_client *client, return 0; } -static int rt5033_battery_remove(struct i2c_client *client) +static void rt5033_battery_remove(struct i2c_client *client) { struct rt5033_battery *battery = i2c_get_clientdata(client); power_supply_unregister(battery->psy); - - return 0; } static const struct i2c_device_id rt5033_battery_id[] = { diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 74ee54320e6a85cba98f1b1b6b00c33160ab720c..72962286d70459eee3c4f3e35ceda5e823c45796 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1698,7 +1698,7 @@ put_usb_notifier: return ret; } -static int rt9455_remove(struct i2c_client *client) +static void rt9455_remove(struct i2c_client *client) { int ret; struct rt9455_info *info = i2c_get_clientdata(client); @@ -1715,8 +1715,6 @@ static int rt9455_remove(struct i2c_client *client) cancel_delayed_work_sync(&info->pwr_rdy_work); cancel_delayed_work_sync(&info->max_charging_time_work); cancel_delayed_work_sync(&info->batt_presence_work); - - return 0; } static const struct i2c_device_id rt9455_i2c_id_table[] = { diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c index 1511f71f937c73ad125001fb3016f529acfa05cb..996a82f8a2a1dd11a9c6e2dd809355f12bc6f981 100644 --- a/drivers/power/supply/smb347-charger.c +++ b/drivers/power/supply/smb347-charger.c @@ -1595,14 +1595,12 @@ static int smb347_probe(struct i2c_client *client, return 0; } -static int smb347_remove(struct i2c_client *client) +static void smb347_remove(struct i2c_client *client) { struct smb347_charger *smb = i2c_get_clientdata(client); smb347_usb_vbus_regulator_disable(smb->usb_rdev); smb347_irq_disable(smb); - - return 0; } static void smb347_shutdown(struct i2c_client *client) diff --git a/drivers/power/supply/tps65217_charger.c b/drivers/power/supply/tps65217_charger.c index ba33d1617e0b6d986a9c81d7c91859af9c933aed..a4bc9f2a10bca1619bbd326ef2e9a2e54ef8c465 100644 --- a/drivers/power/supply/tps65217_charger.c +++ b/drivers/power/supply/tps65217_charger.c @@ -50,7 +50,7 @@ static int tps65217_config_charger(struct tps65217_charger *charger) * tps65217 rev. G, p. 31 (see p. 32 for NTC schematic) * * The device can be configured to support a 100k NTC (B = 3960) by - * setting the the NTC_TYPE bit in register CHGCONFIG1 to 1. However it + * setting the NTC_TYPE bit in register CHGCONFIG1 to 1. However it * is not recommended to do so. In sleep mode, the charger continues * charging the battery, but all register values are reset to default * values. Therefore, the charger would get the wrong temperature diff --git a/drivers/power/supply/z2_battery.c b/drivers/power/supply/z2_battery.c index 7ed4e4bb26ecaad5d880a20c9632ba5d79cdb866..1897c29848600b6c214732f927295ac191961f21 100644 --- a/drivers/power/supply/z2_battery.c +++ b/drivers/power/supply/z2_battery.c @@ -251,7 +251,7 @@ err: return ret; } -static int z2_batt_remove(struct i2c_client *client) +static void z2_batt_remove(struct i2c_client *client) { struct z2_charger *charger = i2c_get_clientdata(client); @@ -263,8 +263,6 @@ static int z2_batt_remove(struct i2c_client *client) free_irq(gpiod_to_irq(charger->charge_gpiod), charger); kfree(charger); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c index a20bf12f3ce31a670e0a8960342fefa529fc5a22..999e218d779397a479cbdc83293afdca1cfc4fa7 100644 --- a/drivers/powercap/idle_inject.c +++ b/drivers/powercap/idle_inject.c @@ -254,7 +254,7 @@ void idle_inject_stop(struct idle_inject_device *ii_dev) iit = per_cpu_ptr(&idle_inject_thread, cpu); iit->should_run = 0; - wait_task_inactive(iit->tsk, 0); + wait_task_inactive(iit->tsk, TASK_ANY); } cpu_hotplug_enable(); diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 21d624f9f5fb2028d780f0072fe93702751a0eb7..26d00b1853b4219a276f80257a11fdf03ed47512 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -994,6 +994,9 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value, y = value & 0x1f; value = (1 << y) * (4 + f) * rp->time_unit / 4; } else { + if (value < rp->time_unit) + return 0; + do_div(value, rp->time_unit); y = ilog2(value); f = div64_u64(4 * (value - (1 << y)), 1 << y); @@ -1035,7 +1038,6 @@ static const struct rapl_defaults rapl_defaults_spr_server = { .check_unit = rapl_check_unit_core, .set_floor_freq = set_floor_freq_default, .compute_time_window = rapl_compute_time_window_core, - .dram_domain_energy_unit = 15300, .psys_domain_energy_unit = 1000000000, .spr_psys_bits = true, }; @@ -1110,6 +1112,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &rapl_defaults_core), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &rapl_defaults_core), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core), diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c index 688cde320bb05fd537b17c3d5e893dbd6d86cfa6..51cae72bb6db270bc94a3c63db11581e8096996d 100644 --- a/drivers/ptp/ptp_clock.c +++ b/drivers/ptp/ptp_clock.c @@ -174,7 +174,7 @@ static void ptp_clock_release(struct device *dev) mutex_destroy(&ptp->tsevq_mux); mutex_destroy(&ptp->pincfg_mux); mutex_destroy(&ptp->n_vclocks_mux); - ida_simple_remove(&ptp_clocks_map, ptp->index); + ida_free(&ptp_clocks_map, ptp->index); kfree(ptp); } @@ -217,7 +217,7 @@ struct ptp_clock *ptp_clock_register(struct ptp_clock_info *info, if (ptp == NULL) goto no_memory; - index = ida_simple_get(&ptp_clocks_map, 0, MINORMASK + 1, GFP_KERNEL); + index = ida_alloc_max(&ptp_clocks_map, MINORMASK, GFP_KERNEL); if (index < 0) { err = index; goto no_slot; @@ -332,7 +332,7 @@ kworker_err: mutex_destroy(&ptp->tsevq_mux); mutex_destroy(&ptp->pincfg_mux); mutex_destroy(&ptp->n_vclocks_mux); - ida_simple_remove(&ptp_clocks_map, index); + ida_free(&ptp_clocks_map, index); no_slot: kfree(ptp); no_memory: diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index e59ea2173aac88786c319e9364847c7a2d503629..a48d9b7d29217d29c4443702a50e6fdb9636774b 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -1311,12 +1311,6 @@ fail: goto out; } -static int -ptp_ocp_firstchild(struct device *dev, void *data) -{ - return 1; -} - static struct device * ptp_ocp_find_flash(struct ptp_ocp *bp) { @@ -1325,7 +1319,7 @@ ptp_ocp_find_flash(struct ptp_ocp *bp) last = NULL; dev = &bp->spi_flash->dev; - while ((dev = device_find_child(dev, NULL, ptp_ocp_firstchild))) { + while ((dev = device_find_any_child(dev))) { if (!strcmp("mtd", dev_bus_name(dev))) break; put_device(last); @@ -3663,6 +3657,7 @@ ptp_ocp_detach_sysfs(struct ptp_ocp *bp) struct device *dev = &bp->dev; sysfs_remove_link(&dev->kobj, "ttyGNSS"); + sysfs_remove_link(&dev->kobj, "ttyGNSS2"); sysfs_remove_link(&dev->kobj, "ttyMAC"); sysfs_remove_link(&dev->kobj, "ptp"); sysfs_remove_link(&dev->kobj, "pps"); diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 0e042410f6b977e03084ce2cf3082dbb2960e624..d333e7422f4a9c9e9619d603934c8f5ba0a4c030 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -678,7 +678,7 @@ static struct pwm_chip *fwnode_to_pwmchip(struct fwnode_handle *fwnode) mutex_lock(&pwm_lock); list_for_each_entry(chip, &pwm_chips, list) - if (chip->dev && dev_fwnode(chip->dev) == fwnode) { + if (chip->dev && device_match_fwnode(chip->dev, fwnode)) { mutex_unlock(&pwm_lock); return chip; } @@ -734,8 +734,8 @@ static struct device_link *pwm_device_link_add(struct device *dev, * Returns: A pointer to the requested PWM device or an ERR_PTR()-encoded * error code on failure. */ -struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np, - const char *con_id) +static struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np, + const char *con_id) { struct pwm_device *pwm = NULL; struct of_phandle_args args; @@ -797,7 +797,6 @@ put: return pwm; } -EXPORT_SYMBOL_GPL(of_pwm_get); /** * acpi_pwm_get() - request a PWM via parsing "pwms" property in ACPI @@ -1070,36 +1069,6 @@ struct pwm_device *devm_pwm_get(struct device *dev, const char *con_id) } EXPORT_SYMBOL_GPL(devm_pwm_get); -/** - * devm_of_pwm_get() - resource managed of_pwm_get() - * @dev: device for PWM consumer - * @np: device node to get the PWM from - * @con_id: consumer name - * - * This function performs like of_pwm_get() but the acquired PWM device will - * automatically be released on driver detach. - * - * Returns: A pointer to the requested PWM device or an ERR_PTR()-encoded - * error code on failure. - */ -struct pwm_device *devm_of_pwm_get(struct device *dev, struct device_node *np, - const char *con_id) -{ - struct pwm_device *pwm; - int ret; - - pwm = of_pwm_get(dev, np, con_id); - if (IS_ERR(pwm)) - return pwm; - - ret = devm_add_action_or_reset(dev, devm_pwm_release, pwm); - if (ret) - return ERR_PTR(ret); - - return pwm; -} -EXPORT_SYMBOL_GPL(devm_of_pwm_get); - /** * devm_fwnode_pwm_get() - request a resource managed PWM from firmware node * @dev: device for PWM consumer diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c index c893ec3d2fb43b8ecc04e6dca76927ac3d2c6239..98413d36433818e97e784230fa0ed715a5e65e80 100644 --- a/drivers/pwm/pwm-lpss-pci.c +++ b/drivers/pwm/pwm-lpss-pci.c @@ -14,35 +14,6 @@ #include "pwm-lpss.h" -/* BayTrail */ -static const struct pwm_lpss_boardinfo pwm_lpss_byt_info = { - .clk_rate = 25000000, - .npwm = 1, - .base_unit_bits = 16, -}; - -/* Braswell */ -static const struct pwm_lpss_boardinfo pwm_lpss_bsw_info = { - .clk_rate = 19200000, - .npwm = 1, - .base_unit_bits = 16, -}; - -/* Broxton */ -static const struct pwm_lpss_boardinfo pwm_lpss_bxt_info = { - .clk_rate = 19200000, - .npwm = 4, - .base_unit_bits = 22, - .bypass = true, -}; - -/* Tangier */ -static const struct pwm_lpss_boardinfo pwm_lpss_tng_info = { - .clk_rate = 19200000, - .npwm = 4, - .base_unit_bits = 22, -}; - static int pwm_lpss_probe_pci(struct pci_dev *pdev, const struct pci_device_id *id) { @@ -54,8 +25,12 @@ static int pwm_lpss_probe_pci(struct pci_dev *pdev, if (err < 0) return err; + err = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); + if (err) + return err; + info = (struct pwm_lpss_boardinfo *)id->driver_data; - lpwm = pwm_lpss_probe(&pdev->dev, &pdev->resource[0], info); + lpwm = pwm_lpss_probe(&pdev->dev, pcim_iomap_table(pdev)[0], info); if (IS_ERR(lpwm)) return PTR_ERR(lpwm); @@ -73,7 +48,6 @@ static void pwm_lpss_remove_pci(struct pci_dev *pdev) pm_runtime_get_sync(&pdev->dev); } -#ifdef CONFIG_PM static int pwm_lpss_runtime_suspend_pci(struct device *dev) { /* @@ -87,12 +61,11 @@ static int pwm_lpss_runtime_resume_pci(struct device *dev) { return 0; } -#endif -static const struct dev_pm_ops pwm_lpss_pci_pm = { - SET_RUNTIME_PM_OPS(pwm_lpss_runtime_suspend_pci, - pwm_lpss_runtime_resume_pci, NULL) -}; +static DEFINE_RUNTIME_DEV_PM_OPS(pwm_lpss_pci_pm, + pwm_lpss_runtime_suspend_pci, + pwm_lpss_runtime_resume_pci, + NULL); static const struct pci_device_id pwm_lpss_pci_ids[] = { { PCI_VDEVICE(INTEL, 0x0ac8), (unsigned long)&pwm_lpss_bxt_info}, @@ -114,10 +87,11 @@ static struct pci_driver pwm_lpss_driver_pci = { .probe = pwm_lpss_probe_pci, .remove = pwm_lpss_remove_pci, .driver = { - .pm = &pwm_lpss_pci_pm, + .pm = pm_ptr(&pwm_lpss_pci_pm), }, }; module_pci_driver(pwm_lpss_driver_pci); MODULE_DESCRIPTION("PWM PCI driver for Intel LPSS"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(PWM_LPSS); diff --git a/drivers/pwm/pwm-lpss-platform.c b/drivers/pwm/pwm-lpss-platform.c index 928570430cef7599b1e72c58e793dfef1b26f3dc..c48c6f2b2cd8f441ccc01e57e1ee32ff6abd0b46 100644 --- a/drivers/pwm/pwm-lpss-platform.c +++ b/drivers/pwm/pwm-lpss-platform.c @@ -7,52 +7,31 @@ * Derived from the original pwm-lpss.c */ -#include #include +#include #include #include #include +#include #include "pwm-lpss.h" -/* BayTrail */ -static const struct pwm_lpss_boardinfo pwm_lpss_byt_info = { - .clk_rate = 25000000, - .npwm = 1, - .base_unit_bits = 16, -}; - -/* Braswell */ -static const struct pwm_lpss_boardinfo pwm_lpss_bsw_info = { - .clk_rate = 19200000, - .npwm = 1, - .base_unit_bits = 16, - .other_devices_aml_touches_pwm_regs = true, -}; - -/* Broxton */ -static const struct pwm_lpss_boardinfo pwm_lpss_bxt_info = { - .clk_rate = 19200000, - .npwm = 4, - .base_unit_bits = 22, - .bypass = true, -}; static int pwm_lpss_probe_platform(struct platform_device *pdev) { const struct pwm_lpss_boardinfo *info; - const struct acpi_device_id *id; struct pwm_lpss_chip *lpwm; - struct resource *r; + void __iomem *base; - id = acpi_match_device(pdev->dev.driver->acpi_match_table, &pdev->dev); - if (!id) + info = device_get_match_data(&pdev->dev); + if (!info) return -ENODEV; - info = (const struct pwm_lpss_boardinfo *)id->driver_data; - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); - lpwm = pwm_lpss_probe(&pdev->dev, r, info); + lpwm = pwm_lpss_probe(&pdev->dev, base, info); if (IS_ERR(lpwm)) return PTR_ERR(lpwm); @@ -110,4 +89,5 @@ module_platform_driver(pwm_lpss_driver_platform); MODULE_DESCRIPTION("PWM platform driver for Intel LPSS"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(PWM_LPSS); MODULE_ALIAS("platform:pwm-lpss"); diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index 36d4e83e6b7901665cf123373bdf2e2be2b6afc6..accdef5dd58e76d3f46dc7143e62caa4a352f2b0 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -10,6 +10,7 @@ * Author: Alan Cox */ +#include #include #include #include @@ -18,17 +19,53 @@ #include #include +#define DEFAULT_SYMBOL_NAMESPACE PWM_LPSS + #include "pwm-lpss.h" #define PWM 0x00000000 #define PWM_ENABLE BIT(31) #define PWM_SW_UPDATE BIT(30) #define PWM_BASE_UNIT_SHIFT 8 -#define PWM_ON_TIME_DIV_MASK 0x000000ff +#define PWM_ON_TIME_DIV_MASK GENMASK(7, 0) /* Size of each PWM register space if multiple */ #define PWM_SIZE 0x400 +/* BayTrail */ +const struct pwm_lpss_boardinfo pwm_lpss_byt_info = { + .clk_rate = 25000000, + .npwm = 1, + .base_unit_bits = 16, +}; +EXPORT_SYMBOL_GPL(pwm_lpss_byt_info); + +/* Braswell */ +const struct pwm_lpss_boardinfo pwm_lpss_bsw_info = { + .clk_rate = 19200000, + .npwm = 1, + .base_unit_bits = 16, + .other_devices_aml_touches_pwm_regs = true, +}; +EXPORT_SYMBOL_GPL(pwm_lpss_bsw_info); + +/* Broxton */ +const struct pwm_lpss_boardinfo pwm_lpss_bxt_info = { + .clk_rate = 19200000, + .npwm = 4, + .base_unit_bits = 22, + .bypass = true, +}; +EXPORT_SYMBOL_GPL(pwm_lpss_bxt_info); + +/* Tangier */ +const struct pwm_lpss_boardinfo pwm_lpss_tng_info = { + .clk_rate = 19200000, + .npwm = 4, + .base_unit_bits = 22, +}; +EXPORT_SYMBOL_GPL(pwm_lpss_tng_info); + static inline struct pwm_lpss_chip *to_lpwm(struct pwm_chip *chip) { return container_of(chip, struct pwm_lpss_chip, chip); @@ -207,7 +244,7 @@ static const struct pwm_ops pwm_lpss_ops = { .owner = THIS_MODULE, }; -struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r, +struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, void __iomem *base, const struct pwm_lpss_boardinfo *info) { struct pwm_lpss_chip *lpwm; @@ -222,10 +259,7 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r, if (!lpwm) return ERR_PTR(-ENOMEM); - lpwm->regs = devm_ioremap_resource(dev, r); - if (IS_ERR(lpwm->regs)) - return ERR_CAST(lpwm->regs); - + lpwm->regs = base; lpwm->info = info; c = lpwm->info->clk_rate; diff --git a/drivers/pwm/pwm-lpss.h b/drivers/pwm/pwm-lpss.h index 8b3476f25e066cfd287d32c503b90ff8900eab66..8e82eb5a7e009159004eb58f62cffab1031234f8 100644 --- a/drivers/pwm/pwm-lpss.h +++ b/drivers/pwm/pwm-lpss.h @@ -25,6 +25,11 @@ struct pwm_lpss_boardinfo { unsigned long clk_rate; unsigned int npwm; unsigned long base_unit_bits; + /* + * Some versions of the IP may stuck in the state machine if enable + * bit is not set, and hence update bit will show busy status till + * the reset. For the rest it may be otherwise. + */ bool bypass; /* * On some devices the _PS0/_PS3 AML code of the GPU (GFX0) device @@ -33,7 +38,12 @@ struct pwm_lpss_boardinfo { bool other_devices_aml_touches_pwm_regs; }; -struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r, +extern const struct pwm_lpss_boardinfo pwm_lpss_byt_info; +extern const struct pwm_lpss_boardinfo pwm_lpss_bsw_info; +extern const struct pwm_lpss_boardinfo pwm_lpss_bxt_info; +extern const struct pwm_lpss_boardinfo pwm_lpss_tng_info; + +struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, void __iomem *base, const struct pwm_lpss_boardinfo *info); #endif /* __PWM_LPSS_H */ diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c index c91fa7f9e33da975a73407e9bd4eac28c4109cfe..f230c10d28bb287401aab6305e73c031337781b6 100644 --- a/drivers/pwm/pwm-pca9685.c +++ b/drivers/pwm/pwm-pca9685.c @@ -598,7 +598,7 @@ static int pca9685_pwm_probe(struct i2c_client *client, return 0; } -static int pca9685_pwm_remove(struct i2c_client *client) +static void pca9685_pwm_remove(struct i2c_client *client) { struct pca9685 *pca = i2c_get_clientdata(client); @@ -610,8 +610,6 @@ static int pca9685_pwm_remove(struct i2c_client *client) } pm_runtime_disable(&client->dev); - - return 0; } static int __maybe_unused pca9685_pwm_runtime_suspend(struct device *dev) diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index f3647b3171522ace7cfb1e82acbd650e68e5431d..a5af859217c19d8fa435071ca9d5a62cca242ef5 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -328,22 +328,16 @@ static int rockchip_pwm_probe(struct platform_device *pdev) else pc->pclk = pc->clk; - if (IS_ERR(pc->pclk)) { - ret = PTR_ERR(pc->pclk); - if (ret != -EPROBE_DEFER) - dev_err(&pdev->dev, "Can't get APB clk: %d\n", ret); - return ret; - } + if (IS_ERR(pc->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pc->pclk), "Can't get APB clk\n"); ret = clk_prepare_enable(pc->clk); - if (ret) { - dev_err(&pdev->dev, "Can't prepare enable PWM clk: %d\n", ret); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Can't prepare enable PWM clk\n"); ret = clk_prepare_enable(pc->pclk); if (ret) { - dev_err(&pdev->dev, "Can't prepare enable APB clk: %d\n", ret); + dev_err_probe(&pdev->dev, ret, "Can't prepare enable APB clk\n"); goto err_clk; } @@ -360,7 +354,7 @@ static int rockchip_pwm_probe(struct platform_device *pdev) ret = pwmchip_add(&pc->chip); if (ret < 0) { - dev_err(&pdev->dev, "pwmchip_add() failed: %d\n", ret); + dev_err_probe(&pdev->dev, ret, "pwmchip_add() failed\n"); goto err_pclk; } diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 9903c3a7ecedc87ee38bed4d4b37da570c2766be..e7db8e45001cf3347d8f0688805672df338c3746 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -42,7 +42,7 @@ static ssize_t period_show(struct device *child, pwm_get_state(pwm, &state); - return sprintf(buf, "%llu\n", state.period); + return sysfs_emit(buf, "%llu\n", state.period); } static ssize_t period_store(struct device *child, @@ -77,7 +77,7 @@ static ssize_t duty_cycle_show(struct device *child, pwm_get_state(pwm, &state); - return sprintf(buf, "%llu\n", state.duty_cycle); + return sysfs_emit(buf, "%llu\n", state.duty_cycle); } static ssize_t duty_cycle_store(struct device *child, @@ -112,7 +112,7 @@ static ssize_t enable_show(struct device *child, pwm_get_state(pwm, &state); - return sprintf(buf, "%d\n", state.enabled); + return sysfs_emit(buf, "%d\n", state.enabled); } static ssize_t enable_store(struct device *child, @@ -171,7 +171,7 @@ static ssize_t polarity_show(struct device *child, break; } - return sprintf(buf, "%s\n", polarity); + return sysfs_emit(buf, "%s\n", polarity); } static ssize_t polarity_store(struct device *child, @@ -212,7 +212,7 @@ static ssize_t capture_show(struct device *child, if (ret) return ret; - return sprintf(buf, "%u %u\n", result.period, result.duty_cycle); + return sysfs_emit(buf, "%u %u\n", result.period, result.duty_cycle); } static DEVICE_ATTR_RW(period); @@ -361,7 +361,7 @@ static ssize_t npwm_show(struct device *parent, struct device_attribute *attr, { const struct pwm_chip *chip = dev_get_drvdata(parent); - return sprintf(buf, "%u\n", chip->npwm); + return sysfs_emit(buf, "%u\n", chip->npwm); } static DEVICE_ATTR_RO(npwm); @@ -433,7 +433,7 @@ static int pwm_class_resume_npwm(struct device *parent, unsigned int npwm) return ret; } -static int __maybe_unused pwm_class_suspend(struct device *parent) +static int pwm_class_suspend(struct device *parent) { struct pwm_chip *chip = dev_get_drvdata(parent); unsigned int i; @@ -464,20 +464,20 @@ static int __maybe_unused pwm_class_suspend(struct device *parent) return ret; } -static int __maybe_unused pwm_class_resume(struct device *parent) +static int pwm_class_resume(struct device *parent) { struct pwm_chip *chip = dev_get_drvdata(parent); return pwm_class_resume_npwm(parent, chip->npwm); } -static SIMPLE_DEV_PM_OPS(pwm_class_pm_ops, pwm_class_suspend, pwm_class_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(pwm_class_pm_ops, pwm_class_suspend, pwm_class_resume); static struct class pwm_class = { .name = "pwm", .owner = THIS_MODULE, .dev_groups = pwm_chip_groups, - .pm = &pwm_class_pm_ops, + .pm = pm_sleep_ptr(&pwm_class_pm_ops), }; static int pwmchip_sysfs_match(struct device *parent, const void *data) diff --git a/drivers/ras/cec.c b/drivers/ras/cec.c index 42f2fc0bc8a98f993fdd61bfcb101a4b52ce0430..321af498ee119d62610b308901932848124b1230 100644 --- a/drivers/ras/cec.c +++ b/drivers/ras/cec.c @@ -556,6 +556,14 @@ static int __init cec_init(void) if (ce_arr.disabled) return -ENODEV; + /* + * Intel systems may avoid uncorrectable errors + * if pages with corrected errors are aggressively + * taken offline. + */ + if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) + action_threshold = 2; + ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL); if (!ce_arr.array) { pr_err("Error allocating CE array page!\n"); diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig index 23e3e4a35cc9440c3809e51169040eff0d9c39e8..070e4403c6c24a1609fc07d25659f2a8111fefd2 100644 --- a/drivers/regulator/Kconfig +++ b/drivers/regulator/Kconfig @@ -787,6 +787,24 @@ config REGULATOR_MT6323 This driver supports the control of different power rails of device through regulator interface. +config REGULATOR_MT6331 + tristate "MediaTek MT6331 PMIC" + depends on MFD_MT6397 + help + Say y here to select this option to enable the power regulator of + MediaTek MT6331 PMIC. + This driver supports the control of different power rails of device + through regulator interface + +config REGULATOR_MT6332 + tristate "MediaTek MT6332 PMIC" + depends on MFD_MT6397 + help + Say y here to select this option to enable the power regulator of + MediaTek MT6332 PMIC. + This driver supports the control of different power rails of device + through regulator interface + config REGULATOR_MT6358 tristate "MediaTek MT6358 PMIC" depends on MFD_MT6397 @@ -1264,6 +1282,7 @@ config REGULATOR_STW481X_VMMC config REGULATOR_SY7636A tristate "Silergy SY7636A voltage regulator" + depends on MFD_SY7636A help This driver supports Silergy SY3686A voltage regulator. @@ -1384,6 +1403,15 @@ config REGULATOR_TPS65218 voltage regulators. It supports software based voltage control for different voltage domains +config REGULATOR_TPS65219 + tristate "TI TPS65219 Power regulators" + depends on MFD_TPS65219 && OF + help + This driver supports TPS65219 voltage regulator chips. + TPS65219 series of PMICs have 3 single phase BUCKs & 4 LDOs + voltage regulators. It supports software based voltage control + for different voltage domains. + config REGULATOR_TPS6524X tristate "TI TPS6524X Power regulators" depends on SPI diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile index fa49bb6cc5442ffed274a2f586703e92e53dd611..5962307e1130dfc0d829d318f9ef3b7693ea0218 100644 --- a/drivers/regulator/Makefile +++ b/drivers/regulator/Makefile @@ -95,6 +95,8 @@ obj-$(CONFIG_REGULATOR_MPQ7920) += mpq7920.o obj-$(CONFIG_REGULATOR_MT6311) += mt6311-regulator.o obj-$(CONFIG_REGULATOR_MT6315) += mt6315-regulator.o obj-$(CONFIG_REGULATOR_MT6323) += mt6323-regulator.o +obj-$(CONFIG_REGULATOR_MT6331) += mt6331-regulator.o +obj-$(CONFIG_REGULATOR_MT6332) += mt6332-regulator.o obj-$(CONFIG_REGULATOR_MT6358) += mt6358-regulator.o obj-$(CONFIG_REGULATOR_MT6359) += mt6359-regulator.o obj-$(CONFIG_REGULATOR_MT6360) += mt6360-regulator.o @@ -162,6 +164,7 @@ obj-$(CONFIG_REGULATOR_TPS65086) += tps65086-regulator.o obj-$(CONFIG_REGULATOR_TPS65090) += tps65090-regulator.o obj-$(CONFIG_REGULATOR_TPS65217) += tps65217-regulator.o obj-$(CONFIG_REGULATOR_TPS65218) += tps65218-regulator.o +obj-$(CONFIG_REGULATOR_TPS65219) += tps65219-regulator.o obj-$(CONFIG_REGULATOR_TPS6524X) += tps6524x-regulator.o obj-$(CONFIG_REGULATOR_TPS6586X) += tps6586x-regulator.o obj-$(CONFIG_REGULATOR_TPS65910) += tps65910-regulator.o diff --git a/drivers/regulator/bd71815-regulator.c b/drivers/regulator/bd71815-regulator.c index acaa6607898ea2e70fcec89d97d3d3ef56209098..c2b8b8be78242ee0f080388c5d75e2fb1d79bce7 100644 --- a/drivers/regulator/bd71815-regulator.c +++ b/drivers/regulator/bd71815-regulator.c @@ -571,11 +571,10 @@ static int bd7181x_probe(struct platform_device *pdev) dev_err(&pdev->dev, "No parent regmap\n"); return -ENODEV; } - ldo4_en = devm_gpiod_get_from_of_node(&pdev->dev, - pdev->dev.parent->of_node, - "rohm,vsel-gpios", 0, - GPIOD_ASIS, "ldo4-en"); + ldo4_en = devm_fwnode_gpiod_get(&pdev->dev, + dev_fwnode(pdev->dev.parent), + "rohm,vsel", GPIOD_ASIS, "ldo4-en"); if (IS_ERR(ldo4_en)) { ret = PTR_ERR(ldo4_en); if (ret != -ENOENT) diff --git a/drivers/regulator/bd9576-regulator.c b/drivers/regulator/bd9576-regulator.c index aa42da4d141edd350a52795a1a4b413c36b9cb42..393c8693b327e2af44e641271e019654919f3572 100644 --- a/drivers/regulator/bd9576-regulator.c +++ b/drivers/regulator/bd9576-regulator.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -939,8 +940,8 @@ static int bd957x_probe(struct platform_device *pdev) } ic_data->regmap = regmap; - vout_mode = of_property_read_bool(pdev->dev.parent->of_node, - "rohm,vout1-en-low"); + vout_mode = device_property_read_bool(pdev->dev.parent, + "rohm,vout1-en-low"); if (vout_mode) { struct gpio_desc *en; @@ -948,10 +949,10 @@ static int bd957x_probe(struct platform_device *pdev) /* VOUT1 enable state judged by VOUT1_EN pin */ /* See if we have GPIO defined */ - en = devm_gpiod_get_from_of_node(&pdev->dev, - pdev->dev.parent->of_node, - "rohm,vout1-en-gpios", 0, - GPIOD_OUT_LOW, "vout1-en"); + en = devm_fwnode_gpiod_get(&pdev->dev, + dev_fwnode(pdev->dev.parent), + "rohm,vout1-en", GPIOD_OUT_LOW, + "vout1-en"); if (!IS_ERR(en)) { /* VOUT1_OPS gpio ctrl */ /* @@ -986,8 +987,8 @@ static int bd957x_probe(struct platform_device *pdev) * like DDR voltage selection. */ platform_set_drvdata(pdev, ic_data); - ddr_sel = of_property_read_bool(pdev->dev.parent->of_node, - "rohm,ddr-sel-low"); + ddr_sel = device_property_read_bool(pdev->dev.parent, + "rohm,ddr-sel-low"); if (ddr_sel) ic_data->regulator_data[2].desc.fixed_uV = 1350000; else diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index d8373cb04f9038af50c1798ea3d16b142a8159a1..bcccad8f751627c98e618f5ab86dde9d0b3df340 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -977,12 +977,27 @@ static int drms_uA_update(struct regulator_dev *rdev) rdev_err(rdev, "failed to set load %d: %pe\n", current_uA, ERR_PTR(err)); } else { + /* + * Unfortunately in some cases the constraints->valid_ops has + * REGULATOR_CHANGE_DRMS but there are no valid modes listed. + * That's not really legit but we won't consider it a fatal + * error here. We'll treat it as if REGULATOR_CHANGE_DRMS + * wasn't set. + */ + if (!rdev->constraints->valid_modes_mask) { + rdev_dbg(rdev, "Can change modes; but no valid mode\n"); + return 0; + } + /* get output voltage */ output_uV = regulator_get_voltage_rdev(rdev); - if (output_uV <= 0) { - rdev_err(rdev, "invalid output voltage found\n"); - return -EINVAL; - } + + /* + * Don't return an error; if regulator driver cares about + * output_uV then it's up to the driver to validate. + */ + if (output_uV <= 0) + rdev_dbg(rdev, "invalid output voltage found\n"); /* get input voltage */ input_uV = 0; @@ -990,10 +1005,13 @@ static int drms_uA_update(struct regulator_dev *rdev) input_uV = regulator_get_voltage(rdev->supply); if (input_uV <= 0) input_uV = rdev->constraints->input_uV; - if (input_uV <= 0) { - rdev_err(rdev, "invalid input voltage found\n"); - return -EINVAL; - } + + /* + * Don't return an error; if regulator driver cares about + * input_uV then it's up to the driver to validate. + */ + if (input_uV <= 0) + rdev_dbg(rdev, "invalid input voltage found\n"); /* now get the optimum mode for our new total regulator load */ mode = rdev->desc->ops->get_optimum_mode(rdev, input_uV, @@ -2681,7 +2699,7 @@ static int _regulator_do_enable(struct regulator_dev *rdev) * return -ETIMEDOUT. */ if (rdev->desc->poll_enabled_time) { - unsigned int time_remaining = delay; + int time_remaining = delay; while (time_remaining > 0) { _regulator_delay_helper(rdev->desc->poll_enabled_time); @@ -2733,13 +2751,18 @@ static int _regulator_do_enable(struct regulator_dev *rdev) */ static int _regulator_handle_consumer_enable(struct regulator *regulator) { + int ret; struct regulator_dev *rdev = regulator->rdev; lockdep_assert_held_once(&rdev->mutex.base); regulator->enable_count++; - if (regulator->uA_load && regulator->enable_count == 1) - return drms_uA_update(rdev); + if (regulator->uA_load && regulator->enable_count == 1) { + ret = drms_uA_update(rdev); + if (ret) + regulator->enable_count--; + return ret; + } return 0; } @@ -3497,10 +3520,8 @@ static int _regulator_set_voltage_time(struct regulator_dev *rdev, (new_uV < old_uV)) return rdev->constraints->settling_time_down; - if (ramp_delay == 0) { - rdev_dbg(rdev, "ramp_delay not set\n"); + if (ramp_delay == 0) return 0; - } return DIV_ROUND_UP(abs(new_uV - old_uV), ramp_delay); } @@ -5393,6 +5414,7 @@ regulator_register(const struct regulator_desc *regulator_desc, bool dangling_of_gpiod = false; struct device *dev; int ret, i; + bool resolved_early = false; if (cfg == NULL) return ERR_PTR(-EINVAL); @@ -5496,24 +5518,10 @@ regulator_register(const struct regulator_desc *regulator_desc, BLOCKING_INIT_NOTIFIER_HEAD(&rdev->notifier); INIT_DELAYED_WORK(&rdev->disable_work, regulator_disable_work); - /* preform any regulator specific init */ - if (init_data && init_data->regulator_init) { - ret = init_data->regulator_init(rdev->reg_data); - if (ret < 0) - goto clean; - } - - if (config->ena_gpiod) { - ret = regulator_ena_gpio_request(rdev, config); - if (ret != 0) { - rdev_err(rdev, "Failed to request enable GPIO: %pe\n", - ERR_PTR(ret)); - goto clean; - } - /* The regulator core took over the GPIO descriptor */ - dangling_cfg_gpiod = false; - dangling_of_gpiod = false; - } + if (init_data && init_data->supply_regulator) + rdev->supply_name = init_data->supply_regulator; + else if (regulator_desc->supply_name) + rdev->supply_name = regulator_desc->supply_name; /* register with sysfs */ rdev->dev.class = ®ulator_class; @@ -5535,13 +5543,38 @@ regulator_register(const struct regulator_desc *regulator_desc, goto wash; } - if (init_data && init_data->supply_regulator) - rdev->supply_name = init_data->supply_regulator; - else if (regulator_desc->supply_name) - rdev->supply_name = regulator_desc->supply_name; + if ((rdev->supply_name && !rdev->supply) && + (rdev->constraints->always_on || + rdev->constraints->boot_on)) { + ret = regulator_resolve_supply(rdev); + if (ret) + rdev_dbg(rdev, "unable to resolve supply early: %pe\n", + ERR_PTR(ret)); + + resolved_early = true; + } + + /* perform any regulator specific init */ + if (init_data && init_data->regulator_init) { + ret = init_data->regulator_init(rdev->reg_data); + if (ret < 0) + goto wash; + } + + if (config->ena_gpiod) { + ret = regulator_ena_gpio_request(rdev, config); + if (ret != 0) { + rdev_err(rdev, "Failed to request enable GPIO: %pe\n", + ERR_PTR(ret)); + goto wash; + } + /* The regulator core took over the GPIO descriptor */ + dangling_cfg_gpiod = false; + dangling_of_gpiod = false; + } ret = set_machine_constraints(rdev); - if (ret == -EPROBE_DEFER) { + if (ret == -EPROBE_DEFER && !resolved_early) { /* Regulator might be in bypass mode and so needs its supply * to set the constraints */ diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c index 76e0e23bf598c7a134507fd0fa4d8e9bdc1d6163..e4c753b830888f2805c6e9186677fd48aa089e57 100644 --- a/drivers/regulator/da9121-regulator.c +++ b/drivers/regulator/da9121-regulator.c @@ -1164,7 +1164,7 @@ error: return ret; } -static int da9121_i2c_remove(struct i2c_client *i2c) +static void da9121_i2c_remove(struct i2c_client *i2c) { struct da9121 *chip = i2c_get_clientdata(i2c); const int mask_all[4] = { 0xFF, 0xFF, 0xFF, 0xFF }; @@ -1176,7 +1176,6 @@ static int da9121_i2c_remove(struct i2c_client *i2c) ret = regmap_bulk_write(chip->regmap, DA9121_REG_SYS_MASK_0, mask_all, 4); if (ret != 0) dev_err(chip->dev, "Failed to set IRQ masks: %d\n", ret); - return 0; } static const struct i2c_device_id da9121_i2c_id[] = { diff --git a/drivers/regulator/devres.c b/drivers/regulator/devres.c index 32823a87fd409a310ddd0417ee4af41869271ce7..3265e75e97ab4cab1dfd4eb16b94c40456c2ee35 100644 --- a/drivers/regulator/devres.c +++ b/drivers/regulator/devres.c @@ -70,6 +70,65 @@ struct regulator *devm_regulator_get_exclusive(struct device *dev, } EXPORT_SYMBOL_GPL(devm_regulator_get_exclusive); +static void regulator_action_disable(void *d) +{ + struct regulator *r = (struct regulator *)d; + + regulator_disable(r); +} + +static int _devm_regulator_get_enable(struct device *dev, const char *id, + int get_type) +{ + struct regulator *r; + int ret; + + r = _devm_regulator_get(dev, id, get_type); + if (IS_ERR(r)) + return PTR_ERR(r); + + ret = regulator_enable(r); + if (!ret) + ret = devm_add_action_or_reset(dev, ®ulator_action_disable, r); + + if (ret) + devm_regulator_put(r); + + return ret; +} + +/** + * devm_regulator_get_enable_optional - Resource managed regulator get and enable + * @dev: device to supply + * @id: supply name or regulator ID. + * + * Get and enable regulator for duration of the device life-time. + * regulator_disable() and regulator_put() are automatically called on driver + * detach. See regulator_get_optional() and regulator_enable() for more + * information. + */ +int devm_regulator_get_enable_optional(struct device *dev, const char *id) +{ + return _devm_regulator_get_enable(dev, id, OPTIONAL_GET); +} +EXPORT_SYMBOL_GPL(devm_regulator_get_enable_optional); + +/** + * devm_regulator_get_enable - Resource managed regulator get and enable + * @dev: device to supply + * @id: supply name or regulator ID. + * + * Get and enable regulator for duration of the device life-time. + * regulator_disable() and regulator_put() are automatically called on driver + * detach. See regulator_get() and regulator_enable() for more + * information. + */ +int devm_regulator_get_enable(struct device *dev, const char *id) +{ + return _devm_regulator_get_enable(dev, id, NORMAL_GET); +} +EXPORT_SYMBOL_GPL(devm_regulator_get_enable); + /** * devm_regulator_get_optional - Resource managed regulator_get_optional() * @dev: device to supply @@ -194,6 +253,111 @@ int devm_regulator_bulk_get_const(struct device *dev, int num_consumers, } EXPORT_SYMBOL_GPL(devm_regulator_bulk_get_const); +static int devm_regulator_bulk_match(struct device *dev, void *res, + void *data) +{ + struct regulator_bulk_devres *match = res; + struct regulator_bulk_data *target = data; + + /* + * We check the put uses same consumer list as the get did. + * We _could_ scan all entries in consumer array and check the + * regulators match but ATM I don't see the need. We can change this + * later if needed. + */ + return match->consumers == target; +} + +/** + * devm_regulator_bulk_put - Resource managed regulator_bulk_put() + * @consumers: consumers to free + * + * Deallocate regulators allocated with devm_regulator_bulk_get(). Normally + * this function will not need to be called and the resource management + * code will ensure that the resource is freed. + */ +void devm_regulator_bulk_put(struct regulator_bulk_data *consumers) +{ + int rc; + struct regulator *regulator = consumers[0].consumer; + + rc = devres_release(regulator->dev, devm_regulator_bulk_release, + devm_regulator_bulk_match, consumers); + if (rc != 0) + WARN_ON(rc); +} +EXPORT_SYMBOL_GPL(devm_regulator_bulk_put); + +static void devm_regulator_bulk_disable(void *res) +{ + struct regulator_bulk_devres *devres = res; + int i; + + for (i = 0; i < devres->num_consumers; i++) + regulator_disable(devres->consumers[i].consumer); +} + +/** + * devm_regulator_bulk_get_enable - managed get'n enable multiple regulators + * + * @dev: device to supply + * @num_consumers: number of consumers to register + * @id: list of supply names or regulator IDs + * + * @return 0 on success, an errno on failure. + * + * This helper function allows drivers to get several regulator + * consumers in one operation with management, the regulators will + * automatically be freed when the device is unbound. If any of the + * regulators cannot be acquired then any regulators that were + * allocated will be freed before returning to the caller. + */ +int devm_regulator_bulk_get_enable(struct device *dev, int num_consumers, + const char * const *id) +{ + struct regulator_bulk_devres *devres; + struct regulator_bulk_data *consumers; + int i, ret; + + devres = devm_kmalloc(dev, sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + devres->consumers = devm_kcalloc(dev, num_consumers, sizeof(*consumers), + GFP_KERNEL); + consumers = devres->consumers; + if (!consumers) + return -ENOMEM; + + devres->num_consumers = num_consumers; + + for (i = 0; i < num_consumers; i++) + consumers[i].supply = id[i]; + + ret = devm_regulator_bulk_get(dev, num_consumers, consumers); + if (ret) + return ret; + + for (i = 0; i < num_consumers; i++) { + ret = regulator_enable(consumers[i].consumer); + if (ret) + goto unwind; + } + + ret = devm_add_action(dev, devm_regulator_bulk_disable, devres); + if (!ret) + return 0; + +unwind: + while (--i >= 0) + regulator_disable(consumers[i].consumer); + + devm_regulator_bulk_put(consumers); + + return ret; +} +EXPORT_SYMBOL_GPL(devm_regulator_bulk_get_enable); + static void devm_rdev_release(struct device *dev, void *res) { regulator_unregister(*(struct regulator_dev **)res); diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c index 5927d4f3eabd751489c71275452d597f14a6f4fc..95e61a2f43f5d11d00f30f764f0068195836b477 100644 --- a/drivers/regulator/gpio-regulator.c +++ b/drivers/regulator/gpio-regulator.c @@ -220,6 +220,9 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np, regtype); } + if (of_find_property(np, "vin-supply", NULL)) + config->input_supply = "vin"; + return config; } @@ -259,6 +262,18 @@ static int gpio_regulator_probe(struct platform_device *pdev) drvdata->gpiods = devm_kzalloc(dev, sizeof(struct gpio_desc *), GFP_KERNEL); + + if (config->input_supply) { + drvdata->desc.supply_name = devm_kstrdup(&pdev->dev, + config->input_supply, + GFP_KERNEL); + if (!drvdata->desc.supply_name) { + dev_err(&pdev->dev, + "Failed to allocate input supply\n"); + return -ENOMEM; + } + } + if (!drvdata->gpiods) return -ENOMEM; for (i = 0; i < config->ngpios; i++) { diff --git a/drivers/regulator/lp8755.c b/drivers/regulator/lp8755.c index 321bec6e3f8dfed983ba38ee959149c3a36ab8b6..31b43426d47c58088e541e343f883b96cc166679 100644 --- a/drivers/regulator/lp8755.c +++ b/drivers/regulator/lp8755.c @@ -422,15 +422,13 @@ err: return ret; } -static int lp8755_remove(struct i2c_client *client) +static void lp8755_remove(struct i2c_client *client) { int icnt; struct lp8755_chip *pchip = i2c_get_clientdata(client); for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++) regmap_write(pchip->regmap, icnt, 0x00); - - return 0; } static const struct i2c_device_id lp8755_id[] = { diff --git a/drivers/regulator/max597x-regulator.c b/drivers/regulator/max597x-regulator.c index 03c6027682d81b3c8451f3e6ef80e7c834afa727..39f803ff0a90a8f086aa247c2cb30db39d508a1f 100644 --- a/drivers/regulator/max597x-regulator.c +++ b/drivers/regulator/max597x-regulator.c @@ -137,7 +137,7 @@ static int max597x_set_ovp(struct regulator_dev *rdev, int lim_uV, int severity, static int max597x_set_ocp(struct regulator_dev *rdev, int lim_uA, int severity, bool enable) { - int ret, val, reg; + int val, reg; unsigned int vthst, vthfst; struct max597x_regulator *data = rdev_get_drvdata(rdev); @@ -183,9 +183,8 @@ static int max597x_set_ocp(struct regulator_dev *rdev, int lim_uA, val = 0xFF; reg = MAX5970_REG_DAC_FAST(rdev_id); - ret = regmap_write(rdev->regmap, reg, val); - return ret; + return regmap_write(rdev->regmap, reg, val); } static int max597x_get_status(struct regulator_dev *rdev) diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c index fdcb0f508984ccce66c74e1debc716aaf93662a4..596cc36aaff6ba7b37c7c2bb17a37a2d4bc35dc7 100644 --- a/drivers/regulator/max8973-regulator.c +++ b/drivers/regulator/max8973-regulator.c @@ -434,9 +434,9 @@ static int max8973_init_dcdc(struct max8973_chip *max, return ret; } -static int max8973_thermal_read_temp(void *data, int *temp) +static int max8973_thermal_read_temp(struct thermal_zone_device *tz, int *temp) { - struct max8973_chip *mchip = data; + struct max8973_chip *mchip = tz->devdata; unsigned int val; int ret; @@ -465,7 +465,7 @@ static irqreturn_t max8973_thermal_irq(int irq, void *data) return IRQ_HANDLED; } -static const struct thermal_zone_of_device_ops max77621_tz_ops = { +static const struct thermal_zone_device_ops max77621_tz_ops = { .get_temp = max8973_thermal_read_temp, }; @@ -479,8 +479,8 @@ static int max8973_thermal_init(struct max8973_chip *mchip) if (mchip->id != MAX77621) return 0; - tzd = devm_thermal_zone_of_sensor_register(mchip->dev, 0, mchip, - &max77621_tz_ops); + tzd = devm_thermal_of_zone_register(mchip->dev, 0, mchip, + &max77621_tz_ops); if (IS_ERR(tzd)) { ret = PTR_ERR(tzd); dev_err(mchip->dev, "Failed to register thermal sensor: %d\n", diff --git a/drivers/regulator/mt6331-regulator.c b/drivers/regulator/mt6331-regulator.c new file mode 100644 index 0000000000000000000000000000000000000000..56be9a3a84ab36c9d45d55aa1d6edabb642e159d --- /dev/null +++ b/drivers/regulator/mt6331-regulator.c @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2022 Collabora Ltd. +// Author: AngeloGioacchino Del Regno +// +// Based on mt6323-regulator.c, +// Copyright (c) 2016 MediaTek Inc. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MT6331_LDO_MODE_NORMAL 0 +#define MT6331_LDO_MODE_LP 1 + +/* + * MT6331 regulators information + * + * @desc: standard fields of regulator description. + * @qi: Mask for query enable signal status of regulators + * @vselon_reg: Register sections for hardware control mode of bucks + * @vselctrl_reg: Register for controlling the buck control mode. + * @vselctrl_mask: Mask for query buck's voltage control mode. + * @status_reg: Register for regulator enable status where qi unavailable + * @status_mask: Mask for querying regulator enable status + */ +struct mt6331_regulator_info { + struct regulator_desc desc; + u32 qi; + u32 vselon_reg; + u32 vselctrl_reg; + u32 vselctrl_mask; + u32 modeset_reg; + u32 modeset_mask; + u32 status_reg; + u32 status_mask; +}; + +#define MT6331_BUCK(match, vreg, min, max, step, volt_ranges, enreg, \ + vosel, vosel_mask, voselon, vosel_ctrl) \ +[MT6331_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6331_volt_range_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6331_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = (max - min)/step + 1, \ + .linear_ranges = volt_ranges, \ + .n_linear_ranges = ARRAY_SIZE(volt_ranges), \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(0), \ + }, \ + .qi = BIT(13), \ + .vselon_reg = voselon, \ + .vselctrl_reg = vosel_ctrl, \ + .vselctrl_mask = BIT(1), \ + .status_mask = 0, \ +} + +#define MT6331_LDO_AO(match, vreg, ldo_volt_table, vosel, vosel_mask) \ +[MT6331_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6331_volt_table_ao_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6331_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = ARRAY_SIZE(ldo_volt_table), \ + .volt_table = ldo_volt_table, \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + }, \ +} + +#define MT6331_LDO_S(match, vreg, ldo_volt_table, enreg, enbit, vosel, \ + vosel_mask, _modeset_reg, _modeset_mask, \ + _status_reg, _status_mask) \ +[MT6331_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6331_volt_table_no_qi_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6331_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = ARRAY_SIZE(ldo_volt_table), \ + .volt_table = ldo_volt_table, \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(enbit), \ + }, \ + .modeset_reg = _modeset_reg, \ + .modeset_mask = _modeset_mask, \ + .status_reg = _status_reg, \ + .status_mask = _status_mask, \ +} + +#define MT6331_LDO(match, vreg, ldo_volt_table, enreg, enbit, vosel, \ + vosel_mask, _modeset_reg, _modeset_mask) \ +[MT6331_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = (_modeset_reg ? \ + &mt6331_volt_table_ops : \ + &mt6331_volt_table_no_ms_ops), \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6331_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = ARRAY_SIZE(ldo_volt_table), \ + .volt_table = ldo_volt_table, \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(enbit), \ + }, \ + .qi = BIT(15), \ + .modeset_reg = _modeset_reg, \ + .modeset_mask = _modeset_mask, \ +} + +#define MT6331_REG_FIXED(match, vreg, enreg, enbit, qibit, volt, \ + _modeset_reg, _modeset_mask) \ +[MT6331_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = (_modeset_reg ? \ + &mt6331_volt_fixed_ops : \ + &mt6331_volt_fixed_no_ms_ops), \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6331_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = 1, \ + .enable_reg = enreg, \ + .enable_mask = BIT(enbit), \ + .min_uV = volt, \ + }, \ + .qi = BIT(qibit), \ + .modeset_reg = _modeset_reg, \ + .modeset_mask = _modeset_mask, \ +} + +static const struct linear_range buck_volt_range[] = { + REGULATOR_LINEAR_RANGE(700000, 0, 0x7f, 6250), +}; + +static const unsigned int ldo_volt_table1[] = { + 2800000, 3000000, 0, 3200000 +}; + +static const unsigned int ldo_volt_table2[] = { + 1500000, 1800000, 2500000, 2800000, +}; + +static const unsigned int ldo_volt_table3[] = { + 1200000, 1300000, 1500000, 1800000, 2000000, 2800000, 3000000, 3300000, +}; + +static const unsigned int ldo_volt_table4[] = { + 0, 0, 1700000, 1800000, 1860000, 2760000, 3000000, 3100000, +}; + +static const unsigned int ldo_volt_table5[] = { + 1800000, 3300000, 1800000, 3300000, +}; + +static const unsigned int ldo_volt_table6[] = { + 3000000, 3300000, +}; + +static const unsigned int ldo_volt_table7[] = { + 1200000, 1600000, 1700000, 1800000, 1900000, 2000000, 2100000, 2200000, +}; + +static const unsigned int ldo_volt_table8[] = { + 900000, 1000000, 1100000, 1220000, 1300000, 1500000, 1500000, 1500000, +}; + +static const unsigned int ldo_volt_table9[] = { + 1000000, 1050000, 1100000, 1150000, 1200000, 1250000, 1300000, 1300000, +}; + +static const unsigned int ldo_volt_table10[] = { + 1200000, 1300000, 1500000, 1800000, +}; + +static const unsigned int ldo_volt_table11[] = { + 1200000, 1300000, 1400000, 1500000, 1600000, 1700000, 1800000, 1800000, +}; + +static int mt6331_get_status(struct regulator_dev *rdev) +{ + struct mt6331_regulator_info *info = rdev_get_drvdata(rdev); + u32 regval; + int ret; + + ret = regmap_read(rdev->regmap, info->desc.enable_reg, ®val); + if (ret != 0) { + dev_err(&rdev->dev, "Failed to get enable reg: %d\n", ret); + return ret; + } + + return (regval & info->qi) ? REGULATOR_STATUS_ON : REGULATOR_STATUS_OFF; +} + +static int mt6331_ldo_set_mode(struct regulator_dev *rdev, unsigned int mode) +{ + struct mt6331_regulator_info *info = rdev_get_drvdata(rdev); + int val; + + switch (mode) { + case REGULATOR_MODE_STANDBY: + val = MT6331_LDO_MODE_LP; + break; + case REGULATOR_MODE_NORMAL: + val = MT6331_LDO_MODE_NORMAL; + break; + default: + return -EINVAL; + } + + val <<= ffs(info->modeset_mask) - 1; + + return regmap_update_bits(rdev->regmap, info->modeset_reg, + info->modeset_mask, val); +} + +static unsigned int mt6331_ldo_get_mode(struct regulator_dev *rdev) +{ + struct mt6331_regulator_info *info = rdev_get_drvdata(rdev); + unsigned int val; + int ret; + + ret = regmap_read(rdev->regmap, info->modeset_reg, &val); + if (ret < 0) + return ret; + + val &= info->modeset_mask; + val >>= ffs(info->modeset_mask) - 1; + + return (val & BIT(0)) ? REGULATOR_MODE_STANDBY : REGULATOR_MODE_NORMAL; +} + +static const struct regulator_ops mt6331_volt_range_ops = { + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6331_get_status, +}; + +static const struct regulator_ops mt6331_volt_table_no_ms_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6331_get_status, +}; + +static const struct regulator_ops mt6331_volt_table_no_qi_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .set_mode = mt6331_ldo_set_mode, + .get_mode = mt6331_ldo_get_mode, +}; + +static const struct regulator_ops mt6331_volt_table_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6331_get_status, + .set_mode = mt6331_ldo_set_mode, + .get_mode = mt6331_ldo_get_mode, +}; + +static const struct regulator_ops mt6331_volt_table_ao_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, +}; + +static const struct regulator_ops mt6331_volt_fixed_no_ms_ops = { + .list_voltage = regulator_list_voltage_linear, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6331_get_status, +}; + +static const struct regulator_ops mt6331_volt_fixed_ops = { + .list_voltage = regulator_list_voltage_linear, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6331_get_status, + .set_mode = mt6331_ldo_set_mode, + .get_mode = mt6331_ldo_get_mode, +}; + +/* The array is indexed by id(MT6331_ID_XXX) */ +static struct mt6331_regulator_info mt6331_regulators[] = { + MT6331_BUCK("buck-vdvfs11", VDVFS11, 700000, 1493750, 6250, + buck_volt_range, MT6331_VDVFS11_CON9, + MT6331_VDVFS11_CON11, GENMASK(6, 0), + MT6331_VDVFS11_CON12, MT6331_VDVFS11_CON7), + MT6331_BUCK("buck-vdvfs12", VDVFS12, 700000, 1493750, 6250, + buck_volt_range, MT6331_VDVFS12_CON9, + MT6331_VDVFS12_CON11, GENMASK(6, 0), + MT6331_VDVFS12_CON12, MT6331_VDVFS12_CON7), + MT6331_BUCK("buck-vdvfs13", VDVFS13, 700000, 1493750, 6250, + buck_volt_range, MT6331_VDVFS13_CON9, + MT6331_VDVFS13_CON11, GENMASK(6, 0), + MT6331_VDVFS13_CON12, MT6331_VDVFS13_CON7), + MT6331_BUCK("buck-vdvfs14", VDVFS14, 700000, 1493750, 6250, + buck_volt_range, MT6331_VDVFS14_CON9, + MT6331_VDVFS14_CON11, GENMASK(6, 0), + MT6331_VDVFS14_CON12, MT6331_VDVFS14_CON7), + MT6331_BUCK("buck-vcore2", VCORE2, 700000, 1493750, 6250, + buck_volt_range, MT6331_VCORE2_CON9, + MT6331_VCORE2_CON11, GENMASK(6, 0), + MT6331_VCORE2_CON12, MT6331_VCORE2_CON7), + MT6331_REG_FIXED("buck-vio18", VIO18, MT6331_VIO18_CON9, 0, 13, 1800000, 0, 0), + MT6331_REG_FIXED("ldo-vrtc", VRTC, MT6331_DIGLDO_CON11, 8, 15, 2800000, 0, 0), + MT6331_REG_FIXED("ldo-vtcxo1", VTCXO1, MT6331_ANALDO_CON1, 10, 15, 2800000, + MT6331_ANALDO_CON1, GENMASK(1, 0)), + MT6331_REG_FIXED("ldo-vtcxo2", VTCXO2, MT6331_ANALDO_CON2, 10, 15, 2800000, + MT6331_ANALDO_CON2, GENMASK(1, 0)), + MT6331_REG_FIXED("ldo-vsram", VSRAM_DVFS1, MT6331_SYSLDO_CON4, 10, 15, 1012500, + MT6331_SYSLDO_CON4, GENMASK(1, 0)), + MT6331_REG_FIXED("ldo-vio28", VIO28, MT6331_DIGLDO_CON1, 10, 15, 2800000, + MT6331_DIGLDO_CON1, GENMASK(1, 0)), + MT6331_LDO("ldo-avdd32aud", AVDD32_AUD, ldo_volt_table1, MT6331_ANALDO_CON3, 10, + MT6331_ANALDO_CON10, GENMASK(6, 5), MT6331_ANALDO_CON3, GENMASK(1, 0)), + MT6331_LDO("ldo-vauxa32", VAUXA32, ldo_volt_table1, MT6331_ANALDO_CON4, 10, + MT6331_ANALDO_CON6, GENMASK(6, 5), MT6331_ANALDO_CON4, GENMASK(1, 0)), + MT6331_LDO("ldo-vemc33", VEMC33, ldo_volt_table6, MT6331_DIGLDO_CON5, 10, + MT6331_DIGLDO_CON17, BIT(6), MT6331_DIGLDO_CON5, GENMASK(1, 0)), + MT6331_LDO("ldo-vibr", VIBR, ldo_volt_table3, MT6331_DIGLDO_CON12, 10, + MT6331_DIGLDO_CON20, GENMASK(6, 4), MT6331_DIGLDO_CON12, GENMASK(1, 0)), + MT6331_LDO("ldo-vmc", VMC, ldo_volt_table5, MT6331_DIGLDO_CON3, 10, + MT6331_DIGLDO_CON15, GENMASK(5, 4), MT6331_DIGLDO_CON3, GENMASK(1, 0)), + MT6331_LDO("ldo-vmch", VMCH, ldo_volt_table6, MT6331_DIGLDO_CON4, 10, + MT6331_DIGLDO_CON16, BIT(6), MT6331_DIGLDO_CON4, GENMASK(1, 0)), + MT6331_LDO("ldo-vmipi", VMIPI, ldo_volt_table3, MT6331_SYSLDO_CON5, 10, + MT6331_SYSLDO_CON13, GENMASK(5, 3), MT6331_SYSLDO_CON5, GENMASK(1, 0)), + MT6331_LDO("ldo-vsim1", VSIM1, ldo_volt_table4, MT6331_DIGLDO_CON8, 10, + MT6331_DIGLDO_CON21, GENMASK(6, 4), MT6331_DIGLDO_CON8, GENMASK(1, 0)), + MT6331_LDO("ldo-vsim2", VSIM2, ldo_volt_table4, MT6331_DIGLDO_CON9, 10, + MT6331_DIGLDO_CON22, GENMASK(6, 4), MT6331_DIGLDO_CON9, GENMASK(1, 0)), + MT6331_LDO("ldo-vusb10", VUSB10, ldo_volt_table9, MT6331_SYSLDO_CON2, 10, + MT6331_SYSLDO_CON10, GENMASK(5, 3), MT6331_SYSLDO_CON2, GENMASK(1, 0)), + MT6331_LDO("ldo-vcama", VCAMA, ldo_volt_table2, MT6331_ANALDO_CON5, 15, + MT6331_ANALDO_CON9, GENMASK(5, 4), 0, 0), + MT6331_LDO_S("ldo-vcamaf", VCAM_AF, ldo_volt_table3, MT6331_DIGLDO_CON2, 10, + MT6331_DIGLDO_CON14, GENMASK(6, 4), MT6331_DIGLDO_CON2, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(0)), + MT6331_LDO_S("ldo-vcamd", VCAMD, ldo_volt_table8, MT6331_SYSLDO_CON1, 15, + MT6331_SYSLDO_CON9, GENMASK(6, 4), MT6331_SYSLDO_CON1, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(11)), + MT6331_LDO_S("ldo-vcamio", VCAM_IO, ldo_volt_table10, MT6331_SYSLDO_CON3, 10, + MT6331_SYSLDO_CON11, GENMASK(4, 3), MT6331_SYSLDO_CON3, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(13)), + MT6331_LDO_S("ldo-vgp1", VGP1, ldo_volt_table3, MT6331_DIGLDO_CON6, 10, + MT6331_DIGLDO_CON19, GENMASK(6, 4), MT6331_DIGLDO_CON6, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(4)), + MT6331_LDO_S("ldo-vgp2", VGP2, ldo_volt_table10, MT6331_SYSLDO_CON6, 10, + MT6331_SYSLDO_CON14, GENMASK(4, 3), MT6331_SYSLDO_CON6, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(15)), + MT6331_LDO_S("ldo-vgp3", VGP3, ldo_volt_table10, MT6331_SYSLDO_CON7, 10, + MT6331_SYSLDO_CON15, GENMASK(4, 3), MT6331_SYSLDO_CON7, GENMASK(1, 0), + MT6331_EN_STATUS2, BIT(0)), + MT6331_LDO_S("ldo-vgp4", VGP4, ldo_volt_table7, MT6331_DIGLDO_CON7, 10, + MT6331_DIGLDO_CON18, GENMASK(6, 4), MT6331_DIGLDO_CON7, GENMASK(1, 0), + MT6331_EN_STATUS1, BIT(5)), + MT6331_LDO_AO("ldo-vdig18", VDIG18, ldo_volt_table11, + MT6331_DIGLDO_CON28, GENMASK(14, 12)), +}; + +static int mt6331_set_buck_vosel_reg(struct platform_device *pdev) +{ + struct mt6397_chip *mt6331 = dev_get_drvdata(pdev->dev.parent); + int i; + u32 regval; + + for (i = 0; i < MT6331_ID_VREG_MAX; i++) { + if (mt6331_regulators[i].vselctrl_reg) { + if (regmap_read(mt6331->regmap, + mt6331_regulators[i].vselctrl_reg, + ®val) < 0) { + dev_err(&pdev->dev, + "Failed to read buck ctrl\n"); + return -EIO; + } + + if (regval & mt6331_regulators[i].vselctrl_mask) { + mt6331_regulators[i].desc.vsel_reg = + mt6331_regulators[i].vselon_reg; + } + } + } + + return 0; +} + +static int mt6331_regulator_probe(struct platform_device *pdev) +{ + struct mt6397_chip *mt6331 = dev_get_drvdata(pdev->dev.parent); + struct regulator_config config = {}; + struct regulator_dev *rdev; + int i; + u32 reg_value; + + /* Query buck controller to select activated voltage register part */ + if (mt6331_set_buck_vosel_reg(pdev)) + return -EIO; + + /* Read PMIC chip revision to update constraints and voltage table */ + if (regmap_read(mt6331->regmap, MT6331_HWCID, ®_value) < 0) { + dev_err(&pdev->dev, "Failed to read Chip ID\n"); + return -EIO; + } + reg_value &= GENMASK(7, 0); + + dev_info(&pdev->dev, "Chip ID = 0x%x\n", reg_value); + + /* + * ChipID 0x10 is "MT6331 E1", has a different voltage table and + * it's currently not supported in this driver. Upon detection of + * this ID, refuse to register the regulators, as we will wrongly + * interpret the VSEL for this revision, potentially overvolting + * some device. + */ + if (reg_value == 0x10) { + dev_err(&pdev->dev, "Chip version not supported. Bailing out.\n"); + return -EINVAL; + } + + for (i = 0; i < MT6331_ID_VREG_MAX; i++) { + config.dev = &pdev->dev; + config.driver_data = &mt6331_regulators[i]; + config.regmap = mt6331->regmap; + rdev = devm_regulator_register(&pdev->dev, + &mt6331_regulators[i].desc, &config); + if (IS_ERR(rdev)) { + dev_err(&pdev->dev, "failed to register %s\n", + mt6331_regulators[i].desc.name); + return PTR_ERR(rdev); + } + } + return 0; +} + +static const struct platform_device_id mt6331_platform_ids[] = { + {"mt6331-regulator", 0}, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(platform, mt6331_platform_ids); + +static struct platform_driver mt6331_regulator_driver = { + .driver = { + .name = "mt6331-regulator", + }, + .probe = mt6331_regulator_probe, + .id_table = mt6331_platform_ids, +}; + +module_platform_driver(mt6331_regulator_driver); + +MODULE_AUTHOR("AngeloGioacchino Del Regno "); +MODULE_DESCRIPTION("Regulator Driver for MediaTek MT6331 PMIC"); +MODULE_LICENSE("GPL"); diff --git a/drivers/regulator/mt6332-regulator.c b/drivers/regulator/mt6332-regulator.c new file mode 100644 index 0000000000000000000000000000000000000000..77a27d8127a354f3222b57c77c8489e55e7e3010 --- /dev/null +++ b/drivers/regulator/mt6332-regulator.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (c) 2022 Collabora Ltd. +// Author: AngeloGioacchino Del Regno +// +// Based on mt6323-regulator.c, +// Copyright (c) 2016 MediaTek Inc. +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MT6332_LDO_MODE_NORMAL 0 +#define MT6332_LDO_MODE_LP 1 + +/* + * MT6332 regulators information + * + * @desc: standard fields of regulator description. + * @qi: Mask for query enable signal status of regulators + * @vselon_reg: Register sections for hardware control mode of bucks + * @vselctrl_reg: Register for controlling the buck control mode. + * @vselctrl_mask: Mask for query buck's voltage control mode. + * @status_reg: Register for regulator enable status where qi unavailable + * @status_mask: Mask for querying regulator enable status + */ +struct mt6332_regulator_info { + struct regulator_desc desc; + u32 qi; + u32 vselon_reg; + u32 vselctrl_reg; + u32 vselctrl_mask; + u32 modeset_reg; + u32 modeset_mask; + u32 status_reg; + u32 status_mask; +}; + +#define MT6332_BUCK(match, vreg, min, max, step, volt_ranges, enreg, \ + vosel, vosel_mask, voselon, vosel_ctrl) \ +[MT6332_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6332_buck_volt_range_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6332_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = (max - min)/step + 1, \ + .linear_ranges = volt_ranges, \ + .n_linear_ranges = ARRAY_SIZE(volt_ranges), \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(0), \ + }, \ + .qi = BIT(13), \ + .vselon_reg = voselon, \ + .vselctrl_reg = vosel_ctrl, \ + .vselctrl_mask = BIT(1), \ + .status_mask = 0, \ +} + +#define MT6332_LDO_LINEAR(match, vreg, min, max, step, volt_ranges, \ + enreg, vosel, vosel_mask, voselon, \ + vosel_ctrl, _modeset_reg, _modeset_mask) \ +[MT6332_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6332_ldo_volt_range_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6332_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = (max - min)/step + 1, \ + .linear_ranges = volt_ranges, \ + .n_linear_ranges = ARRAY_SIZE(volt_ranges), \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(0), \ + }, \ + .qi = BIT(15), \ + .vselon_reg = voselon, \ + .vselctrl_reg = vosel_ctrl, \ + .vselctrl_mask = BIT(1), \ + .modeset_reg = _modeset_reg, \ + .modeset_mask = _modeset_mask, \ + .status_mask = 0, \ +} + +#define MT6332_LDO_AO(match, vreg, ldo_volt_table, vosel, vosel_mask) \ +[MT6332_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6332_volt_table_ao_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6332_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = ARRAY_SIZE(ldo_volt_table), \ + .volt_table = ldo_volt_table, \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + }, \ +} + +#define MT6332_LDO(match, vreg, ldo_volt_table, enreg, enbit, vosel, \ + vosel_mask, _modeset_reg, _modeset_mask) \ +[MT6332_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6332_volt_table_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6332_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = ARRAY_SIZE(ldo_volt_table), \ + .volt_table = ldo_volt_table, \ + .vsel_reg = vosel, \ + .vsel_mask = vosel_mask, \ + .enable_reg = enreg, \ + .enable_mask = BIT(enbit), \ + }, \ + .qi = BIT(15), \ + .modeset_reg = _modeset_reg, \ + .modeset_mask = _modeset_mask, \ + .status_mask = 0, \ +} + +#define MT6332_REG_FIXED(match, vreg, enreg, enbit, qibit, volt, stbit) \ +[MT6332_ID_##vreg] = { \ + .desc = { \ + .name = #vreg, \ + .of_match = of_match_ptr(match), \ + .ops = &mt6332_volt_fixed_ops, \ + .type = REGULATOR_VOLTAGE, \ + .id = MT6332_ID_##vreg, \ + .owner = THIS_MODULE, \ + .n_voltages = 1, \ + .enable_reg = enreg, \ + .enable_mask = BIT(enbit), \ + .min_uV = volt, \ + }, \ + .qi = BIT(qibit), \ + .status_reg = MT6332_EN_STATUS0, \ + .status_mask = BIT(stbit), \ +} + +static const struct linear_range boost_volt_range[] = { + REGULATOR_LINEAR_RANGE(3500000, 0, 0x7f, 31250), +}; + +static const struct linear_range buck_volt_range[] = { + REGULATOR_LINEAR_RANGE(700000, 0, 0x7f, 6250), +}; + +static const struct linear_range buck_pa_volt_range[] = { + REGULATOR_LINEAR_RANGE(500000, 0, 0x3f, 50000), +}; + +static const struct linear_range buck_rf_volt_range[] = { + REGULATOR_LINEAR_RANGE(1050000, 0, 0x7f, 9375), +}; + +static const unsigned int ldo_volt_table1[] = { + 2800000, 3000000, 0, 3200000 +}; + +static const unsigned int ldo_volt_table2[] = { + 1200000, 1300000, 1400000, 1500000, 1600000, 1700000, 1800000, 1800000, +}; + +static int mt6332_get_status(struct regulator_dev *rdev) +{ + struct mt6332_regulator_info *info = rdev_get_drvdata(rdev); + u32 reg, en_mask, regval; + int ret; + + if (info->qi > 0) { + reg = info->desc.enable_reg; + en_mask = info->qi; + } else { + reg = info->status_reg; + en_mask = info->status_mask; + } + + ret = regmap_read(rdev->regmap, reg, ®val); + if (ret != 0) { + dev_err(&rdev->dev, "Failed to get enable reg: %d\n", ret); + return ret; + } + + return (regval & en_mask) ? REGULATOR_STATUS_ON : REGULATOR_STATUS_OFF; +} + +static int mt6332_ldo_set_mode(struct regulator_dev *rdev, unsigned int mode) +{ + struct mt6332_regulator_info *info = rdev_get_drvdata(rdev); + int val; + + switch (mode) { + case REGULATOR_MODE_STANDBY: + val = MT6332_LDO_MODE_LP; + break; + case REGULATOR_MODE_NORMAL: + val = MT6332_LDO_MODE_NORMAL; + break; + default: + return -EINVAL; + } + + val <<= ffs(info->modeset_mask) - 1; + + return regmap_update_bits(rdev->regmap, info->modeset_reg, + info->modeset_mask, val); +} + +static unsigned int mt6332_ldo_get_mode(struct regulator_dev *rdev) +{ + struct mt6332_regulator_info *info = rdev_get_drvdata(rdev); + unsigned int val; + int ret; + + ret = regmap_read(rdev->regmap, info->modeset_reg, &val); + if (ret < 0) + return ret; + + val &= info->modeset_mask; + val >>= ffs(info->modeset_mask) - 1; + + return (val & BIT(0)) ? REGULATOR_MODE_STANDBY : REGULATOR_MODE_NORMAL; +} + +static const struct regulator_ops mt6332_buck_volt_range_ops = { + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6332_get_status, +}; + +static const struct regulator_ops mt6332_ldo_volt_range_ops = { + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6332_get_status, + .set_mode = mt6332_ldo_set_mode, + .get_mode = mt6332_ldo_get_mode, +}; + +static const struct regulator_ops mt6332_volt_table_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6332_get_status, + .set_mode = mt6332_ldo_set_mode, + .get_mode = mt6332_ldo_get_mode, +}; + +static const struct regulator_ops mt6332_volt_table_ao_ops = { + .list_voltage = regulator_list_voltage_table, + .map_voltage = regulator_map_voltage_iterate, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_time_sel = regulator_set_voltage_time_sel, +}; + +static const struct regulator_ops mt6332_volt_fixed_ops = { + .list_voltage = regulator_list_voltage_linear, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .get_status = mt6332_get_status, +}; + +/* The array is indexed by id(MT6332_ID_XXX) */ +static struct mt6332_regulator_info mt6332_regulators[] = { + MT6332_BUCK("buck-vdram", VDRAM, 700000, 1493750, 6250, buck_volt_range, + MT6332_EN_STATUS0, MT6332_VDRAM_CON11, GENMASK(6, 0), + MT6332_VDRAM_CON12, MT6332_VDRAM_CON7), + MT6332_BUCK("buck-vdvfs2", VDVFS2, 700000, 1312500, 6250, buck_volt_range, + MT6332_VDVFS2_CON9, MT6332_VDVFS2_CON11, GENMASK(6, 0), + MT6332_VDVFS2_CON12, MT6332_VDVFS2_CON7), + MT6332_BUCK("buck-vpa", VPA, 500000, 3400000, 50000, buck_pa_volt_range, + MT6332_VPA_CON9, MT6332_VPA_CON11, GENMASK(5, 0), + MT6332_VPA_CON12, MT6332_VPA_CON7), + MT6332_BUCK("buck-vrf18a", VRF1, 1050000, 2240625, 9375, buck_rf_volt_range, + MT6332_VRF1_CON9, MT6332_VRF1_CON11, GENMASK(6, 0), + MT6332_VRF1_CON12, MT6332_VRF1_CON7), + MT6332_BUCK("buck-vrf18b", VRF2, 1050000, 2240625, 9375, buck_rf_volt_range, + MT6332_VRF2_CON9, MT6332_VRF2_CON11, GENMASK(6, 0), + MT6332_VRF2_CON12, MT6332_VRF2_CON7), + MT6332_BUCK("buck-vsbst", VSBST, 3500000, 7468750, 31250, boost_volt_range, + MT6332_VSBST_CON8, MT6332_VSBST_CON12, GENMASK(6, 0), + MT6332_VSBST_CON13, MT6332_VSBST_CON8), + MT6332_LDO("ldo-vauxb32", VAUXB32, ldo_volt_table1, MT6332_LDO_CON1, 10, + MT6332_LDO_CON9, GENMASK(6, 5), MT6332_LDO_CON1, GENMASK(1, 0)), + MT6332_REG_FIXED("ldo-vbif28", VBIF28, MT6332_LDO_CON2, 10, 0, 2800000, 1), + MT6332_REG_FIXED("ldo-vusb33", VUSB33, MT6332_LDO_CON3, 10, 0, 3300000, 2), + MT6332_LDO_LINEAR("ldo-vsram", VSRAM_DVFS2, 700000, 1493750, 6250, buck_volt_range, + MT6332_EN_STATUS0, MT6332_LDO_CON8, GENMASK(15, 9), + MT6332_VDVFS2_CON23, MT6332_VDVFS2_CON22, + MT6332_LDO_CON5, GENMASK(1, 0)), + MT6332_LDO_AO("ldo-vdig18", VDIG18, ldo_volt_table2, MT6332_LDO_CON12, GENMASK(11, 9)), +}; + +static int mt6332_set_buck_vosel_reg(struct platform_device *pdev) +{ + struct mt6397_chip *mt6332 = dev_get_drvdata(pdev->dev.parent); + int i; + u32 regval; + + for (i = 0; i < MT6332_ID_VREG_MAX; i++) { + if (mt6332_regulators[i].vselctrl_reg) { + if (regmap_read(mt6332->regmap, + mt6332_regulators[i].vselctrl_reg, + ®val) < 0) { + dev_err(&pdev->dev, + "Failed to read buck ctrl\n"); + return -EIO; + } + + if (regval & mt6332_regulators[i].vselctrl_mask) { + mt6332_regulators[i].desc.vsel_reg = + mt6332_regulators[i].vselon_reg; + } + } + } + + return 0; +} + +static int mt6332_regulator_probe(struct platform_device *pdev) +{ + struct mt6397_chip *mt6332 = dev_get_drvdata(pdev->dev.parent); + struct regulator_config config = {}; + struct regulator_dev *rdev; + int i; + u32 reg_value; + + /* Query buck controller to select activated voltage register part */ + if (mt6332_set_buck_vosel_reg(pdev)) + return -EIO; + + /* Read PMIC chip revision to update constraints and voltage table */ + if (regmap_read(mt6332->regmap, MT6332_HWCID, ®_value) < 0) { + dev_err(&pdev->dev, "Failed to read Chip ID\n"); + return -EIO; + } + reg_value &= GENMASK(7, 0); + + dev_info(&pdev->dev, "Chip ID = 0x%x\n", reg_value); + + /* + * ChipID 0x10 is "MT6332 E1", has a different voltage table and + * it's currently not supported in this driver. Upon detection of + * this ID, refuse to register the regulators, as we will wrongly + * interpret the VSEL for this revision, potentially overvolting + * some device. + */ + if (reg_value == 0x10) { + dev_err(&pdev->dev, "Chip version not supported. Bailing out.\n"); + return -EINVAL; + } + + for (i = 0; i < MT6332_ID_VREG_MAX; i++) { + config.dev = &pdev->dev; + config.driver_data = &mt6332_regulators[i]; + config.regmap = mt6332->regmap; + rdev = devm_regulator_register(&pdev->dev, + &mt6332_regulators[i].desc, &config); + if (IS_ERR(rdev)) { + dev_err(&pdev->dev, "failed to register %s\n", + mt6332_regulators[i].desc.name); + return PTR_ERR(rdev); + } + } + return 0; +} + +static const struct platform_device_id mt6332_platform_ids[] = { + {"mt6332-regulator", 0}, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(platform, mt6332_platform_ids); + +static struct platform_driver mt6332_regulator_driver = { + .driver = { + .name = "mt6332-regulator", + }, + .probe = mt6332_regulator_probe, + .id_table = mt6332_platform_ids, +}; + +module_platform_driver(mt6332_regulator_driver); + +MODULE_AUTHOR("AngeloGioacchino Del Regno "); +MODULE_DESCRIPTION("Regulator Driver for MediaTek MT6332 PMIC"); +MODULE_LICENSE("GPL"); diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c index e12b681c72e5ebfd8d68d2b4365fe74d0abfa7b0..0aff1c2886b5dd425e4a20299a47dc7833fb64ea 100644 --- a/drivers/regulator/of_regulator.c +++ b/drivers/regulator/of_regulator.c @@ -676,7 +676,7 @@ clean: } /** - * of_parse_coupled regulator - Get regulator_dev pointer from rdev's property + * of_parse_coupled_regulator() - Get regulator_dev pointer from rdev's property * @rdev: Pointer to regulator_dev, whose DTS is used as a source to parse * "regulator-coupled-with" property * @index: Index in phandles array diff --git a/drivers/regulator/pfuze100-regulator.c b/drivers/regulator/pfuze100-regulator.c index 6b617024a67d1b048bf4df59806826e26ef8dd47..d899d6e98fb81d938b5daf81aab4ec17cede294e 100644 --- a/drivers/regulator/pfuze100-regulator.c +++ b/drivers/regulator/pfuze100-regulator.c @@ -766,7 +766,7 @@ static int pfuze100_regulator_probe(struct i2c_client *client, ((pfuze_chip->chip_id == PFUZE3000) ? "3000" : "3001")))); memcpy(pfuze_chip->regulator_descs, pfuze_chip->pfuze_regulators, - sizeof(pfuze_chip->regulator_descs)); + regulator_num * sizeof(struct pfuze_regulator)); ret = pfuze_parse_regulators_dt(pfuze_chip); if (ret) diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c index 561de6b2e6e363a5c119e2a7986e76fc79789811..4158ff126a67a8104bd99c96b812cfe2b6c967ef 100644 --- a/drivers/regulator/qcom-rpmh-regulator.c +++ b/drivers/regulator/qcom-rpmh-regulator.c @@ -306,9 +306,10 @@ static unsigned int rpmh_regulator_vrm_get_mode(struct regulator_dev *rdev) } /** - * rpmh_regulator_vrm_set_load() - set the regulator mode based upon the load - * current requested + * rpmh_regulator_vrm_get_optimum_mode() - get the mode based on the load * @rdev: Regulator device pointer for the rpmh-regulator + * @input_uV: Input voltage + * @output_uV: Output voltage * @load_uA: Aggregated load current in microamps * * This function is used in the regulator_ops for VRM type RPMh regulator @@ -316,17 +317,15 @@ static unsigned int rpmh_regulator_vrm_get_mode(struct regulator_dev *rdev) * * Return: 0 on success, errno on failure */ -static int rpmh_regulator_vrm_set_load(struct regulator_dev *rdev, int load_uA) +static unsigned int rpmh_regulator_vrm_get_optimum_mode( + struct regulator_dev *rdev, int input_uV, int output_uV, int load_uA) { struct rpmh_vreg *vreg = rdev_get_drvdata(rdev); - unsigned int mode; if (load_uA >= vreg->hw_data->hpm_min_load_uA) - mode = REGULATOR_MODE_NORMAL; + return REGULATOR_MODE_NORMAL; else - mode = REGULATOR_MODE_IDLE; - - return rpmh_regulator_vrm_set_mode(rdev, mode); + return REGULATOR_MODE_IDLE; } static int rpmh_regulator_vrm_set_bypass(struct regulator_dev *rdev, @@ -375,7 +374,7 @@ static const struct regulator_ops rpmh_regulator_vrm_drms_ops = { .list_voltage = regulator_list_voltage_linear_range, .set_mode = rpmh_regulator_vrm_set_mode, .get_mode = rpmh_regulator_vrm_get_mode, - .set_load = rpmh_regulator_vrm_set_load, + .get_optimum_mode = rpmh_regulator_vrm_get_optimum_mode, }; static const struct regulator_ops rpmh_regulator_vrm_bypass_ops = { @@ -1199,6 +1198,52 @@ static const struct rpmh_vreg_init_data pmr735a_vreg_data[] = { {} }; +static const struct rpmh_vreg_init_data pm660_vreg_data[] = { + RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", "smp%s3", &pmic4_ftsmps426, "vdd-s3"), + RPMH_VREG("smps4", "smp%s4", &pmic4_hfsmps3, "vdd-s4"), + RPMH_VREG("smps5", "smp%s5", &pmic4_hfsmps3, "vdd-s5"), + RPMH_VREG("smps6", "smp%s6", &pmic4_hfsmps3, "vdd-s6"), + RPMH_VREG("ldo1", "ldo%s1", &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo2", "ldo%s2", &pmic4_nldo, "vdd-l2-l3"), + RPMH_VREG("ldo3", "ldo%s3", &pmic4_nldo, "vdd-l2-l3"), + /* ldo4 is inaccessible on PM660 */ + RPMH_VREG("ldo5", "ldo%s5", &pmic4_nldo, "vdd-l5"), + RPMH_VREG("ldo6", "ldo%s6", &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo7", "ldo%s7", &pmic4_nldo, "vdd-l1-l6-l7"), + RPMH_VREG("ldo8", "ldo%s8", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo9", "ldo%s9", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo10", "ldo%s10", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo11", "ldo%s11", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo12", "ldo%s12", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo13", "ldo%s13", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo14", "ldo%s14", &pmic4_pldo_lv, "vdd-l8-l9-l10-l11-l12-l13-l14"), + RPMH_VREG("ldo15", "ldo%s15", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo16", "ldo%s16", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo17", "ldo%s17", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo18", "ldo%s18", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + RPMH_VREG("ldo19", "ldo%s19", &pmic4_pldo, "vdd-l15-l16-l17-l18-l19"), + {} +}; + +static const struct rpmh_vreg_init_data pm660l_vreg_data[] = { + RPMH_VREG("smps1", "smp%s1", &pmic4_ftsmps426, "vdd-s1"), + RPMH_VREG("smps2", "smp%s2", &pmic4_ftsmps426, "vdd-s2"), + RPMH_VREG("smps3", "smp%s3", &pmic4_ftsmps426, "vdd-s3-s4"), + RPMH_VREG("smps5", "smp%s5", &pmic4_ftsmps426, "vdd-s5"), + RPMH_VREG("ldo1", "ldo%s1", &pmic4_nldo, "vdd-l1-l9-l10"), + RPMH_VREG("ldo2", "ldo%s2", &pmic4_pldo, "vdd-l2"), + RPMH_VREG("ldo3", "ldo%s3", &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo4", "ldo%s4", &pmic4_pldo, "vdd-l4-l6"), + RPMH_VREG("ldo5", "ldo%s5", &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo6", "ldo%s6", &pmic4_pldo, "vdd-l4-l6"), + RPMH_VREG("ldo7", "ldo%s7", &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("ldo8", "ldo%s8", &pmic4_pldo, "vdd-l3-l5-l7-l8"), + RPMH_VREG("bob", "bob%s1", &pmic4_bob, "vdd-bob"), + {} +}; + static int rpmh_regulator_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1321,6 +1366,14 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = { .compatible = "qcom,pmr735a-rpmh-regulators", .data = pmr735a_vreg_data, }, + { + .compatible = "qcom,pm660-rpmh-regulators", + .data = pm660_vreg_data, + }, + { + .compatible = "qcom,pm660l-rpmh-regulators", + .data = pm660l_vreg_data, + }, {} }; MODULE_DEVICE_TABLE(of, rpmh_regulator_match_table); diff --git a/drivers/regulator/qcom_rpm-regulator.c b/drivers/regulator/qcom_rpm-regulator.c index 7f9d66ac37ff8a4bf5c6e57e9574aae5c17f8f84..3c41b71a1f5298004ea7bceeb82d99ed2310ed23 100644 --- a/drivers/regulator/qcom_rpm-regulator.c +++ b/drivers/regulator/qcom_rpm-regulator.c @@ -802,6 +802,12 @@ static const struct rpm_regulator_data rpm_pm8018_regulators[] = { }; static const struct rpm_regulator_data rpm_pm8058_regulators[] = { + { "s0", QCOM_RPM_PM8058_SMPS0, &pm8058_smps, "vdd_s0" }, + { "s1", QCOM_RPM_PM8058_SMPS1, &pm8058_smps, "vdd_s1" }, + { "s2", QCOM_RPM_PM8058_SMPS2, &pm8058_smps, "vdd_s2" }, + { "s3", QCOM_RPM_PM8058_SMPS3, &pm8058_smps, "vdd_s3" }, + { "s4", QCOM_RPM_PM8058_SMPS4, &pm8058_smps, "vdd_s4" }, + { "l0", QCOM_RPM_PM8058_LDO0, &pm8058_nldo, "vdd_l0_l1_lvs" }, { "l1", QCOM_RPM_PM8058_LDO1, &pm8058_nldo, "vdd_l0_l1_lvs" }, { "l2", QCOM_RPM_PM8058_LDO2, &pm8058_pldo, "vdd_l2_l11_l12" }, @@ -829,12 +835,6 @@ static const struct rpm_regulator_data rpm_pm8058_regulators[] = { { "l24", QCOM_RPM_PM8058_LDO24, &pm8058_nldo, "vdd_l23_l24_l25" }, { "l25", QCOM_RPM_PM8058_LDO25, &pm8058_nldo, "vdd_l23_l24_l25" }, - { "s0", QCOM_RPM_PM8058_SMPS0, &pm8058_smps, "vdd_s0" }, - { "s1", QCOM_RPM_PM8058_SMPS1, &pm8058_smps, "vdd_s1" }, - { "s2", QCOM_RPM_PM8058_SMPS2, &pm8058_smps, "vdd_s2" }, - { "s3", QCOM_RPM_PM8058_SMPS3, &pm8058_smps, "vdd_s3" }, - { "s4", QCOM_RPM_PM8058_SMPS4, &pm8058_smps, "vdd_s4" }, - { "lvs0", QCOM_RPM_PM8058_LVS0, &pm8058_switch, "vdd_l0_l1_lvs" }, { "lvs1", QCOM_RPM_PM8058_LVS1, &pm8058_switch, "vdd_l0_l1_lvs" }, @@ -843,6 +843,12 @@ static const struct rpm_regulator_data rpm_pm8058_regulators[] = { }; static const struct rpm_regulator_data rpm_pm8901_regulators[] = { + { "s0", QCOM_RPM_PM8901_SMPS0, &pm8901_ftsmps, "vdd_s0" }, + { "s1", QCOM_RPM_PM8901_SMPS1, &pm8901_ftsmps, "vdd_s1" }, + { "s2", QCOM_RPM_PM8901_SMPS2, &pm8901_ftsmps, "vdd_s2" }, + { "s3", QCOM_RPM_PM8901_SMPS3, &pm8901_ftsmps, "vdd_s3" }, + { "s4", QCOM_RPM_PM8901_SMPS4, &pm8901_ftsmps, "vdd_s4" }, + { "l0", QCOM_RPM_PM8901_LDO0, &pm8901_nldo, "vdd_l0" }, { "l1", QCOM_RPM_PM8901_LDO1, &pm8901_pldo, "vdd_l1" }, { "l2", QCOM_RPM_PM8901_LDO2, &pm8901_pldo, "vdd_l2" }, @@ -851,12 +857,6 @@ static const struct rpm_regulator_data rpm_pm8901_regulators[] = { { "l5", QCOM_RPM_PM8901_LDO5, &pm8901_pldo, "vdd_l5" }, { "l6", QCOM_RPM_PM8901_LDO6, &pm8901_pldo, "vdd_l6" }, - { "s0", QCOM_RPM_PM8901_SMPS0, &pm8901_ftsmps, "vdd_s0" }, - { "s1", QCOM_RPM_PM8901_SMPS1, &pm8901_ftsmps, "vdd_s1" }, - { "s2", QCOM_RPM_PM8901_SMPS2, &pm8901_ftsmps, "vdd_s2" }, - { "s3", QCOM_RPM_PM8901_SMPS3, &pm8901_ftsmps, "vdd_s3" }, - { "s4", QCOM_RPM_PM8901_SMPS4, &pm8901_ftsmps, "vdd_s4" }, - { "lvs0", QCOM_RPM_PM8901_LVS0, &pm8901_switch, "lvs0_in" }, { "lvs1", QCOM_RPM_PM8901_LVS1, &pm8901_switch, "lvs1_in" }, { "lvs2", QCOM_RPM_PM8901_LVS2, &pm8901_switch, "lvs2_in" }, diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c index 59024c6391417f2bf9b0ff5cb1c7b34d4b84a98f..f98168d58dce9883881aaa8b3b8cc67f6c41571d 100644 --- a/drivers/regulator/qcom_smd-regulator.c +++ b/drivers/regulator/qcom_smd-regulator.c @@ -668,6 +668,15 @@ static const struct regulator_desc pm660l_bob = { .ops = &rpm_bob_ops, }; +static const struct regulator_desc pm6125_ftsmps = { + .linear_ranges = (struct linear_range[]) { + REGULATOR_LINEAR_RANGE(300000, 0, 268, 4000), + }, + .n_linear_ranges = 1, + .n_voltages = 269, + .ops = &rpm_smps_ldo_ops, +}; + static const struct regulator_desc pms405_hfsmps3 = { .linear_ranges = (struct linear_range[]) { REGULATOR_LINEAR_RANGE(320000, 0, 215, 8000), @@ -772,6 +781,158 @@ static const struct rpm_regulator_data rpm_mp5496_regulators[] = { {} }; +static const struct rpm_regulator_data rpm_pm2250_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPA, 1, &pm2250_lvftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPA, 2, &pm2250_lvftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPA, 3, &pm2250_lvftsmps, "vdd_s3" }, + { "s4", QCOM_SMD_RPM_SMPA, 4, &pm2250_ftsmps, "vdd_s4" }, + { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, + { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, + { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, + { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, + { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, + { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + { "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, + {} +}; + +static const struct rpm_regulator_data rpm_pm6125_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPA, 1, &pm6125_ftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPA, 2, &pm6125_ftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPA, 3, &pm6125_ftsmps, "vdd_s3" }, + { "s4", QCOM_SMD_RPM_SMPA, 4, &pm6125_ftsmps, "vdd_s4" }, + { "s5", QCOM_SMD_RPM_SMPA, 5, &pm8998_hfsmps, "vdd_s5" }, + { "s6", QCOM_SMD_RPM_SMPA, 6, &pm8998_hfsmps, "vdd_s6" }, + { "s7", QCOM_SMD_RPM_SMPA, 7, &pm8998_hfsmps, "vdd_s7" }, + { "s8", QCOM_SMD_RPM_SMPA, 8, &pm6125_ftsmps, "vdd_s8" }, + { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l7_l17_l18" }, + { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l2_l3_l4" }, + { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3_l4" }, + { "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_nldo660, "vdd_l2_l3_l4" }, + { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l6_l8" }, + { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l7_l17_l18" }, + { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l6_l8" }, + { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l9_l11" }, + { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l10_l13_l14" }, + { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l9_l11" }, + { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l12_l16" }, + { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l10_l13_l14" }, + { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l10_l13_l14" }, + { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l12_l16" }, + { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_nldo660, "vdd_l1_l7_l17_l18" }, + { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_nldo660, "vdd_l1_l7_l17_l18" }, + { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l23", QCOM_SMD_RPM_LDOA, 23, &pm660_pldo660, "vdd_l23_l24" }, + { "l24", QCOM_SMD_RPM_LDOA, 24, &pm660_pldo660, "vdd_l23_l24" }, + { } +}; + +static const struct rpm_regulator_data rpm_pm660_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPA, 1, &pm660_ftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPA, 2, &pm660_ftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPA, 3, &pm660_ftsmps, "vdd_s3" }, + { "s4", QCOM_SMD_RPM_SMPA, 4, &pm660_hfsmps, "vdd_s4" }, + { "s5", QCOM_SMD_RPM_SMPA, 5, &pm660_hfsmps, "vdd_s5" }, + { "s6", QCOM_SMD_RPM_SMPA, 6, &pm660_hfsmps, "vdd_s6" }, + { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l6_l7" }, + { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_ht_nldo, "vdd_l2_l3" }, + { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3" }, + /* l4 is unaccessible on PM660 */ + { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_ht_nldo, "vdd_l5" }, + { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_ht_nldo, "vdd_l1_l6_l7" }, + { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_ht_nldo, "vdd_l1_l6_l7" }, + { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, + { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, + { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, + { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, + { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, + { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, + { } +}; + +static const struct rpm_regulator_data rpm_pm660l_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPB, 1, &pm660_ftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPB, 2, &pm660_ftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_RWCX, 0, &pm660_ftsmps, "vdd_s3_s4" }, + { "s5", QCOM_SMD_RPM_RWMX, 0, &pm660_ftsmps, "vdd_s5" }, + { "l1", QCOM_SMD_RPM_LDOB, 1, &pm660_nldo660, "vdd_l1_l9_l10" }, + { "l2", QCOM_SMD_RPM_LDOB, 2, &pm660_pldo660, "vdd_l2" }, + { "l3", QCOM_SMD_RPM_LDOB, 3, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, + { "l4", QCOM_SMD_RPM_LDOB, 4, &pm660_pldo660, "vdd_l4_l6" }, + { "l5", QCOM_SMD_RPM_LDOB, 5, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, + { "l6", QCOM_SMD_RPM_LDOB, 6, &pm660_pldo660, "vdd_l4_l6" }, + { "l7", QCOM_SMD_RPM_LDOB, 7, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, + { "l8", QCOM_SMD_RPM_LDOB, 8, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, + { "l9", QCOM_SMD_RPM_RWLC, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" }, + { "l10", QCOM_SMD_RPM_RWLM, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" }, + { "bob", QCOM_SMD_RPM_BOBB, 1, &pm660l_bob, "vdd_bob", }, + { } +}; + +static const struct rpm_regulator_data rpm_pm8226_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8226_hfsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8226_ftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPA, 3, &pm8226_hfsmps, "vdd_s3" }, + { "s4", QCOM_SMD_RPM_SMPA, 4, &pm8226_hfsmps, "vdd_s4" }, + { "s5", QCOM_SMD_RPM_SMPA, 5, &pm8226_hfsmps, "vdd_s5" }, + { "l1", QCOM_SMD_RPM_LDOA, 1, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, + { "l2", QCOM_SMD_RPM_LDOA, 2, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, + { "l3", QCOM_SMD_RPM_LDOA, 3, &pm8226_nldo, "vdd_l3_l24_l26" }, + { "l4", QCOM_SMD_RPM_LDOA, 4, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, + { "l5", QCOM_SMD_RPM_LDOA, 5, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, + { "l6", QCOM_SMD_RPM_LDOA, 6, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, + { "l7", QCOM_SMD_RPM_LDOA, 7, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, + { "l8", QCOM_SMD_RPM_LDOA, 8, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, + { "l9", QCOM_SMD_RPM_LDOA, 9, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, + { "l10", QCOM_SMD_RPM_LDOA, 10, &pm8226_pldo, "vdd_l10_l11_l13" }, + { "l11", QCOM_SMD_RPM_LDOA, 11, &pm8226_pldo, "vdd_l10_l11_l13" }, + { "l12", QCOM_SMD_RPM_LDOA, 12, &pm8226_pldo, "vdd_l12_l14" }, + { "l13", QCOM_SMD_RPM_LDOA, 13, &pm8226_pldo, "vdd_l10_l11_l13" }, + { "l14", QCOM_SMD_RPM_LDOA, 14, &pm8226_pldo, "vdd_l12_l14" }, + { "l15", QCOM_SMD_RPM_LDOA, 15, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, + { "l16", QCOM_SMD_RPM_LDOA, 16, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, + { "l17", QCOM_SMD_RPM_LDOA, 17, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, + { "l18", QCOM_SMD_RPM_LDOA, 18, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, + { "l19", QCOM_SMD_RPM_LDOA, 19, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "l20", QCOM_SMD_RPM_LDOA, 20, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "l21", QCOM_SMD_RPM_LDOA, 21, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "l22", QCOM_SMD_RPM_LDOA, 22, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "l23", QCOM_SMD_RPM_LDOA, 23, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "l24", QCOM_SMD_RPM_LDOA, 24, &pm8226_nldo, "vdd_l3_l24_l26" }, + { "l25", QCOM_SMD_RPM_LDOA, 25, &pm8226_pldo, "vdd_l25" }, + { "l26", QCOM_SMD_RPM_LDOA, 26, &pm8226_nldo, "vdd_l3_l24_l26" }, + { "l27", QCOM_SMD_RPM_LDOA, 27, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, + { "l28", QCOM_SMD_RPM_LDOA, 28, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, + { "lvs1", QCOM_SMD_RPM_VSA, 1, &pm8226_switch, "vdd_lvs1" }, + {} +}; + static const struct rpm_regulator_data rpm_pm8841_regulators[] = { { "s1", QCOM_SMD_RPM_SMPB, 1, &pm8x41_hfsmps, "vdd_s1" }, { "s2", QCOM_SMD_RPM_SMPB, 2, &pm8841_ftsmps, "vdd_s2" }, @@ -833,44 +994,6 @@ static const struct rpm_regulator_data rpm_pm8916_regulators[] = { {} }; -static const struct rpm_regulator_data rpm_pm8226_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8226_hfsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8226_ftsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_SMPA, 3, &pm8226_hfsmps, "vdd_s3" }, - { "s4", QCOM_SMD_RPM_SMPA, 4, &pm8226_hfsmps, "vdd_s4" }, - { "s5", QCOM_SMD_RPM_SMPA, 5, &pm8226_hfsmps, "vdd_s5" }, - { "l1", QCOM_SMD_RPM_LDOA, 1, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, - { "l2", QCOM_SMD_RPM_LDOA, 2, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, - { "l3", QCOM_SMD_RPM_LDOA, 3, &pm8226_nldo, "vdd_l3_l24_l26" }, - { "l4", QCOM_SMD_RPM_LDOA, 4, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, - { "l5", QCOM_SMD_RPM_LDOA, 5, &pm8226_nldo, "vdd_l1_l2_l4_l5" }, - { "l6", QCOM_SMD_RPM_LDOA, 6, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, - { "l7", QCOM_SMD_RPM_LDOA, 7, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, - { "l8", QCOM_SMD_RPM_LDOA, 8, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, - { "l9", QCOM_SMD_RPM_LDOA, 9, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, - { "l10", QCOM_SMD_RPM_LDOA, 10, &pm8226_pldo, "vdd_l10_l11_l13" }, - { "l11", QCOM_SMD_RPM_LDOA, 11, &pm8226_pldo, "vdd_l10_l11_l13" }, - { "l12", QCOM_SMD_RPM_LDOA, 12, &pm8226_pldo, "vdd_l12_l14" }, - { "l13", QCOM_SMD_RPM_LDOA, 13, &pm8226_pldo, "vdd_l10_l11_l13" }, - { "l14", QCOM_SMD_RPM_LDOA, 14, &pm8226_pldo, "vdd_l12_l14" }, - { "l15", QCOM_SMD_RPM_LDOA, 15, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, - { "l16", QCOM_SMD_RPM_LDOA, 16, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, - { "l17", QCOM_SMD_RPM_LDOA, 17, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, - { "l18", QCOM_SMD_RPM_LDOA, 18, &pm8226_pldo, "vdd_l15_l16_l17_l18" }, - { "l19", QCOM_SMD_RPM_LDOA, 19, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "l20", QCOM_SMD_RPM_LDOA, 20, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "l21", QCOM_SMD_RPM_LDOA, 21, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "l22", QCOM_SMD_RPM_LDOA, 22, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "l23", QCOM_SMD_RPM_LDOA, 23, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "l24", QCOM_SMD_RPM_LDOA, 24, &pm8226_nldo, "vdd_l3_l24_l26" }, - { "l25", QCOM_SMD_RPM_LDOA, 25, &pm8226_pldo, "vdd_l25" }, - { "l26", QCOM_SMD_RPM_LDOA, 26, &pm8226_nldo, "vdd_l3_l24_l26" }, - { "l27", QCOM_SMD_RPM_LDOA, 27, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" }, - { "l28", QCOM_SMD_RPM_LDOA, 28, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" }, - { "lvs1", QCOM_SMD_RPM_VSA, 1, &pm8226_switch, "vdd_lvs1" }, - {} -}; - static const struct rpm_regulator_data rpm_pm8941_regulators[] = { { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8x41_hfsmps, "vdd_s1" }, { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8x41_hfsmps, "vdd_s2" }, @@ -912,57 +1035,6 @@ static const struct rpm_regulator_data rpm_pm8941_regulators[] = { {} }; -static const struct rpm_regulator_data rpm_pma8084_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPA, 1, &pma8084_ftsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPA, 2, &pma8084_ftsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_SMPA, 3, &pma8084_hfsmps, "vdd_s3" }, - { "s4", QCOM_SMD_RPM_SMPA, 4, &pma8084_hfsmps, "vdd_s4" }, - { "s5", QCOM_SMD_RPM_SMPA, 5, &pma8084_hfsmps, "vdd_s5" }, - { "s6", QCOM_SMD_RPM_SMPA, 6, &pma8084_ftsmps, "vdd_s6" }, - { "s7", QCOM_SMD_RPM_SMPA, 7, &pma8084_ftsmps, "vdd_s7" }, - { "s8", QCOM_SMD_RPM_SMPA, 8, &pma8084_ftsmps, "vdd_s8" }, - { "s9", QCOM_SMD_RPM_SMPA, 9, &pma8084_ftsmps, "vdd_s9" }, - { "s10", QCOM_SMD_RPM_SMPA, 10, &pma8084_ftsmps, "vdd_s10" }, - { "s11", QCOM_SMD_RPM_SMPA, 11, &pma8084_ftsmps, "vdd_s11" }, - { "s12", QCOM_SMD_RPM_SMPA, 12, &pma8084_ftsmps, "vdd_s12" }, - - { "l1", QCOM_SMD_RPM_LDOA, 1, &pma8084_nldo, "vdd_l1_l11" }, - { "l2", QCOM_SMD_RPM_LDOA, 2, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, - { "l3", QCOM_SMD_RPM_LDOA, 3, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, - { "l4", QCOM_SMD_RPM_LDOA, 4, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, - { "l5", QCOM_SMD_RPM_LDOA, 5, &pma8084_pldo, "vdd_l5_l7" }, - { "l6", QCOM_SMD_RPM_LDOA, 6, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, - { "l7", QCOM_SMD_RPM_LDOA, 7, &pma8084_pldo, "vdd_l5_l7" }, - { "l8", QCOM_SMD_RPM_LDOA, 8, &pma8084_pldo, "vdd_l8" }, - { "l9", QCOM_SMD_RPM_LDOA, 9, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l10", QCOM_SMD_RPM_LDOA, 10, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l11", QCOM_SMD_RPM_LDOA, 11, &pma8084_nldo, "vdd_l1_l11" }, - { "l12", QCOM_SMD_RPM_LDOA, 12, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, - { "l13", QCOM_SMD_RPM_LDOA, 13, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l14", QCOM_SMD_RPM_LDOA, 14, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, - { "l15", QCOM_SMD_RPM_LDOA, 15, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, - { "l16", QCOM_SMD_RPM_LDOA, 16, &pma8084_pldo, "vdd_l16_l25" }, - { "l17", QCOM_SMD_RPM_LDOA, 17, &pma8084_pldo, "vdd_l17" }, - { "l18", QCOM_SMD_RPM_LDOA, 18, &pma8084_pldo, "vdd_l18" }, - { "l19", QCOM_SMD_RPM_LDOA, 19, &pma8084_pldo, "vdd_l19" }, - { "l20", QCOM_SMD_RPM_LDOA, 20, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l21", QCOM_SMD_RPM_LDOA, 21, &pma8084_pldo, "vdd_l21" }, - { "l22", QCOM_SMD_RPM_LDOA, 22, &pma8084_pldo, "vdd_l22" }, - { "l23", QCOM_SMD_RPM_LDOA, 23, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l24", QCOM_SMD_RPM_LDOA, 24, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, - { "l25", QCOM_SMD_RPM_LDOA, 25, &pma8084_pldo, "vdd_l16_l25" }, - { "l26", QCOM_SMD_RPM_LDOA, 26, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, - { "l27", QCOM_SMD_RPM_LDOA, 27, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, - - { "lvs1", QCOM_SMD_RPM_VSA, 1, &pma8084_switch }, - { "lvs2", QCOM_SMD_RPM_VSA, 2, &pma8084_switch }, - { "lvs3", QCOM_SMD_RPM_VSA, 3, &pma8084_switch }, - { "lvs4", QCOM_SMD_RPM_VSA, 4, &pma8084_switch }, - { "5vs1", QCOM_SMD_RPM_VSA, 5, &pma8084_switch }, - - {} -}; - static const struct rpm_regulator_data rpm_pm8950_regulators[] = { { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8950_hfsmps, "vdd_s1" }, { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8950_hfsmps, "vdd_s2" }, @@ -1082,14 +1154,6 @@ static const struct rpm_regulator_data rpm_pm8994_regulators[] = { {} }; -static const struct rpm_regulator_data rpm_pmi8994_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPB, 1, &pmi8994_ftsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPB, 2, &pmi8994_hfsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_SMPB, 3, &pmi8994_hfsmps, "vdd_s3" }, - { "boost-bypass", QCOM_SMD_RPM_BBYB, 1, &pmi8994_bby, "vdd_bst_byp" }, - {} -}; - static const struct rpm_regulator_data rpm_pm8998_regulators[] = { { "s1", QCOM_SMD_RPM_SMPA, 1, &pm8998_ftsmps, "vdd_s1" }, { "s2", QCOM_SMD_RPM_SMPA, 2, &pm8998_ftsmps, "vdd_s2" }, @@ -1137,57 +1201,68 @@ static const struct rpm_regulator_data rpm_pm8998_regulators[] = { {} }; -static const struct rpm_regulator_data rpm_pmi8998_regulators[] = { - { "bob", QCOM_SMD_RPM_BOBB, 1, &pmi8998_bob, "vdd_bob" }, +static const struct rpm_regulator_data rpm_pma8084_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPA, 1, &pma8084_ftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPA, 2, &pma8084_ftsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPA, 3, &pma8084_hfsmps, "vdd_s3" }, + { "s4", QCOM_SMD_RPM_SMPA, 4, &pma8084_hfsmps, "vdd_s4" }, + { "s5", QCOM_SMD_RPM_SMPA, 5, &pma8084_hfsmps, "vdd_s5" }, + { "s6", QCOM_SMD_RPM_SMPA, 6, &pma8084_ftsmps, "vdd_s6" }, + { "s7", QCOM_SMD_RPM_SMPA, 7, &pma8084_ftsmps, "vdd_s7" }, + { "s8", QCOM_SMD_RPM_SMPA, 8, &pma8084_ftsmps, "vdd_s8" }, + { "s9", QCOM_SMD_RPM_SMPA, 9, &pma8084_ftsmps, "vdd_s9" }, + { "s10", QCOM_SMD_RPM_SMPA, 10, &pma8084_ftsmps, "vdd_s10" }, + { "s11", QCOM_SMD_RPM_SMPA, 11, &pma8084_ftsmps, "vdd_s11" }, + { "s12", QCOM_SMD_RPM_SMPA, 12, &pma8084_ftsmps, "vdd_s12" }, + + { "l1", QCOM_SMD_RPM_LDOA, 1, &pma8084_nldo, "vdd_l1_l11" }, + { "l2", QCOM_SMD_RPM_LDOA, 2, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, + { "l3", QCOM_SMD_RPM_LDOA, 3, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, + { "l4", QCOM_SMD_RPM_LDOA, 4, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, + { "l5", QCOM_SMD_RPM_LDOA, 5, &pma8084_pldo, "vdd_l5_l7" }, + { "l6", QCOM_SMD_RPM_LDOA, 6, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, + { "l7", QCOM_SMD_RPM_LDOA, 7, &pma8084_pldo, "vdd_l5_l7" }, + { "l8", QCOM_SMD_RPM_LDOA, 8, &pma8084_pldo, "vdd_l8" }, + { "l9", QCOM_SMD_RPM_LDOA, 9, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l10", QCOM_SMD_RPM_LDOA, 10, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l11", QCOM_SMD_RPM_LDOA, 11, &pma8084_nldo, "vdd_l1_l11" }, + { "l12", QCOM_SMD_RPM_LDOA, 12, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, + { "l13", QCOM_SMD_RPM_LDOA, 13, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l14", QCOM_SMD_RPM_LDOA, 14, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, + { "l15", QCOM_SMD_RPM_LDOA, 15, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, + { "l16", QCOM_SMD_RPM_LDOA, 16, &pma8084_pldo, "vdd_l16_l25" }, + { "l17", QCOM_SMD_RPM_LDOA, 17, &pma8084_pldo, "vdd_l17" }, + { "l18", QCOM_SMD_RPM_LDOA, 18, &pma8084_pldo, "vdd_l18" }, + { "l19", QCOM_SMD_RPM_LDOA, 19, &pma8084_pldo, "vdd_l19" }, + { "l20", QCOM_SMD_RPM_LDOA, 20, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l21", QCOM_SMD_RPM_LDOA, 21, &pma8084_pldo, "vdd_l21" }, + { "l22", QCOM_SMD_RPM_LDOA, 22, &pma8084_pldo, "vdd_l22" }, + { "l23", QCOM_SMD_RPM_LDOA, 23, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l24", QCOM_SMD_RPM_LDOA, 24, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" }, + { "l25", QCOM_SMD_RPM_LDOA, 25, &pma8084_pldo, "vdd_l16_l25" }, + { "l26", QCOM_SMD_RPM_LDOA, 26, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" }, + { "l27", QCOM_SMD_RPM_LDOA, 27, &pma8084_nldo, "vdd_l2_l3_l4_l27" }, + + { "lvs1", QCOM_SMD_RPM_VSA, 1, &pma8084_switch }, + { "lvs2", QCOM_SMD_RPM_VSA, 2, &pma8084_switch }, + { "lvs3", QCOM_SMD_RPM_VSA, 3, &pma8084_switch }, + { "lvs4", QCOM_SMD_RPM_VSA, 4, &pma8084_switch }, + { "5vs1", QCOM_SMD_RPM_VSA, 5, &pma8084_switch }, + {} }; -static const struct rpm_regulator_data rpm_pm660_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPA, 1, &pm660_ftsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPA, 2, &pm660_ftsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_SMPA, 3, &pm660_ftsmps, "vdd_s3" }, - { "s4", QCOM_SMD_RPM_SMPA, 4, &pm660_hfsmps, "vdd_s4" }, - { "s5", QCOM_SMD_RPM_SMPA, 5, &pm660_hfsmps, "vdd_s5" }, - { "s6", QCOM_SMD_RPM_SMPA, 6, &pm660_hfsmps, "vdd_s6" }, - { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l6_l7" }, - { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_ht_nldo, "vdd_l2_l3" }, - { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3" }, - /* l4 is unaccessible on PM660 */ - { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_ht_nldo, "vdd_l5" }, - { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_ht_nldo, "vdd_l1_l6_l7" }, - { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_ht_nldo, "vdd_l1_l6_l7" }, - { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" }, - { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, - { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, - { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, - { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, - { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" }, - { } +static const struct rpm_regulator_data rpm_pmi8994_regulators[] = { + { "s1", QCOM_SMD_RPM_SMPB, 1, &pmi8994_ftsmps, "vdd_s1" }, + { "s2", QCOM_SMD_RPM_SMPB, 2, &pmi8994_hfsmps, "vdd_s2" }, + { "s3", QCOM_SMD_RPM_SMPB, 3, &pmi8994_hfsmps, "vdd_s3" }, + { "boost-bypass", QCOM_SMD_RPM_BBYB, 1, &pmi8994_bby, "vdd_bst_byp" }, + {} }; -static const struct rpm_regulator_data rpm_pm660l_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPB, 1, &pm660_ftsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPB, 2, &pm660_ftsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_RWCX, 0, &pm660_ftsmps, "vdd_s3_s4" }, - { "s5", QCOM_SMD_RPM_RWMX, 0, &pm660_ftsmps, "vdd_s5" }, - { "l1", QCOM_SMD_RPM_LDOB, 1, &pm660_nldo660, "vdd_l1_l9_l10" }, - { "l2", QCOM_SMD_RPM_LDOB, 2, &pm660_pldo660, "vdd_l2" }, - { "l3", QCOM_SMD_RPM_LDOB, 3, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, - { "l4", QCOM_SMD_RPM_LDOB, 4, &pm660_pldo660, "vdd_l4_l6" }, - { "l5", QCOM_SMD_RPM_LDOB, 5, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, - { "l6", QCOM_SMD_RPM_LDOB, 6, &pm660_pldo660, "vdd_l4_l6" }, - { "l7", QCOM_SMD_RPM_LDOB, 7, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, - { "l8", QCOM_SMD_RPM_LDOB, 8, &pm660_pldo660, "vdd_l3_l5_l7_l8" }, - { "l9", QCOM_SMD_RPM_RWLC, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" }, - { "l10", QCOM_SMD_RPM_RWLM, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" }, - { "bob", QCOM_SMD_RPM_BOBB, 1, &pm660l_bob, "vdd_bob", }, - { } +static const struct rpm_regulator_data rpm_pmi8998_regulators[] = { + { "bob", QCOM_SMD_RPM_BOBB, 1, &pmi8998_bob, "vdd_bob" }, + {} }; static const struct rpm_regulator_data rpm_pms405_regulators[] = { @@ -1212,54 +1287,25 @@ static const struct rpm_regulator_data rpm_pms405_regulators[] = { {} }; -static const struct rpm_regulator_data rpm_pm2250_regulators[] = { - { "s1", QCOM_SMD_RPM_SMPA, 1, &pm2250_lvftsmps, "vdd_s1" }, - { "s2", QCOM_SMD_RPM_SMPA, 2, &pm2250_lvftsmps, "vdd_s2" }, - { "s3", QCOM_SMD_RPM_SMPA, 3, &pm2250_lvftsmps, "vdd_s3" }, - { "s4", QCOM_SMD_RPM_SMPA, 4, &pm2250_ftsmps, "vdd_s4" }, - { "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" }, - { "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, - { "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, - { "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, - { "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" }, - { "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - { "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" }, - {} -}; - static const struct of_device_id rpm_of_match[] = { { .compatible = "qcom,rpm-mp5496-regulators", .data = &rpm_mp5496_regulators }, + { .compatible = "qcom,rpm-pm2250-regulators", .data = &rpm_pm2250_regulators }, + { .compatible = "qcom,rpm-pm6125-regulators", .data = &rpm_pm6125_regulators }, + { .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators }, + { .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators }, + { .compatible = "qcom,rpm-pm8226-regulators", .data = &rpm_pm8226_regulators }, { .compatible = "qcom,rpm-pm8841-regulators", .data = &rpm_pm8841_regulators }, { .compatible = "qcom,rpm-pm8909-regulators", .data = &rpm_pm8909_regulators }, { .compatible = "qcom,rpm-pm8916-regulators", .data = &rpm_pm8916_regulators }, - { .compatible = "qcom,rpm-pm8226-regulators", .data = &rpm_pm8226_regulators }, { .compatible = "qcom,rpm-pm8941-regulators", .data = &rpm_pm8941_regulators }, { .compatible = "qcom,rpm-pm8950-regulators", .data = &rpm_pm8950_regulators }, { .compatible = "qcom,rpm-pm8953-regulators", .data = &rpm_pm8953_regulators }, { .compatible = "qcom,rpm-pm8994-regulators", .data = &rpm_pm8994_regulators }, { .compatible = "qcom,rpm-pm8998-regulators", .data = &rpm_pm8998_regulators }, - { .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators }, - { .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators }, { .compatible = "qcom,rpm-pma8084-regulators", .data = &rpm_pma8084_regulators }, { .compatible = "qcom,rpm-pmi8994-regulators", .data = &rpm_pmi8994_regulators }, { .compatible = "qcom,rpm-pmi8998-regulators", .data = &rpm_pmi8998_regulators }, { .compatible = "qcom,rpm-pms405-regulators", .data = &rpm_pms405_regulators }, - { .compatible = "qcom,rpm-pm2250-regulators", .data = &rpm_pm2250_regulators }, {} }; MODULE_DEVICE_TABLE(of, rpm_of_match); diff --git a/drivers/regulator/qcom_spmi-regulator.c b/drivers/regulator/qcom_spmi-regulator.c index a2d0292a92fdc625d6838e192d7b18d8ae0bcd9d..3e312729741e766b37859094e69694ad516d631a 100644 --- a/drivers/regulator/qcom_spmi-regulator.c +++ b/drivers/regulator/qcom_spmi-regulator.c @@ -99,6 +99,9 @@ enum spmi_regulator_logical_type { SPMI_REGULATOR_LOGICAL_TYPE_ULT_LDO, SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS426, SPMI_REGULATOR_LOGICAL_TYPE_HFS430, + SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3, + SPMI_REGULATOR_LOGICAL_TYPE_LDO_510, + SPMI_REGULATOR_LOGICAL_TYPE_HFSMPS, }; enum spmi_regulator_type { @@ -166,6 +169,17 @@ enum spmi_regulator_subtype { SPMI_REGULATOR_SUBTYPE_HFS430 = 0x0a, SPMI_REGULATOR_SUBTYPE_HT_P150 = 0x35, SPMI_REGULATOR_SUBTYPE_HT_P600 = 0x3d, + SPMI_REGULATOR_SUBTYPE_HFSMPS_510 = 0x0a, + SPMI_REGULATOR_SUBTYPE_FTSMPS_510 = 0x0b, + SPMI_REGULATOR_SUBTYPE_LV_P150_510 = 0x71, + SPMI_REGULATOR_SUBTYPE_LV_P300_510 = 0x72, + SPMI_REGULATOR_SUBTYPE_LV_P600_510 = 0x73, + SPMI_REGULATOR_SUBTYPE_N300_510 = 0x6a, + SPMI_REGULATOR_SUBTYPE_N600_510 = 0x6b, + SPMI_REGULATOR_SUBTYPE_N1200_510 = 0x6c, + SPMI_REGULATOR_SUBTYPE_MV_P50_510 = 0x7a, + SPMI_REGULATOR_SUBTYPE_MV_P150_510 = 0x7b, + SPMI_REGULATOR_SUBTYPE_MV_P600_510 = 0x7d, }; enum spmi_common_regulator_registers { @@ -193,6 +207,14 @@ enum spmi_ftsmps426_regulator_registers { SPMI_FTSMPS426_REG_VOLTAGE_ULS_MSB = 0x69, }; +/* + * Third common register layout + */ +enum spmi_hfsmps_regulator_registers { + SPMI_HFSMPS_REG_STEP_CTRL = 0x3c, + SPMI_HFSMPS_REG_PULL_DOWN = 0xa0, +}; + enum spmi_vs_registers { SPMI_VS_REG_OCP = 0x4a, SPMI_VS_REG_SOFT_START = 0x4c, @@ -260,6 +282,15 @@ enum spmi_common_control_register_index { #define SPMI_FTSMPS426_MODE_MASK 0x07 +/* Third common regulator mode register values */ +#define SPMI_HFSMPS_MODE_BYPASS_MASK 2 +#define SPMI_HFSMPS_MODE_RETENTION_MASK 3 +#define SPMI_HFSMPS_MODE_LPM_MASK 4 +#define SPMI_HFSMPS_MODE_AUTO_MASK 6 +#define SPMI_HFSMPS_MODE_HPM_MASK 7 + +#define SPMI_HFSMPS_MODE_MASK 0x07 + /* Common regulator pull down control register layout */ #define SPMI_COMMON_PULL_DOWN_ENABLE_MASK 0x80 @@ -305,6 +336,9 @@ enum spmi_common_control_register_index { #define SPMI_FTSMPS_STEP_MARGIN_NUM 4 #define SPMI_FTSMPS_STEP_MARGIN_DEN 5 +/* slew_rate has units of uV/us. */ +#define SPMI_HFSMPS_SLEW_RATE_38p4 38400 + #define SPMI_FTSMPS426_STEP_CTRL_DELAY_MASK 0x03 #define SPMI_FTSMPS426_STEP_CTRL_DELAY_SHIFT 0 @@ -554,6 +588,14 @@ static struct spmi_voltage_range ht_p600_ranges[] = { SPMI_VOLTAGE_RANGE(0, 1704000, 1704000, 1896000, 1896000, 8000), }; +static struct spmi_voltage_range nldo_510_ranges[] = { + SPMI_VOLTAGE_RANGE(0, 320000, 320000, 1304000, 1304000, 8000), +}; + +static struct spmi_voltage_range ftsmps510_ranges[] = { + SPMI_VOLTAGE_RANGE(0, 300000, 300000, 1372000, 1372000, 4000), +}; + static DEFINE_SPMI_SET_POINTS(pldo); static DEFINE_SPMI_SET_POINTS(nldo1); static DEFINE_SPMI_SET_POINTS(nldo2); @@ -576,6 +618,8 @@ static DEFINE_SPMI_SET_POINTS(ht_nldo); static DEFINE_SPMI_SET_POINTS(hfs430); static DEFINE_SPMI_SET_POINTS(ht_p150); static DEFINE_SPMI_SET_POINTS(ht_p600); +static DEFINE_SPMI_SET_POINTS(nldo_510); +static DEFINE_SPMI_SET_POINTS(ftsmps510); static inline int spmi_vreg_read(struct spmi_regulator *vreg, u16 addr, u8 *buf, int len) @@ -1062,6 +1106,23 @@ static unsigned int spmi_regulator_ftsmps426_get_mode(struct regulator_dev *rdev } } +static unsigned int spmi_regulator_hfsmps_get_mode(struct regulator_dev *rdev) +{ + struct spmi_regulator *vreg = rdev_get_drvdata(rdev); + u8 reg; + + spmi_vreg_read(vreg, SPMI_COMMON_REG_MODE, ®, 1); + + switch (reg) { + case SPMI_HFSMPS_MODE_HPM_MASK: + return REGULATOR_MODE_NORMAL; + case SPMI_HFSMPS_MODE_AUTO_MASK: + return REGULATOR_MODE_FAST; + default: + return REGULATOR_MODE_IDLE; + } +} + static int spmi_regulator_common_set_mode(struct regulator_dev *rdev, unsigned int mode) { @@ -1108,6 +1169,33 @@ spmi_regulator_ftsmps426_set_mode(struct regulator_dev *rdev, unsigned int mode) return spmi_vreg_update_bits(vreg, SPMI_COMMON_REG_MODE, val, mask); } +static int +spmi_regulator_hfsmps_set_mode(struct regulator_dev *rdev, unsigned int mode) +{ + struct spmi_regulator *vreg = rdev_get_drvdata(rdev); + u8 mask = SPMI_HFSMPS_MODE_MASK; + u8 val; + + switch (mode) { + case REGULATOR_MODE_NORMAL: + val = SPMI_HFSMPS_MODE_HPM_MASK; + break; + case REGULATOR_MODE_FAST: + val = SPMI_HFSMPS_MODE_AUTO_MASK; + break; + case REGULATOR_MODE_IDLE: + val = vreg->logical_type == + SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3 ? + SPMI_HFSMPS_MODE_RETENTION_MASK : + SPMI_HFSMPS_MODE_LPM_MASK; + break; + default: + return -EINVAL; + } + + return spmi_vreg_update_bits(vreg, SPMI_COMMON_REG_MODE, val, mask); +} + static int spmi_regulator_common_set_load(struct regulator_dev *rdev, int load_uA) { @@ -1131,6 +1219,15 @@ static int spmi_regulator_common_set_pull_down(struct regulator_dev *rdev) mask, mask); } +static int spmi_regulator_hfsmps_set_pull_down(struct regulator_dev *rdev) +{ + struct spmi_regulator *vreg = rdev_get_drvdata(rdev); + unsigned int mask = SPMI_COMMON_PULL_DOWN_ENABLE_MASK; + + return spmi_vreg_update_bits(vreg, SPMI_HFSMPS_REG_PULL_DOWN, + mask, mask); +} + static int spmi_regulator_common_set_soft_start(struct regulator_dev *rdev) { struct spmi_regulator *vreg = rdev_get_drvdata(rdev); @@ -1465,6 +1562,21 @@ static const struct regulator_ops spmi_hfs430_ops = { .get_mode = spmi_regulator_ftsmps426_get_mode, }; +static const struct regulator_ops spmi_hfsmps_ops = { + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .is_enabled = regulator_is_enabled_regmap, + .set_voltage_sel = spmi_regulator_ftsmps426_set_voltage, + .set_voltage_time_sel = spmi_regulator_set_voltage_time_sel, + .get_voltage_sel = spmi_regulator_ftsmps426_get_voltage, + .map_voltage = spmi_regulator_single_map_voltage, + .list_voltage = spmi_regulator_common_list_voltage, + .set_mode = spmi_regulator_hfsmps_set_mode, + .get_mode = spmi_regulator_hfsmps_get_mode, + .set_load = spmi_regulator_common_set_load, + .set_pull_down = spmi_regulator_hfsmps_set_pull_down, +}; + /* Maximum possible digital major revision value */ #define INF 0xFF @@ -1473,7 +1585,8 @@ static const struct spmi_regulator_mapping supported_regulators[] = { SPMI_VREG(LDO, HT_P600, 0, INF, HFS430, hfs430, ht_p600, 10000), SPMI_VREG(LDO, HT_P150, 0, INF, HFS430, hfs430, ht_p150, 10000), SPMI_VREG(BUCK, GP_CTL, 0, INF, SMPS, smps, smps, 100000), - SPMI_VREG(BUCK, HFS430, 0, INF, HFS430, hfs430, hfs430, 10000), + SPMI_VREG(BUCK, HFS430, 0, 3, HFS430, hfs430, hfs430, 10000), + SPMI_VREG(BUCK, HFSMPS_510, 4, INF, HFSMPS, hfsmps, hfs430, 100000), SPMI_VREG(LDO, N300, 0, INF, LDO, ldo, nldo1, 10000), SPMI_VREG(LDO, N600, 0, 0, LDO, ldo, nldo2, 10000), SPMI_VREG(LDO, N1200, 0, 0, LDO, ldo, nldo2, 10000), @@ -1549,6 +1662,16 @@ static const struct spmi_regulator_mapping supported_regulators[] = { SPMI_VREG(ULT_LDO, P300, 0, INF, ULT_LDO, ult_ldo, ult_pldo, 10000), SPMI_VREG(ULT_LDO, P150, 0, INF, ULT_LDO, ult_ldo, ult_pldo, 10000), SPMI_VREG(ULT_LDO, P50, 0, INF, ULT_LDO, ult_ldo, ult_pldo, 5000), + SPMI_VREG(LDO, LV_P150_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000), + SPMI_VREG(LDO, LV_P300_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000), + SPMI_VREG(LDO, LV_P600_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000), + SPMI_VREG(LDO, MV_P50_510, 0, INF, LDO_510, hfsmps, pldo660, 10000), + SPMI_VREG(LDO, MV_P150_510, 0, INF, LDO_510, hfsmps, pldo660, 10000), + SPMI_VREG(LDO, MV_P600_510, 0, INF, LDO_510, hfsmps, pldo660, 10000), + SPMI_VREG(LDO, N300_510, 0, INF, LDO_510, hfsmps, nldo_510, 10000), + SPMI_VREG(LDO, N600_510, 0, INF, LDO_510, hfsmps, nldo_510, 10000), + SPMI_VREG(LDO, N1200_510, 0, INF, LDO_510, hfsmps, nldo_510, 10000), + SPMI_VREG(FTS, FTSMPS_510, 0, INF, FTSMPS3, hfsmps, ftsmps510, 100000), }; static void spmi_calculate_num_voltages(struct spmi_voltage_set_points *points) @@ -1696,6 +1819,26 @@ static int spmi_regulator_init_slew_rate_ftsmps426(struct spmi_regulator *vreg, return ret; } +static int spmi_regulator_init_slew_rate_hfsmps(struct spmi_regulator *vreg) +{ + int ret; + u8 reg = 0; + int delay; + + ret = spmi_vreg_read(vreg, SPMI_HFSMPS_REG_STEP_CTRL, ®, 1); + if (ret) { + dev_err(vreg->dev, "spmi read failed, ret=%d\n", ret); + return ret; + } + + delay = reg & SPMI_FTSMPS426_STEP_CTRL_DELAY_MASK; + delay >>= SPMI_FTSMPS426_STEP_CTRL_DELAY_SHIFT; + + vreg->slew_rate = SPMI_HFSMPS_SLEW_RATE_38p4 >> delay; + + return ret; +} + static int spmi_regulator_init_registers(struct spmi_regulator *vreg, const struct spmi_regulator_init_data *data) { @@ -1846,6 +1989,12 @@ static int spmi_regulator_of_parse(struct device_node *node, if (ret) return ret; break; + case SPMI_REGULATOR_LOGICAL_TYPE_HFSMPS: + case SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3: + ret = spmi_regulator_init_slew_rate_hfsmps(vreg); + if (ret) + return ret; + break; default: break; } @@ -1872,40 +2021,100 @@ static int spmi_regulator_of_parse(struct device_node *node, return 0; } -static const struct spmi_regulator_data pm8941_regulators[] = { +static const struct spmi_regulator_data pm6125_regulators[] = { + { "s1", 0x1400, "vdd_s1" }, + { "s2", 0x1700, "vdd_s2" }, + { "s3", 0x1a00, "vdd_s3" }, + { "s4", 0x1d00, "vdd_s4" }, + { "s5", 0x2000, "vdd_s5" }, + { "s6", 0x2300, "vdd_s6" }, + { "s7", 0x2600, "vdd_s7" }, + { "s8", 0x2900, "vdd_s8" }, + { "l1", 0x4000, "vdd_l1_l7_l17_l18" }, + { "l2", 0x4100, "vdd_l2_l3_l4" }, + { "l3", 0x4200, "vdd_l2_l3_l4" }, + { "l4", 0x4300, "vdd_l2_l3_l4" }, + { "l5", 0x4400, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l6", 0x4500, "vdd_l6_l8" }, + { "l7", 0x4600, "vdd_l1_l7_l17_l18" }, + { "l8", 0x4700, "vdd_l6_l8" }, + { "l9", 0x4800, "vdd_l9_l11" }, + { "l10", 0x4900, "vdd_l10_l13_l14" }, + { "l11", 0x4a00, "vdd_l9_l11" }, + { "l12", 0x4b00, "vdd_l12_l16" }, + { "l13", 0x4c00, "vdd_l10_l13_l14" }, + { "l14", 0x4d00, "vdd_l10_l13_l14" }, + { "l15", 0x4e00, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l16", 0x4f00, "vdd_l12_l16" }, + { "l17", 0x5000, "vdd_l1_l7_l17_l18" }, + { "l18", 0x5100, "vdd_l1_l7_l17_l18" }, + { "l19", 0x5200, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l20", 0x5300, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l21", 0x5400, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l22", 0x5500, "vdd_l5_l15_l19_l20_l21_l22" }, + { "l23", 0x5600, "vdd_l23_l24" }, + { "l24", 0x5700, "vdd_l23_l24" }, +}; + +static const struct spmi_regulator_data pm660_regulators[] = { { "s1", 0x1400, "vdd_s1", }, { "s2", 0x1700, "vdd_s2", }, { "s3", 0x1a00, "vdd_s3", }, - { "s4", 0xa000, }, - { "l1", 0x4000, "vdd_l1_l3", }, - { "l2", 0x4100, "vdd_l2_lvs_1_2_3", }, - { "l3", 0x4200, "vdd_l1_l3", }, - { "l4", 0x4300, "vdd_l4_l11", }, - { "l5", 0x4400, "vdd_l5_l7", NULL, 0x0410 }, - { "l6", 0x4500, "vdd_l6_l12_l14_l15", }, - { "l7", 0x4600, "vdd_l5_l7", NULL, 0x0410 }, - { "l8", 0x4700, "vdd_l8_l16_l18_19", }, - { "l9", 0x4800, "vdd_l9_l10_l17_l22", }, - { "l10", 0x4900, "vdd_l9_l10_l17_l22", }, - { "l11", 0x4a00, "vdd_l4_l11", }, - { "l12", 0x4b00, "vdd_l6_l12_l14_l15", }, - { "l13", 0x4c00, "vdd_l13_l20_l23_l24", }, - { "l14", 0x4d00, "vdd_l6_l12_l14_l15", }, - { "l15", 0x4e00, "vdd_l6_l12_l14_l15", }, - { "l16", 0x4f00, "vdd_l8_l16_l18_19", }, - { "l17", 0x5000, "vdd_l9_l10_l17_l22", }, - { "l18", 0x5100, "vdd_l8_l16_l18_19", }, - { "l19", 0x5200, "vdd_l8_l16_l18_19", }, - { "l20", 0x5300, "vdd_l13_l20_l23_l24", }, - { "l21", 0x5400, "vdd_l21", }, - { "l22", 0x5500, "vdd_l9_l10_l17_l22", }, - { "l23", 0x5600, "vdd_l13_l20_l23_l24", }, - { "l24", 0x5700, "vdd_l13_l20_l23_l24", }, - { "lvs1", 0x8000, "vdd_l2_lvs_1_2_3", }, - { "lvs2", 0x8100, "vdd_l2_lvs_1_2_3", }, - { "lvs3", 0x8200, "vdd_l2_lvs_1_2_3", }, - { "5vs1", 0x8300, "vin_5vs", "ocp-5vs1", }, - { "5vs2", 0x8400, "vin_5vs", "ocp-5vs2", }, + { "s4", 0x1d00, "vdd_s3", }, + { "s5", 0x2000, "vdd_s5", }, + { "s6", 0x2300, "vdd_s6", }, + { "l1", 0x4000, "vdd_l1_l6_l7", }, + { "l2", 0x4100, "vdd_l2_l3", }, + { "l3", 0x4200, "vdd_l2_l3", }, + /* l4 is unaccessible on PM660 */ + { "l5", 0x4400, "vdd_l5", }, + { "l6", 0x4500, "vdd_l1_l6_l7", }, + { "l7", 0x4600, "vdd_l1_l6_l7", }, + { "l8", 0x4700, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l9", 0x4800, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l10", 0x4900, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l11", 0x4a00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l12", 0x4b00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l13", 0x4c00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l14", 0x4d00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, + { "l15", 0x4e00, "vdd_l15_l16_l17_l18_l19", }, + { "l16", 0x4f00, "vdd_l15_l16_l17_l18_l19", }, + { "l17", 0x5000, "vdd_l15_l16_l17_l18_l19", }, + { "l18", 0x5100, "vdd_l15_l16_l17_l18_l19", }, + { "l19", 0x5200, "vdd_l15_l16_l17_l18_l19", }, + { } +}; + +static const struct spmi_regulator_data pm660l_regulators[] = { + { "s1", 0x1400, "vdd_s1", }, + { "s2", 0x1700, "vdd_s2", }, + { "s3", 0x1a00, "vdd_s3", }, + { "s4", 0x1d00, "vdd_s4", }, + { "s5", 0x2000, "vdd_s5", }, + { "l1", 0x4000, "vdd_l1_l9_l10", }, + { "l2", 0x4100, "vdd_l2", }, + { "l3", 0x4200, "vdd_l3_l5_l7_l8", }, + { "l4", 0x4300, "vdd_l4_l6", }, + { "l5", 0x4400, "vdd_l3_l5_l7_l8", }, + { "l6", 0x4500, "vdd_l4_l6", }, + { "l7", 0x4600, "vdd_l3_l5_l7_l8", }, + { "l8", 0x4700, "vdd_l3_l5_l7_l8", }, + { "l9", 0x4800, "vdd_l1_l9_l10", }, + { "l10", 0x4900, "vdd_l1_l9_l10", }, + { } +}; + +static const struct spmi_regulator_data pm8004_regulators[] = { + { "s2", 0x1700, "vdd_s2", }, + { "s5", 0x2000, "vdd_s5", }, + { } +}; + +static const struct spmi_regulator_data pm8005_regulators[] = { + { "s1", 0x1400, "vdd_s1", }, + { "s2", 0x1700, "vdd_s2", }, + { "s3", 0x1a00, "vdd_s3", }, + { "s4", 0x1d00, "vdd_s4", }, { } }; @@ -1985,6 +2194,43 @@ static const struct spmi_regulator_data pm8916_regulators[] = { { } }; +static const struct spmi_regulator_data pm8941_regulators[] = { + { "s1", 0x1400, "vdd_s1", }, + { "s2", 0x1700, "vdd_s2", }, + { "s3", 0x1a00, "vdd_s3", }, + { "s4", 0xa000, }, + { "l1", 0x4000, "vdd_l1_l3", }, + { "l2", 0x4100, "vdd_l2_lvs_1_2_3", }, + { "l3", 0x4200, "vdd_l1_l3", }, + { "l4", 0x4300, "vdd_l4_l11", }, + { "l5", 0x4400, "vdd_l5_l7", NULL, 0x0410 }, + { "l6", 0x4500, "vdd_l6_l12_l14_l15", }, + { "l7", 0x4600, "vdd_l5_l7", NULL, 0x0410 }, + { "l8", 0x4700, "vdd_l8_l16_l18_19", }, + { "l9", 0x4800, "vdd_l9_l10_l17_l22", }, + { "l10", 0x4900, "vdd_l9_l10_l17_l22", }, + { "l11", 0x4a00, "vdd_l4_l11", }, + { "l12", 0x4b00, "vdd_l6_l12_l14_l15", }, + { "l13", 0x4c00, "vdd_l13_l20_l23_l24", }, + { "l14", 0x4d00, "vdd_l6_l12_l14_l15", }, + { "l15", 0x4e00, "vdd_l6_l12_l14_l15", }, + { "l16", 0x4f00, "vdd_l8_l16_l18_19", }, + { "l17", 0x5000, "vdd_l9_l10_l17_l22", }, + { "l18", 0x5100, "vdd_l8_l16_l18_19", }, + { "l19", 0x5200, "vdd_l8_l16_l18_19", }, + { "l20", 0x5300, "vdd_l13_l20_l23_l24", }, + { "l21", 0x5400, "vdd_l21", }, + { "l22", 0x5500, "vdd_l9_l10_l17_l22", }, + { "l23", 0x5600, "vdd_l13_l20_l23_l24", }, + { "l24", 0x5700, "vdd_l13_l20_l23_l24", }, + { "lvs1", 0x8000, "vdd_l2_lvs_1_2_3", }, + { "lvs2", 0x8100, "vdd_l2_lvs_1_2_3", }, + { "lvs3", 0x8200, "vdd_l2_lvs_1_2_3", }, + { "5vs1", 0x8300, "vin_5vs", "ocp-5vs1", }, + { "5vs2", 0x8400, "vin_5vs", "ocp-5vs2", }, + { } +}; + static const struct spmi_regulator_data pm8950_regulators[] = { { "s1", 0x1400, "vdd_s1", }, { "s2", 0x1700, "vdd_s2", }, @@ -2076,69 +2322,6 @@ static const struct spmi_regulator_data pmi8994_regulators[] = { { } }; -static const struct spmi_regulator_data pm660_regulators[] = { - { "s1", 0x1400, "vdd_s1", }, - { "s2", 0x1700, "vdd_s2", }, - { "s3", 0x1a00, "vdd_s3", }, - { "s4", 0x1d00, "vdd_s3", }, - { "s5", 0x2000, "vdd_s5", }, - { "s6", 0x2300, "vdd_s6", }, - { "l1", 0x4000, "vdd_l1_l6_l7", }, - { "l2", 0x4100, "vdd_l2_l3", }, - { "l3", 0x4200, "vdd_l2_l3", }, - /* l4 is unaccessible on PM660 */ - { "l5", 0x4400, "vdd_l5", }, - { "l6", 0x4500, "vdd_l1_l6_l7", }, - { "l7", 0x4600, "vdd_l1_l6_l7", }, - { "l8", 0x4700, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l9", 0x4800, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l10", 0x4900, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l11", 0x4a00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l12", 0x4b00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l13", 0x4c00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l14", 0x4d00, "vdd_l8_l9_l10_l11_l12_l13_l14", }, - { "l15", 0x4e00, "vdd_l15_l16_l17_l18_l19", }, - { "l16", 0x4f00, "vdd_l15_l16_l17_l18_l19", }, - { "l17", 0x5000, "vdd_l15_l16_l17_l18_l19", }, - { "l18", 0x5100, "vdd_l15_l16_l17_l18_l19", }, - { "l19", 0x5200, "vdd_l15_l16_l17_l18_l19", }, - { } -}; - -static const struct spmi_regulator_data pm660l_regulators[] = { - { "s1", 0x1400, "vdd_s1", }, - { "s2", 0x1700, "vdd_s2", }, - { "s3", 0x1a00, "vdd_s3", }, - { "s4", 0x1d00, "vdd_s4", }, - { "s5", 0x2000, "vdd_s5", }, - { "l1", 0x4000, "vdd_l1_l9_l10", }, - { "l2", 0x4100, "vdd_l2", }, - { "l3", 0x4200, "vdd_l3_l5_l7_l8", }, - { "l4", 0x4300, "vdd_l4_l6", }, - { "l5", 0x4400, "vdd_l3_l5_l7_l8", }, - { "l6", 0x4500, "vdd_l4_l6", }, - { "l7", 0x4600, "vdd_l3_l5_l7_l8", }, - { "l8", 0x4700, "vdd_l3_l5_l7_l8", }, - { "l9", 0x4800, "vdd_l1_l9_l10", }, - { "l10", 0x4900, "vdd_l1_l9_l10", }, - { } -}; - - -static const struct spmi_regulator_data pm8004_regulators[] = { - { "s2", 0x1700, "vdd_s2", }, - { "s5", 0x2000, "vdd_s5", }, - { } -}; - -static const struct spmi_regulator_data pm8005_regulators[] = { - { "s1", 0x1400, "vdd_s1", }, - { "s2", 0x1700, "vdd_s2", }, - { "s3", 0x1a00, "vdd_s3", }, - { "s4", 0x1d00, "vdd_s4", }, - { } -}; - static const struct spmi_regulator_data pmp8074_regulators[] = { { "s1", 0x1400, "vdd_s1"}, { "s2", 0x1700, "vdd_s2"}, @@ -2167,6 +2350,9 @@ static const struct spmi_regulator_data pms405_regulators[] = { }; static const struct of_device_id qcom_spmi_regulator_match[] = { + { .compatible = "qcom,pm6125-regulators", .data = &pm6125_regulators }, + { .compatible = "qcom,pm660-regulators", .data = &pm660_regulators }, + { .compatible = "qcom,pm660l-regulators", .data = &pm660l_regulators }, { .compatible = "qcom,pm8004-regulators", .data = &pm8004_regulators }, { .compatible = "qcom,pm8005-regulators", .data = &pm8005_regulators }, { .compatible = "qcom,pm8226-regulators", .data = &pm8226_regulators }, @@ -2176,8 +2362,6 @@ static const struct of_device_id qcom_spmi_regulator_match[] = { { .compatible = "qcom,pm8950-regulators", .data = &pm8950_regulators }, { .compatible = "qcom,pm8994-regulators", .data = &pm8994_regulators }, { .compatible = "qcom,pmi8994-regulators", .data = &pmi8994_regulators }, - { .compatible = "qcom,pm660-regulators", .data = &pm660_regulators }, - { .compatible = "qcom,pm660l-regulators", .data = &pm660l_regulators }, { .compatible = "qcom,pmp8074-regulators", .data = &pmp8074_regulators }, { .compatible = "qcom,pms405-regulators", .data = &pms405_regulators }, { } diff --git a/drivers/regulator/rpi-panel-attiny-regulator.c b/drivers/regulator/rpi-panel-attiny-regulator.c index 105f694a67e681f9cd2557283da7af3824beddc0..308f7972941ba25c0f058cacbbd0c3eb1daccbaa 100644 --- a/drivers/regulator/rpi-panel-attiny-regulator.c +++ b/drivers/regulator/rpi-panel-attiny-regulator.c @@ -381,13 +381,11 @@ error: return ret; } -static int attiny_i2c_remove(struct i2c_client *client) +static void attiny_i2c_remove(struct i2c_client *client) { struct attiny_lcd *state = i2c_get_clientdata(client); mutex_destroy(&state->lock); - - return 0; } static const struct of_device_id attiny_dt_ids[] = { diff --git a/drivers/regulator/ti-abb-regulator.c b/drivers/regulator/ti-abb-regulator.c index ce00db27589ab0b739306512a47d52830781eb12..115345e9fdedcc46fb1c42d4fffbe217584485b2 100644 --- a/drivers/regulator/ti-abb-regulator.c +++ b/drivers/regulator/ti-abb-regulator.c @@ -151,7 +151,7 @@ static inline void ti_abb_clear_txdone(const struct ti_abb *abb) }; /** - * ti_abb_wait_tranx() - waits for ABB tranxdone event + * ti_abb_wait_txdone() - waits for ABB tranxdone event * @dev: device * @abb: pointer to the abb instance * diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c new file mode 100644 index 0000000000000000000000000000000000000000..c484c943e4675d11e25edfe0bf22843e58fea0b8 --- /dev/null +++ b/drivers/regulator/tps65219-regulator.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// tps65219-regulator.c +// +// Regulator driver for TPS65219 PMIC +// +// Copyright (C) 2022 BayLibre Incorporated - https://www.baylibre.com/ +// +// This implementation derived from tps65218 authored by +// "J Keerthy " +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct tps65219_regulator_irq_type { + const char *irq_name; + const char *regulator_name; + const char *event_name; + unsigned long event; +}; + +static struct tps65219_regulator_irq_type tps65219_regulator_irq_types[] = { + { "LDO3_SCG", "LDO3", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "LDO3_OC", "LDO3", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "LDO3_UV", "LDO3", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "LDO4_SCG", "LDO4", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "LDO4_OC", "LDO4", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "LDO4_UV", "LDO4", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "LDO1_SCG", "LDO1", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "LDO1_OC", "LDO1", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "LDO1_UV", "LDO1", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "LDO2_SCG", "LDO2", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "LDO2_OC", "LDO2", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "LDO2_UV", "LDO2", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "BUCK3_SCG", "BUCK3", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "BUCK3_OC", "BUCK3", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK3_NEG_OC", "BUCK3", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK3_UV", "BUCK3", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "BUCK1_SCG", "BUCK1", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "BUCK1_OC", "BUCK1", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK1_NEG_OC", "BUCK1", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK1_UV", "BUCK1", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "BUCK2_SCG", "BUCK2", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT }, + { "BUCK2_OC", "BUCK2", "overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK2_NEG_OC", "BUCK2", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT }, + { "BUCK2_UV", "BUCK2", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE }, + { "BUCK1_RV", "BUCK1", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "BUCK2_RV", "BUCK2", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "BUCK3_RV", "BUCK3", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO1_RV", "LDO1", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO2_RV", "LDO2", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO3_RV", "LDO3", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO4_RV", "LDO4", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "BUCK1_RV_SD", "BUCK1", "residual voltage on shutdown", + REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "BUCK2_RV_SD", "BUCK2", "residual voltage on shutdown", + REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "BUCK3_RV_SD", "BUCK3", "residual voltage on shutdown", + REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO1_RV_SD", "LDO1", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO2_RV_SD", "LDO2", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO3_RV_SD", "LDO3", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "LDO4_RV_SD", "LDO4", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN }, + { "SENSOR_3_WARM", "SENSOR3", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN}, + { "SENSOR_2_WARM", "SENSOR2", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN }, + { "SENSOR_1_WARM", "SENSOR1", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN }, + { "SENSOR_0_WARM", "SENSOR0", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN }, + { "SENSOR_3_HOT", "SENSOR3", "hot temperature", REGULATOR_EVENT_OVER_TEMP}, + { "SENSOR_2_HOT", "SENSOR2", "hot temperature", REGULATOR_EVENT_OVER_TEMP }, + { "SENSOR_1_HOT", "SENSOR1", "hot temperature", REGULATOR_EVENT_OVER_TEMP }, + { "SENSOR_0_HOT", "SENSOR0", "hot temperature", REGULATOR_EVENT_OVER_TEMP }, + { "TIMEOUT", "", "", REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE }, +}; + +struct tps65219_regulator_irq_data { + struct device *dev; + struct tps65219_regulator_irq_type *type; + struct regulator_dev *rdev; +}; + +#define TPS65219_REGULATOR(_name, _of, _id, _type, _ops, _n, _vr, _vm, _er, \ + _em, _cr, _cm, _lr, _nlr, _delay, _fuv, \ + _ct, _ncl, _bpm) \ + { \ + .name = _name, \ + .of_match = _of, \ + .regulators_node = of_match_ptr("regulators"), \ + .supply_name = _of, \ + .id = _id, \ + .ops = &(_ops), \ + .n_voltages = _n, \ + .type = _type, \ + .owner = THIS_MODULE, \ + .vsel_reg = _vr, \ + .vsel_mask = _vm, \ + .csel_reg = _cr, \ + .csel_mask = _cm, \ + .curr_table = _ct, \ + .n_current_limits = _ncl, \ + .enable_reg = _er, \ + .enable_mask = _em, \ + .volt_table = NULL, \ + .linear_ranges = _lr, \ + .n_linear_ranges = _nlr, \ + .ramp_delay = _delay, \ + .fixed_uV = _fuv, \ + .bypass_reg = _vr, \ + .bypass_mask = _bpm, \ + } \ + +static const struct linear_range bucks_ranges[] = { + REGULATOR_LINEAR_RANGE(600000, 0x0, 0x1f, 25000), + REGULATOR_LINEAR_RANGE(1400000, 0x20, 0x33, 100000), + REGULATOR_LINEAR_RANGE(3400000, 0x34, 0x3f, 0), +}; + +static const struct linear_range ldos_1_2_ranges[] = { + REGULATOR_LINEAR_RANGE(600000, 0x0, 0x37, 50000), + REGULATOR_LINEAR_RANGE(3400000, 0x38, 0x3f, 0), +}; + +static const struct linear_range ldos_3_4_ranges[] = { + REGULATOR_LINEAR_RANGE(1200000, 0x0, 0xC, 0), + REGULATOR_LINEAR_RANGE(1250000, 0xD, 0x35, 50000), + REGULATOR_LINEAR_RANGE(3300000, 0x36, 0x3F, 0), +}; + +static int tps65219_set_mode(struct regulator_dev *dev, unsigned int mode) +{ + struct tps65219 *tps = rdev_get_drvdata(dev); + + switch (mode) { + case REGULATOR_MODE_NORMAL: + return regmap_set_bits(tps->regmap, TPS65219_REG_STBY_1_CONFIG, + dev->desc->enable_mask); + + case REGULATOR_MODE_STANDBY: + return regmap_clear_bits(tps->regmap, + TPS65219_REG_STBY_1_CONFIG, + dev->desc->enable_mask); + default: + return -EINVAL; + } +} + +static unsigned int tps65219_get_mode(struct regulator_dev *dev) +{ + struct tps65219 *tps = rdev_get_drvdata(dev); + unsigned int rid = rdev_get_id(dev); + int ret, value = 0; + + ret = regmap_read(tps->regmap, TPS65219_REG_STBY_1_CONFIG, &value); + if (ret) { + dev_dbg(tps->dev, "%s failed for regulator %s: %d ", + __func__, dev->desc->name, ret); + return ret; + } + value = (value & BIT(rid)) >> rid; + if (value) + return REGULATOR_MODE_STANDBY; + else + return REGULATOR_MODE_NORMAL; +} + +/* + * generic regulator_set_bypass_regmap does not fully match requirements + * TPS65219 Requires explicitly that regulator is disabled before switch + */ +static int tps65219_set_bypass(struct regulator_dev *dev, bool enable) +{ + struct tps65219 *tps = rdev_get_drvdata(dev); + unsigned int rid = rdev_get_id(dev); + + if (dev->desc->ops->is_enabled(dev)) { + dev_err(tps->dev, + "%s LDO%d enabled, must be shut down to set bypass ", + __func__, rid); + return -EBUSY; + } + return regulator_set_bypass_regmap(dev, enable); +} + +/* Operations permitted on BUCK1/2/3 */ +static const struct regulator_ops tps65219_bucks_ops = { + .is_enabled = regulator_is_enabled_regmap, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .set_mode = tps65219_set_mode, + .get_mode = tps65219_get_mode, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, + .set_voltage_time_sel = regulator_set_voltage_time_sel, + +}; + +/* Operations permitted on LDO1/2 */ +static const struct regulator_ops tps65219_ldos_1_2_ops = { + .is_enabled = regulator_is_enabled_regmap, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .set_mode = tps65219_set_mode, + .get_mode = tps65219_get_mode, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, + .set_bypass = tps65219_set_bypass, + .get_bypass = regulator_get_bypass_regmap, +}; + +/* Operations permitted on LDO3/4 */ +static const struct regulator_ops tps65219_ldos_3_4_ops = { + .is_enabled = regulator_is_enabled_regmap, + .enable = regulator_enable_regmap, + .disable = regulator_disable_regmap, + .set_mode = tps65219_set_mode, + .get_mode = tps65219_get_mode, + .get_voltage_sel = regulator_get_voltage_sel_regmap, + .set_voltage_sel = regulator_set_voltage_sel_regmap, + .list_voltage = regulator_list_voltage_linear_range, + .map_voltage = regulator_map_voltage_linear_range, +}; + +static const struct regulator_desc regulators[] = { + TPS65219_REGULATOR("BUCK1", "buck1", TPS65219_BUCK_1, + REGULATOR_VOLTAGE, tps65219_bucks_ops, 64, + TPS65219_REG_BUCK1_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_BUCK1_EN_MASK, 0, 0, bucks_ranges, + 3, 4000, 0, NULL, 0, 0), + TPS65219_REGULATOR("BUCK2", "buck2", TPS65219_BUCK_2, + REGULATOR_VOLTAGE, tps65219_bucks_ops, 64, + TPS65219_REG_BUCK2_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_BUCK2_EN_MASK, 0, 0, bucks_ranges, + 3, 4000, 0, NULL, 0, 0), + TPS65219_REGULATOR("BUCK3", "buck3", TPS65219_BUCK_3, + REGULATOR_VOLTAGE, tps65219_bucks_ops, 64, + TPS65219_REG_BUCK3_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_BUCK3_EN_MASK, 0, 0, bucks_ranges, + 3, 0, 0, NULL, 0, 0), + TPS65219_REGULATOR("LDO1", "ldo1", TPS65219_LDO_1, + REGULATOR_VOLTAGE, tps65219_ldos_1_2_ops, 64, + TPS65219_REG_LDO1_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_LDO1_EN_MASK, 0, 0, ldos_1_2_ranges, + 2, 0, 0, NULL, 0, TPS65219_LDOS_BYP_CONFIG_MASK), + TPS65219_REGULATOR("LDO2", "ldo2", TPS65219_LDO_2, + REGULATOR_VOLTAGE, tps65219_ldos_1_2_ops, 64, + TPS65219_REG_LDO2_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_LDO2_EN_MASK, 0, 0, ldos_1_2_ranges, + 2, 0, 0, NULL, 0, TPS65219_LDOS_BYP_CONFIG_MASK), + TPS65219_REGULATOR("LDO3", "ldo3", TPS65219_LDO_3, + REGULATOR_VOLTAGE, tps65219_ldos_3_4_ops, 64, + TPS65219_REG_LDO3_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_LDO3_EN_MASK, 0, 0, ldos_3_4_ranges, + 3, 0, 0, NULL, 0, 0), + TPS65219_REGULATOR("LDO4", "ldo4", TPS65219_LDO_4, + REGULATOR_VOLTAGE, tps65219_ldos_3_4_ops, 64, + TPS65219_REG_LDO4_VOUT, + TPS65219_BUCKS_LDOS_VOUT_VSET_MASK, + TPS65219_REG_ENABLE_CTRL, + TPS65219_ENABLE_LDO4_EN_MASK, 0, 0, ldos_3_4_ranges, + 3, 0, 0, NULL, 0, 0), +}; + +static irqreturn_t tps65219_regulator_irq_handler(int irq, void *data) +{ + struct tps65219_regulator_irq_data *irq_data = data; + + if (irq_data->type->event_name[0] == '\0') { + /* This is the timeout interrupt no specific regulator */ + dev_err(irq_data->dev, + "System was put in shutdown due to timeout during an active or standby transition.\n"); + return IRQ_HANDLED; + } + + regulator_notifier_call_chain(irq_data->rdev, + irq_data->type->event, NULL); + + dev_err(irq_data->dev, "Error IRQ trap %s for %s\n", + irq_data->type->event_name, irq_data->type->regulator_name); + return IRQ_HANDLED; +} + +static int tps65219_get_rdev_by_name(const char *regulator_name, + struct regulator_dev *rdevtbl[7], + struct regulator_dev *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(regulators); i++) { + if (strcmp(regulator_name, regulators[i].name) == 0) { + dev = rdevtbl[i]; + return 0; + } + } + return -EINVAL; +} + +static int tps65219_regulator_probe(struct platform_device *pdev) +{ + struct tps65219 *tps = dev_get_drvdata(pdev->dev.parent); + struct regulator_dev *rdev; + struct regulator_config config = { }; + int i; + int error; + int irq; + struct tps65219_regulator_irq_data *irq_data; + struct tps65219_regulator_irq_type *irq_type; + struct regulator_dev *rdevtbl[7]; + + config.dev = tps->dev; + config.driver_data = tps; + config.regmap = tps->regmap; + + for (i = 0; i < ARRAY_SIZE(regulators); i++) { + dev_dbg(tps->dev, "%s regul i= %d START", __func__, i); + rdev = devm_regulator_register(&pdev->dev, ®ulators[i], + &config); + if (IS_ERR(rdev)) { + dev_err(tps->dev, "failed to register %s regulator\n", + pdev->name); + return PTR_ERR(rdev); + } + rdevtbl[i] = rdev; + dev_dbg(tps->dev, "%s regul i= %d COMPLETED", __func__, i); + } + + irq_data = devm_kmalloc(tps->dev, + ARRAY_SIZE(tps65219_regulator_irq_types) * + sizeof(struct tps65219_regulator_irq_data), + GFP_KERNEL); + if (!irq_data) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(tps65219_regulator_irq_types); ++i) { + irq_type = &tps65219_regulator_irq_types[i]; + + irq = platform_get_irq_byname(pdev, irq_type->irq_name); + if (irq < 0) + return -EINVAL; + + irq_data[i].dev = tps->dev; + irq_data[i].type = irq_type; + + tps65219_get_rdev_by_name(irq_type->regulator_name, rdevtbl, rdev); + if (rdev < 0) { + dev_err(tps->dev, "Failed to get rdev for %s\n", + irq_type->regulator_name); + return -EINVAL; + } + irq_data[i].rdev = rdev; + + error = devm_request_threaded_irq(tps->dev, irq, NULL, + tps65219_regulator_irq_handler, + IRQF_ONESHOT, + irq_type->irq_name, + &irq_data[i]); + if (error) { + dev_err(tps->dev, "failed to request %s IRQ %d: %d\n", + irq_type->irq_name, irq, error); + return error; + } + } + + return 0; +} + +static const struct platform_device_id tps65219_regulator_id_table[] = { + { "tps65219-regulator", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, tps65219_regulator_id_table); + +static struct platform_driver tps65219_regulator_driver = { + .driver = { + .name = "tps65219-pmic", + }, + .probe = tps65219_regulator_probe, + .id_table = tps65219_regulator_id_table, +}; + +module_platform_driver(tps65219_regulator_driver); + +MODULE_AUTHOR("Jerome Neanne "); +MODULE_DESCRIPTION("TPS65219 voltage regulator driver"); +MODULE_ALIAS("platform:tps65219-pmic"); +MODULE_LICENSE("GPL"); diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index ca0817f8e41e9c33a378c9f32aefa05731e4f0e3..899aa8dd12f075e0833e6be4474d1fc9d1817db8 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -599,7 +599,7 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) } /* Register memory region */ - mem = rproc_mem_entry_init(dev, cpu_addr, (dma_addr_t)att->sa, + mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa, att->size, da, NULL, NULL, "dsp_mem"); if (mem) @@ -635,7 +635,7 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) } /* Register memory region */ - mem = rproc_mem_entry_init(dev, cpu_addr, (dma_addr_t)rmem->base, + mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)rmem->base, rmem->size, da, NULL, NULL, it.node->name); if (mem) diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 38383e7de3c1e7861535ef4b053b77b65a845af2..7cc4fd207e2d813a178be99eff1936fdb141d863 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -646,7 +646,6 @@ static int imx_rproc_xtr_mbox_init(struct rproc *rproc) struct imx_rproc *priv = rproc->priv; struct device *dev = priv->dev; struct mbox_client *cl; - int ret; if (!of_get_property(dev->of_node, "mbox-names", NULL)) return 0; @@ -659,18 +658,15 @@ static int imx_rproc_xtr_mbox_init(struct rproc *rproc) cl->rx_callback = imx_rproc_rx_callback; priv->tx_ch = mbox_request_channel_byname(cl, "tx"); - if (IS_ERR(priv->tx_ch)) { - ret = PTR_ERR(priv->tx_ch); - return dev_err_probe(cl->dev, ret, - "failed to request tx mailbox channel: %d\n", ret); - } + if (IS_ERR(priv->tx_ch)) + return dev_err_probe(cl->dev, PTR_ERR(priv->tx_ch), + "failed to request tx mailbox channel\n"); priv->rx_ch = mbox_request_channel_byname(cl, "rx"); if (IS_ERR(priv->rx_ch)) { mbox_free_channel(priv->tx_ch); - ret = PTR_ERR(priv->rx_ch); - return dev_err_probe(cl->dev, ret, - "failed to request rx mailbox channel: %d\n", ret); + return dev_err_probe(cl->dev, PTR_ERR(priv->rx_ch), + "failed to request rx mailbox channel\n"); } return 0; diff --git a/drivers/remoteproc/keystone_remoteproc.c b/drivers/remoteproc/keystone_remoteproc.c index 594a9b43b7ae7c820f61e2f81ceded8ce37a7b4e..95b39741925dbf88a7f7534548ca86b71d2d3eed 100644 --- a/drivers/remoteproc/keystone_remoteproc.c +++ b/drivers/remoteproc/keystone_remoteproc.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -59,10 +59,10 @@ struct keystone_rproc { int num_mems; struct regmap *dev_ctrl; struct reset_control *reset; + struct gpio_desc *kick_gpio; u32 boot_offset; int irq_ring; int irq_fault; - int kick_gpio; struct work_struct workqueue; }; @@ -232,10 +232,10 @@ static void keystone_rproc_kick(struct rproc *rproc, int vqid) { struct keystone_rproc *ksproc = rproc->priv; - if (WARN_ON(ksproc->kick_gpio < 0)) + if (!ksproc->kick_gpio) return; - gpio_set_value(ksproc->kick_gpio, 1); + gpiod_set_value(ksproc->kick_gpio, 1); } /* @@ -432,9 +432,9 @@ static int keystone_rproc_probe(struct platform_device *pdev) goto disable_clk; } - ksproc->kick_gpio = of_get_named_gpio_flags(np, "kick-gpios", 0, NULL); - if (ksproc->kick_gpio < 0) { - ret = ksproc->kick_gpio; + ksproc->kick_gpio = gpiod_get(dev, "kick", GPIOD_ASIS); + ret = PTR_ERR_OR_ZERO(ksproc->kick_gpio); + if (ret) { dev_err(dev, "failed to get gpio for virtio kicks, status = %d\n", ret); goto disable_clk; @@ -466,6 +466,7 @@ static int keystone_rproc_probe(struct platform_device *pdev) release_mem: of_reserved_mem_device_release(dev); + gpiod_put(ksproc->kick_gpio); disable_clk: pm_runtime_put_sync(dev); disable_rpm: @@ -480,6 +481,7 @@ static int keystone_rproc_remove(struct platform_device *pdev) struct keystone_rproc *ksproc = platform_get_drvdata(pdev); rproc_del(ksproc->rproc); + gpiod_put(ksproc->kick_gpio); pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); rproc_free(ksproc->rproc); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index e5279ed9a8d7c7ace6f396795833a0379762177a..8768cb64f560ce1cfc47bc87b33bf4541ea118d0 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -23,9 +23,7 @@ #include #include #include -#include #include -#include /* XXX: pokes into bus_dma_range */ #include #include #include @@ -346,7 +344,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) if (rproc_check_carveout_da(rproc, mem, rsc->vring[i].da, size)) return -ENOMEM; } else { - /* Register carveout in in list */ + /* Register carveout in list */ mem = rproc_mem_entry_init(dev, NULL, 0, size, rsc->vring[i].da, rproc_alloc_carveout, @@ -384,7 +382,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) return 0; } -static int +int rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i) { struct rproc *rproc = rvdev->rproc; @@ -435,57 +433,17 @@ void rproc_free_vring(struct rproc_vring *rvring) } } -static int rproc_vdev_do_start(struct rproc_subdev *subdev) +void rproc_add_rvdev(struct rproc *rproc, struct rproc_vdev *rvdev) { - struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev); - - return rproc_add_virtio_dev(rvdev, rvdev->id); -} - -static void rproc_vdev_do_stop(struct rproc_subdev *subdev, bool crashed) -{ - struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev); - int ret; - - ret = device_for_each_child(&rvdev->dev, NULL, rproc_remove_virtio_dev); - if (ret) - dev_warn(&rvdev->dev, "can't remove vdev child device: %d\n", ret); -} - -/** - * rproc_rvdev_release() - release the existence of a rvdev - * - * @dev: the subdevice's dev - */ -static void rproc_rvdev_release(struct device *dev) -{ - struct rproc_vdev *rvdev = container_of(dev, struct rproc_vdev, dev); - - of_reserved_mem_device_release(dev); - dma_release_coherent_memory(dev); - - kfree(rvdev); + if (rvdev && rproc) + list_add_tail(&rvdev->node, &rproc->rvdevs); } -static int copy_dma_range_map(struct device *to, struct device *from) +void rproc_remove_rvdev(struct rproc_vdev *rvdev) { - const struct bus_dma_region *map = from->dma_range_map, *new_map, *r; - int num_ranges = 0; - - if (!map) - return 0; - - for (r = map; r->size; r++) - num_ranges++; - - new_map = kmemdup(map, array_size(num_ranges + 1, sizeof(*map)), - GFP_KERNEL); - if (!new_map) - return -ENOMEM; - to->dma_range_map = new_map; - return 0; + if (rvdev) + list_del(&rvdev->node); } - /** * rproc_handle_vdev() - handle a vdev fw resource * @rproc: the remote processor @@ -520,12 +478,13 @@ static int rproc_handle_vdev(struct rproc *rproc, void *ptr, struct fw_rsc_vdev *rsc = ptr; struct device *dev = &rproc->dev; struct rproc_vdev *rvdev; - int i, ret; - char name[16]; + size_t rsc_size; + struct rproc_vdev_data rvdev_data; + struct platform_device *pdev; /* make sure resource isn't truncated */ - if (struct_size(rsc, vring, rsc->num_of_vrings) + rsc->config_len > - avail) { + rsc_size = struct_size(rsc, vring, rsc->num_of_vrings); + if (size_add(rsc_size, rsc->config_len) > avail) { dev_err(dev, "vdev rsc is truncated\n"); return -EINVAL; } @@ -545,93 +504,19 @@ static int rproc_handle_vdev(struct rproc *rproc, void *ptr, return -EINVAL; } - rvdev = kzalloc(sizeof(*rvdev), GFP_KERNEL); - if (!rvdev) - return -ENOMEM; - - kref_init(&rvdev->refcount); - - rvdev->id = rsc->id; - rvdev->rproc = rproc; - rvdev->index = rproc->nb_vdev++; + rvdev_data.id = rsc->id; + rvdev_data.index = rproc->nb_vdev++; + rvdev_data.rsc_offset = offset; + rvdev_data.rsc = rsc; - /* Initialise vdev subdevice */ - snprintf(name, sizeof(name), "vdev%dbuffer", rvdev->index); - rvdev->dev.parent = &rproc->dev; - rvdev->dev.release = rproc_rvdev_release; - dev_set_name(&rvdev->dev, "%s#%s", dev_name(rvdev->dev.parent), name); - dev_set_drvdata(&rvdev->dev, rvdev); - - ret = device_register(&rvdev->dev); - if (ret) { - put_device(&rvdev->dev); - return ret; + pdev = platform_device_register_data(dev, "rproc-virtio", rvdev_data.index, &rvdev_data, + sizeof(rvdev_data)); + if (IS_ERR(pdev)) { + dev_err(dev, "failed to create rproc-virtio device\n"); + return PTR_ERR(pdev); } - ret = copy_dma_range_map(&rvdev->dev, rproc->dev.parent); - if (ret) - goto free_rvdev; - - /* Make device dma capable by inheriting from parent's capabilities */ - set_dma_ops(&rvdev->dev, get_dma_ops(rproc->dev.parent)); - - ret = dma_coerce_mask_and_coherent(&rvdev->dev, - dma_get_mask(rproc->dev.parent)); - if (ret) { - dev_warn(dev, - "Failed to set DMA mask %llx. Trying to continue... (%pe)\n", - dma_get_mask(rproc->dev.parent), ERR_PTR(ret)); - } - - /* parse the vrings */ - for (i = 0; i < rsc->num_of_vrings; i++) { - ret = rproc_parse_vring(rvdev, rsc, i); - if (ret) - goto free_rvdev; - } - - /* remember the resource offset*/ - rvdev->rsc_offset = offset; - - /* allocate the vring resources */ - for (i = 0; i < rsc->num_of_vrings; i++) { - ret = rproc_alloc_vring(rvdev, i); - if (ret) - goto unwind_vring_allocations; - } - - list_add_tail(&rvdev->node, &rproc->rvdevs); - - rvdev->subdev.start = rproc_vdev_do_start; - rvdev->subdev.stop = rproc_vdev_do_stop; - - rproc_add_subdev(rproc, &rvdev->subdev); - return 0; - -unwind_vring_allocations: - for (i--; i >= 0; i--) - rproc_free_vring(&rvdev->vring[i]); -free_rvdev: - device_unregister(&rvdev->dev); - return ret; -} - -void rproc_vdev_release(struct kref *ref) -{ - struct rproc_vdev *rvdev = container_of(ref, struct rproc_vdev, refcount); - struct rproc_vring *rvring; - struct rproc *rproc = rvdev->rproc; - int id; - - for (id = 0; id < ARRAY_SIZE(rvdev->vring); id++) { - rvring = &rvdev->vring[id]; - rproc_free_vring(rvring); - } - - rproc_remove_subdev(rproc, &rvdev->subdev); - list_del(&rvdev->node); - device_unregister(&rvdev->dev); } /** @@ -1365,7 +1250,7 @@ void rproc_resource_cleanup(struct rproc *rproc) /* clean up remote vdev entries */ list_for_each_entry_safe(rvdev, rvtmp, &rproc->rvdevs, node) - kref_put(&rvdev->refcount, rproc_vdev_release); + platform_device_unregister(rvdev->pdev); rproc_coredump_cleanup(rproc); } @@ -1885,6 +1770,45 @@ static int __rproc_detach(struct rproc *rproc) return 0; } +static int rproc_attach_recovery(struct rproc *rproc) +{ + int ret; + + ret = __rproc_detach(rproc); + if (ret) + return ret; + + return __rproc_attach(rproc); +} + +static int rproc_boot_recovery(struct rproc *rproc) +{ + const struct firmware *firmware_p; + struct device *dev = &rproc->dev; + int ret; + + ret = rproc_stop(rproc, true); + if (ret) + return ret; + + /* generate coredump */ + rproc->ops->coredump(rproc); + + /* load firmware */ + ret = request_firmware(&firmware_p, rproc->firmware, dev); + if (ret < 0) { + dev_err(dev, "request_firmware failed: %d\n", ret); + return ret; + } + + /* boot the remote processor up again */ + ret = rproc_start(rproc, firmware_p); + + release_firmware(firmware_p); + + return ret; +} + /** * rproc_trigger_recovery() - recover a remoteproc * @rproc: the remote processor @@ -1899,7 +1823,6 @@ static int __rproc_detach(struct rproc *rproc) */ int rproc_trigger_recovery(struct rproc *rproc) { - const struct firmware *firmware_p; struct device *dev = &rproc->dev; int ret; @@ -1913,24 +1836,10 @@ int rproc_trigger_recovery(struct rproc *rproc) dev_err(dev, "recovering %s\n", rproc->name); - ret = rproc_stop(rproc, true); - if (ret) - goto unlock_mutex; - - /* generate coredump */ - rproc->ops->coredump(rproc); - - /* load firmware */ - ret = request_firmware(&firmware_p, rproc->firmware, dev); - if (ret < 0) { - dev_err(dev, "request_firmware failed: %d\n", ret); - goto unlock_mutex; - } - - /* boot the remote processor up again */ - ret = rproc_start(rproc, firmware_p); - - release_firmware(firmware_p); + if (rproc_has_feature(rproc, RPROC_FEAT_ATTACH_ON_RECOVERY)) + ret = rproc_attach_recovery(rproc); + else + ret = rproc_boot_recovery(rproc); unlock_mutex: mutex_unlock(&rproc->lock); diff --git a/drivers/remoteproc/remoteproc_internal.h b/drivers/remoteproc/remoteproc_internal.h index 72d4d3d7d94d173edd0ee4beb17611ad76bb0bb2..d4dbb8d1d80cc9b2735f315b67723f1c13b6e8ee 100644 --- a/drivers/remoteproc/remoteproc_internal.h +++ b/drivers/remoteproc/remoteproc_internal.h @@ -24,16 +24,43 @@ struct rproc_debug_trace { struct rproc_mem_entry trace_mem; }; +/** + * struct rproc_vdev_data - remoteproc virtio device data + * @rsc_offset: offset of the vdev's resource entry + * @id: virtio device id (as in virtio_ids.h) + * @index: vdev position versus other vdev declared in resource table + * @rsc: pointer to the vdev resource entry. Valid only during vdev init as + * the resource can be cached by rproc. + */ +struct rproc_vdev_data { + u32 rsc_offset; + unsigned int id; + u32 index; + struct fw_rsc_vdev *rsc; +}; + +static inline bool rproc_has_feature(struct rproc *rproc, unsigned int feature) +{ + return test_bit(feature, rproc->features); +} + +static inline int rproc_set_feature(struct rproc *rproc, unsigned int feature) +{ + if (feature >= RPROC_MAX_FEATURES) + return -EINVAL; + + set_bit(feature, rproc->features); + + return 0; +} + /* from remoteproc_core.c */ void rproc_release(struct kref *kref); -irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int vq_id); -void rproc_vdev_release(struct kref *ref); int rproc_of_parse_firmware(struct device *dev, int index, const char **fw_name); /* from remoteproc_virtio.c */ -int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id); -int rproc_remove_virtio_dev(struct device *dev, void *data); +irqreturn_t rproc_vq_interrupt(struct rproc *rproc, int vq_id); /* from remoteproc_debugfs.c */ void rproc_remove_trace_file(struct dentry *tfile); @@ -83,6 +110,7 @@ static inline void rproc_char_device_remove(struct rproc *rproc) void rproc_free_vring(struct rproc_vring *rvring); int rproc_alloc_vring(struct rproc_vdev *rvdev, int i); +int rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i); phys_addr_t rproc_va_to_pa(void *cpu_addr); int rproc_trigger_recovery(struct rproc *rproc); @@ -95,6 +123,8 @@ struct resource_table *rproc_elf_find_loaded_rsc_table(struct rproc *rproc, const struct firmware *fw); struct rproc_mem_entry * rproc_find_carveout_by_name(struct rproc *rproc, const char *name, ...); +void rproc_add_rvdev(struct rproc *rproc, struct rproc_vdev *rvdev); +void rproc_remove_rvdev(struct rproc_vdev *rvdev); static inline int rproc_prepare_device(struct rproc *rproc) { diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 0f7706e23eb91f8a578c2db0bea9f56e93351c64..0e95525c1158156cf76b1cb71f78e582ddc74eeb 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -9,9 +9,12 @@ * Brian Swetland */ +#include #include +#include #include #include +#include #include #include #include @@ -23,9 +26,32 @@ #include "remoteproc_internal.h" +static int copy_dma_range_map(struct device *to, struct device *from) +{ + const struct bus_dma_region *map = from->dma_range_map, *new_map, *r; + int num_ranges = 0; + + if (!map) + return 0; + + for (r = map; r->size; r++) + num_ranges++; + + new_map = kmemdup(map, array_size(num_ranges + 1, sizeof(*map)), + GFP_KERNEL); + if (!new_map) + return -ENOMEM; + to->dma_range_map = new_map; + return 0; +} + static struct rproc_vdev *vdev_to_rvdev(struct virtio_device *vdev) { - return container_of(vdev->dev.parent, struct rproc_vdev, dev); + struct platform_device *pdev; + + pdev = container_of(vdev->dev.parent, struct platform_device, dev); + + return platform_get_drvdata(pdev); } static struct rproc *vdev_to_rproc(struct virtio_device *vdev) @@ -322,13 +348,10 @@ static void rproc_virtio_dev_release(struct device *dev) { struct virtio_device *vdev = dev_to_virtio(dev); struct rproc_vdev *rvdev = vdev_to_rvdev(vdev); - struct rproc *rproc = vdev_to_rproc(vdev); kfree(vdev); - kref_put(&rvdev->refcount, rproc_vdev_release); - - put_device(&rproc->dev); + put_device(&rvdev->pdev->dev); } /** @@ -341,10 +364,10 @@ static void rproc_virtio_dev_release(struct device *dev) * * Return: 0 on success or an appropriate error value otherwise */ -int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) +static int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) { struct rproc *rproc = rvdev->rproc; - struct device *dev = &rvdev->dev; + struct device *dev = &rvdev->pdev->dev; struct virtio_device *vdev; struct rproc_mem_entry *mem; int ret; @@ -414,18 +437,8 @@ int rproc_add_virtio_dev(struct rproc_vdev *rvdev, int id) vdev->dev.parent = dev; vdev->dev.release = rproc_virtio_dev_release; - /* - * We're indirectly making a non-temporary copy of the rproc pointer - * here, because drivers probed with this vdev will indirectly - * access the wrapping rproc. - * - * Therefore we must increment the rproc refcount here, and decrement - * it _only_ when the vdev is released. - */ - get_device(&rproc->dev); - /* Reference the vdev and vring allocations */ - kref_get(&rvdev->refcount); + get_device(dev); ret = register_virtio_device(vdev); if (ret) { @@ -449,10 +462,142 @@ out: * * Return: 0 */ -int rproc_remove_virtio_dev(struct device *dev, void *data) +static int rproc_remove_virtio_dev(struct device *dev, void *data) { struct virtio_device *vdev = dev_to_virtio(dev); unregister_virtio_device(vdev); return 0; } + +static int rproc_vdev_do_start(struct rproc_subdev *subdev) +{ + struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev); + + return rproc_add_virtio_dev(rvdev, rvdev->id); +} + +static void rproc_vdev_do_stop(struct rproc_subdev *subdev, bool crashed) +{ + struct rproc_vdev *rvdev = container_of(subdev, struct rproc_vdev, subdev); + struct device *dev = &rvdev->pdev->dev; + int ret; + + ret = device_for_each_child(dev, NULL, rproc_remove_virtio_dev); + if (ret) + dev_warn(dev, "can't remove vdev child device: %d\n", ret); +} + +static int rproc_virtio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rproc_vdev_data *rvdev_data = dev->platform_data; + struct rproc_vdev *rvdev; + struct rproc *rproc = container_of(dev->parent, struct rproc, dev); + struct fw_rsc_vdev *rsc; + int i, ret; + + if (!rvdev_data) + return -EINVAL; + + rvdev = devm_kzalloc(dev, sizeof(*rvdev), GFP_KERNEL); + if (!rvdev) + return -ENOMEM; + + rvdev->id = rvdev_data->id; + rvdev->rproc = rproc; + rvdev->index = rvdev_data->index; + + ret = copy_dma_range_map(dev, rproc->dev.parent); + if (ret) + return ret; + + /* Make device dma capable by inheriting from parent's capabilities */ + set_dma_ops(dev, get_dma_ops(rproc->dev.parent)); + + ret = dma_coerce_mask_and_coherent(dev, dma_get_mask(rproc->dev.parent)); + if (ret) { + dev_warn(dev, "Failed to set DMA mask %llx. Trying to continue... (%pe)\n", + dma_get_mask(rproc->dev.parent), ERR_PTR(ret)); + } + + platform_set_drvdata(pdev, rvdev); + rvdev->pdev = pdev; + + rsc = rvdev_data->rsc; + + /* parse the vrings */ + for (i = 0; i < rsc->num_of_vrings; i++) { + ret = rproc_parse_vring(rvdev, rsc, i); + if (ret) + return ret; + } + + /* remember the resource offset*/ + rvdev->rsc_offset = rvdev_data->rsc_offset; + + /* allocate the vring resources */ + for (i = 0; i < rsc->num_of_vrings; i++) { + ret = rproc_alloc_vring(rvdev, i); + if (ret) + goto unwind_vring_allocations; + } + + rproc_add_rvdev(rproc, rvdev); + + rvdev->subdev.start = rproc_vdev_do_start; + rvdev->subdev.stop = rproc_vdev_do_stop; + + rproc_add_subdev(rproc, &rvdev->subdev); + + /* + * We're indirectly making a non-temporary copy of the rproc pointer + * here, because the platform device or the vdev device will indirectly + * access the wrapping rproc. + * + * Therefore we must increment the rproc refcount here, and decrement + * it _only_ on platform remove. + */ + get_device(&rproc->dev); + + return 0; + +unwind_vring_allocations: + for (i--; i >= 0; i--) + rproc_free_vring(&rvdev->vring[i]); + + return ret; +} + +static int rproc_virtio_remove(struct platform_device *pdev) +{ + struct rproc_vdev *rvdev = dev_get_drvdata(&pdev->dev); + struct rproc *rproc = rvdev->rproc; + struct rproc_vring *rvring; + int id; + + for (id = 0; id < ARRAY_SIZE(rvdev->vring); id++) { + rvring = &rvdev->vring[id]; + rproc_free_vring(rvring); + } + + rproc_remove_subdev(rproc, &rvdev->subdev); + rproc_remove_rvdev(rvdev); + + of_reserved_mem_device_release(&pdev->dev); + dma_release_coherent_memory(&pdev->dev); + + put_device(&rproc->dev); + + return 0; +} + +/* Platform driver */ +static struct platform_driver rproc_virtio_driver = { + .probe = rproc_virtio_probe, + .remove = rproc_virtio_remove, + .driver = { + .name = "rproc-virtio", + }, +}; +builtin_platform_driver(rproc_virtio_driver); diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig index 806773e88832180d9232b3e88f0a9dec90350656..de176c2fbad96d3165fba9172ce71a0531ba8d3b 100644 --- a/drivers/reset/Kconfig +++ b/drivers/reset/Kconfig @@ -152,6 +152,13 @@ config RESET_PISTACHIO help This enables the reset driver for ImgTec Pistachio SoCs. +config RESET_POLARFIRE_SOC + bool "Microchip PolarFire SoC (MPFS) Reset Driver" + depends on AUXILIARY_BUS && MCHP_CLK_MPFS + default MCHP_CLK_MPFS + help + This driver supports peripheral reset for the Microchip PolarFire SoC + config RESET_QCOM_AOSS tristate "Qcom AOSS Reset Driver" depends on ARCH_QCOM || COMPILE_TEST @@ -201,7 +208,7 @@ config RESET_SCMI config RESET_SIMPLE bool "Simple Reset Controller Driver" if COMPILE_TEST || EXPERT - default ARCH_ASPEED || ARCH_BCM4908 || ARCH_BITMAIN || ARCH_REALTEK || ARCH_STM32 || (ARCH_INTEL_SOCFPGA && ARM64) || ARCH_SUNXI || ARC + default ARCH_ASPEED || ARCH_BCMBCA || ARCH_BITMAIN || ARCH_REALTEK || ARCH_STM32 || (ARCH_INTEL_SOCFPGA && ARM64) || ARCH_SUNXI || ARC depends on HAS_IOMEM help This enables a simple reset controller driver for reset lines that diff --git a/drivers/reset/Makefile b/drivers/reset/Makefile index cd5cf8e7c6a74b29b432a8064e58ec10c9796779..3e7e5fd633a8e99787dea9505e495cd65bfbd013 100644 --- a/drivers/reset/Makefile +++ b/drivers/reset/Makefile @@ -22,6 +22,7 @@ obj-$(CONFIG_RESET_MESON_AUDIO_ARB) += reset-meson-audio-arb.o obj-$(CONFIG_RESET_NPCM) += reset-npcm.o obj-$(CONFIG_RESET_OXNAS) += reset-oxnas.o obj-$(CONFIG_RESET_PISTACHIO) += reset-pistachio.o +obj-$(CONFIG_RESET_POLARFIRE_SOC) += reset-mpfs.o obj-$(CONFIG_RESET_QCOM_AOSS) += reset-qcom-aoss.o obj-$(CONFIG_RESET_QCOM_PDC) += reset-qcom-pdc.o obj-$(CONFIG_RESET_RASPBERRYPI) += reset-raspberrypi.o @@ -40,4 +41,3 @@ obj-$(CONFIG_RESET_UNIPHIER) += reset-uniphier.o obj-$(CONFIG_RESET_UNIPHIER_GLUE) += reset-uniphier-glue.o obj-$(CONFIG_RESET_ZYNQ) += reset-zynq.o obj-$(CONFIG_ARCH_ZYNQMP) += reset-zynqmp.o - diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c index 185a333df66c55286cf69b329420db30956f7239..d2408725eb2c3c1376d125ae69650206e5978456 100644 --- a/drivers/reset/reset-imx7.c +++ b/drivers/reset/reset-imx7.c @@ -329,6 +329,7 @@ static int imx8mp_reset_set(struct reset_controller_dev *rcdev, break; case IMX8MP_RESET_PCIE_CTRL_APPS_EN: + case IMX8MP_RESET_PCIEPHY_PERST: value = assert ? 0 : bit; break; } diff --git a/drivers/reset/reset-microchip-sparx5.c b/drivers/reset/reset-microchip-sparx5.c index 00b612a0effa11729c6c71c4165385a769f09cee..f3528dd1d084ea6b8da57120d0b4294011d9696e 100644 --- a/drivers/reset/reset-microchip-sparx5.c +++ b/drivers/reset/reset-microchip-sparx5.c @@ -33,11 +33,8 @@ static struct regmap_config sparx5_reset_regmap_config = { .reg_stride = 4, }; -static int sparx5_switch_reset(struct reset_controller_dev *rcdev, - unsigned long id) +static int sparx5_switch_reset(struct mchp_reset_context *ctx) { - struct mchp_reset_context *ctx = - container_of(rcdev, struct mchp_reset_context, rcdev); u32 val; /* Make sure the core is PROTECTED from reset */ @@ -54,8 +51,14 @@ static int sparx5_switch_reset(struct reset_controller_dev *rcdev, 1, 100); } +static int sparx5_reset_noop(struct reset_controller_dev *rcdev, + unsigned long id) +{ + return 0; +} + static const struct reset_control_ops sparx5_reset_ops = { - .reset = sparx5_switch_reset, + .reset = sparx5_reset_noop, }; static int mchp_sparx5_map_syscon(struct platform_device *pdev, char *name, @@ -122,6 +125,11 @@ static int mchp_sparx5_reset_probe(struct platform_device *pdev) ctx->rcdev.of_node = dn; ctx->props = device_get_match_data(&pdev->dev); + /* Issue the reset very early, our actual reset callback is a noop. */ + err = sparx5_switch_reset(ctx); + if (err) + return err; + return devm_reset_controller_register(&pdev->dev, &ctx->rcdev); } @@ -163,6 +171,10 @@ static int __init mchp_sparx5_reset_init(void) return platform_driver_register(&mchp_sparx5_reset_driver); } +/* + * Because this is a global reset, keep this postcore_initcall() to issue the + * reset as early as possible during the kernel startup. + */ postcore_initcall(mchp_sparx5_reset_init); MODULE_DESCRIPTION("Microchip Sparx5 switch reset driver"); diff --git a/drivers/reset/reset-mpfs.c b/drivers/reset/reset-mpfs.c new file mode 100644 index 0000000000000000000000000000000000000000..e003e50590ec581b050916b312e9318aedff64d3 --- /dev/null +++ b/drivers/reset/reset-mpfs.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PolarFire SoC (MPFS) Peripheral Clock Reset Controller + * + * Author: Conor Dooley + * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries. + * + */ +#include +#include +#include +#include +#include +#include +#include + +/* + * The ENVM reset is the lowest bit in the register & I am using the CLK_FOO + * defines in the dt to make things easier to configure - so this is accounting + * for the offset of 3 there. + */ +#define MPFS_PERIPH_OFFSET CLK_ENVM +#define MPFS_NUM_RESETS 30u +#define MPFS_SLEEP_MIN_US 100 +#define MPFS_SLEEP_MAX_US 200 + +/* block concurrent access to the soft reset register */ +static DEFINE_SPINLOCK(mpfs_reset_lock); + +/* + * Peripheral clock resets + */ + +static int mpfs_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&mpfs_reset_lock, flags); + + reg = mpfs_reset_read(rcdev->dev); + reg |= BIT(id); + mpfs_reset_write(rcdev->dev, reg); + + spin_unlock_irqrestore(&mpfs_reset_lock, flags); + + return 0; +} + +static int mpfs_deassert(struct reset_controller_dev *rcdev, unsigned long id) +{ + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&mpfs_reset_lock, flags); + + reg = mpfs_reset_read(rcdev->dev); + reg &= ~BIT(id); + mpfs_reset_write(rcdev->dev, reg); + + spin_unlock_irqrestore(&mpfs_reset_lock, flags); + + return 0; +} + +static int mpfs_status(struct reset_controller_dev *rcdev, unsigned long id) +{ + u32 reg = mpfs_reset_read(rcdev->dev); + + /* + * It is safe to return here as MPFS_NUM_RESETS makes sure the sign bit + * is never hit. + */ + return (reg & BIT(id)); +} + +static int mpfs_reset(struct reset_controller_dev *rcdev, unsigned long id) +{ + mpfs_assert(rcdev, id); + + usleep_range(MPFS_SLEEP_MIN_US, MPFS_SLEEP_MAX_US); + + mpfs_deassert(rcdev, id); + + return 0; +} + +static const struct reset_control_ops mpfs_reset_ops = { + .reset = mpfs_reset, + .assert = mpfs_assert, + .deassert = mpfs_deassert, + .status = mpfs_status, +}; + +static int mpfs_reset_xlate(struct reset_controller_dev *rcdev, + const struct of_phandle_args *reset_spec) +{ + unsigned int index = reset_spec->args[0]; + + /* + * CLK_RESERVED does not map to a clock, but it does map to a reset, + * so it has to be accounted for here. It is the reset for the fabric, + * so if this reset gets called - do not reset it. + */ + if (index == CLK_RESERVED) { + dev_err(rcdev->dev, "Resetting the fabric is not supported\n"); + return -EINVAL; + } + + if (index < MPFS_PERIPH_OFFSET || index >= (MPFS_PERIPH_OFFSET + rcdev->nr_resets)) { + dev_err(rcdev->dev, "Invalid reset index %u\n", index); + return -EINVAL; + } + + return index - MPFS_PERIPH_OFFSET; +} + +static int mpfs_reset_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct device *dev = &adev->dev; + struct reset_controller_dev *rcdev; + + rcdev = devm_kzalloc(dev, sizeof(*rcdev), GFP_KERNEL); + if (!rcdev) + return -ENOMEM; + + rcdev->dev = dev; + rcdev->dev->parent = dev->parent; + rcdev->ops = &mpfs_reset_ops; + rcdev->of_node = dev->parent->of_node; + rcdev->of_reset_n_cells = 1; + rcdev->of_xlate = mpfs_reset_xlate; + rcdev->nr_resets = MPFS_NUM_RESETS; + + return devm_reset_controller_register(dev, rcdev); +} + +static const struct auxiliary_device_id mpfs_reset_ids[] = { + { + .name = "clk_mpfs.reset-mpfs", + }, + { } +}; +MODULE_DEVICE_TABLE(auxiliary, mpfs_reset_ids); + +static struct auxiliary_driver mpfs_reset_driver = { + .probe = mpfs_reset_probe, + .id_table = mpfs_reset_ids, +}; + +module_auxiliary_driver(mpfs_reset_driver); + +MODULE_DESCRIPTION("Microchip PolarFire SoC Reset Driver"); +MODULE_AUTHOR("Conor Dooley "); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(MCHP_CLK_MPFS); diff --git a/drivers/reset/reset-npcm.c b/drivers/reset/reset-npcm.c index 24c55efa98e556c4e81fbd227b383c2f85de679c..f2333506b0a670de941d50e278269459b69ca1f4 100644 --- a/drivers/reset/reset-npcm.c +++ b/drivers/reset/reset-npcm.c @@ -291,7 +291,7 @@ static void npcm_usb_reset_npcm8xx(struct npcm_rc_data *rc) iprst2 |= ipsrst2_bits; iprst3 |= (ipsrst3_bits | NPCM_IPSRST3_USBPHY1 | NPCM_IPSRST3_USBPHY2); - iprst2 |= ipsrst4_bits; + iprst4 |= ipsrst4_bits; writel(iprst1, rc->base + NPCM_IPSRST1); writel(iprst2, rc->base + NPCM_IPSRST2); diff --git a/drivers/rpmsg/rpmsg_char.c b/drivers/rpmsg/rpmsg_char.c index 4f2189111494a4f8f8913a83db2a1a1cd3f1de80..3e0b8f3496ed806990ecaeeb0b232dab2355bc67 100644 --- a/drivers/rpmsg/rpmsg_char.c +++ b/drivers/rpmsg/rpmsg_char.c @@ -76,7 +76,9 @@ int rpmsg_chrdev_eptdev_destroy(struct device *dev, void *data) mutex_lock(&eptdev->ept_lock); if (eptdev->ept) { - rpmsg_destroy_ept(eptdev->ept); + /* The default endpoint is released by the rpmsg core */ + if (!eptdev->default_ept) + rpmsg_destroy_ept(eptdev->ept); eptdev->ept = NULL; } mutex_unlock(&eptdev->ept_lock); @@ -424,15 +426,12 @@ int rpmsg_chrdev_eptdev_create(struct rpmsg_device *rpdev, struct device *parent struct rpmsg_channel_info chinfo) { struct rpmsg_eptdev *eptdev; - int ret; eptdev = rpmsg_chrdev_eptdev_alloc(rpdev, parent); if (IS_ERR(eptdev)) return PTR_ERR(eptdev); - ret = rpmsg_chrdev_eptdev_add(eptdev, chinfo); - - return ret; + return rpmsg_chrdev_eptdev_add(eptdev, chinfo); } EXPORT_SYMBOL(rpmsg_chrdev_eptdev_create); diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index b8de25118ad09df4e3e40b9d40fe0df413ecd919..bb63edb507da4b7b255b33744444cf219f10054c 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -423,6 +423,7 @@ config RTC_DRV_ISL1208 config RTC_DRV_ISL12022 tristate "Intersil ISL12022" + select REGMAP_I2C help If you say yes here you get support for the Intersil ISL12022 RTC chip. diff --git a/drivers/rtc/rtc-bq32k.c b/drivers/rtc/rtc-bq32k.c index e0bbb11d912e3edd8ae11b7301020ce3c33654b4..6d6a55efb9cc7ae6e14d872564f6c36d2ff49049 100644 --- a/drivers/rtc/rtc-bq32k.c +++ b/drivers/rtc/rtc-bq32k.c @@ -297,11 +297,9 @@ static int bq32k_probe(struct i2c_client *client) return 0; } -static int bq32k_remove(struct i2c_client *client) +static void bq32k_remove(struct i2c_client *client) { bq32k_sysfs_unregister(&client->dev); - - return 0; } static const struct i2c_device_id bq32k_id[] = { diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index bdb1df843c78d074bb182bb6552ea2da6084886b..610413b4e9ca793618874881dfe2b27b9d23666c 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -1352,10 +1352,10 @@ static void cmos_check_acpi_rtc_status(struct device *dev, static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) { - cmos_wake_setup(&pnp->dev); + int irq, ret; if (pnp_port_start(pnp, 0) == 0x70 && !pnp_irq_valid(pnp, 0)) { - unsigned int irq = 0; + irq = 0; #ifdef CONFIG_X86 /* Some machines contain a PNP entry for the RTC, but * don't define the IRQ. It should always be safe to @@ -1364,13 +1364,17 @@ static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) if (nr_legacy_irqs()) irq = RTC_IRQ; #endif - return cmos_do_probe(&pnp->dev, - pnp_get_resource(pnp, IORESOURCE_IO, 0), irq); } else { - return cmos_do_probe(&pnp->dev, - pnp_get_resource(pnp, IORESOURCE_IO, 0), - pnp_irq(pnp, 0)); + irq = pnp_irq(pnp, 0); } + + ret = cmos_do_probe(&pnp->dev, pnp_get_resource(pnp, IORESOURCE_IO, 0), irq); + if (ret) + return ret; + + cmos_wake_setup(&pnp->dev); + + return 0; } static void cmos_pnp_remove(struct pnp_dev *pnp) @@ -1454,10 +1458,9 @@ static inline void cmos_of_init(struct platform_device *pdev) {} static int __init cmos_platform_probe(struct platform_device *pdev) { struct resource *resource; - int irq; + int irq, ret; cmos_of_init(pdev); - cmos_wake_setup(&pdev->dev); if (RTC_IOMAPPED) resource = platform_get_resource(pdev, IORESOURCE_IO, 0); @@ -1467,7 +1470,13 @@ static int __init cmos_platform_probe(struct platform_device *pdev) if (irq < 0) irq = -1; - return cmos_do_probe(&pdev->dev, resource, irq); + ret = cmos_do_probe(&pdev->dev, resource, irq); + if (ret) + return ret; + + cmos_wake_setup(&pdev->dev); + + return 0; } static int cmos_platform_remove(struct platform_device *pdev) diff --git a/drivers/rtc/rtc-ds1374.c b/drivers/rtc/rtc-ds1374.c index b19de5100b1a328750193fd7fda5a3adb99d60a5..7f089f0661634624ba4958c82aa0eed92338a712 100644 --- a/drivers/rtc/rtc-ds1374.c +++ b/drivers/rtc/rtc-ds1374.c @@ -530,7 +530,7 @@ static int ds1374_probe(struct i2c_client *client) return 0; } -static int ds1374_remove(struct i2c_client *client) +static void ds1374_remove(struct i2c_client *client) { struct ds1374 *ds1374 = i2c_get_clientdata(client); @@ -542,8 +542,6 @@ static int ds1374_remove(struct i2c_client *client) devm_free_irq(&client->dev, client->irq, client); cancel_work_sync(&ds1374->work); } - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c index a24331ba8a5fc9bbce488b3e0f2810a6a6f90c2b..5db9c737c022f252d629e7335133351f6bb57471 100644 --- a/drivers/rtc/rtc-ds1685.c +++ b/drivers/rtc/rtc-ds1685.c @@ -132,7 +132,7 @@ ds1685_rtc_bin2bcd(struct ds1685_priv *rtc, u8 val, u8 bin_mask, u8 bcd_mask) } /** - * s1685_rtc_check_mday - check validity of the day of month. + * ds1685_rtc_check_mday - check validity of the day of month. * @rtc: pointer to the ds1685 rtc structure. * @mday: day of month. * diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index c2717bb52b2bee9e5996c543ba2212f657bc09d2..c828bc8e05b9cc48eebdf05343e2eb1b2e62dd8c 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -265,18 +265,17 @@ static int gamecube_rtc_read_offset_from_sram(struct priv *d) * SRAM address as on previous consoles. */ ret = regmap_read(d->regmap, RTC_SRAM_BIAS, &d->rtc_bias); - if (ret) { - pr_err("failed to get the RTC bias\n"); - iounmap(hw_srnprot); - return -1; - } /* Reset SRAM access to how it was before, our job here is done. */ if (old != 0x7bf) iowrite32be(old, hw_srnprot); + iounmap(hw_srnprot); - return 0; + if (ret) + pr_err("failed to get the RTC bias\n"); + + return ret; } static const struct regmap_range rtc_rd_ranges[] = { diff --git a/drivers/rtc/rtc-isl12022.c b/drivers/rtc/rtc-isl12022.c index 79461ded1a4868cfcc4b2e2c7ebc960ebc8523a0..ca677c4265e6c56da41b08b31caf8931f604d81b 100644 --- a/drivers/rtc/rtc-isl12022.c +++ b/drivers/rtc/rtc-isl12022.c @@ -16,6 +16,7 @@ #include #include #include +#include /* ISL register offsets */ #define ISL12022_REG_SC 0x00 @@ -42,83 +43,32 @@ static struct i2c_driver isl12022_driver; struct isl12022 { struct rtc_device *rtc; - - bool write_enabled; /* true if write enable is set */ + struct regmap *regmap; }; - -static int isl12022_read_regs(struct i2c_client *client, uint8_t reg, - uint8_t *data, size_t n) -{ - struct i2c_msg msgs[] = { - { - .addr = client->addr, - .flags = 0, - .len = 1, - .buf = data - }, /* setup read ptr */ - { - .addr = client->addr, - .flags = I2C_M_RD, - .len = n, - .buf = data - } - }; - - int ret; - - data[0] = reg; - ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs)); - if (ret != ARRAY_SIZE(msgs)) { - dev_err(&client->dev, "%s: read error, ret=%d\n", - __func__, ret); - return -EIO; - } - - return 0; -} - - -static int isl12022_write_reg(struct i2c_client *client, - uint8_t reg, uint8_t val) -{ - uint8_t data[2] = { reg, val }; - int err; - - err = i2c_master_send(client, data, sizeof(data)); - if (err != sizeof(data)) { - dev_err(&client->dev, - "%s: err=%d addr=%02x, data=%02x\n", - __func__, err, data[0], data[1]); - return -EIO; - } - - return 0; -} - - /* * In the routines that deal directly with the isl12022 hardware, we use * rtc_time -- month 0-11, hour 0-23, yr = calendar year-epoch. */ static int isl12022_rtc_read_time(struct device *dev, struct rtc_time *tm) { - struct i2c_client *client = to_i2c_client(dev); + struct isl12022 *isl12022 = dev_get_drvdata(dev); + struct regmap *regmap = isl12022->regmap; uint8_t buf[ISL12022_REG_INT + 1]; int ret; - ret = isl12022_read_regs(client, ISL12022_REG_SC, buf, sizeof(buf)); + ret = regmap_bulk_read(regmap, ISL12022_REG_SC, buf, sizeof(buf)); if (ret) return ret; if (buf[ISL12022_REG_SR] & (ISL12022_SR_LBAT85 | ISL12022_SR_LBAT75)) { - dev_warn(&client->dev, + dev_warn(dev, "voltage dropped below %u%%, " "date and time is not reliable.\n", buf[ISL12022_REG_SR] & ISL12022_SR_LBAT85 ? 85 : 75); } - dev_dbg(&client->dev, + dev_dbg(dev, "%s: raw data is sec=%02x, min=%02x, hr=%02x, " "mday=%02x, mon=%02x, year=%02x, wday=%02x, " "sr=%02x, int=%02x", @@ -141,65 +91,25 @@ static int isl12022_rtc_read_time(struct device *dev, struct rtc_time *tm) tm->tm_mon = bcd2bin(buf[ISL12022_REG_MO] & 0x1F) - 1; tm->tm_year = bcd2bin(buf[ISL12022_REG_YR]) + 100; - dev_dbg(&client->dev, "%s: secs=%d, mins=%d, hours=%d, " - "mday=%d, mon=%d, year=%d, wday=%d\n", - __func__, - tm->tm_sec, tm->tm_min, tm->tm_hour, - tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday); + dev_dbg(dev, "%s: %ptR\n", __func__, tm); return 0; } static int isl12022_rtc_set_time(struct device *dev, struct rtc_time *tm) { - struct i2c_client *client = to_i2c_client(dev); - struct isl12022 *isl12022 = i2c_get_clientdata(client); - size_t i; + struct isl12022 *isl12022 = dev_get_drvdata(dev); + struct regmap *regmap = isl12022->regmap; int ret; uint8_t buf[ISL12022_REG_DW + 1]; - dev_dbg(&client->dev, "%s: secs=%d, mins=%d, hours=%d, " - "mday=%d, mon=%d, year=%d, wday=%d\n", - __func__, - tm->tm_sec, tm->tm_min, tm->tm_hour, - tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday); - - if (!isl12022->write_enabled) { - - ret = isl12022_read_regs(client, ISL12022_REG_INT, buf, 1); - if (ret) - return ret; - - /* Check if WRTC (write rtc enable) is set factory default is - * 0 (not set) */ - if (!(buf[0] & ISL12022_INT_WRTC)) { - dev_info(&client->dev, - "init write enable and 24 hour format\n"); - - /* Set the write enable bit. */ - ret = isl12022_write_reg(client, - ISL12022_REG_INT, - buf[0] | ISL12022_INT_WRTC); - if (ret) - return ret; - - /* Write to any RTC register to start RTC, we use the - * HR register, setting the MIL bit to use the 24 hour - * format. */ - ret = isl12022_read_regs(client, ISL12022_REG_HR, - buf, 1); - if (ret) - return ret; - - ret = isl12022_write_reg(client, - ISL12022_REG_HR, - buf[0] | ISL12022_HR_MIL); - if (ret) - return ret; - } - - isl12022->write_enabled = true; - } + dev_dbg(dev, "%s: %ptR\n", __func__, tm); + + /* Ensure the write enable bit is set. */ + ret = regmap_update_bits(regmap, ISL12022_REG_INT, + ISL12022_INT_WRTC, ISL12022_INT_WRTC); + if (ret) + return ret; /* hours, minutes and seconds */ buf[ISL12022_REG_SC] = bin2bcd(tm->tm_sec); @@ -216,15 +126,8 @@ static int isl12022_rtc_set_time(struct device *dev, struct rtc_time *tm) buf[ISL12022_REG_DW] = tm->tm_wday & 0x07; - /* write register's data */ - for (i = 0; i < ARRAY_SIZE(buf); i++) { - ret = isl12022_write_reg(client, ISL12022_REG_SC + i, - buf[ISL12022_REG_SC + i]); - if (ret) - return -EIO; - } - - return 0; + return regmap_bulk_write(isl12022->regmap, ISL12022_REG_SC, + buf, sizeof(buf)); } static const struct rtc_class_ops isl12022_rtc_ops = { @@ -232,6 +135,12 @@ static const struct rtc_class_ops isl12022_rtc_ops = { .set_time = isl12022_rtc_set_time, }; +static const struct regmap_config regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .use_single_write = true, +}; + static int isl12022_probe(struct i2c_client *client) { struct isl12022 *isl12022; @@ -243,13 +152,23 @@ static int isl12022_probe(struct i2c_client *client) GFP_KERNEL); if (!isl12022) return -ENOMEM; + dev_set_drvdata(&client->dev, isl12022); + + isl12022->regmap = devm_regmap_init_i2c(client, ®map_config); + if (IS_ERR(isl12022->regmap)) { + dev_err(&client->dev, "regmap allocation failed\n"); + return PTR_ERR(isl12022->regmap); + } + + isl12022->rtc = devm_rtc_allocate_device(&client->dev); + if (IS_ERR(isl12022->rtc)) + return PTR_ERR(isl12022->rtc); - i2c_set_clientdata(client, isl12022); + isl12022->rtc->ops = &isl12022_rtc_ops; + isl12022->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; + isl12022->rtc->range_max = RTC_TIMESTAMP_END_2099; - isl12022->rtc = devm_rtc_device_register(&client->dev, - isl12022_driver.driver.name, - &isl12022_rtc_ops, THIS_MODULE); - return PTR_ERR_OR_ZERO(isl12022->rtc); + return devm_rtc_register_device(isl12022->rtc); } #ifdef CONFIG_OF diff --git a/drivers/rtc/rtc-isl12026.c b/drivers/rtc/rtc-isl12026.c index 1fc6627d854d375ba0b57ef38691612eb94ef98e..1bfca39079d40efbe71b60cb7da3cfa2424589d5 100644 --- a/drivers/rtc/rtc-isl12026.c +++ b/drivers/rtc/rtc-isl12026.c @@ -472,12 +472,11 @@ static int isl12026_probe_new(struct i2c_client *client) return devm_rtc_register_device(priv->rtc); } -static int isl12026_remove(struct i2c_client *client) +static void isl12026_remove(struct i2c_client *client) { struct isl12026 *priv = i2c_get_clientdata(client); i2c_unregister_device(priv->nvm_client); - return 0; } static const struct of_device_id isl12026_dt_match[] = { diff --git a/drivers/rtc/rtc-jz4740.c b/drivers/rtc/rtc-jz4740.c index 6e51df72fd658147f5e3b033ea1c331962492809..c383719292c7d17c4d38b03a227d93fdb4f1438c 100644 --- a/drivers/rtc/rtc-jz4740.c +++ b/drivers/rtc/rtc-jz4740.c @@ -257,11 +257,6 @@ static void jz4740_rtc_power_off(void) kernel_halt(); } -static void jz4740_rtc_clk_disable(void *data) -{ - clk_disable_unprepare(data); -} - static const struct of_device_id jz4740_rtc_of_match[] = { { .compatible = "ingenic,jz4740-rtc", .data = (void *)ID_JZ4740 }, { .compatible = "ingenic,jz4760-rtc", .data = (void *)ID_JZ4760 }, @@ -329,23 +324,9 @@ static int jz4740_rtc_probe(struct platform_device *pdev) if (IS_ERR(rtc->base)) return PTR_ERR(rtc->base); - clk = devm_clk_get(dev, "rtc"); - if (IS_ERR(clk)) { - dev_err(dev, "Failed to get RTC clock\n"); - return PTR_ERR(clk); - } - - ret = clk_prepare_enable(clk); - if (ret) { - dev_err(dev, "Failed to enable clock\n"); - return ret; - } - - ret = devm_add_action_or_reset(dev, jz4740_rtc_clk_disable, clk); - if (ret) { - dev_err(dev, "Failed to register devm action\n"); - return ret; - } + clk = devm_clk_get_enabled(dev, "rtc"); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "Failed to get RTC clock\n"); spin_lock_init(&rtc->lock); diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c index d868458cd40ed05c200cfe456267df385a5886a5..e0b4d3794320bf5b26113f2e2d02180d75843dd2 100644 --- a/drivers/rtc/rtc-m41t80.c +++ b/drivers/rtc/rtc-m41t80.c @@ -989,7 +989,7 @@ static int m41t80_probe(struct i2c_client *client, return 0; } -static int m41t80_remove(struct i2c_client *client) +static void m41t80_remove(struct i2c_client *client) { #ifdef CONFIG_RTC_DRV_M41T80_WDT struct m41t80_data *clientdata = i2c_get_clientdata(client); @@ -999,8 +999,6 @@ static int m41t80_remove(struct i2c_client *client) unregister_reboot_notifier(&wdt_notifier); } #endif - - return 0; } static struct i2c_driver m41t80_driver = { diff --git a/drivers/rtc/rtc-mpfs.c b/drivers/rtc/rtc-mpfs.c index f14d1925e0c94dfbc48a6eff84a2f63d47e4cf9b..2a479d44f1981e282bb422bdc07f4c4d89bc93f7 100644 --- a/drivers/rtc/rtc-mpfs.c +++ b/drivers/rtc/rtc-mpfs.c @@ -193,23 +193,6 @@ static int mpfs_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled) return 0; } -static inline struct clk *mpfs_rtc_init_clk(struct device *dev) -{ - struct clk *clk; - int ret; - - clk = devm_clk_get(dev, "rtc"); - if (IS_ERR(clk)) - return clk; - - ret = clk_prepare_enable(clk); - if (ret) - return ERR_PTR(ret); - - devm_add_action_or_reset(dev, (void (*) (void *))clk_disable_unprepare, clk); - return clk; -} - static irqreturn_t mpfs_rtc_wakeup_irq_handler(int irq, void *dev) { struct mpfs_rtc_dev *rtcdev = dev; @@ -233,7 +216,7 @@ static int mpfs_rtc_probe(struct platform_device *pdev) { struct mpfs_rtc_dev *rtcdev; struct clk *clk; - u32 prescaler; + unsigned long prescaler; int wakeup_irq, ret; rtcdev = devm_kzalloc(&pdev->dev, sizeof(struct mpfs_rtc_dev), GFP_KERNEL); @@ -251,7 +234,7 @@ static int mpfs_rtc_probe(struct platform_device *pdev) /* range is capped by alarm max, lower reg is 31:0 & upper is 10:0 */ rtcdev->rtc->range_max = GENMASK_ULL(42, 0); - clk = mpfs_rtc_init_clk(&pdev->dev); + clk = devm_clk_get_enabled(&pdev->dev, "rtc"); if (IS_ERR(clk)) return PTR_ERR(clk); @@ -275,14 +258,13 @@ static int mpfs_rtc_probe(struct platform_device *pdev) /* prescaler hardware adds 1 to reg value */ prescaler = clk_get_rate(devm_clk_get(&pdev->dev, "rtcref")) - 1; - if (prescaler > MAX_PRESCALER_COUNT) { - dev_dbg(&pdev->dev, "invalid prescaler %d\n", prescaler); + dev_dbg(&pdev->dev, "invalid prescaler %lu\n", prescaler); return -EINVAL; } writel(prescaler, rtcdev->base + PRESCALER_REG); - dev_info(&pdev->dev, "prescaler set to: 0x%X \r\n", prescaler); + dev_info(&pdev->dev, "prescaler set to: %lu\n", prescaler); device_init_wakeup(&pdev->dev, true); ret = dev_pm_set_wake_irq(&pdev->dev, wakeup_irq); diff --git a/drivers/rtc/rtc-mxc.c b/drivers/rtc/rtc-mxc.c index 53d4e253e81f07216136482dc189edaec6e7f330..762cf03345f1450164b633a59cdefeb29c1420c2 100644 --- a/drivers/rtc/rtc-mxc.c +++ b/drivers/rtc/rtc-mxc.c @@ -291,14 +291,6 @@ static const struct rtc_class_ops mxc_rtc_ops = { .alarm_irq_enable = mxc_rtc_alarm_irq_enable, }; -static void mxc_rtc_action(void *p) -{ - struct rtc_plat_data *pdata = p; - - clk_disable_unprepare(pdata->clk_ref); - clk_disable_unprepare(pdata->clk_ipg); -} - static int mxc_rtc_probe(struct platform_device *pdev) { struct rtc_device *rtc; @@ -341,33 +333,18 @@ static int mxc_rtc_probe(struct platform_device *pdev) rtc->range_max = (1 << 16) * 86400ULL - 1; } - pdata->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); + pdata->clk_ipg = devm_clk_get_enabled(&pdev->dev, "ipg"); if (IS_ERR(pdata->clk_ipg)) { dev_err(&pdev->dev, "unable to get ipg clock!\n"); return PTR_ERR(pdata->clk_ipg); } - ret = clk_prepare_enable(pdata->clk_ipg); - if (ret) - return ret; - - pdata->clk_ref = devm_clk_get(&pdev->dev, "ref"); + pdata->clk_ref = devm_clk_get_enabled(&pdev->dev, "ref"); if (IS_ERR(pdata->clk_ref)) { - clk_disable_unprepare(pdata->clk_ipg); dev_err(&pdev->dev, "unable to get ref clock!\n"); return PTR_ERR(pdata->clk_ref); } - ret = clk_prepare_enable(pdata->clk_ref); - if (ret) { - clk_disable_unprepare(pdata->clk_ipg); - return ret; - } - - ret = devm_add_action_or_reset(&pdev->dev, mxc_rtc_action, pdata); - if (ret) - return ret; - rate = clk_get_rate(pdata->clk_ref); if (rate == 32768) diff --git a/drivers/rtc/rtc-rs5c372.c b/drivers/rtc/rtc-rs5c372.c index cb15983383f55fb47bc96588d79727f5cb47f1cf..9562c477e1c96d2362e57178889bfac1b965686e 100644 --- a/drivers/rtc/rtc-rs5c372.c +++ b/drivers/rtc/rtc-rs5c372.c @@ -910,10 +910,9 @@ exit: return err; } -static int rs5c372_remove(struct i2c_client *client) +static void rs5c372_remove(struct i2c_client *client) { rs5c_sysfs_unregister(&client->dev); - return 0; } static struct i2c_driver rs5c372_driver = { diff --git a/drivers/rtc/rtc-rv3028.c b/drivers/rtc/rtc-rv3028.c index cdc623b3e365bb1b21f6ba477947ff35a54d99cb..dd170e3efd83ed6da15ca91feefc2e39829df923 100644 --- a/drivers/rtc/rtc-rv3028.c +++ b/drivers/rtc/rtc-rv3028.c @@ -521,10 +521,9 @@ static int rv3028_param_get(struct device *dev, struct rtc_param *param) { struct rv3028_data *rv3028 = dev_get_drvdata(dev); int ret; + u32 value; switch(param->param) { - u32 value; - case RTC_PARAM_BACKUP_SWITCH_MODE: ret = regmap_read(rv3028->regmap, RV3028_BACKUP, &value); if (ret < 0) @@ -554,9 +553,9 @@ static int rv3028_param_get(struct device *dev, struct rtc_param *param) static int rv3028_param_set(struct device *dev, struct rtc_param *param) { struct rv3028_data *rv3028 = dev_get_drvdata(dev); + u8 mode; switch(param->param) { - u8 mode; case RTC_PARAM_BACKUP_SWITCH_MODE: switch (param->uvalue) { case RTC_BSM_DISABLED: diff --git a/drivers/rtc/rtc-stmp3xxx.c b/drivers/rtc/rtc-stmp3xxx.c index 40c0f7ed36e0685e489d07a16706e81db3de2c8f..aae40d20d086802fa23c13dd76c09d56db625f9f 100644 --- a/drivers/rtc/rtc-stmp3xxx.c +++ b/drivers/rtc/rtc-stmp3xxx.c @@ -107,6 +107,8 @@ static void stmp3xxx_wdt_register(struct platform_device *rtc_pdev) wdt_pdev->dev.parent = &rtc_pdev->dev; wdt_pdev->dev.platform_data = &wdt_pdata; rc = platform_device_add(wdt_pdev); + if (rc) + platform_device_put(wdt_pdev); } if (rc) diff --git a/drivers/rtc/rtc-ti-k3.c b/drivers/rtc/rtc-ti-k3.c index 7a0f181d3fefe782dd206ae8abe48fe61860a603..ba23163cc0428247d73f163fe50ef046c0238d08 100644 --- a/drivers/rtc/rtc-ti-k3.c +++ b/drivers/rtc/rtc-ti-k3.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -45,14 +46,6 @@ #define K3RTC_MIN_OFFSET (-277761) #define K3RTC_MAX_OFFSET (277778) -/** - * struct ti_k3_rtc_soc_data - Private of compatible data for ti-k3-rtc - * @unlock_irq_erratum: Has erratum for unlock infinite IRQs (erratum i2327) - */ -struct ti_k3_rtc_soc_data { - const bool unlock_irq_erratum; -}; - static const struct regmap_config ti_k3_rtc_regmap_config = { .name = "peripheral-registers", .reg_bits = 32, @@ -118,7 +111,6 @@ static const struct reg_field ti_rtc_reg_fields[] = { * @rtc_dev: rtc device * @regmap: rtc mmio regmap * @r_fields: rtc register fields - * @soc: SoC compatible match data */ struct ti_k3_rtc { unsigned int irq; @@ -127,7 +119,6 @@ struct ti_k3_rtc { struct rtc_device *rtc_dev; struct regmap *regmap; struct regmap_field *r_fields[K3_RTC_MAX_FIELDS]; - const struct ti_k3_rtc_soc_data *soc; }; static int k3rtc_field_read(struct ti_k3_rtc *priv, enum ti_k3_rtc_fields f) @@ -190,11 +181,22 @@ static int k3rtc_unlock_rtc(struct ti_k3_rtc *priv) /* Skip fence since we are going to check the unlock bit as fence */ ret = regmap_field_read_poll_timeout(priv->r_fields[K3RTC_UNLOCK], ret, - !ret, 2, priv->sync_timeout_us); + ret, 2, priv->sync_timeout_us); return ret; } +/* + * This is the list of SoCs affected by TI's i2327 errata causing the RTC + * state-machine to break if not unlocked fast enough during boot. These + * SoCs must have the bootloader unlock this device very early in the + * boot-flow before we (Linux) can use this device. + */ +static const struct soc_device_attribute has_erratum_i2327[] = { + { .family = "AM62X", .revision = "SR1.0" }, + { /* sentinel */ } +}; + static int k3rtc_configure(struct device *dev) { int ret; @@ -208,7 +210,7 @@ static int k3rtc_configure(struct device *dev) * * In such occurrence, it is assumed that the RTC module is unusable */ - if (priv->soc->unlock_irq_erratum) { + if (soc_device_match(has_erratum_i2327)) { ret = k3rtc_check_unlocked(priv); /* If there is an error OR if we are locked, return error */ if (ret) { @@ -513,21 +515,12 @@ static struct nvmem_config ti_k3_rtc_nvmem_config = { static int k3rtc_get_32kclk(struct device *dev, struct ti_k3_rtc *priv) { - int ret; struct clk *clk; - clk = devm_clk_get(dev, "osc32k"); + clk = devm_clk_get_enabled(dev, "osc32k"); if (IS_ERR(clk)) return PTR_ERR(clk); - ret = clk_prepare_enable(clk); - if (ret) - return ret; - - ret = devm_add_action_or_reset(dev, (void (*)(void *))clk_disable_unprepare, clk); - if (ret) - return ret; - priv->rate_32k = clk_get_rate(clk); /* Make sure we are exact 32k clock. Else, try to compensate delay */ @@ -542,24 +535,19 @@ static int k3rtc_get_32kclk(struct device *dev, struct ti_k3_rtc *priv) */ priv->sync_timeout_us = (u32)(DIV_ROUND_UP_ULL(1000000, priv->rate_32k) * 4); - return ret; + return 0; } static int k3rtc_get_vbusclk(struct device *dev, struct ti_k3_rtc *priv) { - int ret; struct clk *clk; /* Note: VBUS isn't a context clock, it is needed for hardware operation */ - clk = devm_clk_get(dev, "vbus"); + clk = devm_clk_get_enabled(dev, "vbus"); if (IS_ERR(clk)) return PTR_ERR(clk); - ret = clk_prepare_enable(clk); - if (ret) - return ret; - - return devm_add_action_or_reset(dev, (void (*)(void *))clk_disable_unprepare, clk); + return 0; } static int ti_k3_rtc_probe(struct platform_device *pdev) @@ -602,8 +590,6 @@ static int ti_k3_rtc_probe(struct platform_device *pdev) if (IS_ERR(priv->rtc_dev)) return PTR_ERR(priv->rtc_dev); - priv->soc = of_device_get_match_data(dev); - priv->rtc_dev->ops = &ti_k3_rtc_ops; priv->rtc_dev->range_max = (1ULL << 48) - 1; /* 48Bit seconds */ ti_k3_rtc_nvmem_config.priv = priv; @@ -635,12 +621,8 @@ static int ti_k3_rtc_probe(struct platform_device *pdev) return devm_rtc_nvmem_register(priv->rtc_dev, &ti_k3_rtc_nvmem_config); } -static const struct ti_k3_rtc_soc_data ti_k3_am62_data = { - .unlock_irq_erratum = true, -}; - static const struct of_device_id ti_k3_rtc_of_match_table[] = { - {.compatible = "ti,am62-rtc", .data = &ti_k3_am62_data}, + {.compatible = "ti,am62-rtc" }, {} }; MODULE_DEVICE_TABLE(of, ti_k3_rtc_of_match_table); diff --git a/drivers/rtc/rtc-x1205.c b/drivers/rtc/rtc-x1205.c index ba0d22a5b421844ac1ec6ca9ebdd42943adedfe9..f587afa843573d2c7fa69a035f5b5886337c4544 100644 --- a/drivers/rtc/rtc-x1205.c +++ b/drivers/rtc/rtc-x1205.c @@ -657,10 +657,9 @@ static int x1205_probe(struct i2c_client *client) return 0; } -static int x1205_remove(struct i2c_client *client) +static void x1205_remove(struct i2c_client *client) { x1205_sysfs_unregister(&client->dev); - return 0; } static const struct i2c_device_id x1205_id[] = { diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index ea82821599f6e1ed9719daf384c01da12889bd93..5a6d9c15395f7acf626552bf4d3dd29bbd1553e5 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -41,15 +41,6 @@ #define DASD_DIAG_MOD "dasd_diag_mod" -static unsigned int queue_depth = 32; -static unsigned int nr_hw_queues = 4; - -module_param(queue_depth, uint, 0444); -MODULE_PARM_DESC(queue_depth, "Default queue depth for new DASD devices"); - -module_param(nr_hw_queues, uint, 0444); -MODULE_PARM_DESC(nr_hw_queues, "Default number of hardware queues for new DASD devices"); - /* * SECTION: exported variables of dasd.c */ @@ -68,8 +59,6 @@ MODULE_LICENSE("GPL"); /* * SECTION: prototypes for static functions of dasd.c */ -static int dasd_alloc_queue(struct dasd_block *); -static void dasd_free_queue(struct dasd_block *); static int dasd_flush_block_queue(struct dasd_block *); static void dasd_device_tasklet(unsigned long); static void dasd_block_tasklet(unsigned long); @@ -198,21 +187,11 @@ EXPORT_SYMBOL_GPL(dasd_free_block); */ static int dasd_state_new_to_known(struct dasd_device *device) { - int rc; - /* * As long as the device is not in state DASD_STATE_NEW we want to * keep the reference count > 0. */ dasd_get_device(device); - - if (device->block) { - rc = dasd_alloc_queue(device->block); - if (rc) { - dasd_put_device(device); - return rc; - } - } device->state = DASD_STATE_KNOWN; return 0; } @@ -226,9 +205,6 @@ static int dasd_state_known_to_new(struct dasd_device *device) dasd_eer_disable(device); device->state = DASD_STATE_NEW; - if (device->block) - dasd_free_queue(device->block); - /* Give up reference we took in dasd_state_new_to_known. */ dasd_put_device(device); return 0; @@ -1591,9 +1567,8 @@ void dasd_generic_handle_state_change(struct dasd_device *device) dasd_schedule_device_bh(device); if (device->block) { dasd_schedule_block_bh(device->block); - if (device->block->request_queue) - blk_mq_run_hw_queues(device->block->request_queue, - true); + if (device->block->gdp) + blk_mq_run_hw_queues(device->block->gdp->queue, true); } } EXPORT_SYMBOL_GPL(dasd_generic_handle_state_change); @@ -2691,7 +2666,7 @@ static void dasd_block_timeout(struct timer_list *t) dasd_device_remove_stop_bits(block->base, DASD_STOPPED_PENDING); spin_unlock_irqrestore(get_ccwdev_lock(block->base->cdev), flags); dasd_schedule_block_bh(block); - blk_mq_run_hw_queues(block->request_queue, true); + blk_mq_run_hw_queues(block->gdp->queue, true); } /* @@ -3239,7 +3214,7 @@ static void dasd_request_done(struct request *req) blk_mq_run_hw_queues(req->q, true); } -static struct blk_mq_ops dasd_mq_ops = { +struct blk_mq_ops dasd_mq_ops = { .queue_rq = do_dasd_request, .complete = dasd_request_done, .timeout = dasd_times_out, @@ -3247,45 +3222,6 @@ static struct blk_mq_ops dasd_mq_ops = { .exit_hctx = dasd_exit_hctx, }; -/* - * Allocate and initialize request queue and default I/O scheduler. - */ -static int dasd_alloc_queue(struct dasd_block *block) -{ - int rc; - - block->tag_set.ops = &dasd_mq_ops; - block->tag_set.cmd_size = sizeof(struct dasd_ccw_req); - block->tag_set.nr_hw_queues = nr_hw_queues; - block->tag_set.queue_depth = queue_depth; - block->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; - block->tag_set.numa_node = NUMA_NO_NODE; - - rc = blk_mq_alloc_tag_set(&block->tag_set); - if (rc) - return rc; - - block->request_queue = blk_mq_init_queue(&block->tag_set); - if (IS_ERR(block->request_queue)) - return PTR_ERR(block->request_queue); - - block->request_queue->queuedata = block; - - return 0; -} - -/* - * Deactivate and free request queue. - */ -static void dasd_free_queue(struct dasd_block *block) -{ - if (block->request_queue) { - blk_mq_destroy_queue(block->request_queue); - blk_mq_free_tag_set(&block->tag_set); - block->request_queue = NULL; - } -} - static int dasd_open(struct block_device *bdev, fmode_t mode) { struct dasd_device *base; @@ -3762,10 +3698,9 @@ int dasd_generic_path_operational(struct dasd_device *device) dasd_schedule_device_bh(device); if (device->block) { dasd_schedule_block_bh(device->block); - if (device->block->request_queue) - blk_mq_run_hw_queues(device->block->request_queue, - true); - } + if (device->block->gdp) + blk_mq_run_hw_queues(device->block->gdp->queue, true); + } if (!device->stopped) wake_up(&generic_waitq); @@ -3916,8 +3851,8 @@ void dasd_generic_space_avail(struct dasd_device *device) if (device->block) { dasd_schedule_block_bh(device->block); - if (device->block->request_queue) - blk_mq_run_hw_queues(device->block->request_queue, true); + if (device->block->gdp) + blk_mq_run_hw_queues(device->block->gdp->queue, true); } if (!device->stopped) wake_up(&generic_waitq); @@ -3927,7 +3862,7 @@ EXPORT_SYMBOL_GPL(dasd_generic_space_avail); /* * clear active requests and requeue them to block layer if possible */ -static int dasd_generic_requeue_all_requests(struct dasd_device *device) +int dasd_generic_requeue_all_requests(struct dasd_device *device) { struct list_head requeue_queue; struct dasd_ccw_req *cqr, *n; @@ -4001,6 +3936,7 @@ static int dasd_generic_requeue_all_requests(struct dasd_device *device) dasd_schedule_device_bh(device); return rc; } +EXPORT_SYMBOL_GPL(dasd_generic_requeue_all_requests); static void do_requeue_requests(struct work_struct *work) { diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 299001ad9a32d8c0ff23e54cb33543ef67f8902b..81d283b3cd3bdc870030cb93ccd09043b05531e8 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -1050,6 +1050,11 @@ dasd_3990_erp_com_rej(struct dasd_ccw_req * erp, char *sense) dev_err(&device->cdev->dev, "An I/O request was rejected" " because writing is inhibited\n"); erp = dasd_3990_erp_cleanup(erp, DASD_CQR_FAILED); + } else if (sense[7] & SNS7_INVALID_ON_SEC) { + dev_err(&device->cdev->dev, "An I/O request was rejected on a copy pair secondary device\n"); + /* suppress dump of sense data for this error */ + set_bit(DASD_CQR_SUPPRESS_CR, &erp->refers->flags); + erp = dasd_3990_erp_cleanup(erp, DASD_CQR_FAILED); } else { /* fatal error - set status to FAILED internal error 09 - Command Reject */ diff --git a/drivers/s390/block/dasd_alias.c b/drivers/s390/block/dasd_alias.c index dc78a523a69f2104d911555b097a6407a8e04cc1..b6b938aa6615894087d20943624e60bac54754a8 100644 --- a/drivers/s390/block/dasd_alias.c +++ b/drivers/s390/block/dasd_alias.c @@ -675,12 +675,12 @@ int dasd_alias_remove_device(struct dasd_device *device) struct dasd_device *dasd_alias_get_start_dev(struct dasd_device *base_device) { struct dasd_eckd_private *alias_priv, *private = base_device->private; - struct alias_pav_group *group = private->pavgroup; struct alias_lcu *lcu = private->lcu; struct dasd_device *alias_device; + struct alias_pav_group *group; unsigned long flags; - if (!group || !lcu) + if (!lcu) return NULL; if (lcu->pav == NO_PAV || lcu->flags & (NEED_UAC_UPDATE | UPDATE_PENDING)) @@ -697,6 +697,11 @@ struct dasd_device *dasd_alias_get_start_dev(struct dasd_device *base_device) } spin_lock_irqsave(&lcu->lock, flags); + group = private->pavgroup; + if (!group) { + spin_unlock_irqrestore(&lcu->lock, flags); + return NULL; + } alias_device = group->next; if (!alias_device) { if (list_empty(&group->aliaslist)) { diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index 811e79c9f59c840bd7201754aee7082a447e458b..cb83f81da41624552a26388efccdaf61cdceb888 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -26,7 +26,6 @@ /* This is ugly... */ #define PRINTK_HEADER "dasd_devmap:" -#define DASD_BUS_ID_SIZE 20 #define DASD_MAX_PARAMS 256 #include "dasd_int.h" @@ -50,6 +49,7 @@ struct dasd_devmap { unsigned int devindex; unsigned short features; struct dasd_device *device; + struct dasd_copy_relation *copy; }; /* @@ -130,7 +130,7 @@ __setup ("dasd=", dasd_call_setup); /* * Read a device busid/devno from a string. */ -static int __init dasd_busid(char *str, int *id0, int *id1, int *devno) +static int dasd_busid(char *str, int *id0, int *id1, int *devno) { unsigned int val; char *tok; @@ -426,7 +426,7 @@ dasd_add_busid(const char *bus_id, int features) if (!devmap) { /* This bus_id is new. */ new->devindex = dasd_max_devindex++; - strlcpy(new->bus_id, bus_id, DASD_BUS_ID_SIZE); + strscpy(new->bus_id, bus_id, DASD_BUS_ID_SIZE); new->features = features; new->device = NULL; list_add(&new->list, &dasd_hashlists[hash]); @@ -438,16 +438,12 @@ dasd_add_busid(const char *bus_id, int features) return devmap; } -/* - * Find devmap for device with given bus_id. - */ static struct dasd_devmap * -dasd_find_busid(const char *bus_id) +dasd_find_busid_locked(const char *bus_id) { struct dasd_devmap *devmap, *tmp; int hash; - spin_lock(&dasd_devmap_lock); devmap = ERR_PTR(-ENODEV); hash = dasd_hash_busid(bus_id); list_for_each_entry(tmp, &dasd_hashlists[hash], list) { @@ -456,6 +452,19 @@ dasd_find_busid(const char *bus_id) break; } } + return devmap; +} + +/* + * Find devmap for device with given bus_id. + */ +static struct dasd_devmap * +dasd_find_busid(const char *bus_id) +{ + struct dasd_devmap *devmap; + + spin_lock(&dasd_devmap_lock); + devmap = dasd_find_busid_locked(bus_id); spin_unlock(&dasd_devmap_lock); return devmap; } @@ -584,6 +593,238 @@ dasd_create_device(struct ccw_device *cdev) return device; } +/* + * allocate a PPRC data structure and call the discipline function to fill + */ +static int dasd_devmap_get_pprc_status(struct dasd_device *device, + struct dasd_pprc_data_sc4 **data) +{ + struct dasd_pprc_data_sc4 *temp; + + if (!device->discipline || !device->discipline->pprc_status) { + dev_warn(&device->cdev->dev, "Unable to query copy relation status\n"); + return -EOPNOTSUPP; + } + temp = kzalloc(sizeof(*temp), GFP_KERNEL); + if (!temp) + return -ENOMEM; + + /* get PPRC information from storage */ + if (device->discipline->pprc_status(device, temp)) { + dev_warn(&device->cdev->dev, "Error during copy relation status query\n"); + kfree(temp); + return -EINVAL; + } + *data = temp; + + return 0; +} + +/* + * find an entry in a PPRC device_info array by a given UID + * depending on the primary/secondary state of the device it has to be + * matched with the respective fields + */ +static int dasd_devmap_entry_from_pprc_data(struct dasd_pprc_data_sc4 *data, + struct dasd_uid uid, + bool primary) +{ + int i; + + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (primary) { + if (data->dev_info[i].prim_cu_ssid == uid.ssid && + data->dev_info[i].primary == uid.real_unit_addr) + return i; + } else { + if (data->dev_info[i].sec_cu_ssid == uid.ssid && + data->dev_info[i].secondary == uid.real_unit_addr) + return i; + } + } + return -1; +} + +/* + * check the consistency of a specified copy relation by checking + * the following things: + * + * - is the given device part of a copy pair setup + * - does the state of the device match the state in the PPRC status data + * - does the device UID match with the UID in the PPRC status data + * - to prevent misrouted IO check if the given device is present in all + * related PPRC status data + */ +static int dasd_devmap_check_copy_relation(struct dasd_device *device, + struct dasd_copy_entry *entry, + struct dasd_pprc_data_sc4 *data, + struct dasd_copy_relation *copy) +{ + struct dasd_pprc_data_sc4 *tmp_dat; + struct dasd_device *tmp_dev; + struct dasd_uid uid; + int i, j; + + if (!device->discipline || !device->discipline->get_uid || + device->discipline->get_uid(device, &uid)) + return 1; + + i = dasd_devmap_entry_from_pprc_data(data, uid, entry->primary); + if (i < 0) { + dev_warn(&device->cdev->dev, "Device not part of a copy relation\n"); + return 1; + } + + /* double check which role the current device has */ + if (entry->primary) { + if (data->dev_info[i].flags & 0x80) { + dev_warn(&device->cdev->dev, "Copy pair secondary is setup as primary\n"); + return 1; + } + if (data->dev_info[i].prim_cu_ssid != uid.ssid || + data->dev_info[i].primary != uid.real_unit_addr) { + dev_warn(&device->cdev->dev, + "Primary device %s does not match copy pair status primary device %04x\n", + dev_name(&device->cdev->dev), + data->dev_info[i].prim_cu_ssid | + data->dev_info[i].primary); + return 1; + } + } else { + if (!(data->dev_info[i].flags & 0x80)) { + dev_warn(&device->cdev->dev, "Copy pair primary is setup as secondary\n"); + return 1; + } + if (data->dev_info[i].sec_cu_ssid != uid.ssid || + data->dev_info[i].secondary != uid.real_unit_addr) { + dev_warn(&device->cdev->dev, + "Secondary device %s does not match copy pair status secondary device %04x\n", + dev_name(&device->cdev->dev), + data->dev_info[i].sec_cu_ssid | + data->dev_info[i].secondary); + return 1; + } + } + + /* + * the current device has to be part of the copy relation of all + * entries to prevent misrouted IO to another copy pair + */ + for (j = 0; j < DASD_CP_ENTRIES; j++) { + if (entry == ©->entry[j]) + tmp_dev = device; + else + tmp_dev = copy->entry[j].device; + + if (!tmp_dev) + continue; + + if (dasd_devmap_get_pprc_status(tmp_dev, &tmp_dat)) + return 1; + + if (dasd_devmap_entry_from_pprc_data(tmp_dat, uid, entry->primary) < 0) { + dev_warn(&tmp_dev->cdev->dev, + "Copy pair relation does not contain device: %s\n", + dev_name(&device->cdev->dev)); + kfree(tmp_dat); + return 1; + } + kfree(tmp_dat); + } + return 0; +} + +/* delete device from copy relation entry */ +static void dasd_devmap_delete_copy_relation_device(struct dasd_device *device) +{ + struct dasd_copy_relation *copy; + int i; + + if (!device->copy) + return; + + copy = device->copy; + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].device == device) + copy->entry[i].device = NULL; + } + dasd_put_device(device); + device->copy = NULL; +} + +/* + * read all required information for a copy relation setup and setup the device + * accordingly + */ +int dasd_devmap_set_device_copy_relation(struct ccw_device *cdev, + bool pprc_enabled) +{ + struct dasd_pprc_data_sc4 *data = NULL; + struct dasd_copy_entry *entry = NULL; + struct dasd_copy_relation *copy; + struct dasd_devmap *devmap; + struct dasd_device *device; + int i, rc = 0; + + devmap = dasd_devmap_from_cdev(cdev); + if (IS_ERR(devmap)) + return PTR_ERR(devmap); + + device = devmap->device; + if (!device) + return -ENODEV; + + copy = devmap->copy; + /* no copy pair setup for this device */ + if (!copy) + goto out; + + rc = dasd_devmap_get_pprc_status(device, &data); + if (rc) + return rc; + + /* print error if PPRC is requested but not enabled on storage server */ + if (!pprc_enabled) { + dev_err(&cdev->dev, "Copy relation not enabled on storage server\n"); + rc = -EINVAL; + goto out; + } + + if (!data->dev_info[0].state) { + dev_warn(&device->cdev->dev, "Copy pair setup requested for device not in copy relation\n"); + rc = -EINVAL; + goto out; + } + /* find entry */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && + strncmp(dev_name(&cdev->dev), + copy->entry[i].busid, DASD_BUS_ID_SIZE) == 0) { + entry = ©->entry[i]; + break; + } + } + if (!entry) { + dev_warn(&device->cdev->dev, "Copy relation entry not found\n"); + rc = -EINVAL; + goto out; + } + /* check if the copy relation is valid */ + if (dasd_devmap_check_copy_relation(device, entry, data, copy)) { + dev_warn(&device->cdev->dev, "Copy relation faulty\n"); + rc = -EINVAL; + goto out; + } + + dasd_get_device(device); + copy->entry[i].device = device; + device->copy = copy; +out: + kfree(data); + return rc; +} +EXPORT_SYMBOL_GPL(dasd_devmap_set_device_copy_relation); + /* * Wait queue for dasd_delete_device waits. */ @@ -617,6 +858,8 @@ dasd_delete_device(struct dasd_device *device) dev_set_drvdata(&device->cdev->dev, NULL); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); + /* Removve copy relation */ + dasd_devmap_delete_copy_relation_device(device); /* * Drop ref_count by 3, one for the devmap reference, one for * the cdev reference and one for the passed reference. @@ -694,6 +937,7 @@ void dasd_add_link_to_gendisk(struct gendisk *gdp, struct dasd_device *device) gdp->private_data = devmap; spin_unlock(&dasd_devmap_lock); } +EXPORT_SYMBOL(dasd_add_link_to_gendisk); struct dasd_device *dasd_device_from_gendisk(struct gendisk *gdp) { @@ -1334,7 +1578,6 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct dasd_device *device; - struct request_queue *q; unsigned long val; device = dasd_device_from_cdev(to_ccwdev(dev)); @@ -1346,15 +1589,13 @@ dasd_timeout_store(struct device *dev, struct device_attribute *attr, dasd_put_device(device); return -EINVAL; } - q = device->block->request_queue; - if (!q) { + if (!device->block->gdp) { dasd_put_device(device); return -ENODEV; } device->blk_timeout = val; - - blk_queue_rq_timeout(q, device->blk_timeout * HZ); + blk_queue_rq_timeout(device->block->gdp->queue, val * HZ); dasd_put_device(device); return count; @@ -1683,6 +1924,347 @@ dasd_path_fcs_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) static struct kobj_attribute path_fcs_attribute = __ATTR(fc_security, 0444, dasd_path_fcs_show, NULL); +/* + * print copy relation in the form + * primary,secondary[1] primary,secondary[2], ... + */ +static ssize_t +dasd_copy_pair_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + char prim_busid[DASD_BUS_ID_SIZE]; + struct dasd_copy_relation *copy; + struct dasd_devmap *devmap; + int len = 0; + int i; + + devmap = dasd_find_busid(dev_name(dev)); + if (IS_ERR(devmap)) + return -ENODEV; + + if (!devmap->copy) + return -ENODEV; + + copy = devmap->copy; + /* find primary */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && copy->entry[i].primary) { + strscpy(prim_busid, copy->entry[i].busid, + DASD_BUS_ID_SIZE); + break; + } + } + if (!copy->entry[i].primary) + goto out; + + /* print all secondary */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && !copy->entry[i].primary) + len += sysfs_emit_at(buf, len, "%s,%s ", prim_busid, + copy->entry[i].busid); + } + + len += sysfs_emit_at(buf, len, "\n"); +out: + return len; +} + +static int dasd_devmap_set_copy_relation(struct dasd_devmap *devmap, + struct dasd_copy_relation *copy, + char *busid, bool primary) +{ + int i; + + /* find free entry */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + /* current bus_id already included, nothing to do */ + if (copy->entry[i].configured && + strncmp(copy->entry[i].busid, busid, DASD_BUS_ID_SIZE) == 0) + return 0; + + if (!copy->entry[i].configured) + break; + } + if (i == DASD_CP_ENTRIES) + return -EINVAL; + + copy->entry[i].configured = true; + strscpy(copy->entry[i].busid, busid, DASD_BUS_ID_SIZE); + if (primary) { + copy->active = ©->entry[i]; + copy->entry[i].primary = true; + } + if (!devmap->copy) + devmap->copy = copy; + + return 0; +} + +static void dasd_devmap_del_copy_relation(struct dasd_copy_relation *copy, + char *busid) +{ + int i; + + spin_lock(&dasd_devmap_lock); + /* find entry */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && + strncmp(copy->entry[i].busid, busid, DASD_BUS_ID_SIZE) == 0) + break; + } + if (i == DASD_CP_ENTRIES || !copy->entry[i].configured) { + spin_unlock(&dasd_devmap_lock); + return; + } + + copy->entry[i].configured = false; + memset(copy->entry[i].busid, 0, DASD_BUS_ID_SIZE); + if (copy->active == ©->entry[i]) { + copy->active = NULL; + copy->entry[i].primary = false; + } + spin_unlock(&dasd_devmap_lock); +} + +static int dasd_devmap_clear_copy_relation(struct device *dev) +{ + struct dasd_copy_relation *copy; + struct dasd_devmap *devmap; + int i, rc = 1; + + devmap = dasd_devmap_from_cdev(to_ccwdev(dev)); + if (IS_ERR(devmap)) + return 1; + + spin_lock(&dasd_devmap_lock); + if (!devmap->copy) + goto out; + + copy = devmap->copy; + /* first check if all secondary devices are offline*/ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (!copy->entry[i].configured) + continue; + + if (copy->entry[i].device == copy->active->device) + continue; + + if (copy->entry[i].device) + goto out; + } + /* clear all devmap entries */ + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (strlen(copy->entry[i].busid) == 0) + continue; + if (copy->entry[i].device) { + dasd_put_device(copy->entry[i].device); + copy->entry[i].device->copy = NULL; + copy->entry[i].device = NULL; + } + devmap = dasd_find_busid_locked(copy->entry[i].busid); + devmap->copy = NULL; + memset(copy->entry[i].busid, 0, DASD_BUS_ID_SIZE); + } + kfree(copy); + rc = 0; +out: + spin_unlock(&dasd_devmap_lock); + return rc; +} + +/* + * parse BUSIDs from a copy pair + */ +static int dasd_devmap_parse_busid(const char *buf, char *prim_busid, + char *sec_busid) +{ + char *primary, *secondary, *tmp, *pt; + int id0, id1, id2; + + pt = kstrdup(buf, GFP_KERNEL); + tmp = pt; + if (!tmp) + return -ENOMEM; + + primary = strsep(&tmp, ","); + if (!primary) { + kfree(pt); + return -EINVAL; + } + secondary = strsep(&tmp, ","); + if (!secondary) { + kfree(pt); + return -EINVAL; + } + if (dasd_busid(primary, &id0, &id1, &id2)) { + kfree(pt); + return -EINVAL; + } + sprintf(prim_busid, "%01x.%01x.%04x", id0, id1, id2); + if (dasd_busid(secondary, &id0, &id1, &id2)) { + kfree(pt); + return -EINVAL; + } + sprintf(sec_busid, "%01x.%01x.%04x", id0, id1, id2); + kfree(pt); + + return 0; +} + +static ssize_t dasd_copy_pair_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_devmap *prim_devmap, *sec_devmap; + char prim_busid[DASD_BUS_ID_SIZE]; + char sec_busid[DASD_BUS_ID_SIZE]; + struct dasd_copy_relation *copy; + struct dasd_device *device; + bool pprc_enabled; + int rc; + + if (strncmp(buf, "clear", strlen("clear")) == 0) { + if (dasd_devmap_clear_copy_relation(dev)) + return -EINVAL; + return count; + } + + rc = dasd_devmap_parse_busid(buf, prim_busid, sec_busid); + if (rc) + return rc; + + if (strncmp(dev_name(dev), prim_busid, DASD_BUS_ID_SIZE) != 0 && + strncmp(dev_name(dev), sec_busid, DASD_BUS_ID_SIZE) != 0) + return -EINVAL; + + /* allocate primary devmap if needed */ + prim_devmap = dasd_find_busid(prim_busid); + if (IS_ERR(prim_devmap)) + prim_devmap = dasd_add_busid(prim_busid, DASD_FEATURE_DEFAULT); + + /* allocate secondary devmap if needed */ + sec_devmap = dasd_find_busid(sec_busid); + if (IS_ERR(sec_devmap)) + sec_devmap = dasd_add_busid(sec_busid, DASD_FEATURE_DEFAULT); + + /* setting copy relation is only allowed for offline secondary */ + if (sec_devmap->device) + return -EINVAL; + + if (prim_devmap->copy) { + copy = prim_devmap->copy; + } else if (sec_devmap->copy) { + copy = sec_devmap->copy; + } else { + copy = kzalloc(sizeof(*copy), GFP_KERNEL); + if (!copy) + return -ENOMEM; + } + spin_lock(&dasd_devmap_lock); + rc = dasd_devmap_set_copy_relation(prim_devmap, copy, prim_busid, true); + if (rc) { + spin_unlock(&dasd_devmap_lock); + return rc; + } + rc = dasd_devmap_set_copy_relation(sec_devmap, copy, sec_busid, false); + if (rc) { + spin_unlock(&dasd_devmap_lock); + return rc; + } + spin_unlock(&dasd_devmap_lock); + + /* if primary device is already online call device setup directly */ + if (prim_devmap->device && !prim_devmap->device->copy) { + device = prim_devmap->device; + if (device->discipline->pprc_enabled) { + pprc_enabled = device->discipline->pprc_enabled(device); + rc = dasd_devmap_set_device_copy_relation(device->cdev, + pprc_enabled); + } else { + rc = -EOPNOTSUPP; + } + } + if (rc) { + dasd_devmap_del_copy_relation(copy, prim_busid); + dasd_devmap_del_copy_relation(copy, sec_busid); + count = rc; + } + + return count; +} +static DEVICE_ATTR(copy_pair, 0644, dasd_copy_pair_show, + dasd_copy_pair_store); + +static ssize_t +dasd_copy_role_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct dasd_copy_relation *copy; + struct dasd_device *device; + int len, i; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + if (!device->copy) { + len = sysfs_emit(buf, "none\n"); + goto out; + } + copy = device->copy; + /* only the active device is primary */ + if (copy->active->device == device) { + len = sysfs_emit(buf, "primary\n"); + goto out; + } + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].device == device) { + len = sysfs_emit(buf, "secondary\n"); + goto out; + } + } + /* not in the list, no COPY role */ + len = sysfs_emit(buf, "none\n"); +out: + dasd_put_device(device); + return len; +} +static DEVICE_ATTR(copy_role, 0444, dasd_copy_role_show, NULL); + +static ssize_t dasd_device_ping(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_device *device; + size_t rc; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + /* + * do not try during offline processing + * early check only + * the sleep_on function itself checks for offline + * processing again + */ + if (test_bit(DASD_FLAG_OFFLINE, &device->flags)) { + rc = -EBUSY; + goto out; + } + if (!device->discipline || !device->discipline->device_ping) { + rc = -EOPNOTSUPP; + goto out; + } + rc = device->discipline->device_ping(device); + if (!rc) + rc = count; +out: + dasd_put_device(device); + return rc; +} +static DEVICE_ATTR(ping, 0200, NULL, dasd_device_ping); + #define DASD_DEFINE_ATTR(_name, _func) \ static ssize_t dasd_##_name##_show(struct device *dev, \ struct device_attribute *attr, \ @@ -1739,6 +2321,9 @@ static struct attribute * dasd_attrs[] = { &dev_attr_hpf.attr, &dev_attr_ese.attr, &dev_attr_fc_security.attr, + &dev_attr_copy_pair.attr, + &dev_attr_copy_role.attr, + &dev_attr_ping.attr, NULL, }; diff --git a/drivers/s390/block/dasd_diag.c b/drivers/s390/block/dasd_diag.c index 94ee59864971e095c1508124efb54cb5babb4ac5..f956a4ac9881a7cdaf1994ffb2e7b71af4565638 100644 --- a/drivers/s390/block/dasd_diag.c +++ b/drivers/s390/block/dasd_diag.c @@ -627,7 +627,7 @@ dasd_diag_dump_sense(struct dasd_device *device, struct dasd_ccw_req * req, static void dasd_diag_setup_blk_queue(struct dasd_block *block) { unsigned int logical_block_size = block->bp_block; - struct request_queue *q = block->request_queue; + struct request_queue *q = block->gdp->queue; int max; max = DIAG_MAX_BLOCKS << block->s2b_shift; diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 3cc93e2e4e15ab4e62ba50ba3420b2581c413973..662730f3b027c6933317eaaba99aced0d1a233a8 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -2012,6 +2012,49 @@ static void dasd_eckd_kick_validate_server(struct dasd_device *device) dasd_put_device(device); } +/* + * return if the device is the copy relation primary if a copy relation is active + */ +static int dasd_device_is_primary(struct dasd_device *device) +{ + if (!device->copy) + return 1; + + if (device->copy->active->device == device) + return 1; + + return 0; +} + +static int dasd_eckd_alloc_block(struct dasd_device *device) +{ + struct dasd_block *block; + struct dasd_uid temp_uid; + + if (!dasd_device_is_primary(device)) + return 0; + + dasd_eckd_get_uid(device, &temp_uid); + if (temp_uid.type == UA_BASE_DEVICE) { + block = dasd_alloc_block(); + if (IS_ERR(block)) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, "%s", + "could not allocate dasd block structure"); + return PTR_ERR(block); + } + device->block = block; + block->base = device; + } + return 0; +} + +static bool dasd_eckd_pprc_enabled(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + + return private->rdc_data.facilities.PPRC_enabled; +} + /* * Check device characteristics. * If the device is accessible using ECKD discipline, the device is enabled. @@ -2020,8 +2063,6 @@ static int dasd_eckd_check_characteristics(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; - struct dasd_block *block; - struct dasd_uid temp_uid; int rc, i; int readonly; unsigned long value; @@ -2079,20 +2120,29 @@ dasd_eckd_check_characteristics(struct dasd_device *device) device->default_expires = value; } - dasd_eckd_get_uid(device, &temp_uid); - if (temp_uid.type == UA_BASE_DEVICE) { - block = dasd_alloc_block(); - if (IS_ERR(block)) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, "%s", - "could not allocate dasd " - "block structure"); - rc = PTR_ERR(block); - goto out_err1; - } - device->block = block; - block->base = device; + /* Read Device Characteristics */ + rc = dasd_generic_read_dev_chars(device, DASD_ECKD_MAGIC, + &private->rdc_data, 64); + if (rc) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, + "Read device characteristic failed, rc=%d", rc); + goto out_err1; + } + + /* setup PPRC for device from devmap */ + rc = dasd_devmap_set_device_copy_relation(device->cdev, + dasd_eckd_pprc_enabled(device)); + if (rc) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, + "copy relation setup failed, rc=%d", rc); + goto out_err1; } + /* check if block device is needed and allocate in case */ + rc = dasd_eckd_alloc_block(device); + if (rc) + goto out_err1; + /* register lcu with alias handling, enable PAV */ rc = dasd_alias_make_device_known_to_lcu(device); if (rc) @@ -2117,15 +2167,6 @@ dasd_eckd_check_characteristics(struct dasd_device *device) /* Read Extent Pool Information */ dasd_eckd_read_ext_pool_info(device); - /* Read Device Characteristics */ - rc = dasd_generic_read_dev_chars(device, DASD_ECKD_MAGIC, - &private->rdc_data, 64); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read device characteristic failed, rc=%d", rc); - goto out_err3; - } - if ((device->features & DASD_FEATURE_USERAW) && !(private->rdc_data.facilities.RT_in_LR)) { dev_err(&device->cdev->dev, "The storage server does not " @@ -6078,6 +6119,207 @@ static int dasd_hosts_print(struct dasd_device *device, struct seq_file *m) return 0; } +static struct dasd_device +*copy_relation_find_device(struct dasd_copy_relation *copy, + char *busid) +{ + int i; + + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && + strncmp(copy->entry[i].busid, busid, DASD_BUS_ID_SIZE) == 0) + return copy->entry[i].device; + } + return NULL; +} + +/* + * set the new active/primary device + */ +static void copy_pair_set_active(struct dasd_copy_relation *copy, char *new_busid, + char *old_busid) +{ + int i; + + for (i = 0; i < DASD_CP_ENTRIES; i++) { + if (copy->entry[i].configured && + strncmp(copy->entry[i].busid, new_busid, + DASD_BUS_ID_SIZE) == 0) { + copy->active = ©->entry[i]; + copy->entry[i].primary = true; + } else if (copy->entry[i].configured && + strncmp(copy->entry[i].busid, old_busid, + DASD_BUS_ID_SIZE) == 0) { + copy->entry[i].primary = false; + } + } +} + +/* + * The function will swap the role of a given copy pair. + * During the swap operation the relation of the blockdevice is disconnected + * from the old primary and connected to the new. + * + * IO is paused on the block queue before swap and may be resumed afterwards. + */ +static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid, + char *sec_busid) +{ + struct dasd_device *primary, *secondary; + struct dasd_copy_relation *copy; + struct dasd_block *block; + struct gendisk *gdp; + + copy = device->copy; + if (!copy) + return DASD_COPYPAIRSWAP_INVALID; + primary = copy->active->device; + if (!primary) + return DASD_COPYPAIRSWAP_INVALID; + /* double check if swap has correct primary */ + if (strncmp(dev_name(&primary->cdev->dev), prim_busid, DASD_BUS_ID_SIZE) != 0) + return DASD_COPYPAIRSWAP_PRIMARY; + + secondary = copy_relation_find_device(copy, sec_busid); + if (!secondary) + return DASD_COPYPAIRSWAP_SECONDARY; + + /* + * usually the device should be quiesced for swap + * for paranoia stop device and requeue requests again + */ + dasd_device_set_stop_bits(primary, DASD_STOPPED_PPRC); + dasd_device_set_stop_bits(secondary, DASD_STOPPED_PPRC); + dasd_generic_requeue_all_requests(primary); + + /* swap DASD internal device <> block assignment */ + block = primary->block; + primary->block = NULL; + secondary->block = block; + block->base = secondary; + /* set new primary device in COPY relation */ + copy_pair_set_active(copy, sec_busid, prim_busid); + + /* swap blocklayer device link */ + gdp = block->gdp; + dasd_add_link_to_gendisk(gdp, secondary); + + /* re-enable device */ + dasd_device_remove_stop_bits(primary, DASD_STOPPED_PPRC); + dasd_device_remove_stop_bits(secondary, DASD_STOPPED_PPRC); + dasd_schedule_device_bh(secondary); + + return DASD_COPYPAIRSWAP_SUCCESS; +} + +/* + * Perform Subsystem Function - Peer-to-Peer Remote Copy Extended Query + */ +static int dasd_eckd_query_pprc_status(struct dasd_device *device, + struct dasd_pprc_data_sc4 *data) +{ + struct dasd_pprc_data_sc4 *pprc_data; + struct dasd_psf_prssd_data *prssdp; + struct dasd_ccw_req *cqr; + struct ccw1 *ccw; + int rc; + + cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, 1 /* PSF */ + 1 /* RSSD */, + sizeof(*prssdp) + sizeof(*pprc_data) + 1, + device, NULL); + if (IS_ERR(cqr)) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, "%s", + "Could not allocate query PPRC status request"); + return PTR_ERR(cqr); + } + cqr->startdev = device; + cqr->memdev = device; + cqr->block = NULL; + cqr->retries = 256; + cqr->expires = 10 * HZ; + + /* Prepare for Read Subsystem Data */ + prssdp = (struct dasd_psf_prssd_data *)cqr->data; + memset(prssdp, 0, sizeof(struct dasd_psf_prssd_data)); + prssdp->order = PSF_ORDER_PRSSD; + prssdp->suborder = PSF_SUBORDER_PPRCEQ; + prssdp->varies[0] = PPRCEQ_SCOPE_4; + pprc_data = (struct dasd_pprc_data_sc4 *)(prssdp + 1); + + ccw = cqr->cpaddr; + ccw->cmd_code = DASD_ECKD_CCW_PSF; + ccw->count = sizeof(struct dasd_psf_prssd_data); + ccw->flags |= CCW_FLAG_CC; + ccw->flags |= CCW_FLAG_SLI; + ccw->cda = (__u32)(addr_t)prssdp; + + /* Read Subsystem Data - query host access */ + ccw++; + ccw->cmd_code = DASD_ECKD_CCW_RSSD; + ccw->count = sizeof(*pprc_data); + ccw->flags |= CCW_FLAG_SLI; + ccw->cda = (__u32)(addr_t)pprc_data; + + cqr->buildclk = get_tod_clock(); + cqr->status = DASD_CQR_FILLED; + + rc = dasd_sleep_on_interruptible(cqr); + if (rc == 0) { + *data = *pprc_data; + } else { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, + "PPRC Extended Query failed with rc=%d\n", + rc); + rc = -EOPNOTSUPP; + } + + dasd_sfree_request(cqr, cqr->memdev); + return rc; +} + +/* + * ECKD NOP - no operation + */ +static int dasd_eckd_nop(struct dasd_device *device) +{ + struct dasd_ccw_req *cqr; + struct ccw1 *ccw; + int rc; + + cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, 1, 1, device, NULL); + if (IS_ERR(cqr)) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, "%s", + "Could not allocate NOP request"); + return PTR_ERR(cqr); + } + cqr->startdev = device; + cqr->memdev = device; + cqr->block = NULL; + cqr->retries = 1; + cqr->expires = 10 * HZ; + + ccw = cqr->cpaddr; + ccw->cmd_code = DASD_ECKD_CCW_NOP; + ccw->flags |= CCW_FLAG_SLI; + + cqr->buildclk = get_tod_clock(); + cqr->status = DASD_CQR_FILLED; + + rc = dasd_sleep_on_interruptible(cqr); + if (rc != 0) { + DBF_EVENT_DEVID(DBF_WARNING, device->cdev, + "NOP failed with rc=%d\n", rc); + rc = -EOPNOTSUPP; + } + dasd_sfree_request(cqr, cqr->memdev); + return rc; +} + +static int dasd_eckd_device_ping(struct dasd_device *device) +{ + return dasd_eckd_nop(device); +} + /* * Perform Subsystem Function - CUIR response */ @@ -6602,7 +6844,7 @@ static void dasd_eckd_handle_hpf_error(struct dasd_device *device, static void dasd_eckd_setup_blk_queue(struct dasd_block *block) { unsigned int logical_block_size = block->bp_block; - struct request_queue *q = block->request_queue; + struct request_queue *q = block->gdp->queue; struct dasd_device *device = block->base; int max; @@ -6697,6 +6939,10 @@ static struct dasd_discipline dasd_eckd_discipline = { .ext_pool_exhaust = dasd_eckd_ext_pool_exhaust, .ese_format = dasd_eckd_ese_format, .ese_read = dasd_eckd_ese_read, + .pprc_status = dasd_eckd_query_pprc_status, + .pprc_enabled = dasd_eckd_pprc_enabled, + .copy_pair_swap = dasd_eckd_copy_pair_swap, + .device_ping = dasd_eckd_device_ping, }; static int __init diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index a91b265441cc4f425531aac2c1af59f04a471717..f9299bd184ba80098375b06981921844670522bf 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -13,6 +13,7 @@ /***************************************************************************** * SECTION: CCW Definitions ****************************************************************************/ +#define DASD_ECKD_CCW_NOP 0x03 #define DASD_ECKD_CCW_WRITE 0x05 #define DASD_ECKD_CCW_READ 0x06 #define DASD_ECKD_CCW_WRITE_HOME_ADDRESS 0x09 @@ -66,9 +67,15 @@ * Perform Subsystem Function / Sub-Orders */ #define PSF_SUBORDER_QHA 0x1C /* Query Host Access */ +#define PSF_SUBORDER_PPRCEQ 0x50 /* PPRC Extended Query */ #define PSF_SUBORDER_VSQ 0x52 /* Volume Storage Query */ #define PSF_SUBORDER_LCQ 0x53 /* Logical Configuration Query */ +/* + * PPRC Extended Query Scopes + */ +#define PPRCEQ_SCOPE_4 0x04 /* Scope 4 for PPRC Extended Query */ + /* * CUIR response condition codes */ @@ -261,7 +268,7 @@ struct dasd_eckd_characteristics { unsigned char reserved3:8; unsigned char defect_wr:1; unsigned char XRC_supported:1; - unsigned char reserved4:1; + unsigned char PPRC_enabled:1; unsigned char striping:1; unsigned char reserved5:4; unsigned char cfw:1; diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c index 5ae64af9ccea3d3effe15c3e772cede643b961a5..d4d31cd11d2615e78cf52484e53d3b0a110c4590 100644 --- a/drivers/s390/block/dasd_eer.c +++ b/drivers/s390/block/dasd_eer.c @@ -313,7 +313,7 @@ static void dasd_eer_write_standard_trigger(struct dasd_device *device, ktime_get_real_ts64(&ts); header.tv_sec = ts.tv_sec; header.tv_usec = ts.tv_nsec / NSEC_PER_USEC; - strlcpy(header.busid, dev_name(&device->cdev->dev), + strscpy(header.busid, dev_name(&device->cdev->dev), DASD_EER_BUSID_SIZE); spin_lock_irqsave(&bufferlock, flags); @@ -356,7 +356,7 @@ static void dasd_eer_write_snss_trigger(struct dasd_device *device, ktime_get_real_ts64(&ts); header.tv_sec = ts.tv_sec; header.tv_usec = ts.tv_nsec / NSEC_PER_USEC; - strlcpy(header.busid, dev_name(&device->cdev->dev), + strscpy(header.busid, dev_name(&device->cdev->dev), DASD_EER_BUSID_SIZE); spin_lock_irqsave(&bufferlock, flags); diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c index 60be7f7bf2d167d02e1526cc7367adfd8e0e1642..cddfb01a3dca133336969b056fed0e9d1932a53e 100644 --- a/drivers/s390/block/dasd_fba.c +++ b/drivers/s390/block/dasd_fba.c @@ -767,7 +767,7 @@ dasd_fba_dump_sense(struct dasd_device *device, struct dasd_ccw_req * req, static void dasd_fba_setup_blk_queue(struct dasd_block *block) { unsigned int logical_block_size = block->bp_block; - struct request_queue *q = block->request_queue; + struct request_queue *q = block->gdp->queue; unsigned int max_bytes, max_discard_sectors; int max; diff --git a/drivers/s390/block/dasd_genhd.c b/drivers/s390/block/dasd_genhd.c index 5a83f0a39901b3816d978224a5673384d4ce340d..998a961e170417ae3fe882a0f80284cbca56b530 100644 --- a/drivers/s390/block/dasd_genhd.c +++ b/drivers/s390/block/dasd_genhd.c @@ -25,7 +25,14 @@ #include "dasd_int.h" -static struct lock_class_key dasd_bio_compl_lkclass; +static unsigned int queue_depth = 32; +static unsigned int nr_hw_queues = 4; + +module_param(queue_depth, uint, 0444); +MODULE_PARM_DESC(queue_depth, "Default queue depth for new DASD devices"); + +module_param(nr_hw_queues, uint, 0444); +MODULE_PARM_DESC(nr_hw_queues, "Default number of hardware queues for new DASD devices"); /* * Allocate and register gendisk structure for device. @@ -41,10 +48,21 @@ int dasd_gendisk_alloc(struct dasd_block *block) if (base->devindex >= DASD_PER_MAJOR) return -EBUSY; - gdp = blk_mq_alloc_disk_for_queue(block->request_queue, - &dasd_bio_compl_lkclass); - if (!gdp) - return -ENOMEM; + block->tag_set.ops = &dasd_mq_ops; + block->tag_set.cmd_size = sizeof(struct dasd_ccw_req); + block->tag_set.nr_hw_queues = nr_hw_queues; + block->tag_set.queue_depth = queue_depth; + block->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; + block->tag_set.numa_node = NUMA_NO_NODE; + rc = blk_mq_alloc_tag_set(&block->tag_set); + if (rc) + return rc; + + gdp = blk_mq_alloc_disk(&block->tag_set, block); + if (IS_ERR(gdp)) { + blk_mq_free_tag_set(&block->tag_set); + return PTR_ERR(gdp); + } /* Initialize gendisk structure. */ gdp->major = DASD_MAJOR; @@ -100,6 +118,7 @@ void dasd_gendisk_free(struct dasd_block *block) block->gdp->private_data = NULL; put_disk(block->gdp); block->gdp = NULL; + blk_mq_free_tag_set(&block->tag_set); } } diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 333a399f754ec00fa5d78342e352ed81a900489d..97adc8a7ae6b115dbed22db8c636101df0b58454 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -259,6 +259,55 @@ struct dasd_uid { char vduit[33]; }; +/* + * PPRC Status data + */ +struct dasd_pprc_header { + __u8 entries; /* 0 Number of device entries */ + __u8 unused; /* 1 unused */ + __u16 entry_length; /* 2-3 Length of device entry */ + __u32 unused2; /* 4-7 unused */ +} __packed; + +struct dasd_pprc_dev_info { + __u8 state; /* 0 Copy State */ + __u8 flags; /* 1 Flags */ + __u8 reserved1[2]; /* 2-3 reserved */ + __u8 prim_lss; /* 4 Primary device LSS */ + __u8 primary; /* 5 Primary device address */ + __u8 sec_lss; /* 6 Secondary device LSS */ + __u8 secondary; /* 7 Secondary device address */ + __u16 pprc_id; /* 8-9 Peer-to-Peer Remote Copy ID */ + __u8 reserved2[12]; /* 10-21 reserved */ + __u16 prim_cu_ssid; /* 22-23 Pimary Control Unit SSID */ + __u8 reserved3[12]; /* 24-35 reserved */ + __u16 sec_cu_ssid; /* 36-37 Secondary Control Unit SSID */ + __u8 reserved4[90]; /* 38-127 reserved */ +} __packed; + +struct dasd_pprc_data_sc4 { + struct dasd_pprc_header header; + struct dasd_pprc_dev_info dev_info[5]; +} __packed; + +#define DASD_BUS_ID_SIZE 20 +#define DASD_CP_ENTRIES 5 + +struct dasd_copy_entry { + char busid[DASD_BUS_ID_SIZE]; + struct dasd_device *device; + bool primary; + bool configured; +}; + +struct dasd_copy_relation { + struct dasd_copy_entry entry[DASD_CP_ENTRIES]; + struct dasd_copy_entry *active; +}; + +int dasd_devmap_set_device_copy_relation(struct ccw_device *, + bool pprc_enabled); + /* * the struct dasd_discipline is * sth like a table of virtual functions, if you think of dasd_eckd @@ -387,6 +436,10 @@ struct dasd_discipline { struct dasd_ccw_req *(*ese_format)(struct dasd_device *, struct dasd_ccw_req *, struct irb *); int (*ese_read)(struct dasd_ccw_req *, struct irb *); + int (*pprc_status)(struct dasd_device *, struct dasd_pprc_data_sc4 *); + bool (*pprc_enabled)(struct dasd_device *); + int (*copy_pair_swap)(struct dasd_device *, char *, char *); + int (*device_ping)(struct dasd_device *); }; extern struct dasd_discipline *dasd_diag_discipline_pointer; @@ -583,12 +636,12 @@ struct dasd_device { struct dasd_profile profile; struct dasd_format_entry format_entry; struct kset *paths_info; + struct dasd_copy_relation *copy; }; struct dasd_block { /* Block device stuff. */ struct gendisk *gdp; - struct request_queue *request_queue; spinlock_t request_queue_lock; struct blk_mq_tag_set tag_set; struct block_device *bdev; @@ -629,6 +682,7 @@ struct dasd_queue { #define DASD_STOPPED_PENDING 4 /* long busy */ #define DASD_STOPPED_DC_WAIT 8 /* disconnected, wait */ #define DASD_STOPPED_SU 16 /* summary unit check handling */ +#define DASD_STOPPED_PPRC 32 /* PPRC swap */ #define DASD_STOPPED_NOSPC 128 /* no space left */ /* per device flags */ @@ -653,6 +707,22 @@ struct dasd_queue { void dasd_put_device_wake(struct dasd_device *); +/* + * return values to be returned from the copy pair swap function + * 0x00: swap successful + * 0x01: swap data invalid + * 0x02: no active device found + * 0x03: wrong primary specified + * 0x04: secondary device not found + * 0x05: swap already running + */ +#define DASD_COPYPAIRSWAP_SUCCESS 0 +#define DASD_COPYPAIRSWAP_INVALID 1 +#define DASD_COPYPAIRSWAP_NOACTIVE 2 +#define DASD_COPYPAIRSWAP_PRIMARY 3 +#define DASD_COPYPAIRSWAP_SECONDARY 4 +#define DASD_COPYPAIRSWAP_MULTIPLE 5 + /* * Reference count inliners */ @@ -779,6 +849,7 @@ extern debug_info_t *dasd_debug_area; extern struct dasd_profile dasd_global_profile; extern unsigned int dasd_global_profile_level; extern const struct block_device_operations dasd_device_operations; +extern struct blk_mq_ops dasd_mq_ops; extern struct kmem_cache *dasd_page_cache; @@ -837,6 +908,8 @@ int dasd_generic_verify_path(struct dasd_device *, __u8); void dasd_generic_space_exhaust(struct dasd_device *, struct dasd_ccw_req *); void dasd_generic_space_avail(struct dasd_device *); +int dasd_generic_requeue_all_requests(struct dasd_device *); + int dasd_generic_read_dev_chars(struct dasd_device *, int, void *, int); char *dasd_get_sense(struct irb *); diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 95349f95758c48726985fa429036afde8e553503..d0ddf2cc97861345252bd38fd0a274a852c98506 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -379,6 +379,56 @@ out_err: return rc; } +/* + * Swap driver iternal copy relation. + */ +static int +dasd_ioctl_copy_pair_swap(struct block_device *bdev, void __user *argp) +{ + struct dasd_copypair_swap_data_t data; + struct dasd_device *device; + int rc; + + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; + + device = dasd_device_from_gendisk(bdev->bd_disk); + if (!device) + return -ENODEV; + + if (copy_from_user(&data, argp, sizeof(struct dasd_copypair_swap_data_t))) { + dasd_put_device(device); + return -EFAULT; + } + if (memchr_inv(data.reserved, 0, sizeof(data.reserved))) { + pr_warn("%s: Ivalid swap data specified.\n", + dev_name(&device->cdev->dev)); + dasd_put_device(device); + return DASD_COPYPAIRSWAP_INVALID; + } + if (bdev_is_partition(bdev)) { + pr_warn("%s: The specified DASD is a partition and cannot be swapped\n", + dev_name(&device->cdev->dev)); + dasd_put_device(device); + return DASD_COPYPAIRSWAP_INVALID; + } + if (!device->copy) { + pr_warn("%s: The specified DASD has no copy pair set up\n", + dev_name(&device->cdev->dev)); + dasd_put_device(device); + return -ENODEV; + } + if (!device->discipline->copy_pair_swap) { + dasd_put_device(device); + return -EOPNOTSUPP; + } + rc = device->discipline->copy_pair_swap(device, data.primary, + data.secondary); + dasd_put_device(device); + + return rc; +} + #ifdef CONFIG_DASD_PROFILE /* * Reset device profile information @@ -637,6 +687,9 @@ int dasd_ioctl(struct block_device *bdev, fmode_t mode, case BIODASDRAS: rc = dasd_ioctl_release_space(bdev, argp); break; + case BIODASDCOPYPAIRSWAP: + rc = dasd_ioctl_copy_pair_swap(bdev, argp); + break; default: /* if the discipline has an ioctl method try it. */ rc = -ENOTTY; diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 5187705bd0f39294026348d354f3f6bce4c578fb..93b80da602772ac5a1d654e15b27ed32f29820bc 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -614,7 +614,7 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char rc = -ENAMETOOLONG; goto seg_list_del; } - strlcpy(local_buf, buf, i + 1); + strscpy(local_buf, buf, i + 1); dev_info->num_of_segments = num_of_segments; rc = dcssblk_is_continuous(dev_info); if (rc < 0) diff --git a/drivers/s390/char/hmcdrv_cache.c b/drivers/s390/char/hmcdrv_cache.c index 1f5bdb23786234450b757ffcbbc85e7b966e0a5f..43df27ceec1108963301a680e2e5e447a7dc4d98 100644 --- a/drivers/s390/char/hmcdrv_cache.c +++ b/drivers/s390/char/hmcdrv_cache.c @@ -154,7 +154,7 @@ static ssize_t hmcdrv_cache_do(const struct hmcdrv_ftp_cmdspec *ftp, /* cache some file info (FTP command, file name and file * size) unconditionally */ - strlcpy(hmcdrv_cache_file.fname, ftp->fname, + strscpy(hmcdrv_cache_file.fname, ftp->fname, HMCDRV_FTP_FIDENT_MAX); hmcdrv_cache_file.id = ftp->id; pr_debug("caching cmd %d, file size %zu for '%s'\n", diff --git a/drivers/s390/char/tape_class.c b/drivers/s390/char/tape_class.c index b58df0dd0039c0d3a7f97fade839d0bd600bfd2c..c21dc68e05a043857e47a70ec0dd447f2d9f1e35 100644 --- a/drivers/s390/char/tape_class.c +++ b/drivers/s390/char/tape_class.c @@ -54,10 +54,10 @@ struct tape_class_device *register_tape_dev( if (!tcd) return ERR_PTR(-ENOMEM); - strlcpy(tcd->device_name, device_name, TAPECLASS_NAME_LEN); + strscpy(tcd->device_name, device_name, TAPECLASS_NAME_LEN); for (s = strchr(tcd->device_name, '/'); s; s = strchr(s, '/')) *s = '!'; - strlcpy(tcd->mode_name, mode_name, TAPECLASS_NAME_LEN); + strscpy(tcd->mode_name, mode_name, TAPECLASS_NAME_LEN); for (s = strchr(tcd->mode_name, '/'); s; s = strchr(s, '/')) *s = '!'; diff --git a/drivers/s390/char/tty3270.c b/drivers/s390/char/tty3270.c index 5c83f71c1d0e16a6dd0249d34d00e87c0539512b..26e3995ac062ce8fcfa0188de0a2ee9b0306d354 100644 --- a/drivers/s390/char/tty3270.c +++ b/drivers/s390/char/tty3270.c @@ -1760,7 +1760,7 @@ tty3270_flush_chars(struct tty_struct *tty) * Check for visible/invisible input switches */ static void -tty3270_set_termios(struct tty_struct *tty, struct ktermios *old) +tty3270_set_termios(struct tty_struct *tty, const struct ktermios *old) { struct tty3270 *tp; int new; diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 68f49e2e964c0d9877d3e1410cee6c1d689afe06..131293f7f152178087d21bffa9a4ba61ac0fb5e5 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -15,12 +15,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include "vmur.h" @@ -78,6 +80,8 @@ static struct ccw_driver ur_driver = { static DEFINE_MUTEX(vmur_mutex); +static void ur_uevent(struct work_struct *ws); + /* * Allocation, freeing, getting and putting of urdev structures * @@ -108,6 +112,7 @@ static struct urdev *urdev_alloc(struct ccw_device *cdev) ccw_device_get_id(cdev, &urd->dev_id); mutex_init(&urd->io_mutex); init_waitqueue_head(&urd->wait); + INIT_WORK(&urd->uevent_work, ur_uevent); spin_lock_init(&urd->open_lock); refcount_set(&urd->ref_count, 1); urd->cdev = cdev; @@ -275,6 +280,18 @@ out: return rc; } +static void ur_uevent(struct work_struct *ws) +{ + struct urdev *urd = container_of(ws, struct urdev, uevent_work); + char *envp[] = { + "EVENT=unsol_de", /* Unsolicited device-end interrupt */ + NULL + }; + + kobject_uevent_env(&urd->cdev->dev.kobj, KOBJ_CHANGE, envp); + urdev_put(urd); +} + /* * ur interrupt handler, called from the ccw_device layer */ @@ -288,12 +305,21 @@ static void ur_int_handler(struct ccw_device *cdev, unsigned long intparm, intparm, irb->scsw.cmd.cstat, irb->scsw.cmd.dstat, irb->scsw.cmd.count); } + urd = dev_get_drvdata(&cdev->dev); if (!intparm) { TRACE("ur_int_handler: unsolicited interrupt\n"); + + if (scsw_dstat(&irb->scsw) & DEV_STAT_DEV_END) { + /* + * Userspace might be interested in a transition to + * device-ready state. + */ + urdev_get(urd); + schedule_work(&urd->uevent_work); + } + return; } - urd = dev_get_drvdata(&cdev->dev); - BUG_ON(!urd); /* On special conditions irb is an error pointer */ if (IS_ERR(irb)) urd->io_request_rc = PTR_ERR(irb); @@ -809,7 +835,6 @@ static int ur_probe(struct ccw_device *cdev) rc = -ENOMEM; goto fail_urdev_put; } - cdev->handler = ur_int_handler; /* validate virtual unit record device */ urd->class = get_urd_class(urd); @@ -823,6 +848,7 @@ static int ur_probe(struct ccw_device *cdev) } spin_lock_irq(get_ccwdev_lock(cdev)); dev_set_drvdata(&cdev->dev, urd); + cdev->handler = ur_int_handler; spin_unlock_irq(get_ccwdev_lock(cdev)); mutex_unlock(&vmur_mutex); @@ -928,6 +954,10 @@ static int ur_set_offline_force(struct ccw_device *cdev, int force) rc = -EBUSY; goto fail_urdev_put; } + if (cancel_work_sync(&urd->uevent_work)) { + /* Work not run yet - need to release reference here */ + urdev_put(urd); + } device_destroy(vmur_class, urd->char_device->dev); cdev_del(urd->char_device); urd->char_device = NULL; @@ -963,6 +993,7 @@ static void ur_remove(struct ccw_device *cdev) spin_lock_irqsave(get_ccwdev_lock(cdev), flags); urdev_put(dev_get_drvdata(&cdev->dev)); dev_set_drvdata(&cdev->dev, NULL); + cdev->handler = NULL; spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags); mutex_unlock(&vmur_mutex); diff --git a/drivers/s390/char/vmur.h b/drivers/s390/char/vmur.h index 608b0719ce17b68301d7f6df47743c3a64b403d2..92d17d7cb47bdb4a95b904ea4bf91cfd8b5a3514 100644 --- a/drivers/s390/char/vmur.h +++ b/drivers/s390/char/vmur.h @@ -13,6 +13,7 @@ #define _VMUR_H_ #include +#include #define DEV_CLASS_UR_I 0x20 /* diag210 unit record input device class */ #define DEV_CLASS_UR_O 0x10 /* diag210 unit record output device class */ @@ -76,6 +77,7 @@ struct urdev { wait_queue_head_t wait; /* wait queue to serialize open */ int open_flag; /* "urdev is open" flag */ spinlock_t open_lock; /* serialize critical sections */ + struct work_struct uevent_work; /* work to send uevent */ }; /* diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index f6da215ccf9f6c2a158173395fdf441423b4096a..6165e6aae762a044150df20f19c7e1e8900b076d 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "sclp.h" #define TRACE(x...) debug_sprintf_event(zcore_dbf, 1, x) diff --git a/drivers/s390/cio/qdio_debug.c b/drivers/s390/cio/qdio_debug.c index 4bb7965daa0fc7b8d048a0839fdb4737e84b4fe3..1a9714af51e430ab61598f5661c7c8b6a1bd2693 100644 --- a/drivers/s390/cio/qdio_debug.c +++ b/drivers/s390/cio/qdio_debug.c @@ -87,7 +87,7 @@ int qdio_allocate_dbf(struct qdio_irq *irq_ptr) debug_unregister(irq_ptr->debug_area); return -ENOMEM; } - strlcpy(new_entry->dbf_name, text, QDIO_DBF_NAME_LEN); + strscpy(new_entry->dbf_name, text, QDIO_DBF_NAME_LEN); new_entry->dbf_info = irq_ptr->debug_area; mutex_lock(&qdio_dbf_list_mutex); list_add(&new_entry->dbf_list, &qdio_dbf_list); diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 86d9e428357b0610b37409d7e45c0146a190d4bd..7f5402fe857a2e6d2fbc5292b1fd88718c3d49d1 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -12,7 +12,6 @@ #include #include -#include #include #include @@ -142,7 +141,6 @@ static struct vfio_ccw_private *vfio_ccw_alloc_private(struct subchannel *sch) INIT_LIST_HEAD(&private->crw); INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); - atomic_set(&private->avail, 1); private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), GFP_KERNEL); @@ -203,7 +201,6 @@ static void vfio_ccw_free_private(struct vfio_ccw_private *private) mutex_destroy(&private->io_mutex); kfree(private); } - static int vfio_ccw_sch_probe(struct subchannel *sch) { struct pmcw *pmcw = &sch->schib.pmcw; @@ -222,7 +219,12 @@ static int vfio_ccw_sch_probe(struct subchannel *sch) dev_set_drvdata(&sch->dev, private); - ret = mdev_register_device(&sch->dev, &vfio_ccw_mdev_driver); + private->mdev_type.sysfs_name = "io"; + private->mdev_type.pretty_name = "I/O subchannel (Non-QDIO)"; + private->mdev_types[0] = &private->mdev_type; + ret = mdev_register_parent(&private->parent, &sch->dev, + &vfio_ccw_mdev_driver, + private->mdev_types, 1); if (ret) goto out_free; @@ -241,7 +243,7 @@ static void vfio_ccw_sch_remove(struct subchannel *sch) { struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev); - mdev_unregister_device(&sch->dev); + mdev_unregister_parent(&private->parent); dev_set_drvdata(&sch->dev, NULL); diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index 4a806a2273b54ea0ed3ad059ed57cae9ac80bc44..6ae4d012d800845bf5cad2b68abefd5bbc4e2164 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -11,7 +11,6 @@ */ #include -#include #include #include @@ -45,47 +44,14 @@ static void vfio_ccw_dma_unmap(struct vfio_device *vdev, u64 iova, u64 length) vfio_ccw_mdev_reset(private); } -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "I/O subchannel (Non-QDIO)\n"); -} -static MDEV_TYPE_ATTR_RO(name); - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_CCW_STRING); -} -static MDEV_TYPE_ATTR_RO(device_api); - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) +static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) { struct vfio_ccw_private *private = - dev_get_drvdata(mtype_get_parent_dev(mtype)); + container_of(vdev, struct vfio_ccw_private, vdev); - return sprintf(buf, "%d\n", atomic_read(&private->avail)); + init_completion(&private->release_comp); + return 0; } -static MDEV_TYPE_ATTR_RO(available_instances); - -static struct attribute *mdev_types_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_available_instances.attr, - NULL, -}; - -static struct attribute_group mdev_type_group = { - .name = "io", - .attrs = mdev_types_attrs, -}; - -static struct attribute_group *mdev_type_groups[] = { - &mdev_type_group, - NULL, -}; static int vfio_ccw_mdev_probe(struct mdev_device *mdev) { @@ -95,12 +61,9 @@ static int vfio_ccw_mdev_probe(struct mdev_device *mdev) if (private->state == VFIO_CCW_STATE_NOT_OPER) return -ENODEV; - if (atomic_dec_if_positive(&private->avail) < 0) - return -EPERM; - - memset(&private->vdev, 0, sizeof(private->vdev)); - vfio_init_group_dev(&private->vdev, &mdev->dev, - &vfio_ccw_dev_ops); + ret = vfio_init_device(&private->vdev, &mdev->dev, &vfio_ccw_dev_ops); + if (ret) + return ret; VFIO_CCW_MSG_EVENT(2, "sch %x.%x.%04x: create\n", private->sch->schid.cssid, @@ -109,16 +72,32 @@ static int vfio_ccw_mdev_probe(struct mdev_device *mdev) ret = vfio_register_emulated_iommu_dev(&private->vdev); if (ret) - goto err_atomic; + goto err_put_vdev; dev_set_drvdata(&mdev->dev, private); return 0; -err_atomic: - vfio_uninit_group_dev(&private->vdev); - atomic_inc(&private->avail); +err_put_vdev: + vfio_put_device(&private->vdev); return ret; } +static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) +{ + struct vfio_ccw_private *private = + container_of(vdev, struct vfio_ccw_private, vdev); + + /* + * We cannot free vfio_ccw_private here because it includes + * parent info which must be free'ed by css driver. + * + * Use a workaround by memset'ing the core device part and + * then notifying the remove path that all active references + * to this device have been released. + */ + memset(vdev, 0, sizeof(*vdev)); + complete(&private->release_comp); +} + static void vfio_ccw_mdev_remove(struct mdev_device *mdev) { struct vfio_ccw_private *private = dev_get_drvdata(mdev->dev.parent); @@ -130,8 +109,16 @@ static void vfio_ccw_mdev_remove(struct mdev_device *mdev) vfio_unregister_group_dev(&private->vdev); - vfio_uninit_group_dev(&private->vdev); - atomic_inc(&private->avail); + vfio_put_device(&private->vdev); + /* + * Wait for all active references on mdev are released so it + * is safe to defer kfree() to a later point. + * + * TODO: the clean fix is to split parent/mdev info from ccw + * private structure so each can be managed in its own life + * cycle. + */ + wait_for_completion(&private->release_comp); } static int vfio_ccw_mdev_open_device(struct vfio_device *vdev) @@ -592,6 +579,8 @@ static void vfio_ccw_mdev_request(struct vfio_device *vdev, unsigned int count) } static const struct vfio_device_ops vfio_ccw_dev_ops = { + .init = vfio_ccw_mdev_init_dev, + .release = vfio_ccw_mdev_release_dev, .open_device = vfio_ccw_mdev_open_device, .close_device = vfio_ccw_mdev_close_device, .read = vfio_ccw_mdev_read, @@ -602,6 +591,8 @@ static const struct vfio_device_ops vfio_ccw_dev_ops = { }; struct mdev_driver vfio_ccw_mdev_driver = { + .device_api = VFIO_DEVICE_API_CCW_STRING, + .max_instances = 1, .driver = { .name = "vfio_ccw_mdev", .owner = THIS_MODULE, @@ -609,5 +600,4 @@ struct mdev_driver vfio_ccw_mdev_driver = { }, .probe = vfio_ccw_mdev_probe, .remove = vfio_ccw_mdev_remove, - .supported_type_groups = mdev_type_groups, }; diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index cd24b7fada91ca11f029b77550d26a947b69c195..bd5fb81456af854c0a87a2b27262e72f66a3d04d 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -72,7 +73,6 @@ struct vfio_ccw_crw { * @sch: pointer to the subchannel * @state: internal state of the device * @completion: synchronization helper of the I/O completion - * @avail: available for creating a mediated device * @io_region: MMIO region to input/output I/O arguments/results * @io_mutex: protect against concurrent update of I/O regions * @region: additional regions for other subchannel operations @@ -88,13 +88,14 @@ struct vfio_ccw_crw { * @req_trigger: eventfd ctx for signaling userspace to return device * @io_work: work for deferral process of I/O handling * @crw_work: work for deferral process of CRW handling + * @release_comp: synchronization helper for vfio device release + * @parent: parent data structures for mdevs created */ struct vfio_ccw_private { struct vfio_device vdev; struct subchannel *sch; int state; struct completion *completion; - atomic_t avail; struct ccw_io_region *io_region; struct mutex io_mutex; struct vfio_ccw_region *region; @@ -113,6 +114,12 @@ struct vfio_ccw_private { struct eventfd_ctx *req_trigger; struct work_struct io_work; struct work_struct crw_work; + + struct completion release_comp; + + struct mdev_parent parent; + struct mdev_type mdev_type; + struct mdev_type *mdev_types[1]; } __aligned(8); int vfio_ccw_sch_quiesce(struct subchannel *sch); diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 6c8c41fac4e14f60f9fa542dbed8434a18b39198..0b4cc8c597ae67845c8d01c434e93ec05c3a5195 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -684,42 +684,41 @@ static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm, AP_DOMAINS); } -static int vfio_ap_mdev_probe(struct mdev_device *mdev) +static int vfio_ap_mdev_init_dev(struct vfio_device *vdev) { - struct ap_matrix_mdev *matrix_mdev; - int ret; - - if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0)) - return -EPERM; - - matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL); - if (!matrix_mdev) { - ret = -ENOMEM; - goto err_dec_available; - } - vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev, - &vfio_ap_matrix_dev_ops); + struct ap_matrix_mdev *matrix_mdev = + container_of(vdev, struct ap_matrix_mdev, vdev); - matrix_mdev->mdev = mdev; + matrix_mdev->mdev = to_mdev_device(vdev->dev); vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix); matrix_mdev->pqap_hook = handle_pqap; vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb); hash_init(matrix_mdev->qtable.queues); + return 0; +} + +static int vfio_ap_mdev_probe(struct mdev_device *mdev) +{ + struct ap_matrix_mdev *matrix_mdev; + int ret; + + matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev, + &vfio_ap_matrix_dev_ops); + if (IS_ERR(matrix_mdev)) + return PTR_ERR(matrix_mdev); + ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) - goto err_list; + goto err_put_vdev; dev_set_drvdata(&mdev->dev, matrix_mdev); mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); mutex_unlock(&matrix_dev->mdevs_lock); return 0; -err_list: - vfio_uninit_group_dev(&matrix_mdev->vdev); - kfree(matrix_mdev); -err_dec_available: - atomic_inc(&matrix_dev->available_instances); +err_put_vdev: + vfio_put_device(&matrix_mdev->vdev); return ret; } @@ -766,6 +765,11 @@ static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev) } } +static void vfio_ap_mdev_release_dev(struct vfio_device *vdev) +{ + vfio_free_device(vdev); +} + static void vfio_ap_mdev_remove(struct mdev_device *mdev) { struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev); @@ -779,54 +783,9 @@ static void vfio_ap_mdev_remove(struct mdev_device *mdev) list_del(&matrix_mdev->node); mutex_unlock(&matrix_dev->mdevs_lock); mutex_unlock(&matrix_dev->guests_lock); - vfio_uninit_group_dev(&matrix_mdev->vdev); - kfree(matrix_mdev); - atomic_inc(&matrix_dev->available_instances); -} - -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT); + vfio_put_device(&matrix_mdev->vdev); } -static MDEV_TYPE_ATTR_RO(name); - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) -{ - return sprintf(buf, "%d\n", - atomic_read(&matrix_dev->available_instances)); -} - -static MDEV_TYPE_ATTR_RO(available_instances); - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING); -} - -static MDEV_TYPE_ATTR_RO(device_api); - -static struct attribute *vfio_ap_mdev_type_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_available_instances.attr, - NULL, -}; - -static struct attribute_group vfio_ap_mdev_hwvirt_type_group = { - .name = VFIO_AP_MDEV_TYPE_HWVIRT, - .attrs = vfio_ap_mdev_type_attrs, -}; - -static struct attribute_group *vfio_ap_mdev_type_groups[] = { - &vfio_ap_mdev_hwvirt_type_group, - NULL, -}; - #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \ "already assigned to %s" @@ -984,6 +943,11 @@ static ssize_t assign_adapter_store(struct device *dev, goto done; } + if (test_bit_inv(apid, matrix_mdev->matrix.apm)) { + ret = count; + goto done; + } + set_bit_inv(apid, matrix_mdev->matrix.apm); ret = vfio_ap_mdev_validate_masks(matrix_mdev); @@ -1109,6 +1073,11 @@ static ssize_t unassign_adapter_store(struct device *dev, goto done; } + if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) { + ret = count; + goto done; + } + clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm); vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid); ret = count; @@ -1183,6 +1152,11 @@ static ssize_t assign_domain_store(struct device *dev, goto done; } + if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { + ret = count; + goto done; + } + set_bit_inv(apqi, matrix_mdev->matrix.aqm); ret = vfio_ap_mdev_validate_masks(matrix_mdev); @@ -1286,6 +1260,11 @@ static ssize_t unassign_domain_store(struct device *dev, goto done; } + if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) { + ret = count; + goto done; + } + clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm); vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi); ret = count; @@ -1329,6 +1308,11 @@ static ssize_t assign_control_domain_store(struct device *dev, goto done; } + if (test_bit_inv(id, matrix_mdev->matrix.adm)) { + ret = count; + goto done; + } + /* Set the bit in the ADM (bitmask) corresponding to the AP control * domain number (id). The bits in the mask, from most significant to * least significant, correspond to IDs 0 up to the one less than the @@ -1378,6 +1362,11 @@ static ssize_t unassign_control_domain_store(struct device *dev, goto done; } + if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) { + ret = count; + goto done; + } + clear_bit_inv(domid, matrix_mdev->matrix.adm); if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) { @@ -1794,6 +1783,8 @@ static const struct attribute_group vfio_queue_attr_group = { }; static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { + .init = vfio_ap_mdev_init_dev, + .release = vfio_ap_mdev_release_dev, .open_device = vfio_ap_mdev_open_device, .close_device = vfio_ap_mdev_close_device, .ioctl = vfio_ap_mdev_ioctl, @@ -1801,6 +1792,8 @@ static const struct vfio_device_ops vfio_ap_matrix_dev_ops = { }; static struct mdev_driver vfio_ap_matrix_driver = { + .device_api = VFIO_DEVICE_API_AP_STRING, + .max_instances = MAX_ZDEV_ENTRIES_EXT, .driver = { .name = "vfio_ap_mdev", .owner = THIS_MODULE, @@ -1809,20 +1802,22 @@ static struct mdev_driver vfio_ap_matrix_driver = { }, .probe = vfio_ap_mdev_probe, .remove = vfio_ap_mdev_remove, - .supported_type_groups = vfio_ap_mdev_type_groups, }; int vfio_ap_mdev_register(void) { int ret; - atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT); - ret = mdev_register_driver(&vfio_ap_matrix_driver); if (ret) return ret; - ret = mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_driver); + matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT; + matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT; + matrix_dev->mdev_types[0] = &matrix_dev->mdev_type; + ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device, + &vfio_ap_matrix_driver, + matrix_dev->mdev_types, 1); if (ret) goto err_driver; return 0; @@ -1834,7 +1829,7 @@ err_driver: void vfio_ap_mdev_unregister(void) { - mdev_unregister_device(&matrix_dev->device); + mdev_unregister_parent(&matrix_dev->parent); mdev_unregister_driver(&vfio_ap_matrix_driver); } diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index d782cf463eabae76ae976f4989746a639e541dd5..2eddd5f34ed34c55e9f0473a8d1c90c0224a2d6f 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -13,7 +13,6 @@ #define _VFIO_AP_PRIVATE_H_ #include -#include #include #include #include @@ -30,7 +29,6 @@ * struct ap_matrix_dev - Contains the data for the matrix device. * * @device: generic device structure associated with the AP matrix device - * @available_instances: number of mediated matrix devices that can be created * @info: the struct containing the output from the PQAP(QCI) instruction * @mdev_list: the list of mediated matrix devices created * @mdevs_lock: mutex for locking the AP matrix device. This lock will be @@ -47,12 +45,14 @@ */ struct ap_matrix_dev { struct device device; - atomic_t available_instances; struct ap_config_info info; struct list_head mdev_list; struct mutex mdevs_lock; /* serializes access to each ap_matrix_mdev */ struct ap_driver *vfio_ap_drv; struct mutex guests_lock; /* serializes access to each KVM guest */ + struct mdev_parent parent; + struct mdev_type mdev_type; + struct mdev_type *mdev_types[]; }; extern struct ap_matrix_dev *matrix_dev; diff --git a/drivers/s390/net/ctcm_main.c b/drivers/s390/net/ctcm_main.c index e0fdd54bfeb70b35168a416d68f82a399c235b46..37b551bd43bffb2d85023c31520607be123e8701 100644 --- a/drivers/s390/net/ctcm_main.c +++ b/drivers/s390/net/ctcm_main.c @@ -1566,7 +1566,7 @@ static int ctcm_new_device(struct ccwgroup_device *cgdev) goto out_dev; } - strlcpy(priv->fsm->name, dev->name, sizeof(priv->fsm->name)); + strscpy(priv->fsm->name, dev->name, sizeof(priv->fsm->name)); dev_info(&dev->dev, "setup OK : r/w = %s/%s, protocol : %d\n", diff --git a/drivers/s390/net/fsm.c b/drivers/s390/net/fsm.c index 98c4864932d221f4ec12b2f1130457f540a5e2ff..0ff61d00feb19c7d6520d14d07a99aac5a4d3454 100644 --- a/drivers/s390/net/fsm.c +++ b/drivers/s390/net/fsm.c @@ -28,7 +28,7 @@ init_fsm(char *name, const char **state_names, const char **event_names, int nr_ "fsm(%s): init_fsm: Couldn't alloc instance\n", name); return NULL; } - strlcpy(this->name, name, sizeof(this->name)); + strscpy(this->name, name, sizeof(this->name)); init_waitqueue_head(&this->wait_q); f = kzalloc(sizeof(fsm), order); diff --git a/drivers/s390/net/qeth_ethtool.c b/drivers/s390/net/qeth_ethtool.c index 9eba0a32e9f97ac19abebd7963b0ce4a96c302ad..e250f49535fac586ef974481fc6e798d487f2dd9 100644 --- a/drivers/s390/net/qeth_ethtool.c +++ b/drivers/s390/net/qeth_ethtool.c @@ -188,9 +188,9 @@ static void qeth_get_drvinfo(struct net_device *dev, { struct qeth_card *card = dev->ml_priv; - strlcpy(info->driver, IS_LAYER2(card) ? "qeth_l2" : "qeth_l3", + strscpy(info->driver, IS_LAYER2(card) ? "qeth_l2" : "qeth_l3", sizeof(info->driver)); - strlcpy(info->fw_version, card->info.mcl_level, + strscpy(info->fw_version, card->info.mcl_level, sizeof(info->fw_version)); snprintf(info->bus_info, sizeof(info->bus_info), "%s/%s/%s", CARD_RDEV_ID(card), CARD_WDEV_ID(card), CARD_DDEV_ID(card)); diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index 2d4436cbcb47bbc87e95766af3760a52b54d0882..9dc935886e9f367984d68ed102195f441ceab2e6 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -1133,7 +1133,7 @@ static int qeth_l2_setup_netdev(struct qeth_card *card) PAGE_SIZE * (QDIO_MAX_ELEMENTS_PER_BUFFER - 1)); } - netif_napi_add(card->dev, &card->napi, qeth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(card->dev, &card->napi, qeth_poll); return register_netdev(card->dev); } @@ -1530,8 +1530,8 @@ static void qeth_addr_change_event(struct qeth_card *card, else INIT_DELAYED_WORK(&data->dwork, qeth_l2_dev2br_worker); data->card = card; - memcpy(&data->ac_event, hostevs, - sizeof(struct qeth_ipacmd_addr_change) + extrasize); + data->ac_event = *hostevs; + memcpy(data->ac_event.entry, hostevs->entry, extrasize); queue_delayed_work(card->event_wq, &data->dwork, 0); } diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c index 8d44bce0477abd2802a81e6e6042851c5b4aa0cf..d8487a10cd5557fca98fe2859bba23696159212e 100644 --- a/drivers/s390/net/qeth_l3_main.c +++ b/drivers/s390/net/qeth_l3_main.c @@ -1910,7 +1910,7 @@ static int qeth_l3_setup_netdev(struct qeth_card *card) netif_set_tso_max_size(card->dev, PAGE_SIZE * (QETH_MAX_BUFFER_ELEMENTS(card) - 1)); - netif_napi_add(card->dev, &card->napi, qeth_poll, NAPI_POLL_WEIGHT); + netif_napi_add(card->dev, &card->napi, qeth_poll); return register_netdev(card->dev); } diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c index fd2f1c31bd2197cc58b3b74e6e9c9d8d3b7d5440..df782646e856fa8ea11e495583b3626af039ec41 100644 --- a/drivers/s390/scsi/zfcp_aux.c +++ b/drivers/s390/scsi/zfcp_aux.c @@ -103,7 +103,7 @@ static void __init zfcp_init_device_setup(char *devstr) token = strsep(&str, ","); if (!token || strlen(token) >= ZFCP_BUS_ID_SIZE) goto err_out; - strlcpy(busid, token, ZFCP_BUS_ID_SIZE); + strscpy(busid, token, ZFCP_BUS_ID_SIZE); token = strsep(&str, ","); if (!token || kstrtoull(token, 0, (unsigned long long *) &wwpn)) diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c index b61acbb09be3ba1a0091fa3b5bc49200e149311e..77917b33987098d7be37ead7852dad74728aba80 100644 --- a/drivers/s390/scsi/zfcp_fc.c +++ b/drivers/s390/scsi/zfcp_fc.c @@ -885,7 +885,7 @@ static int zfcp_fc_gspn(struct zfcp_adapter *adapter, dev_name(&adapter->ccw_device->dev), init_utsname()->nodename); else - strlcpy(fc_host_symbolic_name(adapter->scsi_host), + strscpy(fc_host_symbolic_name(adapter->scsi_host), gspn_rsp->gspn.fp_name, FC_SYMBOLIC_NAME_SIZE); return 0; diff --git a/drivers/scsi/3w-9xxx.c b/drivers/scsi/3w-9xxx.c index cd823ff5deab261682c2693fb63e7eac016ab2a8..6cb9cca9565b9ba076c74891de6001609d793f79 100644 --- a/drivers/scsi/3w-9xxx.c +++ b/drivers/scsi/3w-9xxx.c @@ -2006,7 +2006,7 @@ static int twa_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id) retval = pci_enable_device(pdev); if (retval) { TW_PRINTK(host, TW_DRIVER, 0x34, "Failed to enable pci device"); - goto out_disable_device; + return -ENODEV; } pci_set_master(pdev); diff --git a/drivers/scsi/3w-xxxx.c b/drivers/scsi/3w-xxxx.c index a853c5497af62de98c0792af52e02da0ca31a261..ffdecb12d654c7c37c5c886647ff5c867b8fad24 100644 --- a/drivers/scsi/3w-xxxx.c +++ b/drivers/scsi/3w-xxxx.c @@ -912,7 +912,7 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a data_buffer_length_adjusted = (data_buffer_length + 511) & ~511; /* Now allocate ioctl buf memory */ - cpu_addr = dma_alloc_coherent(&tw_dev->tw_pci_dev->dev, data_buffer_length_adjusted+sizeof(TW_New_Ioctl) - 1, &dma_handle, GFP_KERNEL); + cpu_addr = dma_alloc_coherent(&tw_dev->tw_pci_dev->dev, data_buffer_length_adjusted + sizeof(TW_New_Ioctl), &dma_handle, GFP_KERNEL); if (cpu_addr == NULL) { retval = -ENOMEM; goto out; @@ -921,7 +921,7 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a tw_ioctl = (TW_New_Ioctl *)cpu_addr; /* Now copy down the entire ioctl */ - if (copy_from_user(tw_ioctl, argp, data_buffer_length + sizeof(TW_New_Ioctl) - 1)) + if (copy_from_user(tw_ioctl, argp, data_buffer_length + sizeof(TW_New_Ioctl))) goto out2; passthru = (TW_Passthru *)&tw_ioctl->firmware_command; @@ -966,15 +966,15 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a /* Load the sg list */ switch (TW_SGL_OUT(tw_ioctl->firmware_command.opcode__sgloffset)) { case 2: - tw_ioctl->firmware_command.byte8.param.sgl[0].address = dma_handle + sizeof(TW_New_Ioctl) - 1; + tw_ioctl->firmware_command.byte8.param.sgl[0].address = dma_handle + sizeof(TW_New_Ioctl); tw_ioctl->firmware_command.byte8.param.sgl[0].length = data_buffer_length_adjusted; break; case 3: - tw_ioctl->firmware_command.byte8.io.sgl[0].address = dma_handle + sizeof(TW_New_Ioctl) - 1; + tw_ioctl->firmware_command.byte8.io.sgl[0].address = dma_handle + sizeof(TW_New_Ioctl); tw_ioctl->firmware_command.byte8.io.sgl[0].length = data_buffer_length_adjusted; break; case 5: - passthru->sg_list[0].address = dma_handle + sizeof(TW_New_Ioctl) - 1; + passthru->sg_list[0].address = dma_handle + sizeof(TW_New_Ioctl); passthru->sg_list[0].length = data_buffer_length_adjusted; break; } @@ -1017,12 +1017,12 @@ static long tw_chrdev_ioctl(struct file *file, unsigned int cmd, unsigned long a } /* Now copy the response to userspace */ - if (copy_to_user(argp, tw_ioctl, sizeof(TW_New_Ioctl) + data_buffer_length - 1)) + if (copy_to_user(argp, tw_ioctl, sizeof(TW_New_Ioctl) + data_buffer_length)) goto out2; retval = 0; out2: /* Now free ioctl buf memory */ - dma_free_coherent(&tw_dev->tw_pci_dev->dev, data_buffer_length_adjusted+sizeof(TW_New_Ioctl) - 1, cpu_addr, dma_handle); + dma_free_coherent(&tw_dev->tw_pci_dev->dev, data_buffer_length_adjusted + sizeof(TW_New_Ioctl), cpu_addr, dma_handle); out: mutex_unlock(&tw_dev->ioctl_lock); mutex_unlock(&tw_mutex); diff --git a/drivers/scsi/3w-xxxx.h b/drivers/scsi/3w-xxxx.h index e8f3f081b7d8d06efe1cdc726723b64b6fb25808..120a087bdf3cbcc089ad39a0a9ab275415bf7091 100644 --- a/drivers/scsi/3w-xxxx.h +++ b/drivers/scsi/3w-xxxx.h @@ -348,7 +348,7 @@ typedef struct TAG_TW_New_Ioctl { unsigned int data_buffer_length; unsigned char padding [508]; TW_Command firmware_command; - char data_buffer[1]; + char data_buffer[]; } TW_New_Ioctl; /* GetParam descriptor */ diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 955cb69a54181560f9730bc7cf4debc79a8b2bc7..03e71e3d5e5b34fd580dcad9e824f5745eaa2969 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -2,9 +2,10 @@ menu "SCSI device support" config SCSI_MOD - tristate - default y if SCSI=n || SCSI=y - default m if SCSI=m + tristate + default y if SCSI=n || SCSI=y + default m if SCSI=m + depends on BLOCK config RAID_ATTRS tristate "RAID Transport Class" diff --git a/drivers/scsi/aic7xxx/aic79xx_osm.c b/drivers/scsi/aic7xxx/aic79xx_osm.c index 928099163f0f6f9fbb40322b723fff7ad13204e0..f2f3405cdec5e9a69f0e3f943d2482fbca382572 100644 --- a/drivers/scsi/aic7xxx/aic79xx_osm.c +++ b/drivers/scsi/aic7xxx/aic79xx_osm.c @@ -194,7 +194,7 @@ struct ahd_linux_iocell_opts #define AIC79XX_PRECOMP_INDEX 0 #define AIC79XX_SLEWRATE_INDEX 1 #define AIC79XX_AMPLITUDE_INDEX 2 -static const struct ahd_linux_iocell_opts aic79xx_iocell_info[] = +static struct ahd_linux_iocell_opts aic79xx_iocell_info[] __ro_after_init = { AIC79XX_DEFAULT_IOOPTS, AIC79XX_DEFAULT_IOOPTS, diff --git a/drivers/scsi/csiostor/csio_scsi.c b/drivers/scsi/csiostor/csio_scsi.c index 9aafe0002ab12cc4c76880673aef9d79212d77a1..05e1a63e00c3a11266abca93910879a20ca7c7aa 100644 --- a/drivers/scsi/csiostor/csio_scsi.c +++ b/drivers/scsi/csiostor/csio_scsi.c @@ -1366,9 +1366,9 @@ csio_show_hw_state(struct device *dev, struct csio_hw *hw = csio_lnode_to_hw(ln); if (csio_is_hw_ready(hw)) - return snprintf(buf, PAGE_SIZE, "ready\n"); - else - return snprintf(buf, PAGE_SIZE, "not ready\n"); + return sysfs_emit(buf, "ready\n"); + + return sysfs_emit(buf, "not ready\n"); } /* Device reset */ @@ -1430,7 +1430,7 @@ csio_show_dbg_level(struct device *dev, { struct csio_lnode *ln = shost_priv(class_to_shost(dev)); - return snprintf(buf, PAGE_SIZE, "%x\n", ln->params.log_level); + return sysfs_emit(buf, "%x\n", ln->params.log_level); } /* Store debug level */ @@ -1476,7 +1476,7 @@ csio_show_num_reg_rnodes(struct device *dev, { struct csio_lnode *ln = shost_priv(class_to_shost(dev)); - return snprintf(buf, PAGE_SIZE, "%d\n", ln->num_reg_rnodes); + return sysfs_emit(buf, "%d\n", ln->num_reg_rnodes); } static DEVICE_ATTR(num_reg_rnodes, S_IRUGO, csio_show_num_reg_rnodes, NULL); diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c index 53d91bf9c12a89e0a9b4f2705a248515d2c82868..c07d2e3b4bcff71d41f43eac18596824ecef2ada 100644 --- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c +++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c @@ -254,7 +254,7 @@ static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, } else if (is_t5(lldi->adapter_type)) { struct cpl_t5_act_open_req *req = (struct cpl_t5_act_open_req *)skb->head; - u32 isn = (prandom_u32() & ~7UL) - 1; + u32 isn = (get_random_u32() & ~7UL) - 1; INIT_TP_WR(req, 0); OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, @@ -282,7 +282,7 @@ static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, } else { struct cpl_t6_act_open_req *req = (struct cpl_t6_act_open_req *)skb->head; - u32 isn = (prandom_u32() & ~7UL) - 1; + u32 isn = (get_random_u32() & ~7UL) - 1; INIT_TP_WR(req, 0); OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index e7be95ee7d64473c9f957b41bc3a956ef71f2683..cd1324ec742de4d05ecffdb5d6940b33dd264845 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -132,7 +132,7 @@ static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp) break; case SISL_AFU_RC_OUT_OF_DATA_BUFS: /* Retry */ - scp->result = (DID_ALLOC_FAILURE << 16); + scp->result = (DID_ERROR << 16); break; default: scp->result = (DID_ERROR << 16); diff --git a/drivers/scsi/esas2r/atioctl.h b/drivers/scsi/esas2r/atioctl.h index ff2ad9b385757c1f59eeb28a9e3aa0001adab47e..dd3437412ffcfda94534c501f0c020c15e8125ba 100644 --- a/drivers/scsi/esas2r/atioctl.h +++ b/drivers/scsi/esas2r/atioctl.h @@ -831,6 +831,7 @@ struct __packed atto_hba_trace { u32 total_length; u32 trace_mask; u8 reserved2[48]; + u8 contents[]; }; #define ATTO_FUNC_SCSI_PASS_THRU 0x04 diff --git a/drivers/scsi/esas2r/esas2r_ioctl.c b/drivers/scsi/esas2r/esas2r_ioctl.c index 08f4e43c7d9ee7681d81ab529465635ea684f81d..e003d923acbff6777452ecc10088c212c58ddf92 100644 --- a/drivers/scsi/esas2r/esas2r_ioctl.c +++ b/drivers/scsi/esas2r/esas2r_ioctl.c @@ -947,10 +947,9 @@ static int hba_ioctl_callback(struct esas2r_adapter *a, break; } - memcpy(trc + 1, + memcpy(trc->contents, a->fw_coredump_buff + offset, len); - hi->data_length = len; } else if (trc->trace_func == ATTO_TRC_TF_RESET) { memset(a->fw_coredump_buff, 0, diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c index 39e16eab47aad4d4671ec7dbf58628435d267dbf..ddc048069af25aeac8680512e4bae041bfef9c2b 100644 --- a/drivers/scsi/fcoe/fcoe_ctlr.c +++ b/drivers/scsi/fcoe/fcoe_ctlr.c @@ -2233,7 +2233,7 @@ static void fcoe_ctlr_vn_restart(struct fcoe_ctlr *fip) if (fip->probe_tries < FIP_VN_RLIM_COUNT) { fip->probe_tries++; - wait = prandom_u32() % FIP_VN_PROBE_WAIT; + wait = prandom_u32_max(FIP_VN_PROBE_WAIT); } else wait = FIP_VN_RLIM_INT; mod_timer(&fip->timer, jiffies + msecs_to_jiffies(wait)); @@ -3125,7 +3125,7 @@ static void fcoe_ctlr_vn_timeout(struct fcoe_ctlr *fip) fcoe_all_vn2vn, 0); fip->port_ka_time = jiffies + msecs_to_jiffies(FIP_VN_BEACON_INT + - (prandom_u32() % FIP_VN_BEACON_FUZZ)); + prandom_u32_max(FIP_VN_BEACON_FUZZ)); } if (time_before(fip->port_ka_time, next_time)) next_time = fip->port_ka_time; diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h index 24c83bc4f5dc8c8802acff4168e067979f3d829a..9aebf4a26b132df449adc812426f393e370d41dc 100644 --- a/drivers/scsi/hisi_sas/hisi_sas.h +++ b/drivers/scsi/hisi_sas/hisi_sas.h @@ -649,6 +649,7 @@ extern void hisi_sas_phy_enable(struct hisi_hba *hisi_hba, int phy_no, int enable); extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy, gfp_t gfp_flags); +extern void hisi_sas_phy_bcast(struct hisi_sas_phy *phy); extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba, struct sas_task *task, struct hisi_sas_slot *slot); diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c index 33af5b8dede206725e278263ef0913d80541c6e7..699b07abb6b0b03e460226959fe5c31a8972e23d 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_main.c +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c @@ -1341,6 +1341,7 @@ static void hisi_sas_refresh_port_id(struct hisi_hba *hisi_hba) static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state) { + struct sas_ha_struct *sas_ha = &hisi_hba->sha; struct asd_sas_port *_sas_port = NULL; int phy_no; @@ -1369,6 +1370,12 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 state) hisi_sas_phy_down(hisi_hba, phy_no, 0, GFP_KERNEL); } } + /* + * Ensure any bcast events are processed prior to calling async nexus + * reset calls from hisi_sas_clear_nexus_ha() -> + * hisi_sas_async_I_T_nexus_reset() + */ + sas_drain_work(sas_ha); } static void hisi_sas_reset_init_all_devices(struct hisi_hba *hisi_hba) @@ -1527,9 +1534,9 @@ static int hisi_sas_controller_reset(struct hisi_hba *hisi_hba) clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags); return rc; } + clear_bit(HISI_SAS_HW_FAULT_BIT, &hisi_hba->flags); hisi_sas_controller_reset_done(hisi_hba); - clear_bit(HISI_SAS_HW_FAULT_BIT, &hisi_hba->flags); dev_info(dev, "controller reset complete\n"); return 0; @@ -1816,12 +1823,14 @@ static int hisi_sas_clear_nexus_ha(struct sas_ha_struct *sas_ha) struct hisi_hba *hisi_hba = sas_ha->lldd_ha; HISI_SAS_DECLARE_RST_WORK_ON_STACK(r); ASYNC_DOMAIN_EXCLUSIVE(async); - int i; + int i, ret; queue_work(hisi_hba->wq, &r.work); wait_for_completion(r.completion); - if (!r.done) - return TMF_RESP_FUNC_FAILED; + if (!r.done) { + ret = TMF_RESP_FUNC_FAILED; + goto out; + } for (i = 0; i < HISI_SAS_MAX_DEVICES; i++) { struct hisi_sas_device *sas_dev = &hisi_hba->devices[i]; @@ -1838,7 +1847,9 @@ static int hisi_sas_clear_nexus_ha(struct sas_ha_struct *sas_ha) async_synchronize_full_domain(&async); hisi_sas_release_tasks(hisi_hba); - return TMF_RESP_FUNC_COMPLETE; + ret = TMF_RESP_FUNC_COMPLETE; +out: + return ret; } static int hisi_sas_query_task(struct sas_task *task) @@ -1982,6 +1993,22 @@ void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy, } EXPORT_SYMBOL_GPL(hisi_sas_phy_down); +void hisi_sas_phy_bcast(struct hisi_sas_phy *phy) +{ + struct asd_sas_phy *sas_phy = &phy->sas_phy; + struct hisi_hba *hisi_hba = phy->hisi_hba; + struct sas_ha_struct *sha = &hisi_hba->sha; + + if (test_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags)) + return; + + if (test_bit(SAS_HA_FROZEN, &sha->state)) + return; + + sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(hisi_sas_phy_bcast); + void hisi_sas_sync_irqs(struct hisi_hba *hisi_hba) { int i; diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c index 349546bacb2b0c7d46ad7098c5cff5aebcde35c0..d643c5a49aa94d6929143e26a3d85fd35fc85d72 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c @@ -1412,9 +1412,7 @@ static irqreturn_t int_bcast_v1_hw(int irq, void *p) goto end; } - if (!test_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags)) - sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, - GFP_ATOMIC); + hisi_sas_phy_bcast(phy); end: hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT2, diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c index 70e401fd432a04d46b73ed84ae8c29451044a1f1..cded42f4ca44555a6aba11ff8c74910b23fd90df 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c @@ -2811,15 +2811,12 @@ static irqreturn_t int_phy_updown_v2_hw(int irq_no, void *p) static void phy_bcast_v2_hw(int phy_no, struct hisi_hba *hisi_hba) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; - struct asd_sas_phy *sas_phy = &phy->sas_phy; u32 bcast_status; hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS); - if ((bcast_status & RX_BCAST_CHG_MSK) && - !test_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags)) - sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, - GFP_ATOMIC); + if (bcast_status & RX_BCAST_CHG_MSK) + hisi_sas_phy_bcast(phy); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_RX_BCST_ACK_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); @@ -3537,7 +3534,7 @@ static struct attribute *host_v2_hw_attrs[] = { ATTRIBUTE_GROUPS(host_v2_hw); -static int map_queues_v2_hw(struct Scsi_Host *shost) +static void map_queues_v2_hw(struct Scsi_Host *shost) { struct hisi_hba *hisi_hba = shost_priv(shost); struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT]; @@ -3552,9 +3549,6 @@ static int map_queues_v2_hw(struct Scsi_Host *shost) for_each_cpu(cpu, mask) qmap->mq_map[cpu] = qmap->queue_offset + queue; } - - return 0; - } static struct scsi_host_template sht_v2_hw = { diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index efe8c5be58702c2cc806db9519e5003c96629a3a..d56b4bfd276781770f3f9d55af0694972ab04e8c 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -1626,15 +1626,12 @@ static irqreturn_t phy_down_v3_hw(int phy_no, struct hisi_hba *hisi_hba) static irqreturn_t phy_bcast_v3_hw(int phy_no, struct hisi_hba *hisi_hba) { struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no]; - struct asd_sas_phy *sas_phy = &phy->sas_phy; u32 bcast_status; hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 1); bcast_status = hisi_sas_phy_read32(hisi_hba, phy_no, RX_PRIMS_STATUS); - if ((bcast_status & RX_BCAST_CHG_MSK) && - !test_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags)) - sas_notify_port_event(sas_phy, PORTE_BROADCAST_RCVD, - GFP_ATOMIC); + if (bcast_status & RX_BCAST_CHG_MSK) + hisi_sas_phy_bcast(phy); hisi_sas_phy_write32(hisi_hba, phy_no, CHL_INT0, CHL_INT0_SL_RX_BCST_ACK_MSK); hisi_sas_phy_write32(hisi_hba, phy_no, SL_RX_BCAST_CHK_MSK, 0); @@ -2786,7 +2783,6 @@ static int slave_configure_v3_hw(struct scsi_device *sdev) struct hisi_hba *hisi_hba = shost_priv(shost); int ret = hisi_sas_slave_configure(sdev); struct device *dev = hisi_hba->dev; - unsigned int max_sectors; if (ret) return ret; @@ -2802,12 +2798,6 @@ static int slave_configure_v3_hw(struct scsi_device *sdev) } } - /* Set according to IOMMU IOVA caching limit */ - max_sectors = min_t(size_t, queue_max_hw_sectors(sdev->request_queue), - (PAGE_SIZE * 32) >> SECTOR_SHIFT); - - blk_queue_max_hw_sectors(sdev->request_queue, max_sectors); - return 0; } @@ -3171,13 +3161,12 @@ static int debugfs_set_bist_v3_hw(struct hisi_hba *hisi_hba, bool enable) return 0; } -static int hisi_sas_map_queues(struct Scsi_Host *shost) +static void hisi_sas_map_queues(struct Scsi_Host *shost) { struct hisi_hba *hisi_hba = shost_priv(shost); struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT]; - return blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev, - BASE_VECTORS_V3_HW); + blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev, BASE_VECTORS_V3_HW); } static struct scsi_host_template sht_v3_hw = { diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 0738238ed6cc40e7313adadbcaaa8757bc76b650..9857dba09c951a68cfd94965191fb7dd30972d70 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -182,6 +182,15 @@ void scsi_remove_host(struct Scsi_Host *shost) mutex_unlock(&shost->scan_mutex); scsi_proc_host_rm(shost); + /* + * New SCSI devices cannot be attached anymore because of the SCSI host + * state so drop the tag set refcnt. Wait until the tag set refcnt drops + * to zero because .exit_cmd_priv implementations may need the host + * pointer. + */ + kref_put(&shost->tagset_refcnt, scsi_mq_free_tags); + wait_for_completion(&shost->tagset_freed); + spin_lock_irqsave(shost->host_lock, flags); if (scsi_host_set_state(shost, SHOST_DEL)) BUG_ON(scsi_host_set_state(shost, SHOST_DEL_RECOVERY)); @@ -190,15 +199,6 @@ void scsi_remove_host(struct Scsi_Host *shost) transport_unregister_device(&shost->shost_gendev); device_unregister(&shost->shost_dev); device_del(&shost->shost_gendev); - - /* - * After scsi_remove_host() has returned the scsi LLD module can be - * unloaded and/or the host resources can be released. Hence wait until - * the dependent SCSI targets and devices are gone before returning. - */ - wait_event(shost->targets_wq, atomic_read(&shost->target_count) == 0); - - scsi_mq_destroy_tags(shost); } EXPORT_SYMBOL(scsi_remove_host); @@ -254,6 +254,9 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, if (error) goto fail; + kref_init(&shost->tagset_refcnt); + init_completion(&shost->tagset_freed); + /* * Increase usage count temporarily here so that calling * scsi_autopm_put_host() will trigger runtime idle if there is @@ -309,8 +312,8 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, return error; /* - * Any resources associated with the SCSI host in this function except - * the tag set will be freed by scsi_host_dev_release(). + * Any host allocation in this function will be freed in + * scsi_host_dev_release(). */ out_del_dev: device_del(&shost->shost_dev); @@ -326,7 +329,7 @@ int scsi_add_host_with_dma(struct Scsi_Host *shost, struct device *dev, pm_runtime_disable(&shost->shost_gendev); pm_runtime_set_suspended(&shost->shost_gendev); pm_runtime_put_noidle(&shost->shost_gendev); - scsi_mq_destroy_tags(shost); + kref_put(&shost->tagset_refcnt, scsi_mq_free_tags); fail: return error; } @@ -406,7 +409,6 @@ struct Scsi_Host *scsi_host_alloc(struct scsi_host_template *sht, int privsize) INIT_LIST_HEAD(&shost->starved_list); init_waitqueue_head(&shost->host_wait); mutex_init(&shost->scan_mutex); - init_waitqueue_head(&shost->targets_wq); index = ida_alloc(&host_index_ida, GFP_KERNEL); if (index < 0) { diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index a47bcce3c9c78dc1b4c5d31b18f3672f6dffb962..f8e832b1bc46a716b37f6f1fa5a168fcdd926d3c 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -6233,8 +6233,7 @@ static struct CommandList *cmd_alloc(struct ctlr_info *h) offset = (i + 1) % HPSA_NRESERVED_CMDS; continue; } - set_bit(i & (BITS_PER_LONG - 1), - h->cmd_pool_bits + (i / BITS_PER_LONG)); + set_bit(i, h->cmd_pool_bits); break; /* it's ours now. */ } hpsa_cmd_partial_init(h, i, c); @@ -6261,8 +6260,7 @@ static void cmd_free(struct ctlr_info *h, struct CommandList *c) int i; i = c - h->cmd_pool; - clear_bit(i & (BITS_PER_LONG - 1), - h->cmd_pool_bits + (i / BITS_PER_LONG)); + clear_bit(i, h->cmd_pool_bits); } } @@ -8030,7 +8028,7 @@ out_disable: static void hpsa_free_cmd_pool(struct ctlr_info *h) { - kfree(h->cmd_pool_bits); + bitmap_free(h->cmd_pool_bits); h->cmd_pool_bits = NULL; if (h->cmd_pool) { dma_free_coherent(&h->pdev->dev, @@ -8052,9 +8050,7 @@ static void hpsa_free_cmd_pool(struct ctlr_info *h) static int hpsa_alloc_cmd_pool(struct ctlr_info *h) { - h->cmd_pool_bits = kcalloc(DIV_ROUND_UP(h->nr_cmds, BITS_PER_LONG), - sizeof(unsigned long), - GFP_KERNEL); + h->cmd_pool_bits = bitmap_zalloc(h->nr_cmds, GFP_KERNEL); h->cmd_pool = dma_alloc_coherent(&h->pdev->dev, h->nr_cmds * sizeof(*h->cmd_pool), &h->cmd_pool_dhandle, GFP_KERNEL); diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c index f18b770626e65ed426e013b6a9ec970202ffa152..7e89037182458d0eb275b2dbd92e48d4bae9de5c 100644 --- a/drivers/scsi/hptiop.c +++ b/drivers/scsi/hptiop.c @@ -1044,10 +1044,7 @@ static int hptiop_queuecommand_lck(struct scsi_cmnd *scp) req->channel = scp->device->channel; req->target = scp->device->id; req->lun = scp->device->lun; - req->header.size = cpu_to_le32( - sizeof(struct hpt_iop_request_scsi_command) - - sizeof(struct hpt_iopsg) - + sg_count * sizeof(struct hpt_iopsg)); + req->header.size = cpu_to_le32(struct_size(req, sg_list, sg_count)); memcpy(req->cdb, scp->cmnd, sizeof(req->cdb)); hba->ops->post_req(hba, _req); @@ -1397,8 +1394,8 @@ static int hptiop_probe(struct pci_dev *pcidev, const struct pci_device_id *id) host->cmd_per_lun = le32_to_cpu(iop_config.max_requests); host->max_cmd_len = 16; - req_size = sizeof(struct hpt_iop_request_scsi_command) - + sizeof(struct hpt_iopsg) * (hba->max_sg_descriptors - 1); + req_size = struct_size((struct hpt_iop_request_scsi_command *)0, + sg_list, hba->max_sg_descriptors); if ((req_size & 0x1f) != 0) req_size = (req_size + 0x1f) & ~0x1f; diff --git a/drivers/scsi/hptiop.h b/drivers/scsi/hptiop.h index 363d5a16243f4e264bcb38de3ab13f033b798a27..394ef6aa469e89f4cb4b110f31ba2aa77163f1f2 100644 --- a/drivers/scsi/hptiop.h +++ b/drivers/scsi/hptiop.h @@ -228,7 +228,7 @@ struct hpt_iop_request_scsi_command { u8 pad1; u8 cdb[16]; __le32 dataxfer_length; - struct hpt_iopsg sg_list[1]; + struct hpt_iopsg sg_list[]; }; struct hpt_iop_request_ioctl_command { @@ -237,7 +237,7 @@ struct hpt_iop_request_ioctl_command { __le32 inbuf_size; __le32 outbuf_size; __le32 bytes_returned; - u8 buf[1]; + u8 buf[]; /* out data should be put at buf[(inbuf_size+3)&~3] */ }; diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c index eee1a24f7e15e8c8ac6700d5972b8b439823fe9f..e8770310a64b39e3d0d8b867154ddda2fa9bce67 100644 --- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c +++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c @@ -444,7 +444,7 @@ static void ibmvscsis_disconnect(struct work_struct *work) break; /* - * Can transition from this state to to unconfiguring + * Can transition from this state to unconfiguring * or err disconnect. */ case ERR_DISCONNECT_RECONNECT: diff --git a/drivers/scsi/initio.c b/drivers/scsi/initio.c index f585d6e5fab928f5bc854f6f6db98fd51771a1f1..375261d6761971f46b0db5f7696564d9ba110be2 100644 --- a/drivers/scsi/initio.c +++ b/drivers/scsi/initio.c @@ -1166,7 +1166,7 @@ static void tulip_scsi(struct initio_host * host) return; } if (host->jsint & (TSS_FUNC_COMP | TSS_BUS_SERV)) { /* func complete or Bus service */ - if ((scb = host->active) != NULL) + if (host->active) initio_next_state(host); return; } diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c index 29b1bd755afec7e0a8e336251f876b123c6f0d03..5fb1f364e8155d7a3c857cbff10ad02ca9c89c33 100644 --- a/drivers/scsi/iscsi_tcp.c +++ b/drivers/scsi/iscsi_tcp.c @@ -595,6 +595,8 @@ iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session, INIT_WORK(&conn->recvwork, iscsi_sw_tcp_recv_data_work); tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q; + mutex_init(&tcp_sw_conn->sock_lock); + tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) goto free_conn; @@ -629,11 +631,15 @@ free_conn: static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn) { - struct iscsi_session *session = conn->session; struct iscsi_tcp_conn *tcp_conn = conn->dd_data; struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; struct socket *sock = tcp_sw_conn->sock; + /* + * The iscsi transport class will make sure we are not called in + * parallel with start, stop, bind and destroys. However, this can be + * called twice if userspace does a stop then a destroy. + */ if (!sock) return; @@ -649,9 +655,9 @@ static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn) iscsi_suspend_rx(conn); - spin_lock_bh(&session->frwd_lock); + mutex_lock(&tcp_sw_conn->sock_lock); tcp_sw_conn->sock = NULL; - spin_unlock_bh(&session->frwd_lock); + mutex_unlock(&tcp_sw_conn->sock_lock); sockfd_put(sock); } @@ -703,7 +709,6 @@ iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session, struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, int is_leading) { - struct iscsi_session *session = cls_session->dd_data; struct iscsi_conn *conn = cls_conn->dd_data; struct iscsi_tcp_conn *tcp_conn = conn->dd_data; struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; @@ -723,10 +728,10 @@ iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session, if (err) goto free_socket; - spin_lock_bh(&session->frwd_lock); + mutex_lock(&tcp_sw_conn->sock_lock); /* bind iSCSI connection and socket */ tcp_sw_conn->sock = sock; - spin_unlock_bh(&session->frwd_lock); + mutex_unlock(&tcp_sw_conn->sock_lock); /* setup Socket parameters */ sk = sock->sk; @@ -763,8 +768,15 @@ static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn, break; case ISCSI_PARAM_DATADGST_EN: iscsi_set_param(cls_conn, param, buf, buflen); + + mutex_lock(&tcp_sw_conn->sock_lock); + if (!tcp_sw_conn->sock) { + mutex_unlock(&tcp_sw_conn->sock_lock); + return -ENOTCONN; + } tcp_sw_conn->sendpage = conn->datadgst_en ? sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage; + mutex_unlock(&tcp_sw_conn->sock_lock); break; case ISCSI_PARAM_MAX_R2T: return iscsi_tcp_set_max_r2t(conn, buf); @@ -779,8 +791,8 @@ static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param, char *buf) { struct iscsi_conn *conn = cls_conn->dd_data; - struct iscsi_tcp_conn *tcp_conn = conn->dd_data; - struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; + struct iscsi_sw_tcp_conn *tcp_sw_conn; + struct iscsi_tcp_conn *tcp_conn; struct sockaddr_in6 addr; struct socket *sock; int rc; @@ -790,21 +802,36 @@ static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, case ISCSI_PARAM_CONN_ADDRESS: case ISCSI_PARAM_LOCAL_PORT: spin_lock_bh(&conn->session->frwd_lock); - if (!tcp_sw_conn || !tcp_sw_conn->sock) { + if (!conn->session->leadconn) { spin_unlock_bh(&conn->session->frwd_lock); return -ENOTCONN; } - sock = tcp_sw_conn->sock; - sock_hold(sock->sk); + /* + * The conn has been setup and bound, so just grab a ref + * incase a destroy runs while we are in the net layer. + */ + iscsi_get_conn(conn->cls_conn); spin_unlock_bh(&conn->session->frwd_lock); + tcp_conn = conn->dd_data; + tcp_sw_conn = tcp_conn->dd_data; + + mutex_lock(&tcp_sw_conn->sock_lock); + sock = tcp_sw_conn->sock; + if (!sock) { + rc = -ENOTCONN; + goto sock_unlock; + } + if (param == ISCSI_PARAM_LOCAL_PORT) rc = kernel_getsockname(sock, (struct sockaddr *)&addr); else rc = kernel_getpeername(sock, (struct sockaddr *)&addr); - sock_put(sock->sk); +sock_unlock: + mutex_unlock(&tcp_sw_conn->sock_lock); + iscsi_put_conn(conn->cls_conn); if (rc < 0) return rc; @@ -842,17 +869,21 @@ static int iscsi_sw_tcp_host_get_param(struct Scsi_Host *shost, } tcp_conn = conn->dd_data; tcp_sw_conn = tcp_conn->dd_data; - sock = tcp_sw_conn->sock; - if (!sock) { - spin_unlock_bh(&session->frwd_lock); - return -ENOTCONN; - } - sock_hold(sock->sk); + /* + * The conn has been setup and bound, so just grab a ref + * incase a destroy runs while we are in the net layer. + */ + iscsi_get_conn(conn->cls_conn); spin_unlock_bh(&session->frwd_lock); - rc = kernel_getsockname(sock, - (struct sockaddr *)&addr); - sock_put(sock->sk); + mutex_lock(&tcp_sw_conn->sock_lock); + sock = tcp_sw_conn->sock; + if (!sock) + rc = -ENOTCONN; + else + rc = kernel_getsockname(sock, (struct sockaddr *)&addr); + mutex_unlock(&tcp_sw_conn->sock_lock); + iscsi_put_conn(conn->cls_conn); if (rc < 0) return rc; diff --git a/drivers/scsi/iscsi_tcp.h b/drivers/scsi/iscsi_tcp.h index 850a018aefb9bb6c41aed65f047f42834e751641..68e14a344904f955960b194497db8f5c12883b12 100644 --- a/drivers/scsi/iscsi_tcp.h +++ b/drivers/scsi/iscsi_tcp.h @@ -28,6 +28,9 @@ struct iscsi_sw_tcp_send { struct iscsi_sw_tcp_conn { struct socket *sock; + /* Taken when accessing the sock from the netlink/sysfs interface */ + struct mutex sock_lock; + struct work_struct recvwork; bool queue_recv; diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c index fa2209080cc264815c8520e3c04435b280285429..5ce2518301040a7ce660787a08e717bcd2616680 100644 --- a/drivers/scsi/libsas/sas_expander.c +++ b/drivers/scsi/libsas/sas_expander.c @@ -67,7 +67,7 @@ static int smp_execute_task_sg(struct domain_device *dev, res = i->dft->lldd_execute_task(task, GFP_KERNEL); if (res) { - del_timer(&task->slow_task->timer); + del_timer_sync(&task->slow_task->timer); pr_notice("executing SMP task failed:%d\n", res); break; } diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c index 9c82e5dc4fcc4c4714b31c02400a9b06463e8b4e..a36fa1c128a8412cee130250ac4cf1f456913641 100644 --- a/drivers/scsi/libsas/sas_scsi_host.c +++ b/drivers/scsi/libsas/sas_scsi_host.c @@ -872,7 +872,8 @@ int sas_change_queue_depth(struct scsi_device *sdev, int depth) struct domain_device *dev = sdev_to_domain_dev(sdev); if (dev_is_sata(dev)) - return __ata_change_queue_depth(dev->sata_dev.ap, sdev, depth); + return ata_change_queue_depth(dev->sata_dev.ap, + sas_to_ata_dev(dev), sdev, depth); if (!sdev->tagged_supported) depth = 1; diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index e6a083d098a1a5661c9f76c8f43f0a32af1abe55..9ad233b40a9e2e660a82bd016b20f8be78ae5a6a 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -68,8 +68,6 @@ struct lpfc_sli2_slim; #define LPFC_MIN_TGT_QDEPTH 10 #define LPFC_MAX_TGT_QDEPTH 0xFFFF -#define LPFC_MAX_BUCKET_COUNT 20 /* Maximum no. of buckets for stat data - collection. */ /* * Following time intervals are used of adjusting SCSI device * queue depths when there are driver resource error or Firmware @@ -405,6 +403,7 @@ struct lpfc_trunk_link { link1, link2, link3; + u32 phy_lnk_speed; }; /* Format of congestion module parameters */ @@ -732,8 +731,6 @@ struct lpfc_vport { struct lpfc_debugfs_trc *disc_trc; atomic_t disc_trc_cnt; #endif - uint8_t stat_data_enabled; - uint8_t stat_data_blocked; struct list_head rcv_buffer_list; unsigned long rcv_buffer_time_stamp; uint32_t vport_flag; @@ -1436,13 +1433,6 @@ struct lpfc_hba { */ #define QUE_BUFTAG_BIT (1<<31) uint32_t buffer_tag_count; - /* data structure used for latency data collection */ -#define LPFC_NO_BUCKET 0 -#define LPFC_LINEAR_BUCKET 1 -#define LPFC_POWER2_BUCKET 2 - uint8_t bucket_type; - uint32_t bucket_base; - uint32_t bucket_step; /* Maximum number of events that can be outstanding at any time*/ #define LPFC_MAX_EVT_COUNT 512 @@ -1564,16 +1554,13 @@ struct lpfc_hba { /* cgn_reg_signal and cgn_init_reg_signal use * enum fc_edc_cg_signal_cap_types */ - u16 cgn_fpin_frequency; + u16 cgn_fpin_frequency; /* In units of msecs */ #define LPFC_FPIN_INIT_FREQ 0xffff u32 cgn_sig_freq; u32 cgn_acqe_cnt; /* RX monitor handling for CMF */ - struct rxtable_entry *rxtable; /* RX_monitor information */ - atomic_t rxtable_idx_head; -#define LPFC_RXMONITOR_TABLE_IN_USE (LPFC_MAX_RXMONITOR_ENTRY + 73) - atomic_t rxtable_idx_tail; + struct lpfc_rx_info_monitor *rx_monitor; atomic_t rx_max_read_cnt; /* Maximum read bytes */ uint64_t rx_block_cnt; @@ -1610,10 +1597,11 @@ struct lpfc_hba { char os_host_name[MAXHOSTNAMELEN]; - /* SCSI host template information - for physical port */ - struct scsi_host_template port_template; - /* SCSI host template information - for all vports */ - struct scsi_host_template vport_template; + /* LD Signaling */ + u32 degrade_activate_threshold; + u32 degrade_deactivate_threshold; + u32 fec_degrade_interval; + atomic_t dbg_log_idx; atomic_t dbg_log_cnt; atomic_t dbg_log_dmping; @@ -1622,7 +1610,7 @@ struct lpfc_hba { #define LPFC_MAX_RXMONITOR_ENTRY 800 #define LPFC_MAX_RXMONITOR_DUMP 32 -struct rxtable_entry { +struct rx_info_entry { uint64_t cmf_bytes; /* Total no of read bytes for CMF_SYNC_WQE */ uint64_t total_bytes; /* Total no of read bytes requested */ uint64_t rcv_bytes; /* Total no of read bytes completed */ @@ -1637,6 +1625,13 @@ struct rxtable_entry { uint32_t timer_interval; }; +struct lpfc_rx_info_monitor { + struct rx_info_entry *ring; /* info organized in a circular buffer */ + u32 head_idx, tail_idx; /* index to head/tail of ring */ + spinlock_t lock; /* spinlock for ring */ + u32 entries; /* storing number entries/size of ring */ +}; + static inline struct Scsi_Host * lpfc_shost_from_vport(struct lpfc_vport *vport) { diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 09cf2cd0ae60d6938261c322c169f4d6e5e0232d..ef1481326fd7d37359f578a88f9c2ad8fc495533 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -4093,333 +4093,6 @@ lpfc_static_vport_show(struct device *dev, struct device_attribute *attr, */ static DEVICE_ATTR_RO(lpfc_static_vport); -/** - * lpfc_stat_data_ctrl_store - write call back for lpfc_stat_data_ctrl sysfs file - * @dev: Pointer to class device. - * @attr: Unused. - * @buf: Data buffer. - * @count: Size of the data buffer. - * - * This function get called when a user write to the lpfc_stat_data_ctrl - * sysfs file. This function parse the command written to the sysfs file - * and take appropriate action. These commands are used for controlling - * driver statistical data collection. - * Following are the command this function handles. - * - * setbucket - * = Set the latency buckets. - * destroybucket = destroy all the buckets. - * start = start data collection - * stop = stop data collection - * reset = reset the collected data - **/ -static ssize_t -lpfc_stat_data_ctrl_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct Scsi_Host *shost = class_to_shost(dev); - struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata; - struct lpfc_hba *phba = vport->phba; -#define LPFC_MAX_DATA_CTRL_LEN 1024 - static char bucket_data[LPFC_MAX_DATA_CTRL_LEN]; - unsigned long i; - char *str_ptr, *token; - struct lpfc_vport **vports; - struct Scsi_Host *v_shost; - char *bucket_type_str, *base_str, *step_str; - unsigned long base, step, bucket_type; - - if (!strncmp(buf, "setbucket", strlen("setbucket"))) { - if (strlen(buf) > (LPFC_MAX_DATA_CTRL_LEN - 1)) - return -EINVAL; - - strncpy(bucket_data, buf, LPFC_MAX_DATA_CTRL_LEN); - str_ptr = &bucket_data[0]; - /* Ignore this token - this is command token */ - token = strsep(&str_ptr, "\t "); - if (!token) - return -EINVAL; - - bucket_type_str = strsep(&str_ptr, "\t "); - if (!bucket_type_str) - return -EINVAL; - - if (!strncmp(bucket_type_str, "linear", strlen("linear"))) - bucket_type = LPFC_LINEAR_BUCKET; - else if (!strncmp(bucket_type_str, "power2", strlen("power2"))) - bucket_type = LPFC_POWER2_BUCKET; - else - return -EINVAL; - - base_str = strsep(&str_ptr, "\t "); - if (!base_str) - return -EINVAL; - base = simple_strtoul(base_str, NULL, 0); - - step_str = strsep(&str_ptr, "\t "); - if (!step_str) - return -EINVAL; - step = simple_strtoul(step_str, NULL, 0); - if (!step) - return -EINVAL; - - /* Block the data collection for every vport */ - vports = lpfc_create_vport_work_array(phba); - if (vports == NULL) - return -ENOMEM; - - for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) { - v_shost = lpfc_shost_from_vport(vports[i]); - spin_lock_irq(v_shost->host_lock); - /* Block and reset data collection */ - vports[i]->stat_data_blocked = 1; - if (vports[i]->stat_data_enabled) - lpfc_vport_reset_stat_data(vports[i]); - spin_unlock_irq(v_shost->host_lock); - } - - /* Set the bucket attributes */ - phba->bucket_type = bucket_type; - phba->bucket_base = base; - phba->bucket_step = step; - - for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) { - v_shost = lpfc_shost_from_vport(vports[i]); - - /* Unblock data collection */ - spin_lock_irq(v_shost->host_lock); - vports[i]->stat_data_blocked = 0; - spin_unlock_irq(v_shost->host_lock); - } - lpfc_destroy_vport_work_array(phba, vports); - return strlen(buf); - } - - if (!strncmp(buf, "destroybucket", strlen("destroybucket"))) { - vports = lpfc_create_vport_work_array(phba); - if (vports == NULL) - return -ENOMEM; - - for (i = 0; i <= phba->max_vports && vports[i] != NULL; i++) { - v_shost = lpfc_shost_from_vport(vports[i]); - spin_lock_irq(shost->host_lock); - vports[i]->stat_data_blocked = 1; - lpfc_free_bucket(vport); - vport->stat_data_enabled = 0; - vports[i]->stat_data_blocked = 0; - spin_unlock_irq(shost->host_lock); - } - lpfc_destroy_vport_work_array(phba, vports); - phba->bucket_type = LPFC_NO_BUCKET; - phba->bucket_base = 0; - phba->bucket_step = 0; - return strlen(buf); - } - - if (!strncmp(buf, "start", strlen("start"))) { - /* If no buckets configured return error */ - if (phba->bucket_type == LPFC_NO_BUCKET) - return -EINVAL; - spin_lock_irq(shost->host_lock); - if (vport->stat_data_enabled) { - spin_unlock_irq(shost->host_lock); - return strlen(buf); - } - lpfc_alloc_bucket(vport); - vport->stat_data_enabled = 1; - spin_unlock_irq(shost->host_lock); - return strlen(buf); - } - - if (!strncmp(buf, "stop", strlen("stop"))) { - spin_lock_irq(shost->host_lock); - if (vport->stat_data_enabled == 0) { - spin_unlock_irq(shost->host_lock); - return strlen(buf); - } - lpfc_free_bucket(vport); - vport->stat_data_enabled = 0; - spin_unlock_irq(shost->host_lock); - return strlen(buf); - } - - if (!strncmp(buf, "reset", strlen("reset"))) { - if ((phba->bucket_type == LPFC_NO_BUCKET) - || !vport->stat_data_enabled) - return strlen(buf); - spin_lock_irq(shost->host_lock); - vport->stat_data_blocked = 1; - lpfc_vport_reset_stat_data(vport); - vport->stat_data_blocked = 0; - spin_unlock_irq(shost->host_lock); - return strlen(buf); - } - return -EINVAL; -} - - -/** - * lpfc_stat_data_ctrl_show - Read function for lpfc_stat_data_ctrl sysfs file - * @dev: Pointer to class device. - * @attr: Unused. - * @buf: Data buffer. - * - * This function is the read call back function for - * lpfc_stat_data_ctrl sysfs file. This function report the - * current statistical data collection state. - **/ -static ssize_t -lpfc_stat_data_ctrl_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct Scsi_Host *shost = class_to_shost(dev); - struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata; - struct lpfc_hba *phba = vport->phba; - int index = 0; - int i; - char *bucket_type; - unsigned long bucket_value; - - switch (phba->bucket_type) { - case LPFC_LINEAR_BUCKET: - bucket_type = "linear"; - break; - case LPFC_POWER2_BUCKET: - bucket_type = "power2"; - break; - default: - bucket_type = "No Bucket"; - break; - } - - sprintf(&buf[index], "Statistical Data enabled :%d, " - "blocked :%d, Bucket type :%s, Bucket base :%d," - " Bucket step :%d\nLatency Ranges :", - vport->stat_data_enabled, vport->stat_data_blocked, - bucket_type, phba->bucket_base, phba->bucket_step); - index = strlen(buf); - if (phba->bucket_type != LPFC_NO_BUCKET) { - for (i = 0; i < LPFC_MAX_BUCKET_COUNT; i++) { - if (phba->bucket_type == LPFC_LINEAR_BUCKET) - bucket_value = phba->bucket_base + - phba->bucket_step * i; - else - bucket_value = phba->bucket_base + - (1 << i) * phba->bucket_step; - - if (index + 10 > PAGE_SIZE) - break; - sprintf(&buf[index], "%08ld ", bucket_value); - index = strlen(buf); - } - } - sprintf(&buf[index], "\n"); - return strlen(buf); -} - -/* - * Sysfs attribute to control the statistical data collection. - */ -static DEVICE_ATTR_RW(lpfc_stat_data_ctrl); - -/* - * lpfc_drvr_stat_data: sysfs attr to get driver statistical data. - */ - -/* - * Each Bucket takes 11 characters and 1 new line + 17 bytes WWN - * for each target. - */ -#define STAT_DATA_SIZE_PER_TARGET(NUM_BUCKETS) ((NUM_BUCKETS) * 11 + 18) -#define MAX_STAT_DATA_SIZE_PER_TARGET \ - STAT_DATA_SIZE_PER_TARGET(LPFC_MAX_BUCKET_COUNT) - - -/** - * sysfs_drvr_stat_data_read - Read function for lpfc_drvr_stat_data attribute - * @filp: sysfs file - * @kobj: Pointer to the kernel object - * @bin_attr: Attribute object - * @buf: Buffer pointer - * @off: File offset - * @count: Buffer size - * - * This function is the read call back function for lpfc_drvr_stat_data - * sysfs file. This function export the statistical data to user - * applications. - **/ -static ssize_t -sysfs_drvr_stat_data_read(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, - char *buf, loff_t off, size_t count) -{ - struct device *dev = container_of(kobj, struct device, - kobj); - struct Scsi_Host *shost = class_to_shost(dev); - struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata; - struct lpfc_hba *phba = vport->phba; - int i = 0, index = 0; - unsigned long nport_index; - struct lpfc_nodelist *ndlp = NULL; - nport_index = (unsigned long)off / - MAX_STAT_DATA_SIZE_PER_TARGET; - - if (!vport->stat_data_enabled || vport->stat_data_blocked - || (phba->bucket_type == LPFC_NO_BUCKET)) - return 0; - - spin_lock_irq(shost->host_lock); - list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) { - if (!ndlp->lat_data) - continue; - - if (nport_index > 0) { - nport_index--; - continue; - } - - if ((index + MAX_STAT_DATA_SIZE_PER_TARGET) - > count) - break; - - if (!ndlp->lat_data) - continue; - - /* Print the WWN */ - sprintf(&buf[index], "%02x%02x%02x%02x%02x%02x%02x%02x:", - ndlp->nlp_portname.u.wwn[0], - ndlp->nlp_portname.u.wwn[1], - ndlp->nlp_portname.u.wwn[2], - ndlp->nlp_portname.u.wwn[3], - ndlp->nlp_portname.u.wwn[4], - ndlp->nlp_portname.u.wwn[5], - ndlp->nlp_portname.u.wwn[6], - ndlp->nlp_portname.u.wwn[7]); - - index = strlen(buf); - - for (i = 0; i < LPFC_MAX_BUCKET_COUNT; i++) { - sprintf(&buf[index], "%010u,", - ndlp->lat_data[i].cmd_count); - index = strlen(buf); - } - sprintf(&buf[index], "\n"); - index = strlen(buf); - } - spin_unlock_irq(shost->host_lock); - return index; -} - -static struct bin_attribute sysfs_drvr_stat_data_attr = { - .attr = { - .name = "lpfc_drvr_stat_data", - .mode = S_IRUSR, - }, - .size = LPFC_MAX_TARGET * MAX_STAT_DATA_SIZE_PER_TARGET, - .read = sysfs_drvr_stat_data_read, - .write = NULL, -}; - /* # lpfc_link_speed: Link speed selection for initializing the Fibre Channel # connection. @@ -6273,7 +5946,6 @@ static struct attribute *lpfc_hba_attrs[] = { &dev_attr_lpfc_xlane_priority.attr, &dev_attr_lpfc_sg_seg_cnt.attr, &dev_attr_lpfc_max_scsicmpl_time.attr, - &dev_attr_lpfc_stat_data_ctrl.attr, &dev_attr_lpfc_aer_support.attr, &dev_attr_lpfc_aer_state_cleanup.attr, &dev_attr_lpfc_sriov_nr_virtfn.attr, @@ -6332,7 +6004,6 @@ static struct attribute *lpfc_vport_attrs[] = { &dev_attr_npiv_info.attr, &dev_attr_lpfc_enable_da_id.attr, &dev_attr_lpfc_max_scsicmpl_time.attr, - &dev_attr_lpfc_stat_data_ctrl.attr, &dev_attr_lpfc_static_vport.attr, &dev_attr_cmf_info.attr, NULL, @@ -6545,17 +6216,14 @@ lpfc_alloc_sysfs_attr(struct lpfc_vport *vport) struct Scsi_Host *shost = lpfc_shost_from_vport(vport); int error; - error = sysfs_create_bin_file(&shost->shost_dev.kobj, - &sysfs_drvr_stat_data_attr); - /* Virtual ports do not need ctrl_reg and mbox */ - if (error || vport->port_type == LPFC_NPIV_PORT) - goto out; + if (vport->port_type == LPFC_NPIV_PORT) + return 0; error = sysfs_create_bin_file(&shost->shost_dev.kobj, &sysfs_ctlreg_attr); if (error) - goto out_remove_stat_attr; + goto out; error = sysfs_create_bin_file(&shost->shost_dev.kobj, &sysfs_mbox_attr); @@ -6565,9 +6233,6 @@ lpfc_alloc_sysfs_attr(struct lpfc_vport *vport) return 0; out_remove_ctlreg_attr: sysfs_remove_bin_file(&shost->shost_dev.kobj, &sysfs_ctlreg_attr); -out_remove_stat_attr: - sysfs_remove_bin_file(&shost->shost_dev.kobj, - &sysfs_drvr_stat_data_attr); out: return error; } @@ -6580,8 +6245,7 @@ void lpfc_free_sysfs_attr(struct lpfc_vport *vport) { struct Scsi_Host *shost = lpfc_shost_from_vport(vport); - sysfs_remove_bin_file(&shost->shost_dev.kobj, - &sysfs_drvr_stat_data_attr); + /* Virtual ports do not need ctrl_reg and mbox */ if (vport->port_type == LPFC_NPIV_PORT) return; diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c index 9be3bb01a8ec7630ac2b435ed9a25aae810ccdda..ac0c7ccf2eaee59cfc335d672543b3abfd485bd0 100644 --- a/drivers/scsi/lpfc/lpfc_bsg.c +++ b/drivers/scsi/lpfc/lpfc_bsg.c @@ -1977,8 +1977,6 @@ lpfc_sli4_bsg_set_loopback_mode(struct lpfc_hba *phba, int mode, static int lpfc_sli4_diag_fcport_reg_setup(struct lpfc_hba *phba) { - int rc; - if (phba->pport->fc_flag & FC_VFI_REGISTERED) { lpfc_printf_log(phba, KERN_WARNING, LOG_LIBDFC, "3136 Port still had vfi registered: " @@ -1988,8 +1986,7 @@ lpfc_sli4_diag_fcport_reg_setup(struct lpfc_hba *phba) phba->vpi_ids[phba->pport->vpi]); return -EINVAL; } - rc = lpfc_issue_reg_vfi(phba->pport); - return rc; + return lpfc_issue_reg_vfi(phba->pport); } /** diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index bcad912043282b0795b6815ddc3f536fc9cc85d7..d2d207791056ca1b5938969116f96426bb5ba5cc 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -78,6 +78,7 @@ int lpfc_init_iocb_list(struct lpfc_hba *phba, int cnt); void lpfc_free_iocb_list(struct lpfc_hba *phba); int lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, struct lpfc_queue *drq, int count, int idx); +int lpfc_read_lds_params(struct lpfc_hba *phba); uint32_t lpfc_calc_cmf_latency(struct lpfc_hba *phba); void lpfc_cmf_signal_init(struct lpfc_hba *phba); void lpfc_cmf_start(struct lpfc_hba *phba); @@ -92,6 +93,14 @@ void lpfc_cgn_dump_rxmonitor(struct lpfc_hba *phba); void lpfc_cgn_update_stat(struct lpfc_hba *phba, uint32_t dtag); void lpfc_unblock_requests(struct lpfc_hba *phba); void lpfc_block_requests(struct lpfc_hba *phba); +int lpfc_rx_monitor_create_ring(struct lpfc_rx_info_monitor *rx_monitor, + u32 entries); +void lpfc_rx_monitor_destroy_ring(struct lpfc_rx_info_monitor *rx_monitor); +void lpfc_rx_monitor_record(struct lpfc_rx_info_monitor *rx_monitor, + struct rx_info_entry *entry); +u32 lpfc_rx_monitor_report(struct lpfc_hba *phba, + struct lpfc_rx_info_monitor *rx_monitor, char *buf, + u32 buf_len, u32 max_read_entries); void lpfc_mbx_cmpl_local_config_link(struct lpfc_hba *, LPFC_MBOXQ_t *); void lpfc_mbx_cmpl_reg_login(struct lpfc_hba *, LPFC_MBOXQ_t *); @@ -454,6 +463,7 @@ extern const struct attribute_group *lpfc_hba_groups[]; extern const struct attribute_group *lpfc_vport_groups[]; extern struct scsi_host_template lpfc_template; extern struct scsi_host_template lpfc_template_nvme; +extern struct scsi_host_template lpfc_vport_template; extern struct fc_function_template lpfc_transport_functions; extern struct fc_function_template lpfc_vport_transport_functions; diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 13dfe285493d18153ee0cece721cc51fcc6656fb..75fd2bfc212b240cee649fa679726252f9833905 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -1509,7 +1509,7 @@ lpfc_cmpl_ct_cmd_gft_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_sli_ct_request *CTrsp; int did; struct lpfc_nodelist *ndlp = NULL; - struct lpfc_nodelist *ns_ndlp = NULL; + struct lpfc_nodelist *ns_ndlp = cmdiocb->ndlp; uint32_t fc4_data_0, fc4_data_1; u32 ulp_status = get_job_ulpstatus(phba, rspiocb); u32 ulp_word4 = get_job_word4(phba, rspiocb); @@ -1522,15 +1522,12 @@ lpfc_cmpl_ct_cmd_gft_id(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, ulp_status, ulp_word4, did); /* Ignore response if link flipped after this request was made */ - if ((uint32_t) cmdiocb->event_tag != phba->fc_eventTag) { + if ((uint32_t)cmdiocb->event_tag != phba->fc_eventTag) { lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY, "9046 Event tag mismatch. Ignoring NS rsp\n"); goto out; } - /* Preserve the nameserver node to release the reference. */ - ns_ndlp = cmdiocb->ndlp; - if (ulp_status == IOSTAT_SUCCESS) { /* Good status, continue checking */ CTrsp = (struct lpfc_sli_ct_request *)outp->virt; @@ -2504,420 +2501,298 @@ lpfc_fdmi_change_check(struct lpfc_vport *vport) } } -/* Routines for all individual HBA attributes */ -static int -lpfc_fdmi_hba_attr_wwnn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad) +static inline int +lpfc_fdmi_set_attr_u32(void *attr, uint16_t attrtype, uint32_t attrval) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + struct lpfc_fdmi_attr_u32 *ae = attr; + int size = sizeof(*ae); - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + ae->type = cpu_to_be16(attrtype); + ae->len = cpu_to_be16(size); + ae->value_u32 = cpu_to_be32(attrval); - memcpy(&ae->un.AttrWWN, &vport->fc_sparam.nodeName, - sizeof(struct lpfc_name)); - size = FOURBYTES + sizeof(struct lpfc_name); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_NODENAME); return size; } -static int -lpfc_fdmi_hba_attr_manufacturer(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) + +static inline int +lpfc_fdmi_set_attr_wwn(void *attr, uint16_t attrtype, struct lpfc_name *wwn) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_fdmi_attr_wwn *ae = attr; + int size = sizeof(*ae); - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + ae->type = cpu_to_be16(attrtype); + ae->len = cpu_to_be16(size); + /* WWN's assumed to be bytestreams - Big Endian presentation */ + memcpy(ae->name, wwn, + min_t(size_t, sizeof(struct lpfc_name), sizeof(__be64))); - /* This string MUST be consistent with other FC platforms - * supported by Broadcom. - */ - strncpy(ae->un.AttrString, - "Emulex Corporation", - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_MANUFACTURER); return size; } -static int -lpfc_fdmi_hba_attr_sn(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad) +static inline int +lpfc_fdmi_set_attr_fullwwn(void *attr, uint16_t attrtype, + struct lpfc_name *wwnn, struct lpfc_name *wwpn) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_fdmi_attr_fullwwn *ae = attr; + u8 *nname = ae->nname; + u8 *pname = ae->pname; + int size = sizeof(*ae); - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + ae->type = cpu_to_be16(attrtype); + ae->len = cpu_to_be16(size); + /* WWN's assumed to be bytestreams - Big Endian presentation */ + memcpy(nname, wwnn, + min_t(size_t, sizeof(struct lpfc_name), sizeof(__be64))); + memcpy(pname, wwpn, + min_t(size_t, sizeof(struct lpfc_name), sizeof(__be64))); - strncpy(ae->un.AttrString, phba->SerialNumber, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_SERIAL_NUMBER); return size; } -static int -lpfc_fdmi_hba_attr_model(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +static inline int +lpfc_fdmi_set_attr_string(void *attr, uint16_t attrtype, char *attrstring) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_fdmi_attr_string *ae = attr; + int len, size; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + /* + * We are trusting the caller that if a fdmi string field + * is capped at 64 bytes, the caller passes in a string of + * 64 bytes or less. + */ - strncpy(ae->un.AttrString, phba->ModelName, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, sizeof(ae->un.AttrString)); + strncpy(ae->value_string, attrstring, sizeof(ae->value_string)); + len = strnlen(ae->value_string, sizeof(ae->value_string)); + /* round string length to a 32bit boundary. Ensure there's a NULL */ len += (len & 3) ? (4 - (len & 3)) : 4; + /* size is Type/Len (4 bytes) plus string length */ size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_MODEL); + + ae->type = cpu_to_be16(attrtype); + ae->len = cpu_to_be16(size); + return size; } -static int -lpfc_fdmi_hba_attr_description(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +/* Bitfields for FC4 Types that can be reported */ +#define ATTR_FC4_CT 0x00000001 +#define ATTR_FC4_FCP 0x00000002 +#define ATTR_FC4_NVME 0x00000004 + +static inline int +lpfc_fdmi_set_attr_fc4types(void *attr, uint16_t attrtype, uint32_t typemask) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_fdmi_attr_fc4types *ae = attr; + int size = sizeof(*ae); - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + ae->type = cpu_to_be16(attrtype); + ae->len = cpu_to_be16(size); + + if (typemask & ATTR_FC4_FCP) + ae->value_types[2] = 0x01; /* Type 0x8 - FCP */ + + if (typemask & ATTR_FC4_CT) + ae->value_types[7] = 0x01; /* Type 0x20 - CT */ + + if (typemask & ATTR_FC4_NVME) + ae->value_types[6] = 0x01; /* Type 0x28 - NVME */ - strncpy(ae->un.AttrString, phba->ModelDesc, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_MODEL_DESCRIPTION); return size; } +/* Routines for all individual HBA attributes */ static int -lpfc_fdmi_hba_attr_hdw_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_wwnn(struct lpfc_vport *vport, void *attr) { - struct lpfc_hba *phba = vport->phba; - lpfc_vpd_t *vp = &phba->vpd; - struct lpfc_fdmi_attr_entry *ae; - uint32_t i, j, incr, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - /* Convert JEDEC ID to ascii for hardware version */ - incr = vp->rev.biuRev; - for (i = 0; i < 8; i++) { - j = (incr & 0xf); - if (j <= 9) - ae->un.AttrString[7 - i] = - (char)((uint8_t) 0x30 + - (uint8_t) j); - else - ae->un.AttrString[7 - i] = - (char)((uint8_t) 0x61 + - (uint8_t) (j - 10)); - incr = (incr >> 4); - } - size = FOURBYTES + 8; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_HARDWARE_VERSION); - return size; + return lpfc_fdmi_set_attr_wwn(attr, RHBA_NODENAME, + &vport->fc_sparam.nodeName); } static int -lpfc_fdmi_hba_attr_drvr_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_manufacturer(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + /* This string MUST be consistent with other FC platforms + * supported by Broadcom. + */ + return lpfc_fdmi_set_attr_string(attr, RHBA_MANUFACTURER, + "Emulex Corporation"); +} - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); +static int +lpfc_fdmi_hba_attr_sn(struct lpfc_vport *vport, void *attr) +{ + struct lpfc_hba *phba = vport->phba; - strncpy(ae->un.AttrString, lpfc_release_version, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_DRIVER_VERSION); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_SERIAL_NUMBER, + phba->SerialNumber); } static int -lpfc_fdmi_hba_attr_rom_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_model(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - if (phba->sli_rev == LPFC_SLI_REV4) - lpfc_decode_firmware_rev(phba, ae->un.AttrString, 1); - else - strncpy(ae->un.AttrString, phba->OptionROMVersion, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_OPTION_ROM_VERSION); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_MODEL, + phba->ModelName); } static int -lpfc_fdmi_hba_attr_fmw_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_description(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - lpfc_decode_firmware_rev(phba, ae->un.AttrString, 1); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_FIRMWARE_VERSION); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_MODEL_DESCRIPTION, + phba->ModelDesc); } static int -lpfc_fdmi_hba_attr_os_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_hdw_ver(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_hba *phba = vport->phba; + lpfc_vpd_t *vp = &phba->vpd; + char buf[16] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + snprintf(buf, sizeof(buf), "%08x", vp->rev.biuRev); - snprintf(ae->un.AttrString, sizeof(ae->un.AttrString), "%s %s %s", - init_utsname()->sysname, - init_utsname()->release, - init_utsname()->version); + return lpfc_fdmi_set_attr_string(attr, RHBA_HARDWARE_VERSION, buf); +} - len = strnlen(ae->un.AttrString, sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_OS_NAME_VERSION); - return size; +static int +lpfc_fdmi_hba_attr_drvr_ver(struct lpfc_vport *vport, void *attr) +{ + return lpfc_fdmi_set_attr_string(attr, RHBA_DRIVER_VERSION, + lpfc_release_version); } static int -lpfc_fdmi_hba_attr_ct_len(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_rom_ver(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + struct lpfc_hba *phba = vport->phba; + char buf[64] = { 0 }; - ae = &ad->AttrValue; + if (phba->sli_rev == LPFC_SLI_REV4) { + lpfc_decode_firmware_rev(phba, buf, 1); - ae->un.AttrInt = cpu_to_be32(LPFC_MAX_CT_SIZE); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_MAX_CT_PAYLOAD_LEN); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_OPTION_ROM_VERSION, + buf); + } + + return lpfc_fdmi_set_attr_string(attr, RHBA_OPTION_ROM_VERSION, + phba->OptionROMVersion); } static int -lpfc_fdmi_hba_attr_symbolic_name(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_fmw_ver(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + struct lpfc_hba *phba = vport->phba; + char buf[64] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + lpfc_decode_firmware_rev(phba, buf, 1); - len = lpfc_vport_symbolic_node_name(vport, - ae->un.AttrString, 256); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_SYM_NODENAME); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_FIRMWARE_VERSION, buf); } static int -lpfc_fdmi_hba_attr_vendor_info(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_os_ver(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + char buf[256] = { 0 }; - ae = &ad->AttrValue; + snprintf(buf, sizeof(buf), "%s %s %s", + init_utsname()->sysname, + init_utsname()->release, + init_utsname()->version); - /* Nothing is defined for this currently */ - ae->un.AttrInt = cpu_to_be32(0); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_VENDOR_INFO); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_OS_NAME_VERSION, buf); } static int -lpfc_fdmi_hba_attr_num_ports(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_ct_len(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - - /* Each driver instance corresponds to a single port */ - ae->un.AttrInt = cpu_to_be32(1); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_NUM_PORTS); - return size; + return lpfc_fdmi_set_attr_u32(attr, RHBA_MAX_CT_PAYLOAD_LEN, + LPFC_MAX_CT_SIZE); } static int -lpfc_fdmi_hba_attr_fabric_wwnn(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_symbolic_name(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + char buf[256] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + lpfc_vport_symbolic_node_name(vport, buf, sizeof(buf)); - memcpy(&ae->un.AttrWWN, &vport->fabric_nodename, - sizeof(struct lpfc_name)); - size = FOURBYTES + sizeof(struct lpfc_name); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_FABRIC_WWNN); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_SYM_NODENAME, buf); } static int -lpfc_fdmi_hba_attr_bios_ver(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_vendor_info(struct lpfc_vport *vport, void *attr) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + return lpfc_fdmi_set_attr_u32(attr, RHBA_VENDOR_INFO, 0); +} - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); +static int +lpfc_fdmi_hba_attr_num_ports(struct lpfc_vport *vport, void *attr) +{ + /* Each driver instance corresponds to a single port */ + return lpfc_fdmi_set_attr_u32(attr, RHBA_NUM_PORTS, 1); +} - strlcat(ae->un.AttrString, phba->BIOSVersion, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_BIOS_VERSION); - return size; +static int +lpfc_fdmi_hba_attr_fabric_wwnn(struct lpfc_vport *vport, void *attr) +{ + return lpfc_fdmi_set_attr_wwn(attr, RHBA_FABRIC_WWNN, + &vport->fabric_nodename); } static int -lpfc_fdmi_hba_attr_bios_state(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_bios_ver(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + struct lpfc_hba *phba = vport->phba; - ae = &ad->AttrValue; + return lpfc_fdmi_set_attr_string(attr, RHBA_BIOS_VERSION, + phba->BIOSVersion); +} +static int +lpfc_fdmi_hba_attr_bios_state(struct lpfc_vport *vport, void *attr) +{ /* Driver doesn't have access to this information */ - ae->un.AttrInt = cpu_to_be32(0); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_BIOS_STATE); - return size; + return lpfc_fdmi_set_attr_u32(attr, RHBA_BIOS_STATE, 0); } static int -lpfc_fdmi_hba_attr_vendor_id(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_hba_attr_vendor_id(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - strncpy(ae->un.AttrString, "EMULEX", - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RHBA_VENDOR_ID); - return size; + return lpfc_fdmi_set_attr_string(attr, RHBA_VENDOR_ID, "EMULEX"); } -/* Routines for all individual PORT attributes */ +/* + * Routines for all individual PORT attributes + */ + static int -lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + u32 fc4types; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - ae->un.AttrTypes[2] = 0x01; /* Type 0x8 - FCP */ - ae->un.AttrTypes[7] = 0x01; /* Type 0x20 - CT */ + fc4types = (ATTR_FC4_CT | ATTR_FC4_FCP); /* Check to see if Firmware supports NVME and on physical port */ if ((phba->sli_rev == LPFC_SLI_REV4) && (vport == phba->pport) && phba->sli4_hba.pc_sli4_params.nvme) - ae->un.AttrTypes[6] = 0x01; /* Type 0x28 - NVME */ + fc4types |= ATTR_FC4_NVME; - size = FOURBYTES + 32; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SUPPORTED_FC4_TYPES); - return size; + return lpfc_fdmi_set_attr_fc4types(attr, RPRT_SUPPORTED_FC4_TYPES, + fc4types); } static int -lpfc_fdmi_port_attr_support_speed(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_support_speed(struct lpfc_vport *vport, void *attr) { - struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + struct lpfc_hba *phba = vport->phba; + u32 speeds = 0; u32 tcfg; u8 i, cnt; - ae = &ad->AttrValue; - - ae->un.AttrInt = 0; if (!(phba->hba_flag & HBA_FCOE_MODE)) { cnt = 0; if (phba->sli_rev == LPFC_SLI_REV4) { @@ -2929,539 +2804,314 @@ lpfc_fdmi_port_attr_support_speed(struct lpfc_vport *vport, if (cnt > 2) { /* 4 lane trunk group */ if (phba->lmt & LMT_64Gb) - ae->un.AttrInt |= HBA_PORTSPEED_256GFC; + speeds |= HBA_PORTSPEED_256GFC; if (phba->lmt & LMT_32Gb) - ae->un.AttrInt |= HBA_PORTSPEED_128GFC; + speeds |= HBA_PORTSPEED_128GFC; if (phba->lmt & LMT_16Gb) - ae->un.AttrInt |= HBA_PORTSPEED_64GFC; + speeds |= HBA_PORTSPEED_64GFC; } else if (cnt) { /* 2 lane trunk group */ if (phba->lmt & LMT_128Gb) - ae->un.AttrInt |= HBA_PORTSPEED_256GFC; + speeds |= HBA_PORTSPEED_256GFC; if (phba->lmt & LMT_64Gb) - ae->un.AttrInt |= HBA_PORTSPEED_128GFC; + speeds |= HBA_PORTSPEED_128GFC; if (phba->lmt & LMT_32Gb) - ae->un.AttrInt |= HBA_PORTSPEED_64GFC; + speeds |= HBA_PORTSPEED_64GFC; if (phba->lmt & LMT_16Gb) - ae->un.AttrInt |= HBA_PORTSPEED_32GFC; + speeds |= HBA_PORTSPEED_32GFC; } else { if (phba->lmt & LMT_256Gb) - ae->un.AttrInt |= HBA_PORTSPEED_256GFC; + speeds |= HBA_PORTSPEED_256GFC; if (phba->lmt & LMT_128Gb) - ae->un.AttrInt |= HBA_PORTSPEED_128GFC; + speeds |= HBA_PORTSPEED_128GFC; if (phba->lmt & LMT_64Gb) - ae->un.AttrInt |= HBA_PORTSPEED_64GFC; + speeds |= HBA_PORTSPEED_64GFC; if (phba->lmt & LMT_32Gb) - ae->un.AttrInt |= HBA_PORTSPEED_32GFC; + speeds |= HBA_PORTSPEED_32GFC; if (phba->lmt & LMT_16Gb) - ae->un.AttrInt |= HBA_PORTSPEED_16GFC; + speeds |= HBA_PORTSPEED_16GFC; if (phba->lmt & LMT_10Gb) - ae->un.AttrInt |= HBA_PORTSPEED_10GFC; + speeds |= HBA_PORTSPEED_10GFC; if (phba->lmt & LMT_8Gb) - ae->un.AttrInt |= HBA_PORTSPEED_8GFC; + speeds |= HBA_PORTSPEED_8GFC; if (phba->lmt & LMT_4Gb) - ae->un.AttrInt |= HBA_PORTSPEED_4GFC; + speeds |= HBA_PORTSPEED_4GFC; if (phba->lmt & LMT_2Gb) - ae->un.AttrInt |= HBA_PORTSPEED_2GFC; + speeds |= HBA_PORTSPEED_2GFC; if (phba->lmt & LMT_1Gb) - ae->un.AttrInt |= HBA_PORTSPEED_1GFC; + speeds |= HBA_PORTSPEED_1GFC; } } else { /* FCoE links support only one speed */ switch (phba->fc_linkspeed) { case LPFC_ASYNC_LINK_SPEED_10GBPS: - ae->un.AttrInt = HBA_PORTSPEED_10GE; + speeds = HBA_PORTSPEED_10GE; break; case LPFC_ASYNC_LINK_SPEED_25GBPS: - ae->un.AttrInt = HBA_PORTSPEED_25GE; + speeds = HBA_PORTSPEED_25GE; break; case LPFC_ASYNC_LINK_SPEED_40GBPS: - ae->un.AttrInt = HBA_PORTSPEED_40GE; + speeds = HBA_PORTSPEED_40GE; break; case LPFC_ASYNC_LINK_SPEED_100GBPS: - ae->un.AttrInt = HBA_PORTSPEED_100GE; + speeds = HBA_PORTSPEED_100GE; break; } } - ae->un.AttrInt = cpu_to_be32(ae->un.AttrInt); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SUPPORTED_SPEED); - return size; + + return lpfc_fdmi_set_attr_u32(attr, RPRT_SUPPORTED_SPEED, speeds); } static int -lpfc_fdmi_port_attr_speed(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_speed(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; + u32 speeds = 0; if (!(phba->hba_flag & HBA_FCOE_MODE)) { switch (phba->fc_linkspeed) { case LPFC_LINK_SPEED_1GHZ: - ae->un.AttrInt = HBA_PORTSPEED_1GFC; + speeds = HBA_PORTSPEED_1GFC; break; case LPFC_LINK_SPEED_2GHZ: - ae->un.AttrInt = HBA_PORTSPEED_2GFC; + speeds = HBA_PORTSPEED_2GFC; break; case LPFC_LINK_SPEED_4GHZ: - ae->un.AttrInt = HBA_PORTSPEED_4GFC; + speeds = HBA_PORTSPEED_4GFC; break; case LPFC_LINK_SPEED_8GHZ: - ae->un.AttrInt = HBA_PORTSPEED_8GFC; + speeds = HBA_PORTSPEED_8GFC; break; case LPFC_LINK_SPEED_10GHZ: - ae->un.AttrInt = HBA_PORTSPEED_10GFC; + speeds = HBA_PORTSPEED_10GFC; break; case LPFC_LINK_SPEED_16GHZ: - ae->un.AttrInt = HBA_PORTSPEED_16GFC; + speeds = HBA_PORTSPEED_16GFC; break; case LPFC_LINK_SPEED_32GHZ: - ae->un.AttrInt = HBA_PORTSPEED_32GFC; + speeds = HBA_PORTSPEED_32GFC; break; case LPFC_LINK_SPEED_64GHZ: - ae->un.AttrInt = HBA_PORTSPEED_64GFC; + speeds = HBA_PORTSPEED_64GFC; break; case LPFC_LINK_SPEED_128GHZ: - ae->un.AttrInt = HBA_PORTSPEED_128GFC; + speeds = HBA_PORTSPEED_128GFC; break; case LPFC_LINK_SPEED_256GHZ: - ae->un.AttrInt = HBA_PORTSPEED_256GFC; + speeds = HBA_PORTSPEED_256GFC; break; default: - ae->un.AttrInt = HBA_PORTSPEED_UNKNOWN; + speeds = HBA_PORTSPEED_UNKNOWN; break; } } else { switch (phba->fc_linkspeed) { case LPFC_ASYNC_LINK_SPEED_10GBPS: - ae->un.AttrInt = HBA_PORTSPEED_10GE; + speeds = HBA_PORTSPEED_10GE; break; case LPFC_ASYNC_LINK_SPEED_25GBPS: - ae->un.AttrInt = HBA_PORTSPEED_25GE; + speeds = HBA_PORTSPEED_25GE; break; case LPFC_ASYNC_LINK_SPEED_40GBPS: - ae->un.AttrInt = HBA_PORTSPEED_40GE; + speeds = HBA_PORTSPEED_40GE; break; case LPFC_ASYNC_LINK_SPEED_100GBPS: - ae->un.AttrInt = HBA_PORTSPEED_100GE; + speeds = HBA_PORTSPEED_100GE; break; default: - ae->un.AttrInt = HBA_PORTSPEED_UNKNOWN; + speeds = HBA_PORTSPEED_UNKNOWN; break; } } - ae->un.AttrInt = cpu_to_be32(ae->un.AttrInt); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_PORT_SPEED); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_PORT_SPEED, speeds); } static int -lpfc_fdmi_port_attr_max_frame(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_max_frame(struct lpfc_vport *vport, void *attr) { - struct serv_parm *hsp; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; + struct serv_parm *hsp = (struct serv_parm *)&vport->fc_sparam; - hsp = (struct serv_parm *)&vport->fc_sparam; - ae->un.AttrInt = (((uint32_t) hsp->cmn.bbRcvSizeMsb & 0x0F) << 8) | - (uint32_t) hsp->cmn.bbRcvSizeLsb; - ae->un.AttrInt = cpu_to_be32(ae->un.AttrInt); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_MAX_FRAME_SIZE); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_MAX_FRAME_SIZE, + (((uint32_t)hsp->cmn.bbRcvSizeMsb & 0x0F) << 8) | + (uint32_t)hsp->cmn.bbRcvSizeLsb); } static int -lpfc_fdmi_port_attr_os_devname(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_os_devname(struct lpfc_vport *vport, void *attr) { struct Scsi_Host *shost = lpfc_shost_from_vport(vport); - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + char buf[64] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + snprintf(buf, sizeof(buf), "/sys/class/scsi_host/host%d", + shost->host_no); - snprintf(ae->un.AttrString, sizeof(ae->un.AttrString), - "/sys/class/scsi_host/host%d", shost->host_no); - len = strnlen((char *)ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_OS_DEVICE_NAME); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_OS_DEVICE_NAME, buf); } static int -lpfc_fdmi_port_attr_host_name(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_host_name(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + char buf[64] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + scnprintf(buf, sizeof(buf), "%s", vport->phba->os_host_name); - scnprintf(ae->un.AttrString, sizeof(ae->un.AttrString), "%s", - vport->phba->os_host_name); - - len = strnlen(ae->un.AttrString, sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_HOST_NAME); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_HOST_NAME, buf); } static int -lpfc_fdmi_port_attr_wwnn(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_wwnn(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - memcpy(&ae->un.AttrWWN, &vport->fc_sparam.nodeName, - sizeof(struct lpfc_name)); - size = FOURBYTES + sizeof(struct lpfc_name); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_NODENAME); - return size; + return lpfc_fdmi_set_attr_wwn(attr, RPRT_NODENAME, + &vport->fc_sparam.nodeName); } static int -lpfc_fdmi_port_attr_wwpn(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_wwpn(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - memcpy(&ae->un.AttrWWN, &vport->fc_sparam.portName, - sizeof(struct lpfc_name)); - size = FOURBYTES + sizeof(struct lpfc_name); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_PORTNAME); - return size; + return lpfc_fdmi_set_attr_wwn(attr, RPRT_PORTNAME, + &vport->fc_sparam.portName); } static int -lpfc_fdmi_port_attr_symbolic_name(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_symbolic_name(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; + char buf[256] = { 0 }; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); + lpfc_vport_symbolic_port_name(vport, buf, sizeof(buf)); - len = lpfc_vport_symbolic_port_name(vport, ae->un.AttrString, 256); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SYM_PORTNAME); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_SYM_PORTNAME, buf); } static int -lpfc_fdmi_port_attr_port_type(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_port_type(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - ae = &ad->AttrValue; - if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) - ae->un.AttrInt = cpu_to_be32(LPFC_FDMI_PORTTYPE_NLPORT); - else - ae->un.AttrInt = cpu_to_be32(LPFC_FDMI_PORTTYPE_NPORT); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_PORT_TYPE); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_PORT_TYPE, + (phba->fc_topology == LPFC_TOPOLOGY_LOOP) ? + LPFC_FDMI_PORTTYPE_NLPORT : + LPFC_FDMI_PORTTYPE_NPORT); } static int -lpfc_fdmi_port_attr_class(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_class(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - ae->un.AttrInt = cpu_to_be32(FC_COS_CLASS2 | FC_COS_CLASS3); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SUPPORTED_CLASS); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_SUPPORTED_CLASS, + FC_COS_CLASS2 | FC_COS_CLASS3); } static int -lpfc_fdmi_port_attr_fabric_wwpn(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_fabric_wwpn(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - memcpy(&ae->un.AttrWWN, &vport->fabric_portname, - sizeof(struct lpfc_name)); - size = FOURBYTES + sizeof(struct lpfc_name); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_FABRICNAME); - return size; + return lpfc_fdmi_set_attr_wwn(attr, RPRT_FABRICNAME, + &vport->fabric_portname); } static int -lpfc_fdmi_port_attr_active_fc4type(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_active_fc4type(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; + u32 fc4types; - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - ae->un.AttrTypes[2] = 0x01; /* Type 0x8 - FCP */ - ae->un.AttrTypes[7] = 0x01; /* Type 0x20 - CT */ + fc4types = (ATTR_FC4_CT | ATTR_FC4_FCP); /* Check to see if NVME is configured or not */ if (vport == phba->pport && phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) - ae->un.AttrTypes[6] = 0x1; /* Type 0x28 - NVME */ + fc4types |= ATTR_FC4_NVME; - size = FOURBYTES + 32; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_ACTIVE_FC4_TYPES); - return size; + return lpfc_fdmi_set_attr_fc4types(attr, RPRT_ACTIVE_FC4_TYPES, + fc4types); } static int -lpfc_fdmi_port_attr_port_state(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_port_state(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - /* Link Up - operational */ - ae->un.AttrInt = cpu_to_be32(LPFC_FDMI_PORTSTATE_ONLINE); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_PORT_STATE); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_PORT_STATE, + LPFC_FDMI_PORTSTATE_ONLINE); } static int -lpfc_fdmi_port_attr_num_disc(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_num_disc(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; vport->fdmi_num_disc = lpfc_find_map_node(vport); - ae->un.AttrInt = cpu_to_be32(vport->fdmi_num_disc); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_DISC_PORT); - return size; + + return lpfc_fdmi_set_attr_u32(attr, RPRT_DISC_PORT, + vport->fdmi_num_disc); } static int -lpfc_fdmi_port_attr_nportid(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_port_attr_nportid(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - ae->un.AttrInt = cpu_to_be32(vport->fc_myDID); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_PORT_ID); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_PORT_ID, vport->fc_myDID); } static int -lpfc_fdmi_smart_attr_service(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_service(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - strncpy(ae->un.AttrString, "Smart SAN Initiator", - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_SERVICE); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_SMART_SERVICE, + "Smart SAN Initiator"); } static int -lpfc_fdmi_smart_attr_guid(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_guid(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - memcpy(&ae->un.AttrString, &vport->fc_sparam.nodeName, - sizeof(struct lpfc_name)); - memcpy((((uint8_t *)&ae->un.AttrString) + - sizeof(struct lpfc_name)), - &vport->fc_sparam.portName, sizeof(struct lpfc_name)); - size = FOURBYTES + (2 * sizeof(struct lpfc_name)); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_GUID); - return size; + return lpfc_fdmi_set_attr_fullwwn(attr, RPRT_SMART_GUID, + &vport->fc_sparam.nodeName, + &vport->fc_sparam.portName); } static int -lpfc_fdmi_smart_attr_version(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_version(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - - strncpy(ae->un.AttrString, "Smart SAN Version 2.0", - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, - sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_VERSION); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_SMART_VERSION, + "Smart SAN Version 2.0"); } static int -lpfc_fdmi_smart_attr_model(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_model(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - - ae = &ad->AttrValue; - memset(ae, 0, sizeof(*ae)); - strncpy(ae->un.AttrString, phba->ModelName, - sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_MODEL); - return size; + return lpfc_fdmi_set_attr_string(attr, RPRT_SMART_MODEL, + phba->ModelName); } static int -lpfc_fdmi_smart_attr_port_info(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_port_info(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - /* SRIOV (type 3) is not supported */ - if (vport->vpi) - ae->un.AttrInt = cpu_to_be32(2); /* NPIV */ - else - ae->un.AttrInt = cpu_to_be32(1); /* Physical */ - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_PORT_INFO); - return size; + + return lpfc_fdmi_set_attr_u32(attr, RPRT_SMART_PORT_INFO, + (vport->vpi) ? 2 /* NPIV */ : 1 /* Physical */); } static int -lpfc_fdmi_smart_attr_qos(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_qos(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - ae->un.AttrInt = cpu_to_be32(0); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_QOS); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_SMART_QOS, 0); } static int -lpfc_fdmi_smart_attr_security(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_smart_attr_security(struct lpfc_vport *vport, void *attr) { - struct lpfc_fdmi_attr_entry *ae; - uint32_t size; - - ae = &ad->AttrValue; - ae->un.AttrInt = cpu_to_be32(1); - size = FOURBYTES + sizeof(uint32_t); - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_SMART_SECURITY); - return size; + return lpfc_fdmi_set_attr_u32(attr, RPRT_SMART_SECURITY, 1); } static int -lpfc_fdmi_vendor_attr_mi(struct lpfc_vport *vport, - struct lpfc_fdmi_attr_def *ad) +lpfc_fdmi_vendor_attr_mi(struct lpfc_vport *vport, void *attr) { struct lpfc_hba *phba = vport->phba; - struct lpfc_fdmi_attr_entry *ae; - uint32_t len, size; - char mibrevision[16]; - - ae = (struct lpfc_fdmi_attr_entry *)&ad->AttrValue; - memset(ae, 0, 256); - sprintf(mibrevision, "ELXE2EM:%04d", - phba->sli4_hba.pc_sli4_params.mi_ver); - strncpy(ae->un.AttrString, &mibrevision[0], sizeof(ae->un.AttrString)); - len = strnlen(ae->un.AttrString, sizeof(ae->un.AttrString)); - len += (len & 3) ? (4 - (len & 3)) : 4; - size = FOURBYTES + len; - ad->AttrLen = cpu_to_be16(size); - ad->AttrType = cpu_to_be16(RPRT_VENDOR_MI); - return size; + char buf[32] = { 0 }; + + sprintf(buf, "ELXE2EM:%04d", phba->sli4_hba.pc_sli4_params.mi_ver); + + return lpfc_fdmi_set_attr_string(attr, RPRT_VENDOR_MI, buf); } /* RHBA attribute jump table */ int (*lpfc_fdmi_hba_action[]) - (struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad) = { + (struct lpfc_vport *vport, void *attrbuf) = { /* Action routine Mask bit Attribute type */ lpfc_fdmi_hba_attr_wwnn, /* bit0 RHBA_NODENAME */ lpfc_fdmi_hba_attr_manufacturer, /* bit1 RHBA_MANUFACTURER */ @@ -3485,7 +3135,7 @@ int (*lpfc_fdmi_hba_action[]) /* RPA / RPRT attribute jump table */ int (*lpfc_fdmi_port_action[]) - (struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad) = { + (struct lpfc_vport *vport, void *attrbuf) = { /* Action routine Mask bit Attribute type */ lpfc_fdmi_port_attr_fc4type, /* bit0 RPRT_SUPPORT_FC4_TYPES */ lpfc_fdmi_port_attr_support_speed, /* bit1 RPRT_SUPPORTED_SPEED */ @@ -3527,20 +3177,20 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, int cmdcode, uint32_t new_mask) { struct lpfc_hba *phba = vport->phba; - struct lpfc_dmabuf *mp, *bmp; + struct lpfc_dmabuf *rq, *rsp; struct lpfc_sli_ct_request *CtReq; - struct ulp_bde64 *bpl; + struct ulp_bde64_le *bde; uint32_t bit_pos; - uint32_t size; + uint32_t size, addsz; uint32_t rsp_size; uint32_t mask; struct lpfc_fdmi_reg_hba *rh; struct lpfc_fdmi_port_entry *pe; - struct lpfc_fdmi_reg_portattr *pab = NULL; + struct lpfc_fdmi_reg_portattr *pab = NULL, *base = NULL; struct lpfc_fdmi_attr_block *ab = NULL; - int (*func)(struct lpfc_vport *vport, struct lpfc_fdmi_attr_def *ad); - void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, - struct lpfc_iocbq *); + int (*func)(struct lpfc_vport *vport, void *attrbuf); + void (*cmpl)(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, + struct lpfc_iocbq *rspiocb); if (!ndlp) return 0; @@ -3549,25 +3199,29 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, /* fill in BDEs for command */ /* Allocate buffer for command payload */ - mp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); - if (!mp) + rq = kmalloc(sizeof(*rq), GFP_KERNEL); + if (!rq) goto fdmi_cmd_exit; - mp->virt = lpfc_mbuf_alloc(phba, 0, &(mp->phys)); - if (!mp->virt) - goto fdmi_cmd_free_mp; + rq->virt = lpfc_mbuf_alloc(phba, 0, &rq->phys); + if (!rq->virt) + goto fdmi_cmd_free_rq; /* Allocate buffer for Buffer ptr list */ - bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); - if (!bmp) - goto fdmi_cmd_free_mpvirt; + rsp = kmalloc(sizeof(*rsp), GFP_KERNEL); + if (!rsp) + goto fdmi_cmd_free_rqvirt; - bmp->virt = lpfc_mbuf_alloc(phba, 0, &(bmp->phys)); - if (!bmp->virt) - goto fdmi_cmd_free_bmp; + rsp->virt = lpfc_mbuf_alloc(phba, 0, &rsp->phys); + if (!rsp->virt) + goto fdmi_cmd_free_rsp; - INIT_LIST_HEAD(&mp->list); - INIT_LIST_HEAD(&bmp->list); + INIT_LIST_HEAD(&rq->list); + INIT_LIST_HEAD(&rsp->list); + + /* mbuf buffers are 1K in length - aka LPFC_BPL_SIZE */ + memset(rq->virt, 0, LPFC_BPL_SIZE); + rsp_size = LPFC_BPL_SIZE; /* FDMI request */ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY, @@ -3575,10 +3229,9 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, cmdcode, new_mask, vport->fdmi_port_mask, vport->fc_flag, vport->port_state); - CtReq = (struct lpfc_sli_ct_request *)mp->virt; + CtReq = (struct lpfc_sli_ct_request *)rq->virt; /* First populate the CT_IU preamble */ - memset(CtReq, 0, sizeof(struct lpfc_sli_ct_request)); CtReq->RevisionId.bits.Revision = SLI_CT_REVISION; CtReq->RevisionId.bits.InId = 0; @@ -3586,17 +3239,18 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, CtReq->FsSubType = SLI_CT_FDMI_Subtypes; CtReq->CommandResponse.bits.CmdRsp = cpu_to_be16(cmdcode); - rsp_size = LPFC_BPL_SIZE; + size = 0; /* Next fill in the specific FDMI cmd information */ switch (cmdcode) { case SLI_MGMT_RHAT: case SLI_MGMT_RHBA: - rh = (struct lpfc_fdmi_reg_hba *)&CtReq->un.PortID; + rh = (struct lpfc_fdmi_reg_hba *)&CtReq->un; /* HBA Identifier */ memcpy(&rh->hi.PortName, &phba->pport->fc_sparam.portName, sizeof(struct lpfc_name)); + size += sizeof(struct lpfc_fdmi_hba_ident); if (cmdcode == SLI_MGMT_RHBA) { /* Registered Port List */ @@ -3605,16 +3259,13 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, memcpy(&rh->rpl.pe.PortName, &phba->pport->fc_sparam.portName, sizeof(struct lpfc_name)); - - /* point to the HBA attribute block */ - size = 2 * sizeof(struct lpfc_name) + - FOURBYTES; - } else { - size = sizeof(struct lpfc_name); + size += sizeof(struct lpfc_fdmi_reg_port_list); } + ab = (struct lpfc_fdmi_attr_block *)((uint8_t *)rh + size); ab->EntryCnt = 0; - size += FOURBYTES; + size += FOURBYTES; /* add length of EntryCnt field */ + bit_pos = 0; if (new_mask) mask = new_mask; @@ -3625,11 +3276,13 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, while (mask) { if (mask & 0x1) { func = lpfc_fdmi_hba_action[bit_pos]; - size += func(vport, - (struct lpfc_fdmi_attr_def *) - ((uint8_t *)rh + size)); - ab->EntryCnt++; - if ((size + 256) > + addsz = func(vport, ((uint8_t *)rh + size)); + if (addsz) { + ab->EntryCnt++; + size += addsz; + } + /* check if another attribute fits */ + if ((size + FDMI_MAX_ATTRLEN) > (LPFC_BPL_SIZE - LPFC_CT_PREAMBLE)) goto hba_out; } @@ -3639,7 +3292,7 @@ lpfc_fdmi_cmd(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, hba_out: ab->EntryCnt = cpu_to_be32(ab->EntryCnt); /* Total size */ - size = GID_REQUEST_SZ - 4 + size; + size += GID_REQUEST_SZ - 4; break; case SLI_MGMT_RPRT: @@ -3650,22 +3303,29 @@ hba_out: } fallthrough; case SLI_MGMT_RPA: - pab = (struct lpfc_fdmi_reg_portattr *)&CtReq->un.PortID; + /* Store base ptr right after preamble */ + base = (struct lpfc_fdmi_reg_portattr *)&CtReq->un; + if (cmdcode == SLI_MGMT_RPRT) { - rh = (struct lpfc_fdmi_reg_hba *)pab; + rh = (struct lpfc_fdmi_reg_hba *)base; /* HBA Identifier */ memcpy(&rh->hi.PortName, &phba->pport->fc_sparam.portName, sizeof(struct lpfc_name)); pab = (struct lpfc_fdmi_reg_portattr *) - ((uint8_t *)pab + sizeof(struct lpfc_name)); + ((uint8_t *)base + sizeof(struct lpfc_name)); + size += sizeof(struct lpfc_name); + } else { + pab = base; } memcpy((uint8_t *)&pab->PortName, (uint8_t *)&vport->fc_sparam.portName, sizeof(struct lpfc_name)); - size += sizeof(struct lpfc_name) + FOURBYTES; pab->ab.EntryCnt = 0; + /* add length of name and EntryCnt field */ + size += sizeof(struct lpfc_name) + FOURBYTES; + bit_pos = 0; if (new_mask) mask = new_mask; @@ -3676,11 +3336,13 @@ hba_out: while (mask) { if (mask & 0x1) { func = lpfc_fdmi_port_action[bit_pos]; - size += func(vport, - (struct lpfc_fdmi_attr_def *) - ((uint8_t *)pab + size)); - pab->ab.EntryCnt++; - if ((size + 256) > + addsz = func(vport, ((uint8_t *)base + size)); + if (addsz) { + pab->ab.EntryCnt++; + size += addsz; + } + /* check if another attribute fits */ + if ((size + FDMI_MAX_ATTRLEN) > (LPFC_BPL_SIZE - LPFC_CT_PREAMBLE)) goto port_out; } @@ -3689,10 +3351,7 @@ hba_out: } port_out: pab->ab.EntryCnt = cpu_to_be32(pab->ab.EntryCnt); - /* Total size */ - if (cmdcode == SLI_MGMT_RPRT) - size += sizeof(struct lpfc_name); - size = GID_REQUEST_SZ - 4 + size; + size += GID_REQUEST_SZ - 4; break; case SLI_MGMT_GHAT: @@ -3701,7 +3360,7 @@ port_out: fallthrough; case SLI_MGMT_DHBA: case SLI_MGMT_DHAT: - pe = (struct lpfc_fdmi_port_entry *)&CtReq->un.PortID; + pe = (struct lpfc_fdmi_port_entry *)&CtReq->un; memcpy((uint8_t *)&pe->PortName, (uint8_t *)&vport->fc_sparam.portName, sizeof(struct lpfc_name)); @@ -3720,7 +3379,7 @@ port_out: } fallthrough; case SLI_MGMT_DPA: - pe = (struct lpfc_fdmi_port_entry *)&CtReq->un.PortID; + pe = (struct lpfc_fdmi_port_entry *)&CtReq->un; memcpy((uint8_t *)&pe->PortName, (uint8_t *)&vport->fc_sparam.portName, sizeof(struct lpfc_name)); @@ -3733,31 +3392,32 @@ port_out: lpfc_printf_vlog(vport, KERN_WARNING, LOG_DISCOVERY, "0298 FDMI cmdcode x%x not supported\n", cmdcode); - goto fdmi_cmd_free_bmpvirt; + goto fdmi_cmd_free_rspvirt; } CtReq->CommandResponse.bits.Size = cpu_to_be16(rsp_size); - bpl = (struct ulp_bde64 *)bmp->virt; - bpl->addrHigh = le32_to_cpu(putPaddrHigh(mp->phys)); - bpl->addrLow = le32_to_cpu(putPaddrLow(mp->phys)); - bpl->tus.f.bdeFlags = 0; - bpl->tus.f.bdeSize = size; + bde = (struct ulp_bde64_le *)rsp->virt; + bde->addr_high = cpu_to_le32(putPaddrHigh(rq->phys)); + bde->addr_low = cpu_to_le32(putPaddrLow(rq->phys)); + bde->type_size = cpu_to_le32(ULP_BDE64_TYPE_BDE_64 << + ULP_BDE64_TYPE_SHIFT); + bde->type_size |= cpu_to_le32(size); /* * The lpfc_ct_cmd/lpfc_get_req shall increment ndlp reference count * to hold ndlp reference for the corresponding callback function. */ - if (!lpfc_ct_cmd(vport, mp, bmp, ndlp, cmpl, rsp_size, 0)) + if (!lpfc_ct_cmd(vport, rq, rsp, ndlp, cmpl, rsp_size, 0)) return 0; -fdmi_cmd_free_bmpvirt: - lpfc_mbuf_free(phba, bmp->virt, bmp->phys); -fdmi_cmd_free_bmp: - kfree(bmp); -fdmi_cmd_free_mpvirt: - lpfc_mbuf_free(phba, mp->virt, mp->phys); -fdmi_cmd_free_mp: - kfree(mp); +fdmi_cmd_free_rspvirt: + lpfc_mbuf_free(phba, rsp->virt, rsp->phys); +fdmi_cmd_free_rsp: + kfree(rsp); +fdmi_cmd_free_rqvirt: + lpfc_mbuf_free(phba, rq->virt, rq->phys); +fdmi_cmd_free_rq: + kfree(rq); fdmi_cmd_exit: /* Issue FDMI request failed */ lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY, @@ -3912,6 +3572,7 @@ lpfc_cmpl_ct_cmd_vmid(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_sli_ct_request *ctrsp = outp->virt; u16 rsp = ctrsp->CommandResponse.bits.CmdRsp; struct app_id_object *app; + struct lpfc_nodelist *ndlp = cmdiocb->ndlp; u32 cmd, hash, bucket; struct lpfc_vmid *vmp, *cur; u8 *data = outp->virt; @@ -3923,7 +3584,7 @@ lpfc_cmpl_ct_cmd_vmid(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, if (lpfc_els_chk_latt(vport) || get_job_ulpstatus(phba, rspiocb)) { if (cmd != SLI_CTAS_DALLAPP_ID) - return; + goto free_res; } /* Check for a CT LS_RJT response */ if (rsp == be16_to_cpu(SLI_CT_RESPONSE_FS_RJT)) { @@ -3938,7 +3599,7 @@ lpfc_cmpl_ct_cmd_vmid(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, /* If DALLAPP_ID failed retry later */ if (cmd == SLI_CTAS_DALLAPP_ID) vport->load_flag |= FC_DEREGISTER_ALL_APP_ID; - return; + goto free_res; } } @@ -3952,7 +3613,7 @@ lpfc_cmpl_ct_cmd_vmid(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, app->obj.entity_id_len); if (app->obj.entity_id_len == 0 || app->port_id == 0) - return; + goto free_res; hash = lpfc_vmid_hash_fn(app->obj.entity_id, app->obj.entity_id_len); @@ -3999,6 +3660,9 @@ lpfc_cmpl_ct_cmd_vmid(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, lpfc_printf_vlog(vport, KERN_DEBUG, LOG_DISCOVERY, "8857 Invalid command code\n"); } +free_res: + lpfc_ct_free_iocb(phba, cmdiocb); + lpfc_nlp_put(ndlp); } /** diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 5037ea09a8104ca63557b438cf4be0764d7c9e66..f5252e45a48a28ad229932bfbf14c4ed8b9a2295 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -5156,7 +5156,7 @@ error_out: static int lpfc_idiag_extacc_avail_get(struct lpfc_hba *phba, char *pbuffer, int len) { - uint16_t ext_cnt, ext_size; + uint16_t ext_cnt = 0, ext_size = 0; len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len, "\nAvailable Extents Information:\n"); @@ -5531,7 +5531,7 @@ lpfc_rx_monitor_open(struct inode *inode, struct file *file) if (!debug) goto out; - debug->buffer = vmalloc(MAX_DEBUGFS_RX_TABLE_SIZE); + debug->buffer = vmalloc(MAX_DEBUGFS_RX_INFO_SIZE); if (!debug->buffer) { kfree(debug); goto out; @@ -5552,57 +5552,18 @@ lpfc_rx_monitor_read(struct file *file, char __user *buf, size_t nbytes, struct lpfc_rx_monitor_debug *debug = file->private_data; struct lpfc_hba *phba = (struct lpfc_hba *)debug->i_private; char *buffer = debug->buffer; - struct rxtable_entry *entry; - int i, len = 0, head, tail, last, start; - - head = atomic_read(&phba->rxtable_idx_head); - while (head == LPFC_RXMONITOR_TABLE_IN_USE) { - /* Table is getting updated */ - msleep(20); - head = atomic_read(&phba->rxtable_idx_head); - } - tail = atomic_xchg(&phba->rxtable_idx_tail, head); - if (!phba->rxtable || head == tail) { - len += scnprintf(buffer + len, MAX_DEBUGFS_RX_TABLE_SIZE - len, - "Rxtable is empty\n"); - goto out; - } - last = (head > tail) ? head : LPFC_MAX_RXMONITOR_ENTRY; - start = tail; - - len += scnprintf(buffer + len, MAX_DEBUGFS_RX_TABLE_SIZE - len, - " MaxBPI Tot_Data_CMF Tot_Data_Cmd " - "Tot_Data_Cmpl Lat(us) Avg_IO Max_IO " - "Bsy IO_cnt Info BWutil(ms)\n"); -get_table: - for (i = start; i < last; i++) { - entry = &phba->rxtable[i]; - len += scnprintf(buffer + len, MAX_DEBUGFS_RX_TABLE_SIZE - len, - "%3d:%12lld %12lld %12lld %12lld " - "%7lldus %8lld %7lld " - "%2d %4d %2d %2d(%2d)\n", - i, entry->max_bytes_per_interval, - entry->cmf_bytes, - entry->total_bytes, - entry->rcv_bytes, - entry->avg_io_latency, - entry->avg_io_size, - entry->max_read_cnt, - entry->cmf_busy, - entry->io_cnt, - entry->cmf_info, - entry->timer_utilization, - entry->timer_interval); + if (!phba->rx_monitor) { + scnprintf(buffer, MAX_DEBUGFS_RX_INFO_SIZE, + "Rx Monitor Info is empty.\n"); + } else { + lpfc_rx_monitor_report(phba, phba->rx_monitor, buffer, + MAX_DEBUGFS_RX_INFO_SIZE, + LPFC_MAX_RXMONITOR_ENTRY); } - if (head != last) { - start = 0; - last = head; - goto get_table; - } -out: - return simple_read_from_buffer(buf, nbytes, ppos, buffer, len); + return simple_read_from_buffer(buf, nbytes, ppos, buffer, + strlen(buffer)); } static int diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h index 6dd361c1fd31837be2b82b77abcb6079e40ac095..8d2e8d05bbc05fb85ac7235fa7c0f47131d4a771 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.h +++ b/drivers/scsi/lpfc/lpfc_debugfs.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2007-2011 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -282,7 +282,7 @@ struct lpfc_idiag { void *ptr_private; }; -#define MAX_DEBUGFS_RX_TABLE_SIZE (128 * LPFC_MAX_RXMONITOR_ENTRY) +#define MAX_DEBUGFS_RX_INFO_SIZE (128 * LPFC_MAX_RXMONITOR_ENTRY) struct lpfc_rx_monitor_debug { char *i_private; char *buffer; diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h index 37a4b79010bfc52dbbd59a142f4bdd5b27d03533..f82615d87c4bbb85730848f0c4e14d2823e45f7c 100644 --- a/drivers/scsi/lpfc/lpfc_disc.h +++ b/drivers/scsi/lpfc/lpfc_disc.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2013 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -149,7 +149,6 @@ struct lpfc_nodelist { uint32_t cmd_qdepth; unsigned long last_change_time; unsigned long *active_rrqs_xri_bitmap; - struct lpfc_scsicmd_bkt *lat_data; /* Latency data */ uint32_t fc4_prli_sent; /* flags to keep ndlp alive until special conditions are met */ @@ -188,7 +187,6 @@ struct lpfc_node_rrq { #define NLP_RNID_SND 0x00000400 /* sent RNID request for this entry */ #define NLP_ELS_SND_MASK 0x000007e0 /* sent ELS request for this entry */ #define NLP_NVMET_RECOV 0x00001000 /* NVMET auditing node for recovery. */ -#define NLP_FCP_PRLI_RJT 0x00002000 /* Rport does not support FCP PRLI. */ #define NLP_UNREG_INP 0x00008000 /* UNREG_RPI cmd is in progress */ #define NLP_DROPPED 0x00010000 /* Init ref count has been dropped */ #define NLP_DELAY_TMO 0x00020000 /* delay timeout is running for node */ diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c index 9e69de9eb9923ff163fea3307de731f87b58b3b8..863b2125fed6cebe9a0ffdade153914af88c8a26 100644 --- a/drivers/scsi/lpfc/lpfc_els.c +++ b/drivers/scsi/lpfc/lpfc_els.c @@ -2200,10 +2200,6 @@ lpfc_issue_els_plogi(struct lpfc_vport *vport, uint32_t did, uint8_t retry) if (!elsiocb) return 1; - spin_lock_irq(&ndlp->lock); - ndlp->nlp_flag &= ~NLP_FCP_PRLI_RJT; - spin_unlock_irq(&ndlp->lock); - pcmd = (uint8_t *)elsiocb->cmd_dmabuf->virt; /* For PLOGI request, remainder of payload is service parameters */ @@ -3992,7 +3988,8 @@ lpfc_cmpl_els_edc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, goto out; /* ELS cmd tag completes */ - lpfc_printf_log(phba, KERN_INFO, LOG_ELS | LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_INFO, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "4676 Fabric EDC Rsp: " "0x%02x, 0x%08x\n", edc_rsp->acc_hdr.la_cmd, @@ -4029,18 +4026,18 @@ lpfc_cmpl_els_edc(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, if (bytes_remain < FC_TLV_DESC_SZ_FROM_LENGTH(tlv) || FC_TLV_DESC_SZ_FROM_LENGTH(tlv) != sizeof(struct fc_diag_lnkflt_desc)) { - lpfc_printf_log( - phba, KERN_WARNING, LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "6462 Truncated Link Fault Diagnostic " "descriptor[%d]: %d vs 0x%zx 0x%zx\n", desc_cnt, bytes_remain, FC_TLV_DESC_SZ_FROM_LENGTH(tlv), - sizeof(struct fc_diag_cg_sig_desc)); + sizeof(struct fc_diag_lnkflt_desc)); goto out; } plnkflt = (struct fc_diag_lnkflt_desc *)tlv; - lpfc_printf_log( - phba, KERN_INFO, LOG_ELS | LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_INFO, + LOG_ELS | LOG_LDS_EVENT, "4617 Link Fault Desc Data: 0x%08x 0x%08x " "0x%08x 0x%08x 0x%08x\n", be32_to_cpu(plnkflt->desc_tag), @@ -4120,8 +4117,26 @@ out: } static void -lpfc_format_edc_cgn_desc(struct lpfc_hba *phba, struct fc_diag_cg_sig_desc *cgd) +lpfc_format_edc_lft_desc(struct lpfc_hba *phba, struct fc_tlv_desc *tlv) { + struct fc_diag_lnkflt_desc *lft = (struct fc_diag_lnkflt_desc *)tlv; + + lft->desc_tag = cpu_to_be32(ELS_DTAG_LNK_FAULT_CAP); + lft->desc_len = cpu_to_be32( + FC_TLV_DESC_LENGTH_FROM_SZ(struct fc_diag_lnkflt_desc)); + + lft->degrade_activate_threshold = + cpu_to_be32(phba->degrade_activate_threshold); + lft->degrade_deactivate_threshold = + cpu_to_be32(phba->degrade_deactivate_threshold); + lft->fec_degrade_interval = cpu_to_be32(phba->fec_degrade_interval); +} + +static void +lpfc_format_edc_cgn_desc(struct lpfc_hba *phba, struct fc_tlv_desc *tlv) +{ + struct fc_diag_cg_sig_desc *cgd = (struct fc_diag_cg_sig_desc *)tlv; + /* We are assuming cgd was zero'ed before calling this routine */ /* Configure the congestion detection capability */ @@ -4165,6 +4180,23 @@ lpfc_format_edc_cgn_desc(struct lpfc_hba *phba, struct fc_diag_cg_sig_desc *cgd) cpu_to_be16(EDC_CG_SIGFREQ_MSEC); } +static bool +lpfc_link_is_lds_capable(struct lpfc_hba *phba) +{ + if (!(phba->lmt & LMT_64Gb)) + return false; + if (phba->sli_rev != LPFC_SLI_REV4) + return false; + + if (phba->sli4_hba.conf_trunk) { + if (phba->trunk_link.phy_lnk_speed == LPFC_USER_LINK_SPEED_64G) + return true; + } else if (phba->fc_linkspeed == LPFC_LINK_SPEED_64GHZ) { + return true; + } + return false; +} + /** * lpfc_issue_els_edc - Exchange Diagnostic Capabilities with the fabric. * @vport: pointer to a host virtual N_Port data structure. @@ -4192,12 +4224,12 @@ lpfc_issue_els_edc(struct lpfc_vport *vport, uint8_t retry) { struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *elsiocb; - struct lpfc_els_edc_req *edc_req; - struct fc_diag_cg_sig_desc *cgn_desc; + struct fc_els_edc *edc_req; + struct fc_tlv_desc *tlv; u16 cmdsize; struct lpfc_nodelist *ndlp; u8 *pcmd = NULL; - u32 edc_req_size, cgn_desc_size; + u32 cgn_desc_size, lft_desc_size; int rc; if (vport->port_type == LPFC_NPIV_PORT) @@ -4207,13 +4239,17 @@ lpfc_issue_els_edc(struct lpfc_vport *vport, uint8_t retry) if (!ndlp || ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) return -ENODEV; - /* If HBA doesn't support signals, drop into RDF */ - if (!phba->cgn_init_reg_signal) + cgn_desc_size = (phba->cgn_init_reg_signal) ? + sizeof(struct fc_diag_cg_sig_desc) : 0; + lft_desc_size = (lpfc_link_is_lds_capable(phba)) ? + sizeof(struct fc_diag_lnkflt_desc) : 0; + cmdsize = cgn_desc_size + lft_desc_size; + + /* Skip EDC if no applicable descriptors */ + if (!cmdsize) goto try_rdf; - edc_req_size = sizeof(struct fc_els_edc); - cgn_desc_size = sizeof(struct fc_diag_cg_sig_desc); - cmdsize = edc_req_size + cgn_desc_size; + cmdsize += sizeof(struct fc_els_edc); elsiocb = lpfc_prep_els_iocb(vport, 1, cmdsize, retry, ndlp, ndlp->nlp_DID, ELS_CMD_EDC); if (!elsiocb) @@ -4222,15 +4258,19 @@ lpfc_issue_els_edc(struct lpfc_vport *vport, uint8_t retry) /* Configure the payload for the supported Diagnostics capabilities. */ pcmd = (u8 *)elsiocb->cmd_dmabuf->virt; memset(pcmd, 0, cmdsize); - edc_req = (struct lpfc_els_edc_req *)pcmd; - edc_req->edc.desc_len = cpu_to_be32(cgn_desc_size); - edc_req->edc.edc_cmd = ELS_EDC; - - cgn_desc = &edc_req->cgn_desc; + edc_req = (struct fc_els_edc *)pcmd; + edc_req->desc_len = cpu_to_be32(cgn_desc_size + lft_desc_size); + edc_req->edc_cmd = ELS_EDC; + tlv = edc_req->desc; - lpfc_format_edc_cgn_desc(phba, cgn_desc); + if (cgn_desc_size) { + lpfc_format_edc_cgn_desc(phba, tlv); + phba->cgn_sig_freq = lpfc_fabric_cgn_frequency; + tlv = fc_tlv_next_desc(tlv); + } - phba->cgn_sig_freq = lpfc_fabric_cgn_frequency; + if (lft_desc_size) + lpfc_format_edc_lft_desc(phba, tlv); lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS | LOG_CGN_MGMT, "4623 Xmit EDC to remote " @@ -4676,47 +4716,52 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, } switch (stat.un.b.lsRjtRsnCode) { case LSRJT_UNABLE_TPC: - /* The driver has a VALID PLOGI but the rport has - * rejected the PRLI - can't do it now. Delay - * for 1 second and try again. - * - * However, if explanation is REQ_UNSUPPORTED there's - * no point to retry PRLI. + /* Special case for PRLI LS_RJTs. Recall that lpfc + * uses a single routine to issue both PRLI FC4 types. + * If the PRLI is rejected because that FC4 type + * isn't really supported, don't retry and cause + * multiple transport registrations. Otherwise, parse + * the reason code/reason code explanation and take the + * appropriate action. */ - if ((cmd == ELS_CMD_PRLI || cmd == ELS_CMD_NVMEPRLI) && - stat.un.b.lsRjtRsnCodeExp != - LSEXP_REQ_UNSUPPORTED) { - delay = 1000; - maxretry = lpfc_max_els_tries + 1; - retry = 1; - break; - } - - /* Legacy bug fix code for targets with PLOGI delays. */ - if (stat.un.b.lsRjtRsnCodeExp == - LSEXP_CMD_IN_PROGRESS) { + lpfc_printf_vlog(vport, KERN_INFO, + LOG_DISCOVERY | LOG_ELS | LOG_NODE, + "0153 ELS cmd x%x LS_RJT by x%x. " + "RsnCode x%x RsnCodeExp x%x\n", + cmd, did, stat.un.b.lsRjtRsnCode, + stat.un.b.lsRjtRsnCodeExp); + + switch (stat.un.b.lsRjtRsnCodeExp) { + case LSEXP_CANT_GIVE_DATA: + case LSEXP_CMD_IN_PROGRESS: if (cmd == ELS_CMD_PLOGI) { delay = 1000; maxretry = 48; } retry = 1; break; - } - if (stat.un.b.lsRjtRsnCodeExp == - LSEXP_CANT_GIVE_DATA) { - if (cmd == ELS_CMD_PLOGI) { + case LSEXP_REQ_UNSUPPORTED: + case LSEXP_NO_RSRC_ASSIGN: + /* These explanation codes get no retry. */ + if (cmd == ELS_CMD_PRLI || + cmd == ELS_CMD_NVMEPRLI) + break; + fallthrough; + default: + /* Limit the delay and retry action to a limited + * cmd set. There are other ELS commands where + * a retry is not expected. + */ + if (cmd == ELS_CMD_PLOGI || + cmd == ELS_CMD_PRLI || + cmd == ELS_CMD_NVMEPRLI) { delay = 1000; - maxretry = 48; + maxretry = lpfc_max_els_tries + 1; + retry = 1; } - retry = 1; - break; - } - if (cmd == ELS_CMD_PLOGI) { - delay = 1000; - maxretry = lpfc_max_els_tries + 1; - retry = 1; break; } + if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) && (cmd == ELS_CMD_FDISC) && (stat.un.b.lsRjtRsnCodeExp == LSEXP_OUT_OF_RESOURCE)){ @@ -4797,13 +4842,8 @@ lpfc_els_retry(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, */ if (stat.un.b.lsRjtRsnCodeExp == LSEXP_REQ_UNSUPPORTED) { - if (cmd == ELS_CMD_PRLI) { - spin_lock_irq(&ndlp->lock); - ndlp->nlp_flag |= NLP_FCP_PRLI_RJT; - spin_unlock_irq(&ndlp->lock); - retry = 0; + if (cmd == ELS_CMD_PRLI) goto out_retry; - } } break; } @@ -5784,14 +5824,21 @@ lpfc_issue_els_edc_rsp(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, struct lpfc_nodelist *ndlp) { struct lpfc_hba *phba = vport->phba; - struct lpfc_els_edc_rsp *edc_rsp; + struct fc_els_edc_resp *edc_rsp; + struct fc_tlv_desc *tlv; struct lpfc_iocbq *elsiocb; IOCB_t *icmd, *cmd; union lpfc_wqe128 *wqe; + u32 cgn_desc_size, lft_desc_size; + u16 cmdsize; uint8_t *pcmd; - int cmdsize, rc; + int rc; - cmdsize = sizeof(struct lpfc_els_edc_rsp); + cmdsize = sizeof(struct fc_els_edc_resp); + cgn_desc_size = sizeof(struct fc_diag_cg_sig_desc); + lft_desc_size = (lpfc_link_is_lds_capable(phba)) ? + sizeof(struct fc_diag_lnkflt_desc) : 0; + cmdsize += cgn_desc_size + lft_desc_size; elsiocb = lpfc_prep_els_iocb(vport, 0, cmdsize, cmdiocb->retry, ndlp, ndlp->nlp_DID, ELS_CMD_ACC); if (!elsiocb) @@ -5813,15 +5860,19 @@ lpfc_issue_els_edc_rsp(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, pcmd = elsiocb->cmd_dmabuf->virt; memset(pcmd, 0, cmdsize); - edc_rsp = (struct lpfc_els_edc_rsp *)pcmd; - edc_rsp->edc_rsp.acc_hdr.la_cmd = ELS_LS_ACC; - edc_rsp->edc_rsp.desc_list_len = cpu_to_be32( - FC_TLV_DESC_LENGTH_FROM_SZ(struct lpfc_els_edc_rsp)); - edc_rsp->edc_rsp.lsri.desc_tag = cpu_to_be32(ELS_DTAG_LS_REQ_INFO); - edc_rsp->edc_rsp.lsri.desc_len = cpu_to_be32( + edc_rsp = (struct fc_els_edc_resp *)pcmd; + edc_rsp->acc_hdr.la_cmd = ELS_LS_ACC; + edc_rsp->desc_list_len = cpu_to_be32(sizeof(struct fc_els_lsri_desc) + + cgn_desc_size + lft_desc_size); + edc_rsp->lsri.desc_tag = cpu_to_be32(ELS_DTAG_LS_REQ_INFO); + edc_rsp->lsri.desc_len = cpu_to_be32( FC_TLV_DESC_LENGTH_FROM_SZ(struct fc_els_lsri_desc)); - edc_rsp->edc_rsp.lsri.rqst_w0.cmd = ELS_EDC; - lpfc_format_edc_cgn_desc(phba, &edc_rsp->cgn_desc); + edc_rsp->lsri.rqst_w0.cmd = ELS_EDC; + tlv = edc_rsp->desc; + lpfc_format_edc_cgn_desc(phba, tlv); + tlv = fc_tlv_next_desc(tlv); + if (lft_desc_size) + lpfc_format_edc_lft_desc(phba, tlv); lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_RSP, "Issue EDC ACC: did:x%x flg:x%x refcnt %d", @@ -6006,7 +6057,7 @@ lpfc_els_rsp_prli_acc(struct lpfc_vport *vport, struct lpfc_iocbq *oldiocb, if (prli_fc4_req == PRLI_FCP_TYPE) { cmdsize = sizeof(uint32_t) + sizeof(PRLI); elsrspcmd = (ELS_CMD_ACC | (ELS_CMD_PRLI & ~ELS_RSP_MASK)); - } else if (prli_fc4_req & PRLI_NVME_TYPE) { + } else if (prli_fc4_req == PRLI_NVME_TYPE) { cmdsize = sizeof(uint32_t) + sizeof(struct lpfc_nvme_prli); elsrspcmd = (ELS_CMD_ACC | (ELS_CMD_NVMEPRLI & ~ELS_RSP_MASK)); } else { @@ -6069,7 +6120,7 @@ lpfc_els_rsp_prli_acc(struct lpfc_vport *vport, struct lpfc_iocbq *oldiocb, npr->ConfmComplAllowed = 1; npr->prliType = PRLI_FCP_TYPE; npr->initiatorFunc = 1; - } else if (prli_fc4_req & PRLI_NVME_TYPE) { + } else if (prli_fc4_req == PRLI_NVME_TYPE) { /* Respond with an NVME PRLI Type */ npr_nvme = (struct lpfc_nvme_prli *) pcmd; bf_set(prli_type_code, npr_nvme, PRLI_NVME_TYPE); @@ -9086,7 +9137,7 @@ lpfc_els_rcv_edc(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, uint32_t *ptr, dtag; const char *dtag_nm; int desc_cnt = 0, bytes_remain; - bool rcv_cap_desc = false; + struct fc_diag_lnkflt_desc *plnkflt; payload = cmdiocb->cmd_dmabuf->virt; @@ -9094,7 +9145,8 @@ lpfc_els_rcv_edc(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, bytes_remain = be32_to_cpu(edc_req->desc_len); ptr = (uint32_t *)payload; - lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS | LOG_CGN_MGMT, + lpfc_printf_vlog(vport, KERN_INFO, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "3319 Rcv EDC payload len %d: x%x x%x x%x\n", bytes_remain, be32_to_cpu(*ptr), be32_to_cpu(*(ptr + 1)), be32_to_cpu(*(ptr + 2))); @@ -9113,9 +9165,10 @@ lpfc_els_rcv_edc(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, * cycle through EDC diagnostic descriptors to find the * congestion signaling capability descriptor */ - while (bytes_remain && !rcv_cap_desc) { + while (bytes_remain) { if (bytes_remain < FC_TLV_DESC_HDR_SZ) { - lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "6464 Truncated TLV hdr on " "Diagnostic descriptor[%d]\n", desc_cnt); @@ -9128,16 +9181,27 @@ lpfc_els_rcv_edc(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, if (bytes_remain < FC_TLV_DESC_SZ_FROM_LENGTH(tlv) || FC_TLV_DESC_SZ_FROM_LENGTH(tlv) != sizeof(struct fc_diag_lnkflt_desc)) { - lpfc_printf_log( - phba, KERN_WARNING, LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "6465 Truncated Link Fault Diagnostic " "descriptor[%d]: %d vs 0x%zx 0x%zx\n", desc_cnt, bytes_remain, FC_TLV_DESC_SZ_FROM_LENGTH(tlv), - sizeof(struct fc_diag_cg_sig_desc)); + sizeof(struct fc_diag_lnkflt_desc)); goto out; } - /* No action for Link Fault descriptor for now */ + plnkflt = (struct fc_diag_lnkflt_desc *)tlv; + lpfc_printf_log(phba, KERN_INFO, + LOG_ELS | LOG_LDS_EVENT, + "4626 Link Fault Desc Data: x%08x len x%x " + "da x%x dd x%x interval x%x\n", + be32_to_cpu(plnkflt->desc_tag), + be32_to_cpu(plnkflt->desc_len), + be32_to_cpu( + plnkflt->degrade_activate_threshold), + be32_to_cpu( + plnkflt->degrade_deactivate_threshold), + be32_to_cpu(plnkflt->fec_degrade_interval)); break; case ELS_DTAG_CG_SIGNAL_CAP: if (bytes_remain < FC_TLV_DESC_SZ_FROM_LENGTH(tlv) || @@ -9164,11 +9228,11 @@ lpfc_els_rcv_edc(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb, lpfc_least_capable_settings( phba, (struct fc_diag_cg_sig_desc *)tlv); - rcv_cap_desc = true; break; default: dtag_nm = lpfc_get_tlv_dtag_nm(dtag); - lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT, + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_CGN_MGMT | LOG_LDS_EVENT, "6467 unknown Diagnostic " "Descriptor[%d]: tag x%x (%s)\n", desc_cnt, dtag, dtag_nm); diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 2645def612e6df825aa2020917b188d33952ec1a..d38ebd7281b9b96f4485b58aa8af808d91d5f8a1 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -1242,6 +1242,8 @@ lpfc_linkdown(struct lpfc_hba *phba) phba->trunk_link.link1.state = 0; phba->trunk_link.link2.state = 0; phba->trunk_link.link3.state = 0; + phba->trunk_link.phy_lnk_speed = + LPFC_LINK_SPEED_UNKNOWN; phba->sli4_hba.link_state.logical_speed = LPFC_LINK_SPEED_UNKNOWN; } @@ -1353,8 +1355,13 @@ lpfc_linkup_port(struct lpfc_vport *vport) FCH_EVT_LINKUP, 0); spin_lock_irq(shost->host_lock); - vport->fc_flag &= ~(FC_PT2PT | FC_PT2PT_PLOGI | FC_ABORT_DISCOVERY | - FC_RSCN_MODE | FC_NLP_MORE | FC_RSCN_DISCOVERY); + if (phba->defer_flogi_acc_flag) + vport->fc_flag &= ~(FC_ABORT_DISCOVERY | FC_RSCN_MODE | + FC_NLP_MORE | FC_RSCN_DISCOVERY); + else + vport->fc_flag &= ~(FC_PT2PT | FC_PT2PT_PLOGI | + FC_ABORT_DISCOVERY | FC_RSCN_MODE | + FC_NLP_MORE | FC_RSCN_DISCOVERY); vport->fc_flag |= FC_NDISC_ACTIVE; vport->fc_ns_retry = 0; spin_unlock_irq(shost->host_lock); @@ -1392,7 +1399,6 @@ lpfc_linkup(struct lpfc_hba *phba) /* reinitialize initial HBA flag */ phba->hba_flag &= ~(HBA_FLOGI_ISSUED | HBA_RHBA_CMPL); - phba->defer_flogi_acc_flag = false; return 0; } @@ -2150,8 +2156,8 @@ lpfc_check_pending_fcoe_event(struct lpfc_hba *phba, uint8_t unreg_fcf) * This function makes an running random selection decision on FCF record to * use through a sequence of @fcf_cnt eligible FCF records with equal * probability. To perform integer manunipulation of random numbers with - * size unit32_t, the lower 16 bits of the 32-bit random number returned - * from prandom_u32() are taken as the random random number generated. + * size unit32_t, a 16-bit random number returned from get_random_u16() is + * taken as the random random number generated. * * Returns true when outcome is for the newly read FCF record should be * chosen; otherwise, return false when outcome is for keeping the previously @@ -2163,7 +2169,7 @@ lpfc_sli4_new_fcf_random_select(struct lpfc_hba *phba, uint32_t fcf_cnt) uint32_t rand_num; /* Get 16-bit uniform random number */ - rand_num = 0xFFFF & prandom_u32(); + rand_num = get_random_u16(); /* Decision with probability 1/fcf_cnt */ if ((fcf_cnt * rand_num) < 0xFFFF) @@ -2964,7 +2970,7 @@ lpfc_mbx_cmpl_fcf_rr_read_fcf_rec(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) uint32_t boot_flag, addr_mode; uint16_t next_fcf_index, fcf_index; uint16_t current_fcf_index; - uint16_t vlan_id; + uint16_t vlan_id = LPFC_FCOE_NULL_VID; int rc; /* If link state is not up, stop the roundrobin failover process */ @@ -3069,7 +3075,7 @@ lpfc_mbx_cmpl_read_fcf_rec(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) struct fcf_record *new_fcf_record; uint32_t boot_flag, addr_mode; uint16_t fcf_index, next_fcf_index; - uint16_t vlan_id; + uint16_t vlan_id = LPFC_FCOE_NULL_VID; int rc; /* If link state is not up, no need to proceed */ @@ -3790,6 +3796,9 @@ lpfc_mbx_cmpl_read_topology(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) if (phba->cmf_active_mode != LPFC_CFG_OFF) lpfc_cmf_signal_init(phba); + if (phba->lmt & LMT_64Gb) + lpfc_read_lds_params(phba); + } else if (attn_type == LPFC_ATT_LINK_DOWN || attn_type == LPFC_ATT_UNEXP_WWPN) { phba->fc_stat.LinkDown++; @@ -4389,8 +4398,11 @@ out: rc = lpfc_issue_els_edc(vport, 0); lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_ELS | LOG_DISCOVERY, - "4220 EDC issue error x%x, Data: x%x\n", + "4220 Issue EDC status x%x Data x%x\n", rc, phba->cgn_init_reg_signal); + } else if (phba->lmt & LMT_64Gb) { + /* may send link fault capability descriptor */ + lpfc_issue_els_edc(vport, 0); } else { lpfc_issue_els_rdf(vport, 0); } @@ -4788,22 +4800,6 @@ lpfc_nlp_state_cleanup(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, new_state == NLP_STE_UNMAPPED_NODE) lpfc_nlp_reg_node(vport, ndlp); - if ((new_state == NLP_STE_MAPPED_NODE) && - (vport->stat_data_enabled)) { - /* - * A new target is discovered, if there is no buffer for - * statistical data collection allocate buffer. - */ - ndlp->lat_data = kcalloc(LPFC_MAX_BUCKET_COUNT, - sizeof(struct lpfc_scsicmd_bkt), - GFP_KERNEL); - - if (!ndlp->lat_data) - lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, - "0286 lpfc_nlp_state_cleanup failed to " - "allocate statistical data buffer DID " - "0x%x\n", ndlp->nlp_DID); - } /* * If the node just added to Mapped list was an FCP target, * but the remote port registration failed or assigned a target @@ -6648,7 +6644,6 @@ lpfc_nlp_release(struct kref *kref) ndlp->fc4_xpt_flags = 0; /* free ndlp memory for final ndlp release */ - kfree(ndlp->lat_data); if (ndlp->phba->sli_rev == LPFC_SLI_REV4) mempool_free(ndlp->active_rrqs_xri_bitmap, ndlp->phba->active_rrq_pool); diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h index 071983e2cdfee1eeabdf121a659ef391fee608ad..5c283936ff08eb19cc1ae255081f40750261ea25 100644 --- a/drivers/scsi/lpfc/lpfc_hw.h +++ b/drivers/scsi/lpfc/lpfc_hw.h @@ -703,6 +703,7 @@ struct ls_rjt { /* Structure is in Big Endian format */ #define LSEXP_OUT_OF_RESOURCE 0x29 #define LSEXP_CANT_GIVE_DATA 0x2A #define LSEXP_REQ_UNSUPPORTED 0x2C +#define LSEXP_NO_RSRC_ASSIGN 0x52 uint8_t vendorUnique; /* FC Word 0, bit 0: 7 */ } b; } un; @@ -1441,30 +1442,56 @@ struct lpfc_vmid_gallapp_ident_list { /* Definitions for HBA / Port attribute entries */ -/* Attribute Entry */ -struct lpfc_fdmi_attr_entry { - union { - uint32_t AttrInt; - uint8_t AttrTypes[32]; - uint8_t AttrString[256]; - struct lpfc_name AttrWWN; - } un; +/* Attribute Entry Structures */ + +struct lpfc_fdmi_attr_u32 { + __be16 type; + __be16 len; + __be32 value_u32; }; -struct lpfc_fdmi_attr_def { /* Defined in TLV format */ - /* Structure is in Big Endian format */ - uint32_t AttrType:16; - uint32_t AttrLen:16; - /* Marks start of Value (ATTRIBUTE_ENTRY) */ - struct lpfc_fdmi_attr_entry AttrValue; -} __packed; +struct lpfc_fdmi_attr_wwn { + __be16 type; + __be16 len; + + /* Keep as u8[8] instead of __be64 to avoid accidental zero padding + * by compiler + */ + u8 name[8]; +}; + +struct lpfc_fdmi_attr_fullwwn { + __be16 type; + __be16 len; + + /* Keep as u8[8] instead of __be64 to avoid accidental zero padding + * by compiler + */ + u8 nname[8]; + u8 pname[8]; +}; + +struct lpfc_fdmi_attr_fc4types { + __be16 type; + __be16 len; + u8 value_types[32]; +}; + +struct lpfc_fdmi_attr_string { + __be16 type; + __be16 len; + char value_string[256]; +}; + +/* Maximum FDMI attribute length is Type+Len (4 bytes) + 256 byte string */ +#define FDMI_MAX_ATTRLEN sizeof(struct lpfc_fdmi_attr_string) /* * HBA Attribute Block */ struct lpfc_fdmi_attr_block { uint32_t EntryCnt; /* Number of HBA attribute entries */ - struct lpfc_fdmi_attr_entry Entry; /* Variable-length array */ + /* Variable Length Attribute Entry TLV's follow */ }; /* diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index 4527fef23ae76257d44f76fc468c0dcc0677b474..5288fc69908a6691e5f65ad94ee16b3fbc61d41c 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -738,6 +738,7 @@ struct lpfc_register { #define lpfc_sliport_eqdelay_id_WORD word0 #define LPFC_SEC_TO_USEC 1000000 #define LPFC_SEC_TO_MSEC 1000 +#define LPFC_MSECS_TO_SECS(msecs) ((msecs) / 1000) /* The following Registers apply to SLI4 if_type 0 UCNAs. They typically * reside in BAR 2. @@ -3483,9 +3484,10 @@ struct lpfc_sli4_parameters { #define LPFC_SET_UE_RECOVERY 0x10 #define LPFC_SET_MDS_DIAGS 0x12 -#define LPFC_SET_CGN_SIGNAL 0x1f #define LPFC_SET_DUAL_DUMP 0x1e +#define LPFC_SET_CGN_SIGNAL 0x1f #define LPFC_SET_ENABLE_MI 0x21 +#define LPFC_SET_LD_SIGNAL 0x23 #define LPFC_SET_ENABLE_CMF 0x24 struct lpfc_mbx_set_feature { struct mbox_header header; @@ -3516,13 +3518,17 @@ struct lpfc_mbx_set_feature { #define lpfc_mbx_set_feature_cmf_SHIFT 0 #define lpfc_mbx_set_feature_cmf_MASK 0x00000001 #define lpfc_mbx_set_feature_cmf_WORD word6 +#define lpfc_mbx_set_feature_lds_qry_SHIFT 0 +#define lpfc_mbx_set_feature_lds_qry_MASK 0x00000001 +#define lpfc_mbx_set_feature_lds_qry_WORD word6 +#define LPFC_QUERY_LDS_OP 1 #define lpfc_mbx_set_feature_mi_SHIFT 0 #define lpfc_mbx_set_feature_mi_MASK 0x0000ffff #define lpfc_mbx_set_feature_mi_WORD word6 #define lpfc_mbx_set_feature_milunq_SHIFT 16 #define lpfc_mbx_set_feature_milunq_MASK 0x0000ffff #define lpfc_mbx_set_feature_milunq_WORD word6 - uint32_t word7; + u32 word7; #define lpfc_mbx_set_feature_UERP_SHIFT 0 #define lpfc_mbx_set_feature_UERP_MASK 0x0000ffff #define lpfc_mbx_set_feature_UERP_WORD word7 @@ -3536,6 +3542,8 @@ struct lpfc_mbx_set_feature { #define lpfc_mbx_set_feature_CGN_acqe_freq_SHIFT 0 #define lpfc_mbx_set_feature_CGN_acqe_freq_MASK 0x000000ff #define lpfc_mbx_set_feature_CGN_acqe_freq_WORD word8 + u32 word9; + u32 word10; }; @@ -4313,7 +4321,7 @@ struct lpfc_acqe_cgn_signal { struct lpfc_acqe_sli { uint32_t event_data1; uint32_t event_data2; - uint32_t reserved; + uint32_t event_data3; uint32_t trailer; #define LPFC_SLI_EVENT_TYPE_PORT_ERROR 0x1 #define LPFC_SLI_EVENT_TYPE_OVER_TEMP 0x2 @@ -4326,6 +4334,7 @@ struct lpfc_acqe_sli { #define LPFC_SLI_EVENT_TYPE_MISCONF_FAWWN 0xF #define LPFC_SLI_EVENT_TYPE_EEPROM_FAILURE 0x10 #define LPFC_SLI_EVENT_TYPE_CGN_SIGNAL 0x11 +#define LPFC_SLI_EVENT_TYPE_RD_SIGNAL 0x12 }; /* @@ -4798,6 +4807,9 @@ struct cmf_sync_wqe { #define cmf_sync_cqid_WORD word11 uint32_t read_bytes; uint32_t word13; +#define cmf_sync_period_SHIFT 16 +#define cmf_sync_period_MASK 0x0000ffff +#define cmf_sync_period_WORD word13 uint32_t word14; uint32_t word15; }; @@ -5046,22 +5058,6 @@ struct lpfc_grp_hdr { { FPIN_CONGN_SEVERITY_ERROR, "Alarm" }, \ } -/* EDC supports two descriptors. When allocated, it is the - * size of this structure plus each supported descriptor. - */ -struct lpfc_els_edc_req { - struct fc_els_edc edc; /* hdr up to descriptors */ - struct fc_diag_cg_sig_desc cgn_desc; /* 1st descriptor */ -}; - -/* Minimum structure defines for the EDC response. - * Balance is in buffer. - */ -struct lpfc_els_edc_rsp { - struct fc_els_edc_resp edc_rsp; /* hdr up to descriptors */ - struct fc_diag_cg_sig_desc cgn_desc; /* 1st descriptor */ -}; - /* Used for logging FPIN messages */ #define LPFC_FPIN_WWPN_LINE_SZ 128 #define LPFC_FPIN_WWPN_LINE_CNT 6 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index c69c5a0979ec4cf9bc9af41b0a6b3c5840dc90fe..b49c39569386a860e9bfa26df2d5d9c507616b51 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -325,8 +325,7 @@ lpfc_dump_wakeup_param_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) prog_id_word = pmboxq->u.mb.un.varWords[7]; /* Decode the Option rom version word to a readable string */ - if (prg->dist < 4) - dist = dist_char[prg->dist]; + dist = dist_char[prg->dist]; if ((prg->dist == 3) && (prg->num == 0)) snprintf(phba->OptionROMVersion, 32, "%d.%d%d", @@ -2258,6 +2257,101 @@ lpfc_handle_latt_err_exit: return; } +static void +lpfc_fill_vpd(struct lpfc_hba *phba, uint8_t *vpd, int length, int *pindex) +{ + int i, j; + + while (length > 0) { + /* Look for Serial Number */ + if ((vpd[*pindex] == 'S') && (vpd[*pindex + 1] == 'N')) { + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + j = 0; + length -= (3+i); + while (i--) { + phba->SerialNumber[j++] = vpd[(*pindex)++]; + if (j == 31) + break; + } + phba->SerialNumber[j] = 0; + continue; + } else if ((vpd[*pindex] == 'V') && (vpd[*pindex + 1] == '1')) { + phba->vpd_flag |= VPD_MODEL_DESC; + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + j = 0; + length -= (3+i); + while (i--) { + phba->ModelDesc[j++] = vpd[(*pindex)++]; + if (j == 255) + break; + } + phba->ModelDesc[j] = 0; + continue; + } else if ((vpd[*pindex] == 'V') && (vpd[*pindex + 1] == '2')) { + phba->vpd_flag |= VPD_MODEL_NAME; + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + j = 0; + length -= (3+i); + while (i--) { + phba->ModelName[j++] = vpd[(*pindex)++]; + if (j == 79) + break; + } + phba->ModelName[j] = 0; + continue; + } else if ((vpd[*pindex] == 'V') && (vpd[*pindex + 1] == '3')) { + phba->vpd_flag |= VPD_PROGRAM_TYPE; + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + j = 0; + length -= (3+i); + while (i--) { + phba->ProgramType[j++] = vpd[(*pindex)++]; + if (j == 255) + break; + } + phba->ProgramType[j] = 0; + continue; + } else if ((vpd[*pindex] == 'V') && (vpd[*pindex + 1] == '4')) { + phba->vpd_flag |= VPD_PORT; + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + j = 0; + length -= (3 + i); + while (i--) { + if ((phba->sli_rev == LPFC_SLI_REV4) && + (phba->sli4_hba.pport_name_sta == + LPFC_SLI4_PPNAME_GET)) { + j++; + (*pindex)++; + } else + phba->Port[j++] = vpd[(*pindex)++]; + if (j == 19) + break; + } + if ((phba->sli_rev != LPFC_SLI_REV4) || + (phba->sli4_hba.pport_name_sta == + LPFC_SLI4_PPNAME_NON)) + phba->Port[j] = 0; + continue; + } else { + *pindex += 2; + i = vpd[*pindex]; + *pindex += 1; + *pindex += i; + length -= (3 + i); + } + } +} + /** * lpfc_parse_vpd - Parse VPD (Vital Product Data) * @phba: pointer to lpfc hba data structure. @@ -2277,7 +2371,7 @@ lpfc_parse_vpd(struct lpfc_hba *phba, uint8_t *vpd, int len) { uint8_t lenlo, lenhi; int Length; - int i, j; + int i; int finished = 0; int index = 0; @@ -2310,101 +2404,10 @@ lpfc_parse_vpd(struct lpfc_hba *phba, uint8_t *vpd, int len) Length = ((((unsigned short)lenhi) << 8) + lenlo); if (Length > len - index) Length = len - index; - while (Length > 0) { - /* Look for Serial Number */ - if ((vpd[index] == 'S') && (vpd[index+1] == 'N')) { - index += 2; - i = vpd[index]; - index += 1; - j = 0; - Length -= (3+i); - while(i--) { - phba->SerialNumber[j++] = vpd[index++]; - if (j == 31) - break; - } - phba->SerialNumber[j] = 0; - continue; - } - else if ((vpd[index] == 'V') && (vpd[index+1] == '1')) { - phba->vpd_flag |= VPD_MODEL_DESC; - index += 2; - i = vpd[index]; - index += 1; - j = 0; - Length -= (3+i); - while(i--) { - phba->ModelDesc[j++] = vpd[index++]; - if (j == 255) - break; - } - phba->ModelDesc[j] = 0; - continue; - } - else if ((vpd[index] == 'V') && (vpd[index+1] == '2')) { - phba->vpd_flag |= VPD_MODEL_NAME; - index += 2; - i = vpd[index]; - index += 1; - j = 0; - Length -= (3+i); - while(i--) { - phba->ModelName[j++] = vpd[index++]; - if (j == 79) - break; - } - phba->ModelName[j] = 0; - continue; - } - else if ((vpd[index] == 'V') && (vpd[index+1] == '3')) { - phba->vpd_flag |= VPD_PROGRAM_TYPE; - index += 2; - i = vpd[index]; - index += 1; - j = 0; - Length -= (3+i); - while(i--) { - phba->ProgramType[j++] = vpd[index++]; - if (j == 255) - break; - } - phba->ProgramType[j] = 0; - continue; - } - else if ((vpd[index] == 'V') && (vpd[index+1] == '4')) { - phba->vpd_flag |= VPD_PORT; - index += 2; - i = vpd[index]; - index += 1; - j = 0; - Length -= (3+i); - while(i--) { - if ((phba->sli_rev == LPFC_SLI_REV4) && - (phba->sli4_hba.pport_name_sta == - LPFC_SLI4_PPNAME_GET)) { - j++; - index++; - } else - phba->Port[j++] = vpd[index++]; - if (j == 19) - break; - } - if ((phba->sli_rev != LPFC_SLI_REV4) || - (phba->sli4_hba.pport_name_sta == - LPFC_SLI4_PPNAME_NON)) - phba->Port[j] = 0; - continue; - } - else { - index += 2; - i = vpd[index]; - index += 1; - index += i; - Length -= (3 + i); - } - } - finished = 0; - break; + + lpfc_fill_vpd(phba, vpd, Length, &index); + finished = 0; + break; case 0x78: finished = 1; break; @@ -4614,6 +4617,17 @@ lpfc_get_wwpn(struct lpfc_hba *phba) return rol64(wwn, 32); } +static unsigned short lpfc_get_sg_tablesize(struct lpfc_hba *phba) +{ + if (phba->sli_rev == LPFC_SLI_REV4) + if (phba->cfg_xpsgl && !phba->nvmet_support) + return LPFC_MAX_SG_TABLESIZE; + else + return phba->cfg_scsi_seg_cnt; + else + return phba->cfg_sg_seg_cnt; +} + /** * lpfc_vmid_res_alloc - Allocates resources for VMID * @phba: pointer to lpfc hba data structure. @@ -4716,42 +4730,26 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) /* Seed template for SCSI host registration */ if (dev == &phba->pcidev->dev) { - template = &phba->port_template; - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { /* Seed physical port template */ - memcpy(template, &lpfc_template, sizeof(*template)); + template = &lpfc_template; if (use_no_reset_hba) /* template is for a no reset SCSI Host */ template->eh_host_reset_handler = NULL; - /* Template for all vports this physical port creates */ - memcpy(&phba->vport_template, &lpfc_template, - sizeof(*template)); - phba->vport_template.shost_groups = lpfc_vport_groups; - phba->vport_template.eh_bus_reset_handler = NULL; - phba->vport_template.eh_host_reset_handler = NULL; - phba->vport_template.vendor_id = 0; - - /* Initialize the host templates with updated value */ - if (phba->sli_rev == LPFC_SLI_REV4) { - template->sg_tablesize = phba->cfg_scsi_seg_cnt; - phba->vport_template.sg_tablesize = - phba->cfg_scsi_seg_cnt; - } else { - template->sg_tablesize = phba->cfg_sg_seg_cnt; - phba->vport_template.sg_tablesize = - phba->cfg_sg_seg_cnt; - } - + /* Seed updated value of sg_tablesize */ + template->sg_tablesize = lpfc_get_sg_tablesize(phba); } else { /* NVMET is for physical port only */ - memcpy(template, &lpfc_template_nvme, - sizeof(*template)); + template = &lpfc_template_nvme; } } else { - template = &phba->vport_template; + /* Seed vport template */ + template = &lpfc_vport_template; + + /* Seed updated value of sg_tablesize */ + template->sg_tablesize = lpfc_get_sg_tablesize(phba); } shost = scsi_host_alloc(template, sizeof(struct lpfc_vport)); @@ -4784,11 +4782,6 @@ lpfc_create_port(struct lpfc_hba *phba, int instance, struct device *dev) shost->dma_boundary = phba->sli4_hba.pc_sli4_params.sge_supp_len-1; - - if (phba->cfg_xpsgl && !phba->nvmet_support) - shost->sg_tablesize = LPFC_MAX_SG_TABLESIZE; - else - shost->sg_tablesize = phba->cfg_scsi_seg_cnt; } else /* SLI-3 has a limited number of hardware queues (3), * thus there is only one for FCP processing. @@ -5569,38 +5562,12 @@ lpfc_async_link_speed_to_read_top(struct lpfc_hba *phba, uint8_t speed_code) void lpfc_cgn_dump_rxmonitor(struct lpfc_hba *phba) { - struct rxtable_entry *entry; - int cnt = 0, head, tail, last, start; - - head = atomic_read(&phba->rxtable_idx_head); - tail = atomic_read(&phba->rxtable_idx_tail); - if (!phba->rxtable || head == tail) { - lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT, - "4411 Rxtable is empty\n"); - return; - } - last = tail; - start = head; - - /* Display the last LPFC_MAX_RXMONITOR_DUMP entries from the rxtable */ - while (start != last) { - if (start) - start--; - else - start = LPFC_MAX_RXMONITOR_ENTRY - 1; - entry = &phba->rxtable[start]; + if (!phba->rx_monitor) { lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, - "4410 %02d: MBPI %lld Xmit %lld Cmpl %lld " - "Lat %lld ASz %lld Info %02d BWUtil %d " - "Int %d slot %d\n", - cnt, entry->max_bytes_per_interval, - entry->total_bytes, entry->rcv_bytes, - entry->avg_io_latency, entry->avg_io_size, - entry->cmf_info, entry->timer_utilization, - entry->timer_interval, start); - cnt++; - if (cnt >= LPFC_MAX_RXMONITOR_DUMP) - return; + "4411 Rx Monitor Info is empty.\n"); + } else { + lpfc_rx_monitor_report(phba, phba->rx_monitor, NULL, 0, + LPFC_MAX_RXMONITOR_DUMP); } } @@ -6007,9 +5974,8 @@ lpfc_cmf_timer(struct hrtimer *timer) { struct lpfc_hba *phba = container_of(timer, struct lpfc_hba, cmf_timer); - struct rxtable_entry *entry; + struct rx_info_entry entry; uint32_t io_cnt; - uint32_t head, tail; uint32_t busy, max_read; uint64_t total, rcv, lat, mbpi, extra, cnt; int timer_interval = LPFC_CMF_INTERVAL; @@ -6129,40 +6095,30 @@ lpfc_cmf_timer(struct hrtimer *timer) } /* Save rxmonitor information for debug */ - if (phba->rxtable) { - head = atomic_xchg(&phba->rxtable_idx_head, - LPFC_RXMONITOR_TABLE_IN_USE); - entry = &phba->rxtable[head]; - entry->total_bytes = total; - entry->cmf_bytes = total + extra; - entry->rcv_bytes = rcv; - entry->cmf_busy = busy; - entry->cmf_info = phba->cmf_active_info; + if (phba->rx_monitor) { + entry.total_bytes = total; + entry.cmf_bytes = total + extra; + entry.rcv_bytes = rcv; + entry.cmf_busy = busy; + entry.cmf_info = phba->cmf_active_info; if (io_cnt) { - entry->avg_io_latency = div_u64(lat, io_cnt); - entry->avg_io_size = div_u64(rcv, io_cnt); + entry.avg_io_latency = div_u64(lat, io_cnt); + entry.avg_io_size = div_u64(rcv, io_cnt); } else { - entry->avg_io_latency = 0; - entry->avg_io_size = 0; + entry.avg_io_latency = 0; + entry.avg_io_size = 0; } - entry->max_read_cnt = max_read; - entry->io_cnt = io_cnt; - entry->max_bytes_per_interval = mbpi; + entry.max_read_cnt = max_read; + entry.io_cnt = io_cnt; + entry.max_bytes_per_interval = mbpi; if (phba->cmf_active_mode == LPFC_CFG_MANAGED) - entry->timer_utilization = phba->cmf_last_ts; + entry.timer_utilization = phba->cmf_last_ts; else - entry->timer_utilization = ms; - entry->timer_interval = ms; + entry.timer_utilization = ms; + entry.timer_interval = ms; phba->cmf_last_ts = 0; - /* Increment rxtable index */ - head = (head + 1) % LPFC_MAX_RXMONITOR_ENTRY; - tail = atomic_read(&phba->rxtable_idx_tail); - if (head == tail) { - tail = (tail + 1) % LPFC_MAX_RXMONITOR_ENTRY; - atomic_set(&phba->rxtable_idx_tail, tail); - } - atomic_set(&phba->rxtable_idx_head, head); + lpfc_rx_monitor_record(phba->rx_monitor, &entry); } if (phba->cmf_active_mode == LPFC_CFG_MONITOR) { @@ -6232,6 +6188,7 @@ lpfc_update_trunk_link_status(struct lpfc_hba *phba, { uint8_t port_fault = bf_get(lpfc_acqe_fc_la_trunk_linkmask, acqe_fc); uint8_t err = bf_get(lpfc_acqe_fc_la_trunk_fault, acqe_fc); + u8 cnt = 0; phba->sli4_hba.link_state.speed = lpfc_sli4_port_speed_parse(phba, LPFC_TRAILER_CODE_FC, @@ -6250,26 +6207,36 @@ lpfc_update_trunk_link_status(struct lpfc_hba *phba, bf_get(lpfc_acqe_fc_la_trunk_link_status_port0, acqe_fc) ? LPFC_LINK_UP : LPFC_LINK_DOWN; phba->trunk_link.link0.fault = port_fault & 0x1 ? err : 0; + cnt++; } if (bf_get(lpfc_acqe_fc_la_trunk_config_port1, acqe_fc)) { phba->trunk_link.link1.state = bf_get(lpfc_acqe_fc_la_trunk_link_status_port1, acqe_fc) ? LPFC_LINK_UP : LPFC_LINK_DOWN; phba->trunk_link.link1.fault = port_fault & 0x2 ? err : 0; + cnt++; } if (bf_get(lpfc_acqe_fc_la_trunk_config_port2, acqe_fc)) { phba->trunk_link.link2.state = bf_get(lpfc_acqe_fc_la_trunk_link_status_port2, acqe_fc) ? LPFC_LINK_UP : LPFC_LINK_DOWN; phba->trunk_link.link2.fault = port_fault & 0x4 ? err : 0; + cnt++; } if (bf_get(lpfc_acqe_fc_la_trunk_config_port3, acqe_fc)) { phba->trunk_link.link3.state = bf_get(lpfc_acqe_fc_la_trunk_link_status_port3, acqe_fc) ? LPFC_LINK_UP : LPFC_LINK_DOWN; phba->trunk_link.link3.fault = port_fault & 0x8 ? err : 0; + cnt++; } + if (cnt) + phba->trunk_link.phy_lnk_speed = + phba->sli4_hba.link_state.logical_speed / (cnt * 1000); + else + phba->trunk_link.phy_lnk_speed = LPFC_LINK_SPEED_UNKNOWN; + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2910 Async FC Trunking Event - Speed:%d\n" "\tLogical speed:%d " @@ -6347,7 +6314,7 @@ lpfc_sli4_async_fc_evt(struct lpfc_hba *phba, struct lpfc_acqe_fc_la *acqe_fc) if (bf_get(lpfc_acqe_fc_la_att_type, acqe_fc) == LPFC_FC_LA_TYPE_LINK_DOWN) phba->sli4_hba.link_state.logical_speed = 0; - else if (!phba->sli4_hba.conf_trunk) + else if (!phba->sli4_hba.conf_trunk) phba->sli4_hba.link_state.logical_speed = bf_get(lpfc_acqe_fc_la_llink_spd, acqe_fc) * 10; @@ -6465,7 +6432,7 @@ lpfc_sli4_async_sli_evt(struct lpfc_hba *phba, struct lpfc_acqe_sli *acqe_sli) "2901 Async SLI event - Type:%d, Event Data: x%08x " "x%08x x%08x x%08x\n", evt_type, acqe_sli->event_data1, acqe_sli->event_data2, - acqe_sli->reserved, acqe_sli->trailer); + acqe_sli->event_data3, acqe_sli->trailer); port_name = phba->Port[0]; if (port_name == 0x00) @@ -6494,7 +6461,7 @@ lpfc_sli4_async_sli_evt(struct lpfc_hba *phba, struct lpfc_acqe_sli *acqe_sli) temp_event_data.event_code = LPFC_NORMAL_TEMP; temp_event_data.data = (uint32_t)acqe_sli->event_data1; - lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_LDS_EVENT, "3191 Normal Temperature:%d Celsius - Port Name %c\n", acqe_sli->event_data1, port_name); @@ -6672,6 +6639,15 @@ lpfc_sli4_async_sli_evt(struct lpfc_hba *phba, struct lpfc_acqe_sli *acqe_sli) } } break; + case LPFC_SLI_EVENT_TYPE_RD_SIGNAL: + /* May be accompanied by a temperature event */ + lpfc_printf_log(phba, KERN_INFO, + LOG_SLI | LOG_LINK_EVENT | LOG_LDS_EVENT, + "2902 Remote Degrade Signaling: x%08x x%08x " + "x%08x\n", + acqe_sli->event_data1, acqe_sli->event_data2, + acqe_sli->event_data3); + break; default: lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "3193 Unrecognized SLI event, type: 0x%x", @@ -7085,6 +7061,12 @@ lpfc_cgn_params_val(struct lpfc_hba *phba, struct lpfc_cgn_param *p_cfg_param) spin_unlock_irq(&phba->hbalock); } +static const char * const lpfc_cmf_mode_to_str[] = { + "OFF", + "MANAGED", + "MONITOR", +}; + /** * lpfc_cgn_params_parse - Process a FW cong parm change event * @phba: pointer to lpfc hba data structure. @@ -7104,6 +7086,7 @@ lpfc_cgn_params_parse(struct lpfc_hba *phba, { struct lpfc_cgn_info *cp; uint32_t crc, oldmode; + char acr_string[4] = {0}; /* Make sure the FW has encoded the correct magic number to * validate the congestion parameter in FW memory. @@ -7180,9 +7163,6 @@ lpfc_cgn_params_parse(struct lpfc_hba *phba, lpfc_issue_els_edc(phba->pport, 0); break; case LPFC_CFG_MONITOR: - lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, - "4661 Switch from MANAGED to " - "`MONITOR mode\n"); phba->cmf_max_bytes_per_interval = phba->cmf_link_byte_count; @@ -7201,14 +7181,26 @@ lpfc_cgn_params_parse(struct lpfc_hba *phba, lpfc_issue_els_edc(phba->pport, 0); break; case LPFC_CFG_MANAGED: - lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, - "4662 Switch from MONITOR to " - "MANAGED mode\n"); lpfc_cmf_signal_init(phba); break; } break; } + if (oldmode != LPFC_CFG_OFF || + oldmode != phba->cgn_p.cgn_param_mode) { + if (phba->cgn_p.cgn_param_mode == LPFC_CFG_MANAGED) + scnprintf(acr_string, sizeof(acr_string), "%u", + phba->cgn_p.cgn_param_level0); + else + scnprintf(acr_string, sizeof(acr_string), "NA"); + + dev_info(&phba->pcidev->dev, "%d: " + "4663 CMF: Mode %s acr %s\n", + phba->brd_no, + lpfc_cmf_mode_to_str + [phba->cgn_p.cgn_param_mode], + acr_string); + } } else { lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT, "4669 FW cgn parm buf wrong magic 0x%x " @@ -8053,7 +8045,7 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) /* Allocate device driver memory */ rc = lpfc_mem_alloc(phba, SGL_ALIGN_SZ); if (rc) - return -ENOMEM; + goto out_destroy_workqueue; /* IF Type 2 ports get initialized now. */ if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) >= @@ -8315,8 +8307,10 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) &phba->pcidev->dev, phba->cfg_sg_dma_buf_size, i, 0); - if (!phba->lpfc_sg_dma_buf_pool) + if (!phba->lpfc_sg_dma_buf_pool) { + rc = -ENOMEM; goto out_free_bsmbx; + } phba->lpfc_cmd_rsp_buf_pool = dma_pool_create("lpfc_cmd_rsp_buf_pool", @@ -8324,8 +8318,10 @@ lpfc_sli4_driver_resource_setup(struct lpfc_hba *phba) sizeof(struct fcp_cmnd) + sizeof(struct fcp_rsp), i, 0); - if (!phba->lpfc_cmd_rsp_buf_pool) + if (!phba->lpfc_cmd_rsp_buf_pool) { + rc = -ENOMEM; goto out_free_sg_dma_buf; + } mempool_free(mboxq, phba->mbox_mem_pool); @@ -8481,6 +8477,9 @@ out_free_bsmbx: lpfc_destroy_bootstrap_mbox(phba); out_free_mem: lpfc_mem_free(phba); +out_destroy_workqueue: + destroy_workqueue(phba->wq); + phba->wq = NULL; return rc; } @@ -12413,7 +12412,7 @@ lpfc_hba_eq_hdl_array_init(struct lpfc_hba *phba) for (i = 0; i < phba->cfg_irq_chann; i++) { eqhdl = lpfc_get_eq_hdl(i); - eqhdl->irq = LPFC_VECTOR_MAP_EMPTY; + eqhdl->irq = LPFC_IRQ_EMPTY; eqhdl->phba = phba; } } @@ -12786,7 +12785,7 @@ static void __lpfc_cpuhp_remove(struct lpfc_hba *phba) static void lpfc_cpuhp_remove(struct lpfc_hba *phba) { - if (phba->pport->fc_flag & FC_OFFLINE_MODE) + if (phba->pport && (phba->pport->fc_flag & FC_OFFLINE_MODE)) return; __lpfc_cpuhp_remove(phba); @@ -13050,9 +13049,17 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) LPFC_DRIVER_HANDLER_NAME"%d", index); eqhdl->idx = index; - rc = request_irq(pci_irq_vector(phba->pcidev, index), - &lpfc_sli4_hba_intr_handler, 0, - name, eqhdl); + rc = pci_irq_vector(phba->pcidev, index); + if (rc < 0) { + lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, + "0489 MSI-X fast-path (%d) " + "pci_irq_vec failed (%d)\n", index, rc); + goto cfg_fail_out; + } + eqhdl->irq = rc; + + rc = request_irq(eqhdl->irq, &lpfc_sli4_hba_intr_handler, 0, + name, eqhdl); if (rc) { lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, "0486 MSI-X fast-path (%d) " @@ -13060,8 +13067,6 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba) goto cfg_fail_out; } - eqhdl->irq = pci_irq_vector(phba->pcidev, index); - if (aff_mask) { /* If found a neighboring online cpu, set affinity */ if (cpu_select < nr_cpu_ids) @@ -13178,7 +13183,14 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) } eqhdl = lpfc_get_eq_hdl(0); - eqhdl->irq = pci_irq_vector(phba->pcidev, 0); + rc = pci_irq_vector(phba->pcidev, 0); + if (rc < 0) { + pci_free_irq_vectors(phba->pcidev); + lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, + "0496 MSI pci_irq_vec failed (%d)\n", rc); + return rc; + } + eqhdl->irq = rc; cpu = cpumask_first(cpu_present_mask); lpfc_assign_eq_map_info(phba, 0, LPFC_CPU_FIRST_IRQ, cpu); @@ -13205,8 +13217,8 @@ lpfc_sli4_enable_msi(struct lpfc_hba *phba) * MSI-X -> MSI -> IRQ. * * Return codes - * 0 - successful - * other values - error + * Interrupt mode (2, 1, 0) - successful + * LPFC_INTR_ERROR - error **/ static uint32_t lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) @@ -13251,7 +13263,14 @@ lpfc_sli4_enable_intr(struct lpfc_hba *phba, uint32_t cfg_mode) intr_mode = 0; eqhdl = lpfc_get_eq_hdl(0); - eqhdl->irq = pci_irq_vector(phba->pcidev, 0); + retval = pci_irq_vector(phba->pcidev, 0); + if (retval < 0) { + lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, + "0502 INTR pci_irq_vec failed (%d)\n", + retval); + return LPFC_INTR_ERROR; + } + eqhdl->irq = retval; cpu = cpumask_first(cpu_present_mask); lpfc_assign_eq_map_info(phba, 0, LPFC_CPU_FIRST_IRQ, diff --git a/drivers/scsi/lpfc/lpfc_logmsg.h b/drivers/scsi/lpfc/lpfc_logmsg.h index 4d455da9cd69e0ffc41728a8ce7ade9b0ed0572f..b39cefcd8703f9a1c9cb490ddb9e8e4c3f659610 100644 --- a/drivers/scsi/lpfc/lpfc_logmsg.h +++ b/drivers/scsi/lpfc/lpfc_logmsg.h @@ -35,7 +35,7 @@ #define LOG_FCP_ERROR 0x00001000 /* log errors, not underruns */ #define LOG_LIBDFC 0x00002000 /* Libdfc events */ #define LOG_VPORT 0x00004000 /* NPIV events */ -#define LOG_SECURITY 0x00008000 /* Security events */ +#define LOG_LDS_EVENT 0x00008000 /* Link Degrade Signaling events */ #define LOG_EVENT 0x00010000 /* CT,TEMP,DUMP, logging */ #define LOG_FIP 0x00020000 /* FIP events */ #define LOG_FCP_UNDER 0x00040000 /* FCP underruns errors */ diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c index 870e53b8f81dd2bfcd34d14024b48caf3ea5ff83..89cbeba06aeab675f9f81b86ba3f57ac53253f9b 100644 --- a/drivers/scsi/lpfc/lpfc_mem.c +++ b/drivers/scsi/lpfc/lpfc_mem.c @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2014 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -344,9 +344,12 @@ lpfc_mem_free_all(struct lpfc_hba *phba) phba->cgn_i = NULL; } - /* Free RX table */ - kfree(phba->rxtable); - phba->rxtable = NULL; + /* Free RX Monitor */ + if (phba->rx_monitor) { + lpfc_rx_monitor_destroy_ring(phba->rx_monitor); + kfree(phba->rx_monitor); + phba->rx_monitor = NULL; + } /* Free the iocb lookup array */ kfree(psli->iocbq_lookup); diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c index 084c0f9fdc3a6bc2c7f78385b68e6373b8fd02c8..7a1563564df7f3baf1a8727c8d3fd0a48f917921 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.c +++ b/drivers/scsi/lpfc/lpfc_scsi.c @@ -111,62 +111,6 @@ lpfc_sli4_set_rsp_sgl_last(struct lpfc_hba *phba, #define LPFC_INVALID_REFTAG ((u32)-1) -/** - * lpfc_update_stats - Update statistical data for the command completion - * @vport: The virtual port on which this call is executing. - * @lpfc_cmd: lpfc scsi command object pointer. - * - * This function is called when there is a command completion and this - * function updates the statistical data for the command completion. - **/ -static void -lpfc_update_stats(struct lpfc_vport *vport, struct lpfc_io_buf *lpfc_cmd) -{ - struct lpfc_hba *phba = vport->phba; - struct lpfc_rport_data *rdata; - struct lpfc_nodelist *pnode; - struct scsi_cmnd *cmd = lpfc_cmd->pCmd; - unsigned long flags; - struct Scsi_Host *shost = lpfc_shost_from_vport(vport); - unsigned long latency; - int i; - - if (!vport->stat_data_enabled || - vport->stat_data_blocked || - (cmd->result)) - return; - - latency = jiffies_to_msecs((long)jiffies - (long)lpfc_cmd->start_time); - rdata = lpfc_cmd->rdata; - pnode = rdata->pnode; - - spin_lock_irqsave(shost->host_lock, flags); - if (!pnode || - !pnode->lat_data || - (phba->bucket_type == LPFC_NO_BUCKET)) { - spin_unlock_irqrestore(shost->host_lock, flags); - return; - } - - if (phba->bucket_type == LPFC_LINEAR_BUCKET) { - i = (latency + phba->bucket_step - 1 - phba->bucket_base)/ - phba->bucket_step; - /* check array subscript bounds */ - if (i < 0) - i = 0; - else if (i >= LPFC_MAX_BUCKET_COUNT) - i = LPFC_MAX_BUCKET_COUNT - 1; - } else { - for (i = 0; i < LPFC_MAX_BUCKET_COUNT-1; i++) - if (latency <= (phba->bucket_base + - ((1<bucket_step))) - break; - } - - pnode->lat_data[i].cmd_count++; - spin_unlock_irqrestore(shost->host_lock, flags); -} - /** * lpfc_rampdown_queue_depth - Post RAMP_DOWN_QUEUE event to worker thread * @phba: The Hba for which this call is being executed. @@ -4272,7 +4216,7 @@ lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, lpfc_cmd->result == IOERR_ABORT_REQUESTED || lpfc_cmd->result == IOERR_RPI_SUSPENDED || lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) { - cmd->result = DID_REQUEUE << 16; + cmd->result = DID_TRANSPORT_DISRUPTED << 16; break; } if ((lpfc_cmd->result == IOERR_RX_DMA_FAILED || @@ -4335,8 +4279,6 @@ lpfc_fcp_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, cmd->retries, scsi_get_resid(cmd)); } - lpfc_update_stats(vport, lpfc_cmd); - if (vport->cfg_max_scsicmpl_time && time_after(jiffies, lpfc_cmd->start_time + msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) { @@ -4562,7 +4504,7 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, lpfc_cmd->result == IOERR_NO_RESOURCES || lpfc_cmd->result == IOERR_ABORT_REQUESTED || lpfc_cmd->result == IOERR_SLER_CMD_RCV_FAILURE) { - cmd->result = DID_REQUEUE << 16; + cmd->result = DID_TRANSPORT_DISRUPTED << 16; break; } if ((lpfc_cmd->result == IOERR_RX_DMA_FAILED || @@ -4617,7 +4559,6 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn, scsi_get_resid(cmd)); } - lpfc_update_stats(vport, lpfc_cmd); if (vport->cfg_max_scsicmpl_time && time_after(jiffies, lpfc_cmd->start_time + msecs_to_jiffies(vport->cfg_max_scsicmpl_time))) { @@ -6853,3 +6794,30 @@ struct scsi_host_template lpfc_template = { .change_queue_depth = scsi_change_queue_depth, .track_queue_depth = 1, }; + +struct scsi_host_template lpfc_vport_template = { + .module = THIS_MODULE, + .name = LPFC_DRIVER_NAME, + .proc_name = LPFC_DRIVER_NAME, + .info = lpfc_info, + .queuecommand = lpfc_queuecommand, + .eh_timed_out = fc_eh_timed_out, + .eh_should_retry_cmd = fc_eh_should_retry_cmd, + .eh_abort_handler = lpfc_abort_handler, + .eh_device_reset_handler = lpfc_device_reset_handler, + .eh_target_reset_handler = lpfc_target_reset_handler, + .eh_bus_reset_handler = NULL, + .eh_host_reset_handler = NULL, + .slave_alloc = lpfc_slave_alloc, + .slave_configure = lpfc_slave_configure, + .slave_destroy = lpfc_slave_destroy, + .scan_finished = lpfc_scan_finished, + .this_id = -1, + .sg_tablesize = LPFC_DEFAULT_SG_SEG_CNT, + .cmd_per_lun = LPFC_CMD_PER_LUN, + .shost_groups = lpfc_vport_groups, + .max_sectors = 0xFFFFFFFF, + .vendor_id = 0, + .change_queue_depth = scsi_change_queue_depth, + .track_queue_depth = 1, +}; diff --git a/drivers/scsi/lpfc/lpfc_scsi.h b/drivers/scsi/lpfc/lpfc_scsi.h index 3836d7f6a5757ce3a3285143fd82ccc774ebf51c..eae56944f31b59ff972ba3c5299dca2d0ee268b2 100644 --- a/drivers/scsi/lpfc/lpfc_scsi.h +++ b/drivers/scsi/lpfc/lpfc_scsi.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2021 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -126,10 +126,6 @@ struct fcp_cmnd { }; -struct lpfc_scsicmd_bkt { - uint32_t cmd_count; -}; - #define LPFC_SCSI_DMA_EXT_SIZE 264 #define LPFC_BPL_SIZE 1024 #define MDAC_DIRECT_CMD 0x22 diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 608016725db99da2c62d8ac6e0bd6c0b28727175..99d06dc7ddf6b5e8fa993663a17e191a88546722 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -1916,6 +1916,7 @@ lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total) unsigned long iflags; u32 ret_val; u32 atot, wtot, max; + u16 warn_sync_period = 0; /* First address any alarm / warning activity */ atot = atomic_xchg(&phba->cgn_sync_alarm_cnt, 0); @@ -1970,10 +1971,14 @@ lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total) lpfc_acqe_cgn_frequency; bf_set(cmf_sync_wsigmax, &wqe->cmf_sync, max); bf_set(cmf_sync_wsigcnt, &wqe->cmf_sync, wtot); + warn_sync_period = lpfc_acqe_cgn_frequency; } else { /* We hit a FPIN warning condition */ bf_set(cmf_sync_wfpinmax, &wqe->cmf_sync, 1); bf_set(cmf_sync_wfpincnt, &wqe->cmf_sync, 1); + if (phba->cgn_fpin_frequency != LPFC_FPIN_INIT_FREQ) + warn_sync_period = + LPFC_MSECS_TO_SECS(phba->cgn_fpin_frequency); } } @@ -1989,6 +1994,7 @@ initpath: bf_set(cmf_sync_reqtag, &wqe->cmf_sync, sync_buf->iotag); bf_set(cmf_sync_qosd, &wqe->cmf_sync, 1); + bf_set(cmf_sync_period, &wqe->cmf_sync, warn_sync_period); bf_set(cmf_sync_cmd_type, &wqe->cmf_sync, CMF_SYNC_COMMAND); bf_set(cmf_sync_wqec, &wqe->cmf_sync, 1); @@ -2850,6 +2856,7 @@ void lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) { struct lpfc_vport *vport = pmb->vport; + struct lpfc_dmabuf *mp; struct lpfc_nodelist *ndlp; struct Scsi_Host *shost; uint16_t rpi, vpi; @@ -2862,6 +2869,12 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) if (!(phba->pport->load_flag & FC_UNLOADING) && pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 && !pmb->u.mb.mbxStatus) { + mp = (struct lpfc_dmabuf *)pmb->ctx_buf; + if (mp) { + pmb->ctx_buf = NULL; + lpfc_mbuf_free(phba, mp->virt, mp->phys); + kfree(mp); + } rpi = pmb->u.mb.un.varWords[0]; vpi = pmb->u.mb.un.varRegLogin.vpi; if (phba->sli_rev == LPFC_SLI_REV4) @@ -6202,6 +6215,9 @@ lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type, struct lpfc_mbx_get_rsrc_extent_info *rsrc_info; LPFC_MBOXQ_t *mbox; + *extnt_count = 0; + *extnt_size = 0; + mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -6817,8 +6833,13 @@ lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox, bf_set(lpfc_mbx_set_feature_mi, &mbox->u.mqe.un.set_feature, phba->sli4_hba.pc_sli4_params.mi_ver); break; + case LPFC_SET_LD_SIGNAL: + mbox->u.mqe.un.set_feature.feature = LPFC_SET_LD_SIGNAL; + mbox->u.mqe.un.set_feature.param_len = 16; + bf_set(lpfc_mbx_set_feature_lds_qry, + &mbox->u.mqe.un.set_feature, LPFC_QUERY_LDS_OP); + break; case LPFC_SET_ENABLE_CMF: - bf_set(lpfc_mbx_set_feature_dd, &mbox->u.mqe.un.set_feature, 1); mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_CMF; mbox->u.mqe.un.set_feature.param_len = 4; bf_set(lpfc_mbx_set_feature_cmf, @@ -7813,6 +7834,62 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, return 1; } +static void +lpfc_mbx_cmpl_read_lds_params(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) +{ + union lpfc_sli4_cfg_shdr *shdr; + u32 shdr_status, shdr_add_status; + + shdr = (union lpfc_sli4_cfg_shdr *) + &pmb->u.mqe.un.sli4_config.header.cfg_shdr; + shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); + shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); + if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) { + lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT | LOG_MBOX, + "4622 SET_FEATURE (x%x) mbox failed, " + "status x%x add_status x%x, mbx status x%x\n", + LPFC_SET_LD_SIGNAL, shdr_status, + shdr_add_status, pmb->u.mb.mbxStatus); + phba->degrade_activate_threshold = 0; + phba->degrade_deactivate_threshold = 0; + phba->fec_degrade_interval = 0; + goto out; + } + + phba->degrade_activate_threshold = pmb->u.mqe.un.set_feature.word7; + phba->degrade_deactivate_threshold = pmb->u.mqe.un.set_feature.word8; + phba->fec_degrade_interval = pmb->u.mqe.un.set_feature.word10; + + lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT, + "4624 Success: da x%x dd x%x interval x%x\n", + phba->degrade_activate_threshold, + phba->degrade_deactivate_threshold, + phba->fec_degrade_interval); +out: + mempool_free(pmb, phba->mbox_mem_pool); +} + +int +lpfc_read_lds_params(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + int rc; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + return -ENOMEM; + + lpfc_set_features(phba, mboxq, LPFC_SET_LD_SIGNAL); + mboxq->vport = phba->pport; + mboxq->mbox_cmpl = lpfc_mbx_cmpl_read_lds_params; + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); + if (rc == MBX_NOT_FINISHED) { + mempool_free(mboxq, phba->mbox_mem_pool); + return -EIO; + } + return 0; +} + static void lpfc_mbx_cmpl_cgn_set_ftrs(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) { @@ -7959,6 +8036,172 @@ static void lpfc_sli4_dip(struct lpfc_hba *phba) } } +/** + * lpfc_rx_monitor_create_ring - Initialize ring buffer for rx_monitor + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @entries: Number of rx_info_entry objects to allocate in ring + * + * Return: + * 0 - Success + * ENOMEM - Failure to kmalloc + **/ +int lpfc_rx_monitor_create_ring(struct lpfc_rx_info_monitor *rx_monitor, + u32 entries) +{ + rx_monitor->ring = kmalloc_array(entries, sizeof(struct rx_info_entry), + GFP_KERNEL); + if (!rx_monitor->ring) + return -ENOMEM; + + rx_monitor->head_idx = 0; + rx_monitor->tail_idx = 0; + spin_lock_init(&rx_monitor->lock); + rx_monitor->entries = entries; + + return 0; +} + +/** + * lpfc_rx_monitor_destroy_ring - Free ring buffer for rx_monitor + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + **/ +void lpfc_rx_monitor_destroy_ring(struct lpfc_rx_info_monitor *rx_monitor) +{ + spin_lock(&rx_monitor->lock); + kfree(rx_monitor->ring); + rx_monitor->ring = NULL; + rx_monitor->entries = 0; + rx_monitor->head_idx = 0; + rx_monitor->tail_idx = 0; + spin_unlock(&rx_monitor->lock); +} + +/** + * lpfc_rx_monitor_record - Insert an entry into rx_monitor's ring + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @entry: Pointer to rx_info_entry + * + * Used to insert an rx_info_entry into rx_monitor's ring. Note that this is a + * deep copy of rx_info_entry not a shallow copy of the rx_info_entry ptr. + * + * This is called from lpfc_cmf_timer, which is in timer/softirq context. + * + * In cases of old data overflow, we do a best effort of FIFO order. + **/ +void lpfc_rx_monitor_record(struct lpfc_rx_info_monitor *rx_monitor, + struct rx_info_entry *entry) +{ + struct rx_info_entry *ring = rx_monitor->ring; + u32 *head_idx = &rx_monitor->head_idx; + u32 *tail_idx = &rx_monitor->tail_idx; + spinlock_t *ring_lock = &rx_monitor->lock; + u32 ring_size = rx_monitor->entries; + + spin_lock(ring_lock); + memcpy(&ring[*tail_idx], entry, sizeof(*entry)); + *tail_idx = (*tail_idx + 1) % ring_size; + + /* Best effort of FIFO saved data */ + if (*tail_idx == *head_idx) + *head_idx = (*head_idx + 1) % ring_size; + + spin_unlock(ring_lock); +} + +/** + * lpfc_rx_monitor_report - Read out rx_monitor's ring + * @phba: Pointer to lpfc_hba object + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @buf: Pointer to char buffer that will contain rx monitor info data + * @buf_len: Length buf including null char + * @max_read_entries: Maximum number of entries to read out of ring + * + * Used to dump/read what's in rx_monitor's ring buffer. + * + * If buf is NULL || buf_len == 0, then it is implied that we want to log the + * information to kmsg instead of filling out buf. + * + * Return: + * Number of entries read out of the ring + **/ +u32 lpfc_rx_monitor_report(struct lpfc_hba *phba, + struct lpfc_rx_info_monitor *rx_monitor, char *buf, + u32 buf_len, u32 max_read_entries) +{ + struct rx_info_entry *ring = rx_monitor->ring; + struct rx_info_entry *entry; + u32 *head_idx = &rx_monitor->head_idx; + u32 *tail_idx = &rx_monitor->tail_idx; + spinlock_t *ring_lock = &rx_monitor->lock; + u32 ring_size = rx_monitor->entries; + u32 cnt = 0; + char tmp[DBG_LOG_STR_SZ] = {0}; + bool log_to_kmsg = (!buf || !buf_len) ? true : false; + + if (!log_to_kmsg) { + /* clear the buffer to be sure */ + memset(buf, 0, buf_len); + + scnprintf(buf, buf_len, "\t%-16s%-16s%-16s%-16s%-8s%-8s%-8s" + "%-8s%-8s%-8s%-16s\n", + "MaxBPI", "Tot_Data_CMF", + "Tot_Data_Cmd", "Tot_Data_Cmpl", + "Lat(us)", "Avg_IO", "Max_IO", "Bsy", + "IO_cnt", "Info", "BWutil(ms)"); + } + + /* Needs to be _bh because record is called from timer interrupt + * context + */ + spin_lock_bh(ring_lock); + while (*head_idx != *tail_idx) { + entry = &ring[*head_idx]; + + /* Read out this entry's data. */ + if (!log_to_kmsg) { + /* If !log_to_kmsg, then store to buf. */ + scnprintf(tmp, sizeof(tmp), + "%03d:\t%-16llu%-16llu%-16llu%-16llu%-8llu" + "%-8llu%-8llu%-8u%-8u%-8u%u(%u)\n", + *head_idx, entry->max_bytes_per_interval, + entry->cmf_bytes, entry->total_bytes, + entry->rcv_bytes, entry->avg_io_latency, + entry->avg_io_size, entry->max_read_cnt, + entry->cmf_busy, entry->io_cnt, + entry->cmf_info, entry->timer_utilization, + entry->timer_interval); + + /* Check for buffer overflow */ + if ((strlen(buf) + strlen(tmp)) >= buf_len) + break; + + /* Append entry's data to buffer */ + strlcat(buf, tmp, buf_len); + } else { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "4410 %02u: MBPI %llu Xmit %llu " + "Cmpl %llu Lat %llu ASz %llu Info %02u " + "BWUtil %u Int %u slot %u\n", + cnt, entry->max_bytes_per_interval, + entry->total_bytes, entry->rcv_bytes, + entry->avg_io_latency, + entry->avg_io_size, entry->cmf_info, + entry->timer_utilization, + entry->timer_interval, *head_idx); + } + + *head_idx = (*head_idx + 1) % ring_size; + + /* Don't feed more than max_read_entries */ + cnt++; + if (cnt >= max_read_entries) + break; + } + spin_unlock_bh(ring_lock); + + return cnt; +} + /** * lpfc_cmf_setup - Initialize idle_stat tracking * @phba: Pointer to HBA context object. @@ -8133,19 +8376,29 @@ no_cmf: phba->cmf_interval_rate = LPFC_CMF_INTERVAL; /* Allocate RX Monitor Buffer */ - if (!phba->rxtable) { - phba->rxtable = kmalloc_array(LPFC_MAX_RXMONITOR_ENTRY, - sizeof(struct rxtable_entry), - GFP_KERNEL); - if (!phba->rxtable) { + if (!phba->rx_monitor) { + phba->rx_monitor = kzalloc(sizeof(*phba->rx_monitor), + GFP_KERNEL); + + if (!phba->rx_monitor) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "2644 Failed to alloc memory " "for RX Monitor Buffer\n"); return -ENOMEM; } + + /* Instruct the rx_monitor object to instantiate its ring */ + if (lpfc_rx_monitor_create_ring(phba->rx_monitor, + LPFC_MAX_RXMONITOR_ENTRY)) { + kfree(phba->rx_monitor); + phba->rx_monitor = NULL; + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "2645 Failed to alloc memory " + "for RX Monitor's Ring\n"); + return -ENOMEM; + } } - atomic_set(&phba->rxtable_idx_head, 0); - atomic_set(&phba->rxtable_idx_tail, 0); + return 0; } @@ -10322,12 +10575,10 @@ static int __lpfc_sli_issue_fcp_io_s4(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { - int rc; struct lpfc_io_buf *lpfc_cmd = piocb->io_buf; lpfc_prep_embed_io(phba, lpfc_cmd); - rc = lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, piocb); - return rc; + return lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, piocb); } void diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h index 1ddad5b170a60a88f48a6efe02df7d3c935ae99f..cbb1aa1cf025bacf16c019a6342da43681a7560d 100644 --- a/drivers/scsi/lpfc/lpfc_sli4.h +++ b/drivers/scsi/lpfc/lpfc_sli4.h @@ -489,7 +489,7 @@ struct lpfc_hba; #define LPFC_SLI4_HANDLER_NAME_SZ 16 struct lpfc_hba_eq_hdl { uint32_t idx; - uint16_t irq; + int irq; char handler_name[LPFC_SLI4_HANDLER_NAME_SZ]; struct lpfc_hba *phba; struct lpfc_queue *eq; @@ -611,6 +611,8 @@ struct lpfc_vector_map_info { }; #define LPFC_VECTOR_MAP_EMPTY 0xffff +#define LPFC_IRQ_EMPTY 0xffffffff + /* Multi-XRI pool */ #define XRI_BATCH 8 diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h index 63eba9928e4b8c389469e94005d4bf86faa87435..192d5630a44daf3509e54476f3969e1102ae5ffb 100644 --- a/drivers/scsi/lpfc/lpfc_version.h +++ b/drivers/scsi/lpfc/lpfc_version.h @@ -20,7 +20,7 @@ * included with this package. * *******************************************************************/ -#define LPFC_DRIVER_VERSION "14.2.0.5" +#define LPFC_DRIVER_VERSION "14.2.0.7" #define LPFC_DRIVER_NAME "lpfc" /* Used for SLI 2/3 */ diff --git a/drivers/scsi/lpfc/lpfc_vmid.c b/drivers/scsi/lpfc/lpfc_vmid.c index f64ced04b912582998d5cda5bab361f0f6fb271b..ed1d7f7b88a3813d20b8372d22a4fa06f575eb9a 100644 --- a/drivers/scsi/lpfc/lpfc_vmid.c +++ b/drivers/scsi/lpfc/lpfc_vmid.c @@ -245,9 +245,7 @@ int lpfc_vmid_get_appid(struct lpfc_vport *vport, char *uuid, /* allocate the per cpu variable for holding */ /* the last access time stamp only if VMID is enabled */ if (!vmp->last_io_time) - vmp->last_io_time = __alloc_percpu(sizeof(u64), - __alignof__(struct - lpfc_vmid)); + vmp->last_io_time = alloc_percpu_gfp(u64, GFP_ATOMIC); if (!vmp->last_io_time) { hash_del(&vmp->hnode); vmp->flag = LPFC_VMID_SLOT_FREE; diff --git a/drivers/scsi/lpfc/lpfc_vport.c b/drivers/scsi/lpfc/lpfc_vport.c index e7efb025ed5020579de7908999feae3b5a332a8e..4d171f5c213f7350a2cbf8e9957dbeda17ac6ded 100644 --- a/drivers/scsi/lpfc/lpfc_vport.c +++ b/drivers/scsi/lpfc/lpfc_vport.c @@ -809,74 +809,3 @@ lpfc_destroy_vport_work_array(struct lpfc_hba *phba, struct lpfc_vport **vports) kfree(vports); } - -/** - * lpfc_vport_reset_stat_data - Reset the statistical data for the vport - * @vport: Pointer to vport object. - * - * This function resets the statistical data for the vport. This function - * is called with the host_lock held - **/ -void -lpfc_vport_reset_stat_data(struct lpfc_vport *vport) -{ - struct lpfc_nodelist *ndlp = NULL, *next_ndlp = NULL; - - list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes, nlp_listp) { - if (ndlp->lat_data) - memset(ndlp->lat_data, 0, LPFC_MAX_BUCKET_COUNT * - sizeof(struct lpfc_scsicmd_bkt)); - } -} - - -/** - * lpfc_alloc_bucket - Allocate data buffer required for statistical data - * @vport: Pointer to vport object. - * - * This function allocates data buffer required for all the FC - * nodes of the vport to collect statistical data. - **/ -void -lpfc_alloc_bucket(struct lpfc_vport *vport) -{ - struct lpfc_nodelist *ndlp = NULL, *next_ndlp = NULL; - - list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes, nlp_listp) { - - kfree(ndlp->lat_data); - ndlp->lat_data = NULL; - - if (ndlp->nlp_state == NLP_STE_MAPPED_NODE) { - ndlp->lat_data = kcalloc(LPFC_MAX_BUCKET_COUNT, - sizeof(struct lpfc_scsicmd_bkt), - GFP_ATOMIC); - - if (!ndlp->lat_data) - lpfc_printf_vlog(vport, KERN_ERR, - LOG_TRACE_EVENT, - "0287 lpfc_alloc_bucket failed to " - "allocate statistical data buffer DID " - "0x%x\n", ndlp->nlp_DID); - } - } -} - -/** - * lpfc_free_bucket - Free data buffer required for statistical data - * @vport: Pointer to vport object. - * - * Th function frees statistical data buffer of all the FC - * nodes of the vport. - **/ -void -lpfc_free_bucket(struct lpfc_vport *vport) -{ - struct lpfc_nodelist *ndlp = NULL, *next_ndlp = NULL; - - list_for_each_entry_safe(ndlp, next_ndlp, &vport->fc_nodes, nlp_listp) { - - kfree(ndlp->lat_data); - ndlp->lat_data = NULL; - } -} diff --git a/drivers/scsi/lpfc/lpfc_vport.h b/drivers/scsi/lpfc/lpfc_vport.h index f4b8528dd2e730c26a04bfdc5973c5d8e275adb8..fa60c146c16919815af9fa056e902262f854b5b4 100644 --- a/drivers/scsi/lpfc/lpfc_vport.h +++ b/drivers/scsi/lpfc/lpfc_vport.h @@ -1,7 +1,7 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * + * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term * * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2006 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * @@ -115,8 +115,4 @@ struct vport_cmd_tag { void lpfc_vport_set_state(struct lpfc_vport *vport, enum fc_vport_state new_state); -void lpfc_vport_reset_stat_data(struct lpfc_vport *); -void lpfc_alloc_bucket(struct lpfc_vport *); -void lpfc_free_bucket(struct lpfc_vport *); - #endif /* H_LPFC_VPORT */ diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c index 157c3bdb50be24b8bf75d4b2d45782249c33efac..132de68c14e9886077ef251a788f10a4f220e32e 100644 --- a/drivers/scsi/megaraid/megaraid_mbox.c +++ b/drivers/scsi/megaraid/megaraid_mbox.c @@ -3979,7 +3979,7 @@ megaraid_mbox_app_hndl_show(struct device *dev, struct device_attribute *attr, c app_hndl = mraid_mm_adapter_app_handle(adapter->unique_id); - return snprintf(buf, 8, "%u\n", app_hndl); + return sysfs_emit(buf, "%u\n", app_hndl); } @@ -4048,7 +4048,7 @@ megaraid_mbox_ld_show(struct device *dev, struct device_attribute *attr, char *b } } - return snprintf(buf, 36, "%d %d %d %d\n", scsi_id, logical_drv, + return sysfs_emit(buf, "%d %d %d %d\n", scsi_id, logical_drv, ldid_map, app_hndl); } diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index a3e117a4b8e746981daa7315b716e6e0dc2c8a4e..9be4ba61a076ccff196692b6b50350e6ee6b127b 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -3174,7 +3174,7 @@ megasas_bios_param(struct scsi_device *sdev, struct block_device *bdev, return 0; } -static int megasas_map_queues(struct Scsi_Host *shost) +static void megasas_map_queues(struct Scsi_Host *shost) { struct megasas_instance *instance; int qoff = 0, offset; @@ -3183,7 +3183,7 @@ static int megasas_map_queues(struct Scsi_Host *shost) instance = (struct megasas_instance *)shost->hostdata; if (shost->nr_hw_queues == 1) - return 0; + return; offset = instance->low_latency_index_start; @@ -3209,8 +3209,6 @@ static int megasas_map_queues(struct Scsi_Host *shost) map->queue_offset = qoff; blk_mq_map_queues(map); } - - return 0; } static void megasas_aen_polling(struct work_struct *work); @@ -4023,10 +4021,8 @@ megasas_deplete_reply_queue(struct megasas_instance *instance, u32 mfiStatus; u32 fw_state; - if ((mfiStatus = instance->instancet->check_reset(instance, - instance->reg_set)) == 1) { + if (instance->instancet->check_reset(instance, instance->reg_set) == 1) return IRQ_HANDLED; - } mfiStatus = instance->instancet->clear_intr(instance); if (mfiStatus == 0) { @@ -5157,9 +5153,9 @@ static void megasas_update_ext_vd_details(struct megasas_instance *instance) fusion->current_map_sz = ventura_map_sz; fusion->max_map_sz = ventura_map_sz; } else { - fusion->old_map_sz = sizeof(struct MR_FW_RAID_MAP) + - (sizeof(struct MR_LD_SPAN_MAP) * - (instance->fw_supported_vd_count - 1)); + fusion->old_map_sz = + struct_size((struct MR_FW_RAID_MAP *)0, ldSpanMap, + instance->fw_supported_vd_count); fusion->new_map_sz = sizeof(struct MR_FW_RAID_MAP_EXT); fusion->max_map_sz = @@ -5792,10 +5788,10 @@ megasas_setup_jbod_map(struct megasas_instance *instance) { int i; struct fusion_context *fusion = instance->ctrl_context; - u32 pd_seq_map_sz; + size_t pd_seq_map_sz; - pd_seq_map_sz = sizeof(struct MR_PD_CFG_SEQ_NUM_SYNC) + - (sizeof(struct MR_PD_CFG_SEQ) * (MAX_PHYSICAL_DEVICES - 1)); + pd_seq_map_sz = struct_size((struct MR_PD_CFG_SEQ_NUM_SYNC *)0, seq, + MAX_PHYSICAL_DEVICES); instance->use_seqnum_jbod_fp = instance->support_seqnum_jbod_fp; @@ -7153,22 +7149,18 @@ static int megasas_alloc_ctrl_mem(struct megasas_instance *instance) switch (instance->adapter_type) { case MFI_SERIES: if (megasas_alloc_mfi_ctrl_mem(instance)) - goto fail; + return -ENOMEM; break; case AERO_SERIES: case VENTURA_SERIES: case THUNDERBOLT_SERIES: case INVADER_SERIES: if (megasas_alloc_fusion_context(instance)) - goto fail; + return -ENOMEM; break; } return 0; - fail: - kfree(instance->reply_map); - instance->reply_map = NULL; - return -ENOMEM; } /* @@ -7974,7 +7966,7 @@ static void megasas_detach_one(struct pci_dev *pdev) struct Scsi_Host *host; struct megasas_instance *instance; struct fusion_context *fusion; - u32 pd_seq_map_sz; + size_t pd_seq_map_sz; instance = pci_get_drvdata(pdev); @@ -8046,9 +8038,9 @@ skip_firing_dcmds: if (instance->adapter_type != MFI_SERIES) { megasas_release_fusion(instance); - pd_seq_map_sz = sizeof(struct MR_PD_CFG_SEQ_NUM_SYNC) + - (sizeof(struct MR_PD_CFG_SEQ) * - (MAX_PHYSICAL_DEVICES - 1)); + pd_seq_map_sz = + struct_size((struct MR_PD_CFG_SEQ_NUM_SYNC *)0, + seq, MAX_PHYSICAL_DEVICES); for (i = 0; i < 2 ; i++) { if (fusion->ld_map[i]) dma_free_coherent(&instance->pdev->dev, diff --git a/drivers/scsi/megaraid/megaraid_sas_fp.c b/drivers/scsi/megaraid/megaraid_sas_fp.c index 83f69c33b01a97ff71cb9d005799a56739c7d3db..da1cad1ee12389053fec297e2f9da723d2145fb6 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fp.c +++ b/drivers/scsi/megaraid/megaraid_sas_fp.c @@ -326,9 +326,9 @@ u8 MR_ValidateMapInfo(struct megasas_instance *instance, u64 map_id) else if (instance->supportmax256vd) expected_size = sizeof(struct MR_FW_RAID_MAP_EXT); else - expected_size = - (sizeof(struct MR_FW_RAID_MAP) - sizeof(struct MR_LD_SPAN_MAP) + - (sizeof(struct MR_LD_SPAN_MAP) * le16_to_cpu(pDrvRaidMap->ldCount))); + expected_size = struct_size((struct MR_FW_RAID_MAP *)0, + ldSpanMap, + le16_to_cpu(pDrvRaidMap->ldCount)); if (le32_to_cpu(pDrvRaidMap->totalSize) != expected_size) { dev_dbg(&instance->pdev->dev, "megasas: map info structure size 0x%x", diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index e48d4261d0bcacf3ed63f46b4815d4b8e050c138..6650f8c8e9b046fc845735547fb5e10d88e62067 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -1310,7 +1310,7 @@ megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)]; pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)]; - pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES - 1); + pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES); cmd = megasas_get_cmd(instance); if (!cmd) { @@ -5310,7 +5310,6 @@ megasas_alloc_fusion_context(struct megasas_instance *instance) if (!fusion->log_to_span) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); - kfree(instance->ctrl_context); return -ENOMEM; } } diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.h b/drivers/scsi/megaraid/megaraid_sas_fusion.h index ce84f811e5e10b0bb312b8d94ad6008f9fc97a9f..49e9a9048ee7ec18152e1bf4da60c9f48a733815 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.h +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.h @@ -942,7 +942,7 @@ struct MR_FW_RAID_MAP { u8 reserved2[7]; struct MR_ARRAY_INFO arMapInfo[MAX_RAIDMAP_ARRAYS]; struct MR_DEV_HANDLE_INFO devHndlInfo[MAX_RAIDMAP_PHYSICAL_DEVICES]; - struct MR_LD_SPAN_MAP ldSpanMap[1]; + struct MR_LD_SPAN_MAP ldSpanMap[]; }; struct IO_REQUEST_INFO { @@ -1053,7 +1053,7 @@ struct MR_FW_RAID_MAP_DYNAMIC { struct MR_RAID_MAP_DESC_TABLE raid_map_desc_table[RAID_MAP_DESC_TYPE_COUNT]; /* Variable Size buffer containing all data */ - u32 raid_map_desc_data[1]; + u32 raid_map_desc_data[]; }; /* Dynamicaly sized RAID MAp structure */ #define IEEE_SGE_FLAGS_ADDR_MASK (0x03) @@ -1148,7 +1148,7 @@ typedef struct LOG_BLOCK_SPAN_INFO { struct MR_FW_RAID_MAP_ALL { struct MR_FW_RAID_MAP raidMap; - struct MR_LD_SPAN_MAP ldSpanMap[MAX_LOGICAL_DRIVES - 1]; + struct MR_LD_SPAN_MAP ldSpanMap[MAX_LOGICAL_DRIVES]; } __attribute__ ((packed)); struct MR_DRV_RAID_MAP { @@ -1182,7 +1182,7 @@ struct MR_DRV_RAID_MAP { devHndlInfo[MAX_RAIDMAP_PHYSICAL_DEVICES_DYN]; u16 ldTgtIdToLd[MAX_LOGICAL_DRIVES_DYN]; struct MR_ARRAY_INFO arMapInfo[MAX_API_ARRAYS_DYN]; - struct MR_LD_SPAN_MAP ldSpanMap[1]; + struct MR_LD_SPAN_MAP ldSpanMap[]; }; @@ -1193,7 +1193,7 @@ struct MR_DRV_RAID_MAP { struct MR_DRV_RAID_MAP_ALL { struct MR_DRV_RAID_MAP raidMap; - struct MR_LD_SPAN_MAP ldSpanMap[MAX_LOGICAL_DRIVES_DYN - 1]; + struct MR_LD_SPAN_MAP ldSpanMap[MAX_LOGICAL_DRIVES_DYN]; } __packed; @@ -1249,7 +1249,7 @@ struct MR_PD_CFG_SEQ { struct MR_PD_CFG_SEQ_NUM_SYNC { __le32 size; __le32 count; - struct MR_PD_CFG_SEQ seq[1]; + struct MR_PD_CFG_SEQ seq[]; } __packed; /* stream detection */ diff --git a/drivers/scsi/mpi3mr/Makefile b/drivers/scsi/mpi3mr/Makefile index f5cdbe48c150454dfd419cf5e154e28ed6f5b02b..ef86ca46646b8f93a72adee0aff2c5343d220c3b 100644 --- a/drivers/scsi/mpi3mr/Makefile +++ b/drivers/scsi/mpi3mr/Makefile @@ -3,3 +3,4 @@ obj-m += mpi3mr.o mpi3mr-y += mpi3mr_os.o \ mpi3mr_fw.o \ mpi3mr_app.o \ + mpi3mr_transport.o diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h index 4cd9f24e544cc43c372a84526e376da280616a9c..0a2af48915a5528f2c1a3cf44f516e21dfefe467 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_cnfg.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2017-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2017-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_CNFG_H #define MPI30_CNFG_H 1 @@ -100,6 +99,7 @@ struct mpi3_config_page_header { #define MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK (0xf0) #define MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT (4) #define MPI3_SAS_NEG_LINK_RATE_PHYSICAL_MASK (0x0f) +#define MPI3_SAS_NEG_LINK_RATE_PHYSICAL_SHIFT (0) #define MPI3_SAS_NEG_LINK_RATE_UNKNOWN_LINK_RATE (0x00) #define MPI3_SAS_NEG_LINK_RATE_PHY_DISABLED (0x01) #define MPI3_SAS_NEG_LINK_RATE_NEGOTIATION_FAILED (0x02) @@ -135,6 +135,16 @@ struct mpi3_config_page_header { #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_ACTIVE (0x00000000) #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_PARTIAL (0x08000000) #define MPI3_SAS_PHYINFO_PHY_POWER_CONDITION_SLUMBER (0x10000000) +#define MPI3_SAS_NEG_LINK_RATE_PHYSICAL_SHIFT (0) +#define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_CHANGED_MASK (0x04000000) +#define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_CHANGED_SHIFT (26) +#define MPI3_SAS_PHYINFO_INSIDE_ZPSDS_PERSISTENT_MASK (0x02000000) +#define MPI3_SAS_PHYINFO_INSIDE_ZPSDS_PERSISTENT_SHIFT (25) +#define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_MASK (0x01000000) +#define MPI3_SAS_PHYINFO_REQUESTED_INSIDE_ZPSDS_SHIFT (24) +#define MPI3_SAS_PHYINFO_ZONE_GROUP_PERSISTENT (0x00400000) +#define MPI3_SAS_PHYINFO_INSIDE_ZPSDS_WITHIN (0x00200000) +#define MPI3_SAS_PHYINFO_ZONING_ENABLED (0x00100000) #define MPI3_SAS_PHYINFO_REASON_MASK (0x000f0000) #define MPI3_SAS_PHYINFO_REASON_UNKNOWN (0x00000000) #define MPI3_SAS_PHYINFO_REASON_POWER_ON (0x00010000) @@ -210,7 +220,7 @@ struct mpi3_man_page0 { u8 board_rework_day; u8 board_rework_month; __le16 board_rework_year; - __le64 board_revision; + u8 board_revision[8]; u8 e_pack_fru[16]; u8 product_name[256]; }; @@ -226,6 +236,15 @@ struct mpi3_man_page1 { }; #define MPI3_MAN1_PAGEVERSION (0x00) +struct mpi3_man_page2 { + struct mpi3_config_page_header header; + u8 flags; + u8 reserved09[3]; + __le32 reserved0c[3]; + u8 oem_board_tracer_number[32]; +}; +#define MPI3_MAN2_PAGEVERSION (0x00) +#define MPI3_MAN2_FLAGS_TRACER_PRESENT (0x01) struct mpi3_man5_phy_entry { __le64 ioc_wwid; __le64 device_name; @@ -338,6 +357,8 @@ struct mpi3_man7_receptacle_info { #define MPI3_MAN7_LOCATION_INTERNAL (0x01) #define MPI3_MAN7_LOCATION_EXTERNAL (0x02) #define MPI3_MAN7_LOCATION_VIRTUAL (0x03) +#define MPI3_MAN7_LOCATION_HOST (0x04) +#define MPI3_MAN7_CONNECTOR_TYPE_NO_INFO (0x00) #define MPI3_MAN7_PEDCLK_ROUTING_MASK (0x10) #define MPI3_MAN7_PEDCLK_ROUTING_DIRECT (0x00) #define MPI3_MAN7_PEDCLK_ROUTING_CLOCK_BUFFER (0x10) @@ -369,7 +390,8 @@ struct mpi3_man8_phy_info { __le32 reserved0c; }; -#define MPI3_MAN8_PHY_INFO_RECEPTACLE_ID_HOST_PHY (0xff) +#define MPI3_MAN8_PHY_INFO_RECEPTACLE_ID_NOT_ASSOCIATED (0xff) +#define MPI3_MAN8_PHY_INFO_CONNECTOR_LANE_NOT_ASSOCIATED (0xff) #ifndef MPI3_MAN8_PHY_INFO_MAX #define MPI3_MAN8_PHY_INFO_MAX (1) #endif @@ -536,6 +558,10 @@ struct mpi3_man11_bkplane_spec_non_ubm_format { #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_GROUP_MASK (0xf000) #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_GROUP_SHIFT (12) #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_REFCLK_POLICY_ALWAYS_ENABLED (0x0200) +#define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_LINKWIDTH_MASK (0x00c0) +#define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_LINKWIDTH_4 (0x0000) +#define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_LINKWIDTH_2 (0x0040) +#define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_LINKWIDTH_1 (0x0080) #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_PRESENCE_DETECT_MASK (0x0030) #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_PRESENCE_DETECT_GPIO (0x0000) #define MPI3_MAN11_BKPLANE_NON_UBM_FLAGS_PRESENCE_DETECT_REG (0x0010) @@ -825,19 +851,16 @@ struct mpi3_man_page21 { }; #define MPI3_MAN21_PAGEVERSION (0x00) -#define MPI3_MAN21_FLAGS_HOST_METADATA_CAPABILITY_MASK (0x80) -#define MPI3_MAN21_FLAGS_HOST_METADATA_CAPABILITY_ENABLED (0x80) -#define MPI3_MAN21_FLAGS_HOST_METADATA_CAPABILITY_DISABLED (0x00) -#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_MASK (0x60) -#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_BLOCK (0x00) -#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_ALLOW (0x20) -#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_WARN (0x40) -#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_MASK (0x08) -#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_ALLOW (0x00) -#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_PREVENT (0x08) -#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_MASK (0x01) -#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_DEFAULT (0x00) -#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_OEM_SPECIFIC (0x01) +#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_MASK (0x00000060) +#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_BLOCK (0x00000000) +#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_ALLOW (0x00000020) +#define MPI3_MAN21_FLAGS_UNCERTIFIED_DRIVES_WARN (0x00000040) +#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_MASK (0x00000008) +#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_ALLOW (0x00000000) +#define MPI3_MAN21_FLAGS_BLOCK_SSD_WR_CACHE_CHANGE_PREVENT (0x00000008) +#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_MASK (0x00000001) +#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_DEFAULT (0x00000000) +#define MPI3_MAN21_FLAGS_SES_VPD_ASSOC_OEM_SPECIFIC (0x00000001) #ifndef MPI3_MAN_PROD_SPECIFIC_MAX #define MPI3_MAN_PROD_SPECIFIC_MAX (1) #endif @@ -995,7 +1018,12 @@ struct mpi3_io_unit_page5 { #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SATA_SSD_MASK (0x000c) #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SATA_SSD_SHIFT (2) #define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SAS_SSD_MASK (0x0003) -#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SAA_SSD_SHIFT (0) +#define MPI3_IOUNIT5_DEVICE_SHUTDOWN_SAS_SSD_SHIFT (0) +#define MPI3_IOUNIT5_FLAGS_SATAPUIS_MASK (0x0c) +#define MPI3_IOUNIT5_FLAGS_SATAPUIS_NOT_SUPPORTED (0x00) +#define MPI3_IOUNIT5_FLAGS_SATAPUIS_OS_CONTROLLED (0x04) +#define MPI3_IOUNIT5_FLAGS_SATAPUIS_APP_CONTROLLED (0x08) +#define MPI3_IOUNIT5_FLAGS_SATAPUIS_BLOCKED (0x0c) #define MPI3_IOUNIT5_FLAGS_POWER_CAPABLE_SPINUP (0x02) #define MPI3_IOUNIT5_FLAGS_AUTO_PORT_ENABLE (0x01) #define MPI3_IOUNIT5_PHY_SPINUP_GROUP_MASK (0x03) @@ -1027,7 +1055,8 @@ struct mpi3_io_unit_page8 { u8 slots_available; u8 current_key_encryption_algo; u8 key_digest_hash_algo; - __le32 reserved10[2]; + union mpi3_version_union current_svn; + __le32 reserved14; __le32 current_key[128]; union mpi3_iounit8_digest digest[MPI3_IOUNIT8_DIGEST_MAX]; }; @@ -1036,6 +1065,7 @@ struct mpi3_io_unit_page8 { #define MPI3_IOUNIT8_SBMODE_SECURE_DEBUG (0x04) #define MPI3_IOUNIT8_SBMODE_HARD_SECURE (0x02) #define MPI3_IOUNIT8_SBMODE_CONFIG_SECURE (0x01) +#define MPI3_IOUNIT8_SBSTATE_SVN_UPDATE_PENDING (0x04) #define MPI3_IOUNIT8_SBSTATE_KEY_UPDATE_PENDING (0x02) #define MPI3_IOUNIT8_SBSTATE_SECURE_BOOT_ENABLED (0x01) struct mpi3_io_unit_page9 { @@ -1045,9 +1075,14 @@ struct mpi3_io_unit_page9 { __le16 reserved0e; }; -#define MPI3_IOUNIT9_PAGEVERSION (0x00) -#define MPI3_IOUNIT9_FLAGS_VDFIRST_ENABLED (0x01) -#define MPI3_IOUNIT9_FIRSTDEVICE_UNKNOWN (0xffff) +#define MPI3_IOUNIT9_PAGEVERSION (0x00) +#define MPI3_IOUNIT9_FLAGS_UBM_ENCLOSURE_ORDER_MASK (0x00000006) +#define MPI3_IOUNIT9_FLAGS_UBM_ENCLOSURE_ORDER_SHIFT (1) +#define MPI3_IOUNIT9_FLAGS_UBM_ENCLOSURE_ORDER_NONE (0x00000000) +#define MPI3_IOUNIT9_FLAGS_UBM_ENCLOSURE_ORDER_RECEPTACLE (0x00000002) +#define MPI3_IOUNIT9_FLAGS_UBM_ENCLOSURE_ORDER_BACKPLANE_TYPE (0x00000004) +#define MPI3_IOUNIT9_FLAGS_VDFIRST_ENABLED (0x00000001) +#define MPI3_IOUNIT9_FIRSTDEVICE_UNKNOWN (0xffff) struct mpi3_io_unit_page10 { struct mpi3_config_page_header header; u8 flags; @@ -1090,6 +1125,57 @@ struct mpi3_io_unit_page11 { struct mpi3_iounit11_profile profile[MPI3_IOUNIT11_PROFILE_MAX]; }; #define MPI3_IOUNIT11_PAGEVERSION (0x00) +#ifndef MPI3_IOUNIT12_BUCKET_MAX +#define MPI3_IOUNIT12_BUCKET_MAX (1) +#endif +struct mpi3_iounit12_bucket { + u8 coalescing_depth; + u8 coalescing_timeout; + __le16 io_count_low_boundary; + __le32 reserved04; +}; +struct mpi3_io_unit_page12 { + struct mpi3_config_page_header header; + __le32 flags; + __le32 reserved0c[4]; + u8 num_buckets; + u8 reserved1d[3]; + struct mpi3_iounit12_bucket bucket[MPI3_IOUNIT12_BUCKET_MAX]; +}; +#define MPI3_IOUNIT12_PAGEVERSION (0x00) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_MASK (0x00000300) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_SHIFT (8) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_8 (0x00000000) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_16 (0x00000100) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_32 (0x00000200) +#define MPI3_IOUNIT12_FLAGS_NUMPASSES_64 (0x00000300) +#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_MASK (0x00000003) +#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_DISABLED (0x00000000) +#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_500US (0x00000001) +#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_1MS (0x00000002) +#define MPI3_IOUNIT12_FLAGS_PASSPERIOD_2MS (0x00000003) +#ifndef MPI3_IOUNIT13_FUNC_MAX +#define MPI3_IOUNIT13_FUNC_MAX (1) +#endif +struct mpi3_iounit13_allowed_function { + __le16 sub_function; + u8 function_code; + u8 fuction_flags; +}; +#define MPI3_IOUNIT13_FUNCTION_FLAGS_ADMIN_BLOCKED (0x04) +#define MPI3_IOUNIT13_FUNCTION_FLAGS_OOB_BLOCKED (0x02) +#define MPI3_IOUNIT13_FUNCTION_FLAGS_CHECK_SUBFUNCTION_ENABLED (0x01) +struct mpi3_io_unit_page13 { + struct mpi3_config_page_header header; + __le16 flags; + __le16 reserved0a; + u8 num_allowed_functions; + u8 reserved0d[3]; + struct mpi3_iounit13_allowed_function allowed_function[MPI3_IOUNIT13_FUNC_MAX]; +}; +#define MPI3_IOUNIT13_PAGEVERSION (0x00) +#define MPI3_IOUNIT13_FLAGS_ADMIN_BLOCKED (0x0002) +#define MPI3_IOUNIT13_FLAGS_OOB_BLOCKED (0x0001) struct mpi3_ioc_page0 { struct mpi3_config_page_header header; __le32 reserved08; @@ -1182,6 +1268,7 @@ struct mpi3_driver_page0 { __le32 reserved18; }; #define MPI3_DRIVER0_PAGEVERSION (0x00) +#define MPI3_DRIVER0_BSDOPTS_HEADLESS_MODE_ENABLE (0x00000008) #define MPI3_DRIVER0_BSDOPTS_DIS_HII_CONFIG_UTIL (0x00000004) #define MPI3_DRIVER0_BSDOPTS_REGISTRATION_MASK (0x00000003) #define MPI3_DRIVER0_BSDOPTS_REGISTRATION_IOC_AND_DEVS (0x00000000) @@ -1906,19 +1993,30 @@ struct mpi3_pcie_io_unit_page1 { }; #define MPI3_PCIEIOUNIT1_PAGEVERSION (0x00) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_OVERRIDE_DISABLE (0x80) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_DISABLE (0x40) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_MASK (0x30) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_PERST_OVERRIDE_MASK (0xe0000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_PERST_OVERRIDE_NONE (0x00000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_PERST_OVERRIDE_DEASSERT (0x20000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_PERST_OVERRIDE_ASSERT (0x40000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_PERST_OVERRIDE_BACKPLANE_ERROR (0x60000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_REFCLK_OVERRIDE_MASK (0x1c000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_REFCLK_OVERRIDE_NONE (0x00000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_REFCLK_OVERRIDE_DEASSERT (0x04000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_REFCLK_OVERRIDE_ASSERT (0x08000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_REFCLK_OVERRIDE_BACKPLANE_ERROR (0x0c000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_OVERRIDE_DISABLE (0x00000080) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_DISABLE (0x00000040) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_MASK (0x00000030) #define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SHIFT (4) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRIS_SRNS_DISABLED (0x00) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRIS_ENABLED (0x10) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRNS_ENABLED (0x20) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MASK (0x0f) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_2_5 (0x02) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_5_0 (0x03) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_8_0 (0x04) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_16_0 (0x05) -#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_32_0 (0x06) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRIS_SRNS_DISABLED (0x00000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRIS_ENABLED (0x00000010) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_CLOCK_OVERRIDE_MODE_SRNS_ENABLED (0x00000020) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MASK (0x0000000f) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_USE_BACKPLANE (0x00000000) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_2_5 (0x00000002) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_5_0 (0x00000003) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_8_0 (0x00000004) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_16_0 (0x00000005) +#define MPI3_PCIEIOUNIT1_CONTROL_FLAGS_LINK_RATE_OVERRIDE_MAX_32_0 (0x00000006) #define MPI3_PCIEIOUNIT1_ASPM_SWITCH_MASK (0x0c) #define MPI3_PCIEIOUNIT1_ASPM_SWITCH_SHIFT (2) #define MPI3_PCIEIOUNIT1_ASPM_DIRECT_MASK (0x03) @@ -2169,10 +2267,7 @@ struct mpi3_device0_vd_format { #define MPI3_DEVICE0_VD_DEVICE_INFO_SATA (0x0002) #define MPI3_DEVICE0_VD_DEVICE_INFO_SAS (0x0001) #define MPI3_DEVICE0_VD_FLAGS_IO_THROTTLE_GROUP_QD_MASK (0xf000) -#define MPI3_DEVICE0_VD_FLAGS_METADATA_MODE_MASK (0x0003) -#define MPI3_DEVICE0_VD_FLAGS_METADATA_MODE_NONE (0x0000) -#define MPI3_DEVICE0_VD_FLAGS_METADATA_MODE_HOST (0x0001) -#define MPI3_DEVICE0_VD_FLAGS_METADATA_MODE_IOC (0x0002) +#define MPI3_DEVICE0_VD_FLAGS_IO_THROTTLE_GROUP_QD_SHIFT (12) union mpi3_device0_dev_spec_format { struct mpi3_device0_sas_sata_format sas_sata_format; struct mpi3_device0_pcie_format pcie_format; diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_image.h b/drivers/scsi/mpi3mr/mpi/mpi30_image.h index c29b87de8e18e8a22a862b34948f922b0d73b882..64c58815988a378e04e2a92ad681b8905ec4e15f 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_image.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_image.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2018-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2018-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_IMAGE_H #define MPI30_IMAGE_H 1 @@ -63,6 +62,9 @@ struct mpi3_component_image_header { #define MPI3_IMAGE_HEADER_SIGNATURE1_PBLP (0x504c4250) #define MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST (0x464e414d) #define MPI3_IMAGE_HEADER_SIGNATURE1_OEM (0x204d454f) +#define MPI3_IMAGE_HEADER_SIGNATURE1_RMC (0x20434d52) +#define MPI3_IMAGE_HEADER_SIGNATURE1_SMM (0x204d4d53) +#define MPI3_IMAGE_HEADER_SIGNATURE1_PSW (0x20575350) #define MPI3_IMAGE_HEADER_SIGNATURE2_VALUE (0x50584546) #define MPI3_IMAGE_HEADER_FLAGS_DEVICE_KEY_BASIS_MASK (0x00000030) #define MPI3_IMAGE_HEADER_FLAGS_DEVICE_KEY_BASIS_CDI (0x00000000) diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_init.h b/drivers/scsi/mpi3mr/mpi/mpi30_init.h index aac11c58cca9547bbe3a285ad714ca00e3138359..3c03610ecfa63b40d0ac383f8c3d07e0fb02815b 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_init.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_init.h @@ -1,13 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2016-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2016-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_INIT_H #define MPI30_INIT_H 1 struct mpi3_scsi_io_cdb_eedp32 { u8 cdb[20]; - __be32 primary_reference_tag; + __be32 primary_reference_tag; __le16 primary_application_tag; __le16 primary_application_tag_mask; __le32 transfer_length; diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h index 214e4c65e5760fce5d5339c12307e2ecbd83add2..1c6c6730df5c7aa4c7ae5b156d8acdc93ca263a3 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_ioc.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2016-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2016-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_IOC_H #define MPI30_IOC_H 1 @@ -158,6 +157,7 @@ struct mpi3_ioc_facts_data { #define MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA (0x00000000) #define MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR (0x00000002) #define MPI3_IOCFACTS_IO_THROTTLE_DATA_LENGTH_NOT_REQUIRED (0x0000) +#define MPI3_IOCFACTS_MAX_IO_THROTTLE_GROUP_NOT_REQUIRED (0x0000) struct mpi3_mgmt_passthrough_request { __le16 host_tag; u8 ioc_use_only02; @@ -637,6 +637,23 @@ struct mpi3_event_data_diag_buffer_status_change { #define MPI3_EVENT_DIAG_BUFFER_STATUS_CHANGE_RC_RELEASED (0x01) #define MPI3_EVENT_DIAG_BUFFER_STATUS_CHANGE_RC_PAUSED (0x02) #define MPI3_EVENT_DIAG_BUFFER_STATUS_CHANGE_RC_RESUMED (0x03) +#define MPI3_PEL_LOCALE_FLAGS_NON_BLOCKING_BOOT_EVENT (0x0200) +#define MPI3_PEL_LOCALE_FLAGS_BLOCKING_BOOT_EVENT (0x0100) +#define MPI3_PEL_LOCALE_FLAGS_PCIE (0x0080) +#define MPI3_PEL_LOCALE_FLAGS_CONFIGURATION (0x0040) +#define MPI3_PEL_LOCALE_FLAGS_CONTROLER (0x0020) +#define MPI3_PEL_LOCALE_FLAGS_SAS (0x0010) +#define MPI3_PEL_LOCALE_FLAGS_EPACK (0x0008) +#define MPI3_PEL_LOCALE_FLAGS_ENCLOSURE (0x0004) +#define MPI3_PEL_LOCALE_FLAGS_PD (0x0002) +#define MPI3_PEL_LOCALE_FLAGS_VD (0x0001) +#define MPI3_PEL_CLASS_DEBUG (0x00) +#define MPI3_PEL_CLASS_PROGRESS (0x01) +#define MPI3_PEL_CLASS_INFORMATIONAL (0x02) +#define MPI3_PEL_CLASS_WARNING (0x03) +#define MPI3_PEL_CLASS_CRITICAL (0x04) +#define MPI3_PEL_CLASS_FATAL (0x05) +#define MPI3_PEL_CLASS_FAULT (0x06) #define MPI3_PEL_CLEARTYPE_CLEAR (0x00) #define MPI3_PEL_WAITTIME_INFINITE_WAIT (0x00) #define MPI3_PEL_ACTION_GET_SEQNUM (0x01) @@ -924,6 +941,7 @@ struct mpi3_ci_download_reply { }; #define MPI3_CI_DOWNLOAD_FLAGS_DOWNLOAD_IN_PROGRESS (0x80) +#define MPI3_CI_DOWNLOAD_FLAGS_ACTIVATION_FAILURE (0x40) #define MPI3_CI_DOWNLOAD_FLAGS_OFFLINE_ACTIVATION_REQUIRED (0x20) #define MPI3_CI_DOWNLOAD_FLAGS_KEY_UPDATE_PENDING (0x10) #define MPI3_CI_DOWNLOAD_FLAGS_ACTIVATION_STATUS_MASK (0x0e) diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_pci.h b/drivers/scsi/mpi3mr/mpi/mpi30_pci.h index 901dbd788940097402190df677d2a73b5a6e301e..b7a5df01120da5ae0b201eb3d9c5b6718cf48b2b 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_pci.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_pci.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2016-2021 Broadcom Inc. All rights reserved. + * Copyright 2016-2022 Broadcom Inc. All rights reserved. * */ #ifndef MPI30_PCI_H diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_sas.h b/drivers/scsi/mpi3mr/mpi/mpi30_sas.h index 298d895e374b2ce0dc544a1cea73af4af8321de0..e587f77ccd68d9957d65929867173e05ea64890a 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_sas.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_sas.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2016-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2016-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_SAS_H #define MPI30_SAS_H 1 diff --git a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h index ba05ea57af25bcea699a794e642098b30f24502b..9b76b963275107c665d1e40ef95b512dc2f47304 100644 --- a/drivers/scsi/mpi3mr/mpi/mpi30_transport.h +++ b/drivers/scsi/mpi3mr/mpi/mpi30_transport.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Copyright 2016-2021 Broadcom Inc. All rights reserved. - * + * Copyright 2016-2022 Broadcom Inc. All rights reserved. */ #ifndef MPI30_TRANSPORT_H #define MPI30_TRANSPORT_H 1 @@ -19,8 +18,8 @@ union mpi3_version_union { #define MPI3_VERSION_MAJOR (3) #define MPI3_VERSION_MINOR (0) -#define MPI3_VERSION_UNIT (23) -#define MPI3_VERSION_DEV (1) +#define MPI3_VERSION_UNIT (26) +#define MPI3_VERSION_DEV (0) #define MPI3_DEVHANDLE_INVALID (0xffff) struct mpi3_sysif_oper_queue_indexes { __le16 producer_index; @@ -212,6 +211,7 @@ struct mpi3_default_reply_descriptor { #define MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS (0x1000) #define MPI3_REPLY_DESCRIPT_FLAGS_TYPE_TARGET_COMMAND_BUFFER (0x2000) #define MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS (0x3000) +#define MPI3_REPLY_DESCRIPT_REQUEST_QUEUE_ID_INVALID (0xffff) struct mpi3_address_reply_descriptor { __le64 reply_frame_address; __le16 request_queue_ci; diff --git a/drivers/scsi/mpi3mr/mpi3mr.h b/drivers/scsi/mpi3mr/mpi3mr.h index 0935b2e806623f4b653e266df05f1521922264ad..def4c5e15cd89b65fb0ec59479f00f962c7ce24d 100644 --- a/drivers/scsi/mpi3mr/mpi3mr.h +++ b/drivers/scsi/mpi3mr/mpi3mr.h @@ -39,6 +39,7 @@ #include #include #include +#include #include "mpi/mpi30_transport.h" #include "mpi/mpi30_cnfg.h" @@ -55,8 +56,8 @@ extern struct list_head mrioc_list; extern int prot_mask; extern atomic64_t event_counter; -#define MPI3MR_DRIVER_VERSION "8.0.0.69.0" -#define MPI3MR_DRIVER_RELDATE "16-March-2022" +#define MPI3MR_DRIVER_VERSION "8.2.0.3.0" +#define MPI3MR_DRIVER_RELDATE "08-September-2022" #define MPI3MR_DRIVER_NAME "mpi3mr" #define MPI3MR_DRIVER_LICENSE "GPL" @@ -97,9 +98,11 @@ extern atomic64_t event_counter; #define MPI3MR_HOSTTAG_PEL_ABORT 3 #define MPI3MR_HOSTTAG_PEL_WAIT 4 #define MPI3MR_HOSTTAG_BLK_TMS 5 +#define MPI3MR_HOSTTAG_CFG_CMDS 6 +#define MPI3MR_HOSTTAG_TRANSPORT_CMDS 7 #define MPI3MR_NUM_DEVRMCMD 16 -#define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_BLK_TMS + 1) +#define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_TRANSPORT_CMDS + 1) #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \ MPI3MR_NUM_DEVRMCMD - 1) @@ -115,6 +118,7 @@ extern atomic64_t event_counter; /* command/controller interaction timeout definitions in seconds */ #define MPI3MR_INTADMCMD_TIMEOUT 60 #define MPI3MR_PORTENABLE_TIMEOUT 300 +#define MPI3MR_PORTENABLE_POLL_INTERVAL 5 #define MPI3MR_ABORTTM_TIMEOUT 60 #define MPI3MR_RESETTM_TIMEOUT 60 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5 @@ -126,6 +130,10 @@ extern atomic64_t event_counter; #define MPI3MR_WATCHDOG_INTERVAL 1000 /* in milli seconds */ +#define MPI3MR_DEFAULT_CFG_PAGE_SZ 1024 /* in bytes */ + +#define MPI3MR_RESET_TOPOLOGY_SETTLE_TIME 10 + #define MPI3MR_SCMD_TIMEOUT (60 * HZ) #define MPI3MR_EH_SCMD_TIMEOUT (60 * HZ) @@ -274,6 +282,8 @@ enum mpi3mr_reset_reason { MPI3MR_RESET_FROM_SYSFS = 23, MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24, MPI3MR_RESET_FROM_FIRMWARE = 27, + MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29, + MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT = 30, }; /* Queue type definitions */ @@ -421,12 +431,14 @@ struct op_reply_qinfo { * struct mpi3mr_intr_info - Interrupt cookie information * * @mrioc: Adapter instance reference + * @os_irq: irq number * @msix_index: MSIx index * @op_reply_q: Associated operational reply queue * @name: Dev name for the irq claiming device */ struct mpi3mr_intr_info { struct mpi3mr_ioc *mrioc; + int os_irq; u16 msix_index; struct op_reply_qinfo *op_reply_q; char name[MPI3MR_NAME_LENGTH]; @@ -457,16 +469,138 @@ struct mpi3mr_throttle_group_info { atomic_t pend_large_data_sz; }; +/* HBA port flags */ +#define MPI3MR_HBA_PORT_FLAG_DIRTY 0x01 + +/** + * struct mpi3mr_hba_port - HBA's port information + * @port_id: Port number + * @flags: HBA port flags + */ +struct mpi3mr_hba_port { + struct list_head list; + u8 port_id; + u8 flags; +}; + +/** + * struct mpi3mr_sas_port - Internal SAS port information + * @port_list: List of ports belonging to a SAS node + * @num_phys: Number of phys associated with port + * @marked_responding: used while refresing the sas ports + * @lowest_phy: lowest phy ID of current sas port + * @phy_mask: phy_mask of current sas port + * @hba_port: HBA port entry + * @remote_identify: Attached device identification + * @rphy: SAS transport layer rphy object + * @port: SAS transport layer port object + * @phy_list: mpi3mr_sas_phy objects belonging to this port + */ +struct mpi3mr_sas_port { + struct list_head port_list; + u8 num_phys; + u8 marked_responding; + int lowest_phy; + u32 phy_mask; + struct mpi3mr_hba_port *hba_port; + struct sas_identify remote_identify; + struct sas_rphy *rphy; + struct sas_port *port; + struct list_head phy_list; +}; + +/** + * struct mpi3mr_sas_phy - Internal SAS Phy information + * @port_siblings: List of phys belonging to a port + * @identify: Phy identification + * @remote_identify: Attached device identification + * @phy: SAS transport layer Phy object + * @phy_id: Unique phy id within a port + * @handle: Firmware device handle for this phy + * @attached_handle: Firmware device handle for attached device + * @phy_belongs_to_port: Flag to indicate phy belongs to port + @hba_port: HBA port entry + */ +struct mpi3mr_sas_phy { + struct list_head port_siblings; + struct sas_identify identify; + struct sas_identify remote_identify; + struct sas_phy *phy; + u8 phy_id; + u16 handle; + u16 attached_handle; + u8 phy_belongs_to_port; + struct mpi3mr_hba_port *hba_port; +}; + +/** + * struct mpi3mr_sas_node - SAS host/expander information + * @list: List of sas nodes in a controller + * @parent_dev: Parent device class + * @num_phys: Number phys belonging to sas_node + * @sas_address: SAS address of sas_node + * @handle: Firmware device handle for this sas_host/expander + * @sas_address_parent: SAS address of parent expander or host + * @enclosure_handle: Firmware handle of enclosure of this node + * @device_info: Capabilities of this sas_host/expander + * @non_responding: used to refresh the expander devices during reset + * @host_node: Flag to indicate this is a host_node + * @hba_port: HBA port entry + * @phy: A list of phys that make up this sas_host/expander + * @sas_port_list: List of internal ports of this node + * @rphy: sas_rphy object of this expander node + */ +struct mpi3mr_sas_node { + struct list_head list; + struct device *parent_dev; + u8 num_phys; + u64 sas_address; + u16 handle; + u64 sas_address_parent; + u16 enclosure_handle; + u64 enclosure_logical_id; + u8 non_responding; + u8 host_node; + struct mpi3mr_hba_port *hba_port; + struct mpi3mr_sas_phy *phy; + struct list_head sas_port_list; + struct sas_rphy *rphy; +}; + +/** + * struct mpi3mr_enclosure_node - enclosure information + * @list: List of enclosures + * @pg0: Enclosure page 0; + */ +struct mpi3mr_enclosure_node { + struct list_head list; + struct mpi3_enclosure_page0 pg0; +}; + /** * struct tgt_dev_sas_sata - SAS/SATA device specific * information cached from firmware given data * * @sas_address: World wide unique SAS address + * @sas_address_parent: Sas address of parent expander or host * @dev_info: Device information bits + * @phy_id: Phy identifier provided in device page 0 + * @attached_phy_id: Attached phy identifier provided in device page 0 + * @sas_transport_attached: Is this device exposed to transport + * @pend_sas_rphy_add: Flag to check device is in process of add + * @hba_port: HBA port entry + * @rphy: SAS transport layer rphy object */ struct tgt_dev_sas_sata { u64 sas_address; + u64 sas_address_parent; u16 dev_info; + u8 phy_id; + u8 attached_phy_id; + u8 sas_transport_attached; + u8 pend_sas_rphy_add; + struct mpi3mr_hba_port *hba_port; + struct sas_rphy *rphy; }; /** @@ -531,12 +665,16 @@ union _form_spec_inf { * @slot: Slot number * @encl_handle: FW enclosure handle * @perst_id: FW assigned Persistent ID + * @devpg0_flag: Device Page0 flag * @dev_type: SAS/SATA/PCIE device type * @is_hidden: Should be exposed to upper layers or not * @host_exposed: Already exposed to host or not + * @io_unit_port: IO Unit port ID + * @non_stl: Is this device not to be attached with SAS TL * @io_throttle_enabled: I/O throttling needed or not * @q_depth: Device specific Queue Depth * @wwid: World wide ID + * @enclosure_logical_id: Enclosure logical identifier * @dev_spec: Device type specific information * @ref_count: Reference count */ @@ -548,12 +686,16 @@ struct mpi3mr_tgt_dev { u16 slot; u16 encl_handle; u16 perst_id; + u16 devpg0_flag; u8 dev_type; u8 is_hidden; u8 host_exposed; + u8 io_unit_port; + u8 non_stl; u8 io_throttle_enabled; u16 q_depth; u64 wwid; + u64 enclosure_logical_id; union _form_spec_inf dev_spec; struct kref ref_count; }; @@ -679,6 +821,21 @@ struct mpi3mr_drv_cmd { struct mpi3mr_drv_cmd *drv_cmd); }; +/** + * struct dma_memory_desc - memory descriptor structure to store + * virtual address, dma address and size for any generic dma + * memory allocations in the driver. + * + * @size: buffer size + * @addr: virtual address + * @dma_addr: dma address + */ +struct dma_memory_desc { + u32 size; + void *addr; + dma_addr_t dma_addr; +}; + /** * struct chain_element - memory descriptor structure to store @@ -756,6 +913,7 @@ struct scmd_priv { * @num_op_reply_q: Number of operational reply queues * @op_reply_qinfo: Operational reply queue info pointer * @init_cmds: Command tracker for initialization commands + * @cfg_cmds: Command tracker for configuration requests * @facts: Cached IOC facts data * @op_reply_desc_sz: Operational reply descriptor size * @num_reply_bufs: Number of reply buffers allocated @@ -792,6 +950,7 @@ struct scmd_priv { * @scan_started: Async scan started * @scan_failed: Asycn scan failed * @stop_drv_processing: Stop all command processing + * @device_refresh_on: Don't process the events until devices are refreshed * @max_host_ios: Maximum host I/O count * @chain_buf_count: Chain buffer count * @chain_buf_pool: Chain buffer pool @@ -854,6 +1013,17 @@ struct scmd_priv { * @io_throttle_low: I/O size to stop throttle in 512b blocks * @num_io_throttle_group: Maximum number of throttle groups * @throttle_groups: Pointer to throttle group info structures + * @cfg_page: Default memory for configuration pages + * @cfg_page_dma: Configuration page DMA address + * @cfg_page_sz: Default configuration page memory size + * @sas_transport_enabled: SAS transport enabled or not + * @scsi_device_channel: Channel ID for SCSI devices + * @transport_cmds: Command tracker for SAS transport commands + * @sas_hba: SAS node for the controller + * @sas_expander_list: SAS node list of expanders + * @sas_node_lock: Lock to protect SAS node list + * @hba_port_table_list: List of HBA Ports + * @enclosure_list: List of Enclosure objects */ struct mpi3mr_ioc { struct list_head list; @@ -904,6 +1074,7 @@ struct mpi3mr_ioc { struct op_reply_qinfo *op_reply_qinfo; struct mpi3mr_drv_cmd init_cmds; + struct mpi3mr_drv_cmd cfg_cmds; struct mpi3mr_ioc_facts facts; u16 op_reply_desc_sz; @@ -948,6 +1119,7 @@ struct mpi3mr_ioc { u8 scan_started; u16 scan_failed; u8 stop_drv_processing; + u8 device_refresh_on; u16 max_host_ios; spinlock_t tgtdev_lock; @@ -1025,6 +1197,19 @@ struct mpi3mr_ioc { u32 io_throttle_low; u16 num_io_throttle_group; struct mpi3mr_throttle_group_info *throttle_groups; + + void *cfg_page; + dma_addr_t cfg_page_dma; + u16 cfg_page_sz; + + u8 sas_transport_enabled; + u8 scsi_device_channel; + struct mpi3mr_drv_cmd transport_cmds; + struct mpi3mr_sas_node sas_hba; + struct list_head sas_expander_list; + spinlock_t sas_node_lock; + struct list_head hba_port_table_list; + struct list_head enclosure_list; }; /** @@ -1149,6 +1334,67 @@ int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc, struct mpi3mr_drv_cmd *drv_cmd); void mpi3mr_app_save_logdata(struct mpi3mr_ioc *mrioc, char *event_data, u16 event_data_size); +struct mpi3mr_enclosure_node *mpi3mr_enclosure_find_by_handle( + struct mpi3mr_ioc *mrioc, u16 handle); extern const struct attribute_group *mpi3mr_host_groups[]; extern const struct attribute_group *mpi3mr_dev_groups[]; + +extern struct sas_function_template mpi3mr_transport_functions; +extern struct scsi_transport_template *mpi3mr_transport_template; + +int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec); +int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form, + u32 form_spec); +int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form, + u32 form_spec); +int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form, + u32 form_spec); +int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form, + u32 form_spec); +int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form, + u32 form_spec); +int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz); +int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz); +int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz); +int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_driver_page1 *driver_pg1, u16 pg_sz); + +u8 mpi3mr_is_expander_device(u16 device_info); +int mpi3mr_expander_add(struct mpi3mr_ioc *mrioc, u16 handle); +void mpi3mr_expander_remove(struct mpi3mr_ioc *mrioc, u64 sas_address, + struct mpi3mr_hba_port *hba_port); +struct mpi3mr_sas_node *__mpi3mr_expander_find_by_handle(struct mpi3mr_ioc + *mrioc, u16 handle); +struct mpi3mr_hba_port *mpi3mr_get_hba_port_by_id(struct mpi3mr_ioc *mrioc, + u8 port_id); +void mpi3mr_sas_host_refresh(struct mpi3mr_ioc *mrioc); +void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc); +void mpi3mr_update_links(struct mpi3mr_ioc *mrioc, + u64 sas_address_parent, u16 handle, u8 phy_number, u8 link_rate, + struct mpi3mr_hba_port *hba_port); +void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev); +int mpi3mr_report_tgtdev_to_sas_transport(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev); +void mpi3mr_remove_tgtdev_from_sas_transport(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev); +struct mpi3mr_tgt_dev *__mpi3mr_get_tgtdev_by_addr_and_rphy( + struct mpi3mr_ioc *mrioc, u64 sas_address, struct sas_rphy *rphy); +void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc, + bool device_add); +void mpi3mr_refresh_sas_ports(struct mpi3mr_ioc *mrioc); +void mpi3mr_refresh_expanders(struct mpi3mr_ioc *mrioc); +void mpi3mr_add_event_wait_for_device_refresh(struct mpi3mr_ioc *mrioc); +void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc); +void mpi3mr_flush_cmds_for_unrecovered_controller(struct mpi3mr_ioc *mrioc); +void mpi3mr_free_enclosure_list(struct mpi3mr_ioc *mrioc); #endif /*MPI3MR_H_INCLUDED*/ diff --git a/drivers/scsi/mpi3mr/mpi3mr_debug.h b/drivers/scsi/mpi3mr/mpi3mr_debug.h index 2464c400a5a434232f995337e9566dd65511d8f3..ee6edd8322e60f93ec2222c016396dac61669162 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_debug.h +++ b/drivers/scsi/mpi3mr/mpi3mr_debug.h @@ -23,9 +23,13 @@ #define MPI3_DEBUG_RESET 0x00000020 #define MPI3_DEBUG_SCSI_ERROR 0x00000040 #define MPI3_DEBUG_REPLY 0x00000080 +#define MPI3_DEBUG_CFG_ERROR 0x00000100 +#define MPI3_DEBUG_TRANSPORT_ERROR 0x00000200 #define MPI3_DEBUG_BSG_ERROR 0x00008000 #define MPI3_DEBUG_BSG_INFO 0x00010000 #define MPI3_DEBUG_SCSI_INFO 0x00020000 +#define MPI3_DEBUG_CFG_INFO 0x00040000 +#define MPI3_DEBUG_TRANSPORT_INFO 0x00080000 #define MPI3_DEBUG 0x01000000 #define MPI3_DEBUG_SG 0x02000000 @@ -122,6 +126,29 @@ pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__); \ } while (0) +#define dprint_cfg_info(ioc, fmt, ...) \ + do { \ + if (ioc->logging_level & MPI3_DEBUG_CFG_INFO) \ + pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__); \ + } while (0) + +#define dprint_cfg_err(ioc, fmt, ...) \ + do { \ + if (ioc->logging_level & MPI3_DEBUG_CFG_ERROR) \ + pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__); \ + } while (0) +#define dprint_transport_info(ioc, fmt, ...) \ + do { \ + if (ioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO) \ + pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__); \ + } while (0) + +#define dprint_transport_err(ioc, fmt, ...) \ + do { \ + if (ioc->logging_level & MPI3_DEBUG_TRANSPORT_ERROR) \ + pr_info("%s: " fmt, (ioc)->name, ##__VA_ARGS__); \ + } while (0) + #endif /* MPT3SAS_DEBUG_H_INCLUDED */ /** diff --git a/drivers/scsi/mpi3mr/mpi3mr_fw.c b/drivers/scsi/mpi3mr/mpi3mr_fw.c index 0866dfd433185fe35452ea0d078abd7b7ff82d71..0c4aabaefdcc4c34eba84223093668444c4f6cf0 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_fw.c +++ b/drivers/scsi/mpi3mr/mpi3mr_fw.c @@ -244,6 +244,9 @@ static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc, case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE: desc = "Enclosure Device Status Change"; break; + case MPI3_EVENT_ENCL_DEVICE_ADDED: + desc = "Enclosure Added"; + break; case MPI3_EVENT_HARD_RESET_RECEIVED: desc = "Hard Reset Received"; break; @@ -299,6 +302,8 @@ mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag, switch (host_tag) { case MPI3MR_HOSTTAG_INITCMDS: return &mrioc->init_cmds; + case MPI3MR_HOSTTAG_CFG_CMDS: + return &mrioc->cfg_cmds; case MPI3MR_HOSTTAG_BSG_CMDS: return &mrioc->bsg_cmds; case MPI3MR_HOSTTAG_BLK_TMS: @@ -307,6 +312,8 @@ mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag, return &mrioc->pel_abort_cmd; case MPI3MR_HOSTTAG_PEL_WAIT: return &mrioc->pel_cmds; + case MPI3MR_HOSTTAG_TRANSPORT_CMDS: + return &mrioc->transport_cmds; case MPI3MR_HOSTTAG_INVALID: if (def_reply && def_reply->function == MPI3_FUNCTION_EVENT_NOTIFICATION) @@ -424,6 +431,9 @@ static int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc) return 0; do { + if (mrioc->unrecoverable) + break; + mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci); mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma); if (reply_dma) @@ -509,6 +519,9 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc, } do { + if (mrioc->unrecoverable) + break; + req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1; op_req_q = &mrioc->req_qinfo[req_q_idx]; @@ -530,6 +543,7 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc, if ((le16_to_cpu(reply_desc->reply_flags) & MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) break; +#ifndef CONFIG_PREEMPT_RT /* * Exit completion loop to avoid CPU lockup * Ensure remaining completion happens from threaded ISR. @@ -538,7 +552,7 @@ int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc, op_reply_q->enable_irq_poll = true; break; } - +#endif } while (1); writel(reply_ci, @@ -569,7 +583,8 @@ int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) mrioc = (struct mpi3mr_ioc *)shost->hostdata; - if ((mrioc->reset_in_progress || mrioc->prepare_for_reset)) + if ((mrioc->reset_in_progress || mrioc->prepare_for_reset || + mrioc->unrecoverable)) return 0; num_entries = mpi3mr_process_op_reply_q(mrioc, @@ -607,18 +622,16 @@ static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata) return IRQ_NONE; } +#ifndef CONFIG_PREEMPT_RT + static irqreturn_t mpi3mr_isr(int irq, void *privdata) { struct mpi3mr_intr_info *intr_info = privdata; - struct mpi3mr_ioc *mrioc; - u16 midx; int ret; if (!intr_info) return IRQ_NONE; - mrioc = intr_info->mrioc; - midx = intr_info->msix_index; /* Call primary ISR routine */ ret = mpi3mr_isr_primary(irq, privdata); @@ -633,7 +646,7 @@ static irqreturn_t mpi3mr_isr(int irq, void *privdata) !atomic_read(&intr_info->op_reply_q->pend_ios)) return ret; - disable_irq_nosync(pci_irq_vector(mrioc->pdev, midx)); + disable_irq_nosync(intr_info->os_irq); return IRQ_WAKE_THREAD; } @@ -663,7 +676,7 @@ static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata) /* Poll for pending IOs completions */ do { - if (!mrioc->intr_enabled) + if (!mrioc->intr_enabled || mrioc->unrecoverable) break; if (!midx) @@ -679,11 +692,13 @@ static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata) (num_op_reply < mrioc->max_host_ios)); intr_info->op_reply_q->enable_irq_poll = false; - enable_irq(pci_irq_vector(mrioc->pdev, midx)); + enable_irq(intr_info->os_irq); return IRQ_HANDLED; } +#endif + /** * mpi3mr_request_irq - Request IRQ and register ISR * @mrioc: Adapter instance reference @@ -706,14 +721,20 @@ static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index) snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d", mrioc->driver_name, mrioc->id, index); +#ifndef CONFIG_PREEMPT_RT retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr, mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info); +#else + retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr_primary, + NULL, IRQF_SHARED, intr_info->name, intr_info); +#endif if (retval) { ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n", intr_info->name, pci_irq_vector(pdev, index)); return retval; } + intr_info->os_irq = pci_irq_vector(pdev, index); return retval; } @@ -907,6 +928,8 @@ static const struct { { MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" }, { MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" }, { MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" }, + { MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"}, + { MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" }, }; /** @@ -1130,6 +1153,13 @@ mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc) return -EPERM; } + if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities & + MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) + ioc_err(mrioc, + "critical error: multipath capability is enabled at the\n" + "\tcontroller while sas transport support is enabled at the\n" + "\tdriver, please reboot the system or reload the driver\n"); + dev_handle_bitmap_sz = mrioc->facts.max_devhandle / 8; if (mrioc->facts.max_devhandle % 8) dev_handle_bitmap_sz++; @@ -1194,6 +1224,14 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) msleep(100); } while (--timeout); + if (!pci_device_is_present(mrioc->pdev)) { + mrioc->unrecoverable = 1; + ioc_err(mrioc, + "controller is not present while waiting to reset\n"); + retval = -1; + goto out_device_not_present; + } + ioc_state = mpi3mr_get_iocstate(mrioc); ioc_info(mrioc, "controller is in %s state after waiting to reset\n", @@ -1251,6 +1289,13 @@ static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc) mpi3mr_iocstate_name(ioc_state)); return 0; } + if (!pci_device_is_present(mrioc->pdev)) { + mrioc->unrecoverable = 1; + ioc_err(mrioc, + "controller is not present at the bringup\n"); + retval = -1; + goto out_device_not_present; + } msleep(100); } while (--timeout); @@ -1259,6 +1304,7 @@ out_failed: ioc_err(mrioc, "failed to bring to ready state, current state: %s\n", mpi3mr_iocstate_name(ioc_state)); +out_device_not_present: return retval; } @@ -2163,9 +2209,13 @@ int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc, pi = 0; op_req_q->pi = pi; +#ifndef CONFIG_PREEMPT_RT if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios) > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT) mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true; +#else + atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios); +#endif writel(op_req_q->pi, &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index); @@ -2193,6 +2243,17 @@ void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code) { u32 ioc_status, host_diagnostic, timeout; + if (mrioc->unrecoverable) { + ioc_err(mrioc, "controller is unrecoverable\n"); + return; + } + + if (!pci_device_is_present(mrioc->pdev)) { + mrioc->unrecoverable = 1; + ioc_err(mrioc, "controller is not present\n"); + return; + } + ioc_status = readl(&mrioc->sysif_regs->ioc_status); if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) || (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) { @@ -2384,9 +2445,21 @@ static void mpi3mr_watchdog_work(struct work_struct *work) u32 fault, host_diagnostic, ioc_status; u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH; - if (mrioc->reset_in_progress || mrioc->unrecoverable) + if (mrioc->reset_in_progress) return; + if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) { + ioc_err(mrioc, "watchdog could not detect the controller\n"); + mrioc->unrecoverable = 1; + } + + if (mrioc->unrecoverable) { + ioc_err(mrioc, + "flush pending commands for unrecoverable controller\n"); + mpi3mr_flush_cmds_for_unrecovered_controller(mrioc); + return; + } + if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) { mrioc->ts_update_counter = 0; mpi3mr_sync_timestamp(mrioc); @@ -2426,11 +2499,12 @@ static void mpi3mr_watchdog_work(struct work_struct *work) mrioc->diagsave_timeout = 0; switch (fault) { + case MPI3_SYSIF_FAULT_CODE_COMPLETE_RESET_NEEDED: case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED: - ioc_info(mrioc, + ioc_warn(mrioc, "controller requires system power cycle, marking controller as unrecoverable\n"); mrioc->unrecoverable = 1; - return; + goto schedule_work; case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS: return; case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET: @@ -2853,6 +2927,10 @@ static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc) if (!mrioc->bsg_cmds.reply) goto out_failed; + mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL); + if (!mrioc->transport_cmds.reply) + goto out_failed; + for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) { mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz, GFP_KERNEL); @@ -3362,10 +3440,13 @@ out_failed: static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc, struct mpi3mr_drv_cmd *drv_cmd) { - drv_cmd->state = MPI3MR_CMD_NOTUSED; drv_cmd->callback = NULL; - mrioc->scan_failed = drv_cmd->ioc_status; mrioc->scan_started = 0; + if (drv_cmd->state & MPI3MR_CMD_RESET) + mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR; + else + mrioc->scan_failed = drv_cmd->ioc_status; + drv_cmd->state = MPI3MR_CMD_NOTUSED; } /** @@ -3447,6 +3528,7 @@ static const struct { char *name; } mpi3mr_capabilities[] = { { MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" }, + { MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED, "MultiPath" }, }; /** @@ -3657,6 +3739,7 @@ static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc) mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED); mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE); mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE); + mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED); mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST); mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY); mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR); @@ -3727,6 +3810,14 @@ retry_init: mrioc->max_host_ios = min_t(int, mrioc->max_host_ios, MPI3MR_HOST_IOS_KDUMP); + if (!(mrioc->facts.ioc_capabilities & + MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) { + mrioc->sas_transport_enabled = 1; + mrioc->scsi_device_channel = 1; + mrioc->shost->max_channel = 1; + mrioc->shost->transportt = mpi3mr_transport_template; + } + mrioc->reply_sz = mrioc->facts.reply_sz; retval = mpi3mr_check_reset_dma_mask(mrioc); @@ -3738,6 +3829,14 @@ retry_init: mpi3mr_print_ioc_info(mrioc); + dprint_init(mrioc, "allocating config page buffers\n"); + mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev, + MPI3MR_DEFAULT_CFG_PAGE_SZ, &mrioc->cfg_page_dma, GFP_KERNEL); + if (!mrioc->cfg_page) + goto out_failed_noretry; + + mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ; + retval = mpi3mr_alloc_reply_sense_bufs(mrioc); if (retval) { ioc_err(mrioc, @@ -3795,8 +3894,7 @@ retry_init: if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) { dprint_init(mrioc, "allocating memory for throttle groups\n"); sz = sizeof(struct mpi3mr_throttle_group_info); - mrioc->throttle_groups = (struct mpi3mr_throttle_group_info *) - kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL); + mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL); if (!mrioc->throttle_groups) goto out_failed_noretry; } @@ -3845,8 +3943,12 @@ int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume) int retval = 0; u8 retry = 0; struct mpi3_ioc_facts_data facts_data; + u32 pe_timeout, ioc_status; retry_init: + pe_timeout = + (MPI3MR_PORTENABLE_TIMEOUT / MPI3MR_PORTENABLE_POLL_INTERVAL); + dprint_reset(mrioc, "bringing up the controller to ready state\n"); retval = mpi3mr_bring_ioc_ready(mrioc); if (retval) { @@ -3936,12 +4038,50 @@ retry_init: goto out_failed; } + mrioc->device_refresh_on = 1; + mpi3mr_add_event_wait_for_device_refresh(mrioc); + ioc_info(mrioc, "sending port enable\n"); - retval = mpi3mr_issue_port_enable(mrioc, 0); + retval = mpi3mr_issue_port_enable(mrioc, 1); if (retval) { ioc_err(mrioc, "failed to issue port enable\n"); goto out_failed; } + do { + ssleep(MPI3MR_PORTENABLE_POLL_INTERVAL); + if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED) + break; + if (!pci_device_is_present(mrioc->pdev)) + mrioc->unrecoverable = 1; + if (mrioc->unrecoverable) { + retval = -1; + goto out_failed_noretry; + } + ioc_status = readl(&mrioc->sysif_regs->ioc_status); + if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) || + (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) { + mpi3mr_print_fault_info(mrioc); + mrioc->init_cmds.is_waiting = 0; + mrioc->init_cmds.callback = NULL; + mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED; + goto out_failed; + } + } while (--pe_timeout); + + if (!pe_timeout) { + ioc_err(mrioc, "port enable timed out\n"); + mpi3mr_check_rh_fault_ioc(mrioc, + MPI3MR_RESET_FROM_PE_TIMEOUT); + mrioc->init_cmds.is_waiting = 0; + mrioc->init_cmds.callback = NULL; + mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED; + goto out_failed; + } else if (mrioc->scan_failed) { + ioc_err(mrioc, + "port enable failed with status=0x%04x\n", + mrioc->scan_failed); + } else + ioc_info(mrioc, "port enable completed successfully\n"); ioc_info(mrioc, "controller %s completed successfully\n", (is_resume)?"resume":"re-initialization"); @@ -4042,6 +4182,8 @@ void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc) sizeof(*mrioc->pel_cmds.reply)); memset(mrioc->pel_abort_cmd.reply, 0, sizeof(*mrioc->pel_abort_cmd.reply)); + memset(mrioc->transport_cmds.reply, 0, + sizeof(*mrioc->transport_cmds.reply)); for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) memset(mrioc->dev_rmhs_cmds[i].reply, 0, sizeof(*mrioc->dev_rmhs_cmds[i].reply)); @@ -4102,6 +4244,8 @@ void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc) u16 i; struct mpi3mr_intr_info *intr_info; + mpi3mr_free_enclosure_list(mrioc); + if (mrioc->sense_buf_pool) { if (mrioc->sense_buf) dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf, @@ -4187,6 +4331,9 @@ void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc) kfree(mrioc->chain_bitmap); mrioc->chain_bitmap = NULL; + kfree(mrioc->transport_cmds.reply); + mrioc->transport_cmds.reply = NULL; + for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) { kfree(mrioc->dev_rmhs_cmds[i].reply); mrioc->dev_rmhs_cmds[i].reply = NULL; @@ -4355,13 +4502,17 @@ static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc, * * Return: Nothing. */ -static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc) +void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc) { struct mpi3mr_drv_cmd *cmdptr; u8 i; cmdptr = &mrioc->init_cmds; mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr); + + cmdptr = &mrioc->cfg_cmds; + mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr); + cmdptr = &mrioc->bsg_cmds; mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr); cmdptr = &mrioc->host_tm_cmds; @@ -4383,6 +4534,8 @@ static void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc) cmdptr = &mrioc->pel_abort_cmd; mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr); + cmdptr = &mrioc->transport_cmds; + mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr); } /** @@ -4681,6 +4834,7 @@ int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc, ioc_info(mrioc, "controller reset is triggered by %s\n", mpi3mr_reset_rc_name(reset_reason)); + mrioc->device_refresh_on = 0; mrioc->reset_in_progress = 1; mrioc->stop_bsgs = 1; mrioc->prev_reset_result = -1; @@ -4739,6 +4893,8 @@ int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc, mpi3mr_flush_host_io(mrioc); mpi3mr_cleanup_fwevt_list(mrioc); mpi3mr_invalidate_devhandles(mrioc); + mpi3mr_free_enclosure_list(mrioc); + if (mrioc->prepare_for_reset) { mrioc->prepare_for_reset = 0; mrioc->prepare_for_reset_timeout_counter = 0; @@ -4750,7 +4906,7 @@ int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc, mrioc->name, reset_reason); goto out; } - ssleep(10); + ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME); out: if (!retval) { @@ -4762,7 +4918,8 @@ out: mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds); } - mpi3mr_rfresh_tgtdevs(mrioc); + mrioc->device_refresh_on = 0; + mrioc->ts_update_counter = 0; spin_lock_irqsave(&mrioc->watchdog_lock, flags); if (mrioc->watchdog_work_q) @@ -4776,9 +4933,11 @@ out: } else { mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason); + mrioc->device_refresh_on = 0; mrioc->unrecoverable = 1; mrioc->reset_in_progress = 0; retval = -1; + mpi3mr_flush_cmds_for_unrecovered_controller(mrioc); } mrioc->prev_reset_result = retval; mutex_unlock(&mrioc->reset_mutex); @@ -4786,3 +4945,836 @@ out: ((retval == 0) ? "successful" : "failed")); return retval; } + + +/** + * mpi3mr_free_config_dma_memory - free memory for config page + * @mrioc: Adapter instance reference + * @mem_desc: memory descriptor structure + * + * Check whether the size of the buffer specified by the memory + * descriptor is greater than the default page size if so then + * free the memory pointed by the descriptor. + * + * Return: Nothing. + */ +static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc, + struct dma_memory_desc *mem_desc) +{ + if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) { + dma_free_coherent(&mrioc->pdev->dev, mem_desc->size, + mem_desc->addr, mem_desc->dma_addr); + mem_desc->addr = NULL; + } +} + +/** + * mpi3mr_alloc_config_dma_memory - Alloc memory for config page + * @mrioc: Adapter instance reference + * @mem_desc: Memory descriptor to hold dma memory info + * + * This function allocates new dmaable memory or provides the + * default config page dmaable memory based on the memory size + * described by the descriptor. + * + * Return: 0 on success, non-zero on failure. + */ +static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc, + struct dma_memory_desc *mem_desc) +{ + if (mem_desc->size > mrioc->cfg_page_sz) { + mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev, + mem_desc->size, &mem_desc->dma_addr, GFP_KERNEL); + if (!mem_desc->addr) + return -ENOMEM; + } else { + mem_desc->addr = mrioc->cfg_page; + mem_desc->dma_addr = mrioc->cfg_page_dma; + memset(mem_desc->addr, 0, mrioc->cfg_page_sz); + } + return 0; +} + +/** + * mpi3mr_post_cfg_req - Issue config requests and wait + * @mrioc: Adapter instance reference + * @cfg_req: Configuration request + * @timeout: Timeout in seconds + * @ioc_status: Pointer to return ioc status + * + * A generic function for posting MPI3 configuration request to + * the firmware. This blocks for the completion of request for + * timeout seconds and if the request times out this function + * faults the controller with proper reason code. + * + * On successful completion of the request this function returns + * appropriate ioc status from the firmware back to the caller. + * + * Return: 0 on success, non-zero on failure. + */ +static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc, + struct mpi3_config_request *cfg_req, int timeout, u16 *ioc_status) +{ + int retval = 0; + + mutex_lock(&mrioc->cfg_cmds.mutex); + if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) { + retval = -1; + ioc_err(mrioc, "sending config request failed due to command in use\n"); + mutex_unlock(&mrioc->cfg_cmds.mutex); + goto out; + } + mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING; + mrioc->cfg_cmds.is_waiting = 1; + mrioc->cfg_cmds.callback = NULL; + mrioc->cfg_cmds.ioc_status = 0; + mrioc->cfg_cmds.ioc_loginfo = 0; + + cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS); + cfg_req->function = MPI3_FUNCTION_CONFIG; + + init_completion(&mrioc->cfg_cmds.done); + dprint_cfg_info(mrioc, "posting config request\n"); + if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO) + dprint_dump(cfg_req, sizeof(struct mpi3_config_request), + "mpi3_cfg_req"); + retval = mpi3mr_admin_request_post(mrioc, cfg_req, sizeof(*cfg_req), 1); + if (retval) { + ioc_err(mrioc, "posting config request failed\n"); + goto out_unlock; + } + wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ)); + if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) { + mpi3mr_check_rh_fault_ioc(mrioc, + MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT); + ioc_err(mrioc, "config request timed out\n"); + retval = -1; + goto out_unlock; + } + *ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK; + if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS) + dprint_cfg_err(mrioc, + "cfg_page request returned with ioc_status(0x%04x), log_info(0x%08x)\n", + *ioc_status, mrioc->cfg_cmds.ioc_loginfo); + +out_unlock: + mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED; + mutex_unlock(&mrioc->cfg_cmds.mutex); + +out: + return retval; +} + +/** + * mpi3mr_process_cfg_req - config page request processor + * @mrioc: Adapter instance reference + * @cfg_req: Configuration request + * @cfg_hdr: Configuration page header + * @timeout: Timeout in seconds + * @ioc_status: Pointer to return ioc status + * @cfg_buf: Memory pointer to copy config page or header + * @cfg_buf_sz: Size of the memory to get config page or header + * + * This is handler for config page read, write and config page + * header read operations. + * + * This function expects the cfg_req to be populated with page + * type, page number, action for the header read and with page + * address for all other operations. + * + * The cfg_hdr can be passed as null for reading required header + * details for read/write pages the cfg_hdr should point valid + * configuration page header. + * + * This allocates dmaable memory based on the size of the config + * buffer and set the SGE of the cfg_req. + * + * For write actions, the config page data has to be passed in + * the cfg_buf and size of the data has to be mentioned in the + * cfg_buf_sz. + * + * For read/header actions, on successful completion of the + * request with successful ioc_status the data will be copied + * into the cfg_buf limited to a minimum of actual page size and + * cfg_buf_sz + * + * + * Return: 0 on success, non-zero on failure. + */ +static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc, + struct mpi3_config_request *cfg_req, + struct mpi3_config_page_header *cfg_hdr, int timeout, u16 *ioc_status, + void *cfg_buf, u32 cfg_buf_sz) +{ + struct dma_memory_desc mem_desc; + int retval = -1; + u8 invalid_action = 0; + u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST; + + memset(&mem_desc, 0, sizeof(struct dma_memory_desc)); + + if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER) + mem_desc.size = sizeof(struct mpi3_config_page_header); + else { + if (!cfg_hdr) { + ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n", + cfg_req->action, cfg_req->page_type, + cfg_req->page_number); + goto out; + } + switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) { + case MPI3_CONFIG_PAGEATTR_READ_ONLY: + if (cfg_req->action + != MPI3_CONFIG_ACTION_READ_CURRENT) + invalid_action = 1; + break; + case MPI3_CONFIG_PAGEATTR_CHANGEABLE: + if ((cfg_req->action == + MPI3_CONFIG_ACTION_READ_PERSISTENT) || + (cfg_req->action == + MPI3_CONFIG_ACTION_WRITE_PERSISTENT)) + invalid_action = 1; + break; + case MPI3_CONFIG_PAGEATTR_PERSISTENT: + default: + break; + } + if (invalid_action) { + ioc_err(mrioc, + "config action(%d) is not allowed for page_type(0x%02x), page_num(%d) with page_attribute(0x%02x)\n", + cfg_req->action, cfg_req->page_type, + cfg_req->page_number, cfg_hdr->page_attribute); + goto out; + } + mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4; + cfg_req->page_length = cfg_hdr->page_length; + cfg_req->page_version = cfg_hdr->page_version; + } + if (mpi3mr_alloc_config_dma_memory(mrioc, &mem_desc)) + goto out; + + mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size, + mem_desc.dma_addr); + + if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) || + (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) { + memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size, + cfg_buf_sz)); + dprint_cfg_info(mrioc, "config buffer to be written\n"); + if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO) + dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf"); + } + + if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status)) + goto out; + + retval = 0; + if ((*ioc_status == MPI3_IOCSTATUS_SUCCESS) && + (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) && + (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) { + memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size, + cfg_buf_sz)); + dprint_cfg_info(mrioc, "config buffer read\n"); + if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO) + dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf"); + } + +out: + mpi3mr_free_config_dma_memory(mrioc, &mem_desc); + return retval; +} + +/** + * mpi3mr_cfg_get_dev_pg0 - Read current device page0 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @dev_pg0: Pointer to return device page 0 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like device handle + * + * This is handler for config page read for a specific device + * page0. The ioc_status has the controller returned ioc_status. + * This routine doesn't check ioc_status to decide whether the + * page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(dev_pg0, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DEVICE; + cfg_req.page_number = 0; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "device page0 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_DEVICE_PGAD_FORM_MASK) | + (form_spec & MPI3_DEVICE_PGAD_HANDLE_MASK)); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, dev_pg0, pg_sz)) { + ioc_err(mrioc, "device page0 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + + +/** + * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @phy_pg0: Pointer to return SAS Phy page 0 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like phy number + * + * This is handler for config page read for a specific SAS Phy + * page0. The ioc_status has the controller returned ioc_status. + * This routine doesn't check ioc_status to decide whether the + * page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form, + u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(phy_pg0, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY; + cfg_req.page_number = 0; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "sas phy page0 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) | + (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK)); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg0, pg_sz)) { + ioc_err(mrioc, "sas phy page0 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @phy_pg1: Pointer to return SAS Phy page 1 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like phy number + * + * This is handler for config page read for a specific SAS Phy + * page1. The ioc_status has the controller returned ioc_status. + * This routine doesn't check ioc_status to decide whether the + * page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form, + u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(phy_pg1, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY; + cfg_req.page_number = 1; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "sas phy page1 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) | + (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK)); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg1, pg_sz)) { + ioc_err(mrioc, "sas phy page1 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + + +/** + * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @exp_pg0: Pointer to return SAS Expander page 0 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like device handle + * + * This is handler for config page read for a specific SAS + * Expander page0. The ioc_status has the controller returned + * ioc_status. This routine doesn't check ioc_status to decide + * whether the page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form, + u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(exp_pg0, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER; + cfg_req.page_number = 0; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "expander page0 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) | + (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK | + MPI3_SAS_EXPAND_PGAD_HANDLE_MASK))); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg0, pg_sz)) { + ioc_err(mrioc, "expander page0 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @exp_pg1: Pointer to return SAS Expander page 1 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like phy number + * + * This is handler for config page read for a specific SAS + * Expander page1. The ioc_status has the controller returned + * ioc_status. This routine doesn't check ioc_status to decide + * whether the page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form, + u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(exp_pg1, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER; + cfg_req.page_number = 1; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "expander page1 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) | + (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK | + MPI3_SAS_EXPAND_PGAD_HANDLE_MASK))); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg1, pg_sz)) { + ioc_err(mrioc, "expander page1 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0 + * @mrioc: Adapter instance reference + * @ioc_status: Pointer to return ioc status + * @encl_pg0: Pointer to return Enclosure page 0 + * @pg_sz: Size of the memory allocated to the page pointer + * @form: The form to be used for addressing the page + * @form_spec: Form specific information like device handle + * + * This is handler for config page read for a specific Enclosure + * page0. The ioc_status has the controller returned ioc_status. + * This routine doesn't check ioc_status to decide whether the + * page read is success or not and it is the callers + * responsibility. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status, + struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form, + u32 form_spec) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u32 page_address; + + memset(encl_pg0, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_ENCLOSURE; + cfg_req.page_number = 0; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "enclosure page0 header read failed\n"); + goto out_failed; + } + if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n", + *ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + page_address = ((form & MPI3_ENCLOS_PGAD_FORM_MASK) | + (form_spec & MPI3_ENCLOS_PGAD_HANDLE_MASK)); + cfg_req.page_address = cpu_to_le32(page_address); + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, ioc_status, encl_pg0, pg_sz)) { + ioc_err(mrioc, "enclosure page0 read failed\n"); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + + +/** + * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0 + * @mrioc: Adapter instance reference + * @sas_io_unit_pg0: Pointer to return SAS IO Unit page 0 + * @pg_sz: Size of the memory allocated to the page pointer + * + * This is handler for config page read for the SAS IO Unit + * page0. This routine checks ioc_status to decide whether the + * page read is success or not. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u16 ioc_status = 0; + + memset(sas_io_unit_pg0, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT; + cfg_req.page_number = 0; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "sas io unit page0 header read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg0, pg_sz)) { + ioc_err(mrioc, "sas io unit page0 read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1 + * @mrioc: Adapter instance reference + * @sas_io_unit_pg1: Pointer to return SAS IO Unit page 1 + * @pg_sz: Size of the memory allocated to the page pointer + * + * This is handler for config page read for the SAS IO Unit + * page1. This routine checks ioc_status to decide whether the + * page read is success or not. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u16 ioc_status = 0; + + memset(sas_io_unit_pg1, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT; + cfg_req.page_number = 1; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "sas io unit page1 header read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) { + ioc_err(mrioc, "sas io unit page1 read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1 + * @mrioc: Adapter instance reference + * @sas_io_unit_pg1: Pointer to the SAS IO Unit page 1 to write + * @pg_sz: Size of the memory allocated to the page pointer + * + * This is handler for config page write for the SAS IO Unit + * page1. This routine checks ioc_status to decide whether the + * page read is success or not. This will modify both current + * and persistent page. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u16 ioc_status = 0; + + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT; + cfg_req.page_number = 1; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "sas io unit page1 header read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_WRITE_CURRENT; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) { + ioc_err(mrioc, "sas io unit page1 write current failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + + cfg_req.action = MPI3_CONFIG_ACTION_WRITE_PERSISTENT; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) { + ioc_err(mrioc, "sas io unit page1 write persistent failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + return 0; +out_failed: + return -1; +} + +/** + * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1 + * @mrioc: Adapter instance reference + * @driver_pg1: Pointer to return Driver page 1 + * @pg_sz: Size of the memory allocated to the page pointer + * + * This is handler for config page read for the Driver page1. + * This routine checks ioc_status to decide whether the page + * read is success or not. + * + * Return: 0 on success, non-zero on failure. + */ +int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc, + struct mpi3_driver_page1 *driver_pg1, u16 pg_sz) +{ + struct mpi3_config_page_header cfg_hdr; + struct mpi3_config_request cfg_req; + u16 ioc_status = 0; + + memset(driver_pg1, 0, pg_sz); + memset(&cfg_hdr, 0, sizeof(cfg_hdr)); + memset(&cfg_req, 0, sizeof(cfg_req)); + + cfg_req.function = MPI3_FUNCTION_CONFIG; + cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER; + cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DRIVER; + cfg_req.page_number = 1; + cfg_req.page_address = 0; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) { + ioc_err(mrioc, "driver page1 header read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT; + + if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, driver_pg1, pg_sz)) { + ioc_err(mrioc, "driver page1 read failed\n"); + goto out_failed; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n", + ioc_status); + goto out_failed; + } + return 0; +out_failed: + return -1; +} diff --git a/drivers/scsi/mpi3mr/mpi3mr_os.c b/drivers/scsi/mpi3mr/mpi3mr_os.c index bfa1165e23b67ddb232108ce711d6d0694be6293..f77ee4051b00d140e58938d1512db53486de3494 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_os.c +++ b/drivers/scsi/mpi3mr/mpi3mr_os.c @@ -40,6 +40,8 @@ static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event, #define MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION (0xFFFF) +#define MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH (0xFFFE) + /** * mpi3mr_host_tag_for_scmd - Get host tag for a scmd * @mrioc: Adapter instance reference @@ -422,6 +424,8 @@ void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc) tgt_priv->io_throttle_enabled = 0; tgt_priv->io_divert = 0; tgt_priv->throttle_group = NULL; + if (tgtdev->host_exposed) + atomic_set(&tgt_priv->block_io, 1); } } } @@ -578,6 +582,39 @@ void mpi3mr_flush_host_io(struct mpi3mr_ioc *mrioc) mrioc->flush_io_count); } +/** + * mpi3mr_flush_cmds_for_unrecovered_controller - Flush all pending cmds + * @mrioc: Adapter instance reference + * + * This function waits for currently running IO poll threads to + * exit and then flushes all host I/Os and any internal pending + * cmds. This is executed after controller is marked as + * unrecoverable. + * + * Return: Nothing. + */ +void mpi3mr_flush_cmds_for_unrecovered_controller(struct mpi3mr_ioc *mrioc) +{ + struct Scsi_Host *shost = mrioc->shost; + int i; + + if (!mrioc->unrecoverable) + return; + + if (mrioc->op_reply_qinfo) { + for (i = 0; i < mrioc->num_queues; i++) { + while (atomic_read(&mrioc->op_reply_qinfo[i].in_use)) + udelay(500); + atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0); + } + } + mrioc->flush_io_count = 0; + blk_mq_tagset_busy_iter(&shost->tag_set, + mpi3mr_flush_scmd, (void *)mrioc); + mpi3mr_flush_delayed_cmd_lists(mrioc); + mpi3mr_flush_drv_cmds(mrioc); +} + /** * mpi3mr_alloc_tgtdev - target device allocator * @@ -796,7 +833,7 @@ static void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc, * * Return: None. */ -static void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc, +void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc, bool device_add) { ioc_notice(mrioc, "Device %s was in progress before the reset and\n", @@ -816,7 +853,7 @@ static void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc, * * Return: 0 on success, non zero on failure. */ -static void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc, +void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc, struct mpi3mr_tgt_dev *tgtdev) { struct mpi3mr_stgt_priv_data *tgt_priv; @@ -825,22 +862,29 @@ static void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc, __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid); if (tgtdev->starget && tgtdev->starget->hostdata) { tgt_priv = tgtdev->starget->hostdata; + atomic_set(&tgt_priv->block_io, 0); tgt_priv->dev_handle = MPI3MR_INVALID_DEV_HANDLE; } - if (tgtdev->starget) { - if (mrioc->current_event) - mrioc->current_event->pending_at_sml = 1; - scsi_remove_target(&tgtdev->starget->dev); - tgtdev->host_exposed = 0; - if (mrioc->current_event) { - mrioc->current_event->pending_at_sml = 0; - if (mrioc->current_event->discard) { - mpi3mr_print_device_event_notice(mrioc, false); - return; + if (!mrioc->sas_transport_enabled || (tgtdev->dev_type != + MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl) { + if (tgtdev->starget) { + if (mrioc->current_event) + mrioc->current_event->pending_at_sml = 1; + scsi_remove_target(&tgtdev->starget->dev); + tgtdev->host_exposed = 0; + if (mrioc->current_event) { + mrioc->current_event->pending_at_sml = 0; + if (mrioc->current_event->discard) { + mpi3mr_print_device_event_notice(mrioc, + false); + return; + } } } - } + } else + mpi3mr_remove_tgtdev_from_sas_transport(mrioc, tgtdev); + ioc_info(mrioc, "%s :Removed handle(0x%04x), wwid(0x%016llx)\n", __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid); } @@ -862,21 +906,25 @@ static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc, int retval = 0; struct mpi3mr_tgt_dev *tgtdev; + if (mrioc->reset_in_progress) + return -1; + tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id); if (!tgtdev) { retval = -1; goto out; } - if (tgtdev->is_hidden) { + if (tgtdev->is_hidden || tgtdev->host_exposed) { retval = -1; goto out; } - if (!tgtdev->host_exposed && !mrioc->reset_in_progress) { + if (!mrioc->sas_transport_enabled || (tgtdev->dev_type != + MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl){ tgtdev->host_exposed = 1; if (mrioc->current_event) mrioc->current_event->pending_at_sml = 1; - scsi_scan_target(&mrioc->shost->shost_gendev, 0, - tgtdev->perst_id, + scsi_scan_target(&mrioc->shost->shost_gendev, + mrioc->scsi_device_channel, tgtdev->perst_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); if (!tgtdev->starget) tgtdev->host_exposed = 0; @@ -887,7 +935,8 @@ static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc, goto out; } } - } + } else + mpi3mr_report_tgtdev_to_sas_transport(mrioc, tgtdev); out: if (tgtdev) mpi3mr_tgtdev_put(tgtdev); @@ -1018,18 +1067,29 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, { u16 flags = 0; struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL; + struct mpi3mr_enclosure_node *enclosure_dev = NULL; u8 prot_mask = 0; tgtdev->perst_id = le16_to_cpu(dev_pg0->persistent_id); tgtdev->dev_handle = le16_to_cpu(dev_pg0->dev_handle); tgtdev->dev_type = dev_pg0->device_form; + tgtdev->io_unit_port = dev_pg0->io_unit_port; tgtdev->encl_handle = le16_to_cpu(dev_pg0->enclosure_handle); tgtdev->parent_handle = le16_to_cpu(dev_pg0->parent_dev_handle); tgtdev->slot = le16_to_cpu(dev_pg0->slot); tgtdev->q_depth = le16_to_cpu(dev_pg0->queue_depth); tgtdev->wwid = le64_to_cpu(dev_pg0->wwid); + tgtdev->devpg0_flag = le16_to_cpu(dev_pg0->flags); + + if (tgtdev->encl_handle) + enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc, + tgtdev->encl_handle); + if (enclosure_dev) + tgtdev->enclosure_logical_id = le64_to_cpu( + enclosure_dev->pg0.enclosure_logical_id); + + flags = tgtdev->devpg0_flag; - flags = le16_to_cpu(dev_pg0->flags); tgtdev->is_hidden = (flags & MPI3_DEVICE0_FLAGS_HIDDEN); if (is_added == true) @@ -1045,6 +1105,8 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, scsi_tgt_priv_data->dev_type = tgtdev->dev_type; scsi_tgt_priv_data->io_throttle_enabled = tgtdev->io_throttle_enabled; + if (is_added == true) + atomic_set(&scsi_tgt_priv_data->block_io, 0); } switch (dev_pg0->access_status) { @@ -1068,12 +1130,25 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, tgtdev->dev_spec.sas_sata_inf.dev_info = dev_info; tgtdev->dev_spec.sas_sata_inf.sas_address = le64_to_cpu(sasinf->sas_address); + tgtdev->dev_spec.sas_sata_inf.phy_id = sasinf->phy_num; + tgtdev->dev_spec.sas_sata_inf.attached_phy_id = + sasinf->attached_phy_identifier; if ((dev_info & MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_MASK) != MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_END_DEVICE) tgtdev->is_hidden = 1; else if (!(dev_info & (MPI3_SAS_DEVICE_INFO_STP_SATA_TARGET | MPI3_SAS_DEVICE_INFO_SSP_TARGET))) tgtdev->is_hidden = 1; + + if (((tgtdev->devpg0_flag & + MPI3_DEVICE0_FLAGS_ATT_METHOD_DIR_ATTACHED) + && (tgtdev->devpg0_flag & + MPI3_DEVICE0_FLAGS_ATT_METHOD_VIRTUAL)) || + (tgtdev->parent_handle == 0xFFFF)) + tgtdev->non_stl = 1; + if (tgtdev->dev_spec.sas_sata_inf.hba_port) + tgtdev->dev_spec.sas_sata_inf.hba_port->port_id = + dev_pg0->io_unit_port; break; } case MPI3_DEVICE_DEVFORM_PCIE: @@ -1106,6 +1181,7 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, ((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) != MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_SCSI_DEVICE)) tgtdev->is_hidden = 1; + tgtdev->non_stl = 1; if (!mrioc->shost) break; prot_mask = scsi_host_get_prot(mrioc->shost); @@ -1129,6 +1205,7 @@ static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc, tgtdev->dev_spec.vd_inf.state = vdinf->vd_state; if (vdinf->vd_state == MPI3_DEVICE0_VD_STATE_OFFLINE) tgtdev->is_hidden = 1; + tgtdev->non_stl = 1; tgtdev->dev_spec.vd_inf.tg_id = vdinf_io_throttle_group; tgtdev->dev_spec.vd_inf.tg_high = le16_to_cpu(vdinf->io_throttle_group_high) * 2048; @@ -1257,6 +1334,135 @@ out: mpi3mr_tgtdev_put(tgtdev); } +/** + * mpi3mr_free_enclosure_list - release enclosures + * @mrioc: Adapter instance reference + * + * Free memory allocated during encloure add. + * + * Return nothing. + */ +void mpi3mr_free_enclosure_list(struct mpi3mr_ioc *mrioc) +{ + struct mpi3mr_enclosure_node *enclosure_dev, *enclosure_dev_next; + + list_for_each_entry_safe(enclosure_dev, + enclosure_dev_next, &mrioc->enclosure_list, list) { + list_del(&enclosure_dev->list); + kfree(enclosure_dev); + } +} + +/** + * mpi3mr_enclosure_find_by_handle - enclosure search by handle + * @mrioc: Adapter instance reference + * @handle: Firmware device handle of the enclosure + * + * This searches for enclosure device based on handle, then returns the + * enclosure object. + * + * Return: Enclosure object reference or NULL + */ +struct mpi3mr_enclosure_node *mpi3mr_enclosure_find_by_handle( + struct mpi3mr_ioc *mrioc, u16 handle) +{ + struct mpi3mr_enclosure_node *enclosure_dev, *r = NULL; + + list_for_each_entry(enclosure_dev, &mrioc->enclosure_list, list) { + if (le16_to_cpu(enclosure_dev->pg0.enclosure_handle) != handle) + continue; + r = enclosure_dev; + goto out; + } +out: + return r; +} + +/** + * mpi3mr_encldev_add_chg_evt_debug - debug for enclosure event + * @mrioc: Adapter instance reference + * @encl_pg0: Enclosure page 0. + * @is_added: Added event or not + * + * Return nothing. + */ +static void mpi3mr_encldev_add_chg_evt_debug(struct mpi3mr_ioc *mrioc, + struct mpi3_enclosure_page0 *encl_pg0, u8 is_added) +{ + char *reason_str = NULL; + + if (!(mrioc->logging_level & MPI3_DEBUG_EVENT_WORK_TASK)) + return; + + if (is_added) + reason_str = "enclosure added"; + else + reason_str = "enclosure dev status changed"; + + ioc_info(mrioc, + "%s: handle(0x%04x), enclosure logical id(0x%016llx)\n", + reason_str, le16_to_cpu(encl_pg0->enclosure_handle), + (unsigned long long)le64_to_cpu(encl_pg0->enclosure_logical_id)); + ioc_info(mrioc, + "number of slots(%d), port(%d), flags(0x%04x), present(%d)\n", + le16_to_cpu(encl_pg0->num_slots), encl_pg0->io_unit_port, + le16_to_cpu(encl_pg0->flags), + ((le16_to_cpu(encl_pg0->flags) & + MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4)); +} + +/** + * mpi3mr_encldev_add_chg_evt_bh - Enclosure evt bottomhalf + * @mrioc: Adapter instance reference + * @fwevt: Firmware event reference + * + * Prints information about the Enclosure device status or + * Enclosure add events if logging is enabled and add or remove + * the enclosure from the controller's internal list of + * enclosures. + * + * Return: Nothing. + */ +static void mpi3mr_encldev_add_chg_evt_bh(struct mpi3mr_ioc *mrioc, + struct mpi3mr_fwevt *fwevt) +{ + struct mpi3mr_enclosure_node *enclosure_dev = NULL; + struct mpi3_enclosure_page0 *encl_pg0; + u16 encl_handle; + u8 added, present; + + encl_pg0 = (struct mpi3_enclosure_page0 *) fwevt->event_data; + added = (fwevt->event_id == MPI3_EVENT_ENCL_DEVICE_ADDED) ? 1 : 0; + mpi3mr_encldev_add_chg_evt_debug(mrioc, encl_pg0, added); + + + encl_handle = le16_to_cpu(encl_pg0->enclosure_handle); + present = ((le16_to_cpu(encl_pg0->flags) & + MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4); + + if (encl_handle) + enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc, + encl_handle); + if (!enclosure_dev && present) { + enclosure_dev = + kzalloc(sizeof(struct mpi3mr_enclosure_node), + GFP_KERNEL); + if (!enclosure_dev) + return; + list_add_tail(&enclosure_dev->list, + &mrioc->enclosure_list); + } + if (enclosure_dev) { + if (!present) { + list_del(&enclosure_dev->list); + kfree(enclosure_dev); + } else + memcpy(&enclosure_dev->pg0, encl_pg0, + sizeof(enclosure_dev->pg0)); + + } +} + /** * mpi3mr_sastopochg_evt_debug - SASTopoChange details * @mrioc: Adapter instance reference @@ -1296,8 +1502,9 @@ mpi3mr_sastopochg_evt_debug(struct mpi3mr_ioc *mrioc, ioc_info(mrioc, "%s :sas topology change: (%s)\n", __func__, status_str); ioc_info(mrioc, - "%s :\texpander_handle(0x%04x), enclosure_handle(0x%04x) start_phy(%02d), num_entries(%d)\n", + "%s :\texpander_handle(0x%04x), port(%d), enclosure_handle(0x%04x) start_phy(%02d), num_entries(%d)\n", __func__, le16_to_cpu(event_data->expander_dev_handle), + event_data->io_unit_port, le16_to_cpu(event_data->enclosure_handle), event_data->start_phy_num, event_data->num_entries); for (i = 0; i < event_data->num_entries; i++) { @@ -1355,9 +1562,30 @@ static void mpi3mr_sastopochg_evt_bh(struct mpi3mr_ioc *mrioc, int i; u16 handle; u8 reason_code; + u64 exp_sas_address = 0, parent_sas_address = 0; + struct mpi3mr_hba_port *hba_port = NULL; struct mpi3mr_tgt_dev *tgtdev = NULL; + struct mpi3mr_sas_node *sas_expander = NULL; + unsigned long flags; + u8 link_rate, prev_link_rate, parent_phy_number; mpi3mr_sastopochg_evt_debug(mrioc, event_data); + if (mrioc->sas_transport_enabled) { + hba_port = mpi3mr_get_hba_port_by_id(mrioc, + event_data->io_unit_port); + if (le16_to_cpu(event_data->expander_dev_handle)) { + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + sas_expander = __mpi3mr_expander_find_by_handle(mrioc, + le16_to_cpu(event_data->expander_dev_handle)); + if (sas_expander) { + exp_sas_address = sas_expander->sas_address; + hba_port = sas_expander->hba_port; + } + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + parent_sas_address = exp_sas_address; + } else + parent_sas_address = mrioc->sas_hba.sas_address; + } for (i = 0; i < event_data->num_entries; i++) { if (fwevt->discard) @@ -1379,12 +1607,37 @@ static void mpi3mr_sastopochg_evt_bh(struct mpi3mr_ioc *mrioc, mpi3mr_tgtdev_del_from_list(mrioc, tgtdev); mpi3mr_tgtdev_put(tgtdev); break; + case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING: + case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED: + case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE: + { + if (!mrioc->sas_transport_enabled || tgtdev->non_stl + || tgtdev->is_hidden) + break; + link_rate = event_data->phy_entry[i].link_rate >> 4; + prev_link_rate = event_data->phy_entry[i].link_rate & 0xF; + if (link_rate == prev_link_rate) + break; + if (!parent_sas_address) + break; + parent_phy_number = event_data->start_phy_num + i; + mpi3mr_update_links(mrioc, parent_sas_address, handle, + parent_phy_number, link_rate, hba_port); + break; + } default: break; } if (tgtdev) mpi3mr_tgtdev_put(tgtdev); } + + if (mrioc->sas_transport_enabled && (event_data->exp_status == + MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING)) { + if (sas_expander) + mpi3mr_expander_remove(mrioc, exp_sas_address, + hba_port); + } } /** @@ -1604,28 +1857,54 @@ static void mpi3mr_set_qd_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc, static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc, struct mpi3mr_fwevt *fwevt) { + struct mpi3_device_page0 *dev_pg0 = NULL; + u16 perst_id, handle, dev_info; + struct mpi3_device0_sas_sata_format *sasinf = NULL; + mpi3mr_fwevt_del_from_list(mrioc, fwevt); mrioc->current_event = fwevt; if (mrioc->stop_drv_processing) goto out; + if (mrioc->unrecoverable) { + dprint_event_bh(mrioc, + "ignoring event(0x%02x) in bottom half handler due to unrecoverable controller\n", + fwevt->event_id); + goto out; + } + if (!fwevt->process_evt) goto evt_ack; switch (fwevt->event_id) { case MPI3_EVENT_DEVICE_ADDED: { - struct mpi3_device_page0 *dev_pg0 = - (struct mpi3_device_page0 *)fwevt->event_data; - mpi3mr_report_tgtdev_to_host(mrioc, - le16_to_cpu(dev_pg0->persistent_id)); + dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data; + perst_id = le16_to_cpu(dev_pg0->persistent_id); + handle = le16_to_cpu(dev_pg0->dev_handle); + if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID) + mpi3mr_report_tgtdev_to_host(mrioc, perst_id); + else if (mrioc->sas_transport_enabled && + (dev_pg0->device_form == MPI3_DEVICE_DEVFORM_SAS_SATA)) { + sasinf = &dev_pg0->device_specific.sas_sata_format; + dev_info = le16_to_cpu(sasinf->device_info); + if (!mrioc->sas_hba.num_phys) + mpi3mr_sas_host_add(mrioc); + else + mpi3mr_sas_host_refresh(mrioc); + + if (mpi3mr_is_expander_device(dev_info)) + mpi3mr_expander_add(mrioc, handle); + } break; } case MPI3_EVENT_DEVICE_INFO_CHANGED: { - mpi3mr_devinfochg_evt_bh(mrioc, - (struct mpi3_device_page0 *)fwevt->event_data); + dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data; + perst_id = le16_to_cpu(dev_pg0->persistent_id); + if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID) + mpi3mr_devinfochg_evt_bh(mrioc, dev_pg0); break; } case MPI3_EVENT_DEVICE_STATUS_CHANGE: @@ -1633,6 +1912,13 @@ static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc, mpi3mr_devstatuschg_evt_bh(mrioc, fwevt); break; } + case MPI3_EVENT_ENCL_DEVICE_ADDED: + case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE: + { + mpi3mr_encldev_add_chg_evt_bh(mrioc, fwevt); + break; + } + case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST: { mpi3mr_sastopochg_evt_bh(mrioc, fwevt); @@ -1662,6 +1948,22 @@ static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc, } break; } + case MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH: + { + while (mrioc->device_refresh_on) + msleep(500); + + dprint_event_bh(mrioc, + "scan for non responding and newly added devices after soft reset started\n"); + if (mrioc->sas_transport_enabled) { + mpi3mr_refresh_sas_ports(mrioc); + mpi3mr_refresh_expanders(mrioc); + } + mpi3mr_rfresh_tgtdevs(mrioc); + ioc_info(mrioc, + "scan for non responding and newly added devices after soft reset completed\n"); + break; + } default: break; } @@ -1716,6 +2018,9 @@ static int mpi3mr_create_tgtdev(struct mpi3mr_ioc *mrioc, u16 perst_id = 0; perst_id = le16_to_cpu(dev_pg0->persistent_id); + if (perst_id == MPI3_DEVICE0_PERSISTENTID_INVALID) + return retval; + tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id); if (tgtdev) { mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true); @@ -2428,6 +2733,35 @@ static void mpi3mr_cablemgmt_evt_th(struct mpi3mr_ioc *mrioc, } } +/** + * mpi3mr_add_event_wait_for_device_refresh - Add Wait for Device Refresh Event + * @mrioc: Adapter instance reference + * + * Add driver specific event to make sure that the driver won't process the + * events until all the devices are refreshed during soft reset. + * + * Return: Nothing + */ +void mpi3mr_add_event_wait_for_device_refresh(struct mpi3mr_ioc *mrioc) +{ + struct mpi3mr_fwevt *fwevt = NULL; + + fwevt = mpi3mr_alloc_fwevt(0); + if (!fwevt) { + dprint_event_th(mrioc, + "failed to schedule bottom half handler for event(0x%02x)\n", + MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH); + return; + } + fwevt->mrioc = mrioc; + fwevt->event_id = MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH; + fwevt->send_ack = 0; + fwevt->process_evt = 1; + fwevt->evt_ctx = 0; + fwevt->event_data_size = 0; + mpi3mr_fwevt_add_to_list(mrioc, fwevt); +} + /** * mpi3mr_os_handle_events - Firmware event handler * @mrioc: Adapter instance reference @@ -2494,6 +2828,8 @@ void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc, } case MPI3_EVENT_DEVICE_INFO_CHANGED: case MPI3_EVENT_LOG_DATA: + case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE: + case MPI3_EVENT_ENCL_DEVICE_ADDED: { process_evt_bh = 1; break; @@ -2508,7 +2844,6 @@ void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc, mpi3mr_cablemgmt_evt_th(mrioc, event_reply); break; } - case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE: case MPI3_EVENT_SAS_DISCOVERY: case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR: case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE: @@ -3464,7 +3799,7 @@ static int mpi3mr_bios_param(struct scsi_device *sdev, * * Return: return zero. */ -static int mpi3mr_map_queues(struct Scsi_Host *shost) +static void mpi3mr_map_queues(struct Scsi_Host *shost) { struct mpi3mr_ioc *mrioc = shost_priv(shost); int i, qoff, offset; @@ -3500,9 +3835,6 @@ static int mpi3mr_map_queues(struct Scsi_Host *shost) qoff += map->nr_queues; offset += map->nr_queues; } - - return 0; - } /** @@ -3852,9 +4184,10 @@ static void mpi3mr_slave_destroy(struct scsi_device *sdev) struct Scsi_Host *shost; struct mpi3mr_ioc *mrioc; struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data; - struct mpi3mr_tgt_dev *tgt_dev; + struct mpi3mr_tgt_dev *tgt_dev = NULL; unsigned long flags; struct scsi_target *starget; + struct sas_rphy *rphy = NULL; if (!sdev->hostdata) return; @@ -3867,7 +4200,14 @@ static void mpi3mr_slave_destroy(struct scsi_device *sdev) scsi_tgt_priv_data->num_luns--; spin_lock_irqsave(&mrioc->tgtdev_lock, flags); - tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + if (starget->channel == mrioc->scsi_device_channel) + tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + else if (mrioc->sas_transport_enabled && !starget->channel) { + rphy = dev_to_rphy(starget->dev.parent); + tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + } + if (tgt_dev && (!scsi_tgt_priv_data->num_luns)) tgt_dev->starget = NULL; if (tgt_dev) @@ -3932,16 +4272,23 @@ static int mpi3mr_slave_configure(struct scsi_device *sdev) struct scsi_target *starget; struct Scsi_Host *shost; struct mpi3mr_ioc *mrioc; - struct mpi3mr_tgt_dev *tgt_dev; + struct mpi3mr_tgt_dev *tgt_dev = NULL; unsigned long flags; int retval = 0; + struct sas_rphy *rphy = NULL; starget = scsi_target(sdev); shost = dev_to_shost(&starget->dev); mrioc = shost_priv(shost); spin_lock_irqsave(&mrioc->tgtdev_lock, flags); - tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + if (starget->channel == mrioc->scsi_device_channel) + tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + else if (mrioc->sas_transport_enabled && !starget->channel) { + rphy = dev_to_rphy(starget->dev.parent); + tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + } spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); if (!tgt_dev) return -ENXIO; @@ -3989,11 +4336,12 @@ static int mpi3mr_slave_alloc(struct scsi_device *sdev) struct Scsi_Host *shost; struct mpi3mr_ioc *mrioc; struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data; - struct mpi3mr_tgt_dev *tgt_dev; + struct mpi3mr_tgt_dev *tgt_dev = NULL; struct mpi3mr_sdev_priv_data *scsi_dev_priv_data; unsigned long flags; struct scsi_target *starget; int retval = 0; + struct sas_rphy *rphy = NULL; starget = scsi_target(sdev); shost = dev_to_shost(&starget->dev); @@ -4001,7 +4349,14 @@ static int mpi3mr_slave_alloc(struct scsi_device *sdev) scsi_tgt_priv_data = starget->hostdata; spin_lock_irqsave(&mrioc->tgtdev_lock, flags); - tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + + if (starget->channel == mrioc->scsi_device_channel) + tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + else if (mrioc->sas_transport_enabled && !starget->channel) { + rphy = dev_to_rphy(starget->dev.parent); + tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + } if (tgt_dev) { if (tgt_dev->starget == NULL) @@ -4044,6 +4399,8 @@ static int mpi3mr_target_alloc(struct scsi_target *starget) struct mpi3mr_tgt_dev *tgt_dev; unsigned long flags; int retval = 0; + struct sas_rphy *rphy = NULL; + bool update_stgt_priv_data = false; scsi_tgt_priv_data = kzalloc(sizeof(*scsi_tgt_priv_data), GFP_KERNEL); if (!scsi_tgt_priv_data) @@ -4052,8 +4409,25 @@ static int mpi3mr_target_alloc(struct scsi_target *starget) starget->hostdata = scsi_tgt_priv_data; spin_lock_irqsave(&mrioc->tgtdev_lock, flags); - tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); - if (tgt_dev && !tgt_dev->is_hidden) { + + if (starget->channel == mrioc->scsi_device_channel) { + tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id); + if (tgt_dev && !tgt_dev->is_hidden) + update_stgt_priv_data = true; + else + retval = -ENXIO; + } else if (mrioc->sas_transport_enabled && !starget->channel) { + rphy = dev_to_rphy(starget->dev.parent); + tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + if (tgt_dev && !tgt_dev->is_hidden && !tgt_dev->non_stl && + (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_SAS_SATA)) + update_stgt_priv_data = true; + else + retval = -ENXIO; + } + + if (update_stgt_priv_data) { scsi_tgt_priv_data->starget = starget; scsi_tgt_priv_data->dev_handle = tgt_dev->dev_handle; scsi_tgt_priv_data->perst_id = tgt_dev->perst_id; @@ -4067,8 +4441,7 @@ static int mpi3mr_target_alloc(struct scsi_target *starget) if (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_VD) scsi_tgt_priv_data->throttle_group = tgt_dev->dev_spec.vd_inf.tg; - } else - retval = -ENXIO; + } spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); return retval; @@ -4257,6 +4630,16 @@ static int mpi3mr_qcmd(struct Scsi_Host *shost, stgt_priv_data = sdev_priv_data->tgt_priv_data; + if (atomic_read(&stgt_priv_data->block_io)) { + if (mrioc->stop_drv_processing) { + scmd->result = DID_NO_CONNECT << 16; + scsi_done(scmd); + goto out; + } + retval = SCSI_MLQUEUE_DEVICE_BUSY; + goto out; + } + dev_handle = stgt_priv_data->dev_handle; if (dev_handle == MPI3MR_INVALID_DEV_HANDLE) { scmd->result = DID_NO_CONNECT << 16; @@ -4269,16 +4652,6 @@ static int mpi3mr_qcmd(struct Scsi_Host *shost, goto out; } - if (atomic_read(&stgt_priv_data->block_io)) { - if (mrioc->stop_drv_processing) { - scmd->result = DID_NO_CONNECT << 16; - scsi_done(scmd); - goto out; - } - retval = SCSI_MLQUEUE_DEVICE_BUSY; - goto out; - } - if (stgt_priv_data->dev_type == MPI3_DEVICE_DEVFORM_PCIE) is_pcie_dev = 1; if ((scmd->cmnd[0] == UNMAP) && is_pcie_dev && @@ -4556,16 +4929,23 @@ mpi3mr_probe(struct pci_dev *pdev, const struct pci_device_id *id) spin_lock_init(&mrioc->tgtdev_lock); spin_lock_init(&mrioc->watchdog_lock); spin_lock_init(&mrioc->chain_buf_lock); + spin_lock_init(&mrioc->sas_node_lock); INIT_LIST_HEAD(&mrioc->fwevt_list); INIT_LIST_HEAD(&mrioc->tgtdev_list); INIT_LIST_HEAD(&mrioc->delayed_rmhs_list); INIT_LIST_HEAD(&mrioc->delayed_evtack_cmds_list); + INIT_LIST_HEAD(&mrioc->sas_expander_list); + INIT_LIST_HEAD(&mrioc->hba_port_table_list); + INIT_LIST_HEAD(&mrioc->enclosure_list); mutex_init(&mrioc->reset_mutex); mpi3mr_init_drv_cmd(&mrioc->init_cmds, MPI3MR_HOSTTAG_INITCMDS); mpi3mr_init_drv_cmd(&mrioc->host_tm_cmds, MPI3MR_HOSTTAG_BLK_TMS); mpi3mr_init_drv_cmd(&mrioc->bsg_cmds, MPI3MR_HOSTTAG_BSG_CMDS); + mpi3mr_init_drv_cmd(&mrioc->cfg_cmds, MPI3MR_HOSTTAG_CFG_CMDS); + mpi3mr_init_drv_cmd(&mrioc->transport_cmds, + MPI3MR_HOSTTAG_TRANSPORT_CMDS); for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) mpi3mr_init_drv_cmd(&mrioc->dev_rmhs_cmds[i], @@ -4700,6 +5080,11 @@ static void mpi3mr_remove(struct pci_dev *pdev) while (mrioc->reset_in_progress || mrioc->is_driver_loading) ssleep(1); + if (!pci_device_is_present(mrioc->pdev)) { + mrioc->unrecoverable = 1; + mpi3mr_flush_cmds_for_unrecovered_controller(mrioc); + } + mpi3mr_bsg_exit(mrioc); mrioc->stop_drv_processing = 1; mpi3mr_cleanup_fwevt_list(mrioc); @@ -4709,7 +5094,11 @@ static void mpi3mr_remove(struct pci_dev *pdev) spin_unlock_irqrestore(&mrioc->fwevt_lock, flags); if (wq) destroy_workqueue(wq); - scsi_remove_host(shost); + + if (mrioc->sas_transport_enabled) + sas_remove_host(shost); + else + scsi_remove_host(shost); list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list, list) { @@ -4766,22 +5155,21 @@ static void mpi3mr_shutdown(struct pci_dev *pdev) mpi3mr_cleanup_resources(mrioc); } -#ifdef CONFIG_PM /** * mpi3mr_suspend - PCI power management suspend callback - * @pdev: PCI device instance - * @state: New power state + * @dev: Device struct * * Change the power state to the given value and cleanup the IOC * by issuing MUR and shutdown notification * * Return: 0 always. */ -static int mpi3mr_suspend(struct pci_dev *pdev, pm_message_t state) +static int __maybe_unused +mpi3mr_suspend(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); struct Scsi_Host *shost = pci_get_drvdata(pdev); struct mpi3mr_ioc *mrioc; - pci_power_t device_state; if (!shost) return 0; @@ -4795,27 +5183,26 @@ static int mpi3mr_suspend(struct pci_dev *pdev, pm_message_t state) mpi3mr_stop_watchdog(mrioc); mpi3mr_cleanup_ioc(mrioc); - device_state = pci_choose_state(pdev, state); - ioc_info(mrioc, "pdev=0x%p, slot=%s, entering operating state [D%d]\n", - pdev, pci_name(pdev), device_state); - pci_save_state(pdev); + ioc_info(mrioc, "pdev=0x%p, slot=%s, entering operating state\n", + pdev, pci_name(pdev)); mpi3mr_cleanup_resources(mrioc); - pci_set_power_state(pdev, device_state); return 0; } /** * mpi3mr_resume - PCI power management resume callback - * @pdev: PCI device instance + * @dev: Device struct * * Restore the power state to D0 and reinitialize the controller * and resume I/O operations to the target devices * * Return: 0 on success, non-zero on failure */ -static int mpi3mr_resume(struct pci_dev *pdev) +static int __maybe_unused +mpi3mr_resume(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); struct Scsi_Host *shost = pci_get_drvdata(pdev); struct mpi3mr_ioc *mrioc; pci_power_t device_state = pdev->current_state; @@ -4828,9 +5215,6 @@ static int mpi3mr_resume(struct pci_dev *pdev) ioc_info(mrioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n", pdev, pci_name(pdev), device_state); - pci_set_power_state(pdev, PCI_D0); - pci_enable_wake(pdev, PCI_D0, 0); - pci_restore_state(pdev); mrioc->pdev = pdev; mrioc->cpu_count = num_online_cpus(); r = mpi3mr_setup_resources(mrioc); @@ -4841,18 +5225,21 @@ static int mpi3mr_resume(struct pci_dev *pdev) } mrioc->stop_drv_processing = 0; + mpi3mr_invalidate_devhandles(mrioc); + mpi3mr_free_enclosure_list(mrioc); mpi3mr_memset_buffers(mrioc); r = mpi3mr_reinit_ioc(mrioc, 1); if (r) { ioc_err(mrioc, "resuming controller failed[%d]\n", r); return r; } + ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME); scsi_unblock_requests(shost); + mrioc->device_refresh_on = 0; mpi3mr_start_watchdog(mrioc); return 0; } -#endif static const struct pci_device_id mpi3mr_pci_id_table[] = { { @@ -4863,16 +5250,15 @@ static const struct pci_device_id mpi3mr_pci_id_table[] = { }; MODULE_DEVICE_TABLE(pci, mpi3mr_pci_id_table); +static SIMPLE_DEV_PM_OPS(mpi3mr_pm_ops, mpi3mr_suspend, mpi3mr_resume); + static struct pci_driver mpi3mr_pci_driver = { .name = MPI3MR_DRIVER_NAME, .id_table = mpi3mr_pci_id_table, .probe = mpi3mr_probe, .remove = mpi3mr_remove, .shutdown = mpi3mr_shutdown, -#ifdef CONFIG_PM - .suspend = mpi3mr_suspend, - .resume = mpi3mr_resume, -#endif + .driver.pm = &mpi3mr_pm_ops, }; static ssize_t event_counter_show(struct device_driver *dd, char *buf) @@ -4888,18 +5274,33 @@ static int __init mpi3mr_init(void) pr_info("Loading %s version %s\n", MPI3MR_DRIVER_NAME, MPI3MR_DRIVER_VERSION); + mpi3mr_transport_template = + sas_attach_transport(&mpi3mr_transport_functions); + if (!mpi3mr_transport_template) { + pr_err("%s failed to load due to sas transport attach failure\n", + MPI3MR_DRIVER_NAME); + return -ENODEV; + } + ret_val = pci_register_driver(&mpi3mr_pci_driver); if (ret_val) { pr_err("%s failed to load due to pci register driver failure\n", MPI3MR_DRIVER_NAME); - return ret_val; + goto err_pci_reg_fail; } ret_val = driver_create_file(&mpi3mr_pci_driver.driver, &driver_attr_event_counter); if (ret_val) - pci_unregister_driver(&mpi3mr_pci_driver); + goto err_event_counter; + + return ret_val; + +err_event_counter: + pci_unregister_driver(&mpi3mr_pci_driver); +err_pci_reg_fail: + sas_release_transport(mpi3mr_transport_template); return ret_val; } @@ -4916,6 +5317,7 @@ static void __exit mpi3mr_exit(void) driver_remove_file(&mpi3mr_pci_driver.driver, &driver_attr_event_counter); pci_unregister_driver(&mpi3mr_pci_driver); + sas_release_transport(mpi3mr_transport_template); } module_init(mpi3mr_init); diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c new file mode 100644 index 0000000000000000000000000000000000000000..3fc897336b5e097ac1f80eeb37546b381eb02874 --- /dev/null +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -0,0 +1,3291 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for Broadcom MPI3 Storage Controllers + * + * Copyright (C) 2017-2022 Broadcom Inc. + * (mailto: mpi3mr-linuxdrv.pdl@broadcom.com) + * + */ + +#include "mpi3mr.h" + +static void mpi3mr_expander_node_remove(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *sas_expander); + +/** + * mpi3mr_post_transport_req - Issue transport requests and wait + * @mrioc: Adapter instance reference + * @request: Properly populated MPI3 request + * @request_sz: Size of the MPI3 request + * @reply: Pointer to return MPI3 reply + * @reply_sz: Size of the MPI3 reply buffer + * @timeout: Timeout in seconds + * @ioc_status: Pointer to return ioc status + * + * A generic function for posting MPI3 requests from the SAS + * transport layer that uses transport command infrastructure. + * This blocks for the completion of request for timeout seconds + * and if the request times out this function faults the + * controller with proper reason code. + * + * On successful completion of the request this function returns + * appropriate ioc status from the firmware back to the caller. + * + * Return: 0 on success, non-zero on failure. + */ +static int mpi3mr_post_transport_req(struct mpi3mr_ioc *mrioc, void *request, + u16 request_sz, void *reply, u16 reply_sz, int timeout, + u16 *ioc_status) +{ + int retval = 0; + + mutex_lock(&mrioc->transport_cmds.mutex); + if (mrioc->transport_cmds.state & MPI3MR_CMD_PENDING) { + retval = -1; + ioc_err(mrioc, "sending transport request failed due to command in use\n"); + mutex_unlock(&mrioc->transport_cmds.mutex); + goto out; + } + mrioc->transport_cmds.state = MPI3MR_CMD_PENDING; + mrioc->transport_cmds.is_waiting = 1; + mrioc->transport_cmds.callback = NULL; + mrioc->transport_cmds.ioc_status = 0; + mrioc->transport_cmds.ioc_loginfo = 0; + + init_completion(&mrioc->transport_cmds.done); + dprint_cfg_info(mrioc, "posting transport request\n"); + if (mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO) + dprint_dump(request, request_sz, "transport_req"); + retval = mpi3mr_admin_request_post(mrioc, request, request_sz, 1); + if (retval) { + ioc_err(mrioc, "posting transport request failed\n"); + goto out_unlock; + } + wait_for_completion_timeout(&mrioc->transport_cmds.done, + (timeout * HZ)); + if (!(mrioc->transport_cmds.state & MPI3MR_CMD_COMPLETE)) { + mpi3mr_check_rh_fault_ioc(mrioc, + MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT); + ioc_err(mrioc, "transport request timed out\n"); + retval = -1; + goto out_unlock; + } + *ioc_status = mrioc->transport_cmds.ioc_status & + MPI3_IOCSTATUS_STATUS_MASK; + if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS) + dprint_transport_err(mrioc, + "transport request returned with ioc_status(0x%04x), log_info(0x%08x)\n", + *ioc_status, mrioc->transport_cmds.ioc_loginfo); + + if ((reply) && (mrioc->transport_cmds.state & MPI3MR_CMD_REPLY_VALID)) + memcpy((u8 *)reply, mrioc->transport_cmds.reply, reply_sz); + +out_unlock: + mrioc->transport_cmds.state = MPI3MR_CMD_NOTUSED; + mutex_unlock(&mrioc->transport_cmds.mutex); + +out: + return retval; +} + +/* report manufacture request structure */ +struct rep_manu_request { + u8 smp_frame_type; + u8 function; + u8 reserved; + u8 request_length; +}; + +/* report manufacture reply structure */ +struct rep_manu_reply { + u8 smp_frame_type; /* 0x41 */ + u8 function; /* 0x01 */ + u8 function_result; + u8 response_length; + u16 expander_change_count; + u8 reserved0[2]; + u8 sas_format; + u8 reserved2[3]; + u8 vendor_id[SAS_EXPANDER_VENDOR_ID_LEN]; + u8 product_id[SAS_EXPANDER_PRODUCT_ID_LEN]; + u8 product_rev[SAS_EXPANDER_PRODUCT_REV_LEN]; + u8 component_vendor_id[SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN]; + u16 component_id; + u8 component_revision_id; + u8 reserved3; + u8 vendor_specific[8]; +}; + +/** + * mpi3mr_report_manufacture - obtain SMP report_manufacture + * @mrioc: Adapter instance reference + * @sas_address: SAS address of the expander device + * @edev: SAS transport layer sas_expander_device object + * @port_id: ID of the HBA port + * + * Fills in the sas_expander_device with manufacturing info. + * + * Return: 0 for success, non-zero for failure. + */ +static int mpi3mr_report_manufacture(struct mpi3mr_ioc *mrioc, + u64 sas_address, struct sas_expander_device *edev, u8 port_id) +{ + struct mpi3_smp_passthrough_request mpi_request; + struct mpi3_smp_passthrough_reply mpi_reply; + struct rep_manu_reply *manufacture_reply; + struct rep_manu_request *manufacture_request; + int rc = 0; + void *psge; + void *data_out = NULL; + dma_addr_t data_out_dma; + dma_addr_t data_in_dma; + size_t data_in_sz; + size_t data_out_sz; + u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST; + u16 request_sz = sizeof(struct mpi3_smp_passthrough_request); + u16 reply_sz = sizeof(struct mpi3_smp_passthrough_reply); + u16 ioc_status; + u8 *tmp; + + if (mrioc->reset_in_progress) { + ioc_err(mrioc, "%s: host reset in progress!\n", __func__); + return -EFAULT; + } + + data_out_sz = sizeof(struct rep_manu_request); + data_in_sz = sizeof(struct rep_manu_reply); + data_out = dma_alloc_coherent(&mrioc->pdev->dev, + data_out_sz + data_in_sz, &data_out_dma, GFP_KERNEL); + if (!data_out) { + rc = -ENOMEM; + goto out; + } + + data_in_dma = data_out_dma + data_out_sz; + manufacture_reply = data_out + data_out_sz; + + manufacture_request = data_out; + manufacture_request->smp_frame_type = 0x40; + manufacture_request->function = 1; + manufacture_request->reserved = 0; + manufacture_request->request_length = 0; + + memset(&mpi_request, 0, request_sz); + memset(&mpi_reply, 0, reply_sz); + mpi_request.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_TRANSPORT_CMDS); + mpi_request.function = MPI3_FUNCTION_SMP_PASSTHROUGH; + mpi_request.io_unit_port = (u8) port_id; + mpi_request.sas_address = cpu_to_le64(sas_address); + + psge = &mpi_request.request_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_out_sz, data_out_dma); + + psge = &mpi_request.response_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_in_sz, data_in_dma); + + dprint_transport_info(mrioc, + "sending report manufacturer SMP request to sas_address(0x%016llx), port(%d)\n", + (unsigned long long)sas_address, port_id); + + rc = mpi3mr_post_transport_req(mrioc, &mpi_request, request_sz, + &mpi_reply, reply_sz, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status); + if (rc) + goto out; + + dprint_transport_info(mrioc, + "report manufacturer SMP request completed with ioc_status(0x%04x)\n", + ioc_status); + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + rc = -EINVAL; + goto out; + } + + dprint_transport_info(mrioc, + "report manufacturer - reply data transfer size(%d)\n", + le16_to_cpu(mpi_reply.response_data_length)); + + if (le16_to_cpu(mpi_reply.response_data_length) != + sizeof(struct rep_manu_reply)) { + rc = -EINVAL; + goto out; + } + + strscpy(edev->vendor_id, manufacture_reply->vendor_id, + SAS_EXPANDER_VENDOR_ID_LEN); + strscpy(edev->product_id, manufacture_reply->product_id, + SAS_EXPANDER_PRODUCT_ID_LEN); + strscpy(edev->product_rev, manufacture_reply->product_rev, + SAS_EXPANDER_PRODUCT_REV_LEN); + edev->level = manufacture_reply->sas_format & 1; + if (edev->level) { + strscpy(edev->component_vendor_id, + manufacture_reply->component_vendor_id, + SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN); + tmp = (u8 *)&manufacture_reply->component_id; + edev->component_id = tmp[0] << 8 | tmp[1]; + edev->component_revision_id = + manufacture_reply->component_revision_id; + } + +out: + if (data_out) + dma_free_coherent(&mrioc->pdev->dev, data_out_sz + data_in_sz, + data_out, data_out_dma); + + return rc; +} + +/** + * __mpi3mr_expander_find_by_handle - expander search by handle + * @mrioc: Adapter instance reference + * @handle: Firmware device handle of the expander + * + * Context: The caller should acquire sas_node_lock + * + * This searches for expander device based on handle, then + * returns the sas_node object. + * + * Return: Expander sas_node object reference or NULL + */ +struct mpi3mr_sas_node *__mpi3mr_expander_find_by_handle(struct mpi3mr_ioc + *mrioc, u16 handle) +{ + struct mpi3mr_sas_node *sas_expander, *r; + + r = NULL; + list_for_each_entry(sas_expander, &mrioc->sas_expander_list, list) { + if (sas_expander->handle != handle) + continue; + r = sas_expander; + goto out; + } + out: + return r; +} + +/** + * mpi3mr_is_expander_device - if device is an expander + * @device_info: Bitfield providing information about the device + * + * Return: 1 if the device is expander device, else 0. + */ +u8 mpi3mr_is_expander_device(u16 device_info) +{ + if ((device_info & MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_MASK) == + MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_EXPANDER) + return 1; + else + return 0; +} + +/** + * mpi3mr_get_sas_address - retrieve sas_address for handle + * @mrioc: Adapter instance reference + * @handle: Firmware device handle + * @sas_address: Address to hold sas address + * + * This function issues device page0 read for a given device + * handle and gets the SAS address and return it back + * + * Return: 0 for success, non-zero for failure + */ +static int mpi3mr_get_sas_address(struct mpi3mr_ioc *mrioc, u16 handle, + u64 *sas_address) +{ + struct mpi3_device_page0 dev_pg0; + u16 ioc_status; + struct mpi3_device0_sas_sata_format *sasinf; + + *sas_address = 0; + + if ((mpi3mr_cfg_get_dev_pg0(mrioc, &ioc_status, &dev_pg0, + sizeof(dev_pg0), MPI3_DEVICE_PGAD_FORM_HANDLE, + handle))) { + ioc_err(mrioc, "%s: device page0 read failed\n", __func__); + return -ENXIO; + } + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "device page read failed for handle(0x%04x), with ioc_status(0x%04x) failure at %s:%d/%s()!\n", + handle, ioc_status, __FILE__, __LINE__, __func__); + return -ENXIO; + } + + if (le16_to_cpu(dev_pg0.flags) & + MPI3_DEVICE0_FLAGS_CONTROLLER_DEV_HANDLE) + *sas_address = mrioc->sas_hba.sas_address; + else if (dev_pg0.device_form == MPI3_DEVICE_DEVFORM_SAS_SATA) { + sasinf = &dev_pg0.device_specific.sas_sata_format; + *sas_address = le64_to_cpu(sasinf->sas_address); + } else { + ioc_err(mrioc, "%s: device_form(%d) is not SAS_SATA\n", + __func__, dev_pg0.device_form); + return -ENXIO; + } + return 0; +} + +/** + * __mpi3mr_get_tgtdev_by_addr - target device search + * @mrioc: Adapter instance reference + * @sas_address: SAS address of the device + * @hba_port: HBA port entry + * + * This searches for target device from sas address and hba port + * pointer then return mpi3mr_tgt_dev object. + * + * Return: Valid tget_dev or NULL + */ +static struct mpi3mr_tgt_dev *__mpi3mr_get_tgtdev_by_addr(struct mpi3mr_ioc *mrioc, + u64 sas_address, struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_tgt_dev *tgtdev; + + assert_spin_locked(&mrioc->tgtdev_lock); + + list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) + if ((tgtdev->dev_type == MPI3_DEVICE_DEVFORM_SAS_SATA) && + (tgtdev->dev_spec.sas_sata_inf.sas_address == sas_address) + && (tgtdev->dev_spec.sas_sata_inf.hba_port == hba_port)) + goto found_device; + return NULL; +found_device: + mpi3mr_tgtdev_get(tgtdev); + return tgtdev; +} + +/** + * mpi3mr_get_tgtdev_by_addr - target device search + * @mrioc: Adapter instance reference + * @sas_address: SAS address of the device + * @hba_port: HBA port entry + * + * This searches for target device from sas address and hba port + * pointer then return mpi3mr_tgt_dev object. + * + * Context: This function will acquire tgtdev_lock and will + * release before returning the mpi3mr_tgt_dev object. + * + * Return: Valid tget_dev or NULL + */ +static struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_addr(struct mpi3mr_ioc *mrioc, + u64 sas_address, struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_tgt_dev *tgtdev = NULL; + unsigned long flags; + + if (!hba_port) + goto out; + + spin_lock_irqsave(&mrioc->tgtdev_lock, flags); + tgtdev = __mpi3mr_get_tgtdev_by_addr(mrioc, sas_address, hba_port); + spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); + +out: + return tgtdev; +} + +/** + * mpi3mr_remove_device_by_sas_address - remove the device + * @mrioc: Adapter instance reference + * @sas_address: SAS address of the device + * @hba_port: HBA port entry + * + * This searches for target device using sas address and hba + * port pointer then removes it from the OS. + * + * Return: None + */ +static void mpi3mr_remove_device_by_sas_address(struct mpi3mr_ioc *mrioc, + u64 sas_address, struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_tgt_dev *tgtdev = NULL; + unsigned long flags; + u8 was_on_tgtdev_list = 0; + + if (!hba_port) + return; + + spin_lock_irqsave(&mrioc->tgtdev_lock, flags); + tgtdev = __mpi3mr_get_tgtdev_by_addr(mrioc, + sas_address, hba_port); + if (tgtdev) { + if (!list_empty(&tgtdev->list)) { + list_del_init(&tgtdev->list); + was_on_tgtdev_list = 1; + mpi3mr_tgtdev_put(tgtdev); + } + } + spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); + if (was_on_tgtdev_list) { + if (tgtdev->host_exposed) + mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev); + mpi3mr_tgtdev_put(tgtdev); + } +} + +/** + * __mpi3mr_get_tgtdev_by_addr_and_rphy - target device search + * @mrioc: Adapter instance reference + * @sas_address: SAS address of the device + * @rphy: SAS transport layer rphy object + * + * This searches for target device from sas address and rphy + * pointer then return mpi3mr_tgt_dev object. + * + * Return: Valid tget_dev or NULL + */ +struct mpi3mr_tgt_dev *__mpi3mr_get_tgtdev_by_addr_and_rphy( + struct mpi3mr_ioc *mrioc, u64 sas_address, struct sas_rphy *rphy) +{ + struct mpi3mr_tgt_dev *tgtdev; + + assert_spin_locked(&mrioc->tgtdev_lock); + + list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) + if ((tgtdev->dev_type == MPI3_DEVICE_DEVFORM_SAS_SATA) && + (tgtdev->dev_spec.sas_sata_inf.sas_address == sas_address) + && (tgtdev->dev_spec.sas_sata_inf.rphy == rphy)) + goto found_device; + return NULL; +found_device: + mpi3mr_tgtdev_get(tgtdev); + return tgtdev; +} + +/** + * mpi3mr_expander_find_by_sas_address - sas expander search + * @mrioc: Adapter instance reference + * @sas_address: SAS address of expander + * @hba_port: HBA port entry + * + * Return: A valid SAS expander node or NULL. + * + */ +static struct mpi3mr_sas_node *mpi3mr_expander_find_by_sas_address( + struct mpi3mr_ioc *mrioc, u64 sas_address, + struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_sas_node *sas_expander, *r = NULL; + + if (!hba_port) + goto out; + + list_for_each_entry(sas_expander, &mrioc->sas_expander_list, list) { + if ((sas_expander->sas_address != sas_address) || + (sas_expander->hba_port != hba_port)) + continue; + r = sas_expander; + goto out; + } +out: + return r; +} + +/** + * __mpi3mr_sas_node_find_by_sas_address - sas node search + * @mrioc: Adapter instance reference + * @sas_address: SAS address of expander or sas host + * @hba_port: HBA port entry + * Context: Caller should acquire mrioc->sas_node_lock. + * + * If the SAS address indicates the device is direct attached to + * the controller (controller's SAS address) then the SAS node + * associated with the controller is returned back else the SAS + * address and hba port are used to identify the exact expander + * and the associated sas_node object is returned. If there is + * no match NULL is returned. + * + * Return: A valid SAS node or NULL. + * + */ +static struct mpi3mr_sas_node *__mpi3mr_sas_node_find_by_sas_address( + struct mpi3mr_ioc *mrioc, u64 sas_address, + struct mpi3mr_hba_port *hba_port) +{ + + if (mrioc->sas_hba.sas_address == sas_address) + return &mrioc->sas_hba; + return mpi3mr_expander_find_by_sas_address(mrioc, sas_address, + hba_port); +} + +/** + * mpi3mr_parent_present - Is parent present for a phy + * @mrioc: Adapter instance reference + * @phy: SAS transport layer phy object + * + * Return: 0 if parent is present else non-zero + */ +static int mpi3mr_parent_present(struct mpi3mr_ioc *mrioc, struct sas_phy *phy) +{ + unsigned long flags; + struct mpi3mr_hba_port *hba_port = phy->hostdata; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + if (__mpi3mr_sas_node_find_by_sas_address(mrioc, + phy->identify.sas_address, + hba_port) == NULL) { + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + return -1; + } + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + return 0; +} + +/** + * mpi3mr_convert_phy_link_rate - + * @link_rate: link rate as defined in the MPI header + * + * Convert link_rate from mpi format into sas_transport layer + * form. + * + * Return: A valid SAS transport layer defined link rate + */ +static enum sas_linkrate mpi3mr_convert_phy_link_rate(u8 link_rate) +{ + enum sas_linkrate rc; + + switch (link_rate) { + case MPI3_SAS_NEG_LINK_RATE_1_5: + rc = SAS_LINK_RATE_1_5_GBPS; + break; + case MPI3_SAS_NEG_LINK_RATE_3_0: + rc = SAS_LINK_RATE_3_0_GBPS; + break; + case MPI3_SAS_NEG_LINK_RATE_6_0: + rc = SAS_LINK_RATE_6_0_GBPS; + break; + case MPI3_SAS_NEG_LINK_RATE_12_0: + rc = SAS_LINK_RATE_12_0_GBPS; + break; + case MPI3_SAS_NEG_LINK_RATE_22_5: + rc = SAS_LINK_RATE_22_5_GBPS; + break; + case MPI3_SAS_NEG_LINK_RATE_PHY_DISABLED: + rc = SAS_PHY_DISABLED; + break; + case MPI3_SAS_NEG_LINK_RATE_NEGOTIATION_FAILED: + rc = SAS_LINK_RATE_FAILED; + break; + case MPI3_SAS_NEG_LINK_RATE_PORT_SELECTOR: + rc = SAS_SATA_PORT_SELECTOR; + break; + case MPI3_SAS_NEG_LINK_RATE_SMP_RESET_IN_PROGRESS: + rc = SAS_PHY_RESET_IN_PROGRESS; + break; + case MPI3_SAS_NEG_LINK_RATE_SATA_OOB_COMPLETE: + case MPI3_SAS_NEG_LINK_RATE_UNKNOWN_LINK_RATE: + default: + rc = SAS_LINK_RATE_UNKNOWN; + break; + } + return rc; +} + +/** + * mpi3mr_delete_sas_phy - Remove a single phy from port + * @mrioc: Adapter instance reference + * @mr_sas_port: Internal Port object + * @mr_sas_phy: Internal Phy object + * + * Return: None. + */ +static void mpi3mr_delete_sas_phy(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_port *mr_sas_port, + struct mpi3mr_sas_phy *mr_sas_phy) +{ + u64 sas_address = mr_sas_port->remote_identify.sas_address; + + dev_info(&mr_sas_phy->phy->dev, + "remove: sas_address(0x%016llx), phy(%d)\n", + (unsigned long long) sas_address, mr_sas_phy->phy_id); + + list_del(&mr_sas_phy->port_siblings); + mr_sas_port->num_phys--; + mr_sas_port->phy_mask &= ~(1 << mr_sas_phy->phy_id); + if (mr_sas_port->lowest_phy == mr_sas_phy->phy_id) + mr_sas_port->lowest_phy = ffs(mr_sas_port->phy_mask) - 1; + sas_port_delete_phy(mr_sas_port->port, mr_sas_phy->phy); + mr_sas_phy->phy_belongs_to_port = 0; +} + +/** + * mpi3mr_add_sas_phy - Adding a single phy to a port + * @mrioc: Adapter instance reference + * @mr_sas_port: Internal Port object + * @mr_sas_phy: Internal Phy object + * + * Return: None. + */ +static void mpi3mr_add_sas_phy(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_port *mr_sas_port, + struct mpi3mr_sas_phy *mr_sas_phy) +{ + u64 sas_address = mr_sas_port->remote_identify.sas_address; + + dev_info(&mr_sas_phy->phy->dev, + "add: sas_address(0x%016llx), phy(%d)\n", (unsigned long long) + sas_address, mr_sas_phy->phy_id); + + list_add_tail(&mr_sas_phy->port_siblings, &mr_sas_port->phy_list); + mr_sas_port->num_phys++; + mr_sas_port->phy_mask |= (1 << mr_sas_phy->phy_id); + if (mr_sas_phy->phy_id < mr_sas_port->lowest_phy) + mr_sas_port->lowest_phy = ffs(mr_sas_port->phy_mask) - 1; + sas_port_add_phy(mr_sas_port->port, mr_sas_phy->phy); + mr_sas_phy->phy_belongs_to_port = 1; +} + +/** + * mpi3mr_add_phy_to_an_existing_port - add phy to existing port + * @mrioc: Adapter instance reference + * @mr_sas_node: Internal sas node object (expander or host) + * @mr_sas_phy: Internal Phy object * + * @sas_address: SAS address of device/expander were phy needs + * to be added to + * @hba_port: HBA port entry + * + * Return: None. + */ +static void mpi3mr_add_phy_to_an_existing_port(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *mr_sas_node, struct mpi3mr_sas_phy *mr_sas_phy, + u64 sas_address, struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_sas_port *mr_sas_port; + struct mpi3mr_sas_phy *srch_phy; + + if (mr_sas_phy->phy_belongs_to_port == 1) + return; + + if (!hba_port) + return; + + list_for_each_entry(mr_sas_port, &mr_sas_node->sas_port_list, + port_list) { + if (mr_sas_port->remote_identify.sas_address != + sas_address) + continue; + if (mr_sas_port->hba_port != hba_port) + continue; + list_for_each_entry(srch_phy, &mr_sas_port->phy_list, + port_siblings) { + if (srch_phy == mr_sas_phy) + return; + } + mpi3mr_add_sas_phy(mrioc, mr_sas_port, mr_sas_phy); + return; + } +} + +/** + * mpi3mr_delete_sas_port - helper function to removing a port + * @mrioc: Adapter instance reference + * @mr_sas_port: Internal Port object + * + * Return: None. + */ +static void mpi3mr_delete_sas_port(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_port *mr_sas_port) +{ + u64 sas_address = mr_sas_port->remote_identify.sas_address; + struct mpi3mr_hba_port *hba_port = mr_sas_port->hba_port; + enum sas_device_type device_type = + mr_sas_port->remote_identify.device_type; + + dev_info(&mr_sas_port->port->dev, + "remove: sas_address(0x%016llx)\n", + (unsigned long long) sas_address); + + if (device_type == SAS_END_DEVICE) + mpi3mr_remove_device_by_sas_address(mrioc, sas_address, + hba_port); + + else if (device_type == SAS_EDGE_EXPANDER_DEVICE || + device_type == SAS_FANOUT_EXPANDER_DEVICE) + mpi3mr_expander_remove(mrioc, sas_address, hba_port); +} + +/** + * mpi3mr_del_phy_from_an_existing_port - del phy from a port + * @mrioc: Adapter instance reference + * @mr_sas_node: Internal sas node object (expander or host) + * @mr_sas_phy: Internal Phy object + * + * Return: None. + */ +static void mpi3mr_del_phy_from_an_existing_port(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *mr_sas_node, struct mpi3mr_sas_phy *mr_sas_phy) +{ + struct mpi3mr_sas_port *mr_sas_port, *next; + struct mpi3mr_sas_phy *srch_phy; + + if (mr_sas_phy->phy_belongs_to_port == 0) + return; + + list_for_each_entry_safe(mr_sas_port, next, &mr_sas_node->sas_port_list, + port_list) { + list_for_each_entry(srch_phy, &mr_sas_port->phy_list, + port_siblings) { + if (srch_phy != mr_sas_phy) + continue; + if ((mr_sas_port->num_phys == 1) && + !mrioc->reset_in_progress) + mpi3mr_delete_sas_port(mrioc, mr_sas_port); + else + mpi3mr_delete_sas_phy(mrioc, mr_sas_port, + mr_sas_phy); + return; + } + } +} + +/** + * mpi3mr_sas_port_sanity_check - sanity check while adding port + * @mrioc: Adapter instance reference + * @mr_sas_node: Internal sas node object (expander or host) + * @sas_address: SAS address of device/expander + * @hba_port: HBA port entry + * + * Verifies whether the Phys attached to a device with the given + * SAS address already belongs to an existing sas port if so + * will remove those phys from the sas port + * + * Return: None. + */ +static void mpi3mr_sas_port_sanity_check(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *mr_sas_node, u64 sas_address, + struct mpi3mr_hba_port *hba_port) +{ + int i; + + for (i = 0; i < mr_sas_node->num_phys; i++) { + if ((mr_sas_node->phy[i].remote_identify.sas_address != + sas_address) || (mr_sas_node->phy[i].hba_port != hba_port)) + continue; + if (mr_sas_node->phy[i].phy_belongs_to_port == 1) + mpi3mr_del_phy_from_an_existing_port(mrioc, + mr_sas_node, &mr_sas_node->phy[i]); + } +} + +/** + * mpi3mr_set_identify - set identify for phys and end devices + * @mrioc: Adapter instance reference + * @handle: Firmware device handle + * @identify: SAS transport layer's identify info + * + * Populates sas identify info for a specific device. + * + * Return: 0 for success, non-zero for failure. + */ +static int mpi3mr_set_identify(struct mpi3mr_ioc *mrioc, u16 handle, + struct sas_identify *identify) +{ + + struct mpi3_device_page0 device_pg0; + struct mpi3_device0_sas_sata_format *sasinf; + u16 device_info; + u16 ioc_status; + + if (mrioc->reset_in_progress) { + ioc_err(mrioc, "%s: host reset in progress!\n", __func__); + return -EFAULT; + } + + if ((mpi3mr_cfg_get_dev_pg0(mrioc, &ioc_status, &device_pg0, + sizeof(device_pg0), MPI3_DEVICE_PGAD_FORM_HANDLE, handle))) { + ioc_err(mrioc, "%s: device page0 read failed\n", __func__); + return -ENXIO; + } + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "device page read failed for handle(0x%04x), with ioc_status(0x%04x) failure at %s:%d/%s()!\n", + handle, ioc_status, __FILE__, __LINE__, __func__); + return -EIO; + } + + memset(identify, 0, sizeof(struct sas_identify)); + sasinf = &device_pg0.device_specific.sas_sata_format; + device_info = le16_to_cpu(sasinf->device_info); + + /* sas_address */ + identify->sas_address = le64_to_cpu(sasinf->sas_address); + + /* phy number of the parent device this device is linked to */ + identify->phy_identifier = sasinf->phy_num; + + /* device_type */ + switch (device_info & MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_MASK) { + case MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_NO_DEVICE: + identify->device_type = SAS_PHY_UNUSED; + break; + case MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_END_DEVICE: + identify->device_type = SAS_END_DEVICE; + break; + case MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_EXPANDER: + identify->device_type = SAS_EDGE_EXPANDER_DEVICE; + break; + } + + /* initiator_port_protocols */ + if (device_info & MPI3_SAS_DEVICE_INFO_SSP_INITIATOR) + identify->initiator_port_protocols |= SAS_PROTOCOL_SSP; + /* MPI3.0 doesn't have define for SATA INIT so setting both here*/ + if (device_info & MPI3_SAS_DEVICE_INFO_STP_INITIATOR) + identify->initiator_port_protocols |= (SAS_PROTOCOL_STP | + SAS_PROTOCOL_SATA); + if (device_info & MPI3_SAS_DEVICE_INFO_SMP_INITIATOR) + identify->initiator_port_protocols |= SAS_PROTOCOL_SMP; + + /* target_port_protocols */ + if (device_info & MPI3_SAS_DEVICE_INFO_SSP_TARGET) + identify->target_port_protocols |= SAS_PROTOCOL_SSP; + /* MPI3.0 doesn't have define for STP Target so setting both here*/ + if (device_info & MPI3_SAS_DEVICE_INFO_STP_SATA_TARGET) + identify->target_port_protocols |= (SAS_PROTOCOL_STP | + SAS_PROTOCOL_SATA); + if (device_info & MPI3_SAS_DEVICE_INFO_SMP_TARGET) + identify->target_port_protocols |= SAS_PROTOCOL_SMP; + return 0; +} + +/** + * mpi3mr_add_host_phy - report sas_host phy to SAS transport + * @mrioc: Adapter instance reference + * @mr_sas_phy: Internal Phy object + * @phy_pg0: SAS phy page 0 + * @parent_dev: Prent device class object + * + * Return: 0 for success, non-zero for failure. + */ +static int mpi3mr_add_host_phy(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_phy *mr_sas_phy, struct mpi3_sas_phy_page0 phy_pg0, + struct device *parent_dev) +{ + struct sas_phy *phy; + int phy_index = mr_sas_phy->phy_id; + + + INIT_LIST_HEAD(&mr_sas_phy->port_siblings); + phy = sas_phy_alloc(parent_dev, phy_index); + if (!phy) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + if ((mpi3mr_set_identify(mrioc, mr_sas_phy->handle, + &mr_sas_phy->identify))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + sas_phy_free(phy); + return -1; + } + phy->identify = mr_sas_phy->identify; + mr_sas_phy->attached_handle = le16_to_cpu(phy_pg0.attached_dev_handle); + if (mr_sas_phy->attached_handle) + mpi3mr_set_identify(mrioc, mr_sas_phy->attached_handle, + &mr_sas_phy->remote_identify); + phy->identify.phy_identifier = mr_sas_phy->phy_id; + phy->negotiated_linkrate = mpi3mr_convert_phy_link_rate( + (phy_pg0.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) >> + MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT); + phy->minimum_linkrate_hw = mpi3mr_convert_phy_link_rate( + phy_pg0.hw_link_rate & MPI3_SAS_HWRATE_MIN_RATE_MASK); + phy->maximum_linkrate_hw = mpi3mr_convert_phy_link_rate( + phy_pg0.hw_link_rate >> 4); + phy->minimum_linkrate = mpi3mr_convert_phy_link_rate( + phy_pg0.programmed_link_rate & MPI3_SAS_PRATE_MIN_RATE_MASK); + phy->maximum_linkrate = mpi3mr_convert_phy_link_rate( + phy_pg0.programmed_link_rate >> 4); + phy->hostdata = mr_sas_phy->hba_port; + + if ((sas_phy_add(phy))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + sas_phy_free(phy); + return -1; + } + if ((mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO)) + dev_info(&phy->dev, + "add: handle(0x%04x), sas_address(0x%016llx)\n" + "\tattached_handle(0x%04x), sas_address(0x%016llx)\n", + mr_sas_phy->handle, (unsigned long long) + mr_sas_phy->identify.sas_address, + mr_sas_phy->attached_handle, + (unsigned long long) + mr_sas_phy->remote_identify.sas_address); + mr_sas_phy->phy = phy; + return 0; +} + +/** + * mpi3mr_add_expander_phy - report expander phy to transport + * @mrioc: Adapter instance reference + * @mr_sas_phy: Internal Phy object + * @expander_pg1: SAS Expander page 1 + * @parent_dev: Parent device class object + * + * Return: 0 for success, non-zero for failure. + */ +static int mpi3mr_add_expander_phy(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_phy *mr_sas_phy, + struct mpi3_sas_expander_page1 expander_pg1, + struct device *parent_dev) +{ + struct sas_phy *phy; + int phy_index = mr_sas_phy->phy_id; + + INIT_LIST_HEAD(&mr_sas_phy->port_siblings); + phy = sas_phy_alloc(parent_dev, phy_index); + if (!phy) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + if ((mpi3mr_set_identify(mrioc, mr_sas_phy->handle, + &mr_sas_phy->identify))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + sas_phy_free(phy); + return -1; + } + phy->identify = mr_sas_phy->identify; + mr_sas_phy->attached_handle = + le16_to_cpu(expander_pg1.attached_dev_handle); + if (mr_sas_phy->attached_handle) + mpi3mr_set_identify(mrioc, mr_sas_phy->attached_handle, + &mr_sas_phy->remote_identify); + phy->identify.phy_identifier = mr_sas_phy->phy_id; + phy->negotiated_linkrate = mpi3mr_convert_phy_link_rate( + (expander_pg1.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) >> + MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT); + phy->minimum_linkrate_hw = mpi3mr_convert_phy_link_rate( + expander_pg1.hw_link_rate & MPI3_SAS_HWRATE_MIN_RATE_MASK); + phy->maximum_linkrate_hw = mpi3mr_convert_phy_link_rate( + expander_pg1.hw_link_rate >> 4); + phy->minimum_linkrate = mpi3mr_convert_phy_link_rate( + expander_pg1.programmed_link_rate & MPI3_SAS_PRATE_MIN_RATE_MASK); + phy->maximum_linkrate = mpi3mr_convert_phy_link_rate( + expander_pg1.programmed_link_rate >> 4); + phy->hostdata = mr_sas_phy->hba_port; + + if ((sas_phy_add(phy))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + sas_phy_free(phy); + return -1; + } + if ((mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO)) + dev_info(&phy->dev, + "add: handle(0x%04x), sas_address(0x%016llx)\n" + "\tattached_handle(0x%04x), sas_address(0x%016llx)\n", + mr_sas_phy->handle, (unsigned long long) + mr_sas_phy->identify.sas_address, + mr_sas_phy->attached_handle, + (unsigned long long) + mr_sas_phy->remote_identify.sas_address); + mr_sas_phy->phy = phy; + return 0; +} + +/** + * mpi3mr_alloc_hba_port - alloc hba port object + * @mrioc: Adapter instance reference + * @port_id: Port number + * + * Alloc memory for hba port object. + */ +static struct mpi3mr_hba_port * +mpi3mr_alloc_hba_port(struct mpi3mr_ioc *mrioc, u16 port_id) +{ + struct mpi3mr_hba_port *hba_port; + + hba_port = kzalloc(sizeof(struct mpi3mr_hba_port), + GFP_KERNEL); + if (!hba_port) + return NULL; + hba_port->port_id = port_id; + ioc_info(mrioc, "hba_port entry: %p, port: %d is added to hba_port list\n", + hba_port, hba_port->port_id); + list_add_tail(&hba_port->list, &mrioc->hba_port_table_list); + return hba_port; +} + +/** + * mpi3mr_get_hba_port_by_id - find hba port by id + * @mrioc: Adapter instance reference + * @port_id - Port ID to search + * + * Return: mpi3mr_hba_port reference for the matched port + */ + +struct mpi3mr_hba_port *mpi3mr_get_hba_port_by_id(struct mpi3mr_ioc *mrioc, + u8 port_id) +{ + struct mpi3mr_hba_port *port, *port_next; + + list_for_each_entry_safe(port, port_next, + &mrioc->hba_port_table_list, list) { + if (port->port_id != port_id) + continue; + if (port->flags & MPI3MR_HBA_PORT_FLAG_DIRTY) + continue; + return port; + } + + return NULL; +} + +/** + * mpi3mr_update_links - refreshing SAS phy link changes + * @mrioc: Adapter instance reference + * @sas_address_parent: SAS address of parent expander or host + * @handle: Firmware device handle of attached device + * @phy_number: Phy number + * @link_rate: New link rate + * @hba_port: HBA port entry + * + * Return: None. + */ +void mpi3mr_update_links(struct mpi3mr_ioc *mrioc, + u64 sas_address_parent, u16 handle, u8 phy_number, u8 link_rate, + struct mpi3mr_hba_port *hba_port) +{ + unsigned long flags; + struct mpi3mr_sas_node *mr_sas_node; + struct mpi3mr_sas_phy *mr_sas_phy; + + if (mrioc->reset_in_progress) + return; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + mr_sas_node = __mpi3mr_sas_node_find_by_sas_address(mrioc, + sas_address_parent, hba_port); + if (!mr_sas_node) { + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + return; + } + + mr_sas_phy = &mr_sas_node->phy[phy_number]; + mr_sas_phy->attached_handle = handle; + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + if (handle && (link_rate >= MPI3_SAS_NEG_LINK_RATE_1_5)) { + mpi3mr_set_identify(mrioc, handle, + &mr_sas_phy->remote_identify); + mpi3mr_add_phy_to_an_existing_port(mrioc, mr_sas_node, + mr_sas_phy, mr_sas_phy->remote_identify.sas_address, + hba_port); + } else + memset(&mr_sas_phy->remote_identify, 0, sizeof(struct + sas_identify)); + + if (mr_sas_phy->phy) + mr_sas_phy->phy->negotiated_linkrate = + mpi3mr_convert_phy_link_rate(link_rate); + + if ((mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO)) + dev_info(&mr_sas_phy->phy->dev, + "refresh: parent sas_address(0x%016llx),\n" + "\tlink_rate(0x%02x), phy(%d)\n" + "\tattached_handle(0x%04x), sas_address(0x%016llx)\n", + (unsigned long long)sas_address_parent, + link_rate, phy_number, handle, (unsigned long long) + mr_sas_phy->remote_identify.sas_address); +} + +/** + * mpi3mr_sas_host_refresh - refreshing sas host object contents + * @mrioc: Adapter instance reference + * + * This function refreshes the controllers phy information and + * updates the SAS transport layer with updated information, + * this is executed for each device addition or device info + * change events + * + * Return: None. + */ +void mpi3mr_sas_host_refresh(struct mpi3mr_ioc *mrioc) +{ + int i; + u8 link_rate; + u16 sz, port_id, attached_handle; + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0 = NULL; + + dprint_transport_info(mrioc, + "updating handles for sas_host(0x%016llx)\n", + (unsigned long long)mrioc->sas_hba.sas_address); + + sz = offsetof(struct mpi3_sas_io_unit_page0, phy_data) + + (mrioc->sas_hba.num_phys * + sizeof(struct mpi3_sas_io_unit0_phy_data)); + sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg0) + return; + if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + + mrioc->sas_hba.handle = 0; + for (i = 0; i < mrioc->sas_hba.num_phys; i++) { + if (sas_io_unit_pg0->phy_data[i].phy_flags & + (MPI3_SASIOUNIT0_PHYFLAGS_HOST_PHY | + MPI3_SASIOUNIT0_PHYFLAGS_VIRTUAL_PHY)) + continue; + link_rate = + sas_io_unit_pg0->phy_data[i].negotiated_link_rate >> 4; + if (!mrioc->sas_hba.handle) + mrioc->sas_hba.handle = le16_to_cpu( + sas_io_unit_pg0->phy_data[i].controller_dev_handle); + port_id = sas_io_unit_pg0->phy_data[i].io_unit_port; + if (!(mpi3mr_get_hba_port_by_id(mrioc, port_id))) + if (!mpi3mr_alloc_hba_port(mrioc, port_id)) + goto out; + + mrioc->sas_hba.phy[i].handle = mrioc->sas_hba.handle; + attached_handle = le16_to_cpu( + sas_io_unit_pg0->phy_data[i].attached_dev_handle); + if (attached_handle && link_rate < MPI3_SAS_NEG_LINK_RATE_1_5) + link_rate = MPI3_SAS_NEG_LINK_RATE_1_5; + mrioc->sas_hba.phy[i].hba_port = + mpi3mr_get_hba_port_by_id(mrioc, port_id); + mpi3mr_update_links(mrioc, mrioc->sas_hba.sas_address, + attached_handle, i, link_rate, + mrioc->sas_hba.phy[i].hba_port); + } + out: + kfree(sas_io_unit_pg0); +} + +/** + * mpi3mr_sas_host_add - create sas host object + * @mrioc: Adapter instance reference + * + * This function creates the controllers phy information and + * updates the SAS transport layer with updated information, + * this is executed for first device addition or device info + * change event. + * + * Return: None. + */ +void mpi3mr_sas_host_add(struct mpi3mr_ioc *mrioc) +{ + int i; + u16 sz, num_phys = 1, port_id, ioc_status; + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0 = NULL; + struct mpi3_sas_phy_page0 phy_pg0; + struct mpi3_device_page0 dev_pg0; + struct mpi3_enclosure_page0 encl_pg0; + struct mpi3_device0_sas_sata_format *sasinf; + + sz = offsetof(struct mpi3_sas_io_unit_page0, phy_data) + + (num_phys * sizeof(struct mpi3_sas_io_unit0_phy_data)); + sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg0) + return; + + if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + num_phys = sas_io_unit_pg0->num_phys; + kfree(sas_io_unit_pg0); + + mrioc->sas_hba.host_node = 1; + INIT_LIST_HEAD(&mrioc->sas_hba.sas_port_list); + mrioc->sas_hba.parent_dev = &mrioc->shost->shost_gendev; + mrioc->sas_hba.phy = kcalloc(num_phys, + sizeof(struct mpi3mr_sas_phy), GFP_KERNEL); + if (!mrioc->sas_hba.phy) + return; + + mrioc->sas_hba.num_phys = num_phys; + + sz = offsetof(struct mpi3_sas_io_unit_page0, phy_data) + + (num_phys * sizeof(struct mpi3_sas_io_unit0_phy_data)); + sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg0) + return; + + if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + + mrioc->sas_hba.handle = 0; + for (i = 0; i < mrioc->sas_hba.num_phys; i++) { + if (sas_io_unit_pg0->phy_data[i].phy_flags & + (MPI3_SASIOUNIT0_PHYFLAGS_HOST_PHY | + MPI3_SASIOUNIT0_PHYFLAGS_VIRTUAL_PHY)) + continue; + if (mpi3mr_cfg_get_sas_phy_pg0(mrioc, &ioc_status, &phy_pg0, + sizeof(struct mpi3_sas_phy_page0), + MPI3_SAS_PHY_PGAD_FORM_PHY_NUMBER, i)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + + if (!mrioc->sas_hba.handle) + mrioc->sas_hba.handle = le16_to_cpu( + sas_io_unit_pg0->phy_data[i].controller_dev_handle); + port_id = sas_io_unit_pg0->phy_data[i].io_unit_port; + + if (!(mpi3mr_get_hba_port_by_id(mrioc, port_id))) + if (!mpi3mr_alloc_hba_port(mrioc, port_id)) + goto out; + + mrioc->sas_hba.phy[i].handle = mrioc->sas_hba.handle; + mrioc->sas_hba.phy[i].phy_id = i; + mrioc->sas_hba.phy[i].hba_port = + mpi3mr_get_hba_port_by_id(mrioc, port_id); + mpi3mr_add_host_phy(mrioc, &mrioc->sas_hba.phy[i], + phy_pg0, mrioc->sas_hba.parent_dev); + } + if ((mpi3mr_cfg_get_dev_pg0(mrioc, &ioc_status, &dev_pg0, + sizeof(dev_pg0), MPI3_DEVICE_PGAD_FORM_HANDLE, + mrioc->sas_hba.handle))) { + ioc_err(mrioc, "%s: device page0 read failed\n", __func__); + goto out; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "device page read failed for handle(0x%04x), with ioc_status(0x%04x) failure at %s:%d/%s()!\n", + mrioc->sas_hba.handle, ioc_status, __FILE__, __LINE__, + __func__); + goto out; + } + mrioc->sas_hba.enclosure_handle = + le16_to_cpu(dev_pg0.enclosure_handle); + sasinf = &dev_pg0.device_specific.sas_sata_format; + mrioc->sas_hba.sas_address = + le64_to_cpu(sasinf->sas_address); + ioc_info(mrioc, + "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n", + mrioc->sas_hba.handle, + (unsigned long long) mrioc->sas_hba.sas_address, + mrioc->sas_hba.num_phys); + + if (mrioc->sas_hba.enclosure_handle) { + if (!(mpi3mr_cfg_get_enclosure_pg0(mrioc, &ioc_status, + &encl_pg0, sizeof(dev_pg0), + MPI3_ENCLOS_PGAD_FORM_HANDLE, + mrioc->sas_hba.enclosure_handle)) && + (ioc_status == MPI3_IOCSTATUS_SUCCESS)) + mrioc->sas_hba.enclosure_logical_id = + le64_to_cpu(encl_pg0.enclosure_logical_id); + } + +out: + kfree(sas_io_unit_pg0); +} + +/** + * mpi3mr_sas_port_add - Expose the SAS device to the SAS TL + * @mrioc: Adapter instance reference + * @handle: Firmware device handle of the attached device + * @sas_address_parent: sas address of parent expander or host + * @hba_port: HBA port entry + * + * This function creates a new sas port object for the given end + * device matching sas address and hba_port and adds it to the + * sas_node's sas_port_list and expose the attached sas device + * to the SAS transport layer through sas_rphy_add. + * + * Returns a valid mpi3mr_sas_port reference or NULL. + */ +static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc, + u16 handle, u64 sas_address_parent, struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_sas_phy *mr_sas_phy, *next; + struct mpi3mr_sas_port *mr_sas_port; + unsigned long flags; + struct mpi3mr_sas_node *mr_sas_node; + struct sas_rphy *rphy; + struct mpi3mr_tgt_dev *tgtdev = NULL; + int i; + struct sas_port *port; + + if (!hba_port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return NULL; + } + + mr_sas_port = kzalloc(sizeof(struct mpi3mr_sas_port), GFP_KERNEL); + if (!mr_sas_port) + return NULL; + + INIT_LIST_HEAD(&mr_sas_port->port_list); + INIT_LIST_HEAD(&mr_sas_port->phy_list); + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + mr_sas_node = __mpi3mr_sas_node_find_by_sas_address(mrioc, + sas_address_parent, hba_port); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + if (!mr_sas_node) { + ioc_err(mrioc, "%s:could not find parent sas_address(0x%016llx)!\n", + __func__, (unsigned long long)sas_address_parent); + goto out_fail; + } + + if ((mpi3mr_set_identify(mrioc, handle, + &mr_sas_port->remote_identify))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + + if (mr_sas_port->remote_identify.device_type == SAS_PHY_UNUSED) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + + mr_sas_port->hba_port = hba_port; + mpi3mr_sas_port_sanity_check(mrioc, mr_sas_node, + mr_sas_port->remote_identify.sas_address, hba_port); + + for (i = 0; i < mr_sas_node->num_phys; i++) { + if ((mr_sas_node->phy[i].remote_identify.sas_address != + mr_sas_port->remote_identify.sas_address) || + (mr_sas_node->phy[i].hba_port != hba_port)) + continue; + list_add_tail(&mr_sas_node->phy[i].port_siblings, + &mr_sas_port->phy_list); + mr_sas_port->num_phys++; + mr_sas_port->phy_mask |= (1 << i); + } + + if (!mr_sas_port->num_phys) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + + mr_sas_port->lowest_phy = ffs(mr_sas_port->phy_mask) - 1; + + if (mr_sas_port->remote_identify.device_type == SAS_END_DEVICE) { + tgtdev = mpi3mr_get_tgtdev_by_addr(mrioc, + mr_sas_port->remote_identify.sas_address, + mr_sas_port->hba_port); + + if (!tgtdev) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + tgtdev->dev_spec.sas_sata_inf.pend_sas_rphy_add = 1; + } + + if (!mr_sas_node->parent_dev) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + + port = sas_port_alloc_num(mr_sas_node->parent_dev); + if ((sas_port_add(port))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } + + list_for_each_entry(mr_sas_phy, &mr_sas_port->phy_list, + port_siblings) { + if ((mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO)) + dev_info(&port->dev, + "add: handle(0x%04x), sas_address(0x%016llx), phy(%d)\n", + handle, (unsigned long long) + mr_sas_port->remote_identify.sas_address, + mr_sas_phy->phy_id); + sas_port_add_phy(port, mr_sas_phy->phy); + mr_sas_phy->phy_belongs_to_port = 1; + mr_sas_phy->hba_port = hba_port; + } + + mr_sas_port->port = port; + if (mr_sas_port->remote_identify.device_type == SAS_END_DEVICE) { + rphy = sas_end_device_alloc(port); + tgtdev->dev_spec.sas_sata_inf.rphy = rphy; + } else { + rphy = sas_expander_alloc(port, + mr_sas_port->remote_identify.device_type); + } + rphy->identify = mr_sas_port->remote_identify; + + if (mrioc->current_event) + mrioc->current_event->pending_at_sml = 1; + + if ((sas_rphy_add(rphy))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + } + if (mr_sas_port->remote_identify.device_type == SAS_END_DEVICE) { + tgtdev->dev_spec.sas_sata_inf.pend_sas_rphy_add = 0; + tgtdev->dev_spec.sas_sata_inf.sas_transport_attached = 1; + mpi3mr_tgtdev_put(tgtdev); + } + + dev_info(&rphy->dev, + "%s: added: handle(0x%04x), sas_address(0x%016llx)\n", + __func__, handle, (unsigned long long) + mr_sas_port->remote_identify.sas_address); + + mr_sas_port->rphy = rphy; + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_add_tail(&mr_sas_port->port_list, &mr_sas_node->sas_port_list); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + if (mrioc->current_event) { + mrioc->current_event->pending_at_sml = 0; + if (mrioc->current_event->discard) + mpi3mr_print_device_event_notice(mrioc, true); + } + + /* fill in report manufacture */ + if (mr_sas_port->remote_identify.device_type == + SAS_EDGE_EXPANDER_DEVICE || + mr_sas_port->remote_identify.device_type == + SAS_FANOUT_EXPANDER_DEVICE) + mpi3mr_report_manufacture(mrioc, + mr_sas_port->remote_identify.sas_address, + rphy_to_expander_device(rphy), hba_port->port_id); + + return mr_sas_port; + + out_fail: + list_for_each_entry_safe(mr_sas_phy, next, &mr_sas_port->phy_list, + port_siblings) + list_del(&mr_sas_phy->port_siblings); + kfree(mr_sas_port); + return NULL; +} + +/** + * mpi3mr_sas_port_remove - remove port from the list + * @mrioc: Adapter instance reference + * @sas_address: SAS address of attached device + * @sas_address_parent: SAS address of parent expander or host + * @hba_port: HBA port entry + * + * Removing object and freeing associated memory from the + * sas_port_list. + * + * Return: None + */ +static void mpi3mr_sas_port_remove(struct mpi3mr_ioc *mrioc, u64 sas_address, + u64 sas_address_parent, struct mpi3mr_hba_port *hba_port) +{ + int i; + unsigned long flags; + struct mpi3mr_sas_port *mr_sas_port, *next; + struct mpi3mr_sas_node *mr_sas_node; + u8 found = 0; + struct mpi3mr_sas_phy *mr_sas_phy, *next_phy; + struct mpi3mr_hba_port *srch_port, *hba_port_next = NULL; + + if (!hba_port) + return; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + mr_sas_node = __mpi3mr_sas_node_find_by_sas_address(mrioc, + sas_address_parent, hba_port); + if (!mr_sas_node) { + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + return; + } + list_for_each_entry_safe(mr_sas_port, next, &mr_sas_node->sas_port_list, + port_list) { + if (mr_sas_port->remote_identify.sas_address != sas_address) + continue; + if (mr_sas_port->hba_port != hba_port) + continue; + found = 1; + list_del(&mr_sas_port->port_list); + goto out; + } + + out: + if (!found) { + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + return; + } + + if (mr_sas_node->host_node) { + list_for_each_entry_safe(srch_port, hba_port_next, + &mrioc->hba_port_table_list, list) { + if (srch_port != hba_port) + continue; + ioc_info(mrioc, + "removing hba_port entry: %p port: %d from hba_port list\n", + srch_port, srch_port->port_id); + list_del(&hba_port->list); + kfree(hba_port); + break; + } + } + + for (i = 0; i < mr_sas_node->num_phys; i++) { + if (mr_sas_node->phy[i].remote_identify.sas_address == + sas_address) + memset(&mr_sas_node->phy[i].remote_identify, 0, + sizeof(struct sas_identify)); + } + + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + if (mrioc->current_event) + mrioc->current_event->pending_at_sml = 1; + + list_for_each_entry_safe(mr_sas_phy, next_phy, + &mr_sas_port->phy_list, port_siblings) { + if ((mrioc->logging_level & MPI3_DEBUG_TRANSPORT_INFO)) + dev_info(&mr_sas_port->port->dev, + "remove: sas_address(0x%016llx), phy(%d)\n", + (unsigned long long) + mr_sas_port->remote_identify.sas_address, + mr_sas_phy->phy_id); + mr_sas_phy->phy_belongs_to_port = 0; + if (!mrioc->stop_drv_processing) + sas_port_delete_phy(mr_sas_port->port, + mr_sas_phy->phy); + list_del(&mr_sas_phy->port_siblings); + } + if (!mrioc->stop_drv_processing) + sas_port_delete(mr_sas_port->port); + ioc_info(mrioc, "%s: removed sas_address(0x%016llx)\n", + __func__, (unsigned long long)sas_address); + + if (mrioc->current_event) { + mrioc->current_event->pending_at_sml = 0; + if (mrioc->current_event->discard) + mpi3mr_print_device_event_notice(mrioc, false); + } + + kfree(mr_sas_port); +} + +/** + * struct host_port - host port details + * @sas_address: SAS Address of the attached device + * @phy_mask: phy mask of host port + * @handle: Device Handle of attached device + * @iounit_port_id: port ID + * @used: host port is already matched with sas port from sas_port_list + * @lowest_phy: lowest phy ID of host port + */ +struct host_port { + u64 sas_address; + u32 phy_mask; + u16 handle; + u8 iounit_port_id; + u8 used; + u8 lowest_phy; +}; + +/** + * mpi3mr_update_mr_sas_port - update sas port objects during reset + * @mrioc: Adapter instance reference + * @h_port: host_port object + * @mr_sas_port: sas_port objects which needs to be updated + * + * Update the port ID of sas port object. Also add the phys if new phys got + * added to current sas port and remove the phys if some phys are moved + * out of the current sas port. + * + * Return: Nothing. + */ +static void +mpi3mr_update_mr_sas_port(struct mpi3mr_ioc *mrioc, struct host_port *h_port, + struct mpi3mr_sas_port *mr_sas_port) +{ + struct mpi3mr_sas_phy *mr_sas_phy; + u32 phy_mask_xor; + u64 phys_to_be_added, phys_to_be_removed; + int i; + + h_port->used = 1; + mr_sas_port->marked_responding = 1; + + dev_info(&mr_sas_port->port->dev, + "sas_address(0x%016llx), old: port_id %d phy_mask 0x%x, new: port_id %d phy_mask:0x%x\n", + mr_sas_port->remote_identify.sas_address, + mr_sas_port->hba_port->port_id, mr_sas_port->phy_mask, + h_port->iounit_port_id, h_port->phy_mask); + + mr_sas_port->hba_port->port_id = h_port->iounit_port_id; + mr_sas_port->hba_port->flags &= ~MPI3MR_HBA_PORT_FLAG_DIRTY; + + /* Get the newly added phys bit map & removed phys bit map */ + phy_mask_xor = mr_sas_port->phy_mask ^ h_port->phy_mask; + phys_to_be_added = h_port->phy_mask & phy_mask_xor; + phys_to_be_removed = mr_sas_port->phy_mask & phy_mask_xor; + + /* + * Register these new phys to current mr_sas_port's port. + * if these phys are previously registered with another port + * then delete these phys from that port first. + */ + for_each_set_bit(i, (ulong *) &phys_to_be_added, BITS_PER_TYPE(u32)) { + mr_sas_phy = &mrioc->sas_hba.phy[i]; + if (mr_sas_phy->phy_belongs_to_port) + mpi3mr_del_phy_from_an_existing_port(mrioc, + &mrioc->sas_hba, mr_sas_phy); + mpi3mr_add_phy_to_an_existing_port(mrioc, + &mrioc->sas_hba, mr_sas_phy, + mr_sas_port->remote_identify.sas_address, + mr_sas_port->hba_port); + } + + /* Delete the phys which are not part of current mr_sas_port's port. */ + for_each_set_bit(i, (ulong *) &phys_to_be_removed, BITS_PER_TYPE(u32)) { + mr_sas_phy = &mrioc->sas_hba.phy[i]; + if (mr_sas_phy->phy_belongs_to_port) + mpi3mr_del_phy_from_an_existing_port(mrioc, + &mrioc->sas_hba, mr_sas_phy); + } +} + +/** + * mpi3mr_refresh_sas_ports - update host's sas ports during reset + * @mrioc: Adapter instance reference + * + * Update the host's sas ports during reset by checking whether + * sas ports are still intact or not. Add/remove phys if any hba + * phys are (moved in)/(moved out) of sas port. Also update + * io_unit_port if it got changed during reset. + * + * Return: Nothing. + */ +void +mpi3mr_refresh_sas_ports(struct mpi3mr_ioc *mrioc) +{ + struct host_port h_port[32]; + int i, j, found, host_port_count = 0, port_idx; + u16 sz, attached_handle, ioc_status; + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0 = NULL; + struct mpi3_device_page0 dev_pg0; + struct mpi3_device0_sas_sata_format *sasinf; + struct mpi3mr_sas_port *mr_sas_port; + + sz = offsetof(struct mpi3_sas_io_unit_page0, phy_data) + + (mrioc->sas_hba.num_phys * + sizeof(struct mpi3_sas_io_unit0_phy_data)); + sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg0) + return; + if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + + /* Create a new expander port table */ + for (i = 0; i < mrioc->sas_hba.num_phys; i++) { + attached_handle = le16_to_cpu( + sas_io_unit_pg0->phy_data[i].attached_dev_handle); + if (!attached_handle) + continue; + found = 0; + for (j = 0; j < host_port_count; j++) { + if (h_port[j].handle == attached_handle) { + h_port[j].phy_mask |= (1 << i); + found = 1; + break; + } + } + if (found) + continue; + if ((mpi3mr_cfg_get_dev_pg0(mrioc, &ioc_status, &dev_pg0, + sizeof(dev_pg0), MPI3_DEVICE_PGAD_FORM_HANDLE, + attached_handle))) { + dprint_reset(mrioc, + "failed to read dev_pg0 for handle(0x%04x) at %s:%d/%s()!\n", + attached_handle, __FILE__, __LINE__, __func__); + continue; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + dprint_reset(mrioc, + "ioc_status(0x%x) while reading dev_pg0 for handle(0x%04x) at %s:%d/%s()!\n", + ioc_status, attached_handle, + __FILE__, __LINE__, __func__); + continue; + } + sasinf = &dev_pg0.device_specific.sas_sata_format; + + port_idx = host_port_count; + h_port[port_idx].sas_address = le64_to_cpu(sasinf->sas_address); + h_port[port_idx].handle = attached_handle; + h_port[port_idx].phy_mask = (1 << i); + h_port[port_idx].iounit_port_id = sas_io_unit_pg0->phy_data[i].io_unit_port; + h_port[port_idx].lowest_phy = sasinf->phy_num; + h_port[port_idx].used = 0; + host_port_count++; + } + + if (!host_port_count) + goto out; + + if (mrioc->logging_level & MPI3_DEBUG_RESET) { + ioc_info(mrioc, "Host port details before reset\n"); + list_for_each_entry(mr_sas_port, &mrioc->sas_hba.sas_port_list, + port_list) { + ioc_info(mrioc, + "port_id:%d, sas_address:(0x%016llx), phy_mask:(0x%x), lowest phy id:%d\n", + mr_sas_port->hba_port->port_id, + mr_sas_port->remote_identify.sas_address, + mr_sas_port->phy_mask, mr_sas_port->lowest_phy); + } + mr_sas_port = NULL; + ioc_info(mrioc, "Host port details after reset\n"); + for (i = 0; i < host_port_count; i++) { + ioc_info(mrioc, + "port_id:%d, sas_address:(0x%016llx), phy_mask:(0x%x), lowest phy id:%d\n", + h_port[i].iounit_port_id, h_port[i].sas_address, + h_port[i].phy_mask, h_port[i].lowest_phy); + } + } + + /* mark all host sas port entries as dirty */ + list_for_each_entry(mr_sas_port, &mrioc->sas_hba.sas_port_list, + port_list) { + mr_sas_port->marked_responding = 0; + mr_sas_port->hba_port->flags |= MPI3MR_HBA_PORT_FLAG_DIRTY; + } + + /* First check for matching lowest phy */ + for (i = 0; i < host_port_count; i++) { + mr_sas_port = NULL; + list_for_each_entry(mr_sas_port, &mrioc->sas_hba.sas_port_list, + port_list) { + if (mr_sas_port->marked_responding) + continue; + if (h_port[i].sas_address != mr_sas_port->remote_identify.sas_address) + continue; + if (h_port[i].lowest_phy == mr_sas_port->lowest_phy) { + mpi3mr_update_mr_sas_port(mrioc, &h_port[i], mr_sas_port); + break; + } + } + } + + /* In case if lowest phy is got enabled or disabled during reset */ + for (i = 0; i < host_port_count; i++) { + if (h_port[i].used) + continue; + mr_sas_port = NULL; + list_for_each_entry(mr_sas_port, &mrioc->sas_hba.sas_port_list, + port_list) { + if (mr_sas_port->marked_responding) + continue; + if (h_port[i].sas_address != mr_sas_port->remote_identify.sas_address) + continue; + if (h_port[i].phy_mask & mr_sas_port->phy_mask) { + mpi3mr_update_mr_sas_port(mrioc, &h_port[i], mr_sas_port); + break; + } + } + } + + /* In case if expander cable is removed & connected to another HBA port during reset */ + for (i = 0; i < host_port_count; i++) { + if (h_port[i].used) + continue; + mr_sas_port = NULL; + list_for_each_entry(mr_sas_port, &mrioc->sas_hba.sas_port_list, + port_list) { + if (mr_sas_port->marked_responding) + continue; + if (h_port[i].sas_address != mr_sas_port->remote_identify.sas_address) + continue; + mpi3mr_update_mr_sas_port(mrioc, &h_port[i], mr_sas_port); + break; + } + } +out: + kfree(sas_io_unit_pg0); +} + +/** + * mpi3mr_refresh_expanders - Refresh expander device exposure + * @mrioc: Adapter instance reference + * + * This is executed post controller reset to identify any + * missing expander devices during reset and remove from the upper layers + * or expose any newly detected expander device to the upper layers. + * + * Return: Nothing. + */ +void +mpi3mr_refresh_expanders(struct mpi3mr_ioc *mrioc) +{ + struct mpi3mr_sas_node *sas_expander, *sas_expander_next; + struct mpi3_sas_expander_page0 expander_pg0; + u16 ioc_status, handle; + u64 sas_address; + int i; + unsigned long flags; + struct mpi3mr_hba_port *hba_port; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_for_each_entry(sas_expander, &mrioc->sas_expander_list, list) { + sas_expander->non_responding = 1; + } + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + sas_expander = NULL; + + handle = 0xffff; + + /* Search for responding expander devices and add them if they are newly got added */ + while (true) { + if ((mpi3mr_cfg_get_sas_exp_pg0(mrioc, &ioc_status, &expander_pg0, + sizeof(struct mpi3_sas_expander_page0), + MPI3_SAS_EXPAND_PGAD_FORM_GET_NEXT_HANDLE, handle))) { + dprint_reset(mrioc, + "failed to read exp pg0 for handle(0x%04x) at %s:%d/%s()!\n", + handle, __FILE__, __LINE__, __func__); + break; + } + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + dprint_reset(mrioc, + "ioc_status(0x%x) while reading exp pg0 for handle:(0x%04x), %s:%d/%s()!\n", + ioc_status, handle, __FILE__, __LINE__, __func__); + break; + } + + handle = le16_to_cpu(expander_pg0.dev_handle); + sas_address = le64_to_cpu(expander_pg0.sas_address); + hba_port = mpi3mr_get_hba_port_by_id(mrioc, expander_pg0.io_unit_port); + + if (!hba_port) { + mpi3mr_sas_host_refresh(mrioc); + mpi3mr_expander_add(mrioc, handle); + continue; + } + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + sas_expander = + mpi3mr_expander_find_by_sas_address(mrioc, + sas_address, hba_port); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + if (!sas_expander) { + mpi3mr_sas_host_refresh(mrioc); + mpi3mr_expander_add(mrioc, handle); + continue; + } + + sas_expander->non_responding = 0; + if (sas_expander->handle == handle) + continue; + + sas_expander->handle = handle; + for (i = 0 ; i < sas_expander->num_phys ; i++) + sas_expander->phy[i].handle = handle; + } + + /* + * Delete non responding expander devices and the corresponding + * hba_port if the non responding expander device's parent device + * is a host node. + */ + sas_expander = NULL; + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_for_each_entry_safe_reverse(sas_expander, sas_expander_next, + &mrioc->sas_expander_list, list) { + if (sas_expander->non_responding) { + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + mpi3mr_expander_node_remove(mrioc, sas_expander); + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + } + } + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); +} + +/** + * mpi3mr_expander_node_add - insert an expander to the list. + * @mrioc: Adapter instance reference + * @sas_expander: Expander sas node + * Context: This function will acquire sas_node_lock. + * + * Adding new object to the ioc->sas_expander_list. + * + * Return: None. + */ +static void mpi3mr_expander_node_add(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *sas_expander) +{ + unsigned long flags; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_add_tail(&sas_expander->list, &mrioc->sas_expander_list); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); +} + +/** + * mpi3mr_expander_add - Create expander object + * @mrioc: Adapter instance reference + * @handle: Expander firmware device handle + * + * This function creating expander object, stored in + * sas_expander_list and expose it to the SAS transport + * layer. + * + * Return: 0 for success, non-zero for failure. + */ +int mpi3mr_expander_add(struct mpi3mr_ioc *mrioc, u16 handle) +{ + struct mpi3mr_sas_node *sas_expander; + struct mpi3mr_enclosure_node *enclosure_dev; + struct mpi3_sas_expander_page0 expander_pg0; + struct mpi3_sas_expander_page1 expander_pg1; + u16 ioc_status, parent_handle, temp_handle; + u64 sas_address, sas_address_parent = 0; + int i; + unsigned long flags; + u8 port_id, link_rate; + struct mpi3mr_sas_port *mr_sas_port = NULL; + struct mpi3mr_hba_port *hba_port; + u32 phynum_handle; + int rc = 0; + + if (!handle) + return -1; + + if (mrioc->reset_in_progress) + return -1; + + if ((mpi3mr_cfg_get_sas_exp_pg0(mrioc, &ioc_status, &expander_pg0, + sizeof(expander_pg0), MPI3_SAS_EXPAND_PGAD_FORM_HANDLE, handle))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + + parent_handle = le16_to_cpu(expander_pg0.parent_dev_handle); + if (mpi3mr_get_sas_address(mrioc, parent_handle, &sas_address_parent) + != 0) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + + port_id = expander_pg0.io_unit_port; + hba_port = mpi3mr_get_hba_port_by_id(mrioc, port_id); + if (!hba_port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + + if (sas_address_parent != mrioc->sas_hba.sas_address) { + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + sas_expander = + mpi3mr_expander_find_by_sas_address(mrioc, + sas_address_parent, hba_port); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + if (!sas_expander) { + rc = mpi3mr_expander_add(mrioc, parent_handle); + if (rc != 0) + return rc; + } else { + /* + * When there is a parent expander present, update it's + * phys where child expander is connected with the link + * speed, attached dev handle and sas address. + */ + for (i = 0 ; i < sas_expander->num_phys ; i++) { + phynum_handle = + (i << MPI3_SAS_EXPAND_PGAD_PHYNUM_SHIFT) | + parent_handle; + if (mpi3mr_cfg_get_sas_exp_pg1(mrioc, + &ioc_status, &expander_pg1, + sizeof(expander_pg1), + MPI3_SAS_EXPAND_PGAD_FORM_HANDLE_PHY_NUM, + phynum_handle)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + return rc; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + return rc; + } + temp_handle = le16_to_cpu( + expander_pg1.attached_dev_handle); + if (temp_handle != handle) + continue; + link_rate = (expander_pg1.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) >> + MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT; + mpi3mr_update_links(mrioc, sas_address_parent, + handle, i, link_rate, hba_port); + } + } + } + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + sas_address = le64_to_cpu(expander_pg0.sas_address); + sas_expander = mpi3mr_expander_find_by_sas_address(mrioc, + sas_address, hba_port); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + if (sas_expander) + return 0; + + sas_expander = kzalloc(sizeof(struct mpi3mr_sas_node), + GFP_KERNEL); + if (!sas_expander) + return -1; + + sas_expander->handle = handle; + sas_expander->num_phys = expander_pg0.num_phys; + sas_expander->sas_address_parent = sas_address_parent; + sas_expander->sas_address = sas_address; + sas_expander->hba_port = hba_port; + + ioc_info(mrioc, + "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n", + handle, parent_handle, (unsigned long long) + sas_expander->sas_address, sas_expander->num_phys); + + if (!sas_expander->num_phys) { + rc = -1; + goto out_fail; + } + sas_expander->phy = kcalloc(sas_expander->num_phys, + sizeof(struct mpi3mr_sas_phy), GFP_KERNEL); + if (!sas_expander->phy) { + rc = -1; + goto out_fail; + } + + INIT_LIST_HEAD(&sas_expander->sas_port_list); + mr_sas_port = mpi3mr_sas_port_add(mrioc, handle, sas_address_parent, + sas_expander->hba_port); + if (!mr_sas_port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + goto out_fail; + } + sas_expander->parent_dev = &mr_sas_port->rphy->dev; + sas_expander->rphy = mr_sas_port->rphy; + + for (i = 0 ; i < sas_expander->num_phys ; i++) { + phynum_handle = (i << MPI3_SAS_EXPAND_PGAD_PHYNUM_SHIFT) | + handle; + if (mpi3mr_cfg_get_sas_exp_pg1(mrioc, &ioc_status, + &expander_pg1, sizeof(expander_pg1), + MPI3_SAS_EXPAND_PGAD_FORM_HANDLE_PHY_NUM, + phynum_handle)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + goto out_fail; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + goto out_fail; + } + + sas_expander->phy[i].handle = handle; + sas_expander->phy[i].phy_id = i; + sas_expander->phy[i].hba_port = hba_port; + + if ((mpi3mr_add_expander_phy(mrioc, &sas_expander->phy[i], + expander_pg1, sas_expander->parent_dev))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -1; + goto out_fail; + } + } + + if (sas_expander->enclosure_handle) { + enclosure_dev = + mpi3mr_enclosure_find_by_handle(mrioc, + sas_expander->enclosure_handle); + if (enclosure_dev) + sas_expander->enclosure_logical_id = le64_to_cpu( + enclosure_dev->pg0.enclosure_logical_id); + } + + mpi3mr_expander_node_add(mrioc, sas_expander); + return 0; + +out_fail: + + if (mr_sas_port) + mpi3mr_sas_port_remove(mrioc, + sas_expander->sas_address, + sas_address_parent, sas_expander->hba_port); + kfree(sas_expander->phy); + kfree(sas_expander); + return rc; +} + +/** + * mpi3mr_expander_node_remove - recursive removal of expander. + * @mrioc: Adapter instance reference + * @sas_expander: Expander device object + * + * Removes expander object and freeing associated memory from + * the sas_expander_list and removes the same from SAS TL, if + * one of the attached device is an expander then it recursively + * removes the expander device too. + * + * Return nothing. + */ +static void mpi3mr_expander_node_remove(struct mpi3mr_ioc *mrioc, + struct mpi3mr_sas_node *sas_expander) +{ + struct mpi3mr_sas_port *mr_sas_port, *next; + unsigned long flags; + u8 port_id; + + /* remove sibling ports attached to this expander */ + list_for_each_entry_safe(mr_sas_port, next, + &sas_expander->sas_port_list, port_list) { + if (mrioc->reset_in_progress) + return; + if (mr_sas_port->remote_identify.device_type == + SAS_END_DEVICE) + mpi3mr_remove_device_by_sas_address(mrioc, + mr_sas_port->remote_identify.sas_address, + mr_sas_port->hba_port); + else if (mr_sas_port->remote_identify.device_type == + SAS_EDGE_EXPANDER_DEVICE || + mr_sas_port->remote_identify.device_type == + SAS_FANOUT_EXPANDER_DEVICE) + mpi3mr_expander_remove(mrioc, + mr_sas_port->remote_identify.sas_address, + mr_sas_port->hba_port); + } + + port_id = sas_expander->hba_port->port_id; + mpi3mr_sas_port_remove(mrioc, sas_expander->sas_address, + sas_expander->sas_address_parent, sas_expander->hba_port); + + ioc_info(mrioc, "expander_remove: handle(0x%04x), sas_addr(0x%016llx), port:%d\n", + sas_expander->handle, (unsigned long long) + sas_expander->sas_address, port_id); + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_del(&sas_expander->list); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + + kfree(sas_expander->phy); + kfree(sas_expander); +} + +/** + * mpi3mr_expander_remove - Remove expander object + * @mrioc: Adapter instance reference + * @sas_address: Remove expander sas_address + * @hba_port: HBA port reference + * + * This function remove expander object, stored in + * mrioc->sas_expander_list and removes it from the SAS TL by + * calling mpi3mr_expander_node_remove(). + * + * Return: None + */ +void mpi3mr_expander_remove(struct mpi3mr_ioc *mrioc, u64 sas_address, + struct mpi3mr_hba_port *hba_port) +{ + struct mpi3mr_sas_node *sas_expander; + unsigned long flags; + + if (mrioc->reset_in_progress) + return; + + if (!hba_port) + return; + + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + sas_expander = mpi3mr_expander_find_by_sas_address(mrioc, sas_address, + hba_port); + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + if (sas_expander) + mpi3mr_expander_node_remove(mrioc, sas_expander); + +} + +/** + * mpi3mr_get_sas_negotiated_logical_linkrate - get linkrate + * @mrioc: Adapter instance reference + * @tgtdev: Target device + * + * This function identifies whether the target device is + * attached directly or through expander and issues sas phy + * page0 or expander phy page1 and gets the link rate, if there + * is any failure in reading the pages then this returns link + * rate of 1.5. + * + * Return: logical link rate. + */ +static u8 mpi3mr_get_sas_negotiated_logical_linkrate(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev) +{ + u8 link_rate = MPI3_SAS_NEG_LINK_RATE_1_5, phy_number; + struct mpi3_sas_expander_page1 expander_pg1; + struct mpi3_sas_phy_page0 phy_pg0; + u32 phynum_handle; + u16 ioc_status; + + phy_number = tgtdev->dev_spec.sas_sata_inf.phy_id; + if (!(tgtdev->devpg0_flag & MPI3_DEVICE0_FLAGS_ATT_METHOD_DIR_ATTACHED)) { + phynum_handle = ((phy_number<parent_handle); + if (mpi3mr_cfg_get_sas_exp_pg1(mrioc, &ioc_status, + &expander_pg1, sizeof(expander_pg1), + MPI3_SAS_EXPAND_PGAD_FORM_HANDLE_PHY_NUM, + phynum_handle)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + link_rate = (expander_pg1.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) >> + MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT; + goto out; + } + if (mpi3mr_cfg_get_sas_phy_pg0(mrioc, &ioc_status, &phy_pg0, + sizeof(struct mpi3_sas_phy_page0), + MPI3_SAS_PHY_PGAD_FORM_PHY_NUMBER, phy_number)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out; + } + link_rate = (phy_pg0.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) >> + MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT; +out: + return link_rate; +} + +/** + * mpi3mr_report_tgtdev_to_sas_transport - expose dev to SAS TL + * @mrioc: Adapter instance reference + * @tgtdev: Target device + * + * This function exposes the target device after + * preparing host_phy, setting up link rate etc. + * + * Return: 0 on success, non-zero for failure. + */ +int mpi3mr_report_tgtdev_to_sas_transport(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev) +{ + int retval = 0; + u8 link_rate, parent_phy_number; + u64 sas_address_parent, sas_address; + struct mpi3mr_hba_port *hba_port; + u8 port_id; + + if ((tgtdev->dev_type != MPI3_DEVICE_DEVFORM_SAS_SATA) || + !mrioc->sas_transport_enabled) + return -1; + + sas_address = tgtdev->dev_spec.sas_sata_inf.sas_address; + if (!mrioc->sas_hba.num_phys) + mpi3mr_sas_host_add(mrioc); + else + mpi3mr_sas_host_refresh(mrioc); + + if (mpi3mr_get_sas_address(mrioc, tgtdev->parent_handle, + &sas_address_parent) != 0) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + tgtdev->dev_spec.sas_sata_inf.sas_address_parent = sas_address_parent; + + parent_phy_number = tgtdev->dev_spec.sas_sata_inf.phy_id; + port_id = tgtdev->io_unit_port; + + hba_port = mpi3mr_get_hba_port_by_id(mrioc, port_id); + if (!hba_port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -1; + } + tgtdev->dev_spec.sas_sata_inf.hba_port = hba_port; + + link_rate = mpi3mr_get_sas_negotiated_logical_linkrate(mrioc, tgtdev); + + mpi3mr_update_links(mrioc, sas_address_parent, tgtdev->dev_handle, + parent_phy_number, link_rate, hba_port); + + tgtdev->host_exposed = 1; + if (!mpi3mr_sas_port_add(mrioc, tgtdev->dev_handle, + sas_address_parent, hba_port)) { + tgtdev->host_exposed = 0; + retval = -1; + } else if ((!tgtdev->starget)) { + if (!mrioc->is_driver_loading) + mpi3mr_sas_port_remove(mrioc, sas_address, + sas_address_parent, hba_port); + tgtdev->host_exposed = 0; + retval = -1; + } + return retval; +} + +/** + * mpi3mr_remove_tgtdev_from_sas_transport - remove from SAS TL + * @mrioc: Adapter instance reference + * @tgtdev: Target device + * + * This function removes the target device + * + * Return: None. + */ +void mpi3mr_remove_tgtdev_from_sas_transport(struct mpi3mr_ioc *mrioc, + struct mpi3mr_tgt_dev *tgtdev) +{ + u64 sas_address_parent, sas_address; + struct mpi3mr_hba_port *hba_port; + + if ((tgtdev->dev_type != MPI3_DEVICE_DEVFORM_SAS_SATA) || + !mrioc->sas_transport_enabled) + return; + + hba_port = tgtdev->dev_spec.sas_sata_inf.hba_port; + sas_address = tgtdev->dev_spec.sas_sata_inf.sas_address; + sas_address_parent = tgtdev->dev_spec.sas_sata_inf.sas_address_parent; + mpi3mr_sas_port_remove(mrioc, sas_address, sas_address_parent, + hba_port); + tgtdev->host_exposed = 0; +} + +/** + * mpi3mr_get_port_id_by_sas_phy - Get port ID of the given phy + * @phy: SAS transport layer phy object + * + * Return: Port number for valid ID else 0xFFFF + */ +static inline u8 mpi3mr_get_port_id_by_sas_phy(struct sas_phy *phy) +{ + u8 port_id = 0xFF; + struct mpi3mr_hba_port *hba_port = phy->hostdata; + + if (hba_port) + port_id = hba_port->port_id; + + return port_id; +} + +/** + * mpi3mr_get_port_id_by_rphy - Get Port number from SAS rphy + * + * @mrioc: Adapter instance reference + * @rphy: SAS transport layer remote phy object + * + * Retrieves HBA port number in which the device pointed by the + * rphy object is attached with. + * + * Return: Valid port number on success else OxFFFF. + */ +static u8 mpi3mr_get_port_id_by_rphy(struct mpi3mr_ioc *mrioc, struct sas_rphy *rphy) +{ + struct mpi3mr_sas_node *sas_expander; + struct mpi3mr_tgt_dev *tgtdev; + unsigned long flags; + u8 port_id = 0xFF; + + if (!rphy) + return port_id; + + if (rphy->identify.device_type == SAS_EDGE_EXPANDER_DEVICE || + rphy->identify.device_type == SAS_FANOUT_EXPANDER_DEVICE) { + spin_lock_irqsave(&mrioc->sas_node_lock, flags); + list_for_each_entry(sas_expander, &mrioc->sas_expander_list, + list) { + if (sas_expander->rphy == rphy) { + port_id = sas_expander->hba_port->port_id; + break; + } + } + spin_unlock_irqrestore(&mrioc->sas_node_lock, flags); + } else if (rphy->identify.device_type == SAS_END_DEVICE) { + spin_lock_irqsave(&mrioc->tgtdev_lock, flags); + + tgtdev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + if (tgtdev) { + port_id = + tgtdev->dev_spec.sas_sata_inf.hba_port->port_id; + mpi3mr_tgtdev_put(tgtdev); + } + spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); + } + return port_id; +} + +static inline struct mpi3mr_ioc *phy_to_mrioc(struct sas_phy *phy) +{ + struct Scsi_Host *shost = dev_to_shost(phy->dev.parent); + + return shost_priv(shost); +} + +static inline struct mpi3mr_ioc *rphy_to_mrioc(struct sas_rphy *rphy) +{ + struct Scsi_Host *shost = dev_to_shost(rphy->dev.parent->parent); + + return shost_priv(shost); +} + +/* report phy error log structure */ +struct phy_error_log_request { + u8 smp_frame_type; /* 0x40 */ + u8 function; /* 0x11 */ + u8 allocated_response_length; + u8 request_length; /* 02 */ + u8 reserved_1[5]; + u8 phy_identifier; + u8 reserved_2[2]; +}; + +/* report phy error log reply structure */ +struct phy_error_log_reply { + u8 smp_frame_type; /* 0x41 */ + u8 function; /* 0x11 */ + u8 function_result; + u8 response_length; + __be16 expander_change_count; + u8 reserved_1[3]; + u8 phy_identifier; + u8 reserved_2[2]; + __be32 invalid_dword; + __be32 running_disparity_error; + __be32 loss_of_dword_sync; + __be32 phy_reset_problem; +}; + + +/** + * mpi3mr_get_expander_phy_error_log - return expander counters: + * @mrioc: Adapter instance reference + * @phy: The SAS transport layer phy object + * + * Return: 0 for success, non-zero for failure. + * + */ +static int mpi3mr_get_expander_phy_error_log(struct mpi3mr_ioc *mrioc, + struct sas_phy *phy) +{ + struct mpi3_smp_passthrough_request mpi_request; + struct mpi3_smp_passthrough_reply mpi_reply; + struct phy_error_log_request *phy_error_log_request; + struct phy_error_log_reply *phy_error_log_reply; + int rc; + void *psge; + void *data_out = NULL; + dma_addr_t data_out_dma, data_in_dma; + u32 data_out_sz, data_in_sz, sz; + u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST; + u16 request_sz = sizeof(struct mpi3_smp_passthrough_request); + u16 reply_sz = sizeof(struct mpi3_smp_passthrough_reply); + u16 ioc_status; + + if (mrioc->reset_in_progress) { + ioc_err(mrioc, "%s: host reset in progress!\n", __func__); + return -EFAULT; + } + + data_out_sz = sizeof(struct phy_error_log_request); + data_in_sz = sizeof(struct phy_error_log_reply); + sz = data_out_sz + data_in_sz; + data_out = dma_alloc_coherent(&mrioc->pdev->dev, sz, &data_out_dma, + GFP_KERNEL); + if (!data_out) { + rc = -ENOMEM; + goto out; + } + + data_in_dma = data_out_dma + data_out_sz; + phy_error_log_reply = data_out + data_out_sz; + + rc = -EINVAL; + memset(data_out, 0, sz); + phy_error_log_request = data_out; + phy_error_log_request->smp_frame_type = 0x40; + phy_error_log_request->function = 0x11; + phy_error_log_request->request_length = 2; + phy_error_log_request->allocated_response_length = 0; + phy_error_log_request->phy_identifier = phy->number; + + memset(&mpi_request, 0, request_sz); + memset(&mpi_reply, 0, reply_sz); + mpi_request.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_TRANSPORT_CMDS); + mpi_request.function = MPI3_FUNCTION_SMP_PASSTHROUGH; + mpi_request.io_unit_port = (u8) mpi3mr_get_port_id_by_sas_phy(phy); + mpi_request.sas_address = cpu_to_le64(phy->identify.sas_address); + + psge = &mpi_request.request_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_out_sz, data_out_dma); + + psge = &mpi_request.response_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_in_sz, data_in_dma); + + dprint_transport_info(mrioc, + "sending phy error log SMP request to sas_address(0x%016llx), phy_id(%d)\n", + (unsigned long long)phy->identify.sas_address, phy->number); + + if (mpi3mr_post_transport_req(mrioc, &mpi_request, request_sz, + &mpi_reply, reply_sz, MPI3MR_INTADMCMD_TIMEOUT, &ioc_status)) + goto out; + + dprint_transport_info(mrioc, + "phy error log SMP request completed with ioc_status(0x%04x)\n", + ioc_status); + + if (ioc_status == MPI3_IOCSTATUS_SUCCESS) { + dprint_transport_info(mrioc, + "phy error log - reply data transfer size(%d)\n", + le16_to_cpu(mpi_reply.response_data_length)); + + if (le16_to_cpu(mpi_reply.response_data_length) != + sizeof(struct phy_error_log_reply)) + goto out; + + dprint_transport_info(mrioc, + "phy error log - function_result(%d)\n", + phy_error_log_reply->function_result); + + phy->invalid_dword_count = + be32_to_cpu(phy_error_log_reply->invalid_dword); + phy->running_disparity_error_count = + be32_to_cpu(phy_error_log_reply->running_disparity_error); + phy->loss_of_dword_sync_count = + be32_to_cpu(phy_error_log_reply->loss_of_dword_sync); + phy->phy_reset_problem_count = + be32_to_cpu(phy_error_log_reply->phy_reset_problem); + rc = 0; + } + +out: + if (data_out) + dma_free_coherent(&mrioc->pdev->dev, sz, data_out, + data_out_dma); + + return rc; +} + +/** + * mpi3mr_transport_get_linkerrors - return phy error counters + * @phy: The SAS transport layer phy object + * + * This function retrieves the phy error log information of the + * HBA or expander for which the phy belongs to + * + * Return: 0 for success, non-zero for failure. + */ +static int mpi3mr_transport_get_linkerrors(struct sas_phy *phy) +{ + struct mpi3mr_ioc *mrioc = phy_to_mrioc(phy); + struct mpi3_sas_phy_page1 phy_pg1; + int rc = 0; + u16 ioc_status; + + rc = mpi3mr_parent_present(mrioc, phy); + if (rc) + return rc; + + if (phy->identify.sas_address != mrioc->sas_hba.sas_address) + return mpi3mr_get_expander_phy_error_log(mrioc, phy); + + memset(&phy_pg1, 0, sizeof(struct mpi3_sas_phy_page1)); + /* get hba phy error logs */ + if ((mpi3mr_cfg_get_sas_phy_pg1(mrioc, &ioc_status, &phy_pg1, + sizeof(struct mpi3_sas_phy_page1), + MPI3_SAS_PHY_PGAD_FORM_PHY_NUMBER, phy->number))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -ENXIO; + } + + if (ioc_status != MPI3_IOCSTATUS_SUCCESS) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + return -ENXIO; + } + phy->invalid_dword_count = le32_to_cpu(phy_pg1.invalid_dword_count); + phy->running_disparity_error_count = + le32_to_cpu(phy_pg1.running_disparity_error_count); + phy->loss_of_dword_sync_count = + le32_to_cpu(phy_pg1.loss_dword_synch_count); + phy->phy_reset_problem_count = + le32_to_cpu(phy_pg1.phy_reset_problem_count); + return 0; +} + +/** + * mpi3mr_transport_get_enclosure_identifier - Get Enclosure ID + * @rphy: The SAS transport layer remote phy object + * @identifier: Enclosure identifier to be returned + * + * Returns the enclosure id for the device pointed by the remote + * phy object. + * + * Return: 0 on success or -ENXIO + */ +static int +mpi3mr_transport_get_enclosure_identifier(struct sas_rphy *rphy, + u64 *identifier) +{ + struct mpi3mr_ioc *mrioc = rphy_to_mrioc(rphy); + struct mpi3mr_tgt_dev *tgtdev = NULL; + unsigned long flags; + int rc; + + spin_lock_irqsave(&mrioc->tgtdev_lock, flags); + tgtdev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + if (tgtdev) { + *identifier = + tgtdev->enclosure_logical_id; + rc = 0; + mpi3mr_tgtdev_put(tgtdev); + } else { + *identifier = 0; + rc = -ENXIO; + } + spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); + + return rc; +} + +/** + * mpi3mr_transport_get_bay_identifier - Get bay ID + * @rphy: The SAS transport layer remote phy object + * + * Returns the slot id for the device pointed by the remote phy + * object. + * + * Return: Valid slot ID on success or -ENXIO + */ +static int +mpi3mr_transport_get_bay_identifier(struct sas_rphy *rphy) +{ + struct mpi3mr_ioc *mrioc = rphy_to_mrioc(rphy); + struct mpi3mr_tgt_dev *tgtdev = NULL; + unsigned long flags; + int rc; + + spin_lock_irqsave(&mrioc->tgtdev_lock, flags); + tgtdev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc, + rphy->identify.sas_address, rphy); + if (tgtdev) { + rc = tgtdev->slot; + mpi3mr_tgtdev_put(tgtdev); + } else + rc = -ENXIO; + spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags); + + return rc; +} + +/* phy control request structure */ +struct phy_control_request { + u8 smp_frame_type; /* 0x40 */ + u8 function; /* 0x91 */ + u8 allocated_response_length; + u8 request_length; /* 0x09 */ + u16 expander_change_count; + u8 reserved_1[3]; + u8 phy_identifier; + u8 phy_operation; + u8 reserved_2[13]; + u64 attached_device_name; + u8 programmed_min_physical_link_rate; + u8 programmed_max_physical_link_rate; + u8 reserved_3[6]; +}; + +/* phy control reply structure */ +struct phy_control_reply { + u8 smp_frame_type; /* 0x41 */ + u8 function; /* 0x11 */ + u8 function_result; + u8 response_length; +}; + +#define SMP_PHY_CONTROL_LINK_RESET (0x01) +#define SMP_PHY_CONTROL_HARD_RESET (0x02) +#define SMP_PHY_CONTROL_DISABLE (0x03) + +/** + * mpi3mr_expander_phy_control - expander phy control + * @mrioc: Adapter instance reference + * @phy: The SAS transport layer phy object + * @phy_operation: The phy operation to be executed + * + * Issues SMP passthru phy control request to execute a specific + * phy operation for a given expander device. + * + * Return: 0 for success, non-zero for failure. + */ +static int +mpi3mr_expander_phy_control(struct mpi3mr_ioc *mrioc, + struct sas_phy *phy, u8 phy_operation) +{ + struct mpi3_smp_passthrough_request mpi_request; + struct mpi3_smp_passthrough_reply mpi_reply; + struct phy_control_request *phy_control_request; + struct phy_control_reply *phy_control_reply; + int rc; + void *psge; + void *data_out = NULL; + dma_addr_t data_out_dma; + dma_addr_t data_in_dma; + size_t data_in_sz; + size_t data_out_sz; + u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST; + u16 request_sz = sizeof(struct mpi3_smp_passthrough_request); + u16 reply_sz = sizeof(struct mpi3_smp_passthrough_reply); + u16 ioc_status; + u16 sz; + + if (mrioc->reset_in_progress) { + ioc_err(mrioc, "%s: host reset in progress!\n", __func__); + return -EFAULT; + } + + data_out_sz = sizeof(struct phy_control_request); + data_in_sz = sizeof(struct phy_control_reply); + sz = data_out_sz + data_in_sz; + data_out = dma_alloc_coherent(&mrioc->pdev->dev, sz, &data_out_dma, + GFP_KERNEL); + if (!data_out) { + rc = -ENOMEM; + goto out; + } + + data_in_dma = data_out_dma + data_out_sz; + phy_control_reply = data_out + data_out_sz; + + rc = -EINVAL; + memset(data_out, 0, sz); + + phy_control_request = data_out; + phy_control_request->smp_frame_type = 0x40; + phy_control_request->function = 0x91; + phy_control_request->request_length = 9; + phy_control_request->allocated_response_length = 0; + phy_control_request->phy_identifier = phy->number; + phy_control_request->phy_operation = phy_operation; + phy_control_request->programmed_min_physical_link_rate = + phy->minimum_linkrate << 4; + phy_control_request->programmed_max_physical_link_rate = + phy->maximum_linkrate << 4; + + memset(&mpi_request, 0, request_sz); + memset(&mpi_reply, 0, reply_sz); + mpi_request.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_TRANSPORT_CMDS); + mpi_request.function = MPI3_FUNCTION_SMP_PASSTHROUGH; + mpi_request.io_unit_port = (u8) mpi3mr_get_port_id_by_sas_phy(phy); + mpi_request.sas_address = cpu_to_le64(phy->identify.sas_address); + + psge = &mpi_request.request_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_out_sz, data_out_dma); + + psge = &mpi_request.response_sge; + mpi3mr_add_sg_single(psge, sgl_flags, data_in_sz, data_in_dma); + + dprint_transport_info(mrioc, + "sending phy control SMP request to sas_address(0x%016llx), phy_id(%d) opcode(%d)\n", + (unsigned long long)phy->identify.sas_address, phy->number, + phy_operation); + + if (mpi3mr_post_transport_req(mrioc, &mpi_request, request_sz, + &mpi_reply, reply_sz, MPI3MR_INTADMCMD_TIMEOUT, &ioc_status)) + goto out; + + dprint_transport_info(mrioc, + "phy control SMP request completed with ioc_status(0x%04x)\n", + ioc_status); + + if (ioc_status == MPI3_IOCSTATUS_SUCCESS) { + dprint_transport_info(mrioc, + "phy control - reply data transfer size(%d)\n", + le16_to_cpu(mpi_reply.response_data_length)); + + if (le16_to_cpu(mpi_reply.response_data_length) != + sizeof(struct phy_control_reply)) + goto out; + dprint_transport_info(mrioc, + "phy control - function_result(%d)\n", + phy_control_reply->function_result); + rc = 0; + } + out: + if (data_out) + dma_free_coherent(&mrioc->pdev->dev, sz, data_out, + data_out_dma); + + return rc; +} + +/** + * mpi3mr_transport_phy_reset - Reset a given phy + * @phy: The SAS transport layer phy object + * @hard_reset: Flag to indicate the type of reset + * + * Return: 0 for success, non-zero for failure. + */ +static int +mpi3mr_transport_phy_reset(struct sas_phy *phy, int hard_reset) +{ + struct mpi3mr_ioc *mrioc = phy_to_mrioc(phy); + struct mpi3_iounit_control_request mpi_request; + struct mpi3_iounit_control_reply mpi_reply; + u16 request_sz = sizeof(struct mpi3_iounit_control_request); + u16 reply_sz = sizeof(struct mpi3_iounit_control_reply); + int rc = 0; + u16 ioc_status; + + rc = mpi3mr_parent_present(mrioc, phy); + if (rc) + return rc; + + /* handle expander phys */ + if (phy->identify.sas_address != mrioc->sas_hba.sas_address) + return mpi3mr_expander_phy_control(mrioc, phy, + (hard_reset == 1) ? SMP_PHY_CONTROL_HARD_RESET : + SMP_PHY_CONTROL_LINK_RESET); + + /* handle hba phys */ + memset(&mpi_request, 0, request_sz); + mpi_request.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_TRANSPORT_CMDS); + mpi_request.function = MPI3_FUNCTION_IO_UNIT_CONTROL; + mpi_request.operation = MPI3_CTRL_OP_SAS_PHY_CONTROL; + mpi_request.param8[MPI3_CTRL_OP_SAS_PHY_CONTROL_PARAM8_ACTION_INDEX] = + (hard_reset ? MPI3_CTRL_ACTION_HARD_RESET : + MPI3_CTRL_ACTION_LINK_RESET); + mpi_request.param8[MPI3_CTRL_OP_SAS_PHY_CONTROL_PARAM8_PHY_INDEX] = + phy->number; + + dprint_transport_info(mrioc, + "sending phy reset request to sas_address(0x%016llx), phy_id(%d) hard_reset(%d)\n", + (unsigned long long)phy->identify.sas_address, phy->number, + hard_reset); + + if (mpi3mr_post_transport_req(mrioc, &mpi_request, request_sz, + &mpi_reply, reply_sz, MPI3MR_INTADMCMD_TIMEOUT, &ioc_status)) { + rc = -EAGAIN; + goto out; + } + + dprint_transport_info(mrioc, + "phy reset request completed with ioc_status(0x%04x)\n", + ioc_status); +out: + return rc; +} + +/** + * mpi3mr_transport_phy_enable - enable/disable phys + * @phy: The SAS transport layer phy object + * @enable: flag to enable/disable, enable phy when true + * + * This function enables/disables a given by executing required + * configuration page changes or expander phy control command + * + * Return: 0 for success, non-zero for failure. + */ +static int +mpi3mr_transport_phy_enable(struct sas_phy *phy, int enable) +{ + struct mpi3mr_ioc *mrioc = phy_to_mrioc(phy); + struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0 = NULL; + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1 = NULL; + u16 sz; + int rc = 0; + int i, discovery_active; + + rc = mpi3mr_parent_present(mrioc, phy); + if (rc) + return rc; + + /* handle expander phys */ + if (phy->identify.sas_address != mrioc->sas_hba.sas_address) + return mpi3mr_expander_phy_control(mrioc, phy, + (enable == 1) ? SMP_PHY_CONTROL_LINK_RESET : + SMP_PHY_CONTROL_DISABLE); + + /* handle hba phys */ + sz = offsetof(struct mpi3_sas_io_unit_page0, phy_data) + + (mrioc->sas_hba.num_phys * + sizeof(struct mpi3_sas_io_unit0_phy_data)); + sas_io_unit_pg0 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg0) { + rc = -ENOMEM; + goto out; + } + if (mpi3mr_cfg_get_sas_io_unit_pg0(mrioc, sas_io_unit_pg0, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -ENXIO; + goto out; + } + + /* unable to enable/disable phys when discovery is active */ + for (i = 0, discovery_active = 0; i < mrioc->sas_hba.num_phys ; i++) { + if (sas_io_unit_pg0->phy_data[i].port_flags & + MPI3_SASIOUNIT0_PORTFLAGS_DISC_IN_PROGRESS) { + ioc_err(mrioc, + "discovery is active on port = %d, phy = %d\n" + "\tunable to enable/disable phys, try again later!\n", + sas_io_unit_pg0->phy_data[i].io_unit_port, i); + discovery_active = 1; + } + } + + if (discovery_active) { + rc = -EAGAIN; + goto out; + } + + if ((sas_io_unit_pg0->phy_data[phy->number].phy_flags & + (MPI3_SASIOUNIT0_PHYFLAGS_HOST_PHY | + MPI3_SASIOUNIT0_PHYFLAGS_VIRTUAL_PHY))) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -ENXIO; + goto out; + } + + /* read sas_iounit page 1 */ + sz = offsetof(struct mpi3_sas_io_unit_page1, phy_data) + + (mrioc->sas_hba.num_phys * + sizeof(struct mpi3_sas_io_unit1_phy_data)); + sas_io_unit_pg1 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg1) { + rc = -ENOMEM; + goto out; + } + + if (mpi3mr_cfg_get_sas_io_unit_pg1(mrioc, sas_io_unit_pg1, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -ENXIO; + goto out; + } + + if (enable) + sas_io_unit_pg1->phy_data[phy->number].phy_flags + &= ~MPI3_SASIOUNIT1_PHYFLAGS_PHY_DISABLE; + else + sas_io_unit_pg1->phy_data[phy->number].phy_flags + |= MPI3_SASIOUNIT1_PHYFLAGS_PHY_DISABLE; + + mpi3mr_cfg_set_sas_io_unit_pg1(mrioc, sas_io_unit_pg1, sz); + + /* link reset */ + if (enable) + mpi3mr_transport_phy_reset(phy, 0); + + out: + kfree(sas_io_unit_pg1); + kfree(sas_io_unit_pg0); + return rc; +} + +/** + * mpi3mr_transport_phy_speed - set phy min/max speed + * @phy: The SAS transport later phy object + * @rates: Rates defined as in sas_phy_linkrates + * + * This function sets the link rates given in the rates + * argument to the given phy by executing required configuration + * page changes or expander phy control command + * + * Return: 0 for success, non-zero for failure. + */ +static int +mpi3mr_transport_phy_speed(struct sas_phy *phy, struct sas_phy_linkrates *rates) +{ + struct mpi3mr_ioc *mrioc = phy_to_mrioc(phy); + struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1 = NULL; + struct mpi3_sas_phy_page0 phy_pg0; + u16 sz, ioc_status; + int rc = 0; + + rc = mpi3mr_parent_present(mrioc, phy); + if (rc) + return rc; + + if (!rates->minimum_linkrate) + rates->minimum_linkrate = phy->minimum_linkrate; + else if (rates->minimum_linkrate < phy->minimum_linkrate_hw) + rates->minimum_linkrate = phy->minimum_linkrate_hw; + + if (!rates->maximum_linkrate) + rates->maximum_linkrate = phy->maximum_linkrate; + else if (rates->maximum_linkrate > phy->maximum_linkrate_hw) + rates->maximum_linkrate = phy->maximum_linkrate_hw; + + /* handle expander phys */ + if (phy->identify.sas_address != mrioc->sas_hba.sas_address) { + phy->minimum_linkrate = rates->minimum_linkrate; + phy->maximum_linkrate = rates->maximum_linkrate; + return mpi3mr_expander_phy_control(mrioc, phy, + SMP_PHY_CONTROL_LINK_RESET); + } + + /* handle hba phys */ + sz = offsetof(struct mpi3_sas_io_unit_page1, phy_data) + + (mrioc->sas_hba.num_phys * + sizeof(struct mpi3_sas_io_unit1_phy_data)); + sas_io_unit_pg1 = kzalloc(sz, GFP_KERNEL); + if (!sas_io_unit_pg1) { + rc = -ENOMEM; + goto out; + } + + if (mpi3mr_cfg_get_sas_io_unit_pg1(mrioc, sas_io_unit_pg1, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -ENXIO; + goto out; + } + + sas_io_unit_pg1->phy_data[phy->number].max_min_link_rate = + (rates->minimum_linkrate + (rates->maximum_linkrate << 4)); + + if (mpi3mr_cfg_set_sas_io_unit_pg1(mrioc, sas_io_unit_pg1, sz)) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + rc = -ENXIO; + goto out; + } + + /* link reset */ + mpi3mr_transport_phy_reset(phy, 0); + + /* read phy page 0, then update the rates in the sas transport phy */ + if (!mpi3mr_cfg_get_sas_phy_pg0(mrioc, &ioc_status, &phy_pg0, + sizeof(struct mpi3_sas_phy_page0), + MPI3_SAS_PHY_PGAD_FORM_PHY_NUMBER, phy->number) && + (ioc_status == MPI3_IOCSTATUS_SUCCESS)) { + phy->minimum_linkrate = mpi3mr_convert_phy_link_rate( + phy_pg0.programmed_link_rate & + MPI3_SAS_PRATE_MIN_RATE_MASK); + phy->maximum_linkrate = mpi3mr_convert_phy_link_rate( + phy_pg0.programmed_link_rate >> 4); + phy->negotiated_linkrate = + mpi3mr_convert_phy_link_rate( + (phy_pg0.negotiated_link_rate & + MPI3_SAS_NEG_LINK_RATE_LOGICAL_MASK) + >> MPI3_SAS_NEG_LINK_RATE_LOGICAL_SHIFT); + } + +out: + kfree(sas_io_unit_pg1); + return rc; +} + +/** + * mpi3mr_map_smp_buffer - map BSG dma buffer + * @dev: Generic device reference + * @buf: BSG buffer pointer + * @dma_addr: Physical address holder + * @dma_len: Mapped DMA buffer length. + * @p: Virtual address holder + * + * This function maps the DMAable buffer + * + * Return: 0 on success, non-zero on failure + */ +static int +mpi3mr_map_smp_buffer(struct device *dev, struct bsg_buffer *buf, + dma_addr_t *dma_addr, size_t *dma_len, void **p) +{ + /* Check if the request is split across multiple segments */ + if (buf->sg_cnt > 1) { + *p = dma_alloc_coherent(dev, buf->payload_len, dma_addr, + GFP_KERNEL); + if (!*p) + return -ENOMEM; + *dma_len = buf->payload_len; + } else { + if (!dma_map_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL)) + return -ENOMEM; + *dma_addr = sg_dma_address(buf->sg_list); + *dma_len = sg_dma_len(buf->sg_list); + *p = NULL; + } + + return 0; +} + +/** + * mpi3mr_unmap_smp_buffer - unmap BSG dma buffer + * @dev: Generic device reference + * @buf: BSG buffer pointer + * @dma_addr: Physical address to be unmapped + * @p: Virtual address + * + * This function unmaps the DMAable buffer + */ +static void +mpi3mr_unmap_smp_buffer(struct device *dev, struct bsg_buffer *buf, + dma_addr_t dma_addr, void *p) +{ + if (p) + dma_free_coherent(dev, buf->payload_len, p, dma_addr); + else + dma_unmap_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL); +} + +/** + * mpi3mr_transport_smp_handler - handler for smp passthru + * @job: BSG job reference + * @shost: SCSI host object reference + * @rphy: SAS transport rphy object pointing the expander + * + * This is used primarily by smp utils for sending the SMP + * commands to the expanders attached to the controller + */ +static void +mpi3mr_transport_smp_handler(struct bsg_job *job, struct Scsi_Host *shost, + struct sas_rphy *rphy) +{ + struct mpi3mr_ioc *mrioc = shost_priv(shost); + struct mpi3_smp_passthrough_request mpi_request; + struct mpi3_smp_passthrough_reply mpi_reply; + int rc; + void *psge; + dma_addr_t dma_addr_in; + dma_addr_t dma_addr_out; + void *addr_in = NULL; + void *addr_out = NULL; + size_t dma_len_in; + size_t dma_len_out; + unsigned int reslen = 0; + u16 request_sz = sizeof(struct mpi3_smp_passthrough_request); + u16 reply_sz = sizeof(struct mpi3_smp_passthrough_reply); + u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST; + u16 ioc_status; + + if (mrioc->reset_in_progress) { + ioc_err(mrioc, "%s: host reset in progress!\n", __func__); + rc = -EFAULT; + goto out; + } + + rc = mpi3mr_map_smp_buffer(&mrioc->pdev->dev, &job->request_payload, + &dma_addr_out, &dma_len_out, &addr_out); + if (rc) + goto out; + + if (addr_out) + sg_copy_to_buffer(job->request_payload.sg_list, + job->request_payload.sg_cnt, addr_out, + job->request_payload.payload_len); + + rc = mpi3mr_map_smp_buffer(&mrioc->pdev->dev, &job->reply_payload, + &dma_addr_in, &dma_len_in, &addr_in); + if (rc) + goto unmap_out; + + memset(&mpi_request, 0, request_sz); + memset(&mpi_reply, 0, reply_sz); + mpi_request.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_TRANSPORT_CMDS); + mpi_request.function = MPI3_FUNCTION_SMP_PASSTHROUGH; + mpi_request.io_unit_port = (u8) mpi3mr_get_port_id_by_rphy(mrioc, rphy); + mpi_request.sas_address = ((rphy) ? + cpu_to_le64(rphy->identify.sas_address) : + cpu_to_le64(mrioc->sas_hba.sas_address)); + psge = &mpi_request.request_sge; + mpi3mr_add_sg_single(psge, sgl_flags, dma_len_out - 4, dma_addr_out); + + psge = &mpi_request.response_sge; + mpi3mr_add_sg_single(psge, sgl_flags, dma_len_in - 4, dma_addr_in); + + dprint_transport_info(mrioc, "sending SMP request\n"); + + rc = mpi3mr_post_transport_req(mrioc, &mpi_request, request_sz, + &mpi_reply, reply_sz, + MPI3MR_INTADMCMD_TIMEOUT, &ioc_status); + if (rc) + goto unmap_in; + + dprint_transport_info(mrioc, + "SMP request completed with ioc_status(0x%04x)\n", ioc_status); + + dprint_transport_info(mrioc, + "SMP request - reply data transfer size(%d)\n", + le16_to_cpu(mpi_reply.response_data_length)); + + memcpy(job->reply, &mpi_reply, reply_sz); + job->reply_len = reply_sz; + reslen = le16_to_cpu(mpi_reply.response_data_length); + + if (addr_in) + sg_copy_from_buffer(job->reply_payload.sg_list, + job->reply_payload.sg_cnt, addr_in, + job->reply_payload.payload_len); + + rc = 0; +unmap_in: + mpi3mr_unmap_smp_buffer(&mrioc->pdev->dev, &job->reply_payload, + dma_addr_in, addr_in); +unmap_out: + mpi3mr_unmap_smp_buffer(&mrioc->pdev->dev, &job->request_payload, + dma_addr_out, addr_out); +out: + bsg_job_done(job, rc, reslen); +} + +struct sas_function_template mpi3mr_transport_functions = { + .get_linkerrors = mpi3mr_transport_get_linkerrors, + .get_enclosure_identifier = mpi3mr_transport_get_enclosure_identifier, + .get_bay_identifier = mpi3mr_transport_get_bay_identifier, + .phy_reset = mpi3mr_transport_phy_reset, + .phy_enable = mpi3mr_transport_phy_enable, + .set_phy_speed = mpi3mr_transport_phy_speed, + .smp_handler = mpi3mr_transport_smp_handler, +}; + +struct scsi_transport_template *mpi3mr_transport_template; diff --git a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h index d00431f553e15c39a6a298d38458c3662c3f0c60..4d0be5ab98c1ff577f67896df43fb1d846b60893 100644 --- a/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h +++ b/drivers/scsi/mpt3sas/mpi/mpi2_cnfg.h @@ -534,6 +534,7 @@ typedef struct _MPI2_CONFIG_REPLY { ****************************************************************************/ #define MPI2_MFGPAGE_VENDORID_LSI (0x1000) +#define MPI2_MFGPAGE_VENDORID_ATTO (0x117C) /*MPI v2.0 SAS products */ #define MPI2_MFGPAGE_DEVID_SAS2004 (0x0070) diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 565339a0811dbe92b3f341f0f05379d0e6187d67..8b22df8c1792dbca52b37bf5399ccc6e151aeafc 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -2990,19 +2990,26 @@ static int _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev) { struct sysinfo s; + u64 coherent_dma_mask, dma_mask; - if (ioc->is_mcpu_endpoint || - sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma || - dma_get_required_mask(&pdev->dev) <= 32) + if (ioc->is_mcpu_endpoint || sizeof(dma_addr_t) == 4 || + dma_get_required_mask(&pdev->dev) <= 32) { ioc->dma_mask = 32; + coherent_dma_mask = dma_mask = DMA_BIT_MASK(32); /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */ - else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) + } else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) { ioc->dma_mask = 63; - else + coherent_dma_mask = dma_mask = DMA_BIT_MASK(63); + } else { ioc->dma_mask = 64; + coherent_dma_mask = dma_mask = DMA_BIT_MASK(64); + } - if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)) || - dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask))) + if (ioc->use_32bit_dma) + coherent_dma_mask = DMA_BIT_MASK(32); + + if (dma_set_mask(&pdev->dev, dma_mask) || + dma_set_coherent_mask(&pdev->dev, coherent_dma_mask)) return -ENODEV; if (ioc->dma_mask > 32) { @@ -4313,7 +4320,7 @@ _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SMID = cpu_to_le16(smid); - writel(*request, &ioc->chip->AtomicRequestDescriptorPost); + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); } /** @@ -4335,7 +4342,7 @@ _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SMID = cpu_to_le16(smid); - writel(*request, &ioc->chip->AtomicRequestDescriptorPost); + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); } /** @@ -4358,7 +4365,7 @@ _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, descriptor.MSIxIndex = msix_task; descriptor.SMID = cpu_to_le16(smid); - writel(*request, &ioc->chip->AtomicRequestDescriptorPost); + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); } /** @@ -4379,7 +4386,7 @@ _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid) descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SMID = cpu_to_le16(smid); - writel(*request, &ioc->chip->AtomicRequestDescriptorPost); + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); } /** @@ -5424,6 +5431,151 @@ out: return rc; } +/** + * mpt3sas_atto_validate_nvram - validate the ATTO nvram read from mfg pg1 + * + * @ioc : per adapter object + * @n : ptr to the ATTO nvram structure + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_validate_nvram(struct MPT3SAS_ADAPTER *ioc, + struct ATTO_SAS_NVRAM *n) +{ + int r = -EINVAL; + union ATTO_SAS_ADDRESS *s1; + u32 len; + u8 *pb; + u8 ckSum; + + /* validate nvram checksum */ + pb = (u8 *) n; + ckSum = ATTO_SASNVR_CKSUM_SEED; + len = sizeof(struct ATTO_SAS_NVRAM); + + while (len--) + ckSum = ckSum + pb[len]; + + if (ckSum) { + ioc_err(ioc, "Invalid ATTO NVRAM checksum\n"); + return r; + } + + s1 = (union ATTO_SAS_ADDRESS *) n->SasAddr; + + if (n->Signature[0] != 'E' + || n->Signature[1] != 'S' + || n->Signature[2] != 'A' + || n->Signature[3] != 'S') + ioc_err(ioc, "Invalid ATTO NVRAM signature\n"); + else if (n->Version > ATTO_SASNVR_VERSION) + ioc_info(ioc, "Invalid ATTO NVRAM version"); + else if ((n->SasAddr[7] & (ATTO_SAS_ADDR_ALIGN - 1)) + || s1->b[0] != 0x50 + || s1->b[1] != 0x01 + || s1->b[2] != 0x08 + || (s1->b[3] & 0xF0) != 0x60 + || ((s1->b[3] & 0x0F) | le32_to_cpu(s1->d[1])) == 0) { + ioc_err(ioc, "Invalid ATTO SAS address\n"); + } else + r = 0; + return r; +} + +/** + * mpt3sas_atto_get_sas_addr - get the ATTO SAS address from mfg page 1 + * + * @ioc : per adapter object + * @*sas_addr : return sas address + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_get_sas_addr(struct MPT3SAS_ADAPTER *ioc, union ATTO_SAS_ADDRESS *sas_addr) +{ + Mpi2ManufacturingPage1_t mfg_pg1; + Mpi2ConfigReply_t mpi_reply; + struct ATTO_SAS_NVRAM *nvram; + int r; + __be64 addr; + + r = mpt3sas_config_get_manufacturing_pg1(ioc, &mpi_reply, &mfg_pg1); + if (r) { + ioc_err(ioc, "Failed to read manufacturing page 1\n"); + return r; + } + + /* validate nvram */ + nvram = (struct ATTO_SAS_NVRAM *) mfg_pg1.VPD; + r = mpt3sas_atto_validate_nvram(ioc, nvram); + if (r) + return r; + + addr = *((__be64 *) nvram->SasAddr); + sas_addr->q = cpu_to_le64(be64_to_cpu(addr)); + return r; +} + +/** + * mpt3sas_atto_init - perform initializaion for ATTO branded + * adapter. + * @ioc : per adapter object + *5 + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_init(struct MPT3SAS_ADAPTER *ioc) +{ + int sz = 0; + Mpi2BiosPage4_t *bios_pg4 = NULL; + Mpi2ConfigReply_t mpi_reply; + int r; + int ix; + union ATTO_SAS_ADDRESS sas_addr; + union ATTO_SAS_ADDRESS temp; + union ATTO_SAS_ADDRESS bias; + + r = mpt3sas_atto_get_sas_addr(ioc, &sas_addr); + if (r) + return r; + + /* get header first to get size */ + r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, NULL, 0); + if (r) { + ioc_err(ioc, "Failed to read ATTO bios page 4 header.\n"); + return r; + } + + sz = mpi_reply.Header.PageLength * sizeof(u32); + bios_pg4 = kzalloc(sz, GFP_KERNEL); + if (!bios_pg4) { + ioc_err(ioc, "Failed to allocate memory for ATTO bios page.\n"); + return -ENOMEM; + } + + /* read bios page 4 */ + r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, bios_pg4, sz); + if (r) { + ioc_err(ioc, "Failed to read ATTO bios page 4\n"); + goto out; + } + + /* Update bios page 4 with the ATTO WWID */ + bias.q = sas_addr.q; + bias.b[7] += ATTO_SAS_ADDR_DEVNAME_BIAS; + + for (ix = 0; ix < bios_pg4->NumPhys; ix++) { + temp.q = sas_addr.q; + temp.b[7] += ix; + bios_pg4->Phy[ix].ReassignmentWWID = temp.q; + bios_pg4->Phy[ix].ReassignmentDeviceName = bias.q; + } + r = mpt3sas_config_set_bios_pg4(ioc, &mpi_reply, bios_pg4, sz); + +out: + kfree(bios_pg4); + return r; +} + /** * _base_static_config_pages - static start of day config pages * @ioc: per adapter object @@ -5447,6 +5599,13 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) if (rc) return rc; } + + if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) { + rc = mpt3sas_atto_init(ioc); + if (rc) + return rc; + } + /* * Ensure correct T10 PI operation if vendor left EEDPTagMode * flag unset in NVDATA. @@ -5496,12 +5655,21 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) rc = _base_assign_fw_reported_qd(ioc); if (rc) return rc; - rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); - if (rc) - return rc; - rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); - if (rc) - return rc; + + /* + * ATTO doesn't use bios page 2 and 3 for bios settings. + */ + if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) + ioc->bios_pg3.BiosVersion = 0; + else { + rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); + if (rc) + return rc; + rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); + if (rc) + return rc; + } + rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8); if (rc) return rc; @@ -6895,7 +7063,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, /* send message 32-bits at a time */ for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) { - writel(request[i], &ioc->chip->Doorbell); + writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell); if ((_base_wait_for_doorbell_ack(ioc, 5))) failed = 1; } @@ -6914,16 +7082,16 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, } /* read the first two 16-bits, it gives the total length of the reply */ - reply[0] = ioc->base_readl(&ioc->chip->Doorbell) - & MPI2_DOORBELL_DATA_MASK; + reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) + & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); if ((_base_wait_for_doorbell_int(ioc, 5))) { ioc_err(ioc, "doorbell handshake int failed (line=%d)\n", __LINE__); return -EFAULT; } - reply[1] = ioc->base_readl(&ioc->chip->Doorbell) - & MPI2_DOORBELL_DATA_MASK; + reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) + & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); for (i = 2; i < default_reply->MsgLength * 2; i++) { @@ -6935,8 +7103,9 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, if (i >= reply_bytes/2) /* overflow case */ ioc->base_readl(&ioc->chip->Doorbell); else - reply[i] = ioc->base_readl(&ioc->chip->Doorbell) - & MPI2_DOORBELL_DATA_MASK; + reply[i] = le16_to_cpu( + ioc->base_readl(&ioc->chip->Doorbell) + & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); } diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h index e584cf0ffc238223dd301fab815619c04a98a3a4..05364aa15ecdb86b9da1688adf0d7a50dc38270a 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.h +++ b/drivers/scsi/mpt3sas/mpt3sas_base.h @@ -77,8 +77,8 @@ #define MPT3SAS_DRIVER_NAME "mpt3sas" #define MPT3SAS_AUTHOR "Avago Technologies " #define MPT3SAS_DESCRIPTION "LSI MPT Fusion SAS 3.0 Device Driver" -#define MPT3SAS_DRIVER_VERSION "42.100.00.00" -#define MPT3SAS_MAJOR_VERSION 42 +#define MPT3SAS_DRIVER_VERSION "43.100.00.00" +#define MPT3SAS_MAJOR_VERSION 43 #define MPT3SAS_MINOR_VERSION 100 #define MPT3SAS_BUILD_VERSION 0 #define MPT3SAS_RELEASE_VERSION 00 @@ -1652,6 +1652,32 @@ struct mpt3sas_debugfs_buffer { typedef u8 (*MPT_CALLBACK)(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply); +/* + * struct ATTO_SAS_NVRAM - ATTO NVRAM settings stored + * in Manufacturing page 1 used to get + * ATTO SasAddr. + */ +struct ATTO_SAS_NVRAM { + u8 Signature[4]; + u8 Version; +#define ATTO_SASNVR_VERSION 0 + + u8 Checksum; +#define ATTO_SASNVR_CKSUM_SEED 0x5A + u8 Pad[10]; + u8 SasAddr[8]; +#define ATTO_SAS_ADDR_ALIGN 64 + u8 Reserved[232]; +}; + +#define ATTO_SAS_ADDR_DEVNAME_BIAS 63 + +union ATTO_SAS_ADDRESS { + U8 b[8]; + U16 w[4]; + U32 d[2]; + U64 q; +}; /* base shared API */ extern struct list_head mpt3sas_ioc_list; @@ -1828,6 +1854,9 @@ int mpt3sas_config_get_number_hba_phys(struct MPT3SAS_ADAPTER *ioc, u8 *num_phys); int mpt3sas_config_get_manufacturing_pg0(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t *mpi_reply, Mpi2ManufacturingPage0_t *config_page); +int mpt3sas_config_get_manufacturing_pg1(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2ManufacturingPage1_t *config_page); + int mpt3sas_config_get_manufacturing_pg7(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t *mpi_reply, Mpi2ManufacturingPage7_t *config_page, u16 sz); @@ -1846,6 +1875,12 @@ int mpt3sas_config_get_bios_pg2(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage2_t *config_page); int mpt3sas_config_get_bios_pg3(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage3_t *config_page); +int mpt3sas_config_set_bios_pg4(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage4_t *config_page, + int sz_config_page); +int mpt3sas_config_get_bios_pg4(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage4_t *config_page, + int sz_config_page); int mpt3sas_config_get_iounit_pg0(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage0_t *config_page); int mpt3sas_config_get_sas_device_pg0(struct MPT3SAS_ADAPTER *ioc, diff --git a/drivers/scsi/mpt3sas/mpt3sas_config.c b/drivers/scsi/mpt3sas/mpt3sas_config.c index a8dd14c91efdb2d55be89a647f553589fd84fcc4..d114ef381c4451d788387ea8c5e79a8536431683 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_config.c +++ b/drivers/scsi/mpt3sas/mpt3sas_config.c @@ -540,6 +540,42 @@ mpt3sas_config_get_manufacturing_pg0(struct MPT3SAS_ADAPTER *ioc, return r; } +/** + * mpt3sas_config_get_manufacturing_pg1 - obtain manufacturing page 1 + * @ioc: per adapter object + * @mpi_reply: reply mf payload returned from firmware + * @config_page: contents of the config page + * Context: sleep. + * + * Return: 0 for success, non-zero for failure. + */ +int +mpt3sas_config_get_manufacturing_pg1(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2ManufacturingPage1_t *config_page) +{ + Mpi2ConfigRequest_t mpi_request; + int r; + + memset(&mpi_request, 0, sizeof(Mpi2ConfigRequest_t)); + mpi_request.Function = MPI2_FUNCTION_CONFIG; + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_HEADER; + mpi_request.Header.PageType = MPI2_CONFIG_PAGETYPE_MANUFACTURING; + mpi_request.Header.PageNumber = 1; + mpi_request.Header.PageVersion = MPI2_MANUFACTURING1_PAGEVERSION; + ioc->build_zero_len_sge_mpi(ioc, &mpi_request.PageBufferSGE); + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, NULL, 0); + if (r) + goto out; + + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT; + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, config_page, + sizeof(*config_page)); + out: + return r; +} + /** * mpt3sas_config_get_manufacturing_pg7 - obtain manufacturing page 7 * @ioc: per adapter object @@ -757,10 +793,98 @@ mpt3sas_config_get_bios_pg3(struct MPT3SAS_ADAPTER *ioc, Mpi2ConfigReply_t r = _config_request(ioc, &mpi_request, mpi_reply, MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, config_page, sizeof(*config_page)); + out: return r; } +/** + * mpt3sas_config_set_bios_pg4 - write out bios page 4 + * @ioc: per adapter object + * @mpi_reply: reply mf payload returned from firmware + * @config_page: contents of the config page + * @sz_config_pg: sizeof the config page + * Context: sleep. + * + * Return: 0 for success, non-zero for failure. + */ +int +mpt3sas_config_set_bios_pg4(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage4_t *config_page, + int sz_config_pg) +{ + Mpi2ConfigRequest_t mpi_request; + int r; + + memset(&mpi_request, 0, sizeof(Mpi2ConfigRequest_t)); + + mpi_request.Function = MPI2_FUNCTION_CONFIG; + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_HEADER; + mpi_request.Header.PageType = MPI2_CONFIG_PAGETYPE_BIOS; + mpi_request.Header.PageNumber = 4; + mpi_request.Header.PageVersion = MPI2_BIOSPAGE4_PAGEVERSION; + + ioc->build_zero_len_sge_mpi(ioc, &mpi_request.PageBufferSGE); + + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, NULL, 0); + if (r) + goto out; + + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_WRITE_CURRENT; + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, config_page, + sz_config_pg); + out: + return r; +} + +/** + * mpt3sas_config_get_bios_pg4 - read bios page 4 + * @ioc: per adapter object + * @mpi_reply: reply mf payload returned from firmware + * @config_page: contents of the config page + * @sz_config_pg: sizeof the config page + * Context: sleep. + * + * Return: 0 for success, non-zero for failure. + */ +int +mpt3sas_config_get_bios_pg4(struct MPT3SAS_ADAPTER *ioc, + Mpi2ConfigReply_t *mpi_reply, Mpi2BiosPage4_t *config_page, + int sz_config_pg) +{ + Mpi2ConfigRequest_t mpi_request; + int r; + + memset(&mpi_request, 0, sizeof(Mpi2ConfigRequest_t)); + mpi_request.Function = MPI2_FUNCTION_CONFIG; + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_HEADER; + mpi_request.Header.PageType = MPI2_CONFIG_PAGETYPE_BIOS; + mpi_request.Header.PageNumber = 4; + mpi_request.Header.PageVersion = MPI2_BIOSPAGE4_PAGEVERSION; + ioc->build_zero_len_sge_mpi(ioc, &mpi_request.PageBufferSGE); + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, NULL, 0); + if (r) + goto out; + + /* + * The sizeof the page is variable. Allow for just the + * size to be returned + */ + if (config_page && sz_config_pg) { + mpi_request.Action = MPI2_CONFIG_ACTION_PAGE_READ_CURRENT; + + r = _config_request(ioc, &mpi_request, mpi_reply, + MPT3_CONFIG_PAGE_DEFAULT_TIMEOUT, config_page, + sz_config_pg); + } + +out: + return r; +} + /** * mpt3sas_config_get_iounit_pg0 - obtain iounit page 0 * @ioc: per adapter object diff --git a/drivers/scsi/mpt3sas/mpt3sas_ctl.c b/drivers/scsi/mpt3sas/mpt3sas_ctl.c index 84c87c2c3e7e86acca900db239a62ed8a76e23c0..0d8b1e942dedaa186a3dc709fc236745182e916c 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_ctl.c +++ b/drivers/scsi/mpt3sas/mpt3sas_ctl.c @@ -948,6 +948,14 @@ _ctl_do_mpt_command(struct MPT3SAS_ADAPTER *ioc, struct mpt3_ioctl_command karg, break; } case MPI2_FUNCTION_FW_DOWNLOAD: + { + if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) { + ioc_info(ioc, "Firmware download not supported for ATTO HBA.\n"); + ret = -EPERM; + break; + } + fallthrough; + } case MPI2_FUNCTION_FW_UPLOAD: { ioc->build_sg(ioc, psge, data_out_dma, data_out_sz, data_in_dma, @@ -1686,6 +1694,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc, ioc->ctl_cmds.status = MPT3_CMD_PENDING; memset(ioc->ctl_cmds.reply, 0, ioc->reply_sz); mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); + memset(mpi_request, 0, ioc->request_sz); ioc->ctl_cmds.smid = smid; request_data = ioc->diag_buffer[buffer_type]; @@ -1787,6 +1796,7 @@ _ctl_diag_register_2(struct MPT3SAS_ADAPTER *ioc, if (rc && request_data) { dma_free_coherent(&ioc->pdev->dev, request_data_sz, request_data, request_data_dma); + ioc->diag_buffer[buffer_type] = NULL; ioc->diag_buffer_status[buffer_type] &= ~MPT3_DIAG_BUFFER_IS_DRIVER_ALLOCATED; } @@ -2163,6 +2173,7 @@ mpt3sas_send_diag_release(struct MPT3SAS_ADAPTER *ioc, u8 buffer_type, ioc->ctl_cmds.status = MPT3_CMD_PENDING; memset(ioc->ctl_cmds.reply, 0, ioc->reply_sz); mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); + memset(mpi_request, 0, ioc->request_sz); ioc->ctl_cmds.smid = smid; mpi_request->Function = MPI2_FUNCTION_DIAG_RELEASE; @@ -2417,6 +2428,7 @@ _ctl_diag_read_buffer(struct MPT3SAS_ADAPTER *ioc, void __user *arg) ioc->ctl_cmds.status = MPT3_CMD_PENDING; memset(ioc->ctl_cmds.reply, 0, ioc->reply_sz); mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); + memset(mpi_request, 0, ioc->request_sz); ioc->ctl_cmds.smid = smid; mpi_request->Function = MPI2_FUNCTION_DIAG_BUFFER_POST; diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index def37a7e59807dcc1cb0472a5f3fb95dc5290f82..8e24ebcebfe52cf9d18bfbe4704ef705c33b864b 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -3670,6 +3670,7 @@ static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc) fw_event = list_first_entry(&ioc->fw_event_list, struct fw_event_work, list); list_del_init(&fw_event->list); + fw_event_work_put(fw_event); } spin_unlock_irqrestore(&ioc->fw_event_lock, flags); @@ -3751,7 +3752,6 @@ _scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc) if (cancel_work_sync(&fw_event->work)) fw_event_work_put(fw_event); - fw_event_work_put(fw_event); } ioc->fw_events_cleanup = 0; } @@ -5156,6 +5156,19 @@ scsih_qcmd(struct Scsi_Host *shost, struct scsi_cmnd *scmd) /* invalid device handle */ handle = sas_target_priv_data->handle; + + /* + * Avoid error handling escallation when device is disconnected + */ + if (handle == MPT3SAS_INVALID_DEVICE_HANDLE || sas_device_priv_data->block) { + if (scmd->device->host->shost_state == SHOST_RECOVERY && + scmd->cmnd[0] == TEST_UNIT_READY) { + scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07); + scsi_done(scmd); + return 0; + } + } + if (handle == MPT3SAS_INVALID_DEVICE_HANDLE) { scmd->result = DID_NO_CONNECT << 16; scsi_done(scmd); @@ -11872,7 +11885,7 @@ out: * scsih_map_queues - map reply queues with request queues * @shost: SCSI host pointer */ -static int scsih_map_queues(struct Scsi_Host *shost) +static void scsih_map_queues(struct Scsi_Host *shost) { struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)shost->hostdata; @@ -11882,7 +11895,7 @@ static int scsih_map_queues(struct Scsi_Host *shost) int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors; if (shost->nr_hw_queues == 1) - return 0; + return; for (i = 0, qoff = 0; i < shost->nr_maps; i++) { map = &shost->tag_set.map[i]; @@ -11910,7 +11923,6 @@ static int scsih_map_queues(struct Scsi_Host *shost) qoff += map->nr_queues; } - return 0; } /* shost template for SAS 2.0 HBA devices */ @@ -11975,7 +11987,7 @@ static struct scsi_host_template mpt3sas_driver_template = { .sg_tablesize = MPT3SAS_SG_DEPTH, .max_sectors = 32767, .max_segment_size = 0xffffffff, - .cmd_per_lun = 7, + .cmd_per_lun = 128, .shost_groups = mpt3sas_host_groups, .sdev_groups = mpt3sas_dev_groups, .track_queue_depth = 1, @@ -12732,6 +12744,12 @@ static const struct pci_device_id mpt3sas_pci_table[] = { { MPI2_MFGPAGE_VENDORID_LSI, MPI26_MFGPAGE_DEVID_HARD_SEC_3816, PCI_ANY_ID, PCI_ANY_ID }, + /* + * ATTO Branded ExpressSAS H12xx GT + */ + { MPI2_MFGPAGE_VENDORID_ATTO, MPI26_MFGPAGE_DEVID_HARD_SEC_3816, + PCI_ANY_ID, PCI_ANY_ID }, + /* * Sea SI –> 0x00E4 Invalid, 0x00E7 Tampered */ diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c index 91d78d0a38fe55d700216673c356a8b54a6fd51f..628b08ba6770bff66f861aae9dc09c02a1a48b96 100644 --- a/drivers/scsi/pm8001/pm8001_hwi.c +++ b/drivers/scsi/pm8001/pm8001_hwi.c @@ -3612,6 +3612,10 @@ int pm8001_mpi_task_abort_resp(struct pm8001_hba_info *pm8001_ha, void *piomb) pm8001_dbg(pm8001_ha, FAIL, " TASK NULL. RETURNING !!!\n"); return -1; } + + if (t->task_proto == SAS_PROTOCOL_INTERNAL_ABORT) + atomic_dec(&pm8001_dev->running_req); + ts = &t->task_status; if (status != 0) pm8001_dbg(pm8001_ha, FAIL, "task abort failed status 0x%x ,tag = 0x%x, scp= 0x%x\n", diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index a0028e130a7e45aecc057a7bc2f85963d3ea465e..2ff2fac1e403d4099fe9e90fa488d8f85eed3ce7 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -81,7 +81,7 @@ LIST_HEAD(hba_list); struct workqueue_struct *pm8001_wq; -static int pm8001_map_queues(struct Scsi_Host *shost) +static void pm8001_map_queues(struct Scsi_Host *shost) { struct sas_ha_struct *sha = SHOST_TO_SAS_HA(shost); struct pm8001_hba_info *pm8001_ha = sha->lldd_ha; diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h index c5e3f380a01cd1390496d3723b979e8e0826d1af..b08f52673889c682ec5fb003797fc0c5d1b4af4b 100644 --- a/drivers/scsi/pm8001/pm8001_sas.h +++ b/drivers/scsi/pm8001/pm8001_sas.h @@ -612,7 +612,7 @@ struct fw_control_info { operations.*/ u32 reserved;/* padding required for 64 bit alignment */ - u8 buffer[1];/* Start of buffer */ + u8 buffer[];/* Start of buffer */ }; struct fw_control_ex { struct fw_control_info *fw_control; diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c index 3d6b137314f3f215862f898c9506a6c07f6aba54..e045c6e25090281d31da505aac7f06f66b19d193 100644 --- a/drivers/scsi/qedf/qedf_main.c +++ b/drivers/scsi/qedf/qedf_main.c @@ -1921,6 +1921,27 @@ static int qedf_vport_create(struct fc_vport *vport, bool disabled) fc_vport_setlink(vn_port); } + /* Set symbolic node name */ + if (base_qedf->pdev->device == QL45xxx) + snprintf(fc_host_symbolic_name(vn_port->host), 256, + "Marvell FastLinQ 45xxx FCoE v%s", QEDF_VERSION); + + if (base_qedf->pdev->device == QL41xxx) + snprintf(fc_host_symbolic_name(vn_port->host), 256, + "Marvell FastLinQ 41xxx FCoE v%s", QEDF_VERSION); + + /* Set supported speed */ + fc_host_supported_speeds(vn_port->host) = n_port->link_supported_speeds; + + /* Set speed */ + vn_port->link_speed = n_port->link_speed; + + /* Set port type */ + fc_host_port_type(vn_port->host) = FC_PORTTYPE_NPIV; + + /* Set maxframe size */ + fc_host_maxframe_size(vn_port->host) = n_port->mfs; + QEDF_INFO(&(base_qedf->dbg_ctx), QEDF_LOG_NPIV, "vn_port=%p.\n", vn_port); @@ -3686,11 +3707,6 @@ err2: err1: scsi_host_put(lport->host); err0: - if (qedf) { - QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC, "Probe done.\n"); - - clear_bit(QEDF_PROBING, &qedf->flags); - } return rc; } diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c index cecfb2cb4c7beb87cc61d8519c20be53753fa2c9..df2fe7bd26d1b72b0ecd622038c6afe73e29d327 100644 --- a/drivers/scsi/qedi/qedi_main.c +++ b/drivers/scsi/qedi/qedi_main.c @@ -618,7 +618,7 @@ static int qedi_cm_alloc_mem(struct qedi_ctx *qedi) sizeof(struct qedi_endpoint *)), GFP_KERNEL); if (!qedi->ep_tbl) return -ENOMEM; - port_id = prandom_u32() % QEDI_LOCAL_PORT_RANGE; + port_id = prandom_u32_max(QEDI_LOCAL_PORT_RANGE); if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE, QEDI_LOCAL_PORT_MIN, port_id)) { qedi_cm_free_mem(qedi); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index 5db9bf69dcff04ec68bac1579dff5b461518761c..cd75b179410d78e2add57805abac420927a2d4e9 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -2519,19 +2519,23 @@ qla2x00_get_flash_image_status(struct bsg_job *bsg_job) qla27xx_get_active_image(vha, &active_regions); regions.global_image = active_regions.global; + if (IS_QLA27XX(ha)) + regions.nvme_params = QLA27XX_PRIMARY_IMAGE; + if (IS_QLA28XX(ha)) { qla28xx_get_aux_images(vha, &active_regions); regions.board_config = active_regions.aux.board_config; regions.vpd_nvram = active_regions.aux.vpd_nvram; regions.npiv_config_0_1 = active_regions.aux.npiv_config_0_1; regions.npiv_config_2_3 = active_regions.aux.npiv_config_2_3; + regions.nvme_params = active_regions.aux.nvme_params; } ql_dbg(ql_dbg_user, vha, 0x70e1, - "%s(%lu): FW=%u BCFG=%u VPDNVR=%u NPIV01=%u NPIV02=%u\n", + "%s(%lu): FW=%u BCFG=%u VPDNVR=%u NPIV01=%u NPIV02=%u NVME_PARAMS=%u\n", __func__, vha->host_no, regions.global_image, regions.board_config, regions.vpd_nvram, - regions.npiv_config_0_1, regions.npiv_config_2_3); + regions.npiv_config_0_1, regions.npiv_config_2_3, regions.nvme_params); sg_copy_from_buffer(bsg_job->reply_payload.sg_list, bsg_job->reply_payload.sg_cnt, ®ions, sizeof(regions)); diff --git a/drivers/scsi/qla2xxx/qla_bsg.h b/drivers/scsi/qla2xxx/qla_bsg.h index bb64b9c5a74bbe7a39f62d23128f8ca9705836a9..d38dab0a07e83357b8bbf522e5398312b0793d9d 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.h +++ b/drivers/scsi/qla2xxx/qla_bsg.h @@ -314,7 +314,8 @@ struct qla_active_regions { uint8_t vpd_nvram; uint8_t npiv_config_0_1; uint8_t npiv_config_2_3; - uint8_t reserved[32]; + uint8_t nvme_params; + uint8_t reserved[31]; } __packed; #include "qla_edif_bsg.h" diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c index 7cf1f78cbaeee6e0fec9f0ba8460f8d187b6cac0..d7e8454304ceea98feddf77802104a32ba1e0c1c 100644 --- a/drivers/scsi/qla2xxx/qla_dbg.c +++ b/drivers/scsi/qla2xxx/qla_dbg.c @@ -2455,7 +2455,7 @@ qla83xx_fw_dump_failed_0: /****************************************************************************/ /* Write the debug message prefix into @pbuf. */ -static void ql_dbg_prefix(char *pbuf, int pbuf_size, +static void ql_dbg_prefix(char *pbuf, int pbuf_size, struct pci_dev *pdev, const scsi_qla_host_t *vha, uint msg_id) { if (vha) { @@ -2464,6 +2464,9 @@ static void ql_dbg_prefix(char *pbuf, int pbuf_size, /* []-:: */ snprintf(pbuf, pbuf_size, "%s [%s]-%04x:%lu: ", QL_MSGHDR, dev_name(&(pdev->dev)), msg_id, vha->host_no); + } else if (pdev) { + snprintf(pbuf, pbuf_size, "%s [%s]-%04x: : ", QL_MSGHDR, + dev_name(&pdev->dev), msg_id); } else { /* []-: : */ snprintf(pbuf, pbuf_size, "%s [%s]-%04x: : ", QL_MSGHDR, @@ -2491,20 +2494,20 @@ ql_dbg(uint level, scsi_qla_host_t *vha, uint id, const char *fmt, ...) struct va_format vaf; char pbuf[64]; - if (!ql_mask_match(level) && !trace_ql_dbg_log_enabled()) + ql_ktrace(1, level, pbuf, NULL, vha, id, fmt); + + if (!ql_mask_match(level)) return; + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, vha, id); + va_start(va, fmt); vaf.fmt = fmt; vaf.va = &va; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), vha, id); - - if (!ql_mask_match(level)) - trace_ql_dbg_log(pbuf, &vaf); - else - pr_warn("%s%pV", pbuf, &vaf); + pr_warn("%s%pV", pbuf, &vaf); va_end(va); @@ -2533,6 +2536,9 @@ ql_dbg_pci(uint level, struct pci_dev *pdev, uint id, const char *fmt, ...) if (pdev == NULL) return; + + ql_ktrace(1, level, pbuf, pdev, NULL, id, fmt); + if (!ql_mask_match(level)) return; @@ -2541,7 +2547,9 @@ ql_dbg_pci(uint level, struct pci_dev *pdev, uint id, const char *fmt, ...) vaf.fmt = fmt; vaf.va = &va; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, id + ql_dbg_offset); + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), pdev, NULL, + id + ql_dbg_offset); pr_warn("%s%pV", pbuf, &vaf); va_end(va); @@ -2570,7 +2578,10 @@ ql_log(uint level, scsi_qla_host_t *vha, uint id, const char *fmt, ...) if (level > ql_errlev) return; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), vha, id); + ql_ktrace(0, level, pbuf, NULL, vha, id, fmt); + + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, vha, id); va_start(va, fmt); @@ -2621,7 +2632,10 @@ ql_log_pci(uint level, struct pci_dev *pdev, uint id, const char *fmt, ...) if (level > ql_errlev) return; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, id); + ql_ktrace(0, level, pbuf, pdev, NULL, id, fmt); + + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), pdev, NULL, id); va_start(va, fmt); @@ -2716,7 +2730,11 @@ ql_log_qp(uint32_t level, struct qla_qpair *qpair, int32_t id, if (level > ql_errlev) return; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), qpair ? qpair->vha : NULL, id); + ql_ktrace(0, level, pbuf, NULL, qpair ? qpair->vha : NULL, id, fmt); + + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, + qpair ? qpair->vha : NULL, id); va_start(va, fmt); @@ -2762,6 +2780,8 @@ ql_dbg_qp(uint32_t level, struct qla_qpair *qpair, int32_t id, struct va_format vaf; char pbuf[128]; + ql_ktrace(1, level, pbuf, NULL, qpair ? qpair->vha : NULL, id, fmt); + if (!ql_mask_match(level)) return; @@ -2770,8 +2790,10 @@ ql_dbg_qp(uint32_t level, struct qla_qpair *qpair, int32_t id, vaf.fmt = fmt; vaf.va = &va; - ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), qpair ? qpair->vha : NULL, - id + ql_dbg_offset); + if (!pbuf[0]) /* set by ql_ktrace */ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), NULL, + qpair ? qpair->vha : NULL, id + ql_dbg_offset); + pr_warn("%s%pV", pbuf, &vaf); va_end(va); diff --git a/drivers/scsi/qla2xxx/qla_dbg.h b/drivers/scsi/qla2xxx/qla_dbg.h index feeb1666227f1235de63b4c891195c9d95e76226..70482b55d240ecf720ada456713c32ab22c2c856 100644 --- a/drivers/scsi/qla2xxx/qla_dbg.h +++ b/drivers/scsi/qla2xxx/qla_dbg.h @@ -385,3 +385,46 @@ ql_mask_match(uint level) return level && ((level & ql2xextended_error_logging) == level); } + +static inline int +ql_mask_match_ext(uint level, int *log_tunable) +{ + if (*log_tunable == 1) + *log_tunable = QL_DBG_DEFAULT1_MASK; + + return (level & *log_tunable) == level; +} + +/* Assumes local variable pbuf and pbuf_ready present. */ +#define ql_ktrace(dbg_msg, level, pbuf, pdev, vha, id, fmt) do { \ + struct va_format _vaf; \ + va_list _va; \ + u32 dbg_off = dbg_msg ? ql_dbg_offset : 0; \ + \ + pbuf[0] = 0; \ + if (!trace_ql_dbg_log_enabled()) \ + break; \ + \ + if (dbg_msg && !ql_mask_match_ext(level, \ + &ql2xextended_error_logging_ktrace)) \ + break; \ + \ + ql_dbg_prefix(pbuf, ARRAY_SIZE(pbuf), pdev, vha, id + dbg_off); \ + \ + va_start(_va, fmt); \ + _vaf.fmt = fmt; \ + _vaf.va = &_va; \ + \ + trace_ql_dbg_log(pbuf, &_vaf); \ + \ + va_end(_va); \ +} while (0) + +#define QLA_ENABLE_KERNEL_TRACING + +#ifdef QLA_ENABLE_KERNEL_TRACING +#define QLA_TRACE_ENABLE(_tr) \ + trace_array_set_clr_event(_tr, "qla", NULL, true) +#else /* QLA_ENABLE_KERNEL_TRACING */ +#define QLA_TRACE_ENABLE(_tr) +#endif /* QLA_ENABLE_KERNEL_TRACING */ diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index 3ec6a200942ee90f60e5783e6e8223790a6cbc33..802eec6407d9accf0a9b480487550e8be97940bd 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -35,6 +35,11 @@ #include +#define QLA_DFS_DEFINE_DENTRY(_debugfs_file_name) \ + struct dentry *dfs_##_debugfs_file_name +#define QLA_DFS_ROOT_DEFINE_DENTRY(_debugfs_file_name) \ + struct dentry *qla_dfs_##_debugfs_file_name + /* Big endian Fibre Channel S_ID (source ID) or D_ID (destination ID). */ typedef struct { uint8_t domain; @@ -4768,6 +4773,7 @@ struct active_regions { uint8_t vpd_nvram; uint8_t npiv_config_0_1; uint8_t npiv_config_2_3; + uint8_t nvme_params; } aux; }; @@ -5052,6 +5058,7 @@ struct qla27xx_image_status { #define QLA28XX_AUX_IMG_VPD_NVRAM BIT_1 #define QLA28XX_AUX_IMG_NPIV_CONFIG_0_1 BIT_2 #define QLA28XX_AUX_IMG_NPIV_CONFIG_2_3 BIT_3 +#define QLA28XX_AUX_IMG_NVME_PARAMS BIT_4 #define SET_VP_IDX 1 #define SET_AL_PA 2 diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index 85bd0e468d43edaff3610ad40f0fc6bea4bbc924..777808af56347311363692124d3f0dbe802e09da 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -489,6 +489,99 @@ qla_dfs_naqp_show(struct seq_file *s, void *unused) return 0; } +/* + * Helper macros for setting up debugfs entries. + * _name: The name of the debugfs entry + * _ctx_struct: The context that was passed when creating the debugfs file + * + * QLA_DFS_SETUP_RD could be used when there is only a show function. + * - show function take the name qla_dfs__show + * + * QLA_DFS_SETUP_RW could be used when there are both show and write functions. + * - show function take the name qla_dfs__show + * - write function take the name qla_dfs__write + * + * To have a new debugfs entry, do: + * 1. Create a "struct dentry *" in the appropriate structure in the format + * dfs_ + * 2. Setup debugfs entries using QLA_DFS_SETUP_RD / QLA_DFS_SETUP_RW + * 3. Create debugfs file in qla2x00_dfs_setup() using QLA_DFS_CREATE_FILE + * or QLA_DFS_ROOT_CREATE_FILE + * 4. Remove debugfs file in qla2x00_dfs_remove() using QLA_DFS_REMOVE_FILE + * or QLA_DFS_ROOT_REMOVE_FILE + * + * Example for creating "TEST" sysfs file: + * 1. struct qla_hw_data { ... struct dentry *dfs_TEST; } + * 2. QLA_DFS_SETUP_RD(TEST, scsi_qla_host_t); + * 3. In qla2x00_dfs_setup(): + * QLA_DFS_CREATE_FILE(ha, TEST, 0600, ha->dfs_dir, vha); + * 4. In qla2x00_dfs_remove(): + * QLA_DFS_REMOVE_FILE(ha, TEST); + */ +#define QLA_DFS_SETUP_RD(_name, _ctx_struct) \ +static int \ +qla_dfs_##_name##_open(struct inode *inode, struct file *file) \ +{ \ + _ctx_struct *__ctx = inode->i_private; \ + \ + return single_open(file, qla_dfs_##_name##_show, __ctx); \ +} \ + \ +static const struct file_operations qla_dfs_##_name##_ops = { \ + .open = qla_dfs_##_name##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +}; + +#define QLA_DFS_SETUP_RW(_name, _ctx_struct) \ +static int \ +qla_dfs_##_name##_open(struct inode *inode, struct file *file) \ +{ \ + _ctx_struct *__ctx = inode->i_private; \ + \ + return single_open(file, qla_dfs_##_name##_show, __ctx); \ +} \ + \ +static const struct file_operations qla_dfs_##_name##_ops = { \ + .open = qla_dfs_##_name##_open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ + .write = qla_dfs_##_name##_write, \ +}; + +#define QLA_DFS_ROOT_CREATE_FILE(_name, _perm, _ctx) \ + do { \ + if (!qla_dfs_##_name) \ + qla_dfs_##_name = debugfs_create_file(#_name, \ + _perm, qla2x00_dfs_root, _ctx, \ + &qla_dfs_##_name##_ops); \ + } while (0) + +#define QLA_DFS_ROOT_REMOVE_FILE(_name) \ + do { \ + if (qla_dfs_##_name) { \ + debugfs_remove(qla_dfs_##_name); \ + qla_dfs_##_name = NULL; \ + } \ + } while (0) + +#define QLA_DFS_CREATE_FILE(_struct, _name, _perm, _parent, _ctx) \ + do { \ + (_struct)->dfs_##_name = debugfs_create_file(#_name, \ + _perm, _parent, _ctx, \ + &qla_dfs_##_name##_ops) \ + } while (0) + +#define QLA_DFS_REMOVE_FILE(_struct, _name) \ + do { \ + if ((_struct)->dfs_##_name) { \ + debugfs_remove((_struct)->dfs_##_name); \ + (_struct)->dfs_##_name = NULL; \ + } \ + } while (0) + static int qla_dfs_naqp_open(struct inode *inode, struct file *file) { diff --git a/drivers/scsi/qla2xxx/qla_edif.c b/drivers/scsi/qla2xxx/qla_edif.c index 400a8b6f39821f0e4c713f0447b899d05fa308e1..00ccc41cef147279e884e5eab3ba5143ac0a1905 100644 --- a/drivers/scsi/qla2xxx/qla_edif.c +++ b/drivers/scsi/qla2xxx/qla_edif.c @@ -1551,7 +1551,7 @@ qla24xx_sadb_update(struct bsg_job *bsg_job) ql_dbg(ql_dbg_edif, vha, 0x70a3, "Failed to find port= %06x\n", sa_frame.port_id.b24); rval = -EINVAL; - SET_DID_STATUS(bsg_reply->result, DID_TARGET_FAILURE); + SET_DID_STATUS(bsg_reply->result, DID_NO_CONNECT); goto done; } diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h index 361015b5763ef5e944c50d74907384bdeea69168..f307beed9d29d9ccbb61edad8c40216db6bf6046 100644 --- a/drivers/scsi/qla2xxx/qla_fw.h +++ b/drivers/scsi/qla2xxx/qla_fw.h @@ -1675,6 +1675,7 @@ struct qla_flt_location { #define FLT_REG_VPD_SEC_27XX_1 0x52 #define FLT_REG_VPD_SEC_27XX_2 0xD8 #define FLT_REG_VPD_SEC_27XX_3 0xDA +#define FLT_REG_NVME_PARAMS_27XX 0x21 /* 28xx */ #define FLT_REG_AUX_IMG_PRI_28XX 0x125 @@ -1691,6 +1692,8 @@ struct qla_flt_location { #define FLT_REG_MPI_SEC_28XX 0xF0 #define FLT_REG_PEP_PRI_28XX 0xD1 #define FLT_REG_PEP_SEC_28XX 0xF1 +#define FLT_REG_NVME_PARAMS_PRI_28XX 0x14E +#define FLT_REG_NVME_PARAMS_SEC_28XX 0x179 struct qla_flt_region { __le16 code; diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h index 5dd2932382ee3357f76e36436c4326c8de19def0..e3256e721be1476debceaef96a2dbd2afeab6b65 100644 --- a/drivers/scsi/qla2xxx/qla_gbl.h +++ b/drivers/scsi/qla2xxx/qla_gbl.h @@ -70,8 +70,6 @@ extern int qla2x00_async_prlo(struct scsi_qla_host *, fc_port_t *); extern int qla2x00_async_adisc(struct scsi_qla_host *, fc_port_t *, uint16_t *); extern int qla2x00_async_tm_cmd(fc_port_t *, uint32_t, uint32_t, uint32_t); -extern void qla2x00_async_login_done(struct scsi_qla_host *, fc_port_t *, - uint16_t *); struct qla_work_evt *qla2x00_alloc_work(struct scsi_qla_host *, enum qla_work_type); extern int qla24xx_async_gnl(struct scsi_qla_host *, fc_port_t *); @@ -163,6 +161,7 @@ extern int ql2xrdpenable; extern int ql2xsmartsan; extern int ql2xallocfwdump; extern int ql2xextended_error_logging; +extern int ql2xextended_error_logging_ktrace; extern int ql2xiidmaenable; extern int ql2xmqsupport; extern int ql2xfwloadbin; @@ -193,8 +192,6 @@ extern int ql2xsecenable; extern int ql2xenforce_iocb_limit; extern int ql2xabts_wait_nvme; extern u32 ql2xnvme_queues; -extern int ql2xrspq_follow_inptr; -extern int ql2xrspq_follow_inptr_legacy; extern int qla2x00_loop_reset(scsi_qla_host_t *); extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int); @@ -279,7 +276,6 @@ extern int qla24xx_vport_create_req_sanity_check(struct fc_vport *); extern scsi_qla_host_t *qla24xx_create_vhost(struct fc_vport *); extern void qla2x00_sp_free_dma(srb_t *sp); -extern char *qla2x00_get_fw_version_str(struct scsi_qla_host *, char *); extern void qla2x00_mark_device_lost(scsi_qla_host_t *, fc_port_t *, int); extern void qla2x00_mark_all_devices_lost(scsi_qla_host_t *); @@ -616,7 +612,6 @@ void __qla_consume_iocb(struct scsi_qla_host *vha, void **pkt, struct rsp_que ** /* * Global Function Prototypes in qla_sup.c source file. */ -extern void qla2x00_release_nvram_protection(scsi_qla_host_t *); extern int qla24xx_read_flash_data(scsi_qla_host_t *, uint32_t *, uint32_t, uint32_t); extern uint8_t *qla2x00_read_nvram_data(scsi_qla_host_t *, void *, uint32_t, @@ -788,12 +783,6 @@ extern void qla2x00_init_response_q_entries(struct rsp_que *); extern int qla25xx_delete_req_que(struct scsi_qla_host *, struct req_que *); extern int qla25xx_delete_rsp_que(struct scsi_qla_host *, struct rsp_que *); extern int qla25xx_delete_queues(struct scsi_qla_host *); -extern uint16_t qla24xx_rd_req_reg(struct qla_hw_data *, uint16_t); -extern uint16_t qla25xx_rd_req_reg(struct qla_hw_data *, uint16_t); -extern void qla24xx_wrt_req_reg(struct qla_hw_data *, uint16_t, uint16_t); -extern void qla25xx_wrt_req_reg(struct qla_hw_data *, uint16_t, uint16_t); -extern void qla25xx_wrt_rsp_reg(struct qla_hw_data *, uint16_t, uint16_t); -extern void qla24xx_wrt_rsp_reg(struct qla_hw_data *, uint16_t, uint16_t); /* qlafx00 related functions */ extern int qlafx00_pci_config(struct scsi_qla_host *); @@ -878,8 +867,6 @@ extern void qla82xx_init_flags(struct qla_hw_data *); extern void qla82xx_set_drv_active(scsi_qla_host_t *); extern int qla82xx_wr_32(struct qla_hw_data *, ulong, u32); extern int qla82xx_rd_32(struct qla_hw_data *, ulong); -extern int qla82xx_rdmem(struct qla_hw_data *, u64, void *, int); -extern int qla82xx_wrmem(struct qla_hw_data *, u64, void *, int); /* ISP 8021 IDC */ extern void qla82xx_clear_drv_active(struct qla_hw_data *); diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index e7fe0e52c11d4a4bbf4986427dc25ec590055ed1..e12db95de6883f351bed957f8ce0b35f871ba7d6 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -7933,6 +7933,9 @@ qla28xx_component_status( active_regions->aux.npiv_config_2_3 = qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_NPIV_CONFIG_2_3); + + active_regions->aux.nvme_params = + qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_NVME_PARAMS); } static int @@ -8041,11 +8044,12 @@ check_valid_image: } ql_dbg(ql_dbg_init, vha, 0x018f, - "aux images active: BCFG=%u VPD/NVR=%u NPIV0/1=%u NPIV2/3=%u\n", + "aux images active: BCFG=%u VPD/NVR=%u NPIV0/1=%u NPIV2/3=%u, NVME=%u\n", active_regions->aux.board_config, active_regions->aux.vpd_nvram, active_regions->aux.npiv_config_0_1, - active_regions->aux.npiv_config_2_3); + active_regions->aux.npiv_config_2_3, + active_regions->aux.nvme_params); } void diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 76e79f350a22657abcdaf9e7b8adee000c59c00d..e19fde304e5c6891b44db73496f9fefe69643398 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -3764,7 +3764,7 @@ void qla24xx_process_response_queue(struct scsi_qla_host *vha, struct purex_entry_24xx *purex_entry; struct purex_item *pure_item; u16 rsp_in = 0, cur_ring_index; - int follow_inptr, is_shadow_hba; + int is_shadow_hba; if (!ha->flags.fw_started) return; @@ -3774,25 +3774,18 @@ void qla24xx_process_response_queue(struct scsi_qla_host *vha, qla_cpu_update(rsp->qpair, smp_processor_id()); } -#define __update_rsp_in(_update, _is_shadow_hba, _rsp, _rsp_in) \ +#define __update_rsp_in(_is_shadow_hba, _rsp, _rsp_in) \ do { \ - if (_update) { \ - _rsp_in = _is_shadow_hba ? *(_rsp)->in_ptr : \ + _rsp_in = _is_shadow_hba ? *(_rsp)->in_ptr : \ rd_reg_dword_relaxed((_rsp)->rsp_q_in); \ - } \ } while (0) is_shadow_hba = IS_SHADOW_REG_CAPABLE(ha); - follow_inptr = is_shadow_hba ? ql2xrspq_follow_inptr : - ql2xrspq_follow_inptr_legacy; - __update_rsp_in(follow_inptr, is_shadow_hba, rsp, rsp_in); + __update_rsp_in(is_shadow_hba, rsp, rsp_in); - while ((likely(follow_inptr && - rsp->ring_index != rsp_in && - rsp->ring_ptr->signature != RESPONSE_PROCESSED)) || - (!follow_inptr && - rsp->ring_ptr->signature != RESPONSE_PROCESSED)) { + while (rsp->ring_index != rsp_in && + rsp->ring_ptr->signature != RESPONSE_PROCESSED) { pkt = (struct sts_entry_24xx *)rsp->ring_ptr; cur_ring_index = rsp->ring_index; @@ -3906,8 +3899,7 @@ process_err: } pure_item = qla27xx_copy_fpin_pkt(vha, (void **)&pkt, &rsp); - __update_rsp_in(follow_inptr, is_shadow_hba, - rsp, rsp_in); + __update_rsp_in(is_shadow_hba, rsp, rsp_in); if (!pure_item) break; qla24xx_queue_purex_item(vha, pure_item, diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 7450c3458be7e0f541e8a4d11e02222ddc1c5d54..02fdeb0d31ec4afed59a923a75670a8a1dbf6ef3 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -684,12 +684,8 @@ static void qla_nvme_map_queues(struct nvme_fc_local_port *lport, struct blk_mq_queue_map *map) { struct scsi_qla_host *vha = lport->private; - int rc; - rc = blk_mq_pci_map_queues(map, vha->hw->pdev, vha->irq_offset); - if (rc) - ql_log(ql_log_warn, vha, 0x21de, - "pci map queue failed 0x%x", rc); + blk_mq_pci_map_queues(map, vha->hw->pdev, vha->irq_offset); } static void qla_nvme_localport_delete(struct nvme_fc_local_port *lport) diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 0bd0fd1042dfe24ce86ac8addb7dc2f2cc866b7a..2c85f3cce7264d82f3c18ccebeec5eac09656226 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include @@ -35,6 +37,8 @@ static int apidev_major; */ struct kmem_cache *srb_cachep; +static struct trace_array *qla_trc_array; + int ql2xfulldump_on_mpifail; module_param(ql2xfulldump_on_mpifail, int, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(ql2xfulldump_on_mpifail, @@ -117,6 +121,11 @@ MODULE_PARM_DESC(ql2xextended_error_logging, "ql2xextended_error_logging=1).\n" "\t\tDo LOGICAL OR of the value to enable more than one level"); +int ql2xextended_error_logging_ktrace = 1; +module_param(ql2xextended_error_logging_ktrace, int, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(ql2xextended_error_logging_ktrace, + "Same BIT definition as ql2xextended_error_logging, but used to control logging to kernel trace buffer (default=1).\n"); + int ql2xshiftctondsd = 6; module_param(ql2xshiftctondsd, int, S_IRUGO); MODULE_PARM_DESC(ql2xshiftctondsd, @@ -333,24 +342,14 @@ MODULE_PARM_DESC(ql2xabts_wait_nvme, "To wait for ABTS response on I/O timeouts for NVMe. (default: 1)"); -u32 ql2xdelay_before_pci_error_handling = 5; +static u32 ql2xdelay_before_pci_error_handling = 5; module_param(ql2xdelay_before_pci_error_handling, uint, 0644); MODULE_PARM_DESC(ql2xdelay_before_pci_error_handling, "Number of seconds delayed before qla begin PCI error self-handling (default: 5).\n"); -int ql2xrspq_follow_inptr = 1; -module_param(ql2xrspq_follow_inptr, int, 0644); -MODULE_PARM_DESC(ql2xrspq_follow_inptr, - "Follow RSP IN pointer for RSP updates for HBAs 27xx and newer (default: 1)."); - -int ql2xrspq_follow_inptr_legacy = 1; -module_param(ql2xrspq_follow_inptr_legacy, int, 0644); -MODULE_PARM_DESC(ql2xrspq_follow_inptr_legacy, - "Follow RSP IN pointer for RSP updates for HBAs older than 27XX. (default: 1)."); - static void qla2x00_clear_drv_active(struct qla_hw_data *); static void qla2x00_free_device(scsi_qla_host_t *); -static int qla2xxx_map_queues(struct Scsi_Host *shost); +static void qla2xxx_map_queues(struct Scsi_Host *shost); static void qla2x00_destroy_deferred_work(struct qla_hw_data *); u32 ql2xnvme_queues = DEF_NVME_HW_QUEUES; @@ -2849,6 +2848,27 @@ static void qla2x00_iocb_work_fn(struct work_struct *work) spin_unlock_irqrestore(&vha->work_lock, flags); } +static void +qla_trace_init(void) +{ + qla_trc_array = trace_array_get_by_name("qla2xxx"); + if (!qla_trc_array) { + ql_log(ql_log_fatal, NULL, 0x0001, + "Unable to create qla2xxx trace instance, instance logging will be disabled.\n"); + return; + } + + QLA_TRACE_ENABLE(qla_trc_array); +} + +static void +qla_trace_uninit(void) +{ + if (!qla_trc_array) + return; + trace_array_put(qla_trc_array); +} + /* * PCI driver interface */ @@ -3530,7 +3550,7 @@ skip_dpc: qla_dual_mode_enabled(base_vha)) scsi_scan_host(host); else - ql_dbg(ql_dbg_init, base_vha, 0x0122, + ql_log(ql_log_info, base_vha, 0x0122, "skipping scsi_scan_host() for non-initiator port\n"); qla2x00_alloc_sysfs_attr(base_vha); @@ -7994,17 +8014,15 @@ qla_pci_reset_done(struct pci_dev *pdev) clear_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags); } -static int qla2xxx_map_queues(struct Scsi_Host *shost) +static void qla2xxx_map_queues(struct Scsi_Host *shost) { - int rc; scsi_qla_host_t *vha = (scsi_qla_host_t *)shost->hostdata; struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT]; if (USER_CTRL_IRQ(vha->hw) || !vha->hw->mqiobase) - rc = blk_mq_map_queues(qmap); + blk_mq_map_queues(qmap); else - rc = blk_mq_pci_map_queues(qmap, vha->hw->pdev, vha->irq_offset); - return rc; + blk_mq_pci_map_queues(qmap, vha->hw->pdev, vha->irq_offset); } struct scsi_host_template qla2xxx_driver_template = { @@ -8191,6 +8209,8 @@ qla2x00_module_init(void) BUILD_BUG_ON(sizeof(sw_info_t) != 32); BUILD_BUG_ON(sizeof(target_id_t) != 2); + qla_trace_init(); + /* Allocate cache for SRBs. */ srb_cachep = kmem_cache_create("qla2xxx_srbs", sizeof(srb_t), 0, SLAB_HWCACHE_ALIGN, NULL); @@ -8269,6 +8289,8 @@ qlt_exit: destroy_cache: kmem_cache_destroy(srb_cachep); + + qla_trace_uninit(); return ret; } @@ -8287,6 +8309,7 @@ qla2x00_module_exit(void) fc_release_transport(qla2xxx_transport_template); qlt_exit(); kmem_cache_destroy(srb_cachep); + qla_trace_uninit(); } module_init(qla2x00_module_init); diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index 2b2f682883752956f49281d0c91ae35c5fc97884..bb754a95080231b95eb226b3efe7bcfc4b1d1beb 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -1557,11 +1557,11 @@ int qlt_stop_phase1(struct qla_tgt *tgt) ql_dbg(ql_dbg_tgt_mgt, vha, 0xf009, "Waiting for sess works (tgt %p)", tgt); spin_lock_irqsave(&tgt->sess_work_lock, flags); - while (!list_empty(&tgt->sess_works_list)) { + do { spin_unlock_irqrestore(&tgt->sess_work_lock, flags); - flush_scheduled_work(); + flush_work(&tgt->sess_work); spin_lock_irqsave(&tgt->sess_work_lock, flags); - } + } while (!list_empty(&tgt->sess_works_list)); spin_unlock_irqrestore(&tgt->sess_work_lock, flags); ql_dbg(ql_dbg_tgt_mgt, vha, 0xf00a, @@ -2151,8 +2151,10 @@ static int __qlt_24xx_handle_abts(struct scsi_qla_host *vha, abort_cmd = ha->tgt.tgt_ops->find_cmd_by_tag(sess, le32_to_cpu(abts->exchange_addr_to_abort)); - if (!abort_cmd) + if (!abort_cmd) { + mempool_free(mcmd, qla_tgt_mgmt_cmd_mempool); return -EIO; + } mcmd->unpacked_lun = abort_cmd->se_cmd.orig_fe_lun; if (abort_cmd->qpair) { @@ -6334,69 +6336,6 @@ out_term: spin_unlock_irqrestore(&ha->hardware_lock, flags); } -static void qlt_tmr_work(struct qla_tgt *tgt, - struct qla_tgt_sess_work_param *prm) -{ - struct atio_from_isp *a = &prm->tm_iocb2; - struct scsi_qla_host *vha = tgt->vha; - struct qla_hw_data *ha = vha->hw; - struct fc_port *sess; - unsigned long flags; - be_id_t s_id; - int rc; - u64 unpacked_lun; - int fn; - void *iocb; - - spin_lock_irqsave(&ha->tgt.sess_lock, flags); - - if (tgt->tgt_stop) - goto out_term2; - - s_id = prm->tm_iocb2.u.isp24.fcp_hdr.s_id; - sess = ha->tgt.tgt_ops->find_sess_by_s_id(vha, s_id); - if (!sess) { - spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); - - sess = qlt_make_local_sess(vha, s_id); - /* sess has got an extra creation ref */ - - spin_lock_irqsave(&ha->tgt.sess_lock, flags); - if (!sess) - goto out_term2; - } else { - if (sess->deleted) { - goto out_term2; - } - - if (!kref_get_unless_zero(&sess->sess_kref)) { - ql_dbg(ql_dbg_tgt_tmr, vha, 0xf020, - "%s: kref_get fail %8phC\n", - __func__, sess->port_name); - goto out_term2; - } - } - - iocb = a; - fn = a->u.isp24.fcp_cmnd.task_mgmt_flags; - unpacked_lun = - scsilun_to_int((struct scsi_lun *)&a->u.isp24.fcp_cmnd.lun); - - rc = qlt_issue_task_mgmt(sess, unpacked_lun, fn, iocb, 0); - spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); - - ha->tgt.tgt_ops->put_sess(sess); - - if (rc != 0) - goto out_term; - return; - -out_term2: - spin_unlock_irqrestore(&ha->tgt.sess_lock, flags); -out_term: - qlt_send_term_exchange(ha->base_qpair, NULL, &prm->tm_iocb2, 1, 0); -} - static void qlt_sess_work_fn(struct work_struct *work) { struct qla_tgt *tgt = container_of(work, struct qla_tgt, sess_work); @@ -6423,9 +6362,6 @@ static void qlt_sess_work_fn(struct work_struct *work) case QLA_TGT_SESS_WORK_ABORT: qlt_abort_work(tgt, prm); break; - case QLA_TGT_SESS_WORK_TM: - qlt_tmr_work(tgt, prm); - break; default: BUG_ON(1); break; @@ -6512,7 +6448,6 @@ int qlt_add_target(struct qla_hw_data *ha, struct scsi_qla_host *base_vha) tgt->ha = ha; tgt->vha = base_vha; init_waitqueue_head(&tgt->waitQ); - INIT_LIST_HEAD(&tgt->del_sess_list); spin_lock_init(&tgt->sess_work_lock); INIT_WORK(&tgt->sess_work, qlt_sess_work_fn); INIT_LIST_HEAD(&tgt->sess_works_list); @@ -6935,14 +6870,8 @@ qlt_24xx_config_rings(struct scsi_qla_host *vha) if (ha->flags.msix_enabled) { if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) { - if (IS_QLA2071(ha)) { - /* 4 ports Baker: Enable Interrupt Handshake */ - icb->msix_atio = 0; - icb->firmware_options_2 |= cpu_to_le32(BIT_26); - } else { - icb->msix_atio = cpu_to_le16(msix->entry); - icb->firmware_options_2 &= cpu_to_le32(~BIT_26); - } + icb->msix_atio = cpu_to_le16(msix->entry); + icb->firmware_options_2 &= cpu_to_le32(~BIT_26); ql_dbg(ql_dbg_init, vha, 0xf072, "Registering ICB vector 0x%x for atio que.\n", msix->entry); diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h index de3942b8efc46462798088f73d4797704ae9c15c..7df86578214f1b6949f9f4d2288e685ce24e44ab 100644 --- a/drivers/scsi/qla2xxx/qla_target.h +++ b/drivers/scsi/qla2xxx/qla_target.h @@ -813,9 +813,6 @@ struct qla_tgt { /* Count of sessions refering qla_tgt. Protected by hardware_lock. */ int sess_count; - /* Protected by hardware_lock */ - struct list_head del_sess_list; - spinlock_t sess_work_lock; struct list_head sess_works_list; struct work_struct sess_work; @@ -945,7 +942,6 @@ struct qla_tgt_sess_work_param { struct list_head sess_works_list_entry; #define QLA_TGT_SESS_WORK_ABORT 1 -#define QLA_TGT_SESS_WORK_TM 2 int type; union { @@ -1079,8 +1075,6 @@ extern void qlt_81xx_config_nvram_stage2(struct scsi_qla_host *, struct init_cb_81xx *); extern void qlt_81xx_config_nvram_stage1(struct scsi_qla_host *, struct nvram_81xx *); -extern int qlt_24xx_process_response_error(struct scsi_qla_host *, - struct sts_entry_24xx *); extern void qlt_modify_vp_config(struct scsi_qla_host *, struct vp_config_entry_24xx *); extern void qlt_probe_one_stage1(struct scsi_qla_host *, struct qla_hw_data *); diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h index f3257d46b6d2e86e3f5394c4dbb06cda1d5503f3..03f3e2cd62b5d23dca4d0d452f1f2d436a259013 100644 --- a/drivers/scsi/qla2xxx/qla_version.h +++ b/drivers/scsi/qla2xxx/qla_version.h @@ -6,9 +6,9 @@ /* * Driver version */ -#define QLA2XXX_VERSION "10.02.07.800-k" +#define QLA2XXX_VERSION "10.02.07.900-k" #define QLA_DRIVER_MAJOR_VER 10 #define QLA_DRIVER_MINOR_VER 2 #define QLA_DRIVER_PATCH_VER 7 -#define QLA_DRIVER_BETA_VER 800 +#define QLA_DRIVER_BETA_VER 900 diff --git a/drivers/scsi/qlogicpti.c b/drivers/scsi/qlogicpti.c index 57f2f4135a060557d34486d1e0fba2b21c1284ec..8c961ff03fcdfe27beac2ebaafedc03c8525ca8b 100644 --- a/drivers/scsi/qlogicpti.c +++ b/drivers/scsi/qlogicpti.c @@ -909,7 +909,8 @@ static inline int load_cmd(struct scsi_cmnd *Cmnd, struct Command_Entry *cmd, sg_count = dma_map_sg(&qpti->op->dev, sg, scsi_sg_count(Cmnd), Cmnd->sc_data_direction); - + if (!sg_count) + return -1; ds = cmd->dataseg; cmd->segment_cnt = sg_count; diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c index 086ec5b5862d0e1ec5860440f5e7c5cb761598b2..c59eac7a32f2a087f0491b9d594a9e00f7950d47 100644 --- a/drivers/scsi/scsi.c +++ b/drivers/scsi/scsi.c @@ -586,13 +586,10 @@ EXPORT_SYMBOL(scsi_device_get); */ void scsi_device_put(struct scsi_device *sdev) { - /* - * Decreasing the module reference count before the device reference - * count is safe since scsi_remove_host() only returns after all - * devices have been removed. - */ - module_put(sdev->host->hostt->module); + struct module *mod = sdev->host->hostt->module; + put_device(&sdev->sdev_gendev); + module_put(mod); } EXPORT_SYMBOL(scsi_device_put); diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index b8a76b89f85a3ce9db6320fbe1ff26c67a7d958c..697fc57bc711fb0e8492234c5cf65ac90d80b9c9 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -7474,12 +7474,12 @@ static int resp_not_ready(struct scsi_cmnd *scp, struct sdebug_dev_info *devip) return check_condition_result; } -static int sdebug_map_queues(struct Scsi_Host *shost) +static void sdebug_map_queues(struct Scsi_Host *shost) { int i, qoff; if (shost->nr_hw_queues == 1) - return 0; + return; for (i = 0, qoff = 0; i < HCTX_MAX_TYPES; i++) { struct blk_mq_queue_map *map = &shost->tag_set.map[i]; @@ -7501,9 +7501,6 @@ static int sdebug_map_queues(struct Scsi_Host *shost) qoff += map->nr_queues; } - - return 0; - } static int sdebug_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 448748e3fba5e156243d6f0e54276739ac734c82..6995c89792300e3cda4e8e27b4ead6fadd98aefc 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -334,6 +334,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req) trace_scsi_dispatch_cmd_timeout(scmd); scsi_log_completion(scmd, TIMEOUT_ERROR); + atomic_inc(&scmd->device->iotmo_cnt); if (host->eh_deadline != -1 && !host->last_reset) host->last_reset = jiffies; @@ -514,6 +515,11 @@ static void scsi_report_sense(struct scsi_device *sdev, } } +static inline void set_scsi_ml_byte(struct scsi_cmnd *cmd, u8 status) +{ + cmd->result = (cmd->result & 0xffff00ff) | (status << 8); +} + /** * scsi_check_sense - Examine scsi cmd sense * @scmd: Cmd to have sense checked. @@ -644,7 +650,7 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd) case DATA_PROTECT: if (sshdr.asc == 0x27 && sshdr.ascq == 0x07) { /* Thin provisioning hard threshold reached */ - set_host_byte(scmd, DID_ALLOC_FAILURE); + set_scsi_ml_byte(scmd, SCSIML_STAT_NOSPC); return SUCCESS; } fallthrough; @@ -652,14 +658,14 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd) case VOLUME_OVERFLOW: case MISCOMPARE: case BLANK_CHECK: - set_host_byte(scmd, DID_TARGET_FAILURE); + set_scsi_ml_byte(scmd, SCSIML_STAT_TGT_FAILURE); return SUCCESS; case MEDIUM_ERROR: if (sshdr.asc == 0x11 || /* UNRECOVERED READ ERR */ sshdr.asc == 0x13 || /* AMNF DATA FIELD */ sshdr.asc == 0x14) { /* RECORD NOT FOUND */ - set_host_byte(scmd, DID_MEDIUM_ERROR); + set_scsi_ml_byte(scmd, SCSIML_STAT_MED_ERROR); return SUCCESS; } return NEEDS_RETRY; @@ -668,7 +674,7 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd) if (scmd->device->retry_hwerror) return ADD_TO_MLQUEUE; else - set_host_byte(scmd, DID_TARGET_FAILURE); + set_scsi_ml_byte(scmd, SCSIML_STAT_TGT_FAILURE); fallthrough; case ILLEGAL_REQUEST: @@ -678,7 +684,7 @@ enum scsi_disposition scsi_check_sense(struct scsi_cmnd *scmd) sshdr.asc == 0x24 || /* Invalid field in cdb */ sshdr.asc == 0x26 || /* Parameter value invalid */ sshdr.asc == 0x27) { /* Write protected */ - set_host_byte(scmd, DID_TARGET_FAILURE); + set_scsi_ml_byte(scmd, SCSIML_STAT_TGT_FAILURE); } return SUCCESS; @@ -1983,7 +1989,7 @@ enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *scmd) case SAM_STAT_RESERVATION_CONFLICT: sdev_printk(KERN_INFO, scmd->device, "reservation conflict\n"); - set_host_byte(scmd, DID_NEXUS_FAILURE); + set_scsi_ml_byte(scmd, SCSIML_STAT_RESV_CONFLICT); return SUCCESS; /* causes immediate i/o error */ } return FAILED; @@ -2004,9 +2010,11 @@ maybe_retry: } } -static void eh_lock_door_done(struct request *req, blk_status_t status) +static enum rq_end_io_ret eh_lock_door_done(struct request *req, + blk_status_t status) { blk_mq_free_request(req); + return RQ_END_IO_NONE; } /** diff --git a/drivers/scsi/scsi_ioctl.c b/drivers/scsi/scsi_ioctl.c index 729e309e60346f6424522240d46ab9a7c5f3217f..2d20da55fb64086ebc66d4702b37ac28c7cc1bea 100644 --- a/drivers/scsi/scsi_ioctl.c +++ b/drivers/scsi/scsi_ioctl.c @@ -449,25 +449,9 @@ static int sg_io(struct scsi_device *sdev, struct sg_io_hdr *hdr, fmode_t mode) if (ret < 0) goto out_put_request; - ret = 0; - if (hdr->iovec_count && hdr->dxfer_len) { - struct iov_iter i; - struct iovec *iov = NULL; - - ret = import_iovec(rq_data_dir(rq), hdr->dxferp, - hdr->iovec_count, 0, &iov, &i); - if (ret < 0) - goto out_put_request; - - /* SG_IO howto says that the shorter of the two wins */ - iov_iter_truncate(&i, hdr->dxfer_len); - - ret = blk_rq_map_user_iov(rq->q, rq, NULL, &i, GFP_KERNEL); - kfree(iov); - } else if (hdr->dxfer_len) - ret = blk_rq_map_user(rq->q, rq, NULL, hdr->dxferp, - hdr->dxfer_len, GFP_KERNEL); - + ret = blk_rq_map_user_io(rq, NULL, hdr->dxferp, hdr->dxfer_len, + GFP_KERNEL, hdr->iovec_count && hdr->dxfer_len, + hdr->iovec_count, 0, rq_data_dir(rq)); if (ret) goto out_put_request; diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 4dbd29ab1dcc37792688849a468e5b8e5dc72ede..8b89fab7c4206353e419dbf1f6e56771a17699f9 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -111,7 +111,7 @@ scsi_set_blocked(struct scsi_cmnd *cmd, int reason) } } -static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) +static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd, unsigned long msecs) { struct request *rq = scsi_cmd_to_rq(cmd); @@ -121,7 +121,12 @@ static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) } else { WARN_ON_ONCE(true); } - blk_mq_requeue_request(rq, true); + + if (msecs) { + blk_mq_requeue_request(rq, false); + blk_mq_delay_kick_requeue_list(rq->q, msecs); + } else + blk_mq_requeue_request(rq, true); } /** @@ -576,16 +581,36 @@ static bool scsi_end_request(struct request *req, blk_status_t error, return false; } +static inline u8 get_scsi_ml_byte(int result) +{ + return (result >> 8) & 0xff; +} + /** * scsi_result_to_blk_status - translate a SCSI result code into blk_status_t - * @cmd: SCSI command * @result: scsi error code * - * Translate a SCSI result code into a blk_status_t value. May reset the host - * byte of @cmd->result. + * Translate a SCSI result code into a blk_status_t value. */ -static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result) +static blk_status_t scsi_result_to_blk_status(int result) { + /* + * Check the scsi-ml byte first in case we converted a host or status + * byte. + */ + switch (get_scsi_ml_byte(result)) { + case SCSIML_STAT_OK: + break; + case SCSIML_STAT_RESV_CONFLICT: + return BLK_STS_NEXUS; + case SCSIML_STAT_NOSPC: + return BLK_STS_NOSPC; + case SCSIML_STAT_MED_ERROR: + return BLK_STS_MEDIUM; + case SCSIML_STAT_TGT_FAILURE: + return BLK_STS_TARGET; + } + switch (host_byte(result)) { case DID_OK: if (scsi_status_is_good(result)) @@ -594,18 +619,6 @@ static blk_status_t scsi_result_to_blk_status(struct scsi_cmnd *cmd, int result) case DID_TRANSPORT_FAILFAST: case DID_TRANSPORT_MARGINAL: return BLK_STS_TRANSPORT; - case DID_TARGET_FAILURE: - set_host_byte(cmd, DID_OK); - return BLK_STS_TARGET; - case DID_NEXUS_FAILURE: - set_host_byte(cmd, DID_OK); - return BLK_STS_NEXUS; - case DID_ALLOC_FAILURE: - set_host_byte(cmd, DID_OK); - return BLK_STS_NOSPC; - case DID_MEDIUM_ERROR: - set_host_byte(cmd, DID_OK); - return BLK_STS_MEDIUM; default: return BLK_STS_IOERR; } @@ -651,14 +664,6 @@ static unsigned int scsi_rq_err_bytes(const struct request *rq) return bytes; } -/* Helper for scsi_io_completion() when "reprep" action required. */ -static void scsi_io_completion_reprep(struct scsi_cmnd *cmd, - struct request_queue *q) -{ - /* A new command will be prepared and issued. */ - scsi_mq_requeue_cmd(cmd); -} - static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd) { struct request *req = scsi_cmd_to_rq(cmd); @@ -676,14 +681,21 @@ static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd) return false; } +/* + * When ALUA transition state is returned, reprep the cmd to + * use the ALUA handler's transition timeout. Delay the reprep + * 1 sec to avoid aggressive retries of the target in that + * state. + */ +#define ALUA_TRANSITION_REPREP_DELAY 1000 + /* Helper for scsi_io_completion() when special action required. */ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) { - struct request_queue *q = cmd->device->request_queue; struct request *req = scsi_cmd_to_rq(cmd); int level = 0; - enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY, - ACTION_DELAYED_RETRY} action; + enum {ACTION_FAIL, ACTION_REPREP, ACTION_DELAYED_REPREP, + ACTION_RETRY, ACTION_DELAYED_RETRY} action; struct scsi_sense_hdr sshdr; bool sense_valid; bool sense_current = true; /* false implies "deferred sense" */ @@ -693,7 +705,7 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) if (sense_valid) sense_current = !scsi_sense_is_deferred(&sshdr); - blk_stat = scsi_result_to_blk_status(cmd, result); + blk_stat = scsi_result_to_blk_status(result); if (host_byte(result) == DID_RESET) { /* Third party bus reset or reset for error recovery @@ -772,8 +784,8 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) action = ACTION_DELAYED_RETRY; break; case 0x0a: /* ALUA state transition */ - blk_stat = BLK_STS_TRANSPORT; - fallthrough; + action = ACTION_DELAYED_REPREP; + break; default: action = ACTION_FAIL; break; @@ -832,7 +844,10 @@ static void scsi_io_completion_action(struct scsi_cmnd *cmd, int result) return; fallthrough; case ACTION_REPREP: - scsi_io_completion_reprep(cmd, q); + scsi_mq_requeue_cmd(cmd, 0); + break; + case ACTION_DELAYED_REPREP: + scsi_mq_requeue_cmd(cmd, ALUA_TRANSITION_REPREP_DELAY); break; case ACTION_RETRY: /* Retry the same command immediately */ @@ -871,14 +886,14 @@ static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result, SCSI_SENSE_BUFFERSIZE); } if (sense_current) - *blk_statp = scsi_result_to_blk_status(cmd, result); + *blk_statp = scsi_result_to_blk_status(result); } else if (blk_rq_bytes(req) == 0 && sense_current) { /* * Flush commands do not transfers any data, and thus cannot use * good_bytes != blk_rq_bytes(req) as the signal for an error. * This sets *blk_statp explicitly for the problem case. */ - *blk_statp = scsi_result_to_blk_status(cmd, result); + *blk_statp = scsi_result_to_blk_status(result); } /* * Recovered errors need reporting, but they're always treated as @@ -926,7 +941,7 @@ static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result, * command block will be released and the queue function will be goosed. If we * are not done then we have to figure out what to do next: * - * a) We can call scsi_io_completion_reprep(). The request will be + * a) We can call scsi_mq_requeue_cmd(). The request will be * unprepared and put back on the queue. Then a new command will * be created for it. This should be used if we made forward * progress, or if we want to switch from READ(10) to READ(6) for @@ -942,7 +957,6 @@ static int scsi_io_completion_nz_result(struct scsi_cmnd *cmd, int result, void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) { int result = cmd->result; - struct request_queue *q = cmd->device->request_queue; struct request *req = scsi_cmd_to_rq(cmd); blk_status_t blk_stat = BLK_STS_OK; @@ -979,7 +993,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes) * request just queue the command up again. */ if (likely(result == 0)) - scsi_io_completion_reprep(cmd, q); + scsi_mq_requeue_cmd(cmd, 0); else scsi_io_completion_action(cmd, result); } @@ -1542,7 +1556,6 @@ static blk_status_t scsi_prepare_cmd(struct request *req) scsi_init_command(sdev, cmd); cmd->eh_eflags = 0; - cmd->allowed = 0; cmd->prot_type = 0; cmd->prot_flags = 0; cmd->submitter = 0; @@ -1593,6 +1606,8 @@ static blk_status_t scsi_prepare_cmd(struct request *req) return ret; } + /* Usually overridden by the ULP */ + cmd->allowed = 0; memset(cmd->cmnd, 0, sizeof(cmd->cmnd)); return scsi_cmd_to_driver(cmd)->init_command(cmd); } @@ -1849,13 +1864,13 @@ static int scsi_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, return 0; } -static int scsi_map_queues(struct blk_mq_tag_set *set) +static void scsi_map_queues(struct blk_mq_tag_set *set) { struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set); if (shost->hostt->map_queues) return shost->hostt->map_queues(shost); - return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); + blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); } void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q) @@ -1976,9 +1991,13 @@ int scsi_mq_setup_tags(struct Scsi_Host *shost) return blk_mq_alloc_tag_set(tag_set); } -void scsi_mq_destroy_tags(struct Scsi_Host *shost) +void scsi_mq_free_tags(struct kref *kref) { + struct Scsi_Host *shost = container_of(kref, typeof(*shost), + tagset_refcnt); + blk_mq_free_tag_set(&shost->tag_set); + complete(&shost->tagset_freed); } /** diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h index 429663bd78ecf74fc3c160d85e2ba8144d21fb01..c52de9a973e46622ef521f9418a1b4859c2df5a3 100644 --- a/drivers/scsi/scsi_priv.h +++ b/drivers/scsi/scsi_priv.h @@ -18,6 +18,17 @@ struct scsi_nl_hdr; #define SCSI_CMD_RETRIES_NO_LIMIT -1 +/* + * Error codes used by scsi-ml internally. These must not be used by drivers. + */ +enum scsi_ml_status { + SCSIML_STAT_OK = 0x00, + SCSIML_STAT_RESV_CONFLICT = 0x01, /* Reservation conflict */ + SCSIML_STAT_NOSPC = 0x02, /* Space allocation on the dev failed */ + SCSIML_STAT_MED_ERROR = 0x03, /* Medium error */ + SCSIML_STAT_TGT_FAILURE = 0x04, /* Permanent target failure */ +}; + /* * Scsi Error Handler Flags */ @@ -94,7 +105,7 @@ extern void scsi_run_host_queues(struct Scsi_Host *shost); extern void scsi_requeue_run_queue(struct work_struct *work); extern void scsi_start_queue(struct scsi_device *sdev); extern int scsi_mq_setup_tags(struct Scsi_Host *shost); -extern void scsi_mq_destroy_tags(struct Scsi_Host *shost); +extern void scsi_mq_free_tags(struct kref *kref); extern void scsi_exit_queue(void); extern void scsi_evt_thread(struct work_struct *work); diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c index ac6059702d13514d8bb9ed704e644b62117109fb..5d27f5196de6fd49537ba29d9d3814207380e0ea 100644 --- a/drivers/scsi/scsi_scan.c +++ b/drivers/scsi/scsi_scan.c @@ -340,6 +340,7 @@ static struct scsi_device *scsi_alloc_sdev(struct scsi_target *starget, kfree(sdev); goto out; } + kref_get(&sdev->host->tagset_refcnt); sdev->request_queue = q; q->queuedata = sdev; __scsi_init_queue(sdev->host, q); @@ -406,14 +407,9 @@ static void scsi_target_destroy(struct scsi_target *starget) static void scsi_target_dev_release(struct device *dev) { struct device *parent = dev->parent; - struct Scsi_Host *shost = dev_to_shost(parent); struct scsi_target *starget = to_scsi_target(dev); kfree(starget); - - if (atomic_dec_return(&shost->target_count) == 0) - wake_up(&shost->targets_wq); - put_device(parent); } @@ -526,10 +522,6 @@ static struct scsi_target *scsi_alloc_target(struct device *parent, starget->state = STARGET_CREATED; starget->scsi_level = SCSI_2; starget->max_target_blocked = SCSI_DEFAULT_TARGET_BLOCKED; - init_waitqueue_head(&starget->sdev_wq); - - atomic_inc(&shost->target_count); - retry: spin_lock_irqsave(shost->host_lock, flags); diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c index 9dad2fd5297fa6c74bac5d79e9e3b15b3d48f8ee..c95177ca6ed2645e9720d76e6f3c2764a459e5b9 100644 --- a/drivers/scsi/scsi_sysfs.c +++ b/drivers/scsi/scsi_sysfs.c @@ -443,15 +443,18 @@ static void scsi_device_cls_release(struct device *class_dev) static void scsi_device_dev_release_usercontext(struct work_struct *work) { - struct scsi_device *sdev = container_of(work, struct scsi_device, - ew.work); - struct scsi_target *starget = sdev->sdev_target; + struct scsi_device *sdev; struct device *parent; struct list_head *this, *tmp; struct scsi_vpd *vpd_pg80 = NULL, *vpd_pg83 = NULL; struct scsi_vpd *vpd_pg0 = NULL, *vpd_pg89 = NULL; struct scsi_vpd *vpd_pgb0 = NULL, *vpd_pgb1 = NULL, *vpd_pgb2 = NULL; unsigned long flags; + struct module *mod; + + sdev = container_of(work, struct scsi_device, ew.work); + + mod = sdev->host->hostt->module; scsi_dh_release_device(sdev); @@ -513,16 +516,19 @@ static void scsi_device_dev_release_usercontext(struct work_struct *work) kfree(sdev->inquiry); kfree(sdev); - if (starget && atomic_dec_return(&starget->sdev_count) == 0) - wake_up(&starget->sdev_wq); - if (parent) put_device(parent); + module_put(mod); } static void scsi_device_dev_release(struct device *dev) { struct scsi_device *sdp = to_scsi_device(dev); + + /* Set module pointer as NULL in case of module unloading */ + if (!try_module_get(sdp->host->hostt->module)) + sdp->host->hostt->module = NULL; + execute_in_process_context(scsi_device_dev_release_usercontext, &sdp->ew); } @@ -970,6 +976,7 @@ static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL) show_sdev_iostat(iorequest_cnt); show_sdev_iostat(iodone_cnt); show_sdev_iostat(ioerr_cnt); +show_sdev_iostat(iotmo_cnt); static ssize_t sdev_show_modalias(struct device *dev, struct device_attribute *attr, char *buf) @@ -1289,6 +1296,7 @@ static struct attribute *scsi_sdev_attrs[] = { &dev_attr_iorequest_cnt.attr, &dev_attr_iodone_cnt.attr, &dev_attr_ioerr_cnt.attr, + &dev_attr_iotmo_cnt.attr, &dev_attr_modalias.attr, &dev_attr_queue_depth.attr, &dev_attr_queue_type.attr, @@ -1470,6 +1478,7 @@ void __scsi_remove_device(struct scsi_device *sdev) mutex_unlock(&sdev->state_mutex); blk_mq_destroy_queue(sdev->request_queue); + kref_put(&sdev->host->tagset_refcnt, scsi_mq_free_tags); cancel_work_sync(&sdev->requeue_work); if (sdev->host->hostt->slave_destroy) @@ -1529,14 +1538,6 @@ static void __scsi_remove_target(struct scsi_target *starget) goto restart; } spin_unlock_irqrestore(shost->host_lock, flags); - - /* - * After scsi_remove_target() returns its caller can remove resources - * associated with @starget, e.g. an rport or session. Wait until all - * devices associated with @starget have been removed to prevent that - * a SCSI error handling callback function triggers a use-after-free. - */ - wait_event(starget->sdev_wq, atomic_read(&starget->sdev_count) == 0); } /** @@ -1647,9 +1648,6 @@ void scsi_sysfs_device_initialize(struct scsi_device *sdev) list_add_tail(&sdev->same_target_siblings, &starget->devices); list_add_tail(&sdev->siblings, &shost->__devices); spin_unlock_irqrestore(shost->host_lock, flags); - - atomic_inc(&starget->sdev_count); - /* * device can now only be removed via __scsi_remove_device() so hold * the target. Target will be held in CREATED state until something diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c index a2524106206db77f1a40a176789d2f04dfc307fe..8934160c4a33bf013033b76368fb04edb9b732be 100644 --- a/drivers/scsi/scsi_transport_fc.c +++ b/drivers/scsi/scsi_transport_fc.c @@ -543,7 +543,7 @@ fc_host_post_fc_event(struct Scsi_Host *shost, u32 event_number, struct nlmsghdr *nlh; struct fc_nl_event *event; const char *name; - u32 len; + size_t len, padding; int err; if (!data_buf || data_len < 4) @@ -554,7 +554,7 @@ fc_host_post_fc_event(struct Scsi_Host *shost, u32 event_number, goto send_fail; } - len = FC_NL_MSGALIGN(sizeof(*event) + data_len); + len = FC_NL_MSGALIGN(sizeof(*event) - sizeof(event->event_data) + data_len); skb = nlmsg_new(len, GFP_KERNEL); if (!skb) { @@ -578,7 +578,9 @@ fc_host_post_fc_event(struct Scsi_Host *shost, u32 event_number, event->event_num = event_number; event->event_code = event_code; if (data_len) - memcpy(&event->event_data, data_buf, data_len); + memcpy(event->event_data_flex, data_buf, data_len); + padding = len - offsetof(typeof(*event), event_data_flex) - data_len; + memset(event->event_data_flex + data_len, 0, padding); nlmsg_multicast(scsi_nl_sock, skb, 0, SCSI_NL_GRP_FC_EVENTS, GFP_KERNEL); @@ -1170,7 +1172,7 @@ static int fc_rport_set_dev_loss_tmo(struct fc_rport *rport, return 0; } -fc_rport_show_function(dev_loss_tmo, "%d\n", 20, ) +fc_rport_show_function(dev_loss_tmo, "%u\n", 20, ) static ssize_t store_fc_rport_dev_loss_tmo(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) diff --git a/drivers/scsi/scsi_transport_spi.c b/drivers/scsi/scsi_transport_spi.c index bd72c38d7bfc47ee4635089d21963419988d934f..f569cf0095c28ba660cbc5608331175d642e8528 100644 --- a/drivers/scsi/scsi_transport_spi.c +++ b/drivers/scsi/scsi_transport_spi.c @@ -998,8 +998,9 @@ void spi_dv_device(struct scsi_device *sdev) { struct scsi_target *starget = sdev->sdev_target; - u8 *buffer; const int len = SPI_MAX_ECHO_BUFFER_SIZE*2; + unsigned int sleep_flags; + u8 *buffer; /* * Because this function and the power management code both call @@ -1007,7 +1008,7 @@ spi_dv_device(struct scsi_device *sdev) * while suspend or resume is in progress. Hence the * lock/unlock_system_sleep() calls. */ - lock_system_sleep(); + sleep_flags = lock_system_sleep(); if (scsi_autopm_get_device(sdev)) goto unlock_system_sleep; @@ -1058,7 +1059,7 @@ put_autopm: scsi_autopm_put_device(sdev); unlock_system_sleep: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); } EXPORT_SYMBOL(spi_dv_device); diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 8f79fa6318fec25bd7a50e5e9c3b19addc3361cd..eb76ba0550216d7d55d998e4609ed7ea5d4bdb9a 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -103,7 +103,6 @@ static void sd_config_discard(struct scsi_disk *, unsigned int); static void sd_config_write_same(struct scsi_disk *); static int sd_revalidate_disk(struct gendisk *); static void sd_unlock_native_capacity(struct gendisk *disk); -static void sd_start_done_work(struct work_struct *work); static int sd_probe(struct device *); static int sd_remove(struct device *); static void sd_shutdown(struct device *); @@ -3471,7 +3470,6 @@ static int sd_probe(struct device *dev) sdkp->max_retries = SD_MAX_RETRIES; atomic_set(&sdkp->openers, 0); atomic_set(&sdkp->device->ioerr_cnt, 0); - INIT_WORK(&sdkp->start_done_work, sd_start_done_work); if (!sdp->request_queue->rq_timeout) { if (sdp->type != TYPE_MOD) @@ -3594,69 +3592,12 @@ static void scsi_disk_release(struct device *dev) kfree(sdkp); } -/* Process sense data after a START command finished. */ -static void sd_start_done_work(struct work_struct *work) -{ - struct scsi_disk *sdkp = container_of(work, typeof(*sdkp), - start_done_work); - struct scsi_sense_hdr sshdr; - int res = sdkp->start_result; - - if (res == 0) - return; - - sd_print_result(sdkp, "Start/Stop Unit failed", res); - - if (res < 0) - return; - - if (scsi_normalize_sense(sdkp->start_sense_buffer, - sdkp->start_sense_len, &sshdr)) - sd_print_sense_hdr(sdkp, &sshdr); -} - -/* A START command finished. May be called from interrupt context. */ -static void sd_start_done(struct request *req, blk_status_t status) -{ - const struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); - struct scsi_disk *sdkp = scsi_disk(req->q->disk); - - sdkp->start_result = scmd->result; - WARN_ON_ONCE(scmd->sense_len > SCSI_SENSE_BUFFERSIZE); - sdkp->start_sense_len = scmd->sense_len; - memcpy(sdkp->start_sense_buffer, scmd->sense_buffer, - ARRAY_SIZE(sdkp->start_sense_buffer)); - WARN_ON_ONCE(!schedule_work(&sdkp->start_done_work)); -} - -/* Submit a START command asynchronously. */ -static int sd_submit_start(struct scsi_disk *sdkp, u8 cmd[], u8 cmd_len) -{ - struct scsi_device *sdev = sdkp->device; - struct request_queue *q = sdev->request_queue; - struct request *req; - struct scsi_cmnd *scmd; - - req = scsi_alloc_request(q, REQ_OP_DRV_IN, BLK_MQ_REQ_PM); - if (IS_ERR(req)) - return PTR_ERR(req); - - scmd = blk_mq_rq_to_pdu(req); - scmd->cmd_len = cmd_len; - memcpy(scmd->cmnd, cmd, cmd_len); - scmd->allowed = sdkp->max_retries; - req->timeout = SD_TIMEOUT; - req->rq_flags |= RQF_PM | RQF_QUIET; - req->end_io = sd_start_done; - blk_execute_rq_nowait(req, /*at_head=*/true); - - return 0; -} - static int sd_start_stop_device(struct scsi_disk *sdkp, int start) { unsigned char cmd[6] = { START_STOP }; /* START_VALID */ + struct scsi_sense_hdr sshdr; struct scsi_device *sdp = sdkp->device; + int res; if (start) cmd[4] |= 1; /* START */ @@ -3667,10 +3608,23 @@ static int sd_start_stop_device(struct scsi_disk *sdkp, int start) if (!scsi_device_online(sdp)) return -ENODEV; - /* Wait until processing of sense data has finished. */ - flush_work(&sdkp->start_done_work); + res = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr, + SD_TIMEOUT, sdkp->max_retries, 0, RQF_PM, NULL); + if (res) { + sd_print_result(sdkp, "Start/Stop Unit failed", res); + if (res > 0 && scsi_sense_valid(&sshdr)) { + sd_print_sense_hdr(sdkp, &sshdr); + /* 0x3a is medium not present */ + if (sshdr.asc == 0x3a) + res = 0; + } + } - return sd_submit_start(sdkp, cmd, sizeof(cmd)); + /* SCSI error codes must not go to the generic layer */ + if (res) + return -EIO; + + return 0; } /* @@ -3697,8 +3651,6 @@ static void sd_shutdown(struct device *dev) sd_printk(KERN_NOTICE, sdkp, "Stopping disk\n"); sd_start_stop_device(sdkp, 0); } - - flush_work(&sdkp->start_done_work); } static int sd_suspend_common(struct device *dev, bool ignore_stop_errors) diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index b89187761d61f33af307bc0a91b656b1df4373bc..5eea762f84d188e15b516b05f0f98ba3892209a5 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -150,11 +150,6 @@ struct scsi_disk { unsigned urswrz : 1; unsigned security : 1; unsigned ignore_medium_access_errors : 1; - - int start_result; - u32 start_sense_len; - u8 start_sense_buffer[SCSI_SENSE_BUFFERSIZE]; - struct work_struct start_done_work; }; #define to_scsi_disk(obj) container_of(obj, struct scsi_disk, disk_dev) diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 340b050ad28d15dc61ae4d5864e6d1f36e32d083..ce34a8ad53b4e9d99501e043be2d81c2476748f2 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -177,7 +177,7 @@ typedef struct sg_device { /* holds the state of each scsi generic device */ } Sg_device; /* tasklet or soft irq callback */ -static void sg_rq_end_io(struct request *rq, blk_status_t status); +static enum rq_end_io_ret sg_rq_end_io(struct request *rq, blk_status_t status); static int sg_start_req(Sg_request *srp, unsigned char *cmd); static int sg_finish_rem_req(Sg_request * srp); static int sg_build_indirect(Sg_scatter_hold * schp, Sg_fd * sfp, int buff_size); @@ -1311,7 +1311,7 @@ sg_rq_end_io_usercontext(struct work_struct *work) * This function is a "bottom half" handler that is called by the mid * level when a command is completed (or has failed). */ -static void +static enum rq_end_io_ret sg_rq_end_io(struct request *rq, blk_status_t status) { struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq); @@ -1324,11 +1324,11 @@ sg_rq_end_io(struct request *rq, blk_status_t status) int result, resid, done = 1; if (WARN_ON(srp->done != 0)) - return; + return RQ_END_IO_NONE; sfp = srp->parentfp; if (WARN_ON(sfp == NULL)) - return; + return RQ_END_IO_NONE; sdp = sfp->parentdp; if (unlikely(atomic_read(&sdp->detaching))) @@ -1406,6 +1406,7 @@ sg_rq_end_io(struct request *rq, blk_status_t status) INIT_WORK(&srp->ew.work, sg_rq_end_io_usercontext); schedule_work(&srp->ew.work); } + return RQ_END_IO_NONE; } static const struct file_operations sg_fops = { @@ -1803,26 +1804,8 @@ sg_start_req(Sg_request *srp, unsigned char *cmd) md->from_user = 0; } - if (iov_count) { - struct iovec *iov = NULL; - struct iov_iter i; - - res = import_iovec(rw, hp->dxferp, iov_count, 0, &iov, &i); - if (res < 0) - return res; - - iov_iter_truncate(&i, hp->dxfer_len); - if (!iov_iter_count(&i)) { - kfree(iov); - return -EINVAL; - } - - res = blk_rq_map_user_iov(q, rq, md, &i, GFP_ATOMIC); - kfree(iov); - } else - res = blk_rq_map_user(q, rq, md, hp->dxferp, - hp->dxfer_len, GFP_ATOMIC); - + res = blk_rq_map_user_io(rq, md, hp->dxferp, hp->dxfer_len, + GFP_ATOMIC, iov_count, iov_count, 1, rw); if (!res) { srp->bio = rq->bio; diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 7a8c2c75acbaf31bd8036929e12f106c474c7013..b971fbe3b3a173db160702cdcb07095b073bf96e 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -6436,12 +6436,12 @@ static int pqi_slave_alloc(struct scsi_device *sdev) return 0; } -static int pqi_map_queues(struct Scsi_Host *shost) +static void pqi_map_queues(struct Scsi_Host *shost) { struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost); - return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], - ctrl_info->pci_dev, 0); + blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT], + ctrl_info->pci_dev, 0); } static inline bool pqi_is_tape_changer_device(struct pqi_scsi_dev *device) diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index 850172a2b8f14083f80e7a4a4ad40cf8dfebdae6..b90a440e135dc493eefac95c8a108629f534f9b3 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -512,7 +512,8 @@ static void st_do_stats(struct scsi_tape *STp, struct request *req) atomic64_dec(&STp->stats->in_flight); } -static void st_scsi_execute_end(struct request *req, blk_status_t status) +static enum rq_end_io_ret st_scsi_execute_end(struct request *req, + blk_status_t status) { struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); struct st_request *SRpnt = req->end_io_data; @@ -532,6 +533,7 @@ static void st_scsi_execute_end(struct request *req, blk_status_t status) blk_rq_unmap_user(tmp); blk_mq_free_request(req); + return RQ_END_IO_NONE; } static int st_scsi_execute(struct st_request *SRpnt, const unsigned char *cmd, @@ -4246,11 +4248,10 @@ static int st_probe(struct device *dev) struct st_partstat *STps; struct st_buffer *buffer; int i, error; - char *stp; if (SDp->type != TYPE_TAPE) return -ENODEV; - if ((stp = st_incompatible(SDp))) { + if (st_incompatible(SDp)) { sdev_printk(KERN_INFO, SDp, "OnStream tapes are no longer supported;\n"); sdev_printk(KERN_INFO, SDp, diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c index e6420f2127ce1c4eb9725b8f7711893ccb06ab1f..8def242675ef3fa515fbaf850969598c5ea86692 100644 --- a/drivers/scsi/stex.c +++ b/drivers/scsi/stex.c @@ -665,16 +665,17 @@ static int stex_queuecommand_lck(struct scsi_cmnd *cmd) return 0; case PASSTHRU_CMD: if (cmd->cmnd[1] == PASSTHRU_GET_DRVVER) { - struct st_drvver ver; + const struct st_drvver ver = { + .major = ST_VER_MAJOR, + .minor = ST_VER_MINOR, + .oem = ST_OEM, + .build = ST_BUILD_VER, + .signature[0] = PASSTHRU_SIGNATURE, + .console_id = host->max_id - 1, + .host_no = hba->host->host_no, + }; size_t cp_len = sizeof(ver); - ver.major = ST_VER_MAJOR; - ver.minor = ST_VER_MINOR; - ver.oem = ST_OEM; - ver.build = ST_BUILD_VER; - ver.signature[0] = PASSTHRU_SIGNATURE; - ver.console_id = host->max_id - 1; - ver.host_no = hba->host->host_no; cp_len = scsi_sg_copy_from_buffer(cmd, &ver, cp_len); if (sizeof(ver) == cp_len) cmd->result = DID_OK << 16; diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index fe000da113327021c93f6c255acda683e86673af..bc46721aa01cbf58ad99ee16cffe0ec2bb3fa2f6 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -60,6 +60,9 @@ #define VMSTOR_PROTO_VERSION_WIN8_1 VMSTOR_PROTO_VERSION(6, 0) #define VMSTOR_PROTO_VERSION_WIN10 VMSTOR_PROTO_VERSION(6, 2) +/* channel callback timeout in ms */ +#define CALLBACK_TIMEOUT 2 + /* Packet structure describing virtual storage requests. */ enum vstor_packet_operation { VSTOR_OPERATION_COMPLETE_IO = 1, @@ -1029,7 +1032,7 @@ do_work: */ wrk = kmalloc(sizeof(struct storvsc_scan_work), GFP_ATOMIC); if (!wrk) { - set_host_byte(scmnd, DID_TARGET_FAILURE); + set_host_byte(scmnd, DID_BAD_TARGET); return; } @@ -1204,6 +1207,7 @@ static void storvsc_on_channel_callback(void *context) struct hv_device *device; struct storvsc_device *stor_device; struct Scsi_Host *shost; + unsigned long time_limit = jiffies + msecs_to_jiffies(CALLBACK_TIMEOUT); if (channel->primary_channel != NULL) device = channel->primary_channel->device_obj; @@ -1224,6 +1228,11 @@ static void storvsc_on_channel_callback(void *context) u32 minlen = rqst_id ? sizeof(struct vstor_packet) : sizeof(enum vstor_packet_operation); + if (unlikely(time_after(jiffies, time_limit))) { + hv_pkt_iter_close(channel); + return; + } + if (pktlen < minlen) { dev_err(&device->device, "Invalid pkt: id=%llu, len=%u, minlen=%u\n", @@ -2012,7 +2021,7 @@ static int storvsc_probe(struct hv_device *device, */ host_dev->handle_error_wq = alloc_ordered_workqueue("storvsc_error_wq_%d", - WQ_MEM_RECLAIM, + 0, host->host_no); if (!host_dev->handle_error_wq) { ret = -ENOMEM; @@ -2059,7 +2068,7 @@ err_out3: err_out2: /* * Once we have connected with the host, we would need to - * to invoke storvsc_dev_remove() to rollback this state and + * invoke storvsc_dev_remove() to rollback this state and * this call also frees up the stor_device; hence the jump around * err_out1 label. */ diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c index 578c4b6d0f7d97b1caddb46d8e0baef6f0b21822..2a79ab16134b15d449adc947abf7d22b1eb169e8 100644 --- a/drivers/scsi/virtio_scsi.c +++ b/drivers/scsi/virtio_scsi.c @@ -141,10 +141,10 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf) set_host_byte(sc, DID_TRANSPORT_DISRUPTED); break; case VIRTIO_SCSI_S_TARGET_FAILURE: - set_host_byte(sc, DID_TARGET_FAILURE); + set_host_byte(sc, DID_BAD_TARGET); break; case VIRTIO_SCSI_S_NEXUS_FAILURE: - set_host_byte(sc, DID_NEXUS_FAILURE); + set_status_byte(sc, SAM_STAT_RESERVATION_CONFLICT); break; default: scmd_printk(KERN_WARNING, sc, "Unknown response %d", @@ -711,12 +711,12 @@ static int virtscsi_abort(struct scsi_cmnd *sc) return virtscsi_tmf(vscsi, cmd); } -static int virtscsi_map_queues(struct Scsi_Host *shost) +static void virtscsi_map_queues(struct Scsi_Host *shost) { struct virtio_scsi *vscsi = shost_priv(shost); struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT]; - return blk_mq_virtio_map_queues(qmap, vscsi->vdev, 2); + blk_mq_virtio_map_queues(qmap, vscsi->vdev, 2); } static void virtscsi_commit_rqs(struct Scsi_Host *shost, u16 hwq) diff --git a/drivers/scsi/wd33c93.c b/drivers/scsi/wd33c93.c index 3fe562047d85fa11001feedcdb632cd206773fb6..e4fafc77bd2010d6221e10932676312a1d1930a9 100644 --- a/drivers/scsi/wd33c93.c +++ b/drivers/scsi/wd33c93.c @@ -162,65 +162,6 @@ module_param(setup_strings, charp, 0); static void wd33c93_execute(struct Scsi_Host *instance); -#ifdef CONFIG_WD33C93_PIO -static inline uchar -read_wd33c93(const wd33c93_regs regs, uchar reg_num) -{ - uchar data; - - outb(reg_num, regs.SASR); - data = inb(regs.SCMD); - return data; -} - -static inline unsigned long -read_wd33c93_count(const wd33c93_regs regs) -{ - unsigned long value; - - outb(WD_TRANSFER_COUNT_MSB, regs.SASR); - value = inb(regs.SCMD) << 16; - value |= inb(regs.SCMD) << 8; - value |= inb(regs.SCMD); - return value; -} - -static inline uchar -read_aux_stat(const wd33c93_regs regs) -{ - return inb(regs.SASR); -} - -static inline void -write_wd33c93(const wd33c93_regs regs, uchar reg_num, uchar value) -{ - outb(reg_num, regs.SASR); - outb(value, regs.SCMD); -} - -static inline void -write_wd33c93_count(const wd33c93_regs regs, unsigned long value) -{ - outb(WD_TRANSFER_COUNT_MSB, regs.SASR); - outb((value >> 16) & 0xff, regs.SCMD); - outb((value >> 8) & 0xff, regs.SCMD); - outb( value & 0xff, regs.SCMD); -} - -#define write_wd33c93_cmd(regs, cmd) \ - write_wd33c93((regs), WD_COMMAND, (cmd)) - -static inline void -write_wd33c93_cdb(const wd33c93_regs regs, uint len, uchar cmnd[]) -{ - int i; - - outb(WD_CDB_1, regs.SASR); - for (i=0; ipdev->dev.of_node = node; ctrl->ngd = ngd; - platform_device_add(ngd->pdev); + ret = platform_device_add(ngd->pdev); + if (ret) { + platform_device_put(ngd->pdev); + kfree(ngd); + of_node_put(node); + return ret; + } ngd->base = ctrl->base + ngd->id * data->offset + (ngd->id - 1) * data->size; @@ -1543,10 +1549,8 @@ static int qcom_slim_ngd_ctrl_probe(struct platform_device *pdev) ret = devm_request_irq(dev, ret, qcom_slim_ngd_interrupt, IRQF_TRIGGER_HIGH, "slim-ngd", ctrl); - if (ret) { - dev_err(&pdev->dev, "request IRQ failed\n"); - return ret; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "request IRQ failed\n"); ctrl->nb.notifier_call = qcom_slim_ngd_ssr_notify; ctrl->notifier = qcom_register_ssr_notifier("lpass", &ctrl->nb); @@ -1575,18 +1579,27 @@ static int qcom_slim_ngd_ctrl_probe(struct platform_device *pdev) ctrl->pdr = pdr_handle_alloc(slim_pd_status, ctrl); if (IS_ERR(ctrl->pdr)) { - dev_err(dev, "Failed to init PDR handle\n"); - return PTR_ERR(ctrl->pdr); + ret = dev_err_probe(dev, PTR_ERR(ctrl->pdr), + "Failed to init PDR handle\n"); + goto err_pdr_alloc; } pds = pdr_add_lookup(ctrl->pdr, "avs/audio", "msm/adsp/audio_pd"); if (IS_ERR(pds) && PTR_ERR(pds) != -EALREADY) { - dev_err(dev, "pdr add lookup failed: %d\n", ret); - return PTR_ERR(pds); + ret = dev_err_probe(dev, PTR_ERR(pds), "pdr add lookup failed\n"); + goto err_pdr_lookup; } platform_driver_register(&qcom_slim_ngd_driver); return of_qcom_slim_ngd_register(dev, ctrl); + +err_pdr_alloc: + qcom_unregister_ssr_notifier(ctrl->notifier, &ctrl->nb); + +err_pdr_lookup: + pdr_handle_release(ctrl->pdr); + + return ret; } static int qcom_slim_ngd_ctrl_remove(struct platform_device *pdev) diff --git a/drivers/soc/amlogic/meson-ee-pwrc.c b/drivers/soc/amlogic/meson-ee-pwrc.c index 2be3afe6c2e3d2970dc89bd736fee356c2d5f69e..dd5f2a13ceb52fe7ad707ef0c63e7ffdf477f621 100644 --- a/drivers/soc/amlogic/meson-ee-pwrc.c +++ b/drivers/soc/amlogic/meson-ee-pwrc.c @@ -469,6 +469,7 @@ static int meson_ee_pwrc_probe(struct platform_device *pdev) { const struct meson_ee_pwrc_domain_data *match; struct regmap *regmap_ao, *regmap_hhi; + struct device_node *parent_np; struct meson_ee_pwrc *pwrc; int i, ret; @@ -495,7 +496,9 @@ static int meson_ee_pwrc_probe(struct platform_device *pdev) pwrc->xlate.num_domains = match->count; - regmap_hhi = syscon_node_to_regmap(of_get_parent(pdev->dev.of_node)); + parent_np = of_get_parent(pdev->dev.of_node); + regmap_hhi = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap_hhi)) { dev_err(&pdev->dev, "failed to get HHI regmap\n"); return PTR_ERR(regmap_hhi); diff --git a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c index b4615b2886254dace71994695f90d586c7938cad..312fd9afccb01c1e7dc8a86999f173733dd5eeff 100644 --- a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c +++ b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c @@ -273,6 +273,7 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev) const struct meson_gx_pwrc_vpu *vpu_pd_match; struct regmap *regmap_ao, *regmap_hhi; struct meson_gx_pwrc_vpu *vpu_pd; + struct device_node *parent_np; struct reset_control *rstc; struct clk *vpu_clk; struct clk *vapb_clk; @@ -291,7 +292,9 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev) memcpy(vpu_pd, vpu_pd_match, sizeof(*vpu_pd)); - regmap_ao = syscon_node_to_regmap(of_get_parent(pdev->dev.of_node)); + parent_np = of_get_parent(pdev->dev.of_node); + regmap_ao = syscon_node_to_regmap(parent_np); + of_node_put(parent_np); if (IS_ERR(regmap_ao)) { dev_err(&pdev->dev, "failed to get regmap\n"); return PTR_ERR(regmap_ao); diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index cf1129e9f76b147d13e6710bc24210a4a547145f..031ec4aa06d55042d8a7208376d4d66cd894d59d 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -660,6 +660,12 @@ int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message, } EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait); +int apple_rtkit_poll(struct apple_rtkit *rtk) +{ + return mbox_client_peek_data(rtk->mbox_chan); +} +EXPORT_SYMBOL_GPL(apple_rtkit_poll); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; diff --git a/drivers/soc/bcm/bcm63xx/Kconfig b/drivers/soc/bcm/bcm63xx/Kconfig index 9e501c8ac5ce116cb524f8094de22ecdeeba15a6..355c34482076e18a1be43554fd8dd133db986287 100644 --- a/drivers/soc/bcm/bcm63xx/Kconfig +++ b/drivers/soc/bcm/bcm63xx/Kconfig @@ -13,8 +13,8 @@ endif # SOC_BCM63XX config BCM_PMB bool "Broadcom PMB (Power Management Bus) driver" - depends on ARCH_BCM4908 || (COMPILE_TEST && OF) - default ARCH_BCM4908 + depends on ARCH_BCMBCA || (COMPILE_TEST && OF) + default ARCH_BCMBCA select PM_GENERIC_DOMAINS if PM help This enables support for the Broadcom's PMB (Power Management Bus) that diff --git a/drivers/soc/bcm/brcmstb/biuctrl.c b/drivers/soc/bcm/brcmstb/biuctrl.c index 1467bbd59690a4fc49b51e4c55e9f39569d69083..e1d7b45432485a0601d26886a7834cc58e4c2963 100644 --- a/drivers/soc/bcm/brcmstb/biuctrl.c +++ b/drivers/soc/bcm/brcmstb/biuctrl.c @@ -288,7 +288,6 @@ static int __init setup_hifcpubiuctrl_regs(struct device_node *np) if (BRCM_ID(family_id) == 0x7260 && BRCM_REV(family_id) == 0) cpubiuctrl_regs = b53_cpubiuctrl_no_wb_regs; out: - of_node_put(np); return ret; } diff --git a/drivers/soc/bcm/brcmstb/pm/pm-arm.c b/drivers/soc/bcm/brcmstb/pm/pm-arm.c index d6b30d521307db7762cd060e5a3378c38c30ac6f..d681cd24c6e1419c35de1a637d6f51e7f91a6f98 100644 --- a/drivers/soc/bcm/brcmstb/pm/pm-arm.c +++ b/drivers/soc/bcm/brcmstb/pm/pm-arm.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -664,7 +663,20 @@ static void __iomem *brcmstb_ioremap_match(const struct of_device_id *matches, return of_io_request_and_map(dn, index, dn->full_name); } - +/* + * The AON is a small domain in the SoC that can retain its state across + * various system wide sleep states and specific reset conditions; the + * AON DATA RAM is a small RAM of a few words (< 1KB) which can store + * persistent information across such events. + * + * The purpose of the below panic notifier is to help with notifying + * the bootloader that a panic occurred and so that it should try its + * best to preserve the DRAM contents holding that buffer for recovery + * by the kernel as opposed to wiping out DRAM clean again. + * + * Reference: comment from Florian Fainelli, at + * https://lore.kernel.org/lkml/781cafb0-8d06-8b56-907a-5175c2da196a@gmail.com + */ static int brcmstb_pm_panic_notify(struct notifier_block *nb, unsigned long action, void *data) { @@ -684,13 +696,14 @@ static int brcmstb_pm_probe(struct platform_device *pdev) const struct of_device_id *of_id = NULL; struct device_node *dn; void __iomem *base; - int ret, i; + int ret, i, s; /* AON ctrl registers */ base = brcmstb_ioremap_match(aon_ctrl_dt_ids, 0, NULL); if (IS_ERR(base)) { pr_err("error mapping AON_CTRL\n"); - return PTR_ERR(base); + ret = PTR_ERR(base); + goto aon_err; } ctrl.aon_ctrl_base = base; @@ -700,8 +713,10 @@ static int brcmstb_pm_probe(struct platform_device *pdev) /* Assume standard offset */ ctrl.aon_sram = ctrl.aon_ctrl_base + AON_CTRL_SYSTEM_DATA_RAM_OFS; + s = 0; } else { ctrl.aon_sram = base; + s = 1; } writel_relaxed(0, ctrl.aon_sram + AON_REG_PANIC); @@ -711,7 +726,8 @@ static int brcmstb_pm_probe(struct platform_device *pdev) (const void **)&ddr_phy_data); if (IS_ERR(base)) { pr_err("error mapping DDR PHY\n"); - return PTR_ERR(base); + ret = PTR_ERR(base); + goto ddr_phy_err; } ctrl.support_warm_boot = ddr_phy_data->supports_warm_boot; ctrl.pll_status_offset = ddr_phy_data->pll_status_offset; @@ -731,17 +747,20 @@ static int brcmstb_pm_probe(struct platform_device *pdev) for_each_matching_node(dn, ddr_shimphy_dt_ids) { i = ctrl.num_memc; if (i >= MAX_NUM_MEMC) { + of_node_put(dn); pr_warn("too many MEMCs (max %d)\n", MAX_NUM_MEMC); break; } base = of_io_request_and_map(dn, 0, dn->full_name); if (IS_ERR(base)) { + of_node_put(dn); if (!ctrl.support_warm_boot) break; pr_err("error mapping DDR SHIMPHY %d\n", i); - return PTR_ERR(base); + ret = PTR_ERR(base); + goto ddr_shimphy_err; } ctrl.memcs[i].ddr_shimphy_base = base; ctrl.num_memc++; @@ -752,14 +771,18 @@ static int brcmstb_pm_probe(struct platform_device *pdev) for_each_matching_node(dn, brcmstb_memc_of_match) { base = of_iomap(dn, 0); if (!base) { + of_node_put(dn); pr_err("error mapping DDR Sequencer %d\n", i); - return -ENOMEM; + ret = -ENOMEM; + goto brcmstb_memc_err; } of_id = of_match_node(brcmstb_memc_of_match, dn); if (!of_id) { iounmap(base); - return -EINVAL; + of_node_put(dn); + ret = -EINVAL; + goto brcmstb_memc_err; } ddr_seq_data = of_id->data; @@ -779,21 +802,24 @@ static int brcmstb_pm_probe(struct platform_device *pdev) dn = of_find_matching_node(NULL, sram_dt_ids); if (!dn) { pr_err("SRAM not found\n"); - return -EINVAL; + ret = -EINVAL; + goto brcmstb_memc_err; } ret = brcmstb_init_sram(dn); of_node_put(dn); if (ret) { pr_err("error setting up SRAM for PM\n"); - return ret; + goto brcmstb_memc_err; } ctrl.pdev = pdev; ctrl.s3_params = kmalloc(sizeof(*ctrl.s3_params), GFP_KERNEL); - if (!ctrl.s3_params) - return -ENOMEM; + if (!ctrl.s3_params) { + ret = -ENOMEM; + goto s3_params_err; + } ctrl.s3_params_pa = dma_map_single(&pdev->dev, ctrl.s3_params, sizeof(*ctrl.s3_params), DMA_TO_DEVICE); @@ -813,7 +839,21 @@ static int brcmstb_pm_probe(struct platform_device *pdev) out: kfree(ctrl.s3_params); - +s3_params_err: + iounmap(ctrl.boot_sram); +brcmstb_memc_err: + for (i--; i >= 0; i--) + iounmap(ctrl.memcs[i].ddr_ctrl); +ddr_shimphy_err: + for (i = 0; i < ctrl.num_memc; i++) + iounmap(ctrl.memcs[i].ddr_shimphy_base); + + iounmap(ctrl.memcs[0].ddr_phy_base); +ddr_phy_err: + iounmap(ctrl.aon_ctrl_base); + if (s) + iounmap(ctrl.aon_sram); +aon_err: pr_warn("PM: initialization failed with code %d\n", ret); return ret; diff --git a/drivers/soc/fsl/Kconfig b/drivers/soc/fsl/Kconfig index 07d52cafbb3133476bf1e338de898dee9a95b123..fcec6ed83d5e275342274a59db5f7ad48e0bbfb9 100644 --- a/drivers/soc/fsl/Kconfig +++ b/drivers/soc/fsl/Kconfig @@ -24,6 +24,7 @@ config FSL_MC_DPIO tristate "QorIQ DPAA2 DPIO driver" depends on FSL_MC_BUS select SOC_BUS + select FSL_GUTS select DIMLIB help Driver for the DPAA2 DPIO object. A DPIO provides queue and diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c index fde4edd83c14cb33d009c518070167240b2344c4..739e4eee6b75ca666abe04004184fc451a09ad21 100644 --- a/drivers/soc/fsl/qbman/qman.c +++ b/drivers/soc/fsl/qbman/qman.c @@ -2483,13 +2483,8 @@ out: } EXPORT_SYMBOL(qman_create_cgr); -int qman_delete_cgr(struct qman_cgr *cgr) +static struct qman_portal *qman_cgr_get_affine_portal(struct qman_cgr *cgr) { - unsigned long irqflags; - struct qm_mcr_querycgr cgr_state; - struct qm_mcc_initcgr local_opts; - int ret = 0; - struct qman_cgr *i; struct qman_portal *p = get_affine_portal(); if (cgr->chan != p->config->channel) { @@ -2497,10 +2492,25 @@ int qman_delete_cgr(struct qman_cgr *cgr) dev_err(p->config->dev, "CGR not owned by current portal"); dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n", cgr->chan, p->config->channel); - - ret = -EINVAL; - goto put_portal; + put_affine_portal(); + return NULL; } + + return p; +} + +int qman_delete_cgr(struct qman_cgr *cgr) +{ + unsigned long irqflags; + struct qm_mcr_querycgr cgr_state; + struct qm_mcc_initcgr local_opts; + int ret = 0; + struct qman_cgr *i; + struct qman_portal *p = qman_cgr_get_affine_portal(cgr); + + if (!p) + return -EINVAL; + memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); spin_lock_irqsave(&p->cgr_lock, irqflags); list_del(&cgr->node); @@ -2528,7 +2538,6 @@ int qman_delete_cgr(struct qman_cgr *cgr) list_add(&cgr->node, &p->cgr_cbs); release_lock: spin_unlock_irqrestore(&p->cgr_lock, irqflags); -put_portal: put_affine_portal(); return ret; } @@ -2559,6 +2568,54 @@ void qman_delete_cgr_safe(struct qman_cgr *cgr) } EXPORT_SYMBOL(qman_delete_cgr_safe); +static int qman_update_cgr(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts) +{ + int ret; + unsigned long irqflags; + struct qman_portal *p = qman_cgr_get_affine_portal(cgr); + + if (!p) + return -EINVAL; + + spin_lock_irqsave(&p->cgr_lock, irqflags); + ret = qm_modify_cgr(cgr, 0, opts); + spin_unlock_irqrestore(&p->cgr_lock, irqflags); + put_affine_portal(); + return ret; +} + +struct update_cgr_params { + struct qman_cgr *cgr; + struct qm_mcc_initcgr *opts; + int ret; +}; + +static void qman_update_cgr_smp_call(void *p) +{ + struct update_cgr_params *params = p; + + params->ret = qman_update_cgr(params->cgr, params->opts); +} + +int qman_update_cgr_safe(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts) +{ + struct update_cgr_params params = { + .cgr = cgr, + .opts = opts, + }; + + preempt_disable(); + if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) + smp_call_function_single(qman_cgr_cpus[cgr->cgrid], + qman_update_cgr_smp_call, ¶ms, + true); + else + params.ret = qman_update_cgr(cgr, opts); + preempt_enable(); + return params.ret; +} +EXPORT_SYMBOL(qman_update_cgr_safe); + /* Cleanup FQs */ static int _qm_mr_consume_and_match_verb(struct qm_portal *p, int v) diff --git a/drivers/soc/imx/Kconfig b/drivers/soc/imx/Kconfig index a840494e849a032b715f1c646296e70cb25287c8..4b906791d6c7d7b1cf276a4d3d367817833fe6bc 100644 --- a/drivers/soc/imx/Kconfig +++ b/drivers/soc/imx/Kconfig @@ -20,4 +20,12 @@ config SOC_IMX8M support, it will provide the SoC info like SoC family, ID and revision etc. +config SOC_IMX9 + tristate "i.MX9 SoC family support" + depends on ARCH_MXC || COMPILE_TEST + default ARCH_MXC && ARM64 + select SOC_BUS + help + If you say yes here, you get support for the NXP i.MX9 family + endmenu diff --git a/drivers/soc/imx/Makefile b/drivers/soc/imx/Makefile index 63cd29f6d4d21cb1c7add53b547d63771956bad4..7b4099ceafd6257722d85d9ad180e162608fdbf7 100644 --- a/drivers/soc/imx/Makefile +++ b/drivers/soc/imx/Makefile @@ -7,3 +7,5 @@ obj-$(CONFIG_IMX_GPCV2_PM_DOMAINS) += gpcv2.o obj-$(CONFIG_SOC_IMX8M) += soc-imx8m.o obj-$(CONFIG_SOC_IMX8M) += imx8m-blk-ctrl.o obj-$(CONFIG_SOC_IMX8M) += imx8mp-blk-ctrl.o +obj-$(CONFIG_SOC_IMX9) += imx93-src.o imx93-pd.o +obj-$(CONFIG_SOC_IMX9) += imx93-blk-ctrl.o diff --git a/drivers/soc/imx/gpcv2.c b/drivers/soc/imx/gpcv2.c index 6383a4edc3607fe44018665f3baa08c5d7cbcaae..88aee59730e397eb86105a3bcfbd8072586bd9c3 100644 --- a/drivers/soc/imx/gpcv2.c +++ b/drivers/soc/imx/gpcv2.c @@ -335,6 +335,8 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd) } } + reset_control_assert(domain->reset); + /* Enable reset clocks for all devices in the domain */ ret = clk_bulk_prepare_enable(domain->num_clks, domain->clks); if (ret) { @@ -342,7 +344,8 @@ static int imx_pgc_power_up(struct generic_pm_domain *genpd) goto out_regulator_disable; } - reset_control_assert(domain->reset); + /* delays for reset to propagate */ + udelay(5); if (domain->bits.pxx) { /* request the domain to power up */ diff --git a/drivers/soc/imx/imx8m-blk-ctrl.c b/drivers/soc/imx/imx8m-blk-ctrl.c index dff7529268e4dcec93b2d12a61e125799fc96e1f..00879615a701588f3c52ef025958ba3dd3b3a4fc 100644 --- a/drivers/soc/imx/imx8m-blk-ctrl.c +++ b/drivers/soc/imx/imx8m-blk-ctrl.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -37,6 +38,8 @@ struct imx8m_blk_ctrl_domain_data { const char *name; const char * const *clk_names; int num_clks; + const char * const *path_names; + int num_paths; const char *gpc_name; u32 rst_mask; u32 clk_mask; @@ -52,13 +55,16 @@ struct imx8m_blk_ctrl_domain_data { }; #define DOMAIN_MAX_CLKS 4 +#define DOMAIN_MAX_PATHS 4 struct imx8m_blk_ctrl_domain { struct generic_pm_domain genpd; const struct imx8m_blk_ctrl_domain_data *data; struct clk_bulk_data clks[DOMAIN_MAX_CLKS]; + struct icc_bulk_data paths[DOMAIN_MAX_PATHS]; struct device *power_dev; struct imx8m_blk_ctrl *bc; + int num_paths; }; struct imx8m_blk_ctrl_data { @@ -117,6 +123,10 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd) if (data->mipi_phy_rst_mask) regmap_set_bits(bc->regmap, BLK_MIPI_RESET_DIV, data->mipi_phy_rst_mask); + ret = icc_bulk_set_bw(domain->num_paths, domain->paths); + if (ret) + dev_err(bc->dev, "failed to set icc bw\n"); + /* disable upstream clocks */ clk_bulk_disable_unprepare(data->num_clks, domain->clks); @@ -152,19 +162,6 @@ static int imx8m_blk_ctrl_power_off(struct generic_pm_domain *genpd) return 0; } -static struct generic_pm_domain * -imx8m_blk_ctrl_xlate(struct of_phandle_args *args, void *data) -{ - struct genpd_onecell_data *onecell_data = data; - unsigned int index = args->args[0]; - - if (args->args_count != 1 || - index >= onecell_data->num_domains) - return ERR_PTR(-EINVAL); - - return onecell_data->domains[index]; -} - static struct lock_class_key blk_ctrl_genpd_lock_class; static int imx8m_blk_ctrl_probe(struct platform_device *pdev) @@ -206,7 +203,6 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) return -ENOMEM; bc->onecell_data.num_domains = bc_data->num_domains; - bc->onecell_data.xlate = imx8m_blk_ctrl_xlate; bc->onecell_data.domains = devm_kcalloc(dev, bc_data->num_domains, sizeof(struct generic_pm_domain *), GFP_KERNEL); @@ -224,10 +220,29 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) int j; domain->data = data; + domain->num_paths = data->num_paths; for (j = 0; j < data->num_clks; j++) domain->clks[j].id = data->clk_names[j]; + for (j = 0; j < data->num_paths; j++) { + domain->paths[j].name = data->path_names[j]; + /* Fake value for now, just let ICC could configure NoC mode/priority */ + domain->paths[j].avg_bw = 1; + domain->paths[j].peak_bw = 1; + } + + ret = devm_of_icc_bulk_get(dev, data->num_paths, domain->paths); + if (ret) { + if (ret != -EPROBE_DEFER) { + dev_warn_once(dev, "Could not get interconnect paths, NoC will stay unconfigured!\n"); + domain->num_paths = 0; + } else { + dev_err_probe(dev, ret, "failed to get noc entries\n"); + goto cleanup_pds; + } + } + ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks); if (ret) { dev_err_probe(dev, ret, "failed to get clock\n"); @@ -243,7 +258,6 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) ret = PTR_ERR(domain->power_dev); goto cleanup_pds; } - dev_set_name(domain->power_dev, "%s", data->name); domain->genpd.name = data->name; domain->genpd.power_on = imx8m_blk_ctrl_power_on; @@ -455,6 +469,46 @@ static const struct imx8m_blk_ctrl_data imx8mm_vpu_blk_ctl_dev_data = { .num_domains = ARRAY_SIZE(imx8mm_vpu_blk_ctl_domain_data), }; +static const struct imx8m_blk_ctrl_domain_data imx8mp_vpu_blk_ctl_domain_data[] = { + [IMX8MP_VPUBLK_PD_G1] = { + .name = "vpublk-g1", + .clk_names = (const char *[]){ "g1", }, + .num_clks = 1, + .gpc_name = "g1", + .rst_mask = BIT(1), + .clk_mask = BIT(1), + .path_names = (const char *[]){"g1"}, + .num_paths = 1, + }, + [IMX8MP_VPUBLK_PD_G2] = { + .name = "vpublk-g2", + .clk_names = (const char *[]){ "g2", }, + .num_clks = 1, + .gpc_name = "g2", + .rst_mask = BIT(0), + .clk_mask = BIT(0), + .path_names = (const char *[]){"g2"}, + .num_paths = 1, + }, + [IMX8MP_VPUBLK_PD_VC8000E] = { + .name = "vpublk-vc8000e", + .clk_names = (const char *[]){ "vc8000e", }, + .num_clks = 1, + .gpc_name = "vc8000e", + .rst_mask = BIT(2), + .clk_mask = BIT(2), + .path_names = (const char *[]){"vc8000e"}, + .num_paths = 1, + }, +}; + +static const struct imx8m_blk_ctrl_data imx8mp_vpu_blk_ctl_dev_data = { + .max_reg = 0x18, + .power_notifier_fn = imx8mm_vpu_power_notifier, + .domains = imx8mp_vpu_blk_ctl_domain_data, + .num_domains = ARRAY_SIZE(imx8mp_vpu_blk_ctl_domain_data), +}; + static int imx8mm_disp_power_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -650,6 +704,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[ .gpc_name = "lcdif1", .rst_mask = BIT(4) | BIT(5) | BIT(23), .clk_mask = BIT(4) | BIT(5) | BIT(23), + .path_names = (const char *[]){"lcdif-rd", "lcdif-wr"}, + .num_paths = 2, }, [IMX8MP_MEDIABLK_PD_ISI] = { .name = "mediablk-isi", @@ -658,6 +714,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[ .gpc_name = "isi", .rst_mask = BIT(6) | BIT(7), .clk_mask = BIT(6) | BIT(7), + .path_names = (const char *[]){"isi0", "isi1", "isi2"}, + .num_paths = 3, }, [IMX8MP_MEDIABLK_PD_MIPI_CSI2_2] = { .name = "mediablk-mipi-csi2-2", @@ -675,6 +733,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[ .gpc_name = "lcdif2", .rst_mask = BIT(11) | BIT(12) | BIT(24), .clk_mask = BIT(11) | BIT(12) | BIT(24), + .path_names = (const char *[]){"lcdif-rd", "lcdif-wr"}, + .num_paths = 2, }, [IMX8MP_MEDIABLK_PD_ISP] = { .name = "mediablk-isp", @@ -683,6 +743,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[ .gpc_name = "isp", .rst_mask = BIT(16) | BIT(17) | BIT(18), .clk_mask = BIT(16) | BIT(17) | BIT(18), + .path_names = (const char *[]){"isp0", "isp1"}, + .num_paths = 2, }, [IMX8MP_MEDIABLK_PD_DWE] = { .name = "mediablk-dwe", @@ -691,6 +753,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[ .gpc_name = "dwe", .rst_mask = BIT(19) | BIT(20) | BIT(21), .clk_mask = BIT(19) | BIT(20) | BIT(21), + .path_names = (const char *[]){"dwe"}, + .num_paths = 1, }, [IMX8MP_MEDIABLK_PD_MIPI_DSI_2] = { .name = "mediablk-mipi-dsi-2", @@ -788,6 +852,9 @@ static const struct of_device_id imx8m_blk_ctrl_of_match[] = { }, { .compatible = "fsl,imx8mq-vpu-blk-ctrl", .data = &imx8mq_vpu_blk_ctl_dev_data + }, { + .compatible = "fsl,imx8mp-vpu-blk-ctrl", + .data = &imx8mp_vpu_blk_ctl_dev_data }, { /* Sentinel */ } diff --git a/drivers/soc/imx/imx8mp-blk-ctrl.c b/drivers/soc/imx/imx8mp-blk-ctrl.c index 4ca2ede6871bded76cbce17239f1eed25163aa56..0e3b6ba22f943e0b3bd2a1c64189e3a1e964d137 100644 --- a/drivers/soc/imx/imx8mp-blk-ctrl.c +++ b/drivers/soc/imx/imx8mp-blk-ctrl.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #define GPR_REG0 0x0 #define PCIE_CLOCK_MODULE_EN BIT(0) #define USB_CLOCK_MODULE_EN BIT(1) +#define PCIE_PHY_APB_RST BIT(4) +#define PCIE_PHY_INIT_RST BIT(5) struct imx8mp_blk_ctrl_domain; @@ -36,17 +39,22 @@ struct imx8mp_blk_ctrl_domain_data { const char *name; const char * const *clk_names; int num_clks; + const char * const *path_names; + int num_paths; const char *gpc_name; }; #define DOMAIN_MAX_CLKS 2 +#define DOMAIN_MAX_PATHS 3 struct imx8mp_blk_ctrl_domain { struct generic_pm_domain genpd; const struct imx8mp_blk_ctrl_domain_data *data; struct clk_bulk_data clks[DOMAIN_MAX_CLKS]; + struct icc_bulk_data paths[DOMAIN_MAX_PATHS]; struct device *power_dev; struct imx8mp_blk_ctrl *bc; + int num_paths; int id; }; @@ -75,6 +83,10 @@ static void imx8mp_hsio_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc, case IMX8MP_HSIOBLK_PD_PCIE: regmap_set_bits(bc->regmap, GPR_REG0, PCIE_CLOCK_MODULE_EN); break; + case IMX8MP_HSIOBLK_PD_PCIE_PHY: + regmap_set_bits(bc->regmap, GPR_REG0, + PCIE_PHY_APB_RST | PCIE_PHY_INIT_RST); + break; default: break; } @@ -90,6 +102,10 @@ static void imx8mp_hsio_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc, case IMX8MP_HSIOBLK_PD_PCIE: regmap_clear_bits(bc->regmap, GPR_REG0, PCIE_CLOCK_MODULE_EN); break; + case IMX8MP_HSIOBLK_PD_PCIE_PHY: + regmap_clear_bits(bc->regmap, GPR_REG0, + PCIE_PHY_APB_RST | PCIE_PHY_INIT_RST); + break; default: break; } @@ -144,6 +160,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hsio_domain_data[] = { .clk_names = (const char *[]){ "usb" }, .num_clks = 1, .gpc_name = "usb", + .path_names = (const char *[]){"usb1", "usb2"}, + .num_paths = 2, }, [IMX8MP_HSIOBLK_PD_USB_PHY1] = { .name = "hsioblk-usb-phy1", @@ -158,6 +176,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hsio_domain_data[] = { .clk_names = (const char *[]){ "pcie" }, .num_clks = 1, .gpc_name = "pcie", + .path_names = (const char *[]){"noc-pcie", "pcie"}, + .num_paths = 2, }, [IMX8MP_HSIOBLK_PD_PCIE_PHY] = { .name = "hsioblk-pcie-phy", @@ -225,6 +245,13 @@ static void imx8mp_hdmi_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc, regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12)); regmap_clear_bits(bc->regmap, HDMI_TX_CONTROL0, BIT(3)); break; + case IMX8MP_HDMIBLK_PD_HDCP: + regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); + break; + case IMX8MP_HDMIBLK_PD_HRV: + regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); + regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); + break; default: break; } @@ -273,6 +300,13 @@ static void imx8mp_hdmi_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc, regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12)); regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(22) | BIT(24)); break; + case IMX8MP_HDMIBLK_PD_HDCP: + regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11)); + break; + case IMX8MP_HDMIBLK_PD_HRV: + regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15)); + regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5)); + break; default: break; } @@ -322,6 +356,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { .clk_names = (const char *[]){ "axi", "apb" }, .num_clks = 2, .gpc_name = "lcdif", + .path_names = (const char *[]){"lcdif-hdmi"}, + .num_paths = 1, }, [IMX8MP_HDMIBLK_PD_PAI] = { .name = "hdmiblk-pai", @@ -353,6 +389,22 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = { .num_clks = 2, .gpc_name = "hdmi-tx-phy", }, + [IMX8MP_HDMIBLK_PD_HRV] = { + .name = "hdmiblk-hrv", + .clk_names = (const char *[]){ "axi", "apb" }, + .num_clks = 2, + .gpc_name = "hrv", + .path_names = (const char *[]){"hrv"}, + .num_paths = 1, + }, + [IMX8MP_HDMIBLK_PD_HDCP] = { + .name = "hdmiblk-hdcp", + .clk_names = (const char *[]){ "axi", "apb" }, + .num_clks = 2, + .gpc_name = "hdcp", + .path_names = (const char *[]){"hdcp"}, + .num_paths = 1, + }, }; static const struct imx8mp_blk_ctrl_data imx8mp_hdmi_blk_ctl_dev_data = { @@ -395,6 +447,10 @@ static int imx8mp_blk_ctrl_power_on(struct generic_pm_domain *genpd) goto clk_disable; } + ret = icc_bulk_set_bw(domain->num_paths, domain->paths); + if (ret) + dev_err(bc->dev, "failed to set icc bw\n"); + clk_bulk_disable_unprepare(data->num_clks, domain->clks); return 0; @@ -434,19 +490,6 @@ static int imx8mp_blk_ctrl_power_off(struct generic_pm_domain *genpd) return 0; } -static struct generic_pm_domain * -imx8m_blk_ctrl_xlate(struct of_phandle_args *args, void *data) -{ - struct genpd_onecell_data *onecell_data = data; - unsigned int index = args->args[0]; - - if (args->args_count != 1 || - index >= onecell_data->num_domains) - return ERR_PTR(-EINVAL); - - return onecell_data->domains[index]; -} - static struct lock_class_key blk_ctrl_genpd_lock_class; static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) @@ -489,7 +532,6 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) return -ENOMEM; bc->onecell_data.num_domains = num_domains; - bc->onecell_data.xlate = imx8m_blk_ctrl_xlate; bc->onecell_data.domains = devm_kcalloc(dev, num_domains, sizeof(struct generic_pm_domain *), GFP_KERNEL); @@ -510,10 +552,29 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev) int j; domain->data = data; + domain->num_paths = data->num_paths; for (j = 0; j < data->num_clks; j++) domain->clks[j].id = data->clk_names[j]; + for (j = 0; j < data->num_paths; j++) { + domain->paths[j].name = data->path_names[j]; + /* Fake value for now, just let ICC could configure NoC mode/priority */ + domain->paths[j].avg_bw = 1; + domain->paths[j].peak_bw = 1; + } + + ret = devm_of_icc_bulk_get(dev, data->num_paths, domain->paths); + if (ret) { + if (ret != -EPROBE_DEFER) { + dev_warn_once(dev, "Could not get interconnect paths, NoC will stay unconfigured!\n"); + domain->num_paths = 0; + } else { + dev_err_probe(dev, ret, "failed to get noc entries\n"); + goto cleanup_pds; + } + } + ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks); if (ret) { dev_err_probe(dev, ret, "failed to get clock\n"); diff --git a/drivers/soc/imx/imx93-blk-ctrl.c b/drivers/soc/imx/imx93-blk-ctrl.c new file mode 100644 index 0000000000000000000000000000000000000000..2c600329436cf288250e76df5d91fd1b8851dacb --- /dev/null +++ b/drivers/soc/imx/imx93-blk-ctrl.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2022 NXP, Peng Fan + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define BLK_SFT_RSTN 0x0 +#define BLK_CLK_EN 0x4 +#define BLK_MAX_CLKS 4 + +#define DOMAIN_MAX_CLKS 4 + +#define LCDIF_QOS_REG 0xC +#define LCDIF_DEFAULT_QOS_OFF 12 +#define LCDIF_CFG_QOS_OFF 8 + +#define PXP_QOS_REG 0x10 +#define PXP_R_DEFAULT_QOS_OFF 28 +#define PXP_R_CFG_QOS_OFF 24 +#define PXP_W_DEFAULT_QOS_OFF 20 +#define PXP_W_CFG_QOS_OFF 16 + +#define ISI_CACHE_REG 0x14 + +#define ISI_QOS_REG 0x1C +#define ISI_V_DEFAULT_QOS_OFF 28 +#define ISI_V_CFG_QOS_OFF 24 +#define ISI_U_DEFAULT_QOS_OFF 20 +#define ISI_U_CFG_QOS_OFF 16 +#define ISI_Y_R_DEFAULT_QOS_OFF 12 +#define ISI_Y_R_CFG_QOS_OFF 8 +#define ISI_Y_W_DEFAULT_QOS_OFF 4 +#define ISI_Y_W_CFG_QOS_OFF 0 + +#define PRIO_MASK 0xF + +#define PRIO(X) (X) + +struct imx93_blk_ctrl_domain; + +struct imx93_blk_ctrl { + struct device *dev; + struct regmap *regmap; + int num_clks; + struct clk_bulk_data clks[BLK_MAX_CLKS]; + struct imx93_blk_ctrl_domain *domains; + struct genpd_onecell_data onecell_data; +}; + +#define DOMAIN_MAX_QOS 4 + +struct imx93_blk_ctrl_qos { + u32 reg; + u32 cfg_off; + u32 default_prio; + u32 cfg_prio; +}; + +struct imx93_blk_ctrl_domain_data { + const char *name; + const char * const *clk_names; + int num_clks; + u32 rst_mask; + u32 clk_mask; + int num_qos; + struct imx93_blk_ctrl_qos qos[DOMAIN_MAX_QOS]; +}; + +struct imx93_blk_ctrl_domain { + struct generic_pm_domain genpd; + const struct imx93_blk_ctrl_domain_data *data; + struct clk_bulk_data clks[DOMAIN_MAX_CLKS]; + struct imx93_blk_ctrl *bc; +}; + +struct imx93_blk_ctrl_data { + const struct imx93_blk_ctrl_domain_data *domains; + int num_domains; + const char * const *clk_names; + int num_clks; + const struct regmap_access_table *reg_access_table; +}; + +static inline struct imx93_blk_ctrl_domain * +to_imx93_blk_ctrl_domain(struct generic_pm_domain *genpd) +{ + return container_of(genpd, struct imx93_blk_ctrl_domain, genpd); +} + +static int imx93_blk_ctrl_set_qos(struct imx93_blk_ctrl_domain *domain) +{ + const struct imx93_blk_ctrl_domain_data *data = domain->data; + struct imx93_blk_ctrl *bc = domain->bc; + const struct imx93_blk_ctrl_qos *qos; + u32 val, mask; + int i; + + for (i = 0; i < data->num_qos; i++) { + qos = &data->qos[i]; + + mask = PRIO_MASK << qos->cfg_off; + mask |= PRIO_MASK << (qos->cfg_off + 4); + val = qos->cfg_prio << qos->cfg_off; + val |= qos->default_prio << (qos->cfg_off + 4); + + regmap_write_bits(bc->regmap, qos->reg, mask, val); + + dev_dbg(bc->dev, "data->qos[i].reg 0x%x 0x%x\n", qos->reg, val); + } + + return 0; +} + +static int imx93_blk_ctrl_power_on(struct generic_pm_domain *genpd) +{ + struct imx93_blk_ctrl_domain *domain = to_imx93_blk_ctrl_domain(genpd); + const struct imx93_blk_ctrl_domain_data *data = domain->data; + struct imx93_blk_ctrl *bc = domain->bc; + int ret; + + ret = clk_bulk_prepare_enable(bc->num_clks, bc->clks); + if (ret) { + dev_err(bc->dev, "failed to enable bus clocks\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(data->num_clks, domain->clks); + if (ret) { + clk_bulk_disable_unprepare(bc->num_clks, bc->clks); + dev_err(bc->dev, "failed to enable clocks\n"); + return ret; + } + + ret = pm_runtime_get_sync(bc->dev); + if (ret < 0) { + pm_runtime_put_noidle(bc->dev); + dev_err(bc->dev, "failed to power up domain\n"); + goto disable_clk; + } + + /* ungate clk */ + regmap_clear_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); + + /* release reset */ + regmap_set_bits(bc->regmap, BLK_SFT_RSTN, data->rst_mask); + + dev_dbg(bc->dev, "pd_on: name: %s\n", genpd->name); + + return imx93_blk_ctrl_set_qos(domain); + +disable_clk: + clk_bulk_disable_unprepare(data->num_clks, domain->clks); + + clk_bulk_disable_unprepare(bc->num_clks, bc->clks); + + return ret; +} + +static int imx93_blk_ctrl_power_off(struct generic_pm_domain *genpd) +{ + struct imx93_blk_ctrl_domain *domain = to_imx93_blk_ctrl_domain(genpd); + const struct imx93_blk_ctrl_domain_data *data = domain->data; + struct imx93_blk_ctrl *bc = domain->bc; + + dev_dbg(bc->dev, "pd_off: name: %s\n", genpd->name); + + regmap_clear_bits(bc->regmap, BLK_SFT_RSTN, data->rst_mask); + regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask); + + pm_runtime_put(bc->dev); + + clk_bulk_disable_unprepare(data->num_clks, domain->clks); + + clk_bulk_disable_unprepare(bc->num_clks, bc->clks); + + return 0; +} + +static int imx93_blk_ctrl_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct imx93_blk_ctrl_data *bc_data = of_device_get_match_data(dev); + struct imx93_blk_ctrl *bc; + void __iomem *base; + int i, ret; + + struct regmap_config regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .rd_table = bc_data->reg_access_table, + .wr_table = bc_data->reg_access_table, + .max_register = SZ_4K, + }; + + bc = devm_kzalloc(dev, sizeof(*bc), GFP_KERNEL); + if (!bc) + return -ENOMEM; + + bc->dev = dev; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + bc->regmap = devm_regmap_init_mmio(dev, base, ®map_config); + if (IS_ERR(bc->regmap)) + return dev_err_probe(dev, PTR_ERR(bc->regmap), + "failed to init regmap\n"); + + bc->domains = devm_kcalloc(dev, bc_data->num_domains, + sizeof(struct imx93_blk_ctrl_domain), + GFP_KERNEL); + if (!bc->domains) + return -ENOMEM; + + bc->onecell_data.num_domains = bc_data->num_domains; + bc->onecell_data.domains = + devm_kcalloc(dev, bc_data->num_domains, + sizeof(struct generic_pm_domain *), GFP_KERNEL); + if (!bc->onecell_data.domains) + return -ENOMEM; + + for (i = 0; i < bc_data->num_clks; i++) + bc->clks[i].id = bc_data->clk_names[i]; + bc->num_clks = bc_data->num_clks; + + ret = devm_clk_bulk_get(dev, bc->num_clks, bc->clks); + if (ret) { + dev_err_probe(dev, ret, "failed to get bus clock\n"); + return ret; + } + + for (i = 0; i < bc_data->num_domains; i++) { + const struct imx93_blk_ctrl_domain_data *data = &bc_data->domains[i]; + struct imx93_blk_ctrl_domain *domain = &bc->domains[i]; + int j; + + domain->data = data; + + for (j = 0; j < data->num_clks; j++) + domain->clks[j].id = data->clk_names[j]; + + ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks); + if (ret) { + dev_err_probe(dev, ret, "failed to get clock\n"); + goto cleanup_pds; + } + + domain->genpd.name = data->name; + domain->genpd.power_on = imx93_blk_ctrl_power_on; + domain->genpd.power_off = imx93_blk_ctrl_power_off; + domain->bc = bc; + + ret = pm_genpd_init(&domain->genpd, NULL, true); + if (ret) { + dev_err_probe(dev, ret, "failed to init power domain\n"); + goto cleanup_pds; + } + + bc->onecell_data.domains[i] = &domain->genpd; + } + + pm_runtime_enable(dev); + + ret = of_genpd_add_provider_onecell(dev->of_node, &bc->onecell_data); + if (ret) { + dev_err_probe(dev, ret, "failed to add power domain provider\n"); + goto cleanup_pds; + } + + dev_set_drvdata(dev, bc); + + return 0; + +cleanup_pds: + for (i--; i >= 0; i--) + pm_genpd_remove(&bc->domains[i].genpd); + + return ret; +} + +static int imx93_blk_ctrl_remove(struct platform_device *pdev) +{ + struct imx93_blk_ctrl *bc = dev_get_drvdata(&pdev->dev); + int i; + + of_genpd_del_provider(pdev->dev.of_node); + + for (i = 0; bc->onecell_data.num_domains; i++) { + struct imx93_blk_ctrl_domain *domain = &bc->domains[i]; + + pm_genpd_remove(&domain->genpd); + } + + return 0; +} + +static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] = { + [IMX93_MEDIABLK_PD_MIPI_DSI] = { + .name = "mediablk-mipi-dsi", + .clk_names = (const char *[]){ "dsi" }, + .num_clks = 1, + .rst_mask = BIT(11) | BIT(12), + .clk_mask = BIT(11) | BIT(12), + }, + [IMX93_MEDIABLK_PD_MIPI_CSI] = { + .name = "mediablk-mipi-csi", + .clk_names = (const char *[]){ "cam", "csi" }, + .num_clks = 2, + .rst_mask = BIT(9) | BIT(10), + .clk_mask = BIT(9) | BIT(10), + }, + [IMX93_MEDIABLK_PD_PXP] = { + .name = "mediablk-pxp", + .clk_names = (const char *[]){ "pxp" }, + .num_clks = 1, + .rst_mask = BIT(7) | BIT(8), + .clk_mask = BIT(7) | BIT(8), + .num_qos = 2, + .qos = { + { + .reg = PXP_QOS_REG, + .cfg_off = PXP_R_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(6), + }, { + .reg = PXP_QOS_REG, + .cfg_off = PXP_W_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(6), + } + } + }, + [IMX93_MEDIABLK_PD_LCDIF] = { + .name = "mediablk-lcdif", + .clk_names = (const char *[]){ "disp", "lcdif" }, + .num_clks = 2, + .rst_mask = BIT(4) | BIT(5) | BIT(6), + .clk_mask = BIT(4) | BIT(5) | BIT(6), + .num_qos = 1, + .qos = { + { + .reg = LCDIF_QOS_REG, + .cfg_off = LCDIF_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(7), + } + } + }, + [IMX93_MEDIABLK_PD_ISI] = { + .name = "mediablk-isi", + .clk_names = (const char *[]){ "isi" }, + .num_clks = 1, + .rst_mask = BIT(2) | BIT(3), + .clk_mask = BIT(2) | BIT(3), + .num_qos = 4, + .qos = { + { + .reg = ISI_QOS_REG, + .cfg_off = ISI_Y_W_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(7), + }, { + .reg = ISI_QOS_REG, + .cfg_off = ISI_Y_R_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(7), + }, { + .reg = ISI_QOS_REG, + .cfg_off = ISI_U_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(7), + }, { + .reg = ISI_QOS_REG, + .cfg_off = ISI_V_CFG_QOS_OFF, + .default_prio = PRIO(3), + .cfg_prio = PRIO(7), + } + } + }, +}; + +static const struct regmap_range imx93_media_blk_ctl_yes_ranges[] = { + regmap_reg_range(BLK_SFT_RSTN, BLK_CLK_EN), + regmap_reg_range(LCDIF_QOS_REG, ISI_CACHE_REG), + regmap_reg_range(ISI_QOS_REG, ISI_QOS_REG), +}; + +static const struct regmap_access_table imx93_media_blk_ctl_access_table = { + .yes_ranges = imx93_media_blk_ctl_yes_ranges, + .n_yes_ranges = ARRAY_SIZE(imx93_media_blk_ctl_yes_ranges), +}; + +static const struct imx93_blk_ctrl_data imx93_media_blk_ctl_dev_data = { + .domains = imx93_media_blk_ctl_domain_data, + .num_domains = ARRAY_SIZE(imx93_media_blk_ctl_domain_data), + .clk_names = (const char *[]){ "axi", "apb", "nic", }, + .num_clks = 3, + .reg_access_table = &imx93_media_blk_ctl_access_table, +}; + +static const struct of_device_id imx93_blk_ctrl_of_match[] = { + { + .compatible = "fsl,imx93-media-blk-ctrl", + .data = &imx93_media_blk_ctl_dev_data + }, { + /* Sentinel */ + } +}; +MODULE_DEVICE_TABLE(of, imx93_blk_ctrl_of_match); + +static struct platform_driver imx93_blk_ctrl_driver = { + .probe = imx93_blk_ctrl_probe, + .remove = imx93_blk_ctrl_remove, + .driver = { + .name = "imx93-blk-ctrl", + .of_match_table = imx93_blk_ctrl_of_match, + }, +}; +module_platform_driver(imx93_blk_ctrl_driver); + +MODULE_AUTHOR("Peng Fan "); +MODULE_DESCRIPTION("i.MX93 BLK CTRL driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/imx/imx93-pd.c b/drivers/soc/imx/imx93-pd.c new file mode 100644 index 0000000000000000000000000000000000000000..1f3d7039c1de9c674deda7ad0e9f473f2cb44e74 --- /dev/null +++ b/drivers/soc/imx/imx93-pd.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2022 NXP + */ + +#include +#include +#include +#include +#include +#include +#include + +#define MIX_SLICE_SW_CTRL_OFF 0x20 +#define SLICE_SW_CTRL_PSW_CTRL_OFF_MASK BIT(4) +#define SLICE_SW_CTRL_PDN_SOFT_MASK BIT(31) + +#define MIX_FUNC_STAT_OFF 0xB4 + +#define FUNC_STAT_PSW_STAT_MASK BIT(0) +#define FUNC_STAT_RST_STAT_MASK BIT(2) +#define FUNC_STAT_ISO_STAT_MASK BIT(4) + +struct imx93_power_domain { + struct generic_pm_domain genpd; + struct device *dev; + void __iomem *addr; + struct clk_bulk_data *clks; + int num_clks; + bool init_off; +}; + +#define to_imx93_pd(_genpd) container_of(_genpd, struct imx93_power_domain, genpd) + +static int imx93_pd_on(struct generic_pm_domain *genpd) +{ + struct imx93_power_domain *domain = to_imx93_pd(genpd); + void __iomem *addr = domain->addr; + u32 val; + int ret; + + ret = clk_bulk_prepare_enable(domain->num_clks, domain->clks); + if (ret) { + dev_err(domain->dev, "failed to enable clocks for domain: %s\n", genpd->name); + return ret; + } + + val = readl(addr + MIX_SLICE_SW_CTRL_OFF); + val &= ~SLICE_SW_CTRL_PDN_SOFT_MASK; + writel(val, addr + MIX_SLICE_SW_CTRL_OFF); + + ret = readl_poll_timeout(addr + MIX_FUNC_STAT_OFF, val, + !(val & FUNC_STAT_ISO_STAT_MASK), 1, 10000); + if (ret) { + dev_err(domain->dev, "pd_on timeout: name: %s, stat: %x\n", genpd->name, val); + return ret; + } + + return 0; +} + +static int imx93_pd_off(struct generic_pm_domain *genpd) +{ + struct imx93_power_domain *domain = to_imx93_pd(genpd); + void __iomem *addr = domain->addr; + int ret; + u32 val; + + /* Power off MIX */ + val = readl(addr + MIX_SLICE_SW_CTRL_OFF); + val |= SLICE_SW_CTRL_PDN_SOFT_MASK; + writel(val, addr + MIX_SLICE_SW_CTRL_OFF); + + ret = readl_poll_timeout(addr + MIX_FUNC_STAT_OFF, val, + val & FUNC_STAT_PSW_STAT_MASK, 1, 1000); + if (ret) { + dev_err(domain->dev, "pd_off timeout: name: %s, stat: %x\n", genpd->name, val); + return ret; + } + + clk_bulk_disable_unprepare(domain->num_clks, domain->clks); + + return 0; +}; + +static int imx93_pd_remove(struct platform_device *pdev) +{ + struct imx93_power_domain *domain = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + + if (!domain->init_off) + clk_bulk_disable_unprepare(domain->num_clks, domain->clks); + + of_genpd_del_provider(np); + pm_genpd_remove(&domain->genpd); + + return 0; +} + +static int imx93_pd_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct imx93_power_domain *domain; + int ret; + + domain = devm_kzalloc(dev, sizeof(*domain), GFP_KERNEL); + if (!domain) + return -ENOMEM; + + domain->addr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(domain->addr)) + return PTR_ERR(domain->addr); + + domain->num_clks = devm_clk_bulk_get_all(dev, &domain->clks); + if (domain->num_clks < 0) + return dev_err_probe(dev, domain->num_clks, "Failed to get domain's clocks\n"); + + domain->genpd.name = dev_name(dev); + domain->genpd.power_off = imx93_pd_off; + domain->genpd.power_on = imx93_pd_on; + domain->dev = dev; + + domain->init_off = readl(domain->addr + MIX_FUNC_STAT_OFF) & FUNC_STAT_ISO_STAT_MASK; + /* Just to sync the status of hardware */ + if (!domain->init_off) { + ret = clk_bulk_prepare_enable(domain->num_clks, domain->clks); + if (ret) { + dev_err(domain->dev, "failed to enable clocks for domain: %s\n", + domain->genpd.name); + return ret; + } + } + + ret = pm_genpd_init(&domain->genpd, NULL, domain->init_off); + if (ret) + return ret; + + platform_set_drvdata(pdev, domain); + + return of_genpd_add_provider_simple(np, &domain->genpd); +} + +static const struct of_device_id imx93_pd_ids[] = { + { .compatible = "fsl,imx93-src-slice" }, + { } +}; +MODULE_DEVICE_TABLE(of, imx93_pd_ids); + +static struct platform_driver imx93_power_domain_driver = { + .driver = { + .name = "imx93_power_domain", + .owner = THIS_MODULE, + .of_match_table = imx93_pd_ids, + }, + .probe = imx93_pd_probe, + .remove = imx93_pd_remove, +}; +module_platform_driver(imx93_power_domain_driver); + +MODULE_AUTHOR("Peng Fan "); +MODULE_DESCRIPTION("NXP i.MX93 power domain driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/imx/imx93-src.c b/drivers/soc/imx/imx93-src.c new file mode 100644 index 0000000000000000000000000000000000000000..4d74921cae0f1773fa524eb37595296d5f1dae5f --- /dev/null +++ b/drivers/soc/imx/imx93-src.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2022 NXP + */ + +#include +#include +#include + +static int imx93_src_probe(struct platform_device *pdev) +{ + return devm_of_platform_populate(&pdev->dev); +} + +static const struct of_device_id imx93_src_ids[] = { + { .compatible = "fsl,imx93-src" }, + { } +}; +MODULE_DEVICE_TABLE(of, imx93_src_ids); + +static struct platform_driver imx93_src_driver = { + .driver = { + .name = "imx93_src", + .owner = THIS_MODULE, + .of_match_table = imx93_src_ids, + }, + .probe = imx93_src_probe, +}; +module_platform_driver(imx93_src_driver); + +MODULE_AUTHOR("Peng Fan "); +MODULE_DESCRIPTION("NXP i.MX93 src driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig index 3c3eedea35f729b86217e92ad027b2188867c165..40d0cc600cae33ed84f3b6f5525cfd41ac593c89 100644 --- a/drivers/soc/mediatek/Kconfig +++ b/drivers/soc/mediatek/Kconfig @@ -37,6 +37,7 @@ config MTK_INFRACFG config MTK_PMIC_WRAP tristate "MediaTek PMIC Wrapper Support" depends on RESET_CONTROLLER + depends on OF select REGMAP help Say yes here to add support for MediaTek PMIC Wrapper found @@ -46,6 +47,7 @@ config MTK_PMIC_WRAP config MTK_SCPSYS bool "MediaTek SCPSYS Support" default ARCH_MEDIATEK + depends on OF select REGMAP select MTK_INFRACFG select PM_GENERIC_DOMAINS if PM @@ -75,7 +77,7 @@ config MTK_MMSYS config MTK_SVS tristate "MediaTek Smart Voltage Scaling(SVS)" - depends on MTK_EFUSE && NVMEM + depends on NVMEM_MTK_EFUSE && NVMEM help The Smart Voltage Scaling(SVS) engine is a piece of hardware which has several controllers(banks) for calculating suitable diff --git a/drivers/soc/mediatek/mt8186-mmsys.h b/drivers/soc/mediatek/mt8186-mmsys.h index eb1ad9c37a9cb0177758c527569752beef8f3643..09b1ccbc0093c1a9f700e72982fe1c8607596ac9 100644 --- a/drivers/soc/mediatek/mt8186-mmsys.h +++ b/drivers/soc/mediatek/mt8186-mmsys.h @@ -3,6 +3,12 @@ #ifndef __SOC_MEDIATEK_MT8186_MMSYS_H #define __SOC_MEDIATEK_MT8186_MMSYS_H +/* Values for DPI configuration in MMSYS address space */ +#define MT8186_MMSYS_DPI_OUTPUT_FORMAT 0x400 +#define DPI_FORMAT_MASK 0x1 +#define DPI_RGB888_DDR_CON BIT(0) +#define DPI_RGB565_SDR_CON BIT(1) + #define MT8186_MMSYS_OVL_CON 0xF04 #define MT8186_MMSYS_OVL0_CON_MASK 0x3 #define MT8186_MMSYS_OVL0_2L_CON_MASK 0xC diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c index 06d8e83a2cb56f5afdc25ab423083b7d733befe8..d2c7a87aab87e1dd7d53f69aab51231b88933fd2 100644 --- a/drivers/soc/mediatek/mtk-mmsys.c +++ b/drivers/soc/mediatek/mtk-mmsys.c @@ -227,6 +227,26 @@ void mtk_mmsys_ddp_disconnect(struct device *dev, } EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_disconnect); +static void mtk_mmsys_update_bits(struct mtk_mmsys *mmsys, u32 offset, u32 mask, u32 val) +{ + u32 tmp; + + tmp = readl_relaxed(mmsys->regs + offset); + tmp = (tmp & ~mask) | val; + writel_relaxed(tmp, mmsys->regs + offset); +} + +void mtk_mmsys_ddp_dpi_fmt_config(struct device *dev, u32 val) +{ + if (val) + mtk_mmsys_update_bits(dev_get_drvdata(dev), MT8186_MMSYS_DPI_OUTPUT_FORMAT, + DPI_RGB888_DDR_CON, DPI_FORMAT_MASK); + else + mtk_mmsys_update_bits(dev_get_drvdata(dev), MT8186_MMSYS_DPI_OUTPUT_FORMAT, + DPI_RGB565_SDR_CON, DPI_FORMAT_MASK); +} +EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_dpi_fmt_config); + static int mtk_mmsys_reset_update(struct reset_controller_dev *rcdev, unsigned long id, bool assert) { diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c index 5ea43de4e410662d7e3530987013cc773442a925..c1a33d52038e0c91dd403c9b009330e4d8c9326c 100644 --- a/drivers/soc/mediatek/mtk-mutex.c +++ b/drivers/soc/mediatek/mtk-mutex.c @@ -91,6 +91,15 @@ #define MT8183_MUTEX_MOD_MDP_AAL0 23 #define MT8183_MUTEX_MOD_MDP_CCORR0 24 +#define MT8186_MUTEX_MOD_MDP_RDMA0 0 +#define MT8186_MUTEX_MOD_MDP_AAL0 2 +#define MT8186_MUTEX_MOD_MDP_HDR0 4 +#define MT8186_MUTEX_MOD_MDP_RSZ0 5 +#define MT8186_MUTEX_MOD_MDP_RSZ1 6 +#define MT8186_MUTEX_MOD_MDP_WROT0 7 +#define MT8186_MUTEX_MOD_MDP_TDSHP0 9 +#define MT8186_MUTEX_MOD_MDP_COLOR0 14 + #define MT8173_MUTEX_MOD_DISP_OVL0 11 #define MT8173_MUTEX_MOD_DISP_OVL1 12 #define MT8173_MUTEX_MOD_DISP_RDMA0 13 @@ -324,6 +333,17 @@ static const unsigned int mt8186_mutex_mod[DDP_COMPONENT_ID_MAX] = { [DDP_COMPONENT_RDMA1] = MT8186_MUTEX_MOD_DISP_RDMA1, }; +static const unsigned int mt8186_mdp_mutex_table_mod[MUTEX_MOD_IDX_MAX] = { + [MUTEX_MOD_IDX_MDP_RDMA0] = MT8186_MUTEX_MOD_MDP_RDMA0, + [MUTEX_MOD_IDX_MDP_RSZ0] = MT8186_MUTEX_MOD_MDP_RSZ0, + [MUTEX_MOD_IDX_MDP_RSZ1] = MT8186_MUTEX_MOD_MDP_RSZ1, + [MUTEX_MOD_IDX_MDP_TDSHP0] = MT8186_MUTEX_MOD_MDP_TDSHP0, + [MUTEX_MOD_IDX_MDP_WROT0] = MT8186_MUTEX_MOD_MDP_WROT0, + [MUTEX_MOD_IDX_MDP_HDR0] = MT8186_MUTEX_MOD_MDP_HDR0, + [MUTEX_MOD_IDX_MDP_AAL0] = MT8186_MUTEX_MOD_MDP_AAL0, + [MUTEX_MOD_IDX_MDP_COLOR0] = MT8186_MUTEX_MOD_MDP_COLOR0, +}; + static const unsigned int mt8192_mutex_mod[DDP_COMPONENT_ID_MAX] = { [DDP_COMPONENT_AAL0] = MT8192_MUTEX_MOD_DISP_AAL0, [DDP_COMPONENT_CCORR] = MT8192_MUTEX_MOD_DISP_CCORR0, @@ -380,6 +400,13 @@ static const unsigned int mt2712_mutex_sof[DDP_MUTEX_SOF_MAX] = { [MUTEX_SOF_DSI3] = MUTEX_SOF_DSI3, }; +static const unsigned int mt6795_mutex_sof[DDP_MUTEX_SOF_MAX] = { + [MUTEX_SOF_SINGLE_MODE] = MUTEX_SOF_SINGLE_MODE, + [MUTEX_SOF_DSI0] = MUTEX_SOF_DSI0, + [MUTEX_SOF_DSI1] = MUTEX_SOF_DSI1, + [MUTEX_SOF_DPI0] = MUTEX_SOF_DPI0, +}; + static const unsigned int mt8167_mutex_sof[DDP_MUTEX_SOF_MAX] = { [MUTEX_SOF_SINGLE_MODE] = MUTEX_SOF_SINGLE_MODE, [MUTEX_SOF_DSI0] = MUTEX_SOF_DSI0, @@ -434,6 +461,13 @@ static const struct mtk_mutex_data mt2712_mutex_driver_data = { .mutex_sof_reg = MT2701_MUTEX0_SOF0, }; +static const struct mtk_mutex_data mt6795_mutex_driver_data = { + .mutex_mod = mt8173_mutex_mod, + .mutex_sof = mt6795_mutex_sof, + .mutex_mod_reg = MT2701_MUTEX0_MOD0, + .mutex_sof_reg = MT2701_MUTEX0_SOF0, +}; + static const struct mtk_mutex_data mt8167_mutex_driver_data = { .mutex_mod = mt8167_mutex_mod, .mutex_sof = mt8167_mutex_sof, @@ -458,6 +492,12 @@ static const struct mtk_mutex_data mt8183_mutex_driver_data = { .no_clk = true, }; +static const struct mtk_mutex_data mt8186_mdp_mutex_driver_data = { + .mutex_mod_reg = MT8183_MUTEX0_MOD0, + .mutex_sof_reg = MT8183_MUTEX0_SOF0, + .mutex_table_mod = mt8186_mdp_mutex_table_mod, +}; + static const struct mtk_mutex_data mt8186_mutex_driver_data = { .mutex_mod = mt8186_mutex_mod, .mutex_sof = mt8186_mutex_sof, @@ -802,6 +842,8 @@ static const struct of_device_id mutex_driver_dt_match[] = { .data = &mt2701_mutex_driver_data}, { .compatible = "mediatek,mt2712-disp-mutex", .data = &mt2712_mutex_driver_data}, + { .compatible = "mediatek,mt6795-disp-mutex", + .data = &mt6795_mutex_driver_data}, { .compatible = "mediatek,mt8167-disp-mutex", .data = &mt8167_mutex_driver_data}, { .compatible = "mediatek,mt8173-disp-mutex", @@ -810,6 +852,8 @@ static const struct of_device_id mutex_driver_dt_match[] = { .data = &mt8183_mutex_driver_data}, { .compatible = "mediatek,mt8186-disp-mutex", .data = &mt8186_mutex_driver_data}, + { .compatible = "mediatek,mt8186-mdp3-mutex", + .data = &mt8186_mdp_mutex_driver_data}, { .compatible = "mediatek,mt8192-disp-mutex", .data = &mt8192_mutex_driver_data}, { .compatible = "mediatek,mt8195-disp-mutex", diff --git a/drivers/soc/mediatek/mtk-pm-domains.c b/drivers/soc/mediatek/mtk-pm-domains.c index 9734f1091c695f07b40804c764bfaa8497db0494..09e3c38b8466434b5dfd9eb732494e028670c59d 100644 --- a/drivers/soc/mediatek/mtk-pm-domains.c +++ b/drivers/soc/mediatek/mtk-pm-domains.c @@ -393,7 +393,7 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no if (IS_ERR(clk)) { ret = PTR_ERR(clk); dev_err_probe(scpsys->dev, ret, - "%pOF: failed to get clk at index %d: %d\n", node, i, ret); + "%pOF: failed to get clk at index %d\n", node, i); goto err_put_clocks; } @@ -405,8 +405,8 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no if (IS_ERR(clk)) { ret = PTR_ERR(clk); dev_err_probe(scpsys->dev, ret, - "%pOF: failed to get clk at index %d: %d\n", node, - i + clk_ind, ret); + "%pOF: failed to get clk at index %d\n", node, + i + clk_ind); goto err_put_subsys_clocks; } diff --git a/drivers/soc/mediatek/mtk-pmic-wrap.c b/drivers/soc/mediatek/mtk-pmic-wrap.c index d8cb0f8336452c9aec1b5662ae673b51cf874370..eb82ae06697faa2b83f2b1c1a3127c8b9c5d32e8 100644 --- a/drivers/soc/mediatek/mtk-pmic-wrap.c +++ b/drivers/soc/mediatek/mtk-pmic-wrap.c @@ -2316,7 +2316,7 @@ err_out1: static struct platform_driver pwrap_drv = { .driver = { .name = "mt-pmic-pwrap", - .of_match_table = of_match_ptr(of_pwrap_match_tbl), + .of_match_table = of_pwrap_match_tbl, }, .probe = pwrap_probe, }; diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c index ca75b14931ec9ed925c5760269a005f0a453c84a..7a668888111c24a6c993d0c8026fff27101e3cbd 100644 --- a/drivers/soc/mediatek/mtk-scpsys.c +++ b/drivers/soc/mediatek/mtk-scpsys.c @@ -1141,7 +1141,7 @@ static struct platform_driver scpsys_drv = { .name = "mtk-scpsys", .suppress_bind_attrs = true, .owner = THIS_MODULE, - .of_match_table = of_match_ptr(of_scpsys_match_tbl), + .of_match_table = of_scpsys_match_tbl, }, }; builtin_platform_driver(scpsys_drv); diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c index dee8664a12fd6f080c39406bbf729be9245ed210..0469c9dfeb04ea86fb871c51b5fdd0fa711f8ac0 100644 --- a/drivers/soc/mediatek/mtk-svs.c +++ b/drivers/soc/mediatek/mtk-svs.c @@ -3,6 +3,7 @@ * Copyright (C) 2022 MediaTek Inc. */ +#include #include #include #include @@ -53,22 +54,79 @@ #define SVSB_MON_VOLT_IGNORE BIT(16) #define SVSB_REMOVE_DVTFIXED_VOLT BIT(24) -/* svs bank register common configuration */ -#define SVSB_DET_MAX 0xffff +/* svs bank register fields and common configuration */ +#define SVSB_PTPCONFIG_DETMAX GENMASK(15, 0) +#define SVSB_DET_MAX FIELD_PREP(SVSB_PTPCONFIG_DETMAX, 0xffff) #define SVSB_DET_WINDOW 0xa28 -#define SVSB_DTHI 0x1 -#define SVSB_DTLO 0xfe -#define SVSB_EN_INIT01 0x1 -#define SVSB_EN_INIT02 0x5 -#define SVSB_EN_MON 0x2 -#define SVSB_EN_OFF 0x0 -#define SVSB_INTEN_INIT0x 0x00005f01 -#define SVSB_INTEN_MONVOPEN 0x00ff0000 -#define SVSB_INTSTS_CLEAN 0x00ffffff -#define SVSB_INTSTS_COMPLETE 0x1 -#define SVSB_INTSTS_MONVOP 0x00ff0000 + +/* DESCHAR */ +#define SVSB_DESCHAR_FLD_MDES GENMASK(7, 0) +#define SVSB_DESCHAR_FLD_BDES GENMASK(15, 8) + +/* TEMPCHAR */ +#define SVSB_TEMPCHAR_FLD_DVT_FIXED GENMASK(7, 0) +#define SVSB_TEMPCHAR_FLD_MTDES GENMASK(15, 8) +#define SVSB_TEMPCHAR_FLD_VCO GENMASK(23, 16) + +/* DETCHAR */ +#define SVSB_DETCHAR_FLD_DCMDET GENMASK(7, 0) +#define SVSB_DETCHAR_FLD_DCBDET GENMASK(15, 8) + +/* SVSEN (PTPEN) */ +#define SVSB_PTPEN_INIT01 BIT(0) +#define SVSB_PTPEN_MON BIT(1) +#define SVSB_PTPEN_INIT02 (SVSB_PTPEN_INIT01 | BIT(2)) +#define SVSB_PTPEN_OFF 0x0 + +/* FREQPCTS */ +#define SVSB_FREQPCTS_FLD_PCT0_4 GENMASK(7, 0) +#define SVSB_FREQPCTS_FLD_PCT1_5 GENMASK(15, 8) +#define SVSB_FREQPCTS_FLD_PCT2_6 GENMASK(23, 16) +#define SVSB_FREQPCTS_FLD_PCT3_7 GENMASK(31, 24) + +/* INTSTS */ +#define SVSB_INTSTS_VAL_CLEAN 0x00ffffff +#define SVSB_INTSTS_F0_COMPLETE BIT(0) +#define SVSB_INTSTS_FLD_MONVOP GENMASK(23, 16) #define SVSB_RUNCONFIG_DEFAULT 0x80000000 +/* LIMITVALS */ +#define SVSB_LIMITVALS_FLD_DTLO GENMASK(7, 0) +#define SVSB_LIMITVALS_FLD_DTHI GENMASK(15, 8) +#define SVSB_LIMITVALS_FLD_VMIN GENMASK(23, 16) +#define SVSB_LIMITVALS_FLD_VMAX GENMASK(31, 24) +#define SVSB_VAL_DTHI 0x1 +#define SVSB_VAL_DTLO 0xfe + +/* INTEN */ +#define SVSB_INTEN_F0EN BIT(0) +#define SVSB_INTEN_DACK0UPEN BIT(8) +#define SVSB_INTEN_DC0EN BIT(9) +#define SVSB_INTEN_DC1EN BIT(10) +#define SVSB_INTEN_DACK0LOEN BIT(11) +#define SVSB_INTEN_INITPROD_OVF_EN BIT(12) +#define SVSB_INTEN_INITSUM_OVF_EN BIT(14) +#define SVSB_INTEN_MONVOPEN GENMASK(23, 16) +#define SVSB_INTEN_INIT0x (SVSB_INTEN_F0EN | SVSB_INTEN_DACK0UPEN | \ + SVSB_INTEN_DC0EN | SVSB_INTEN_DC1EN | \ + SVSB_INTEN_DACK0LOEN | \ + SVSB_INTEN_INITPROD_OVF_EN | \ + SVSB_INTEN_INITSUM_OVF_EN) + +/* TSCALCS */ +#define SVSB_TSCALCS_FLD_MTS GENMASK(11, 0) +#define SVSB_TSCALCS_FLD_BTS GENMASK(23, 12) + +/* INIT2VALS */ +#define SVSB_INIT2VALS_FLD_DCVOFFSETIN GENMASK(15, 0) +#define SVSB_INIT2VALS_FLD_AGEVOFFSETIN GENMASK(31, 16) + +/* VOPS */ +#define SVSB_VOPS_FLD_VOP0_4 GENMASK(7, 0) +#define SVSB_VOPS_FLD_VOP1_5 GENMASK(15, 8) +#define SVSB_VOPS_FLD_VOP2_6 GENMASK(23, 16) +#define SVSB_VOPS_FLD_VOP3_7 GENMASK(31, 24) + /* svs bank related setting */ #define BITS8 8 #define MAX_OPP_ENTRIES 16 @@ -262,7 +320,6 @@ static const u32 svs_regs_v2[] = { * @rst: svs platform reset control * @efuse_parsing: svs platform efuse parsing function pointer * @probe: svs platform probe function pointer - * @irqflags: svs platform irq settings flags * @efuse_max: total number of svs efuse * @tefuse_max: total number of thermal efuse * @regs: svs platform registers map @@ -280,7 +337,6 @@ struct svs_platform { struct reset_control *rst; bool (*efuse_parsing)(struct svs_platform *svsp); int (*probe)(struct svs_platform *svsp); - unsigned long irqflags; size_t efuse_max; size_t tefuse_max; const u32 *regs; @@ -294,7 +350,6 @@ struct svs_platform_data { struct svs_bank *banks; bool (*efuse_parsing)(struct svs_platform *svsp); int (*probe)(struct svs_platform *svsp); - unsigned long irqflags; const u32 *regs; u32 bank_max; }; @@ -668,8 +723,8 @@ static ssize_t svs_enable_debug_write(struct file *filp, svsp->pbank = svsb; svsb->mode_support = SVSB_MODE_ALL_DISABLE; svs_switch_bank(svsp); - svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN); - svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS); + svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN); + svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS); spin_unlock_irqrestore(&svs_lock, flags); svsb->phase = SVSB_PHASE_ERROR; @@ -830,7 +885,7 @@ static void svs_get_bank_volts_v3(struct svs_platform *svsp) } else if (svsb->type == SVSB_LOW) { /* volt[turn_pt] + volt[j] ~ volt[opp_count - 1] */ j = svsb->opp_count - 7; - svsb->volt[turn_pt] = vop30 & GENMASK(7, 0); + svsb->volt[turn_pt] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, vop30); shift_byte++; for (i = j; i < svsb->opp_count; i++) { b_sft = BITS8 * (shift_byte % REG_BYTES); @@ -852,7 +907,7 @@ static void svs_get_bank_volts_v3(struct svs_platform *svsp) if (svsb->type == SVSB_HIGH) { /* volt[0] + volt[j] ~ volt[turn_pt - 1] */ j = turn_pt - 7; - svsb->volt[0] = vop30 & GENMASK(7, 0); + svsb->volt[0] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, vop30); shift_byte++; for (i = j; i < turn_pt; i++) { b_sft = BITS8 * (shift_byte % REG_BYTES); @@ -983,16 +1038,16 @@ static void svs_get_bank_volts_v2(struct svs_platform *svsp) u32 temp, i; temp = svs_readl_relaxed(svsp, VOP74); - svsb->volt[14] = (temp >> 24) & GENMASK(7, 0); - svsb->volt[12] = (temp >> 16) & GENMASK(7, 0); - svsb->volt[10] = (temp >> 8) & GENMASK(7, 0); - svsb->volt[8] = (temp & GENMASK(7, 0)); + svsb->volt[14] = FIELD_GET(SVSB_VOPS_FLD_VOP3_7, temp); + svsb->volt[12] = FIELD_GET(SVSB_VOPS_FLD_VOP2_6, temp); + svsb->volt[10] = FIELD_GET(SVSB_VOPS_FLD_VOP1_5, temp); + svsb->volt[8] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, temp); temp = svs_readl_relaxed(svsp, VOP30); - svsb->volt[6] = (temp >> 24) & GENMASK(7, 0); - svsb->volt[4] = (temp >> 16) & GENMASK(7, 0); - svsb->volt[2] = (temp >> 8) & GENMASK(7, 0); - svsb->volt[0] = (temp & GENMASK(7, 0)); + svsb->volt[6] = FIELD_GET(SVSB_VOPS_FLD_VOP3_7, temp); + svsb->volt[4] = FIELD_GET(SVSB_VOPS_FLD_VOP2_6, temp); + svsb->volt[2] = FIELD_GET(SVSB_VOPS_FLD_VOP1_5, temp); + svsb->volt[0] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, temp); for (i = 0; i <= 12; i += 2) svsb->volt[i + 1] = interpolate(svsb->freq_pct[i], @@ -1014,20 +1069,20 @@ static void svs_get_bank_volts_v2(struct svs_platform *svsp) static void svs_set_bank_freq_pct_v2(struct svs_platform *svsp) { struct svs_bank *svsb = svsp->pbank; + u32 freqpct74_val, freqpct30_val; + + freqpct74_val = FIELD_PREP(SVSB_FREQPCTS_FLD_PCT0_4, svsb->freq_pct[8]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT1_5, svsb->freq_pct[10]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT2_6, svsb->freq_pct[12]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT3_7, svsb->freq_pct[14]); - svs_writel_relaxed(svsp, - (svsb->freq_pct[14] << 24) | - (svsb->freq_pct[12] << 16) | - (svsb->freq_pct[10] << 8) | - svsb->freq_pct[8], - FREQPCT74); - - svs_writel_relaxed(svsp, - (svsb->freq_pct[6] << 24) | - (svsb->freq_pct[4] << 16) | - (svsb->freq_pct[2] << 8) | - svsb->freq_pct[0], - FREQPCT30); + freqpct30_val = FIELD_PREP(SVSB_FREQPCTS_FLD_PCT0_4, svsb->freq_pct[0]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT1_5, svsb->freq_pct[2]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT2_6, svsb->freq_pct[4]) | + FIELD_PREP(SVSB_FREQPCTS_FLD_PCT3_7, svsb->freq_pct[6]); + + svs_writel_relaxed(svsp, freqpct74_val, FREQPCT74); + svs_writel_relaxed(svsp, freqpct30_val, FREQPCT30); } static void svs_set_bank_phase(struct svs_platform *svsp, @@ -1038,13 +1093,17 @@ static void svs_set_bank_phase(struct svs_platform *svsp, svs_switch_bank(svsp); - des_char = (svsb->bdes << 8) | svsb->mdes; + des_char = FIELD_PREP(SVSB_DESCHAR_FLD_BDES, svsb->bdes) | + FIELD_PREP(SVSB_DESCHAR_FLD_MDES, svsb->mdes); svs_writel_relaxed(svsp, des_char, DESCHAR); - temp_char = (svsb->vco << 16) | (svsb->mtdes << 8) | svsb->dvt_fixed; + temp_char = FIELD_PREP(SVSB_TEMPCHAR_FLD_VCO, svsb->vco) | + FIELD_PREP(SVSB_TEMPCHAR_FLD_MTDES, svsb->mtdes) | + FIELD_PREP(SVSB_TEMPCHAR_FLD_DVT_FIXED, svsb->dvt_fixed); svs_writel_relaxed(svsp, temp_char, TEMPCHAR); - det_char = (svsb->dcbdet << 8) | svsb->dcmdet; + det_char = FIELD_PREP(SVSB_DETCHAR_FLD_DCBDET, svsb->dcbdet) | + FIELD_PREP(SVSB_DETCHAR_FLD_DCMDET, svsb->dcmdet); svs_writel_relaxed(svsp, det_char, DETCHAR); svs_writel_relaxed(svsp, svsb->dc_config, DCCONFIG); @@ -1053,33 +1112,37 @@ static void svs_set_bank_phase(struct svs_platform *svsp, svsb->set_freq_pct(svsp); - limit_vals = (svsb->vmax << 24) | (svsb->vmin << 16) | - (SVSB_DTHI << 8) | SVSB_DTLO; + limit_vals = FIELD_PREP(SVSB_LIMITVALS_FLD_DTLO, SVSB_VAL_DTLO) | + FIELD_PREP(SVSB_LIMITVALS_FLD_DTHI, SVSB_VAL_DTHI) | + FIELD_PREP(SVSB_LIMITVALS_FLD_VMIN, svsb->vmin) | + FIELD_PREP(SVSB_LIMITVALS_FLD_VMAX, svsb->vmax); svs_writel_relaxed(svsp, limit_vals, LIMITVALS); svs_writel_relaxed(svsp, SVSB_DET_WINDOW, DETWINDOW); svs_writel_relaxed(svsp, SVSB_DET_MAX, CONFIG); svs_writel_relaxed(svsp, svsb->chk_shift, CHKSHIFT); svs_writel_relaxed(svsp, svsb->ctl0, CTL0); - svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS); + svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS); switch (target_phase) { case SVSB_PHASE_INIT01: svs_writel_relaxed(svsp, svsb->vboot, VBOOT); svs_writel_relaxed(svsp, SVSB_INTEN_INIT0x, INTEN); - svs_writel_relaxed(svsp, SVSB_EN_INIT01, SVSEN); + svs_writel_relaxed(svsp, SVSB_PTPEN_INIT01, SVSEN); break; case SVSB_PHASE_INIT02: + init2vals = FIELD_PREP(SVSB_INIT2VALS_FLD_AGEVOFFSETIN, svsb->age_voffset_in) | + FIELD_PREP(SVSB_INIT2VALS_FLD_DCVOFFSETIN, svsb->dc_voffset_in); svs_writel_relaxed(svsp, SVSB_INTEN_INIT0x, INTEN); - init2vals = (svsb->age_voffset_in << 16) | svsb->dc_voffset_in; svs_writel_relaxed(svsp, init2vals, INIT2VALS); - svs_writel_relaxed(svsp, SVSB_EN_INIT02, SVSEN); + svs_writel_relaxed(svsp, SVSB_PTPEN_INIT02, SVSEN); break; case SVSB_PHASE_MON: - ts_calcs = (svsb->bts << 12) | svsb->mts; + ts_calcs = FIELD_PREP(SVSB_TSCALCS_FLD_BTS, svsb->bts) | + FIELD_PREP(SVSB_TSCALCS_FLD_MTS, svsb->mts); svs_writel_relaxed(svsp, ts_calcs, TSCALCS); svs_writel_relaxed(svsp, SVSB_INTEN_MONVOPEN, INTEN); - svs_writel_relaxed(svsp, SVSB_EN_MON, SVSEN); + svs_writel_relaxed(svsp, SVSB_PTPEN_MON, SVSEN); break; default: dev_err(svsb->dev, "requested unknown target phase: %u\n", @@ -1115,8 +1178,8 @@ static inline void svs_error_isr_handler(struct svs_platform *svsp) svs_save_bank_register_data(svsp, SVSB_PHASE_ERROR); svsb->phase = SVSB_PHASE_ERROR; - svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN); - svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS); + svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN); + svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS); } static inline void svs_init01_isr_handler(struct svs_platform *svsp) @@ -1141,8 +1204,8 @@ static inline void svs_init01_isr_handler(struct svs_platform *svsp) svsb->age_voffset_in = svs_readl_relaxed(svsp, AGEVALUES) & GENMASK(15, 0); - svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN); - svs_writel_relaxed(svsp, SVSB_INTSTS_COMPLETE, INTSTS); + svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN); + svs_writel_relaxed(svsp, SVSB_INTSTS_F0_COMPLETE, INTSTS); svsb->core_sel &= ~SVSB_DET_CLK_EN; } @@ -1160,8 +1223,8 @@ static inline void svs_init02_isr_handler(struct svs_platform *svsp) svsb->phase = SVSB_PHASE_INIT02; svsb->get_volts(svsp); - svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN); - svs_writel_relaxed(svsp, SVSB_INTSTS_COMPLETE, INTSTS); + svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN); + svs_writel_relaxed(svsp, SVSB_INTSTS_F0_COMPLETE, INTSTS); } static inline void svs_mon_mode_isr_handler(struct svs_platform *svsp) @@ -1174,7 +1237,7 @@ static inline void svs_mon_mode_isr_handler(struct svs_platform *svsp) svsb->get_volts(svsp); svsb->temp = svs_readl_relaxed(svsp, TEMP) & GENMASK(7, 0); - svs_writel_relaxed(svsp, SVSB_INTSTS_MONVOP, INTSTS); + svs_writel_relaxed(svsp, SVSB_INTSTS_FLD_MONVOP, INTSTS); } static irqreturn_t svs_isr(int irq, void *data) @@ -1201,13 +1264,13 @@ static irqreturn_t svs_isr(int irq, void *data) int_sts = svs_readl_relaxed(svsp, INTSTS); svs_en = svs_readl_relaxed(svsp, SVSEN); - if (int_sts == SVSB_INTSTS_COMPLETE && - svs_en == SVSB_EN_INIT01) + if (int_sts == SVSB_INTSTS_F0_COMPLETE && + svs_en == SVSB_PTPEN_INIT01) svs_init01_isr_handler(svsp); - else if (int_sts == SVSB_INTSTS_COMPLETE && - svs_en == SVSB_EN_INIT02) + else if (int_sts == SVSB_INTSTS_F0_COMPLETE && + svs_en == SVSB_PTPEN_INIT02) svs_init02_isr_handler(svsp); - else if (int_sts & SVSB_INTSTS_MONVOP) + else if (int_sts & SVSB_INTSTS_FLD_MONVOP) svs_mon_mode_isr_handler(svsp); else svs_error_isr_handler(svsp); @@ -1493,8 +1556,8 @@ static int svs_suspend(struct device *dev) spin_lock_irqsave(&svs_lock, flags); svsp->pbank = svsb; svs_switch_bank(svsp); - svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN); - svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS); + svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN); + svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS); spin_unlock_irqrestore(&svs_lock, flags); svsb->phase = SVSB_PHASE_ERROR; @@ -1589,7 +1652,7 @@ static int svs_bank_resource_setup(struct svs_platform *svsp) dev_set_drvdata(svsb->dev, svsp); - ret = dev_pm_opp_of_add_table(svsb->opp_dev); + ret = devm_pm_opp_of_add_table(svsb->opp_dev); if (ret) { dev_err(svsb->dev, "add opp table fail: %d\n", ret); return ret; @@ -1644,11 +1707,36 @@ static int svs_bank_resource_setup(struct svs_platform *svsp) return 0; } +static int svs_thermal_efuse_get_data(struct svs_platform *svsp) +{ + struct nvmem_cell *cell; + + /* Thermal efuse parsing */ + cell = nvmem_cell_get(svsp->dev, "t-calibration-data"); + if (IS_ERR_OR_NULL(cell)) { + dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n", PTR_ERR(cell)); + return PTR_ERR(cell); + } + + svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max); + if (IS_ERR(svsp->tefuse)) { + dev_err(svsp->dev, "cannot read thermal efuse: %ld\n", + PTR_ERR(svsp->tefuse)); + nvmem_cell_put(cell); + return PTR_ERR(svsp->tefuse); + } + + svsp->tefuse_max /= sizeof(u32); + nvmem_cell_put(cell); + + return 0; +} + static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp) { struct svs_bank *svsb; - struct nvmem_cell *cell; u32 idx, i, vmin, golden_temp; + int ret; for (i = 0; i < svsp->efuse_max; i++) if (svsp->efuse[i]) @@ -1686,24 +1774,9 @@ static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp) svsb->vmax += svsb->dvt_fixed; } - /* Thermal efuse parsing */ - cell = nvmem_cell_get(svsp->dev, "t-calibration-data"); - if (IS_ERR_OR_NULL(cell)) { - dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n", - PTR_ERR(cell)); - return false; - } - - svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max); - if (IS_ERR(svsp->tefuse)) { - dev_err(svsp->dev, "cannot read thermal efuse: %ld\n", - PTR_ERR(svsp->tefuse)); - nvmem_cell_put(cell); + ret = svs_thermal_efuse_get_data(svsp); + if (ret) return false; - } - - svsp->tefuse_max /= sizeof(u32); - nvmem_cell_put(cell); for (i = 0; i < svsp->tefuse_max; i++) if (svsp->tefuse[i] != 0) @@ -1726,11 +1799,11 @@ static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp) static bool svs_mt8183_efuse_parsing(struct svs_platform *svsp) { struct svs_bank *svsb; - struct nvmem_cell *cell; int format[6], x_roomt[6], o_vtsmcu[5], o_vtsabb, tb_roomt = 0; int adc_ge_t, adc_oe_t, ge, oe, gain, degc_cali, adc_cali_en_t; int o_slope, o_slope_sign, ts_id; u32 idx, i, ft_pgm, mts, temp0, temp1, temp2; + int ret; for (i = 0; i < svsp->efuse_max; i++) if (svsp->efuse[i]) @@ -1806,24 +1879,9 @@ static bool svs_mt8183_efuse_parsing(struct svs_platform *svsp) } } - /* Get thermal efuse by nvmem */ - cell = nvmem_cell_get(svsp->dev, "t-calibration-data"); - if (IS_ERR(cell)) { - dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n", - PTR_ERR(cell)); - goto remove_mt8183_svsb_mon_mode; - } - - svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max); - if (IS_ERR(svsp->tefuse)) { - dev_err(svsp->dev, "cannot read thermal efuse: %ld\n", - PTR_ERR(svsp->tefuse)); - nvmem_cell_put(cell); - goto remove_mt8183_svsb_mon_mode; - } - - svsp->tefuse_max /= sizeof(u32); - nvmem_cell_put(cell); + ret = svs_thermal_efuse_get_data(svsp); + if (ret) + return false; /* Thermal efuse parsing */ adc_ge_t = (svsp->tefuse[1] >> 22) & GENMASK(9, 0); @@ -2244,7 +2302,6 @@ static const struct svs_platform_data svs_mt8192_platform_data = { .banks = svs_mt8192_banks, .efuse_parsing = svs_mt8192_efuse_parsing, .probe = svs_mt8192_platform_probe, - .irqflags = IRQF_TRIGGER_HIGH, .regs = svs_regs_v2, .bank_max = ARRAY_SIZE(svs_mt8192_banks), }; @@ -2254,7 +2311,6 @@ static const struct svs_platform_data svs_mt8183_platform_data = { .banks = svs_mt8183_banks, .efuse_parsing = svs_mt8183_efuse_parsing, .probe = svs_mt8183_platform_probe, - .irqflags = IRQF_TRIGGER_LOW, .regs = svs_regs_v2, .bank_max = ARRAY_SIZE(svs_mt8183_banks), }; @@ -2292,7 +2348,6 @@ static struct svs_platform *svs_platform_probe(struct platform_device *pdev) svsp->banks = svsp_data->banks; svsp->efuse_parsing = svsp_data->efuse_parsing; svsp->probe = svsp_data->probe; - svsp->irqflags = svsp_data->irqflags; svsp->regs = svsp_data->regs; svsp->bank_max = svsp_data->bank_max; @@ -2306,8 +2361,7 @@ static struct svs_platform *svs_platform_probe(struct platform_device *pdev) static int svs_probe(struct platform_device *pdev) { struct svs_platform *svsp; - unsigned int svsp_irq; - int ret; + int svsp_irq, ret; svsp = svs_platform_probe(pdev); if (IS_ERR(svsp)) @@ -2325,10 +2379,14 @@ static int svs_probe(struct platform_device *pdev) goto svs_probe_free_resource; } - svsp_irq = irq_of_parse_and_map(svsp->dev->of_node, 0); + svsp_irq = platform_get_irq(pdev, 0); + if (svsp_irq < 0) { + ret = svsp_irq; + goto svs_probe_free_resource; + } + ret = devm_request_threaded_irq(svsp->dev, svsp_irq, NULL, svs_isr, - svsp->irqflags | IRQF_ONESHOT, - svsp->name, svsp); + IRQF_ONESHOT, svsp->name, svsp); if (ret) { dev_err(svsp->dev, "register irq(%d) failed: %d\n", svsp_irq, ret); @@ -2392,7 +2450,7 @@ static struct platform_driver svs_driver = { .driver = { .name = "mtk-svs", .pm = &svs_pm_ops, - .of_match_table = of_match_ptr(svs_of_match), + .of_match_table = svs_of_match, }, }; diff --git a/drivers/soc/pxa/ssp.c b/drivers/soc/pxa/ssp.c index 563440315acd31437c77ff19aca8af2e575e16d4..93449fb3519e87512fc2fb7d1e2b6d3583d373dd 100644 --- a/drivers/soc/pxa/ssp.c +++ b/drivers/soc/pxa/ssp.c @@ -180,11 +180,7 @@ static int pxa_ssp_probe(struct platform_device *pdev) static int pxa_ssp_remove(struct platform_device *pdev) { - struct ssp_device *ssp; - - ssp = platform_get_drvdata(pdev); - if (ssp == NULL) - return -ENODEV; + struct ssp_device *ssp = platform_get_drvdata(pdev); mutex_lock(&ssp_lock); list_del(&ssp->node); diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig index e0d7a54595627797545f53cb7b1927f8bad4cd27..024e420f1bb77b2ddf7b4c8c4790d083fa4f56e6 100644 --- a/drivers/soc/qcom/Kconfig +++ b/drivers/soc/qcom/Kconfig @@ -129,7 +129,7 @@ config QCOM_RPMHPD config QCOM_RPMPD tristate "Qualcomm RPM Power domain driver" - depends on PM + depends on PM && OF depends on QCOM_SMD_RPM select PM_GENERIC_DOMAINS select PM_GENERIC_DOMAINS_OF diff --git a/drivers/soc/qcom/icc-bwmon.c b/drivers/soc/qcom/icc-bwmon.c index 7f8aca533cd3ee58d5ef8e98473ea6c7923bce45..d07be3700db607a93a46708b13ac5466ad98022a 100644 --- a/drivers/soc/qcom/icc-bwmon.c +++ b/drivers/soc/qcom/icc-bwmon.c @@ -5,6 +5,8 @@ * Author: Krzysztof Kozlowski , based on * previous work of Thara Gopinath and msm-4.9 downstream sources. */ + +#include #include #include #include @@ -13,6 +15,7 @@ #include #include #include +#include #include /* @@ -31,33 +34,44 @@ /* Internal sampling clock frequency */ #define HW_TIMER_HZ 19200000 -#define BWMON_GLOBAL_IRQ_STATUS 0x0 -#define BWMON_GLOBAL_IRQ_CLEAR 0x8 -#define BWMON_GLOBAL_IRQ_ENABLE 0xc -#define BWMON_GLOBAL_IRQ_ENABLE_ENABLE BIT(0) - -#define BWMON_IRQ_STATUS 0x100 -#define BWMON_IRQ_STATUS_ZONE_SHIFT 4 -#define BWMON_IRQ_CLEAR 0x108 -#define BWMON_IRQ_ENABLE 0x10c -#define BWMON_IRQ_ENABLE_ZONE1_SHIFT 5 -#define BWMON_IRQ_ENABLE_ZONE2_SHIFT 6 -#define BWMON_IRQ_ENABLE_ZONE3_SHIFT 7 -#define BWMON_IRQ_ENABLE_MASK (BIT(BWMON_IRQ_ENABLE_ZONE1_SHIFT) | \ - BIT(BWMON_IRQ_ENABLE_ZONE3_SHIFT)) - -#define BWMON_ENABLE 0x2a0 +#define BWMON_V4_GLOBAL_IRQ_CLEAR 0x008 +#define BWMON_V4_GLOBAL_IRQ_ENABLE 0x00c +/* + * All values here and further are matching regmap fields, so without absolute + * register offsets. + */ +#define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE BIT(0) + +#define BWMON_V4_IRQ_STATUS 0x100 +#define BWMON_V4_IRQ_CLEAR 0x108 + +#define BWMON_V4_IRQ_ENABLE 0x10c +#define BWMON_IRQ_ENABLE_MASK (BIT(1) | BIT(3)) +#define BWMON_V5_IRQ_STATUS 0x000 +#define BWMON_V5_IRQ_CLEAR 0x008 +#define BWMON_V5_IRQ_ENABLE 0x00c + +#define BWMON_V4_ENABLE 0x2a0 +#define BWMON_V5_ENABLE 0x010 #define BWMON_ENABLE_ENABLE BIT(0) -#define BWMON_CLEAR 0x2a4 +#define BWMON_V4_CLEAR 0x2a4 +#define BWMON_V5_CLEAR 0x014 #define BWMON_CLEAR_CLEAR BIT(0) +#define BWMON_CLEAR_CLEAR_ALL BIT(1) -#define BWMON_SAMPLE_WINDOW 0x2a8 -#define BWMON_THRESHOLD_HIGH 0x2ac -#define BWMON_THRESHOLD_MED 0x2b0 -#define BWMON_THRESHOLD_LOW 0x2b4 +#define BWMON_V4_SAMPLE_WINDOW 0x2a8 +#define BWMON_V5_SAMPLE_WINDOW 0x020 -#define BWMON_ZONE_ACTIONS 0x2b8 +#define BWMON_V4_THRESHOLD_HIGH 0x2ac +#define BWMON_V4_THRESHOLD_MED 0x2b0 +#define BWMON_V4_THRESHOLD_LOW 0x2b4 +#define BWMON_V5_THRESHOLD_HIGH 0x024 +#define BWMON_V5_THRESHOLD_MED 0x028 +#define BWMON_V5_THRESHOLD_LOW 0x02c + +#define BWMON_V4_ZONE_ACTIONS 0x2b8 +#define BWMON_V5_ZONE_ACTIONS 0x030 /* * Actions to perform on some zone 'z' when current zone hits the threshold: * Increment counter of zone 'z' @@ -83,55 +97,244 @@ BWMON_ZONE_ACTIONS_CLEAR(2) | \ BWMON_ZONE_ACTIONS_CLEAR(1) | \ BWMON_ZONE_ACTIONS_CLEAR(0)) -/* Value for BWMON_ZONE_ACTIONS */ -#define BWMON_ZONE_ACTIONS_DEFAULT (BWMON_ZONE_ACTIONS_ZONE0 | \ - BWMON_ZONE_ACTIONS_ZONE1 << 8 | \ - BWMON_ZONE_ACTIONS_ZONE2 << 16 | \ - BWMON_ZONE_ACTIONS_ZONE3 << 24) /* - * There is no clear documentation/explanation of BWMON_THRESHOLD_COUNT + * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT * register. Based on observations, this is number of times one threshold has to * be reached, to trigger interrupt in given zone. * * 0xff are maximum values meant to ignore the zones 0 and 2. */ -#define BWMON_THRESHOLD_COUNT 0x2bc -#define BWMON_THRESHOLD_COUNT_ZONE1_SHIFT 8 -#define BWMON_THRESHOLD_COUNT_ZONE2_SHIFT 16 -#define BWMON_THRESHOLD_COUNT_ZONE3_SHIFT 24 +#define BWMON_V4_THRESHOLD_COUNT 0x2bc +#define BWMON_V5_THRESHOLD_COUNT 0x034 #define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT 0xff #define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT 0xff -/* BWMONv4 count registers use count unit of 64 kB */ -#define BWMON_COUNT_UNIT_KB 64 -#define BWMON_ZONE_COUNT 0x2d8 -#define BWMON_ZONE_MAX(zone) (0x2e0 + 4 * (zone)) +#define BWMON_V4_ZONE_MAX(zone) (0x2e0 + 4 * (zone)) +#define BWMON_V5_ZONE_MAX(zone) (0x044 + 4 * (zone)) + +/* Quirks for specific BWMON types */ +#define BWMON_HAS_GLOBAL_IRQ BIT(0) +#define BWMON_NEEDS_FORCE_CLEAR BIT(1) + +enum bwmon_fields { + F_GLOBAL_IRQ_CLEAR, + F_GLOBAL_IRQ_ENABLE, + F_IRQ_STATUS, + F_IRQ_CLEAR, + F_IRQ_ENABLE, + F_ENABLE, + F_CLEAR, + F_SAMPLE_WINDOW, + F_THRESHOLD_HIGH, + F_THRESHOLD_MED, + F_THRESHOLD_LOW, + F_ZONE_ACTIONS_ZONE0, + F_ZONE_ACTIONS_ZONE1, + F_ZONE_ACTIONS_ZONE2, + F_ZONE_ACTIONS_ZONE3, + F_THRESHOLD_COUNT_ZONE0, + F_THRESHOLD_COUNT_ZONE1, + F_THRESHOLD_COUNT_ZONE2, + F_THRESHOLD_COUNT_ZONE3, + F_ZONE0_MAX, + F_ZONE1_MAX, + F_ZONE2_MAX, + F_ZONE3_MAX, + + F_NUM_FIELDS +}; struct icc_bwmon_data { unsigned int sample_ms; + unsigned int count_unit_kb; /* kbytes */ unsigned int default_highbw_kbps; unsigned int default_medbw_kbps; unsigned int default_lowbw_kbps; u8 zone1_thres_count; u8 zone3_thres_count; + unsigned int quirks; + + const struct regmap_config *regmap_cfg; + const struct reg_field *regmap_fields; }; struct icc_bwmon { struct device *dev; - void __iomem *base; + const struct icc_bwmon_data *data; int irq; - unsigned int default_lowbw_kbps; - unsigned int sample_ms; + struct regmap *regmap; + struct regmap_field *regs[F_NUM_FIELDS]; + unsigned int max_bw_kbps; unsigned int min_bw_kbps; unsigned int target_kbps; unsigned int current_kbps; }; -static void bwmon_clear_counters(struct icc_bwmon *bwmon) +/* BWMON v4 */ +static const struct reg_field msm8998_bwmon_reg_fields[] = { + [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0), + [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0), + [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7), + [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7), + [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7), + /* F_ENABLE covers entire register to disable other features */ + [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31), + [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1), + [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23), + [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11), + [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11), + [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11), + [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7), + [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15), + [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23), + [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31), + [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7), + [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15), + [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23), + [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31), + [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11), + [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11), + [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11), + [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11), +}; + +static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = { + regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR), + regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR), + regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR), +}; + +static const struct regmap_access_table msm8998_bwmon_reg_read_table = { + .no_ranges = msm8998_bwmon_reg_noread_ranges, + .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges), +}; + +static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = { + regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS), + regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)), +}; + +static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = { + .yes_ranges = msm8998_bwmon_reg_volatile_ranges, + .n_yes_ranges = ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges), +}; + +/* + * Fill the cache for non-readable registers only as rest does not really + * matter and can be read from the device. + */ +static const struct reg_default msm8998_bwmon_reg_defaults[] = { + { BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 }, + { BWMON_V4_IRQ_CLEAR, 0x0 }, + { BWMON_V4_CLEAR, 0x0 }, +}; + +static const struct regmap_config msm8998_bwmon_regmap_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + /* + * No concurrent access expected - driver has one interrupt handler, + * regmap is not shared, no driver or user-space API. + */ + .disable_locking = true, + .rd_table = &msm8998_bwmon_reg_read_table, + .volatile_table = &msm8998_bwmon_reg_volatile_table, + .reg_defaults = msm8998_bwmon_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_reg_defaults), + /* + * Cache is necessary for using regmap fields with non-readable + * registers. + */ + .cache_type = REGCACHE_RBTREE, +}; + +/* BWMON v5 */ +static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = { + [F_GLOBAL_IRQ_CLEAR] = {}, + [F_GLOBAL_IRQ_ENABLE] = {}, + [F_IRQ_STATUS] = REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3), + [F_IRQ_CLEAR] = REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3), + [F_IRQ_ENABLE] = REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3), + /* F_ENABLE covers entire register to disable other features */ + [F_ENABLE] = REG_FIELD(BWMON_V5_ENABLE, 0, 31), + [F_CLEAR] = REG_FIELD(BWMON_V5_CLEAR, 0, 1), + [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19), + [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11), + [F_THRESHOLD_MED] = REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11), + [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11), + [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7), + [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15), + [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23), + [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31), + [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7), + [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15), + [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23), + [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31), + [F_ZONE0_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11), + [F_ZONE1_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11), + [F_ZONE2_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11), + [F_ZONE3_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11), +}; + +static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = { + regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR), + regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR), +}; + +static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = { + .no_ranges = sdm845_llcc_bwmon_reg_noread_ranges, + .n_no_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges), +}; + +static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = { + regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS), + regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)), +}; + +static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = { + .yes_ranges = sdm845_llcc_bwmon_reg_volatile_ranges, + .n_yes_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges), +}; + +/* + * Fill the cache for non-readable registers only as rest does not really + * matter and can be read from the device. + */ +static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = { + { BWMON_V5_IRQ_CLEAR, 0x0 }, + { BWMON_V5_CLEAR, 0x0 }, +}; + +static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + /* + * No concurrent access expected - driver has one interrupt handler, + * regmap is not shared, no driver or user-space API. + */ + .disable_locking = true, + .rd_table = &sdm845_llcc_bwmon_reg_read_table, + .volatile_table = &sdm845_llcc_bwmon_reg_volatile_table, + .reg_defaults = sdm845_llcc_bwmon_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults), + /* + * Cache is necessary for using regmap fields with non-readable + * registers. + */ + .cache_type = REGCACHE_RBTREE, +}; + +static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all) { + unsigned int val = BWMON_CLEAR_CLEAR; + + if (clear_all) + val |= BWMON_CLEAR_CLEAR_ALL; /* * Clear counters. The order and barriers are * important. Quoting downstream Qualcomm msm-4.9 tree: @@ -140,7 +343,9 @@ static void bwmon_clear_counters(struct icc_bwmon *bwmon) * region. So, we need to make sure the counter clear is completed * before we try to clear the IRQ or do any other counter operations. */ - writel(BWMON_CLEAR_CLEAR, bwmon->base + BWMON_CLEAR); + regmap_field_force_write(bwmon->regs[F_CLEAR], val); + if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) + regmap_field_force_write(bwmon->regs[F_CLEAR], 0); } static void bwmon_clear_irq(struct icc_bwmon *bwmon) @@ -161,76 +366,91 @@ static void bwmon_clear_irq(struct icc_bwmon *bwmon) * clearing here so that local writes don't happen before the * interrupt is cleared. */ - writel(BWMON_IRQ_ENABLE_MASK, bwmon->base + BWMON_IRQ_CLEAR); - writel(BIT(0), bwmon->base + BWMON_GLOBAL_IRQ_CLEAR); + regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK); + if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) + regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0); + if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) + regmap_field_force_write(bwmon->regs[F_GLOBAL_IRQ_CLEAR], + BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); } static void bwmon_disable(struct icc_bwmon *bwmon) { /* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */ - writel(0x0, bwmon->base + BWMON_GLOBAL_IRQ_ENABLE); - writel(0x0, bwmon->base + BWMON_IRQ_ENABLE); + if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) + regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE], 0x0); + regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0); /* * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious * IRQ. */ - writel(0x0, bwmon->base + BWMON_ENABLE); + regmap_field_write(bwmon->regs[F_ENABLE], 0x0); } static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable) { /* Enable interrupts */ - writel(BWMON_GLOBAL_IRQ_ENABLE_ENABLE, - bwmon->base + BWMON_GLOBAL_IRQ_ENABLE); - writel(irq_enable, bwmon->base + BWMON_IRQ_ENABLE); + if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) + regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE], + BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); + regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable); /* Enable bwmon */ - writel(BWMON_ENABLE_ENABLE, bwmon->base + BWMON_ENABLE); + regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE); } -static unsigned int bwmon_kbps_to_count(unsigned int kbps) +static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon, + unsigned int kbps) { - return kbps / BWMON_COUNT_UNIT_KB; + return kbps / bwmon->data->count_unit_kb; } -static void bwmon_set_threshold(struct icc_bwmon *bwmon, unsigned int reg, - unsigned int kbps) +static void bwmon_set_threshold(struct icc_bwmon *bwmon, + struct regmap_field *reg, unsigned int kbps) { unsigned int thres; - thres = mult_frac(bwmon_kbps_to_count(kbps), bwmon->sample_ms, - MSEC_PER_SEC); - writel_relaxed(thres, bwmon->base + reg); + thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps), + bwmon->data->sample_ms, MSEC_PER_SEC); + regmap_field_write(reg, thres); } -static void bwmon_start(struct icc_bwmon *bwmon, - const struct icc_bwmon_data *data) +static void bwmon_start(struct icc_bwmon *bwmon) { - unsigned int thres_count; + const struct icc_bwmon_data *data = bwmon->data; int window; - bwmon_clear_counters(bwmon); + bwmon_clear_counters(bwmon, true); - window = mult_frac(bwmon->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC); - /* Maximum sampling window: 0xfffff */ - writel_relaxed(window, bwmon->base + BWMON_SAMPLE_WINDOW); + window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC); + /* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */ + regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window); - bwmon_set_threshold(bwmon, BWMON_THRESHOLD_HIGH, + bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], data->default_highbw_kbps); - bwmon_set_threshold(bwmon, BWMON_THRESHOLD_MED, + bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], data->default_medbw_kbps); - bwmon_set_threshold(bwmon, BWMON_THRESHOLD_LOW, + bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW], data->default_lowbw_kbps); - thres_count = data->zone3_thres_count << BWMON_THRESHOLD_COUNT_ZONE3_SHIFT | - BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT << BWMON_THRESHOLD_COUNT_ZONE2_SHIFT | - data->zone1_thres_count << BWMON_THRESHOLD_COUNT_ZONE1_SHIFT | - BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT; - writel_relaxed(thres_count, bwmon->base + BWMON_THRESHOLD_COUNT); - writel_relaxed(BWMON_ZONE_ACTIONS_DEFAULT, - bwmon->base + BWMON_ZONE_ACTIONS); - /* Write barriers in bwmon_clear_irq() */ + regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0], + BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT); + regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1], + data->zone1_thres_count); + regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2], + BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT); + regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3], + data->zone3_thres_count); + + regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0], + BWMON_ZONE_ACTIONS_ZONE0); + regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1], + BWMON_ZONE_ACTIONS_ZONE1); + regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2], + BWMON_ZONE_ACTIONS_ZONE2); + regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3], + BWMON_ZONE_ACTIONS_ZONE3); bwmon_clear_irq(bwmon); bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK); @@ -242,7 +462,9 @@ static irqreturn_t bwmon_intr(int irq, void *dev_id) unsigned int status, max; int zone; - status = readl(bwmon->base + BWMON_IRQ_STATUS); + if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status)) + return IRQ_NONE; + status &= BWMON_IRQ_ENABLE_MASK; if (!status) { /* @@ -259,15 +481,18 @@ static irqreturn_t bwmon_intr(int irq, void *dev_id) bwmon_disable(bwmon); - zone = get_bitmask_order(status >> BWMON_IRQ_STATUS_ZONE_SHIFT) - 1; + zone = get_bitmask_order(status) - 1; /* * Zone max bytes count register returns count units within sampling * window. Downstream kernel for BWMONv4 (called BWMON type 2 in * downstream) always increments the max bytes count by one. */ - max = readl(bwmon->base + BWMON_ZONE_MAX(zone)) + 1; - max *= BWMON_COUNT_UNIT_KB; - bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->sample_ms); + if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max)) + return IRQ_NONE; + + max += 1; + max *= bwmon->data->count_unit_kb; + bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms); return IRQ_WAKE_THREAD; } @@ -297,16 +522,17 @@ static irqreturn_t bwmon_intr_thread(int irq, void *dev_id) up_kbps = bwmon->target_kbps + 1; if (bwmon->target_kbps >= bwmon->max_bw_kbps) - irq_enable = BIT(BWMON_IRQ_ENABLE_ZONE1_SHIFT); + irq_enable = BIT(1); else if (bwmon->target_kbps <= bwmon->min_bw_kbps) - irq_enable = BIT(BWMON_IRQ_ENABLE_ZONE3_SHIFT); + irq_enable = BIT(3); else irq_enable = BWMON_IRQ_ENABLE_MASK; - bwmon_set_threshold(bwmon, BWMON_THRESHOLD_HIGH, up_kbps); - bwmon_set_threshold(bwmon, BWMON_THRESHOLD_MED, down_kbps); - /* Write barriers in bwmon_clear_counters() */ - bwmon_clear_counters(bwmon); + bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], + up_kbps); + bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], + down_kbps); + bwmon_clear_counters(bwmon, false); bwmon_clear_irq(bwmon); bwmon_enable(bwmon, irq_enable); @@ -324,25 +550,47 @@ out: return IRQ_HANDLED; } +static int bwmon_init_regmap(struct platform_device *pdev, + struct icc_bwmon *bwmon) +{ + struct device *dev = &pdev->dev; + void __iomem *base; + struct regmap *map; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return dev_err_probe(dev, PTR_ERR(base), + "failed to map bwmon registers\n"); + + map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg); + if (IS_ERR(map)) + return dev_err_probe(dev, PTR_ERR(map), + "failed to initialize regmap\n"); + + BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS); + BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS); + + return devm_regmap_field_bulk_alloc(dev, map, bwmon->regs, + bwmon->data->regmap_fields, + F_NUM_FIELDS); +} + static int bwmon_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct dev_pm_opp *opp; struct icc_bwmon *bwmon; - const struct icc_bwmon_data *data; int ret; bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL); if (!bwmon) return -ENOMEM; - data = of_device_get_match_data(dev); + bwmon->data = of_device_get_match_data(dev); - bwmon->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(bwmon->base)) { - dev_err(dev, "failed to map bwmon registers\n"); - return PTR_ERR(bwmon->base); - } + ret = bwmon_init_regmap(pdev, bwmon); + if (ret) + return ret; bwmon->irq = platform_get_irq(pdev, 0); if (bwmon->irq < 0) @@ -362,8 +610,6 @@ static int bwmon_probe(struct platform_device *pdev) if (IS_ERR(opp)) return dev_err_probe(dev, ret, "failed to find min peak bandwidth\n"); - bwmon->sample_ms = data->sample_ms; - bwmon->default_lowbw_kbps = data->default_lowbw_kbps; bwmon->dev = dev; bwmon_disable(bwmon); @@ -374,7 +620,7 @@ static int bwmon_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "failed to request IRQ\n"); platform_set_drvdata(pdev, bwmon); - bwmon_start(bwmon, data); + bwmon_start(bwmon); return 0; } @@ -388,18 +634,55 @@ static int bwmon_remove(struct platform_device *pdev) return 0; } -/* BWMON v4 */ static const struct icc_bwmon_data msm8998_bwmon_data = { .sample_ms = 4, + .count_unit_kb = 64, .default_highbw_kbps = 4800 * 1024, /* 4.8 GBps */ .default_medbw_kbps = 512 * 1024, /* 512 MBps */ .default_lowbw_kbps = 0, .zone1_thres_count = 16, .zone3_thres_count = 1, + .quirks = BWMON_HAS_GLOBAL_IRQ, + .regmap_fields = msm8998_bwmon_reg_fields, + .regmap_cfg = &msm8998_bwmon_regmap_cfg, +}; + +static const struct icc_bwmon_data sdm845_llcc_bwmon_data = { + .sample_ms = 4, + .count_unit_kb = 1024, + .default_highbw_kbps = 800 * 1024, /* 800 MBps */ + .default_medbw_kbps = 256 * 1024, /* 256 MBps */ + .default_lowbw_kbps = 0, + .zone1_thres_count = 16, + .zone3_thres_count = 1, + .regmap_fields = sdm845_llcc_bwmon_reg_fields, + .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, +}; + +static const struct icc_bwmon_data sc7280_llcc_bwmon_data = { + .sample_ms = 4, + .count_unit_kb = 64, + .default_highbw_kbps = 800 * 1024, /* 800 MBps */ + .default_medbw_kbps = 256 * 1024, /* 256 MBps */ + .default_lowbw_kbps = 0, + .zone1_thres_count = 16, + .zone3_thres_count = 1, + .quirks = BWMON_NEEDS_FORCE_CLEAR, + .regmap_fields = sdm845_llcc_bwmon_reg_fields, + .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, }; static const struct of_device_id bwmon_of_match[] = { - { .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data }, + { + .compatible = "qcom,msm8998-bwmon", + .data = &msm8998_bwmon_data + }, { + .compatible = "qcom,sdm845-llcc-bwmon", + .data = &sdm845_llcc_bwmon_data + }, { + .compatible = "qcom,sc7280-llcc-bwmon", + .data = &sc7280_llcc_bwmon_data + }, {} }; MODULE_DEVICE_TABLE(of, bwmon_of_match); diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index 38d7296315a25ee20466bfeed7f79ed1abcead86..8b7e8118f3cec2424acaaed5d4e56244a4e2e4f1 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -104,6 +104,7 @@ struct qcom_llcc_config { int size; bool need_llcc_cfg; const u32 *reg_offset; + const struct llcc_edac_reg_offset *edac_reg_offset; }; enum llcc_reg_offset { @@ -296,12 +297,68 @@ static const struct llcc_slice_config sm8450_data[] = { {LLCC_AENPU, 8, 2048, 1, 1, 0xFFFF, 0x0, 0, 0, 0, 0, 0, 0, 0 }, }; -static const u32 llcc_v1_2_reg_offset[] = { +static const struct llcc_edac_reg_offset llcc_v1_edac_reg_offset = { + .trp_ecc_error_status0 = 0x20344, + .trp_ecc_error_status1 = 0x20348, + .trp_ecc_sb_err_syn0 = 0x2304c, + .trp_ecc_db_err_syn0 = 0x20370, + .trp_ecc_error_cntr_clear = 0x20440, + .trp_interrupt_0_status = 0x20480, + .trp_interrupt_0_clear = 0x20484, + .trp_interrupt_0_enable = 0x20488, + + /* LLCC Common registers */ + .cmn_status0 = 0x3000c, + .cmn_interrupt_0_enable = 0x3001c, + .cmn_interrupt_2_enable = 0x3003c, + + /* LLCC DRP registers */ + .drp_ecc_error_cfg = 0x40000, + .drp_ecc_error_cntr_clear = 0x40004, + .drp_interrupt_status = 0x41000, + .drp_interrupt_clear = 0x41008, + .drp_interrupt_enable = 0x4100c, + .drp_ecc_error_status0 = 0x42044, + .drp_ecc_error_status1 = 0x42048, + .drp_ecc_sb_err_syn0 = 0x4204c, + .drp_ecc_db_err_syn0 = 0x42070, +}; + +static const struct llcc_edac_reg_offset llcc_v2_1_edac_reg_offset = { + .trp_ecc_error_status0 = 0x20344, + .trp_ecc_error_status1 = 0x20348, + .trp_ecc_sb_err_syn0 = 0x2034c, + .trp_ecc_db_err_syn0 = 0x20370, + .trp_ecc_error_cntr_clear = 0x20440, + .trp_interrupt_0_status = 0x20480, + .trp_interrupt_0_clear = 0x20484, + .trp_interrupt_0_enable = 0x20488, + + /* LLCC Common registers */ + .cmn_status0 = 0x3400c, + .cmn_interrupt_0_enable = 0x3401c, + .cmn_interrupt_2_enable = 0x3403c, + + /* LLCC DRP registers */ + .drp_ecc_error_cfg = 0x50000, + .drp_ecc_error_cntr_clear = 0x50004, + .drp_interrupt_status = 0x50020, + .drp_interrupt_clear = 0x50028, + .drp_interrupt_enable = 0x5002c, + .drp_ecc_error_status0 = 0x520f4, + .drp_ecc_error_status1 = 0x520f8, + .drp_ecc_sb_err_syn0 = 0x520fc, + .drp_ecc_db_err_syn0 = 0x52120, +}; + +/* LLCC register offset starting from v1.0.0 */ +static const u32 llcc_v1_reg_offset[] = { [LLCC_COMMON_HW_INFO] = 0x00030000, [LLCC_COMMON_STATUS0] = 0x0003000c, }; -static const u32 llcc_v21_reg_offset[] = { +/* LLCC register offset starting from v2.0.1 */ +static const u32 llcc_v2_1_reg_offset[] = { [LLCC_COMMON_HW_INFO] = 0x00034000, [LLCC_COMMON_STATUS0] = 0x0003400c, }; @@ -310,70 +367,80 @@ static const struct qcom_llcc_config sc7180_cfg = { .sct_data = sc7180_data, .size = ARRAY_SIZE(sc7180_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sc7280_cfg = { .sct_data = sc7280_data, .size = ARRAY_SIZE(sc7280_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sc8180x_cfg = { .sct_data = sc8180x_data, .size = ARRAY_SIZE(sc8180x_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sc8280xp_cfg = { .sct_data = sc8280xp_data, .size = ARRAY_SIZE(sc8280xp_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sdm845_cfg = { .sct_data = sdm845_data, .size = ARRAY_SIZE(sdm845_data), .need_llcc_cfg = false, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sm6350_cfg = { .sct_data = sm6350_data, .size = ARRAY_SIZE(sm6350_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sm8150_cfg = { .sct_data = sm8150_data, .size = ARRAY_SIZE(sm8150_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sm8250_cfg = { .sct_data = sm8250_data, .size = ARRAY_SIZE(sm8250_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sm8350_cfg = { .sct_data = sm8350_data, .size = ARRAY_SIZE(sm8350_data), .need_llcc_cfg = true, - .reg_offset = llcc_v1_2_reg_offset, + .reg_offset = llcc_v1_reg_offset, + .edac_reg_offset = &llcc_v1_edac_reg_offset, }; static const struct qcom_llcc_config sm8450_cfg = { .sct_data = sm8450_data, .size = ARRAY_SIZE(sm8450_data), .need_llcc_cfg = true, - .reg_offset = llcc_v21_reg_offset, + .reg_offset = llcc_v2_1_reg_offset, + .edac_reg_offset = &llcc_v2_1_edac_reg_offset, }; static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER; @@ -774,6 +841,7 @@ static int qcom_llcc_probe(struct platform_device *pdev) drv_data->cfg = llcc_cfg; drv_data->cfg_size = sz; + drv_data->edac_reg_offset = cfg->edac_reg_offset; mutex_init(&drv_data->lock); platform_set_drvdata(pdev, drv_data); diff --git a/drivers/soc/qcom/qcom_stats.c b/drivers/soc/qcom/qcom_stats.c index d6bfd1bbdc2a8932fda0d7eb9e27c435671796e9..121ea409fafcdce952b488571490d7ecc8614d0d 100644 --- a/drivers/soc/qcom/qcom_stats.c +++ b/drivers/soc/qcom/qcom_stats.c @@ -246,6 +246,14 @@ static const struct stats_config rpm_data_dba0 = { .subsystem_stats_in_smem = false, }; +static const struct stats_config rpmh_data_sdm845 = { + .stats_offset = 0x48, + .num_records = 2, + .appended_stats_avail = false, + .dynamic_offset = false, + .subsystem_stats_in_smem = true, +}; + static const struct stats_config rpmh_data = { .stats_offset = 0x48, .num_records = 3, @@ -261,6 +269,7 @@ static const struct of_device_id qcom_stats_table[] = { { .compatible = "qcom,msm8974-rpm-stats", .data = &rpm_data_dba0 }, { .compatible = "qcom,rpm-stats", .data = &rpm_data }, { .compatible = "qcom,rpmh-stats", .data = &rpmh_data }, + { .compatible = "qcom,sdm845-rpmh-stats", .data = &rpmh_data_sdm845 }, { } }; MODULE_DEVICE_TABLE(of, qcom_stats_table); diff --git a/drivers/soc/qcom/qmi_encdec.c b/drivers/soc/qcom/qmi_encdec.c index 328cc82371919ff97d4bbe785dc4a988dbd7f48c..b7158e3c3a0bdda4e75a5d5cf21809290519f798 100644 --- a/drivers/soc/qcom/qmi_encdec.c +++ b/drivers/soc/qcom/qmi_encdec.c @@ -57,11 +57,11 @@ do { \ #define TLV_TYPE_SIZE sizeof(u8) #define OPTIONAL_TLV_TYPE_START 0x10 -static int qmi_encode(struct qmi_elem_info *ei_array, void *out_buf, +static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf, const void *in_c_struct, u32 out_buf_len, int enc_level); -static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct, +static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct, const void *in_buf, u32 in_buf_len, int dec_level); /** @@ -76,10 +76,10 @@ static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct, * * Return: struct info of the next element that can be encoded. */ -static struct qmi_elem_info *skip_to_next_elem(struct qmi_elem_info *ei_array, - int level) +static const struct qmi_elem_info * +skip_to_next_elem(const struct qmi_elem_info *ei_array, int level) { - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; u8 tlv_type; if (level > 1) { @@ -101,11 +101,11 @@ static struct qmi_elem_info *skip_to_next_elem(struct qmi_elem_info *ei_array, * * Return: Expected minimum length of the QMI message or 0 on error. */ -static int qmi_calc_min_msg_len(struct qmi_elem_info *ei_array, +static int qmi_calc_min_msg_len(const struct qmi_elem_info *ei_array, int level) { int min_msg_len = 0; - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; if (!ei_array) return min_msg_len; @@ -194,13 +194,13 @@ static int qmi_encode_basic_elem(void *buf_dst, const void *buf_src, * Return: The number of bytes of encoded information on success or negative * errno on error. */ -static int qmi_encode_struct_elem(struct qmi_elem_info *ei_array, +static int qmi_encode_struct_elem(const struct qmi_elem_info *ei_array, void *buf_dst, const void *buf_src, u32 elem_len, u32 out_buf_len, int enc_level) { int i, rc, encoded_bytes = 0; - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; for (i = 0; i < elem_len; i++) { rc = qmi_encode(temp_ei->ei_array, buf_dst, buf_src, @@ -233,13 +233,13 @@ static int qmi_encode_struct_elem(struct qmi_elem_info *ei_array, * Return: The number of bytes of encoded information on success or negative * errno on error. */ -static int qmi_encode_string_elem(struct qmi_elem_info *ei_array, +static int qmi_encode_string_elem(const struct qmi_elem_info *ei_array, void *buf_dst, const void *buf_src, u32 out_buf_len, int enc_level) { int rc; int encoded_bytes = 0; - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; u32 string_len = 0; u32 string_len_sz = 0; @@ -289,11 +289,11 @@ static int qmi_encode_string_elem(struct qmi_elem_info *ei_array, * Return: The number of bytes of encoded information on success or negative * errno on error. */ -static int qmi_encode(struct qmi_elem_info *ei_array, void *out_buf, +static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf, const void *in_c_struct, u32 out_buf_len, int enc_level) { - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; u8 opt_flag_value = 0; u32 data_len_value = 0, data_len_sz; u8 *buf_dst = (u8 *)out_buf; @@ -468,13 +468,13 @@ static int qmi_decode_basic_elem(void *buf_dst, const void *buf_src, * Return: The total size of the decoded data elements on success, negative * errno on error. */ -static int qmi_decode_struct_elem(struct qmi_elem_info *ei_array, +static int qmi_decode_struct_elem(const struct qmi_elem_info *ei_array, void *buf_dst, const void *buf_src, u32 elem_len, u32 tlv_len, int dec_level) { int i, rc, decoded_bytes = 0; - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; for (i = 0; i < elem_len && decoded_bytes < tlv_len; i++) { rc = qmi_decode(temp_ei->ei_array, buf_dst, buf_src, @@ -514,7 +514,7 @@ static int qmi_decode_struct_elem(struct qmi_elem_info *ei_array, * Return: The total size of the decoded data elements on success, negative * errno on error. */ -static int qmi_decode_string_elem(struct qmi_elem_info *ei_array, +static int qmi_decode_string_elem(const struct qmi_elem_info *ei_array, void *buf_dst, const void *buf_src, u32 tlv_len, int dec_level) { @@ -522,7 +522,7 @@ static int qmi_decode_string_elem(struct qmi_elem_info *ei_array, int decoded_bytes = 0; u32 string_len = 0; u32 string_len_sz = 0; - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; if (dec_level == 1) { string_len = tlv_len; @@ -564,10 +564,10 @@ static int qmi_decode_string_elem(struct qmi_elem_info *ei_array, * * Return: Pointer to struct info, if found */ -static struct qmi_elem_info *find_ei(struct qmi_elem_info *ei_array, - u32 type) +static const struct qmi_elem_info *find_ei(const struct qmi_elem_info *ei_array, + u32 type) { - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; while (temp_ei->data_type != QMI_EOTI) { if (temp_ei->tlv_type == (u8)type) @@ -590,11 +590,11 @@ static struct qmi_elem_info *find_ei(struct qmi_elem_info *ei_array, * Return: The number of bytes of decoded information on success, negative * errno on error. */ -static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct, +static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct, const void *in_buf, u32 in_buf_len, int dec_level) { - struct qmi_elem_info *temp_ei = ei_array; + const struct qmi_elem_info *temp_ei = ei_array; u8 opt_flag_value = 1; u32 data_len_value = 0, data_len_sz = 0; u8 *buf_dst = out_c_struct; @@ -713,7 +713,7 @@ static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct, * Return: Buffer with encoded message, or negative ERR_PTR() on error */ void *qmi_encode_message(int type, unsigned int msg_id, size_t *len, - unsigned int txn_id, struct qmi_elem_info *ei, + unsigned int txn_id, const struct qmi_elem_info *ei, const void *c_struct) { struct qmi_header *hdr; @@ -767,7 +767,7 @@ EXPORT_SYMBOL(qmi_encode_message); * errno on error. */ int qmi_decode_message(const void *buf, size_t len, - struct qmi_elem_info *ei, void *c_struct) + const struct qmi_elem_info *ei, void *c_struct) { if (!ei) return -EINVAL; @@ -781,7 +781,7 @@ int qmi_decode_message(const void *buf, size_t len, EXPORT_SYMBOL(qmi_decode_message); /* Common header in all QMI responses */ -struct qmi_elem_info qmi_response_type_v01_ei[] = { +const struct qmi_elem_info qmi_response_type_v01_ei[] = { { .data_type = QMI_SIGNED_2_BYTE_ENUM, .elem_len = 1, diff --git a/drivers/soc/qcom/qmi_interface.c b/drivers/soc/qcom/qmi_interface.c index c8c4c730b1353fb48528be8908e0c281b49920b0..57052726299db275d74bd137f8016df7bba7e0bd 100644 --- a/drivers/soc/qcom/qmi_interface.c +++ b/drivers/soc/qcom/qmi_interface.c @@ -305,7 +305,7 @@ EXPORT_SYMBOL(qmi_add_server); * Return: Transaction id on success, negative errno on failure. */ int qmi_txn_init(struct qmi_handle *qmi, struct qmi_txn *txn, - struct qmi_elem_info *ei, void *c_struct) + const struct qmi_elem_info *ei, void *c_struct) { int ret; @@ -736,7 +736,8 @@ EXPORT_SYMBOL(qmi_handle_release); static ssize_t qmi_send_message(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, struct qmi_txn *txn, int type, int msg_id, size_t len, - struct qmi_elem_info *ei, const void *c_struct) + const struct qmi_elem_info *ei, + const void *c_struct) { struct msghdr msghdr = {}; struct kvec iv; @@ -787,7 +788,7 @@ static ssize_t qmi_send_message(struct qmi_handle *qmi, */ ssize_t qmi_send_request(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, struct qmi_txn *txn, int msg_id, size_t len, - struct qmi_elem_info *ei, const void *c_struct) + const struct qmi_elem_info *ei, const void *c_struct) { return qmi_send_message(qmi, sq, txn, QMI_REQUEST, msg_id, len, ei, c_struct); @@ -808,7 +809,7 @@ EXPORT_SYMBOL(qmi_send_request); */ ssize_t qmi_send_response(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, struct qmi_txn *txn, int msg_id, size_t len, - struct qmi_elem_info *ei, const void *c_struct) + const struct qmi_elem_info *ei, const void *c_struct) { return qmi_send_message(qmi, sq, txn, QMI_RESPONSE, msg_id, len, ei, c_struct); @@ -827,7 +828,8 @@ EXPORT_SYMBOL(qmi_send_response); * Return: 0 on success, negative errno on failure. */ ssize_t qmi_send_indication(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, - int msg_id, size_t len, struct qmi_elem_info *ei, + int msg_id, size_t len, + const struct qmi_elem_info *ei, const void *c_struct) { struct qmi_txn txn; diff --git a/drivers/soc/qcom/rpmpd.c b/drivers/soc/qcom/rpmpd.c index 5803038c744e58b24b641ad4c47fa7ffca7de392..337b1ad1cd3bfecd4119dd417a4676655f9ee7bf 100644 --- a/drivers/soc/qcom/rpmpd.c +++ b/drivers/soc/qcom/rpmpd.c @@ -29,6 +29,7 @@ #define RPMPD_RWLM 0x6d6c7772 #define RPMPD_RWSC 0x63737772 #define RPMPD_RWSM 0x6d737772 +#define RPMPD_RWGX 0x78677772 /* Operation Keys */ #define KEY_CORNER 0x6e726f63 /* corn */ @@ -433,6 +434,26 @@ static const struct rpmpd_desc sm6125_desc = { .max_state = RPM_SMD_LEVEL_BINNING, }; +DEFINE_RPMPD_PAIR(sm6375, vddgx, vddgx_ao, RWGX, LEVEL, 0); +static struct rpmpd *sm6375_rpmpds[] = { + [SM6375_VDDCX] = &sm6125_vddcx, + [SM6375_VDDCX_AO] = &sm6125_vddcx_ao, + [SM6375_VDDCX_VFL] = &sm6125_vddcx_vfl, + [SM6375_VDDMX] = &sm6125_vddmx, + [SM6375_VDDMX_AO] = &sm6125_vddmx_ao, + [SM6375_VDDMX_VFL] = &sm6125_vddmx_vfl, + [SM6375_VDDGX] = &sm6375_vddgx, + [SM6375_VDDGX_AO] = &sm6375_vddgx_ao, + [SM6375_VDD_LPI_CX] = &sm6115_vdd_lpi_cx, + [SM6375_VDD_LPI_MX] = &sm6115_vdd_lpi_mx, +}; + +static const struct rpmpd_desc sm6375_desc = { + .rpmpds = sm6375_rpmpds, + .num_pds = ARRAY_SIZE(sm6375_rpmpds), + .max_state = RPM_SMD_LEVEL_TURBO_NO_CPR, +}; + static struct rpmpd *qcm2290_rpmpds[] = { [QCM2290_VDDCX] = &sm6115_vddcx, [QCM2290_VDDCX_AO] = &sm6115_vddcx_ao, @@ -466,6 +487,7 @@ static const struct of_device_id rpmpd_match_table[] = { { .compatible = "qcom,sdm660-rpmpd", .data = &sdm660_desc }, { .compatible = "qcom,sm6115-rpmpd", .data = &sm6115_desc }, { .compatible = "qcom,sm6125-rpmpd", .data = &sm6125_desc }, + { .compatible = "qcom,sm6375-rpmpd", .data = &sm6375_desc }, { } }; MODULE_DEVICE_TABLE(of, rpmpd_match_table); diff --git a/drivers/soc/qcom/smem_state.c b/drivers/soc/qcom/smem_state.c index 31faf4aa868e60d5c9d06df9354772a7ff549464..e848cc9a3cf801be1743e4ac1bcfc2701ae993df 100644 --- a/drivers/soc/qcom/smem_state.c +++ b/drivers/soc/qcom/smem_state.c @@ -136,6 +136,7 @@ static void qcom_smem_state_release(struct kref *ref) struct qcom_smem_state *state = container_of(ref, struct qcom_smem_state, refcount); list_del(&state->list); + of_node_put(state->of_node); kfree(state); } @@ -205,7 +206,7 @@ struct qcom_smem_state *qcom_smem_state_register(struct device_node *of_node, kref_init(&state->refcount); - state->of_node = of_node; + state->of_node = of_node_get(of_node); state->ops = *ops; state->priv = priv; diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c index 9df9bba242f3e24435bc83157da965f01a93c78a..3e8994d6110e68ddaa1fcc4a59c8eb4f0b7c62de 100644 --- a/drivers/soc/qcom/smsm.c +++ b/drivers/soc/qcom/smsm.c @@ -526,7 +526,7 @@ static int qcom_smsm_probe(struct platform_device *pdev) for (id = 0; id < smsm->num_hosts; id++) { ret = smsm_parse_ipc(smsm, id); if (ret < 0) - return ret; + goto out_put; } /* Acquire the main SMSM state vector */ @@ -534,13 +534,14 @@ static int qcom_smsm_probe(struct platform_device *pdev) smsm->num_entries * sizeof(u32)); if (ret < 0 && ret != -EEXIST) { dev_err(&pdev->dev, "unable to allocate shared state entry\n"); - return ret; + goto out_put; } states = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE, NULL); if (IS_ERR(states)) { dev_err(&pdev->dev, "Unable to acquire shared state entry\n"); - return PTR_ERR(states); + ret = PTR_ERR(states); + goto out_put; } /* Acquire the list of interrupt mask vectors */ @@ -548,13 +549,14 @@ static int qcom_smsm_probe(struct platform_device *pdev) ret = qcom_smem_alloc(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, size); if (ret < 0 && ret != -EEXIST) { dev_err(&pdev->dev, "unable to allocate smsm interrupt mask\n"); - return ret; + goto out_put; } intr_mask = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, NULL); if (IS_ERR(intr_mask)) { dev_err(&pdev->dev, "unable to acquire shared memory interrupt mask\n"); - return PTR_ERR(intr_mask); + ret = PTR_ERR(intr_mask); + goto out_put; } /* Setup the reference to the local state bits */ @@ -565,7 +567,8 @@ static int qcom_smsm_probe(struct platform_device *pdev) smsm->state = qcom_smem_state_register(local_node, &smsm_state_ops, smsm); if (IS_ERR(smsm->state)) { dev_err(smsm->dev, "failed to register qcom_smem_state\n"); - return PTR_ERR(smsm->state); + ret = PTR_ERR(smsm->state); + goto out_put; } /* Register handlers for remote processor entries of interest. */ @@ -595,16 +598,19 @@ static int qcom_smsm_probe(struct platform_device *pdev) } platform_set_drvdata(pdev, smsm); + of_node_put(local_node); return 0; unwind_interfaces: + of_node_put(node); for (id = 0; id < smsm->num_entries; id++) if (smsm->entries[id].domain) irq_domain_remove(smsm->entries[id].domain); qcom_smem_state_unregister(smsm->state); - +out_put: + of_node_put(local_node); return ret; } diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c index 4554fb8655d34e393d3c60d25f8200f696dec81f..aa37e1bad095c91a718d40ec86a5b3696f7dbdc1 100644 --- a/drivers/soc/qcom/socinfo.c +++ b/drivers/soc/qcom/socinfo.c @@ -104,6 +104,7 @@ static const char *const pmic_models[] = { [36] = "PM8009", [38] = "PM8150C", [41] = "SMB2351", + [45] = "PM6125", [47] = "PMK8350", [48] = "PM8350", [49] = "PM8350C", @@ -334,6 +335,7 @@ static const struct soc_id soc_id[] = { { 482, "SM8450" }, { 487, "SC7280" }, { 495, "SC7180P" }, + { 507, "SM6375" }, }; static const char *socinfo_machine(struct device *dev, unsigned int id) diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index c50a6ce1b99d1ed9dc634e32d47ab3244966f0eb..f95a1337450d2a3f23ade96752cdb0f24b18119b 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -44,6 +44,7 @@ config ARCH_RZG2L bool select PM select PM_GENERIC_DOMAINS + select RENESAS_RZG2L_IRQC config ARCH_RZN1 bool @@ -332,6 +333,16 @@ config ARCH_R9A09G011 endif # ARM64 +if RISCV + +config ARCH_R9A07G043 + bool "RISC-V Platform support for RZ/Five" + select ARCH_RZG2L + help + This enables support for the Renesas RZ/Five SoC. + +endif # RISCV + config RST_RCAR bool "Reset Controller support for R-Car" if COMPILE_TEST diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c index d171f1b635c7ecfb0543682e6a4fa62df50121b2..621ceaa047d472bedfe81d6b8ddc87d4fadc0e81 100644 --- a/drivers/soc/renesas/renesas-soc.c +++ b/drivers/soc/renesas/renesas-soc.c @@ -50,6 +50,10 @@ static const struct renesas_family fam_rza2 __initconst __maybe_unused = { .name = "RZ/A2", }; +static const struct renesas_family fam_rzfive __initconst __maybe_unused = { + .name = "RZ/Five", +}; + static const struct renesas_family fam_rzg1 __initconst __maybe_unused = { .name = "RZ/G1", .reg = 0xff000044, /* PRR (Product Register) */ @@ -102,6 +106,11 @@ static const struct renesas_soc soc_rmobile_a1 __initconst __maybe_unused = { .id = 0x40, }; +static const struct renesas_soc soc_rz_five __initconst __maybe_unused = { + .family = &fam_rzfive, + .id = 0x847c447, +}; + static const struct renesas_soc soc_rz_g1h __initconst __maybe_unused = { .family = &fam_rzg1, .id = 0x45, @@ -320,6 +329,7 @@ static const struct of_device_id renesas_socs[] __initconst = { { .compatible = "renesas,r8a779m0", .data = &soc_rcar_h3 }, { .compatible = "renesas,r8a779m1", .data = &soc_rcar_h3 }, { .compatible = "renesas,r8a779m8", .data = &soc_rcar_h3 }, + { .compatible = "renesas,r8a779mb", .data = &soc_rcar_h3 }, #endif #ifdef CONFIG_ARCH_R8A77960 { .compatible = "renesas,r8a7796", .data = &soc_rcar_m3_w }, @@ -358,8 +368,12 @@ static const struct of_device_id renesas_socs[] __initconst = { { .compatible = "renesas,r8a779g0", .data = &soc_rcar_v4h }, #endif #if defined(CONFIG_ARCH_R9A07G043) +#ifdef CONFIG_RISCV + { .compatible = "renesas,r9a07g043", .data = &soc_rz_five }, +#else { .compatible = "renesas,r9a07g043", .data = &soc_rz_g2ul }, #endif +#endif #if defined(CONFIG_ARCH_R9A07G044) { .compatible = "renesas,r9a07g044", .data = &soc_rz_g2l }, #endif diff --git a/drivers/soc/rockchip/io-domain.c b/drivers/soc/rockchip/io-domain.c index 9df513d1219bc5466b8dd4dcc85ee62e5f0d20d3..6619256c2d1180f2a24e5c1a595bafe1d2c65c16 100644 --- a/drivers/soc/rockchip/io-domain.c +++ b/drivers/soc/rockchip/io-domain.c @@ -491,6 +491,22 @@ static const struct rockchip_iodomain_soc_data soc_data_rv1108_pmu = { }, }; +static const struct rockchip_iodomain_soc_data soc_data_rv1126_pmu = { + .grf_offset = 0x140, + .supply_names = { + NULL, + "vccio1", + "vccio2", + "vccio3", + "vccio4", + "vccio5", + "vccio6", + "vccio7", + "pmuio0", + "pmuio1", + }, +}; + static const struct of_device_id rockchip_iodomain_match[] = { { .compatible = "rockchip,px30-io-voltage-domain", @@ -544,6 +560,10 @@ static const struct of_device_id rockchip_iodomain_match[] = { .compatible = "rockchip,rv1108-pmu-io-voltage-domain", .data = &soc_data_rv1108_pmu }, + { + .compatible = "rockchip,rv1126-pmu-io-voltage-domain", + .data = &soc_data_rv1126_pmu + }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, rockchip_iodomain_match); diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c index 89795abac9518f440c060edaef2955b35ee478a4..84bc022f9e5b71bbb19b2d6129bfac7909d71d4d 100644 --- a/drivers/soc/rockchip/pm_domains.c +++ b/drivers/soc/rockchip/pm_domains.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include #include #include +#include struct rockchip_domain_info { const char *name; @@ -41,6 +43,9 @@ struct rockchip_domain_info { bool active_wakeup; int pwr_w_mask; int req_w_mask; + int repair_status_mask; + u32 pwr_offset; + u32 req_offset; }; struct rockchip_pmu_info { @@ -49,6 +54,7 @@ struct rockchip_pmu_info { u32 req_offset; u32 idle_offset; u32 ack_offset; + u32 repair_status_offset; u32 core_pwrcnt_offset; u32 gpu_pwrcnt_offset; @@ -113,6 +119,22 @@ struct rockchip_pmu { .active_wakeup = wakeup, \ } +#define DOMAIN_M_O_R(_name, p_offset, pwr, status, r_status, r_offset, req, idle, ack, wakeup) \ +{ \ + .name = _name, \ + .pwr_offset = p_offset, \ + .pwr_w_mask = (pwr) << 16, \ + .pwr_mask = (pwr), \ + .status_mask = (status), \ + .repair_status_mask = (r_status), \ + .req_offset = r_offset, \ + .req_w_mask = (req) << 16, \ + .req_mask = (req), \ + .idle_mask = (idle), \ + .ack_mask = (ack), \ + .active_wakeup = wakeup, \ +} + #define DOMAIN_RK3036(_name, req, ack, idle, wakeup) \ { \ .name = _name, \ @@ -126,6 +148,9 @@ struct rockchip_pmu { #define DOMAIN_PX30(name, pwr, status, req, wakeup) \ DOMAIN_M(name, pwr, status, req, (req) << 16, req, wakeup) +#define DOMAIN_RV1126(name, pwr, req, idle, wakeup) \ + DOMAIN_M(name, pwr, pwr, req, idle, idle, wakeup) + #define DOMAIN_RK3288(name, pwr, status, req, wakeup) \ DOMAIN(name, pwr, status, req, req, (req) << 16, wakeup) @@ -244,6 +269,9 @@ void rockchip_pmu_unblock(void) } EXPORT_SYMBOL_GPL(rockchip_pmu_unblock); +#define DOMAIN_RK3588(name, p_offset, pwr, status, r_status, r_offset, req, idle, wakeup) \ + DOMAIN_M_O_R(name, p_offset, pwr, status, r_status, r_offset, req, idle, idle, wakeup) + static bool rockchip_pmu_domain_is_idle(struct rockchip_pm_domain *pd) { struct rockchip_pmu *pmu = pd->pmu; @@ -268,6 +296,7 @@ static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd, const struct rockchip_domain_info *pd_info = pd->info; struct generic_pm_domain *genpd = &pd->genpd; struct rockchip_pmu *pmu = pd->pmu; + u32 pd_req_offset = pd_info->req_offset; unsigned int target_ack; unsigned int val; bool is_idle; @@ -276,11 +305,11 @@ static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd, if (pd_info->req_mask == 0) return 0; else if (pd_info->req_w_mask) - regmap_write(pmu->regmap, pmu->info->req_offset, + regmap_write(pmu->regmap, pmu->info->req_offset + pd_req_offset, idle ? (pd_info->req_mask | pd_info->req_w_mask) : pd_info->req_w_mask); else - regmap_update_bits(pmu->regmap, pmu->info->req_offset, + regmap_update_bits(pmu->regmap, pmu->info->req_offset + pd_req_offset, pd_info->req_mask, idle ? -1U : 0); wmb(); @@ -363,6 +392,12 @@ static bool rockchip_pmu_domain_is_on(struct rockchip_pm_domain *pd) struct rockchip_pmu *pmu = pd->pmu; unsigned int val; + if (pd->info->repair_status_mask) { + regmap_read(pmu->regmap, pmu->info->repair_status_offset, &val); + /* 1'b1: power on, 1'b0: power off */ + return val & pd->info->repair_status_mask; + } + /* check idle status for idle-only domains */ if (pd->info->status_mask == 0) return !rockchip_pmu_domain_is_idle(pd); @@ -378,16 +413,17 @@ static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd, { struct rockchip_pmu *pmu = pd->pmu; struct generic_pm_domain *genpd = &pd->genpd; + u32 pd_pwr_offset = pd->info->pwr_offset; bool is_on; if (pd->info->pwr_mask == 0) return; else if (pd->info->pwr_w_mask) - regmap_write(pmu->regmap, pmu->info->pwr_offset, + regmap_write(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset, on ? pd->info->pwr_w_mask : (pd->info->pwr_mask | pd->info->pwr_w_mask)); else - regmap_update_bits(pmu->regmap, pmu->info->pwr_offset, + regmap_update_bits(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset, pd->info->pwr_mask, on ? 0 : -1U); wmb(); @@ -514,6 +550,9 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu, node, id); return -EINVAL; } + /* RK3588 has domains with two parents (RKVDEC0/RKVDEC1) */ + if (pmu->genpd_data.domains[id]) + return 0; pd_info = &pmu->info->domain_info[id]; if (!pd_info) { @@ -595,14 +634,6 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu, } } - error = rockchip_pd_power(pd, true); - if (error) { - dev_err(pmu->dev, - "failed to power on domain '%pOFn': %d\n", - node, error); - goto err_unprepare_clocks; - } - if (pd->info->name) pd->genpd.name = pd->info->name; else @@ -614,7 +645,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu, pd->genpd.flags = GENPD_FLAG_PM_CLK; if (pd_info->active_wakeup) pd->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; - pm_genpd_init(&pd->genpd, NULL, false); + pm_genpd_init(&pd->genpd, NULL, !rockchip_pmu_domain_is_on(pd)); pmu->genpd_data.domains[id] = &pd->genpd; return 0; @@ -855,6 +886,16 @@ static const struct rockchip_domain_info px30_pm_domains[] = { [PX30_PD_GPU] = DOMAIN_PX30("gpu", BIT(15), BIT(15), BIT(2), false), }; +static const struct rockchip_domain_info rv1126_pm_domains[] = { + [RV1126_PD_VEPU] = DOMAIN_RV1126("vepu", BIT(2), BIT(9), BIT(9), false), + [RV1126_PD_VI] = DOMAIN_RV1126("vi", BIT(4), BIT(6), BIT(6), false), + [RV1126_PD_ISPP] = DOMAIN_RV1126("ispp", BIT(1), BIT(8), BIT(8), false), + [RV1126_PD_VDPU] = DOMAIN_RV1126("vdpu", BIT(3), BIT(10), BIT(10), false), + [RV1126_PD_NVM] = DOMAIN_RV1126("nvm", BIT(7), BIT(11), BIT(11), false), + [RV1126_PD_SDIO] = DOMAIN_RV1126("sdio", BIT(8), BIT(13), BIT(13), false), + [RV1126_PD_USB] = DOMAIN_RV1126("usb", BIT(9), BIT(15), BIT(15), false), +}; + static const struct rockchip_domain_info rk3036_pm_domains[] = { [RK3036_PD_MSCH] = DOMAIN_RK3036("msch", BIT(14), BIT(23), BIT(30), true), [RK3036_PD_CORE] = DOMAIN_RK3036("core", BIT(13), BIT(17), BIT(24), false), @@ -982,6 +1023,38 @@ static const struct rockchip_domain_info rk3568_pm_domains[] = { [RK3568_PD_PIPE] = DOMAIN_RK3568("pipe", BIT(8), BIT(11), false), }; +static const struct rockchip_domain_info rk3588_pm_domains[] = { + [RK3588_PD_GPU] = DOMAIN_RK3588("gpu", 0x0, BIT(0), 0, BIT(1), 0x0, BIT(0), BIT(0), false), + [RK3588_PD_NPU] = DOMAIN_RK3588("npu", 0x0, BIT(1), BIT(1), 0, 0x0, 0, 0, false), + [RK3588_PD_VCODEC] = DOMAIN_RK3588("vcodec", 0x0, BIT(2), BIT(2), 0, 0x0, 0, 0, false), + [RK3588_PD_NPUTOP] = DOMAIN_RK3588("nputop", 0x0, BIT(3), 0, BIT(2), 0x0, BIT(1), BIT(1), false), + [RK3588_PD_NPU1] = DOMAIN_RK3588("npu1", 0x0, BIT(4), 0, BIT(3), 0x0, BIT(2), BIT(2), false), + [RK3588_PD_NPU2] = DOMAIN_RK3588("npu2", 0x0, BIT(5), 0, BIT(4), 0x0, BIT(3), BIT(3), false), + [RK3588_PD_VENC0] = DOMAIN_RK3588("venc0", 0x0, BIT(6), 0, BIT(5), 0x0, BIT(4), BIT(4), false), + [RK3588_PD_VENC1] = DOMAIN_RK3588("venc1", 0x0, BIT(7), 0, BIT(6), 0x0, BIT(5), BIT(5), false), + [RK3588_PD_RKVDEC0] = DOMAIN_RK3588("rkvdec0", 0x0, BIT(8), 0, BIT(7), 0x0, BIT(6), BIT(6), false), + [RK3588_PD_RKVDEC1] = DOMAIN_RK3588("rkvdec1", 0x0, BIT(9), 0, BIT(8), 0x0, BIT(7), BIT(7), false), + [RK3588_PD_VDPU] = DOMAIN_RK3588("vdpu", 0x0, BIT(10), 0, BIT(9), 0x0, BIT(8), BIT(8), false), + [RK3588_PD_RGA30] = DOMAIN_RK3588("rga30", 0x0, BIT(11), 0, BIT(10), 0x0, 0, 0, false), + [RK3588_PD_AV1] = DOMAIN_RK3588("av1", 0x0, BIT(12), 0, BIT(11), 0x0, BIT(9), BIT(9), false), + [RK3588_PD_VI] = DOMAIN_RK3588("vi", 0x0, BIT(13), 0, BIT(12), 0x0, BIT(10), BIT(10), false), + [RK3588_PD_FEC] = DOMAIN_RK3588("fec", 0x0, BIT(14), 0, BIT(13), 0x0, 0, 0, false), + [RK3588_PD_ISP1] = DOMAIN_RK3588("isp1", 0x0, BIT(15), 0, BIT(14), 0x0, BIT(11), BIT(11), false), + [RK3588_PD_RGA31] = DOMAIN_RK3588("rga31", 0x4, BIT(0), 0, BIT(15), 0x0, BIT(12), BIT(12), false), + [RK3588_PD_VOP] = DOMAIN_RK3588("vop", 0x4, BIT(1), 0, BIT(16), 0x0, BIT(13) | BIT(14), BIT(13) | BIT(14), false), + [RK3588_PD_VO0] = DOMAIN_RK3588("vo0", 0x4, BIT(2), 0, BIT(17), 0x0, BIT(15), BIT(15), false), + [RK3588_PD_VO1] = DOMAIN_RK3588("vo1", 0x4, BIT(3), 0, BIT(18), 0x4, BIT(0), BIT(16), false), + [RK3588_PD_AUDIO] = DOMAIN_RK3588("audio", 0x4, BIT(4), 0, BIT(19), 0x4, BIT(1), BIT(17), false), + [RK3588_PD_PHP] = DOMAIN_RK3588("php", 0x4, BIT(5), 0, BIT(20), 0x4, BIT(5), BIT(21), false), + [RK3588_PD_GMAC] = DOMAIN_RK3588("gmac", 0x4, BIT(6), 0, BIT(21), 0x0, 0, 0, false), + [RK3588_PD_PCIE] = DOMAIN_RK3588("pcie", 0x4, BIT(7), 0, BIT(22), 0x0, 0, 0, true), + [RK3588_PD_NVM] = DOMAIN_RK3588("nvm", 0x4, BIT(8), BIT(24), 0, 0x4, BIT(2), BIT(18), false), + [RK3588_PD_NVM0] = DOMAIN_RK3588("nvm0", 0x4, BIT(9), 0, BIT(23), 0x0, 0, 0, false), + [RK3588_PD_SDIO] = DOMAIN_RK3588("sdio", 0x4, BIT(10), 0, BIT(24), 0x4, BIT(3), BIT(19), false), + [RK3588_PD_USB] = DOMAIN_RK3588("usb", 0x4, BIT(11), 0, BIT(25), 0x4, BIT(4), BIT(20), true), + [RK3588_PD_SDMMC] = DOMAIN_RK3588("sdmmc", 0x4, BIT(13), 0, BIT(26), 0x0, 0, 0, false), +}; + static const struct rockchip_pmu_info px30_pmu = { .pwr_offset = 0x18, .status_offset = 0x20, @@ -1128,6 +1201,29 @@ static const struct rockchip_pmu_info rk3568_pmu = { .domain_info = rk3568_pm_domains, }; +static const struct rockchip_pmu_info rk3588_pmu = { + .pwr_offset = 0x14c, + .status_offset = 0x180, + .req_offset = 0x10c, + .idle_offset = 0x120, + .ack_offset = 0x118, + .repair_status_offset = 0x290, + + .num_domains = ARRAY_SIZE(rk3588_pm_domains), + .domain_info = rk3588_pm_domains, +}; + +static const struct rockchip_pmu_info rv1126_pmu = { + .pwr_offset = 0x110, + .status_offset = 0x108, + .req_offset = 0xc0, + .idle_offset = 0xd8, + .ack_offset = 0xd0, + + .num_domains = ARRAY_SIZE(rv1126_pm_domains), + .domain_info = rv1126_pm_domains, +}; + static const struct of_device_id rockchip_pm_domain_dt_match[] = { { .compatible = "rockchip,px30-power-controller", @@ -1177,6 +1273,14 @@ static const struct of_device_id rockchip_pm_domain_dt_match[] = { .compatible = "rockchip,rk3568-power-controller", .data = (void *)&rk3568_pmu, }, + { + .compatible = "rockchip,rk3588-power-controller", + .data = (void *)&rk3588_pmu, + }, + { + .compatible = "rockchip,rv1126-power-controller", + .data = (void *)&rv1126_pmu, + }, { /* sentinel */ }, }; diff --git a/drivers/soc/sifive/Kconfig b/drivers/soc/sifive/Kconfig index 58cf8c40d08d53e86fa27ba482a5cc3b1d1d1a55..ed4c571f8771b355f76e079185f38c312b25907a 100644 --- a/drivers/soc/sifive/Kconfig +++ b/drivers/soc/sifive/Kconfig @@ -2,9 +2,9 @@ if SOC_SIFIVE -config SIFIVE_L2 - bool "Sifive L2 Cache controller" +config SIFIVE_CCACHE + bool "Sifive Composable Cache controller" help - Support for the L2 cache controller on SiFive platforms. + Support for the composable cache controller on SiFive platforms. endif diff --git a/drivers/soc/sifive/Makefile b/drivers/soc/sifive/Makefile index b5caff77938f6f61014810ed1f170cfcf7781764..1f5dc339bf82763574cc1d015d412f086aff9114 100644 --- a/drivers/soc/sifive/Makefile +++ b/drivers/soc/sifive/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_SIFIVE_L2) += sifive_l2_cache.o +obj-$(CONFIG_SIFIVE_CCACHE) += sifive_ccache.o diff --git a/drivers/soc/sifive/sifive_ccache.c b/drivers/soc/sifive/sifive_ccache.c new file mode 100644 index 0000000000000000000000000000000000000000..1c171150e878d5bb0481c020d4fba2c73c92f879 --- /dev/null +++ b/drivers/soc/sifive/sifive_ccache.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SiFive composable cache controller Driver + * + * Copyright (C) 2018-2022 SiFive, Inc. + * + */ + +#define pr_fmt(fmt) "CCACHE: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 +#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 +#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 + +#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 +#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 +#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 + +#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 +#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 +#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 + +#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 +#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 +#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 + +#define SIFIVE_CCACHE_CONFIG 0x00 +#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) +#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) +#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) +#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) + +#define SIFIVE_CCACHE_WAYENABLE 0x08 +#define SIFIVE_CCACHE_ECCINJECTERR 0x40 + +#define SIFIVE_CCACHE_MAX_ECCINTR 4 + +static void __iomem *ccache_base; +static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; +static struct riscv_cacheinfo_ops ccache_cache_ops; +static int level; + +enum { + DIR_CORR = 0, + DATA_CORR, + DATA_UNCORR, + DIR_UNCORR, +}; + +#ifdef CONFIG_DEBUG_FS +static struct dentry *sifive_test; + +static ssize_t ccache_write(struct file *file, const char __user *data, + size_t count, loff_t *ppos) +{ + unsigned int val; + + if (kstrtouint_from_user(data, count, 0, &val)) + return -EINVAL; + if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) + writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); + else + return -EINVAL; + return count; +} + +static const struct file_operations ccache_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ccache_write +}; + +static void setup_sifive_debug(void) +{ + sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); + + debugfs_create_file("sifive_debug_inject_error", 0200, + sifive_test, NULL, &ccache_fops); +} +#endif + +static void ccache_config_read(void) +{ + u32 cfg; + + cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); + pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", + FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), + FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), + BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), + BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); + + cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); + pr_info("Index of the largest way enabled: %u\n", cfg); +} + +static const struct of_device_id sifive_ccache_ids[] = { + { .compatible = "sifive,fu540-c000-ccache" }, + { .compatible = "sifive,fu740-c000-ccache" }, + { .compatible = "sifive,ccache0" }, + { /* end of table */ } +}; + +static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); + +int register_sifive_ccache_error_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_register(&ccache_err_chain, nb); +} +EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); + +int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_unregister(&ccache_err_chain, nb); +} +EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); + +static int ccache_largest_wayenabled(void) +{ + return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; +} + +static ssize_t number_of_ways_enabled_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", ccache_largest_wayenabled()); +} + +static DEVICE_ATTR_RO(number_of_ways_enabled); + +static struct attribute *priv_attrs[] = { + &dev_attr_number_of_ways_enabled.attr, + NULL, +}; + +static const struct attribute_group priv_attr_group = { + .attrs = priv_attrs, +}; + +static const struct attribute_group *ccache_get_priv_group(struct cacheinfo + *this_leaf) +{ + /* We want to use private group for composable cache only */ + if (this_leaf->level == level) + return &priv_attr_group; + else + return NULL; +} + +static irqreturn_t ccache_int_handler(int irq, void *device) +{ + unsigned int add_h, add_l; + + if (irq == g_irq[DIR_CORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); + pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DirError interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_CE, + "DirECCFix"); + } + if (irq == g_irq[DIR_UNCORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); + /* Reading this register clears the DirFail interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_UE, + "DirECCFail"); + panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); + } + if (irq == g_irq[DATA_CORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); + pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DataError interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_CE, + "DatECCFix"); + } + if (irq == g_irq[DATA_UNCORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); + pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DataFail interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_UE, + "DatECCFail"); + } + + return IRQ_HANDLED; +} + +static int __init sifive_ccache_init(void) +{ + struct device_node *np; + struct resource res; + int i, rc, intr_num; + + np = of_find_matching_node(NULL, sifive_ccache_ids); + if (!np) + return -ENODEV; + + if (of_address_to_resource(np, 0, &res)) + return -ENODEV; + + ccache_base = ioremap(res.start, resource_size(&res)); + if (!ccache_base) + return -ENOMEM; + + if (of_property_read_u32(np, "cache-level", &level)) + return -ENOENT; + + intr_num = of_property_count_u32_elems(np, "interrupts"); + if (!intr_num) { + pr_err("No interrupts property\n"); + return -ENODEV; + } + + for (i = 0; i < intr_num; i++) { + g_irq[i] = irq_of_parse_and_map(np, i); + rc = request_irq(g_irq[i], ccache_int_handler, 0, "ccache_ecc", + NULL); + if (rc) { + pr_err("Could not request IRQ %d\n", g_irq[i]); + return rc; + } + } + + ccache_config_read(); + + ccache_cache_ops.get_priv_group = ccache_get_priv_group; + riscv_set_cacheinfo_ops(&ccache_cache_ops); + +#ifdef CONFIG_DEBUG_FS + setup_sifive_debug(); +#endif + return 0; +} + +device_initcall(sifive_ccache_init); diff --git a/drivers/soc/sifive/sifive_l2_cache.c b/drivers/soc/sifive/sifive_l2_cache.c deleted file mode 100644 index 59640a1d0b28a1a9606a6ddc6dc702df82bc97b5..0000000000000000000000000000000000000000 --- a/drivers/soc/sifive/sifive_l2_cache.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * SiFive L2 cache controller Driver - * - * Copyright (C) 2018-2019 SiFive, Inc. - * - */ -#include -#include -#include -#include -#include -#include -#include - -#define SIFIVE_L2_DIRECCFIX_LOW 0x100 -#define SIFIVE_L2_DIRECCFIX_HIGH 0x104 -#define SIFIVE_L2_DIRECCFIX_COUNT 0x108 - -#define SIFIVE_L2_DIRECCFAIL_LOW 0x120 -#define SIFIVE_L2_DIRECCFAIL_HIGH 0x124 -#define SIFIVE_L2_DIRECCFAIL_COUNT 0x128 - -#define SIFIVE_L2_DATECCFIX_LOW 0x140 -#define SIFIVE_L2_DATECCFIX_HIGH 0x144 -#define SIFIVE_L2_DATECCFIX_COUNT 0x148 - -#define SIFIVE_L2_DATECCFAIL_LOW 0x160 -#define SIFIVE_L2_DATECCFAIL_HIGH 0x164 -#define SIFIVE_L2_DATECCFAIL_COUNT 0x168 - -#define SIFIVE_L2_CONFIG 0x00 -#define SIFIVE_L2_WAYENABLE 0x08 -#define SIFIVE_L2_ECCINJECTERR 0x40 - -#define SIFIVE_L2_MAX_ECCINTR 4 - -static void __iomem *l2_base; -static int g_irq[SIFIVE_L2_MAX_ECCINTR]; -static struct riscv_cacheinfo_ops l2_cache_ops; - -enum { - DIR_CORR = 0, - DATA_CORR, - DATA_UNCORR, - DIR_UNCORR, -}; - -#ifdef CONFIG_DEBUG_FS -static struct dentry *sifive_test; - -static ssize_t l2_write(struct file *file, const char __user *data, - size_t count, loff_t *ppos) -{ - unsigned int val; - - if (kstrtouint_from_user(data, count, 0, &val)) - return -EINVAL; - if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) - writel(val, l2_base + SIFIVE_L2_ECCINJECTERR); - else - return -EINVAL; - return count; -} - -static const struct file_operations l2_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .write = l2_write -}; - -static void setup_sifive_debug(void) -{ - sifive_test = debugfs_create_dir("sifive_l2_cache", NULL); - - debugfs_create_file("sifive_debug_inject_error", 0200, - sifive_test, NULL, &l2_fops); -} -#endif - -static void l2_config_read(void) -{ - u32 regval, val; - - regval = readl(l2_base + SIFIVE_L2_CONFIG); - val = regval & 0xFF; - pr_info("L2CACHE: No. of Banks in the cache: %d\n", val); - val = (regval & 0xFF00) >> 8; - pr_info("L2CACHE: No. of ways per bank: %d\n", val); - val = (regval & 0xFF0000) >> 16; - pr_info("L2CACHE: Sets per bank: %llu\n", (uint64_t)1 << val); - val = (regval & 0xFF000000) >> 24; - pr_info("L2CACHE: Bytes per cache block: %llu\n", (uint64_t)1 << val); - - regval = readl(l2_base + SIFIVE_L2_WAYENABLE); - pr_info("L2CACHE: Index of the largest way enabled: %d\n", regval); -} - -static const struct of_device_id sifive_l2_ids[] = { - { .compatible = "sifive,fu540-c000-ccache" }, - { .compatible = "sifive,fu740-c000-ccache" }, - { /* end of table */ }, -}; - -static ATOMIC_NOTIFIER_HEAD(l2_err_chain); - -int register_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_register(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(register_sifive_l2_error_notifier); - -int unregister_sifive_l2_error_notifier(struct notifier_block *nb) -{ - return atomic_notifier_chain_unregister(&l2_err_chain, nb); -} -EXPORT_SYMBOL_GPL(unregister_sifive_l2_error_notifier); - -static int l2_largest_wayenabled(void) -{ - return readl(l2_base + SIFIVE_L2_WAYENABLE) & 0xFF; -} - -static ssize_t number_of_ways_enabled_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - return sprintf(buf, "%u\n", l2_largest_wayenabled()); -} - -static DEVICE_ATTR_RO(number_of_ways_enabled); - -static struct attribute *priv_attrs[] = { - &dev_attr_number_of_ways_enabled.attr, - NULL, -}; - -static const struct attribute_group priv_attr_group = { - .attrs = priv_attrs, -}; - -static const struct attribute_group *l2_get_priv_group(struct cacheinfo *this_leaf) -{ - /* We want to use private group for L2 cache only */ - if (this_leaf->level == 2) - return &priv_attr_group; - else - return NULL; -} - -static irqreturn_t l2_int_handler(int irq, void *device) -{ - unsigned int add_h, add_l; - - if (irq == g_irq[DIR_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DIRECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DIRECCFIX_LOW); - pr_err("L2CACHE: DirError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DirError interrupt sig */ - readl(l2_base + SIFIVE_L2_DIRECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DirECCFix"); - } - if (irq == g_irq[DIR_UNCORR]) { - add_h = readl(l2_base + SIFIVE_L2_DIRECCFAIL_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DIRECCFAIL_LOW); - /* Reading this register clears the DirFail interrupt sig */ - readl(l2_base + SIFIVE_L2_DIRECCFAIL_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE, - "DirECCFail"); - panic("L2CACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); - } - if (irq == g_irq[DATA_CORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFIX_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFIX_LOW); - pr_err("L2CACHE: DataError @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataError interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFIX_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_CE, - "DatECCFix"); - } - if (irq == g_irq[DATA_UNCORR]) { - add_h = readl(l2_base + SIFIVE_L2_DATECCFAIL_HIGH); - add_l = readl(l2_base + SIFIVE_L2_DATECCFAIL_LOW); - pr_err("L2CACHE: DataFail @ 0x%08X.%08X\n", add_h, add_l); - /* Reading this register clears the DataFail interrupt sig */ - readl(l2_base + SIFIVE_L2_DATECCFAIL_COUNT); - atomic_notifier_call_chain(&l2_err_chain, SIFIVE_L2_ERR_TYPE_UE, - "DatECCFail"); - } - - return IRQ_HANDLED; -} - -static int __init sifive_l2_init(void) -{ - struct device_node *np; - struct resource res; - int i, rc, intr_num; - - np = of_find_matching_node(NULL, sifive_l2_ids); - if (!np) - return -ENODEV; - - if (of_address_to_resource(np, 0, &res)) - return -ENODEV; - - l2_base = ioremap(res.start, resource_size(&res)); - if (!l2_base) - return -ENOMEM; - - intr_num = of_property_count_u32_elems(np, "interrupts"); - if (!intr_num) { - pr_err("L2CACHE: no interrupts property\n"); - return -ENODEV; - } - - for (i = 0; i < intr_num; i++) { - g_irq[i] = irq_of_parse_and_map(np, i); - rc = request_irq(g_irq[i], l2_int_handler, 0, "l2_ecc", NULL); - if (rc) { - pr_err("L2CACHE: Could not request IRQ %d\n", g_irq[i]); - return rc; - } - } - - l2_config_read(); - - l2_cache_ops.get_priv_group = l2_get_priv_group; - riscv_set_cacheinfo_ops(&l2_cache_ops); - -#ifdef CONFIG_DEBUG_FS - setup_sifive_debug(); -#endif - return 0; -} -device_initcall(sifive_l2_init); diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c index a8f3876963a087fe805932a8b4678d68d8f437a8..92f9186c1c4248dd427a738a3625c9bfc23a9af8 100644 --- a/drivers/soc/sunxi/sunxi_sram.c +++ b/drivers/soc/sunxi/sunxi_sram.c @@ -78,8 +78,8 @@ static struct sunxi_sram_desc sun4i_a10_sram_d = { static struct sunxi_sram_desc sun50i_a64_sram_c = { .data = SUNXI_SRAM_DATA("C", 0x4, 24, 1, - SUNXI_SRAM_MAP(0, 1, "cpu"), - SUNXI_SRAM_MAP(1, 0, "de2")), + SUNXI_SRAM_MAP(1, 0, "cpu"), + SUNXI_SRAM_MAP(0, 1, "de2")), }; static const struct of_device_id sunxi_sram_dt_ids[] = { @@ -254,36 +254,36 @@ int sunxi_sram_claim(struct device *dev) writel(val | ((device << sram_data->offset) & mask), base + sram_data->reg); + sram_desc->claimed = true; spin_unlock(&sram_lock); return 0; } EXPORT_SYMBOL(sunxi_sram_claim); -int sunxi_sram_release(struct device *dev) +void sunxi_sram_release(struct device *dev) { const struct sunxi_sram_data *sram_data; struct sunxi_sram_desc *sram_desc; if (!dev || !dev->of_node) - return -EINVAL; + return; sram_data = sunxi_sram_of_parse(dev->of_node, NULL); if (IS_ERR(sram_data)) - return -EINVAL; + return; sram_desc = to_sram_desc(sram_data); spin_lock(&sram_lock); sram_desc->claimed = false; spin_unlock(&sram_lock); - - return 0; } EXPORT_SYMBOL(sunxi_sram_release); struct sunxi_sramc_variant { int num_emac_clocks; + bool has_ldo_ctrl; }; static const struct sunxi_sramc_variant sun4i_a10_sramc_variant = { @@ -294,6 +294,11 @@ static const struct sunxi_sramc_variant sun8i_h3_sramc_variant = { .num_emac_clocks = 1, }; +static const struct sunxi_sramc_variant sun20i_d1_sramc_variant = { + .num_emac_clocks = 1, + .has_ldo_ctrl = true, +}; + static const struct sunxi_sramc_variant sun50i_a64_sramc_variant = { .num_emac_clocks = 1, }; @@ -303,37 +308,38 @@ static const struct sunxi_sramc_variant sun50i_h616_sramc_variant = { }; #define SUNXI_SRAM_EMAC_CLOCK_REG 0x30 +#define SUNXI_SYS_LDO_CTRL_REG 0x150 + static bool sunxi_sram_regmap_accessible_reg(struct device *dev, unsigned int reg) { - const struct sunxi_sramc_variant *variant; - - variant = of_device_get_match_data(dev); + const struct sunxi_sramc_variant *variant = dev_get_drvdata(dev); - if (reg < SUNXI_SRAM_EMAC_CLOCK_REG) - return false; - if (reg > SUNXI_SRAM_EMAC_CLOCK_REG + variant->num_emac_clocks * 4) - return false; + if (reg >= SUNXI_SRAM_EMAC_CLOCK_REG && + reg < SUNXI_SRAM_EMAC_CLOCK_REG + variant->num_emac_clocks * 4) + return true; + if (reg == SUNXI_SYS_LDO_CTRL_REG && variant->has_ldo_ctrl) + return true; - return true; + return false; } -static struct regmap_config sunxi_sram_emac_clock_regmap = { +static struct regmap_config sunxi_sram_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, /* last defined register */ - .max_register = SUNXI_SRAM_EMAC_CLOCK_REG + 4, + .max_register = SUNXI_SYS_LDO_CTRL_REG, /* other devices have no business accessing other registers */ .readable_reg = sunxi_sram_regmap_accessible_reg, .writeable_reg = sunxi_sram_regmap_accessible_reg, }; -static int sunxi_sram_probe(struct platform_device *pdev) +static int __init sunxi_sram_probe(struct platform_device *pdev) { - struct dentry *d; - struct regmap *emac_clock; const struct sunxi_sramc_variant *variant; + struct device *dev = &pdev->dev; + struct regmap *regmap; sram_dev = &pdev->dev; @@ -341,24 +347,21 @@ static int sunxi_sram_probe(struct platform_device *pdev) if (!variant) return -EINVAL; + dev_set_drvdata(dev, (struct sunxi_sramc_variant *)variant); + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); - of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); - - d = debugfs_create_file("sram", S_IRUGO, NULL, NULL, - &sunxi_sram_fops); - if (!d) - return -ENOMEM; + if (variant->num_emac_clocks || variant->has_ldo_ctrl) { + regmap = devm_regmap_init_mmio(dev, base, &sunxi_sram_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + } - if (variant->num_emac_clocks > 0) { - emac_clock = devm_regmap_init_mmio(&pdev->dev, base, - &sunxi_sram_emac_clock_regmap); + of_platform_populate(dev->of_node, NULL, NULL, dev); - if (IS_ERR(emac_clock)) - return PTR_ERR(emac_clock); - } + debugfs_create_file("sram", 0444, NULL, NULL, &sunxi_sram_fops); return 0; } @@ -384,6 +387,10 @@ static const struct of_device_id sunxi_sram_dt_match[] = { .compatible = "allwinner,sun8i-h3-system-control", .data = &sun8i_h3_sramc_variant, }, + { + .compatible = "allwinner,sun20i-d1-system-control", + .data = &sun20i_d1_sramc_variant, + }, { .compatible = "allwinner,sun50i-a64-sram-controller", .data = &sun50i_a64_sramc_variant, @@ -409,9 +416,8 @@ static struct platform_driver sunxi_sram_driver = { .name = "sunxi-sram", .of_match_table = sunxi_sram_dt_match, }, - .probe = sunxi_sram_probe, }; -module_platform_driver(sunxi_sram_driver); +builtin_platform_driver_probe(sunxi_sram_driver, sunxi_sram_probe); MODULE_AUTHOR("Maxime Ripard "); MODULE_DESCRIPTION("Allwinner sunXi SRAM Controller Driver"); diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig index 5725c8ef0406af29d208a0a09de7e67e5b65d223..d1ecadffa1bba5070e1fb456ce4c41fa34c415b6 100644 --- a/drivers/soc/tegra/Kconfig +++ b/drivers/soc/tegra/Kconfig @@ -136,7 +136,6 @@ config SOC_TEGRA_FUSE def_bool y depends on ARCH_TEGRA select SOC_BUS - select TEGRA20_APB_DMA if ARCH_TEGRA_2x_SOC config SOC_TEGRA_FLOWCTRL bool @@ -162,3 +161,12 @@ config SOC_TEGRA30_VOLTAGE_COUPLER bool "Voltage scaling support for Tegra30 SoCs" depends on ARCH_TEGRA_3x_SOC || COMPILE_TEST depends on REGULATOR + +config SOC_TEGRA_CBB + tristate "Tegra driver to handle error from CBB" + depends on ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC + default y + help + Support for handling error from Tegra Control Backbone(CBB). + This driver handles the errors from CBB and prints debug + information about the failed transactions. diff --git a/drivers/soc/tegra/Makefile b/drivers/soc/tegra/Makefile index 054e862b63d8928d364816b31c253eb142baa365..d722f512dc9d38d0ae8b608a8e19ccc9bc0fc8cc 100644 --- a/drivers/soc/tegra/Makefile +++ b/drivers/soc/tegra/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-y += fuse/ +obj-y += cbb/ obj-y += common.o obj-$(CONFIG_SOC_TEGRA_FLOWCTRL) += flowctrl.o diff --git a/drivers/soc/tegra/cbb/Makefile b/drivers/soc/tegra/cbb/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e3ac6cdddf5c9e84feee345c033d1014f2079f31 --- /dev/null +++ b/drivers/soc/tegra/cbb/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Control Backbone Driver code. +# +ifdef CONFIG_SOC_TEGRA_CBB +obj-y += tegra-cbb.o +obj-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra194-cbb.o +obj-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra234-cbb.o +endif diff --git a/drivers/soc/tegra/cbb/tegra-cbb.c b/drivers/soc/tegra/cbb/tegra-cbb.c new file mode 100644 index 0000000000000000000000000000000000000000..d200937353c7ca970799852af06ba07381d9b5a4 --- /dev/null +++ b/drivers/soc/tegra/cbb/tegra-cbb.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void tegra_cbb_print_err(struct seq_file *file, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + if (file) { + seq_vprintf(file, fmt, args); + } else { + vaf.fmt = fmt; + vaf.va = &args; + pr_crit("%pV", &vaf); + } + + va_end(args); +} + +void tegra_cbb_print_cache(struct seq_file *file, u32 cache) +{ + const char *buff_str, *mod_str, *rd_str, *wr_str; + + buff_str = (cache & BIT(0)) ? "Bufferable " : ""; + mod_str = (cache & BIT(1)) ? "Modifiable " : ""; + rd_str = (cache & BIT(2)) ? "Read-Allocate " : ""; + wr_str = (cache & BIT(3)) ? "Write-Allocate" : ""; + + if (cache == 0x0) + buff_str = "Device Non-Bufferable"; + + tegra_cbb_print_err(file, "\t Cache\t\t\t: 0x%x -- %s%s%s%s\n", + cache, buff_str, mod_str, rd_str, wr_str); +} + +void tegra_cbb_print_prot(struct seq_file *file, u32 prot) +{ + const char *data_str, *secure_str, *priv_str; + + data_str = (prot & 0x4) ? "Instruction" : "Data"; + secure_str = (prot & 0x2) ? "Non-Secure" : "Secure"; + priv_str = (prot & 0x1) ? "Privileged" : "Unprivileged"; + + tegra_cbb_print_err(file, "\t Protection\t\t: 0x%x -- %s, %s, %s Access\n", + prot, priv_str, secure_str, data_str); +} + +static int tegra_cbb_err_show(struct seq_file *file, void *data) +{ + struct tegra_cbb *cbb = file->private; + + return cbb->ops->debugfs_show(cbb, file, data); +} + +static int tegra_cbb_err_open(struct inode *inode, struct file *file) +{ + return single_open(file, tegra_cbb_err_show, inode->i_private); +} + +static const struct file_operations tegra_cbb_err_fops = { + .open = tegra_cbb_err_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release +}; + +static int tegra_cbb_err_debugfs_init(struct tegra_cbb *cbb) +{ + static struct dentry *root; + + if (!root) { + root = debugfs_create_file("tegra_cbb_err", 0444, NULL, cbb, &tegra_cbb_err_fops); + if (IS_ERR_OR_NULL(root)) { + pr_err("%s(): could not create debugfs node\n", __func__); + return PTR_ERR(root); + } + } + + return 0; +} + +void tegra_cbb_stall_enable(struct tegra_cbb *cbb) +{ + if (cbb->ops->stall_enable) + cbb->ops->stall_enable(cbb); +} + +void tegra_cbb_fault_enable(struct tegra_cbb *cbb) +{ + if (cbb->ops->fault_enable) + cbb->ops->fault_enable(cbb); +} + +void tegra_cbb_error_clear(struct tegra_cbb *cbb) +{ + if (cbb->ops->error_clear) + cbb->ops->error_clear(cbb); +} + +u32 tegra_cbb_get_status(struct tegra_cbb *cbb) +{ + if (cbb->ops->get_status) + return cbb->ops->get_status(cbb); + + return 0; +} + +int tegra_cbb_get_irq(struct platform_device *pdev, unsigned int *nonsec_irq, + unsigned int *sec_irq) +{ + unsigned int index = 0; + int num_intr = 0, irq; + + num_intr = platform_irq_count(pdev); + if (!num_intr) + return -EINVAL; + + if (num_intr == 2) { + irq = platform_get_irq(pdev, index); + if (irq <= 0) { + dev_err(&pdev->dev, "failed to get non-secure IRQ: %d\n", irq); + return -ENOENT; + } + + *nonsec_irq = irq; + index++; + } + + irq = platform_get_irq(pdev, index); + if (irq <= 0) { + dev_err(&pdev->dev, "failed to get secure IRQ: %d\n", irq); + return -ENOENT; + } + + *sec_irq = irq; + + if (num_intr == 1) + dev_dbg(&pdev->dev, "secure IRQ: %u\n", *sec_irq); + + if (num_intr == 2) + dev_dbg(&pdev->dev, "secure IRQ: %u, non-secure IRQ: %u\n", *sec_irq, *nonsec_irq); + + return 0; +} + +int tegra_cbb_register(struct tegra_cbb *cbb) +{ + int ret; + + if (IS_ENABLED(CONFIG_DEBUG_FS)) { + ret = tegra_cbb_err_debugfs_init(cbb); + if (ret) { + dev_err(cbb->dev, "failed to create debugfs\n"); + return ret; + } + } + + /* register interrupt handler for errors due to different initiators */ + ret = cbb->ops->interrupt_enable(cbb); + if (ret < 0) { + dev_err(cbb->dev, "Failed to register CBB Interrupt ISR"); + return ret; + } + + cbb->ops->error_enable(cbb); + dsb(sy); + + return 0; +} diff --git a/drivers/soc/tegra/cbb/tegra194-cbb.c b/drivers/soc/tegra/cbb/tegra194-cbb.c new file mode 100644 index 0000000000000000000000000000000000000000..1ae0bd9a1ac1b5a1ce6ffe5588c602ad32d6e839 --- /dev/null +++ b/drivers/soc/tegra/cbb/tegra194-cbb.c @@ -0,0 +1,2364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved + * + * The driver handles Error's from Control Backbone(CBB) generated due to + * illegal accesses. When an error is reported from a NOC within CBB, + * the driver checks ErrVld status of all three Error Logger's of that NOC. + * It then prints debug information about failed transaction using ErrLog + * registers of error logger which has ErrVld set. Currently, SLV, DEC, + * TMO, SEC, UNS are the codes which are supported by CBB. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ERRLOGGER_0_ID_COREID_0 0x00000000 +#define ERRLOGGER_0_ID_REVISIONID_0 0x00000004 +#define ERRLOGGER_0_FAULTEN_0 0x00000008 +#define ERRLOGGER_0_ERRVLD_0 0x0000000c +#define ERRLOGGER_0_ERRCLR_0 0x00000010 +#define ERRLOGGER_0_ERRLOG0_0 0x00000014 +#define ERRLOGGER_0_ERRLOG1_0 0x00000018 +#define ERRLOGGER_0_RSVD_00_0 0x0000001c +#define ERRLOGGER_0_ERRLOG3_0 0x00000020 +#define ERRLOGGER_0_ERRLOG4_0 0x00000024 +#define ERRLOGGER_0_ERRLOG5_0 0x00000028 +#define ERRLOGGER_0_STALLEN_0 0x00000038 + +#define ERRLOGGER_1_ID_COREID_0 0x00000080 +#define ERRLOGGER_1_ID_REVISIONID_0 0x00000084 +#define ERRLOGGER_1_FAULTEN_0 0x00000088 +#define ERRLOGGER_1_ERRVLD_0 0x0000008c +#define ERRLOGGER_1_ERRCLR_0 0x00000090 +#define ERRLOGGER_1_ERRLOG0_0 0x00000094 +#define ERRLOGGER_1_ERRLOG1_0 0x00000098 +#define ERRLOGGER_1_RSVD_00_0 0x0000009c +#define ERRLOGGER_1_ERRLOG3_0 0x000000a0 +#define ERRLOGGER_1_ERRLOG4_0 0x000000a4 +#define ERRLOGGER_1_ERRLOG5_0 0x000000a8 +#define ERRLOGGER_1_STALLEN_0 0x000000b8 + +#define ERRLOGGER_2_ID_COREID_0 0x00000100 +#define ERRLOGGER_2_ID_REVISIONID_0 0x00000104 +#define ERRLOGGER_2_FAULTEN_0 0x00000108 +#define ERRLOGGER_2_ERRVLD_0 0x0000010c +#define ERRLOGGER_2_ERRCLR_0 0x00000110 +#define ERRLOGGER_2_ERRLOG0_0 0x00000114 +#define ERRLOGGER_2_ERRLOG1_0 0x00000118 +#define ERRLOGGER_2_RSVD_00_0 0x0000011c +#define ERRLOGGER_2_ERRLOG3_0 0x00000120 +#define ERRLOGGER_2_ERRLOG4_0 0x00000124 +#define ERRLOGGER_2_ERRLOG5_0 0x00000128 +#define ERRLOGGER_2_STALLEN_0 0x00000138 + +#define CBB_NOC_INITFLOW GENMASK(23, 20) +#define CBB_NOC_TARGFLOW GENMASK(19, 16) +#define CBB_NOC_TARG_SUBRANGE GENMASK(15, 9) +#define CBB_NOC_SEQID GENMASK(8, 0) + +#define BPMP_NOC_INITFLOW GENMASK(20, 18) +#define BPMP_NOC_TARGFLOW GENMASK(17, 13) +#define BPMP_NOC_TARG_SUBRANGE GENMASK(12, 9) +#define BPMP_NOC_SEQID GENMASK(8, 0) + +#define AON_NOC_INITFLOW GENMASK(22, 21) +#define AON_NOC_TARGFLOW GENMASK(20, 15) +#define AON_NOC_TARG_SUBRANGE GENMASK(14, 9) +#define AON_NOC_SEQID GENMASK(8, 0) + +#define SCE_NOC_INITFLOW GENMASK(21, 19) +#define SCE_NOC_TARGFLOW GENMASK(18, 14) +#define SCE_NOC_TARG_SUBRANGE GENMASK(13, 9) +#define SCE_NOC_SEQID GENMASK(8, 0) + +#define CBB_NOC_AXCACHE GENMASK(3, 0) +#define CBB_NOC_NON_MOD GENMASK(4, 4) +#define CBB_NOC_AXPROT GENMASK(7, 5) +#define CBB_NOC_FALCONSEC GENMASK(9, 8) +#define CBB_NOC_GRPSEC GENMASK(16, 10) +#define CBB_NOC_VQC GENMASK(18, 17) +#define CBB_NOC_MSTR_ID GENMASK(22, 19) +#define CBB_NOC_AXI_ID GENMASK(30, 23) + +#define CLUSTER_NOC_AXCACHE GENMASK(3, 0) +#define CLUSTER_NOC_AXPROT GENMASK(6, 4) +#define CLUSTER_NOC_FALCONSEC GENMASK(8, 7) +#define CLUSTER_NOC_GRPSEC GENMASK(15, 9) +#define CLUSTER_NOC_VQC GENMASK(17, 16) +#define CLUSTER_NOC_MSTR_ID GENMASK(21, 18) + +#define USRBITS_MSTR_ID GENMASK(21, 18) + +#define CBB_ERR_OPC GENMASK(4, 1) +#define CBB_ERR_ERRCODE GENMASK(10, 8) +#define CBB_ERR_LEN1 GENMASK(27, 16) + +#define DMAAPB_X_RAW_INTERRUPT_STATUS 0x2ec + +struct tegra194_cbb_packet_header { + bool lock; // [0] + u8 opc; // [4:1] + u8 errcode; // [10:8]= RD, RDW, RDL, RDX, WR, WRW, WRC, PRE, URG + u16 len1; // [27:16] + bool format; // [31] = 1 -> FlexNoC versions 2.7 & above +}; + +struct tegra194_cbb_aperture { + u8 initflow; + u8 targflow; + u8 targ_subrange; + u8 init_mapping; + u32 init_localaddress; + u8 targ_mapping; + u32 targ_localaddress; + u16 seqid; +}; + +struct tegra194_cbb_userbits { + u8 axcache; + u8 non_mod; + u8 axprot; + u8 falconsec; + u8 grpsec; + u8 vqc; + u8 mstr_id; + u8 axi_id; +}; + +struct tegra194_cbb_noc_data { + const char *name; + bool erd_mask_inband_err; + const char * const *master_id; + unsigned int max_aperture; + const struct tegra194_cbb_aperture *noc_aperture; + const char * const *routeid_initflow; + const char * const *routeid_targflow; + void (*parse_routeid)(struct tegra194_cbb_aperture *info, u64 routeid); + void (*parse_userbits)(struct tegra194_cbb_userbits *usrbits, u32 elog_5); +}; + +struct tegra194_axi2apb_bridge { + struct resource res; + void __iomem *base; +}; + +struct tegra194_cbb { + struct tegra_cbb base; + + const struct tegra194_cbb_noc_data *noc; + struct resource *res; + + void __iomem *regs; + unsigned int num_intr; + unsigned int sec_irq; + unsigned int nonsec_irq; + u32 errlog0; + u32 errlog1; + u32 errlog2; + u32 errlog3; + u32 errlog4; + u32 errlog5; + + struct tegra194_axi2apb_bridge *bridges; + unsigned int num_bridges; +}; + +static inline struct tegra194_cbb *to_tegra194_cbb(struct tegra_cbb *cbb) +{ + return container_of(cbb, struct tegra194_cbb, base); +} + +static LIST_HEAD(cbb_list); +static DEFINE_SPINLOCK(cbb_lock); + +static const char * const tegra194_cbb_trantype[] = { + "RD - Read, Incrementing", + "RDW - Read, Wrap", /* Not Supported */ + "RDX - Exclusive Read", /* Not Supported */ + "RDL - Linked Read", /* Not Supported */ + "WR - Write, Incrementing", + "WRW - Write, Wrap", /* Not Supported */ + "WRC - Exclusive Write", /* Not Supported */ + "PRE - Preamble Sequence for Fixed Accesses" +}; + +static const char * const tegra194_axi2apb_error[] = { + "SFIFONE - Status FIFO Not Empty interrupt", + "SFIFOF - Status FIFO Full interrupt", + "TIM - Timer(Timeout) interrupt", + "SLV - SLVERR interrupt", + "NULL", + "ERBF - Early response buffer Full interrupt", + "NULL", + "RDFIFOF - Read Response FIFO Full interrupt", + "WRFIFOF - Write Response FIFO Full interrupt", + "CH0DFIFOF - Ch0 Data FIFO Full interrupt", + "CH1DFIFOF - Ch1 Data FIFO Full interrupt", + "CH2DFIFOF - Ch2 Data FIFO Full interrupt", + "UAT - Unsupported alignment type error", + "UBS - Unsupported burst size error", + "UBE - Unsupported Byte Enable error", + "UBT - Unsupported burst type error", + "BFS - Block Firewall security error", + "ARFS - Address Range Firewall security error", + "CH0RFIFOF - Ch0 Request FIFO Full interrupt", + "CH1RFIFOF - Ch1 Request FIFO Full interrupt", + "CH2RFIFOF - Ch2 Request FIFO Full interrupt" +}; + +static const char * const tegra194_master_id[] = { + [0x0] = "CCPLEX", + [0x1] = "CCPLEX_DPMU", + [0x2] = "BPMP", + [0x3] = "AON", + [0x4] = "SCE", + [0x5] = "GPCDMA_PERIPHERAL", + [0x6] = "TSECA", + [0x7] = "TSECB", + [0x8] = "JTAGM_DFT", + [0x9] = "CORESIGHT_AXIAP", + [0xa] = "APE", + [0xb] = "PEATR", + [0xc] = "NVDEC", + [0xd] = "RCE", + [0xe] = "NVDEC1" +}; + +static const struct tegra_cbb_error tegra194_cbb_errors[] = { + { + .code = "SLV", + .source = "Target", + .desc = "Target error detected by CBB slave" + }, { + .code = "DEC", + .source = "Initiator NIU", + .desc = "Address decode error" + }, { + .code = "UNS", + .source = "Target NIU", + .desc = "Unsupported request. Not a valid transaction" + }, { + .code = "DISC", /* Not Supported by CBB */ + .source = "Power Disconnect", + .desc = "Disconnected target or domain" + }, { + .code = "SEC", + .source = "Initiator NIU or Firewall", + .desc = "Security violation. Firewall error" + }, { + .code = "HIDE", /* Not Supported by CBB */ + .source = "Firewall", + .desc = "Hidden security violation, reported as OK to initiator" + }, { + .code = "TMO", + .source = "Target NIU", + .desc = "Target time-out error" + }, { + .code = "RSV", + .source = "None", + .desc = "Reserved" + } +}; + +/* + * CBB NOC aperture lookup table as per file "cbb_central_noc_Structure.info". + */ +static const char * const tegra194_cbbcentralnoc_routeid_initflow[] = { + [0x0] = "aon_p2ps/I/aon", + [0x1] = "ape_p2ps/I/ape_p2ps", + [0x2] = "bpmp_p2ps/I/bpmp_p2ps", + [0x3] = "ccroc_p2ps/I/ccroc_p2ps", + [0x4] = "csite_p2ps/I/0", + [0x5] = "gpcdma_mmio_p2ps/I/0", + [0x6] = "jtag_p2ps/I/0", + [0x7] = "nvdec1_p2ps/I/0", + [0x8] = "nvdec_p2ps/I/0", + [0x9] = "rce_p2ps/I/rce_p2ps", + [0xa] = "sce_p2ps/I/sce_p2ps", + [0xb] = "tseca_p2ps/I/0", + [0xc] = "tsecb_p2ps/I/0", + [0xd] = "RESERVED", + [0xe] = "RESERVED", + [0xf] = "RESERVED" +}; + +static const char * const tegra194_cbbcentralnoc_routeid_targflow[] = { + [0x0] = "SVC/T/intreg", + [0x1] = "axis_satellite_axi2apb_p2pm/T/axis_satellite_axi2apb_p2pm", + [0x2] = "axis_satellite_grout/T/axis_satellite_grout", + [0x3] = "cbb_firewall/T/cbb_firewall", + [0x4] = "gpu_p2pm/T/gpu_p2pm", + [0x5] = "host1x_p2pm/T/host1x_p2pm", + [0x6] = "sapb_3_p2pm/T/sapb_3_p2pm", + [0x7] = "smmu0_p2pm/T/smmu0_p2pm", + [0x8] = "smmu1_p2pm/T/smmu1_p2pm", + [0x9] = "smmu2_p2pm/T/smmu2_p2pm", + [0xa] = "stm_p2pm/T/stm_p2pm", + [0xb] = "RESERVED", + [0xc] = "RESERVED", + [0xd] = "RESERVED", + [0xe] = "RESERVED", + [0xf] = "RESERVED" +}; + +/* + * Fields of CBB NOC lookup table: + * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress, + * Targ mapping, Targ localAddress + * ---------------------------------------------------------------------------- + */ +static const struct tegra194_cbb_aperture tegra194_cbbcentralnoc_apert_lookup[] = { + { 0x0, 0x0, 0x00, 0x0, 0x02300000, 0, 0x00000000 }, + { 0x0, 0x1, 0x00, 0x0, 0x02003000, 0, 0x02003000 }, + { 0x0, 0x1, 0x01, 0x0, 0x02006000, 2, 0x02006000 }, + { 0x0, 0x1, 0x02, 0x0, 0x02016000, 3, 0x02016000 }, + { 0x0, 0x1, 0x03, 0x0, 0x0201d000, 4, 0x0201d000 }, + { 0x0, 0x1, 0x04, 0x0, 0x0202b000, 6, 0x0202b000 }, + { 0x0, 0x1, 0x05, 0x0, 0x02434000, 20, 0x02434000 }, + { 0x0, 0x1, 0x06, 0x0, 0x02436000, 21, 0x02436000 }, + { 0x0, 0x1, 0x07, 0x0, 0x02438000, 22, 0x02438000 }, + { 0x0, 0x1, 0x08, 0x0, 0x02445000, 24, 0x02445000 }, + { 0x0, 0x1, 0x09, 0x0, 0x02446000, 25, 0x02446000 }, + { 0x0, 0x1, 0x0a, 0x0, 0x02004000, 1, 0x02004000 }, + { 0x0, 0x1, 0x0b, 0x0, 0x0201e000, 5, 0x0201e000 }, + { 0x0, 0x1, 0x0c, 0x0, 0x0202c000, 7, 0x0202c000 }, + { 0x0, 0x1, 0x0d, 0x0, 0x02204000, 8, 0x02204000 }, + { 0x0, 0x1, 0x0e, 0x0, 0x02214000, 9, 0x02214000 }, + { 0x0, 0x1, 0x0f, 0x0, 0x02224000, 10, 0x02224000 }, + { 0x0, 0x1, 0x10, 0x0, 0x02234000, 11, 0x02234000 }, + { 0x0, 0x1, 0x11, 0x0, 0x02244000, 12, 0x02244000 }, + { 0x0, 0x1, 0x12, 0x0, 0x02254000, 13, 0x02254000 }, + { 0x0, 0x1, 0x13, 0x0, 0x02264000, 14, 0x02264000 }, + { 0x0, 0x1, 0x14, 0x0, 0x02274000, 15, 0x02274000 }, + { 0x0, 0x1, 0x15, 0x0, 0x02284000, 16, 0x02284000 }, + { 0x0, 0x1, 0x16, 0x0, 0x0243a000, 23, 0x0243a000 }, + { 0x0, 0x1, 0x17, 0x0, 0x02370000, 17, 0x02370000 }, + { 0x0, 0x1, 0x18, 0x0, 0x023d0000, 18, 0x023d0000 }, + { 0x0, 0x1, 0x19, 0x0, 0x023e0000, 19, 0x023e0000 }, + { 0x0, 0x1, 0x1a, 0x0, 0x02450000, 26, 0x02450000 }, + { 0x0, 0x1, 0x1b, 0x0, 0x02460000, 27, 0x02460000 }, + { 0x0, 0x1, 0x1c, 0x0, 0x02490000, 28, 0x02490000 }, + { 0x0, 0x1, 0x1d, 0x0, 0x03130000, 31, 0x03130000 }, + { 0x0, 0x1, 0x1e, 0x0, 0x03160000, 32, 0x03160000 }, + { 0x0, 0x1, 0x1f, 0x0, 0x03270000, 33, 0x03270000 }, + { 0x0, 0x1, 0x20, 0x0, 0x032e0000, 35, 0x032e0000 }, + { 0x0, 0x1, 0x21, 0x0, 0x03300000, 36, 0x03300000 }, + { 0x0, 0x1, 0x22, 0x0, 0x13090000, 40, 0x13090000 }, + { 0x0, 0x1, 0x23, 0x0, 0x20120000, 43, 0x20120000 }, + { 0x0, 0x1, 0x24, 0x0, 0x20170000, 44, 0x20170000 }, + { 0x0, 0x1, 0x25, 0x0, 0x20190000, 45, 0x20190000 }, + { 0x0, 0x1, 0x26, 0x0, 0x201b0000, 46, 0x201b0000 }, + { 0x0, 0x1, 0x27, 0x0, 0x20250000, 47, 0x20250000 }, + { 0x0, 0x1, 0x28, 0x0, 0x20260000, 48, 0x20260000 }, + { 0x0, 0x1, 0x29, 0x0, 0x20420000, 49, 0x20420000 }, + { 0x0, 0x1, 0x2a, 0x0, 0x20460000, 50, 0x20460000 }, + { 0x0, 0x1, 0x2b, 0x0, 0x204f0000, 51, 0x204f0000 }, + { 0x0, 0x1, 0x2c, 0x0, 0x20520000, 52, 0x20520000 }, + { 0x0, 0x1, 0x2d, 0x0, 0x20580000, 53, 0x20580000 }, + { 0x0, 0x1, 0x2e, 0x0, 0x205a0000, 54, 0x205a0000 }, + { 0x0, 0x1, 0x2f, 0x0, 0x205c0000, 55, 0x205c0000 }, + { 0x0, 0x1, 0x30, 0x0, 0x20690000, 56, 0x20690000 }, + { 0x0, 0x1, 0x31, 0x0, 0x20770000, 57, 0x20770000 }, + { 0x0, 0x1, 0x32, 0x0, 0x20790000, 58, 0x20790000 }, + { 0x0, 0x1, 0x33, 0x0, 0x20880000, 59, 0x20880000 }, + { 0x0, 0x1, 0x34, 0x0, 0x20990000, 62, 0x20990000 }, + { 0x0, 0x1, 0x35, 0x0, 0x20e10000, 65, 0x20e10000 }, + { 0x0, 0x1, 0x36, 0x0, 0x20e70000, 66, 0x20e70000 }, + { 0x0, 0x1, 0x37, 0x0, 0x20e80000, 67, 0x20e80000 }, + { 0x0, 0x1, 0x38, 0x0, 0x20f30000, 68, 0x20f30000 }, + { 0x0, 0x1, 0x39, 0x0, 0x20f50000, 69, 0x20f50000 }, + { 0x0, 0x1, 0x3a, 0x0, 0x20fc0000, 70, 0x20fc0000 }, + { 0x0, 0x1, 0x3b, 0x0, 0x21110000, 72, 0x21110000 }, + { 0x0, 0x1, 0x3c, 0x0, 0x21270000, 73, 0x21270000 }, + { 0x0, 0x1, 0x3d, 0x0, 0x21290000, 74, 0x21290000 }, + { 0x0, 0x1, 0x3e, 0x0, 0x21840000, 75, 0x21840000 }, + { 0x0, 0x1, 0x3f, 0x0, 0x21880000, 76, 0x21880000 }, + { 0x0, 0x1, 0x40, 0x0, 0x218d0000, 77, 0x218d0000 }, + { 0x0, 0x1, 0x41, 0x0, 0x21950000, 78, 0x21950000 }, + { 0x0, 0x1, 0x42, 0x0, 0x21960000, 79, 0x21960000 }, + { 0x0, 0x1, 0x43, 0x0, 0x21a10000, 80, 0x21a10000 }, + { 0x0, 0x1, 0x44, 0x0, 0x024a0000, 29, 0x024a0000 }, + { 0x0, 0x1, 0x45, 0x0, 0x024c0000, 30, 0x024c0000 }, + { 0x0, 0x1, 0x46, 0x0, 0x032c0000, 34, 0x032c0000 }, + { 0x0, 0x1, 0x47, 0x0, 0x03400000, 37, 0x03400000 }, + { 0x0, 0x1, 0x48, 0x0, 0x130a0000, 41, 0x130a0000 }, + { 0x0, 0x1, 0x49, 0x0, 0x130c0000, 42, 0x130c0000 }, + { 0x0, 0x1, 0x4a, 0x0, 0x208a0000, 60, 0x208a0000 }, + { 0x0, 0x1, 0x4b, 0x0, 0x208c0000, 61, 0x208c0000 }, + { 0x0, 0x1, 0x4c, 0x0, 0x209a0000, 63, 0x209a0000 }, + { 0x0, 0x1, 0x4d, 0x0, 0x21a40000, 81, 0x21a40000 }, + { 0x0, 0x1, 0x4e, 0x0, 0x03440000, 38, 0x03440000 }, + { 0x0, 0x1, 0x4f, 0x0, 0x20d00000, 64, 0x20d00000 }, + { 0x0, 0x1, 0x50, 0x0, 0x21000000, 71, 0x21000000 }, + { 0x0, 0x1, 0x51, 0x0, 0x0b000000, 39, 0x0b000000 }, + { 0x0, 0x2, 0x00, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x0, 0x3, 0x00, 0x0, 0x02340000, 0, 0x00000000 }, + { 0x0, 0x4, 0x00, 0x0, 0x17000000, 0, 0x17000000 }, + { 0x0, 0x4, 0x01, 0x0, 0x18000000, 1, 0x18000000 }, + { 0x0, 0x5, 0x00, 0x0, 0x13e80000, 1, 0x13e80000 }, + { 0x0, 0x5, 0x01, 0x0, 0x15810000, 12, 0x15810000 }, + { 0x0, 0x5, 0x02, 0x0, 0x15840000, 14, 0x15840000 }, + { 0x0, 0x5, 0x03, 0x0, 0x15a40000, 17, 0x15a40000 }, + { 0x0, 0x5, 0x04, 0x0, 0x13f00000, 3, 0x13f00000 }, + { 0x0, 0x5, 0x05, 0x0, 0x15820000, 13, 0x15820000 }, + { 0x0, 0x5, 0x06, 0x0, 0x13ec0000, 2, 0x13ec0000 }, + { 0x0, 0x5, 0x07, 0x0, 0x15200000, 6, 0x15200000 }, + { 0x0, 0x5, 0x08, 0x0, 0x15340000, 7, 0x15340000 }, + { 0x0, 0x5, 0x09, 0x0, 0x15380000, 8, 0x15380000 }, + { 0x0, 0x5, 0x0a, 0x0, 0x15500000, 10, 0x15500000 }, + { 0x0, 0x5, 0x0b, 0x0, 0x155c0000, 11, 0x155c0000 }, + { 0x0, 0x5, 0x0c, 0x0, 0x15a00000, 16, 0x15a00000 }, + { 0x0, 0x5, 0x0d, 0x0, 0x13e00000, 0, 0x13e00000 }, + { 0x0, 0x5, 0x0e, 0x0, 0x15100000, 5, 0x15100000 }, + { 0x0, 0x5, 0x0f, 0x0, 0x15480000, 9, 0x15480000 }, + { 0x0, 0x5, 0x10, 0x0, 0x15880000, 15, 0x15880000 }, + { 0x0, 0x5, 0x11, 0x0, 0x15a80000, 18, 0x15a80000 }, + { 0x0, 0x5, 0x12, 0x0, 0x15b00000, 19, 0x15b00000 }, + { 0x0, 0x5, 0x13, 0x0, 0x14800000, 4, 0x14800000 }, + { 0x0, 0x5, 0x14, 0x0, 0x15c00000, 20, 0x15c00000 }, + { 0x0, 0x5, 0x15, 0x0, 0x16000000, 21, 0x16000000 }, + { 0x0, 0x6, 0x00, 0x0, 0x02000000, 4, 0x02000000 }, + { 0x0, 0x6, 0x01, 0x0, 0x02007000, 5, 0x02007000 }, + { 0x0, 0x6, 0x02, 0x0, 0x02008000, 6, 0x02008000 }, + { 0x0, 0x6, 0x03, 0x0, 0x02013000, 7, 0x02013000 }, + { 0x0, 0x6, 0x04, 0x0, 0x0201c000, 8, 0x0201c000 }, + { 0x0, 0x6, 0x05, 0x0, 0x02020000, 9, 0x02020000 }, + { 0x0, 0x6, 0x06, 0x0, 0x0202a000, 10, 0x0202a000 }, + { 0x0, 0x6, 0x07, 0x0, 0x0202e000, 11, 0x0202e000 }, + { 0x0, 0x6, 0x08, 0x0, 0x06400000, 33, 0x06400000 }, + { 0x0, 0x6, 0x09, 0x0, 0x02038000, 12, 0x02038000 }, + { 0x0, 0x6, 0x0a, 0x0, 0x00100000, 0, 0x00100000 }, + { 0x0, 0x6, 0x0b, 0x0, 0x023b0000, 13, 0x023b0000 }, + { 0x0, 0x6, 0x0c, 0x0, 0x02800000, 16, 0x02800000 }, + { 0x0, 0x6, 0x0d, 0x0, 0x030e0000, 22, 0x030e0000 }, + { 0x0, 0x6, 0x0e, 0x0, 0x03800000, 23, 0x03800000 }, + { 0x0, 0x6, 0x0f, 0x0, 0x03980000, 25, 0x03980000 }, + { 0x0, 0x6, 0x10, 0x0, 0x03a60000, 26, 0x03a60000 }, + { 0x0, 0x6, 0x11, 0x0, 0x03d80000, 31, 0x03d80000 }, + { 0x0, 0x6, 0x12, 0x0, 0x20000000, 36, 0x20000000 }, + { 0x0, 0x6, 0x13, 0x0, 0x20050000, 38, 0x20050000 }, + { 0x0, 0x6, 0x14, 0x0, 0x201e0000, 40, 0x201e0000 }, + { 0x0, 0x6, 0x15, 0x0, 0x20280000, 42, 0x20280000 }, + { 0x0, 0x6, 0x16, 0x0, 0x202c0000, 43, 0x202c0000 }, + { 0x0, 0x6, 0x17, 0x0, 0x20390000, 44, 0x20390000 }, + { 0x0, 0x6, 0x18, 0x0, 0x20430000, 45, 0x20430000 }, + { 0x0, 0x6, 0x19, 0x0, 0x20440000, 46, 0x20440000 }, + { 0x0, 0x6, 0x1a, 0x0, 0x204e0000, 47, 0x204e0000 }, + { 0x0, 0x6, 0x1b, 0x0, 0x20550000, 48, 0x20550000 }, + { 0x0, 0x6, 0x1c, 0x0, 0x20570000, 49, 0x20570000 }, + { 0x0, 0x6, 0x1d, 0x0, 0x20590000, 50, 0x20590000 }, + { 0x0, 0x6, 0x1e, 0x0, 0x20730000, 52, 0x20730000 }, + { 0x0, 0x6, 0x1f, 0x0, 0x209f0000, 54, 0x209f0000 }, + { 0x0, 0x6, 0x20, 0x0, 0x20e20000, 55, 0x20e20000 }, + { 0x0, 0x6, 0x21, 0x0, 0x20ed0000, 56, 0x20ed0000 }, + { 0x0, 0x6, 0x22, 0x0, 0x20fd0000, 57, 0x20fd0000 }, + { 0x0, 0x6, 0x23, 0x0, 0x21120000, 59, 0x21120000 }, + { 0x0, 0x6, 0x24, 0x0, 0x211a0000, 60, 0x211a0000 }, + { 0x0, 0x6, 0x25, 0x0, 0x21850000, 61, 0x21850000 }, + { 0x0, 0x6, 0x26, 0x0, 0x21860000, 62, 0x21860000 }, + { 0x0, 0x6, 0x27, 0x0, 0x21890000, 63, 0x21890000 }, + { 0x0, 0x6, 0x28, 0x0, 0x21970000, 64, 0x21970000 }, + { 0x0, 0x6, 0x29, 0x0, 0x21990000, 65, 0x21990000 }, + { 0x0, 0x6, 0x2a, 0x0, 0x21a00000, 66, 0x21a00000 }, + { 0x0, 0x6, 0x2b, 0x0, 0x21a90000, 68, 0x21a90000 }, + { 0x0, 0x6, 0x2c, 0x0, 0x21ac0000, 70, 0x21ac0000 }, + { 0x0, 0x6, 0x2d, 0x0, 0x01f80000, 3, 0x01f80000 }, + { 0x0, 0x6, 0x2e, 0x0, 0x024e0000, 14, 0x024e0000 }, + { 0x0, 0x6, 0x2f, 0x0, 0x030c0000, 21, 0x030c0000 }, + { 0x0, 0x6, 0x30, 0x0, 0x03820000, 24, 0x03820000 }, + { 0x0, 0x6, 0x31, 0x0, 0x03aa0000, 27, 0x03aa0000 }, + { 0x0, 0x6, 0x32, 0x0, 0x03c80000, 29, 0x03c80000 }, + { 0x0, 0x6, 0x33, 0x0, 0x130e0000, 34, 0x130e0000 }, + { 0x0, 0x6, 0x34, 0x0, 0x20020000, 37, 0x20020000 }, + { 0x0, 0x6, 0x35, 0x0, 0x20060000, 39, 0x20060000 }, + { 0x0, 0x6, 0x36, 0x0, 0x20200000, 41, 0x20200000 }, + { 0x0, 0x6, 0x37, 0x0, 0x206a0000, 51, 0x206a0000 }, + { 0x0, 0x6, 0x38, 0x0, 0x20740000, 53, 0x20740000 }, + { 0x0, 0x6, 0x39, 0x0, 0x20fe0000, 58, 0x20fe0000 }, + { 0x0, 0x6, 0x3a, 0x0, 0x21a20000, 67, 0x21a20000 }, + { 0x0, 0x6, 0x3b, 0x0, 0x21aa0000, 69, 0x21aa0000 }, + { 0x0, 0x6, 0x3c, 0x0, 0x02b80000, 17, 0x02b80000 }, + { 0x0, 0x6, 0x3d, 0x0, 0x03080000, 20, 0x03080000 }, + { 0x0, 0x6, 0x3e, 0x0, 0x13100000, 35, 0x13100000 }, + { 0x0, 0x6, 0x3f, 0x0, 0x01f00000, 2, 0x01f00000 }, + { 0x0, 0x6, 0x40, 0x0, 0x03000000, 19, 0x03000000 }, + { 0x0, 0x6, 0x41, 0x0, 0x03c00000, 28, 0x03c00000 }, + { 0x0, 0x6, 0x42, 0x0, 0x03d00000, 30, 0x03d00000 }, + { 0x0, 0x6, 0x43, 0x0, 0x01700000, 1, 0x01700000 }, + { 0x0, 0x6, 0x44, 0x0, 0x02c00000, 18, 0x02c00000 }, + { 0x0, 0x6, 0x45, 0x0, 0x02600000, 15, 0x02600000 }, + { 0x0, 0x6, 0x46, 0x0, 0x06000000, 32, 0x06000000 }, + { 0x0, 0x6, 0x47, 0x0, 0x24000000, 71, 0x24000000 }, + { 0x0, 0x7, 0x00, 0x0, 0x12000000, 0, 0x12000000 }, + { 0x0, 0x8, 0x00, 0x0, 0x11000000, 0, 0x11000000 }, + { 0x0, 0x9, 0x00, 0x0, 0x10000000, 0, 0x10000000 }, + { 0x0, 0xa, 0x00, 0x0, 0x22000000, 0, 0x22000000 } +}; + +/* + * BPMP NOC aperture lookup table as per file "BPMP_NOC_Structure.info". + */ +static const char * const tegra194_bpmpnoc_routeid_initflow[] = { + [0x0] = "cbb_i/I/0", + [0x1] = "cpu_m_i/I/0", + [0x2] = "cpu_p_i/I/0", + [0x3] = "cvc_i/I/0", + [0x4] = "dma_m_i/I/0", + [0x5] = "dma_p_i/I/0", + [0x6] = "RESERVED", + [0x7] = "RESERVED" +}; + +static const char * const tegra194_bpmpnoc_routeid_targflow[] = { + [0x00] = "multiport0_t/T/actmon", + [0x01] = "multiport0_t/T/ast_0", + [0x02] = "multiport0_t/T/ast_1", + [0x03] = "multiport0_t/T/atcm_cfg", + [0x04] = "multiport0_t/T/car", + [0x05] = "multiport0_t/T/central_pwr_mgr", + [0x06] = "multiport0_t/T/central_vtg_ctlr", + [0x07] = "multiport0_t/T/cfg", + [0x08] = "multiport0_t/T/dma", + [0x09] = "multiport0_t/T/err_collator", + [0x0a] = "multiport0_t/T/err_collator_car", + [0x0b] = "multiport0_t/T/fpga_misc", + [0x0c] = "multiport0_t/T/fpga_uart", + [0x0d] = "multiport0_t/T/gte", + [0x0e] = "multiport0_t/T/hsp", + [0x0f] = "multiport0_t/T/misc", + [0x10] = "multiport0_t/T/pm", + [0x11] = "multiport0_t/T/simon0", + [0x12] = "multiport0_t/T/simon1", + [0x13] = "multiport0_t/T/simon2", + [0x14] = "multiport0_t/T/simon3", + [0x15] = "multiport0_t/T/simon4", + [0x16] = "multiport0_t/T/soc_therm", + [0x17] = "multiport0_t/T/tke", + [0x18] = "multiport0_t/T/vic_0", + [0x19] = "multiport0_t/T/vic_1", + [0x1a] = "ast0_t/T/0", + [0x1b] = "ast1_t/T/0", + [0x1c] = "bpmp_noc_firewall/T/0", + [0x1d] = "cbb_t/T/0", + [0x1e] = "cpu_t/T/0", + [0x1f] = "svc_t/T/0" +}; + +/* + * Fields of BPMP NOC lookup table: + * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress, + * Targ mapping, Targ localAddress + * ---------------------------------------------------------------------------- + */ +static const struct tegra194_cbb_aperture tegra194_bpmpnoc_apert_lookup[] = { + { 0x0, 0x1c, 0x0, 0x0, 0x0d640000, 0, 0x00000000 }, + { 0x0, 0x1e, 0x0, 0x0, 0x0d400000, 0, 0x0d400000 }, + { 0x0, 0x00, 0x0, 0x0, 0x0d230000, 0, 0x00000000 }, + { 0x0, 0x01, 0x0, 0x0, 0x0d040000, 0, 0x00000000 }, + { 0x0, 0x02, 0x0, 0x0, 0x0d050000, 0, 0x00000000 }, + { 0x0, 0x03, 0x0, 0x0, 0x0d000000, 0, 0x00000000 }, + { 0x0, 0x04, 0x0, 0x0, 0x20ae0000, 3, 0x000e0000 }, + { 0x0, 0x04, 0x1, 0x0, 0x20ac0000, 2, 0x000c0000 }, + { 0x0, 0x04, 0x2, 0x0, 0x20a80000, 1, 0x00080000 }, + { 0x0, 0x04, 0x3, 0x0, 0x20a00000, 0, 0x00000000 }, + { 0x0, 0x05, 0x0, 0x0, 0x0d2a0000, 0, 0x00000000 }, + { 0x0, 0x06, 0x0, 0x0, 0x0d290000, 0, 0x00000000 }, + { 0x0, 0x07, 0x0, 0x0, 0x0d2c0000, 0, 0x00000000 }, + { 0x0, 0x08, 0x0, 0x0, 0x0d0e0000, 4, 0x00080000 }, + { 0x0, 0x08, 0x1, 0x0, 0x0d060000, 0, 0x00000000 }, + { 0x0, 0x08, 0x2, 0x0, 0x0d080000, 1, 0x00020000 }, + { 0x0, 0x08, 0x3, 0x0, 0x0d0a0000, 2, 0x00040000 }, + { 0x0, 0x08, 0x4, 0x0, 0x0d0c0000, 3, 0x00060000 }, + { 0x0, 0x09, 0x0, 0x0, 0x0d650000, 0, 0x00000000 }, + { 0x0, 0x0a, 0x0, 0x0, 0x20af0000, 0, 0x00000000 }, + { 0x0, 0x0b, 0x0, 0x0, 0x0d3e0000, 0, 0x00000000 }, + { 0x0, 0x0c, 0x0, 0x0, 0x0d3d0000, 0, 0x00000000 }, + { 0x0, 0x0d, 0x0, 0x0, 0x0d1e0000, 0, 0x00000000 }, + { 0x0, 0x0e, 0x0, 0x0, 0x0d150000, 0, 0x00000000 }, + { 0x0, 0x0e, 0x1, 0x0, 0x0d160000, 1, 0x00010000 }, + { 0x0, 0x0e, 0x2, 0x0, 0x0d170000, 2, 0x00020000 }, + { 0x0, 0x0e, 0x3, 0x0, 0x0d180000, 3, 0x00030000 }, + { 0x0, 0x0e, 0x4, 0x0, 0x0d190000, 4, 0x00040000 }, + { 0x0, 0x0e, 0x5, 0x0, 0x0d1a0000, 5, 0x00050000 }, + { 0x0, 0x0e, 0x6, 0x0, 0x0d1b0000, 6, 0x00060000 }, + { 0x0, 0x0e, 0x7, 0x0, 0x0d1c0000, 7, 0x00070000 }, + { 0x0, 0x0e, 0x8, 0x0, 0x0d1d0000, 8, 0x00080000 }, + { 0x0, 0x0f, 0x0, 0x0, 0x0d660000, 0, 0x00000000 }, + { 0x0, 0x10, 0x0, 0x0, 0x0d1f0000, 0, 0x00000000 }, + { 0x0, 0x10, 0x1, 0x0, 0x0d200000, 1, 0x00010000 }, + { 0x0, 0x10, 0x2, 0x0, 0x0d210000, 2, 0x00020000 }, + { 0x0, 0x10, 0x3, 0x0, 0x0d220000, 3, 0x00030000 }, + { 0x0, 0x11, 0x0, 0x0, 0x0d240000, 0, 0x00000000 }, + { 0x0, 0x12, 0x0, 0x0, 0x0d250000, 0, 0x00000000 }, + { 0x0, 0x13, 0x0, 0x0, 0x0d260000, 0, 0x00000000 }, + { 0x0, 0x14, 0x0, 0x0, 0x0d270000, 0, 0x00000000 }, + { 0x0, 0x15, 0x0, 0x0, 0x0d2b0000, 0, 0x00000000 }, + { 0x0, 0x16, 0x0, 0x0, 0x0d280000, 0, 0x00000000 }, + { 0x0, 0x17, 0x0, 0x0, 0x0d0f0000, 0, 0x00000000 }, + { 0x0, 0x17, 0x1, 0x0, 0x0d100000, 1, 0x00010000 }, + { 0x0, 0x17, 0x2, 0x0, 0x0d110000, 2, 0x00020000 }, + { 0x0, 0x17, 0x3, 0x0, 0x0d120000, 3, 0x00030000 }, + { 0x0, 0x17, 0x4, 0x0, 0x0d130000, 4, 0x00040000 }, + { 0x0, 0x17, 0x5, 0x0, 0x0d140000, 5, 0x00050000 }, + { 0x0, 0x18, 0x0, 0x0, 0x0d020000, 0, 0x00000000 }, + { 0x0, 0x19, 0x0, 0x0, 0x0d030000, 0, 0x00000000 }, + { 0x0, 0x1f, 0x0, 0x0, 0x0d600000, 0, 0x00000000 }, + { 0x0, 0x1f, 0x1, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x1, 0x1a, 0x0, 0x0, 0x40000000, 0, 0x40000000 }, + { 0x1, 0x1a, 0x1, 0x1, 0x80000000, 1, 0x80000000 }, + { 0x1, 0x1a, 0x2, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x2, 0x1c, 0x0, 0x0, 0x0d640000, 0, 0x00000000 }, + { 0x2, 0x1d, 0x0, 0x0, 0x20b00000, 8, 0x20b00000 }, + { 0x2, 0x1d, 0x1, 0x0, 0x20800000, 7, 0x20800000 }, + { 0x2, 0x1d, 0x2, 0x0, 0x20c00000, 9, 0x20c00000 }, + { 0x2, 0x1d, 0x3, 0x0, 0x0d800000, 3, 0x0d800000 }, + { 0x2, 0x1d, 0x4, 0x0, 0x20000000, 6, 0x20000000 }, + { 0x2, 0x1d, 0x5, 0x0, 0x0c000000, 2, 0x0c000000 }, + { 0x2, 0x1d, 0x6, 0x0, 0x21000000, 10, 0x21000000 }, + { 0x2, 0x1d, 0x7, 0x0, 0x0e000000, 4, 0x0e000000 }, + { 0x2, 0x1d, 0x8, 0x0, 0x22000000, 11, 0x22000000 }, + { 0x2, 0x1d, 0x9, 0x0, 0x08000000, 1, 0x08000000 }, + { 0x2, 0x1d, 0xa, 0x0, 0x24000000, 12, 0x24000000 }, + { 0x2, 0x1d, 0xb, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x2, 0x1d, 0xc, 0x0, 0x28000000, 13, 0x28000000 }, + { 0x2, 0x1d, 0xd, 0x0, 0x10000000, 5, 0x10000000 }, + { 0x2, 0x1d, 0xe, 0x0, 0x30000000, 14, 0x30000000 }, + { 0x2, 0x00, 0x0, 0x0, 0x0d230000, 0, 0x00000000 }, + { 0x2, 0x01, 0x0, 0x0, 0x0d040000, 0, 0x00000000 }, + { 0x2, 0x02, 0x0, 0x0, 0x0d050000, 0, 0x00000000 }, + { 0x2, 0x03, 0x0, 0x0, 0x0d000000, 0, 0x00000000 }, + { 0x2, 0x04, 0x0, 0x0, 0x20ae0000, 3, 0x000e0000 }, + { 0x2, 0x04, 0x1, 0x0, 0x20ac0000, 2, 0x000c0000 }, + { 0x2, 0x04, 0x2, 0x0, 0x20a80000, 1, 0x00080000 }, + { 0x2, 0x04, 0x3, 0x0, 0x20a00000, 0, 0x00000000 }, + { 0x2, 0x05, 0x0, 0x0, 0x0d2a0000, 0, 0x00000000 }, + { 0x2, 0x06, 0x0, 0x0, 0x0d290000, 0, 0x00000000 }, + { 0x2, 0x07, 0x0, 0x0, 0x0d2c0000, 0, 0x00000000 }, + { 0x2, 0x08, 0x0, 0x0, 0x0d0e0000, 4, 0x00080000 }, + { 0x2, 0x08, 0x1, 0x0, 0x0d060000, 0, 0x00000000 }, + { 0x2, 0x08, 0x2, 0x0, 0x0d080000, 1, 0x00020000 }, + { 0x2, 0x08, 0x3, 0x0, 0x0d0a0000, 2, 0x00040000 }, + { 0x2, 0x08, 0x4, 0x0, 0x0d0c0000, 3, 0x00060000 }, + { 0x2, 0x09, 0x0, 0x0, 0x0d650000, 0, 0x00000000 }, + { 0x2, 0x0a, 0x0, 0x0, 0x20af0000, 0, 0x00000000 }, + { 0x2, 0x0b, 0x0, 0x0, 0x0d3e0000, 0, 0x00000000 }, + { 0x2, 0x0c, 0x0, 0x0, 0x0d3d0000, 0, 0x00000000 }, + { 0x2, 0x0d, 0x0, 0x0, 0x0d1e0000, 0, 0x00000000 }, + { 0x2, 0x0e, 0x0, 0x0, 0x0d150000, 0, 0x00000000 }, + { 0x2, 0x0e, 0x1, 0x0, 0x0d160000, 1, 0x00010000 }, + { 0x2, 0x0e, 0x2, 0x0, 0x0d170000, 2, 0x00020000 }, + { 0x2, 0x0e, 0x3, 0x0, 0x0d180000, 3, 0x00030000 }, + { 0x2, 0x0e, 0x4, 0x0, 0x0d190000, 4, 0x00040000 }, + { 0x2, 0x0e, 0x5, 0x0, 0x0d1a0000, 5, 0x00050000 }, + { 0x2, 0x0e, 0x6, 0x0, 0x0d1b0000, 6, 0x00060000 }, + { 0x2, 0x0e, 0x7, 0x0, 0x0d1c0000, 7, 0x00070000 }, + { 0x2, 0x0e, 0x8, 0x0, 0x0d1d0000, 8, 0x00080000 }, + { 0x2, 0x0f, 0x0, 0x0, 0x0d660000, 0, 0x00000000 }, + { 0x2, 0x10, 0x0, 0x0, 0x0d1f0000, 0, 0x00000000 }, + { 0x2, 0x10, 0x1, 0x0, 0x0d200000, 1, 0x00010000 }, + { 0x2, 0x10, 0x2, 0x0, 0x0d210000, 2, 0x00020000 }, + { 0x2, 0x10, 0x3, 0x0, 0x0d220000, 3, 0x00030000 }, + { 0x2, 0x11, 0x0, 0x0, 0x0d240000, 0, 0x00000000 }, + { 0x2, 0x12, 0x0, 0x0, 0x0d250000, 0, 0x00000000 }, + { 0x2, 0x13, 0x0, 0x0, 0x0d260000, 0, 0x00000000 }, + { 0x2, 0x14, 0x0, 0x0, 0x0d270000, 0, 0x00000000 }, + { 0x2, 0x15, 0x0, 0x0, 0x0d2b0000, 0, 0x00000000 }, + { 0x2, 0x16, 0x0, 0x0, 0x0d280000, 0, 0x00000000 }, + { 0x2, 0x17, 0x0, 0x0, 0x0d0f0000, 0, 0x00000000 }, + { 0x2, 0x17, 0x1, 0x0, 0x0d100000, 1, 0x00010000 }, + { 0x2, 0x17, 0x2, 0x0, 0x0d110000, 2, 0x00020000 }, + { 0x2, 0x17, 0x3, 0x0, 0x0d120000, 3, 0x00030000 }, + { 0x2, 0x17, 0x4, 0x0, 0x0d130000, 4, 0x00040000 }, + { 0x2, 0x17, 0x5, 0x0, 0x0d140000, 5, 0x00050000 }, + { 0x2, 0x18, 0x0, 0x0, 0x0d020000, 0, 0x00000000 }, + { 0x2, 0x19, 0x0, 0x0, 0x0d030000, 0, 0x00000000 }, + { 0x2, 0x1f, 0x0, 0x0, 0x0d600000, 0, 0x00000000 }, + { 0x2, 0x1f, 0x1, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x1b, 0x0, 0x0, 0x40000000, 0, 0x40000000 }, + { 0x3, 0x1b, 0x1, 0x1, 0x80000000, 1, 0x80000000 }, + { 0x3, 0x1c, 0x0, 0x2, 0x0d640000, 0, 0x00000000 }, + { 0x3, 0x1d, 0x0, 0x2, 0x20b00000, 8, 0x20b00000 }, + { 0x3, 0x1d, 0x1, 0x2, 0x20800000, 7, 0x20800000 }, + { 0x3, 0x1d, 0x2, 0x2, 0x20c00000, 9, 0x20c00000 }, + { 0x3, 0x1d, 0x3, 0x2, 0x0d800000, 3, 0x0d800000 }, + { 0x3, 0x1d, 0x4, 0x2, 0x20000000, 6, 0x20000000 }, + { 0x3, 0x1d, 0x5, 0x2, 0x0c000000, 2, 0x0c000000 }, + { 0x3, 0x1d, 0x6, 0x2, 0x21000000, 10, 0x21000000 }, + { 0x3, 0x1d, 0x7, 0x2, 0x0e000000, 4, 0x0e000000 }, + { 0x3, 0x1d, 0x8, 0x2, 0x22000000, 11, 0x22000000 }, + { 0x3, 0x1d, 0x9, 0x2, 0x08000000, 1, 0x08000000 }, + { 0x3, 0x1d, 0xa, 0x2, 0x24000000, 12, 0x24000000 }, + { 0x3, 0x1d, 0xb, 0x2, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x1d, 0xc, 0x2, 0x28000000, 13, 0x28000000 }, + { 0x3, 0x1d, 0xd, 0x2, 0x10000000, 5, 0x10000000 }, + { 0x3, 0x1d, 0xe, 0x2, 0x30000000, 14, 0x30000000 }, + { 0x3, 0x1e, 0x0, 0x2, 0x0d400000, 0, 0x0d400000 }, + { 0x3, 0x00, 0x0, 0x2, 0x0d230000, 0, 0x00000000 }, + { 0x3, 0x01, 0x0, 0x2, 0x0d040000, 0, 0x00000000 }, + { 0x3, 0x02, 0x0, 0x2, 0x0d050000, 0, 0x00000000 }, + { 0x3, 0x03, 0x0, 0x2, 0x0d000000, 0, 0x00000000 }, + { 0x3, 0x04, 0x0, 0x2, 0x20ae0000, 3, 0x000e0000 }, + { 0x3, 0x04, 0x1, 0x2, 0x20ac0000, 2, 0x000c0000 }, + { 0x3, 0x04, 0x2, 0x2, 0x20a80000, 1, 0x00080000 }, + { 0x3, 0x04, 0x3, 0x2, 0x20a00000, 0, 0x00000000 }, + { 0x3, 0x05, 0x0, 0x2, 0x0d2a0000, 0, 0x00000000 }, + { 0x3, 0x06, 0x0, 0x2, 0x0d290000, 0, 0x00000000 }, + { 0x3, 0x07, 0x0, 0x2, 0x0d2c0000, 0, 0x00000000 }, + { 0x3, 0x08, 0x0, 0x2, 0x0d0e0000, 4, 0x00080000 }, + { 0x3, 0x08, 0x1, 0x2, 0x0d060000, 0, 0x00000000 }, + { 0x3, 0x08, 0x2, 0x2, 0x0d080000, 1, 0x00020000 }, + { 0x3, 0x08, 0x3, 0x2, 0x0d0a0000, 2, 0x00040000 }, + { 0x3, 0x08, 0x4, 0x2, 0x0d0c0000, 3, 0x00060000 }, + { 0x3, 0x09, 0x0, 0x2, 0x0d650000, 0, 0x00000000 }, + { 0x3, 0x0a, 0x0, 0x2, 0x20af0000, 0, 0x00000000 }, + { 0x3, 0x0b, 0x0, 0x2, 0x0d3e0000, 0, 0x00000000 }, + { 0x3, 0x0c, 0x0, 0x2, 0x0d3d0000, 0, 0x00000000 }, + { 0x3, 0x0d, 0x0, 0x2, 0x0d1e0000, 0, 0x00000000 }, + { 0x3, 0x0e, 0x0, 0x2, 0x0d150000, 0, 0x00000000 }, + { 0x3, 0x0e, 0x1, 0x2, 0x0d160000, 1, 0x00010000 }, + { 0x3, 0x0e, 0x2, 0x2, 0x0d170000, 2, 0x00020000 }, + { 0x3, 0x0e, 0x3, 0x2, 0x0d180000, 3, 0x00030000 }, + { 0x3, 0x0e, 0x4, 0x2, 0x0d190000, 4, 0x00040000 }, + { 0x3, 0x0e, 0x5, 0x2, 0x0d1a0000, 5, 0x00050000 }, + { 0x3, 0x0e, 0x6, 0x2, 0x0d1b0000, 6, 0x00060000 }, + { 0x3, 0x0e, 0x7, 0x2, 0x0d1c0000, 7, 0x00070000 }, + { 0x3, 0x0e, 0x8, 0x2, 0x0d1d0000, 8, 0x00080000 }, + { 0x3, 0x0f, 0x0, 0x2, 0x0d660000, 0, 0x00000000 }, + { 0x3, 0x10, 0x0, 0x2, 0x0d1f0000, 0, 0x00000000 }, + { 0x3, 0x10, 0x1, 0x2, 0x0d200000, 1, 0x00010000 }, + { 0x3, 0x10, 0x2, 0x2, 0x0d210000, 2, 0x00020000 }, + { 0x3, 0x10, 0x3, 0x2, 0x0d220000, 3, 0x00030000 }, + { 0x3, 0x11, 0x0, 0x2, 0x0d240000, 0, 0x00000000 }, + { 0x3, 0x12, 0x0, 0x2, 0x0d250000, 0, 0x00000000 }, + { 0x3, 0x13, 0x0, 0x2, 0x0d260000, 0, 0x00000000 }, + { 0x3, 0x14, 0x0, 0x2, 0x0d270000, 0, 0x00000000 }, + { 0x3, 0x15, 0x0, 0x2, 0x0d2b0000, 0, 0x00000000 }, + { 0x3, 0x16, 0x0, 0x2, 0x0d280000, 0, 0x00000000 }, + { 0x3, 0x17, 0x0, 0x2, 0x0d0f0000, 0, 0x00000000 }, + { 0x3, 0x17, 0x1, 0x2, 0x0d100000, 1, 0x00010000 }, + { 0x3, 0x17, 0x2, 0x2, 0x0d110000, 2, 0x00020000 }, + { 0x3, 0x17, 0x3, 0x2, 0x0d120000, 3, 0x00030000 }, + { 0x3, 0x17, 0x4, 0x2, 0x0d130000, 4, 0x00040000 }, + { 0x3, 0x17, 0x5, 0x2, 0x0d140000, 5, 0x00050000 }, + { 0x3, 0x18, 0x0, 0x2, 0x0d020000, 0, 0x00000000 }, + { 0x3, 0x19, 0x0, 0x2, 0x0d030000, 0, 0x00000000 }, + { 0x3, 0x1f, 0x0, 0x2, 0x0d600000, 0, 0x00000000 }, + { 0x3, 0x1f, 0x1, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x4, 0x1b, 0x0, 0x0, 0x40000000, 0, 0x40000000 }, + { 0x4, 0x1b, 0x1, 0x1, 0x80000000, 1, 0x80000000 }, + { 0x4, 0x1e, 0x0, 0x2, 0x0d400000, 0, 0x0d400000 }, + { 0x4, 0x1e, 0x1, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x5, 0x1c, 0x0, 0x0, 0x0d640000, 0, 0x00000000 }, + { 0x5, 0x1d, 0x0, 0x0, 0x20b00000, 8, 0x20b00000 }, + { 0x5, 0x1d, 0x1, 0x0, 0x20800000, 7, 0x20800000 }, + { 0x5, 0x1d, 0x2, 0x0, 0x20c00000, 9, 0x20c00000 }, + { 0x5, 0x1d, 0x3, 0x0, 0x0d800000, 3, 0x0d800000 }, + { 0x5, 0x1d, 0x4, 0x0, 0x20000000, 6, 0x20000000 }, + { 0x5, 0x1d, 0x5, 0x0, 0x0c000000, 2, 0x0c000000 }, + { 0x5, 0x1d, 0x6, 0x0, 0x21000000, 10, 0x21000000 }, + { 0x5, 0x1d, 0x7, 0x0, 0x0e000000, 4, 0x0e000000 }, + { 0x5, 0x1d, 0x8, 0x0, 0x22000000, 11, 0x22000000 }, + { 0x5, 0x1d, 0x9, 0x0, 0x08000000, 1, 0x08000000 }, + { 0x5, 0x1d, 0xa, 0x0, 0x24000000, 12, 0x24000000 }, + { 0x5, 0x1d, 0xb, 0x0, 0x00000000, 0, 0x00000000 }, + { 0x5, 0x1d, 0xc, 0x0, 0x28000000, 13, 0x28000000 }, + { 0x5, 0x1d, 0xd, 0x0, 0x10000000, 5, 0x10000000 }, + { 0x5, 0x1d, 0xe, 0x0, 0x30000000, 14, 0x30000000 }, + { 0x5, 0x00, 0x0, 0x0, 0x0d230000, 0, 0x00000000 }, + { 0x5, 0x01, 0x0, 0x0, 0x0d040000, 0, 0x00000000 }, + { 0x5, 0x02, 0x0, 0x0, 0x0d050000, 0, 0x00000000 }, + { 0x5, 0x03, 0x0, 0x0, 0x0d000000, 0, 0x00000000 }, + { 0x5, 0x04, 0x0, 0x0, 0x20ae0000, 3, 0x000e0000 }, + { 0x5, 0x04, 0x1, 0x0, 0x20ac0000, 2, 0x000c0000 }, + { 0x5, 0x04, 0x2, 0x0, 0x20a80000, 1, 0x00080000 }, + { 0x5, 0x04, 0x3, 0x0, 0x20a00000, 0, 0x00000000 }, + { 0x5, 0x05, 0x0, 0x0, 0x0d2a0000, 0, 0x00000000 }, + { 0x5, 0x06, 0x0, 0x0, 0x0d290000, 0, 0x00000000 }, + { 0x5, 0x07, 0x0, 0x0, 0x0d2c0000, 0, 0x00000000 }, + { 0x5, 0x08, 0x0, 0x0, 0x0d0e0000, 4, 0x00080000 }, + { 0x5, 0x08, 0x1, 0x0, 0x0d060000, 0, 0x00000000 }, + { 0x5, 0x08, 0x2, 0x0, 0x0d080000, 1, 0x00020000 }, + { 0x5, 0x08, 0x3, 0x0, 0x0d0a0000, 2, 0x00040000 }, + { 0x5, 0x08, 0x4, 0x0, 0x0d0c0000, 3, 0x00060000 }, + { 0x5, 0x09, 0x0, 0x0, 0x0d650000, 0, 0x00000000 }, + { 0x5, 0x0a, 0x0, 0x0, 0x20af0000, 0, 0x00000000 }, + { 0x5, 0x0b, 0x0, 0x0, 0x0d3e0000, 0, 0x00000000 }, + { 0x5, 0x0c, 0x0, 0x0, 0x0d3d0000, 0, 0x00000000 }, + { 0x5, 0x0d, 0x0, 0x0, 0x0d1e0000, 0, 0x00000000 }, + { 0x5, 0x0e, 0x0, 0x0, 0x0d150000, 0, 0x00000000 }, + { 0x5, 0x0e, 0x1, 0x0, 0x0d160000, 1, 0x00010000 }, + { 0x5, 0x0e, 0x2, 0x0, 0x0d170000, 2, 0x00020000 }, + { 0x5, 0x0e, 0x3, 0x0, 0x0d180000, 3, 0x00030000 }, + { 0x5, 0x0e, 0x4, 0x0, 0x0d190000, 4, 0x00040000 }, + { 0x5, 0x0e, 0x5, 0x0, 0x0d1a0000, 5, 0x00050000 }, + { 0x5, 0x0e, 0x6, 0x0, 0x0d1b0000, 6, 0x00060000 }, + { 0x5, 0x0e, 0x7, 0x0, 0x0d1c0000, 7, 0x00070000 }, + { 0x5, 0x0e, 0x8, 0x0, 0x0d1d0000, 8, 0x00080000 }, + { 0x5, 0x0f, 0x0, 0x0, 0x0d660000, 0, 0x00000000 }, + { 0x5, 0x10, 0x0, 0x0, 0x0d1f0000, 0, 0x00000000 }, + { 0x5, 0x10, 0x1, 0x0, 0x0d200000, 1, 0x00010000 }, + { 0x5, 0x10, 0x2, 0x0, 0x0d210000, 2, 0x00020000 }, + { 0x5, 0x10, 0x3, 0x0, 0x0d220000, 3, 0x00030000 }, + { 0x5, 0x11, 0x0, 0x0, 0x0d240000, 0, 0x00000000 }, + { 0x5, 0x12, 0x0, 0x0, 0x0d250000, 0, 0x00000000 }, + { 0x5, 0x13, 0x0, 0x0, 0x0d260000, 0, 0x00000000 }, + { 0x5, 0x14, 0x0, 0x0, 0x0d270000, 0, 0x00000000 }, + { 0x5, 0x15, 0x0, 0x0, 0x0d2b0000, 0, 0x00000000 }, + { 0x5, 0x16, 0x0, 0x0, 0x0d280000, 0, 0x00000000 }, + { 0x5, 0x17, 0x0, 0x0, 0x0d0f0000, 0, 0x00000000 }, + { 0x5, 0x17, 0x1, 0x0, 0x0d100000, 1, 0x00010000 }, + { 0x5, 0x17, 0x2, 0x0, 0x0d110000, 2, 0x00020000 }, + { 0x5, 0x17, 0x3, 0x0, 0x0d120000, 3, 0x00030000 }, + { 0x5, 0x17, 0x4, 0x0, 0x0d130000, 4, 0x00040000 }, + { 0x5, 0x17, 0x5, 0x0, 0x0d140000, 5, 0x00050000 }, + { 0x5, 0x18, 0x0, 0x0, 0x0d020000, 0, 0x00000000 }, + { 0x5, 0x19, 0x0, 0x0, 0x0d030000, 0, 0x00000000 }, + { 0x5, 0x1f, 0x0, 0x0, 0x0d600000, 0, 0x00000000 }, + { 0x5, 0x1f, 0x1, 0x0, 0x00000000, 0, 0x00000000 } +}; + +/* + * AON NOC aperture lookup table as per file "AON_NOC_Structure.info". + */ +static const char * const tegra194_aonnoc_routeid_initflow[] = { + [0x0] = "cbb_i/I/0", + [0x1] = "cpu_p_i/I/0", + [0x2] = "dma_m_i/I/0", + [0x3] = "dma_p_i/I/0" +}; + +static const char * const tegra194_aonnoc_routeid_targflow[] = { + [0x00] = "multiport1_t/T/aon_misc", + [0x01] = "multiport1_t/T/avic0", + [0x02] = "multiport1_t/T/avic1", + [0x03] = "multiport1_t/T/can1", + [0x04] = "multiport1_t/T/can2", + [0x05] = "multiport1_t/T/dma", + [0x06] = "multiport1_t/T/dmic", + [0x07] = "multiport1_t/T/err_collator", + [0x08] = "multiport1_t/T/fpga_misc", + [0x09] = "multiport1_t/T/gte", + [0x0a] = "multiport1_t/T/hsp", + [0x0b] = "multiport1_t/T/i2c2", + [0x0c] = "multiport1_t/T/i2c8", + [0x0d] = "multiport1_t/T/pwm", + [0x0e] = "multiport1_t/T/spi2", + [0x0f] = "multiport1_t/T/tke", + [0x10] = "multiport1_t/T/uartg", + [0x11] = "RESERVED", + [0x12] = "RESERVED", + [0x13] = "RESERVED", + [0x14] = "RESERVED", + [0x15] = "RESERVED", + [0x16] = "RESERVED", + [0x17] = "RESERVED", + [0x18] = "RESERVED", + [0x19] = "RESERVED", + [0x1a] = "RESERVED", + [0x1b] = "RESERVED", + [0x1c] = "RESERVED", + [0x1d] = "RESERVED", + [0x1e] = "RESERVED", + [0x1f] = "RESERVED", + [0x20] = "multiport0_t/T/aovc", + [0x21] = "multiport0_t/T/atcm", + [0x22] = "multiport0_t/T/cast", + [0x23] = "multiport0_t/T/dast", + [0x24] = "multiport0_t/T/err_collator_car", + [0x25] = "multiport0_t/T/gpio", + [0x26] = "multiport0_t/T/i2c10", + [0x27] = "multiport0_t/T/mss", + [0x28] = "multiport0_t/T/padctl_a12", + [0x29] = "multiport0_t/T/padctl_a14", + [0x2a] = "multiport0_t/T/padctl_a15", + [0x2b] = "multiport0_t/T/rtc", + [0x2c] = "multiport0_t/T/tsc", + [0x2d] = "RESERVED", + [0x2e] = "RESERVED", + [0x2f] = "RESERVED", + [0x30] = "multiport2_t/T/aon_vref_ro", + [0x31] = "multiport2_t/T/aopm", + [0x32] = "multiport2_t/T/car", + [0x33] = "multiport2_t/T/pmc", + [0x34] = "ast1_t/T/0", + [0x35] = "cbb_t/T/0", + [0x36] = "cpu_t/T/0", + [0x37] = "firewall_t/T/0", + [0x38] = "svc_t/T/0", + [0x39] = "uartc/T/uartc", + [0x3a] = "RESERVED", + [0x3b] = "RESERVED", + [0x3c] = "RESERVED", + [0x3d] = "RESERVED", + [0x3e] = "RESERVED", + [0x3f] = "RESERVED" +}; + +/* + * Fields of AON NOC lookup table: + * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress, + * Targ mapping, Targ localAddress + * ---------------------------------------------------------------------------- + */ +static const struct tegra194_cbb_aperture tegra194_aonnoc_aperture_lookup[] = { + { 0x0, 0x37, 0x00, 0, 0x0c640000, 0, 0x00000000 }, + { 0x0, 0x20, 0x00, 0, 0x0c3b0000, 0, 0x00000000 }, + { 0x0, 0x21, 0x00, 0, 0x0c000000, 0, 0x00000000 }, + { 0x0, 0x22, 0x00, 0, 0x0c040000, 0, 0x00000000 }, + { 0x0, 0x23, 0x00, 0, 0x0c050000, 0, 0x00000000 }, + { 0x0, 0x24, 0x00, 0, 0x20cf0000, 0, 0x00000000 }, + { 0x0, 0x25, 0x00, 0, 0x0c2f0000, 0, 0x00000000 }, + { 0x0, 0x26, 0x00, 0, 0x0c230000, 0, 0x00000000 }, + { 0x0, 0x27, 0x00, 0, 0x0c350000, 0, 0x00000000 }, + { 0x0, 0x28, 0x00, 0, 0x0c301000, 0, 0x00000000 }, + { 0x0, 0x29, 0x00, 0, 0x0c302000, 0, 0x00000000 }, + { 0x0, 0x2a, 0x00, 0, 0x0c303000, 0, 0x00000000 }, + { 0x0, 0x2b, 0x00, 0, 0x0c2a0000, 0, 0x00000000 }, + { 0x0, 0x2c, 0x00, 0, 0x0c2b0000, 0, 0x00000000 }, + { 0x0, 0x2c, 0x01, 0, 0x0c2c0000, 1, 0x00010000 }, + { 0x0, 0x2c, 0x02, 0, 0x0c2d0000, 2, 0x00020000 }, + { 0x0, 0x2c, 0x03, 0, 0x0c2e0000, 3, 0x00030000 }, + { 0x0, 0x00, 0x00, 0, 0x0c660000, 0, 0x00000000 }, + { 0x0, 0x01, 0x00, 0, 0x0c020000, 0, 0x00000000 }, + { 0x0, 0x02, 0x00, 0, 0x0c030000, 0, 0x00000000 }, + { 0x0, 0x03, 0x00, 0, 0x0c310000, 0, 0x00000000 }, + { 0x0, 0x04, 0x00, 0, 0x0c320000, 0, 0x00000000 }, + { 0x0, 0x05, 0x00, 0, 0x0c0a0000, 2, 0x00040000 }, + { 0x0, 0x05, 0x01, 0, 0x0c0b0000, 3, 0x00050000 }, + { 0x0, 0x05, 0x02, 0, 0x0c0e0000, 5, 0x00080000 }, + { 0x0, 0x05, 0x03, 0, 0x0c060000, 0, 0x00000000 }, + { 0x0, 0x05, 0x04, 0, 0x0c080000, 1, 0x00020000 }, + { 0x0, 0x05, 0x05, 0, 0x0c0c0000, 4, 0x00060000 }, + { 0x0, 0x06, 0x00, 0, 0x0c330000, 0, 0x00000000 }, + { 0x0, 0x07, 0x00, 0, 0x0c650000, 0, 0x00000000 }, + { 0x0, 0x08, 0x00, 0, 0x0c3e0000, 0, 0x00000000 }, + { 0x0, 0x09, 0x00, 0, 0x0c1e0000, 0, 0x00000000 }, + { 0x0, 0x0a, 0x00, 0, 0x0c150000, 0, 0x00000000 }, + { 0x0, 0x0a, 0x01, 0, 0x0c160000, 1, 0x00010000 }, + { 0x0, 0x0a, 0x02, 0, 0x0c170000, 2, 0x00020000 }, + { 0x0, 0x0a, 0x03, 0, 0x0c180000, 3, 0x00030000 }, + { 0x0, 0x0a, 0x04, 0, 0x0c190000, 4, 0x00040000 }, + { 0x0, 0x0a, 0x05, 0, 0x0c1a0000, 5, 0x00050000 }, + { 0x0, 0x0a, 0x06, 0, 0x0c1b0000, 6, 0x00060000 }, + { 0x0, 0x0a, 0x07, 0, 0x0c1c0000, 7, 0x00070000 }, + { 0x0, 0x0a, 0x08, 0, 0x0c1d0000, 8, 0x00080000 }, + { 0x0, 0x0b, 0x00, 0, 0x0c240000, 0, 0x00000000 }, + { 0x0, 0x0c, 0x00, 0, 0x0c250000, 0, 0x00000000 }, + { 0x0, 0x0d, 0x00, 0, 0x0c340000, 0, 0x00000000 }, + { 0x0, 0x0e, 0x00, 0, 0x0c260000, 0, 0x00000000 }, + { 0x0, 0x0f, 0x00, 0, 0x0c0f0000, 0, 0x00000000 }, + { 0x0, 0x0f, 0x01, 0, 0x0c100000, 1, 0x00010000 }, + { 0x0, 0x0f, 0x02, 0, 0x0c110000, 2, 0x00020000 }, + { 0x0, 0x0f, 0x03, 0, 0x0c120000, 3, 0x00030000 }, + { 0x0, 0x0f, 0x04, 0, 0x0c130000, 4, 0x00040000 }, + { 0x0, 0x0f, 0x05, 0, 0x0c140000, 5, 0x00050000 }, + { 0x0, 0x10, 0x00, 0, 0x0c290000, 0, 0x00000000 }, + { 0x0, 0x30, 0x00, 0, 0x20ce0000, 0, 0x00000000 }, + { 0x0, 0x31, 0x00, 0, 0x0c1f0000, 0, 0x00000000 }, + { 0x0, 0x31, 0x01, 0, 0x0c200000, 1, 0x00010000 }, + { 0x0, 0x31, 0x02, 0, 0x0c210000, 2, 0x00020000 }, + { 0x0, 0x31, 0x03, 0, 0x0c220000, 3, 0x00030000 }, + { 0x0, 0x32, 0x00, 0, 0x20cc0000, 3, 0x001c0000 }, + { 0x0, 0x32, 0x01, 0, 0x20c80000, 2, 0x00180000 }, + { 0x0, 0x32, 0x02, 0, 0x20c00000, 1, 0x00100000 }, + { 0x0, 0x32, 0x03, 0, 0x20b00000, 0, 0x00000000 }, + { 0x0, 0x33, 0x00, 0, 0x0c360000, 0, 0x00000000 }, + { 0x0, 0x33, 0x01, 0, 0x0c370000, 1, 0x00010000 }, + { 0x0, 0x33, 0x02, 0, 0x0c3a0000, 3, 0x00040000 }, + { 0x0, 0x33, 0x03, 0, 0x0c380000, 2, 0x00020000 }, + { 0x0, 0x38, 0x00, 0, 0x0c600000, 0, 0x00000000 }, + { 0x0, 0x38, 0x01, 0, 0x00000000, 0, 0x00000000 }, + { 0x0, 0x39, 0x00, 0, 0x0c280000, 0, 0x00000000 }, + { 0x1, 0x35, 0x00, 0, 0x00000000, 0, 0x00000000 }, + { 0x1, 0x35, 0x01, 0, 0x00100000, 1, 0x00100000 }, + { 0x1, 0x35, 0x02, 0, 0x05a00000, 11, 0x05a00000 }, + { 0x1, 0x35, 0x03, 0, 0x05b00000, 32, 0x05b00000 }, + { 0x1, 0x35, 0x04, 0, 0x05c00000, 33, 0x05c00000 }, + { 0x1, 0x35, 0x05, 0, 0x05d00000, 12, 0x05d00000 }, + { 0x1, 0x35, 0x06, 0, 0x20000000, 19, 0x20000000 }, + { 0x1, 0x35, 0x07, 0, 0x20100000, 20, 0x20100000 }, + { 0x1, 0x35, 0x08, 0, 0x20a00000, 24, 0x20a00000 }, + { 0x1, 0x35, 0x09, 0, 0x20d00000, 25, 0x20d00000 }, + { 0x1, 0x35, 0x0a, 0, 0x00200000, 2, 0x00200000 }, + { 0x1, 0x35, 0x0b, 0, 0x05800000, 10, 0x05800000 }, + { 0x1, 0x35, 0x0c, 0, 0x05e00000, 13, 0x05e00000 }, + { 0x1, 0x35, 0x0d, 0, 0x20200000, 21, 0x20200000 }, + { 0x1, 0x35, 0x0e, 0, 0x20800000, 23, 0x20800000 }, + { 0x1, 0x35, 0x0f, 0, 0x20e00000, 26, 0x20e00000 }, + { 0x1, 0x35, 0x10, 0, 0x00400000, 3, 0x00400000 }, + { 0x1, 0x35, 0x11, 0, 0x20400000, 22, 0x20400000 }, + { 0x1, 0x35, 0x12, 0, 0x00800000, 4, 0x00800000 }, + { 0x1, 0x35, 0x13, 0, 0x05000000, 9, 0x05000000 }, + { 0x1, 0x35, 0x14, 0, 0x0c800000, 34, 0x0c800000 }, + { 0x1, 0x35, 0x15, 0, 0x01000000, 5, 0x01000000 }, + { 0x1, 0x35, 0x16, 0, 0x03000000, 7, 0x03000000 }, + { 0x1, 0x35, 0x17, 0, 0x04000000, 8, 0x04000000 }, + { 0x1, 0x35, 0x18, 0, 0x0d000000, 16, 0x0d000000 }, + { 0x1, 0x35, 0x19, 0, 0x21000000, 27, 0x21000000 }, + { 0x1, 0x35, 0x1a, 0, 0x02000000, 6, 0x02000000 }, + { 0x1, 0x35, 0x1b, 0, 0x06000000, 14, 0x06000000 }, + { 0x1, 0x35, 0x1c, 0, 0x0e000000, 17, 0x0e000000 }, + { 0x1, 0x35, 0x1d, 0, 0x22000000, 28, 0x22000000 }, + { 0x1, 0x35, 0x1e, 0, 0x08000000, 15, 0x08000000 }, + { 0x1, 0x35, 0x1f, 0, 0x24000000, 29, 0x24000000 }, + { 0x1, 0x35, 0x20, 0, 0x28000000, 30, 0x28000000 }, + { 0x1, 0x35, 0x21, 0, 0x10000000, 18, 0x10000000 }, + { 0x1, 0x35, 0x22, 0, 0x30000000, 31, 0x30000000 }, + { 0x1, 0x37, 0x00, 0, 0x0c640000, 0, 0x00000000 }, + { 0x1, 0x20, 0x00, 0, 0x0c3b0000, 0, 0x00000000 }, + { 0x1, 0x21, 0x00, 0, 0x0c000000, 0, 0x00000000 }, + { 0x1, 0x22, 0x00, 0, 0x0c040000, 0, 0x00000000 }, + { 0x1, 0x23, 0x00, 0, 0x0c050000, 0, 0x00000000 }, + { 0x1, 0x24, 0x00, 0, 0x20cf0000, 0, 0x00000000 }, + { 0x1, 0x25, 0x00, 0, 0x0c2f0000, 0, 0x00000000 }, + { 0x1, 0x26, 0x00, 0, 0x0c230000, 0, 0x00000000 }, + { 0x1, 0x27, 0x00, 0, 0x0c350000, 0, 0x00000000 }, + { 0x1, 0x28, 0x00, 0, 0x0c301000, 0, 0x00000000 }, + { 0x1, 0x29, 0x00, 0, 0x0c302000, 0, 0x00000000 }, + { 0x1, 0x2a, 0x00, 0, 0x0c303000, 0, 0x00000000 }, + { 0x1, 0x2b, 0x00, 0, 0x0c2a0000, 0, 0x00000000 }, + { 0x1, 0x2c, 0x00, 0, 0x0c2b0000, 0, 0x00000000 }, + { 0x1, 0x2c, 0x01, 0, 0x0c2c0000, 1, 0x00010000 }, + { 0x1, 0x2c, 0x02, 0, 0x0c2d0000, 2, 0x00020000 }, + { 0x1, 0x2c, 0x03, 0, 0x0c2e0000, 3, 0x00030000 }, + { 0x1, 0x00, 0x00, 0, 0x0c660000, 0, 0x00000000 }, + { 0x1, 0x01, 0x00, 0, 0x0c020000, 0, 0x00000000 }, + { 0x1, 0x02, 0x00, 0, 0x0c030000, 0, 0x00000000 }, + { 0x1, 0x03, 0x00, 0, 0x0c310000, 0, 0x00000000 }, + { 0x1, 0x04, 0x00, 0, 0x0c320000, 0, 0x00000000 }, + { 0x1, 0x05, 0x00, 0, 0x0c0a0000, 2, 0x00040000 }, + { 0x1, 0x05, 0x01, 0, 0x0c0b0000, 3, 0x00050000 }, + { 0x1, 0x05, 0x02, 0, 0x0c0e0000, 5, 0x00080000 }, + { 0x1, 0x05, 0x03, 0, 0x0c060000, 0, 0x00000000 }, + { 0x1, 0x05, 0x04, 0, 0x0c080000, 1, 0x00020000 }, + { 0x1, 0x05, 0x05, 0, 0x0c0c0000, 4, 0x00060000 }, + { 0x1, 0x06, 0x00, 0, 0x0c330000, 0, 0x00000000 }, + { 0x1, 0x07, 0x00, 0, 0x0c650000, 0, 0x00000000 }, + { 0x1, 0x08, 0x00, 0, 0x0c3e0000, 0, 0x00000000 }, + { 0x1, 0x09, 0x00, 0, 0x0c1e0000, 0, 0x00000000 }, + { 0x1, 0x0a, 0x00, 0, 0x0c150000, 0, 0x00000000 }, + { 0x1, 0x0a, 0x01, 0, 0x0c160000, 1, 0x00010000 }, + { 0x1, 0x0a, 0x02, 0, 0x0c170000, 2, 0x00020000 }, + { 0x1, 0x0a, 0x03, 0, 0x0c180000, 3, 0x00030000 }, + { 0x1, 0x0a, 0x04, 0, 0x0c190000, 4, 0x00040000 }, + { 0x1, 0x0a, 0x05, 0, 0x0c1a0000, 5, 0x00050000 }, + { 0x1, 0x0a, 0x06, 0, 0x0c1b0000, 6, 0x00060000 }, + { 0x1, 0x0a, 0x07, 0, 0x0c1c0000, 7, 0x00070000 }, + { 0x1, 0x0a, 0x08, 0, 0x0c1d0000, 8, 0x00080000 }, + { 0x1, 0x0b, 0x00, 0, 0x0c240000, 0, 0x00000000 }, + { 0x1, 0x0c, 0x00, 0, 0x0c250000, 0, 0x00000000 }, + { 0x1, 0x0d, 0x00, 0, 0x0c340000, 0, 0x00000000 }, + { 0x1, 0x0e, 0x00, 0, 0x0c260000, 0, 0x00000000 }, + { 0x1, 0x0f, 0x00, 0, 0x0c0f0000, 0, 0x00000000 }, + { 0x1, 0x0f, 0x01, 0, 0x0c100000, 1, 0x00010000 }, + { 0x1, 0x0f, 0x02, 0, 0x0c110000, 2, 0x00020000 }, + { 0x1, 0x0f, 0x03, 0, 0x0c120000, 3, 0x00030000 }, + { 0x1, 0x0f, 0x04, 0, 0x0c130000, 4, 0x00040000 }, + { 0x1, 0x0f, 0x05, 0, 0x0c140000, 5, 0x00050000 }, + { 0x1, 0x10, 0x00, 0, 0x0c290000, 0, 0x00000000 }, + { 0x1, 0x30, 0x00, 0, 0x20ce0000, 0, 0x00000000 }, + { 0x1, 0x31, 0x00, 0, 0x0c1f0000, 0, 0x00000000 }, + { 0x1, 0x31, 0x01, 0, 0x0c200000, 1, 0x00010000 }, + { 0x1, 0x31, 0x02, 0, 0x0c210000, 2, 0x00020000 }, + { 0x1, 0x31, 0x03, 0, 0x0c220000, 3, 0x00030000 }, + { 0x1, 0x32, 0x00, 0, 0x20cc0000, 3, 0x001c0000 }, + { 0x1, 0x32, 0x01, 0, 0x20c80000, 2, 0x00180000 }, + { 0x1, 0x32, 0x02, 0, 0x20c00000, 1, 0x00100000 }, + { 0x1, 0x32, 0x03, 0, 0x20b00000, 0, 0x00000000 }, + { 0x1, 0x33, 0x00, 0, 0x0c360000, 0, 0x00000000 }, + { 0x1, 0x33, 0x01, 0, 0x0c370000, 1, 0x00010000 }, + { 0x1, 0x33, 0x02, 0, 0x0c3a0000, 3, 0x00040000 }, + { 0x1, 0x33, 0x03, 0, 0x0c380000, 2, 0x00020000 }, + { 0x1, 0x38, 0x00, 0, 0x0c600000, 0, 0x00000000 }, + { 0x1, 0x38, 0x01, 0, 0x00000000, 0, 0x00000000 }, + { 0x1, 0x39, 0x00, 0, 0x0c280000, 0, 0x00000000 }, + { 0x2, 0x34, 0x00, 0, 0x40000000, 0, 0x40000000 }, + { 0x2, 0x34, 0x01, 0, 0x80000000, 1, 0x80000000 }, + { 0x2, 0x36, 0x00, 0, 0x0c400000, 0, 0x0c400000 }, + { 0x2, 0x36, 0x01, 0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x35, 0x00, 0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x35, 0x01, 0, 0x00100000, 1, 0x00100000 }, + { 0x3, 0x35, 0x02, 0, 0x05a00000, 11, 0x05a00000 }, + { 0x3, 0x35, 0x03, 0, 0x05b00000, 32, 0x05b00000 }, + { 0x3, 0x35, 0x04, 0, 0x05c00000, 33, 0x05c00000 }, + { 0x3, 0x35, 0x05, 0, 0x05d00000, 12, 0x05d00000 }, + { 0x3, 0x35, 0x06, 0, 0x20000000, 19, 0x20000000 }, + { 0x3, 0x35, 0x07, 0, 0x20100000, 20, 0x20100000 }, + { 0x3, 0x35, 0x08, 0, 0x20a00000, 24, 0x20a00000 }, + { 0x3, 0x35, 0x09, 0, 0x20d00000, 25, 0x20d00000 }, + { 0x3, 0x35, 0x0a, 0, 0x00200000, 2, 0x00200000 }, + { 0x3, 0x35, 0x0b, 0, 0x05800000, 10, 0x05800000 }, + { 0x3, 0x35, 0x0c, 0, 0x05e00000, 13, 0x05e00000 }, + { 0x3, 0x35, 0x0d, 0, 0x20200000, 21, 0x20200000 }, + { 0x3, 0x35, 0x0e, 0, 0x20800000, 23, 0x20800000 }, + { 0x3, 0x35, 0x0f, 0, 0x20e00000, 26, 0x20e00000 }, + { 0x3, 0x35, 0x10, 0, 0x00400000, 3, 0x00400000 }, + { 0x3, 0x35, 0x11, 0, 0x20400000, 22, 0x20400000 }, + { 0x3, 0x35, 0x12, 0, 0x00800000, 4, 0x00800000 }, + { 0x3, 0x35, 0x13, 0, 0x50000000, 9, 0x05000000 }, + { 0x3, 0x35, 0x14, 0, 0xc0800000, 34, 0x0c800000 }, + { 0x3, 0x35, 0x15, 0, 0x10000000, 5, 0x01000000 }, + { 0x3, 0x35, 0x16, 0, 0x30000000, 7, 0x03000000 }, + { 0x3, 0x35, 0x17, 0, 0x04000000, 8, 0x04000000 }, + { 0x3, 0x35, 0x18, 0, 0x0d000000, 16, 0x0d000000 }, + { 0x3, 0x35, 0x19, 0, 0x21000000, 27, 0x21000000 }, + { 0x3, 0x35, 0x1a, 0, 0x02000000, 6, 0x02000000 }, + { 0x3, 0x35, 0x1b, 0, 0x06000000, 14, 0x06000000 }, + { 0x3, 0x35, 0x1c, 0, 0x0e000000, 17, 0x0e000000 }, + { 0x3, 0x35, 0x1d, 0, 0x22000000, 28, 0x22000000 }, + { 0x3, 0x35, 0x1e, 0, 0x08000000, 15, 0x08000000 }, + { 0x3, 0x35, 0x1f, 0, 0x24000000, 29, 0x24000000 }, + { 0x3, 0x35, 0x20, 0, 0x28000000, 30, 0x28000000 }, + { 0x3, 0x35, 0x21, 0, 0x10000000, 18, 0x10000000 }, + { 0x3, 0x35, 0x22, 0, 0x30000000, 31, 0x30000000 }, + { 0x3, 0x37, 0x00, 0, 0x0c640000, 0, 0x00000000 }, + { 0x3, 0x20, 0x00, 0, 0x0c3b0000, 0, 0x00000000 }, + { 0x3, 0x21, 0x00, 0, 0x0c000000, 0, 0x00000000 }, + { 0x3, 0x22, 0x00, 0, 0x0c040000, 0, 0x00000000 }, + { 0x3, 0x23, 0x00, 0, 0x0c050000, 0, 0x00000000 }, + { 0x3, 0x24, 0x00, 0, 0x20cf0000, 0, 0x00000000 }, + { 0x3, 0x25, 0x00, 0, 0x0c2f0000, 0, 0x00000000 }, + { 0x3, 0x26, 0x00, 0, 0x0c230000, 0, 0x00000000 }, + { 0x3, 0x27, 0x00, 0, 0x0c350000, 0, 0x00000000 }, + { 0x3, 0x28, 0x00, 0, 0x0c301000, 0, 0x00000000 }, + { 0x3, 0x29, 0x00, 0, 0x0c302000, 0, 0x00000000 }, + { 0x3, 0x2a, 0x00, 0, 0x0c303000, 0, 0x00000000 }, + { 0x3, 0x2b, 0x00, 0, 0x0c2a0000, 0, 0x00000000 }, + { 0x3, 0x2c, 0x00, 0, 0x0c2b0000, 0, 0x00000000 }, + { 0x3, 0x2c, 0x01, 0, 0x0c2c0000, 1, 0x00010000 }, + { 0x3, 0x2c, 0x02, 0, 0x0c2d0000, 2, 0x00020000 }, + { 0x3, 0x2c, 0x03, 0, 0x0c2e0000, 3, 0x00030000 }, + { 0x3, 0x00, 0x00, 0, 0x0c660000, 0, 0x00000000 }, + { 0x3, 0x01, 0x00, 0, 0x0c020000, 0, 0x00000000 }, + { 0x3, 0x02, 0x00, 0, 0x0c030000, 0, 0x00000000 }, + { 0x3, 0x03, 0x00, 0, 0x0c310000, 0, 0x00000000 }, + { 0x3, 0x04, 0x00, 0, 0x0c320000, 0, 0x00000000 }, + { 0x3, 0x05, 0x00, 0, 0x0c0a0000, 2, 0x00040000 }, + { 0x3, 0x05, 0x01, 0, 0x0c0b0000, 3, 0x00050000 }, + { 0x3, 0x05, 0x02, 0, 0x0c0e0000, 5, 0x00080000 }, + { 0x3, 0x05, 0x03, 0, 0x0c060000, 0, 0x00000000 }, + { 0x3, 0x05, 0x04, 0, 0x0c080000, 1, 0x00020000 }, + { 0x3, 0x05, 0x05, 0, 0x0c0c0000, 4, 0x00060000 }, + { 0x3, 0x06, 0x00, 0, 0x0c330000, 0, 0x00000000 }, + { 0x3, 0x07, 0x00, 0, 0x0c650000, 0, 0x00000000 }, + { 0x3, 0x08, 0x00, 0, 0x0c3e0000, 0, 0x00000000 }, + { 0x3, 0x09, 0x00, 0, 0x0c1e0000, 0, 0x00000000 }, + { 0x3, 0x0a, 0x00, 0, 0x0c150000, 0, 0x00000000 }, + { 0x3, 0x0a, 0x01, 0, 0x0c160000, 1, 0x00010000 }, + { 0x3, 0x0a, 0x02, 0, 0x0c170000, 2, 0x00020000 }, + { 0x3, 0x0a, 0x03, 0, 0x0c180000, 3, 0x00030000 }, + { 0x3, 0x0a, 0x04, 0, 0x0c190000, 4, 0x00040000 }, + { 0x3, 0x0a, 0x05, 0, 0x0c1a0000, 5, 0x00050000 }, + { 0x3, 0x0a, 0x06, 0, 0x0c1b0000, 6, 0x00060000 }, + { 0x3, 0x0a, 0x07, 0, 0x0c1c0000, 7, 0x00070000 }, + { 0x3, 0x0a, 0x08, 0, 0x0c1d0000, 8, 0x00080000 }, + { 0x3, 0x0b, 0x00, 0, 0x0c240000, 0, 0x00000000 }, + { 0x3, 0x0c, 0x00, 0, 0x0c250000, 0, 0x00000000 }, + { 0x3, 0x0d, 0x00, 0, 0x0c340000, 0, 0x00000000 }, + { 0x3, 0x0e, 0x00, 0, 0x0c260000, 0, 0x00000000 }, + { 0x3, 0x0f, 0x00, 0, 0x0c0f0000, 0, 0x00000000 }, + { 0x3, 0x0f, 0x01, 0, 0x0c100000, 1, 0x00010000 }, + { 0x3, 0x0f, 0x02, 0, 0x0c110000, 2, 0x00020000 }, + { 0x3, 0x0f, 0x03, 0, 0x0c120000, 3, 0x00030000 }, + { 0x3, 0x0f, 0x04, 0, 0x0c130000, 4, 0x00040000 }, + { 0x3, 0x0f, 0x05, 0, 0x0c140000, 5, 0x00050000 }, + { 0x3, 0x10, 0x00, 0, 0x0c290000, 0, 0x00000000 }, + { 0x3, 0x30, 0x00, 0, 0x20ce0000, 0, 0x00000000 }, + { 0x3, 0x31, 0x00, 0, 0x0c1f0000, 0, 0x00000000 }, + { 0x3, 0x31, 0x01, 0, 0x0c200000, 1, 0x00010000 }, + { 0x3, 0x31, 0x02, 0, 0x0c210000, 2, 0x00020000 }, + { 0x3, 0x31, 0x03, 0, 0x0c220000, 3, 0x00030000 }, + { 0x3, 0x32, 0x00, 0, 0x20cc0000, 3, 0x001c0000 }, + { 0x3, 0x32, 0x01, 0, 0x20c80000, 2, 0x00180000 }, + { 0x3, 0x32, 0x02, 0, 0x20c00000, 1, 0x00100000 }, + { 0x3, 0x32, 0x03, 0, 0x20b00000, 0, 0x00000000 }, + { 0x3, 0x33, 0x00, 0, 0x0c360000, 0, 0x00000000 }, + { 0x3, 0x33, 0x01, 0, 0x0c370000, 1, 0x00010000 }, + { 0x3, 0x33, 0x02, 0, 0x0c3a0000, 3, 0x00040000 }, + { 0x3, 0x33, 0x03, 0, 0x0c380000, 2, 0x00020000 }, + { 0x3, 0x38, 0x00, 0, 0x0c600000, 0, 0x00000000 }, + { 0x3, 0x38, 0x01, 0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x39, 0x00, 0, 0x0c280000, 0, 0x00000000 } +}; + +/* + * SCE/RCE NOC aperture lookup table as per file "AON_NOC_Structure.info". + */ +static const char * const tegra194_scenoc_routeid_initflow[] = { + [0x0] = "cbb_i/I/0", + [0x1] = "cpu_m_i/I/0", + [0x2] = "cpu_p_i/I/0", + [0x3] = "dma_m_i/I/0", + [0x4] = "dma_p_i/I/0", + [0x5] = "RESERVED", + [0x6] = "RESERVED", + [0x7] = "RESERVED" +}; + +static const char * const tegra194_scenoc_routeid_targflow[] = { + [0x00] = "multiport0_t/T/atcm_cfg", + [0x01] = "multiport0_t/T/car", + [0x02] = "multiport0_t/T/cast", + [0x03] = "multiport0_t/T/cfg", + [0x04] = "multiport0_t/T/dast", + [0x05] = "multiport0_t/T/dma", + [0x06] = "multiport0_t/T/err_collator", + [0x07] = "multiport0_t/T/err_collator_car", + [0x08] = "multiport0_t/T/fpga_misc", + [0x09] = "multiport0_t/T/fpga_uart", + [0x0a] = "multiport0_t/T/gte", + [0x0b] = "multiport0_t/T/hsp", + [0x0c] = "multiport0_t/T/misc", + [0x0d] = "multiport0_t/T/pm", + [0x0e] = "multiport0_t/T/tke", + [0x0f] = "RESERVED", + [0x10] = "multiport1_t/T/hsm", + [0x11] = "multiport1_t/T/vic0", + [0x12] = "multiport1_t/T/vic1", + [0x13] = "ast0_t/T/0", + [0x14] = "ast1_t/T/0", + [0x15] = "cbb_t/T/0", + [0x16] = "cpu_t/T/0", + [0x17] = "sce_noc_firewall/T/0", + [0x18] = "svc_t/T/0", + [0x19] = "RESERVED", + [0x1a] = "RESERVED", + [0x1b] = "RESERVED", + [0x1c] = "RESERVED", + [0x1d] = "RESERVED", + [0x1e] = "RESERVED", + [0x1f] = "RESERVED" +}; + +/* + * Fields of SCE/RCE NOC lookup table: + * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress, + * Targ mapping, Targ localAddress + * ---------------------------------------------------------------------------- + */ +static const struct tegra194_cbb_aperture tegra194_scenoc_apert_lookup[] = { + { 0x0, 0x16, 0x0, 0, 0x0b400000, 0, 0x0b400000 }, + { 0x0, 0x16, 0x1, 0, 0x0bc00000, 1, 0x0bc00000 }, + { 0x0, 0x0, 0x0, 0, 0x0b000000, 0, 0x00000000 }, + { 0x0, 0x0, 0x1, 0, 0x0b800000, 1, 0x00000000 }, + { 0x0, 0x1, 0x0, 0, 0x20de0000, 3, 0x000e0000 }, + { 0x0, 0x1, 0x1, 0, 0x210e0000, 7, 0x000e0000 }, + { 0x0, 0x1, 0x2, 0, 0x20dc0000, 2, 0x000c0000 }, + { 0x0, 0x1, 0x3, 0, 0x210c0000, 6, 0x000c0000 }, + { 0x0, 0x1, 0x4, 0, 0x20d80000, 1, 0x00080000 }, + { 0x0, 0x1, 0x5, 0, 0x21080000, 5, 0x00080000 }, + { 0x0, 0x1, 0x6, 0, 0x20d00000, 0, 0x00000000 }, + { 0x0, 0x1, 0x7, 0, 0x21000000, 4, 0x00000000 }, + { 0x0, 0x2, 0x0, 0, 0x0b040000, 0, 0x00000000 }, + { 0x0, 0x2, 0x1, 0, 0x0b840000, 1, 0x00000000 }, + { 0x0, 0x3, 0x0, 0, 0x0b230000, 0, 0x00000000 }, + { 0x0, 0x3, 0x1, 0, 0x0ba30000, 1, 0x00000000 }, + { 0x0, 0x4, 0x0, 0, 0x0b050000, 0, 0x00000000 }, + { 0x0, 0x4, 0x1, 0, 0x0b850000, 1, 0x00000000 }, + { 0x0, 0x5, 0x0, 0, 0x0b060000, 0, 0x00000000 }, + { 0x0, 0x5, 0x1, 0, 0x0b070000, 1, 0x00010000 }, + { 0x0, 0x5, 0x2, 0, 0x0b080000, 2, 0x00020000 }, + { 0x0, 0x5, 0x3, 0, 0x0b090000, 3, 0x00030000 }, + { 0x0, 0x5, 0x4, 0, 0x0b0a0000, 4, 0x00040000 }, + { 0x0, 0x5, 0x5, 0, 0x0b0b0000, 5, 0x00050000 }, + { 0x0, 0x5, 0x6, 0, 0x0b0c0000, 6, 0x00060000 }, + { 0x0, 0x5, 0x7, 0, 0x0b0d0000, 7, 0x00070000 }, + { 0x0, 0x5, 0x8, 0, 0x0b0e0000, 8, 0x00080000 }, + { 0x0, 0x5, 0x9, 0, 0x0b860000, 9, 0x00000000 }, + { 0x0, 0x5, 0xa, 0, 0x0b870000, 10, 0x00010000 }, + { 0x0, 0x5, 0xb, 0, 0x0b880000, 11, 0x00020000 }, + { 0x0, 0x5, 0xc, 0, 0x0b890000, 12, 0x00030000 }, + { 0x0, 0x5, 0xd, 0, 0x0b8a0000, 13, 0x00040000 }, + { 0x0, 0x5, 0xe, 0, 0x0b8b0000, 14, 0x00050000 }, + { 0x0, 0x5, 0xf, 0, 0x0b8c0000, 15, 0x00060000 }, + { 0x0, 0x5, 0x10, 0, 0x0b8d0000, 16, 0x00070000 }, + { 0x0, 0x5, 0x11, 0, 0x0b8e0000, 17, 0x00080000 }, + { 0x0, 0x6, 0x0, 0, 0x0b650000, 0, 0x00000000 }, + { 0x0, 0x6, 0x1, 0, 0x0be50000, 1, 0x00000000 }, + { 0x0, 0x7, 0x0, 0, 0x20df0000, 0, 0x00000000 }, + { 0x0, 0x7, 0x1, 0, 0x210f0000, 1, 0x00000000 }, + { 0x0, 0x8, 0x0, 0, 0x0b3e0000, 0, 0x00000000 }, + { 0x0, 0x8, 0x1, 0, 0x0bbe0000, 1, 0x00000000 }, + { 0x0, 0x9, 0x0, 0, 0x0b3d0000, 0, 0x00000000 }, + { 0x0, 0x9, 0x1, 0, 0x0bbd0000, 1, 0x00000000 }, + { 0x0, 0xa, 0x0, 0, 0x0b1e0000, 0, 0x00000000 }, + { 0x0, 0xa, 0x1, 0, 0x0b9e0000, 1, 0x00000000 }, + { 0x0, 0xb, 0x0, 0, 0x0b150000, 0, 0x00000000 }, + { 0x0, 0xb, 0x1, 0, 0x0b160000, 1, 0x00010000 }, + { 0x0, 0xb, 0x2, 0, 0x0b170000, 2, 0x00020000 }, + { 0x0, 0xb, 0x3, 0, 0x0b180000, 3, 0x00030000 }, + { 0x0, 0xb, 0x4, 0, 0x0b190000, 4, 0x00040000 }, + { 0x0, 0xb, 0x5, 0, 0x0b1a0000, 5, 0x00050000 }, + { 0x0, 0xb, 0x6, 0, 0x0b1b0000, 6, 0x00060000 }, + { 0x0, 0xb, 0x7, 0, 0x0b1c0000, 7, 0x00070000 }, + { 0x0, 0xb, 0x8, 0, 0x0b1d0000, 8, 0x00080000 }, + { 0x0, 0xb, 0x9, 0, 0x0b950000, 9, 0x00000000 }, + { 0x0, 0xb, 0xa, 0, 0x0b960000, 10, 0x00010000 }, + { 0x0, 0xb, 0xb, 0, 0x0b970000, 11, 0x00020000 }, + { 0x0, 0xb, 0xc, 0, 0x0b980000, 12, 0x00030000 }, + { 0x0, 0xb, 0xd, 0, 0x0b990000, 13, 0x00040000 }, + { 0x0, 0xb, 0xe, 0, 0x0b9a0000, 14, 0x00050000 }, + { 0x0, 0xb, 0xf, 0, 0x0b9b0000, 15, 0x00060000 }, + { 0x0, 0xb, 0x10, 0, 0x0b9c0000, 16, 0x00070000 }, + { 0x0, 0xb, 0x11, 0, 0x0b9d0000, 17, 0x00080000 }, + { 0x0, 0xc, 0x0, 0, 0x0b660000, 0, 0x00000000 }, + { 0x0, 0xc, 0x1, 0, 0x0be60000, 1, 0x00000000 }, + { 0x0, 0xd, 0x0, 0, 0x0b1f0000, 0, 0x00000000 }, + { 0x0, 0xd, 0x1, 0, 0x0b200000, 1, 0x00010000 }, + { 0x0, 0xd, 0x2, 0, 0x0b210000, 2, 0x00020000 }, + { 0x0, 0xd, 0x3, 0, 0x0b220000, 3, 0x00030000 }, + { 0x0, 0xd, 0x4, 0, 0x0b9f0000, 4, 0x00000000 }, + { 0x0, 0xd, 0x5, 0, 0x0ba00000, 5, 0x00010000 }, + { 0x0, 0xd, 0x6, 0, 0x0ba10000, 6, 0x00020000 }, + { 0x0, 0xd, 0x7, 0, 0x0ba20000, 7, 0x00030000 }, + { 0x0, 0xe, 0x0, 0, 0x0b0f0000, 0, 0x00000000 }, + { 0x0, 0xe, 0x1, 0, 0x0b100000, 1, 0x00010000 }, + { 0x0, 0xe, 0x2, 0, 0x0b110000, 2, 0x00020000 }, + { 0x0, 0xe, 0x3, 0, 0x0b120000, 3, 0x00030000 }, + { 0x0, 0xe, 0x4, 0, 0x0b130000, 4, 0x00040000 }, + { 0x0, 0xe, 0x5, 0, 0x0b140000, 5, 0x00050000 }, + { 0x0, 0xe, 0x6, 0, 0x0b8f0000, 6, 0x00000000 }, + { 0x0, 0xe, 0x7, 0, 0x0b900000, 7, 0x00010000 }, + { 0x0, 0xe, 0x8, 0, 0x0b910000, 8, 0x00020000 }, + { 0x0, 0xe, 0x9, 0, 0x0b920000, 9, 0x00030000 }, + { 0x0, 0xe, 0xa, 0, 0x0b930000, 10, 0x00040000 }, + { 0x0, 0xe, 0xb, 0, 0x0b940000, 11, 0x00050000 }, + { 0x0, 0x10, 0x0, 0, 0x0b240000, 0, 0x00000000 }, + { 0x0, 0x10, 0x1, 0, 0x0ba40000, 1, 0x00000000 }, + { 0x0, 0x11, 0x0, 0, 0x0b020000, 0, 0x00000000 }, + { 0x0, 0x11, 0x1, 0, 0x0b820000, 1, 0x00000000 }, + { 0x0, 0x12, 0x0, 0, 0x0b030000, 0, 0x00000000 }, + { 0x0, 0x12, 0x1, 0, 0x0b830000, 1, 0x00000000 }, + { 0x0, 0x17, 0x0, 0, 0x0b640000, 0, 0x00000000 }, + { 0x0, 0x17, 0x1, 0, 0x0be40000, 1, 0x00000000 }, + { 0x0, 0x18, 0x0, 0, 0x0b600000, 0, 0x00000000 }, + { 0x0, 0x18, 0x1, 0, 0x0be00000, 1, 0x00000000 }, + { 0x0, 0x18, 0x2, 0, 0x00000000, 0, 0x00000000 }, + { 0x0, 0x18, 0x3, 0, 0x00000000, 0, 0x00000000 }, + { 0x1, 0x13, 0x0, 0, 0x40000000, 0, 0x40000000 }, + { 0x1, 0x13, 0x1, 1, 0x80000000, 1, 0x80000000 }, + { 0x1, 0x13, 0x2, 0, 0x00000000, 0, 0x00000000 }, + { 0x2, 0x15, 0x0, 0, 0x20c00000, 8, 0x20c00000 }, + { 0x2, 0x15, 0x1, 0, 0x21100000, 22, 0x21100000 }, + { 0x2, 0x15, 0x2, 0, 0x20e00000, 9, 0x20e00000 }, + { 0x2, 0x15, 0x3, 0, 0x21200000, 23, 0x21200000 }, + { 0x2, 0x15, 0x4, 0, 0x20800000, 7, 0x20800000 }, + { 0x2, 0x15, 0x5, 0, 0x21400000, 24, 0x21400000 }, + { 0x2, 0x15, 0x6, 0, 0x0b000000, 18, 0x0b000000 }, + { 0x2, 0x15, 0x7, 0, 0x0b800000, 3, 0x0b800000 }, + { 0x2, 0x15, 0x8, 0, 0x20000000, 6, 0x20000000 }, + { 0x2, 0x15, 0x9, 0, 0x21800000, 25, 0x21800000 }, + { 0x2, 0x15, 0xa, 0, 0x0a000000, 2, 0x0a000000 }, + { 0x2, 0x15, 0xb, 0, 0x0a000000, 17, 0x0a000000 }, + { 0x2, 0x15, 0xc, 0, 0x20000000, 21, 0x20000000 }, + { 0x2, 0x15, 0xd, 0, 0x21000000, 10, 0x21000000 }, + { 0x2, 0x15, 0xe, 0, 0x08000000, 1, 0x08000000 }, + { 0x2, 0x15, 0xf, 0, 0x08000000, 16, 0x08000000 }, + { 0x2, 0x15, 0x10, 0, 0x22000000, 11, 0x22000000 }, + { 0x2, 0x15, 0x11, 0, 0x22000000, 26, 0x22000000 }, + { 0x2, 0x15, 0x12, 0, 0x0c000000, 4, 0x0c000000 }, + { 0x2, 0x15, 0x13, 0, 0x0c000000, 19, 0x0c000000 }, + { 0x2, 0x15, 0x14, 0, 0x24000000, 12, 0x24000000 }, + { 0x2, 0x15, 0x15, 0, 0x24000000, 27, 0x24000000 }, + { 0x2, 0x15, 0x16, 0, 0x00000000, 0, 0x00000000 }, + { 0x2, 0x15, 0x17, 0, 0x00000000, 15, 0x00000000 }, + { 0x2, 0x15, 0x18, 0, 0x28000000, 13, 0x28000000 }, + { 0x2, 0x15, 0x19, 0, 0x28000000, 28, 0x28000000 }, + { 0x2, 0x15, 0x1a, 0, 0x10000000, 5, 0x10000000 }, + { 0x2, 0x15, 0x1b, 0, 0x10000000, 20, 0x10000000 }, + { 0x2, 0x15, 0x1c, 0, 0x30000000, 14, 0x30000000 }, + { 0x2, 0x15, 0x1d, 0, 0x30000000, 29, 0x30000000 }, + { 0x2, 0x0, 0x0, 0, 0x0b000000, 0, 0x00000000 }, + { 0x2, 0x0, 0x1, 0, 0x0b800000, 1, 0x00000000 }, + { 0x2, 0x1, 0x0, 0, 0x20de0000, 3, 0x000e0000 }, + { 0x2, 0x1, 0x1, 0, 0x210e0000, 7, 0x000e0000 }, + { 0x2, 0x1, 0x2, 0, 0x20dc0000, 2, 0x000c0000 }, + { 0x2, 0x1, 0x3, 0, 0x210c0000, 6, 0x000c0000 }, + { 0x2, 0x1, 0x4, 0, 0x20d80000, 1, 0x00080000 }, + { 0x2, 0x1, 0x5, 0, 0x21080000, 5, 0x00080000 }, + { 0x2, 0x1, 0x6, 0, 0x20d00000, 0, 0x00000000 }, + { 0x2, 0x1, 0x7, 0, 0x21000000, 4, 0x00000000 }, + { 0x2, 0x2, 0x0, 0, 0x0b040000, 0, 0x00000000 }, + { 0x2, 0x2, 0x1, 0, 0x0b840000, 1, 0x00000000 }, + { 0x2, 0x3, 0x0, 0, 0x0b230000, 0, 0x00000000 }, + { 0x2, 0x3, 0x1, 0, 0x0ba30000, 1, 0x00000000 }, + { 0x2, 0x4, 0x0, 0, 0x0b050000, 0, 0x00000000 }, + { 0x2, 0x4, 0x1, 0, 0x0b850000, 1, 0x00000000 }, + { 0x2, 0x5, 0x0, 0, 0x0b060000, 0, 0x00000000 }, + { 0x2, 0x5, 0x1, 0, 0x0b070000, 1, 0x00010000 }, + { 0x2, 0x5, 0x2, 0, 0x0b080000, 2, 0x00020000 }, + { 0x2, 0x5, 0x3, 0, 0x0b090000, 3, 0x00030000 }, + { 0x2, 0x5, 0x4, 0, 0x0b0a0000, 4, 0x00040000 }, + { 0x2, 0x5, 0x5, 0, 0x0b0b0000, 5, 0x00050000 }, + { 0x2, 0x5, 0x6, 0, 0x0b0c0000, 6, 0x00060000 }, + { 0x2, 0x5, 0x7, 0, 0x0b0d0000, 7, 0x00070000 }, + { 0x2, 0x5, 0x8, 0, 0x0b0e0000, 8, 0x00080000 }, + { 0x2, 0x5, 0x9, 0, 0x0b860000, 9, 0x00000000 }, + { 0x2, 0x5, 0xa, 0, 0x0b870000, 10, 0x00010000 }, + { 0x2, 0x5, 0xb, 0, 0x0b880000, 11, 0x00020000 }, + { 0x2, 0x5, 0xc, 0, 0x0b890000, 12, 0x00030000 }, + { 0x2, 0x5, 0xd, 0, 0x0b8a0000, 13, 0x00040000 }, + { 0x2, 0x5, 0xe, 0, 0x0b8b0000, 14, 0x00050000 }, + { 0x2, 0x5, 0xf, 0, 0x0b8c0000, 15, 0x00060000 }, + { 0x2, 0x5, 0x10, 0, 0x0b8d0000, 16, 0x00070000 }, + { 0x2, 0x5, 0x11, 0, 0x0b8e0000, 17, 0x00080000 }, + { 0x2, 0x6, 0x0, 0, 0x0b650000, 0, 0x00000000 }, + { 0x2, 0x6, 0x1, 0, 0x0be50000, 1, 0x00000000 }, + { 0x2, 0x7, 0x0, 0, 0x20df0000, 0, 0x00000000 }, + { 0x2, 0x7, 0x1, 0, 0x210f0000, 1, 0x00000000 }, + { 0x2, 0x8, 0x0, 0, 0x0b3e0000, 0, 0x00000000 }, + { 0x2, 0x8, 0x1, 0, 0x0bbe0000, 1, 0x00000000 }, + { 0x2, 0x9, 0x0, 0, 0x0b3d0000, 0, 0x00000000 }, + { 0x2, 0x9, 0x1, 0, 0x0bbd0000, 1, 0x00000000 }, + { 0x2, 0xa, 0x0, 0, 0x0b1e0000, 0, 0x00000000 }, + { 0x2, 0xa, 0x1, 0, 0x0b9e0000, 1, 0x00000000 }, + { 0x2, 0xb, 0x0, 0, 0x0b150000, 0, 0x00000000 }, + { 0x2, 0xb, 0x1, 0, 0x0b160000, 1, 0x00010000 }, + { 0x2, 0xb, 0x2, 0, 0x0b170000, 2, 0x00020000 }, + { 0x2, 0xb, 0x3, 0, 0x0b180000, 3, 0x00030000 }, + { 0x2, 0xb, 0x4, 0, 0x0b190000, 4, 0x00040000 }, + { 0x2, 0xb, 0x5, 0, 0x0b1a0000, 5, 0x00050000 }, + { 0x2, 0xb, 0x6, 0, 0x0b1b0000, 6, 0x00060000 }, + { 0x2, 0xb, 0x7, 0, 0x0b1c0000, 7, 0x00070000 }, + { 0x2, 0xb, 0x8, 0, 0x0b1d0000, 8, 0x00080000 }, + { 0x2, 0xb, 0x9, 0, 0x0b950000, 9, 0x00000000 }, + { 0x2, 0xb, 0xa, 0, 0x0b960000, 10, 0x00010000 }, + { 0x2, 0xb, 0xb, 0, 0x0b970000, 11, 0x00020000 }, + { 0x2, 0xb, 0xc, 0, 0x0b980000, 12, 0x00030000 }, + { 0x2, 0xb, 0xd, 0, 0x0b990000, 13, 0x00040000 }, + { 0x2, 0xb, 0xe, 0, 0x0b9a0000, 14, 0x00050000 }, + { 0x2, 0xb, 0xf, 0, 0x0b9b0000, 15, 0x00060000 }, + { 0x2, 0xb, 0x10, 0, 0x0b9c0000, 16, 0x00070000 }, + { 0x2, 0xb, 0x11, 0, 0x0b9d0000, 17, 0x00080000 }, + { 0x2, 0xc, 0x0, 0, 0x0b660000, 0, 0x00000000 }, + { 0x2, 0xc, 0x1, 0, 0x0be60000, 1, 0x00000000 }, + { 0x2, 0xd, 0x0, 0, 0x0b1f0000, 0, 0x00000000 }, + { 0x2, 0xd, 0x1, 0, 0x0b200000, 1, 0x00010000 }, + { 0x2, 0xd, 0x2, 0, 0x0b210000, 2, 0x00020000 }, + { 0x2, 0xd, 0x3, 0, 0x0b220000, 3, 0x00030000 }, + { 0x2, 0xd, 0x4, 0, 0x0b9f0000, 4, 0x00000000 }, + { 0x2, 0xd, 0x5, 0, 0x0ba00000, 5, 0x00010000 }, + { 0x2, 0xd, 0x6, 0, 0x0ba10000, 6, 0x00020000 }, + { 0x2, 0xd, 0x7, 0, 0x0ba20000, 7, 0x00030000 }, + { 0x2, 0xe, 0x0, 0, 0x0b0f0000, 0, 0x00000000 }, + { 0x2, 0xe, 0x1, 0, 0x0b100000, 1, 0x00010000 }, + { 0x2, 0xe, 0x2, 0, 0x0b110000, 2, 0x00020000 }, + { 0x2, 0xe, 0x3, 0, 0x0b120000, 3, 0x00030000 }, + { 0x2, 0xe, 0x4, 0, 0x0b130000, 4, 0x00040000 }, + { 0x2, 0xe, 0x5, 0, 0x0b140000, 5, 0x00050000 }, + { 0x2, 0xe, 0x6, 0, 0x0b8f0000, 6, 0x00000000 }, + { 0x2, 0xe, 0x7, 0, 0x0b900000, 7, 0x00010000 }, + { 0x2, 0xe, 0x8, 0, 0x0b910000, 8, 0x00020000 }, + { 0x2, 0xe, 0x9, 0, 0x0b920000, 9, 0x00030000 }, + { 0x2, 0xe, 0xa, 0, 0x0b930000, 10, 0x00040000 }, + { 0x2, 0xe, 0xb, 0, 0x0b940000, 11, 0x00050000 }, + { 0x2, 0x10, 0x0, 0, 0x0b240000, 0, 0x00000000 }, + { 0x2, 0x10, 0x1, 0, 0x0ba40000, 1, 0x00000000 }, + { 0x2, 0x11, 0x0, 0, 0x0b020000, 0, 0x00000000 }, + { 0x2, 0x11, 0x1, 0, 0x0b820000, 1, 0x00000000 }, + { 0x2, 0x12, 0x0, 0, 0x0b030000, 0, 0x00000000 }, + { 0x2, 0x12, 0x1, 0, 0x0b830000, 1, 0x00000000 }, + { 0x2, 0x17, 0x0, 0, 0x0b640000, 0, 0x00000000 }, + { 0x2, 0x17, 0x1, 0, 0x0be40000, 1, 0x00000000 }, + { 0x2, 0x18, 0x0, 0, 0x0b600000, 0, 0x00000000 }, + { 0x2, 0x18, 0x1, 0, 0x0be00000, 1, 0x00000000 }, + { 0x2, 0x18, 0x2, 0, 0x00000000, 0, 0x00000000 }, + { 0x2, 0x18, 0x3, 0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x14, 0x0, 0, 0x40000000, 0, 0x40000000 }, + { 0x3, 0x14, 0x1, 1, 0x80000000, 1, 0x80000000 }, + { 0x3, 0x16, 0x0, 2, 0x0b400000, 0, 0x0b400000 }, + { 0x3, 0x16, 0x1, 2, 0x0bc00000, 1, 0x0bc00000 }, + { 0x3, 0x16, 0x2, 0, 0x00000000, 0, 0x00000000 }, + { 0x3, 0x16, 0x3, 0, 0x00000000, 0, 0x00000000 }, + { 0x4, 0x15, 0x0, 0, 0x20c00000, 8, 0x20c00000 }, + { 0x4, 0x15, 0x1, 0, 0x21100000, 22, 0x21100000 }, + { 0x4, 0x15, 0x2, 0, 0x20e00000, 9, 0x20e00000 }, + { 0x4, 0x15, 0x3, 0, 0x21200000, 23, 0x21200000 }, + { 0x4, 0x15, 0x4, 0, 0x20800000, 7, 0x20800000 }, + { 0x4, 0x15, 0x5, 0, 0x21400000, 24, 0x21400000 }, + { 0x4, 0x15, 0x6, 0, 0x0b000000, 18, 0x0b000000 }, + { 0x4, 0x15, 0x7, 0, 0x0b800000, 3, 0x0b800000 }, + { 0x4, 0x15, 0x8, 0, 0x20000000, 6, 0x20000000 }, + { 0x4, 0x15, 0x9, 0, 0x21800000, 25, 0x21800000 }, + { 0x4, 0x15, 0xa, 0, 0x0a000000, 2, 0x0a000000 }, + { 0x4, 0x15, 0xb, 0, 0x0a000000, 17, 0x0a000000 }, + { 0x4, 0x15, 0xc, 0, 0x20000000, 21, 0x20000000 }, + { 0x4, 0x15, 0xd, 0, 0x21000000, 10, 0x21000000 }, + { 0x4, 0x15, 0xe, 0, 0x08000000, 1, 0x08000000 }, + { 0x4, 0x15, 0xf, 0, 0x08000000, 16, 0x08000000 }, + { 0x4, 0x15, 0x10, 0, 0x22000000, 11, 0x22000000 }, + { 0x4, 0x15, 0x11, 0, 0x22000000, 26, 0x22000000 }, + { 0x4, 0x15, 0x12, 0, 0x0c000000, 4, 0x0c000000 }, + { 0x4, 0x15, 0x13, 0, 0x0c000000, 19, 0x0c000000 }, + { 0x4, 0x15, 0x14, 0, 0x24000000, 12, 0x24000000 }, + { 0x4, 0x15, 0x15, 0, 0x24000000, 27, 0x24000000 }, + { 0x4, 0x15, 0x16, 0, 0x00000000, 0, 0x00000000 }, + { 0x4, 0x15, 0x17, 0, 0x00000000, 15, 0x00000000 }, + { 0x4, 0x15, 0x18, 0, 0x28000000, 13, 0x28000000 }, + { 0x4, 0x15, 0x19, 0, 0x28000000, 28, 0x28000000 }, + { 0x4, 0x15, 0x1a, 0, 0x10000000, 5, 0x10000000 }, + { 0x4, 0x15, 0x1b, 0, 0x10000000, 20, 0x10000000 }, + { 0x4, 0x15, 0x1c, 0, 0x30000000, 14, 0x30000000 }, + { 0x4, 0x15, 0x1d, 0, 0x30000000, 29, 0x30000000 }, + { 0x4, 0x0, 0x0, 0, 0x0b000000, 0, 0x00000000 }, + { 0x4, 0x0, 0x1, 0, 0x0b800000, 1, 0x00000000 }, + { 0x4, 0x1, 0x0, 0, 0x20de0000, 3, 0x000e0000 }, + { 0x4, 0x1, 0x1, 0, 0x210e0000, 7, 0x000e0000 }, + { 0x4, 0x1, 0x2, 0, 0x20dc0000, 2, 0x000c0000 }, + { 0x4, 0x1, 0x3, 0, 0x210c0000, 6, 0x000c0000 }, + { 0x4, 0x1, 0x4, 0, 0x20d80000, 1, 0x00080000 }, + { 0x4, 0x1, 0x5, 0, 0x21080000, 5, 0x00080000 }, + { 0x4, 0x1, 0x6, 0, 0x20d00000, 0, 0x00000000 }, + { 0x4, 0x1, 0x7, 0, 0x21000000, 4, 0x00000000 }, + { 0x4, 0x2, 0x0, 0, 0x0b040000, 0, 0x00000000 }, + { 0x4, 0x2, 0x1, 0, 0x0b840000, 1, 0x00000000 }, + { 0x4, 0x3, 0x0, 0, 0x0b230000, 0, 0x00000000 }, + { 0x4, 0x3, 0x1, 0, 0x0ba30000, 1, 0x00000000 }, + { 0x4, 0x4, 0x0, 0, 0x0b050000, 0, 0x00000000 }, + { 0x4, 0x4, 0x1, 0, 0x0b850000, 1, 0x00000000 }, + { 0x4, 0x5, 0x0, 0, 0x0b060000, 0, 0x00000000 }, + { 0x4, 0x5, 0x1, 0, 0x0b070000, 1, 0x00010000 }, + { 0x4, 0x5, 0x2, 0, 0x0b080000, 2, 0x00020000 }, + { 0x4, 0x5, 0x3, 0, 0x0b090000, 3, 0x00030000 }, + { 0x4, 0x5, 0x4, 0, 0x0b0a0000, 4, 0x00040000 }, + { 0x4, 0x5, 0x5, 0, 0x0b0b0000, 5, 0x00050000 }, + { 0x4, 0x5, 0x6, 0, 0x0b0c0000, 6, 0x00060000 }, + { 0x4, 0x5, 0x7, 0, 0x0b0d0000, 7, 0x00070000 }, + { 0x4, 0x5, 0x8, 0, 0x0b0e0000, 8, 0x00080000 }, + { 0x4, 0x5, 0x9, 0, 0x0b860000, 9, 0x00000000 }, + { 0x4, 0x5, 0xa, 0, 0x0b870000, 10, 0x00010000 }, + { 0x4, 0x5, 0xb, 0, 0x0b880000, 11, 0x00020000 }, + { 0x4, 0x5, 0xc, 0, 0x0b890000, 12, 0x00030000 }, + { 0x4, 0x5, 0xd, 0, 0x0b8a0000, 13, 0x00040000 }, + { 0x4, 0x5, 0xe, 0, 0x0b8b0000, 14, 0x00050000 }, + { 0x4, 0x5, 0xf, 0, 0x0b8c0000, 15, 0x00060000 }, + { 0x4, 0x5, 0x10, 0, 0x0b8d0000, 16, 0x00070000 }, + { 0x4, 0x5, 0x11, 0, 0x0b8e0000, 17, 0x00080000 }, + { 0x4, 0x6, 0x0, 0, 0x0b650000, 0, 0x00000000 }, + { 0x4, 0x6, 0x1, 0, 0x0be50000, 1, 0x00000000 }, + { 0x4, 0x7, 0x0, 0, 0x20df0000, 0, 0x00000000 }, + { 0x4, 0x7, 0x1, 0, 0x210f0000, 1, 0x00000000 }, + { 0x4, 0x8, 0x0, 0, 0x0b3e0000, 0, 0x00000000 }, + { 0x4, 0x8, 0x1, 0, 0x0bbe0000, 1, 0x00000000 }, + { 0x4, 0x9, 0x0, 0, 0x0b3d0000, 0, 0x00000000 }, + { 0x4, 0x9, 0x1, 0, 0x0bbd0000, 1, 0x00000000 }, + { 0x4, 0xa, 0x0, 0, 0x0b1e0000, 0, 0x00000000 }, + { 0x4, 0xa, 0x1, 0, 0x0b9e0000, 1, 0x00000000 }, + { 0x4, 0xb, 0x0, 0, 0x0b150000, 0, 0x00000000 }, + { 0x4, 0xb, 0x1, 0, 0x0b160000, 1, 0x00010000 }, + { 0x4, 0xb, 0x2, 0, 0x0b170000, 2, 0x00020000 }, + { 0x4, 0xb, 0x3, 0, 0x0b180000, 3, 0x00030000 }, + { 0x4, 0xb, 0x4, 0, 0x0b190000, 4, 0x00040000 }, + { 0x4, 0xb, 0x5, 0, 0x0b1a0000, 5, 0x00050000 }, + { 0x4, 0xb, 0x6, 0, 0x0b1b0000, 6, 0x00060000 }, + { 0x4, 0xb, 0x7, 0, 0x0b1c0000, 7, 0x00070000 }, + { 0x4, 0xb, 0x8, 0, 0x0b1d0000, 8, 0x00080000 }, + { 0x4, 0xb, 0x9, 0, 0x0b950000, 9, 0x00000000 }, + { 0x4, 0xb, 0xa, 0, 0x0b960000, 10, 0x00010000 }, + { 0x4, 0xb, 0xb, 0, 0x0b970000, 11, 0x00020000 }, + { 0x4, 0xb, 0xc, 0, 0x0b980000, 12, 0x00030000 }, + { 0x4, 0xb, 0xd, 0, 0x0b990000, 13, 0x00040000 }, + { 0x4, 0xb, 0xe, 0, 0x0b9a0000, 14, 0x00050000 }, + { 0x4, 0xb, 0xf, 0, 0x0b9b0000, 15, 0x00060000 }, + { 0x4, 0xb, 0x10, 0, 0x0b9c0000, 16, 0x00070000 }, + { 0x4, 0xb, 0x11, 0, 0x0b9d0000, 17, 0x00080000 }, + { 0x4, 0xc, 0x0, 0, 0x0b660000, 0, 0x00000000 }, + { 0x4, 0xc, 0x1, 0, 0x0be60000, 1, 0x00000000 }, + { 0x4, 0xd, 0x0, 0, 0x0b1f0000, 0, 0x00000000 }, + { 0x4, 0xd, 0x1, 0, 0x0b200000, 1, 0x00010000 }, + { 0x4, 0xd, 0x2, 0, 0x0b210000, 2, 0x00020000 }, + { 0x4, 0xd, 0x3, 0, 0x0b220000, 3, 0x00030000 }, + { 0x4, 0xd, 0x4, 0, 0x0b9f0000, 4, 0x00000000 }, + { 0x4, 0xd, 0x5, 0, 0x0ba00000, 5, 0x00010000 }, + { 0x4, 0xd, 0x6, 0, 0x0ba10000, 6, 0x00020000 }, + { 0x4, 0xd, 0x7, 0, 0x0ba20000, 7, 0x00030000 }, + { 0x4, 0xe, 0x0, 0, 0x0b0f0000, 0, 0x00000000 }, + { 0x4, 0xe, 0x1, 0, 0x0b100000, 1, 0x00010000 }, + { 0x4, 0xe, 0x2, 0, 0x0b110000, 2, 0x00020000 }, + { 0x4, 0xe, 0x3, 0, 0x0b120000, 3, 0x00030000 }, + { 0x4, 0xe, 0x4, 0, 0x0b130000, 4, 0x00040000 }, + { 0x4, 0xe, 0x5, 0, 0x0b140000, 5, 0x00050000 }, + { 0x4, 0xe, 0x6, 0, 0x0b8f0000, 6, 0x00000000 }, + { 0x4, 0xe, 0x7, 0, 0x0b900000, 7, 0x00010000 }, + { 0x4, 0xe, 0x8, 0, 0x0b910000, 8, 0x00020000 }, + { 0x4, 0xe, 0x9, 0, 0x0b920000, 9, 0x00030000 }, + { 0x4, 0xe, 0xa, 0, 0x0b930000, 10, 0x00040000 }, + { 0x4, 0xe, 0xb, 0, 0x0b940000, 11, 0x00050000 }, + { 0x4, 0x10, 0x0, 0, 0x0b240000, 0, 0x00000000 }, + { 0x4, 0x10, 0x1, 0, 0x0ba40000, 1, 0x00000000 }, + { 0x4, 0x11, 0x0, 0, 0x0b020000, 0, 0x00000000 }, + { 0x4, 0x11, 0x1, 0, 0x0b820000, 1, 0x00000000 }, + { 0x4, 0x12, 0x0, 0, 0x0b030000, 0, 0x00000000 }, + { 0x4, 0x12, 0x1, 0, 0x0b830000, 1, 0x00000000 }, + { 0x4, 0x17, 0x0, 0, 0x0b640000, 0, 0x00000000 }, + { 0x4, 0x17, 0x1, 0, 0x0be40000, 1, 0x00000000 }, + { 0x4, 0x18, 0x0, 0, 0x0b600000, 0, 0x00000000 }, + { 0x4, 0x18, 0x1, 0, 0x0be00000, 1, 0x00000000 }, + { 0x4, 0x18, 0x2, 0, 0x00000000, 0, 0x00000000 }, + { 0x4, 0x18, 0x3, 0, 0x00000000, 0, 0x00000000 } +}; + +static void cbbcentralnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid) +{ + info->initflow = FIELD_GET(CBB_NOC_INITFLOW, routeid); + info->targflow = FIELD_GET(CBB_NOC_TARGFLOW, routeid); + info->targ_subrange = FIELD_GET(CBB_NOC_TARG_SUBRANGE, routeid); + info->seqid = FIELD_GET(CBB_NOC_SEQID, routeid); +} + +static void bpmpnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid) +{ + info->initflow = FIELD_GET(BPMP_NOC_INITFLOW, routeid); + info->targflow = FIELD_GET(BPMP_NOC_TARGFLOW, routeid); + info->targ_subrange = FIELD_GET(BPMP_NOC_TARG_SUBRANGE, routeid); + info->seqid = FIELD_GET(BPMP_NOC_SEQID, routeid); +} + +static void aonnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid) +{ + info->initflow = FIELD_GET(AON_NOC_INITFLOW, routeid); + info->targflow = FIELD_GET(AON_NOC_TARGFLOW, routeid); + info->targ_subrange = FIELD_GET(AON_NOC_TARG_SUBRANGE, routeid); + info->seqid = FIELD_GET(AON_NOC_SEQID, routeid); +} + +static void scenoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid) +{ + info->initflow = FIELD_GET(SCE_NOC_INITFLOW, routeid); + info->targflow = FIELD_GET(SCE_NOC_TARGFLOW, routeid); + info->targ_subrange = FIELD_GET(SCE_NOC_TARG_SUBRANGE, routeid); + info->seqid = FIELD_GET(SCE_NOC_SEQID, routeid); +} + +static void cbbcentralnoc_parse_userbits(struct tegra194_cbb_userbits *usrbits, u32 elog_5) +{ + usrbits->axcache = FIELD_GET(CBB_NOC_AXCACHE, elog_5); + usrbits->non_mod = FIELD_GET(CBB_NOC_NON_MOD, elog_5); + usrbits->axprot = FIELD_GET(CBB_NOC_AXPROT, elog_5); + usrbits->falconsec = FIELD_GET(CBB_NOC_FALCONSEC, elog_5); + usrbits->grpsec = FIELD_GET(CBB_NOC_GRPSEC, elog_5); + usrbits->vqc = FIELD_GET(CBB_NOC_VQC, elog_5); + usrbits->mstr_id = FIELD_GET(CBB_NOC_MSTR_ID, elog_5) - 1; + usrbits->axi_id = FIELD_GET(CBB_NOC_AXI_ID, elog_5); +} + +static void clusternoc_parse_userbits(struct tegra194_cbb_userbits *usrbits, u32 elog_5) +{ + usrbits->axcache = FIELD_GET(CLUSTER_NOC_AXCACHE, elog_5); + usrbits->axprot = FIELD_GET(CLUSTER_NOC_AXCACHE, elog_5); + usrbits->falconsec = FIELD_GET(CLUSTER_NOC_FALCONSEC, elog_5); + usrbits->grpsec = FIELD_GET(CLUSTER_NOC_GRPSEC, elog_5); + usrbits->vqc = FIELD_GET(CLUSTER_NOC_VQC, elog_5); + usrbits->mstr_id = FIELD_GET(CLUSTER_NOC_MSTR_ID, elog_5) - 1; +} + +static void tegra194_cbb_fault_enable(struct tegra_cbb *cbb) +{ + struct tegra194_cbb *priv = to_tegra194_cbb(cbb); + + writel(1, priv->regs + ERRLOGGER_0_FAULTEN_0); + writel(1, priv->regs + ERRLOGGER_1_FAULTEN_0); + writel(1, priv->regs + ERRLOGGER_2_FAULTEN_0); +} + +static void tegra194_cbb_stall_enable(struct tegra_cbb *cbb) +{ + struct tegra194_cbb *priv = to_tegra194_cbb(cbb); + + writel(1, priv->regs + ERRLOGGER_0_STALLEN_0); + writel(1, priv->regs + ERRLOGGER_1_STALLEN_0); + writel(1, priv->regs + ERRLOGGER_2_STALLEN_0); +} + +static void tegra194_cbb_error_clear(struct tegra_cbb *cbb) +{ + struct tegra194_cbb *priv = to_tegra194_cbb(cbb); + + writel(1, priv->regs + ERRLOGGER_0_ERRCLR_0); + writel(1, priv->regs + ERRLOGGER_1_ERRCLR_0); + writel(1, priv->regs + ERRLOGGER_2_ERRCLR_0); + dsb(sy); +} + +static u32 tegra194_cbb_get_status(struct tegra_cbb *cbb) +{ + struct tegra194_cbb *priv = to_tegra194_cbb(cbb); + u32 value; + + value = readl(priv->regs + ERRLOGGER_0_ERRVLD_0); + value |= (readl(priv->regs + ERRLOGGER_1_ERRVLD_0) << 1); + value |= (readl(priv->regs + ERRLOGGER_2_ERRVLD_0) << 2); + + dsb(sy); + return value; +} + +static u32 tegra194_axi2apb_status(void __iomem *addr) +{ + u32 value; + + value = readl(addr + DMAAPB_X_RAW_INTERRUPT_STATUS); + writel(0xffffffff, addr + DMAAPB_X_RAW_INTERRUPT_STATUS); + + return value; +} + +static bool tegra194_axi2apb_fatal(struct seq_file *file, unsigned int bridge, u32 status) +{ + bool is_fatal = true; + size_t i; + + for (i = 0; i < ARRAY_SIZE(tegra194_axi2apb_error); i++) { + if (status & BIT(i)) { + tegra_cbb_print_err(file, "\t AXI2APB_%d bridge error: %s\n", + bridge + 1, tegra194_axi2apb_error[i]); + if (strstr(tegra194_axi2apb_error[i], "Firewall")) + is_fatal = false; + } + } + + return is_fatal; +} + +/* + * Fetch InitlocalAddress from NOC Aperture lookup table + * using Targflow, Targsubrange + */ +static u32 get_init_localaddress(const struct tegra194_cbb_aperture *info, + const struct tegra194_cbb_aperture *aper, unsigned int max) +{ + unsigned int t_f = 0, t_sr = 0; + u32 addr = 0; + + for (t_f = 0; t_f < max; t_f++) { + if (aper[t_f].targflow == info->targflow) { + t_sr = t_f; + + do { + if (aper[t_sr].targ_subrange == info->targ_subrange) { + addr = aper[t_sr].init_localaddress; + return addr; + } + + if (t_sr >= max) + return 0; + + t_sr++; + } while (aper[t_sr].targflow == aper[t_sr - 1].targflow); + + t_f = t_sr; + } + } + + return addr; +} + +static void print_errlog5(struct seq_file *file, struct tegra194_cbb *cbb) +{ + struct tegra194_cbb_userbits userbits; + + cbb->noc->parse_userbits(&userbits, cbb->errlog5); + + if (!strcmp(cbb->noc->name, "cbb-noc")) { + tegra_cbb_print_err(file, "\t Non-Modify\t\t: %#x\n", userbits.non_mod); + tegra_cbb_print_err(file, "\t AXI ID\t\t: %#x\n", userbits.axi_id); + } + + tegra_cbb_print_err(file, "\t Master ID\t\t: %s\n", + cbb->noc->master_id[userbits.mstr_id]); + tegra_cbb_print_err(file, "\t Security Group(GRPSEC): %#x\n", userbits.grpsec); + tegra_cbb_print_cache(file, userbits.axcache); + tegra_cbb_print_prot(file, userbits.axprot); + tegra_cbb_print_err(file, "\t FALCONSEC\t\t: %#x\n", userbits.falconsec); + tegra_cbb_print_err(file, "\t Virtual Queuing Channel(VQC): %#x\n", userbits.vqc); +} + +/* + * Fetch Base Address/InitlocalAddress from NOC aperture lookup table using TargFlow & + * Targ_subRange extracted from RouteId. Perform address reconstruction as below: + * + * Address = Base Address + (ErrLog3 + ErrLog4) + */ +static void +print_errlog3_4(struct seq_file *file, u32 errlog3, u32 errlog4, + const struct tegra194_cbb_aperture *info, + const struct tegra194_cbb_aperture *aperture, unsigned int max) +{ + u64 addr = (u64)errlog4 << 32 | errlog3; + + /* + * If errlog4[7] = "1", then it's a joker entry. Joker entries are a rare phenomenon and + * such addresses are not reliable. Debugging should be done using only the RouteId + * information. + */ + if (errlog4 & 0x80) + tegra_cbb_print_err(file, "\t debug using RouteId alone as below address is a " + "joker entry and not reliable"); + + addr += get_init_localaddress(info, aperture, max); + + tegra_cbb_print_err(file, "\t Address accessed\t: %#llx\n", addr); +} + +/* + * Get RouteId from ErrLog1+ErrLog2 registers and fetch values of + * InitFlow, TargFlow, Targ_subRange and SeqId values from RouteId + */ +static void +print_errlog1_2(struct seq_file *file, struct tegra194_cbb *cbb, + struct tegra194_cbb_aperture *info) +{ + u64 routeid = (u64)cbb->errlog2 << 32 | cbb->errlog1; + u32 seqid = 0; + + tegra_cbb_print_err(file, "\t RouteId\t\t: %#llx\n", routeid); + + cbb->noc->parse_routeid(info, routeid); + + tegra_cbb_print_err(file, "\t InitFlow\t\t: %s\n", + cbb->noc->routeid_initflow[info->initflow]); + + tegra_cbb_print_err(file, "\t Targflow\t\t: %s\n", + cbb->noc->routeid_targflow[info->targflow]); + + tegra_cbb_print_err(file, "\t TargSubRange\t\t: %d\n", info->targ_subrange); + tegra_cbb_print_err(file, "\t SeqId\t\t\t: %d\n", seqid); +} + +/* + * Print transcation type, error code and description from ErrLog0 for all + * errors. For NOC slave errors, all relevant error info is printed using + * ErrLog0 only. But additional information is printed for errors from + * APB slaves because for them: + * - All errors are logged as SLV(slave) errors due to APB having only single + * bit pslverr to report all errors. + * - Exact cause is printed by reading DMAAPB_X_RAW_INTERRUPT_STATUS register. + * - The driver prints information showing AXI2APB bridge and exact error + * only if there is error in any AXI2APB slave. + * - There is still no way to disambiguate a DEC error from SLV error type. + */ +static bool print_errlog0(struct seq_file *file, struct tegra194_cbb *cbb) +{ + struct tegra194_cbb_packet_header hdr; + bool is_fatal = true; + + hdr.lock = cbb->errlog0 & 0x1; + hdr.opc = FIELD_GET(CBB_ERR_OPC, cbb->errlog0); + hdr.errcode = FIELD_GET(CBB_ERR_ERRCODE, cbb->errlog0); + hdr.len1 = FIELD_GET(CBB_ERR_LEN1, cbb->errlog0); + hdr.format = (cbb->errlog0 >> 31); + + tegra_cbb_print_err(file, "\t Transaction Type\t: %s\n", + tegra194_cbb_trantype[hdr.opc]); + tegra_cbb_print_err(file, "\t Error Code\t\t: %s\n", + tegra194_cbb_errors[hdr.errcode].code); + tegra_cbb_print_err(file, "\t Error Source\t\t: %s\n", + tegra194_cbb_errors[hdr.errcode].source); + tegra_cbb_print_err(file, "\t Error Description\t: %s\n", + tegra194_cbb_errors[hdr.errcode].desc); + + /* + * Do not crash system for errors which are only notifications to indicate a transaction + * was not allowed to be attempted. + */ + if (!strcmp(tegra194_cbb_errors[hdr.errcode].code, "SEC") || + !strcmp(tegra194_cbb_errors[hdr.errcode].code, "DEC") || + !strcmp(tegra194_cbb_errors[hdr.errcode].code, "UNS") || + !strcmp(tegra194_cbb_errors[hdr.errcode].code, "DISC")) { + is_fatal = false; + } else if (!strcmp(tegra194_cbb_errors[hdr.errcode].code, "SLV") && + cbb->num_bridges > 0) { + unsigned int i; + u32 status; + + /* For all SLV errors, read DMAAPB_X_RAW_INTERRUPT_STATUS + * register to get error status for all AXI2APB bridges. + * Print bridge details if a bit is set in a bridge's + * status register due to error in a APB slave connected + * to that bridge. For other NOC slaves, none of the status + * register will be set. + */ + + for (i = 0; i < cbb->num_bridges; i++) { + status = tegra194_axi2apb_status(cbb->bridges[i].base); + + if (status) + is_fatal = tegra194_axi2apb_fatal(file, i, status); + } + } + + tegra_cbb_print_err(file, "\t Packet header Lock\t: %d\n", hdr.lock); + tegra_cbb_print_err(file, "\t Packet header Len1\t: %d\n", hdr.len1); + + if (hdr.format) + tegra_cbb_print_err(file, "\t NOC protocol version\t: %s\n", + "version >= 2.7"); + else + tegra_cbb_print_err(file, "\t NOC protocol version\t: %s\n", + "version < 2.7"); + + return is_fatal; +} + +/* + * Print debug information about failed transaction using + * ErrLog registers of error loggger having ErrVld set + */ +static bool print_errloggerX_info(struct seq_file *file, struct tegra194_cbb *cbb, + int errloggerX) +{ + struct tegra194_cbb_aperture info = { 0, }; + bool is_fatal = true; + + tegra_cbb_print_err(file, "\tError Logger\t\t: %d\n", errloggerX); + + if (errloggerX == 0) { + cbb->errlog0 = readl(cbb->regs + ERRLOGGER_0_ERRLOG0_0); + cbb->errlog1 = readl(cbb->regs + ERRLOGGER_0_ERRLOG1_0); + cbb->errlog2 = readl(cbb->regs + ERRLOGGER_0_RSVD_00_0); + cbb->errlog3 = readl(cbb->regs + ERRLOGGER_0_ERRLOG3_0); + cbb->errlog4 = readl(cbb->regs + ERRLOGGER_0_ERRLOG4_0); + cbb->errlog5 = readl(cbb->regs + ERRLOGGER_0_ERRLOG5_0); + } else if (errloggerX == 1) { + cbb->errlog0 = readl(cbb->regs + ERRLOGGER_1_ERRLOG0_0); + cbb->errlog1 = readl(cbb->regs + ERRLOGGER_1_ERRLOG1_0); + cbb->errlog2 = readl(cbb->regs + ERRLOGGER_1_RSVD_00_0); + cbb->errlog3 = readl(cbb->regs + ERRLOGGER_1_ERRLOG3_0); + cbb->errlog4 = readl(cbb->regs + ERRLOGGER_1_ERRLOG4_0); + cbb->errlog5 = readl(cbb->regs + ERRLOGGER_1_ERRLOG5_0); + } else if (errloggerX == 2) { + cbb->errlog0 = readl(cbb->regs + ERRLOGGER_2_ERRLOG0_0); + cbb->errlog1 = readl(cbb->regs + ERRLOGGER_2_ERRLOG1_0); + cbb->errlog2 = readl(cbb->regs + ERRLOGGER_2_RSVD_00_0); + cbb->errlog3 = readl(cbb->regs + ERRLOGGER_2_ERRLOG3_0); + cbb->errlog4 = readl(cbb->regs + ERRLOGGER_2_ERRLOG4_0); + cbb->errlog5 = readl(cbb->regs + ERRLOGGER_2_ERRLOG5_0); + } + + tegra_cbb_print_err(file, "\tErrLog0\t\t\t: %#x\n", cbb->errlog0); + is_fatal = print_errlog0(file, cbb); + + tegra_cbb_print_err(file, "\tErrLog1\t\t\t: %#x\n", cbb->errlog1); + tegra_cbb_print_err(file, "\tErrLog2\t\t\t: %#x\n", cbb->errlog2); + print_errlog1_2(file, cbb, &info); + + tegra_cbb_print_err(file, "\tErrLog3\t\t\t: %#x\n", cbb->errlog3); + tegra_cbb_print_err(file, "\tErrLog4\t\t\t: %#x\n", cbb->errlog4); + print_errlog3_4(file, cbb->errlog3, cbb->errlog4, &info, cbb->noc->noc_aperture, + cbb->noc->max_aperture); + + tegra_cbb_print_err(file, "\tErrLog5\t\t\t: %#x\n", cbb->errlog5); + + if (cbb->errlog5) + print_errlog5(file, cbb); + + return is_fatal; +} + +static bool print_errlog(struct seq_file *file, struct tegra194_cbb *cbb, u32 errvld) +{ + bool is_fatal = true; + + pr_crit("**************************************\n"); + pr_crit("CPU:%d, Error:%s\n", smp_processor_id(), cbb->noc->name); + + if (errvld & 0x1) + is_fatal = print_errloggerX_info(file, cbb, 0); + else if (errvld & 0x2) + is_fatal = print_errloggerX_info(file, cbb, 1); + else if (errvld & 0x4) + is_fatal = print_errloggerX_info(file, cbb, 2); + + tegra_cbb_error_clear(&cbb->base); + tegra_cbb_print_err(file, "\t**************************************\n"); + return is_fatal; +} + +#ifdef CONFIG_DEBUG_FS +static DEFINE_MUTEX(cbb_err_mutex); + +static int tegra194_cbb_debugfs_show(struct tegra_cbb *cbb, struct seq_file *file, void *data) +{ + struct tegra_cbb *noc; + + mutex_lock(&cbb_err_mutex); + + list_for_each_entry(noc, &cbb_list, node) { + struct tegra194_cbb *priv = to_tegra194_cbb(noc); + u32 status; + + status = tegra_cbb_get_status(noc); + if (status) + print_errlog(file, priv, status); + } + + mutex_unlock(&cbb_err_mutex); + + return 0; +} +#endif + +/* + * Handler for CBB errors from different initiators + */ +static irqreturn_t tegra194_cbb_err_isr(int irq, void *data) +{ + bool is_inband_err = false, is_fatal = false; + //struct tegra194_cbb *cbb = data; + struct tegra_cbb *noc; + unsigned long flags; + u8 mstr_id = 0; + + spin_lock_irqsave(&cbb_lock, flags); + + /* XXX only process interrupts for "cbb" instead of iterating over all NOCs? */ + list_for_each_entry(noc, &cbb_list, node) { + struct tegra194_cbb *priv = to_tegra194_cbb(noc); + u32 status = 0; + + status = tegra_cbb_get_status(noc); + + if (status && ((irq == priv->sec_irq) || (irq == priv->nonsec_irq))) { + tegra_cbb_print_err(NULL, "CPU:%d, Error: %s@%llx, irq=%d\n", + smp_processor_id(), priv->noc->name, priv->res->start, + irq); + + mstr_id = FIELD_GET(USRBITS_MSTR_ID, priv->errlog5) - 1; + is_fatal = print_errlog(NULL, priv, status); + + /* + * If illegal request is from CCPLEX(0x1) + * initiator then call BUG() to crash system. + */ + if ((mstr_id == 0x1) && priv->noc->erd_mask_inband_err) + is_inband_err = 1; + } + } + + spin_unlock_irqrestore(&cbb_lock, flags); + + if (is_inband_err) { + if (is_fatal) + BUG(); + else + WARN(true, "Warning due to CBB Error\n"); + } + + return IRQ_HANDLED; +} + +/* + * Register handler for CBB_NONSECURE & CBB_SECURE interrupts + * for reporting CBB errors + */ +static int tegra194_cbb_interrupt_enable(struct tegra_cbb *cbb) +{ + struct tegra194_cbb *priv = to_tegra194_cbb(cbb); + struct device *dev = cbb->dev; + int err; + + if (priv->sec_irq) { + err = devm_request_irq(dev, priv->sec_irq, tegra194_cbb_err_isr, 0, dev_name(dev), + priv); + if (err) { + dev_err(dev, "failed to register interrupt %u: %d\n", priv->sec_irq, err); + return err; + } + } + + if (priv->nonsec_irq) { + err = devm_request_irq(dev, priv->nonsec_irq, tegra194_cbb_err_isr, 0, + dev_name(dev), priv); + if (err) { + dev_err(dev, "failed to register interrupt %u: %d\n", priv->nonsec_irq, + err); + return err; + } + } + + return 0; +} + +static void tegra194_cbb_error_enable(struct tegra_cbb *cbb) +{ + /* + * Set “StallEn=1” to enable queuing of error packets till + * first is served & cleared + */ + tegra_cbb_stall_enable(cbb); + + /* set “FaultEn=1” to enable error reporting signal “Fault” */ + tegra_cbb_fault_enable(cbb); +} + +static const struct tegra_cbb_ops tegra194_cbb_ops = { + .get_status = tegra194_cbb_get_status, + .error_clear = tegra194_cbb_error_clear, + .fault_enable = tegra194_cbb_fault_enable, + .stall_enable = tegra194_cbb_stall_enable, + .error_enable = tegra194_cbb_error_enable, + .interrupt_enable = tegra194_cbb_interrupt_enable, +#ifdef CONFIG_DEBUG_FS + .debugfs_show = tegra194_cbb_debugfs_show, +#endif +}; + +static struct tegra194_cbb_noc_data tegra194_cbb_central_noc_data = { + .name = "cbb-noc", + .erd_mask_inband_err = true, + .master_id = tegra194_master_id, + .noc_aperture = tegra194_cbbcentralnoc_apert_lookup, + .max_aperture = ARRAY_SIZE(tegra194_cbbcentralnoc_apert_lookup), + .routeid_initflow = tegra194_cbbcentralnoc_routeid_initflow, + .routeid_targflow = tegra194_cbbcentralnoc_routeid_targflow, + .parse_routeid = cbbcentralnoc_parse_routeid, + .parse_userbits = cbbcentralnoc_parse_userbits +}; + +static struct tegra194_cbb_noc_data tegra194_aon_noc_data = { + .name = "aon-noc", + .erd_mask_inband_err = false, + .master_id = tegra194_master_id, + .noc_aperture = tegra194_aonnoc_aperture_lookup, + .max_aperture = ARRAY_SIZE(tegra194_aonnoc_aperture_lookup), + .routeid_initflow = tegra194_aonnoc_routeid_initflow, + .routeid_targflow = tegra194_aonnoc_routeid_targflow, + .parse_routeid = aonnoc_parse_routeid, + .parse_userbits = clusternoc_parse_userbits +}; + +static struct tegra194_cbb_noc_data tegra194_bpmp_noc_data = { + .name = "bpmp-noc", + .erd_mask_inband_err = false, + .master_id = tegra194_master_id, + .noc_aperture = tegra194_bpmpnoc_apert_lookup, + .max_aperture = ARRAY_SIZE(tegra194_bpmpnoc_apert_lookup), + .routeid_initflow = tegra194_bpmpnoc_routeid_initflow, + .routeid_targflow = tegra194_bpmpnoc_routeid_targflow, + .parse_routeid = bpmpnoc_parse_routeid, + .parse_userbits = clusternoc_parse_userbits +}; + +static struct tegra194_cbb_noc_data tegra194_rce_noc_data = { + .name = "rce-noc", + .erd_mask_inband_err = false, + .master_id = tegra194_master_id, + .noc_aperture = tegra194_scenoc_apert_lookup, + .max_aperture = ARRAY_SIZE(tegra194_scenoc_apert_lookup), + .routeid_initflow = tegra194_scenoc_routeid_initflow, + .routeid_targflow = tegra194_scenoc_routeid_targflow, + .parse_routeid = scenoc_parse_routeid, + .parse_userbits = clusternoc_parse_userbits +}; + +static struct tegra194_cbb_noc_data tegra194_sce_noc_data = { + .name = "sce-noc", + .erd_mask_inband_err = false, + .master_id = tegra194_master_id, + .noc_aperture = tegra194_scenoc_apert_lookup, + .max_aperture = ARRAY_SIZE(tegra194_scenoc_apert_lookup), + .routeid_initflow = tegra194_scenoc_routeid_initflow, + .routeid_targflow = tegra194_scenoc_routeid_targflow, + .parse_routeid = scenoc_parse_routeid, + .parse_userbits = clusternoc_parse_userbits +}; + +static const struct of_device_id tegra194_cbb_match[] = { + { .compatible = "nvidia,tegra194-cbb-noc", .data = &tegra194_cbb_central_noc_data }, + { .compatible = "nvidia,tegra194-aon-noc", .data = &tegra194_aon_noc_data }, + { .compatible = "nvidia,tegra194-bpmp-noc", .data = &tegra194_bpmp_noc_data }, + { .compatible = "nvidia,tegra194-rce-noc", .data = &tegra194_rce_noc_data }, + { .compatible = "nvidia,tegra194-sce-noc", .data = &tegra194_sce_noc_data }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, tegra194_cbb_match); + +static int tegra194_cbb_get_bridges(struct tegra194_cbb *cbb, struct device_node *np) +{ + struct tegra_cbb *entry; + struct resource res; + unsigned long flags; + unsigned int i; + int err; + + spin_lock_irqsave(&cbb_lock, flags); + + list_for_each_entry(entry, &cbb_list, node) { + struct tegra194_cbb *priv = to_tegra194_cbb(entry); + + if (priv->bridges) { + cbb->num_bridges = priv->num_bridges; + cbb->bridges = priv->bridges; + break; + } + } + + spin_unlock_irqrestore(&cbb_lock, flags); + + if (!cbb->bridges) { + while (of_address_to_resource(np, cbb->num_bridges, &res) == 0) + cbb->num_bridges++; + + cbb->bridges = devm_kcalloc(cbb->base.dev, cbb->num_bridges, + sizeof(*cbb->bridges), GFP_KERNEL); + if (!cbb->bridges) + return -ENOMEM; + + for (i = 0; i < cbb->num_bridges; i++) { + err = of_address_to_resource(np, i, &cbb->bridges[i].res); + if (err < 0) + return err; + + cbb->bridges[i].base = devm_ioremap_resource(cbb->base.dev, + &cbb->bridges[i].res); + if (IS_ERR(cbb->bridges[i].base)) { + dev_err(cbb->base.dev, "failed to map AXI2APB range\n"); + return PTR_ERR(cbb->bridges[i].base); + } + } + } + + if (cbb->num_bridges > 0) { + dev_dbg(cbb->base.dev, "AXI2APB bridge info present:\n"); + + for (i = 0; i < cbb->num_bridges; i++) + dev_dbg(cbb->base.dev, " %u: %pR\n", i, &cbb->bridges[i].res); + } + + return 0; +} + +static int tegra194_cbb_probe(struct platform_device *pdev) +{ + const struct tegra194_cbb_noc_data *noc; + struct tegra194_cbb *cbb; + struct device_node *np; + unsigned long flags; + int err; + + noc = of_device_get_match_data(&pdev->dev); + + if (noc->erd_mask_inband_err) { + /* + * Set Error Response Disable(ERD) bit to mask SError/inband + * error and only trigger interrupts for illegal access from + * CCPLEX initiator. + */ + err = tegra194_miscreg_mask_serror(); + if (err) { + dev_err(&pdev->dev, "couldn't mask inband errors\n"); + return err; + } + } + + cbb = devm_kzalloc(&pdev->dev, sizeof(*cbb), GFP_KERNEL); + if (!cbb) + return -ENOMEM; + + INIT_LIST_HEAD(&cbb->base.node); + cbb->base.ops = &tegra194_cbb_ops; + cbb->base.dev = &pdev->dev; + cbb->noc = noc; + + cbb->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &cbb->res); + if (IS_ERR(cbb->regs)) + return PTR_ERR(cbb->regs); + + err = tegra_cbb_get_irq(pdev, &cbb->nonsec_irq, &cbb->sec_irq); + if (err) + return err; + + np = of_parse_phandle(pdev->dev.of_node, "nvidia,axi2apb", 0); + if (np) { + err = tegra194_cbb_get_bridges(cbb, np); + of_node_put(np); + if (err < 0) + return err; + } + + platform_set_drvdata(pdev, cbb); + + spin_lock_irqsave(&cbb_lock, flags); + list_add(&cbb->base.node, &cbb_list); + spin_unlock_irqrestore(&cbb_lock, flags); + + return tegra_cbb_register(&cbb->base); +} + +static int tegra194_cbb_remove(struct platform_device *pdev) +{ + struct tegra194_cbb *cbb = platform_get_drvdata(pdev); + struct tegra_cbb *noc, *tmp; + unsigned long flags; + + spin_lock_irqsave(&cbb_lock, flags); + + list_for_each_entry_safe(noc, tmp, &cbb_list, node) { + struct tegra194_cbb *priv = to_tegra194_cbb(noc); + + if (cbb->res->start == priv->res->start) { + list_del(&noc->node); + break; + } + } + + spin_unlock_irqrestore(&cbb_lock, flags); + + return 0; +} + +static int __maybe_unused tegra194_cbb_resume_noirq(struct device *dev) +{ + struct tegra194_cbb *cbb = dev_get_drvdata(dev); + + tegra194_cbb_error_enable(&cbb->base); + dsb(sy); + + dev_dbg(dev, "%s resumed\n", cbb->noc->name); + return 0; +} + +static const struct dev_pm_ops tegra194_cbb_pm = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, tegra194_cbb_resume_noirq) +}; + +static struct platform_driver tegra194_cbb_driver = { + .probe = tegra194_cbb_probe, + .remove = tegra194_cbb_remove, + .driver = { + .name = "tegra194-cbb", + .of_match_table = of_match_ptr(tegra194_cbb_match), + .pm = &tegra194_cbb_pm, + }, +}; + +static int __init tegra194_cbb_init(void) +{ + return platform_driver_register(&tegra194_cbb_driver); +} +pure_initcall(tegra194_cbb_init); + +static void __exit tegra194_cbb_exit(void) +{ + platform_driver_unregister(&tegra194_cbb_driver); +} +module_exit(tegra194_cbb_exit); + +MODULE_AUTHOR("Sumit Gupta "); +MODULE_DESCRIPTION("Control Backbone error handling driver for Tegra194"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/tegra/cbb/tegra234-cbb.c b/drivers/soc/tegra/cbb/tegra234-cbb.c new file mode 100644 index 0000000000000000000000000000000000000000..3528f9e15d5c0cd9eac88dbb8f9ea614d9596221 --- /dev/null +++ b/drivers/soc/tegra/cbb/tegra234-cbb.c @@ -0,0 +1,1113 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved + * + * The driver handles Error's from Control Backbone(CBB) version 2.0. + * generated due to illegal accesses. The driver prints debug information + * about failed transaction on receiving interrupt from Error Notifier. + * Error types supported by CBB2.0 are: + * UNSUPPORTED_ERR, PWRDOWN_ERR, TIMEOUT_ERR, FIREWALL_ERR, DECODE_ERR, + * SLAVE_ERR + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define FABRIC_EN_CFG_INTERRUPT_ENABLE_0_0 0x0 +#define FABRIC_EN_CFG_STATUS_0_0 0x40 +#define FABRIC_EN_CFG_ADDR_INDEX_0_0 0x60 +#define FABRIC_EN_CFG_ADDR_LOW_0 0x80 +#define FABRIC_EN_CFG_ADDR_HI_0 0x84 + +#define FABRIC_MN_MASTER_ERR_EN_0 0x200 +#define FABRIC_MN_MASTER_ERR_FORCE_0 0x204 +#define FABRIC_MN_MASTER_ERR_STATUS_0 0x208 +#define FABRIC_MN_MASTER_ERR_OVERFLOW_STATUS_0 0x20c + +#define FABRIC_MN_MASTER_LOG_ERR_STATUS_0 0x300 +#define FABRIC_MN_MASTER_LOG_ADDR_LOW_0 0x304 +#define FABRIC_MN_MASTER_LOG_ADDR_HIGH_0 0x308 +#define FABRIC_MN_MASTER_LOG_ATTRIBUTES0_0 0x30c +#define FABRIC_MN_MASTER_LOG_ATTRIBUTES1_0 0x310 +#define FABRIC_MN_MASTER_LOG_ATTRIBUTES2_0 0x314 +#define FABRIC_MN_MASTER_LOG_USER_BITS0_0 0x318 + +#define AXI_SLV_TIMEOUT_STATUS_0_0 0x8 +#define APB_BLOCK_TMO_STATUS_0 0xc00 +#define APB_BLOCK_NUM_TMO_OFFSET 0x20 + +#define FAB_EM_EL_MSTRID GENMASK(29, 24) +#define FAB_EM_EL_VQC GENMASK(17, 16) +#define FAB_EM_EL_GRPSEC GENMASK(14, 8) +#define FAB_EM_EL_FALCONSEC GENMASK(1, 0) + +#define FAB_EM_EL_FABID GENMASK(20, 16) +#define FAB_EM_EL_SLAVEID GENMASK(7, 0) + +#define FAB_EM_EL_ACCESSID GENMASK(7, 0) + +#define FAB_EM_EL_AXCACHE GENMASK(27, 24) +#define FAB_EM_EL_AXPROT GENMASK(22, 20) +#define FAB_EM_EL_BURSTLENGTH GENMASK(19, 12) +#define FAB_EM_EL_BURSTTYPE GENMASK(9, 8) +#define FAB_EM_EL_BEATSIZE GENMASK(6, 4) +#define FAB_EM_EL_ACCESSTYPE GENMASK(0, 0) + +#define USRBITS_MSTR_ID GENMASK(29, 24) + +#define REQ_SOCKET_ID GENMASK(27, 24) + +enum tegra234_cbb_fabric_ids { + CBB_FAB_ID, + SCE_FAB_ID, + RCE_FAB_ID, + DCE_FAB_ID, + AON_FAB_ID, + PSC_FAB_ID, + BPMP_FAB_ID, + FSI_FAB_ID, + MAX_FAB_ID, +}; + +struct tegra234_slave_lookup { + const char *name; + unsigned int offset; +}; + +struct tegra234_cbb_fabric { + const char *name; + phys_addr_t off_mask_erd; + bool erd_mask_inband_err; + const char * const *master_id; + unsigned int notifier_offset; + const struct tegra_cbb_error *errors; + const struct tegra234_slave_lookup *slave_map; +}; + +struct tegra234_cbb { + struct tegra_cbb base; + + const struct tegra234_cbb_fabric *fabric; + struct resource *res; + void __iomem *regs; + + int num_intr; + int sec_irq; + + /* record */ + void __iomem *mon; + unsigned int type; + u32 mask; + u64 access; + u32 mn_attr0; + u32 mn_attr1; + u32 mn_attr2; + u32 mn_user_bits; +}; + +static inline struct tegra234_cbb *to_tegra234_cbb(struct tegra_cbb *cbb) +{ + return container_of(cbb, struct tegra234_cbb, base); +} + +static LIST_HEAD(cbb_list); +static DEFINE_SPINLOCK(cbb_lock); + +static void tegra234_cbb_fault_enable(struct tegra_cbb *cbb) +{ + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + void __iomem *addr; + + addr = priv->regs + priv->fabric->notifier_offset; + writel(0x1ff, addr + FABRIC_EN_CFG_INTERRUPT_ENABLE_0_0); + dsb(sy); +} + +static void tegra234_cbb_error_clear(struct tegra_cbb *cbb) +{ + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + + writel(0x3f, priv->mon + FABRIC_MN_MASTER_ERR_STATUS_0); + dsb(sy); +} + +static u32 tegra234_cbb_get_status(struct tegra_cbb *cbb) +{ + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + void __iomem *addr; + u32 value; + + addr = priv->regs + priv->fabric->notifier_offset; + value = readl(addr + FABRIC_EN_CFG_STATUS_0_0); + dsb(sy); + + return value; +} + +static void tegra234_cbb_mask_serror(struct tegra234_cbb *cbb) +{ + writel(0x1, cbb->regs + cbb->fabric->off_mask_erd); + dsb(sy); +} + +static u32 tegra234_cbb_get_tmo_slv(void __iomem *addr) +{ + u32 timeout; + + timeout = readl(addr); + return timeout; +} + +static void tegra234_cbb_tmo_slv(struct seq_file *file, const char *slave, void __iomem *addr, + u32 status) +{ + tegra_cbb_print_err(file, "\t %s : %#x\n", slave, status); +} + +static void tegra234_cbb_lookup_apbslv(struct seq_file *file, const char *slave, + void __iomem *base) +{ + unsigned int block = 0; + void __iomem *addr; + char name[64]; + u32 status; + + status = tegra234_cbb_get_tmo_slv(base); + if (status) + tegra_cbb_print_err(file, "\t %s_BLOCK_TMO_STATUS : %#x\n", slave, status); + + while (status) { + if (status & BIT(0)) { + u32 timeout, clients, client = 0; + + addr = base + APB_BLOCK_NUM_TMO_OFFSET + (block * 4); + timeout = tegra234_cbb_get_tmo_slv(addr); + clients = timeout; + + while (timeout) { + if (timeout & BIT(0)) { + if (clients != 0xffffffff) + clients &= BIT(client); + + sprintf(name, "%s_BLOCK%d_TMO", slave, block); + + tegra234_cbb_tmo_slv(file, name, addr, clients); + } + + timeout >>= 1; + client++; + } + } + + status >>= 1; + block++; + } +} + +static void tegra234_lookup_slave_timeout(struct seq_file *file, struct tegra234_cbb *cbb, + u8 slave_id, u8 fab_id) +{ + const struct tegra234_slave_lookup *map = cbb->fabric->slave_map; + void __iomem *addr; + + /* + * 1) Get slave node name and address mapping using slave_id. + * 2) Check if the timed out slave node is APB or AXI. + * 3) If AXI, then print timeout register and reset axi slave + * using _SN_<>_SLV_TIMEOUT_STATUS_0_0 register. + * 4) If APB, then perform an additional lookup to find the client + * which timed out. + * a) Get block number from the index of set bit in + * _SN_AXI2APB_<>_BLOCK_TMO_STATUS_0 register. + * b) Get address of register repective to block number i.e. + * _SN_AXI2APB_<>_BLOCK_TMO_0. + * c) Read the register in above step to get client_id which + * timed out as per the set bits. + * d) Reset the timedout client and print details. + * e) Goto step-a till all bits are set. + */ + + addr = cbb->regs + map[slave_id].offset; + + if (strstr(map[slave_id].name, "AXI2APB")) { + addr += APB_BLOCK_TMO_STATUS_0; + + tegra234_cbb_lookup_apbslv(file, map[slave_id].name, addr); + } else { + char name[64]; + u32 status; + + addr += AXI_SLV_TIMEOUT_STATUS_0_0; + + status = tegra234_cbb_get_tmo_slv(addr); + if (status) { + sprintf(name, "%s_SLV_TIMEOUT_STATUS", map[slave_id].name); + tegra234_cbb_tmo_slv(file, name, addr, status); + } + } +} + +static void tegra234_cbb_print_error(struct seq_file *file, struct tegra234_cbb *cbb, u32 status, + u32 overflow) +{ + unsigned int type = 0; + + if (status & (status - 1)) + tegra_cbb_print_err(file, "\t Multiple type of errors reported\n"); + + while (status) { + if (status & 0x1) + tegra_cbb_print_err(file, "\t Error Code\t\t: %s\n", + cbb->fabric->errors[type].code); + + status >>= 1; + type++; + } + + type = 0; + + while (overflow) { + if (overflow & 0x1) + tegra_cbb_print_err(file, "\t Overflow\t\t: Multiple %s\n", + cbb->fabric->errors[type].code); + + overflow >>= 1; + type++; + } +} + +static void print_errlog_err(struct seq_file *file, struct tegra234_cbb *cbb) +{ + u8 cache_type, prot_type, burst_length, mstr_id, grpsec, vqc, falconsec, beat_size; + u8 access_type, access_id, requester_socket_id, local_socket_id, slave_id, fab_id; + char fabric_name[20]; + bool is_numa = false; + u8 burst_type; + + if (num_possible_nodes() > 1) + is_numa = true; + + mstr_id = FIELD_GET(FAB_EM_EL_MSTRID, cbb->mn_user_bits); + vqc = FIELD_GET(FAB_EM_EL_VQC, cbb->mn_user_bits); + grpsec = FIELD_GET(FAB_EM_EL_GRPSEC, cbb->mn_user_bits); + falconsec = FIELD_GET(FAB_EM_EL_FALCONSEC, cbb->mn_user_bits); + + /* + * For SOC with multiple NUMA nodes, print cross socket access + * errors only if initiator/master_id is CCPLEX, CPMU or GPU. + */ + if (is_numa) { + local_socket_id = numa_node_id(); + requester_socket_id = FIELD_GET(REQ_SOCKET_ID, cbb->mn_attr2); + + if (requester_socket_id != local_socket_id) { + if ((mstr_id != 0x1) && (mstr_id != 0x2) && (mstr_id != 0xB)) + return; + } + } + + fab_id = FIELD_GET(FAB_EM_EL_FABID, cbb->mn_attr2); + slave_id = FIELD_GET(FAB_EM_EL_SLAVEID, cbb->mn_attr2); + + access_id = FIELD_GET(FAB_EM_EL_ACCESSID, cbb->mn_attr1); + + cache_type = FIELD_GET(FAB_EM_EL_AXCACHE, cbb->mn_attr0); + prot_type = FIELD_GET(FAB_EM_EL_AXPROT, cbb->mn_attr0); + burst_length = FIELD_GET(FAB_EM_EL_BURSTLENGTH, cbb->mn_attr0); + burst_type = FIELD_GET(FAB_EM_EL_BURSTTYPE, cbb->mn_attr0); + beat_size = FIELD_GET(FAB_EM_EL_BEATSIZE, cbb->mn_attr0); + access_type = FIELD_GET(FAB_EM_EL_ACCESSTYPE, cbb->mn_attr0); + + tegra_cbb_print_err(file, "\n"); + tegra_cbb_print_err(file, "\t Error Code\t\t: %s\n", + cbb->fabric->errors[cbb->type].code); + + tegra_cbb_print_err(file, "\t MASTER_ID\t\t: %s\n", cbb->fabric->master_id[mstr_id]); + tegra_cbb_print_err(file, "\t Address\t\t: %#llx\n", cbb->access); + + tegra_cbb_print_cache(file, cache_type); + tegra_cbb_print_prot(file, prot_type); + + tegra_cbb_print_err(file, "\t Access_Type\t\t: %s", (access_type) ? "Write\n" : "Read\n"); + tegra_cbb_print_err(file, "\t Access_ID\t\t: %#x", access_id); + + if (fab_id == PSC_FAB_ID) + strcpy(fabric_name, "psc-fabric"); + else if (fab_id == FSI_FAB_ID) + strcpy(fabric_name, "fsi-fabric"); + else + strcpy(fabric_name, cbb->fabric->name); + + if (is_numa) { + tegra_cbb_print_err(file, "\t Requester_Socket_Id\t: %#x\n", + requester_socket_id); + tegra_cbb_print_err(file, "\t Local_Socket_Id\t: %#x\n", + local_socket_id); + tegra_cbb_print_err(file, "\t No. of NUMA_NODES\t: %#x\n", + num_possible_nodes()); + } + + tegra_cbb_print_err(file, "\t Fabric\t\t: %s\n", fabric_name); + tegra_cbb_print_err(file, "\t Slave_Id\t\t: %#x\n", slave_id); + tegra_cbb_print_err(file, "\t Burst_length\t\t: %#x\n", burst_length); + tegra_cbb_print_err(file, "\t Burst_type\t\t: %#x\n", burst_type); + tegra_cbb_print_err(file, "\t Beat_size\t\t: %#x\n", beat_size); + tegra_cbb_print_err(file, "\t VQC\t\t\t: %#x\n", vqc); + tegra_cbb_print_err(file, "\t GRPSEC\t\t: %#x\n", grpsec); + tegra_cbb_print_err(file, "\t FALCONSEC\t\t: %#x\n", falconsec); + + if ((fab_id == PSC_FAB_ID) || (fab_id == FSI_FAB_ID)) + return; + + if (!strcmp(cbb->fabric->errors[cbb->type].code, "TIMEOUT_ERR")) { + tegra234_lookup_slave_timeout(file, cbb, slave_id, fab_id); + return; + } + + tegra_cbb_print_err(file, "\t Slave\t\t\t: %s\n", cbb->fabric->slave_map[slave_id].name); +} + +static int print_errmonX_info(struct seq_file *file, struct tegra234_cbb *cbb) +{ + u32 overflow, status, error; + + status = readl(cbb->mon + FABRIC_MN_MASTER_ERR_STATUS_0); + if (!status) { + pr_err("Error Notifier received a spurious notification\n"); + return -ENODATA; + } + + if (status == 0xffffffff) { + pr_err("CBB registers returning all 1's which is invalid\n"); + return -EINVAL; + } + + overflow = readl(cbb->mon + FABRIC_MN_MASTER_ERR_OVERFLOW_STATUS_0); + + tegra234_cbb_print_error(file, cbb, status, overflow); + + error = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ERR_STATUS_0); + if (!error) { + pr_info("Error Monitor doesn't have Error Logger\n"); + return -EINVAL; + } + + cbb->type = 0; + + while (error) { + if (error & BIT(0)) { + u32 hi, lo; + + hi = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ADDR_HIGH_0); + lo = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ADDR_LOW_0); + + cbb->access = (u64)hi << 32 | lo; + + cbb->mn_attr0 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES0_0); + cbb->mn_attr1 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES1_0); + cbb->mn_attr2 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES2_0); + cbb->mn_user_bits = readl(cbb->mon + FABRIC_MN_MASTER_LOG_USER_BITS0_0); + + print_errlog_err(file, cbb); + } + + cbb->type++; + error >>= 1; + } + + return 0; +} + +static int print_err_notifier(struct seq_file *file, struct tegra234_cbb *cbb, u32 status) +{ + unsigned int index = 0; + int err; + + pr_crit("**************************************\n"); + pr_crit("CPU:%d, Error:%s, Errmon:%d\n", smp_processor_id(), + cbb->fabric->name, status); + + while (status) { + if (status & BIT(0)) { + unsigned int notifier = cbb->fabric->notifier_offset; + u32 hi, lo, mask = BIT(index); + phys_addr_t addr; + u64 offset; + + writel(mask, cbb->regs + notifier + FABRIC_EN_CFG_ADDR_INDEX_0_0); + hi = readl(cbb->regs + notifier + FABRIC_EN_CFG_ADDR_HI_0); + lo = readl(cbb->regs + notifier + FABRIC_EN_CFG_ADDR_LOW_0); + + addr = (u64)hi << 32 | lo; + + offset = addr - cbb->res->start; + cbb->mon = cbb->regs + offset; + cbb->mask = BIT(index); + + err = print_errmonX_info(file, cbb); + tegra234_cbb_error_clear(&cbb->base); + if (err) + return err; + } + + status >>= 1; + index++; + } + + tegra_cbb_print_err(file, "\t**************************************\n"); + return 0; +} + +#ifdef CONFIG_DEBUG_FS +static DEFINE_MUTEX(cbb_debugfs_mutex); + +static int tegra234_cbb_debugfs_show(struct tegra_cbb *cbb, struct seq_file *file, void *data) +{ + int err = 0; + + mutex_lock(&cbb_debugfs_mutex); + + list_for_each_entry(cbb, &cbb_list, node) { + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + u32 status; + + status = tegra_cbb_get_status(&priv->base); + if (status) { + err = print_err_notifier(file, priv, status); + if (err) + break; + } + } + + mutex_unlock(&cbb_debugfs_mutex); + return err; +} +#endif + +/* + * Handler for CBB errors + */ +static irqreturn_t tegra234_cbb_isr(int irq, void *data) +{ + bool is_inband_err = false; + struct tegra_cbb *cbb; + unsigned long flags; + u8 mstr_id; + int err; + + spin_lock_irqsave(&cbb_lock, flags); + + list_for_each_entry(cbb, &cbb_list, node) { + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + u32 status = tegra_cbb_get_status(cbb); + + if (status && (irq == priv->sec_irq)) { + tegra_cbb_print_err(NULL, "CPU:%d, Error: %s@%llx, irq=%d\n", + smp_processor_id(), priv->fabric->name, + priv->res->start, irq); + + err = print_err_notifier(NULL, priv, status); + if (err) + goto unlock; + + mstr_id = FIELD_GET(USRBITS_MSTR_ID, priv->mn_user_bits); + + /* + * If illegal request is from CCPLEX(id:0x1) master then call BUG() to + * crash system. + */ + if ((mstr_id == 0x1) && priv->fabric->off_mask_erd) + is_inband_err = 1; + } + } + +unlock: + spin_unlock_irqrestore(&cbb_lock, flags); + WARN_ON(is_inband_err); + return IRQ_HANDLED; +} + +/* + * Register handler for CBB_SECURE interrupt for reporting errors + */ +static int tegra234_cbb_interrupt_enable(struct tegra_cbb *cbb) +{ + struct tegra234_cbb *priv = to_tegra234_cbb(cbb); + + if (priv->sec_irq) { + int err = devm_request_irq(cbb->dev, priv->sec_irq, tegra234_cbb_isr, 0, + dev_name(cbb->dev), priv); + if (err) { + dev_err(cbb->dev, "failed to register interrupt %u: %d\n", priv->sec_irq, + err); + return err; + } + } + + return 0; +} + +static void tegra234_cbb_error_enable(struct tegra_cbb *cbb) +{ + tegra_cbb_fault_enable(cbb); +} + +static const struct tegra_cbb_ops tegra234_cbb_ops = { + .get_status = tegra234_cbb_get_status, + .error_clear = tegra234_cbb_error_clear, + .fault_enable = tegra234_cbb_fault_enable, + .error_enable = tegra234_cbb_error_enable, + .interrupt_enable = tegra234_cbb_interrupt_enable, +#ifdef CONFIG_DEBUG_FS + .debugfs_show = tegra234_cbb_debugfs_show, +#endif +}; + +static const char * const tegra234_master_id[] = { + [0x00] = "TZ", + [0x01] = "CCPLEX", + [0x02] = "CCPMU", + [0x03] = "BPMP_FW", + [0x04] = "AON", + [0x05] = "SCE", + [0x06] = "GPCDMA_P", + [0x07] = "TSECA_NONSECURE", + [0x08] = "TSECA_LIGHTSECURE", + [0x09] = "TSECA_HEAVYSECURE", + [0x0a] = "CORESIGHT", + [0x0b] = "APE", + [0x0c] = "PEATRANS", + [0x0d] = "JTAGM_DFT", + [0x0e] = "RCE", + [0x0f] = "DCE", + [0x10] = "PSC_FW_USER", + [0x11] = "PSC_FW_SUPERVISOR", + [0x12] = "PSC_FW_MACHINE", + [0x13] = "PSC_BOOT", + [0x14] = "BPMP_BOOT", + [0x15] = "NVDEC_NONSECURE", + [0x16] = "NVDEC_LIGHTSECURE", + [0x17] = "NVDEC_HEAVYSECURE", + [0x18] = "CBB_INTERNAL", + [0x19] = "RSVD" +}; + +static const struct tegra_cbb_error tegra234_cbb_errors[] = { + { + .code = "SLAVE_ERR", + .desc = "Slave being accessed responded with an error" + }, { + .code = "DECODE_ERR", + .desc = "Attempt to access an address hole" + }, { + .code = "FIREWALL_ERR", + .desc = "Attempt to access a region which is firewall protected" + }, { + .code = "TIMEOUT_ERR", + .desc = "No response returned by slave" + }, { + .code = "PWRDOWN_ERR", + .desc = "Attempt to access a portion of fabric that is powered down" + }, { + .code = "UNSUPPORTED_ERR", + .desc = "Attempt to access a slave through an unsupported access" + } +}; + +static const struct tegra234_slave_lookup tegra234_aon_slave_map[] = { + { "AXI2APB", 0x00000 }, + { "AST", 0x14000 }, + { "CBB", 0x15000 }, + { "CPU", 0x16000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_aon_fabric = { + .name = "aon-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_aon_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x17000, +}; + +static const struct tegra234_slave_lookup tegra234_bpmp_slave_map[] = { + { "AXI2APB", 0x00000 }, + { "AST0", 0x15000 }, + { "AST1", 0x16000 }, + { "CBB", 0x17000 }, + { "CPU", 0x18000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_bpmp_fabric = { + .name = "bpmp-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_bpmp_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x19000, +}; + +static const struct tegra234_slave_lookup tegra234_cbb_slave_map[] = { + { "AON", 0x40000 }, + { "BPMP", 0x41000 }, + { "CBB", 0x42000 }, + { "HOST1X", 0x43000 }, + { "STM", 0x44000 }, + { "FSI", 0x45000 }, + { "PSC", 0x46000 }, + { "PCIE_C1", 0x47000 }, + { "PCIE_C2", 0x48000 }, + { "PCIE_C3", 0x49000 }, + { "PCIE_C0", 0x4a000 }, + { "PCIE_C4", 0x4b000 }, + { "GPU", 0x4c000 }, + { "SMMU0", 0x4d000 }, + { "SMMU1", 0x4e000 }, + { "SMMU2", 0x4f000 }, + { "SMMU3", 0x50000 }, + { "SMMU4", 0x51000 }, + { "PCIE_C10", 0x52000 }, + { "PCIE_C7", 0x53000 }, + { "PCIE_C8", 0x54000 }, + { "PCIE_C9", 0x55000 }, + { "PCIE_C5", 0x56000 }, + { "PCIE_C6", 0x57000 }, + { "DCE", 0x58000 }, + { "RCE", 0x59000 }, + { "SCE", 0x5a000 }, + { "AXI2APB_1", 0x70000 }, + { "AXI2APB_10", 0x71000 }, + { "AXI2APB_11", 0x72000 }, + { "AXI2APB_12", 0x73000 }, + { "AXI2APB_13", 0x74000 }, + { "AXI2APB_14", 0x75000 }, + { "AXI2APB_15", 0x76000 }, + { "AXI2APB_16", 0x77000 }, + { "AXI2APB_17", 0x78000 }, + { "AXI2APB_18", 0x79000 }, + { "AXI2APB_19", 0x7a000 }, + { "AXI2APB_2", 0x7b000 }, + { "AXI2APB_20", 0x7c000 }, + { "AXI2APB_21", 0x7d000 }, + { "AXI2APB_22", 0x7e000 }, + { "AXI2APB_23", 0x7f000 }, + { "AXI2APB_25", 0x80000 }, + { "AXI2APB_26", 0x81000 }, + { "AXI2APB_27", 0x82000 }, + { "AXI2APB_28", 0x83000 }, + { "AXI2APB_29", 0x84000 }, + { "AXI2APB_30", 0x85000 }, + { "AXI2APB_31", 0x86000 }, + { "AXI2APB_32", 0x87000 }, + { "AXI2APB_33", 0x88000 }, + { "AXI2APB_34", 0x89000 }, + { "AXI2APB_35", 0x92000 }, + { "AXI2APB_4", 0x8b000 }, + { "AXI2APB_5", 0x8c000 }, + { "AXI2APB_6", 0x8d000 }, + { "AXI2APB_7", 0x8e000 }, + { "AXI2APB_8", 0x8f000 }, + { "AXI2APB_9", 0x90000 }, + { "AXI2APB_3", 0x91000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_cbb_fabric = { + .name = "cbb-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_cbb_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x60000, + .off_mask_erd = 0x3a004 +}; + +static const struct tegra234_slave_lookup tegra234_dce_slave_map[] = { + { "AXI2APB", 0x00000 }, + { "AST0", 0x15000 }, + { "AST1", 0x16000 }, + { "CPU", 0x18000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_dce_fabric = { + .name = "dce-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_dce_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x19000, +}; + +static const struct tegra234_slave_lookup tegra234_rce_slave_map[] = { + { "AXI2APB", 0x00000 }, + { "AST0", 0x15000 }, + { "AST1", 0x16000 }, + { "CPU", 0x18000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_rce_fabric = { + .name = "rce-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_rce_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x19000, +}; + +static const struct tegra234_slave_lookup tegra234_sce_slave_map[] = { + { "AXI2APB", 0x00000 }, + { "AST0", 0x15000 }, + { "AST1", 0x16000 }, + { "CBB", 0x17000 }, + { "CPU", 0x18000 }, +}; + +static const struct tegra234_cbb_fabric tegra234_sce_fabric = { + .name = "sce-fabric", + .master_id = tegra234_master_id, + .slave_map = tegra234_sce_slave_map, + .errors = tegra234_cbb_errors, + .notifier_offset = 0x19000, +}; + +static const char * const tegra241_master_id[] = { + [0x0] = "TZ", + [0x1] = "CCPLEX", + [0x2] = "CCPMU", + [0x3] = "BPMP_FW", + [0x4] = "PSC_FW_USER", + [0x5] = "PSC_FW_SUPERVISOR", + [0x6] = "PSC_FW_MACHINE", + [0x7] = "PSC_BOOT", + [0x8] = "BPMP_BOOT", + [0x9] = "JTAGM_DFT", + [0xa] = "CORESIGHT", + [0xb] = "GPU", + [0xc] = "PEATRANS", + [0xd ... 0x3f] = "RSVD" +}; + +/* + * Possible causes for Slave and Timeout errors. + * SLAVE_ERR: + * Slave being accessed responded with an error. Slave could return + * an error for various cases : + * Unsupported access, clamp setting when power gated, register + * level firewall(SCR), address hole within the slave, etc + * + * TIMEOUT_ERR: + * No response returned by slave. Can be due to slave being clock + * gated, under reset, powered down or slave inability to respond + * for an internal slave issue + */ +static const struct tegra_cbb_error tegra241_cbb_errors[] = { + { + .code = "SLAVE_ERR", + .desc = "Slave being accessed responded with an error." + }, { + .code = "DECODE_ERR", + .desc = "Attempt to access an address hole or Reserved region of memory." + }, { + .code = "FIREWALL_ERR", + .desc = "Attempt to access a region which is firewalled." + }, { + .code = "TIMEOUT_ERR", + .desc = "No response returned by slave." + }, { + .code = "PWRDOWN_ERR", + .desc = "Attempt to access a portion of the fabric that is powered down." + }, { + .code = "UNSUPPORTED_ERR", + .desc = "Attempt to access a slave through an unsupported access." + }, { + .code = "POISON_ERR", + .desc = "Slave responds with poison error to indicate error in data." + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "RSVD" + }, { + .code = "NO_SUCH_ADDRESS_ERR", + .desc = "The address belongs to the pri_target range but there is no register " + "implemented at the address." + }, { + .code = "TASK_ERR", + .desc = "Attempt to update a PRI task when the current task has still not " + "completed." + }, { + .code = "EXTERNAL_ERR", + .desc = "Indicates that an external PRI register access met with an error due to " + "any issue in the unit." + }, { + .code = "INDEX_ERR", + .desc = "Applicable to PRI index aperture pair, when the programmed index is " + "outside the range defined in the manual." + }, { + .code = "RESET_ERR", + .desc = "Target in Reset Error: Attempt to access a SubPri or external PRI " + "register but they are in reset." + }, { + .code = "REGISTER_RST_ERR", + .desc = "Attempt to access a PRI register but the register is partial or " + "completely in reset." + }, { + .code = "POWER_GATED_ERR", + .desc = "Returned by external PRI client when the external access goes to a power " + "gated domain." + }, { + .code = "SUBPRI_FS_ERR", + .desc = "Subpri is floorswept: Attempt to access a subpri through the main pri " + "target but subPri logic is floorswept." + }, { + .code = "SUBPRI_CLK_OFF_ERR", + .desc = "Subpri clock is off: Attempt to access a subpri through the main pri " + "target but subPris clock is gated/off." + }, +}; + +static const struct tegra234_slave_lookup tegra241_cbb_slave_map[] = { + { "CCPLEX", 0x50000 }, + { "PCIE_C8", 0x51000 }, + { "PCIE_C9", 0x52000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "AON", 0x5b000 }, + { "BPMP", 0x5c000 }, + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "PSC", 0x5d000 }, + { "STM", 0x5e000 }, + { "AXI2APB_1", 0x70000 }, + { "AXI2APB_10", 0x71000 }, + { "AXI2APB_11", 0x72000 }, + { "AXI2APB_12", 0x73000 }, + { "AXI2APB_13", 0x74000 }, + { "AXI2APB_14", 0x75000 }, + { "AXI2APB_15", 0x76000 }, + { "AXI2APB_16", 0x77000 }, + { "AXI2APB_17", 0x78000 }, + { "AXI2APB_18", 0x79000 }, + { "AXI2APB_19", 0x7a000 }, + { "AXI2APB_2", 0x7b000 }, + { "AXI2APB_20", 0x7c000 }, + { "AXI2APB_4", 0x87000 }, + { "AXI2APB_5", 0x88000 }, + { "AXI2APB_6", 0x89000 }, + { "AXI2APB_7", 0x8a000 }, + { "AXI2APB_8", 0x8b000 }, + { "AXI2APB_9", 0x8c000 }, + { "AXI2APB_3", 0x8d000 }, + { "AXI2APB_21", 0x7d000 }, + { "AXI2APB_22", 0x7e000 }, + { "AXI2APB_23", 0x7f000 }, + { "AXI2APB_24", 0x80000 }, + { "AXI2APB_25", 0x81000 }, + { "AXI2APB_26", 0x82000 }, + { "AXI2APB_27", 0x83000 }, + { "AXI2APB_28", 0x84000 }, + { "PCIE_C4", 0x53000 }, + { "PCIE_C5", 0x54000 }, + { "PCIE_C6", 0x55000 }, + { "PCIE_C7", 0x56000 }, + { "PCIE_C2", 0x57000 }, + { "PCIE_C3", 0x58000 }, + { "PCIE_C0", 0x59000 }, + { "PCIE_C1", 0x5a000 }, + { "AXI2APB_29", 0x85000 }, + { "AXI2APB_30", 0x86000 }, +}; + +static const struct tegra234_cbb_fabric tegra241_cbb_fabric = { + .name = "cbb-fabric", + .master_id = tegra241_master_id, + .slave_map = tegra241_cbb_slave_map, + .errors = tegra241_cbb_errors, + .notifier_offset = 0x60000, + .off_mask_erd = 0x40004, +}; + +static const struct tegra234_slave_lookup tegra241_bpmp_slave_map[] = { + { "RSVD", 0x00000 }, + { "RSVD", 0x00000 }, + { "CBB", 0x15000 }, + { "CPU", 0x16000 }, + { "AXI2APB", 0x00000 }, + { "DBB0", 0x17000 }, + { "DBB1", 0x18000 }, +}; + +static const struct tegra234_cbb_fabric tegra241_bpmp_fabric = { + .name = "bpmp-fabric", + .master_id = tegra241_master_id, + .slave_map = tegra241_bpmp_slave_map, + .errors = tegra241_cbb_errors, + .notifier_offset = 0x19000, +}; + +static const struct of_device_id tegra234_cbb_dt_ids[] = { + { .compatible = "nvidia,tegra234-cbb-fabric", .data = &tegra234_cbb_fabric }, + { .compatible = "nvidia,tegra234-aon-fabric", .data = &tegra234_aon_fabric }, + { .compatible = "nvidia,tegra234-bpmp-fabric", .data = &tegra234_bpmp_fabric }, + { .compatible = "nvidia,tegra234-dce-fabric", .data = &tegra234_dce_fabric }, + { .compatible = "nvidia,tegra234-rce-fabric", .data = &tegra234_rce_fabric }, + { .compatible = "nvidia,tegra234-sce-fabric", .data = &tegra234_sce_fabric }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, tegra234_cbb_dt_ids); + +struct tegra234_cbb_acpi_uid { + const char *hid; + const char *uid; + const struct tegra234_cbb_fabric *fabric; +}; + +static const struct tegra234_cbb_acpi_uid tegra234_cbb_acpi_uids[] = { + { "NVDA1070", "1", &tegra241_cbb_fabric }, + { "NVDA1070", "2", &tegra241_bpmp_fabric }, + { }, +}; + +static const struct +tegra234_cbb_fabric *tegra234_cbb_acpi_get_fabric(struct acpi_device *adev) +{ + const struct tegra234_cbb_acpi_uid *entry; + + for (entry = tegra234_cbb_acpi_uids; entry->hid; entry++) { + if (acpi_dev_hid_uid_match(adev, entry->hid, entry->uid)) + return entry->fabric; + } + + return NULL; +} + +static const struct acpi_device_id tegra241_cbb_acpi_ids[] = { + { "NVDA1070" }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, tegra241_cbb_acpi_ids); + +static int tegra234_cbb_probe(struct platform_device *pdev) +{ + const struct tegra234_cbb_fabric *fabric; + struct tegra234_cbb *cbb; + unsigned long flags = 0; + int err; + + if (pdev->dev.of_node) { + fabric = of_device_get_match_data(&pdev->dev); + } else { + struct acpi_device *device = ACPI_COMPANION(&pdev->dev); + if (!device) + return -ENODEV; + + fabric = tegra234_cbb_acpi_get_fabric(device); + if (!fabric) { + dev_err(&pdev->dev, "no device match found\n"); + return -ENODEV; + } + } + + cbb = devm_kzalloc(&pdev->dev, sizeof(*cbb), GFP_KERNEL); + if (!cbb) + return -ENOMEM; + + INIT_LIST_HEAD(&cbb->base.node); + cbb->base.ops = &tegra234_cbb_ops; + cbb->base.dev = &pdev->dev; + cbb->fabric = fabric; + + cbb->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &cbb->res); + if (IS_ERR(cbb->regs)) + return PTR_ERR(cbb->regs); + + err = tegra_cbb_get_irq(pdev, NULL, &cbb->sec_irq); + if (err) + return err; + + platform_set_drvdata(pdev, cbb); + + spin_lock_irqsave(&cbb_lock, flags); + list_add(&cbb->base.node, &cbb_list); + spin_unlock_irqrestore(&cbb_lock, flags); + + /* set ERD bit to mask SError and generate interrupt to report error */ + if (cbb->fabric->off_mask_erd) + tegra234_cbb_mask_serror(cbb); + + return tegra_cbb_register(&cbb->base); +} + +static int tegra234_cbb_remove(struct platform_device *pdev) +{ + return 0; +} + +static int __maybe_unused tegra234_cbb_resume_noirq(struct device *dev) +{ + struct tegra234_cbb *cbb = dev_get_drvdata(dev); + + tegra234_cbb_error_enable(&cbb->base); + + dev_dbg(dev, "%s resumed\n", cbb->fabric->name); + + return 0; +} + +static const struct dev_pm_ops tegra234_cbb_pm = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, tegra234_cbb_resume_noirq) +}; + +static struct platform_driver tegra234_cbb_driver = { + .probe = tegra234_cbb_probe, + .remove = tegra234_cbb_remove, + .driver = { + .name = "tegra234-cbb", + .of_match_table = tegra234_cbb_dt_ids, + .acpi_match_table = tegra241_cbb_acpi_ids, + .pm = &tegra234_cbb_pm, + }, +}; + +static int __init tegra234_cbb_init(void) +{ + return platform_driver_register(&tegra234_cbb_driver); +} +pure_initcall(tegra234_cbb_init); + +static void __exit tegra234_cbb_exit(void) +{ + platform_driver_unregister(&tegra234_cbb_driver); +} +module_exit(tegra234_cbb_exit); + +MODULE_DESCRIPTION("Control Backbone 2.0 error handling driver for Tegra234"); +MODULE_LICENSE("GPL"); diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c index b0a8405dbdb199141aff73392a550050f2aab28a..6542267a224d2c45413463bf5811d6a65b464131 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra.c +++ b/drivers/soc/tegra/fuse/fuse-tegra.c @@ -568,6 +568,7 @@ static int __init tegra_init_fuse(void) np = of_find_matching_node(NULL, car_match); if (np) { void __iomem *base = of_iomap(np, 0); + of_node_put(np); if (base) { tegra_enable_fuse_clk(base); iounmap(base); diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c index 590c862538d0aa3d342eb780c346f0af3b88785c..3351bd872ab27293149416fa6f487ebee89878d9 100644 --- a/drivers/soc/tegra/fuse/tegra-apbmisc.c +++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c @@ -16,12 +16,16 @@ #define FUSE_SKU_INFO 0x10 +#define ERD_ERR_CONFIG 0x120c +#define ERD_MASK_INBAND_ERR 0x1 + #define PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT 4 #define PMC_STRAPPING_OPT_A_RAM_CODE_MASK_LONG \ (0xf << PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT) #define PMC_STRAPPING_OPT_A_RAM_CODE_MASK_SHORT \ (0x3 << PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT) +static void __iomem *apbmisc_base; static bool long_ram_code; static u32 strapping; static u32 chipid; @@ -93,6 +97,28 @@ u32 tegra_read_ram_code(void) } EXPORT_SYMBOL_GPL(tegra_read_ram_code); +/* + * The function sets ERD(Error Response Disable) bit. + * This allows to mask inband errors and always send an + * OKAY response from CBB to the master which caused error. + */ +int tegra194_miscreg_mask_serror(void) +{ + if (!apbmisc_base) + return -EPROBE_DEFER; + + if (!of_machine_is_compatible("nvidia,tegra194")) { + WARN(1, "Only supported for Tegra194 devices!\n"); + return -EOPNOTSUPP; + } + + writel_relaxed(ERD_MASK_INBAND_ERR, + apbmisc_base + ERD_ERR_CONFIG); + + return 0; +} +EXPORT_SYMBOL(tegra194_miscreg_mask_serror); + static const struct of_device_id apbmisc_match[] __initconst = { { .compatible = "nvidia,tegra20-apbmisc", }, { .compatible = "nvidia,tegra186-misc", }, @@ -134,7 +160,7 @@ void __init tegra_init_revision(void) void __init tegra_init_apbmisc(void) { - void __iomem *apbmisc_base, *strapping_base; + void __iomem *strapping_base; struct resource apbmisc, straps; struct device_node *np; @@ -182,12 +208,12 @@ void __init tegra_init_apbmisc(void) */ if (of_address_to_resource(np, 0, &apbmisc) < 0) { pr_err("failed to get APBMISC registers\n"); - return; + goto put; } if (of_address_to_resource(np, 1, &straps) < 0) { pr_err("failed to get strapping options registers\n"); - return; + goto put; } } @@ -196,7 +222,6 @@ void __init tegra_init_apbmisc(void) pr_err("failed to map APBMISC registers\n"); } else { chipid = readl_relaxed(apbmisc_base + 4); - iounmap(apbmisc_base); } strapping_base = ioremap(straps.start, resource_size(&straps)); @@ -208,4 +233,7 @@ void __init tegra_init_apbmisc(void) } long_ram_code = of_property_read_bool(np, "nvidia,long-ram-code"); + +put: + of_node_put(np); } diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index 6a4b8f7e79480d8207d3f72dd711778bd1bde2eb..678e8bc8a45d1045ca1421c829418ab0c2f20d85 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -296,6 +296,17 @@ struct tegra_wake_event { } gpio; }; +#define TEGRA_WAKE_SIMPLE(_name, _id) \ + { \ + .name = _name, \ + .id = _id, \ + .irq = 0, \ + .gpio = { \ + .instance = UINT_MAX, \ + .pin = UINT_MAX, \ + }, \ + } + #define TEGRA_WAKE_IRQ(_name, _id, _irq) \ { \ .name = _name, \ @@ -2239,6 +2250,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq, for (i = 0; i < soc->num_wake_events; i++) { const struct tegra_wake_event *event = &soc->wake_events[i]; + /* IRQ and simple wake events */ if (fwspec->param_count == 2) { struct irq_fwspec spec; @@ -2251,6 +2263,12 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq, if (err < 0) break; + /* simple hierarchies stop at the PMC level */ + if (event->irq == 0) { + err = irq_domain_disconnect_hierarchy(domain->parent, virq); + break; + } + spec.fwnode = &pmc->dev->of_node->fwnode; spec.param_count = 3; spec.param[0] = GIC_SPI; @@ -2263,6 +2281,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq, break; } + /* GPIO wake events */ if (fwspec->param_count == 3) { if (event->gpio.instance != fwspec->param[0] || event->gpio.pin != fwspec->param[1]) @@ -2274,7 +2293,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq, /* GPIO hierarchies stop at the PMC level */ if (!err && domain->parent) - err = irq_domain_disconnect_hierarchy(domain->parent, + err = irq_domain_disconnect_hierarchy(domain->parent, virq); break; } @@ -2885,17 +2904,10 @@ static int tegra_pmc_probe(struct platform_device *pdev) pmc->scratch = base; } - pmc->clk = devm_clk_get(&pdev->dev, "pclk"); - if (IS_ERR(pmc->clk)) { - err = PTR_ERR(pmc->clk); - - if (err != -ENOENT) { - dev_err(&pdev->dev, "failed to get pclk: %d\n", err); - return err; - } - - pmc->clk = NULL; - } + pmc->clk = devm_clk_get_optional(&pdev->dev, "pclk"); + if (IS_ERR(pmc->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(pmc->clk), + "failed to get pclk\n"); /* * PMC should be last resort for restarting since it soft-resets @@ -3757,6 +3769,13 @@ static const struct tegra_wake_event tegra194_wake_events[] = { TEGRA_WAKE_IRQ("pmu", 24, 209), TEGRA_WAKE_GPIO("power", 29, 1, TEGRA194_AON_GPIO(EE, 4)), TEGRA_WAKE_IRQ("rtc", 73, 10), + TEGRA_WAKE_SIMPLE("usb3-port-0", 76), + TEGRA_WAKE_SIMPLE("usb3-port-1", 77), + TEGRA_WAKE_SIMPLE("usb3-port-2-3", 78), + TEGRA_WAKE_SIMPLE("usb2-port-0", 79), + TEGRA_WAKE_SIMPLE("usb2-port-1", 80), + TEGRA_WAKE_SIMPLE("usb2-port-2", 81), + TEGRA_WAKE_SIMPLE("usb2-port-3", 82), }; static const struct tegra_pmc_soc tegra194_pmc_soc = { @@ -4025,7 +4044,7 @@ static int __init tegra_pmc_early_init(void) return -ENXIO; } - if (np) { + if (of_device_is_available(np)) { pmc->soc = match->data; if (pmc->soc->maybe_tz_only) diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 8d4000664fa34f712fa60a3f4b7d985bb90f94b6..76515c33e639ed646209b40b612fab517d3bea43 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -11,11 +11,12 @@ #include "bus.h" #include "sysfs_local.h" -static DEFINE_IDA(sdw_ida); +static DEFINE_IDA(sdw_bus_ida); +static DEFINE_IDA(sdw_peripheral_ida); static int sdw_get_id(struct sdw_bus *bus) { - int rc = ida_alloc(&sdw_ida, GFP_KERNEL); + int rc = ida_alloc(&sdw_bus_ida, GFP_KERNEL); if (rc < 0) return rc; @@ -75,7 +76,6 @@ int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent, /* * Initialize multi_link flag - * TODO: populate this flag by reading property from FW node */ bus->multi_link = false; if (bus->ops->read_prop) { @@ -157,9 +157,11 @@ static int sdw_delete_slave(struct device *dev, void *data) mutex_lock(&bus->bus_lock); - if (slave->dev_num) /* clear dev_num if assigned */ + if (slave->dev_num) { /* clear dev_num if assigned */ clear_bit(slave->dev_num, bus->assigned); - + if (bus->dev_num_ida_min) + ida_free(&sdw_peripheral_ida, slave->dev_num); + } list_del_init(&slave->node); mutex_unlock(&bus->bus_lock); @@ -179,7 +181,7 @@ void sdw_bus_master_delete(struct sdw_bus *bus) sdw_master_device_del(bus); sdw_bus_debugfs_exit(bus); - ida_free(&sdw_ida, bus->id); + ida_free(&sdw_bus_ida, bus->id); } EXPORT_SYMBOL(sdw_bus_master_delete); @@ -297,6 +299,38 @@ int sdw_transfer(struct sdw_bus *bus, struct sdw_msg *msg) return ret; } +/** + * sdw_show_ping_status() - Direct report of PING status, to be used by Peripheral drivers + * @bus: SDW bus + * @sync_delay: Delay before reading status + */ +void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay) +{ + u32 status; + + if (!bus->ops->read_ping_status) + return; + + /* + * wait for peripheral to sync if desired. 10-15ms should be more than + * enough in most cases. + */ + if (sync_delay) + usleep_range(10000, 15000); + + mutex_lock(&bus->msg_lock); + + status = bus->ops->read_ping_status(bus); + + mutex_unlock(&bus->msg_lock); + + if (!status) + dev_warn(bus->dev, "%s: no peripherals attached\n", __func__); + else + dev_dbg(bus->dev, "PING status: %#x\n", status); +} +EXPORT_SYMBOL(sdw_show_ping_status); + /** * sdw_transfer_defer() - Asynchronously transfer message to a SDW Slave device * @bus: SDW bus @@ -639,10 +673,18 @@ static int sdw_get_device_num(struct sdw_slave *slave) { int bit; - bit = find_first_zero_bit(slave->bus->assigned, SDW_MAX_DEVICES); - if (bit == SDW_MAX_DEVICES) { - bit = -ENODEV; - goto err; + if (slave->bus->dev_num_ida_min) { + bit = ida_alloc_range(&sdw_peripheral_ida, + slave->bus->dev_num_ida_min, SDW_MAX_DEVICES, + GFP_KERNEL); + if (bit < 0) + goto err; + } else { + bit = find_first_zero_bit(slave->bus->assigned, SDW_MAX_DEVICES); + if (bit == SDW_MAX_DEVICES) { + bit = -ENODEV; + goto err; + } } /* @@ -719,7 +761,7 @@ void sdw_extract_slave_id(struct sdw_bus *bus, } EXPORT_SYMBOL(sdw_extract_slave_id); -static int sdw_program_device_num(struct sdw_bus *bus) +static int sdw_program_device_num(struct sdw_bus *bus, bool *programmed) { u8 buf[SDW_NUM_DEV_ID_REGISTERS] = {0}; struct sdw_slave *slave, *_s; @@ -729,6 +771,8 @@ static int sdw_program_device_num(struct sdw_bus *bus) int count = 0, ret; u64 addr; + *programmed = false; + /* No Slave, so use raw xfer api */ ret = sdw_fill_msg(&msg, NULL, SDW_SCP_DEVID_0, SDW_NUM_DEV_ID_REGISTERS, 0, SDW_MSG_FLAG_READ, buf); @@ -763,6 +807,16 @@ static int sdw_program_device_num(struct sdw_bus *bus) if (sdw_compare_devid(slave, id) == 0) { found = true; + /* + * To prevent skipping state-machine stages don't + * program a device until we've seen it UNATTACH. + * Must return here because no other device on #0 + * can be detected until this one has been + * assigned a device ID. + */ + if (slave->status != SDW_SLAVE_UNATTACHED) + return 0; + /* * Assign a new dev_num to this Slave and * not mark it present. It will be marked @@ -777,6 +831,8 @@ static int sdw_program_device_num(struct sdw_bus *bus) return ret; } + *programmed = true; + break; } } @@ -816,13 +872,13 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, mutex_lock(&bus->bus_lock); dev_vdbg(bus->dev, - "%s: changing status slave %d status %d new status %d\n", - __func__, slave->dev_num, slave->status, status); + "changing status slave %d status %d new status %d\n", + slave->dev_num, slave->status, status); if (status == SDW_SLAVE_UNATTACHED) { dev_dbg(&slave->dev, - "%s: initializing enumeration and init completion for Slave %d\n", - __func__, slave->dev_num); + "initializing enumeration and init completion for Slave %d\n", + slave->dev_num); init_completion(&slave->enumeration_complete); init_completion(&slave->initialization_complete); @@ -830,8 +886,8 @@ static void sdw_modify_slave_status(struct sdw_slave *slave, } else if ((status == SDW_SLAVE_ATTACHED) && (slave->status == SDW_SLAVE_UNATTACHED)) { dev_dbg(&slave->dev, - "%s: signaling enumeration completion for Slave %d\n", - __func__, slave->dev_num); + "signaling enumeration completion for Slave %d\n", + slave->dev_num); complete(&slave->enumeration_complete); } @@ -1598,7 +1654,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) port = buf2[0] & SDW_SCP_INTSTAT2_PORT4_10; for_each_set_bit(bit, &port, 8) { /* scp2 ports start from 4 */ - port_num = bit + 3; + port_num = bit + 4; sdw_handle_port_interrupt(slave, port_num, &port_status[port_num]); @@ -1610,7 +1666,7 @@ static int sdw_handle_slave_alerts(struct sdw_slave *slave) port = buf2[1] & SDW_SCP_INTSTAT3_PORT11_14; for_each_set_bit(bit, &port, 8) { /* scp3 ports start from 11 */ - port_num = bit + 10; + port_num = bit + 11; sdw_handle_port_interrupt(slave, port_num, &port_status[port_num]); @@ -1736,7 +1792,7 @@ int sdw_handle_slave_status(struct sdw_bus *bus, { enum sdw_slave_status prev_status; struct sdw_slave *slave; - bool attached_initializing; + bool attached_initializing, id_programmed; int i, ret = 0; /* first check if any Slaves fell off the bus */ @@ -1757,19 +1813,33 @@ int sdw_handle_slave_status(struct sdw_bus *bus, dev_warn(&slave->dev, "Slave %d state check1: UNATTACHED, status was %d\n", i, slave->status); sdw_modify_slave_status(slave, SDW_SLAVE_UNATTACHED); + + /* Ensure driver knows that peripheral unattached */ + ret = sdw_update_slave_status(slave, status[i]); + if (ret < 0) + dev_warn(&slave->dev, "Update Slave status failed:%d\n", ret); } } if (status[0] == SDW_SLAVE_ATTACHED) { dev_dbg(bus->dev, "Slave attached, programming device number\n"); - ret = sdw_program_device_num(bus); - if (ret < 0) - dev_err(bus->dev, "Slave attach failed: %d\n", ret); + /* - * programming a device number will have side effects, - * so we deal with other devices at a later time + * Programming a device number will have side effects, + * so we deal with other devices at a later time. + * This relies on those devices reporting ATTACHED, which will + * trigger another call to this function. This will only + * happen if at least one device ID was programmed. + * Error returns from sdw_program_device_num() are currently + * ignored because there's no useful recovery that can be done. + * Returning the error here could result in the current status + * of other devices not being handled, because if no device IDs + * were programmed there's nothing to guarantee a status change + * to trigger another call to this function. */ - return ret; + sdw_program_device_num(bus, &id_programmed); + if (id_programmed) + return 0; } /* Continue to check other slave statuses */ @@ -1838,8 +1908,8 @@ int sdw_handle_slave_status(struct sdw_bus *bus, "Update Slave status failed:%d\n", ret); if (attached_initializing) { dev_dbg(&slave->dev, - "%s: signaling initialization completion for Slave %d\n", - __func__, slave->dev_num); + "signaling initialization completion for Slave %d\n", + slave->dev_num); complete(&slave->initialization_complete); diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index 4fbb19557f5edab12d863a7d3b313ab589171962..93929f19d0831b511fb83c0c4d9c6bd428b8af36 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -544,9 +544,12 @@ cdns_fill_msg_resp(struct sdw_cdns *cdns, return SDW_CMD_IGNORED; } - /* fill response */ - for (i = 0; i < count; i++) - msg->buf[i + offset] = FIELD_GET(CDNS_MCP_RESP_RDATA, cdns->response_buf[i]); + if (msg->flags == SDW_MSG_FLAG_READ) { + /* fill response */ + for (i = 0; i < count; i++) + msg->buf[i + offset] = FIELD_GET(CDNS_MCP_RESP_RDATA, + cdns->response_buf[i]); + } return SDW_CMD_OK; } @@ -566,7 +569,7 @@ _cdns_xfer_msg(struct sdw_cdns *cdns, struct sdw_msg *msg, int cmd, } base = CDNS_MCP_CMD_BASE; - addr = msg->addr; + addr = msg->addr + offset; for (i = 0; i < count; i++) { data = FIELD_PREP(CDNS_MCP_CMD_DEV_ADDR, msg->dev_num); @@ -705,18 +708,15 @@ cdns_xfer_msg(struct sdw_bus *bus, struct sdw_msg *msg) for (i = 0; i < msg->len / CDNS_MCP_CMD_LEN; i++) { ret = _cdns_xfer_msg(cdns, msg, cmd, i * CDNS_MCP_CMD_LEN, CDNS_MCP_CMD_LEN, false); - if (ret < 0) - goto exit; + if (ret != SDW_CMD_OK) + return ret; } if (!(msg->len % CDNS_MCP_CMD_LEN)) - goto exit; - - ret = _cdns_xfer_msg(cdns, msg, cmd, i * CDNS_MCP_CMD_LEN, - msg->len % CDNS_MCP_CMD_LEN, false); + return SDW_CMD_OK; -exit: - return ret; + return _cdns_xfer_msg(cdns, msg, cmd, i * CDNS_MCP_CMD_LEN, + msg->len % CDNS_MCP_CMD_LEN, false); } EXPORT_SYMBOL(cdns_xfer_msg); @@ -756,6 +756,14 @@ cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num) } EXPORT_SYMBOL(cdns_reset_page_addr); +u32 cdns_read_ping_status(struct sdw_bus *bus) +{ + struct sdw_cdns *cdns = bus_to_cdns(bus); + + return cdns_readl(cdns, CDNS_MCP_SLAVE_STAT); +} +EXPORT_SYMBOL(cdns_read_ping_status); + /* * IRQ handling */ @@ -782,6 +790,7 @@ static int cdns_update_slave_status(struct sdw_cdns *cdns, enum sdw_slave_status status[SDW_MAX_DEVICES + 1]; bool is_slave = false; u32 mask; + u32 val; int i, set_status; memset(status, 0, sizeof(status)); @@ -789,41 +798,38 @@ static int cdns_update_slave_status(struct sdw_cdns *cdns, for (i = 0; i <= SDW_MAX_DEVICES; i++) { mask = (slave_intstat >> (i * CDNS_MCP_SLAVE_STATUS_NUM)) & CDNS_MCP_SLAVE_STATUS_BITS; - if (!mask) - continue; - is_slave = true; set_status = 0; - if (mask & CDNS_MCP_SLAVE_INTSTAT_RESERVED) { - status[i] = SDW_SLAVE_RESERVED; - set_status++; - } - - if (mask & CDNS_MCP_SLAVE_INTSTAT_ATTACHED) { - status[i] = SDW_SLAVE_ATTACHED; - set_status++; - } + if (mask) { + is_slave = true; - if (mask & CDNS_MCP_SLAVE_INTSTAT_ALERT) { - status[i] = SDW_SLAVE_ALERT; - set_status++; - } + if (mask & CDNS_MCP_SLAVE_INTSTAT_RESERVED) { + status[i] = SDW_SLAVE_RESERVED; + set_status++; + } - if (mask & CDNS_MCP_SLAVE_INTSTAT_NPRESENT) { - status[i] = SDW_SLAVE_UNATTACHED; - set_status++; - } + if (mask & CDNS_MCP_SLAVE_INTSTAT_ATTACHED) { + status[i] = SDW_SLAVE_ATTACHED; + set_status++; + } - /* first check if Slave reported multiple status */ - if (set_status > 1) { - u32 val; + if (mask & CDNS_MCP_SLAVE_INTSTAT_ALERT) { + status[i] = SDW_SLAVE_ALERT; + set_status++; + } - dev_warn_ratelimited(cdns->dev, - "Slave %d reported multiple Status: %d\n", - i, mask); + if (mask & CDNS_MCP_SLAVE_INTSTAT_NPRESENT) { + status[i] = SDW_SLAVE_UNATTACHED; + set_status++; + } + } - /* check latest status extracted from PING commands */ + /* + * check that there was a single reported Slave status and when + * there is not use the latest status extracted from PING commands + */ + if (set_status != 1) { val = cdns_readl(cdns, CDNS_MCP_SLAVE_STAT); val >>= (i * 2); @@ -842,11 +848,6 @@ static int cdns_update_slave_status(struct sdw_cdns *cdns, status[i] = SDW_SLAVE_RESERVED; break; } - - dev_warn_ratelimited(cdns->dev, - "Slave %d status updated to %d\n", - i, status[i]); - } } @@ -961,9 +962,22 @@ static void cdns_update_slave_status_work(struct work_struct *work) u32 device0_status; int retry_count = 0; + /* + * Clear main interrupt first so we don't lose any assertions + * that happen during this function. + */ + cdns_writel(cdns, CDNS_MCP_INTSTAT, CDNS_MCP_INT_SLAVE_MASK); + slave0 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT0); slave1 = cdns_readl(cdns, CDNS_MCP_SLAVE_INTSTAT1); + /* + * Clear the bits before handling so we don't lose any + * bits that re-assert. + */ + cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT0, slave0); + cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT1, slave1); + /* combine the two status */ slave_intstat = ((u64)slave1 << 32) | slave0; @@ -971,8 +985,6 @@ static void cdns_update_slave_status_work(struct work_struct *work) update_status: cdns_update_slave_status(cdns, slave_intstat); - cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT0, slave0); - cdns_writel(cdns, CDNS_MCP_SLAVE_INTSTAT1, slave1); /* * When there is more than one peripheral per link, it's @@ -989,6 +1001,11 @@ update_status: * attention with PING commands. There is no need to check for * ALERTS since they are not allowed until a non-zero * device_number is assigned. + * + * Do not clear the INTSTAT0/1. While looping to enumerate devices on + * #0 there could be status changes on other devices - these must + * be kept in the INTSTAT so they can be handled when all #0 devices + * have been handled. */ device0_status = cdns_readl(cdns, CDNS_MCP_SLAVE_STAT); @@ -1008,8 +1025,7 @@ update_status: } } - /* clear and unmask Slave interrupt now */ - cdns_writel(cdns, CDNS_MCP_INTSTAT, CDNS_MCP_INT_SLAVE_MASK); + /* unmask Slave interrupt now */ cdns_updatel(cdns, CDNS_MCP_INTMASK, CDNS_MCP_INT_SLAVE_MASK, CDNS_MCP_INT_SLAVE_MASK); diff --git a/drivers/soundwire/cadence_master.h b/drivers/soundwire/cadence_master.h index 595d72c15d977e8a694dcfa9bb187e459ab73741..ca9e805bab88f383fdd5d1a8e2acae535389c0a0 100644 --- a/drivers/soundwire/cadence_master.h +++ b/drivers/soundwire/cadence_master.h @@ -177,6 +177,8 @@ enum sdw_command_response cdns_xfer_msg_defer(struct sdw_bus *bus, struct sdw_msg *msg, struct sdw_defer *defer); +u32 cdns_read_ping_status(struct sdw_bus *bus); + int cdns_bus_conf(struct sdw_bus *bus, struct sdw_bus_params *params); int cdns_set_sdw_stream(struct snd_soc_dai *dai, diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 747983743a14b44d0a659fe14b44f1baa6e4e246..f81cdd83ec26e6b1f59cffee8f48fbdb66c220b8 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -55,7 +55,26 @@ static const struct adr_remap dell_sku_0A3E[] = { {} }; +/* + * The HP Omen 16-k0005TX does not expose the correct version of RT711 on link0 + * and does not expose a RT1316 on link3 + */ +static const struct adr_remap hp_omen_16[] = { + /* rt711-sdca on link0 */ + { + 0x000020025d071100ull, + 0x000030025d071101ull + }, + /* rt1316-sdca on link3 */ + { + 0x000120025d071100ull, + 0x000330025d131601ull + }, + {} +}; + static const struct dmi_system_id adr_remap_quirk_table[] = { + /* TGL devices */ { .matches = { DMI_MATCH(DMI_SYS_VENDOR, "HP"), @@ -78,6 +97,14 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)dell_sku_0A3E, }, + /* ADL devices */ + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "OMEN by HP Gaming Laptop 16-k0xxx"), + }, + .driver_data = (void *)hp_omen_16, + }, {} }; diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 89d1d0d021fc72ab7a3724fc625fcb25bf0ec2bd..244209358784ffb2c5f9348a497d12b4a2bf2052 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -22,6 +22,9 @@ #include "bus.h" #include "intel.h" +/* IDA min selected to avoid conflicts with HDaudio/iDISP SDI values */ +#define INTEL_DEV_NUM_IDA_MIN 4 + #define INTEL_MASTER_SUSPEND_DELAY_MS 3000 #define INTEL_MASTER_RESET_ITERATIONS 10 @@ -135,7 +138,7 @@ static int intel_reg_show(struct seq_file *s_file, void *data) if (!buf) return -ENOMEM; - links = intel_readl(s, SDW_SHIM_LCAP) & GENMASK(2, 0); + links = intel_readl(s, SDW_SHIM_LCAP) & SDW_SHIM_LCAP_LCOUNT_MASK; ret = scnprintf(buf, RD_BUF, "Register Value\n"); ret += scnprintf(buf + ret, RD_BUF - ret, "\nShim\n"); @@ -167,9 +170,8 @@ static int intel_reg_show(struct seq_file *s_file, void *data) ret += intel_sprintf(s, false, buf, ret, SDW_SHIM_PCMSYCHC(i, j)); } - ret += scnprintf(buf + ret, RD_BUF - ret, "\n PDMSCAP, IOCTL, CTMCTL\n"); + ret += scnprintf(buf + ret, RD_BUF - ret, "\n IOCTL, CTMCTL\n"); - ret += intel_sprintf(s, false, buf, ret, SDW_SHIM_PDMSCAP(i)); ret += intel_sprintf(s, false, buf, ret, SDW_SHIM_IOCTL(i)); ret += intel_sprintf(s, false, buf, ret, SDW_SHIM_CTMCTL(i)); } @@ -258,86 +260,6 @@ static void intel_debugfs_exit(struct sdw_intel *sdw) {} /* * shim ops */ - -static int intel_link_power_up(struct sdw_intel *sdw) -{ - unsigned int link_id = sdw->instance; - void __iomem *shim = sdw->link_res->shim; - u32 *shim_mask = sdw->link_res->shim_mask; - struct sdw_bus *bus = &sdw->cdns.bus; - struct sdw_master_prop *prop = &bus->prop; - u32 spa_mask, cpa_mask; - u32 link_control; - int ret = 0; - u32 syncprd; - u32 sync_reg; - - mutex_lock(sdw->link_res->shim_lock); - - /* - * The hardware relies on an internal counter, typically 4kHz, - * to generate the SoundWire SSP - which defines a 'safe' - * synchronization point between commands and audio transport - * and allows for multi link synchronization. The SYNCPRD value - * is only dependent on the oscillator clock provided to - * the IP, so adjust based on _DSD properties reported in DSDT - * tables. The values reported are based on either 24MHz - * (CNL/CML) or 38.4 MHz (ICL/TGL+). - */ - if (prop->mclk_freq % 6000000) - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; - else - syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24; - - if (!*shim_mask) { - dev_dbg(sdw->cdns.dev, "%s: powering up all links\n", __func__); - - /* we first need to program the SyncPRD/CPU registers */ - dev_dbg(sdw->cdns.dev, - "%s: first link up, programming SYNCPRD\n", __func__); - - /* set SyncPRD period */ - sync_reg = intel_readl(shim, SDW_SHIM_SYNC); - u32p_replace_bits(&sync_reg, syncprd, SDW_SHIM_SYNC_SYNCPRD); - - /* Set SyncCPU bit */ - sync_reg |= SDW_SHIM_SYNC_SYNCCPU; - intel_writel(shim, SDW_SHIM_SYNC, sync_reg); - - /* Link power up sequence */ - link_control = intel_readl(shim, SDW_SHIM_LCTL); - - /* only power-up enabled links */ - spa_mask = FIELD_PREP(SDW_SHIM_LCTL_SPA_MASK, sdw->link_res->link_mask); - cpa_mask = FIELD_PREP(SDW_SHIM_LCTL_CPA_MASK, sdw->link_res->link_mask); - - link_control |= spa_mask; - - ret = intel_set_bit(shim, SDW_SHIM_LCTL, link_control, cpa_mask); - if (ret < 0) { - dev_err(sdw->cdns.dev, "Failed to power up link: %d\n", ret); - goto out; - } - - /* SyncCPU will change once link is active */ - ret = intel_wait_bit(shim, SDW_SHIM_SYNC, - SDW_SHIM_SYNC_SYNCCPU, 0); - if (ret < 0) { - dev_err(sdw->cdns.dev, - "Failed to set SHIM_SYNC: %d\n", ret); - goto out; - } - } - - *shim_mask |= BIT(link_id); - - sdw->cdns.link_up = true; -out: - mutex_unlock(sdw->link_res->shim_lock); - - return ret; -} - /* this needs to be called with shim_lock */ static void intel_shim_glue_to_master_ip(struct sdw_intel *sdw) { @@ -389,15 +311,13 @@ static void intel_shim_master_ip_to_glue(struct sdw_intel *sdw) /* at this point Integration Glue has full control of the I/Os */ } -static int intel_shim_init(struct sdw_intel *sdw, bool clock_stop) +/* this needs to be called with shim_lock */ +static void intel_shim_init(struct sdw_intel *sdw) { void __iomem *shim = sdw->link_res->shim; unsigned int link_id = sdw->instance; - int ret = 0; u16 ioctl = 0, act = 0; - mutex_lock(sdw->link_res->shim_lock); - /* Initialize Shim */ ioctl |= SDW_SHIM_IOCTL_BKE; intel_writew(shim, SDW_SHIM_IOCTL(link_id), ioctl); @@ -422,10 +342,17 @@ static int intel_shim_init(struct sdw_intel *sdw, bool clock_stop) act |= SDW_SHIM_CTMCTL_DODS; intel_writew(shim, SDW_SHIM_CTMCTL(link_id), act); usleep_range(10, 15); +} - mutex_unlock(sdw->link_res->shim_lock); +static int intel_shim_check_wake(struct sdw_intel *sdw) +{ + void __iomem *shim; + u16 wake_sts; - return ret; + shim = sdw->link_res->shim; + wake_sts = intel_readw(shim, SDW_SHIM_WAKESTS); + + return wake_sts & BIT(sdw->instance); } static void intel_shim_wake(struct sdw_intel *sdw, bool wake_enable) @@ -454,6 +381,88 @@ static void intel_shim_wake(struct sdw_intel *sdw, bool wake_enable) mutex_unlock(sdw->link_res->shim_lock); } +static int intel_link_power_up(struct sdw_intel *sdw) +{ + unsigned int link_id = sdw->instance; + void __iomem *shim = sdw->link_res->shim; + u32 *shim_mask = sdw->link_res->shim_mask; + struct sdw_bus *bus = &sdw->cdns.bus; + struct sdw_master_prop *prop = &bus->prop; + u32 spa_mask, cpa_mask; + u32 link_control; + int ret = 0; + u32 syncprd; + u32 sync_reg; + + mutex_lock(sdw->link_res->shim_lock); + + /* + * The hardware relies on an internal counter, typically 4kHz, + * to generate the SoundWire SSP - which defines a 'safe' + * synchronization point between commands and audio transport + * and allows for multi link synchronization. The SYNCPRD value + * is only dependent on the oscillator clock provided to + * the IP, so adjust based on _DSD properties reported in DSDT + * tables. The values reported are based on either 24MHz + * (CNL/CML) or 38.4 MHz (ICL/TGL+). + */ + if (prop->mclk_freq % 6000000) + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_38_4; + else + syncprd = SDW_SHIM_SYNC_SYNCPRD_VAL_24; + + if (!*shim_mask) { + dev_dbg(sdw->cdns.dev, "powering up all links\n"); + + /* we first need to program the SyncPRD/CPU registers */ + dev_dbg(sdw->cdns.dev, + "first link up, programming SYNCPRD\n"); + + /* set SyncPRD period */ + sync_reg = intel_readl(shim, SDW_SHIM_SYNC); + u32p_replace_bits(&sync_reg, syncprd, SDW_SHIM_SYNC_SYNCPRD); + + /* Set SyncCPU bit */ + sync_reg |= SDW_SHIM_SYNC_SYNCCPU; + intel_writel(shim, SDW_SHIM_SYNC, sync_reg); + + /* Link power up sequence */ + link_control = intel_readl(shim, SDW_SHIM_LCTL); + + /* only power-up enabled links */ + spa_mask = FIELD_PREP(SDW_SHIM_LCTL_SPA_MASK, sdw->link_res->link_mask); + cpa_mask = FIELD_PREP(SDW_SHIM_LCTL_CPA_MASK, sdw->link_res->link_mask); + + link_control |= spa_mask; + + ret = intel_set_bit(shim, SDW_SHIM_LCTL, link_control, cpa_mask); + if (ret < 0) { + dev_err(sdw->cdns.dev, "Failed to power up link: %d\n", ret); + goto out; + } + + /* SyncCPU will change once link is active */ + ret = intel_wait_bit(shim, SDW_SHIM_SYNC, + SDW_SHIM_SYNC_SYNCCPU, 0); + if (ret < 0) { + dev_err(sdw->cdns.dev, + "Failed to set SHIM_SYNC: %d\n", ret); + goto out; + } + } + + *shim_mask |= BIT(link_id); + + sdw->cdns.link_up = true; + + intel_shim_init(sdw); + +out: + mutex_unlock(sdw->link_res->shim_lock); + + return ret; +} + static int intel_link_power_down(struct sdw_intel *sdw) { u32 link_control, spa_mask, cpa_mask; @@ -476,7 +485,7 @@ static int intel_link_power_down(struct sdw_intel *sdw) if (!*shim_mask) { - dev_dbg(sdw->cdns.dev, "%s: powering down all links\n", __func__); + dev_dbg(sdw->cdns.dev, "powering down all links\n"); /* Link power down sequence */ link_control = intel_readl(shim, SDW_SHIM_LCTL); @@ -1169,11 +1178,20 @@ static int intel_create_dai(struct sdw_cdns *cdns, static int intel_register_dai(struct sdw_intel *sdw) { + struct sdw_cdns_stream_config config; struct sdw_cdns *cdns = &sdw->cdns; struct sdw_cdns_streams *stream; struct snd_soc_dai_driver *dais; int num_dai, ret, off = 0; + /* Read the PDI config and initialize cadence PDI */ + intel_pdi_init(sdw, &config); + ret = sdw_cdns_pdi_init(cdns, config); + if (ret) + return ret; + + intel_pdi_ch_update(sdw); + /* DAIs are created based on total number of PDIs supported */ num_dai = cdns->pcm.num_pdi; @@ -1201,8 +1219,208 @@ static int intel_register_dai(struct sdw_intel *sdw) if (ret) return ret; - return snd_soc_register_component(cdns->dev, &dai_component, - dais, num_dai); + return devm_snd_soc_register_component(cdns->dev, &dai_component, + dais, num_dai); +} + +static int intel_start_bus(struct sdw_intel *sdw) +{ + struct device *dev = sdw->cdns.dev; + struct sdw_cdns *cdns = &sdw->cdns; + struct sdw_bus *bus = &cdns->bus; + int ret; + + ret = sdw_cdns_enable_interrupt(cdns, true); + if (ret < 0) { + dev_err(dev, "%s: cannot enable interrupts: %d\n", __func__, ret); + return ret; + } + + /* + * follow recommended programming flows to avoid timeouts when + * gsync is enabled + */ + if (bus->multi_link) + intel_shim_sync_arm(sdw); + + ret = sdw_cdns_init(cdns); + if (ret < 0) { + dev_err(dev, "%s: unable to initialize Cadence IP: %d\n", __func__, ret); + goto err_interrupt; + } + + ret = sdw_cdns_exit_reset(cdns); + if (ret < 0) { + dev_err(dev, "%s: unable to exit bus reset sequence: %d\n", __func__, ret); + goto err_interrupt; + } + + if (bus->multi_link) { + ret = intel_shim_sync_go(sdw); + if (ret < 0) { + dev_err(dev, "%s: sync go failed: %d\n", __func__, ret); + goto err_interrupt; + } + } + sdw_cdns_check_self_clearing_bits(cdns, __func__, + true, INTEL_MASTER_RESET_ITERATIONS); + + return 0; + +err_interrupt: + sdw_cdns_enable_interrupt(cdns, false); + return ret; +} + +static int intel_start_bus_after_reset(struct sdw_intel *sdw) +{ + struct device *dev = sdw->cdns.dev; + struct sdw_cdns *cdns = &sdw->cdns; + struct sdw_bus *bus = &cdns->bus; + bool clock_stop0; + int status; + int ret; + + /* + * An exception condition occurs for the CLK_STOP_BUS_RESET + * case if one or more masters remain active. In this condition, + * all the masters are powered on for they are in the same power + * domain. Master can preserve its context for clock stop0, so + * there is no need to clear slave status and reset bus. + */ + clock_stop0 = sdw_cdns_is_clock_stop(&sdw->cdns); + + if (!clock_stop0) { + + /* + * make sure all Slaves are tagged as UNATTACHED and + * provide reason for reinitialization + */ + + status = SDW_UNATTACH_REQUEST_MASTER_RESET; + sdw_clear_slave_status(bus, status); + + ret = sdw_cdns_enable_interrupt(cdns, true); + if (ret < 0) { + dev_err(dev, "cannot enable interrupts during resume\n"); + return ret; + } + + /* + * follow recommended programming flows to avoid + * timeouts when gsync is enabled + */ + if (bus->multi_link) + intel_shim_sync_arm(sdw); + + /* + * Re-initialize the IP since it was powered-off + */ + sdw_cdns_init(&sdw->cdns); + + } else { + ret = sdw_cdns_enable_interrupt(cdns, true); + if (ret < 0) { + dev_err(dev, "cannot enable interrupts during resume\n"); + return ret; + } + } + + ret = sdw_cdns_clock_restart(cdns, !clock_stop0); + if (ret < 0) { + dev_err(dev, "unable to restart clock during resume\n"); + goto err_interrupt; + } + + if (!clock_stop0) { + ret = sdw_cdns_exit_reset(cdns); + if (ret < 0) { + dev_err(dev, "unable to exit bus reset sequence during resume\n"); + goto err_interrupt; + } + + if (bus->multi_link) { + ret = intel_shim_sync_go(sdw); + if (ret < 0) { + dev_err(sdw->cdns.dev, "sync go failed during resume\n"); + goto err_interrupt; + } + } + } + sdw_cdns_check_self_clearing_bits(cdns, __func__, true, INTEL_MASTER_RESET_ITERATIONS); + + return 0; + +err_interrupt: + sdw_cdns_enable_interrupt(cdns, false); + return ret; +} + +static void intel_check_clock_stop(struct sdw_intel *sdw) +{ + struct device *dev = sdw->cdns.dev; + bool clock_stop0; + + clock_stop0 = sdw_cdns_is_clock_stop(&sdw->cdns); + if (!clock_stop0) + dev_err(dev, "%s: invalid configuration, clock was not stopped\n", __func__); +} + +static int intel_start_bus_after_clock_stop(struct sdw_intel *sdw) +{ + struct device *dev = sdw->cdns.dev; + struct sdw_cdns *cdns = &sdw->cdns; + int ret; + + ret = sdw_cdns_enable_interrupt(cdns, true); + if (ret < 0) { + dev_err(dev, "%s: cannot enable interrupts: %d\n", __func__, ret); + return ret; + } + + ret = sdw_cdns_clock_restart(cdns, false); + if (ret < 0) { + dev_err(dev, "%s: unable to restart clock: %d\n", __func__, ret); + sdw_cdns_enable_interrupt(cdns, false); + return ret; + } + + sdw_cdns_check_self_clearing_bits(cdns, "intel_resume_runtime no_quirks", + true, INTEL_MASTER_RESET_ITERATIONS); + + return 0; +} + +static int intel_stop_bus(struct sdw_intel *sdw, bool clock_stop) +{ + struct device *dev = sdw->cdns.dev; + struct sdw_cdns *cdns = &sdw->cdns; + bool wake_enable = false; + int ret; + + if (clock_stop) { + ret = sdw_cdns_clock_stop(cdns, true); + if (ret < 0) + dev_err(dev, "%s: cannot stop clock: %d\n", __func__, ret); + else + wake_enable = true; + } + + ret = sdw_cdns_enable_interrupt(cdns, false); + if (ret < 0) { + dev_err(dev, "%s: cannot disable interrupts: %d\n", __func__, ret); + return ret; + } + + ret = intel_link_power_down(sdw); + if (ret) { + dev_err(dev, "%s: Link power down failed: %d\n", __func__, ret); + return ret; + } + + intel_shim_wake(sdw, wake_enable); + + return 0; } static int sdw_master_read_intel_prop(struct sdw_bus *bus) @@ -1254,7 +1472,7 @@ static int intel_prop_read(struct sdw_bus *bus) } static struct sdw_master_ops sdw_intel_ops = { - .read_prop = sdw_master_read_prop, + .read_prop = intel_prop_read, .override_adr = sdw_dmi_override_adr, .xfer_msg = cdns_xfer_msg, .xfer_msg_defer = cdns_xfer_msg_defer, @@ -1262,22 +1480,9 @@ static struct sdw_master_ops sdw_intel_ops = { .set_bus_conf = cdns_bus_conf, .pre_bank_switch = intel_pre_bank_switch, .post_bank_switch = intel_post_bank_switch, + .read_ping_status = cdns_read_ping_status, }; -static int intel_init(struct sdw_intel *sdw) -{ - bool clock_stop; - - /* Initialize shim and controller */ - intel_link_power_up(sdw); - - clock_stop = sdw_cdns_is_clock_stop(&sdw->cdns); - - intel_shim_init(sdw, clock_stop); - - return 0; -} - /* * probe and init (aux_dev_id argument is required by function prototype but not used) */ @@ -1307,11 +1512,11 @@ static int intel_link_probe(struct auxiliary_device *auxdev, cdns->msg_count = 0; bus->link_id = auxdev->id; + bus->dev_num_ida_min = INTEL_DEV_NUM_IDA_MIN; sdw_cdns_probe(cdns); - /* Set property read ops */ - sdw_intel_ops.read_prop = intel_prop_read; + /* Set ops */ bus->ops = &sdw_intel_ops; /* set driver data, accessed by snd_soc_dai_get_drvdata() */ @@ -1344,7 +1549,6 @@ static int intel_link_probe(struct auxiliary_device *auxdev, int intel_link_startup(struct auxiliary_device *auxdev) { - struct sdw_cdns_stream_config config; struct device *dev = &auxdev->dev; struct sdw_cdns *cdns = auxiliary_get_drvdata(auxdev); struct sdw_intel *sdw = cdns_to_intel(cdns); @@ -1365,7 +1569,6 @@ int intel_link_startup(struct auxiliary_device *auxdev) multi_link = !(link_flags & SDW_INTEL_MASTER_DISABLE_MULTI_LINK); if (!multi_link) { dev_dbg(dev, "Multi-link is disabled\n"); - bus->multi_link = false; } else { /* * hardware-based synchronization is required regardless @@ -1373,68 +1576,31 @@ int intel_link_startup(struct auxiliary_device *auxdev) * synchronization is gated by gsync when the multi-master * mode is set. */ - bus->multi_link = true; bus->hw_sync_min_links = 1; } + bus->multi_link = multi_link; /* Initialize shim, controller */ - ret = intel_init(sdw); - if (ret) - goto err_init; - - /* Read the PDI config and initialize cadence PDI */ - intel_pdi_init(sdw, &config); - ret = sdw_cdns_pdi_init(cdns, config); + ret = intel_link_power_up(sdw); if (ret) goto err_init; - intel_pdi_ch_update(sdw); - - ret = sdw_cdns_enable_interrupt(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable interrupts\n"); - goto err_init; - } - - /* - * follow recommended programming flows to avoid timeouts when - * gsync is enabled - */ - if (multi_link) - intel_shim_sync_arm(sdw); - - ret = sdw_cdns_init(cdns); - if (ret < 0) { - dev_err(dev, "unable to initialize Cadence IP\n"); - goto err_interrupt; - } - - ret = sdw_cdns_exit_reset(cdns); - if (ret < 0) { - dev_err(dev, "unable to exit bus reset sequence\n"); - goto err_interrupt; - } - - if (multi_link) { - ret = intel_shim_sync_go(sdw); - if (ret < 0) { - dev_err(dev, "sync go failed: %d\n", ret); - goto err_interrupt; - } - } - sdw_cdns_check_self_clearing_bits(cdns, __func__, - true, INTEL_MASTER_RESET_ITERATIONS); - /* Register DAIs */ ret = intel_register_dai(sdw); if (ret) { dev_err(dev, "DAI registration failed: %d\n", ret); - snd_soc_unregister_component(dev); - goto err_interrupt; + goto err_power_up; } intel_debugfs_init(sdw); + /* start bus */ + ret = intel_start_bus(sdw); + if (ret) { + dev_err(dev, "bus start failed: %d\n", ret); + goto err_power_up; + } + /* Enable runtime PM */ if (!(link_flags & SDW_INTEL_MASTER_DISABLE_PM_RUNTIME)) { pm_runtime_set_autosuspend_delay(dev, @@ -1479,15 +1645,14 @@ int intel_link_startup(struct auxiliary_device *auxdev) sdw->startup_done = true; return 0; -err_interrupt: - sdw_cdns_enable_interrupt(cdns, false); +err_power_up: + intel_link_power_down(sdw); err_init: return ret; } static void intel_link_remove(struct auxiliary_device *auxdev) { - struct device *dev = &auxdev->dev; struct sdw_cdns *cdns = auxiliary_get_drvdata(auxdev); struct sdw_intel *sdw = cdns_to_intel(cdns); struct sdw_bus *bus = &cdns->bus; @@ -1500,7 +1665,6 @@ static void intel_link_remove(struct auxiliary_device *auxdev) if (!bus->prop.hw_disabled) { intel_debugfs_exit(sdw); sdw_cdns_enable_interrupt(cdns, false); - snd_soc_unregister_component(dev); } sdw_bus_master_delete(bus); } @@ -1510,8 +1674,6 @@ int intel_link_process_wakeen_event(struct auxiliary_device *auxdev) struct device *dev = &auxdev->dev; struct sdw_intel *sdw; struct sdw_bus *bus; - void __iomem *shim; - u16 wake_sts; sdw = auxiliary_get_drvdata(auxdev); bus = &sdw->cdns.bus; @@ -1522,10 +1684,7 @@ int intel_link_process_wakeen_event(struct auxiliary_device *auxdev) return 0; } - shim = sdw->link_res->shim; - wake_sts = intel_readw(shim, SDW_SHIM_WAKESTS); - - if (!(wake_sts & BIT(sdw->instance))) + if (!intel_shim_check_wake(sdw)) return 0; /* disable WAKEEN interrupt ASAP to prevent interrupt flood */ @@ -1553,11 +1712,11 @@ static int intel_resume_child_device(struct device *dev, void *data) struct sdw_slave *slave = dev_to_sdw_dev(dev); if (!slave->probed) { - dev_dbg(dev, "%s: skipping device, no probed driver\n", __func__); + dev_dbg(dev, "skipping device, no probed driver\n"); return 0; } if (!slave->dev_num_sticky) { - dev_dbg(dev, "%s: skipping device, never detected on bus\n", __func__); + dev_dbg(dev, "skipping device, never detected on bus\n"); return 0; } @@ -1643,7 +1802,7 @@ static int __maybe_unused intel_suspend(struct device *dev) } if (pm_runtime_suspended(dev)) { - dev_dbg(dev, "%s: pm_runtime status: suspended\n", __func__); + dev_dbg(dev, "pm_runtime status: suspended\n"); clock_stop_quirks = sdw->link_res->clock_stop_quirks; @@ -1664,20 +1823,12 @@ static int __maybe_unused intel_suspend(struct device *dev) return 0; } - ret = sdw_cdns_enable_interrupt(cdns, false); + ret = intel_stop_bus(sdw, false); if (ret < 0) { - dev_err(dev, "cannot disable interrupts on suspend\n"); - return ret; - } - - ret = intel_link_power_down(sdw); - if (ret) { - dev_err(dev, "Link power down failed: %d\n", ret); + dev_err(dev, "%s: cannot stop bus: %d\n", __func__, ret); return ret; } - intel_shim_wake(sdw, false); - return 0; } @@ -1698,44 +1849,19 @@ static int __maybe_unused intel_suspend_runtime(struct device *dev) clock_stop_quirks = sdw->link_res->clock_stop_quirks; if (clock_stop_quirks & SDW_INTEL_CLK_STOP_TEARDOWN) { - - ret = sdw_cdns_enable_interrupt(cdns, false); + ret = intel_stop_bus(sdw, false); if (ret < 0) { - dev_err(dev, "cannot disable interrupts on suspend\n"); - return ret; - } - - ret = intel_link_power_down(sdw); - if (ret) { - dev_err(dev, "Link power down failed: %d\n", ret); + dev_err(dev, "%s: cannot stop bus during teardown: %d\n", + __func__, ret); return ret; } - - intel_shim_wake(sdw, false); - - } else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET || - !clock_stop_quirks) { - bool wake_enable = true; - - ret = sdw_cdns_clock_stop(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable clock stop on suspend\n"); - wake_enable = false; - } - - ret = sdw_cdns_enable_interrupt(cdns, false); + } else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET || !clock_stop_quirks) { + ret = intel_stop_bus(sdw, true); if (ret < 0) { - dev_err(dev, "cannot disable interrupts on suspend\n"); - return ret; - } - - ret = intel_link_power_down(sdw); - if (ret) { - dev_err(dev, "Link power down failed: %d\n", ret); + dev_err(dev, "%s: cannot stop bus during clock_stop: %d\n", + __func__, ret); return ret; } - - intel_shim_wake(sdw, wake_enable); } else { dev_err(dev, "%s clock_stop_quirks %x unsupported\n", __func__, clock_stop_quirks); @@ -1751,7 +1877,6 @@ static int __maybe_unused intel_resume(struct device *dev) struct sdw_intel *sdw = cdns_to_intel(cdns); struct sdw_bus *bus = &cdns->bus; int link_flags; - bool multi_link; int ret; if (bus->prop.hw_disabled || !sdw->startup_done) { @@ -1761,10 +1886,9 @@ static int __maybe_unused intel_resume(struct device *dev) } link_flags = md_flags >> (bus->link_id * 8); - multi_link = !(link_flags & SDW_INTEL_MASTER_DISABLE_MULTI_LINK); if (pm_runtime_suspended(dev)) { - dev_dbg(dev, "%s: pm_runtime status was suspended, forcing active\n", __func__); + dev_dbg(dev, "pm_runtime status was suspended, forcing active\n"); /* follow required sequence from runtime_pm.rst */ pm_runtime_disable(dev); @@ -1778,7 +1902,7 @@ static int __maybe_unused intel_resume(struct device *dev) pm_runtime_idle(dev); } - ret = intel_init(sdw); + ret = intel_link_power_up(sdw); if (ret) { dev_err(dev, "%s failed: %d\n", __func__, ret); return ret; @@ -1790,41 +1914,13 @@ static int __maybe_unused intel_resume(struct device *dev) */ sdw_clear_slave_status(bus, SDW_UNATTACH_REQUEST_MASTER_RESET); - ret = sdw_cdns_enable_interrupt(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable interrupts during resume\n"); - return ret; - } - - /* - * follow recommended programming flows to avoid timeouts when - * gsync is enabled - */ - if (multi_link) - intel_shim_sync_arm(sdw); - - ret = sdw_cdns_init(&sdw->cdns); - if (ret < 0) { - dev_err(dev, "unable to initialize Cadence IP during resume\n"); - return ret; - } - - ret = sdw_cdns_exit_reset(cdns); + ret = intel_start_bus(sdw); if (ret < 0) { - dev_err(dev, "unable to exit bus reset sequence during resume\n"); + dev_err(dev, "cannot start bus during resume\n"); + intel_link_power_down(sdw); return ret; } - if (multi_link) { - ret = intel_shim_sync_go(sdw); - if (ret < 0) { - dev_err(dev, "sync go failed during resume\n"); - return ret; - } - } - sdw_cdns_check_self_clearing_bits(cdns, __func__, - true, INTEL_MASTER_RESET_ITERATIONS); - /* * after system resume, the pm_runtime suspend() may kick in * during the enumeration, before any children device force the @@ -1837,7 +1933,7 @@ static int __maybe_unused intel_resume(struct device *dev) */ pm_runtime_mark_last_busy(dev); - return ret; + return 0; } static int __maybe_unused intel_resume_runtime(struct device *dev) @@ -1846,10 +1942,6 @@ static int __maybe_unused intel_resume_runtime(struct device *dev) struct sdw_intel *sdw = cdns_to_intel(cdns); struct sdw_bus *bus = &cdns->bus; u32 clock_stop_quirks; - bool clock_stop0; - int link_flags; - bool multi_link; - int status; int ret; if (bus->prop.hw_disabled || !sdw->startup_done) { @@ -1861,15 +1953,12 @@ static int __maybe_unused intel_resume_runtime(struct device *dev) /* unconditionally disable WAKEEN interrupt */ intel_shim_wake(sdw, false); - link_flags = md_flags >> (bus->link_id * 8); - multi_link = !(link_flags & SDW_INTEL_MASTER_DISABLE_MULTI_LINK); - clock_stop_quirks = sdw->link_res->clock_stop_quirks; if (clock_stop_quirks & SDW_INTEL_CLK_STOP_TEARDOWN) { - ret = intel_init(sdw); + ret = intel_link_power_up(sdw); if (ret) { - dev_err(dev, "%s failed: %d\n", __func__, ret); + dev_err(dev, "%s: power_up failed after teardown: %d\n", __func__, ret); return ret; } @@ -1879,145 +1968,45 @@ static int __maybe_unused intel_resume_runtime(struct device *dev) */ sdw_clear_slave_status(bus, SDW_UNATTACH_REQUEST_MASTER_RESET); - ret = sdw_cdns_enable_interrupt(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable interrupts during resume\n"); - return ret; - } - - /* - * follow recommended programming flows to avoid - * timeouts when gsync is enabled - */ - if (multi_link) - intel_shim_sync_arm(sdw); - - ret = sdw_cdns_init(&sdw->cdns); - if (ret < 0) { - dev_err(dev, "unable to initialize Cadence IP during resume\n"); - return ret; - } - - ret = sdw_cdns_exit_reset(cdns); + ret = intel_start_bus(sdw); if (ret < 0) { - dev_err(dev, "unable to exit bus reset sequence during resume\n"); + dev_err(dev, "%s: cannot start bus after teardown: %d\n", __func__, ret); + intel_link_power_down(sdw); return ret; } - if (multi_link) { - ret = intel_shim_sync_go(sdw); - if (ret < 0) { - dev_err(dev, "sync go failed during resume\n"); - return ret; - } - } - sdw_cdns_check_self_clearing_bits(cdns, "intel_resume_runtime TEARDOWN", - true, INTEL_MASTER_RESET_ITERATIONS); } else if (clock_stop_quirks & SDW_INTEL_CLK_STOP_BUS_RESET) { - ret = intel_init(sdw); + ret = intel_link_power_up(sdw); if (ret) { - dev_err(dev, "%s failed: %d\n", __func__, ret); + dev_err(dev, "%s: power_up failed after bus reset: %d\n", __func__, ret); return ret; } - /* - * An exception condition occurs for the CLK_STOP_BUS_RESET - * case if one or more masters remain active. In this condition, - * all the masters are powered on for they are in the same power - * domain. Master can preserve its context for clock stop0, so - * there is no need to clear slave status and reset bus. - */ - clock_stop0 = sdw_cdns_is_clock_stop(&sdw->cdns); - - if (!clock_stop0) { - - /* - * make sure all Slaves are tagged as UNATTACHED and - * provide reason for reinitialization - */ - - status = SDW_UNATTACH_REQUEST_MASTER_RESET; - sdw_clear_slave_status(bus, status); - - ret = sdw_cdns_enable_interrupt(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable interrupts during resume\n"); - return ret; - } - - /* - * follow recommended programming flows to avoid - * timeouts when gsync is enabled - */ - if (multi_link) - intel_shim_sync_arm(sdw); - - /* - * Re-initialize the IP since it was powered-off - */ - sdw_cdns_init(&sdw->cdns); - - } else { - ret = sdw_cdns_enable_interrupt(cdns, true); - if (ret < 0) { - dev_err(dev, "cannot enable interrupts during resume\n"); - return ret; - } - } - - ret = sdw_cdns_clock_restart(cdns, !clock_stop0); + ret = intel_start_bus_after_reset(sdw); if (ret < 0) { - dev_err(dev, "unable to restart clock during resume\n"); + dev_err(dev, "%s: cannot start bus after reset: %d\n", __func__, ret); + intel_link_power_down(sdw); return ret; } - - if (!clock_stop0) { - ret = sdw_cdns_exit_reset(cdns); - if (ret < 0) { - dev_err(dev, "unable to exit bus reset sequence during resume\n"); - return ret; - } - - if (multi_link) { - ret = intel_shim_sync_go(sdw); - if (ret < 0) { - dev_err(sdw->cdns.dev, "sync go failed during resume\n"); - return ret; - } - } - } - sdw_cdns_check_self_clearing_bits(cdns, "intel_resume_runtime BUS_RESET", - true, INTEL_MASTER_RESET_ITERATIONS); - } else if (!clock_stop_quirks) { - clock_stop0 = sdw_cdns_is_clock_stop(&sdw->cdns); - if (!clock_stop0) - dev_err(dev, "%s invalid configuration, clock was not stopped", __func__); + intel_check_clock_stop(sdw); - ret = intel_init(sdw); + ret = intel_link_power_up(sdw); if (ret) { - dev_err(dev, "%s failed: %d\n", __func__, ret); + dev_err(dev, "%s: power_up failed: %d\n", __func__, ret); return ret; } - ret = sdw_cdns_enable_interrupt(cdns, true); + ret = intel_start_bus_after_clock_stop(sdw); if (ret < 0) { - dev_err(dev, "cannot enable interrupts during resume\n"); + dev_err(dev, "%s: cannot start bus after clock stop: %d\n", __func__, ret); + intel_link_power_down(sdw); return ret; } - - ret = sdw_cdns_clock_restart(cdns, false); - if (ret < 0) { - dev_err(dev, "unable to resume master during resume\n"); - return ret; - } - - sdw_cdns_check_self_clearing_bits(cdns, "intel_resume_runtime no_quirks", - true, INTEL_MASTER_RESET_ITERATIONS); } else { - dev_err(dev, "%s clock_stop_quirks %x unsupported\n", + dev_err(dev, "%s: clock_stop_quirks %x unsupported\n", __func__, clock_stop_quirks); ret = -EINVAL; } diff --git a/drivers/soundwire/intel_init.c b/drivers/soundwire/intel_init.c index 824f4f32d4dc368533439730852d4afb849d9dec..d091513919df517a14e73765ac847f084a936fba 100644 --- a/drivers/soundwire/intel_init.c +++ b/drivers/soundwire/intel_init.c @@ -306,7 +306,7 @@ sdw_intel_startup_controller(struct sdw_intel_ctx *ctx) /* Check SNDWLCAP.LCOUNT */ caps = ioread32(ctx->mmio_base + ctx->shim_base + SDW_SHIM_LCAP); - caps &= GENMASK(2, 0); + caps &= SDW_SHIM_LCAP_LCOUNT_MASK; /* Check HW supported vs property value */ if (caps < ctx->count) { diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 9df970eeca454828d037a5836b09afbbfbef4cd1..b33d5db494a54ebbcebceaa675c2db423098d515 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -169,7 +169,7 @@ struct qcom_swrm_ctrl { u8 wcmd_id; struct qcom_swrm_port_config pconfig[QCOM_SDW_MAX_PORTS]; struct sdw_stream_runtime *sruntime[SWRM_MAX_DAIS]; - enum sdw_slave_status status[SDW_MAX_DEVICES]; + enum sdw_slave_status status[SDW_MAX_DEVICES + 1]; int (*reg_read)(struct qcom_swrm_ctrl *ctrl, int reg, u32 *val); int (*reg_write)(struct qcom_swrm_ctrl *ctrl, int reg, int val); u32 slave_status; @@ -420,7 +420,7 @@ static int qcom_swrm_get_alert_slave_dev_num(struct qcom_swrm_ctrl *ctrl) ctrl->reg_read(ctrl, SWRM_MCP_SLV_STATUS, &val); - for (dev_num = 0; dev_num < SDW_MAX_DEVICES; dev_num++) { + for (dev_num = 1; dev_num <= SDW_MAX_DEVICES; dev_num++) { status = (val >> (dev_num * SWRM_MCP_SLV_STATUS_SZ)); if ((status & SWRM_MCP_SLV_STATUS_MASK) == SDW_SLAVE_ALERT) { @@ -440,7 +440,7 @@ static void qcom_swrm_get_device_status(struct qcom_swrm_ctrl *ctrl) ctrl->reg_read(ctrl, SWRM_MCP_SLV_STATUS, &val); ctrl->slave_status = val; - for (i = 0; i < SDW_MAX_DEVICES; i++) { + for (i = 1; i <= SDW_MAX_DEVICES; i++) { u32 s; s = (val >> (i * 2)); @@ -573,11 +573,10 @@ static irqreturn_t qcom_swrm_irq_handler(int irq, void *dev_id) break; case SWRM_INTERRUPT_STATUS_NEW_SLAVE_ATTACHED: case SWRM_INTERRUPT_STATUS_CHANGE_ENUM_SLAVE_STATUS: - dev_err_ratelimited(swrm->dev, "%s: SWR new slave attached\n", - __func__); + dev_dbg_ratelimited(swrm->dev, "SWR new slave attached\n"); swrm->reg_read(swrm, SWRM_MCP_SLV_STATUS, &slave_status); if (swrm->slave_status == slave_status) { - dev_err(swrm->dev, "Slave status not changed %x\n", + dev_dbg(swrm->dev, "Slave status not changed %x\n", slave_status); } else { qcom_swrm_get_device_status(swrm); @@ -1356,10 +1355,6 @@ static int qcom_swrm_probe(struct platform_device *pdev) ctrl->bus.compute_params = &qcom_swrm_compute_params; ctrl->bus.clk_stop_timeout = 300; - ctrl->audio_cgcr = devm_reset_control_get_exclusive(dev, "swr_audio_cgcr"); - if (IS_ERR(ctrl->audio_cgcr)) - dev_err(dev, "Failed to get audio_cgcr reset required for soundwire-v1.6.0\n"); - ret = qcom_swrm_get_port_config(ctrl); if (ret) goto err_clk; diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index e32f6a2058aed8baa0eb1c9c5ec0105d23ee95d1..d1bb62f7368b7452b58367a2b82f77f461fe31de 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -591,6 +591,15 @@ config SPI_MICROCHIP_CORE PolarFire SoC. If built as a module, it will be called spi-microchip-core. +config SPI_MICROCHIP_CORE_QSPI + tristate "Microchip FPGA QSPI controllers" + depends on SPI_MASTER + help + This enables the QSPI driver for Microchip FPGA QSPI controllers. + Say Y or M here if you want to use the QSPI controllers on + PolarFire SoC. + If built as a module, it will be called spi-microchip-core-qspi. + config SPI_MT65XX tristate "MediaTek SPI controller" depends on ARCH_MEDIATEK || COMPILE_TEST diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index 15d2f3835e45af8c04a6332e7a848eaf49c337f9..4b34e855c84125782d310da1adb6cbc61591fa60 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_SPI_LP8841_RTC) += spi-lp8841-rtc.o obj-$(CONFIG_SPI_MESON_SPICC) += spi-meson-spicc.o obj-$(CONFIG_SPI_MESON_SPIFC) += spi-meson-spifc.o obj-$(CONFIG_SPI_MICROCHIP_CORE) += spi-microchip-core.o +obj-$(CONFIG_SPI_MICROCHIP_CORE_QSPI) += spi-microchip-core-qspi.o obj-$(CONFIG_SPI_MPC512x_PSC) += spi-mpc512x-psc.o obj-$(CONFIG_SPI_MPC52xx_PSC) += spi-mpc52xx-psc.o obj-$(CONFIG_SPI_MPC52xx) += spi-mpc52xx.o diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c index 08df4f8d0531ca767f56a0e33d3d1b8f55f4647b..e23121456c706a3cd43e0821e5beca46a045fb61 100644 --- a/drivers/spi/spi-amd.c +++ b/drivers/spi/spi-amd.c @@ -36,9 +36,17 @@ #define AMD_SPI_FIFO_SIZE 70 #define AMD_SPI_MEM_SIZE 200 -/* M_CMD OP codes for SPI */ -#define AMD_SPI_XFER_TX 1 -#define AMD_SPI_XFER_RX 2 +#define AMD_SPI_ENA_REG 0x20 +#define AMD_SPI_ALT_SPD_SHIFT 20 +#define AMD_SPI_ALT_SPD_MASK GENMASK(23, AMD_SPI_ALT_SPD_SHIFT) +#define AMD_SPI_SPI100_SHIFT 0 +#define AMD_SPI_SPI100_MASK GENMASK(AMD_SPI_SPI100_SHIFT, AMD_SPI_SPI100_SHIFT) +#define AMD_SPI_SPEED_REG 0x6C +#define AMD_SPI_SPD7_SHIFT 8 +#define AMD_SPI_SPD7_MASK GENMASK(13, AMD_SPI_SPD7_SHIFT) + +#define AMD_SPI_MAX_HZ 100000000 +#define AMD_SPI_MIN_HZ 800000 /** * enum amd_spi_versions - SPI controller versions @@ -50,14 +58,41 @@ enum amd_spi_versions { AMD_SPI_V2, }; +enum amd_spi_speed { + F_66_66MHz, + F_33_33MHz, + F_22_22MHz, + F_16_66MHz, + F_100MHz, + F_800KHz, + SPI_SPD7, + F_50MHz = 0x4, + F_4MHz = 0x32, + F_3_17MHz = 0x3F +}; + +/** + * struct amd_spi_freq - Matches device speed with values to write in regs + * @speed_hz: Device frequency + * @enable_val: Value to be written to "enable register" + * @spd7_val: Some frequencies requires to have a value written at SPISPEED register + */ +struct amd_spi_freq { + u32 speed_hz; + u32 enable_val; + u32 spd7_val; +}; + /** * struct amd_spi - SPI driver instance * @io_remap_addr: Start address of the SPI controller registers * @version: SPI controller hardware version + * @speed_hz: Device frequency */ struct amd_spi { void __iomem *io_remap_addr; enum amd_spi_versions version; + unsigned int speed_hz; }; static inline u8 amd_spi_readreg8(struct amd_spi *amd_spi, int idx) @@ -189,65 +224,125 @@ static int amd_spi_master_setup(struct spi_device *spi) return 0; } +static const struct amd_spi_freq amd_spi_freq[] = { + { AMD_SPI_MAX_HZ, F_100MHz, 0}, + { 66660000, F_66_66MHz, 0}, + { 50000000, SPI_SPD7, F_50MHz}, + { 33330000, F_33_33MHz, 0}, + { 22220000, F_22_22MHz, 0}, + { 16660000, F_16_66MHz, 0}, + { 4000000, SPI_SPD7, F_4MHz}, + { 3170000, SPI_SPD7, F_3_17MHz}, + { AMD_SPI_MIN_HZ, F_800KHz, 0}, +}; + +static int amd_set_spi_freq(struct amd_spi *amd_spi, u32 speed_hz) +{ + unsigned int i, spd7_val, alt_spd; + + if (speed_hz < AMD_SPI_MIN_HZ) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(amd_spi_freq); i++) + if (speed_hz >= amd_spi_freq[i].speed_hz) + break; + + if (amd_spi->speed_hz == amd_spi_freq[i].speed_hz) + return 0; + + amd_spi->speed_hz = amd_spi_freq[i].speed_hz; + + alt_spd = (amd_spi_freq[i].enable_val << AMD_SPI_ALT_SPD_SHIFT) + & AMD_SPI_ALT_SPD_MASK; + amd_spi_setclear_reg32(amd_spi, AMD_SPI_ENA_REG, alt_spd, + AMD_SPI_ALT_SPD_MASK); + + if (amd_spi->speed_hz == AMD_SPI_MAX_HZ) + amd_spi_setclear_reg32(amd_spi, AMD_SPI_ENA_REG, 1, + AMD_SPI_SPI100_MASK); + + if (amd_spi_freq[i].spd7_val) { + spd7_val = (amd_spi_freq[i].spd7_val << AMD_SPI_SPD7_SHIFT) + & AMD_SPI_SPD7_MASK; + amd_spi_setclear_reg32(amd_spi, AMD_SPI_SPEED_REG, spd7_val, + AMD_SPI_SPD7_MASK); + } + + return 0; +} + static inline int amd_spi_fifo_xfer(struct amd_spi *amd_spi, struct spi_master *master, struct spi_message *message) { struct spi_transfer *xfer = NULL; - u8 cmd_opcode; + struct spi_device *spi = message->spi; + u8 cmd_opcode = 0, fifo_pos = AMD_SPI_FIFO_BASE; u8 *buf = NULL; - u32 m_cmd = 0; u32 i = 0; u32 tx_len = 0, rx_len = 0; list_for_each_entry(xfer, &message->transfers, transfer_list) { - if (xfer->rx_buf) - m_cmd = AMD_SPI_XFER_RX; - if (xfer->tx_buf) - m_cmd = AMD_SPI_XFER_TX; + if (xfer->speed_hz) + amd_set_spi_freq(amd_spi, xfer->speed_hz); + else + amd_set_spi_freq(amd_spi, spi->max_speed_hz); - if (m_cmd & AMD_SPI_XFER_TX) { + if (xfer->tx_buf) { buf = (u8 *)xfer->tx_buf; - tx_len = xfer->len - 1; - cmd_opcode = *(u8 *)xfer->tx_buf; - buf++; - amd_spi_set_opcode(amd_spi, cmd_opcode); + if (!tx_len) { + cmd_opcode = *(u8 *)xfer->tx_buf; + buf++; + xfer->len--; + } + tx_len += xfer->len; /* Write data into the FIFO. */ - for (i = 0; i < tx_len; i++) { - iowrite8(buf[i], ((u8 __iomem *)amd_spi->io_remap_addr + - AMD_SPI_FIFO_BASE + i)); - } + for (i = 0; i < xfer->len; i++) + amd_spi_writereg8(amd_spi, fifo_pos + i, buf[i]); - amd_spi_set_tx_count(amd_spi, tx_len); - amd_spi_clear_fifo_ptr(amd_spi); - /* Execute command */ - amd_spi_execute_opcode(amd_spi); - } - if (m_cmd & AMD_SPI_XFER_RX) { - /* - * Store no. of bytes to be received from - * FIFO - */ - rx_len = xfer->len; - buf = (u8 *)xfer->rx_buf; - amd_spi_set_rx_count(amd_spi, rx_len); - amd_spi_clear_fifo_ptr(amd_spi); - /* Execute command */ - amd_spi_execute_opcode(amd_spi); - amd_spi_busy_wait(amd_spi); - /* Read data from FIFO to receive buffer */ - for (i = 0; i < rx_len; i++) - buf[i] = amd_spi_readreg8(amd_spi, AMD_SPI_FIFO_BASE + tx_len + i); + fifo_pos += xfer->len; } + + /* Store no. of bytes to be received from FIFO */ + if (xfer->rx_buf) + rx_len += xfer->len; + } + + if (!buf) { + message->status = -EINVAL; + goto fin_msg; + } + + amd_spi_set_opcode(amd_spi, cmd_opcode); + amd_spi_set_tx_count(amd_spi, tx_len); + amd_spi_set_rx_count(amd_spi, rx_len); + + /* Execute command */ + message->status = amd_spi_execute_opcode(amd_spi); + if (message->status) + goto fin_msg; + + if (rx_len) { + message->status = amd_spi_busy_wait(amd_spi); + if (message->status) + goto fin_msg; + + list_for_each_entry(xfer, &message->transfers, transfer_list) + if (xfer->rx_buf) { + buf = (u8 *)xfer->rx_buf; + /* Read data from FIFO to receive buffer */ + for (i = 0; i < xfer->len; i++) + buf[i] = amd_spi_readreg8(amd_spi, fifo_pos + i); + fifo_pos += xfer->len; + } } /* Update statistics */ message->actual_length = tx_len + rx_len + 1; - /* complete the transaction */ - message->status = 0; +fin_msg: switch (amd_spi->version) { case AMD_SPI_V1: break; @@ -260,7 +355,7 @@ static inline int amd_spi_fifo_xfer(struct amd_spi *amd_spi, spi_finalize_current_message(master); - return 0; + return message->status; } static int amd_spi_master_transfer(struct spi_master *master, @@ -275,9 +370,7 @@ static int amd_spi_master_transfer(struct spi_master *master, * Extract spi_transfers from the spi message and * program the controller. */ - amd_spi_fifo_xfer(amd_spi, master, msg); - - return 0; + return amd_spi_fifo_xfer(amd_spi, master, msg); } static size_t amd_spi_max_transfer_size(struct spi_device *spi) @@ -312,6 +405,8 @@ static int amd_spi_probe(struct platform_device *pdev) master->num_chipselect = 4; master->mode_bits = 0; master->flags = SPI_MASTER_HALF_DUPLEX; + master->max_speed_hz = AMD_SPI_MAX_HZ; + master->min_speed_hz = AMD_SPI_MIN_HZ; master->setup = amd_spi_master_setup; master->transfer_one_message = amd_spi_master_transfer; master->max_transfer_size = amd_spi_max_transfer_size; diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c index 3e891bf22470e0610c64c5e15bf0a2f3f1e956eb..a334e89add869e1c6fe0970e300ca8a3c8cc1937 100644 --- a/drivers/spi/spi-aspeed-smc.c +++ b/drivers/spi/spi-aspeed-smc.c @@ -736,10 +736,8 @@ static int aspeed_spi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); aspi->regs = devm_ioremap_resource(dev, res); - if (IS_ERR(aspi->regs)) { - dev_err(dev, "missing AHB register window\n"); + if (IS_ERR(aspi->regs)) return PTR_ERR(aspi->regs); - } res = platform_get_resource(pdev, IORESOURCE_MEM, 1); aspi->ahb_base = devm_ioremap_resource(dev, res); diff --git a/drivers/spi/spi-bitbang-txrx.h b/drivers/spi/spi-bitbang-txrx.h index 267342dfa73880d3db4f1573f8a79ba32436e728..2dcbe166df63e31d782be489a1d59a65b3165e6e 100644 --- a/drivers/spi/spi-bitbang-txrx.h +++ b/drivers/spi/spi-bitbang-txrx.h @@ -116,6 +116,7 @@ bitbang_txrx_le_cpha0(struct spi_device *spi, { /* if (cpol == 0) this is SPI_MODE_0; else this is SPI_MODE_2 */ + u8 rxbit = bits - 1; u32 oldbit = !(word & 1); /* clock starts at inactive polarity */ for (; likely(bits); bits--) { @@ -135,7 +136,7 @@ bitbang_txrx_le_cpha0(struct spi_device *spi, /* sample LSB (from slave) on leading edge */ word >>= 1; if ((flags & SPI_MASTER_NO_RX) == 0) - word |= getmiso(spi) << (bits - 1); + word |= getmiso(spi) << rxbit; setsck(spi, cpol); } return word; @@ -148,6 +149,7 @@ bitbang_txrx_le_cpha1(struct spi_device *spi, { /* if (cpol == 0) this is SPI_MODE_1; else this is SPI_MODE_3 */ + u8 rxbit = bits - 1; u32 oldbit = !(word & 1); /* clock starts at inactive polarity */ for (; likely(bits); bits--) { @@ -168,7 +170,7 @@ bitbang_txrx_le_cpha1(struct spi_device *spi, /* sample LSB (from slave) on trailing edge */ word >>= 1; if ((flags & SPI_MASTER_NO_RX) == 0) - word |= getmiso(spi) << (bits - 1); + word |= getmiso(spi) << rxbit; } return word; } diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 72b1a5a2298c551c34a80aefd56af16d98949f40..44723054794524f1b0e933eb219055784178b2f1 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -39,6 +39,7 @@ #define CQSPI_DISABLE_DAC_MODE BIT(1) #define CQSPI_SUPPORT_EXTERNAL_DMA BIT(2) #define CQSPI_NO_SUPPORT_WR_COMPLETION BIT(3) +#define CQSPI_SLOW_SRAM BIT(4) /* Capabilities */ #define CQSPI_SUPPORTS_OCTAL BIT(0) @@ -87,6 +88,7 @@ struct cqspi_st { bool use_dma_read; u32 pd_dev_id; bool wr_completion; + bool slow_sram; }; struct cqspi_driver_platdata { @@ -333,7 +335,10 @@ static irqreturn_t cqspi_irq_handler(int this_irq, void *dev) } } - irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR; + else if (!cqspi->slow_sram) + irq_status &= CQSPI_IRQ_MASK_RD | CQSPI_IRQ_MASK_WR; + else + irq_status &= CQSPI_REG_IRQ_WATERMARK | CQSPI_IRQ_MASK_WR; if (irq_status) complete(&cqspi->transfer_complete); @@ -673,7 +678,18 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, /* Clear all interrupts. */ writel(CQSPI_IRQ_STATUS_MASK, reg_base + CQSPI_REG_IRQSTATUS); - writel(CQSPI_IRQ_MASK_RD, reg_base + CQSPI_REG_IRQMASK); + /* + * On SoCFPGA platform reading the SRAM is slow due to + * hardware limitation and causing read interrupt storm to CPU, + * so enabling only watermark interrupt to disable all read + * interrupts later as we want to run "bytes to read" loop with + * all the read interrupts disabled for max performance. + */ + + if (!cqspi->slow_sram) + writel(CQSPI_IRQ_MASK_RD, reg_base + CQSPI_REG_IRQMASK); + else + writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK); reinit_completion(&cqspi->transfer_complete); writel(CQSPI_REG_INDIRECTRD_START_MASK, @@ -684,6 +700,13 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, msecs_to_jiffies(CQSPI_READ_TIMEOUT_MS))) ret = -ETIMEDOUT; + /* + * Disable all read interrupts until + * we are out of "bytes to read" + */ + if (cqspi->slow_sram) + writel(0x0, reg_base + CQSPI_REG_IRQMASK); + bytes_to_read = cqspi_get_rd_sram_level(cqspi); if (ret && bytes_to_read == 0) { @@ -715,8 +738,11 @@ static int cqspi_indirect_read_execute(struct cqspi_flash_pdata *f_pdata, bytes_to_read = cqspi_get_rd_sram_level(cqspi); } - if (remaining > 0) + if (remaining > 0) { reinit_completion(&cqspi->transfer_complete); + if (cqspi->slow_sram) + writel(CQSPI_REG_IRQ_WATERMARK, reg_base + CQSPI_REG_IRQMASK); + } } /* Check indirect done status */ @@ -1619,7 +1645,7 @@ static int cqspi_probe(struct platform_device *pdev) pm_runtime_enable(dev); ret = pm_runtime_resume_and_get(dev); if (ret < 0) - return ret; + goto probe_pm_failed; ret = clk_prepare_enable(cqspi->clk); if (ret) { @@ -1667,6 +1693,8 @@ static int cqspi_probe(struct platform_device *pdev) cqspi->use_dma_read = true; if (ddata->quirks & CQSPI_NO_SUPPORT_WR_COMPLETION) cqspi->wr_completion = false; + if (ddata->quirks & CQSPI_SLOW_SRAM) + cqspi->slow_sram = true; if (of_device_is_compatible(pdev->dev.of_node, "xlnx,versal-ospi-1.0")) @@ -1712,6 +1740,7 @@ probe_reset_failed: clk_disable_unprepare(cqspi->clk); probe_clk_failed: pm_runtime_put_sync(dev); +probe_pm_failed: pm_runtime_disable(dev); return ret; } @@ -1779,7 +1808,9 @@ static const struct cqspi_driver_platdata intel_lgm_qspi = { }; static const struct cqspi_driver_platdata socfpga_qspi = { - .quirks = CQSPI_DISABLE_DAC_MODE | CQSPI_NO_SUPPORT_WR_COMPLETION, + .quirks = CQSPI_DISABLE_DAC_MODE + | CQSPI_NO_SUPPORT_WR_COMPLETION + | CQSPI_SLOW_SRAM, }; static const struct cqspi_driver_platdata versal_ospi = { diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c index 3ab19be8309514db4b96bcc94321b620fd9d4db4..9e187f9c6c95eb29ec76080e4aeb0221896ed70b 100644 --- a/drivers/spi/spi-cadence-xspi.c +++ b/drivers/spi/spi-cadence-xspi.c @@ -565,10 +565,8 @@ static int cdns_xspi_probe(struct platform_device *pdev) res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sdma"); cdns_xspi->sdmabase = devm_ioremap_resource(dev, res); - if (IS_ERR(cdns_xspi->sdmabase)) { - dev_err(dev, "Failed to remap SDMA address\n"); + if (IS_ERR(cdns_xspi->sdmabase)) return PTR_ERR(cdns_xspi->sdmabase); - } cdns_xspi->sdmasize = resource_size(res); cdns_xspi->auxbase = devm_platform_ioremap_resource_byname(pdev, "aux"); diff --git a/drivers/spi/spi-dw-bt1.c b/drivers/spi/spi-dw-bt1.c index c0655341612371d88ca14e46d1d60a61983fc526..3fb89dee595e7ac44d85b0a1aadea341623c9122 100644 --- a/drivers/spi/spi-dw-bt1.c +++ b/drivers/spi/spi-dw-bt1.c @@ -293,8 +293,10 @@ static int dw_spi_bt1_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); ret = dw_spi_add_host(&pdev->dev, dws); - if (ret) + if (ret) { + pm_runtime_disable(&pdev->dev); goto err_disable_clk; + } platform_set_drvdata(pdev, dwsbt1); diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c index f87d97ccd2d68330f456c0ce81f5bf62e0c95dc0..99edddf9958b956c1bbe03f1a71998e806c78e7e 100644 --- a/drivers/spi/spi-dw-core.c +++ b/drivers/spi/spi-dw-core.c @@ -955,7 +955,7 @@ int dw_spi_add_host(struct device *dev, struct dw_spi *dws) ret = spi_register_controller(master); if (ret) { - dev_err(&master->dev, "problem registering spi master\n"); + dev_err_probe(dev, ret, "problem registering spi master\n"); goto err_dma_exit; } diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index fd004c9db9dc0dcab9ffd939c7f7ef3660270ccc..a33e547b7d39597dbb1e5bc02003fbd3af91acc8 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -1294,8 +1294,7 @@ static int dspi_probe(struct platform_device *pdev) else ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(&pdev->dev, res); + base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(base)) { ret = PTR_ERR(base); goto out_ctlr_put; diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index 19b1f3d881b08ef9953597cad182cf780bf7fc94..e8c1c8a4c6c82f509cffce8fc924b627a8c33458 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -855,8 +855,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev) init_completion(&fsl_lpspi->xfer_done); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - fsl_lpspi->base = devm_ioremap_resource(&pdev->dev, res); + fsl_lpspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(fsl_lpspi->base)) { ret = PTR_ERR(fsl_lpspi->base); goto out_controller_put; @@ -912,7 +911,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev) ret = devm_spi_register_controller(&pdev->dev, controller); if (ret < 0) { - dev_err_probe(&pdev->dev, ret, "spi_register_controller error: %i\n", ret); + dev_err_probe(&pdev->dev, ret, "spi_register_controller error\n"); goto free_dma; } @@ -947,11 +946,8 @@ static int fsl_lpspi_remove(struct platform_device *pdev) static int __maybe_unused fsl_lpspi_suspend(struct device *dev) { - int ret; - pinctrl_pm_select_sleep_state(dev); - ret = pm_runtime_force_suspend(dev); - return ret; + return pm_runtime_force_suspend(dev); } static int __maybe_unused fsl_lpspi_resume(struct device *dev) diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c index 46ae46a944c5ce89818ddef2f2a622a96ee95036..85cc71ba624a9070228b189e1d00d7673803a474 100644 --- a/drivers/spi/spi-fsl-qspi.c +++ b/drivers/spi/spi-fsl-qspi.c @@ -867,8 +867,7 @@ static int fsl_qspi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, q); /* find the resources */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "QuadSPI"); - q->iobase = devm_ioremap_resource(dev, res); + q->iobase = devm_platform_ioremap_resource_byname(pdev, "QuadSPI"); if (IS_ERR(q->iobase)) { ret = PTR_ERR(q->iobase); goto err_put_ctrl; diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c index bdf94cc7be1afe18c4939e39262c47a752c6d945..731624f157fc0fa2122065b44bffa0b751d3451d 100644 --- a/drivers/spi/spi-fsl-spi.c +++ b/drivers/spi/spi-fsl-spi.c @@ -111,32 +111,6 @@ static void fsl_spi_change_mode(struct spi_device *spi) local_irq_restore(flags); } -static void fsl_spi_chipselect(struct spi_device *spi, int value) -{ - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master); - struct fsl_spi_platform_data *pdata; - struct spi_mpc8xxx_cs *cs = spi->controller_state; - - pdata = spi->dev.parent->parent->platform_data; - - if (value == BITBANG_CS_INACTIVE) { - if (pdata->cs_control) - pdata->cs_control(spi, false); - } - - if (value == BITBANG_CS_ACTIVE) { - mpc8xxx_spi->rx_shift = cs->rx_shift; - mpc8xxx_spi->tx_shift = cs->tx_shift; - mpc8xxx_spi->get_rx = cs->get_rx; - mpc8xxx_spi->get_tx = cs->get_tx; - - fsl_spi_change_mode(spi); - - if (pdata->cs_control) - pdata->cs_control(spi, true); - } -} - static void fsl_spi_qe_cpu_set_shifts(u32 *rx_shift, u32 *tx_shift, int bits_per_word, int msb_first) { @@ -354,15 +328,11 @@ static int fsl_spi_bufs(struct spi_device *spi, struct spi_transfer *t, return mpc8xxx_spi->count; } -static int fsl_spi_do_one_msg(struct spi_master *master, - struct spi_message *m) +static int fsl_spi_prepare_message(struct spi_controller *ctlr, + struct spi_message *m) { - struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master); - struct spi_device *spi = m->spi; - struct spi_transfer *t, *first; - unsigned int cs_change; - const int nsecs = 50; - int status, last_bpw; + struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(ctlr); + struct spi_transfer *t; /* * In CPU mode, optimize large byte transfers to use larger @@ -378,62 +348,30 @@ static int fsl_spi_do_one_msg(struct spi_master *master, t->bits_per_word = 16; } } + return 0; +} - /* Don't allow changes if CS is active */ - cs_change = 1; - list_for_each_entry(t, &m->transfers, transfer_list) { - if (cs_change) - first = t; - cs_change = t->cs_change; - if (first->speed_hz != t->speed_hz) { - dev_err(&spi->dev, - "speed_hz cannot change while CS is active\n"); - return -EINVAL; - } - } - - last_bpw = -1; - cs_change = 1; - status = -EINVAL; - list_for_each_entry(t, &m->transfers, transfer_list) { - if (cs_change || last_bpw != t->bits_per_word) - status = fsl_spi_setup_transfer(spi, t); - if (status < 0) - break; - last_bpw = t->bits_per_word; - - if (cs_change) { - fsl_spi_chipselect(spi, BITBANG_CS_ACTIVE); - ndelay(nsecs); - } - cs_change = t->cs_change; - if (t->len) - status = fsl_spi_bufs(spi, t, m->is_dma_mapped); - if (status) { - status = -EMSGSIZE; - break; - } - m->actual_length += t->len; - - spi_transfer_delay_exec(t); - - if (cs_change) { - ndelay(nsecs); - fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE); - ndelay(nsecs); - } - } +static int fsl_spi_transfer_one(struct spi_controller *controller, + struct spi_device *spi, + struct spi_transfer *t) +{ + int status; - m->status = status; + status = fsl_spi_setup_transfer(spi, t); + if (status < 0) + return status; + if (t->len) + status = fsl_spi_bufs(spi, t, !!t->tx_dma || !!t->rx_dma); + if (status > 0) + return -EMSGSIZE; - if (status || !cs_change) { - ndelay(nsecs); - fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE); - } + return status; +} - fsl_spi_setup_transfer(spi, NULL); - spi_finalize_current_message(master); - return 0; +static int fsl_spi_unprepare_message(struct spi_controller *controller, + struct spi_message *msg) +{ + return fsl_spi_setup_transfer(msg->spi, NULL); } static int fsl_spi_setup(struct spi_device *spi) @@ -482,9 +420,6 @@ static int fsl_spi_setup(struct spi_device *spi) return retval; } - /* Initialize chipselect - might be active for SPI_CS_HIGH mode */ - fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE); - return 0; } @@ -557,9 +492,7 @@ static void fsl_spi_grlib_cs_control(struct spi_device *spi, bool on) u32 slvsel; u16 cs = spi->chip_select; - if (spi->cs_gpiod) { - gpiod_set_value(spi->cs_gpiod, on); - } else if (cs < mpc8xxx_spi->native_chipselects) { + if (cs < mpc8xxx_spi->native_chipselects) { slvsel = mpc8xxx_spi_read_reg(®_base->slvsel); slvsel = on ? (slvsel | (1 << cs)) : (slvsel & ~(1 << cs)); mpc8xxx_spi_write_reg(®_base->slvsel, slvsel); @@ -568,7 +501,6 @@ static void fsl_spi_grlib_cs_control(struct spi_device *spi, bool on) static void fsl_spi_grlib_probe(struct device *dev) { - struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); struct spi_master *master = dev_get_drvdata(dev); struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master); struct fsl_spi_reg __iomem *reg_base = mpc8xxx_spi->reg_base; @@ -588,7 +520,18 @@ static void fsl_spi_grlib_probe(struct device *dev) mpc8xxx_spi_write_reg(®_base->slvsel, 0xffffffff); } master->num_chipselect = mpc8xxx_spi->native_chipselects; - pdata->cs_control = fsl_spi_grlib_cs_control; + master->set_cs = fsl_spi_grlib_cs_control; +} + +static void fsl_spi_cs_control(struct spi_device *spi, bool on) +{ + struct device *dev = spi->dev.parent->parent; + struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); + struct mpc8xxx_spi_probe_info *pinfo = to_of_pinfo(pdata); + + if (WARN_ON_ONCE(!pinfo->immr_spi_cs)) + return; + iowrite32be(on ? 0 : SPI_BOOT_SEL_BIT, pinfo->immr_spi_cs); } static struct spi_master *fsl_spi_probe(struct device *dev, @@ -613,8 +556,11 @@ static struct spi_master *fsl_spi_probe(struct device *dev, master->setup = fsl_spi_setup; master->cleanup = fsl_spi_cleanup; - master->transfer_one_message = fsl_spi_do_one_msg; + master->prepare_message = fsl_spi_prepare_message; + master->transfer_one = fsl_spi_transfer_one; + master->unprepare_message = fsl_spi_unprepare_message; master->use_gpio_descriptors = true; + master->set_cs = fsl_spi_cs_control; mpc8xxx_spi = spi_master_get_devdata(master); mpc8xxx_spi->max_bits_per_word = 32; @@ -688,21 +634,6 @@ err: return ERR_PTR(ret); } -static void fsl_spi_cs_control(struct spi_device *spi, bool on) -{ - if (spi->cs_gpiod) { - gpiod_set_value(spi->cs_gpiod, on); - } else { - struct device *dev = spi->dev.parent->parent; - struct fsl_spi_platform_data *pdata = dev_get_platdata(dev); - struct mpc8xxx_spi_probe_info *pinfo = to_of_pinfo(pdata); - - if (WARN_ON_ONCE(!pinfo->immr_spi_cs)) - return; - iowrite32be(on ? 0 : SPI_BOOT_SEL_BIT, pinfo->immr_spi_cs); - } -} - static int of_fsl_spi_probe(struct platform_device *ofdev) { struct device *dev = &ofdev->dev; @@ -744,12 +675,10 @@ static int of_fsl_spi_probe(struct platform_device *ofdev) ret = gpiod_count(dev, "cs"); if (ret < 0) ret = 0; - if (ret == 0 && !spisel_boot) { + if (ret == 0 && !spisel_boot) pdata->max_chipselect = 1; - } else { + else pdata->max_chipselect = ret + spisel_boot; - pdata->cs_control = fsl_spi_cs_control; - } } ret = of_address_to_resource(np, 0, &mem); diff --git a/drivers/spi/spi-gxp.c b/drivers/spi/spi-gxp.c index 9ea355f7d64f90eee202329da9e78c30e8997dee..15b11018383909763ddf45991c3bf61dd74d647d 100644 --- a/drivers/spi/spi-gxp.c +++ b/drivers/spi/spi-gxp.c @@ -254,7 +254,6 @@ static int gxp_spifi_probe(struct platform_device *pdev) const struct gxp_spi_data *data; struct spi_controller *ctlr; struct gxp_spi *spifi; - struct resource *res; int ret; data = of_device_get_match_data(&pdev->dev); @@ -269,18 +268,15 @@ static int gxp_spifi_probe(struct platform_device *pdev) spifi->data = data; spifi->dev = dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - spifi->reg_base = devm_ioremap_resource(&pdev->dev, res); + spifi->reg_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(spifi->reg_base)) return PTR_ERR(spifi->reg_base); - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - spifi->dat_base = devm_ioremap_resource(&pdev->dev, res); + spifi->dat_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(spifi->dat_base)) return PTR_ERR(spifi->dat_base); - res = platform_get_resource(pdev, IORESOURCE_MEM, 2); - spifi->dir_base = devm_ioremap_resource(&pdev->dev, res); + spifi->dir_base = devm_platform_ioremap_resource(pdev, 2); if (IS_ERR(spifi->dir_base)) return PTR_ERR(spifi->dir_base); diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index 71376b6df89dbb279dd7f5dda18ed41f305b9155..bfd12247f17344d9cd9c7f0e6dae4408c8baf35b 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -730,11 +730,9 @@ static int img_spfi_resume(struct device *dev) struct img_spfi *spfi = spi_master_get_devdata(master); int ret; - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_noidle(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) return ret; - } spfi_reset(spfi); pm_runtime_put(dev); diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 66063687ae2714745f869ddb78c903b32fcf031b..55f4ee2db002ce9d808e7e957b1018d78976c5f5 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -116,6 +116,22 @@ #define ERASE_64K_OPCODE_SHIFT 16 #define ERASE_64K_OPCODE_MASK (0xff << ERASE_OPCODE_SHIFT) +/* Flash descriptor fields */ +#define FLVALSIG_MAGIC 0x0ff0a55a +#define FLMAP0_NC_MASK GENMASK(9, 8) +#define FLMAP0_NC_SHIFT 8 +#define FLMAP0_FCBA_MASK GENMASK(7, 0) + +#define FLCOMP_C0DEN_MASK GENMASK(3, 0) +#define FLCOMP_C0DEN_512K 0x00 +#define FLCOMP_C0DEN_1M 0x01 +#define FLCOMP_C0DEN_2M 0x02 +#define FLCOMP_C0DEN_4M 0x03 +#define FLCOMP_C0DEN_8M 0x04 +#define FLCOMP_C0DEN_16M 0x05 +#define FLCOMP_C0DEN_32M 0x06 +#define FLCOMP_C0DEN_64M 0x07 + #define INTEL_SPI_TIMEOUT 5000 /* ms */ #define INTEL_SPI_FIFO_SZ 64 @@ -129,6 +145,7 @@ * @master: Pointer to the SPI controller structure * @nregions: Maximum number of regions * @pr_num: Maximum number of protected range registers + * @chip0_size: Size of the first flash chip in bytes * @locked: Is SPI setting locked * @swseq_reg: Use SW sequencer in register reads/writes * @swseq_erase: Use SW sequencer in erase operation @@ -146,6 +163,7 @@ struct intel_spi { struct spi_controller *master; size_t nregions; size_t pr_num; + size_t chip0_size; bool locked; bool swseq_reg; bool swseq_erase; @@ -158,6 +176,7 @@ struct intel_spi_mem_op { struct spi_mem_op mem_op; u32 replacement_op; int (*exec_op)(struct intel_spi *ispi, + const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op); }; @@ -441,7 +460,16 @@ static int intel_spi_sw_cycle(struct intel_spi *ispi, u8 opcode, size_t len, return 0; } -static int intel_spi_read_reg(struct intel_spi *ispi, +static u32 intel_spi_chip_addr(const struct intel_spi *ispi, + const struct spi_mem *mem) +{ + /* Pick up the correct start address */ + if (!mem) + return 0; + return mem->spi->chip_select == 1 ? ispi->chip0_size : 0; +} + +static int intel_spi_read_reg(struct intel_spi *ispi, const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op) { @@ -449,8 +477,7 @@ static int intel_spi_read_reg(struct intel_spi *ispi, u8 opcode = op->cmd.opcode; int ret; - /* Address of the first chip */ - writel(0, ispi->base + FADDR); + writel(intel_spi_chip_addr(ispi, mem), ispi->base + FADDR); if (ispi->swseq_reg) ret = intel_spi_sw_cycle(ispi, opcode, nbytes, @@ -464,7 +491,7 @@ static int intel_spi_read_reg(struct intel_spi *ispi, return intel_spi_read_block(ispi, op->data.buf.in, nbytes); } -static int intel_spi_write_reg(struct intel_spi *ispi, +static int intel_spi_write_reg(struct intel_spi *ispi, const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op) { @@ -511,7 +538,7 @@ static int intel_spi_write_reg(struct intel_spi *ispi, if (opcode == SPINOR_OP_WRDI) return 0; - writel(0, ispi->base + FADDR); + writel(intel_spi_chip_addr(ispi, mem), ispi->base + FADDR); /* Write the value beforehand */ ret = intel_spi_write_block(ispi, op->data.buf.out, nbytes); @@ -524,13 +551,13 @@ static int intel_spi_write_reg(struct intel_spi *ispi, return intel_spi_hw_cycle(ispi, opcode, nbytes); } -static int intel_spi_read(struct intel_spi *ispi, +static int intel_spi_read(struct intel_spi *ispi, const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op) { - void *read_buf = op->data.buf.in; + u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val; size_t block_size, nbytes = op->data.nbytes; - u32 addr = op->addr.val; + void *read_buf = op->data.buf.in; u32 val, status; int ret; @@ -585,13 +612,13 @@ static int intel_spi_read(struct intel_spi *ispi, return 0; } -static int intel_spi_write(struct intel_spi *ispi, +static int intel_spi_write(struct intel_spi *ispi, const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op) { + u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val; size_t block_size, nbytes = op->data.nbytes; const void *write_buf = op->data.buf.out; - u32 addr = op->addr.val; u32 val, status; int ret; @@ -648,12 +675,12 @@ static int intel_spi_write(struct intel_spi *ispi, return 0; } -static int intel_spi_erase(struct intel_spi *ispi, +static int intel_spi_erase(struct intel_spi *ispi, const struct spi_mem *mem, const struct intel_spi_mem_op *iop, const struct spi_mem_op *op) { + u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val; u8 opcode = op->cmd.opcode; - u32 addr = op->addr.val; u32 val, status; int ret; @@ -765,7 +792,7 @@ static int intel_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *o if (!iop) return -EOPNOTSUPP; - return iop->exec_op(ispi, iop, op); + return iop->exec_op(ispi, mem, iop, op); } static const char *intel_spi_get_name(struct spi_mem *mem) @@ -805,7 +832,7 @@ static ssize_t intel_spi_dirmap_read(struct spi_mem_dirmap_desc *desc, u64 offs, op.data.nbytes = len; op.data.buf.in = buf; - ret = iop->exec_op(ispi, iop, &op); + ret = iop->exec_op(ispi, desc->mem, iop, &op); return ret ? ret : len; } @@ -821,7 +848,7 @@ static ssize_t intel_spi_dirmap_write(struct spi_mem_dirmap_desc *desc, u64 offs op.data.nbytes = len; op.data.buf.out = buf; - ret = iop->exec_op(ispi, iop, &op); + ret = iop->exec_op(ispi, desc->mem, iop, &op); return ret ? ret : len; } @@ -1073,6 +1100,7 @@ static int intel_spi_init(struct intel_spi *ispi) ispi->pregs = ispi->base + CNL_PR; ispi->nregions = CNL_FREG_NUM; ispi->pr_num = CNL_PR_NUM; + erase_64k = true; break; default: @@ -1226,10 +1254,98 @@ static void intel_spi_fill_partition(struct intel_spi *ispi, } } +static int intel_spi_read_desc(struct intel_spi *ispi) +{ + struct spi_mem_op op = + SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_READ, 0), + SPI_MEM_OP_ADDR(3, 0, 0), + SPI_MEM_OP_NO_DUMMY, + SPI_MEM_OP_DATA_IN(0, NULL, 0)); + u32 buf[2], nc, fcba, flcomp; + ssize_t ret; + + op.addr.val = 0x10; + op.data.buf.in = buf; + op.data.nbytes = sizeof(buf); + + ret = intel_spi_read(ispi, NULL, NULL, &op); + if (ret) { + dev_warn(ispi->dev, "failed to read descriptor\n"); + return ret; + } + + dev_dbg(ispi->dev, "FLVALSIG=0x%08x\n", buf[0]); + dev_dbg(ispi->dev, "FLMAP0=0x%08x\n", buf[1]); + + if (buf[0] != FLVALSIG_MAGIC) { + dev_warn(ispi->dev, "descriptor signature not valid\n"); + return -ENODEV; + } + + fcba = (buf[1] & FLMAP0_FCBA_MASK) << 4; + dev_dbg(ispi->dev, "FCBA=%#x\n", fcba); + + op.addr.val = fcba; + op.data.buf.in = &flcomp; + op.data.nbytes = sizeof(flcomp); + + ret = intel_spi_read(ispi, NULL, NULL, &op); + if (ret) { + dev_warn(ispi->dev, "failed to read FLCOMP\n"); + return -ENODEV; + } + + dev_dbg(ispi->dev, "FLCOMP=0x%08x\n", flcomp); + + switch (flcomp & FLCOMP_C0DEN_MASK) { + case FLCOMP_C0DEN_512K: + ispi->chip0_size = SZ_512K; + break; + case FLCOMP_C0DEN_1M: + ispi->chip0_size = SZ_1M; + break; + case FLCOMP_C0DEN_2M: + ispi->chip0_size = SZ_2M; + break; + case FLCOMP_C0DEN_4M: + ispi->chip0_size = SZ_4M; + break; + case FLCOMP_C0DEN_8M: + ispi->chip0_size = SZ_8M; + break; + case FLCOMP_C0DEN_16M: + ispi->chip0_size = SZ_16M; + break; + case FLCOMP_C0DEN_32M: + ispi->chip0_size = SZ_32M; + break; + case FLCOMP_C0DEN_64M: + ispi->chip0_size = SZ_64M; + break; + default: + return -EINVAL; + } + + dev_dbg(ispi->dev, "chip0 size %zd KB\n", ispi->chip0_size / SZ_1K); + + nc = (buf[1] & FLMAP0_NC_MASK) >> FLMAP0_NC_SHIFT; + if (!nc) + ispi->master->num_chipselect = 1; + else if (nc == 1) + ispi->master->num_chipselect = 2; + else + return -EINVAL; + + dev_dbg(ispi->dev, "%u flash components found\n", + ispi->master->num_chipselect); + return 0; +} + static int intel_spi_populate_chip(struct intel_spi *ispi) { struct flash_platform_data *pdata; struct spi_board_info chip; + int ret; pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -1247,7 +1363,23 @@ static int intel_spi_populate_chip(struct intel_spi *ispi) snprintf(chip.modalias, 8, "spi-nor"); chip.platform_data = pdata; - return spi_new_device(ispi->master, &chip) ? 0 : -ENODEV; + if (!spi_new_device(ispi->master, &chip)) + return -ENODEV; + + /* Add the second chip if present */ + if (ispi->master->num_chipselect < 2) + return 0; + + ret = intel_spi_read_desc(ispi); + if (ret) + return ret; + + chip.platform_data = NULL; + chip.chip_select = 1; + + if (!spi_new_device(ispi->master, &chip)) + return -ENODEV; + return 0; } /** diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c index 4d4f77a186a98a3a34f56f70e3b7d20d1d316860..dd7de8fa37d0477d1a0072ba5e88168d5f2f601a 100644 --- a/drivers/spi/spi-loopback-test.c +++ b/drivers/spi/spi-loopback-test.c @@ -313,6 +313,33 @@ static struct spi_test spi_tests[] = { }, }, }, + { + .description = "three tx+rx transfers with overlapping cache lines", + .fill_option = FILL_COUNT_8, + /* + * This should be large enough for the controller driver to + * choose to transfer it with DMA. + */ + .iterate_len = { 512, -1 }, + .iterate_transfer_mask = BIT(1), + .transfer_count = 3, + .transfers = { + { + .len = 1, + .tx_buf = TX(0), + .rx_buf = RX(0), + }, + { + .tx_buf = TX(1), + .rx_buf = RX(1), + }, + { + .len = 1, + .tx_buf = TX(513), + .rx_buf = RX(513), + }, + }, + }, { /* end of tests sequence */ } }; diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index e4cb52e1fe2619d7699413d05d5dacc1cbf32a0e..bad201510a99261eb7d4804358e7ba0696f6cf59 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -537,7 +537,7 @@ static unsigned long meson_spicc_pow2_recalc_rate(struct clk_hw *hw, struct clk_divider *divider = to_clk_divider(hw); struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider); - if (!spicc->master->cur_msg || !spicc->master->busy) + if (!spicc->master->cur_msg) return 0; return clk_divider_ops.recalc_rate(hw, parent_rate); @@ -549,7 +549,7 @@ static int meson_spicc_pow2_determine_rate(struct clk_hw *hw, struct clk_divider *divider = to_clk_divider(hw); struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider); - if (!spicc->master->cur_msg || !spicc->master->busy) + if (!spicc->master->cur_msg) return -EINVAL; return clk_divider_ops.determine_rate(hw, req); @@ -561,13 +561,13 @@ static int meson_spicc_pow2_set_rate(struct clk_hw *hw, unsigned long rate, struct clk_divider *divider = to_clk_divider(hw); struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider); - if (!spicc->master->cur_msg || !spicc->master->busy) + if (!spicc->master->cur_msg) return -EINVAL; return clk_divider_ops.set_rate(hw, rate, parent_rate); } -const struct clk_ops meson_spicc_pow2_clk_ops = { +static const struct clk_ops meson_spicc_pow2_clk_ops = { .recalc_rate = meson_spicc_pow2_recalc_rate, .determine_rate = meson_spicc_pow2_determine_rate, .set_rate = meson_spicc_pow2_set_rate, diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c new file mode 100644 index 0000000000000000000000000000000000000000..19a6a46829f6d963ee671701e9b5a299fc6cdc6e --- /dev/null +++ b/drivers/spi/spi-microchip-core-qspi.c @@ -0,0 +1,600 @@ +// SPDX-License-Identifier: (GPL-2.0) +/* + * Microchip coreQSPI QSPI controller driver + * + * Copyright (C) 2018-2022 Microchip Technology Inc. and its subsidiaries + * + * Author: Naga Sureshkumar Relli + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * QSPI Control register mask defines + */ +#define CONTROL_ENABLE BIT(0) +#define CONTROL_MASTER BIT(1) +#define CONTROL_XIP BIT(2) +#define CONTROL_XIPADDR BIT(3) +#define CONTROL_CLKIDLE BIT(10) +#define CONTROL_SAMPLE_MASK GENMASK(12, 11) +#define CONTROL_MODE0 BIT(13) +#define CONTROL_MODE12_MASK GENMASK(15, 14) +#define CONTROL_MODE12_EX_RO BIT(14) +#define CONTROL_MODE12_EX_RW BIT(15) +#define CONTROL_MODE12_FULL GENMASK(15, 14) +#define CONTROL_FLAGSX4 BIT(16) +#define CONTROL_CLKRATE_MASK GENMASK(27, 24) +#define CONTROL_CLKRATE_SHIFT 24 + +/* + * QSPI Frames register mask defines + */ +#define FRAMES_TOTALBYTES_MASK GENMASK(15, 0) +#define FRAMES_CMDBYTES_MASK GENMASK(24, 16) +#define FRAMES_CMDBYTES_SHIFT 16 +#define FRAMES_SHIFT 25 +#define FRAMES_IDLE_MASK GENMASK(29, 26) +#define FRAMES_IDLE_SHIFT 26 +#define FRAMES_FLAGBYTE BIT(30) +#define FRAMES_FLAGWORD BIT(31) + +/* + * QSPI Interrupt Enable register mask defines + */ +#define IEN_TXDONE BIT(0) +#define IEN_RXDONE BIT(1) +#define IEN_RXAVAILABLE BIT(2) +#define IEN_TXAVAILABLE BIT(3) +#define IEN_RXFIFOEMPTY BIT(4) +#define IEN_TXFIFOFULL BIT(5) + +/* + * QSPI Status register mask defines + */ +#define STATUS_TXDONE BIT(0) +#define STATUS_RXDONE BIT(1) +#define STATUS_RXAVAILABLE BIT(2) +#define STATUS_TXAVAILABLE BIT(3) +#define STATUS_RXFIFOEMPTY BIT(4) +#define STATUS_TXFIFOFULL BIT(5) +#define STATUS_READY BIT(7) +#define STATUS_FLAGSX4 BIT(8) +#define STATUS_MASK GENMASK(8, 0) + +#define BYTESUPPER_MASK GENMASK(31, 16) +#define BYTESLOWER_MASK GENMASK(15, 0) + +#define MAX_DIVIDER 16 +#define MIN_DIVIDER 0 +#define MAX_DATA_CMD_LEN 256 + +/* QSPI ready time out value */ +#define TIMEOUT_MS 500 + +/* + * QSPI Register offsets. + */ +#define REG_CONTROL (0x00) +#define REG_FRAMES (0x04) +#define REG_IEN (0x0c) +#define REG_STATUS (0x10) +#define REG_DIRECT_ACCESS (0x14) +#define REG_UPPER_ACCESS (0x18) +#define REG_RX_DATA (0x40) +#define REG_TX_DATA (0x44) +#define REG_X4_RX_DATA (0x48) +#define REG_X4_TX_DATA (0x4c) +#define REG_FRAMESUP (0x50) + +/** + * struct mchp_coreqspi - Defines qspi driver instance + * @regs: Virtual address of the QSPI controller registers + * @clk: QSPI Operating clock + * @data_completion: completion structure + * @op_lock: lock access to the device + * @txbuf: TX buffer + * @rxbuf: RX buffer + * @irq: IRQ number + * @tx_len: Number of bytes left to transfer + * @rx_len: Number of bytes left to receive + */ +struct mchp_coreqspi { + void __iomem *regs; + struct clk *clk; + struct completion data_completion; + struct mutex op_lock; /* lock access to the device */ + u8 *txbuf; + u8 *rxbuf; + int irq; + int tx_len; + int rx_len; +}; + +static int mchp_coreqspi_set_mode(struct mchp_coreqspi *qspi, const struct spi_mem_op *op) +{ + u32 control = readl_relaxed(qspi->regs + REG_CONTROL); + + /* + * The operating mode can be configured based on the command that needs to be send. + * bits[15:14]: Sets whether multiple bit SPI operates in normal, extended or full modes. + * 00: Normal (single DQ0 TX and single DQ1 RX lines) + * 01: Extended RO (command and address bytes on DQ0 only) + * 10: Extended RW (command byte on DQ0 only) + * 11: Full. (command and address are on all DQ lines) + * bit[13]: Sets whether multiple bit SPI uses 2 or 4 bits of data + * 0: 2-bits (BSPI) + * 1: 4-bits (QSPI) + */ + if (op->data.buswidth == 4 || op->data.buswidth == 2) { + control &= ~CONTROL_MODE12_MASK; + if (op->cmd.buswidth == 1 && (op->addr.buswidth == 1 || op->addr.buswidth == 0)) + control |= CONTROL_MODE12_EX_RO; + else if (op->cmd.buswidth == 1) + control |= CONTROL_MODE12_EX_RW; + else + control |= CONTROL_MODE12_FULL; + + control |= CONTROL_MODE0; + } else { + control &= ~(CONTROL_MODE12_MASK | + CONTROL_MODE0); + } + + writel_relaxed(control, qspi->regs + REG_CONTROL); + + return 0; +} + +static inline void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi) +{ + u32 control, data; + + if (!qspi->rx_len) + return; + + control = readl_relaxed(qspi->regs + REG_CONTROL); + + /* + * Read 4-bytes from the SPI FIFO in single transaction and then read + * the reamaining data byte wise. + */ + control |= CONTROL_FLAGSX4; + writel_relaxed(control, qspi->regs + REG_CONTROL); + + while (qspi->rx_len >= 4) { + while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_RXFIFOEMPTY) + ; + data = readl_relaxed(qspi->regs + REG_X4_RX_DATA); + *(u32 *)qspi->rxbuf = data; + qspi->rxbuf += 4; + qspi->rx_len -= 4; + } + + control &= ~CONTROL_FLAGSX4; + writel_relaxed(control, qspi->regs + REG_CONTROL); + + while (qspi->rx_len--) { + while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_RXFIFOEMPTY) + ; + data = readl_relaxed(qspi->regs + REG_RX_DATA); + *qspi->rxbuf++ = (data & 0xFF); + } +} + +static inline void mchp_coreqspi_write_op(struct mchp_coreqspi *qspi, bool word) +{ + u32 control, data; + + control = readl_relaxed(qspi->regs + REG_CONTROL); + control |= CONTROL_FLAGSX4; + writel_relaxed(control, qspi->regs + REG_CONTROL); + + while (qspi->tx_len >= 4) { + while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_TXFIFOFULL) + ; + data = *(u32 *)qspi->txbuf; + qspi->txbuf += 4; + qspi->tx_len -= 4; + writel_relaxed(data, qspi->regs + REG_X4_TX_DATA); + } + + control &= ~CONTROL_FLAGSX4; + writel_relaxed(control, qspi->regs + REG_CONTROL); + + while (qspi->tx_len--) { + while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_TXFIFOFULL) + ; + data = *qspi->txbuf++; + writel_relaxed(data, qspi->regs + REG_TX_DATA); + } +} + +static void mchp_coreqspi_enable_ints(struct mchp_coreqspi *qspi) +{ + u32 mask = IEN_TXDONE | + IEN_RXDONE | + IEN_RXAVAILABLE; + + writel_relaxed(mask, qspi->regs + REG_IEN); +} + +static void mchp_coreqspi_disable_ints(struct mchp_coreqspi *qspi) +{ + writel_relaxed(0, qspi->regs + REG_IEN); +} + +static irqreturn_t mchp_coreqspi_isr(int irq, void *dev_id) +{ + struct mchp_coreqspi *qspi = (struct mchp_coreqspi *)dev_id; + irqreturn_t ret = IRQ_NONE; + int intfield = readl_relaxed(qspi->regs + REG_STATUS) & STATUS_MASK; + + if (intfield == 0) + return ret; + + if (intfield & IEN_TXDONE) { + writel_relaxed(IEN_TXDONE, qspi->regs + REG_STATUS); + ret = IRQ_HANDLED; + } + + if (intfield & IEN_RXAVAILABLE) { + writel_relaxed(IEN_RXAVAILABLE, qspi->regs + REG_STATUS); + mchp_coreqspi_read_op(qspi); + ret = IRQ_HANDLED; + } + + if (intfield & IEN_RXDONE) { + writel_relaxed(IEN_RXDONE, qspi->regs + REG_STATUS); + complete(&qspi->data_completion); + ret = IRQ_HANDLED; + } + + return ret; +} + +static int mchp_coreqspi_setup_clock(struct mchp_coreqspi *qspi, struct spi_device *spi) +{ + unsigned long clk_hz; + u32 control, baud_rate_val = 0; + + clk_hz = clk_get_rate(qspi->clk); + if (!clk_hz) + return -EINVAL; + + baud_rate_val = DIV_ROUND_UP(clk_hz, 2 * spi->max_speed_hz); + if (baud_rate_val > MAX_DIVIDER || baud_rate_val < MIN_DIVIDER) { + dev_err(&spi->dev, + "could not configure the clock for spi clock %d Hz & system clock %ld Hz\n", + spi->max_speed_hz, clk_hz); + return -EINVAL; + } + + control = readl_relaxed(qspi->regs + REG_CONTROL); + control |= baud_rate_val << CONTROL_CLKRATE_SHIFT; + writel_relaxed(control, qspi->regs + REG_CONTROL); + control = readl_relaxed(qspi->regs + REG_CONTROL); + + if ((spi->mode & SPI_CPOL) && (spi->mode & SPI_CPHA)) + control |= CONTROL_CLKIDLE; + else + control &= ~CONTROL_CLKIDLE; + + writel_relaxed(control, qspi->regs + REG_CONTROL); + + return 0; +} + +static int mchp_coreqspi_setup_op(struct spi_device *spi_dev) +{ + struct spi_controller *ctlr = spi_dev->master; + struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr); + u32 control = readl_relaxed(qspi->regs + REG_CONTROL); + + control |= (CONTROL_MASTER | CONTROL_ENABLE); + control &= ~CONTROL_CLKIDLE; + writel_relaxed(control, qspi->regs + REG_CONTROL); + + return 0; +} + +static inline void mchp_coreqspi_config_op(struct mchp_coreqspi *qspi, const struct spi_mem_op *op) +{ + u32 idle_cycles = 0; + int total_bytes, cmd_bytes, frames, ctrl; + + cmd_bytes = op->cmd.nbytes + op->addr.nbytes; + total_bytes = cmd_bytes + op->data.nbytes; + + /* + * As per the coreQSPI IP spec,the number of command and data bytes are + * controlled by the frames register for each SPI sequence. This supports + * the SPI flash memory read and writes sequences as below. so configure + * the cmd and total bytes accordingly. + * --------------------------------------------------------------------- + * TOTAL BYTES | CMD BYTES | What happens | + * ______________________________________________________________________ + * | | | + * 1 | 1 | The SPI core will transmit a single byte | + * | | and receive data is discarded | + * | | | + * 1 | 0 | The SPI core will transmit a single byte | + * | | and return a single byte | + * | | | + * 10 | 4 | The SPI core will transmit 4 command | + * | | bytes discarding the receive data and | + * | | transmits 6 dummy bytes returning the 6 | + * | | received bytes and return a single byte | + * | | | + * 10 | 10 | The SPI core will transmit 10 command | + * | | | + * 10 | 0 | The SPI core will transmit 10 command | + * | | bytes and returning 10 received bytes | + * ______________________________________________________________________ + */ + if (!(op->data.dir == SPI_MEM_DATA_IN)) + cmd_bytes = total_bytes; + + frames = total_bytes & BYTESUPPER_MASK; + writel_relaxed(frames, qspi->regs + REG_FRAMESUP); + frames = total_bytes & BYTESLOWER_MASK; + frames |= cmd_bytes << FRAMES_CMDBYTES_SHIFT; + + if (op->dummy.buswidth) + idle_cycles = op->dummy.nbytes * 8 / op->dummy.buswidth; + + frames |= idle_cycles << FRAMES_IDLE_SHIFT; + ctrl = readl_relaxed(qspi->regs + REG_CONTROL); + + if (ctrl & CONTROL_MODE12_MASK) + frames |= (1 << FRAMES_SHIFT); + + frames |= FRAMES_FLAGWORD; + writel_relaxed(frames, qspi->regs + REG_FRAMES); +} + +static int mchp_qspi_wait_for_ready(struct spi_mem *mem) +{ + struct mchp_coreqspi *qspi = spi_controller_get_devdata + (mem->spi->master); + u32 status; + int ret; + + ret = readl_poll_timeout(qspi->regs + REG_STATUS, status, + (status & STATUS_READY), 0, + TIMEOUT_MS); + if (ret) { + dev_err(&mem->spi->dev, + "Timeout waiting on QSPI ready.\n"); + return -ETIMEDOUT; + } + + return ret; +} + +static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct mchp_coreqspi *qspi = spi_controller_get_devdata + (mem->spi->master); + u32 address = op->addr.val; + u8 opcode = op->cmd.opcode; + u8 opaddr[5]; + int err, i; + + mutex_lock(&qspi->op_lock); + err = mchp_qspi_wait_for_ready(mem); + if (err) + goto error; + + err = mchp_coreqspi_setup_clock(qspi, mem->spi); + if (err) + goto error; + + err = mchp_coreqspi_set_mode(qspi, op); + if (err) + goto error; + + reinit_completion(&qspi->data_completion); + mchp_coreqspi_config_op(qspi, op); + if (op->cmd.opcode) { + qspi->txbuf = &opcode; + qspi->rxbuf = NULL; + qspi->tx_len = op->cmd.nbytes; + qspi->rx_len = 0; + mchp_coreqspi_write_op(qspi, false); + } + + qspi->txbuf = &opaddr[0]; + if (op->addr.nbytes) { + for (i = 0; i < op->addr.nbytes; i++) + qspi->txbuf[i] = address >> (8 * (op->addr.nbytes - i - 1)); + + qspi->rxbuf = NULL; + qspi->tx_len = op->addr.nbytes; + qspi->rx_len = 0; + mchp_coreqspi_write_op(qspi, false); + } + + if (op->data.nbytes) { + if (op->data.dir == SPI_MEM_DATA_OUT) { + qspi->txbuf = (u8 *)op->data.buf.out; + qspi->rxbuf = NULL; + qspi->rx_len = 0; + qspi->tx_len = op->data.nbytes; + mchp_coreqspi_write_op(qspi, true); + } else { + qspi->txbuf = NULL; + qspi->rxbuf = (u8 *)op->data.buf.in; + qspi->rx_len = op->data.nbytes; + qspi->tx_len = 0; + } + } + + mchp_coreqspi_enable_ints(qspi); + + if (!wait_for_completion_timeout(&qspi->data_completion, msecs_to_jiffies(1000))) + err = -ETIMEDOUT; + +error: + mutex_unlock(&qspi->op_lock); + mchp_coreqspi_disable_ints(qspi); + + return err; +} + +static bool mchp_coreqspi_supports_op(struct spi_mem *mem, const struct spi_mem_op *op) +{ + if (!spi_mem_default_supports_op(mem, op)) + return false; + + if ((op->data.buswidth == 4 || op->data.buswidth == 2) && + (op->cmd.buswidth == 1 && (op->addr.buswidth == 1 || op->addr.buswidth == 0))) { + /* + * If the command and address are on DQ0 only, then this + * controller doesn't support sending data on dual and + * quad lines. but it supports reading data on dual and + * quad lines with same configuration as command and + * address on DQ0. + * i.e. The control register[15:13] :EX_RO(read only) is + * meant only for the command and address are on DQ0 but + * not to write data, it is just to read. + * Ex: 0x34h is Quad Load Program Data which is not + * supported. Then the spi-mem layer will iterate over + * each command and it will chose the supported one. + */ + if (op->data.dir == SPI_MEM_DATA_OUT) + return false; + } + + return true; +} + +static int mchp_coreqspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) +{ + if (op->data.dir == SPI_MEM_DATA_OUT || op->data.dir == SPI_MEM_DATA_IN) { + if (op->data.nbytes > MAX_DATA_CMD_LEN) + op->data.nbytes = MAX_DATA_CMD_LEN; + } + + return 0; +} + +static const struct spi_controller_mem_ops mchp_coreqspi_mem_ops = { + .adjust_op_size = mchp_coreqspi_adjust_op_size, + .supports_op = mchp_coreqspi_supports_op, + .exec_op = mchp_coreqspi_exec_op, +}; + +static int mchp_coreqspi_probe(struct platform_device *pdev) +{ + struct spi_controller *ctlr; + struct mchp_coreqspi *qspi; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + int ret; + + ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*qspi)); + if (!ctlr) + return dev_err_probe(&pdev->dev, -ENOMEM, + "unable to allocate master for QSPI controller\n"); + + qspi = spi_controller_get_devdata(ctlr); + platform_set_drvdata(pdev, qspi); + + qspi->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(qspi->regs)) + return dev_err_probe(&pdev->dev, PTR_ERR(qspi->regs), + "failed to map registers\n"); + + qspi->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(qspi->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(qspi->clk), + "could not get clock\n"); + + ret = clk_prepare_enable(qspi->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to enable clock\n"); + + init_completion(&qspi->data_completion); + mutex_init(&qspi->op_lock); + + qspi->irq = platform_get_irq(pdev, 0); + if (qspi->irq < 0) { + ret = qspi->irq; + goto out; + } + + ret = devm_request_irq(&pdev->dev, qspi->irq, mchp_coreqspi_isr, + IRQF_SHARED, pdev->name, qspi); + if (ret) { + dev_err(&pdev->dev, "request_irq failed %d\n", ret); + goto out; + } + + ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + ctlr->mem_ops = &mchp_coreqspi_mem_ops; + ctlr->setup = mchp_coreqspi_setup_op; + ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD | + SPI_TX_DUAL | SPI_TX_QUAD; + ctlr->dev.of_node = np; + + ret = devm_spi_register_controller(&pdev->dev, ctlr); + if (ret) { + dev_err_probe(&pdev->dev, ret, + "spi_register_controller failed\n"); + goto out; + } + + return 0; + +out: + clk_disable_unprepare(qspi->clk); + + return ret; +} + +static int mchp_coreqspi_remove(struct platform_device *pdev) +{ + struct mchp_coreqspi *qspi = platform_get_drvdata(pdev); + u32 control = readl_relaxed(qspi->regs + REG_CONTROL); + + mchp_coreqspi_disable_ints(qspi); + control &= ~CONTROL_ENABLE; + writel_relaxed(control, qspi->regs + REG_CONTROL); + clk_disable_unprepare(qspi->clk); + + return 0; +} + +static const struct of_device_id mchp_coreqspi_of_match[] = { + { .compatible = "microchip,coreqspi-rtl-v2" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, mchp_coreqspi_of_match); + +static struct platform_driver mchp_coreqspi_driver = { + .probe = mchp_coreqspi_probe, + .driver = { + .name = "microchip,coreqspi", + .of_match_table = mchp_coreqspi_of_match, + }, + .remove = mchp_coreqspi_remove, +}; +module_platform_driver(mchp_coreqspi_driver); + +MODULE_AUTHOR("Naga Sureshkumar Relli dev), master); if (ret) return dev_err_probe(&pdev->dev, ret, - "could not request irq: %d\n", ret); + "could not request irq\n"); spi->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(spi->clk)) return dev_err_probe(&pdev->dev, PTR_ERR(spi->clk), - "could not get clk: %d\n", ret); + "could not get clk\n"); ret = clk_prepare_enable(spi->clk); if (ret) diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c index bc5e36fd42887edb0459b9d8772e0a8d3f8a1159..cb075c1acbee30d5f931f6490c2bd8a8e44287e6 100644 --- a/drivers/spi/spi-mpc52xx.c +++ b/drivers/spi/spi-mpc52xx.c @@ -11,13 +11,14 @@ */ #include +#include #include #include #include #include +#include #include #include -#include #include #include #include @@ -89,7 +90,7 @@ struct mpc52xx_spi { const u8 *tx_buf; int cs_change; int gpio_cs_count; - unsigned int *gpio_cs; + struct gpio_desc **gpio_cs; }; /* @@ -101,9 +102,10 @@ static void mpc52xx_spi_chipsel(struct mpc52xx_spi *ms, int value) if (ms->gpio_cs_count > 0) { cs = ms->message->spi->chip_select; - gpio_set_value(ms->gpio_cs[cs], value ? 0 : 1); - } else + gpiod_set_value(ms->gpio_cs[cs], value); + } else { out_8(ms->regs + SPI_PORTDATA, value ? 0 : 0x08); + } } /* @@ -385,10 +387,10 @@ static int mpc52xx_spi_probe(struct platform_device *op) { struct spi_master *master; struct mpc52xx_spi *ms; + struct gpio_desc *gpio_cs; void __iomem *regs; u8 ctrl1; int rc, i = 0; - int gpio_cs; /* MMIO registers */ dev_dbg(&op->dev, "probing mpc5200 SPI device\n"); @@ -438,7 +440,7 @@ static int mpc52xx_spi_probe(struct platform_device *op) ms->irq1 = irq_of_parse_and_map(op->dev.of_node, 1); ms->state = mpc52xx_spi_fsmstate_idle; ms->ipb_freq = mpc5xxx_get_bus_frequency(&op->dev); - ms->gpio_cs_count = of_gpio_count(op->dev.of_node); + ms->gpio_cs_count = gpiod_count(&op->dev, NULL); if (ms->gpio_cs_count > 0) { master->num_chipselect = ms->gpio_cs_count; ms->gpio_cs = kmalloc_array(ms->gpio_cs_count, @@ -450,23 +452,16 @@ static int mpc52xx_spi_probe(struct platform_device *op) } for (i = 0; i < ms->gpio_cs_count; i++) { - gpio_cs = of_get_gpio(op->dev.of_node, i); - if (!gpio_is_valid(gpio_cs)) { - dev_err(&op->dev, - "could not parse the gpio field in oftree\n"); - rc = -ENODEV; - goto err_gpio; - } - - rc = gpio_request(gpio_cs, dev_name(&op->dev)); + gpio_cs = gpiod_get_index(&op->dev, + NULL, i, GPIOD_OUT_LOW); + rc = PTR_ERR_OR_ZERO(gpio_cs); if (rc) { dev_err(&op->dev, - "can't request spi cs gpio #%d on gpio line %d\n", - i, gpio_cs); + "failed to get spi cs gpio #%d: %d\n", + i, rc); goto err_gpio; } - gpio_direction_output(gpio_cs, 1); ms->gpio_cs[i] = gpio_cs; } } @@ -507,7 +502,7 @@ static int mpc52xx_spi_probe(struct platform_device *op) dev_err(&ms->master->dev, "initialization failed\n"); err_gpio: while (i-- > 0) - gpio_free(ms->gpio_cs[i]); + gpiod_put(ms->gpio_cs[i]); kfree(ms->gpio_cs); err_alloc_gpio: @@ -528,7 +523,7 @@ static int mpc52xx_spi_remove(struct platform_device *op) free_irq(ms->irq1, ms); for (i = 0; i < ms->gpio_cs_count; i++) - gpio_free(ms->gpio_cs[i]); + gpiod_put(ms->gpio_cs[i]); kfree(ms->gpio_cs); spi_unregister_master(master); diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 0a3b9f7eed30f1edf0dfc8868b453698908061c3..11aeae7fe7fc9fd59e7e40e0f1e3e162a1b54914 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -1184,6 +1184,11 @@ static int mtk_spi_probe(struct platform_device *pdev) if (!dev->dma_mask) dev->dma_mask = &dev->coherent_dma_mask; + if (mdata->dev_comp->ipm_design) + dma_set_max_seg_size(dev, SZ_16M); + else + dma_set_max_seg_size(dev, SZ_256K); + ret = devm_request_irq(dev, irq, mtk_spi_interrupt, IRQF_TRIGGER_NONE, dev_name(dev), master); if (ret) diff --git a/drivers/spi/spi-mt7621.c b/drivers/spi/spi-mt7621.c index b4b9b7309b5e949faf09aa129368a4c03da7aac2..c4cc8e2f85e2f85ce36674410b89500f7fd4accc 100644 --- a/drivers/spi/spi-mt7621.c +++ b/drivers/spi/spi-mt7621.c @@ -55,7 +55,6 @@ struct mt7621_spi { void __iomem *base; unsigned int sys_freq; unsigned int speed; - struct clk *clk; int pending_write; }; @@ -327,7 +326,6 @@ static int mt7621_spi_probe(struct platform_device *pdev) struct spi_controller *master; struct mt7621_spi *rs; void __iomem *base; - int status = 0; struct clk *clk; int ret; @@ -339,21 +337,14 @@ static int mt7621_spi_probe(struct platform_device *pdev) if (IS_ERR(base)) return PTR_ERR(base); - clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(clk)) { - dev_err(&pdev->dev, "unable to get SYS clock, err=%d\n", - status); - return PTR_ERR(clk); - } - - status = clk_prepare_enable(clk); - if (status) - return status; + clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk), + "unable to get SYS clock\n"); master = devm_spi_alloc_master(&pdev->dev, sizeof(*rs)); if (!master) { dev_info(&pdev->dev, "master allocation failed\n"); - clk_disable_unprepare(clk); return -ENOMEM; } @@ -369,38 +360,18 @@ static int mt7621_spi_probe(struct platform_device *pdev) rs = spi_controller_get_devdata(master); rs->base = base; - rs->clk = clk; rs->master = master; - rs->sys_freq = clk_get_rate(rs->clk); + rs->sys_freq = clk_get_rate(clk); rs->pending_write = 0; dev_info(&pdev->dev, "sys_freq: %u\n", rs->sys_freq); ret = device_reset(&pdev->dev); if (ret) { dev_err(&pdev->dev, "SPI reset failed!\n"); - clk_disable_unprepare(clk); return ret; } - ret = spi_register_controller(master); - if (ret) - clk_disable_unprepare(clk); - - return ret; -} - -static int mt7621_spi_remove(struct platform_device *pdev) -{ - struct spi_controller *master; - struct mt7621_spi *rs; - - master = dev_get_drvdata(&pdev->dev); - rs = spi_controller_get_devdata(master); - - spi_unregister_controller(master); - clk_disable_unprepare(rs->clk); - - return 0; + return devm_spi_register_controller(&pdev->dev, master); } MODULE_ALIAS("platform:" DRIVER_NAME); @@ -411,7 +382,6 @@ static struct platform_driver mt7621_spi_driver = { .of_match_table = mt7621_spi_match, }, .probe = mt7621_spi_probe, - .remove = mt7621_spi_remove, }; module_platform_driver(mt7621_spi_driver); diff --git a/drivers/spi/spi-mux.c b/drivers/spi/spi-mux.c index f5d32ec4634e341d51b20446e041886b07dbb871..0709e987bd5ab28ce521e13e550cca6e1dd293ec 100644 --- a/drivers/spi/spi-mux.c +++ b/drivers/spi/spi-mux.c @@ -161,6 +161,7 @@ static int spi_mux_probe(struct spi_device *spi) ctlr->num_chipselect = mux_control_states(priv->mux); ctlr->bus_num = -1; ctlr->dev.of_node = spi->dev.of_node; + ctlr->must_async = true; ret = devm_spi_register_controller(&spi->dev, ctlr); if (ret) diff --git a/drivers/spi/spi-npcm-pspi.c b/drivers/spi/spi-npcm-pspi.c index 1668a347e003ddc5e7ef39da0b987123fc8889aa..7f2e4d1b0d43371f8cf31f96ee0505f495ac578c 100644 --- a/drivers/spi/spi-npcm-pspi.c +++ b/drivers/spi/spi-npcm-pspi.c @@ -443,6 +443,7 @@ static int npcm_pspi_remove(struct platform_device *pdev) static const struct of_device_id npcm_pspi_match[] = { { .compatible = "nuvoton,npcm750-pspi", .data = NULL }, + { .compatible = "nuvoton,npcm845-pspi", .data = NULL }, {} }; MODULE_DEVICE_TABLE(of, npcm_pspi_match); diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index 2b0301fc971c6cb98d4140f2cca1d44955a695e8..d6a65a989ef80533f53c2ef3c988ccf635803cd8 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -588,7 +588,7 @@ static int nxp_fspi_clk_prep_enable(struct nxp_fspi *f) { int ret; - if (is_acpi_node(f->dev->fwnode)) + if (is_acpi_node(dev_fwnode(f->dev))) return 0; ret = clk_prepare_enable(f->clk_en); @@ -606,7 +606,7 @@ static int nxp_fspi_clk_prep_enable(struct nxp_fspi *f) static int nxp_fspi_clk_disable_unprep(struct nxp_fspi *f) { - if (is_acpi_node(f->dev->fwnode)) + if (is_acpi_node(dev_fwnode(f->dev))) return 0; clk_disable_unprepare(f->clk); @@ -1100,7 +1100,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, f); /* find the resources - configuration register address space */ - if (is_acpi_node(f->dev->fwnode)) + if (is_acpi_node(dev_fwnode(f->dev))) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); else res = platform_get_resource_byname(pdev, @@ -1113,7 +1113,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) } /* find the resources - controller memory mapped space */ - if (is_acpi_node(f->dev->fwnode)) + if (is_acpi_node(dev_fwnode(f->dev))) res = platform_get_resource(pdev, IORESOURCE_MEM, 1); else res = platform_get_resource_byname(pdev, diff --git a/drivers/spi/spi-omap-100k.c b/drivers/spi/spi-omap-100k.c index 20b0471729651d2398b526d790f5f1ce46bea4fd..061f7394e5b9b37b701d941e0f530a0b38170037 100644 --- a/drivers/spi/spi-omap-100k.c +++ b/drivers/spi/spi-omap-100k.c @@ -412,6 +412,7 @@ static int omap1_spi100k_probe(struct platform_device *pdev) return status; err_fck: + pm_runtime_disable(&pdev->dev); clk_disable_unprepare(spi100k->fck); err_ick: clk_disable_unprepare(spi100k->ick); diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c index c42e59df38fedf3af18bf866382a8d4bb1499276..6ba9b0d7710b0e66a3736adc168a83ec63da68a0 100644 --- a/drivers/spi/spi-omap2-mcspi.c +++ b/drivers/spi/spi-omap2-mcspi.c @@ -1509,10 +1509,8 @@ static int omap2_mcspi_probe(struct platform_device *pdev) } status = platform_get_irq(pdev, 0); - if (status == -EPROBE_DEFER) - goto free_master; if (status < 0) { - dev_err(&pdev->dev, "no irq resource found\n"); + dev_err_probe(&pdev->dev, status, "no irq resource found\n"); goto free_master; } init_completion(&mcspi->txdone); diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c index 838d12e65144af77c431cf65a3da68bea2ee5c2d..2bf21c2e7a52a1b3019d5b91964fb6a6d3ca5b41 100644 --- a/drivers/spi/spi-pxa2xx.c +++ b/drivers/spi/spi-pxa2xx.c @@ -1441,31 +1441,6 @@ static const struct of_device_id pxa2xx_spi_of_match[] = { }; MODULE_DEVICE_TABLE(of, pxa2xx_spi_of_match); -#ifdef CONFIG_ACPI - -static int pxa2xx_spi_get_port_id(struct device *dev) -{ - struct acpi_device *adev; - unsigned int devid; - int port_id = -1; - - adev = ACPI_COMPANION(dev); - if (adev && adev->pnp.unique_id && - !kstrtouint(adev->pnp.unique_id, 0, &devid)) - port_id = devid; - return port_id; -} - -#else /* !CONFIG_ACPI */ - -static int pxa2xx_spi_get_port_id(struct device *dev) -{ - return -1; -} - -#endif /* CONFIG_ACPI */ - - #ifdef CONFIG_PCI static bool pxa2xx_spi_idma_filter(struct dma_chan *chan, void *param) @@ -1479,13 +1454,16 @@ static struct pxa2xx_spi_controller * pxa2xx_spi_init_pdata(struct platform_device *pdev) { struct pxa2xx_spi_controller *pdata; + struct device *dev = &pdev->dev; + struct device *parent = dev->parent; struct ssp_device *ssp; struct resource *res; - struct device *parent = pdev->dev.parent; struct pci_dev *pcidev = dev_is_pci(parent) ? to_pci_dev(parent) : NULL; const struct pci_device_id *pcidev_id = NULL; enum pxa_ssp_type type; const void *match; + int status; + u64 uid; if (pcidev) pcidev_id = pci_match_id(pxa2xx_spi_pci_compound_match, pcidev); @@ -1529,7 +1507,12 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev) ssp->type = type; ssp->dev = &pdev->dev; - ssp->port_id = pxa2xx_spi_get_port_id(&pdev->dev); + + status = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &uid); + if (status) + ssp->port_id = -1; + else + ssp->port_id = uid; pdata->is_slave = device_property_read_bool(&pdev->dev, "spi-slave"); pdata->num_chipselect = 1; @@ -1873,10 +1856,8 @@ static int pxa2xx_spi_runtime_suspend(struct device *dev) static int pxa2xx_spi_runtime_resume(struct device *dev) { struct driver_data *drv_data = dev_get_drvdata(dev); - int status; - status = clk_prepare_enable(drv_data->ssp->clk); - return status; + return clk_prepare_enable(drv_data->ssp->clk); } #endif diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c index 00d6084306b4a366d89ff7643daa586504371419..7d89510dc3f00d26ee7210edb2ed1e28758c24c9 100644 --- a/drivers/spi/spi-qup.c +++ b/drivers/spi/spi-qup.c @@ -1198,8 +1198,10 @@ static int spi_qup_pm_resume_runtime(struct device *device) return ret; ret = clk_prepare_enable(controller->cclk); - if (ret) + if (ret) { + clk_disable_unprepare(controller->iclk); return ret; + } /* Disable clocks auto gaiting */ config = readl_relaxed(controller->base + QUP_CONFIG); @@ -1245,14 +1247,25 @@ static int spi_qup_resume(struct device *device) return ret; ret = clk_prepare_enable(controller->cclk); - if (ret) + if (ret) { + clk_disable_unprepare(controller->iclk); return ret; + } ret = spi_qup_set_state(controller, QUP_STATE_RESET); if (ret) - return ret; + goto disable_clk; - return spi_master_resume(master); + ret = spi_master_resume(master); + if (ret) + goto disable_clk; + + return 0; + +disable_clk: + clk_disable_unprepare(controller->cclk); + clk_disable_unprepare(controller->iclk); + return ret; } #endif /* CONFIG_PM_SLEEP */ diff --git a/drivers/spi/spi-s3c24xx.c b/drivers/spi/spi-s3c24xx.c index 660aa866af06fe099199e99651e91d0b80e516b0..ef25b5e939008cfc25e47cb640d03fbfd92b71d1 100644 --- a/drivers/spi/spi-s3c24xx.c +++ b/drivers/spi/spi-s3c24xx.c @@ -449,7 +449,7 @@ static int s3c24xx_spi_probe(struct platform_device *pdev) struct spi_master *master; int err = 0; - master = spi_alloc_master(&pdev->dev, sizeof(struct s3c24xx_spi)); + master = devm_spi_alloc_master(&pdev->dev, sizeof(struct s3c24xx_spi)); if (master == NULL) { dev_err(&pdev->dev, "No memory for spi_master\n"); return -ENOMEM; @@ -463,8 +463,7 @@ static int s3c24xx_spi_probe(struct platform_device *pdev) if (pdata == NULL) { dev_err(&pdev->dev, "No platform data supplied\n"); - err = -ENOENT; - goto err_no_pdata; + return -ENOENT; } platform_set_drvdata(pdev, hw); @@ -499,29 +498,24 @@ static int s3c24xx_spi_probe(struct platform_device *pdev) /* find and map our resources */ hw->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(hw->regs)) { - err = PTR_ERR(hw->regs); - goto err_no_pdata; - } + if (IS_ERR(hw->regs)) + return PTR_ERR(hw->regs); hw->irq = platform_get_irq(pdev, 0); - if (hw->irq < 0) { - err = -ENOENT; - goto err_no_pdata; - } + if (hw->irq < 0) + return -ENOENT; err = devm_request_irq(&pdev->dev, hw->irq, s3c24xx_spi_irq, 0, pdev->name, hw); if (err) { dev_err(&pdev->dev, "Cannot claim IRQ\n"); - goto err_no_pdata; + return err; } hw->clk = devm_clk_get(&pdev->dev, "spi"); if (IS_ERR(hw->clk)) { dev_err(&pdev->dev, "No clock for device\n"); - err = PTR_ERR(hw->clk); - goto err_no_pdata; + return PTR_ERR(hw->clk); } s3c24xx_spi_initialsetup(hw); @@ -539,8 +533,6 @@ static int s3c24xx_spi_probe(struct platform_device *pdev) err_register: clk_disable(hw->clk); - err_no_pdata: - spi_master_put(hw->master); return err; } diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index 7f346866614abbd48ba430543da349e7ab2bb7ac..71d324ec9a70af0efd166efc750787555defce40 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -84,6 +84,7 @@ #define S3C64XX_SPI_ST_TX_FIFORDY (1<<0) #define S3C64XX_SPI_PACKET_CNT_EN (1<<16) +#define S3C64XX_SPI_PACKET_CNT_MASK GENMASK(15, 0) #define S3C64XX_SPI_PND_TX_UNDERRUN_CLR (1<<4) #define S3C64XX_SPI_PND_TX_OVERRUN_CLR (1<<3) @@ -389,8 +390,8 @@ static int s3c64xx_spi_unprepare_transfer(struct spi_master *spi) if (sdd->rx_dma.ch && sdd->tx_dma.ch) { dma_release_channel(sdd->rx_dma.ch); dma_release_channel(sdd->tx_dma.ch); - sdd->rx_dma.ch = 0; - sdd->tx_dma.ch = 0; + sdd->rx_dma.ch = NULL; + sdd->tx_dma.ch = NULL; } return 0; @@ -711,6 +712,13 @@ static int s3c64xx_spi_prepare_message(struct spi_master *master, return 0; } +static size_t s3c64xx_spi_max_transfer_size(struct spi_device *spi) +{ + struct spi_controller *ctlr = spi->controller; + + return ctlr->can_dma ? S3C64XX_SPI_PACKET_CNT_MASK : SIZE_MAX; +} + static int s3c64xx_spi_transfer_one(struct spi_master *master, struct spi_device *spi, struct spi_transfer *xfer) @@ -1152,6 +1160,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) master->unprepare_transfer_hardware = s3c64xx_spi_unprepare_transfer; master->prepare_message = s3c64xx_spi_prepare_message; master->transfer_one = s3c64xx_spi_transfer_one; + master->max_transfer_size = s3c64xx_spi_max_transfer_size; master->num_chipselect = sci->num_cs; master->use_gpio_descriptors = true; master->dma_alignment = 8; diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index d0012b30410c2cf9218c69f9f3a391a70b17a4f4..9bca3d076f053d21dcf59b61f9ae244d45ef58f0 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -1085,6 +1085,7 @@ static const struct of_device_id sh_msiof_match[] = { { .compatible = "renesas,rcar-gen2-msiof", .data = &rcar_gen2_data }, { .compatible = "renesas,msiof-r8a7796", .data = &rcar_gen3_data }, { .compatible = "renesas,rcar-gen3-msiof", .data = &rcar_gen3_data }, + { .compatible = "renesas,rcar-gen4-msiof", .data = &rcar_gen3_data }, { .compatible = "renesas,sh-msiof", .data = &sh_data }, /* Deprecated */ {}, }; diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c index f3fe92300639f35a1cb220d945e5eec1e0ec7f14..9131660c1afb25e4283e25b7c03e44711dd74565 100644 --- a/drivers/spi/spi-stm32-qspi.c +++ b/drivers/spi/spi-stm32-qspi.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -355,10 +356,10 @@ static int stm32_qspi_get_mode(u8 buswidth) return buswidth; } -static int stm32_qspi_send(struct spi_mem *mem, const struct spi_mem_op *op) +static int stm32_qspi_send(struct spi_device *spi, const struct spi_mem_op *op) { - struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master); - struct stm32_qspi_flash *flash = &qspi->flash[mem->spi->chip_select]; + struct stm32_qspi *qspi = spi_controller_get_devdata(spi->master); + struct stm32_qspi_flash *flash = &qspi->flash[spi->chip_select]; u32 ccr, cr; int timeout, err = 0, err_poll_status = 0; @@ -465,7 +466,7 @@ static int stm32_qspi_poll_status(struct spi_mem *mem, const struct spi_mem_op * qspi->fmode = CCR_FMODE_APM; qspi->status_timeout = timeout_ms; - ret = stm32_qspi_send(mem, op); + ret = stm32_qspi_send(mem->spi, op); mutex_unlock(&qspi->lock); pm_runtime_mark_last_busy(qspi->dev); @@ -489,7 +490,7 @@ static int stm32_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) else qspi->fmode = CCR_FMODE_INDW; - ret = stm32_qspi_send(mem, op); + ret = stm32_qspi_send(mem->spi, op); mutex_unlock(&qspi->lock); pm_runtime_mark_last_busy(qspi->dev); @@ -545,7 +546,7 @@ static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc, else qspi->fmode = CCR_FMODE_INDR; - ret = stm32_qspi_send(desc->mem, &op); + ret = stm32_qspi_send(desc->mem->spi, &op); mutex_unlock(&qspi->lock); pm_runtime_mark_last_busy(qspi->dev); @@ -554,12 +555,96 @@ static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc, return ret ?: len; } +static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl, + struct spi_message *msg) +{ + struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl); + struct spi_transfer *transfer; + struct spi_device *spi = msg->spi; + struct spi_mem_op op; + int ret = 0; + + if (!spi->cs_gpiod) + return -EOPNOTSUPP; + + ret = pm_runtime_resume_and_get(qspi->dev); + if (ret < 0) + return ret; + + mutex_lock(&qspi->lock); + + gpiod_set_value_cansleep(spi->cs_gpiod, true); + + list_for_each_entry(transfer, &msg->transfers, transfer_list) { + u8 dummy_bytes = 0; + + memset(&op, 0, sizeof(op)); + + dev_dbg(qspi->dev, "tx_buf:%p tx_nbits:%d rx_buf:%p rx_nbits:%d len:%d dummy_data:%d\n", + transfer->tx_buf, transfer->tx_nbits, + transfer->rx_buf, transfer->rx_nbits, + transfer->len, transfer->dummy_data); + + /* + * QSPI hardware supports dummy bytes transfer. + * If current transfer is dummy byte, merge it with the next + * transfer in order to take into account QSPI block constraint + */ + if (transfer->dummy_data) { + op.dummy.buswidth = transfer->tx_nbits; + op.dummy.nbytes = transfer->len; + dummy_bytes = transfer->len; + + /* if happens, means that message is not correctly built */ + if (list_is_last(&transfer->transfer_list, &msg->transfers)) { + ret = -EINVAL; + goto end_of_transfer; + } + + transfer = list_next_entry(transfer, transfer_list); + } + + op.data.nbytes = transfer->len; + + if (transfer->rx_buf) { + qspi->fmode = CCR_FMODE_INDR; + op.data.buswidth = transfer->rx_nbits; + op.data.dir = SPI_MEM_DATA_IN; + op.data.buf.in = transfer->rx_buf; + } else { + qspi->fmode = CCR_FMODE_INDW; + op.data.buswidth = transfer->tx_nbits; + op.data.dir = SPI_MEM_DATA_OUT; + op.data.buf.out = transfer->tx_buf; + } + + ret = stm32_qspi_send(spi, &op); + if (ret) + goto end_of_transfer; + + msg->actual_length += transfer->len + dummy_bytes; + } + +end_of_transfer: + gpiod_set_value_cansleep(spi->cs_gpiod, false); + + mutex_unlock(&qspi->lock); + + msg->status = ret; + spi_finalize_current_message(ctrl); + + pm_runtime_mark_last_busy(qspi->dev); + pm_runtime_put_autosuspend(qspi->dev); + + return ret; +} + static int stm32_qspi_setup(struct spi_device *spi) { struct spi_controller *ctrl = spi->master; struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl); struct stm32_qspi_flash *flash; - u32 presc; + u32 presc, mode; int ret; if (ctrl->busy) @@ -568,6 +653,16 @@ static int stm32_qspi_setup(struct spi_device *spi) if (!spi->max_speed_hz) return -EINVAL; + mode = spi->mode & (SPI_TX_OCTAL | SPI_RX_OCTAL); + if ((mode == SPI_TX_OCTAL || mode == SPI_RX_OCTAL) || + ((mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) && + gpiod_count(qspi->dev, "cs") == -ENOENT)) { + dev_err(qspi->dev, "spi-rx-bus-width\\/spi-tx-bus-width\\/cs-gpios\n"); + dev_err(qspi->dev, "configuration not supported\n"); + + return -EINVAL; + } + ret = pm_runtime_resume_and_get(qspi->dev); if (ret < 0) return ret; @@ -580,6 +675,16 @@ static int stm32_qspi_setup(struct spi_device *spi) mutex_lock(&qspi->lock); qspi->cr_reg = CR_APMS | 3 << CR_FTHRES_SHIFT | CR_SSHIFT | CR_EN; + + /* + * Dual flash mode is only enable in case SPI_TX_OCTAL and SPI_TX_OCTAL + * are both set in spi->mode and "cs-gpios" properties is found in DT + */ + if (mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) { + qspi->cr_reg |= CR_DFM; + dev_dbg(qspi->dev, "Dual flash mode enable"); + } + writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR); /* set dcr fsize to max address */ @@ -741,11 +846,13 @@ static int stm32_qspi_probe(struct platform_device *pdev) mutex_init(&qspi->lock); - ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD - | SPI_TX_DUAL | SPI_TX_QUAD; + ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_OCTAL + | SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_OCTAL; ctrl->setup = stm32_qspi_setup; ctrl->bus_num = -1; ctrl->mem_ops = &stm32_qspi_mem_ops; + ctrl->use_gpio_descriptors = true; + ctrl->transfer_one_message = stm32_qspi_transfer_one_message; ctrl->num_chipselect = STM32_QSPI_MAX_NORCHIP; ctrl->dev.of_node = dev->of_node; diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 523edfdf5dcd181fd91fba0351f528c18691a7bd..7377d3b81302282b28b9e6494162c1302583c5f5 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -421,7 +421,7 @@ static int xilinx_spi_probe(struct platform_device *pdev) return -EINVAL; } - master = spi_alloc_master(&pdev->dev, sizeof(struct xilinx_spi)); + master = devm_spi_alloc_master(&pdev->dev, sizeof(struct xilinx_spi)); if (!master) return -ENODEV; @@ -439,10 +439,8 @@ static int xilinx_spi_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); xspi->regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(xspi->regs)) { - ret = PTR_ERR(xspi->regs); - goto put_master; - } + if (IS_ERR(xspi->regs)) + return PTR_ERR(xspi->regs); master->bus_num = pdev->id; master->num_chipselect = num_cs; @@ -472,14 +470,13 @@ static int xilinx_spi_probe(struct platform_device *pdev) xspi->irq = platform_get_irq(pdev, 0); if (xspi->irq < 0 && xspi->irq != -ENXIO) { - ret = xspi->irq; - goto put_master; + return xspi->irq; } else if (xspi->irq >= 0) { /* Register for SPI Interrupt */ ret = devm_request_irq(&pdev->dev, xspi->irq, xilinx_spi_irq, 0, dev_name(&pdev->dev), xspi); if (ret) - goto put_master; + return ret; } /* SPI controller initializations */ @@ -488,7 +485,7 @@ static int xilinx_spi_probe(struct platform_device *pdev) ret = spi_bitbang_start(&xspi->bitbang); if (ret) { dev_err(&pdev->dev, "spi_bitbang_start FAILED\n"); - goto put_master; + return ret; } dev_info(&pdev->dev, "at %pR, irq=%d\n", res, xspi->irq); @@ -500,11 +497,6 @@ static int xilinx_spi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, master); return 0; - -put_master: - spi_master_put(master); - - return ret; } static int xilinx_spi_remove(struct platform_device *pdev) diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c index fc2b5eb7d614e49931d5ee8bd0ab3310ad5c1978..2fa7608f94cd1c0c0ea788b7967e5f7eedad5af4 100644 --- a/drivers/spi/spi-xtensa-xtfpga.c +++ b/drivers/spi/spi-xtensa-xtfpga.c @@ -83,7 +83,7 @@ static int xtfpga_spi_probe(struct platform_device *pdev) int ret; struct spi_master *master; - master = spi_alloc_master(&pdev->dev, sizeof(struct xtfpga_spi)); + master = devm_spi_alloc_master(&pdev->dev, sizeof(struct xtfpga_spi)); if (!master) return -ENOMEM; @@ -97,30 +97,24 @@ static int xtfpga_spi_probe(struct platform_device *pdev) xspi->bitbang.chipselect = xtfpga_spi_chipselect; xspi->bitbang.txrx_word[SPI_MODE_0] = xtfpga_spi_txrx_word; xspi->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(xspi->regs)) { - ret = PTR_ERR(xspi->regs); - goto err; - } + if (IS_ERR(xspi->regs)) + return PTR_ERR(xspi->regs); xtfpga_spi_write32(xspi, XTFPGA_SPI_START, 0); usleep_range(1000, 2000); if (xtfpga_spi_read32(xspi, XTFPGA_SPI_BUSY)) { dev_err(&pdev->dev, "Device stuck in busy state\n"); - ret = -EBUSY; - goto err; + return -EBUSY; } ret = spi_bitbang_start(&xspi->bitbang); if (ret < 0) { dev_err(&pdev->dev, "spi_bitbang_start failed\n"); - goto err; + return ret; } platform_set_drvdata(pdev, master); return 0; -err: - spi_master_put(master); - return ret; } static int xtfpga_spi_remove(struct platform_device *pdev) diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 83da8862b8f22ee414bc7f92af8550285e43e4d1..5f9aedd1f0b65e4f356c1a2a466196efd91c4a29 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -753,7 +753,7 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr, proxy->max_speed_hz = chip->max_speed_hz; proxy->mode = chip->mode; proxy->irq = chip->irq; - strlcpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias)); + strscpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias)); proxy->dev.platform_data = (void *) chip->platform_data; proxy->controller_data = chip->controller_data; proxy->controller_state = NULL; @@ -1010,9 +1010,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force) } #ifdef CONFIG_HAS_DMA -int spi_map_buf(struct spi_controller *ctlr, struct device *dev, - struct sg_table *sgt, void *buf, size_t len, - enum dma_data_direction dir) +static int spi_map_buf_attrs(struct spi_controller *ctlr, struct device *dev, + struct sg_table *sgt, void *buf, size_t len, + enum dma_data_direction dir, unsigned long attrs) { const bool vmalloced_buf = is_vmalloc_addr(buf); unsigned int max_seg_size = dma_get_max_seg_size(dev); @@ -1078,28 +1078,41 @@ int spi_map_buf(struct spi_controller *ctlr, struct device *dev, sg = sg_next(sg); } - ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir); - if (!ret) - ret = -ENOMEM; + ret = dma_map_sgtable(dev, sgt, dir, attrs); if (ret < 0) { sg_free_table(sgt); return ret; } - sgt->nents = ret; - return 0; } -void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev, - struct sg_table *sgt, enum dma_data_direction dir) +int spi_map_buf(struct spi_controller *ctlr, struct device *dev, + struct sg_table *sgt, void *buf, size_t len, + enum dma_data_direction dir) +{ + return spi_map_buf_attrs(ctlr, dev, sgt, buf, len, dir, 0); +} + +static void spi_unmap_buf_attrs(struct spi_controller *ctlr, + struct device *dev, struct sg_table *sgt, + enum dma_data_direction dir, + unsigned long attrs) { if (sgt->orig_nents) { - dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); + dma_unmap_sgtable(dev, sgt, dir, attrs); sg_free_table(sgt); + sgt->orig_nents = 0; + sgt->nents = 0; } } +void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev, + struct sg_table *sgt, enum dma_data_direction dir) +{ + spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0); +} + static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) { struct device *tx_dev, *rx_dev; @@ -1124,29 +1137,37 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) rx_dev = ctlr->dev.parent; list_for_each_entry(xfer, &msg->transfers, transfer_list) { + /* The sync is done before each transfer. */ + unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC; + if (!ctlr->can_dma(ctlr, msg->spi, xfer)) continue; if (xfer->tx_buf != NULL) { - ret = spi_map_buf(ctlr, tx_dev, &xfer->tx_sg, - (void *)xfer->tx_buf, xfer->len, - DMA_TO_DEVICE); + ret = spi_map_buf_attrs(ctlr, tx_dev, &xfer->tx_sg, + (void *)xfer->tx_buf, + xfer->len, DMA_TO_DEVICE, + attrs); if (ret != 0) return ret; } if (xfer->rx_buf != NULL) { - ret = spi_map_buf(ctlr, rx_dev, &xfer->rx_sg, - xfer->rx_buf, xfer->len, - DMA_FROM_DEVICE); + ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg, + xfer->rx_buf, xfer->len, + DMA_FROM_DEVICE, attrs); if (ret != 0) { - spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg, - DMA_TO_DEVICE); + spi_unmap_buf_attrs(ctlr, tx_dev, + &xfer->tx_sg, DMA_TO_DEVICE, + attrs); + return ret; } } } + ctlr->cur_rx_dma_dev = rx_dev; + ctlr->cur_tx_dma_dev = tx_dev; ctlr->cur_msg_mapped = true; return 0; @@ -1154,38 +1175,60 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg) { + struct device *rx_dev = ctlr->cur_rx_dma_dev; + struct device *tx_dev = ctlr->cur_tx_dma_dev; struct spi_transfer *xfer; - struct device *tx_dev, *rx_dev; if (!ctlr->cur_msg_mapped || !ctlr->can_dma) return 0; - if (ctlr->dma_tx) - tx_dev = ctlr->dma_tx->device->dev; - else if (ctlr->dma_map_dev) - tx_dev = ctlr->dma_map_dev; - else - tx_dev = ctlr->dev.parent; - - if (ctlr->dma_rx) - rx_dev = ctlr->dma_rx->device->dev; - else if (ctlr->dma_map_dev) - rx_dev = ctlr->dma_map_dev; - else - rx_dev = ctlr->dev.parent; - list_for_each_entry(xfer, &msg->transfers, transfer_list) { + /* The sync has already been done after each transfer. */ + unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC; + if (!ctlr->can_dma(ctlr, msg->spi, xfer)) continue; - spi_unmap_buf(ctlr, rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE); - spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg, DMA_TO_DEVICE); + spi_unmap_buf_attrs(ctlr, rx_dev, &xfer->rx_sg, + DMA_FROM_DEVICE, attrs); + spi_unmap_buf_attrs(ctlr, tx_dev, &xfer->tx_sg, + DMA_TO_DEVICE, attrs); } ctlr->cur_msg_mapped = false; return 0; } + +static void spi_dma_sync_for_device(struct spi_controller *ctlr, + struct spi_transfer *xfer) +{ + struct device *rx_dev = ctlr->cur_rx_dma_dev; + struct device *tx_dev = ctlr->cur_tx_dma_dev; + + if (!ctlr->cur_msg_mapped) + return; + + if (xfer->tx_sg.orig_nents) + dma_sync_sgtable_for_device(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE); + if (xfer->rx_sg.orig_nents) + dma_sync_sgtable_for_device(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE); +} + +static void spi_dma_sync_for_cpu(struct spi_controller *ctlr, + struct spi_transfer *xfer) +{ + struct device *rx_dev = ctlr->cur_rx_dma_dev; + struct device *tx_dev = ctlr->cur_tx_dma_dev; + + if (!ctlr->cur_msg_mapped) + return; + + if (xfer->rx_sg.orig_nents) + dma_sync_sgtable_for_cpu(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE); + if (xfer->tx_sg.orig_nents) + dma_sync_sgtable_for_cpu(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE); +} #else /* !CONFIG_HAS_DMA */ static inline int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) @@ -1198,6 +1241,16 @@ static inline int __spi_unmap_msg(struct spi_controller *ctlr, { return 0; } + +static void spi_dma_sync_for_device(struct spi_controller *ctrl, + struct spi_transfer *xfer) +{ +} + +static void spi_dma_sync_for_cpu(struct spi_controller *ctrl, + struct spi_transfer *xfer) +{ +} #endif /* !CONFIG_HAS_DMA */ static inline int spi_unmap_msg(struct spi_controller *ctlr, @@ -1435,7 +1488,8 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, struct spi_statistics __percpu *statm = ctlr->pcpu_statistics; struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics; - spi_set_cs(msg->spi, true, false); + xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list); + spi_set_cs(msg->spi, !xfer->cs_off, false); SPI_STATISTICS_INCREMENT_FIELD(statm, messages); SPI_STATISTICS_INCREMENT_FIELD(stats, messages); @@ -1455,8 +1509,11 @@ static int spi_transfer_one_message(struct spi_controller *ctlr, reinit_completion(&ctlr->xfer_completion); fallback_pio: + spi_dma_sync_for_device(ctlr, xfer); ret = ctlr->transfer_one(ctlr, msg->spi, xfer); if (ret < 0) { + spi_dma_sync_for_cpu(ctlr, xfer); + if (ctlr->cur_msg_mapped && (xfer->error & SPI_TRANS_FAIL_NO_START)) { __spi_unmap_msg(ctlr, msg); @@ -1479,6 +1536,8 @@ fallback_pio: if (ret < 0) msg->status = ret; } + + spi_dma_sync_for_cpu(ctlr, xfer); } else { if (xfer->len) dev_err(&msg->spi->dev, @@ -1503,10 +1562,15 @@ fallback_pio: &msg->transfers)) { keep_cs = true; } else { - spi_set_cs(msg->spi, false, false); + if (!xfer->cs_off) + spi_set_cs(msg->spi, false, false); _spi_transfer_cs_change_delay(msg, xfer); - spi_set_cs(msg->spi, true, false); + if (!list_next_entry(xfer, transfer_list)->cs_off) + spi_set_cs(msg->spi, true, false); } + } else if (!list_is_last(&xfer->transfer_list, &msg->transfers) && + xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) { + spi_set_cs(msg->spi, xfer->cs_off, false); } msg->actual_length += xfer->len; @@ -1587,6 +1651,15 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr, trace_spi_message_start(msg); + ret = spi_split_transfers_maxsize(ctlr, msg, + spi_max_transfer_size(msg->spi), + GFP_KERNEL | GFP_DMA); + if (ret) { + msg->status = ret; + spi_finalize_current_message(ctlr); + return ret; + } + if (ctlr->prepare_message) { ret = ctlr->prepare_message(ctlr, msg); if (ret) { @@ -1727,8 +1800,7 @@ static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread) spin_unlock_irqrestore(&ctlr->queue_lock, flags); ret = __spi_pump_transfer_message(ctlr, msg, was_busy); - if (!ret) - kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); + kthread_queue_work(ctlr->kworker, &ctlr->pump_messages); ctlr->cur_msg = NULL; ctlr->fallback = false; @@ -2330,7 +2402,7 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi, goto err_out; } - strlcpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias)); + strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias)); /* Use provided chip-select for ancillary device */ ancillary->chip_select = chip_select; @@ -2726,7 +2798,7 @@ static ssize_t slave_store(struct device *dev, struct device_attribute *attr, if (!spi) return -ENOMEM; - strlcpy(spi->modalias, name, sizeof(spi->modalias)); + strscpy(spi->modalias, name, sizeof(spi->modalias)); rc = spi_add_device(spi); if (rc) { @@ -4033,7 +4105,7 @@ static int __spi_sync(struct spi_device *spi, struct spi_message *message) * guard against reentrancy from a different context. The io_mutex * will catch those cases. */ - if (READ_ONCE(ctlr->queue_empty)) { + if (READ_ONCE(ctlr->queue_empty) && !ctlr->must_async) { message->actual_length = 0; message->status = -EINPROGRESS; @@ -4375,7 +4447,7 @@ static int acpi_spi_notify(struct notifier_block *nb, unsigned long value, switch (value) { case ACPI_RECONFIG_DEVICE_ADD: - ctlr = acpi_spi_find_controller_by_adev(adev->parent); + ctlr = acpi_spi_find_controller_by_adev(acpi_dev_parent(adev)); if (!ctlr) break; diff --git a/drivers/spmi/spmi-pmic-arb.c b/drivers/spmi/spmi-pmic-arb.c index 2113be40b5a973a46574c07d1c0f781271ed7259..2cf3203b2397e88e9014fd3e705d6cd6e9c795ad 100644 --- a/drivers/spmi/spmi-pmic-arb.c +++ b/drivers/spmi/spmi-pmic-arb.c @@ -91,7 +91,7 @@ enum pmic_arb_channel { /* Maximum number of support PMIC peripherals */ #define PMIC_ARB_MAX_PERIPHS 512 -#define PMIC_ARB_TIMEOUT_US 100 +#define PMIC_ARB_TIMEOUT_US 1000 #define PMIC_ARB_MAX_TRANS_BYTES (8) #define PMIC_ARB_APID_MASK 0xFF @@ -590,23 +590,16 @@ static void cleanup_irq(struct spmi_pmic_arb *pmic_arb, u16 apid, int id) u8 per = ppid & 0xFF; u8 irq_mask = BIT(id); + dev_err_ratelimited(&pmic_arb->spmic->dev, "%s apid=%d sid=0x%x per=0x%x irq=%d\n", + __func__, apid, sid, per, id); writel_relaxed(irq_mask, pmic_arb->ver_ops->irq_clear(pmic_arb, apid)); - - if (pmic_arb_write_cmd(pmic_arb->spmic, SPMI_CMD_EXT_WRITEL, sid, - (per << 8) + QPNPINT_REG_LATCHED_CLR, &irq_mask, 1)) - dev_err_ratelimited(&pmic_arb->spmic->dev, "failed to ack irq_mask = 0x%x for ppid = %x\n", - irq_mask, ppid); - - if (pmic_arb_write_cmd(pmic_arb->spmic, SPMI_CMD_EXT_WRITEL, sid, - (per << 8) + QPNPINT_REG_EN_CLR, &irq_mask, 1)) - dev_err_ratelimited(&pmic_arb->spmic->dev, "failed to ack irq_mask = 0x%x for ppid = %x\n", - irq_mask, ppid); } -static void periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid) +static int periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid) { unsigned int irq; u32 status, id; + int handled = 0; u8 sid = (pmic_arb->apid_data[apid].ppid >> 8) & 0xF; u8 per = pmic_arb->apid_data[apid].ppid & 0xFF; @@ -621,7 +614,10 @@ static void periph_interrupt(struct spmi_pmic_arb *pmic_arb, u16 apid) continue; } generic_handle_irq(irq); + handled++; } + + return handled; } static void pmic_arb_chained_irq(struct irq_desc *desc) @@ -629,28 +625,66 @@ static void pmic_arb_chained_irq(struct irq_desc *desc) struct spmi_pmic_arb *pmic_arb = irq_desc_get_handler_data(desc); const struct pmic_arb_ver_ops *ver_ops = pmic_arb->ver_ops; struct irq_chip *chip = irq_desc_get_chip(desc); - int first = pmic_arb->min_apid >> 5; - int last = pmic_arb->max_apid >> 5; + int first = pmic_arb->min_apid; + int last = pmic_arb->max_apid; u8 ee = pmic_arb->ee; - u32 status, enable; + u32 status, enable, handled = 0; int i, id, apid; + /* status based dispatch */ + bool acc_valid = false; + u32 irq_status = 0; chained_irq_enter(chip, desc); - for (i = first; i <= last; ++i) { + for (i = first >> 5; i <= last >> 5; ++i) { status = readl_relaxed( ver_ops->owner_acc_status(pmic_arb, ee, i)); + if (status) + acc_valid = true; + while (status) { id = ffs(status) - 1; status &= ~BIT(id); apid = id + i * 32; + if (apid < first || apid > last) { + WARN_ONCE(true, "spurious spmi irq received for apid=%d\n", + apid); + continue; + } enable = readl_relaxed( ver_ops->acc_enable(pmic_arb, apid)); if (enable & SPMI_PIC_ACC_ENABLE_BIT) - periph_interrupt(pmic_arb, apid); + if (periph_interrupt(pmic_arb, apid) != 0) + handled++; + } + } + + /* ACC_STATUS is empty but IRQ fired check IRQ_STATUS */ + if (!acc_valid) { + for (i = first; i <= last; i++) { + /* skip if APPS is not irq owner */ + if (pmic_arb->apid_data[i].irq_ee != pmic_arb->ee) + continue; + + irq_status = readl_relaxed( + ver_ops->irq_status(pmic_arb, i)); + if (irq_status) { + enable = readl_relaxed( + ver_ops->acc_enable(pmic_arb, i)); + if (enable & SPMI_PIC_ACC_ENABLE_BIT) { + dev_dbg(&pmic_arb->spmic->dev, + "Dispatching IRQ for apid=%d status=%x\n", + i, irq_status); + if (periph_interrupt(pmic_arb, i) != 0) + handled++; + } + } } } + if (handled == 0) + handle_bad_irq(desc); + chained_irq_exit(chip, desc); } @@ -770,6 +804,7 @@ static int qpnpint_irq_domain_activate(struct irq_domain *domain, u16 apid = hwirq_to_apid(d->hwirq); u16 sid = hwirq_to_sid(d->hwirq); u16 irq = hwirq_to_irq(d->hwirq); + u8 buf; if (pmic_arb->apid_data[apid].irq_ee != pmic_arb->ee) { dev_err(&pmic_arb->spmic->dev, "failed to xlate sid = %#x, periph = %#x, irq = %u: ee=%u but owner=%u\n", @@ -778,6 +813,10 @@ static int qpnpint_irq_domain_activate(struct irq_domain *domain, return -ENODEV; } + buf = BIT(irq); + qpnpint_spmi_write(d, QPNPINT_REG_EN_CLR, &buf, 1); + qpnpint_spmi_write(d, QPNPINT_REG_LATCHED_CLR, &buf, 1); + return 0; } @@ -992,7 +1031,8 @@ static int pmic_arb_read_apid_map_v5(struct spmi_pmic_arb *pmic_arb) * version 5, there is more than one APID mapped to each PPID. * The owner field for each of these mappings specifies the EE which is * allowed to write to the APID. The owner of the last (highest) APID - * for a given PPID will receive interrupts from the PPID. + * which has the IRQ owner bit set for a given PPID will receive + * interrupts from the PPID. */ for (i = 0; ; i++, apidd++) { offset = pmic_arb->ver_ops->apid_map_offset(i); @@ -1015,16 +1055,16 @@ static int pmic_arb_read_apid_map_v5(struct spmi_pmic_arb *pmic_arb) apid = pmic_arb->ppid_to_apid[ppid] & ~PMIC_ARB_APID_VALID; prev_apidd = &pmic_arb->apid_data[apid]; - if (valid && is_irq_ee && - prev_apidd->write_ee == pmic_arb->ee) { + if (!valid || apidd->write_ee == pmic_arb->ee) { + /* First PPID mapping or one for this EE */ + pmic_arb->ppid_to_apid[ppid] = i | PMIC_ARB_APID_VALID; + } else if (valid && is_irq_ee && + prev_apidd->write_ee == pmic_arb->ee) { /* * Duplicate PPID mapping after the one for this EE; * override the irq owner */ prev_apidd->irq_ee = apidd->irq_ee; - } else if (!valid || is_irq_ee) { - /* First PPID mapping or duplicate for another EE */ - pmic_arb->ppid_to_apid[ppid] = i | PMIC_ARB_APID_VALID; } apidd->ppid = ppid; @@ -1093,6 +1133,11 @@ static int pmic_arb_offset_v5(struct spmi_pmic_arb *pmic_arb, u8 sid, u16 addr, offset = 0x10000 * pmic_arb->ee + 0x80 * apid; break; case PMIC_ARB_CHANNEL_RW: + if (pmic_arb->apid_data[apid].write_ee != pmic_arb->ee) { + dev_err(&pmic_arb->spmic->dev, "disallowed SPMI write to sid=%u, addr=0x%04X\n", + sid, addr); + return -EPERM; + } offset = 0x10000 * apid; break; } diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index a456ce5141e16379ecb7b21553178527e30260e9..55381592bb5a698b8cf10cc6f54cfe21753b22ac 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -35,7 +35,7 @@ static void spmi_ctrl_release(struct device *dev) { struct spmi_controller *ctrl = to_spmi_controller(dev); - ida_simple_remove(&ctrl_ida, ctrl->nr); + ida_free(&ctrl_ida, ctrl->nr); kfree(ctrl); } @@ -457,7 +457,7 @@ struct spmi_controller *spmi_controller_alloc(struct device *parent, ctrl->dev.of_node = parent->of_node; spmi_controller_set_drvdata(ctrl, &ctrl[1]); - id = ida_simple_get(&ctrl_ida, 0, 0, GFP_KERNEL); + id = ida_alloc(&ctrl_ida, GFP_KERNEL); if (id < 0) { dev_err(parent, "unable to allocate SPMI controller identifier.\n"); diff --git a/drivers/ssb/driver_gpio.c b/drivers/ssb/driver_gpio.c index 2de3896489c84eb764e0e47b75819587f0c184be..897cb8db5084ff8019cf7e265ffad2a81cd33bba 100644 --- a/drivers/ssb/driver_gpio.c +++ b/drivers/ssb/driver_gpio.c @@ -132,7 +132,8 @@ static irqreturn_t ssb_gpio_irq_chipco_handler(int irq, void *dev_id) return IRQ_NONE; for_each_set_bit(gpio, &irqs, bus->gpio.ngpio) - generic_handle_irq(ssb_gpio_to_irq(&bus->gpio, gpio)); + generic_handle_domain_irq_safe(bus->irq_domain, gpio); + ssb_chipco_gpio_polarity(chipco, irqs, val & irqs); return IRQ_HANDLED; @@ -330,7 +331,8 @@ static irqreturn_t ssb_gpio_irq_extif_handler(int irq, void *dev_id) return IRQ_NONE; for_each_set_bit(gpio, &irqs, bus->gpio.ngpio) - generic_handle_irq(ssb_gpio_to_irq(&bus->gpio, gpio)); + generic_handle_domain_irq_safe(bus->irq_domain, gpio); + ssb_extif_gpio_polarity(extif, irqs, val & irqs); return IRQ_HANDLED; diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 3bd80f9695ac069df5f63fd899461af4c2d51eed..5cfabd5376cc233ab154c43c29a3278d709e4220 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -60,10 +60,6 @@ source "drivers/staging/board/Kconfig" source "drivers/staging/gdm724x/Kconfig" -source "drivers/staging/fwserial/Kconfig" - -source "drivers/staging/clocking-wizard/Kconfig" - source "drivers/staging/fbtft/Kconfig" source "drivers/staging/most/Kconfig" diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index 1d9ae39fea14a14258edf6fc313e15a7a1b54eae..f8c3aa9c241820d0f52f06c1851b042a0db154dc 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -20,8 +20,6 @@ obj-$(CONFIG_USB_EMXX) += emxx_udc/ obj-$(CONFIG_MFD_NVEC) += nvec/ obj-$(CONFIG_STAGING_BOARD) += board/ obj-$(CONFIG_LTE_GDM724X) += gdm724x/ -obj-$(CONFIG_FIREWIRE_SERIAL) += fwserial/ -obj-$(CONFIG_COMMON_CLK_XLNX_CLKWZRD) += clocking-wizard/ obj-$(CONFIG_FB_TFT) += fbtft/ obj-$(CONFIG_MOST) += most/ obj-$(CONFIG_KS7010) += ks7010/ diff --git a/drivers/staging/clocking-wizard/Kconfig b/drivers/staging/clocking-wizard/Kconfig deleted file mode 100644 index 2324b5d737886ffe959504245500f44894e4483f..0000000000000000000000000000000000000000 --- a/drivers/staging/clocking-wizard/Kconfig +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Xilinx Clocking Wizard Driver -# - -config COMMON_CLK_XLNX_CLKWZRD - tristate "Xilinx Clocking Wizard" - depends on COMMON_CLK && OF && HAS_IOMEM - help - Support for the Xilinx Clocking Wizard IP core clock generator. diff --git a/drivers/staging/clocking-wizard/Makefile b/drivers/staging/clocking-wizard/Makefile deleted file mode 100644 index b1f915224d96ea4757ae9255685c93a55dd32e6e..0000000000000000000000000000000000000000 --- a/drivers/staging/clocking-wizard/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_COMMON_CLK_XLNX_CLKWZRD) += clk-xlnx-clock-wizard.o diff --git a/drivers/staging/clocking-wizard/TODO b/drivers/staging/clocking-wizard/TODO deleted file mode 100644 index c7e1dc58dfbac1262f8aa055de4c6eec4c7f62a3..0000000000000000000000000000000000000000 --- a/drivers/staging/clocking-wizard/TODO +++ /dev/null @@ -1,13 +0,0 @@ -TODO: - - support for fractional multiplier - - support for fractional divider (output 0 only) - - support for set_rate() operations (may benefit from Stephen Boyd's - refactoring of the clk primitives: - https://lore.kernel.org/lkml/1409957256-23729-1-git-send-email-sboyd@codeaurora.org) - - review arithmetic - - overflow after multiplication? - - maximize accuracy before divisions - -Patches to: - Greg Kroah-Hartman - Sören Brinkmann diff --git a/drivers/staging/clocking-wizard/dt-binding.txt b/drivers/staging/clocking-wizard/dt-binding.txt deleted file mode 100644 index efb67ff9f76c4ec710ae39de69da67ef700237c7..0000000000000000000000000000000000000000 --- a/drivers/staging/clocking-wizard/dt-binding.txt +++ /dev/null @@ -1,30 +0,0 @@ -Binding for Xilinx Clocking Wizard IP Core - -This binding uses the common clock binding[1]. Details about the devices can be -found in the product guide[2]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt -[2] Clocking Wizard Product Guide -https://www.xilinx.com/support/documentation/ip_documentation/clk_wiz/v5_1/pg065-clk-wiz.pdf - -Required properties: - - compatible: Must be 'xlnx,clocking-wizard' - - reg: Base and size of the cores register space - - clocks: Handle to input clock - - clock-names: Tuple containing 'clk_in1' and 's_axi_aclk' - - clock-output-names: Names for the output clocks - -Optional properties: - - speed-grade: Speed grade of the device (valid values are 1..3) - -Example: - clock-generator@40040000 { - reg = <0x40040000 0x1000>; - compatible = "xlnx,clocking-wizard"; - speed-grade = <1>; - clock-names = "clk_in1", "s_axi_aclk"; - clocks = <&clkc 15>, <&clkc 15>; - clock-output-names = "clk_out0", "clk_out1", "clk_out2", - "clk_out3", "clk_out4", "clk_out5", - "clk_out6", "clk_out7"; - }; diff --git a/drivers/staging/fwserial/Kconfig b/drivers/staging/fwserial/Kconfig deleted file mode 100644 index 6964aac2a7edd525d800819f2a8b22d7700bba44..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/Kconfig +++ /dev/null @@ -1,32 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -config FIREWIRE_SERIAL - tristate "TTY over Firewire" - depends on FIREWIRE && TTY - help - This enables TTY over IEEE 1394, providing high-speed serial - connectivity to cabled peers. This driver implements a - ad-hoc transport protocol and is currently limited to - Linux-to-Linux communication. - - To compile this driver as a module, say M here: the module will - be called firewire-serial. - -if FIREWIRE_SERIAL - -config FWTTY_MAX_TOTAL_PORTS - int "Maximum number of serial ports supported" - default "64" - help - Set this to the maximum number of serial ports you want the - firewire-serial driver to support. - -config FWTTY_MAX_CARD_PORTS - int "Maximum number of serial ports supported per adapter" - range 0 FWTTY_MAX_TOTAL_PORTS - default "32" - help - Set this to the maximum number of serial ports each firewire - adapter supports. The actual number of serial ports registered - is set with the module parameter "ttys". - -endif diff --git a/drivers/staging/fwserial/Makefile b/drivers/staging/fwserial/Makefile deleted file mode 100644 index 1cd5c5c7e8056de6262904362d3396838571ff25..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_FIREWIRE_SERIAL) += firewire-serial.o -firewire-serial-objs := fwserial.o dma_fifo.o diff --git a/drivers/staging/fwserial/TODO b/drivers/staging/fwserial/TODO deleted file mode 100644 index 382a7959407c4c9cafc3f8ce94a281480816287b..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/TODO +++ /dev/null @@ -1,14 +0,0 @@ -TODOs prior to this driver moving out of staging ------------------------------------------------- -1. Implement retries for RCODE_BUSY, RCODE_NO_ACK and RCODE_SEND_ERROR - - I/O is handled asynchronously which presents some issues when error - conditions occur. -2. Implement _robust_ console on top of this. The existing prototype console - driver is not ready for the big leagues yet. -3. Expose means of controlling attach/detach of peers via sysfs. Include - GUID-to-port matching/whitelist/blacklist. - --- Issues with firewire stack -- -1. This driver uses the same unregistered vendor id that the firewire core does - (0xd00d1e). Perhaps this could be exposed as a define in - firewire.h? diff --git a/drivers/staging/fwserial/dma_fifo.c b/drivers/staging/fwserial/dma_fifo.c deleted file mode 100644 index 5dcbab6fd6224c2f8822e0fcb56aa5fb4585c897..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/dma_fifo.c +++ /dev/null @@ -1,294 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * DMA-able FIFO implementation - * - * Copyright (C) 2012 Peter Hurley - */ - -#include -#include -#include -#include - -#include "dma_fifo.h" - -#ifdef DEBUG_TRACING -#define df_trace(s, args...) pr_debug(s, ##args) -#else -#define df_trace(s, args...) -#endif - -#define FAIL(fifo, condition, format...) ({ \ - fifo->corrupt = !!(condition); \ - WARN(fifo->corrupt, format); \ -}) - -/* - * private helper fn to determine if check is in open interval (lo,hi) - */ -static bool addr_check(unsigned int check, unsigned int lo, unsigned int hi) -{ - return check - (lo + 1) < (hi - 1) - lo; -} - -/** - * dma_fifo_init: initialize the fifo to a valid but inoperative state - * @fifo: address of in-place "struct dma_fifo" object - */ -void dma_fifo_init(struct dma_fifo *fifo) -{ - memset(fifo, 0, sizeof(*fifo)); - INIT_LIST_HEAD(&fifo->pending); -} - -/** - * dma_fifo_alloc - initialize and allocate dma_fifo - * @fifo: address of in-place "struct dma_fifo" object - * @size: 'apparent' size, in bytes, of fifo - * @align: dma alignment to maintain (should be at least cpu cache alignment), - * must be power of 2 - * @tx_limit: maximum # of bytes transmissible per dma (rounded down to - * multiple of alignment, but at least align size) - * @open_limit: maximum # of outstanding dma transactions allowed - * @gfp_mask: get_free_pages mask, passed to kmalloc() - * - * The 'apparent' size will be rounded up to next greater aligned size. - * Returns 0 if no error, otherwise an error code - */ -int dma_fifo_alloc(struct dma_fifo *fifo, int size, unsigned int align, - int tx_limit, int open_limit, gfp_t gfp_mask) -{ - int capacity; - - if (!is_power_of_2(align) || size < 0) - return -EINVAL; - - size = round_up(size, align); - capacity = size + align * open_limit + align * DMA_FIFO_GUARD; - fifo->data = kmalloc(capacity, gfp_mask); - if (!fifo->data) - return -ENOMEM; - - fifo->in = 0; - fifo->out = 0; - fifo->done = 0; - fifo->size = size; - fifo->avail = size; - fifo->align = align; - fifo->tx_limit = max_t(int, round_down(tx_limit, align), align); - fifo->open = 0; - fifo->open_limit = open_limit; - fifo->guard = size + align * open_limit; - fifo->capacity = capacity; - fifo->corrupt = 0; - - return 0; -} - -/** - * dma_fifo_free - frees the fifo - * @fifo: address of in-place "struct dma_fifo" to free - * - * Also reinits the fifo to a valid but inoperative state. This - * allows the fifo to be reused with a different target requiring - * different fifo parameters. - */ -void dma_fifo_free(struct dma_fifo *fifo) -{ - struct dma_pending *pending, *next; - - if (!fifo->data) - return; - - list_for_each_entry_safe(pending, next, &fifo->pending, link) - list_del_init(&pending->link); - kfree(fifo->data); - fifo->data = NULL; -} - -/** - * dma_fifo_reset - dumps the fifo contents and reinits for reuse - * @fifo: address of in-place "struct dma_fifo" to reset - */ -void dma_fifo_reset(struct dma_fifo *fifo) -{ - struct dma_pending *pending, *next; - - if (!fifo->data) - return; - - list_for_each_entry_safe(pending, next, &fifo->pending, link) - list_del_init(&pending->link); - fifo->in = 0; - fifo->out = 0; - fifo->done = 0; - fifo->avail = fifo->size; - fifo->open = 0; - fifo->corrupt = 0; -} - -/** - * dma_fifo_in - copies data into the fifo - * @fifo: address of in-place "struct dma_fifo" to write to - * @src: buffer to copy from - * @n: # of bytes to copy - * - * Returns the # of bytes actually copied, which can be less than requested if - * the fifo becomes full. If < 0, return is error code. - */ -int dma_fifo_in(struct dma_fifo *fifo, const void *src, int n) -{ - int ofs, l; - - if (!fifo->data) - return -ENOENT; - if (fifo->corrupt) - return -ENXIO; - - if (n > fifo->avail) - n = fifo->avail; - if (n <= 0) - return 0; - - ofs = fifo->in % fifo->capacity; - l = min(n, fifo->capacity - ofs); - memcpy(fifo->data + ofs, src, l); - memcpy(fifo->data, src + l, n - l); - - if (FAIL(fifo, addr_check(fifo->done, fifo->in, fifo->in + n) || - fifo->avail < n, - "fifo corrupt: in:%u out:%u done:%u n:%d avail:%d", - fifo->in, fifo->out, fifo->done, n, fifo->avail)) - return -ENXIO; - - fifo->in += n; - fifo->avail -= n; - - df_trace("in:%u out:%u done:%u n:%d avail:%d", fifo->in, fifo->out, - fifo->done, n, fifo->avail); - - return n; -} - -/** - * dma_fifo_out_pend - gets address/len of next avail read and marks as pended - * @fifo: address of in-place "struct dma_fifo" to read from - * @pended: address of structure to fill with read address/len - * The data/len fields will be NULL/0 if no dma is pended. - * - * Returns the # of used bytes remaining in fifo (ie, if > 0, more data - * remains in the fifo that was not pended). If < 0, return is error code. - */ -int dma_fifo_out_pend(struct dma_fifo *fifo, struct dma_pending *pended) -{ - unsigned int len, n, ofs, l, limit; - - if (!fifo->data) - return -ENOENT; - if (fifo->corrupt) - return -ENXIO; - - pended->len = 0; - pended->data = NULL; - pended->out = fifo->out; - - len = fifo->in - fifo->out; - if (!len) - return -ENODATA; - if (fifo->open == fifo->open_limit) - return -EAGAIN; - - n = len; - ofs = fifo->out % fifo->capacity; - l = fifo->capacity - ofs; - limit = min_t(unsigned int, l, fifo->tx_limit); - if (n > limit) { - n = limit; - fifo->out += limit; - } else if (ofs + n > fifo->guard) { - fifo->out += l; - fifo->in = fifo->out; - } else { - fifo->out += round_up(n, fifo->align); - fifo->in = fifo->out; - } - - df_trace("in: %u out: %u done: %u n: %d len: %u avail: %d", fifo->in, - fifo->out, fifo->done, n, len, fifo->avail); - - pended->len = n; - pended->data = fifo->data + ofs; - pended->next = fifo->out; - list_add_tail(&pended->link, &fifo->pending); - ++fifo->open; - - if (FAIL(fifo, fifo->open > fifo->open_limit, - "past open limit:%d (limit:%d)", - fifo->open, fifo->open_limit)) - return -ENXIO; - if (FAIL(fifo, fifo->out & (fifo->align - 1), - "fifo out unaligned:%u (align:%u)", - fifo->out, fifo->align)) - return -ENXIO; - - return len - n; -} - -/** - * dma_fifo_out_complete - marks pended dma as completed - * @fifo: address of in-place "struct dma_fifo" which was read from - * @complete: address of structure for previously pended dma to mark completed - */ -int dma_fifo_out_complete(struct dma_fifo *fifo, struct dma_pending *complete) -{ - struct dma_pending *pending, *next, *tmp; - - if (!fifo->data) - return -ENOENT; - if (fifo->corrupt) - return -ENXIO; - if (list_empty(&fifo->pending) && fifo->open == 0) - return -EINVAL; - - if (FAIL(fifo, list_empty(&fifo->pending) != (fifo->open == 0), - "pending list disagrees with open count:%d", - fifo->open)) - return -ENXIO; - - tmp = complete->data; - *tmp = *complete; - list_replace(&complete->link, &tmp->link); - dp_mark_completed(tmp); - - /* Only update the fifo in the original pended order */ - list_for_each_entry_safe(pending, next, &fifo->pending, link) { - if (!dp_is_completed(pending)) { - df_trace("still pending: saved out: %u len: %d", - pending->out, pending->len); - break; - } - - if (FAIL(fifo, pending->out != fifo->done || - addr_check(fifo->in, fifo->done, pending->next), - "in:%u out:%u done:%u saved:%u next:%u", - fifo->in, fifo->out, fifo->done, pending->out, - pending->next)) - return -ENXIO; - - list_del_init(&pending->link); - fifo->done = pending->next; - fifo->avail += pending->len; - --fifo->open; - - df_trace("in: %u out: %u done: %u len: %u avail: %d", fifo->in, - fifo->out, fifo->done, pending->len, fifo->avail); - } - - if (FAIL(fifo, fifo->open < 0, "open dma:%d < 0", fifo->open)) - return -ENXIO; - if (FAIL(fifo, fifo->avail > fifo->size, "fifo avail:%d > size:%d", - fifo->avail, fifo->size)) - return -ENXIO; - - return 0; -} diff --git a/drivers/staging/fwserial/dma_fifo.h b/drivers/staging/fwserial/dma_fifo.h deleted file mode 100644 index c46a06336975617ba2e166ce6a3cc052d3cf9d18..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/dma_fifo.h +++ /dev/null @@ -1,117 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * DMA-able FIFO interface - * - * Copyright (C) 2012 Peter Hurley - */ - -#ifndef _DMA_FIFO_H_ -#define _DMA_FIFO_H_ - -/** - * The design basis for the DMA FIFO is to provide an output side that - * complies with the streaming DMA API design that can be DMA'd from directly - * (without additional copying), coupled with an input side that maintains a - * logically consistent 'apparent' size (ie, bytes in + bytes avail is static - * for the lifetime of the FIFO). - * - * DMA output transactions originate on a cache line boundary and can be - * variably-sized. DMA output transactions can be retired out-of-order but - * the FIFO will only advance the output in the original input sequence. - * This means the FIFO will eventually stall if a transaction is never retired. - * - * Chunking the output side into cache line multiples means that some FIFO - * memory is unused. For example, if all the avail input has been pended out, - * then the in and out markers are re-aligned to the next cache line. - * The maximum possible waste is - * (cache line alignment - 1) * (max outstanding dma transactions) - * This potential waste requires additional hidden capacity within the FIFO - * to be able to accept input while the 'apparent' size has not been reached. - * - * Additional cache lines (ie, guard area) are used to minimize DMA - * fragmentation when wrapping at the end of the FIFO. Input is allowed into the - * guard area, but the in and out FIFO markers are wrapped when DMA is pended. - */ - -#define DMA_FIFO_GUARD 3 /* # of cache lines to reserve for the guard area */ - -struct dma_fifo { - unsigned int in; - unsigned int out; /* updated when dma is pended */ - unsigned int done; /* updated upon dma completion */ - struct { - unsigned corrupt:1; - }; - int size; /* 'apparent' size of fifo */ - int guard; /* ofs of guard area */ - int capacity; /* size + reserved */ - int avail; /* # of unused bytes in fifo */ - unsigned int align; /* must be power of 2 */ - int tx_limit; /* max # of bytes per dma transaction */ - int open_limit; /* max # of outstanding allowed */ - int open; /* # of outstanding dma transactions */ - struct list_head pending; /* fifo markers for outstanding dma */ - void *data; -}; - -struct dma_pending { - struct list_head link; - void *data; - unsigned int len; - unsigned int next; - unsigned int out; -}; - -static inline void dp_mark_completed(struct dma_pending *dp) -{ - dp->data += 1; -} - -static inline bool dp_is_completed(struct dma_pending *dp) -{ - return (unsigned long)dp->data & 1UL; -} - -void dma_fifo_init(struct dma_fifo *fifo); -int dma_fifo_alloc(struct dma_fifo *fifo, int size, unsigned int align, - int tx_limit, int open_limit, gfp_t gfp_mask); -void dma_fifo_free(struct dma_fifo *fifo); -void dma_fifo_reset(struct dma_fifo *fifo); -int dma_fifo_in(struct dma_fifo *fifo, const void *src, int n); -int dma_fifo_out_pend(struct dma_fifo *fifo, struct dma_pending *pended); -int dma_fifo_out_complete(struct dma_fifo *fifo, - struct dma_pending *complete); - -/* returns the # of used bytes in the fifo */ -static inline int dma_fifo_level(struct dma_fifo *fifo) -{ - return fifo->size - fifo->avail; -} - -/* returns the # of bytes ready for output in the fifo */ -static inline int dma_fifo_out_level(struct dma_fifo *fifo) -{ - return fifo->in - fifo->out; -} - -/* returns the # of unused bytes in the fifo */ -static inline int dma_fifo_avail(struct dma_fifo *fifo) -{ - return fifo->avail; -} - -/* returns true if fifo has max # of outstanding dmas */ -static inline bool dma_fifo_busy(struct dma_fifo *fifo) -{ - return fifo->open == fifo->open_limit; -} - -/* changes the max size of dma returned from dma_fifo_out_pend() */ -static inline int dma_fifo_change_tx_limit(struct dma_fifo *fifo, int tx_limit) -{ - tx_limit = round_down(tx_limit, fifo->align); - fifo->tx_limit = max_t(int, tx_limit, fifo->align); - return 0; -} - -#endif /* _DMA_FIFO_H_ */ diff --git a/drivers/staging/fwserial/fwserial.c b/drivers/staging/fwserial/fwserial.c deleted file mode 100644 index e8fa7f53cd5ee2333bbe7213bd7e6b23ba04f6bd..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/fwserial.c +++ /dev/null @@ -1,2890 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * FireWire Serial driver - * - * Copyright (C) 2012 Peter Hurley - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fwserial.h" - -inline u64 be32_to_u64(__be32 hi, __be32 lo) -{ - return ((u64)be32_to_cpu(hi) << 32 | be32_to_cpu(lo)); -} - -#define LINUX_VENDOR_ID 0xd00d1eU /* same id used in card root directory */ -#define FWSERIAL_VERSION 0x00e81cU /* must be unique within LINUX_VENDOR_ID */ - -/* configurable options */ -static int num_ttys = 4; /* # of std ttys to create per fw_card */ - /* - doubles as loopback port index */ -static bool auto_connect = true; /* try to VIRT_CABLE to every peer */ -static bool create_loop_dev = true; /* create a loopback device for each card */ - -module_param_named(ttys, num_ttys, int, 0644); -module_param_named(auto, auto_connect, bool, 0644); -module_param_named(loop, create_loop_dev, bool, 0644); - -/* - * Threshold below which the tty is woken for writing - * - should be equal to WAKEUP_CHARS in drivers/tty/n_tty.c because - * even if the writer is woken, n_tty_poll() won't set EPOLLOUT until - * our fifo is below this level - */ -#define WAKEUP_CHARS 256 - -/* - * fwserial_list: list of every fw_serial created for each fw_card - * See discussion in fwserial_probe. - */ -static LIST_HEAD(fwserial_list); -static DEFINE_MUTEX(fwserial_list_mutex); - -/* - * port_table: array of tty ports allocated to each fw_card - * - * tty ports are allocated during probe when an fw_serial is first - * created for a given fw_card. Ports are allocated in a contiguous block, - * each block consisting of 'num_ports' ports. - */ -static struct fwtty_port *port_table[MAX_TOTAL_PORTS]; -static DEFINE_MUTEX(port_table_lock); -static bool port_table_corrupt; -#define FWTTY_INVALID_INDEX MAX_TOTAL_PORTS - -#define loop_idx(port) (((port)->index) / num_ports) -#define table_idx(loop) ((loop) * num_ports + num_ttys) - -/* total # of tty ports created per fw_card */ -static int num_ports; - -/* slab used as pool for struct fwtty_transactions */ -static struct kmem_cache *fwtty_txn_cache; - -struct tty_driver *fwtty_driver; -static struct tty_driver *fwloop_driver; - -static struct dentry *fwserial_debugfs; - -struct fwtty_transaction; -typedef void (*fwtty_transaction_cb)(struct fw_card *card, int rcode, - void *data, size_t length, - struct fwtty_transaction *txn); - -struct fwtty_transaction { - struct fw_transaction fw_txn; - fwtty_transaction_cb callback; - struct fwtty_port *port; - union { - struct dma_pending dma_pended; - }; -}; - -#define to_device(a, b) (a->b) -#define fwtty_err(p, fmt, ...) \ - dev_err(to_device(p, device), fmt, ##__VA_ARGS__) -#define fwtty_info(p, fmt, ...) \ - dev_info(to_device(p, device), fmt, ##__VA_ARGS__) -#define fwtty_notice(p, fmt, ...) \ - dev_notice(to_device(p, device), fmt, ##__VA_ARGS__) -#define fwtty_dbg(p, fmt, ...) \ - dev_dbg(to_device(p, device), "%s: " fmt, __func__, ##__VA_ARGS__) -#define fwtty_err_ratelimited(p, fmt, ...) \ - dev_err_ratelimited(to_device(p, device), fmt, ##__VA_ARGS__) - -#ifdef DEBUG -static inline void debug_short_write(struct fwtty_port *port, int c, int n) -{ - int avail; - - if (n < c) { - spin_lock_bh(&port->lock); - avail = dma_fifo_avail(&port->tx_fifo); - spin_unlock_bh(&port->lock); - fwtty_dbg(port, "short write: avail:%d req:%d wrote:%d\n", - avail, c, n); - } -} -#else -#define debug_short_write(port, c, n) -#endif - -static struct fwtty_peer *__fwserial_peer_by_node_id(struct fw_card *card, - int generation, int id); - -#ifdef FWTTY_PROFILING - -static void fwtty_profile_fifo(struct fwtty_port *port, unsigned int *stat) -{ - spin_lock_bh(&port->lock); - fwtty_profile_data(stat, dma_fifo_avail(&port->tx_fifo)); - spin_unlock_bh(&port->lock); -} - -static void fwtty_dump_profile(struct seq_file *m, struct stats *stats) -{ - /* for each stat, print sum of 0 to 2^k, then individually */ - int k = 4; - unsigned int sum; - int j; - char t[10]; - - snprintf(t, 10, "< %d", 1 << k); - seq_printf(m, "\n%14s %6s", " ", t); - for (j = k + 1; j < DISTRIBUTION_MAX_INDEX; ++j) - seq_printf(m, "%6d", 1 << j); - - ++k; - for (j = 0, sum = 0; j <= k; ++j) - sum += stats->reads[j]; - seq_printf(m, "\n%14s: %6d", "reads", sum); - for (j = k + 1; j <= DISTRIBUTION_MAX_INDEX; ++j) - seq_printf(m, "%6d", stats->reads[j]); - - for (j = 0, sum = 0; j <= k; ++j) - sum += stats->writes[j]; - seq_printf(m, "\n%14s: %6d", "writes", sum); - for (j = k + 1; j <= DISTRIBUTION_MAX_INDEX; ++j) - seq_printf(m, "%6d", stats->writes[j]); - - for (j = 0, sum = 0; j <= k; ++j) - sum += stats->txns[j]; - seq_printf(m, "\n%14s: %6d", "txns", sum); - for (j = k + 1; j <= DISTRIBUTION_MAX_INDEX; ++j) - seq_printf(m, "%6d", stats->txns[j]); - - for (j = 0, sum = 0; j <= k; ++j) - sum += stats->unthrottle[j]; - seq_printf(m, "\n%14s: %6d", "avail @ unthr", sum); - for (j = k + 1; j <= DISTRIBUTION_MAX_INDEX; ++j) - seq_printf(m, "%6d", stats->unthrottle[j]); -} - -#else -#define fwtty_profile_fifo(port, stat) -#define fwtty_dump_profile(m, stats) -#endif - -/* - * Returns the max receive packet size for the given node - * Devices which are OHCI v1.0/ v1.1/ v1.2-draft or RFC 2734 compliant - * are required by specification to support max_rec of 8 (512 bytes) or more. - */ -static inline int device_max_receive(struct fw_device *fw_device) -{ - /* see IEEE 1394-2008 table 8-8 */ - return min(2 << fw_device->max_rec, 4096); -} - -static void fwtty_log_tx_error(struct fwtty_port *port, int rcode) -{ - switch (rcode) { - case RCODE_SEND_ERROR: - fwtty_err_ratelimited(port, "card busy\n"); - break; - case RCODE_ADDRESS_ERROR: - fwtty_err_ratelimited(port, "bad unit addr or write length\n"); - break; - case RCODE_DATA_ERROR: - fwtty_err_ratelimited(port, "failed rx\n"); - break; - case RCODE_NO_ACK: - fwtty_err_ratelimited(port, "missing ack\n"); - break; - case RCODE_BUSY: - fwtty_err_ratelimited(port, "remote busy\n"); - break; - default: - fwtty_err_ratelimited(port, "failed tx: %d\n", rcode); - } -} - -static void fwtty_common_callback(struct fw_card *card, int rcode, - void *payload, size_t len, void *cb_data) -{ - struct fwtty_transaction *txn = cb_data; - struct fwtty_port *port = txn->port; - - if (port && rcode != RCODE_COMPLETE) - fwtty_log_tx_error(port, rcode); - if (txn->callback) - txn->callback(card, rcode, payload, len, txn); - kmem_cache_free(fwtty_txn_cache, txn); -} - -static int fwtty_send_data_async(struct fwtty_peer *peer, int tcode, - unsigned long long addr, void *payload, - size_t len, fwtty_transaction_cb callback, - struct fwtty_port *port) -{ - struct fwtty_transaction *txn; - int generation; - - txn = kmem_cache_alloc(fwtty_txn_cache, GFP_ATOMIC); - if (!txn) - return -ENOMEM; - - txn->callback = callback; - txn->port = port; - - generation = peer->generation; - smp_rmb(); - fw_send_request(peer->serial->card, &txn->fw_txn, tcode, - peer->node_id, generation, peer->speed, addr, payload, - len, fwtty_common_callback, txn); - return 0; -} - -static void fwtty_send_txn_async(struct fwtty_peer *peer, - struct fwtty_transaction *txn, int tcode, - unsigned long long addr, void *payload, - size_t len, fwtty_transaction_cb callback, - struct fwtty_port *port) -{ - int generation; - - txn->callback = callback; - txn->port = port; - - generation = peer->generation; - smp_rmb(); - fw_send_request(peer->serial->card, &txn->fw_txn, tcode, - peer->node_id, generation, peer->speed, addr, payload, - len, fwtty_common_callback, txn); -} - -static void __fwtty_restart_tx(struct fwtty_port *port) -{ - int len, avail; - - len = dma_fifo_out_level(&port->tx_fifo); - if (len) - schedule_delayed_work(&port->drain, 0); - avail = dma_fifo_avail(&port->tx_fifo); - - fwtty_dbg(port, "fifo len: %d avail: %d\n", len, avail); -} - -static void fwtty_restart_tx(struct fwtty_port *port) -{ - spin_lock_bh(&port->lock); - __fwtty_restart_tx(port); - spin_unlock_bh(&port->lock); -} - -/* - * fwtty_update_port_status - decodes & dispatches line status changes - * - * Note: in loopback, the port->lock is being held. Only use functions that - * don't attempt to reclaim the port->lock. - */ -static void fwtty_update_port_status(struct fwtty_port *port, - unsigned int status) -{ - unsigned int delta; - struct tty_struct *tty; - - /* simulated LSR/MSR status from remote */ - status &= ~MCTRL_MASK; - delta = (port->mstatus ^ status) & ~MCTRL_MASK; - delta &= ~(status & TIOCM_RNG); - port->mstatus = status; - - if (delta & TIOCM_RNG) - ++port->icount.rng; - if (delta & TIOCM_DSR) - ++port->icount.dsr; - if (delta & TIOCM_CAR) - ++port->icount.dcd; - if (delta & TIOCM_CTS) - ++port->icount.cts; - - fwtty_dbg(port, "status: %x delta: %x\n", status, delta); - - if (delta & TIOCM_CAR) { - tty = tty_port_tty_get(&port->port); - if (tty && !C_CLOCAL(tty)) { - if (status & TIOCM_CAR) - wake_up_interruptible(&port->port.open_wait); - else - schedule_work(&port->hangup); - } - tty_kref_put(tty); - } - - if (delta & TIOCM_CTS) { - tty = tty_port_tty_get(&port->port); - if (tty && C_CRTSCTS(tty)) { - if (tty->hw_stopped) { - if (status & TIOCM_CTS) { - tty->hw_stopped = 0; - if (port->loopback) - __fwtty_restart_tx(port); - else - fwtty_restart_tx(port); - } - } else { - if (~status & TIOCM_CTS) - tty->hw_stopped = 1; - } - } - tty_kref_put(tty); - - } else if (delta & OOB_TX_THROTTLE) { - tty = tty_port_tty_get(&port->port); - if (tty) { - if (tty->hw_stopped) { - if (~status & OOB_TX_THROTTLE) { - tty->hw_stopped = 0; - if (port->loopback) - __fwtty_restart_tx(port); - else - fwtty_restart_tx(port); - } - } else { - if (status & OOB_TX_THROTTLE) - tty->hw_stopped = 1; - } - } - tty_kref_put(tty); - } - - if (delta & (UART_LSR_BI << 24)) { - if (status & (UART_LSR_BI << 24)) { - port->break_last = jiffies; - schedule_delayed_work(&port->emit_breaks, 0); - } else { - /* run emit_breaks one last time (if pending) */ - mod_delayed_work(system_wq, &port->emit_breaks, 0); - } - } - - if (delta & (TIOCM_DSR | TIOCM_CAR | TIOCM_CTS | TIOCM_RNG)) - wake_up_interruptible(&port->port.delta_msr_wait); -} - -/* - * __fwtty_port_line_status - generate 'line status' for indicated port - * - * This function returns a remote 'MSR' state based on the local 'MCR' state, - * as if a null modem cable was attached. The actual status is a mangling - * of TIOCM_* bits suitable for sending to a peer's status_addr. - * - * Note: caller must be holding port lock - */ -static unsigned int __fwtty_port_line_status(struct fwtty_port *port) -{ - unsigned int status = 0; - - /* TODO: add module param to tie RNG to DTR as well */ - - if (port->mctrl & TIOCM_DTR) - status |= TIOCM_DSR | TIOCM_CAR; - if (port->mctrl & TIOCM_RTS) - status |= TIOCM_CTS; - if (port->mctrl & OOB_RX_THROTTLE) - status |= OOB_TX_THROTTLE; - /* emulate BRK as add'l line status */ - if (port->break_ctl) - status |= UART_LSR_BI << 24; - - return status; -} - -/* - * __fwtty_write_port_status - send the port line status to peer - * - * Note: caller must be holding the port lock. - */ -static int __fwtty_write_port_status(struct fwtty_port *port) -{ - struct fwtty_peer *peer; - int err = -ENOENT; - unsigned int status = __fwtty_port_line_status(port); - - rcu_read_lock(); - peer = rcu_dereference(port->peer); - if (peer) { - err = fwtty_send_data_async(peer, TCODE_WRITE_QUADLET_REQUEST, - peer->status_addr, &status, - sizeof(status), NULL, port); - } - rcu_read_unlock(); - - return err; -} - -/* - * fwtty_write_port_status - same as above but locked by port lock - */ -static int fwtty_write_port_status(struct fwtty_port *port) -{ - int err; - - spin_lock_bh(&port->lock); - err = __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); - return err; -} - -static void fwtty_throttle_port(struct fwtty_port *port) -{ - struct tty_struct *tty; - unsigned int old; - - tty = tty_port_tty_get(&port->port); - if (!tty) - return; - - spin_lock_bh(&port->lock); - - old = port->mctrl; - port->mctrl |= OOB_RX_THROTTLE; - if (C_CRTSCTS(tty)) - port->mctrl &= ~TIOCM_RTS; - if (~old & OOB_RX_THROTTLE) - __fwtty_write_port_status(port); - - spin_unlock_bh(&port->lock); - - tty_kref_put(tty); -} - -/* - * fwtty_do_hangup - wait for ldisc to deliver all pending rx; only then hangup - * - * When the remote has finished tx, and all in-flight rx has been received and - * pushed to the flip buffer, the remote may close its device. This will - * drop DTR on the remote which will drop carrier here. Typically, the tty is - * hung up when carrier is dropped or lost. - * - * However, there is a race between the hang up and the line discipline - * delivering its data to the reader. A hangup will cause the ldisc to flush - * (ie., clear) the read buffer and flip buffer. Because of firewire's - * relatively high throughput, the ldisc frequently lags well behind the driver, - * resulting in lost data (which has already been received and written to - * the flip buffer) when the remote closes its end. - * - * Unfortunately, since the flip buffer offers no direct method for determining - * if it holds data, ensuring the ldisc has delivered all data is problematic. - */ - -/* FIXME: drop this workaround when __tty_hangup waits for ldisc completion */ -static void fwtty_do_hangup(struct work_struct *work) -{ - struct fwtty_port *port = to_port(work, hangup); - struct tty_struct *tty; - - schedule_timeout_uninterruptible(msecs_to_jiffies(50)); - - tty = tty_port_tty_get(&port->port); - if (tty) - tty_vhangup(tty); - tty_kref_put(tty); -} - -static void fwtty_emit_breaks(struct work_struct *work) -{ - struct fwtty_port *port = to_port(to_delayed_work(work), emit_breaks); - static const char buf[16]; - unsigned long now = jiffies; - unsigned long elapsed = now - port->break_last; - int n, t, c, brk = 0; - - /* generate breaks at the line rate (but at least 1) */ - n = (elapsed * port->cps) / HZ + 1; - port->break_last = now; - - fwtty_dbg(port, "sending %d brks\n", n); - - while (n) { - t = min(n, 16); - c = tty_insert_flip_string_fixed_flag(&port->port, buf, - TTY_BREAK, t); - n -= c; - brk += c; - if (c < t) - break; - } - tty_flip_buffer_push(&port->port); - - if (port->mstatus & (UART_LSR_BI << 24)) - schedule_delayed_work(&port->emit_breaks, FREQ_BREAKS); - port->icount.brk += brk; -} - -static int fwtty_rx(struct fwtty_port *port, unsigned char *data, size_t len) -{ - int c, n = len; - unsigned int lsr; - int err = 0; - - fwtty_dbg(port, "%d\n", n); - fwtty_profile_data(port->stats.reads, n); - - if (port->write_only) { - n = 0; - goto out; - } - - /* disregard break status; breaks are generated by emit_breaks work */ - lsr = (port->mstatus >> 24) & ~UART_LSR_BI; - - if (port->overrun) - lsr |= UART_LSR_OE; - - if (lsr & UART_LSR_OE) - ++port->icount.overrun; - - lsr &= port->status_mask; - if (lsr & ~port->ignore_mask & UART_LSR_OE) { - if (!tty_insert_flip_char(&port->port, 0, TTY_OVERRUN)) { - err = -EIO; - goto out; - } - } - port->overrun = false; - - if (lsr & port->ignore_mask & ~UART_LSR_OE) { - /* TODO: don't drop SAK and Magic SysRq here */ - n = 0; - goto out; - } - - c = tty_insert_flip_string_fixed_flag(&port->port, data, TTY_NORMAL, n); - if (c > 0) - tty_flip_buffer_push(&port->port); - n -= c; - - if (n) { - port->overrun = true; - err = -EIO; - fwtty_err_ratelimited(port, "flip buffer overrun\n"); - - } else { - /* throttle the sender if remaining flip buffer space has - * reached high watermark to avoid losing data which may be - * in-flight. Since the AR request context is 32k, that much - * data may have _already_ been acked. - */ - if (tty_buffer_space_avail(&port->port) < HIGH_WATERMARK) - fwtty_throttle_port(port); - } - -out: - port->icount.rx += len; - port->stats.lost += n; - return err; -} - -/* - * fwtty_port_handler - bus address handler for port reads/writes - * - * This handler is responsible for handling inbound read/write dma from remotes. - */ -static void fwtty_port_handler(struct fw_card *card, - struct fw_request *request, - int tcode, int destination, int source, - int generation, - unsigned long long addr, - void *data, size_t len, - void *callback_data) -{ - struct fwtty_port *port = callback_data; - struct fwtty_peer *peer; - int err; - int rcode; - - /* Only accept rx from the peer virtual-cabled to this port */ - rcu_read_lock(); - peer = __fwserial_peer_by_node_id(card, generation, source); - rcu_read_unlock(); - if (!peer || peer != rcu_access_pointer(port->peer)) { - rcode = RCODE_ADDRESS_ERROR; - fwtty_err_ratelimited(port, "ignoring unauthenticated data\n"); - goto respond; - } - - switch (tcode) { - case TCODE_WRITE_QUADLET_REQUEST: - if (addr != port->rx_handler.offset || len != 4) { - rcode = RCODE_ADDRESS_ERROR; - } else { - fwtty_update_port_status(port, *(unsigned int *)data); - rcode = RCODE_COMPLETE; - } - break; - - case TCODE_WRITE_BLOCK_REQUEST: - if (addr != port->rx_handler.offset + 4 || - len > port->rx_handler.length - 4) { - rcode = RCODE_ADDRESS_ERROR; - } else { - err = fwtty_rx(port, data, len); - switch (err) { - case 0: - rcode = RCODE_COMPLETE; - break; - case -EIO: - rcode = RCODE_DATA_ERROR; - break; - default: - rcode = RCODE_CONFLICT_ERROR; - break; - } - } - break; - - default: - rcode = RCODE_TYPE_ERROR; - } - -respond: - fw_send_response(card, request, rcode); -} - -/* - * fwtty_tx_complete - callback for tx dma - * @data: ignored, has no meaning for write txns - * @length: ignored, has no meaning for write txns - * - * The writer must be woken here if the fifo has been emptied because it - * may have slept if chars_in_buffer was != 0 - */ -static void fwtty_tx_complete(struct fw_card *card, int rcode, - void *data, size_t length, - struct fwtty_transaction *txn) -{ - struct fwtty_port *port = txn->port; - int len; - - fwtty_dbg(port, "rcode: %d\n", rcode); - - switch (rcode) { - case RCODE_COMPLETE: - spin_lock_bh(&port->lock); - dma_fifo_out_complete(&port->tx_fifo, &txn->dma_pended); - len = dma_fifo_level(&port->tx_fifo); - spin_unlock_bh(&port->lock); - - port->icount.tx += txn->dma_pended.len; - break; - - default: - /* TODO: implement retries */ - spin_lock_bh(&port->lock); - dma_fifo_out_complete(&port->tx_fifo, &txn->dma_pended); - len = dma_fifo_level(&port->tx_fifo); - spin_unlock_bh(&port->lock); - - port->stats.dropped += txn->dma_pended.len; - } - - if (len < WAKEUP_CHARS) - tty_port_tty_wakeup(&port->port); -} - -static int fwtty_tx(struct fwtty_port *port, bool drain) -{ - struct fwtty_peer *peer; - struct fwtty_transaction *txn; - struct tty_struct *tty; - int n, len; - - tty = tty_port_tty_get(&port->port); - if (!tty) - return -ENOENT; - - rcu_read_lock(); - peer = rcu_dereference(port->peer); - if (!peer) { - n = -EIO; - goto out; - } - - if (test_and_set_bit(IN_TX, &port->flags)) { - n = -EALREADY; - goto out; - } - - /* try to write as many dma transactions out as possible */ - n = -EAGAIN; - while (!tty->flow.stopped && !tty->hw_stopped && - !test_bit(STOP_TX, &port->flags)) { - txn = kmem_cache_alloc(fwtty_txn_cache, GFP_ATOMIC); - if (!txn) { - n = -ENOMEM; - break; - } - - spin_lock_bh(&port->lock); - n = dma_fifo_out_pend(&port->tx_fifo, &txn->dma_pended); - spin_unlock_bh(&port->lock); - - fwtty_dbg(port, "out: %u rem: %d\n", txn->dma_pended.len, n); - - if (n < 0) { - kmem_cache_free(fwtty_txn_cache, txn); - if (n == -EAGAIN) { - ++port->stats.tx_stall; - } else if (n == -ENODATA) { - fwtty_profile_data(port->stats.txns, 0); - } else { - ++port->stats.fifo_errs; - fwtty_err_ratelimited(port, "fifo err: %d\n", - n); - } - break; - } - - fwtty_profile_data(port->stats.txns, txn->dma_pended.len); - - fwtty_send_txn_async(peer, txn, TCODE_WRITE_BLOCK_REQUEST, - peer->fifo_addr, txn->dma_pended.data, - txn->dma_pended.len, fwtty_tx_complete, - port); - ++port->stats.sent; - - /* - * Stop tx if the 'last view' of the fifo is empty or if - * this is the writer and there's not enough data to bother - */ - if (n == 0 || (!drain && n < WRITER_MINIMUM)) - break; - } - - if (n >= 0 || n == -EAGAIN || n == -ENOMEM || n == -ENODATA) { - spin_lock_bh(&port->lock); - len = dma_fifo_out_level(&port->tx_fifo); - if (len) { - unsigned long delay = (n == -ENOMEM) ? HZ : 1; - - schedule_delayed_work(&port->drain, delay); - } - len = dma_fifo_level(&port->tx_fifo); - spin_unlock_bh(&port->lock); - - /* wakeup the writer */ - if (drain && len < WAKEUP_CHARS) - tty_wakeup(tty); - } - - clear_bit(IN_TX, &port->flags); - wake_up_interruptible(&port->wait_tx); - -out: - rcu_read_unlock(); - tty_kref_put(tty); - return n; -} - -static void fwtty_drain_tx(struct work_struct *work) -{ - struct fwtty_port *port = to_port(to_delayed_work(work), drain); - - fwtty_tx(port, true); -} - -static void fwtty_write_xchar(struct fwtty_port *port, char ch) -{ - struct fwtty_peer *peer; - - ++port->stats.xchars; - - fwtty_dbg(port, "%02x\n", ch); - - rcu_read_lock(); - peer = rcu_dereference(port->peer); - if (peer) { - fwtty_send_data_async(peer, TCODE_WRITE_BLOCK_REQUEST, - peer->fifo_addr, &ch, sizeof(ch), - NULL, port); - } - rcu_read_unlock(); -} - -static struct fwtty_port *fwtty_port_get(unsigned int index) -{ - struct fwtty_port *port; - - if (index >= MAX_TOTAL_PORTS) - return NULL; - - mutex_lock(&port_table_lock); - port = port_table[index]; - if (port) - kref_get(&port->serial->kref); - mutex_unlock(&port_table_lock); - return port; -} - -static int fwtty_ports_add(struct fw_serial *serial) -{ - int err = -EBUSY; - int i, j; - - if (port_table_corrupt) - return err; - - mutex_lock(&port_table_lock); - for (i = 0; i + num_ports <= MAX_TOTAL_PORTS; i += num_ports) { - if (!port_table[i]) { - for (j = 0; j < num_ports; ++i, ++j) { - serial->ports[j]->index = i; - port_table[i] = serial->ports[j]; - } - err = 0; - break; - } - } - mutex_unlock(&port_table_lock); - return err; -} - -static void fwserial_destroy(struct kref *kref) -{ - struct fw_serial *serial = to_serial(kref, kref); - struct fwtty_port **ports = serial->ports; - int j, i = ports[0]->index; - - synchronize_rcu(); - - mutex_lock(&port_table_lock); - for (j = 0; j < num_ports; ++i, ++j) { - port_table_corrupt |= port_table[i] != ports[j]; - WARN_ONCE(port_table_corrupt, "port_table[%d]: %p != ports[%d]: %p", - i, port_table[i], j, ports[j]); - - port_table[i] = NULL; - } - mutex_unlock(&port_table_lock); - - for (j = 0; j < num_ports; ++j) { - fw_core_remove_address_handler(&ports[j]->rx_handler); - tty_port_destroy(&ports[j]->port); - kfree(ports[j]); - } - kfree(serial); -} - -static void fwtty_port_put(struct fwtty_port *port) -{ - kref_put(&port->serial->kref, fwserial_destroy); -} - -static void fwtty_port_dtr_rts(struct tty_port *tty_port, int on) -{ - struct fwtty_port *port = to_port(tty_port, port); - - fwtty_dbg(port, "on/off: %d\n", on); - - spin_lock_bh(&port->lock); - /* Don't change carrier state if this is a console */ - if (!port->port.console) { - if (on) - port->mctrl |= TIOCM_DTR | TIOCM_RTS; - else - port->mctrl &= ~(TIOCM_DTR | TIOCM_RTS); - } - - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); -} - -/* - * fwtty_port_carrier_raised: required tty_port operation - * - * This port operation is polled after a tty has been opened and is waiting for - * carrier detect -- see drivers/tty/tty_port:tty_port_block_til_ready(). - */ -static int fwtty_port_carrier_raised(struct tty_port *tty_port) -{ - struct fwtty_port *port = to_port(tty_port, port); - int rc; - - rc = (port->mstatus & TIOCM_CAR); - - fwtty_dbg(port, "%d\n", rc); - - return rc; -} - -static unsigned int set_termios(struct fwtty_port *port, struct tty_struct *tty) -{ - unsigned int baud, frame; - - baud = tty_termios_baud_rate(&tty->termios); - tty_termios_encode_baud_rate(&tty->termios, baud, baud); - - /* compute bit count of 2 frames */ - frame = 12 + ((C_CSTOPB(tty)) ? 4 : 2) + ((C_PARENB(tty)) ? 2 : 0); - - switch (C_CSIZE(tty)) { - case CS5: - frame -= (C_CSTOPB(tty)) ? 1 : 0; - break; - case CS6: - frame += 2; - break; - case CS7: - frame += 4; - break; - case CS8: - frame += 6; - break; - } - - port->cps = (baud << 1) / frame; - - port->status_mask = UART_LSR_OE; - if (_I_FLAG(tty, BRKINT | PARMRK)) - port->status_mask |= UART_LSR_BI; - - port->ignore_mask = 0; - if (I_IGNBRK(tty)) { - port->ignore_mask |= UART_LSR_BI; - if (I_IGNPAR(tty)) - port->ignore_mask |= UART_LSR_OE; - } - - port->write_only = !C_CREAD(tty); - - /* turn off echo and newline xlat if loopback */ - if (port->loopback) { - tty->termios.c_lflag &= ~(ECHO | ECHOE | ECHOK | ECHOKE | - ECHONL | ECHOPRT | ECHOCTL); - tty->termios.c_oflag &= ~ONLCR; - } - - return baud; -} - -static int fwtty_port_activate(struct tty_port *tty_port, - struct tty_struct *tty) -{ - struct fwtty_port *port = to_port(tty_port, port); - unsigned int baud; - int err; - - set_bit(TTY_IO_ERROR, &tty->flags); - - err = dma_fifo_alloc(&port->tx_fifo, FWTTY_PORT_TXFIFO_LEN, - cache_line_size(), - port->max_payload, - FWTTY_PORT_MAX_PEND_DMA, - GFP_KERNEL); - if (err) - return err; - - spin_lock_bh(&port->lock); - - baud = set_termios(port, tty); - - /* if console, don't change carrier state */ - if (!port->port.console) { - port->mctrl = 0; - if (baud != 0) - port->mctrl = TIOCM_DTR | TIOCM_RTS; - } - - if (C_CRTSCTS(tty) && ~port->mstatus & TIOCM_CTS) - tty->hw_stopped = 1; - - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); - - clear_bit(TTY_IO_ERROR, &tty->flags); - - return 0; -} - -/* - * fwtty_port_shutdown - * - * Note: the tty port core ensures this is not the console and - * manages TTY_IO_ERROR properly - */ -static void fwtty_port_shutdown(struct tty_port *tty_port) -{ - struct fwtty_port *port = to_port(tty_port, port); - - /* TODO: cancel outstanding transactions */ - - cancel_delayed_work_sync(&port->emit_breaks); - cancel_delayed_work_sync(&port->drain); - - spin_lock_bh(&port->lock); - port->flags = 0; - port->break_ctl = 0; - port->overrun = 0; - __fwtty_write_port_status(port); - dma_fifo_free(&port->tx_fifo); - spin_unlock_bh(&port->lock); -} - -static int fwtty_open(struct tty_struct *tty, struct file *fp) -{ - struct fwtty_port *port = tty->driver_data; - - return tty_port_open(&port->port, tty, fp); -} - -static void fwtty_close(struct tty_struct *tty, struct file *fp) -{ - struct fwtty_port *port = tty->driver_data; - - tty_port_close(&port->port, tty, fp); -} - -static void fwtty_hangup(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - - tty_port_hangup(&port->port); -} - -static void fwtty_cleanup(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - - tty->driver_data = NULL; - fwtty_port_put(port); -} - -static int fwtty_install(struct tty_driver *driver, struct tty_struct *tty) -{ - struct fwtty_port *port = fwtty_port_get(tty->index); - int err; - - err = tty_standard_install(driver, tty); - if (!err) - tty->driver_data = port; - else - fwtty_port_put(port); - return err; -} - -static int fwloop_install(struct tty_driver *driver, struct tty_struct *tty) -{ - struct fwtty_port *port = fwtty_port_get(table_idx(tty->index)); - int err; - - err = tty_standard_install(driver, tty); - if (!err) - tty->driver_data = port; - else - fwtty_port_put(port); - return err; -} - -static int fwtty_write(struct tty_struct *tty, const unsigned char *buf, int c) -{ - struct fwtty_port *port = tty->driver_data; - int n, len; - - fwtty_dbg(port, "%d\n", c); - fwtty_profile_data(port->stats.writes, c); - - spin_lock_bh(&port->lock); - n = dma_fifo_in(&port->tx_fifo, buf, c); - len = dma_fifo_out_level(&port->tx_fifo); - if (len < DRAIN_THRESHOLD) - schedule_delayed_work(&port->drain, 1); - spin_unlock_bh(&port->lock); - - if (len >= DRAIN_THRESHOLD) - fwtty_tx(port, false); - - debug_short_write(port, c, n); - - return (n < 0) ? 0 : n; -} - -static unsigned int fwtty_write_room(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - unsigned int n; - - spin_lock_bh(&port->lock); - n = dma_fifo_avail(&port->tx_fifo); - spin_unlock_bh(&port->lock); - - fwtty_dbg(port, "%u\n", n); - - return n; -} - -static unsigned int fwtty_chars_in_buffer(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - unsigned int n; - - spin_lock_bh(&port->lock); - n = dma_fifo_level(&port->tx_fifo); - spin_unlock_bh(&port->lock); - - fwtty_dbg(port, "%u\n", n); - - return n; -} - -static void fwtty_send_xchar(struct tty_struct *tty, char ch) -{ - struct fwtty_port *port = tty->driver_data; - - fwtty_dbg(port, "%02x\n", ch); - - fwtty_write_xchar(port, ch); -} - -static void fwtty_throttle(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - - /* - * Ignore throttling (but not unthrottling). - * It only makes sense to throttle when data will no longer be - * accepted by the tty flip buffer. For example, it is - * possible for received data to overflow the tty buffer long - * before the line discipline ever has a chance to throttle the driver. - * Additionally, the driver may have already completed the I/O - * but the tty buffer is still emptying, so the line discipline is - * throttling and unthrottling nothing. - */ - - ++port->stats.throttled; -} - -static void fwtty_unthrottle(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - - fwtty_dbg(port, "CRTSCTS: %d\n", C_CRTSCTS(tty) != 0); - - fwtty_profile_fifo(port, port->stats.unthrottle); - - spin_lock_bh(&port->lock); - port->mctrl &= ~OOB_RX_THROTTLE; - if (C_CRTSCTS(tty)) - port->mctrl |= TIOCM_RTS; - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); -} - -static int check_msr_delta(struct fwtty_port *port, unsigned long mask, - struct async_icount *prev) -{ - struct async_icount now; - int delta; - - now = port->icount; - - delta = ((mask & TIOCM_RNG && prev->rng != now.rng) || - (mask & TIOCM_DSR && prev->dsr != now.dsr) || - (mask & TIOCM_CAR && prev->dcd != now.dcd) || - (mask & TIOCM_CTS && prev->cts != now.cts)); - - *prev = now; - - return delta; -} - -static int wait_msr_change(struct fwtty_port *port, unsigned long mask) -{ - struct async_icount prev; - - prev = port->icount; - - return wait_event_interruptible(port->port.delta_msr_wait, - check_msr_delta(port, mask, &prev)); -} - -static int get_serial_info(struct tty_struct *tty, - struct serial_struct *ss) -{ - struct fwtty_port *port = tty->driver_data; - - mutex_lock(&port->port.mutex); - ss->line = port->index; - ss->baud_base = 400000000; - ss->close_delay = jiffies_to_msecs(port->port.close_delay) / 10; - ss->closing_wait = 3000; - mutex_unlock(&port->port.mutex); - - return 0; -} - -static int set_serial_info(struct tty_struct *tty, - struct serial_struct *ss) -{ - struct fwtty_port *port = tty->driver_data; - unsigned int cdelay; - - cdelay = msecs_to_jiffies(ss->close_delay * 10); - - mutex_lock(&port->port.mutex); - if (!capable(CAP_SYS_ADMIN)) { - if (cdelay != port->port.close_delay || - ((ss->flags & ~ASYNC_USR_MASK) != - (port->port.flags & ~ASYNC_USR_MASK))) { - mutex_unlock(&port->port.mutex); - return -EPERM; - } - } - port->port.close_delay = cdelay; - mutex_unlock(&port->port.mutex); - - return 0; -} - -static int fwtty_ioctl(struct tty_struct *tty, unsigned int cmd, - unsigned long arg) -{ - struct fwtty_port *port = tty->driver_data; - int err; - - switch (cmd) { - case TIOCMIWAIT: - err = wait_msr_change(port, arg); - break; - - default: - err = -ENOIOCTLCMD; - } - - return err; -} - -static void fwtty_set_termios(struct tty_struct *tty, struct ktermios *old) -{ - struct fwtty_port *port = tty->driver_data; - unsigned int baud; - - spin_lock_bh(&port->lock); - baud = set_termios(port, tty); - - if ((baud == 0) && (old->c_cflag & CBAUD)) { - port->mctrl &= ~(TIOCM_DTR | TIOCM_RTS); - } else if ((baud != 0) && !(old->c_cflag & CBAUD)) { - if (C_CRTSCTS(tty) || !tty_throttled(tty)) - port->mctrl |= TIOCM_DTR | TIOCM_RTS; - else - port->mctrl |= TIOCM_DTR; - } - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); - - if (old->c_cflag & CRTSCTS) { - if (!C_CRTSCTS(tty)) { - tty->hw_stopped = 0; - fwtty_restart_tx(port); - } - } else if (C_CRTSCTS(tty) && ~port->mstatus & TIOCM_CTS) { - tty->hw_stopped = 1; - } -} - -/* - * fwtty_break_ctl - start/stop sending breaks - * - * Signals the remote to start or stop generating simulated breaks. - * First, stop dequeueing from the fifo and wait for writer/drain to leave tx - * before signalling the break line status. This guarantees any pending rx will - * be queued to the line discipline before break is simulated on the remote. - * Conversely, turning off break_ctl requires signalling the line status change, - * then enabling tx. - */ -static int fwtty_break_ctl(struct tty_struct *tty, int state) -{ - struct fwtty_port *port = tty->driver_data; - long ret; - - fwtty_dbg(port, "%d\n", state); - - if (state == -1) { - set_bit(STOP_TX, &port->flags); - ret = wait_event_interruptible_timeout(port->wait_tx, - !test_bit(IN_TX, &port->flags), - 10); - if (ret == 0 || ret == -ERESTARTSYS) { - clear_bit(STOP_TX, &port->flags); - fwtty_restart_tx(port); - return -EINTR; - } - } - - spin_lock_bh(&port->lock); - port->break_ctl = (state == -1); - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); - - if (state == 0) { - spin_lock_bh(&port->lock); - dma_fifo_reset(&port->tx_fifo); - clear_bit(STOP_TX, &port->flags); - spin_unlock_bh(&port->lock); - } - return 0; -} - -static int fwtty_tiocmget(struct tty_struct *tty) -{ - struct fwtty_port *port = tty->driver_data; - unsigned int tiocm; - - spin_lock_bh(&port->lock); - tiocm = (port->mctrl & MCTRL_MASK) | (port->mstatus & ~MCTRL_MASK); - spin_unlock_bh(&port->lock); - - fwtty_dbg(port, "%x\n", tiocm); - - return tiocm; -} - -static int fwtty_tiocmset(struct tty_struct *tty, - unsigned int set, unsigned int clear) -{ - struct fwtty_port *port = tty->driver_data; - - fwtty_dbg(port, "set: %x clear: %x\n", set, clear); - - /* TODO: simulate loopback if TIOCM_LOOP set */ - - spin_lock_bh(&port->lock); - port->mctrl &= ~(clear & MCTRL_MASK & 0xffff); - port->mctrl |= set & MCTRL_MASK & 0xffff; - __fwtty_write_port_status(port); - spin_unlock_bh(&port->lock); - return 0; -} - -static int fwtty_get_icount(struct tty_struct *tty, - struct serial_icounter_struct *icount) -{ - struct fwtty_port *port = tty->driver_data; - struct stats stats; - - memcpy(&stats, &port->stats, sizeof(stats)); - if (port->port.console) - (*port->fwcon_ops->stats)(&stats, port->con_data); - - icount->cts = port->icount.cts; - icount->dsr = port->icount.dsr; - icount->rng = port->icount.rng; - icount->dcd = port->icount.dcd; - icount->rx = port->icount.rx; - icount->tx = port->icount.tx + stats.xchars; - icount->frame = port->icount.frame; - icount->overrun = port->icount.overrun; - icount->parity = port->icount.parity; - icount->brk = port->icount.brk; - icount->buf_overrun = port->icount.overrun; - return 0; -} - -static void fwtty_proc_show_port(struct seq_file *m, struct fwtty_port *port) -{ - struct stats stats; - - memcpy(&stats, &port->stats, sizeof(stats)); - if (port->port.console) - (*port->fwcon_ops->stats)(&stats, port->con_data); - - seq_printf(m, " addr:%012llx tx:%d rx:%d", port->rx_handler.offset, - port->icount.tx + stats.xchars, port->icount.rx); - seq_printf(m, " cts:%d dsr:%d rng:%d dcd:%d", port->icount.cts, - port->icount.dsr, port->icount.rng, port->icount.dcd); - seq_printf(m, " fe:%d oe:%d pe:%d brk:%d", port->icount.frame, - port->icount.overrun, port->icount.parity, port->icount.brk); -} - -static void fwtty_debugfs_show_port(struct seq_file *m, struct fwtty_port *port) -{ - struct stats stats; - - memcpy(&stats, &port->stats, sizeof(stats)); - if (port->port.console) - (*port->fwcon_ops->stats)(&stats, port->con_data); - - seq_printf(m, " dr:%d st:%d err:%d lost:%d", stats.dropped, - stats.tx_stall, stats.fifo_errs, stats.lost); - seq_printf(m, " pkts:%d thr:%d", stats.sent, stats.throttled); - - if (port->port.console) { - seq_puts(m, "\n "); - (*port->fwcon_ops->proc_show)(m, port->con_data); - } - - fwtty_dump_profile(m, &port->stats); -} - -static void fwtty_debugfs_show_peer(struct seq_file *m, struct fwtty_peer *peer) -{ - int generation = peer->generation; - - smp_rmb(); - seq_printf(m, " %s:", dev_name(&peer->unit->device)); - seq_printf(m, " node:%04x gen:%d", peer->node_id, generation); - seq_printf(m, " sp:%d max:%d guid:%016llx", peer->speed, - peer->max_payload, (unsigned long long)peer->guid); - seq_printf(m, " mgmt:%012llx", (unsigned long long)peer->mgmt_addr); - seq_printf(m, " addr:%012llx", (unsigned long long)peer->status_addr); - seq_putc(m, '\n'); -} - -static int fwtty_proc_show(struct seq_file *m, void *v) -{ - struct fwtty_port *port; - int i; - - seq_puts(m, "fwserinfo: 1.0 driver: 1.0\n"); - for (i = 0; i < MAX_TOTAL_PORTS && (port = fwtty_port_get(i)); ++i) { - seq_printf(m, "%2d:", i); - if (capable(CAP_SYS_ADMIN)) - fwtty_proc_show_port(m, port); - fwtty_port_put(port); - seq_puts(m, "\n"); - } - return 0; -} - -static int fwtty_stats_show(struct seq_file *m, void *v) -{ - struct fw_serial *serial = m->private; - struct fwtty_port *port; - int i; - - for (i = 0; i < num_ports; ++i) { - port = fwtty_port_get(serial->ports[i]->index); - if (port) { - seq_printf(m, "%2d:", port->index); - fwtty_proc_show_port(m, port); - fwtty_debugfs_show_port(m, port); - fwtty_port_put(port); - seq_puts(m, "\n"); - } - } - return 0; -} -DEFINE_SHOW_ATTRIBUTE(fwtty_stats); - -static int fwtty_peers_show(struct seq_file *m, void *v) -{ - struct fw_serial *serial = m->private; - struct fwtty_peer *peer; - - rcu_read_lock(); - seq_printf(m, "card: %s guid: %016llx\n", - dev_name(serial->card->device), - (unsigned long long)serial->card->guid); - list_for_each_entry_rcu(peer, &serial->peer_list, list) - fwtty_debugfs_show_peer(m, peer); - rcu_read_unlock(); - return 0; -} -DEFINE_SHOW_ATTRIBUTE(fwtty_peers); - -static const struct tty_port_operations fwtty_port_ops = { - .dtr_rts = fwtty_port_dtr_rts, - .carrier_raised = fwtty_port_carrier_raised, - .shutdown = fwtty_port_shutdown, - .activate = fwtty_port_activate, -}; - -static const struct tty_operations fwtty_ops = { - .open = fwtty_open, - .close = fwtty_close, - .hangup = fwtty_hangup, - .cleanup = fwtty_cleanup, - .install = fwtty_install, - .write = fwtty_write, - .write_room = fwtty_write_room, - .chars_in_buffer = fwtty_chars_in_buffer, - .send_xchar = fwtty_send_xchar, - .throttle = fwtty_throttle, - .unthrottle = fwtty_unthrottle, - .ioctl = fwtty_ioctl, - .set_termios = fwtty_set_termios, - .break_ctl = fwtty_break_ctl, - .tiocmget = fwtty_tiocmget, - .tiocmset = fwtty_tiocmset, - .get_icount = fwtty_get_icount, - .set_serial = set_serial_info, - .get_serial = get_serial_info, - .proc_show = fwtty_proc_show, -}; - -static const struct tty_operations fwloop_ops = { - .open = fwtty_open, - .close = fwtty_close, - .hangup = fwtty_hangup, - .cleanup = fwtty_cleanup, - .install = fwloop_install, - .write = fwtty_write, - .write_room = fwtty_write_room, - .chars_in_buffer = fwtty_chars_in_buffer, - .send_xchar = fwtty_send_xchar, - .throttle = fwtty_throttle, - .unthrottle = fwtty_unthrottle, - .ioctl = fwtty_ioctl, - .set_termios = fwtty_set_termios, - .break_ctl = fwtty_break_ctl, - .tiocmget = fwtty_tiocmget, - .tiocmset = fwtty_tiocmset, - .get_icount = fwtty_get_icount, - .set_serial = set_serial_info, - .get_serial = get_serial_info, -}; - -static inline int mgmt_pkt_expected_len(__be16 code) -{ - static const struct fwserial_mgmt_pkt pkt; - - switch (be16_to_cpu(code)) { - case FWSC_VIRT_CABLE_PLUG: - return sizeof(pkt.hdr) + sizeof(pkt.plug_req); - - case FWSC_VIRT_CABLE_PLUG_RSP: /* | FWSC_RSP_OK */ - return sizeof(pkt.hdr) + sizeof(pkt.plug_rsp); - - case FWSC_VIRT_CABLE_UNPLUG: - case FWSC_VIRT_CABLE_UNPLUG_RSP: - case FWSC_VIRT_CABLE_PLUG_RSP | FWSC_RSP_NACK: - case FWSC_VIRT_CABLE_UNPLUG_RSP | FWSC_RSP_NACK: - return sizeof(pkt.hdr); - - default: - return -1; - } -} - -static inline void fill_plug_params(struct virt_plug_params *params, - struct fwtty_port *port) -{ - u64 status_addr = port->rx_handler.offset; - u64 fifo_addr = port->rx_handler.offset + 4; - size_t fifo_len = port->rx_handler.length - 4; - - params->status_hi = cpu_to_be32(status_addr >> 32); - params->status_lo = cpu_to_be32(status_addr); - params->fifo_hi = cpu_to_be32(fifo_addr >> 32); - params->fifo_lo = cpu_to_be32(fifo_addr); - params->fifo_len = cpu_to_be32(fifo_len); -} - -static inline void fill_plug_req(struct fwserial_mgmt_pkt *pkt, - struct fwtty_port *port) -{ - pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_PLUG); - pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code)); - fill_plug_params(&pkt->plug_req, port); -} - -static inline void fill_plug_rsp_ok(struct fwserial_mgmt_pkt *pkt, - struct fwtty_port *port) -{ - pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_PLUG_RSP); - pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code)); - fill_plug_params(&pkt->plug_rsp, port); -} - -static inline void fill_plug_rsp_nack(struct fwserial_mgmt_pkt *pkt) -{ - pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_PLUG_RSP | FWSC_RSP_NACK); - pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code)); -} - -static inline void fill_unplug_rsp_nack(struct fwserial_mgmt_pkt *pkt) -{ - pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_UNPLUG_RSP | FWSC_RSP_NACK); - pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code)); -} - -static inline void fill_unplug_rsp_ok(struct fwserial_mgmt_pkt *pkt) -{ - pkt->hdr.code = cpu_to_be16(FWSC_VIRT_CABLE_UNPLUG_RSP); - pkt->hdr.len = cpu_to_be16(mgmt_pkt_expected_len(pkt->hdr.code)); -} - -static void fwserial_virt_plug_complete(struct fwtty_peer *peer, - struct virt_plug_params *params) -{ - struct fwtty_port *port = peer->port; - - peer->status_addr = be32_to_u64(params->status_hi, params->status_lo); - peer->fifo_addr = be32_to_u64(params->fifo_hi, params->fifo_lo); - peer->fifo_len = be32_to_cpu(params->fifo_len); - peer_set_state(peer, FWPS_ATTACHED); - - /* reconfigure tx_fifo optimally for this peer */ - spin_lock_bh(&port->lock); - port->max_payload = min(peer->max_payload, peer->fifo_len); - dma_fifo_change_tx_limit(&port->tx_fifo, port->max_payload); - spin_unlock_bh(&peer->port->lock); - - if (port->port.console && port->fwcon_ops->notify) - (*port->fwcon_ops->notify)(FWCON_NOTIFY_ATTACH, port->con_data); - - fwtty_info(&peer->unit, "peer (guid:%016llx) connected on %s\n", - (unsigned long long)peer->guid, dev_name(port->device)); -} - -static inline int fwserial_send_mgmt_sync(struct fwtty_peer *peer, - struct fwserial_mgmt_pkt *pkt) -{ - int generation; - int rcode, tries = 5; - - do { - generation = peer->generation; - smp_rmb(); - - rcode = fw_run_transaction(peer->serial->card, - TCODE_WRITE_BLOCK_REQUEST, - peer->node_id, - generation, peer->speed, - peer->mgmt_addr, - pkt, be16_to_cpu(pkt->hdr.len)); - if (rcode == RCODE_BUSY || rcode == RCODE_SEND_ERROR || - rcode == RCODE_GENERATION) { - fwtty_dbg(&peer->unit, "mgmt write error: %d\n", rcode); - continue; - } else { - break; - } - } while (--tries > 0); - return rcode; -} - -/* - * fwserial_claim_port - attempt to claim port @ index for peer - * - * Returns ptr to claimed port or error code (as ERR_PTR()) - * Can sleep - must be called from process context - */ -static struct fwtty_port *fwserial_claim_port(struct fwtty_peer *peer, - int index) -{ - struct fwtty_port *port; - - if (index < 0 || index >= num_ports) - return ERR_PTR(-EINVAL); - - /* must guarantee that previous port releases have completed */ - synchronize_rcu(); - - port = peer->serial->ports[index]; - spin_lock_bh(&port->lock); - if (!rcu_access_pointer(port->peer)) - rcu_assign_pointer(port->peer, peer); - else - port = ERR_PTR(-EBUSY); - spin_unlock_bh(&port->lock); - - return port; -} - -/* - * fwserial_find_port - find avail port and claim for peer - * - * Returns ptr to claimed port or NULL if none avail - * Can sleep - must be called from process context - */ -static struct fwtty_port *fwserial_find_port(struct fwtty_peer *peer) -{ - struct fwtty_port **ports = peer->serial->ports; - int i; - - /* must guarantee that previous port releases have completed */ - synchronize_rcu(); - - /* TODO: implement optional GUID-to-specific port # matching */ - - /* find an unattached port (but not the loopback port, if present) */ - for (i = 0; i < num_ttys; ++i) { - spin_lock_bh(&ports[i]->lock); - if (!ports[i]->peer) { - /* claim port */ - rcu_assign_pointer(ports[i]->peer, peer); - spin_unlock_bh(&ports[i]->lock); - return ports[i]; - } - spin_unlock_bh(&ports[i]->lock); - } - return NULL; -} - -static void fwserial_release_port(struct fwtty_port *port, bool reset) -{ - /* drop carrier (and all other line status) */ - if (reset) - fwtty_update_port_status(port, 0); - - spin_lock_bh(&port->lock); - - /* reset dma fifo max transmission size back to S100 */ - port->max_payload = link_speed_to_max_payload(SCODE_100); - dma_fifo_change_tx_limit(&port->tx_fifo, port->max_payload); - - RCU_INIT_POINTER(port->peer, NULL); - spin_unlock_bh(&port->lock); - - if (port->port.console && port->fwcon_ops->notify) - (*port->fwcon_ops->notify)(FWCON_NOTIFY_DETACH, port->con_data); -} - -static void fwserial_plug_timeout(struct timer_list *t) -{ - struct fwtty_peer *peer = from_timer(peer, t, timer); - struct fwtty_port *port; - - spin_lock_bh(&peer->lock); - if (peer->state != FWPS_PLUG_PENDING) { - spin_unlock_bh(&peer->lock); - return; - } - - port = peer_revert_state(peer); - spin_unlock_bh(&peer->lock); - - if (port) - fwserial_release_port(port, false); -} - -/* - * fwserial_connect_peer - initiate virtual cable with peer - * - * Returns 0 if VIRT_CABLE_PLUG request was successfully sent, - * otherwise error code. Must be called from process context. - */ -static int fwserial_connect_peer(struct fwtty_peer *peer) -{ - struct fwtty_port *port; - struct fwserial_mgmt_pkt *pkt; - int err, rcode; - - pkt = kmalloc(sizeof(*pkt), GFP_KERNEL); - if (!pkt) - return -ENOMEM; - - port = fwserial_find_port(peer); - if (!port) { - fwtty_err(&peer->unit, "avail ports in use\n"); - err = -EBUSY; - goto free_pkt; - } - - spin_lock_bh(&peer->lock); - - /* only initiate VIRT_CABLE_PLUG if peer is currently not attached */ - if (peer->state != FWPS_NOT_ATTACHED) { - err = -EBUSY; - goto release_port; - } - - peer->port = port; - peer_set_state(peer, FWPS_PLUG_PENDING); - - fill_plug_req(pkt, peer->port); - - mod_timer(&peer->timer, jiffies + VIRT_CABLE_PLUG_TIMEOUT); - spin_unlock_bh(&peer->lock); - - rcode = fwserial_send_mgmt_sync(peer, pkt); - - spin_lock_bh(&peer->lock); - if (peer->state == FWPS_PLUG_PENDING && rcode != RCODE_COMPLETE) { - if (rcode == RCODE_CONFLICT_ERROR) - err = -EAGAIN; - else - err = -EIO; - goto cancel_timer; - } - spin_unlock_bh(&peer->lock); - - kfree(pkt); - return 0; - -cancel_timer: - del_timer(&peer->timer); - peer_revert_state(peer); -release_port: - spin_unlock_bh(&peer->lock); - fwserial_release_port(port, false); -free_pkt: - kfree(pkt); - return err; -} - -/* - * fwserial_close_port - - * HUP the tty (if the tty exists) and unregister the tty device. - * Only used by the unit driver upon unit removal to disconnect and - * cleanup all attached ports - * - * The port reference is put by fwtty_cleanup (if a reference was - * ever taken). - */ -static void fwserial_close_port(struct tty_driver *driver, - struct fwtty_port *port) -{ - struct tty_struct *tty; - - mutex_lock(&port->port.mutex); - tty = tty_port_tty_get(&port->port); - if (tty) { - tty_vhangup(tty); - tty_kref_put(tty); - } - mutex_unlock(&port->port.mutex); - - if (driver == fwloop_driver) - tty_unregister_device(driver, loop_idx(port)); - else - tty_unregister_device(driver, port->index); -} - -/** - * fwserial_lookup - finds first fw_serial associated with card - * @card: fw_card to match - * - * NB: caller must be holding fwserial_list_mutex - */ -static struct fw_serial *fwserial_lookup(struct fw_card *card) -{ - struct fw_serial *serial; - - list_for_each_entry(serial, &fwserial_list, list) { - if (card == serial->card) - return serial; - } - - return NULL; -} - -/** - * __fwserial_lookup_rcu - finds first fw_serial associated with card - * @card: fw_card to match - * - * NB: caller must be inside rcu_read_lock() section - */ -static struct fw_serial *__fwserial_lookup_rcu(struct fw_card *card) -{ - struct fw_serial *serial; - - list_for_each_entry_rcu(serial, &fwserial_list, list) { - if (card == serial->card) - return serial; - } - - return NULL; -} - -/* - * __fwserial_peer_by_node_id - finds a peer matching the given generation + id - * - * If a matching peer could not be found for the specified generation/node id, - * this could be because: - * a) the generation has changed and one of the nodes hasn't updated yet - * b) the remote node has created its remote unit device before this - * local node has created its corresponding remote unit device - * In either case, the remote node should retry - * - * Note: caller must be in rcu_read_lock() section - */ -static struct fwtty_peer *__fwserial_peer_by_node_id(struct fw_card *card, - int generation, int id) -{ - struct fw_serial *serial; - struct fwtty_peer *peer; - - serial = __fwserial_lookup_rcu(card); - if (!serial) { - /* - * Something is very wrong - there should be a matching - * fw_serial structure for every fw_card. Maybe the remote node - * has created its remote unit device before this driver has - * been probed for any unit devices... - */ - fwtty_err(card, "unknown card (guid %016llx)\n", - (unsigned long long)card->guid); - return NULL; - } - - list_for_each_entry_rcu(peer, &serial->peer_list, list) { - int g = peer->generation; - - smp_rmb(); - if (generation == g && id == peer->node_id) - return peer; - } - - return NULL; -} - -#ifdef DEBUG -static void __dump_peer_list(struct fw_card *card) -{ - struct fw_serial *serial; - struct fwtty_peer *peer; - - serial = __fwserial_lookup_rcu(card); - if (!serial) - return; - - list_for_each_entry_rcu(peer, &serial->peer_list, list) { - int g = peer->generation; - - smp_rmb(); - fwtty_dbg(card, "peer(%d:%x) guid: %016llx\n", - g, peer->node_id, (unsigned long long)peer->guid); - } -} -#else -#define __dump_peer_list(s) -#endif - -static void fwserial_auto_connect(struct work_struct *work) -{ - struct fwtty_peer *peer = to_peer(to_delayed_work(work), connect); - int err; - - err = fwserial_connect_peer(peer); - if (err == -EAGAIN && ++peer->connect_retries < MAX_CONNECT_RETRIES) - schedule_delayed_work(&peer->connect, CONNECT_RETRY_DELAY); -} - -static void fwserial_peer_workfn(struct work_struct *work) -{ - struct fwtty_peer *peer = to_peer(work, work); - - peer->workfn(work); -} - -/** - * fwserial_add_peer - add a newly probed 'serial' unit device as a 'peer' - * @serial: aggregate representing the specific fw_card to add the peer to - * @unit: 'peer' to create and add to peer_list of serial - * - * Adds a 'peer' (ie, a local or remote 'serial' unit device) to the list of - * peers for a specific fw_card. Optionally, auto-attach this peer to an - * available tty port. This function is called either directly or indirectly - * as a result of a 'serial' unit device being created & probed. - * - * Note: this function is serialized with fwserial_remove_peer() by the - * fwserial_list_mutex held in fwserial_probe(). - * - * A 1:1 correspondence between an fw_unit and an fwtty_peer is maintained - * via the dev_set_drvdata() for the device of the fw_unit. - */ -static int fwserial_add_peer(struct fw_serial *serial, struct fw_unit *unit) -{ - struct device *dev = &unit->device; - struct fw_device *parent = fw_parent_device(unit); - struct fwtty_peer *peer; - struct fw_csr_iterator ci; - int key, val; - int generation; - - peer = kzalloc(sizeof(*peer), GFP_KERNEL); - if (!peer) - return -ENOMEM; - - peer_set_state(peer, FWPS_NOT_ATTACHED); - - dev_set_drvdata(dev, peer); - peer->unit = unit; - peer->guid = (u64)parent->config_rom[3] << 32 | parent->config_rom[4]; - peer->speed = parent->max_speed; - peer->max_payload = min(device_max_receive(parent), - link_speed_to_max_payload(peer->speed)); - - generation = parent->generation; - smp_rmb(); - peer->node_id = parent->node_id; - smp_wmb(); - peer->generation = generation; - - /* retrieve the mgmt bus addr from the unit directory */ - fw_csr_iterator_init(&ci, unit->directory); - while (fw_csr_iterator_next(&ci, &key, &val)) { - if (key == (CSR_OFFSET | CSR_DEPENDENT_INFO)) { - peer->mgmt_addr = CSR_REGISTER_BASE + 4 * val; - break; - } - } - if (peer->mgmt_addr == 0ULL) { - /* - * No mgmt address effectively disables VIRT_CABLE_PLUG - - * this peer will not be able to attach to a remote - */ - peer_set_state(peer, FWPS_NO_MGMT_ADDR); - } - - spin_lock_init(&peer->lock); - peer->port = NULL; - - timer_setup(&peer->timer, fwserial_plug_timeout, 0); - INIT_WORK(&peer->work, fwserial_peer_workfn); - INIT_DELAYED_WORK(&peer->connect, fwserial_auto_connect); - - /* associate peer with specific fw_card */ - peer->serial = serial; - list_add_rcu(&peer->list, &serial->peer_list); - - fwtty_info(&peer->unit, "peer added (guid:%016llx)\n", - (unsigned long long)peer->guid); - - /* identify the local unit & virt cable to loopback port */ - if (parent->is_local) { - serial->self = peer; - if (create_loop_dev) { - struct fwtty_port *port; - - port = fwserial_claim_port(peer, num_ttys); - if (!IS_ERR(port)) { - struct virt_plug_params params; - - spin_lock_bh(&peer->lock); - peer->port = port; - fill_plug_params(¶ms, port); - fwserial_virt_plug_complete(peer, ¶ms); - spin_unlock_bh(&peer->lock); - - fwtty_write_port_status(port); - } - } - - } else if (auto_connect) { - /* auto-attach to remote units only (if policy allows) */ - schedule_delayed_work(&peer->connect, 1); - } - - return 0; -} - -/* - * fwserial_remove_peer - remove a 'serial' unit device as a 'peer' - * - * Remove a 'peer' from its list of peers. This function is only - * called by fwserial_remove() on bus removal of the unit device. - * - * Note: this function is serialized with fwserial_add_peer() by the - * fwserial_list_mutex held in fwserial_remove(). - */ -static void fwserial_remove_peer(struct fwtty_peer *peer) -{ - struct fwtty_port *port; - - spin_lock_bh(&peer->lock); - peer_set_state(peer, FWPS_GONE); - spin_unlock_bh(&peer->lock); - - cancel_delayed_work_sync(&peer->connect); - cancel_work_sync(&peer->work); - - spin_lock_bh(&peer->lock); - /* if this unit is the local unit, clear link */ - if (peer == peer->serial->self) - peer->serial->self = NULL; - - /* cancel the request timeout timer (if running) */ - del_timer(&peer->timer); - - port = peer->port; - peer->port = NULL; - - list_del_rcu(&peer->list); - - fwtty_info(&peer->unit, "peer removed (guid:%016llx)\n", - (unsigned long long)peer->guid); - - spin_unlock_bh(&peer->lock); - - if (port) - fwserial_release_port(port, true); - - synchronize_rcu(); - kfree(peer); -} - -/** - * fwserial_create - init everything to create TTYs for a specific fw_card - * @unit: fw_unit for first 'serial' unit device probed for this fw_card - * - * This function inits the aggregate structure (an fw_serial instance) - * used to manage the TTY ports registered by a specific fw_card. Also, the - * unit device is added as the first 'peer'. - * - * This unit device may represent a local unit device (as specified by the - * config ROM unit directory) or it may represent a remote unit device - * (as specified by the reading of the remote node's config ROM). - * - * Returns 0 to indicate "ownership" of the unit device, or a negative errno - * value to indicate which error. - */ -static int fwserial_create(struct fw_unit *unit) -{ - struct fw_device *parent = fw_parent_device(unit); - struct fw_card *card = parent->card; - struct fw_serial *serial; - struct fwtty_port *port; - struct device *tty_dev; - int i, j; - int err; - - serial = kzalloc(sizeof(*serial), GFP_KERNEL); - if (!serial) - return -ENOMEM; - - kref_init(&serial->kref); - serial->card = card; - INIT_LIST_HEAD(&serial->peer_list); - - for (i = 0; i < num_ports; ++i) { - port = kzalloc(sizeof(*port), GFP_KERNEL); - if (!port) { - err = -ENOMEM; - goto free_ports; - } - tty_port_init(&port->port); - port->index = FWTTY_INVALID_INDEX; - port->port.ops = &fwtty_port_ops; - port->serial = serial; - tty_buffer_set_limit(&port->port, 128 * 1024); - - spin_lock_init(&port->lock); - INIT_DELAYED_WORK(&port->drain, fwtty_drain_tx); - INIT_DELAYED_WORK(&port->emit_breaks, fwtty_emit_breaks); - INIT_WORK(&port->hangup, fwtty_do_hangup); - init_waitqueue_head(&port->wait_tx); - port->max_payload = link_speed_to_max_payload(SCODE_100); - dma_fifo_init(&port->tx_fifo); - - RCU_INIT_POINTER(port->peer, NULL); - serial->ports[i] = port; - - /* get unique bus addr region for port's status & recv fifo */ - port->rx_handler.length = FWTTY_PORT_RXFIFO_LEN + 4; - port->rx_handler.address_callback = fwtty_port_handler; - port->rx_handler.callback_data = port; - /* - * XXX: use custom memory region above cpu physical memory addrs - * this will ease porting to 64-bit firewire adapters - */ - err = fw_core_add_address_handler(&port->rx_handler, - &fw_high_memory_region); - if (err) { - tty_port_destroy(&port->port); - kfree(port); - goto free_ports; - } - } - /* preserve i for error cleanup */ - - err = fwtty_ports_add(serial); - if (err) { - fwtty_err(&unit, "no space in port table\n"); - goto free_ports; - } - - for (j = 0; j < num_ttys; ++j) { - tty_dev = tty_port_register_device(&serial->ports[j]->port, - fwtty_driver, - serial->ports[j]->index, - card->device); - if (IS_ERR(tty_dev)) { - err = PTR_ERR(tty_dev); - fwtty_err(&unit, "register tty device error (%d)\n", - err); - goto unregister_ttys; - } - - serial->ports[j]->device = tty_dev; - } - /* preserve j for error cleanup */ - - if (create_loop_dev) { - struct device *loop_dev; - - loop_dev = tty_port_register_device(&serial->ports[j]->port, - fwloop_driver, - loop_idx(serial->ports[j]), - card->device); - if (IS_ERR(loop_dev)) { - err = PTR_ERR(loop_dev); - fwtty_err(&unit, "create loop device failed (%d)\n", - err); - goto unregister_ttys; - } - serial->ports[j]->device = loop_dev; - serial->ports[j]->loopback = true; - } - - if (!IS_ERR_OR_NULL(fwserial_debugfs)) { - serial->debugfs = debugfs_create_dir(dev_name(&unit->device), - fwserial_debugfs); - if (!IS_ERR_OR_NULL(serial->debugfs)) { - debugfs_create_file("peers", 0444, serial->debugfs, - serial, &fwtty_peers_fops); - debugfs_create_file("stats", 0444, serial->debugfs, - serial, &fwtty_stats_fops); - } - } - - list_add_rcu(&serial->list, &fwserial_list); - - fwtty_notice(&unit, "TTY over FireWire on device %s (guid %016llx)\n", - dev_name(card->device), (unsigned long long)card->guid); - - err = fwserial_add_peer(serial, unit); - if (!err) - return 0; - - fwtty_err(&unit, "unable to add peer unit device (%d)\n", err); - - /* fall-through to error processing */ - debugfs_remove_recursive(serial->debugfs); - - list_del_rcu(&serial->list); - if (create_loop_dev) - tty_unregister_device(fwloop_driver, - loop_idx(serial->ports[j])); -unregister_ttys: - for (--j; j >= 0; --j) - tty_unregister_device(fwtty_driver, serial->ports[j]->index); - kref_put(&serial->kref, fwserial_destroy); - return err; - -free_ports: - for (--i; i >= 0; --i) { - fw_core_remove_address_handler(&serial->ports[i]->rx_handler); - tty_port_destroy(&serial->ports[i]->port); - kfree(serial->ports[i]); - } - kfree(serial); - return err; -} - -/* - * fwserial_probe: bus probe function for firewire 'serial' unit devices - * - * A 'serial' unit device is created and probed as a result of: - * - declaring a ieee1394 bus id table for 'devices' matching a fabricated - * 'serial' unit specifier id - * - adding a unit directory to the config ROM(s) for a 'serial' unit - * - * The firewire core registers unit devices by enumerating unit directories - * of a node's config ROM after reading the config ROM when a new node is - * added to the bus topology after a bus reset. - * - * The practical implications of this are: - * - this probe is called for both local and remote nodes that have a 'serial' - * unit directory in their config ROM (that matches the specifiers in - * fwserial_id_table). - * - no specific order is enforced for local vs. remote unit devices - * - * This unit driver copes with the lack of specific order in the same way the - * firewire net driver does -- each probe, for either a local or remote unit - * device, is treated as a 'peer' (has a struct fwtty_peer instance) and the - * first peer created for a given fw_card (tracked by the global fwserial_list) - * creates the underlying TTYs (aggregated in a fw_serial instance). - * - * NB: an early attempt to differentiate local & remote unit devices by creating - * peers only for remote units and fw_serial instances (with their - * associated TTY devices) only for local units was discarded. Managing - * the peer lifetimes on device removal proved too complicated. - * - * fwserial_probe/fwserial_remove are effectively serialized by the - * fwserial_list_mutex. This is necessary because the addition of the first peer - * for a given fw_card will trigger the creation of the fw_serial for that - * fw_card, which must not simultaneously contend with the removal of the - * last peer for a given fw_card triggering the destruction of the same - * fw_serial for the same fw_card. - */ -static int fwserial_probe(struct fw_unit *unit, - const struct ieee1394_device_id *id) -{ - struct fw_serial *serial; - int err; - - mutex_lock(&fwserial_list_mutex); - serial = fwserial_lookup(fw_parent_device(unit)->card); - if (!serial) - err = fwserial_create(unit); - else - err = fwserial_add_peer(serial, unit); - mutex_unlock(&fwserial_list_mutex); - return err; -} - -/* - * fwserial_remove: bus removal function for firewire 'serial' unit devices - * - * The corresponding 'peer' for this unit device is removed from the list of - * peers for the associated fw_serial (which has a 1:1 correspondence with a - * specific fw_card). If this is the last peer being removed, then trigger - * the destruction of the underlying TTYs. - */ -static void fwserial_remove(struct fw_unit *unit) -{ - struct fwtty_peer *peer = dev_get_drvdata(&unit->device); - struct fw_serial *serial = peer->serial; - int i; - - mutex_lock(&fwserial_list_mutex); - fwserial_remove_peer(peer); - - if (list_empty(&serial->peer_list)) { - /* unlink from the fwserial_list here */ - list_del_rcu(&serial->list); - - debugfs_remove_recursive(serial->debugfs); - - for (i = 0; i < num_ttys; ++i) - fwserial_close_port(fwtty_driver, serial->ports[i]); - if (create_loop_dev) - fwserial_close_port(fwloop_driver, serial->ports[i]); - kref_put(&serial->kref, fwserial_destroy); - } - mutex_unlock(&fwserial_list_mutex); -} - -/* - * fwserial_update: bus update function for 'firewire' serial unit devices - * - * Updates the new node_id and bus generation for this peer. Note that locking - * is unnecessary; but careful memory barrier usage is important to enforce the - * load and store order of generation & node_id. - * - * The fw-core orders the write of node_id before generation in the parent - * fw_device to ensure that a stale node_id cannot be used with a current - * bus generation. So the generation value must be read before the node_id. - * - * In turn, this orders the write of node_id before generation in the peer to - * also ensure a stale node_id cannot be used with a current bus generation. - */ -static void fwserial_update(struct fw_unit *unit) -{ - struct fw_device *parent = fw_parent_device(unit); - struct fwtty_peer *peer = dev_get_drvdata(&unit->device); - int generation; - - generation = parent->generation; - smp_rmb(); - peer->node_id = parent->node_id; - smp_wmb(); - peer->generation = generation; -} - -static const struct ieee1394_device_id fwserial_id_table[] = { - { - .match_flags = IEEE1394_MATCH_SPECIFIER_ID | - IEEE1394_MATCH_VERSION, - .specifier_id = LINUX_VENDOR_ID, - .version = FWSERIAL_VERSION, - }, - { } -}; - -static struct fw_driver fwserial_driver = { - .driver = { - .owner = THIS_MODULE, - .name = KBUILD_MODNAME, - .bus = &fw_bus_type, - }, - .probe = fwserial_probe, - .update = fwserial_update, - .remove = fwserial_remove, - .id_table = fwserial_id_table, -}; - -#define FW_UNIT_SPECIFIER(id) ((CSR_SPECIFIER_ID << 24) | (id)) -#define FW_UNIT_VERSION(ver) ((CSR_VERSION << 24) | (ver)) -#define FW_UNIT_ADDRESS(ofs) (((CSR_OFFSET | CSR_DEPENDENT_INFO) << 24) \ - | (((ofs) - CSR_REGISTER_BASE) >> 2)) -/* XXX: config ROM definitons could be improved with semi-automated offset - * and length calculation - */ -#define FW_ROM_LEN(quads) ((quads) << 16) -#define FW_ROM_DESCRIPTOR(ofs) (((CSR_LEAF | CSR_DESCRIPTOR) << 24) | (ofs)) - -struct fwserial_unit_directory_data { - u32 len_crc; - u32 unit_specifier; - u32 unit_sw_version; - u32 unit_addr_offset; - u32 desc1_ofs; - u32 desc1_len_crc; - u32 desc1_data[5]; -} __packed; - -static struct fwserial_unit_directory_data fwserial_unit_directory_data = { - .len_crc = FW_ROM_LEN(4), - .unit_specifier = FW_UNIT_SPECIFIER(LINUX_VENDOR_ID), - .unit_sw_version = FW_UNIT_VERSION(FWSERIAL_VERSION), - .desc1_ofs = FW_ROM_DESCRIPTOR(1), - .desc1_len_crc = FW_ROM_LEN(5), - .desc1_data = { - 0x00000000, /* type = text */ - 0x00000000, /* enc = ASCII, lang EN */ - 0x4c696e75, /* 'Linux TTY' */ - 0x78205454, - 0x59000000, - }, -}; - -static struct fw_descriptor fwserial_unit_directory = { - .length = sizeof(fwserial_unit_directory_data) / sizeof(u32), - .key = (CSR_DIRECTORY | CSR_UNIT) << 24, - .data = (u32 *)&fwserial_unit_directory_data, -}; - -/* - * The management address is in the unit space region but above other known - * address users (to keep wild writes from causing havoc) - */ -static const struct fw_address_region fwserial_mgmt_addr_region = { - .start = CSR_REGISTER_BASE + 0x1e0000ULL, - .end = 0x1000000000000ULL, -}; - -static struct fw_address_handler fwserial_mgmt_addr_handler; - -/** - * fwserial_handle_plug_req - handle VIRT_CABLE_PLUG request work - * @work: ptr to peer->work - * - * Attempts to complete the VIRT_CABLE_PLUG handshake sequence for this peer. - * - * This checks for a collided request-- ie, that a VIRT_CABLE_PLUG request was - * already sent to this peer. If so, the collision is resolved by comparing - * guid values; the loser sends the plug response. - * - * Note: if an error prevents a response, don't do anything -- the - * remote will timeout its request. - */ -static void fwserial_handle_plug_req(struct work_struct *work) -{ - struct fwtty_peer *peer = to_peer(work, work); - struct virt_plug_params *plug_req = &peer->work_params.plug_req; - struct fwtty_port *port; - struct fwserial_mgmt_pkt *pkt; - int rcode; - - pkt = kmalloc(sizeof(*pkt), GFP_KERNEL); - if (!pkt) - return; - - port = fwserial_find_port(peer); - - spin_lock_bh(&peer->lock); - - switch (peer->state) { - case FWPS_NOT_ATTACHED: - if (!port) { - fwtty_err(&peer->unit, "no more ports avail\n"); - fill_plug_rsp_nack(pkt); - } else { - peer->port = port; - fill_plug_rsp_ok(pkt, peer->port); - peer_set_state(peer, FWPS_PLUG_RESPONDING); - /* don't release claimed port */ - port = NULL; - } - break; - - case FWPS_PLUG_PENDING: - if (peer->serial->card->guid > peer->guid) - goto cleanup; - - /* We lost - hijack the already-claimed port and send ok */ - del_timer(&peer->timer); - fill_plug_rsp_ok(pkt, peer->port); - peer_set_state(peer, FWPS_PLUG_RESPONDING); - break; - - default: - fill_plug_rsp_nack(pkt); - } - - spin_unlock_bh(&peer->lock); - if (port) - fwserial_release_port(port, false); - - rcode = fwserial_send_mgmt_sync(peer, pkt); - - spin_lock_bh(&peer->lock); - if (peer->state == FWPS_PLUG_RESPONDING) { - if (rcode == RCODE_COMPLETE) { - struct fwtty_port *tmp = peer->port; - - fwserial_virt_plug_complete(peer, plug_req); - spin_unlock_bh(&peer->lock); - - fwtty_write_port_status(tmp); - spin_lock_bh(&peer->lock); - } else { - fwtty_err(&peer->unit, "PLUG_RSP error (%d)\n", rcode); - port = peer_revert_state(peer); - } - } -cleanup: - spin_unlock_bh(&peer->lock); - if (port) - fwserial_release_port(port, false); - kfree(pkt); -} - -static void fwserial_handle_unplug_req(struct work_struct *work) -{ - struct fwtty_peer *peer = to_peer(work, work); - struct fwtty_port *port = NULL; - struct fwserial_mgmt_pkt *pkt; - int rcode; - - pkt = kmalloc(sizeof(*pkt), GFP_KERNEL); - if (!pkt) - return; - - spin_lock_bh(&peer->lock); - - switch (peer->state) { - case FWPS_ATTACHED: - fill_unplug_rsp_ok(pkt); - peer_set_state(peer, FWPS_UNPLUG_RESPONDING); - break; - - case FWPS_UNPLUG_PENDING: - if (peer->serial->card->guid > peer->guid) - goto cleanup; - - /* We lost - send unplug rsp */ - del_timer(&peer->timer); - fill_unplug_rsp_ok(pkt); - peer_set_state(peer, FWPS_UNPLUG_RESPONDING); - break; - - default: - fill_unplug_rsp_nack(pkt); - } - - spin_unlock_bh(&peer->lock); - - rcode = fwserial_send_mgmt_sync(peer, pkt); - - spin_lock_bh(&peer->lock); - if (peer->state == FWPS_UNPLUG_RESPONDING) { - if (rcode != RCODE_COMPLETE) - fwtty_err(&peer->unit, "UNPLUG_RSP error (%d)\n", - rcode); - port = peer_revert_state(peer); - } -cleanup: - spin_unlock_bh(&peer->lock); - if (port) - fwserial_release_port(port, true); - kfree(pkt); -} - -static int fwserial_parse_mgmt_write(struct fwtty_peer *peer, - struct fwserial_mgmt_pkt *pkt, - unsigned long long addr, - size_t len) -{ - struct fwtty_port *port = NULL; - bool reset = false; - int rcode; - - if (addr != fwserial_mgmt_addr_handler.offset || len < sizeof(pkt->hdr)) - return RCODE_ADDRESS_ERROR; - - if (len != be16_to_cpu(pkt->hdr.len) || - len != mgmt_pkt_expected_len(pkt->hdr.code)) - return RCODE_DATA_ERROR; - - spin_lock_bh(&peer->lock); - if (peer->state == FWPS_GONE) { - /* - * This should never happen - it would mean that the - * remote unit that just wrote this transaction was - * already removed from the bus -- and the removal was - * processed before we rec'd this transaction - */ - fwtty_err(&peer->unit, "peer already removed\n"); - spin_unlock_bh(&peer->lock); - return RCODE_ADDRESS_ERROR; - } - - rcode = RCODE_COMPLETE; - - fwtty_dbg(&peer->unit, "mgmt: hdr.code: %04x\n", pkt->hdr.code); - - switch (be16_to_cpu(pkt->hdr.code) & FWSC_CODE_MASK) { - case FWSC_VIRT_CABLE_PLUG: - if (work_pending(&peer->work)) { - fwtty_err(&peer->unit, "plug req: busy\n"); - rcode = RCODE_CONFLICT_ERROR; - - } else { - peer->work_params.plug_req = pkt->plug_req; - peer->workfn = fwserial_handle_plug_req; - queue_work(system_unbound_wq, &peer->work); - } - break; - - case FWSC_VIRT_CABLE_PLUG_RSP: - if (peer->state != FWPS_PLUG_PENDING) { - rcode = RCODE_CONFLICT_ERROR; - - } else if (be16_to_cpu(pkt->hdr.code) & FWSC_RSP_NACK) { - fwtty_notice(&peer->unit, "NACK plug rsp\n"); - port = peer_revert_state(peer); - - } else { - struct fwtty_port *tmp = peer->port; - - fwserial_virt_plug_complete(peer, &pkt->plug_rsp); - spin_unlock_bh(&peer->lock); - - fwtty_write_port_status(tmp); - spin_lock_bh(&peer->lock); - } - break; - - case FWSC_VIRT_CABLE_UNPLUG: - if (work_pending(&peer->work)) { - fwtty_err(&peer->unit, "unplug req: busy\n"); - rcode = RCODE_CONFLICT_ERROR; - } else { - peer->workfn = fwserial_handle_unplug_req; - queue_work(system_unbound_wq, &peer->work); - } - break; - - case FWSC_VIRT_CABLE_UNPLUG_RSP: - if (peer->state != FWPS_UNPLUG_PENDING) { - rcode = RCODE_CONFLICT_ERROR; - } else { - if (be16_to_cpu(pkt->hdr.code) & FWSC_RSP_NACK) - fwtty_notice(&peer->unit, "NACK unplug?\n"); - port = peer_revert_state(peer); - reset = true; - } - break; - - default: - fwtty_err(&peer->unit, "unknown mgmt code %d\n", - be16_to_cpu(pkt->hdr.code)); - rcode = RCODE_DATA_ERROR; - } - spin_unlock_bh(&peer->lock); - - if (port) - fwserial_release_port(port, reset); - - return rcode; -} - -/* - * fwserial_mgmt_handler: bus address handler for mgmt requests - * - * This handler is responsible for handling virtual cable requests from remotes - * for all cards. - */ -static void fwserial_mgmt_handler(struct fw_card *card, - struct fw_request *request, - int tcode, int destination, int source, - int generation, - unsigned long long addr, - void *data, size_t len, - void *callback_data) -{ - struct fwserial_mgmt_pkt *pkt = data; - struct fwtty_peer *peer; - int rcode; - - rcu_read_lock(); - peer = __fwserial_peer_by_node_id(card, generation, source); - if (!peer) { - fwtty_dbg(card, "peer(%d:%x) not found\n", generation, source); - __dump_peer_list(card); - rcode = RCODE_CONFLICT_ERROR; - - } else { - switch (tcode) { - case TCODE_WRITE_BLOCK_REQUEST: - rcode = fwserial_parse_mgmt_write(peer, pkt, addr, len); - break; - - default: - rcode = RCODE_TYPE_ERROR; - } - } - - rcu_read_unlock(); - fw_send_response(card, request, rcode); -} - -static int __init fwserial_init(void) -{ - int err, num_loops = !!(create_loop_dev); - - /* XXX: placeholder for a "firewire" debugfs node */ - fwserial_debugfs = debugfs_create_dir(KBUILD_MODNAME, NULL); - - /* num_ttys/num_ports must not be set above the static alloc avail */ - if (num_ttys + num_loops > MAX_CARD_PORTS) - num_ttys = MAX_CARD_PORTS - num_loops; - - num_ports = num_ttys + num_loops; - - fwtty_driver = tty_alloc_driver(MAX_TOTAL_PORTS, TTY_DRIVER_REAL_RAW - | TTY_DRIVER_DYNAMIC_DEV); - if (IS_ERR(fwtty_driver)) { - err = PTR_ERR(fwtty_driver); - goto remove_debugfs; - } - - fwtty_driver->driver_name = KBUILD_MODNAME; - fwtty_driver->name = tty_dev_name; - fwtty_driver->major = 0; - fwtty_driver->minor_start = 0; - fwtty_driver->type = TTY_DRIVER_TYPE_SERIAL; - fwtty_driver->subtype = SERIAL_TYPE_NORMAL; - fwtty_driver->init_termios = tty_std_termios; - fwtty_driver->init_termios.c_cflag |= CLOCAL; - tty_set_operations(fwtty_driver, &fwtty_ops); - - err = tty_register_driver(fwtty_driver); - if (err) { - pr_err("register tty driver failed (%d)\n", err); - goto put_tty; - } - - if (create_loop_dev) { - fwloop_driver = tty_alloc_driver(MAX_TOTAL_PORTS / num_ports, - TTY_DRIVER_REAL_RAW - | TTY_DRIVER_DYNAMIC_DEV); - if (IS_ERR(fwloop_driver)) { - err = PTR_ERR(fwloop_driver); - goto unregister_driver; - } - - fwloop_driver->driver_name = KBUILD_MODNAME "_loop"; - fwloop_driver->name = loop_dev_name; - fwloop_driver->major = 0; - fwloop_driver->minor_start = 0; - fwloop_driver->type = TTY_DRIVER_TYPE_SERIAL; - fwloop_driver->subtype = SERIAL_TYPE_NORMAL; - fwloop_driver->init_termios = tty_std_termios; - fwloop_driver->init_termios.c_cflag |= CLOCAL; - tty_set_operations(fwloop_driver, &fwloop_ops); - - err = tty_register_driver(fwloop_driver); - if (err) { - pr_err("register loop driver failed (%d)\n", err); - goto put_loop; - } - } - - fwtty_txn_cache = kmem_cache_create("fwtty_txn_cache", - sizeof(struct fwtty_transaction), - 0, 0, NULL); - if (!fwtty_txn_cache) { - err = -ENOMEM; - goto unregister_loop; - } - - /* - * Ideally, this address handler would be registered per local node - * (rather than the same handler for all local nodes). However, - * since the firewire core requires the config rom descriptor *before* - * the local unit device(s) are created, a single management handler - * must suffice for all local serial units. - */ - fwserial_mgmt_addr_handler.length = sizeof(struct fwserial_mgmt_pkt); - fwserial_mgmt_addr_handler.address_callback = fwserial_mgmt_handler; - - err = fw_core_add_address_handler(&fwserial_mgmt_addr_handler, - &fwserial_mgmt_addr_region); - if (err) { - pr_err("add management handler failed (%d)\n", err); - goto destroy_cache; - } - - fwserial_unit_directory_data.unit_addr_offset = - FW_UNIT_ADDRESS(fwserial_mgmt_addr_handler.offset); - err = fw_core_add_descriptor(&fwserial_unit_directory); - if (err) { - pr_err("add unit descriptor failed (%d)\n", err); - goto remove_handler; - } - - err = driver_register(&fwserial_driver.driver); - if (err) { - pr_err("register fwserial driver failed (%d)\n", err); - goto remove_descriptor; - } - - return 0; - -remove_descriptor: - fw_core_remove_descriptor(&fwserial_unit_directory); -remove_handler: - fw_core_remove_address_handler(&fwserial_mgmt_addr_handler); -destroy_cache: - kmem_cache_destroy(fwtty_txn_cache); -unregister_loop: - if (create_loop_dev) - tty_unregister_driver(fwloop_driver); -put_loop: - if (create_loop_dev) - tty_driver_kref_put(fwloop_driver); -unregister_driver: - tty_unregister_driver(fwtty_driver); -put_tty: - tty_driver_kref_put(fwtty_driver); -remove_debugfs: - debugfs_remove_recursive(fwserial_debugfs); - - return err; -} - -static void __exit fwserial_exit(void) -{ - driver_unregister(&fwserial_driver.driver); - fw_core_remove_descriptor(&fwserial_unit_directory); - fw_core_remove_address_handler(&fwserial_mgmt_addr_handler); - kmem_cache_destroy(fwtty_txn_cache); - if (create_loop_dev) { - tty_unregister_driver(fwloop_driver); - tty_driver_kref_put(fwloop_driver); - } - tty_unregister_driver(fwtty_driver); - tty_driver_kref_put(fwtty_driver); - debugfs_remove_recursive(fwserial_debugfs); -} - -module_init(fwserial_init); -module_exit(fwserial_exit); - -MODULE_AUTHOR("Peter Hurley (peter@hurleysoftware.com)"); -MODULE_DESCRIPTION("FireWire Serial TTY Driver"); -MODULE_LICENSE("GPL"); -MODULE_DEVICE_TABLE(ieee1394, fwserial_id_table); -MODULE_PARM_DESC(ttys, "Number of ttys to create for each local firewire node"); -MODULE_PARM_DESC(auto, "Auto-connect a tty to each firewire node discovered"); -MODULE_PARM_DESC(loop, "Create a loopback device, fwloop, with ttys"); diff --git a/drivers/staging/fwserial/fwserial.h b/drivers/staging/fwserial/fwserial.h deleted file mode 100644 index 1d15f183e0fad820ef8e6379ee94dfb1ad3ea7d6..0000000000000000000000000000000000000000 --- a/drivers/staging/fwserial/fwserial.h +++ /dev/null @@ -1,359 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _FIREWIRE_FWSERIAL_H -#define _FIREWIRE_FWSERIAL_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dma_fifo.h" - -#ifdef FWTTY_PROFILING -#define DISTRIBUTION_MAX_SIZE 8192 -#define DISTRIBUTION_MAX_INDEX (ilog2(DISTRIBUTION_MAX_SIZE) + 1) -static inline void fwtty_profile_data(unsigned int stat[], unsigned int val) -{ - int n = (val) ? min(ilog2(val) + 1, DISTRIBUTION_MAX_INDEX) : 0; - ++stat[n]; -} -#else -#define DISTRIBUTION_MAX_INDEX 0 -#define fwtty_profile_data(st, n) -#endif - -/* Parameters for both VIRT_CABLE_PLUG & VIRT_CABLE_PLUG_RSP mgmt codes */ -struct virt_plug_params { - __be32 status_hi; - __be32 status_lo; - __be32 fifo_hi; - __be32 fifo_lo; - __be32 fifo_len; -}; - -struct peer_work_params { - union { - struct virt_plug_params plug_req; - }; -}; - -/** - * fwtty_peer: structure representing local & remote unit devices - * @unit: unit child device of fw_device node - * @serial: back pointer to associated fw_serial aggregate - * @guid: unique 64-bit guid for this unit device - * @generation: most recent bus generation - * @node_id: most recent node_id - * @speed: link speed of peer (0 = S100, 2 = S400, ... 5 = S3200) - * @mgmt_addr: bus addr region to write mgmt packets to - * @status_addr: bus addr register to write line status to - * @fifo_addr: bus addr region to write serial output to - * @fifo_len: max length for single write to fifo_addr - * @list: link for insertion into fw_serial's peer_list - * @rcu: for deferring peer reclamation - * @lock: spinlock to synchonize changes to state & port fields - * @work: only one work item can be queued at any one time - * Note: pending work is canceled prior to removal, so this - * peer is valid for at least the lifetime of the work function - * @work_params: parameter block for work functions - * @timer: timer for resetting peer state if remote request times out - * @state: current state - * @connect: work item for auto-connecting - * @connect_retries: # of connections already attempted - * @port: associated tty_port (usable if state == FWSC_ATTACHED) - */ -struct fwtty_peer { - struct fw_unit *unit; - struct fw_serial *serial; - u64 guid; - int generation; - int node_id; - unsigned int speed; - int max_payload; - u64 mgmt_addr; - - /* these are usable only if state == FWSC_ATTACHED */ - u64 status_addr; - u64 fifo_addr; - int fifo_len; - - struct list_head list; - struct rcu_head rcu; - - spinlock_t lock; - work_func_t workfn; - struct work_struct work; - struct peer_work_params work_params; - struct timer_list timer; - int state; - struct delayed_work connect; - int connect_retries; - - struct fwtty_port *port; -}; - -#define to_peer(ptr, field) (container_of(ptr, struct fwtty_peer, field)) - -/* state values for fwtty_peer.state field */ -enum fwtty_peer_state { - FWPS_GONE, - FWPS_NOT_ATTACHED, - FWPS_ATTACHED, - FWPS_PLUG_PENDING, - FWPS_PLUG_RESPONDING, - FWPS_UNPLUG_PENDING, - FWPS_UNPLUG_RESPONDING, - - FWPS_NO_MGMT_ADDR = -1, -}; - -#define CONNECT_RETRY_DELAY HZ -#define MAX_CONNECT_RETRIES 10 - -/* must be holding peer lock for these state funclets */ -static inline void peer_set_state(struct fwtty_peer *peer, int new) -{ - peer->state = new; -} - -static inline struct fwtty_port *peer_revert_state(struct fwtty_peer *peer) -{ - struct fwtty_port *port = peer->port; - - peer->port = NULL; - peer_set_state(peer, FWPS_NOT_ATTACHED); - return port; -} - -struct fwserial_mgmt_pkt { - struct { - __be16 len; - __be16 code; - } hdr; - union { - struct virt_plug_params plug_req; - struct virt_plug_params plug_rsp; - }; -} __packed; - -/* fwserial_mgmt_packet codes */ -#define FWSC_RSP_OK 0x0000 -#define FWSC_RSP_NACK 0x8000 -#define FWSC_CODE_MASK 0x0fff - -#define FWSC_VIRT_CABLE_PLUG 1 -#define FWSC_VIRT_CABLE_UNPLUG 2 -#define FWSC_VIRT_CABLE_PLUG_RSP 3 -#define FWSC_VIRT_CABLE_UNPLUG_RSP 4 - -/* 1 min. plug timeout -- suitable for userland authorization */ -#define VIRT_CABLE_PLUG_TIMEOUT (60 * HZ) - -struct stats { - unsigned int xchars; - unsigned int dropped; - unsigned int tx_stall; - unsigned int fifo_errs; - unsigned int sent; - unsigned int lost; - unsigned int throttled; - unsigned int reads[DISTRIBUTION_MAX_INDEX + 1]; - unsigned int writes[DISTRIBUTION_MAX_INDEX + 1]; - unsigned int txns[DISTRIBUTION_MAX_INDEX + 1]; - unsigned int unthrottle[DISTRIBUTION_MAX_INDEX + 1]; -}; - -struct fwconsole_ops { - void (*notify)(int code, void *data); - void (*stats)(struct stats *stats, void *data); - void (*proc_show)(struct seq_file *m, void *data); -}; - -/* codes for console ops notify */ -#define FWCON_NOTIFY_ATTACH 1 -#define FWCON_NOTIFY_DETACH 2 - -/** - * fwtty_port: structure used to track/represent underlying tty_port - * @port: underlying tty_port - * @device: tty device - * @index: index into port_table for this particular port - * note: minor = index + minor_start assigned by tty_alloc_driver() - * @serial: back pointer to the containing fw_serial - * @rx_handler: bus address handler for unique addr region used by remotes - * to communicate with this port. Every port uses - * fwtty_port_handler() for per port transactions. - * @fwcon_ops: ops for attached fw_console (if any) - * @con_data: private data for fw_console - * @wait_tx: waitqueue for sleeping until writer/drain completes tx - * @emit_breaks: delayed work responsible for generating breaks when the - * break line status is active - * @cps : characters per second computed from the termios settings - * @break_last: timestamp in jiffies from last emit_breaks - * @hangup: work responsible for HUPing when carrier is dropped/lost - * @mstatus: loose virtualization of LSR/MSR - * bits 15..0 correspond to TIOCM_* bits - * bits 19..16 reserved for mctrl - * bit 20 OOB_TX_THROTTLE - * bits 23..21 reserved - * bits 31..24 correspond to UART_LSR_* bits - * @lock: spinlock for protecting concurrent access to fields below it - * @mctrl: loose virtualization of MCR - * bits 15..0 correspond to TIOCM_* bits - * bit 16 OOB_RX_THROTTLE - * bits 19..17 reserved - * bits 31..20 reserved for mstatus - * @drain: delayed work scheduled to ensure that writes are flushed. - * The work can race with the writer but concurrent sending is - * prevented with the IN_TX flag. Scheduled under lock to - * limit scheduling when fifo has just been drained. - * @tx_fifo: fifo used to store & block-up writes for dma to remote - * @max_payload: max bytes transmissible per dma (based on peer's max_payload) - * @status_mask: UART_LSR_* bitmask significant to rx (based on termios) - * @ignore_mask: UART_LSR_* bitmask of states to ignore (also based on termios) - * @break_ctl: if set, port is 'sending break' to remote - * @write_only: self-explanatory - * @overrun: previous rx was lost (partially or completely) - * @loopback: if set, port is in loopback mode - * @flags: atomic bit flags - * bit 0: IN_TX - gate to allow only one cpu to send from the dma fifo - * at a time. - * bit 1: STOP_TX - force tx to exit while sending - * @peer: rcu-pointer to associated fwtty_peer (if attached) - * NULL if no peer attached - * @icount: predefined statistics reported by the TIOCGICOUNT ioctl - * @stats: additional statistics reported in /proc/tty/driver/firewire_serial - */ -struct fwtty_port { - struct tty_port port; - struct device *device; - unsigned int index; - struct fw_serial *serial; - struct fw_address_handler rx_handler; - - struct fwconsole_ops *fwcon_ops; - void *con_data; - - wait_queue_head_t wait_tx; - struct delayed_work emit_breaks; - unsigned int cps; - unsigned long break_last; - - struct work_struct hangup; - - unsigned int mstatus; - - spinlock_t lock; - unsigned int mctrl; - struct delayed_work drain; - struct dma_fifo tx_fifo; - int max_payload; - unsigned int status_mask; - unsigned int ignore_mask; - unsigned int break_ctl:1, - write_only:1, - overrun:1, - loopback:1; - unsigned long flags; - - struct fwtty_peer __rcu *peer; - - struct async_icount icount; - struct stats stats; -}; - -#define to_port(ptr, field) (container_of(ptr, struct fwtty_port, field)) - -/* bit #s for flags field */ -#define IN_TX 0 -#define STOP_TX 1 - -/* bitmasks for special mctrl/mstatus bits */ -#define OOB_RX_THROTTLE 0x00010000 -#define MCTRL_RSRVD 0x000e0000 -#define OOB_TX_THROTTLE 0x00100000 -#define MSTATUS_RSRVD 0x00e00000 - -#define MCTRL_MASK (TIOCM_DTR | TIOCM_RTS | TIOCM_OUT1 | TIOCM_OUT2 | \ - TIOCM_LOOP | OOB_RX_THROTTLE | MCTRL_RSRVD) - -/* XXX even every 1/50th secs. may be unnecessarily accurate */ -/* delay in jiffies between brk emits */ -#define FREQ_BREAKS (HZ / 50) - -/* Ports are allocated in blocks of num_ports for each fw_card */ -#define MAX_CARD_PORTS CONFIG_FWTTY_MAX_CARD_PORTS -#define MAX_TOTAL_PORTS CONFIG_FWTTY_MAX_TOTAL_PORTS - -/* tuning parameters */ -#define FWTTY_PORT_TXFIFO_LEN 4096 -#define FWTTY_PORT_MAX_PEND_DMA 8 /* costs a cache line per pend */ -#define DRAIN_THRESHOLD 1024 -#define MAX_ASYNC_PAYLOAD 4096 /* ohci-defined limit */ -#define WRITER_MINIMUM 128 -/* TODO: how to set watermark to AR context size? see fwtty_rx() */ -#define HIGH_WATERMARK 32768 /* AR context is 32K */ - -/* - * Size of bus addr region above 4GB used per port as the recv addr - * - must be at least as big as the MAX_ASYNC_PAYLOAD - */ -#define FWTTY_PORT_RXFIFO_LEN MAX_ASYNC_PAYLOAD - -/** - * fw_serial: aggregate used to associate tty ports with specific fw_card - * @card: fw_card associated with this fw_serial device (1:1 association) - * @kref: reference-counted multi-port management allows delayed destroy - * @self: local unit device as 'peer'. Not valid until local unit device - * is enumerated. - * @list: link for insertion into fwserial_list - * @peer_list: list of local & remote unit devices attached to this card - * @ports: fixed array of tty_ports provided by this serial device - */ -struct fw_serial { - struct fw_card *card; - struct kref kref; - - struct dentry *debugfs; - struct fwtty_peer *self; - - struct list_head list; - struct list_head peer_list; - - struct fwtty_port *ports[MAX_CARD_PORTS]; -}; - -#define to_serial(ptr, field) (container_of(ptr, struct fw_serial, field)) - -#define TTY_DEV_NAME "fwtty" /* ttyFW was taken */ -static const char tty_dev_name[] = TTY_DEV_NAME; -static const char loop_dev_name[] = "fwloop"; - -extern struct tty_driver *fwtty_driver; - -/* - * Returns the max send async payload size in bytes based on the unit device - * link speed. Self-limiting asynchronous bandwidth (via reducing the payload) - * is not necessary and does not work, because - * 1) asynchronous traffic will absorb all available bandwidth (less that - * being used for isochronous traffic) - * 2) isochronous arbitration always wins. - */ -static inline int link_speed_to_max_payload(unsigned int speed) -{ - /* Max async payload is 4096 - see IEEE 1394-2008 tables 6-4, 16-18 */ - return min(512 << speed, 4096); -} - -#endif /* _FIREWIRE_FWSERIAL_H */ diff --git a/drivers/staging/greybus/audio_helper.c b/drivers/staging/greybus/audio_helper.c index 05e91e6bc2a0869ed21309b6f496592f00a1befd..223987616e074e7d8b972b65cd78ce9f27c9b0bc 100644 --- a/drivers/staging/greybus/audio_helper.c +++ b/drivers/staging/greybus/audio_helper.c @@ -3,7 +3,6 @@ * Greybus Audio Sound SoC helper APIs */ -#include #include #include #include @@ -116,10 +115,6 @@ int gbaudio_dapm_free_controls(struct snd_soc_dapm_context *dapm, { int i; struct snd_soc_dapm_widget *w, *tmp_w; -#ifdef CONFIG_DEBUG_FS - struct dentry *parent = dapm->debugfs_dapm; - struct dentry *debugfs_w = NULL; -#endif mutex_lock(&dapm->card->dapm_mutex); for (i = 0; i < num; i++) { @@ -139,12 +134,6 @@ int gbaudio_dapm_free_controls(struct snd_soc_dapm_context *dapm, continue; } widget++; -#ifdef CONFIG_DEBUG_FS - if (!parent) - debugfs_w = debugfs_lookup(w->name, parent); - debugfs_remove(debugfs_w); - debugfs_w = NULL; -#endif gbaudio_dapm_free_widget(w); } mutex_unlock(&dapm->card->dapm_mutex); diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index dc4ed0ff1ae277fc0041276f3340aecc6ec0059e..90ff07f2cbf76b168c503e318c20065c1bebcd50 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -480,7 +480,7 @@ static int gb_tty_break_ctl(struct tty_struct *tty, int state) } static void gb_tty_set_termios(struct tty_struct *tty, - struct ktermios *termios_old) + const struct ktermios *termios_old) { struct gb_uart_set_line_coding_request newline; struct gb_tty *gb_tty = tty->driver_data; diff --git a/drivers/staging/iio/Kconfig b/drivers/staging/iio/Kconfig index a8e970db179deb48d84c23e8523170d2eb0c2269..afd05bf3345ee1b74175d558006d0ed762269251 100644 --- a/drivers/staging/iio/Kconfig +++ b/drivers/staging/iio/Kconfig @@ -8,7 +8,6 @@ menu "IIO staging drivers" source "drivers/staging/iio/accel/Kconfig" source "drivers/staging/iio/adc/Kconfig" source "drivers/staging/iio/addac/Kconfig" -source "drivers/staging/iio/cdc/Kconfig" source "drivers/staging/iio/frequency/Kconfig" source "drivers/staging/iio/impedance-analyzer/Kconfig" source "drivers/staging/iio/meter/Kconfig" diff --git a/drivers/staging/iio/Makefile b/drivers/staging/iio/Makefile index b15904b99581843398f07626ae572c38fe785333..5ed56fe57e14cf5f2d3cfe34a51a31c1f6c334f8 100644 --- a/drivers/staging/iio/Makefile +++ b/drivers/staging/iio/Makefile @@ -6,7 +6,6 @@ obj-y += accel/ obj-y += adc/ obj-y += addac/ -obj-y += cdc/ obj-y += frequency/ obj-y += impedance-analyzer/ obj-y += meter/ diff --git a/drivers/staging/iio/cdc/Kconfig b/drivers/staging/iio/cdc/Kconfig deleted file mode 100644 index a7386bbbcb79cbd9bcb50772b6f6df810949c3c1..0000000000000000000000000000000000000000 --- a/drivers/staging/iio/cdc/Kconfig +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# CDC drivers -# -menu "Capacitance to digital converters" - -config AD7746 - tristate "Analog Devices AD7745, AD7746 AD7747 capacitive sensor driver" - depends on I2C - help - Say yes here to build support for Analog Devices capacitive sensors. - (AD7745, AD7746, AD7747) Provides direct access via sysfs. - - To compile this driver as a module, choose M here: the - module will be called ad7746. - -endmenu diff --git a/drivers/staging/iio/cdc/Makefile b/drivers/staging/iio/cdc/Makefile deleted file mode 100644 index afb7499a7090aea9e0111eb0e8afc51ebf54ce29..0000000000000000000000000000000000000000 --- a/drivers/staging/iio/cdc/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for industrial I/O CDC drivers -# - -obj-$(CONFIG_AD7746) += ad7746.o diff --git a/drivers/staging/iio/frequency/ad9832.c b/drivers/staging/iio/frequency/ad9832.c index f43464db618a339f1e48d43d2f3e98fad67b7b66..6f9eebd6c7eecc410aa79286eeb187b6ca57feff 100644 --- a/drivers/staging/iio/frequency/ad9832.c +++ b/drivers/staging/iio/frequency/ad9832.c @@ -112,10 +112,10 @@ struct ad9832_state { * transfer buffers to live in their own cache lines. */ union { - __be16 freq_data[4]____cacheline_aligned; + __be16 freq_data[4]; __be16 phase_data[2]; __be16 data; - }; + } __aligned(IIO_DMA_MINALIGN); }; static unsigned long ad9832_calc_freqreg(unsigned long mclk, unsigned long fout) diff --git a/drivers/staging/iio/frequency/ad9834.c b/drivers/staging/iio/frequency/ad9834.c index 94b131ef8a22c0942260207b8f3dac7e9d7b7b9a..2b4267a87e65e153e9b5fa55cf17d3b9e3677720 100644 --- a/drivers/staging/iio/frequency/ad9834.c +++ b/drivers/staging/iio/frequency/ad9834.c @@ -83,7 +83,7 @@ struct ad9834_state { * DMA (thus cache coherency maintenance) requires the * transfer buffers to live in their own cache lines. */ - __be16 data ____cacheline_aligned; + __be16 data __aligned(IIO_DMA_MINALIGN); __be16 freq_data[2]; }; diff --git a/drivers/staging/iio/meter/ade7854.h b/drivers/staging/iio/meter/ade7854.h index a51e6e3183d38958d14b8482abef8dcfa14dfca6..7a49f8f1016fc8ae8cfc430921aff833fa9a4caa 100644 --- a/drivers/staging/iio/meter/ade7854.h +++ b/drivers/staging/iio/meter/ade7854.h @@ -162,7 +162,7 @@ struct ade7854_state { int bits); int irq; struct mutex buf_lock; - u8 tx[ADE7854_MAX_TX] ____cacheline_aligned; + u8 tx[ADE7854_MAX_TX] __aligned(IIO_DMA_MINALIGN); u8 rx[ADE7854_MAX_RX]; }; diff --git a/drivers/staging/iio/resolver/ad2s1210.c b/drivers/staging/iio/resolver/ad2s1210.c index c0b2716d05112ade9fb66ede9467c1ddf0f9892e..e4cf42438487d2f0e622b6f9a938fe31eaa9f92b 100644 --- a/drivers/staging/iio/resolver/ad2s1210.c +++ b/drivers/staging/iio/resolver/ad2s1210.c @@ -94,8 +94,8 @@ struct ad2s1210_state { bool hysteresis; u8 resolution; enum ad2s1210_mode mode; - u8 rx[2] ____cacheline_aligned; - u8 tx[2] ____cacheline_aligned; + u8 rx[2] __aligned(IIO_DMA_MINALIGN); + u8 tx[2]; }; static const int ad2s1210_mode_vals[4][2] = { diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig index 421ce9dbf44c2faaa0e7da25a8dd94f46cd7e058..d4f03b203ae54e930418537b068a1b9fbb3864d5 100644 --- a/drivers/staging/media/Kconfig +++ b/drivers/staging/media/Kconfig @@ -22,10 +22,6 @@ if STAGING_MEDIA && MEDIA_SUPPORT # Please keep them in alphabetic order source "drivers/staging/media/atomisp/Kconfig" -source "drivers/staging/media/av7110/Kconfig" - -source "drivers/staging/media/hantro/Kconfig" - source "drivers/staging/media/imx/Kconfig" source "drivers/staging/media/ipu3/Kconfig" @@ -38,12 +34,31 @@ source "drivers/staging/media/omap4iss/Kconfig" source "drivers/staging/media/rkvdec/Kconfig" -source "drivers/staging/media/stkwebcam/Kconfig" - source "drivers/staging/media/sunxi/Kconfig" source "drivers/staging/media/tegra-video/Kconfig" -source "drivers/staging/media/zoran/Kconfig" +menuconfig STAGING_MEDIA_DEPRECATED + bool "Media staging drivers (DEPRECATED)" + default n + help + This option enables deprecated media drivers that are + scheduled for future removal from the kernel. + + If you wish to work on these drivers to prevent their removal, + then contact the linux-media@vger.kernel.org mailing list. + + If in doubt, say N here. + +if STAGING_MEDIA_DEPRECATED +source "drivers/staging/media/deprecated/cpia2/Kconfig" +source "drivers/staging/media/deprecated/fsl-viu/Kconfig" +source "drivers/staging/media/deprecated/meye/Kconfig" +source "drivers/staging/media/deprecated/saa7146/Kconfig" +source "drivers/staging/media/deprecated/stkwebcam/Kconfig" +source "drivers/staging/media/deprecated/tm6000/Kconfig" +source "drivers/staging/media/deprecated/vpfe_capture/Kconfig" +source "drivers/staging/media/deprecated/zr364xx/Kconfig" +endif endif diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile index 950e96f10aad7026e3acea446573a3733d4411b4..a387692b84f299cde0b2cee08e12f469b52d2953 100644 --- a/drivers/staging/media/Makefile +++ b/drivers/staging/media/Makefile @@ -1,14 +1,18 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_INTEL_ATOMISP) += atomisp/ +obj-$(CONFIG_VIDEO_CPIA2) += deprecated/cpia2/ obj-$(CONFIG_VIDEO_IMX_MEDIA) += imx/ obj-$(CONFIG_VIDEO_MAX96712) += max96712/ obj-$(CONFIG_VIDEO_MESON_VDEC) += meson/vdec/ +obj-$(CONFIG_VIDEO_MEYE) += deprecated/meye/ obj-$(CONFIG_VIDEO_OMAP4) += omap4iss/ obj-$(CONFIG_VIDEO_ROCKCHIP_VDEC) += rkvdec/ -obj-$(CONFIG_VIDEO_STKWEBCAM) += stkwebcam/ +obj-$(CONFIG_VIDEO_STKWEBCAM) += deprecated/stkwebcam/ obj-$(CONFIG_VIDEO_SUNXI) += sunxi/ obj-$(CONFIG_VIDEO_TEGRA) += tegra-video/ -obj-$(CONFIG_VIDEO_HANTRO) += hantro/ obj-$(CONFIG_VIDEO_IPU3_IMGU) += ipu3/ -obj-$(CONFIG_VIDEO_ZORAN) += zoran/ -obj-$(CONFIG_DVB_AV7110) += av7110/ +obj-$(CONFIG_VIDEO_TM6000) += deprecated/tm6000/ +obj-$(CONFIG_VIDEO_VIU) += deprecated/fsl-viu/ +obj-$(CONFIG_USB_ZR364XX) += deprecated/zr364xx/ +obj-y += deprecated/vpfe_capture/ +obj-y += deprecated/saa7146/ diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c index cbc8b1d9199554d10abfe1e4e9da00732684eb7b..783f1b88ebf2075d8d35e936ccce9bdb22824d0b 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c @@ -1194,7 +1194,7 @@ static const struct v4l2_subdev_ops gc0310_ops = { .sensor = &gc0310_sensor_ops, }; -static int gc0310_remove(struct i2c_client *client) +static void gc0310_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct gc0310_device *dev = to_gc0310_sensor(sd); @@ -1207,8 +1207,6 @@ static int gc0310_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); v4l2_ctrl_handler_free(&dev->ctrl_handler); kfree(dev); - - return 0; } static int gc0310_probe(struct i2c_client *client) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c index 0e6b2e6100d15a58e0d17af2e14bcc853d216374..4d5a7e335f85fa72b0760129ba1be7eb2233b8b6 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c @@ -952,7 +952,7 @@ static const struct v4l2_subdev_ops gc2235_ops = { .sensor = &gc2235_sensor_ops, }; -static int gc2235_remove(struct i2c_client *client) +static void gc2235_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct gc2235_device *dev = to_gc2235_sensor(sd); @@ -965,8 +965,6 @@ static int gc2235_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); v4l2_ctrl_handler_free(&dev->ctrl_handler); kfree(dev); - - return 0; } static int gc2235_probe(struct i2c_client *client) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c index e046489cd253b386e912a5326c01c6cf59110ca0..75d16b525294f3058c17858d55df1fd0677c1f04 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c @@ -910,7 +910,7 @@ free_flash: return err; } -static int lm3554_remove(struct i2c_client *client) +static void lm3554_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct lm3554 *flash = to_lm3554(sd); @@ -926,8 +926,6 @@ static int lm3554_remove(struct i2c_client *client) lm3554_gpio_uninit(client); kfree(flash); - - return 0; } static const struct dev_pm_ops lm3554_pm_ops = { diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c index 3c81ab73cdae07124f272b614470af0c215a11bc..a0e8e94b241256977524af322ab54d3dd98375e3 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c @@ -1713,7 +1713,7 @@ static const struct v4l2_subdev_ops mt9m114_ops = { .sensor = &mt9m114_sensor_ops, }; -static int mt9m114_remove(struct i2c_client *client) +static void mt9m114_remove(struct i2c_client *client) { struct mt9m114_device *dev; struct v4l2_subdev *sd = i2c_get_clientdata(client); @@ -1724,7 +1724,6 @@ static int mt9m114_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); v4l2_ctrl_handler_free(&dev->ctrl_handler); kfree(dev); - return 0; } static int mt9m114_probe(struct i2c_client *client) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c index 4ba99c6606814345abc1541e83f77fc9cc0add89..8f48b23be3aac6e23ac910784f4d14dd21d8230f 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c @@ -1135,7 +1135,7 @@ static const struct v4l2_subdev_ops ov2680_ops = { .sensor = &ov2680_sensor_ops, }; -static int ov2680_remove(struct i2c_client *client) +static void ov2680_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2680_device *dev = to_ov2680_sensor(sd); @@ -1148,8 +1148,6 @@ static int ov2680_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); v4l2_ctrl_handler_free(&dev->ctrl_handler); kfree(dev); - - return 0; } static int ov2680_probe(struct i2c_client *client) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c index d5d099ac1b7077065cd269518335c10ab6caf820..887b6f99f6cab75a583bde4cb825fe033b21df11 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c @@ -1090,7 +1090,7 @@ static const struct v4l2_subdev_ops ov2722_ops = { .sensor = &ov2722_sensor_ops, }; -static int ov2722_remove(struct i2c_client *client) +static void ov2722_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2722_device *dev = to_ov2722_sensor(sd); @@ -1103,8 +1103,6 @@ static int ov2722_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); kfree(dev); - - return 0; } static int __ov2722_init_ctrl_handler(struct ov2722_device *dev) diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c index 6c95f57a52e97f9c331028942d9248289160659b..c1cd631455e6930a434bf6980b60e57a57dd1458 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c +++ b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c @@ -1877,7 +1877,7 @@ static const struct v4l2_subdev_ops ov5693_ops = { .pad = &ov5693_pad_ops, }; -static int ov5693_remove(struct i2c_client *client) +static void ov5693_remove(struct i2c_client *client) { struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov5693_device *dev = to_ov5693_sensor(sd); @@ -1893,8 +1893,6 @@ static int ov5693_remove(struct i2c_client *client) media_entity_cleanup(&dev->sd.entity); v4l2_ctrl_handler_free(&dev->ctrl_handler); kfree(dev); - - return 0; } static int ov5693_probe(struct i2c_client *client) diff --git a/drivers/staging/media/av7110/TODO b/drivers/staging/media/av7110/TODO deleted file mode 100644 index 60062d8441b345a7f8710f8434f3a84cd5d1e9b0..0000000000000000000000000000000000000000 --- a/drivers/staging/media/av7110/TODO +++ /dev/null @@ -1,3 +0,0 @@ -- This driver is too old and relies on a different API. - Drop it from Kernel on a couple of versions. -- Cleanup patches for the drivers here won't be accepted. diff --git a/drivers/media/usb/cpia2/Kconfig b/drivers/staging/media/deprecated/cpia2/Kconfig similarity index 66% rename from drivers/media/usb/cpia2/Kconfig rename to drivers/staging/media/deprecated/cpia2/Kconfig index da2c6862b4a2746ff721dcdc42428f2528374d9f..ee3b25a759d4865d904ed8a3baacf1bac50e25b6 100644 --- a/drivers/media/usb/cpia2/Kconfig +++ b/drivers/staging/media/deprecated/cpia2/Kconfig @@ -1,10 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only config VIDEO_CPIA2 - tristate "CPiA2 Video For Linux" + tristate "CPiA2 Video For Linux (DEPRECATED)" depends on USB && VIDEO_DEV help This is the video4linux driver for cameras based on Vision's CPiA2 (Colour Processor Interface ASIC), such as the Digital Blue QX5 Microscope. If you have one of these cameras, say Y here + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + This driver is also available as a module (cpia2). diff --git a/drivers/media/usb/cpia2/Makefile b/drivers/staging/media/deprecated/cpia2/Makefile similarity index 100% rename from drivers/media/usb/cpia2/Makefile rename to drivers/staging/media/deprecated/cpia2/Makefile diff --git a/drivers/staging/media/deprecated/cpia2/TODO b/drivers/staging/media/deprecated/cpia2/TODO new file mode 100644 index 0000000000000000000000000000000000000000..92ac8718d164610ea9701371911bdda10d7291f7 --- /dev/null +++ b/drivers/staging/media/deprecated/cpia2/TODO @@ -0,0 +1,6 @@ +The cpia2 driver does not use the vb2 framework for streaming +video, instead it implements this in the driver. + +To prevent removal of this driver early 2023 it has to be +converted to use vb2. Contact the linux-media@vger.kernel.org +mailing list if you want to do this. diff --git a/drivers/media/usb/cpia2/cpia2.h b/drivers/staging/media/deprecated/cpia2/cpia2.h similarity index 100% rename from drivers/media/usb/cpia2/cpia2.h rename to drivers/staging/media/deprecated/cpia2/cpia2.h diff --git a/drivers/media/usb/cpia2/cpia2_core.c b/drivers/staging/media/deprecated/cpia2/cpia2_core.c similarity index 100% rename from drivers/media/usb/cpia2/cpia2_core.c rename to drivers/staging/media/deprecated/cpia2/cpia2_core.c diff --git a/drivers/media/usb/cpia2/cpia2_registers.h b/drivers/staging/media/deprecated/cpia2/cpia2_registers.h similarity index 100% rename from drivers/media/usb/cpia2/cpia2_registers.h rename to drivers/staging/media/deprecated/cpia2/cpia2_registers.h diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/staging/media/deprecated/cpia2/cpia2_usb.c similarity index 100% rename from drivers/media/usb/cpia2/cpia2_usb.c rename to drivers/staging/media/deprecated/cpia2/cpia2_usb.c diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/staging/media/deprecated/cpia2/cpia2_v4l.c similarity index 100% rename from drivers/media/usb/cpia2/cpia2_v4l.c rename to drivers/staging/media/deprecated/cpia2/cpia2_v4l.c diff --git a/drivers/staging/media/deprecated/fsl-viu/Kconfig b/drivers/staging/media/deprecated/fsl-viu/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..399892c69a188af51cb503b12ee52ccec8041c26 --- /dev/null +++ b/drivers/staging/media/deprecated/fsl-viu/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VIDEO_VIU + tristate "NXP VIU Video Driver (DEPRECATED)" + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV && (PPC_MPC512x || COMPILE_TEST) && I2C + select VIDEOBUF_DMA_CONTIG + help + Support for Freescale VIU video driver. This device captures + video data, or overlays video on DIU frame buffer. + + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + + Say Y here if you want to enable VIU device on MPC5121e Rev2+. + In doubt, say N. diff --git a/drivers/staging/media/deprecated/fsl-viu/Makefile b/drivers/staging/media/deprecated/fsl-viu/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..931ec56ad08c679289e224d50a61d279ce9405a9 --- /dev/null +++ b/drivers/staging/media/deprecated/fsl-viu/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_VIDEO_VIU) += fsl-viu.o diff --git a/drivers/staging/media/deprecated/fsl-viu/TODO b/drivers/staging/media/deprecated/fsl-viu/TODO new file mode 100644 index 0000000000000000000000000000000000000000..ecb30a4296893a9d99d7d30e354a8b7246c28a94 --- /dev/null +++ b/drivers/staging/media/deprecated/fsl-viu/TODO @@ -0,0 +1,7 @@ +This is one of the few drivers still not using the vb2 +framework, so this driver is now deprecated with the intent of +removing it altogether by the beginning of 2023. + +In order to keep this driver it has to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/platform/nxp/fsl-viu.c b/drivers/staging/media/deprecated/fsl-viu/fsl-viu.c similarity index 100% rename from drivers/media/platform/nxp/fsl-viu.c rename to drivers/staging/media/deprecated/fsl-viu/fsl-viu.c diff --git a/drivers/media/pci/meye/Kconfig b/drivers/staging/media/deprecated/meye/Kconfig similarity index 73% rename from drivers/media/pci/meye/Kconfig rename to drivers/staging/media/deprecated/meye/Kconfig index 3e69b66f1a5b3652959cf37dde39fcb4e3612c79..f135f8568c85ef0502b9e0b21a0a2db0a269c33c 100644 --- a/drivers/media/pci/meye/Kconfig +++ b/drivers/staging/media/deprecated/meye/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config VIDEO_MEYE - tristate "Sony Vaio Picturebook Motion Eye Video For Linux" + tristate "Sony Vaio Picturebook Motion Eye Video For Linux (DEPRECATED)" depends on PCI && VIDEO_DEV depends on SONY_LAPTOP depends on X86 || COMPILE_TEST @@ -12,5 +12,8 @@ config VIDEO_MEYE If you say Y or M here, you need to say Y or M to "Sony Laptop Extras" in the misc device section. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + To compile this driver as a module, choose M here: the module will be called meye. diff --git a/drivers/media/pci/meye/Makefile b/drivers/staging/media/deprecated/meye/Makefile similarity index 100% rename from drivers/media/pci/meye/Makefile rename to drivers/staging/media/deprecated/meye/Makefile diff --git a/drivers/staging/media/deprecated/meye/TODO b/drivers/staging/media/deprecated/meye/TODO new file mode 100644 index 0000000000000000000000000000000000000000..6d1d1433d5a00244e7fcdc2abd1b098953a5c69e --- /dev/null +++ b/drivers/staging/media/deprecated/meye/TODO @@ -0,0 +1,6 @@ +The meye driver does not use the vb2 framework for streaming +video, instead it implements this in the driver. + +To prevent removal of this driver early 2023 it has to be +converted to use vb2. Contact the linux-media@vger.kernel.org +mailing list if you want to do this. diff --git a/drivers/media/pci/meye/meye.c b/drivers/staging/media/deprecated/meye/meye.c similarity index 100% rename from drivers/media/pci/meye/meye.c rename to drivers/staging/media/deprecated/meye/meye.c diff --git a/drivers/media/pci/meye/meye.h b/drivers/staging/media/deprecated/meye/meye.h similarity index 100% rename from drivers/media/pci/meye/meye.h rename to drivers/staging/media/deprecated/meye/meye.h diff --git a/drivers/staging/media/deprecated/saa7146/Kconfig b/drivers/staging/media/deprecated/saa7146/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..54154da79f59a457c7a99d2953a91246f7f36b40 --- /dev/null +++ b/drivers/staging/media/deprecated/saa7146/Kconfig @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +source "drivers/staging/media/deprecated/saa7146/common/Kconfig" +source "drivers/staging/media/deprecated/saa7146/av7110/Kconfig" +source "drivers/staging/media/deprecated/saa7146/saa7146/Kconfig" +source "drivers/staging/media/deprecated/saa7146/ttpci/Kconfig" diff --git a/drivers/staging/media/deprecated/saa7146/Makefile b/drivers/staging/media/deprecated/saa7146/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..68e7aa10c639542afdcb5de77980d386b528cd86 --- /dev/null +++ b/drivers/staging/media/deprecated/saa7146/Makefile @@ -0,0 +1,2 @@ + # SPDX-License-Identifier: GPL-2.0-only +obj-y += common/ av7110/ saa7146/ ttpci/ diff --git a/drivers/staging/media/av7110/Kconfig b/drivers/staging/media/deprecated/saa7146/av7110/Kconfig similarity index 82% rename from drivers/staging/media/av7110/Kconfig rename to drivers/staging/media/deprecated/saa7146/av7110/Kconfig index 9faf9d2d40010b8e42d21ca98e7834aa5e988a6d..1571eab3192665c4f1818f331feb359b519b4506 100644 --- a/drivers/staging/media/av7110/Kconfig +++ b/drivers/staging/media/deprecated/saa7146/av7110/Kconfig @@ -5,7 +5,7 @@ config DVB_AV7110_IR default DVB_AV7110 config DVB_AV7110 - tristate "AV7110 cards" + tristate "AV7110 cards (DEPRECATED)" depends on DVB_CORE && PCI && I2C select TTPCI_EEPROM select VIDEO_SAA7146_VV @@ -35,10 +35,13 @@ config DVB_AV7110 kernel image by adding the filename to the EXTRA_FIRMWARE configuration option string. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a card and want to use it. config DVB_AV7110_OSD - bool "AV7110 OSD support" + bool "AV7110 OSD support (DEPRECATED)" depends on DVB_AV7110 default y if DVB_AV7110=y || DVB_AV7110=m help @@ -49,10 +52,13 @@ config DVB_AV7110_OSD Anyway, some popular DVB software like VDR uses this OSD to render its menus, so say Y if you want to use this software. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + All other people say N. config DVB_BUDGET_PATCH - tristate "AV7110 cards with Budget Patch" + tristate "AV7110 cards with Budget Patch (DEPRECATED)" depends on DVB_BUDGET_CORE && I2C depends on DVB_AV7110 select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT @@ -68,6 +74,9 @@ config DVB_BUDGET_PATCH standard AV7110 driver prior to loading this driver. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a card and want to use it. To compile this driver as a module, choose M here: the @@ -80,7 +89,7 @@ if DVB_AV7110 # it if we drop support for AV7110, as no other driver will use it. config DVB_SP8870 - tristate "Spase sp8870 based" + tristate "Spase sp8870 based (DEPRECATED)" depends on DVB_CORE && I2C default m if !MEDIA_SUBDRV_AUTOSELECT help @@ -91,4 +100,7 @@ config DVB_SP8870 download/extract it, and then copy it to /usr/lib/hotplug/firmware or /lib/firmware (depending on configuration of firmware hotplug). + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + endif diff --git a/drivers/staging/media/av7110/Makefile b/drivers/staging/media/deprecated/saa7146/av7110/Makefile similarity index 78% rename from drivers/staging/media/av7110/Makefile rename to drivers/staging/media/deprecated/saa7146/av7110/Makefile index 307b267598ea67f5c17408031c21fe3c13370638..c04cd0a59109e37bc9e47de98eb78dbf11a87f0b 100644 --- a/drivers/staging/media/av7110/Makefile +++ b/drivers/staging/media/deprecated/saa7146/av7110/Makefile @@ -18,5 +18,6 @@ obj-$(CONFIG_DVB_SP8870) += sp8870.o ccflags-y += -I $(srctree)/drivers/media/dvb-frontends ccflags-y += -I $(srctree)/drivers/media/tuners -ccflags-y += -I $(srctree)/drivers/media/pci/ttpci ccflags-y += -I $(srctree)/drivers/media/common +ccflags-y += -I $(srctree)/drivers/staging/media/deprecated/saa7146/ttpci +ccflags-y += -I $(srctree)/drivers/staging/media/deprecated/saa7146/common diff --git a/drivers/staging/media/deprecated/saa7146/av7110/TODO b/drivers/staging/media/deprecated/saa7146/av7110/TODO new file mode 100644 index 0000000000000000000000000000000000000000..38817e04bb678481bda9fb228a8ae3bf5fed2bae --- /dev/null +++ b/drivers/staging/media/deprecated/saa7146/av7110/TODO @@ -0,0 +1,9 @@ +- This driver is too old and relies on a different API. + Drop it from Kernel on a couple of versions. +- Cleanup patches for the drivers here won't be accepted. + +These drivers are now deprecated with the intent of +removing them altogether by the beginning of 2023. + +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/staging/media/av7110/audio-bilingual-channel-select.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-bilingual-channel-select.rst similarity index 100% rename from drivers/staging/media/av7110/audio-bilingual-channel-select.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-bilingual-channel-select.rst diff --git a/drivers/staging/media/av7110/audio-channel-select.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-channel-select.rst similarity index 100% rename from drivers/staging/media/av7110/audio-channel-select.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-channel-select.rst diff --git a/drivers/staging/media/av7110/audio-clear-buffer.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-clear-buffer.rst similarity index 100% rename from drivers/staging/media/av7110/audio-clear-buffer.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-clear-buffer.rst diff --git a/drivers/staging/media/av7110/audio-continue.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-continue.rst similarity index 100% rename from drivers/staging/media/av7110/audio-continue.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-continue.rst diff --git a/drivers/staging/media/av7110/audio-fclose.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-fclose.rst similarity index 100% rename from drivers/staging/media/av7110/audio-fclose.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-fclose.rst diff --git a/drivers/staging/media/av7110/audio-fopen.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-fopen.rst similarity index 100% rename from drivers/staging/media/av7110/audio-fopen.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-fopen.rst diff --git a/drivers/staging/media/av7110/audio-fwrite.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-fwrite.rst similarity index 100% rename from drivers/staging/media/av7110/audio-fwrite.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-fwrite.rst diff --git a/drivers/staging/media/av7110/audio-get-capabilities.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-get-capabilities.rst similarity index 100% rename from drivers/staging/media/av7110/audio-get-capabilities.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-get-capabilities.rst diff --git a/drivers/staging/media/av7110/audio-get-status.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-get-status.rst similarity index 100% rename from drivers/staging/media/av7110/audio-get-status.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-get-status.rst diff --git a/drivers/staging/media/av7110/audio-pause.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-pause.rst similarity index 100% rename from drivers/staging/media/av7110/audio-pause.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-pause.rst diff --git a/drivers/staging/media/av7110/audio-play.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-play.rst similarity index 100% rename from drivers/staging/media/av7110/audio-play.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-play.rst diff --git a/drivers/staging/media/av7110/audio-select-source.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-select-source.rst similarity index 100% rename from drivers/staging/media/av7110/audio-select-source.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-select-source.rst diff --git a/drivers/staging/media/av7110/audio-set-av-sync.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-av-sync.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-av-sync.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-av-sync.rst diff --git a/drivers/staging/media/av7110/audio-set-bypass-mode.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-bypass-mode.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-bypass-mode.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-bypass-mode.rst diff --git a/drivers/staging/media/av7110/audio-set-id.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-id.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-id.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-id.rst diff --git a/drivers/staging/media/av7110/audio-set-mixer.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-mixer.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-mixer.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-mixer.rst diff --git a/drivers/staging/media/av7110/audio-set-mute.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-mute.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-mute.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-mute.rst diff --git a/drivers/staging/media/av7110/audio-set-streamtype.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-set-streamtype.rst similarity index 100% rename from drivers/staging/media/av7110/audio-set-streamtype.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-set-streamtype.rst diff --git a/drivers/staging/media/av7110/audio-stop.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio-stop.rst similarity index 100% rename from drivers/staging/media/av7110/audio-stop.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio-stop.rst diff --git a/drivers/staging/media/av7110/audio.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio.rst similarity index 100% rename from drivers/staging/media/av7110/audio.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio.rst diff --git a/drivers/staging/media/av7110/audio_data_types.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio_data_types.rst similarity index 100% rename from drivers/staging/media/av7110/audio_data_types.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio_data_types.rst diff --git a/drivers/staging/media/av7110/audio_function_calls.rst b/drivers/staging/media/deprecated/saa7146/av7110/audio_function_calls.rst similarity index 100% rename from drivers/staging/media/av7110/audio_function_calls.rst rename to drivers/staging/media/deprecated/saa7146/av7110/audio_function_calls.rst diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110.c similarity index 100% rename from drivers/staging/media/av7110/av7110.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110.c diff --git a/drivers/staging/media/av7110/av7110.h b/drivers/staging/media/deprecated/saa7146/av7110/av7110.h similarity index 99% rename from drivers/staging/media/av7110/av7110.h rename to drivers/staging/media/deprecated/saa7146/av7110/av7110.h index 809d938ae1667f1c4c8fad802cd3592a8697609d..9fde69b38f1ce3a2769c935e4bc5d8bb3d6f521a 100644 --- a/drivers/staging/media/av7110/av7110.h +++ b/drivers/staging/media/deprecated/saa7146/av7110/av7110.h @@ -33,7 +33,7 @@ #include "stv0297.h" #include "l64781.h" -#include +#include "saa7146_vv.h" #define ANALOG_TUNER_VES1820 1 diff --git a/drivers/staging/media/av7110/av7110_av.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_av.c similarity index 99% rename from drivers/staging/media/av7110/av7110_av.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_av.c index ab7cf496b4540e36524d8953f22278c874910090..0bf513c26b6b5e363ea9fab610ac3b43626ab0d6 100644 --- a/drivers/staging/media/av7110/av7110_av.c +++ b/drivers/staging/media/deprecated/saa7146/av7110/av7110_av.c @@ -106,7 +106,7 @@ int av7110_av_start_record(struct av7110 *av7110, int av, int ret = 0; struct dvb_demux *dvbdmx = dvbdmxfeed->demux; - dprintk(2, "av7110:%p, , dvb_demux_feed:%p\n", av7110, dvbdmxfeed); + dprintk(2, "av7110:%p, dvb_demux_feed:%p\n", av7110, dvbdmxfeed); if (av7110->playing || (av7110->rec_mode & av)) return -EBUSY; diff --git a/drivers/staging/media/av7110/av7110_av.h b/drivers/staging/media/deprecated/saa7146/av7110/av7110_av.h similarity index 100% rename from drivers/staging/media/av7110/av7110_av.h rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_av.h diff --git a/drivers/staging/media/av7110/av7110_ca.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_ca.c similarity index 100% rename from drivers/staging/media/av7110/av7110_ca.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_ca.c diff --git a/drivers/staging/media/av7110/av7110_ca.h b/drivers/staging/media/deprecated/saa7146/av7110/av7110_ca.h similarity index 100% rename from drivers/staging/media/av7110/av7110_ca.h rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_ca.h diff --git a/drivers/staging/media/av7110/av7110_hw.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_hw.c similarity index 100% rename from drivers/staging/media/av7110/av7110_hw.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_hw.c diff --git a/drivers/staging/media/av7110/av7110_hw.h b/drivers/staging/media/deprecated/saa7146/av7110/av7110_hw.h similarity index 100% rename from drivers/staging/media/av7110/av7110_hw.h rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_hw.h diff --git a/drivers/staging/media/av7110/av7110_ipack.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_ipack.c similarity index 100% rename from drivers/staging/media/av7110/av7110_ipack.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_ipack.c diff --git a/drivers/staging/media/av7110/av7110_ipack.h b/drivers/staging/media/deprecated/saa7146/av7110/av7110_ipack.h similarity index 100% rename from drivers/staging/media/av7110/av7110_ipack.h rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_ipack.h diff --git a/drivers/staging/media/av7110/av7110_ir.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_ir.c similarity index 100% rename from drivers/staging/media/av7110/av7110_ir.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_ir.c diff --git a/drivers/staging/media/av7110/av7110_v4l.c b/drivers/staging/media/deprecated/saa7146/av7110/av7110_v4l.c similarity index 100% rename from drivers/staging/media/av7110/av7110_v4l.c rename to drivers/staging/media/deprecated/saa7146/av7110/av7110_v4l.c diff --git a/drivers/staging/media/av7110/budget-patch.c b/drivers/staging/media/deprecated/saa7146/av7110/budget-patch.c similarity index 100% rename from drivers/staging/media/av7110/budget-patch.c rename to drivers/staging/media/deprecated/saa7146/av7110/budget-patch.c diff --git a/drivers/staging/media/av7110/dvb_filter.c b/drivers/staging/media/deprecated/saa7146/av7110/dvb_filter.c similarity index 100% rename from drivers/staging/media/av7110/dvb_filter.c rename to drivers/staging/media/deprecated/saa7146/av7110/dvb_filter.c diff --git a/drivers/staging/media/av7110/dvb_filter.h b/drivers/staging/media/deprecated/saa7146/av7110/dvb_filter.h similarity index 100% rename from drivers/staging/media/av7110/dvb_filter.h rename to drivers/staging/media/deprecated/saa7146/av7110/dvb_filter.h diff --git a/drivers/staging/media/av7110/sp8870.c b/drivers/staging/media/deprecated/saa7146/av7110/sp8870.c similarity index 100% rename from drivers/staging/media/av7110/sp8870.c rename to drivers/staging/media/deprecated/saa7146/av7110/sp8870.c diff --git a/drivers/staging/media/av7110/sp8870.h b/drivers/staging/media/deprecated/saa7146/av7110/sp8870.h similarity index 100% rename from drivers/staging/media/av7110/sp8870.h rename to drivers/staging/media/deprecated/saa7146/av7110/sp8870.h diff --git a/drivers/staging/media/av7110/video-clear-buffer.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-clear-buffer.rst similarity index 100% rename from drivers/staging/media/av7110/video-clear-buffer.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-clear-buffer.rst diff --git a/drivers/staging/media/av7110/video-command.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-command.rst similarity index 100% rename from drivers/staging/media/av7110/video-command.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-command.rst diff --git a/drivers/staging/media/av7110/video-continue.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-continue.rst similarity index 100% rename from drivers/staging/media/av7110/video-continue.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-continue.rst diff --git a/drivers/staging/media/av7110/video-fast-forward.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-fast-forward.rst similarity index 100% rename from drivers/staging/media/av7110/video-fast-forward.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-fast-forward.rst diff --git a/drivers/staging/media/av7110/video-fclose.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-fclose.rst similarity index 100% rename from drivers/staging/media/av7110/video-fclose.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-fclose.rst diff --git a/drivers/staging/media/av7110/video-fopen.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-fopen.rst similarity index 100% rename from drivers/staging/media/av7110/video-fopen.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-fopen.rst diff --git a/drivers/staging/media/av7110/video-freeze.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-freeze.rst similarity index 100% rename from drivers/staging/media/av7110/video-freeze.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-freeze.rst diff --git a/drivers/staging/media/av7110/video-fwrite.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-fwrite.rst similarity index 100% rename from drivers/staging/media/av7110/video-fwrite.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-fwrite.rst diff --git a/drivers/staging/media/av7110/video-get-capabilities.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-capabilities.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-capabilities.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-capabilities.rst diff --git a/drivers/staging/media/av7110/video-get-event.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-event.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-event.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-event.rst diff --git a/drivers/staging/media/av7110/video-get-frame-count.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-frame-count.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-frame-count.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-frame-count.rst diff --git a/drivers/staging/media/av7110/video-get-pts.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-pts.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-pts.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-pts.rst diff --git a/drivers/staging/media/av7110/video-get-size.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-size.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-size.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-size.rst diff --git a/drivers/staging/media/av7110/video-get-status.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-get-status.rst similarity index 100% rename from drivers/staging/media/av7110/video-get-status.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-get-status.rst diff --git a/drivers/staging/media/av7110/video-play.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-play.rst similarity index 100% rename from drivers/staging/media/av7110/video-play.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-play.rst diff --git a/drivers/staging/media/av7110/video-select-source.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-select-source.rst similarity index 100% rename from drivers/staging/media/av7110/video-select-source.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-select-source.rst diff --git a/drivers/staging/media/av7110/video-set-blank.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-set-blank.rst similarity index 100% rename from drivers/staging/media/av7110/video-set-blank.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-set-blank.rst diff --git a/drivers/staging/media/av7110/video-set-display-format.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-set-display-format.rst similarity index 100% rename from drivers/staging/media/av7110/video-set-display-format.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-set-display-format.rst diff --git a/drivers/staging/media/av7110/video-set-format.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-set-format.rst similarity index 100% rename from drivers/staging/media/av7110/video-set-format.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-set-format.rst diff --git a/drivers/staging/media/av7110/video-set-streamtype.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-set-streamtype.rst similarity index 100% rename from drivers/staging/media/av7110/video-set-streamtype.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-set-streamtype.rst diff --git a/drivers/staging/media/av7110/video-slowmotion.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-slowmotion.rst similarity index 100% rename from drivers/staging/media/av7110/video-slowmotion.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-slowmotion.rst diff --git a/drivers/staging/media/av7110/video-stillpicture.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-stillpicture.rst similarity index 100% rename from drivers/staging/media/av7110/video-stillpicture.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-stillpicture.rst diff --git a/drivers/staging/media/av7110/video-stop.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-stop.rst similarity index 100% rename from drivers/staging/media/av7110/video-stop.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-stop.rst diff --git a/drivers/staging/media/av7110/video-try-command.rst b/drivers/staging/media/deprecated/saa7146/av7110/video-try-command.rst similarity index 100% rename from drivers/staging/media/av7110/video-try-command.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video-try-command.rst diff --git a/drivers/staging/media/av7110/video.rst b/drivers/staging/media/deprecated/saa7146/av7110/video.rst similarity index 100% rename from drivers/staging/media/av7110/video.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video.rst diff --git a/drivers/staging/media/av7110/video_function_calls.rst b/drivers/staging/media/deprecated/saa7146/av7110/video_function_calls.rst similarity index 100% rename from drivers/staging/media/av7110/video_function_calls.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video_function_calls.rst diff --git a/drivers/staging/media/av7110/video_types.rst b/drivers/staging/media/deprecated/saa7146/av7110/video_types.rst similarity index 100% rename from drivers/staging/media/av7110/video_types.rst rename to drivers/staging/media/deprecated/saa7146/av7110/video_types.rst diff --git a/drivers/media/common/saa7146/Kconfig b/drivers/staging/media/deprecated/saa7146/common/Kconfig similarity index 100% rename from drivers/media/common/saa7146/Kconfig rename to drivers/staging/media/deprecated/saa7146/common/Kconfig diff --git a/drivers/media/common/saa7146/Makefile b/drivers/staging/media/deprecated/saa7146/common/Makefile similarity index 100% rename from drivers/media/common/saa7146/Makefile rename to drivers/staging/media/deprecated/saa7146/common/Makefile diff --git a/include/media/drv-intf/saa7146.h b/drivers/staging/media/deprecated/saa7146/common/saa7146.h similarity index 100% rename from include/media/drv-intf/saa7146.h rename to drivers/staging/media/deprecated/saa7146/common/saa7146.h diff --git a/drivers/media/common/saa7146/saa7146_core.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_core.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_core.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_core.c index e50fa0ff7c5d5486fce4548e9fefa40f532bac6a..da21d346b87005ed9dce63270f1bfd9a91679dea 100644 --- a/drivers/media/common/saa7146/saa7146_core.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_core.c @@ -8,8 +8,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include +#include "saa7146.h" static int saa7146_num; diff --git a/drivers/media/common/saa7146/saa7146_fops.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_fops.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_fops.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_fops.c index e9a15de6126e90d117628254dbd2da5c15947706..aa14698a9c54562cd5b78b051dac5bdd3066cac7 100644 --- a/drivers/media/common/saa7146/saa7146_fops.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_fops.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include +#include "saa7146_vv.h" /****************************************************************************/ /* resource management functions, shamelessly stolen from saa7134 driver */ diff --git a/drivers/media/common/saa7146/saa7146_hlp.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_hlp.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_hlp.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_hlp.c index 6c9946a402eefec8d2941119fd127a9b8768388f..b1222a4cfa4a82111e8eb0236fe6fa395b90b0dd 100644 --- a/drivers/media/common/saa7146/saa7146_hlp.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_hlp.c @@ -3,7 +3,7 @@ #include #include -#include +#include "saa7146_vv.h" static void calculate_output_format_register(struct saa7146_dev* saa, u32 palette, u32* clip_format) { diff --git a/drivers/media/common/saa7146/saa7146_i2c.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_i2c.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_i2c.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_i2c.c index df9ebe2a168cb05c24bc32123d108510f79cbad5..7a33fe51775a28b98df4bc60367d3c9fa44cc238 100644 --- a/drivers/media/common/saa7146/saa7146_i2c.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_i2c.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include +#include "saa7146_vv.h" static u32 saa7146_i2c_func(struct i2c_adapter *adapter) { diff --git a/drivers/media/common/saa7146/saa7146_vbi.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_vbi.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_vbi.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_vbi.c index bd442b984423feb1c8dfc6ce25760c642ed08f58..2d4a05d7bc5b8b18999e4bad0d5e8a865c306b18 100644 --- a/drivers/media/common/saa7146/saa7146_vbi.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_vbi.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include "saa7146_vv.h" static int vbi_pixel_to_capture = 720 * 2; diff --git a/drivers/media/common/saa7146/saa7146_video.c b/drivers/staging/media/deprecated/saa7146/common/saa7146_video.c similarity index 99% rename from drivers/media/common/saa7146/saa7146_video.c rename to drivers/staging/media/deprecated/saa7146/common/saa7146_video.c index 2296765079a41d503cf77e94f2a4ded828924147..4598a44231fa8ab9e37621ef3b4d3adf98f06584 100644 --- a/drivers/media/common/saa7146/saa7146_video.c +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_video.c @@ -1,10 +1,10 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include #include #include #include #include +#include "saa7146_vv.h" static int max_memory = 32; diff --git a/include/media/drv-intf/saa7146_vv.h b/drivers/staging/media/deprecated/saa7146/common/saa7146_vv.h similarity index 99% rename from include/media/drv-intf/saa7146_vv.h rename to drivers/staging/media/deprecated/saa7146/common/saa7146_vv.h index 635805fb35e876e754bb02cc17770c8784453d48..d7bd916fe3adb59d523b6d495dc996274b601a58 100644 --- a/include/media/drv-intf/saa7146_vv.h +++ b/drivers/staging/media/deprecated/saa7146/common/saa7146_vv.h @@ -5,8 +5,8 @@ #include #include #include -#include #include +#include "saa7146.h" #define MAX_SAA7146_CAPTURE_BUFFERS 32 /* arbitrary */ #define BUFFER_TIMEOUT (HZ/2) /* 0.5 seconds */ diff --git a/drivers/media/pci/saa7146/Kconfig b/drivers/staging/media/deprecated/saa7146/saa7146/Kconfig similarity index 67% rename from drivers/media/pci/saa7146/Kconfig rename to drivers/staging/media/deprecated/saa7146/saa7146/Kconfig index 3bbb68a0ed7bc9d85a8cf29f82aa80b1d5c9a72d..228e8d3f8d2b38a3173577d8385b2651a876c76d 100644 --- a/drivers/media/pci/saa7146/Kconfig +++ b/drivers/staging/media/deprecated/saa7146/saa7146/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config VIDEO_HEXIUM_GEMINI - tristate "Hexium Gemini frame grabber" + tristate "Hexium Gemini frame grabber (DEPRECATED)" depends on PCI && VIDEO_DEV && I2C select VIDEO_SAA7146_VV help @@ -8,22 +8,28 @@ config VIDEO_HEXIUM_GEMINI grabber card by Hexium. Please note that the Gemini Dual card is *not* fully supported. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + To compile this driver as a module, choose M here: the module will be called hexium_gemini. config VIDEO_HEXIUM_ORION - tristate "Hexium HV-PCI6 and Orion frame grabber" + tristate "Hexium HV-PCI6 and Orion frame grabber (DEPRECATED)" depends on PCI && VIDEO_DEV && I2C select VIDEO_SAA7146_VV help This is a video4linux driver for the Hexium HV-PCI6 and Orion frame grabber cards by Hexium. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + To compile this driver as a module, choose M here: the module will be called hexium_orion. config VIDEO_MXB - tristate "Siemens-Nixdorf 'Multimedia eXtension Board'" + tristate "Siemens-Nixdorf 'Multimedia eXtension Board' (DEPRECATED)" depends on PCI && VIDEO_DEV && I2C select VIDEO_SAA7146_VV select VIDEO_TUNER @@ -35,5 +41,8 @@ config VIDEO_MXB This is a video4linux driver for the 'Multimedia eXtension Board' TV card by Siemens-Nixdorf. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + To compile this driver as a module, choose M here: the module will be called mxb. diff --git a/drivers/media/pci/saa7146/Makefile b/drivers/staging/media/deprecated/saa7146/saa7146/Makefile similarity index 100% rename from drivers/media/pci/saa7146/Makefile rename to drivers/staging/media/deprecated/saa7146/saa7146/Makefile diff --git a/drivers/staging/media/deprecated/saa7146/saa7146/TODO b/drivers/staging/media/deprecated/saa7146/saa7146/TODO new file mode 100644 index 0000000000000000000000000000000000000000..c9ae2ec79ceafe05d6e81c4506f827a93f437ed7 --- /dev/null +++ b/drivers/staging/media/deprecated/saa7146/saa7146/TODO @@ -0,0 +1,7 @@ +The saa7146-based drivers are one of the few drivers still not using +the vb2 framework, so these drivers are now deprecated with the intent of +removing them altogether by the beginning of 2023. + +In order to keep these drivers they have to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/staging/media/deprecated/saa7146/saa7146/hexium_gemini.c similarity index 99% rename from drivers/media/pci/saa7146/hexium_gemini.c rename to drivers/staging/media/deprecated/saa7146/saa7146/hexium_gemini.c index 3947701cd6c7e76835d3d8e58321a5a75c7d9c46..124e82bd45079b0415aade24b1f598c9c8159be1 100644 --- a/drivers/media/pci/saa7146/hexium_gemini.c +++ b/drivers/staging/media/deprecated/saa7146/saa7146/hexium_gemini.c @@ -13,9 +13,9 @@ #define DEBUG_VARIABLE debug -#include #include #include +#include "../common/saa7146_vv.h" static int debug; module_param(debug, int, 0); diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/staging/media/deprecated/saa7146/saa7146/hexium_orion.c similarity index 99% rename from drivers/media/pci/saa7146/hexium_orion.c rename to drivers/staging/media/deprecated/saa7146/saa7146/hexium_orion.c index 2eb4bee16b71f28175854cc636472c903f0f4228..ebd63998ac79912a826ff8049aaf4d289b046812 100644 --- a/drivers/media/pci/saa7146/hexium_orion.c +++ b/drivers/staging/media/deprecated/saa7146/saa7146/hexium_orion.c @@ -13,9 +13,9 @@ #define DEBUG_VARIABLE debug -#include #include #include +#include "../common/saa7146_vv.h" static int debug; module_param(debug, int, 0); diff --git a/drivers/media/pci/saa7146/mxb.c b/drivers/staging/media/deprecated/saa7146/saa7146/mxb.c similarity index 99% rename from drivers/media/pci/saa7146/mxb.c rename to drivers/staging/media/deprecated/saa7146/saa7146/mxb.c index 7ded8f5b05cb32ca8adbbdc21e00dee5ec155e80..3e568f952dae5827f3590bfa8ece24c614aaa969 100644 --- a/drivers/media/pci/saa7146/mxb.c +++ b/drivers/staging/media/deprecated/saa7146/saa7146/mxb.c @@ -13,13 +13,13 @@ #define DEBUG_VARIABLE debug -#include #include #include #include #include #include +#include "../common/saa7146_vv.h" #include "tea6415c.h" #include "tea6420.h" diff --git a/drivers/media/pci/ttpci/Kconfig b/drivers/staging/media/deprecated/saa7146/ttpci/Kconfig similarity index 83% rename from drivers/media/pci/ttpci/Kconfig rename to drivers/staging/media/deprecated/saa7146/ttpci/Kconfig index 65a6832a6b9630f77d77a407c8b4a018246b3d4f..8c85ed58e9385fc440b0e2f56763f332aad04c81 100644 --- a/drivers/media/pci/ttpci/Kconfig +++ b/drivers/staging/media/deprecated/saa7146/ttpci/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config DVB_BUDGET_CORE - tristate "SAA7146 DVB cards (aka Budget, Nova-PCI)" + tristate "SAA7146 DVB cards (aka Budget, Nova-PCI) (DEPRECATED)" depends on DVB_CORE && PCI && I2C select VIDEO_SAA7146 select TTPCI_EEPROM @@ -10,7 +10,7 @@ config DVB_BUDGET_CORE MPEG2 decoder. config DVB_BUDGET - tristate "Budget cards" + tristate "Budget cards (DEPRECATED)" depends on DVB_BUDGET_CORE && I2C select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT select DVB_VES1X93 if MEDIA_SUBDRV_AUTOSELECT @@ -30,13 +30,16 @@ config DVB_BUDGET or Nova-PCI cards) without onboard MPEG2 decoder, and without analog inputs or an onboard Common Interface connector. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a card and want to use it. To compile this driver as a module, choose M here: the module will be called budget. config DVB_BUDGET_CI - tristate "Budget cards with onboard CI connector" + tristate "Budget cards with onboard CI connector (DEPRECATED)" depends on DVB_BUDGET_CORE && I2C select DVB_STV0297 if MEDIA_SUBDRV_AUTOSELECT select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT @@ -57,13 +60,16 @@ config DVB_BUDGET_CI Note: The Common Interface is not yet supported by this driver due to lack of information from the vendor. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a card and want to use it. To compile this driver as a module, choose M here: the module will be called budget-ci. config DVB_BUDGET_AV - tristate "Budget cards with analog video inputs" + tristate "Budget cards with analog video inputs (DEPRECATED)" depends on DVB_BUDGET_CORE && I2C select VIDEO_SAA7146_VV depends on VIDEO_DEV # dependencies of VIDEO_SAA7146_VV @@ -80,6 +86,9 @@ config DVB_BUDGET_AV (so called Budget- or Nova-PCI cards) without onboard MPEG2 decoder, but with one or more analog video inputs. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a card and want to use it. To compile this driver as a module, choose M here: the diff --git a/drivers/media/pci/ttpci/Makefile b/drivers/staging/media/deprecated/saa7146/ttpci/Makefile similarity index 100% rename from drivers/media/pci/ttpci/Makefile rename to drivers/staging/media/deprecated/saa7146/ttpci/Makefile diff --git a/drivers/staging/media/deprecated/saa7146/ttpci/TODO b/drivers/staging/media/deprecated/saa7146/ttpci/TODO new file mode 100644 index 0000000000000000000000000000000000000000..c9ae2ec79ceafe05d6e81c4506f827a93f437ed7 --- /dev/null +++ b/drivers/staging/media/deprecated/saa7146/ttpci/TODO @@ -0,0 +1,7 @@ +The saa7146-based drivers are one of the few drivers still not using +the vb2 framework, so these drivers are now deprecated with the intent of +removing them altogether by the beginning of 2023. + +In order to keep these drivers they have to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/pci/ttpci/budget-av.c b/drivers/staging/media/deprecated/saa7146/ttpci/budget-av.c similarity index 99% rename from drivers/media/pci/ttpci/budget-av.c rename to drivers/staging/media/deprecated/saa7146/ttpci/budget-av.c index 3cb83005cf09b6c186a081b144ceb2e3657cfab8..0c61a2dec2211afa19ac1765196b37aefc9860af 100644 --- a/drivers/media/pci/ttpci/budget-av.c +++ b/drivers/staging/media/deprecated/saa7146/ttpci/budget-av.c @@ -29,7 +29,7 @@ #include "tda1004x.h" #include "tua6100.h" #include "dvb-pll.h" -#include +#include "../common/saa7146_vv.h" #include #include #include diff --git a/drivers/media/pci/ttpci/budget-ci.c b/drivers/staging/media/deprecated/saa7146/ttpci/budget-ci.c similarity index 100% rename from drivers/media/pci/ttpci/budget-ci.c rename to drivers/staging/media/deprecated/saa7146/ttpci/budget-ci.c diff --git a/drivers/media/pci/ttpci/budget-core.c b/drivers/staging/media/deprecated/saa7146/ttpci/budget-core.c similarity index 100% rename from drivers/media/pci/ttpci/budget-core.c rename to drivers/staging/media/deprecated/saa7146/ttpci/budget-core.c diff --git a/drivers/media/pci/ttpci/budget.c b/drivers/staging/media/deprecated/saa7146/ttpci/budget.c similarity index 100% rename from drivers/media/pci/ttpci/budget.c rename to drivers/staging/media/deprecated/saa7146/ttpci/budget.c diff --git a/drivers/media/pci/ttpci/budget.h b/drivers/staging/media/deprecated/saa7146/ttpci/budget.h similarity index 98% rename from drivers/media/pci/ttpci/budget.h rename to drivers/staging/media/deprecated/saa7146/ttpci/budget.h index bd87432e6cde021b5ac33cb6b85d4e90efa66b32..82cc0df492b3975c95a15d33f42c372a2efdcb55 100644 --- a/drivers/media/pci/ttpci/budget.h +++ b/drivers/staging/media/deprecated/saa7146/ttpci/budget.h @@ -13,7 +13,7 @@ #include #include -#include +#include "../common/saa7146.h" extern int budget_debug; diff --git a/drivers/staging/media/stkwebcam/Kconfig b/drivers/staging/media/deprecated/stkwebcam/Kconfig similarity index 100% rename from drivers/staging/media/stkwebcam/Kconfig rename to drivers/staging/media/deprecated/stkwebcam/Kconfig diff --git a/drivers/staging/media/stkwebcam/Makefile b/drivers/staging/media/deprecated/stkwebcam/Makefile similarity index 100% rename from drivers/staging/media/stkwebcam/Makefile rename to drivers/staging/media/deprecated/stkwebcam/Makefile diff --git a/drivers/staging/media/stkwebcam/TODO b/drivers/staging/media/deprecated/stkwebcam/TODO similarity index 100% rename from drivers/staging/media/stkwebcam/TODO rename to drivers/staging/media/deprecated/stkwebcam/TODO diff --git a/drivers/staging/media/stkwebcam/stk-sensor.c b/drivers/staging/media/deprecated/stkwebcam/stk-sensor.c similarity index 100% rename from drivers/staging/media/stkwebcam/stk-sensor.c rename to drivers/staging/media/deprecated/stkwebcam/stk-sensor.c diff --git a/drivers/staging/media/stkwebcam/stk-webcam.c b/drivers/staging/media/deprecated/stkwebcam/stk-webcam.c similarity index 100% rename from drivers/staging/media/stkwebcam/stk-webcam.c rename to drivers/staging/media/deprecated/stkwebcam/stk-webcam.c diff --git a/drivers/staging/media/stkwebcam/stk-webcam.h b/drivers/staging/media/deprecated/stkwebcam/stk-webcam.h similarity index 100% rename from drivers/staging/media/stkwebcam/stk-webcam.h rename to drivers/staging/media/deprecated/stkwebcam/stk-webcam.h diff --git a/drivers/media/usb/tm6000/Kconfig b/drivers/staging/media/deprecated/tm6000/Kconfig similarity index 84% rename from drivers/media/usb/tm6000/Kconfig rename to drivers/staging/media/deprecated/tm6000/Kconfig index 56e977deba815cd92a073d2e5688131724dc2460..73d72e49eb2850750d5636ff4b04ad9acf9d7f47 100644 --- a/drivers/media/usb/tm6000/Kconfig +++ b/drivers/staging/media/deprecated/tm6000/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only config VIDEO_TM6000 - tristate "TV Master TM5600/6000/6010 driver" + tristate "TV Master TM5600/6000/6010 driver (DEPRECATED)" depends on VIDEO_DEV && I2C && INPUT && RC_CORE && USB select VIDEO_TUNER select MEDIA_TUNER_XC2028 @@ -13,6 +13,9 @@ config VIDEO_TM6000 only compressed MPEG data over the usb bus, so you need an external software decoder to watch TV on your computer. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + Say Y if you own such a device and want to use it. config VIDEO_TM6000_ALSA diff --git a/drivers/media/usb/tm6000/Makefile b/drivers/staging/media/deprecated/tm6000/Makefile similarity index 100% rename from drivers/media/usb/tm6000/Makefile rename to drivers/staging/media/deprecated/tm6000/Makefile diff --git a/drivers/staging/media/deprecated/tm6000/TODO b/drivers/staging/media/deprecated/tm6000/TODO new file mode 100644 index 0000000000000000000000000000000000000000..ecb30a4296893a9d99d7d30e354a8b7246c28a94 --- /dev/null +++ b/drivers/staging/media/deprecated/tm6000/TODO @@ -0,0 +1,7 @@ +This is one of the few drivers still not using the vb2 +framework, so this driver is now deprecated with the intent of +removing it altogether by the beginning of 2023. + +In order to keep this driver it has to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/usb/tm6000/tm6000-alsa.c b/drivers/staging/media/deprecated/tm6000/tm6000-alsa.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-alsa.c rename to drivers/staging/media/deprecated/tm6000/tm6000-alsa.c diff --git a/drivers/media/usb/tm6000/tm6000-cards.c b/drivers/staging/media/deprecated/tm6000/tm6000-cards.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-cards.c rename to drivers/staging/media/deprecated/tm6000/tm6000-cards.c diff --git a/drivers/media/usb/tm6000/tm6000-core.c b/drivers/staging/media/deprecated/tm6000/tm6000-core.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-core.c rename to drivers/staging/media/deprecated/tm6000/tm6000-core.c diff --git a/drivers/media/usb/tm6000/tm6000-dvb.c b/drivers/staging/media/deprecated/tm6000/tm6000-dvb.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-dvb.c rename to drivers/staging/media/deprecated/tm6000/tm6000-dvb.c diff --git a/drivers/media/usb/tm6000/tm6000-i2c.c b/drivers/staging/media/deprecated/tm6000/tm6000-i2c.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-i2c.c rename to drivers/staging/media/deprecated/tm6000/tm6000-i2c.c diff --git a/drivers/media/usb/tm6000/tm6000-input.c b/drivers/staging/media/deprecated/tm6000/tm6000-input.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-input.c rename to drivers/staging/media/deprecated/tm6000/tm6000-input.c diff --git a/drivers/media/usb/tm6000/tm6000-regs.h b/drivers/staging/media/deprecated/tm6000/tm6000-regs.h similarity index 100% rename from drivers/media/usb/tm6000/tm6000-regs.h rename to drivers/staging/media/deprecated/tm6000/tm6000-regs.h diff --git a/drivers/media/usb/tm6000/tm6000-stds.c b/drivers/staging/media/deprecated/tm6000/tm6000-stds.c similarity index 100% rename from drivers/media/usb/tm6000/tm6000-stds.c rename to drivers/staging/media/deprecated/tm6000/tm6000-stds.c diff --git a/drivers/media/usb/tm6000/tm6000-usb-isoc.h b/drivers/staging/media/deprecated/tm6000/tm6000-usb-isoc.h similarity index 100% rename from drivers/media/usb/tm6000/tm6000-usb-isoc.h rename to drivers/staging/media/deprecated/tm6000/tm6000-usb-isoc.h diff --git a/drivers/media/usb/tm6000/tm6000-video.c b/drivers/staging/media/deprecated/tm6000/tm6000-video.c similarity index 99% rename from drivers/media/usb/tm6000/tm6000-video.c rename to drivers/staging/media/deprecated/tm6000/tm6000-video.c index d855a19551f38d64912d686d61bf176ecc73a27e..e06ed21edbddaf3625a9f68945f51bb68a48bee0 100644 --- a/drivers/media/usb/tm6000/tm6000-video.c +++ b/drivers/staging/media/deprecated/tm6000/tm6000-video.c @@ -916,8 +916,6 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv, return -EINVAL; } - field = f->fmt.pix.field; - field = V4L2_FIELD_INTERLACED; tm6000_get_std_res(dev); diff --git a/drivers/media/usb/tm6000/tm6000.h b/drivers/staging/media/deprecated/tm6000/tm6000.h similarity index 100% rename from drivers/media/usb/tm6000/tm6000.h rename to drivers/staging/media/deprecated/tm6000/tm6000.h diff --git a/drivers/staging/media/deprecated/vpfe_capture/Kconfig b/drivers/staging/media/deprecated/vpfe_capture/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..10250e7e566b16f12fddc93e2296619f1b8da4f2 --- /dev/null +++ b/drivers/staging/media/deprecated/vpfe_capture/Kconfig @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0-only +config VIDEO_DM6446_CCDC + tristate "TI DM6446 CCDC video capture driver" + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV + depends on ARCH_DAVINCI || COMPILE_TEST + depends on I2C + select VIDEOBUF_DMA_CONTIG + help + Enables DaVinci CCD hw module. DaVinci CCDC hw interfaces + with decoder modules such as TVP5146 over BT656 or + sensor module such as MT9T001 over a raw interface. This + module configures the interface and CCDC/ISIF to do + video frame capture from slave decoders. + + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + + To compile this driver as a module, choose M here. There will + be two modules called vpfe_capture.ko and dm644x_ccdc.ko + +config VIDEO_DM355_CCDC + tristate "TI DM355 CCDC video capture driver" + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV + depends on ARCH_DAVINCI || COMPILE_TEST + depends on I2C + select VIDEOBUF_DMA_CONTIG + help + Enables DM355 CCD hw module. DM355 CCDC hw interfaces + with decoder modules such as TVP5146 over BT656 or + sensor module such as MT9T001 over a raw interface. This + module configures the interface and CCDC/ISIF to do + video frame capture from a slave decoders + + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + + To compile this driver as a module, choose M here. There will + be two modules called vpfe_capture.ko and dm355_ccdc.ko + +config VIDEO_DM365_ISIF + tristate "TI DM365 ISIF video capture driver" + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV + depends on ARCH_DAVINCI || COMPILE_TEST + depends on I2C + select VIDEOBUF_DMA_CONTIG + help + Enables ISIF hw module. This is the hardware module for + configuring ISIF in VPFE to capture Raw Bayer RGB data from + a image sensor or YUV data from a YUV source. + + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + + To compile this driver as a module, choose M here. There will + be two modules called vpfe_capture.ko and isif.ko diff --git a/drivers/staging/media/deprecated/vpfe_capture/Makefile b/drivers/staging/media/deprecated/vpfe_capture/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..609e8dc09ce74c01664ce7027f9f106b654b38a5 --- /dev/null +++ b/drivers/staging/media/deprecated/vpfe_capture/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_VIDEO_DM6446_CCDC) += vpfe_capture.o dm644x_ccdc.o +obj-$(CONFIG_VIDEO_DM355_CCDC) += vpfe_capture.o dm355_ccdc.o +obj-$(CONFIG_VIDEO_DM365_ISIF) += vpfe_capture.o isif.o diff --git a/drivers/staging/media/deprecated/vpfe_capture/TODO b/drivers/staging/media/deprecated/vpfe_capture/TODO new file mode 100644 index 0000000000000000000000000000000000000000..ce654d7337afa382f7e6a7ccf2f6d2c2018b3281 --- /dev/null +++ b/drivers/staging/media/deprecated/vpfe_capture/TODO @@ -0,0 +1,7 @@ +These are one of the few drivers still not using the vb2 +framework, so these drivers are now deprecated with the intent of +removing them altogether by the beginning of 2023. + +In order to keep these drivers they have to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/platform/ti/davinci/ccdc_hw_device.h b/drivers/staging/media/deprecated/vpfe_capture/ccdc_hw_device.h similarity index 100% rename from drivers/media/platform/ti/davinci/ccdc_hw_device.h rename to drivers/staging/media/deprecated/vpfe_capture/ccdc_hw_device.h diff --git a/drivers/media/platform/ti/davinci/dm355_ccdc.c b/drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc.c similarity index 99% rename from drivers/media/platform/ti/davinci/dm355_ccdc.c rename to drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc.c index 8fe55d1b972cf6cba25115f7b87e104bb6cbe1f5..da8db53e9498fe4a3f2c04cc866c917ccf756d94 100644 --- a/drivers/media/platform/ti/davinci/dm355_ccdc.c +++ b/drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc.c @@ -22,7 +22,7 @@ #include #include -#include +#include "dm355_ccdc.h" #include #include "dm355_ccdc_regs.h" diff --git a/include/media/davinci/dm355_ccdc.h b/drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc.h similarity index 100% rename from include/media/davinci/dm355_ccdc.h rename to drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc.h diff --git a/drivers/media/platform/ti/davinci/dm355_ccdc_regs.h b/drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc_regs.h similarity index 100% rename from drivers/media/platform/ti/davinci/dm355_ccdc_regs.h rename to drivers/staging/media/deprecated/vpfe_capture/dm355_ccdc_regs.h diff --git a/drivers/media/platform/ti/davinci/dm644x_ccdc.c b/drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc.c similarity index 99% rename from drivers/media/platform/ti/davinci/dm644x_ccdc.c rename to drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc.c index e4073e99914c077aa3b1bc5c58d547cf313b673b..4a93e5ad64153d732b35039f4689f6d0630af872 100644 --- a/drivers/media/platform/ti/davinci/dm644x_ccdc.c +++ b/drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc.c @@ -24,7 +24,7 @@ #include #include -#include +#include "dm644x_ccdc.h" #include #include "dm644x_ccdc_regs.h" diff --git a/include/media/davinci/dm644x_ccdc.h b/drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc.h similarity index 100% rename from include/media/davinci/dm644x_ccdc.h rename to drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc.h diff --git a/drivers/media/platform/ti/davinci/dm644x_ccdc_regs.h b/drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc_regs.h similarity index 100% rename from drivers/media/platform/ti/davinci/dm644x_ccdc_regs.h rename to drivers/staging/media/deprecated/vpfe_capture/dm644x_ccdc_regs.h diff --git a/drivers/media/platform/ti/davinci/isif.c b/drivers/staging/media/deprecated/vpfe_capture/isif.c similarity index 99% rename from drivers/media/platform/ti/davinci/isif.c rename to drivers/staging/media/deprecated/vpfe_capture/isif.c index 69e862de014fd9891b702f84e28c007277e71fdc..4059891c28240bb8002aac00fe0056bb19f12672 100644 --- a/drivers/media/platform/ti/davinci/isif.c +++ b/drivers/staging/media/deprecated/vpfe_capture/isif.c @@ -22,7 +22,7 @@ #include #include -#include +#include "isif.h" #include #include "isif_regs.h" diff --git a/include/media/davinci/isif.h b/drivers/staging/media/deprecated/vpfe_capture/isif.h similarity index 100% rename from include/media/davinci/isif.h rename to drivers/staging/media/deprecated/vpfe_capture/isif.h diff --git a/drivers/media/platform/ti/davinci/isif_regs.h b/drivers/staging/media/deprecated/vpfe_capture/isif_regs.h similarity index 100% rename from drivers/media/platform/ti/davinci/isif_regs.h rename to drivers/staging/media/deprecated/vpfe_capture/isif_regs.h diff --git a/drivers/media/platform/ti/davinci/vpfe_capture.c b/drivers/staging/media/deprecated/vpfe_capture/vpfe_capture.c similarity index 100% rename from drivers/media/platform/ti/davinci/vpfe_capture.c rename to drivers/staging/media/deprecated/vpfe_capture/vpfe_capture.c diff --git a/drivers/media/usb/zr364xx/Kconfig b/drivers/staging/media/deprecated/zr364xx/Kconfig similarity index 65% rename from drivers/media/usb/zr364xx/Kconfig rename to drivers/staging/media/deprecated/zr364xx/Kconfig index a9fb02566c4b63d024485c2536350911bc0fc1e7..ea29c9d8dca24628be1c72de36227bc07779be23 100644 --- a/drivers/media/usb/zr364xx/Kconfig +++ b/drivers/staging/media/deprecated/zr364xx/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config USB_ZR364XX - tristate "USB ZR364XX Camera support" - depends on VIDEO_DEV + tristate "USB ZR364XX Camera support (DEPRECATED)" + depends on USB && VIDEO_DEV select VIDEOBUF_GEN select VIDEOBUF_VMALLOC help @@ -10,6 +10,9 @@ config USB_ZR364XX See for more info and list of supported cameras. + This driver is deprecated and is scheduled for removal by + the beginning of 2023. See the TODO file for more information. + To compile this driver as a module, choose M here: the module will be called zr364xx. diff --git a/drivers/media/usb/zr364xx/Makefile b/drivers/staging/media/deprecated/zr364xx/Makefile similarity index 100% rename from drivers/media/usb/zr364xx/Makefile rename to drivers/staging/media/deprecated/zr364xx/Makefile diff --git a/drivers/staging/media/deprecated/zr364xx/TODO b/drivers/staging/media/deprecated/zr364xx/TODO new file mode 100644 index 0000000000000000000000000000000000000000..ecb30a4296893a9d99d7d30e354a8b7246c28a94 --- /dev/null +++ b/drivers/staging/media/deprecated/zr364xx/TODO @@ -0,0 +1,7 @@ +This is one of the few drivers still not using the vb2 +framework, so this driver is now deprecated with the intent of +removing it altogether by the beginning of 2023. + +In order to keep this driver it has to be converted to vb2. +If someone is interested in doing this work, then contact the +linux-media mailinglist (https://linuxtv.org/lists.php). diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/staging/media/deprecated/zr364xx/zr364xx.c similarity index 100% rename from drivers/media/usb/zr364xx/zr364xx.c rename to drivers/staging/media/deprecated/zr364xx/zr364xx.c diff --git a/drivers/staging/media/hantro/TODO b/drivers/staging/media/hantro/TODO deleted file mode 100644 index 8483ff482146000cecd293b648d0a25d4526d04c..0000000000000000000000000000000000000000 --- a/drivers/staging/media/hantro/TODO +++ /dev/null @@ -1,2 +0,0 @@ -The V4L controls for the HEVC CODEC are not yet part of the stable uABI, -we are keeping this driver in staging until the HEVC uABI has been merged. diff --git a/drivers/staging/media/imx/imx7-media-csi.c b/drivers/staging/media/imx/imx7-media-csi.c index a0553c24cce4b4f8b1a804cf0f1560fc32d70870..cbc66ef0eda8ea7ec14efe9b4940e70cdb6010fc 100644 --- a/drivers/staging/media/imx/imx7-media-csi.c +++ b/drivers/staging/media/imx/imx7-media-csi.c @@ -160,7 +160,7 @@ #define IMX7_CSI_VIDEO_NAME "imx-capture" /* In bytes, per queue */ -#define IMX7_CSI_VIDEO_MEM_LIMIT SZ_64M +#define IMX7_CSI_VIDEO_MEM_LIMIT SZ_512M #define IMX7_CSI_VIDEO_EOF_TIMEOUT 2000 #define IMX7_CSI_DEF_MBUS_CODE MEDIA_BUS_FMT_UYVY8_2X8 diff --git a/drivers/staging/media/max96712/max96712.c b/drivers/staging/media/max96712/max96712.c index 6b5abd958bff278fd838a3673ad3586dcb497ac1..99b333b681987ed49023bf83dbebca745ea42d4b 100644 --- a/drivers/staging/media/max96712/max96712.c +++ b/drivers/staging/media/max96712/max96712.c @@ -407,15 +407,13 @@ static int max96712_probe(struct i2c_client *client) return max96712_v4l2_register(priv); } -static int max96712_remove(struct i2c_client *client) +static void max96712_remove(struct i2c_client *client) { struct max96712_priv *priv = i2c_get_clientdata(client); v4l2_async_unregister_subdev(&priv->sd); gpiod_set_value_cansleep(priv->gpiod_pwdn, 0); - - return 0; } static const struct of_device_id max96712_of_table[] = { diff --git a/drivers/staging/media/meson/vdec/vdec_hevc.c b/drivers/staging/media/meson/vdec/vdec_hevc.c index 9530e580e57a20e25bfc043cba4abc5c79907a43..afced435c9070240885970a82c40280b4c6fc6f5 100644 --- a/drivers/staging/media/meson/vdec/vdec_hevc.c +++ b/drivers/staging/media/meson/vdec/vdec_hevc.c @@ -167,8 +167,12 @@ static int vdec_hevc_start(struct amvdec_session *sess) clk_set_rate(core->vdec_hevc_clk, 666666666); ret = clk_prepare_enable(core->vdec_hevc_clk); - if (ret) + if (ret) { + if (core->platform->revision == VDEC_REVISION_G12A || + core->platform->revision == VDEC_REVISION_SM1) + clk_disable_unprepare(core->vdec_hevcf_clk); return ret; + } if (core->platform->revision == VDEC_REVISION_SM1) regmap_update_bits(core->regmap_ao, AO_RTI_GEN_PWR_SLEEP0, diff --git a/drivers/staging/media/omap4iss/iss_video.c b/drivers/staging/media/omap4iss/iss_video.c index 9512cd3314f2b28ad0760c79977f53c76bd0f22c..842509dcfedff53915473378511b3196ae006c5f 100644 --- a/drivers/staging/media/omap4iss/iss_video.c +++ b/drivers/staging/media/omap4iss/iss_video.c @@ -843,7 +843,7 @@ iss_video_dqbuf(struct file *file, void *fh, struct v4l2_buffer *b) * processing might be possible but requires more testing. * * Stream start must be delayed until buffers are available at both the input - * and output. The pipeline must be started in the videobuf queue callback with + * and output. The pipeline must be started in the vb2 queue callback with * the buffers queue spinlock held. The modules subdev set stream operation must * not sleep. */ diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c index 4af5a831bde01df914a60a2bc40c1a26a3211483..4fc167b42cf0c74fcc7adad5b867d9d7cd8af29b 100644 --- a/drivers/staging/media/rkvdec/rkvdec-h264.c +++ b/drivers/staging/media/rkvdec/rkvdec-h264.c @@ -1162,8 +1162,8 @@ static int rkvdec_h264_run(struct rkvdec_ctx *ctx) schedule_delayed_work(&rkvdec->watchdog_work, msecs_to_jiffies(2000)); - writel(0xffffffff, rkvdec->regs + RKVDEC_REG_STRMD_ERR_EN); - writel(0xffffffff, rkvdec->regs + RKVDEC_REG_H264_ERR_E); + writel(0, rkvdec->regs + RKVDEC_REG_STRMD_ERR_EN); + writel(0, rkvdec->regs + RKVDEC_REG_H264_ERR_E); writel(1, rkvdec->regs + RKVDEC_REG_PREF_LUMA_CACHE_COMMAND); writel(1, rkvdec->regs + RKVDEC_REG_PREF_CHR_CACHE_COMMAND); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 960a0130cd62026c14f5c93c4967fe278bf22078..55c54dfdc585c66c1801e95c70a51fd3727bc539 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -448,6 +448,8 @@ static int cedrus_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; + platform_set_drvdata(pdev, dev); + dev->vfd = cedrus_video_device; dev->dev = &pdev->dev; dev->pdev = pdev; @@ -521,8 +523,6 @@ static int cedrus_probe(struct platform_device *pdev) goto err_m2m_mc; } - platform_set_drvdata(pdev, dev); - return 0; err_m2m_mc: diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.h b/drivers/staging/media/sunxi/cedrus/cedrus.h index 0841930193505b2320cf8ccd129df7137bf875a6..93a2196006f7378cc293ea152b5dac5e26aaaa46 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.h +++ b/drivers/staging/media/sunxi/cedrus/cedrus.h @@ -237,19 +237,23 @@ static inline dma_addr_t cedrus_buf_addr(struct vb2_buffer *buf, } static inline dma_addr_t cedrus_dst_buf_addr(struct cedrus_ctx *ctx, - int index, unsigned int plane) + struct vb2_buffer *buf, + unsigned int plane) { - struct vb2_buffer *buf = NULL; - struct vb2_queue *vq; - - if (index < 0) - return 0; + return buf ? cedrus_buf_addr(buf, &ctx->dst_fmt, plane) : 0; +} - vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE); - if (vq) - buf = vb2_get_buffer(vq, index); +static inline void cedrus_write_ref_buf_addr(struct cedrus_ctx *ctx, + struct vb2_queue *q, + u64 timestamp, + u32 luma_reg, + u32 chroma_reg) +{ + struct cedrus_dev *dev = ctx->dev; + struct vb2_buffer *buf = vb2_find_buffer(q, timestamp); - return buf ? cedrus_buf_addr(buf, &ctx->dst_fmt, plane) : 0; + cedrus_write(dev, luma_reg, cedrus_dst_buf_addr(ctx, buf, 0)); + cedrus_write(dev, chroma_reg, cedrus_dst_buf_addr(ctx, buf, 1)); } static inline struct cedrus_buffer * diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c index 3b6aa78a2985f0ac820b31400fa9831df4258fdb..e7f7602a5ab403c2ee31e4bc75de8466aacdcd1c 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c @@ -106,11 +106,11 @@ void cedrus_device_run(void *priv) /* Trigger decoding if setup went well, bail out otherwise. */ if (!error) { - dev->dec_ops[ctx->current_codec]->trigger(ctx); - /* Start the watchdog timer. */ schedule_delayed_work(&dev->watchdog_work, msecs_to_jiffies(2000)); + + dev->dec_ops[ctx->current_codec]->trigger(ctx); } else { v4l2_m2m_buf_done_and_job_finish(ctx->dev->m2m_dev, ctx->fh.m2m_ctx, diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c index c345e67ba9bc74a2db3d4ef695ca774b316850e6..a8b236cd38005608420ccfa80a122f6e2ac769b7 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c @@ -111,16 +111,16 @@ static void cedrus_write_frame_list(struct cedrus_ctx *ctx, for (i = 0; i < ARRAY_SIZE(decode->dpb); i++) { const struct v4l2_h264_dpb_entry *dpb = &decode->dpb[i]; struct cedrus_buffer *cedrus_buf; - int buf_idx; + struct vb2_buffer *buf; if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_VALID)) continue; - buf_idx = vb2_find_timestamp(cap_q, dpb->reference_ts, 0); - if (buf_idx < 0) + buf = vb2_find_buffer(cap_q, dpb->reference_ts); + if (!buf) continue; - cedrus_buf = vb2_to_cedrus_buffer(cap_q->bufs[buf_idx]); + cedrus_buf = vb2_to_cedrus_buffer(buf); position = cedrus_buf->codec.h264.position; used_dpbs |= BIT(position); @@ -186,7 +186,7 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx, const struct v4l2_h264_dpb_entry *dpb; const struct cedrus_buffer *cedrus_buf; unsigned int position; - int buf_idx; + struct vb2_buffer *buf; u8 dpb_idx; dpb_idx = ref_list[i].index; @@ -195,11 +195,11 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx, if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) continue; - buf_idx = vb2_find_timestamp(cap_q, dpb->reference_ts, 0); - if (buf_idx < 0) + buf = vb2_find_buffer(cap_q, dpb->reference_ts); + if (!buf) continue; - cedrus_buf = vb2_to_cedrus_buffer(cap_q->bufs[buf_idx]); + cedrus_buf = vb2_to_cedrus_buffer(buf); position = cedrus_buf->codec.h264.position; sram_array[i] |= position << 1; diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c index 687f87598f780ad4d92ba38a4a5c0f73be826b0e..4952fc17f3e6d9f23f235e47a2b9ba44b9da563d 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c @@ -102,14 +102,14 @@ static void cedrus_h265_frame_info_write_single(struct cedrus_ctx *ctx, unsigned int index, bool field_pic, u32 pic_order_cnt[], - int buffer_index) + struct vb2_buffer *buf) { struct cedrus_dev *dev = ctx->dev; - dma_addr_t dst_luma_addr = cedrus_dst_buf_addr(ctx, buffer_index, 0); - dma_addr_t dst_chroma_addr = cedrus_dst_buf_addr(ctx, buffer_index, 1); + dma_addr_t dst_luma_addr = cedrus_dst_buf_addr(ctx, buf, 0); + dma_addr_t dst_chroma_addr = cedrus_dst_buf_addr(ctx, buf, 1); dma_addr_t mv_col_buf_addr[2] = { - cedrus_h265_frame_info_mv_col_buf_addr(ctx, buffer_index, 0), - cedrus_h265_frame_info_mv_col_buf_addr(ctx, buffer_index, + cedrus_h265_frame_info_mv_col_buf_addr(ctx, buf->index, 0), + cedrus_h265_frame_info_mv_col_buf_addr(ctx, buf->index, field_pic ? 1 : 0) }; u32 offset = VE_DEC_H265_SRAM_OFFSET_FRAME_INFO + @@ -141,18 +141,18 @@ static void cedrus_h265_frame_info_write_dpb(struct cedrus_ctx *ctx, unsigned int i; for (i = 0; i < num_active_dpb_entries; i++) { - int buffer_index = vb2_find_timestamp(vq, dpb[i].timestamp, 0); + struct vb2_buffer *buf = vb2_find_buffer(vq, dpb[i].timestamp); u32 pic_order_cnt[2] = { dpb[i].pic_order_cnt_val, dpb[i].pic_order_cnt_val }; - if (buffer_index < 0) + if (!buf) continue; cedrus_h265_frame_info_write_single(ctx, i, dpb[i].field_pic, pic_order_cnt, - buffer_index); + buf); } } @@ -234,8 +234,9 @@ static void cedrus_h265_skip_bits(struct cedrus_dev *dev, int num) cedrus_write(dev, VE_DEC_H265_TRIGGER, VE_DEC_H265_TRIGGER_FLUSH_BITS | VE_DEC_H265_TRIGGER_TYPE_N_BITS(tmp)); - while (cedrus_read(dev, VE_DEC_H265_STATUS) & VE_DEC_H265_STATUS_VLD_BUSY) - udelay(1); + + if (cedrus_wait_for(dev, VE_DEC_H265_STATUS, VE_DEC_H265_STATUS_VLD_BUSY)) + dev_err_ratelimited(dev->dev, "timed out waiting to skip bits\n"); count += tmp; } @@ -751,7 +752,7 @@ static int cedrus_h265_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) cedrus_h265_frame_info_write_single(ctx, output_pic_list_index, slice_params->pic_struct != 0, pic_order_cnt, - run->dst->vb2_buf.index); + &run->dst->vb2_buf); cedrus_write(dev, VE_DEC_H265_OUTPUT_FRAME_IDX, output_pic_list_index); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c b/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c index 4cfc4a3c8a7f72f7021d1c3649e67e97e8de7d7c..c1128d2cd5559e951a5e7713cbe164450d0a77d5 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c @@ -54,13 +54,9 @@ static int cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) const struct v4l2_ctrl_mpeg2_picture *pic; const struct v4l2_ctrl_mpeg2_quantisation *quantisation; dma_addr_t src_buf_addr, dst_luma_addr, dst_chroma_addr; - dma_addr_t fwd_luma_addr, fwd_chroma_addr; - dma_addr_t bwd_luma_addr, bwd_chroma_addr; struct cedrus_dev *dev = ctx->dev; struct vb2_queue *vq; const u8 *matrix; - int forward_idx; - int backward_idx; unsigned int i; u32 reg; @@ -123,27 +119,19 @@ static int cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) cedrus_write(dev, VE_DEC_MPEG_PICBOUNDSIZE, reg); /* Forward and backward prediction reference buffers. */ - vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE); - forward_idx = vb2_find_timestamp(vq, pic->forward_ref_ts, 0); - fwd_luma_addr = cedrus_dst_buf_addr(ctx, forward_idx, 0); - fwd_chroma_addr = cedrus_dst_buf_addr(ctx, forward_idx, 1); - - cedrus_write(dev, VE_DEC_MPEG_FWD_REF_LUMA_ADDR, fwd_luma_addr); - cedrus_write(dev, VE_DEC_MPEG_FWD_REF_CHROMA_ADDR, fwd_chroma_addr); - - backward_idx = vb2_find_timestamp(vq, pic->backward_ref_ts, 0); - bwd_luma_addr = cedrus_dst_buf_addr(ctx, backward_idx, 0); - bwd_chroma_addr = cedrus_dst_buf_addr(ctx, backward_idx, 1); - - cedrus_write(dev, VE_DEC_MPEG_BWD_REF_LUMA_ADDR, bwd_luma_addr); - cedrus_write(dev, VE_DEC_MPEG_BWD_REF_CHROMA_ADDR, bwd_chroma_addr); + cedrus_write_ref_buf_addr(ctx, vq, pic->forward_ref_ts, + VE_DEC_MPEG_FWD_REF_LUMA_ADDR, + VE_DEC_MPEG_FWD_REF_CHROMA_ADDR); + cedrus_write_ref_buf_addr(ctx, vq, pic->backward_ref_ts, + VE_DEC_MPEG_BWD_REF_LUMA_ADDR, + VE_DEC_MPEG_BWD_REF_CHROMA_ADDR); /* Destination luma and chroma buffers. */ - dst_luma_addr = cedrus_dst_buf_addr(ctx, run->dst->vb2_buf.index, 0); - dst_chroma_addr = cedrus_dst_buf_addr(ctx, run->dst->vb2_buf.index, 1); + dst_luma_addr = cedrus_dst_buf_addr(ctx, &run->dst->vb2_buf, 0); + dst_chroma_addr = cedrus_dst_buf_addr(ctx, &run->dst->vb2_buf, 1); cedrus_write(dev, VE_DEC_MPEG_REC_LUMA, dst_luma_addr); cedrus_write(dev, VE_DEC_MPEG_REC_CHROMA, dst_chroma_addr); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_vp8.c b/drivers/staging/media/sunxi/cedrus/cedrus_vp8.c index 3f750d1795b6275cdcec8483bf2abb94faf809df..f7714baae37dfd056fcfbc2947f174322ed8d531 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_vp8.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_vp8.c @@ -660,7 +660,6 @@ static int cedrus_vp8_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) dma_addr_t luma_addr, chroma_addr; dma_addr_t src_buf_addr; int header_size; - int qindex; u32 reg; cedrus_engine_enable(ctx, CEDRUS_CODEC_VP8); @@ -804,43 +803,17 @@ static int cedrus_vp8_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) reg |= VE_VP8_LF_DELTA0(slice->lf.mb_mode_delta[0]); cedrus_write(dev, VE_VP8_MODE_LF_DELTA, reg); - luma_addr = cedrus_dst_buf_addr(ctx, run->dst->vb2_buf.index, 0); - chroma_addr = cedrus_dst_buf_addr(ctx, run->dst->vb2_buf.index, 1); + luma_addr = cedrus_dst_buf_addr(ctx, &run->dst->vb2_buf, 0); + chroma_addr = cedrus_dst_buf_addr(ctx, &run->dst->vb2_buf, 1); cedrus_write(dev, VE_VP8_REC_LUMA, luma_addr); cedrus_write(dev, VE_VP8_REC_CHROMA, chroma_addr); - qindex = vb2_find_timestamp(cap_q, slice->last_frame_ts, 0); - if (qindex >= 0) { - luma_addr = cedrus_dst_buf_addr(ctx, qindex, 0); - chroma_addr = cedrus_dst_buf_addr(ctx, qindex, 1); - cedrus_write(dev, VE_VP8_FWD_LUMA, luma_addr); - cedrus_write(dev, VE_VP8_FWD_CHROMA, chroma_addr); - } else { - cedrus_write(dev, VE_VP8_FWD_LUMA, 0); - cedrus_write(dev, VE_VP8_FWD_CHROMA, 0); - } - - qindex = vb2_find_timestamp(cap_q, slice->golden_frame_ts, 0); - if (qindex >= 0) { - luma_addr = cedrus_dst_buf_addr(ctx, qindex, 0); - chroma_addr = cedrus_dst_buf_addr(ctx, qindex, 1); - cedrus_write(dev, VE_VP8_BWD_LUMA, luma_addr); - cedrus_write(dev, VE_VP8_BWD_CHROMA, chroma_addr); - } else { - cedrus_write(dev, VE_VP8_BWD_LUMA, 0); - cedrus_write(dev, VE_VP8_BWD_CHROMA, 0); - } - - qindex = vb2_find_timestamp(cap_q, slice->alt_frame_ts, 0); - if (qindex >= 0) { - luma_addr = cedrus_dst_buf_addr(ctx, qindex, 0); - chroma_addr = cedrus_dst_buf_addr(ctx, qindex, 1); - cedrus_write(dev, VE_VP8_ALT_LUMA, luma_addr); - cedrus_write(dev, VE_VP8_ALT_CHROMA, chroma_addr); - } else { - cedrus_write(dev, VE_VP8_ALT_LUMA, 0); - cedrus_write(dev, VE_VP8_ALT_CHROMA, 0); - } + cedrus_write_ref_buf_addr(ctx, cap_q, slice->last_frame_ts, + VE_VP8_FWD_LUMA, VE_VP8_FWD_CHROMA); + cedrus_write_ref_buf_addr(ctx, cap_q, slice->golden_frame_ts, + VE_VP8_BWD_LUMA, VE_VP8_BWD_CHROMA); + cedrus_write_ref_buf_addr(ctx, cap_q, slice->alt_frame_ts, + VE_VP8_ALT_LUMA, VE_VP8_ALT_CHROMA); cedrus_write(dev, VE_H264_CTRL, VE_H264_CTRL_VP8 | VE_H264_CTRL_DECODE_ERR_INT | diff --git a/drivers/staging/media/zoran/TODO b/drivers/staging/media/zoran/TODO deleted file mode 100644 index 6992540d3e5370d5f903d037200787d9653b9615..0000000000000000000000000000000000000000 --- a/drivers/staging/media/zoran/TODO +++ /dev/null @@ -1,19 +0,0 @@ - -How to test the zoran driver: -- RAW capture - mplayer tv:///dev/video0 -tv driver=v4l2 - -- MJPEG capture (compression) - mplayer tv:///dev/video0 -tv driver=v4l2:outfmt=mjpeg - TODO: need two test for both Dcim path - -- MJPEG play (decompression) - ffmpeg -i test.avi -vcodec mjpeg -an -f v4l2 /dev/video0 - Note: only recent ffmpeg has the ability of sending non-raw video via v4l2 - - The original way of sending video was via mplayer vo_zr/vo_zr2, but it does not compile - anymore and is a dead end (usage of some old private ffmpeg structures). - -TODO -- fix the v4l compliance "TRY_FMT cannot handle an invalid pixelformat" -- Filter JPEG data to made output work diff --git a/drivers/staging/media/zoran/zoran_device.h b/drivers/staging/media/zoran/zoran_device.h deleted file mode 100644 index 322b04c55d410ed2b557430811c994866b5f05b0..0000000000000000000000000000000000000000 --- a/drivers/staging/media/zoran/zoran_device.h +++ /dev/null @@ -1,60 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Zoran zr36057/zr36067 PCI controller driver, for the - * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux - * Media Labs LML33/LML33R10. - * - * This part handles card-specific data and detection - * - * Copyright (C) 2000 Serguei Miridonov - */ - -#ifndef __ZORAN_DEVICE_H__ -#define __ZORAN_DEVICE_H__ - -/* general purpose I/O */ -extern void GPIO(struct zoran *zr, int bit, unsigned int value); - -/* codec (or actually: guest bus) access */ -extern int post_office_wait(struct zoran *zr); -extern int post_office_write(struct zoran *zr, unsigned int guest, unsigned int reg, unsigned int value); -extern int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg); - -extern void jpeg_codec_sleep(struct zoran *zr, int sleep); -extern int jpeg_codec_reset(struct zoran *zr); - -/* zr360x7 access to raw capture */ -extern void zr36057_overlay(struct zoran *zr, int on); -extern void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count); -extern void zr36057_set_memgrab(struct zoran *zr, int mode); -extern int wait_grab_pending(struct zoran *zr); - -/* interrupts */ -extern void print_interrupts(struct zoran *zr); -extern void clear_interrupt_counters(struct zoran *zr); -extern irqreturn_t zoran_irq(int irq, void *dev_id); - -/* JPEG codec access */ -extern void jpeg_start(struct zoran *zr); -extern void zr36057_enable_jpg(struct zoran *zr, - enum zoran_codec_mode mode); -extern void zoran_feed_stat_com(struct zoran *zr); - -/* general */ -extern void zoran_set_pci_master(struct zoran *zr, int set_master); -extern void zoran_init_hardware(struct zoran *zr); -extern void zr36057_restart(struct zoran *zr); - -extern const struct zoran_format zoran_formats[]; - -extern int v4l_bufsize; -extern int jpg_bufsize; -extern int pass_through; - -/* i2c */ -#define decoder_call(zr, o, f, args...) \ - v4l2_subdev_call(zr->decoder, o, f, ##args) -#define encoder_call(zr, o, f, args...) \ - v4l2_subdev_call(zr->encoder, o, f, ##args) - -#endif /* __ZORAN_DEVICE_H__ */ diff --git a/drivers/staging/most/i2c/i2c.c b/drivers/staging/most/i2c/i2c.c index 7042f10887bb67f9dd3e7188ff9bf2b5f288cc9f..285a071f02be18bbc0150d74acdb93a8129f3e02 100644 --- a/drivers/staging/most/i2c/i2c.c +++ b/drivers/staging/most/i2c/i2c.c @@ -340,14 +340,12 @@ static int i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) * * Unregister the i2c client device as a MOST interface */ -static int i2c_remove(struct i2c_client *client) +static void i2c_remove(struct i2c_client *client) { struct hdm_i2c *dev = i2c_get_clientdata(client); most_deregister_interface(&dev->most_iface); kfree(dev); - - return 0; } static const struct i2c_device_id i2c_id[] = { diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c index 1ad94c5060b522527a57289dfcaf7c06eec701f3..a36e36701c74365bdc4888ecf90311a0b85efd56 100644 --- a/drivers/staging/octeon/ethernet-tx.c +++ b/drivers/staging/octeon/ethernet-tx.c @@ -125,7 +125,7 @@ static void cvm_oct_free_tx_skbs(struct net_device *dev) * * Returns Always returns NETDEV_TX_OK */ -int cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev) +netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev) { union cvmx_pko_command_word0 pko_command; union cvmx_buf_ptr hw_buffer; @@ -506,7 +506,7 @@ skip_xmit: * @dev: Device info structure * Returns Always returns zero */ -int cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev) +netdev_tx_t cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev) { struct octeon_ethernet *priv = netdev_priv(dev); void *packet_buffer; diff --git a/drivers/staging/octeon/ethernet-tx.h b/drivers/staging/octeon/ethernet-tx.h index 78936e9b33b0abd53d1aec2be1bb363e3e5ed5ab..6c524668f65a5182a2220d2643d01ec0aa335380 100644 --- a/drivers/staging/octeon/ethernet-tx.h +++ b/drivers/staging/octeon/ethernet-tx.h @@ -5,8 +5,8 @@ * Copyright (c) 2003-2007 Cavium Networks */ -int cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev); -int cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev); +netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev); +netdev_tx_t cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev); int cvm_oct_transmit_qos(struct net_device *dev, void *work_queue_entry, int do_free, int qos); void cvm_oct_tx_initialize(void); diff --git a/drivers/staging/olpc_dcon/olpc_dcon.c b/drivers/staging/olpc_dcon/olpc_dcon.c index 9363c5cfe50f6773e4b5f7ae27dd968bce607bf3..4fb9b9f1079962c2ef3e946c714c85167df9109c 100644 --- a/drivers/staging/olpc_dcon/olpc_dcon.c +++ b/drivers/staging/olpc_dcon/olpc_dcon.c @@ -668,7 +668,7 @@ static int dcon_probe(struct i2c_client *client, const struct i2c_device_id *id) return rc; } -static int dcon_remove(struct i2c_client *client) +static void dcon_remove(struct i2c_client *client) { struct dcon_priv *dcon = i2c_get_clientdata(client); @@ -684,8 +684,6 @@ static int dcon_remove(struct i2c_client *client) cancel_work_sync(&dcon->switch_source); kfree(dcon); - - return 0; } #ifdef CONFIG_PM diff --git a/drivers/staging/pi433/pi433_if.c b/drivers/staging/pi433/pi433_if.c index df02335fdbab8f243a59088e50006e338e9c1403..d4e06a3929f3d8e236e1c024108f4f0015f1c087 100644 --- a/drivers/staging/pi433/pi433_if.c +++ b/drivers/staging/pi433/pi433_if.c @@ -1149,19 +1149,7 @@ out_unlock: return ret; } - -static int pi433_debugfs_regs_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, pi433_debugfs_regs_show, inode->i_private); -} - -static const struct file_operations debugfs_fops = { - .llseek = seq_lseek, - .open = pi433_debugfs_regs_open, - .owner = THIS_MODULE, - .read = seq_read, - .release = single_release -}; +DEFINE_SHOW_ATTRIBUTE(pi433_debugfs_regs); /*-------------------------------------------------------------------------*/ @@ -1320,7 +1308,7 @@ static int pi433_probe(struct spi_device *spi) entry = debugfs_create_dir(dev_name(device->dev), debugfs_lookup(KBUILD_MODNAME, NULL)); - debugfs_create_file("regs", 0400, entry, device, &debugfs_fops); + debugfs_create_file("regs", 0400, entry, device, &pi433_debugfs_regs_fops); return 0; diff --git a/drivers/staging/pi433/rf69.c b/drivers/staging/pi433/rf69.c index 659c8c1b38fda1ed33661b7797db17741eabf2ec..8c7fab6a46bb291399948ffb129f69933e0c2aa5 100644 --- a/drivers/staging/pi433/rf69.c +++ b/drivers/staging/pi433/rf69.c @@ -816,7 +816,7 @@ int rf69_write_fifo(struct spi_device *spi, u8 *buffer, unsigned int size) if (size > FIFO_SIZE) { dev_dbg(&spi->dev, - "read fifo: passed in buffer bigger then internal buffer\n"); + "write fifo: passed in buffer bigger then internal buffer\n"); return -EMSGSIZE; } diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c index ca6b966f5dd3d021aa274cdad9f11786225cd103..1ead7793062a66e5b3d4525fb1d012d38d498001 100644 --- a/drivers/staging/qlge/qlge_main.c +++ b/drivers/staging/qlge/qlge_main.c @@ -3041,8 +3041,8 @@ static int qlge_start_rx_ring(struct qlge_adapter *qdev, struct rx_ring *rx_ring /* Inbound completion handling rx_rings run in * separate NAPI contexts. */ - netif_napi_add_weight(qdev->ndev, &rx_ring->napi, - qlge_napi_poll_msix, 64); + netif_napi_add(qdev->ndev, &rx_ring->napi, + qlge_napi_poll_msix); cqicb->irq_delay = cpu_to_le16(qdev->rx_coalesce_usecs); cqicb->pkt_delay = cpu_to_le16(qdev->rx_max_coalesced_frames); } else { diff --git a/drivers/staging/r8188eu/Makefile b/drivers/staging/r8188eu/Makefile index eea16eb7caa0331a6270710c818a1429bffc1b56..fd494c2299e69bf98b982a682511c691d34161d4 100644 --- a/drivers/staging/r8188eu/Makefile +++ b/drivers/staging/r8188eu/Makefile @@ -10,7 +10,6 @@ r8188eu-y = \ hal/hal_com.o \ hal/odm.o \ hal/odm_HWConfig.o \ - hal/odm_RegConfig8188E.o \ hal/odm_RTL8188E.o \ hal/rtl8188e_cmd.o \ hal/rtl8188e_dm.o \ @@ -18,19 +17,14 @@ r8188eu-y = \ hal/rtl8188e_phycfg.o \ hal/rtl8188e_rf6052.o \ hal/rtl8188e_rxdesc.o \ - hal/rtl8188e_xmit.o \ - hal/rtl8188eu_recv.o \ hal/rtl8188eu_xmit.o \ hal/usb_halinit.o \ hal/usb_ops_linux.o \ os_dep/ioctl_linux.o \ - os_dep/mlme_linux.o \ os_dep/os_intfs.o \ os_dep/osdep_service.o \ - os_dep/recv_linux.o \ os_dep/usb_intf.o \ os_dep/usb_ops_linux.o \ - os_dep/xmit_linux.o \ core/rtw_ap.o \ core/rtw_br_ext.o \ core/rtw_cmd.o \ diff --git a/drivers/staging/r8188eu/core/rtw_ap.c b/drivers/staging/r8188eu/core/rtw_ap.c index 5bd9dfa57cc5c1424cc4bd0511e761f377fb0c4c..24eb8dce9bfeb042362098516c1db86b123e548a 100644 --- a/drivers/staging/r8188eu/core/rtw_ap.c +++ b/drivers/staging/r8188eu/core/rtw_ap.c @@ -935,6 +935,48 @@ u8 bss_cap_update_on_sta_leave(struct adapter *padapter, struct sta_info *psta) return beacon_updated; } +void rtw_indicate_sta_assoc_event(struct adapter *padapter, struct sta_info *psta) +{ + union iwreq_data wrqu; + struct sta_priv *pstapriv = &padapter->stapriv; + + if (!psta) + return; + + if (psta->aid > NUM_STA) + return; + + if (pstapriv->sta_aid[psta->aid - 1] != psta) + return; + + wrqu.addr.sa_family = ARPHRD_ETHER; + + memcpy(wrqu.addr.sa_data, psta->hwaddr, ETH_ALEN); + + wireless_send_event(padapter->pnetdev, IWEVREGISTERED, &wrqu, NULL); +} + +static void rtw_indicate_sta_disassoc_event(struct adapter *padapter, struct sta_info *psta) +{ + union iwreq_data wrqu; + struct sta_priv *pstapriv = &padapter->stapriv; + + if (!psta) + return; + + if (psta->aid > NUM_STA) + return; + + if (pstapriv->sta_aid[psta->aid - 1] != psta) + return; + + wrqu.addr.sa_family = ARPHRD_ETHER; + + memcpy(wrqu.addr.sa_data, psta->hwaddr, ETH_ALEN); + + wireless_send_event(padapter->pnetdev, IWEVEXPIRED, &wrqu, NULL); +} + u8 ap_free_sta(struct adapter *padapter, struct sta_info *psta, bool active, u16 reason) { diff --git a/drivers/staging/r8188eu/core/rtw_br_ext.c b/drivers/staging/r8188eu/core/rtw_br_ext.c index bca20fe5c9833b5a617bb9f7d83114c29a138d56..4c5f30792a46cd680fa5479087d85de540ebc887 100644 --- a/drivers/staging/r8188eu/core/rtw_br_ext.c +++ b/drivers/staging/r8188eu/core/rtw_br_ext.c @@ -12,7 +12,6 @@ #include "../include/drv_types.h" #include "../include/rtw_br_ext.h" #include "../include/usb_osintf.h" -#include "../include/recv_osdep.h" #ifndef csum_ipv6_magic #include "../include/net/ip6_checksum.h" diff --git a/drivers/staging/r8188eu/core/rtw_cmd.c b/drivers/staging/r8188eu/core/rtw_cmd.c index 5b6a891b5d67edb6c4a26f2c289e2ed07e27dc9b..3fadace33de6524762861b80980d736d0ae82509 100644 --- a/drivers/staging/r8188eu/core/rtw_cmd.c +++ b/drivers/staging/r8188eu/core/rtw_cmd.c @@ -5,8 +5,6 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/mlme_osdep.h" #include "../include/rtw_br_ext.h" #include "../include/rtw_mlme_ext.h" #include "../include/rtl8188e_dm.h" @@ -58,8 +56,6 @@ exit: u32 rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) { - u32 res = _SUCCESS; - init_completion(&pcmdpriv->enqueue_cmd); /* sema_init(&(pcmdpriv->cmd_done_sema), 0); */ init_completion(&pcmdpriv->start_cmd_thread); @@ -74,27 +70,24 @@ u32 rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) pcmdpriv->cmd_allocated_buf = kzalloc(MAX_CMDSZ + CMDBUFF_ALIGN_SZ, GFP_KERNEL); - if (!pcmdpriv->cmd_allocated_buf) { - res = _FAIL; - goto exit; - } + if (!pcmdpriv->cmd_allocated_buf) + return _FAIL; pcmdpriv->cmd_buf = pcmdpriv->cmd_allocated_buf + CMDBUFF_ALIGN_SZ - ((size_t)(pcmdpriv->cmd_allocated_buf) & (CMDBUFF_ALIGN_SZ - 1)); pcmdpriv->rsp_allocated_buf = kzalloc(MAX_RSPSZ + 4, GFP_KERNEL); if (!pcmdpriv->rsp_allocated_buf) { - res = _FAIL; - goto exit; + kfree(pcmdpriv->cmd_allocated_buf); + return _FAIL; } pcmdpriv->rsp_buf = pcmdpriv->rsp_allocated_buf + 4 - ((size_t)(pcmdpriv->rsp_allocated_buf) & 3); pcmdpriv->cmd_done_cnt = 0; pcmdpriv->rsp_cnt = 0; -exit: - return res; + return _SUCCESS; } u32 rtw_init_evt_priv(struct evt_priv *pevtpriv) @@ -288,8 +281,7 @@ post_process: * ### NOTE:#### (!!!!) * MUST TAKE CARE THAT BEFORE CALLING THIS FUNC, YOU SHOULD HAVE LOCKED pmlmepriv->lock */ -u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, int ssid_num, - struct rtw_ieee80211_channel *ch, int ch_num) +u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, int ssid_num) { u8 res = _FAIL; struct cmd_obj *ph2c; @@ -331,17 +323,6 @@ u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, } } - /* prepare channel list */ - if (ch) { - int i; - for (i = 0; i < ch_num && i < RTW_CHANNEL_SCAN_AMOUNT; i++) { - if (ch[i].hw_value && !(ch[i].flags & RTW_IEEE80211_CHAN_DISABLED)) { - memcpy(&psurveyPara->ch[i], &ch[i], sizeof(struct rtw_ieee80211_channel)); - psurveyPara->ch_num++; - } - } - } - set_fwstate(pmlmepriv, _FW_UNDER_SURVEY); res = rtw_enqueue_cmd(pcmdpriv, ph2c); @@ -1290,6 +1271,66 @@ exit: return res; } +/* C2H event format: + * Field TRIGGER CONTENT CMD_SEQ CMD_LEN CMD_ID + * BITS [127:120] [119:16] [15:8] [7:4] [3:0] + */ +static s32 c2h_evt_read(struct adapter *adapter, u8 *buf) +{ + s32 ret = _FAIL; + struct c2h_evt_hdr *c2h_evt; + int i; + u8 trigger; + + if (!buf) + goto exit; + + ret = rtw_read8(adapter, REG_C2HEVT_CLEAR, &trigger); + if (ret) + return _FAIL; + + if (trigger == C2H_EVT_HOST_CLOSE) + goto exit; /* Not ready */ + else if (trigger != C2H_EVT_FW_CLOSE) + goto clear_evt; /* Not a valid value */ + + c2h_evt = (struct c2h_evt_hdr *)buf; + + memset(c2h_evt, 0, 16); + + ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL, buf); + if (ret) { + ret = _FAIL; + goto clear_evt; + } + + ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL + 1, buf + 1); + if (ret) { + ret = _FAIL; + goto clear_evt; + } + /* Read the content */ + for (i = 0; i < c2h_evt->plen; i++) { + ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL + + sizeof(*c2h_evt) + i, c2h_evt->payload + i); + if (ret) { + ret = _FAIL; + goto clear_evt; + } + } + + ret = _SUCCESS; + +clear_evt: + /* Clear event to notify FW we have read the command. + * If this field isn't clear, the FW won't update the next + * command message. + */ + rtw_write8(adapter, REG_C2HEVT_CLEAR, C2H_EVT_HOST_CLOSE); +exit: + return ret; +} + static void c2h_evt_hdl(struct adapter *adapter, struct c2h_evt_hdr *c2h_evt, c2h_id_filter filter) { u8 buf[16]; diff --git a/drivers/staging/r8188eu/core/rtw_fw.c b/drivers/staging/r8188eu/core/rtw_fw.c index 95534f9c7a0f477467ea0ae45e8ba468f884c902..682c65b1e04c2024888ea95747c95e794c5746b8 100644 --- a/drivers/staging/r8188eu/core/rtw_fw.c +++ b/drivers/staging/r8188eu/core/rtw_fw.c @@ -236,7 +236,7 @@ static int load_firmware(struct rt_firmware *rtfw, struct device *device) { int ret = _SUCCESS; const struct firmware *fw; - const char *fw_name = "rtlwifi/rtl8188eufw.bin"; + const char *fw_name = FW_RTL8188EU; int err = request_firmware(&fw, fw_name, device); if (err) { diff --git a/drivers/staging/r8188eu/core/rtw_ioctl_set.c b/drivers/staging/r8188eu/core/rtw_ioctl_set.c index 17f6bcbeebf428878f834fc1de4968a9dc661b21..55e6b0f41dc32bb27b2788919b96d240449e3983 100644 --- a/drivers/staging/r8188eu/core/rtw_ioctl_set.c +++ b/drivers/staging/r8188eu/core/rtw_ioctl_set.c @@ -11,8 +11,6 @@ #include "../include/usb_osintf.h" #include "../include/usb_ops.h" -extern void indicate_wx_scan_complete_event(struct adapter *padapter); - u8 rtw_do_join(struct adapter *padapter) { struct list_head *plist, *phead; @@ -43,7 +41,7 @@ u8 rtw_do_join(struct adapter *padapter) if (!pmlmepriv->LinkDetectInfo.bBusyTraffic || pmlmepriv->to_roaming > 0) { /* submit site_survey_cmd */ - ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0); + ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1); if (ret != _SUCCESS) pmlmepriv->to_join = false; } else { @@ -89,7 +87,7 @@ u8 rtw_do_join(struct adapter *padapter) /* we try to issue sitesurvey firstly */ if (!pmlmepriv->LinkDetectInfo.bBusyTraffic || pmlmepriv->to_roaming > 0) { - ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1, NULL, 0); + ret = rtw_sitesurvey_cmd(padapter, &pmlmepriv->assoc_ssid, 1); if (ret != _SUCCESS) pmlmepriv->to_join = false; } else { @@ -353,14 +351,9 @@ u8 rtw_set_802_11_bssid_list_scan(struct adapter *padapter, struct ndis_802_11_s /* Scan or linking is in progress, do nothing. */ res = true; } else { - if (rtw_is_scan_deny(padapter)) { - indicate_wx_scan_complete_event(padapter); - return _SUCCESS; - } - spin_lock_bh(&pmlmepriv->lock); - res = rtw_sitesurvey_cmd(padapter, pssid, ssid_max_num, NULL, 0); + res = rtw_sitesurvey_cmd(padapter, pssid, ssid_max_num); spin_unlock_bh(&pmlmepriv->lock); } diff --git a/drivers/staging/r8188eu/core/rtw_led.c b/drivers/staging/r8188eu/core/rtw_led.c index d5c6c5e296215df4195d9724835678b7084de711..1e316e6358ea2deda4c80ef0bb8710fc156167e6 100644 --- a/drivers/staging/r8188eu/core/rtw_led.c +++ b/drivers/staging/r8188eu/core/rtw_led.c @@ -25,9 +25,7 @@ static void ResetLedStatus(struct led_priv *pLed) pLed->bLedWPSBlinkInProgress = false; pLed->BlinkTimes = 0; /* Number of times to toggle led state for blinking. */ - pLed->BlinkingLedState = LED_UNKNOWN; /* Next state for blinking, either RTW_LED_ON or RTW_LED_OFF are. */ - pLed->bLedNoLinkBlinkInProgress = false; pLed->bLedLinkBlinkInProgress = false; pLed->bLedScanBlinkInProgress = false; } @@ -37,7 +35,7 @@ static void SwLedOn(struct adapter *padapter, struct led_priv *pLed) u8 LedCfg; int res; - if (padapter->bSurpriseRemoved || padapter->bDriverStopped) + if (padapter->bDriverStopped) return; res = rtw_read8(padapter, REG_LEDCFG2, &LedCfg); @@ -53,7 +51,7 @@ static void SwLedOff(struct adapter *padapter, struct led_priv *pLed) u8 LedCfg; int res; - if (padapter->bSurpriseRemoved || padapter->bDriverStopped) + if (padapter->bDriverStopped) goto exit; res = rtw_read8(padapter, REG_LEDCFG2, &LedCfg);/* 0x4E */ @@ -79,41 +77,25 @@ static void blink_work(struct work_struct *work) struct adapter *padapter = pLed->padapter; struct mlme_priv *pmlmepriv = &padapter->mlmepriv; - if ((padapter->bSurpriseRemoved) || (padapter->bDriverStopped)) - return; - if (padapter->pwrctrlpriv.rf_pwrstate != rf_on) { SwLedOff(padapter, pLed); ResetLedStatus(pLed); return; } - /* Change LED according to BlinkingLedState specified. */ - if (pLed->BlinkingLedState == RTW_LED_ON) - SwLedOn(padapter, pLed); - else + if (pLed->bLedOn) SwLedOff(padapter, pLed); + else + SwLedOn(padapter, pLed); switch (pLed->CurrLedState) { case LED_BLINK_SLOWLY: - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); break; case LED_BLINK_NORMAL: - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); break; case LED_BLINK_SCAN: - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; pLed->BlinkTimes--; if (pLed->BlinkTimes == 0) { if (check_fwstate(pmlmepriv, _FW_LINKED)) { @@ -121,7 +103,6 @@ static void blink_work(struct work_struct *work) pLed->CurrLedState = LED_BLINK_NORMAL; schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); } else { - pLed->bLedNoLinkBlinkInProgress = true; pLed->CurrLedState = LED_BLINK_SLOWLY; schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); } @@ -131,10 +112,6 @@ static void blink_work(struct work_struct *work) } break; case LED_BLINK_TXRX: - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; pLed->BlinkTimes--; if (pLed->BlinkTimes == 0) { if (check_fwstate(pmlmepriv, _FW_LINKED)) { @@ -142,7 +119,6 @@ static void blink_work(struct work_struct *work) pLed->CurrLedState = LED_BLINK_NORMAL; schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); } else { - pLed->bLedNoLinkBlinkInProgress = true; pLed->CurrLedState = LED_BLINK_SLOWLY; schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); } @@ -152,25 +128,16 @@ static void blink_work(struct work_struct *work) } break; case LED_BLINK_WPS: - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, LED_BLINK_SCAN_INTVL); break; case LED_BLINK_WPS_STOP: /* WPS success */ - if (pLed->BlinkingLedState != RTW_LED_ON) { + if (!pLed->bLedOn) { pLed->bLedLinkBlinkInProgress = true; pLed->CurrLedState = LED_BLINK_NORMAL; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); pLed->bLedWPSBlinkInProgress = false; } else { - pLed->BlinkingLedState = RTW_LED_OFF; schedule_delayed_work(&pLed->blink_work, LED_BLINK_WPS_SUCESS_INTVL); } break; @@ -217,192 +184,110 @@ void rtw_led_control(struct adapter *padapter, enum LED_CTL_MODE LedAction) switch (LedAction) { case LED_CTL_START_TO_LINK: case LED_CTL_NO_LINK: - if (!pLed->bLedNoLinkBlinkInProgress) { - if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) - return; - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } + if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) + return; - pLed->bLedNoLinkBlinkInProgress = true; - pLed->CurrLedState = LED_BLINK_SLOWLY; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; - schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); - } + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = false; + + pLed->CurrLedState = LED_BLINK_SLOWLY; + schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); break; case LED_CTL_LINK: - if (!pLed->bLedLinkBlinkInProgress) { - if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) - return; - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } - pLed->bLedLinkBlinkInProgress = true; - pLed->CurrLedState = LED_BLINK_NORMAL; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; - schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); - } + if (!pLed->bLedLinkBlinkInProgress) + return; + + if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) + return; + + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedBlinkInProgress = false; + pLed->bLedLinkBlinkInProgress = true; + + pLed->CurrLedState = LED_BLINK_NORMAL; + schedule_delayed_work(&pLed->blink_work, LED_BLINK_LINK_INTVL); break; case LED_CTL_SITE_SURVEY: - if ((pmlmepriv->LinkDetectInfo.bBusyTraffic) && (check_fwstate(pmlmepriv, _FW_LINKED))) { - ; - } else if (!pLed->bLedScanBlinkInProgress) { - if (IS_LED_WPS_BLINKING(pLed)) - return; - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } - pLed->bLedScanBlinkInProgress = true; - pLed->CurrLedState = LED_BLINK_SCAN; - pLed->BlinkTimes = 24; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; - schedule_delayed_work(&pLed->blink_work, LED_BLINK_SCAN_INTVL); - } + if ((pmlmepriv->LinkDetectInfo.bBusyTraffic) && (check_fwstate(pmlmepriv, _FW_LINKED))) + return; + + if (pLed->bLedScanBlinkInProgress) + return; + + if (IS_LED_WPS_BLINKING(pLed)) + return; + + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = false; + pLed->bLedScanBlinkInProgress = true; + + pLed->CurrLedState = LED_BLINK_SCAN; + pLed->BlinkTimes = 24; + schedule_delayed_work(&pLed->blink_work, LED_BLINK_SCAN_INTVL); break; case LED_CTL_TX: case LED_CTL_RX: - if (!pLed->bLedBlinkInProgress) { - if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) - return; - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - pLed->bLedBlinkInProgress = true; - pLed->CurrLedState = LED_BLINK_TXRX; - pLed->BlinkTimes = 2; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; - schedule_delayed_work(&pLed->blink_work, LED_BLINK_FASTER_INTVL); - } + if (pLed->bLedBlinkInProgress) + return; + + if (pLed->CurrLedState == LED_BLINK_SCAN || IS_LED_WPS_BLINKING(pLed)) + return; + + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = true; + + pLed->CurrLedState = LED_BLINK_TXRX; + pLed->BlinkTimes = 2; + schedule_delayed_work(&pLed->blink_work, LED_BLINK_FASTER_INTVL); break; case LED_CTL_START_WPS: /* wait until xinpin finish */ - if (!pLed->bLedWPSBlinkInProgress) { - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } - if (pLed->bLedScanBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedScanBlinkInProgress = false; - } - pLed->bLedWPSBlinkInProgress = true; - pLed->CurrLedState = LED_BLINK_WPS; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; - schedule_delayed_work(&pLed->blink_work, LED_BLINK_SCAN_INTVL); - } + if (pLed->bLedWPSBlinkInProgress) + return; + + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = false; + pLed->bLedScanBlinkInProgress = false; + pLed->bLedWPSBlinkInProgress = true; + pLed->CurrLedState = LED_BLINK_WPS; + schedule_delayed_work(&pLed->blink_work, LED_BLINK_SCAN_INTVL); break; case LED_CTL_STOP_WPS: - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } - if (pLed->bLedScanBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedScanBlinkInProgress = false; - } - if (pLed->bLedWPSBlinkInProgress) - cancel_delayed_work(&pLed->blink_work); - else - pLed->bLedWPSBlinkInProgress = true; + cancel_delayed_work(&pLed->blink_work); + + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = false; + pLed->bLedScanBlinkInProgress = false; + pLed->bLedWPSBlinkInProgress = true; + pLed->CurrLedState = LED_BLINK_WPS_STOP; if (pLed->bLedOn) { - pLed->BlinkingLedState = RTW_LED_OFF; schedule_delayed_work(&pLed->blink_work, LED_BLINK_WPS_SUCESS_INTVL); } else { - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, 0); } break; case LED_CTL_STOP_WPS_FAIL: - if (pLed->bLedWPSBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedWPSBlinkInProgress = false; - } - pLed->bLedNoLinkBlinkInProgress = true; + cancel_delayed_work(&pLed->blink_work); + pLed->bLedWPSBlinkInProgress = false; pLed->CurrLedState = LED_BLINK_SLOWLY; - if (pLed->bLedOn) - pLed->BlinkingLedState = RTW_LED_OFF; - else - pLed->BlinkingLedState = RTW_LED_ON; schedule_delayed_work(&pLed->blink_work, LED_BLINK_NO_LINK_INTVL); break; case LED_CTL_POWER_OFF: pLed->CurrLedState = RTW_LED_OFF; - pLed->BlinkingLedState = RTW_LED_OFF; - if (pLed->bLedNoLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedNoLinkBlinkInProgress = false; - } - if (pLed->bLedLinkBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedLinkBlinkInProgress = false; - } - if (pLed->bLedBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedBlinkInProgress = false; - } - if (pLed->bLedWPSBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedWPSBlinkInProgress = false; - } - if (pLed->bLedScanBlinkInProgress) { - cancel_delayed_work(&pLed->blink_work); - pLed->bLedScanBlinkInProgress = false; - } + pLed->bLedLinkBlinkInProgress = false; + pLed->bLedBlinkInProgress = false; + pLed->bLedWPSBlinkInProgress = false; + pLed->bLedScanBlinkInProgress = false; + cancel_delayed_work(&pLed->blink_work); SwLedOff(padapter, pLed); break; default: diff --git a/drivers/staging/r8188eu/core/rtw_mlme.c b/drivers/staging/r8188eu/core/rtw_mlme.c index 2705c9d87b1443dd453c2ac442eed554e6710313..5ca03d6cac32a2f1b849bab76835321c46d4ee5c 100644 --- a/drivers/staging/r8188eu/core/rtw_mlme.c +++ b/drivers/staging/r8188eu/core/rtw_mlme.c @@ -5,10 +5,7 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/xmit_osdep.h" #include "../include/hal_intf.h" -#include "../include/mlme_osdep.h" #include "../include/sta_info.h" #include "../include/wifi.h" #include "../include/wlan_bssdef.h" @@ -190,6 +187,37 @@ u8 *rtw_get_beacon_interval_from_ie(u8 *ie) return ie + 8; } +static void rtw_join_timeout_handler(struct timer_list *t) +{ + struct adapter *adapter = from_timer(adapter, t, mlmepriv.assoc_timer); + + _rtw_join_timeout_handler(adapter); +} + +static void _rtw_scan_timeout_handler(struct timer_list *t) +{ + struct adapter *adapter = from_timer(adapter, t, mlmepriv.scan_to_timer); + + rtw_scan_timeout_handler(adapter); +} + +static void _dynamic_check_timer_handlder(struct timer_list *t) +{ + struct adapter *adapter = from_timer(adapter, t, mlmepriv.dynamic_chk_timer); + + rtw_dynamic_check_timer_handlder(adapter); + _set_timer(&adapter->mlmepriv.dynamic_chk_timer, 2000); +} + +static void rtw_init_mlme_timer(struct adapter *padapter) +{ + struct mlme_priv *pmlmepriv = &padapter->mlmepriv; + + timer_setup(&pmlmepriv->assoc_timer, rtw_join_timeout_handler, 0); + timer_setup(&pmlmepriv->scan_to_timer, _rtw_scan_timeout_handler, 0); + timer_setup(&pmlmepriv->dynamic_chk_timer, _dynamic_check_timer_handlder, 0); +} + int rtw_init_mlme_priv(struct adapter *padapter)/* struct mlme_priv *pmlmepriv) */ { int i; @@ -235,8 +263,6 @@ int rtw_init_mlme_priv(struct adapter *padapter)/* struct mlme_priv *pmlmepriv) /* allocate DMA-able/Non-Page memory for cmd_buf and rsp_buf */ - rtw_clear_scan_deny(padapter); - rtw_init_mlme_timer(padapter); exit: @@ -641,6 +667,23 @@ exit: spin_unlock_bh(&pmlmepriv->lock); } +static void rtw_xmit_schedule(struct adapter *padapter) +{ + struct xmit_priv *pxmitpriv; + + if (!padapter) + return; + + pxmitpriv = &padapter->xmitpriv; + + spin_lock_bh(&pxmitpriv->lock); + + if (rtw_txframes_pending(padapter)) + tasklet_hi_schedule(&pxmitpriv->xmit_tasklet); + + spin_unlock_bh(&pxmitpriv->lock); +} + void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf) { struct mlme_priv *pmlmepriv = &adapter->mlmepriv; @@ -697,7 +740,7 @@ void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf) } else { if (rtw_to_roaming(adapter) != 0) { if (--pmlmepriv->to_roaming == 0 || - rtw_sitesurvey_cmd(adapter, &pmlmepriv->assoc_ssid, 1, NULL, 0) != _SUCCESS) { + rtw_sitesurvey_cmd(adapter, &pmlmepriv->assoc_ssid, 1) != _SUCCESS) { rtw_set_roaming(adapter, 0); rtw_free_assoc_resources(adapter, 1); rtw_indicate_disconnect(adapter); @@ -719,7 +762,7 @@ void rtw_surveydone_event_callback(struct adapter *adapter, u8 *pbuf) if (check_fwstate(pmlmepriv, _FW_LINKED)) p2p_ps_wk_cmd(adapter, P2P_PS_SCAN_DONE, 0); - rtw_os_xmit_schedule(adapter); + rtw_xmit_schedule(adapter); } static void free_scanqueue(struct mlme_priv *pmlmepriv) @@ -795,6 +838,48 @@ void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue) } +static struct rt_pmkid_list backup_pmkid[NUM_PMKID_CACHE]; + +static void rtw_reset_securitypriv(struct adapter *adapter) +{ + u8 backup_index; + u8 backup_counter; + u32 backup_time; + + if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { + /* 802.1x */ + /* We have to backup the PMK information for WiFi PMK Caching test item. */ + /* Backup the btkip_countermeasure information. */ + /* When the countermeasure is trigger, the driver have to disconnect with AP for 60 seconds. */ + memcpy(&backup_pmkid[0], &adapter->securitypriv.PMKIDList[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE); + backup_index = adapter->securitypriv.PMKIDIndex; + backup_counter = adapter->securitypriv.btkip_countermeasure; + backup_time = adapter->securitypriv.btkip_countermeasure_time; + memset((unsigned char *)&adapter->securitypriv, 0, sizeof(struct security_priv)); + + /* Restore the PMK information to securitypriv structure for the following connection. */ + memcpy(&adapter->securitypriv.PMKIDList[0], + &backup_pmkid[0], + sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE); + adapter->securitypriv.PMKIDIndex = backup_index; + adapter->securitypriv.btkip_countermeasure = backup_counter; + adapter->securitypriv.btkip_countermeasure_time = backup_time; + adapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen; + adapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled; + } else { + /* reset values in securitypriv */ + struct security_priv *psec_priv = &adapter->securitypriv; + + psec_priv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */ + psec_priv->dot11PrivacyAlgrthm = _NO_PRIVACY_; + psec_priv->dot11PrivacyKeyIndex = 0; + psec_priv->dot118021XGrpPrivacy = _NO_PRIVACY_; + psec_priv->dot118021XGrpKeyid = 1; + psec_priv->ndisauthtype = Ndis802_11AuthModeOpen; + psec_priv->ndisencryptstatus = Ndis802_11WEPDisabled; + } +} + /* *rtw_indicate_connect: the caller has to lock pmlmepriv->lock */ @@ -809,12 +894,13 @@ void rtw_indicate_connect(struct adapter *padapter) rtw_led_control(padapter, LED_CTL_LINK); - rtw_os_indicate_connect(padapter); + rtw_indicate_wx_assoc_event(padapter); + netif_carrier_on(padapter->pnetdev); + if (padapter->pid[2] != 0) + rtw_signal_process(padapter->pid[2], SIGALRM); } pmlmepriv->to_roaming = 0; - - rtw_set_scan_deny(padapter, 3000); } /* @@ -831,11 +917,14 @@ void rtw_indicate_disconnect(struct adapter *padapter) if (check_fwstate(&padapter->mlmepriv, _FW_LINKED) || (pmlmepriv->to_roaming <= 0)) { - rtw_os_indicate_disconnect(padapter); + /* Do it first for tx broadcast pkt after disconnection issue! */ + netif_carrier_off(padapter->pnetdev); + + rtw_indicate_wx_disassoc_event(padapter); + rtw_reset_securitypriv(padapter); _clr_fwstate_(pmlmepriv, _FW_LINKED); rtw_led_control(padapter, LED_CTL_NO_LINK); - rtw_clear_scan_deny(padapter); } p2p_ps_wk_cmd(padapter, P2P_PS_DISABLE, 1); @@ -843,9 +932,9 @@ void rtw_indicate_disconnect(struct adapter *padapter) } -inline void rtw_indicate_scan_done(struct adapter *padapter, bool aborted) +inline void rtw_indicate_scan_done(struct adapter *padapter) { - rtw_os_indicate_scan_done(padapter, aborted); + indicate_wx_scan_complete_event(padapter); } static struct sta_info *rtw_joinbss_update_stainfo(struct adapter *padapter, struct wlan_network *pnetwork) @@ -1068,8 +1157,7 @@ void rtw_joinbss_event_callback(struct adapter *adapter, u8 *pbuf) mlmeext_joinbss_event_callback(adapter, pnetwork->join_res); - rtw_os_xmit_schedule(adapter); - + rtw_xmit_schedule(adapter); } void rtw_set_max_rpt_macid(struct adapter *adapter, u8 macid) @@ -1316,7 +1404,7 @@ void rtw_scan_timeout_handler (struct adapter *adapter) spin_lock_bh(&pmlmepriv->lock); _clr_fwstate_(pmlmepriv, _FW_UNDER_SURVEY); spin_unlock_bh(&pmlmepriv->lock); - rtw_indicate_scan_done(adapter, true); + rtw_indicate_scan_done(adapter); } static void rtw_auto_scan_handler(struct adapter *padapter) @@ -1442,10 +1530,6 @@ int rtw_select_and_join_from_scanned_queue(struct mlme_priv *pmlmepriv) pmlmepriv->pscanned = phead->next; while (phead != pmlmepriv->pscanned) { pnetwork = container_of(pmlmepriv->pscanned, struct wlan_network, list); - if (!pnetwork) { - ret = _FAIL; - goto exit; - } pmlmepriv->pscanned = pmlmepriv->pscanned->next; rtw_check_join_candidate(pmlmepriv, &candidate, pnetwork); } @@ -1639,6 +1723,33 @@ static int rtw_append_pmkid(struct adapter *Adapter, int iEntry, u8 *ie, uint ie return ie_len; } +static void rtw_report_sec_ie(struct adapter *adapter, u8 authmode, u8 *sec_ie) +{ + uint len; + u8 *buff, *p, i; + union iwreq_data wrqu; + + buff = NULL; + if (authmode == _WPA_IE_ID_) { + buff = kzalloc(IW_CUSTOM_MAX, GFP_ATOMIC); + if (!buff) + return; + p = buff; + p += sprintf(p, "ASSOCINFO(ReqIEs ="); + len = sec_ie[1] + 2; + len = (len < IW_CUSTOM_MAX) ? len : IW_CUSTOM_MAX; + for (i = 0; i < len; i++) + p += sprintf(p, "%02x", sec_ie[i]); + p += sprintf(p, ")"); + memset(&wrqu, 0, sizeof(wrqu)); + wrqu.data.length = p - buff; + wrqu.data.length = (wrqu.data.length < IW_CUSTOM_MAX) ? + wrqu.data.length : IW_CUSTOM_MAX; + wireless_send_event(adapter->pnetdev, IWEVCUSTOM, &wrqu, buff); + kfree(buff); + } +} + int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len) { u8 authmode = 0; diff --git a/drivers/staging/r8188eu/core/rtw_mlme_ext.c b/drivers/staging/r8188eu/core/rtw_mlme_ext.c index 32d0e101d0c246788d24f16b1970378439f1e326..07905e2ae8e055fb982c30397387ebd8ab9a6a98 100644 --- a/drivers/staging/r8188eu/core/rtw_mlme_ext.c +++ b/drivers/staging/r8188eu/core/rtw_mlme_ext.c @@ -9,8 +9,6 @@ #include "../include/wifi.h" #include "../include/rtw_mlme_ext.h" #include "../include/wlan_bssdef.h" -#include "../include/mlme_osdep.h" -#include "../include/recv_osdep.h" #include "../include/rtl8188e_xmit.h" #include "../include/rtl8188e_dm.h" @@ -334,6 +332,28 @@ static u8 init_channel_set(struct adapter *padapter, u8 ChannelPlan, struct rt_c return chanset_size; } +static void _survey_timer_hdl(struct timer_list *t) +{ + struct adapter *padapter = from_timer(padapter, t, mlmeextpriv.survey_timer); + + survey_timer_hdl(padapter); +} + +static void _link_timer_hdl(struct timer_list *t) +{ + struct adapter *padapter = from_timer(padapter, t, mlmeextpriv.link_timer); + + link_timer_hdl(padapter); +} + +static void init_mlme_ext_timer(struct adapter *padapter) +{ + struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; + + timer_setup(&pmlmeext->survey_timer, _survey_timer_hdl, 0); + timer_setup(&pmlmeext->link_timer, _link_timer_hdl, 0); +} + void init_mlme_ext_priv(struct adapter *padapter) { struct registry_priv *pregistrypriv = &padapter->registrypriv; @@ -910,6 +930,46 @@ authclnt_fail: return _FAIL; } +static void UpdateBrateTbl(u8 *mbrate) +{ + u8 i; + u8 rate; + + /* 1M, 2M, 5.5M, 11M, 6M, 12M, 24M are mandatory. */ + for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) { + rate = mbrate[i] & 0x7f; + switch (rate) { + case IEEE80211_CCK_RATE_1MB: + case IEEE80211_CCK_RATE_2MB: + case IEEE80211_CCK_RATE_5MB: + case IEEE80211_CCK_RATE_11MB: + case IEEE80211_OFDM_RATE_6MB: + case IEEE80211_OFDM_RATE_12MB: + case IEEE80211_OFDM_RATE_24MB: + mbrate[i] |= IEEE80211_BASIC_RATE_MASK; + break; + } + } +} + +static void UpdateBrateTblForSoftAP(u8 *bssrateset, u32 bssratelen) +{ + u8 i; + u8 rate; + + for (i = 0; i < bssratelen; i++) { + rate = bssrateset[i] & 0x7f; + switch (rate) { + case IEEE80211_CCK_RATE_1MB: + case IEEE80211_CCK_RATE_2MB: + case IEEE80211_CCK_RATE_5MB: + case IEEE80211_CCK_RATE_11MB: + bssrateset[i] |= IEEE80211_BASIC_RATE_MASK; + break; + } + } +} + unsigned int OnAssocReq(struct adapter *padapter, struct recv_frame *precv_frame) { u16 capab_info; @@ -1320,9 +1380,9 @@ OnAssocReqFail: unsigned int OnAssocRsp(struct adapter *padapter, struct recv_frame *precv_frame) { + struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)precv_frame->rx_data; uint i; int res; - unsigned short status; struct ndis_802_11_var_ie *pIE; struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info; @@ -1331,7 +1391,7 @@ unsigned int OnAssocRsp(struct adapter *padapter, struct recv_frame *precv_frame uint pkt_len = precv_frame->len; /* check A1 matches or not */ - if (memcmp(myid(&padapter->eeprompriv), get_da(pframe), ETH_ALEN)) + if (memcmp(myid(&padapter->eeprompriv), mgmt->da, ETH_ALEN)) return _SUCCESS; if (!(pmlmeinfo->state & (WIFI_FW_AUTH_SUCCESS | WIFI_FW_ASSOC_STATE))) @@ -1342,28 +1402,24 @@ unsigned int OnAssocRsp(struct adapter *padapter, struct recv_frame *precv_frame _cancel_timer_ex(&pmlmeext->link_timer); - /* status */ - status = le16_to_cpu(*(__le16 *)(pframe + WLAN_HDR_A3_LEN + 2)); - if (status > 0) { + if (le16_to_cpu(mgmt->u.assoc_resp.status_code) > 0) { pmlmeinfo->state = WIFI_FW_NULL_STATE; res = -4; goto report_assoc_result; } - /* get capabilities */ - pmlmeinfo->capability = le16_to_cpu(*(__le16 *)(pframe + WLAN_HDR_A3_LEN)); + pmlmeinfo->capability = le16_to_cpu(mgmt->u.assoc_resp.capab_info); /* set slot time */ pmlmeinfo->slotTime = (pmlmeinfo->capability & BIT(10)) ? 9 : 20; - /* AID */ - pmlmeinfo->aid = (int)(le16_to_cpu(*(__le16 *)(pframe + WLAN_HDR_A3_LEN + 4)) & 0x3fff); + pmlmeinfo->aid = le16_to_cpu(mgmt->u.assoc_resp.aid) & 0x3fff; res = pmlmeinfo->aid; /* following are moved to join event callback function */ /* to handle HT, WMM, rate adaptive, update MAC reg */ /* for not to handle the synchronous IO in the tasklet */ - for (i = (6 + WLAN_HDR_A3_LEN); i < pkt_len;) { + for (i = offsetof(struct ieee80211_mgmt, u.assoc_resp.variable); i < pkt_len;) { pIE = (struct ndis_802_11_var_ie *)(pframe + i); switch (pIE->ElementID) { @@ -1391,7 +1447,7 @@ unsigned int OnAssocRsp(struct adapter *padapter, struct recv_frame *precv_frame pmlmeinfo->state |= WIFI_FW_ASSOC_SUCCESS; /* Update Basic Rate Table for spec, 2010-12-28 , by thomas */ - UpdateBrateTbl(padapter, pmlmeinfo->network.SupportedRates); + UpdateBrateTbl(pmlmeinfo->network.SupportedRates); report_assoc_result: report_join_res(padapter, res); @@ -7858,7 +7914,7 @@ u8 tx_beacon_hdl(struct adapter *padapter, unsigned char *pbuf) spin_unlock_bh(&psta_bmc->sleep_q.lock); if (rtl8188eu_hal_xmit(padapter, pxmitframe)) - rtw_os_xmit_complete(padapter, pxmitframe); + rtw_xmit_complete(padapter, pxmitframe); spin_lock_bh(&psta_bmc->sleep_q.lock); } spin_unlock_bh(&psta_bmc->sleep_q.lock); diff --git a/drivers/staging/r8188eu/core/rtw_p2p.c b/drivers/staging/r8188eu/core/rtw_p2p.c index bd654d4ff8b4b8cc62dcb9388f5f75979fb74dd4..dc159e58f428852abec8f0fb859eea3d458e9a0d 100644 --- a/drivers/staging/r8188eu/core/rtw_p2p.c +++ b/drivers/staging/r8188eu/core/rtw_p2p.c @@ -1883,15 +1883,14 @@ void init_wifidirect_info(struct adapter *padapter, enum P2P_ROLE role) int rtw_p2p_enable(struct adapter *padapter, enum P2P_ROLE role) { - int ret = _SUCCESS; + int ret; struct wifidirect_info *pwdinfo = &padapter->wdinfo; if (role == P2P_ROLE_DEVICE || role == P2P_ROLE_CLIENT || role == P2P_ROLE_GO) { /* leave IPS/Autosuspend */ - if (rtw_pwr_wakeup(padapter)) { - ret = _FAIL; - goto exit; - } + ret = rtw_pwr_wakeup(padapter); + if (ret) + return ret; /* Added by Albert 2011/03/22 */ /* In the P2P mode, the driver should not support the b mode. */ @@ -1902,10 +1901,9 @@ int rtw_p2p_enable(struct adapter *padapter, enum P2P_ROLE role) init_wifidirect_info(padapter, role); } else if (role == P2P_ROLE_DISABLE) { - if (rtw_pwr_wakeup(padapter)) { - ret = _FAIL; - goto exit; - } + ret = rtw_pwr_wakeup(padapter); + if (ret) + return ret; /* Disable P2P function */ if (!rtw_p2p_chk_state(pwdinfo, P2P_STATE_NONE)) { @@ -1923,6 +1921,5 @@ int rtw_p2p_enable(struct adapter *padapter, enum P2P_ROLE role) update_tx_basic_rate(padapter, padapter->registrypriv.wireless_mode); } -exit: - return ret; + return 0; } diff --git a/drivers/staging/r8188eu/core/rtw_pwrctrl.c b/drivers/staging/r8188eu/core/rtw_pwrctrl.c index 10550bd2c16dbb1fcf23bd5338042d144f7ba441..870d81735b8dc5b83d21cf25ef87481bf38189c7 100644 --- a/drivers/staging/r8188eu/core/rtw_pwrctrl.c +++ b/drivers/staging/r8188eu/core/rtw_pwrctrl.c @@ -89,7 +89,7 @@ static bool rtw_pwr_unassociated_idle(struct adapter *adapter) struct wifidirect_info *pwdinfo = &adapter->wdinfo; bool ret = false; - if (adapter->pwrctrlpriv.ips_deny_time >= jiffies) + if (time_after_eq(adapter->pwrctrlpriv.ips_deny_time, jiffies)) goto exit; if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE | WIFI_SITE_MONITOR) || diff --git a/drivers/staging/r8188eu/core/rtw_recv.c b/drivers/staging/r8188eu/core/rtw_recv.c index e5a7b7dfc3879a9e1eca6a368bb8d8608458c9e8..bb5c3b3888e0828dff2cf9846f3628c61de24c84 100644 --- a/drivers/staging/r8188eu/core/rtw_recv.c +++ b/drivers/staging/r8188eu/core/rtw_recv.c @@ -6,8 +6,6 @@ #include #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/mlme_osdep.h" #include "../include/usb_ops.h" #include "../include/wifi.h" #include "../include/rtl8188e_recv.h" @@ -37,6 +35,69 @@ void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv) } +static int rtl8188eu_init_recv_priv(struct adapter *padapter) +{ + struct recv_priv *precvpriv = &padapter->recvpriv; + int i, res = _SUCCESS; + struct recv_buf *precvbuf; + + tasklet_init(&precvpriv->recv_tasklet, + rtl8188eu_recv_tasklet, + (unsigned long)padapter); + + /* init recv_buf */ + rtw_init_queue(&precvpriv->free_recv_buf_queue); + + precvpriv->pallocated_recv_buf = kzalloc(NR_RECVBUFF * sizeof(struct recv_buf) + 4, + GFP_KERNEL); + if (!precvpriv->pallocated_recv_buf) { + res = _FAIL; + goto exit; + } + + precvpriv->precv_buf = (u8 *)ALIGN((size_t)(precvpriv->pallocated_recv_buf), 4); + + precvbuf = (struct recv_buf *)precvpriv->precv_buf; + + for (i = 0; i < NR_RECVBUFF; i++) { + precvbuf->pskb = NULL; + precvbuf->reuse = false; + precvbuf->purb = usb_alloc_urb(0, GFP_KERNEL); + if (!precvbuf->purb) { + res = _FAIL; + break; + } + precvbuf->adapter = padapter; + precvbuf++; + } + precvpriv->free_recv_buf_queue_cnt = NR_RECVBUFF; + skb_queue_head_init(&precvpriv->rx_skb_queue); + { + int i; + size_t tmpaddr = 0; + size_t alignment = 0; + struct sk_buff *pskb = NULL; + + skb_queue_head_init(&precvpriv->free_recv_skb_queue); + + for (i = 0; i < NR_PREALLOC_RECV_SKB; i++) { + pskb = __netdev_alloc_skb(padapter->pnetdev, + MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ, GFP_KERNEL); + if (pskb) { + pskb->dev = padapter->pnetdev; + tmpaddr = (size_t)pskb->data; + alignment = tmpaddr & (RECVBUFF_ALIGN_SZ - 1); + skb_reserve(pskb, (RECVBUFF_ALIGN_SZ - alignment)); + + skb_queue_tail(&precvpriv->free_recv_skb_queue, pskb); + } + pskb = NULL; + } + } +exit: + return res; +} + int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter) { int i; @@ -91,6 +152,26 @@ exit: return res; } +static void rtl8188eu_free_recv_priv(struct adapter *padapter) +{ + int i; + struct recv_buf *precvbuf; + struct recv_priv *precvpriv = &padapter->recvpriv; + + precvbuf = (struct recv_buf *)precvpriv->precv_buf; + + for (i = 0; i < NR_RECVBUFF; i++) { + usb_free_urb(precvbuf->purb); + precvbuf++; + } + + kfree(precvpriv->pallocated_recv_buf); + + skb_queue_purge(&precvpriv->rx_skb_queue); + + skb_queue_purge(&precvpriv->free_recv_skb_queue); +} + void _rtw_free_recv_priv(struct recv_priv *precvpriv) { struct adapter *padapter = precvpriv->adapter; @@ -244,6 +325,42 @@ u32 rtw_free_uc_swdec_pending_queue(struct adapter *adapter) return cnt; } +static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup) +{ + union iwreq_data wrqu; + struct iw_michaelmicfailure ev; + struct mlme_priv *pmlmepriv = &padapter->mlmepriv; + struct security_priv *psecuritypriv = &padapter->securitypriv; + u32 cur_time = 0; + + if (psecuritypriv->last_mic_err_time == 0) { + psecuritypriv->last_mic_err_time = jiffies; + } else { + cur_time = jiffies; + + if (cur_time - psecuritypriv->last_mic_err_time < 60 * HZ) { + psecuritypriv->btkip_countermeasure = true; + psecuritypriv->last_mic_err_time = 0; + psecuritypriv->btkip_countermeasure_time = cur_time; + } else { + psecuritypriv->last_mic_err_time = jiffies; + } + } + + memset(&ev, 0x00, sizeof(ev)); + if (bgroup) + ev.flags |= IW_MICFAILURE_GROUP; + else + ev.flags |= IW_MICFAILURE_PAIRWISE; + + ev.src_addr.sa_family = ARPHRD_ETHER; + memcpy(ev.src_addr.sa_data, &pmlmepriv->assoc_bssid[0], ETH_ALEN); + memset(&wrqu, 0x00, sizeof(wrqu)); + wrqu.data.length = sizeof(ev); + wireless_send_event(padapter->pnetdev, IWEVMICHAELMICFAILURE, + &wrqu, (char *)&ev); +} + static int recvframe_chkmic(struct adapter *adapter, struct recv_frame *precvframe) { int i, res = _SUCCESS; @@ -1294,7 +1411,6 @@ static int amsdu_to_msdu(struct adapter *padapter, struct recv_frame *prframe) u8 nr_subframes, i; unsigned char *pdata; struct rx_pkt_attrib *pattrib; - unsigned char *data_ptr; struct sk_buff *sub_skb, *subframes[MAX_SUBFRAME_COUNT]; struct recv_priv *precvpriv = &padapter->recvpriv; @@ -1329,8 +1445,7 @@ static int amsdu_to_msdu(struct adapter *padapter, struct recv_frame *prframe) sub_skb = dev_alloc_skb(nSubframe_Length + 12); if (sub_skb) { skb_reserve(sub_skb, 12); - data_ptr = (u8 *)skb_put(sub_skb, nSubframe_Length); - memcpy(data_ptr, pdata, nSubframe_Length); + skb_put_data(sub_skb, pdata, nSubframe_Length); } else { sub_skb = skb_clone(prframe->pkt, GFP_ATOMIC); if (sub_skb) { @@ -1460,6 +1575,85 @@ static bool enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, s return true; } +static int rtw_recv_indicatepkt(struct adapter *padapter, struct recv_frame *precv_frame) +{ + struct recv_priv *precvpriv; + struct __queue *pfree_recv_queue; + struct sk_buff *skb; + struct mlme_priv *pmlmepriv = &padapter->mlmepriv; + + precvpriv = &padapter->recvpriv; + pfree_recv_queue = &precvpriv->free_recv_queue; + + skb = precv_frame->pkt; + if (!skb) + goto _recv_indicatepkt_drop; + + skb->data = precv_frame->rx_data; + + skb_set_tail_pointer(skb, precv_frame->len); + + skb->len = precv_frame->len; + + if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) { + struct sk_buff *pskb2 = NULL; + struct sta_info *psta = NULL; + struct sta_priv *pstapriv = &padapter->stapriv; + struct rx_pkt_attrib *pattrib = &precv_frame->attrib; + bool bmcast = is_multicast_ether_addr(pattrib->dst); + + if (memcmp(pattrib->dst, myid(&padapter->eeprompriv), ETH_ALEN)) { + if (bmcast) { + psta = rtw_get_bcmc_stainfo(padapter); + pskb2 = skb_clone(skb, GFP_ATOMIC); + } else { + psta = rtw_get_stainfo(pstapriv, pattrib->dst); + } + + if (psta) { + struct net_device *pnetdev; + + pnetdev = (struct net_device *)padapter->pnetdev; + skb->dev = pnetdev; + skb_set_queue_mapping(skb, rtw_recv_select_queue(skb)); + + rtw_xmit_entry(skb, pnetdev); + + if (bmcast) + skb = pskb2; + else + goto _recv_indicatepkt_end; + } + } + } + + rcu_read_lock(); + rcu_dereference(padapter->pnetdev->rx_handler_data); + rcu_read_unlock(); + + skb->ip_summed = CHECKSUM_NONE; + skb->dev = padapter->pnetdev; + skb->protocol = eth_type_trans(skb, padapter->pnetdev); + + netif_rx(skb); + +_recv_indicatepkt_end: + + /* pointers to NULL before rtw_free_recvframe() */ + precv_frame->pkt = NULL; + + rtw_free_recvframe(precv_frame, pfree_recv_queue); + + return _SUCCESS; + +_recv_indicatepkt_drop: + + /* enqueue back to free_recv_queue */ + rtw_free_recvframe(precv_frame, pfree_recv_queue); + + return _FAIL; +} + static bool recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reorder_ctrl *preorder_ctrl, int bforced) { struct list_head *phead, *plist; diff --git a/drivers/staging/r8188eu/core/rtw_sta_mgt.c b/drivers/staging/r8188eu/core/rtw_sta_mgt.c index 357f98e22d8a12104b931f42a788c301c335b530..98eeb16cab6c89dad9c1b4c3d3e0b767cb5199f4 100644 --- a/drivers/staging/r8188eu/core/rtw_sta_mgt.c +++ b/drivers/staging/r8188eu/core/rtw_sta_mgt.c @@ -5,9 +5,6 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/xmit_osdep.h" -#include "../include/mlme_osdep.h" #include "../include/sta_info.h" static void _rtw_init_stainfo(struct sta_info *psta) @@ -141,6 +138,31 @@ void _rtw_free_sta_priv(struct sta_priv *pstapriv) } } +static void _rtw_reordering_ctrl_timeout_handler(struct timer_list *t) +{ + struct recv_reorder_ctrl *preorder_ctrl; + + preorder_ctrl = from_timer(preorder_ctrl, t, reordering_ctrl_timer); + rtw_reordering_ctrl_timeout_handler(preorder_ctrl); +} + +static void rtw_init_recv_timer(struct recv_reorder_ctrl *preorder_ctrl) +{ + timer_setup(&preorder_ctrl->reordering_ctrl_timer, _rtw_reordering_ctrl_timeout_handler, 0); +} + +static void _addba_timer_hdl(struct timer_list *t) +{ + struct sta_info *psta = from_timer(psta, t, addba_retry_timer); + + addba_timer_hdl(psta); +} + +static void init_addba_retry_timer(struct adapter *padapter, struct sta_info *psta) +{ + timer_setup(&psta->addba_retry_timer, _addba_timer_hdl, 0); +} + struct sta_info *rtw_alloc_stainfo(struct sta_priv *pstapriv, u8 *hwaddr) { s32 index; diff --git a/drivers/staging/r8188eu/core/rtw_wlan_util.c b/drivers/staging/r8188eu/core/rtw_wlan_util.c index 3a002cb6834fa0199a13e7ba806bca106b9fef21..e50631848cab6bba2b13def50e0816e87bd4c933 100644 --- a/drivers/staging/r8188eu/core/rtw_wlan_util.c +++ b/drivers/staging/r8188eu/core/rtw_wlan_util.c @@ -222,46 +222,6 @@ void get_rate_set(struct adapter *padapter, unsigned char *pbssrate, int *bssrat memcpy(pbssrate, supportedrates, *bssrate_len); } -void UpdateBrateTbl(struct adapter *Adapter, u8 *mbrate) -{ - u8 i; - u8 rate; - - /* 1M, 2M, 5.5M, 11M, 6M, 12M, 24M are mandatory. */ - for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) { - rate = mbrate[i] & 0x7f; - switch (rate) { - case IEEE80211_CCK_RATE_1MB: - case IEEE80211_CCK_RATE_2MB: - case IEEE80211_CCK_RATE_5MB: - case IEEE80211_CCK_RATE_11MB: - case IEEE80211_OFDM_RATE_6MB: - case IEEE80211_OFDM_RATE_12MB: - case IEEE80211_OFDM_RATE_24MB: - mbrate[i] |= IEEE80211_BASIC_RATE_MASK; - break; - } - } -} - -void UpdateBrateTblForSoftAP(u8 *bssrateset, u32 bssratelen) -{ - u8 i; - u8 rate; - - for (i = 0; i < bssratelen; i++) { - rate = bssrateset[i] & 0x7f; - switch (rate) { - case IEEE80211_CCK_RATE_1MB: - case IEEE80211_CCK_RATE_2MB: - case IEEE80211_CCK_RATE_5MB: - case IEEE80211_CCK_RATE_11MB: - bssrateset[i] |= IEEE80211_BASIC_RATE_MASK; - break; - } - } -} - void Save_DM_Func_Flag(struct adapter *padapter) { struct hal_data_8188e *haldata = &padapter->haldata; @@ -1578,10 +1538,8 @@ void beacon_timing_control(struct adapter *padapter) static struct adapter *pbuddy_padapter; -int rtw_handle_dualmac(struct adapter *adapter, bool init) +void rtw_handle_dualmac(struct adapter *adapter, bool init) { - int status = _SUCCESS; - if (init) { if (!pbuddy_padapter) { pbuddy_padapter = adapter; @@ -1594,5 +1552,4 @@ int rtw_handle_dualmac(struct adapter *adapter, bool init) } else { pbuddy_padapter = NULL; } - return status; } diff --git a/drivers/staging/r8188eu/core/rtw_xmit.c b/drivers/staging/r8188eu/core/rtw_xmit.c index 24401f3ae2a050e23ee0a5692e589147fddd4702..873d2c5c36344a761f7eb0ae0a40b58957a1dad5 100644 --- a/drivers/staging/r8188eu/core/rtw_xmit.c +++ b/drivers/staging/r8188eu/core/rtw_xmit.c @@ -33,6 +33,32 @@ void _rtw_init_sta_xmit_priv(struct sta_xmit_priv *psta_xmitpriv) INIT_LIST_HEAD(&psta_xmitpriv->apsd); } +static int rtw_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, + u32 alloc_sz) +{ + pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL); + if (!pxmitbuf->pallocated_buf) + return _FAIL; + + pxmitbuf->pbuf = (u8 *)ALIGN((size_t)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ); + pxmitbuf->dma_transfer_addr = 0; + + pxmitbuf->pxmit_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!pxmitbuf->pxmit_urb) { + kfree(pxmitbuf->pallocated_buf); + return _FAIL; + } + + return _SUCCESS; +} + +static void rtw_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, + u32 free_sz) +{ + usb_free_urb(pxmitbuf->pxmit_urb); + kfree(pxmitbuf->pallocated_buf); +} + s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) { int i; @@ -108,7 +134,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) if (!pxmitpriv->pallocated_xmitbuf) { res = _FAIL; - goto exit; + goto free_frame_buf; } pxmitpriv->pxmitbuf = (u8 *)ALIGN((size_t)(pxmitpriv->pallocated_xmitbuf), 4); @@ -125,12 +151,12 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) pxmitbuf->ext_tag = false; /* Tx buf allocation may fail sometimes, so sleep and retry. */ - res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); + res = rtw_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); if (res == _FAIL) { msleep(10); - res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); + res = rtw_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); if (res == _FAIL) - goto exit; + goto free_xmitbuf; } pxmitbuf->flags = XMIT_VO_QUEUE; @@ -148,7 +174,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) if (!pxmitpriv->pallocated_xmit_extbuf) { res = _FAIL; - goto exit; + goto free_xmitbuf; } pxmitpriv->pxmit_extbuf = (u8 *)ALIGN((size_t)(pxmitpriv->pallocated_xmit_extbuf), 4); @@ -162,10 +188,10 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) pxmitbuf->padapter = padapter; pxmitbuf->ext_tag = true; - res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, max_xmit_extbuf_size + XMITBUF_ALIGN_SZ); + res = rtw_xmit_resource_alloc(padapter, pxmitbuf, max_xmit_extbuf_size + XMITBUF_ALIGN_SZ); if (res == _FAIL) { res = _FAIL; - goto exit; + goto free_xmit_extbuf; } list_add_tail(&pxmitbuf->list, &pxmitpriv->free_xmit_extbuf_queue.queue); @@ -176,7 +202,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) if (rtw_alloc_hwxmits(padapter)) { res = _FAIL; - goto exit; + goto free_xmit_extbuf; } rtw_init_hwxmits(pxmitpriv->hwxmits, pxmitpriv->hwxmit_entry); @@ -200,11 +226,54 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) rtl8188eu_init_xmit_priv(padapter); -exit: + return _SUCCESS; +free_xmit_extbuf: + pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf; + while (i--) { + rtw_xmit_resource_free(padapter, pxmitbuf, (max_xmit_extbuf_size + XMITBUF_ALIGN_SZ)); + pxmitbuf++; + } + vfree(pxmitpriv->pallocated_xmit_extbuf); + i = NR_XMITBUFF; +free_xmitbuf: + pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf; + while (i--) { + rtw_xmit_resource_free(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); + pxmitbuf++; + } + vfree(pxmitpriv->pallocated_xmitbuf); +free_frame_buf: + vfree(pxmitpriv->pallocated_frame_buf); +exit: return res; } +static void rtw_pkt_complete(struct adapter *padapter, struct sk_buff *pkt) +{ + u16 queue; + struct xmit_priv *pxmitpriv = &padapter->xmitpriv; + + queue = skb_get_queue_mapping(pkt); + if (padapter->registrypriv.wifi_spec) { + if (__netif_subqueue_stopped(padapter->pnetdev, queue) && + (pxmitpriv->hwxmits[queue].accnt < WMM_XMIT_THRESHOLD)) + netif_wake_subqueue(padapter->pnetdev, queue); + } else { + if (__netif_subqueue_stopped(padapter->pnetdev, queue)) + netif_wake_subqueue(padapter->pnetdev, queue); + } + + dev_kfree_skb_any(pkt); +} + +void rtw_xmit_complete(struct adapter *padapter, struct xmit_frame *pxframe) +{ + if (pxframe->pkt) + rtw_pkt_complete(padapter, pxframe->pkt); + pxframe->pkt = NULL; +} + void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv) { int i; @@ -218,13 +287,13 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv) return; for (i = 0; i < NR_XMITFRAME; i++) { - rtw_os_xmit_complete(padapter, pxmitframe); + rtw_xmit_complete(padapter, pxmitframe); pxmitframe++; } for (i = 0; i < NR_XMITBUFF; i++) { - rtw_os_xmit_resource_free(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); + rtw_xmit_resource_free(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ)); pxmitbuf++; } @@ -234,7 +303,7 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv) pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmit_extbuf; for (i = 0; i < num_xmit_extbuf; i++) { - rtw_os_xmit_resource_free(padapter, pxmitbuf, (max_xmit_extbuf_size + XMITBUF_ALIGN_SZ)); + rtw_xmit_resource_free(padapter, pxmitbuf, (max_xmit_extbuf_size + XMITBUF_ALIGN_SZ)); pxmitbuf++; } @@ -378,18 +447,59 @@ u8 qos_acm(u8 acm_mask, u8 priority) return change_priority; } +static void rtw_open_pktfile(struct sk_buff *pktptr, struct pkt_file *pfile) +{ + if (!pktptr) { + pr_err("8188eu: pktptr is NULL\n"); + return; + } + if (!pfile) { + pr_err("8188eu: pfile is NULL\n"); + return; + } + pfile->pkt = pktptr; + pfile->cur_addr = pktptr->data; + pfile->buf_start = pktptr->data; + pfile->pkt_len = pktptr->len; + pfile->buf_len = pktptr->len; + + pfile->cur_buffer = pfile->buf_start; +} + +static uint rtw_remainder_len(struct pkt_file *pfile) +{ + return pfile->buf_len - ((size_t)(pfile->cur_addr) - + (size_t)(pfile->buf_start)); +} + +static uint rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen) +{ + uint len; + + len = rtw_remainder_len(pfile); + len = (rlen > len) ? len : rlen; + + if (rmem) + skb_copy_bits(pfile->pkt, pfile->buf_len - pfile->pkt_len, rmem, len); + + pfile->cur_addr += len; + pfile->pkt_len -= len; + + return len; +} + static void set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib) { struct ethhdr etherhdr; struct iphdr ip_hdr; s32 user_prio = 0; - _rtw_open_pktfile(ppktfile->pkt, ppktfile); - _rtw_pktfile_read(ppktfile, (unsigned char *)ðerhdr, ETH_HLEN); + rtw_open_pktfile(ppktfile->pkt, ppktfile); + rtw_pktfile_read(ppktfile, (unsigned char *)ðerhdr, ETH_HLEN); /* get user_prio from IP hdr */ if (pattrib->ether_type == 0x0800) { - _rtw_pktfile_read(ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr)); + rtw_pktfile_read(ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr)); /* user_prio = (ntohs(ip_hdr.tos) >> 5) & 0x3; */ user_prio = ip_hdr.tos >> 5; } else if (pattrib->ether_type == 0x888e) { @@ -418,8 +528,8 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p - _rtw_open_pktfile(pkt, &pktfile); - _rtw_pktfile_read(&pktfile, (u8 *)ðerhdr, ETH_HLEN); + rtw_open_pktfile(pkt, &pktfile); + rtw_pktfile_read(&pktfile, (u8 *)ðerhdr, ETH_HLEN); pattrib->ether_type = ntohs(etherhdr.h_proto); @@ -447,7 +557,7 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p /* to prevent DHCP protocol fail */ u8 tmp[24]; - _rtw_pktfile_read(&pktfile, &tmp[0], 24); + rtw_pktfile_read(&pktfile, &tmp[0], 24); pattrib->dhcp_pkt = 0; if (pktfile.pkt_len > 282) {/* MINIMUM_DHCP_PACKET_SIZE) { */ if (((tmp[21] == 68) && (tmp[23] == 67)) || @@ -460,9 +570,6 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p } } - if ((pattrib->ether_type == 0x888e) || (pattrib->dhcp_pkt == 1)) - rtw_set_scan_deny(padapter, 3000); - /* If EAPOL , ARP , OR DHCP packet, driver must be in active mode. */ if ((pattrib->ether_type == 0x0806) || (pattrib->ether_type == 0x888e) || (pattrib->dhcp_pkt == 1)) rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_SPECIAL_PACKET, 1); @@ -897,8 +1004,8 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct goto exit; } - _rtw_open_pktfile(pkt, &pktfile); - _rtw_pktfile_read(&pktfile, NULL, pattrib->pkt_hdrlen); + rtw_open_pktfile(pkt, &pktfile); + rtw_pktfile_read(&pktfile, NULL, pattrib->pkt_hdrlen); frg_inx = 0; frg_len = pxmitpriv->frag_len - 4;/* 2346-4 = 2342 */ @@ -956,9 +1063,9 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct if (bmcst) { /* don't do fragment to broadcast/multicast packets */ - mem_sz = _rtw_pktfile_read(&pktfile, pframe, pattrib->pktlen); + mem_sz = rtw_pktfile_read(&pktfile, pframe, pattrib->pktlen); } else { - mem_sz = _rtw_pktfile_read(&pktfile, pframe, mpdu_len); + mem_sz = rtw_pktfile_read(&pktfile, pframe, mpdu_len); } pframe += mem_sz; @@ -970,7 +1077,7 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct frg_inx++; - if (bmcst || rtw_endofpktfile(&pktfile)) { + if (bmcst || pktfile.pkt_len == 0) { pattrib->nr_frags = frg_inx; pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len + ((pattrib->nr_frags == 1) ? llc_sz : 0) + @@ -1286,7 +1393,7 @@ s32 rtw_free_xmitframe(struct xmit_priv *pxmitpriv, struct xmit_frame *pxmitfram spin_unlock_bh(&pfree_xmit_queue->lock); if (pndis_pkt) - rtw_os_pkt_complete(padapter, pndis_pkt); + rtw_pkt_complete(padapter, pndis_pkt); exit: @@ -1945,7 +2052,7 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta) spin_unlock_bh(&psta->sleep_q.lock); if (rtl8188eu_hal_xmit(padapter, pxmitframe)) - rtw_os_xmit_complete(padapter, pxmitframe); + rtw_xmit_complete(padapter, pxmitframe); spin_lock_bh(&psta->sleep_q.lock); } @@ -1995,7 +2102,7 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta) spin_unlock_bh(&psta_bmc->sleep_q.lock); if (rtl8188eu_hal_xmit(padapter, pxmitframe)) - rtw_os_xmit_complete(padapter, pxmitframe); + rtw_xmit_complete(padapter, pxmitframe); spin_lock_bh(&psta_bmc->sleep_q.lock); } @@ -2069,7 +2176,7 @@ void xmit_delivery_enabled_frames(struct adapter *padapter, struct sta_info *pst pxmitframe->attrib.triggered = 1; if (rtl8188eu_hal_xmit(padapter, pxmitframe)) - rtw_os_xmit_complete(padapter, pxmitframe); + rtw_xmit_complete(padapter, pxmitframe); if ((psta->sleepq_ac_len == 0) && (!psta->has_legacy_ac) && (wmmps_ac)) { pstapriv->tim_bitmap &= ~BIT(psta->aid); @@ -2136,3 +2243,105 @@ void rtw_ack_tx_done(struct xmit_priv *pxmitpriv, int status) if (pxmitpriv->ack_tx) rtw_sctx_done_err(&pack_tx_ops, status); } + +static void rtw_check_xmit_resource(struct adapter *padapter, struct sk_buff *pkt) +{ + struct xmit_priv *pxmitpriv = &padapter->xmitpriv; + u16 queue; + + queue = skb_get_queue_mapping(pkt); + if (padapter->registrypriv.wifi_spec) { + /* No free space for Tx, tx_worker is too slow */ + if (pxmitpriv->hwxmits[queue].accnt > WMM_XMIT_THRESHOLD) + netif_stop_subqueue(padapter->pnetdev, queue); + } else { + if (pxmitpriv->free_xmitframe_cnt <= 4) { + if (!netif_tx_queue_stopped(netdev_get_tx_queue(padapter->pnetdev, queue))) + netif_stop_subqueue(padapter->pnetdev, queue); + } + } +} + +static int rtw_mlcst2unicst(struct adapter *padapter, struct sk_buff *skb) +{ + struct sta_priv *pstapriv = &padapter->stapriv; + struct xmit_priv *pxmitpriv = &padapter->xmitpriv; + struct list_head *phead, *plist; + struct sk_buff *newskb; + struct sta_info *psta = NULL; + s32 res; + + spin_lock_bh(&pstapriv->asoc_list_lock); + phead = &pstapriv->asoc_list; + plist = phead->next; + + /* free sta asoc_queue */ + while (phead != plist) { + psta = container_of(plist, struct sta_info, asoc_list); + + plist = plist->next; + + /* avoid come from STA1 and send back STA1 */ + if (!memcmp(psta->hwaddr, &skb->data[6], 6)) + continue; + + newskb = skb_copy(skb, GFP_ATOMIC); + + if (newskb) { + memcpy(newskb->data, psta->hwaddr, 6); + res = rtw_xmit(padapter, &newskb); + if (res < 0) { + pxmitpriv->tx_drop++; + dev_kfree_skb_any(newskb); + } else { + pxmitpriv->tx_pkts++; + } + } else { + pxmitpriv->tx_drop++; + + spin_unlock_bh(&pstapriv->asoc_list_lock); + return false; /* Caller shall tx this multicast frame via normal way. */ + } + } + + spin_unlock_bh(&pstapriv->asoc_list_lock); + dev_kfree_skb_any(skb); + return true; +} + +netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) +{ + struct adapter *padapter = (struct adapter *)rtw_netdev_priv(pnetdev); + struct xmit_priv *pxmitpriv = &padapter->xmitpriv; + struct mlme_priv *pmlmepriv = &padapter->mlmepriv; + s32 res = 0; + + if (!rtw_if_up(padapter)) + goto drop_packet; + + rtw_check_xmit_resource(padapter, pkt); + + if (!rtw_mc2u_disable && check_fwstate(pmlmepriv, WIFI_AP_STATE) && + (IP_MCAST_MAC(pkt->data) || ICMPV6_MCAST_MAC(pkt->data)) && + (padapter->registrypriv.wifi_spec == 0)) { + if (pxmitpriv->free_xmitframe_cnt > (NR_XMITFRAME / 4)) { + res = rtw_mlcst2unicst(padapter, pkt); + if (res) + goto exit; + } + } + + res = rtw_xmit(padapter, &pkt); + if (res < 0) + goto drop_packet; + + pxmitpriv->tx_pkts++; + goto exit; + +drop_packet: + pxmitpriv->tx_drop++; + dev_kfree_skb_any(pkt); + +exit: + return NETDEV_TX_OK; +} diff --git a/drivers/staging/r8188eu/hal/HalHWImg8188E_BB.c b/drivers/staging/r8188eu/hal/HalHWImg8188E_BB.c index 7901d0afa2e772134769e44ccc3142f681a8562b..23b7205722b5b1cfea44dcc0c6afa951ec763832 100644 --- a/drivers/staging/r8188eu/hal/HalHWImg8188E_BB.c +++ b/drivers/staging/r8188eu/hal/HalHWImg8188E_BB.c @@ -166,7 +166,14 @@ static u32 array_agc_tab_1t_8188e[] = { 0xC78, 0x407F0001, }; -enum HAL_STATUS ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *dm_odm) +static void odm_ConfigBB_AGC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Bitmask, u32 Data) +{ + rtl8188e_PHY_SetBBReg(pDM_Odm->Adapter, Addr, Bitmask, Data); + /* Add 1us delay between BB/RF register setting. */ + udelay(1); +} + +int ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *dm_odm) { u32 hex = 0; u32 i = 0; @@ -176,7 +183,6 @@ enum HAL_STATUS ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *dm_odm) struct adapter *adapter = dm_odm->Adapter; struct xmit_frame *pxmit_frame = NULL; u8 bndy_cnt = 1; - enum HAL_STATUS rst = HAL_STATUS_SUCCESS; hex += ODM_ITRF_USB << 8; hex += ODM_CE << 16; @@ -187,7 +193,7 @@ enum HAL_STATUS ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *dm_odm) pxmit_frame = rtw_IOL_accquire_xmit_frame(adapter); if (!pxmit_frame) { pr_info("rtw_IOL_accquire_xmit_frame failed\n"); - return HAL_STATUS_FAILURE; + return -ENOMEM; } } @@ -238,10 +244,10 @@ enum HAL_STATUS ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *dm_odm) if (biol) { if (!rtl8188e_IOL_exec_cmds_sync(dm_odm->Adapter, pxmit_frame, 1000, bndy_cnt)) { printk("~~~ %s IOL_exec_cmds Failed !!!\n", __func__); - rst = HAL_STATUS_FAILURE; + return -1; } } - return rst; + return 0; } /****************************************************************************** @@ -442,7 +448,31 @@ static u32 array_phy_reg_1t_8188e[] = { 0xF00, 0x00000300, }; -enum HAL_STATUS ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *dm_odm) +static void odm_ConfigBB_PHY_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Bitmask, u32 Data) +{ + if (Addr == 0xfe) { + msleep(50); + } else if (Addr == 0xfd) { + mdelay(5); + } else if (Addr == 0xfc) { + mdelay(1); + } else if (Addr == 0xfb) { + udelay(50); + } else if (Addr == 0xfa) { + udelay(5); + } else if (Addr == 0xf9) { + udelay(1); + } else { + if (Addr == 0xa24) + pDM_Odm->RFCalibrateInfo.RegA24 = Data; + rtl8188e_PHY_SetBBReg(pDM_Odm->Adapter, Addr, Bitmask, Data); + + /* Add 1us delay between BB/RF register setting. */ + udelay(1); + } +} + +int ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *dm_odm) { u32 hex = 0; u32 i = 0; @@ -452,7 +482,6 @@ enum HAL_STATUS ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *dm_odm) struct adapter *adapter = dm_odm->Adapter; struct xmit_frame *pxmit_frame = NULL; u8 bndy_cnt = 1; - enum HAL_STATUS rst = HAL_STATUS_SUCCESS; hex += ODM_ITRF_USB << 8; hex += ODM_CE << 16; hex += 0xFF000000; @@ -462,7 +491,7 @@ enum HAL_STATUS ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *dm_odm) pxmit_frame = rtw_IOL_accquire_xmit_frame(adapter); if (!pxmit_frame) { pr_info("rtw_IOL_accquire_xmit_frame failed\n"); - return HAL_STATUS_FAILURE; + return -ENOMEM; } } @@ -544,11 +573,11 @@ enum HAL_STATUS ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *dm_odm) } if (biol) { if (!rtl8188e_IOL_exec_cmds_sync(dm_odm->Adapter, pxmit_frame, 1000, bndy_cnt)) { - rst = HAL_STATUS_FAILURE; pr_info("~~~ IOL Config %s Failed !!!\n", __func__); + return -1; } } - return rst; + return 0; } /****************************************************************************** @@ -647,6 +676,25 @@ static u32 array_phy_reg_pg_8188e[] = { }; +static void odm_ConfigBB_PHY_REG_PG_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Bitmask, + u32 Data) +{ + if (Addr == 0xfe) + msleep(50); + else if (Addr == 0xfd) + mdelay(5); + else if (Addr == 0xfc) + mdelay(1); + else if (Addr == 0xfb) + udelay(50); + else if (Addr == 0xfa) + udelay(5); + else if (Addr == 0xf9) + udelay(1); + else + storePwrIndexDiffRateOffset(pDM_Odm->Adapter, Addr, Bitmask, Data); +} + void ODM_ReadAndConfig_PHY_REG_PG_8188E(struct odm_dm_struct *dm_odm) { u32 hex; diff --git a/drivers/staging/r8188eu/hal/HalHWImg8188E_MAC.c b/drivers/staging/r8188eu/hal/HalHWImg8188E_MAC.c index 77b25885c63b718ac7d2e4c4e27673db90c69371..da71867bcca3be60139e275eb130dad57cc8e747 100644 --- a/drivers/staging/r8188eu/hal/HalHWImg8188E_MAC.c +++ b/drivers/staging/r8188eu/hal/HalHWImg8188E_MAC.c @@ -126,7 +126,12 @@ static u32 array_MAC_REG_8188E[] = { 0x70B, 0x00000087, }; -enum HAL_STATUS ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *dm_odm) +static void odm_ConfigMAC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u8 Data) +{ + rtw_write8(pDM_Odm->Adapter, Addr, Data); +} + +int ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *dm_odm) { #define READ_NEXT_PAIR(v1, v2, i) do { i += 2; v1 = array[i]; v2 = array[i + 1]; } while (0) @@ -139,7 +144,6 @@ enum HAL_STATUS ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *dm_odm) struct adapter *adapt = dm_odm->Adapter; struct xmit_frame *pxmit_frame = NULL; u8 bndy_cnt = 1; - enum HAL_STATUS rst = HAL_STATUS_SUCCESS; hex += ODM_ITRF_USB << 8; hex += ODM_CE << 16; hex += 0xFF000000; @@ -150,7 +154,7 @@ enum HAL_STATUS ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *dm_odm) pxmit_frame = rtw_IOL_accquire_xmit_frame(adapt); if (!pxmit_frame) { pr_info("rtw_IOL_accquire_xmit_frame failed\n"); - return HAL_STATUS_FAILURE; + return -ENOMEM; } } @@ -201,8 +205,8 @@ enum HAL_STATUS ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *dm_odm) if (biol) { if (!rtl8188e_IOL_exec_cmds_sync(dm_odm->Adapter, pxmit_frame, 1000, bndy_cnt)) { pr_info("~~~ MAC IOL_exec_cmds Failed !!!\n"); - rst = HAL_STATUS_FAILURE; + return -1; } } - return rst; + return 0; } diff --git a/drivers/staging/r8188eu/hal/HalHWImg8188E_RF.c b/drivers/staging/r8188eu/hal/HalHWImg8188E_RF.c index 08cbfce3808de2be44ebaeb622ce27c993157e54..a4c3d3d149f743132fa8e88c5130f60853f06e94 100644 --- a/drivers/staging/r8188eu/hal/HalHWImg8188E_RF.c +++ b/drivers/staging/r8188eu/hal/HalHWImg8188E_RF.c @@ -130,7 +130,37 @@ static u32 Array_RadioA_1T_8188E[] = { 0x000, 0x00033E60, }; -enum HAL_STATUS ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *pDM_Odm) +static void odm_ConfigRFReg_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, + u32 Data, u32 RegAddr) +{ + if (Addr == 0xffe) { + msleep(50); + } else if (Addr == 0xfd) { + mdelay(5); + } else if (Addr == 0xfc) { + mdelay(1); + } else if (Addr == 0xfb) { + udelay(50); + } else if (Addr == 0xfa) { + udelay(5); + } else if (Addr == 0xf9) { + udelay(1); + } else { + rtl8188e_PHY_SetRFReg(pDM_Odm->Adapter, RegAddr, bRFRegOffsetMask, Data); + /* Add 1us delay between BB/RF register setting. */ + udelay(1); + } +} + +static void odm_ConfigRF_RadioA_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Data) +{ + u32 content = 0x1000; /* RF_Content: radioa_txt */ + u32 maskforPhySet = (u32)(content & 0xE000); + + odm_ConfigRFReg_8188E(pDM_Odm, Addr, Data, Addr | maskforPhySet); +} + +int ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *pDM_Odm) { #define READ_NEXT_PAIR(v1, v2, i) do \ { i += 2; v1 = Array[i]; \ @@ -144,7 +174,6 @@ enum HAL_STATUS ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *pDM_Odm) struct adapter *Adapter = pDM_Odm->Adapter; struct xmit_frame *pxmit_frame = NULL; u8 bndy_cnt = 1; - enum HAL_STATUS rst = HAL_STATUS_SUCCESS; hex += ODM_ITRF_USB << 8; hex += ODM_CE << 16; @@ -155,7 +184,7 @@ enum HAL_STATUS ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *pDM_Odm) pxmit_frame = rtw_IOL_accquire_xmit_frame(Adapter); if (!pxmit_frame) { pr_info("rtw_IOL_accquire_xmit_frame failed\n"); - return HAL_STATUS_FAILURE; + return -ENOMEM; } } @@ -232,9 +261,9 @@ enum HAL_STATUS ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *pDM_Odm) } if (biol) { if (!rtl8188e_IOL_exec_cmds_sync(pDM_Odm->Adapter, pxmit_frame, 1000, bndy_cnt)) { - rst = HAL_STATUS_FAILURE; pr_info("~~~ IOL Config %s Failed !!!\n", __func__); + return -1; } } - return rst; + return 0; } diff --git a/drivers/staging/r8188eu/hal/hal_com.c b/drivers/staging/r8188eu/hal/hal_com.c index 6a1cdc67335b37ea8b21b31458c4c2e2e4649f09..33967eb3c0d0e8cbb7f84b9665ba63c4f22c6c1f 100644 --- a/drivers/staging/r8188eu/hal/hal_com.c +++ b/drivers/staging/r8188eu/hal/hal_com.c @@ -137,176 +137,3 @@ void HalSetBrateCfg(struct adapter *adapt, u8 *brates, u16 *rate_cfg) } } } - -static void one_out_pipe(struct adapter *adapter) -{ - struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); - - pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ - pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];/* VI */ - pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[0];/* BE */ - pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0];/* BK */ - - pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ - pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ - pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ - pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ -} - -static void two_out_pipe(struct adapter *adapter, bool wifi_cfg) -{ - struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); - - if (wifi_cfg) { /* WMM */ - /* BK, BE, VI, VO, BCN, CMD, MGT, HIGH, HCCA */ - /* 0, 1, 0, 1, 0, 0, 0, 0, 0}; */ - /* 0:H, 1:L */ - - pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[1];/* VO */ - pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];/* VI */ - pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[1];/* BE */ - pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0];/* BK */ - - pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ - pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ - pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ - pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ - - } else {/* typical setting */ - /* BK, BE, VI, VO, BCN, CMD, MGT, HIGH, HCCA */ - /* 1, 1, 0, 0, 0, 0, 0, 0, 0}; */ - /* 0:H, 1:L */ - - pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ - pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];/* VI */ - pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[1];/* BE */ - pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[1];/* BK */ - - pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ - pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ - pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ - pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ - } -} - -static void three_out_pipe(struct adapter *adapter, bool wifi_cfg) -{ - struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); - - if (wifi_cfg) {/* for WMM */ - /* BK, BE, VI, VO, BCN, CMD, MGT, HIGH, HCCA */ - /* 1, 2, 1, 0, 0, 0, 0, 0, 0}; */ - /* 0:H, 1:N, 2:L */ - - pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ - pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[1];/* VI */ - pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[2];/* BE */ - pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[1];/* BK */ - - pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ - pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ - pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ - pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ - - } else {/* typical setting */ - /* BK, BE, VI, VO, BCN, CMD, MGT, HIGH, HCCA */ - /* 2, 2, 1, 0, 0, 0, 0, 0, 0}; */ - /* 0:H, 1:N, 2:L */ - - pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ - pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[1];/* VI */ - pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[2];/* BE */ - pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[2];/* BK */ - - pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ - pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ - pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ - pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ - } -} - -bool Hal_MappingOutPipe(struct adapter *adapter, u8 numoutpipe) -{ - struct registry_priv *pregistrypriv = &adapter->registrypriv; - bool wifi_cfg = pregistrypriv->wifi_spec; - bool result = true; - - switch (numoutpipe) { - case 2: - two_out_pipe(adapter, wifi_cfg); - break; - case 3: - three_out_pipe(adapter, wifi_cfg); - break; - case 1: - one_out_pipe(adapter); - break; - default: - result = false; - break; - } - return result; -} - -/* -* C2H event format: -* Field TRIGGER CONTENT CMD_SEQ CMD_LEN CMD_ID -* BITS [127:120] [119:16] [15:8] [7:4] [3:0] -*/ - -s32 c2h_evt_read(struct adapter *adapter, u8 *buf) -{ - s32 ret = _FAIL; - struct c2h_evt_hdr *c2h_evt; - int i; - u8 trigger; - - if (!buf) - goto exit; - - ret = rtw_read8(adapter, REG_C2HEVT_CLEAR, &trigger); - if (ret) - return _FAIL; - - if (trigger == C2H_EVT_HOST_CLOSE) - goto exit; /* Not ready */ - else if (trigger != C2H_EVT_FW_CLOSE) - goto clear_evt; /* Not a valid value */ - - c2h_evt = (struct c2h_evt_hdr *)buf; - - memset(c2h_evt, 0, 16); - - ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL, buf); - if (ret) { - ret = _FAIL; - goto clear_evt; - } - - ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL + 1, buf + 1); - if (ret) { - ret = _FAIL; - goto clear_evt; - } - /* Read the content */ - for (i = 0; i < c2h_evt->plen; i++) { - ret = rtw_read8(adapter, REG_C2HEVT_MSG_NORMAL + - sizeof(*c2h_evt) + i, c2h_evt->payload + i); - if (ret) { - ret = _FAIL; - goto clear_evt; - } - } - - ret = _SUCCESS; - -clear_evt: - /* - * Clear event to notify FW we have read the command. - * If this field isn't clear, the FW won't update the next - * command message. - */ - rtw_write8(adapter, REG_C2HEVT_CLEAR, C2H_EVT_HOST_CLOSE); -exit: - return ret; -} diff --git a/drivers/staging/r8188eu/hal/odm_HWConfig.c b/drivers/staging/r8188eu/hal/odm_HWConfig.c index 54cc3d7789cd406779e217b124893818efa7d4c7..38f357e8aedaa8ce88e4816cebc0d3891265ac2e 100644 --- a/drivers/staging/r8188eu/hal/odm_HWConfig.c +++ b/drivers/staging/r8188eu/hal/odm_HWConfig.c @@ -3,38 +3,38 @@ #include "../include/drv_types.h" -static u8 odm_QueryRxPwrPercentage(s8 AntPower) +static u8 odm_query_rxpwrpercentage(s8 antpower) { - if ((AntPower <= -100) || (AntPower >= 20)) - return 0; - else if (AntPower >= 0) - return 100; + if ((antpower <= -100) || (antpower >= 20)) + return 0; + else if (antpower >= 0) + return 100; else - return 100 + AntPower; + return 100 + antpower; } -static s32 odm_SignalScaleMapping(struct odm_dm_struct *dm_odm, s32 CurrSig) +static s32 odm_signal_scale_mapping(struct odm_dm_struct *dm_odm, s32 currsig) { - s32 RetSig = 0; - - if (CurrSig >= 51 && CurrSig <= 100) - RetSig = 100; - else if (CurrSig >= 41 && CurrSig <= 50) - RetSig = 80 + ((CurrSig - 40) * 2); - else if (CurrSig >= 31 && CurrSig <= 40) - RetSig = 66 + (CurrSig - 30); - else if (CurrSig >= 21 && CurrSig <= 30) - RetSig = 54 + (CurrSig - 20); - else if (CurrSig >= 10 && CurrSig <= 20) - RetSig = 42 + (((CurrSig - 10) * 2) / 3); - else if (CurrSig >= 5 && CurrSig <= 9) - RetSig = 22 + (((CurrSig - 5) * 3) / 2); - else if (CurrSig >= 1 && CurrSig <= 4) - RetSig = 6 + (((CurrSig - 1) * 3) / 2); + s32 retsig; + + if (currsig >= 51 && currsig <= 100) + retsig = 100; + else if (currsig >= 41 && currsig <= 50) + retsig = 80 + ((currsig - 40) * 2); + else if (currsig >= 31 && currsig <= 40) + retsig = 66 + (currsig - 30); + else if (currsig >= 21 && currsig <= 30) + retsig = 54 + (currsig - 20); + else if (currsig >= 10 && currsig <= 20) + retsig = 42 + (((currsig - 10) * 2) / 3); + else if (currsig >= 5 && currsig <= 9) + retsig = 22 + (((currsig - 5) * 3) / 2); + else if (currsig >= 1 && currsig <= 4) + retsig = 6 + (((currsig - 1) * 3) / 2); else - RetSig = CurrSig; + retsig = currsig; - return RetSig; + return retsig; } static u8 odm_evm_db_to_percentage(s8 value) @@ -117,7 +117,7 @@ static void odm_RxPhyStatus92CSeries_Parsing(struct odm_dm_struct *dm_odm, break; } rx_pwr_all += 6; - PWDB_ALL = odm_QueryRxPwrPercentage(rx_pwr_all); + PWDB_ALL = odm_query_rxpwrpercentage(rx_pwr_all); if (!cck_highpwr) { if (PWDB_ALL >= 80) PWDB_ALL = ((PWDB_ALL - 80) << 1) + ((PWDB_ALL - 80) >> 1) + 80; @@ -162,7 +162,7 @@ static void odm_RxPhyStatus92CSeries_Parsing(struct odm_dm_struct *dm_odm, pPhyInfo->RxPwr[i] = rx_pwr[i]; /* Translate DBM to percentage. */ - RSSI = odm_QueryRxPwrPercentage(rx_pwr[i]); + RSSI = odm_query_rxpwrpercentage(rx_pwr[i]); total_rssi += RSSI; pPhyInfo->RxMIMOSignalStrength[i] = (u8)RSSI; @@ -173,7 +173,7 @@ static void odm_RxPhyStatus92CSeries_Parsing(struct odm_dm_struct *dm_odm, /* (2)PWDB, Average PWDB calculated by hardware (for rate adaptive) */ rx_pwr_all = (((pPhyStaRpt->cck_sig_qual_ofdm_pwdb_all) >> 1) & 0x7f) - 110; - PWDB_ALL = odm_QueryRxPwrPercentage(rx_pwr_all); + PWDB_ALL = odm_query_rxpwrpercentage(rx_pwr_all); pPhyInfo->RxPWDBAll = PWDB_ALL; pPhyInfo->RxPower = rx_pwr_all; @@ -200,10 +200,10 @@ static void odm_RxPhyStatus92CSeries_Parsing(struct odm_dm_struct *dm_odm, /* UI BSS List signal strength(in percentage), make it good looking, from 0~100. */ /* It is assigned to the BSS List in GetValueFromBeaconOrProbeRsp(). */ if (isCCKrate) { - pPhyInfo->SignalStrength = (u8)(odm_SignalScaleMapping(dm_odm, PWDB_ALL));/* PWDB_ALL; */ + pPhyInfo->SignalStrength = (u8)(odm_signal_scale_mapping(dm_odm, PWDB_ALL));/* PWDB_ALL; */ } else { if (rf_rx_num != 0) - pPhyInfo->SignalStrength = (u8)(odm_SignalScaleMapping(dm_odm, total_rssi /= rf_rx_num)); + pPhyInfo->SignalStrength = (u8)(odm_signal_scale_mapping(dm_odm, total_rssi /= rf_rx_num)); } /* For 88E HW Antenna Diversity */ @@ -347,8 +347,3 @@ void ODM_PhyStatusQuery(struct odm_dm_struct *dm_odm, odm_RxPhyStatus92CSeries_Parsing(dm_odm, pPhyInfo, pPhyStatus, pPktinfo, adapt); odm_Process_RSSIForDM(dm_odm, pPhyInfo, pPktinfo); } - -enum HAL_STATUS ODM_ConfigRFWithHeaderFile(struct odm_dm_struct *dm_odm) -{ - return ODM_ReadAndConfig_RadioA_1T_8188E(dm_odm); -} diff --git a/drivers/staging/r8188eu/hal/odm_RegConfig8188E.c b/drivers/staging/r8188eu/hal/odm_RegConfig8188E.c deleted file mode 100644 index 0fa17a99f9e937c688ec0c50f6ca347b25059554..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/hal/odm_RegConfig8188E.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#include "../include/drv_types.h" - -static void odm_ConfigRFReg_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, - u32 Data, u32 RegAddr) -{ - if (Addr == 0xffe) { - msleep(50); - } else if (Addr == 0xfd) { - mdelay(5); - } else if (Addr == 0xfc) { - mdelay(1); - } else if (Addr == 0xfb) { - udelay(50); - } else if (Addr == 0xfa) { - udelay(5); - } else if (Addr == 0xf9) { - udelay(1); - } else { - rtl8188e_PHY_SetRFReg(pDM_Odm->Adapter, RegAddr, bRFRegOffsetMask, Data); - /* Add 1us delay between BB/RF register setting. */ - udelay(1); - } -} - -void odm_ConfigRF_RadioA_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Data) -{ - u32 content = 0x1000; /* RF_Content: radioa_txt */ - u32 maskforPhySet = (u32)(content & 0xE000); - - odm_ConfigRFReg_8188E(pDM_Odm, Addr, Data, Addr | maskforPhySet); -} - -void odm_ConfigMAC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u8 Data) -{ - rtw_write8(pDM_Odm->Adapter, Addr, Data); -} - -void odm_ConfigBB_AGC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Bitmask, u32 Data) -{ - rtl8188e_PHY_SetBBReg(pDM_Odm->Adapter, Addr, Bitmask, Data); - /* Add 1us delay between BB/RF register setting. */ - udelay(1); -} - -void odm_ConfigBB_PHY_REG_PG_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, - u32 Bitmask, u32 Data) -{ - if (Addr == 0xfe) - msleep(50); - else if (Addr == 0xfd) - mdelay(5); - else if (Addr == 0xfc) - mdelay(1); - else if (Addr == 0xfb) - udelay(50); - else if (Addr == 0xfa) - udelay(5); - else if (Addr == 0xf9) - udelay(1); - else - storePwrIndexDiffRateOffset(pDM_Odm->Adapter, Addr, Bitmask, Data); -} - -void odm_ConfigBB_PHY_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u32 Bitmask, u32 Data) -{ - if (Addr == 0xfe) { - msleep(50); - } else if (Addr == 0xfd) { - mdelay(5); - } else if (Addr == 0xfc) { - mdelay(1); - } else if (Addr == 0xfb) { - udelay(50); - } else if (Addr == 0xfa) { - udelay(5); - } else if (Addr == 0xf9) { - udelay(1); - } else { - if (Addr == 0xa24) - pDM_Odm->RFCalibrateInfo.RegA24 = Data; - rtl8188e_PHY_SetBBReg(pDM_Odm->Adapter, Addr, Bitmask, Data); - - /* Add 1us delay between BB/RF register setting. */ - udelay(1); - } -} diff --git a/drivers/staging/r8188eu/hal/rtl8188e_cmd.c b/drivers/staging/r8188eu/hal/rtl8188e_cmd.c index b01ee1695fee2aafbcd28f99bf6cd2a6942bf0fb..8310d7f539822822d83b6e7018a114cdbfc2dd4c 100644 --- a/drivers/staging/r8188eu/hal/rtl8188e_cmd.c +++ b/drivers/staging/r8188eu/hal/rtl8188e_cmd.c @@ -5,8 +5,6 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/mlme_osdep.h" #include "../include/rtw_ioctl_set.h" #include "../include/rtl8188e_hal.h" diff --git a/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c b/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c index 5b8f1a912bbb75bd8405acb89e9a7f94a2c1d559..158260547f2b27942032364385cc4c780a120251 100644 --- a/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c +++ b/drivers/staging/r8188eu/hal/rtl8188e_hal_init.c @@ -526,43 +526,38 @@ void rtl8188e_ReadEFuse(struct adapter *Adapter, u16 _size_byte, u8 *pbuf) Hal_EfuseReadEFuse88E(Adapter, 0, _size_byte, pbuf); } -static void dump_chip_info(struct HAL_VERSION chip_vers) +static void dump_chip_info(struct adapter *adapter, struct HAL_VERSION chip_vers) { - uint cnt = 0; - char buf[128]; - - cnt += sprintf((buf + cnt), "Chip Version Info: CHIP_8188E_"); - cnt += sprintf((buf + cnt), "%s_", IS_NORMAL_CHIP(chip_vers) ? - "Normal_Chip" : "Test_Chip"); - cnt += sprintf((buf + cnt), "%s_", IS_CHIP_VENDOR_TSMC(chip_vers) ? - "TSMC" : "UMC"); + struct net_device *netdev = adapter->pnetdev; + char *cut = NULL; + char buf[25]; switch (chip_vers.CUTVersion) { case A_CUT_VERSION: - cnt += sprintf((buf + cnt), "A_CUT_"); + cut = "A_CUT"; break; case B_CUT_VERSION: - cnt += sprintf((buf + cnt), "B_CUT_"); + cut = "B_CUT"; break; case C_CUT_VERSION: - cnt += sprintf((buf + cnt), "C_CUT_"); + cut = "C_CUT"; break; case D_CUT_VERSION: - cnt += sprintf((buf + cnt), "D_CUT_"); + cut = "D_CUT"; break; case E_CUT_VERSION: - cnt += sprintf((buf + cnt), "E_CUT_"); + cut = "E_CUT"; break; default: - cnt += sprintf((buf + cnt), "UNKNOWN_CUT(%d)_", chip_vers.CUTVersion); + snprintf(buf, sizeof(buf), "UNKNOWN_CUT(%d)", chip_vers.CUTVersion); + cut = buf; break; } - cnt += sprintf((buf + cnt), "1T1R_"); - - cnt += sprintf((buf + cnt), "RomVer(%d)\n", 0); - - pr_info("%s", buf); + netdev_dbg(netdev, "Chip Version Info: CHIP_8188E_%s_%s_%s_1T1R_RomVer(%d)\n", + IS_NORMAL_CHIP(chip_vers) ? "Normal_Chip" : "Test_Chip", + IS_CHIP_VENDOR_TSMC(chip_vers) ? "TSMC" : "UMC", + cut, 0); } void rtl8188e_read_chip_version(struct adapter *padapter) @@ -581,7 +576,7 @@ void rtl8188e_read_chip_version(struct adapter *padapter) ChipVersion.VendorType = ((value32 & VENDOR_ID) ? CHIP_VENDOR_UMC : CHIP_VENDOR_TSMC); ChipVersion.CUTVersion = (value32 & CHIP_VER_RTL_MASK) >> CHIP_VER_RTL_SHIFT; /* IC version (CUT) */ - dump_chip_info(ChipVersion); + dump_chip_info(padapter, ChipVersion); pHalData->VersionID = ChipVersion; } @@ -688,6 +683,7 @@ Hal_EfuseParseIDCode88E( ) { struct eeprom_priv *pEEPROM = &padapter->eeprompriv; + struct net_device *netdev = padapter->pnetdev; u16 EEPROMId; /* Check 0x8129 again for making sure autoload status!! */ @@ -699,7 +695,7 @@ Hal_EfuseParseIDCode88E( pEEPROM->bautoload_fail_flag = false; } - pr_info("EEPROM ID = 0x%04x\n", EEPROMId); + netdev_dbg(netdev, "EEPROM ID = 0x%04x\n", EEPROMId); } static void Hal_ReadPowerValueFromPROM_8188E(struct txpowerinfo24g *pwrInfo24G, u8 *PROMContent, bool AutoLoadFail) diff --git a/drivers/staging/r8188eu/hal/rtl8188e_phycfg.c b/drivers/staging/r8188eu/hal/rtl8188e_phycfg.c index dea6d915a1f4042d8e0c5e99b4f6023b78a28f0b..532c63bce0bf02f1ae639adcee9275606444c261 100644 --- a/drivers/staging/r8188eu/hal/rtl8188e_phycfg.c +++ b/drivers/staging/r8188eu/hal/rtl8188e_phycfg.c @@ -12,26 +12,12 @@ /* 1. BB register R/W API */ /* */ -/** -* Function: phy_CalculateBitShift -* -* OverView: Get shifted position of the BitMask -* -* Input: -* u32 BitMask, -* -* Output: none -* Return: u32 Return the shift bit bit position of the mask -*/ -static u32 phy_CalculateBitShift(u32 BitMask) +/* Get shifted position of the bit mask */ +static u32 phy_calculate_bit_shift(u32 bitmask) { - u32 i; + u32 i = ffs(bitmask); - for (i = 0; i <= 31; i++) { - if (((BitMask >> i) & 0x1) == 1) - break; - } - return i; + return i ? i - 1 : 32; } /** @@ -62,7 +48,7 @@ rtl8188e_PHY_QueryBBReg( if (res) return 0; - BitShift = phy_CalculateBitShift(BitMask); + BitShift = phy_calculate_bit_shift(BitMask); ReturnValue = (OriginalValue & BitMask) >> BitShift; return ReturnValue; } @@ -95,7 +81,7 @@ void rtl8188e_PHY_SetBBReg(struct adapter *Adapter, u32 RegAddr, u32 BitMask, u3 if (res) return; - BitShift = phy_CalculateBitShift(BitMask); + BitShift = phy_calculate_bit_shift(BitMask); Data = ((OriginalValue & (~BitMask)) | (Data << BitShift)); } @@ -267,7 +253,7 @@ u32 rtl8188e_PHY_QueryRFReg(struct adapter *Adapter, u32 RegAddr, u32 BitMask) Original_Value = phy_RFSerialRead(Adapter, RegAddr); - BitShift = phy_CalculateBitShift(BitMask); + BitShift = phy_calculate_bit_shift(BitMask); Readback_Value = (Original_Value & BitMask) >> BitShift; return Readback_Value; } @@ -302,7 +288,7 @@ rtl8188e_PHY_SetRFReg( /* RF data is 12 bits only */ if (BitMask != bRFRegOffsetMask) { Original_Value = phy_RFSerialRead(Adapter, RegAddr); - BitShift = phy_CalculateBitShift(BitMask); + BitShift = phy_calculate_bit_shift(BitMask); Data = ((Original_Value & (~BitMask)) | (Data << BitShift)); } @@ -337,7 +323,7 @@ s32 PHY_MACConfig8188E(struct adapter *Adapter) /* */ /* Config MAC */ /* */ - if (HAL_STATUS_FAILURE == ODM_ReadAndConfig_MAC_REG_8188E(&pHalData->odmpriv)) + if (ODM_ReadAndConfig_MAC_REG_8188E(&pHalData->odmpriv)) rtStatus = _FAIL; /* 2010.07.13 AMPDU aggregation number B */ @@ -469,7 +455,7 @@ static int phy_BB8188E_Config_ParaFile(struct adapter *Adapter) /* 1. Read PHY_REG.TXT BB INIT!! */ /* We will separate as 88C / 92C according to chip version */ /* */ - if (HAL_STATUS_FAILURE == ODM_ReadAndConfig_PHY_REG_1T_8188E(&pHalData->odmpriv)) + if (ODM_ReadAndConfig_PHY_REG_1T_8188E(&pHalData->odmpriv)) return _FAIL; /* 2. If EEPROM or EFUSE autoload OK, We must config by PHY_REG_PG.txt */ @@ -479,7 +465,7 @@ static int phy_BB8188E_Config_ParaFile(struct adapter *Adapter) } /* 3. BB AGC table Initialization */ - if (HAL_STATUS_FAILURE == ODM_ReadAndConfig_AGC_TAB_1T_8188E(&pHalData->odmpriv)) + if (ODM_ReadAndConfig_AGC_TAB_1T_8188E(&pHalData->odmpriv)) return _FAIL; return _SUCCESS; @@ -521,15 +507,6 @@ PHY_BBConfig8188E( return rtStatus; } -int PHY_RFConfig8188E(struct adapter *Adapter) -{ - int rtStatus = _SUCCESS; - - /* RF config */ - rtStatus = PHY_RF6052_Config8188E(Adapter); - return rtStatus; -} - static void getTxPowerIndex88E(struct adapter *Adapter, u8 channel, u8 *cckPowerLevel, u8 *ofdmPowerLevel, u8 *BW20PowerLevel, u8 *BW40PowerLevel) diff --git a/drivers/staging/r8188eu/hal/rtl8188e_rf6052.c b/drivers/staging/r8188eu/hal/rtl8188e_rf6052.c index d043b7bc41423fa83cbc19aa923f5fee35ba843b..e5ec6e563fbda6c662a09ac3750c44b7944c0783 100644 --- a/drivers/staging/r8188eu/hal/rtl8188e_rf6052.c +++ b/drivers/staging/r8188eu/hal/rtl8188e_rf6052.c @@ -366,7 +366,7 @@ rtl8188e_PHY_RF6052SetOFDMTxPower( } } -static int phy_RF6052_Config_ParaFile(struct adapter *Adapter) +int phy_RF6052_Config_ParaFile(struct adapter *Adapter) { struct bb_reg_def *pPhyReg; struct hal_data_8188e *pHalData = &Adapter->haldata; @@ -396,7 +396,7 @@ static int phy_RF6052_Config_ParaFile(struct adapter *Adapter) udelay(1);/* PlatformStallExecution(1); */ /*----Initialize RF fom connfiguration file----*/ - if (HAL_STATUS_FAILURE == ODM_ConfigRFWithHeaderFile(&pHalData->odmpriv)) + if (ODM_ReadAndConfig_RadioA_1T_8188E(&pHalData->odmpriv)) rtStatus = _FAIL; /*----Restore RFENV control type----*/; @@ -404,14 +404,3 @@ static int phy_RF6052_Config_ParaFile(struct adapter *Adapter) return rtStatus; } - -int PHY_RF6052_Config8188E(struct adapter *Adapter) -{ - int rtStatus = _SUCCESS; - - /* */ - /* Config BB and RF */ - /* */ - rtStatus = phy_RF6052_Config_ParaFile(Adapter); - return rtStatus; -} diff --git a/drivers/staging/r8188eu/hal/rtl8188e_xmit.c b/drivers/staging/r8188eu/hal/rtl8188e_xmit.c deleted file mode 100644 index 46b871f3f631b85eb007483bd6d1dc7d52cefa84..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/hal/rtl8188e_xmit.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#define _RTL8188E_XMIT_C_ - -#include "../include/osdep_service.h" -#include "../include/drv_types.h" -#include "../include/rtl8188e_hal.h" - -void handle_txrpt_ccx_88e(struct adapter *adapter, u8 *buf) -{ - struct txrpt_ccx_88e *txrpt_ccx = (struct txrpt_ccx_88e *)buf; - - if (txrpt_ccx->int_ccx) { - if (txrpt_ccx->pkt_ok) - rtw_ack_tx_done(&adapter->xmitpriv, - RTW_SCTX_DONE_SUCCESS); - else - rtw_ack_tx_done(&adapter->xmitpriv, - RTW_SCTX_DONE_CCX_PKT_FAIL); - } -} diff --git a/drivers/staging/r8188eu/hal/rtl8188eu_recv.c b/drivers/staging/r8188eu/hal/rtl8188eu_recv.c deleted file mode 100644 index def6d0d6e4022e86bda0d29046935d66fa388eaf..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/hal/rtl8188eu_recv.c +++ /dev/null @@ -1,91 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#define _RTL8188EU_RECV_C_ -#include "../include/osdep_service.h" -#include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/mlme_osdep.h" - -#include "../include/usb_ops.h" -#include "../include/wifi.h" - -#include "../include/rtl8188e_hal.h" - -int rtl8188eu_init_recv_priv(struct adapter *padapter) -{ - struct recv_priv *precvpriv = &padapter->recvpriv; - int i, res = _SUCCESS; - struct recv_buf *precvbuf; - - tasklet_init(&precvpriv->recv_tasklet, - rtl8188eu_recv_tasklet, - (unsigned long)padapter); - - /* init recv_buf */ - rtw_init_queue(&precvpriv->free_recv_buf_queue); - - precvpriv->pallocated_recv_buf = kzalloc(NR_RECVBUFF * sizeof(struct recv_buf) + 4, - GFP_KERNEL); - if (!precvpriv->pallocated_recv_buf) { - res = _FAIL; - goto exit; - } - - precvpriv->precv_buf = (u8 *)ALIGN((size_t)(precvpriv->pallocated_recv_buf), 4); - - precvbuf = (struct recv_buf *)precvpriv->precv_buf; - - for (i = 0; i < NR_RECVBUFF; i++) { - res = rtw_os_recvbuf_resource_alloc(padapter, precvbuf); - if (res == _FAIL) - break; - precvbuf->adapter = padapter; - precvbuf++; - } - precvpriv->free_recv_buf_queue_cnt = NR_RECVBUFF; - skb_queue_head_init(&precvpriv->rx_skb_queue); - { - int i; - size_t tmpaddr = 0; - size_t alignment = 0; - struct sk_buff *pskb = NULL; - - skb_queue_head_init(&precvpriv->free_recv_skb_queue); - - for (i = 0; i < NR_PREALLOC_RECV_SKB; i++) { - pskb = __netdev_alloc_skb(padapter->pnetdev, MAX_RECVBUF_SZ + RECVBUFF_ALIGN_SZ, GFP_KERNEL); - if (pskb) { - pskb->dev = padapter->pnetdev; - tmpaddr = (size_t)pskb->data; - alignment = tmpaddr & (RECVBUFF_ALIGN_SZ - 1); - skb_reserve(pskb, (RECVBUFF_ALIGN_SZ - alignment)); - - skb_queue_tail(&precvpriv->free_recv_skb_queue, pskb); - } - pskb = NULL; - } - } -exit: - return res; -} - -void rtl8188eu_free_recv_priv(struct adapter *padapter) -{ - int i; - struct recv_buf *precvbuf; - struct recv_priv *precvpriv = &padapter->recvpriv; - - precvbuf = (struct recv_buf *)precvpriv->precv_buf; - - for (i = 0; i < NR_RECVBUFF; i++) { - rtw_os_recvbuf_resource_free(padapter, precvbuf); - precvbuf++; - } - - kfree(precvpriv->pallocated_recv_buf); - - skb_queue_purge(&precvpriv->rx_skb_queue); - - skb_queue_purge(&precvpriv->free_recv_skb_queue); -} diff --git a/drivers/staging/r8188eu/hal/rtl8188eu_xmit.c b/drivers/staging/r8188eu/hal/rtl8188eu_xmit.c index bdfa5194928976aeead1b80b20375ab8c5d0fff8..8e4a5acc0b18885d3c93d4d3664e8597c1760263 100644 --- a/drivers/staging/r8188eu/hal/rtl8188eu_xmit.c +++ b/drivers/staging/r8188eu/hal/rtl8188eu_xmit.c @@ -431,7 +431,7 @@ bool rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmit rtw_xmitframe_coalesce(adapt, pxmitframe->pkt, pxmitframe); /* always return ndis_packet after rtw_xmitframe_coalesce */ - rtw_os_xmit_complete(adapt, pxmitframe); + rtw_xmit_complete(adapt, pxmitframe); /* 3 2. aggregate same priority and same DA(AP or STA) frames */ pfirstframe = pxmitframe; @@ -501,7 +501,7 @@ bool rtl8188eu_xmitframe_complete(struct adapter *adapt, struct xmit_priv *pxmit rtw_xmitframe_coalesce(adapt, pxmitframe->pkt, pxmitframe); /* always return ndis_packet after rtw_xmitframe_coalesce */ - rtw_os_xmit_complete(adapt, pxmitframe); + rtw_xmit_complete(adapt, pxmitframe); /* (len - TXDESC_SIZE) == pxmitframe->attrib.last_txcmdsz */ update_txdesc(pxmitframe, pxmitframe->buf_addr, pxmitframe->attrib.last_txcmdsz, true); diff --git a/drivers/staging/r8188eu/hal/usb_halinit.c b/drivers/staging/r8188eu/hal/usb_halinit.c index ff074d246dab764e4215b60898703192a0573e5b..d28b4dc2a7676124155003585d44d6c78be7d5b6 100644 --- a/drivers/staging/r8188eu/hal/usb_halinit.c +++ b/drivers/staging/r8188eu/hal/usb_halinit.c @@ -13,40 +13,77 @@ #include "../include/usb_osintf.h" #include "../include/HalPwrSeqCmd.h" -static void _ConfigNormalChipOutEP_8188E(struct adapter *adapt, u8 NumOutPipe) +static void one_out_pipe(struct adapter *adapter) { - struct hal_data_8188e *haldata = &adapt->haldata; + struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); - switch (NumOutPipe) { - case 3: - haldata->OutEpQueueSel = TX_SELE_HQ | TX_SELE_LQ | TX_SELE_NQ; - haldata->OutEpNumber = 3; - break; - case 2: - haldata->OutEpQueueSel = TX_SELE_HQ | TX_SELE_NQ; - haldata->OutEpNumber = 2; - break; - case 1: - haldata->OutEpQueueSel = TX_SELE_HQ; - haldata->OutEpNumber = 1; - break; - default: - break; + pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ + pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];/* VI */ + pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[0];/* BE */ + pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0];/* BK */ +} + +static void two_out_pipe(struct adapter *adapter, bool wifi_cfg) +{ + struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); + + /* 0:H, 1:L */ + + pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[0];/* VI */ + pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[1];/* BE */ + + if (wifi_cfg) { + pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[1];/* VO */ + pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[0];/* BK */ + } else { + pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ + pdvobjpriv->Queue2Pipe[3] = pdvobjpriv->RtOutPipe[1];/* BK */ } } -static bool HalUsbSetQueuePipeMapping8188EUsb(struct adapter *adapt, u8 NumOutPipe) +static void three_out_pipe(struct adapter *adapter, bool wifi_cfg) { + struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapter); + + /* 0:H, 1:N, 2:L */ - _ConfigNormalChipOutEP_8188E(adapt, NumOutPipe); - return Hal_MappingOutPipe(adapt, NumOutPipe); + pdvobjpriv->Queue2Pipe[0] = pdvobjpriv->RtOutPipe[0];/* VO */ + pdvobjpriv->Queue2Pipe[1] = pdvobjpriv->RtOutPipe[1];/* VI */ + pdvobjpriv->Queue2Pipe[2] = pdvobjpriv->RtOutPipe[2];/* BE */ + + pdvobjpriv->Queue2Pipe[3] = wifi_cfg ? + pdvobjpriv->RtOutPipe[1] : pdvobjpriv->RtOutPipe[2];/* BK */ } -void rtl8188eu_interface_configure(struct adapter *adapt) +int rtl8188eu_interface_configure(struct adapter *adapt) { + struct registry_priv *pregistrypriv = &adapt->registrypriv; struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(adapt); + struct hal_data_8188e *haldata = &adapt->haldata; + bool wifi_cfg = pregistrypriv->wifi_spec; - HalUsbSetQueuePipeMapping8188EUsb(adapt, pdvobjpriv->RtNumOutPipes); + pdvobjpriv->Queue2Pipe[4] = pdvobjpriv->RtOutPipe[0];/* BCN */ + pdvobjpriv->Queue2Pipe[5] = pdvobjpriv->RtOutPipe[0];/* MGT */ + pdvobjpriv->Queue2Pipe[6] = pdvobjpriv->RtOutPipe[0];/* HIGH */ + pdvobjpriv->Queue2Pipe[7] = pdvobjpriv->RtOutPipe[0];/* TXCMD */ + + switch (pdvobjpriv->RtNumOutPipes) { + case 3: + haldata->out_ep_extra_queues = TX_SELE_LQ | TX_SELE_NQ; + three_out_pipe(adapt, wifi_cfg); + break; + case 2: + haldata->out_ep_extra_queues = TX_SELE_NQ; + two_out_pipe(adapt, wifi_cfg); + break; + case 1: + one_out_pipe(adapt); + break; + default: + return -ENXIO; + } + + return 0; } u32 rtl8188eu_InitPowerOn(struct adapter *adapt) @@ -116,32 +153,24 @@ static void _InitQueueReservedPage(struct adapter *Adapter) { struct hal_data_8188e *haldata = &Adapter->haldata; struct registry_priv *pregistrypriv = &Adapter->registrypriv; - u32 numHQ = 0; - u32 numLQ = 0; - u32 numNQ = 0; - u32 numPubQ; - u32 value32; - u8 value8; - bool bWiFiConfig = pregistrypriv->wifi_spec; - - if (bWiFiConfig) { - if (haldata->OutEpQueueSel & TX_SELE_HQ) - numHQ = 0x29; + u8 numLQ = 0; + u8 numNQ = 0; + u8 numPubQ; - if (haldata->OutEpQueueSel & TX_SELE_LQ) + if (pregistrypriv->wifi_spec) { + if (haldata->out_ep_extra_queues & TX_SELE_LQ) numLQ = 0x1C; /* NOTE: This step shall be proceed before writing REG_RQPN. */ - if (haldata->OutEpQueueSel & TX_SELE_NQ) + if (haldata->out_ep_extra_queues & TX_SELE_NQ) numNQ = 0x1C; - value8 = (u8)_NPQ(numNQ); - rtw_write8(Adapter, REG_RQPN_NPQ, value8); - numPubQ = 0xA8 - numHQ - numLQ - numNQ; + rtw_write8(Adapter, REG_RQPN_NPQ, numNQ); + + numPubQ = 0xA8 - NUM_HQ - numLQ - numNQ; /* TX DMA */ - value32 = _HPQ(numHQ) | _LPQ(numLQ) | _PUBQ(numPubQ) | LD_RQPN; - rtw_write32(Adapter, REG_RQPN, value32); + rtw_write32(Adapter, REG_RQPN, LD_RQPN | numPubQ << 16 | numLQ << 8 | NUM_HQ); } else { rtw_write16(Adapter, REG_RQPN_NPQ, 0x0000);/* Just follow MP Team,??? Georgia 03/28 */ rtw_write16(Adapter, REG_RQPN_NPQ, 0x0d); @@ -187,69 +216,20 @@ static void _InitNormalChipRegPriority(struct adapter *Adapter, u16 beQ, rtw_write16(Adapter, REG_TRXDMA_CTRL, value16); } -static void _InitNormalChipOneOutEpPriority(struct adapter *Adapter) -{ - struct hal_data_8188e *haldata = &Adapter->haldata; - - u16 value = 0; - switch (haldata->OutEpQueueSel) { - case TX_SELE_HQ: - value = QUEUE_HIGH; - break; - case TX_SELE_LQ: - value = QUEUE_LOW; - break; - case TX_SELE_NQ: - value = QUEUE_NORMAL; - break; - default: - break; - } - _InitNormalChipRegPriority(Adapter, value, value, value, value, - value, value); -} - static void _InitNormalChipTwoOutEpPriority(struct adapter *Adapter) { - struct hal_data_8188e *haldata = &Adapter->haldata; struct registry_priv *pregistrypriv = &Adapter->registrypriv; - u16 beQ, bkQ, viQ, voQ, mgtQ, hiQ; - u16 valueHi = 0; - u16 valueLow = 0; - - switch (haldata->OutEpQueueSel) { - case (TX_SELE_HQ | TX_SELE_LQ): - valueHi = QUEUE_HIGH; - valueLow = QUEUE_LOW; - break; - case (TX_SELE_NQ | TX_SELE_LQ): - valueHi = QUEUE_NORMAL; - valueLow = QUEUE_LOW; - break; - case (TX_SELE_HQ | TX_SELE_NQ): - valueHi = QUEUE_HIGH; - valueLow = QUEUE_NORMAL; - break; - default: - break; - } + u16 bkQ, voQ; if (!pregistrypriv->wifi_spec) { - beQ = valueLow; - bkQ = valueLow; - viQ = valueHi; - voQ = valueHi; - mgtQ = valueHi; - hiQ = valueHi; + bkQ = QUEUE_NORMAL; + voQ = QUEUE_HIGH; } else {/* for WMM ,CONFIG_OUT_EP_WIFI_MODE */ - beQ = valueLow; - bkQ = valueHi; - viQ = valueHi; - voQ = valueLow; - mgtQ = valueHi; - hiQ = valueHi; + bkQ = QUEUE_HIGH; + voQ = QUEUE_NORMAL; } - _InitNormalChipRegPriority(Adapter, beQ, bkQ, viQ, voQ, mgtQ, hiQ); + _InitNormalChipRegPriority(Adapter, QUEUE_NORMAL, bkQ, QUEUE_HIGH, + voQ, QUEUE_HIGH, QUEUE_HIGH); } static void _InitNormalChipThreeOutEpPriority(struct adapter *Adapter) @@ -277,11 +257,12 @@ static void _InitNormalChipThreeOutEpPriority(struct adapter *Adapter) static void _InitQueuePriority(struct adapter *Adapter) { - struct hal_data_8188e *haldata = &Adapter->haldata; + struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(Adapter); - switch (haldata->OutEpNumber) { + switch (pdvobjpriv->RtNumOutPipes) { case 1: - _InitNormalChipOneOutEpPriority(Adapter); + _InitNormalChipRegPriority(Adapter, QUEUE_HIGH, QUEUE_HIGH, QUEUE_HIGH, + QUEUE_HIGH, QUEUE_HIGH, QUEUE_HIGH); break; case 2: _InitNormalChipTwoOutEpPriority(Adapter); @@ -515,8 +496,7 @@ static int _InitBeaconParameters(struct adapter *Adapter) return 0; } -static void _BeaconFunctionEnable(struct adapter *Adapter, - bool Enable, bool Linked) +static void _BeaconFunctionEnable(struct adapter *Adapter) { rtw_write8(Adapter, REG_BCN_CTRL, (BIT(4) | BIT(3) | BIT(1))); @@ -567,7 +547,6 @@ u32 rtl8188eu_hal_init(struct adapter *Adapter) { u8 value8 = 0; u16 value16; - u8 txpktbuf_bndy; u32 status = _SUCCESS; int res; struct hal_data_8188e *haldata = &Adapter->haldata; @@ -600,13 +579,6 @@ u32 rtl8188eu_hal_init(struct adapter *Adapter) /* HW GPIO pin. Before PHY_RFConfig8192C. */ /* 2010/08/26 MH If Efuse does not support sective suspend then disable the function. */ - if (!pregistrypriv->wifi_spec) { - txpktbuf_bndy = TX_PAGE_BOUNDARY_88E; - } else { - /* for WMM */ - txpktbuf_bndy = WMM_NORMAL_TX_PAGE_BOUNDARY_88E; - } - _InitQueueReservedPage(Adapter); _InitQueuePriority(Adapter); _InitPageBoundary(Adapter); @@ -639,7 +611,7 @@ u32 rtl8188eu_hal_init(struct adapter *Adapter) if (status == _FAIL) goto exit; - status = PHY_RFConfig8188E(Adapter); + status = phy_RF6052_Config_ParaFile(Adapter); if (status == _FAIL) goto exit; @@ -647,9 +619,9 @@ u32 rtl8188eu_hal_init(struct adapter *Adapter) if (status == _FAIL) goto exit; - _InitTxBufferBoundary(Adapter, txpktbuf_bndy); + _InitTxBufferBoundary(Adapter, TX_PAGE_BOUNDARY_88E); - status = InitLLTTable(Adapter, txpktbuf_bndy); + status = InitLLTTable(Adapter, TX_PAGE_BOUNDARY_88E); if (status == _FAIL) goto exit; @@ -922,7 +894,7 @@ static void Hal_EfuseParseMACAddr_8188EU(struct adapter *adapt, u8 *hwinfo, bool } } -void ReadAdapterInfo8188EU(struct adapter *Adapter) +int ReadAdapterInfo8188EU(struct adapter *Adapter) { struct eeprom_priv *eeprom = &Adapter->eeprompriv; struct led_priv *ledpriv = &Adapter->ledpriv; @@ -933,13 +905,13 @@ void ReadAdapterInfo8188EU(struct adapter *Adapter) /* check system boot selection */ res = rtw_read8(Adapter, REG_9346CR, &eeValue); if (res) - return; + return res; eeprom->bautoload_fail_flag = !(eeValue & EEPROM_EN); efuse_buf = kmalloc(EFUSE_MAP_LEN_88E, GFP_KERNEL); if (!efuse_buf) - return; + return -ENOMEM; memset(efuse_buf, 0xFF, EFUSE_MAP_LEN_88E); if (!(eeValue & BOOT_FROM_EEPROM) && !eeprom->bautoload_fail_flag) { @@ -961,6 +933,7 @@ void ReadAdapterInfo8188EU(struct adapter *Adapter) ledpriv->bRegUseLed = true; kfree(efuse_buf); + return 0; } void UpdateHalRAMask8188EUsb(struct adapter *adapt, u32 mac_id, u8 rssi_level) @@ -1069,7 +1042,7 @@ void SetBeaconRelatedRegisters8188EUsb(struct adapter *adapt) rtw_write8(adapt, REG_RXTSF_OFFSET_CCK, 0x50); rtw_write8(adapt, REG_RXTSF_OFFSET_OFDM, 0x50); - _BeaconFunctionEnable(adapt, true, true); + _BeaconFunctionEnable(adapt); rtw_resume_tx_beacon(adapt); diff --git a/drivers/staging/r8188eu/hal/usb_ops_linux.c b/drivers/staging/r8188eu/hal/usb_ops_linux.c index c1a4d023f6279ed3cecbf680a72f179966f1e40d..7c72f5e04d9ba519693d52aee05d13ce66c570d2 100644 --- a/drivers/staging/r8188eu/hal/usb_ops_linux.c +++ b/drivers/staging/r8188eu/hal/usb_ops_linux.c @@ -5,7 +5,6 @@ #include "../include/drv_types.h" #include "../include/osdep_intf.h" #include "../include/usb_ops.h" -#include "../include/recv_osdep.h" #include "../include/rtl8188e_hal.h" static int usb_read(struct intf_hdl *intf, u16 value, void *data, u8 size) @@ -190,6 +189,20 @@ int rtw_writeN(struct adapter *adapter, u32 addr, u32 length, u8 *data) return RTW_STATUS_CODE(ret); } +static void handle_txrpt_ccx_88e(struct adapter *adapter, u8 *buf) +{ + struct txrpt_ccx_88e *txrpt_ccx = (struct txrpt_ccx_88e *)buf; + + if (txrpt_ccx->int_ccx) { + if (txrpt_ccx->pkt_ok) + rtw_ack_tx_done(&adapter->xmitpriv, + RTW_SCTX_DONE_SUCCESS); + else + rtw_ack_tx_done(&adapter->xmitpriv, + RTW_SCTX_DONE_CCX_PKT_FAIL); + } +} + static int recvbuf2recvframe(struct adapter *adapt, struct sk_buff *pskb) { u8 *pbuf; diff --git a/drivers/staging/r8188eu/include/Hal8188EPhyCfg.h b/drivers/staging/r8188eu/include/Hal8188EPhyCfg.h index 9e6f2361b09051d5589a49c38bcfc744f654f259..4a0b782c33bec27e86f088759a669af07fee70a4 100644 --- a/drivers/staging/r8188eu/include/Hal8188EPhyCfg.h +++ b/drivers/staging/r8188eu/include/Hal8188EPhyCfg.h @@ -80,7 +80,6 @@ void rtl8188e_PHY_SetRFReg(struct adapter *adapter, u32 regaddr, u32 mask, u32 d /* MAC/BB/RF HAL config */ int PHY_MACConfig8188E(struct adapter *adapter); int PHY_BBConfig8188E(struct adapter *adapter); -int PHY_RFConfig8188E(struct adapter *adapter); /* BB TX Power R/W */ void PHY_SetTxPowerLevel8188E(struct adapter *adapter, u8 channel); diff --git a/drivers/staging/r8188eu/include/HalHWImg8188E_BB.h b/drivers/staging/r8188eu/include/HalHWImg8188E_BB.h index 8270fdbc284430aae87bfca430826e6cb561a0cb..0a290bc31c4d79f9cf81b37fbcf8e3bb56f9da94 100644 --- a/drivers/staging/r8188eu/include/HalHWImg8188E_BB.h +++ b/drivers/staging/r8188eu/include/HalHWImg8188E_BB.h @@ -10,13 +10,13 @@ * AGC_TAB_1T.TXT ******************************************************************************/ -enum HAL_STATUS ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *odm); +int ODM_ReadAndConfig_AGC_TAB_1T_8188E(struct odm_dm_struct *odm); /****************************************************************************** * PHY_REG_1T.TXT ******************************************************************************/ -enum HAL_STATUS ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *odm); +int ODM_ReadAndConfig_PHY_REG_1T_8188E(struct odm_dm_struct *odm); /****************************************************************************** * PHY_REG_PG.TXT diff --git a/drivers/staging/r8188eu/include/HalHWImg8188E_MAC.h b/drivers/staging/r8188eu/include/HalHWImg8188E_MAC.h index 391c1754b0b651424ad571017d4cadf5206666e4..b3d67c1a80503296c5111b3b311396321b67f630 100644 --- a/drivers/staging/r8188eu/include/HalHWImg8188E_MAC.h +++ b/drivers/staging/r8188eu/include/HalHWImg8188E_MAC.h @@ -7,7 +7,6 @@ /****************************************************************************** * MAC_REG.TXT ******************************************************************************/ - -enum HAL_STATUS ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *pDM_Odm); +int ODM_ReadAndConfig_MAC_REG_8188E(struct odm_dm_struct *pDM_Odm); #endif /* end of HWIMG_SUPPORT */ diff --git a/drivers/staging/r8188eu/include/HalHWImg8188E_RF.h b/drivers/staging/r8188eu/include/HalHWImg8188E_RF.h index 0c67c3df20b99554fa810b0f62f754a4011e0edf..880feadb43407cd8dd9fafd843609ebbc531d359 100644 --- a/drivers/staging/r8188eu/include/HalHWImg8188E_RF.h +++ b/drivers/staging/r8188eu/include/HalHWImg8188E_RF.h @@ -8,6 +8,6 @@ * RadioA_1T.TXT ******************************************************************************/ -enum HAL_STATUS ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *odm); +int ODM_ReadAndConfig_RadioA_1T_8188E(struct odm_dm_struct *odm); #endif /* end of HWIMG_SUPPORT */ diff --git a/drivers/staging/r8188eu/include/drv_types.h b/drivers/staging/r8188eu/include/drv_types.h index bba88a0ede61cd08f7cba7906ca2a879a6babde8..1bd0c8f3a35886308c9e0fad4530d72116cfaa00 100644 --- a/drivers/staging/r8188eu/include/drv_types.h +++ b/drivers/staging/r8188eu/include/drv_types.h @@ -10,8 +10,6 @@ #ifndef __DRV_TYPES_H__ #define __DRV_TYPES_H__ -#define DRV_NAME "r8188eu" - #include "osdep_service.h" #include "wlan_bssdef.h" #include "rtw_ht.h" @@ -36,10 +34,9 @@ #include "rtl8188e_hal.h" #include "rtw_fw.h" -#define DRIVERVERSION "v4.1.4_6773.20130222" +#define FW_RTL8188EU "rtlwifi/rtl8188eufw.bin" struct registry_priv { - u8 chip_version; u8 rfintfs; u8 lbkmode; u8 hci; @@ -222,7 +219,7 @@ struct adapter { #define adapter_to_dvobj(adapter) (adapter->dvobj) -int rtw_handle_dualmac(struct adapter *adapter, bool init); +void rtw_handle_dualmac(struct adapter *adapter, bool init); static inline u8 *myid(struct eeprom_priv *peepriv) { diff --git a/drivers/staging/r8188eu/include/hal_com.h b/drivers/staging/r8188eu/include/hal_com.h index d7e333f6ce39b3abe4a294dbc7633b199fd0e0ee..cd3f845e146a4f573e4d1a8a802a1a242e3de2e0 100644 --- a/drivers/staging/r8188eu/include/hal_com.h +++ b/drivers/staging/r8188eu/include/hal_com.h @@ -143,8 +143,4 @@ u8 MRateToHwRate(u8 rate); void HalSetBrateCfg(struct adapter *Adapter, u8 *mBratesOS, u16 *pBrateCfg); -bool Hal_MappingOutPipe(struct adapter *pAdapter, u8 NumOutPipe); - -s32 c2h_evt_read(struct adapter *adapter, u8 *buf); - #endif /* __HAL_COMMON_H__ */ diff --git a/drivers/staging/r8188eu/include/hal_intf.h b/drivers/staging/r8188eu/include/hal_intf.h index ab6856d8a0909b17a570b4e299a2e1bb4b34c5a5..ac6e3f95c5b76cf50c5387148f6fda31e001f529 100644 --- a/drivers/staging/r8188eu/include/hal_intf.h +++ b/drivers/staging/r8188eu/include/hal_intf.h @@ -10,8 +10,8 @@ typedef s32 (*c2h_id_filter)(u8 id); -void rtl8188eu_interface_configure(struct adapter *adapt); -void ReadAdapterInfo8188EU(struct adapter *Adapter); +int rtl8188eu_interface_configure(struct adapter *adapt); +int ReadAdapterInfo8188EU(struct adapter *Adapter); void rtl8188eu_init_default_value(struct adapter *adapt); void rtl8188e_SetHalODMVar(struct adapter *Adapter, void *pValue1, bool bSet); u32 rtl8188eu_InitPowerOn(struct adapter *adapt); @@ -39,7 +39,6 @@ void rtw_hal_update_ra_mask(struct adapter *padapter, u32 mac_id, u8 level); void rtw_hal_clone_data(struct adapter *dst_adapt, struct adapter *src_adapt); -void indicate_wx_scan_complete_event(struct adapter *padapter); u8 rtw_do_join(struct adapter *padapter); #endif /* __HAL_INTF_H__ */ diff --git a/drivers/staging/r8188eu/include/ioctl_cfg80211.h b/drivers/staging/r8188eu/include/ioctl_cfg80211.h deleted file mode 100644 index 738f645f9bbc15de27da33683166f8364b9a7bc4..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/include/ioctl_cfg80211.h +++ /dev/null @@ -1,89 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -/* Copyright(c) 2007 - 2011 Realtek Corporation. i*/ - -#ifndef __IOCTL_CFG80211_H__ -#define __IOCTL_CFG80211_H__ - -struct rtw_wdev_invit_info { - u8 token; - u8 flags; - u8 status; - u8 req_op_ch; - u8 rsp_op_ch; -}; - -#define rtw_wdev_invit_info_init(invit_info) \ - do { \ - (invit_info)->token = 0; \ - (invit_info)->flags = 0x00; \ - (invit_info)->status = 0xff; \ - (invit_info)->req_op_ch = 0; \ - (invit_info)->rsp_op_ch = 0; \ - } while (0) - -struct rtw_wdev_priv { - struct wireless_dev *rtw_wdev; - - struct adapter *padapter; - - struct cfg80211_scan_request *scan_request; - spinlock_t scan_req_lock; - - struct net_device *pmon_ndev;/* for monitor interface */ - char ifname_mon[IFNAMSIZ + 1]; /* name of monitor interface */ - - u8 p2p_enabled; - - u8 provdisc_req_issued; - - struct rtw_wdev_invit_info invit_info; - - u8 bandroid_scan; - bool block; - bool power_mgmt; -}; - -#define wdev_to_priv(w) ((struct rtw_wdev_priv *)(wdev_priv(w))) - -#define wiphy_to_wdev(x) \ -((struct wireless_dev *)(((struct rtw_wdev_priv *)wiphy_priv(x))->rtw_wdev)) - -int rtw_wdev_alloc(struct adapter *padapter, struct device *dev); -void rtw_wdev_free(struct wireless_dev *wdev); -void rtw_wdev_unregister(struct wireless_dev *wdev); - -void rtw_cfg80211_init_wiphy(struct adapter *padapter); - -void rtw_cfg80211_surveydone_event_callback(struct adapter *padapter); - -void rtw_cfg80211_indicate_connect(struct adapter *padapter); -void rtw_cfg80211_indicate_disconnect(struct adapter *padapter); -void rtw_cfg80211_indicate_scan_done(struct rtw_wdev_priv *pwdev_priv, - bool aborted); - -void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, - u8 *pmgmt_frame, uint frame_len); -void rtw_cfg80211_indicate_sta_disassoc(struct adapter *padapter, - unsigned char *da, - unsigned short reason); - -void rtw_cfg80211_issue_p2p_provision_request(struct adapter *padapter, - const u8 *buf, size_t len); -void rtw_cfg80211_rx_p2p_action_public(struct adapter *padapter, - u8 *pmgmt_frame, uint frame_len); -void rtw_cfg80211_rx_action_p2p(struct adapter *padapter, u8 *pmgmt_frame, - uint frame_len); -void rtw_cfg80211_rx_action(struct adapter *adapter, u8 *frame, - uint frame_len, const char *msg); - -int rtw_cfg80211_set_mgnt_wpsp2pie(struct net_device *net, - char *buf, int len, int type); - -bool rtw_cfg80211_pwr_mgmt(struct adapter *adapter); - -#define rtw_cfg80211_rx_mgmt(dev, freq, sig_dbm, buf, len, gfp) \ - cfg80211_rx_mgmt(dev, freq, sig_dbm, buf, len, gfp) -#define rtw_cfg80211_send_rx_assoc(dev, bss, buf, len) \ - cfg80211_send_rx_assoc(dev, bss, buf, len) - -#endif /* __IOCTL_CFG80211_H__ */ diff --git a/drivers/staging/r8188eu/include/mlme_osdep.h b/drivers/staging/r8188eu/include/mlme_osdep.h deleted file mode 100644 index 5b9f688f942461c87a1c0f0199e7aa6844874541..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/include/mlme_osdep.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#ifndef __MLME_OSDEP_H_ -#define __MLME_OSDEP_H_ - -#include "osdep_service.h" -#include "drv_types.h" - -void rtw_init_mlme_timer(struct adapter *padapter); -void rtw_os_indicate_disconnect(struct adapter *adapter); -void rtw_os_indicate_connect(struct adapter *adapter); -void rtw_os_indicate_scan_done(struct adapter *padapter, bool aborted); -void rtw_report_sec_ie(struct adapter *adapter, u8 authmode, u8 *sec_ie); - -void rtw_reset_securitypriv(struct adapter *adapter); -void indicate_wx_scan_complete_event(struct adapter *padapter); - -#endif /* _MLME_OSDEP_H_ */ diff --git a/drivers/staging/r8188eu/include/odm_HWConfig.h b/drivers/staging/r8188eu/include/odm_HWConfig.h index b37962edb2edebaa675a8d2978eacce9ef7b66a6..3f7185780e879925508070ec0cfc0d0e91208fbc 100644 --- a/drivers/staging/r8188eu/include/odm_HWConfig.h +++ b/drivers/staging/r8188eu/include/odm_HWConfig.h @@ -66,5 +66,4 @@ void ODM_PhyStatusQuery(struct odm_dm_struct *pDM_Odm, struct odm_per_pkt_info *pPktinfo, struct adapter *adapt); -enum HAL_STATUS ODM_ConfigRFWithHeaderFile(struct odm_dm_struct *pDM_Odm); #endif diff --git a/drivers/staging/r8188eu/include/odm_RegConfig8188E.h b/drivers/staging/r8188eu/include/odm_RegConfig8188E.h deleted file mode 100644 index 683fa4a079562fcf0a00735109d326d68cc9931e..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/include/odm_RegConfig8188E.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#ifndef __INC_ODM_REGCONFIG_H_8188E -#define __INC_ODM_REGCONFIG_H_8188E - -void odm_ConfigRF_RadioA_8188E(struct odm_dm_struct *pDM_Odm, - u32 Addr, u32 Data); - -void odm_ConfigMAC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, u8 Data); - -void odm_ConfigBB_AGC_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, - u32 Bitmask, u32 Data); - -void odm_ConfigBB_PHY_REG_PG_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, - u32 Bitmask, u32 Data); - -void odm_ConfigBB_PHY_8188E(struct odm_dm_struct *pDM_Odm, u32 Addr, - u32 Bitmask, u32 Data); - -#endif diff --git a/drivers/staging/r8188eu/include/odm_types.h b/drivers/staging/r8188eu/include/odm_types.h index 08ba7a418ba881a51495249258a99ef057ea097e..76302df4b330b014508c658cbe90b329bf5579ae 100644 --- a/drivers/staging/r8188eu/include/odm_types.h +++ b/drivers/staging/r8188eu/include/odm_types.h @@ -6,11 +6,6 @@ #define ODM_CE 0x04 /* BIT(2) */ -enum HAL_STATUS { - HAL_STATUS_SUCCESS, - HAL_STATUS_FAILURE, -}; - #define SET_TX_DESC_ANTSEL_A_88E(__ptxdesc, __value) \ le32p_replace_bits((__le32 *)(__ptxdesc + 8), __value, BIT(24)) #define SET_TX_DESC_ANTSEL_B_88E(__ptxdesc, __value) \ diff --git a/drivers/staging/r8188eu/include/osdep_intf.h b/drivers/staging/r8188eu/include/osdep_intf.h index 0d7009269aabc32aa68cabc5787c1fe30ddff345..36511c469546147530c25baecb8647e3405f9a12 100644 --- a/drivers/staging/r8188eu/include/osdep_intf.h +++ b/drivers/staging/r8188eu/include/osdep_intf.h @@ -39,6 +39,9 @@ The protection mechanism is through the pending queue. u8 bio_timer_cancel; }; +int netdev_open(struct net_device *pnetdev); +int netdev_close(struct net_device *pnetdev); + u8 rtw_init_drv_sw(struct adapter *padapter); u8 rtw_free_drv_sw(struct adapter *padapter); u8 rtw_reset_drv_sw(struct adapter *padapter); diff --git a/drivers/staging/r8188eu/include/recv_osdep.h b/drivers/staging/r8188eu/include/recv_osdep.h deleted file mode 100644 index ca8a613508fd15b6a369fe8f5086e4f27695d869..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/include/recv_osdep.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#ifndef __RECV_OSDEP_H_ -#define __RECV_OSDEP_H_ - -#include "osdep_service.h" -#include "drv_types.h" - -int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter); -void _rtw_free_recv_priv(struct recv_priv *precvpriv); - -s32 rtw_recv_entry(struct recv_frame *precv_frame); -int rtw_recv_indicatepkt(struct adapter *adapter, struct recv_frame *recv_frame); -void rtw_recv_returnpacket(struct net_device *cnxt, struct sk_buff *retpkt); - -void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup); - -int rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter); -void rtw_free_recv_priv(struct recv_priv *precvpriv); - -int rtw_os_recvbuf_resource_alloc(struct adapter *adapt, struct recv_buf *buf); -int rtw_os_recvbuf_resource_free(struct adapter *adapt, struct recv_buf *buf); - -void rtw_init_recv_timer(struct recv_reorder_ctrl *preorder_ctrl); -int _netdev_open(struct net_device *pnetdev); -int netdev_open(struct net_device *pnetdev); -int netdev_close(struct net_device *pnetdev); - -#endif /* */ diff --git a/drivers/staging/r8188eu/include/rtl8188e_hal.h b/drivers/staging/r8188eu/include/rtl8188e_hal.h index 5cd62b21672037353192bc28420bd07f6aac704a..ed4091e7cc7e6c0a21f0ebbe49ef6f8e03397a7a 100644 --- a/drivers/staging/r8188eu/include/rtl8188e_hal.h +++ b/drivers/staging/r8188eu/include/rtl8188e_hal.h @@ -23,7 +23,6 @@ #include "HalHWImg8188E_MAC.h" #include "HalHWImg8188E_RF.h" #include "HalHWImg8188E_BB.h" -#include "odm_RegConfig8188E.h" #include "odm_RTL8188E.h" #define DRVINFO_SZ 4 /* unit is 8bytes */ @@ -36,7 +35,6 @@ 0x2400 /* 9k for 88E nornal chip , MaxRxBuff=10k-max(TxReportSize(64*8), * WOLPattern(16*24)) */ -#define TX_SELE_HQ BIT(0) /* High Queue */ #define TX_SELE_LQ BIT(1) /* Low Queue */ #define TX_SELE_NQ BIT(2) /* Normal Queue */ @@ -51,12 +49,6 @@ #define TX_PAGE_BOUNDARY_88E (TX_TOTAL_PAGE_NUMBER_88E + 1) -/* Note: For Normal Chip Setting ,modify later */ -#define WMM_NORMAL_TX_TOTAL_PAGE_NUMBER \ - TX_TOTAL_PAGE_NUMBER_88E /* 0xA9 , 0xb0=>176=>22k */ -#define WMM_NORMAL_TX_PAGE_BOUNDARY_88E \ - (WMM_NORMAL_TX_TOTAL_PAGE_NUMBER + 1) /* 0xA9 */ - #include "HalVerDef.h" #include "hal_com.h" @@ -155,8 +147,7 @@ struct hal_data_8188e { u8 AntDivCfg; u8 TRxAntDivType; - u8 OutEpQueueSel; - u8 OutEpNumber; + u8 out_ep_extra_queues; struct P2P_PS_Offload_t p2p_ps_offload; diff --git a/drivers/staging/r8188eu/include/rtl8188e_recv.h b/drivers/staging/r8188eu/include/rtl8188e_recv.h index b752c5c06309be1da6b09bde737cb1d91b674636..dc4f358f646d46334117745de3183a84acd2e2d4 100644 --- a/drivers/staging/r8188eu/include/rtl8188e_recv.h +++ b/drivers/staging/r8188eu/include/rtl8188e_recv.h @@ -33,8 +33,6 @@ enum rx_packet_type { HIS_REPORT,/* USB HISR RPT */ }; -s32 rtl8188eu_init_recv_priv(struct adapter *padapter); -void rtl8188eu_free_recv_priv(struct adapter * padapter); void rtl8188eu_recv_tasklet(unsigned long priv); void update_recvframe_phyinfo_88e(struct recv_frame *fra, struct phy_stat *phy); void update_recvframe_attrib_88e(struct recv_frame *fra, struct recv_stat *stat); diff --git a/drivers/staging/r8188eu/include/rtl8188e_rf.h b/drivers/staging/r8188eu/include/rtl8188e_rf.h index 04556496baad86ff3db4cafe3aa76cfeef52d4cf..63ac0acc68fd8c4ec0c9db8ce618821617d7ebaa 100644 --- a/drivers/staging/r8188eu/include/rtl8188e_rf.h +++ b/drivers/staging/r8188eu/include/rtl8188e_rf.h @@ -8,7 +8,7 @@ #define RF6052_MAX_REG 0x3F #define RF6052_MAX_PATH 2 -int PHY_RF6052_Config8188E(struct adapter *Adapter); +int phy_RF6052_Config_ParaFile(struct adapter *Adapter); void rtl8188e_PHY_RF6052SetBandwidth(struct adapter *Adapter, enum ht_channel_width Bandwidth); void rtl8188e_PHY_RF6052SetCckTxPower(struct adapter *Adapter, u8 *level); diff --git a/drivers/staging/r8188eu/include/rtl8188e_spec.h b/drivers/staging/r8188eu/include/rtl8188e_spec.h index 9e7b1f89037ceea0afe81b22579f20060647be72..e34619140e33cb759bd7ab8d3b3ce33cc9867ae6 100644 --- a/drivers/staging/r8188eu/include/rtl8188e_spec.h +++ b/drivers/staging/r8188eu/include/rtl8188e_spec.h @@ -924,15 +924,9 @@ Current IOREG MAP #define _LLT_OP_VALUE(x) (((x) >> 30) & 0x3) /* 0x0200h ~ 0x027Fh TXDMA Configuration */ -/* 2RQPN */ -#define _HPQ(x) ((x) & 0xFF) -#define _LPQ(x) (((x) & 0xFF) << 8) -#define _PUBQ(x) (((x) & 0xFF) << 16) -/* NOTE: in RQPN_NPQ register */ -#define _NPQ(x) ((x) & 0xFF) - -#define HPQ_PUBLIC_DIS BIT(24) -#define LPQ_PUBLIC_DIS BIT(25) + +#define NUM_HQ 0x29 + #define LD_RQPN BIT(31) /* 2TDECTRL */ diff --git a/drivers/staging/r8188eu/include/rtl8188e_xmit.h b/drivers/staging/r8188eu/include/rtl8188e_xmit.h index 8adb672f7a07dda169cf1d38f7da7a2a4951eae9..6db7fabebea92bae85d8c60c55cee12eeb9b2687 100644 --- a/drivers/staging/r8188eu/include/rtl8188e_xmit.h +++ b/drivers/staging/r8188eu/include/rtl8188e_xmit.h @@ -83,12 +83,6 @@ /* OFFSET 20 */ #define RTY_LMT_EN BIT(17) -enum TXDESC_SC { - SC_DONT_CARE = 0x00, - SC_UPPER = 0x01, - SC_LOWER = 0x02, - SC_DUPLICATE = 0x03 -}; /* OFFSET 20 */ #define SGI BIT(6) #define USB_TXAGG_NUM_SHT 24 @@ -147,6 +141,4 @@ bool rtl8188eu_xmitframe_complete(struct adapter *padapter, struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf); -void handle_txrpt_ccx_88e(struct adapter *adapter, u8 *buf); - #endif /* __RTL8188E_XMIT_H__ */ diff --git a/drivers/staging/r8188eu/include/rtw_ap.h b/drivers/staging/r8188eu/include/rtw_ap.h index 724229fe84aad99820707f36b1e32a42f8fdc00c..8b4134eb3095629e934ec2a3f1fdad4ae6e98989 100644 --- a/drivers/staging/r8188eu/include/rtw_ap.h +++ b/drivers/staging/r8188eu/include/rtw_ap.h @@ -10,8 +10,6 @@ /* external function */ void rtw_indicate_sta_assoc_event(struct adapter *padapter, struct sta_info *psta); -void rtw_indicate_sta_disassoc_event(struct adapter *padapter, - struct sta_info *psta); void init_mlme_ap_info(struct adapter *padapter); void free_mlme_ap_info(struct adapter *padapter); void update_beacon(struct adapter *padapter, u8 ie_id, diff --git a/drivers/staging/r8188eu/include/rtw_cmd.h b/drivers/staging/r8188eu/include/rtw_cmd.h index 6b6d560d71436edb359e1ee16c23c8737b6ec57d..9a76aa85de940b3815c2948281960a5fcdb87a3f 100644 --- a/drivers/staging/r8188eu/include/rtw_cmd.h +++ b/drivers/staging/r8188eu/include/rtw_cmd.h @@ -730,9 +730,7 @@ Result: #define H2C_CMD_OVERFLOW 0x06 #define H2C_RESERVED 0x07 -u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, - int ssid_num, struct rtw_ieee80211_channel *ch, - int ch_num); +u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, int ssid_num); u8 rtw_createbss_cmd(struct adapter *padapter); u8 rtw_setstakey_cmd(struct adapter *padapter, u8 *psta, u8 unicast_key); u8 rtw_clearstakey_cmd(struct adapter *padapter, u8 *psta, u8 entry, u8 enqueue); diff --git a/drivers/staging/r8188eu/include/rtw_led.h b/drivers/staging/r8188eu/include/rtw_led.h index d6b0c1c2f9a2cdb10192e863fae746e6fe4e2c8c..8520f022a67f25d2d3c02cd6ca16f6ae3b335078 100644 --- a/drivers/staging/r8188eu/include/rtw_led.h +++ b/drivers/staging/r8188eu/include/rtw_led.h @@ -21,20 +21,15 @@ enum LED_CTL_MODE { }; enum LED_STATE_871x { - LED_UNKNOWN = 0, - RTW_LED_ON = 1, RTW_LED_OFF = 2, LED_BLINK_NORMAL = 3, LED_BLINK_SLOWLY = 4, LED_BLINK_SCAN = 6, /* LED is blinking during scanning period, * the # of times to blink is depend on time * for scanning. */ - LED_BLINK_StartToBlink = 8,/* Customzied for Sercomm Printer - * Server case */ LED_BLINK_TXRX = 9, LED_BLINK_WPS = 10, /* LED is blinkg during WPS communication */ LED_BLINK_WPS_STOP = 11, - LED_BLINK_RUNTOP = 13, /* Customized for RunTop */ }; struct led_priv { @@ -43,8 +38,6 @@ struct led_priv { bool bRegUseLed; enum LED_STATE_871x CurrLedState; /* Current LED state. */ - enum LED_STATE_871x BlinkingLedState; /* Next state for blinking, - * either RTW_LED_ON or RTW_LED_OFF are. */ bool bLedOn; /* true if LED is ON, false if LED is OFF. */ @@ -54,7 +47,6 @@ struct led_priv { u32 BlinkTimes; /* Number of times to toggle led state for blinking. */ - bool bLedNoLinkBlinkInProgress; bool bLedLinkBlinkInProgress; bool bLedScanBlinkInProgress; struct delayed_work blink_work; diff --git a/drivers/staging/r8188eu/include/rtw_mlme.h b/drivers/staging/r8188eu/include/rtw_mlme.h index d81668498e46252d127d098a2b5610664ee0546d..b69989cbab21d6da14b00bc65db399022b0ba09e 100644 --- a/drivers/staging/r8188eu/include/rtw_mlme.h +++ b/drivers/staging/r8188eu/include/rtw_mlme.h @@ -5,7 +5,6 @@ #define __RTW_MLME_H_ #include "osdep_service.h" -#include "mlme_osdep.h" #include "drv_types.h" #include "wlan_bssdef.h" @@ -64,17 +63,6 @@ enum rt_scan_type { SCAN_MIX, }; -enum SCAN_RESULT_TYPE { - SCAN_RESULT_P2P_ONLY = 0, /* Will return all the P2P devices. */ - SCAN_RESULT_ALL = 1, /* Will return all the scanned device, - * include AP. */ - SCAN_RESULT_WFD_TYPE = 2 /* Will just return the correct WFD - * device. */ - /* If this device is Miracast sink - * device, it will just return all the - * Miracast source devices. */ -}; - /* there are several "locks" in mlme_priv, since mlme_priv is a shared resource between many threads, @@ -433,8 +421,6 @@ void indicate_wx_scan_complete_event(struct adapter *padapter); void rtw_indicate_wx_assoc_event(struct adapter *padapter); void rtw_indicate_wx_disassoc_event(struct adapter *padapter); int event_thread(void *context); -void rtw_join_timeout_handler (struct timer_list *t); -void _rtw_scan_timeout_handler (struct timer_list *t); void rtw_free_network_queue(struct adapter *adapter, u8 isfreeall); int rtw_init_mlme_priv(struct adapter *adapter); void rtw_free_mlme_priv (struct mlme_priv *pmlmepriv); @@ -537,7 +523,7 @@ struct wlan_network *rtw_get_oldest_wlan_network(struct __queue *scanned_queue); void rtw_free_assoc_resources(struct adapter *adapter, int lock_scanned_queue); void rtw_indicate_disconnect(struct adapter *adapter); void rtw_indicate_connect(struct adapter *adapter); -void rtw_indicate_scan_done( struct adapter *padapter, bool aborted); +void rtw_indicate_scan_done(struct adapter *padapter); int rtw_restruct_sec_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_len); @@ -551,10 +537,6 @@ void _rtw_join_timeout_handler(struct adapter *adapter); void rtw_scan_timeout_handler(struct adapter *adapter); void rtw_dynamic_check_timer_handlder(struct adapter *adapter); -#define rtw_is_scan_deny(adapter) false -#define rtw_clear_scan_deny(adapter) do {} while (0) -#define rtw_set_scan_deny_timer_hdl(adapter) do {} while (0) -#define rtw_set_scan_deny(adapter, ms) do {} while (0) void rtw_free_mlme_priv_ie_data(struct mlme_priv *pmlmepriv); diff --git a/drivers/staging/r8188eu/include/rtw_mlme_ext.h b/drivers/staging/r8188eu/include/rtw_mlme_ext.h index 343ce1ce4b3d5c7aadf113d6513bdef65f93205f..b322d0848db9b3ce4880541256e768ab56e525dc 100644 --- a/drivers/staging/r8188eu/include/rtw_mlme_ext.h +++ b/drivers/staging/r8188eu/include/rtw_mlme_ext.h @@ -388,15 +388,11 @@ struct mlme_ext_priv { void init_mlme_ext_priv(struct adapter *adapter); int init_hw_mlme_ext(struct adapter *padapter); void free_mlme_ext_priv (struct mlme_ext_priv *pmlmeext); -extern void init_mlme_ext_timer(struct adapter *padapter); -extern void init_addba_retry_timer(struct adapter *adapt, struct sta_info *sta); extern struct xmit_frame *alloc_mgtxmitframe(struct xmit_priv *pxmitpriv); unsigned char networktype_to_raid(unsigned char network_type); u8 judge_network_type(struct adapter *padapter, unsigned char *rate, int len); void get_rate_set(struct adapter *padapter, unsigned char *pbssrate, int *len); -void UpdateBrateTbl(struct adapter *padapter, u8 *mBratesOS); -void UpdateBrateTblForSoftAP(u8 *bssrateset, u32 bssratelen); void Save_DM_Func_Flag(struct adapter *padapter); void Restore_DM_Func_Flag(struct adapter *padapter); diff --git a/drivers/staging/r8188eu/include/rtw_recv.h b/drivers/staging/r8188eu/include/rtw_recv.h index 66d240a7123d0a06c99a57c3234c6c1f4342aa18..7768b0c5988c9f43dd2c7201a62771c7b1451caa 100644 --- a/drivers/staging/r8188eu/include/rtw_recv.h +++ b/drivers/staging/r8188eu/include/rtw_recv.h @@ -243,6 +243,9 @@ struct recv_frame { struct recv_reorder_ctrl *preorder_ctrl; }; +int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *padapter); +void _rtw_free_recv_priv(struct recv_priv *precvpriv); +s32 rtw_recv_entry(struct recv_frame *precv_frame); struct recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue); struct recv_frame *rtw_alloc_recvframe(struct __queue *pfree_recv_queue); int rtw_free_recvframe(struct recv_frame *precvframe, diff --git a/drivers/staging/r8188eu/include/rtw_xmit.h b/drivers/staging/r8188eu/include/rtw_xmit.h index 034a9f8f51c97b4837a2feb56741000c3abd3644..82efcd54af3f910007ab4b1493d522cf09d94a16 100644 --- a/drivers/staging/r8188eu/include/rtw_xmit.h +++ b/drivers/staging/r8188eu/include/rtw_xmit.h @@ -7,6 +7,9 @@ #include "osdep_service.h" #include "drv_types.h" +#define NR_XMITFRAME 256 +#define WMM_XMIT_THRESHOLD (NR_XMITFRAME * 2 / 5) + #define MAX_XMITBUF_SZ (20480) /* 20k */ #define NR_XMITBUFF (4) @@ -304,6 +307,15 @@ struct xmit_priv { struct submit_ctx ack_tx_ops; }; +struct pkt_file { + struct sk_buff *pkt; + size_t pkt_len; /* the remainder length of the open_file */ + unsigned char *cur_buffer; + u8 *buf_start; + u8 *cur_addr; + size_t buf_len; +}; + struct xmit_buf *rtw_alloc_xmitbuf_ext(struct xmit_priv *pxmitpriv); s32 rtw_free_xmitbuf_ext(struct xmit_priv *pxmitpriv, struct xmit_buf *pxmitbuf); @@ -355,7 +367,7 @@ u32 rtw_get_ff_hwaddr(struct xmit_frame *pxmitframe); int rtw_ack_tx_wait(struct xmit_priv *pxmitpriv, u32 timeout_ms); void rtw_ack_tx_done(struct xmit_priv *pxmitpriv, int status); -/* include after declaring struct xmit_buf, in order to avoid warning */ -#include "xmit_osdep.h" +void rtw_xmit_complete(struct adapter *padapter, struct xmit_frame *pxframe); +netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); #endif /* _RTL871X_XMIT_H_ */ diff --git a/drivers/staging/r8188eu/include/wlan_bssdef.h b/drivers/staging/r8188eu/include/wlan_bssdef.h index 9d1c9e763287a68bbe582aabc4ac341b633d645c..81bda91a4136bab4eb32aa32113044b5f0f3d98c 100644 --- a/drivers/staging/r8188eu/include/wlan_bssdef.h +++ b/drivers/staging/r8188eu/include/wlan_bssdef.h @@ -133,10 +133,6 @@ struct ndis_802_11_assoc_info { u32 OffsetResponseIEs; }; -enum ndis_802_11_reload_def { - Ndis802_11ReloadWEPKeys -}; - /* Key mapping keys require a BSSID */ struct ndis_802_11_key { u32 Length; /* Length of this structure */ diff --git a/drivers/staging/r8188eu/include/xmit_osdep.h b/drivers/staging/r8188eu/include/xmit_osdep.h deleted file mode 100644 index 00658681fef9d0d8a171216fdfc0f41075cbfb2d..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/include/xmit_osdep.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#ifndef __XMIT_OSDEP_H_ -#define __XMIT_OSDEP_H_ - -#include "osdep_service.h" -#include "drv_types.h" - -struct pkt_file { - struct sk_buff *pkt; - size_t pkt_len; /* the remainder length of the open_file */ - unsigned char *cur_buffer; - u8 *buf_start; - u8 *cur_addr; - size_t buf_len; -}; - -extern int rtw_ht_enable; -extern int rtw_cbw40_enable; -extern int rtw_ampdu_enable;/* for enable tx_ampdu */ - -#define NR_XMITFRAME 256 - -struct xmit_priv; -struct pkt_attrib; -struct sta_xmit_priv; -struct xmit_frame; -struct xmit_buf; - -int rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); - -void rtw_os_xmit_schedule(struct adapter *padapter); - -int rtw_os_xmit_resource_alloc(struct adapter *padapter, - struct xmit_buf *pxmitbuf, u32 alloc_sz); -void rtw_os_xmit_resource_free(struct adapter *padapter, - struct xmit_buf *pxmitbuf, u32 free_sz); - -uint rtw_remainder_len(struct pkt_file *pfile); -void _rtw_open_pktfile(struct sk_buff *pkt, struct pkt_file *pfile); -uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen); -bool rtw_endofpktfile(struct pkt_file *pfile); - -void rtw_os_pkt_complete(struct adapter *padapter, struct sk_buff *pkt); -void rtw_os_xmit_complete(struct adapter *padapter, - struct xmit_frame *pxframe); - -#endif /* __XMIT_OSDEP_H_ */ diff --git a/drivers/staging/r8188eu/os_dep/ioctl_linux.c b/drivers/staging/r8188eu/os_dep/ioctl_linux.c index 7f91dac2e41bcb8f545895d5bc726a4331c9c9ba..2de2e1e32738a7569cba0fd82e71497a8d1e1c3c 100644 --- a/drivers/staging/r8188eu/os_dep/ioctl_linux.c +++ b/drivers/staging/r8188eu/os_dep/ioctl_linux.c @@ -1099,7 +1099,7 @@ static int rtw_wx_set_scan(struct net_device *dev, struct iw_request_info *a, spin_lock_bh(&pmlmepriv->lock); - _status = rtw_sitesurvey_cmd(padapter, ssid, 1, NULL, 0); + _status = rtw_sitesurvey_cmd(padapter, ssid, 1); spin_unlock_bh(&pmlmepriv->lock); } @@ -1836,7 +1836,7 @@ static int rtw_wx_set_enc_ext(struct net_device *dev, goto out; } - strlcpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN); + strscpy((char *)param->u.crypt.alg, alg_name, IEEE_CRYPT_ALG_NAME_LEN); if (pext->ext_flags & IW_ENCODE_EXT_SET_TX_KEY) param->u.crypt.set_tx = 1; @@ -2079,7 +2079,7 @@ static int rtw_wext_p2p_enable(struct net_device *dev, struct iw_request_info *info, union iwreq_data *wrqu, char *extra) { - int ret = 0; + int ret; struct adapter *padapter = (struct adapter *)rtw_netdev_priv(dev); struct wifidirect_info *pwdinfo = &padapter->wdinfo; struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; @@ -2094,10 +2094,9 @@ static int rtw_wext_p2p_enable(struct net_device *dev, else if (*extra == '3') init_role = P2P_ROLE_GO; - if (_FAIL == rtw_p2p_enable(padapter, init_role)) { - ret = -EFAULT; - goto exit; - } + ret = rtw_p2p_enable(padapter, init_role); + if (ret) + return ret; /* set channel/bandwidth */ if (init_role != P2P_ROLE_DISABLE) { @@ -2121,8 +2120,7 @@ static int rtw_wext_p2p_enable(struct net_device *dev, set_channel_bwmode(padapter, channel, ch_offset, bwmode); } -exit: - return ret; + return 0; } static void rtw_p2p_set_go_nego_ssid(struct net_device *dev, diff --git a/drivers/staging/r8188eu/os_dep/mlme_linux.c b/drivers/staging/r8188eu/os_dep/mlme_linux.c deleted file mode 100644 index 899d8e9c383449ac4f92a45003534c2f6ff78900..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/os_dep/mlme_linux.c +++ /dev/null @@ -1,205 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2011 Realtek Corporation. i*/ - -#define _MLME_OSDEP_C_ - -#include "../include/osdep_service.h" -#include "../include/drv_types.h" -#include "../include/mlme_osdep.h" - -void rtw_join_timeout_handler (struct timer_list *t) -{ - struct adapter *adapter = from_timer(adapter, t, mlmepriv.assoc_timer); - - _rtw_join_timeout_handler(adapter); -} - -void _rtw_scan_timeout_handler (struct timer_list *t) -{ - struct adapter *adapter = from_timer(adapter, t, mlmepriv.scan_to_timer); - - rtw_scan_timeout_handler(adapter); -} - -static void _dynamic_check_timer_handlder(struct timer_list *t) -{ - struct adapter *adapter = from_timer(adapter, t, mlmepriv.dynamic_chk_timer); - - rtw_dynamic_check_timer_handlder(adapter); - _set_timer(&adapter->mlmepriv.dynamic_chk_timer, 2000); -} - -void rtw_init_mlme_timer(struct adapter *padapter) -{ - struct mlme_priv *pmlmepriv = &padapter->mlmepriv; - - timer_setup(&pmlmepriv->assoc_timer, rtw_join_timeout_handler, 0); - timer_setup(&pmlmepriv->scan_to_timer, _rtw_scan_timeout_handler, 0); - timer_setup(&pmlmepriv->dynamic_chk_timer, _dynamic_check_timer_handlder, 0); -} - -void rtw_os_indicate_connect(struct adapter *adapter) -{ - - rtw_indicate_wx_assoc_event(adapter); - netif_carrier_on(adapter->pnetdev); - if (adapter->pid[2] != 0) - rtw_signal_process(adapter->pid[2], SIGALRM); - -} - -void rtw_os_indicate_scan_done(struct adapter *padapter, bool aborted) -{ - indicate_wx_scan_complete_event(padapter); -} - -static struct rt_pmkid_list backup_pmkid[NUM_PMKID_CACHE]; - -void rtw_reset_securitypriv(struct adapter *adapter) -{ - u8 backup_index = 0; - u8 backup_counter = 0x00; - u32 backup_time = 0; - - if (adapter->securitypriv.dot11AuthAlgrthm == dot11AuthAlgrthm_8021X) { - /* 802.1x */ - /* We have to backup the PMK information for WiFi PMK Caching test item. */ - /* Backup the btkip_countermeasure information. */ - /* When the countermeasure is trigger, the driver have to disconnect with AP for 60 seconds. */ - memcpy(&backup_pmkid[0], &adapter->securitypriv.PMKIDList[0], sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE); - backup_index = adapter->securitypriv.PMKIDIndex; - backup_counter = adapter->securitypriv.btkip_countermeasure; - backup_time = adapter->securitypriv.btkip_countermeasure_time; - memset((unsigned char *)&adapter->securitypriv, 0, sizeof(struct security_priv)); - - /* Restore the PMK information to securitypriv structure for the following connection. */ - memcpy(&adapter->securitypriv.PMKIDList[0], - &backup_pmkid[0], - sizeof(struct rt_pmkid_list) * NUM_PMKID_CACHE); - adapter->securitypriv.PMKIDIndex = backup_index; - adapter->securitypriv.btkip_countermeasure = backup_counter; - adapter->securitypriv.btkip_countermeasure_time = backup_time; - adapter->securitypriv.ndisauthtype = Ndis802_11AuthModeOpen; - adapter->securitypriv.ndisencryptstatus = Ndis802_11WEPDisabled; - } else { - /* reset values in securitypriv */ - struct security_priv *psec_priv = &adapter->securitypriv; - - psec_priv->dot11AuthAlgrthm = dot11AuthAlgrthm_Open; /* open system */ - psec_priv->dot11PrivacyAlgrthm = _NO_PRIVACY_; - psec_priv->dot11PrivacyKeyIndex = 0; - psec_priv->dot118021XGrpPrivacy = _NO_PRIVACY_; - psec_priv->dot118021XGrpKeyid = 1; - psec_priv->ndisauthtype = Ndis802_11AuthModeOpen; - psec_priv->ndisencryptstatus = Ndis802_11WEPDisabled; - } -} - -void rtw_os_indicate_disconnect(struct adapter *adapter) -{ - - netif_carrier_off(adapter->pnetdev); /* Do it first for tx broadcast pkt after disconnection issue! */ - rtw_indicate_wx_disassoc_event(adapter); - rtw_reset_securitypriv(adapter); -} - -void rtw_report_sec_ie(struct adapter *adapter, u8 authmode, u8 *sec_ie) -{ - uint len; - u8 *buff, *p, i; - union iwreq_data wrqu; - - buff = NULL; - if (authmode == _WPA_IE_ID_) { - buff = kzalloc(IW_CUSTOM_MAX, GFP_ATOMIC); - if (!buff) - return; - p = buff; - p += sprintf(p, "ASSOCINFO(ReqIEs ="); - len = sec_ie[1] + 2; - len = (len < IW_CUSTOM_MAX) ? len : IW_CUSTOM_MAX; - for (i = 0; i < len; i++) - p += sprintf(p, "%02x", sec_ie[i]); - p += sprintf(p, ")"); - memset(&wrqu, 0, sizeof(wrqu)); - wrqu.data.length = p - buff; - wrqu.data.length = (wrqu.data.length < IW_CUSTOM_MAX) ? - wrqu.data.length : IW_CUSTOM_MAX; - wireless_send_event(adapter->pnetdev, IWEVCUSTOM, &wrqu, buff); - kfree(buff); - } -} - -static void _survey_timer_hdl(struct timer_list *t) -{ - struct adapter *padapter = from_timer(padapter, t, mlmeextpriv.survey_timer); - - survey_timer_hdl(padapter); -} - -static void _link_timer_hdl(struct timer_list *t) -{ - struct adapter *padapter = from_timer(padapter, t, mlmeextpriv.link_timer); - link_timer_hdl(padapter); -} - -static void _addba_timer_hdl(struct timer_list *t) -{ - struct sta_info *psta = from_timer(psta, t, addba_retry_timer); - addba_timer_hdl(psta); -} - -void init_addba_retry_timer(struct adapter *padapter, struct sta_info *psta) -{ - timer_setup(&psta->addba_retry_timer, _addba_timer_hdl, 0); -} - -void init_mlme_ext_timer(struct adapter *padapter) -{ - struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; - - timer_setup(&pmlmeext->survey_timer, _survey_timer_hdl, 0); - timer_setup(&pmlmeext->link_timer, _link_timer_hdl, 0); -} - -void rtw_indicate_sta_assoc_event(struct adapter *padapter, struct sta_info *psta) -{ - union iwreq_data wrqu; - struct sta_priv *pstapriv = &padapter->stapriv; - - if (!psta) - return; - - if (psta->aid > NUM_STA) - return; - - if (pstapriv->sta_aid[psta->aid - 1] != psta) - return; - - wrqu.addr.sa_family = ARPHRD_ETHER; - - memcpy(wrqu.addr.sa_data, psta->hwaddr, ETH_ALEN); - - wireless_send_event(padapter->pnetdev, IWEVREGISTERED, &wrqu, NULL); -} - -void rtw_indicate_sta_disassoc_event(struct adapter *padapter, struct sta_info *psta) -{ - union iwreq_data wrqu; - struct sta_priv *pstapriv = &padapter->stapriv; - - if (!psta) - return; - - if (psta->aid > NUM_STA) - return; - - if (pstapriv->sta_aid[psta->aid - 1] != psta) - return; - - wrqu.addr.sa_family = ARPHRD_ETHER; - - memcpy(wrqu.addr.sa_data, psta->hwaddr, ETH_ALEN); - - wireless_send_event(padapter->pnetdev, IWEVEXPIRED, &wrqu, NULL); -} diff --git a/drivers/staging/r8188eu/os_dep/os_intfs.c b/drivers/staging/r8188eu/os_dep/os_intfs.c index cac9553666e6df97dc5f7a24126bd6d3b4199174..6a45315d01a29b9c607147298430830120190240 100644 --- a/drivers/staging/r8188eu/os_dep/os_intfs.c +++ b/drivers/staging/r8188eu/os_dep/os_intfs.c @@ -5,8 +5,6 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/xmit_osdep.h" -#include "../include/recv_osdep.h" #include "../include/hal_intf.h" #include "../include/rtw_ioctl.h" #include "../include/usb_osintf.h" @@ -17,13 +15,12 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Realtek Wireless Lan Driver"); MODULE_AUTHOR("Realtek Semiconductor Corp."); -MODULE_VERSION(DRIVERVERSION); +MODULE_FIRMWARE(FW_RTL8188EU); #define CONFIG_BR_EXT_BRNAME "br0" #define RTW_NOTCH_FILTER 0 /* 0:Disable, 1:Enable, */ /* module param defaults */ -static int rtw_chip_version = 0x00; static int rtw_rfintfs = HWPI; static int rtw_lbkmode;/* RTL8712_AIR_TRX; */ static int rtw_network_mode = Ndis802_11IBSS;/* Ndis802_11Infrastructure; infra, ad-hoc, auto */ @@ -66,9 +63,9 @@ static int rtw_uapsd_acvo_en; static int rtw_led_enable = 1; -int rtw_ht_enable = 1; -int rtw_cbw40_enable = 3; /* 0 :disable, bit(0): enable 2.4g, bit(1): enable 5g */ -int rtw_ampdu_enable = 1;/* for enable tx_ampdu */ +static int rtw_ht_enable = 1; +static int rtw_cbw40_enable = 3; /* 0 :disable, bit(0): enable 2.4g, bit(1): enable 5g */ +static int rtw_ampdu_enable = 1;/* for enable tx_ampdu */ static int rtw_rx_stbc = 1;/* 0: disable, bit(0):enable 2.4g, bit(1):enable 5g, default is set to enable 2.4GHZ for IOT issue with bufflao's AP at 5GHZ */ static int rtw_ampdu_amsdu;/* 0: disabled, 1:enabled, 2:auto */ @@ -105,7 +102,6 @@ char *rtw_initmac; /* temp mac address if users want to use instead of the mac module_param(rtw_initmac, charp, 0644); module_param(rtw_channel_plan, int, 0644); -module_param(rtw_chip_version, int, 0644); module_param(rtw_rfintfs, int, 0644); module_param(rtw_lbkmode, int, 0644); module_param(rtw_network_mode, int, 0644); @@ -152,7 +148,6 @@ static uint loadparam(struct adapter *padapter) { struct registry_priv *registry_par = &padapter->registrypriv; - registry_par->chip_version = (u8)rtw_chip_version; registry_par->rfintfs = (u8)rtw_rfintfs; registry_par->lbkmode = (u8)rtw_lbkmode; registry_par->network_mode = (u8)rtw_network_mode; @@ -621,7 +616,7 @@ void netdev_br_init(struct net_device *netdev) rcu_read_unlock(); } -int _netdev_open(struct net_device *pnetdev) +static int _netdev_open(struct net_device *pnetdev) { uint status; struct adapter *padapter = (struct adapter *)rtw_netdev_priv(pnetdev); @@ -635,7 +630,7 @@ int _netdev_open(struct net_device *pnetdev) if (status == _FAIL) goto netdev_open_error; - pr_info("MAC Address = %pM\n", pnetdev->dev_addr); + netdev_dbg(pnetdev, "MAC Address = %pM\n", pnetdev->dev_addr); status = rtw_start_drv_threads(padapter); if (status == _FAIL) { diff --git a/drivers/staging/r8188eu/os_dep/osdep_service.c b/drivers/staging/r8188eu/os_dep/osdep_service.c index 3504a0a9ba87cb31eaebfbb30660b1ce8ff6eb80..88271f956b52189dc3f14242760fc79b0134e991 100644 --- a/drivers/staging/r8188eu/os_dep/osdep_service.c +++ b/drivers/staging/r8188eu/os_dep/osdep_service.c @@ -5,7 +5,6 @@ #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" #include "../include/rtw_ioctl_set.h" /* @@ -54,14 +53,13 @@ struct net_device *rtw_alloc_etherdev_with_old_priv(int sizeof_priv, pnetdev = alloc_etherdev_mq(sizeof(struct rtw_netdev_priv_indicator), 4); if (!pnetdev) - goto RETURN; + return NULL; pnetdev->dev.type = &wlan_type; pnpi = netdev_priv(pnetdev); pnpi->priv = old_priv; pnpi->sizeof_priv = sizeof_priv; -RETURN: return pnetdev; } @@ -72,19 +70,18 @@ struct net_device *rtw_alloc_etherdev(int sizeof_priv) pnetdev = alloc_etherdev_mq(sizeof(struct rtw_netdev_priv_indicator), 4); if (!pnetdev) - goto RETURN; + return NULL; pnpi = netdev_priv(pnetdev); pnpi->priv = vzalloc(sizeof_priv); if (!pnpi->priv) { free_netdev(pnetdev); - pnetdev = NULL; - goto RETURN; + return NULL; } pnpi->sizeof_priv = sizeof_priv; -RETURN: + return pnetdev; } diff --git a/drivers/staging/r8188eu/os_dep/recv_linux.c b/drivers/staging/r8188eu/os_dep/recv_linux.c deleted file mode 100644 index 1e14b6d49795e7dd7ac9f511e47decd9db855dde..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/os_dep/recv_linux.c +++ /dev/null @@ -1,165 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2011 Realtek Corporation. */ - -#define _RECV_OSDEP_C_ - -#include "../include/osdep_service.h" -#include "../include/drv_types.h" - -#include "../include/wifi.h" -#include "../include/recv_osdep.h" - -#include "../include/osdep_intf.h" -#include "../include/usb_ops.h" - -/* alloc os related resource in struct recv_buf */ -int rtw_os_recvbuf_resource_alloc(struct adapter *padapter, - struct recv_buf *precvbuf) -{ - int res = _SUCCESS; - - precvbuf->purb = usb_alloc_urb(0, GFP_KERNEL); - if (!precvbuf->purb) - res = _FAIL; - precvbuf->pskb = NULL; - precvbuf->reuse = false; - return res; -} - -/* free os related resource in struct recv_buf */ -int rtw_os_recvbuf_resource_free(struct adapter *padapter, - struct recv_buf *precvbuf) -{ - usb_free_urb(precvbuf->purb); - return _SUCCESS; -} - -void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup) -{ - union iwreq_data wrqu; - struct iw_michaelmicfailure ev; - struct mlme_priv *pmlmepriv = &padapter->mlmepriv; - struct security_priv *psecuritypriv = &padapter->securitypriv; - u32 cur_time = 0; - - if (psecuritypriv->last_mic_err_time == 0) { - psecuritypriv->last_mic_err_time = jiffies; - } else { - cur_time = jiffies; - - if (cur_time - psecuritypriv->last_mic_err_time < 60 * HZ) { - psecuritypriv->btkip_countermeasure = true; - psecuritypriv->last_mic_err_time = 0; - psecuritypriv->btkip_countermeasure_time = cur_time; - } else { - psecuritypriv->last_mic_err_time = jiffies; - } - } - - memset(&ev, 0x00, sizeof(ev)); - if (bgroup) - ev.flags |= IW_MICFAILURE_GROUP; - else - ev.flags |= IW_MICFAILURE_PAIRWISE; - - ev.src_addr.sa_family = ARPHRD_ETHER; - memcpy(ev.src_addr.sa_data, &pmlmepriv->assoc_bssid[0], ETH_ALEN); - memset(&wrqu, 0x00, sizeof(wrqu)); - wrqu.data.length = sizeof(ev); - wireless_send_event(padapter->pnetdev, IWEVMICHAELMICFAILURE, - &wrqu, (char *)&ev); -} - -int rtw_recv_indicatepkt(struct adapter *padapter, - struct recv_frame *precv_frame) -{ - struct recv_priv *precvpriv; - struct __queue *pfree_recv_queue; - struct sk_buff *skb; - struct mlme_priv *pmlmepriv = &padapter->mlmepriv; - - precvpriv = &padapter->recvpriv; - pfree_recv_queue = &precvpriv->free_recv_queue; - - skb = precv_frame->pkt; - if (!skb) - goto _recv_indicatepkt_drop; - - skb->data = precv_frame->rx_data; - - skb_set_tail_pointer(skb, precv_frame->len); - - skb->len = precv_frame->len; - - if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) { - struct sk_buff *pskb2 = NULL; - struct sta_info *psta = NULL; - struct sta_priv *pstapriv = &padapter->stapriv; - struct rx_pkt_attrib *pattrib = &precv_frame->attrib; - bool bmcast = is_multicast_ether_addr(pattrib->dst); - - if (memcmp(pattrib->dst, myid(&padapter->eeprompriv), - ETH_ALEN)) { - if (bmcast) { - psta = rtw_get_bcmc_stainfo(padapter); - pskb2 = skb_clone(skb, GFP_ATOMIC); - } else { - psta = rtw_get_stainfo(pstapriv, pattrib->dst); - } - - if (psta) { - struct net_device *pnetdev; - - pnetdev = (struct net_device *)padapter->pnetdev; - skb->dev = pnetdev; - skb_set_queue_mapping(skb, rtw_recv_select_queue(skb)); - - rtw_xmit_entry(skb, pnetdev); - - if (bmcast) - skb = pskb2; - else - goto _recv_indicatepkt_end; - } - } - } - - rcu_read_lock(); - rcu_dereference(padapter->pnetdev->rx_handler_data); - rcu_read_unlock(); - - skb->ip_summed = CHECKSUM_NONE; - skb->dev = padapter->pnetdev; - skb->protocol = eth_type_trans(skb, padapter->pnetdev); - - netif_rx(skb); - -_recv_indicatepkt_end: - - /* pointers to NULL before rtw_free_recvframe() */ - precv_frame->pkt = NULL; - - rtw_free_recvframe(precv_frame, pfree_recv_queue); - - return _SUCCESS; - -_recv_indicatepkt_drop: - - /* enqueue back to free_recv_queue */ - rtw_free_recvframe(precv_frame, pfree_recv_queue); - - return _FAIL; -} - -static void _rtw_reordering_ctrl_timeout_handler(struct timer_list *t) -{ - struct recv_reorder_ctrl *preorder_ctrl; - - preorder_ctrl = from_timer(preorder_ctrl, t, reordering_ctrl_timer); - rtw_reordering_ctrl_timeout_handler(preorder_ctrl); -} - -void rtw_init_recv_timer(struct recv_reorder_ctrl *preorder_ctrl) -{ - timer_setup(&preorder_ctrl->reordering_ctrl_timer, _rtw_reordering_ctrl_timeout_handler, 0); -} diff --git a/drivers/staging/r8188eu/os_dep/usb_intf.c b/drivers/staging/r8188eu/os_dep/usb_intf.c index cc2b44f60c468a8ac11093821ec8e63fea28ad20..5fbfbcd95de289552242b598e5c656bf59e8190d 100644 --- a/drivers/staging/r8188eu/os_dep/usb_intf.c +++ b/drivers/staging/r8188eu/os_dep/usb_intf.c @@ -4,8 +4,6 @@ #include #include "../include/osdep_service.h" #include "../include/drv_types.h" -#include "../include/recv_osdep.h" -#include "../include/xmit_osdep.h" #include "../include/hal_intf.h" #include "../include/osdep_intf.h" #include "../include/usb_ops.h" @@ -28,6 +26,7 @@ static struct usb_device_id rtw_usb_id_tbl[] = { /*=== Realtek demoboard ===*/ {USB_DEVICE(USB_VENDER_ID_REALTEK, 0x8179)}, /* 8188EUS */ {USB_DEVICE(USB_VENDER_ID_REALTEK, 0x0179)}, /* 8188ETV */ + {USB_DEVICE(USB_VENDER_ID_REALTEK, 0xffef)}, /* Rosewill USB-N150 Nano */ /*=== Customer ID ===*/ /****** 8188EUS ********/ {USB_DEVICE(0x07B8, 0x8179)}, /* Abocom - Abocom */ @@ -54,7 +53,7 @@ struct rtw_usb_drv { }; static struct rtw_usb_drv rtl8188e_usb_drv = { - .usbdrv.name = "r8188eu", + .usbdrv.name = KBUILD_MODNAME, .usbdrv.probe = rtw_drv_init, .usbdrv.disconnect = rtw_dev_remove, .usbdrv.id_table = rtw_usb_id_tbl, @@ -231,7 +230,7 @@ static int rtw_suspend(struct usb_interface *pusb_intf, pm_message_t message) mutex_unlock(&pwrpriv->lock); if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) - rtw_indicate_scan_done(padapter, 1); + rtw_indicate_scan_done(padapter); if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) rtw_indicate_disconnect(padapter); @@ -287,17 +286,17 @@ exit: * We accept the new device by returning 0. */ -static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj, - struct usb_interface *pusb_intf) +static int rtw_usb_if1_init(struct dvobj_priv *dvobj, struct usb_interface *pusb_intf) { struct adapter *padapter = NULL; struct net_device *pnetdev = NULL; struct io_priv *piopriv; struct intf_hdl *pintf; + int ret; padapter = vzalloc(sizeof(*padapter)); if (!padapter) - return NULL; + return -ENOMEM; padapter->dvobj = dvobj; dvobj->if1 = padapter; @@ -306,12 +305,13 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj, padapter->hw_init_mutex = &usb_drv->hw_init_mutex; - if (rtw_handle_dualmac(padapter, 1) != _SUCCESS) - goto free_adapter; + rtw_handle_dualmac(padapter, 1); pnetdev = rtw_init_netdev(padapter); - if (!pnetdev) + if (!pnetdev) { + ret = -ENODEV; goto handle_dualmac; + } SET_NETDEV_DEV(pnetdev, dvobj_to_dev(dvobj)); padapter = rtw_netdev_priv(pnetdev); @@ -329,14 +329,20 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj, rtl8188e_read_chip_version(padapter); /* step usb endpoint mapping */ - rtl8188eu_interface_configure(padapter); + ret = rtl8188eu_interface_configure(padapter); + if (ret) + goto handle_dualmac; /* step read efuse/eeprom data and get mac_addr */ - ReadAdapterInfo8188EU(padapter); + ret = ReadAdapterInfo8188EU(padapter); + if (ret) + goto handle_dualmac; /* step 5. */ - if (rtw_init_drv_sw(padapter) == _FAIL) + if (rtw_init_drv_sw(padapter) == _FAIL) { + ret = -ENODEV; goto handle_dualmac; + } #ifdef CONFIG_PM if (padapter->pwrctrlpriv.bSupportRemoteWakeup) { @@ -351,7 +357,8 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj, usb_autopm_get_interface(pusb_intf); /* alloc dev name after read efuse. */ - if (rtw_init_netdev_name(pnetdev, padapter->registrypriv.ifname) < 0) + ret = rtw_init_netdev_name(pnetdev, padapter->registrypriv.ifname); + if (ret) goto free_drv_sw; rtw_macaddr_cfg(padapter->eeprompriv.mac_addr); rtw_init_wifidirect_addrs(padapter, padapter->eeprompriv.mac_addr, @@ -359,23 +366,23 @@ static struct adapter *rtw_usb_if1_init(struct dvobj_priv *dvobj, eth_hw_addr_set(pnetdev, padapter->eeprompriv.mac_addr); /* step 6. Tell the network stack we exist */ - if (register_netdev(pnetdev) != 0) + ret = register_netdev(pnetdev); + if (ret) goto free_drv_sw; - return padapter; + return 0; free_drv_sw: rtw_cancel_all_timer(padapter); rtw_free_drv_sw(padapter); handle_dualmac: rtw_handle_dualmac(padapter, 0); -free_adapter: if (pnetdev) rtw_free_netdev(pnetdev); else vfree(padapter); - return NULL; + return ret; } static void rtw_usb_if1_deinit(struct adapter *if1) @@ -403,27 +410,24 @@ static void rtw_usb_if1_deinit(struct adapter *if1) static int rtw_drv_init(struct usb_interface *pusb_intf, const struct usb_device_id *pdid) { - struct adapter *if1 = NULL; struct dvobj_priv *dvobj; + int ret; /* Initialize dvobj_priv */ dvobj = usb_dvobj_init(pusb_intf); if (!dvobj) - goto err; + return -ENODEV; - if1 = rtw_usb_if1_init(dvobj, pusb_intf); - if (!if1) - goto free_dvobj; + ret = rtw_usb_if1_init(dvobj, pusb_intf); + if (ret) { + usb_dvobj_deinit(pusb_intf); + return ret; + } if (ui_pid[1] != 0) rtw_signal_process(ui_pid[1], SIGUSR2); return 0; - -free_dvobj: - usb_dvobj_deinit(pusb_intf); -err: - return -ENODEV; } /* diff --git a/drivers/staging/r8188eu/os_dep/xmit_linux.c b/drivers/staging/r8188eu/os_dep/xmit_linux.c deleted file mode 100644 index 91a1e4e3219ac60120eb9bada2fe19bd3843b15f..0000000000000000000000000000000000000000 --- a/drivers/staging/r8188eu/os_dep/xmit_linux.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright(c) 2007 - 2012 Realtek Corporation. */ - -#define _XMIT_OSDEP_C_ - -#include "../include/osdep_service.h" -#include "../include/drv_types.h" -#include "../include/wifi.h" -#include "../include/mlme_osdep.h" -#include "../include/xmit_osdep.h" -#include "../include/osdep_intf.h" -#include "../include/usb_osintf.h" - -uint rtw_remainder_len(struct pkt_file *pfile) -{ - return pfile->buf_len - ((size_t)(pfile->cur_addr) - - (size_t)(pfile->buf_start)); -} - -void _rtw_open_pktfile(struct sk_buff *pktptr, struct pkt_file *pfile) -{ - - if (!pktptr) { - pr_err("8188eu: pktptr is NULL\n"); - return; - } - if (!pfile) { - pr_err("8188eu: pfile is NULL\n"); - return; - } - pfile->pkt = pktptr; - pfile->cur_addr = pktptr->data; - pfile->buf_start = pktptr->data; - pfile->pkt_len = pktptr->len; - pfile->buf_len = pktptr->len; - - pfile->cur_buffer = pfile->buf_start; - -} - -uint _rtw_pktfile_read(struct pkt_file *pfile, u8 *rmem, uint rlen) -{ - uint len = 0; - - len = rtw_remainder_len(pfile); - len = (rlen > len) ? len : rlen; - - if (rmem) - skb_copy_bits(pfile->pkt, pfile->buf_len - pfile->pkt_len, rmem, len); - - pfile->cur_addr += len; - pfile->pkt_len -= len; - - return len; -} - -bool rtw_endofpktfile(struct pkt_file *pfile) -{ - - if (pfile->pkt_len == 0) { - - return true; - } - - return false; -} - -int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz) -{ - pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL); - if (!pxmitbuf->pallocated_buf) - return _FAIL; - - pxmitbuf->pbuf = (u8 *)ALIGN((size_t)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ); - pxmitbuf->dma_transfer_addr = 0; - - pxmitbuf->pxmit_urb = usb_alloc_urb(0, GFP_KERNEL); - if (!pxmitbuf->pxmit_urb) - return _FAIL; - - return _SUCCESS; -} - -void rtw_os_xmit_resource_free(struct adapter *padapter, - struct xmit_buf *pxmitbuf, u32 free_sz) -{ - usb_free_urb(pxmitbuf->pxmit_urb); - - kfree(pxmitbuf->pallocated_buf); -} - -#define WMM_XMIT_THRESHOLD (NR_XMITFRAME * 2 / 5) - -void rtw_os_pkt_complete(struct adapter *padapter, struct sk_buff *pkt) -{ - u16 queue; - struct xmit_priv *pxmitpriv = &padapter->xmitpriv; - - queue = skb_get_queue_mapping(pkt); - if (padapter->registrypriv.wifi_spec) { - if (__netif_subqueue_stopped(padapter->pnetdev, queue) && - (pxmitpriv->hwxmits[queue].accnt < WMM_XMIT_THRESHOLD)) - netif_wake_subqueue(padapter->pnetdev, queue); - } else { - if (__netif_subqueue_stopped(padapter->pnetdev, queue)) - netif_wake_subqueue(padapter->pnetdev, queue); - } - - dev_kfree_skb_any(pkt); -} - -void rtw_os_xmit_complete(struct adapter *padapter, struct xmit_frame *pxframe) -{ - if (pxframe->pkt) - rtw_os_pkt_complete(padapter, pxframe->pkt); - pxframe->pkt = NULL; -} - -void rtw_os_xmit_schedule(struct adapter *padapter) -{ - struct xmit_priv *pxmitpriv; - - if (!padapter) - return; - - pxmitpriv = &padapter->xmitpriv; - - spin_lock_bh(&pxmitpriv->lock); - - if (rtw_txframes_pending(padapter)) - tasklet_hi_schedule(&pxmitpriv->xmit_tasklet); - - spin_unlock_bh(&pxmitpriv->lock); -} - -static void rtw_check_xmit_resource(struct adapter *padapter, struct sk_buff *pkt) -{ - struct xmit_priv *pxmitpriv = &padapter->xmitpriv; - u16 queue; - - queue = skb_get_queue_mapping(pkt); - if (padapter->registrypriv.wifi_spec) { - /* No free space for Tx, tx_worker is too slow */ - if (pxmitpriv->hwxmits[queue].accnt > WMM_XMIT_THRESHOLD) - netif_stop_subqueue(padapter->pnetdev, queue); - } else { - if (pxmitpriv->free_xmitframe_cnt <= 4) { - if (!netif_tx_queue_stopped(netdev_get_tx_queue(padapter->pnetdev, queue))) - netif_stop_subqueue(padapter->pnetdev, queue); - } - } -} - -static int rtw_mlcst2unicst(struct adapter *padapter, struct sk_buff *skb) -{ - struct sta_priv *pstapriv = &padapter->stapriv; - struct xmit_priv *pxmitpriv = &padapter->xmitpriv; - struct list_head *phead, *plist; - struct sk_buff *newskb; - struct sta_info *psta = NULL; - s32 res; - - spin_lock_bh(&pstapriv->asoc_list_lock); - phead = &pstapriv->asoc_list; - plist = phead->next; - - /* free sta asoc_queue */ - while (phead != plist) { - psta = container_of(plist, struct sta_info, asoc_list); - - plist = plist->next; - - /* avoid come from STA1 and send back STA1 */ - if (!memcmp(psta->hwaddr, &skb->data[6], 6)) - continue; - - newskb = skb_copy(skb, GFP_ATOMIC); - - if (newskb) { - memcpy(newskb->data, psta->hwaddr, 6); - res = rtw_xmit(padapter, &newskb); - if (res < 0) { - pxmitpriv->tx_drop++; - dev_kfree_skb_any(newskb); - } else { - pxmitpriv->tx_pkts++; - } - } else { - pxmitpriv->tx_drop++; - - spin_unlock_bh(&pstapriv->asoc_list_lock); - return false; /* Caller shall tx this multicast frame via normal way. */ - } - } - - spin_unlock_bh(&pstapriv->asoc_list_lock); - dev_kfree_skb_any(skb); - return true; -} - -int rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) -{ - struct adapter *padapter = (struct adapter *)rtw_netdev_priv(pnetdev); - struct xmit_priv *pxmitpriv = &padapter->xmitpriv; - struct mlme_priv *pmlmepriv = &padapter->mlmepriv; - s32 res = 0; - - if (!rtw_if_up(padapter)) - goto drop_packet; - - rtw_check_xmit_resource(padapter, pkt); - - if (!rtw_mc2u_disable && check_fwstate(pmlmepriv, WIFI_AP_STATE) && - (IP_MCAST_MAC(pkt->data) || ICMPV6_MCAST_MAC(pkt->data)) && - (padapter->registrypriv.wifi_spec == 0)) { - if (pxmitpriv->free_xmitframe_cnt > (NR_XMITFRAME / 4)) { - res = rtw_mlcst2unicst(padapter, pkt); - if (res) - goto exit; - } - } - - res = rtw_xmit(padapter, &pkt); - if (res < 0) - goto drop_packet; - - pxmitpriv->tx_pkts++; - goto exit; - -drop_packet: - pxmitpriv->tx_drop++; - dev_kfree_skb_any(pkt); - -exit: - - return 0; -} diff --git a/drivers/staging/rtl8192e/Kconfig b/drivers/staging/rtl8192e/Kconfig index 39f5a6a7346a558cf6db85fb425dfefb02966c08..e06c189b4ce442caa523aec52a808dfd19bc9e96 100644 --- a/drivers/staging/rtl8192e/Kconfig +++ b/drivers/staging/rtl8192e/Kconfig @@ -10,6 +10,9 @@ config RTLLIB If unsure, say N. + This driver adds support for rtllib wireless cards. + Only the rtl8192e is supported as of now. + if RTLLIB config RTLLIB_CRYPTO_CCMP @@ -23,6 +26,8 @@ config RTLLIB_CRYPTO_CCMP CCMP crypto driver for rtllib. If you enabled RTLLIB, you want this. + Adds support for the CCM mode Protocol crypto driver for + use in wireless cards (including rtllib cards). config RTLLIB_CRYPTO_TKIP tristate "Support for rtllib TKIP crypto" @@ -35,6 +40,8 @@ config RTLLIB_CRYPTO_TKIP TKIP crypto driver for rtllib. If you enabled RTLLIB, you want this. + Adds support for the Temporal Key Integrity Protocol for + the IEEE 802.11i standard for use on wireless cards. config RTLLIB_CRYPTO_WEP tristate "Support for rtllib WEP crypto" @@ -42,9 +49,12 @@ config RTLLIB_CRYPTO_WEP depends on RTLLIB default y help - TKIP crypto driver for rtllib. + WEP crypto driver for rtllib. If you enabled RTLLIB, you want this. + Adds support for the (now weak) Wired Equivalent Privacy + (WEP) crypto protocol for wireless cards. + NOTE: This protocol is now considered insecure. source "drivers/staging/rtl8192e/rtl8192e/Kconfig" diff --git a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c index 4abec7b429939246c4c56bab23096dc09d41a914..ab2e9b7298830a341c128a1aaa3b46b4a579a478 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c +++ b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.c @@ -10,7 +10,7 @@ #include "r8190P_rtl8256.h" void rtl92e_set_bandwidth(struct net_device *dev, - enum ht_channel_width Bandwidth) + enum ht_channel_width bandwidth) { u8 eRFPath; struct r8192_priv *priv = rtllib_priv(dev); @@ -25,7 +25,7 @@ void rtl92e_set_bandwidth(struct net_device *dev, if (!rtl92e_is_legal_rf_path(dev, eRFPath)) continue; - switch (Bandwidth) { + switch (bandwidth) { case HT_CHANNEL_WIDTH_20: rtl92e_set_rf_reg(dev, (enum rf90_radio_path)eRFPath, 0x0b, bMask12Bits, 0x100); @@ -44,7 +44,7 @@ void rtl92e_set_bandwidth(struct net_device *dev, break; default: netdev_err(dev, "%s(): Unknown bandwidth: %#X\n", - __func__, Bandwidth); + __func__, bandwidth); break; } } @@ -115,10 +115,6 @@ bool rtl92e_config_rf(struct net_device *dev) (enum rf90_radio_path)eRFPath, RegOffSetToBeCheck, bMask12Bits); - RT_TRACE(COMP_RF, - "RF %d %d register final value: %x\n", - eRFPath, RegOffSetToBeCheck, - RF3_Final_Value); RetryTimes--; } @@ -142,8 +138,6 @@ bool rtl92e_config_rf(struct net_device *dev) goto fail; } } - - RT_TRACE(COMP_PHY, "PHY Initialization Success\n"); return true; fail: diff --git a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.h b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.h index 4cb483f1a152a0d65aa784cdb314a43ae5e99c22..3c52e2b4309599c7a3b5e8415854e3c67fae13de 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.h +++ b/drivers/staging/rtl8192e/rtl8192e/r8190P_rtl8256.h @@ -9,7 +9,7 @@ #define RTL819X_TOTAL_RF_PATH 2 void rtl92e_set_bandwidth(struct net_device *dev, - enum ht_channel_width Bandwidth); + enum ht_channel_width bandwidth); bool rtl92e_config_rf(struct net_device *dev); void rtl92e_set_cck_tx_power(struct net_device *dev, u8 powerlevel); void rtl92e_set_ofdm_tx_power(struct net_device *dev, u8 powerlevel); diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c index cd8bbc358d016fe2e9eca3447fcdfe6396f98257..8bf06f736ffb8a0414468a38445bdfc1b5770ef9 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c +++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_cmdpkt.c @@ -21,8 +21,6 @@ bool rtl92e_send_cmd_pkt(struct net_device *dev, u32 type, const void *data, struct tx_fwinfo_8190pci *pTxFwInfo = NULL; - RT_TRACE(COMP_CMDPKT, "%s(),buffer_len is %d\n", __func__, len); - do { if ((len - frag_offset) > CMDPACKET_FRAG_SIZE) { frag_length = CMDPACKET_FRAG_SIZE; @@ -61,8 +59,7 @@ bool rtl92e_send_cmd_pkt(struct net_device *dev, u32 type, const void *data, tcb_desc->txbuf_size = frag_length; } - seg_ptr = skb_put(skb, frag_length); - memcpy(seg_ptr, data, (u32)frag_length); + skb_put_data(skb, data, frag_length); if (type == DESC_PACKET_TYPE_INIT && (!priv->rtllib->check_nic_enough_desc(dev, TXCMD_QUEUE) || diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c index 4b9249195b5aac8a3c44e5cf791182f21de433a3..18e4e5d84878a4b89b17082dd453dfa4dc71685a 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c +++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_dev.c @@ -186,8 +186,6 @@ void rtl92e_set_reg(struct net_device *dev, u8 variable, u8 *val) AC_PARAM_ECW_MIN_OFFSET) | (((u32)u1bAIFS) << AC_PARAM_AIFS_OFFSET); - RT_TRACE(COMP_DBG, "%s():HW_VAR_AC_PARAM eACI:%x:%x\n", - __func__, eACI, u4bAcParam); switch (eACI) { case AC1_BK: rtl92e_writel(dev, EDCAPARA_BK, u4bAcParam); @@ -226,8 +224,6 @@ void rtl92e_set_reg(struct net_device *dev, u8 variable, u8 *val) u8 acm = pAciAifsn->f.acm; u8 AcmCtrl = rtl92e_readb(dev, AcmHwCtrl); - RT_TRACE(COMP_DBG, "===========>%s():HW_VAR_ACM_CTRL:%x\n", - __func__, eACI); AcmCtrl = AcmCtrl | ((priv->AcmMethod == 2) ? 0x0 : 0x1); if (acm) { @@ -243,12 +239,6 @@ void rtl92e_set_reg(struct net_device *dev, u8 variable, u8 *val) case AC3_VO: AcmCtrl |= AcmHw_VoqEn; break; - - default: - RT_TRACE(COMP_QOS, - "SetHwReg8185(): [HW_VAR_ACM_CTRL] acm set failed: eACI is %d\n", - eACI); - break; } } else { switch (eACI) { @@ -268,10 +258,6 @@ void rtl92e_set_reg(struct net_device *dev, u8 variable, u8 *val) break; } } - - RT_TRACE(COMP_QOS, - "SetHwReg8190pci(): [HW_VAR_ACM_CTRL] Write 0x%X\n", - AcmCtrl); rtl92e_writeb(dev, AcmHwCtrl, AcmCtrl); break; } @@ -304,8 +290,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) u16 i, usValue, IC_Version; u16 EEPROMId; - RT_TRACE(COMP_INIT, "====> %s\n", __func__); - EEPROMId = rtl92e_eeprom_read(dev, 0); if (EEPROMId != RTL8190_EEPROM_ID) { netdev_err(dev, "%s(): Invalid EEPROM ID: %x\n", __func__, @@ -329,8 +313,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) ICVer8192 = IC_Version & 0xf; ICVer8256 = (IC_Version & 0xf0)>>4; - RT_TRACE(COMP_INIT, "\nICVer8192 = 0x%x\n", ICVer8192); - RT_TRACE(COMP_INIT, "\nICVer8256 = 0x%x\n", ICVer8256); if (ICVer8192 == 0x2) { if (ICVer8256 == 0x5) priv->card_8192_version = VERSION_8190_BE; @@ -343,22 +325,14 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) priv->card_8192_version = VERSION_8190_BD; break; } - RT_TRACE(COMP_INIT, "\nIC Version = 0x%x\n", - priv->card_8192_version); } else { priv->card_8192_version = VERSION_8190_BD; priv->eeprom_vid = 0; priv->eeprom_did = 0; priv->eeprom_CustomerID = 0; priv->eeprom_ChannelPlan = 0; - RT_TRACE(COMP_INIT, "\nIC Version = 0x%x\n", 0xff); } - RT_TRACE(COMP_INIT, "EEPROM VID = 0x%4x\n", priv->eeprom_vid); - RT_TRACE(COMP_INIT, "EEPROM DID = 0x%4x\n", priv->eeprom_did); - RT_TRACE(COMP_INIT, "EEPROM Customer ID: 0x%2x\n", - priv->eeprom_CustomerID); - if (!priv->AutoloadFailFlag) { u8 addr[ETH_ALEN]; @@ -372,9 +346,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) eth_hw_addr_set(dev, bMac_Tmp_Addr); } - RT_TRACE(COMP_INIT, "Permanent Address = %pM\n", - dev->dev_addr); - if (priv->card_8192_version > VERSION_8190_BD) priv->bTXPowerDataReadFromEEPORM = true; else @@ -395,8 +366,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) } else { priv->EEPROMLegacyHTTxPowerDiff = 0x04; } - RT_TRACE(COMP_INIT, "EEPROMLegacyHTTxPowerDiff = %d\n", - priv->EEPROMLegacyHTTxPowerDiff); if (!priv->AutoloadFailFlag) priv->EEPROMThermalMeter = ((rtl92e_eeprom_read(dev, @@ -404,8 +373,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) 0xff00) >> 8; else priv->EEPROMThermalMeter = EEPROM_Default_ThermalMeter; - RT_TRACE(COMP_INIT, "ThermalMeter = %d\n", - priv->EEPROMThermalMeter); priv->TSSI_13dBm = priv->EEPROMThermalMeter * 100; if (priv->epromtype == EEPROM_93C46) { @@ -421,10 +388,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) priv->EEPROMCrystalCap = EEPROM_Default_TxPwDiff_CrystalCap; } - RT_TRACE(COMP_INIT, "EEPROMAntPwDiff = %d\n", - priv->EEPROMAntPwDiff); - RT_TRACE(COMP_INIT, "EEPROMCrystalCap = %d\n", - priv->EEPROMCrystalCap); for (i = 0; i < 14; i += 2) { if (!priv->AutoloadFailFlag) @@ -434,12 +397,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) usValue = EEPROM_Default_TxPower; *((u16 *)(&priv->EEPROMTxPowerLevelCCK[i])) = usValue; - RT_TRACE(COMP_INIT, - "CCK Tx Power Level, Index %d = 0x%02x\n", - i, priv->EEPROMTxPowerLevelCCK[i]); - RT_TRACE(COMP_INIT, - "CCK Tx Power Level, Index %d = 0x%02x\n", - i+1, priv->EEPROMTxPowerLevelCCK[i+1]); } for (i = 0; i < 14; i += 2) { if (!priv->AutoloadFailFlag) @@ -449,13 +406,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) usValue = EEPROM_Default_TxPower; *((u16 *)(&priv->EEPROMTxPowerLevelOFDM24G[i])) = usValue; - RT_TRACE(COMP_INIT, - "OFDM 2.4G Tx Power Level, Index %d = 0x%02x\n", - i, priv->EEPROMTxPowerLevelOFDM24G[i]); - RT_TRACE(COMP_INIT, - "OFDM 2.4G Tx Power Level, Index %d = 0x%02x\n", - i + 1, - priv->EEPROMTxPowerLevelOFDM24G[i+1]); } } if (priv->epromtype == EEPROM_93C46) { @@ -508,22 +458,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) priv->TxPowerLevelOFDM24G_C[i] = priv->EEPROMRfCOfdmChnlTxPwLevel[2]; } - for (i = 0; i < 14; i++) - RT_TRACE(COMP_INIT, - "priv->TxPowerLevelCCK_A[%d] = 0x%x\n", - i, priv->TxPowerLevelCCK_A[i]); - for (i = 0; i < 14; i++) - RT_TRACE(COMP_INIT, - "priv->TxPowerLevelOFDM24G_A[%d] = 0x%x\n", - i, priv->TxPowerLevelOFDM24G_A[i]); - for (i = 0; i < 14; i++) - RT_TRACE(COMP_INIT, - "priv->TxPowerLevelCCK_C[%d] = 0x%x\n", - i, priv->TxPowerLevelCCK_C[i]); - for (i = 0; i < 14; i++) - RT_TRACE(COMP_INIT, - "priv->TxPowerLevelOFDM24G_C[%d] = 0x%x\n", - i, priv->TxPowerLevelOFDM24G_C[i]); priv->LegacyHTTxPowerDiff = priv->EEPROMLegacyHTTxPowerDiff; priv->AntennaTxPwDiff[0] = 0; @@ -536,13 +470,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) } } - if (priv->rf_type == RF_1T2R) { - /* no matter what checkpatch says, the braces are needed */ - RT_TRACE(COMP_INIT, "\n1T2R config\n"); - } else if (priv->rf_type == RF_2T4R) { - RT_TRACE(COMP_INIT, "\n2T4R config\n"); - } - rtl92e_init_adaptive_rate(dev); priv->rf_chip = RF_8256; @@ -574,8 +501,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) priv->ChannelPlan = priv->eeprom_ChannelPlan&0x7f; else priv->ChannelPlan = 0x0; - RT_TRACE(COMP_INIT, "Toshiba ChannelPlan = 0x%x\n", - priv->ChannelPlan); break; case EEPROM_CID_Nettronix: priv->ScanDelay = 100; @@ -602,10 +527,6 @@ static void _rtl92e_read_eeprom_info(struct net_device *dev) priv->rtllib->bSupportRemoteWakeUp = true; else priv->rtllib->bSupportRemoteWakeUp = false; - - RT_TRACE(COMP_INIT, "RegChannelPlan(%d)\n", priv->RegChannelPlan); - RT_TRACE(COMP_INIT, "ChannelPlan = %d\n", priv->ChannelPlan); - RT_TRACE(COMP_TRACE, "<==== ReadAdapterInfo\n"); } void rtl92e_get_eeprom_size(struct net_device *dev) @@ -613,14 +534,9 @@ void rtl92e_get_eeprom_size(struct net_device *dev) u16 curCR; struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_INIT, "===========>%s()\n", __func__); curCR = rtl92e_readw(dev, EPROM_CMD); - RT_TRACE(COMP_INIT, "read from Reg Cmd9346CR(%x):%x\n", EPROM_CMD, - curCR); priv->epromtype = (curCR & EPROM_CMD_9356SEL) ? EEPROM_93C56 : EEPROM_93C46; - RT_TRACE(COMP_INIT, "<===========%s(), epromtype:%d\n", __func__, - priv->epromtype); _rtl92e_read_eeprom_info(dev); } @@ -697,7 +613,6 @@ bool rtl92e_start_adapter(struct net_device *dev) int i = 0; u32 retry_times = 0; - RT_TRACE(COMP_INIT, "====>%s()\n", __func__); priv->being_init_adapter = true; start: @@ -710,7 +625,7 @@ start: priv->pFirmware->status = FW_STATUS_0_INIT; if (priv->RegRfOff) - priv->rtllib->eRFPowerState = eRfOff; + priv->rtllib->rf_power_state = rf_off; ulRegRead = rtl92e_readl(dev, CPU_GEN); if (priv->pFirmware->status == FW_STATUS_0_INIT) @@ -732,13 +647,11 @@ start: rtl92e_writeb(dev, SWREGULATOR, 0xb8); } } - RT_TRACE(COMP_INIT, "BB Config Start!\n"); rtStatus = rtl92e_config_bb(dev); if (!rtStatus) { netdev_warn(dev, "%s(): Failed to configure BB\n", __func__); return rtStatus; } - RT_TRACE(COMP_INIT, "BB Config Finished!\n"); priv->LoopbackMode = RTL819X_NO_LOOPBACK; if (priv->ResetProgress == RESET_TYPE_NORESET) { @@ -818,19 +731,7 @@ start: tmpvalue = rtl92e_readb(dev, IC_VERRSION); priv->IC_Cut = tmpvalue; - RT_TRACE(COMP_INIT, "priv->IC_Cut= 0x%x\n", priv->IC_Cut); - if (priv->IC_Cut >= IC_VersionCut_D) { - if (priv->IC_Cut == IC_VersionCut_D) { - /* no matter what checkpatch says, braces are needed */ - RT_TRACE(COMP_INIT, "D-cut\n"); - } else if (priv->IC_Cut == IC_VersionCut_E) { - RT_TRACE(COMP_INIT, "E-cut\n"); - } - } else { - RT_TRACE(COMP_INIT, "Before C-cut\n"); - } - RT_TRACE(COMP_INIT, "Load Firmware!\n"); bfirmwareok = rtl92e_init_fw(dev); if (!bfirmwareok) { if (retry_times < 10) { @@ -841,15 +742,13 @@ start: goto end; } } - RT_TRACE(COMP_INIT, "Load Firmware finished!\n"); + if (priv->ResetProgress == RESET_TYPE_NORESET) { - RT_TRACE(COMP_INIT, "RF Config Started!\n"); rtStatus = rtl92e_config_phy(dev); if (!rtStatus) { netdev_info(dev, "RF Config failed\n"); return rtStatus; } - RT_TRACE(COMP_INIT, "RF Config Finished!\n"); } rtl92e_set_bb_reg(dev, rFPGA0_RFMOD, bCCKEn, 0x1); @@ -858,25 +757,14 @@ start: rtl92e_writeb(dev, 0x87, 0x0); if (priv->RegRfOff) { - RT_TRACE((COMP_INIT | COMP_RF | COMP_POWER), - "%s(): Turn off RF for RegRfOff ----------\n", - __func__); - rtl92e_set_rf_state(dev, eRfOff, RF_CHANGE_BY_SW); - } else if (priv->rtllib->RfOffReason > RF_CHANGE_BY_PS) { - RT_TRACE((COMP_INIT|COMP_RF|COMP_POWER), - "%s(): Turn off RF for RfOffReason(%d) ----------\n", - __func__, priv->rtllib->RfOffReason); - rtl92e_set_rf_state(dev, eRfOff, priv->rtllib->RfOffReason); - } else if (priv->rtllib->RfOffReason >= RF_CHANGE_BY_IPS) { - RT_TRACE((COMP_INIT|COMP_RF|COMP_POWER), - "%s(): Turn off RF for RfOffReason(%d) ----------\n", - __func__, priv->rtllib->RfOffReason); - rtl92e_set_rf_state(dev, eRfOff, priv->rtllib->RfOffReason); + rtl92e_set_rf_state(dev, rf_off, RF_CHANGE_BY_SW); + } else if (priv->rtllib->rf_off_reason > RF_CHANGE_BY_PS) { + rtl92e_set_rf_state(dev, rf_off, priv->rtllib->rf_off_reason); + } else if (priv->rtllib->rf_off_reason >= RF_CHANGE_BY_IPS) { + rtl92e_set_rf_state(dev, rf_off, priv->rtllib->rf_off_reason); } else { - RT_TRACE((COMP_INIT|COMP_RF|COMP_POWER), "%s(): RF-ON\n", - __func__); - priv->rtllib->eRFPowerState = eRfOn; - priv->rtllib->RfOffReason = 0; + priv->rtllib->rf_power_state = rf_on; + priv->rtllib->rf_off_reason = 0; } if (priv->rtllib->FwRWRF) @@ -915,18 +803,6 @@ start: priv->CCKPresentAttentuation_difference = 0; priv->CCKPresentAttentuation = priv->CCKPresentAttentuation_20Mdefault; - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex_initial = %d\n", - priv->rfa_txpowertrackingindex); - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex_real__initial = %d\n", - priv->rfa_txpowertrackingindex_real); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation_difference_initial = %d\n", - priv->CCKPresentAttentuation_difference); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation_initial = %d\n", - priv->CCKPresentAttentuation); priv->btxpower_tracking = false; } } @@ -946,7 +822,7 @@ static void _rtl92e_net_update(struct net_device *dev) net = &priv->rtllib->current_network; rtl92e_config_rate(dev, &rate_config); - priv->dot11CurrentPreambleMode = PREAMBLE_AUTO; + priv->dot11_current_preamble_mode = PREAMBLE_AUTO; priv->basic_rate = rate_config &= 0x15f; rtl92e_writew(dev, BSSIDR, *(u16 *)net->bssid); rtl92e_writel(dev, BSSIDR + 2, *(u32 *)(net->bssid + 2)); @@ -1237,7 +1113,6 @@ void rtl92e_fill_tx_desc(struct net_device *dev, struct tx_desc *pdesc, static u8 tmp; if (!tmp) { - RT_TRACE(COMP_DBG, "==>================hw sec\n"); tmp = 1; } switch (priv->rtllib->pairwise_key_type) { @@ -1350,12 +1225,6 @@ static u8 _rtl92e_rate_hw_to_mgn(bool bIsHT, u8 rate) case DESC90_RATE54M: ret_rate = MGN_54M; break; - - default: - RT_TRACE(COMP_RECV, - "%s: Non supportedRate [%x], bIsHT = %d!!!\n", - __func__, rate, bIsHT); - break; } } else { @@ -1411,12 +1280,6 @@ static u8 _rtl92e_rate_hw_to_mgn(bool bIsHT, u8 rate) case DESC90_RATEMCS32: ret_rate = 0x80 | 0x20; break; - - default: - RT_TRACE(COMP_RECV, - "%s: Non supported Rate [%x], bIsHT = %d!!!\n", - __func__, rate, bIsHT); - break; } } @@ -1721,9 +1584,6 @@ static void _rtl92e_process_phyinfo(struct r8192_priv *priv, u8 *buffer, for (rfpath = RF90_PATH_A; rfpath < RF90_PATH_C; rfpath++) { if (!rtl92e_is_legal_rf_path(priv->rtllib->dev, rfpath)) continue; - RT_TRACE(COMP_DBG, - "Jacken -> pPreviousstats->RxMIMOSignalStrength[rfpath] = %d\n", - prev_st->RxMIMOSignalStrength[rfpath]); if (priv->stats.rx_rssi_percentage[rfpath] == 0) { priv->stats.rx_rssi_percentage[rfpath] = prev_st->RxMIMOSignalStrength[rfpath]; @@ -1745,9 +1605,6 @@ static void _rtl92e_process_phyinfo(struct r8192_priv *priv, u8 *buffer, (prev_st->RxMIMOSignalStrength[rfpath])) / (RX_SMOOTH); } - RT_TRACE(COMP_DBG, - "Jacken -> priv->RxStats.RxRSSIPercentage[rfPath] = %d\n", - priv->stats.rx_rssi_percentage[rfpath]); } } @@ -1772,11 +1629,6 @@ static void _rtl92e_process_phyinfo(struct r8192_priv *priv, u8 *buffer, if (prev_st->RxPWDBAll >= 3) prev_st->RxPWDBAll -= 3; } - - RT_TRACE(COMP_RXDESC, "Smooth %s PWDB = %d\n", - prev_st->bIsCCK ? "CCK" : "OFDM", - prev_st->RxPWDBAll); - if (prev_st->bPacketToSelf || prev_st->bPacketBeacon || prev_st->bToSelfBA) { if (priv->undecorated_smoothed_pwdb < 0) @@ -2052,11 +1904,6 @@ bool rtl92e_get_rx_stats(struct net_device *dev, struct rtllib_rx_stats *stats, stats->RxIs40MHzPacket = pDrvInfo->BW; _rtl92e_translate_rx_signal_stats(dev, skb, stats, pdesc, pDrvInfo); - - if (pDrvInfo->FirstAGGR == 1 || pDrvInfo->PartAggr == 1) - RT_TRACE(COMP_RXDESC, - "pDrvInfo->FirstAGGR = %d, pDrvInfo->PartAggr = %d\n", - pDrvInfo->FirstAGGR, pDrvInfo->PartAggr); skb_trim(skb, skb->len - 4/*sCrcLng*/); @@ -2138,7 +1985,7 @@ void rtl92e_update_ratr_table(struct net_device *dev) break; case IEEE_N_24G: case IEEE_N_5G: - if (ieee->pHTInfo->PeerMimoPs == 0) { + if (ieee->pHTInfo->peer_mimo_ps == 0) { ratr_value &= 0x0007F007; } else { if (priv->rf_type == RF_1T2R) @@ -2151,10 +1998,10 @@ void rtl92e_update_ratr_table(struct net_device *dev) break; } ratr_value &= 0x0FFFFFFF; - if (ieee->pHTInfo->bCurTxBW40MHz && + if (ieee->pHTInfo->cur_tx_bw40mhz && ieee->pHTInfo->bCurShortGI40MHz) ratr_value |= 0x80000000; - else if (!ieee->pHTInfo->bCurTxBW40MHz && + else if (!ieee->pHTInfo->cur_tx_bw40mhz && ieee->pHTInfo->bCurShortGI20MHz) ratr_value |= 0x80000000; rtl92e_writel(dev, RATR0+rate_index*4, ratr_value); @@ -2261,9 +2108,6 @@ bool rtl92e_is_rx_stuck(struct net_device *dev) u8 i; u8 SilentResetRxSoltNum = 4; - RT_TRACE(COMP_RESET, "%s(): RegRxCounter is %d, RxCounter is %d\n", - __func__, RegRxCounter, priv->RxCounter); - rx_chk_cnt++; if (priv->undecorated_smoothed_pwdb >= (RateAdaptiveTH_High+5)) { rx_chk_cnt = 0; @@ -2321,9 +2165,6 @@ bool rtl92e_is_tx_stuck(struct net_device *dev) bool bStuck = false; u16 RegTxCounter = rtl92e_readw(dev, 0x128); - RT_TRACE(COMP_RESET, "%s():RegTxCounter is %d,TxCounter is %d\n", - __func__, RegTxCounter, priv->TxCounter); - if (priv->TxCounter == RegTxCounter) bStuck = true; diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c index 38110fa4f36de9a62b785fc41b3e5946604b0fcb..789d288d75037810926fbe06fe3f3fa7dcc8cb2a 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c +++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_firmware.c @@ -77,10 +77,6 @@ static bool _rtl92e_fw_check_ready(struct net_device *dev, rt_status = _rtl92e_wait_for_fw(dev, CPU_GEN_FIRM_RDY, 20); if (rt_status) pfirmware->status = FW_STATUS_5_READY; - else - RT_TRACE(COMP_FIRMWARE, - "_rtl92e_is_fw_ready fail(%d)!\n", - rt_status); break; default: rt_status = false; @@ -149,9 +145,6 @@ bool rtl92e_init_fw(struct net_device *dev) } else if (pfirmware->status == FW_STATUS_5_READY) { rst_opt = OPT_FIRMWARE_RESET; starting_state = FW_INIT_STEP2_DATA; - } else { - RT_TRACE(COMP_FIRMWARE, - "PlatformInitFirmware: undefined firmware state\n"); } for (i = starting_state; i <= FW_INIT_STEP2_DATA; i++) { diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c index f92551094738fc25f8f46e31c043cf7952e54f95..1b592258e64061c477ddbb5cf29c77b879b2b0f2 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c +++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.c @@ -117,8 +117,6 @@ static u32 _rtl92e_phy_rf_read(struct net_device *dev, } else NewOffset = Offset; } else { - RT_TRACE((COMP_PHY|COMP_ERR), - "check RF type here, need to be 8256\n"); NewOffset = Offset; } rtl92e_set_bb_reg(dev, pPhyReg->rfHSSIPara2, bLSSIReadAddress, @@ -173,8 +171,6 @@ static void _rtl92e_phy_rf_write(struct net_device *dev, } else NewOffset = Offset; } else { - RT_TRACE((COMP_PHY|COMP_ERR), - "check RF type here, need to be 8256\n"); NewOffset = Offset; } @@ -204,10 +200,9 @@ void rtl92e_set_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath, if (!rtl92e_is_legal_rf_path(dev, eRFPath)) return; - if (priv->rtllib->eRFPowerState != eRfOn && !priv->being_init_adapter) + if (priv->rtllib->rf_power_state != rf_on && !priv->being_init_adapter) return; - RT_TRACE(COMP_PHY, "FW RF CTRL is not ready now\n"); if (priv->Rf_Mode == RF_OP_By_FW) { if (BitMask != bMask12Bits) { Original_Value = _rtl92e_phy_rf_fw_read(dev, eRFPath, @@ -242,7 +237,7 @@ u32 rtl92e_get_rf_reg(struct net_device *dev, enum rf90_radio_path eRFPath, if (!rtl92e_is_legal_rf_path(dev, eRFPath)) return 0; - if (priv->rtllib->eRFPowerState != eRfOn && !priv->being_init_adapter) + if (priv->rtllib->rf_power_state != rf_on && !priv->being_init_adapter) return 0; mutex_lock(&priv->rf_mutex); if (priv->Rf_Mode == RF_OP_By_FW) { @@ -312,19 +307,14 @@ void rtl92e_config_mac(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); if (priv->bTXPowerDataReadFromEEPORM) { - RT_TRACE(COMP_PHY, "Rtl819XMACPHY_Array_PG\n"); dwArrayLen = MACPHY_Array_PGLength; pdwArray = Rtl819XMACPHY_Array_PG; } else { - RT_TRACE(COMP_PHY, "Read rtl819XMACPHY_Array\n"); dwArrayLen = MACPHY_ArrayLength; pdwArray = Rtl819XMACPHY_Array; } for (i = 0; i < dwArrayLen; i += 3) { - RT_TRACE(COMP_DBG, - "The Rtl8190MACPHY_Array[0] is %x Rtl8190MACPHY_Array[1] is %x Rtl8190MACPHY_Array[2] is %x\n", - pdwArray[i], pdwArray[i+1], pdwArray[i+2]); if (pdwArray[i] == 0x318) pdwArray[i+2] = 0x00000800; rtl92e_set_bb_reg(dev, pdwArray[i], pdwArray[i+1], @@ -357,20 +347,12 @@ static void _rtl92e_phy_config_bb(struct net_device *dev, u8 ConfigType) rtl92e_set_bb_reg(dev, Rtl819XPHY_REGArray_Table[i], bMaskDWord, Rtl819XPHY_REGArray_Table[i+1]); - RT_TRACE(COMP_DBG, - "i: %x, The Rtl819xUsbPHY_REGArray[0] is %x Rtl819xUsbPHY_REGArray[1] is %x\n", - i, Rtl819XPHY_REGArray_Table[i], - Rtl819XPHY_REGArray_Table[i+1]); } } else if (ConfigType == BaseBand_Config_AGC_TAB) { for (i = 0; i < AGCTAB_ArrayLen; i += 2) { rtl92e_set_bb_reg(dev, Rtl819XAGCTAB_Array_Table[i], bMaskDWord, Rtl819XAGCTAB_Array_Table[i+1]); - RT_TRACE(COMP_DBG, - "i:%x, The rtl819XAGCTAB_Array[0] is %x rtl819XAGCTAB_Array[1] is %x\n", - i, Rtl819XAGCTAB_Array_Table[i], - Rtl819XAGCTAB_Array_Table[i+1]); } } } @@ -478,8 +460,6 @@ bool rtl92e_check_bb_and_rf(struct net_device *dev, enum hw90_block CheckBlock, WriteAddr[HW90_BLOCK_PHY0] = 0x900; WriteAddr[HW90_BLOCK_PHY1] = 0x800; WriteAddr[HW90_BLOCK_RF] = 0x3; - RT_TRACE(COMP_PHY, "=======>%s(), CheckBlock:%d\n", __func__, - CheckBlock); if (CheckBlock == HW90_BLOCK_MAC) { netdev_warn(dev, "%s(): No checks available for MAC block.\n", @@ -543,9 +523,6 @@ static bool _rtl92e_bb_config_para_file(struct net_device *dev) (enum hw90_block)eCheckItem, (enum rf90_radio_path)0); if (!rtStatus) { - RT_TRACE((COMP_ERR | COMP_PHY), - "rtl92e_config_rf():Check PHY%d Fail!!\n", - eCheckItem-1); return rtStatus; } } @@ -602,15 +579,9 @@ void rtl92e_get_tx_power(struct net_device *dev) priv->DefaultInitialGain[1] = rtl92e_readb(dev, rOFDM0_XBAGCCore1); priv->DefaultInitialGain[2] = rtl92e_readb(dev, rOFDM0_XCAGCCore1); priv->DefaultInitialGain[3] = rtl92e_readb(dev, rOFDM0_XDAGCCore1); - RT_TRACE(COMP_INIT, - "Default initial gain (c50=0x%x, c58=0x%x, c60=0x%x, c68=0x%x)\n", - priv->DefaultInitialGain[0], priv->DefaultInitialGain[1], - priv->DefaultInitialGain[2], priv->DefaultInitialGain[3]); priv->framesync = rtl92e_readb(dev, rOFDM0_RxDetector3); priv->framesyncC34 = rtl92e_readl(dev, rOFDM0_RxDetector2); - RT_TRACE(COMP_INIT, "Default framesync (0x%x) = 0x%x\n", - rOFDM0_RxDetector3, priv->framesync); priv->SifsTime = rtl92e_readw(dev, SIFS); } @@ -813,9 +784,6 @@ static u8 _rtl92e_phy_switch_channel_step(struct net_device *dev, u8 channel, struct sw_chnl_cmd *CurrentCmd = NULL; u8 eRFPath; - RT_TRACE(COMP_TRACE, "====>%s()====stage:%d, step:%d, channel:%d\n", - __func__, *stage, *step, channel); - if (!rtllib_legal_channel(priv->rtllib, channel)) { netdev_err(dev, "Invalid channel requested: %d\n", channel); return true; @@ -976,21 +944,13 @@ static void _rtl92e_phy_switch_channel_work_item(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_TRACE, "==> SwChnlCallback819xUsbWorkItem()\n"); - - RT_TRACE(COMP_TRACE, "=====>--%s(), set chan:%d, priv:%p\n", __func__, - priv->chan, priv); - _rtl92e_phy_switch_channel(dev, priv->chan); - - RT_TRACE(COMP_TRACE, "<== SwChnlCallback819xUsbWorkItem()\n"); } u8 rtl92e_set_channel(struct net_device *dev, u8 channel) { struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_PHY, "=====>%s()\n", __func__); if (!priv->up) { netdev_err(dev, "%s(): Driver is not initialized\n", __func__); return false; @@ -1060,10 +1020,6 @@ static void _rtl92e_cck_tx_power_track_bw_switch_tssi(struct net_device *dev) if (priv->CCKPresentAttentuation < 0) priv->CCKPresentAttentuation = 0; - RT_TRACE(COMP_POWER_TRACKING, - "20M, priv->CCKPresentAttentuation = %d\n", - priv->CCKPresentAttentuation); - if (priv->rtllib->current_network.channel == 14 && !priv->bcck_in_ch14) { priv->bcck_in_ch14 = true; @@ -1082,9 +1038,6 @@ static void _rtl92e_cck_tx_power_track_bw_switch_tssi(struct net_device *dev) priv->CCKPresentAttentuation_40Mdefault + priv->CCKPresentAttentuation_difference; - RT_TRACE(COMP_POWER_TRACKING, - "40M, priv->CCKPresentAttentuation = %d\n", - priv->CCKPresentAttentuation); if (priv->CCKPresentAttentuation > (CCKTxBBGainTableLength - 1)) priv->CCKPresentAttentuation = @@ -1123,16 +1076,10 @@ static void _rtl92e_cck_tx_power_track_bw_switch_thermal(struct net_device *dev) if (priv->Record_CCK_20Mindex == 0) priv->Record_CCK_20Mindex = 6; priv->CCK_index = priv->Record_CCK_20Mindex; - RT_TRACE(COMP_POWER_TRACKING, - "20MHz, %s,CCK_index = %d\n", __func__, - priv->CCK_index); break; case HT_CHANNEL_WIDTH_20_40: priv->CCK_index = priv->Record_CCK_40Mindex; - RT_TRACE(COMP_POWER_TRACKING, - "40MHz, %s, CCK_index = %d\n", __func__, - priv->CCK_index); break; } rtl92e_dm_cck_txpower_adjust(dev, priv->bcck_in_ch14); @@ -1154,12 +1101,6 @@ static void _rtl92e_set_bw_mode_work_item(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); u8 regBwOpMode; - RT_TRACE(COMP_SWBW, - "==>%s Switch to %s bandwidth\n", __func__, - priv->CurrentChannelBW == HT_CHANNEL_WIDTH_20 ? - "20MHz" : "40MHz"); - - if (priv->rf_chip == RF_PSEUDO_11N) { priv->SetBWModeInProgress = false; return; @@ -1251,11 +1192,9 @@ static void _rtl92e_set_bw_mode_work_item(struct net_device *dev) atomic_dec(&(priv->rtllib->atm_swbw)); priv->SetBWModeInProgress = false; - - RT_TRACE(COMP_SWBW, "<==SetBWMode819xUsb()"); } -void rtl92e_set_bw_mode(struct net_device *dev, enum ht_channel_width Bandwidth, +void rtl92e_set_bw_mode(struct net_device *dev, enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset) { struct r8192_priv *priv = rtllib_priv(dev); @@ -1267,7 +1206,7 @@ void rtl92e_set_bw_mode(struct net_device *dev, enum ht_channel_width Bandwidth, atomic_inc(&(priv->rtllib->atm_swbw)); priv->SetBWModeInProgress = true; - priv->CurrentChannelBW = Bandwidth; + priv->CurrentChannelBW = bandwidth; if (Offset == HT_EXTCHNL_OFFSET_LOWER) priv->nCur40MhzPrimeSC = HAL_PRIME_CHNL_OFFSET_UPPER; @@ -1291,8 +1230,6 @@ void rtl92e_init_gain(struct net_device *dev, u8 Operation) if (priv->up) { switch (Operation) { case IG_Backup: - RT_TRACE(COMP_SCAN, - "IG_Backup, backup the initial gain.\n"); initial_gain = SCAN_RX_INITIAL_GAIN; BitMask = bMaskByte0; if (dm_digtable.dig_algorithm == @@ -1314,35 +1251,13 @@ void rtl92e_init_gain(struct net_device *dev, u8 Operation) priv->initgain_backup.cca = (u8)rtl92e_get_bb_reg(dev, rCCK0_CCA, BitMask); - RT_TRACE(COMP_SCAN, - "Scan InitialGainBackup 0xc50 is %x\n", - priv->initgain_backup.xaagccore1); - RT_TRACE(COMP_SCAN, - "Scan InitialGainBackup 0xc58 is %x\n", - priv->initgain_backup.xbagccore1); - RT_TRACE(COMP_SCAN, - "Scan InitialGainBackup 0xc60 is %x\n", - priv->initgain_backup.xcagccore1); - RT_TRACE(COMP_SCAN, - "Scan InitialGainBackup 0xc68 is %x\n", - priv->initgain_backup.xdagccore1); - RT_TRACE(COMP_SCAN, - "Scan InitialGainBackup 0xa0a is %x\n", - priv->initgain_backup.cca); - - RT_TRACE(COMP_SCAN, "Write scan initial gain = 0x%x\n", - initial_gain); rtl92e_writeb(dev, rOFDM0_XAAGCCore1, initial_gain); rtl92e_writeb(dev, rOFDM0_XBAGCCore1, initial_gain); rtl92e_writeb(dev, rOFDM0_XCAGCCore1, initial_gain); rtl92e_writeb(dev, rOFDM0_XDAGCCore1, initial_gain); - RT_TRACE(COMP_SCAN, "Write scan 0xa0a = 0x%x\n", - POWER_DETECTION_TH); rtl92e_writeb(dev, 0xa0a, POWER_DETECTION_TH); break; case IG_Restore: - RT_TRACE(COMP_SCAN, - "IG_Restore, restore the initial gain.\n"); BitMask = 0x7f; if (dm_digtable.dig_algorithm == DIG_ALGO_BY_FALSE_ALARM) @@ -1360,22 +1275,6 @@ void rtl92e_init_gain(struct net_device *dev, u8 Operation) rtl92e_set_bb_reg(dev, rCCK0_CCA, BitMask, (u32)priv->initgain_backup.cca); - RT_TRACE(COMP_SCAN, - "Scan BBInitialGainRestore 0xc50 is %x\n", - priv->initgain_backup.xaagccore1); - RT_TRACE(COMP_SCAN, - "Scan BBInitialGainRestore 0xc58 is %x\n", - priv->initgain_backup.xbagccore1); - RT_TRACE(COMP_SCAN, - "Scan BBInitialGainRestore 0xc60 is %x\n", - priv->initgain_backup.xcagccore1); - RT_TRACE(COMP_SCAN, - "Scan BBInitialGainRestore 0xc68 is %x\n", - priv->initgain_backup.xdagccore1); - RT_TRACE(COMP_SCAN, - "Scan BBInitialGainRestore 0xa0a is %x\n", - priv->initgain_backup.cca); - rtl92e_set_tx_power(dev, priv->rtllib->current_network.channel); @@ -1383,9 +1282,6 @@ void rtl92e_init_gain(struct net_device *dev, u8 Operation) DIG_ALGO_BY_FALSE_ALARM) rtl92e_set_bb_reg(dev, UFWP, bMaskByte1, 0x1); break; - default: - RT_TRACE(COMP_SCAN, "Unknown IG Operation.\n"); - break; } } } @@ -1405,7 +1301,7 @@ void rtl92e_set_rf_off(struct net_device *dev) } static bool _rtl92e_set_rf_power_state(struct net_device *dev, - enum rt_rf_power_state eRFPowerState) + enum rt_rf_power_state rf_power_state) { struct r8192_priv *priv = rtllib_priv(dev); struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) @@ -1416,15 +1312,13 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, if (priv->SetRFPowerStateInProgress) return false; - RT_TRACE(COMP_PS, "===========> %s!\n", __func__); priv->SetRFPowerStateInProgress = true; switch (priv->rf_chip) { case RF_8256: - switch (eRFPowerState) { - case eRfOn: - RT_TRACE(COMP_PS, "%s eRfOn!\n", __func__); - if ((priv->rtllib->eRFPowerState == eRfOff) && + switch (rf_power_state) { + case rf_on: + if ((priv->rtllib->rf_power_state == rf_off) && RT_IN_PS_LEVEL(pPSC, RT_RF_OFF_LEVL_HALT_NIC)) { bool rtstatus; u32 InitilizeCount = 3; @@ -1469,8 +1363,8 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, break; - case eRfSleep: - if (priv->rtllib->eRFPowerState == eRfOff) + case rf_sleep: + if (priv->rtllib->rf_power_state == rf_off) break; @@ -1481,25 +1375,18 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, QueueID++; continue; } else { - RT_TRACE((COMP_POWER|COMP_RF), - "eRf Off/Sleep: %d times TcbBusyQueue[%d] !=0 before doze!\n", - (i+1), QueueID); udelay(10); i++; } if (i >= MAX_DOZE_WAITING_TIMES_9x) { - RT_TRACE(COMP_POWER, "\n\n\n TimeOut!! %s: eRfOff: %d times TcbBusyQueue[%d] != 0 !!!\n", - __func__, MAX_DOZE_WAITING_TIMES_9x, QueueID); break; } } rtl92e_set_rf_off(dev); break; - case eRfOff: - RT_TRACE(COMP_PS, "%s eRfOff/Sleep !\n", __func__); - + case rf_off: for (QueueID = 0, i = 0; QueueID < MAX_TX_QUEUE; ) { ring = &priv->tx_ring[QueueID]; @@ -1507,18 +1394,11 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, QueueID++; continue; } else { - RT_TRACE(COMP_POWER, - "eRf Off/Sleep: %d times TcbBusyQueue[%d] !=0 before doze!\n", - (i+1), QueueID); udelay(10); i++; } if (i >= MAX_DOZE_WAITING_TIMES_9x) { - RT_TRACE(COMP_POWER, - "\n\n\n SetZebra: RFPowerState8185B(): eRfOff: %d times TcbBusyQueue[%d] != 0 !!!\n", - MAX_DOZE_WAITING_TIMES_9x, - QueueID); break; } } @@ -1538,7 +1418,7 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, bResult = false; netdev_warn(dev, "%s(): Unknown state requested: 0x%X.\n", - __func__, eRFPowerState); + __func__, rf_power_state); break; } @@ -1550,7 +1430,7 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, } if (bResult) { - priv->rtllib->eRFPowerState = eRFPowerState; + priv->rtllib->rf_power_state = rf_power_state; switch (priv->rf_chip) { case RF_8256: @@ -1563,30 +1443,22 @@ static bool _rtl92e_set_rf_power_state(struct net_device *dev, } priv->SetRFPowerStateInProgress = false; - RT_TRACE(COMP_PS, "<=========== %s bResult = %d!\n", __func__, bResult); return bResult; } bool rtl92e_set_rf_power_state(struct net_device *dev, - enum rt_rf_power_state eRFPowerState) + enum rt_rf_power_state rf_power_state) { struct r8192_priv *priv = rtllib_priv(dev); bool bResult = false; - RT_TRACE(COMP_PS, - "---------> %s: eRFPowerState(%d)\n", __func__, eRFPowerState); - if (eRFPowerState == priv->rtllib->eRFPowerState && + if (rf_power_state == priv->rtllib->rf_power_state && priv->bHwRfOffAction == 0) { - RT_TRACE(COMP_PS, "<--------- %s: discard the request for eRFPowerState(%d) is the same.\n", - __func__, eRFPowerState); return bResult; } - bResult = _rtl92e_set_rf_power_state(dev, eRFPowerState); - - RT_TRACE(COMP_PS, "<--------- %s: bResult(%d)\n", __func__, bResult); - + bResult = _rtl92e_set_rf_power_state(dev, rf_power_state); return bResult; } @@ -1603,10 +1475,6 @@ void rtl92e_scan_op_backup(struct net_device *dev, u8 Operation) case SCAN_OPT_RESTORE: priv->rtllib->InitialGainHandler(dev, IG_Restore); break; - - default: - RT_TRACE(COMP_SCAN, "Unknown Scan Backup Operation.\n"); - break; } } } diff --git a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h index 7c9148e033d8c766aac976a7ec65e141d732842c..75629f5df954d2ee20ebbee50c1d8ad0f0f77aaf 100644 --- a/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h +++ b/drivers/staging/rtl8192e/rtl8192e/r8192E_phy.h @@ -75,15 +75,14 @@ u8 rtl92e_config_rf_path(struct net_device *dev, enum rf90_radio_path eRFPath); u8 rtl92e_set_channel(struct net_device *dev, u8 channel); void rtl92e_set_bw_mode(struct net_device *dev, - enum ht_channel_width Bandwidth, + enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset); void rtl92e_init_gain(struct net_device *dev, u8 Operation); void rtl92e_set_rf_off(struct net_device *dev); bool rtl92e_set_rf_power_state(struct net_device *dev, - enum rt_rf_power_state eRFPowerState); -#define PHY_SetRFPowerState rtl92e_set_rf_power_state + enum rt_rf_power_state rf_power_state); void rtl92e_scan_op_backup(struct net_device *dev, u8 Operation); diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c b/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c index d7630f02a9107d49674bc1226b177fc9faafef22..41faeb4b9b9b434989d6c2badbf090c9045a0954 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_cam.c @@ -42,14 +42,10 @@ void rtl92e_enable_hw_security_config(struct net_device *dev) ieee->hwsec_active = 1; - if ((ieee->pHTInfo->IOTAction&HT_IOT_ACT_PURE_N_MODE) || !hwwep) { + if ((ieee->pHTInfo->iot_action & HT_IOT_ACT_PURE_N_MODE) || !hwwep) { ieee->hwsec_active = 0; SECR_value &= ~SCR_RxDecEnable; } - - RT_TRACE(COMP_SEC, "%s:, hwsec:%d, pairwise_key:%d, SECR_value:%x\n", - __func__, ieee->hwsec_active, ieee->pairwise_key_type, - SECR_value); rtl92e_writeb(dev, SECR, SECR_value); } @@ -60,10 +56,6 @@ void rtl92e_set_swcam(struct net_device *dev, u8 EntryNo, u8 KeyIndex, struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = priv->rtllib; - RT_TRACE(COMP_DBG, - "===========>%s():EntryNo is %d,KeyIndex is %d,KeyType is %d,is_mesh is %d\n", - __func__, EntryNo, KeyIndex, KeyType, is_mesh); - if (EntryNo >= TOTAL_CAM_ENTRY) return; @@ -86,12 +78,12 @@ void rtl92e_set_key(struct net_device *dev, u8 EntryNo, u8 KeyIndex, u16 usConfig = 0; u8 i; struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev); - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; - rtState = priv->rtllib->eRFPowerState; + rt_state = priv->rtllib->rf_power_state; if (priv->rtllib->PowerSaveControl.bInactivePs) { - if (rtState == eRfOff) { - if (priv->rtllib->RfOffReason > RF_CHANGE_BY_IPS) { + if (rt_state == rf_off) { + if (priv->rtllib->rf_off_reason > RF_CHANGE_BY_IPS) { netdev_warn(dev, "%s(): RF is OFF.\n", __func__); return; @@ -107,10 +99,6 @@ void rtl92e_set_key(struct net_device *dev, u8 EntryNo, u8 KeyIndex, return; } - RT_TRACE(COMP_SEC, - "====>to %s, dev:%p, EntryNo:%d, KeyIndex:%d,KeyType:%d, MacAddr %pM\n", - __func__, dev, EntryNo, KeyIndex, KeyType, MacAddr); - if (DefaultKey) usConfig |= BIT15 | (KeyType<<2); else @@ -144,7 +132,6 @@ void rtl92e_set_key(struct net_device *dev, u8 EntryNo, u8 KeyIndex, } } } - RT_TRACE(COMP_SEC, "=========>after set key, usconfig:%x\n", usConfig); } void rtl92e_cam_restore(struct net_device *dev) @@ -163,9 +150,6 @@ void rtl92e_cam_restore(struct net_device *dev) 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - RT_TRACE(COMP_SEC, "%s:\n", __func__); - - if ((priv->rtllib->pairwise_key_type == KEY_TYPE_WEP40) || (priv->rtllib->pairwise_key_type == KEY_TYPE_WEP104)) { diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c index b9ce71848023a3e3fa27d6cfedbe28f3dfe954da..89bc989cffbae1e14e11bf471df90bf0a39c4d3e 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.c @@ -25,7 +25,6 @@ int hwwep = 1; static char *ifname = "wlan%d"; - static const struct rtl819x_ops rtl819xp_ops = { .nic_type = NIC_8192E, .get_eeprom_size = rtl92e_get_eeprom_size, @@ -44,8 +43,8 @@ static const struct rtl819x_ops rtl819xp_ops = { .rx_enable = rtl92e_enable_rx, .tx_enable = rtl92e_enable_tx, .interrupt_recognized = rtl92e_ack_irq, - .TxCheckStuckHandler = rtl92e_is_tx_stuck, - .RxCheckStuckHandler = rtl92e_is_rx_stuck, + .tx_check_stuck_handler = rtl92e_is_tx_stuck, + .rx_check_stuck_handler = rtl92e_is_rx_stuck, }; static struct pci_device_id rtl8192_pci_id_tbl[] = { @@ -133,36 +132,27 @@ void rtl92e_writew(struct net_device *dev, int x, u16 y) * -----------------------------GENERAL FUNCTION------------------------- ****************************************************************************/ bool rtl92e_set_rf_state(struct net_device *dev, - enum rt_rf_power_state StateToSet, - RT_RF_CHANGE_SOURCE ChangeSource) + enum rt_rf_power_state state_to_set, + RT_RF_CHANGE_SOURCE change_source) { struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = priv->rtllib; - bool bActionAllowed = false; - bool bConnectBySSID = false; - enum rt_rf_power_state rtState; - u16 RFWaitCounter = 0; + bool action_allowed = false; + bool connect_by_ssid = false; + enum rt_rf_power_state rt_state; + u16 rf_wait_counter = 0; unsigned long flag; - RT_TRACE((COMP_PS | COMP_RF), - "===>%s: StateToSet(%d)\n", __func__, StateToSet); - while (true) { spin_lock_irqsave(&priv->rf_ps_lock, flag); - if (priv->RFChangeInProgress) { + if (priv->rf_change_in_progress) { spin_unlock_irqrestore(&priv->rf_ps_lock, flag); - RT_TRACE((COMP_PS | COMP_RF), - "%s: RF Change in progress! Wait to set..StateToSet(%d).\n", - __func__, StateToSet); - - while (priv->RFChangeInProgress) { - RFWaitCounter++; - RT_TRACE((COMP_PS | COMP_RF), - "%s: Wait 1 ms (%d times)...\n", - __func__, RFWaitCounter); + + while (priv->rf_change_in_progress) { + rf_wait_counter++; mdelay(1); - if (RFWaitCounter > 100) { + if (rf_wait_counter > 100) { netdev_warn(dev, "%s(): Timeout waiting for RF change.\n", __func__); @@ -170,43 +160,37 @@ bool rtl92e_set_rf_state(struct net_device *dev, } } } else { - priv->RFChangeInProgress = true; + priv->rf_change_in_progress = true; spin_unlock_irqrestore(&priv->rf_ps_lock, flag); break; } } - rtState = priv->rtllib->eRFPowerState; + rt_state = priv->rtllib->rf_power_state; - switch (StateToSet) { - case eRfOn: - priv->rtllib->RfOffReason &= (~ChangeSource); + switch (state_to_set) { + case rf_on: + priv->rtllib->rf_off_reason &= (~change_source); - if ((ChangeSource == RF_CHANGE_BY_HW) && priv->bHwRadioOff) - priv->bHwRadioOff = false; + if ((change_source == RF_CHANGE_BY_HW) && priv->hw_radio_off) + priv->hw_radio_off = false; - if (!priv->rtllib->RfOffReason) { - priv->rtllib->RfOffReason = 0; - bActionAllowed = true; - - - if (rtState == eRfOff && - ChangeSource >= RF_CHANGE_BY_HW) - bConnectBySSID = true; - } else { - RT_TRACE((COMP_PS | COMP_RF), - "%s - eRfon reject pMgntInfo->RfOffReason= 0x%x, ChangeSource=0x%X\n", - __func__, priv->rtllib->RfOffReason, ChangeSource); - } + if (!priv->rtllib->rf_off_reason) { + priv->rtllib->rf_off_reason = 0; + action_allowed = true; + if (rt_state == rf_off && + change_source >= RF_CHANGE_BY_HW) + connect_by_ssid = true; + } break; - case eRfOff: + case rf_off: if ((priv->rtllib->iw_mode == IW_MODE_INFRA) || (priv->rtllib->iw_mode == IW_MODE_ADHOC)) { - if ((priv->rtllib->RfOffReason > RF_CHANGE_BY_IPS) || - (ChangeSource > RF_CHANGE_BY_IPS)) { + if ((priv->rtllib->rf_off_reason > RF_CHANGE_BY_IPS) || + (change_source > RF_CHANGE_BY_IPS)) { if (ieee->state == RTLLIB_LINKED) priv->blinked_ingpio = true; else @@ -215,46 +199,36 @@ bool rtl92e_set_rf_state(struct net_device *dev, WLAN_REASON_DISASSOC_STA_HAS_LEFT); } } - if ((ChangeSource == RF_CHANGE_BY_HW) && !priv->bHwRadioOff) - priv->bHwRadioOff = true; - priv->rtllib->RfOffReason |= ChangeSource; - bActionAllowed = true; + if ((change_source == RF_CHANGE_BY_HW) && !priv->hw_radio_off) + priv->hw_radio_off = true; + priv->rtllib->rf_off_reason |= change_source; + action_allowed = true; break; - case eRfSleep: - priv->rtllib->RfOffReason |= ChangeSource; - bActionAllowed = true; + case rf_sleep: + priv->rtllib->rf_off_reason |= change_source; + action_allowed = true; break; default: break; } - if (bActionAllowed) { - RT_TRACE((COMP_PS | COMP_RF), - "%s: Action is allowed.... StateToSet(%d), RfOffReason(%#X)\n", - __func__, StateToSet, priv->rtllib->RfOffReason); - PHY_SetRFPowerState(dev, StateToSet); - if (StateToSet == eRfOn) { - - if (bConnectBySSID && priv->blinked_ingpio) { + if (action_allowed) { + rtl92e_set_rf_power_state(dev, state_to_set); + if (state_to_set == rf_on) { + if (connect_by_ssid && priv->blinked_ingpio) { schedule_delayed_work( &ieee->associate_procedure_wq, 0); priv->blinked_ingpio = false; } } - } else { - RT_TRACE((COMP_PS | COMP_RF), - "%s: Action is rejected.... StateToSet(%d), ChangeSource(%#X), RfOffReason(%#X)\n", - __func__, StateToSet, ChangeSource, priv->rtllib->RfOffReason); } spin_lock_irqsave(&priv->rf_ps_lock, flag); - priv->RFChangeInProgress = false; + priv->rf_change_in_progress = false; spin_unlock_irqrestore(&priv->rf_ps_lock, flag); - - RT_TRACE((COMP_PS | COMP_RF), "<===%s\n", __func__); - return bActionAllowed; + return action_allowed; } static short _rtl92e_check_nic_enough_desc(struct net_device *dev, int prio) @@ -297,7 +271,6 @@ static void _rtl92e_set_chan(struct net_device *dev, short ch) { struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_CH, "=====>%s()====ch:%d\n", __func__, ch); if (priv->chan_forced) return; @@ -314,22 +287,16 @@ static void _rtl92e_update_cap(struct net_device *dev, u16 cap) bool ShortPreamble; if (cap & WLAN_CAPABILITY_SHORT_PREAMBLE) { - if (priv->dot11CurrentPreambleMode != PREAMBLE_SHORT) { + if (priv->dot11_current_preamble_mode != PREAMBLE_SHORT) { ShortPreamble = true; - priv->dot11CurrentPreambleMode = PREAMBLE_SHORT; - RT_TRACE(COMP_DBG, - "%s(): WLAN_CAPABILITY_SHORT_PREAMBLE\n", - __func__); + priv->dot11_current_preamble_mode = PREAMBLE_SHORT; priv->rtllib->SetHwRegHandler(dev, HW_VAR_ACK_PREAMBLE, (unsigned char *)&ShortPreamble); } } else { - if (priv->dot11CurrentPreambleMode != PREAMBLE_LONG) { + if (priv->dot11_current_preamble_mode != PREAMBLE_LONG) { ShortPreamble = false; - priv->dot11CurrentPreambleMode = PREAMBLE_LONG; - RT_TRACE(COMP_DBG, - "%s(): WLAN_CAPABILITY_LONG_PREAMBLE\n", - __func__); + priv->dot11_current_preamble_mode = PREAMBLE_LONG; priv->rtllib->SetHwRegHandler(dev, HW_VAR_ACK_PREAMBLE, (unsigned char *)&ShortPreamble); } @@ -337,17 +304,17 @@ static void _rtl92e_update_cap(struct net_device *dev, u16 cap) if (net->mode & (IEEE_G | IEEE_N_24G)) { u8 slot_time_val; - u8 CurSlotTime = priv->slot_time; + u8 cur_slot_time = priv->slot_time; if ((cap & WLAN_CAPABILITY_SHORT_SLOT_TIME) && - (!priv->rtllib->pHTInfo->bCurrentRT2RTLongSlotTime)) { - if (CurSlotTime != SHORT_SLOT_TIME) { + (!priv->rtllib->pHTInfo->current_rt2rt_long_slot_time)) { + if (cur_slot_time != SHORT_SLOT_TIME) { slot_time_val = SHORT_SLOT_TIME; priv->rtllib->SetHwRegHandler(dev, HW_VAR_SLOT_TIME, &slot_time_val); } } else { - if (CurSlotTime != NON_SHORT_SLOT_TIME) { + if (cur_slot_time != NON_SHORT_SLOT_TIME) { slot_time_val = NON_SHORT_SLOT_TIME; priv->rtllib->SetHwRegHandler(dev, HW_VAR_SLOT_TIME, &slot_time_val); @@ -374,7 +341,7 @@ static void _rtl92e_update_beacon(void *data) if (ieee->pHTInfo->bCurrentHTSupport) HT_update_self_and_peer_setting(ieee, net); - ieee->pHTInfo->bCurrentRT2RTLongSlotTime = net->bssht.bd_rt2rt_long_slot_time; + ieee->pHTInfo->current_rt2rt_long_slot_time = net->bssht.bd_rt2rt_long_slot_time; ieee->pHTInfo->RT2RT_HT_Mode = net->bssht.rt2rt_ht_mode; _rtl92e_update_cap(dev, net->capability); } @@ -389,13 +356,10 @@ static void _rtl92e_qos_activate(void *data) mutex_lock(&priv->mutex); if (priv->rtllib->state != RTLLIB_LINKED) goto success; - RT_TRACE(COMP_QOS, - "qos active process with associate response received\n"); for (i = 0; i < QOS_QUEUE_NUM; i++) priv->rtllib->SetHwRegHandler(dev, HW_VAR_AC_PARAM, (u8 *)(&i)); - success: mutex_unlock(&priv->mutex); } @@ -426,18 +390,14 @@ static int _rtl92e_qos_handle_probe_response(struct r8192_priv *priv, network->qos_data.param_count; priv->rtllib->wmm_acm = network->qos_data.wmm_acm; schedule_work(&priv->qos_activate); - RT_TRACE(COMP_QOS, - "QoS parameters change call qos_activate\n"); } } else { memcpy(&priv->rtllib->current_network.qos_data.parameters, &def_qos_parameters, size); - if ((network->qos_data.active == 1) && (active_network == 1)) { + if ((network->qos_data.active == 1) && (active_network == 1)) schedule_work(&priv->qos_activate); - RT_TRACE(COMP_QOS, - "QoS was disabled call qos_activate\n"); - } + network->qos_data.active = 0; network->qos_data.supported = 0; } @@ -455,7 +415,6 @@ static int _rtl92e_handle_beacon(struct net_device *dev, schedule_delayed_work(&priv->update_beacon_wq, 0); return 0; - } static int _rtl92e_qos_assoc_resp(struct r8192_priv *priv, @@ -496,8 +455,6 @@ static int _rtl92e_qos_assoc_resp(struct r8192_priv *priv, spin_unlock_irqrestore(&priv->rtllib->lock, flags); - RT_TRACE(COMP_QOS, "%s: network->flags = %d,%d\n", __func__, - network->flags, priv->rtllib->current_network.qos_data.active); if (set_qos_param == 1) { rtl92e_dm_init_edca_turbo(priv->rtllib->dev); schedule_work(&priv->qos_activate); @@ -716,15 +673,9 @@ void rtl92e_set_wireless_mode(struct net_device *dev, u8 wireless_mode) if ((wireless_mode == WIRELESS_MODE_N_24G) || (wireless_mode == WIRELESS_MODE_N_5G)) { priv->rtllib->pHTInfo->bEnableHT = 1; - RT_TRACE(COMP_DBG, "%s(), wireless_mode:%x, bEnableHT = 1\n", - __func__, wireless_mode); } else { priv->rtllib->pHTInfo->bEnableHT = 0; - RT_TRACE(COMP_DBG, "%s(), wireless_mode:%x, bEnableHT = 0\n", - __func__, wireless_mode); } - - RT_TRACE(COMP_INIT, "Current Wireless Mode is %x\n", wireless_mode); _rtl92e_refresh_support_rate(priv); } @@ -742,7 +693,6 @@ static int _rtl92e_sta_up(struct net_device *dev, bool is_silent_reset) priv->rtllib->ieee_up = 1; priv->up_first_time = 0; - RT_TRACE(COMP_INIT, "Bringing up iface"); priv->bfirst_init = true; init_status = priv->ops->initialize_adapter(dev); if (!init_status) { @@ -751,7 +701,6 @@ static int _rtl92e_sta_up(struct net_device *dev, bool is_silent_reset) return -1; } - RT_TRACE(COMP_INIT, "start adapter finished\n"); RT_CLEAR_PS_LEVEL(pPSC, RT_RF_OFF_LEVL_HALT_NIC); priv->bfirst_init = false; @@ -790,7 +739,6 @@ static int _rtl92e_sta_down(struct net_device *dev, bool shutdownrf) priv->up = 0; priv->rtllib->ieee_up = 0; priv->bfirst_after_down = true; - RT_TRACE(COMP_DOWN, "==========>%s()\n", __func__); if (!netif_queue_stopped(dev)) netif_stop_queue(dev); @@ -807,29 +755,25 @@ static int _rtl92e_sta_down(struct net_device *dev, bool shutdownrf) rtllib_softmac_stop_protocol(priv->rtllib, 0, true); spin_lock_irqsave(&priv->rf_ps_lock, flags); - while (priv->RFChangeInProgress) { + while (priv->rf_change_in_progress) { spin_unlock_irqrestore(&priv->rf_ps_lock, flags); if (RFInProgressTimeOut > 100) { spin_lock_irqsave(&priv->rf_ps_lock, flags); break; } - RT_TRACE(COMP_DBG, - "===>%s():RF is in progress, need to wait until rf change is done.\n", - __func__); mdelay(1); RFInProgressTimeOut++; spin_lock_irqsave(&priv->rf_ps_lock, flags); } - priv->RFChangeInProgress = true; + priv->rf_change_in_progress = true; spin_unlock_irqrestore(&priv->rf_ps_lock, flags); priv->ops->stop_adapter(dev, false); spin_lock_irqsave(&priv->rf_ps_lock, flags); - priv->RFChangeInProgress = false; + priv->rf_change_in_progress = false; spin_unlock_irqrestore(&priv->rf_ps_lock, flags); udelay(100); memset(&priv->rtllib->current_network, 0, offsetof(struct rtllib_network, list)); - RT_TRACE(COMP_DOWN, "<==========%s()\n", __func__); return 0; } @@ -883,14 +827,13 @@ static void _rtl92e_init_priv_constant(struct net_device *dev) pPSC->RegMaxLPSAwakeIntvl = 5; } - static void _rtl92e_init_priv_variable(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); u8 i; priv->AcmMethod = eAcmWay2_SW; - priv->dot11CurrentPreambleMode = PREAMBLE_AUTO; + priv->dot11_current_preamble_mode = PREAMBLE_AUTO; priv->rtllib->status = 0; priv->polling_timer_on = 0; priv->up_first_time = 1; @@ -935,12 +878,12 @@ static void _rtl92e_init_priv_variable(struct net_device *dev) memset(&priv->InterruptLog, 0, sizeof(struct log_int_8190)); priv->RxCounter = 0; priv->rtllib->wx_set_enc = 0; - priv->bHwRadioOff = false; + priv->hw_radio_off = false; priv->RegRfOff = false; priv->isRFOff = false; priv->bInPowerSaveMode = false; - priv->rtllib->RfOffReason = 0; - priv->RFChangeInProgress = false; + priv->rtllib->rf_off_reason = 0; + priv->rf_change_in_progress = false; priv->bHwRfOffAction = 0; priv->SetRFPowerStateInProgress = false; priv->rtllib->PowerSaveControl.bInactivePs = true; @@ -949,7 +892,7 @@ static void _rtl92e_init_priv_variable(struct net_device *dev) priv->rtllib->PowerSaveControl.bFwCtrlLPS = false; priv->rtllib->LPSDelayCnt = 0; priv->rtllib->sta_sleep = LPS_IS_WAKE; - priv->rtllib->eRFPowerState = eRfOn; + priv->rtllib->rf_power_state = rf_on; priv->rtllib->current_network.beacon_interval = DEFAULT_BEACONINTERVAL; priv->rtllib->iw_mode = IW_MODE_INFRA; @@ -1032,7 +975,6 @@ static short _rtl92e_get_channel_map(struct net_device *dev) "rtl819x_init:Error channel plan! Set to default.\n"); priv->ChannelPlan = COUNTRY_CODE_FCC; } - RT_TRACE(COMP_INIT, "Channel plan is %d\n", priv->ChannelPlan); dot11d_init(priv->rtllib); dot11d_channel_map(priv->ChannelPlan, priv->rtllib); for (i = 1; i <= 11; i++) @@ -1072,7 +1014,6 @@ static short _rtl92e_init(struct net_device *dev) } priv->irq = dev->irq; - RT_TRACE(COMP_INIT, "IRQ %d\n", dev->irq); if (_rtl92e_pci_initdescring(dev) != 0) { netdev_err(dev, "Endopoints initialization failed"); @@ -1149,11 +1090,8 @@ static enum reset_type _rtl92e_tx_check_stuck(struct net_device *dev) spin_unlock_irqrestore(&priv->irq_th_lock, flags); if (bCheckFwTxCnt) { - if (priv->ops->TxCheckStuckHandler(dev)) { - RT_TRACE(COMP_RESET, - "TxCheckStuck(): Fw indicates no Tx condition!\n"); + if (priv->ops->tx_check_stuck_handler(dev)) return RESET_TYPE_SILENT; - } } return RESET_TYPE_NORESET; @@ -1163,10 +1101,8 @@ static enum reset_type _rtl92e_rx_check_stuck(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); - if (priv->ops->RxCheckStuckHandler(dev)) { - RT_TRACE(COMP_RESET, "RxStuck Condition\n"); + if (priv->ops->rx_check_stuck_handler(dev)) return RESET_TYPE_SILENT; - } return RESET_TYPE_NORESET; } @@ -1178,12 +1114,12 @@ static enum reset_type _rtl92e_if_check_reset(struct net_device *dev) enum reset_type RxResetType = RESET_TYPE_NORESET; enum rt_rf_power_state rfState; - rfState = priv->rtllib->eRFPowerState; + rfState = priv->rtllib->rf_power_state; - if (rfState == eRfOn) + if (rfState == rf_on) TxResetType = _rtl92e_tx_check_stuck(dev); - if (rfState == eRfOn && + if (rfState == rf_on && (priv->rtllib->iw_mode == IW_MODE_INFRA) && (priv->rtllib->state == RTLLIB_LINKED)) RxResetType = _rtl92e_rx_check_stuck(dev); @@ -1201,7 +1137,6 @@ static enum reset_type _rtl92e_if_check_reset(struct net_device *dev) } else { return RESET_TYPE_NORESET; } - } static void _rtl92e_if_silent_reset(struct net_device *dev) @@ -1213,17 +1148,14 @@ static void _rtl92e_if_silent_reset(struct net_device *dev) unsigned long flag; if (priv->ResetProgress == RESET_TYPE_NORESET) { - - RT_TRACE(COMP_RESET, "=========>Reset progress!!\n"); - priv->ResetProgress = RESET_TYPE_SILENT; spin_lock_irqsave(&priv->rf_ps_lock, flag); - if (priv->RFChangeInProgress) { + if (priv->rf_change_in_progress) { spin_unlock_irqrestore(&priv->rf_ps_lock, flag); goto END; } - priv->RFChangeInProgress = true; + priv->rf_change_in_progress = true; priv->bResetInProgress = true; spin_unlock_irqrestore(&priv->rf_ps_lock, flag); @@ -1242,12 +1174,7 @@ RESET_START: } priv->up = 0; - RT_TRACE(COMP_RESET, "%s():======>start to down the driver\n", - __func__); mdelay(1000); - RT_TRACE(COMP_RESET, - "%s():111111111111111111111111======>start to down the driver\n", - __func__); if (!netif_queue_stopped(dev)) netif_stop_queue(dev); @@ -1275,16 +1202,8 @@ RESET_START: rtl92e_dm_backup_state(dev); mutex_unlock(&priv->wx_mutex); - RT_TRACE(COMP_RESET, - "%s():<==========down process is finished\n", - __func__); - - RT_TRACE(COMP_RESET, "%s():<===========up process start\n", - __func__); reset_status = _rtl92e_up(dev, true); - RT_TRACE(COMP_RESET, - "%s():<===========up process is finished\n", __func__); if (reset_status == -1) { if (reset_times < 3) { reset_times++; @@ -1298,7 +1217,7 @@ RESET_START: ieee->is_silent_reset = 1; spin_lock_irqsave(&priv->rf_ps_lock, flag); - priv->RFChangeInProgress = false; + priv->rf_change_in_progress = false; spin_unlock_irqrestore(&priv->rf_ps_lock, flag); rtl92e_enable_hw_security_config(dev); @@ -1333,8 +1252,6 @@ END: priv->bResetInProgress = false; rtl92e_writeb(dev, UFWP, 1); - RT_TRACE(COMP_RESET, "Reset finished!! ====>[%d]\n", - priv->reset_count); } } @@ -1375,7 +1292,7 @@ static void _rtl92e_watchdog_wq_cb(void *data) bool bHigherBusyRxTraffic = false; bool bEnterPS = false; - if (!priv->up || priv->bHwRadioOff) + if (!priv->up || priv->hw_radio_off) return; if (priv->rtllib->state >= RTLLIB_LINKED) { @@ -1390,13 +1307,11 @@ static void _rtl92e_watchdog_wq_cb(void *data) if (!rtllib_act_scanning(priv->rtllib, false)) { if ((ieee->iw_mode == IW_MODE_INFRA) && (ieee->state == RTLLIB_NOLINK) && - (ieee->eRFPowerState == eRfOn) && !ieee->is_set_key && + (ieee->rf_power_state == rf_on) && !ieee->is_set_key && (!ieee->proto_stoppping) && !ieee->wx_set_enc) { if ((ieee->PowerSaveControl.ReturnPoint == IPS_CALLBACK_NONE) && (!ieee->bNetPromiscuousMode)) { - RT_TRACE(COMP_PS, - "====================>haha: rtl92e_ips_enter()\n"); rtl92e_ips_enter(dev); } } @@ -1407,7 +1322,6 @@ static void _rtl92e_watchdog_wq_cb(void *data) ieee->LinkDetectInfo.NumTxOkInPeriod > 100) bBusyTraffic = true; - if (ieee->LinkDetectInfo.NumRxOkInPeriod > 4000 || ieee->LinkDetectInfo.NumTxOkInPeriod > 4000) { bHigherBusyTraffic = true; @@ -1433,7 +1347,6 @@ static void _rtl92e_watchdog_wq_cb(void *data) rtl92e_leisure_ps_leave(dev); } else { - RT_TRACE(COMP_LPS, "====>no link LPS leave\n"); rtl92e_leisure_ps_leave(dev); } @@ -1456,9 +1369,8 @@ static void _rtl92e_watchdog_wq_cb(void *data) else priv->check_roaming_cnt = 0; - if (priv->check_roaming_cnt > 0) { - if (ieee->eRFPowerState == eRfOff) + if (ieee->rf_power_state == rf_off) netdev_info(dev, "%s(): RF is off\n", __func__); netdev_info(dev, @@ -1487,12 +1399,11 @@ static void _rtl92e_watchdog_wq_cb(void *data) } ieee->LinkDetectInfo.NumRecvBcnInPeriod = 0; ieee->LinkDetectInfo.NumRecvDataInPeriod = 0; - } spin_lock_irqsave(&priv->tx_lock, flags); if ((check_reset_cnt++ >= 3) && (!ieee->is_roaming) && - (!priv->RFChangeInProgress) && (!pPSC->bSwRfProcessing)) { + (!priv->rf_change_in_progress) && (!pPSC->bSwRfProcessing)) { ResetType = _rtl92e_if_check_reset(dev); check_reset_cnt = 3; } @@ -1500,7 +1411,6 @@ static void _rtl92e_watchdog_wq_cb(void *data) if (!priv->bDisableNormalResetCheck && ResetType == RESET_TYPE_NORMAL) { priv->ResetProgress = RESET_TYPE_NORMAL; - RT_TRACE(COMP_RESET, "%s(): NOMAL RESET\n", __func__); return; } @@ -1510,7 +1420,6 @@ static void _rtl92e_watchdog_wq_cb(void *data) priv->force_reset = false; priv->bForcedSilentReset = false; priv->bResetInProgress = false; - RT_TRACE(COMP_TRACE, " <==RtUsbCheckForHangWorkItemCallback()\n"); } static void _rtl92e_watchdog_timer_cb(struct timer_list *t) @@ -1541,7 +1450,6 @@ void rtl92e_tx_enable(struct net_device *dev) rtllib_reset_queue(priv->rtllib); } - static void _rtl92e_free_rx_ring(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); @@ -1599,7 +1507,7 @@ static void _rtl92e_hard_data_xmit(struct sk_buff *skb, struct net_device *dev, MAX_DEV_ADDR_SIZE); u8 queue_index = tcb_desc->queue_index; - if ((priv->rtllib->eRFPowerState == eRfOff) || !priv->up || + if ((priv->rtllib->rf_power_state == rf_off) || !priv->up || priv->bResetInProgress) { kfree_skb(skb); return; @@ -1632,7 +1540,7 @@ static int _rtl92e_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) u8 queue_index = tcb_desc->queue_index; if (queue_index != TXCMD_QUEUE) { - if ((priv->rtllib->eRFPowerState == eRfOff) || + if ((priv->rtllib->rf_power_state == rf_off) || !priv->up || priv->bResetInProgress) { kfree_skb(skb); return 0; @@ -1936,13 +1844,11 @@ long rtl92e_translate_to_dbm(struct r8192_priv *priv, u8 signal_strength_index) return signal_power; } - void rtl92e_update_rx_statistics(struct r8192_priv *priv, struct rtllib_rx_stats *pprevious_stats) { int weighting = 0; - if (priv->stats.recv_signal_power == 0) priv->stats.recv_signal_power = pprevious_stats->RecvSignalPower; @@ -1985,8 +1891,6 @@ void rtl92e_copy_mpdu_stats(struct rtllib_rx_stats *psrc_stats, ptarget_stats->bFirstMPDU = psrc_stats->bFirstMPDU; } - - static void _rtl92e_rx_normal(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); @@ -2086,7 +1990,6 @@ done: priv->rx_idx[rx_queue_idx] = (priv->rx_idx[rx_queue_idx] + 1) % priv->rxringcount; } - } static void _rtl92e_tx_resume(struct net_device *dev) @@ -2151,7 +2054,6 @@ static int _rtl92e_open(struct net_device *dev) ret = _rtl92e_try_up(dev); mutex_unlock(&priv->wx_mutex); return ret; - } static int _rtl92e_try_up(struct net_device *dev) @@ -2163,7 +2065,6 @@ static int _rtl92e_try_up(struct net_device *dev) return _rtl92e_up(dev, false); } - static int _rtl92e_close(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); @@ -2181,7 +2082,6 @@ static int _rtl92e_close(struct net_device *dev) mutex_unlock(&priv->wx_mutex); return ret; - } static int _rtl92e_down(struct net_device *dev, bool shutdownrf) @@ -2224,10 +2124,8 @@ static void _rtl92e_set_multicast(struct net_device *dev) promisc = (dev->flags & IFF_PROMISC) ? 1 : 0; priv->promisc = promisc; - } - static int _rtl92e_set_mac_adr(struct net_device *dev, void *mac) { struct r8192_priv *priv = rtllib_priv(dev); @@ -2278,21 +2176,13 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev) goto done; } - if (inta & IMR_TBDOK) { - RT_TRACE(COMP_INTR, "beacon ok interrupt!\n"); + if (inta & IMR_TBDOK) priv->stats.txbeaconokint++; - } - if (inta & IMR_TBDER) { - RT_TRACE(COMP_INTR, "beacon ok interrupt!\n"); + if (inta & IMR_TBDER) priv->stats.txbeaconerr++; - } - - if (inta & IMR_BDOK) - RT_TRACE(COMP_INTR, "beacon interrupt!\n"); if (inta & IMR_MGNTDOK) { - RT_TRACE(COMP_INTR, "Manage ok interrupt!\n"); priv->stats.txmanageokint++; _rtl92e_tx_isr(dev, MGNT_QUEUE); spin_unlock_irqrestore(&priv->irq_th_lock, flags); @@ -2319,13 +2209,10 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev) tasklet_schedule(&priv->irq_rx_tasklet); } - if (inta & IMR_BcnInt) { - RT_TRACE(COMP_INTR, "prepare beacon for interrupt!\n"); + if (inta & IMR_BcnInt) tasklet_schedule(&priv->irq_prepare_beacon_tasklet); - } if (inta & IMR_RDU) { - RT_TRACE(COMP_INTR, "rx descriptor unavailable!\n"); priv->stats.rxrdu++; rtl92e_writel(dev, INTA_MASK, rtl92e_readl(dev, INTA_MASK) & ~IMR_RDU); @@ -2333,7 +2220,6 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev) } if (inta & IMR_RXFOVW) { - RT_TRACE(COMP_INTR, "rx overflow !\n"); priv->stats.rxoverflow++; tasklet_schedule(&priv->irq_rx_tasklet); } @@ -2342,21 +2228,18 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev) priv->stats.txoverflow++; if (inta & IMR_BKDOK) { - RT_TRACE(COMP_INTR, "BK Tx OK interrupt!\n"); priv->stats.txbkokint++; priv->rtllib->LinkDetectInfo.NumTxOkInPeriod++; _rtl92e_tx_isr(dev, BK_QUEUE); } if (inta & IMR_BEDOK) { - RT_TRACE(COMP_INTR, "BE TX OK interrupt!\n"); priv->stats.txbeokint++; priv->rtllib->LinkDetectInfo.NumTxOkInPeriod++; _rtl92e_tx_isr(dev, BE_QUEUE); } if (inta & IMR_VIDOK) { - RT_TRACE(COMP_INTR, "VI TX OK interrupt!\n"); priv->stats.txviokint++; priv->rtllib->LinkDetectInfo.NumTxOkInPeriod++; _rtl92e_tx_isr(dev, VI_QUEUE); @@ -2364,7 +2247,6 @@ static irqreturn_t _rtl92e_irq(int irq, void *netdev) if (inta & IMR_VODOK) { priv->stats.txvookint++; - RT_TRACE(COMP_INTR, "Vo TX OK interrupt!\n"); priv->rtllib->LinkDetectInfo.NumTxOkInPeriod++; _rtl92e_tx_isr(dev, VO_QUEUE); } @@ -2376,8 +2258,6 @@ done: return IRQ_HANDLED; } - - /**************************************************************************** * ---------------------------- PCI_STUFF--------------------------- ****************************************************************************/ @@ -2402,8 +2282,6 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev, int err = -ENOMEM; u8 revision_id; - RT_TRACE(COMP_INIT, "Configuring chip resources"); - if (pci_enable_device(pdev)) { dev_err(&pdev->dev, "Failed to enable PCI device"); return -EIO; @@ -2452,7 +2330,6 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev, goto err_rel_rtllib; } - ioaddr = (unsigned long)ioremap(pmem_start, pmem_len); if (ioaddr == (unsigned long)NULL) { netdev_err(dev, "ioremap failed!"); @@ -2483,13 +2360,9 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev, dev->type = ARPHRD_ETHER; dev->watchdog_timeo = HZ * 3; - if (dev_alloc_name(dev, ifname) < 0) { - RT_TRACE(COMP_INIT, - "Oops: devname already taken! Trying wlan%%d...\n"); + if (dev_alloc_name(dev, ifname) < 0) dev_alloc_name(dev, ifname); - } - RT_TRACE(COMP_INIT, "Driver probe completed1\n"); if (_rtl92e_init(dev) != 0) { netdev_warn(dev, "Initialization failed"); goto err_free_irq; @@ -2500,12 +2373,10 @@ static int _rtl92e_pci_probe(struct pci_dev *pdev, if (register_netdev(dev)) goto err_free_irq; - RT_TRACE(COMP_INIT, "dev name: %s\n", dev->name); if (priv->polling_timer_on == 0) rtl92e_check_rfctrl_gpio_timer(&priv->gpio_polling_timer); - RT_TRACE(COMP_INIT, "Driver probe completed\n"); return 0; err_free_irq: @@ -2560,7 +2431,6 @@ static void _rtl92e_pci_disconnect(struct pci_dev *pdev) } pci_disable_device(pdev); - RT_TRACE(COMP_DOWN, "wlan driver removed\n"); } bool rtl92e_enable_nic(struct net_device *dev) @@ -2576,7 +2446,6 @@ bool rtl92e_enable_nic(struct net_device *dev) return false; } - RT_TRACE(COMP_PS, "===========>%s()\n", __func__); priv->bfirst_init = true; init_status = priv->ops->initialize_adapter(dev); if (!init_status) { @@ -2584,13 +2453,11 @@ bool rtl92e_enable_nic(struct net_device *dev) priv->bdisable_nic = false; return false; } - RT_TRACE(COMP_INIT, "start adapter finished\n"); RT_CLEAR_PS_LEVEL(pPSC, RT_RF_OFF_LEVL_HALT_NIC); priv->bfirst_init = false; rtl92e_irq_enable(dev); priv->bdisable_nic = false; - RT_TRACE(COMP_PS, "<===========%s()\n", __func__); return init_status; } @@ -2599,7 +2466,6 @@ bool rtl92e_disable_nic(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); u8 tmp_state = 0; - RT_TRACE(COMP_PS, "=========>%s()\n", __func__); priv->bdisable_nic = true; tmp_state = priv->rtllib->state; rtllib_softmac_stop_protocol(priv->rtllib, 0, false); @@ -2608,8 +2474,6 @@ bool rtl92e_disable_nic(struct net_device *dev) rtl92e_irq_disable(dev); priv->ops->stop_adapter(dev, false); - RT_TRACE(COMP_PS, "<=========%s()\n", __func__); - return true; } diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h index 698552a921009855e65f51a15d51abde1b7625d4..7021f9c435d96b40d7823a3adf9d304cdda7a33d 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_core.h +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_core.h @@ -299,8 +299,8 @@ struct rtl819x_ops { void (*tx_enable)(struct net_device *dev); void (*interrupt_recognized)(struct net_device *dev, u32 *p_inta, u32 *p_intb); - bool (*TxCheckStuckHandler)(struct net_device *dev); - bool (*RxCheckStuckHandler)(struct net_device *dev); + bool (*tx_check_stuck_handler)(struct net_device *dev); + bool (*rx_check_stuck_handler)(struct net_device *dev); }; struct r8192_priv { @@ -392,7 +392,7 @@ struct r8192_priv { u16 ShortRetryLimit; u16 LongRetryLimit; - bool bHwRadioOff; + bool hw_radio_off; bool blinked_ingpio; u8 polling_timer_on; @@ -430,7 +430,7 @@ struct r8192_priv { u16 basic_rate; u8 short_preamble; - u8 dot11CurrentPreambleMode; + u8 dot11_current_preamble_mode; u8 slot_time; u16 SifsTime; @@ -478,7 +478,7 @@ struct r8192_priv { bool bInPowerSaveMode; u8 bHwRfOffAction; - bool RFChangeInProgress; + bool rf_change_in_progress; bool SetRFPowerStateInProgress; bool bdisable_nic; @@ -598,6 +598,6 @@ bool rtl92e_enable_nic(struct net_device *dev); bool rtl92e_disable_nic(struct net_device *dev); bool rtl92e_set_rf_state(struct net_device *dev, - enum rt_rf_power_state StateToSet, - RT_RF_CHANGE_SOURCE ChangeSource); + enum rt_rf_power_state state_to_set, + RT_RF_CHANGE_SOURCE change_source); #endif diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c index d58800d06e8f62d9354306fd810d3361c4e26c63..702551056227dc3eede31dbeb3e42ffe382f0ff6 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.c @@ -268,8 +268,6 @@ static void _rtl92e_dm_check_ac_dc_power(struct net_device *dev) NULL}; if (priv->ResetProgress == RESET_TYPE_SILENT) { - RT_TRACE((COMP_INIT | COMP_POWER | COMP_RF), - "GPIOChangeRFWorkItemCallBack(): Silent Reset!!!!!!!\n"); return; } @@ -333,8 +331,6 @@ static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev) static u8 ping_rssi_state; if (!priv->up) { - RT_TRACE(COMP_RATE, - "<---- %s: driver is going to unload\n", __func__); return; } @@ -347,9 +343,9 @@ static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev) if (priv->rtllib->state == RTLLIB_LINKED) { - bshort_gi_enabled = (pHTInfo->bCurTxBW40MHz && + bshort_gi_enabled = (pHTInfo->cur_tx_bw40mhz && pHTInfo->bCurShortGI40MHz) || - (!pHTInfo->bCurTxBW40MHz && + (!pHTInfo->cur_tx_bw40mhz && pHTInfo->bCurShortGI20MHz); pra->upper_rssi_threshold_ratr = @@ -423,9 +419,6 @@ static void _rtl92e_dm_check_rate_adaptive(struct net_device *dev) u32 ratr_value; ratr_value = targetRATR; - RT_TRACE(COMP_RATE, - "currentRATR = %x, targetRATR = %x\n", - currentRATR, targetRATR); if (priv->rf_type == RF_1T2R) ratr_value &= ~(RATE_ALL_OFDM_2SS); rtl92e_writel(dev, RATR0, ratr_value); @@ -628,7 +621,6 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) u16 Avg_TSSI_Meas, TSSI_13dBm, Avg_TSSI_Meas_from_driver = 0; u32 delta = 0; - RT_TRACE(COMP_POWER_TRACKING, "%s()\n", __func__); rtl92e_writeb(dev, Pw_Track_Flag, 0); rtl92e_writeb(dev, FW_Busy_Flag, 0); priv->rtllib->bdynamic_txpower_enable = false; @@ -637,10 +629,6 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) RF_Type = priv->rf_type; Value = (RF_Type<<8) | powerlevelOFDM24G; - RT_TRACE(COMP_POWER_TRACKING, "powerlevelOFDM24G = %x\n", - powerlevelOFDM24G); - - for (j = 0; j <= 30; j++) { tx_cmd.Op = TXCMD_SET_TX_PWR_TRACKING; @@ -656,15 +644,11 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) mdelay(1); if (priv->bResetInProgress) { - RT_TRACE(COMP_POWER_TRACKING, - "we are in silent reset progress, so return\n"); rtl92e_writeb(dev, Pw_Track_Flag, 0); rtl92e_writeb(dev, FW_Busy_Flag, 0); return; } - if (priv->rtllib->eRFPowerState != eRfOn) { - RT_TRACE(COMP_POWER_TRACKING, - "we are in power save, so return\n"); + if (priv->rtllib->rf_power_state != rf_on) { rtl92e_writeb(dev, Pw_Track_Flag, 0); rtl92e_writeb(dev, FW_Busy_Flag, 0); return; @@ -689,10 +673,6 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) tmp_report[k] = rtl92e_readb(dev, Tssi_Report_Value2); - RT_TRACE(COMP_POWER_TRACKING, - "TSSI_report_value = %d\n", - tmp_report[k]); - if (tmp_report[k] <= 20) { viviflag = true; break; @@ -702,8 +682,6 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) if (viviflag) { rtl92e_writeb(dev, Pw_Track_Flag, 0); viviflag = false; - RT_TRACE(COMP_POWER_TRACKING, - "we filted this data\n"); for (k = 0; k < 5; k++) tmp_report[k] = 0; break; @@ -713,12 +691,7 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) Avg_TSSI_Meas_from_driver += tmp_report[k]; Avg_TSSI_Meas_from_driver *= 100 / 5; - RT_TRACE(COMP_POWER_TRACKING, - "Avg_TSSI_Meas_from_driver = %d\n", - Avg_TSSI_Meas_from_driver); TSSI_13dBm = priv->TSSI_13dBm; - RT_TRACE(COMP_POWER_TRACKING, "TSSI_13dBm = %d\n", - TSSI_13dBm); if (Avg_TSSI_Meas_from_driver > TSSI_13dBm) delta = Avg_TSSI_Meas_from_driver - TSSI_13dBm; @@ -729,20 +702,6 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) priv->rtllib->bdynamic_txpower_enable = true; rtl92e_writeb(dev, Pw_Track_Flag, 0); rtl92e_writeb(dev, FW_Busy_Flag, 0); - RT_TRACE(COMP_POWER_TRACKING, - "tx power track is done\n"); - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex = %d\n", - priv->rfa_txpowertrackingindex); - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex_real = %d\n", - priv->rfa_txpowertrackingindex_real); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation_difference = %d\n", - priv->CCKPresentAttentuation_difference); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation = %d\n", - priv->CCKPresentAttentuation); return; } if (Avg_TSSI_Meas_from_driver < TSSI_13dBm - E_FOR_TX_POWER_TRACK) @@ -785,26 +744,12 @@ static void _rtl92e_dm_tx_power_tracking_callback_tssi(struct net_device *dev) } else rtl92e_dm_cck_txpower_adjust(dev, priv->bcck_in_ch14); } - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex = %d\n", - priv->rfa_txpowertrackingindex); - RT_TRACE(COMP_POWER_TRACKING, - "priv->rfa_txpowertrackingindex_real = %d\n", - priv->rfa_txpowertrackingindex_real); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation_difference = %d\n", - priv->CCKPresentAttentuation_difference); - RT_TRACE(COMP_POWER_TRACKING, - "priv->CCKPresentAttentuation = %d\n", - priv->CCKPresentAttentuation); if (priv->CCKPresentAttentuation_difference <= -12 || priv->CCKPresentAttentuation_difference >= 24) { priv->rtllib->bdynamic_txpower_enable = true; rtl92e_writeb(dev, Pw_Track_Flag, 0); rtl92e_writeb(dev, FW_Busy_Flag, 0); - RT_TRACE(COMP_POWER_TRACKING, - "tx power track--->limited\n"); return; } @@ -834,10 +779,6 @@ static void _rtl92e_dm_tx_power_tracking_cb_thermal(struct net_device *dev) for (i = 0; i < OFDM_Table_Length; i++) { if (tmpRegA == OFDMSwingTable[i]) { priv->OFDM_index[0] = i; - RT_TRACE(COMP_POWER_TRACKING, - "Initial reg0x%x = 0x%x, OFDM_index = 0x%x\n", - rOFDM0_XATxIQImbalance, tmpRegA, - priv->OFDM_index[0]); } } @@ -845,10 +786,6 @@ static void _rtl92e_dm_tx_power_tracking_cb_thermal(struct net_device *dev) for (i = 0; i < CCK_Table_length; i++) { if (TempCCk == (u32)CCKSwingTable_Ch1_Ch13[i][0]) { priv->CCK_index = i; - RT_TRACE(COMP_POWER_TRACKING, - "Initial reg0x%x = 0x%x, CCK_index = 0x%x\n", - rCCK0_TxFilter1, TempCCk, - priv->CCK_index); break; } } @@ -857,12 +794,10 @@ static void _rtl92e_dm_tx_power_tracking_cb_thermal(struct net_device *dev) } tmpRegA = rtl92e_get_rf_reg(dev, RF90_PATH_A, 0x12, 0x078); - RT_TRACE(COMP_POWER_TRACKING, "Readback ThermalMeterA = %d\n", tmpRegA); if (tmpRegA < 3 || tmpRegA > 13) return; if (tmpRegA >= 12) tmpRegA = 12; - RT_TRACE(COMP_POWER_TRACKING, "Valid ThermalMeterA = %d\n", tmpRegA); priv->ThermalMeter[0] = ThermalMeterVal; priv->ThermalMeter[1] = ThermalMeterVal; @@ -894,9 +829,6 @@ static void _rtl92e_dm_tx_power_tracking_cb_thermal(struct net_device *dev) priv->Record_CCK_20Mindex = tmpCCK20Mindex; priv->Record_CCK_40Mindex = tmpCCK40Mindex; - RT_TRACE(COMP_POWER_TRACKING, - "Record_CCK_20Mindex / Record_CCK_40Mindex = %d / %d.\n", - priv->Record_CCK_20Mindex, priv->Record_CCK_40Mindex); if (priv->rtllib->current_network.channel == 14 && !priv->bcck_in_ch14) { @@ -919,9 +851,6 @@ static void _rtl92e_dm_tx_power_tracking_cb_thermal(struct net_device *dev) priv->OFDM_index[0] = tmpOFDMindex; rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance, bMaskDWord, OFDMSwingTable[priv->OFDM_index[0]]); - RT_TRACE(COMP_POWER_TRACKING, "Update OFDMSwing[%d] = 0x%x\n", - priv->OFDM_index[0], - OFDMSwingTable[priv->OFDM_index[0]]); } priv->txpower_count = 0; } @@ -960,8 +889,6 @@ static void _rtl92e_dm_init_tx_power_tracking_thermal(struct net_device *dev) priv->btxpower_tracking = false; priv->txpower_count = 0; priv->btxpower_trackingInit = false; - RT_TRACE(COMP_POWER_TRACKING, "pMgntInfo->bTXPowerTracking = %d\n", - priv->btxpower_tracking); } void rtl92e_dm_init_txpower_tracking(struct net_device *dev) @@ -979,7 +906,6 @@ static void _rtl92e_dm_check_tx_power_tracking_tssi(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); static u32 tx_power_track_counter; - RT_TRACE(COMP_POWER_TRACKING, "%s()\n", __func__); if (rtl92e_readb(dev, 0x11e) == 1) return; if (!priv->btxpower_tracking) @@ -1086,44 +1012,29 @@ static void _rtl92e_dm_cck_tx_power_adjust_thermal_meter(struct net_device *dev, TempVal = CCKSwingTable_Ch1_Ch13[priv->CCK_index][0] + (CCKSwingTable_Ch1_Ch13[priv->CCK_index][1] << 8); rtl92e_set_bb_reg(dev, rCCK0_TxFilter1, bMaskHWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, - "CCK not chnl 14, reg 0x%x = 0x%x\n", rCCK0_TxFilter1, - TempVal); TempVal = CCKSwingTable_Ch1_Ch13[priv->CCK_index][2] + (CCKSwingTable_Ch1_Ch13[priv->CCK_index][3] << 8) + (CCKSwingTable_Ch1_Ch13[priv->CCK_index][4] << 16)+ (CCKSwingTable_Ch1_Ch13[priv->CCK_index][5] << 24); rtl92e_set_bb_reg(dev, rCCK0_TxFilter2, bMaskDWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, - "CCK not chnl 14, reg 0x%x = 0x%x\n", rCCK0_TxFilter2, - TempVal); TempVal = CCKSwingTable_Ch1_Ch13[priv->CCK_index][6] + (CCKSwingTable_Ch1_Ch13[priv->CCK_index][7] << 8); rtl92e_set_bb_reg(dev, rCCK0_DebugPort, bMaskLWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, - "CCK not chnl 14, reg 0x%x = 0x%x\n", rCCK0_DebugPort, - TempVal); } else { TempVal = CCKSwingTable_Ch14[priv->CCK_index][0] + (CCKSwingTable_Ch14[priv->CCK_index][1] << 8); rtl92e_set_bb_reg(dev, rCCK0_TxFilter1, bMaskHWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, "CCK chnl 14, reg 0x%x = 0x%x\n", - rCCK0_TxFilter1, TempVal); TempVal = CCKSwingTable_Ch14[priv->CCK_index][2] + (CCKSwingTable_Ch14[priv->CCK_index][3] << 8) + (CCKSwingTable_Ch14[priv->CCK_index][4] << 16)+ (CCKSwingTable_Ch14[priv->CCK_index][5] << 24); rtl92e_set_bb_reg(dev, rCCK0_TxFilter2, bMaskDWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, "CCK chnl 14, reg 0x%x = 0x%x\n", - rCCK0_TxFilter2, TempVal); TempVal = CCKSwingTable_Ch14[priv->CCK_index][6] + (CCKSwingTable_Ch14[priv->CCK_index][7]<<8); rtl92e_set_bb_reg(dev, rCCK0_DebugPort, bMaskLWord, TempVal); - RT_TRACE(COMP_POWER_TRACKING, "CCK chnl 14, reg 0x%x = 0x%x\n", - rCCK0_DebugPort, TempVal); } } @@ -1141,32 +1052,12 @@ static void _rtl92e_dm_tx_power_reset_recovery(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_POWER_TRACKING, "Start Reset Recovery ==>\n"); rtl92e_set_bb_reg(dev, rOFDM0_XATxIQImbalance, bMaskDWord, dm_tx_bb_gain[priv->rfa_txpowertrackingindex]); - RT_TRACE(COMP_POWER_TRACKING, "Reset Recovery: Fill in 0xc80 is %08x\n", - dm_tx_bb_gain[priv->rfa_txpowertrackingindex]); - RT_TRACE(COMP_POWER_TRACKING, - "Reset Recovery: Fill in RFA_txPowerTrackingIndex is %x\n", - priv->rfa_txpowertrackingindex); - RT_TRACE(COMP_POWER_TRACKING, - "Reset Recovery : RF A I/Q Amplify Gain is %d\n", - dm_tx_bb_gain_idx_to_amplify(priv->rfa_txpowertrackingindex)); - RT_TRACE(COMP_POWER_TRACKING, - "Reset Recovery: CCK Attenuation is %d dB\n", - priv->CCKPresentAttentuation); rtl92e_dm_cck_txpower_adjust(dev, priv->bcck_in_ch14); rtl92e_set_bb_reg(dev, rOFDM0_XCTxIQImbalance, bMaskDWord, dm_tx_bb_gain[priv->rfc_txpowertrackingindex]); - RT_TRACE(COMP_POWER_TRACKING, "Reset Recovery: Fill in 0xc90 is %08x\n", - dm_tx_bb_gain[priv->rfc_txpowertrackingindex]); - RT_TRACE(COMP_POWER_TRACKING, - "Reset Recovery: Fill in RFC_txPowerTrackingIndex is %x\n", - priv->rfc_txpowertrackingindex); - RT_TRACE(COMP_POWER_TRACKING, - "Reset Recovery : RF C I/Q Amplify Gain is %d\n", - dm_tx_bb_gain_idx_to_amplify(priv->rfc_txpowertrackingindex)); } void rtl92e_dm_restore_state(struct net_device *dev) @@ -1176,8 +1067,6 @@ void rtl92e_dm_restore_state(struct net_device *dev) u32 ratr_value; if (!priv->up) { - RT_TRACE(COMP_RATE, - "<---- %s: driver is going to unload\n", __func__); return; } @@ -1218,17 +1107,6 @@ static void _rtl92e_dm_bb_initialgain_restore(struct net_device *dev) bit_mask = bMaskByte2; rtl92e_set_bb_reg(dev, rCCK0_CCA, bit_mask, (u32)priv->initgain_backup.cca); - - RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xc50 is %x\n", - priv->initgain_backup.xaagccore1); - RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xc58 is %x\n", - priv->initgain_backup.xbagccore1); - RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xc60 is %x\n", - priv->initgain_backup.xcagccore1); - RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xc68 is %x\n", - priv->initgain_backup.xdagccore1); - RT_TRACE(COMP_DIG, "dm_BBInitialGainRestore 0xa0a is %x\n", - priv->initgain_backup.cca); rtl92e_set_bb_reg(dev, UFWP, bMaskByte1, 0x1); } @@ -1251,17 +1129,6 @@ void rtl92e_dm_backup_state(struct net_device *dev) priv->initgain_backup.xdagccore1 = rtl92e_get_bb_reg(dev, rOFDM0_XDAGCCore1, bit_mask); bit_mask = bMaskByte2; priv->initgain_backup.cca = (u8)rtl92e_get_bb_reg(dev, rCCK0_CCA, bit_mask); - - RT_TRACE(COMP_DIG, "BBInitialGainBackup 0xc50 is %x\n", - priv->initgain_backup.xaagccore1); - RT_TRACE(COMP_DIG, "BBInitialGainBackup 0xc58 is %x\n", - priv->initgain_backup.xbagccore1); - RT_TRACE(COMP_DIG, "BBInitialGainBackup 0xc60 is %x\n", - priv->initgain_backup.xcagccore1); - RT_TRACE(COMP_DIG, "BBInitialGainBackup 0xc68 is %x\n", - priv->initgain_backup.xdagccore1); - RT_TRACE(COMP_DIG, "BBInitialGainBackup 0xa0a is %x\n", - priv->initgain_backup.cca); } static void _rtl92e_dm_dig_init(struct net_device *dev) @@ -1681,13 +1548,13 @@ static void _rtl92e_dm_check_edca_turbo(struct net_device *dev) goto dm_CheckEdcaTurbo_EXIT; if (priv->rtllib->state != RTLLIB_LINKED) goto dm_CheckEdcaTurbo_EXIT; - if (priv->rtllib->pHTInfo->IOTAction & HT_IOT_ACT_DISABLE_EDCA_TURBO) + if (priv->rtllib->pHTInfo->iot_action & HT_IOT_ACT_DISABLE_EDCA_TURBO) goto dm_CheckEdcaTurbo_EXIT; if (!priv->rtllib->bis_any_nonbepkts) { curTxOkCnt = priv->stats.txbytesunicast - lastTxOkCnt; curRxOkCnt = priv->stats.rxbytesunicast - lastRxOkCnt; - if (pHTInfo->IOTAction & HT_IOT_ACT_EDCA_BIAS_ON_RX) { + if (pHTInfo->iot_action & HT_IOT_ACT_EDCA_BIAS_ON_RX) { if (curTxOkCnt > 4*curRxOkCnt) { if (priv->bis_cur_rdlstate || !priv->bcurrent_turbo_EDCA) { @@ -1766,16 +1633,16 @@ static void _rtl92e_dm_cts_to_self(struct net_device *dev) unsigned long curRxOkCnt = 0; if (!priv->rtllib->bCTSToSelfEnable) { - pHTInfo->IOTAction &= ~HT_IOT_ACT_FORCED_CTS2SELF; + pHTInfo->iot_action &= ~HT_IOT_ACT_FORCED_CTS2SELF; return; } if (pHTInfo->IOTPeer == HT_IOT_PEER_BROADCOM) { curTxOkCnt = priv->stats.txbytesunicast - lastTxOkCnt; curRxOkCnt = priv->stats.rxbytesunicast - lastRxOkCnt; if (curRxOkCnt > 4*curTxOkCnt) - pHTInfo->IOTAction &= ~HT_IOT_ACT_FORCED_CTS2SELF; + pHTInfo->iot_action &= ~HT_IOT_ACT_FORCED_CTS2SELF; else - pHTInfo->IOTAction |= HT_IOT_ACT_FORCED_CTS2SELF; + pHTInfo->iot_action |= HT_IOT_ACT_FORCED_CTS2SELF; lastTxOkCnt = priv->stats.txbytesunicast; lastRxOkCnt = priv->stats.rxbytesunicast; @@ -1798,7 +1665,7 @@ static void _rtl92e_dm_check_rf_ctrl_gpio(void *data) struct r8192_priv, gpio_change_rf_wq); struct net_device *dev = priv->rtllib->dev; u8 tmp1byte; - enum rt_rf_power_state eRfPowerStateToSet; + enum rt_rf_power_state rf_power_state_to_set; bool bActuallySet = false; char *argv[3]; static const char RadioPowerPath[] = "/etc/acpi/events/RadioPower.sh"; @@ -1817,25 +1684,23 @@ static void _rtl92e_dm_check_rf_ctrl_gpio(void *data) tmp1byte = rtl92e_readb(dev, GPI); - eRfPowerStateToSet = (tmp1byte&BIT1) ? eRfOn : eRfOff; + rf_power_state_to_set = (tmp1byte&BIT1) ? rf_on : rf_off; - if (priv->bHwRadioOff && (eRfPowerStateToSet == eRfOn)) { - RT_TRACE(COMP_RF, "gpiochangeRF - HW Radio ON\n"); + if (priv->hw_radio_off && (rf_power_state_to_set == rf_on)) { netdev_info(dev, "gpiochangeRF - HW Radio ON\n"); - priv->bHwRadioOff = false; + priv->hw_radio_off = false; bActuallySet = true; - } else if (!priv->bHwRadioOff && (eRfPowerStateToSet == eRfOff)) { - RT_TRACE(COMP_RF, "gpiochangeRF - HW Radio OFF\n"); + } else if (!priv->hw_radio_off && (rf_power_state_to_set == rf_off)) { netdev_info(dev, "gpiochangeRF - HW Radio OFF\n"); - priv->bHwRadioOff = true; + priv->hw_radio_off = true; bActuallySet = true; } if (bActuallySet) { mdelay(1000); priv->bHwRfOffAction = 1; - rtl92e_set_rf_state(dev, eRfPowerStateToSet, RF_CHANGE_BY_HW); - if (priv->bHwRadioOff) + rtl92e_set_rf_state(dev, rf_power_state_to_set, RF_CHANGE_BY_HW); + if (priv->hw_radio_off) argv[1] = "RFOFF"; else argv[1] = "RFON"; @@ -2132,7 +1997,7 @@ static void _rtl92e_dm_fsync_timer_callback(struct timer_list *t) if (priv->rtllib->state == RTLLIB_LINKED && priv->rtllib->bfsync_enable && - (priv->rtllib->pHTInfo->IOTAction & HT_IOT_ACT_CDD_FSYNC)) { + (priv->rtllib->pHTInfo->iot_action & HT_IOT_ACT_CDD_FSYNC)) { u32 rate_bitmap; for (rate_index = 0; rate_index <= 27; rate_index++) { @@ -2173,10 +2038,6 @@ static void _rtl92e_dm_fsync_timer_callback(struct timer_list *t) } priv->rate_record = rate_count; priv->rateCountDiffRecord = rate_count_diff; - RT_TRACE(COMP_HALDM, - "rateRecord %d rateCount %d, rateCountdiff %d bSwitchFsync %d\n", - priv->rate_record, rate_count, rate_count_diff, - priv->bswitch_fsync); if (priv->undecorated_smoothed_pwdb > priv->rtllib->fsync_rssi_threshold && bSwitchFromCountDiff) { @@ -2220,11 +2081,6 @@ static void _rtl92e_dm_fsync_timer_callback(struct timer_list *t) priv->ContinueDiffCount = 0; rtl92e_writel(dev, rOFDM0_RxDetector2, 0x465c52cd); } - RT_TRACE(COMP_HALDM, "ContinueDiffCount %d\n", priv->ContinueDiffCount); - RT_TRACE(COMP_HALDM, - "rateRecord %d rateCount %d, rateCountdiff %d bSwitchFsync %d\n", - priv->rate_record, rate_count, rate_count_diff, - priv->bswitch_fsync); } static void _rtl92e_dm_start_hw_fsync(struct net_device *dev) @@ -2232,7 +2088,6 @@ static void _rtl92e_dm_start_hw_fsync(struct net_device *dev) u8 rf_timing = 0x77; struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_HALDM, "%s\n", __func__); rtl92e_writel(dev, rOFDM0_RxDetector2, 0x465c12cf); priv->rtllib->SetHwRegHandler(dev, HW_VAR_RF_TIMING, (u8 *)(&rf_timing)); @@ -2244,7 +2099,6 @@ static void _rtl92e_dm_end_hw_fsync(struct net_device *dev) u8 rf_timing = 0xaa; struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_HALDM, "%s\n", __func__); rtl92e_writel(dev, rOFDM0_RxDetector2, 0x465c52cd); priv->rtllib->SetHwRegHandler(dev, HW_VAR_RF_TIMING, (u8 *) (&rf_timing)); @@ -2255,7 +2109,6 @@ static void _rtl92e_dm_end_sw_fsync(struct net_device *dev) { struct r8192_priv *priv = rtllib_priv(dev); - RT_TRACE(COMP_HALDM, "%s\n", __func__); del_timer_sync(&(priv->fsync_timer)); if (priv->bswitch_fsync) { @@ -2276,7 +2129,6 @@ static void _rtl92e_dm_start_sw_fsync(struct net_device *dev) u32 rateIndex; u32 rateBitmap; - RT_TRACE(COMP_HALDM, "%s\n", __func__); priv->rate_record = 0; priv->ContinueDiffCount = 0; priv->rateCountDiffRecord = 0; @@ -2315,17 +2167,6 @@ static void _rtl92e_dm_check_fsync(struct net_device *dev) static u8 reg_c38_State = RegC38_Default; static u32 reset_cnt; - RT_TRACE(COMP_HALDM, - "RSSI %d TimeInterval %d MultipleTimeInterval %d\n", - priv->rtllib->fsync_rssi_threshold, - priv->rtllib->fsync_time_interval, - priv->rtllib->fsync_multiple_timeinterval); - RT_TRACE(COMP_HALDM, - "RateBitmap 0x%x FirstDiffRateThreshold %d SecondDiffRateThreshold %d\n", - priv->rtllib->fsync_rate_bitmap, - priv->rtllib->fsync_firstdiff_ratethreshold, - priv->rtllib->fsync_seconddiff_ratethreshold); - if (priv->rtllib->state == RTLLIB_LINKED && priv->rtllib->pHTInfo->IOTPeer == HT_IOT_PEER_BROADCOM) { if (priv->rtllib->bfsync_enable == 0) { @@ -2461,9 +2302,6 @@ static void _rtl92e_dm_dynamic_tx_power(struct net_device *dev) txlowpower_threshold = TX_POWER_NEAR_FIELD_THRESH_LOW; } - RT_TRACE(COMP_TXAGC, "priv->undecorated_smoothed_pwdb = %ld\n", - priv->undecorated_smoothed_pwdb); - if (priv->rtllib->state == RTLLIB_LINKED) { if (priv->undecorated_smoothed_pwdb >= txhipower_threshold) { priv->bDynamicTxHighPower = true; @@ -2484,9 +2322,6 @@ static void _rtl92e_dm_dynamic_tx_power(struct net_device *dev) if ((priv->bDynamicTxHighPower != priv->bLastDTPFlag_High) || (priv->bDynamicTxLowPower != priv->bLastDTPFlag_Low)) { - RT_TRACE(COMP_TXAGC, "SetTxPowerLevel8190() channel = %d\n", - priv->rtllib->current_network.channel); - rtl92e_set_tx_power(dev, priv->rtllib->current_network.channel); } priv->bLastDTPFlag_High = priv->bDynamicTxHighPower; @@ -2499,14 +2334,9 @@ static void _rtl92e_dm_check_txrateandretrycount(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = priv->rtllib; - ieee->softmac_stats.CurrentShowTxate = rtl92e_readb(dev, - Current_Tx_Rate_Reg); - - ieee->softmac_stats.last_packet_rate = rtl92e_readb(dev, - Initial_Tx_Rate_Reg); - - ieee->softmac_stats.txretrycount = rtl92e_readl(dev, - Tx_Retry_Count_Reg); + ieee->softmac_stats.CurrentShowTxate = rtl92e_readb(dev, CURRENT_TX_RATE_REG); + ieee->softmac_stats.last_packet_rate = rtl92e_readb(dev, INITIAL_TX_RATE_REG); + ieee->softmac_stats.txretrycount = rtl92e_readl(dev, TX_RETRY_COUNT_REG); } static void _rtl92e_dm_send_rssi_to_fw(struct net_device *dev) diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.h b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.h index ea1b14bbcdcd90dac5a26687633d27a93cf099eb..51e295d389a839311e522700ca3c287954beecbf 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_dm.h +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_dm.h @@ -42,9 +42,9 @@ #define TX_POWER_ATHEROAP_THRESH_HIGH 78 #define TX_POWER_ATHEROAP_THRESH_LOW 72 -#define Current_Tx_Rate_Reg 0x1e0 -#define Initial_Tx_Rate_Reg 0x1e1 -#define Tx_Retry_Count_Reg 0x1ac +#define CURRENT_TX_RATE_REG 0x1e0 +#define INITIAL_TX_RATE_REG 0x1e1 +#define TX_RETRY_COUNT_REG 0x1ac #define RegC38_TH 20 #define DM_Type_ByDriver 1 diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c b/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c index 1d992d5c4e17f69cf33a090aa1668828ea2cb0bb..81e1bb856c6076a9c3b85a1c3394ede0513cd631 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_pci.c @@ -16,11 +16,9 @@ static void _rtl92e_parse_pci_configuration(struct pci_dev *pdev, struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev); u8 tmp; - u16 LinkCtrlReg; + u16 link_ctrl_reg; - pcie_capability_read_word(priv->pdev, PCI_EXP_LNKCTL, &LinkCtrlReg); - - RT_TRACE(COMP_INIT, "Link Control Register =%x\n", LinkCtrlReg); + pcie_capability_read_word(priv->pdev, PCI_EXP_LNKCTL, &link_ctrl_reg); pci_read_config_byte(pdev, 0x98, &tmp); tmp |= BIT4; @@ -33,28 +31,28 @@ static void _rtl92e_parse_pci_configuration(struct pci_dev *pdev, bool rtl92e_check_adapter(struct pci_dev *pdev, struct net_device *dev) { struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev); - u16 DeviceID; - u8 RevisionID; - u16 IrqLine; + u16 device_id; + u8 revision_id; + u16 irq_line; - DeviceID = pdev->device; - RevisionID = pdev->revision; - pci_read_config_word(pdev, 0x3C, &IrqLine); + device_id = pdev->device; + revision_id = pdev->revision; + pci_read_config_word(pdev, 0x3C, &irq_line); priv->card_8192 = priv->ops->nic_type; - if (DeviceID == 0x8192) { - switch (RevisionID) { + if (device_id == 0x8192) { + switch (revision_id) { case HAL_HW_PCI_REVISION_ID_8192PCIE: dev_info(&pdev->dev, "Adapter(8192 PCI-E) is found - DeviceID=%x\n", - DeviceID); + device_id); priv->card_8192 = NIC_8192E; break; case HAL_HW_PCI_REVISION_ID_8192SE: dev_info(&pdev->dev, "Adapter(8192SE) is found - DeviceID=%x\n", - DeviceID); + device_id); priv->card_8192 = NIC_8192SE; break; default: diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_pm.c b/drivers/staging/rtl8192e/rtl8192e/rtl_pm.c index 5575186caebddd1d166765762c62914bce048179..82b45c61ac7558cfe8ee1468b4ed3fa28bee6690 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_pm.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_pm.c @@ -32,7 +32,7 @@ int rtl92e_suspend(struct device *dev_d) netif_device_detach(dev); if (!priv->rtllib->bSupportRemoteWakeUp) { - rtl92e_set_rf_state(dev, eRfOff, RF_CHANGE_BY_INIT); + rtl92e_set_rf_state(dev, rf_off, RF_CHANGE_BY_INIT); ulRegRead = rtl92e_readl(dev, CPU_GEN); ulRegRead |= CPU_GEN_SYSTEM_RESET; rtl92e_writel(dev, CPU_GEN, ulRegRead); @@ -83,10 +83,9 @@ int rtl92e_resume(struct device *dev_d) dev->netdev_ops->ndo_open(dev); if (!priv->rtllib->bSupportRemoteWakeUp) - rtl92e_set_rf_state(dev, eRfOn, RF_CHANGE_BY_INIT); + rtl92e_set_rf_state(dev, rf_on, RF_CHANGE_BY_INIT); out: - RT_TRACE(COMP_POWER, "<================r8192E resume call.\n"); return 0; } diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c index c5e89eb403420983df23019af71c2c7c5afbb79e..8c00b111ddb2be32a366cb5f63797c699f156214 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_ps.c @@ -21,16 +21,12 @@ static void _rtl92e_hw_sleep(struct net_device *dev) unsigned long flags = 0; spin_lock_irqsave(&priv->rf_ps_lock, flags); - if (priv->RFChangeInProgress) { + if (priv->rf_change_in_progress) { spin_unlock_irqrestore(&priv->rf_ps_lock, flags); - RT_TRACE(COMP_DBG, - "%s(): RF Change in progress!\n", __func__); return; } spin_unlock_irqrestore(&priv->rf_ps_lock, flags); - RT_TRACE(COMP_DBG, "%s()============>come to sleep down\n", __func__); - - rtl92e_set_rf_state(dev, eRfSleep, RF_CHANGE_BY_PS); + rtl92e_set_rf_state(dev, rf_sleep, RF_CHANGE_BY_PS); } void rtl92e_hw_sleep_wq(void *data) @@ -48,17 +44,14 @@ void rtl92e_hw_wakeup(struct net_device *dev) unsigned long flags = 0; spin_lock_irqsave(&priv->rf_ps_lock, flags); - if (priv->RFChangeInProgress) { + if (priv->rf_change_in_progress) { spin_unlock_irqrestore(&priv->rf_ps_lock, flags); - RT_TRACE(COMP_DBG, - "%s(): RF Change in progress!\n", __func__); schedule_delayed_work(&priv->rtllib->hw_wakeup_wq, msecs_to_jiffies(10)); return; } spin_unlock_irqrestore(&priv->rf_ps_lock, flags); - RT_TRACE(COMP_PS, "%s()============>come to wake up\n", __func__); - rtl92e_set_rf_state(dev, eRfOn, RF_CHANGE_BY_PS); + rtl92e_set_rf_state(dev, rf_on, RF_CHANGE_BY_PS); } void rtl92e_hw_wakeup_wq(void *data) @@ -110,15 +103,10 @@ static void _rtl92e_ps_update_rf_state(struct net_device *dev) struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) &(priv->rtllib->PowerSaveControl); - RT_TRACE(COMP_PS, "%s() --------->\n", __func__); pPSC->bSwRfProcessing = true; - - RT_TRACE(COMP_PS, "%s(): Set RF to %s.\n", __func__, - pPSC->eInactivePowerState == eRfOff ? "OFF" : "ON"); rtl92e_set_rf_state(dev, pPSC->eInactivePowerState, RF_CHANGE_BY_IPS); pPSC->bSwRfProcessing = false; - RT_TRACE(COMP_PS, "%s() <---------\n", __func__); } void rtl92e_ips_enter(struct net_device *dev) @@ -126,15 +114,14 @@ void rtl92e_ips_enter(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) &(priv->rtllib->PowerSaveControl); - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; if (pPSC->bInactivePs) { - rtState = priv->rtllib->eRFPowerState; - if (rtState == eRfOn && !pPSC->bSwRfProcessing && + rt_state = priv->rtllib->rf_power_state; + if (rt_state == rf_on && !pPSC->bSwRfProcessing && (priv->rtllib->state != RTLLIB_LINKED) && (priv->rtllib->iw_mode != IW_MODE_MASTER)) { - RT_TRACE(COMP_PS, "%s(): Turn off RF.\n", __func__); - pPSC->eInactivePowerState = eRfOff; + pPSC->eInactivePowerState = rf_off; priv->isRFOff = true; priv->bInPowerSaveMode = true; _rtl92e_ps_update_rf_state(dev); @@ -147,14 +134,13 @@ void rtl92e_ips_leave(struct net_device *dev) struct r8192_priv *priv = rtllib_priv(dev); struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) &(priv->rtllib->PowerSaveControl); - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; if (pPSC->bInactivePs) { - rtState = priv->rtllib->eRFPowerState; - if (rtState != eRfOn && !pPSC->bSwRfProcessing && - priv->rtllib->RfOffReason <= RF_CHANGE_BY_IPS) { - RT_TRACE(COMP_PS, "%s(): Turn on RF.\n", __func__); - pPSC->eInactivePowerState = eRfOn; + rt_state = priv->rtllib->rf_power_state; + if (rt_state != rf_on && !pPSC->bSwRfProcessing && + priv->rtllib->rf_off_reason <= RF_CHANGE_BY_IPS) { + pPSC->eInactivePowerState = rf_on; priv->bInPowerSaveMode = false; _rtl92e_ps_update_rf_state(dev); } @@ -176,13 +162,13 @@ void rtl92e_ips_leave_wq(void *data) void rtl92e_rtllib_ips_leave_wq(struct net_device *dev) { struct r8192_priv *priv = (struct r8192_priv *)rtllib_priv(dev); - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; - rtState = priv->rtllib->eRFPowerState; + rt_state = priv->rtllib->rf_power_state; if (priv->rtllib->PowerSaveControl.bInactivePs) { - if (rtState == eRfOff) { - if (priv->rtllib->RfOffReason > RF_CHANGE_BY_IPS) { + if (rt_state == rf_off) { + if (priv->rtllib->rf_off_reason > RF_CHANGE_BY_IPS) { netdev_warn(dev, "%s(): RF is OFF.\n", __func__); return; @@ -210,7 +196,6 @@ static bool _rtl92e_ps_set_mode(struct net_device *dev, u8 rtPsMode) if (priv->rtllib->iw_mode == IW_MODE_ADHOC) return false; - RT_TRACE(COMP_LPS, "%s(): set ieee->ps = %x\n", __func__, rtPsMode); if (!priv->ps_force) priv->rtllib->ps = rtPsMode; if (priv->rtllib->sta_sleep != LPS_IS_WAKE && @@ -221,8 +206,6 @@ static bool _rtl92e_ps_set_mode(struct net_device *dev, u8 rtPsMode) priv->rtllib->sta_sleep = LPS_IS_WAKE; spin_lock_irqsave(&(priv->rtllib->mgmt_tx_lock), flags); - RT_TRACE(COMP_DBG, - "LPS leave: notify AP we are awaked ++++++++++ SendNullFunctionData\n"); rtllib_sta_ps_send_null_frame(priv->rtllib, 0); spin_unlock_irqrestore(&(priv->rtllib->mgmt_tx_lock), flags); } @@ -236,12 +219,6 @@ void rtl92e_leisure_ps_enter(struct net_device *dev) struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) &(priv->rtllib->PowerSaveControl); - RT_TRACE(COMP_PS, "%s()...\n", __func__); - RT_TRACE(COMP_PS, - "pPSC->bLeisurePs = %d, ieee->ps = %d,pPSC->LpsIdleCount is %d,RT_CHECK_FOR_HANG_PERIOD is %d\n", - pPSC->bLeisurePs, priv->rtllib->ps, pPSC->LpsIdleCount, - RT_CHECK_FOR_HANG_PERIOD); - if (!((priv->rtllib->iw_mode == IW_MODE_INFRA) && (priv->rtllib->state == RTLLIB_LINKED)) || (priv->rtllib->iw_mode == IW_MODE_ADHOC) || @@ -252,10 +229,6 @@ void rtl92e_leisure_ps_enter(struct net_device *dev) if (pPSC->LpsIdleCount >= RT_CHECK_FOR_HANG_PERIOD) { if (priv->rtllib->ps == RTLLIB_PS_DISABLED) { - - RT_TRACE(COMP_LPS, - "%s(): Enter 802.11 power save mode...\n", __func__); - if (!pPSC->bFwCtrlLPS) { if (priv->rtllib->SetFwCmdHandler) priv->rtllib->SetFwCmdHandler( @@ -275,15 +248,8 @@ void rtl92e_leisure_ps_leave(struct net_device *dev) struct rt_pwr_save_ctrl *pPSC = (struct rt_pwr_save_ctrl *) &(priv->rtllib->PowerSaveControl); - - RT_TRACE(COMP_PS, "%s()...\n", __func__); - RT_TRACE(COMP_PS, "pPSC->bLeisurePs = %d, ieee->ps = %d\n", - pPSC->bLeisurePs, priv->rtllib->ps); - if (pPSC->bLeisurePs) { if (priv->rtllib->ps != RTLLIB_PS_DISABLED) { - RT_TRACE(COMP_LPS, - "%s(): Busy Traffic , Leave 802.11 power save..\n", __func__); _rtl92e_ps_set_mode(dev, RTLLIB_PS_DISABLED); if (!pPSC->bFwCtrlLPS) { diff --git a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c index 407effde5e71aff2d31107c6988241a53e4fec3d..4920cb49e3811799cc933da7bce1d0be3b159ee5 100644 --- a/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c +++ b/drivers/staging/rtl8192e/rtl8192e/rtl_wx.c @@ -52,7 +52,7 @@ static int _rtl92e_wx_set_rate(struct net_device *dev, int ret; struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -71,7 +71,7 @@ static int _rtl92e_wx_set_rts(struct net_device *dev, int ret; struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -99,7 +99,7 @@ static int _rtl92e_wx_set_power(struct net_device *dev, int ret; struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) { + if (priv->hw_radio_off) { netdev_warn(dev, "%s(): Can't set Power: Radio is Off.\n", __func__); return 0; @@ -129,7 +129,7 @@ static int _rtl92e_wx_set_rawtx(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); int ret; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -149,8 +149,6 @@ static int _rtl92e_wx_force_reset(struct net_device *dev, mutex_lock(&priv->wx_mutex); - RT_TRACE(COMP_DBG, "%s(): force reset ! extra is %d\n", - __func__, *extra); priv->force_reset = *extra; mutex_unlock(&priv->wx_mutex); return 0; @@ -167,8 +165,6 @@ static int _rtl92e_wx_adapter_power_status(struct net_device *dev, mutex_lock(&priv->wx_mutex); - RT_TRACE(COMP_POWER, "%s(): %s\n", __func__, (*extra == 6) ? - "DC power" : "AC power"); if (*extra || priv->force_lps) { priv->ps_force = false; pPSC->bLeisurePs = true; @@ -228,7 +224,7 @@ static int _rtl92e_wx_set_debug(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); u8 c = *extra; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; netdev_info(dev, "=====>%s(), *extra:%x, debugflag:%x\n", __func__, @@ -247,18 +243,18 @@ static int _rtl92e_wx_set_mode(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = netdev_priv_rsl(dev); - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; int ret; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; - rtState = priv->rtllib->eRFPowerState; + rt_state = priv->rtllib->rf_power_state; mutex_lock(&priv->wx_mutex); if (wrqu->mode == IW_MODE_ADHOC || wrqu->mode == IW_MODE_MONITOR || ieee->bNetPromiscuousMode) { if (priv->rtllib->PowerSaveControl.bInactivePs) { - if (rtState == eRfOff) { - if (priv->rtllib->RfOffReason > + if (rt_state == rf_off) { + if (priv->rtllib->rf_off_reason > RF_CHANGE_BY_IPS) { netdev_warn(dev, "%s(): RF is OFF.\n", __func__); @@ -379,7 +375,7 @@ static int _rtl92e_wx_set_scan(struct net_device *dev, { struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = priv->rtllib; - enum rt_rf_power_state rtState; + enum rt_rf_power_state rt_state; int ret; if (!(ieee->softmac_features & IEEE_SOFTMAC_SCAN)) { @@ -391,12 +387,12 @@ static int _rtl92e_wx_set_scan(struct net_device *dev, return 0; } - if (priv->bHwRadioOff) { + if (priv->hw_radio_off) { netdev_info(dev, "================>%s(): hwradio off\n", __func__); return 0; } - rtState = priv->rtllib->eRFPowerState; + rt_state = priv->rtllib->rf_power_state; if (!priv->up) return -ENETDOWN; if (priv->rtllib->LinkDetectInfo.bBusyTraffic == true) @@ -419,17 +415,14 @@ static int _rtl92e_wx_set_scan(struct net_device *dev, if (priv->rtllib->state != RTLLIB_LINKED) { if (priv->rtllib->PowerSaveControl.bInactivePs) { - if (rtState == eRfOff) { - if (priv->rtllib->RfOffReason > + if (rt_state == rf_off) { + if (priv->rtllib->rf_off_reason > RF_CHANGE_BY_IPS) { netdev_warn(dev, "%s(): RF is OFF.\n", __func__); mutex_unlock(&priv->wx_mutex); return -1; } - RT_TRACE(COMP_PS, - "=========>%s(): rtl92e_ips_leave\n", - __func__); mutex_lock(&priv->rtllib->ips_mutex); rtl92e_ips_leave(dev); mutex_unlock(&priv->rtllib->ips_mutex); @@ -440,7 +433,7 @@ static int _rtl92e_wx_set_scan(struct net_device *dev, priv->rtllib->LedControlHandler(dev, LED_CTL_SITE_SURVEY); - if (priv->rtllib->eRFPowerState != eRfOff) { + if (priv->rtllib->rf_power_state != rf_off) { priv->rtllib->actscanning = true; if (ieee->ScanOperationBackupHandler) @@ -473,7 +466,7 @@ static int _rtl92e_wx_get_scan(struct net_device *dev, if (!priv->up) return -ENETDOWN; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -492,9 +485,9 @@ static int _rtl92e_wx_set_essid(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); int ret; - if (priv->bHwRadioOff) { + if (priv->hw_radio_off) { netdev_info(dev, - "=========>%s():hw radio off,or Rf state is eRfOff, return\n", + "=========>%s():hw radio off,or Rf state is rf_off, return\n", __func__); return 0; } @@ -560,7 +553,7 @@ static int _rtl92e_wx_set_freq(struct net_device *dev, int ret; struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -586,7 +579,7 @@ static int _rtl92e_wx_set_frag(struct net_device *dev, { struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; if (wrqu->frag.disabled) @@ -622,7 +615,7 @@ static int _rtl92e_wx_set_wap(struct net_device *dev, int ret; struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -669,7 +662,7 @@ static int _rtl92e_wx_set_enc(struct net_device *dev, {0x00, 0x00, 0x00, 0x00, 0x00, 0x03} }; int i; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; if (!priv->up) @@ -681,7 +674,6 @@ static int _rtl92e_wx_set_enc(struct net_device *dev, mutex_unlock(&priv->rtllib->ips_mutex); mutex_lock(&priv->wx_mutex); - RT_TRACE(COMP_SEC, "Setting SW wep key"); ret = rtllib_wx_set_encode(priv->rtllib, info, wrqu, key); mutex_unlock(&priv->wx_mutex); @@ -754,7 +746,7 @@ static int _rtl92e_wx_set_scan_type(struct net_device *dev, int *parms = (int *)p; int mode = parms[0]; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; priv->rtllib->active_scan = mode; @@ -770,7 +762,7 @@ static int _rtl92e_wx_set_retry(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); int err = 0; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -843,7 +835,7 @@ static int _rtl92e_wx_set_sens(struct net_device *dev, short err = 0; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -870,7 +862,7 @@ static int _rtl92e_wx_set_encode_ext(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); struct rtllib_device *ieee = priv->rtllib; - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -950,7 +942,7 @@ static int _rtl92e_wx_set_auth(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -967,7 +959,7 @@ static int _rtl92e_wx_set_mlme(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); @@ -984,7 +976,7 @@ static int _rtl92e_wx_set_gen_ie(struct net_device *dev, struct r8192_priv *priv = rtllib_priv(dev); - if (priv->bHwRadioOff) + if (priv->hw_radio_off) return 0; mutex_lock(&priv->wx_mutex); diff --git a/drivers/staging/rtl8192e/rtl819x_BAProc.c b/drivers/staging/rtl8192e/rtl819x_BAProc.c index 7d04966afdd92ca967cac56294acbb0269f821a2..19d13b3fcecf72d78521d122a24cbaa8cc2441d3 100644 --- a/drivers/staging/rtl8192e/rtl819x_BAProc.c +++ b/drivers/staging/rtl8192e/rtl819x_BAProc.c @@ -100,8 +100,6 @@ static struct sk_buff *rtllib_ADDBA(struct rtllib_device *ieee, u8 *Dst, *tag++ = pBA->dialog_token; if (type == ACT_ADDBARSP) { - RT_TRACE(COMP_DBG, "====>to send ADDBARSP\n"); - put_unaligned_le16(StatusCode, tag); tag += 2; } @@ -183,7 +181,6 @@ static void rtllib_send_ADDBAReq(struct rtllib_device *ieee, u8 *dst, skb = rtllib_ADDBA(ieee, dst, pBA, 0, ACT_ADDBAREQ); if (skb) { - RT_TRACE(COMP_DBG, "====>to send ADDBAREQ!!!!!\n"); softmac_mgmt_xmit(skb, ieee); } else { netdev_dbg(ieee->dev, "Failed to generate ADDBAReq packet.\n"); @@ -247,10 +244,9 @@ int rtllib_rx_ADDBAReq(struct rtllib_device *ieee, struct sk_buff *skb) pBaTimeoutVal = (u16 *)(tag + 5); pBaStartSeqCtrl = (union sequence_control *)(req + 7); - RT_TRACE(COMP_DBG, "====>rx ADDBAREQ from : %pM\n", dst); if (!ieee->current_network.qos_data.active || !ieee->pHTInfo->bCurrentHTSupport || - (ieee->pHTInfo->IOTAction & HT_IOT_ACT_REJECT_ADDBA_REQ)) { + (ieee->pHTInfo->iot_action & HT_IOT_ACT_REJECT_ADDBA_REQ)) { rc = ADDBA_STATUS_REFUSED; netdev_warn(ieee->dev, "Failed to reply on ADDBA_REQ as some capability is not ready(%d, %d)\n", @@ -282,7 +278,7 @@ int rtllib_rx_ADDBAReq(struct rtllib_device *ieee, struct sk_buff *skb) pBA->ba_start_seq_ctrl = *pBaStartSeqCtrl; if (ieee->GetHalfNmodeSupportByAPsHandler(ieee->dev) || - (ieee->pHTInfo->IOTAction & HT_IOT_ACT_ALLOW_PEER_AGG_ONE_PKT)) + (ieee->pHTInfo->iot_action & HT_IOT_ACT_ALLOW_PEER_AGG_ONE_PKT)) pBA->ba_param_set.field.buffer_size = 1; else pBA->ba_param_set.field.buffer_size = 32; @@ -330,7 +326,6 @@ int rtllib_rx_ADDBARsp(struct rtllib_device *ieee, struct sk_buff *skb) pBaParamSet = (union ba_param_set *)(tag + 5); pBaTimeoutVal = (u16 *)(tag + 7); - RT_TRACE(COMP_DBG, "====>rx ADDBARSP from : %pM\n", dst); if (!ieee->current_network.qos_data.active || !ieee->pHTInfo->bCurrentHTSupport || !ieee->pHTInfo->bCurrentAMPDUEnable) { diff --git a/drivers/staging/rtl8192e/rtl819x_HT.h b/drivers/staging/rtl8192e/rtl819x_HT.h index ce13b41074a7b1c44907d4d4b94b458a5bc4c1cd..76bc9c5a6d83d0e3a2ffa02726373cba926b3e30 100644 --- a/drivers/staging/rtl8192e/rtl819x_HT.h +++ b/drivers/staging/rtl8192e/rtl819x_HT.h @@ -131,51 +131,40 @@ struct rt_hi_throughput { u8 AMPDU_Factor; u8 CurrentAMPDUFactor; u8 MPDU_Density; - u8 CurrentMPDUDensity; + u8 current_mpdu_density; enum ht_aggre_mode ForcedAMPDUMode; - u8 ForcedAMPDUFactor; - u8 ForcedMPDUDensity; + u8 forced_ampdu_factor; + u8 forced_mpdu_density; enum ht_aggre_mode ForcedAMSDUMode; - u16 ForcedAMSDUMaxSize; + u8 forced_short_gi; - u8 bForcedShortGI; + u8 current_op_mode; - u8 CurrentOpMode; - - u8 SelfMimoPs; - u8 PeerMimoPs; + u8 self_mimo_ps; + u8 peer_mimo_ps; enum ht_extchnl_offset CurSTAExtChnlOffset; - u8 bCurTxBW40MHz; - u8 PeerBandwidth; - - u8 bSwBwInProgress; - u8 SwBwStep; - - u8 bRegRT2RTAggregation; + u8 cur_tx_bw40mhz; + u8 sw_bw_in_progress; + u8 reg_rt2rt_aggregation; u8 RT2RT_HT_Mode; - u8 bCurrentRT2RTAggregation; - u8 bCurrentRT2RTLongSlotTime; - u8 szRT2RTAggBuffer[10]; - - u8 bRegRxReorderEnable; - u8 bCurRxReorderEnable; - u8 RxReorderWinSize; - u8 RxReorderPendingTime; - u16 RxReorderDropCounter; - - u8 bIsPeerBcm; - + u8 current_rt2rt_aggregation; + u8 current_rt2rt_long_slot_time; + u8 sz_rt2rt_agg_buf[10]; + + u8 reg_rx_reorder_enable; + u8 cur_rx_reorder_enable; + u8 rx_reorder_win_size; + u8 rx_reorder_pending_time; + u16 rx_reorder_drop_counter; u8 IOTPeer; - u32 IOTAction; - u8 IOTRaFunc; + u32 iot_action; + u8 iot_ra_func; u8 bWAIotBroadcom; u8 WAIotTH; - - u8 bAcceptAddbaReq; } __packed; struct bss_ht { diff --git a/drivers/staging/rtl8192e/rtl819x_HTProc.c b/drivers/staging/rtl8192e/rtl819x_HTProc.c index 3b8efaf9b88c9d372b8ac5c1c3657f7d5cd5b77d..ef3dca51cf99b3af2fea79c5ab454ddbc3b889b4 100644 --- a/drivers/staging/rtl8192e/rtl819x_HTProc.c +++ b/drivers/staging/rtl8192e/rtl819x_HTProc.c @@ -70,9 +70,6 @@ static u8 LINKSYS_MARVELL_4400N[3] = {0x00, 0x14, 0xa4}; void HTUpdateDefaultSetting(struct rtllib_device *ieee) { struct rt_hi_throughput *pHTInfo = ieee->pHTInfo; - - pHTInfo->bAcceptAddbaReq = 1; - pHTInfo->bRegShortGI20MHz = 1; pHTInfo->bRegShortGI40MHz = 1; @@ -90,19 +87,19 @@ void HTUpdateDefaultSetting(struct rtllib_device *ieee) pHTInfo->AMPDU_Factor = 2; pHTInfo->MPDU_Density = 0; - pHTInfo->SelfMimoPs = 3; - if (pHTInfo->SelfMimoPs == 2) - pHTInfo->SelfMimoPs = 3; + pHTInfo->self_mimo_ps = 3; + if (pHTInfo->self_mimo_ps == 2) + pHTInfo->self_mimo_ps = 3; ieee->bTxDisableRateFallBack = 0; ieee->bTxUseDriverAssingedRate = 0; ieee->bTxEnableFwCalcDur = 1; - pHTInfo->bRegRT2RTAggregation = 1; + pHTInfo->reg_rt2rt_aggregation = 1; - pHTInfo->bRegRxReorderEnable = 1; - pHTInfo->RxReorderWinSize = 64; - pHTInfo->RxReorderPendingTime = 30; + pHTInfo->reg_rx_reorder_enable = 1; + pHTInfo->rx_reorder_win_size = 64; + pHTInfo->rx_reorder_pending_time = 30; } static u16 HTMcsToDataRate(struct rtllib_device *ieee, u8 nMcsRate) @@ -254,20 +251,20 @@ static void HTIOTActDetermineRaFunc(struct rtllib_device *ieee, bool bPeerRx2ss) { struct rt_hi_throughput *pHTInfo = ieee->pHTInfo; - pHTInfo->IOTRaFunc &= HT_IOT_RAFUNC_DISABLE_ALL; + pHTInfo->iot_ra_func &= HT_IOT_RAFUNC_DISABLE_ALL; if (pHTInfo->IOTPeer == HT_IOT_PEER_RALINK && !bPeerRx2ss) - pHTInfo->IOTRaFunc |= HT_IOT_RAFUNC_PEER_1R; + pHTInfo->iot_ra_func |= HT_IOT_RAFUNC_PEER_1R; - if (pHTInfo->IOTAction & HT_IOT_ACT_AMSDU_ENABLE) - pHTInfo->IOTRaFunc |= HT_IOT_RAFUNC_TX_AMSDU; + if (pHTInfo->iot_action & HT_IOT_ACT_AMSDU_ENABLE) + pHTInfo->iot_ra_func |= HT_IOT_RAFUNC_TX_AMSDU; } void HTResetIOTSetting(struct rt_hi_throughput *pHTInfo) { - pHTInfo->IOTAction = 0; + pHTInfo->iot_action = 0; pHTInfo->IOTPeer = HT_IOT_PEER_UNKNOWN; - pHTInfo->IOTRaFunc = 0; + pHTInfo->iot_ra_func = 0; } void HTConstructCapabilityElement(struct rtllib_device *ieee, u8 *posHTCap, @@ -300,7 +297,7 @@ void HTConstructCapabilityElement(struct rtllib_device *ieee, u8 *posHTCap, else pCapELE->ChlWidth = (pHT->bRegBW40MHz ? 1 : 0); - pCapELE->MimoPwrSave = pHT->SelfMimoPs; + pCapELE->MimoPwrSave = pHT->self_mimo_ps; pCapELE->GreenField = 0; pCapELE->ShortGI20Mhz = 1; pCapELE->ShortGI40Mhz = 1; @@ -332,16 +329,16 @@ void HTConstructCapabilityElement(struct rtllib_device *ieee, u8 *posHTCap, pCapELE->ASCap = 0; if (bAssoc) { - if (pHT->IOTAction & HT_IOT_ACT_DISABLE_MCS15) + if (pHT->iot_action & HT_IOT_ACT_DISABLE_MCS15) pCapELE->MCS[1] &= 0x7f; - if (pHT->IOTAction & HT_IOT_ACT_DISABLE_MCS14) + if (pHT->iot_action & HT_IOT_ACT_DISABLE_MCS14) pCapELE->MCS[1] &= 0xbf; - if (pHT->IOTAction & HT_IOT_ACT_DISABLE_ALL_2SS) + if (pHT->iot_action & HT_IOT_ACT_DISABLE_ALL_2SS) pCapELE->MCS[1] &= 0x00; - if (pHT->IOTAction & HT_IOT_ACT_DISABLE_RX_40MHZ_SHORT_GI) + if (pHT->iot_action & HT_IOT_ACT_DISABLE_RX_40MHZ_SHORT_GI) pCapELE->ShortGI40Mhz = 0; if (ieee->GetHalfNmodeSupportByAPsHandler(ieee->dev)) { @@ -377,7 +374,7 @@ void HTConstructInfoElement(struct rtllib_device *ieee, u8 *posHTInfo, pHTInfoEle->RIFS = 0; pHTInfoEle->PSMPAccessOnly = 0; pHTInfoEle->SrvIntGranularity = 0; - pHTInfoEle->OptMode = pHT->CurrentOpMode; + pHTInfoEle->OptMode = pHT->current_op_mode; pHTInfoEle->NonGFDevPresent = 0; pHTInfoEle->DualBeacon = 0; pHTInfoEle->SecondaryBeacon = 0; @@ -506,7 +503,7 @@ static u8 HTFilterMCSRate(struct rtllib_device *ieee, u8 *pSupportMCS, } void HTSetConnectBwMode(struct rtllib_device *ieee, - enum ht_channel_width Bandwidth, + enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset); void HTOnAssocRsp(struct rtllib_device *ieee) @@ -543,7 +540,7 @@ void HTOnAssocRsp(struct rtllib_device *ieee) #endif HTSetConnectBwMode(ieee, (enum ht_channel_width)(pPeerHTCap->ChlWidth), (enum ht_extchnl_offset)(pPeerHTInfo->ExtChlOffset)); - pHTInfo->bCurTxBW40MHz = ((pPeerHTInfo->RecommemdedTxWidth == 1) ? + pHTInfo->cur_tx_bw40mhz = ((pPeerHTInfo->RecommemdedTxWidth == 1) ? true : false); pHTInfo->bCurShortGI20MHz = ((pHTInfo->bRegShortGI20MHz) ? @@ -574,7 +571,7 @@ void HTOnAssocRsp(struct rtllib_device *ieee) pHTInfo->bCurrentAMPDUEnable = false; } - if (!pHTInfo->bRegRT2RTAggregation) { + if (!pHTInfo->reg_rt2rt_aggregation) { if (pHTInfo->AMPDU_Factor > pPeerHTCap->MaxRxAMPDUFactor) pHTInfo->CurrentAMPDUFactor = pPeerHTCap->MaxRxAMPDUFactor; @@ -597,15 +594,14 @@ void HTOnAssocRsp(struct rtllib_device *ieee) } } if (pHTInfo->MPDU_Density > pPeerHTCap->MPDUDensity) - pHTInfo->CurrentMPDUDensity = pHTInfo->MPDU_Density; + pHTInfo->current_mpdu_density = pHTInfo->MPDU_Density; else - pHTInfo->CurrentMPDUDensity = pPeerHTCap->MPDUDensity; - if (pHTInfo->IOTAction & HT_IOT_ACT_TX_USE_AMSDU_8K) { + pHTInfo->current_mpdu_density = pPeerHTCap->MPDUDensity; + if (pHTInfo->iot_action & HT_IOT_ACT_TX_USE_AMSDU_8K) { pHTInfo->bCurrentAMPDUEnable = false; pHTInfo->ForcedAMSDUMode = HT_AGG_FORCE_ENABLE; - pHTInfo->ForcedAMSDUMaxSize = 7935; } - pHTInfo->bCurRxReorderEnable = pHTInfo->bRegRxReorderEnable; + pHTInfo->cur_rx_reorder_enable = pHTInfo->reg_rx_reorder_enable; if (pPeerHTCap->MCS[0] == 0) pPeerHTCap->MCS[0] = 0xff; @@ -614,8 +610,8 @@ void HTOnAssocRsp(struct rtllib_device *ieee) HTFilterMCSRate(ieee, pPeerHTCap->MCS, ieee->dot11HTOperationalRateSet); - pHTInfo->PeerMimoPs = pPeerHTCap->MimoPwrSave; - if (pHTInfo->PeerMimoPs == MIMO_PS_STATIC) + pHTInfo->peer_mimo_ps = pPeerHTCap->MimoPwrSave; + if (pHTInfo->peer_mimo_ps == MIMO_PS_STATIC) pMcsFilter = MCS_FILTER_1SS; else pMcsFilter = MCS_FILTER_ALL; @@ -623,7 +619,7 @@ void HTOnAssocRsp(struct rtllib_device *ieee) ieee->dot11HTOperationalRateSet, pMcsFilter); ieee->HTCurrentOperaRate = ieee->HTHighestOperaRate; - pHTInfo->CurrentOpMode = pPeerHTInfo->OptMode; + pHTInfo->current_op_mode = pPeerHTInfo->OptMode; } void HTInitializeHTInfo(struct rtllib_device *ieee) @@ -633,17 +629,17 @@ void HTInitializeHTInfo(struct rtllib_device *ieee) pHTInfo->bCurrentHTSupport = false; pHTInfo->bCurBW40MHz = false; - pHTInfo->bCurTxBW40MHz = false; + pHTInfo->cur_tx_bw40mhz = false; pHTInfo->bCurShortGI20MHz = false; pHTInfo->bCurShortGI40MHz = false; - pHTInfo->bForcedShortGI = false; + pHTInfo->forced_short_gi = false; pHTInfo->bCurSuppCCK = true; pHTInfo->bCurrent_AMSDU_Support = false; pHTInfo->nCurrent_AMSDU_MaxSize = pHTInfo->nAMSDU_MaxSize; - pHTInfo->CurrentMPDUDensity = pHTInfo->MPDU_Density; + pHTInfo->current_mpdu_density = pHTInfo->MPDU_Density; pHTInfo->CurrentAMPDUFactor = pHTInfo->AMPDU_Factor; memset((void *)(&(pHTInfo->SelfHTCap)), 0, @@ -655,17 +651,17 @@ void HTInitializeHTInfo(struct rtllib_device *ieee) memset((void *)(&(pHTInfo->PeerHTInfoBuf)), 0, sizeof(pHTInfo->PeerHTInfoBuf)); - pHTInfo->bSwBwInProgress = false; + pHTInfo->sw_bw_in_progress = false; pHTInfo->ePeerHTSpecVer = HT_SPEC_VER_IEEE; - pHTInfo->bCurrentRT2RTAggregation = false; - pHTInfo->bCurrentRT2RTLongSlotTime = false; + pHTInfo->current_rt2rt_aggregation = false; + pHTInfo->current_rt2rt_long_slot_time = false; pHTInfo->RT2RT_HT_Mode = (enum rt_ht_capability)0; pHTInfo->IOTPeer = 0; - pHTInfo->IOTAction = 0; - pHTInfo->IOTRaFunc = 0; + pHTInfo->iot_action = 0; + pHTInfo->iot_ra_func = 0; { u8 *RegHTSuppRateSets = &(ieee->RegHTSuppRateSet[0]); @@ -717,51 +713,51 @@ void HTResetSelfAndSavePeerSetting(struct rtllib_device *ieee, pNetwork->bssht.bd_ht_info_buf, pNetwork->bssht.bd_ht_info_len); - if (pHTInfo->bRegRT2RTAggregation) { - pHTInfo->bCurrentRT2RTAggregation = + if (pHTInfo->reg_rt2rt_aggregation) { + pHTInfo->current_rt2rt_aggregation = pNetwork->bssht.bd_rt2rt_aggregation; - pHTInfo->bCurrentRT2RTLongSlotTime = + pHTInfo->current_rt2rt_long_slot_time = pNetwork->bssht.bd_rt2rt_long_slot_time; pHTInfo->RT2RT_HT_Mode = pNetwork->bssht.rt2rt_ht_mode; } else { - pHTInfo->bCurrentRT2RTAggregation = false; - pHTInfo->bCurrentRT2RTLongSlotTime = false; + pHTInfo->current_rt2rt_aggregation = false; + pHTInfo->current_rt2rt_long_slot_time = false; pHTInfo->RT2RT_HT_Mode = (enum rt_ht_capability)0; } HTIOTPeerDetermine(ieee); - pHTInfo->IOTAction = 0; + pHTInfo->iot_action = 0; bIOTAction = HTIOTActIsDisableMCS14(ieee, pNetwork->bssid); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_DISABLE_MCS14; + pHTInfo->iot_action |= HT_IOT_ACT_DISABLE_MCS14; bIOTAction = HTIOTActIsDisableMCS15(ieee); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_DISABLE_MCS15; + pHTInfo->iot_action |= HT_IOT_ACT_DISABLE_MCS15; bIOTAction = HTIOTActIsDisableMCSTwoSpatialStream(ieee); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_DISABLE_ALL_2SS; + pHTInfo->iot_action |= HT_IOT_ACT_DISABLE_ALL_2SS; bIOTAction = HTIOTActIsDisableEDCATurbo(ieee, pNetwork->bssid); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_DISABLE_EDCA_TURBO; + pHTInfo->iot_action |= HT_IOT_ACT_DISABLE_EDCA_TURBO; bIOTAction = HTIOTActIsMgntUseCCK6M(ieee, pNetwork); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_MGNT_USE_CCK_6M; + pHTInfo->iot_action |= HT_IOT_ACT_MGNT_USE_CCK_6M; bIOTAction = HTIOTActIsCCDFsync(ieee); if (bIOTAction) - pHTInfo->IOTAction |= HT_IOT_ACT_CDD_FSYNC; + pHTInfo->iot_action |= HT_IOT_ACT_CDD_FSYNC; } else { pHTInfo->bCurrentHTSupport = false; - pHTInfo->bCurrentRT2RTAggregation = false; - pHTInfo->bCurrentRT2RTLongSlotTime = false; + pHTInfo->current_rt2rt_aggregation = false; + pHTInfo->current_rt2rt_long_slot_time = false; pHTInfo->RT2RT_HT_Mode = (enum rt_ht_capability)0; - pHTInfo->IOTAction = 0; - pHTInfo->IOTRaFunc = 0; + pHTInfo->iot_action = 0; + pHTInfo->iot_ra_func = 0; } } @@ -774,7 +770,7 @@ void HT_update_self_and_peer_setting(struct rtllib_device *ieee, if (pHTInfo->bCurrentHTSupport) { if (pNetwork->bssht.bd_ht_info_len != 0) - pHTInfo->CurrentOpMode = pPeerHTInfo->OptMode; + pHTInfo->current_op_mode = pPeerHTInfo->OptMode; } } EXPORT_SYMBOL(HT_update_self_and_peer_setting); @@ -801,7 +797,7 @@ void HTUseDefaultSetting(struct rtllib_device *ieee) pHTInfo->bCurrentAMPDUEnable = pHTInfo->bAMPDUEnable; pHTInfo->CurrentAMPDUFactor = pHTInfo->AMPDU_Factor; - pHTInfo->CurrentMPDUDensity = pHTInfo->CurrentMPDUDensity; + pHTInfo->current_mpdu_density = pHTInfo->current_mpdu_density; HTFilterMCSRate(ieee, ieee->Regdot11TxHTOperationalRateSet, ieee->dot11HTOperationalRateSet); @@ -850,11 +846,11 @@ static void HTSetConnectBwModeCallback(struct rtllib_device *ieee) HT_EXTCHNL_OFFSET_NO_EXT); } - pHTInfo->bSwBwInProgress = false; + pHTInfo->sw_bw_in_progress = false; } void HTSetConnectBwMode(struct rtllib_device *ieee, - enum ht_channel_width Bandwidth, + enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset) { struct rt_hi_throughput *pHTInfo = ieee->pHTInfo; @@ -863,13 +859,13 @@ void HTSetConnectBwMode(struct rtllib_device *ieee, return; if (ieee->GetHalfNmodeSupportByAPsHandler(ieee->dev)) - Bandwidth = HT_CHANNEL_WIDTH_20; + bandwidth = HT_CHANNEL_WIDTH_20; - if (pHTInfo->bSwBwInProgress) { - pr_info("%s: bSwBwInProgress!!\n", __func__); + if (pHTInfo->sw_bw_in_progress) { + pr_info("%s: sw_bw_in_progress!!\n", __func__); return; } - if (Bandwidth == HT_CHANNEL_WIDTH_20_40) { + if (bandwidth == HT_CHANNEL_WIDTH_20_40) { if (ieee->current_network.channel < 2 && Offset == HT_EXTCHNL_OFFSET_LOWER) Offset = HT_EXTCHNL_OFFSET_NO_EXT; @@ -889,7 +885,7 @@ void HTSetConnectBwMode(struct rtllib_device *ieee, netdev_dbg(ieee->dev, "%s():pHTInfo->bCurBW40MHz:%x\n", __func__, pHTInfo->bCurBW40MHz); - pHTInfo->bSwBwInProgress = true; + pHTInfo->sw_bw_in_progress = true; HTSetConnectBwModeCallback(ieee); } diff --git a/drivers/staging/rtl8192e/rtl819x_TSProc.c b/drivers/staging/rtl8192e/rtl819x_TSProc.c index 34b00a76b6bd7c116ca435ec3555a0da5a512cd5..05c7e822f37278eb66656ebdede07badd35a9011 100644 --- a/drivers/staging/rtl8192e/rtl819x_TSProc.c +++ b/drivers/staging/rtl8192e/rtl819x_TSProc.c @@ -83,7 +83,7 @@ static void RxPktPendingTimeout(struct timer_list *t) if (bPktInBuf && (pRxTs->rx_timeout_indicate_seq == 0xffff)) { pRxTs->rx_timeout_indicate_seq = pRxTs->rx_indicate_seq; mod_timer(&pRxTs->rx_pkt_pending_timer, jiffies + - msecs_to_jiffies(ieee->pHTInfo->RxReorderPendingTime) + msecs_to_jiffies(ieee->pHTInfo->rx_reorder_pending_time) ); } spin_unlock_irqrestore(&(ieee->reorder_spinlock), flags); diff --git a/drivers/staging/rtl8192e/rtllib.h b/drivers/staging/rtl8192e/rtllib.h index 0ecd81a818665c0ea301a5e4e57b63a167a5a9de..3c72ed2a30a410d1edd5010749634d896ce50bf1 100644 --- a/drivers/staging/rtl8192e/rtllib.h +++ b/drivers/staging/rtl8192e/rtllib.h @@ -1244,9 +1244,9 @@ enum ips_callback_function { }; enum rt_rf_power_state { - eRfOn, - eRfSleep, - eRfOff + rf_on, + rf_sleep, + rf_off }; struct rt_pwr_save_ctrl { @@ -1434,8 +1434,8 @@ struct rtllib_device { bool FirstIe_InScan; bool be_scan_inprogress; bool beinretry; - enum rt_rf_power_state eRFPowerState; - RT_RF_CHANGE_SOURCE RfOffReason; + enum rt_rf_power_state rf_power_state; + RT_RF_CHANGE_SOURCE rf_off_reason; bool is_set_key; bool wx_set_enc; struct rt_hi_throughput *pHTInfo; @@ -1765,7 +1765,7 @@ struct rtllib_device { /* check whether Tx hw resource available */ short (*check_nic_enough_desc)(struct net_device *dev, int queue_index); void (*SetBWModeHandler)(struct net_device *dev, - enum ht_channel_width Bandwidth, + enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset); bool (*GetNmodeSupportBySecCfg)(struct net_device *dev); void (*SetWirelessMode)(struct net_device *dev, u8 wireless_mode); @@ -1938,7 +1938,7 @@ int rtllib_encrypt_fragment( struct sk_buff *frag, int hdr_len); -int rtllib_xmit(struct sk_buff *skb, struct net_device *dev); +netdev_tx_t rtllib_xmit(struct sk_buff *skb, struct net_device *dev); void rtllib_txb_free(struct rtllib_txb *txb); /* rtllib_rx.c */ @@ -2073,7 +2073,7 @@ int rtllib_wx_get_rts(struct rtllib_device *ieee, struct iw_request_info *info, #define MAX_RECEIVE_BUFFER_SIZE 9100 void HTSetConnectBwMode(struct rtllib_device *ieee, - enum ht_channel_width Bandwidth, + enum ht_channel_width bandwidth, enum ht_extchnl_offset Offset); void HTUpdateDefaultSetting(struct rtllib_device *ieee); void HTConstructCapabilityElement(struct rtllib_device *ieee, diff --git a/drivers/staging/rtl8192e/rtllib_debug.h b/drivers/staging/rtl8192e/rtllib_debug.h index e3e8302945eb5333e482ee66bc3ec146cae55c03..f6b23defe22577c3ea880a68dc2242dde5005638 100644 --- a/drivers/staging/rtl8192e/rtllib_debug.h +++ b/drivers/staging/rtl8192e/rtllib_debug.h @@ -46,10 +46,4 @@ enum RTL_DEBUG { COMP_ERR = BIT(31) }; -#define RT_TRACE(component, x, args...) \ -do { \ - if (rt_global_debug_component & component) \ - printk(KERN_DEBUG DRV_NAME ":" x "\n", ##args);\ -} while (0) - #endif diff --git a/drivers/staging/rtl8192e/rtllib_rx.c b/drivers/staging/rtl8192e/rtllib_rx.c index abe5c153f74e88fd8663cde12429d79db2710076..46d75e925ee9b780e07fddf53f29c1c5b6e76a2e 100644 --- a/drivers/staging/rtl8192e/rtllib_rx.c +++ b/drivers/staging/rtl8192e/rtllib_rx.c @@ -569,7 +569,7 @@ static void RxReorderIndicatePacket(struct rtllib_device *ieee, { struct rt_hi_throughput *pHTInfo = ieee->pHTInfo; struct rx_reorder_entry *pReorderEntry = NULL; - u8 WinSize = pHTInfo->RxReorderWinSize; + u8 WinSize = pHTInfo->rx_reorder_win_size; u16 WinEnd = 0; u8 index = 0; bool bMatchWinStart = false, bPktInBuf = false; @@ -591,7 +591,7 @@ static void RxReorderIndicatePacket(struct rtllib_device *ieee, netdev_dbg(ieee->dev, "Packet Drop! IndicateSeq: %d, NewSeq: %d\n", pTS->rx_indicate_seq, SeqNum); - pHTInfo->RxReorderDropCounter++; + pHTInfo->rx_reorder_drop_counter++; { int i; @@ -755,7 +755,7 @@ static void RxReorderIndicatePacket(struct rtllib_device *ieee, netdev_dbg(ieee->dev, "%s(): SET rx timeout timer\n", __func__); pTS->rx_timeout_indicate_seq = pTS->rx_indicate_seq; mod_timer(&pTS->rx_pkt_pending_timer, jiffies + - msecs_to_jiffies(pHTInfo->RxReorderPendingTime)); + msecs_to_jiffies(pHTInfo->rx_reorder_pending_time)); } spin_unlock_irqrestore(&(ieee->reorder_spinlock), flags); } @@ -924,7 +924,7 @@ static int rtllib_rx_check_duplicate(struct rtllib_device *ieee, sc = le16_to_cpu(hdr->seq_ctl); frag = WLAN_GET_SEQ_FRAG(sc); - if (!ieee->pHTInfo->bCurRxReorderEnable || + if (!ieee->pHTInfo->cur_rx_reorder_enable || !ieee->current_network.qos_data.active || !IsDataFrame(skb->data) || IsLegacyDataFrame(skb->data)) { @@ -1442,7 +1442,7 @@ static int rtllib_rx_InfraAdhoc(struct rtllib_device *ieee, struct sk_buff *skb, } /* Indicate packets to upper layer or Rx Reorder */ - if (!ieee->pHTInfo->bCurRxReorderEnable || pTS == NULL || bToOtherSTA) + if (!ieee->pHTInfo->cur_rx_reorder_enable || pTS == NULL || bToOtherSTA) rtllib_rx_indicate_pkt_legacy(ieee, rx_stats, rxb, dst, src); else RxReorderIndicatePacket(ieee, rxb, pTS, SeqNum); diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c index b5f4d35954a9dc5e222fb51a6da8d40be535795d..1a3ca3e57623a860849e6b5f62514323c88d870a 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac.c +++ b/drivers/staging/rtl8192e/rtllib_softmac.c @@ -180,7 +180,7 @@ static u8 MgntQuery_MgntFrameTxRate(struct rtllib_device *ieee) struct rt_hi_throughput *pHTInfo = ieee->pHTInfo; u8 rate; - if (pHTInfo->IOTAction & HT_IOT_ACT_MGNT_USE_CCK_6M) + if (pHTInfo->iot_action & HT_IOT_ACT_MGNT_USE_CCK_6M) rate = 0x0c; else rate = ieee->basic_rate & 0x7f; @@ -586,9 +586,9 @@ static void rtllib_softmac_scan_wq(void *data) mutex_lock(&ieee->scan_mutex); - if (ieee->eRFPowerState == eRfOff) { + if (ieee->rf_power_state == rf_off) { netdev_info(ieee->dev, - "======>%s():rf state is eRfOff, return\n", + "======>%s():rf state is rf_off, return\n", __func__); goto out1; } @@ -865,10 +865,10 @@ static struct sk_buff *rtllib_probe_resp(struct rtllib_device *ieee, HTConstructInfoElement(ieee, tmp_ht_info_buf, &tmp_ht_info_len, encrypt); - if (pHTInfo->bRegRT2RTAggregation) { - tmp_generic_ie_buf = ieee->pHTInfo->szRT2RTAggBuffer; + if (pHTInfo->reg_rt2rt_aggregation) { + tmp_generic_ie_buf = ieee->pHTInfo->sz_rt2rt_agg_buf; tmp_generic_ie_len = - sizeof(ieee->pHTInfo->szRT2RTAggBuffer); + sizeof(ieee->pHTInfo->sz_rt2rt_agg_buf); HTConstructRT2RTAggElement(ieee, tmp_generic_ie_buf, &tmp_generic_ie_len); } @@ -1189,10 +1189,10 @@ rtllib_association_req(struct rtllib_network *beacon, ht_cap_len = sizeof(ieee->pHTInfo->SelfHTCap); HTConstructCapabilityElement(ieee, ht_cap_buf, &ht_cap_len, encrypt, true); - if (ieee->pHTInfo->bCurrentRT2RTAggregation) { - realtek_ie_buf = ieee->pHTInfo->szRT2RTAggBuffer; + if (ieee->pHTInfo->current_rt2rt_aggregation) { + realtek_ie_buf = ieee->pHTInfo->sz_rt2rt_agg_buf; realtek_ie_len = - sizeof(ieee->pHTInfo->szRT2RTAggBuffer); + sizeof(ieee->pHTInfo->sz_rt2rt_agg_buf); HTConstructRT2RTAggElement(ieee, realtek_ie_buf, &realtek_ie_len); } @@ -1368,7 +1368,7 @@ rtllib_association_req(struct rtllib_network *beacon, tag += ht_cap_len - 2; } - if (ieee->pHTInfo->bCurrentRT2RTAggregation) { + if (ieee->pHTInfo->current_rt2rt_aggregation) { tag = skb_put(skb, realtek_ie_len); *tag++ = MFIE_TYPE_GENERIC; *tag++ = realtek_ie_len - 2; @@ -1584,13 +1584,8 @@ static void rtllib_associate_procedure_wq(void *data) ieee->data_hard_stop(ieee->dev); rtllib_stop_scan(ieee); - RT_TRACE(COMP_DBG, "===>%s(), chan:%d\n", __func__, - ieee->current_network.channel); HTSetConnectBwMode(ieee, HT_CHANNEL_WIDTH_20, HT_EXTCHNL_OFFSET_NO_EXT); - if (ieee->eRFPowerState == eRfOff) { - RT_TRACE(COMP_DBG, - "=============>%s():Rf state is eRfOff, schedule ipsleave wq again,return\n", - __func__); + if (ieee->rf_power_state == rf_off) { if (ieee->rtllib_ips_leave_wq != NULL) ieee->rtllib_ips_leave_wq(ieee->dev); mutex_unlock(&ieee->wx_mutex); @@ -1611,7 +1606,7 @@ inline void rtllib_softmac_new_net(struct rtllib_device *ieee, short apset, ssidset, ssidbroad, apmatch, ssidmatch; - /* we are interested in new new only if we are not associated + /* we are interested in new only if we are not associated * and we are not associating / authenticating */ if (ieee->state != RTLLIB_NOLINK) @@ -1899,7 +1894,7 @@ static inline u16 assoc_parse(struct rtllib_device *ieee, struct sk_buff *skb, ((ieee->mode == IEEE_G) && (ieee->current_network.mode == IEEE_N_24G) && (ieee->AsocRetryCount++ < (RT_ASOC_RETRY_LIMIT-1)))) { - ieee->pHTInfo->IOTAction |= HT_IOT_ACT_PURE_N_MODE; + ieee->pHTInfo->iot_action |= HT_IOT_ACT_PURE_N_MODE; } else { ieee->AsocRetryCount = 0; } @@ -2062,9 +2057,6 @@ static inline void rtllib_sta_ps(struct work_struct *work) if ((ieee->ps == RTLLIB_PS_DISABLED || ieee->iw_mode != IW_MODE_INFRA || ieee->state != RTLLIB_LINKED)) { - RT_TRACE(COMP_DBG, - "=====>%s(): no need to ps,wake up!! ieee->ps is %d, ieee->iw_mode is %d, ieee->state is %d\n", - __func__, ieee->ps, ieee->iw_mode, ieee->state); spin_lock_irqsave(&ieee->mgmt_tx_lock, flags2); rtllib_sta_wakeup(ieee, 1); @@ -2109,7 +2101,7 @@ static void rtllib_sta_wakeup(struct rtllib_device *ieee, short nl) { if (ieee->sta_sleep == LPS_IS_WAKE) { if (nl) { - if (ieee->pHTInfo->IOTAction & + if (ieee->pHTInfo->iot_action & HT_IOT_ACT_NULL_DATA_POWER_SAVING) { ieee->ack_tx_to_ieee = 1; rtllib_sta_ps_send_null_frame(ieee, 0); @@ -2125,7 +2117,7 @@ static void rtllib_sta_wakeup(struct rtllib_device *ieee, short nl) if (ieee->sta_sleep == LPS_IS_SLEEP) ieee->sta_wake_up(ieee->dev); if (nl) { - if (ieee->pHTInfo->IOTAction & + if (ieee->pHTInfo->iot_action & HT_IOT_ACT_NULL_DATA_POWER_SAVING) { ieee->ack_tx_to_ieee = 1; rtllib_sta_ps_send_null_frame(ieee, 0); @@ -2160,7 +2152,7 @@ void rtllib_ps_tx_ack(struct rtllib_device *ieee, short success) if ((ieee->sta_sleep == LPS_IS_WAKE) && !success) { spin_lock_irqsave(&ieee->mgmt_tx_lock, flags2); - if (ieee->pHTInfo->IOTAction & + if (ieee->pHTInfo->iot_action & HT_IOT_ACT_NULL_DATA_POWER_SAVING) rtllib_sta_ps_send_null_frame(ieee, 0); else @@ -2304,7 +2296,7 @@ static void rtllib_rx_auth_resp(struct rtllib_device *ieee, struct sk_buff *skb) if (ieee->open_wep || !challenge) { ieee->state = RTLLIB_ASSOCIATING_AUTHENTICATED; ieee->softmac_stats.rx_auth_rs_ok++; - if (!(ieee->pHTInfo->IOTAction & HT_IOT_ACT_PURE_N_MODE)) { + if (!(ieee->pHTInfo->iot_action & HT_IOT_ACT_PURE_N_MODE)) { if (!ieee->GetNmodeSupportBySecCfg(ieee->dev)) { if (IsHTHalfNmodeAPs(ieee)) { bSupportNmode = true; diff --git a/drivers/staging/rtl8192e/rtllib_softmac_wx.c b/drivers/staging/rtl8192e/rtllib_softmac_wx.c index 70a62ca0f69a36fea55ea4eadb5a4abe88c91182..f9589c5b62bacca0802e2cb0dc034e336e89884c 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac_wx.c +++ b/drivers/staging/rtl8192e/rtllib_softmac_wx.c @@ -364,8 +364,6 @@ void rtllib_wx_sync_scan_wq(void *data) b40M = 1; chan_offset = ieee->pHTInfo->CurSTAExtChnlOffset; bandwidth = (enum ht_channel_width)ieee->pHTInfo->bCurBW40MHz; - RT_TRACE(COMP_DBG, "Scan in 40M, force to 20M first:%d, %d\n", - chan_offset, bandwidth); ieee->SetBWModeHandler(ieee->dev, HT_CHANNEL_WIDTH_20, HT_EXTCHNL_OFFSET_NO_EXT); } @@ -373,7 +371,6 @@ void rtllib_wx_sync_scan_wq(void *data) rtllib_start_scan_syncro(ieee, 0); if (b40M) { - RT_TRACE(COMP_DBG, "Scan in 20M, back to 40M\n"); if (chan_offset == HT_EXTCHNL_OFFSET_UPPER) ieee->set_chan(ieee->dev, chan + 2); else if (chan_offset == HT_EXTCHNL_OFFSET_LOWER) @@ -571,14 +568,11 @@ int rtllib_wx_set_power(struct rtllib_device *ieee, mutex_lock(&ieee->wx_mutex); if (wrqu->power.disabled) { - RT_TRACE(COMP_DBG, "===>%s(): power disable\n", __func__); ieee->ps = RTLLIB_PS_DISABLED; goto exit; } if (wrqu->power.flags & IW_POWER_TIMEOUT) { ieee->ps_timeout = wrqu->power.value / 1000; - RT_TRACE(COMP_DBG, "===>%s():ps_timeout is %d\n", __func__, - ieee->ps_timeout); } if (wrqu->power.flags & IW_POWER_PERIOD) diff --git a/drivers/staging/rtl8192e/rtllib_tx.c b/drivers/staging/rtl8192e/rtllib_tx.c index 42f81b23a144fad06ff5619cd624c36e4bed1651..e307020580a0571755ed0cdeaca7958dfcec9851 100644 --- a/drivers/staging/rtl8192e/rtllib_tx.c +++ b/drivers/staging/rtl8192e/rtllib_tx.c @@ -284,7 +284,7 @@ static void rtllib_tx_query_agg_cap(struct rtllib_device *ieee, if (tcb_desc->bdhcp || ieee->CntAfterLink < 2) return; - if (pHTInfo->IOTAction & HT_IOT_ACT_TX_NO_AGGREGATION) + if (pHTInfo->iot_action & HT_IOT_ACT_TX_NO_AGGREGATION) return; if (!ieee->GetNmodeSupportBySecCfg(ieee->dev)) @@ -315,7 +315,7 @@ static void rtllib_tx_query_agg_cap(struct rtllib_device *ieee, if (ieee->iw_mode == IW_MODE_INFRA) { tcb_desc->bAMPDUEnable = true; tcb_desc->ampdu_factor = pHTInfo->CurrentAMPDUFactor; - tcb_desc->ampdu_density = pHTInfo->CurrentMPDUDensity; + tcb_desc->ampdu_density = pHTInfo->current_mpdu_density; } } FORCED_AGG_SETTING: @@ -325,8 +325,8 @@ FORCED_AGG_SETTING: case HT_AGG_FORCE_ENABLE: tcb_desc->bAMPDUEnable = true; - tcb_desc->ampdu_density = pHTInfo->ForcedMPDUDensity; - tcb_desc->ampdu_factor = pHTInfo->ForcedAMPDUFactor; + tcb_desc->ampdu_density = pHTInfo->forced_mpdu_density; + tcb_desc->ampdu_factor = pHTInfo->forced_ampdu_factor; break; case HT_AGG_FORCE_DISABLE: @@ -358,7 +358,7 @@ static void rtllib_query_HTCapShortGI(struct rtllib_device *ieee, if (!pHTInfo->bCurrentHTSupport || !pHTInfo->bEnableHT) return; - if (pHTInfo->bForcedShortGI) { + if (pHTInfo->forced_short_gi) { tcb_desc->bUseShortGI = true; return; } @@ -384,7 +384,7 @@ static void rtllib_query_BandwidthMode(struct rtllib_device *ieee, if ((tcb_desc->data_rate & 0x80) == 0) return; - if (pHTInfo->bCurBW40MHz && pHTInfo->bCurTxBW40MHz && + if (pHTInfo->bCurBW40MHz && pHTInfo->cur_tx_bw40mhz && !ieee->bandwidth_auto_switch.bforced_tx20Mhz) tcb_desc->bPacketBW = true; } @@ -422,12 +422,12 @@ static void rtllib_query_protectionmode(struct rtllib_device *ieee, pHTInfo = ieee->pHTInfo; while (true) { - if (pHTInfo->IOTAction & HT_IOT_ACT_FORCED_CTS2SELF) { + if (pHTInfo->iot_action & HT_IOT_ACT_FORCED_CTS2SELF) { tcb_desc->bCTSEnable = true; tcb_desc->rts_rate = MGN_24M; tcb_desc->bRTSEnable = true; break; - } else if (pHTInfo->IOTAction & (HT_IOT_ACT_FORCED_RTS | + } else if (pHTInfo->iot_action & (HT_IOT_ACT_FORCED_RTS | HT_IOT_ACT_PURE_N_MODE)) { tcb_desc->bRTSEnable = true; tcb_desc->rts_rate = MGN_24M; @@ -440,7 +440,7 @@ static void rtllib_query_protectionmode(struct rtllib_device *ieee, break; } if (pHTInfo->bCurrentHTSupport && pHTInfo->bEnableHT) { - u8 HTOpMode = pHTInfo->CurrentOpMode; + u8 HTOpMode = pHTInfo->current_op_mode; if ((pHTInfo->bCurBW40MHz && (HTOpMode == 2 || HTOpMode == 3)) || @@ -885,7 +885,7 @@ static int rtllib_xmit_inter(struct sk_buff *skb, struct net_device *dev) tcb_desc->priority = skb->priority; if (ether_type == ETH_P_PAE) { - if (ieee->pHTInfo->IOTAction & + if (ieee->pHTInfo->iot_action & HT_IOT_ACT_WA_IOT_Broadcom) { tcb_desc->data_rate = MgntQuery_TxRateExcludeCCKRates(ieee); @@ -910,7 +910,7 @@ static int rtllib_xmit_inter(struct sk_buff *skb, struct net_device *dev) tcb_desc->data_rate = rtllib_current_rate(ieee); if (bdhcp) { - if (ieee->pHTInfo->IOTAction & + if (ieee->pHTInfo->iot_action & HT_IOT_ACT_WA_IOT_Broadcom) { tcb_desc->data_rate = MgntQuery_TxRateExcludeCCKRates(ieee); @@ -962,9 +962,9 @@ static int rtllib_xmit_inter(struct sk_buff *skb, struct net_device *dev) } -int rtllib_xmit(struct sk_buff *skb, struct net_device *dev) +netdev_tx_t rtllib_xmit(struct sk_buff *skb, struct net_device *dev) { memset(skb->cb, 0, sizeof(skb->cb)); - return rtllib_xmit_inter(skb, dev); + return rtllib_xmit_inter(skb, dev) ? NETDEV_TX_BUSY : NETDEV_TX_OK; } EXPORT_SYMBOL(rtllib_xmit); diff --git a/drivers/staging/rtl8192u/Makefile b/drivers/staging/rtl8192u/Makefile index 0be7426b6ebc696b4e512c608f859e1e9c98a07c..d32dfd89a606ad95b4cde1781a716fa0d322445b 100644 --- a/drivers/staging/rtl8192u/Makefile +++ b/drivers/staging/rtl8192u/Makefile @@ -8,6 +8,7 @@ ccflags-y += -DTHOMAS_BEACON -DTHOMAS_TASKLET -DTHOMAS_SKB -DTHOMAS_TURBO r8192u_usb-y := r8192U_core.o r8180_93cx6.o r8192U_wx.o \ r8190_rtl8256.o r819xU_phy.o r819xU_firmware.o \ r819xU_cmdpkt.o r8192U_dm.o r819xU_firmware_img.o \ + r8192U_debugfs.o \ ieee80211/ieee80211_crypt.o \ ieee80211/ieee80211_crypt_tkip.o \ ieee80211/ieee80211_crypt_ccmp.o \ diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211.h b/drivers/staging/rtl8192u/ieee80211/ieee80211.h index b577f9c81f8580362f2c0f43e0f023cfea66e4d0..9cd4b18967459db6e5be3d4dcabcef751219b0b7 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211.h +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211.h @@ -2178,7 +2178,7 @@ int ieee80211_set_encryption(struct ieee80211_device *ieee); int ieee80211_encrypt_fragment(struct ieee80211_device *ieee, struct sk_buff *frag, int hdr_len); -int ieee80211_xmit(struct sk_buff *skb, struct net_device *dev); +netdev_tx_t ieee80211_xmit(struct sk_buff *skb, struct net_device *dev); void ieee80211_txb_free(struct ieee80211_txb *txb); diff --git a/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c b/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c index 8602e3a6c8370db2ac98e9606bed5ac5cea44181..e4b6454809a0f3757b3500f176d81365f2e779e5 100644 --- a/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c +++ b/drivers/staging/rtl8192u/ieee80211/ieee80211_tx.c @@ -526,7 +526,7 @@ static void ieee80211_query_seqnum(struct ieee80211_device *ieee, } } -int ieee80211_xmit(struct sk_buff *skb, struct net_device *dev) +netdev_tx_t ieee80211_xmit(struct sk_buff *skb, struct net_device *dev) { struct ieee80211_device *ieee = netdev_priv(dev); struct ieee80211_txb *txb = NULL; @@ -822,13 +822,13 @@ int ieee80211_xmit(struct sk_buff *skb, struct net_device *dev) if ((*ieee->hard_start_xmit)(txb, dev) == 0) { stats->tx_packets++; stats->tx_bytes += __le16_to_cpu(txb->payload_size); - return 0; + return NETDEV_TX_OK; } ieee80211_txb_free(txb); } } - return 0; + return NETDEV_TX_OK; failed: spin_unlock_irqrestore(&ieee->lock, flags); diff --git a/drivers/staging/rtl8192u/r8192U.h b/drivers/staging/rtl8192u/r8192U.h index 1942cb84937484056270607eed131d3f1c768118..ff0ada00bf4142ab81851f909edbf09c01654d40 100644 --- a/drivers/staging/rtl8192u/r8192U.h +++ b/drivers/staging/rtl8192u/r8192U.h @@ -1061,6 +1061,9 @@ typedef struct r8192_priv { struct delayed_work gpio_change_rf_wq; struct delayed_work initialgain_operate_wq; struct workqueue_struct *priv_wq; + + /* debugfs */ + struct dentry *debugfs_dir; } r8192_priv; /* For rtl8187B */ @@ -1117,4 +1120,10 @@ void EnableHWSecurityConfig8192(struct net_device *dev); void setKey(struct net_device *dev, u8 EntryNo, u8 KeyIndex, u16 KeyType, const u8 *MacAddr, u8 DefaultKey, u32 *KeyContent); +void rtl8192_debugfs_init_one(struct net_device *dev); +void rtl8192_debugfs_exit_one(struct net_device *dev); +void rtl8192_debugfs_rename_one(struct net_device *dev); +void rtl8192_debugfs_init(void); +void rtl8192_debugfs_exit(void); + #endif diff --git a/drivers/staging/rtl8192u/r8192U_core.c b/drivers/staging/rtl8192u/r8192U_core.c index 2ca925f35830a09140768d28063f3939d6e50a24..0a60ef20107c4301d4b5e8e6ba37fe3178195752 100644 --- a/drivers/staging/rtl8192u/r8192U_core.c +++ b/drivers/staging/rtl8192u/r8192U_core.c @@ -56,7 +56,6 @@ double __extendsfdf2(float a) #include "r8192U_dm.h" #include #include -#include #include /* FIXME: check if 2.6.7 is ok */ @@ -452,179 +451,6 @@ static struct net_device_stats *rtl8192_stats(struct net_device *dev); static void rtl8192_restart(struct work_struct *work); static void watch_dog_timer_callback(struct timer_list *t); -/**************************************************************************** - * -----------------------------PROCFS STUFF------------------------- - ****************************************************************************/ - -static struct proc_dir_entry *rtl8192_proc; - -static int __maybe_unused proc_get_stats_ap(struct seq_file *m, void *v) -{ - struct net_device *dev = m->private; - struct r8192_priv *priv = (struct r8192_priv *)ieee80211_priv(dev); - struct ieee80211_device *ieee = priv->ieee80211; - struct ieee80211_network *target; - - list_for_each_entry(target, &ieee->network_list, list) { - const char *wpa = "non_WPA"; - - if (target->wpa_ie_len > 0 || target->rsn_ie_len > 0) - wpa = "WPA"; - - seq_printf(m, "%s %s\n", target->ssid, wpa); - } - - return 0; -} - -static int __maybe_unused proc_get_registers(struct seq_file *m, void *v) -{ - struct net_device *dev = m->private; - int i, n, max = 0xff; - u8 byte_rd; - - seq_puts(m, "\n####################page 0##################\n "); - - for (n = 0; n <= max;) { - seq_printf(m, "\nD: %2x > ", n); - - for (i = 0; i < 16 && n <= max; i++, n++) { - read_nic_byte(dev, 0x000 | n, &byte_rd); - seq_printf(m, "%2x ", byte_rd); - } - } - - seq_puts(m, "\n####################page 1##################\n "); - for (n = 0; n <= max;) { - seq_printf(m, "\nD: %2x > ", n); - - for (i = 0; i < 16 && n <= max; i++, n++) { - read_nic_byte(dev, 0x100 | n, &byte_rd); - seq_printf(m, "%2x ", byte_rd); - } - } - - seq_puts(m, "\n####################page 3##################\n "); - for (n = 0; n <= max;) { - seq_printf(m, "\nD: %2x > ", n); - - for (i = 0; i < 16 && n <= max; i++, n++) { - read_nic_byte(dev, 0x300 | n, &byte_rd); - seq_printf(m, "%2x ", byte_rd); - } - } - - seq_putc(m, '\n'); - return 0; -} - -static int __maybe_unused proc_get_stats_tx(struct seq_file *m, void *v) -{ - struct net_device *dev = m->private; - struct r8192_priv *priv = (struct r8192_priv *)ieee80211_priv(dev); - - seq_printf(m, - "TX VI priority ok int: %lu\n" - "TX VI priority error int: %lu\n" - "TX VO priority ok int: %lu\n" - "TX VO priority error int: %lu\n" - "TX BE priority ok int: %lu\n" - "TX BE priority error int: %lu\n" - "TX BK priority ok int: %lu\n" - "TX BK priority error int: %lu\n" - "TX MANAGE priority ok int: %lu\n" - "TX MANAGE priority error int: %lu\n" - "TX BEACON priority ok int: %lu\n" - "TX BEACON priority error int: %lu\n" - "TX queue resume: %lu\n" - "TX queue stopped?: %d\n" - "TX fifo overflow: %lu\n" - "TX VI queue: %d\n" - "TX VO queue: %d\n" - "TX BE queue: %d\n" - "TX BK queue: %d\n" - "TX VI dropped: %lu\n" - "TX VO dropped: %lu\n" - "TX BE dropped: %lu\n" - "TX BK dropped: %lu\n" - "TX total data packets %lu\n", - priv->stats.txviokint, - priv->stats.txvierr, - priv->stats.txvookint, - priv->stats.txvoerr, - priv->stats.txbeokint, - priv->stats.txbeerr, - priv->stats.txbkokint, - priv->stats.txbkerr, - priv->stats.txmanageokint, - priv->stats.txmanageerr, - priv->stats.txbeaconokint, - priv->stats.txbeaconerr, - priv->stats.txresumed, - netif_queue_stopped(dev), - priv->stats.txoverflow, - atomic_read(&(priv->tx_pending[VI_PRIORITY])), - atomic_read(&(priv->tx_pending[VO_PRIORITY])), - atomic_read(&(priv->tx_pending[BE_PRIORITY])), - atomic_read(&(priv->tx_pending[BK_PRIORITY])), - priv->stats.txvidrop, - priv->stats.txvodrop, - priv->stats.txbedrop, - priv->stats.txbkdrop, - priv->stats.txdatapkt - ); - - return 0; -} - -static int __maybe_unused proc_get_stats_rx(struct seq_file *m, void *v) -{ - struct net_device *dev = m->private; - struct r8192_priv *priv = (struct r8192_priv *)ieee80211_priv(dev); - - seq_printf(m, - "RX packets: %lu\n" - "RX urb status error: %lu\n" - "RX invalid urb error: %lu\n", - priv->stats.rxoktotal, - priv->stats.rxstaterr, - priv->stats.rxurberr); - - return 0; -} - -static void rtl8192_proc_module_init(void) -{ - RT_TRACE(COMP_INIT, "Initializing proc filesystem"); - rtl8192_proc = proc_mkdir(RTL819XU_MODULE_NAME, init_net.proc_net); -} - -static void rtl8192_proc_init_one(struct net_device *dev) -{ - struct proc_dir_entry *dir; - - if (!rtl8192_proc) - return; - - dir = proc_mkdir_data(dev->name, 0, rtl8192_proc, dev); - if (!dir) - return; - - proc_create_single("stats-rx", S_IFREG | 0444, dir, - proc_get_stats_rx); - proc_create_single("stats-tx", S_IFREG | 0444, dir, - proc_get_stats_tx); - proc_create_single("stats-ap", S_IFREG | 0444, dir, - proc_get_stats_ap); - proc_create_single("registers", S_IFREG | 0444, dir, - proc_get_registers); -} - -static void rtl8192_proc_remove_one(struct net_device *dev) -{ - remove_proc_subtree(dev->name, rtl8192_proc); -} - /**************************************************************************** * -----------------------------MISC STUFF------------------------- *****************************************************************************/ @@ -4730,7 +4556,7 @@ static int rtl8192_usb_probe(struct usb_interface *intf, goto fail2; RT_TRACE(COMP_INIT, "dev name=======> %s\n", dev->name); - rtl8192_proc_init_one(dev); + rtl8192_debugfs_init_one(dev); RT_TRACE(COMP_INIT, "Driver probe completed\n"); return 0; @@ -4764,10 +4590,11 @@ static void rtl8192_usb_disconnect(struct usb_interface *intf) struct net_device *dev = usb_get_intfdata(intf); struct r8192_priv *priv = ieee80211_priv(dev); - unregister_netdev(dev); RT_TRACE(COMP_DOWN, "=============>wlan driver to be removed\n"); - rtl8192_proc_remove_one(dev); + rtl8192_debugfs_exit_one(dev); + + unregister_netdev(dev); rtl8192_down(dev); kfree(priv->pFirmware); @@ -4779,6 +4606,30 @@ static void rtl8192_usb_disconnect(struct usb_interface *intf) RT_TRACE(COMP_DOWN, "wlan driver removed\n"); } +static int rtl8192_usb_netdev_event(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct net_device *netdev = netdev_notifier_info_to_dev(data); + + if (netdev->netdev_ops != &rtl8192_netdev_ops) + goto out; + + switch (event) { + case NETDEV_CHANGENAME: + rtl8192_debugfs_rename_one(netdev); + break; + default: + break; + } + +out: + return NOTIFY_DONE; +} + +static struct notifier_block rtl8192_usb_netdev_notifier = { + .notifier_call = rtl8192_usb_netdev_event, +}; + static int __init rtl8192_usb_module_init(void) { int ret; @@ -4788,10 +4639,17 @@ static int __init rtl8192_usb_module_init(void) RT_TRACE(COMP_INIT, "Initializing module"); RT_TRACE(COMP_INIT, "Wireless extensions version %d", WIRELESS_EXT); + ret = register_netdevice_notifier(&rtl8192_usb_netdev_notifier); + if (ret) { + pr_err("register_netdevice_notifier failed %d\n", ret); + return ret; + } + + rtl8192_debugfs_init(); ret = ieee80211_debug_init(); if (ret) { pr_err("ieee80211_debug_init() failed %d\n", ret); - return ret; + goto debugfs_exit; } ret = ieee80211_crypto_init(); @@ -4818,14 +4676,12 @@ static int __init rtl8192_usb_module_init(void) goto crypto_ccmp_exit; } - rtl8192_proc_module_init(); ret = usb_register(&rtl8192_usb_driver); if (ret) - goto rtl8192_proc_module_exit; + goto crypto_wep_exit; return ret; -rtl8192_proc_module_exit: - remove_proc_entry(RTL819XU_MODULE_NAME, init_net.proc_net); +crypto_wep_exit: ieee80211_crypto_wep_exit(); crypto_ccmp_exit: ieee80211_crypto_ccmp_exit(); @@ -4835,18 +4691,22 @@ crypto_exit: ieee80211_crypto_deinit(); debug_exit: ieee80211_debug_exit(); +debugfs_exit: + rtl8192_debugfs_exit(); + unregister_netdevice_notifier(&rtl8192_usb_netdev_notifier); return ret; } static void __exit rtl8192_usb_module_exit(void) { usb_deregister(&rtl8192_usb_driver); - remove_proc_entry(RTL819XU_MODULE_NAME, init_net.proc_net); ieee80211_crypto_wep_exit(); ieee80211_crypto_ccmp_exit(); ieee80211_crypto_tkip_exit(); ieee80211_crypto_deinit(); ieee80211_debug_exit(); + rtl8192_debugfs_exit(); + unregister_netdevice_notifier(&rtl8192_usb_netdev_notifier); RT_TRACE(COMP_DOWN, "Exiting"); } diff --git a/drivers/staging/rtl8192u/r8192U_debugfs.c b/drivers/staging/rtl8192u/r8192U_debugfs.c new file mode 100644 index 0000000000000000000000000000000000000000..fe8ef72506ee1c11ac4d1afeee7b31ee774bc13e --- /dev/null +++ b/drivers/staging/rtl8192u/r8192U_debugfs.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/**************************************************************************** + * -----------------------------DEGUGFS STUFF------------------------- + ****************************************************************************/ +#include +#include +#include "r8192U.h" + +#define KBUILD_MODNAME "r8192u_usb" + +static int rtl8192_usb_stats_ap_show(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct r8192_priv *priv = ieee80211_priv(dev); + struct ieee80211_device *ieee = priv->ieee80211; + struct ieee80211_network *target; + + list_for_each_entry(target, &ieee->network_list, list) { + const char *wpa = "non_WPA"; + + if (target->wpa_ie_len > 0 || target->rsn_ie_len > 0) + wpa = "WPA"; + + seq_printf(m, "%s %s\n", target->ssid, wpa); + } + + return 0; +} + +static int rtl8192_usb_registers_show(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + int i, n, max = 0xff; + u8 byte_rd; + + seq_puts(m, "\n####################page 0##################\n "); + + for (n = 0; n <= max;) { + seq_printf(m, "\nD: %2x > ", n); + + for (i = 0; i < 16 && n <= max; i++, n++) { + read_nic_byte(dev, 0x000 | n, &byte_rd); + seq_printf(m, "%2x ", byte_rd); + } + } + + seq_puts(m, "\n####################page 1##################\n "); + for (n = 0; n <= max;) { + seq_printf(m, "\nD: %2x > ", n); + + for (i = 0; i < 16 && n <= max; i++, n++) { + read_nic_byte(dev, 0x100 | n, &byte_rd); + seq_printf(m, "%2x ", byte_rd); + } + } + + seq_puts(m, "\n####################page 3##################\n "); + for (n = 0; n <= max;) { + seq_printf(m, "\nD: %2x > ", n); + + for (i = 0; i < 16 && n <= max; i++, n++) { + read_nic_byte(dev, 0x300 | n, &byte_rd); + seq_printf(m, "%2x ", byte_rd); + } + } + + seq_putc(m, '\n'); + return 0; +} + +static int rtl8192_usb_stats_tx_show(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct r8192_priv *priv = ieee80211_priv(dev); + + seq_printf(m, + "TX VI priority ok int: %lu\n" + "TX VI priority error int: %lu\n" + "TX VO priority ok int: %lu\n" + "TX VO priority error int: %lu\n" + "TX BE priority ok int: %lu\n" + "TX BE priority error int: %lu\n" + "TX BK priority ok int: %lu\n" + "TX BK priority error int: %lu\n" + "TX MANAGE priority ok int: %lu\n" + "TX MANAGE priority error int: %lu\n" + "TX BEACON priority ok int: %lu\n" + "TX BEACON priority error int: %lu\n" + "TX queue resume: %lu\n" + "TX queue stopped?: %d\n" + "TX fifo overflow: %lu\n" + "TX VI queue: %d\n" + "TX VO queue: %d\n" + "TX BE queue: %d\n" + "TX BK queue: %d\n" + "TX VI dropped: %lu\n" + "TX VO dropped: %lu\n" + "TX BE dropped: %lu\n" + "TX BK dropped: %lu\n" + "TX total data packets %lu\n", + priv->stats.txviokint, + priv->stats.txvierr, + priv->stats.txvookint, + priv->stats.txvoerr, + priv->stats.txbeokint, + priv->stats.txbeerr, + priv->stats.txbkokint, + priv->stats.txbkerr, + priv->stats.txmanageokint, + priv->stats.txmanageerr, + priv->stats.txbeaconokint, + priv->stats.txbeaconerr, + priv->stats.txresumed, + netif_queue_stopped(dev), + priv->stats.txoverflow, + atomic_read(&(priv->tx_pending[VI_PRIORITY])), + atomic_read(&(priv->tx_pending[VO_PRIORITY])), + atomic_read(&(priv->tx_pending[BE_PRIORITY])), + atomic_read(&(priv->tx_pending[BK_PRIORITY])), + priv->stats.txvidrop, + priv->stats.txvodrop, + priv->stats.txbedrop, + priv->stats.txbkdrop, + priv->stats.txdatapkt + ); + + return 0; +} + +static int rtl8192_usb_stats_rx_show(struct seq_file *m, void *v) +{ + struct net_device *dev = m->private; + struct r8192_priv *priv = ieee80211_priv(dev); + + seq_printf(m, + "RX packets: %lu\n" + "RX urb status error: %lu\n" + "RX invalid urb error: %lu\n", + priv->stats.rxoktotal, + priv->stats.rxstaterr, + priv->stats.rxurberr); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(rtl8192_usb_stats_rx); +DEFINE_SHOW_ATTRIBUTE(rtl8192_usb_stats_tx); +DEFINE_SHOW_ATTRIBUTE(rtl8192_usb_stats_ap); +DEFINE_SHOW_ATTRIBUTE(rtl8192_usb_registers); + +void rtl8192_debugfs_init_one(struct net_device *dev) +{ + struct r8192_priv *priv = ieee80211_priv(dev); + struct dentry *parent_dir = debugfs_lookup(KBUILD_MODNAME, NULL); + struct dentry *dir = debugfs_create_dir(dev->name, parent_dir); + + debugfs_create_file("stats-rx", 0444, dir, dev, &rtl8192_usb_stats_rx_fops); + debugfs_create_file("stats-tx", 0444, dir, dev, &rtl8192_usb_stats_tx_fops); + debugfs_create_file("stats-ap", 0444, dir, dev, &rtl8192_usb_stats_ap_fops); + debugfs_create_file("registers", 0444, dir, dev, &rtl8192_usb_registers_fops); + + priv->debugfs_dir = dir; +} + +void rtl8192_debugfs_exit_one(struct net_device *dev) +{ + struct r8192_priv *priv = ieee80211_priv(dev); + + debugfs_remove_recursive(priv->debugfs_dir); +} + +void rtl8192_debugfs_rename_one(struct net_device *dev) +{ + struct r8192_priv *priv = ieee80211_priv(dev); + struct dentry *parent_dir = debugfs_lookup(KBUILD_MODNAME, NULL); + + debugfs_rename(parent_dir, priv->debugfs_dir, parent_dir, dev->name); +} + +void rtl8192_debugfs_init(void) +{ + debugfs_create_dir(KBUILD_MODNAME, NULL); +} + +void rtl8192_debugfs_exit(void) +{ + debugfs_remove_recursive(debugfs_lookup(KBUILD_MODNAME, NULL)); +} diff --git a/drivers/staging/rtl8712/rtl8712_cmd.c b/drivers/staging/rtl8712/rtl8712_cmd.c index 2326aae6709e233fe8f90e1950b9e4174fc1a7ab..bb7db96ed8219f250cbc039ba374637de5026796 100644 --- a/drivers/staging/rtl8712/rtl8712_cmd.c +++ b/drivers/staging/rtl8712/rtl8712_cmd.c @@ -117,34 +117,6 @@ static void r871x_internal_cmd_hdl(struct _adapter *padapter, u8 *pbuf) kfree(pdrvcmd->pbuf); } -static u8 read_macreg_hdl(struct _adapter *padapter, u8 *pbuf) -{ - void (*pcmd_callback)(struct _adapter *dev, struct cmd_obj *pcmd); - struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; - - /* invoke cmd->callback function */ - pcmd_callback = cmd_callback[pcmd->cmdcode].callback; - if (!pcmd_callback) - r8712_free_cmd_obj(pcmd); - else - pcmd_callback(padapter, pcmd); - return H2C_SUCCESS; -} - -static u8 write_macreg_hdl(struct _adapter *padapter, u8 *pbuf) -{ - void (*pcmd_callback)(struct _adapter *dev, struct cmd_obj *pcmd); - struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; - - /* invoke cmd->callback function */ - pcmd_callback = cmd_callback[pcmd->cmdcode].callback; - if (!pcmd_callback) - r8712_free_cmd_obj(pcmd); - else - pcmd_callback(padapter, pcmd); - return H2C_SUCCESS; -} - static u8 read_bbreg_hdl(struct _adapter *padapter, u8 *pbuf) { struct cmd_obj *pcmd = (struct cmd_obj *)pbuf; @@ -213,14 +185,6 @@ static struct cmd_obj *cmd_hdl_filter(struct _adapter *padapter, pcmd_r = NULL; switch (pcmd->cmdcode) { - case GEN_CMD_CODE(_Read_MACREG): - read_macreg_hdl(padapter, (u8 *)pcmd); - pcmd_r = pcmd; - break; - case GEN_CMD_CODE(_Write_MACREG): - write_macreg_hdl(padapter, (u8 *)pcmd); - pcmd_r = pcmd; - break; case GEN_CMD_CODE(_Read_BBREG): read_bbreg_hdl(padapter, (u8 *)pcmd); break; diff --git a/drivers/staging/rtl8712/xmit_linux.c b/drivers/staging/rtl8712/xmit_linux.c index 4a93839bf9474de9e4d4fe699809a5c1195c44b7..132afbf49dde952c72c1da2d4402fd3bdc719dde 100644 --- a/drivers/staging/rtl8712/xmit_linux.c +++ b/drivers/staging/rtl8712/xmit_linux.c @@ -66,16 +66,16 @@ void r8712_set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib) { struct ethhdr etherhdr; struct iphdr ip_hdr; - u16 UserPriority = 0; + u16 user_priority = 0; _r8712_open_pktfile(ppktfile->pkt, ppktfile); _r8712_pktfile_read(ppktfile, (unsigned char *)ðerhdr, ETH_HLEN); - /* get UserPriority from IP hdr*/ + /* get user_priority from IP hdr*/ if (pattrib->ether_type == 0x0800) { _r8712_pktfile_read(ppktfile, (u8 *)&ip_hdr, sizeof(ip_hdr)); - /*UserPriority = (ntohs(ip_hdr.tos) >> 5) & 0x3 ;*/ - UserPriority = ip_hdr.tos >> 5; + /*user_priority = (ntohs(ip_hdr.tos) >> 5) & 0x3 ;*/ + user_priority = ip_hdr.tos >> 5; } else { /* "When priority processing of data frames is supported, * a STA's SME should send EAPOL-Key frames at the highest @@ -83,9 +83,9 @@ void r8712_set_qos(struct pkt_file *ppktfile, struct pkt_attrib *pattrib) */ if (pattrib->ether_type == 0x888e) - UserPriority = 7; + user_priority = 7; } - pattrib->priority = UserPriority; + pattrib->priority = user_priority; pattrib->hdrlen = WLAN_HDR_A3_QOS_LEN; pattrib->subtype = WIFI_QOS_DATA_TYPE; } @@ -140,7 +140,7 @@ void r8712_xmit_complete(struct _adapter *padapter, struct xmit_frame *pxframe) pxframe->pkt = NULL; } -int r8712_xmit_entry(_pkt *pkt, struct net_device *netdev) +netdev_tx_t r8712_xmit_entry(_pkt *pkt, struct net_device *netdev) { struct xmit_frame *xmitframe = NULL; struct _adapter *adapter = netdev_priv(netdev); @@ -165,11 +165,11 @@ int r8712_xmit_entry(_pkt *pkt, struct net_device *netdev) } xmitpriv->tx_pkts++; xmitpriv->tx_bytes += xmitframe->attrib.last_txcmdsz; - return 0; + return NETDEV_TX_OK; _xmit_entry_drop: if (xmitframe) r8712_free_xmitframe(xmitpriv, xmitframe); xmitpriv->tx_drop++; dev_kfree_skb_any(pkt); - return 0; + return NETDEV_TX_OK; } diff --git a/drivers/staging/rtl8712/xmit_osdep.h b/drivers/staging/rtl8712/xmit_osdep.h index b76021b568f8e920f87412ddc809ddcf82f644fc..1ad42658c8831c7126b1b8b4d973e79304a994c7 100644 --- a/drivers/staging/rtl8712/xmit_osdep.h +++ b/drivers/staging/rtl8712/xmit_osdep.h @@ -34,7 +34,7 @@ struct sta_xmit_priv; struct xmit_frame; struct xmit_buf; -int r8712_xmit_entry(_pkt *pkt, struct net_device *pnetdev); +netdev_tx_t r8712_xmit_entry(_pkt *pkt, struct net_device *pnetdev); void r8712_SetFilter(struct work_struct *work); int r8712_xmit_resource_alloc(struct _adapter *padapter, struct xmit_buf *pxmitbuf); diff --git a/drivers/staging/rtl8723bs/Makefile b/drivers/staging/rtl8723bs/Makefile index 159ca1b9016b7139db0b972831eac686dc98fbf5..590bde02058c7a6608c7c7374f211c3481d6053d 100644 --- a/drivers/staging/rtl8723bs/Makefile +++ b/drivers/staging/rtl8723bs/Makefile @@ -10,7 +10,6 @@ r8723bs-y = \ core/rtw_ieee80211.o \ core/rtw_mlme.o \ core/rtw_mlme_ext.o \ - core/rtw_odm.o \ core/rtw_pwrctrl.o \ core/rtw_recv.o \ core/rtw_rf.o \ @@ -33,7 +32,6 @@ r8723bs-y = \ hal/odm_DynamicTxPower.o \ hal/odm_EdcaTurboCheck.o \ hal/odm_HWConfig.o \ - hal/odm_NoiseMonitor.o \ hal/odm_RegConfig8723B.o \ hal/rtl8723b_cmd.o \ hal/rtl8723b_dm.o \ diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c index b4170f64d11868556a946bf92fab0ed2ce0dab65..d3f10a3cf972a3037d1d4df5aed4cdbc0bb8c972 100644 --- a/drivers/staging/rtl8723bs/core/rtw_cmd.c +++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c @@ -161,8 +161,6 @@ static struct cmd_hdl wlancmds[] = { int rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) { - int res = 0; - init_completion(&pcmdpriv->cmd_queue_comp); init_completion(&pcmdpriv->terminate_cmdthread_comp); @@ -175,18 +173,16 @@ int rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) pcmdpriv->cmd_allocated_buf = rtw_zmalloc(MAX_CMDSZ + CMDBUFF_ALIGN_SZ); - if (!pcmdpriv->cmd_allocated_buf) { - res = -ENOMEM; - goto exit; - } + if (!pcmdpriv->cmd_allocated_buf) + return -ENOMEM; pcmdpriv->cmd_buf = pcmdpriv->cmd_allocated_buf + CMDBUFF_ALIGN_SZ - ((SIZE_PTR)(pcmdpriv->cmd_allocated_buf) & (CMDBUFF_ALIGN_SZ-1)); pcmdpriv->rsp_allocated_buf = rtw_zmalloc(MAX_RSPSZ + 4); if (!pcmdpriv->rsp_allocated_buf) { - res = -ENOMEM; - goto exit; + kfree(pcmdpriv->cmd_allocated_buf); + return -ENOMEM; } pcmdpriv->rsp_buf = pcmdpriv->rsp_allocated_buf + 4 - ((SIZE_PTR)(pcmdpriv->rsp_allocated_buf) & 3); @@ -196,8 +192,8 @@ int rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) pcmdpriv->rsp_cnt = 0; mutex_init(&pcmdpriv->sctx_mutex); -exit: - return res; + + return 0; } static void c2h_wk_callback(struct work_struct *work); @@ -593,35 +589,6 @@ u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, return res; } -u8 rtw_setdatarate_cmd(struct adapter *padapter, u8 *rateset) -{ - struct cmd_obj *ph2c; - struct setdatarate_parm *pbsetdataratepara; - struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - u8 res = _SUCCESS; - - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); - if (!ph2c) { - res = _FAIL; - goto exit; - } - - pbsetdataratepara = rtw_zmalloc(sizeof(struct setdatarate_parm)); - if (!pbsetdataratepara) { - kfree(ph2c); - res = _FAIL; - goto exit; - } - - init_h2fwcmd_w_parm_no_rsp(ph2c, pbsetdataratepara, GEN_CMD_CODE(_SetDataRate)); - pbsetdataratepara->mac_id = 5; - memcpy(pbsetdataratepara->datarates, rateset, NumRates); - - res = rtw_enqueue_cmd(pcmdpriv, ph2c); -exit: - return res; -} - void rtw_getbbrfreg_cmdrsp_callback(struct adapter *padapter, struct cmd_obj *pcmd) { /* rtw_free_cmd_obj(pcmd); */ @@ -1140,61 +1107,6 @@ exit: return res; } -u8 rtw_set_chplan_cmd(struct adapter *padapter, u8 chplan, u8 enqueue, u8 swconfig) -{ - struct cmd_obj *pcmdobj; - struct SetChannelPlan_param *setChannelPlan_param; - struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - - u8 res = _SUCCESS; - - /* check if allow software config */ - if (swconfig && rtw_hal_is_disable_sw_channel_plan(padapter)) { - res = _FAIL; - goto exit; - } - - /* check input parameter */ - if (!rtw_is_channel_plan_valid(chplan)) { - res = _FAIL; - goto exit; - } - - /* prepare cmd parameter */ - setChannelPlan_param = rtw_zmalloc(sizeof(struct SetChannelPlan_param)); - if (!setChannelPlan_param) { - res = _FAIL; - goto exit; - } - setChannelPlan_param->channel_plan = chplan; - - if (enqueue) { - /* need enqueue, prepare cmd_obj and enqueue */ - pcmdobj = rtw_zmalloc(sizeof(struct cmd_obj)); - if (!pcmdobj) { - kfree(setChannelPlan_param); - res = _FAIL; - goto exit; - } - - init_h2fwcmd_w_parm_no_rsp(pcmdobj, setChannelPlan_param, GEN_CMD_CODE(_SetChannelPlan)); - res = rtw_enqueue_cmd(pcmdpriv, pcmdobj); - } else { - /* no need to enqueue, do the cmd hdl directly and free cmd parameter */ - if (set_chplan_hdl(padapter, (unsigned char *)setChannelPlan_param) != H2C_SUCCESS) - res = _FAIL; - - kfree(setChannelPlan_param); - } - - /* do something based on res... */ - if (res == _SUCCESS) - padapter->mlmepriv.ChannelPlan = chplan; - -exit: - return res; -} - static void collect_traffic_statistics(struct adapter *padapter) { struct dvobj_priv *pdvobjpriv = adapter_to_dvobj(padapter); diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index 68e41d99679d875f25e0dbb1d4af5edd2adc0e85..3d8a64f694481b21357e7185334faf78180f2687 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -634,23 +634,6 @@ void rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie } } -u8 rtw_is_wps_ie(u8 *ie_ptr, uint *wps_ielen) -{ - u8 match = false; - u8 eid, wps_oui[4] = {0x0, 0x50, 0xf2, 0x04}; - - if (!ie_ptr) - return match; - - eid = ie_ptr[0]; - - if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&ie_ptr[2], wps_oui, 4))) { - *wps_ielen = ie_ptr[1]+2; - match = true; - } - return match; -} - /** * rtw_get_wps_ie - Search WPS IE from a series of IEs * @in_ie: Address of IEs to search diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index f2242cf2dfb4b6396c250ba718f2be134f71a60b..6498fd17e1d3e2ca28fd2b9e3934bd9e06482951 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -2521,7 +2521,7 @@ void rtw_issue_addbareq_cmd(struct adapter *padapter, struct xmit_frame *pxmitfr { u8 issued; int priority; - struct sta_info *psta = NULL; + struct sta_info *psta; struct ht_priv *phtpriv; struct pkt_attrib *pattrib = &pxmitframe->attrib; s32 bmcst = IS_MCAST(pattrib->ra); diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index f878b04076d8697702ce03484a853b12e9cc39f3..8e74b4f47b9473bcb2fd290b4584609fc391e63f 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -5945,27 +5945,6 @@ int rtw_chk_start_clnt_join(struct adapter *padapter, u8 *ch, u8 *bw, u8 *offset return connect_allow ? _SUCCESS : _FAIL; } -/* Find union about ch, bw, ch_offset of all linked/linking interfaces */ -int rtw_get_ch_setting_union(struct adapter *adapter, u8 *ch, u8 *bw, u8 *offset) -{ - struct dvobj_priv *dvobj = adapter_to_dvobj(adapter); - struct adapter *iface; - - if (ch) - *ch = 0; - if (bw) - *bw = CHANNEL_WIDTH_20; - if (offset) - *offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE; - - iface = dvobj->padapters; - - if (!check_fwstate(&iface->mlmepriv, _FW_LINKED|_FW_UNDER_LINKING)) - return 0; - - return 1; -} - u8 set_ch_hdl(struct adapter *padapter, u8 *pbuf) { struct set_ch_parm *set_ch_parm; diff --git a/drivers/staging/rtl8723bs/core/rtw_odm.c b/drivers/staging/rtl8723bs/core/rtw_odm.c deleted file mode 100644 index f6b73a2a0270fb5e601b192596e72e780a5254a2..0000000000000000000000000000000000000000 --- a/drivers/staging/rtl8723bs/core/rtw_odm.c +++ /dev/null @@ -1,195 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/****************************************************************************** - * - * Copyright(c) 2013 Realtek Corporation. All rights reserved. - * - ******************************************************************************/ - -#include -#include -#include -#include - -static const char * const odm_comp_str[] = { - /* BIT0 */"ODM_COMP_DIG", - /* BIT1 */"ODM_COMP_RA_MASK", - /* BIT2 */"ODM_COMP_DYNAMIC_TXPWR", - /* BIT3 */"ODM_COMP_FA_CNT", - /* BIT4 */"ODM_COMP_RSSI_MONITOR", - /* BIT5 */"ODM_COMP_CCK_PD", - /* BIT6 */"ODM_COMP_ANT_DIV", - /* BIT7 */"ODM_COMP_PWR_SAVE", - /* BIT8 */"ODM_COMP_PWR_TRAIN", - /* BIT9 */"ODM_COMP_RATE_ADAPTIVE", - /* BIT10 */"ODM_COMP_PATH_DIV", - /* BIT11 */"ODM_COMP_PSD", - /* BIT12 */"ODM_COMP_DYNAMIC_PRICCA", - /* BIT13 */"ODM_COMP_RXHP", - /* BIT14 */"ODM_COMP_MP", - /* BIT15 */"ODM_COMP_DYNAMIC_ATC", - /* BIT16 */"ODM_COMP_EDCA_TURBO", - /* BIT17 */"ODM_COMP_EARLY_MODE", - /* BIT18 */NULL, - /* BIT19 */NULL, - /* BIT20 */NULL, - /* BIT21 */NULL, - /* BIT22 */NULL, - /* BIT23 */NULL, - /* BIT24 */"ODM_COMP_TX_PWR_TRACK", - /* BIT25 */"ODM_COMP_RX_GAIN_TRACK", - /* BIT26 */"ODM_COMP_CALIBRATION", - /* BIT27 */NULL, - /* BIT28 */NULL, - /* BIT29 */NULL, - /* BIT30 */"ODM_COMP_COMMON", - /* BIT31 */"ODM_COMP_INIT", -}; - -#define RTW_ODM_COMP_MAX 32 - -static const char * const odm_ability_str[] = { - /* BIT0 */"ODM_BB_DIG", - /* BIT1 */"ODM_BB_RA_MASK", - /* BIT2 */"ODM_BB_DYNAMIC_TXPWR", - /* BIT3 */"ODM_BB_FA_CNT", - /* BIT4 */"ODM_BB_RSSI_MONITOR", - /* BIT5 */"ODM_BB_CCK_PD", - /* BIT6 */"ODM_BB_ANT_DIV", - /* BIT7 */"ODM_BB_PWR_SAVE", - /* BIT8 */"ODM_BB_PWR_TRAIN", - /* BIT9 */"ODM_BB_RATE_ADAPTIVE", - /* BIT10 */"ODM_BB_PATH_DIV", - /* BIT11 */"ODM_BB_PSD", - /* BIT12 */"ODM_BB_RXHP", - /* BIT13 */"ODM_BB_ADAPTIVITY", - /* BIT14 */"ODM_BB_DYNAMIC_ATC", - /* BIT15 */NULL, - /* BIT16 */"ODM_MAC_EDCA_TURBO", - /* BIT17 */"ODM_MAC_EARLY_MODE", - /* BIT18 */NULL, - /* BIT19 */NULL, - /* BIT20 */NULL, - /* BIT21 */NULL, - /* BIT22 */NULL, - /* BIT23 */NULL, - /* BIT24 */"ODM_RF_TX_PWR_TRACK", - /* BIT25 */"ODM_RF_RX_GAIN_TRACK", - /* BIT26 */"ODM_RF_CALIBRATION", -}; - -#define RTW_ODM_ABILITY_MAX 27 - -static const char * const odm_dbg_level_str[] = { - NULL, - "ODM_DBG_OFF", - "ODM_DBG_SERIOUS", - "ODM_DBG_WARNING", - "ODM_DBG_LOUD", - "ODM_DBG_TRACE", -}; - -#define RTW_ODM_DBG_LEVEL_NUM 6 - -void rtw_odm_dbg_comp_msg(struct adapter *adapter) -{ - u64 dbg_comp; - int i; - - rtw_hal_get_def_var(adapter, HW_DEF_ODM_DBG_FLAG, &dbg_comp); - netdev_dbg(adapter->pnetdev, "odm.DebugComponents = 0x%016llx\n", - dbg_comp); - for (i = 0; i < RTW_ODM_COMP_MAX; i++) { - if (odm_comp_str[i]) - netdev_dbg(adapter->pnetdev, "%cBIT%-2d %s\n", - (BIT0 << i) & dbg_comp ? '+' : ' ', i, - odm_comp_str[i]); - } -} - -inline void rtw_odm_dbg_comp_set(struct adapter *adapter, u64 comps) -{ - rtw_hal_set_def_var(adapter, HW_DEF_ODM_DBG_FLAG, &comps); -} - -void rtw_odm_dbg_level_msg(void *sel, struct adapter *adapter) -{ - u32 dbg_level; - int i; - - rtw_hal_get_def_var(adapter, HW_DEF_ODM_DBG_LEVEL, &dbg_level); - netdev_dbg(adapter->pnetdev, "odm.DebugLevel = %u\n", dbg_level); - for (i = 0; i < RTW_ODM_DBG_LEVEL_NUM; i++) { - if (odm_dbg_level_str[i]) - netdev_dbg(adapter->pnetdev, "%u %s\n", i, - odm_dbg_level_str[i]); - } -} - -inline void rtw_odm_dbg_level_set(struct adapter *adapter, u32 level) -{ - rtw_hal_set_def_var(adapter, HW_DEF_ODM_DBG_LEVEL, &level); -} - -void rtw_odm_ability_msg(void *sel, struct adapter *adapter) -{ - u32 ability = 0; - int i; - - rtw_hal_get_hwreg(adapter, HW_VAR_DM_FLAG, (u8 *)&ability); - netdev_dbg(adapter->pnetdev, "odm.SupportAbility = 0x%08x\n", ability); - for (i = 0; i < RTW_ODM_ABILITY_MAX; i++) { - if (odm_ability_str[i]) - netdev_dbg(adapter->pnetdev, "%cBIT%-2d %s\n", - (BIT0 << i) & ability ? '+' : ' ', i, - odm_ability_str[i]); - } -} - -inline void rtw_odm_ability_set(struct adapter *adapter, u32 ability) -{ - rtw_hal_set_hwreg(adapter, HW_VAR_DM_FLAG, (u8 *)&ability); -} - -void rtw_odm_adaptivity_parm_msg(void *sel, struct adapter *adapter) -{ - struct hal_com_data *pHalData = GET_HAL_DATA(adapter); - struct dm_odm_t *odm = &pHalData->odmpriv; - - netdev_dbg(adapter->pnetdev, "%10s %16s %8s %10s %11s %14s\n", - "TH_L2H_ini", "TH_EDCCA_HL_diff", "IGI_Base", "ForceEDCCA", - "AdapEn_RSSI", "IGI_LowerBound"); - netdev_dbg(adapter->pnetdev, - "0x%-8x %-16d 0x%-6x %-10d %-11u %-14u\n", - (u8)odm->TH_L2H_ini, - odm->TH_EDCCA_HL_diff, - odm->IGI_Base, - odm->ForceEDCCA, - odm->AdapEn_RSSI, - odm->IGI_LowerBound); -} - -void rtw_odm_adaptivity_parm_set(struct adapter *adapter, s8 TH_L2H_ini, - s8 TH_EDCCA_HL_diff, s8 IGI_Base, - bool ForceEDCCA, u8 AdapEn_RSSI, - u8 IGI_LowerBound) -{ - struct hal_com_data *pHalData = GET_HAL_DATA(adapter); - struct dm_odm_t *odm = &pHalData->odmpriv; - - odm->TH_L2H_ini = TH_L2H_ini; - odm->TH_EDCCA_HL_diff = TH_EDCCA_HL_diff; - odm->IGI_Base = IGI_Base; - odm->ForceEDCCA = ForceEDCCA; - odm->AdapEn_RSSI = AdapEn_RSSI; - odm->IGI_LowerBound = IGI_LowerBound; -} - -void rtw_odm_get_perpkt_rssi(void *sel, struct adapter *adapter) -{ - struct hal_com_data *hal_data = GET_HAL_DATA(adapter); - struct dm_odm_t *odm = &hal_data->odmpriv; - - netdev_dbg(adapter->pnetdev, - "RxRate = %s, RSSI_A = %d(%%), RSSI_B = %d(%%)\n", - HDATA_RATE(odm->RxRate), odm->RSSI_A, odm->RSSI_B); -} diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c index d8d394b67eebc7fad95051445406536e0adb1173..2825375bff94289351d5cf65fe9bce6b5b21f48b 100644 --- a/drivers/staging/rtl8723bs/core/rtw_recv.c +++ b/drivers/staging/rtl8723bs/core/rtw_recv.c @@ -203,22 +203,12 @@ signed int rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *q } /* -signed int rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *queue) -{ - return rtw_free_recvframe(precvframe, queue); -} -*/ - - - - -/* -caller : defrag ; recvframe_chk_defrag in recv_thread (passive) -pframequeue: defrag_queue : will be accessed in recv_thread (passive) - -using spinlock to protect - -*/ + * caller : defrag ; recvframe_chk_defrag in recv_thread (passive) + * pframequeue: defrag_queue : will be accessed in recv_thread (passive) + * + * using spinlock to protect + * + */ void rtw_free_recvframe_queue(struct __queue *pframequeue, struct __queue *pfree_recv_queue) { @@ -245,6 +235,7 @@ u32 rtw_free_uc_swdec_pending_queue(struct adapter *adapter) { u32 cnt = 0; union recv_frame *pending_frame; + while ((pending_frame = rtw_alloc_recvframe(&adapter->recvpriv.uc_swdec_pending_queue))) { rtw_free_recvframe(pending_frame, &adapter->recvpriv.free_recv_queue); cnt++; @@ -397,6 +388,7 @@ static union recv_frame *decryptor(struct adapter *padapter, union recv_frame *p if (prxattrib->encrypt > 0) { u8 *iv = precv_frame->u.hdr.rx_data+prxattrib->hdrlen; + prxattrib->key_index = (((iv[3])>>6)&0x3); if (prxattrib->key_index > WEP_KEYS) { @@ -882,6 +874,7 @@ static signed int sta2ap_data_frame(struct adapter *adapter, union recv_frame *p } } else { u8 *myhwaddr = myid(&adapter->eeprompriv); + if (memcmp(pattrib->ra, myhwaddr, ETH_ALEN)) { ret = RTW_RX_HANDLED; goto exit; @@ -1125,6 +1118,7 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re psta = rtw_get_stainfo(pstapriv, psta_addr); if (!psta) { u8 type = GetFrameType(pfhdr->rx_data); + if (type != WIFI_DATA_TYPE) { psta = rtw_get_bcmc_stainfo(padapter); pdefrag_q = &psta->sta_recvpriv.defrag_q; @@ -1207,6 +1201,7 @@ static signed int validate_recv_mgnt_frame(struct adapter *padapter, union recv_ { /* for rx pkt statistics */ struct sta_info *psta = rtw_get_stainfo(&padapter->stapriv, GetAddr2Ptr(precv_frame->u.hdr.rx_data)); + if (psta) { psta->sta_stats.rx_mgnt_pkts++; if (GetFrameSubType(precv_frame->u.hdr.rx_data) == WIFI_BEACON) @@ -1374,9 +1369,8 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame /* actual management data frame body */ data_len = pattrib->pkt_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; mgmt_DATA = rtw_zmalloc(data_len); - if (!mgmt_DATA) { + if (!mgmt_DATA) goto validate_80211w_fail; - } precv_frame = decryptor(adapter, precv_frame); /* save actual management data frame body */ memcpy(mgmt_DATA, ptr+pattrib->hdrlen+pattrib->iv_len, data_len); @@ -1385,9 +1379,8 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame /* remove the iv and icv length */ pattrib->pkt_len = pattrib->pkt_len - pattrib->iv_len - pattrib->icv_len; kfree(mgmt_DATA); - if (!precv_frame) { + if (!precv_frame) goto validate_80211w_fail; - } } else if (IS_MCAST(GetAddr1Ptr(ptr)) && (subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC)) { signed int BIP_ret = _SUCCESS; @@ -1480,6 +1473,7 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame retval = validate_recv_data_frame(adapter, precv_frame); if (retval == _FAIL) { struct recv_priv *precvpriv = &adapter->recvpriv; + precvpriv->rx_drop++; } else if (retval == _SUCCESS) { #ifdef DBG_RX_DUMP_EAP @@ -1651,14 +1645,12 @@ static int check_indicate_seq(struct recv_reorder_ctrl *preorder_ctrl, u16 seq_n u16 wend = (preorder_ctrl->indicate_seq + wsize - 1) & 0xFFF;/* 4096; */ /* Rx Reorder initialize condition. */ - if (preorder_ctrl->indicate_seq == 0xFFFF) { + if (preorder_ctrl->indicate_seq == 0xFFFF) preorder_ctrl->indicate_seq = seq_num; - } /* Drop out the packet which SeqNum is smaller than WinStart */ - if (SN_LESS(seq_num, preorder_ctrl->indicate_seq)) { + if (SN_LESS(seq_num, preorder_ctrl->indicate_seq)) return false; - } /* */ /* Sliding window manipulation. Conditions includes: */ @@ -2084,10 +2076,8 @@ s32 rtw_recv_entry(union recv_frame *precvframe) precvpriv = &padapter->recvpriv; ret = recv_func(padapter, precvframe); - if (ret == _FAIL) { + if (ret == _FAIL) goto _recv_entry_drop; - } - precvpriv->rx_pkts++; diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c index 9acd49323c7c6dd4f0b3a15f8888956beedcd0fc..e36f8c369a0487250871dbdf9a1e296eb9848e7d 100644 --- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c +++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c @@ -1283,11 +1283,6 @@ s32 hal_btcoex_IsBTCoexCtrlAMPDUSize(struct adapter *padapter) return (s32)GLBtCoexist.btInfo.bBtCtrlAggBufSize; } -void hal_btcoex_SetManualControl(struct adapter *padapter, u8 bmanual) -{ - GLBtCoexist.bManualControl = bmanual; -} - bool hal_btcoex_IsBtControlLps(struct adapter *padapter) { if (!hal_btcoex_IsBtExist(padapter)) diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c index 909b37bcc8974778e9e49d0ffd9c3b1e542b0997..e42556d03bce96229e5fb008cd69b6bbe232bef9 100644 --- a/drivers/staging/rtl8723bs/hal/hal_com.c +++ b/drivers/staging/rtl8723bs/hal/hal_com.c @@ -859,25 +859,6 @@ bool eqNByte(u8 *str1, u8 *str2, u32 num) return true; } -/* */ -/* Description: */ -/* Return true if chTmp is represent for hex digit and */ -/* false otherwise. */ -/* */ -/* */ -bool IsHexDigit(char chTmp) -{ - if ( - (chTmp >= '0' && chTmp <= '9') || - (chTmp >= 'a' && chTmp <= 'f') || - (chTmp >= 'A' && chTmp <= 'F') - ) - return true; - else - return false; -} - - /* */ /* Description: */ /* Translate a character to hex digit. */ @@ -894,106 +875,6 @@ u32 MapCharToHexDigit(char chTmp) return 0; } - - -/* Description: */ -/* Parse hex number from the string pucStr. */ -bool GetHexValueFromString(char *szStr, u32 *pu4bVal, u32 *pu4bMove) -{ - char *szScan = szStr; - - /* Check input parameter. */ - if (!szStr || !pu4bVal || !pu4bMove) - return false; - - /* Initialize output. */ - *pu4bMove = 0; - *pu4bVal = 0; - - /* Skip leading space. */ - while (*szScan != '\0' && (*szScan == ' ' || *szScan == '\t')) { - szScan++; - (*pu4bMove)++; - } - - /* Skip leading '0x' or '0X'. */ - if (*szScan == '0' && (*(szScan+1) == 'x' || *(szScan+1) == 'X')) { - szScan += 2; - (*pu4bMove) += 2; - } - - /* Check if szScan is now pointer to a character for hex digit, */ - /* if not, it means this is not a valid hex number. */ - if (!IsHexDigit(*szScan)) - return false; - - /* Parse each digit. */ - do { - (*pu4bVal) <<= 4; - *pu4bVal += MapCharToHexDigit(*szScan); - - szScan++; - (*pu4bMove)++; - } while (IsHexDigit(*szScan)); - - return true; -} - -bool GetFractionValueFromString( - char *szStr, u8 *pInteger, u8 *pFraction, u32 *pu4bMove -) -{ - char *szScan = szStr; - - /* Initialize output. */ - *pu4bMove = 0; - *pInteger = 0; - *pFraction = 0; - - /* Skip leading space. */ - while (*szScan != '\0' && (*szScan == ' ' || *szScan == '\t')) { - ++szScan; - ++(*pu4bMove); - } - - /* Parse each digit. */ - do { - (*pInteger) *= 10; - *pInteger += (*szScan - '0'); - - ++szScan; - ++(*pu4bMove); - - if (*szScan == '.') { - ++szScan; - ++(*pu4bMove); - - if (*szScan < '0' || *szScan > '9') - return false; - else { - *pFraction = *szScan - '0'; - ++szScan; - ++(*pu4bMove); - return true; - } - } - } while (*szScan >= '0' && *szScan <= '9'); - - return true; -} - -/* */ -/* Description: */ -/* Return true if szStr is comment out with leading "//". */ -/* */ -bool IsCommentString(char *szStr) -{ - if (*szStr == '/' && *(szStr+1) == '/') - return true; - else - return false; -} - bool GetU1ByteIntegerFromStringInDecimal(char *Str, u8 *pInt) { u16 i = 0; diff --git a/drivers/staging/rtl8723bs/hal/hal_intf.c b/drivers/staging/rtl8723bs/hal/hal_intf.c index 94ecefb9113d7b2443fb8a60557e49c49136bc39..6bb0ff8d7c78ac2bbc913e0040a4ab3552be8ef1 100644 --- a/drivers/staging/rtl8723bs/hal/hal_intf.c +++ b/drivers/staging/rtl8723bs/hal/hal_intf.c @@ -400,11 +400,6 @@ c2h_id_filter rtw_hal_c2h_id_filter_ccx(struct adapter *adapter) return adapter->HalFunc.c2h_id_filter_ccx; } -s32 rtw_hal_is_disable_sw_channel_plan(struct adapter *padapter) -{ - return GET_HAL_DATA(padapter)->bDisableSWChannelPlan; -} - s32 rtw_hal_macid_sleep(struct adapter *padapter, u32 macid) { u8 support; diff --git a/drivers/staging/rtl8723bs/hal/odm.h b/drivers/staging/rtl8723bs/hal/odm.h index 19cfc2915458e6c7ffc48136a43c66f77a2a3e50..fe9782d2d4fd23699569c3e6ae68fe9ace665a7f 100644 --- a/drivers/staging/rtl8723bs/hal/odm.h +++ b/drivers/staging/rtl8723bs/hal/odm.h @@ -14,7 +14,6 @@ #include "odm_DynamicBBPowerSaving.h" #include "odm_DynamicTxPower.h" #include "odm_CfoTracking.h" -#include "odm_NoiseMonitor.h" #define TP_MODE 0 #define RSSI_MODE 1 @@ -863,7 +862,6 @@ struct dm_odm_t { /* DM_Out_Source_Dynamic_Mechanism_Structure */ u8 Adaptivity_IGI_upper; u8 NHM_cnt_0; - struct odm_noise_monitor noise_level;/* ODM_MAX_CHANNEL_NUM]; */ /* */ /* 2 Define STA info. */ /* _ODM_STA_INFO */ diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.c b/drivers/staging/rtl8723bs/hal/odm_DIG.c index 7e92c373cddbe966c258194fb4b221b948855720..07edf74ccfe5224c0f9b2156b38c0147aaef7f3b 100644 --- a/drivers/staging/rtl8723bs/hal/odm_DIG.c +++ b/drivers/staging/rtl8723bs/hal/odm_DIG.c @@ -309,63 +309,6 @@ void ODM_Write_DIG(void *pDM_VOID, u8 CurrentIGI) } -void odm_PauseDIG( - void *pDM_VOID, - enum ODM_Pause_DIG_TYPE PauseType, - u8 IGIValue -) -{ - struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID; - struct dig_t *pDM_DigTable = &pDM_Odm->DM_DigTable; - static bool bPaused; - - if ( - (pDM_Odm->SupportAbility & ODM_BB_ADAPTIVITY) && - pDM_Odm->TxHangFlg == true - ) { - return; - } - - if ( - !bPaused && (!(pDM_Odm->SupportAbility & ODM_BB_DIG) || - !(pDM_Odm->SupportAbility & ODM_BB_FA_CNT)) - ){ - return; - } - - switch (PauseType) { - /* 1 Pause DIG */ - case ODM_PAUSE_DIG: - /* 2 Disable DIG */ - ODM_CmnInfoUpdate(pDM_Odm, ODM_CMNINFO_ABILITY, pDM_Odm->SupportAbility & (~ODM_BB_DIG)); - - /* 2 Backup IGI value */ - if (!bPaused) { - pDM_DigTable->IGIBackup = pDM_DigTable->CurIGValue; - bPaused = true; - } - - /* 2 Write new IGI value */ - ODM_Write_DIG(pDM_Odm, IGIValue); - break; - - /* 1 Resume DIG */ - case ODM_RESUME_DIG: - if (bPaused) { - /* 2 Write backup IGI value */ - ODM_Write_DIG(pDM_Odm, pDM_DigTable->IGIBackup); - bPaused = false; - - /* 2 Enable DIG */ - ODM_CmnInfoUpdate(pDM_Odm, ODM_CMNINFO_ABILITY, pDM_Odm->SupportAbility | ODM_BB_DIG); - } - break; - - default: - break; - } -} - bool odm_DigAbort(void *pDM_VOID) { struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID; diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.h b/drivers/staging/rtl8723bs/hal/odm_DIG.h index 88cfd542df16bef4c1577aa3e7fcfefcc64dd70e..a5b041101c895ee8733bfca9418d8c3f64eef7c1 100644 --- a/drivers/staging/rtl8723bs/hal/odm_DIG.h +++ b/drivers/staging/rtl8723bs/hal/odm_DIG.h @@ -141,8 +141,6 @@ void odm_Adaptivity(void *pDM_VOID, u8 IGI); void ODM_Write_DIG(void *pDM_VOID, u8 CurrentIGI); -void odm_PauseDIG(void *pDM_VOID, enum ODM_Pause_DIG_TYPE PauseType, u8 IGIValue); - void odm_DIGInit(void *pDM_VOID); void odm_DIG(void *pDM_VOID); diff --git a/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.c b/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.c deleted file mode 100644 index 392cc8a398f55805c3f74206423a15d07c37f4dc..0000000000000000000000000000000000000000 --- a/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.c +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/****************************************************************************** - * - * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved. - * - ******************************************************************************/ - -#include "odm_precomp.h" - -/* This function is for inband noise test utility only */ -/* To obtain the inband noise level(dbm), do the following. */ -/* 1. disable DIG and Power Saving */ -/* 2. Set initial gain = 0x1a */ -/* 3. Stop updating idle time pwer report (for driver read) */ -/* - 0x80c[25] */ - -#define Valid_Min -35 -#define Valid_Max 10 -#define ValidCnt 5 - -static s16 odm_InbandNoise_Monitor_NSeries( - struct dm_odm_t *pDM_Odm, - u8 bPauseDIG, - u8 IGIValue, - u32 max_time -) -{ - u32 tmp4b; - u8 max_rf_path = 0, rf_path; - u8 reg_c50, reg_c58, valid_done = 0; - struct noise_level noise_data; - u32 start = 0; - - pDM_Odm->noise_level.noise_all = 0; - - max_rf_path = 1; - - memset(&noise_data, 0, sizeof(struct noise_level)); - - /* */ - /* Step 1. Disable DIG && Set initial gain. */ - /* */ - - if (bPauseDIG) - odm_PauseDIG(pDM_Odm, ODM_PAUSE_DIG, IGIValue); - /* */ - /* Step 2. Disable all power save for read registers */ - /* */ - /* dcmd_DebugControlPowerSave(padapter, PSDisable); */ - - /* */ - /* Step 3. Get noise power level */ - /* */ - start = jiffies; - while (1) { - - /* Stop updating idle time pwer report (for driver read) */ - PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_TxGainStage, BIT25, 1); - - /* Read Noise Floor Report */ - tmp4b = PHY_QueryBBReg(pDM_Odm->Adapter, 0x8f8, bMaskDWord); - - /* PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XAAGCCore1, bMaskByte0, TestInitialGain); */ - /* if (max_rf_path == 2) */ - /* PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XBAGCCore1, bMaskByte0, TestInitialGain); */ - - /* update idle time pwer report per 5us */ - PHY_SetBBReg(pDM_Odm->Adapter, rFPGA0_TxGainStage, BIT25, 0); - - noise_data.value[RF_PATH_A] = (u8)(tmp4b&0xff); - noise_data.value[RF_PATH_B] = (u8)((tmp4b&0xff00)>>8); - - for (rf_path = RF_PATH_A; rf_path < max_rf_path; rf_path++) { - noise_data.sval[rf_path] = (s8)noise_data.value[rf_path]; - noise_data.sval[rf_path] /= 2; - } - /* mdelay(10); */ - /* msleep(10); */ - - for (rf_path = RF_PATH_A; rf_path < max_rf_path; rf_path++) { - if ((noise_data.valid_cnt[rf_path] < ValidCnt) && (noise_data.sval[rf_path] < Valid_Max && noise_data.sval[rf_path] >= Valid_Min)) { - noise_data.valid_cnt[rf_path]++; - noise_data.sum[rf_path] += noise_data.sval[rf_path]; - if (noise_data.valid_cnt[rf_path] == ValidCnt) { - valid_done++; - } - - } - - } - - /* printk("####### valid_done:%d #############\n", valid_done); */ - if ((valid_done == max_rf_path) || (jiffies_to_msecs(jiffies - start) > max_time)) { - for (rf_path = RF_PATH_A; rf_path < max_rf_path; rf_path++) { - /* printk("%s PATH_%d - sum = %d, valid_cnt = %d\n", __func__, rf_path, noise_data.sum[rf_path], noise_data.valid_cnt[rf_path]); */ - if (noise_data.valid_cnt[rf_path]) - noise_data.sum[rf_path] /= noise_data.valid_cnt[rf_path]; - else - noise_data.sum[rf_path] = 0; - } - break; - } - } - reg_c50 = (s32)PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XAAGCCore1, bMaskByte0); - reg_c50 &= ~BIT7; - pDM_Odm->noise_level.noise[RF_PATH_A] = -110 + reg_c50 + noise_data.sum[RF_PATH_A]; - pDM_Odm->noise_level.noise_all += pDM_Odm->noise_level.noise[RF_PATH_A]; - - if (max_rf_path == 2) { - reg_c58 = (s32)PHY_QueryBBReg(pDM_Odm->Adapter, rOFDM0_XBAGCCore1, bMaskByte0); - reg_c58 &= ~BIT7; - pDM_Odm->noise_level.noise[RF_PATH_B] = -110 + reg_c58 + noise_data.sum[RF_PATH_B]; - pDM_Odm->noise_level.noise_all += pDM_Odm->noise_level.noise[RF_PATH_B]; - } - pDM_Odm->noise_level.noise_all /= max_rf_path; - - /* */ - /* Step 4. Recover the Dig */ - /* */ - if (bPauseDIG) - odm_PauseDIG(pDM_Odm, ODM_RESUME_DIG, IGIValue); - - return pDM_Odm->noise_level.noise_all; - -} - -s16 ODM_InbandNoise_Monitor(void *pDM_VOID, u8 bPauseDIG, u8 IGIValue, u32 max_time) -{ - return odm_InbandNoise_Monitor_NSeries(pDM_VOID, bPauseDIG, IGIValue, max_time); -} diff --git a/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.h b/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.h deleted file mode 100644 index ab114543f39c3ff567e5dbdb4fa7deafd4b7d7eb..0000000000000000000000000000000000000000 --- a/drivers/staging/rtl8723bs/hal/odm_NoiseMonitor.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/****************************************************************************** - * - * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved. - * - *****************************************************************************/ -#ifndef __ODMNOISEMONITOR_H__ -#define __ODMNOISEMONITOR_H__ - -#define ODM_MAX_CHANNEL_NUM 38/* 14+24 */ -struct noise_level { - /* u8 value_a, value_b; */ - u8 value[MAX_RF_PATH]; - /* s8 sval_a, sval_b; */ - s8 sval[MAX_RF_PATH]; - - /* s32 noise_a = 0, noise_b = 0, sum_a = 0, sum_b = 0; */ - /* s32 noise[ODM_RF_PATH_MAX]; */ - s32 sum[MAX_RF_PATH]; - /* u8 valid_cnt_a = 0, valid_cnt_b = 0, */ - u8 valid[MAX_RF_PATH]; - u8 valid_cnt[MAX_RF_PATH]; - -}; - - -struct odm_noise_monitor { - s8 noise[MAX_RF_PATH]; - s16 noise_all; -}; - -s16 ODM_InbandNoise_Monitor( - void *pDM_VOID, - u8 bPauseDIG, - u8 IGIValue, - u32 max_time -); - -#endif diff --git a/drivers/staging/rtl8723bs/hal/odm_precomp.h b/drivers/staging/rtl8723bs/hal/odm_precomp.h index edce506022a5c98ee5be10a0dca64b89f2d04046..2987857a8761fa7a64751ca853e590965be571fd 100644 --- a/drivers/staging/rtl8723bs/hal/odm_precomp.h +++ b/drivers/staging/rtl8723bs/hal/odm_precomp.h @@ -33,7 +33,6 @@ #include "odm_DynamicBBPowerSaving.h" #include "odm_DynamicTxPower.h" #include "odm_CfoTracking.h" -#include "odm_NoiseMonitor.h" #include "HalPhyRf.h" #include "HalPhyRf_8723B.h"/* for IQK, LCK, Power-tracking */ #include "rtl8723b_hal.h" diff --git a/drivers/staging/rtl8723bs/include/drv_types.h b/drivers/staging/rtl8723bs/include/drv_types.h index 0bbbdebdf15719b90da5bab40917ed5afa0041ed..82159e1c7f9b63d57e3e2fd6a32f105eecfe0f4b 100644 --- a/drivers/staging/rtl8723bs/include/drv_types.h +++ b/drivers/staging/rtl8723bs/include/drv_types.h @@ -50,7 +50,6 @@ #include #include #include -#include #include "ioctl_cfg80211.h" @@ -493,8 +492,6 @@ static inline u8 *myid(struct eeprom_priv *peepriv) #include -int rtw_change_ifname(struct adapter *padapter, const char *ifname); - extern char *rtw_initmac; extern int rtw_mc2u_disable; extern int rtw_ht_enable; diff --git a/drivers/staging/rtl8723bs/include/hal_btcoex.h b/drivers/staging/rtl8723bs/include/hal_btcoex.h index 78599d3521bf374429446f5f5b450fd58b86d020..fb167642da01d3e5b3e82e687e21c8ddad3acfca 100644 --- a/drivers/staging/rtl8723bs/include/hal_btcoex.h +++ b/drivers/staging/rtl8723bs/include/hal_btcoex.h @@ -45,7 +45,6 @@ void hal_btcoex_HaltNotify(struct adapter *padapter); void hal_btcoex_Handler(struct adapter *padapter); s32 hal_btcoex_IsBTCoexCtrlAMPDUSize(struct adapter *padapter); -void hal_btcoex_SetManualControl(struct adapter *padapter, u8 bmanual); bool hal_btcoex_IsBtControlLps(struct adapter *padapter); bool hal_btcoex_IsLpsOn(struct adapter *padapter); u8 hal_btcoex_RpwmVal(struct adapter *); diff --git a/drivers/staging/rtl8723bs/include/hal_com.h b/drivers/staging/rtl8723bs/include/hal_com.h index 7be0ea20bca4ac132a7cef41ec0b28838fb4e08d..6356b8c2ef814a32d2fa20a0b8cd39dd392d18ac 100644 --- a/drivers/staging/rtl8723bs/include/hal_com.h +++ b/drivers/staging/rtl8723bs/include/hal_com.h @@ -147,17 +147,8 @@ u8 GetHalDefVar(struct adapter *adapter, enum hal_def_variable variable, bool eqNByte(u8 *str1, u8 *str2, u32 num); -bool IsHexDigit(char chTmp); - u32 MapCharToHexDigit(char chTmp); -bool GetHexValueFromString(char *szStr, u32 *pu4bVal, u32 *pu4bMove); - -bool GetFractionValueFromString(char *szStr, u8 *pInteger, u8 *pFraction, - u32 *pu4bMove); - -bool IsCommentString(char *szStr); - bool ParseQualifiedString(char *In, u32 *Start, char *Out, char LeftQualifier, char RightQualifier); diff --git a/drivers/staging/rtl8723bs/include/hal_intf.h b/drivers/staging/rtl8723bs/include/hal_intf.h index 45bebbadb7ca9b77193d420aef3d4475350144d7..5cffab2d06ffc1488b0ea029568ca62a2aa1e59d 100644 --- a/drivers/staging/rtl8723bs/include/hal_intf.h +++ b/drivers/staging/rtl8723bs/include/hal_intf.h @@ -353,8 +353,6 @@ bool rtw_hal_c2h_valid(struct adapter *adapter, u8 *buf); s32 rtw_hal_c2h_handler(struct adapter *adapter, u8 *c2h_evt); c2h_id_filter rtw_hal_c2h_id_filter_ccx(struct adapter *adapter); -s32 rtw_hal_is_disable_sw_channel_plan(struct adapter *padapter); - s32 rtw_hal_macid_sleep(struct adapter *padapter, u32 macid); s32 rtw_hal_macid_wakeup(struct adapter *padapter, u32 macid); diff --git a/drivers/staging/rtl8723bs/include/ieee80211.h b/drivers/staging/rtl8723bs/include/ieee80211.h index 1e627dc0044d43343445b6fa14a2045180367037..9041d8dc5fb1d19e400da724eae6ed76153670f5 100644 --- a/drivers/staging/rtl8723bs/include/ieee80211.h +++ b/drivers/staging/rtl8723bs/include/ieee80211.h @@ -746,7 +746,6 @@ int rtw_parse_wpa2_ie(u8 *wpa_ie, int wpa_ie_len, int *group_cipher, int *pairwi void rtw_get_sec_ie(u8 *in_ie, uint in_len, u8 *rsn_ie, u16 *rsn_len, u8 *wpa_ie, u16 *wpa_len); -u8 rtw_is_wps_ie(u8 *ie_ptr, uint *wps_ielen); u8 *rtw_get_wps_ie(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen); u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_attr, u32 *len_attr); u8 *rtw_get_wps_attr_content(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_content, uint *len_content); diff --git a/drivers/staging/rtl8723bs/include/rtw_cmd.h b/drivers/staging/rtl8723bs/include/rtw_cmd.h index 1bf030cbbbbee1ba90d87cc83962dd8b392ef370..fe1b0310120352ab0053bee2c6ec87f75638da58 100644 --- a/drivers/staging/rtl8723bs/include/rtw_cmd.h +++ b/drivers/staging/rtl8723bs/include/rtw_cmd.h @@ -591,7 +591,6 @@ extern u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 extern u8 rtw_joinbss_cmd(struct adapter *padapter, struct wlan_network *pnetwork); u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueue); extern u8 rtw_setopmode_cmd(struct adapter *padapter, enum ndis_802_11_network_infrastructure networktype, bool enqueue); -extern u8 rtw_setdatarate_cmd(struct adapter *padapter, u8 *rateset); extern u8 rtw_setrfintfs_cmd(struct adapter *padapter, u8 mode); extern u8 rtw_gettssi_cmd(struct adapter *padapter, u8 offset, u8 *pval); @@ -613,8 +612,6 @@ extern u8 rtw_ps_cmd(struct adapter *padapter); u8 rtw_chk_hi_queue_cmd(struct adapter *padapter); -extern u8 rtw_set_chplan_cmd(struct adapter *padapter, u8 chplan, u8 enqueue, u8 swconfig); - extern u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length); extern u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt); diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h index 89b389d4c44ba843e5ee3759725d4d723ee96207..65e138a5238f57fb01db3a5fa9859baca94547c0 100644 --- a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h +++ b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h @@ -662,7 +662,6 @@ extern void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint extern u8 traffic_status_watchdog(struct adapter *padapter, u8 from_timer); int rtw_chk_start_clnt_join(struct adapter *padapter, u8 *ch, u8 *bw, u8 *offset); -int rtw_get_ch_setting_union(struct adapter *adapter, u8 *ch, u8 *bw, u8 *offset); struct cmd_hdl { uint parmsize; diff --git a/drivers/staging/rtl8723bs/include/rtw_odm.h b/drivers/staging/rtl8723bs/include/rtw_odm.h deleted file mode 100644 index 94fc68a5c424fa0d4fab0b2529b2662c139d1088..0000000000000000000000000000000000000000 --- a/drivers/staging/rtl8723bs/include/rtw_odm.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/****************************************************************************** - * - * Copyright(c) 2013 Realtek Corporation. All rights reserved. - * - ******************************************************************************/ -#ifndef __RTW_ODM_H__ -#define __RTW_ODM_H__ - -#include - -/* -* This file provides utilities/wrappers for rtw driver to use ODM -*/ - -void rtw_odm_dbg_comp_msg(struct adapter *adapter); -void rtw_odm_dbg_comp_set(struct adapter *adapter, u64 comps); -void rtw_odm_dbg_level_msg(void *sel, struct adapter *adapter); -void rtw_odm_dbg_level_set(struct adapter *adapter, u32 level); - -void rtw_odm_ability_msg(void *sel, struct adapter *adapter); -void rtw_odm_ability_set(struct adapter *adapter, u32 ability); - -void rtw_odm_adaptivity_parm_msg(void *sel, struct adapter *adapter); -void rtw_odm_adaptivity_parm_set(struct adapter *adapter, s8 TH_L2H_ini, s8 TH_EDCCA_HL_diff, - s8 IGI_Base, bool ForceEDCCA, u8 AdapEn_RSSI, u8 IGI_LowerBound); -void rtw_odm_get_perpkt_rssi(void *sel, struct adapter *adapter); -#endif /* __RTW_ODM_H__ */ diff --git a/drivers/staging/rtl8723bs/include/xmit_osdep.h b/drivers/staging/rtl8723bs/include/xmit_osdep.h index e781cd5dfd01eb864173533825b8a4739d82671a..8704dced593a199f02474364abf025b75e8a8b74 100644 --- a/drivers/staging/rtl8723bs/include/xmit_osdep.h +++ b/drivers/staging/rtl8723bs/include/xmit_osdep.h @@ -25,8 +25,8 @@ struct sta_xmit_priv; struct xmit_frame; struct xmit_buf; -extern int _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); -extern int rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); +extern void _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); +extern netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev); void rtw_os_xmit_schedule(struct adapter *padapter); diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index cb6d287f580dc24e39beef7a8626bf36acdf152b..6aeb169c6ebf0a0effdc474a0fcd1f3be8fbce84 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -198,7 +198,7 @@ static int rtw_ieee80211_channel_to_frequency(int chan, int band) if (band == NL80211_BAND_2GHZ) { if (chan == 14) return 2484; - else if (chan < 14) + else if (chan < 14) return 2407 + chan * 5; } @@ -810,7 +810,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param memcpy(padapter->securitypriv.dot118021XGrpKey[param->u.crypt.idx].skey, param->u.crypt.key, (param->u.crypt.key_len > 16 ? 16 : param->u.crypt.key_len)); memcpy(padapter->securitypriv.dot118021XGrptxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[16]), 8); memcpy(padapter->securitypriv.dot118021XGrprxmickey[param->u.crypt.idx].skey, &(param->u.crypt.key[24]), 8); - padapter->securitypriv.binstallGrpkey = true; + padapter->securitypriv.binstallGrpkey = true; padapter->securitypriv.dot118021XGrpKeyid = param->u.crypt.idx; rtw_set_key(padapter, &padapter->securitypriv, param->u.crypt.idx, 1, true); @@ -850,8 +850,8 @@ exit: } static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params) { char *alg_name; u32 param_len; @@ -920,9 +920,9 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev, ret = rtw_cfg80211_ap_set_encryption(ndev, param, param_len); } else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true - || check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) { - ret = rtw_cfg80211_set_encryption(ndev, param, param_len); - } + || check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) { + ret = rtw_cfg80211_set_encryption(ndev, param, param_len); + } addkey_end: kfree(param); @@ -932,8 +932,8 @@ addkey_end: } static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, const u8 *mac_addr, - void *cookie, + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params*)) { @@ -941,7 +941,8 @@ static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev, } static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { struct adapter *padapter = rtw_netdev_priv(ndev); struct security_priv *psecuritypriv = &padapter->securitypriv; @@ -955,7 +956,7 @@ static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev, } static int cfg80211_rtw_set_default_key(struct wiphy *wiphy, - struct net_device *ndev, u8 key_index + struct net_device *ndev, int link_id, u8 key_index , bool unicast, bool multicast ) { @@ -1065,7 +1066,7 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy, } } - if (_FAIL == rtw_pwr_wakeup(padapter)) { + if (rtw_pwr_wakeup(padapter) == _FAIL) { ret = -EPERM; goto exit; } @@ -1239,7 +1240,7 @@ static int cfg80211_rtw_scan(struct wiphy *wiphy } rtw_ps_deny(padapter, PS_DENY_SCAN); - if (_FAIL == rtw_pwr_wakeup(padapter)) { + if (rtw_pwr_wakeup(padapter) == _FAIL) { need_indicate_scan_done = true; goto check_need_indicate_scan_done; } @@ -1499,49 +1500,49 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel pairwise_cipher = WPA_CIPHER_NONE; switch (group_cipher) { - case WPA_CIPHER_NONE: - padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; - break; - case WPA_CIPHER_WEP40: - padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; - break; - case WPA_CIPHER_TKIP: - padapter->securitypriv.dot118021XGrpPrivacy = _TKIP_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled; - break; - case WPA_CIPHER_CCMP: - padapter->securitypriv.dot118021XGrpPrivacy = _AES_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled; - break; - case WPA_CIPHER_WEP104: - padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; - break; + case WPA_CIPHER_NONE: + padapter->securitypriv.dot118021XGrpPrivacy = _NO_PRIVACY_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; + break; + case WPA_CIPHER_WEP40: + padapter->securitypriv.dot118021XGrpPrivacy = _WEP40_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; + break; + case WPA_CIPHER_TKIP: + padapter->securitypriv.dot118021XGrpPrivacy = _TKIP_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled; + break; + case WPA_CIPHER_CCMP: + padapter->securitypriv.dot118021XGrpPrivacy = _AES_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled; + break; + case WPA_CIPHER_WEP104: + padapter->securitypriv.dot118021XGrpPrivacy = _WEP104_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; + break; } switch (pairwise_cipher) { - case WPA_CIPHER_NONE: - padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; - break; - case WPA_CIPHER_WEP40: - padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; - break; - case WPA_CIPHER_TKIP: - padapter->securitypriv.dot11PrivacyAlgrthm = _TKIP_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled; - break; - case WPA_CIPHER_CCMP: - padapter->securitypriv.dot11PrivacyAlgrthm = _AES_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled; - break; - case WPA_CIPHER_WEP104: - padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_; - padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; - break; + case WPA_CIPHER_NONE: + padapter->securitypriv.dot11PrivacyAlgrthm = _NO_PRIVACY_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11EncryptionDisabled; + break; + case WPA_CIPHER_WEP40: + padapter->securitypriv.dot11PrivacyAlgrthm = _WEP40_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; + break; + case WPA_CIPHER_TKIP: + padapter->securitypriv.dot11PrivacyAlgrthm = _TKIP_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption2Enabled; + break; + case WPA_CIPHER_CCMP: + padapter->securitypriv.dot11PrivacyAlgrthm = _AES_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption3Enabled; + break; + case WPA_CIPHER_WEP104: + padapter->securitypriv.dot11PrivacyAlgrthm = _WEP104_; + padapter->securitypriv.ndisencryptstatus = Ndis802_11Encryption1Enabled; + break; } {/* handle wps_ie */ @@ -1582,7 +1583,7 @@ static int cfg80211_rtw_join_ibss(struct wiphy *wiphy, struct net_device *ndev, struct mlme_priv *pmlmepriv = &padapter->mlmepriv; int ret = 0; - if (_FAIL == rtw_pwr_wakeup(padapter)) { + if (rtw_pwr_wakeup(padapter) == _FAIL) { ret = -EPERM; goto exit; } @@ -1673,7 +1674,7 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev, } rtw_ps_deny(padapter, PS_DENY_JOIN); - if (_FAIL == rtw_pwr_wakeup(padapter)) { + if (rtw_pwr_wakeup(padapter) == _FAIL) { ret = -EPERM; goto exit; } @@ -1848,6 +1849,7 @@ static int cfg80211_rtw_get_txpower(struct wiphy *wiphy, inline bool rtw_cfg80211_pwr_mgmt(struct adapter *adapter) { struct rtw_wdev_priv *rtw_wdev_priv = adapter_wdev_data(adapter); + return rtw_wdev_priv->power_mgmt; } @@ -1953,6 +1955,7 @@ void rtw_cfg80211_indicate_sta_assoc(struct adapter *padapter, u8 *pmgmt_frame, { struct station_info sinfo = {}; u8 ie_offset; + if (GetFrameSubType(pmgmt_frame) == WIFI_ASSOCREQ) ie_offset = _ASOCREQ_IE_OFFSET_; else /* WIFI_REASSOCREQ */ @@ -2084,7 +2087,8 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc memcpy(pdata + sizeof(dst_mac_addr), src_mac_addr, sizeof(src_mac_addr)); /* Use the real net device to transmit the packet */ - return _rtw_xmit_entry(skb, padapter->pnetdev); + _rtw_xmit_entry(skb, padapter->pnetdev); + return NETDEV_TX_OK; } else if ((frame_control & (IEEE80211_FCTL_FTYPE|IEEE80211_FCTL_STYPE)) == (IEEE80211_FTYPE_MGMT|IEEE80211_STYPE_ACTION)) { @@ -2347,7 +2351,7 @@ static int cfg80211_rtw_start_ap(struct wiphy *wiphy, struct net_device *ndev, } static int cfg80211_rtw_change_beacon(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_beacon_data *info) + struct cfg80211_beacon_data *info) { struct adapter *adapter = rtw_netdev_priv(ndev); @@ -2466,7 +2470,7 @@ static int cfg80211_rtw_dump_station(struct wiphy *wiphy, struct net_device *nde spin_lock_bh(&pstapriv->asoc_list_lock); psta = rtw_sta_info_get_by_idx(idx, pstapriv); spin_unlock_bh(&pstapriv->asoc_list_lock); - if (NULL == psta) { + if (psta == NULL) { ret = -ENOENT; goto exit; } @@ -2601,7 +2605,7 @@ static int cfg80211_rtw_mgmt_tx(struct wiphy *wiphy, goto exit; rtw_ps_deny(padapter, PS_DENY_MGNT_TX); - if (_FAIL == rtw_pwr_wakeup(padapter)) { + if (rtw_pwr_wakeup(padapter) == _FAIL) { ret = -EFAULT; goto cancel_ps_deny; } diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c index 380d8c9e1239eaa50a9e4979071e174684abc624..68bba3c0e757a61edde01857db0feeecc7c4cafc 100644 --- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c +++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c @@ -664,51 +664,36 @@ void rtw_reset_drv_sw(struct adapter *padapter) u8 rtw_init_drv_sw(struct adapter *padapter) { - u8 ret8 = _SUCCESS; - rtw_init_default_value(padapter); rtw_init_hal_com_default_value(padapter); - if (rtw_init_cmd_priv(&padapter->cmdpriv)) { - ret8 = _FAIL; - goto exit; - } + if (rtw_init_cmd_priv(&padapter->cmdpriv)) + return _FAIL; padapter->cmdpriv.padapter = padapter; - if (rtw_init_evt_priv(&padapter->evtpriv)) { - ret8 = _FAIL; - goto exit; - } + if (rtw_init_evt_priv(&padapter->evtpriv)) + goto free_cmd_priv; - - if (rtw_init_mlme_priv(padapter) == _FAIL) { - ret8 = _FAIL; - goto exit; - } + if (rtw_init_mlme_priv(padapter) == _FAIL) + goto free_evt_priv; init_mlme_ext_priv(padapter); - if (_rtw_init_xmit_priv(&padapter->xmitpriv, padapter) == _FAIL) { - ret8 = _FAIL; - goto exit; - } + if (_rtw_init_xmit_priv(&padapter->xmitpriv, padapter) == _FAIL) + goto free_mlme_ext; - if (_rtw_init_recv_priv(&padapter->recvpriv, padapter) == _FAIL) { - ret8 = _FAIL; - goto exit; - } + if (_rtw_init_recv_priv(&padapter->recvpriv, padapter) == _FAIL) + goto free_xmit_priv; /* add for CONFIG_IEEE80211W, none 11w also can use */ spin_lock_init(&padapter->security_key_mutex); /* We don't need to memset padapter->XXX to zero, because adapter is allocated by vzalloc(). */ /* memset((unsigned char *)&padapter->securitypriv, 0, sizeof (struct security_priv)); */ - if (_rtw_init_sta_priv(&padapter->stapriv) == _FAIL) { - ret8 = _FAIL; - goto exit; - } + if (_rtw_init_sta_priv(&padapter->stapriv) == _FAIL) + goto free_recv_priv; padapter->stapriv.padapter = padapter; padapter->setband = GHZ24_50; @@ -719,9 +704,26 @@ u8 rtw_init_drv_sw(struct adapter *padapter) rtw_hal_dm_init(padapter); -exit: + return _SUCCESS; + +free_recv_priv: + _rtw_free_recv_priv(&padapter->recvpriv); + +free_xmit_priv: + _rtw_free_xmit_priv(&padapter->xmitpriv); + +free_mlme_ext: + free_mlme_ext_priv(&padapter->mlmeextpriv); - return ret8; + rtw_free_mlme_priv(&padapter->mlmepriv); + +free_evt_priv: + rtw_free_evt_priv(&padapter->evtpriv); + +free_cmd_priv: + rtw_free_cmd_priv(&padapter->cmdpriv); + + return _FAIL; } void rtw_cancel_all_timer(struct adapter *padapter) diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c index 4fbfa75c05d74eb39be89227180a26ea62648fbb..f09c1324c39c6e7af231e4083febac14d4dc4980 100644 --- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c +++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c @@ -108,56 +108,6 @@ RETURN: return; } -int rtw_change_ifname(struct adapter *padapter, const char *ifname) -{ - struct net_device *pnetdev; - struct net_device *cur_pnetdev; - struct rereg_nd_name_data *rereg_priv; - int ret; - - if (!padapter) - goto error; - - cur_pnetdev = padapter->pnetdev; - rereg_priv = &padapter->rereg_nd_name_priv; - - /* free the old_pnetdev */ - if (rereg_priv->old_pnetdev) { - free_netdev(rereg_priv->old_pnetdev); - rereg_priv->old_pnetdev = NULL; - } - - if (!rtnl_is_locked()) - unregister_netdev(cur_pnetdev); - else - unregister_netdevice(cur_pnetdev); - - rereg_priv->old_pnetdev = cur_pnetdev; - - pnetdev = rtw_init_netdev(padapter); - if (!pnetdev) - goto error; - - SET_NETDEV_DEV(pnetdev, dvobj_to_dev(adapter_to_dvobj(padapter))); - - rtw_init_netdev_name(pnetdev, ifname); - - eth_hw_addr_set(pnetdev, padapter->eeprompriv.mac_addr); - - if (!rtnl_is_locked()) - ret = register_netdev(pnetdev); - else - ret = register_netdevice(pnetdev); - - if (ret != 0) - goto error; - - return 0; - -error: - return -1; -} - void rtw_buf_free(u8 **buf, u32 *buf_len) { if (!buf || !buf_len) diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c index 530e7a6c67c51f357232a787eb4045f75d48dc15..1eeabfffd6d2449c15129709897b46253fee05d8 100644 --- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c +++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c @@ -181,7 +181,7 @@ static int rtw_mlcst2unicst(struct adapter *padapter, struct sk_buff *skb) return true; } -int _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) +void _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) { struct adapter *padapter = rtw_netdev_priv(pnetdev); struct xmit_priv *pxmitpriv = &padapter->xmitpriv; @@ -202,7 +202,7 @@ int _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) if (pxmitpriv->free_xmitframe_cnt > (NR_XMITFRAME / 4)) { res = rtw_mlcst2unicst(padapter, pkt); if (res) - goto exit; + return; } } @@ -210,22 +210,17 @@ int _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) if (res < 0) goto drop_packet; - goto exit; + return; drop_packet: pxmitpriv->tx_drop++; dev_kfree_skb_any(pkt); - -exit: - return 0; } -int rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) +netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev) { - int ret = 0; - if (pkt) - ret = _rtw_xmit_entry(pkt, pnetdev); + _rtw_xmit_entry(pkt, pnetdev); - return ret; + return NETDEV_TX_OK; } diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index ce04c38f6afd0fba33565c31b23bfae4fd32f019..168ae2e9005d7606e31dfec4c28983ab88dd8ecd 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -387,7 +387,8 @@ static int lynxfb_ops_set_par(struct fb_info *info) ret = lynxfb_set_color_offsets(info); - var->height = var->width = -1; + var->height = -1; + var->width = -1; var->accel_flags = 0;/*FB_ACCELF_TEXT;*/ if (ret) { @@ -499,7 +500,8 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var, return ret; } - var->height = var->width = -1; + var->height = -1; + var->width = -1; var->accel_flags = 0;/* FB_ACCELF_TEXT; */ /* check if current fb's video memory big enough to hold the onscreen*/ @@ -724,7 +726,8 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index) 0x800f0 + (int)crtc->channel * 0x140; pr_info("crtc->cursor.mmio = %p\n", crtc->cursor.mmio); - crtc->cursor.max_h = crtc->cursor.max_w = 64; + crtc->cursor.max_h = 64; + crtc->cursor.max_w = 64; crtc->cursor.size = crtc->cursor.max_h * crtc->cursor.max_w * 2 / 8; crtc->cursor.vstart = sm750_dev->pvMem + crtc->cursor.offset; @@ -1022,7 +1025,8 @@ static int lynxfb_pci_probe(struct pci_dev *pdev, if (!sm750_dev) return err; - sm750_dev->fbinfo[0] = sm750_dev->fbinfo[1] = NULL; + sm750_dev->fbinfo[0] = NULL; + sm750_dev->fbinfo[1] = NULL; sm750_dev->devid = pdev->device; sm750_dev->revid = pdev->revision; sm750_dev->pdev = pdev; diff --git a/drivers/staging/vme_user/vme_tsi148.c b/drivers/staging/vme_user/vme_tsi148.c index 9564762132415526165324008a9d105bd96636d9..020e0b3bce64b49d54d96250c31fc301085d1fe5 100644 --- a/drivers/staging/vme_user/vme_tsi148.c +++ b/drivers/staging/vme_user/vme_tsi148.c @@ -125,8 +125,8 @@ static u32 tsi148_MB_irqhandler(struct vme_bridge *tsi148_bridge, u32 stat) for (i = 0; i < 4; i++) { if (stat & TSI148_LCSR_INTS_MBS[i]) { val = ioread32be(bridge->base + TSI148_GCSR_MBOX[i]); - dev_err(tsi148_bridge->parent, "VME Mailbox %d received" - ": 0x%x\n", i, val); + dev_err(tsi148_bridge->parent, "VME Mailbox %d received: 0x%x\n", + i, val); serviced |= TSI148_LCSR_INTC_MBC[i]; } } @@ -143,14 +143,12 @@ static u32 tsi148_PERR_irqhandler(struct vme_bridge *tsi148_bridge) bridge = tsi148_bridge->driver_priv; - dev_err(tsi148_bridge->parent, "PCI Exception at address: 0x%08x:%08x, " - "attributes: %08x\n", + dev_err(tsi148_bridge->parent, "PCI Exception at address: 0x%08x:%08x, attributes: %08x\n", ioread32be(bridge->base + TSI148_LCSR_EDPAU), ioread32be(bridge->base + TSI148_LCSR_EDPAL), ioread32be(bridge->base + TSI148_LCSR_EDPAT)); - dev_err(tsi148_bridge->parent, "PCI-X attribute reg: %08x, PCI-X split " - "completion reg: %08x\n", + dev_err(tsi148_bridge->parent, "PCI-X attribute reg: %08x, PCI-X split completion reg: %08x\n", ioread32be(bridge->base + TSI148_LCSR_EDPXA), ioread32be(bridge->base + TSI148_LCSR_EDPXS)); @@ -180,10 +178,8 @@ static u32 tsi148_VERR_irqhandler(struct vme_bridge *tsi148_bridge) reg_join(error_addr_high, error_addr_low, &error_addr); /* Check for exception register overflow (we have lost error data) */ - if (error_attrib & TSI148_LCSR_VEAT_VEOF) { - dev_err(tsi148_bridge->parent, "VME Bus Exception Overflow " - "Occurred\n"); - } + if (error_attrib & TSI148_LCSR_VEAT_VEOF) + dev_err(tsi148_bridge->parent, "VME Bus Exception Overflow Occurred\n"); if (err_chk) vme_bus_error_handler(tsi148_bridge, error_addr, error_am); @@ -317,8 +313,8 @@ static int tsi148_irq_init(struct vme_bridge *tsi148_bridge) IRQF_SHARED, driver_name, tsi148_bridge); if (result) { - dev_err(tsi148_bridge->parent, "Can't get assigned pci irq " - "vector %02X\n", pdev->irq); + dev_err(tsi148_bridge->parent, "Can't get assigned pci irq vector %02X\n", + pdev->irq); return result; } @@ -529,8 +525,7 @@ static int tsi148_slave_set(struct vme_slave_resource *image, int enabled, return -EINVAL; } if (pci_offset_low & (granularity - 1)) { - dev_err(tsi148_bridge->parent, "Invalid PCI Offset " - "alignment\n"); + dev_err(tsi148_bridge->parent, "Invalid PCI Offset alignment\n"); return -EINVAL; } @@ -588,7 +583,7 @@ static int tsi148_slave_set(struct vme_slave_resource *image, int enabled, temp_ctl &= ~0xF; if (cycle & VME_SUPER) - temp_ctl |= TSI148_LCSR_ITAT_SUPR ; + temp_ctl |= TSI148_LCSR_ITAT_SUPR; if (cycle & VME_USER) temp_ctl |= TSI148_LCSR_ITAT_NPRIV; if (cycle & VME_PROG) @@ -762,8 +757,7 @@ static int tsi148_alloc_resource(struct vme_master_resource *image, &image->bus_resource, size, 0x10000, PCIBIOS_MIN_MEM, 0, NULL, NULL); if (retval) { - dev_err(tsi148_bridge->parent, "Failed to allocate mem " - "resource for window %d size 0x%lx start 0x%lx\n", + dev_err(tsi148_bridge->parent, "Failed to allocate mem resource for window %d size 0x%lx start 0x%lx\n", image->number, (unsigned long)size, (unsigned long)image->bus_resource.start); goto err_resource; @@ -827,15 +821,13 @@ static int tsi148_master_set(struct vme_master_resource *image, int enabled, /* Verify input data */ if (vme_base & 0xFFFF) { - dev_err(tsi148_bridge->parent, "Invalid VME Window " - "alignment\n"); + dev_err(tsi148_bridge->parent, "Invalid VME Window alignment\n"); retval = -EINVAL; goto err_window; } if ((size == 0) && (enabled != 0)) { - dev_err(tsi148_bridge->parent, "Size must be non-zero for " - "enabled windows\n"); + dev_err(tsi148_bridge->parent, "Size must be non-zero for enabled windows\n"); retval = -EINVAL; goto err_window; } @@ -849,8 +841,7 @@ static int tsi148_master_set(struct vme_master_resource *image, int enabled, retval = tsi148_alloc_resource(image, size); if (retval) { spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Unable to allocate memory for " - "resource\n"); + dev_err(tsi148_bridge->parent, "Unable to allocate memory for resource\n"); goto err_res; } @@ -890,8 +881,7 @@ static int tsi148_master_set(struct vme_master_resource *image, int enabled, } if (vme_offset_low & 0xFFFF) { spin_unlock(&image->lock); - dev_err(tsi148_bridge->parent, "Invalid VME Offset " - "alignment\n"); + dev_err(tsi148_bridge->parent, "Invalid VME Offset alignment\n"); retval = -EINVAL; goto err_gran; } @@ -937,8 +927,7 @@ static int tsi148_master_set(struct vme_master_resource *image, int enabled, temp_ctl |= TSI148_LCSR_OTAT_TM_2eSST; } if (cycle & VME_2eSSTB) { - dev_warn(tsi148_bridge->parent, "Currently not setting " - "Broadcast Select Registers\n"); + dev_warn(tsi148_bridge->parent, "Currently not setting Broadcast Select Registers\n"); temp_ctl &= ~TSI148_LCSR_OTAT_TM_M; temp_ctl |= TSI148_LCSR_OTAT_TM_2eSSTB; } @@ -1451,8 +1440,7 @@ static int tsi148_dma_set_vme_src_attributes(struct device *dev, __be32 *attr, val |= TSI148_LCSR_DSAT_TM_2eSST; if (cycle & VME_2eSSTB) { - dev_err(dev, "Currently not setting Broadcast Select " - "Registers\n"); + dev_err(dev, "Currently not setting Broadcast Select Registers\n"); val |= TSI148_LCSR_DSAT_TM_2eSSTB; } @@ -1550,8 +1538,7 @@ static int tsi148_dma_set_vme_dest_attributes(struct device *dev, __be32 *attr, val |= TSI148_LCSR_DDAT_TM_2eSST; if (cycle & VME_2eSSTB) { - dev_err(dev, "Currently not setting Broadcast Select " - "Registers\n"); + dev_err(dev, "Currently not setting Broadcast Select Registers\n"); val |= TSI148_LCSR_DDAT_TM_2eSSTB; } @@ -1639,8 +1626,7 @@ static int tsi148_dma_list_add(struct vme_dma_list *list, /* Test descriptor alignment */ if ((unsigned long)&entry->descriptor & 0x7) { - dev_err(tsi148_bridge->parent, "Descriptor not aligned to 8 " - "byte boundary as required: %p\n", + dev_err(tsi148_bridge->parent, "Descriptor not aligned to 8 byte boundary as required: %p\n", &entry->descriptor); retval = -EINVAL; goto err_align; @@ -1827,10 +1813,10 @@ static int tsi148_dma_list_exec(struct vme_dma_list *list) /* Need to add to pending here */ mutex_unlock(&ctrlr->mtx); return -EBUSY; - } else { - list_add(&list->list, &ctrlr->running); } + list_add(&list->list, &ctrlr->running); + /* Get first bus address and write into registers */ entry = list_first_entry(&list->entries, struct tsi148_dma_entry, list); @@ -1935,8 +1921,7 @@ static int tsi148_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, for (i = 0; i < lm->monitors; i++) { if (bridge->lm_callback[i]) { mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Location monitor " - "callback attached, can't reset\n"); + dev_err(tsi148_bridge->parent, "Location monitor callback attached, can't reset\n"); return -EBUSY; } } @@ -1961,7 +1946,7 @@ static int tsi148_lm_set(struct vme_lm_resource *lm, unsigned long long lm_base, } if (cycle & VME_SUPER) - lm_ctl |= TSI148_LCSR_LMAT_SUPR ; + lm_ctl |= TSI148_LCSR_LMAT_SUPR; if (cycle & VME_USER) lm_ctl |= TSI148_LCSR_LMAT_NPRIV; if (cycle & VME_PROG) @@ -2051,8 +2036,7 @@ static int tsi148_lm_attach(struct vme_lm_resource *lm, int monitor, lm_ctl = ioread32be(bridge->base + TSI148_LCSR_LMAT); if ((lm_ctl & (TSI148_LCSR_LMAT_PGM | TSI148_LCSR_LMAT_DATA)) == 0) { mutex_unlock(&lm->mtx); - dev_err(tsi148_bridge->parent, "Location monitor not properly " - "configured\n"); + dev_err(tsi148_bridge->parent, "Location monitor not properly configured\n"); return -EINVAL; } @@ -2196,8 +2180,7 @@ static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge, VME_CRCSR_BUF_SIZE, &bridge->crcsr_bus, GFP_KERNEL); if (!bridge->crcsr_kernel) { - dev_err(tsi148_bridge->parent, "Failed to allocate memory for " - "CR/CSR image\n"); + dev_err(tsi148_bridge->parent, "Failed to allocate memory for CR/CSR image\n"); return -ENOMEM; } @@ -2237,8 +2220,7 @@ static int tsi148_crcsr_init(struct vme_bridge *tsi148_bridge, (vstat * 0x80000), 0x80000, VME_CRCSR, VME_SCT, VME_D16); if (retval) - dev_err(tsi148_bridge->parent, "Configuring flush image" - " failed\n"); + dev_err(tsi148_bridge->parent, "Configuring flush image failed\n"); } return 0; diff --git a/drivers/staging/vt6655/baseband.c b/drivers/staging/vt6655/baseband.c index 5de841cb776cf97b8456c1800dc4c43f4fb4b9b4..6ce41983dcf433cf7700b60324e6e187f9c42f19 100644 --- a/drivers/staging/vt6655/baseband.c +++ b/drivers/staging/vt6655/baseband.c @@ -2083,7 +2083,7 @@ bool bb_vt3253_init(struct vnt_private *priv) priv->dbm_threshold[2] = 0; priv->dbm_threshold[3] = 0; /* Fix VT3226 DFC system timing issue */ - MACvSetRFLE_LatchBase(iobase); + vt6655_mac_word_reg_bits_on(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_RFLEOPT); /* {{ RobertYu: 20050104 */ } else { /* No VGA Table now */ diff --git a/drivers/staging/vt6655/card.c b/drivers/staging/vt6655/card.c index 846469cc06bb9d4afab391960f6ca513228772d0..c680925b9c92c5664673697e2b5b60f6822361dc 100644 --- a/drivers/staging/vt6655/card.c +++ b/drivers/staging/vt6655/card.c @@ -55,9 +55,15 @@ static const unsigned short cwRXBCNTSFOff[MAX_RATE] = { /*--------------------- Static Functions --------------------------*/ -static void s_vCalculateOFDMRParameter(unsigned char rate, u8 bb_type, - unsigned char *pbyTxRate, - unsigned char *pbyRsvTime); +static void vt6655_mac_set_bb_type(void __iomem *iobase, u32 mask) +{ + u32 reg_value; + + reg_value = ioread32(iobase + MAC_REG_ENCFG); + reg_value = reg_value & ~ENCFG_BBTYPE_MASK; + reg_value = reg_value | mask; + iowrite32(reg_value, iobase + MAC_REG_ENCFG); +} /*--------------------- Export Functions --------------------------*/ @@ -186,21 +192,21 @@ bool CARDbSetPhyParameter(struct vnt_private *priv, u8 bb_type) /* Set SIFS, DIFS, EIFS, SlotTime, CwMin */ if (bb_type == BB_TYPE_11A) { - MACvSetBBType(priv->port_offset, BB_TYPE_11A); + vt6655_mac_set_bb_type(priv->port_offset, BB_TYPE_11A); bb_write_embedded(priv, 0x88, 0x03); bySlot = C_SLOT_SHORT; bySIFS = C_SIFS_A; byDIFS = C_SIFS_A + 2 * C_SLOT_SHORT; byCWMaxMin = 0xA4; } else if (bb_type == BB_TYPE_11B) { - MACvSetBBType(priv->port_offset, BB_TYPE_11B); + vt6655_mac_set_bb_type(priv->port_offset, BB_TYPE_11B); bb_write_embedded(priv, 0x88, 0x02); bySlot = C_SLOT_LONG; bySIFS = C_SIFS_BG; byDIFS = C_SIFS_BG + 2 * C_SLOT_LONG; byCWMaxMin = 0xA5; } else { /* PK_TYPE_11GA & PK_TYPE_11GB */ - MACvSetBBType(priv->port_offset, BB_TYPE_11G); + vt6655_mac_set_bb_type(priv->port_offset, BB_TYPE_11G); bb_write_embedded(priv, 0x88, 0x08); bySIFS = C_SIFS_BG; @@ -403,9 +409,9 @@ void CARDvSafeResetTx(struct vnt_private *priv) } /* set MAC TD pointer */ - MACvSetCurrTXDescAddr(TYPE_TXDMA0, priv, priv->td0_pool_dma); + vt6655_mac_set_curr_tx_desc_addr(TYPE_TXDMA0, priv, priv->td0_pool_dma); - MACvSetCurrTXDescAddr(TYPE_AC0DMA, priv, priv->td1_pool_dma); + vt6655_mac_set_curr_tx_desc_addr(TYPE_AC0DMA, priv, priv->td1_pool_dma); /* set MAC Beacon TX pointer */ iowrite32((u32)priv->tx_beacon_dma, priv->port_offset + MAC_REG_BCNDMAPTR); @@ -452,9 +458,9 @@ void CARDvSafeResetRx(struct vnt_private *priv) iowrite32(RX_PERPKT, priv->port_offset + MAC_REG_RXDMACTL0); iowrite32(RX_PERPKT, priv->port_offset + MAC_REG_RXDMACTL1); /* set MAC RD pointer */ - MACvSetCurrRx0DescAddr(priv, priv->rd0_pool_dma); + vt6655_mac_set_curr_rx_0_desc_addr(priv, priv->rd0_pool_dma); - MACvSetCurrRx1DescAddr(priv, priv->rd1_pool_dma); + vt6655_mac_set_curr_rx_1_desc_addr(priv, priv->rd1_pool_dma); } /* @@ -539,7 +545,7 @@ void CARDvSetRSPINF(struct vnt_private *priv, u8 bb_type) spin_lock_irqsave(&priv->lock, flags); /* Set to Page1 */ - MACvSelectPage1(priv->port_offset); + VT6655_MAC_SELECT_PAGE1(priv->port_offset); /* RSPINF_b_1 */ vnt_get_phy_field(priv, 14, @@ -637,7 +643,7 @@ void CARDvSetRSPINF(struct vnt_private *priv, u8 bb_type) &byRsvTime); iowrite16(MAKEWORD(byTxRate, byRsvTime), priv->port_offset + MAC_REG_RSPINF_A_72); /* Set to Page0 */ - MACvSelectPage0(priv->port_offset); + VT6655_MAC_SELECT_PAGE0(priv->port_offset); spin_unlock_irqrestore(&priv->lock, flags); } diff --git a/drivers/staging/vt6655/channel.c b/drivers/staging/vt6655/channel.c index e926f9829a1590fa5d1aa0f640e24b1183fc2d72..4122875ebcaaab15fbbcfa61b1cbf50e243d8e01 100644 --- a/drivers/staging/vt6655/channel.c +++ b/drivers/staging/vt6655/channel.c @@ -116,12 +116,12 @@ bool set_channel(struct vnt_private *priv, struct ieee80211_channel *ch) spin_lock_irqsave(&priv->lock, flags); /* set HW default power register */ - MACvSelectPage1(priv->port_offset); + VT6655_MAC_SELECT_PAGE1(priv->port_offset); RFbSetPower(priv, RATE_1M, priv->byCurrentCh); iowrite8(priv->byCurPwr, priv->port_offset + MAC_REG_PWRCCK); RFbSetPower(priv, RATE_6M, priv->byCurrentCh); iowrite8(priv->byCurPwr, priv->port_offset + MAC_REG_PWROFDM); - MACvSelectPage0(priv->port_offset); + VT6655_MAC_SELECT_PAGE0(priv->port_offset); spin_unlock_irqrestore(&priv->lock, flags); } diff --git a/drivers/staging/vt6655/device_main.c b/drivers/staging/vt6655/device_main.c index bab08a40fe6691997dbd3c8131e57a6c02eac8f2..56c3cf3ba53d91e976e978a1324e846bdb12e70b 100644 --- a/drivers/staging/vt6655/device_main.c +++ b/drivers/staging/vt6655/device_main.c @@ -205,6 +205,55 @@ static void vt6655_mac_read_ether_addr(void __iomem *iobase, u8 *mac_addr) iowrite8(0, iobase + MAC_REG_PAGE1SEL); } +static void vt6655_mac_dma_ctl(void __iomem *iobase, u8 reg_index) +{ + u32 reg_value; + + reg_value = ioread32(iobase + reg_index); + if (reg_value & DMACTL_RUN) + iowrite32(DMACTL_WAKE, iobase + reg_index); + else + iowrite32(DMACTL_RUN, iobase + reg_index); +} + +static void vt6655_mac_set_bits(void __iomem *iobase, u32 mask) +{ + u32 reg_value; + + reg_value = ioread32(iobase + MAC_REG_ENCFG); + reg_value = reg_value | mask; + iowrite32(reg_value, iobase + MAC_REG_ENCFG); +} + +static void vt6655_mac_clear_bits(void __iomem *iobase, u32 mask) +{ + u32 reg_value; + + reg_value = ioread32(iobase + MAC_REG_ENCFG); + reg_value = reg_value & ~mask; + iowrite32(reg_value, iobase + MAC_REG_ENCFG); +} + +static void vt6655_mac_en_protect_md(void __iomem *iobase) +{ + vt6655_mac_set_bits(iobase, ENCFG_PROTECTMD); +} + +static void vt6655_mac_dis_protect_md(void __iomem *iobase) +{ + vt6655_mac_clear_bits(iobase, ENCFG_PROTECTMD); +} + +static void vt6655_mac_en_barker_preamble_md(void __iomem *iobase) +{ + vt6655_mac_set_bits(iobase, ENCFG_BARKERPREAM); +} + +static void vt6655_mac_dis_barker_preamble_md(void __iomem *iobase) +{ + vt6655_mac_clear_bits(iobase, ENCFG_BARKERPREAM); +} + /* * Initialisation of MAC & BBP registers */ @@ -351,11 +400,11 @@ static void device_init_registers(struct vnt_private *priv) } if (priv->local_id > REV_ID_VT3253_B1) { - MACvSelectPage1(priv->port_offset); + VT6655_MAC_SELECT_PAGE1(priv->port_offset); iowrite8(MSRCTL1_TXPWR | MSRCTL1_CSAPAREN, priv->port_offset + MAC_REG_MSRCTL + 1); - MACvSelectPage0(priv->port_offset); + VT6655_MAC_SELECT_PAGE0(priv->port_offset); } /* use relative tx timeout and 802.11i D4 */ @@ -363,7 +412,7 @@ static void device_init_registers(struct vnt_private *priv) (CFG_TKIPOPT | CFG_NOTXTIMEOUT)); /* set performance parameter by registry */ - MACvSetShortRetryLimit(priv, priv->byShortRetryLimit); + vt6655_mac_set_short_retry_limit(priv, priv->byShortRetryLimit); MACvSetLongRetryLimit(priv, priv->byLongRetryLimit); /* reset TSF counter */ @@ -420,8 +469,8 @@ static void device_init_registers(struct vnt_private *priv) vt6655_mac_reg_bits_on(priv->port_offset, MAC_REG_RCR, RCR_WPAERR); /* Turn On Rx DMA */ - MACvReceive0(priv->port_offset); - MACvReceive1(priv->port_offset); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_RXDMACTL0); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_RXDMACTL1); /* start the adapter */ iowrite8(HOSTCR_MACEN | HOSTCR_RXON | HOSTCR_TXON, priv->port_offset + MAC_REG_HOSTCR); @@ -537,13 +586,12 @@ static void device_free_rings(struct vnt_private *priv) priv->opts.tx_descs[1] * sizeof(struct vnt_tx_desc), priv->aRD0Ring, priv->pool_dma); - if (priv->tx0_bufs) - dma_free_coherent(&priv->pcid->dev, - priv->opts.tx_descs[0] * PKT_BUF_SZ + - priv->opts.tx_descs[1] * PKT_BUF_SZ + - CB_BEACON_BUF_SIZE + - CB_MAX_BUF_SIZE, - priv->tx0_bufs, priv->tx_bufs_dma0); + dma_free_coherent(&priv->pcid->dev, + priv->opts.tx_descs[0] * PKT_BUF_SZ + + priv->opts.tx_descs[1] * PKT_BUF_SZ + + CB_BEACON_BUF_SIZE + + CB_MAX_BUF_SIZE, + priv->tx0_bufs, priv->tx_bufs_dma0); } static int device_init_rd0_ring(struct vnt_private *priv) @@ -583,7 +631,7 @@ err_free_rd: kfree(desc->rd_info); err_free_desc: - while (--i) { + while (i--) { desc = &priv->aRD0Ring[i]; device_free_rx_buf(priv, desc); kfree(desc->rd_info); @@ -629,7 +677,7 @@ err_free_rd: kfree(desc->rd_info); err_free_desc: - while (--i) { + while (i--) { desc = &priv->aRD1Ring[i]; device_free_rx_buf(priv, desc); kfree(desc->rd_info); @@ -694,7 +742,7 @@ static int device_init_td0_ring(struct vnt_private *priv) return 0; err_free_desc: - while (--i) { + while (i--) { desc = &priv->apTD0Rings[i]; kfree(desc->td_info); } @@ -734,7 +782,7 @@ static int device_init_td1_ring(struct vnt_private *priv) return 0; err_free_desc: - while (--i) { + while (i--) { desc = &priv->apTD1Rings[i]; kfree(desc->td_info); } @@ -1135,8 +1183,8 @@ static void vnt_interrupt_process(struct vnt_private *priv) isr = ioread32(priv->port_offset + MAC_REG_ISR); - MACvReceive0(priv->port_offset); - MACvReceive1(priv->port_offset); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_RXDMACTL0); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_RXDMACTL1); if (max_count > priv->opts.int_works) break; @@ -1218,9 +1266,9 @@ static int vnt_tx_packet(struct vnt_private *priv, struct sk_buff *skb) wmb(); /* second memory barrier */ if (head_td->td_info->flags & TD_FLAGS_NETIF_SKB) - MACvTransmitAC0(priv->port_offset); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_AC0DMACTL); else - MACvTransmit0(priv->port_offset); + vt6655_mac_dma_ctl(priv->port_offset, MAC_REG_TXDMACTL0); priv->iTDUsed[dma_idx]++; @@ -1440,19 +1488,19 @@ static void vnt_bss_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_ERP_PREAMBLE) { if (conf->use_short_preamble) { - MACvEnableBarkerPreambleMd(priv->port_offset); + vt6655_mac_en_barker_preamble_md(priv->port_offset); priv->preamble_type = true; } else { - MACvDisableBarkerPreambleMd(priv->port_offset); + vt6655_mac_dis_barker_preamble_md(priv->port_offset); priv->preamble_type = false; } } if (changed & BSS_CHANGED_ERP_CTS_PROT) { if (conf->use_cts_prot) - MACvEnableProtectMD(priv->port_offset); + vt6655_mac_en_protect_md(priv->port_offset); else - MACvDisableProtectMD(priv->port_offset); + vt6655_mac_dis_protect_md(priv->port_offset); } if (changed & BSS_CHANGED_ERP_SLOT) { @@ -1538,21 +1586,21 @@ static void vnt_configure(struct ieee80211_hw *hw, spin_lock_irqsave(&priv->lock, flags); if (priv->mc_list_count > 2) { - MACvSelectPage1(priv->port_offset); + VT6655_MAC_SELECT_PAGE1(priv->port_offset); iowrite32(0xffffffff, priv->port_offset + MAC_REG_MAR0); iowrite32(0xffffffff, priv->port_offset + MAC_REG_MAR0 + 4); - MACvSelectPage0(priv->port_offset); + VT6655_MAC_SELECT_PAGE0(priv->port_offset); } else { - MACvSelectPage1(priv->port_offset); + VT6655_MAC_SELECT_PAGE1(priv->port_offset); multicast = le64_to_cpu(multicast); iowrite32((u32)multicast, priv->port_offset + MAC_REG_MAR0); iowrite32((u32)(multicast >> 32), priv->port_offset + MAC_REG_MAR0 + 4); - MACvSelectPage0(priv->port_offset); + VT6655_MAC_SELECT_PAGE0(priv->port_offset); } spin_unlock_irqrestore(&priv->lock, flags); diff --git a/drivers/staging/vt6655/mac.c b/drivers/staging/vt6655/mac.c index dcc649532737d8b390750645597e12324b25bee5..b4ebc7d319619065cbdf1153581c7ce9212772b2 100644 --- a/drivers/staging/vt6655/mac.c +++ b/drivers/staging/vt6655/mac.c @@ -10,17 +10,16 @@ * Date: May 21, 1996 * * Functions: - * MACbIsRegBitsOff - Test if All test Bits Off - * MACbIsIntDisable - Test if MAC interrupt disable - * MACvSetShortRetryLimit - Set 802.11 Short Retry limit + * vt6655_mac_is_reg_bits_off - Test if All test Bits Off + * vt6655_mac_set_short_retry_limit - Set 802.11 Short Retry limit * MACvSetLongRetryLimit - Set 802.11 Long Retry limit - * MACvSetLoopbackMode - Set MAC Loopback Mode - * MACvSaveContext - Save Context of MAC Registers - * MACvRestoreContext - Restore Context of MAC Registers + * vt6655_mac_set_loopback_mode - Set MAC Loopback Mode + * vt6655_mac_save_context - Save Context of MAC Registers + * vt6655_mac_restore_context - Restore Context of MAC Registers * MACbSoftwareReset - Software Reset MAC - * MACbSafeRxOff - Turn Off MAC Rx - * MACbSafeTxOff - Turn Off MAC Tx - * MACbSafeStop - Stop MAC function + * vt6655_mac_safe_rx_off - Turn Off MAC Rx + * vt6655_mac_safe_tx_off - Turn Off MAC Tx + * vt6655_mac_safe_stop - Stop MAC function * MACbShutdown - Shut down MAC * MACvInitialize - Initialize MAC * MACvSetCurrRxDescAddr - Set Rx Descriptors Address @@ -86,43 +85,21 @@ static void vt6655_mac_clear_stck_ds(void __iomem *iobase) * Parameters: * In: * io_base - Base Address for MAC - * byRegOfs - Offset of MAC Register - * byTestBits - Test bits + * reg_offset - Offset of MAC Register + * mask - Test bits * Out: * none * * Return Value: true if all test bits Off; otherwise false * */ -bool MACbIsRegBitsOff(struct vnt_private *priv, unsigned char byRegOfs, - unsigned char byTestBits) +static bool vt6655_mac_is_reg_bits_off(struct vnt_private *priv, + unsigned char reg_offset, + unsigned char mask) { void __iomem *io_base = priv->port_offset; - return !(ioread8(io_base + byRegOfs) & byTestBits); -} - -/* - * Description: - * Test if MAC interrupt disable - * - * Parameters: - * In: - * io_base - Base Address for MAC - * Out: - * none - * - * Return Value: true if interrupt is disable; otherwise false - * - */ -bool MACbIsIntDisable(struct vnt_private *priv) -{ - void __iomem *io_base = priv->port_offset; - - if (ioread32(io_base + MAC_REG_IMR)) - return false; - - return true; + return !(ioread8(io_base + reg_offset) & mask); } /* @@ -132,19 +109,18 @@ bool MACbIsIntDisable(struct vnt_private *priv) * Parameters: * In: * io_base - Base Address for MAC - * byRetryLimit- Retry Limit + * retry_limit - Retry Limit * Out: * none * * Return Value: none * */ -void MACvSetShortRetryLimit(struct vnt_private *priv, - unsigned char byRetryLimit) +void vt6655_mac_set_short_retry_limit(struct vnt_private *priv, unsigned char retry_limit) { void __iomem *io_base = priv->port_offset; /* set SRT */ - iowrite8(byRetryLimit, io_base + MAC_REG_SRT); + iowrite8(retry_limit, io_base + MAC_REG_SRT); } /* @@ -176,21 +152,20 @@ void MACvSetLongRetryLimit(struct vnt_private *priv, * Parameters: * In: * io_base - Base Address for MAC - * byLoopbackMode - Loopback Mode + * loopback_mode - Loopback Mode * Out: * none * * Return Value: none * */ -void MACvSetLoopbackMode(struct vnt_private *priv, unsigned char byLoopbackMode) +static void vt6655_mac_set_loopback_mode(struct vnt_private *priv, u8 loopback_mode) { void __iomem *io_base = priv->port_offset; - byLoopbackMode <<= 6; + loopback_mode <<= 6; /* set TCR */ - iowrite8((ioread8(io_base + MAC_REG_TEST) & 0x3f) | byLoopbackMode, - io_base + MAC_REG_TEST); + iowrite8((ioread8(io_base + MAC_REG_TEST) & 0x3f) | loopback_mode, io_base + MAC_REG_TEST); } /* @@ -206,20 +181,20 @@ void MACvSetLoopbackMode(struct vnt_private *priv, unsigned char byLoopbackMode) * Return Value: none * */ -void MACvSaveContext(struct vnt_private *priv, unsigned char *cxt_buf) +static void vt6655_mac_save_context(struct vnt_private *priv, u8 *cxt_buf) { void __iomem *io_base = priv->port_offset; /* read page0 register */ memcpy_fromio(cxt_buf, io_base, MAC_MAX_CONTEXT_SIZE_PAGE0); - MACvSelectPage1(io_base); + VT6655_MAC_SELECT_PAGE1(io_base); /* read page1 register */ memcpy_fromio(cxt_buf + MAC_MAX_CONTEXT_SIZE_PAGE0, io_base, MAC_MAX_CONTEXT_SIZE_PAGE1); - MACvSelectPage0(io_base); + VT6655_MAC_SELECT_PAGE0(io_base); } /* @@ -236,16 +211,16 @@ void MACvSaveContext(struct vnt_private *priv, unsigned char *cxt_buf) * Return Value: none * */ -void MACvRestoreContext(struct vnt_private *priv, unsigned char *cxt_buf) +static void vt6655_mac_restore_context(struct vnt_private *priv, u8 *cxt_buf) { void __iomem *io_base = priv->port_offset; - MACvSelectPage1(io_base); + VT6655_MAC_SELECT_PAGE1(io_base); /* restore page1 */ memcpy_toio(io_base, cxt_buf + MAC_MAX_CONTEXT_SIZE_PAGE0, MAC_MAX_CONTEXT_SIZE_PAGE1); - MACvSelectPage0(io_base); + VT6655_MAC_SELECT_PAGE0(io_base); /* restore RCR,TCR,IMR... */ memcpy_toio(io_base + MAC_REG_RCR, cxt_buf + MAC_REG_RCR, @@ -318,23 +293,20 @@ bool MACbSoftwareReset(struct vnt_private *priv) * Return Value: true if success; otherwise false * */ -bool MACbSafeSoftwareReset(struct vnt_private *priv) +static void vt6655_mac_save_soft_reset(struct vnt_private *priv) { - unsigned char abyTmpRegData[MAC_MAX_CONTEXT_SIZE_PAGE0 + MAC_MAX_CONTEXT_SIZE_PAGE1]; - bool bRetVal; + u8 tmp_reg_data[MAC_MAX_CONTEXT_SIZE_PAGE0 + MAC_MAX_CONTEXT_SIZE_PAGE1]; /* PATCH.... * save some important register's value, then do * reset, then restore register's value */ /* save MAC context */ - MACvSaveContext(priv, abyTmpRegData); + vt6655_mac_save_context(priv, tmp_reg_data); /* do reset */ - bRetVal = MACbSoftwareReset(priv); + MACbSoftwareReset(priv); /* restore MAC context, except CR0 */ - MACvRestoreContext(priv, abyTmpRegData); - - return bRetVal; + vt6655_mac_restore_context(priv, tmp_reg_data); } /* @@ -350,7 +322,7 @@ bool MACbSafeSoftwareReset(struct vnt_private *priv) * Return Value: true if success; otherwise false * */ -bool MACbSafeRxOff(struct vnt_private *priv) +static bool vt6655_mac_safe_rx_off(struct vnt_private *priv) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -404,7 +376,7 @@ bool MACbSafeRxOff(struct vnt_private *priv) * Return Value: true if success; otherwise false * */ -bool MACbSafeTxOff(struct vnt_private *priv) +static bool vt6655_mac_safe_tx_off(struct vnt_private *priv) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -460,20 +432,20 @@ bool MACbSafeTxOff(struct vnt_private *priv) * Return Value: true if success; otherwise false * */ -bool MACbSafeStop(struct vnt_private *priv) +static bool vt6655_mac_safe_stop(struct vnt_private *priv) { void __iomem *io_base = priv->port_offset; vt6655_mac_reg_bits_off(io_base, MAC_REG_TCR, TCR_AUTOBCNTX); - if (!MACbSafeRxOff(priv)) { - pr_debug(" MACbSafeRxOff == false)\n"); - MACbSafeSoftwareReset(priv); + if (!vt6655_mac_safe_rx_off(priv)) { + pr_debug(" vt6655_mac_safe_rx_off == false)\n"); + vt6655_mac_save_soft_reset(priv); return false; } - if (!MACbSafeTxOff(priv)) { - pr_debug(" MACbSafeTxOff == false)\n"); - MACbSafeSoftwareReset(priv); + if (!vt6655_mac_safe_tx_off(priv)) { + pr_debug(" vt6655_mac_safe_tx_off == false)\n"); + vt6655_mac_save_soft_reset(priv); return false; } @@ -500,13 +472,13 @@ bool MACbShutdown(struct vnt_private *priv) void __iomem *io_base = priv->port_offset; /* disable MAC IMR */ iowrite32(0, io_base + MAC_REG_IMR); - MACvSetLoopbackMode(priv, MAC_LB_INTERNAL); + vt6655_mac_set_loopback_mode(priv, MAC_LB_INTERNAL); /* stop the adapter */ - if (!MACbSafeStop(priv)) { - MACvSetLoopbackMode(priv, MAC_LB_NONE); + if (!vt6655_mac_safe_stop(priv)) { + vt6655_mac_set_loopback_mode(priv, MAC_LB_NONE); return false; } - MACvSetLoopbackMode(priv, MAC_LB_NONE); + vt6655_mac_set_loopback_mode(priv, MAC_LB_NONE); return true; } @@ -555,7 +527,7 @@ void MACvInitialize(struct vnt_private *priv) * Return Value: none * */ -void MACvSetCurrRx0DescAddr(struct vnt_private *priv, u32 curr_desc_addr) +void vt6655_mac_set_curr_rx_0_desc_addr(struct vnt_private *priv, u32 curr_desc_addr) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -589,7 +561,7 @@ void MACvSetCurrRx0DescAddr(struct vnt_private *priv, u32 curr_desc_addr) * Return Value: none * */ -void MACvSetCurrRx1DescAddr(struct vnt_private *priv, u32 curr_desc_addr) +void vt6655_mac_set_curr_rx_1_desc_addr(struct vnt_private *priv, u32 curr_desc_addr) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -623,8 +595,7 @@ void MACvSetCurrRx1DescAddr(struct vnt_private *priv, u32 curr_desc_addr) * Return Value: none * */ -void MACvSetCurrTx0DescAddrEx(struct vnt_private *priv, - u32 curr_desc_addr) +static void vt6655_mac_set_curr_tx_0_desc_addr_ex(struct vnt_private *priv, u32 curr_desc_addr) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -659,8 +630,7 @@ void MACvSetCurrTx0DescAddrEx(struct vnt_private *priv, * */ /* TxDMA1 = AC0DMA */ -void MACvSetCurrAC0DescAddrEx(struct vnt_private *priv, - u32 curr_desc_addr) +static void vt6655_mac_set_curr_ac_0_desc_addr_ex(struct vnt_private *priv, u32 curr_desc_addr) { void __iomem *io_base = priv->port_offset; unsigned short ww; @@ -681,13 +651,12 @@ void MACvSetCurrAC0DescAddrEx(struct vnt_private *priv, iowrite8(DMACTL_RUN, io_base + MAC_REG_AC0DMACTL); } -void MACvSetCurrTXDescAddr(int iTxType, struct vnt_private *priv, - u32 curr_desc_addr) +void vt6655_mac_set_curr_tx_desc_addr(int tx_type, struct vnt_private *priv, u32 curr_desc_addr) { - if (iTxType == TYPE_AC0DMA) - MACvSetCurrAC0DescAddrEx(priv, curr_desc_addr); - else if (iTxType == TYPE_TXDMA0) - MACvSetCurrTx0DescAddrEx(priv, curr_desc_addr); + if (tx_type == TYPE_AC0DMA) + vt6655_mac_set_curr_ac_0_desc_addr_ex(priv, curr_desc_addr); + else if (tx_type == TYPE_TXDMA0) + vt6655_mac_set_curr_tx_0_desc_addr_ex(priv, curr_desc_addr); } /* @@ -767,7 +736,7 @@ bool MACbPSWakeup(struct vnt_private *priv) void __iomem *io_base = priv->port_offset; unsigned int ww; /* Read PSCTL */ - if (MACbIsRegBitsOff(priv, MAC_REG_PSCTL, PSCTL_PS)) + if (vt6655_mac_is_reg_bits_off(priv, MAC_REG_PSCTL, PSCTL_PS)) return true; /* Disable PS */ diff --git a/drivers/staging/vt6655/mac.h b/drivers/staging/vt6655/mac.h index 0122c4603c6646882a71d1480500300f5aa75e9d..acf931c3f5fd9b65b1c03d871f046c5ec9832c38 100644 --- a/drivers/staging/vt6655/mac.h +++ b/drivers/staging/vt6655/mac.h @@ -12,7 +12,7 @@ * Revision History: * 07-01-2003 Bryan YC Fan: Re-write codes to support VT3253 spec. * 08-25-2003 Kyle Hsu: Porting MAC functions from sim53. - * 09-03-2003 Bryan YC Fan: Add MACvDisableProtectMD & MACvEnableProtectMD + * 09-03-2003 Bryan YC Fan: Add vt6655_mac_dis_protect_md & vt6655_mac_en_protect_md */ #ifndef __MAC_H__ @@ -537,95 +537,9 @@ /*--------------------- Export Macros ------------------------------*/ -#define MACvReceive0(iobase) \ -do { \ - unsigned long dwData; \ - dwData = ioread32(iobase + MAC_REG_RXDMACTL0); \ - if (dwData & DMACTL_RUN) \ - iowrite32(DMACTL_WAKE, iobase + MAC_REG_RXDMACTL0); \ - else \ - iowrite32(DMACTL_RUN, iobase + MAC_REG_RXDMACTL0); \ -} while (0) - -#define MACvReceive1(iobase) \ -do { \ - unsigned long dwData; \ - dwData = ioread32(iobase + MAC_REG_RXDMACTL1); \ - if (dwData & DMACTL_RUN) \ - iowrite32(DMACTL_WAKE, iobase + MAC_REG_RXDMACTL1); \ - else \ - iowrite32(DMACTL_RUN, iobase + MAC_REG_RXDMACTL1); \ -} while (0) - -#define MACvTransmit0(iobase) \ -do { \ - unsigned long dwData; \ - dwData = ioread32(iobase + MAC_REG_TXDMACTL0); \ - if (dwData & DMACTL_RUN) \ - iowrite32(DMACTL_WAKE, iobase + MAC_REG_TXDMACTL0); \ - else \ - iowrite32(DMACTL_RUN, iobase + MAC_REG_TXDMACTL0); \ -} while (0) - -#define MACvTransmitAC0(iobase) \ -do { \ - unsigned long dwData; \ - dwData = ioread32(iobase + MAC_REG_AC0DMACTL); \ - if (dwData & DMACTL_RUN) \ - iowrite32(DMACTL_WAKE, iobase + MAC_REG_AC0DMACTL); \ - else \ - iowrite32(DMACTL_RUN, iobase + MAC_REG_AC0DMACTL); \ -} while (0) - -#define MACvSelectPage0(iobase) \ - iowrite8(0, iobase + MAC_REG_PAGE1SEL) - -#define MACvSelectPage1(iobase) \ - iowrite8(1, iobase + MAC_REG_PAGE1SEL) - -#define MACvEnableProtectMD(iobase) \ -do { \ - unsigned long dwOrgValue; \ - dwOrgValue = ioread32(iobase + MAC_REG_ENCFG); \ - dwOrgValue = dwOrgValue | ENCFG_PROTECTMD; \ - iowrite32((u32)dwOrgValue, iobase + MAC_REG_ENCFG); \ -} while (0) - -#define MACvDisableProtectMD(iobase) \ -do { \ - unsigned long dwOrgValue; \ - dwOrgValue = ioread32(iobase + MAC_REG_ENCFG); \ - dwOrgValue = dwOrgValue & ~ENCFG_PROTECTMD; \ - iowrite32((u32)dwOrgValue, iobase + MAC_REG_ENCFG); \ -} while (0) - -#define MACvEnableBarkerPreambleMd(iobase) \ -do { \ - unsigned long dwOrgValue; \ - dwOrgValue = ioread32(iobase + MAC_REG_ENCFG); \ - dwOrgValue = dwOrgValue | ENCFG_BARKERPREAM; \ - iowrite32((u32)dwOrgValue, iobase + MAC_REG_ENCFG); \ -} while (0) - -#define MACvDisableBarkerPreambleMd(iobase) \ -do { \ - unsigned long dwOrgValue; \ - dwOrgValue = ioread32(iobase + MAC_REG_ENCFG); \ - dwOrgValue = dwOrgValue & ~ENCFG_BARKERPREAM; \ - iowrite32((u32)dwOrgValue, iobase + MAC_REG_ENCFG); \ -} while (0) - -#define MACvSetBBType(iobase, byTyp) \ -do { \ - unsigned long dwOrgValue; \ - dwOrgValue = ioread32(iobase + MAC_REG_ENCFG); \ - dwOrgValue = dwOrgValue & ~ENCFG_BBTYPE_MASK; \ - dwOrgValue = dwOrgValue | (unsigned long)byTyp; \ - iowrite32((u32)dwOrgValue, iobase + MAC_REG_ENCFG); \ -} while (0) - -#define MACvSetRFLE_LatchBase(iobase) \ - vt6655_mac_word_reg_bits_on(iobase, MAC_REG_SOFTPWRCTL, SOFTPWRCTL_RFLEOPT) +#define VT6655_MAC_SELECT_PAGE0(iobase) iowrite8(0, iobase + MAC_REG_PAGE1SEL) + +#define VT6655_MAC_SELECT_PAGE1(iobase) iowrite8(1, iobase + MAC_REG_PAGE1SEL) #define MAKEWORD(lb, hb) \ ((unsigned short)(((unsigned char)(lb)) | (((unsigned short)((unsigned char)(hb))) << 8))) @@ -635,38 +549,16 @@ void vt6655_mac_word_reg_bits_on(void __iomem *iobase, const u8 reg_offset, cons void vt6655_mac_reg_bits_off(void __iomem *iobase, const u8 reg_offset, const u8 bit_mask); void vt6655_mac_word_reg_bits_off(void __iomem *iobase, const u8 reg_offset, const u16 bit_mask); -bool MACbIsRegBitsOff(struct vnt_private *priv, unsigned char byRegOfs, - unsigned char byTestBits); - -bool MACbIsIntDisable(struct vnt_private *priv); - -void MACvSetShortRetryLimit(struct vnt_private *priv, - unsigned char byRetryLimit); +void vt6655_mac_set_short_retry_limit(struct vnt_private *priv, unsigned char retry_limit); void MACvSetLongRetryLimit(struct vnt_private *priv, unsigned char byRetryLimit); -void MACvSetLoopbackMode(struct vnt_private *priv, unsigned char byLoopbackMode); - -void MACvSaveContext(struct vnt_private *priv, unsigned char *cxt_buf); -void MACvRestoreContext(struct vnt_private *priv, unsigned char *cxt_buf); - bool MACbSoftwareReset(struct vnt_private *priv); -bool MACbSafeSoftwareReset(struct vnt_private *priv); -bool MACbSafeRxOff(struct vnt_private *priv); -bool MACbSafeTxOff(struct vnt_private *priv); -bool MACbSafeStop(struct vnt_private *priv); bool MACbShutdown(struct vnt_private *priv); void MACvInitialize(struct vnt_private *priv); -void MACvSetCurrRx0DescAddr(struct vnt_private *priv, - u32 curr_desc_addr); -void MACvSetCurrRx1DescAddr(struct vnt_private *priv, - u32 curr_desc_addr); -void MACvSetCurrTXDescAddr(int iTxType, struct vnt_private *priv, - u32 curr_desc_addr); -void MACvSetCurrTx0DescAddrEx(struct vnt_private *priv, - u32 curr_desc_addr); -void MACvSetCurrAC0DescAddrEx(struct vnt_private *priv, - u32 curr_desc_addr); +void vt6655_mac_set_curr_rx_0_desc_addr(struct vnt_private *priv, u32 curr_desc_addr); +void vt6655_mac_set_curr_rx_1_desc_addr(struct vnt_private *priv, u32 curr_desc_addr); +void vt6655_mac_set_curr_tx_desc_addr(int tx_type, struct vnt_private *priv, u32 curr_desc_addr); void MACvSetCurrSyncDescAddrEx(struct vnt_private *priv, u32 curr_desc_addr); void MACvSetCurrATIMDescAddrEx(struct vnt_private *priv, diff --git a/drivers/staging/wlan-ng/cfg80211.c b/drivers/staging/wlan-ng/cfg80211.c index b7b56d8406d1c60c8b6a617f81dc8ac0362f421e..471bb310176fb890e6cf0ce8eb56b50a1c2c5a91 100644 --- a/drivers/staging/wlan-ng/cfg80211.c +++ b/drivers/staging/wlan-ng/cfg80211.c @@ -143,8 +143,8 @@ exit: } static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct wlandevice *wlandev = dev->ml_priv; u32 did; @@ -172,7 +172,7 @@ static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev, } static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool pairwise, + int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params*)) { @@ -202,7 +202,8 @@ static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev, } static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr) { struct wlandevice *wlandev = dev->ml_priv; u32 did; @@ -227,7 +228,8 @@ static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev, } static int prism2_set_default_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_index, bool unicast, bool multicast) + int link_id, u8 key_index, bool unicast, + bool multicast) { struct wlandevice *wlandev = dev->ml_priv; diff --git a/drivers/staging/wlan-ng/p80211netdev.h b/drivers/staging/wlan-ng/p80211netdev.h index 5654dc54ae9108eb8d8123af4d113bd12593b412..1cee51a1075ed6e646d32f70bc04b556e3d0a2de 100644 --- a/drivers/staging/wlan-ng/p80211netdev.h +++ b/drivers/staging/wlan-ng/p80211netdev.h @@ -137,8 +137,6 @@ struct p80211_frmrx { /* called by /proc/net/wireless */ struct iw_statistics *p80211wext_get_wireless_stats(struct net_device *dev); -/* wireless extensions' ioctls */ -extern struct iw_handler_def p80211wext_handler_def; /* WEP stuff */ #define NUM_WEPKEYS 4 diff --git a/drivers/target/iscsi/cxgbit/cxgbit_cm.c b/drivers/target/iscsi/cxgbit/cxgbit_cm.c index 3336d2b78bf77e51e792dec7838df7ba0f92b7d4..d9204c590d9aba62b34f3ee3f536852c2ac9c089 100644 --- a/drivers/target/iscsi/cxgbit/cxgbit_cm.c +++ b/drivers/target/iscsi/cxgbit/cxgbit_cm.c @@ -1202,7 +1202,7 @@ cxgbit_pass_accept_rpl(struct cxgbit_sock *csk, struct cpl_pass_accept_req *req) opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO); opt2 |= T5_ISS_F; - rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1); + rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1); opt2 |= T5_OPT_2_VALID_F; diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c index fb91423a4e2e48e19d48c869242d2a1e916fd0ac..c8470e7c0e108d2634ca2209f108e89be27b41d2 100644 --- a/drivers/target/target_core_alua.c +++ b/drivers/target/target_core_alua.c @@ -164,6 +164,9 @@ target_emulate_report_target_port_groups(struct se_cmd *cmd) spin_lock(&dev->t10_alua.tg_pt_gps_lock); list_for_each_entry(tg_pt_gp, &dev->t10_alua.tg_pt_gps_list, tg_pt_gp_list) { + /* Skip empty port groups */ + if (!tg_pt_gp->tg_pt_gp_members) + continue; /* * Check if the Target port group and Target port descriptor list * based on tg_pt_gp_members count will fit into the response payload. diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h index a889a6237d9c1818944b24af2d9b3596a8c7d443..30fcf69e1a1d59e7436776496cdc63ab6dc7e1ca 100644 --- a/drivers/target/target_core_internal.h +++ b/drivers/target/target_core_internal.h @@ -133,8 +133,6 @@ struct se_node_acl *core_tpg_add_initiator_node_acl(struct se_portal_group *tpg, void core_tpg_del_initiator_node_acl(struct se_node_acl *acl); /* target_core_transport.c */ -extern struct kmem_cache *se_tmr_req_cache; - int init_se_kmem_caches(void); void release_se_kmem_caches(void); u32 scsi_get_new_index(scsi_index_t); diff --git a/drivers/target/target_core_pscsi.c b/drivers/target/target_core_pscsi.c index e6a967ddc08ce6d4de42edb9d9e28bc6bafb7429..69a4c9581e80e50394c29bfb12ed24afc301f748 100644 --- a/drivers/target/target_core_pscsi.c +++ b/drivers/target/target_core_pscsi.c @@ -39,7 +39,7 @@ static inline struct pscsi_dev_virt *PSCSI_DEV(struct se_device *dev) } static sense_reason_t pscsi_execute_cmd(struct se_cmd *cmd); -static void pscsi_req_done(struct request *, blk_status_t); +static enum rq_end_io_ret pscsi_req_done(struct request *, blk_status_t); /* pscsi_attach_hba(): * @@ -500,7 +500,7 @@ static int pscsi_configure_device(struct se_device *dev) continue; /* * Functions will release the held struct scsi_host->host_lock - * before calling calling pscsi_add_device_to_list() to register + * before calling pscsi_add_device_to_list() to register * struct scsi_device with target_core_mod. */ switch (sd->type) { @@ -1002,7 +1002,8 @@ static sector_t pscsi_get_blocks(struct se_device *dev) return 0; } -static void pscsi_req_done(struct request *req, blk_status_t status) +static enum rq_end_io_ret pscsi_req_done(struct request *req, + blk_status_t status) { struct se_cmd *cmd = req->end_io_data; struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); @@ -1029,6 +1030,7 @@ static void pscsi_req_done(struct request *req, blk_status_t status) } blk_mq_free_request(req); + return RQ_END_IO_NONE; } static const struct target_backend_ops pscsi_ops = { diff --git a/drivers/target/target_core_spc.c b/drivers/target/target_core_spc.c index c14441c89bedd4a6a1d09aef67e2be71a87a529b..7cca3b15472b3a676046cf7cef541a1933c9ba10 100644 --- a/drivers/target/target_core_spc.c +++ b/drivers/target/target_core_spc.c @@ -115,6 +115,12 @@ spc_emulate_inquiry_std(struct se_cmd *cmd, unsigned char *buf) buf[5] |= 0x1; } + /* + * Set MULTIP bit to indicate presence of multiple SCSI target ports + */ + if (dev->export_count > 1) + buf[6] |= 0x10; + buf[7] = 0x2; /* CmdQue=1 */ /* diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 3deaeecb712e35e1b8720cf40c3cf7fd44ec4706..2940559c30860bf36ab4314aa234425610b82917 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -486,6 +486,7 @@ static struct genl_family tcmu_genl_family __ro_after_init = { .netnsok = true, .small_ops = tcmu_genl_ops, .n_small_ops = ARRAY_SIZE(tcmu_genl_ops), + .resv_start_op = TCMU_CMD_SET_FEATURES + 1, }; #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index)) diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index 28f87cd8b3ede04cdbe3208a23837a8ea3f2236a..290b1bb0e9cd724fe51188154aef53da808cfbd1 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -492,15 +492,18 @@ static bool is_normal_memory(pgprot_t p) #endif } -static int __check_mem_type(struct vm_area_struct *vma, unsigned long end) +static int __check_mem_type(struct mm_struct *mm, unsigned long start, + unsigned long end) { - while (vma && is_normal_memory(vma->vm_page_prot)) { - if (vma->vm_end >= end) - return 0; - vma = vma->vm_next; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, start); + + for_each_vma_range(vmi, vma, end) { + if (!is_normal_memory(vma->vm_page_prot)) + return -EINVAL; } - return -EINVAL; + return 0; } int optee_check_mem_type(unsigned long start, size_t num_pages) @@ -516,8 +519,7 @@ int optee_check_mem_type(unsigned long start, size_t num_pages) return 0; mmap_read_lock(mm); - rc = __check_mem_type(find_vma(mm, start), - start + num_pages * PAGE_SIZE); + rc = __check_mem_type(mm, start, start + num_pages * PAGE_SIZE); mmap_read_unlock(mm); return rc; diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c index 7ab31740cff84b9c88c738b1f3c05c77d160d531..0828240f27e62403ae80b0d3ef881ea7826dddfd 100644 --- a/drivers/tee/optee/ffa_abi.c +++ b/drivers/tee/optee/ffa_abi.c @@ -271,8 +271,8 @@ static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm, unsigned long start) { struct optee *optee = tee_get_drvdata(ctx->teedev); - const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops; struct ffa_device *ffa_dev = optee->ffa.ffa_dev; + const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops; struct ffa_mem_region_attributes mem_attr = { .receiver = ffa_dev->vm_id, .attrs = FFA_MEM_RW, @@ -294,14 +294,14 @@ static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm, if (rc) return rc; args.sg = sgt.sgl; - rc = ffa_ops->memory_share(ffa_dev, &args); + rc = mem_ops->memory_share(&args); sg_free_table(&sgt); if (rc) return rc; rc = optee_shm_add_ffa_handle(optee, shm, args.g_handle); if (rc) { - ffa_ops->memory_reclaim(args.g_handle, 0); + mem_ops->memory_reclaim(args.g_handle, 0); return rc; } @@ -314,8 +314,9 @@ static int optee_ffa_shm_unregister(struct tee_context *ctx, struct tee_shm *shm) { struct optee *optee = tee_get_drvdata(ctx->teedev); - const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops; struct ffa_device *ffa_dev = optee->ffa.ffa_dev; + const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops; + const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops; u64 global_handle = shm->sec_world_id; struct ffa_send_direct_data data = { .data0 = OPTEE_FFA_UNREGISTER_SHM, @@ -327,11 +328,11 @@ static int optee_ffa_shm_unregister(struct tee_context *ctx, optee_shm_rem_ffa_handle(optee, global_handle); shm->sec_world_id = 0; - rc = ffa_ops->sync_send_receive(ffa_dev, &data); + rc = msg_ops->sync_send_receive(ffa_dev, &data); if (rc) pr_err("Unregister SHM id 0x%llx rc %d\n", global_handle, rc); - rc = ffa_ops->memory_reclaim(global_handle, 0); + rc = mem_ops->memory_reclaim(global_handle, 0); if (rc) pr_err("mem_reclaim: 0x%llx %d", global_handle, rc); @@ -342,7 +343,7 @@ static int optee_ffa_shm_unregister_supp(struct tee_context *ctx, struct tee_shm *shm) { struct optee *optee = tee_get_drvdata(ctx->teedev); - const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops; + const struct ffa_mem_ops *mem_ops; u64 global_handle = shm->sec_world_id; int rc; @@ -353,7 +354,8 @@ static int optee_ffa_shm_unregister_supp(struct tee_context *ctx, */ optee_shm_rem_ffa_handle(optee, global_handle); - rc = ffa_ops->memory_reclaim(global_handle, 0); + mem_ops = optee->ffa.ffa_dev->ops->mem_ops; + rc = mem_ops->memory_reclaim(global_handle, 0); if (rc) pr_err("mem_reclaim: 0x%llx %d", global_handle, rc); @@ -529,8 +531,8 @@ static int optee_ffa_yielding_call(struct tee_context *ctx, struct optee_msg_arg *rpc_arg) { struct optee *optee = tee_get_drvdata(ctx->teedev); - const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops; struct ffa_device *ffa_dev = optee->ffa.ffa_dev; + const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops; struct optee_call_waiter w; u32 cmd = data->data0; u32 w4 = data->data1; @@ -541,7 +543,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx, /* Initialize waiter */ optee_cq_wait_init(&optee->call_queue, &w); while (true) { - rc = ffa_ops->sync_send_receive(ffa_dev, data); + rc = msg_ops->sync_send_receive(ffa_dev, data); if (rc) goto done; @@ -576,7 +578,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx, * OP-TEE has returned with a RPC request. * * Note that data->data4 (passed in register w7) is already - * filled in by ffa_ops->sync_send_receive() returning + * filled in by ffa_mem_ops->sync_send_receive() returning * above. */ cond_resched(); @@ -652,14 +654,15 @@ static int optee_ffa_do_call_with_arg(struct tee_context *ctx, */ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev, - const struct ffa_dev_ops *ops) + const struct ffa_ops *ops) { + const struct ffa_msg_ops *msg_ops = ops->msg_ops; struct ffa_send_direct_data data = { OPTEE_FFA_GET_API_VERSION }; int rc; - ops->mode_32bit_set(ffa_dev); + msg_ops->mode_32bit_set(ffa_dev); - rc = ops->sync_send_receive(ffa_dev, &data); + rc = msg_ops->sync_send_receive(ffa_dev, &data); if (rc) { pr_err("Unexpected error %d\n", rc); return false; @@ -672,7 +675,7 @@ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev, } data = (struct ffa_send_direct_data){ OPTEE_FFA_GET_OS_VERSION }; - rc = ops->sync_send_receive(ffa_dev, &data); + rc = msg_ops->sync_send_receive(ffa_dev, &data); if (rc) { pr_err("Unexpected error %d\n", rc); return false; @@ -687,14 +690,14 @@ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev, } static bool optee_ffa_exchange_caps(struct ffa_device *ffa_dev, - const struct ffa_dev_ops *ops, + const struct ffa_ops *ops, u32 *sec_caps, unsigned int *rpc_param_count) { struct ffa_send_direct_data data = { OPTEE_FFA_EXCHANGE_CAPABILITIES }; int rc; - rc = ops->sync_send_receive(ffa_dev, &data); + rc = ops->msg_ops->sync_send_receive(ffa_dev, &data); if (rc) { pr_err("Unexpected error %d", rc); return false; @@ -783,7 +786,7 @@ static void optee_ffa_remove(struct ffa_device *ffa_dev) static int optee_ffa_probe(struct ffa_device *ffa_dev) { - const struct ffa_dev_ops *ffa_ops; + const struct ffa_ops *ffa_ops; unsigned int rpc_param_count; struct tee_shm_pool *pool; struct tee_device *teedev; @@ -793,11 +796,7 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev) u32 sec_caps; int rc; - ffa_ops = ffa_dev_ops_get(ffa_dev); - if (!ffa_ops) { - pr_warn("failed \"method\" init: ffa\n"); - return -ENOENT; - } + ffa_ops = ffa_dev->ops; if (!optee_ffa_api_is_compatbile(ffa_dev, ffa_ops)) return -EINVAL; @@ -821,7 +820,6 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev) optee->ops = &optee_ffa_ops; optee->ffa.ffa_dev = ffa_dev; - optee->ffa.ffa_ops = ffa_ops; optee->rpc_param_count = rpc_param_count; teedev = tee_device_alloc(&optee_ffa_clnt_desc, NULL, optee->pool, diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h index a33d98d17cfd5f7a5dfcd10e261363b4f8e3c32d..04ae58892608c6b7e2627b156030fb39a3c5b5a2 100644 --- a/drivers/tee/optee/optee_private.h +++ b/drivers/tee/optee/optee_private.h @@ -111,7 +111,6 @@ struct optee_smc { */ struct optee_ffa { struct ffa_device *ffa_dev; - const struct ffa_dev_ops *ffa_ops; /* Serializes access to @global_ids */ struct mutex mutex; struct rhashtable global_ids; diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 1175f3a46859f74fe55a93985cb86c40274f4ce8..27295bda3e0bd25ec08fcbdbedf165df220ac139 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "tee_private.h" diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index def8e1a0399c33fd8a847a860c747ca686d6145b..2506c6c8ca83a07efd38ff5a970a21a8286c4c63 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -52,7 +52,7 @@ obj-$(CONFIG_DA9062_THERMAL) += da9062-thermal.o obj-y += intel/ obj-$(CONFIG_TI_SOC_THERMAL) += ti-soc-thermal/ obj-y += st/ -obj-$(CONFIG_QCOM_TSENS) += qcom/ +obj-y += qcom/ obj-y += tegra/ obj-$(CONFIG_HISI_THERMAL) += hisi_thermal.o obj-$(CONFIG_MTK_THERMAL) += mtk_thermal.o diff --git a/drivers/thermal/amlogic_thermal.c b/drivers/thermal/amlogic_thermal.c index e61b91d14ad1e59f35bbaf771490064e3aa9de84..d30cb791e63c86a32c12318c28dbf7e86397bd56 100644 --- a/drivers/thermal/amlogic_thermal.c +++ b/drivers/thermal/amlogic_thermal.c @@ -179,12 +179,12 @@ static int amlogic_thermal_disable(struct amlogic_thermal *data) return 0; } -static int amlogic_thermal_get_temp(void *data, int *temp) +static int amlogic_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { unsigned int tval; - struct amlogic_thermal *pdata = data; + struct amlogic_thermal *pdata = tz->devdata; - if (!data) + if (!pdata) return -EINVAL; regmap_read(pdata->regmap, TSENSOR_STAT0, &tval); @@ -195,7 +195,7 @@ static int amlogic_thermal_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops amlogic_thermal_ops = { +static const struct thermal_zone_device_ops amlogic_thermal_ops = { .get_temp = amlogic_thermal_get_temp, }; @@ -276,10 +276,10 @@ static int amlogic_thermal_probe(struct platform_device *pdev) return PTR_ERR(pdata->sec_ao_map); } - pdata->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, - 0, - pdata, - &amlogic_thermal_ops); + pdata->tzd = devm_thermal_of_zone_register(&pdev->dev, + 0, + pdata, + &amlogic_thermal_ops); if (IS_ERR(pdata->tzd)) { ret = PTR_ERR(pdata->tzd); dev_err(dev, "Failed to register tsensor: %d\n", ret); diff --git a/drivers/thermal/armada_thermal.c b/drivers/thermal/armada_thermal.c index c2ebfb5be4b3dc715c49b1bfec275df6836a34ba..52d63b3997fe107750a9f0be221cd9aa1c8c4d00 100644 --- a/drivers/thermal/armada_thermal.c +++ b/drivers/thermal/armada_thermal.c @@ -420,9 +420,9 @@ static struct thermal_zone_device_ops legacy_ops = { .get_temp = armada_get_temp_legacy, }; -static int armada_get_temp(void *_sensor, int *temp) +static int armada_get_temp(struct thermal_zone_device *tz, int *temp) { - struct armada_thermal_sensor *sensor = _sensor; + struct armada_thermal_sensor *sensor = tz->devdata; struct armada_thermal_priv *priv = sensor->priv; int ret; @@ -450,7 +450,7 @@ unlock_mutex: return ret; } -static const struct thermal_zone_of_device_ops of_ops = { +static const struct thermal_zone_device_ops of_ops = { .get_temp = armada_get_temp, }; @@ -928,9 +928,9 @@ static int armada_thermal_probe(struct platform_device *pdev) /* Register the sensor */ sensor->priv = priv; sensor->id = sensor_id; - tz = devm_thermal_zone_of_sensor_register(&pdev->dev, - sensor->id, sensor, - &of_ops); + tz = devm_thermal_of_zone_register(&pdev->dev, + sensor->id, sensor, + &of_ops); if (IS_ERR(tz)) { dev_info(&pdev->dev, "Thermal sensor %d unavailable\n", sensor_id); diff --git a/drivers/thermal/broadcom/bcm2711_thermal.c b/drivers/thermal/broadcom/bcm2711_thermal.c index e9bef5c3414b6fe56cc7be0568cea422f95fc4c9..1f8651d15160c972cf477a745b4142a2d7e938b1 100644 --- a/drivers/thermal/broadcom/bcm2711_thermal.c +++ b/drivers/thermal/broadcom/bcm2711_thermal.c @@ -31,11 +31,11 @@ struct bcm2711_thermal_priv { struct thermal_zone_device *thermal; }; -static int bcm2711_get_temp(void *data, int *temp) +static int bcm2711_get_temp(struct thermal_zone_device *tz, int *temp) { - struct bcm2711_thermal_priv *priv = data; - int slope = thermal_zone_get_slope(priv->thermal); - int offset = thermal_zone_get_offset(priv->thermal); + struct bcm2711_thermal_priv *priv = tz->devdata; + int slope = thermal_zone_get_slope(tz); + int offset = thermal_zone_get_offset(tz); u32 val; int ret; @@ -54,7 +54,7 @@ static int bcm2711_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops bcm2711_thermal_of_ops = { +static const struct thermal_zone_device_ops bcm2711_thermal_of_ops = { .get_temp = bcm2711_get_temp, }; @@ -88,8 +88,8 @@ static int bcm2711_thermal_probe(struct platform_device *pdev) } priv->regmap = regmap; - thermal = devm_thermal_zone_of_sensor_register(dev, 0, priv, - &bcm2711_thermal_of_ops); + thermal = devm_thermal_of_zone_register(dev, 0, priv, + &bcm2711_thermal_of_ops); if (IS_ERR(thermal)) { ret = PTR_ERR(thermal); dev_err(dev, "could not register sensor: %d\n", ret); diff --git a/drivers/thermal/broadcom/bcm2835_thermal.c b/drivers/thermal/broadcom/bcm2835_thermal.c index c8e4344d5a3d8b7795614b8495fcf794a82566b1..2c67841a111514f0dde39297154aa00727369d5a 100644 --- a/drivers/thermal/broadcom/bcm2835_thermal.c +++ b/drivers/thermal/broadcom/bcm2835_thermal.c @@ -88,9 +88,9 @@ static int bcm2835_thermal_temp2adc(int temp, int offset, int slope) return temp; } -static int bcm2835_thermal_get_temp(void *d, int *temp) +static int bcm2835_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct bcm2835_thermal_data *data = d; + struct bcm2835_thermal_data *data = tz->devdata; u32 val = readl(data->regs + BCM2835_TS_TSENSSTAT); if (!(val & BCM2835_TS_TSENSSTAT_VALID)) @@ -135,7 +135,7 @@ static void bcm2835_thermal_debugfs(struct platform_device *pdev) debugfs_create_regset32("regset", 0444, data->debugfsdir, regset); } -static const struct thermal_zone_of_device_ops bcm2835_thermal_ops = { +static const struct thermal_zone_device_ops bcm2835_thermal_ops = { .get_temp = bcm2835_thermal_get_temp, }; @@ -206,8 +206,8 @@ static int bcm2835_thermal_probe(struct platform_device *pdev) data->clk, rate); /* register of thermal sensor and get info from DT */ - tz = thermal_zone_of_sensor_register(&pdev->dev, 0, data, - &bcm2835_thermal_ops); + tz = devm_thermal_of_zone_register(&pdev->dev, 0, data, + &bcm2835_thermal_ops); if (IS_ERR(tz)) { err = PTR_ERR(tz); dev_err(&pdev->dev, @@ -277,7 +277,7 @@ static int bcm2835_thermal_probe(struct platform_device *pdev) return 0; err_tz: - thermal_zone_of_sensor_unregister(&pdev->dev, tz); + thermal_of_zone_unregister(tz); err_clk: clk_disable_unprepare(data->clk); @@ -290,7 +290,7 @@ static int bcm2835_thermal_remove(struct platform_device *pdev) struct thermal_zone_device *tz = data->tz; debugfs_remove_recursive(data->debugfsdir); - thermal_zone_of_sensor_unregister(&pdev->dev, tz); + thermal_of_zone_unregister(tz); clk_disable_unprepare(data->clk); return 0; diff --git a/drivers/thermal/broadcom/brcmstb_thermal.c b/drivers/thermal/broadcom/brcmstb_thermal.c index 0cedb8b4f00a40626764314ea3268ab65f28f4b7..c79c6cfdd74d504390121ab71697932e230dff93 100644 --- a/drivers/thermal/broadcom/brcmstb_thermal.c +++ b/drivers/thermal/broadcom/brcmstb_thermal.c @@ -105,7 +105,7 @@ static struct avs_tmon_trip avs_tmon_trips[] = { struct brcmstb_thermal_params { unsigned int offset; unsigned int mult; - const struct thermal_zone_of_device_ops *of_ops; + const struct thermal_zone_device_ops *of_ops; }; struct brcmstb_thermal_priv { @@ -150,9 +150,9 @@ static inline u32 avs_tmon_temp_to_code(struct brcmstb_thermal_priv *priv, return (u32)((offset - temp) / mult); } -static int brcmstb_get_temp(void *data, int *temp) +static int brcmstb_get_temp(struct thermal_zone_device *tz, int *temp) { - struct brcmstb_thermal_priv *priv = data; + struct brcmstb_thermal_priv *priv = tz->devdata; u32 val; long t; @@ -260,9 +260,9 @@ static irqreturn_t brcmstb_tmon_irq_thread(int irq, void *data) return IRQ_HANDLED; } -static int brcmstb_set_trips(void *data, int low, int high) +static int brcmstb_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct brcmstb_thermal_priv *priv = data; + struct brcmstb_thermal_priv *priv = tz->devdata; dev_dbg(priv->dev, "set trips %d <--> %d\n", low, high); @@ -288,7 +288,7 @@ static int brcmstb_set_trips(void *data, int low, int high) return 0; } -static const struct thermal_zone_of_device_ops brcmstb_16nm_of_ops = { +static const struct thermal_zone_device_ops brcmstb_16nm_of_ops = { .get_temp = brcmstb_get_temp, }; @@ -298,7 +298,7 @@ static const struct brcmstb_thermal_params brcmstb_16nm_params = { .of_ops = &brcmstb_16nm_of_ops, }; -static const struct thermal_zone_of_device_ops brcmstb_28nm_of_ops = { +static const struct thermal_zone_device_ops brcmstb_28nm_of_ops = { .get_temp = brcmstb_get_temp, .set_trips = brcmstb_set_trips, }; @@ -318,7 +318,7 @@ MODULE_DEVICE_TABLE(of, brcmstb_thermal_id_table); static int brcmstb_thermal_probe(struct platform_device *pdev) { - const struct thermal_zone_of_device_ops *of_ops; + const struct thermal_zone_device_ops *of_ops; struct thermal_zone_device *thermal; struct brcmstb_thermal_priv *priv; struct resource *res; @@ -341,8 +341,8 @@ static int brcmstb_thermal_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); of_ops = priv->temp_params->of_ops; - thermal = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, priv, - of_ops); + thermal = devm_thermal_of_zone_register(&pdev->dev, 0, priv, + of_ops); if (IS_ERR(thermal)) { ret = PTR_ERR(thermal); dev_err(&pdev->dev, "could not register sensor: %d\n", ret); diff --git a/drivers/thermal/broadcom/ns-thermal.c b/drivers/thermal/broadcom/ns-thermal.c index c9468ba9d449baed3961678f2b206d573c6ee757..07a8a3f49bd0f98c5bdede4bc12d85d25b823fcd 100644 --- a/drivers/thermal/broadcom/ns-thermal.c +++ b/drivers/thermal/broadcom/ns-thermal.c @@ -14,19 +14,14 @@ #define PVTMON_CONTROL0_SEL_TEST_MODE 0x0000000e #define PVTMON_STATUS 0x08 -struct ns_thermal { - struct thermal_zone_device *tz; - void __iomem *pvtmon; -}; - -static int ns_thermal_get_temp(void *data, int *temp) +static int ns_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct ns_thermal *ns_thermal = data; - int offset = thermal_zone_get_offset(ns_thermal->tz); - int slope = thermal_zone_get_slope(ns_thermal->tz); + void __iomem *pvtmon = tz->devdata; + int offset = thermal_zone_get_offset(tz); + int slope = thermal_zone_get_slope(tz); u32 val; - val = readl(ns_thermal->pvtmon + PVTMON_CONTROL0); + val = readl(pvtmon + PVTMON_CONTROL0); if ((val & PVTMON_CONTROL0_SEL_MASK) != PVTMON_CONTROL0_SEL_TEMP_MONITOR) { /* Clear current mode selection */ val &= ~PVTMON_CONTROL0_SEL_MASK; @@ -34,50 +29,47 @@ static int ns_thermal_get_temp(void *data, int *temp) /* Set temp monitor mode (it's the default actually) */ val |= PVTMON_CONTROL0_SEL_TEMP_MONITOR; - writel(val, ns_thermal->pvtmon + PVTMON_CONTROL0); + writel(val, pvtmon + PVTMON_CONTROL0); } - val = readl(ns_thermal->pvtmon + PVTMON_STATUS); + val = readl(pvtmon + PVTMON_STATUS); *temp = slope * val + offset; return 0; } -static const struct thermal_zone_of_device_ops ns_thermal_ops = { +static const struct thermal_zone_device_ops ns_thermal_ops = { .get_temp = ns_thermal_get_temp, }; static int ns_thermal_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct ns_thermal *ns_thermal; - - ns_thermal = devm_kzalloc(dev, sizeof(*ns_thermal), GFP_KERNEL); - if (!ns_thermal) - return -ENOMEM; + struct thermal_zone_device *tz; + void __iomem *pvtmon; - ns_thermal->pvtmon = of_iomap(dev_of_node(dev), 0); - if (WARN_ON(!ns_thermal->pvtmon)) + pvtmon = of_iomap(dev_of_node(dev), 0); + if (WARN_ON(!pvtmon)) return -ENOENT; - ns_thermal->tz = devm_thermal_zone_of_sensor_register(dev, 0, - ns_thermal, - &ns_thermal_ops); - if (IS_ERR(ns_thermal->tz)) { - iounmap(ns_thermal->pvtmon); - return PTR_ERR(ns_thermal->tz); + tz = devm_thermal_of_zone_register(dev, 0, + pvtmon, + &ns_thermal_ops); + if (IS_ERR(tz)) { + iounmap(pvtmon); + return PTR_ERR(tz); } - platform_set_drvdata(pdev, ns_thermal); + platform_set_drvdata(pdev, pvtmon); return 0; } static int ns_thermal_remove(struct platform_device *pdev) { - struct ns_thermal *ns_thermal = platform_get_drvdata(pdev); + void __iomem *pvtmon = platform_get_drvdata(pdev); - iounmap(ns_thermal->pvtmon); + iounmap(pvtmon); return 0; } diff --git a/drivers/thermal/broadcom/sr-thermal.c b/drivers/thermal/broadcom/sr-thermal.c index 85ab9edd580cc7151824bcb1e1681e1d63a7259c..2b93502543ff4be8254d3bee4f016d9b318deb5c 100644 --- a/drivers/thermal/broadcom/sr-thermal.c +++ b/drivers/thermal/broadcom/sr-thermal.c @@ -19,7 +19,6 @@ #define SR_TMON_MAX_LIST 6 struct sr_tmon { - struct thermal_zone_device *tz; unsigned int crit_temp; unsigned int tmon_id; struct sr_thermal *priv; @@ -31,9 +30,9 @@ struct sr_thermal { struct sr_tmon tmon[SR_TMON_MAX_LIST]; }; -static int sr_get_temp(void *data, int *temp) +static int sr_get_temp(struct thermal_zone_device *tz, int *temp) { - struct sr_tmon *tmon = data; + struct sr_tmon *tmon = tz->devdata; struct sr_thermal *sr_thermal = tmon->priv; *temp = readl(sr_thermal->regs + SR_TMON_TEMP_BASE(tmon->tmon_id)); @@ -41,13 +40,14 @@ static int sr_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops sr_tz_ops = { +static const struct thermal_zone_device_ops sr_tz_ops = { .get_temp = sr_get_temp, }; static int sr_thermal_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + struct thermal_zone_device *tz; struct sr_thermal *sr_thermal; struct sr_tmon *tmon; struct resource *res; @@ -84,10 +84,10 @@ static int sr_thermal_probe(struct platform_device *pdev) writel(0, sr_thermal->regs + SR_TMON_TEMP_BASE(i)); tmon->tmon_id = i; tmon->priv = sr_thermal; - tmon->tz = devm_thermal_zone_of_sensor_register(dev, i, tmon, - &sr_tz_ops); - if (IS_ERR(tmon->tz)) - return PTR_ERR(tmon->tz); + tz = devm_thermal_of_zone_register(dev, i, tmon, + &sr_tz_ops); + if (IS_ERR(tz)) + return PTR_ERR(tz); dev_dbg(dev, "thermal sensor %d registered\n", i); } diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index b76293cc989c434feae8da029256c267a9b96b0c..9f8b438fcf8f826edcd7befb9fd52813ec971dd8 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -475,7 +475,7 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev, /** * __cpufreq_cooling_register - helper function to create cpufreq cooling device - * @np: a valid struct device_node to the cooling device device tree node + * @np: a valid struct device_node to the cooling device tree node * @policy: cpufreq policy * Normally this should be same as cpufreq policy->related_cpus. * @em: Energy Model of the cpufreq policy @@ -501,17 +501,17 @@ __cpufreq_cooling_register(struct device_node *np, struct thermal_cooling_device_ops *cooling_ops; char *name; + if (IS_ERR_OR_NULL(policy)) { + pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy); + return ERR_PTR(-EINVAL); + } + dev = get_cpu_device(policy->cpu); if (unlikely(!dev)) { pr_warn("No cpu device for cpu %d\n", policy->cpu); return ERR_PTR(-ENODEV); } - if (IS_ERR_OR_NULL(policy)) { - pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy); - return ERR_PTR(-EINVAL); - } - i = cpufreq_table_count_valid_entries(policy); if (!i) { pr_debug("%s: CPUFreq table not found or has no valid entries\n", diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c index 180edec34e079b5a44c6565bf65aa18540c7cfeb..7dcfde7a9f2c1994d3abaccfc04598b11650f07b 100644 --- a/drivers/thermal/da9062-thermal.c +++ b/drivers/thermal/da9062-thermal.c @@ -248,10 +248,9 @@ static int da9062_thermal_probe(struct platform_device *pdev) jiffies_to_msecs(thermal->zone->passive_delay_jiffies)); ret = platform_get_irq_byname(pdev, "THERMAL"); - if (ret < 0) { - dev_err(&pdev->dev, "Failed to get platform IRQ.\n"); + if (ret < 0) goto err_zone; - } + thermal->irq = ret; ret = request_threaded_irq(thermal->irq, NULL, diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c index 121cf853e545e41611b2ba9f76e8b6ffb8548b12..cb10e280681fcc027cd1a800d684a6cfaf4a2db7 100644 --- a/drivers/thermal/db8500_thermal.c +++ b/drivers/thermal/db8500_thermal.c @@ -58,9 +58,9 @@ struct db8500_thermal_zone { }; /* Callback to get current temperature */ -static int db8500_thermal_get_temp(void *data, int *temp) +static int db8500_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct db8500_thermal_zone *th = data; + struct db8500_thermal_zone *th = tz->devdata; /* * TODO: There is no PRCMU interface to get temperature data currently, @@ -72,7 +72,7 @@ static int db8500_thermal_get_temp(void *data, int *temp) return 0; } -static struct thermal_zone_of_device_ops thdev_ops = { +static const struct thermal_zone_device_ops thdev_ops = { .get_temp = db8500_thermal_get_temp, }; @@ -182,7 +182,7 @@ static int db8500_thermal_probe(struct platform_device *pdev) } /* register of thermal sensor and get info from DT */ - th->tz = devm_thermal_zone_of_sensor_register(dev, 0, th, &thdev_ops); + th->tz = devm_thermal_of_zone_register(dev, 0, th, &thdev_ops); if (IS_ERR(th->tz)) { dev_err(dev, "register thermal zone sensor failed\n"); return PTR_ERR(th->tz); diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c index 991a1c54296de0dd577016305d0514ce474e2c9e..a08bbe33be9679c4b90c7dd2c03ed2b7b756d18c 100644 --- a/drivers/thermal/gov_bang_bang.c +++ b/drivers/thermal/gov_bang_bang.c @@ -31,8 +31,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip) trip, trip_temp, tz->temperature, trip_hyst); - mutex_lock(&tz->lock); - list_for_each_entry(instance, &tz->thermal_instances, tz_node) { if (instance->trip != trip) continue; @@ -65,8 +63,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip) instance->cdev->updated = false; /* cdev needs update */ mutex_unlock(&instance->cdev->lock); } - - mutex_unlock(&tz->lock); } /** @@ -100,15 +96,13 @@ static int bang_bang_control(struct thermal_zone_device *tz, int trip) { struct thermal_instance *instance; - thermal_zone_trip_update(tz, trip); + lockdep_assert_held(&tz->lock); - mutex_lock(&tz->lock); + thermal_zone_trip_update(tz, trip); list_for_each_entry(instance, &tz->thermal_instances, tz_node) thermal_cdev_update(instance->cdev); - mutex_unlock(&tz->lock); - return 0; } diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c index 6a2abcfc648f7f9b08464603ff89a2afa704a09c..a4ee4661e9cc44b84b0c29619cf9c908d3a19d71 100644 --- a/drivers/thermal/gov_fair_share.c +++ b/drivers/thermal/gov_fair_share.c @@ -82,7 +82,7 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip) int total_instance = 0; int cur_trip_level = get_trip_level(tz); - mutex_lock(&tz->lock); + lockdep_assert_held(&tz->lock); list_for_each_entry(instance, &tz->thermal_instances, tz_node) { if (instance->trip != trip) @@ -112,7 +112,6 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip) mutex_unlock(&cdev->lock); } - mutex_unlock(&tz->lock); return 0; } diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c index 1d505247096728f97cddb105384b0e95f21d4414..2d1aeaba38a889b0a62025ca7b089aa430f40cbf 100644 --- a/drivers/thermal/gov_power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c @@ -392,8 +392,6 @@ static int allocate_power(struct thermal_zone_device *tz, int i, num_actors, total_weight, ret = 0; int trip_max_desired_temperature = params->trip_max_desired_temperature; - mutex_lock(&tz->lock); - num_actors = 0; total_weight = 0; list_for_each_entry(instance, &tz->thermal_instances, tz_node) { @@ -404,10 +402,8 @@ static int allocate_power(struct thermal_zone_device *tz, } } - if (!num_actors) { - ret = -ENODEV; - goto unlock; - } + if (!num_actors) + return -ENODEV; /* * We need to allocate five arrays of the same size: @@ -421,10 +417,8 @@ static int allocate_power(struct thermal_zone_device *tz, BUILD_BUG_ON(sizeof(*req_power) != sizeof(*extra_actor_power)); BUILD_BUG_ON(sizeof(*req_power) != sizeof(*weighted_req_power)); req_power = kcalloc(num_actors * 5, sizeof(*req_power), GFP_KERNEL); - if (!req_power) { - ret = -ENOMEM; - goto unlock; - } + if (!req_power) + return -ENOMEM; max_power = &req_power[num_actors]; granted_power = &req_power[2 * num_actors]; @@ -496,8 +490,6 @@ static int allocate_power(struct thermal_zone_device *tz, control_temp - tz->temperature); kfree(req_power); -unlock: - mutex_unlock(&tz->lock); return ret; } @@ -576,7 +568,6 @@ static void allow_maximum_power(struct thermal_zone_device *tz, bool update) struct power_allocator_params *params = tz->governor_data; u32 req_power; - mutex_lock(&tz->lock); list_for_each_entry(instance, &tz->thermal_instances, tz_node) { struct thermal_cooling_device *cdev = instance->cdev; @@ -598,7 +589,6 @@ static void allow_maximum_power(struct thermal_zone_device *tz, bool update) mutex_unlock(&instance->cdev->lock); } - mutex_unlock(&tz->lock); } /** @@ -712,6 +702,8 @@ static int power_allocator_throttle(struct thermal_zone_device *tz, int trip) struct power_allocator_params *params = tz->governor_data; bool update; + lockdep_assert_held(&tz->lock); + /* * We get called for every trip point but we only need to do * our calculations once diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c index 9729b46d0258aacfcd7506937e6059377e7f7d7e..cdd3354bc27fee2418a2c9ee449c7c83c89d63f1 100644 --- a/drivers/thermal/gov_step_wise.c +++ b/drivers/thermal/gov_step_wise.c @@ -117,8 +117,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip) dev_dbg(&tz->device, "Trip%d[type=%d,temp=%d]:trend=%d,throttle=%d\n", trip, trip_type, trip_temp, trend, throttle); - mutex_lock(&tz->lock); - list_for_each_entry(instance, &tz->thermal_instances, tz_node) { if (instance->trip != trip) continue; @@ -145,8 +143,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip) instance->cdev->updated = false; /* cdev needs update */ mutex_unlock(&instance->cdev->lock); } - - mutex_unlock(&tz->lock); } /** @@ -164,15 +160,13 @@ static int step_wise_throttle(struct thermal_zone_device *tz, int trip) { struct thermal_instance *instance; - thermal_zone_trip_update(tz, trip); + lockdep_assert_held(&tz->lock); - mutex_lock(&tz->lock); + thermal_zone_trip_update(tz, trip); list_for_each_entry(instance, &tz->thermal_instances, tz_node) thermal_cdev_update(instance->cdev); - mutex_unlock(&tz->lock); - return 0; } diff --git a/drivers/thermal/gov_user_space.c b/drivers/thermal/gov_user_space.c index a62a4e90bd3f5b8e2ce57073d4b1aa2df9892aba..8bc1c22aaf03807345cf90cb7a413e91326f59a1 100644 --- a/drivers/thermal/gov_user_space.c +++ b/drivers/thermal/gov_user_space.c @@ -34,7 +34,8 @@ static int notify_user_space(struct thermal_zone_device *tz, int trip) char *thermal_prop[5]; int i; - mutex_lock(&tz->lock); + lockdep_assert_held(&tz->lock); + thermal_prop[0] = kasprintf(GFP_KERNEL, "NAME=%s", tz->type); thermal_prop[1] = kasprintf(GFP_KERNEL, "TEMP=%d", tz->temperature); thermal_prop[2] = kasprintf(GFP_KERNEL, "TRIP=%d", trip); @@ -43,7 +44,7 @@ static int notify_user_space(struct thermal_zone_device *tz, int trip) kobject_uevent_env(&tz->device.kobj, KOBJ_CHANGE, thermal_prop); for (i = 0; i < 4; ++i) kfree(thermal_prop[i]); - mutex_unlock(&tz->lock); + return 0; } diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c index 19a242c69ce63dd8d1f6441f4e3dd77ec0c66083..d6974db7aaf7635a7331c94cb160bd1e98322080 100644 --- a/drivers/thermal/hisi_thermal.c +++ b/drivers/thermal/hisi_thermal.c @@ -434,9 +434,9 @@ static int hi3660_thermal_probe(struct hisi_thermal_data *data) return 0; } -static int hisi_thermal_get_temp(void *__data, int *temp) +static int hisi_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct hisi_thermal_sensor *sensor = __data; + struct hisi_thermal_sensor *sensor = tz->devdata; struct hisi_thermal_data *data = sensor->data; *temp = data->ops->get_temp(sensor); @@ -447,7 +447,7 @@ static int hisi_thermal_get_temp(void *__data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops hisi_of_thermal_ops = { +static const struct thermal_zone_device_ops hisi_of_thermal_ops = { .get_temp = hisi_thermal_get_temp, }; @@ -459,7 +459,7 @@ static irqreturn_t hisi_thermal_alarm_irq_thread(int irq, void *dev) data->ops->irq_handler(sensor); - hisi_thermal_get_temp(sensor, &temp); + temp = data->ops->get_temp(sensor); if (temp >= sensor->thres_temp) { dev_crit(&data->pdev->dev, @@ -484,9 +484,9 @@ static int hisi_thermal_register_sensor(struct platform_device *pdev, int ret, i; const struct thermal_trip *trip; - sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, - sensor->id, sensor, - &hisi_of_thermal_ops); + sensor->tzd = devm_thermal_of_zone_register(&pdev->dev, + sensor->id, sensor, + &hisi_of_thermal_ops); if (IS_ERR(sensor->tzd)) { ret = PTR_ERR(sensor->tzd); sensor->tzd = NULL; diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c index af666bd9e8d4d5de2379893f03ab3c151acce014..e2c2673025a7ac45754574d2d824fa84fed1c37d 100644 --- a/drivers/thermal/imx8mm_thermal.c +++ b/drivers/thermal/imx8mm_thermal.c @@ -96,15 +96,15 @@ static int imx8mp_tmu_get_temp(void *data, int *temp) return 0; } -static int tmu_get_temp(void *data, int *temp) +static int tmu_get_temp(struct thermal_zone_device *tz, int *temp) { - struct tmu_sensor *sensor = data; + struct tmu_sensor *sensor = tz->devdata; struct imx8mm_tmu *tmu = sensor->priv; - return tmu->socdata->get_temp(data, temp); + return tmu->socdata->get_temp(sensor, temp); } -static struct thermal_zone_of_device_ops tmu_tz_ops = { +static const struct thermal_zone_device_ops tmu_tz_ops = { .get_temp = tmu_get_temp, }; @@ -165,9 +165,9 @@ static int imx8mm_tmu_probe(struct platform_device *pdev) for (i = 0; i < data->num_sensors; i++) { tmu->sensors[i].priv = tmu; tmu->sensors[i].tzd = - devm_thermal_zone_of_sensor_register(&pdev->dev, i, - &tmu->sensors[i], - &tmu_tz_ops); + devm_thermal_of_zone_register(&pdev->dev, i, + &tmu->sensors[i], + &tmu_tz_ops); if (IS_ERR(tmu->sensors[i].tzd)) { ret = PTR_ERR(tmu->sensors[i].tzd); dev_err(&pdev->dev, diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c index 331a241eb0ef315606b24bb7c841706d21c710b1..5d92b70a5d53a3f1be716f6adcb16aff3323a618 100644 --- a/drivers/thermal/imx_sc_thermal.c +++ b/drivers/thermal/imx_sc_thermal.c @@ -43,11 +43,11 @@ struct imx_sc_msg_misc_get_temp { } data; } __packed __aligned(4); -static int imx_sc_thermal_get_temp(void *data, int *temp) +static int imx_sc_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { struct imx_sc_msg_misc_get_temp msg; struct imx_sc_rpc_msg *hdr = &msg.hdr; - struct imx_sc_sensor *sensor = data; + struct imx_sc_sensor *sensor = tz->devdata; int ret; msg.data.req.resource_id = sensor->resource_id; @@ -70,65 +70,61 @@ static int imx_sc_thermal_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops imx_sc_thermal_ops = { +static const struct thermal_zone_device_ops imx_sc_thermal_ops = { .get_temp = imx_sc_thermal_get_temp, }; static int imx_sc_thermal_probe(struct platform_device *pdev) { - struct device_node *np, *child, *sensor_np; struct imx_sc_sensor *sensor; - int ret; + const int *resource_id; + int i, ret; ret = imx_scu_get_handle(&thermal_ipc_handle); if (ret) return ret; - np = of_find_node_by_name(NULL, "thermal-zones"); - if (!np) - return -ENODEV; + resource_id = of_device_get_match_data(&pdev->dev); + if (!resource_id) + return -EINVAL; - sensor_np = of_node_get(pdev->dev.of_node); + for (i = 0; resource_id[i] > 0; i++) { - for_each_available_child_of_node(np, child) { sensor = devm_kzalloc(&pdev->dev, sizeof(*sensor), GFP_KERNEL); - if (!sensor) { - of_node_put(child); - ret = -ENOMEM; - goto put_node; - } + if (!sensor) + return -ENOMEM; - ret = thermal_zone_of_get_sensor_id(child, - sensor_np, - &sensor->resource_id); - if (ret < 0) { - dev_err(&pdev->dev, - "failed to get valid sensor resource id: %d\n", - ret); - of_node_put(child); - break; - } + sensor->resource_id = resource_id[i]; - sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, - sensor->resource_id, - sensor, - &imx_sc_thermal_ops); + sensor->tzd = devm_thermal_of_zone_register(&pdev->dev, sensor->resource_id, + sensor, &imx_sc_thermal_ops); if (IS_ERR(sensor->tzd)) { - dev_err(&pdev->dev, "failed to register thermal zone\n"); + /* + * Save the error value before freeing the + * sensor pointer, otherwise we endup with a + * use-after-free error + */ ret = PTR_ERR(sensor->tzd); - of_node_put(child); - break; + + devm_kfree(&pdev->dev, sensor); + + /* + * The thermal framework notifies us there is + * no thermal zone description for such a + * sensor id + */ + if (ret == -ENODEV) + continue; + + dev_err(&pdev->dev, "failed to register thermal zone\n"); + return ret; } if (devm_thermal_add_hwmon_sysfs(sensor->tzd)) dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n"); } -put_node: - of_node_put(sensor_np); - of_node_put(np); - - return ret; + return 0; } static int imx_sc_thermal_remove(struct platform_device *pdev) @@ -136,8 +132,10 @@ static int imx_sc_thermal_remove(struct platform_device *pdev) return 0; } +static int imx_sc_sensors[] = { IMX_SC_R_SYSTEM, IMX_SC_R_PMIC_0, -1 }; + static const struct of_device_id imx_sc_thermal_table[] = { - { .compatible = "fsl,imx-sc-thermal", }, + { .compatible = "fsl,imx-sc-thermal", .data = imx_sc_sensors }, {} }; MODULE_DEVICE_TABLE(of, imx_sc_thermal_table); diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index 80d4e0676083ac56ade7bea96834203f1d95eed0..db8a6f63657d660a4992469d19932a2b1128a537 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -527,7 +527,7 @@ static void int3400_setup_gddv(struct int3400_thermal_priv *priv) priv->data_vault = kmemdup(obj->package.elements[0].buffer.pointer, obj->package.elements[0].buffer.length, GFP_KERNEL); - if (!priv->data_vault) + if (ZERO_OR_NULL_PTR(priv->data_vault)) goto out_free; bin_attr_data_vault.private = priv->data_vault; @@ -597,7 +597,7 @@ static int int3400_thermal_probe(struct platform_device *pdev) goto free_imok; } - if (priv->data_vault) { + if (!ZERO_OR_NULL_PTR(priv->data_vault)) { result = sysfs_create_group(&pdev->dev.kobj, &data_attribute_group); if (result) @@ -614,7 +614,7 @@ static int int3400_thermal_probe(struct platform_device *pdev) free_sysfs: cleanup_odvp(priv); - if (priv->data_vault) { + if (!ZERO_OR_NULL_PTR(priv->data_vault)) { sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group); kfree(priv->data_vault); } @@ -647,7 +647,7 @@ static int int3400_thermal_remove(struct platform_device *pdev) if (!priv->rel_misc_dev_res) acpi_thermal_rel_misc_device_remove(priv->adev->handle); - if (priv->data_vault) + if (!ZERO_OR_NULL_PTR(priv->data_vault)) sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group); sysfs_remove_group(&pdev->dev.kobj, &uuid_attribute_group); sysfs_remove_group(&pdev->dev.kobj, &imok_attribute_group); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index c2dc4c158b9dcca6fb0d1c5e0a0c03a221a4ac18..bf1b1cdfade4a27d691e7c20e36beb0373e721ee 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -373,18 +373,7 @@ static struct pci_driver proc_thermal_pci_driver = { .driver.pm = &proc_thermal_pci_pm, }; -static int __init proc_thermal_init(void) -{ - return pci_register_driver(&proc_thermal_pci_driver); -} - -static void __exit proc_thermal_exit(void) -{ - pci_unregister_driver(&proc_thermal_pci_driver); -} - -module_init(proc_thermal_init); -module_exit(proc_thermal_exit); +module_pci_driver(proc_thermal_pci_driver); MODULE_AUTHOR("Srinivas Pandruvada "); MODULE_DESCRIPTION("Processor Thermal Reporting Device Driver"); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c index 4571a1a53b841f22eba3a7c0fada9503fe2287bd..09e032f822f30e24ee119ee5724fa0d4fcf094f1 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c @@ -151,18 +151,7 @@ static struct pci_driver proc_thermal_pci_driver = { .driver.pm = &proc_thermal_pci_pm, }; -static int __init proc_thermal_init(void) -{ - return pci_register_driver(&proc_thermal_pci_driver); -} - -static void __exit proc_thermal_exit(void) -{ - pci_unregister_driver(&proc_thermal_pci_driver); -} - -module_init(proc_thermal_init); -module_exit(proc_thermal_exit); +module_pci_driver(proc_thermal_pci_driver); MODULE_AUTHOR("Srinivas Pandruvada "); MODULE_DESCRIPTION("Processor Thermal Reporting Device Driver"); diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c index c841ab37e7c6d9e34b9f9604222e6a77071ff13e..2a5570b9799a9afcba4115b7df91ce06e87532eb 100644 --- a/drivers/thermal/intel/intel_powerclamp.c +++ b/drivers/thermal/intel/intel_powerclamp.c @@ -62,8 +62,7 @@ static struct dentry *debug_dir; static unsigned int set_target_ratio; static unsigned int current_ratio; static bool should_skip; -static bool reduce_irq; -static atomic_t idle_wakeup_counter; + static unsigned int control_cpu; /* The cpu assigned to collect stat and update * control parameters. default to BSP but BSP * can be offlined. @@ -285,9 +284,6 @@ static unsigned int get_compensation(int ratio) cal_data[ratio + 1].steady_comp) / 3; } - /* REVISIT: simple penalty of double idle injection */ - if (reduce_irq) - comp = ratio; /* do not exceed limit */ if (comp + ratio >= MAX_TARGET_RATIO) comp = MAX_TARGET_RATIO - ratio - 1; @@ -301,13 +297,9 @@ static void adjust_compensation(int target_ratio, unsigned int win) struct powerclamp_calibration_data *d = &cal_data[target_ratio]; /* - * adjust compensations if confidence level has not been reached or - * there are too many wakeups during the last idle injection period, we - * cannot trust the data for compensation. + * adjust compensations if confidence level has not been reached. */ - if (d->confidence >= CONFIDENCE_OK || - atomic_read(&idle_wakeup_counter) > - win * num_online_cpus()) + if (d->confidence >= CONFIDENCE_OK) return; delta = set_target_ratio - current_ratio; @@ -347,14 +339,7 @@ static bool powerclamp_adjust_controls(unsigned int target_ratio, tsc_last = tsc_now; adjust_compensation(target_ratio, win); - /* - * too many external interrupts, set flag such - * that we can take measure later. - */ - reduce_irq = atomic_read(&idle_wakeup_counter) >= - 2 * win * num_online_cpus(); - atomic_set(&idle_wakeup_counter, 0); /* if we are above target+guard, skip */ return set_target_ratio + guard <= current_ratio; } @@ -532,8 +517,10 @@ static int start_power_clamp(void) /* prefer BSP */ control_cpu = 0; - if (!cpu_online(control_cpu)) - control_cpu = smp_processor_id(); + if (!cpu_online(control_cpu)) { + control_cpu = get_cpu(); + put_cpu(); + } clamping = true; schedule_delayed_work(&poll_pkg_cstate_work, 0); diff --git a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c index 5d0b3ffc6f46c65943373743e89567e2dcf7fb1f..22c9bcb899c37b21dde6ee48cb19b3cb93554fcb 100644 --- a/drivers/thermal/k3_bandgap.c +++ b/drivers/thermal/k3_bandgap.c @@ -139,9 +139,9 @@ static int k3_bgp_read_temp(struct k3_thermal_data *devdata, return 0; } -static int k3_thermal_get_temp(void *devdata, int *temp) +static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct k3_thermal_data *data = devdata; + struct k3_thermal_data *data = tz->devdata; int ret = 0; ret = k3_bgp_read_temp(data, temp); @@ -151,7 +151,7 @@ static int k3_thermal_get_temp(void *devdata, int *temp) return ret; } -static const struct thermal_zone_of_device_ops k3_of_thermal_ops = { +static const struct thermal_zone_device_ops k3_of_thermal_ops = { .get_temp = k3_thermal_get_temp, }; @@ -213,9 +213,9 @@ static int k3_bandgap_probe(struct platform_device *pdev) writel(val, data[id].bgp->base + data[id].ctrl_offset); data[id].tzd = - devm_thermal_zone_of_sensor_register(dev, id, - &data[id], - &k3_of_thermal_ops); + devm_thermal_of_zone_register(dev, id, + &data[id], + &k3_of_thermal_ops); if (IS_ERR(data[id].tzd)) { dev_err(dev, "thermal zone device is NULL\n"); ret = PTR_ERR(data[id].tzd); diff --git a/drivers/thermal/k3_j72xx_bandgap.c b/drivers/thermal/k3_j72xx_bandgap.c index 115a44eb4fbfbff274fb66af5dcf3c196e709340..16b6bcf1bf4fa171b9e7a0aa413a943f0cef4394 100644 --- a/drivers/thermal/k3_j72xx_bandgap.c +++ b/drivers/thermal/k3_j72xx_bandgap.c @@ -247,9 +247,9 @@ static inline int k3_bgp_read_temp(struct k3_thermal_data *devdata, } /* Get temperature callback function for thermal zone */ -static int k3_thermal_get_temp(void *devdata, int *temp) +static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct k3_thermal_data *data = devdata; + struct k3_thermal_data *data = tz->devdata; int ret = 0; ret = k3_bgp_read_temp(data, temp); @@ -259,7 +259,7 @@ static int k3_thermal_get_temp(void *devdata, int *temp) return ret; } -static const struct thermal_zone_of_device_ops k3_of_thermal_ops = { +static const struct thermal_zone_device_ops k3_of_thermal_ops = { .get_temp = k3_thermal_get_temp, }; @@ -474,10 +474,8 @@ static int k3_j72xx_bandgap_probe(struct platform_device *pdev) writel(val, data[id].bgp->cfg2_base + data[id].ctrl_offset); bgp->ts_data[id] = &data[id]; - ti_thermal = - devm_thermal_zone_of_sensor_register(bgp->dev, id, - &data[id], - &k3_of_thermal_ops); + ti_thermal = devm_thermal_of_zone_register(bgp->dev, id, &data[id], + &k3_of_thermal_ops); if (IS_ERR(ti_thermal)) { dev_err(bgp->dev, "thermal zone device is NULL\n"); ret = PTR_ERR(ti_thermal); diff --git a/drivers/thermal/max77620_thermal.c b/drivers/thermal/max77620_thermal.c index 82d06c7411eb327871e404402f49315140903658..6451a55eb5823abdb4f0b986ef1d77f39ec2e9bc 100644 --- a/drivers/thermal/max77620_thermal.c +++ b/drivers/thermal/max77620_thermal.c @@ -44,9 +44,9 @@ struct max77620_therm_info { * Return 0 on success otherwise error number to show reason of failure. */ -static int max77620_thermal_read_temp(void *data, int *temp) +static int max77620_thermal_read_temp(struct thermal_zone_device *tz, int *temp) { - struct max77620_therm_info *mtherm = data; + struct max77620_therm_info *mtherm = tz->devdata; unsigned int val; int ret; @@ -66,7 +66,7 @@ static int max77620_thermal_read_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops max77620_thermal_ops = { +static const struct thermal_zone_device_ops max77620_thermal_ops = { .get_temp = max77620_thermal_read_temp, }; @@ -114,7 +114,7 @@ static int max77620_thermal_probe(struct platform_device *pdev) */ device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); - mtherm->tz_device = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, + mtherm->tz_device = devm_thermal_of_zone_register(&pdev->dev, 0, mtherm, &max77620_thermal_ops); if (IS_ERR(mtherm->tz_device)) { ret = PTR_ERR(mtherm->tz_device); diff --git a/drivers/thermal/mtk_thermal.c b/drivers/thermal/mtk_thermal.c index ede94eaddddae085440e5dcbaf51a317c1c1ee19..8440692e3890d279538ca4a0c89d3ca578dc1d51 100644 --- a/drivers/thermal/mtk_thermal.c +++ b/drivers/thermal/mtk_thermal.c @@ -679,9 +679,9 @@ static int mtk_thermal_bank_temperature(struct mtk_thermal_bank *bank) return max; } -static int mtk_read_temp(void *data, int *temperature) +static int mtk_read_temp(struct thermal_zone_device *tz, int *temperature) { - struct mtk_thermal *mt = data; + struct mtk_thermal *mt = tz->devdata; int i; int tempmax = INT_MIN; @@ -700,7 +700,7 @@ static int mtk_read_temp(void *data, int *temperature) return 0; } -static const struct thermal_zone_of_device_ops mtk_thermal_ops = { +static const struct thermal_zone_device_ops mtk_thermal_ops = { .get_temp = mtk_read_temp, }; @@ -1082,8 +1082,8 @@ static int mtk_thermal_probe(struct platform_device *pdev) platform_set_drvdata(pdev, mt); - tzdev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, mt, - &mtk_thermal_ops); + tzdev = devm_thermal_of_zone_register(&pdev->dev, 0, mt, + &mtk_thermal_ops); if (IS_ERR(tzdev)) { ret = PTR_ERR(tzdev); goto err_disable_clk_peri_therm; diff --git a/drivers/thermal/qcom/Kconfig b/drivers/thermal/qcom/Kconfig index bfd889422dd328abf0eb0ed1738185f478bc68c7..2c7f3f9a26ebbba41b89c8aa5d310048e8d6c792 100644 --- a/drivers/thermal/qcom/Kconfig +++ b/drivers/thermal/qcom/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config QCOM_TSENS tristate "Qualcomm TSENS Temperature Alarm" - depends on QCOM_QFPROM + depends on NVMEM_QCOM_QFPROM depends on ARCH_QCOM || COMPILE_TEST help This enables the thermal sysfs driver for the TSENS device. It shows diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c index 073943cbcc2bea777b5e7665fc91c984d40968f0..1b2c43eab27d12b00235e27c09b0f2745ff62107 100644 --- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c +++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c @@ -357,9 +357,9 @@ static irqreturn_t adc_tm5_gen2_isr(int irq, void *data) return IRQ_HANDLED; } -static int adc_tm5_get_temp(void *data, int *temp) +static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp) { - struct adc_tm5_channel *channel = data; + struct adc_tm5_channel *channel = tz->devdata; int ret; if (!channel || !channel->iio) @@ -639,9 +639,9 @@ config_fail: return ret; } -static int adc_tm5_set_trips(void *data, int low, int high) +static int adc_tm5_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct adc_tm5_channel *channel = data; + struct adc_tm5_channel *channel = tz->devdata; struct adc_tm5_chip *chip; int ret; @@ -660,7 +660,7 @@ static int adc_tm5_set_trips(void *data, int low, int high) return ret; } -static struct thermal_zone_of_device_ops adc_tm5_thermal_ops = { +static const struct thermal_zone_device_ops adc_tm5_thermal_ops = { .get_temp = adc_tm5_get_temp, .set_trips = adc_tm5_set_trips, }; @@ -672,11 +672,10 @@ static int adc_tm5_register_tzd(struct adc_tm5_chip *adc_tm) for (i = 0; i < adc_tm->nchannels; i++) { adc_tm->channels[i].chip = adc_tm; - - tzd = devm_thermal_zone_of_sensor_register(adc_tm->dev, - adc_tm->channels[i].channel, - &adc_tm->channels[i], - &adc_tm5_thermal_ops); + tzd = devm_thermal_of_zone_register(adc_tm->dev, + adc_tm->channels[i].channel, + &adc_tm->channels[i], + &adc_tm5_thermal_ops); if (IS_ERR(tzd)) { if (PTR_ERR(tzd) == -ENODEV) { dev_warn(adc_tm->dev, "thermal sensor on channel %d is not used\n", @@ -830,7 +829,8 @@ static int adc_tm5_get_dt_channel_data(struct adc_tm5_chip *adc_tm, } channel->adc_channel = args.args[0]; - channel->iio = devm_of_iio_channel_get_by_name(adc_tm->dev, node, NULL); + channel->iio = devm_fwnode_iio_channel_get_by_name(adc_tm->dev, + of_fwnode_handle(node), NULL); if (IS_ERR(channel->iio)) { ret = PTR_ERR(channel->iio); if (ret != -EPROBE_DEFER) @@ -1026,10 +1026,8 @@ static int adc_tm5_probe(struct platform_device *pdev) adc_tm->base = reg; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(dev, "get_irq failed: %d\n", irq); + if (irq < 0) return irq; - } ret = adc_tm5_get_dt_data(adc_tm, node); if (ret) { diff --git a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c index 770f82cc9bca8130cd93af772cb2a84186649d0c..be785ab37e53d5f877212980b166605e7afba62e 100644 --- a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c +++ b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c @@ -186,9 +186,9 @@ static int qpnp_tm_update_temp_no_adc(struct qpnp_tm_chip *chip) return 0; } -static int qpnp_tm_get_temp(void *data, int *temp) +static int qpnp_tm_get_temp(struct thermal_zone_device *tz, int *temp) { - struct qpnp_tm_chip *chip = data; + struct qpnp_tm_chip *chip = tz->devdata; int ret, mili_celsius; if (!temp) @@ -263,9 +263,9 @@ skip: return qpnp_tm_write(chip, QPNP_TM_REG_SHUTDOWN_CTRL1, reg); } -static int qpnp_tm_set_trip_temp(void *data, int trip, int temp) +static int qpnp_tm_set_trip_temp(struct thermal_zone_device *tz, int trip, int temp) { - struct qpnp_tm_chip *chip = data; + struct qpnp_tm_chip *chip = tz->devdata; const struct thermal_trip *trip_points; int ret; @@ -283,7 +283,7 @@ static int qpnp_tm_set_trip_temp(void *data, int trip, int temp) return ret; } -static const struct thermal_zone_of_device_ops qpnp_tm_sensor_ops = { +static const struct thermal_zone_device_ops qpnp_tm_sensor_ops = { .get_temp = qpnp_tm_get_temp, .set_trip_temp = qpnp_tm_set_trip_temp, }; @@ -446,7 +446,7 @@ static int qpnp_tm_probe(struct platform_device *pdev) * read the trip points. get_temp() returns the default temperature * before the hardware initialization is completed. */ - chip->tz_dev = devm_thermal_zone_of_sensor_register( + chip->tz_dev = devm_thermal_of_zone_register( &pdev->dev, 0, chip, &qpnp_tm_sensor_ops); if (IS_ERR(chip->tz_dev)) { dev_err(&pdev->dev, "failed to register sensor\n"); diff --git a/drivers/thermal/qcom/tsens-v0_1.c b/drivers/thermal/qcom/tsens-v0_1.c index f136cb3502384d70ab06ecfe04166dc458276237..327f37202c69fff1b853dda82d396c730107529a 100644 --- a/drivers/thermal/qcom/tsens-v0_1.c +++ b/drivers/thermal/qcom/tsens-v0_1.c @@ -604,7 +604,7 @@ static const struct tsens_ops ops_8939 = { struct tsens_plat_data data_8939 = { .num_sensors = 10, .ops = &ops_8939, - .hw_ids = (unsigned int []){ 0, 1, 2, 4, 5, 6, 7, 8, 9, 10 }, + .hw_ids = (unsigned int []){ 0, 1, 2, 3, 5, 6, 7, 8, 9, 10 }, .feat = &tsens_v0_1_feat, .fields = tsens_v0_1_regfields, diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index e49f58e8351375c0c55f9d7733e9706d601af1f7..b1b10005fb286e81833f0d52df9e58d7289c3b7d 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -532,9 +532,9 @@ static irqreturn_t tsens_irq_thread(int irq, void *data) return IRQ_HANDLED; } -static int tsens_set_trips(void *_sensor, int low, int high) +static int tsens_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct tsens_sensor *s = _sensor; + struct tsens_sensor *s = tz->devdata; struct tsens_priv *priv = s->priv; struct device *dev = priv->dev; struct tsens_irq_data d; @@ -925,9 +925,9 @@ err_put_device: return ret; } -static int tsens_get_temp(void *data, int *temp) +static int tsens_get_temp(struct thermal_zone_device *tz, int *temp) { - struct tsens_sensor *s = data; + struct tsens_sensor *s = tz->devdata; struct tsens_priv *priv = s->priv; return priv->ops->get_temp(s, temp); @@ -991,7 +991,7 @@ static const struct of_device_id tsens_table[] = { }; MODULE_DEVICE_TABLE(of, tsens_table); -static const struct thermal_zone_of_device_ops tsens_of_ops = { +static const struct thermal_zone_device_ops tsens_of_ops = { .get_temp = tsens_get_temp, .set_trips = tsens_set_trips, }; @@ -1044,9 +1044,9 @@ static int tsens_register(struct tsens_priv *priv) for (i = 0; i < priv->num_sensors; i++) { priv->sensor[i].priv = priv; - tzd = devm_thermal_zone_of_sensor_register(priv->dev, priv->sensor[i].hw_id, - &priv->sensor[i], - &tsens_of_ops); + tzd = devm_thermal_of_zone_register(priv->dev, priv->sensor[i].hw_id, + &priv->sensor[i], + &tsens_of_ops); if (IS_ERR(tzd)) continue; priv->sensor[i].tzd = tzd; diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 73049f9bea252dd0cce3ef2a00be6edf3cd4552d..d111e218f362ebbfd85a6d191ea15f82a891643e 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -82,9 +82,9 @@ static struct qoriq_tmu_data *qoriq_sensor_to_data(struct qoriq_sensor *s) return container_of(s, struct qoriq_tmu_data, sensor[s->id]); } -static int tmu_get_temp(void *p, int *temp) +static int tmu_get_temp(struct thermal_zone_device *tz, int *temp) { - struct qoriq_sensor *qsensor = p; + struct qoriq_sensor *qsensor = tz->devdata; struct qoriq_tmu_data *qdata = qoriq_sensor_to_data(qsensor); u32 val; /* @@ -122,7 +122,7 @@ static int tmu_get_temp(void *p, int *temp) return 0; } -static const struct thermal_zone_of_device_ops tmu_tz_ops = { +static const struct thermal_zone_device_ops tmu_tz_ops = { .get_temp = tmu_get_temp, }; @@ -146,9 +146,9 @@ static int qoriq_tmu_register_tmu_zone(struct device *dev, sensor->id = id; - tzd = devm_thermal_zone_of_sensor_register(dev, id, - sensor, - &tmu_tz_ops); + tzd = devm_thermal_of_zone_register(dev, id, + sensor, + &tmu_tz_ops); ret = PTR_ERR_OR_ZERO(tzd); if (ret) { if (ret == -ENODEV) diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c index cda7c52f2319b9abc71fb42ed29a317ed83b1a6c..4c1c6f89aa2fb9740443034e691e182bcdc477df 100644 --- a/drivers/thermal/rcar_gen3_thermal.c +++ b/drivers/thermal/rcar_gen3_thermal.c @@ -164,9 +164,9 @@ static int rcar_gen3_thermal_round(int temp) return result * RCAR3_THERMAL_GRAN; } -static int rcar_gen3_thermal_get_temp(void *devdata, int *temp) +static int rcar_gen3_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct rcar_gen3_thermal_tsc *tsc = devdata; + struct rcar_gen3_thermal_tsc *tsc = tz->devdata; int mcelsius, val; int reg; @@ -203,9 +203,9 @@ static int rcar_gen3_thermal_mcelsius_to_temp(struct rcar_gen3_thermal_tsc *tsc, return INT_FIXPT(val); } -static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high) +static int rcar_gen3_thermal_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct rcar_gen3_thermal_tsc *tsc = devdata; + struct rcar_gen3_thermal_tsc *tsc = tz->devdata; u32 irqmsk = 0; if (low != -INT_MAX) { @@ -225,7 +225,7 @@ static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high) return 0; } -static struct thermal_zone_of_device_ops rcar_gen3_tz_of_ops = { +static struct thermal_zone_device_ops rcar_gen3_tz_of_ops = { .get_temp = rcar_gen3_thermal_get_temp, .set_trips = rcar_gen3_thermal_set_trips, }; @@ -508,8 +508,8 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev) for (i = 0; i < priv->num_tscs; i++) { struct rcar_gen3_thermal_tsc *tsc = priv->tscs[i]; - zone = devm_thermal_zone_of_sensor_register(dev, i, tsc, - &rcar_gen3_tz_of_ops); + zone = devm_thermal_of_zone_register(dev, i, tsc, + &rcar_gen3_tz_of_ops); if (IS_ERR(zone)) { dev_err(dev, "Sensor %u: Can't register thermal zone\n", i); ret = PTR_ERR(zone); @@ -560,7 +560,7 @@ static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev) priv->thermal_init(tsc); if (zone->ops->set_trips) - rcar_gen3_thermal_set_trips(tsc, zone->prev_low_trip, + rcar_gen3_thermal_set_trips(zone, zone->prev_low_trip, zone->prev_high_trip); } diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index 1d729ed4d68567971964b5f11160dc735d8a6a81..61c2b8855cb8dc2bee0eede62bdd109c32eef0e8 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -271,13 +271,6 @@ static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv, return 0; } -static int rcar_thermal_of_get_temp(void *data, int *temp) -{ - struct rcar_thermal_priv *priv = data; - - return rcar_thermal_get_current_temp(priv, temp); -} - static int rcar_thermal_get_temp(struct thermal_zone_device *zone, int *temp) { struct rcar_thermal_priv *priv = rcar_zone_to_priv(zone); @@ -323,8 +316,8 @@ static int rcar_thermal_get_trip_temp(struct thermal_zone_device *zone, return 0; } -static const struct thermal_zone_of_device_ops rcar_thermal_zone_of_ops = { - .get_temp = rcar_thermal_of_get_temp, +static const struct thermal_zone_device_ops rcar_thermal_zone_of_ops = { + .get_temp = rcar_thermal_get_temp, }; static struct thermal_zone_device_ops rcar_thermal_zone_ops = { @@ -534,7 +527,7 @@ static int rcar_thermal_probe(struct platform_device *pdev) goto error_unregister; if (chip->use_of_thermal) { - priv->zone = devm_thermal_zone_of_sensor_register( + priv->zone = devm_thermal_of_zone_register( dev, i, priv, &rcar_thermal_zone_of_ops); } else { diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c index dc3a9c276a09bcbca86e18dc9ef530ec31d8df10..819e059cde710f4e9467f200e20a2083fbabfd56 100644 --- a/drivers/thermal/rockchip_thermal.c +++ b/drivers/thermal/rockchip_thermal.c @@ -1211,9 +1211,9 @@ static irqreturn_t rockchip_thermal_alarm_irq_thread(int irq, void *dev) return IRQ_HANDLED; } -static int rockchip_thermal_set_trips(void *_sensor, int low, int high) +static int rockchip_thermal_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct rockchip_thermal_sensor *sensor = _sensor; + struct rockchip_thermal_sensor *sensor = tz->devdata; struct rockchip_thermal_data *thermal = sensor->thermal; const struct rockchip_tsadc_chip *tsadc = thermal->chip; @@ -1224,9 +1224,9 @@ static int rockchip_thermal_set_trips(void *_sensor, int low, int high) sensor->id, thermal->regs, high); } -static int rockchip_thermal_get_temp(void *_sensor, int *out_temp) +static int rockchip_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp) { - struct rockchip_thermal_sensor *sensor = _sensor; + struct rockchip_thermal_sensor *sensor = tz->devdata; struct rockchip_thermal_data *thermal = sensor->thermal; const struct rockchip_tsadc_chip *tsadc = sensor->thermal->chip; int retval; @@ -1239,7 +1239,7 @@ static int rockchip_thermal_get_temp(void *_sensor, int *out_temp) return retval; } -static const struct thermal_zone_of_device_ops rockchip_of_thermal_ops = { +static const struct thermal_zone_device_ops rockchip_of_thermal_ops = { .get_temp = rockchip_thermal_get_temp, .set_trips = rockchip_thermal_set_trips, }; @@ -1326,8 +1326,8 @@ rockchip_thermal_register_sensor(struct platform_device *pdev, sensor->thermal = thermal; sensor->id = id; - sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, id, - sensor, &rockchip_of_thermal_ops); + sensor->tzd = devm_thermal_of_zone_register(&pdev->dev, id, sensor, + &rockchip_of_thermal_ops); if (IS_ERR(sensor->tzd)) { error = PTR_ERR(sensor->tzd); dev_err(&pdev->dev, "failed to register sensor %d: %d\n", diff --git a/drivers/thermal/rzg2l_thermal.c b/drivers/thermal/rzg2l_thermal.c index 51ae80eda6af4631bc3810fc779568cff8883870..2e0649f3850607574ddf74a0891fe9c34a246958 100644 --- a/drivers/thermal/rzg2l_thermal.c +++ b/drivers/thermal/rzg2l_thermal.c @@ -73,9 +73,9 @@ static inline void rzg2l_thermal_write(struct rzg2l_thermal_priv *priv, u32 reg, iowrite32(data, priv->base + reg); } -static int rzg2l_thermal_get_temp(void *devdata, int *temp) +static int rzg2l_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct rzg2l_thermal_priv *priv = devdata; + struct rzg2l_thermal_priv *priv = tz->devdata; u32 result = 0, dsensor, ts_code_ave; int val, i; @@ -114,7 +114,7 @@ static int rzg2l_thermal_get_temp(void *devdata, int *temp) return 0; } -static const struct thermal_zone_of_device_ops rzg2l_tz_of_ops = { +static const struct thermal_zone_device_ops rzg2l_tz_of_ops = { .get_temp = rzg2l_thermal_get_temp, }; @@ -207,8 +207,8 @@ static int rzg2l_thermal_probe(struct platform_device *pdev) goto err; } - zone = devm_thermal_zone_of_sensor_register(dev, 0, priv, - &rzg2l_tz_of_ops); + zone = devm_thermal_of_zone_register(dev, 0, priv, + &rzg2l_tz_of_ops); if (IS_ERR(zone)) { dev_err(dev, "Can't register thermal zone"); ret = PTR_ERR(zone); diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c index f4ab4c5b4b626cfff4f9591a84b9ad14403375a7..51874d0a284cbab1c51f7a8a66128bf55d3e2f8a 100644 --- a/drivers/thermal/samsung/exynos_tmu.c +++ b/drivers/thermal/samsung/exynos_tmu.c @@ -650,9 +650,9 @@ static void exynos7_tmu_control(struct platform_device *pdev, bool on) writel(con, data->base + EXYNOS_TMU_REG_CONTROL); } -static int exynos_get_temp(void *p, int *temp) +static int exynos_get_temp(struct thermal_zone_device *tz, int *temp) { - struct exynos_tmu_data *data = p; + struct exynos_tmu_data *data = tz->devdata; int value, ret = 0; if (!data || !data->tmu_read) @@ -728,9 +728,9 @@ static void exynos4412_tmu_set_emulation(struct exynos_tmu_data *data, writel(val, data->base + emul_con); } -static int exynos_tmu_set_emulation(void *drv_data, int temp) +static int exynos_tmu_set_emulation(struct thermal_zone_device *tz, int temp) { - struct exynos_tmu_data *data = drv_data; + struct exynos_tmu_data *data = tz->devdata; int ret = -EINVAL; if (data->soc == SOC_ARCH_EXYNOS4210) @@ -750,7 +750,7 @@ out: } #else #define exynos4412_tmu_set_emulation NULL -static int exynos_tmu_set_emulation(void *drv_data, int temp) +static int exynos_tmu_set_emulation(struct thermal_zone_device *tz, int temp) { return -EINVAL; } #endif /* CONFIG_THERMAL_EMULATION */ @@ -997,7 +997,7 @@ static int exynos_map_dt_data(struct platform_device *pdev) return 0; } -static const struct thermal_zone_of_device_ops exynos_sensor_ops = { +static const struct thermal_zone_device_ops exynos_sensor_ops = { .get_temp = exynos_get_temp, .set_emul_temp = exynos_tmu_set_emulation, }; @@ -1091,8 +1091,8 @@ static int exynos_tmu_probe(struct platform_device *pdev) * data->tzd must be registered before calling exynos_tmu_initialize(), * requesting irq and calling exynos_tmu_control(). */ - data->tzd = thermal_zone_of_sensor_register(&pdev->dev, 0, data, - &exynos_sensor_ops); + data->tzd = devm_thermal_of_zone_register(&pdev->dev, 0, data, + &exynos_sensor_ops); if (IS_ERR(data->tzd)) { ret = PTR_ERR(data->tzd); if (ret != -EPROBE_DEFER) @@ -1104,21 +1104,19 @@ static int exynos_tmu_probe(struct platform_device *pdev) ret = exynos_tmu_initialize(pdev); if (ret) { dev_err(&pdev->dev, "Failed to initialize TMU\n"); - goto err_thermal; + goto err_sclk; } ret = devm_request_irq(&pdev->dev, data->irq, exynos_tmu_irq, IRQF_TRIGGER_RISING | IRQF_SHARED, dev_name(&pdev->dev), data); if (ret) { dev_err(&pdev->dev, "Failed to request irq: %d\n", data->irq); - goto err_thermal; + goto err_sclk; } exynos_tmu_control(pdev, true); return 0; -err_thermal: - thermal_zone_of_sensor_unregister(&pdev->dev, data->tzd); err_sclk: clk_disable_unprepare(data->sclk); err_clk: @@ -1136,9 +1134,7 @@ err_sensor: static int exynos_tmu_remove(struct platform_device *pdev) { struct exynos_tmu_data *data = platform_get_drvdata(pdev); - struct thermal_zone_device *tzd = data->tzd; - thermal_zone_of_sensor_unregister(&pdev->dev, tzd); exynos_tmu_control(pdev, false); clk_disable_unprepare(data->sclk); diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c index fff80fc180028416e864a16ee82fcfae39f8d510..ac884514f116a0f38741bd2f1c948f140a794a75 100644 --- a/drivers/thermal/sprd_thermal.c +++ b/drivers/thermal/sprd_thermal.c @@ -204,9 +204,9 @@ static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen) return clamp(val, val, (u32)(SPRD_THM_RAW_DATA_HIGH - 1)); } -static int sprd_thm_read_temp(void *devdata, int *temp) +static int sprd_thm_read_temp(struct thermal_zone_device *tz, int *temp) { - struct sprd_thermal_sensor *sen = devdata; + struct sprd_thermal_sensor *sen = tz->devdata; u32 data; data = readl(sen->data->base + SPRD_THM_TEMP(sen->id)) & @@ -217,7 +217,7 @@ static int sprd_thm_read_temp(void *devdata, int *temp) return 0; } -static const struct thermal_zone_of_device_ops sprd_thm_ops = { +static const struct thermal_zone_device_ops sprd_thm_ops = { .get_temp = sprd_thm_read_temp, }; @@ -408,10 +408,10 @@ static int sprd_thm_probe(struct platform_device *pdev) sprd_thm_sensor_init(thm, sen); - sen->tzd = devm_thermal_zone_of_sensor_register(sen->dev, - sen->id, - sen, - &sprd_thm_ops); + sen->tzd = devm_thermal_of_zone_register(sen->dev, + sen->id, + sen, + &sprd_thm_ops); if (IS_ERR(sen->tzd)) { dev_err(&pdev->dev, "register thermal zone failed %d\n", sen->id); @@ -523,8 +523,8 @@ static int sprd_thm_remove(struct platform_device *pdev) for (i = 0; i < thm->nr_sensors; i++) { sprd_thm_toggle_sensor(thm->sensor[i], false); - devm_thermal_zone_of_sensor_unregister(&pdev->dev, - thm->sensor[i]->tzd); + devm_thermal_of_zone_unregister(&pdev->dev, + thm->sensor[i]->tzd); } clk_disable_unprepare(thm->clk); diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c index 5fd3fb8912a6a796f0a5d8953b3e2def32ded5d3..78feb802a87d2b1a344d548091afd8da39e149fa 100644 --- a/drivers/thermal/st/stm_thermal.c +++ b/drivers/thermal/st/stm_thermal.c @@ -302,9 +302,9 @@ static int stm_disable_irq(struct stm_thermal_sensor *sensor) return 0; } -static int stm_thermal_set_trips(void *data, int low, int high) +static int stm_thermal_set_trips(struct thermal_zone_device *tz, int low, int high) { - struct stm_thermal_sensor *sensor = data; + struct stm_thermal_sensor *sensor = tz->devdata; u32 itr1, th; int ret; @@ -350,9 +350,9 @@ static int stm_thermal_set_trips(void *data, int low, int high) } /* Callback to get temperature from HW */ -static int stm_thermal_get_temp(void *data, int *temp) +static int stm_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct stm_thermal_sensor *sensor = data; + struct stm_thermal_sensor *sensor = tz->devdata; u32 periods; int freqM, ret; @@ -474,7 +474,7 @@ static int stm_thermal_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(stm_thermal_pm_ops, stm_thermal_suspend, stm_thermal_resume); -static const struct thermal_zone_of_device_ops stm_tz_ops = { +static const struct thermal_zone_device_ops stm_tz_ops = { .get_temp = stm_thermal_get_temp, .set_trips = stm_thermal_set_trips, }; @@ -539,9 +539,9 @@ static int stm_thermal_probe(struct platform_device *pdev) return ret; } - sensor->th_dev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, - sensor, - &stm_tz_ops); + sensor->th_dev = devm_thermal_of_zone_register(&pdev->dev, 0, + sensor, + &stm_tz_ops); if (IS_ERR(sensor->th_dev)) { dev_err(&pdev->dev, "%s: thermal zone sensor registering KO\n", @@ -572,7 +572,6 @@ static int stm_thermal_probe(struct platform_device *pdev) return 0; err_tz: - thermal_zone_of_sensor_unregister(&pdev->dev, sensor->th_dev); return ret; } @@ -582,7 +581,6 @@ static int stm_thermal_remove(struct platform_device *pdev) stm_thermal_sensor_off(sensor); thermal_remove_hwmon_sysfs(sensor->th_dev); - thermal_zone_of_sensor_unregister(&pdev->dev, sensor->th_dev); return 0; } diff --git a/drivers/thermal/sun8i_thermal.c b/drivers/thermal/sun8i_thermal.c index 212c87e63a667c02b686b2d7b075be9069c1ab7e..e64d06d1328ce0b3e3575b867dd87842a6cb56c9 100644 --- a/drivers/thermal/sun8i_thermal.c +++ b/drivers/thermal/sun8i_thermal.c @@ -108,9 +108,9 @@ static int sun50i_h5_calc_temp(struct ths_device *tmdev, return -1590 * reg / 10 + 276000; } -static int sun8i_ths_get_temp(void *data, int *temp) +static int sun8i_ths_get_temp(struct thermal_zone_device *tz, int *temp) { - struct tsensor *s = data; + struct tsensor *s = tz->devdata; struct ths_device *tmdev = s->tmdev; int val = 0; @@ -135,7 +135,7 @@ static int sun8i_ths_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops ths_ops = { +static const struct thermal_zone_device_ops ths_ops = { .get_temp = sun8i_ths_get_temp, }; @@ -468,10 +468,10 @@ static int sun8i_ths_register(struct ths_device *tmdev) tmdev->sensor[i].tmdev = tmdev; tmdev->sensor[i].id = i; tmdev->sensor[i].tzd = - devm_thermal_zone_of_sensor_register(tmdev->dev, - i, - &tmdev->sensor[i], - &ths_ops); + devm_thermal_of_zone_register(tmdev->dev, + i, + &tmdev->sensor[i], + &ths_ops); if (IS_ERR(tmdev->sensor[i].tzd)) return PTR_ERR(tmdev->sensor[i].tzd); diff --git a/drivers/thermal/tegra/soctherm.c b/drivers/thermal/tegra/soctherm.c index 825eab5266196761f491deba692c6e84d47aa6de..1efe470f31e9a301b4c51d2194781b4cb3019793 100644 --- a/drivers/thermal/tegra/soctherm.c +++ b/drivers/thermal/tegra/soctherm.c @@ -421,9 +421,9 @@ static int translate_temp(u16 val) return t; } -static int tegra_thermctl_get_temp(void *data, int *out_temp) +static int tegra_thermctl_get_temp(struct thermal_zone_device *tz, int *out_temp) { - struct tegra_thermctl_zone *zone = data; + struct tegra_thermctl_zone *zone = tz->devdata; u32 val; val = readl(zone->reg); @@ -582,10 +582,9 @@ static int tsensor_group_thermtrip_get(struct tegra_soctherm *ts, int id) return temp; } -static int tegra_thermctl_set_trip_temp(void *data, int trip, int temp) +static int tegra_thermctl_set_trip_temp(struct thermal_zone_device *tz, int trip, int temp) { - struct tegra_thermctl_zone *zone = data; - struct thermal_zone_device *tz = zone->tz; + struct tegra_thermctl_zone *zone = tz->devdata; struct tegra_soctherm *ts = zone->ts; const struct tegra_tsensor_group *sg = zone->sg; struct device *dev = zone->dev; @@ -657,9 +656,9 @@ static void thermal_irq_disable(struct tegra_thermctl_zone *zn) mutex_unlock(&zn->ts->thermctl_lock); } -static int tegra_thermctl_set_trips(void *data, int lo, int hi) +static int tegra_thermctl_set_trips(struct thermal_zone_device *tz, int lo, int hi) { - struct tegra_thermctl_zone *zone = data; + struct tegra_thermctl_zone *zone = tz->devdata; u32 r; thermal_irq_disable(zone); @@ -682,7 +681,7 @@ static int tegra_thermctl_set_trips(void *data, int lo, int hi) return 0; } -static const struct thermal_zone_of_device_ops tegra_of_thermal_ops = { +static const struct thermal_zone_device_ops tegra_of_thermal_ops = { .get_temp = tegra_thermctl_get_temp, .set_trip_temp = tegra_thermctl_set_trip_temp, .set_trips = tegra_thermctl_set_trips, @@ -2194,9 +2193,9 @@ static int tegra_soctherm_probe(struct platform_device *pdev) zone->sg = soc->ttgs[i]; zone->ts = tegra; - z = devm_thermal_zone_of_sensor_register(&pdev->dev, - soc->ttgs[i]->id, zone, - &tegra_of_thermal_ops); + z = devm_thermal_of_zone_register(&pdev->dev, + soc->ttgs[i]->id, zone, + &tegra_of_thermal_ops); if (IS_ERR(z)) { err = PTR_ERR(z); dev_err(&pdev->dev, "failed to register sensor: %d\n", diff --git a/drivers/thermal/tegra/tegra-bpmp-thermal.c b/drivers/thermal/tegra/tegra-bpmp-thermal.c index 5affc3d196be828d0451af4b6f273e4527f793e2..eb84f0b9dc7c2b91a55383dda6c756db938fd14c 100644 --- a/drivers/thermal/tegra/tegra-bpmp-thermal.c +++ b/drivers/thermal/tegra/tegra-bpmp-thermal.c @@ -30,9 +30,9 @@ struct tegra_bpmp_thermal { struct tegra_bpmp_thermal_zone **zones; }; -static int tegra_bpmp_thermal_get_temp(void *data, int *out_temp) +static int __tegra_bpmp_thermal_get_temp(struct tegra_bpmp_thermal_zone *zone, + int *out_temp) { - struct tegra_bpmp_thermal_zone *zone = data; struct mrq_thermal_host_to_bpmp_request req; union mrq_thermal_bpmp_to_host_response reply; struct tegra_bpmp_message msg; @@ -60,9 +60,14 @@ static int tegra_bpmp_thermal_get_temp(void *data, int *out_temp) return 0; } -static int tegra_bpmp_thermal_set_trips(void *data, int low, int high) +static int tegra_bpmp_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp) { - struct tegra_bpmp_thermal_zone *zone = data; + return __tegra_bpmp_thermal_get_temp(tz->devdata, out_temp); +} + +static int tegra_bpmp_thermal_set_trips(struct thermal_zone_device *tz, int low, int high) +{ + struct tegra_bpmp_thermal_zone *zone = tz->devdata; struct mrq_thermal_host_to_bpmp_request req; struct tegra_bpmp_message msg; int err; @@ -157,7 +162,7 @@ static int tegra_bpmp_thermal_get_num_zones(struct tegra_bpmp *bpmp, return 0; } -static const struct thermal_zone_of_device_ops tegra_bpmp_of_thermal_ops = { +static const struct thermal_zone_device_ops tegra_bpmp_of_thermal_ops = { .get_temp = tegra_bpmp_thermal_get_temp, .set_trips = tegra_bpmp_thermal_set_trips, }; @@ -200,13 +205,13 @@ static int tegra_bpmp_thermal_probe(struct platform_device *pdev) zone->idx = i; zone->tegra = tegra; - err = tegra_bpmp_thermal_get_temp(zone, &temp); + err = __tegra_bpmp_thermal_get_temp(zone, &temp); if (err < 0) { devm_kfree(&pdev->dev, zone); continue; } - tzd = devm_thermal_zone_of_sensor_register( + tzd = devm_thermal_of_zone_register( &pdev->dev, i, zone, &tegra_bpmp_of_thermal_ops); if (IS_ERR(tzd)) { if (PTR_ERR(tzd) == -EPROBE_DEFER) diff --git a/drivers/thermal/tegra/tegra30-tsensor.c b/drivers/thermal/tegra/tegra30-tsensor.c index 05886684f4295674df0eb3665773ee5e15a2707c..c34501287e96c1b3881b16d5197904a708b93541 100644 --- a/drivers/thermal/tegra/tegra30-tsensor.c +++ b/drivers/thermal/tegra/tegra30-tsensor.c @@ -159,9 +159,9 @@ static void devm_tegra_tsensor_hw_disable(void *data) tegra_tsensor_hw_disable(ts); } -static int tegra_tsensor_get_temp(void *data, int *temp) +static int tegra_tsensor_get_temp(struct thermal_zone_device *tz, int *temp) { - const struct tegra_tsensor_channel *tsc = data; + const struct tegra_tsensor_channel *tsc = tz->devdata; const struct tegra_tsensor *ts = tsc->ts; int err, c1, c2, c3, c4, counter; u32 val; @@ -217,9 +217,9 @@ static int tegra_tsensor_temp_to_counter(const struct tegra_tsensor *ts, int tem return DIV_ROUND_CLOSEST(c2 * 1000000 - ts->calib.b, ts->calib.a); } -static int tegra_tsensor_set_trips(void *data, int low, int high) +static int tegra_tsensor_set_trips(struct thermal_zone_device *tz, int low, int high) { - const struct tegra_tsensor_channel *tsc = data; + const struct tegra_tsensor_channel *tsc = tz->devdata; const struct tegra_tsensor *ts = tsc->ts; u32 val; @@ -240,7 +240,7 @@ static int tegra_tsensor_set_trips(void *data, int low, int high) return 0; } -static const struct thermal_zone_of_device_ops ops = { +static const struct thermal_zone_device_ops ops = { .get_temp = tegra_tsensor_get_temp, .set_trips = tegra_tsensor_set_trips, }; @@ -516,7 +516,7 @@ static int tegra_tsensor_register_channel(struct tegra_tsensor *ts, tsc->id = id; tsc->regs = ts->regs + 0x40 * (hw_id + 1); - tsc->tzd = devm_thermal_zone_of_sensor_register(ts->dev, id, tsc, &ops); + tsc->tzd = devm_thermal_of_zone_register(ts->dev, id, tsc, &ops); if (IS_ERR(tsc->tzd)) { if (PTR_ERR(tsc->tzd) != -ENODEV) return dev_err_probe(ts->dev, PTR_ERR(tsc->tzd), diff --git a/drivers/thermal/thermal-generic-adc.c b/drivers/thermal/thermal-generic-adc.c index 73665c3ccfe0b6de5aa11a423067877bf823c6d6..323e273e329824bc4fa705974ae9686498917ea4 100644 --- a/drivers/thermal/thermal-generic-adc.c +++ b/drivers/thermal/thermal-generic-adc.c @@ -52,9 +52,9 @@ static int gadc_thermal_adc_to_temp(struct gadc_thermal_info *gti, int val) return temp; } -static int gadc_thermal_get_temp(void *data, int *temp) +static int gadc_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { - struct gadc_thermal_info *gti = data; + struct gadc_thermal_info *gti = tz->devdata; int val; int ret; @@ -68,7 +68,7 @@ static int gadc_thermal_get_temp(void *data, int *temp) return 0; } -static const struct thermal_zone_of_device_ops gadc_thermal_ops = { +static const struct thermal_zone_device_ops gadc_thermal_ops = { .get_temp = gadc_thermal_get_temp, }; @@ -143,8 +143,8 @@ static int gadc_thermal_probe(struct platform_device *pdev) gti->dev = &pdev->dev; platform_set_drvdata(pdev, gti); - gti->tz_dev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, gti, - &gadc_thermal_ops); + gti->tz_dev = devm_thermal_of_zone_register(&pdev->dev, 0, gti, + &gadc_thermal_ops); if (IS_ERR(gti->tz_dev)) { ret = PTR_ERR(gti->tz_dev); if (ret != -EPROBE_DEFER) diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 6a5d0ae5d7a4c0c12ed980dd434e676878284025..117eeaf7dd241e82ca1401ae39501a4c46896bb5 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -295,27 +295,14 @@ static void thermal_zone_device_set_polling(struct thermal_zone_device *tz, cancel_delayed_work(&tz->poll_queue); } -static inline bool should_stop_polling(struct thermal_zone_device *tz) -{ - return !thermal_zone_device_is_enabled(tz); -} - static void monitor_thermal_zone(struct thermal_zone_device *tz) { - bool stop; - - stop = should_stop_polling(tz); - - mutex_lock(&tz->lock); - - if (!stop && tz->passive) + if (tz->mode != THERMAL_DEVICE_ENABLED) + thermal_zone_device_set_polling(tz, 0); + else if (tz->passive) thermal_zone_device_set_polling(tz, tz->passive_delay_jiffies); - else if (!stop && tz->polling_delay_jiffies) + else if (tz->polling_delay_jiffies) thermal_zone_device_set_polling(tz, tz->polling_delay_jiffies); - else - thermal_zone_device_set_polling(tz, 0); - - mutex_unlock(&tz->lock); } static void handle_non_critical_trips(struct thermal_zone_device *tz, int trip) @@ -383,18 +370,13 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, int trip) handle_critical_trips(tz, trip, trip_temp, type); else handle_non_critical_trips(tz, trip); - /* - * Alright, we handled this trip successfully. - * So, start monitoring again. - */ - monitor_thermal_zone(tz); } static void update_temperature(struct thermal_zone_device *tz) { int temp, ret; - ret = thermal_zone_get_temp(tz, &temp); + ret = __thermal_zone_get_temp(tz, &temp); if (ret) { if (ret != -EAGAIN) dev_warn(&tz->device, @@ -403,10 +385,8 @@ static void update_temperature(struct thermal_zone_device *tz) return; } - mutex_lock(&tz->lock); tz->last_temperature = tz->temperature; tz->temperature = temp; - mutex_unlock(&tz->lock); trace_thermal_temperature(tz); @@ -469,15 +449,9 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_disable); int thermal_zone_device_is_enabled(struct thermal_zone_device *tz) { - enum thermal_device_mode mode; - - mutex_lock(&tz->lock); - - mode = tz->mode; + lockdep_assert_held(&tz->lock); - mutex_unlock(&tz->lock); - - return mode == THERMAL_DEVICE_ENABLED; + return tz->mode == THERMAL_DEVICE_ENABLED; } void thermal_zone_device_update(struct thermal_zone_device *tz, @@ -485,9 +459,6 @@ void thermal_zone_device_update(struct thermal_zone_device *tz, { int count; - if (should_stop_polling(tz)) - return; - if (atomic_read(&in_suspend)) return; @@ -495,14 +466,23 @@ void thermal_zone_device_update(struct thermal_zone_device *tz, "'get_temp' ops set\n", __func__)) return; + mutex_lock(&tz->lock); + + if (!thermal_zone_device_is_enabled(tz)) + goto out; + update_temperature(tz); - thermal_zone_set_trips(tz); + __thermal_zone_set_trips(tz); tz->notify_event = event; for (count = 0; count < tz->num_trips; count++) handle_thermal_trip(tz, count); + + monitor_thermal_zone(tz); +out: + mutex_unlock(&tz->lock); } EXPORT_SYMBOL_GPL(thermal_zone_device_update); @@ -1206,13 +1186,26 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t return ERR_PTR(-EINVAL); } - if (type && strlen(type) >= THERMAL_NAME_LENGTH) { + if (strlen(type) >= THERMAL_NAME_LENGTH) { pr_err("Thermal zone name (%s) too long, should be under %d chars\n", type, THERMAL_NAME_LENGTH); return ERR_PTR(-EINVAL); } - if (num_trips > THERMAL_MAX_TRIPS || num_trips < 0 || mask >> num_trips) { + /* + * Max trip count can't exceed 31 as the "mask >> num_trips" condition. + * For example, shifting by 32 will result in compiler warning: + * warning: right shift count >= width of type [-Wshift-count- overflow] + * + * Also "mask >> num_trips" will always be true with 32 bit shift. + * E.g. mask = 0x80000000 for trip id 31 to be RW. Then + * mask >> 32 = 0x80000000 + * This will result in failure for the below condition. + * + * Check will be true when the bit 31 of the mask is set. + * 32 bit shift will cause overflow of 4 byte integer. + */ + if (num_trips > (BITS_PER_TYPE(int) - 1) || num_trips < 0 || mask >> num_trips) { pr_err("Incorrect number of thermal trips\n"); return ERR_PTR(-EINVAL); } @@ -1239,7 +1232,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t } tz->id = id; - strlcpy(tz->type, type, sizeof(tz->type)); + strscpy(tz->type, type, sizeof(tz->type)); result = dev_set_name(&tz->device, "thermal_zone%d", tz->id); if (result) @@ -1329,6 +1322,7 @@ free_tz: kfree(tz); return ERR_PTR(result); } +EXPORT_SYMBOL_GPL(thermal_zone_device_register_with_trips); struct thermal_zone_device *thermal_zone_device_register(const char *type, int ntrips, int mask, void *devdata, struct thermal_zone_device_ops *ops, @@ -1457,9 +1451,6 @@ static int thermal_pm_notify(struct notifier_block *nb, case PM_POST_SUSPEND: atomic_set(&in_suspend, 0); list_for_each_entry(tz, &thermal_tz_list, node) { - if (!thermal_zone_device_is_enabled(tz)) - continue; - thermal_zone_device_init(tz); thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED); @@ -1491,10 +1482,6 @@ static int __init thermal_init(void) if (result) goto unregister_governors; - result = of_parse_thermal_zones(); - if (result) - goto unregister_class; - result = register_pm_notifier(&thermal_pm_nb); if (result) pr_warn("Thermal: Can not register suspend notifier, return %d\n", @@ -1502,8 +1489,6 @@ static int __init thermal_init(void) return 0; -unregister_class: - class_unregister(&thermal_class); unregister_governors: thermal_unregister_governors(); error: diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index c991bb290512ca38a7ad19b31a70ddeb9bb28683..1571917bd3c8917a4d1115bee42e7a3230babade 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -112,6 +112,8 @@ int thermal_build_list_of_policies(char *buf); /* Helpers */ void thermal_zone_set_trips(struct thermal_zone_device *tz); +void __thermal_zone_set_trips(struct thermal_zone_device *tz); +int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); /* sysfs I/F */ int thermal_zone_create_device_groups(struct thermal_zone_device *, int); @@ -135,13 +137,11 @@ thermal_cooling_device_stats_update(struct thermal_cooling_device *cdev, /* device tree support */ #ifdef CONFIG_THERMAL_OF -int of_parse_thermal_zones(void); int of_thermal_get_ntrips(struct thermal_zone_device *); bool of_thermal_is_trip_valid(struct thermal_zone_device *, int); const struct thermal_trip * of_thermal_get_trip_points(struct thermal_zone_device *); #else -static inline int of_parse_thermal_zones(void) { return 0; } static inline int of_thermal_get_ntrips(struct thermal_zone_device *tz) { return 0; diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 690890f054a39743758bb68cf628b0b682429065..c65cdce8f856e5112033317af2321e1ac3313d4c 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -64,27 +64,17 @@ get_thermal_instance(struct thermal_zone_device *tz, } EXPORT_SYMBOL(get_thermal_instance); -/** - * thermal_zone_get_temp() - returns the temperature of a thermal zone - * @tz: a valid pointer to a struct thermal_zone_device - * @temp: a valid pointer to where to store the resulting temperature. - * - * When a valid thermal zone reference is passed, it will fetch its - * temperature and fill @temp. - * - * Return: On success returns 0, an error code otherwise - */ -int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) +int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) { int ret = -EINVAL; int count; int crit_temp = INT_MAX; enum thermal_trip_type type; - if (!tz || IS_ERR(tz) || !tz->ops->get_temp) - goto exit; + lockdep_assert_held(&tz->lock); - mutex_lock(&tz->lock); + if (!tz || IS_ERR(tz) || !tz->ops->get_temp) + return -EINVAL; ret = tz->ops->get_temp(tz, temp); @@ -107,35 +97,42 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) *temp = tz->emul_temperature; } - mutex_unlock(&tz->lock); -exit: return ret; } -EXPORT_SYMBOL_GPL(thermal_zone_get_temp); /** - * thermal_zone_set_trips - Computes the next trip points for the driver - * @tz: a pointer to a thermal zone device structure + * thermal_zone_get_temp() - returns the temperature of a thermal zone + * @tz: a valid pointer to a struct thermal_zone_device + * @temp: a valid pointer to where to store the resulting temperature. * - * The function computes the next temperature boundaries by browsing - * the trip points. The result is the closer low and high trip points - * to the current temperature. These values are passed to the backend - * driver to let it set its own notification mechanism (usually an - * interrupt). + * When a valid thermal zone reference is passed, it will fetch its + * temperature and fill @temp. * - * It does not return a value + * Return: On success returns 0, an error code otherwise */ -void thermal_zone_set_trips(struct thermal_zone_device *tz) +int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp) +{ + int ret; + + mutex_lock(&tz->lock); + ret = __thermal_zone_get_temp(tz, temp); + mutex_unlock(&tz->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(thermal_zone_get_temp); + +void __thermal_zone_set_trips(struct thermal_zone_device *tz) { int low = -INT_MAX; int high = INT_MAX; int trip_temp, hysteresis; int i, ret; - mutex_lock(&tz->lock); + lockdep_assert_held(&tz->lock); if (!tz->ops->set_trips || !tz->ops->get_trip_hyst) - goto exit; + return; for (i = 0; i < tz->num_trips; i++) { int trip_low; @@ -154,7 +151,7 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz) /* No need to change trip points */ if (tz->prev_low_trip == low && tz->prev_high_trip == high) - goto exit; + return; tz->prev_low_trip = low; tz->prev_high_trip = high; @@ -169,8 +166,24 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz) ret = tz->ops->set_trips(tz, low, high); if (ret) dev_err(&tz->device, "Failed to set trips: %d\n", ret); +} -exit: +/** + * thermal_zone_set_trips - Computes the next trip points for the driver + * @tz: a pointer to a thermal zone device structure + * + * The function computes the next temperature boundaries by browsing + * the trip points. The result is the closer low and high trip points + * to the current temperature. These values are passed to the backend + * driver to let it set its own notification mechanism (usually an + * interrupt). + * + * It does not return a value + */ +void thermal_zone_set_trips(struct thermal_zone_device *tz) +{ + mutex_lock(&tz->lock); + __thermal_zone_set_trips(tz); mutex_unlock(&tz->lock); } diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index 09e49ec8b6f48926e5a80838509fec73d7da900a..f53f4ceb6a5de010f3e7b229d75d605736830121 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -147,7 +147,7 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) return -ENOMEM; INIT_LIST_HEAD(&hwmon->tz_list); - strlcpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH); + strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH); strreplace(hwmon->type, '-', '_'); hwmon->device = hwmon_device_register_for_thermal(&tz->device, hwmon->type, hwmon); diff --git a/drivers/thermal/thermal_mmio.c b/drivers/thermal/thermal_mmio.c index 360b0dfdc3b07efcf99d622ab89f344221ffd1a4..39c921415989e524ed8b5db09d4977b5523b3592 100644 --- a/drivers/thermal/thermal_mmio.c +++ b/drivers/thermal/thermal_mmio.c @@ -20,11 +20,10 @@ static u32 thermal_mmio_readb(void __iomem *mmio_base) return readb(mmio_base); } -static int thermal_mmio_get_temperature(void *private, int *temp) +static int thermal_mmio_get_temperature(struct thermal_zone_device *tz, int *temp) { int t; - struct thermal_mmio *sensor = - (struct thermal_mmio *)private; + struct thermal_mmio *sensor = tz->devdata; t = sensor->read_mmio(sensor->mmio_base) & sensor->mask; t *= sensor->factor; @@ -34,7 +33,7 @@ static int thermal_mmio_get_temperature(void *private, int *temp) return 0; } -static const struct thermal_zone_of_device_ops thermal_mmio_ops = { +static const struct thermal_zone_device_ops thermal_mmio_ops = { .get_temp = thermal_mmio_get_temperature, }; @@ -68,10 +67,10 @@ static int thermal_mmio_probe(struct platform_device *pdev) } } - thermal_zone = devm_thermal_zone_of_sensor_register(&pdev->dev, - 0, - sensor, - &thermal_mmio_ops); + thermal_zone = devm_thermal_of_zone_register(&pdev->dev, + 0, + sensor, + &thermal_mmio_ops); if (IS_ERR(thermal_zone)) { dev_err(&pdev->dev, "failed to register sensor (%ld)\n", @@ -79,7 +78,7 @@ static int thermal_mmio_probe(struct platform_device *pdev) return PTR_ERR(thermal_zone); } - thermal_mmio_get_temperature(sensor, &temperature); + thermal_mmio_get_temperature(thermal_zone, &temperature); dev_info(&pdev->dev, "thermal mmio sensor %s registered, current temperature: %d\n", pdev->name, temperature); @@ -107,7 +106,7 @@ static struct platform_driver thermal_mmio_driver = { .probe = thermal_mmio_probe, .driver = { .name = "thermal-mmio", - .of_match_table = of_match_ptr(thermal_mmio_id_table), + .of_match_table = thermal_mmio_id_table, }, }; diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c index 050d243a5fa1f9620ec69be700e11f2c03f97889..e2d78a996b5f320c3b9ae63a6575e1fa23ea272a 100644 --- a/drivers/thermal/thermal_netlink.c +++ b/drivers/thermal/thermal_netlink.c @@ -693,6 +693,7 @@ static struct genl_family thermal_gnl_family __ro_after_init = { .policy = thermal_genl_policy, .small_ops = thermal_genl_ops, .n_small_ops = ARRAY_SIZE(thermal_genl_ops), + .resv_start_op = THERMAL_GENL_CMD_CDEV_GET + 1, .mcgrps = thermal_genl_mcgrps, .n_mcgrps = ARRAY_SIZE(thermal_genl_mcgrps), }; diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c index 802c30b72a925da4dc8c1cb08ad8b0d9456fb9f4..d4b6335ace15faa1954691b05138efe5f9b7557d 100644 --- a/drivers/thermal/thermal_of.c +++ b/drivers/thermal/thermal_of.c @@ -19,93 +19,6 @@ #include "thermal_core.h" -/*** Private data structures to represent thermal device tree data ***/ - -/** - * struct __thermal_cooling_bind_param - a cooling device for a trip point - * @cooling_device: a pointer to identify the referred cooling device - * @min: minimum cooling state used at this trip point - * @max: maximum cooling state used at this trip point - */ - -struct __thermal_cooling_bind_param { - struct device_node *cooling_device; - unsigned long min; - unsigned long max; -}; - -/** - * struct __thermal_bind_params - a match between trip and cooling device - * @tcbp: a pointer to an array of cooling devices - * @count: number of elements in array - * @trip_id: the trip point index - * @usage: the percentage (from 0 to 100) of cooling contribution - */ - -struct __thermal_bind_params { - struct __thermal_cooling_bind_param *tcbp; - unsigned int count; - unsigned int trip_id; - unsigned int usage; -}; - -/** - * struct __thermal_zone - internal representation of a thermal zone - * @passive_delay: polling interval while passive cooling is activated - * @polling_delay: zone polling interval - * @slope: slope of the temperature adjustment curve - * @offset: offset of the temperature adjustment curve - * @ntrips: number of trip points - * @trips: an array of trip points (0..ntrips - 1) - * @num_tbps: number of thermal bind params - * @tbps: an array of thermal bind params (0..num_tbps - 1) - * @sensor_data: sensor private data used while reading temperature and trend - * @ops: set of callbacks to handle the thermal zone based on DT - */ - -struct __thermal_zone { - int passive_delay; - int polling_delay; - int slope; - int offset; - - /* trip data */ - int ntrips; - struct thermal_trip *trips; - - /* cooling binding data */ - int num_tbps; - struct __thermal_bind_params *tbps; - - /* sensor interface */ - void *sensor_data; - const struct thermal_zone_of_device_ops *ops; -}; - -/*** DT thermal zone device callbacks ***/ - -static int of_thermal_get_temp(struct thermal_zone_device *tz, - int *temp) -{ - struct __thermal_zone *data = tz->devdata; - - if (!data->ops || !data->ops->get_temp) - return -EINVAL; - - return data->ops->get_temp(data->sensor_data, temp); -} - -static int of_thermal_set_trips(struct thermal_zone_device *tz, - int low, int high) -{ - struct __thermal_zone *data = tz->devdata; - - if (!data->ops || !data->ops->set_trips) - return -EINVAL; - - return data->ops->set_trips(data->sensor_data, low, high); -} - /** * of_thermal_get_ntrips - function to export number of available trip * points. @@ -158,114 +71,6 @@ of_thermal_get_trip_points(struct thermal_zone_device *tz) } EXPORT_SYMBOL_GPL(of_thermal_get_trip_points); -/** - * of_thermal_set_emul_temp - function to set emulated temperature - * - * @tz: pointer to a thermal zone - * @temp: temperature to set - * - * This function gives the ability to set emulated value of temperature, - * which is handy for debugging - * - * Return: zero on success, error code otherwise - */ -static int of_thermal_set_emul_temp(struct thermal_zone_device *tz, - int temp) -{ - struct __thermal_zone *data = tz->devdata; - - if (!data->ops || !data->ops->set_emul_temp) - return -EINVAL; - - return data->ops->set_emul_temp(data->sensor_data, temp); -} - -static int of_thermal_get_trend(struct thermal_zone_device *tz, int trip, - enum thermal_trend *trend) -{ - struct __thermal_zone *data = tz->devdata; - - if (!data->ops || !data->ops->get_trend) - return -EINVAL; - - return data->ops->get_trend(data->sensor_data, trip, trend); -} - -static int of_thermal_change_mode(struct thermal_zone_device *tz, - enum thermal_device_mode mode) -{ - struct __thermal_zone *data = tz->devdata; - - return data->ops->change_mode(data->sensor_data, mode); -} - -static int of_thermal_bind(struct thermal_zone_device *thermal, - struct thermal_cooling_device *cdev) -{ - struct __thermal_zone *data = thermal->devdata; - struct __thermal_bind_params *tbp; - struct __thermal_cooling_bind_param *tcbp; - int i, j; - - if (!data || IS_ERR(data)) - return -ENODEV; - - /* find where to bind */ - for (i = 0; i < data->num_tbps; i++) { - tbp = data->tbps + i; - - for (j = 0; j < tbp->count; j++) { - tcbp = tbp->tcbp + j; - - if (tcbp->cooling_device == cdev->np) { - int ret; - - ret = thermal_zone_bind_cooling_device(thermal, - tbp->trip_id, cdev, - tcbp->max, - tcbp->min, - tbp->usage); - if (ret) - return ret; - } - } - } - - return 0; -} - -static int of_thermal_unbind(struct thermal_zone_device *thermal, - struct thermal_cooling_device *cdev) -{ - struct __thermal_zone *data = thermal->devdata; - struct __thermal_bind_params *tbp; - struct __thermal_cooling_bind_param *tcbp; - int i, j; - - if (!data || IS_ERR(data)) - return -ENODEV; - - /* find where to unbind */ - for (i = 0; i < data->num_tbps; i++) { - tbp = data->tbps + i; - - for (j = 0; j < tbp->count; j++) { - tcbp = tbp->tcbp + j; - - if (tcbp->cooling_device == cdev->np) { - int ret; - - ret = thermal_zone_unbind_cooling_device(thermal, - tbp->trip_id, cdev); - if (ret) - return ret; - } - } - } - - return 0; -} - static int of_thermal_get_trip_type(struct thermal_zone_device *tz, int trip, enum thermal_trip_type *type) { @@ -288,28 +93,6 @@ static int of_thermal_get_trip_temp(struct thermal_zone_device *tz, int trip, return 0; } -static int of_thermal_set_trip_temp(struct thermal_zone_device *tz, int trip, - int temp) -{ - struct __thermal_zone *data = tz->devdata; - - if (trip >= tz->num_trips || trip < 0) - return -EDOM; - - if (data->ops && data->ops->set_trip_temp) { - int ret; - - ret = data->ops->set_trip_temp(data->sensor_data, trip, temp); - if (ret) - return ret; - } - - /* thermal framework should take care of data->mask & (1 << trip) */ - tz->trips[trip].temperature = temp; - - return 0; -} - static int of_thermal_get_trip_hyst(struct thermal_zone_device *tz, int trip, int *hyst) { @@ -347,307 +130,6 @@ static int of_thermal_get_crit_temp(struct thermal_zone_device *tz, return -EINVAL; } -static struct thermal_zone_device_ops of_thermal_ops = { - .get_trip_type = of_thermal_get_trip_type, - .get_trip_temp = of_thermal_get_trip_temp, - .set_trip_temp = of_thermal_set_trip_temp, - .get_trip_hyst = of_thermal_get_trip_hyst, - .set_trip_hyst = of_thermal_set_trip_hyst, - .get_crit_temp = of_thermal_get_crit_temp, - - .bind = of_thermal_bind, - .unbind = of_thermal_unbind, -}; - -/*** sensor API ***/ - -static struct thermal_zone_device * -thermal_zone_of_add_sensor(struct device_node *zone, - struct device_node *sensor, void *data, - const struct thermal_zone_of_device_ops *ops) -{ - struct thermal_zone_device *tzd; - struct __thermal_zone *tz; - - tzd = thermal_zone_get_zone_by_name(zone->name); - if (IS_ERR(tzd)) - return ERR_PTR(-EPROBE_DEFER); - - tz = tzd->devdata; - - if (!ops) - return ERR_PTR(-EINVAL); - - mutex_lock(&tzd->lock); - tz->ops = ops; - tz->sensor_data = data; - - tzd->ops->get_temp = of_thermal_get_temp; - tzd->ops->get_trend = of_thermal_get_trend; - - /* - * The thermal zone core will calculate the window if they have set the - * optional set_trips pointer. - */ - if (ops->set_trips) - tzd->ops->set_trips = of_thermal_set_trips; - - if (ops->set_emul_temp) - tzd->ops->set_emul_temp = of_thermal_set_emul_temp; - - if (ops->change_mode) - tzd->ops->change_mode = of_thermal_change_mode; - - mutex_unlock(&tzd->lock); - - return tzd; -} - -/** - * thermal_zone_of_get_sensor_id - get sensor ID from a DT thermal zone - * @tz_np: a valid thermal zone device node. - * @sensor_np: a sensor node of a valid sensor device. - * @id: the sensor ID returned if success. - * - * This function will get sensor ID from a given thermal zone node and - * the sensor node must match the temperature provider @sensor_np. - * - * Return: 0 on success, proper error code otherwise. - */ - -int thermal_zone_of_get_sensor_id(struct device_node *tz_np, - struct device_node *sensor_np, - u32 *id) -{ - struct of_phandle_args sensor_specs; - int ret; - - ret = of_parse_phandle_with_args(tz_np, - "thermal-sensors", - "#thermal-sensor-cells", - 0, - &sensor_specs); - if (ret) - return ret; - - if (sensor_specs.np != sensor_np) { - of_node_put(sensor_specs.np); - return -ENODEV; - } - - if (sensor_specs.args_count > 1) - pr_warn("%pOFn: too many cells in sensor specifier %d\n", - sensor_specs.np, sensor_specs.args_count); - - *id = sensor_specs.args_count ? sensor_specs.args[0] : 0; - - of_node_put(sensor_specs.np); - - return 0; -} -EXPORT_SYMBOL_GPL(thermal_zone_of_get_sensor_id); - -/** - * thermal_zone_of_sensor_register - registers a sensor to a DT thermal zone - * @dev: a valid struct device pointer of a sensor device. Must contain - * a valid .of_node, for the sensor node. - * @sensor_id: a sensor identifier, in case the sensor IP has more - * than one sensors - * @data: a private pointer (owned by the caller) that will be passed - * back, when a temperature reading is needed. - * @ops: struct thermal_zone_of_device_ops *. Must contain at least .get_temp. - * - * This function will search the list of thermal zones described in device - * tree and look for the zone that refer to the sensor device pointed by - * @dev->of_node as temperature providers. For the zone pointing to the - * sensor node, the sensor will be added to the DT thermal zone device. - * - * The thermal zone temperature is provided by the @get_temp function - * pointer. When called, it will have the private pointer @data back. - * - * The thermal zone temperature trend is provided by the @get_trend function - * pointer. When called, it will have the private pointer @data back. - * - * TODO: - * 01 - This function must enqueue the new sensor instead of using - * it as the only source of temperature values. - * - * 02 - There must be a way to match the sensor with all thermal zones - * that refer to it. - * - * Return: On success returns a valid struct thermal_zone_device, - * otherwise, it returns a corresponding ERR_PTR(). Caller must - * check the return value with help of IS_ERR() helper. - */ -struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int sensor_id, void *data, - const struct thermal_zone_of_device_ops *ops) -{ - struct device_node *np, *child, *sensor_np; - struct thermal_zone_device *tzd = ERR_PTR(-ENODEV); - - np = of_find_node_by_name(NULL, "thermal-zones"); - if (!np) - return ERR_PTR(-ENODEV); - - if (!dev || !dev->of_node) { - of_node_put(np); - return ERR_PTR(-ENODEV); - } - - sensor_np = of_node_get(dev->of_node); - - for_each_available_child_of_node(np, child) { - int ret, id; - - /* For now, thermal framework supports only 1 sensor per zone */ - ret = thermal_zone_of_get_sensor_id(child, sensor_np, &id); - if (ret) - continue; - - if (id == sensor_id) { - tzd = thermal_zone_of_add_sensor(child, sensor_np, - data, ops); - if (!IS_ERR(tzd)) - thermal_zone_device_enable(tzd); - - of_node_put(child); - goto exit; - } - } -exit: - of_node_put(sensor_np); - of_node_put(np); - - return tzd; -} -EXPORT_SYMBOL_GPL(thermal_zone_of_sensor_register); - -/** - * thermal_zone_of_sensor_unregister - unregisters a sensor from a DT thermal zone - * @dev: a valid struct device pointer of a sensor device. Must contain - * a valid .of_node, for the sensor node. - * @tzd: a pointer to struct thermal_zone_device where the sensor is registered. - * - * This function removes the sensor callbacks and private data from the - * thermal zone device registered with thermal_zone_of_sensor_register() - * API. It will also silent the zone by remove the .get_temp() and .get_trend() - * thermal zone device callbacks. - * - * TODO: When the support to several sensors per zone is added, this - * function must search the sensor list based on @dev parameter. - * - */ -void thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tzd) -{ - struct __thermal_zone *tz; - - if (!dev || !tzd || !tzd->devdata) - return; - - tz = tzd->devdata; - - /* no __thermal_zone, nothing to be done */ - if (!tz) - return; - - /* stop temperature polling */ - thermal_zone_device_disable(tzd); - - mutex_lock(&tzd->lock); - tzd->ops->get_temp = NULL; - tzd->ops->get_trend = NULL; - tzd->ops->set_emul_temp = NULL; - tzd->ops->change_mode = NULL; - - tz->ops = NULL; - tz->sensor_data = NULL; - mutex_unlock(&tzd->lock); -} -EXPORT_SYMBOL_GPL(thermal_zone_of_sensor_unregister); - -static void devm_thermal_zone_of_sensor_release(struct device *dev, void *res) -{ - thermal_zone_of_sensor_unregister(dev, - *(struct thermal_zone_device **)res); -} - -static int devm_thermal_zone_of_sensor_match(struct device *dev, void *res, - void *data) -{ - struct thermal_zone_device **r = res; - - if (WARN_ON(!r || !*r)) - return 0; - - return *r == data; -} - -/** - * devm_thermal_zone_of_sensor_register - Resource managed version of - * thermal_zone_of_sensor_register() - * @dev: a valid struct device pointer of a sensor device. Must contain - * a valid .of_node, for the sensor node. - * @sensor_id: a sensor identifier, in case the sensor IP has more - * than one sensors - * @data: a private pointer (owned by the caller) that will be passed - * back, when a temperature reading is needed. - * @ops: struct thermal_zone_of_device_ops *. Must contain at least .get_temp. - * - * Refer thermal_zone_of_sensor_register() for more details. - * - * Return: On success returns a valid struct thermal_zone_device, - * otherwise, it returns a corresponding ERR_PTR(). Caller must - * check the return value with help of IS_ERR() helper. - * Registered thermal_zone_device device will automatically be - * released when device is unbounded. - */ -struct thermal_zone_device *devm_thermal_zone_of_sensor_register( - struct device *dev, int sensor_id, - void *data, const struct thermal_zone_of_device_ops *ops) -{ - struct thermal_zone_device **ptr, *tzd; - - ptr = devres_alloc(devm_thermal_zone_of_sensor_release, sizeof(*ptr), - GFP_KERNEL); - if (!ptr) - return ERR_PTR(-ENOMEM); - - tzd = thermal_zone_of_sensor_register(dev, sensor_id, data, ops); - if (IS_ERR(tzd)) { - devres_free(ptr); - return tzd; - } - - *ptr = tzd; - devres_add(dev, ptr); - - return tzd; -} -EXPORT_SYMBOL_GPL(devm_thermal_zone_of_sensor_register); - -/** - * devm_thermal_zone_of_sensor_unregister - Resource managed version of - * thermal_zone_of_sensor_unregister(). - * @dev: Device for which which resource was allocated. - * @tzd: a pointer to struct thermal_zone_device where the sensor is registered. - * - * This function removes the sensor callbacks and private data from the - * thermal zone device registered with devm_thermal_zone_of_sensor_register() - * API. It will also silent the zone by remove the .get_temp() and .get_trend() - * thermal zone device callbacks. - * Normally this function will not need to be called and the resource - * management code will ensure that the resource is freed. - */ -void devm_thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tzd) -{ - WARN_ON(devres_release(dev, devm_thermal_zone_of_sensor_release, - devm_thermal_zone_of_sensor_match, tzd)); -} -EXPORT_SYMBOL_GPL(devm_thermal_zone_of_sensor_unregister); - /*** functions parsing device tree nodes ***/ static int of_find_trip_id(struct device_node *np, struct device_node *trip) @@ -679,98 +161,6 @@ out: return i; } -/** - * thermal_of_populate_bind_params - parse and fill cooling map data - * @np: DT node containing a cooling-map node - * @__tbp: data structure to be filled with cooling map info - * @trips: array of thermal zone trip points - * @ntrips: number of trip points inside trips. - * - * This function parses a cooling-map type of node represented by - * @np parameter and fills the read data into @__tbp data structure. - * It needs the already parsed array of trip points of the thermal zone - * in consideration. - * - * Return: 0 on success, proper error code otherwise - */ -static int thermal_of_populate_bind_params(struct device_node *tz_np, - struct device_node *np, - struct __thermal_bind_params *__tbp) -{ - struct of_phandle_args cooling_spec; - struct __thermal_cooling_bind_param *__tcbp; - struct device_node *trip; - int ret, i, count; - int trip_id; - u32 prop; - - /* Default weight. Usage is optional */ - __tbp->usage = THERMAL_WEIGHT_DEFAULT; - ret = of_property_read_u32(np, "contribution", &prop); - if (ret == 0) - __tbp->usage = prop; - - trip = of_parse_phandle(np, "trip", 0); - if (!trip) { - pr_err("missing trip property\n"); - return -ENODEV; - } - - trip_id = of_find_trip_id(tz_np, trip); - if (trip_id < 0) { - ret = trip_id; - goto end; - } - - __tbp->trip_id = trip_id; - - count = of_count_phandle_with_args(np, "cooling-device", - "#cooling-cells"); - if (count <= 0) { - pr_err("Add a cooling_device property with at least one device\n"); - ret = -ENOENT; - goto end; - } - - __tcbp = kcalloc(count, sizeof(*__tcbp), GFP_KERNEL); - if (!__tcbp) { - ret = -ENOMEM; - goto end; - } - - for (i = 0; i < count; i++) { - ret = of_parse_phandle_with_args(np, "cooling-device", - "#cooling-cells", i, &cooling_spec); - if (ret < 0) { - pr_err("Invalid cooling-device entry\n"); - goto free_tcbp; - } - - __tcbp[i].cooling_device = cooling_spec.np; - - if (cooling_spec.args_count >= 2) { /* at least min and max */ - __tcbp[i].min = cooling_spec.args[0]; - __tcbp[i].max = cooling_spec.args[1]; - } else { - pr_err("wrong reference to cooling device, missing limits\n"); - } - } - - __tbp->tcbp = __tcbp; - __tbp->count = count; - - goto end; - -free_tcbp: - for (i = i - 1; i >= 0; i--) - of_node_put(__tcbp[i].cooling_device); - kfree(__tcbp); -end: - of_node_put(trip); - - return ret; -} - /* * It maps 'enum thermal_trip_type' found in include/linux/thermal.h * into the device tree binding of 'trip', property type. @@ -811,16 +201,6 @@ static int thermal_of_get_trip_type(struct device_node *np, return -ENODEV; } -/** - * thermal_of_populate_trip - parse and fill one trip point data - * @np: DT node containing a trip point node - * @trip: trip point data structure to be filled up - * - * This function parses a trip point type of node represented by - * @np parameter and fills the read data into @trip data structure. - * - * Return: 0 on success, proper error code otherwise - */ static int thermal_of_populate_trip(struct device_node *np, struct thermal_trip *trip) { @@ -897,258 +277,458 @@ out_of_node_put: return ERR_PTR(ret); } -/** - * thermal_of_build_thermal_zone - parse and fill one thermal zone data - * @np: DT node containing a thermal zone node - * - * This function parses a thermal zone type of node represented by - * @np parameter and fills the read data into a __thermal_zone data structure - * and return this pointer. - * - * TODO: Missing properties to parse: thermal-sensor-names - * - * Return: On success returns a valid struct __thermal_zone, - * otherwise, it returns a corresponding ERR_PTR(). Caller must - * check the return value with help of IS_ERR() helper. - */ -static struct __thermal_zone -__init *thermal_of_build_thermal_zone(struct device_node *np) +static struct device_node *of_thermal_zone_find(struct device_node *sensor, int id) { - struct device_node *child = NULL, *gchild; - struct __thermal_zone *tz; - int ret, i; - u32 prop, coef[2]; + struct device_node *np, *tz; + struct of_phandle_args sensor_specs; + np = of_find_node_by_name(NULL, "thermal-zones"); if (!np) { - pr_err("no thermal zone np\n"); - return ERR_PTR(-EINVAL); + pr_debug("No thermal zones description\n"); + return ERR_PTR(-ENODEV); } - tz = kzalloc(sizeof(*tz), GFP_KERNEL); - if (!tz) - return ERR_PTR(-ENOMEM); + /* + * Search for each thermal zone, a defined sensor + * corresponding to the one passed as parameter + */ + for_each_available_child_of_node(np, tz) { + + int count, i; + + count = of_count_phandle_with_args(tz, "thermal-sensors", + "#thermal-sensor-cells"); + if (count <= 0) { + pr_err("%pOFn: missing thermal sensor\n", tz); + tz = ERR_PTR(-EINVAL); + goto out; + } - ret = of_property_read_u32(np, "polling-delay-passive", &prop); + for (i = 0; i < count; i++) { + + int ret; + + ret = of_parse_phandle_with_args(tz, "thermal-sensors", + "#thermal-sensor-cells", + i, &sensor_specs); + if (ret < 0) { + pr_err("%pOFn: Failed to read thermal-sensors cells: %d\n", tz, ret); + tz = ERR_PTR(ret); + goto out; + } + + if ((sensor == sensor_specs.np) && id == (sensor_specs.args_count ? + sensor_specs.args[0] : 0)) { + pr_debug("sensor %pOFn id=%d belongs to %pOFn\n", sensor, id, tz); + goto out; + } + } + } + tz = ERR_PTR(-ENODEV); +out: + of_node_put(np); + return tz; +} + +static int thermal_of_monitor_init(struct device_node *np, int *delay, int *pdelay) +{ + int ret; + + ret = of_property_read_u32(np, "polling-delay-passive", pdelay); if (ret < 0) { pr_err("%pOFn: missing polling-delay-passive property\n", np); - goto free_tz; + return ret; } - tz->passive_delay = prop; - ret = of_property_read_u32(np, "polling-delay", &prop); + ret = of_property_read_u32(np, "polling-delay", delay); if (ret < 0) { pr_err("%pOFn: missing polling-delay property\n", np); - goto free_tz; + return ret; } - tz->polling_delay = prop; + + return 0; +} + +static struct thermal_zone_params *thermal_of_parameters_init(struct device_node *np) +{ + struct thermal_zone_params *tzp; + int coef[2]; + int ncoef = ARRAY_SIZE(coef); + int prop, ret; + + tzp = kzalloc(sizeof(*tzp), GFP_KERNEL); + if (!tzp) + return ERR_PTR(-ENOMEM); + + tzp->no_hwmon = true; + + if (!of_property_read_u32(np, "sustainable-power", &prop)) + tzp->sustainable_power = prop; /* - * REVIST: for now, the thermal framework supports only - * one sensor per thermal zone. Thus, we are considering - * only the first two values as slope and offset. + * For now, the thermal framework supports only one sensor per + * thermal zone. Thus, we are considering only the first two + * values as slope and offset. */ - ret = of_property_read_u32_array(np, "coefficients", coef, 2); - if (ret == 0) { - tz->slope = coef[0]; - tz->offset = coef[1]; - } else { - tz->slope = 1; - tz->offset = 0; + ret = of_property_read_u32_array(np, "coefficients", coef, ncoef); + if (ret) { + coef[0] = 1; + coef[1] = 0; } - tz->trips = thermal_of_trips_init(np, &tz->ntrips); - if (IS_ERR(tz->trips)) { - ret = PTR_ERR(tz->trips); - goto finish; + tzp->slope = coef[0]; + tzp->offset = coef[1]; + + return tzp; +} + +static struct device_node *thermal_of_zone_get_by_name(struct thermal_zone_device *tz) +{ + struct device_node *np, *tz_np; + + np = of_find_node_by_name(NULL, "thermal-zones"); + if (!np) + return ERR_PTR(-ENODEV); + + tz_np = of_get_child_by_name(np, tz->type); + + of_node_put(np); + + if (!tz_np) + return ERR_PTR(-ENODEV); + + return tz_np; +} + +static int __thermal_of_unbind(struct device_node *map_np, int index, int trip_id, + struct thermal_zone_device *tz, struct thermal_cooling_device *cdev) +{ + struct of_phandle_args cooling_spec; + int ret; + + ret = of_parse_phandle_with_args(map_np, "cooling-device", "#cooling-cells", + index, &cooling_spec); + + of_node_put(cooling_spec.np); + + if (ret < 0) { + pr_err("Invalid cooling-device entry\n"); + return ret; } - /* cooling-maps */ - child = of_get_child_by_name(np, "cooling-maps"); + if (cooling_spec.args_count < 2) { + pr_err("wrong reference to cooling device, missing limits\n"); + return -EINVAL; + } - /* cooling-maps not provided */ - if (!child) - goto finish; + if (cooling_spec.np != cdev->np) + return 0; - tz->num_tbps = of_get_child_count(child); - if (tz->num_tbps == 0) - goto finish; + ret = thermal_zone_unbind_cooling_device(tz, trip_id, cdev); + if (ret) + pr_err("Failed to unbind '%s' with '%s': %d\n", tz->type, cdev->type, ret); - tz->tbps = kcalloc(tz->num_tbps, sizeof(*tz->tbps), GFP_KERNEL); - if (!tz->tbps) { - ret = -ENOMEM; - goto free_trips; + return ret; +} + +static int __thermal_of_bind(struct device_node *map_np, int index, int trip_id, + struct thermal_zone_device *tz, struct thermal_cooling_device *cdev) +{ + struct of_phandle_args cooling_spec; + int ret, weight = THERMAL_WEIGHT_DEFAULT; + + of_property_read_u32(map_np, "contribution", &weight); + + ret = of_parse_phandle_with_args(map_np, "cooling-device", "#cooling-cells", + index, &cooling_spec); + + of_node_put(cooling_spec.np); + + if (ret < 0) { + pr_err("Invalid cooling-device entry\n"); + return ret; } - i = 0; - for_each_child_of_node(child, gchild) { - ret = thermal_of_populate_bind_params(np, gchild, &tz->tbps[i++]); - if (ret) { - of_node_put(gchild); - goto free_tbps; - } + if (cooling_spec.args_count < 2) { + pr_err("wrong reference to cooling device, missing limits\n"); + return -EINVAL; } -finish: - of_node_put(child); + if (cooling_spec.np != cdev->np) + return 0; - return tz; + ret = thermal_zone_bind_cooling_device(tz, trip_id, cdev, cooling_spec.args[1], + cooling_spec.args[0], + weight); + if (ret) + pr_err("Failed to bind '%s' with '%s': %d\n", tz->type, cdev->type, ret); -free_tbps: - for (i = i - 1; i >= 0; i--) { - struct __thermal_bind_params *tbp = tz->tbps + i; - int j; + return ret; +} + +static int thermal_of_for_each_cooling_device(struct device_node *tz_np, struct device_node *map_np, + struct thermal_zone_device *tz, struct thermal_cooling_device *cdev, + int (*action)(struct device_node *, int, int, + struct thermal_zone_device *, struct thermal_cooling_device *)) +{ + struct device_node *tr_np; + int count, i, trip_id; - for (j = 0; j < tbp->count; j++) - of_node_put(tbp->tcbp[j].cooling_device); + tr_np = of_parse_phandle(map_np, "trip", 0); + if (!tr_np) + return -ENODEV; - kfree(tbp->tcbp); + trip_id = of_find_trip_id(tz_np, tr_np); + if (trip_id < 0) + return trip_id; + + count = of_count_phandle_with_args(map_np, "cooling-device", "#cooling-cells"); + if (count <= 0) { + pr_err("Add a cooling_device property with at least one device\n"); + return -ENOENT; } - kfree(tz->tbps); -free_trips: - kfree(tz->trips); -free_tz: - kfree(tz); - of_node_put(child); + /* + * At this point, we don't want to bail out when there is an + * error, we will try to bind/unbind as many as possible + * cooling devices + */ + for (i = 0; i < count; i++) + action(map_np, i, trip_id, tz, cdev); - return ERR_PTR(ret); + return 0; } -static __init void of_thermal_free_zone(struct __thermal_zone *tz) +static int thermal_of_for_each_cooling_maps(struct thermal_zone_device *tz, + struct thermal_cooling_device *cdev, + int (*action)(struct device_node *, int, int, + struct thermal_zone_device *, struct thermal_cooling_device *)) { - struct __thermal_bind_params *tbp; - int i, j; + struct device_node *tz_np, *cm_np, *child; + int ret = 0; - for (i = 0; i < tz->num_tbps; i++) { - tbp = tz->tbps + i; + tz_np = thermal_of_zone_get_by_name(tz); + if (IS_ERR(tz_np)) { + pr_err("Failed to get node tz by name\n"); + return PTR_ERR(tz_np); + } - for (j = 0; j < tbp->count; j++) - of_node_put(tbp->tcbp[j].cooling_device); + cm_np = of_get_child_by_name(tz_np, "cooling-maps"); + if (!cm_np) + goto out; - kfree(tbp->tcbp); + for_each_child_of_node(cm_np, child) { + ret = thermal_of_for_each_cooling_device(tz_np, child, tz, cdev, action); + if (ret) + break; } - kfree(tz->tbps); - kfree(tz->trips); - kfree(tz); + of_node_put(cm_np); +out: + of_node_put(tz_np); + + return ret; +} + +static int thermal_of_bind(struct thermal_zone_device *tz, + struct thermal_cooling_device *cdev) +{ + return thermal_of_for_each_cooling_maps(tz, cdev, __thermal_of_bind); +} + +static int thermal_of_unbind(struct thermal_zone_device *tz, + struct thermal_cooling_device *cdev) +{ + return thermal_of_for_each_cooling_maps(tz, cdev, __thermal_of_unbind); } /** - * of_thermal_destroy_zones - remove all zones parsed and allocated resources + * thermal_of_zone_unregister - Cleanup the specific allocated ressources * - * Finds all zones parsed and added to the thermal framework and remove them - * from the system, together with their resources. + * This function disables the thermal zone and frees the different + * ressources allocated specific to the thermal OF. * + * @tz: a pointer to the thermal zone structure */ -static __init void of_thermal_destroy_zones(void) +void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - struct device_node *np, *child; + struct thermal_trip *trips = tz->trips; + struct thermal_zone_params *tzp = tz->tzp; + struct thermal_zone_device_ops *ops = tz->ops; - np = of_find_node_by_name(NULL, "thermal-zones"); - if (!np) { - pr_debug("unable to find thermal zones\n"); - return; + thermal_zone_device_disable(tz); + thermal_zone_device_unregister(tz); + kfree(trips); + kfree(tzp); + kfree(ops); +} +EXPORT_SYMBOL_GPL(thermal_of_zone_unregister); + +/** + * thermal_of_zone_register - Register a thermal zone with device node + * sensor + * + * The thermal_of_zone_register() parses a device tree given a device + * node sensor and identifier. It searches for the thermal zone + * associated to the couple sensor/id and retrieves all the thermal + * zone properties and registers new thermal zone with those + * properties. + * + * @sensor: A device node pointer corresponding to the sensor in the device tree + * @id: An integer as sensor identifier + * @data: A private data to be stored in the thermal zone dedicated private area + * @ops: A set of thermal sensor ops + * + * Return: a valid thermal zone structure pointer on success. + * - EINVAL: if the device tree thermal description is malformed + * - ENOMEM: if one structure can not be allocated + * - Other negative errors are returned by the underlying called functions + */ +struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, + const struct thermal_zone_device_ops *ops) +{ + struct thermal_zone_device *tz; + struct thermal_trip *trips; + struct thermal_zone_params *tzp; + struct thermal_zone_device_ops *of_ops; + struct device_node *np; + int delay, pdelay; + int ntrips, mask; + int ret; + + of_ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL); + if (!of_ops) + return ERR_PTR(-ENOMEM); + + np = of_thermal_zone_find(sensor, id); + if (IS_ERR(np)) { + if (PTR_ERR(np) != -ENODEV) + pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id); + return ERR_CAST(np); } - for_each_available_child_of_node(np, child) { - struct thermal_zone_device *zone; + trips = thermal_of_trips_init(np, &ntrips); + if (IS_ERR(trips)) { + pr_err("Failed to find trip points for %pOFn id=%d\n", sensor, id); + return ERR_CAST(trips); + } - zone = thermal_zone_get_zone_by_name(child->name); - if (IS_ERR(zone)) - continue; + ret = thermal_of_monitor_init(np, &delay, &pdelay); + if (ret) { + pr_err("Failed to initialize monitoring delays from %pOFn\n", np); + goto out_kfree_trips; + } - thermal_zone_device_unregister(zone); - kfree(zone->tzp); - kfree(zone->ops); - of_thermal_free_zone(zone->devdata); + tzp = thermal_of_parameters_init(np); + if (IS_ERR(tzp)) { + ret = PTR_ERR(tzp); + pr_err("Failed to initialize parameter from %pOFn: %d\n", np, ret); + goto out_kfree_trips; } - of_node_put(np); + + of_ops->get_trip_type = of_ops->get_trip_type ? : of_thermal_get_trip_type; + of_ops->get_trip_temp = of_ops->get_trip_temp ? : of_thermal_get_trip_temp; + of_ops->get_trip_hyst = of_ops->get_trip_hyst ? : of_thermal_get_trip_hyst; + of_ops->set_trip_hyst = of_ops->set_trip_hyst ? : of_thermal_set_trip_hyst; + of_ops->get_crit_temp = of_ops->get_crit_temp ? : of_thermal_get_crit_temp; + of_ops->bind = thermal_of_bind; + of_ops->unbind = thermal_of_unbind; + + mask = GENMASK_ULL((ntrips) - 1, 0); + + tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips, + mask, data, of_ops, tzp, + pdelay, delay); + if (IS_ERR(tz)) { + ret = PTR_ERR(tz); + pr_err("Failed to register thermal zone %pOFn: %d\n", np, ret); + goto out_kfree_tzp; + } + + ret = thermal_zone_device_enable(tz); + if (ret) { + pr_err("Failed to enabled thermal zone '%s', id=%d: %d\n", + tz->type, tz->id, ret); + thermal_of_zone_unregister(tz); + return ERR_PTR(ret); + } + + return tz; + +out_kfree_tzp: + kfree(tzp); +out_kfree_trips: + kfree(trips); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(thermal_of_zone_register); + +static void devm_thermal_of_zone_release(struct device *dev, void *res) +{ + thermal_of_zone_unregister(*(struct thermal_zone_device **)res); +} + +static int devm_thermal_of_zone_match(struct device *dev, void *res, + void *data) +{ + struct thermal_zone_device **r = res; + + if (WARN_ON(!r || !*r)) + return 0; + + return *r == data; } /** - * of_parse_thermal_zones - parse device tree thermal data - * - * Initialization function that can be called by machine initialization - * code to parse thermal data and populate the thermal framework - * with hardware thermal zones info. This function only parses thermal zones. - * Cooling devices and sensor devices nodes are supposed to be parsed - * by their respective drivers. + * devm_thermal_of_zone_register - register a thermal tied with the sensor life cycle * - * Return: 0 on success, proper error code otherwise + * This function is the device version of the thermal_of_zone_register() function. * + * @dev: a device structure pointer to sensor to be tied with the thermal zone OF life cycle + * @sensor_id: the sensor identifier + * @data: a pointer to a private data to be stored in the thermal zone 'devdata' field + * @ops: a pointer to the ops structure associated with the sensor */ -int __init of_parse_thermal_zones(void) +struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int sensor_id, void *data, + const struct thermal_zone_device_ops *ops) { - struct device_node *np, *child; - struct __thermal_zone *tz; - struct thermal_zone_device_ops *ops; - - np = of_find_node_by_name(NULL, "thermal-zones"); - if (!np) { - pr_debug("unable to find thermal zones\n"); - return 0; /* Run successfully on systems without thermal DT */ - } - - for_each_available_child_of_node(np, child) { - struct thermal_zone_device *zone; - struct thermal_zone_params *tzp; - int i, mask = 0; - u32 prop; - - tz = thermal_of_build_thermal_zone(child); - if (IS_ERR(tz)) { - pr_err("failed to build thermal zone %pOFn: %ld\n", - child, - PTR_ERR(tz)); - continue; - } - - ops = kmemdup(&of_thermal_ops, sizeof(*ops), GFP_KERNEL); - if (!ops) - goto exit_free; + struct thermal_zone_device **ptr, *tzd; - tzp = kzalloc(sizeof(*tzp), GFP_KERNEL); - if (!tzp) { - kfree(ops); - goto exit_free; - } + ptr = devres_alloc(devm_thermal_of_zone_release, sizeof(*ptr), + GFP_KERNEL); + if (!ptr) + return ERR_PTR(-ENOMEM); - /* No hwmon because there might be hwmon drivers registering */ - tzp->no_hwmon = true; - - if (!of_property_read_u32(child, "sustainable-power", &prop)) - tzp->sustainable_power = prop; - - for (i = 0; i < tz->ntrips; i++) - mask |= 1 << i; - - /* these two are left for temperature drivers to use */ - tzp->slope = tz->slope; - tzp->offset = tz->offset; - - zone = thermal_zone_device_register_with_trips(child->name, tz->trips, tz->ntrips, - mask, tz, ops, tzp, tz->passive_delay, - tz->polling_delay); - if (IS_ERR(zone)) { - pr_err("Failed to build %pOFn zone %ld\n", child, - PTR_ERR(zone)); - kfree(tzp); - kfree(ops); - of_thermal_free_zone(tz); - /* attempting to build remaining zones still */ - } + tzd = thermal_of_zone_register(dev->of_node, sensor_id, data, ops); + if (IS_ERR(tzd)) { + devres_free(ptr); + return tzd; } - of_node_put(np); - return 0; - -exit_free: - of_node_put(child); - of_node_put(np); - of_thermal_free_zone(tz); + *ptr = tzd; + devres_add(dev, ptr); - /* no memory available, so free what we have built */ - of_thermal_destroy_zones(); + return tzd; +} +EXPORT_SYMBOL_GPL(devm_thermal_of_zone_register); - return -ENOMEM; +/** + * devm_thermal_of_zone_unregister - Resource managed version of + * thermal_of_zone_unregister(). + * @dev: Device for which which resource was allocated. + * @tz: a pointer to struct thermal_zone where the sensor is registered. + * + * This function removes the sensor callbacks and private data from the + * thermal zone device registered with devm_thermal_zone_of_sensor_register() + * API. It will also silent the zone by remove the .get_temp() and .get_trend() + * thermal zone device callbacks. + * Normally this function will not need to be called and the resource + * management code will ensure that the resource is freed. + */ +void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz) +{ + WARN_ON(devres_release(dev, devm_thermal_of_zone_release, + devm_thermal_of_zone_match, tz)); } +EXPORT_SYMBOL_GPL(devm_thermal_of_zone_unregister); diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index 3a8d6e747c255dd639100762b2f8c5ecdcd3650b..ec495c7dff035384d8312a6fa74ea16b40d15b56 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -49,7 +49,11 @@ static ssize_t mode_show(struct device *dev, struct device_attribute *attr, char *buf) { struct thermal_zone_device *tz = to_thermal_zone(dev); - int enabled = thermal_zone_device_is_enabled(tz); + int enabled; + + mutex_lock(&tz->lock); + enabled = thermal_zone_device_is_enabled(tz); + mutex_unlock(&tz->lock); return sprintf(buf, "%s\n", enabled ? "enabled" : "disabled"); } @@ -115,7 +119,7 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, int temperature, hyst = 0; enum thermal_trip_type type; - if (!tz->ops->set_trip_temp) + if (!tz->ops->set_trip_temp && !tz->trips) return -EPERM; if (sscanf(attr->attr.name, "trip_point_%d_temp", &trip) != 1) @@ -124,9 +128,14 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr, if (kstrtoint(buf, 10, &temperature)) return -EINVAL; - ret = tz->ops->set_trip_temp(tz, trip, temperature); - if (ret) - return ret; + if (tz->ops->set_trip_temp) { + ret = tz->ops->set_trip_temp(tz, trip, temperature); + if (ret) + return ret; + } + + if (tz->trips) + tz->trips[trip].temperature = temperature; if (tz->ops->get_trip_hyst) { ret = tz->ops->get_trip_hyst(tz, trip, &hyst); diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c index 703039d8b937f664c5aaf9643281c8f1fa7c05c0..8a9055bd376ec16a0487c4d4484ac9968f3d59d5 100644 --- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c +++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c @@ -65,10 +65,10 @@ static inline int ti_thermal_hotspot_temperature(int t, int s, int c) /* thermal zone ops */ /* Get temperature callback function for thermal zone */ -static inline int __ti_thermal_get_temp(void *devdata, int *temp) +static inline int __ti_thermal_get_temp(struct thermal_zone_device *tz, int *temp) { struct thermal_zone_device *pcb_tz = NULL; - struct ti_thermal_data *data = devdata; + struct ti_thermal_data *data = tz->devdata; struct ti_bandgap *bgp; const struct ti_temp_sensor *s; int ret, tmp, slope, constant; @@ -85,8 +85,8 @@ static inline int __ti_thermal_get_temp(void *devdata, int *temp) return ret; /* Default constants */ - slope = thermal_zone_get_slope(data->ti_thermal); - constant = thermal_zone_get_offset(data->ti_thermal); + slope = thermal_zone_get_slope(tz); + constant = thermal_zone_get_offset(tz); pcb_tz = data->pcb_tz; /* In case pcb zone is available, use the extrapolation rule with it */ @@ -107,9 +107,9 @@ static inline int __ti_thermal_get_temp(void *devdata, int *temp) return ret; } -static int __ti_thermal_get_trend(void *p, int trip, enum thermal_trend *trend) +static int __ti_thermal_get_trend(struct thermal_zone_device *tz, int trip, enum thermal_trend *trend) { - struct ti_thermal_data *data = p; + struct ti_thermal_data *data = tz->devdata; struct ti_bandgap *bgp; int id, tr, ret = 0; @@ -130,7 +130,7 @@ static int __ti_thermal_get_trend(void *p, int trip, enum thermal_trend *trend) return 0; } -static const struct thermal_zone_of_device_ops ti_of_thermal_ops = { +static const struct thermal_zone_device_ops ti_of_thermal_ops = { .get_temp = __ti_thermal_get_temp, .get_trend = __ti_thermal_get_trend, }; @@ -170,7 +170,7 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id, return -EINVAL; /* in case this is specified by DT */ - data->ti_thermal = devm_thermal_zone_of_sensor_register(bgp->dev, id, + data->ti_thermal = devm_thermal_of_zone_register(bgp->dev, id, data, &ti_of_thermal_ops); if (IS_ERR(data->ti_thermal)) { dev_err(bgp->dev, "thermal zone device is NULL\n"); diff --git a/drivers/thermal/uniphier_thermal.c b/drivers/thermal/uniphier_thermal.c index 4cae5561a2a3848335f1898cc5fdebeaf513337c..4111d99ef50e302f9bf32c8a80aa6bf3098e88f7 100644 --- a/drivers/thermal/uniphier_thermal.c +++ b/drivers/thermal/uniphier_thermal.c @@ -187,9 +187,9 @@ static void uniphier_tm_disable_sensor(struct uniphier_tm_dev *tdev) usleep_range(1000, 2000); /* The spec note says at least 1ms */ } -static int uniphier_tm_get_temp(void *data, int *out_temp) +static int uniphier_tm_get_temp(struct thermal_zone_device *tz, int *out_temp) { - struct uniphier_tm_dev *tdev = data; + struct uniphier_tm_dev *tdev = tz->devdata; struct regmap *map = tdev->regmap; int ret; u32 temp; @@ -204,7 +204,7 @@ static int uniphier_tm_get_temp(void *data, int *out_temp) return 0; } -static const struct thermal_zone_of_device_ops uniphier_of_thermal_ops = { +static const struct thermal_zone_device_ops uniphier_of_thermal_ops = { .get_temp = uniphier_tm_get_temp, }; @@ -289,8 +289,8 @@ static int uniphier_tm_probe(struct platform_device *pdev) platform_set_drvdata(pdev, tdev); - tdev->tz_dev = devm_thermal_zone_of_sensor_register(dev, 0, tdev, - &uniphier_of_thermal_ops); + tdev->tz_dev = devm_thermal_of_zone_register(dev, 0, tdev, + &uniphier_of_thermal_ops); if (IS_ERR(tdev->tz_dev)) { dev_err(dev, "failed to register sensor device\n"); return PTR_ERR(tdev->tz_dev); diff --git a/drivers/thunderbolt/Kconfig b/drivers/thunderbolt/Kconfig index e76a6c1736376b976ad4f737dd98139eff5eafb8..448fd2ec8f6e2e6e263f96722803b69fc36ee9d8 100644 --- a/drivers/thunderbolt/Kconfig +++ b/drivers/thunderbolt/Kconfig @@ -27,10 +27,19 @@ config USB4_DEBUGFS_WRITE Only enable this if you know what you are doing! Never enable this for production systems or distro kernels. +config USB4_DEBUGFS_MARGINING + bool "Expose receiver lane margining operations under USB4 ports (DANGEROUS)" + depends on DEBUG_FS + depends on USB4_DEBUGFS_WRITE + help + Enables hardware and software based receiver lane margining support + under each USB4 port. Used for electrical quality and robustness + validation during manufacturing. Should not be enabled by distro + kernels. + config USB4_KUNIT_TEST bool "KUnit tests" if !KUNIT_ALL_TESTS - depends on (USB4=m || KUNIT=y) - depends on KUNIT + depends on USB4 && KUNIT=y default KUNIT_ALL_TESTS config USB4_DMA_TEST diff --git a/drivers/thunderbolt/acpi.c b/drivers/thunderbolt/acpi.c index b1f0dc8df47cde840a403f62699b9f5326b72c06..7a8adf5ad5a09d66032b32b06bea154e8a745368 100644 --- a/drivers/thunderbolt/acpi.c +++ b/drivers/thunderbolt/acpi.c @@ -42,7 +42,7 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data, */ dev = acpi_get_first_physical_node(adev); while (!dev) { - adev = adev->parent; + adev = acpi_dev_parent(adev); if (!adev) break; dev = acpi_get_first_physical_node(adev); diff --git a/drivers/thunderbolt/ctl.c b/drivers/thunderbolt/ctl.c index e5ede5debfb0e251fbaa49e4cfa4df7a49913a8a..0c661a706160fe579e7e64f4d692691d41cf8380 100644 --- a/drivers/thunderbolt/ctl.c +++ b/drivers/thunderbolt/ctl.c @@ -407,7 +407,7 @@ static void tb_ctl_rx_submit(struct ctl_pkg *pkg) static int tb_async_error(const struct ctl_pkg *pkg) { - const struct cfg_error_pkg *error = (const struct cfg_error_pkg *)pkg; + const struct cfg_error_pkg *error = pkg->buffer; if (pkg->frame.eof != TB_CFG_PKG_ERROR) return false; diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c index c850b0ac098c7171309ccf7680b80e77433bc686..834bcad42e9fe8cfa5a1e065bed7b512f817bb0d 100644 --- a/drivers/thunderbolt/debugfs.c +++ b/drivers/thunderbolt/debugfs.c @@ -12,6 +12,7 @@ #include #include "tb.h" +#include "sb_regs.h" #define PORT_CAP_PCIE_LEN 1 #define PORT_CAP_POWER_LEN 2 @@ -187,6 +188,828 @@ static ssize_t switch_regs_write(struct file *file, const char __user *user_buf, #define DEBUGFS_MODE 0400 #endif +#if IS_ENABLED(CONFIG_USB4_DEBUGFS_MARGINING) +/** + * struct tb_margining - Lane margining support + * @caps: Port lane margining capabilities + * @results: Last lane margining results + * @lanes: %0, %1 or %7 (all) + * @min_ber_level: Minimum supported BER level contour value + * @max_ber_level: Maximum supported BER level contour value + * @ber_level: Current BER level contour value + * @voltage_steps: Number of mandatory voltage steps + * @max_voltage_offset: Maximum mandatory voltage offset (in mV) + * @time_steps: Number of time margin steps + * @max_time_offset: Maximum time margin offset (in mUI) + * @software: %true if software margining is used instead of hardware + * @time: %true if time margining is used instead of voltage + * @right_high: %false if left/low margin test is performed, %true if + * right/high + */ +struct tb_margining { + u32 caps[2]; + u32 results[2]; + unsigned int lanes; + unsigned int min_ber_level; + unsigned int max_ber_level; + unsigned int ber_level; + unsigned int voltage_steps; + unsigned int max_voltage_offset; + unsigned int time_steps; + unsigned int max_time_offset; + bool software; + bool time; + bool right_high; +}; + +static bool supports_software(const struct usb4_port *usb4) +{ + return usb4->margining->caps[0] & USB4_MARGIN_CAP_0_MODES_SW; +} + +static bool supports_hardware(const struct usb4_port *usb4) +{ + return usb4->margining->caps[0] & USB4_MARGIN_CAP_0_MODES_HW; +} + +static bool both_lanes(const struct usb4_port *usb4) +{ + return usb4->margining->caps[0] & USB4_MARGIN_CAP_0_2_LANES; +} + +static unsigned int independent_voltage_margins(const struct usb4_port *usb4) +{ + return (usb4->margining->caps[0] & USB4_MARGIN_CAP_0_VOLTAGE_INDP_MASK) >> + USB4_MARGIN_CAP_0_VOLTAGE_INDP_SHIFT; +} + +static bool supports_time(const struct usb4_port *usb4) +{ + return usb4->margining->caps[0] & USB4_MARGIN_CAP_0_TIME; +} + +/* Only applicable if supports_time() returns true */ +static unsigned int independent_time_margins(const struct usb4_port *usb4) +{ + return (usb4->margining->caps[1] & USB4_MARGIN_CAP_1_TIME_INDP_MASK) >> + USB4_MARGIN_CAP_1_TIME_INDP_SHIFT; +} + +static ssize_t +margining_ber_level_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + unsigned int val; + int ret = 0; + char *buf; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + if (usb4->margining->software) { + ret = -EINVAL; + goto out_unlock; + } + + buf = validate_and_copy_from_user(user_buf, &count); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto out_unlock; + } + + buf[count - 1] = '\0'; + + ret = kstrtouint(buf, 10, &val); + if (ret) + goto out_free; + + if (val < usb4->margining->min_ber_level || + val > usb4->margining->max_ber_level) { + ret = -EINVAL; + goto out_free; + } + + usb4->margining->ber_level = val; + +out_free: + free_page((unsigned long)buf); +out_unlock: + mutex_unlock(&tb->lock); + + return ret < 0 ? ret : count; +} + +static void ber_level_show(struct seq_file *s, unsigned int val) +{ + if (val % 2) + seq_printf(s, "3 * 1e%d (%u)\n", -12 + (val + 1) / 2, val); + else + seq_printf(s, "1e%d (%u)\n", -12 + val / 2, val); +} + +static int margining_ber_level_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + + if (usb4->margining->software) + return -EINVAL; + ber_level_show(s, usb4->margining->ber_level); + return 0; +} +DEBUGFS_ATTR_RW(margining_ber_level); + +static int margining_caps_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + u32 cap0, cap1; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + /* Dump the raw caps first */ + cap0 = usb4->margining->caps[0]; + seq_printf(s, "0x%08x\n", cap0); + cap1 = usb4->margining->caps[1]; + seq_printf(s, "0x%08x\n", cap1); + + seq_printf(s, "# software margining: %s\n", + supports_software(usb4) ? "yes" : "no"); + if (supports_hardware(usb4)) { + seq_puts(s, "# hardware margining: yes\n"); + seq_puts(s, "# minimum BER level contour: "); + ber_level_show(s, usb4->margining->min_ber_level); + seq_puts(s, "# maximum BER level contour: "); + ber_level_show(s, usb4->margining->max_ber_level); + } else { + seq_puts(s, "# hardware margining: no\n"); + } + + seq_printf(s, "# both lanes simultaneously: %s\n", + both_lanes(usb4) ? "yes" : "no"); + seq_printf(s, "# voltage margin steps: %u\n", + usb4->margining->voltage_steps); + seq_printf(s, "# maximum voltage offset: %u mV\n", + usb4->margining->max_voltage_offset); + + switch (independent_voltage_margins(usb4)) { + case USB4_MARGIN_CAP_0_VOLTAGE_MIN: + seq_puts(s, "# returns minimum between high and low voltage margins\n"); + break; + case USB4_MARGIN_CAP_0_VOLTAGE_HL: + seq_puts(s, "# returns high or low voltage margin\n"); + break; + case USB4_MARGIN_CAP_0_VOLTAGE_BOTH: + seq_puts(s, "# returns both high and low margins\n"); + break; + } + + if (supports_time(usb4)) { + seq_puts(s, "# time margining: yes\n"); + seq_printf(s, "# time margining is destructive: %s\n", + cap1 & USB4_MARGIN_CAP_1_TIME_DESTR ? "yes" : "no"); + + switch (independent_time_margins(usb4)) { + case USB4_MARGIN_CAP_1_TIME_MIN: + seq_puts(s, "# returns minimum between left and right time margins\n"); + break; + case USB4_MARGIN_CAP_1_TIME_LR: + seq_puts(s, "# returns left or right margin\n"); + break; + case USB4_MARGIN_CAP_1_TIME_BOTH: + seq_puts(s, "# returns both left and right margins\n"); + break; + } + + seq_printf(s, "# time margin steps: %u\n", + usb4->margining->time_steps); + seq_printf(s, "# maximum time offset: %u mUI\n", + usb4->margining->max_time_offset); + } else { + seq_puts(s, "# time margining: no\n"); + } + + mutex_unlock(&tb->lock); + return 0; +} +DEBUGFS_ATTR_RO(margining_caps); + +static ssize_t +margining_lanes_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + int ret = 0; + char *buf; + + buf = validate_and_copy_from_user(user_buf, &count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + buf[count - 1] = '\0'; + + if (mutex_lock_interruptible(&tb->lock)) { + ret = -ERESTARTSYS; + goto out_free; + } + + if (!strcmp(buf, "0")) { + usb4->margining->lanes = 0; + } else if (!strcmp(buf, "1")) { + usb4->margining->lanes = 1; + } else if (!strcmp(buf, "all")) { + /* Needs to be supported */ + if (both_lanes(usb4)) + usb4->margining->lanes = 7; + else + ret = -EINVAL; + } else { + ret = -EINVAL; + } + + mutex_unlock(&tb->lock); + +out_free: + free_page((unsigned long)buf); + return ret < 0 ? ret : count; +} + +static int margining_lanes_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + unsigned int lanes; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + lanes = usb4->margining->lanes; + if (both_lanes(usb4)) { + if (!lanes) + seq_puts(s, "[0] 1 all\n"); + else if (lanes == 1) + seq_puts(s, "0 [1] all\n"); + else + seq_puts(s, "0 1 [all]\n"); + } else { + if (!lanes) + seq_puts(s, "[0] 1\n"); + else + seq_puts(s, "0 [1]\n"); + } + + mutex_unlock(&tb->lock); + return 0; +} +DEBUGFS_ATTR_RW(margining_lanes); + +static ssize_t margining_mode_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + int ret = 0; + char *buf; + + buf = validate_and_copy_from_user(user_buf, &count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + buf[count - 1] = '\0'; + + if (mutex_lock_interruptible(&tb->lock)) { + ret = -ERESTARTSYS; + goto out_free; + } + + if (!strcmp(buf, "software")) { + if (supports_software(usb4)) + usb4->margining->software = true; + else + ret = -EINVAL; + } else if (!strcmp(buf, "hardware")) { + if (supports_hardware(usb4)) + usb4->margining->software = false; + else + ret = -EINVAL; + } else { + ret = -EINVAL; + } + + mutex_unlock(&tb->lock); + +out_free: + free_page((unsigned long)buf); + return ret ? ret : count; +} + +static int margining_mode_show(struct seq_file *s, void *not_used) +{ + const struct tb_port *port = s->private; + const struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + const char *space = ""; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + if (supports_software(usb4)) { + if (usb4->margining->software) + seq_puts(s, "[software]"); + else + seq_puts(s, "software"); + space = " "; + } + if (supports_hardware(usb4)) { + if (usb4->margining->software) + seq_printf(s, "%shardware", space); + else + seq_printf(s, "%s[hardware]", space); + } + + mutex_unlock(&tb->lock); + + seq_puts(s, "\n"); + return 0; +} +DEBUGFS_ATTR_RW(margining_mode); + +static int margining_run_write(void *data, u64 val) +{ + struct tb_port *port = data; + struct usb4_port *usb4 = port->usb4; + struct tb_switch *sw = port->sw; + struct tb_margining *margining; + struct tb *tb = sw->tb; + int ret; + + if (val != 1) + return -EINVAL; + + pm_runtime_get_sync(&sw->dev); + + if (mutex_lock_interruptible(&tb->lock)) { + ret = -ERESTARTSYS; + goto out_rpm_put; + } + + /* + * CL states may interfere with lane margining so inform the user know + * and bail out. + */ + if (tb_port_is_clx_enabled(port, TB_CL1 | TB_CL2)) { + tb_port_warn(port, + "CL states are enabled, Disable them with clx=0 and re-connect\n"); + ret = -EINVAL; + goto out_unlock; + } + + margining = usb4->margining; + + if (margining->software) { + tb_port_dbg(port, "running software %s lane margining for lanes %u\n", + margining->time ? "time" : "voltage", margining->lanes); + ret = usb4_port_sw_margin(port, margining->lanes, margining->time, + margining->right_high, + USB4_MARGIN_SW_COUNTER_CLEAR); + if (ret) + goto out_unlock; + + ret = usb4_port_sw_margin_errors(port, &margining->results[0]); + } else { + tb_port_dbg(port, "running hardware %s lane margining for lanes %u\n", + margining->time ? "time" : "voltage", margining->lanes); + /* Clear the results */ + margining->results[0] = 0; + margining->results[1] = 0; + ret = usb4_port_hw_margin(port, margining->lanes, + margining->ber_level, margining->time, + margining->right_high, margining->results); + } + +out_unlock: + mutex_unlock(&tb->lock); +out_rpm_put: + pm_runtime_mark_last_busy(&sw->dev); + pm_runtime_put_autosuspend(&sw->dev); + + return ret; +} +DEFINE_DEBUGFS_ATTRIBUTE(margining_run_fops, NULL, margining_run_write, + "%llu\n"); + +static ssize_t margining_results_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + /* Just clear the results */ + usb4->margining->results[0] = 0; + usb4->margining->results[1] = 0; + + mutex_unlock(&tb->lock); + return count; +} + +static void voltage_margin_show(struct seq_file *s, + const struct tb_margining *margining, u8 val) +{ + unsigned int tmp, voltage; + + tmp = val & USB4_MARGIN_HW_RES_1_MARGIN_MASK; + voltage = tmp * margining->max_voltage_offset / margining->voltage_steps; + seq_printf(s, "%u mV (%u)", voltage, tmp); + if (val & USB4_MARGIN_HW_RES_1_EXCEEDS) + seq_puts(s, " exceeds maximum"); + seq_puts(s, "\n"); +} + +static void time_margin_show(struct seq_file *s, + const struct tb_margining *margining, u8 val) +{ + unsigned int tmp, interval; + + tmp = val & USB4_MARGIN_HW_RES_1_MARGIN_MASK; + interval = tmp * margining->max_time_offset / margining->time_steps; + seq_printf(s, "%u mUI (%u)", interval, tmp); + if (val & USB4_MARGIN_HW_RES_1_EXCEEDS) + seq_puts(s, " exceeds maximum"); + seq_puts(s, "\n"); +} + +static int margining_results_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb_margining *margining; + struct tb *tb = port->sw->tb; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + margining = usb4->margining; + /* Dump the raw results first */ + seq_printf(s, "0x%08x\n", margining->results[0]); + /* Only the hardware margining has two result dwords */ + if (!margining->software) { + unsigned int val; + + seq_printf(s, "0x%08x\n", margining->results[1]); + + if (margining->time) { + if (!margining->lanes || margining->lanes == 7) { + val = margining->results[1]; + seq_puts(s, "# lane 0 right time margin: "); + time_margin_show(s, margining, val); + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L0_LL_MARGIN_SHIFT; + seq_puts(s, "# lane 0 left time margin: "); + time_margin_show(s, margining, val); + } + if (margining->lanes == 1 || margining->lanes == 7) { + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L1_RH_MARGIN_SHIFT; + seq_puts(s, "# lane 1 right time margin: "); + time_margin_show(s, margining, val); + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L1_LL_MARGIN_SHIFT; + seq_puts(s, "# lane 1 left time margin: "); + time_margin_show(s, margining, val); + } + } else { + if (!margining->lanes || margining->lanes == 7) { + val = margining->results[1]; + seq_puts(s, "# lane 0 high voltage margin: "); + voltage_margin_show(s, margining, val); + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L0_LL_MARGIN_SHIFT; + seq_puts(s, "# lane 0 low voltage margin: "); + voltage_margin_show(s, margining, val); + } + if (margining->lanes == 1 || margining->lanes == 7) { + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L1_RH_MARGIN_SHIFT; + seq_puts(s, "# lane 1 high voltage margin: "); + voltage_margin_show(s, margining, val); + val = margining->results[1] >> + USB4_MARGIN_HW_RES_1_L1_LL_MARGIN_SHIFT; + seq_puts(s, "# lane 1 low voltage margin: "); + voltage_margin_show(s, margining, val); + } + } + } + + mutex_unlock(&tb->lock); + return 0; +} +DEBUGFS_ATTR_RW(margining_results); + +static ssize_t margining_test_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + int ret = 0; + char *buf; + + buf = validate_and_copy_from_user(user_buf, &count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + buf[count - 1] = '\0'; + + if (mutex_lock_interruptible(&tb->lock)) { + ret = -ERESTARTSYS; + goto out_free; + } + + if (!strcmp(buf, "time") && supports_time(usb4)) + usb4->margining->time = true; + else if (!strcmp(buf, "voltage")) + usb4->margining->time = false; + else + ret = -EINVAL; + + mutex_unlock(&tb->lock); + +out_free: + free_page((unsigned long)buf); + return ret ? ret : count; +} + +static int margining_test_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + if (supports_time(usb4)) { + if (usb4->margining->time) + seq_puts(s, "voltage [time]\n"); + else + seq_puts(s, "[voltage] time\n"); + } else { + seq_puts(s, "[voltage]\n"); + } + + mutex_unlock(&tb->lock); + return 0; +} +DEBUGFS_ATTR_RW(margining_test); + +static ssize_t margining_margin_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + int ret = 0; + char *buf; + + buf = validate_and_copy_from_user(user_buf, &count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + buf[count - 1] = '\0'; + + if (mutex_lock_interruptible(&tb->lock)) { + ret = -ERESTARTSYS; + goto out_free; + } + + if (usb4->margining->time) { + if (!strcmp(buf, "left")) + usb4->margining->right_high = false; + else if (!strcmp(buf, "right")) + usb4->margining->right_high = true; + else + ret = -EINVAL; + } else { + if (!strcmp(buf, "low")) + usb4->margining->right_high = false; + else if (!strcmp(buf, "high")) + usb4->margining->right_high = true; + else + ret = -EINVAL; + } + + mutex_unlock(&tb->lock); + +out_free: + free_page((unsigned long)buf); + return ret ? ret : count; +} + +static int margining_margin_show(struct seq_file *s, void *not_used) +{ + struct tb_port *port = s->private; + struct usb4_port *usb4 = port->usb4; + struct tb *tb = port->sw->tb; + + if (mutex_lock_interruptible(&tb->lock)) + return -ERESTARTSYS; + + if (usb4->margining->time) { + if (usb4->margining->right_high) + seq_puts(s, "left [right]\n"); + else + seq_puts(s, "[left] right\n"); + } else { + if (usb4->margining->right_high) + seq_puts(s, "low [high]\n"); + else + seq_puts(s, "[low] high\n"); + } + + mutex_unlock(&tb->lock); + return 0; +} +DEBUGFS_ATTR_RW(margining_margin); + +static void margining_port_init(struct tb_port *port) +{ + struct tb_margining *margining; + struct dentry *dir, *parent; + struct usb4_port *usb4; + char dir_name[10]; + unsigned int val; + int ret; + + usb4 = port->usb4; + if (!usb4) + return; + + snprintf(dir_name, sizeof(dir_name), "port%d", port->port); + parent = debugfs_lookup(dir_name, port->sw->debugfs_dir); + + margining = kzalloc(sizeof(*margining), GFP_KERNEL); + if (!margining) + return; + + ret = usb4_port_margining_caps(port, margining->caps); + if (ret) { + kfree(margining); + return; + } + + usb4->margining = margining; + + /* Set the initial mode */ + if (supports_software(usb4)) + margining->software = true; + + val = (margining->caps[0] & USB4_MARGIN_CAP_0_VOLTAGE_STEPS_MASK) >> + USB4_MARGIN_CAP_0_VOLTAGE_STEPS_SHIFT; + margining->voltage_steps = val; + val = (margining->caps[0] & USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_MASK) >> + USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_SHIFT; + margining->max_voltage_offset = 74 + val * 2; + + if (supports_time(usb4)) { + val = (margining->caps[1] & USB4_MARGIN_CAP_1_TIME_STEPS_MASK) >> + USB4_MARGIN_CAP_1_TIME_STEPS_SHIFT; + margining->time_steps = val; + val = (margining->caps[1] & USB4_MARGIN_CAP_1_TIME_OFFSET_MASK) >> + USB4_MARGIN_CAP_1_TIME_OFFSET_SHIFT; + /* + * Store it as mUI (milli Unit Interval) because we want + * to keep it as integer. + */ + margining->max_time_offset = 200 + 10 * val; + } + + dir = debugfs_create_dir("margining", parent); + if (supports_hardware(usb4)) { + val = (margining->caps[1] & USB4_MARGIN_CAP_1_MIN_BER_MASK) >> + USB4_MARGIN_CAP_1_MIN_BER_SHIFT; + margining->min_ber_level = val; + val = (margining->caps[1] & USB4_MARGIN_CAP_1_MAX_BER_MASK) >> + USB4_MARGIN_CAP_1_MAX_BER_SHIFT; + margining->max_ber_level = val; + + /* Set the default to minimum */ + margining->ber_level = margining->min_ber_level; + + debugfs_create_file("ber_level_contour", 0400, dir, port, + &margining_ber_level_fops); + } + debugfs_create_file("caps", 0400, dir, port, &margining_caps_fops); + debugfs_create_file("lanes", 0600, dir, port, &margining_lanes_fops); + debugfs_create_file("mode", 0600, dir, port, &margining_mode_fops); + debugfs_create_file("run", 0600, dir, port, &margining_run_fops); + debugfs_create_file("results", 0600, dir, port, &margining_results_fops); + debugfs_create_file("test", 0600, dir, port, &margining_test_fops); + if (independent_voltage_margins(usb4) || + (supports_time(usb4) && independent_time_margins(usb4))) + debugfs_create_file("margin", 0600, dir, port, &margining_margin_fops); +} + +static void margining_port_remove(struct tb_port *port) +{ + struct dentry *parent; + char dir_name[10]; + + if (!port->usb4) + return; + + snprintf(dir_name, sizeof(dir_name), "port%d", port->port); + parent = debugfs_lookup(dir_name, port->sw->debugfs_dir); + debugfs_remove_recursive(debugfs_lookup("margining", parent)); + + kfree(port->usb4->margining); + port->usb4->margining = NULL; +} + +static void margining_switch_init(struct tb_switch *sw) +{ + struct tb_port *upstream, *downstream; + struct tb_switch *parent_sw; + u64 route = tb_route(sw); + + if (!route) + return; + + upstream = tb_upstream_port(sw); + parent_sw = tb_switch_parent(sw); + downstream = tb_port_at(route, parent_sw); + + margining_port_init(downstream); + margining_port_init(upstream); +} + +static void margining_switch_remove(struct tb_switch *sw) +{ + struct tb_switch *parent_sw; + struct tb_port *downstream; + u64 route = tb_route(sw); + + if (!route) + return; + + /* + * Upstream is removed with the router itself but we need to + * remove the downstream port margining directory. + */ + parent_sw = tb_switch_parent(sw); + downstream = tb_port_at(route, parent_sw); + margining_port_remove(downstream); +} + +static void margining_xdomain_init(struct tb_xdomain *xd) +{ + struct tb_switch *parent_sw; + struct tb_port *downstream; + + parent_sw = tb_xdomain_parent(xd); + downstream = tb_port_at(xd->route, parent_sw); + + margining_port_init(downstream); +} + +static void margining_xdomain_remove(struct tb_xdomain *xd) +{ + struct tb_switch *parent_sw; + struct tb_port *downstream; + + parent_sw = tb_xdomain_parent(xd); + downstream = tb_port_at(xd->route, parent_sw); + margining_port_remove(downstream); +} +#else +static inline void margining_switch_init(struct tb_switch *sw) { } +static inline void margining_switch_remove(struct tb_switch *sw) { } +static inline void margining_xdomain_init(struct tb_xdomain *xd) { } +static inline void margining_xdomain_remove(struct tb_xdomain *xd) { } +#endif + static int port_clear_all_counters(struct tb_port *port) { u32 *buf; @@ -689,6 +1512,8 @@ void tb_switch_debugfs_init(struct tb_switch *sw) debugfs_create_file("counters", 0600, debugfs_dir, port, &counters_fops); } + + margining_switch_init(sw); } /** @@ -699,9 +1524,20 @@ void tb_switch_debugfs_init(struct tb_switch *sw) */ void tb_switch_debugfs_remove(struct tb_switch *sw) { + margining_switch_remove(sw); debugfs_remove_recursive(sw->debugfs_dir); } +void tb_xdomain_debugfs_init(struct tb_xdomain *xd) +{ + margining_xdomain_init(xd); +} + +void tb_xdomain_debugfs_remove(struct tb_xdomain *xd) +{ + margining_xdomain_remove(xd); +} + /** * tb_service_debugfs_init() - Add debugfs directory for service * @svc: Thunderbolt service pointer diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c index 99211f35a5cd2dd3504b658671cc315d82209b8f..ec7b5f65804e49e82533d6493c4cb90df514f2e2 100644 --- a/drivers/thunderbolt/domain.c +++ b/drivers/thunderbolt/domain.c @@ -144,11 +144,9 @@ static ssize_t boot_acl_show(struct device *dev, struct device_attribute *attr, for (ret = 0, i = 0; i < tb->nboot_acl; i++) { if (!uuid_is_null(&uuids[i])) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%pUb", - &uuids[i]); + ret += sysfs_emit_at(buf, ret, "%pUb", &uuids[i]); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s", - i < tb->nboot_acl - 1 ? "," : "\n"); + ret += sysfs_emit_at(buf, ret, "%s", i < tb->nboot_acl - 1 ? "," : "\n"); } out: @@ -247,7 +245,7 @@ static ssize_t deauthorization_show(struct device *dev, tb->security_level == TB_SECURITY_SECURE) deauthorization = !!tb->cm_ops->disapprove_switch; - return sprintf(buf, "%d\n", deauthorization); + return sysfs_emit(buf, "%d\n", deauthorization); } static DEVICE_ATTR_RO(deauthorization); @@ -270,7 +268,7 @@ static ssize_t security_show(struct device *dev, struct device_attribute *attr, if (tb->security_level < ARRAY_SIZE(tb_security_names)) name = tb_security_names[tb->security_level]; - return sprintf(buf, "%s\n", name); + return sysfs_emit(buf, "%s\n", name); } static DEVICE_ATTR_RO(security); diff --git a/drivers/thunderbolt/icm.c b/drivers/thunderbolt/icm.c index ae38f0d25a8d4b410b4d549c615a6796a009f629..86521ebb257942b6826348de39645354d3c310a1 100644 --- a/drivers/thunderbolt/icm.c +++ b/drivers/thunderbolt/icm.c @@ -2518,6 +2518,9 @@ struct tb *icm_probe(struct tb_nhi *nhi) case PCI_DEVICE_ID_INTEL_ADL_NHI1: case PCI_DEVICE_ID_INTEL_RPL_NHI0: case PCI_DEVICE_ID_INTEL_RPL_NHI1: + case PCI_DEVICE_ID_INTEL_MTL_M_NHI0: + case PCI_DEVICE_ID_INTEL_MTL_P_NHI0: + case PCI_DEVICE_ID_INTEL_MTL_P_NHI1: icm->is_supported = icm_tgl_is_supported; icm->driver_ready = icm_icl_driver_ready; icm->set_uuid = icm_icl_set_uuid; @@ -2529,6 +2532,7 @@ struct tb *icm_probe(struct tb_nhi *nhi) tb->cm_ops = &icm_icl_ops; break; + case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI: case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI: icm->is_supported = icm_tgl_is_supported; icm->get_mode = icm_ar_get_mode; diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index cb8c9c4ae93a26f1dccffeb19f6cacc070a5b06e..4dce2edd86ea0f9a4d223f2f6047e8a7db03d42e 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -28,7 +28,11 @@ #define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring") #define RING_FIRST_USABLE_HOPID 1 - +/* + * Used with QUIRK_E2E to specify an unused HopID the Rx credits are + * transferred. + */ +#define RING_E2E_RESERVED_HOPID RING_FIRST_USABLE_HOPID /* * Minimal number of vectors when we use MSI-X. Two for control channel * Rx/Tx and the rest four are for cross domain DMA paths. @@ -38,7 +42,9 @@ #define NHI_MAILBOX_TIMEOUT 500 /* ms */ +/* Host interface quirks */ #define QUIRK_AUTO_CLEAR_INT BIT(0) +#define QUIRK_E2E BIT(1) static int ring_interrupt_index(struct tb_ring *ring) { @@ -458,8 +464,18 @@ static void ring_release_msix(struct tb_ring *ring) static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring) { + unsigned int start_hop = RING_FIRST_USABLE_HOPID; int ret = 0; + if (nhi->quirks & QUIRK_E2E) { + start_hop = RING_FIRST_USABLE_HOPID + 1; + if (ring->flags & RING_FLAG_E2E && !ring->is_tx) { + dev_dbg(&nhi->pdev->dev, "quirking E2E TX HopID %u -> %u\n", + ring->e2e_tx_hop, RING_E2E_RESERVED_HOPID); + ring->e2e_tx_hop = RING_E2E_RESERVED_HOPID; + } + } + spin_lock_irq(&nhi->lock); if (ring->hop < 0) { @@ -469,7 +485,7 @@ static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring) * Automatically allocate HopID from the non-reserved * range 1 .. hop_count - 1. */ - for (i = RING_FIRST_USABLE_HOPID; i < nhi->hop_count; i++) { + for (i = start_hop; i < nhi->hop_count; i++) { if (ring->is_tx) { if (!nhi->tx_rings[i]) { ring->hop = i; @@ -484,6 +500,11 @@ static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring) } } + if (ring->hop > 0 && ring->hop < start_hop) { + dev_warn(&nhi->pdev->dev, "invalid hop: %d\n", ring->hop); + ret = -EINVAL; + goto err_unlock; + } if (ring->hop < 0 || ring->hop >= nhi->hop_count) { dev_warn(&nhi->pdev->dev, "invalid hop: %d\n", ring->hop); ret = -EINVAL; @@ -1097,12 +1118,26 @@ static void nhi_shutdown(struct tb_nhi *nhi) static void nhi_check_quirks(struct tb_nhi *nhi) { - /* - * Intel hardware supports auto clear of the interrupt status - * reqister right after interrupt is being issued. - */ - if (nhi->pdev->vendor == PCI_VENDOR_ID_INTEL) + if (nhi->pdev->vendor == PCI_VENDOR_ID_INTEL) { + /* + * Intel hardware supports auto clear of the interrupt + * status register right after interrupt is being + * issued. + */ nhi->quirks |= QUIRK_AUTO_CLEAR_INT; + + switch (nhi->pdev->device) { + case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI: + case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI: + /* + * Falcon Ridge controller needs the end-to-end + * flow control workaround to avoid losing Rx + * packets when RING_FLAG_E2E is set. + */ + nhi->quirks |= QUIRK_E2E; + break; + } + } } static int nhi_check_iommu_pdev(struct pci_dev *pdev, void *data) @@ -1149,6 +1184,7 @@ static void nhi_check_iommu(struct tb_nhi *nhi) static int nhi_init_msi(struct tb_nhi *nhi) { struct pci_dev *pdev = nhi->pdev; + struct device *dev = &pdev->dev; int res, irq, nvec; /* In case someone left them on. */ @@ -1179,10 +1215,8 @@ static int nhi_init_msi(struct tb_nhi *nhi) res = devm_request_irq(&pdev->dev, irq, nhi_msi, IRQF_NO_SUSPEND, "thunderbolt", nhi); - if (res) { - dev_err(&pdev->dev, "request_irq failed, aborting\n"); - return res; - } + if (res) + return dev_err_probe(dev, res, "request_irq failed, aborting\n"); } return 0; @@ -1223,26 +1257,21 @@ static struct tb *nhi_select_cm(struct tb_nhi *nhi) static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct device *dev = &pdev->dev; struct tb_nhi *nhi; struct tb *tb; int res; - if (!nhi_imr_valid(pdev)) { - dev_warn(&pdev->dev, "firmware image not valid, aborting\n"); - return -ENODEV; - } + if (!nhi_imr_valid(pdev)) + return dev_err_probe(dev, -ENODEV, "firmware image not valid, aborting\n"); res = pcim_enable_device(pdev); - if (res) { - dev_err(&pdev->dev, "cannot enable PCI device, aborting\n"); - return res; - } + if (res) + return dev_err_probe(dev, res, "cannot enable PCI device, aborting\n"); res = pcim_iomap_regions(pdev, 1 << 0, "thunderbolt"); - if (res) { - dev_err(&pdev->dev, "cannot obtain PCI resources, aborting\n"); - return res; - } + if (res) + return dev_err_probe(dev, res, "cannot obtain PCI resources, aborting\n"); nhi = devm_kzalloc(&pdev->dev, sizeof(*nhi), GFP_KERNEL); if (!nhi) @@ -1253,7 +1282,7 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* cannot fail - table is allocated in pcim_iomap_regions */ nhi->iobase = pcim_iomap_table(pdev)[0]; nhi->hop_count = ioread32(nhi->iobase + REG_HOP_COUNT) & 0x3ff; - dev_dbg(&pdev->dev, "total paths: %d\n", nhi->hop_count); + dev_dbg(dev, "total paths: %d\n", nhi->hop_count); nhi->tx_rings = devm_kcalloc(&pdev->dev, nhi->hop_count, sizeof(*nhi->tx_rings), GFP_KERNEL); @@ -1266,18 +1295,14 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) nhi_check_iommu(nhi); res = nhi_init_msi(nhi); - if (res) { - dev_err(&pdev->dev, "cannot enable MSI, aborting\n"); - return res; - } + if (res) + return dev_err_probe(dev, res, "cannot enable MSI, aborting\n"); spin_lock_init(&nhi->lock); res = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (res) { - dev_err(&pdev->dev, "failed to set DMA mask\n"); - return res; - } + if (res) + return dev_err_probe(dev, res, "failed to set DMA mask\n"); pci_set_master(pdev); @@ -1288,13 +1313,11 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) } tb = nhi_select_cm(nhi); - if (!tb) { - dev_err(&nhi->pdev->dev, + if (!tb) + return dev_err_probe(dev, -ENODEV, "failed to determine connection manager, aborting\n"); - return -ENODEV; - } - dev_dbg(&nhi->pdev->dev, "NHI initialized, starting thunderbolt\n"); + dev_dbg(dev, "NHI initialized, starting thunderbolt\n"); res = tb_domain_add(tb); if (res) { @@ -1398,6 +1421,7 @@ static struct pci_device_id nhi_ids[] = { .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, + /* Thunderbolt 4 */ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI1), @@ -1414,6 +1438,12 @@ static struct pci_device_id nhi_ids[] = { .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_M_NHI0), + .driver_data = (kernel_ulong_t)&icl_nhi_ops }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_P_NHI0), + .driver_data = (kernel_ulong_t)&icl_nhi_ops }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_P_NHI1), + .driver_data = (kernel_ulong_t)&icl_nhi_ops }, /* Any USB4 compliant host */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_USB4, ~0) }, diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index f09da5b62233b1204c807e2bd53e0f10326d94d3..b0718020c6f597031beb497060861e33c32831be 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -55,6 +55,7 @@ extern const struct tb_nhi_ops icl_nhi_ops; * need for the PCI quirk anymore as we will use ICM also on Apple * hardware. */ +#define PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI 0x1134 #define PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI 0x1137 #define PCI_DEVICE_ID_INTEL_WIN_RIDGE_2C_NHI 0x157d #define PCI_DEVICE_ID_INTEL_WIN_RIDGE_2C_BRIDGE 0x157e @@ -74,6 +75,9 @@ extern const struct tb_nhi_ops icl_nhi_ops; #define PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_BRIDGE 0x15ef #define PCI_DEVICE_ID_INTEL_ADL_NHI0 0x463e #define PCI_DEVICE_ID_INTEL_ADL_NHI1 0x466d +#define PCI_DEVICE_ID_INTEL_MTL_M_NHI0 0x7eb2 +#define PCI_DEVICE_ID_INTEL_MTL_P_NHI0 0x7ec2 +#define PCI_DEVICE_ID_INTEL_MTL_P_NHI1 0x7ec3 #define PCI_DEVICE_ID_INTEL_ICL_NHI1 0x8a0d #define PCI_DEVICE_ID_INTEL_ICL_NHI0 0x8a17 #define PCI_DEVICE_ID_INTEL_TGL_NHI0 0x9a1b diff --git a/drivers/thunderbolt/nvm.c b/drivers/thunderbolt/nvm.c index b3f31038937864993e2d861b07dbddb627d8b3ab..3dd5f81bd629355ab2274848b02bb08b29d8941b 100644 --- a/drivers/thunderbolt/nvm.c +++ b/drivers/thunderbolt/nvm.c @@ -12,19 +12,315 @@ #include "tb.h" +/* Intel specific NVM offsets */ +#define INTEL_NVM_DEVID 0x05 +#define INTEL_NVM_VERSION 0x08 +#define INTEL_NVM_CSS 0x10 +#define INTEL_NVM_FLASH_SIZE 0x45 + +/* ASMedia specific NVM offsets */ +#define ASMEDIA_NVM_DATE 0x1c +#define ASMEDIA_NVM_VERSION 0x28 + static DEFINE_IDA(nvm_ida); +/** + * struct tb_nvm_vendor_ops - Vendor specific NVM operations + * @read_version: Reads out NVM version from the flash + * @validate: Validates the NVM image before update (optional) + * @write_headers: Writes headers before the rest of the image (optional) + */ +struct tb_nvm_vendor_ops { + int (*read_version)(struct tb_nvm *nvm); + int (*validate)(struct tb_nvm *nvm); + int (*write_headers)(struct tb_nvm *nvm); +}; + +/** + * struct tb_nvm_vendor - Vendor to &struct tb_nvm_vendor_ops mapping + * @vendor: Vendor ID + * @vops: Vendor specific NVM operations + * + * Maps vendor ID to NVM vendor operations. If there is no mapping then + * NVM firmware upgrade is disabled for the device. + */ +struct tb_nvm_vendor { + u16 vendor; + const struct tb_nvm_vendor_ops *vops; +}; + +static int intel_switch_nvm_version(struct tb_nvm *nvm) +{ + struct tb_switch *sw = tb_to_switch(nvm->dev); + u32 val, nvm_size, hdr_size; + int ret; + + /* + * If the switch is in safe-mode the only accessible portion of + * the NVM is the non-active one where userspace is expected to + * write new functional NVM. + */ + if (sw->safe_mode) + return 0; + + ret = tb_switch_nvm_read(sw, INTEL_NVM_FLASH_SIZE, &val, sizeof(val)); + if (ret) + return ret; + + hdr_size = sw->generation < 3 ? SZ_8K : SZ_16K; + nvm_size = (SZ_1M << (val & 7)) / 8; + nvm_size = (nvm_size - hdr_size) / 2; + + ret = tb_switch_nvm_read(sw, INTEL_NVM_VERSION, &val, sizeof(val)); + if (ret) + return ret; + + nvm->major = (val >> 16) & 0xff; + nvm->minor = (val >> 8) & 0xff; + nvm->active_size = nvm_size; + + return 0; +} + +static int intel_switch_nvm_validate(struct tb_nvm *nvm) +{ + struct tb_switch *sw = tb_to_switch(nvm->dev); + unsigned int image_size, hdr_size; + u16 ds_size, device_id; + u8 *buf = nvm->buf; + + image_size = nvm->buf_data_size; + + /* + * FARB pointer must point inside the image and must at least + * contain parts of the digital section we will be reading here. + */ + hdr_size = (*(u32 *)buf) & 0xffffff; + if (hdr_size + INTEL_NVM_DEVID + 2 >= image_size) + return -EINVAL; + + /* Digital section start should be aligned to 4k page */ + if (!IS_ALIGNED(hdr_size, SZ_4K)) + return -EINVAL; + + /* + * Read digital section size and check that it also fits inside + * the image. + */ + ds_size = *(u16 *)(buf + hdr_size); + if (ds_size >= image_size) + return -EINVAL; + + if (sw->safe_mode) + return 0; + + /* + * Make sure the device ID in the image matches the one + * we read from the switch config space. + */ + device_id = *(u16 *)(buf + hdr_size + INTEL_NVM_DEVID); + if (device_id != sw->config.device_id) + return -EINVAL; + + /* Skip headers in the image */ + nvm->buf_data_start = buf + hdr_size; + nvm->buf_data_size = image_size - hdr_size; + + return 0; +} + +static int intel_switch_nvm_write_headers(struct tb_nvm *nvm) +{ + struct tb_switch *sw = tb_to_switch(nvm->dev); + + if (sw->generation < 3) { + int ret; + + /* Write CSS headers first */ + ret = dma_port_flash_write(sw->dma_port, + DMA_PORT_CSS_ADDRESS, nvm->buf + INTEL_NVM_CSS, + DMA_PORT_CSS_MAX_SIZE); + if (ret) + return ret; + } + + return 0; +} + +static const struct tb_nvm_vendor_ops intel_switch_nvm_ops = { + .read_version = intel_switch_nvm_version, + .validate = intel_switch_nvm_validate, + .write_headers = intel_switch_nvm_write_headers, +}; + +static int asmedia_switch_nvm_version(struct tb_nvm *nvm) +{ + struct tb_switch *sw = tb_to_switch(nvm->dev); + u32 val; + int ret; + + ret = tb_switch_nvm_read(sw, ASMEDIA_NVM_VERSION, &val, sizeof(val)); + if (ret) + return ret; + + nvm->major = (val << 16) & 0xff0000; + nvm->major |= val & 0x00ff00; + nvm->major |= (val >> 16) & 0x0000ff; + + ret = tb_switch_nvm_read(sw, ASMEDIA_NVM_DATE, &val, sizeof(val)); + if (ret) + return ret; + + nvm->minor = (val << 16) & 0xff0000; + nvm->minor |= val & 0x00ff00; + nvm->minor |= (val >> 16) & 0x0000ff; + + /* ASMedia NVM size is fixed to 512k */ + nvm->active_size = SZ_512K; + + return 0; +} + +static const struct tb_nvm_vendor_ops asmedia_switch_nvm_ops = { + .read_version = asmedia_switch_nvm_version, +}; + +/* Router vendor NVM support table */ +static const struct tb_nvm_vendor switch_nvm_vendors[] = { + { 0x174c, &asmedia_switch_nvm_ops }, + { PCI_VENDOR_ID_INTEL, &intel_switch_nvm_ops }, + { 0x8087, &intel_switch_nvm_ops }, +}; + +static int intel_retimer_nvm_version(struct tb_nvm *nvm) +{ + struct tb_retimer *rt = tb_to_retimer(nvm->dev); + u32 val, nvm_size; + int ret; + + ret = tb_retimer_nvm_read(rt, INTEL_NVM_VERSION, &val, sizeof(val)); + if (ret) + return ret; + + nvm->major = (val >> 16) & 0xff; + nvm->minor = (val >> 8) & 0xff; + + ret = tb_retimer_nvm_read(rt, INTEL_NVM_FLASH_SIZE, &val, sizeof(val)); + if (ret) + return ret; + + nvm_size = (SZ_1M << (val & 7)) / 8; + nvm_size = (nvm_size - SZ_16K) / 2; + nvm->active_size = nvm_size; + + return 0; +} + +static int intel_retimer_nvm_validate(struct tb_nvm *nvm) +{ + struct tb_retimer *rt = tb_to_retimer(nvm->dev); + unsigned int image_size, hdr_size; + u8 *buf = nvm->buf; + u16 ds_size, device; + + image_size = nvm->buf_data_size; + + /* + * FARB pointer must point inside the image and must at least + * contain parts of the digital section we will be reading here. + */ + hdr_size = (*(u32 *)buf) & 0xffffff; + if (hdr_size + INTEL_NVM_DEVID + 2 >= image_size) + return -EINVAL; + + /* Digital section start should be aligned to 4k page */ + if (!IS_ALIGNED(hdr_size, SZ_4K)) + return -EINVAL; + + /* + * Read digital section size and check that it also fits inside + * the image. + */ + ds_size = *(u16 *)(buf + hdr_size); + if (ds_size >= image_size) + return -EINVAL; + + /* + * Make sure the device ID in the image matches the retimer + * hardware. + */ + device = *(u16 *)(buf + hdr_size + INTEL_NVM_DEVID); + if (device != rt->device) + return -EINVAL; + + /* Skip headers in the image */ + nvm->buf_data_start = buf + hdr_size; + nvm->buf_data_size = image_size - hdr_size; + + return 0; +} + +static const struct tb_nvm_vendor_ops intel_retimer_nvm_ops = { + .read_version = intel_retimer_nvm_version, + .validate = intel_retimer_nvm_validate, +}; + +/* Retimer vendor NVM support table */ +static const struct tb_nvm_vendor retimer_nvm_vendors[] = { + { 0x8087, &intel_retimer_nvm_ops }, +}; + /** * tb_nvm_alloc() - Allocate new NVM structure * @dev: Device owning the NVM * * Allocates new NVM structure with unique @id and returns it. In case - * of error returns ERR_PTR(). + * of error returns ERR_PTR(). Specifically returns %-EOPNOTSUPP if the + * NVM format of the @dev is not known by the kernel. */ struct tb_nvm *tb_nvm_alloc(struct device *dev) { + const struct tb_nvm_vendor_ops *vops = NULL; struct tb_nvm *nvm; - int ret; + int ret, i; + + if (tb_is_switch(dev)) { + const struct tb_switch *sw = tb_to_switch(dev); + + for (i = 0; i < ARRAY_SIZE(switch_nvm_vendors); i++) { + const struct tb_nvm_vendor *v = &switch_nvm_vendors[i]; + + if (v->vendor == sw->config.vendor_id) { + vops = v->vops; + break; + } + } + + if (!vops) { + tb_sw_dbg(sw, "router NVM format of vendor %#x unknown\n", + sw->config.vendor_id); + return ERR_PTR(-EOPNOTSUPP); + } + } else if (tb_is_retimer(dev)) { + const struct tb_retimer *rt = tb_to_retimer(dev); + + for (i = 0; i < ARRAY_SIZE(retimer_nvm_vendors); i++) { + const struct tb_nvm_vendor *v = &retimer_nvm_vendors[i]; + + if (v->vendor == rt->vendor) { + vops = v->vops; + break; + } + } + + if (!vops) { + dev_dbg(dev, "retimer NVM format of vendor %#x unknown\n", + rt->vendor); + return ERR_PTR(-EOPNOTSUPP); + } + } else { + return ERR_PTR(-EOPNOTSUPP); + } nvm = kzalloc(sizeof(*nvm), GFP_KERNEL); if (!nvm) @@ -38,14 +334,85 @@ struct tb_nvm *tb_nvm_alloc(struct device *dev) nvm->id = ret; nvm->dev = dev; + nvm->vops = vops; return nvm; } +/** + * tb_nvm_read_version() - Read and populate NVM version + * @nvm: NVM structure + * + * Uses vendor specific means to read out and fill in the existing + * active NVM version. Returns %0 in case of success and negative errno + * otherwise. + */ +int tb_nvm_read_version(struct tb_nvm *nvm) +{ + const struct tb_nvm_vendor_ops *vops = nvm->vops; + + if (vops && vops->read_version) + return vops->read_version(nvm); + + return -EOPNOTSUPP; +} + +/** + * tb_nvm_validate() - Validate new NVM image + * @nvm: NVM structure + * + * Runs vendor specific validation over the new NVM image and if all + * checks pass returns %0. As side effect updates @nvm->buf_data_start + * and @nvm->buf_data_size fields to match the actual data to be written + * to the NVM. + * + * If the validation does not pass then returns negative errno. + */ +int tb_nvm_validate(struct tb_nvm *nvm) +{ + const struct tb_nvm_vendor_ops *vops = nvm->vops; + unsigned int image_size; + u8 *buf = nvm->buf; + + if (!buf) + return -EINVAL; + if (!vops) + return -EOPNOTSUPP; + + /* Just do basic image size checks */ + image_size = nvm->buf_data_size; + if (image_size < NVM_MIN_SIZE || image_size > NVM_MAX_SIZE) + return -EINVAL; + + /* + * Set the default data start in the buffer. The validate method + * below can change this if needed. + */ + nvm->buf_data_start = buf; + + return vops->validate ? vops->validate(nvm) : 0; +} + +/** + * tb_nvm_write_headers() - Write headers before the rest of the image + * @nvm: NVM structure + * + * If the vendor NVM format requires writing headers before the rest of + * the image, this function does that. Can be called even if the device + * does not need this. + * + * Returns %0 in case of success and negative errno otherwise. + */ +int tb_nvm_write_headers(struct tb_nvm *nvm) +{ + const struct tb_nvm_vendor_ops *vops = nvm->vops; + + return vops->write_headers ? vops->write_headers(nvm) : 0; +} + /** * tb_nvm_add_active() - Adds active NVMem device to NVM * @nvm: NVM structure - * @size: Size of the active NVM in bytes * @reg_read: Pointer to the function to read the NVM (passed directly to the * NVMem device) * @@ -54,7 +421,7 @@ struct tb_nvm *tb_nvm_alloc(struct device *dev) * needed. The first parameter passed to @reg_read is @nvm structure. * Returns %0 in success and negative errno otherwise. */ -int tb_nvm_add_active(struct tb_nvm *nvm, size_t size, nvmem_reg_read_t reg_read) +int tb_nvm_add_active(struct tb_nvm *nvm, nvmem_reg_read_t reg_read) { struct nvmem_config config; struct nvmem_device *nvmem; @@ -67,7 +434,7 @@ int tb_nvm_add_active(struct tb_nvm *nvm, size_t size, nvmem_reg_read_t reg_read config.id = nvm->id; config.stride = 4; config.word_size = 4; - config.size = size; + config.size = nvm->active_size; config.dev = nvm->dev; config.owner = THIS_MODULE; config.priv = nvm; @@ -109,17 +476,17 @@ int tb_nvm_write_buf(struct tb_nvm *nvm, unsigned int offset, void *val, /** * tb_nvm_add_non_active() - Adds non-active NVMem device to NVM * @nvm: NVM structure - * @size: Size of the non-active NVM in bytes * @reg_write: Pointer to the function to write the NVM (passed directly * to the NVMem device) * * Registers new non-active NVmem device for @nvm. The @reg_write is called * directly from NVMem so it must handle possible concurrent access if * needed. The first parameter passed to @reg_write is @nvm structure. + * The size of the NVMem device is set to %NVM_MAX_SIZE. + * * Returns %0 in success and negative errno otherwise. */ -int tb_nvm_add_non_active(struct tb_nvm *nvm, size_t size, - nvmem_reg_write_t reg_write) +int tb_nvm_add_non_active(struct tb_nvm *nvm, nvmem_reg_write_t reg_write) { struct nvmem_config config; struct nvmem_device *nvmem; @@ -132,7 +499,7 @@ int tb_nvm_add_non_active(struct tb_nvm *nvm, size_t size, config.id = nvm->id; config.stride = 4; config.word_size = 4; - config.size = size; + config.size = NVM_MAX_SIZE; config.dev = nvm->dev; config.owner = THIS_MODULE; config.priv = nvm; diff --git a/drivers/thunderbolt/retimer.c b/drivers/thunderbolt/retimer.c index 8c29bd556ae040710469081fbdf3d7dc8c06984a..81252e31014a18aa907c158572e6b50108a12cc7 100644 --- a/drivers/thunderbolt/retimer.c +++ b/drivers/thunderbolt/retimer.c @@ -16,8 +16,23 @@ #define TB_MAX_RETIMER_INDEX 6 -static int tb_retimer_nvm_read(void *priv, unsigned int offset, void *val, - size_t bytes) +/** + * tb_retimer_nvm_read() - Read contents of retimer NVM + * @rt: Retimer device + * @address: NVM address (in bytes) to start reading + * @buf: Data read from NVM is stored here + * @size: Number of bytes to read + * + * Reads retimer NVM and copies the contents to @buf. Returns %0 if the + * read was successful and negative errno in case of failure. + */ +int tb_retimer_nvm_read(struct tb_retimer *rt, unsigned int address, void *buf, + size_t size) +{ + return usb4_port_retimer_nvm_read(rt->port, rt->index, address, buf, size); +} + +static int nvm_read(void *priv, unsigned int offset, void *val, size_t bytes) { struct tb_nvm *nvm = priv; struct tb_retimer *rt = tb_to_retimer(nvm->dev); @@ -30,7 +45,7 @@ static int tb_retimer_nvm_read(void *priv, unsigned int offset, void *val, goto out; } - ret = usb4_port_retimer_nvm_read(rt->port, rt->index, offset, val, bytes); + ret = tb_retimer_nvm_read(rt, offset, val, bytes); mutex_unlock(&rt->tb->lock); out: @@ -40,8 +55,7 @@ out: return ret; } -static int tb_retimer_nvm_write(void *priv, unsigned int offset, void *val, - size_t bytes) +static int nvm_write(void *priv, unsigned int offset, void *val, size_t bytes) { struct tb_nvm *nvm = priv; struct tb_retimer *rt = tb_to_retimer(nvm->dev); @@ -59,34 +73,23 @@ static int tb_retimer_nvm_write(void *priv, unsigned int offset, void *val, static int tb_retimer_nvm_add(struct tb_retimer *rt) { struct tb_nvm *nvm; - u32 val, nvm_size; int ret; nvm = tb_nvm_alloc(&rt->dev); - if (IS_ERR(nvm)) - return PTR_ERR(nvm); - - ret = usb4_port_retimer_nvm_read(rt->port, rt->index, NVM_VERSION, &val, - sizeof(val)); - if (ret) + if (IS_ERR(nvm)) { + ret = PTR_ERR(nvm) == -EOPNOTSUPP ? 0 : PTR_ERR(nvm); goto err_nvm; + } - nvm->major = val >> 16; - nvm->minor = val >> 8; - - ret = usb4_port_retimer_nvm_read(rt->port, rt->index, NVM_FLASH_SIZE, - &val, sizeof(val)); + ret = tb_nvm_read_version(nvm); if (ret) goto err_nvm; - nvm_size = (SZ_1M << (val & 7)) / 8; - nvm_size = (nvm_size - SZ_16K) / 2; - - ret = tb_nvm_add_active(nvm, nvm_size, tb_retimer_nvm_read); + ret = tb_nvm_add_active(nvm, nvm_read); if (ret) goto err_nvm; - ret = tb_nvm_add_non_active(nvm, NVM_MAX_SIZE, tb_retimer_nvm_write); + ret = tb_nvm_add_non_active(nvm, nvm_write); if (ret) goto err_nvm; @@ -94,59 +97,33 @@ static int tb_retimer_nvm_add(struct tb_retimer *rt) return 0; err_nvm: - tb_nvm_free(nvm); + dev_dbg(&rt->dev, "NVM upgrade disabled\n"); + if (!IS_ERR(nvm)) + tb_nvm_free(nvm); + return ret; } static int tb_retimer_nvm_validate_and_write(struct tb_retimer *rt) { - unsigned int image_size, hdr_size; - const u8 *buf = rt->nvm->buf; - u16 ds_size, device; + unsigned int image_size; + const u8 *buf; int ret; - image_size = rt->nvm->buf_data_size; - if (image_size < NVM_MIN_SIZE || image_size > NVM_MAX_SIZE) - return -EINVAL; - - /* - * FARB pointer must point inside the image and must at least - * contain parts of the digital section we will be reading here. - */ - hdr_size = (*(u32 *)buf) & 0xffffff; - if (hdr_size + NVM_DEVID + 2 >= image_size) - return -EINVAL; - - /* Digital section start should be aligned to 4k page */ - if (!IS_ALIGNED(hdr_size, SZ_4K)) - return -EINVAL; - - /* - * Read digital section size and check that it also fits inside - * the image. - */ - ds_size = *(u16 *)(buf + hdr_size); - if (ds_size >= image_size) - return -EINVAL; - - /* - * Make sure the device ID in the image matches the retimer - * hardware. - */ - device = *(u16 *)(buf + hdr_size + NVM_DEVID); - if (device != rt->device) - return -EINVAL; + ret = tb_nvm_validate(rt->nvm); + if (ret) + return ret; - /* Skip headers in the image */ - buf += hdr_size; - image_size -= hdr_size; + buf = rt->nvm->buf_data_start; + image_size = rt->nvm->buf_data_size; ret = usb4_port_retimer_nvm_write(rt->port, rt->index, 0, buf, image_size); - if (!ret) - rt->nvm->flushed = true; + if (ret) + return ret; - return ret; + rt->nvm->flushed = true; + return 0; } static int tb_retimer_nvm_authenticate(struct tb_retimer *rt, bool auth_only) @@ -185,7 +162,7 @@ static ssize_t device_show(struct device *dev, struct device_attribute *attr, { struct tb_retimer *rt = tb_to_retimer(dev); - return sprintf(buf, "%#x\n", rt->device); + return sysfs_emit(buf, "%#x\n", rt->device); } static DEVICE_ATTR_RO(device); @@ -200,8 +177,10 @@ static ssize_t nvm_authenticate_show(struct device *dev, if (!rt->nvm) ret = -EAGAIN; + else if (rt->no_nvm_upgrade) + ret = -EOPNOTSUPP; else - ret = sprintf(buf, "%#x\n", rt->auth_status); + ret = sysfs_emit(buf, "%#x\n", rt->auth_status); mutex_unlock(&rt->tb->lock); @@ -276,7 +255,7 @@ static ssize_t nvm_version_show(struct device *dev, if (!rt->nvm) ret = -EAGAIN; else - ret = sprintf(buf, "%x.%x\n", rt->nvm->major, rt->nvm->minor); + ret = sysfs_emit(buf, "%x.%x\n", rt->nvm->major, rt->nvm->minor); mutex_unlock(&rt->tb->lock); return ret; @@ -288,7 +267,7 @@ static ssize_t vendor_show(struct device *dev, struct device_attribute *attr, { struct tb_retimer *rt = tb_to_retimer(dev); - return sprintf(buf, "%#x\n", rt->vendor); + return sysfs_emit(buf, "%#x\n", rt->vendor); } static DEVICE_ATTR_RO(vendor); diff --git a/drivers/thunderbolt/sb_regs.h b/drivers/thunderbolt/sb_regs.h index bda889ff3bda5a2702b10ca9125c1ed6d1429145..5185cf3e4d978f0216ea8f8cd3327c3fc0be32ef 100644 --- a/drivers/thunderbolt/sb_regs.h +++ b/drivers/thunderbolt/sb_regs.h @@ -26,10 +26,68 @@ enum usb4_sb_opcode { USB4_SB_OPCODE_NVM_BLOCK_WRITE = 0x574b4c42, /* "BLKW" */ USB4_SB_OPCODE_NVM_AUTH_WRITE = 0x48545541, /* "AUTH" */ USB4_SB_OPCODE_NVM_READ = 0x52524641, /* "AFRR" */ + USB4_SB_OPCODE_READ_LANE_MARGINING_CAP = 0x50434452, /* "RDCP" */ + USB4_SB_OPCODE_RUN_HW_LANE_MARGINING = 0x474d4852, /* "RHMG" */ + USB4_SB_OPCODE_RUN_SW_LANE_MARGINING = 0x474d5352, /* "RSMG" */ + USB4_SB_OPCODE_READ_SW_MARGIN_ERR = 0x57534452, /* "RDSW" */ }; #define USB4_SB_METADATA 0x09 #define USB4_SB_METADATA_NVM_AUTH_WRITE_MASK GENMASK(5, 0) #define USB4_SB_DATA 0x12 +/* USB4_SB_OPCODE_READ_LANE_MARGINING_CAP */ +#define USB4_MARGIN_CAP_0_MODES_HW BIT(0) +#define USB4_MARGIN_CAP_0_MODES_SW BIT(1) +#define USB4_MARGIN_CAP_0_2_LANES BIT(2) +#define USB4_MARGIN_CAP_0_VOLTAGE_INDP_MASK GENMASK(4, 3) +#define USB4_MARGIN_CAP_0_VOLTAGE_INDP_SHIFT 3 +#define USB4_MARGIN_CAP_0_VOLTAGE_MIN 0x0 +#define USB4_MARGIN_CAP_0_VOLTAGE_HL 0x1 +#define USB4_MARGIN_CAP_0_VOLTAGE_BOTH 0x2 +#define USB4_MARGIN_CAP_0_TIME BIT(5) +#define USB4_MARGIN_CAP_0_VOLTAGE_STEPS_MASK GENMASK(12, 6) +#define USB4_MARGIN_CAP_0_VOLTAGE_STEPS_SHIFT 6 +#define USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_MASK GENMASK(18, 13) +#define USB4_MARGIN_CAP_0_MAX_VOLTAGE_OFFSET_SHIFT 13 +#define USB4_MARGIN_CAP_1_TIME_DESTR BIT(8) +#define USB4_MARGIN_CAP_1_TIME_INDP_MASK GENMASK(10, 9) +#define USB4_MARGIN_CAP_1_TIME_INDP_SHIFT 9 +#define USB4_MARGIN_CAP_1_TIME_MIN 0x0 +#define USB4_MARGIN_CAP_1_TIME_LR 0x1 +#define USB4_MARGIN_CAP_1_TIME_BOTH 0x2 +#define USB4_MARGIN_CAP_1_TIME_STEPS_MASK GENMASK(15, 11) +#define USB4_MARGIN_CAP_1_TIME_STEPS_SHIFT 11 +#define USB4_MARGIN_CAP_1_TIME_OFFSET_MASK GENMASK(20, 16) +#define USB4_MARGIN_CAP_1_TIME_OFFSET_SHIFT 16 +#define USB4_MARGIN_CAP_1_MIN_BER_MASK GENMASK(25, 21) +#define USB4_MARGIN_CAP_1_MIN_BER_SHIFT 21 +#define USB4_MARGIN_CAP_1_MAX_BER_MASK GENMASK(30, 26) +#define USB4_MARGIN_CAP_1_MAX_BER_SHIFT 26 +#define USB4_MARGIN_CAP_1_MAX_BER_SHIFT 26 + +/* USB4_SB_OPCODE_RUN_HW_LANE_MARGINING */ +#define USB4_MARGIN_HW_TIME BIT(3) +#define USB4_MARGIN_HW_RH BIT(4) +#define USB4_MARGIN_HW_BER_MASK GENMASK(9, 5) +#define USB4_MARGIN_HW_BER_SHIFT 5 + +/* Applicable to all margin values */ +#define USB4_MARGIN_HW_RES_1_MARGIN_MASK GENMASK(6, 0) +#define USB4_MARGIN_HW_RES_1_EXCEEDS BIT(7) +/* Different lane margin shifts */ +#define USB4_MARGIN_HW_RES_1_L0_LL_MARGIN_SHIFT 8 +#define USB4_MARGIN_HW_RES_1_L1_RH_MARGIN_SHIFT 16 +#define USB4_MARGIN_HW_RES_1_L1_LL_MARGIN_SHIFT 24 + +/* USB4_SB_OPCODE_RUN_SW_LANE_MARGINING */ +#define USB4_MARGIN_SW_TIME BIT(3) +#define USB4_MARGIN_SW_RH BIT(4) +#define USB4_MARGIN_SW_COUNTER_MASK GENMASK(14, 13) +#define USB4_MARGIN_SW_COUNTER_SHIFT 13 +#define USB4_MARGIN_SW_COUNTER_NOP 0x0 +#define USB4_MARGIN_SW_COUNTER_CLEAR 0x1 +#define USB4_MARGIN_SW_COUNTER_START 0x2 +#define USB4_MARGIN_SW_COUNTER_STOP 0x3 + #endif diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 244f8cd38b25917389b05d0b197f6eaab1db3f85..60da5c23ccaf437c568066620121219f470aa69b 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -19,8 +19,6 @@ /* Switch NVM support */ -#define NVM_CSS 0x10 - struct nvm_auth_status { struct list_head list; uuid_t uuid; @@ -102,70 +100,30 @@ static void nvm_clear_auth_status(const struct tb_switch *sw) static int nvm_validate_and_write(struct tb_switch *sw) { - unsigned int image_size, hdr_size; - const u8 *buf = sw->nvm->buf; - u16 ds_size; + unsigned int image_size; + const u8 *buf; int ret; - if (!buf) - return -EINVAL; - - image_size = sw->nvm->buf_data_size; - if (image_size < NVM_MIN_SIZE || image_size > NVM_MAX_SIZE) - return -EINVAL; - - /* - * FARB pointer must point inside the image and must at least - * contain parts of the digital section we will be reading here. - */ - hdr_size = (*(u32 *)buf) & 0xffffff; - if (hdr_size + NVM_DEVID + 2 >= image_size) - return -EINVAL; - - /* Digital section start should be aligned to 4k page */ - if (!IS_ALIGNED(hdr_size, SZ_4K)) - return -EINVAL; - - /* - * Read digital section size and check that it also fits inside - * the image. - */ - ds_size = *(u16 *)(buf + hdr_size); - if (ds_size >= image_size) - return -EINVAL; - - if (!sw->safe_mode) { - u16 device_id; + ret = tb_nvm_validate(sw->nvm); + if (ret) + return ret; - /* - * Make sure the device ID in the image matches the one - * we read from the switch config space. - */ - device_id = *(u16 *)(buf + hdr_size + NVM_DEVID); - if (device_id != sw->config.device_id) - return -EINVAL; - - if (sw->generation < 3) { - /* Write CSS headers first */ - ret = dma_port_flash_write(sw->dma_port, - DMA_PORT_CSS_ADDRESS, buf + NVM_CSS, - DMA_PORT_CSS_MAX_SIZE); - if (ret) - return ret; - } + ret = tb_nvm_write_headers(sw->nvm); + if (ret) + return ret; - /* Skip headers in the image */ - buf += hdr_size; - image_size -= hdr_size; - } + buf = sw->nvm->buf_data_start; + image_size = sw->nvm->buf_data_size; if (tb_switch_is_usb4(sw)) ret = usb4_switch_nvm_write(sw, 0, buf, image_size); else ret = dma_port_flash_write(sw->dma_port, 0, buf, image_size); - if (!ret) - sw->nvm->flushed = true; - return ret; + if (ret) + return ret; + + sw->nvm->flushed = true; + return 0; } static int nvm_authenticate_host_dma_port(struct tb_switch *sw) @@ -300,14 +258,6 @@ static inline bool nvm_upgradeable(struct tb_switch *sw) return nvm_readable(sw); } -static inline int nvm_read(struct tb_switch *sw, unsigned int address, - void *buf, size_t size) -{ - if (tb_switch_is_usb4(sw)) - return usb4_switch_nvm_read(sw, address, buf, size); - return dma_port_flash_read(sw->dma_port, address, buf, size); -} - static int nvm_authenticate(struct tb_switch *sw, bool auth_only) { int ret; @@ -335,8 +285,26 @@ static int nvm_authenticate(struct tb_switch *sw, bool auth_only) return ret; } -static int tb_switch_nvm_read(void *priv, unsigned int offset, void *val, - size_t bytes) +/** + * tb_switch_nvm_read() - Read router NVM + * @sw: Router whose NVM to read + * @address: Start address on the NVM + * @buf: Buffer where the read data is copied + * @size: Size of the buffer in bytes + * + * Reads from router NVM and returns the requested data in @buf. Locking + * is up to the caller. Returns %0 in success and negative errno in case + * of failure. + */ +int tb_switch_nvm_read(struct tb_switch *sw, unsigned int address, void *buf, + size_t size) +{ + if (tb_switch_is_usb4(sw)) + return usb4_switch_nvm_read(sw, address, buf, size); + return dma_port_flash_read(sw->dma_port, address, buf, size); +} + +static int nvm_read(void *priv, unsigned int offset, void *val, size_t bytes) { struct tb_nvm *nvm = priv; struct tb_switch *sw = tb_to_switch(nvm->dev); @@ -349,7 +317,7 @@ static int tb_switch_nvm_read(void *priv, unsigned int offset, void *val, goto out; } - ret = nvm_read(sw, offset, val, bytes); + ret = tb_switch_nvm_read(sw, offset, val, bytes); mutex_unlock(&sw->tb->lock); out: @@ -359,8 +327,7 @@ out: return ret; } -static int tb_switch_nvm_write(void *priv, unsigned int offset, void *val, - size_t bytes) +static int nvm_write(void *priv, unsigned int offset, void *val, size_t bytes) { struct tb_nvm *nvm = priv; struct tb_switch *sw = tb_to_switch(nvm->dev); @@ -384,28 +351,20 @@ static int tb_switch_nvm_write(void *priv, unsigned int offset, void *val, static int tb_switch_nvm_add(struct tb_switch *sw) { struct tb_nvm *nvm; - u32 val; int ret; if (!nvm_readable(sw)) return 0; - /* - * The NVM format of non-Intel hardware is not known so - * currently restrict NVM upgrade for Intel hardware. We may - * relax this in the future when we learn other NVM formats. - */ - if (sw->config.vendor_id != PCI_VENDOR_ID_INTEL && - sw->config.vendor_id != 0x8087) { - dev_info(&sw->dev, - "NVM format of vendor %#x is not known, disabling NVM upgrade\n", - sw->config.vendor_id); - return 0; + nvm = tb_nvm_alloc(&sw->dev); + if (IS_ERR(nvm)) { + ret = PTR_ERR(nvm) == -EOPNOTSUPP ? 0 : PTR_ERR(nvm); + goto err_nvm; } - nvm = tb_nvm_alloc(&sw->dev); - if (IS_ERR(nvm)) - return PTR_ERR(nvm); + ret = tb_nvm_read_version(nvm); + if (ret) + goto err_nvm; /* * If the switch is in safe-mode the only accessible portion of @@ -413,31 +372,13 @@ static int tb_switch_nvm_add(struct tb_switch *sw) * write new functional NVM. */ if (!sw->safe_mode) { - u32 nvm_size, hdr_size; - - ret = nvm_read(sw, NVM_FLASH_SIZE, &val, sizeof(val)); - if (ret) - goto err_nvm; - - hdr_size = sw->generation < 3 ? SZ_8K : SZ_16K; - nvm_size = (SZ_1M << (val & 7)) / 8; - nvm_size = (nvm_size - hdr_size) / 2; - - ret = nvm_read(sw, NVM_VERSION, &val, sizeof(val)); - if (ret) - goto err_nvm; - - nvm->major = val >> 16; - nvm->minor = val >> 8; - - ret = tb_nvm_add_active(nvm, nvm_size, tb_switch_nvm_read); + ret = tb_nvm_add_active(nvm, nvm_read); if (ret) goto err_nvm; } if (!sw->no_nvm_upgrade) { - ret = tb_nvm_add_non_active(nvm, NVM_MAX_SIZE, - tb_switch_nvm_write); + ret = tb_nvm_add_non_active(nvm, nvm_write); if (ret) goto err_nvm; } @@ -446,7 +387,11 @@ static int tb_switch_nvm_add(struct tb_switch *sw) return 0; err_nvm: - tb_nvm_free(nvm); + tb_sw_dbg(sw, "NVM upgrade disabled\n"); + sw->no_nvm_upgrade = true; + if (!IS_ERR(nvm)) + tb_nvm_free(nvm); + return ret; } @@ -1229,6 +1174,135 @@ int tb_port_update_credits(struct tb_port *port) return tb_port_do_update_credits(port->dual_link_port); } +static int __tb_port_pm_secondary_set(struct tb_port *port, bool secondary) +{ + u32 phy; + int ret; + + ret = tb_port_read(port, &phy, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_1, 1); + if (ret) + return ret; + + if (secondary) + phy |= LANE_ADP_CS_1_PMS; + else + phy &= ~LANE_ADP_CS_1_PMS; + + return tb_port_write(port, &phy, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_1, 1); +} + +static int tb_port_pm_secondary_enable(struct tb_port *port) +{ + return __tb_port_pm_secondary_set(port, true); +} + +static int tb_port_pm_secondary_disable(struct tb_port *port) +{ + return __tb_port_pm_secondary_set(port, false); +} + +/* Called for USB4 or Titan Ridge routers only */ +static bool tb_port_clx_supported(struct tb_port *port, unsigned int clx_mask) +{ + u32 val, mask = 0; + bool ret; + + /* Don't enable CLx in case of two single-lane links */ + if (!port->bonded && port->dual_link_port) + return false; + + /* Don't enable CLx in case of inter-domain link */ + if (port->xdomain) + return false; + + if (tb_switch_is_usb4(port->sw)) { + if (!usb4_port_clx_supported(port)) + return false; + } else if (!tb_lc_is_clx_supported(port)) { + return false; + } + + if (clx_mask & TB_CL1) { + /* CL0s and CL1 are enabled and supported together */ + mask |= LANE_ADP_CS_0_CL0S_SUPPORT | LANE_ADP_CS_0_CL1_SUPPORT; + } + if (clx_mask & TB_CL2) + mask |= LANE_ADP_CS_0_CL2_SUPPORT; + + ret = tb_port_read(port, &val, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_0, 1); + if (ret) + return false; + + return !!(val & mask); +} + +static int __tb_port_clx_set(struct tb_port *port, enum tb_clx clx, bool enable) +{ + u32 phy, mask; + int ret; + + /* CL0s and CL1 are enabled and supported together */ + if (clx == TB_CL1) + mask = LANE_ADP_CS_1_CL0S_ENABLE | LANE_ADP_CS_1_CL1_ENABLE; + else + /* For now we support only CL0s and CL1. Not CL2 */ + return -EOPNOTSUPP; + + ret = tb_port_read(port, &phy, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_1, 1); + if (ret) + return ret; + + if (enable) + phy |= mask; + else + phy &= ~mask; + + return tb_port_write(port, &phy, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_1, 1); +} + +static int tb_port_clx_disable(struct tb_port *port, enum tb_clx clx) +{ + return __tb_port_clx_set(port, clx, false); +} + +static int tb_port_clx_enable(struct tb_port *port, enum tb_clx clx) +{ + return __tb_port_clx_set(port, clx, true); +} + +/** + * tb_port_is_clx_enabled() - Is given CL state enabled + * @port: USB4 port to check + * @clx_mask: Mask of CL states to check + * + * Returns true if any of the given CL states is enabled for @port. + */ +bool tb_port_is_clx_enabled(struct tb_port *port, unsigned int clx_mask) +{ + u32 val, mask = 0; + int ret; + + if (!tb_port_clx_supported(port, clx_mask)) + return false; + + if (clx_mask & TB_CL1) + mask |= LANE_ADP_CS_1_CL0S_ENABLE | LANE_ADP_CS_1_CL1_ENABLE; + if (clx_mask & TB_CL2) + mask |= LANE_ADP_CS_1_CL2_ENABLE; + + ret = tb_port_read(port, &val, TB_CFG_PORT, + port->cap_phy + LANE_ADP_CS_1, 1); + if (ret) + return false; + + return !!(val & mask); +} + static int tb_port_start_lane_initialization(struct tb_port *port) { int ret; @@ -1620,7 +1694,7 @@ static ssize_t authorized_show(struct device *dev, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%u\n", sw->authorized); + return sysfs_emit(buf, "%u\n", sw->authorized); } static int disapprove_switch(struct device *dev, void *not_used) @@ -1730,7 +1804,7 @@ static ssize_t boot_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%u\n", sw->boot); + return sysfs_emit(buf, "%u\n", sw->boot); } static DEVICE_ATTR_RO(boot); @@ -1739,7 +1813,7 @@ static ssize_t device_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%#x\n", sw->device); + return sysfs_emit(buf, "%#x\n", sw->device); } static DEVICE_ATTR_RO(device); @@ -1748,7 +1822,7 @@ device_name_show(struct device *dev, struct device_attribute *attr, char *buf) { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%s\n", sw->device_name ? sw->device_name : ""); + return sysfs_emit(buf, "%s\n", sw->device_name ?: ""); } static DEVICE_ATTR_RO(device_name); @@ -1757,7 +1831,7 @@ generation_show(struct device *dev, struct device_attribute *attr, char *buf) { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%u\n", sw->generation); + return sysfs_emit(buf, "%u\n", sw->generation); } static DEVICE_ATTR_RO(generation); @@ -1771,9 +1845,9 @@ static ssize_t key_show(struct device *dev, struct device_attribute *attr, return restart_syscall(); if (sw->key) - ret = sprintf(buf, "%*phN\n", TB_SWITCH_KEY_SIZE, sw->key); + ret = sysfs_emit(buf, "%*phN\n", TB_SWITCH_KEY_SIZE, sw->key); else - ret = sprintf(buf, "\n"); + ret = sysfs_emit(buf, "\n"); mutex_unlock(&sw->tb->lock); return ret; @@ -1818,7 +1892,7 @@ static ssize_t speed_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%u.0 Gb/s\n", sw->link_speed); + return sysfs_emit(buf, "%u.0 Gb/s\n", sw->link_speed); } /* @@ -1833,7 +1907,7 @@ static ssize_t lanes_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%u\n", sw->link_width); + return sysfs_emit(buf, "%u\n", sw->link_width); } /* @@ -1850,7 +1924,7 @@ static ssize_t nvm_authenticate_show(struct device *dev, u32 status; nvm_get_auth_status(sw, &status); - return sprintf(buf, "%#x\n", status); + return sysfs_emit(buf, "%#x\n", status); } static ssize_t nvm_authenticate_sysfs(struct device *dev, const char *buf, @@ -1866,6 +1940,11 @@ static ssize_t nvm_authenticate_sysfs(struct device *dev, const char *buf, goto exit_rpm; } + if (sw->no_nvm_upgrade) { + ret = -EOPNOTSUPP; + goto exit_unlock; + } + /* If NVMem devices are not yet added */ if (!sw->nvm) { ret = -EAGAIN; @@ -1954,7 +2033,7 @@ static ssize_t nvm_version_show(struct device *dev, else if (!sw->nvm) ret = -EAGAIN; else - ret = sprintf(buf, "%x.%x\n", sw->nvm->major, sw->nvm->minor); + ret = sysfs_emit(buf, "%x.%x\n", sw->nvm->major, sw->nvm->minor); mutex_unlock(&sw->tb->lock); @@ -1967,7 +2046,7 @@ static ssize_t vendor_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%#x\n", sw->vendor); + return sysfs_emit(buf, "%#x\n", sw->vendor); } static DEVICE_ATTR_RO(vendor); @@ -1976,7 +2055,7 @@ vendor_name_show(struct device *dev, struct device_attribute *attr, char *buf) { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%s\n", sw->vendor_name ? sw->vendor_name : ""); + return sysfs_emit(buf, "%s\n", sw->vendor_name ?: ""); } static DEVICE_ATTR_RO(vendor_name); @@ -1985,7 +2064,7 @@ static ssize_t unique_id_show(struct device *dev, struct device_attribute *attr, { struct tb_switch *sw = tb_to_switch(dev); - return sprintf(buf, "%pUb\n", sw->uuid); + return sysfs_emit(buf, "%pUb\n", sw->uuid); } static DEVICE_ATTR_RO(unique_id); @@ -2413,6 +2492,7 @@ int tb_switch_configure(struct tb_switch *sw) * additional capabilities. */ sw->config.cmuv = USB4_VERSION_1_0; + sw->config.plug_events_delay = 0xa; /* Enumerate the switch */ ret = tb_sw_write(sw, (u32 *)&sw->config + 1, TB_CFG_SWITCH, @@ -2821,6 +2901,26 @@ static void tb_switch_credits_init(struct tb_switch *sw) tb_sw_info(sw, "failed to determine preferred buffer allocation, using defaults\n"); } +static int tb_switch_port_hotplug_enable(struct tb_switch *sw) +{ + struct tb_port *port; + + if (tb_switch_is_icm(sw)) + return 0; + + tb_switch_for_each_port(sw, port) { + int res; + + if (!port->cap_usb4) + continue; + + res = usb4_port_hotplug_enable(port); + if (res) + return res; + } + return 0; +} + /** * tb_switch_add() - Add a switch to the domain * @sw: Switch to add @@ -2890,6 +2990,10 @@ int tb_switch_add(struct tb_switch *sw) return ret; } + ret = tb_switch_port_hotplug_enable(sw); + if (ret) + return ret; + ret = device_add(&sw->dev); if (ret) { dev_err(&sw->dev, "failed to add device: %d\n", ret); @@ -3361,35 +3465,6 @@ struct tb_port *tb_switch_find_port(struct tb_switch *sw, return NULL; } -static int __tb_port_pm_secondary_set(struct tb_port *port, bool secondary) -{ - u32 phy; - int ret; - - ret = tb_port_read(port, &phy, TB_CFG_PORT, - port->cap_phy + LANE_ADP_CS_1, 1); - if (ret) - return ret; - - if (secondary) - phy |= LANE_ADP_CS_1_PMS; - else - phy &= ~LANE_ADP_CS_1_PMS; - - return tb_port_write(port, &phy, TB_CFG_PORT, - port->cap_phy + LANE_ADP_CS_1, 1); -} - -static int tb_port_pm_secondary_enable(struct tb_port *port) -{ - return __tb_port_pm_secondary_set(port, true); -} - -static int tb_port_pm_secondary_disable(struct tb_port *port) -{ - return __tb_port_pm_secondary_set(port, false); -} - static int tb_switch_pm_secondary_resolve(struct tb_switch *sw) { struct tb_switch *parent = tb_switch_parent(sw); @@ -3408,83 +3483,6 @@ static int tb_switch_pm_secondary_resolve(struct tb_switch *sw) return tb_port_pm_secondary_disable(down); } -/* Called for USB4 or Titan Ridge routers only */ -static bool tb_port_clx_supported(struct tb_port *port, enum tb_clx clx) -{ - u32 mask, val; - bool ret; - - /* Don't enable CLx in case of two single-lane links */ - if (!port->bonded && port->dual_link_port) - return false; - - /* Don't enable CLx in case of inter-domain link */ - if (port->xdomain) - return false; - - if (tb_switch_is_usb4(port->sw)) { - if (!usb4_port_clx_supported(port)) - return false; - } else if (!tb_lc_is_clx_supported(port)) { - return false; - } - - switch (clx) { - case TB_CL1: - /* CL0s and CL1 are enabled and supported together */ - mask = LANE_ADP_CS_0_CL0S_SUPPORT | LANE_ADP_CS_0_CL1_SUPPORT; - break; - - /* For now we support only CL0s and CL1. Not CL2 */ - case TB_CL2: - default: - return false; - } - - ret = tb_port_read(port, &val, TB_CFG_PORT, - port->cap_phy + LANE_ADP_CS_0, 1); - if (ret) - return false; - - return !!(val & mask); -} - -static int __tb_port_clx_set(struct tb_port *port, enum tb_clx clx, bool enable) -{ - u32 phy, mask; - int ret; - - /* CL0s and CL1 are enabled and supported together */ - if (clx == TB_CL1) - mask = LANE_ADP_CS_1_CL0S_ENABLE | LANE_ADP_CS_1_CL1_ENABLE; - else - /* For now we support only CL0s and CL1. Not CL2 */ - return -EOPNOTSUPP; - - ret = tb_port_read(port, &phy, TB_CFG_PORT, - port->cap_phy + LANE_ADP_CS_1, 1); - if (ret) - return ret; - - if (enable) - phy |= mask; - else - phy &= ~mask; - - return tb_port_write(port, &phy, TB_CFG_PORT, - port->cap_phy + LANE_ADP_CS_1, 1); -} - -static int tb_port_clx_disable(struct tb_port *port, enum tb_clx clx) -{ - return __tb_port_clx_set(port, clx, false); -} - -static int tb_port_clx_enable(struct tb_port *port, enum tb_clx clx) -{ - return __tb_port_clx_set(port, clx, true); -} - static int __tb_switch_enable_clx(struct tb_switch *sw, enum tb_clx clx) { struct tb_switch *parent = tb_switch_parent(sw); @@ -3786,14 +3784,18 @@ int tb_switch_pcie_l1_enable(struct tb_switch *sw) */ int tb_switch_xhci_connect(struct tb_switch *sw) { - bool usb_port1, usb_port3, xhci_port1, xhci_port3; struct tb_port *port1, *port3; int ret; + if (sw->generation != 3) + return 0; + port1 = &sw->ports[1]; port3 = &sw->ports[3]; if (tb_switch_is_alpine_ridge(sw)) { + bool usb_port1, usb_port3, xhci_port1, xhci_port3; + usb_port1 = tb_lc_is_usb_plugged(port1); usb_port3 = tb_lc_is_usb_plugged(port3); xhci_port1 = tb_lc_is_xhci_connected(port1); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 9853f6c7e81d710156c833cb08ba4346219b305f..4628458044270afca638a14d8d22f3403b1a1137 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -105,6 +105,32 @@ static void tb_remove_dp_resources(struct tb_switch *sw) } } +static void tb_discover_dp_resource(struct tb *tb, struct tb_port *port) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_port *p; + + list_for_each_entry(p, &tcm->dp_resources, list) { + if (p == port) + return; + } + + tb_port_dbg(port, "DP %s resource available discovered\n", + tb_port_is_dpin(port) ? "IN" : "OUT"); + list_add_tail(&port->list, &tcm->dp_resources); +} + +static void tb_discover_dp_resources(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + struct tb_tunnel *tunnel; + + list_for_each_entry(tunnel, &tcm->tunnel_list, list) { + if (tb_tunnel_is_dp(tunnel)) + tb_discover_dp_resource(tb, tunnel->dst_port); + } +} + static void tb_switch_discover_tunnels(struct tb_switch *sw, struct list_head *list, bool alloc_hopids) @@ -174,10 +200,10 @@ static void tb_discover_tunnels(struct tb *tb) } } -static int tb_port_configure_xdomain(struct tb_port *port) +static int tb_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd) { if (tb_switch_is_usb4(port->sw)) - return usb4_port_configure_xdomain(port); + return usb4_port_configure_xdomain(port, xd); return tb_lc_configure_xdomain(port); } @@ -212,7 +238,7 @@ static void tb_scan_xdomain(struct tb_port *port) NULL); if (xd) { tb_port_at(route, sw)->xdomain = xd; - tb_port_configure_xdomain(port); + tb_port_configure_xdomain(port, xd); tb_xdomain_add(xd); } } @@ -1416,8 +1442,11 @@ static int tb_start(struct tb *tb) * ICM firmware upgrade needs running firmware and in native * mode that is not available so disable firmware upgrade of the * root switch. + * + * However, USB4 routers support NVM firmware upgrade if they + * implement the necessary router operations. */ - tb->root_switch->no_nvm_upgrade = true; + tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch); /* All USB4 routers support runtime PM */ tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch); @@ -1446,6 +1475,8 @@ static int tb_start(struct tb *tb) tb_scan_switch(tb->root_switch); /* Find out tunnels created by the boot firmware */ tb_discover_tunnels(tb); + /* Add DP resources from the DP tunnels created by the boot firmware */ + tb_discover_dp_resources(tb); /* * If the boot firmware did not create USB 3.x tunnels create them * now for the whole topology. @@ -1516,7 +1547,7 @@ static void tb_restore_children(struct tb_switch *sw) tb_restore_children(port->remote->sw); } else if (port->xdomain) { - tb_port_configure_xdomain(port); + tb_port_configure_xdomain(port, port->xdomain); } } } diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 5db76de40cc1c632de4538a966de1ae162d8889e..f9786976f5ecfc55bd6c5e53452cba97b9c1c9ca 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -23,11 +23,6 @@ #define NVM_MAX_SIZE SZ_512K #define NVM_DATA_DWORDS 16 -/* Intel specific NVM offsets */ -#define NVM_DEVID 0x05 -#define NVM_VERSION 0x08 -#define NVM_FLASH_SIZE 0x45 - /** * struct tb_nvm - Structure holding NVM information * @dev: Owner of the NVM @@ -35,28 +30,35 @@ * @minor: Minor version number of the active NVM portion * @id: Identifier used with both NVM portions * @active: Active portion NVMem device + * @active_size: Size in bytes of the active NVM * @non_active: Non-active portion NVMem device * @buf: Buffer where the NVM image is stored before it is written to * the actual NVM flash device + * @buf_data_start: Where the actual image starts after skipping + * possible headers * @buf_data_size: Number of bytes actually consumed by the new NVM * image * @authenticating: The device is authenticating the new NVM * @flushed: The image has been flushed to the storage area + * @vops: Router vendor specific NVM operations (optional) * * The user of this structure needs to handle serialization of possible * concurrent access. */ struct tb_nvm { struct device *dev; - u8 major; - u8 minor; + u32 major; + u32 minor; int id; struct nvmem_device *active; + size_t active_size; struct nvmem_device *non_active; void *buf; + void *buf_data_start; size_t buf_data_size; bool authenticating; bool flushed; + const struct tb_nvm_vendor_ops *vops; }; enum tb_nvm_write_ops { @@ -113,8 +115,8 @@ struct tb_switch_tmu { enum tb_clx { TB_CLX_DISABLE, /* CL0s and CL1 are enabled and supported together */ - TB_CL1, - TB_CL2, + TB_CL1 = BIT(0), + TB_CL2 = BIT(1), }; /** @@ -279,12 +281,16 @@ struct tb_port { * @can_offline: Does the port have necessary platform support to moved * it into offline mode and back * @offline: The port is currently in offline mode + * @margining: Pointer to margining structure if enabled */ struct usb4_port { struct device dev; struct tb_port *port; bool can_offline; bool offline; +#ifdef CONFIG_USB4_DEBUGFS_MARGINING + struct tb_margining *margining; +#endif }; /** @@ -296,6 +302,7 @@ struct usb4_port { * @device: Device ID of the retimer * @port: Pointer to the lane 0 adapter * @nvm: Pointer to the NVM if the retimer has one (%NULL otherwise) + * @no_nvm_upgrade: Prevent NVM upgrade of this retimer * @auth_status: Status of last NVM authentication */ struct tb_retimer { @@ -306,6 +313,7 @@ struct tb_retimer { u32 device; struct tb_port *port; struct tb_nvm *nvm; + bool no_nvm_upgrade; u32 auth_status; }; @@ -737,11 +745,13 @@ static inline void tb_domain_put(struct tb *tb) } struct tb_nvm *tb_nvm_alloc(struct device *dev); -int tb_nvm_add_active(struct tb_nvm *nvm, size_t size, nvmem_reg_read_t reg_read); +int tb_nvm_read_version(struct tb_nvm *nvm); +int tb_nvm_validate(struct tb_nvm *nvm); +int tb_nvm_write_headers(struct tb_nvm *nvm); +int tb_nvm_add_active(struct tb_nvm *nvm, nvmem_reg_read_t reg_read); int tb_nvm_write_buf(struct tb_nvm *nvm, unsigned int offset, void *val, size_t bytes); -int tb_nvm_add_non_active(struct tb_nvm *nvm, size_t size, - nvmem_reg_write_t reg_write); +int tb_nvm_add_non_active(struct tb_nvm *nvm, nvmem_reg_write_t reg_write); void tb_nvm_free(struct tb_nvm *nvm); void tb_nvm_exit(void); @@ -755,6 +765,8 @@ int tb_nvm_write_data(unsigned int address, const void *buf, size_t size, unsigned int retries, write_block_fn write_next_block, void *write_block_data); +int tb_switch_nvm_read(struct tb_switch *sw, unsigned int address, void *buf, + size_t size); struct tb_switch *tb_switch_alloc(struct tb *tb, struct device *parent, u64 route); struct tb_switch *tb_switch_alloc_safe_mode(struct tb *tb, @@ -1035,6 +1047,7 @@ void tb_port_lane_bonding_disable(struct tb_port *port); int tb_port_wait_for_link_width(struct tb_port *port, int width, int timeout_msec); int tb_port_update_credits(struct tb_port *port); +bool tb_port_is_clx_enabled(struct tb_port *port, enum tb_clx clx); int tb_switch_find_vse_cap(struct tb_switch *sw, enum tb_switch_vse_cap vsec); int tb_switch_find_cap(struct tb_switch *sw, enum tb_switch_cap cap); @@ -1132,6 +1145,13 @@ void tb_xdomain_remove(struct tb_xdomain *xd); struct tb_xdomain *tb_xdomain_find_by_link_depth(struct tb *tb, u8 link, u8 depth); +static inline struct tb_switch *tb_xdomain_parent(struct tb_xdomain *xd) +{ + return tb_to_switch(xd->dev.parent); +} + +int tb_retimer_nvm_read(struct tb_retimer *rt, unsigned int address, void *buf, + size_t size); int tb_retimer_scan(struct tb_port *port, bool add); void tb_retimer_remove_all(struct tb_port *port); @@ -1174,14 +1194,22 @@ int usb4_switch_add_ports(struct tb_switch *sw); void usb4_switch_remove_ports(struct tb_switch *sw); int usb4_port_unlock(struct tb_port *port); +int usb4_port_hotplug_enable(struct tb_port *port); int usb4_port_configure(struct tb_port *port); void usb4_port_unconfigure(struct tb_port *port); -int usb4_port_configure_xdomain(struct tb_port *port); +int usb4_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd); void usb4_port_unconfigure_xdomain(struct tb_port *port); int usb4_port_router_offline(struct tb_port *port); int usb4_port_router_online(struct tb_port *port); int usb4_port_enumerate_retimers(struct tb_port *port); bool usb4_port_clx_supported(struct tb_port *port); +int usb4_port_margining_caps(struct tb_port *port, u32 *caps); +int usb4_port_hw_margin(struct tb_port *port, unsigned int lanes, + unsigned int ber_level, bool timing, bool right_high, + u32 *results); +int usb4_port_sw_margin(struct tb_port *port, unsigned int lanes, bool timing, + bool right_high, u32 counter); +int usb4_port_sw_margin_errors(struct tb_port *port, u32 *errors); int usb4_port_retimer_set_inbound_sbtx(struct tb_port *port, u8 index); int usb4_port_retimer_read(struct tb_port *port, u8 index, u8 reg, void *buf, @@ -1264,6 +1292,8 @@ void tb_debugfs_init(void); void tb_debugfs_exit(void); void tb_switch_debugfs_init(struct tb_switch *sw); void tb_switch_debugfs_remove(struct tb_switch *sw); +void tb_xdomain_debugfs_init(struct tb_xdomain *xd); +void tb_xdomain_debugfs_remove(struct tb_xdomain *xd); void tb_service_debugfs_init(struct tb_service *svc); void tb_service_debugfs_remove(struct tb_service *svc); #else @@ -1271,6 +1301,8 @@ static inline void tb_debugfs_init(void) { } static inline void tb_debugfs_exit(void) { } static inline void tb_switch_debugfs_init(struct tb_switch *sw) { } static inline void tb_switch_debugfs_remove(struct tb_switch *sw) { } +static inline void tb_xdomain_debugfs_init(struct tb_xdomain *xd) { } +static inline void tb_xdomain_debugfs_remove(struct tb_xdomain *xd) { } static inline void tb_service_debugfs_init(struct tb_service *svc) { } static inline void tb_service_debugfs_remove(struct tb_service *svc) { } #endif diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 16605411038830935e12920a2231bf5e72e0e200..86319dca0f8ca90585879dbd6e7b3f4c489f85d0 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -308,6 +308,7 @@ struct tb_regs_port_header { #define ADP_CS_5 0x05 #define ADP_CS_5_LCA_MASK GENMASK(28, 22) #define ADP_CS_5_LCA_SHIFT 22 +#define ADP_CS_5_DHP BIT(31) /* TMU adapter registers */ #define TMU_ADP_CS_3 0x03 @@ -324,6 +325,7 @@ struct tb_regs_port_header { #define LANE_ADP_CS_0_SUPPORTED_WIDTH_DUAL 0x2 #define LANE_ADP_CS_0_CL0S_SUPPORT BIT(26) #define LANE_ADP_CS_0_CL1_SUPPORT BIT(27) +#define LANE_ADP_CS_0_CL2_SUPPORT BIT(28) #define LANE_ADP_CS_1 0x01 #define LANE_ADP_CS_1_TARGET_SPEED_MASK GENMASK(3, 0) #define LANE_ADP_CS_1_TARGET_SPEED_GEN3 0xc @@ -333,6 +335,7 @@ struct tb_regs_port_header { #define LANE_ADP_CS_1_TARGET_WIDTH_DUAL 0x3 #define LANE_ADP_CS_1_CL0S_ENABLE BIT(10) #define LANE_ADP_CS_1_CL1_ENABLE BIT(11) +#define LANE_ADP_CS_1_CL2_ENABLE BIT(12) #define LANE_ADP_CS_1_LD BIT(14) #define LANE_ADP_CS_1_LB BIT(15) #define LANE_ADP_CS_1_CURRENT_SPEED_MASK GENMASK(19, 16) diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 3a2e7126db9dc894b65b5a92642b9a0cd0a895bd..f986854aa207ade99a87afa2e205269d15176e0d 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -1046,6 +1046,26 @@ int usb4_port_unlock(struct tb_port *port) return tb_port_write(port, &val, TB_CFG_PORT, ADP_CS_4, 1); } +/** + * usb4_port_hotplug_enable() - Enables hotplug for a port + * @port: USB4 port to operate on + * + * Enables hot plug events on a given port. This is only intended + * to be used on lane, DP-IN, and DP-OUT adapters. + */ +int usb4_port_hotplug_enable(struct tb_port *port) +{ + int ret; + u32 val; + + ret = tb_port_read(port, &val, TB_CFG_PORT, ADP_CS_5, 1); + if (ret) + return ret; + + val &= ~ADP_CS_5_DHP; + return tb_port_write(port, &val, TB_CFG_PORT, ADP_CS_5, 1); +} + static int usb4_port_set_configured(struct tb_port *port, bool configured) { int ret; @@ -1115,12 +1135,14 @@ static int usb4_set_xdomain_configured(struct tb_port *port, bool configured) /** * usb4_port_configure_xdomain() - Configure port for XDomain * @port: USB4 port connected to another host + * @xd: XDomain that is connected to the port * - * Marks the USB4 port as being connected to another host. Returns %0 in - * success and negative errno in failure. + * Marks the USB4 port as being connected to another host and updates + * the link type. Returns %0 in success and negative errno in failure. */ -int usb4_port_configure_xdomain(struct tb_port *port) +int usb4_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd) { + xd->link_usb4 = link_is_usb4(port); return usb4_set_xdomain_configured(port, true); } @@ -1384,6 +1406,126 @@ bool usb4_port_clx_supported(struct tb_port *port) return !!(val & PORT_CS_18_CPS); } +/** + * usb4_port_margining_caps() - Read USB4 port marginig capabilities + * @port: USB4 port + * @caps: Array with at least two elements to hold the results + * + * Reads the USB4 port lane margining capabilities into @caps. + */ +int usb4_port_margining_caps(struct tb_port *port, u32 *caps) +{ + int ret; + + ret = usb4_port_sb_op(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_OPCODE_READ_LANE_MARGINING_CAP, 500); + if (ret) + return ret; + + return usb4_port_sb_read(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_DATA, caps, sizeof(*caps) * 2); +} + +/** + * usb4_port_hw_margin() - Run hardware lane margining on port + * @port: USB4 port + * @lanes: Which lanes to run (must match the port capabilities). Can be + * %0, %1 or %7. + * @ber_level: BER level contour value + * @timing: Perform timing margining instead of voltage + * @right_high: Use Right/high margin instead of left/low + * @results: Array with at least two elements to hold the results + * + * Runs hardware lane margining on USB4 port and returns the result in + * @results. + */ +int usb4_port_hw_margin(struct tb_port *port, unsigned int lanes, + unsigned int ber_level, bool timing, bool right_high, + u32 *results) +{ + u32 val; + int ret; + + val = lanes; + if (timing) + val |= USB4_MARGIN_HW_TIME; + if (right_high) + val |= USB4_MARGIN_HW_RH; + if (ber_level) + val |= (ber_level << USB4_MARGIN_HW_BER_SHIFT) & + USB4_MARGIN_HW_BER_MASK; + + ret = usb4_port_sb_write(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_METADATA, &val, sizeof(val)); + if (ret) + return ret; + + ret = usb4_port_sb_op(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_OPCODE_RUN_HW_LANE_MARGINING, 2500); + if (ret) + return ret; + + return usb4_port_sb_read(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_DATA, results, sizeof(*results) * 2); +} + +/** + * usb4_port_sw_margin() - Run software lane margining on port + * @port: USB4 port + * @lanes: Which lanes to run (must match the port capabilities). Can be + * %0, %1 or %7. + * @timing: Perform timing margining instead of voltage + * @right_high: Use Right/high margin instead of left/low + * @counter: What to do with the error counter + * + * Runs software lane margining on USB4 port. Read back the error + * counters by calling usb4_port_sw_margin_errors(). Returns %0 in + * success and negative errno otherwise. + */ +int usb4_port_sw_margin(struct tb_port *port, unsigned int lanes, bool timing, + bool right_high, u32 counter) +{ + u32 val; + int ret; + + val = lanes; + if (timing) + val |= USB4_MARGIN_SW_TIME; + if (right_high) + val |= USB4_MARGIN_SW_RH; + val |= (counter << USB4_MARGIN_SW_COUNTER_SHIFT) & + USB4_MARGIN_SW_COUNTER_MASK; + + ret = usb4_port_sb_write(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_METADATA, &val, sizeof(val)); + if (ret) + return ret; + + return usb4_port_sb_op(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_OPCODE_RUN_SW_LANE_MARGINING, 2500); +} + +/** + * usb4_port_sw_margin_errors() - Read the software margining error counters + * @port: USB4 port + * @errors: Error metadata is copied here. + * + * This reads back the software margining error counters from the port. + * Returns %0 in success and negative errno otherwise. + */ +int usb4_port_sw_margin_errors(struct tb_port *port, u32 *errors) +{ + int ret; + + ret = usb4_port_sb_op(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_OPCODE_READ_SW_MARGIN_ERR, 150); + if (ret) + return ret; + + return usb4_port_sb_read(port, USB4_SB_TARGET_ROUTER, 0, + USB4_SB_METADATA, errors, sizeof(*errors)); +} + static inline int usb4_port_retimer_op(struct tb_port *port, u8 index, enum usb4_sb_opcode opcode, int timeout_msec) diff --git a/drivers/thunderbolt/usb4_port.c b/drivers/thunderbolt/usb4_port.c index 6b02945624ee070130281d068a463faab1774d4d..1a30c0a2328655cc8669ccbc062e05f6152f8c5d 100644 --- a/drivers/thunderbolt/usb4_port.c +++ b/drivers/thunderbolt/usb4_port.c @@ -53,6 +53,8 @@ static ssize_t link_show(struct device *dev, struct device_attribute *attr, link = port->sw->link_usb4 ? "usb4" : "tbt"; else if (tb_port_has_remote(port)) link = port->remote->sw->link_usb4 ? "usb4" : "tbt"; + else if (port->xdomain) + link = port->xdomain->link_usb4 ? "usb4" : "tbt"; else link = "none"; diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c index c31c0d94d8b35c1d0c7ff98cbeab479ec3da79fc..f00b2f62d8e3c92841ba8bb6276d583724e7c31d 100644 --- a/drivers/thunderbolt/xdomain.c +++ b/drivers/thunderbolt/xdomain.c @@ -877,7 +877,7 @@ static ssize_t key_show(struct device *dev, struct device_attribute *attr, * It should be null terminated but anything else is pretty much * allowed. */ - return sprintf(buf, "%*pE\n", (int)strlen(svc->key), svc->key); + return sysfs_emit(buf, "%*pE\n", (int)strlen(svc->key), svc->key); } static DEVICE_ATTR_RO(key); @@ -903,7 +903,7 @@ static ssize_t prtcid_show(struct device *dev, struct device_attribute *attr, { struct tb_service *svc = container_of(dev, struct tb_service, dev); - return sprintf(buf, "%u\n", svc->prtcid); + return sysfs_emit(buf, "%u\n", svc->prtcid); } static DEVICE_ATTR_RO(prtcid); @@ -912,7 +912,7 @@ static ssize_t prtcvers_show(struct device *dev, struct device_attribute *attr, { struct tb_service *svc = container_of(dev, struct tb_service, dev); - return sprintf(buf, "%u\n", svc->prtcvers); + return sysfs_emit(buf, "%u\n", svc->prtcvers); } static DEVICE_ATTR_RO(prtcvers); @@ -921,7 +921,7 @@ static ssize_t prtcrevs_show(struct device *dev, struct device_attribute *attr, { struct tb_service *svc = container_of(dev, struct tb_service, dev); - return sprintf(buf, "%u\n", svc->prtcrevs); + return sysfs_emit(buf, "%u\n", svc->prtcrevs); } static DEVICE_ATTR_RO(prtcrevs); @@ -930,7 +930,7 @@ static ssize_t prtcstns_show(struct device *dev, struct device_attribute *attr, { struct tb_service *svc = container_of(dev, struct tb_service, dev); - return sprintf(buf, "0x%08x\n", svc->prtcstns); + return sysfs_emit(buf, "0x%08x\n", svc->prtcstns); } static DEVICE_ATTR_RO(prtcstns); @@ -1131,11 +1131,6 @@ static int populate_properties(struct tb_xdomain *xd, return 0; } -static inline struct tb_switch *tb_xdomain_parent(struct tb_xdomain *xd) -{ - return tb_to_switch(xd->dev.parent); -} - static int tb_xdomain_update_link_attributes(struct tb_xdomain *xd) { bool change = false; @@ -1440,6 +1435,8 @@ static int tb_xdomain_get_properties(struct tb_xdomain *xd) if (xd->vendor_name && xd->device_name) dev_info(&xd->dev, "%s %s\n", xd->vendor_name, xd->device_name); + + tb_xdomain_debugfs_init(xd); } else { kobject_uevent(&xd->dev.kobj, KOBJ_CHANGE); } @@ -1664,7 +1661,7 @@ static ssize_t device_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%#x\n", xd->device); + return sysfs_emit(buf, "%#x\n", xd->device); } static DEVICE_ATTR_RO(device); @@ -1676,7 +1673,7 @@ device_name_show(struct device *dev, struct device_attribute *attr, char *buf) if (mutex_lock_interruptible(&xd->lock)) return -ERESTARTSYS; - ret = sprintf(buf, "%s\n", xd->device_name ? xd->device_name : ""); + ret = sysfs_emit(buf, "%s\n", xd->device_name ?: ""); mutex_unlock(&xd->lock); return ret; @@ -1688,7 +1685,7 @@ static ssize_t maxhopid_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%d\n", xd->remote_max_hopid); + return sysfs_emit(buf, "%d\n", xd->remote_max_hopid); } static DEVICE_ATTR_RO(maxhopid); @@ -1697,7 +1694,7 @@ static ssize_t vendor_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%#x\n", xd->vendor); + return sysfs_emit(buf, "%#x\n", xd->vendor); } static DEVICE_ATTR_RO(vendor); @@ -1709,7 +1706,7 @@ vendor_name_show(struct device *dev, struct device_attribute *attr, char *buf) if (mutex_lock_interruptible(&xd->lock)) return -ERESTARTSYS; - ret = sprintf(buf, "%s\n", xd->vendor_name ? xd->vendor_name : ""); + ret = sysfs_emit(buf, "%s\n", xd->vendor_name ?: ""); mutex_unlock(&xd->lock); return ret; @@ -1721,7 +1718,7 @@ static ssize_t unique_id_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%pUb\n", xd->remote_uuid); + return sysfs_emit(buf, "%pUb\n", xd->remote_uuid); } static DEVICE_ATTR_RO(unique_id); @@ -1730,7 +1727,7 @@ static ssize_t speed_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%u.0 Gb/s\n", xd->link_speed); + return sysfs_emit(buf, "%u.0 Gb/s\n", xd->link_speed); } static DEVICE_ATTR(rx_speed, 0444, speed_show, NULL); @@ -1741,7 +1738,7 @@ static ssize_t lanes_show(struct device *dev, struct device_attribute *attr, { struct tb_xdomain *xd = container_of(dev, struct tb_xdomain, dev); - return sprintf(buf, "%u\n", xd->link_width); + return sysfs_emit(buf, "%u\n", xd->link_width); } static DEVICE_ATTR(rx_lanes, 0444, lanes_show, NULL); @@ -1940,6 +1937,8 @@ static int unregister_service(struct device *dev, void *data) */ void tb_xdomain_remove(struct tb_xdomain *xd) { + tb_xdomain_debugfs_remove(xd); + stop_handshake(xd); device_for_each_child_reverse(&xd->dev, xd, unregister_service); @@ -2438,7 +2437,7 @@ int tb_xdomain_init(void) tb_property_add_immediate(xdomain_property_dir, "deviceid", 0x1); tb_property_add_immediate(xdomain_property_dir, "devicerv", 0x80000100); - xdomain_property_block_gen = prandom_u32(); + xdomain_property_block_gen = get_random_u32(); return 0; } diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index 81e7f64c173934640b2bdb3e50187098176586e4..f52266766df99ffcce2876d683006da583a15fa4 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -94,7 +94,7 @@ static struct tty_driver *serial_driver; static unsigned char current_ctl_bits; static void change_speed(struct tty_struct *tty, struct serial_state *info, - struct ktermios *old); + const struct ktermios *old); static void rs_wait_until_sent(struct tty_struct *tty, int timeout); @@ -566,7 +566,7 @@ static void shutdown(struct tty_struct *tty, struct serial_state *info) * the specified baud rate for a serial port. */ static void change_speed(struct tty_struct *tty, struct serial_state *info, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct tty_port *port = &info->tport; int quot = 0, baud_base, baud; @@ -1169,7 +1169,7 @@ static int rs_ioctl(struct tty_struct *tty, return 0; } -static void rs_set_termios(struct tty_struct *tty, struct ktermios *old_termios) +static void rs_set_termios(struct tty_struct *tty, const struct ktermios *old_termios) { struct serial_state *info = tty->driver_data; unsigned long flags; diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c index 32366caca6623e23ffb19a8d40108fbf6daeb896..7d49a872de48ab89033063492622165180afda78 100644 --- a/drivers/tty/hvc/hvc_iucv.c +++ b/drivers/tty/hvc/hvc_iucv.c @@ -29,7 +29,6 @@ /* General device driver settings */ -#define HVC_IUCV_MAGIC 0xc9e4c3e5 #define MAX_HVC_IUCV_LINES HVC_ALLOC_TTY_ADAPTERS #define MEMPOOL_MIN_NR (PAGE_SIZE / sizeof(struct iucv_tty_buffer)/4) @@ -131,9 +130,9 @@ static struct iucv_handler hvc_iucv_handler = { */ static struct hvc_iucv_private *hvc_iucv_get_private(uint32_t num) { - if ((num < HVC_IUCV_MAGIC) || (num - HVC_IUCV_MAGIC > hvc_iucv_devices)) + if (num > hvc_iucv_devices) return NULL; - return hvc_iucv_table[num - HVC_IUCV_MAGIC]; + return hvc_iucv_table[num]; } /** @@ -1072,8 +1071,8 @@ static int __init hvc_iucv_alloc(int id, unsigned int is_console) priv->is_console = is_console; /* allocate hvc device */ - priv->hvc = hvc_alloc(HVC_IUCV_MAGIC + id, /* PAGE_SIZE */ - HVC_IUCV_MAGIC + id, &hvc_iucv_ops, 256); + priv->hvc = hvc_alloc(id, /* PAGE_SIZE */ + id, &hvc_iucv_ops, 256); if (IS_ERR(priv->hvc)) { rc = PTR_ERR(priv->hvc); goto out_error_hvc; @@ -1371,7 +1370,7 @@ static int __init hvc_iucv_init(void) /* register the first terminal device as console * (must be done before allocating hvc terminal devices) */ - rc = hvc_instantiate(HVC_IUCV_MAGIC, IUCV_HVC_CON_IDX, &hvc_iucv_ops); + rc = hvc_instantiate(0, IUCV_HVC_CON_IDX, &hvc_iucv_ops); if (rc) { pr_err("Registering HVC terminal device as " "Linux console failed\n"); diff --git a/drivers/tty/hvc/hvcs.c b/drivers/tty/hvc/hvcs.c index 9b7e8246a464a98a5bab02f4bdf6c4e071e5c3fc..4ba24963685e2dc8c5a7a7a4e415ef708197066e 100644 --- a/drivers/tty/hvc/hvcs.c +++ b/drivers/tty/hvc/hvcs.c @@ -69,6 +69,7 @@ #include #include #include +#include #include /* @@ -839,7 +840,7 @@ static void hvcs_set_pi(struct hvcs_partner_info *pi, struct hvcs_struct *hvcsd) hvcsd->p_partition_ID = pi->partition_ID; /* copy the null-term char too */ - strlcpy(hvcsd->p_location_code, pi->location_code, + strscpy(hvcsd->p_location_code, pi->location_code, sizeof(hvcsd->p_location_code)); } diff --git a/drivers/tty/moxa.c b/drivers/tty/moxa.c index f3c72ab1476c34ba6d591609aecc97ca843e5e94..35b6fddf0341b16aa340a6c0dceb69966f0bdb92 100644 --- a/drivers/tty/moxa.c +++ b/drivers/tty/moxa.c @@ -491,7 +491,7 @@ static int moxa_write(struct tty_struct *, const unsigned char *, int); static unsigned int moxa_write_room(struct tty_struct *); static void moxa_flush_buffer(struct tty_struct *); static unsigned int moxa_chars_in_buffer(struct tty_struct *); -static void moxa_set_termios(struct tty_struct *, struct ktermios *); +static void moxa_set_termios(struct tty_struct *, const struct ktermios *); static void moxa_stop(struct tty_struct *); static void moxa_start(struct tty_struct *); static void moxa_hangup(struct tty_struct *); @@ -499,7 +499,7 @@ static int moxa_tiocmget(struct tty_struct *tty); static int moxa_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); static void moxa_poll(struct timer_list *); -static void moxa_set_tty_param(struct tty_struct *, struct ktermios *); +static void moxa_set_tty_param(struct tty_struct *, const struct ktermios *); static void moxa_shutdown(struct tty_port *); static int moxa_carrier_raised(struct tty_port *); static void moxa_dtr_rts(struct tty_port *, int); @@ -1602,7 +1602,7 @@ static int moxa_tiocmset(struct tty_struct *tty, } static void moxa_set_termios(struct tty_struct *tty, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct moxa_port *ch = tty->driver_data; @@ -1761,7 +1761,8 @@ static void moxa_poll(struct timer_list *unused) /******************************************************************************/ -static void moxa_set_tty_param(struct tty_struct *tty, struct ktermios *old_termios) +static void moxa_set_tty_param(struct tty_struct *tty, + const struct ktermios *old_termios) { register struct ktermios *ts = &tty->termios; struct moxa_port *ch = tty->driver_data; diff --git a/drivers/tty/mxser.c b/drivers/tty/mxser.c index 70b982b2c6b24c8f93975f2503bc1fc0bd93fed5..2436e0b10f9a56b56719b93f621f0111706cedf0 100644 --- a/drivers/tty/mxser.c +++ b/drivers/tty/mxser.c @@ -398,7 +398,7 @@ static enum mxser_must_hwid mxser_must_get_hwid(unsigned long io) oldmcr = inb(io + UART_MCR); outb(0, io + UART_MCR); mxser_set_must_xon1_value(io, 0x11); - if ((hwid = inb(io + UART_MCR)) != 0) { + if (inb(io + UART_MCR) != 0) { outb(oldmcr, io + UART_MCR); return MOXA_OTHER_UART; } @@ -571,7 +571,8 @@ static void mxser_handle_cts(struct tty_struct *tty, struct mxser_port *info, * This routine is called to set the UART divisor registers to match * the specified baud rate for a serial port. */ -static void mxser_change_speed(struct tty_struct *tty, struct ktermios *old_termios) +static void mxser_change_speed(struct tty_struct *tty, + const struct ktermios *old_termios) { struct mxser_port *info = tty->driver_data; unsigned cflag, cval; @@ -1348,7 +1349,8 @@ static void mxser_start(struct tty_struct *tty) spin_unlock_irqrestore(&info->slock, flags); } -static void mxser_set_termios(struct tty_struct *tty, struct ktermios *old_termios) +static void mxser_set_termios(struct tty_struct *tty, + const struct ktermios *old_termios) { struct mxser_port *info = tty->driver_data; unsigned long flags; diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index caa5c14ed57f0269bf3ac5640f2ffdeb5703e70c..5e516f5cac5a32171428d72d510765586b61ad3e 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -63,6 +63,14 @@ static int debug; module_param(debug, int, 0600); +/* Module debug bits */ +#define DBG_DUMP BIT(0) /* Data transmission dump. */ +#define DBG_CD_ON BIT(1) /* Always assume CD line on. */ +#define DBG_DATA BIT(2) /* Data transmission details. */ +#define DBG_ERRORS BIT(3) /* Details for fail conditions. */ +#define DBG_TTY BIT(4) /* Transmission statistics for DLCI TTYs. */ +#define DBG_PAYLOAD BIT(5) /* Limits DBG_DUMP to payload frames. */ + /* Defaults: these are from the specification */ #define T1 10 /* 100mS */ @@ -164,6 +172,9 @@ struct gsm_dlci { struct net_device *net; /* network interface, if created */ }; +/* Total number of supported devices */ +#define GSM_TTY_MINORS 256 + /* DLCI 0, 62/63 are special or reserved see gsmtty_open */ #define NUM_DLCI 64 @@ -184,6 +195,11 @@ struct gsm_control { int error; /* Error if any */ }; +enum gsm_encoding { + GSM_BASIC_OPT, + GSM_ADV_OPT, +}; + enum gsm_mux_state { GSM_SEARCH, GSM_START, @@ -230,7 +246,7 @@ struct gsm_mux { unsigned int address; unsigned int count; bool escape; - int encoding; + enum gsm_encoding encoding; u8 control; u8 fcs; u8 *txframe; /* TX framing buffer */ @@ -248,7 +264,7 @@ struct gsm_mux { bool constipated; /* Asked by remote to shut up */ bool has_devices; /* Devices were registered */ - spinlock_t tx_lock; + struct mutex tx_mutex; unsigned int tx_bytes; /* TX data outstanding */ #define TX_THRESH_HI 8192 #define TX_THRESH_LO 2048 @@ -256,7 +272,7 @@ struct gsm_mux { struct list_head tx_data_list; /* Pending data packets */ /* Control messages */ - struct timer_list kick_timer; /* Kick TX queuing on timeout */ + struct delayed_work kick_timeout; /* Kick TX queuing on timeout */ struct timer_list t2_timer; /* Retransmit timer for commands */ int cretries; /* Command retry counter */ struct gsm_control *pending_cmd;/* Our current pending command */ @@ -527,7 +543,7 @@ static int gsm_register_devices(struct tty_driver *driver, unsigned int index) */ dev = tty_register_device(gsm_tty_driver, base + i, NULL); if (IS_ERR(dev)) { - if (debug & 8) + if (debug & DBG_ERRORS) pr_info("%s failed to register device minor %u", __func__, base + i); for (i--; i >= 1; i--) @@ -581,8 +597,12 @@ static void gsm_unregister_devices(struct tty_driver *driver, static void gsm_print_packet(const char *hdr, int addr, int cr, u8 control, const u8 *data, int dlen) { - if (!(debug & 1)) + if (!(debug & DBG_DUMP)) return; + /* Only show user payload frames if debug & DBG_PAYLOAD */ + if (!(debug & DBG_PAYLOAD) && addr != 0) + if ((control & ~PF) == UI || (control & ~PF) == UIH) + return; pr_info("%s %d) %c: ", hdr, addr, "RC"[cr]); @@ -680,7 +700,6 @@ static int gsm_send(struct gsm_mux *gsm, int addr, int cr, int control) struct gsm_msg *msg; u8 *dp; int ocr; - unsigned long flags; msg = gsm_data_alloc(gsm, addr, 0, control); if (!msg) @@ -694,7 +713,7 @@ static int gsm_send(struct gsm_mux *gsm, int addr, int cr, int control) *dp++ = (addr << 2) | (ocr << 1) | EA; *dp++ = control; - if (gsm->encoding == 0) + if (gsm->encoding == GSM_BASIC_OPT) *dp++ = EA; /* Length of data = 0 */ *dp = 0xFF - gsm_fcs_add_block(INIT_FCS, msg->data, dp - msg->data); @@ -702,10 +721,10 @@ static int gsm_send(struct gsm_mux *gsm, int addr, int cr, int control) gsm_print_packet("Q->", addr, cr, control, NULL, 0); - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); list_add_tail(&msg->list, &gsm->tx_ctrl_list); gsm->tx_bytes += msg->len; - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); gsmld_write_trigger(gsm); return 0; @@ -730,7 +749,7 @@ static void gsm_dlci_clear_queues(struct gsm_mux *gsm, struct gsm_dlci *dlci) spin_unlock_irqrestore(&dlci->lock, flags); /* Clear data packets in MUX write queue */ - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); list_for_each_entry_safe(msg, nmsg, &gsm->tx_data_list, list) { if (msg->addr != addr) continue; @@ -738,7 +757,7 @@ static void gsm_dlci_clear_queues(struct gsm_mux *gsm, struct gsm_dlci *dlci) list_del(&msg->list); kfree(msg); } - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); } /** @@ -813,7 +832,7 @@ static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg) int len, ret; - if (gsm->encoding == 0) { + if (gsm->encoding == GSM_BASIC_OPT) { gsm->txframe[0] = GSM0_SOF; memcpy(gsm->txframe + 1, msg->data, msg->len); gsm->txframe[msg->len + 1] = GSM0_SOF; @@ -825,7 +844,7 @@ static int gsm_send_packet(struct gsm_mux *gsm, struct gsm_msg *msg) len += 2; } - if (debug & 4) + if (debug & DBG_DATA) gsm_hex_dump_bytes(__func__, gsm->txframe, len); gsm_print_packet("-->", msg->addr, gsm->initiator, msg->ctrl, msg->data, msg->len); @@ -965,7 +984,7 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) u8 *fcs = dp + msg->len; /* Fill in the header */ - if (gsm->encoding == 0) { + if (gsm->encoding == GSM_BASIC_OPT) { if (msg->len < 128) *--dp = (msg->len << 1) | EA; else { @@ -1009,7 +1028,7 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) gsm->tx_bytes += msg->len; gsmld_write_trigger(gsm); - mod_timer(&gsm->kick_timer, jiffies + 10 * gsm->t1 * HZ / 100); + schedule_delayed_work(&gsm->kick_timeout, 10 * gsm->t1 * HZ / 100); } /** @@ -1024,10 +1043,9 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) static void gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) { - unsigned long flags; - spin_lock_irqsave(&dlci->gsm->tx_lock, flags); + mutex_lock(&dlci->gsm->tx_mutex); __gsm_data_queue(dlci, msg); - spin_unlock_irqrestore(&dlci->gsm->tx_lock, flags); + mutex_unlock(&dlci->gsm->tx_mutex); } /** @@ -1039,7 +1057,7 @@ static void gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) * is data. Keep to the MRU of the mux. This path handles the usual tty * interface which is a byte stream with optional modem data. * - * Caller must hold the tx_lock of the mux. + * Caller must hold the tx_mutex of the mux. */ static int gsm_dlci_data_output(struct gsm_mux *gsm, struct gsm_dlci *dlci) @@ -1099,7 +1117,7 @@ static int gsm_dlci_data_output(struct gsm_mux *gsm, struct gsm_dlci *dlci) * is data. Keep to the MRU of the mux. This path handles framed data * queued as skbuffs to the DLCI. * - * Caller must hold the tx_lock of the mux. + * Caller must hold the tx_mutex of the mux. */ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, @@ -1115,7 +1133,7 @@ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, if (dlci->adaption == 4) overhead = 1; - /* dlci->skb is locked by tx_lock */ + /* dlci->skb is locked by tx_mutex */ if (dlci->skb == NULL) { dlci->skb = skb_dequeue_tail(&dlci->skb_list); if (dlci->skb == NULL) @@ -1169,7 +1187,7 @@ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, * Push an empty frame in to the transmit queue to update the modem status * bits and to transmit an optional break. * - * Caller must hold the tx_lock of the mux. + * Caller must hold the tx_mutex of the mux. */ static int gsm_dlci_modem_output(struct gsm_mux *gsm, struct gsm_dlci *dlci, @@ -1283,13 +1301,12 @@ static int gsm_dlci_data_sweep(struct gsm_mux *gsm) static void gsm_dlci_data_kick(struct gsm_dlci *dlci) { - unsigned long flags; int sweep; if (dlci->constipated) return; - spin_lock_irqsave(&dlci->gsm->tx_lock, flags); + mutex_lock(&dlci->gsm->tx_mutex); /* If we have nothing running then we need to fire up */ sweep = (dlci->gsm->tx_bytes < TX_THRESH_LO); if (dlci->gsm->tx_bytes == 0) { @@ -1300,7 +1317,7 @@ static void gsm_dlci_data_kick(struct gsm_dlci *dlci) } if (sweep) gsm_dlci_data_sweep(dlci->gsm); - spin_unlock_irqrestore(&dlci->gsm->tx_lock, flags); + mutex_unlock(&dlci->gsm->tx_mutex); } /* @@ -1308,6 +1325,31 @@ static void gsm_dlci_data_kick(struct gsm_dlci *dlci) */ +/** + * gsm_control_command - send a command frame to a control + * @gsm: gsm channel + * @cmd: the command to use + * @data: data to follow encoded info + * @dlen: length of data + * + * Encode up and queue a UI/UIH frame containing our command. + */ +static int gsm_control_command(struct gsm_mux *gsm, int cmd, const u8 *data, + int dlen) +{ + struct gsm_msg *msg = gsm_data_alloc(gsm, 0, dlen + 2, gsm->ftype); + + if (msg == NULL) + return -ENOMEM; + + msg->data[0] = (cmd << 1) | CR | EA; /* Set C/R */ + msg->data[1] = (dlen << 1) | EA; + memcpy(msg->data + 2, data, dlen); + gsm_data_queue(gsm->dlci[0], msg); + + return 0; +} + /** * gsm_control_reply - send a response frame to a control * @gsm: gsm channel @@ -1410,18 +1452,12 @@ static void gsm_control_modem(struct gsm_mux *gsm, const u8 *data, int clen) unsigned int modem = 0; struct gsm_dlci *dlci; int len = clen; - int slen; + int cl = clen; const u8 *dp = data; struct tty_struct *tty; - while (gsm_read_ea(&addr, *dp++) == 0) { - len--; - if (len == 0) - return; - } - /* Must be at least one byte following the EA */ - len--; - if (len <= 0) + len = gsm_read_ea_val(&addr, data, cl); + if (len < 1) return; addr >>= 1; @@ -1430,15 +1466,20 @@ static void gsm_control_modem(struct gsm_mux *gsm, const u8 *data, int clen) return; dlci = gsm->dlci[addr]; - slen = len; - while (gsm_read_ea(&modem, *dp++) == 0) { - len--; - if (len == 0) - return; - } - len--; + /* Must be at least one byte following the EA */ + if ((cl - len) < 1) + return; + + dp += len; + cl -= len; + + /* get the modem status */ + len = gsm_read_ea_val(&modem, dp, cl); + if (len < 1) + return; + tty = tty_port_tty_get(&dlci->port); - gsm_process_modem(tty, dlci, modem, slen - len); + gsm_process_modem(tty, dlci, modem, cl); if (tty) { tty_wakeup(tty); tty_kref_put(tty); @@ -1614,13 +1655,7 @@ static void gsm_control_response(struct gsm_mux *gsm, unsigned int command, static void gsm_control_transmit(struct gsm_mux *gsm, struct gsm_control *ctrl) { - struct gsm_msg *msg = gsm_data_alloc(gsm, 0, ctrl->len + 2, gsm->ftype); - if (msg == NULL) - return; - msg->data[0] = (ctrl->cmd << 1) | CR | EA; /* command */ - msg->data[1] = (ctrl->len << 1) | EA; - memcpy(msg->data + 2, ctrl->data, ctrl->len); - gsm_data_queue(gsm->dlci[0], msg); + gsm_control_command(gsm, ctrl->cmd, ctrl->data, ctrl->len); } /** @@ -1740,7 +1775,7 @@ static int gsm_control_wait(struct gsm_mux *gsm, struct gsm_control *control) static void gsm_dlci_close(struct gsm_dlci *dlci) { del_timer(&dlci->t1); - if (debug & 8) + if (debug & DBG_ERRORS) pr_debug("DLCI %d goes closed.\n", dlci->addr); dlci->state = DLCI_CLOSED; /* Prevent us from sending data before the link is up again */ @@ -1774,7 +1809,7 @@ static void gsm_dlci_open(struct gsm_dlci *dlci) /* This will let a tty open continue */ dlci->state = DLCI_OPEN; dlci->constipated = false; - if (debug & 8) + if (debug & DBG_ERRORS) pr_debug("DLCI %d goes open.\n", dlci->addr); /* Send current modem state */ if (dlci->addr) @@ -1810,7 +1845,7 @@ static void gsm_dlci_t1(struct timer_list *t) gsm_command(dlci->gsm, dlci->addr, SABM|PF); mod_timer(&dlci->t1, jiffies + gsm->t1 * HZ / 100); } else if (!dlci->addr && gsm->control == (DM | PF)) { - if (debug & 8) + if (debug & DBG_ERRORS) pr_info("DLCI %d opening in ADM mode.\n", dlci->addr); dlci->mode = DLCI_MODE_ADM; @@ -1913,11 +1948,10 @@ static void gsm_dlci_data(struct gsm_dlci *dlci, const u8 *data, int clen) struct tty_port *port = &dlci->port; struct tty_struct *tty; unsigned int modem = 0; - int len = clen; - int slen = 0; + int len; - if (debug & 16) - pr_debug("%d bytes for tty\n", len); + if (debug & DBG_TTY) + pr_debug("%d bytes for tty\n", clen); switch (dlci->adaption) { /* Unsupported types */ case 4: /* Packetised interruptible data */ @@ -1925,24 +1959,22 @@ static void gsm_dlci_data(struct gsm_dlci *dlci, const u8 *data, int clen) case 3: /* Packetised uininterruptible voice/data */ break; case 2: /* Asynchronous serial with line state in each frame */ - while (gsm_read_ea(&modem, *data++) == 0) { - len--; - slen++; - if (len == 0) - return; - } - len--; - slen++; + len = gsm_read_ea_val(&modem, data, clen); + if (len < 1) + return; tty = tty_port_tty_get(port); if (tty) { - gsm_process_modem(tty, dlci, modem, slen); + gsm_process_modem(tty, dlci, modem, len); tty_wakeup(tty); tty_kref_put(tty); } + /* Skip processed modem data */ + data += len; + clen -= len; fallthrough; case 1: /* Line state will go via DLCI 0 controls only */ default: - tty_insert_flip_string(port, data, len); + tty_insert_flip_string(port, data, clen); tty_flip_buffer_push(port); } } @@ -1963,47 +1995,49 @@ static void gsm_dlci_command(struct gsm_dlci *dlci, const u8 *data, int len) { /* See what command is involved */ unsigned int command = 0; - while (len-- > 0) { - if (gsm_read_ea(&command, *data++) == 1) { - int clen = *data++; - len--; - /* FIXME: this is properly an EA */ - clen >>= 1; - /* Malformed command ? */ - if (clen > len) - return; - if (command & 1) - gsm_control_message(dlci->gsm, command, - data, clen); - else - gsm_control_response(dlci->gsm, command, - data, clen); - return; - } - } + unsigned int clen = 0; + unsigned int dlen; + + /* read the command */ + dlen = gsm_read_ea_val(&command, data, len); + len -= dlen; + data += dlen; + + /* read any control data */ + dlen = gsm_read_ea_val(&clen, data, len); + len -= dlen; + data += dlen; + + /* Malformed command? */ + if (clen > len) + return; + + if (command & 1) + gsm_control_message(dlci->gsm, command, data, clen); + else + gsm_control_response(dlci->gsm, command, data, clen); } /** - * gsm_kick_timer - transmit if possible - * @t: timer contained in our gsm object + * gsm_kick_timeout - transmit if possible + * @work: work contained in our gsm object * * Transmit data from DLCIs if the queue is empty. We can't rely on * a tty wakeup except when we filled the pipe so we need to fire off * new data ourselves in other cases. */ -static void gsm_kick_timer(struct timer_list *t) +static void gsm_kick_timeout(struct work_struct *work) { - struct gsm_mux *gsm = from_timer(gsm, t, kick_timer); - unsigned long flags; + struct gsm_mux *gsm = container_of(work, struct gsm_mux, kick_timeout.work); int sent = 0; - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); /* If we have nothing running then we need to fire up */ if (gsm->tx_bytes < TX_THRESH_LO) sent = gsm_dlci_data_sweep(gsm); - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); - if (sent && debug & 4) + if (sent && debug & DBG_DATA) pr_info("%s TX queue stalled\n", __func__); } @@ -2137,7 +2171,7 @@ static void gsm_queue(struct gsm_mux *gsm) if (gsm->fcs != GOOD_FCS) { gsm->bad_fcs++; - if (debug & 4) + if (debug & DBG_DATA) pr_debug("BAD FCS %02x\n", gsm->fcs); return; } @@ -2458,7 +2492,7 @@ static void gsm_cleanup_mux(struct gsm_mux *gsm, bool disc) } /* Finish outstanding timers, making sure they are done */ - del_timer_sync(&gsm->kick_timer); + cancel_delayed_work_sync(&gsm->kick_timeout); del_timer_sync(&gsm->t2_timer); /* Finish writing to ldisc */ @@ -2501,14 +2535,7 @@ static int gsm_activate_mux(struct gsm_mux *gsm) if (dlci == NULL) return -ENOMEM; - timer_setup(&gsm->kick_timer, gsm_kick_timer, 0); - timer_setup(&gsm->t2_timer, gsm_control_retransmit, 0); - INIT_WORK(&gsm->tx_work, gsmld_write_task); - init_waitqueue_head(&gsm->event); - spin_lock_init(&gsm->control_lock); - spin_lock_init(&gsm->tx_lock); - - if (gsm->encoding == 0) + if (gsm->encoding == GSM_BASIC_OPT) gsm->receive = gsm0_receive; else gsm->receive = gsm1_receive; @@ -2538,6 +2565,7 @@ static void gsm_free_mux(struct gsm_mux *gsm) break; } } + mutex_destroy(&gsm->tx_mutex); mutex_destroy(&gsm->mutex); kfree(gsm->txframe); kfree(gsm->buf); @@ -2609,16 +2637,22 @@ static struct gsm_mux *gsm_alloc_mux(void) } spin_lock_init(&gsm->lock); mutex_init(&gsm->mutex); + mutex_init(&gsm->tx_mutex); kref_init(&gsm->ref); INIT_LIST_HEAD(&gsm->tx_ctrl_list); INIT_LIST_HEAD(&gsm->tx_data_list); + INIT_DELAYED_WORK(&gsm->kick_timeout, gsm_kick_timeout); + timer_setup(&gsm->t2_timer, gsm_control_retransmit, 0); + INIT_WORK(&gsm->tx_work, gsmld_write_task); + init_waitqueue_head(&gsm->event); + spin_lock_init(&gsm->control_lock); gsm->t1 = T1; gsm->t2 = T2; gsm->n2 = N2; gsm->ftype = UIH; gsm->adaption = 1; - gsm->encoding = 1; + gsm->encoding = GSM_ADV_OPT; gsm->mru = 64; /* Default to encoding 1 so these should be 64 */ gsm->mtu = 64; gsm->dead = true; /* Avoid early tty opens */ @@ -2636,6 +2670,7 @@ static struct gsm_mux *gsm_alloc_mux(void) } spin_unlock(&gsm_mux_lock); if (i == MAX_MUX) { + mutex_destroy(&gsm->tx_mutex); mutex_destroy(&gsm->mutex); kfree(gsm->txframe); kfree(gsm->buf); @@ -2719,7 +2754,7 @@ static int gsm_config(struct gsm_mux *gsm, struct gsm_config *c) gsm->initiator = c->initiator; gsm->mru = c->mru; gsm->mtu = c->mtu; - gsm->encoding = c->encapsulation; + gsm->encoding = c->encapsulation ? GSM_ADV_OPT : GSM_BASIC_OPT; gsm->adaption = c->adaption; gsm->n2 = c->n2; @@ -2763,7 +2798,7 @@ static int gsmld_output(struct gsm_mux *gsm, u8 *data, int len) set_bit(TTY_DO_WRITE_WAKEUP, &gsm->tty->flags); return -ENOSPC; } - if (debug & 4) + if (debug & DBG_DATA) gsm_hex_dump_bytes(__func__, data, len); return gsm->tty->ops->write(gsm->tty, data, len); } @@ -2791,17 +2826,16 @@ static void gsmld_write_trigger(struct gsm_mux *gsm) static void gsmld_write_task(struct work_struct *work) { struct gsm_mux *gsm = container_of(work, struct gsm_mux, tx_work); - unsigned long flags; int i, ret; /* All outstanding control channel and control messages and one data * frame is sent. */ ret = -ENODEV; - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); if (gsm->tty) ret = gsm_data_kick(gsm); - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); if (ret >= 0) for (i = 0; i < NUM_DLCI; i++) @@ -2850,7 +2884,7 @@ static void gsmld_receive_buf(struct tty_struct *tty, const unsigned char *cp, struct gsm_mux *gsm = tty->disc_data; char flags = TTY_NORMAL; - if (debug & 4) + if (debug & DBG_DATA) gsm_hex_dump_bytes(__func__, cp, count); for (; count; count--, cp++) { @@ -2858,7 +2892,8 @@ static void gsmld_receive_buf(struct tty_struct *tty, const unsigned char *cp, flags = *fp++; switch (flags) { case TTY_NORMAL: - gsm->receive(gsm, *cp); + if (gsm->receive) + gsm->receive(gsm, *cp); break; case TTY_OVERRUN: case TTY_BREAK: @@ -2942,14 +2977,9 @@ static int gsmld_open(struct tty_struct *tty) tty->receive_room = 65536; /* Attach the initial passive connection */ - gsm->encoding = 1; - + gsm->encoding = GSM_ADV_OPT; gsmld_attach_gsm(tty, gsm); - timer_setup(&gsm->kick_timer, gsm_kick_timer, 0); - timer_setup(&gsm->t2_timer, gsm_control_retransmit, 0); - INIT_WORK(&gsm->tx_work, gsmld_write_task); - return 0; } @@ -3012,7 +3042,6 @@ static ssize_t gsmld_write(struct tty_struct *tty, struct file *file, const unsigned char *buf, size_t nr) { struct gsm_mux *gsm = tty->disc_data; - unsigned long flags; int space; int ret; @@ -3020,13 +3049,13 @@ static ssize_t gsmld_write(struct tty_struct *tty, struct file *file, return -ENODEV; ret = -ENOBUFS; - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); space = tty_write_room(tty); if (space >= nr) ret = tty->ops->write(tty, buf, nr); else set_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); return ret; } @@ -3323,14 +3352,13 @@ static struct tty_ldisc_ops tty_ldisc_packet = { static void gsm_modem_upd_via_data(struct gsm_dlci *dlci, u8 brk) { struct gsm_mux *gsm = dlci->gsm; - unsigned long flags; if (dlci->state != DLCI_OPEN || dlci->adaption != 2) return; - spin_lock_irqsave(&gsm->tx_lock, flags); + mutex_lock(&gsm->tx_mutex); gsm_dlci_modem_output(gsm, dlci, brk); - spin_unlock_irqrestore(&gsm->tx_lock, flags); + mutex_unlock(&gsm->tx_mutex); } /** @@ -3345,7 +3373,7 @@ static int gsm_modem_upd_via_msc(struct gsm_dlci *dlci, u8 brk) struct gsm_control *ctrl; int len = 2; - if (dlci->gsm->encoding != 0) + if (dlci->gsm->encoding != GSM_BASIC_OPT) return 0; modembits[0] = (dlci->addr << 2) | 2 | EA; /* DLCI, Valid, EA */ @@ -3374,7 +3402,7 @@ static int gsm_modem_update(struct gsm_dlci *dlci, u8 brk) /* Send convergence layer type 2 empty data frame. */ gsm_modem_upd_via_data(dlci, brk); return 0; - } else if (dlci->gsm->encoding == 0) { + } else if (dlci->gsm->encoding == GSM_BASIC_OPT) { /* Send as MSC control message. */ return gsm_modem_upd_via_msc(dlci, brk); } @@ -3391,15 +3419,15 @@ static int gsm_carrier_raised(struct tty_port *port) /* Not yet open so no carrier info */ if (dlci->state != DLCI_OPEN) return 0; - if (debug & 2) + if (debug & DBG_CD_ON) return 1; /* * Basic mode with control channel in ADM mode may not respond * to CMD_MSC at all and modem_rx is empty. */ - if (gsm->encoding == 0 && gsm->dlci[0]->mode == DLCI_MODE_ADM && - !dlci->modem_rx) + if (gsm->encoding == GSM_BASIC_OPT && + gsm->dlci[0]->mode == DLCI_MODE_ADM && !dlci->modem_rx) return 1; return dlci->modem_rx & TIOCM_CD; @@ -3647,7 +3675,8 @@ static int gsmtty_ioctl(struct tty_struct *tty, } } -static void gsmtty_set_termios(struct tty_struct *tty, struct ktermios *old) +static void gsmtty_set_termios(struct tty_struct *tty, + const struct ktermios *old) { struct gsm_dlci *dlci = tty->driver_data; if (dlci->state == DLCI_CLOSED) @@ -3745,7 +3774,7 @@ static int __init gsm_init(void) return status; } - gsm_tty_driver = tty_alloc_driver(256, TTY_DRIVER_REAL_RAW | + gsm_tty_driver = tty_alloc_driver(GSM_TTY_MINORS, TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV | TTY_DRIVER_HARDWARE_BREAK); if (IS_ERR(gsm_tty_driver)) { pr_err("gsm_init: tty allocation failed.\n"); diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c index 94c1ec2dd7544c6b0d6aee3520b26b72e4662fe1..46b09bfb6f3a1c6af3e48a738be0183c79ea46dc 100644 --- a/drivers/tty/n_hdlc.c +++ b/drivers/tty/n_hdlc.c @@ -76,8 +76,6 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ -#define HDLC_MAGIC 0x239e - #include #include #include @@ -98,7 +96,6 @@ #include #include -#include #include #include "tty.h" @@ -124,7 +121,6 @@ struct n_hdlc_buf_list { /** * struct n_hdlc - per device instance data structure - * @magic: magic value for structure * @tbusy: reentrancy flag for tx wakeup code * @woke_up: tx wakeup needs to be run again as it was called while @tbusy * @tx_buf_list: list of pending transmit frame buffers @@ -133,7 +129,6 @@ struct n_hdlc_buf_list { * @rx_free_buf_list: list unused received frame buffers */ struct n_hdlc { - int magic; bool tbusy; bool woke_up; struct n_hdlc_buf_list tx_buf_list; @@ -200,10 +195,6 @@ static void n_hdlc_tty_close(struct tty_struct *tty) { struct n_hdlc *n_hdlc = tty->disc_data; - if (n_hdlc->magic != HDLC_MAGIC) { - pr_warn("n_hdlc: trying to close unopened tty!\n"); - return; - } #if defined(TTY_NO_WRITE_SPLIT) clear_bit(TTY_NO_WRITE_SPLIT, &tty->flags); #endif @@ -386,12 +377,6 @@ static void n_hdlc_tty_receive(struct tty_struct *tty, const __u8 *data, pr_debug("%s() called count=%d\n", __func__, count); - /* verify line is using HDLC discipline */ - if (n_hdlc->magic != HDLC_MAGIC) { - pr_err("line not using HDLC discipline\n"); - return; - } - if (count > maxframe) { pr_debug("rx count>maxframesize, data discarded\n"); return; @@ -542,9 +527,6 @@ static ssize_t n_hdlc_tty_write(struct tty_struct *tty, struct file *file, pr_debug("%s() called count=%zd\n", __func__, count); - if (n_hdlc->magic != HDLC_MAGIC) - return -EIO; - /* verify frame size */ if (count > maxframe) { pr_debug("%s: truncating user packet from %zu to %d\n", @@ -609,10 +591,6 @@ static int n_hdlc_tty_ioctl(struct tty_struct *tty, unsigned int cmd, pr_debug("%s() called %d\n", __func__, cmd); - /* Verify the status of the device */ - if (n_hdlc->magic != HDLC_MAGIC) - return -EBADF; - switch (cmd) { case FIONREAD: /* report count of read data available */ @@ -673,9 +651,6 @@ static __poll_t n_hdlc_tty_poll(struct tty_struct *tty, struct file *filp, struct n_hdlc *n_hdlc = tty->disc_data; __poll_t mask = 0; - if (n_hdlc->magic != HDLC_MAGIC) - return 0; - /* * queue the current process into any wait queue that may awaken in the * future (read and write) @@ -739,9 +714,6 @@ static struct n_hdlc *n_hdlc_alloc(void) n_hdlc_alloc_buf(&n_hdlc->rx_free_buf_list, DEFAULT_RX_BUF_COUNT, "rx"); n_hdlc_alloc_buf(&n_hdlc->tx_free_buf_list, DEFAULT_TX_BUF_COUNT, "tx"); - /* Initialize the control block */ - n_hdlc->magic = HDLC_MAGIC; - return n_hdlc; } /* end of n_hdlc_alloc() */ diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c index 3afdd9033a9c50b82b9f6e5c33471c7d908afed9..597019690ae62b088faffdae4d2e7be0165dcae0 100644 --- a/drivers/tty/n_tty.c +++ b/drivers/tty/n_tty.c @@ -1758,7 +1758,7 @@ static int n_tty_receive_buf2(struct tty_struct *tty, const unsigned char *cp, * * Locking: Caller holds @tty->termios_rwsem */ -static void n_tty_set_termios(struct tty_struct *tty, struct ktermios *old) +static void n_tty_set_termios(struct tty_struct *tty, const struct ktermios *old) { struct n_tty_data *ldata = tty->disc_data; diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c index 752dab3356d72b6e9c7f3d25ad91eae31ae8fc83..07394fdaf5225d36417a1eb45dcd08ebe0a55236 100644 --- a/drivers/tty/pty.c +++ b/drivers/tty/pty.c @@ -240,7 +240,7 @@ out: } static void pty_set_termios(struct tty_struct *tty, - struct ktermios *old_termios) + const struct ktermios *old_termios) { /* See if packet mode change of state. */ if (tty->link && tty->link->ctrl.packet) { diff --git a/drivers/tty/serial/21285.c b/drivers/tty/serial/21285.c index 7520cc02fd4dc05a214cfc210e4a47f7ce5acb86..c7d34823f7155ae253332f6d2051879c3e87c731 100644 --- a/drivers/tty/serial/21285.c +++ b/drivers/tty/serial/21285.c @@ -243,7 +243,7 @@ static void serial21285_shutdown(struct uart_port *port) static void serial21285_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, quot, h_lcr, b; @@ -461,9 +461,6 @@ static int __init serial21285_console_setup(struct console *co, char *options) int parity = 'n'; int flow = 'n'; - if (machine_is_personal_server()) - baud = 57600; - /* * Check whether an invalid uart number has been specified, and * if so, search for the first available port that does have diff --git a/drivers/tty/serial/8250/8250_bcm7271.c b/drivers/tty/serial/8250/8250_bcm7271.c index 8efdc271eb75f20b6b3a4eee92608da318417b1e..fa8ccf204d860223e4c5339791295e43d9575e9b 100644 --- a/drivers/tty/serial/8250/8250_bcm7271.c +++ b/drivers/tty/serial/8250/8250_bcm7271.c @@ -755,7 +755,7 @@ static void set_clock_mux(struct uart_port *up, struct brcmuart_priv *priv, static void brcmstb_set_termios(struct uart_port *up, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_8250_port *p8250 = up_to_u8250p(up); struct brcmuart_priv *priv = up->private_data; diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index 2e83e7367441c4b8686613aba3dda09a87a8886f..94fbf0add2ce25f42c0139c85ec6dc01e22c5799 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -298,10 +298,9 @@ static void serial8250_backup_timeout(struct timer_list *t) jiffies + uart_poll_timeout(&up->port) + HZ / 5); } -static int univ8250_setup_irq(struct uart_8250_port *up) +static void univ8250_setup_timer(struct uart_8250_port *up) { struct uart_port *port = &up->port; - int retval = 0; /* * The above check will only give an accurate result the first time @@ -322,10 +321,16 @@ static int univ8250_setup_irq(struct uart_8250_port *up) */ if (!port->irq) mod_timer(&up->timer, jiffies + uart_poll_timeout(port)); - else - retval = serial_link_irq_chain(up); +} - return retval; +static int univ8250_setup_irq(struct uart_8250_port *up) +{ + struct uart_port *port = &up->port; + + if (port->irq) + return serial_link_irq_chain(up); + + return 0; } static void univ8250_release_irq(struct uart_8250_port *up) @@ -381,6 +386,7 @@ static struct uart_ops univ8250_port_ops; static const struct uart_8250_ops univ8250_driver_ops = { .setup_irq = univ8250_setup_irq, .release_irq = univ8250_release_irq, + .setup_timer = univ8250_setup_timer, }; static struct uart_8250_port serial8250_ports[UART_NR]; diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c index a8dba4a0a8fb70b823abd89cbeed677f84079429..b85c82616e8cbb8d7853dfd5c0fef202299803b8 100644 --- a/drivers/tty/serial/8250/8250_dma.c +++ b/drivers/tty/serial/8250/8250_dma.c @@ -26,9 +26,7 @@ static void __dma_tx_complete(void *param) dma->tx_running = 0; - xmit->tail += dma->tx_size; - xmit->tail &= UART_XMIT_SIZE - 1; - p->port.icount.tx += dma->tx_size; + uart_xmit_advance(&p->port, dma->tx_size); if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) uart_write_wakeup(&p->port); @@ -107,8 +105,7 @@ int serial8250_tx_dma(struct uart_8250_port *p) dma_async_issue_pending(dma->txchan); serial8250_clear_THRI(p); - if (dma->tx_err) - dma->tx_err = 0; + dma->tx_err = 0; return 0; err: diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c index a604b42e4458517f35588bc1a195d8defe373fd9..7db51781289edb0c5aee1ee60048840b810b793e 100644 --- a/drivers/tty/serial/8250/8250_dw.c +++ b/drivers/tty/serial/8250/8250_dw.c @@ -350,7 +350,7 @@ dw8250_do_pm(struct uart_port *port, unsigned int state, unsigned int old) } static void dw8250_set_termios(struct uart_port *p, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long newrate = tty_termios_baud_rate(termios) * 16; struct dw8250_data *d = to_dw8250_data(p->private_data); diff --git a/drivers/tty/serial/8250/8250_dwlib.c b/drivers/tty/serial/8250/8250_dwlib.c index dbe4d44f60d4ce6d72e23d26dbd419b526dd947e..75f32f054ebb15e64b8fedc80bb743c431f3375f 100644 --- a/drivers/tty/serial/8250/8250_dwlib.c +++ b/drivers/tty/serial/8250/8250_dwlib.c @@ -92,7 +92,8 @@ static void dw8250_set_divisor(struct uart_port *p, unsigned int baud, serial8250_do_set_divisor(p, baud, quot, quot_frac); } -void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, struct ktermios *old) +void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, + const struct ktermios *old) { p->status &= ~UPSTAT_AUTOCTS; if (termios->c_cflag & CRTSCTS) diff --git a/drivers/tty/serial/8250/8250_dwlib.h b/drivers/tty/serial/8250/8250_dwlib.h index 055bfdc87985736aee5ad3bf20360099aa066ee9..f13e91f2cace9ccb119745f803e5e7cf92065d9c 100644 --- a/drivers/tty/serial/8250/8250_dwlib.h +++ b/drivers/tty/serial/8250/8250_dwlib.h @@ -47,7 +47,7 @@ struct dw8250_data { unsigned int uart_16550_compatible:1; }; -void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, struct ktermios *old); +void dw8250_do_set_termios(struct uart_port *p, struct ktermios *termios, const struct ktermios *old); void dw8250_setup_port(struct uart_port *p); static inline struct dw8250_data *to_dw8250_data(struct dw8250_port_data *data) diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c index 65b6b3cbaff6cb02eea8bd4eec06e2788a6a23d7..e2aa2a1a02ddf53997fb7425a216a7df3d8f517e 100644 --- a/drivers/tty/serial/8250/8250_fintek.c +++ b/drivers/tty/serial/8250/8250_fintek.c @@ -278,7 +278,7 @@ static void fintek_8250_set_max_fifo(struct fintek_8250 *pdata) static void fintek_8250_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct fintek_8250 *pdata = port->private_data; unsigned int baud = tty_termios_baud_rate(termios); diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c index 4ba43bef99336f27883c1138920a2402037c9d8f..44cc755b1a29a90988f292b9844dcec98fd5fdc3 100644 --- a/drivers/tty/serial/8250/8250_lpss.c +++ b/drivers/tty/serial/8250/8250_lpss.c @@ -70,7 +70,7 @@ static inline struct lpss8250 *to_lpss8250(struct dw8250_port_data *data) } static void byt_set_termios(struct uart_port *p, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud = tty_termios_baud_rate(termios); struct lpss8250 *lpss = to_lpss8250(p->private_data); diff --git a/drivers/tty/serial/8250/8250_men_mcb.c b/drivers/tty/serial/8250/8250_men_mcb.c index 737c4c31e8a05c82d72ce110bf88d7c59a72f090..f46ca13ff4aac77e4170cfcdbd6afe2deb8a3129 100644 --- a/drivers/tty/serial/8250/8250_men_mcb.c +++ b/drivers/tty/serial/8250/8250_men_mcb.c @@ -7,7 +7,6 @@ #include #include #include -#include #define MEN_UART_ID_Z025 0x19 #define MEN_UART_ID_Z057 0x39 diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c index a2a03acb04ad3e28b601257dea88fa5469f2a722..2cc78a4bf7a19cd435fc561330ba1b0908eaf690 100644 --- a/drivers/tty/serial/8250/8250_mid.c +++ b/drivers/tty/serial/8250/8250_mid.c @@ -206,9 +206,8 @@ static void dnv_exit(struct mid8250 *mid) /*****************************************************************************/ -static void mid8250_set_termios(struct uart_port *p, - struct ktermios *termios, - struct ktermios *old) +static void mid8250_set_termios(struct uart_port *p, struct ktermios *termios, + const struct ktermios *old) { unsigned int baud = tty_termios_baud_rate(termios); struct mid8250 *mid = p->private_data; diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c index 54051ec7b49927e57b730e48fb0123257ab9f716..fb1d5ec0940e65a3ff3b58d4c2283bc7bae72eed 100644 --- a/drivers/tty/serial/8250/8250_mtk.c +++ b/drivers/tty/serial/8250/8250_mtk.c @@ -291,7 +291,7 @@ static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode) static void mtk8250_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { static const unsigned short fraction_L_mapping[] = { 0, 1, 0x5, 0x15, 0x55, 0x57, 0x57, 0x77, 0x7F, 0xFF, 0xFF diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 0dcecbbc3967aa691756e9c3fbf6e919f79e9d75..41b8c6b27136a9cdb624e5c7c7c8bbedaeb26b53 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -342,6 +342,9 @@ static void omap8250_restore_regs(struct uart_8250_port *up) omap8250_update_mdr1(up, priv); up->port.ops->set_mctrl(&up->port, up->port.mctrl); + + if (up->port.rs485.flags & SER_RS485_ENABLED) + serial8250_em485_stop_tx(up); } /* @@ -350,7 +353,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) */ static void omap_8250_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_8250_port *up = up_to_u8250p(port); struct omap8250_priv *priv = up->port.private_data; @@ -984,9 +987,7 @@ static void omap_8250_dma_tx_complete(void *param) dma->tx_running = 0; - xmit->tail += dma->tx_size; - xmit->tail &= UART_XMIT_SIZE - 1; - p->port.icount.tx += dma->tx_size; + uart_xmit_advance(&p->port, dma->tx_size); if (priv->delayed_restore) { priv->delayed_restore = 0; @@ -1334,6 +1335,7 @@ static int omap8250_probe(struct platform_device *pdev) up.port.throttle = omap_8250_throttle; up.port.unthrottle = omap_8250_unthrottle; up.port.rs485_config = serial8250_em485_config; + up.port.rs485_supported = serial8250_em485_supported; up.rs485_start_tx = serial8250_em485_start_tx; up.rs485_stop_tx = serial8250_em485_stop_tx; up.port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE); diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c index 6f66dc2ebacc7acec27131b2b9f19b1dae24b350..8e9f247590bd4da1a0d7044f048d461e099f7958 100644 --- a/drivers/tty/serial/8250/8250_pci.c +++ b/drivers/tty/serial/8250/8250_pci.c @@ -1232,6 +1232,10 @@ static void pci_oxsemi_tornado_set_mctrl(struct uart_port *port, serial8250_do_set_mctrl(port, mctrl); } +/* + * We require EFR features for clock programming, so set UPF_FULL_PROBE + * for full probing regardless of CONFIG_SERIAL_8250_16550A_VARIANTS setting. + */ static int pci_oxsemi_tornado_setup(struct serial_private *priv, const struct pciserial_board *board, struct uart_8250_port *up, int idx) @@ -1239,6 +1243,7 @@ static int pci_oxsemi_tornado_setup(struct serial_private *priv, struct pci_dev *dev = priv->dev; if (pci_oxsemi_tornado_p(dev)) { + up->port.flags |= UPF_FULL_PROBE; up->port.get_divisor = pci_oxsemi_tornado_get_divisor; up->port.set_divisor = pci_oxsemi_tornado_set_divisor; up->port.set_mctrl = pci_oxsemi_tornado_set_mctrl; @@ -1627,7 +1632,6 @@ static int pci_fintek_init(struct pci_dev *dev) resource_size_t bar_data[3]; u8 config_base; struct serial_private *priv = pci_get_drvdata(dev); - struct uart_8250_port *port; if (!(pci_resource_flags(dev, 5) & IORESOURCE_IO) || !(pci_resource_flags(dev, 4) & IORESOURCE_IO) || @@ -1674,13 +1678,7 @@ static int pci_fintek_init(struct pci_dev *dev) pci_write_config_byte(dev, config_base + 0x06, dev->irq); - if (priv) { - /* re-apply RS232/485 mode when - * pciserial_resume_ports() - */ - port = serial8250_get_port(priv->line[i]); - uart_rs485_config(&port->port); - } else { + if (!priv) { /* First init without port data * force init to RS232 Mode */ diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 39b35a61958c04e43e1862b84f9ff7829324cbf5..fe8662cd9402411ec4be7dd6f7e161ae4e7725bb 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -600,7 +600,7 @@ EXPORT_SYMBOL_GPL(serial8250_rpm_put); static int serial8250_em485_init(struct uart_8250_port *p) { if (p->em485) - return 0; + goto deassert_rts; p->em485 = kmalloc(sizeof(struct uart_8250_em485), GFP_ATOMIC); if (!p->em485) @@ -616,7 +616,9 @@ static int serial8250_em485_init(struct uart_8250_port *p) p->em485->active_timer = NULL; p->em485->tx_stopped = true; - p->rs485_stop_tx(p); +deassert_rts: + if (p->em485->tx_stopped) + p->rs485_stop_tx(p); return 0; } @@ -752,6 +754,14 @@ static void serial8250_set_sleep(struct uart_8250_port *p, int sleep) serial8250_rpm_put(p); } +static void serial8250_clear_IER(struct uart_8250_port *up) +{ + if (up->capabilities & UART_CAP_UUE) + serial_out(up, UART_IER, UART_IER_UUE); + else + serial_out(up, UART_IER, 0); +} + #ifdef CONFIG_SERIAL_8250_RSA /* * Attempts to turn on the RSA FIFO. Returns zero on failure. @@ -1021,7 +1031,8 @@ static void autoconfig_16550a(struct uart_8250_port *up) up->port.type = PORT_16550A; up->capabilities |= UART_CAP_FIFO; - if (!IS_ENABLED(CONFIG_SERIAL_8250_16550A_VARIANTS)) + if (!IS_ENABLED(CONFIG_SERIAL_8250_16550A_VARIANTS) && + !(up->port.flags & UPF_FULL_PROBE)) return; /* @@ -1133,7 +1144,7 @@ static void autoconfig_16550a(struct uart_8250_port *up) * internal UARTs. * We're going to explicitly set the UUE bit to 0 before * trying to write and read a 1 just to make sure it's not - * already a 1 and maybe locked there before we even start start. + * already a 1 and maybe locked there before we even start. */ iersave = serial_in(up, UART_IER); serial_out(up, UART_IER, iersave & ~UART_IER_UUE); @@ -1329,10 +1340,7 @@ static void autoconfig(struct uart_8250_port *up) serial8250_out_MCR(up, save_mcr); serial8250_clear_fifos(up); serial_in(up, UART_RX); - if (up->capabilities & UART_CAP_UUE) - serial_out(up, UART_IER, UART_IER_UUE); - else - serial_out(up, UART_IER, 0); + serial8250_clear_IER(up); out_unlock: spin_unlock_irqrestore(&port->lock, flags); @@ -2042,6 +2050,9 @@ EXPORT_SYMBOL_GPL(serial8250_do_set_mctrl); static void serial8250_set_mctrl(struct uart_port *port, unsigned int mctrl) { + if (port->rs485.flags & SER_RS485_ENABLED) + return; + if (port->set_mctrl) port->set_mctrl(port, mctrl); else @@ -2142,10 +2153,7 @@ static void serial8250_put_poll_char(struct uart_port *port, * First save the IER then disable the interrupts */ ier = serial_port_in(port, UART_IER); - if (up->capabilities & UART_CAP_UUE) - serial_port_out(port, UART_IER, UART_IER_UUE); - else - serial_port_out(port, UART_IER, 0); + serial8250_clear_IER(up); wait_for_xmitr(up, UART_LSR_BOTH_EMPTY); /* @@ -2294,6 +2302,10 @@ int serial8250_do_startup(struct uart_port *port) if (port->irq && (up->port.flags & UPF_SHARE_IRQ)) up->port.irqflags |= IRQF_SHARED; + retval = up->ops->setup_irq(up); + if (retval) + goto out; + if (port->irq && !(up->port.flags & UPF_NO_THRE_TEST)) { unsigned char iir1; @@ -2336,9 +2348,7 @@ int serial8250_do_startup(struct uart_port *port) } } - retval = up->ops->setup_irq(up); - if (retval) - goto out; + up->ops->setup_timer(up); /* * Now, initialize the UART @@ -2651,7 +2661,7 @@ static void serial8250_set_divisor(struct uart_port *port, unsigned int baud, static unsigned int serial8250_get_baud_rate(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int tolerance = port->uartclk / 100; unsigned int min; @@ -2737,7 +2747,7 @@ EXPORT_SYMBOL_GPL(serial8250_update_uartclk); void serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_8250_port *up = up_to_u8250p(port); unsigned char cval; @@ -2875,7 +2885,7 @@ EXPORT_SYMBOL(serial8250_do_set_termios); static void serial8250_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { if (port->set_termios) port->set_termios(port, termios, old); @@ -3187,9 +3197,6 @@ static void serial8250_config_port(struct uart_port *port, int flags) if (flags & UART_CONFIG_TYPE) autoconfig(up); - if (port->rs485.flags & SER_RS485_ENABLED) - uart_rs485_config(port); - /* if access method is AU, it is a 16550 with a quirk */ if (port->type == PORT_16550A && port->iotype == UPIO_AU) up->bugs |= UART_BUG_NOMSR; @@ -3314,8 +3321,13 @@ static void serial8250_console_restore(struct uart_8250_port *up) unsigned int baud, quot, frac = 0; termios.c_cflag = port->cons->cflag; - if (port->state->port.tty && termios.c_cflag == 0) + termios.c_ispeed = port->cons->ispeed; + termios.c_ospeed = port->cons->ospeed; + if (port->state->port.tty && termios.c_cflag == 0) { termios.c_cflag = port->state->port.tty->termios.c_cflag; + termios.c_ispeed = port->state->port.tty->termios.c_ispeed; + termios.c_ospeed = port->state->port.tty->termios.c_ospeed; + } baud = serial8250_get_baud_rate(port, &termios, NULL); quot = serial8250_get_divisor(port, baud, &frac); @@ -3383,11 +3395,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, * First save the IER then disable the interrupts */ ier = serial_port_in(port, UART_IER); - - if (up->capabilities & UART_CAP_UUE) - serial_port_out(port, UART_IER, UART_IER_UUE); - else - serial_port_out(port, UART_IER, 0); + serial8250_clear_IER(up); /* check scratch reg to see if port powered off during system sleep */ if (up->canary && (up->canary != serial_port_in(port, UART_SCR))) { diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index 877173907c5369a58d48f563612164f251ee8de5..434f83168546cfeafd0d4735045c9d7b8b2d93ab 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -127,6 +127,7 @@ config SERIAL_SB1250_DUART_CONSOLE config SERIAL_ATMEL bool "AT91 on-chip serial port support" + depends on COMMON_CLK depends on ARCH_AT91 || COMPILE_TEST select SERIAL_CORE select SERIAL_MCTRL_GPIO if GPIOLIB @@ -427,7 +428,7 @@ config SERIAL_PXA config SERIAL_PXA_NON8250 bool - depends on !SERIAL_8250 + depends on !SERIAL_8250 || COMPILE_TEST config SERIAL_PXA_CONSOLE bool "Console on PXA serial port (DEPRECATED)" @@ -602,21 +603,6 @@ config SERIAL_MUX_CONSOLE select SERIAL_CORE_CONSOLE default y -config PDC_CONSOLE - bool "PDC software console support" - depends on PARISC && !SERIAL_MUX && VT - help - Saying Y here will enable the software based PDC console to be - used as the system console. This is useful for machines in - which the hardware based console has not been written yet. The - following steps must be completed to use the PDC console: - - 1. create the device entry (mknod /dev/ttyB0 c 11 0) - 2. Edit the /etc/inittab to start a getty listening on /dev/ttyB0 - 3. Add device ttyB0 to /etc/securetty (if you want to log on as - root on this console.) - 4. Change the kernel command console parameter to: console=ttyB0 - config SERIAL_SUNSAB tristate "Sun Siemens SAB82532 serial support" depends on SPARC && PCI @@ -1083,8 +1069,8 @@ config SERIAL_TIMBERDALE config SERIAL_BCM63XX tristate "Broadcom BCM63xx/BCM33xx UART support" select SERIAL_CORE - depends on ARCH_BCM4908 || ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC || COMPILE_TEST - default ARCH_BCM4908 || ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC + depends on ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC || COMPILE_TEST + default ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC help This enables the driver for the onchip UART core found on the following chipsets: @@ -1325,7 +1311,7 @@ config SERIAL_FSL_LPUART config SERIAL_FSL_LPUART_CONSOLE bool "Console on Freescale lpuart serial port" - depends on SERIAL_FSL_LPUART=y + depends on SERIAL_FSL_LPUART select SERIAL_CORE_CONSOLE select SERIAL_EARLYCON help diff --git a/drivers/tty/serial/altera_jtaguart.c b/drivers/tty/serial/altera_jtaguart.c index cb791c5149a32e6dac62adecfa510c153c9840c3..c2d154d78e545904a3a6ac9c2d5ede39dda9be6e 100644 --- a/drivers/tty/serial/altera_jtaguart.c +++ b/drivers/tty/serial/altera_jtaguart.c @@ -9,6 +9,7 @@ * (C) Copyright 2010, Tobias Klauser */ +#include #include #include #include @@ -48,7 +49,6 @@ #define ALTERA_JTAGUART_CONTROL_WI_MSK 0x00000200 #define ALTERA_JTAGUART_CONTROL_AC_MSK 0x00000400 #define ALTERA_JTAGUART_CONTROL_WSPACE_MSK 0xFFFF0000 -#define ALTERA_JTAGUART_CONTROL_WSPACE_OFF 16 /* * Local per-uart structure. @@ -59,10 +59,19 @@ struct altera_jtaguart { unsigned long imr; /* Local IMR mirror */ }; +static unsigned int altera_jtaguart_tx_space(struct uart_port *port, u32 *ctlp) +{ + u32 ctl = readl(port->membase + ALTERA_JTAGUART_CONTROL_REG); + + if (ctlp) + *ctlp = ctl; + + return FIELD_GET(ALTERA_JTAGUART_CONTROL_WSPACE_MSK, ctl); +} + static unsigned int altera_jtaguart_tx_empty(struct uart_port *port) { - return (readl(port->membase + ALTERA_JTAGUART_CONTROL_REG) & - ALTERA_JTAGUART_CONTROL_WSPACE_MSK) ? TIOCSER_TEMT : 0; + return altera_jtaguart_tx_space(port, NULL) ? TIOCSER_TEMT : 0; } static unsigned int altera_jtaguart_get_mctrl(struct uart_port *port) @@ -106,8 +115,8 @@ static void altera_jtaguart_break_ctl(struct uart_port *port, int break_state) } static void altera_jtaguart_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { /* Just copy the old termios settings back */ if (old) @@ -150,9 +159,7 @@ static void altera_jtaguart_tx_chars(struct altera_jtaguart *pp) pending = uart_circ_chars_pending(xmit); if (pending > 0) { - count = (readl(port->membase + ALTERA_JTAGUART_CONTROL_REG) & - ALTERA_JTAGUART_CONTROL_WSPACE_MSK) >> - ALTERA_JTAGUART_CONTROL_WSPACE_OFF; + count = altera_jtaguart_tx_space(port, NULL); if (count > pending) count = pending; if (count > 0) { @@ -298,17 +305,17 @@ static struct altera_jtaguart altera_jtaguart_ports[ALTERA_JTAGUART_MAXPORTS]; #if defined(CONFIG_SERIAL_ALTERA_JTAGUART_CONSOLE_BYPASS) static void altera_jtaguart_console_putc(struct uart_port *port, unsigned char c) { - unsigned long status; unsigned long flags; + u32 status; spin_lock_irqsave(&port->lock, flags); - while (((status = readl(port->membase + ALTERA_JTAGUART_CONTROL_REG)) & - ALTERA_JTAGUART_CONTROL_WSPACE_MSK) == 0) { + while (!altera_jtaguart_tx_space(port, &status)) { + spin_unlock_irqrestore(&port->lock, flags); + if ((status & ALTERA_JTAGUART_CONTROL_AC_MSK) == 0) { - spin_unlock_irqrestore(&port->lock, flags); return; /* no connection activity */ } - spin_unlock_irqrestore(&port->lock, flags); + cpu_relax(); spin_lock_irqsave(&port->lock, flags); } @@ -321,8 +328,7 @@ static void altera_jtaguart_console_putc(struct uart_port *port, unsigned char c unsigned long flags; spin_lock_irqsave(&port->lock, flags); - while ((readl(port->membase + ALTERA_JTAGUART_CONTROL_REG) & - ALTERA_JTAGUART_CONTROL_WSPACE_MSK) == 0) { + while (!altera_jtaguart_tx_space(port, NULL)) { spin_unlock_irqrestore(&port->lock, flags); cpu_relax(); spin_lock_irqsave(&port->lock, flags); diff --git a/drivers/tty/serial/altera_uart.c b/drivers/tty/serial/altera_uart.c index 8b749ed557c67e15741eff8b196a23a1f9617aca..82f2790de28d198e5611f571cfadd2610505bcf0 100644 --- a/drivers/tty/serial/altera_uart.c +++ b/drivers/tty/serial/altera_uart.c @@ -175,7 +175,7 @@ static void altera_uart_break_ctl(struct uart_port *port, int break_state) static void altera_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, baudclk; @@ -199,9 +199,8 @@ static void altera_uart_set_termios(struct uart_port *port, */ } -static void altera_uart_rx_chars(struct altera_uart *pp) +static void altera_uart_rx_chars(struct uart_port *port) { - struct uart_port *port = &pp->port; unsigned char ch, flag; unsigned short status; @@ -246,9 +245,8 @@ static void altera_uart_rx_chars(struct altera_uart *pp) tty_flip_buffer_push(&port->state->port); } -static void altera_uart_tx_chars(struct altera_uart *pp) +static void altera_uart_tx_chars(struct uart_port *port) { - struct uart_port *port = &pp->port; struct circ_buf *xmit = &port->state->xmit; if (port->x_char) { @@ -272,10 +270,8 @@ static void altera_uart_tx_chars(struct altera_uart *pp) if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) uart_write_wakeup(port); - if (xmit->head == xmit->tail) { - pp->imr &= ~ALTERA_UART_CONTROL_TRDY_MSK; - altera_uart_update_ctrl_reg(pp); - } + if (uart_circ_empty(xmit)) + altera_uart_stop_tx(port); } static irqreturn_t altera_uart_interrupt(int irq, void *data) @@ -288,9 +284,9 @@ static irqreturn_t altera_uart_interrupt(int irq, void *data) spin_lock(&port->lock); if (isr & ALTERA_UART_STATUS_RRDY_MSK) - altera_uart_rx_chars(pp); + altera_uart_rx_chars(port); if (isr & ALTERA_UART_STATUS_TRDY_MSK) - altera_uart_tx_chars(pp); + altera_uart_tx_chars(port); spin_unlock(&port->lock); return IRQ_RETVAL(isr); diff --git a/drivers/tty/serial/amba-pl010.c b/drivers/tty/serial/amba-pl010.c index fae0b581ff422523f27afc9f6e8b4b92f895231f..af27fb8ec145161cb190e69c012d4da5264ac78d 100644 --- a/drivers/tty/serial/amba-pl010.c +++ b/drivers/tty/serial/amba-pl010.c @@ -370,7 +370,7 @@ static void pl010_shutdown(struct uart_port *port) static void pl010_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int lcr_h, old_cr; unsigned long flags; diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 15f0e4d88c5a0346d1506fb409b468cee84664f5..5cdced39eafdb92758d5b4f79a123e2d76bf5f6b 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2030,7 +2030,7 @@ pl011_setup_status_masks(struct uart_port *port, struct ktermios *termios) static void pl011_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_amba_port *uap = container_of(port, struct uart_amba_port, port); @@ -2162,7 +2162,7 @@ pl011_set_termios(struct uart_port *port, struct ktermios *termios, static void sbsa_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_amba_port *uap = container_of(port, struct uart_amba_port, port); @@ -2777,6 +2777,7 @@ static int pl011_probe(struct amba_device *dev, const struct amba_id *id) struct uart_amba_port *uap; struct vendor_data *vendor = id->data; int portnr, ret; + u32 val; portnr = pl011_find_free_port(); if (portnr < 0) @@ -2801,6 +2802,21 @@ static int pl011_probe(struct amba_device *dev, const struct amba_id *id) uap->port.rs485_supported = pl011_rs485_supported; snprintf(uap->type, sizeof(uap->type), "PL011 rev%u", amba_rev(dev)); + if (device_property_read_u32(&dev->dev, "reg-io-width", &val) == 0) { + switch (val) { + case 1: + uap->port.iotype = UPIO_MEM; + break; + case 4: + uap->port.iotype = UPIO_MEM32; + break; + default: + dev_warn(&dev->dev, "unsupported reg-io-width (%d)\n", + val); + return -EINVAL; + } + } + ret = pl011_setup_port(&dev->dev, uap, &dev->res, portnr); if (ret) return ret; diff --git a/drivers/tty/serial/apbuart.c b/drivers/tty/serial/apbuart.c index 9ef82d870ff25a1b0464fcc3e9e5a796731ebe64..450f4edfda0f00a1fb36910d26b2ad5202489015 100644 --- a/drivers/tty/serial/apbuart.c +++ b/drivers/tty/serial/apbuart.c @@ -228,7 +228,7 @@ static void apbuart_shutdown(struct uart_port *port) } static void apbuart_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, const struct ktermios *old) { unsigned int cr; unsigned long flags; diff --git a/drivers/tty/serial/ar933x_uart.c b/drivers/tty/serial/ar933x_uart.c index 32caeac1298587fe52c7d4884e906163c560e8a6..925484a42c821e8ea70acbefe443b164f5001d4e 100644 --- a/drivers/tty/serial/ar933x_uart.c +++ b/drivers/tty/serial/ar933x_uart.c @@ -283,7 +283,7 @@ static void ar933x_uart_get_scale_step(unsigned int clk, static void ar933x_uart_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { struct ar933x_uart_port *up = container_of(port, struct ar933x_uart_port, port); @@ -583,6 +583,13 @@ static const struct uart_ops ar933x_uart_ops = { static int ar933x_config_rs485(struct uart_port *port, struct ktermios *termios, struct serial_rs485 *rs485conf) { + struct ar933x_uart_port *up = + container_of(port, struct ar933x_uart_port, port); + + if (port->rs485.flags & SER_RS485_ENABLED) + gpiod_set_value(up->rts_gpiod, + !!(rs485conf->flags & SER_RS485_RTS_AFTER_SEND)); + return 0; } diff --git a/drivers/tty/serial/arc_uart.c b/drivers/tty/serial/arc_uart.c index 2a09e92ef9ede92fc01cbeb9495607f281642719..2a65ea2660e10c5d903a19a46511b15d45c4514c 100644 --- a/drivers/tty/serial/arc_uart.c +++ b/drivers/tty/serial/arc_uart.c @@ -351,7 +351,7 @@ static void arc_serial_shutdown(struct uart_port *port) static void arc_serial_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { struct arc_uart_port *uart = to_arc_port(port); unsigned int baud, uartl, uarth, hw_val; diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c index 30ba9eef7b395d302507d6cb67f99ef7a1924480..bd07f79a2df91b2263df5d0f1f17cbb1343e304f 100644 --- a/drivers/tty/serial/atmel_serial.c +++ b/drivers/tty/serial/atmel_serial.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,7 @@ struct atmel_uart_char { struct atmel_uart_port { struct uart_port uart; /* uart */ struct clk *clk; /* uart clock */ + struct clk *gclk; /* uart generic clock */ int may_wakeup; /* cached value of device_may_wakeup for times we need to disable it */ u32 backup_imr; /* IMR saved during suspend */ int break_active; /* break being received */ @@ -150,6 +152,7 @@ struct atmel_uart_port { u32 rts_low; bool ms_irq_enabled; u32 rtor; /* address of receiver timeout register if it exists */ + bool is_usart; bool has_frac_baudrate; bool has_hw_timer; struct timer_list uart_timer; @@ -228,6 +231,11 @@ static inline int atmel_uart_is_half_duplex(struct uart_port *port) (port->iso7816.flags & SER_ISO7816_ENABLED); } +static inline int atmel_error_rate(int desired_value, int actual_value) +{ + return 100 - (desired_value * 100) / actual_value; +} + #ifdef CONFIG_SERIAL_ATMEL_PDC static bool atmel_use_pdc_rx(struct uart_port *port) { @@ -294,9 +302,6 @@ static int atmel_config_rs485(struct uart_port *port, struct ktermios *termios, mode = atmel_uart_readl(port, ATMEL_US_MR); - /* Resetting serial mode to RS232 (0x0) */ - mode &= ~ATMEL_US_USMODE; - if (rs485conf->flags & SER_RS485_ENABLED) { dev_dbg(port->dev, "Setting UART to RS485\n"); if (rs485conf->flags & SER_RS485_RX_DURING_TX) @@ -306,6 +311,7 @@ static int atmel_config_rs485(struct uart_port *port, struct ktermios *termios, atmel_uart_writel(port, ATMEL_US_TTGR, rs485conf->delay_rts_after_send); + mode &= ~ATMEL_US_USMODE; mode |= ATMEL_US_USMODE_RS485; } else { dev_dbg(port->dev, "Setting UART to RS232\n"); @@ -1827,6 +1833,7 @@ static void atmel_get_ip_name(struct uart_port *port) */ atmel_port->has_frac_baudrate = false; atmel_port->has_hw_timer = false; + atmel_port->is_usart = false; if (name == new_uart) { dev_dbg(port->dev, "Uart with hw timer"); @@ -1836,6 +1843,7 @@ static void atmel_get_ip_name(struct uart_port *port) dev_dbg(port->dev, "Usart\n"); atmel_port->has_frac_baudrate = true; atmel_port->has_hw_timer = true; + atmel_port->is_usart = true; atmel_port->rtor = ATMEL_US_RTOR; version = atmel_uart_readl(port, ATMEL_US_VERSION); switch (version) { @@ -1865,6 +1873,7 @@ static void atmel_get_ip_name(struct uart_port *port) dev_dbg(port->dev, "This version is usart\n"); atmel_port->has_frac_baudrate = true; atmel_port->has_hw_timer = true; + atmel_port->is_usart = true; atmel_port->rtor = ATMEL_US_RTOR; break; case 0x203: @@ -2115,6 +2124,8 @@ static void atmel_serial_pm(struct uart_port *port, unsigned int state, * This is called on uart_close() or a suspend event. */ clk_disable_unprepare(atmel_port->clk); + if (__clk_is_enabled(atmel_port->gclk)) + clk_disable_unprepare(atmel_port->gclk); break; default: dev_err(port->dev, "atmel_serial: unknown pm %d\n", state); @@ -2124,19 +2135,25 @@ static void atmel_serial_pm(struct uart_port *port, unsigned int state, /* * Change the port parameters */ -static void atmel_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) +static void atmel_set_termios(struct uart_port *port, + struct ktermios *termios, + const struct ktermios *old) { struct atmel_uart_port *atmel_port = to_atmel_uart_port(port); unsigned long flags; - unsigned int old_mode, mode, imr, quot, baud, div, cd, fp = 0; + unsigned int old_mode, mode, imr, quot, div, cd, fp = 0; + unsigned int baud, actual_baud, gclk_rate; + int ret; /* save the current mode register */ mode = old_mode = atmel_uart_readl(port, ATMEL_US_MR); /* reset the mode, clock divisor, parity, stop bits and data size */ - mode &= ~(ATMEL_US_USCLKS | ATMEL_US_CHRL | ATMEL_US_NBSTOP | - ATMEL_US_PAR | ATMEL_US_USMODE); + if (atmel_port->is_usart) + mode &= ~(ATMEL_US_NBSTOP | ATMEL_US_PAR | ATMEL_US_CHRL | + ATMEL_US_USCLKS | ATMEL_US_USMODE); + else + mode &= ~(ATMEL_UA_BRSRCCK | ATMEL_US_PAR | ATMEL_UA_FILTER); baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16); @@ -2284,10 +2301,60 @@ static void atmel_set_termios(struct uart_port *port, struct ktermios *termios, cd = uart_get_divisor(port, baud); } - if (cd > 65535) { /* BRGR is 16-bit, so switch to slower clock */ + /* + * If the current value of the Clock Divisor surpasses the 16 bit + * ATMEL_US_CD mask and the IP is USART, switch to the Peripheral + * Clock implicitly divided by 8. + * If the IP is UART however, keep the highest possible value for + * the CD and avoid needless division of CD, since UART IP's do not + * support implicit division of the Peripheral Clock. + */ + if (atmel_port->is_usart && cd > ATMEL_US_CD) { cd /= 8; mode |= ATMEL_US_USCLKS_MCK_DIV8; + } else { + cd = min_t(unsigned int, cd, ATMEL_US_CD); + } + + /* + * If there is no Fractional Part, there is a high chance that + * we may be able to generate a baudrate closer to the desired one + * if we use the GCLK as the clock source driving the baudrate + * generator. + */ + if (!atmel_port->has_frac_baudrate) { + if (__clk_is_enabled(atmel_port->gclk)) + clk_disable_unprepare(atmel_port->gclk); + gclk_rate = clk_round_rate(atmel_port->gclk, 16 * baud); + actual_baud = clk_get_rate(atmel_port->clk) / (16 * cd); + if (gclk_rate && abs(atmel_error_rate(baud, actual_baud)) > + abs(atmel_error_rate(baud, gclk_rate / 16))) { + clk_set_rate(atmel_port->gclk, 16 * baud); + ret = clk_prepare_enable(atmel_port->gclk); + if (ret) + goto gclk_fail; + + if (atmel_port->is_usart) { + mode &= ~ATMEL_US_USCLKS; + mode |= ATMEL_US_USCLKS_GCLK; + } else { + mode |= ATMEL_UA_BRSRCCK; + } + + /* + * Set the Clock Divisor for GCLK to 1. + * Since we were able to generate the smallest + * multiple of the desired baudrate times 16, + * then we surely can generate a bigger multiple + * with the exact error rate for an equally increased + * CD. Thus no need to take into account + * a higher value for CD. + */ + cd = 1; + } } + +gclk_fail: quot = cd | fp << ATMEL_US_FP_OFFSET; if (!(port->iso7816.flags & SER_ISO7816_ENABLED)) @@ -2883,6 +2950,12 @@ static int atmel_serial_probe(struct platform_device *pdev) if (ret) goto err; + atmel_port->gclk = devm_clk_get_optional(&pdev->dev, "gclk"); + if (IS_ERR(atmel_port->gclk)) { + ret = PTR_ERR(atmel_port->gclk); + goto err_clk_disable_unprepare; + } + ret = atmel_init_port(atmel_port, pdev); if (ret) goto err_clk_disable_unprepare; diff --git a/drivers/tty/serial/atmel_serial.h b/drivers/tty/serial/atmel_serial.h index 0d8a0f9cc5c33330a644f670f4b6be1d26d0aa06..87f8f79963070c7d09ee3c9a58bba3445f58860c 100644 --- a/drivers/tty/serial/atmel_serial.h +++ b/drivers/tty/serial/atmel_serial.h @@ -9,6 +9,8 @@ * Based on AT91RM9200 datasheet revision E. */ +#include + #ifndef ATMEL_SERIAL_H #define ATMEL_SERIAL_H @@ -39,39 +41,42 @@ #define ATMEL_US_MR 0x04 /* Mode Register */ #define ATMEL_US_USMODE GENMASK(3, 0) /* Mode of the USART */ -#define ATMEL_US_USMODE_NORMAL 0 -#define ATMEL_US_USMODE_RS485 1 -#define ATMEL_US_USMODE_HWHS 2 -#define ATMEL_US_USMODE_MODEM 3 -#define ATMEL_US_USMODE_ISO7816_T0 4 -#define ATMEL_US_USMODE_ISO7816_T1 6 -#define ATMEL_US_USMODE_IRDA 8 +#define ATMEL_US_USMODE_NORMAL FIELD_PREP(ATMEL_US_USMODE, 0) +#define ATMEL_US_USMODE_RS485 FIELD_PREP(ATMEL_US_USMODE, 1) +#define ATMEL_US_USMODE_HWHS FIELD_PREP(ATMEL_US_USMODE, 2) +#define ATMEL_US_USMODE_MODEM FIELD_PREP(ATMEL_US_USMODE, 3) +#define ATMEL_US_USMODE_ISO7816_T0 FIELD_PREP(ATMEL_US_USMODE, 4) +#define ATMEL_US_USMODE_ISO7816_T1 FIELD_PREP(ATMEL_US_USMODE, 6) +#define ATMEL_US_USMODE_IRDA FIELD_PREP(ATMEL_US_USMODE, 8) #define ATMEL_US_USCLKS GENMASK(5, 4) /* Clock Selection */ -#define ATMEL_US_USCLKS_MCK (0 << 4) -#define ATMEL_US_USCLKS_MCK_DIV8 (1 << 4) -#define ATMEL_US_USCLKS_SCK (3 << 4) +#define ATMEL_US_USCLKS_MCK FIELD_PREP(ATMEL_US_USCLKS, 0) +#define ATMEL_US_USCLKS_MCK_DIV8 FIELD_PREP(ATMEL_US_USCLKS, 1) +#define ATMEL_US_USCLKS_GCLK FIELD_PREP(ATMEL_US_USCLKS, 2) +#define ATMEL_US_USCLKS_SCK FIELD_PREP(ATMEL_US_USCLKS, 3) +#define ATMEL_UA_FILTER BIT(4) #define ATMEL_US_CHRL GENMASK(7, 6) /* Character Length */ -#define ATMEL_US_CHRL_5 (0 << 6) -#define ATMEL_US_CHRL_6 (1 << 6) -#define ATMEL_US_CHRL_7 (2 << 6) -#define ATMEL_US_CHRL_8 (3 << 6) +#define ATMEL_US_CHRL_5 FIELD_PREP(ATMEL_US_CHRL, 0) +#define ATMEL_US_CHRL_6 FIELD_PREP(ATMEL_US_CHRL, 1) +#define ATMEL_US_CHRL_7 FIELD_PREP(ATMEL_US_CHRL, 2) +#define ATMEL_US_CHRL_8 FIELD_PREP(ATMEL_US_CHRL, 3) #define ATMEL_US_SYNC BIT(8) /* Synchronous Mode Select */ #define ATMEL_US_PAR GENMASK(11, 9) /* Parity Type */ -#define ATMEL_US_PAR_EVEN (0 << 9) -#define ATMEL_US_PAR_ODD (1 << 9) -#define ATMEL_US_PAR_SPACE (2 << 9) -#define ATMEL_US_PAR_MARK (3 << 9) -#define ATMEL_US_PAR_NONE (4 << 9) -#define ATMEL_US_PAR_MULTI_DROP (6 << 9) +#define ATMEL_US_PAR_EVEN FIELD_PREP(ATMEL_US_PAR, 0) +#define ATMEL_US_PAR_ODD FIELD_PREP(ATMEL_US_PAR, 1) +#define ATMEL_US_PAR_SPACE FIELD_PREP(ATMEL_US_PAR, 2) +#define ATMEL_US_PAR_MARK FIELD_PREP(ATMEL_US_PAR, 3) +#define ATMEL_US_PAR_NONE FIELD_PREP(ATMEL_US_PAR, 4) +#define ATMEL_US_PAR_MULTI_DROP FIELD_PREP(ATMEL_US_PAR, 6) #define ATMEL_US_NBSTOP GENMASK(13, 12) /* Number of Stop Bits */ -#define ATMEL_US_NBSTOP_1 (0 << 12) -#define ATMEL_US_NBSTOP_1_5 (1 << 12) -#define ATMEL_US_NBSTOP_2 (2 << 12) +#define ATMEL_US_NBSTOP_1 FIELD_PREP(ATMEL_US_NBSTOP, 0) +#define ATMEL_US_NBSTOP_1_5 FIELD_PREP(ATMEL_US_NBSTOP, 1) +#define ATMEL_US_NBSTOP_2 FIELD_PREP(ATMEL_US_NBSTOP, 2) +#define ATMEL_UA_BRSRCCK BIT(12) /* Clock Selection for UART */ #define ATMEL_US_CHMODE GENMASK(15, 14) /* Channel Mode */ -#define ATMEL_US_CHMODE_NORMAL (0 << 14) -#define ATMEL_US_CHMODE_ECHO (1 << 14) -#define ATMEL_US_CHMODE_LOC_LOOP (2 << 14) -#define ATMEL_US_CHMODE_REM_LOOP (3 << 14) +#define ATMEL_US_CHMODE_NORMAL FIELD_PREP(ATMEL_US_CHMODE, 0) +#define ATMEL_US_CHMODE_ECHO FIELD_PREP(ATMEL_US_CHMODE, 1) +#define ATMEL_US_CHMODE_LOC_LOOP FIELD_PREP(ATMEL_US_CHMODE, 2) +#define ATMEL_US_CHMODE_REM_LOOP FIELD_PREP(ATMEL_US_CHMODE, 3) #define ATMEL_US_MSBF BIT(16) /* Bit Order */ #define ATMEL_US_MODE9 BIT(17) /* 9-bit Character Length */ #define ATMEL_US_CLKO BIT(18) /* Clock Output Select */ @@ -79,7 +84,7 @@ #define ATMEL_US_INACK BIT(20) /* Inhibit Non Acknowledge */ #define ATMEL_US_DSNACK BIT(21) /* Disable Successive NACK */ #define ATMEL_US_MAX_ITER_MASK GENMASK(26, 24) /* Max Iterations */ -#define ATMEL_US_MAX_ITER(n) (((n) << 24) & ATMEL_US_MAX_ITER_MASK) +#define ATMEL_US_MAX_ITER(n) FIELD_PREP(ATMEL_US_MAX_ITER_MASK, (n)) #define ATMEL_US_FILTER BIT(28) /* Infrared Receive Line Filter */ #define ATMEL_US_IER 0x08 /* Interrupt Enable Register */ @@ -131,19 +136,19 @@ #define ATMEL_US_CMPR 0x90 /* Comparaison Register */ #define ATMEL_US_FMR 0xa0 /* FIFO Mode Register */ -#define ATMEL_US_TXRDYM(data) (((data) & 0x3) << 0) /* TX Ready Mode */ -#define ATMEL_US_RXRDYM(data) (((data) & 0x3) << 4) /* RX Ready Mode */ +#define ATMEL_US_TXRDYM(data) FIELD_PREP(GENMASK(1, 0), (data)) /* TX Ready Mode */ +#define ATMEL_US_RXRDYM(data) FIELD_PREP(GENMASK(5, 4), (data)) /* RX Ready Mode */ #define ATMEL_US_ONE_DATA 0x0 #define ATMEL_US_TWO_DATA 0x1 #define ATMEL_US_FOUR_DATA 0x2 #define ATMEL_US_FRTSC BIT(7) /* FIFO RTS pin Control */ -#define ATMEL_US_TXFTHRES(thr) (((thr) & 0x3f) << 8) /* TX FIFO Threshold */ -#define ATMEL_US_RXFTHRES(thr) (((thr) & 0x3f) << 16) /* RX FIFO Threshold */ -#define ATMEL_US_RXFTHRES2(thr) (((thr) & 0x3f) << 24) /* RX FIFO Threshold2 */ +#define ATMEL_US_TXFTHRES(thr) FIELD_PREP(GENMASK(13, 8), (thr)) /* TX FIFO Threshold */ +#define ATMEL_US_RXFTHRES(thr) FIELD_PREP(GENMASK(21, 16), (thr)) /* RX FIFO Threshold */ +#define ATMEL_US_RXFTHRES2(thr) FIELD_PREP(GENMASK(29, 24), (thr)) /* RX FIFO Threshold2 */ #define ATMEL_US_FLR 0xa4 /* FIFO Level Register */ -#define ATMEL_US_TXFL(reg) (((reg) >> 0) & 0x3f) /* TX FIFO Level */ -#define ATMEL_US_RXFL(reg) (((reg) >> 16) & 0x3f) /* RX FIFO Level */ +#define ATMEL_US_TXFL(reg) FIELD_GET(GENMASK(5, 0), (reg)) /* TX FIFO Level */ +#define ATMEL_US_RXFL(reg) FIELD_GET(GENMASK(21, 16), (reg)) /* RX FIFO Level */ #define ATMEL_US_FIER 0xa8 /* FIFO Interrupt Enable Register */ #define ATMEL_US_FIDR 0xac /* FIFO Interrupt Disable Register */ diff --git a/drivers/tty/serial/bcm63xx_uart.c b/drivers/tty/serial/bcm63xx_uart.c index 53b43174aa40955c4d271690efec3e912d7fa790..5d9737c2d1f2c8ec0f2ef1bd73061ad9aea623e4 100644 --- a/drivers/tty/serial/bcm63xx_uart.c +++ b/drivers/tty/serial/bcm63xx_uart.c @@ -492,9 +492,8 @@ static void bcm_uart_shutdown(struct uart_port *port) /* * serial core request to change current uart setting */ -static void bcm_uart_set_termios(struct uart_port *port, - struct ktermios *new, - struct ktermios *old) +static void bcm_uart_set_termios(struct uart_port *port, struct ktermios *new, + const struct ktermios *old) { unsigned int ctl, baud, quot, ier; unsigned long flags; diff --git a/drivers/tty/serial/clps711x.c b/drivers/tty/serial/clps711x.c index b9b66ad31a082b125cc5249637d56e85ac3ef422..404b43a5ae33fc94935a892f073c145470c25926 100644 --- a/drivers/tty/serial/clps711x.c +++ b/drivers/tty/serial/clps711x.c @@ -251,7 +251,7 @@ static void uart_clps711x_shutdown(struct uart_port *port) static void uart_clps711x_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { u32 ubrlcr; unsigned int baud, quot; diff --git a/drivers/tty/serial/cpm_uart/cpm_uart.h b/drivers/tty/serial/cpm_uart/cpm_uart.h index 8c582779cf224b5bb84c1fb520b950c4343a8353..0577618e78c041af601b2f238f86b67de4ab2f1a 100644 --- a/drivers/tty/serial/cpm_uart/cpm_uart.h +++ b/drivers/tty/serial/cpm_uart/cpm_uart.h @@ -87,7 +87,6 @@ struct uart_cpm_port { struct gpio_desc *gpios[NUM_GPIOS]; }; -extern int cpm_uart_nr; extern struct uart_cpm_port cpm_uart_ports[UART_NR]; /* these are located in their respective files */ diff --git a/drivers/tty/serial/cpm_uart/cpm_uart_core.c b/drivers/tty/serial/cpm_uart/cpm_uart_core.c index db07d6a5d764d60379a9d0617ae103087f2c762e..b4369ed45ae2d101db021e5adca4710d62ba2b51 100644 --- a/drivers/tty/serial/cpm_uart/cpm_uart_core.c +++ b/drivers/tty/serial/cpm_uart/cpm_uart_core.c @@ -484,12 +484,11 @@ static void cpm_uart_shutdown(struct uart_port *port) static void cpm_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { int baud; unsigned long flags; u16 cval, scval, prev_mode; - int bits, sbits; struct uart_cpm_port *pinfo = container_of(port, struct uart_cpm_port, port); smc_t __iomem *smcp = pinfo->smcp; @@ -515,28 +514,17 @@ static void cpm_uart_set_termios(struct uart_port *port, if (maxidl > 0x10) maxidl = 0x10; - /* Character length programmed into the mode register is the - * sum of: 1 start bit, number of data bits, 0 or 1 parity bit, - * 1 or 2 stop bits, minus 1. - * The value 'bits' counts this for us. - */ cval = 0; scval = 0; - /* byte size */ - bits = tty_get_char_size(termios->c_cflag); - sbits = bits - 5; - if (termios->c_cflag & CSTOPB) { cval |= SMCMR_SL; /* Two stops */ scval |= SCU_PSMR_SL; - bits++; } if (termios->c_cflag & PARENB) { cval |= SMCMR_PEN; scval |= SCU_PSMR_PEN; - bits++; if (!(termios->c_cflag & PARODD)) { cval |= SMCMR_PM_EVEN; scval |= (SCU_PSMR_REVP | SCU_PSMR_TEVP); @@ -580,12 +568,9 @@ static void cpm_uart_set_termios(struct uart_port *port, spin_lock_irqsave(&port->lock, flags); - /* Start bit has not been added (so don't, because we would just - * subtract it later), and we need to add one for the number of - * stops bits (there is always at least one). - */ - bits++; if (IS_SMC(pinfo)) { + unsigned int bits = tty_get_frame_size(termios->c_cflag); + /* * MRBLR can be changed while an SMC/SCC is operating only * if it is done in a single bus cycle with one 16-bit move @@ -604,13 +589,17 @@ static void cpm_uart_set_termios(struct uart_port *port, */ prev_mode = in_be16(&smcp->smc_smcmr) & (SMCMR_REN | SMCMR_TEN); /* Output in *one* operation, so we don't interrupt RX/TX if they - * were already enabled. */ - out_be16(&smcp->smc_smcmr, smcr_mk_clen(bits) | cval | - SMCMR_SM_UART | prev_mode); + * were already enabled. + * Character length programmed into the register is frame bits minus 1. + */ + out_be16(&smcp->smc_smcmr, smcr_mk_clen(bits - 1) | cval | + SMCMR_SM_UART | prev_mode); } else { + unsigned int bits = tty_get_char_size(termios->c_cflag); + out_be16(&pinfo->sccup->scc_genscc.scc_mrblr, pinfo->rx_fifosize); out_be16(&pinfo->sccup->scc_maxidl, maxidl); - out_be16(&sccp->scc_psmr, (sbits << 12) | scval); + out_be16(&sccp->scc_psmr, (UART_LCR_WLEN(bits) << 12) | scval); } if (pinfo->clk) @@ -1214,12 +1203,6 @@ static int cpm_uart_init_port(struct device_node *np, pinfo->port.fifosize = pinfo->tx_nrfifos * pinfo->tx_fifosize; spin_lock_init(&pinfo->port.lock); - pinfo->port.irq = irq_of_parse_and_map(np, 0); - if (pinfo->port.irq == NO_IRQ) { - ret = -EINVAL; - goto out_pram; - } - for (i = 0; i < NUM_GPIOS; i++) { struct gpio_desc *gpiod; @@ -1229,7 +1212,7 @@ static int cpm_uart_init_port(struct device_node *np, if (IS_ERR(gpiod)) { ret = PTR_ERR(gpiod); - goto out_irq; + goto out_pram; } if (gpiod) { @@ -1255,8 +1238,6 @@ static int cpm_uart_init_port(struct device_node *np, return cpm_uart_request_port(&pinfo->port); -out_irq: - irq_dispose_mapping(pinfo->port.irq); out_pram: cpm_uart_unmap_pram(pinfo, pram); out_mem: @@ -1436,11 +1417,17 @@ static int cpm_uart_probe(struct platform_device *ofdev) /* initialize the device pointer for the port */ pinfo->port.dev = &ofdev->dev; + pinfo->port.irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); + if (!pinfo->port.irq) + return -EINVAL; + ret = cpm_uart_init_port(ofdev->dev.of_node, pinfo); - if (ret) - return ret; + if (!ret) + return uart_add_one_port(&cpm_reg, &pinfo->port); + + irq_dispose_mapping(pinfo->port.irq); - return uart_add_one_port(&cpm_reg, &pinfo->port); + return ret; } static int cpm_uart_remove(struct platform_device *ofdev) diff --git a/drivers/tty/serial/digicolor-usart.c b/drivers/tty/serial/digicolor-usart.c index af951e6a2ef4e2d7fc4ec2a0fc03cc80c3eb2209..0c0a62346f23195b546f7707f8fab518ba2cb51d 100644 --- a/drivers/tty/serial/digicolor-usart.c +++ b/drivers/tty/serial/digicolor-usart.c @@ -287,7 +287,7 @@ static void digicolor_uart_shutdown(struct uart_port *port) static void digicolor_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud, divisor; u8 config = 0; diff --git a/drivers/tty/serial/dz.c b/drivers/tty/serial/dz.c index 2e21acf397202494e2b8c241374810ba6509c544..829b452daee9d4b813342484e5c7c8548a953c4e 100644 --- a/drivers/tty/serial/dz.c +++ b/drivers/tty/serial/dz.c @@ -559,7 +559,7 @@ static void dz_reset(struct dz_port *dport) } static void dz_set_termios(struct uart_port *uport, struct ktermios *termios, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct dz_port *dport = to_dport(uport); unsigned long flags; @@ -592,9 +592,12 @@ static void dz_set_termios(struct uart_port *uport, struct ktermios *termios, baud = uart_get_baud_rate(uport, termios, old_termios, 50, 9600); bflag = dz_encode_baud_rate(baud); - if (bflag < 0) { /* Try to keep unchanged. */ - baud = uart_get_baud_rate(uport, old_termios, NULL, 50, 9600); - bflag = dz_encode_baud_rate(baud); + if (bflag < 0) { + if (old_termios) { + /* Keep unchanged. */ + baud = tty_termios_baud_rate(old_termios); + bflag = dz_encode_baud_rate(baud); + } if (bflag < 0) { /* Resort to 9600. */ baud = 9600; bflag = DZ_B9600; diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c index 88d08ba1ca83752bbe446b50359df6ba2dd017db..a5f380584cdae7a1835495eb6e4fc7d6bd003783 100644 --- a/drivers/tty/serial/earlycon.c +++ b/drivers/tty/serial/earlycon.c @@ -67,7 +67,7 @@ static void __init earlycon_init(struct earlycon_device *device, if (*s) earlycon->index = simple_strtoul(s, NULL, 10); len = s - name; - strlcpy(earlycon->name, name, min(len + 1, sizeof(earlycon->name))); + strscpy(earlycon->name, name, min(len + 1, sizeof(earlycon->name))); earlycon->data = &early_console_dev; } @@ -123,7 +123,7 @@ static int __init parse_options(struct earlycon_device *device, char *options) device->baud = simple_strtoul(options, NULL, 0); length = min(strcspn(options, " ") + 1, (size_t)(sizeof(device->options))); - strlcpy(device->options, options, length); + strscpy(device->options, options, length); } return 0; @@ -304,7 +304,7 @@ int __init of_setup_earlycon(const struct earlycon_id *match, if (options) { early_console_dev.baud = simple_strtoul(options, NULL, 0); - strlcpy(early_console_dev.options, options, + strscpy(early_console_dev.options, options, sizeof(early_console_dev.options)); } earlycon_init(&early_console_dev, match->name); diff --git a/drivers/tty/serial/fsl_linflexuart.c b/drivers/tty/serial/fsl_linflexuart.c index 98bb0c315e133612c0dcc63564c12581576286f2..84e8153e54200f388520344b9ef55e8435af4cf6 100644 --- a/drivers/tty/serial/fsl_linflexuart.c +++ b/drivers/tty/serial/fsl_linflexuart.c @@ -401,7 +401,7 @@ static void linflex_shutdown(struct uart_port *port) static void linflex_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned long cr, old_cr, cr1; diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index f6c33cd228c8a07011cb477de017b1ab8b8ddadd..67fa113f77d475a621bc53bec3b33752abc0ad2a 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -1284,17 +1284,12 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport) struct dma_slave_config dma_rx_sconfig = {}; struct circ_buf *ring = &sport->rx_ring; int ret, nent; - int bits, baud; struct tty_port *port = &sport->port.state->port; struct tty_struct *tty = port->tty; struct ktermios *termios = &tty->termios; struct dma_chan *chan = sport->dma_rx_chan; - - baud = tty_get_baud_rate(tty); - - bits = (termios->c_cflag & CSIZE) == CS7 ? 9 : 10; - if (termios->c_cflag & PARENB) - bits++; + unsigned int bits = tty_get_frame_size(termios->c_cflag); + unsigned int baud = tty_get_baud_rate(tty); /* * Calculate length of one DMA buffer size to keep latency below @@ -1394,9 +1389,9 @@ static int lpuart_config_rs485(struct uart_port *port, struct ktermios *termios, * Note: UART is assumed to be active high. */ if (rs485->flags & SER_RS485_RTS_ON_SEND) - modem &= ~UARTMODEM_TXRTSPOL; - else if (rs485->flags & SER_RS485_RTS_AFTER_SEND) modem |= UARTMODEM_TXRTSPOL; + else if (rs485->flags & SER_RS485_RTS_AFTER_SEND) + modem &= ~UARTMODEM_TXRTSPOL; } writeb(modem, sport->port.membase + UARTMODEM); @@ -1776,6 +1771,7 @@ static void lpuart_dma_shutdown(struct lpuart_port *sport) if (sport->lpuart_dma_rx_use) { del_timer_sync(&sport->lpuart_timer); lpuart_dma_rx_free(&sport->port); + sport->lpuart_dma_rx_use = false; } if (sport->lpuart_dma_tx_use) { @@ -1784,6 +1780,7 @@ static void lpuart_dma_shutdown(struct lpuart_port *sport) sport->dma_tx_in_progress = false; dmaengine_terminate_all(sport->dma_tx_chan); } + sport->lpuart_dma_tx_use = false; } if (sport->dma_tx_chan) @@ -1833,7 +1830,7 @@ static void lpuart32_shutdown(struct uart_port *port) static void lpuart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct lpuart_port *sport = container_of(port, struct lpuart_port, port); unsigned long flags; @@ -2073,7 +2070,7 @@ static void lpuart32_serial_setbrg(struct lpuart_port *sport, static void lpuart32_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct lpuart_port *sport = container_of(port, struct lpuart_port, port); unsigned long flags; @@ -2191,6 +2188,7 @@ lpuart32_set_termios(struct uart_port *port, struct ktermios *termios, uart_update_timeout(port, termios->c_cflag, baud); /* wait transmit engin complete */ + lpuart32_write(&sport->port, 0, UARTMODIR); lpuart32_wait_bit_set(&sport->port, UARTSTAT, UARTSTAT_TC); /* disable transmit and receive */ @@ -2723,9 +2721,6 @@ static int lpuart_probe(struct platform_device *pdev) lpuart_reg.cons = LPUART_CONSOLE; handler = lpuart_int; } - ret = uart_add_one_port(&lpuart_reg, &sport->port); - if (ret) - goto failed_attach_port; ret = lpuart_global_reset(sport); if (ret) @@ -2735,7 +2730,9 @@ static int lpuart_probe(struct platform_device *pdev) if (ret) goto failed_get_rs485; - uart_rs485_config(&sport->port); + ret = uart_add_one_port(&lpuart_reg, &sport->port); + if (ret) + goto failed_attach_port; ret = devm_request_irq(&pdev->dev, sport->port.irq, handler, 0, DRIVER_NAME, sport); @@ -2745,10 +2742,10 @@ static int lpuart_probe(struct platform_device *pdev) return 0; failed_irq_request: -failed_get_rs485: -failed_reset: uart_remove_one_port(&lpuart_reg, &sport->port); failed_attach_port: +failed_get_rs485: +failed_reset: lpuart_disable_clks(sport); return ret; } @@ -2798,7 +2795,7 @@ static int __maybe_unused lpuart_suspend(struct device *dev) * EDMA driver during suspend will forcefully release any * non-idle DMA channels. If port wakeup is enabled or if port * is console port or 'no_console_suspend' is set the Rx DMA - * cannot resume as as expected, hence gracefully release the + * cannot resume as expected, hence gracefully release the * Rx DMA path before suspend and start Rx DMA path on resume. */ if (irq_wake) { diff --git a/drivers/tty/serial/icom.c b/drivers/tty/serial/icom.c index 45df29947fe88848b6a999bf518fa109f845af45..819f957b6b849be626278f48701f26c3c9a7c82a 100644 --- a/drivers/tty/serial/icom.c +++ b/drivers/tty/serial/icom.c @@ -1351,9 +1351,8 @@ static void icom_close(struct uart_port *port) kref_put(&icom_port->adapter->kref, icom_kref_release); } -static void icom_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old_termios) +static void icom_set_termios(struct uart_port *port, struct ktermios *termios, + const struct ktermios *old_termios) { struct icom_port *icom_port = to_icom_port(port); int baud; diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 522445a8f666e5b09a7f2931c820d849d7162ff3..05b432dc7a85c79065efd53d1a2d6fe6bed27b6c 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -380,8 +380,7 @@ static void imx_uart_rts_active(struct imx_port *sport, u32 *ucr2) { *ucr2 &= ~(UCR2_CTSC | UCR2_CTS); - sport->port.mctrl |= TIOCM_RTS; - mctrl_gpio_set(sport->gpios, sport->port.mctrl); + mctrl_gpio_set(sport->gpios, sport->port.mctrl | TIOCM_RTS); } /* called with port.lock taken and irqs caller dependent */ @@ -390,8 +389,7 @@ static void imx_uart_rts_inactive(struct imx_port *sport, u32 *ucr2) *ucr2 &= ~UCR2_CTSC; *ucr2 |= UCR2_CTS; - sport->port.mctrl &= ~TIOCM_RTS; - mctrl_gpio_set(sport->gpios, sport->port.mctrl); + mctrl_gpio_set(sport->gpios, sport->port.mctrl & ~TIOCM_RTS); } static void start_hrtimer_ms(struct hrtimer *hrt, unsigned long msec) @@ -1620,7 +1618,7 @@ static void imx_uart_flush_buffer(struct uart_port *port) static void imx_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct imx_port *sport = (struct imx_port *)port; unsigned long flags; @@ -2347,8 +2345,6 @@ static int imx_uart_probe(struct platform_device *pdev) dev_err(&pdev->dev, "low-active RTS not possible when receiver is off, enabling receiver\n"); - uart_rs485_config(&sport->port); - /* Disable interrupts before requesting them */ ucr1 = imx_uart_readl(sport, UCR1); ucr1 &= ~(UCR1_ADEN | UCR1_TRDYEN | UCR1_IDEN | UCR1_RRDYEN | UCR1_RTSDEN); diff --git a/drivers/tty/serial/ip22zilog.c b/drivers/tty/serial/ip22zilog.c index 655e64b26852961eb38ab1e07254172ea9b51f4d..dd0a8915ce4ff1a25a6663d9be29fec00465176d 100644 --- a/drivers/tty/serial/ip22zilog.c +++ b/drivers/tty/serial/ip22zilog.c @@ -873,7 +873,7 @@ ip22zilog_convert_to_zs(struct uart_ip22zilog_port *up, unsigned int cflag, /* The port lock is not held. */ static void ip22zilog_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_ip22zilog_port *up = container_of(port, struct uart_ip22zilog_port, port); diff --git a/drivers/tty/serial/jsm/jsm_driver.c b/drivers/tty/serial/jsm/jsm_driver.c index 0ea799bf8dbb1c58161e617929dcbb785d9f29dc..417a5b6bffc34d836bb6eddc99d5134264fe0685 100644 --- a/drivers/tty/serial/jsm/jsm_driver.c +++ b/drivers/tty/serial/jsm/jsm_driver.c @@ -211,7 +211,8 @@ static int jsm_probe_one(struct pci_dev *pdev, const struct pci_device_id *ent) break; default: - return -ENXIO; + rc = -ENXIO; + goto out_kfree_brd; } rc = request_irq(brd->irq, brd->bd_ops->intr, IRQF_SHARED, "JSM", brd); diff --git a/drivers/tty/serial/jsm/jsm_tty.c b/drivers/tty/serial/jsm/jsm_tty.c index cb58bdec2f43fe1e0d8d8478c810a212aad4e49e..222afc270c88d6fcb292be4c0bb9a554a54be4f6 100644 --- a/drivers/tty/serial/jsm/jsm_tty.c +++ b/drivers/tty/serial/jsm/jsm_tty.c @@ -300,8 +300,8 @@ static void jsm_tty_close(struct uart_port *port) } static void jsm_tty_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old_termios) + struct ktermios *termios, + const struct ktermios *old_termios) { unsigned long lock_flags; struct jsm_channel *channel = diff --git a/drivers/tty/serial/lantiq.c b/drivers/tty/serial/lantiq.c index a3120c3347dd632438366772d9abb387215e6eee..c892f3c7d1abc1b6f2ee7e2755e09f01db20de60 100644 --- a/drivers/tty/serial/lantiq.c +++ b/drivers/tty/serial/lantiq.c @@ -8,6 +8,7 @@ * Copyright (C) 2010 Thomas Langer, */ +#include #include #include #include @@ -93,7 +94,6 @@ #define ASCFSTAT_RXFFLMASK 0x003F #define ASCFSTAT_TXFFLMASK 0x3F00 #define ASCFSTAT_TXFREEMASK 0x3F000000 -#define ASCFSTAT_TXFREEOFF 24 static void lqasc_tx_chars(struct uart_port *port); static struct ltq_uart_port *lqasc_port[MAXPORTS]; @@ -139,6 +139,13 @@ lqasc_stop_tx(struct uart_port *port) return; } +static bool lqasc_tx_ready(struct uart_port *port) +{ + u32 fstat = __raw_readl(port->membase + LTQ_ASC_FSTAT); + + return FIELD_GET(ASCFSTAT_TXFREEMASK, fstat); +} + static void lqasc_start_tx(struct uart_port *port) { @@ -228,8 +235,7 @@ lqasc_tx_chars(struct uart_port *port) return; } - while (((__raw_readl(port->membase + LTQ_ASC_FSTAT) & - ASCFSTAT_TXFREEMASK) >> ASCFSTAT_TXFREEOFF) != 0) { + while (lqasc_tx_ready(port)) { if (port->x_char) { writeb(port->x_char, port->membase + LTQ_ASC_TBUF); port->icount.tx++; @@ -405,8 +411,8 @@ lqasc_shutdown(struct uart_port *port) } static void -lqasc_set_termios(struct uart_port *port, - struct ktermios *new, struct ktermios *old) +lqasc_set_termios(struct uart_port *port, struct ktermios *new, + const struct ktermios *old) { unsigned int cflag; unsigned int iflag; @@ -600,15 +606,12 @@ static const struct uart_ops lqasc_pops = { static void lqasc_console_putchar(struct uart_port *port, unsigned char ch) { - int fifofree; - if (!port->membase) return; - do { - fifofree = (__raw_readl(port->membase + LTQ_ASC_FSTAT) - & ASCFSTAT_TXFREEMASK) >> ASCFSTAT_TXFREEOFF; - } while (fifofree == 0); + while (!lqasc_tx_ready(port)) + ; + writeb(ch, port->membase + LTQ_ASC_TBUF); } diff --git a/drivers/tty/serial/liteuart.c b/drivers/tty/serial/liteuart.c index 328b50521f144c04336de3c43bdd31af828aace4..4c0604325ee910746f79a8c1807d15fb9867cd82 100644 --- a/drivers/tty/serial/liteuart.c +++ b/drivers/tty/serial/liteuart.c @@ -178,7 +178,7 @@ static void liteuart_shutdown(struct uart_port *port) } static void liteuart_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud; unsigned long flags; diff --git a/drivers/tty/serial/lpc32xx_hs.c b/drivers/tty/serial/lpc32xx_hs.c index 93140cac1ca11115153798bc667c2654da740aa8..ed47f476833831bc7eb5cd57f23cea22ac04a523 100644 --- a/drivers/tty/serial/lpc32xx_hs.c +++ b/drivers/tty/serial/lpc32xx_hs.c @@ -278,6 +278,13 @@ static void __serial_lpc32xx_rx(struct uart_port *port) static void serial_lpc32xx_stop_tx(struct uart_port *port); +static bool serial_lpc32xx_tx_ready(struct uart_port *port) +{ + u32 level = readl(LPC32XX_HSUART_LEVEL(port->membase)); + + return LPC32XX_HSU_TX_LEV(level) < 64; +} + static void __serial_lpc32xx_tx(struct uart_port *port) { struct circ_buf *xmit = &port->state->xmit; @@ -293,8 +300,7 @@ static void __serial_lpc32xx_tx(struct uart_port *port) goto exit_tx; /* Transfer data */ - while (LPC32XX_HSU_TX_LEV(readl( - LPC32XX_HSUART_LEVEL(port->membase))) < 64) { + while (serial_lpc32xx_tx_ready(port)) { writel((u32) xmit->buf[xmit->tail], LPC32XX_HSUART_FIFO(port->membase)); xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); @@ -493,7 +499,7 @@ static void serial_lpc32xx_shutdown(struct uart_port *port) /* port->lock is not held. */ static void serial_lpc32xx_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, quot; diff --git a/drivers/tty/serial/max3100.c b/drivers/tty/serial/max3100.c index 0b5f21fbb53ddf0af5ecdeaa3d808947a4b369f9..c69602f356fdc81aff6c5c65fcea514f2c1c7a1c 100644 --- a/drivers/tty/serial/max3100.c +++ b/drivers/tty/serial/max3100.c @@ -418,7 +418,7 @@ static void max3100_set_mctrl(struct uart_port *port, unsigned int mctrl) static void max3100_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct max3100_port *s = container_of(port, struct max3100_port, diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c index ab10ca4a45b5ff269203f2e4b3a6a4bd088a38e5..fbf6e2b3161c5a30287b8454bb1c057349f42516 100644 --- a/drivers/tty/serial/max310x.c +++ b/drivers/tty/serial/max310x.c @@ -906,7 +906,7 @@ static void max310x_break_ctl(struct uart_port *port, int break_state) static void max310x_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int lcr = 0, flow = 0; int baud; @@ -1616,11 +1616,9 @@ static int max310x_i2c_probe(struct i2c_client *client) regmaps, client->irq); } -static int max310x_i2c_remove(struct i2c_client *client) +static void max310x_i2c_remove(struct i2c_client *client) { max310x_remove(&client->dev); - - return 0; } static struct i2c_driver max310x_i2c_driver = { diff --git a/drivers/tty/serial/mcf.c b/drivers/tty/serial/mcf.c index f4aaaadd07422aa022ebe1f604d7089f45801dcd..b1cd9a76dd93bc4d49c8d6b1d10d3f8ba4168dc1 100644 --- a/drivers/tty/serial/mcf.c +++ b/drivers/tty/serial/mcf.c @@ -192,7 +192,7 @@ static void mcf_shutdown(struct uart_port *port) /****************************************************************************/ static void mcf_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, baudclk; diff --git a/drivers/tty/serial/men_z135_uart.c b/drivers/tty/serial/men_z135_uart.c index 12117b596e736e04ab869788cf33019ad2dfa4c7..3690f5cf0f4342334fe601f80e550a571eb06ccb 100644 --- a/drivers/tty/serial/men_z135_uart.c +++ b/drivers/tty/serial/men_z135_uart.c @@ -646,8 +646,8 @@ static void men_z135_shutdown(struct uart_port *port) } static void men_z135_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { struct men_z135_port *uart = to_men_z135(port); unsigned int baud; diff --git a/drivers/tty/serial/meson_uart.c b/drivers/tty/serial/meson_uart.c index 6c8db19fd5720dc2a6243c3f97540afa1f0751f4..056243c12836c307c6268a9a62eead9130513c81 100644 --- a/drivers/tty/serial/meson_uart.c +++ b/drivers/tty/serial/meson_uart.c @@ -335,7 +335,7 @@ static void meson_uart_change_speed(struct uart_port *port, unsigned long baud) static void meson_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int cflags, iflags, baud; unsigned long flags; @@ -667,29 +667,6 @@ static struct uart_driver meson_uart_driver = { .cons = MESON_SERIAL_CONSOLE, }; -static inline struct clk *meson_uart_probe_clock(struct device *dev, - const char *id) -{ - struct clk *clk = NULL; - int ret; - - clk = devm_clk_get(dev, id); - if (IS_ERR(clk)) - return clk; - - ret = clk_prepare_enable(clk); - if (ret) { - dev_err(dev, "couldn't enable clk\n"); - return ERR_PTR(ret); - } - - devm_add_action_or_reset(dev, - (void(*)(void *))clk_disable_unprepare, - clk); - - return clk; -} - static int meson_uart_probe_clocks(struct platform_device *pdev, struct uart_port *port) { @@ -697,15 +674,15 @@ static int meson_uart_probe_clocks(struct platform_device *pdev, struct clk *clk_pclk = NULL; struct clk *clk_baud = NULL; - clk_pclk = meson_uart_probe_clock(&pdev->dev, "pclk"); + clk_pclk = devm_clk_get_enabled(&pdev->dev, "pclk"); if (IS_ERR(clk_pclk)) return PTR_ERR(clk_pclk); - clk_xtal = meson_uart_probe_clock(&pdev->dev, "xtal"); + clk_xtal = devm_clk_get_enabled(&pdev->dev, "xtal"); if (IS_ERR(clk_xtal)) return PTR_ERR(clk_xtal); - clk_baud = meson_uart_probe_clock(&pdev->dev, "baud"); + clk_baud = devm_clk_get_enabled(&pdev->dev, "baud"); if (IS_ERR(clk_baud)) return PTR_ERR(clk_baud); diff --git a/drivers/tty/serial/milbeaut_usio.c b/drivers/tty/serial/milbeaut_usio.c index 347088bb380e5d0f99ce364a0c1f06b8dfe6e924..c15e0d84dc7e32fd35b0f8f2dc14f6ce602a4e48 100644 --- a/drivers/tty/serial/milbeaut_usio.c +++ b/drivers/tty/serial/milbeaut_usio.c @@ -298,7 +298,8 @@ static void mlb_usio_shutdown(struct uart_port *port) } static void mlb_usio_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { unsigned int escr, smr = MLB_USIO_SMR_SOE; unsigned long flags, baud, quot; diff --git a/drivers/tty/serial/mpc52xx_uart.c b/drivers/tty/serial/mpc52xx_uart.c index 3f1986c89694816ce82c9afba8271382d85c3967..73362d4bc45d3cd5daee86cbafdea1a516f0584d 100644 --- a/drivers/tty/serial/mpc52xx_uart.c +++ b/drivers/tty/serial/mpc52xx_uart.c @@ -101,7 +101,7 @@ struct psc_ops { void (*cw_restore_ints)(struct uart_port *port); unsigned int (*set_baudrate)(struct uart_port *port, struct ktermios *new, - struct ktermios *old); + const struct ktermios *old); int (*clock_alloc)(struct uart_port *port); void (*clock_relse)(struct uart_port *port); int (*clock)(struct uart_port *port, int enable); @@ -287,7 +287,7 @@ static void mpc52xx_psc_cw_restore_ints(struct uart_port *port) static unsigned int mpc5200_psc_set_baudrate(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud; unsigned int divisor; @@ -305,7 +305,7 @@ static unsigned int mpc5200_psc_set_baudrate(struct uart_port *port, static unsigned int mpc5200b_psc_set_baudrate(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud; unsigned int divisor; @@ -533,7 +533,7 @@ static void mpc512x_psc_cw_restore_ints(struct uart_port *port) static unsigned int mpc512x_psc_set_baudrate(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud; unsigned int divisor; @@ -880,7 +880,7 @@ static inline void mpc5125_set_divisor(struct mpc5125_psc __iomem *psc, static unsigned int mpc5125_psc_set_baudrate(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud; unsigned int divisor; @@ -1167,7 +1167,7 @@ mpc52xx_uart_shutdown(struct uart_port *port) static void mpc52xx_uart_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned char mr1, mr2; @@ -1364,7 +1364,7 @@ static const struct uart_ops mpc52xx_uart_ops = { /* Interrupt handling */ /* ======================================================================== */ -static inline unsigned int +static inline bool mpc52xx_uart_int_rx_chars(struct uart_port *port) { struct tty_port *tport = &port->state->port; @@ -1425,7 +1425,7 @@ mpc52xx_uart_int_rx_chars(struct uart_port *port) return psc_ops->raw_rx_rdy(port); } -static inline int +static inline bool mpc52xx_uart_int_tx_chars(struct uart_port *port) { struct circ_buf *xmit = &port->state->xmit; @@ -1435,13 +1435,13 @@ mpc52xx_uart_int_tx_chars(struct uart_port *port) psc_ops->write_char(port, port->x_char); port->icount.tx++; port->x_char = 0; - return 1; + return true; } /* Nothing to do ? */ if (uart_circ_empty(xmit) || uart_tx_stopped(port)) { mpc52xx_uart_stop_tx(port); - return 0; + return false; } /* Send chars */ @@ -1460,23 +1460,23 @@ mpc52xx_uart_int_tx_chars(struct uart_port *port) /* Maybe we're done after all */ if (uart_circ_empty(xmit)) { mpc52xx_uart_stop_tx(port); - return 0; + return false; } - return 1; + return true; } static irqreturn_t mpc5xxx_uart_process_int(struct uart_port *port) { unsigned long pass = ISR_PASS_LIMIT; - unsigned int keepgoing; + bool keepgoing; u8 status; /* While we have stuff to do, we continue */ do { /* If we don't find anything to do, we stop */ - keepgoing = 0; + keepgoing = false; psc_ops->rx_clr_irq(port); if (psc_ops->rx_rdy(port)) @@ -1495,7 +1495,7 @@ mpc5xxx_uart_process_int(struct uart_port *port) /* Limit number of iteration */ if (!(--pass)) - keepgoing = 0; + keepgoing = false; } while (keepgoing); diff --git a/drivers/tty/serial/mps2-uart.c b/drivers/tty/serial/mps2-uart.c index 5e9429dcc51f308338772d4c101233c1a0c57a2c..2e3e6cf1681784c484d6a8653dbda5f9670ca122 100644 --- a/drivers/tty/serial/mps2-uart.c +++ b/drivers/tty/serial/mps2-uart.c @@ -358,7 +358,7 @@ static void mps2_uart_shutdown(struct uart_port *port) static void mps2_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, bauddiv; diff --git a/drivers/tty/serial/msm_serial.c b/drivers/tty/serial/msm_serial.c index 3159889ddae1bf884ef4001eb2d9ef07aecf94a4..7dd19a2815794e9f3fb9264600843ac082cf6666 100644 --- a/drivers/tty/serial/msm_serial.c +++ b/drivers/tty/serial/msm_serial.c @@ -1263,7 +1263,7 @@ static void msm_shutdown(struct uart_port *port) } static void msm_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct msm_port *msm_port = to_msm_port(port); struct msm_dma *dma = &msm_port->rx_dma; diff --git a/drivers/tty/serial/mux.c b/drivers/tty/serial/mux.c index 0ba0f4d9459d5d5f2eb6810159fb8ad6b5b31c45..ed0e763f622a880c448da1b323c05eeb507e10c7 100644 --- a/drivers/tty/serial/mux.c +++ b/drivers/tty/serial/mux.c @@ -289,7 +289,7 @@ static void mux_shutdown(struct uart_port *port) */ static void mux_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { } diff --git a/drivers/tty/serial/mvebu-uart.c b/drivers/tty/serial/mvebu-uart.c index 65eaecd10b7ca9a6d42f5b0cb609cde5a7159f4a..ba16e1da6bd3e74cf2a4996a297d5313ef7acc8d 100644 --- a/drivers/tty/serial/mvebu-uart.c +++ b/drivers/tty/serial/mvebu-uart.c @@ -564,7 +564,7 @@ static unsigned int mvebu_uart_baud_rate_set(struct uart_port *port, unsigned in static void mvebu_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned long flags; unsigned int baud, min_baud, max_baud; diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c index 1944daf8593acdf34397d48b10dfd54e3f36d72b..d21a4f3ef2fe614de096df2ac694ee102a1878fc 100644 --- a/drivers/tty/serial/mxs-auart.c +++ b/drivers/tty/serial/mxs-auart.c @@ -959,7 +959,7 @@ err_out: #define CTS_AT_AUART() !mctrl_gpio_to_gpiod(s->gpios, UART_GPIO_CTS) static void mxs_auart_settermios(struct uart_port *u, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct mxs_auart_port *s = to_auart_port(u); u32 ctrl, ctrl2, div; diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 0aa666e247d5781ec0276b37aab52ba38dc65639..7d0d2718ef5953962917617395e630621f5044e3 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -300,8 +300,7 @@ static void serial_omap_stop_tx(struct uart_port *port) serial_out(up, UART_OMAP_SCR, up->scr); res = (port->rs485.flags & SER_RS485_RTS_AFTER_SEND) ? 1 : 0; - if (up->rts_gpiod && - gpiod_get_value(up->rts_gpiod) != res) { + if (gpiod_get_value(up->rts_gpiod) != res) { if (port->rs485.delay_rts_after_send > 0) mdelay( port->rs485.delay_rts_after_send); @@ -337,19 +336,24 @@ static void serial_omap_stop_rx(struct uart_port *port) serial_out(up, UART_IER, up->ier); } +static void serial_omap_put_char(struct uart_omap_port *up, unsigned char ch) +{ + serial_out(up, UART_TX, ch); + + if ((up->port.rs485.flags & SER_RS485_ENABLED) && + !(up->port.rs485.flags & SER_RS485_RX_DURING_TX)) + up->rs485_tx_filter_count++; +} + static void transmit_chars(struct uart_omap_port *up, unsigned int lsr) { struct circ_buf *xmit = &up->port.state->xmit; int count; if (up->port.x_char) { - serial_out(up, UART_TX, up->port.x_char); + serial_omap_put_char(up, up->port.x_char); up->port.icount.tx++; up->port.x_char = 0; - if ((up->port.rs485.flags & SER_RS485_ENABLED) && - !(up->port.rs485.flags & SER_RS485_RX_DURING_TX)) - up->rs485_tx_filter_count++; - return; } if (uart_circ_empty(xmit) || uart_tx_stopped(&up->port)) { @@ -358,12 +362,9 @@ static void transmit_chars(struct uart_omap_port *up, unsigned int lsr) } count = up->port.fifosize / 4; do { - serial_out(up, UART_TX, xmit->buf[xmit->tail]); + serial_omap_put_char(up, xmit->buf[xmit->tail]); xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1); up->port.icount.tx++; - if ((up->port.rs485.flags & SER_RS485_ENABLED) && - !(up->port.rs485.flags & SER_RS485_RX_DURING_TX)) - up->rs485_tx_filter_count++; if (uart_circ_empty(xmit)) break; @@ -397,7 +398,7 @@ static void serial_omap_start_tx(struct uart_port *port) /* if rts not already enabled */ res = (port->rs485.flags & SER_RS485_RTS_ON_SEND) ? 1 : 0; - if (up->rts_gpiod && gpiod_get_value(up->rts_gpiod) != res) { + if (gpiod_get_value(up->rts_gpiod) != res) { gpiod_set_value(up->rts_gpiod, res); if (port->rs485.delay_rts_before_send > 0) mdelay(port->rs485.delay_rts_before_send); @@ -802,7 +803,7 @@ static void serial_omap_uart_qos_work(struct work_struct *work) static void serial_omap_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_omap_port *up = to_uart_omap_port(port); unsigned char cval = 0; @@ -1336,13 +1337,11 @@ serial_omap_config_rs485(struct uart_port *port, struct ktermios *termios, up->ier = 0; serial_out(up, UART_IER, 0); - if (up->rts_gpiod) { - /* enable / disable rts */ - val = (rs485->flags & SER_RS485_ENABLED) ? - SER_RS485_RTS_AFTER_SEND : SER_RS485_RTS_ON_SEND; - val = (rs485->flags & val) ? 1 : 0; - gpiod_set_value(up->rts_gpiod, val); - } + /* enable / disable rts */ + val = (rs485->flags & SER_RS485_ENABLED) ? + SER_RS485_RTS_AFTER_SEND : SER_RS485_RTS_ON_SEND; + val = (rs485->flags & val) ? 1 : 0; + gpiod_set_value(up->rts_gpiod, val); /* Enable interrupts */ up->ier = mode; @@ -1547,11 +1546,13 @@ static int serial_omap_probe_rs485(struct uart_omap_port *up, ret = PTR_ERR(up->rts_gpiod); if (ret == -EPROBE_DEFER) return ret; - /* - * FIXME: the code historically ignored any other error than - * -EPROBE_DEFER and just went on without GPIO. - */ + up->rts_gpiod = NULL; + up->port.rs485_supported = (const struct serial_rs485) { }; + if (rs485conf->flags & SER_RS485_ENABLED) { + dev_err(dev, "disabling RS-485 (rts-gpio missing in device tree)\n"); + memset(rs485conf, 0, sizeof(*rs485conf)); + } } else { gpiod_set_consumer_name(up->rts_gpiod, "omap-serial"); } diff --git a/drivers/tty/serial/owl-uart.c b/drivers/tty/serial/owl-uart.c index 888e17e3f25f58c006ebb40d55ff2a53300bcee1..fde39cc1145db53caf5f562abe0db63c037935c6 100644 --- a/drivers/tty/serial/owl-uart.c +++ b/drivers/tty/serial/owl-uart.c @@ -328,7 +328,7 @@ static void owl_uart_change_baudrate(struct owl_uart_port *owl_port, static void owl_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct owl_uart_port *owl_port = to_owl_uart_port(port); unsigned int baud; diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c index 8a9065e4a903b2c297b71f13b73992420a122931..c59ce788657999dbd1931b5e8aef8f3e7ddd118e 100644 --- a/drivers/tty/serial/pch_uart.c +++ b/drivers/tty/serial/pch_uart.c @@ -898,9 +898,7 @@ static unsigned int dma_handle_tx(struct eg20t_port *priv) fifo_size--; } - bytes = min((int)CIRC_CNT(xmit->head, xmit->tail, - UART_XMIT_SIZE), CIRC_CNT_TO_END(xmit->head, - xmit->tail, UART_XMIT_SIZE)); + bytes = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE); if (!bytes) { dev_dbg(priv->port.dev, "%s 0 bytes return\n", __func__); pch_uart_hal_disable_interrupt(priv, PCH_UART_HAL_TX_INT); @@ -1301,7 +1299,8 @@ static void pch_uart_shutdown(struct uart_port *port) *bits. Update read_status_mask and ignore_status_mask to indicate *the types of events we are interested in receiving. */ static void pch_uart_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { int rtn; unsigned int baud, parity, bits, stb; diff --git a/drivers/tty/serial/pic32_uart.c b/drivers/tty/serial/pic32_uart.c index f418f1de66b35c9a8c11dd83c9ebcde01cb90394..2beada66c8245edac1edf7dbfc270bb73af0d6ee 100644 --- a/drivers/tty/serial/pic32_uart.c +++ b/drivers/tty/serial/pic32_uart.c @@ -50,7 +50,7 @@ * @irq_rx_name: irq rx name * @irq_tx: virtual tx interrupt number * @irq_tx_name: irq tx name - * @cts_gpio: clear to send gpio + * @cts_gpiod: clear to send GPIO * @dev: device descriptor **/ struct pic32_sport { @@ -65,8 +65,7 @@ struct pic32_sport { const char *irq_tx_name; bool enable_tx_irq; - bool hw_flow_ctrl; - int cts_gpio; + struct gpio_desc *cts_gpiod; struct clk *clk; @@ -158,25 +157,16 @@ static void pic32_uart_set_mctrl(struct uart_port *port, unsigned int mctrl) PIC32_UART_MODE_LPBK); } -/* get the state of CTS input pin for this port */ -static unsigned int get_cts_state(struct pic32_sport *sport) -{ - /* read and invert UxCTS */ - if (gpio_is_valid(sport->cts_gpio)) - return !gpio_get_value(sport->cts_gpio); - - return 1; -} - /* serial core request to return the state of misc UART input pins */ static unsigned int pic32_uart_get_mctrl(struct uart_port *port) { struct pic32_sport *sport = to_pic32_sport(port); unsigned int mctrl = 0; - if (!sport->hw_flow_ctrl) + /* get the state of CTS input pin for this port */ + if (!sport->cts_gpiod) mctrl |= TIOCM_CTS; - else if (get_cts_state(sport)) + else if (gpiod_get_value(sport->cts_gpiod)) mctrl |= TIOCM_CTS; /* DSR and CD are not supported in PIC32, so return 1 @@ -609,7 +599,7 @@ static void pic32_uart_shutdown(struct uart_port *port) /* serial core request to change current uart setting */ static void pic32_uart_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { struct pic32_sport *sport = to_pic32_sport(port); unsigned int baud; @@ -648,7 +638,7 @@ static void pic32_uart_set_termios(struct uart_port *port, PIC32_UART_MODE_PDSEL0); } /* if hw flow ctrl, then the pins must be specified in device tree */ - if ((new->c_cflag & CRTSCTS) && sport->hw_flow_ctrl) { + if ((new->c_cflag & CRTSCTS) && sport->cts_gpiod) { /* enable hardware flow control */ pic32_uart_writel(sport, PIC32_SET(PIC32_UART_MODE), PIC32_UART_MODE_UEN1); @@ -875,7 +865,8 @@ static struct uart_driver pic32_uart_driver = { static int pic32_uart_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; struct pic32_sport *sport; int uart_idx = 0; struct resource *res_mem; @@ -904,25 +895,10 @@ static int pic32_uart_probe(struct platform_device *pdev) /* Hardware flow control: gpios * !Note: Basically, CTS is needed for reading the status. */ - sport->hw_flow_ctrl = false; - sport->cts_gpio = of_get_named_gpio(np, "cts-gpios", 0); - if (gpio_is_valid(sport->cts_gpio)) { - sport->hw_flow_ctrl = true; - - ret = devm_gpio_request(sport->dev, - sport->cts_gpio, "CTS"); - if (ret) { - dev_err(&pdev->dev, - "error requesting CTS GPIO\n"); - goto err; - } - - ret = gpio_direction_input(sport->cts_gpio); - if (ret) { - dev_err(&pdev->dev, "error setting CTS GPIO\n"); - goto err; - } - } + sport->cts_gpiod = devm_gpiod_get_optional(dev, "cts", GPIOD_IN); + if (IS_ERR(sport->cts_gpiod)) + return dev_err_probe(dev, PTR_ERR(sport->cts_gpiod), "error requesting CTS GPIO\n"); + gpiod_set_consumer_name(sport->cts_gpiod, "CTS"); pic32_sports[uart_idx] = sport; port = &sport->port; @@ -943,7 +919,7 @@ static int pic32_uart_probe(struct platform_device *pdev) } #ifdef CONFIG_SERIAL_PIC32_CONSOLE - if (uart_console(port) && (pic32_console.flags & CON_ENABLED)) { + if (uart_console_enabled(port)) { /* The peripheral clock has been enabled by console_setup, * so disable it till the port is used. */ diff --git a/drivers/tty/serial/pmac_zilog.c b/drivers/tty/serial/pmac_zilog.c index f63257b8e872c40bbd932bbf8554acb01ef42f26..fe2e4ec423f7948e5fd005c97b0d94e62a580b3e 100644 --- a/drivers/tty/serial/pmac_zilog.c +++ b/drivers/tty/serial/pmac_zilog.c @@ -1202,7 +1202,7 @@ static void pmz_irda_setup(struct uart_pmac_port *uap, unsigned long *baud) static void __pmz_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_pmac_port *uap = to_pmz(port); unsigned long baud; @@ -1244,7 +1244,7 @@ static void __pmz_set_termios(struct uart_port *port, struct ktermios *termios, /* The port lock is not held. */ static void pmz_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_pmac_port *uap = to_pmz(port); unsigned long flags; diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c index 9309ffd87c8e041eb303e2eb60d1e529924e6de9..2d25231fad847c849f0e44b76f495eb602b5bf4d 100644 --- a/drivers/tty/serial/pxa.c +++ b/drivers/tty/serial/pxa.c @@ -423,7 +423,7 @@ static void serial_pxa_shutdown(struct uart_port *port) static void serial_pxa_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_pxa_port *up = (struct uart_pxa_port *)port; unsigned char cval, fcr = 0; diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index f4698a064a4dd663e4b3fd7d19739a8be3720f30..83b66b73303afde411c03690a2303a7395888dc3 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -22,6 +22,7 @@ #include #include #include +#include /* UART specific GENI registers */ #define SE_UART_LOOPBACK_CFG 0x22c @@ -1005,7 +1006,8 @@ static unsigned long get_clk_div_rate(struct clk *clk, unsigned int baud, } static void qcom_geni_serial_set_termios(struct uart_port *uport, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { unsigned int baud; u32 bits_per_char; @@ -1524,7 +1526,7 @@ static int __maybe_unused qcom_geni_serial_sys_suspend(struct device *dev) * even with no_console_suspend */ if (uart_console(uport)) { - geni_icc_set_tag(&port->se, 0x3); + geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ACTIVE_ONLY); geni_icc_set_bw(&port->se); } return uart_suspend_port(private_data->drv, uport); @@ -1539,7 +1541,7 @@ static int __maybe_unused qcom_geni_serial_sys_resume(struct device *dev) ret = uart_resume_port(private_data->drv, uport); if (uart_console(uport)) { - geni_icc_set_tag(&port->se, 0x7); + geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS); geni_icc_set_bw(&port->se); } return ret; diff --git a/drivers/tty/serial/rda-uart.c b/drivers/tty/serial/rda-uart.c index feb2054aba37e6abecdf2de2fc02b1c8ec895993..0e387e2144fa25937ddfde1d26cf8fb47a68c5ea 100644 --- a/drivers/tty/serial/rda-uart.c +++ b/drivers/tty/serial/rda-uart.c @@ -238,7 +238,7 @@ static void rda_uart_change_baudrate(struct rda_uart_port *rda_port, static void rda_uart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct rda_uart_port *rda_port = to_rda_uart_port(port); unsigned long flags; diff --git a/drivers/tty/serial/rp2.c b/drivers/tty/serial/rp2.c index 6689d8add8f7aa66926f80cd9d316e5a4a5882e2..b81afb06f1f40ae8d6cd9fd86a6b657bdfc460cf 100644 --- a/drivers/tty/serial/rp2.c +++ b/drivers/tty/serial/rp2.c @@ -370,9 +370,8 @@ static void __rp2_uart_set_termios(struct rp2_uart_port *up, up->ucode + RP2_RX_SWFLOW); } -static void rp2_uart_set_termios(struct uart_port *port, - struct ktermios *new, - struct ktermios *old) +static void rp2_uart_set_termios(struct uart_port *port, struct ktermios *new, + const struct ktermios *old) { struct rp2_uart_port *up = port_to_up(port); unsigned long flags; diff --git a/drivers/tty/serial/sa1100.c b/drivers/tty/serial/sa1100.c index e64e42a19d1a042e679e4d864339b646364a2a6b..dd9e3253cab4487d7774528ed87f3a1a35de634e 100644 --- a/drivers/tty/serial/sa1100.c +++ b/drivers/tty/serial/sa1100.c @@ -409,7 +409,7 @@ static void sa1100_shutdown(struct uart_port *port) static void sa1100_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct sa1100_port *sport = container_of(port, struct sa1100_port, port); diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index b7a4b47ce74e599fe9d130622830f20f14483287..77d1363029f558b76d1e84dfb6b38a25ec44afdc 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -1530,7 +1530,7 @@ static const u16 udivslot_table[16] = { static void s3c24xx_serial_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { const struct s3c2410_uartcfg *cfg = s3c24xx_port_to_cfg(port); struct s3c24xx_uart_port *ourport = to_ourport(port); diff --git a/drivers/tty/serial/sb1250-duart.c b/drivers/tty/serial/sb1250-duart.c index 2cf8533ef7609200a4250172c4f00e2000a725c4..c5d2b6cdcb4a1f44cfa0d0c64dae17a0d3551b10 100644 --- a/drivers/tty/serial/sb1250-duart.c +++ b/drivers/tty/serial/sb1250-duart.c @@ -531,7 +531,7 @@ static void sbd_init_port(struct sbd_port *sport) } static void sbd_set_termios(struct uart_port *uport, struct ktermios *termios, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct sbd_port *sport = to_sport(uport); unsigned int mode1 = 0, mode2 = 0, aux = 0; diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 259e08cc347cc4c6528fb8508cf31f759abab29e..524921360ca786bea6c0298f3b319f7170b28d76 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1015,7 +1015,7 @@ static void sc16is7xx_break_ctl(struct uart_port *port, int break_state) static void sc16is7xx_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct sc16is7xx_port *s = dev_get_drvdata(port->dev); struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); @@ -1689,11 +1689,9 @@ static int sc16is7xx_i2c_probe(struct i2c_client *i2c, return sc16is7xx_probe(&i2c->dev, devtype, regmap, i2c->irq); } -static int sc16is7xx_i2c_remove(struct i2c_client *client) +static void sc16is7xx_i2c_remove(struct i2c_client *client) { sc16is7xx_remove(&client->dev); - - return 0; } static const struct i2c_device_id sc16is7xx_i2c_id_table[] = { diff --git a/drivers/tty/serial/sccnxp.c b/drivers/tty/serial/sccnxp.c index c56de2e104d480fc55e7f37ddcb0d66caec2337c..dd98509f52e5a866e175f87de8b1723d0b1eafb2 100644 --- a/drivers/tty/serial/sccnxp.c +++ b/drivers/tty/serial/sccnxp.c @@ -636,7 +636,8 @@ static void sccnxp_break_ctl(struct uart_port *port, int break_state) } static void sccnxp_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { struct sccnxp_port *s = dev_get_drvdata(port->dev); unsigned long flags; diff --git a/drivers/tty/serial/serial-tegra.c b/drivers/tty/serial/serial-tegra.c index ad4f3567ff90696cbcc3373c9936ee31f695199f..b7170cb9a544f7321f9731ca5bb673b588f1f766 100644 --- a/drivers/tty/serial/serial-tegra.c +++ b/drivers/tty/serial/serial-tegra.c @@ -525,7 +525,7 @@ static void tegra_uart_tx_dma_complete(void *args) count = tup->tx_bytes_requested - state.residue; async_tx_ack(tup->tx_dma_desc); spin_lock_irqsave(&tup->uport.lock, flags); - xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1); + uart_xmit_advance(&tup->uport, count); tup->tx_in_progress = 0; if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS) uart_write_wakeup(&tup->uport); @@ -613,7 +613,6 @@ static unsigned int tegra_uart_tx_empty(struct uart_port *u) static void tegra_uart_stop_tx(struct uart_port *u) { struct tegra_uart_port *tup = to_tegra_uport(u); - struct circ_buf *xmit = &tup->uport.state->xmit; struct dma_tx_state state; unsigned int count; @@ -624,7 +623,7 @@ static void tegra_uart_stop_tx(struct uart_port *u) dmaengine_tx_status(tup->tx_dma_chan, tup->tx_cookie, &state); count = tup->tx_bytes_requested - state.residue; async_tx_ack(tup->tx_dma_desc); - xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1); + uart_xmit_advance(&tup->uport, count); tup->tx_in_progress = 0; } @@ -1271,14 +1270,14 @@ static void tegra_uart_enable_ms(struct uart_port *u) } static void tegra_uart_set_termios(struct uart_port *u, - struct ktermios *termios, struct ktermios *oldtermios) + struct ktermios *termios, + const struct ktermios *oldtermios) { struct tegra_uart_port *tup = to_tegra_uport(u); unsigned int baud; unsigned long flags; unsigned int lcr; unsigned char char_bits; - int symb_bit = 1; struct clk *parent_clk = clk_get_parent(tup->uart_clk); unsigned long parent_clk_rate = clk_get_rate(parent_clk); int max_divider = (tup->cdata->support_clk_src_div) ? 0x7FFF : 0xFFFF; @@ -1305,7 +1304,6 @@ static void tegra_uart_set_termios(struct uart_port *u, termios->c_cflag &= ~CMSPAR; if ((termios->c_cflag & PARENB) == PARENB) { - symb_bit++; if (termios->c_cflag & PARODD) { lcr |= UART_LCR_PARITY; lcr &= ~UART_LCR_EPAR; @@ -1318,22 +1316,18 @@ static void tegra_uart_set_termios(struct uart_port *u, } char_bits = tty_get_char_size(termios->c_cflag); - symb_bit += char_bits; lcr &= ~UART_LCR_WLEN8; lcr |= UART_LCR_WLEN(char_bits); /* Stop bits */ - if (termios->c_cflag & CSTOPB) { + if (termios->c_cflag & CSTOPB) lcr |= UART_LCR_STOP; - symb_bit += 2; - } else { + else lcr &= ~UART_LCR_STOP; - symb_bit++; - } tegra_uart_write(tup, lcr, UART_LCR); tup->lcr_shadow = lcr; - tup->symb_bit = symb_bit; + tup->symb_bit = tty_get_frame_size(termios->c_cflag); /* Baud rate. */ baud = uart_get_baud_rate(u, termios, oldtermios, diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 12c87cd201a7659e958258629f99214eacd01b94..179ee199df343ef28d00e0327c1909cbddf5fbc3 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -49,7 +49,7 @@ static struct lock_class_key port_lock_key; #define RS485_MAX_RTS_DELAY 100 /* msecs */ static void uart_change_speed(struct tty_struct *tty, struct uart_state *state, - struct ktermios *old_termios); + const struct ktermios *old_termios); static void uart_wait_until_sent(struct tty_struct *tty, int timeout); static void uart_change_pm(struct uart_state *state, enum uart_pm_state pm_state); @@ -158,15 +158,10 @@ uart_update_mctrl(struct uart_port *port, unsigned int set, unsigned int clear) unsigned long flags; unsigned int old; - if (port->rs485.flags & SER_RS485_ENABLED) { - set &= ~TIOCM_RTS; - clear &= ~TIOCM_RTS; - } - spin_lock_irqsave(&port->lock, flags); old = port->mctrl; port->mctrl = (old & ~clear) | set; - if (old != port->mctrl) + if (old != port->mctrl && !(port->rs485.flags & SER_RS485_ENABLED)) port->ops->set_mctrl(port, port->mctrl); spin_unlock_irqrestore(&port->lock, flags); } @@ -380,7 +375,7 @@ EXPORT_SYMBOL(uart_update_timeout); */ unsigned int uart_get_baud_rate(struct uart_port *port, struct ktermios *termios, - struct ktermios *old, unsigned int min, unsigned int max) + const struct ktermios *old, unsigned int min, unsigned int max) { unsigned int try; unsigned int baud; @@ -492,7 +487,7 @@ EXPORT_SYMBOL(uart_get_divisor); /* Caller holds port mutex */ static void uart_change_speed(struct tty_struct *tty, struct uart_state *state, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct uart_port *uport = uart_port_check(state); struct ktermios *termios; @@ -1391,7 +1386,7 @@ static void uart_set_rs485_termination(struct uart_port *port, !!(rs485->flags & SER_RS485_TERMINATE_BUS)); } -int uart_rs485_config(struct uart_port *port) +static int uart_rs485_config(struct uart_port *port) { struct serial_rs485 *rs485 = &port->rs485; int ret; @@ -1405,7 +1400,6 @@ int uart_rs485_config(struct uart_port *port) return ret; } -EXPORT_SYMBOL_GPL(uart_rs485_config); static int uart_get_rs485_config(struct uart_port *port, struct serial_rs485 __user *rs485) @@ -1444,8 +1438,13 @@ static int uart_set_rs485_config(struct tty_struct *tty, struct uart_port *port, spin_lock_irqsave(&port->lock, flags); ret = port->rs485_config(port, &tty->termios, &rs485); - if (!ret) + if (!ret) { port->rs485 = rs485; + + /* Reset RTS and other mctrl lines when disabling RS485 */ + if (!(rs485.flags & SER_RS485_ENABLED)) + port->ops->set_mctrl(port, port->mctrl); + } spin_unlock_irqrestore(&port->lock, flags); if (ret) return ret; @@ -1619,7 +1618,7 @@ static void uart_set_ldisc(struct tty_struct *tty) } static void uart_set_termios(struct tty_struct *tty, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct uart_state *state = tty->driver_data; struct uart_port *uport; @@ -2352,7 +2351,8 @@ int uart_suspend_port(struct uart_driver *drv, struct uart_port *uport) spin_lock_irq(&uport->lock); ops->stop_tx(uport); - ops->set_mctrl(uport, 0); + if (!(uport->rs485.flags & SER_RS485_ENABLED)) + ops->set_mctrl(uport, 0); /* save mctrl so it can be restored on resume */ mctrl = uport->mctrl; uport->mctrl = 0; @@ -2440,7 +2440,8 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport) uart_change_pm(state, UART_PM_STATE_ON); spin_lock_irq(&uport->lock); - ops->set_mctrl(uport, 0); + if (!(uport->rs485.flags & SER_RS485_ENABLED)) + ops->set_mctrl(uport, 0); spin_unlock_irq(&uport->lock); if (console_suspend_enabled || !uart_console(uport)) { /* Protected by port mutex for now */ @@ -2451,7 +2452,10 @@ int uart_resume_port(struct uart_driver *drv, struct uart_port *uport) if (tty) uart_change_speed(tty, state, NULL); spin_lock_irq(&uport->lock); - ops->set_mctrl(uport, uport->mctrl); + if (!(uport->rs485.flags & SER_RS485_ENABLED)) + ops->set_mctrl(uport, uport->mctrl); + else + uart_rs485_config(uport); ops->start_tx(uport); spin_unlock_irq(&uport->lock); tty_port_set_initialized(port, 1); @@ -2497,7 +2501,7 @@ uart_report_port(struct uart_driver *drv, struct uart_port *port) "MMIO 0x%llx", (unsigned long long)port->mapbase); break; default: - strlcpy(address, "*unknown*", sizeof(address)); + strscpy(address, "*unknown*", sizeof(address)); break; } @@ -2558,10 +2562,10 @@ uart_configure_port(struct uart_driver *drv, struct uart_state *state, */ spin_lock_irqsave(&port->lock, flags); port->mctrl &= TIOCM_DTR; - if (port->rs485.flags & SER_RS485_ENABLED && - !(port->rs485.flags & SER_RS485_RTS_AFTER_SEND)) - port->mctrl |= TIOCM_RTS; - port->ops->set_mctrl(port, port->mctrl); + if (!(port->rs485.flags & SER_RS485_ENABLED)) + port->ops->set_mctrl(port, port->mctrl); + else + uart_rs485_config(port); spin_unlock_irqrestore(&port->lock, flags); /* diff --git a/drivers/tty/serial/serial_txx9.c b/drivers/tty/serial/serial_txx9.c index 228e380db0804de1e83355c66113ed18a5411a9e..e12f1dc18c38bc7627cf18a74920a889222ae0e9 100644 --- a/drivers/tty/serial/serial_txx9.c +++ b/drivers/tty/serial/serial_txx9.c @@ -594,7 +594,7 @@ static void serial_txx9_shutdown(struct uart_port *up) static void serial_txx9_set_termios(struct uart_port *up, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int cval, fcr = 0; unsigned long flags; diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c index 0075a14200057ab21524f181759a84bc8510e0a3..62f773286d44bb312f67f664799494fb2405276a 100644 --- a/drivers/tty/serial/sh-sci.c +++ b/drivers/tty/serial/sh-sci.c @@ -1407,10 +1407,8 @@ static void sci_dma_tx_work_fn(struct work_struct *work) spin_lock_irq(&port->lock); head = xmit->head; tail = xmit->tail; - buf = s->tx_dma_addr + (tail & (UART_XMIT_SIZE - 1)); - s->tx_dma_len = min_t(unsigned int, - CIRC_CNT(head, tail, UART_XMIT_SIZE), - CIRC_CNT_TO_END(head, tail, UART_XMIT_SIZE)); + buf = s->tx_dma_addr + tail; + s->tx_dma_len = CIRC_CNT_TO_END(head, tail, UART_XMIT_SIZE); if (!s->tx_dma_len) { /* Transmit buffer has been flushed */ spin_unlock_irq(&port->lock); @@ -2367,7 +2365,7 @@ static void sci_reset(struct uart_port *port) } static void sci_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud, smr_val = SCSMR_ASYNC, scr_val = 0, i, bits; unsigned int brr = 255, cks = 0, srr = 15, dl = 0, sccks = 0; diff --git a/drivers/tty/serial/sifive.c b/drivers/tty/serial/sifive.c index 5c3a07546a582f36c4ceafc394f38449abc8d667..7fb6760b5c37ad87a1740fb77ead0715afc8210b 100644 --- a/drivers/tty/serial/sifive.c +++ b/drivers/tty/serial/sifive.c @@ -646,7 +646,7 @@ static int sifive_serial_clk_notifier(struct notifier_block *nb, static void sifive_serial_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct sifive_serial_port *ssp = port_to_sifive_serial_port(port); unsigned long flags; @@ -945,7 +945,7 @@ static int sifive_serial_probe(struct platform_device *pdev) return PTR_ERR(base); } - clk = devm_clk_get(&pdev->dev, NULL); + clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(clk)) { dev_err(&pdev->dev, "unable to find controller clock\n"); return PTR_ERR(clk); diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c index 4329b9c9cbf0e7358d7ef5ff20ce4cc429cf8d0a..342a87967631575f1d857201328e76c9b8fdad55 100644 --- a/drivers/tty/serial/sprd_serial.c +++ b/drivers/tty/serial/sprd_serial.c @@ -771,9 +771,8 @@ static void sprd_shutdown(struct uart_port *port) devm_free_irq(port->dev, port->irq, port); } -static void sprd_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old) +static void sprd_set_termios(struct uart_port *port, struct ktermios *termios, + const struct ktermios *old) { unsigned int baud, quot; unsigned int lcr = 0, fc; diff --git a/drivers/tty/serial/st-asc.c b/drivers/tty/serial/st-asc.c index cce42f4c9bc2219a794fa7adfa8c160e1ea7ec32..fcecea689a0ddd3e52bcebf887b7c6c027c515e8 100644 --- a/drivers/tty/serial/st-asc.c +++ b/drivers/tty/serial/st-asc.c @@ -500,7 +500,7 @@ static void asc_pm(struct uart_port *port, unsigned int state, } static void asc_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct asc_port *ascport = to_asc_port(port); struct gpio_desc *gpiod; diff --git a/drivers/tty/serial/stm32-usart.c b/drivers/tty/serial/stm32-usart.c index 2c85dbf165c4a23737f05087df286d39e997a0bb..dfdbcf092facc2a5afe79194e4ab4ef2038da4e5 100644 --- a/drivers/tty/serial/stm32-usart.c +++ b/drivers/tty/serial/stm32-usart.c @@ -37,7 +37,7 @@ /* Register offsets */ -static struct stm32_usart_info stm32f4_info = { +static struct stm32_usart_info __maybe_unused stm32f4_info = { .ofs = { .isr = 0x00, .rdr = 0x04, @@ -58,7 +58,7 @@ static struct stm32_usart_info stm32f4_info = { } }; -static struct stm32_usart_info stm32f7_info = { +static struct stm32_usart_info __maybe_unused stm32f7_info = { .ofs = { .cr1 = 0x00, .cr2 = 0x04, @@ -80,7 +80,7 @@ static struct stm32_usart_info stm32f7_info = { } }; -static struct stm32_usart_info stm32h7_info = { +static struct stm32_usart_info __maybe_unused stm32h7_info = { .ofs = { .cr1 = 0x00, .cr2 = 0x04, @@ -131,6 +131,53 @@ static void stm32_usart_clr_bits(struct uart_port *port, u32 reg, u32 bits) writel_relaxed(val, port->membase + reg); } +static unsigned int stm32_usart_tx_empty(struct uart_port *port) +{ + struct stm32_port *stm32_port = to_stm32_port(port); + const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; + + if (readl_relaxed(port->membase + ofs->isr) & USART_SR_TC) + return TIOCSER_TEMT; + + return 0; +} + +static void stm32_usart_rs485_rts_enable(struct uart_port *port) +{ + struct stm32_port *stm32_port = to_stm32_port(port); + struct serial_rs485 *rs485conf = &port->rs485; + + if (stm32_port->hw_flow_control || + !(rs485conf->flags & SER_RS485_ENABLED)) + return; + + if (rs485conf->flags & SER_RS485_RTS_ON_SEND) { + mctrl_gpio_set(stm32_port->gpios, + stm32_port->port.mctrl | TIOCM_RTS); + } else { + mctrl_gpio_set(stm32_port->gpios, + stm32_port->port.mctrl & ~TIOCM_RTS); + } +} + +static void stm32_usart_rs485_rts_disable(struct uart_port *port) +{ + struct stm32_port *stm32_port = to_stm32_port(port); + struct serial_rs485 *rs485conf = &port->rs485; + + if (stm32_port->hw_flow_control || + !(rs485conf->flags & SER_RS485_ENABLED)) + return; + + if (rs485conf->flags & SER_RS485_RTS_ON_SEND) { + mctrl_gpio_set(stm32_port->gpios, + stm32_port->port.mctrl & ~TIOCM_RTS); + } else { + mctrl_gpio_set(stm32_port->gpios, + stm32_port->port.mctrl | TIOCM_RTS); + } +} + static void stm32_usart_config_reg_rs485(u32 *cr1, u32 *cr3, u32 delay_ADE, u32 delay_DDE, u32 baud) { @@ -214,6 +261,12 @@ static int stm32_usart_config_rs485(struct uart_port *port, struct ktermios *ter stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit)); + /* Adjust RTS polarity in case it's driven in software */ + if (stm32_usart_tx_empty(port)) + stm32_usart_rs485_rts_disable(port); + else + stm32_usart_rs485_rts_enable(port); + return 0; } @@ -529,42 +582,6 @@ static void stm32_usart_tc_interrupt_disable(struct uart_port *port) stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_TCIE); } -static void stm32_usart_rs485_rts_enable(struct uart_port *port) -{ - struct stm32_port *stm32_port = to_stm32_port(port); - struct serial_rs485 *rs485conf = &port->rs485; - - if (stm32_port->hw_flow_control || - !(rs485conf->flags & SER_RS485_ENABLED)) - return; - - if (rs485conf->flags & SER_RS485_RTS_ON_SEND) { - mctrl_gpio_set(stm32_port->gpios, - stm32_port->port.mctrl | TIOCM_RTS); - } else { - mctrl_gpio_set(stm32_port->gpios, - stm32_port->port.mctrl & ~TIOCM_RTS); - } -} - -static void stm32_usart_rs485_rts_disable(struct uart_port *port) -{ - struct stm32_port *stm32_port = to_stm32_port(port); - struct serial_rs485 *rs485conf = &port->rs485; - - if (stm32_port->hw_flow_control || - !(rs485conf->flags & SER_RS485_ENABLED)) - return; - - if (rs485conf->flags & SER_RS485_RTS_ON_SEND) { - mctrl_gpio_set(stm32_port->gpios, - stm32_port->port.mctrl & ~TIOCM_RTS); - } else { - mctrl_gpio_set(stm32_port->gpios, - stm32_port->port.mctrl | TIOCM_RTS); - } -} - static void stm32_usart_transmit_chars_pio(struct uart_port *port) { struct stm32_port *stm32_port = to_stm32_port(port); @@ -807,17 +824,6 @@ static irqreturn_t stm32_usart_threaded_interrupt(int irq, void *ptr) return IRQ_HANDLED; } -static unsigned int stm32_usart_tx_empty(struct uart_port *port) -{ - struct stm32_port *stm32_port = to_stm32_port(port); - const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; - - if (readl_relaxed(port->membase + ofs->isr) & USART_SR_TC) - return TIOCSER_TEMT; - - return 0; -} - static void stm32_usart_set_mctrl(struct uart_port *port, unsigned int mctrl) { struct stm32_port *stm32_port = to_stm32_port(port); @@ -1089,7 +1095,7 @@ static void stm32_usart_shutdown(struct uart_port *port) static void stm32_usart_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct stm32_port *stm32_port = to_stm32_port(port); const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs; diff --git a/drivers/tty/serial/sunhv.c b/drivers/tty/serial/sunhv.c index eafada8fb6fad8755104ffa4fdc7cb40c8cc1d6e..1938ba5e98c0eee0bed60379b59c83ea76fce577 100644 --- a/drivers/tty/serial/sunhv.c +++ b/drivers/tty/serial/sunhv.c @@ -323,7 +323,7 @@ static void sunhv_shutdown(struct uart_port *port) /* port->lock is not held. */ static void sunhv_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud = uart_get_baud_rate(port, termios, old, 0, 4000000); unsigned int quot = uart_get_divisor(port, baud); diff --git a/drivers/tty/serial/sunplus-uart.c b/drivers/tty/serial/sunplus-uart.c index 60c73662f955c3276deb43e51ba31dcad0606567..7afe61a0e72e42e53089f831aa3b5e765c4a9b43 100644 --- a/drivers/tty/serial/sunplus-uart.c +++ b/drivers/tty/serial/sunplus-uart.c @@ -333,7 +333,7 @@ static void sunplus_shutdown(struct uart_port *port) static void sunplus_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *oldtermios) + const struct ktermios *oldtermios) { u32 ext, div, div_l, div_h, baud, lcr; u32 clk = port->uartclk; diff --git a/drivers/tty/serial/sunsab.c b/drivers/tty/serial/sunsab.c index 6ea52293d9f35821b9d0987b88be8128c5ead290..99608b2a2b74f2b0482d4b235f927f9a8affbf78 100644 --- a/drivers/tty/serial/sunsab.c +++ b/drivers/tty/serial/sunsab.c @@ -681,27 +681,23 @@ static void sunsab_convert_to_sab(struct uart_sunsab_port *up, unsigned int cfla unsigned int quot) { unsigned char dafo; - int bits, n, m; + int n, m; /* Byte size and parity */ switch (cflag & CSIZE) { - case CS5: dafo = SAB82532_DAFO_CHL5; bits = 7; break; - case CS6: dafo = SAB82532_DAFO_CHL6; bits = 8; break; - case CS7: dafo = SAB82532_DAFO_CHL7; bits = 9; break; - case CS8: dafo = SAB82532_DAFO_CHL8; bits = 10; break; + case CS5: dafo = SAB82532_DAFO_CHL5; break; + case CS6: dafo = SAB82532_DAFO_CHL6; break; + case CS7: dafo = SAB82532_DAFO_CHL7; break; + case CS8: dafo = SAB82532_DAFO_CHL8; break; /* Never happens, but GCC is too dumb to figure it out */ - default: dafo = SAB82532_DAFO_CHL5; bits = 7; break; + default: dafo = SAB82532_DAFO_CHL5; break; } - if (cflag & CSTOPB) { + if (cflag & CSTOPB) dafo |= SAB82532_DAFO_STOP; - bits++; - } - if (cflag & PARENB) { + if (cflag & PARENB) dafo |= SAB82532_DAFO_PARE; - bits++; - } if (cflag & PARODD) { dafo |= SAB82532_DAFO_PAR_ODD; @@ -776,7 +772,7 @@ static void sunsab_convert_to_sab(struct uart_sunsab_port *up, unsigned int cfla /* port->lock is not held. */ static void sunsab_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_sunsab_port *up = container_of(port, struct uart_sunsab_port, port); diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c index 84d545e5a8c7b31b94980e26121ea7d0ea31b266..9ea7e567540d60131abce88902a5c2442ce8edc1 100644 --- a/drivers/tty/serial/sunsu.c +++ b/drivers/tty/serial/sunsu.c @@ -897,7 +897,7 @@ sunsu_change_speed(struct uart_port *port, unsigned int cflag, static void sunsu_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { unsigned int baud, quot; @@ -1217,13 +1217,13 @@ static int sunsu_kbd_ms_init(struct uart_sunsu_port *up) serio->id.type = SERIO_RS232; if (up->su_type == SU_PORT_KBD) { serio->id.proto = SERIO_SUNKBD; - strlcpy(serio->name, "sukbd", sizeof(serio->name)); + strscpy(serio->name, "sukbd", sizeof(serio->name)); } else { serio->id.proto = SERIO_SUN; serio->id.extra = 1; - strlcpy(serio->name, "sums", sizeof(serio->name)); + strscpy(serio->name, "sums", sizeof(serio->name)); } - strlcpy(serio->phys, + strscpy(serio->phys, (!(up->port.line & 1) ? "su/serio0" : "su/serio1"), sizeof(serio->phys)); diff --git a/drivers/tty/serial/sunzilog.c b/drivers/tty/serial/sunzilog.c index c14275d83b0b4ca8919c0bd2d2f8a2ee61926c3d..87425290687d5b79c6eca4db66d69b0e1e6df9d7 100644 --- a/drivers/tty/serial/sunzilog.c +++ b/drivers/tty/serial/sunzilog.c @@ -938,7 +938,7 @@ sunzilog_convert_to_zs(struct uart_sunzilog_port *up, unsigned int cflag, /* The port lock is not held. */ static void sunzilog_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct uart_sunzilog_port *up = container_of(port, struct uart_sunzilog_port, port); @@ -1307,13 +1307,13 @@ static void sunzilog_register_serio(struct uart_sunzilog_port *up) serio->id.type = SERIO_RS232; if (up->flags & SUNZILOG_FLAG_CONS_KEYB) { serio->id.proto = SERIO_SUNKBD; - strlcpy(serio->name, "zskbd", sizeof(serio->name)); + strscpy(serio->name, "zskbd", sizeof(serio->name)); } else { serio->id.proto = SERIO_SUN; serio->id.extra = 1; - strlcpy(serio->name, "zsms", sizeof(serio->name)); + strscpy(serio->name, "zsms", sizeof(serio->name)); } - strlcpy(serio->phys, + strscpy(serio->phys, ((up->flags & SUNZILOG_FLAG_CONS_KEYB) ? "zs/serio0" : "zs/serio1"), sizeof(serio->phys)); diff --git a/drivers/tty/serial/tegra-tcu.c b/drivers/tty/serial/tegra-tcu.c index 4877c54c613d1f0d899855350b1e9a81a1a001df..23500b342da7105b1caea3dce9ee6e0d1243b92d 100644 --- a/drivers/tty/serial/tegra-tcu.c +++ b/drivers/tty/serial/tegra-tcu.c @@ -101,7 +101,7 @@ static void tegra_tcu_uart_start_tx(struct uart_port *port) break; tegra_tcu_write(tcu, &xmit->buf[xmit->tail], count); - xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1); + uart_xmit_advance(port, count); } uart_write_wakeup(port); @@ -126,7 +126,7 @@ static void tegra_tcu_uart_shutdown(struct uart_port *port) static void tegra_tcu_uart_set_termios(struct uart_port *port, struct ktermios *new, - struct ktermios *old) + const struct ktermios *old) { } diff --git a/drivers/tty/serial/timbuart.c b/drivers/tty/serial/timbuart.c index 08941eabe7b14c21cc3c8fa5a7bd5f7de2608273..bb19ed012def74097e87e29193d701fb7af28983 100644 --- a/drivers/tty/serial/timbuart.c +++ b/drivers/tty/serial/timbuart.c @@ -275,8 +275,8 @@ static int get_bindex(int baud) } static void timbuart_set_termios(struct uart_port *port, - struct ktermios *termios, - struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { unsigned int baud; short bindex; diff --git a/drivers/tty/serial/uartlite.c b/drivers/tty/serial/uartlite.c index 880e2afbb97b652892604117b66a290ebd8296ad..eca41ac5477cb8b8a1722c3a643c1720f2ef4b56 100644 --- a/drivers/tty/serial/uartlite.c +++ b/drivers/tty/serial/uartlite.c @@ -314,8 +314,9 @@ static void ulite_shutdown(struct uart_port *port) clk_disable(pdata->clk); } -static void ulite_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) +static void ulite_set_termios(struct uart_port *port, + struct ktermios *termios, + const struct ktermios *old) { unsigned long flags; struct uartlite_data *pdata = port->private_data; diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c index 3cc9ef08455c2a60e689bd5144d5aeba44b842fc..82cf14dd3d433f83a5b07c66b076622614cc4f68 100644 --- a/drivers/tty/serial/ucc_uart.c +++ b/drivers/tty/serial/ucc_uart.c @@ -843,7 +843,8 @@ static void qe_uart_shutdown(struct uart_port *port) * Set the serial port parameters. */ static void qe_uart_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { struct uart_qe_port *qe_port = container_of(port, struct uart_qe_port, port); @@ -853,13 +854,6 @@ static void qe_uart_set_termios(struct uart_port *port, u16 upsmr = ioread16be(&uccp->upsmr); struct ucc_uart_pram __iomem *uccup = qe_port->uccup; u16 supsmr = ioread16be(&uccup->supsmr); - u8 char_length = 2; /* 1 + CL + PEN + 1 + SL */ - - /* Character length programmed into the mode register is the - * sum of: 1 start bit, number of data bits, 0 or 1 parity bit, - * 1 or 2 stop bits, minus 1. - * The value 'bits' counts this for us. - */ /* byte size */ upsmr &= UCC_UART_UPSMR_CL_MASK; @@ -869,22 +863,18 @@ static void qe_uart_set_termios(struct uart_port *port, case CS5: upsmr |= UCC_UART_UPSMR_CL_5; supsmr |= UCC_UART_SUPSMR_CL_5; - char_length += 5; break; case CS6: upsmr |= UCC_UART_UPSMR_CL_6; supsmr |= UCC_UART_SUPSMR_CL_6; - char_length += 6; break; case CS7: upsmr |= UCC_UART_UPSMR_CL_7; supsmr |= UCC_UART_SUPSMR_CL_7; - char_length += 7; break; default: /* case CS8 */ upsmr |= UCC_UART_UPSMR_CL_8; supsmr |= UCC_UART_SUPSMR_CL_8; - char_length += 8; break; } @@ -892,13 +882,11 @@ static void qe_uart_set_termios(struct uart_port *port, if (termios->c_cflag & CSTOPB) { upsmr |= UCC_UART_UPSMR_SL; supsmr |= UCC_UART_SUPSMR_SL; - char_length++; /* + SL */ } if (termios->c_cflag & PARENB) { upsmr |= UCC_UART_UPSMR_PEN; supsmr |= UCC_UART_SUPSMR_PEN; - char_length++; /* + PEN */ if (!(termios->c_cflag & PARODD)) { upsmr &= ~(UCC_UART_UPSMR_RPM_MASK | @@ -953,7 +941,7 @@ static void qe_uart_set_termios(struct uart_port *port, iowrite16be(upsmr, &uccp->upsmr); if (soft_uart) { iowrite16be(supsmr, &uccup->supsmr); - iowrite8(char_length, &uccup->rx_length); + iowrite8(tty_get_frame_size(termios->c_cflag), &uccup->rx_length); /* Soft-UART requires a 1X multiplier for TX */ qe_setbrg(qe_port->us_info.rx_clock, baud, 16); diff --git a/drivers/tty/serial/vt8500_serial.c b/drivers/tty/serial/vt8500_serial.c index 6f08136ce78a019e23324ae65758923c1c4d5c86..10fbdb09965f3f63aadccb2ad376728e793600f1 100644 --- a/drivers/tty/serial/vt8500_serial.c +++ b/drivers/tty/serial/vt8500_serial.c @@ -187,6 +187,13 @@ static void handle_rx(struct uart_port *port) tty_flip_buffer_push(tport); } +static unsigned int vt8500_tx_empty(struct uart_port *port) +{ + unsigned int idx = vt8500_read(port, VT8500_URFIDX) & 0x1f; + + return idx < 16 ? TIOCSER_TEMT : 0; +} + static void handle_tx(struct uart_port *port) { struct circ_buf *xmit = &port->state->xmit; @@ -201,7 +208,7 @@ static void handle_tx(struct uart_port *port) return; } - while ((vt8500_read(port, VT8500_URFIDX) & 0x1f) < 16) { + while (vt8500_tx_empty(port)) { if (uart_circ_empty(xmit)) break; @@ -260,12 +267,6 @@ static irqreturn_t vt8500_irq(int irq, void *dev_id) return IRQ_HANDLED; } -static unsigned int vt8500_tx_empty(struct uart_port *port) -{ - return (vt8500_read(port, VT8500_URFIDX) & 0x1f) < 16 ? - TIOCSER_TEMT : 0; -} - static unsigned int vt8500_get_mctrl(struct uart_port *port) { unsigned int usr; @@ -355,7 +356,7 @@ static void vt8500_shutdown(struct uart_port *port) static void vt8500_set_termios(struct uart_port *port, struct ktermios *termios, - struct ktermios *old) + const struct ktermios *old) { struct vt8500_port *vt8500_port = container_of(port, struct vt8500_port, uart); diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c index 9e01fe6c0ab8c217b1d4ec3015b38639feb62e82..2eff7cff57c48ccd9f03096d8d992e56c3eafce6 100644 --- a/drivers/tty/serial/xilinx_uartps.c +++ b/drivers/tty/serial/xilinx_uartps.c @@ -2,7 +2,7 @@ /* * Cadence UART driver (found in Xilinx Zynq) * - * 2011 - 2014 (C) Xilinx Inc. + * Copyright (c) 2011 - 2014 Xilinx, Inc. * * This driver has originally been pushed by Xilinx using a Zynq-branding. This * still shows in the naming of this file, the kconfig symbols and some symbols @@ -361,6 +361,8 @@ static irqreturn_t cdns_uart_isr(int irq, void *dev_id) isrstatus &= ~CDNS_UART_IXR_TXEMPTY; } + isrstatus &= port->read_status_mask; + isrstatus &= ~port->ignore_status_mask; /* * Skip RX processing if RX is disabled as RXEMPTY will never be set * as read bytes will not be removed from the FIFO. @@ -675,7 +677,8 @@ static void cdns_uart_break_ctl(struct uart_port *port, int ctl) * @old: Values of the previously saved termios structure */ static void cdns_uart_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old) + struct ktermios *termios, + const struct ktermios *old) { u32 cval = 0; unsigned int baud, minbaud, maxbaud; @@ -1130,8 +1133,35 @@ static struct uart_driver cdns_uart_uart_driver; */ static void cdns_uart_console_putchar(struct uart_port *port, unsigned char ch) { - while (readl(port->membase + CDNS_UART_SR) & CDNS_UART_SR_TXFULL) + unsigned int ctrl_reg; + unsigned long timeout; + + timeout = jiffies + msecs_to_jiffies(1000); + while (1) { + ctrl_reg = readl(port->membase + CDNS_UART_CR); + if (!(ctrl_reg & CDNS_UART_CR_TX_DIS)) + break; + if (time_after(jiffies, timeout)) { + dev_warn(port->dev, + "timeout waiting for Enable\n"); + return; + } + cpu_relax(); + } + + timeout = jiffies + msecs_to_jiffies(1000); + while (1) { + ctrl_reg = readl(port->membase + CDNS_UART_SR); + + if (!(ctrl_reg & CDNS_UART_SR_TXFULL)) + break; + if (time_after(jiffies, timeout)) { + dev_warn(port->dev, + "timeout waiting for TX fifo\n"); + return; + } cpu_relax(); + } writel(ch, port->membase + CDNS_UART_FIFO); } @@ -1329,12 +1359,20 @@ static int cdns_uart_resume(struct device *device) unsigned long flags; u32 ctrl_reg; int may_wake; + int ret; may_wake = device_may_wakeup(device); if (console_suspend_enabled && uart_console(port) && !may_wake) { - clk_enable(cdns_uart->pclk); - clk_enable(cdns_uart->uartclk); + ret = clk_enable(cdns_uart->pclk); + if (ret) + return ret; + + ret = clk_enable(cdns_uart->uartclk); + if (ret) { + clk_disable(cdns_uart->pclk); + return ret; + } spin_lock_irqsave(&port->lock, flags); @@ -1383,9 +1421,17 @@ static int __maybe_unused cdns_runtime_resume(struct device *dev) { struct uart_port *port = dev_get_drvdata(dev); struct cdns_uart *cdns_uart = port->private_data; + int ret; - clk_enable(cdns_uart->pclk); - clk_enable(cdns_uart->uartclk); + ret = clk_enable(cdns_uart->pclk); + if (ret) + return ret; + + ret = clk_enable(cdns_uart->uartclk); + if (ret) { + clk_disable(cdns_uart->pclk); + return ret; + } return 0; }; @@ -1551,6 +1597,8 @@ static int cdns_uart_probe(struct platform_device *pdev) port->dev = &pdev->dev; port->uartclk = clk_get_rate(cdns_uart_data->uartclk); port->private_data = cdns_uart_data; + port->read_status_mask = CDNS_UART_IXR_TXEMPTY | CDNS_UART_IXR_RXTRIG | + CDNS_UART_IXR_OVERRUN | CDNS_UART_IXR_TOUT; cdns_uart_data->port = port; platform_set_drvdata(pdev, port); diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c index 5bc58591665a0c7c971142ca212e846731faf2f6..688db7d8b7488bdb91886ebfb3e5ca6c95065c00 100644 --- a/drivers/tty/serial/zs.c +++ b/drivers/tty/serial/zs.c @@ -846,7 +846,7 @@ static void zs_reset(struct zs_port *zport) } static void zs_set_termios(struct uart_port *uport, struct ktermios *termios, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct zs_port *zport = to_zport(uport); struct zs_scc *scc = zport->scc; diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index 9bc2a926527726bb39ea282ba1c497a552d1425b..25e9befdda3a8f07ff4b13bf4e315bc132bc5f79 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -91,7 +91,6 @@ static char *driver_name = "SyncLink GT"; static char *slgt_driver_name = "synclink_gt"; static char *tty_dev_prefix = "ttySLG"; MODULE_LICENSE("GPL"); -#define MGSL_MAGIC 0x5401 #define MAX_DEVICES 32 static const struct pci_device_id pci_table[] = { @@ -215,8 +214,6 @@ struct slgt_info { struct slgt_info *next_device; /* device list link */ - int magic; - char device_name[25]; struct pci_dev *pdev; @@ -554,10 +551,6 @@ static inline int sanity_check(struct slgt_info *info, char *devname, const char printk("null struct slgt_info for (%s) in %s\n", devname, name); return 1; } - if (info->magic != MGSL_MAGIC) { - printk("bad magic number struct slgt_info (%s) in %s\n", devname, name); - return 1; - } #else if (!info) return 1; @@ -707,7 +700,8 @@ static void hangup(struct tty_struct *tty) wake_up_interruptible(&info->port.open_wait); } -static void set_termios(struct tty_struct *tty, struct ktermios *old_termios) +static void set_termios(struct tty_struct *tty, + const struct ktermios *old_termios) { struct slgt_info *info = tty->driver_data; unsigned long flags; @@ -3498,7 +3492,6 @@ static struct slgt_info *alloc_dev(int adapter_num, int port_num, struct pci_dev } else { tty_port_init(&info->port); info->port.ops = &slgt_port_ops; - info->magic = MGSL_MAGIC; INIT_WORK(&info->task, bh_handler); info->max_frame_size = 4096; info->base_clock = 14745600; diff --git a/drivers/tty/tty.h b/drivers/tty/tty.h index f310a8274df1539a9a71c2220fbd7fe2b9067f17..1c08c9b67b16ca3f9990a8fd8b2cdefa619b3786 100644 --- a/drivers/tty/tty.h +++ b/drivers/tty/tty.h @@ -73,7 +73,7 @@ void tty_buffer_set_lock_subclass(struct tty_port *port); bool tty_buffer_restart_work(struct tty_port *port); bool tty_buffer_cancel_work(struct tty_port *port); void tty_buffer_flush_work(struct tty_port *port); -speed_t tty_termios_input_baud_rate(struct ktermios *termios); +speed_t tty_termios_input_baud_rate(const struct ktermios *termios); void tty_ldisc_hangup(struct tty_struct *tty, bool reset); int tty_ldisc_reinit(struct tty_struct *tty, int disc); long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg); diff --git a/drivers/tty/tty_baudrate.c b/drivers/tty/tty_baudrate.c index 3cd99ed7c7102c9e1db051e97716766a0b56bd06..f9b49939c27b6f61ea0eab3e9917216dc6194c99 100644 --- a/drivers/tty/tty_baudrate.c +++ b/drivers/tty/tty_baudrate.c @@ -49,13 +49,13 @@ static int n_baud_table = ARRAY_SIZE(baud_table); * * Convert termios baud rate data into a speed. This should be called * with the termios lock held if this termios is a terminal termios - * structure. May change the termios data. Device drivers can call this - * function but should use ->c_[io]speed directly as they are updated. + * structure. Device drivers can call this function but should use + * ->c_[io]speed directly as they are updated. * * Locking: none */ -speed_t tty_termios_baud_rate(struct ktermios *termios) +speed_t tty_termios_baud_rate(const struct ktermios *termios) { unsigned int cbaud; @@ -67,11 +67,7 @@ speed_t tty_termios_baud_rate(struct ktermios *termios) if (cbaud & CBAUDEX) { cbaud &= ~CBAUDEX; - - if (cbaud < 1 || cbaud + 15 > n_baud_table) - termios->c_cflag &= ~CBAUDEX; - else - cbaud += 15; + cbaud += 15; } return cbaud >= n_baud_table ? 0 : baud_table[cbaud]; } @@ -83,30 +79,26 @@ EXPORT_SYMBOL(tty_termios_baud_rate); * * Convert termios baud rate data into a speed. This should be called * with the termios lock held if this termios is a terminal termios - * structure. May change the termios data. Device drivers can call this - * function but should use ->c_[io]speed directly as they are updated. + * structure. Device drivers can call this function but should use + * ->c_[io]speed directly as they are updated. * * Locking: none */ -speed_t tty_termios_input_baud_rate(struct ktermios *termios) +speed_t tty_termios_input_baud_rate(const struct ktermios *termios) { unsigned int cbaud = (termios->c_cflag >> IBSHIFT) & CBAUD; if (cbaud == B0) return tty_termios_baud_rate(termios); - /* Magic token for arbitrary speed via c_ispeed*/ + /* Magic token for arbitrary speed via c_ispeed */ if (cbaud == BOTHER) return termios->c_ispeed; if (cbaud & CBAUDEX) { cbaud &= ~CBAUDEX; - - if (cbaud < 1 || cbaud + 15 > n_baud_table) - termios->c_cflag &= ~(CBAUDEX << IBSHIFT); - else - cbaud += 15; + cbaud += 15; } return cbaud >= n_baud_table ? 0 : baud_table[cbaud]; } diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c index 9fdecc795b6beb914d746ca22151f3669c510159..5e287dedce0185e5221e0d6b8a71adad32ca39b7 100644 --- a/drivers/tty/tty_buffer.c +++ b/drivers/tty/tty_buffer.c @@ -470,7 +470,6 @@ static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) while (head) { struct tty_buffer *next; - unsigned char *p, *f = NULL; unsigned int count; /* @@ -489,11 +488,16 @@ static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) continue; } - p = char_buf_ptr(head, head->lookahead); - if (~head->flags & TTYB_NORMAL) - f = flag_buf_ptr(head, head->lookahead); + if (port->client_ops->lookahead_buf) { + unsigned char *p, *f = NULL; + + p = char_buf_ptr(head, head->lookahead); + if (~head->flags & TTYB_NORMAL) + f = flag_buf_ptr(head, head->lookahead); + + port->client_ops->lookahead_buf(port, p, f, count); + } - port->client_ops->lookahead_buf(port, p, f, count); head->lookahead += count; } } diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 82a8855981f7cc8d784a37f4df238f06344c0964..de06c3c2ff70ac79a7bd5dff790611534d3f2da6 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -99,8 +99,8 @@ #include #include #include - #include +#include #include #include @@ -170,7 +170,6 @@ static void free_tty_struct(struct tty_struct *tty) tty_ldisc_deinit(tty); put_device(tty->dev); kvfree(tty->write_buf); - tty->magic = 0xDEADDEAD; kfree(tty); } @@ -265,11 +264,6 @@ static int tty_paranoia_check(struct tty_struct *tty, struct inode *inode, imajor(inode), iminor(inode), routine); return 1; } - if (tty->magic != TTY_MAGIC) { - pr_warn("(%d:%d): %s: bad magic number\n", - imajor(inode), iminor(inode), routine); - return 1; - } #endif return 0; } @@ -1533,7 +1527,6 @@ static void release_one_tty(struct work_struct *work) if (tty->ops->cleanup) tty->ops->cleanup(tty); - tty->magic = 0; tty_driver_kref_put(driver); module_put(owner); @@ -3093,7 +3086,6 @@ struct tty_struct *alloc_tty_struct(struct tty_driver *driver, int idx) return NULL; kref_init(&tty->kref); - tty->magic = TTY_MAGIC; if (tty_ldisc_init(tty)) { kfree(tty); return NULL; @@ -3329,7 +3321,6 @@ struct tty_driver *__tty_alloc_driver(unsigned int lines, struct module *owner, return ERR_PTR(-ENOMEM); kref_init(&driver->kref); - driver->magic = TTY_DRIVER_MAGIC; driver->num = lines; driver->owner = owner; driver->flags = flags; diff --git a/drivers/tty/tty_ioctl.c b/drivers/tty/tty_ioctl.c index 2a76b330e1089eb935da2a4439f4835ed6394209..ce511557b98b1f6c58f9b9e608693fbe0c00fd76 100644 --- a/drivers/tty/tty_ioctl.c +++ b/drivers/tty/tty_ioctl.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "tty.h" #include @@ -219,7 +220,7 @@ EXPORT_SYMBOL(tty_wait_until_sent); * Termios Helper Methods */ -static void unset_locked_termios(struct tty_struct *tty, struct ktermios *old) +static void unset_locked_termios(struct tty_struct *tty, const struct ktermios *old) { struct ktermios *termios = &tty->termios; struct ktermios *locked = &tty->termios_locked; @@ -249,7 +250,7 @@ static void unset_locked_termios(struct tty_struct *tty, struct ktermios *old) * in some cases where only minimal reconfiguration is supported */ -void tty_termios_copy_hw(struct ktermios *new, struct ktermios *old) +void tty_termios_copy_hw(struct ktermios *new, const struct ktermios *old) { /* The bits a dumb device handles in software. Smart devices need to always provide a set_termios method */ @@ -374,6 +375,80 @@ int tty_set_termios(struct tty_struct *tty, struct ktermios *new_termios) } EXPORT_SYMBOL_GPL(tty_set_termios); + +/* + * Translate a "termio" structure into a "termios". Ugh. + */ +__weak int user_termio_to_kernel_termios(struct ktermios *termios, + struct termio __user *termio) +{ + struct termio v; + + if (copy_from_user(&v, termio, sizeof(struct termio))) + return -EFAULT; + + termios->c_iflag = (0xffff0000 & termios->c_iflag) | v.c_iflag; + termios->c_oflag = (0xffff0000 & termios->c_oflag) | v.c_oflag; + termios->c_cflag = (0xffff0000 & termios->c_cflag) | v.c_cflag; + termios->c_lflag = (0xffff0000 & termios->c_lflag) | v.c_lflag; + termios->c_line = (0xffff0000 & termios->c_lflag) | v.c_line; + memcpy(termios->c_cc, v.c_cc, NCC); + return 0; +} + +/* + * Translate a "termios" structure into a "termio". Ugh. + */ +__weak int kernel_termios_to_user_termio(struct termio __user *termio, + struct ktermios *termios) +{ + struct termio v; + memset(&v, 0, sizeof(struct termio)); + v.c_iflag = termios->c_iflag; + v.c_oflag = termios->c_oflag; + v.c_cflag = termios->c_cflag; + v.c_lflag = termios->c_lflag; + v.c_line = termios->c_line; + memcpy(v.c_cc, termios->c_cc, NCC); + return copy_to_user(termio, &v, sizeof(struct termio)); +} + +#ifdef TCGETS2 +__weak int user_termios_to_kernel_termios(struct ktermios *k, + struct termios2 __user *u) +{ + return copy_from_user(k, u, sizeof(struct termios2)); +} +__weak int kernel_termios_to_user_termios(struct termios2 __user *u, + struct ktermios *k) +{ + return copy_to_user(u, k, sizeof(struct termios2)); +} +__weak int user_termios_to_kernel_termios_1(struct ktermios *k, + struct termios __user *u) +{ + return copy_from_user(k, u, sizeof(struct termios)); +} +__weak int kernel_termios_to_user_termios_1(struct termios __user *u, + struct ktermios *k) +{ + return copy_to_user(u, k, sizeof(struct termios)); +} + +#else + +__weak int user_termios_to_kernel_termios(struct ktermios *k, + struct termios __user *u) +{ + return copy_from_user(k, u, sizeof(struct termios)); +} +__weak int kernel_termios_to_user_termios(struct termios __user *u, + struct ktermios *k) +{ + return copy_to_user(u, k, sizeof(struct termios)); +} +#endif /* TCGETS2 */ + /** * set_termios - set termios values for a tty * @tty: terminal device diff --git a/drivers/tty/tty_mutex.c b/drivers/tty/tty_mutex.c index 393518a24cfe230e55e623697a5fe10dcd703665..784e46a0a3b1013d9eaf07f50d9ce8c4bd882fa7 100644 --- a/drivers/tty/tty_mutex.c +++ b/drivers/tty/tty_mutex.c @@ -14,8 +14,6 @@ void tty_lock(struct tty_struct *tty) { - if (WARN(tty->magic != TTY_MAGIC, "L Bad %p\n", tty)) - return; tty_kref_get(tty); mutex_lock(&tty->legacy_mutex); } @@ -25,8 +23,6 @@ int tty_lock_interruptible(struct tty_struct *tty) { int ret; - if (WARN(tty->magic != TTY_MAGIC, "L Bad %p\n", tty)) - return -EIO; tty_kref_get(tty); ret = mutex_lock_interruptible(&tty->legacy_mutex); if (ret) @@ -36,8 +32,6 @@ int tty_lock_interruptible(struct tty_struct *tty) void tty_unlock(struct tty_struct *tty) { - if (WARN(tty->magic != TTY_MAGIC, "U Bad %p\n", tty)) - return; mutex_unlock(&tty->legacy_mutex); tty_kref_put(tty); } diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c index e11383ae1e7e317c61d70a83aea67edde4023c28..34ba6e54789a7fafbadb9334c788f18984c60771 100644 --- a/drivers/tty/vcc.c +++ b/drivers/tty/vcc.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index ae9c926acd6f962fc22d6e72fa3cb3cd4983d869..981d2bfcf9a5ba2fb89f364d6e112fcfc0966e73 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -154,10 +154,10 @@ static void console_callback(struct work_struct *ignored); static void con_driver_unregister_callback(struct work_struct *ignored); static void blank_screen_t(struct timer_list *unused); static void set_palette(struct vc_data *vc); +static void unblank_screen(void); #define vt_get_kmsg_redirect() vt_kmsg_redirect(-1) -static int printable; /* Is console ready for printing? */ int default_utf8 = true; module_param(default_utf8, int, S_IRUGO | S_IWUSR); int global_cursor_default = -1; @@ -3084,9 +3084,9 @@ static void vt_console_print(struct console *co, const char *b, unsigned count) ushort start_x, cnt; int kmsg_console; - /* console busy or not yet initialized */ - if (!printable) - return; + WARN_CONSOLE_UNLOCKED(); + + /* this protects against concurrent oops only */ if (!spin_trylock(&printing_lock)) return; @@ -3537,7 +3537,6 @@ static int __init con_init(void) pr_info("Console: %s %s %dx%d\n", vc->vc_can_do_color ? "colour" : "mono", display_desc, vc->vc_cols, vc->vc_rows); - printable = 1; console_unlock(); @@ -4452,7 +4451,7 @@ EXPORT_SYMBOL(do_unblank_screen); * call it with 1 as an argument and so force a mode restore... that may kill * X or at least garbage the screen but would also make the Oops visible... */ -void unblank_screen(void) +static void unblank_screen(void) { do_unblank_screen(0); } @@ -4662,9 +4661,11 @@ static int con_font_set(struct vc_data *vc, struct console_font_op *op) console_lock(); if (vc->vc_mode != KD_TEXT) rc = -EINVAL; - else if (vc->vc_sw->con_font_set) + else if (vc->vc_sw->con_font_set) { + if (vc_is_sel(vc)) + clear_selection(); rc = vc->vc_sw->con_font_set(vc, &font, op->flags); - else + } else rc = -ENOSYS; console_unlock(); kfree(font.data); @@ -4691,9 +4692,11 @@ static int con_font_default(struct vc_data *vc, struct console_font_op *op) console_unlock(); return -EINVAL; } - if (vc->vc_sw->con_font_default) + if (vc->vc_sw->con_font_default) { + if (vc_is_sel(vc)) + clear_selection(); rc = vc->vc_sw->con_font_default(vc, &font, s); - else + } else rc = -ENOSYS; console_unlock(); if (!rc) { diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c index 0a088b47d5570047de5457fc93bfc53953f4ebea..53aea56d1de135bf39ef2d013c3fe52b85cbe9d1 100644 --- a/drivers/ufs/core/ufs-sysfs.c +++ b/drivers/ufs/core/ufs-sysfs.c @@ -225,12 +225,13 @@ static ssize_t wb_on_store(struct device *dev, struct device_attribute *attr, unsigned int wb_enable; ssize_t res; - if (!ufshcd_is_wb_allowed(hba) || ufshcd_is_clkscaling_supported(hba)) { + if (!ufshcd_is_wb_allowed(hba) || (ufshcd_is_clkscaling_supported(hba) + && ufshcd_enable_wb_if_scaling_up(hba))) { /* * If the platform supports UFSHCD_CAP_CLK_SCALING, turn WB * on/off will be done while clock scaling up/down. */ - dev_warn(dev, "To control WB through wb_on is not allowed!\n"); + dev_warn(dev, "It is not allowed to configure WB!\n"); return -EOPNOTSUPP; } @@ -254,6 +255,49 @@ out: return res < 0 ? res : count; } +static ssize_t enable_wb_buf_flush_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", hba->dev_info.wb_buf_flush_enabled); +} + +static ssize_t enable_wb_buf_flush_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + unsigned int enable_wb_buf_flush; + ssize_t res; + + if (!ufshcd_is_wb_buf_flush_allowed(hba)) { + dev_warn(dev, "It is not allowed to configure WB buf flushing!\n"); + return -EOPNOTSUPP; + } + + if (kstrtouint(buf, 0, &enable_wb_buf_flush)) + return -EINVAL; + + if (enable_wb_buf_flush != 0 && enable_wb_buf_flush != 1) + return -EINVAL; + + down(&hba->host_sem); + if (!ufshcd_is_user_access_allowed(hba)) { + res = -EBUSY; + goto out; + } + + ufshcd_rpm_get_sync(hba); + res = ufshcd_wb_toggle_buf_flush(hba, enable_wb_buf_flush); + ufshcd_rpm_put_sync(hba); + +out: + up(&hba->host_sem); + return res < 0 ? res : count; +} + static DEVICE_ATTR_RW(rpm_lvl); static DEVICE_ATTR_RO(rpm_target_dev_state); static DEVICE_ATTR_RO(rpm_target_link_state); @@ -262,6 +306,7 @@ static DEVICE_ATTR_RO(spm_target_dev_state); static DEVICE_ATTR_RO(spm_target_link_state); static DEVICE_ATTR_RW(auto_hibern8); static DEVICE_ATTR_RW(wb_on); +static DEVICE_ATTR_RW(enable_wb_buf_flush); static struct attribute *ufs_sysfs_ufshcd_attrs[] = { &dev_attr_rpm_lvl.attr, @@ -272,6 +317,7 @@ static struct attribute *ufs_sysfs_ufshcd_attrs[] = { &dev_attr_spm_target_link_state.attr, &dev_attr_auto_hibern8.attr, &dev_attr_wb_on.attr, + &dev_attr_enable_wb_buf_flush.attr, NULL }; @@ -279,6 +325,40 @@ static const struct attribute_group ufs_sysfs_default_group = { .attrs = ufs_sysfs_ufshcd_attrs, }; +static ssize_t clock_scaling_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", ufshcd_is_clkscaling_supported(hba)); +} + +static ssize_t write_booster_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", ufshcd_is_wb_allowed(hba)); +} + +static DEVICE_ATTR_RO(clock_scaling); +static DEVICE_ATTR_RO(write_booster); + +/* + * See Documentation/ABI/testing/sysfs-driver-ufs for the semantics of this + * group. + */ +static struct attribute *ufs_sysfs_capabilities_attrs[] = { + &dev_attr_clock_scaling.attr, + &dev_attr_write_booster.attr, + NULL +}; + +static const struct attribute_group ufs_sysfs_capabilities_group = { + .name = "capabilities", + .attrs = ufs_sysfs_capabilities_attrs, +}; + static ssize_t monitor_enable_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -1134,6 +1214,7 @@ static const struct attribute_group ufs_sysfs_attributes_group = { static const struct attribute_group *ufs_sysfs_groups[] = { &ufs_sysfs_default_group, + &ufs_sysfs_capabilities_group, &ufs_sysfs_monitor_group, &ufs_sysfs_device_descriptor_group, &ufs_sysfs_interconnect_descriptor_group, diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 8f67db202d7b7bc8747247348244e74ae8a2c846..f68ca33f6ac78c7ce2fdb656ed51421f7841e102 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -26,6 +26,12 @@ static inline u8 ufshcd_wb_get_query_index(struct ufs_hba *hba) return 0; } +static inline bool ufshcd_is_wb_buf_flush_allowed(struct ufs_hba *hba) +{ + return ufshcd_is_wb_allowed(hba) && + !(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL); +} + #ifdef CONFIG_SCSI_UFS_HWMON void ufs_hwmon_probe(struct ufs_hba *hba, u8 mask); void ufs_hwmon_remove(struct ufs_hba *hba); @@ -36,6 +42,11 @@ static inline void ufs_hwmon_remove(struct ufs_hba *hba) {} static inline void ufs_hwmon_notify_event(struct ufs_hba *hba, u8 ee_mask) {} #endif +int ufshcd_query_descriptor_retry(struct ufs_hba *hba, + enum query_opcode opcode, + enum desc_idn idn, u8 index, + u8 selector, + u8 *desc_buf, int *buf_len); int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, int desc_index, diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 6bc679d22927998eacbd779d69d4cfa9a9f9b6c5..7256e6c43ca68de9261af07b9a99427126584515 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -265,8 +266,8 @@ static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, struct ufs_vreg *vreg); static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); -static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set); -static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable); +static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, + bool enable); static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); @@ -286,16 +287,17 @@ static inline void ufshcd_disable_irq(struct ufs_hba *hba) } } -static inline void ufshcd_wb_config(struct ufs_hba *hba) +static void ufshcd_configure_wb(struct ufs_hba *hba) { if (!ufshcd_is_wb_allowed(hba)) return; ufshcd_wb_toggle(hba, true); - ufshcd_wb_toggle_flush_during_h8(hba, true); - if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)) - ufshcd_wb_toggle_flush(hba, true); + ufshcd_wb_toggle_buf_flush_during_h8(hba, true); + + if (ufshcd_is_wb_buf_flush_allowed(hba)) + ufshcd_wb_toggle_buf_flush(hba, true); } static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba) @@ -457,7 +459,7 @@ static void ufshcd_print_evt(struct ufs_hba *hba, u32 id, if (e->tstamp[p] == 0) continue; dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p, - e->val[p], ktime_to_us(e->tstamp[p])); + e->val[p], div_u64(e->tstamp[p], 1000)); found = true; } @@ -502,9 +504,9 @@ void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt) lrbp = &hba->lrb[tag]; dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", - tag, ktime_to_us(lrbp->issue_time_stamp)); + tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", - tag, ktime_to_us(lrbp->compl_time_stamp)); + tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); dev_err(hba->dev, "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", tag, (u64)lrbp->utrd_dma_addr); @@ -566,10 +568,10 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state); dev_err(hba->dev, "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", - ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp), + div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), hba->ufs_stats.hibern8_exit_cnt); dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n", - ktime_to_us(hba->ufs_stats.last_intr_ts), + div_u64(hba->ufs_stats.last_intr_ts, 1000), hba->ufs_stats.last_intr_status); dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", hba->eh_flags, hba->req_abort_count); @@ -1298,9 +1300,11 @@ static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) } /* Enable Write Booster if we have scaled up else disable it */ - downgrade_write(&hba->clk_scaling_lock); - is_writelock = false; - ufshcd_wb_toggle(hba, scale_up); + if (ufshcd_enable_wb_if_scaling_up(hba)) { + downgrade_write(&hba->clk_scaling_lock); + is_writelock = false; + ufshcd_wb_toggle(hba, scale_up); + } out_unprepare: ufshcd_clock_scaling_unprepare(hba, is_writelock); @@ -2140,7 +2144,9 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) unsigned long flags; lrbp->issue_time_stamp = ktime_get(); + lrbp->issue_time_stamp_local_clock = local_clock(); lrbp->compl_time_stamp = ktime_set(0, 0); + lrbp->compl_time_stamp_local_clock = 0; ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); ufshcd_clk_scaling_start_busy(hba); if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) @@ -2701,9 +2707,9 @@ static inline bool is_device_wlun(struct scsi_device *sdev) * Associate the UFS controller queue with the default and poll HCTX types. * Initialize the mq_map[] arrays. */ -static int ufshcd_map_queues(struct Scsi_Host *shost) +static void ufshcd_map_queues(struct Scsi_Host *shost) { - int i, ret; + int i; for (i = 0; i < shost->nr_maps; i++) { struct blk_mq_queue_map *map = &shost->tag_set.map[i]; @@ -2720,11 +2726,8 @@ static int ufshcd_map_queues(struct Scsi_Host *shost) WARN_ON_ONCE(true); } map->queue_offset = 0; - ret = blk_mq_map_queues(map); - WARN_ON_ONCE(ret); + blk_mq_map_queues(map); } - - return 0; } static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) @@ -4222,7 +4225,7 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) } else { ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, POST_CHANGE); - hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get(); + hba->ufs_stats.last_hibern8_exit_tstamp = local_clock(); hba->ufs_stats.hibern8_exit_cnt++; } @@ -4724,7 +4727,7 @@ void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val) e = &hba->ufs_stats.event[id]; e->val[e->pos] = val; - e->tstamp[e->pos] = ktime_get(); + e->tstamp[e->pos] = local_clock(); e->cnt += 1; e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH; @@ -5357,6 +5360,7 @@ static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, for_each_set_bit(index, &completed_reqs, hba->nutrs) { lrbp = &hba->lrb[index]; lrbp->compl_time_stamp = ktime_get(); + lrbp->compl_time_stamp_local_clock = local_clock(); cmd = lrbp->cmd; if (cmd) { if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) @@ -5752,60 +5756,60 @@ int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable) { int ret; - if (!ufshcd_is_wb_allowed(hba)) - return 0; - - if (!(enable ^ hba->dev_info.wb_enabled)) + if (!ufshcd_is_wb_allowed(hba) || + hba->dev_info.wb_enabled == enable) return 0; ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN); if (ret) { - dev_err(hba->dev, "%s Write Booster %s failed %d\n", - __func__, enable ? "enable" : "disable", ret); + dev_err(hba->dev, "%s: Write Booster %s failed %d\n", + __func__, enable ? "enabling" : "disabling", ret); return ret; } hba->dev_info.wb_enabled = enable; - dev_dbg(hba->dev, "%s Write Booster %s\n", + dev_dbg(hba->dev, "%s: Write Booster %s\n", __func__, enable ? "enabled" : "disabled"); return ret; } -static void ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set) +static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, + bool enable) { int ret; - ret = __ufshcd_wb_toggle(hba, set, + ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8); if (ret) { - dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed: %d\n", - __func__, set ? "enable" : "disable", ret); + dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n", + __func__, enable ? "enabling" : "disabling", ret); return; } - dev_dbg(hba->dev, "%s WB-Buf Flush during H8 %s\n", - __func__, set ? "enabled" : "disabled"); + dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n", + __func__, enable ? "enabled" : "disabled"); } -static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable) +int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) { int ret; if (!ufshcd_is_wb_allowed(hba) || hba->dev_info.wb_buf_flush_enabled == enable) - return; + return 0; ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN); if (ret) { - dev_err(hba->dev, "%s WB-Buf Flush %s failed %d\n", __func__, - enable ? "enable" : "disable", ret); - return; + dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n", + __func__, enable ? "enabling" : "disabling", ret); + return ret; } hba->dev_info.wb_buf_flush_enabled = enable; - - dev_dbg(hba->dev, "%s WB-Buf Flush %s\n", + dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n", __func__, enable ? "enabled" : "disabled"); + + return ret; } static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, @@ -5820,7 +5824,7 @@ static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE, index, 0, &cur_buf); if (ret) { - dev_err(hba->dev, "%s dCurWriteBoosterBufferSize read failed %d\n", + dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n", __func__, ret); return false; } @@ -5836,10 +5840,10 @@ static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, static void ufshcd_wb_force_disable(struct ufs_hba *hba) { - if (!(hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)) - ufshcd_wb_toggle_flush(hba, false); + if (ufshcd_is_wb_buf_flush_allowed(hba)) + ufshcd_wb_toggle_buf_flush(hba, false); - ufshcd_wb_toggle_flush_during_h8(hba, false); + ufshcd_wb_toggle_buf_flush_during_h8(hba, false); ufshcd_wb_toggle(hba, false); hba->caps &= ~UFSHCD_CAP_WB_EN; @@ -5905,7 +5909,7 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE, index, 0, &avail_buf); if (ret) { - dev_warn(hba->dev, "%s dAvailableWriteBoosterBufferSize read failed %d\n", + dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n", __func__, ret); return false; } @@ -6645,7 +6649,7 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba) intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); hba->ufs_stats.last_intr_status = intr_status; - hba->ufs_stats.last_intr_ts = ktime_get(); + hba->ufs_stats.last_intr_ts = local_clock(); /* * There could be max of hba->nutrs reqs in flight and in worst case @@ -8236,7 +8240,9 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) */ ufshcd_set_active_icc_lvl(hba); - ufshcd_wb_config(hba); + /* Enable UFS Write Booster if supported */ + ufshcd_configure_wb(hba); + if (hba->ee_usr_mask) ufshcd_write_ee_control(hba); /* Enable Auto-Hibernate if configured */ @@ -8741,6 +8747,8 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, struct scsi_device *sdp; unsigned long flags; int ret, retries; + unsigned long deadline; + int32_t remaining; spin_lock_irqsave(hba->host->host_lock, flags); sdp = hba->ufs_device_wlun; @@ -8773,9 +8781,14 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * callbacks hence set the RQF_PM flag so that it doesn't resume the * already suspended childs. */ + deadline = jiffies + 10 * HZ; for (retries = 3; retries > 0; --retries) { + ret = -ETIMEDOUT; + remaining = deadline - jiffies; + if (remaining <= 0) + break; ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr, - START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL); + remaining / HZ, 0, 0, RQF_PM, NULL); if (!scsi_status_is_check_condition(ret) || !scsi_sense_valid(&sshdr) || sshdr.sense_key != UNIT_ATTENTION) diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c index a1a7a1175a5a61da3f35d39d93fc196fc896529a..3d69a81c5b1783e48b5b92a0f37f3ab8af912590 100644 --- a/drivers/ufs/core/ufshpb.c +++ b/drivers/ufs/core/ufshpb.c @@ -613,14 +613,17 @@ static void ufshpb_activate_subregion(struct ufshpb_lu *hpb, srgn->srgn_state = HPB_SRGN_VALID; } -static void ufshpb_umap_req_compl_fn(struct request *req, blk_status_t error) +static enum rq_end_io_ret ufshpb_umap_req_compl_fn(struct request *req, + blk_status_t error) { struct ufshpb_req *umap_req = (struct ufshpb_req *)req->end_io_data; ufshpb_put_req(umap_req->hpb, umap_req); + return RQ_END_IO_NONE; } -static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) +static enum rq_end_io_ret ufshpb_map_req_compl_fn(struct request *req, + blk_status_t error) { struct ufshpb_req *map_req = (struct ufshpb_req *) req->end_io_data; struct ufshpb_lu *hpb = map_req->hpb; @@ -636,6 +639,7 @@ static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); ufshpb_put_map_req(map_req->hpb, map_req); + return RQ_END_IO_NONE; } static void ufshpb_set_unmap_cmd(unsigned char *cdb, struct ufshpb_region *rgn) diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c index eced9753808203f1258b89e4246f4431f6469698..c3628a8645a565a2e373a46d16bf2aa13c2cc012 100644 --- a/drivers/ufs/host/ufs-exynos.c +++ b/drivers/ufs/host/ufs-exynos.c @@ -1711,7 +1711,7 @@ static struct exynos_ufs_uic_attr fsd_uic_attr = { .pa_dbg_option_suite = 0x2E820183, }; -struct exynos_ufs_drv_data fsd_ufs_drvs = { +static const struct exynos_ufs_drv_data fsd_ufs_drvs = { .uic_attr = &fsd_uic_attr, .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | diff --git a/drivers/ufs/host/ufs-mediatek-trace.h b/drivers/ufs/host/ufs-mediatek-trace.h index 7e010848dc993c23ed7e8d6e90c4ebd49b9aeb1e..b5f2ec3140748f0c4ba1f14156299f47f2584987 100644 --- a/drivers/ufs/host/ufs-mediatek-trace.h +++ b/drivers/ufs/host/ufs-mediatek-trace.h @@ -24,9 +24,32 @@ TRACE_EVENT(ufs_mtk_event, __entry->data = data; ), - TP_printk("ufs:event=%u data=%u", + TP_printk("ufs: event=%u data=%u", __entry->type, __entry->data) - ); +); + +TRACE_EVENT(ufs_mtk_clk_scale, + TP_PROTO(const char *name, bool scale_up, unsigned long clk_rate), + TP_ARGS(name, scale_up, clk_rate), + + TP_STRUCT__entry( + __field(const char*, name) + __field(bool, scale_up) + __field(unsigned long, clk_rate) + ), + + TP_fast_assign( + __entry->name = name; + __entry->scale_up = scale_up; + __entry->clk_rate = clk_rate; + ), + + TP_printk("ufs: clk (%s) scaled %s @ %lu", + __entry->name, + __entry->scale_up ? "up" : "down", + __entry->clk_rate) +); + #endif #undef TRACE_INCLUDE_PATH diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index c958279bdd8fb4b04c4b0c0fb8e016ddca60a60d..7309f3f87eacf1982d60fb8721aad2573bddb694 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -47,6 +46,44 @@ static const struct of_device_id ufs_mtk_of_match[] = { {}, }; +/* + * Details of UIC Errors + */ +static const char *const ufs_uic_err_str[] = { + "PHY Adapter Layer", + "Data Link Layer", + "Network Link Layer", + "Transport Link Layer", + "DME" +}; + +static const char *const ufs_uic_pa_err_str[] = { + "PHY error on Lane 0", + "PHY error on Lane 1", + "PHY error on Lane 2", + "PHY error on Lane 3", + "Generic PHY Adapter Error. This should be the LINERESET indication" +}; + +static const char *const ufs_uic_dl_err_str[] = { + "NAC_RECEIVED", + "TCx_REPLAY_TIMER_EXPIRED", + "AFCx_REQUEST_TIMER_EXPIRED", + "FCx_PROTECTION_TIMER_EXPIRED", + "CRC_ERROR", + "RX_BUFFER_OVERFLOW", + "MAX_FRAME_LENGTH_EXCEEDED", + "WRONG_SEQUENCE_NUMBER", + "AFC_FRAME_SYNTAX_ERROR", + "NAC_FRAME_SYNTAX_ERROR", + "EOF_SYNTAX_ERROR", + "FRAME_SYNTAX_ERROR", + "BAD_CTRL_SYMBOL_TYPE", + "PA_INIT_ERROR", + "PA_ERROR_IND_RECEIVED", + "PA_INIT" +}; + static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); @@ -598,6 +635,12 @@ static void ufs_mtk_boost_pm_qos(struct ufs_hba *hba, bool boost) boost ? 0 : PM_QOS_DEFAULT_VALUE); } +static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool scale_up) +{ + ufs_mtk_boost_crypt(hba, scale_up); + ufs_mtk_boost_pm_qos(hba, scale_up); +} + static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on) { struct ufs_mtk_host *host = ufshcd_get_variant(hba); @@ -605,11 +648,11 @@ static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on) if (on) { phy_power_on(host->mphy); ufs_mtk_setup_ref_clk(hba, on); - ufs_mtk_boost_crypt(hba, on); - ufs_mtk_boost_pm_qos(hba, on); + if (!ufshcd_is_clkscaling_supported(hba)) + ufs_mtk_scale_perf(hba, on); } else { - ufs_mtk_boost_pm_qos(hba, on); - ufs_mtk_boost_crypt(hba, on); + if (!ufshcd_is_clkscaling_supported(hba)) + ufs_mtk_scale_perf(hba, on); ufs_mtk_setup_ref_clk(hba, on); phy_power_off(host->mphy); } @@ -695,6 +738,46 @@ static u32 ufs_mtk_get_ufs_hci_version(struct ufs_hba *hba) return hba->ufs_version; } +/** + * ufs_mtk_init_clocks - Init mtk driver private clocks + * + * @hba: per adapter instance + */ +static void ufs_mtk_init_clocks(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct list_head *head = &hba->clk_list_head; + struct ufs_mtk_clk *mclk = &host->mclk; + struct ufs_clk_info *clki, *clki_tmp; + + /* + * Find private clocks and store them in struct ufs_mtk_clk. + * Remove "ufs_sel_min_src" and "ufs_sel_min_src" from list to avoid + * being switched on/off in clock gating. + */ + list_for_each_entry_safe(clki, clki_tmp, head, list) { + if (!strcmp(clki->name, "ufs_sel")) { + host->mclk.ufs_sel_clki = clki; + } else if (!strcmp(clki->name, "ufs_sel_max_src")) { + host->mclk.ufs_sel_max_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } else if (!strcmp(clki->name, "ufs_sel_min_src")) { + host->mclk.ufs_sel_min_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } + } + + if (!mclk->ufs_sel_clki || !mclk->ufs_sel_max_clki || + !mclk->ufs_sel_min_clki) { + hba->caps &= ~UFSHCD_CAP_CLK_SCALING; + dev_info(hba->dev, + "%s: Clk-scaling not ready. Feature disabled.", + __func__); + } +} + #define MAX_VCC_NAME 30 static int ufs_mtk_vreg_fix_vcc(struct ufs_hba *hba) { @@ -815,12 +898,18 @@ static int ufs_mtk_init(struct ufs_hba *hba) /* Enable WriteBooster */ hba->caps |= UFSHCD_CAP_WB_EN; + + /* Enable clk scaling*/ + hba->caps |= UFSHCD_CAP_CLK_SCALING; + hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL; hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80); if (host->caps & UFS_MTK_CAP_DISABLE_AH8) hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; + ufs_mtk_init_clocks(hba); + /* * ufshcd_vops_init() is invoked after * ufshcd_setup_clock(true) in ufshcd_hba_init() thus @@ -833,6 +922,10 @@ static int ufs_mtk_init(struct ufs_hba *hba) host->ip_ver = ufshcd_readl(hba, REG_UFS_MTK_IP_VER); + /* Initialize pm-qos request */ + cpu_latency_qos_add_request(&host->pm_qos_req, PM_QOS_DEFAULT_VALUE); + host->pm_qos_init = true; + goto out; out_variant_clear: @@ -1247,13 +1340,16 @@ fail: static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba) { - ufshcd_dump_regs(hba, REG_UFS_REFCLK_CTRL, 0x4, "Ref-Clk Ctrl "); + /* Dump ufshci register 0x140 ~ 0x14C */ + ufshcd_dump_regs(hba, REG_UFS_XOUFS_CTRL, 0x10, + "XOUFS Ctrl (0x140): "); ufshcd_dump_regs(hba, REG_UFS_EXTREG, 0x4, "Ext Reg "); + /* Dump ufshci register 0x2200 ~ 0x22AC */ ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL, REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4, - "MPHY Ctrl "); + "MPHY Ctrl (0x2200): "); /* Direct debugging information to REG_MTK_PROBE */ ufs_mtk_dbg_sel(hba); @@ -1310,8 +1406,101 @@ static void ufs_mtk_event_notify(struct ufs_hba *hba, enum ufs_event_type evt, void *data) { unsigned int val = *(u32 *)data; + unsigned long reg; + u8 bit; trace_ufs_mtk_event(evt, val); + + /* Print details of UIC Errors */ + if (evt <= UFS_EVT_DME_ERR) { + dev_info(hba->dev, + "Host UIC Error Code (%s): %08x\n", + ufs_uic_err_str[evt], val); + reg = val; + } + + if (evt == UFS_EVT_PA_ERR) { + for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_pa_err_str)) + dev_info(hba->dev, "%s\n", ufs_uic_pa_err_str[bit]); + } + + if (evt == UFS_EVT_DL_ERR) { + for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_dl_err_str)) + dev_info(hba->dev, "%s\n", ufs_uic_dl_err_str[bit]); + } +} + +static void ufs_mtk_config_scaling_param(struct ufs_hba *hba, + struct devfreq_dev_profile *profile, + struct devfreq_simple_ondemand_data *data) +{ + /* Customize min gear in clk scaling */ + hba->clk_scaling.min_gear = UFS_HS_G4; + + hba->vps->devfreq_profile.polling_ms = 200; + hba->vps->ondemand_data.upthreshold = 50; + hba->vps->ondemand_data.downdifferential = 20; +} + +/** + * ufs_mtk_clk_scale - Internal clk scaling operation + * + * MTK platform supports clk scaling by switching parent of ufs_sel(mux). + * The ufs_sel downstream to ufs_ck which feeds directly to UFS hardware. + * Max and min clocks rate of ufs_sel defined in dts should match rate of + * "ufs_sel_max_src" and "ufs_sel_min_src" respectively. + * This prevent changing rate of pll clock that is shared between modules. + * + * @hba: per adapter instance + * @scale_up: True for scaling up and false for scaling down + */ +static void ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_clk *mclk = &host->mclk; + struct ufs_clk_info *clki = mclk->ufs_sel_clki; + int ret = 0; + + ret = clk_prepare_enable(clki->clk); + if (ret) { + dev_info(hba->dev, + "clk_prepare_enable() fail, ret: %d\n", ret); + return; + } + + if (scale_up) { + ret = clk_set_parent(clki->clk, mclk->ufs_sel_max_clki->clk); + clki->curr_freq = clki->max_freq; + } else { + ret = clk_set_parent(clki->clk, mclk->ufs_sel_min_clki->clk); + clki->curr_freq = clki->min_freq; + } + + if (ret) { + dev_info(hba->dev, + "Failed to set ufs_sel_clki, ret: %d\n", ret); + } + + clk_disable_unprepare(clki->clk); + + trace_ufs_mtk_clk_scale(clki->name, scale_up, clk_get_rate(clki->clk)); +} + +static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up, + enum ufs_notify_change_status status) +{ + if (!ufshcd_is_clkscaling_supported(hba)) + return 0; + + if (status == PRE_CHANGE) { + /* Switch parent before clk_set_rate() */ + ufs_mtk_clk_scale(hba, scale_up); + } else { + /* Request interrupt latency QoS accordingly */ + ufs_mtk_scale_perf(hba, scale_up); + } + + return 0; } /* @@ -1335,6 +1524,8 @@ static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = { .dbg_register_dump = ufs_mtk_dbg_register_dump, .device_reset = ufs_mtk_device_reset, .event_notify = ufs_mtk_event_notify, + .config_scaling_param = ufs_mtk_config_scaling_param, + .clk_scale_notify = ufs_mtk_clk_scale_notify, }; /** diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h index aa26d415527b05e0a1ef894b9e3dd8b69b98eab5..2fc6d7b87694ecc06b45728629ba3a4db4b50fad 100644 --- a/drivers/ufs/host/ufs-mediatek.h +++ b/drivers/ufs/host/ufs-mediatek.h @@ -124,6 +124,12 @@ struct ufs_mtk_crypt_cfg { int vcore_volt; }; +struct ufs_mtk_clk { + struct ufs_clk_info *ufs_sel_clki; /* Mux */ + struct ufs_clk_info *ufs_sel_max_clki; /* Max src */ + struct ufs_clk_info *ufs_sel_min_clki; /* Min src */ +}; + struct ufs_mtk_hw_ver { u8 step; u8 minor; @@ -139,6 +145,7 @@ struct ufs_mtk_host { struct reset_control *crypto_reset; struct ufs_hba *hba; struct ufs_mtk_crypt_cfg *crypt; + struct ufs_mtk_clk mclk; struct ufs_mtk_hw_ver hw_ver; enum ufs_mtk_host_caps caps; bool mphy_powered_on; diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c index 473fad83701ea8e3153726d4816f1cb6b9406c75..8ad1415e10b637ac927b4aca3a5ed7c4de801ab4 100644 --- a/drivers/ufs/host/ufs-qcom.c +++ b/drivers/ufs/host/ufs-qcom.c @@ -846,7 +846,7 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba) struct ufs_qcom_host *host = ufshcd_get_variant(hba); hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; - hba->caps |= UFSHCD_CAP_CLK_SCALING; + hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING; hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND; hba->caps |= UFSHCD_CAP_WB_EN; hba->caps |= UFSHCD_CAP_CRYPTO; diff --git a/drivers/uio/uio_dfl.c b/drivers/uio/uio_dfl.c index 8f39cc8bb034f1f2b931766f52e22717a918d6f6..69e93f3e7faf4bf1e1cdbd587305b198cd68be6c 100644 --- a/drivers/uio/uio_dfl.c +++ b/drivers/uio/uio_dfl.c @@ -46,10 +46,12 @@ static int uio_dfl_probe(struct dfl_device *ddev) #define FME_FEATURE_ID_ETH_GROUP 0x10 #define FME_FEATURE_ID_HSSI_SUBSYS 0x15 +#define PORT_FEATURE_ID_IOPLL_USRCLK 0x14 static const struct dfl_device_id uio_dfl_ids[] = { { FME_ID, FME_FEATURE_ID_ETH_GROUP }, { FME_ID, FME_FEATURE_ID_HSSI_SUBSYS }, + { PORT_ID, PORT_FEATURE_ID_IOPLL_USRCLK }, { } }; MODULE_DEVICE_TABLE(dfl, uio_dfl_ids); diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c index 362217189ef315654cc6b8f245954c4d42740302..1cdb8758ae01b1c83d47023e67569bdc0096f50f 100644 --- a/drivers/usb/atm/usbatm.c +++ b/drivers/usb/atm/usbatm.c @@ -1026,7 +1026,7 @@ int usbatm_usb_probe(struct usb_interface *intf, const struct usb_device_id *id, /* public fields */ instance->driver = driver; - strlcpy(instance->driver_name, driver->driver_name, + strscpy(instance->driver_name, driver->driver_name, sizeof(instance->driver_name)); instance->usb_dev = usb_dev; diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c index d21b69997e750aad5f47697f50391076172dd73d..5adcb349718c33128b0970b6448080dc458f8b5a 100644 --- a/drivers/usb/cdns3/cdns3-gadget.c +++ b/drivers/usb/cdns3/cdns3-gadget.c @@ -1530,7 +1530,8 @@ static void cdns3_transfer_completed(struct cdns3_device *priv_dev, TRB_LEN(le32_to_cpu(trb->length)); if (priv_req->num_of_trb > 1 && - le32_to_cpu(trb->control) & TRB_SMM) + le32_to_cpu(trb->control) & TRB_SMM && + le32_to_cpu(trb->control) & TRB_CHAIN) transfer_end = true; cdns3_ep_inc_deq(priv_ep); @@ -1690,6 +1691,7 @@ static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep) ep_cfg &= ~EP_CFG_ENABLE; writel(ep_cfg, &priv_dev->regs->ep_cfg); priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; + priv_ep->flags |= EP_UPDATE_EP_TRBADDR; } cdns3_transfer_completed(priv_dev, priv_ep); } else if (!(priv_ep->flags & EP_STALLED) && diff --git a/drivers/usb/cdns3/cdns3-plat.c b/drivers/usb/cdns3/cdns3-plat.c index dc068e940ed59450131cbb790786b7bfda0ef1b1..2bc5d094548b6b5edb4093dc633910f78a65fa08 100644 --- a/drivers/usb/cdns3/cdns3-plat.c +++ b/drivers/usb/cdns3/cdns3-plat.c @@ -110,8 +110,6 @@ static int cdns3_plat_probe(struct platform_device *pdev) cdns->wakeup_irq = platform_get_irq_byname_optional(pdev, "wakeup"); if (cdns->wakeup_irq == -EPROBE_DEFER) return cdns->wakeup_irq; - else if (cdns->wakeup_irq == 0) - return -EINVAL; if (cdns->wakeup_irq < 0) { dev_dbg(dev, "couldn't get wakeup irq\n"); diff --git a/drivers/usb/chipidea/Kconfig b/drivers/usb/chipidea/Kconfig index 661818e8fed6b8daef8d9a90e1f2320e8c7b832f..c815824a0b2d9c0d1a07a29853fd6d748ad072dd 100644 --- a/drivers/usb/chipidea/Kconfig +++ b/drivers/usb/chipidea/Kconfig @@ -34,26 +34,26 @@ config USB_CHIPIDEA_HOST ChipIdea driver. config USB_CHIPIDEA_PCI - tristate "Enable PCI glue driver" if EMBEDDED + tristate "Enable PCI glue driver" if EXPERT depends on USB_PCI depends on NOP_USB_XCEIV default USB_CHIPIDEA config USB_CHIPIDEA_MSM - tristate "Enable MSM hsusb glue driver" if EMBEDDED + tristate "Enable MSM hsusb glue driver" if EXPERT default USB_CHIPIDEA config USB_CHIPIDEA_IMX - tristate "Enable i.MX USB glue driver" if EMBEDDED + tristate "Enable i.MX USB glue driver" if EXPERT depends on OF default USB_CHIPIDEA config USB_CHIPIDEA_GENERIC - tristate "Enable generic USB2 glue driver" if EMBEDDED + tristate "Enable generic USB2 glue driver" if EXPERT default USB_CHIPIDEA config USB_CHIPIDEA_TEGRA - tristate "Enable Tegra USB glue driver" if EMBEDDED + tristate "Enable Tegra USB glue driver" if EXPERT depends on OF default USB_CHIPIDEA diff --git a/drivers/usb/chipidea/ci_hdrc_usb2.c b/drivers/usb/chipidea/ci_hdrc_usb2.c index 89e1d82d739b1618429b02e5967890f4ee672306..dc86b12060b56a35692ca8c082d5dc220ac03c0b 100644 --- a/drivers/usb/chipidea/ci_hdrc_usb2.c +++ b/drivers/usb/chipidea/ci_hdrc_usb2.c @@ -30,6 +30,7 @@ static const struct ci_hdrc_platform_data ci_default_pdata = { static const struct ci_hdrc_platform_data ci_zynq_pdata = { .capoffset = DEF_CAPOFFSET, + .flags = CI_HDRC_PHY_VBUS_CONTROL, }; static const struct ci_hdrc_platform_data ci_zevio_pdata = { diff --git a/drivers/usb/chipidea/host.c b/drivers/usb/chipidea/host.c index bdc3885c0d493c14e8abb6fe0c284cd5938eb137..bc3634a54c6b7f5e5e3aa95bac1e7af0e41e642e 100644 --- a/drivers/usb/chipidea/host.c +++ b/drivers/usb/chipidea/host.c @@ -63,6 +63,13 @@ static int ehci_ci_portpower(struct usb_hcd *hcd, int portnum, bool enable) priv->enabled = enable; } + if (ci->platdata->flags & CI_HDRC_PHY_VBUS_CONTROL) { + if (enable) + usb_phy_vbus_on(ci->usb_phy); + else + usb_phy_vbus_off(ci->usb_phy); + } + if (enable && (ci->platdata->phy_mode == USBPHY_INTERFACE_MODE_HSIC)) { /* * Marvell 28nm HSIC PHY requires forcing the port to HS mode. diff --git a/drivers/usb/chipidea/otg_fsm.c b/drivers/usb/chipidea/otg_fsm.c index 61b157b9c662689667b42eeb7fa55eca9e74d35f..ada78daba6df9cd89a616aa30f72cfee2f3d58c1 100644 --- a/drivers/usb/chipidea/otg_fsm.c +++ b/drivers/usb/chipidea/otg_fsm.c @@ -471,6 +471,10 @@ static void ci_otg_drv_vbus(struct otg_fsm *fsm, int on) return; } } + + if (ci->platdata->flags & CI_HDRC_PHY_VBUS_CONTROL) + usb_phy_vbus_on(ci->usb_phy); + /* Disable data pulse irq */ hw_write_otgsc(ci, OTGSC_DPIE, 0); @@ -480,6 +484,9 @@ static void ci_otg_drv_vbus(struct otg_fsm *fsm, int on) if (ci->platdata->reg_vbus) regulator_disable(ci->platdata->reg_vbus); + if (ci->platdata->flags & CI_HDRC_PHY_VBUS_CONTROL) + usb_phy_vbus_off(ci->usb_phy); + fsm->a_bus_drop = 1; fsm->a_bus_req = 0; } diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 483bcb1213f73b4838c4e7436d456f54998da022..36bf051b345b8c87d886265422af1317e86cc3af 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -51,7 +51,7 @@ static DEFINE_IDR(acm_minors); static DEFINE_MUTEX(acm_minors_lock); static void acm_tty_set_termios(struct tty_struct *tty, - struct ktermios *termios_old); + const struct ktermios *termios_old); /* * acm_minors accessors @@ -1049,7 +1049,7 @@ static int acm_tty_ioctl(struct tty_struct *tty, } static void acm_tty_set_termios(struct tty_struct *tty, - struct ktermios *termios_old) + const struct ktermios *termios_old) { struct acm *acm = tty->driver_data; struct ktermios *termios = &tty->termios; @@ -1810,6 +1810,9 @@ static const struct usb_device_id acm_ids[] = { { USB_DEVICE(0x09d8, 0x0320), /* Elatec GmbH TWN3 */ .driver_info = NO_UNION_NORMAL, /* has misplaced union descriptor */ }, + { USB_DEVICE(0x0c26, 0x0020), /* Icom ICF3400 Serie */ + .driver_info = NO_UNION_NORMAL, /* reports zero length descriptor */ + }, { USB_DEVICE(0x0ca6, 0xa050), /* Castles VEGA3000 */ .driver_info = NO_UNION_NORMAL, /* reports zero length descriptor */ }, diff --git a/drivers/usb/class/cdc-wdm.c b/drivers/usb/class/cdc-wdm.c index eebe782380fb9b55f1efb199615ea07524ca43f2..1f0951be15ab77b7097c321b71d940a739d108cf 100644 --- a/drivers/usb/class/cdc-wdm.c +++ b/drivers/usb/class/cdc-wdm.c @@ -958,7 +958,7 @@ static void wdm_wwan_rx(struct wdm_device *desc, int length) if (!skb) return; - memcpy(skb_put(skb, length), desc->inbuf, length); + skb_put_data(skb, desc->inbuf, length); wwan_port_rx(port, skb); /* inbuf has been copied, it is safe to check for outstanding data */ diff --git a/drivers/usb/common/debug.c b/drivers/usb/common/debug.c index 075f6b1b2a1a123f10e4ed74c65970d3e4be8af0..f204cec8d380aa63a60c7be0903ab5565612b811 100644 --- a/drivers/usb/common/debug.c +++ b/drivers/usb/common/debug.c @@ -208,30 +208,28 @@ static void usb_decode_set_isoch_delay(__u8 wValue, char *str, size_t size) snprintf(str, size, "Set Isochronous Delay(Delay = %d ns)", wValue); } -/** - * usb_decode_ctrl - Returns human readable representation of control request. - * @str: buffer to return a human-readable representation of control request. - * This buffer should have about 200 bytes. - * @size: size of str buffer. - * @bRequestType: matches the USB bmRequestType field - * @bRequest: matches the USB bRequest field - * @wValue: matches the USB wValue field (CPU byte order) - * @wIndex: matches the USB wIndex field (CPU byte order) - * @wLength: matches the USB wLength field (CPU byte order) - * - * Function returns decoded, formatted and human-readable description of - * control request packet. - * - * The usage scenario for this is for tracepoints, so function as a return - * use the same value as in parameters. This approach allows to use this - * function in TP_printk - * - * Important: wValue, wIndex, wLength parameters before invoking this function - * should be processed by le16_to_cpu macro. - */ -const char *usb_decode_ctrl(char *str, size_t size, __u8 bRequestType, - __u8 bRequest, __u16 wValue, __u16 wIndex, - __u16 wLength) +static void usb_decode_ctrl_generic(char *str, size_t size, __u8 bRequestType, + __u8 bRequest, __u16 wValue, __u16 wIndex, + __u16 wLength) +{ + u8 recip = bRequestType & USB_RECIP_MASK; + u8 type = bRequestType & USB_TYPE_MASK; + + snprintf(str, size, + "Type=%s Recipient=%s Dir=%s bRequest=%u wValue=%u wIndex=%u wLength=%u", + (type == USB_TYPE_STANDARD) ? "Standard" : + (type == USB_TYPE_VENDOR) ? "Vendor" : + (type == USB_TYPE_CLASS) ? "Class" : "Unknown", + (recip == USB_RECIP_DEVICE) ? "Device" : + (recip == USB_RECIP_INTERFACE) ? "Interface" : + (recip == USB_RECIP_ENDPOINT) ? "Endpoint" : "Unknown", + (bRequestType & USB_DIR_IN) ? "IN" : "OUT", + bRequest, wValue, wIndex, wLength); +} + +static void usb_decode_ctrl_standard(char *str, size_t size, __u8 bRequestType, + __u8 bRequest, __u16 wValue, __u16 wIndex, + __u16 wLength) { switch (bRequest) { case USB_REQ_GET_STATUS: @@ -272,14 +270,48 @@ const char *usb_decode_ctrl(char *str, size_t size, __u8 bRequestType, usb_decode_set_isoch_delay(wValue, str, size); break; default: - snprintf(str, size, "%02x %02x %02x %02x %02x %02x %02x %02x", - bRequestType, bRequest, - (u8)(cpu_to_le16(wValue) & 0xff), - (u8)(cpu_to_le16(wValue) >> 8), - (u8)(cpu_to_le16(wIndex) & 0xff), - (u8)(cpu_to_le16(wIndex) >> 8), - (u8)(cpu_to_le16(wLength) & 0xff), - (u8)(cpu_to_le16(wLength) >> 8)); + usb_decode_ctrl_generic(str, size, bRequestType, bRequest, + wValue, wIndex, wLength); + break; + } +} + +/** + * usb_decode_ctrl - Returns human readable representation of control request. + * @str: buffer to return a human-readable representation of control request. + * This buffer should have about 200 bytes. + * @size: size of str buffer. + * @bRequestType: matches the USB bmRequestType field + * @bRequest: matches the USB bRequest field + * @wValue: matches the USB wValue field (CPU byte order) + * @wIndex: matches the USB wIndex field (CPU byte order) + * @wLength: matches the USB wLength field (CPU byte order) + * + * Function returns decoded, formatted and human-readable description of + * control request packet. + * + * The usage scenario for this is for tracepoints, so function as a return + * use the same value as in parameters. This approach allows to use this + * function in TP_printk + * + * Important: wValue, wIndex, wLength parameters before invoking this function + * should be processed by le16_to_cpu macro. + */ +const char *usb_decode_ctrl(char *str, size_t size, __u8 bRequestType, + __u8 bRequest, __u16 wValue, __u16 wIndex, + __u16 wLength) +{ + switch (bRequestType & USB_TYPE_MASK) { + case USB_TYPE_STANDARD: + usb_decode_ctrl_standard(str, size, bRequestType, bRequest, + wValue, wIndex, wLength); + break; + case USB_TYPE_VENDOR: + case USB_TYPE_CLASS: + default: + usb_decode_ctrl_generic(str, size, bRequestType, bRequest, + wValue, wIndex, wLength); + break; } return str; diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index 0a4f441aff8f01d74c3b6de83d383912d5c68df2..d7c8461976ce09133aeeaed474b826ce149738c2 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -233,7 +233,7 @@ err: return 0; } -static int ulpi_regs_read(struct seq_file *seq, void *data) +static int ulpi_regs_show(struct seq_file *seq, void *data) { struct ulpi *ulpi = seq->private; @@ -269,21 +269,7 @@ static int ulpi_regs_read(struct seq_file *seq, void *data) return 0; } - -static int ulpi_regs_open(struct inode *inode, struct file *f) -{ - struct ulpi *ulpi = inode->i_private; - - return single_open(f, ulpi_regs_read, ulpi); -} - -static const struct file_operations ulpi_regs_ops = { - .owner = THIS_MODULE, - .open = ulpi_regs_open, - .release = single_release, - .read = seq_read, - .llseek = seq_lseek -}; +DEFINE_SHOW_ATTRIBUTE(ulpi_regs); #define ULPI_ROOT debugfs_lookup(KBUILD_MODNAME, NULL) @@ -316,7 +302,7 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi) } root = debugfs_create_dir(dev_name(dev), ULPI_ROOT); - debugfs_create_file("regs", 0444, root, ulpi, &ulpi_regs_ops); + debugfs_create_file("regs", 0444, root, ulpi, &ulpi_regs_fops); dev_dbg(&ulpi->dev, "registered ULPI PHY: vendor %04x, product %04x\n", ulpi->id.vendor, ulpi->id.product); diff --git a/drivers/usb/common/usb-conn-gpio.c b/drivers/usb/common/usb-conn-gpio.c index b39c9f1c375d6e358ae7d62c81aa280f9a007be6..e20874caba363c7178fbe247186f1e34d9bf88a5 100644 --- a/drivers/usb/common/usb-conn-gpio.c +++ b/drivers/usb/common/usb-conn-gpio.c @@ -208,10 +208,8 @@ static int usb_conn_probe(struct platform_device *pdev) if (PTR_ERR(info->vbus) == -ENODEV) info->vbus = NULL; - if (IS_ERR(info->vbus)) { - ret = PTR_ERR(info->vbus); - return dev_err_probe(dev, ret, "failed to get vbus :%d\n", ret); - } + if (IS_ERR(info->vbus)) + return dev_err_probe(dev, PTR_ERR(info->vbus), "failed to get vbus\n"); info->role_sw = usb_role_switch_get(dev); if (IS_ERR(info->role_sw)) diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index b5b85bf80329888b6b3fc60d200d5bddf1bb1af3..837f3e57f5809bf574d6443f418871bc2cc8fd39 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1434,7 +1434,7 @@ static int proc_getdriver(struct usb_dev_state *ps, void __user *arg) if (!intf || !intf->dev.driver) ret = -ENODATA; else { - strlcpy(gd.driver, intf->dev.driver->name, + strscpy(gd.driver, intf->dev.driver->name, sizeof(gd.driver)); ret = (copy_to_user(arg, &gd, sizeof(gd)) ? -EFAULT : 0); } diff --git a/drivers/usb/core/hcd-pci.c b/drivers/usb/core/hcd-pci.c index 482dae72ef1cbb7b14b083daca3a068967a5c648..9b77f49b3560c1f61927f9b53134c99649949d1a 100644 --- a/drivers/usb/core/hcd-pci.c +++ b/drivers/usb/core/hcd-pci.c @@ -157,7 +157,6 @@ static void ehci_wait_for_companions(struct pci_dev *pdev, struct usb_hcd *hcd, /** * usb_hcd_pci_probe - initialize PCI-based HCDs * @dev: USB Host Controller being probed - * @id: pci hotplug id connecting controller to HCD framework * @driver: USB HC driver handle * * Context: task context, might sleep @@ -170,8 +169,7 @@ static void ehci_wait_for_companions(struct pci_dev *pdev, struct usb_hcd *hcd, * * Return: 0 if successful. */ -int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id, - const struct hc_driver *driver) +int usb_hcd_pci_probe(struct pci_dev *dev, const struct hc_driver *driver) { struct usb_hcd *hcd; int retval; @@ -180,9 +178,6 @@ int usb_hcd_pci_probe(struct pci_dev *dev, const struct pci_device_id *id, if (usb_disabled()) return -ENODEV; - if (!id) - return -EINVAL; - if (!driver) return -EINVAL; diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index 94b305bbd621b4b41d90b778c7ebded548ff62df..faeaace0d197dbb88739cebf8bd06617a5712171 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -1474,7 +1474,7 @@ int usb_hcd_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb, urb->sg, urb->num_sgs, dir); - if (n <= 0) + if (!n) ret = -EAGAIN; else urb->transfer_flags |= URB_DMA_MAP_SG; @@ -2158,21 +2158,14 @@ static struct urb *request_single_step_set_feature_urb( { struct urb *urb; struct usb_hcd *hcd = bus_to_hcd(udev->bus); - struct usb_host_endpoint *ep; urb = usb_alloc_urb(0, GFP_KERNEL); if (!urb) return NULL; urb->pipe = usb_rcvctrlpipe(udev, 0); - ep = (usb_pipein(urb->pipe) ? udev->ep_in : udev->ep_out) - [usb_pipeendpoint(urb->pipe)]; - if (!ep) { - usb_free_urb(urb); - return NULL; - } - urb->ep = ep; + urb->ep = &udev->ep0; urb->dev = udev; urb->setup_packet = (void *)dr; urb->transfer_buffer = buf; diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 2633acde7ac1de9e0943f4fdf45b21d2a994e254..bbab424b0d5594a84144cfa4352307983fa7b735 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -6038,6 +6038,11 @@ re_enumerate: * the reset is over (using their post_reset method). * * Return: The same as for usb_reset_and_verify_device(). + * However, if a reset is already in progress (for instance, if a + * driver doesn't have pre_reset() or post_reset() callbacks, and while + * being unbound or re-bound during the ongoing reset its disconnect() + * or probe() routine tries to perform a second, nested reset), the + * routine returns -EINPROGRESS. * * Note: * The caller must own the device lock. For example, it's safe to use @@ -6071,6 +6076,10 @@ int usb_reset_device(struct usb_device *udev) return -EISDIR; } + if (udev->reset_in_progress) + return -EINPROGRESS; + udev->reset_in_progress = 1; + port_dev = hub->ports[udev->portnum - 1]; /* @@ -6135,6 +6144,7 @@ int usb_reset_device(struct usb_device *udev) usb_autosuspend_device(udev); memalloc_noio_restore(noio_flag); + udev->reset_in_progress = 0; return ret; } EXPORT_SYMBOL_GPL(usb_reset_device); diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index f99a65a64588fe9b4deca18c593bfe0f690726f5..0722d213130550a9fec9e3e39362a63042021ca9 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -388,6 +388,15 @@ static const struct usb_device_id usb_quirk_list[] = { /* Kingston DataTraveler 3.0 */ { USB_DEVICE(0x0951, 0x1666), .driver_info = USB_QUIRK_NO_LPM }, + /* NVIDIA Jetson devices in Force Recovery mode */ + { USB_DEVICE(0x0955, 0x7018), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7019), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7418), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7721), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7c18), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7e19), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x0955, 0x7f21), .driver_info = USB_QUIRK_RESET_RESUME }, + /* X-Rite/Gretag-Macbeth Eye-One Pro display colorimeter */ { USB_DEVICE(0x0971, 0x2000), .driver_info = USB_QUIRK_NO_SET_INTF }, @@ -437,6 +446,10 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x1532, 0x0116), .driver_info = USB_QUIRK_LINEAR_UFRAME_INTR_BINTERVAL }, + /* Lenovo ThinkPad OneLink+ Dock twin hub controllers (VIA Labs VL812) */ + { USB_DEVICE(0x17ef, 0x1018), .driver_info = USB_QUIRK_RESET_RESUME }, + { USB_DEVICE(0x17ef, 0x1019), .driver_info = USB_QUIRK_RESET_RESUME }, + /* Lenovo USB-C to Ethernet Adapter RTL8153-04 */ { USB_DEVICE(0x17ef, 0x720c), .driver_info = USB_QUIRK_NO_LPM }, diff --git a/drivers/usb/core/urb.c b/drivers/usb/core/urb.c index 33d62d7e3929f1396ab8ca201d8214446fc5c144..9f3c54032556edadf5cda078a2c93c584cf37de9 100644 --- a/drivers/usb/core/urb.c +++ b/drivers/usb/core/urb.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -426,6 +427,7 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags) URB_SETUP_MAP_SINGLE | URB_SETUP_MAP_LOCAL | URB_DMA_SG_COMBINED); urb->transfer_flags |= (is_out ? URB_DIR_OUT : URB_DIR_IN); + kmsan_handle_urb(urb, is_out); if (xfertype != USB_ENDPOINT_XFER_CONTROL && dev->state < USB_STATE_CONFIGURED) diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index dc4fc72ab1b66be50f781aff194e169e0958393e..5635e4d7ec880e70b80ee938ed8e732b10ea168e 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -3,36 +3,6 @@ * core.c - DesignWare HS OTG Controller common routines * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h index 0683852e47e41801d2dcc3eef995bc90a0ebc49b..40cf2880d7e595354a25fafd056f6758f19d5ecc 100644 --- a/drivers/usb/dwc2/core.h +++ b/drivers/usb/dwc2/core.h @@ -3,36 +3,6 @@ * core.h - DesignWare HS OTG Controller common declarations * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __DWC2_CORE_H__ diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c index a5c52b237e723753e9574dfe7236d0a445cf06cb..158ede7538548e4d6daba4fc08d543f434b3ec6d 100644 --- a/drivers/usb/dwc2/core_intr.c +++ b/drivers/usb/dwc2/core_intr.c @@ -3,36 +3,6 @@ * core_intr.c - DesignWare HS OTG Controller common interrupt handling * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c index aaf7b9fc4d3478835660d79cf8d803ddf93a14db..657f1f659ffaf83d0d0d96953d4017ff55f1da3b 100644 --- a/drivers/usb/dwc2/hcd.c +++ b/drivers/usb/dwc2/hcd.c @@ -3,36 +3,6 @@ * hcd.c - DesignWare HS OTG Controller host-mode routines * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/hcd.h b/drivers/usb/dwc2/hcd.h index ea02ee63ac6d8a9db833ec46c2317eee87378e98..b7254d94fdc39c975469c4985f4c0806ce137b1f 100644 --- a/drivers/usb/dwc2/hcd.h +++ b/drivers/usb/dwc2/hcd.h @@ -3,37 +3,8 @@ * hcd.h - DesignWare HS OTG Controller host-mode declarations * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef __DWC2_HCD_H__ #define __DWC2_HCD_H__ diff --git a/drivers/usb/dwc2/hcd_ddma.c b/drivers/usb/dwc2/hcd_ddma.c index a858b5f9c1d60280db272ea38e868a6b9f1217eb..6b4d825e97a2d91bbafd04ada5fdce4a1ab52c0c 100644 --- a/drivers/usb/dwc2/hcd_ddma.c +++ b/drivers/usb/dwc2/hcd_ddma.c @@ -3,36 +3,6 @@ * hcd_ddma.c - DesignWare HS OTG Controller descriptor DMA routines * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/hcd_intr.c b/drivers/usb/dwc2/hcd_intr.c index d5f4ec1b73b1545f5b517793e3868a58141bdd29..c9740caa5974b9b92652c232ab3616c6dee09d17 100644 --- a/drivers/usb/dwc2/hcd_intr.c +++ b/drivers/usb/dwc2/hcd_intr.c @@ -3,36 +3,6 @@ * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/hcd_queue.c b/drivers/usb/dwc2/hcd_queue.c index 24beff610cf2c65b057f8f2b12f1a04bf80b0685..0a1145592fc766924dc0978d4caf6c66ca9e5a93 100644 --- a/drivers/usb/dwc2/hcd_queue.c +++ b/drivers/usb/dwc2/hcd_queue.c @@ -3,36 +3,6 @@ * hcd_queue.c - DesignWare HS OTG Controller host queuing routines * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/hw.h b/drivers/usb/dwc2/hw.h index 6b16fbf98bc6ec0183147c1325450c018776f634..13abdd5f6752999cc20603fa404b02c3d84bac20 100644 --- a/drivers/usb/dwc2/hw.h +++ b/drivers/usb/dwc2/hw.h @@ -3,36 +3,6 @@ * hw.h - DesignWare HS OTG Controller hardware definitions * * Copyright 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __DWC2_HW_H__ diff --git a/drivers/usb/dwc2/params.c b/drivers/usb/dwc2/params.c index fdb8a42fff86038cf8497e08dd117c7377910250..8eab5f38b1101debbc8a5872ffac4b257506767e 100644 --- a/drivers/usb/dwc2/params.c +++ b/drivers/usb/dwc2/params.c @@ -1,36 +1,6 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* * Copyright (C) 2004-2016 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include diff --git a/drivers/usb/dwc2/pci.c b/drivers/usb/dwc2/pci.c index a93559b4ecdbc33a4b9d107d07f40890cf358c66..b7306ed8be4c1b38eed5f7b74db1a7ee657bf908 100644 --- a/drivers/usb/dwc2/pci.c +++ b/drivers/usb/dwc2/pci.c @@ -3,36 +3,6 @@ * pci.c - DesignWare HS OTG Controller PCI driver * * Copyright (C) 2004-2013 Synopsys, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c index c8ba87df7abef7d4885a97e97e4a0ae9c287c865..ec4ace0107f5f5723fc46435c688b5fe0a3c7f31 100644 --- a/drivers/usb/dwc2/platform.c +++ b/drivers/usb/dwc2/platform.c @@ -3,36 +3,6 @@ * platform.c - DesignWare HS OTG Controller platform driver * * Copyright (C) Matthijs Kooijman - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions, and the following disclaimer, - * without modification. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The names of the above-listed copyright holders may not be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any - * later version. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS - * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, - * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include @@ -154,9 +124,9 @@ static int __dwc2_lowlevel_hw_enable(struct dwc2_hsotg *hsotg) } else if (hsotg->plat && hsotg->plat->phy_init) { ret = hsotg->plat->phy_init(pdev, hsotg->plat->phy_type); } else { - ret = phy_power_on(hsotg->phy); + ret = phy_init(hsotg->phy); if (ret == 0) - ret = phy_init(hsotg->phy); + ret = phy_power_on(hsotg->phy); } return ret; @@ -188,9 +158,9 @@ static int __dwc2_lowlevel_hw_disable(struct dwc2_hsotg *hsotg) } else if (hsotg->plat && hsotg->plat->phy_exit) { ret = hsotg->plat->phy_exit(pdev, hsotg->plat->phy_type); } else { - ret = phy_exit(hsotg->phy); + ret = phy_power_off(hsotg->phy); if (ret == 0) - ret = phy_power_off(hsotg->phy); + ret = phy_exit(hsotg->phy); } if (ret) return ret; diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index c5c238ab3083ece84a6678b34877ba5c11ec67c4..ea51624461b5b7c77adffc08d452804151af3c65 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -23,7 +23,6 @@ #include #include #include -#include #include #include #include @@ -86,7 +85,7 @@ static int dwc3_get_dr_mode(struct dwc3 *dwc) * mode. If the controller supports DRD but the dr_mode is not * specified or set to OTG, then set the mode to peripheral. */ - if (mode == USB_DR_MODE_OTG && !dwc->edev && + if (mode == USB_DR_MODE_OTG && (!IS_ENABLED(CONFIG_USB_ROLE_SWITCH) || !device_property_read_bool(dwc->dev, "usb-role-switch")) && !DWC3_VER_IS_PRIOR(DWC3, 330A)) @@ -408,6 +407,10 @@ static void dwc3_ref_clk_period(struct dwc3 *dwc) reg |= FIELD_PREP(DWC3_GFLADJ_REFCLK_FLADJ_MASK, fladj) | FIELD_PREP(DWC3_GFLADJ_240MHZDECR, decr >> 1) | FIELD_PREP(DWC3_GFLADJ_240MHZDECR_PLS1, decr & 1); + + if (dwc->gfladj_refclk_lpm_sel) + reg |= DWC3_GFLADJ_REFCLK_LPM_SEL; + dwc3_writel(dwc->regs, DWC3_GFLADJ, reg); } @@ -789,7 +792,7 @@ static int dwc3_phy_setup(struct dwc3 *dwc) else reg |= DWC3_GUSB2PHYCFG_ENBLSLPM; - if (dwc->dis_u2_freeclk_exists_quirk) + if (dwc->dis_u2_freeclk_exists_quirk || dwc->gfladj_refclk_lpm_sel) reg &= ~DWC3_GUSB2PHYCFG_U2_FREECLK_EXISTS; dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); @@ -833,15 +836,16 @@ static void dwc3_core_exit(struct dwc3 *dwc) { dwc3_event_buffers_cleanup(dwc); + usb_phy_set_suspend(dwc->usb2_phy, 1); + usb_phy_set_suspend(dwc->usb3_phy, 1); + phy_power_off(dwc->usb2_generic_phy); + phy_power_off(dwc->usb3_generic_phy); + usb_phy_shutdown(dwc->usb2_phy); usb_phy_shutdown(dwc->usb3_phy); phy_exit(dwc->usb2_generic_phy); phy_exit(dwc->usb3_generic_phy); - usb_phy_set_suspend(dwc->usb2_phy, 1); - usb_phy_set_suspend(dwc->usb3_phy, 1); - phy_power_off(dwc->usb2_generic_phy); - phy_power_off(dwc->usb3_generic_phy); dwc3_clk_disable(dwc); reset_control_assert(dwc->reset); } @@ -1179,6 +1183,21 @@ static int dwc3_core_init(struct dwc3 *dwc) dwc3_writel(dwc->regs, DWC3_GUCTL2, reg); } + /* + * When configured in HOST mode, after issuing U3/L2 exit controller + * fails to send proper CRC checksum in CRC5 feild. Because of this + * behaviour Transaction Error is generated, resulting in reset and + * re-enumeration of usb device attached. All the termsel, xcvrsel, + * opmode becomes 0 during end of resume. Enabling bit 10 of GUCTL1 + * will correct this problem. This option is to support certain + * legacy ULPI PHYs. + */ + if (dwc->resume_hs_terminations) { + reg = dwc3_readl(dwc->regs, DWC3_GUCTL1); + reg |= DWC3_GUCTL1_RESUME_OPMODE_HS_HOST; + dwc3_writel(dwc->regs, DWC3_GUCTL1, reg); + } + if (!DWC3_VER_IS_PRIOR(DWC3, 250A)) { reg = dwc3_readl(dwc->regs, DWC3_GUCTL1); @@ -1522,8 +1541,12 @@ static void dwc3_get_properties(struct dwc3 *dwc) "snps,dis-del-phy-power-chg-quirk"); dwc->dis_tx_ipgap_linecheck_quirk = device_property_read_bool(dev, "snps,dis-tx-ipgap-linecheck-quirk"); + dwc->resume_hs_terminations = device_property_read_bool(dev, + "snps,resume-hs-terminations"); dwc->parkmode_disable_ss_quirk = device_property_read_bool(dev, "snps,parkmode-disable-ss-quirk"); + dwc->gfladj_refclk_lpm_sel = device_property_read_bool(dev, + "snps,gfladj-refclk-lpm-sel-quirk"); dwc->tx_de_emphasis_quirk = device_property_read_bool(dev, "snps,tx_de_emphasis_quirk"); @@ -1667,46 +1690,6 @@ static void dwc3_check_params(struct dwc3 *dwc) } } -static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc) -{ - struct device *dev = dwc->dev; - struct device_node *np_phy; - struct extcon_dev *edev = NULL; - const char *name; - - if (device_property_read_bool(dev, "extcon")) - return extcon_get_edev_by_phandle(dev, 0); - - /* - * Device tree platforms should get extcon via phandle. - * On ACPI platforms, we get the name from a device property. - * This device property is for kernel internal use only and - * is expected to be set by the glue code. - */ - if (device_property_read_string(dev, "linux,extcon-name", &name) == 0) - return extcon_get_extcon_dev(name); - - /* - * Try to get an extcon device from the USB PHY controller's "port" - * node. Check if it has the "port" node first, to avoid printing the - * error message from underlying code, as it's a valid case: extcon - * device (and "port" node) may be missing in case of "usb-role-switch" - * or OTG mode. - */ - np_phy = of_parse_phandle(dev->of_node, "phys", 0); - if (of_graph_is_present(np_phy)) { - struct device_node *np_conn; - - np_conn = of_graph_get_remote_node(np_phy, -1, -1); - if (np_conn) - edev = extcon_find_edev_by_node(np_conn); - of_node_put(np_conn); - } - of_node_put(np_phy); - - return edev; -} - static int dwc3_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1751,15 +1734,11 @@ static int dwc3_probe(struct platform_device *pdev) dwc3_get_properties(dwc); - if (!dwc->sysdev_is_parent) { - ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64)); - if (ret) - return ret; - } - dwc->reset = devm_reset_control_array_get_optional_shared(dev); - if (IS_ERR(dwc->reset)) - return PTR_ERR(dwc->reset); + if (IS_ERR(dwc->reset)) { + ret = PTR_ERR(dwc->reset); + goto put_usb_psy; + } if (dev->of_node) { /* @@ -1769,45 +1748,57 @@ static int dwc3_probe(struct platform_device *pdev) * check for them to retain backwards compatibility. */ dwc->bus_clk = devm_clk_get_optional(dev, "bus_early"); - if (IS_ERR(dwc->bus_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->bus_clk), - "could not get bus clock\n"); + if (IS_ERR(dwc->bus_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->bus_clk), + "could not get bus clock\n"); + goto put_usb_psy; + } if (dwc->bus_clk == NULL) { dwc->bus_clk = devm_clk_get_optional(dev, "bus_clk"); - if (IS_ERR(dwc->bus_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->bus_clk), - "could not get bus clock\n"); + if (IS_ERR(dwc->bus_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->bus_clk), + "could not get bus clock\n"); + goto put_usb_psy; + } } dwc->ref_clk = devm_clk_get_optional(dev, "ref"); - if (IS_ERR(dwc->ref_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->ref_clk), - "could not get ref clock\n"); + if (IS_ERR(dwc->ref_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->ref_clk), + "could not get ref clock\n"); + goto put_usb_psy; + } if (dwc->ref_clk == NULL) { dwc->ref_clk = devm_clk_get_optional(dev, "ref_clk"); - if (IS_ERR(dwc->ref_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->ref_clk), - "could not get ref clock\n"); + if (IS_ERR(dwc->ref_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->ref_clk), + "could not get ref clock\n"); + goto put_usb_psy; + } } dwc->susp_clk = devm_clk_get_optional(dev, "suspend"); - if (IS_ERR(dwc->susp_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->susp_clk), - "could not get suspend clock\n"); + if (IS_ERR(dwc->susp_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->susp_clk), + "could not get suspend clock\n"); + goto put_usb_psy; + } if (dwc->susp_clk == NULL) { dwc->susp_clk = devm_clk_get_optional(dev, "suspend_clk"); - if (IS_ERR(dwc->susp_clk)) - return dev_err_probe(dev, PTR_ERR(dwc->susp_clk), - "could not get suspend clock\n"); + if (IS_ERR(dwc->susp_clk)) { + ret = dev_err_probe(dev, PTR_ERR(dwc->susp_clk), + "could not get suspend clock\n"); + goto put_usb_psy; + } } } ret = reset_control_deassert(dwc->reset); if (ret) - return ret; + goto put_usb_psy; ret = dwc3_clk_enable(dwc); if (ret) @@ -1821,7 +1812,13 @@ static int dwc3_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dwc); dwc3_cache_hwparams(dwc); - device_init_wakeup(&pdev->dev, of_property_read_bool(dev->of_node, "wakeup-source")); + + if (!dwc->sysdev_is_parent && + DWC3_GHWPARAMS0_AWIDTH(dwc->hwparams.hwparams0) == 64) { + ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64)); + if (ret) + goto disable_clks; + } spin_lock_init(&dwc->lock); mutex_init(&dwc->mutex); @@ -1843,13 +1840,6 @@ static int dwc3_probe(struct platform_device *pdev) goto err2; } - dwc->edev = dwc3_get_extcon(dwc); - if (IS_ERR(dwc->edev)) { - ret = PTR_ERR(dwc->edev); - dev_err_probe(dwc->dev, ret, "failed to get extcon\n"); - goto err3; - } - ret = dwc3_get_dr_mode(dwc); if (ret) goto err3; @@ -1879,16 +1869,16 @@ err5: dwc3_debugfs_exit(dwc); dwc3_event_buffers_cleanup(dwc); - usb_phy_shutdown(dwc->usb2_phy); - usb_phy_shutdown(dwc->usb3_phy); - phy_exit(dwc->usb2_generic_phy); - phy_exit(dwc->usb3_generic_phy); - usb_phy_set_suspend(dwc->usb2_phy, 1); usb_phy_set_suspend(dwc->usb3_phy, 1); phy_power_off(dwc->usb2_generic_phy); phy_power_off(dwc->usb3_generic_phy); + usb_phy_shutdown(dwc->usb2_phy); + usb_phy_shutdown(dwc->usb3_phy); + phy_exit(dwc->usb2_generic_phy); + phy_exit(dwc->usb3_generic_phy); + dwc3_ulpi_exit(dwc); err4: @@ -1908,7 +1898,7 @@ disable_clks: dwc3_clk_disable(dwc); assert_reset: reset_control_assert(dwc->reset); - +put_usb_psy: if (dwc->usb_psy) power_supply_put(dwc->usb_psy); @@ -1976,14 +1966,12 @@ static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg) case DWC3_GCTL_PRTCAP_DEVICE: if (pm_runtime_suspended(dwc->dev)) break; - spin_lock_irqsave(&dwc->lock, flags); dwc3_gadget_suspend(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); synchronize_irq(dwc->irq_gadget); dwc3_core_exit(dwc); break; case DWC3_GCTL_PRTCAP_HOST: - if (!PMSG_IS_AUTO(msg) && !device_can_wakeup(dwc->dev)) { + if (!PMSG_IS_AUTO(msg) && !device_may_wakeup(dwc->dev)) { dwc3_core_exit(dwc); break; } @@ -2039,12 +2027,10 @@ static int dwc3_resume_common(struct dwc3 *dwc, pm_message_t msg) return ret; dwc3_set_prtcap(dwc, DWC3_GCTL_PRTCAP_DEVICE); - spin_lock_irqsave(&dwc->lock, flags); dwc3_gadget_resume(dwc); - spin_unlock_irqrestore(&dwc->lock, flags); break; case DWC3_GCTL_PRTCAP_HOST: - if (!PMSG_IS_AUTO(msg) && !device_can_wakeup(dwc->dev)) { + if (!PMSG_IS_AUTO(msg) && !device_may_wakeup(dwc->dev)) { ret = dwc3_core_init_for_resume(dwc); if (ret) return ret; diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h index 4fe4287dc934e7f441166f89018a87774268d44f..8f9959ba9fd4646effe5c436140748d38d60f7b2 100644 --- a/drivers/usb/dwc3/core.h +++ b/drivers/usb/dwc3/core.h @@ -263,6 +263,7 @@ #define DWC3_GUCTL1_DEV_FORCE_20_CLK_FOR_30_CLK BIT(26) #define DWC3_GUCTL1_DEV_L1_EXIT_BY_HW BIT(24) #define DWC3_GUCTL1_PARKMODE_DISABLE_SS BIT(17) +#define DWC3_GUCTL1_RESUME_OPMODE_HS_HOST BIT(10) /* Global Status Register */ #define DWC3_GSTS_OTG_IP BIT(10) @@ -391,6 +392,7 @@ #define DWC3_GFLADJ_30MHZ_SDBND_SEL BIT(7) #define DWC3_GFLADJ_30MHZ_MASK 0x3f #define DWC3_GFLADJ_REFCLK_FLADJ_MASK GENMASK(21, 8) +#define DWC3_GFLADJ_REFCLK_LPM_SEL BIT(23) #define DWC3_GFLADJ_240MHZDECR GENMASK(30, 24) #define DWC3_GFLADJ_240MHZDECR_PLS1 BIT(31) @@ -1096,6 +1098,8 @@ struct dwc3_scratchpad_array { * change quirk. * @dis_tx_ipgap_linecheck_quirk: set if we disable u2mac linestate * check during HS transmit. + * @resume-hs-terminations: Set if we enable quirk for fixing improper crc + * generation after resume from suspend. * @parkmode_disable_ss_quirk: set if we need to disable all SuperSpeed * instances in park mode. * @tx_de_emphasis_quirk: set if we enable Tx de-emphasis quirk @@ -1311,7 +1315,9 @@ struct dwc3 { unsigned dis_u2_freeclk_exists_quirk:1; unsigned dis_del_phy_power_chg_quirk:1; unsigned dis_tx_ipgap_linecheck_quirk:1; + unsigned resume_hs_terminations:1; unsigned parkmode_disable_ss_quirk:1; + unsigned gfladj_refclk_lpm_sel:1; unsigned tx_de_emphasis_quirk:1; unsigned tx_de_emphasis:2; @@ -1560,6 +1566,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, u32 param); void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc); +void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status); #else static inline int dwc3_gadget_init(struct dwc3 *dwc) { return 0; } diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h index d223c54115f4a16abcb8a72a5642497da33c8dea..48b44b88dc25295693002dc81cca29a121119fe0 100644 --- a/drivers/usb/dwc3/debug.h +++ b/drivers/usb/dwc3/debug.h @@ -278,7 +278,7 @@ static inline const char *dwc3_ep_event_string(char *str, size_t size, break; case DWC3_DEPEVT_XFERINPROGRESS: scnprintf(str + len, size - len, - "Transfer In Progress [%d] (%c%c%c)", + "Transfer In Progress [%08x] (%c%c%c)", event->parameters, status & DEPEVT_STATUS_SHORT ? 'S' : 's', status & DEPEVT_STATUS_IOC ? 'I' : 'i', @@ -286,7 +286,7 @@ static inline const char *dwc3_ep_event_string(char *str, size_t size, break; case DWC3_DEPEVT_XFERNOTREADY: len += scnprintf(str + len, size - len, - "Transfer Not Ready [%d]%s", + "Transfer Not Ready [%08x]%s", event->parameters, status & DEPEVT_STATUS_TRANSFER_ACTIVE ? " (Active)" : " (Not Active)"); diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c index 039bf241769afb789f7f09130e7af91eba62f12d..8cad9e7d3368725145fc27457c70efa641764a13 100644 --- a/drivers/usb/dwc3/drd.c +++ b/drivers/usb/dwc3/drd.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -438,6 +439,51 @@ static int dwc3_drd_notifier(struct notifier_block *nb, return NOTIFY_DONE; } +static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc) +{ + struct device *dev = dwc->dev; + struct device_node *np_phy; + struct extcon_dev *edev = NULL; + const char *name; + + if (device_property_read_bool(dev, "extcon")) + return extcon_get_edev_by_phandle(dev, 0); + + /* + * Device tree platforms should get extcon via phandle. + * On ACPI platforms, we get the name from a device property. + * This device property is for kernel internal use only and + * is expected to be set by the glue code. + */ + if (device_property_read_string(dev, "linux,extcon-name", &name) == 0) { + edev = extcon_get_extcon_dev(name); + if (!edev) + return ERR_PTR(-EPROBE_DEFER); + + return edev; + } + + /* + * Try to get an extcon device from the USB PHY controller's "port" + * node. Check if it has the "port" node first, to avoid printing the + * error message from underlying code, as it's a valid case: extcon + * device (and "port" node) may be missing in case of "usb-role-switch" + * or OTG mode. + */ + np_phy = of_parse_phandle(dev->of_node, "phys", 0); + if (of_graph_is_present(np_phy)) { + struct device_node *np_conn; + + np_conn = of_graph_get_remote_node(np_phy, -1, -1); + if (np_conn) + edev = extcon_find_edev_by_node(np_conn); + of_node_put(np_conn); + } + of_node_put(np_phy); + + return edev; +} + #if IS_ENABLED(CONFIG_USB_ROLE_SWITCH) #define ROLE_SWITCH 1 static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, @@ -542,6 +588,10 @@ int dwc3_drd_init(struct dwc3 *dwc) device_property_read_bool(dwc->dev, "usb-role-switch")) return dwc3_setup_role_switch(dwc); + dwc->edev = dwc3_get_extcon(dwc); + if (IS_ERR(dwc->edev)) + return PTR_ERR(dwc->edev); + if (dwc->edev) { dwc->edev_nb.notifier_call = dwc3_drd_notifier; ret = extcon_register_notifier(dwc->edev, EXTCON_USB_HOST, diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 6b018048fe2e15b9a11aa907db76773771c32b28..fb14511b1e10f98ce8b39841fb49a1f33a98646c 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -40,10 +40,12 @@ #define PCI_DEVICE_ID_INTEL_TGPLP 0xa0ee #define PCI_DEVICE_ID_INTEL_TGPH 0x43ee #define PCI_DEVICE_ID_INTEL_JSP 0x4dee -#define PCI_DEVICE_ID_INTEL_ADL 0x465e -#define PCI_DEVICE_ID_INTEL_ADLP 0x51ee -#define PCI_DEVICE_ID_INTEL_ADLM 0x54ee +#define PCI_DEVICE_ID_INTEL_ADL 0x460e +#define PCI_DEVICE_ID_INTEL_ADL_PCH 0x51ee +#define PCI_DEVICE_ID_INTEL_ADLN 0x465e +#define PCI_DEVICE_ID_INTEL_ADLN_PCH 0x54ee #define PCI_DEVICE_ID_INTEL_ADLS 0x7ae1 +#define PCI_DEVICE_ID_INTEL_RPL 0x460e #define PCI_DEVICE_ID_INTEL_RPLS 0x7a61 #define PCI_DEVICE_ID_INTEL_MTLP 0x7ec1 #define PCI_DEVICE_ID_INTEL_MTL 0x7e7e @@ -447,15 +449,21 @@ static const struct pci_device_id dwc3_pci_id_table[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLP), + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_PCH), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLM), + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLN), + (kernel_ulong_t) &dwc3_pci_intel_swnode, }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLN_PCH), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADLS), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL), + (kernel_ulong_t) &dwc3_pci_intel_swnode, }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPLS), (kernel_ulong_t) &dwc3_pci_intel_swnode, }, diff --git a/drivers/usb/dwc3/dwc3-qcom.c b/drivers/usb/dwc3/dwc3-qcom.c index c5e482f53e9db13f8232f6519fa7016a645d7887..7c40f3ffc054415aae6b2d03ea1fac0440a66bb6 100644 --- a/drivers/usb/dwc3/dwc3-qcom.c +++ b/drivers/usb/dwc3/dwc3-qcom.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -244,6 +243,7 @@ static int dwc3_qcom_interconnect_disable(struct dwc3_qcom *qcom) */ static int dwc3_qcom_interconnect_init(struct dwc3_qcom *qcom) { + enum usb_device_speed max_speed; struct device *dev = qcom->dev; int ret; @@ -253,7 +253,7 @@ static int dwc3_qcom_interconnect_init(struct dwc3_qcom *qcom) qcom->icc_path_ddr = of_icc_get(dev, "usb-ddr"); if (IS_ERR(qcom->icc_path_ddr)) { dev_err(dev, "failed to get usb-ddr path: %ld\n", - PTR_ERR(qcom->icc_path_ddr)); + PTR_ERR(qcom->icc_path_ddr)); return PTR_ERR(qcom->icc_path_ddr); } @@ -264,21 +264,20 @@ static int dwc3_qcom_interconnect_init(struct dwc3_qcom *qcom) return PTR_ERR(qcom->icc_path_apps); } - if (usb_get_maximum_speed(&qcom->dwc3->dev) >= USB_SPEED_SUPER || - usb_get_maximum_speed(&qcom->dwc3->dev) == USB_SPEED_UNKNOWN) + max_speed = usb_get_maximum_speed(&qcom->dwc3->dev); + if (max_speed >= USB_SPEED_SUPER || max_speed == USB_SPEED_UNKNOWN) { ret = icc_set_bw(qcom->icc_path_ddr, - USB_MEMORY_AVG_SS_BW, USB_MEMORY_PEAK_SS_BW); - else + USB_MEMORY_AVG_SS_BW, USB_MEMORY_PEAK_SS_BW); + } else { ret = icc_set_bw(qcom->icc_path_ddr, - USB_MEMORY_AVG_HS_BW, USB_MEMORY_PEAK_HS_BW); - + USB_MEMORY_AVG_HS_BW, USB_MEMORY_PEAK_HS_BW); + } if (ret) { dev_err(dev, "failed to set bandwidth for usb-ddr path: %d\n", ret); return ret; } - ret = icc_set_bw(qcom->icc_path_apps, - APPS_USB_AVG_BW, APPS_USB_PEAK_BW); + ret = icc_set_bw(qcom->icc_path_apps, APPS_USB_AVG_BW, APPS_USB_PEAK_BW); if (ret) { dev_err(dev, "failed to set bandwidth for apps-usb path: %d\n", ret); return ret; @@ -299,11 +298,24 @@ static void dwc3_qcom_interconnect_exit(struct dwc3_qcom *qcom) icc_put(qcom->icc_path_apps); } +/* Only usable in contexts where the role can not change. */ +static bool dwc3_qcom_is_host(struct dwc3_qcom *qcom) +{ + struct dwc3 *dwc = platform_get_drvdata(qcom->dwc3); + + return dwc->xhci; +} + static enum usb_device_speed dwc3_qcom_read_usb2_speed(struct dwc3_qcom *qcom) { struct dwc3 *dwc = platform_get_drvdata(qcom->dwc3); - struct usb_hcd *hcd = platform_get_drvdata(dwc->xhci); struct usb_device *udev; + struct usb_hcd __maybe_unused *hcd; + + /* + * FIXME: Fix this layering violation. + */ + hcd = platform_get_drvdata(dwc->xhci); /* * It is possible to query the speed of all children of @@ -311,8 +323,11 @@ static enum usb_device_speed dwc3_qcom_read_usb2_speed(struct dwc3_qcom *qcom) * currently supports only 1 port per controller. So * this is sufficient. */ +#ifdef CONFIG_USB udev = usb_hub_find_child(hcd->self.root_hub, 1); - +#else + udev = NULL; +#endif if (!udev) return USB_SPEED_UNKNOWN; @@ -387,7 +402,7 @@ static void dwc3_qcom_enable_interrupts(struct dwc3_qcom *qcom) dwc3_qcom_enable_wakeup_irq(qcom->ss_phy_irq, 0); } -static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) +static int dwc3_qcom_suspend(struct dwc3_qcom *qcom, bool wakeup) { u32 val; int i, ret; @@ -406,7 +421,11 @@ static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) if (ret) dev_warn(qcom->dev, "failed to disable interconnect: %d\n", ret); - if (device_may_wakeup(qcom->dev)) { + /* + * The role is stable during suspend as role switching is done from a + * freezable workqueue. + */ + if (dwc3_qcom_is_host(qcom) && wakeup) { qcom->usb2_speed = dwc3_qcom_read_usb2_speed(qcom); dwc3_qcom_enable_interrupts(qcom); } @@ -416,7 +435,7 @@ static int dwc3_qcom_suspend(struct dwc3_qcom *qcom) return 0; } -static int dwc3_qcom_resume(struct dwc3_qcom *qcom) +static int dwc3_qcom_resume(struct dwc3_qcom *qcom, bool wakeup) { int ret; int i; @@ -424,7 +443,7 @@ static int dwc3_qcom_resume(struct dwc3_qcom *qcom) if (!qcom->is_suspended) return 0; - if (device_may_wakeup(qcom->dev)) + if (dwc3_qcom_is_host(qcom) && wakeup) dwc3_qcom_disable_interrupts(qcom); for (i = 0; i < qcom->num_clocks; i++) { @@ -458,7 +477,11 @@ static irqreturn_t qcom_dwc3_resume_irq(int irq, void *data) if (qcom->pm_suspended) return IRQ_HANDLED; - if (dwc->xhci) + /* + * This is safe as role switching is done from a freezable workqueue + * and the wakeup interrupts are disabled as part of resume. + */ + if (dwc3_qcom_is_host(qcom)) pm_runtime_resume(&dwc->xhci->dev); return IRQ_HANDLED; @@ -757,13 +780,13 @@ dwc3_qcom_create_urs_usb_platdev(struct device *dev) static int dwc3_qcom_probe(struct platform_device *pdev) { - struct device_node *np = pdev->dev.of_node; - struct device *dev = &pdev->dev; - struct dwc3_qcom *qcom; - struct resource *res, *parent_res = NULL; - int ret, i; - bool ignore_pipe_clk; - struct generic_pm_domain *genpd; + struct device_node *np = pdev->dev.of_node; + struct device *dev = &pdev->dev; + struct dwc3_qcom *qcom; + struct resource *res, *parent_res = NULL; + int ret, i; + bool ignore_pipe_clk; + bool wakeup_source; qcom = devm_kzalloc(&pdev->dev, sizeof(*qcom), GFP_KERNEL); if (!qcom) @@ -772,8 +795,6 @@ static int dwc3_qcom_probe(struct platform_device *pdev) platform_set_drvdata(pdev, qcom); qcom->dev = &pdev->dev; - genpd = pd_to_genpd(qcom->dev->pm_domain); - if (has_acpi_companion(dev)) { qcom->acpi_pdata = acpi_device_get_match_data(dev); if (!qcom->acpi_pdata) { @@ -881,16 +902,9 @@ static int dwc3_qcom_probe(struct platform_device *pdev) if (ret) goto interconnect_exit; - if (device_can_wakeup(&qcom->dwc3->dev)) { - /* - * Setting GENPD_FLAG_ALWAYS_ON flag takes care of keeping - * genpd on in both runtime suspend and system suspend cases. - */ - genpd->flags |= GENPD_FLAG_ALWAYS_ON; - device_init_wakeup(&pdev->dev, true); - } else { - genpd->flags |= GENPD_FLAG_RPM_ALWAYS_ON; - } + wakeup_source = of_property_read_bool(dev->of_node, "wakeup-source"); + device_init_wakeup(&pdev->dev, wakeup_source); + device_init_wakeup(&qcom->dwc3->dev, wakeup_source); qcom->is_suspended = false; pm_runtime_set_active(dev); @@ -944,39 +958,45 @@ static int dwc3_qcom_remove(struct platform_device *pdev) static int __maybe_unused dwc3_qcom_pm_suspend(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); - int ret = 0; + bool wakeup = device_may_wakeup(dev); + int ret; - ret = dwc3_qcom_suspend(qcom); - if (!ret) - qcom->pm_suspended = true; + ret = dwc3_qcom_suspend(qcom, wakeup); + if (ret) + return ret; - return ret; + qcom->pm_suspended = true; + + return 0; } static int __maybe_unused dwc3_qcom_pm_resume(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); + bool wakeup = device_may_wakeup(dev); int ret; - ret = dwc3_qcom_resume(qcom); - if (!ret) - qcom->pm_suspended = false; + ret = dwc3_qcom_resume(qcom, wakeup); + if (ret) + return ret; - return ret; + qcom->pm_suspended = false; + + return 0; } static int __maybe_unused dwc3_qcom_runtime_suspend(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); - return dwc3_qcom_suspend(qcom); + return dwc3_qcom_suspend(qcom, true); } static int __maybe_unused dwc3_qcom_runtime_resume(struct device *dev) { struct dwc3_qcom *qcom = dev_get_drvdata(dev); - return dwc3_qcom_resume(qcom); + return dwc3_qcom_resume(qcom, true); } static const struct dev_pm_ops dwc3_qcom_dev_pm_ops = { @@ -987,10 +1007,6 @@ static const struct dev_pm_ops dwc3_qcom_dev_pm_ops = { static const struct of_device_id dwc3_qcom_of_match[] = { { .compatible = "qcom,dwc3" }, - { .compatible = "qcom,msm8996-dwc3" }, - { .compatible = "qcom,msm8998-dwc3" }, - { .compatible = "qcom,sdm660-dwc3" }, - { .compatible = "qcom,sdm845-dwc3" }, { } }; MODULE_DEVICE_TABLE(of, dwc3_qcom_of_match); diff --git a/drivers/usb/dwc3/dwc3-st.c b/drivers/usb/dwc3/dwc3-st.c index 166b5bde45cbc8b2ee40e8fc502c3a32e8ba3109..6c14a79279f9a677a34e0cea71ae651ae7f36d15 100644 --- a/drivers/usb/dwc3/dwc3-st.c +++ b/drivers/usb/dwc3/dwc3-st.c @@ -251,7 +251,7 @@ static int st_dwc3_probe(struct platform_device *pdev) /* Manage SoftReset */ reset_control_deassert(dwc3_data->rstc_rst); - child = of_get_child_by_name(node, "dwc3"); + child = of_get_child_by_name(node, "usb"); if (!child) { dev_err(&pdev->dev, "failed to find dwc3 core node\n"); ret = -ENODEV; diff --git a/drivers/usb/dwc3/dwc3-xilinx.c b/drivers/usb/dwc3/dwc3-xilinx.c index 67b237c7a76a10c2736b063f8932e91b8369392c..8607d4c23283c4db9d8afe8b00fdd75b97701bde 100644 --- a/drivers/usb/dwc3/dwc3-xilinx.c +++ b/drivers/usb/dwc3/dwc3-xilinx.c @@ -47,6 +47,7 @@ struct dwc3_xlnx { struct device *dev; void __iomem *regs; int (*pltfm_init)(struct dwc3_xlnx *data); + struct phy *usb3_phy; }; static void dwc3_xlnx_mask_phy_rst(struct dwc3_xlnx *priv_data, bool mask) @@ -100,13 +101,12 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data) struct device *dev = priv_data->dev; struct reset_control *crst, *hibrst, *apbrst; struct gpio_desc *reset_gpio; - struct phy *usb3_phy; int ret = 0; u32 reg; - usb3_phy = devm_phy_optional_get(dev, "usb3-phy"); - if (IS_ERR(usb3_phy)) { - ret = PTR_ERR(usb3_phy); + priv_data->usb3_phy = devm_phy_optional_get(dev, "usb3-phy"); + if (IS_ERR(priv_data->usb3_phy)) { + ret = PTR_ERR(priv_data->usb3_phy); dev_err_probe(dev, ret, "failed to get USB3 PHY\n"); goto err; @@ -121,7 +121,7 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data) * in use but the usb3-phy entry is missing from the device tree. * Therefore, skip these operations in this case. */ - if (!usb3_phy) + if (!priv_data->usb3_phy) goto skip_usb3_phy; crst = devm_reset_control_get_exclusive(dev, "usb_crst"); @@ -166,9 +166,9 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data) goto err; } - ret = phy_init(usb3_phy); + ret = phy_init(priv_data->usb3_phy); if (ret < 0) { - phy_exit(usb3_phy); + phy_exit(priv_data->usb3_phy); goto err; } @@ -196,9 +196,9 @@ static int dwc3_xlnx_init_zynqmp(struct dwc3_xlnx *priv_data) goto err; } - ret = phy_power_on(usb3_phy); + ret = phy_power_on(priv_data->usb3_phy); if (ret < 0) { - phy_exit(usb3_phy); + phy_exit(priv_data->usb3_phy); goto err; } @@ -322,7 +322,7 @@ static int dwc3_xlnx_remove(struct platform_device *pdev) return 0; } -static int __maybe_unused dwc3_xlnx_suspend_common(struct device *dev) +static int __maybe_unused dwc3_xlnx_runtime_suspend(struct device *dev) { struct dwc3_xlnx *priv_data = dev_get_drvdata(dev); @@ -331,7 +331,7 @@ static int __maybe_unused dwc3_xlnx_suspend_common(struct device *dev) return 0; } -static int __maybe_unused dwc3_xlnx_resume_common(struct device *dev) +static int __maybe_unused dwc3_xlnx_runtime_resume(struct device *dev) { struct dwc3_xlnx *priv_data = dev_get_drvdata(dev); @@ -346,8 +346,45 @@ static int __maybe_unused dwc3_xlnx_runtime_idle(struct device *dev) return 0; } -static UNIVERSAL_DEV_PM_OPS(dwc3_xlnx_dev_pm_ops, dwc3_xlnx_suspend_common, - dwc3_xlnx_resume_common, dwc3_xlnx_runtime_idle); +static int __maybe_unused dwc3_xlnx_suspend(struct device *dev) +{ + struct dwc3_xlnx *priv_data = dev_get_drvdata(dev); + + phy_exit(priv_data->usb3_phy); + + /* Disable the clocks */ + clk_bulk_disable(priv_data->num_clocks, priv_data->clks); + + return 0; +} + +static int __maybe_unused dwc3_xlnx_resume(struct device *dev) +{ + struct dwc3_xlnx *priv_data = dev_get_drvdata(dev); + int ret; + + ret = clk_bulk_enable(priv_data->num_clocks, priv_data->clks); + if (ret) + return ret; + + ret = phy_init(priv_data->usb3_phy); + if (ret < 0) + return ret; + + ret = phy_power_on(priv_data->usb3_phy); + if (ret < 0) { + phy_exit(priv_data->usb3_phy); + return ret; + } + + return 0; +} + +static const struct dev_pm_ops dwc3_xlnx_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(dwc3_xlnx_suspend, dwc3_xlnx_resume) + SET_RUNTIME_PM_OPS(dwc3_xlnx_runtime_suspend, + dwc3_xlnx_runtime_resume, dwc3_xlnx_runtime_idle) +}; static struct platform_driver dwc3_xlnx_driver = { .probe = dwc3_xlnx_probe, diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index 197af63f8d05fa5cb8774cbc5f51a43070034a6c..61de693461da4780bb9017b67a0d37e823b1b7ee 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -197,7 +197,7 @@ int dwc3_gadget_ep0_queue(struct usb_ep *ep, struct usb_request *request, int ret; spin_lock_irqsave(&dwc->lock, flags); - if (!dep->endpoint.desc || !dwc->pullups_connected) { + if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", dep->name); ret = -ESHUTDOWN; @@ -293,7 +293,10 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) continue; dwc3_ep->flags &= ~DWC3_EP_DELAY_STOP; - dwc3_stop_active_transfer(dwc3_ep, true, true); + if (dwc->connected) + dwc3_stop_active_transfer(dwc3_ep, true, true); + else + dwc3_remove_requests(dwc, dwc3_ep, -ESHUTDOWN); } } @@ -815,7 +818,7 @@ static void dwc3_ep0_inspect_setup(struct dwc3 *dwc, int ret = -EINVAL; u32 len; - if (!dwc->gadget_driver || !dwc->connected) + if (!dwc->gadget_driver || !dwc->softconnect || !dwc->connected) goto out; trace_dwc3_ctrl_req(ctrl); @@ -1118,6 +1121,8 @@ static void dwc3_ep0_xfernotready(struct dwc3 *dwc, { switch (event->status) { case DEPEVT_STATUS_CONTROL_DATA: + if (!dwc->softconnect || !dwc->connected) + return; /* * We already have a DATA transfer in the controller's cache, * if we receive a XferNotReady(DATA) we will ignore it, unless diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index aeeec751c53c7b9b8423e2962d6fe463b94dc0c7..079cd333632e149c68accdf9e672c41285410a29 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -366,7 +366,9 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); - if (!(cmd & DWC3_DEPCMD_CMDACT)) { + if (!(cmd & DWC3_DEPCMD_CMDACT) || + (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && + !(cmd & DWC3_DEPCMD_CMDIOC))) { ret = 0; goto skip_status; } @@ -965,29 +967,33 @@ out: return 0; } -static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep) +void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) { struct dwc3_request *req; dwc3_stop_active_transfer(dep, true, false); + /* If endxfer is delayed, avoid unmapping requests */ + if (dep->flags & DWC3_EP_DELAY_STOP) + return; + /* - giveback all requests to gadget driver */ while (!list_empty(&dep->started_list)) { req = next_request(&dep->started_list); - dwc3_gadget_giveback(dep, req, -ESHUTDOWN); + dwc3_gadget_giveback(dep, req, status); } while (!list_empty(&dep->pending_list)) { req = next_request(&dep->pending_list); - dwc3_gadget_giveback(dep, req, -ESHUTDOWN); + dwc3_gadget_giveback(dep, req, status); } while (!list_empty(&dep->cancelled_list)) { req = next_request(&dep->cancelled_list); - dwc3_gadget_giveback(dep, req, -ESHUTDOWN); + dwc3_gadget_giveback(dep, req, status); } } @@ -1005,6 +1011,7 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) { struct dwc3 *dwc = dep->dwc; u32 reg; + u32 mask; trace_dwc3_gadget_ep_disable(dep); @@ -1022,11 +1029,19 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) dep->endpoint.desc = NULL; } - dwc3_remove_requests(dwc, dep); + dwc3_remove_requests(dwc, dep, -ECONNRESET); dep->stream_capable = false; dep->type = 0; - dep->flags &= DWC3_EP_TXFIFO_RESIZED; + mask = DWC3_EP_TXFIFO_RESIZED; + /* + * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is + * set. Do not clear DEP flags, so that the end transfer command will + * be reattempted during the next SETUP stage. + */ + if (dep->flags & DWC3_EP_DELAY_STOP) + mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED); + dep->flags &= mask; return 0; } @@ -2340,7 +2355,7 @@ static void dwc3_stop_active_transfers(struct dwc3 *dwc) if (!dep) continue; - dwc3_remove_requests(dwc, dep); + dwc3_remove_requests(dwc, dep, -ESHUTDOWN); } } @@ -2440,7 +2455,7 @@ static void __dwc3_gadget_set_speed(struct dwc3 *dwc) static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend) { u32 reg; - u32 timeout = 500; + u32 timeout = 2000; if (pm_runtime_suspended(dwc->dev)) return 0; @@ -2473,6 +2488,7 @@ static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend) dwc3_gadget_dctl_write_safe(dwc, reg); do { + usleep_range(1000, 2000); reg = dwc3_readl(dwc->regs, DWC3_DSTS); reg &= DWC3_DSTS_DEVCTRLHLT; } while (--timeout && !(!is_on ^ !reg)); @@ -2501,6 +2517,9 @@ static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) if (dwc->ep0state != EP0_SETUP_PHASE) { int ret; + if (dwc->delayed_status) + dwc3_ep0_send_delayed_status(dwc); + reinit_completion(&dwc->ep0_in_setup); spin_unlock_irqrestore(&dwc->lock, flags); @@ -2539,9 +2558,6 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) is_on = !!is_on; - if (dwc->pullups_connected == is_on) - return 0; - dwc->softconnect = is_on; /* @@ -2566,6 +2582,13 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) return 0; } + if (dwc->pullups_connected == is_on) { + pm_runtime_put(dwc->dev); + return 0; + } + + synchronize_irq(dwc->irq_gadget); + if (!is_on) { ret = dwc3_gadget_soft_disconnect(dwc); } else { @@ -2716,6 +2739,7 @@ static int __dwc3_gadget_start(struct dwc3 *dwc) dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); dep = dwc->eps[0]; + dep->flags = 0; ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); @@ -2723,6 +2747,7 @@ static int __dwc3_gadget_start(struct dwc3 *dwc) } dep = dwc->eps[1]; + dep->flags = 0; ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); @@ -3566,7 +3591,7 @@ static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, * streams are updated, and the device controller will not be * triggered to generate ERDY to move the next stream data. To * workaround this and maintain compatibility with various - * hosts, force to reinitate the stream until the host is ready + * hosts, force to reinitiate the stream until the host is ready * instead of waiting for the host to prime the endpoint. */ if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) { @@ -3594,11 +3619,12 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc, dep = dwc->eps[epnum]; if (!(dep->flags & DWC3_EP_ENABLED)) { - if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) + if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) return; /* Handle only EPCMDCMPLT when EP disabled */ - if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) + if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && + !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) return; } @@ -3693,7 +3719,7 @@ void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, * timeout. Delay issuing the End Transfer command until the Setup TRB is * prepared. */ - if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { + if (dwc->ep0state != EP0_SETUP_PHASE) { dep->flags |= DWC3_EP_DELAY_STOP; return; } @@ -3761,13 +3787,24 @@ static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) reg &= ~DWC3_DCTL_INITU2ENA; dwc3_gadget_dctl_write_safe(dwc, reg); + dwc->connected = false; + dwc3_disconnect_gadget(dwc); dwc->gadget->speed = USB_SPEED_UNKNOWN; dwc->setup_packet_pending = false; usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); - dwc->connected = false; + if (dwc->ep0state != EP0_SETUP_PHASE) { + unsigned int dir; + + dir = !!dwc->ep0_expect_in; + if (dwc->ep0state == EP0_DATA_PHASE) + dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); + else + dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); + dwc3_ep0_stall_and_restart(dwc); + } } static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) @@ -3865,6 +3902,9 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) u8 lanes = 1; u8 speed; + if (!dwc->softconnect) + return; + reg = dwc3_readl(dwc->regs, DWC3_DSTS); speed = reg & DWC3_DSTS_CONNECTSPD; dwc->speed = speed; @@ -4127,7 +4167,7 @@ static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc, unsigned int is_ss = evtinfo & BIT(4); /* - * WORKAROUND: DWC3 revison 2.20a with hibernation support + * WORKAROUND: DWC3 revision 2.20a with hibernation support * have a known issue which can cause USB CV TD.9.23 to fail * randomly. * @@ -4505,12 +4545,17 @@ void dwc3_gadget_exit(struct dwc3 *dwc) int dwc3_gadget_suspend(struct dwc3 *dwc) { + unsigned long flags; + if (!dwc->gadget_driver) return 0; dwc3_gadget_run_stop(dwc, false, false); + + spin_lock_irqsave(&dwc->lock, flags); dwc3_disconnect_gadget(dwc); __dwc3_gadget_stop(dwc); + spin_unlock_irqrestore(&dwc->lock, flags); return 0; } diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index f56c30cf151e4e5eddce3c3d0d9f7d2f730ad32a..a7154fe8206d147fdc00d3f6affbcb92b5772093 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -11,8 +11,13 @@ #include #include +#include "../host/xhci-plat.h" #include "core.h" +static const struct xhci_plat_priv dwc3_xhci_plat_priv = { + .quirks = XHCI_SKIP_PHY_INIT, +}; + static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, int irq, char *name) { @@ -92,6 +97,11 @@ int dwc3_host_init(struct dwc3 *dwc) goto err; } + ret = platform_device_add_data(xhci, &dwc3_xhci_plat_priv, + sizeof(dwc3_xhci_plat_priv)); + if (ret) + goto err; + memset(props, 0, sizeof(struct property_entry) * ARRAY_SIZE(props)); if (dwc->usb3_lpm_capable) @@ -135,4 +145,5 @@ err: void dwc3_host_exit(struct dwc3 *dwc) { platform_device_unregister(dwc->xhci); + dwc->xhci = NULL; } diff --git a/drivers/usb/dwc3/trace.h b/drivers/usb/dwc3/trace.h index cb998ba50fea9dc4912a925b60878379fe967f78..1975aec8d36d853261733ed9e189f8c29d7ad621 100644 --- a/drivers/usb/dwc3/trace.h +++ b/drivers/usb/dwc3/trace.h @@ -241,7 +241,7 @@ DECLARE_EVENT_CLASS(dwc3_log_trb, __entry->enqueue = dep->trb_enqueue; __entry->dequeue = dep->trb_dequeue; ), - TP_printk("%s: trb %p (E%d:D%d) buf %08x%08x size %s%d ctrl %08x (%c%c%c%c:%c%c:%s)", + TP_printk("%s: trb %p (E%d:D%d) buf %08x%08x size %s%d ctrl %08x sofn %08x (%c%c%c%c:%c%c:%s)", __get_str(name), __entry->trb, __entry->enqueue, __entry->dequeue, __entry->bph, __entry->bpl, ({char *s; @@ -267,6 +267,7 @@ DECLARE_EVENT_CLASS(dwc3_log_trb, s = ""; } s; }), DWC3_TRB_SIZE_LENGTH(__entry->size), __entry->ctrl, + DWC3_TRB_CTRL_GET_SID_SOFN(__entry->ctrl), __entry->ctrl & DWC3_TRB_CTRL_HWO ? 'H' : 'h', __entry->ctrl & DWC3_TRB_CTRL_LST ? 'L' : 'l', __entry->ctrl & DWC3_TRB_CTRL_CHN ? 'C' : 'c', diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index e0fa4b186ec6d21e037eeb76737b49c09c3ee88a..73dc10a77cdeadda5a3ef63b5e8d5be32959a3ff 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -2645,10 +2645,10 @@ static int __ffs_data_got_strings(struct ffs_data *ffs, unsigned i = 0; vla_group(d); vla_item(d, struct usb_gadget_strings *, stringtabs, - lang_count + 1); + size_add(lang_count, 1)); vla_item(d, struct usb_gadget_strings, stringtab, lang_count); vla_item(d, struct usb_string, strings, - lang_count*(needed_count+1)); + size_mul(lang_count, (needed_count + 1))); char *vlabuf = kmalloc(vla_group_size(d), GFP_KERNEL); @@ -3700,7 +3700,7 @@ int ffs_name_dev(struct ffs_dev *dev, const char *name) existing = _ffs_do_find_dev(name); if (!existing) - strlcpy(dev->name, name, ARRAY_SIZE(dev->name)); + strscpy(dev->name, name, ARRAY_SIZE(dev->name)); else if (existing != dev) ret = -EBUSY; diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 925e99f9775c723d567215258978c824263d7671..3abf7f586e2afba4f08be76bbcb29ea6c7fcdfcd 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -2662,11 +2662,16 @@ static ssize_t forced_eject_store(struct device *dev, } static DEVICE_ATTR_RW(nofua); -/* mode wil be set in fsg_lun_attr_is_visible() */ -static DEVICE_ATTR(ro, 0, ro_show, ro_store); -static DEVICE_ATTR(file, 0, file_show, file_store); static DEVICE_ATTR_WO(forced_eject); +/* + * Mode of the ro and file attribute files will be overridden in + * fsg_lun_dev_is_visible() depending on if this is a cdrom, or if it is a + * removable device. + */ +static DEVICE_ATTR_RW(ro); +static DEVICE_ATTR_RW(file); + /****************************** FSG COMMON ******************************/ static void fsg_lun_release(struct device *dev) diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index dc8f078f918c515540f1ad95031d41ef6bbc57d7..c36bcfa0e9b46f17e85e7731fe12ca1a28a769a9 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -450,39 +450,35 @@ struct ndp_parser_opts { unsigned next_ndp_index; }; -#define INIT_NDP16_OPTS { \ - .nth_sign = USB_CDC_NCM_NTH16_SIGN, \ - .ndp_sign = USB_CDC_NCM_NDP16_NOCRC_SIGN, \ - .nth_size = sizeof(struct usb_cdc_ncm_nth16), \ - .ndp_size = sizeof(struct usb_cdc_ncm_ndp16), \ - .dpe_size = sizeof(struct usb_cdc_ncm_dpe16), \ - .ndplen_align = 4, \ - .dgram_item_len = 1, \ - .block_length = 1, \ - .ndp_index = 1, \ - .reserved1 = 0, \ - .reserved2 = 0, \ - .next_ndp_index = 1, \ - } - - -#define INIT_NDP32_OPTS { \ - .nth_sign = USB_CDC_NCM_NTH32_SIGN, \ - .ndp_sign = USB_CDC_NCM_NDP32_NOCRC_SIGN, \ - .nth_size = sizeof(struct usb_cdc_ncm_nth32), \ - .ndp_size = sizeof(struct usb_cdc_ncm_ndp32), \ - .dpe_size = sizeof(struct usb_cdc_ncm_dpe32), \ - .ndplen_align = 8, \ - .dgram_item_len = 2, \ - .block_length = 2, \ - .ndp_index = 2, \ - .reserved1 = 1, \ - .reserved2 = 2, \ - .next_ndp_index = 2, \ - } +static const struct ndp_parser_opts ndp16_opts = { + .nth_sign = USB_CDC_NCM_NTH16_SIGN, + .ndp_sign = USB_CDC_NCM_NDP16_NOCRC_SIGN, + .nth_size = sizeof(struct usb_cdc_ncm_nth16), + .ndp_size = sizeof(struct usb_cdc_ncm_ndp16), + .dpe_size = sizeof(struct usb_cdc_ncm_dpe16), + .ndplen_align = 4, + .dgram_item_len = 1, + .block_length = 1, + .ndp_index = 1, + .reserved1 = 0, + .reserved2 = 0, + .next_ndp_index = 1, +}; -static const struct ndp_parser_opts ndp16_opts = INIT_NDP16_OPTS; -static const struct ndp_parser_opts ndp32_opts = INIT_NDP32_OPTS; +static const struct ndp_parser_opts ndp32_opts = { + .nth_sign = USB_CDC_NCM_NTH32_SIGN, + .ndp_sign = USB_CDC_NCM_NDP32_NOCRC_SIGN, + .nth_size = sizeof(struct usb_cdc_ncm_nth32), + .ndp_size = sizeof(struct usb_cdc_ncm_ndp32), + .dpe_size = sizeof(struct usb_cdc_ncm_dpe32), + .ndplen_align = 8, + .dgram_item_len = 2, + .block_length = 2, + .ndp_index = 2, + .reserved1 = 1, + .reserved2 = 2, + .next_ndp_index = 2, +}; static inline void put_ncm(__le16 **p, unsigned size, unsigned val) { diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c index abec5c58f5251ae3e7c3a34a8521b1b9465ea86e..a881c69b1f2bf56cf20fda287efcd708e7abdbe3 100644 --- a/drivers/usb/gadget/function/f_printer.c +++ b/drivers/usb/gadget/function/f_printer.c @@ -89,7 +89,7 @@ struct printer_dev { u8 printer_cdev_open; wait_queue_head_t wait; unsigned q_len; - char *pnp_string; /* We don't own memory! */ + char **pnp_string; /* We don't own memory! */ struct usb_function function; }; @@ -1000,16 +1000,16 @@ static int printer_func_setup(struct usb_function *f, if ((wIndex>>8) != dev->interface) break; - if (!dev->pnp_string) { + if (!*dev->pnp_string) { value = 0; break; } - value = strlen(dev->pnp_string); + value = strlen(*dev->pnp_string); buf[0] = (value >> 8) & 0xFF; buf[1] = value & 0xFF; - memcpy(buf + 2, dev->pnp_string, value); + memcpy(buf + 2, *dev->pnp_string, value); DBG(dev, "1284 PNP String: %x %s\n", value, - dev->pnp_string); + *dev->pnp_string); break; case GET_PORT_STATUS: /* Get Port Status */ @@ -1475,7 +1475,7 @@ static struct usb_function *gprinter_alloc(struct usb_function_instance *fi) kref_init(&dev->kref); ++opts->refcnt; dev->minor = opts->minor; - dev->pnp_string = opts->pnp_string; + dev->pnp_string = &opts->pnp_string; dev->q_len = opts->q_len; mutex_unlock(&opts->lock); diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 8e17ac831be00e14015c41bb9d44b31c8bb90adf..658e2e21fdd0deb4b9eac1812ab5792e81a360d2 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -2306,7 +2306,7 @@ static struct usb_function *tcm_alloc(struct usb_function_instance *fi) DECLARE_USB_FUNCTION(tcm, tcm_alloc_inst, tcm_alloc); -static int tcm_init(void) +static int __init tcm_init(void) { int ret; @@ -2322,7 +2322,7 @@ static int tcm_init(void) } module_init(tcm_init); -static void tcm_exit(void) +static void __exit tcm_exit(void) { target_unregister_template(&usbg_ops); usb_function_unregister(&tcmusb_func); diff --git a/drivers/usb/gadget/function/f_uac2.c b/drivers/usb/gadget/function/f_uac2.c index 1905a8d8e0c9fa51bbc2b96edadd2fe0f3df216d..08726e4c68a56d4ae865a22096dda7ec44697228 100644 --- a/drivers/usb/gadget/function/f_uac2.c +++ b/drivers/usb/gadget/function/f_uac2.c @@ -291,6 +291,12 @@ static struct usb_endpoint_descriptor ss_ep_int_desc = { .bInterval = 4, }; +static struct usb_ss_ep_comp_descriptor ss_ep_int_desc_comp = { + .bLength = sizeof(ss_ep_int_desc_comp), + .bDescriptorType = USB_DT_SS_ENDPOINT_COMP, + .wBytesPerInterval = cpu_to_le16(6), +}; + /* Audio Streaming OUT Interface - Alt0 */ static struct usb_interface_descriptor std_as_out_if0_desc = { .bLength = sizeof std_as_out_if0_desc, @@ -604,7 +610,8 @@ static struct usb_descriptor_header *ss_audio_desc[] = { (struct usb_descriptor_header *)&in_feature_unit_desc, (struct usb_descriptor_header *)&io_out_ot_desc, - (struct usb_descriptor_header *)&ss_ep_int_desc, + (struct usb_descriptor_header *)&ss_ep_int_desc, + (struct usb_descriptor_header *)&ss_ep_int_desc_comp, (struct usb_descriptor_header *)&std_as_out_if0_desc, (struct usb_descriptor_header *)&std_as_out_if1_desc, @@ -800,6 +807,7 @@ static void setup_headers(struct f_uac2_opts *opts, struct usb_ss_ep_comp_descriptor *epout_desc_comp = NULL; struct usb_ss_ep_comp_descriptor *epin_desc_comp = NULL; struct usb_ss_ep_comp_descriptor *epin_fback_desc_comp = NULL; + struct usb_ss_ep_comp_descriptor *ep_int_desc_comp = NULL; struct usb_endpoint_descriptor *epout_desc; struct usb_endpoint_descriptor *epin_desc; struct usb_endpoint_descriptor *epin_fback_desc; @@ -827,6 +835,7 @@ static void setup_headers(struct f_uac2_opts *opts, epin_fback_desc = &ss_epin_fback_desc; epin_fback_desc_comp = &ss_epin_fback_desc_comp; ep_int_desc = &ss_ep_int_desc; + ep_int_desc_comp = &ss_ep_int_desc_comp; } i = 0; @@ -855,8 +864,11 @@ static void setup_headers(struct f_uac2_opts *opts, if (EPOUT_EN(opts)) headers[i++] = USBDHDR(&io_out_ot_desc); - if (FUOUT_EN(opts) || FUIN_EN(opts)) + if (FUOUT_EN(opts) || FUIN_EN(opts)) { headers[i++] = USBDHDR(ep_int_desc); + if (ep_int_desc_comp) + headers[i++] = USBDHDR(ep_int_desc_comp); + } if (EPOUT_EN(opts)) { headers[i++] = USBDHDR(&std_as_out_if0_desc); diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 71669e0e4d0074d4125acee5800f143515ad0007..6e196e06181ecf21b2bf225adbadefed9e0c9f2b 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -421,7 +421,7 @@ uvc_register_video(struct uvc_device *uvc) int ret; /* TODO reference counting. */ - memset(&uvc->vdev, 0, sizeof(uvc->video)); + memset(&uvc->vdev, 0, sizeof(uvc->vdev)); uvc->vdev.v4l2_dev = &uvc->v4l2_dev; uvc->vdev.v4l2_dev->dev = &cdev->gadget->dev; uvc->vdev.fops = &uvc_v4l2_fops; @@ -430,7 +430,7 @@ uvc_register_video(struct uvc_device *uvc) uvc->vdev.vfl_dir = VFL_DIR_TX; uvc->vdev.lock = &uvc->video.mutex; uvc->vdev.device_caps = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING; - strlcpy(uvc->vdev.name, cdev->gadget->name, sizeof(uvc->vdev.name)); + strscpy(uvc->vdev.name, cdev->gadget->name, sizeof(uvc->vdev.name)); video_set_drvdata(&uvc->vdev, uvc); @@ -888,6 +888,7 @@ static void uvc_free(struct usb_function *f) struct uvc_device *uvc = to_uvc(f); struct f_uvc_opts *opts = container_of(f->fi, struct f_uvc_opts, func_inst); + config_item_put(&uvc->header->item); --opts->refcnt; kfree(uvc); } @@ -897,10 +898,14 @@ static void uvc_function_unbind(struct usb_configuration *c, { struct usb_composite_dev *cdev = c->cdev; struct uvc_device *uvc = to_uvc(f); + struct uvc_video *video = &uvc->video; long wait_ret = 1; uvcg_info(f, "%s()\n", __func__); + if (video->async_wq) + destroy_workqueue(video->async_wq); + /* * If we know we're connected via v4l2, then there should be a cleanup * of the device from userspace either via UVC_EVENT_DISCONNECT or @@ -941,6 +946,7 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) struct uvc_device *uvc; struct f_uvc_opts *opts; struct uvc_descriptor_header **strm_cls; + struct config_item *streaming, *header, *h; uvc = kzalloc(sizeof(*uvc), GFP_KERNEL); if (uvc == NULL) @@ -973,6 +979,28 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) uvc->desc.fs_streaming = opts->fs_streaming; uvc->desc.hs_streaming = opts->hs_streaming; uvc->desc.ss_streaming = opts->ss_streaming; + + streaming = config_group_find_item(&opts->func_inst.group, "streaming"); + if (!streaming) + goto err_config; + + header = config_group_find_item(to_config_group(streaming), "header"); + config_item_put(streaming); + if (!header) + goto err_config; + + h = config_group_find_item(to_config_group(header), "h"); + config_item_put(header); + if (!h) + goto err_config; + + uvc->header = to_uvcg_streaming_header(h); + if (!uvc->header->linked) { + mutex_unlock(&opts->lock); + kfree(uvc); + return ERR_PTR(-EBUSY); + } + ++opts->refcnt; mutex_unlock(&opts->lock); @@ -988,6 +1016,11 @@ static struct usb_function *uvc_alloc(struct usb_function_instance *fi) uvc->func.bind_deactivated = true; return &uvc->func; + +err_config: + mutex_unlock(&opts->lock); + kfree(uvc); + return ERR_PTR(-ENOENT); } DECLARE_USB_FUNCTION_INIT(uvc, uvc_alloc_inst, uvc_alloc); diff --git a/drivers/usb/gadget/function/rndis.c b/drivers/usb/gadget/function/rndis.c index 713efd9aefde8d17e1e32d1200d2345370b6c9bb..29bf8664bf582dfc685a2dca5b8ad21ffcc08ebf 100644 --- a/drivers/usb/gadget/function/rndis.c +++ b/drivers/usb/gadget/function/rndis.c @@ -869,7 +869,7 @@ EXPORT_SYMBOL_GPL(rndis_msg_parser); static inline int rndis_get_nr(void) { - return ida_simple_get(&rndis_ida, 0, 0, GFP_KERNEL); + return ida_simple_get(&rndis_ida, 0, 1000, GFP_KERNEL); } static inline void rndis_put_nr(int nr) @@ -1105,7 +1105,7 @@ static int rndis_proc_show(struct seq_file *m, void *v) "used : %s\n" "state : %s\n" "medium : 0x%08X\n" - "speed : %d\n" + "speed : %u\n" "cable : %s\n" "vendor ID : 0x%08X\n" "vendor : %s\n", diff --git a/drivers/usb/gadget/function/storage_common.c b/drivers/usb/gadget/function/storage_common.c index 03035dbbe97b87a67e098b047f8461363cfe848c..208c6a92780ab28744b31932c2cdd9961277453a 100644 --- a/drivers/usb/gadget/function/storage_common.c +++ b/drivers/usb/gadget/function/storage_common.c @@ -294,8 +294,10 @@ EXPORT_SYMBOL_GPL(fsg_lun_fsync_sub); void store_cdrom_address(u8 *dest, int msf, u32 addr) { if (msf) { - /* Convert to Minutes-Seconds-Frames */ - addr >>= 2; /* Convert to 2048-byte frames */ + /* + * Convert to Minutes-Seconds-Frames. + * Sector size is already set to 2048 bytes. + */ addr += 2*75; /* Lead-in occupies 2 seconds */ dest[3] = addr % 75; /* Frames */ addr /= 75; diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c index 7887def05dc2d8e2ab79dd7b7947defce527665a..e06022873df16a357a2ba5b6f82f64b7c4cb081e 100644 --- a/drivers/usb/gadget/function/u_ether.c +++ b/drivers/usb/gadget/function/u_ether.c @@ -144,10 +144,10 @@ static void eth_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *p) { struct eth_dev *dev = netdev_priv(net); - strlcpy(p->driver, "g_ether", sizeof(p->driver)); - strlcpy(p->version, UETH__VERSION, sizeof(p->version)); - strlcpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); - strlcpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); + strscpy(p->driver, "g_ether", sizeof(p->driver)); + strscpy(p->version, UETH__VERSION, sizeof(p->version)); + strscpy(p->fw_version, dev->gadget->name, sizeof(p->fw_version)); + strscpy(p->bus_info, dev_name(&dev->gadget->dev), sizeof(p->bus_info)); } /* REVISIT can also support: diff --git a/drivers/usb/gadget/function/u_serial.c b/drivers/usb/gadget/function/u_serial.c index 6f68cbeeee7c027272dd4cf0925ce3ab2bb80a5a..7538279f98179a17b7ced9b182ebb36b6f655304 100644 --- a/drivers/usb/gadget/function/u_serial.c +++ b/drivers/usb/gadget/function/u_serial.c @@ -1443,7 +1443,7 @@ void gserial_resume(struct gserial *gser) } EXPORT_SYMBOL_GPL(gserial_resume); -static int userial_init(void) +static int __init userial_init(void) { struct tty_driver *driver; unsigned i; @@ -1496,7 +1496,7 @@ fail: } module_init(userial_init); -static void userial_cleanup(void) +static void __exit userial_cleanup(void) { tty_unregister_driver(gs_tty_driver); tty_driver_kref_put(gs_tty_driver); diff --git a/drivers/usb/gadget/function/u_uac1_legacy.c b/drivers/usb/gadget/function/u_uac1_legacy.c index 60ae8b2d3f6a2f78f71b3d624ff15f18f66df493..dd21c251542cb4e417ef9b755652b0aa8ad2286d 100644 --- a/drivers/usb/gadget/function/u_uac1_legacy.c +++ b/drivers/usb/gadget/function/u_uac1_legacy.c @@ -158,8 +158,8 @@ size_t u_audio_playback(struct gaudio *card, void *buf, size_t count) snd_pcm_sframes_t frames; try_again: - if (runtime->status->state == SNDRV_PCM_STATE_XRUN || - runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) { + if (runtime->state == SNDRV_PCM_STATE_XRUN || + runtime->state == SNDRV_PCM_STATE_SUSPENDED) { result = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_PREPARE, NULL); if (result < 0) { diff --git a/drivers/usb/gadget/function/uvc.h b/drivers/usb/gadget/function/uvc.h index 58e383afdd4406c85fb262fd4a5309388e244dbc..40226b1f7e148afbbae99c74d5478617ca3f9581 100644 --- a/drivers/usb/gadget/function/uvc.h +++ b/drivers/usb/gadget/function/uvc.h @@ -88,6 +88,7 @@ struct uvc_video { struct usb_ep *ep; struct work_struct pump; + struct workqueue_struct *async_wq; /* Frame parameters */ u8 bpp; @@ -133,6 +134,8 @@ struct uvc_device { bool func_connected; wait_queue_head_t func_connected_queue; + struct uvcg_streaming_header *header; + /* Descriptors */ struct { const struct uvc_descriptor_header * const *fs_control; diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index fd8f73bb726dd1589f885b096562934af511df1f..c4ed48d6b8a407a744299d4a2c4b07e0495103aa 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -18,12 +18,161 @@ #include #include #include +#include #include "f_uvc.h" #include "uvc.h" #include "uvc_queue.h" #include "uvc_video.h" #include "uvc_v4l2.h" +#include "uvc_configfs.h" + +static struct uvc_format_desc *to_uvc_format(struct uvcg_format *uformat) +{ + char guid[16] = UVC_GUID_FORMAT_MJPEG; + struct uvc_format_desc *format; + struct uvcg_uncompressed *unc; + + if (uformat->type == UVCG_UNCOMPRESSED) { + unc = to_uvcg_uncompressed(&uformat->group.cg_item); + if (!unc) + return ERR_PTR(-EINVAL); + + memcpy(guid, unc->desc.guidFormat, sizeof(guid)); + } + + format = uvc_format_by_guid(guid); + if (!format) + return ERR_PTR(-EINVAL); + + return format; +} + +static int uvc_v4l2_get_bytesperline(struct uvcg_format *uformat, + struct uvcg_frame *uframe) +{ + struct uvcg_uncompressed *u; + + if (uformat->type == UVCG_UNCOMPRESSED) { + u = to_uvcg_uncompressed(&uformat->group.cg_item); + if (!u) + return 0; + + return u->desc.bBitsPerPixel * uframe->frame.w_width / 8; + } + + return 0; +} + +static int uvc_get_frame_size(struct uvcg_format *uformat, + struct uvcg_frame *uframe) +{ + unsigned int bpl = uvc_v4l2_get_bytesperline(uformat, uframe); + + return bpl ? bpl * uframe->frame.w_height : + uframe->frame.dw_max_video_frame_buffer_size; +} + +static struct uvcg_format *find_format_by_index(struct uvc_device *uvc, int index) +{ + struct uvcg_format_ptr *format; + struct uvcg_format *uformat = NULL; + int i = 1; + + list_for_each_entry(format, &uvc->header->formats, entry) { + if (index == i) { + uformat = format->fmt; + break; + } + i++; + } + + return uformat; +} + +static struct uvcg_frame *find_frame_by_index(struct uvc_device *uvc, + struct uvcg_format *uformat, + int index) +{ + struct uvcg_format_ptr *format; + struct uvcg_frame_ptr *frame; + struct uvcg_frame *uframe = NULL; + + list_for_each_entry(format, &uvc->header->formats, entry) { + if (format->fmt->type != uformat->type) + continue; + list_for_each_entry(frame, &format->fmt->frames, entry) { + if (index == frame->frm->frame.b_frame_index) { + uframe = frame->frm; + break; + } + } + } + + return uframe; +} + +static struct uvcg_format *find_format_by_pix(struct uvc_device *uvc, + u32 pixelformat) +{ + struct uvcg_format_ptr *format; + struct uvcg_format *uformat = NULL; + + list_for_each_entry(format, &uvc->header->formats, entry) { + struct uvc_format_desc *fmtdesc = to_uvc_format(format->fmt); + + if (fmtdesc->fcc == pixelformat) { + uformat = format->fmt; + break; + } + } + + return uformat; +} + +static struct uvcg_frame *find_closest_frame_by_size(struct uvc_device *uvc, + struct uvcg_format *uformat, + u16 rw, u16 rh) +{ + struct uvc_video *video = &uvc->video; + struct uvcg_format_ptr *format; + struct uvcg_frame_ptr *frame; + struct uvcg_frame *uframe = NULL; + unsigned int d, maxd; + + /* Find the closest image size. The distance between image sizes is + * the size in pixels of the non-overlapping regions between the + * requested size and the frame-specified size. + */ + maxd = (unsigned int)-1; + + list_for_each_entry(format, &uvc->header->formats, entry) { + if (format->fmt->type != uformat->type) + continue; + + list_for_each_entry(frame, &format->fmt->frames, entry) { + u16 w, h; + + w = frame->frm->frame.w_width; + h = frame->frm->frame.w_height; + + d = min(w, rw) * min(h, rh); + d = w*h + rw*rh - 2*d; + if (d < maxd) { + maxd = d; + uframe = frame->frm; + } + + if (maxd == 0) + break; + } + } + + if (!uframe) + uvcg_dbg(&video->uvc->func, "Unsupported size %ux%u\n", rw, rh); + + return uframe; +} /* -------------------------------------------------------------------------- * Requests handling @@ -67,9 +216,9 @@ uvc_v4l2_querycap(struct file *file, void *fh, struct v4l2_capability *cap) struct uvc_device *uvc = video_get_drvdata(vdev); struct usb_composite_dev *cdev = uvc->func.config->cdev; - strlcpy(cap->driver, "g_uvc", sizeof(cap->driver)); - strlcpy(cap->card, cdev->gadget->name, sizeof(cap->card)); - strlcpy(cap->bus_info, dev_name(&cdev->gadget->dev), + strscpy(cap->driver, "g_uvc", sizeof(cap->driver)); + strscpy(cap->card, cdev->gadget->name, sizeof(cap->card)); + strscpy(cap->bus_info, dev_name(&cdev->gadget->dev), sizeof(cap->bus_info)); return 0; } @@ -134,6 +283,139 @@ uvc_v4l2_set_format(struct file *file, void *fh, struct v4l2_format *fmt) return 0; } +static int +uvc_v4l2_try_format(struct file *file, void *fh, struct v4l2_format *fmt) +{ + struct video_device *vdev = video_devdata(file); + struct uvc_device *uvc = video_get_drvdata(vdev); + struct uvc_video *video = &uvc->video; + struct uvcg_format *uformat; + struct uvcg_frame *uframe; + u8 *fcc; + + if (fmt->type != video->queue.queue.type) + return -EINVAL; + + fcc = (u8 *)&fmt->fmt.pix.pixelformat; + uvcg_dbg(&uvc->func, "Trying format 0x%08x (%c%c%c%c): %ux%u\n", + fmt->fmt.pix.pixelformat, + fcc[0], fcc[1], fcc[2], fcc[3], + fmt->fmt.pix.width, fmt->fmt.pix.height); + + uformat = find_format_by_pix(uvc, fmt->fmt.pix.pixelformat); + if (!uformat) + return -EINVAL; + + uframe = find_closest_frame_by_size(uvc, uformat, + fmt->fmt.pix.width, fmt->fmt.pix.height); + if (!uframe) + return -EINVAL; + + fmt->fmt.pix.width = uframe->frame.w_width; + fmt->fmt.pix.height = uframe->frame.w_height; + fmt->fmt.pix.field = V4L2_FIELD_NONE; + fmt->fmt.pix.bytesperline = uvc_v4l2_get_bytesperline(uformat, uframe); + fmt->fmt.pix.sizeimage = uvc_get_frame_size(uformat, uframe); + fmt->fmt.pix.pixelformat = to_uvc_format(uformat)->fcc; + fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SRGB; + fmt->fmt.pix.priv = 0; + + return 0; +} + +static int +uvc_v4l2_enum_frameintervals(struct file *file, void *fh, + struct v4l2_frmivalenum *fival) +{ + struct video_device *vdev = video_devdata(file); + struct uvc_device *uvc = video_get_drvdata(vdev); + struct uvcg_format *uformat = NULL; + struct uvcg_frame *uframe = NULL; + struct uvcg_frame_ptr *frame; + + uformat = find_format_by_pix(uvc, fival->pixel_format); + if (!uformat) + return -EINVAL; + + list_for_each_entry(frame, &uformat->frames, entry) { + if (frame->frm->frame.w_width == fival->width && + frame->frm->frame.w_height == fival->height) { + uframe = frame->frm; + break; + } + } + if (!uframe) + return -EINVAL; + + if (fival->index >= uframe->frame.b_frame_interval_type) + return -EINVAL; + + fival->discrete.numerator = + uframe->dw_frame_interval[fival->index]; + + /* TODO: handle V4L2_FRMIVAL_TYPE_STEPWISE */ + fival->type = V4L2_FRMIVAL_TYPE_DISCRETE; + fival->discrete.denominator = 10000000; + v4l2_simplify_fraction(&fival->discrete.numerator, + &fival->discrete.denominator, 8, 333); + + return 0; +} + +static int +uvc_v4l2_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *fsize) +{ + struct video_device *vdev = video_devdata(file); + struct uvc_device *uvc = video_get_drvdata(vdev); + struct uvcg_format *uformat = NULL; + struct uvcg_frame *uframe = NULL; + + uformat = find_format_by_pix(uvc, fsize->pixel_format); + if (!uformat) + return -EINVAL; + + if (fsize->index >= uformat->num_frames) + return -EINVAL; + + uframe = find_frame_by_index(uvc, uformat, fsize->index + 1); + if (!uframe) + return -EINVAL; + + fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE; + fsize->discrete.width = uframe->frame.w_width; + fsize->discrete.height = uframe->frame.w_height; + + return 0; +} + +static int +uvc_v4l2_enum_format(struct file *file, void *fh, struct v4l2_fmtdesc *f) +{ + struct video_device *vdev = video_devdata(file); + struct uvc_device *uvc = video_get_drvdata(vdev); + struct uvc_format_desc *fmtdesc; + struct uvcg_format *uformat; + + if (f->index >= uvc->header->num_fmt) + return -EINVAL; + + uformat = find_format_by_index(uvc, f->index + 1); + if (!uformat) + return -EINVAL; + + if (uformat->type != UVCG_UNCOMPRESSED) + f->flags |= V4L2_FMT_FLAG_COMPRESSED; + + fmtdesc = to_uvc_format(uformat); + f->pixelformat = fmtdesc->fcc; + + strscpy(f->description, fmtdesc->name, sizeof(f->description)); + f->description[strlen(fmtdesc->name) - 1] = 0; + + return 0; +} + static int uvc_v4l2_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *b) { @@ -170,7 +452,7 @@ uvc_v4l2_qbuf(struct file *file, void *fh, struct v4l2_buffer *b) return ret; if (uvc->state == UVC_STATE_STREAMING) - schedule_work(&video->pump); + queue_work(video->async_wq, &video->pump); return ret; } @@ -298,8 +580,12 @@ uvc_v4l2_ioctl_default(struct file *file, void *fh, bool valid_prio, const struct v4l2_ioctl_ops uvc_v4l2_ioctl_ops = { .vidioc_querycap = uvc_v4l2_querycap, + .vidioc_try_fmt_vid_out = uvc_v4l2_try_format, .vidioc_g_fmt_vid_out = uvc_v4l2_get_format, .vidioc_s_fmt_vid_out = uvc_v4l2_set_format, + .vidioc_enum_frameintervals = uvc_v4l2_enum_frameintervals, + .vidioc_enum_framesizes = uvc_v4l2_enum_framesizes, + .vidioc_enum_fmt_vid_out = uvc_v4l2_enum_format, .vidioc_reqbufs = uvc_v4l2_reqbufs, .vidioc_querybuf = uvc_v4l2_querybuf, .vidioc_qbuf = uvc_v4l2_qbuf, diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index c00ce0e91f5d5cf6357c3ec311423b3f171a4bc8..bb037fcc90e69eeeee5ca930935c57f7ab7e614e 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -277,7 +277,7 @@ uvc_video_complete(struct usb_ep *ep, struct usb_request *req) spin_unlock_irqrestore(&video->req_lock, flags); if (uvc->state == UVC_STATE_STREAMING) - schedule_work(&video->pump); + queue_work(video->async_wq, &video->pump); } static int @@ -485,7 +485,7 @@ int uvcg_video_enable(struct uvc_video *video, int enable) video->req_int_count = 0; - schedule_work(&video->pump); + queue_work(video->async_wq, &video->pump); return ret; } @@ -499,6 +499,11 @@ int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc) spin_lock_init(&video->req_lock); INIT_WORK(&video->pump, uvcg_video_pump); + /* Allocate a work queue for asynchronous video pump handler. */ + video->async_wq = alloc_workqueue("uvcgadget", WQ_UNBOUND | WQ_HIGHPRI, 0); + if (!video->async_wq) + return -EINVAL; + video->uvc = uvc; video->fcc = V4L2_PIX_FMT_YUYV; video->bpp = 16; diff --git a/drivers/usb/gadget/udc/at91_udc.c b/drivers/usb/gadget/udc/at91_udc.c index 728987280373a6c219545f4c1c10f0c504398693..a9a7b3fc60ec961c544b994d1cb476c5217cbb40 100644 --- a/drivers/usb/gadget/udc/at91_udc.c +++ b/drivers/usb/gadget/udc/at91_udc.c @@ -994,7 +994,7 @@ static const struct usb_gadget_ops at91_udc_ops = { .udc_stop = at91_stop, /* - * VBUS-powered devices may also also want to support bigger + * VBUS-powered devices may also want to support bigger * power budgets after an appropriate SET_CONFIGURATION. */ /* .vbus_power = at91_vbus_power, */ @@ -1779,12 +1779,14 @@ static void at91udc_of_init(struct at91_udc *udc, struct device_node *np) if (of_property_read_u32(np, "atmel,vbus-polled", &val) == 0) board->vbus_polled = 1; - board->vbus_pin = gpiod_get_from_of_node(np, "atmel,vbus-gpio", 0, - GPIOD_IN, "udc_vbus"); + board->vbus_pin = fwnode_gpiod_get_index(of_fwnode_handle(np), + "atmel,vbus", 0, GPIOD_IN, + "udc_vbus"); if (IS_ERR(board->vbus_pin)) board->vbus_pin = NULL; - board->pullup_pin = gpiod_get_from_of_node(np, "atmel,pullup-gpio", 0, + board->pullup_pin = fwnode_gpiod_get_index(of_fwnode_handle(np), + "atmel,pullup", 0, GPIOD_ASIS, "udc_pullup"); if (IS_ERR(board->pullup_pin)) board->pullup_pin = NULL; diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index cafcf260394cd8d24cd3c1bdffddeb7e94c7f7d8..c63c0c2cf649d531da684c377a051bc35f742902 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -736,7 +736,10 @@ int usb_gadget_disconnect(struct usb_gadget *gadget) ret = gadget->ops->pullup(gadget, 0); if (!ret) { gadget->connected = 0; - gadget->udc->driver->disconnect(gadget); + mutex_lock(&udc_lock); + if (gadget->udc->driver) + gadget->udc->driver->disconnect(gadget); + mutex_unlock(&udc_lock); } out: @@ -1489,7 +1492,6 @@ static int gadget_bind_driver(struct device *dev) usb_gadget_udc_set_speed(udc, driver->max_speed); - mutex_lock(&udc_lock); ret = driver->bind(udc->gadget, driver); if (ret) goto err_bind; @@ -1499,7 +1501,6 @@ static int gadget_bind_driver(struct device *dev) goto err_start; usb_gadget_enable_async_callbacks(udc); usb_udc_connect_control(udc); - mutex_unlock(&udc_lock); kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE); return 0; @@ -1512,6 +1513,7 @@ static int gadget_bind_driver(struct device *dev) dev_err(&udc->dev, "failed to start %s: %d\n", driver->function, ret); + mutex_lock(&udc_lock); udc->driver = NULL; driver->is_bound = false; mutex_unlock(&udc_lock); @@ -1529,7 +1531,6 @@ static void gadget_unbind_driver(struct device *dev) kobject_uevent(&udc->dev.kobj, KOBJ_CHANGE); - mutex_lock(&udc_lock); usb_gadget_disconnect(gadget); usb_gadget_disable_async_callbacks(udc); if (gadget->irq) @@ -1537,6 +1538,7 @@ static void gadget_unbind_driver(struct device *dev) udc->driver->unbind(gadget); usb_gadget_udc_stop(udc); + mutex_lock(&udc_lock); driver->is_bound = false; udc->driver = NULL; mutex_unlock(&udc_lock); @@ -1612,7 +1614,7 @@ static ssize_t soft_connect_store(struct device *dev, struct usb_udc *udc = container_of(dev, struct usb_udc, dev); ssize_t ret; - mutex_lock(&udc_lock); + device_lock(&udc->gadget->dev); if (!udc->driver) { dev_err(dev, "soft-connect without a gadget driver\n"); ret = -EOPNOTSUPP; @@ -1633,7 +1635,7 @@ static ssize_t soft_connect_store(struct device *dev, ret = n; out: - mutex_unlock(&udc_lock); + device_unlock(&udc->gadget->dev); return ret; } static DEVICE_ATTR_WO(soft_connect); @@ -1652,11 +1654,15 @@ static ssize_t function_show(struct device *dev, struct device_attribute *attr, char *buf) { struct usb_udc *udc = container_of(dev, struct usb_udc, dev); - struct usb_gadget_driver *drv = udc->driver; + struct usb_gadget_driver *drv; + int rc = 0; - if (!drv || !drv->function) - return 0; - return scnprintf(buf, PAGE_SIZE, "%s\n", drv->function); + mutex_lock(&udc_lock); + drv = udc->driver; + if (drv && drv->function) + rc = scnprintf(buf, PAGE_SIZE, "%s\n", drv->function); + mutex_unlock(&udc_lock); + return rc; } static DEVICE_ATTR_RO(function); diff --git a/drivers/usb/gadget/udc/net2272.c b/drivers/usb/gadget/udc/net2272.c index c97cd4bc817c67731dd70bef9b131d8080459331..84605a4d0715f53e927a800dbc485b8fd0a7da27 100644 --- a/drivers/usb/gadget/udc/net2272.c +++ b/drivers/usb/gadget/udc/net2272.c @@ -91,7 +91,7 @@ module_param(dma_mode, ushort, 0644); * mode 2 == ep-a 1k, ep-b 1k, ep-c 512db * mode 3 == ep-a 1k, ep-b disabled, ep-c 512db */ -static ushort fifo_mode = 0; +static ushort fifo_mode; module_param(fifo_mode, ushort, 0644); /* @@ -100,7 +100,7 @@ module_param(fifo_mode, ushort, 0644); * USB suspend requests will be ignored. This is acceptable for * self-powered devices. For bus powered devices set this to 1. */ -static ushort enable_suspend = 0; +static ushort enable_suspend; module_param(enable_suspend, ushort, 0644); static void assert_out_naking(struct net2272_ep *ep, const char *where) diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c index 61cabb9de6ae1b5b5638ef3684eb7cdc138c7d98..bea346e362b2b72df7910c417995d4756aa94927 100644 --- a/drivers/usb/gadget/udc/omap_udc.c +++ b/drivers/usb/gadget/udc/omap_udc.c @@ -2234,7 +2234,7 @@ static int proc_otg_show(struct seq_file *s) char *ctrl_name = "(UNKNOWN)"; tmp = omap_readl(OTG_REV); - ctrl_name = "tranceiver_ctrl"; + ctrl_name = "transceiver_ctrl"; trans = omap_readw(USB_TRANSCEIVER_CTRL); seq_printf(s, "\nOTG rev %d.%d, %s %05x\n", tmp >> 4, tmp & 0xf, ctrl_name, trans); @@ -2558,7 +2558,7 @@ omap_ep_setup(char *name, u8 addr, u8 type, /* set up driver data structures */ BUG_ON(strlen(name) >= sizeof ep->name); - strlcpy(ep->name, name, sizeof ep->name); + strscpy(ep->name, name, sizeof(ep->name)); INIT_LIST_HEAD(&ep->queue); INIT_LIST_HEAD(&ep->iso); ep->bEndpointAddress = addr; diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c index 648be3fd476a5038ae5057e6b1757775911dcbc2..615ba0a6fbee1b0b57e6b74aa8b559c71402deef 100644 --- a/drivers/usb/gadget/udc/renesas_usb3.c +++ b/drivers/usb/gadget/udc/renesas_usb3.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -38,16 +39,16 @@ #define USB3_USB20_CON 0x204 #define USB3_USB30_CON 0x208 #define USB3_USB_STA 0x210 -#define USB3_DRD_CON 0x218 +#define USB3_DRD_CON(p) ((p)->is_rzv2m ? 0x400 : 0x218) #define USB3_USB_INT_STA_1 0x220 #define USB3_USB_INT_STA_2 0x224 #define USB3_USB_INT_ENA_1 0x228 #define USB3_USB_INT_ENA_2 0x22c #define USB3_STUP_DAT_0 0x230 #define USB3_STUP_DAT_1 0x234 -#define USB3_USB_OTG_STA 0x268 -#define USB3_USB_OTG_INT_STA 0x26c -#define USB3_USB_OTG_INT_ENA 0x270 +#define USB3_USB_OTG_STA(p) ((p)->is_rzv2m ? 0x410 : 0x268) +#define USB3_USB_OTG_INT_STA(p) ((p)->is_rzv2m ? 0x414 : 0x26c) +#define USB3_USB_OTG_INT_ENA(p) ((p)->is_rzv2m ? 0x418 : 0x270) #define USB3_P0_MOD 0x280 #define USB3_P0_CON 0x288 #define USB3_P0_STA 0x28c @@ -135,6 +136,8 @@ #define USB_STA_VBUS_STA BIT(0) /* DRD_CON */ +#define DRD_CON_PERI_RST BIT(31) /* rzv2m only */ +#define DRD_CON_HOST_RST BIT(30) /* rzv2m only */ #define DRD_CON_PERI_CON BIT(24) #define DRD_CON_VBOUT BIT(0) @@ -155,7 +158,7 @@ #define USB_INT_2_PIPE(n) BIT(n) /* USB_OTG_STA, USB_OTG_INT_STA and USB_OTG_INT_ENA */ -#define USB_OTG_IDMON BIT(4) +#define USB_OTG_IDMON(p) ((p)->is_rzv2m ? BIT(0) : BIT(4)) /* P0_MOD */ #define P0_MOD_DIR BIT(6) @@ -255,7 +258,7 @@ #define USB3_EP0_SS_MAX_PACKET_SIZE 512 #define USB3_EP0_HSFS_MAX_PACKET_SIZE 64 #define USB3_EP0_BUF_SIZE 8 -#define USB3_MAX_NUM_PIPES 6 /* This includes PIPE 0 */ +#define USB3_MAX_NUM_PIPES(p) ((p)->is_rzv2m ? 16 : 6) /* This includes PIPE 0 */ #define USB3_WAIT_US 3 #define USB3_DMA_NUM_SETTING_AREA 4 /* @@ -326,10 +329,13 @@ struct renesas_usb3_priv { int num_ramif; int ramsize_per_pipe; /* unit = bytes */ bool workaround_for_vbus; /* if true, don't check vbus signal */ + bool is_rzv2m; /* if true, RZ/V2M SoC */ }; struct renesas_usb3 { void __iomem *reg; + struct reset_control *drd_rstc; + struct reset_control *usbp_rstc; struct usb_gadget gadget; struct usb_gadget_driver *driver; @@ -363,6 +369,7 @@ struct renesas_usb3 { bool forced_b_device; bool start_to_connect; bool role_sw_by_connector; + bool is_rzv2m; }; #define gadget_to_renesas_usb3(_gadget) \ @@ -467,7 +474,7 @@ static void usb3_disable_pipe_irq(struct renesas_usb3 *usb3, int num) static bool usb3_is_host(struct renesas_usb3 *usb3) { - return !(usb3_read(usb3, USB3_DRD_CON) & DRD_CON_PERI_CON); + return !(usb3_read(usb3, USB3_DRD_CON(usb3)) & DRD_CON_PERI_CON); } static void usb3_init_axi_bridge(struct renesas_usb3 *usb3) @@ -674,10 +681,20 @@ static void renesas_usb3_role_work(struct work_struct *work) static void usb3_set_mode(struct renesas_usb3 *usb3, bool host) { + if (usb3->is_rzv2m) { + if (host) { + usb3_set_bit(usb3, DRD_CON_PERI_RST, USB3_DRD_CON(usb3)); + usb3_clear_bit(usb3, DRD_CON_HOST_RST, USB3_DRD_CON(usb3)); + } else { + usb3_set_bit(usb3, DRD_CON_HOST_RST, USB3_DRD_CON(usb3)); + usb3_clear_bit(usb3, DRD_CON_PERI_RST, USB3_DRD_CON(usb3)); + } + } + if (host) - usb3_clear_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON); + usb3_clear_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON(usb3)); else - usb3_set_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON); + usb3_set_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON(usb3)); } static void usb3_set_mode_by_role_sw(struct renesas_usb3 *usb3, bool host) @@ -693,9 +710,9 @@ static void usb3_set_mode_by_role_sw(struct renesas_usb3 *usb3, bool host) static void usb3_vbus_out(struct renesas_usb3 *usb3, bool enable) { if (enable) - usb3_set_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON); + usb3_set_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON(usb3)); else - usb3_clear_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON); + usb3_clear_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON(usb3)); } static void usb3_mode_config(struct renesas_usb3 *usb3, bool host, bool a_dev) @@ -716,7 +733,7 @@ static void usb3_mode_config(struct renesas_usb3 *usb3, bool host, bool a_dev) static bool usb3_is_a_device(struct renesas_usb3 *usb3) { - return !(usb3_read(usb3, USB3_USB_OTG_STA) & USB_OTG_IDMON); + return !(usb3_read(usb3, USB3_USB_OTG_STA(usb3)) & USB_OTG_IDMON(usb3)); } static void usb3_check_id(struct renesas_usb3 *usb3) @@ -739,8 +756,8 @@ static void renesas_usb3_init_controller(struct renesas_usb3 *usb3) usb3_set_bit(usb3, USB_COM_CON_PN_WDATAIF_NL | USB_COM_CON_PN_RDATAIF_NL | USB_COM_CON_PN_LSTTR_PP, USB3_USB_COM_CON); - usb3_write(usb3, USB_OTG_IDMON, USB3_USB_OTG_INT_STA); - usb3_write(usb3, USB_OTG_IDMON, USB3_USB_OTG_INT_ENA); + usb3_write(usb3, USB_OTG_IDMON(usb3), USB3_USB_OTG_INT_STA(usb3)); + usb3_write(usb3, USB_OTG_IDMON(usb3), USB3_USB_OTG_INT_ENA(usb3)); usb3_check_id(usb3); usb3_check_vbus(usb3); @@ -750,7 +767,7 @@ static void renesas_usb3_stop_controller(struct renesas_usb3 *usb3) { usb3_disconnect(usb3); usb3_write(usb3, 0, USB3_P0_INT_ENA); - usb3_write(usb3, 0, USB3_USB_OTG_INT_ENA); + usb3_write(usb3, 0, USB3_USB_OTG_INT_ENA(usb3)); usb3_write(usb3, 0, USB3_USB_INT_ENA_1); usb3_write(usb3, 0, USB3_USB_INT_ENA_2); usb3_write(usb3, 0, USB3_AXI_INT_ENA); @@ -2005,9 +2022,15 @@ static void usb3_irq_idmon_change(struct renesas_usb3 *usb3) usb3_check_id(usb3); } -static void usb3_irq_otg_int(struct renesas_usb3 *usb3, u32 otg_int_sta) +static void usb3_irq_otg_int(struct renesas_usb3 *usb3) { - if (otg_int_sta & USB_OTG_IDMON) + u32 otg_int_sta = usb3_read(usb3, USB3_USB_OTG_INT_STA(usb3)); + + otg_int_sta &= usb3_read(usb3, USB3_USB_OTG_INT_ENA(usb3)); + if (otg_int_sta) + usb3_write(usb3, otg_int_sta, USB3_USB_OTG_INT_STA(usb3)); + + if (otg_int_sta & USB_OTG_IDMON(usb3)) usb3_irq_idmon_change(usb3); } @@ -2015,7 +2038,6 @@ static void usb3_irq_epc(struct renesas_usb3 *usb3) { u32 int_sta_1 = usb3_read(usb3, USB3_USB_INT_STA_1); u32 int_sta_2 = usb3_read(usb3, USB3_USB_INT_STA_2); - u32 otg_int_sta = usb3_read(usb3, USB3_USB_OTG_INT_STA); int_sta_1 &= usb3_read(usb3, USB3_USB_INT_ENA_1); if (int_sta_1) { @@ -2027,11 +2049,8 @@ static void usb3_irq_epc(struct renesas_usb3 *usb3) if (int_sta_2) usb3_irq_epc_int_2(usb3, int_sta_2); - otg_int_sta &= usb3_read(usb3, USB3_USB_OTG_INT_ENA); - if (otg_int_sta) { - usb3_write(usb3, otg_int_sta, USB3_USB_OTG_INT_STA); - usb3_irq_otg_int(usb3, otg_int_sta); - } + if (!usb3->is_rzv2m) + usb3_irq_otg_int(usb3); } static void usb3_irq_dma_int(struct renesas_usb3 *usb3, u32 dma_sta) @@ -2085,6 +2104,15 @@ static irqreturn_t renesas_usb3_irq(int irq, void *_usb3) return ret; } +static irqreturn_t renesas_usb3_otg_irq(int irq, void *_usb3) +{ + struct renesas_usb3 *usb3 = _usb3; + + usb3_irq_otg_int(usb3); + + return IRQ_HANDLED; +} + static void usb3_write_pn_mod(struct renesas_usb3_ep *usb3_ep, const struct usb_endpoint_descriptor *desc) { @@ -2571,6 +2599,8 @@ static int renesas_usb3_remove(struct platform_device *pdev) usb_role_switch_unregister(usb3->role_sw); usb_del_gadget_udc(&usb3->gadget); + reset_control_assert(usb3->usbp_rstc); + reset_control_assert(usb3->drd_rstc); renesas_usb3_dma_free_prd(usb3, &pdev->dev); __renesas_usb3_ep_free_request(usb3->ep0_req); @@ -2589,8 +2619,8 @@ static int renesas_usb3_init_ep(struct renesas_usb3 *usb3, struct device *dev, usb3->num_usb3_eps = priv->ramsize_per_ramif * priv->num_ramif * 2 / priv->ramsize_per_pipe + 1; - if (usb3->num_usb3_eps > USB3_MAX_NUM_PIPES) - usb3->num_usb3_eps = USB3_MAX_NUM_PIPES; + if (usb3->num_usb3_eps > USB3_MAX_NUM_PIPES(usb3)) + usb3->num_usb3_eps = USB3_MAX_NUM_PIPES(usb3); usb3->usb3_ep = devm_kcalloc(dev, usb3->num_usb3_eps, sizeof(*usb3_ep), @@ -2707,6 +2737,13 @@ static const struct renesas_usb3_priv renesas_usb3_priv_r8a77990 = { .workaround_for_vbus = true, }; +static const struct renesas_usb3_priv renesas_usb3_priv_rzv2m = { + .ramsize_per_ramif = SZ_16K, + .num_ramif = 1, + .ramsize_per_pipe = SZ_4K, + .is_rzv2m = true, +}; + static const struct of_device_id usb3_of_match[] = { { .compatible = "renesas,r8a774c0-usb3-peri", @@ -2717,6 +2754,9 @@ static const struct of_device_id usb3_of_match[] = { }, { .compatible = "renesas,r8a77990-usb3-peri", .data = &renesas_usb3_priv_r8a77990, + }, { + .compatible = "renesas,rzv2m-usb3-peri", + .data = &renesas_usb3_priv_rzv2m, }, { .compatible = "renesas,rcar-gen3-usb3-peri", .data = &renesas_usb3_priv_gen3, @@ -2748,7 +2788,7 @@ static struct usb_role_switch_desc renesas_usb3_role_switch_desc = { static int renesas_usb3_probe(struct platform_device *pdev) { struct renesas_usb3 *usb3; - int irq, ret; + int irq, drd_irq, ret; const struct renesas_usb3_priv *priv; const struct soc_device_attribute *attr; @@ -2762,10 +2802,18 @@ static int renesas_usb3_probe(struct platform_device *pdev) if (irq < 0) return irq; + if (priv->is_rzv2m) { + drd_irq = platform_get_irq_byname(pdev, "drd"); + if (drd_irq < 0) + return drd_irq; + } + usb3 = devm_kzalloc(&pdev->dev, sizeof(*usb3), GFP_KERNEL); if (!usb3) return -ENOMEM; + usb3->is_rzv2m = priv->is_rzv2m; + usb3->reg = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(usb3->reg)) return PTR_ERR(usb3->reg); @@ -2787,6 +2835,14 @@ static int renesas_usb3_probe(struct platform_device *pdev) if (ret < 0) return ret; + if (usb3->is_rzv2m) { + ret = devm_request_irq(&pdev->dev, drd_irq, + renesas_usb3_otg_irq, 0, + dev_name(&pdev->dev), usb3); + if (ret < 0) + return ret; + } + INIT_WORK(&usb3->extcon_work, renesas_usb3_extcon_work); usb3->extcon = devm_extcon_dev_allocate(&pdev->dev, renesas_usb3_cable); if (IS_ERR(usb3->extcon)) @@ -2817,10 +2873,27 @@ static int renesas_usb3_probe(struct platform_device *pdev) goto err_add_udc; } + usb3->drd_rstc = devm_reset_control_get_optional_shared(&pdev->dev, + "drd_reset"); + if (IS_ERR(usb3->drd_rstc)) { + ret = PTR_ERR(usb3->drd_rstc); + goto err_add_udc; + } + + usb3->usbp_rstc = devm_reset_control_get_optional_shared(&pdev->dev, + "aresetn_p"); + if (IS_ERR(usb3->usbp_rstc)) { + ret = PTR_ERR(usb3->usbp_rstc); + goto err_add_udc; + } + + reset_control_deassert(usb3->drd_rstc); + reset_control_deassert(usb3->usbp_rstc); + pm_runtime_enable(&pdev->dev); ret = usb_add_gadget_udc(&pdev->dev, &usb3->gadget); if (ret < 0) - goto err_add_udc; + goto err_reset; ret = device_create_file(&pdev->dev, &dev_attr_role); if (ret < 0) @@ -2858,6 +2931,10 @@ static int renesas_usb3_probe(struct platform_device *pdev) err_dev_create: usb_del_gadget_udc(&usb3->gadget); +err_reset: + reset_control_assert(usb3->usbp_rstc); + reset_control_assert(usb3->drd_rstc); + err_add_udc: renesas_usb3_dma_free_prd(usb3, &pdev->dev); diff --git a/drivers/usb/gadget/udc/s3c2410_udc.c b/drivers/usb/gadget/udc/s3c2410_udc.c index c6625aeb7bca25e5304dcf179e4732277a48a282..8c57b191e52b04f0c1846d0b8b4d1c2a4bfe658a 100644 --- a/drivers/usb/gadget/udc/s3c2410_udc.c +++ b/drivers/usb/gadget/udc/s3c2410_udc.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include @@ -1419,8 +1419,7 @@ static int s3c2410_udc_set_pullup(struct s3c2410_udc *udc, int is_on) { dprintk(DEBUG_NORMAL, "%s()\n", __func__); - if (udc_info && (udc_info->udc_command || - gpio_is_valid(udc_info->pullup_pin))) { + if (udc_info && (udc_info->udc_command || udc->pullup_gpiod)) { if (is_on) s3c2410_udc_enable(udc); @@ -1467,9 +1466,7 @@ static irqreturn_t s3c2410_udc_vbus_irq(int irq, void *_dev) dprintk(DEBUG_NORMAL, "%s()\n", __func__); - value = gpio_get_value(udc_info->vbus_pin) ? 1 : 0; - if (udc_info->vbus_pin_inverted) - value = !value; + value = gpiod_get_value(dev->vbus_gpiod); if (value != dev->vbus) s3c2410_udc_vbus_session(&dev->gadget, value); @@ -1504,14 +1501,15 @@ static const struct usb_gadget_ops s3c2410_ops = { .udc_stop = s3c2410_udc_stop, }; -static void s3c2410_udc_command(enum s3c2410_udc_cmd_e cmd) +static void s3c2410_udc_command(struct s3c2410_udc *udc, + enum s3c2410_udc_cmd_e cmd) { if (!udc_info) return; if (udc_info->udc_command) { udc_info->udc_command(cmd); - } else if (gpio_is_valid(udc_info->pullup_pin)) { + } else if (udc->pullup_gpiod) { int value; switch (cmd) { @@ -1524,9 +1522,8 @@ static void s3c2410_udc_command(enum s3c2410_udc_cmd_e cmd) default: return; } - value ^= udc_info->pullup_pin_inverted; - gpio_set_value(udc_info->pullup_pin, value); + gpiod_set_value(udc->pullup_gpiod, value); } } @@ -1551,7 +1548,7 @@ static void s3c2410_udc_disable(struct s3c2410_udc *dev) udc_write(0x1F, S3C2410_UDC_EP_INT_REG); /* Good bye, cruel world */ - s3c2410_udc_command(S3C2410_UDC_P_DISABLE); + s3c2410_udc_command(dev, S3C2410_UDC_P_DISABLE); /* Set speed to unknown */ dev->gadget.speed = USB_SPEED_UNKNOWN; @@ -1613,7 +1610,7 @@ static void s3c2410_udc_enable(struct s3c2410_udc *dev) udc_write(S3C2410_UDC_INT_EP0, S3C2410_UDC_EP_INT_EN_REG); /* time to say "hello, world" */ - s3c2410_udc_command(S3C2410_UDC_P_ENABLE); + s3c2410_udc_command(dev, S3C2410_UDC_P_ENABLE); } static int s3c2410_udc_start(struct usb_gadget *g, @@ -1802,14 +1799,15 @@ static int s3c2410_udc_probe(struct platform_device *pdev) dev_dbg(dev, "got irq %i\n", irq_usbd); - if (udc_info && udc_info->vbus_pin > 0) { - retval = gpio_request(udc_info->vbus_pin, "udc vbus"); - if (retval < 0) { - dev_err(dev, "cannot claim vbus pin\n"); - goto err_int; - } + udc->vbus_gpiod = gpiod_get_optional(dev, "vbus", GPIOD_IN); + if (IS_ERR(udc->vbus_gpiod)) { + retval = PTR_ERR(udc->vbus_gpiod); + goto err_int; + } + if (udc->vbus_gpiod) { + gpiod_set_consumer_name(udc->vbus_gpiod, "udc vbus"); - irq = gpio_to_irq(udc_info->vbus_pin); + irq = gpiod_to_irq(udc->vbus_gpiod); if (irq < 0) { dev_err(dev, "no irq for gpio vbus pin\n"); retval = irq; @@ -1833,16 +1831,12 @@ static int s3c2410_udc_probe(struct platform_device *pdev) udc->vbus = 1; } - if (udc_info && !udc_info->udc_command && - gpio_is_valid(udc_info->pullup_pin)) { - - retval = gpio_request_one(udc_info->pullup_pin, - udc_info->vbus_pin_inverted ? - GPIOF_OUT_INIT_HIGH : GPIOF_OUT_INIT_LOW, - "udc pullup"); - if (retval) - goto err_vbus_irq; + udc->pullup_gpiod = gpiod_get_optional(dev, "pullup", GPIOD_OUT_LOW); + if (IS_ERR(udc->pullup_gpiod)) { + retval = PTR_ERR(udc->pullup_gpiod); + goto err_vbus_irq; } + gpiod_set_consumer_name(udc->pullup_gpiod, "udc pullup"); retval = usb_add_gadget_udc(&pdev->dev, &udc->gadget); if (retval) @@ -1856,15 +1850,10 @@ static int s3c2410_udc_probe(struct platform_device *pdev) return 0; err_add_udc: - if (udc_info && !udc_info->udc_command && - gpio_is_valid(udc_info->pullup_pin)) - gpio_free(udc_info->pullup_pin); err_vbus_irq: - if (udc_info && udc_info->vbus_pin > 0) - free_irq(gpio_to_irq(udc_info->vbus_pin), udc); + if (udc->vbus_gpiod) + free_irq(gpiod_to_irq(udc->vbus_gpiod), udc); err_gpio_claim: - if (udc_info && udc_info->vbus_pin > 0) - gpio_free(udc_info->vbus_pin); err_int: free_irq(irq_usbd, udc); err_udc_clk: @@ -1885,7 +1874,6 @@ err_usb_bus_clk: static int s3c2410_udc_remove(struct platform_device *pdev) { struct s3c2410_udc *udc = platform_get_drvdata(pdev); - unsigned int irq; dev_dbg(&pdev->dev, "%s()\n", __func__); @@ -1895,14 +1883,8 @@ static int s3c2410_udc_remove(struct platform_device *pdev) usb_del_gadget_udc(&udc->gadget); debugfs_remove(debugfs_lookup("registers", s3c2410_udc_debugfs_root)); - if (udc_info && !udc_info->udc_command && - gpio_is_valid(udc_info->pullup_pin)) - gpio_free(udc_info->pullup_pin); - - if (udc_info && udc_info->vbus_pin > 0) { - irq = gpio_to_irq(udc_info->vbus_pin); - free_irq(irq, udc); - } + if (udc->vbus_gpiod) + free_irq(gpiod_to_irq(udc->vbus_gpiod), udc); free_irq(irq_usbd, udc); @@ -1926,14 +1908,18 @@ static int s3c2410_udc_remove(struct platform_device *pdev) static int s3c2410_udc_suspend(struct platform_device *pdev, pm_message_t message) { - s3c2410_udc_command(S3C2410_UDC_P_DISABLE); + struct s3c2410_udc *udc = platform_get_drvdata(pdev); + + s3c2410_udc_command(udc, S3C2410_UDC_P_DISABLE); return 0; } static int s3c2410_udc_resume(struct platform_device *pdev) { - s3c2410_udc_command(S3C2410_UDC_P_ENABLE); + struct s3c2410_udc *udc = platform_get_drvdata(pdev); + + s3c2410_udc_command(udc, S3C2410_UDC_P_ENABLE); return 0; } diff --git a/drivers/usb/gadget/udc/s3c2410_udc.h b/drivers/usb/gadget/udc/s3c2410_udc.h index 135a5bff3c743f5c1b3a5e157fe2bf64fe9996ca..cdbf202e5ee8b81104677e5a7a628c3f533526bd 100644 --- a/drivers/usb/gadget/udc/s3c2410_udc.h +++ b/drivers/usb/gadget/udc/s3c2410_udc.h @@ -83,6 +83,9 @@ struct s3c2410_udc { u32 port_status; int ep0state; + struct gpio_desc *vbus_gpiod; + struct gpio_desc *pullup_gpiod; + unsigned got_irq : 1; unsigned req_std : 1; diff --git a/drivers/usb/gadget/udc/tegra-xudc.c b/drivers/usb/gadget/udc/tegra-xudc.c index 3c37effdfa643c66ef0360794a97c8dcbf409f47..76919d7570d238f81d80064950131237655fc850 100644 --- a/drivers/usb/gadget/udc/tegra-xudc.c +++ b/drivers/usb/gadget/udc/tegra-xudc.c @@ -2,7 +2,7 @@ /* * NVIDIA Tegra XUSB device mode controller * - * Copyright (c) 2013-2019, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2013-2022, NVIDIA CORPORATION. All rights reserved. * Copyright (c) 2015, Google Inc. */ @@ -702,6 +702,8 @@ static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc) pm_runtime_get_sync(xudc->dev); + tegra_phy_xusb_utmi_pad_power_on(xudc->curr_utmi_phy); + err = phy_power_on(xudc->curr_utmi_phy); if (err < 0) dev_err(xudc->dev, "UTMI power on failed: %d\n", err); @@ -756,6 +758,8 @@ static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc) /* Make sure interrupt handler has completed before powergating. */ synchronize_irq(xudc->irq); + tegra_phy_xusb_utmi_pad_power_down(xudc->curr_utmi_phy); + err = phy_power_off(xudc->curr_utmi_phy); if (err < 0) dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err); diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index fd9264cf6c875c79b1901676742b323a2af0653e..247568bc17a2b7a7c828cb4f795463454634b6ad 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -205,12 +205,12 @@ config USB_EHCI_FSL Variation of ARC USB block used in some Freescale chips. config USB_EHCI_HCD_NPCM7XX - tristate "Support for Nuvoton NPCM7XX on-chip EHCI USB controller" - depends on (USB_EHCI_HCD && ARCH_NPCM7XX) || COMPILE_TEST - default y if (USB_EHCI_HCD && ARCH_NPCM7XX) + tristate "Support for Nuvoton NPCM on-chip EHCI USB controller" + depends on (USB_EHCI_HCD && ARCH_NPCM) || COMPILE_TEST + default y if (USB_EHCI_HCD && ARCH_NPCM) help Enables support for the on-chip EHCI controller on - Nuvoton NPCM7XX chips. + Nuvoton NPCM chips. config USB_EHCI_HCD_OMAP tristate "EHCI support for OMAP3 and later chips" diff --git a/drivers/usb/host/ehci-atmel.c b/drivers/usb/host/ehci-atmel.c index 05d41fd65f254838ef7c917e498f61a056aefb5c..8b775e7bab066541f192f658ee81c57b53997829 100644 --- a/drivers/usb/host/ehci-atmel.c +++ b/drivers/usb/host/ehci-atmel.c @@ -25,8 +25,6 @@ #define DRIVER_DESC "EHCI Atmel driver" -static const char hcd_name[] = "ehci-atmel"; - #define EHCI_INSNREG(index) ((index) * 4 + 0x90) #define EHCI_INSNREG08_HSIC_EN BIT(2) @@ -239,7 +237,6 @@ static int __init ehci_atmel_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); ehci_init_driver(&ehci_atmel_hc_driver, &ehci_atmel_drv_overrides); return platform_driver_register(&ehci_atmel_driver); } diff --git a/drivers/usb/host/ehci-exynos.c b/drivers/usb/host/ehci-exynos.c index 1a9b7572e17fe681056f054ac764a98ba31919eb..a333231616f437b80d8ba04f0f10dbc48d589bc8 100644 --- a/drivers/usb/host/ehci-exynos.c +++ b/drivers/usb/host/ehci-exynos.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -32,7 +32,6 @@ (EHCI_INSNREG00_ENA_INCR16 | EHCI_INSNREG00_ENA_INCR8 | \ EHCI_INSNREG00_ENA_INCR4 | EHCI_INSNREG00_ENA_INCRX_ALIGN) -static const char hcd_name[] = "ehci-exynos"; static struct hc_driver __read_mostly exynos_ehci_hc_driver; #define PHY_NUMBER 3 @@ -132,20 +131,13 @@ static void exynos_ehci_phy_disable(struct device *dev) static void exynos_setup_vbus_gpio(struct device *dev) { + struct gpio_desc *gpio; int err; - int gpio; - if (!dev->of_node) - return; - - gpio = of_get_named_gpio(dev->of_node, "samsung,vbus-gpio", 0); - if (!gpio_is_valid(gpio)) - return; - - err = devm_gpio_request_one(dev, gpio, GPIOF_OUT_INIT_HIGH, - "ehci_vbus_gpio"); + gpio = devm_gpiod_get_optional(dev, "samsung,vbus", GPIOD_OUT_HIGH); + err = PTR_ERR_OR_ZERO(gpio); if (err) - dev_err(dev, "can't request ehci vbus gpio %d", gpio); + dev_err(dev, "can't request ehci vbus gpio: %d\n", err); } static int exynos_ehci_probe(struct platform_device *pdev) @@ -347,7 +339,6 @@ static int __init ehci_exynos_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); ehci_init_driver(&exynos_ehci_hc_driver, &exynos_overrides); return platform_driver_register(&exynos_ehci_driver); } diff --git a/drivers/usb/host/ehci-fsl.c b/drivers/usb/host/ehci-fsl.c index 896c0d107f725d8ecd5d2d525b2b643d1c192a44..9cea785934e596434a0ff380667f29771913168a 100644 --- a/drivers/usb/host/ehci-fsl.c +++ b/drivers/usb/host/ehci-fsl.c @@ -722,8 +722,6 @@ static int __init ehci_fsl_init(void) if (usb_disabled()) return -ENODEV; - pr_info(DRV_NAME ": " DRIVER_DESC "\n"); - ehci_init_driver(&fsl_ehci_hc_driver, &ehci_fsl_overrides); fsl_ehci_hc_driver.product_desc = diff --git a/drivers/usb/host/ehci-hcd.c b/drivers/usb/host/ehci-hcd.c index 684164fa97169d3da8f8c50e65c0cc86b089797b..a1930db0da1c3c82af1e6becd92ac8f34f5efe48 100644 --- a/drivers/usb/host/ehci-hcd.c +++ b/drivers/usb/host/ehci-hcd.c @@ -1351,7 +1351,6 @@ static int __init ehci_hcd_init(void) if (usb_disabled()) return -ENODEV; - printk(KERN_INFO "%s: " DRIVER_DESC "\n", hcd_name); set_bit(USB_EHCI_LOADED, &usb_hcds_loaded); if (test_bit(USB_UHCI_LOADED, &usb_hcds_loaded) || test_bit(USB_OHCI_LOADED, &usb_hcds_loaded)) diff --git a/drivers/usb/host/ehci-npcm7xx.c b/drivers/usb/host/ehci-npcm7xx.c index 6b5a7a873e015191c58eb825835e27af15beab69..63af1a827fcbc2cb34041aa2bad95a38057afac4 100644 --- a/drivers/usb/host/ehci-npcm7xx.c +++ b/drivers/usb/host/ehci-npcm7xx.c @@ -22,19 +22,8 @@ #include "ehci.h" -#include -#include - #define DRIVER_DESC "EHCI npcm7xx driver" -static const char hcd_name[] = "npcm7xx-ehci"; - -#define USB2PHYCTL_OFFSET 0x144 - -#define IPSRST2_OFFSET 0x24 -#define IPSRST3_OFFSET 0x34 - - static struct hc_driver __read_mostly ehci_npcm7xx_hc_driver; static int __maybe_unused ehci_npcm7xx_drv_suspend(struct device *dev) @@ -60,52 +49,12 @@ static int npcm7xx_ehci_hcd_drv_probe(struct platform_device *pdev) { struct usb_hcd *hcd; struct resource *res; - struct regmap *gcr_regmap; - struct regmap *rst_regmap; const struct hc_driver *driver = &ehci_npcm7xx_hc_driver; int irq; int retval; dev_dbg(&pdev->dev, "initializing npcm7xx ehci USB Controller\n"); - gcr_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-gcr"); - if (IS_ERR(gcr_regmap)) { - dev_err(&pdev->dev, "%s: failed to find nuvoton,npcm750-gcr\n", - __func__); - return PTR_ERR(gcr_regmap); - } - - rst_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-rst"); - if (IS_ERR(rst_regmap)) { - dev_err(&pdev->dev, "%s: failed to find nuvoton,npcm750-rst\n", - __func__); - return PTR_ERR(rst_regmap); - } - - /********* phy init ******/ - // reset usb host - regmap_update_bits(rst_regmap, IPSRST2_OFFSET, - (0x1 << 26), (0x1 << 26)); - regmap_update_bits(rst_regmap, IPSRST3_OFFSET, - (0x1 << 25), (0x1 << 25)); - regmap_update_bits(gcr_regmap, USB2PHYCTL_OFFSET, - (0x1 << 28), 0); - - udelay(1); - - // enable phy - regmap_update_bits(rst_regmap, IPSRST3_OFFSET, - (0x1 << 25), 0); - - udelay(50); // enable phy - - regmap_update_bits(gcr_regmap, USB2PHYCTL_OFFSET, - (0x1 << 28), (0x1 << 28)); - - // enable host - regmap_update_bits(rst_regmap, IPSRST2_OFFSET, - (0x1 << 26), 0); - if (usb_disabled()) return -ENODEV; @@ -191,8 +140,6 @@ static int __init ehci_npcm7xx_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_npcm7xx_hc_driver, NULL); return platform_driver_register(&npcm7xx_ehci_hcd_driver); } diff --git a/drivers/usb/host/ehci-omap.c b/drivers/usb/host/ehci-omap.c index 8c45bc17a580eedf334bbd3878b89636c2001e93..7dd984722a7f7daeafb66504475bf3b91d8ff6ed 100644 --- a/drivers/usb/host/ehci-omap.c +++ b/drivers/usb/host/ehci-omap.c @@ -284,8 +284,6 @@ static int __init ehci_omap_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_omap_hc_driver, &ehci_omap_overrides); return platform_driver_register(&ehci_hcd_omap_driver); } diff --git a/drivers/usb/host/ehci-orion.c b/drivers/usb/host/ehci-orion.c index 3626758b3e2aa1ccd27881509e97b2b1ff67214a..a3454a3ea4e0006f74d0e8ee5357895eb6cd4d75 100644 --- a/drivers/usb/host/ehci-orion.c +++ b/drivers/usb/host/ehci-orion.c @@ -65,8 +65,6 @@ struct orion_ehci_hcd { struct phy *phy; }; -static const char hcd_name[] = "ehci-orion"; - static struct hc_driver __read_mostly ehci_orion_hc_driver; /* @@ -361,8 +359,6 @@ static int __init ehci_orion_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_orion_hc_driver, &orion_overrides); return platform_driver_register(&ehci_orion_driver); } diff --git a/drivers/usb/host/ehci-pci.c b/drivers/usb/host/ehci-pci.c index 9937c5a7efc2d139cf759355fa0530f0dfc0b180..17f8b6ea0c356da98441e76f35bb0c7270ee4d0e 100644 --- a/drivers/usb/host/ehci-pci.c +++ b/drivers/usb/host/ehci-pci.c @@ -382,7 +382,7 @@ static int ehci_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { if (is_bypassed_id(pdev)) return -ENODEV; - return usb_hcd_pci_probe(pdev, id, &ehci_pci_hc_driver); + return usb_hcd_pci_probe(pdev, &ehci_pci_hc_driver); } static void ehci_pci_remove(struct pci_dev *pdev) @@ -423,8 +423,6 @@ static int __init ehci_pci_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_pci_hc_driver, &pci_overrides); /* Entries for the PCI suspend/resume callbacks are special */ diff --git a/drivers/usb/host/ehci-platform.c b/drivers/usb/host/ehci-platform.c index 6924f0316e9a8375456d0401b2b3fa0299ff2aad..fe497c876d76899bfde7535bc40447d0e4625d90 100644 --- a/drivers/usb/host/ehci-platform.c +++ b/drivers/usb/host/ehci-platform.c @@ -53,8 +53,6 @@ struct ehci_platform_priv { struct delayed_work poll_work; }; -static const char hcd_name[] = "ehci-platform"; - static int ehci_platform_reset(struct usb_hcd *hcd) { struct platform_device *pdev = to_platform_device(hcd->self.controller); @@ -529,8 +527,6 @@ static int __init ehci_platform_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_platform_hc_driver, &platform_overrides); return platform_driver_register(&ehci_platform_driver); } diff --git a/drivers/usb/host/ehci-q.c b/drivers/usb/host/ehci-q.c index 807e64991e3eabdf3441a7f4a6dbc9170f4403bc..666f5c4db25a866115f0eb6f646471d1dec66596 100644 --- a/drivers/usb/host/ehci-q.c +++ b/drivers/usb/host/ehci-q.c @@ -645,7 +645,7 @@ qh_urb_transaction ( token |= (1 /* "in" */ << 8); /* else it's already initted to "out" pid (0 << 8) */ - maxpacket = usb_maxpacket(urb->dev, urb->pipe); + maxpacket = usb_endpoint_maxp(&urb->ep->desc); /* * buffer gets wrapped in one or more qtds; @@ -1218,7 +1218,7 @@ static int ehci_submit_single_step_set_feature( token |= (1 /* "in" */ << 8); /*This is IN stage*/ - maxpacket = usb_maxpacket(urb->dev, urb->pipe); + maxpacket = usb_endpoint_maxp(&urb->ep->desc); qtd_fill(ehci, qtd, buf, len, token, maxpacket); diff --git a/drivers/usb/host/ehci-spear.c b/drivers/usb/host/ehci-spear.c index 3694e450a11a1be32ba3389219fd4c6c0b4d3ec6..c4ddd1022f60c2576718d04c62fdb7b780a4e01e 100644 --- a/drivers/usb/host/ehci-spear.c +++ b/drivers/usb/host/ehci-spear.c @@ -24,8 +24,6 @@ #define DRIVER_DESC "EHCI SPEAr driver" -static const char hcd_name[] = "SPEAr-ehci"; - struct spear_ehci { struct clk *clk; }; @@ -167,8 +165,6 @@ static int __init ehci_spear_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_spear_hc_driver, &spear_overrides); return platform_driver_register(&spear_ehci_hcd_driver); } diff --git a/drivers/usb/host/ehci-st.c b/drivers/usb/host/ehci-st.c index f74433aac94860e70bdf8fb705593f1a6b67dde0..f731dc98c5331ab7ac2e0338787598e1d20d3d0d 100644 --- a/drivers/usb/host/ehci-st.c +++ b/drivers/usb/host/ehci-st.c @@ -42,8 +42,6 @@ struct st_ehci_platform_priv { #define hcd_to_ehci_priv(h) \ ((struct st_ehci_platform_priv *)hcd_to_ehci(h)->priv) -static const char hcd_name[] = "ehci-st"; - #define EHCI_CAPS_SIZE 0x10 #define AHB2STBUS_INSREG01 (EHCI_CAPS_SIZE + 0x84) @@ -346,8 +344,6 @@ static int __init ehci_platform_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ehci_init_driver(&ehci_platform_hc_driver, &platform_overrides); return platform_driver_register(&ehci_platform_driver); } diff --git a/drivers/usb/host/fhci-hcd.c b/drivers/usb/host/fhci-hcd.c index 2ba09c3fbc2f2d5a74879771f1e95aff57b0f5ec..95a44462bed04bb55066ab363cda28aed389e745 100644 --- a/drivers/usb/host/fhci-hcd.c +++ b/drivers/usb/host/fhci-hcd.c @@ -25,8 +25,8 @@ #include #include #include -#include #include +#include #include #include #include "fhci.h" @@ -150,15 +150,15 @@ int fhci_ioports_check_bus_state(struct fhci_hcd *fhci) u8 bits = 0; /* check USBOE,if transmitting,exit */ - if (!gpio_get_value(fhci->gpios[GPIO_USBOE])) + if (!gpiod_get_value(fhci->gpiods[GPIO_USBOE])) return -1; /* check USBRP */ - if (gpio_get_value(fhci->gpios[GPIO_USBRP])) + if (gpiod_get_value(fhci->gpiods[GPIO_USBRP])) bits |= 0x2; /* check USBRN */ - if (gpio_get_value(fhci->gpios[GPIO_USBRN])) + if (gpiod_get_value(fhci->gpiods[GPIO_USBRN])) bits |= 0x1; return bits; @@ -630,40 +630,23 @@ static int of_fhci_probe(struct platform_device *ofdev) /* GPIOs and pins */ for (i = 0; i < NUM_GPIOS; i++) { - int gpio; - enum of_gpio_flags flags; - - gpio = of_get_gpio_flags(node, i, &flags); - fhci->gpios[i] = gpio; - fhci->alow_gpios[i] = flags & OF_GPIO_ACTIVE_LOW; - - if (!gpio_is_valid(gpio)) { - if (i < GPIO_SPEED) { - dev_err(dev, "incorrect GPIO%d: %d\n", - i, gpio); - goto err_gpios; - } else { - dev_info(dev, "assuming board doesn't have " - "%s gpio\n", i == GPIO_SPEED ? - "speed" : "power"); - continue; - } - } + if (i < GPIO_SPEED) + fhci->gpiods[i] = devm_gpiod_get_index(dev, + NULL, i, GPIOD_IN); + + else + fhci->gpiods[i] = devm_gpiod_get_index_optional(dev, + NULL, i, GPIOD_OUT_LOW); - ret = gpio_request(gpio, dev_name(dev)); - if (ret) { - dev_err(dev, "failed to request gpio %d", i); + if (IS_ERR(fhci->gpiods[i])) { + dev_err(dev, "incorrect GPIO%d: %ld\n", + i, PTR_ERR(fhci->gpiods[i])); goto err_gpios; } - - if (i >= GPIO_SPEED) { - ret = gpio_direction_output(gpio, 0); - if (ret) { - dev_err(dev, "failed to set gpio %d as " - "an output\n", i); - i++; - goto err_gpios; - } + if (!fhci->gpiods[i]) { + dev_info(dev, "assuming board doesn't have " + "%s gpio\n", i == GPIO_SPEED ? + "speed" : "power"); } } @@ -766,10 +749,6 @@ err_pins: while (--j >= 0) qe_pin_free(fhci->pins[j]); err_gpios: - while (--i >= 0) { - if (gpio_is_valid(fhci->gpios[i])) - gpio_free(fhci->gpios[i]); - } cpm_muram_free(pram_addr); err_pram: iounmap(hcd->regs); @@ -782,18 +761,12 @@ static int fhci_remove(struct device *dev) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct fhci_hcd *fhci = hcd_to_fhci(hcd); - int i; int j; usb_remove_hcd(hcd); free_irq(fhci->timer->irq, hcd); gtm_put_timer16(fhci->timer); cpm_muram_free(cpm_muram_offset(fhci->pram)); - for (i = 0; i < NUM_GPIOS; i++) { - if (!gpio_is_valid(fhci->gpios[i])) - continue; - gpio_free(fhci->gpios[i]); - } for (j = 0; j < NUM_PINS; j++) qe_pin_free(fhci->pins[j]); fhci_dfs_destroy(fhci); diff --git a/drivers/usb/host/fhci-hub.c b/drivers/usb/host/fhci-hub.c index c359dcdb9b13787d20ebead7d42ff479ee49ce00..5f48660ebdfa299d0c51a8b1f3aa1303f3b69d18 100644 --- a/drivers/usb/host/fhci-hub.c +++ b/drivers/usb/host/fhci-hub.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include "fhci.h" @@ -38,13 +38,12 @@ static u8 root_hub_des[] = { static void fhci_gpio_set_value(struct fhci_hcd *fhci, int gpio_nr, bool on) { - int gpio = fhci->gpios[gpio_nr]; - bool alow = fhci->alow_gpios[gpio_nr]; + struct gpio_desc *gpiod = fhci->gpiods[gpio_nr]; - if (!gpio_is_valid(gpio)) + if (!gpiod) return; - gpio_set_value(gpio, on ^ alow); + gpiod_set_value(gpiod, on); mdelay(5); } @@ -129,9 +128,9 @@ void fhci_io_port_generate_reset(struct fhci_hcd *fhci) { fhci_dbg(fhci, "-> %s\n", __func__); - gpio_direction_output(fhci->gpios[GPIO_USBOE], 0); - gpio_direction_output(fhci->gpios[GPIO_USBTP], 0); - gpio_direction_output(fhci->gpios[GPIO_USBTN], 0); + gpiod_direction_output(fhci->gpiods[GPIO_USBOE], 0); + gpiod_direction_output(fhci->gpiods[GPIO_USBTP], 0); + gpiod_direction_output(fhci->gpiods[GPIO_USBTN], 0); mdelay(5); diff --git a/drivers/usb/host/fhci.h b/drivers/usb/host/fhci.h index 81fbc019a9b3e979d7f441f903cd08d2a75c56a8..1f57b0989485a59f2a13bbd0c8a7828556d44616 100644 --- a/drivers/usb/host/fhci.h +++ b/drivers/usb/host/fhci.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -242,8 +243,7 @@ struct fhci_hcd { enum qe_clock fullspeed_clk; enum qe_clock lowspeed_clk; struct qe_pin *pins[NUM_PINS]; - int gpios[NUM_GPIOS]; - bool alow_gpios[NUM_GPIOS]; + struct gpio_desc *gpiods[NUM_GPIOS]; struct qe_usb_ctlr __iomem *regs; /* I/O memory used to communicate */ struct fhci_pram __iomem *pram; /* Parameter RAM */ diff --git a/drivers/usb/host/fotg210-hcd.c b/drivers/usb/host/fotg210-hcd.c index f8c111e08a0d5fc11e57dbc55a3022c012b4a68d..3d1dbcf4c07326ff39a09fe1b25f7a9363e4bfc8 100644 --- a/drivers/usb/host/fotg210-hcd.c +++ b/drivers/usb/host/fotg210-hcd.c @@ -5692,7 +5692,6 @@ static int __init fotg210_hcd_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); set_bit(USB_EHCI_LOADED, &usb_hcds_loaded); if (test_bit(USB_UHCI_LOADED, &usb_hcds_loaded) || test_bit(USB_OHCI_LOADED, &usb_hcds_loaded)) diff --git a/drivers/usb/host/ohci-at91.c b/drivers/usb/host/ohci-at91.c index 98326465e2dc25a4abb3fc4d73c8955507a3bc05..533537ef3c21db8a7c54d4116b2c8cb2dcd2d599 100644 --- a/drivers/usb/host/ohci-at91.c +++ b/drivers/usb/host/ohci-at91.c @@ -62,8 +62,6 @@ struct ohci_at91_priv { #define DRIVER_DESC "OHCI Atmel driver" -static const char hcd_name[] = "ohci-atmel"; - static struct hc_driver __read_mostly ohci_at91_hc_driver; static const struct ohci_driver_overrides ohci_at91_drv_overrides __initconst = { @@ -699,7 +697,6 @@ static int __init ohci_at91_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); ohci_init_driver(&ohci_at91_hc_driver, &ohci_at91_drv_overrides); /* diff --git a/drivers/usb/host/ohci-da8xx.c b/drivers/usb/host/ohci-da8xx.c index 1371b0c249ece0d4985222e74f91ea6caf637128..d4818e8d652b2eb8d870c741e7a28b3c934749d2 100644 --- a/drivers/usb/host/ohci-da8xx.c +++ b/drivers/usb/host/ohci-da8xx.c @@ -551,7 +551,6 @@ static int __init ohci_da8xx_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", DRV_NAME); ohci_init_driver(&ohci_da8xx_hc_driver, &da8xx_overrides); /* diff --git a/drivers/usb/host/ohci-exynos.c b/drivers/usb/host/ohci-exynos.c index 5f5e8a64c8e2e01761fbf710c298d6e2fe2cc831..8d7977fd5d3bd502960b6cc81b9ad227a5389f1e 100644 --- a/drivers/usb/host/ohci-exynos.c +++ b/drivers/usb/host/ohci-exynos.c @@ -21,7 +21,6 @@ #define DRIVER_DESC "OHCI Exynos driver" -static const char hcd_name[] = "ohci-exynos"; static struct hc_driver __read_mostly exynos_ohci_hc_driver; #define to_exynos_ohci(hcd) (struct exynos_ohci_hcd *)(hcd_to_ohci(hcd)->priv) @@ -310,7 +309,6 @@ static int __init ohci_exynos_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); ohci_init_driver(&exynos_ohci_hc_driver, &exynos_overrides); return platform_driver_register(&exynos_ohci_driver); } diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index c4c821c2288c262d88aa2864bb6a252937011a5f..0457dd9f6c19ab7985524ead90bc81294531e4d5 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -1276,7 +1276,6 @@ static int __init ohci_hcd_mod_init(void) if (usb_disabled()) return -ENODEV; - printk(KERN_INFO "%s: " DRIVER_DESC "\n", hcd_name); pr_debug ("%s: block sizes: ed %zd td %zd\n", hcd_name, sizeof (struct ed), sizeof (struct td)); set_bit(USB_OHCI_LOADED, &usb_hcds_loaded); diff --git a/drivers/usb/host/ohci-nxp.c b/drivers/usb/host/ohci-nxp.c index 106a6bcefb0871991099c0e229e8b379151ae0d0..5b32e683e367d3dd10aa95350d6f136962c9f0d3 100644 --- a/drivers/usb/host/ohci-nxp.c +++ b/drivers/usb/host/ohci-nxp.c @@ -275,8 +275,6 @@ static int __init ohci_nxp_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_nxp_hc_driver, NULL); return platform_driver_register(&ohci_hcd_nxp_driver); } diff --git a/drivers/usb/host/ohci-omap.c b/drivers/usb/host/ohci-omap.c index f5bc9c8bdc9a3edc24b962e777cbb7352f92437f..cb29701df911c5fe29d77f5838e2de6203aec72b 100644 --- a/drivers/usb/host/ohci-omap.c +++ b/drivers/usb/host/ohci-omap.c @@ -423,8 +423,6 @@ static int __init ohci_omap_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_omap_hc_driver, &omap_overrides); return platform_driver_register(&ohci_hcd_omap_driver); } diff --git a/drivers/usb/host/ohci-pci.c b/drivers/usb/host/ohci-pci.c index 41efe927d8f3f5ec4cfcdd7fd20e43fc09408b2f..d7b4f40f9ff4eb82178020573011b2b64ab9c0bd 100644 --- a/drivers/usb/host/ohci-pci.c +++ b/drivers/usb/host/ohci-pci.c @@ -282,7 +282,7 @@ MODULE_DEVICE_TABLE (pci, pci_ids); static int ohci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { - return usb_hcd_pci_probe(dev, id, &ohci_pci_hc_driver); + return usb_hcd_pci_probe(dev, &ohci_pci_hc_driver); } /* pci driver glue; this is a "new style" PCI driver module */ @@ -306,8 +306,6 @@ static int __init ohci_pci_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_pci_hc_driver, &pci_overrides); #ifdef CONFIG_PM diff --git a/drivers/usb/host/ohci-platform.c b/drivers/usb/host/ohci-platform.c index 0adae626512766e0f32cb7c9c61e2bace05df5a7..a84305091c434874a4e3d3555fbfe40cc91b60f3 100644 --- a/drivers/usb/host/ohci-platform.c +++ b/drivers/usb/host/ohci-platform.c @@ -41,8 +41,6 @@ struct ohci_platform_priv { struct reset_control *resets; }; -static const char hcd_name[] = "ohci-platform"; - static int ohci_platform_power_on(struct platform_device *dev) { struct usb_hcd *hcd = platform_get_drvdata(dev); @@ -289,7 +287,7 @@ static int ohci_platform_suspend(struct device *dev) return ret; } -static int ohci_platform_resume(struct device *dev) +static int ohci_platform_resume_common(struct device *dev, bool hibernated) { struct usb_hcd *hcd = dev_get_drvdata(dev); struct usb_ohci_pdata *pdata = dev_get_platdata(dev); @@ -301,7 +299,7 @@ static int ohci_platform_resume(struct device *dev) return err; } - ohci_resume(hcd, false); + ohci_resume(hcd, hibernated); pm_runtime_disable(dev); pm_runtime_set_active(dev); @@ -309,6 +307,16 @@ static int ohci_platform_resume(struct device *dev) return 0; } + +static int ohci_platform_resume(struct device *dev) +{ + return ohci_platform_resume_common(dev, false); +} + +static int ohci_platform_restore(struct device *dev) +{ + return ohci_platform_resume_common(dev, true); +} #endif /* CONFIG_PM_SLEEP */ static const struct of_device_id ohci_platform_ids[] = { @@ -325,8 +333,16 @@ static const struct platform_device_id ohci_platform_table[] = { }; MODULE_DEVICE_TABLE(platform, ohci_platform_table); -static SIMPLE_DEV_PM_OPS(ohci_platform_pm_ops, ohci_platform_suspend, - ohci_platform_resume); +#ifdef CONFIG_PM_SLEEP +static const struct dev_pm_ops ohci_platform_pm_ops = { + .suspend = ohci_platform_suspend, + .resume = ohci_platform_resume, + .freeze = ohci_platform_suspend, + .thaw = ohci_platform_resume, + .poweroff = ohci_platform_suspend, + .restore = ohci_platform_restore, +}; +#endif static struct platform_driver ohci_platform_driver = { .id_table = ohci_platform_table, @@ -335,7 +351,9 @@ static struct platform_driver ohci_platform_driver = { .shutdown = usb_hcd_platform_shutdown, .driver = { .name = "ohci-platform", +#ifdef CONFIG_PM_SLEEP .pm = &ohci_platform_pm_ops, +#endif .of_match_table = ohci_platform_ids, .probe_type = PROBE_PREFER_ASYNCHRONOUS, } @@ -346,8 +364,6 @@ static int __init ohci_platform_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_platform_hc_driver, &platform_overrides); return platform_driver_register(&ohci_platform_driver); } diff --git a/drivers/usb/host/ohci-pxa27x.c b/drivers/usb/host/ohci-pxa27x.c index ab4f610a0140f2bc87bd283c17344dac1d6f0130..a1dad8745622c003035cc1c3e85128855b4b0553 100644 --- a/drivers/usb/host/ohci-pxa27x.c +++ b/drivers/usb/host/ohci-pxa27x.c @@ -114,8 +114,6 @@ #define PXA_UHC_MAX_PORTNUM 3 -static const char hcd_name[] = "ohci-pxa27x"; - static struct hc_driver __read_mostly ohci_pxa27x_hc_driver; struct pxa27x_ohci { @@ -608,8 +606,6 @@ static int __init ohci_pxa27x_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_pxa27x_hc_driver, &pxa27x_overrides); ohci_pxa27x_hc_driver.hub_control = pxa27x_ohci_hub_control; diff --git a/drivers/usb/host/ohci-s3c2410.c b/drivers/usb/host/ohci-s3c2410.c index 12264c0486013fb22c32485a7b4ac92788ebd904..85a0a9ae0095402c4152f74800bd5f74fbdc1afc 100644 --- a/drivers/usb/host/ohci-s3c2410.c +++ b/drivers/usb/host/ohci-s3c2410.c @@ -39,8 +39,6 @@ #define DRIVER_DESC "OHCI S3C2410 driver" -static const char hcd_name[] = "ohci-s3c2410"; - static struct clk *clk; static struct clk *usb_clk; @@ -474,7 +472,6 @@ static int __init ohci_s3c2410_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); ohci_init_driver(&ohci_s3c2410_hc_driver, NULL); /* diff --git a/drivers/usb/host/ohci-spear.c b/drivers/usb/host/ohci-spear.c index 9b81f420656d6b8ec5b2fd3fbda9bdc8f8d44655..196951a27f3f27356e61f2c7fb3a8e181725e2d6 100644 --- a/drivers/usb/host/ohci-spear.c +++ b/drivers/usb/host/ohci-spear.c @@ -23,7 +23,6 @@ #define DRIVER_DESC "OHCI SPEAr driver" -static const char hcd_name[] = "SPEAr-ohci"; struct spear_ohci { struct clk *clk; }; @@ -179,8 +178,6 @@ static int __init ohci_spear_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_spear_hc_driver, &spear_overrides); return platform_driver_register(&spear_ohci_hcd_driver); } diff --git a/drivers/usb/host/ohci-st.c b/drivers/usb/host/ohci-st.c index ac796ccd93ef45e3a1b233b1cb53c4b49cb0dfa7..82eef3c62e11d28ae0afbdcac1cf9f398a09317f 100644 --- a/drivers/usb/host/ohci-st.c +++ b/drivers/usb/host/ohci-st.c @@ -40,8 +40,6 @@ struct st_ohci_platform_priv { #define hcd_to_ohci_priv(h) \ ((struct st_ohci_platform_priv *)hcd_to_ohci(h)->priv) -static const char hcd_name[] = "ohci-st"; - static int st_ohci_platform_power_on(struct platform_device *dev) { struct usb_hcd *hcd = platform_get_drvdata(dev); @@ -324,8 +322,6 @@ static int __init ohci_platform_init(void) if (usb_disabled()) return -ENODEV; - pr_info("%s: " DRIVER_DESC "\n", hcd_name); - ohci_init_driver(&ohci_platform_hc_driver, &platform_overrides); return platform_driver_register(&ohci_platform_driver); } diff --git a/drivers/usb/host/u132-hcd.c b/drivers/usb/host/u132-hcd.c index d879d6af57105a3df049d56056beb7f05e2843ef..95240c9c45bdbb2772cf12df6ff51d2102e554ff 100644 --- a/drivers/usb/host/u132-hcd.c +++ b/drivers/usb/host/u132-hcd.c @@ -3190,7 +3190,6 @@ static int __init u132_hcd_init(void) u132_exiting = 0; if (usb_disabled()) return -ENODEV; - printk(KERN_INFO "driver %s\n", hcd_name); workqueue = create_singlethread_workqueue("u132"); if (!workqueue) return -ENOMEM; diff --git a/drivers/usb/host/uhci-hcd.c b/drivers/usb/host/uhci-hcd.c index d90b869f5f409a801e3cab39304fd09219aaa6e8..c22b51af83fcb9d0fb8630d4181494750014af8e 100644 --- a/drivers/usb/host/uhci-hcd.c +++ b/drivers/usb/host/uhci-hcd.c @@ -867,8 +867,6 @@ static int __init uhci_hcd_init(void) if (usb_disabled()) return -ENODEV; - printk(KERN_INFO "uhci_hcd: " DRIVER_DESC "%s\n", - ignore_oc ? ", overcurrent ignored" : ""); set_bit(USB_UHCI_LOADED, &usb_hcds_loaded); #ifdef CONFIG_DYNAMIC_DEBUG diff --git a/drivers/usb/host/uhci-pci.c b/drivers/usb/host/uhci-pci.c index 9b88745d247f5d258b851323d56902e92c5be7e2..3592f757fe05ddfbce622b7bf91b93cc76b30ce2 100644 --- a/drivers/usb/host/uhci-pci.c +++ b/drivers/usb/host/uhci-pci.c @@ -294,7 +294,7 @@ MODULE_DEVICE_TABLE(pci, uhci_pci_ids); static int uhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { - return usb_hcd_pci_probe(dev, id, &uhci_driver); + return usb_hcd_pci_probe(dev, &uhci_driver); } static struct pci_driver uhci_pci_driver = { diff --git a/drivers/usb/host/xhci-dbgcap.c b/drivers/usb/host/xhci-dbgcap.c index e61155fa6379619eab9018a340eade99d59022e9..f1367b53b2600934d922d5127e0268547eee269b 100644 --- a/drivers/usb/host/xhci-dbgcap.c +++ b/drivers/usb/host/xhci-dbgcap.c @@ -988,7 +988,7 @@ xhci_alloc_dbc(struct device *dev, void __iomem *base, const struct dbc_driver * dbc->driver = driver; if (readl(&dbc->regs->control) & DBC_CTRL_DBC_ENABLE) - return NULL; + goto err; INIT_DELAYED_WORK(&dbc->event_work, xhci_dbc_handle_events); spin_lock_init(&dbc->lock); diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 0fdc014c940117e4d70237bd011c6756f01a2921..4619d5e89d5be1f0b5c3c52d05a29583ccf60e92 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -652,7 +652,7 @@ struct xhci_hub *xhci_get_rhub(struct usb_hcd *hcd) * It will release and re-aquire the lock while calling ACPI * method. */ -void xhci_set_port_power(struct xhci_hcd *xhci, struct usb_hcd *hcd, +static void xhci_set_port_power(struct xhci_hcd *xhci, struct usb_hcd *hcd, u16 index, bool on, unsigned long *flags) __must_hold(&xhci->lock) { @@ -1648,6 +1648,17 @@ int xhci_hub_status_data(struct usb_hcd *hcd, char *buf) status = bus_state->resuming_ports; + /* + * SS devices are only visible to roothub after link training completes. + * Keep polling roothubs for a grace period after xHC start + */ + if (xhci->run_graceperiod) { + if (time_before(jiffies, xhci->run_graceperiod)) + status = 1; + else + xhci->run_graceperiod = 0; + } + mask = PORT_CSC | PORT_PEC | PORT_OCC | PORT_PLC | PORT_WRC | PORT_CEC; /* For each port, did anything change? If so, set that bit in buf. */ diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 8c19e151a945471a461d5074b0b76800f941b14e..9e56aa28efcd4879efe987255518e7d5ec6c018f 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -641,7 +641,7 @@ struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, num_stream_ctxs, &stream_info->ctx_array_dma, mem_flags); if (!stream_info->stream_ctx_array) - goto cleanup_ctx; + goto cleanup_ring_array; memset(stream_info->stream_ctx_array, 0, sizeof(struct xhci_stream_ctx)*num_stream_ctxs); @@ -702,6 +702,11 @@ cleanup_rings: } xhci_free_command(xhci, stream_info->free_streams_command); cleanup_ctx: + xhci_free_stream_ctx(xhci, + stream_info->num_stream_ctxs, + stream_info->stream_ctx_array, + stream_info->ctx_array_dma); +cleanup_ring_array: kfree(stream_info->stream_rings); cleanup_info: kfree(stream_info); diff --git a/drivers/usb/host/xhci-mtk-sch.c b/drivers/usb/host/xhci-mtk-sch.c index 06a6b19acaae6869f045197aca926d744c645d5b..579899eb24c15245f3f70120b9f4df1e2a1de547 100644 --- a/drivers/usb/host/xhci-mtk-sch.c +++ b/drivers/usb/host/xhci-mtk-sch.c @@ -425,7 +425,6 @@ static int check_fs_bus_bw(struct mu3h_sch_ep_info *sch_ep, int offset) static int check_sch_tt(struct mu3h_sch_ep_info *sch_ep, u32 offset) { - u32 extra_cs_count; u32 start_ss, last_ss; u32 start_cs, last_cs; @@ -461,18 +460,12 @@ static int check_sch_tt(struct mu3h_sch_ep_info *sch_ep, u32 offset) if (last_cs > 7) return -ESCH_CS_OVERFLOW; - if (sch_ep->ep_type == ISOC_IN_EP) - extra_cs_count = (last_cs == 7) ? 1 : 2; - else /* ep_type : INTR IN / INTR OUT */ - extra_cs_count = 1; - - cs_count += extra_cs_count; if (cs_count > 7) cs_count = 7; /* HW limit */ sch_ep->cs_count = cs_count; - /* one for ss, the other for idle */ - sch_ep->num_budget_microframes = cs_count + 2; + /* ss, idle are ignored */ + sch_ep->num_budget_microframes = cs_count; /* * if interval=1, maxp >752, num_budge_micoframe is larger @@ -771,8 +764,8 @@ int xhci_mtk_drop_ep(struct usb_hcd *hcd, struct usb_device *udev, if (ret) return ret; - if (ep->hcpriv) - drop_ep_quirk(hcd, udev, ep); + /* needn't check @ep->hcpriv, xhci_endpoint_disable set it NULL */ + drop_ep_quirk(hcd, udev, ep); return 0; } diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index dce6c0ec8d340370e6bbd1626b9240b53cc771ba..40228a3d77a0b780df0f2ba14bcedab62870a222 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -431,7 +431,7 @@ static int xhci_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) * to say USB 2.0, but I'm not sure what the implications would be in * the other parts of the HCD code. */ - retval = usb_hcd_pci_probe(dev, id, &xhci_pci_hc_driver); + retval = usb_hcd_pci_probe(dev, &xhci_pci_hc_driver); if (retval) goto put_runtime_pm; diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index 044855818cb11c0e142576453faf16cb12c64656..5fb55bf19493150f55eb02c4727a99a3999462f8 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -123,7 +123,7 @@ static const struct xhci_plat_priv xhci_plat_renesas_rcar_gen3 = { }; static const struct xhci_plat_priv xhci_plat_brcm = { - .quirks = XHCI_RESET_ON_RESUME, + .quirks = XHCI_RESET_ON_RESUME | XHCI_SUSPEND_RESUME_CLKS, }; static const struct of_device_id usb_xhci_of_match[] = { @@ -398,12 +398,17 @@ static int xhci_plat_remove(struct platform_device *dev) pm_runtime_get_sync(&dev->dev); xhci->xhc_state |= XHCI_STATE_REMOVING; - usb_remove_hcd(shared_hcd); - xhci->shared_hcd = NULL; + if (shared_hcd) { + usb_remove_hcd(shared_hcd); + xhci->shared_hcd = NULL; + } + usb_phy_shutdown(hcd->usb_phy); usb_remove_hcd(hcd); - usb_put_hcd(shared_hcd); + + if (shared_hcd) + usb_put_hcd(shared_hcd); clk_disable_unprepare(clk); clk_disable_unprepare(reg_clk); @@ -432,7 +437,16 @@ static int __maybe_unused xhci_plat_suspend(struct device *dev) * xhci_suspend() needs `do_wakeup` to know whether host is allowed * to do wakeup during suspend. */ - return xhci_suspend(xhci, device_may_wakeup(dev)); + ret = xhci_suspend(xhci, device_may_wakeup(dev)); + if (ret) + return ret; + + if (!device_may_wakeup(dev) && (xhci->quirks & XHCI_SUSPEND_RESUME_CLKS)) { + clk_disable_unprepare(xhci->clk); + clk_disable_unprepare(xhci->reg_clk); + } + + return 0; } static int __maybe_unused xhci_plat_resume(struct device *dev) @@ -441,6 +455,11 @@ static int __maybe_unused xhci_plat_resume(struct device *dev) struct xhci_hcd *xhci = hcd_to_xhci(hcd); int ret; + if (!device_may_wakeup(dev) && (xhci->quirks & XHCI_SUSPEND_RESUME_CLKS)) { + clk_prepare_enable(xhci->clk); + clk_prepare_enable(xhci->reg_clk); + } + ret = xhci_priv_resume_quirk(hcd); if (ret) return ret; diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 65858f607437795bee3272f7ef680b8e39114f71..5176765c40131bb5666239a334bf10039798521d 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -151,9 +151,11 @@ int xhci_start(struct xhci_hcd *xhci) xhci_err(xhci, "Host took too long to start, " "waited %u microseconds.\n", XHCI_MAX_HALT_USEC); - if (!ret) + if (!ret) { /* clear state flags. Including dying, halted or removing */ xhci->xhc_state = 0; + xhci->run_graceperiod = jiffies + msecs_to_jiffies(500); + } return ret; } @@ -791,8 +793,6 @@ static void xhci_stop(struct usb_hcd *hcd) void xhci_shutdown(struct usb_hcd *hcd) { struct xhci_hcd *xhci = hcd_to_xhci(hcd); - unsigned long flags; - int i; if (xhci->quirks & XHCI_SPURIOUS_REBOOT) usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev)); @@ -808,21 +808,12 @@ void xhci_shutdown(struct usb_hcd *hcd) del_timer_sync(&xhci->shared_hcd->rh_timer); } - spin_lock_irqsave(&xhci->lock, flags); + spin_lock_irq(&xhci->lock); xhci_halt(xhci); - - /* Power off USB2 ports*/ - for (i = 0; i < xhci->usb2_rhub.num_ports; i++) - xhci_set_port_power(xhci, xhci->main_hcd, i, false, &flags); - - /* Power off USB3 ports*/ - for (i = 0; i < xhci->usb3_rhub.num_ports; i++) - xhci_set_port_power(xhci, xhci->shared_hcd, i, false, &flags); - /* Workaround for spurious wakeups at shutdown with HSW */ if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) xhci_reset(xhci, XHCI_RESET_SHORT_USEC); - spin_unlock_irqrestore(&xhci->lock, flags); + spin_unlock_irq(&xhci->lock); xhci_cleanup_msix(xhci); @@ -1192,7 +1183,8 @@ int xhci_resume(struct xhci_hcd *xhci, bool hibernated) /* re-initialize the HC on Restore Error, or Host Controller Error */ if (temp & (STS_SRE | STS_HCE)) { reinit_xhc = true; - xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp); + if (!xhci->broken_suspend) + xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp); } if (reinit_xhc) { @@ -1491,7 +1483,7 @@ EXPORT_SYMBOL_GPL(xhci_get_endpoint_index); /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint * address from the XHCI endpoint index. */ -unsigned int xhci_get_endpoint_address(unsigned int ep_index) +static unsigned int xhci_get_endpoint_address(unsigned int ep_index) { unsigned int number = DIV_ROUND_UP(ep_index, 2); unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN; @@ -4104,7 +4096,8 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev) slot_id = command->slot_id; if (!slot_id || command->status != COMP_SUCCESS) { - xhci_err(xhci, "Error while assigning device slot ID\n"); + xhci_err(xhci, "Error while assigning device slot ID: %s\n", + xhci_trb_comp_code_string(command->status)); xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n", HCS_MAX_SLOTS( readl(&xhci->cap_regs->hcs_params1))); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 1960b47acfb28cb570b24dcb13e195c823da286e..c0964fe8ac12fac16f8b3de71751f6ca185fe85f 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1807,8 +1807,6 @@ struct xhci_hcd { struct xhci_erst erst; /* Scratchpad */ struct xhci_scratchpad *scratchpad; - /* Store LPM test failed devices' information */ - struct list_head lpm_failed_devs; /* slot enabling and address device helpers */ /* these are not thread safe so use mutex */ @@ -1826,8 +1824,7 @@ struct xhci_hcd { /* Host controller watchdog timer structures */ unsigned int xhc_state; - - u32 command; + unsigned long run_graceperiod; struct s3_save s3; /* Host controller is dying - not responding to commands. "I'm not dead yet!" * @@ -1899,6 +1896,7 @@ struct xhci_hcd { #define XHCI_NO_SOFT_RETRY BIT_ULL(40) #define XHCI_BROKEN_D3COLD BIT_ULL(41) #define XHCI_EP_CTX_BROKEN_DCS BIT_ULL(42) +#define XHCI_SUSPEND_RESUME_CLKS BIT_ULL(43) unsigned int num_active_eps; unsigned int limit_active_eps; @@ -2041,7 +2039,6 @@ int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *ud void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, struct usb_device *udev); unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc); -unsigned int xhci_get_endpoint_address(unsigned int ep_index); unsigned int xhci_last_valid_endpoint(u32 added_ctxs); void xhci_endpoint_zero(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, struct usb_host_endpoint *ep); void xhci_update_tt_active_eps(struct xhci_hcd *xhci, @@ -2196,8 +2193,6 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, int xhci_hub_status_data(struct usb_hcd *hcd, char *buf); int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1); struct xhci_hub *xhci_get_rhub(struct usb_hcd *hcd); -void xhci_set_port_power(struct xhci_hcd *xhci, struct usb_hcd *hcd, u16 index, - bool on, unsigned long *flags); void xhci_hc_died(struct xhci_hcd *xhci); diff --git a/drivers/usb/misc/idmouse.c b/drivers/usb/misc/idmouse.c index e9437a176518a5f137a4df32d393b483e6d3b23c..ea39243efee396ea4ebb62c69f4bb1fabe9dad72 100644 --- a/drivers/usb/misc/idmouse.c +++ b/drivers/usb/misc/idmouse.c @@ -177,10 +177,6 @@ static int idmouse_create_image(struct usb_idmouse *dev) bytes_read += bulk_read; } - /* reset the device */ -reset: - ftip_command(dev, FTIP_RELEASE, 0, 0); - /* check for valid image */ /* right border should be black (0x00) */ for (bytes_read = sizeof(HEADER)-1 + WIDTH-1; bytes_read < IMGSIZE; bytes_read += WIDTH) @@ -192,6 +188,10 @@ reset: if (dev->bulk_in_buffer[bytes_read] != 0xFF) return -EAGAIN; + /* reset the device */ +reset: + ftip_command(dev, FTIP_RELEASE, 0, 0); + /* should be IMGSIZE == 65040 */ dev_dbg(&dev->interface->dev, "read %d bytes fingerprint data\n", bytes_read); diff --git a/drivers/usb/misc/onboard_usb_hub.c b/drivers/usb/misc/onboard_usb_hub.c index d1df153e7f5a6cd06e6240e4797e57b02d14990b..d63c63942af1b983ceddc9314c2a98a02d17d9fa 100644 --- a/drivers/usb/misc/onboard_usb_hub.c +++ b/drivers/usb/misc/onboard_usb_hub.c @@ -71,10 +71,7 @@ static int onboard_hub_power_off(struct onboard_hub *hub) { int err; - if (hub->reset_gpio) { - gpiod_set_value_cansleep(hub->reset_gpio, 1); - fsleep(hub->pdata->reset_us); - } + gpiod_set_value_cansleep(hub->reset_gpio, 1); err = regulator_disable(hub->vdd); if (err) { diff --git a/drivers/usb/misc/usb251xb.c b/drivers/usb/misc/usb251xb.c index 04c4e3fed09460968c4e9af6ccb283b19a37bbaa..54337d72bb9ff2c1c9e3f1ed901e659aeef4140f 100644 --- a/drivers/usb/misc/usb251xb.c +++ b/drivers/usb/misc/usb251xb.c @@ -400,7 +400,7 @@ static int usb251xb_get_ofdata(struct usb251xb *hub, { struct device *dev = hub->dev; struct device_node *np = dev->of_node; - int len, err; + int len; u32 property_u32 = 0; const char *cproperty_char; char str[USB251XB_STRING_BUFSIZE / 2]; @@ -416,13 +416,9 @@ static int usb251xb_get_ofdata(struct usb251xb *hub, hub->skip_config = 0; hub->gpio_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); - if (PTR_ERR(hub->gpio_reset) == -EPROBE_DEFER) { - return -EPROBE_DEFER; - } else if (IS_ERR(hub->gpio_reset)) { - err = PTR_ERR(hub->gpio_reset); - dev_err(dev, "unable to request GPIO reset pin (%d)\n", err); - return err; - } + if (IS_ERR(hub->gpio_reset)) + return dev_err_probe(dev, PTR_ERR(hub->gpio_reset), + "unable to request GPIO reset pin\n"); if (of_property_read_u16_array(np, "vendor-id", &hub->vendor_id, 1)) hub->vendor_id = USB251XB_DEF_VENDOR_ID; @@ -547,7 +543,7 @@ static int usb251xb_get_ofdata(struct usb251xb *hub, hub->boost_up = USB251XB_DEF_BOOST_UP; cproperty_char = of_get_property(np, "manufacturer", NULL); - strlcpy(str, cproperty_char ? : USB251XB_DEF_MANUFACTURER_STRING, + strscpy(str, cproperty_char ? : USB251XB_DEF_MANUFACTURER_STRING, sizeof(str)); hub->manufacturer_len = strlen(str) & 0xFF; memset(hub->manufacturer, 0, USB251XB_STRING_BUFSIZE); @@ -557,7 +553,7 @@ static int usb251xb_get_ofdata(struct usb251xb *hub, USB251XB_STRING_BUFSIZE); cproperty_char = of_get_property(np, "product", NULL); - strlcpy(str, cproperty_char ? : data->product_str, sizeof(str)); + strscpy(str, cproperty_char ? : data->product_str, sizeof(str)); hub->product_len = strlen(str) & 0xFF; memset(hub->product, 0, USB251XB_STRING_BUFSIZE); len = min_t(size_t, USB251XB_STRING_BUFSIZE / 2, strlen(str)); @@ -566,7 +562,7 @@ static int usb251xb_get_ofdata(struct usb251xb *hub, USB251XB_STRING_BUFSIZE); cproperty_char = of_get_property(np, "serial", NULL); - strlcpy(str, cproperty_char ? : USB251XB_DEF_SERIAL_STRING, + strscpy(str, cproperty_char ? : USB251XB_DEF_SERIAL_STRING, sizeof(str)); hub->serial_len = strlen(str) & 0xFF; memset(hub->serial, 0, USB251XB_STRING_BUFSIZE); diff --git a/drivers/usb/misc/usb3503.c b/drivers/usb/misc/usb3503.c index 330f494cd15867ba92bbae789acf38b7811e3def..c70ca475c7c7e139aa1413edec5d154e4ad7969a 100644 --- a/drivers/usb/misc/usb3503.c +++ b/drivers/usb/misc/usb3503.c @@ -160,6 +160,7 @@ static int usb3503_probe(struct usb3503 *hub) struct usb3503_platform_data *pdata = dev_get_platdata(dev); struct device_node *np = dev->of_node; int err; + bool is_clk_enabled = false; u32 mode = USB3503_MODE_HUB; const u32 *property; enum gpiod_flags flags; @@ -217,6 +218,7 @@ static int usb3503_probe(struct usb3503 *hub) return err; } + is_clk_enabled = true; property = of_get_property(np, "disabled-ports", &len); if (property && (len / sizeof(u32)) > 0) { int i; @@ -236,20 +238,26 @@ static int usb3503_probe(struct usb3503 *hub) else flags = GPIOD_OUT_HIGH; hub->intn = devm_gpiod_get_optional(dev, "intn", flags); - if (IS_ERR(hub->intn)) - return PTR_ERR(hub->intn); + if (IS_ERR(hub->intn)) { + err = PTR_ERR(hub->intn); + goto err_clk; + } if (hub->intn) gpiod_set_consumer_name(hub->intn, "usb3503 intn"); hub->connect = devm_gpiod_get_optional(dev, "connect", GPIOD_OUT_LOW); - if (IS_ERR(hub->connect)) - return PTR_ERR(hub->connect); + if (IS_ERR(hub->connect)) { + err = PTR_ERR(hub->connect); + goto err_clk; + } if (hub->connect) gpiod_set_consumer_name(hub->connect, "usb3503 connect"); hub->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); - if (IS_ERR(hub->reset)) - return PTR_ERR(hub->reset); + if (IS_ERR(hub->reset)) { + err = PTR_ERR(hub->reset); + goto err_clk; + } if (hub->reset) { /* Datasheet defines a hardware reset to be at least 100us */ usleep_range(100, 10000); @@ -265,6 +273,11 @@ static int usb3503_probe(struct usb3503 *hub) (hub->mode == USB3503_MODE_HUB) ? "hub" : "standby"); return 0; + +err_clk: + if (is_clk_enabled) + clk_disable_unprepare(hub->clk); + return err; } static int usb3503_i2c_probe(struct i2c_client *i2c, @@ -289,14 +302,12 @@ static int usb3503_i2c_probe(struct i2c_client *i2c, return usb3503_probe(hub); } -static int usb3503_i2c_remove(struct i2c_client *i2c) +static void usb3503_i2c_remove(struct i2c_client *i2c) { struct usb3503 *hub; hub = i2c_get_clientdata(i2c); clk_disable_unprepare(hub->clk); - - return 0; } static int usb3503_platform_probe(struct platform_device *pdev) diff --git a/drivers/usb/misc/uss720.c b/drivers/usb/misc/uss720.c index 0be8efcda15d5791aafd7cf48e81440afb3744b5..b00d92db5dfd1b952df5474d34c40bb6c61f33b8 100644 --- a/drivers/usb/misc/uss720.c +++ b/drivers/usb/misc/uss720.c @@ -502,7 +502,7 @@ static size_t parport_uss720_epp_write_data(struct parport *pp, const void *buf, #else struct parport_uss720_private *priv = pp->private_data; struct usb_device *usbdev = priv->usbdev; - int rlen; + int rlen = 0; int i; if (!usbdev) @@ -563,7 +563,7 @@ static size_t parport_uss720_ecp_write_data(struct parport *pp, const void *buff { struct parport_uss720_private *priv = pp->private_data; struct usb_device *usbdev = priv->usbdev; - int rlen; + int rlen = 0; int i; if (!usbdev) @@ -581,7 +581,7 @@ static size_t parport_uss720_ecp_read_data(struct parport *pp, void *buffer, siz { struct parport_uss720_private *priv = pp->private_data; struct usb_device *usbdev = priv->usbdev; - int rlen; + int rlen = 0; int i; if (!usbdev) @@ -614,7 +614,7 @@ static size_t parport_uss720_write_compat(struct parport *pp, const void *buffer { struct parport_uss720_private *priv = pp->private_data; struct usb_device *usbdev = priv->usbdev; - int rlen; + int rlen = 0; int i; if (!usbdev) diff --git a/drivers/usb/mon/mon_bin.c b/drivers/usb/mon/mon_bin.c index f48a23adbc35ddbbc66c5227d8c59e3413791b05..094e812e9e69223d3c7a866bb1f416ef73a9aeac 100644 --- a/drivers/usb/mon/mon_bin.c +++ b/drivers/usb/mon/mon_bin.c @@ -1268,6 +1268,11 @@ static int mon_bin_mmap(struct file *filp, struct vm_area_struct *vma) { /* don't do anything here: "fault" will set up page table entries */ vma->vm_ops = &mon_bin_vm_ops; + + if (vma->vm_flags & VM_WRITE) + return -EPERM; + + vma->vm_flags &= ~VM_MAYWRITE; vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; vma->vm_private_data = filp->private_data; mon_bin_vma_open(vma); diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c index 0ca173af87bb643c15202a924b50659730ddfee5..a3a6282893d091aaa90eebd984342ddbc234d324 100644 --- a/drivers/usb/mtu3/mtu3_core.c +++ b/drivers/usb/mtu3/mtu3_core.c @@ -978,8 +978,6 @@ int ssusb_gadget_init(struct ssusb_mtk *ssusb) goto irq_err; } - device_init_wakeup(dev, true); - /* power down device IP for power saving by default */ mtu3_stop(mtu); diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c index 4cb65346789d688e47c93df05a1d3c3cc1e89d4a..d78ae52b4e2615eef0d90f3862598b933c1052d9 100644 --- a/drivers/usb/mtu3/mtu3_plat.c +++ b/drivers/usb/mtu3/mtu3_plat.c @@ -356,6 +356,8 @@ static int mtu3_probe(struct platform_device *pdev) pm_runtime_enable(dev); pm_runtime_get_sync(dev); + device_init_wakeup(dev, true); + ret = ssusb_rscs_init(ssusb); if (ret) goto comm_init_err; diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig index f906dfd360d37c223a6d38f54740cdd8166c61ed..6c8f7763e75e49799a14d603e0582f3067dc2a98 100644 --- a/drivers/usb/musb/Kconfig +++ b/drivers/usb/musb/Kconfig @@ -86,7 +86,7 @@ config USB_MUSB_TUSB6010 tristate "TUSB6010" depends on HAS_IOMEM depends on ARCH_OMAP2PLUS || COMPILE_TEST - depends on NOP_USB_XCEIV = USB_MUSB_HDRC # both built-in or both modules + depends on NOP_USB_XCEIV!=m || USB_MUSB_HDRC=m config USB_MUSB_OMAP2PLUS tristate "OMAP2430 and onwards" diff --git a/drivers/usb/musb/da8xx.c b/drivers/usb/musb/da8xx.c index fd4ae2dd24e5044291d2d63a7de0f9b9d3048a0c..a4e55b0c52cffd467d7c5b4d354bcaf2c64a00e0 100644 --- a/drivers/usb/musb/da8xx.c +++ b/drivers/usb/musb/da8xx.c @@ -523,11 +523,9 @@ static int da8xx_probe(struct platform_device *pdev) } glue->phy = devm_phy_get(&pdev->dev, "usb-phy"); - if (IS_ERR(glue->phy)) { - if (PTR_ERR(glue->phy) != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to get phy\n"); - return PTR_ERR(glue->phy); - } + if (IS_ERR(glue->phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(glue->phy), + "failed to get phy\n"); glue->dev = &pdev->dev; glue->clk = clk; diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c index 417c30bff9cabe770634a1f779f4d17518bbd61a..d1e4e0deb7535f6ccb153c602c9f3760b65f6241 100644 --- a/drivers/usb/musb/jz4740.c +++ b/drivers/usb/musb/jz4740.c @@ -105,7 +105,6 @@ static int jz4740_musb_init(struct musb *musb) .driver_data = glue, .fwnode = dev_fwnode(dev), }; - int err; glue->musb = musb; @@ -113,12 +112,9 @@ static int jz4740_musb_init(struct musb *musb) musb->xceiv = devm_usb_get_phy_by_phandle(dev, "phys", 0); else musb->xceiv = devm_usb_get_phy(dev, USB_PHY_TYPE_USB2); - if (IS_ERR(musb->xceiv)) { - err = PTR_ERR(musb->xceiv); - if (err != -EPROBE_DEFER) - dev_err(dev, "No transceiver configured: %d\n", err); - return err; - } + if (IS_ERR(musb->xceiv)) + return dev_err_probe(dev, PTR_ERR(musb->xceiv), + "No transceiver configured\n"); glue->role_sw = usb_role_switch_register(dev, &role_sw_desc); if (IS_ERR(glue->role_sw)) { diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c index bbbcfd49fb35b80f62965b45f5fa2eed9917c5ba..03027c6fa3aba3e7205af52ab13e08f87dc99506 100644 --- a/drivers/usb/musb/musb_core.c +++ b/drivers/usb/musb/musb_core.c @@ -2595,9 +2595,7 @@ fail2: musb_platform_exit(musb); fail1: - if (status != -EPROBE_DEFER) - dev_err(musb->controller, - "%s failed with status %d\n", __func__, status); + dev_err_probe(musb->controller, status, "%s failed\n", __func__); musb_free(musb); diff --git a/drivers/usb/musb/musb_cppi41.c b/drivers/usb/musb/musb_cppi41.c index c963cb8565f2b3dfdaf2f8ada3a9c807a4e8408b..9589243e89518c1d012428b0201d513c45e21cc4 100644 --- a/drivers/usb/musb/musb_cppi41.c +++ b/drivers/usb/musb/musb_cppi41.c @@ -718,10 +718,8 @@ static int cppi41_dma_controller_start(struct cppi41_dma_controller *controller) dc = dma_request_chan(dev->parent, str); if (IS_ERR(dc)) { - ret = PTR_ERR(dc); - if (ret != -EPROBE_DEFER) - dev_err(dev, "Failed to request %s: %d.\n", - str, ret); + ret = dev_err_probe(dev, PTR_ERR(dc), + "Failed to request %s.\n", str); goto err; } diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c index daada4b66a922bb16303e3344b9ac0dbad5d8101..6704a62a166592f4df150dfbc71ff0a676599aa8 100644 --- a/drivers/usb/musb/musb_gadget.c +++ b/drivers/usb/musb/musb_gadget.c @@ -760,6 +760,9 @@ static void rxstate(struct musb *musb, struct musb_request *req) musb_writew(epio, MUSB_RXCSR, csr); buffer_aint_mapped: + fifo_count = min_t(unsigned int, + request->length - request->actual, + (unsigned int)fifo_count); musb_read_fifo(musb_ep->hw_ep, fifo_count, (u8 *) (request->buf + request->actual)); request->actual += fifo_count; diff --git a/drivers/usb/musb/sunxi.c b/drivers/usb/musb/sunxi.c index 961c858fb349e1fa7a85031eeaa1196e2177df45..7f9a999cd5ff142ec2ceaf1e549a15944f4bed3f 100644 --- a/drivers/usb/musb/sunxi.c +++ b/drivers/usb/musb/sunxi.c @@ -743,31 +743,20 @@ static int sunxi_musb_probe(struct platform_device *pdev) if (test_bit(SUNXI_MUSB_FL_HAS_RESET, &glue->flags)) { glue->rst = devm_reset_control_get(&pdev->dev, NULL); - if (IS_ERR(glue->rst)) { - if (PTR_ERR(glue->rst) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(&pdev->dev, "Error getting reset %ld\n", - PTR_ERR(glue->rst)); - return PTR_ERR(glue->rst); - } + if (IS_ERR(glue->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(glue->rst), + "Error getting reset\n"); } glue->extcon = extcon_get_edev_by_phandle(&pdev->dev, 0); - if (IS_ERR(glue->extcon)) { - if (PTR_ERR(glue->extcon) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(&pdev->dev, "Invalid or missing extcon\n"); - return PTR_ERR(glue->extcon); - } + if (IS_ERR(glue->extcon)) + return dev_err_probe(&pdev->dev, PTR_ERR(glue->extcon), + "Invalid or missing extcon\n"); glue->phy = devm_phy_get(&pdev->dev, "usb"); - if (IS_ERR(glue->phy)) { - if (PTR_ERR(glue->phy) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(&pdev->dev, "Error getting phy %ld\n", - PTR_ERR(glue->phy)); - return PTR_ERR(glue->phy); - } + if (IS_ERR(glue->phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(glue->phy), + "Error getting phy\n"); glue->usb_phy = usb_phy_generic_register(); if (IS_ERR(glue->usb_phy)) { diff --git a/drivers/usb/phy/phy-generic.c b/drivers/usb/phy/phy-generic.c index 34b9f8140187182f334014125c1f9eb745c627b0..3dc5c04e7cbf91bedf49d6387d609c6239d76ee0 100644 --- a/drivers/usb/phy/phy-generic.c +++ b/drivers/usb/phy/phy-generic.c @@ -230,12 +230,9 @@ int usb_phy_gen_create_phy(struct device *dev, struct usb_phy_generic *nop) err = PTR_ERR_OR_ZERO(nop->gpiod_vbus); } - if (err == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (err) { - dev_err(dev, "Error requesting RESET or VBUS GPIO\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, + "Error requesting RESET or VBUS GPIO\n"); if (nop->gpiod_reset) gpiod_direction_output(nop->gpiod_reset, 1); diff --git a/drivers/usb/phy/phy-isp1301-omap.c b/drivers/usb/phy/phy-isp1301-omap.c index f8bd93fe69cda126283da7c201f0b48609527cbc..e5d3f206097c7034ac2d32db5f4dab35f70b4796 100644 --- a/drivers/usb/phy/phy-isp1301-omap.c +++ b/drivers/usb/phy/phy-isp1301-omap.c @@ -1196,7 +1196,7 @@ static void isp1301_release(struct device *dev) static struct isp1301 *the_transceiver; -static int isp1301_remove(struct i2c_client *i2c) +static void isp1301_remove(struct i2c_client *i2c) { struct isp1301 *isp; @@ -1214,8 +1214,6 @@ static int isp1301_remove(struct i2c_client *i2c) put_device(&i2c->dev); the_transceiver = NULL; - - return 0; } /*-------------------------------------------------------------------------*/ diff --git a/drivers/usb/phy/phy-isp1301.c b/drivers/usb/phy/phy-isp1301.c index ad3d57f1c273cf1db9667562096d712e55e5b84c..c2777a5c1f4ed2516ffdf5018f1d85b5495ed3c0 100644 --- a/drivers/usb/phy/phy-isp1301.c +++ b/drivers/usb/phy/phy-isp1301.c @@ -120,14 +120,12 @@ static int isp1301_probe(struct i2c_client *client, return 0; } -static int isp1301_remove(struct i2c_client *client) +static void isp1301_remove(struct i2c_client *client) { struct isp1301 *isp = i2c_get_clientdata(client); usb_remove_phy(&isp->phy); isp1301_i2c_client = NULL; - - return 0; } static struct i2c_driver isp1301_driver = { diff --git a/drivers/usb/phy/phy-jz4770.c b/drivers/usb/phy/phy-jz4770.c index 4025da20b3fdbbe30ce029dc45e09d699bd2c77f..f16adcacdce3f7223bfc9b6b36af605bf1c862f0 100644 --- a/drivers/usb/phy/phy-jz4770.c +++ b/drivers/usb/phy/phy-jz4770.c @@ -321,27 +321,18 @@ static int jz4770_phy_probe(struct platform_device *pdev) } priv->clk = devm_clk_get(dev, NULL); - if (IS_ERR(priv->clk)) { - err = PTR_ERR(priv->clk); - if (err != -EPROBE_DEFER) - dev_err(dev, "Failed to get clock\n"); - return err; - } + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), + "Failed to get clock\n"); priv->vcc_supply = devm_regulator_get(dev, "vcc"); - if (IS_ERR(priv->vcc_supply)) { - err = PTR_ERR(priv->vcc_supply); - if (err != -EPROBE_DEFER) - dev_err(dev, "Failed to get regulator\n"); - return err; - } + if (IS_ERR(priv->vcc_supply)) + return dev_err_probe(dev, PTR_ERR(priv->vcc_supply), + "Failed to get regulator\n"); err = usb_add_phy(&priv->phy, USB_PHY_TYPE_USB2); - if (err) { - if (err != -EPROBE_DEFER) - dev_err(dev, "Unable to register PHY\n"); - return err; - } + if (err) + return dev_err_probe(dev, err, "Unable to register PHY\n"); return devm_add_action_or_reset(dev, ingenic_usb_phy_remove, &priv->phy); } diff --git a/drivers/usb/phy/phy-mxs-usb.c b/drivers/usb/phy/phy-mxs-usb.c index 8a262c5a0408ffa1f3f76994e03bf6f10e1cb5a0..d2836ef5d15c7609277f18ceaab5e1f91f148f8a 100644 --- a/drivers/usb/phy/phy-mxs-usb.c +++ b/drivers/usb/phy/phy-mxs-usb.c @@ -144,8 +144,8 @@ #define MXS_PHY_NEED_IP_FIX BIT(3) /* Minimum and maximum values for device tree entries */ -#define MXS_PHY_TX_CAL45_MIN 30 -#define MXS_PHY_TX_CAL45_MAX 55 +#define MXS_PHY_TX_CAL45_MIN 35 +#define MXS_PHY_TX_CAL45_MAX 54 #define MXS_PHY_TX_D_CAL_MIN 79 #define MXS_PHY_TX_D_CAL_MAX 119 diff --git a/drivers/usb/phy/phy-tegra-usb.c b/drivers/usb/phy/phy-tegra-usb.c index 68cd4b68e3a207f66a7a2fbdd9d6295cc4210111..f0240107edb15270d32ec67e19fe5dc200838665 100644 --- a/drivers/usb/phy/phy-tegra-usb.c +++ b/drivers/usb/phy/phy-tegra-usb.c @@ -1440,16 +1440,22 @@ static int tegra_usb_phy_probe(struct platform_device *pdev) return err; } - gpiod = devm_gpiod_get_from_of_node(&pdev->dev, np, - "nvidia,phy-reset-gpio", - 0, GPIOD_OUT_HIGH, - "ulpi_phy_reset_b"); + gpiod = devm_gpiod_get(&pdev->dev, "nvidia,phy-reset", + GPIOD_OUT_HIGH); err = PTR_ERR_OR_ZERO(gpiod); if (err) { dev_err(&pdev->dev, "Request failed for reset GPIO: %d\n", err); return err; } + + err = gpiod_set_consumer_name(gpiod, "ulpi_phy_reset_b"); + if (err) { + dev_err(&pdev->dev, + "Failed to set up reset GPIO name: %d\n", err); + return err; + } + tegra_phy->reset_gpio = gpiod; phy = devm_otg_ulpi_create(&pdev->dev, diff --git a/drivers/usb/serial/ark3116.c b/drivers/usb/serial/ark3116.c index 39eaa7b97c404bc324fc6e9585c2ef115708e337..9452291f1703321515c66a2660db93bcd84f95a8 100644 --- a/drivers/usb/serial/ark3116.c +++ b/drivers/usb/serial/ark3116.c @@ -189,7 +189,7 @@ static void ark3116_port_remove(struct usb_serial_port *port) static void ark3116_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct ark3116_private *priv = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/belkin_sa.c b/drivers/usb/serial/belkin_sa.c index 8107e4b5b03b17f22fb3e2ba0e6332cee68e9c7e..9331a562dac0ba6df722d47ae3ab8a789df19ead 100644 --- a/drivers/usb/serial/belkin_sa.c +++ b/drivers/usb/serial/belkin_sa.c @@ -44,7 +44,8 @@ static void belkin_sa_close(struct usb_serial_port *port); static void belkin_sa_read_int_callback(struct urb *urb); static void belkin_sa_process_read_urb(struct urb *urb); static void belkin_sa_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios * old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static void belkin_sa_break_ctl(struct tty_struct *tty, int break_state); static int belkin_sa_tiocmget(struct tty_struct *tty); static int belkin_sa_tiocmset(struct tty_struct *tty, @@ -273,7 +274,8 @@ static void belkin_sa_process_read_urb(struct urb *urb) } static void belkin_sa_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct belkin_sa_private *priv = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/ch341.c b/drivers/usb/serial/ch341.c index 2798fca712612bee6e572779c6abad7f063cf0bb..6e1b87e67304616ce04448d3a0d3fd3ebb5d0c56 100644 --- a/drivers/usb/serial/ch341.c +++ b/drivers/usb/serial/ch341.c @@ -97,13 +97,16 @@ struct ch341_private { u8 mcr; u8 msr; u8 lcr; + unsigned long quirks; + u8 version; + unsigned long break_end; }; static void ch341_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios); + const struct ktermios *old_termios); static int ch341_control_out(struct usb_device *dev, u8 request, u16 value, u16 index) @@ -250,8 +253,12 @@ static int ch341_set_baudrate_lcr(struct usb_device *dev, /* * CH341A buffers data until a full endpoint-size packet (32 bytes) * has been received unless bit 7 is set. + * + * At least one device with version 0x27 appears to have this bit + * inverted. */ - val |= BIT(7); + if (priv->version > 0x27) + val |= BIT(7); r = ch341_control_out(dev, CH341_REQ_WRITE_REG, CH341_REG_DIVISOR << 8 | CH341_REG_PRESCALER, @@ -265,6 +272,9 @@ static int ch341_set_baudrate_lcr(struct usb_device *dev, * (stop bits, parity and word length). Version 0x30 and above use * CH341_REG_LCR only and CH341_REG_LCR2 is always set to zero. */ + if (priv->version < 0x30) + return 0; + r = ch341_control_out(dev, CH341_REQ_WRITE_REG, CH341_REG_LCR2 << 8 | CH341_REG_LCR, lcr); if (r) @@ -308,7 +318,9 @@ static int ch341_configure(struct usb_device *dev, struct ch341_private *priv) r = ch341_control_in(dev, CH341_REQ_READ_VERSION, 0, 0, buffer, size); if (r) return r; - dev_dbg(&dev->dev, "Chip version: 0x%02x\n", buffer[0]); + + priv->version = buffer[0]; + dev_dbg(&dev->dev, "Chip version: 0x%02x\n", priv->version); r = ch341_control_out(dev, CH341_REQ_SERIAL_INIT, 0, 0); if (r < 0) @@ -470,7 +482,8 @@ err_kill_interrupt_urb: * tty->termios contains the new setting to be used. */ static void ch341_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct ch341_private *priv = usb_get_serial_port_data(port); unsigned baud_rate; diff --git a/drivers/usb/serial/console.c b/drivers/usb/serial/console.c index b97aa40ca4d1f7af83389a2a1d437329603a3b66..da19a5fa414fc1615a97c99b37d68e970fd929dd 100644 --- a/drivers/usb/serial/console.c +++ b/drivers/usb/serial/console.c @@ -189,8 +189,8 @@ static int usb_console_setup(struct console *co, char *options) info->port = NULL; usb_autopm_put_interface(serial->interface); error_get_interface: - usb_serial_put(serial); mutex_unlock(&serial->disc_mutex); + usb_serial_put(serial); return retval; } diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c index c374620a486f0a53f3772fb34effb11220fe3624..3bcec419f463200c5174da993f2050d94865f6b6 100644 --- a/drivers/usb/serial/cp210x.c +++ b/drivers/usb/serial/cp210x.c @@ -31,9 +31,9 @@ static int cp210x_open(struct tty_struct *tty, struct usb_serial_port *); static void cp210x_close(struct usb_serial_port *); static void cp210x_change_speed(struct tty_struct *, struct usb_serial_port *, - struct ktermios *); + const struct ktermios *); static void cp210x_set_termios(struct tty_struct *, struct usb_serial_port *, - struct ktermios*); + const struct ktermios *); static bool cp210x_tx_empty(struct usb_serial_port *port); static int cp210x_tiocmget(struct tty_struct *); static int cp210x_tiocmset(struct tty_struct *, unsigned int, unsigned int); @@ -130,6 +130,7 @@ static const struct usb_device_id id_table[] = { { USB_DEVICE(0x10C4, 0x83AA) }, /* Mark-10 Digital Force Gauge */ { USB_DEVICE(0x10C4, 0x83D8) }, /* DekTec DTA Plus VHF/UHF Booster/Attenuator */ { USB_DEVICE(0x10C4, 0x8411) }, /* Kyocera GPS Module */ + { USB_DEVICE(0x10C4, 0x8414) }, /* Decagon USB Cable Adapter */ { USB_DEVICE(0x10C4, 0x8418) }, /* IRZ Automation Teleport SG-10 GSM/GPRS Modem */ { USB_DEVICE(0x10C4, 0x846E) }, /* BEI USB Sensor Interface (VCP) */ { USB_DEVICE(0x10C4, 0x8470) }, /* Juniper Networks BX Series System Console */ @@ -1039,7 +1040,8 @@ static speed_t cp210x_get_actual_rate(speed_t baud) * otherwise. */ static void cp210x_change_speed(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct cp210x_serial_private *priv = usb_get_serial_data(serial); @@ -1121,7 +1123,8 @@ static bool cp210x_termios_change(const struct ktermios *a, const struct ktermio } static void cp210x_set_flow_control(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct cp210x_serial_private *priv = usb_get_serial_data(port->serial); struct cp210x_port_private *port_priv = usb_get_serial_port_data(port); @@ -1231,7 +1234,8 @@ out_unlock: } static void cp210x_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct cp210x_serial_private *priv = usb_get_serial_data(port->serial); u16 bits; diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c index 5fbcc155e8f5c17ddb4a38020d4efff04fb73c79..1e0c028c5ec95fed144a54091cd14132bb7eddd9 100644 --- a/drivers/usb/serial/cypress_m8.c +++ b/drivers/usb/serial/cypress_m8.c @@ -125,7 +125,8 @@ static void cypress_send(struct usb_serial_port *port); static unsigned int cypress_write_room(struct tty_struct *tty); static void cypress_earthmate_init_termios(struct tty_struct *tty); static void cypress_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static int cypress_tiocmget(struct tty_struct *tty); static int cypress_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); @@ -859,7 +860,8 @@ static void cypress_earthmate_init_termios(struct tty_struct *tty) } static void cypress_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct cypress_private *priv = usb_get_serial_port_data(port); struct device *dev = &port->dev; diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c index af65eb863d70d4f94c837ca6575ee9ddf77952e6..45d688e9b93f689332e8ebae3985deedc93231f8 100644 --- a/drivers/usb/serial/digi_acceleport.c +++ b/drivers/usb/serial/digi_acceleport.c @@ -215,7 +215,8 @@ static int digi_transmit_idle(struct usb_serial_port *port, static void digi_rx_throttle(struct tty_struct *tty); static void digi_rx_unthrottle(struct tty_struct *tty); static void digi_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); static void digi_break_ctl(struct tty_struct *tty, int break_state); static int digi_tiocmget(struct tty_struct *tty); static int digi_tiocmset(struct tty_struct *tty, unsigned int set, @@ -649,7 +650,8 @@ static void digi_rx_unthrottle(struct tty_struct *tty) static void digi_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct digi_port *priv = usb_get_serial_port_data(port); struct device *dev = &port->dev; diff --git a/drivers/usb/serial/f81232.c b/drivers/usb/serial/f81232.c index d9f20256a6a8eaf86b97f9c4b5b3706ed569626a..2dd58cd9f0ccbae0cc67a2bc7a1c7b881d66fa4f 100644 --- a/drivers/usb/serial/f81232.c +++ b/drivers/usb/serial/f81232.c @@ -603,7 +603,8 @@ static int f81232_port_disable(struct usb_serial_port *port) } static void f81232_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct f81232_private *priv = usb_get_serial_port_data(port); u8 new_lcr = 0; diff --git a/drivers/usb/serial/f81534.c b/drivers/usb/serial/f81534.c index d789c1ec87b353cf4b9a6d3c46dbf5f9478a0dc3..ddfcd72eb0ae7ae9b1b934c8b1dcad646230dd87 100644 --- a/drivers/usb/serial/f81534.c +++ b/drivers/usb/serial/f81534.c @@ -944,8 +944,8 @@ static int f81534_calc_num_ports(struct usb_serial *serial, } static void f81534_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { u8 new_lcr = 0; int status; diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index d5a3986dfee755bc7418e2c998403dad96977e41..05e28a5ce42b1196db9f592618afdcc350fed059 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -47,10 +47,27 @@ #define DRIVER_AUTHOR "Greg Kroah-Hartman , Bill Ryder , Kuba Ober , Andreas Mohr, Johan Hovold " #define DRIVER_DESC "USB FTDI Serial Converters Driver" +enum ftdi_chip_type { + SIO, + FT232A, + FT232B, + FT2232C, + FT232R, + FT232H, + FT2232H, + FT4232H, + FT4232HA, + FT232HP, + FT233HP, + FT2232HP, + FT2233HP, + FT4232HP, + FT4233HP, + FTX, +}; struct ftdi_private { enum ftdi_chip_type chip_type; - /* type of device, either SIO or FT8U232AM */ int baud_base; /* baud base clock for divisor setting */ int custom_divisor; /* custom_divisor kludge, this is for baud_base (different from what goes to the @@ -62,8 +79,7 @@ struct ftdi_private { unsigned long last_dtr_rts; /* saved modem control outputs */ char prev_status; /* Used for TIOCMIWAIT */ char transmit_empty; /* If transmitter is empty or not */ - u16 interface; /* FT2232C, FT2232H or FT4232H port interface - (0 for FT232/245) */ + u16 channel; /* channel index, or 0 for legacy types */ speed_t force_baud; /* if non-zero, force the baud rate to this value */ @@ -84,8 +100,7 @@ struct ftdi_private { #endif }; -/* struct ftdi_sio_quirk is used by devices requiring special attention. */ -struct ftdi_sio_quirk { +struct ftdi_quirk { int (*probe)(struct usb_serial *); /* Special settings for probed ports. */ void (*port_probe)(struct ftdi_private *); @@ -98,27 +113,27 @@ static int ftdi_8u2232c_probe(struct usb_serial *serial); static void ftdi_USB_UIRT_setup(struct ftdi_private *priv); static void ftdi_HE_TIRA1_setup(struct ftdi_private *priv); -static const struct ftdi_sio_quirk ftdi_jtag_quirk = { +static const struct ftdi_quirk ftdi_jtag_quirk = { .probe = ftdi_jtag_probe, }; -static const struct ftdi_sio_quirk ftdi_NDI_device_quirk = { +static const struct ftdi_quirk ftdi_NDI_device_quirk = { .probe = ftdi_NDI_device_setup, }; -static const struct ftdi_sio_quirk ftdi_USB_UIRT_quirk = { +static const struct ftdi_quirk ftdi_USB_UIRT_quirk = { .port_probe = ftdi_USB_UIRT_setup, }; -static const struct ftdi_sio_quirk ftdi_HE_TIRA1_quirk = { +static const struct ftdi_quirk ftdi_HE_TIRA1_quirk = { .port_probe = ftdi_HE_TIRA1_setup, }; -static const struct ftdi_sio_quirk ftdi_stmclite_quirk = { +static const struct ftdi_quirk ftdi_stmclite_quirk = { .probe = ftdi_stmclite_probe, }; -static const struct ftdi_sio_quirk ftdi_8u2232c_quirk = { +static const struct ftdi_quirk ftdi_8u2232c_quirk = { .probe = ftdi_8u2232c_probe, }; @@ -180,6 +195,13 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE(FTDI_VID, FTDI_4232H_PID) }, { USB_DEVICE(FTDI_VID, FTDI_232H_PID) }, { USB_DEVICE(FTDI_VID, FTDI_FTX_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT2233HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT4233HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT2232HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT4232HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT233HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT232HP_PID) }, + { USB_DEVICE(FTDI_VID, FTDI_FT4232HA_PID) }, { USB_DEVICE(FTDI_VID, FTDI_MICRO_CHAMELEON_PID) }, { USB_DEVICE(FTDI_VID, FTDI_RELAIS_PID) }, { USB_DEVICE(FTDI_VID, FTDI_OPENDCC_PID) }, @@ -1045,6 +1067,8 @@ static const struct usb_device_id id_table_combined[] = { /* IDS GmbH devices */ { USB_DEVICE(IDS_VID, IDS_SI31A_PID) }, { USB_DEVICE(IDS_VID, IDS_CM31A_PID) }, + /* Omron devices */ + { USB_DEVICE(OMRON_VID, OMRON_CS1W_CIF31_PID) }, /* U-Blox devices */ { USB_DEVICE(UBLOX_VID, UBLOX_C099F9P_ZED_PID) }, { USB_DEVICE(UBLOX_VID, UBLOX_C099F9P_ODIN_PID) }, @@ -1059,15 +1083,22 @@ static const struct usb_device_id id_table_combined[] = { MODULE_DEVICE_TABLE(usb, id_table_combined); static const char *ftdi_chip_name[] = { - [SIO] = "SIO", /* the serial part of FT8U100AX */ - [FT8U232AM] = "FT8U232AM", - [FT232BM] = "FT232BM", - [FT2232C] = "FT2232C", - [FT232RL] = "FT232RL", - [FT2232H] = "FT2232H", - [FT4232H] = "FT4232H", - [FT232H] = "FT232H", - [FTX] = "FT-X" + [SIO] = "SIO", /* the serial part of FT8U100AX */ + [FT232A] = "FT232A", + [FT232B] = "FT232B", + [FT2232C] = "FT2232C/D", + [FT232R] = "FT232R", + [FT232H] = "FT232H", + [FT2232H] = "FT2232H", + [FT4232H] = "FT4232H", + [FT4232HA] = "FT4232HA", + [FT232HP] = "FT232HP", + [FT233HP] = "FT233HP", + [FT2232HP] = "FT2232HP", + [FT2233HP] = "FT2233HP", + [FT4232HP] = "FT4232HP", + [FT4233HP] = "FT4233HP", + [FTX] = "FT-X", }; @@ -1076,74 +1107,12 @@ static const char *ftdi_chip_name[] = { #define FTDI_STATUS_B1_MASK (FTDI_RS_BI) /* End TIOCMIWAIT */ -/* function prototypes for a FTDI serial converter */ -static int ftdi_sio_probe(struct usb_serial *serial, - const struct usb_device_id *id); -static int ftdi_sio_port_probe(struct usb_serial_port *port); -static void ftdi_sio_port_remove(struct usb_serial_port *port); -static int ftdi_open(struct tty_struct *tty, struct usb_serial_port *port); -static void ftdi_dtr_rts(struct usb_serial_port *port, int on); -static void ftdi_process_read_urb(struct urb *urb); -static int ftdi_prepare_write_buffer(struct usb_serial_port *port, - void *dest, size_t size); static void ftdi_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); -static int ftdi_tiocmget(struct tty_struct *tty); -static int ftdi_tiocmset(struct tty_struct *tty, - unsigned int set, unsigned int clear); -static int ftdi_ioctl(struct tty_struct *tty, - unsigned int cmd, unsigned long arg); -static void get_serial_info(struct tty_struct *tty, struct serial_struct *ss); -static int set_serial_info(struct tty_struct *tty, - struct serial_struct *ss); -static void ftdi_break_ctl(struct tty_struct *tty, int break_state); -static bool ftdi_tx_empty(struct usb_serial_port *port); + struct usb_serial_port *port, + const struct ktermios *old_termios); static int ftdi_get_modem_status(struct usb_serial_port *port, unsigned char status[2]); -static unsigned short int ftdi_232am_baud_base_to_divisor(int baud, int base); -static unsigned short int ftdi_232am_baud_to_divisor(int baud); -static u32 ftdi_232bm_baud_base_to_divisor(int baud, int base); -static u32 ftdi_232bm_baud_to_divisor(int baud); -static u32 ftdi_2232h_baud_base_to_divisor(int baud, int base); -static u32 ftdi_2232h_baud_to_divisor(int baud); - -static struct usb_serial_driver ftdi_sio_device = { - .driver = { - .owner = THIS_MODULE, - .name = "ftdi_sio", - }, - .description = "FTDI USB Serial Device", - .id_table = id_table_combined, - .num_ports = 1, - .bulk_in_size = 512, - .bulk_out_size = 256, - .probe = ftdi_sio_probe, - .port_probe = ftdi_sio_port_probe, - .port_remove = ftdi_sio_port_remove, - .open = ftdi_open, - .dtr_rts = ftdi_dtr_rts, - .throttle = usb_serial_generic_throttle, - .unthrottle = usb_serial_generic_unthrottle, - .process_read_urb = ftdi_process_read_urb, - .prepare_write_buffer = ftdi_prepare_write_buffer, - .tiocmget = ftdi_tiocmget, - .tiocmset = ftdi_tiocmset, - .tiocmiwait = usb_serial_generic_tiocmiwait, - .get_icount = usb_serial_generic_get_icount, - .ioctl = ftdi_ioctl, - .get_serial = get_serial_info, - .set_serial = set_serial_info, - .set_termios = ftdi_set_termios, - .break_ctl = ftdi_break_ctl, - .tx_empty = ftdi_tx_empty, -}; - -static struct usb_serial_driver * const serial_drivers[] = { - &ftdi_sio_device, NULL -}; - - #define WDR_TIMEOUT 5000 /* default urb timeout */ #define WDR_SHORT_TIMEOUT 1000 /* shorter urb timeout */ @@ -1259,7 +1228,7 @@ static int update_mctrl(struct usb_serial_port *port, unsigned int set, usb_sndctrlpipe(port->serial->dev, 0), FTDI_SIO_SET_MODEM_CTRL_REQUEST, FTDI_SIO_SET_MODEM_CTRL_REQUEST_TYPE, - value, priv->interface, + value, priv->channel, NULL, 0, WDR_TIMEOUT); if (rv < 0) { dev_dbg(dev, "%s Error from MODEM_CTRL urb: DTR %s, RTS %s\n", @@ -1305,7 +1274,7 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, if (!baud) baud = 9600; switch (priv->chip_type) { - case SIO: /* SIO chip */ + case SIO: switch (baud) { case 300: div_value = ftdi_sio_b300; break; case 600: div_value = ftdi_sio_b600; break; @@ -1317,8 +1286,7 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, case 38400: div_value = ftdi_sio_b38400; break; case 57600: div_value = ftdi_sio_b57600; break; case 115200: div_value = ftdi_sio_b115200; break; - } /* baud */ - if (div_value == 0) { + default: dev_dbg(dev, "%s - Baudrate (%d) requested is not supported\n", __func__, baud); div_value = ftdi_sio_b9600; @@ -1326,7 +1294,7 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, div_okay = 0; } break; - case FT8U232AM: /* 8U232AM chip */ + case FT232A: if (baud <= 3000000) { div_value = ftdi_232am_baud_to_divisor(baud); } else { @@ -1336,10 +1304,10 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, div_okay = 0; } break; - case FT232BM: /* FT232BM chip */ - case FT2232C: /* FT2232C chip */ - case FT232RL: /* FT232RL chip */ - case FTX: /* FT-X series */ + case FT232B: + case FT2232C: + case FT232R: + case FTX: if (baud <= 3000000) { u16 product_id = le16_to_cpu( port->serial->dev->descriptor.idProduct); @@ -1359,9 +1327,7 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, baud = 9600; } break; - case FT2232H: /* FT2232H chip */ - case FT4232H: /* FT4232H chip */ - case FT232H: /* FT232H chip */ + default: if ((baud <= 12000000) && (baud >= 1200)) { div_value = ftdi_2232h_baud_to_divisor(baud); } else if (baud < 1200) { @@ -1373,7 +1339,7 @@ static u32 get_ftdi_divisor(struct tty_struct *tty, baud = 9600; } break; - } /* priv->chip_type */ + } if (div_okay) { dev_dbg(dev, "%s - Baud rate set to %d (divisor 0x%lX) on chip %s\n", @@ -1396,13 +1362,8 @@ static int change_speed(struct tty_struct *tty, struct usb_serial_port *port) index_value = get_ftdi_divisor(tty, port); value = (u16)index_value; index = (u16)(index_value >> 16); - if (priv->chip_type == FT2232C || priv->chip_type == FT2232H || - priv->chip_type == FT4232H || priv->chip_type == FT232H || - priv->chip_type == FTX) { - /* Probably the BM type needs the MSB of the encoded fractional - * divider also moved like for the chips above. Any infos? */ - index = (u16)((index << 8) | priv->interface); - } + if (priv->channel) + index = (u16)((index << 8) | priv->channel); rv = usb_control_msg(port->serial->dev, usb_sndctrlpipe(port->serial->dev, 0), @@ -1420,7 +1381,7 @@ static int write_latency_timer(struct usb_serial_port *port) int rv; int l = priv->latency; - if (priv->chip_type == SIO || priv->chip_type == FT8U232AM) + if (priv->chip_type == SIO || priv->chip_type == FT232A) return -EINVAL; if (priv->flags & ASYNC_LOW_LATENCY) @@ -1432,7 +1393,7 @@ static int write_latency_timer(struct usb_serial_port *port) usb_sndctrlpipe(udev, 0), FTDI_SIO_SET_LATENCY_TIMER_REQUEST, FTDI_SIO_SET_LATENCY_TIMER_REQUEST_TYPE, - l, priv->interface, + l, priv->channel, NULL, 0, WDR_TIMEOUT); if (rv < 0) dev_err(&port->dev, "Unable to write latency timer: %i\n", rv); @@ -1448,7 +1409,7 @@ static int _read_latency_timer(struct usb_serial_port *port) rv = usb_control_msg_recv(udev, 0, FTDI_SIO_GET_LATENCY_TIMER_REQUEST, FTDI_SIO_GET_LATENCY_TIMER_REQUEST_TYPE, 0, - priv->interface, &buf, 1, WDR_TIMEOUT, + priv->channel, &buf, 1, WDR_TIMEOUT, GFP_KERNEL); if (rv == 0) rv = buf; @@ -1461,7 +1422,7 @@ static int read_latency_timer(struct usb_serial_port *port) struct ftdi_private *priv = usb_get_serial_port_data(port); int rv; - if (priv->chip_type == SIO || priv->chip_type == FT8U232AM) + if (priv->chip_type == SIO || priv->chip_type == FT232A) return -EINVAL; rv = _read_latency_timer(port); @@ -1536,90 +1497,97 @@ static int get_lsr_info(struct usb_serial_port *port, return 0; } - -/* Determine type of FTDI chip based on USB config and descriptor. */ -static void ftdi_determine_type(struct usb_serial_port *port) +static int ftdi_determine_type(struct usb_serial_port *port) { struct ftdi_private *priv = usb_get_serial_port_data(port); struct usb_serial *serial = port->serial; struct usb_device *udev = serial->dev; - unsigned version; - unsigned interfaces; - - /* Assume it is not the original SIO device for now. */ - priv->baud_base = 48000000 / 2; + unsigned int version, ifnum; version = le16_to_cpu(udev->descriptor.bcdDevice); - interfaces = udev->actconfig->desc.bNumInterfaces; - dev_dbg(&port->dev, "%s: bcdDevice = 0x%x, bNumInterfaces = %u\n", __func__, - version, interfaces); - if (interfaces > 1) { - struct usb_interface *intf = serial->interface; - int ifnum = intf->cur_altsetting->desc.bInterfaceNumber; - - /* Multiple interfaces.*/ - if (version == 0x0800) { - priv->chip_type = FT4232H; - /* Hi-speed - baud clock runs at 120MHz */ - priv->baud_base = 120000000 / 2; - } else if (version == 0x0700) { - priv->chip_type = FT2232H; - /* Hi-speed - baud clock runs at 120MHz */ - priv->baud_base = 120000000 / 2; - } else - priv->chip_type = FT2232C; - - /* Determine interface code. */ - if (ifnum == 0) - priv->interface = INTERFACE_A; - else if (ifnum == 1) - priv->interface = INTERFACE_B; - else if (ifnum == 2) - priv->interface = INTERFACE_C; - else if (ifnum == 3) - priv->interface = INTERFACE_D; - - /* BM-type devices have a bug where bcdDevice gets set - * to 0x200 when iSerialNumber is 0. */ - if (version < 0x500) { - dev_dbg(&port->dev, - "%s: something fishy - bcdDevice too low for multi-interface device\n", - __func__); - } - } else if (version < 0x200) { - /* Old device. Assume it's the original SIO. */ - priv->chip_type = SIO; - priv->baud_base = 12000000 / 16; - } else if (version < 0x400) { - /* Assume it's an FT8U232AM (or FT8U245AM) */ - priv->chip_type = FT8U232AM; + ifnum = serial->interface->cur_altsetting->desc.bInterfaceNumber; + + /* Assume Hi-Speed type */ + priv->baud_base = 120000000 / 2; + priv->channel = CHANNEL_A + ifnum; + + switch (version) { + case 0x200: + priv->chip_type = FT232A; + priv->baud_base = 48000000 / 2; + priv->channel = 0; /* - * It might be a BM type because of the iSerialNumber bug. - * If iSerialNumber==0 and the latency timer is readable, - * assume it is BM type. + * FT232B devices have a bug where bcdDevice gets set to 0x200 + * when iSerialNumber is 0. Assume it is an FT232B in case the + * latency timer is readable. */ if (udev->descriptor.iSerialNumber == 0 && _read_latency_timer(port) >= 0) { - dev_dbg(&port->dev, - "%s: has latency timer so not an AM type\n", - __func__); - priv->chip_type = FT232BM; + priv->chip_type = FT232B; } - } else if (version < 0x600) { - /* Assume it's an FT232BM (or FT245BM) */ - priv->chip_type = FT232BM; - } else if (version < 0x900) { - /* Assume it's an FT232RL */ - priv->chip_type = FT232RL; - } else if (version < 0x1000) { - /* Assume it's an FT232H */ + break; + case 0x400: + priv->chip_type = FT232B; + priv->baud_base = 48000000 / 2; + priv->channel = 0; + break; + case 0x500: + priv->chip_type = FT2232C; + priv->baud_base = 48000000 / 2; + break; + case 0x600: + priv->chip_type = FT232R; + priv->baud_base = 48000000 / 2; + priv->channel = 0; + break; + case 0x700: + priv->chip_type = FT2232H; + break; + case 0x800: + priv->chip_type = FT4232H; + break; + case 0x900: priv->chip_type = FT232H; - } else { - /* Assume it's an FT-X series device */ + break; + case 0x1000: priv->chip_type = FTX; + priv->baud_base = 48000000 / 2; + break; + case 0x2800: + priv->chip_type = FT2233HP; + break; + case 0x2900: + priv->chip_type = FT4233HP; + break; + case 0x3000: + priv->chip_type = FT2232HP; + break; + case 0x3100: + priv->chip_type = FT4232HP; + break; + case 0x3200: + priv->chip_type = FT233HP; + break; + case 0x3300: + priv->chip_type = FT232HP; + break; + case 0x3600: + priv->chip_type = FT4232HA; + break; + default: + if (version < 0x200) { + priv->chip_type = SIO; + priv->baud_base = 12000000 / 16; + priv->channel = 0; + } else { + dev_err(&port->dev, "unknown device type: 0x%02x\n", version); + return -ENODEV; + } } dev_info(&udev->dev, "Detected %s\n", ftdi_chip_name[priv->chip_type]); + + return 0; } @@ -1718,7 +1686,7 @@ static ssize_t event_char_store(struct device *dev, usb_sndctrlpipe(udev, 0), FTDI_SIO_SET_EVENT_CHAR_REQUEST, FTDI_SIO_SET_EVENT_CHAR_REQUEST_TYPE, - v, priv->interface, + v, priv->channel, NULL, 0, WDR_TIMEOUT); if (rv < 0) { dev_dbg(&port->dev, "Unable to write event character: %i\n", rv); @@ -1729,51 +1697,42 @@ static ssize_t event_char_store(struct device *dev, } static DEVICE_ATTR_WO(event_char); -static int create_sysfs_attrs(struct usb_serial_port *port) -{ - struct ftdi_private *priv = usb_get_serial_port_data(port); - int retval = 0; - - /* XXX I've no idea if the original SIO supports the event_char - * sysfs parameter, so I'm playing it safe. */ - if (priv->chip_type != SIO) { - dev_dbg(&port->dev, "sysfs attributes for %s\n", ftdi_chip_name[priv->chip_type]); - retval = device_create_file(&port->dev, &dev_attr_event_char); - if ((!retval) && - (priv->chip_type == FT232BM || - priv->chip_type == FT2232C || - priv->chip_type == FT232RL || - priv->chip_type == FT2232H || - priv->chip_type == FT4232H || - priv->chip_type == FT232H || - priv->chip_type == FTX)) { - retval = device_create_file(&port->dev, - &dev_attr_latency_timer); - } - } - return retval; -} +static struct attribute *ftdi_attrs[] = { + &dev_attr_event_char.attr, + &dev_attr_latency_timer.attr, + NULL +}; -static void remove_sysfs_attrs(struct usb_serial_port *port) +static umode_t ftdi_is_visible(struct kobject *kobj, struct attribute *attr, int idx) { + struct device *dev = kobj_to_dev(kobj); + struct usb_serial_port *port = to_usb_serial_port(dev); struct ftdi_private *priv = usb_get_serial_port_data(port); + enum ftdi_chip_type type = priv->chip_type; - /* XXX see create_sysfs_attrs */ - if (priv->chip_type != SIO) { - device_remove_file(&port->dev, &dev_attr_event_char); - if (priv->chip_type == FT232BM || - priv->chip_type == FT2232C || - priv->chip_type == FT232RL || - priv->chip_type == FT2232H || - priv->chip_type == FT4232H || - priv->chip_type == FT232H || - priv->chip_type == FTX) { - device_remove_file(&port->dev, &dev_attr_latency_timer); - } + if (attr == &dev_attr_event_char.attr) { + if (type == SIO) + return 0; + } + + if (attr == &dev_attr_latency_timer.attr) { + if (type == SIO || type == FT232A) + return 0; } + return attr->mode; } +static const struct attribute_group ftdi_group = { + .attrs = ftdi_attrs, + .is_visible = ftdi_is_visible, +}; + +static const struct attribute_group *ftdi_groups[] = { + &ftdi_group, + NULL +}; + #ifdef CONFIG_GPIOLIB static int ftdi_set_bitmode(struct usb_serial_port *port, u8 mode) @@ -1792,7 +1751,7 @@ static int ftdi_set_bitmode(struct usb_serial_port *port, u8 mode) usb_sndctrlpipe(serial->dev, 0), FTDI_SIO_SET_BITMODE_REQUEST, FTDI_SIO_SET_BITMODE_REQUEST_TYPE, val, - priv->interface, NULL, 0, WDR_TIMEOUT); + priv->channel, NULL, 0, WDR_TIMEOUT); if (result < 0) { dev_err(&serial->interface->dev, "bitmode request failed for value 0x%04x: %d\n", @@ -1856,7 +1815,7 @@ static int ftdi_read_cbus_pins(struct usb_serial_port *port) result = usb_control_msg_recv(serial->dev, 0, FTDI_SIO_READ_PINS_REQUEST, FTDI_SIO_READ_PINS_REQUEST_TYPE, 0, - priv->interface, &buf, 1, WDR_TIMEOUT, + priv->channel, &buf, 1, WDR_TIMEOUT, GFP_KERNEL); if (result == 0) result = buf; @@ -2141,7 +2100,7 @@ static int ftdi_gpio_init(struct usb_serial_port *port) case FT232H: result = ftdi_gpio_init_ft232h(port); break; - case FT232RL: + case FT232R: result = ftdi_gpio_init_ft232r(port); break; case FTX: @@ -2211,12 +2170,9 @@ static void ftdi_gpio_remove(struct usb_serial_port *port) { } * *************************************************************************** */ -/* Probe function to check for special devices */ -static int ftdi_sio_probe(struct usb_serial *serial, - const struct usb_device_id *id) +static int ftdi_probe(struct usb_serial *serial, const struct usb_device_id *id) { - const struct ftdi_sio_quirk *quirk = - (struct ftdi_sio_quirk *)id->driver_info; + const struct ftdi_quirk *quirk = (struct ftdi_quirk *)id->driver_info; if (quirk && quirk->probe) { int ret = quirk->probe(serial); @@ -2229,10 +2185,10 @@ static int ftdi_sio_probe(struct usb_serial *serial, return 0; } -static int ftdi_sio_port_probe(struct usb_serial_port *port) +static int ftdi_port_probe(struct usb_serial_port *port) { + const struct ftdi_quirk *quirk = usb_get_serial_data(port->serial); struct ftdi_private *priv; - const struct ftdi_sio_quirk *quirk = usb_get_serial_data(port->serial); int result; priv = kzalloc(sizeof(struct ftdi_private), GFP_KERNEL); @@ -2246,12 +2202,14 @@ static int ftdi_sio_port_probe(struct usb_serial_port *port) usb_set_serial_port_data(port, priv); - ftdi_determine_type(port); + result = ftdi_determine_type(port); + if (result) + goto err_free; + ftdi_set_max_packet_size(port); if (read_latency_timer(port) < 0) priv->latency = 16; write_latency_timer(port); - create_sysfs_attrs(port); result = ftdi_gpio_init(port); if (result < 0) { @@ -2261,6 +2219,11 @@ static int ftdi_sio_port_probe(struct usb_serial_port *port) } return 0; + +err_free: + kfree(priv); + + return result; } /* Setup for the USB-UIRT device, which requires hardwired @@ -2371,14 +2334,12 @@ static int ftdi_stmclite_probe(struct usb_serial *serial) return 0; } -static void ftdi_sio_port_remove(struct usb_serial_port *port) +static void ftdi_port_remove(struct usb_serial_port *port) { struct ftdi_private *priv = usb_get_serial_port_data(port); ftdi_gpio_remove(port); - remove_sysfs_attrs(port); - kfree(priv); } @@ -2392,7 +2353,7 @@ static int ftdi_open(struct tty_struct *tty, struct usb_serial_port *port) usb_control_msg(dev, usb_sndctrlpipe(dev, 0), FTDI_SIO_RESET_REQUEST, FTDI_SIO_RESET_REQUEST_TYPE, FTDI_SIO_RESET_SIO, - priv->interface, NULL, 0, WDR_TIMEOUT); + priv->channel, NULL, 0, WDR_TIMEOUT); /* Termios defaults are set by usb_serial_init. We don't change port->tty->termios - this would lose speed settings, etc. @@ -2415,7 +2376,7 @@ static void ftdi_dtr_rts(struct usb_serial_port *port, int on) usb_sndctrlpipe(port->serial->dev, 0), FTDI_SIO_SET_FLOW_CTRL_REQUEST, FTDI_SIO_SET_FLOW_CTRL_REQUEST_TYPE, - 0, priv->interface, NULL, 0, + 0, priv->channel, NULL, 0, WDR_TIMEOUT) < 0) { dev_err(&port->dev, "error from flowcontrol urb\n"); } @@ -2608,7 +2569,7 @@ static void ftdi_break_ctl(struct tty_struct *tty, int break_state) usb_sndctrlpipe(port->serial->dev, 0), FTDI_SIO_SET_DATA_REQUEST, FTDI_SIO_SET_DATA_REQUEST_TYPE, - value , priv->interface, + value, priv->channel, NULL, 0, WDR_TIMEOUT) < 0) { dev_err(&port->dev, "%s FAILED to enable/disable break state (state was %d)\n", __func__, break_state); @@ -2638,7 +2599,8 @@ static bool ftdi_tx_empty(struct usb_serial_port *port) * WARNING: set_termios calls this with old_termios in kernel space */ static void ftdi_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_device *dev = port->serial->dev; struct device *ddev = &port->dev; @@ -2744,7 +2706,7 @@ no_skip: if (usb_control_msg(dev, usb_sndctrlpipe(dev, 0), FTDI_SIO_SET_DATA_REQUEST, FTDI_SIO_SET_DATA_REQUEST_TYPE, - value , priv->interface, + value, priv->channel, NULL, 0, WDR_SHORT_TIMEOUT) < 0) { dev_err(ddev, "%s FAILED to set databits/stopbits/parity\n", __func__); @@ -2757,7 +2719,7 @@ no_data_parity_stop_changes: if (usb_control_msg(dev, usb_sndctrlpipe(dev, 0), FTDI_SIO_SET_FLOW_CTRL_REQUEST, FTDI_SIO_SET_FLOW_CTRL_REQUEST_TYPE, - 0, priv->interface, + 0, priv->channel, NULL, 0, WDR_TIMEOUT) < 0) { dev_err(ddev, "%s error from disable flowcontrol urb\n", __func__); @@ -2791,7 +2753,7 @@ no_c_cflag_changes: index = FTDI_SIO_DISABLE_FLOW_CTRL; } - index |= priv->interface; + index |= priv->channel; ret = usb_control_msg(dev, usb_sndctrlpipe(dev, 0), FTDI_SIO_SET_FLOW_CTRL_REQUEST, @@ -2819,33 +2781,19 @@ static int ftdi_get_modem_status(struct usb_serial_port *port, if (!buf) return -ENOMEM; /* - * The 8U232AM returns a two byte value (the SIO a 1 byte value) in - * the same format as the data returned from the in point. + * The device returns a two byte value (the SIO a 1 byte value) in the + * same format as the data returned from the IN endpoint. */ - switch (priv->chip_type) { - case SIO: + if (priv->chip_type == SIO) len = 1; - break; - case FT8U232AM: - case FT232BM: - case FT2232C: - case FT232RL: - case FT2232H: - case FT4232H: - case FT232H: - case FTX: + else len = 2; - break; - default: - ret = -EFAULT; - goto out; - } ret = usb_control_msg(port->serial->dev, usb_rcvctrlpipe(port->serial->dev, 0), FTDI_SIO_GET_MODEM_STATUS_REQUEST, FTDI_SIO_GET_MODEM_STATUS_REQUEST_TYPE, - 0, priv->interface, + 0, priv->channel, buf, len, WDR_TIMEOUT); /* NOTE: We allow short responses and handle that below. */ @@ -2915,6 +2863,41 @@ static int ftdi_ioctl(struct tty_struct *tty, return -ENOIOCTLCMD; } +static struct usb_serial_driver ftdi_device = { + .driver = { + .owner = THIS_MODULE, + .name = "ftdi_sio", + .dev_groups = ftdi_groups, + }, + .description = "FTDI USB Serial Device", + .id_table = id_table_combined, + .num_ports = 1, + .bulk_in_size = 512, + .bulk_out_size = 256, + .probe = ftdi_probe, + .port_probe = ftdi_port_probe, + .port_remove = ftdi_port_remove, + .open = ftdi_open, + .dtr_rts = ftdi_dtr_rts, + .throttle = usb_serial_generic_throttle, + .unthrottle = usb_serial_generic_unthrottle, + .process_read_urb = ftdi_process_read_urb, + .prepare_write_buffer = ftdi_prepare_write_buffer, + .tiocmget = ftdi_tiocmget, + .tiocmset = ftdi_tiocmset, + .tiocmiwait = usb_serial_generic_tiocmiwait, + .get_icount = usb_serial_generic_get_icount, + .ioctl = ftdi_ioctl, + .get_serial = get_serial_info, + .set_serial = set_serial_info, + .set_termios = ftdi_set_termios, + .break_ctl = ftdi_break_ctl, + .tx_empty = ftdi_tx_empty, +}; + +static struct usb_serial_driver * const serial_drivers[] = { + &ftdi_device, NULL +}; module_usb_serial_driver(serial_drivers, id_table_combined); MODULE_AUTHOR(DRIVER_AUTHOR); diff --git a/drivers/usb/serial/ftdi_sio.h b/drivers/usb/serial/ftdi_sio.h index be1641e0408b0d7411b8ac4874c0f05c0678ed7c..55ea61264f9190094829f2998d89356b3e0834f0 100644 --- a/drivers/usb/serial/ftdi_sio.h +++ b/drivers/usb/serial/ftdi_sio.h @@ -40,11 +40,11 @@ #define FTDI_SIO_READ_PINS 0x0c /* Read immediate value of pins */ #define FTDI_SIO_READ_EEPROM 0x90 /* Read EEPROM */ -/* Interface indices for FT2232, FT2232H and FT4232H devices */ -#define INTERFACE_A 1 -#define INTERFACE_B 2 -#define INTERFACE_C 3 -#define INTERFACE_D 4 +/* Channel indices for FT2232, FT2232H and FT4232H devices */ +#define CHANNEL_A 1 +#define CHANNEL_B 2 +#define CHANNEL_C 3 +#define CHANNEL_D 4 /* @@ -153,18 +153,6 @@ * not supported by the FT8U232AM). */ -enum ftdi_chip_type { - SIO = 1, - FT8U232AM = 2, - FT232BM = 3, - FT2232C = 4, - FT232RL = 5, - FT2232H = 6, - FT4232H = 7, - FT232H = 8, - FTX = 9, -}; - enum ftdi_sio_baudrate { ftdi_sio_b300 = 0, ftdi_sio_b600 = 1, diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index 4e92c165c86bf0aa199e00bee12e7f8a226ae865..e2099445db708f90ab5122e6180150e4c7d947fc 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -25,6 +25,13 @@ #define FTDI_4232H_PID 0x6011 /* Quad channel hi-speed device */ #define FTDI_232H_PID 0x6014 /* Single channel hi-speed device */ #define FTDI_FTX_PID 0x6015 /* FT-X series (FT201X, FT230X, FT231X, etc) */ +#define FTDI_FT2233HP_PID 0x6040 /* Dual channel hi-speed device with PD */ +#define FTDI_FT4233HP_PID 0x6041 /* Quad channel hi-speed device with PD */ +#define FTDI_FT2232HP_PID 0x6042 /* Dual channel hi-speed device with PD */ +#define FTDI_FT4232HP_PID 0x6043 /* Quad channel hi-speed device with PD */ +#define FTDI_FT233HP_PID 0x6044 /* Dual channel hi-speed device with PD */ +#define FTDI_FT232HP_PID 0x6045 /* Dual channel hi-speed device with PD */ +#define FTDI_FT4232HA_PID 0x6048 /* Quad channel automotive grade hi-speed device */ #define FTDI_SIO_PID 0x8372 /* Product Id SIO application of 8U100AX */ #define FTDI_232RL_PID 0xFBFA /* Product ID for FT232RL */ @@ -661,6 +668,12 @@ #define INFINEON_TRIBOARD_TC1798_PID 0x0028 /* DAS JTAG TriBoard TC1798 V1.0 */ #define INFINEON_TRIBOARD_TC2X7_PID 0x0043 /* DAS JTAG TriBoard TC2X7 V1.0 */ +/* + * Omron corporation (https://www.omron.com) + */ + #define OMRON_VID 0x0590 + #define OMRON_CS1W_CIF31_PID 0x00b2 + /* * Acton Research Corp. */ diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index ffa622539a25cced45e1e70db59cf5d9f4650341..3a4c0febf33589a88e5e3fc2e64e95c5f2c56504 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -281,7 +281,7 @@ static int send_iosp_ext_cmd(struct edgeport_port *edge_port, __u8 command, static int calc_baud_rate_divisor(struct device *dev, int baud_rate, int *divisor); static void change_port_settings(struct tty_struct *tty, struct edgeport_port *edge_port, - struct ktermios *old_termios); + const struct ktermios *old_termios); static int send_cmd_write_uart_register(struct edgeport_port *edge_port, __u8 regNum, __u8 regValue); static int write_cmd_usb(struct edgeport_port *edge_port, @@ -1441,7 +1441,8 @@ static void edge_unthrottle(struct tty_struct *tty) * the termios structure *****************************************************************************/ static void edge_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct edgeport_port *edge_port = usb_get_serial_port_data(port); @@ -2325,7 +2326,7 @@ static int send_cmd_write_uart_register(struct edgeport_port *edge_port, *****************************************************************************/ static void change_port_settings(struct tty_struct *tty, - struct edgeport_port *edge_port, struct ktermios *old_termios) + struct edgeport_port *edge_port, const struct ktermios *old_termios) { struct device *dev = &edge_port->port->dev; struct edgeport_serial *edge_serial = diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c index feba2a8d1233a902f20886f1d7050f5ede11fa2f..bc3c24ea42c1bffb0cafe277e03dceacd087c42f 100644 --- a/drivers/usb/serial/io_ti.c +++ b/drivers/usb/serial/io_ti.c @@ -221,7 +221,8 @@ static void stop_read(struct edgeport_port *edge_port); static int restart_read(struct edgeport_port *edge_port); static void edge_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); static void edge_send(struct usb_serial_port *port, struct tty_struct *tty); static int do_download_mode(struct edgeport_serial *serial, @@ -2210,7 +2211,7 @@ static int restart_read(struct edgeport_port *edge_port) } static void change_port_settings(struct tty_struct *tty, - struct edgeport_port *edge_port, struct ktermios *old_termios) + struct edgeport_port *edge_port, const struct ktermios *old_termios) { struct device *dev = &edge_port->port->dev; struct ump_uart_config *config; @@ -2351,7 +2352,8 @@ static void change_port_settings(struct tty_struct *tty, } static void edge_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct edgeport_port *edge_port = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/ir-usb.c b/drivers/usb/serial/ir-usb.c index 7b44dbea95cd300780f42a1afb1c76f2137ab0dc..82f108134e6fad8237ec75d86b2d3eb81e6c8f57 100644 --- a/drivers/usb/serial/ir-usb.c +++ b/drivers/usb/serial/ir-usb.c @@ -51,7 +51,8 @@ static unsigned int ir_write_room(struct tty_struct *tty); static void ir_write_bulk_callback(struct urb *urb); static void ir_process_read_urb(struct urb *urb); static void ir_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); /* Not that this lot means you can only have one per system */ static u8 ir_baud; @@ -376,7 +377,8 @@ static void ir_process_read_urb(struct urb *urb) } static void ir_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_device *udev = port->serial->dev; unsigned char *transfer_buffer; diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c index 0be3b5e1eaf352d56593415ace2f775ce981d707..77cba71bcccb5de3bc85c55bf7a48cf6a0129919 100644 --- a/drivers/usb/serial/iuu_phoenix.c +++ b/drivers/usb/serial/iuu_phoenix.c @@ -879,7 +879,8 @@ static int iuu_uart_baud(struct usb_serial_port *port, u32 baud_base, } static void iuu_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { const u32 supported_mask = CMSPAR|PARENB|PARODD; struct iuu_private *priv = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/keyspan.c b/drivers/usb/serial/keyspan.c index 1cfcd805f286830710865be969ae42abd39b85bd..2966e0c4941ec6abdd736fa363de044f0a86e5e7 100644 --- a/drivers/usb/serial/keyspan.c +++ b/drivers/usb/serial/keyspan.c @@ -616,7 +616,8 @@ static void keyspan_break_ctl(struct tty_struct *tty, int break_state) static void keyspan_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { int baud_rate, device_port; struct keyspan_port_private *p_priv; diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c index 3e7628becdcd81a4bc6b01fa5ab54a7e53d72c64..6fd15cd9e1eb05872278b49bce30742467293ea3 100644 --- a/drivers/usb/serial/keyspan_pda.c +++ b/drivers/usb/serial/keyspan_pda.c @@ -321,7 +321,8 @@ static void keyspan_pda_break_ctl(struct tty_struct *tty, int break_state) } static void keyspan_pda_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; speed_t speed; diff --git a/drivers/usb/serial/kl5kusb105.c b/drivers/usb/serial/kl5kusb105.c index edcc57bd9b5e6dc4365630774cf017e15def3b78..394b3189e003a1b25cb2dac1868acdaa72bd4d31 100644 --- a/drivers/usb/serial/kl5kusb105.c +++ b/drivers/usb/serial/kl5kusb105.c @@ -56,7 +56,8 @@ static void klsi_105_port_remove(struct usb_serial_port *port); static int klsi_105_open(struct tty_struct *tty, struct usb_serial_port *port); static void klsi_105_close(struct usb_serial_port *port); static void klsi_105_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static int klsi_105_tiocmget(struct tty_struct *tty); static void klsi_105_process_read_urb(struct urb *urb); static int klsi_105_prepare_write_buffer(struct usb_serial_port *port, @@ -366,7 +367,7 @@ static void klsi_105_process_read_urb(struct urb *urb) static void klsi_105_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct klsi_105_private *priv = usb_get_serial_port_data(port); struct device *dev = &port->dev; diff --git a/drivers/usb/serial/kobil_sct.c b/drivers/usb/serial/kobil_sct.c index 4ed8b8b0a3616f2891a3539ffb7a4f559df1f832..5e775f68fcb8574136195f5fb97f4df050f3a64b 100644 --- a/drivers/usb/serial/kobil_sct.c +++ b/drivers/usb/serial/kobil_sct.c @@ -62,7 +62,8 @@ static int kobil_tiocmset(struct tty_struct *tty, static void kobil_read_int_callback(struct urb *urb); static void kobil_write_int_callback(struct urb *urb); static void kobil_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old); static void kobil_init_termios(struct tty_struct *tty); static const struct usb_device_id id_table[] = { @@ -474,7 +475,8 @@ static int kobil_tiocmset(struct tty_struct *tty, } static void kobil_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old) + struct usb_serial_port *port, + const struct ktermios *old) { struct kobil_private *priv; int result; diff --git a/drivers/usb/serial/mct_u232.c b/drivers/usb/serial/mct_u232.c index ecd5b921e3746b8fa1d5f5c5e3c2ee6af0c964f6..d3852feb81a4a2607272fa2d2a01e8fdb8af7610 100644 --- a/drivers/usb/serial/mct_u232.c +++ b/drivers/usb/serial/mct_u232.c @@ -45,7 +45,8 @@ static void mct_u232_close(struct usb_serial_port *port); static void mct_u232_dtr_rts(struct usb_serial_port *port, int on); static void mct_u232_read_int_callback(struct urb *urb); static void mct_u232_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static void mct_u232_break_ctl(struct tty_struct *tty, int break_state); static int mct_u232_tiocmget(struct tty_struct *tty); static int mct_u232_tiocmset(struct tty_struct *tty, @@ -593,7 +594,7 @@ exit: static void mct_u232_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct mct_u232_private *priv = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c index 23ccbba716c7e439ca0f79d7c056a3fd91b1e77a..1d1f85fabc284c812b8c080301437dbbbfbeec65 100644 --- a/drivers/usb/serial/mos7720.c +++ b/drivers/usb/serial/mos7720.c @@ -1356,7 +1356,7 @@ static int send_cmd_write_baud_rate(struct moschip_port *mos7720_port, */ static void change_port_settings(struct tty_struct *tty, struct moschip_port *mos7720_port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct usb_serial_port *port; struct usb_serial *serial; @@ -1494,7 +1494,8 @@ static void change_port_settings(struct tty_struct *tty, * termios structure. */ static void mos7720_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { int status; struct moschip_port *mos7720_port; diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c index 925067a7978d4ad395f644be59b205b8b685643c..6b12bb4648b8326085b58375d452c2beaff0cb85 100644 --- a/drivers/usb/serial/mos7840.c +++ b/drivers/usb/serial/mos7840.c @@ -1188,7 +1188,8 @@ static int mos7840_send_cmd_write_baud_rate(struct moschip_port *mos7840_port, *****************************************************************************/ static void mos7840_change_port_settings(struct tty_struct *tty, - struct moschip_port *mos7840_port, struct ktermios *old_termios) + struct moschip_port *mos7840_port, + const struct ktermios *old_termios) { struct usb_serial_port *port = mos7840_port->port; int baud; @@ -1330,7 +1331,7 @@ static void mos7840_change_port_settings(struct tty_struct *tty, static void mos7840_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct moschip_port *mos7840_port = usb_get_serial_port_data(port); int status; diff --git a/drivers/usb/serial/mxuport.c b/drivers/usb/serial/mxuport.c index eb45a9b0005c8886786a0f8f73369543cb23b13c..faa0eedfe245f9634cc0cacb6f8c172d64a529be 100644 --- a/drivers/usb/serial/mxuport.c +++ b/drivers/usb/serial/mxuport.c @@ -760,7 +760,7 @@ static int mxuport_tiocmget(struct tty_struct *tty) } static int mxuport_set_termios_flow(struct tty_struct *tty, - struct ktermios *old_termios, + const struct ktermios *old_termios, struct usb_serial_port *port, struct usb_serial *serial) { @@ -834,7 +834,7 @@ out: static void mxuport_set_termios(struct tty_struct *tty, struct usb_serial_port *port, - struct ktermios *old_termios) + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; u8 *buf; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index de59fa919540a3c895f59f82b62c0ddc34f059f5..697683e3fbffaff030640e9124027459c04e6c73 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -253,8 +253,10 @@ static void option_instat_callback(struct urb *urb); #define QUECTEL_PRODUCT_BG96 0x0296 #define QUECTEL_PRODUCT_EP06 0x0306 #define QUECTEL_PRODUCT_EM05G 0x030a +#define QUECTEL_PRODUCT_EM060K 0x030b #define QUECTEL_PRODUCT_EM12 0x0512 #define QUECTEL_PRODUCT_RM500Q 0x0800 +#define QUECTEL_PRODUCT_RM520N 0x0801 #define QUECTEL_PRODUCT_EC200S_CN 0x6002 #define QUECTEL_PRODUCT_EC200T 0x6026 #define QUECTEL_PRODUCT_RM500K 0x7001 @@ -438,6 +440,8 @@ static void option_instat_callback(struct urb *urb); #define CINTERION_PRODUCT_MV31_2_RMNET 0x00b9 #define CINTERION_PRODUCT_MV32_WA 0x00f1 #define CINTERION_PRODUCT_MV32_WB 0x00f2 +#define CINTERION_PRODUCT_MV32_WA_RMNET 0x00f3 +#define CINTERION_PRODUCT_MV32_WB_RMNET 0x00f4 /* Olivetti products */ #define OLIVETTI_VENDOR_ID 0x0b3c @@ -573,6 +577,10 @@ static void option_instat_callback(struct urb *urb); #define WETELECOM_PRODUCT_6802 0x6802 #define WETELECOM_PRODUCT_WMD300 0x6803 +/* OPPO products */ +#define OPPO_VENDOR_ID 0x22d9 +#define OPPO_PRODUCT_R11 0x276c + /* Device flags */ @@ -1131,6 +1139,8 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG95, 0xff, 0xff, 0xff), .driver_info = NUMEP2 }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EG95, 0xff, 0, 0) }, + { USB_DEVICE_INTERFACE_CLASS(QUECTEL_VENDOR_ID, 0x0203, 0xff), /* BG95-M3 */ + .driver_info = ZLP }, { USB_DEVICE(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_BG96), .driver_info = RSVD(4) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EP06, 0xff, 0xff, 0xff), @@ -1138,6 +1148,9 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EP06, 0xff, 0, 0) }, { USB_DEVICE_INTERFACE_CLASS(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM05G, 0xff), .driver_info = RSVD(6) | ZLP }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM060K, 0xff, 0x00, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM060K, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM060K, 0xff, 0xff, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM12, 0xff, 0xff, 0xff), .driver_info = RSVD(1) | RSVD(2) | RSVD(3) | RSVD(4) | NUMEP2 }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EM12, 0xff, 0, 0) }, @@ -1149,6 +1162,9 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500Q, 0xff, 0xff, 0x10), .driver_info = ZLP }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM520N, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM520N, 0xff, 0, 0x40) }, + { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM520N, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200S_CN, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_EC200T, 0xff, 0, 0) }, { USB_DEVICE_AND_INTERFACE_INFO(QUECTEL_VENDOR_ID, QUECTEL_PRODUCT_RM500K, 0xff, 0x00, 0x00) }, @@ -1993,8 +2009,12 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(0)}, { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WA, 0xff), .driver_info = RSVD(3)}, + { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WA_RMNET, 0xff), + .driver_info = RSVD(0) }, { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WB, 0xff), .driver_info = RSVD(3)}, + { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WB_RMNET, 0xff), + .driver_info = RSVD(0) }, { USB_DEVICE(OLIVETTI_VENDOR_ID, OLIVETTI_PRODUCT_OLICARD100), .driver_info = RSVD(4) }, { USB_DEVICE(OLIVETTI_VENDOR_ID, OLIVETTI_PRODUCT_OLICARD120), @@ -2155,6 +2175,7 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) }, /* GosunCn GM500 RNDIS */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1406, 0xff) }, /* GosunCn GM500 ECM/NCM */ + { USB_DEVICE_AND_INTERFACE_INFO(OPPO_VENDOR_ID, OPPO_PRODUCT_R11, 0xff, 0xff, 0x30) }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(usb, option_ids); diff --git a/drivers/usb/serial/oti6858.c b/drivers/usb/serial/oti6858.c index a5caedbe72e2972c1ee769f827a7d2a0a88a059a..6365cfe5402cb52aa6a177195a5c0c0ff8a52451 100644 --- a/drivers/usb/serial/oti6858.c +++ b/drivers/usb/serial/oti6858.c @@ -119,7 +119,8 @@ struct oti6858_control_pkt { static int oti6858_open(struct tty_struct *tty, struct usb_serial_port *port); static void oti6858_close(struct usb_serial_port *port); static void oti6858_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static void oti6858_init_termios(struct tty_struct *tty); static void oti6858_read_int_callback(struct urb *urb); static void oti6858_read_bulk_callback(struct urb *urb); @@ -395,7 +396,8 @@ static void oti6858_init_termios(struct tty_struct *tty) } static void oti6858_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct oti6858_private *priv = usb_get_serial_port_data(port); unsigned long flags; diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c index 40b1ab3d284dcb1ae1a7cc0e85cebcbb0bb774ac..8949c1891164bd9511720627c054bbd1339e8452 100644 --- a/drivers/usb/serial/pl2303.c +++ b/drivers/usb/serial/pl2303.c @@ -789,7 +789,8 @@ static bool pl2303_enable_xonxoff(struct tty_struct *tty, const struct pl2303_ty } static void pl2303_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct pl2303_serial_private *spriv = usb_get_serial_data(serial); diff --git a/drivers/usb/serial/qcserial.c b/drivers/usb/serial/qcserial.c index 586ef5551e76e41e5ac0a552429e3979bb03185b..b1e844bf31f81f7984a976bf4ca5dcd8f01b3a97 100644 --- a/drivers/usb/serial/qcserial.c +++ b/drivers/usb/serial/qcserial.c @@ -177,6 +177,7 @@ static const struct usb_device_id id_table[] = { {DEVICE_SWI(0x413c, 0x81b3)}, /* Dell Wireless 5809e Gobi(TM) 4G LTE Mobile Broadband Card (rev3) */ {DEVICE_SWI(0x413c, 0x81b5)}, /* Dell Wireless 5811e QDL */ {DEVICE_SWI(0x413c, 0x81b6)}, /* Dell Wireless 5811e QDL */ + {DEVICE_SWI(0x413c, 0x81c2)}, /* Dell Wireless 5811e */ {DEVICE_SWI(0x413c, 0x81cb)}, /* Dell Wireless 5816e QDL */ {DEVICE_SWI(0x413c, 0x81cc)}, /* Dell Wireless 5816e */ {DEVICE_SWI(0x413c, 0x81cf)}, /* Dell Wireless 5819 */ diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c index 36b1e064e51fb5d25bb230e047747b317a911c07..6fca40ace83a01fb341318f265c6e44d2cf8dc31 100644 --- a/drivers/usb/serial/quatech2.c +++ b/drivers/usb/serial/quatech2.c @@ -261,8 +261,8 @@ static int qt2_calc_num_ports(struct usb_serial *serial, } static void qt2_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_device *dev = port->serial->dev; struct qt2_port_private *port_priv; diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c index 7039dc91882775cba850c2b9697d3d2d49092436..09a972a838ee4c18b85f10008af2a08ab69342fe 100644 --- a/drivers/usb/serial/spcp8x5.c +++ b/drivers/usb/serial/spcp8x5.c @@ -283,7 +283,8 @@ static void spcp8x5_init_termios(struct tty_struct *tty) } static void spcp8x5_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_serial *serial = port->serial; struct spcp8x5_private *priv = usb_get_serial_port_data(port); diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c index 181e302136a5b8444004a51a9be79a58c90823b1..1e1888b66305d846484f300853048b2015e99faa 100644 --- a/drivers/usb/serial/ssu100.c +++ b/drivers/usb/serial/ssu100.c @@ -214,8 +214,8 @@ out: kfree(data); static void ssu100_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct usb_device *dev = port->serial->dev; struct ktermios *termios = &tty->termios; diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c index 18c0bd8533924439b73f9579a1a5234f249e48d5..b99f78224846d55e088cc73cfcf6d1cef662f84d 100644 --- a/drivers/usb/serial/ti_usb_3410_5052.c +++ b/drivers/usb/serial/ti_usb_3410_5052.c @@ -314,7 +314,8 @@ static bool ti_tx_empty(struct usb_serial_port *port); static void ti_throttle(struct tty_struct *tty); static void ti_unthrottle(struct tty_struct *tty); static void ti_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); static int ti_tiocmget(struct tty_struct *tty); static int ti_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); @@ -892,7 +893,8 @@ static void ti_unthrottle(struct tty_struct *tty) } static void ti_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct ti_port *tport = usb_get_serial_port_data(port); struct ti_uart_config *config; diff --git a/drivers/usb/serial/upd78f0730.c b/drivers/usb/serial/upd78f0730.c index 63d4a784ae457ccae525ba0cd24089c05a4ab959..c47439bd90fa67b2914169d56cdf6c5fa94ca8a3 100644 --- a/drivers/usb/serial/upd78f0730.c +++ b/drivers/usb/serial/upd78f0730.c @@ -296,8 +296,8 @@ static speed_t upd78f0730_get_baud_rate(struct tty_struct *tty) } static void upd78f0730_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct device *dev = &port->dev; struct upd78f0730_line_control request; diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c index e35bea2235c1cad5055838b894940af104b9c370..164521ee10c6b49f48aecf2400eb757405dec34f 100644 --- a/drivers/usb/serial/usb-serial.c +++ b/drivers/usb/serial/usb-serial.c @@ -519,7 +519,8 @@ static int serial_ioctl(struct tty_struct *tty, return retval; } -static void serial_set_termios(struct tty_struct *tty, struct ktermios *old) +static void serial_set_termios(struct tty_struct *tty, + const struct ktermios *old) { struct usb_serial_port *port = tty->driver_data; diff --git a/drivers/usb/serial/whiteheat.c b/drivers/usb/serial/whiteheat.c index 332fb92ae575f9bd0e62f66abf435879386af595..7f82d40753ee08949081097cf38455e30cc3921b 100644 --- a/drivers/usb/serial/whiteheat.c +++ b/drivers/usb/serial/whiteheat.c @@ -82,7 +82,8 @@ static void whiteheat_close(struct usb_serial_port *port); static void whiteheat_get_serial(struct tty_struct *tty, struct serial_struct *ss); static void whiteheat_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + struct usb_serial_port *port, + const struct ktermios *old_termios); static int whiteheat_tiocmget(struct tty_struct *tty); static int whiteheat_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); @@ -442,7 +443,8 @@ static void whiteheat_get_serial(struct tty_struct *tty, struct serial_struct *s static void whiteheat_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { firm_setup_port(tty); } diff --git a/drivers/usb/serial/xr_serial.c b/drivers/usb/serial/xr_serial.c index 6853cd56d8dce50c8d7377e6806d0eaada759add..f3811e060a44164ff110c7cd2601ce10ac25d795 100644 --- a/drivers/usb/serial/xr_serial.c +++ b/drivers/usb/serial/xr_serial.c @@ -104,7 +104,8 @@ static int xr21v141x_uart_enable(struct usb_serial_port *port); static int xr21v141x_uart_disable(struct usb_serial_port *port); static int xr21v141x_fifo_reset(struct usb_serial_port *port); static void xr21v141x_set_line_settings(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); struct xr_type { int reg_width; @@ -133,8 +134,8 @@ struct xr_type { int (*disable)(struct usb_serial_port *port); int (*fifo_reset)(struct usb_serial_port *port); void (*set_line_settings)(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios); + struct usb_serial_port *port, + const struct ktermios *old_termios); }; enum xr_type_id { @@ -622,8 +623,8 @@ static int xr21v141x_set_baudrate(struct tty_struct *tty, struct usb_serial_port } static void xr_set_flow_mode(struct tty_struct *tty, - struct usb_serial_port *port, - struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct xr_data *data = usb_get_serial_port_data(port); const struct xr_type *type = data->type; @@ -674,7 +675,8 @@ static void xr_set_flow_mode(struct tty_struct *tty, } static void xr21v141x_set_line_settings(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct ktermios *termios = &tty->termios; u8 bits = 0; @@ -732,7 +734,8 @@ static void xr21v141x_set_line_settings(struct tty_struct *tty, } static void xr_cdc_set_line_coding(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct xr_data *data = usb_get_serial_port_data(port); struct usb_host_interface *alt = port->serial->interface->cur_altsetting; @@ -809,7 +812,8 @@ static void xr_cdc_set_line_coding(struct tty_struct *tty, } static void xr_set_termios(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old_termios) + struct usb_serial_port *port, + const struct ktermios *old_termios) { struct xr_data *data = usb_get_serial_port_data(port); diff --git a/drivers/usb/storage/onetouch.c b/drivers/usb/storage/onetouch.c index 1db2eefeea22f0bb3528dc413dbf76c97989414a..01f3c2779ccf3e76df5ba41c0ca2f0ee8a40e7f8 100644 --- a/drivers/usb/storage/onetouch.c +++ b/drivers/usb/storage/onetouch.c @@ -201,7 +201,7 @@ static int onetouch_connect_input(struct us_data *ss) onetouch->dev = input_dev; if (udev->manufacturer) - strlcpy(onetouch->name, udev->manufacturer, + strscpy(onetouch->name, udev->manufacturer, sizeof(onetouch->name)); if (udev->product) { if (udev->manufacturer) diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c index 84dc270f6f73c4e12daea56091b43b211e9a52a2..de3836412bf32ecc97b12971ae3b7303fbb01fd3 100644 --- a/drivers/usb/storage/uas.c +++ b/drivers/usb/storage/uas.c @@ -283,7 +283,7 @@ static bool uas_evaluate_response_iu(struct response_iu *riu, struct scsi_cmnd * set_host_byte(cmnd, DID_OK); break; case RC_TMF_NOT_SUPPORTED: - set_host_byte(cmnd, DID_TARGET_FAILURE); + set_host_byte(cmnd, DID_BAD_TARGET); break; default: uas_log_cmd_state(cmnd, "response iu", response_code); diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 1a05e3dcfec8a10652ec437bc0d80ce38c401d50..20dcbccb290b36b9200a33c61a5fe2e35b0fe151 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -1275,12 +1275,6 @@ UNUSUAL_DEV( 0x090a, 0x1200, 0x0000, 0x9999, USB_SC_RBC, USB_PR_BULK, NULL, 0 ), -UNUSUAL_DEV(0x090c, 0x1000, 0x1100, 0x1100, - "Samsung", - "Flash Drive FIT", - USB_SC_DEVICE, USB_PR_DEVICE, NULL, - US_FL_MAX_SECTORS_64), - /* aeb */ UNUSUAL_DEV( 0x090c, 0x1132, 0x0000, 0xffff, "Feiya", @@ -2294,6 +2288,13 @@ UNUSUAL_DEV( 0x1e74, 0x4621, 0x0000, 0x0000, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_BULK_IGNORE_TAG | US_FL_MAX_SECTORS_64 ), +/* Reported by Witold Lipieta */ +UNUSUAL_DEV( 0x1fc9, 0x0117, 0x0100, 0x0100, + "NXP Semiconductors", + "PN7462AU", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_RESIDUE ), + /* Supplied with some Castlewood ORB removable drives */ UNUSUAL_DEV( 0x2027, 0xa001, 0x0000, 0x9999, "Double-H Technology", diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h index 4051c8cd0cd8ad20623f26f236e12cc58b4a1bad..251778d14e2dd41f848286fa5d2ea50c93c40e61 100644 --- a/drivers/usb/storage/unusual_uas.h +++ b/drivers/usb/storage/unusual_uas.h @@ -52,6 +52,13 @@ UNUSUAL_DEV(0x059f, 0x1061, 0x0000, 0x9999, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NO_REPORT_OPCODES | US_FL_NO_SAME), +/* Reported-by: Hongling Zeng */ +UNUSUAL_DEV(0x090c, 0x2000, 0x0000, 0x9999, + "Hiksemi", + "External HDD", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_UAS), + /* * Apricorn USB3 dongle sometimes returns "USBSUSBSUSBS" in response to SCSI * commands in UAS mode. Observed with the 1.28 firmware; are there others? @@ -62,6 +69,13 @@ UNUSUAL_DEV(0x0984, 0x0301, 0x0128, 0x0128, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_IGNORE_UAS), +/* Reported-by: Tom Hu */ +UNUSUAL_DEV(0x0b05, 0x1932, 0x0000, 0x9999, + "ASUS", + "External HDD", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_UAS), + /* Reported-by: David Webb */ UNUSUAL_DEV(0x0bc2, 0x331a, 0x0000, 0x9999, "Seagate", @@ -69,6 +83,13 @@ UNUSUAL_DEV(0x0bc2, 0x331a, 0x0000, 0x9999, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NO_REPORT_LUNS), +/* Reported-by: Hongling Zeng */ +UNUSUAL_DEV(0x0bda, 0x9210, 0x0000, 0x9999, + "Hiksemi", + "External HDD", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_UAS), + /* Reported-by: Benjamin Tissoires */ UNUSUAL_DEV(0x13fd, 0x3940, 0x0000, 0x9999, "Initio Corporation", @@ -111,6 +132,13 @@ UNUSUAL_DEV(0x154b, 0xf00d, 0x0000, 0x9999, USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_NO_ATA_1X), +/* Reported-by: Hongling Zeng */ +UNUSUAL_DEV(0x17ef, 0x3899, 0x0000, 0x9999, + "Thinkplus", + "External HDD", + USB_SC_DEVICE, USB_PR_DEVICE, NULL, + US_FL_IGNORE_UAS), + /* Reported-by: Hans de Goede */ UNUSUAL_DEV(0x2109, 0x0711, 0x0000, 0x9999, "VIA", diff --git a/drivers/usb/typec/Kconfig b/drivers/usb/typec/Kconfig index 5defdfead6538b34a86608d7ea5e3a344075c16a..831e7049977df59b9dcdfe848ce5f805733b731e 100644 --- a/drivers/usb/typec/Kconfig +++ b/drivers/usb/typec/Kconfig @@ -56,6 +56,7 @@ config TYPEC_ANX7411 tristate "Analogix ANX7411 Type-C DRP Port controller driver" depends on I2C depends on USB_ROLE_SWITCH + depends on POWER_SUPPLY help Say Y or M here if your system has Analogix ANX7411 Type-C DRP Port controller driver. diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c index c1d8c23baa399cdafcb64c628a49e7ab9e3909fd..de66a2949e33b2b2f029056af449228b851904f6 100644 --- a/drivers/usb/typec/altmodes/displayport.c +++ b/drivers/usb/typec/altmodes/displayport.c @@ -99,8 +99,8 @@ static int dp_altmode_configure(struct dp_altmode *dp, u8 con) case DP_STATUS_CON_UFP_D: case DP_STATUS_CON_BOTH: /* NOTE: First acting as DP source */ conf |= DP_CONF_UFP_U_AS_UFP_D; - pin_assign = DP_CAP_DFP_D_PIN_ASSIGN(dp->alt->vdo) & - DP_CAP_UFP_D_PIN_ASSIGN(dp->port->vdo); + pin_assign = DP_CAP_PIN_ASSIGN_UFP_D(dp->alt->vdo) & + DP_CAP_PIN_ASSIGN_DFP_D(dp->port->vdo); break; default: break; diff --git a/drivers/usb/typec/anx7411.c b/drivers/usb/typec/anx7411.c index c0f0842d443c68a5de19034dcbe30ad7dc0e5735..b8f3b75fd7eb910d09042fc63b1bba004f5eed0d 100644 --- a/drivers/usb/typec/anx7411.c +++ b/drivers/usb/typec/anx7411.c @@ -1105,7 +1105,7 @@ static int anx7411_typec_switch_probe(struct anx7411_data *ctx, int ret; struct device_node *node; - node = of_find_node_by_name(dev->of_node, "orientation_switch"); + node = of_get_child_by_name(dev->of_node, "orientation_switch"); if (!node) return 0; @@ -1115,7 +1115,7 @@ static int anx7411_typec_switch_probe(struct anx7411_data *ctx, return ret; } - node = of_find_node_by_name(dev->of_node, "mode_switch"); + node = of_get_child_by_name(dev->of_node, "mode_switch"); if (!node) { dev_err(dev, "no typec mux exist"); ret = -ENODEV; @@ -1541,7 +1541,7 @@ free_i2c_dummy: return ret; } -static int anx7411_i2c_remove(struct i2c_client *client) +static void anx7411_i2c_remove(struct i2c_client *client) { struct anx7411_data *plat = i2c_get_clientdata(client); @@ -1565,8 +1565,6 @@ static int anx7411_i2c_remove(struct i2c_client *client) typec_unregister_port(plat->typec.port); anx7411_port_unregister_altmodes(plat->typec.port_amode); - - return 0; } static const struct i2c_device_id anx7411_id[] = { diff --git a/drivers/usb/typec/class.c b/drivers/usb/typec/class.c index ebc29ec20e3fb4306e8b66059840d7802dfd5172..bd5e5dd70431357001f5e22a3e037772859c1e1e 100644 --- a/drivers/usb/typec/class.c +++ b/drivers/usb/typec/class.c @@ -2346,6 +2346,7 @@ static void __exit typec_exit(void) ida_destroy(&typec_index_ida); bus_unregister(&typec_bus); class_unregister(&typec_mux_class); + class_unregister(&retimer_class); } module_exit(typec_exit); diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index cd47c3597e197cc4565f11fa621a37fe8131f804..2a58185fb14c4f1edeef173afa76c25db038ebf0 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -245,14 +245,12 @@ err_put_fwnode: return ret; } -static int hd3ss3220_remove(struct i2c_client *client) +static void hd3ss3220_remove(struct i2c_client *client) { struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client); typec_unregister_port(hd3ss3220->port); usb_role_switch_put(hd3ss3220->role_sw); - - return 0; } static const struct of_device_id dev_ids[] = { diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 464330776cd615eb09d6dd0a332f297fa2f9545c..941735c731619c9841d57d32599006c2180bbe31 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -29,7 +29,7 @@ static int switch_fwnode_match(struct device *dev, const void *fwnode) if (!is_typec_switch_dev(dev)) return 0; - return dev_fwnode(dev) == fwnode; + return device_match_fwnode(dev, fwnode); } static void *typec_switch_match(struct fwnode_handle *fwnode, const char *id, @@ -259,7 +259,7 @@ static int mux_fwnode_match(struct device *dev, const void *fwnode) if (!is_typec_mux_dev(dev)) return 0; - return dev_fwnode(dev) == fwnode; + return device_match_fwnode(dev, fwnode); } static void *typec_mux_match(struct fwnode_handle *fwnode, const char *id, diff --git a/drivers/usb/typec/mux/fsa4480.c b/drivers/usb/typec/mux/fsa4480.c index 6184f536719056b8755f44d4e07a1e00a4cb8064..d6495e533e585a88a317bd0bda0974d0dd81513f 100644 --- a/drivers/usb/typec/mux/fsa4480.c +++ b/drivers/usb/typec/mux/fsa4480.c @@ -181,14 +181,12 @@ static int fsa4480_probe(struct i2c_client *client) return 0; } -static int fsa4480_remove(struct i2c_client *client) +static void fsa4480_remove(struct i2c_client *client) { struct fsa4480 *fsa = i2c_get_clientdata(client); typec_mux_unregister(fsa->mux); typec_switch_unregister(fsa->sw); - - return 0; } static const struct i2c_device_id fsa4480_table[] = { diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c index 47b733f78fb0dbe86fb0066a254278e343c46762..e1f4df7238bf4d1b29c502814d07651daaa81ab1 100644 --- a/drivers/usb/typec/mux/intel_pmc_mux.c +++ b/drivers/usb/typec/mux/intel_pmc_mux.c @@ -569,13 +569,6 @@ err_unregister_switch: return ret; } -static int is_memory(struct acpi_resource *res, void *data) -{ - struct resource r; - - return !acpi_dev_resource_memory(res, &r); -} - /* IOM ACPI IDs and IOM_PORT_STATUS_OFFSET */ static const struct acpi_device_id iom_acpi_ids[] = { /* TigerLake */ @@ -583,6 +576,9 @@ static const struct acpi_device_id iom_acpi_ids[] = { /* AlderLake */ { "INTC1079", 0x160, }, + + /* Meteor Lake */ + { "INTC107A", 0x160, }, {} }; @@ -606,7 +602,7 @@ static int pmc_usb_probe_iom(struct pmc_usb *pmc) return -ENODEV; INIT_LIST_HEAD(&resource_list); - ret = acpi_dev_get_resources(adev, &resource_list, is_memory, NULL); + ret = acpi_dev_get_memory_resources(adev, &resource_list); if (ret < 0) return ret; diff --git a/drivers/usb/typec/mux/pi3usb30532.c b/drivers/usb/typec/mux/pi3usb30532.c index 6ce9f282594eaf91b45e7a231da9805cbd1d913f..1cd388b55c307748ac5841b759194311105ef8ff 100644 --- a/drivers/usb/typec/mux/pi3usb30532.c +++ b/drivers/usb/typec/mux/pi3usb30532.c @@ -160,13 +160,12 @@ static int pi3usb30532_probe(struct i2c_client *client) return 0; } -static int pi3usb30532_remove(struct i2c_client *client) +static void pi3usb30532_remove(struct i2c_client *client) { struct pi3usb30532 *pi = i2c_get_clientdata(client); typec_mux_unregister(pi->mux); typec_switch_unregister(pi->sw); - return 0; } static const struct i2c_device_id pi3usb30532_table[] = { diff --git a/drivers/usb/typec/qcom-pmic-typec.c b/drivers/usb/typec/qcom-pmic-typec.c index a0454a80c4a21b3d609ae3506c6dee03da4bae10..432ea62f1bab66791dc7b85344c50526188f0c3c 100644 --- a/drivers/usb/typec/qcom-pmic-typec.c +++ b/drivers/usb/typec/qcom-pmic-typec.c @@ -195,9 +195,8 @@ static int qcom_pmic_typec_probe(struct platform_device *pdev) qcom_usb->role_sw = fwnode_usb_role_switch_get(dev_fwnode(qcom_usb->dev)); if (IS_ERR(qcom_usb->role_sw)) { - if (PTR_ERR(qcom_usb->role_sw) != -EPROBE_DEFER) - dev_err(dev, "failed to get role switch\n"); - ret = PTR_ERR(qcom_usb->role_sw); + ret = dev_err_probe(dev, PTR_ERR(qcom_usb->role_sw), + "failed to get role switch\n"); goto err_typec_port; } diff --git a/drivers/usb/typec/retimer.c b/drivers/usb/typec/retimer.c index 2003731f1bee148a471b285652852561b4af6779..ee94dbbe4745306115bb4be5e0796e247b4b9084 100644 --- a/drivers/usb/typec/retimer.c +++ b/drivers/usb/typec/retimer.c @@ -31,7 +31,7 @@ static bool dev_name_ends_with(struct device *dev, const char *suffix) static int retimer_fwnode_match(struct device *dev, const void *fwnode) { - return dev_fwnode(dev) == fwnode && dev_name_ends_with(dev, "-retimer"); + return device_match_fwnode(dev, fwnode) && dev_name_ends_with(dev, "-retimer"); } static void *typec_retimer_match(struct fwnode_handle *fwnode, const char *id, void *data) diff --git a/drivers/usb/typec/rt1719.c b/drivers/usb/typec/rt1719.c index f1b698edd7ebfdc879db33b8d01efa157debf97a..ea8b700b0cebc15e6917432964266a3a4c35f1c6 100644 --- a/drivers/usb/typec/rt1719.c +++ b/drivers/usb/typec/rt1719.c @@ -930,14 +930,12 @@ err_fwnode_put: return ret; } -static int rt1719_remove(struct i2c_client *i2c) +static void rt1719_remove(struct i2c_client *i2c) { struct rt1719_data *data = i2c_get_clientdata(i2c); typec_unregister_port(data->port); usb_role_switch_put(data->role_sw); - - return 0; } static const struct of_device_id __maybe_unused rt1719_device_table[] = { diff --git a/drivers/usb/typec/stusb160x.c b/drivers/usb/typec/stusb160x.c index e7745d1c2a5c4683ba9f769629886a5cd8651b8c..494b371151e0a1d627ce3e8f7f306124f61d76ed 100644 --- a/drivers/usb/typec/stusb160x.c +++ b/drivers/usb/typec/stusb160x.c @@ -750,11 +750,8 @@ static int stusb160x_probe(struct i2c_client *client) if (client->irq) { chip->role_sw = fwnode_usb_role_switch_get(fwnode); if (IS_ERR(chip->role_sw)) { - ret = PTR_ERR(chip->role_sw); - if (ret != -EPROBE_DEFER) - dev_err(chip->dev, - "Failed to get usb role switch: %d\n", - ret); + ret = dev_err_probe(chip->dev, PTR_ERR(chip->role_sw), + "Failed to get usb role switch\n"); goto port_unregister; } @@ -801,7 +798,7 @@ fwnode_put: return ret; } -static int stusb160x_remove(struct i2c_client *client) +static void stusb160x_remove(struct i2c_client *client) { struct stusb160x *chip = i2c_get_clientdata(client); @@ -823,8 +820,6 @@ static int stusb160x_remove(struct i2c_client *client) if (chip->main_supply) regulator_disable(chip->main_supply); - - return 0; } static int __maybe_unused stusb160x_suspend(struct device *dev) diff --git a/drivers/usb/typec/tcpm/Kconfig b/drivers/usb/typec/tcpm/Kconfig index 073fd2ea5e0bbda1c4f8c41c4e0b46762aab1d64..e6b88ca4a4b94662e0957ba79e91ab3cd3ce7877 100644 --- a/drivers/usb/typec/tcpm/Kconfig +++ b/drivers/usb/typec/tcpm/Kconfig @@ -35,6 +35,17 @@ config TYPEC_MT6360 USB Type-C. It works with Type-C Port Controller Manager to provide USB PD and USB Type-C functionalities. +config TYPEC_TCPCI_MT6370 + tristate "MediaTek MT6370 Type-C driver" + depends on MFD_MT6370 + help + MediaTek MT6370 is a multi-functional IC that includes + USB Type-C. It works with Type-C Port Controller Manager + to provide USB PD and USB Type-C functionalities. + + This driver can also be built as a module. The module + will be called "tcpci_mt6370". + config TYPEC_TCPCI_MAXIM tristate "Maxim TCPCI based Type-C chip driver" help diff --git a/drivers/usb/typec/tcpm/Makefile b/drivers/usb/typec/tcpm/Makefile index 7d499f3569fdeb943c9e18c324630521e5caf506..906d9dced8e7712087e1a8d50c1b3b9d27e6127d 100644 --- a/drivers/usb/typec/tcpm/Makefile +++ b/drivers/usb/typec/tcpm/Makefile @@ -6,4 +6,5 @@ typec_wcove-y := wcove.o obj-$(CONFIG_TYPEC_TCPCI) += tcpci.o obj-$(CONFIG_TYPEC_RT1711H) += tcpci_rt1711h.o obj-$(CONFIG_TYPEC_MT6360) += tcpci_mt6360.o +obj-$(CONFIG_TYPEC_TCPCI_MT6370) += tcpci_mt6370.o obj-$(CONFIG_TYPEC_TCPCI_MAXIM) += tcpci_maxim.o diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 96c55eaf3f808de4ccac0e84192a426ec2273d33..721b2a548084c5a58ff6e0c0167020b5dfcaf7b8 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -151,7 +151,7 @@ static void _fusb302_log(struct fusb302_chip *chip, const char *fmt, if (fusb302_log_full(chip)) { chip->logbuffer_head = max(chip->logbuffer_head - 1, 0); - strlcpy(tmpbuffer, "overflow", sizeof(tmpbuffer)); + strscpy(tmpbuffer, "overflow", sizeof(tmpbuffer)); } if (chip->logbuffer_head < 0 || @@ -1743,9 +1743,8 @@ static int fusb302_probe(struct i2c_client *client, chip->tcpm_port = tcpm_register_port(&client->dev, &chip->tcpc_dev); if (IS_ERR(chip->tcpm_port)) { fwnode_handle_put(chip->tcpc_dev.fwnode); - ret = PTR_ERR(chip->tcpm_port); - if (ret != -EPROBE_DEFER) - dev_err(dev, "cannot register tcpm port, ret=%d", ret); + ret = dev_err_probe(dev, PTR_ERR(chip->tcpm_port), + "cannot register tcpm port\n"); goto destroy_workqueue; } @@ -1771,7 +1770,7 @@ destroy_workqueue: return ret; } -static int fusb302_remove(struct i2c_client *client) +static void fusb302_remove(struct i2c_client *client) { struct fusb302_chip *chip = i2c_get_clientdata(client); @@ -1783,8 +1782,6 @@ static int fusb302_remove(struct i2c_client *client) fwnode_handle_put(chip->tcpc_dev.fwnode); destroy_workqueue(chip->wq); fusb302_debugfs_exit(chip); - - return 0; } static int fusb302_pm_suspend(struct device *dev) diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 812784702d539ec3311179c03dee87a1aa7cb5b5..b2bfcebe218f04f8009b026fea352cae6e6f32d8 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -27,11 +27,6 @@ #define VPPS_VALID_MIN_MV 100 #define VSINKDISCONNECT_PD_MIN_PERCENT 90 -#define tcpc_presenting_rd(reg, cc) \ - (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ - (((reg) & (TCPC_ROLE_CTRL_## cc ##_MASK << TCPC_ROLE_CTRL_## cc ##_SHIFT)) == \ - (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_## cc ##_SHIFT))) - struct tcpci { struct device *dev; @@ -218,23 +213,6 @@ static int tcpci_start_toggling(struct tcpc_dev *tcpc, TCPC_CMD_LOOK4CONNECTION); } -static enum typec_cc_status tcpci_to_typec_cc(unsigned int cc, bool sink) -{ - switch (cc) { - case 0x1: - return sink ? TYPEC_CC_RP_DEF : TYPEC_CC_RA; - case 0x2: - return sink ? TYPEC_CC_RP_1_5 : TYPEC_CC_RD; - case 0x3: - if (sink) - return TYPEC_CC_RP_3_0; - fallthrough; - case 0x0: - default: - return TYPEC_CC_OPEN; - } -} - static int tcpci_get_cc(struct tcpc_dev *tcpc, enum typec_cc_status *cc1, enum typec_cc_status *cc2) { @@ -868,7 +846,7 @@ static int tcpci_probe(struct i2c_client *client, return 0; } -static int tcpci_remove(struct i2c_client *client) +static void tcpci_remove(struct i2c_client *client) { struct tcpci_chip *chip = i2c_get_clientdata(client); int err; @@ -879,8 +857,6 @@ static int tcpci_remove(struct i2c_client *client) dev_warn(&client->dev, "Failed to disable irqs (%pe)\n", ERR_PTR(err)); tcpci_unregister_port(chip->tcpci); - - return 0; } static const struct i2c_device_id tcpci_id[] = { diff --git a/drivers/usb/typec/tcpm/tcpci_maxim.c b/drivers/usb/typec/tcpm/tcpci_maxim.c index 4b6705f3d7b78000a1855ef7441de8a9676a3492..03f89e6f1a78e53d72db9d6ee6d742cb24a69161 100644 --- a/drivers/usb/typec/tcpm/tcpci_maxim.c +++ b/drivers/usb/typec/tcpm/tcpci_maxim.c @@ -492,14 +492,12 @@ unreg_port: return ret; } -static int max_tcpci_remove(struct i2c_client *client) +static void max_tcpci_remove(struct i2c_client *client) { struct max_tcpci_chip *chip = i2c_get_clientdata(client); if (!IS_ERR_OR_NULL(chip->tcpci)) tcpci_unregister_port(chip->tcpci); - - return 0; } static const struct i2c_device_id max_tcpci_id[] = { diff --git a/drivers/usb/typec/tcpm/tcpci_mt6370.c b/drivers/usb/typec/tcpm/tcpci_mt6370.c new file mode 100644 index 0000000000000000000000000000000000000000..c5bb201a5163c251bea110406f6a9fa2d3ee4a15 --- /dev/null +++ b/drivers/usb/typec/tcpm/tcpci_mt6370.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * + * Author: ChiYuan Huang + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MT6370_REG_SYSCTRL8 0x9B + +#define MT6370_AUTOIDLE_MASK BIT(3) + +#define MT6370_VENDOR_ID 0x29CF +#define MT6370_TCPC_DID_A 0x2170 + +struct mt6370_priv { + struct device *dev; + struct regulator *vbus; + struct tcpci *tcpci; + struct tcpci_data tcpci_data; +}; + +static const struct reg_sequence mt6370_reg_init[] = { + REG_SEQ(0xA0, 0x1, 1000), + REG_SEQ(0x81, 0x38, 0), + REG_SEQ(0x82, 0x82, 0), + REG_SEQ(0xBA, 0xFC, 0), + REG_SEQ(0xBB, 0x50, 0), + REG_SEQ(0x9E, 0x8F, 0), + REG_SEQ(0xA1, 0x5, 0), + REG_SEQ(0xA2, 0x4, 0), + REG_SEQ(0xA3, 0x4A, 0), + REG_SEQ(0xA4, 0x01, 0), + REG_SEQ(0x95, 0x01, 0), + REG_SEQ(0x80, 0x71, 0), + REG_SEQ(0x9B, 0x3A, 1000), +}; + +static int mt6370_tcpc_init(struct tcpci *tcpci, struct tcpci_data *data) +{ + u16 did; + int ret; + + ret = regmap_register_patch(data->regmap, mt6370_reg_init, + ARRAY_SIZE(mt6370_reg_init)); + if (ret) + return ret; + + ret = regmap_raw_read(data->regmap, TCPC_BCD_DEV, &did, sizeof(u16)); + if (ret) + return ret; + + if (did == MT6370_TCPC_DID_A) + return regmap_write(data->regmap, TCPC_FAULT_CTRL, 0x80); + + return 0; +} + +static int mt6370_tcpc_set_vconn(struct tcpci *tcpci, struct tcpci_data *data, + bool enable) +{ + return regmap_update_bits(data->regmap, MT6370_REG_SYSCTRL8, + MT6370_AUTOIDLE_MASK, + enable ? 0 : MT6370_AUTOIDLE_MASK); +} + +static int mt6370_tcpc_set_vbus(struct tcpci *tcpci, struct tcpci_data *data, + bool source, bool sink) +{ + struct mt6370_priv *priv = container_of(data, struct mt6370_priv, + tcpci_data); + int ret; + + ret = regulator_is_enabled(priv->vbus); + if (ret < 0) + return ret; + + if (ret && !source) + return regulator_disable(priv->vbus); + + if (!ret && source) + return regulator_enable(priv->vbus); + + return 0; +} + +static irqreturn_t mt6370_irq_handler(int irq, void *dev_id) +{ + struct mt6370_priv *priv = dev_id; + + return tcpci_irq(priv->tcpci); +} + +static int mt6370_check_vendor_info(struct mt6370_priv *priv) +{ + struct regmap *regmap = priv->tcpci_data.regmap; + u16 vid; + int ret; + + ret = regmap_raw_read(regmap, TCPC_VENDOR_ID, &vid, sizeof(u16)); + if (ret) + return ret; + + if (vid != MT6370_VENDOR_ID) + return dev_err_probe(priv->dev, -ENODEV, + "Vendor ID not correct 0x%02x\n", vid); + + return 0; +} + +static void mt6370_unregister_tcpci_port(void *tcpci) +{ + tcpci_unregister_port(tcpci); +} + +static int mt6370_tcpc_probe(struct platform_device *pdev) +{ + struct mt6370_priv *priv; + struct device *dev = &pdev->dev; + int irq, ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + + priv->tcpci_data.regmap = dev_get_regmap(dev->parent, NULL); + if (!priv->tcpci_data.regmap) + return dev_err_probe(dev, -ENODEV, "Failed to init regmap\n"); + + ret = mt6370_check_vendor_info(priv); + if (ret) + return ret; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return dev_err_probe(dev, irq, "Failed to get irq\n"); + + /* Assign TCPCI feature and ops */ + priv->tcpci_data.auto_discharge_disconnect = 1; + priv->tcpci_data.init = mt6370_tcpc_init; + priv->tcpci_data.set_vconn = mt6370_tcpc_set_vconn; + + priv->vbus = devm_regulator_get_optional(dev, "vbus"); + if (!IS_ERR(priv->vbus)) + priv->tcpci_data.set_vbus = mt6370_tcpc_set_vbus; + + priv->tcpci = tcpci_register_port(dev, &priv->tcpci_data); + if (IS_ERR(priv->tcpci)) + return dev_err_probe(dev, PTR_ERR(priv->tcpci), + "Failed to register tcpci port\n"); + + ret = devm_add_action_or_reset(dev, mt6370_unregister_tcpci_port, priv->tcpci); + if (ret) + return ret; + + ret = devm_request_threaded_irq(dev, irq, NULL, mt6370_irq_handler, + IRQF_ONESHOT, dev_name(dev), priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to allocate irq\n"); + + device_init_wakeup(dev, true); + dev_pm_set_wake_irq(dev, irq); + + return 0; +} + +static int mt6370_tcpc_remove(struct platform_device *pdev) +{ + dev_pm_clear_wake_irq(&pdev->dev); + device_init_wakeup(&pdev->dev, false); + + return 0; +} + +static const struct of_device_id mt6370_tcpc_devid_table[] = { + { .compatible = "mediatek,mt6370-tcpc" }, + {} +}; +MODULE_DEVICE_TABLE(of, mt6370_tcpc_devid_table); + +static struct platform_driver mt6370_tcpc_driver = { + .driver = { + .name = "mt6370-tcpc", + .of_match_table = mt6370_tcpc_devid_table, + }, + .probe = mt6370_tcpc_probe, + .remove = mt6370_tcpc_remove, +}; +module_platform_driver(mt6370_tcpc_driver); + +MODULE_AUTHOR("ChiYuan Huang "); +MODULE_DESCRIPTION("MT6370 USB Type-C Port Controller Interface Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c index 3291ca4948da7861cc2f540deb79b6f2a5465272..7b217c712c11a668ec902907cd7717babfda555d 100644 --- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c +++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c @@ -5,6 +5,7 @@ * Richtek RT1711H Type-C Chip Driver */ +#include #include #include #include @@ -13,16 +14,27 @@ #include #include #include +#include #define RT1711H_VID 0x29CF #define RT1711H_PID 0x1711 +#define RT1711H_DID 0x2171 +#define RT1715_DID 0x2173 -#define RT1711H_RTCTRL8 0x9B +#define RT1711H_PHYCTRL1 0x80 +#define RT1711H_PHYCTRL2 0x81 + +#define RT1711H_RTCTRL4 0x93 +/* rx threshold of rd/rp: 1b0 for level 0.4V/0.7V, 1b1 for 0.35V/0.75V */ +#define RT1711H_BMCIO_RXDZSEL BIT(0) +#define RT1711H_RTCTRL8 0x9B /* Autoidle timeout = (tout * 2 + 1) * 6.4ms */ #define RT1711H_RTCTRL8_SET(ck300, ship_off, auto_idle, tout) \ (((ck300) << 7) | ((ship_off) << 5) | \ ((auto_idle) << 3) | ((tout) & 0x07)) +#define RT1711H_AUTOIDLEEN BIT(3) +#define RT1711H_ENEXTMSG BIT(4) #define RT1711H_RTCTRL11 0x9E @@ -35,10 +47,17 @@ #define RT1711H_RTCTRL15 0xA2 #define RT1711H_RTCTRL16 0xA3 +#define RT1711H_RTCTRL18 0xAF +/* 1b0 as fixed rx threshold of rd/rp 0.55V, 1b1 depends on RTCRTL4[0] */ +#define BMCIO_RXDZEN BIT(0) + struct rt1711h_chip { struct tcpci_data data; struct tcpci *tcpci; struct device *dev; + struct regulator *vbus; + bool src_en; + u16 did; }; static int rt1711h_read16(struct rt1711h_chip *chip, unsigned int reg, u16 *val) @@ -75,8 +94,9 @@ static struct rt1711h_chip *tdata_to_rt1711h(struct tcpci_data *tdata) static int rt1711h_init(struct tcpci *tcpci, struct tcpci_data *tdata) { - int ret; struct rt1711h_chip *chip = tdata_to_rt1711h(tdata); + struct regmap *regmap = chip->data.regmap; + int ret; /* CK 300K from 320K, shipping off, auto_idle enable, tout = 32ms */ ret = rt1711h_write8(chip, RT1711H_RTCTRL8, @@ -84,6 +104,14 @@ static int rt1711h_init(struct tcpci *tcpci, struct tcpci_data *tdata) if (ret < 0) return ret; + /* Enable PD30 extended message for RT1715 */ + if (chip->did == RT1715_DID) { + ret = regmap_update_bits(regmap, RT1711H_RTCTRL8, + RT1711H_ENEXTMSG, RT1711H_ENEXTMSG); + if (ret < 0) + return ret; + } + /* I2C reset : (val + 1) * 12.5ms */ ret = rt1711h_write8(chip, RT1711H_RTCTRL11, RT1711H_RTCTRL11_SET(1, 0x0F)); @@ -101,7 +129,37 @@ static int rt1711h_init(struct tcpci *tcpci, struct tcpci_data *tdata) return ret; /* dcSRC.DRP : 33% */ - return rt1711h_write16(chip, RT1711H_RTCTRL16, 330); + ret = rt1711h_write16(chip, RT1711H_RTCTRL16, 330); + if (ret < 0) + return ret; + + /* Enable phy discard retry, retry count 7, rx filter deglitch 100 us */ + ret = rt1711h_write8(chip, RT1711H_PHYCTRL1, 0xF1); + if (ret < 0) + return ret; + + /* Decrease wait time of BMC-encoded 1 bit from 2.67us to 2.55us */ + /* wait time : (val * .4167) us */ + return rt1711h_write8(chip, RT1711H_PHYCTRL2, 62); +} + +static int rt1711h_set_vbus(struct tcpci *tcpci, struct tcpci_data *tdata, + bool src, bool snk) +{ + struct rt1711h_chip *chip = tdata_to_rt1711h(tdata); + int ret; + + if (chip->src_en == src) + return 0; + + if (src) + ret = regulator_enable(chip->vbus); + else + ret = regulator_disable(chip->vbus); + + if (!ret) + chip->src_en = src; + return ret; } static int rt1711h_set_vconn(struct tcpci *tcpci, struct tcpci_data *tdata, @@ -109,8 +167,55 @@ static int rt1711h_set_vconn(struct tcpci *tcpci, struct tcpci_data *tdata, { struct rt1711h_chip *chip = tdata_to_rt1711h(tdata); - return rt1711h_write8(chip, RT1711H_RTCTRL8, - RT1711H_RTCTRL8_SET(0, 1, !enable, 2)); + return regmap_update_bits(chip->data.regmap, RT1711H_RTCTRL8, + RT1711H_AUTOIDLEEN, enable ? 0 : RT1711H_AUTOIDLEEN); +} + +/* + * Selects the CC PHY noise filter voltage level according to the remote current + * CC voltage level. + * + * @status: The port's current cc status read from IC + * Return 0 if writes succeed; failure code otherwise + */ +static inline int rt1711h_init_cc_params(struct rt1711h_chip *chip, u8 status) +{ + int ret, cc1, cc2; + u8 role = 0; + u32 rxdz_en, rxdz_sel; + + ret = rt1711h_read8(chip, TCPC_ROLE_CTRL, &role); + if (ret < 0) + return ret; + + cc1 = tcpci_to_typec_cc((status >> TCPC_CC_STATUS_CC1_SHIFT) & + TCPC_CC_STATUS_CC1_MASK, + status & TCPC_CC_STATUS_TERM || + tcpc_presenting_rd(role, CC1)); + cc2 = tcpci_to_typec_cc((status >> TCPC_CC_STATUS_CC2_SHIFT) & + TCPC_CC_STATUS_CC2_MASK, + status & TCPC_CC_STATUS_TERM || + tcpc_presenting_rd(role, CC2)); + + if ((cc1 >= TYPEC_CC_RP_1_5 && cc2 < TYPEC_CC_RP_DEF) || + (cc2 >= TYPEC_CC_RP_1_5 && cc1 < TYPEC_CC_RP_DEF)) { + rxdz_en = BMCIO_RXDZEN; + if (chip->did == RT1715_DID) + rxdz_sel = RT1711H_BMCIO_RXDZSEL; + else + rxdz_sel = 0; + } else { + rxdz_en = 0; + rxdz_sel = RT1711H_BMCIO_RXDZSEL; + } + + ret = regmap_update_bits(chip->data.regmap, RT1711H_RTCTRL18, + BMCIO_RXDZEN, rxdz_en); + if (ret < 0) + return ret; + + return regmap_update_bits(chip->data.regmap, RT1711H_RTCTRL4, + RT1711H_BMCIO_RXDZSEL, rxdz_sel); } static int rt1711h_start_drp_toggling(struct tcpci *tcpci, @@ -173,6 +278,8 @@ static irqreturn_t rt1711h_irq(int irq, void *dev_id) /* Clear cc change event triggered by starting toggling */ if (status & TCPC_CC_STATUS_TOGGLING) rt1711h_write8(chip, TCPC_ALERT, TCPC_ALERT_CC_STATUS); + else + rt1711h_init_cc_params(chip, status); } out: @@ -191,7 +298,7 @@ static int rt1711h_sw_reset(struct rt1711h_chip *chip) return 0; } -static int rt1711h_check_revision(struct i2c_client *i2c) +static int rt1711h_check_revision(struct i2c_client *i2c, struct rt1711h_chip *chip) { int ret; @@ -209,7 +316,15 @@ static int rt1711h_check_revision(struct i2c_client *i2c) dev_err(&i2c->dev, "pid is not correct, 0x%04x\n", ret); return -ENODEV; } - return 0; + ret = i2c_smbus_read_word_data(i2c, TCPC_BCD_DEV); + if (ret < 0) + return ret; + if (ret != chip->did) { + dev_err(&i2c->dev, "did is not correct, 0x%04x\n", ret); + return -ENODEV; + } + dev_dbg(&i2c->dev, "did is 0x%04x\n", ret); + return ret; } static int rt1711h_probe(struct i2c_client *client, @@ -218,16 +333,18 @@ static int rt1711h_probe(struct i2c_client *client, int ret; struct rt1711h_chip *chip; - ret = rt1711h_check_revision(client); + chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + chip->did = (size_t)device_get_match_data(&client->dev); + + ret = rt1711h_check_revision(client, chip); if (ret < 0) { dev_err(&client->dev, "check vid/pid fail\n"); return ret; } - chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); - if (!chip) - return -ENOMEM; - chip->data.regmap = devm_regmap_init_i2c(client, &rt1711h_regmap_config); if (IS_ERR(chip->data.regmap)) @@ -245,7 +362,12 @@ static int rt1711h_probe(struct i2c_client *client, if (ret < 0) return ret; + chip->vbus = devm_regulator_get(&client->dev, "vbus"); + if (IS_ERR(chip->vbus)) + return PTR_ERR(chip->vbus); + chip->data.init = rt1711h_init; + chip->data.set_vbus = rt1711h_set_vbus; chip->data.set_vconn = rt1711h_set_vconn; chip->data.start_drp_toggling = rt1711h_start_drp_toggling; chip->tcpci = tcpci_register_port(chip->dev, &chip->data); @@ -263,23 +385,24 @@ static int rt1711h_probe(struct i2c_client *client, return 0; } -static int rt1711h_remove(struct i2c_client *client) +static void rt1711h_remove(struct i2c_client *client) { struct rt1711h_chip *chip = i2c_get_clientdata(client); tcpci_unregister_port(chip->tcpci); - return 0; } static const struct i2c_device_id rt1711h_id[] = { { "rt1711h", 0 }, + { "rt1715", 0 }, { } }; MODULE_DEVICE_TABLE(i2c, rt1711h_id); #ifdef CONFIG_OF static const struct of_device_id rt1711h_of_match[] = { - { .compatible = "richtek,rt1711h", }, + { .compatible = "richtek,rt1711h", .data = (void *)RT1711H_DID }, + { .compatible = "richtek,rt1715", .data = (void *)RT1715_DID }, {}, }; MODULE_DEVICE_TABLE(of, rt1711h_of_match); diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index ea5a917c51b1be7a763d739e047aa6d3997972ad..904c7b4ce2f0c278ed5f36f5d7ccb91ac02ff685 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -6320,6 +6320,13 @@ static int tcpm_psy_set_prop(struct power_supply *psy, struct tcpm_port *port = power_supply_get_drvdata(psy); int ret; + /* + * All the properties below are related to USB PD. The check needs to be + * property specific when a non-pd related property is added. + */ + if (!port->pd_supported) + return -EOPNOTSUPP; + switch (psp) { case POWER_SUPPLY_PROP_ONLINE: ret = tcpm_psy_set_online(port, val); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index dfbba5ae9487b7ba4d2aa2278348853e5fee8c72..b637e8b378b3a00210548d064b00715400a3562d 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -857,15 +857,13 @@ err_clear_mask: return ret; } -static int tps6598x_remove(struct i2c_client *client) +static void tps6598x_remove(struct i2c_client *client) { struct tps6598x *tps = i2c_get_clientdata(client); tps6598x_disconnect(tps, 0); typec_unregister_port(tps->port); usb_role_switch_put(tps->role_sw); - - return 0; } static const struct of_device_id tps6598x_of_match[] = { diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 1aea46493b852eaff005440be1a5f6995fb0a810..74fb5a4c6f21b05c02204c61f58927ce2aa364d2 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -588,8 +588,6 @@ static int ucsi_get_pdos(struct ucsi_connector *con, int is_partner, num_pdos * sizeof(u32)); if (ret < 0 && ret != -ETIMEDOUT) dev_err(ucsi->dev, "UCSI_GET_PDOS failed (%d)\n", ret); - if (ret == 0 && offset == 0) - dev_warn(ucsi->dev, "UCSI_GET_PDOS returned 0 bytes\n"); return ret; } @@ -1069,11 +1067,9 @@ static int ucsi_register_port(struct ucsi *ucsi, int index) cap->fwnode = ucsi_find_fwnode(con); con->usb_role_sw = fwnode_usb_role_switch_get(cap->fwnode); - if (IS_ERR(con->usb_role_sw)) { - dev_err(ucsi->dev, "con%d: failed to get usb role switch\n", - con->num); - return PTR_ERR(con->usb_role_sw); - } + if (IS_ERR(con->usb_role_sw)) + return dev_err_probe(ucsi->dev, PTR_ERR(con->usb_role_sw), + "con%d: failed to get usb role switch\n", con->num); /* Delay other interactions with the con until registration is complete */ mutex_lock(&con->lock); @@ -1200,32 +1196,6 @@ out_unlock: return ret; } -static void ucsi_unregister_connectors(struct ucsi *ucsi) -{ - struct ucsi_connector *con; - int i; - - if (!ucsi->connector) - return; - - for (i = 0; i < ucsi->cap.num_connectors; i++) { - con = &ucsi->connector[i]; - - if (!con->wq) - break; - - cancel_work_sync(&con->work); - ucsi_unregister_partner(con); - ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON); - ucsi_unregister_port_psy(con); - destroy_workqueue(con->wq); - typec_unregister_port(con->port); - } - - kfree(ucsi->connector); - ucsi->connector = NULL; -} - /** * ucsi_init - Initialize UCSI interface * @ucsi: UCSI to be initialized @@ -1234,6 +1204,7 @@ static void ucsi_unregister_connectors(struct ucsi *ucsi) */ static int ucsi_init(struct ucsi *ucsi) { + struct ucsi_connector *con; u64 command; int ret; int i; @@ -1264,7 +1235,7 @@ static int ucsi_init(struct ucsi *ucsi) } /* Allocate the connectors. Released in ucsi_unregister() */ - ucsi->connector = kcalloc(ucsi->cap.num_connectors, + ucsi->connector = kcalloc(ucsi->cap.num_connectors + 1, sizeof(*ucsi->connector), GFP_KERNEL); if (!ucsi->connector) { ret = -ENOMEM; @@ -1288,7 +1259,15 @@ static int ucsi_init(struct ucsi *ucsi) return 0; err_unregister: - ucsi_unregister_connectors(ucsi); + for (con = ucsi->connector; con->port; con++) { + ucsi_unregister_partner(con); + ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON); + ucsi_unregister_port_psy(con); + if (con->wq) + destroy_workqueue(con->wq); + typec_unregister_port(con->port); + con->port = NULL; + } err_reset: memset(&ucsi->cap, 0, sizeof(ucsi->cap)); @@ -1402,6 +1381,7 @@ EXPORT_SYMBOL_GPL(ucsi_register); void ucsi_unregister(struct ucsi *ucsi) { u64 cmd = UCSI_SET_NOTIFICATION_ENABLE; + int i; /* Make sure that we are not in the middle of driver initialization */ cancel_delayed_work_sync(&ucsi->work); @@ -1409,7 +1389,18 @@ void ucsi_unregister(struct ucsi *ucsi) /* Disable notifications */ ucsi->ops->async_write(ucsi, UCSI_CONTROL, &cmd, sizeof(cmd)); - ucsi_unregister_connectors(ucsi); + for (i = 0; i < ucsi->cap.num_connectors; i++) { + cancel_work_sync(&ucsi->connector[i].work); + ucsi_unregister_partner(&ucsi->connector[i]); + ucsi_unregister_altmodes(&ucsi->connector[i], + UCSI_RECIPIENT_CON); + ucsi_unregister_port_psy(&ucsi->connector[i]); + if (ucsi->connector[i].wq) + destroy_workqueue(ucsi->connector[i].wq); + typec_unregister_port(ucsi->connector[i].port); + } + + kfree(ucsi->connector); } EXPORT_SYMBOL_GPL(ucsi_unregister); diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c index 5c0bf48be766e9acf41bfad5c1c741089d677081..835f1c4372ba24c4278ea3e605dfb309fe2859ff 100644 --- a/drivers/usb/typec/ucsi/ucsi_ccg.c +++ b/drivers/usb/typec/ucsi/ucsi_ccg.c @@ -125,6 +125,11 @@ struct version_format { #define CCG_FW_BUILD_NVIDIA (('n' << 8) | 'v') #define CCG_OLD_FW_VERSION (CCG_VERSION(0x31) | CCG_VERSION_PATCH(10)) +/* Firmware for Tegra doesn't support UCSI ALT command, built + * for NVIDIA has known issue of reporting wrong capability info + */ +#define CCG_FW_BUILD_NVIDIA_TEGRA (('g' << 8) | 'n') + /* Altmode offset for NVIDIA Function Test Board (FTB) */ #define NVIDIA_FTB_DP_OFFSET (2) #define NVIDIA_FTB_DBG_OFFSET (3) @@ -513,6 +518,7 @@ static int ucsi_ccg_read(struct ucsi *ucsi, unsigned int offset, { struct ucsi_ccg *uc = ucsi_get_drvdata(ucsi); u16 reg = CCGX_RAB_UCSI_DATA_BLOCK(offset); + struct ucsi_capability *cap; struct ucsi_altmode *alt; int ret; @@ -536,6 +542,12 @@ static int ucsi_ccg_read(struct ucsi *ucsi, unsigned int offset, ucsi_ccg_nvidia_altmode(uc, alt); } break; + case UCSI_GET_CAPABILITY: + if (uc->fw_build == CCG_FW_BUILD_NVIDIA_TEGRA) { + cap = val; + cap->features &= ~UCSI_CAP_ALT_MODE_DETAILS; + } + break; default: break; } @@ -1403,7 +1415,7 @@ out_ucsi_destroy: return status; } -static int ucsi_ccg_remove(struct i2c_client *client) +static void ucsi_ccg_remove(struct i2c_client *client) { struct ucsi_ccg *uc = i2c_get_clientdata(client); @@ -1413,8 +1425,6 @@ static int ucsi_ccg_remove(struct i2c_client *client) ucsi_unregister(uc->ucsi); ucsi_destroy(uc->ucsi); free_irq(uc->irq, uc); - - return 0; } static const struct i2c_device_id ucsi_ccg_device_id[] = { diff --git a/drivers/usb/typec/ucsi/ucsi_stm32g0.c b/drivers/usb/typec/ucsi/ucsi_stm32g0.c index 061551d464f12eb42ef37dafc6a89718603a8fad..7b92f0c8de7088aa5f9ab81d6abf5a4c3fd78e2c 100644 --- a/drivers/usb/typec/ucsi/ucsi_stm32g0.c +++ b/drivers/usb/typec/ucsi/ucsi_stm32g0.c @@ -599,7 +599,7 @@ static int ucsi_stm32g0_probe_bootloader(struct ucsi *ucsi) g0->i2c_bl = i2c_new_dummy_device(g0->client->adapter, STM32G0_I2C_BL_ADDR); if (IS_ERR(g0->i2c_bl)) { ret = dev_err_probe(g0->dev, PTR_ERR(g0->i2c_bl), - "Failed to register booloader I2C address\n"); + "Failed to register bootloader I2C address\n"); return ret; } } @@ -688,7 +688,7 @@ destroy: return ret; } -static int ucsi_stm32g0_remove(struct i2c_client *client) +static void ucsi_stm32g0_remove(struct i2c_client *client) { struct ucsi_stm32g0 *g0 = i2c_get_clientdata(client); @@ -697,8 +697,6 @@ static int ucsi_stm32g0_remove(struct i2c_client *client) if (g0->fw_name) i2c_unregister_device(g0->i2c_bl); ucsi_destroy(g0->ucsi); - - return 0; } static int ucsi_stm32g0_suspend(struct device *dev) diff --git a/drivers/usb/typec/wusb3801.c b/drivers/usb/typec/wusb3801.c index e63509f8b01ed3955eb35f6887af5bb0dcfb57d6..3cc7a15ecbd31d56c755acc774c455132672e3ca 100644 --- a/drivers/usb/typec/wusb3801.c +++ b/drivers/usb/typec/wusb3801.c @@ -399,7 +399,7 @@ err_put_connector: return ret; } -static int wusb3801_remove(struct i2c_client *client) +static void wusb3801_remove(struct i2c_client *client) { struct wusb3801 *wusb3801 = i2c_get_clientdata(client); @@ -411,8 +411,6 @@ static int wusb3801_remove(struct i2c_client *client) if (wusb3801->vbus_on) regulator_disable(wusb3801->vbus_supply); - - return 0; } static const struct of_device_id wusb3801_of_match[] = { diff --git a/drivers/usb/usbip/stub_main.c b/drivers/usb/usbip/stub_main.c index 77a5b3f8736af1e760a807894da0f5b2a0c9cbbb..e8c3131a85431082e035f378d2a84d763e1b9a72 100644 --- a/drivers/usb/usbip/stub_main.c +++ b/drivers/usb/usbip/stub_main.c @@ -100,7 +100,7 @@ static int add_match_busid(char *busid) for (i = 0; i < MAX_BUSID; i++) { spin_lock(&busid_table[i].busid_lock); if (!busid_table[i].name[0]) { - strlcpy(busid_table[i].name, busid, BUSID_SIZE); + strscpy(busid_table[i].name, busid, BUSID_SIZE); if ((busid_table[i].status != STUB_BUSID_ALLOC) && (busid_table[i].status != STUB_BUSID_REMOV)) busid_table[i].status = STUB_BUSID_ADDED; diff --git a/drivers/usb/usbip/stub_rx.c b/drivers/usb/usbip/stub_rx.c index 5dd41e8215e0febd22ede7bb8cd35296bf04d214..fc01b31bbb875d1b200c68ba32f219168035a532 100644 --- a/drivers/usb/usbip/stub_rx.c +++ b/drivers/usb/usbip/stub_rx.c @@ -464,7 +464,7 @@ static void stub_recv_cmd_submit(struct stub_device *sdev, int nents; int num_urbs = 1; int pipe = get_pipe(sdev, pdu); - int use_sg = pdu->u.cmd_submit.transfer_flags & URB_DMA_MAP_SG; + int use_sg = pdu->u.cmd_submit.transfer_flags & USBIP_URB_DMA_MAP_SG; int support_sg = 1; int np = 0; int ret, i; @@ -514,7 +514,7 @@ static void stub_recv_cmd_submit(struct stub_device *sdev, num_urbs = nents; priv->completed_urbs = 0; pdu->u.cmd_submit.transfer_flags &= - ~URB_DMA_MAP_SG; + ~USBIP_URB_DMA_MAP_SG; } } else { buffer = kzalloc(buf_len, GFP_KERNEL); diff --git a/drivers/usb/usbip/usbip_common.c b/drivers/usb/usbip/usbip_common.c index 2ab99244bc314dd42c0921105181fbbecdc3047e..053a2bca4c475a54ba7081b979b331652124edea 100644 --- a/drivers/usb/usbip/usbip_common.c +++ b/drivers/usb/usbip/usbip_common.c @@ -344,6 +344,91 @@ static unsigned int tweak_transfer_flags(unsigned int flags) return flags; } +/* + * USBIP driver packs URB transfer flags in PDUs that are exchanged + * between Server (usbip_host) and Client (vhci_hcd). URB_* flags + * are internal to kernel and could change. Where as USBIP URB flags + * exchanged in PDUs are USBIP user API must not change. + * + * USBIP_URB* flags are exported as explicit API and client and server + * do mapping from kernel flags to USBIP_URB*. Details as follows: + * + * Client tx path (USBIP_CMD_SUBMIT): + * - Maps URB_* to USBIP_URB_* when it sends USBIP_CMD_SUBMIT packet. + * + * Server rx path (USBIP_CMD_SUBMIT): + * - Maps USBIP_URB_* to URB_* when it receives USBIP_CMD_SUBMIT packet. + * + * Flags aren't included in USBIP_CMD_UNLINK and USBIP_RET_SUBMIT packets + * and no special handling is needed for them in the following cases: + * - Server rx path (USBIP_CMD_UNLINK) + * - Client rx path & Server tx path (USBIP_RET_SUBMIT) + * + * Code paths: + * usbip_pack_pdu() is the common routine that handles packing pdu from + * urb and unpack pdu to an urb. + * + * usbip_pack_cmd_submit() and usbip_pack_ret_submit() handle + * USBIP_CMD_SUBMIT and USBIP_RET_SUBMIT respectively. + * + * usbip_map_urb_to_usbip() and usbip_map_usbip_to_urb() are used + * by usbip_pack_cmd_submit() and usbip_pack_ret_submit() to map + * flags. + */ + +struct urb_to_usbip_flags { + u32 urb_flag; + u32 usbip_flag; +}; + +#define NUM_USBIP_FLAGS 17 + +static const struct urb_to_usbip_flags flag_map[NUM_USBIP_FLAGS] = { + {URB_SHORT_NOT_OK, USBIP_URB_SHORT_NOT_OK}, + {URB_ISO_ASAP, USBIP_URB_ISO_ASAP}, + {URB_NO_TRANSFER_DMA_MAP, USBIP_URB_NO_TRANSFER_DMA_MAP}, + {URB_ZERO_PACKET, USBIP_URB_ZERO_PACKET}, + {URB_NO_INTERRUPT, USBIP_URB_NO_INTERRUPT}, + {URB_FREE_BUFFER, USBIP_URB_FREE_BUFFER}, + {URB_DIR_IN, USBIP_URB_DIR_IN}, + {URB_DIR_OUT, USBIP_URB_DIR_OUT}, + {URB_DIR_MASK, USBIP_URB_DIR_MASK}, + {URB_DMA_MAP_SINGLE, USBIP_URB_DMA_MAP_SINGLE}, + {URB_DMA_MAP_PAGE, USBIP_URB_DMA_MAP_PAGE}, + {URB_DMA_MAP_SG, USBIP_URB_DMA_MAP_SG}, + {URB_MAP_LOCAL, USBIP_URB_MAP_LOCAL}, + {URB_SETUP_MAP_SINGLE, USBIP_URB_SETUP_MAP_SINGLE}, + {URB_SETUP_MAP_LOCAL, USBIP_URB_SETUP_MAP_LOCAL}, + {URB_DMA_SG_COMBINED, USBIP_URB_DMA_SG_COMBINED}, + {URB_ALIGNED_TEMP_BUFFER, USBIP_URB_ALIGNED_TEMP_BUFFER}, +}; + +static unsigned int urb_to_usbip(unsigned int flags) +{ + unsigned int map_flags = 0; + int loop; + + for (loop = 0; loop < NUM_USBIP_FLAGS; loop++) { + if (flags & flag_map[loop].urb_flag) + map_flags |= flag_map[loop].usbip_flag; + } + + return map_flags; +} + +static unsigned int usbip_to_urb(unsigned int flags) +{ + unsigned int map_flags = 0; + int loop; + + for (loop = 0; loop < NUM_USBIP_FLAGS; loop++) { + if (flags & flag_map[loop].usbip_flag) + map_flags |= flag_map[loop].urb_flag; + } + + return map_flags; +} + static void usbip_pack_cmd_submit(struct usbip_header *pdu, struct urb *urb, int pack) { @@ -354,14 +439,14 @@ static void usbip_pack_cmd_submit(struct usbip_header *pdu, struct urb *urb, * will be discussed when usbip is ported to other operating systems. */ if (pack) { - spdu->transfer_flags = - tweak_transfer_flags(urb->transfer_flags); + /* map after tweaking the urb flags */ + spdu->transfer_flags = urb_to_usbip(tweak_transfer_flags(urb->transfer_flags)); spdu->transfer_buffer_length = urb->transfer_buffer_length; spdu->start_frame = urb->start_frame; spdu->number_of_packets = urb->number_of_packets; spdu->interval = urb->interval; } else { - urb->transfer_flags = spdu->transfer_flags; + urb->transfer_flags = usbip_to_urb(spdu->transfer_flags); urb->transfer_buffer_length = spdu->transfer_buffer_length; urb->start_frame = spdu->start_frame; urb->number_of_packets = spdu->number_of_packets; diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 75a703b803a240bdb966acac2d6bfe0bc8cdded7..3e4486bfa0b71266c43eb5920c778ba40e9cface 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -323,7 +323,7 @@ u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) u32 q_pair_id; ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; - q_pair_id = qid / hw->nr_vring; + q_pair_id = qid / 2; avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; last_avail_idx = vp_ioread16(avail_idx_addr); @@ -337,7 +337,7 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) u32 q_pair_id; ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; - q_pair_id = qid / hw->nr_vring; + q_pair_id = qid / 2; avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; hw->vring[qid].last_avail_idx = num; vp_iowrite16(num, avail_idx_addr); diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index ed100a35e5969a4c602bc595f0ddf487a07dbc92..90913365def43de809ba8c9ef4de1675c656816a 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1320,6 +1320,8 @@ static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue * static int create_rqt(struct mlx5_vdpa_net *ndev) { + int rqt_table_size = roundup_pow_of_two(ndev->rqt_size); + int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2); __be32 *list; void *rqtc; int inlen; @@ -1327,7 +1329,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev) int i, j; int err; - inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); + inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num); in = kzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -1336,12 +1338,12 @@ static int create_rqt(struct mlx5_vdpa_net *ndev) rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); - MLX5_SET(rqtc, rqtc, rqt_max_size, ndev->rqt_size); + MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size); list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); - for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) + for (i = 0, j = 0; i < act_sz; i++, j += 2) list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id); - MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); + MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn); kfree(in); if (err) @@ -1354,6 +1356,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev) static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) { + int act_sz = roundup_pow_of_two(num / 2); __be32 *list; void *rqtc; int inlen; @@ -1361,7 +1364,7 @@ static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) int i, j; int err; - inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num); + inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num); in = kzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -1372,10 +1375,10 @@ static int modify_rqt(struct mlx5_vdpa_net *ndev, int num) MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q); list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]); - for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2) + for (i = 0, j = 0; i < act_sz; i++, j = j + 2) list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id); - MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size); + MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz); err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn); kfree(in); if (err) diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index c06c02704461006baa4d2a60db2a30640b100b42..febdc99b51a7b782d801a6564860a3ee091f1e0a 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -600,6 +600,11 @@ static int vdpa_nl_cmd_dev_add_set_doit(struct sk_buff *skb, struct genl_info *i } config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); } + if (nl_attrs[VDPA_ATTR_DEV_FEATURES]) { + config.device_features = + nla_get_u64(nl_attrs[VDPA_ATTR_DEV_FEATURES]); + config.mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES); + } /* Skip checking capability if user didn't prefer to configure any * device networking attributes. It is likely that user might have used @@ -799,51 +804,76 @@ static int vdpa_nl_cmd_dev_get_dumpit(struct sk_buff *msg, struct netlink_callba return msg->len; } -static int vdpa_dev_net_mq_config_fill(struct vdpa_device *vdev, - struct sk_buff *msg, u64 features, +static int vdpa_dev_net_mq_config_fill(struct sk_buff *msg, u64 features, const struct virtio_net_config *config) { u16 val_u16; - if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0) + if ((features & BIT_ULL(VIRTIO_NET_F_MQ)) == 0 && + (features & BIT_ULL(VIRTIO_NET_F_RSS)) == 0) return 0; - val_u16 = le16_to_cpu(config->max_virtqueue_pairs); + val_u16 = __virtio16_to_cpu(true, config->max_virtqueue_pairs); + return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, val_u16); } +static int vdpa_dev_net_mtu_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_net_config *config) +{ + u16 val_u16; + + if ((features & BIT_ULL(VIRTIO_NET_F_MTU)) == 0) + return 0; + + val_u16 = __virtio16_to_cpu(true, config->mtu); + + return nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16); +} + +static int vdpa_dev_net_mac_config_fill(struct sk_buff *msg, u64 features, + const struct virtio_net_config *config) +{ + if ((features & BIT_ULL(VIRTIO_NET_F_MAC)) == 0) + return 0; + + return nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR, + sizeof(config->mac), config->mac); +} + static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *msg) { struct virtio_net_config config = {}; - u64 features; + u64 features_device; u16 val_u16; - vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config)); - - if (nla_put(msg, VDPA_ATTR_DEV_NET_CFG_MACADDR, sizeof(config.mac), - config.mac)) - return -EMSGSIZE; + vdev->config->get_config(vdev, 0, &config, sizeof(config)); val_u16 = __virtio16_to_cpu(true, config.status); if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16)) return -EMSGSIZE; - val_u16 = __virtio16_to_cpu(true, config.mtu); - if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16)) - return -EMSGSIZE; + features_device = vdev->config->get_device_features(vdev); - features = vdev->config->get_driver_features(vdev); - if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features, + if (nla_put_u64_64bit(msg, VDPA_ATTR_VDPA_DEV_SUPPORTED_FEATURES, features_device, VDPA_ATTR_PAD)) return -EMSGSIZE; - return vdpa_dev_net_mq_config_fill(vdev, msg, features, &config); + if (vdpa_dev_net_mtu_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + + if (vdpa_dev_net_mac_config_fill(msg, features_device, &config)) + return -EMSGSIZE; + + return vdpa_dev_net_mq_config_fill(msg, features_device, &config); } static int vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, u32 seq, int flags, struct netlink_ext_ack *extack) { + u64 features_driver; + u8 status = 0; u32 device_id; void *hdr; int err; @@ -867,6 +897,17 @@ vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, goto msg_err; } + /* only read driver features after the feature negotiation is done */ + status = vdev->config->get_status(vdev); + if (status & VIRTIO_CONFIG_S_FEATURES_OK) { + features_driver = vdev->config->get_driver_features(vdev); + if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_NEGOTIATED_FEATURES, features_driver, + VDPA_ATTR_PAD)) { + err = -EMSGSIZE; + goto msg_err; + } + } + switch (device_id) { case VIRTIO_ID_NET: err = vdpa_dev_net_config_fill(vdev, msg); @@ -1183,6 +1224,7 @@ static struct genl_family vdpa_nl_family __ro_after_init = { .module = THIS_MODULE, .ops = vdpa_nl_ops, .n_ops = ARRAY_SIZE(vdpa_nl_ops), + .resv_start_op = VDPA_CMD_DEV_VSTATS_GET + 1, }; static int vdpa_init(void) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 225b7f5d8be353c16dbcbeb98790f2486ab943a8..b071f0d842fbade2db1ab9a2d10dc9dd03cb80ee 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -18,6 +18,7 @@ #include #include #include +#include #include "vdpa_sim.h" @@ -245,13 +246,22 @@ static const struct dma_map_ops vdpasim_dma_ops = { static const struct vdpa_config_ops vdpasim_config_ops; static const struct vdpa_config_ops vdpasim_batch_config_ops; -struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) +struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, + const struct vdpa_dev_set_config *config) { const struct vdpa_config_ops *ops; struct vdpasim *vdpasim; struct device *dev; int i, ret = -ENOMEM; + if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { + if (config->device_features & + ~dev_attr->supported_features) + return ERR_PTR(-EINVAL); + dev_attr->supported_features = + config->device_features; + } + if (batch_mapping) ops = &vdpasim_batch_config_ops; else diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h index 061986f30911a73f9b3fd2d3f47a79e926f23bb1..0e78737dcc166effb6f7f8e8b3b141e7a2cc9155 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h @@ -71,7 +71,8 @@ struct vdpasim { spinlock_t iommu_lock; }; -struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr); +struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *attr, + const struct vdpa_dev_set_config *config); /* TODO: cross-endian support */ static inline bool vdpasim_is_little_endian(struct vdpasim *vdpasim) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index c8bfea3b7db230a16623e5016b706a23e370571a..c6db1a1baf768594f43cf265b51530bb8880db12 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -383,7 +383,7 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, dev_attr.work_fn = vdpasim_blk_work; dev_attr.buffer_size = VDPASIM_BLK_CAPACITY << SECTOR_SHIFT; - simdev = vdpasim_create(&dev_attr); + simdev = vdpasim_create(&dev_attr, config); if (IS_ERR(simdev)) return PTR_ERR(simdev); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c index 886449e885026ab2b7ee1e4ca3681bb59edf2cbd..c3cb225ea4693af4fa59269000c150c7b0376c25 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c @@ -254,7 +254,7 @@ static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, dev_attr.work_fn = vdpasim_net_work; dev_attr.buffer_size = PAGE_SIZE; - simdev = vdpasim_create(&dev_attr); + simdev = vdpasim_create(&dev_attr, config); if (IS_ERR(simdev)) return PTR_ERR(simdev); @@ -294,7 +294,8 @@ static struct vdpa_mgmt_dev mgmt_dev = { .id_table = id_table, .ops = &vdpasim_net_mgmtdev_ops, .config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR | - 1 << VDPA_ATTR_DEV_NET_CFG_MTU), + 1 << VDPA_ATTR_DEV_NET_CFG_MTU | + 1 << VDPA_ATTR_DEV_FEATURES), .max_supported_vqs = VDPASIM_NET_VQ_NUM, .supported_features = VDPASIM_NET_FEATURES, }; diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 41c0b29739f16484b1afdc14d50ece2e05c1bcb1..35dceee3ed56086af3e105a9451dc91b8af5d743 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -673,10 +673,15 @@ static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, { struct vduse_dev *dev = vdpa_to_vduse(vdpa); - if (offset > dev->config_size || - len > dev->config_size - offset) + /* Initialize the buffer in case of partial copy. */ + memset(buf, 0, len); + + if (offset > dev->config_size) return; + if (len > dev->config_size - offset) + len = dev->config_size - offset; + memcpy(buf, dev->config + offset, len); } diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c index 04522077735b24b306ef52c6f1358006f95e0716..d448db0c4de3f81810bc94008beea0345e202458 100644 --- a/drivers/vdpa/virtio_pci/vp_vdpa.c +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c @@ -17,6 +17,7 @@ #include #include #include +#include #define VP_VDPA_QUEUE_MAX 256 #define VP_VDPA_DRIVER_NAME "vp_vdpa" @@ -35,6 +36,7 @@ struct vp_vdpa { struct virtio_pci_modern_device *mdev; struct vp_vring *vring; struct vdpa_callback config_cb; + u64 device_features; char msix_name[VP_VDPA_NAME_SIZE]; int config_irq; int queues; @@ -66,9 +68,9 @@ static struct virtio_pci_modern_device *vp_vdpa_to_mdev(struct vp_vdpa *vp_vdpa) static u64 vp_vdpa_get_device_features(struct vdpa_device *vdpa) { - struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa); + struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - return vp_modern_get_features(mdev); + return vp_vdpa->device_features; } static int vp_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) @@ -475,6 +477,7 @@ static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, struct pci_dev *pdev = mdev->pci_dev; struct device *dev = &pdev->dev; struct vp_vdpa *vp_vdpa = NULL; + u64 device_features; int ret, i; vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, @@ -491,6 +494,20 @@ static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, vp_vdpa->queues = vp_modern_get_num_queues(mdev); vp_vdpa->mdev = mdev; + device_features = vp_modern_get_features(mdev); + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { + if (add_config->device_features & ~device_features) { + ret = -EINVAL; + dev_err(&pdev->dev, "Try to provision features " + "that are not supported by the device: " + "device_features 0x%llx provisioned 0x%llx\n", + device_features, add_config->device_features); + goto err; + } + device_features = add_config->device_features; + } + vp_vdpa->device_features = device_features; + ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev); if (ret) { dev_err(&pdev->dev, @@ -599,6 +616,7 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) mgtdev->id_table = mdev_id; mgtdev->max_supported_vqs = vp_modern_get_num_queues(mdev); mgtdev->supported_features = vp_modern_get_features(mdev); + mgtdev->config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES); pci_set_master(pdev); pci_set_drvdata(pdev, vp_vdpa_mgtdev); diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 6130d00252ed7f8b835e67c31ced67085565b8bf..86c381ceb9a1e95e95643dcce8a20fc74c5b1559 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -3,6 +3,7 @@ menuconfig VFIO tristate "VFIO Non-Privileged userspace driver framework" select IOMMU_API select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64) + select INTERVAL_TREE help VFIO provides a framework for secure userspace device drivers. See Documentation/driver-api/vfio.rst for more details. diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile index 1a32357592e3eadf7e0a3d4775e1fceb049d8db6..b693a1169286f8771f7d019e600268e45d208ba5 100644 --- a/drivers/vfio/Makefile +++ b/drivers/vfio/Makefile @@ -1,9 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 vfio_virqfd-y := virqfd.o -vfio-y += vfio_main.o - obj-$(CONFIG_VFIO) += vfio.o + +vfio-y += vfio_main.o \ + iova_bitmap.o \ + container.o + obj-$(CONFIG_VFIO_VIRQFD) += vfio_virqfd.o obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o diff --git a/drivers/vfio/container.c b/drivers/vfio/container.c new file mode 100644 index 0000000000000000000000000000000000000000..d74164abbf401d42cbcf42b6f5f7282dd7769de7 --- /dev/null +++ b/drivers/vfio/container.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Red Hat, Inc. All rights reserved. + * + * VFIO container (/dev/vfio/vfio) + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vfio.h" + +struct vfio_container { + struct kref kref; + struct list_head group_list; + struct rw_semaphore group_lock; + struct vfio_iommu_driver *iommu_driver; + void *iommu_data; + bool noiommu; +}; + +static struct vfio { + struct list_head iommu_drivers_list; + struct mutex iommu_drivers_lock; +} vfio; + +#ifdef CONFIG_VFIO_NOIOMMU +bool vfio_noiommu __read_mostly; +module_param_named(enable_unsafe_noiommu_mode, + vfio_noiommu, bool, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); +#endif + +static void *vfio_noiommu_open(unsigned long arg) +{ + if (arg != VFIO_NOIOMMU_IOMMU) + return ERR_PTR(-EINVAL); + if (!capable(CAP_SYS_RAWIO)) + return ERR_PTR(-EPERM); + + return NULL; +} + +static void vfio_noiommu_release(void *iommu_data) +{ +} + +static long vfio_noiommu_ioctl(void *iommu_data, + unsigned int cmd, unsigned long arg) +{ + if (cmd == VFIO_CHECK_EXTENSION) + return vfio_noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0; + + return -ENOTTY; +} + +static int vfio_noiommu_attach_group(void *iommu_data, + struct iommu_group *iommu_group, enum vfio_group_type type) +{ + return 0; +} + +static void vfio_noiommu_detach_group(void *iommu_data, + struct iommu_group *iommu_group) +{ +} + +static const struct vfio_iommu_driver_ops vfio_noiommu_ops = { + .name = "vfio-noiommu", + .owner = THIS_MODULE, + .open = vfio_noiommu_open, + .release = vfio_noiommu_release, + .ioctl = vfio_noiommu_ioctl, + .attach_group = vfio_noiommu_attach_group, + .detach_group = vfio_noiommu_detach_group, +}; + +/* + * Only noiommu containers can use vfio-noiommu and noiommu containers can only + * use vfio-noiommu. + */ +static bool vfio_iommu_driver_allowed(struct vfio_container *container, + const struct vfio_iommu_driver *driver) +{ + if (!IS_ENABLED(CONFIG_VFIO_NOIOMMU)) + return true; + return container->noiommu == (driver->ops == &vfio_noiommu_ops); +} + +/* + * IOMMU driver registration + */ +int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops) +{ + struct vfio_iommu_driver *driver, *tmp; + + if (WARN_ON(!ops->register_device != !ops->unregister_device)) + return -EINVAL; + + driver = kzalloc(sizeof(*driver), GFP_KERNEL); + if (!driver) + return -ENOMEM; + + driver->ops = ops; + + mutex_lock(&vfio.iommu_drivers_lock); + + /* Check for duplicates */ + list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) { + if (tmp->ops == ops) { + mutex_unlock(&vfio.iommu_drivers_lock); + kfree(driver); + return -EINVAL; + } + } + + list_add(&driver->vfio_next, &vfio.iommu_drivers_list); + + mutex_unlock(&vfio.iommu_drivers_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(vfio_register_iommu_driver); + +void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops) +{ + struct vfio_iommu_driver *driver; + + mutex_lock(&vfio.iommu_drivers_lock); + list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) { + if (driver->ops == ops) { + list_del(&driver->vfio_next); + mutex_unlock(&vfio.iommu_drivers_lock); + kfree(driver); + return; + } + } + mutex_unlock(&vfio.iommu_drivers_lock); +} +EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver); + +/* + * Container objects - containers are created when /dev/vfio/vfio is + * opened, but their lifecycle extends until the last user is done, so + * it's freed via kref. Must support container/group/device being + * closed in any order. + */ +static void vfio_container_release(struct kref *kref) +{ + struct vfio_container *container; + container = container_of(kref, struct vfio_container, kref); + + kfree(container); +} + +static void vfio_container_get(struct vfio_container *container) +{ + kref_get(&container->kref); +} + +static void vfio_container_put(struct vfio_container *container) +{ + kref_put(&container->kref, vfio_container_release); +} + +void vfio_device_container_register(struct vfio_device *device) +{ + struct vfio_iommu_driver *iommu_driver = + device->group->container->iommu_driver; + + if (iommu_driver && iommu_driver->ops->register_device) + iommu_driver->ops->register_device( + device->group->container->iommu_data, device); +} + +void vfio_device_container_unregister(struct vfio_device *device) +{ + struct vfio_iommu_driver *iommu_driver = + device->group->container->iommu_driver; + + if (iommu_driver && iommu_driver->ops->unregister_device) + iommu_driver->ops->unregister_device( + device->group->container->iommu_data, device); +} + +long vfio_container_ioctl_check_extension(struct vfio_container *container, + unsigned long arg) +{ + struct vfio_iommu_driver *driver; + long ret = 0; + + down_read(&container->group_lock); + + driver = container->iommu_driver; + + switch (arg) { + /* No base extensions yet */ + default: + /* + * If no driver is set, poll all registered drivers for + * extensions and return the first positive result. If + * a driver is already set, further queries will be passed + * only to that driver. + */ + if (!driver) { + mutex_lock(&vfio.iommu_drivers_lock); + list_for_each_entry(driver, &vfio.iommu_drivers_list, + vfio_next) { + + if (!list_empty(&container->group_list) && + !vfio_iommu_driver_allowed(container, + driver)) + continue; + if (!try_module_get(driver->ops->owner)) + continue; + + ret = driver->ops->ioctl(NULL, + VFIO_CHECK_EXTENSION, + arg); + module_put(driver->ops->owner); + if (ret > 0) + break; + } + mutex_unlock(&vfio.iommu_drivers_lock); + } else + ret = driver->ops->ioctl(container->iommu_data, + VFIO_CHECK_EXTENSION, arg); + } + + up_read(&container->group_lock); + + return ret; +} + +/* hold write lock on container->group_lock */ +static int __vfio_container_attach_groups(struct vfio_container *container, + struct vfio_iommu_driver *driver, + void *data) +{ + struct vfio_group *group; + int ret = -ENODEV; + + list_for_each_entry(group, &container->group_list, container_next) { + ret = driver->ops->attach_group(data, group->iommu_group, + group->type); + if (ret) + goto unwind; + } + + return ret; + +unwind: + list_for_each_entry_continue_reverse(group, &container->group_list, + container_next) { + driver->ops->detach_group(data, group->iommu_group); + } + + return ret; +} + +static long vfio_ioctl_set_iommu(struct vfio_container *container, + unsigned long arg) +{ + struct vfio_iommu_driver *driver; + long ret = -ENODEV; + + down_write(&container->group_lock); + + /* + * The container is designed to be an unprivileged interface while + * the group can be assigned to specific users. Therefore, only by + * adding a group to a container does the user get the privilege of + * enabling the iommu, which may allocate finite resources. There + * is no unset_iommu, but by removing all the groups from a container, + * the container is deprivileged and returns to an unset state. + */ + if (list_empty(&container->group_list) || container->iommu_driver) { + up_write(&container->group_lock); + return -EINVAL; + } + + mutex_lock(&vfio.iommu_drivers_lock); + list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) { + void *data; + + if (!vfio_iommu_driver_allowed(container, driver)) + continue; + if (!try_module_get(driver->ops->owner)) + continue; + + /* + * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION, + * so test which iommu driver reported support for this + * extension and call open on them. We also pass them the + * magic, allowing a single driver to support multiple + * interfaces if they'd like. + */ + if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) { + module_put(driver->ops->owner); + continue; + } + + data = driver->ops->open(arg); + if (IS_ERR(data)) { + ret = PTR_ERR(data); + module_put(driver->ops->owner); + continue; + } + + ret = __vfio_container_attach_groups(container, driver, data); + if (ret) { + driver->ops->release(data); + module_put(driver->ops->owner); + continue; + } + + container->iommu_driver = driver; + container->iommu_data = data; + break; + } + + mutex_unlock(&vfio.iommu_drivers_lock); + up_write(&container->group_lock); + + return ret; +} + +static long vfio_fops_unl_ioctl(struct file *filep, + unsigned int cmd, unsigned long arg) +{ + struct vfio_container *container = filep->private_data; + struct vfio_iommu_driver *driver; + void *data; + long ret = -EINVAL; + + if (!container) + return ret; + + switch (cmd) { + case VFIO_GET_API_VERSION: + ret = VFIO_API_VERSION; + break; + case VFIO_CHECK_EXTENSION: + ret = vfio_container_ioctl_check_extension(container, arg); + break; + case VFIO_SET_IOMMU: + ret = vfio_ioctl_set_iommu(container, arg); + break; + default: + driver = container->iommu_driver; + data = container->iommu_data; + + if (driver) /* passthrough all unrecognized ioctls */ + ret = driver->ops->ioctl(data, cmd, arg); + } + + return ret; +} + +static int vfio_fops_open(struct inode *inode, struct file *filep) +{ + struct vfio_container *container; + + container = kzalloc(sizeof(*container), GFP_KERNEL); + if (!container) + return -ENOMEM; + + INIT_LIST_HEAD(&container->group_list); + init_rwsem(&container->group_lock); + kref_init(&container->kref); + + filep->private_data = container; + + return 0; +} + +static int vfio_fops_release(struct inode *inode, struct file *filep) +{ + struct vfio_container *container = filep->private_data; + struct vfio_iommu_driver *driver = container->iommu_driver; + + if (driver && driver->ops->notify) + driver->ops->notify(container->iommu_data, + VFIO_IOMMU_CONTAINER_CLOSE); + + filep->private_data = NULL; + + vfio_container_put(container); + + return 0; +} + +static const struct file_operations vfio_fops = { + .owner = THIS_MODULE, + .open = vfio_fops_open, + .release = vfio_fops_release, + .unlocked_ioctl = vfio_fops_unl_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +struct vfio_container *vfio_container_from_file(struct file *file) +{ + struct vfio_container *container; + + /* Sanity check, is this really our fd? */ + if (file->f_op != &vfio_fops) + return NULL; + + container = file->private_data; + WARN_ON(!container); /* fget ensures we don't race vfio_release */ + return container; +} + +static struct miscdevice vfio_dev = { + .minor = VFIO_MINOR, + .name = "vfio", + .fops = &vfio_fops, + .nodename = "vfio/vfio", + .mode = S_IRUGO | S_IWUGO, +}; + +int vfio_container_attach_group(struct vfio_container *container, + struct vfio_group *group) +{ + struct vfio_iommu_driver *driver; + int ret = 0; + + lockdep_assert_held(&group->group_lock); + + if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) + return -EPERM; + + down_write(&container->group_lock); + + /* Real groups and fake groups cannot mix */ + if (!list_empty(&container->group_list) && + container->noiommu != (group->type == VFIO_NO_IOMMU)) { + ret = -EPERM; + goto out_unlock_container; + } + + if (group->type == VFIO_IOMMU) { + ret = iommu_group_claim_dma_owner(group->iommu_group, group); + if (ret) + goto out_unlock_container; + } + + driver = container->iommu_driver; + if (driver) { + ret = driver->ops->attach_group(container->iommu_data, + group->iommu_group, + group->type); + if (ret) { + if (group->type == VFIO_IOMMU) + iommu_group_release_dma_owner( + group->iommu_group); + goto out_unlock_container; + } + } + + group->container = container; + group->container_users = 1; + container->noiommu = (group->type == VFIO_NO_IOMMU); + list_add(&group->container_next, &container->group_list); + + /* Get a reference on the container and mark a user within the group */ + vfio_container_get(container); + +out_unlock_container: + up_write(&container->group_lock); + return ret; +} + +void vfio_group_detach_container(struct vfio_group *group) +{ + struct vfio_container *container = group->container; + struct vfio_iommu_driver *driver; + + lockdep_assert_held(&group->group_lock); + WARN_ON(group->container_users != 1); + + down_write(&container->group_lock); + + driver = container->iommu_driver; + if (driver) + driver->ops->detach_group(container->iommu_data, + group->iommu_group); + + if (group->type == VFIO_IOMMU) + iommu_group_release_dma_owner(group->iommu_group); + + group->container = NULL; + group->container_users = 0; + list_del(&group->container_next); + + /* Detaching the last group deprivileges a container, remove iommu */ + if (driver && list_empty(&container->group_list)) { + driver->ops->release(container->iommu_data); + module_put(driver->ops->owner); + container->iommu_driver = NULL; + container->iommu_data = NULL; + } + + up_write(&container->group_lock); + + vfio_container_put(container); +} + +int vfio_device_assign_container(struct vfio_device *device) +{ + struct vfio_group *group = device->group; + + lockdep_assert_held(&group->group_lock); + + if (!group->container || !group->container->iommu_driver || + WARN_ON(!group->container_users)) + return -EINVAL; + + if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) + return -EPERM; + + get_file(group->opened_file); + group->container_users++; + return 0; +} + +void vfio_device_unassign_container(struct vfio_device *device) +{ + mutex_lock(&device->group->group_lock); + WARN_ON(device->group->container_users <= 1); + device->group->container_users--; + fput(device->group->opened_file); + mutex_unlock(&device->group->group_lock); +} + +/* + * Pin contiguous user pages and return their associated host pages for local + * domain only. + * @device [in] : device + * @iova [in] : starting IOVA of user pages to be pinned. + * @npage [in] : count of pages to be pinned. This count should not + * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. + * @prot [in] : protection flags + * @pages[out] : array of host pages + * Return error or number of pages pinned. + * + * A driver may only call this function if the vfio_device was created + * by vfio_register_emulated_iommu_dev(). + */ +int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, + int npage, int prot, struct page **pages) +{ + struct vfio_container *container; + struct vfio_group *group = device->group; + struct vfio_iommu_driver *driver; + int ret; + + if (!pages || !npage || !vfio_assert_device_open(device)) + return -EINVAL; + + if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) + return -E2BIG; + + /* group->container cannot change while a vfio device is open */ + container = group->container; + driver = container->iommu_driver; + if (likely(driver && driver->ops->pin_pages)) + ret = driver->ops->pin_pages(container->iommu_data, + group->iommu_group, iova, + npage, prot, pages); + else + ret = -ENOTTY; + + return ret; +} +EXPORT_SYMBOL(vfio_pin_pages); + +/* + * Unpin contiguous host pages for local domain only. + * @device [in] : device + * @iova [in] : starting address of user pages to be unpinned. + * @npage [in] : count of pages to be unpinned. This count should not + * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. + */ +void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) +{ + struct vfio_container *container; + struct vfio_iommu_driver *driver; + + if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES)) + return; + + if (WARN_ON(!vfio_assert_device_open(device))) + return; + + /* group->container cannot change while a vfio device is open */ + container = device->group->container; + driver = container->iommu_driver; + + driver->ops->unpin_pages(container->iommu_data, iova, npage); +} +EXPORT_SYMBOL(vfio_unpin_pages); + +/* + * This interface allows the CPUs to perform some sort of virtual DMA on + * behalf of the device. + * + * CPUs read/write from/into a range of IOVAs pointing to user space memory + * into/from a kernel buffer. + * + * As the read/write of user space memory is conducted via the CPUs and is + * not a real device DMA, it is not necessary to pin the user space memory. + * + * @device [in] : VFIO device + * @iova [in] : base IOVA of a user space buffer + * @data [in] : pointer to kernel buffer + * @len [in] : kernel buffer length + * @write : indicate read or write + * Return error code on failure or 0 on success. + */ +int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, + size_t len, bool write) +{ + struct vfio_container *container; + struct vfio_iommu_driver *driver; + int ret = 0; + + if (!data || len <= 0 || !vfio_assert_device_open(device)) + return -EINVAL; + + /* group->container cannot change while a vfio device is open */ + container = device->group->container; + driver = container->iommu_driver; + + if (likely(driver && driver->ops->dma_rw)) + ret = driver->ops->dma_rw(container->iommu_data, + iova, data, len, write); + else + ret = -ENOTTY; + return ret; +} +EXPORT_SYMBOL(vfio_dma_rw); + +int __init vfio_container_init(void) +{ + int ret; + + mutex_init(&vfio.iommu_drivers_lock); + INIT_LIST_HEAD(&vfio.iommu_drivers_list); + + ret = misc_register(&vfio_dev); + if (ret) { + pr_err("vfio: misc device register failed\n"); + return ret; + } + + if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) { + ret = vfio_register_iommu_driver(&vfio_noiommu_ops); + if (ret) + goto err_misc; + } + return 0; + +err_misc: + misc_deregister(&vfio_dev); + return ret; +} + +void vfio_container_cleanup(void) +{ + if (IS_ENABLED(CONFIG_VFIO_NOIOMMU)) + vfio_unregister_iommu_driver(&vfio_noiommu_ops); + misc_deregister(&vfio_dev); + mutex_destroy(&vfio.iommu_drivers_lock); +} diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index 3feff729f3ce825cf759421241bdabe08e6d29a8..b16874e913e4f527457f5864be5b942d60729a16 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -108,9 +108,9 @@ static void vfio_fsl_mc_close_device(struct vfio_device *core_vdev) /* reset the device before cleaning up the interrupts */ ret = vfio_fsl_mc_reset_device(vdev); - if (WARN_ON(ret)) + if (ret) dev_warn(&mc_cont->dev, - "VFIO_FLS_MC: reset device has failed (%d)\n", ret); + "VFIO_FSL_MC: reset device has failed (%d)\n", ret); vfio_fsl_mc_irqs_cleanup(vdev); @@ -418,16 +418,7 @@ static int vfio_fsl_mc_mmap(struct vfio_device *core_vdev, return vfio_fsl_mc_mmap_mmio(vdev->regions[index], vma); } -static const struct vfio_device_ops vfio_fsl_mc_ops = { - .name = "vfio-fsl-mc", - .open_device = vfio_fsl_mc_open_device, - .close_device = vfio_fsl_mc_close_device, - .ioctl = vfio_fsl_mc_ioctl, - .read = vfio_fsl_mc_read, - .write = vfio_fsl_mc_write, - .mmap = vfio_fsl_mc_mmap, -}; - +static const struct vfio_device_ops vfio_fsl_mc_ops; static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -518,35 +509,43 @@ static void vfio_fsl_uninit_device(struct vfio_fsl_mc_device *vdev) bus_unregister_notifier(&fsl_mc_bus_type, &vdev->nb); } -static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) +static int vfio_fsl_mc_init_dev(struct vfio_device *core_vdev) { - struct vfio_fsl_mc_device *vdev; - struct device *dev = &mc_dev->dev; + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); + struct fsl_mc_device *mc_dev = to_fsl_mc_device(core_vdev->dev); int ret; - vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); - if (!vdev) - return -ENOMEM; - - vfio_init_group_dev(&vdev->vdev, dev, &vfio_fsl_mc_ops); vdev->mc_dev = mc_dev; mutex_init(&vdev->igate); if (is_fsl_mc_bus_dprc(mc_dev)) - ret = vfio_assign_device_set(&vdev->vdev, &mc_dev->dev); + ret = vfio_assign_device_set(core_vdev, &mc_dev->dev); else - ret = vfio_assign_device_set(&vdev->vdev, mc_dev->dev.parent); - if (ret) - goto out_uninit; + ret = vfio_assign_device_set(core_vdev, mc_dev->dev.parent); - ret = vfio_fsl_mc_init_device(vdev); if (ret) - goto out_uninit; + return ret; + + /* device_set is released by vfio core if @init fails */ + return vfio_fsl_mc_init_device(vdev); +} + +static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) +{ + struct vfio_fsl_mc_device *vdev; + struct device *dev = &mc_dev->dev; + int ret; + + vdev = vfio_alloc_device(vfio_fsl_mc_device, vdev, dev, + &vfio_fsl_mc_ops); + if (IS_ERR(vdev)) + return PTR_ERR(vdev); ret = vfio_register_group_dev(&vdev->vdev); if (ret) { dev_err(dev, "VFIO_FSL_MC: Failed to add to vfio group\n"); - goto out_device; + goto out_put_vdev; } ret = vfio_fsl_mc_scan_container(mc_dev); @@ -557,30 +556,44 @@ static int vfio_fsl_mc_probe(struct fsl_mc_device *mc_dev) out_group_dev: vfio_unregister_group_dev(&vdev->vdev); -out_device: - vfio_fsl_uninit_device(vdev); -out_uninit: - vfio_uninit_group_dev(&vdev->vdev); - kfree(vdev); +out_put_vdev: + vfio_put_device(&vdev->vdev); return ret; } +static void vfio_fsl_mc_release_dev(struct vfio_device *core_vdev) +{ + struct vfio_fsl_mc_device *vdev = + container_of(core_vdev, struct vfio_fsl_mc_device, vdev); + + vfio_fsl_uninit_device(vdev); + mutex_destroy(&vdev->igate); + vfio_free_device(core_vdev); +} + static int vfio_fsl_mc_remove(struct fsl_mc_device *mc_dev) { struct device *dev = &mc_dev->dev; struct vfio_fsl_mc_device *vdev = dev_get_drvdata(dev); vfio_unregister_group_dev(&vdev->vdev); - mutex_destroy(&vdev->igate); - dprc_remove_devices(mc_dev, NULL, 0); - vfio_fsl_uninit_device(vdev); - - vfio_uninit_group_dev(&vdev->vdev); - kfree(vdev); + vfio_put_device(&vdev->vdev); return 0; } +static const struct vfio_device_ops vfio_fsl_mc_ops = { + .name = "vfio-fsl-mc", + .init = vfio_fsl_mc_init_dev, + .release = vfio_fsl_mc_release_dev, + .open_device = vfio_fsl_mc_open_device, + .close_device = vfio_fsl_mc_close_device, + .ioctl = vfio_fsl_mc_ioctl, + .read = vfio_fsl_mc_read, + .write = vfio_fsl_mc_write, + .mmap = vfio_fsl_mc_mmap, +}; + static struct fsl_mc_driver vfio_fsl_mc_driver = { .probe = vfio_fsl_mc_probe, .remove = vfio_fsl_mc_remove, diff --git a/drivers/vfio/iova_bitmap.c b/drivers/vfio/iova_bitmap.c new file mode 100644 index 0000000000000000000000000000000000000000..6631e8befe1b2711d58fad544eac41dc829bd8ad --- /dev/null +++ b/drivers/vfio/iova_bitmap.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022, Oracle and/or its affiliates. + * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ +#include +#include +#include + +#define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) + +/* + * struct iova_bitmap_map - A bitmap representing an IOVA range + * + * Main data structure for tracking mapped user pages of bitmap data. + * + * For example, for something recording dirty IOVAs, it will be provided a + * struct iova_bitmap structure, as a general structure for iterating the + * total IOVA range. The struct iova_bitmap_map, though, represents the + * subset of said IOVA space that is pinned by its parent structure (struct + * iova_bitmap). + * + * The user does not need to exact location of the bits in the bitmap. + * From user perspective the only API available is iova_bitmap_set() which + * records the IOVA *range* in the bitmap by setting the corresponding + * bits. + * + * The bitmap is an array of u64 whereas each bit represents an IOVA of + * range of (1 << pgshift). Thus formula for the bitmap data to be set is: + * + * data[(iova / page_size) / 64] & (1ULL << (iova % 64)) + */ +struct iova_bitmap_map { + /* base IOVA representing bit 0 of the first page */ + unsigned long iova; + + /* page size order that each bit granules to */ + unsigned long pgshift; + + /* page offset of the first user page pinned */ + unsigned long pgoff; + + /* number of pages pinned */ + unsigned long npages; + + /* pinned pages representing the bitmap data */ + struct page **pages; +}; + +/* + * struct iova_bitmap - The IOVA bitmap object + * + * Main data structure for iterating over the bitmap data. + * + * Abstracts the pinning work and iterates in IOVA ranges. + * It uses a windowing scheme and pins the bitmap in relatively + * big ranges e.g. + * + * The bitmap object uses one base page to store all the pinned pages + * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores + * 512 struct page pointers which, if the base page size is 4K, it means + * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * also 4K then the range window to iterate is 64G. + * + * For example iterating on a total IOVA range of 4G..128G, it will walk + * through this set of ranges: + * + * 4G - 68G-1 (64G) + * 68G - 128G-1 (64G) + * + * An example of the APIs on how to use/iterate over the IOVA bitmap: + * + * bitmap = iova_bitmap_alloc(iova, length, page_size, data); + * if (IS_ERR(bitmap)) + * return PTR_ERR(bitmap); + * + * ret = iova_bitmap_for_each(bitmap, arg, dirty_reporter_fn); + * + * iova_bitmap_free(bitmap); + * + * Each iteration of the @dirty_reporter_fn is called with a unique @iova + * and @length argument, indicating the current range available through the + * iova_bitmap. The @dirty_reporter_fn uses iova_bitmap_set() to mark dirty + * areas (@iova_length) within that provided range, as following: + * + * iova_bitmap_set(bitmap, iova, iova_length); + * + * The internals of the object uses an index @mapped_base_index that indexes + * which u64 word of the bitmap is mapped, up to @mapped_total_index. + * Those keep being incremented until @mapped_total_index is reached while + * mapping up to PAGE_SIZE / sizeof(struct page*) maximum of pages. + * + * The IOVA bitmap is usually located on what tracks DMA mapped ranges or + * some form of IOVA range tracking that co-relates to the user passed + * bitmap. + */ +struct iova_bitmap { + /* IOVA range representing the currently mapped bitmap data */ + struct iova_bitmap_map mapped; + + /* userspace address of the bitmap */ + u64 __user *bitmap; + + /* u64 index that @mapped points to */ + unsigned long mapped_base_index; + + /* how many u64 can we walk in total */ + unsigned long mapped_total_index; + + /* base IOVA of the whole bitmap */ + unsigned long iova; + + /* length of the IOVA range for the whole bitmap */ + size_t length; +}; + +/* + * Converts a relative IOVA to a bitmap index. + * This function provides the index into the u64 array (bitmap::bitmap) + * for a given IOVA offset. + * Relative IOVA means relative to the bitmap::mapped base IOVA + * (stored in mapped::iova). All computations in this file are done using + * relative IOVAs and thus avoid an extra subtraction against mapped::iova. + * The user API iova_bitmap_set() always uses a regular absolute IOVAs. + */ +static unsigned long iova_bitmap_offset_to_index(struct iova_bitmap *bitmap, + unsigned long iova) +{ + unsigned long pgsize = 1 << bitmap->mapped.pgshift; + + return iova / (BITS_PER_TYPE(*bitmap->bitmap) * pgsize); +} + +/* + * Converts a bitmap index to a *relative* IOVA. + */ +static unsigned long iova_bitmap_index_to_offset(struct iova_bitmap *bitmap, + unsigned long index) +{ + unsigned long pgshift = bitmap->mapped.pgshift; + + return (index * BITS_PER_TYPE(*bitmap->bitmap)) << pgshift; +} + +/* + * Returns the base IOVA of the mapped range. + */ +static unsigned long iova_bitmap_mapped_iova(struct iova_bitmap *bitmap) +{ + unsigned long skip = bitmap->mapped_base_index; + + return bitmap->iova + iova_bitmap_index_to_offset(bitmap, skip); +} + +/* + * Pins the bitmap user pages for the current range window. + * This is internal to IOVA bitmap and called when advancing the + * index (@mapped_base_index) or allocating the bitmap. + */ +static int iova_bitmap_get(struct iova_bitmap *bitmap) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long npages; + u64 __user *addr; + long ret; + + /* + * @mapped_base_index is the index of the currently mapped u64 words + * that we have access. Anything before @mapped_base_index is not + * mapped. The range @mapped_base_index .. @mapped_total_index-1 is + * mapped but capped at a maximum number of pages. + */ + npages = DIV_ROUND_UP((bitmap->mapped_total_index - + bitmap->mapped_base_index) * + sizeof(*bitmap->bitmap), PAGE_SIZE); + + /* + * We always cap at max number of 'struct page' a base page can fit. + * This is, for example, on x86 means 2M of bitmap data max. + */ + npages = min(npages, PAGE_SIZE / sizeof(struct page *)); + + /* + * Bitmap address to be pinned is calculated via pointer arithmetic + * with bitmap u64 word index. + */ + addr = bitmap->bitmap + bitmap->mapped_base_index; + + ret = pin_user_pages_fast((unsigned long)addr, npages, + FOLL_WRITE, mapped->pages); + if (ret <= 0) + return -EFAULT; + + mapped->npages = (unsigned long)ret; + /* Base IOVA where @pages point to i.e. bit 0 of the first page */ + mapped->iova = iova_bitmap_mapped_iova(bitmap); + + /* + * offset of the page where pinned pages bit 0 is located. + * This handles the case where the bitmap is not PAGE_SIZE + * aligned. + */ + mapped->pgoff = offset_in_page(addr); + return 0; +} + +/* + * Unpins the bitmap user pages and clears @npages + * (un)pinning is abstracted from API user and it's done when advancing + * the index or freeing the bitmap. + */ +static void iova_bitmap_put(struct iova_bitmap *bitmap) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + + if (mapped->npages) { + unpin_user_pages(mapped->pages, mapped->npages); + mapped->npages = 0; + } +} + +/** + * iova_bitmap_alloc() - Allocates an IOVA bitmap object + * @iova: Start address of the IOVA range + * @length: Length of the IOVA range + * @page_size: Page size of the IOVA bitmap. It defines what each bit + * granularity represents + * @data: Userspace address of the bitmap + * + * Allocates an IOVA object and initializes all its fields including the + * first user pages of @data. + * + * Return: A pointer to a newly allocated struct iova_bitmap + * or ERR_PTR() on error. + */ +struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, + unsigned long page_size, u64 __user *data) +{ + struct iova_bitmap_map *mapped; + struct iova_bitmap *bitmap; + int rc; + + bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); + if (!bitmap) + return ERR_PTR(-ENOMEM); + + mapped = &bitmap->mapped; + mapped->pgshift = __ffs(page_size); + bitmap->bitmap = data; + bitmap->mapped_total_index = + iova_bitmap_offset_to_index(bitmap, length - 1) + 1; + bitmap->iova = iova; + bitmap->length = length; + mapped->iova = iova; + mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); + if (!mapped->pages) { + rc = -ENOMEM; + goto err; + } + + rc = iova_bitmap_get(bitmap); + if (rc) + goto err; + return bitmap; + +err: + iova_bitmap_free(bitmap); + return ERR_PTR(rc); +} + +/** + * iova_bitmap_free() - Frees an IOVA bitmap object + * @bitmap: IOVA bitmap to free + * + * It unpins and releases pages array memory and clears any leftover + * state. + */ +void iova_bitmap_free(struct iova_bitmap *bitmap) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + + iova_bitmap_put(bitmap); + + if (mapped->pages) { + free_page((unsigned long)mapped->pages); + mapped->pages = NULL; + } + + kfree(bitmap); +} + +/* + * Returns the remaining bitmap indexes from mapped_total_index to process for + * the currently pinned bitmap pages. + */ +static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) +{ + unsigned long remaining; + + remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; + remaining = min_t(unsigned long, remaining, + (bitmap->mapped.npages << PAGE_SHIFT) / sizeof(*bitmap->bitmap)); + + return remaining; +} + +/* + * Returns the length of the mapped IOVA range. + */ +static unsigned long iova_bitmap_mapped_length(struct iova_bitmap *bitmap) +{ + unsigned long max_iova = bitmap->iova + bitmap->length - 1; + unsigned long iova = iova_bitmap_mapped_iova(bitmap); + unsigned long remaining; + + /* + * iova_bitmap_mapped_remaining() returns a number of indexes which + * when converted to IOVA gives us a max length that the bitmap + * pinned data can cover. Afterwards, that is capped to + * only cover the IOVA range in @bitmap::iova .. @bitmap::length. + */ + remaining = iova_bitmap_index_to_offset(bitmap, + iova_bitmap_mapped_remaining(bitmap)); + + if (iova + remaining - 1 > max_iova) + remaining -= ((iova + remaining - 1) - max_iova); + + return remaining; +} + +/* + * Returns true if there's not more data to iterate. + */ +static bool iova_bitmap_done(struct iova_bitmap *bitmap) +{ + return bitmap->mapped_base_index >= bitmap->mapped_total_index; +} + +/* + * Advances to the next range, releases the current pinned + * pages and pins the next set of bitmap pages. + * Returns 0 on success or otherwise errno. + */ +static int iova_bitmap_advance(struct iova_bitmap *bitmap) +{ + unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1; + unsigned long count = iova_bitmap_offset_to_index(bitmap, iova) + 1; + + bitmap->mapped_base_index += count; + + iova_bitmap_put(bitmap); + if (iova_bitmap_done(bitmap)) + return 0; + + /* When advancing the index we pin the next set of bitmap pages */ + return iova_bitmap_get(bitmap); +} + +/** + * iova_bitmap_for_each() - Iterates over the bitmap + * @bitmap: IOVA bitmap to iterate + * @opaque: Additional argument to pass to the callback + * @fn: Function that gets called for each IOVA range + * + * Helper function to iterate over bitmap data representing a portion of IOVA + * space. It hides the complexity of iterating bitmaps and translating the + * mapped bitmap user pages into IOVA ranges to process. + * + * Return: 0 on success, and an error on failure either upon + * iteration or when the callback returns an error. + */ +int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, + iova_bitmap_fn_t fn) +{ + int ret = 0; + + for (; !iova_bitmap_done(bitmap) && !ret; + ret = iova_bitmap_advance(bitmap)) { + ret = fn(bitmap, iova_bitmap_mapped_iova(bitmap), + iova_bitmap_mapped_length(bitmap), opaque); + if (ret) + break; + } + + return ret; +} + +/** + * iova_bitmap_set() - Records an IOVA range in bitmap + * @bitmap: IOVA bitmap + * @iova: IOVA to start + * @length: IOVA range length + * + * Set the bits corresponding to the range [iova .. iova+length-1] in + * the user bitmap. + * + * Return: The number of bits set. + */ +void iova_bitmap_set(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long offset = (iova - mapped->iova) >> mapped->pgshift; + unsigned long nbits = max_t(unsigned long, 1, length >> mapped->pgshift); + unsigned long page_idx = offset / BITS_PER_PAGE; + unsigned long page_offset = mapped->pgoff; + void *kaddr; + + offset = offset % BITS_PER_PAGE; + + do { + unsigned long size = min(BITS_PER_PAGE - offset, nbits); + + kaddr = kmap_local_page(mapped->pages[page_idx]); + bitmap_set(kaddr + page_offset, offset, size); + kunmap_local(kaddr); + page_offset = offset = 0; + nbits -= size; + page_idx++; + } while (nbits > 0); +} +EXPORT_SYMBOL_GPL(iova_bitmap_set); diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c index b8b9e7911e55916ecc631bcae0dac7583d352a95..58f91b3bd670cc4fc88608dad02f9ab7eb24cb76 100644 --- a/drivers/vfio/mdev/mdev_core.c +++ b/drivers/vfio/mdev/mdev_core.c @@ -8,9 +8,7 @@ */ #include -#include #include -#include #include #include @@ -20,71 +18,11 @@ #define DRIVER_AUTHOR "NVIDIA Corporation" #define DRIVER_DESC "Mediated device Core Driver" -static LIST_HEAD(parent_list); -static DEFINE_MUTEX(parent_list_lock); static struct class_compat *mdev_bus_compat_class; static LIST_HEAD(mdev_list); static DEFINE_MUTEX(mdev_list_lock); -struct device *mdev_parent_dev(struct mdev_device *mdev) -{ - return mdev->type->parent->dev; -} -EXPORT_SYMBOL(mdev_parent_dev); - -/* - * Return the index in supported_type_groups that this mdev_device was created - * from. - */ -unsigned int mdev_get_type_group_id(struct mdev_device *mdev) -{ - return mdev->type->type_group_id; -} -EXPORT_SYMBOL(mdev_get_type_group_id); - -/* - * Used in mdev_type_attribute sysfs functions to return the index in the - * supported_type_groups that the sysfs is called from. - */ -unsigned int mtype_get_type_group_id(struct mdev_type *mtype) -{ - return mtype->type_group_id; -} -EXPORT_SYMBOL(mtype_get_type_group_id); - -/* - * Used in mdev_type_attribute sysfs functions to return the parent struct - * device - */ -struct device *mtype_get_parent_dev(struct mdev_type *mtype) -{ - return mtype->parent->dev; -} -EXPORT_SYMBOL(mtype_get_parent_dev); - -/* Should be called holding parent_list_lock */ -static struct mdev_parent *__find_parent_device(struct device *dev) -{ - struct mdev_parent *parent; - - list_for_each_entry(parent, &parent_list, next) { - if (parent->dev == dev) - return parent; - } - return NULL; -} - -void mdev_release_parent(struct kref *kref) -{ - struct mdev_parent *parent = container_of(kref, struct mdev_parent, - ref); - struct device *dev = parent->dev; - - kfree(parent); - put_device(dev); -} - /* Caller must hold parent unreg_sem read or write lock */ static void mdev_device_remove_common(struct mdev_device *mdev) { @@ -99,145 +37,96 @@ static void mdev_device_remove_common(struct mdev_device *mdev) static int mdev_device_remove_cb(struct device *dev, void *data) { - struct mdev_device *mdev = mdev_from_dev(dev); - - if (mdev) - mdev_device_remove_common(mdev); + if (dev->bus == &mdev_bus_type) + mdev_device_remove_common(to_mdev_device(dev)); return 0; } /* - * mdev_register_device : Register a device + * mdev_register_parent: Register a device as parent for mdevs + * @parent: parent structure registered * @dev: device structure representing parent device. * @mdev_driver: Device driver to bind to the newly created mdev + * @types: Array of supported mdev types + * @nr_types: Number of entries in @types + * + * Registers the @parent stucture as a parent for mdev types and thus mdev + * devices. The caller needs to hold a reference on @dev that must not be + * released until after the call to mdev_unregister_parent(). * - * Add device to list of registered parent devices. * Returns a negative value on error, otherwise 0. */ -int mdev_register_device(struct device *dev, struct mdev_driver *mdev_driver) +int mdev_register_parent(struct mdev_parent *parent, struct device *dev, + struct mdev_driver *mdev_driver, struct mdev_type **types, + unsigned int nr_types) { - int ret; - struct mdev_parent *parent; char *env_string = "MDEV_STATE=registered"; char *envp[] = { env_string, NULL }; + int ret; - /* check for mandatory ops */ - if (!mdev_driver->supported_type_groups) - return -EINVAL; - - dev = get_device(dev); - if (!dev) - return -EINVAL; - - mutex_lock(&parent_list_lock); - - /* Check for duplicate */ - parent = __find_parent_device(dev); - if (parent) { - parent = NULL; - ret = -EEXIST; - goto add_dev_err; - } - - parent = kzalloc(sizeof(*parent), GFP_KERNEL); - if (!parent) { - ret = -ENOMEM; - goto add_dev_err; - } - - kref_init(&parent->ref); + memset(parent, 0, sizeof(*parent)); init_rwsem(&parent->unreg_sem); - parent->dev = dev; parent->mdev_driver = mdev_driver; + parent->types = types; + parent->nr_types = nr_types; + atomic_set(&parent->available_instances, mdev_driver->max_instances); if (!mdev_bus_compat_class) { mdev_bus_compat_class = class_compat_register("mdev_bus"); - if (!mdev_bus_compat_class) { - ret = -ENOMEM; - goto add_dev_err; - } + if (!mdev_bus_compat_class) + return -ENOMEM; } ret = parent_create_sysfs_files(parent); if (ret) - goto add_dev_err; + return ret; ret = class_compat_create_link(mdev_bus_compat_class, dev, NULL); if (ret) dev_warn(dev, "Failed to create compatibility class link\n"); - list_add(&parent->next, &parent_list); - mutex_unlock(&parent_list_lock); - dev_info(dev, "MDEV: Registered\n"); kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp); - return 0; - -add_dev_err: - mutex_unlock(&parent_list_lock); - if (parent) - mdev_put_parent(parent); - else - put_device(dev); - return ret; } -EXPORT_SYMBOL(mdev_register_device); +EXPORT_SYMBOL(mdev_register_parent); /* - * mdev_unregister_device : Unregister a parent device - * @dev: device structure representing parent device. - * - * Remove device from list of registered parent devices. Give a chance to free - * existing mediated devices for given device. + * mdev_unregister_parent : Unregister a parent device + * @parent: parent structure to unregister */ - -void mdev_unregister_device(struct device *dev) +void mdev_unregister_parent(struct mdev_parent *parent) { - struct mdev_parent *parent; char *env_string = "MDEV_STATE=unregistered"; char *envp[] = { env_string, NULL }; - mutex_lock(&parent_list_lock); - parent = __find_parent_device(dev); - - if (!parent) { - mutex_unlock(&parent_list_lock); - return; - } - dev_info(dev, "MDEV: Unregistering\n"); - - list_del(&parent->next); - mutex_unlock(&parent_list_lock); + dev_info(parent->dev, "MDEV: Unregistering\n"); down_write(&parent->unreg_sem); - - class_compat_remove_link(mdev_bus_compat_class, dev, NULL); - - device_for_each_child(dev, NULL, mdev_device_remove_cb); - + class_compat_remove_link(mdev_bus_compat_class, parent->dev, NULL); + device_for_each_child(parent->dev, NULL, mdev_device_remove_cb); parent_remove_sysfs_files(parent); up_write(&parent->unreg_sem); - mdev_put_parent(parent); - - /* We still have the caller's reference to use for the uevent */ - kobject_uevent_env(&dev->kobj, KOBJ_CHANGE, envp); + kobject_uevent_env(&parent->dev->kobj, KOBJ_CHANGE, envp); } -EXPORT_SYMBOL(mdev_unregister_device); +EXPORT_SYMBOL(mdev_unregister_parent); static void mdev_device_release(struct device *dev) { struct mdev_device *mdev = to_mdev_device(dev); - - /* Pairs with the get in mdev_device_create() */ - kobject_put(&mdev->type->kobj); + struct mdev_parent *parent = mdev->type->parent; mutex_lock(&mdev_list_lock); list_del(&mdev->next); + if (!parent->mdev_driver->get_available) + atomic_inc(&parent->available_instances); mutex_unlock(&mdev_list_lock); + /* Pairs with the get in mdev_device_create() */ + kobject_put(&mdev->type->kobj); + dev_dbg(&mdev->dev, "MDEV: destroying\n"); kfree(mdev); } @@ -259,6 +148,18 @@ int mdev_device_create(struct mdev_type *type, const guid_t *uuid) } } + if (!drv->get_available) { + /* + * Note: that non-atomic read and dec is fine here because + * all modifications are under mdev_list_lock. + */ + if (!atomic_read(&parent->available_instances)) { + mutex_unlock(&mdev_list_lock); + return -EUSERS; + } + atomic_dec(&parent->available_instances); + } + mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); if (!mdev) { mutex_unlock(&mdev_list_lock); diff --git a/drivers/vfio/mdev/mdev_driver.c b/drivers/vfio/mdev/mdev_driver.c index 9c2af59809e2e4fc8d958fb3dbe47c8e3e74566b..7825d83a55f8c2777f64e6afae987e1bb6b7a7cf 100644 --- a/drivers/vfio/mdev/mdev_driver.c +++ b/drivers/vfio/mdev/mdev_driver.c @@ -7,7 +7,6 @@ * Kirti Wankhede */ -#include #include #include @@ -47,7 +46,6 @@ struct bus_type mdev_bus_type = { .remove = mdev_remove, .match = mdev_match, }; -EXPORT_SYMBOL_GPL(mdev_bus_type); /** * mdev_register_driver - register a new MDEV driver @@ -57,10 +55,11 @@ EXPORT_SYMBOL_GPL(mdev_bus_type); **/ int mdev_register_driver(struct mdev_driver *drv) { + if (!drv->device_api) + return -EINVAL; + /* initialize common driver fields */ drv->driver.bus = &mdev_bus_type; - - /* register with core */ return driver_register(&drv->driver); } EXPORT_SYMBOL(mdev_register_driver); diff --git a/drivers/vfio/mdev/mdev_private.h b/drivers/vfio/mdev/mdev_private.h index 7c9fc79f3d838405fd5a4fe20adb5a754717eef2..af457b27f6074621bb182c80b7657d2b696e6f61 100644 --- a/drivers/vfio/mdev/mdev_private.h +++ b/drivers/vfio/mdev/mdev_private.h @@ -13,25 +13,7 @@ int mdev_bus_register(void); void mdev_bus_unregister(void); -struct mdev_parent { - struct device *dev; - struct mdev_driver *mdev_driver; - struct kref ref; - struct list_head next; - struct kset *mdev_types_kset; - struct list_head type_list; - /* Synchronize device creation/removal with parent unregistration */ - struct rw_semaphore unreg_sem; -}; - -struct mdev_type { - struct kobject kobj; - struct kobject *devices_kobj; - struct mdev_parent *parent; - struct list_head next; - unsigned int type_group_id; -}; - +extern struct bus_type mdev_bus_type; extern const struct attribute_group *mdev_device_groups[]; #define to_mdev_type_attr(_attr) \ @@ -48,16 +30,4 @@ void mdev_remove_sysfs_files(struct mdev_device *mdev); int mdev_device_create(struct mdev_type *kobj, const guid_t *uuid); int mdev_device_remove(struct mdev_device *dev); -void mdev_release_parent(struct kref *kref); - -static inline void mdev_get_parent(struct mdev_parent *parent) -{ - kref_get(&parent->ref); -} - -static inline void mdev_put_parent(struct mdev_parent *parent) -{ - kref_put(&parent->ref, mdev_release_parent); -} - #endif /* MDEV_PRIVATE_H */ diff --git a/drivers/vfio/mdev/mdev_sysfs.c b/drivers/vfio/mdev/mdev_sysfs.c index 0ccfeb3dda2455fbe5555c283c41581999caccb9..abe3359dd477fc67364084d39141d6018ebad9e0 100644 --- a/drivers/vfio/mdev/mdev_sysfs.c +++ b/drivers/vfio/mdev/mdev_sysfs.c @@ -9,14 +9,24 @@ #include #include -#include #include -#include #include #include "mdev_private.h" -/* Static functions */ +struct mdev_type_attribute { + struct attribute attr; + ssize_t (*show)(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf); + ssize_t (*store)(struct mdev_type *mtype, + struct mdev_type_attribute *attr, const char *buf, + size_t count); +}; + +#define MDEV_TYPE_ATTR_RO(_name) \ + struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RO(_name) +#define MDEV_TYPE_ATTR_WO(_name) \ + struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_WO(_name) static ssize_t mdev_type_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) @@ -74,152 +84,156 @@ static ssize_t create_store(struct mdev_type *mtype, return count; } - static MDEV_TYPE_ATTR_WO(create); +static ssize_t device_api_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", mtype->parent->mdev_driver->device_api); +} +static MDEV_TYPE_ATTR_RO(device_api); + +static ssize_t name_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + return sprintf(buf, "%s\n", + mtype->pretty_name ? mtype->pretty_name : mtype->sysfs_name); +} + +static MDEV_TYPE_ATTR_RO(name); + +static ssize_t available_instances_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, + char *buf) +{ + struct mdev_driver *drv = mtype->parent->mdev_driver; + + if (drv->get_available) + return sysfs_emit(buf, "%u\n", drv->get_available(mtype)); + return sysfs_emit(buf, "%u\n", + atomic_read(&mtype->parent->available_instances)); +} +static MDEV_TYPE_ATTR_RO(available_instances); + +static ssize_t description_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, + char *buf) +{ + return mtype->parent->mdev_driver->show_description(mtype, buf); +} +static MDEV_TYPE_ATTR_RO(description); + +static struct attribute *mdev_types_core_attrs[] = { + &mdev_type_attr_create.attr, + &mdev_type_attr_device_api.attr, + &mdev_type_attr_name.attr, + &mdev_type_attr_available_instances.attr, + &mdev_type_attr_description.attr, + NULL, +}; + +static umode_t mdev_types_core_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + if (attr == &mdev_type_attr_description.attr && + !to_mdev_type(kobj)->parent->mdev_driver->show_description) + return 0; + return attr->mode; +} + +static struct attribute_group mdev_type_core_group = { + .attrs = mdev_types_core_attrs, + .is_visible = mdev_types_core_is_visible, +}; + +static const struct attribute_group *mdev_type_groups[] = { + &mdev_type_core_group, + NULL, +}; + static void mdev_type_release(struct kobject *kobj) { struct mdev_type *type = to_mdev_type(kobj); pr_debug("Releasing group %s\n", kobj->name); /* Pairs with the get in add_mdev_supported_type() */ - mdev_put_parent(type->parent); - kfree(type); + put_device(type->parent->dev); } static struct kobj_type mdev_type_ktype = { - .sysfs_ops = &mdev_type_sysfs_ops, - .release = mdev_type_release, + .sysfs_ops = &mdev_type_sysfs_ops, + .release = mdev_type_release, + .default_groups = mdev_type_groups, }; -static struct mdev_type *add_mdev_supported_type(struct mdev_parent *parent, - unsigned int type_group_id) +static int mdev_type_add(struct mdev_parent *parent, struct mdev_type *type) { - struct mdev_type *type; - struct attribute_group *group = - parent->mdev_driver->supported_type_groups[type_group_id]; int ret; - if (!group->name) { - pr_err("%s: Type name empty!\n", __func__); - return ERR_PTR(-EINVAL); - } - - type = kzalloc(sizeof(*type), GFP_KERNEL); - if (!type) - return ERR_PTR(-ENOMEM); - type->kobj.kset = parent->mdev_types_kset; type->parent = parent; /* Pairs with the put in mdev_type_release() */ - mdev_get_parent(parent); - type->type_group_id = type_group_id; + get_device(parent->dev); ret = kobject_init_and_add(&type->kobj, &mdev_type_ktype, NULL, "%s-%s", dev_driver_string(parent->dev), - group->name); + type->sysfs_name); if (ret) { kobject_put(&type->kobj); - return ERR_PTR(ret); + return ret; } - ret = sysfs_create_file(&type->kobj, &mdev_type_attr_create.attr); - if (ret) - goto attr_create_failed; - type->devices_kobj = kobject_create_and_add("devices", &type->kobj); if (!type->devices_kobj) { ret = -ENOMEM; goto attr_devices_failed; } - ret = sysfs_create_files(&type->kobj, - (const struct attribute **)group->attrs); - if (ret) { - ret = -ENOMEM; - goto attrs_failed; - } - return type; + return 0; -attrs_failed: - kobject_put(type->devices_kobj); attr_devices_failed: - sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr); -attr_create_failed: kobject_del(&type->kobj); kobject_put(&type->kobj); - return ERR_PTR(ret); + return ret; } -static void remove_mdev_supported_type(struct mdev_type *type) +static void mdev_type_remove(struct mdev_type *type) { - struct attribute_group *group = - type->parent->mdev_driver->supported_type_groups[type->type_group_id]; - - sysfs_remove_files(&type->kobj, - (const struct attribute **)group->attrs); kobject_put(type->devices_kobj); - sysfs_remove_file(&type->kobj, &mdev_type_attr_create.attr); kobject_del(&type->kobj); kobject_put(&type->kobj); } -static int add_mdev_supported_type_groups(struct mdev_parent *parent) -{ - int i; - - for (i = 0; parent->mdev_driver->supported_type_groups[i]; i++) { - struct mdev_type *type; - - type = add_mdev_supported_type(parent, i); - if (IS_ERR(type)) { - struct mdev_type *ltype, *tmp; - - list_for_each_entry_safe(ltype, tmp, &parent->type_list, - next) { - list_del(<ype->next); - remove_mdev_supported_type(ltype); - } - return PTR_ERR(type); - } - list_add(&type->next, &parent->type_list); - } - return 0; -} - /* mdev sysfs functions */ void parent_remove_sysfs_files(struct mdev_parent *parent) { - struct mdev_type *type, *tmp; - - list_for_each_entry_safe(type, tmp, &parent->type_list, next) { - list_del(&type->next); - remove_mdev_supported_type(type); - } + int i; + for (i = 0; i < parent->nr_types; i++) + mdev_type_remove(parent->types[i]); kset_unregister(parent->mdev_types_kset); } int parent_create_sysfs_files(struct mdev_parent *parent) { - int ret; + int ret, i; parent->mdev_types_kset = kset_create_and_add("mdev_supported_types", NULL, &parent->dev->kobj); - if (!parent->mdev_types_kset) return -ENOMEM; - INIT_LIST_HEAD(&parent->type_list); - - ret = add_mdev_supported_type_groups(parent); - if (ret) - goto create_err; + for (i = 0; i < parent->nr_types; i++) { + ret = mdev_type_add(parent, parent->types[i]); + if (ret) + goto out_err; + } return 0; -create_err: - kset_unregister(parent->mdev_types_kset); - return ret; +out_err: + while (--i >= 0) + mdev_type_remove(parent->types[i]); + return 0; } static ssize_t remove_store(struct device *dev, struct device_attribute *attr, diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c index ea762e28c1cc6ece32f624740d6cf8b74013651e..39eeca18a0f7c884a18827fbe6d4ccf2584920b9 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c @@ -16,7 +16,7 @@ #include "hisi_acc_vfio_pci.h" -/* return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ +/* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */ static int qm_wait_dev_not_ready(struct hisi_qm *qm) { u32 val; @@ -189,7 +189,7 @@ static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data) struct device *dev = &qm->pdev->dev; int ret; - /* check VF state */ + /* Check VF state */ if (unlikely(hisi_qm_wait_mb_ready(qm))) { dev_err(&qm->pdev->dev, "QM device is not ready to write\n"); return -EBUSY; @@ -337,16 +337,7 @@ static int vf_qm_cache_wb(struct hisi_qm *qm) return 0; } -static struct hisi_acc_vf_core_device *hssi_acc_drvdata(struct pci_dev *pdev) -{ - struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); - - return container_of(core_device, struct hisi_acc_vf_core_device, - core_device); -} - -static void vf_qm_fun_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev, - struct hisi_qm *qm) +static void vf_qm_fun_reset(struct hisi_qm *qm) { int i; @@ -382,7 +373,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL; } - /* vf qp num check */ + /* VF qp num check */ ret = qm_get_vft(vf_qm, &vf_qm->qp_base); if (ret <= 0) { dev_err(dev, "failed to get vft qp nums\n"); @@ -396,7 +387,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, vf_qm->qp_num = ret; - /* vf isolation state check */ + /* VF isolation state check */ ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1); if (ret) { dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n"); @@ -405,7 +396,7 @@ static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev, if (vf_data->que_iso_cfg != que_iso_state) { dev_err(dev, "failed to match isolation state\n"); - return ret; + return -EINVAL; } ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1); @@ -427,10 +418,10 @@ static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, int ret; vf_data->acc_magic = ACC_DEV_MAGIC; - /* save device id */ + /* Save device id */ vf_data->dev_id = hisi_acc_vdev->vf_dev->device; - /* vf qp num save from PF */ + /* VF qp num save from PF */ ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base); if (ret <= 0) { dev_err(dev, "failed to get vft qp nums!\n"); @@ -474,19 +465,19 @@ static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev, ret = qm_set_regs(qm, vf_data); if (ret) { - dev_err(dev, "Set VF regs failed\n"); + dev_err(dev, "set VF regs failed\n"); return ret; } ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); if (ret) { - dev_err(dev, "Set sqc failed\n"); + dev_err(dev, "set sqc failed\n"); return ret; } ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); if (ret) { - dev_err(dev, "Set cqc failed\n"); + dev_err(dev, "set cqc failed\n"); return ret; } @@ -528,12 +519,12 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, return -EINVAL; /* Every reg is 32 bit, the dma address is 64 bit. */ - vf_data->eqe_dma = vf_data->qm_eqc_dw[2]; + vf_data->eqe_dma = vf_data->qm_eqc_dw[1]; vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET; - vf_data->eqe_dma |= vf_data->qm_eqc_dw[1]; - vf_data->aeqe_dma = vf_data->qm_aeqc_dw[2]; + vf_data->eqe_dma |= vf_data->qm_eqc_dw[0]; + vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1]; vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET; - vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[1]; + vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0]; /* Through SQC_BT/CQC_BT to get sqc and cqc address */ ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma); @@ -552,6 +543,14 @@ static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev, return 0; } +static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev) +{ + struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev); + + return container_of(core_device, struct hisi_acc_vf_core_device, + core_device); +} + /* Check the PF's RAS state and Function INT state */ static int hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) @@ -662,7 +661,10 @@ static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vd if (hisi_acc_vdev->vf_qm_state != QM_READY) return; - vf_qm_fun_reset(hisi_acc_vdev, vf_qm); + /* Make sure the device is enabled */ + qm_dev_cmd_init(vf_qm); + + vf_qm_fun_reset(vf_qm); } static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev) @@ -970,7 +972,7 @@ hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev, static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev) { - struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev); + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); if (hisi_acc_vdev->core_device.vdev.migration_flags != VFIO_MIGRATION_STOP_COPY) @@ -1213,8 +1215,28 @@ static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = { .migration_get_state = hisi_acc_vfio_pci_get_device_state, }; +static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev) +{ + struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev, + struct hisi_acc_vf_core_device, core_device.vdev); + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); + struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev); + + hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1; + hisi_acc_vdev->pf_qm = pf_qm; + hisi_acc_vdev->vf_dev = pdev; + mutex_init(&hisi_acc_vdev->state_mutex); + + core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY; + core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops; + + return vfio_pci_core_init_dev(core_vdev); +} + static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { .name = "hisi-acc-vfio-pci-migration", + .init = hisi_acc_vfio_pci_migrn_init_dev, + .release = vfio_pci_core_release_dev, .open_device = hisi_acc_vfio_pci_open_device, .close_device = hisi_acc_vfio_pci_close_device, .ioctl = hisi_acc_vfio_pci_ioctl, @@ -1228,6 +1250,8 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = { static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { .name = "hisi-acc-vfio-pci", + .init = vfio_pci_core_init_dev, + .release = vfio_pci_core_release_dev, .open_device = hisi_acc_vfio_pci_open_device, .close_device = vfio_pci_core_close_device, .ioctl = vfio_pci_core_ioctl, @@ -1239,73 +1263,45 @@ static const struct vfio_device_ops hisi_acc_vfio_pci_ops = { .match = vfio_pci_core_match, }; -static int -hisi_acc_vfio_pci_migrn_init(struct hisi_acc_vf_core_device *hisi_acc_vdev, - struct pci_dev *pdev, struct hisi_qm *pf_qm) -{ - int vf_id; - - vf_id = pci_iov_vf_id(pdev); - if (vf_id < 0) - return vf_id; - - hisi_acc_vdev->vf_id = vf_id + 1; - hisi_acc_vdev->core_device.vdev.migration_flags = - VFIO_MIGRATION_STOP_COPY; - hisi_acc_vdev->pf_qm = pf_qm; - hisi_acc_vdev->vf_dev = pdev; - mutex_init(&hisi_acc_vdev->state_mutex); - - return 0; -} - static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct hisi_acc_vf_core_device *hisi_acc_vdev; + const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops; struct hisi_qm *pf_qm; + int vf_id; int ret; - hisi_acc_vdev = kzalloc(sizeof(*hisi_acc_vdev), GFP_KERNEL); - if (!hisi_acc_vdev) - return -ENOMEM; - pf_qm = hisi_acc_get_pf_qm(pdev); if (pf_qm && pf_qm->ver >= QM_HW_V3) { - ret = hisi_acc_vfio_pci_migrn_init(hisi_acc_vdev, pdev, pf_qm); - if (!ret) { - vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, - &hisi_acc_vfio_pci_migrn_ops); - hisi_acc_vdev->core_device.vdev.mig_ops = - &hisi_acc_vfio_pci_migrn_state_ops; - } else { + vf_id = pci_iov_vf_id(pdev); + if (vf_id >= 0) + ops = &hisi_acc_vfio_pci_migrn_ops; + else pci_warn(pdev, "migration support failed, continue with generic interface\n"); - vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, - &hisi_acc_vfio_pci_ops); - } - } else { - vfio_pci_core_init_device(&hisi_acc_vdev->core_device, pdev, - &hisi_acc_vfio_pci_ops); } + hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device, + core_device.vdev, &pdev->dev, ops); + if (IS_ERR(hisi_acc_vdev)) + return PTR_ERR(hisi_acc_vdev); + dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device); ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device); if (ret) - goto out_free; + goto out_put_vdev; return 0; -out_free: - vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device); - kfree(hisi_acc_vdev); +out_put_vdev: + vfio_put_device(&hisi_acc_vdev->core_device.vdev); return ret; } static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev) { - struct hisi_acc_vf_core_device *hisi_acc_vdev = hssi_acc_drvdata(pdev); + struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev); vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device); - vfio_pci_core_uninit_device(&hisi_acc_vdev->core_device); - kfree(hisi_acc_vdev); + vfio_put_device(&hisi_acc_vdev->core_device.vdev); } static const struct pci_device_id hisi_acc_vfio_pci_table[] = { diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h index 5494f4983bbe4858298ab4edcb3b2ab7f53c60dc..67343325b320165499a6e17bea7e6d9cc9b7e7c3 100644 --- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h +++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.h @@ -16,7 +16,6 @@ #define SEC_CORE_INT_STATUS 0x301008 #define HPRE_HAC_INT_STATUS 0x301800 #define HZIP_CORE_INT_STATUS 0x3010AC -#define QM_QUE_ISO_CFG 0x301154 #define QM_VFT_CFG_RDY 0x10006c #define QM_VFT_CFG_OP_WR 0x100058 @@ -80,7 +79,7 @@ struct acc_vf_data { /* QM reserved 5 regs */ u32 qm_rsv_regs[5]; u32 padding; - /* qm memory init information */ + /* QM memory init information */ u64 eqe_dma; u64 aeqe_dma; u64 sqc_dma; @@ -99,7 +98,7 @@ struct hisi_acc_vf_migration_file { struct hisi_acc_vf_core_device { struct vfio_pci_core_device core_device; u8 deferred_reset:1; - /* for migration state */ + /* For migration state */ struct mutex state_mutex; enum vfio_device_mig_state mig_state; struct pci_dev *pf_dev; @@ -108,7 +107,7 @@ struct hisi_acc_vf_core_device { struct hisi_qm vf_qm; u32 vf_qm_state; int vf_id; - /* for reset handler */ + /* For reset handler */ spinlock_t reset_lock; struct hisi_acc_vf_migration_file *resuming_migf; struct hisi_acc_vf_migration_file *saving_migf; diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c index dd5d7bfe0a498422ccb9098d8b18f3ccfc370f1b..c604b70437a5d5e56f6f8d71808a8c61c1aa0490 100644 --- a/drivers/vfio/pci/mlx5/cmd.c +++ b/drivers/vfio/pci/mlx5/cmd.c @@ -5,8 +5,12 @@ #include "cmd.h" +enum { CQ_OK = 0, CQ_EMPTY = -1, CQ_POLL_ERR = -2 }; + static int mlx5vf_cmd_get_vhca_id(struct mlx5_core_dev *mdev, u16 function_id, u16 *vhca_id); +static void +_mlx5vf_free_page_tracker_resources(struct mlx5vf_pci_core_device *mvdev); int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod) { @@ -66,25 +70,35 @@ int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, return 0; } +static void set_tracker_error(struct mlx5vf_pci_core_device *mvdev) +{ + /* Mark the tracker under an error and wake it up if it's running */ + mvdev->tracker.is_err = true; + complete(&mvdev->tracker_comp); +} + static int mlx5fv_vf_event(struct notifier_block *nb, unsigned long event, void *data) { struct mlx5vf_pci_core_device *mvdev = container_of(nb, struct mlx5vf_pci_core_device, nb); - mutex_lock(&mvdev->state_mutex); switch (event) { case MLX5_PF_NOTIFY_ENABLE_VF: + mutex_lock(&mvdev->state_mutex); mvdev->mdev_detach = false; + mlx5vf_state_mutex_unlock(mvdev); break; case MLX5_PF_NOTIFY_DISABLE_VF: - mlx5vf_disable_fds(mvdev); + mlx5vf_cmd_close_migratable(mvdev); + mutex_lock(&mvdev->state_mutex); mvdev->mdev_detach = true; + mlx5vf_state_mutex_unlock(mvdev); break; default: break; } - mlx5vf_state_mutex_unlock(mvdev); + return 0; } @@ -93,8 +107,11 @@ void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev) if (!mvdev->migrate_cap) return; + /* Must be done outside the lock to let it progress */ + set_tracker_error(mvdev); mutex_lock(&mvdev->state_mutex); mlx5vf_disable_fds(mvdev); + _mlx5vf_free_page_tracker_resources(mvdev); mlx5vf_state_mutex_unlock(mvdev); } @@ -109,7 +126,8 @@ void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev) } void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, - const struct vfio_migration_ops *mig_ops) + const struct vfio_migration_ops *mig_ops, + const struct vfio_log_ops *log_ops) { struct pci_dev *pdev = mvdev->core_device.pdev; int ret; @@ -151,6 +169,9 @@ void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; mvdev->core_device.vdev.mig_ops = mig_ops; + init_completion(&mvdev->tracker_comp); + if (MLX5_CAP_GEN(mvdev->mdev, adv_virtualization)) + mvdev->core_device.vdev.log_ops = log_ops; end: mlx5_vf_put_core_dev(mvdev->mdev); @@ -188,11 +209,13 @@ err_exec: return ret; } -static int _create_state_mkey(struct mlx5_core_dev *mdev, u32 pdn, - struct mlx5_vf_migration_file *migf, u32 *mkey) +static int _create_mkey(struct mlx5_core_dev *mdev, u32 pdn, + struct mlx5_vf_migration_file *migf, + struct mlx5_vhca_recv_buf *recv_buf, + u32 *mkey) { - size_t npages = DIV_ROUND_UP(migf->total_length, PAGE_SIZE); - struct sg_dma_page_iter dma_iter; + size_t npages = migf ? DIV_ROUND_UP(migf->total_length, PAGE_SIZE) : + recv_buf->npages; int err = 0, inlen; __be64 *mtt; void *mkc; @@ -209,8 +232,17 @@ static int _create_state_mkey(struct mlx5_core_dev *mdev, u32 pdn, DIV_ROUND_UP(npages, 2)); mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); - for_each_sgtable_dma_page(&migf->table.sgt, &dma_iter, 0) - *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter)); + if (migf) { + struct sg_dma_page_iter dma_iter; + + for_each_sgtable_dma_page(&migf->table.sgt, &dma_iter, 0) + *mtt++ = cpu_to_be64(sg_page_iter_dma_address(&dma_iter)); + } else { + int i; + + for (i = 0; i < npages; i++) + *mtt++ = cpu_to_be64(recv_buf->dma_addrs[i]); + } mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); @@ -223,7 +255,8 @@ static int _create_state_mkey(struct mlx5_core_dev *mdev, u32 pdn, MLX5_SET(mkc, mkc, qpn, 0xffffff); MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); MLX5_SET(mkc, mkc, translations_octword_size, DIV_ROUND_UP(npages, 2)); - MLX5_SET64(mkc, mkc, len, migf->total_length); + MLX5_SET64(mkc, mkc, len, + migf ? migf->total_length : (npages * PAGE_SIZE)); err = mlx5_core_create_mkey(mdev, mkey, in, inlen); kvfree(in); return err; @@ -297,7 +330,7 @@ int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, if (err) goto err_dma_map; - err = _create_state_mkey(mdev, pdn, migf, &mkey); + err = _create_mkey(mdev, pdn, migf, NULL, &mkey); if (err) goto err_create_mkey; @@ -369,7 +402,7 @@ int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, if (err) goto err_reg; - err = _create_state_mkey(mdev, pdn, migf, &mkey); + err = _create_mkey(mdev, pdn, migf, NULL, &mkey); if (err) goto err_mkey; @@ -391,3 +424,939 @@ end: mutex_unlock(&migf->lock); return err; } + +static void combine_ranges(struct rb_root_cached *root, u32 cur_nodes, + u32 req_nodes) +{ + struct interval_tree_node *prev, *curr, *comb_start, *comb_end; + unsigned long min_gap; + unsigned long curr_gap; + + /* Special shortcut when a single range is required */ + if (req_nodes == 1) { + unsigned long last; + + curr = comb_start = interval_tree_iter_first(root, 0, ULONG_MAX); + while (curr) { + last = curr->last; + prev = curr; + curr = interval_tree_iter_next(curr, 0, ULONG_MAX); + if (prev != comb_start) + interval_tree_remove(prev, root); + } + comb_start->last = last; + return; + } + + /* Combine ranges which have the smallest gap */ + while (cur_nodes > req_nodes) { + prev = NULL; + min_gap = ULONG_MAX; + curr = interval_tree_iter_first(root, 0, ULONG_MAX); + while (curr) { + if (prev) { + curr_gap = curr->start - prev->last; + if (curr_gap < min_gap) { + min_gap = curr_gap; + comb_start = prev; + comb_end = curr; + } + } + prev = curr; + curr = interval_tree_iter_next(curr, 0, ULONG_MAX); + } + comb_start->last = comb_end->last; + interval_tree_remove(comb_end, root); + cur_nodes--; + } +} + +static int mlx5vf_create_tracker(struct mlx5_core_dev *mdev, + struct mlx5vf_pci_core_device *mvdev, + struct rb_root_cached *ranges, u32 nnodes) +{ + int max_num_range = + MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_max_num_range); + struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker; + int record_size = MLX5_ST_SZ_BYTES(page_track_range); + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; + struct interval_tree_node *node = NULL; + u64 total_ranges_len = 0; + u32 num_ranges = nnodes; + u8 log_addr_space_size; + void *range_list_ptr; + void *obj_context; + void *cmd_hdr; + int inlen; + void *in; + int err; + int i; + + if (num_ranges > max_num_range) { + combine_ranges(ranges, nnodes, max_num_range); + num_ranges = max_num_range; + } + + inlen = MLX5_ST_SZ_BYTES(create_page_track_obj_in) + + record_size * num_ranges; + in = kzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + cmd_hdr = MLX5_ADDR_OF(create_page_track_obj_in, in, + general_obj_in_cmd_hdr); + MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, + MLX5_OBJ_TYPE_PAGE_TRACK); + obj_context = MLX5_ADDR_OF(create_page_track_obj_in, in, obj_context); + MLX5_SET(page_track, obj_context, vhca_id, mvdev->vhca_id); + MLX5_SET(page_track, obj_context, track_type, 1); + MLX5_SET(page_track, obj_context, log_page_size, + ilog2(tracker->host_qp->tracked_page_size)); + MLX5_SET(page_track, obj_context, log_msg_size, + ilog2(tracker->host_qp->max_msg_size)); + MLX5_SET(page_track, obj_context, reporting_qpn, tracker->fw_qp->qpn); + MLX5_SET(page_track, obj_context, num_ranges, num_ranges); + + range_list_ptr = MLX5_ADDR_OF(page_track, obj_context, track_range); + node = interval_tree_iter_first(ranges, 0, ULONG_MAX); + for (i = 0; i < num_ranges; i++) { + void *addr_range_i_base = range_list_ptr + record_size * i; + unsigned long length = node->last - node->start; + + MLX5_SET64(page_track_range, addr_range_i_base, start_address, + node->start); + MLX5_SET64(page_track_range, addr_range_i_base, length, length); + total_ranges_len += length; + node = interval_tree_iter_next(node, 0, ULONG_MAX); + } + + WARN_ON(node); + log_addr_space_size = ilog2(total_ranges_len); + if (log_addr_space_size < + (MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_log_min_addr_space)) || + log_addr_space_size > + (MLX5_CAP_ADV_VIRTUALIZATION(mdev, pg_track_log_max_addr_space))) { + err = -EOPNOTSUPP; + goto out; + } + + MLX5_SET(page_track, obj_context, log_addr_space_size, + log_addr_space_size); + err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); + if (err) + goto out; + + tracker->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); +out: + kfree(in); + return err; +} + +static int mlx5vf_cmd_destroy_tracker(struct mlx5_core_dev *mdev, + u32 tracker_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_PAGE_TRACK); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, tracker_id); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static int mlx5vf_cmd_modify_tracker(struct mlx5_core_dev *mdev, + u32 tracker_id, unsigned long iova, + unsigned long length, u32 tracker_state) +{ + u32 in[MLX5_ST_SZ_DW(modify_page_track_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; + void *obj_context; + void *cmd_hdr; + + cmd_hdr = MLX5_ADDR_OF(modify_page_track_obj_in, in, general_obj_in_cmd_hdr); + MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_PAGE_TRACK); + MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, tracker_id); + + obj_context = MLX5_ADDR_OF(modify_page_track_obj_in, in, obj_context); + MLX5_SET64(page_track, obj_context, modify_field_select, 0x3); + MLX5_SET64(page_track, obj_context, range_start_address, iova); + MLX5_SET64(page_track, obj_context, length, length); + MLX5_SET(page_track, obj_context, state, tracker_state); + + return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); +} + +static int alloc_cq_frag_buf(struct mlx5_core_dev *mdev, + struct mlx5_vhca_cq_buf *buf, int nent, + int cqe_size) +{ + struct mlx5_frag_buf *frag_buf = &buf->frag_buf; + u8 log_wq_stride = 6 + (cqe_size == 128 ? 1 : 0); + u8 log_wq_sz = ilog2(cqe_size); + int err; + + err = mlx5_frag_buf_alloc_node(mdev, nent * cqe_size, frag_buf, + mdev->priv.numa_node); + if (err) + return err; + + mlx5_init_fbc(frag_buf->frags, log_wq_stride, log_wq_sz, &buf->fbc); + buf->cqe_size = cqe_size; + buf->nent = nent; + return 0; +} + +static void init_cq_frag_buf(struct mlx5_vhca_cq_buf *buf) +{ + struct mlx5_cqe64 *cqe64; + void *cqe; + int i; + + for (i = 0; i < buf->nent; i++) { + cqe = mlx5_frag_buf_get_wqe(&buf->fbc, i); + cqe64 = buf->cqe_size == 64 ? cqe : cqe + 64; + cqe64->op_own = MLX5_CQE_INVALID << 4; + } +} + +static void mlx5vf_destroy_cq(struct mlx5_core_dev *mdev, + struct mlx5_vhca_cq *cq) +{ + mlx5_core_destroy_cq(mdev, &cq->mcq); + mlx5_frag_buf_free(mdev, &cq->buf.frag_buf); + mlx5_db_free(mdev, &cq->db); +} + +static void mlx5vf_cq_event(struct mlx5_core_cq *mcq, enum mlx5_event type) +{ + if (type != MLX5_EVENT_TYPE_CQ_ERROR) + return; + + set_tracker_error(container_of(mcq, struct mlx5vf_pci_core_device, + tracker.cq.mcq)); +} + +static int mlx5vf_event_notifier(struct notifier_block *nb, unsigned long type, + void *data) +{ + struct mlx5_vhca_page_tracker *tracker = + mlx5_nb_cof(nb, struct mlx5_vhca_page_tracker, nb); + struct mlx5vf_pci_core_device *mvdev = container_of( + tracker, struct mlx5vf_pci_core_device, tracker); + struct mlx5_eqe *eqe = data; + u8 event_type = (u8)type; + u8 queue_type; + int qp_num; + + switch (event_type) { + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + queue_type = eqe->data.qp_srq.type; + if (queue_type != MLX5_EVENT_QUEUE_TYPE_QP) + break; + qp_num = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; + if (qp_num != tracker->host_qp->qpn && + qp_num != tracker->fw_qp->qpn) + break; + set_tracker_error(mvdev); + break; + default: + break; + } + + return NOTIFY_OK; +} + +static void mlx5vf_cq_complete(struct mlx5_core_cq *mcq, + struct mlx5_eqe *eqe) +{ + struct mlx5vf_pci_core_device *mvdev = + container_of(mcq, struct mlx5vf_pci_core_device, + tracker.cq.mcq); + + complete(&mvdev->tracker_comp); +} + +static int mlx5vf_create_cq(struct mlx5_core_dev *mdev, + struct mlx5_vhca_page_tracker *tracker, + size_t ncqe) +{ + int cqe_size = cache_line_size() == 128 ? 128 : 64; + u32 out[MLX5_ST_SZ_DW(create_cq_out)]; + struct mlx5_vhca_cq *cq; + int inlen, err, eqn; + void *cqc, *in; + __be64 *pas; + int vector; + + cq = &tracker->cq; + ncqe = roundup_pow_of_two(ncqe); + err = mlx5_db_alloc_node(mdev, &cq->db, mdev->priv.numa_node); + if (err) + return err; + + cq->ncqe = ncqe; + cq->mcq.set_ci_db = cq->db.db; + cq->mcq.arm_db = cq->db.db + 1; + cq->mcq.cqe_sz = cqe_size; + err = alloc_cq_frag_buf(mdev, &cq->buf, ncqe, cqe_size); + if (err) + goto err_db_free; + + init_cq_frag_buf(&cq->buf); + inlen = MLX5_ST_SZ_BYTES(create_cq_in) + + MLX5_FLD_SZ_BYTES(create_cq_in, pas[0]) * + cq->buf.frag_buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_buff; + } + + vector = raw_smp_processor_id() % mlx5_comp_vectors_count(mdev); + err = mlx5_vector2eqn(mdev, vector, &eqn); + if (err) + goto err_vec; + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + MLX5_SET(cqc, cqc, log_cq_size, ilog2(ncqe)); + MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); + MLX5_SET(cqc, cqc, uar_page, tracker->uar->index); + MLX5_SET(cqc, cqc, log_page_size, cq->buf.frag_buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(cqc, cqc, dbr_addr, cq->db.dma); + pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); + mlx5_fill_page_frag_array(&cq->buf.frag_buf, pas); + cq->mcq.comp = mlx5vf_cq_complete; + cq->mcq.event = mlx5vf_cq_event; + err = mlx5_core_create_cq(mdev, &cq->mcq, in, inlen, out, sizeof(out)); + if (err) + goto err_vec; + + mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, tracker->uar->map, + cq->mcq.cons_index); + kvfree(in); + return 0; + +err_vec: + kvfree(in); +err_buff: + mlx5_frag_buf_free(mdev, &cq->buf.frag_buf); +err_db_free: + mlx5_db_free(mdev, &cq->db); + return err; +} + +static struct mlx5_vhca_qp * +mlx5vf_create_rc_qp(struct mlx5_core_dev *mdev, + struct mlx5_vhca_page_tracker *tracker, u32 max_recv_wr) +{ + u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {}; + struct mlx5_vhca_qp *qp; + u8 log_rq_stride; + u8 log_rq_sz; + void *qpc; + int inlen; + void *in; + int err; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + return ERR_PTR(-ENOMEM); + + qp->rq.wqe_cnt = roundup_pow_of_two(max_recv_wr); + log_rq_stride = ilog2(MLX5_SEND_WQE_DS); + log_rq_sz = ilog2(qp->rq.wqe_cnt); + err = mlx5_db_alloc_node(mdev, &qp->db, mdev->priv.numa_node); + if (err) + goto err_free; + + if (max_recv_wr) { + err = mlx5_frag_buf_alloc_node(mdev, + wq_get_byte_sz(log_rq_sz, log_rq_stride), + &qp->buf, mdev->priv.numa_node); + if (err) + goto err_db_free; + mlx5_init_fbc(qp->buf.frags, log_rq_stride, log_rq_sz, &qp->rq.fbc); + } + + qp->rq.db = &qp->db.db[MLX5_RCV_DBR]; + inlen = MLX5_ST_SZ_BYTES(create_qp_in) + + MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * + qp->buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_in; + } + + qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); + MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); + MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); + MLX5_SET(qpc, qpc, pd, tracker->pdn); + MLX5_SET(qpc, qpc, uar_page, tracker->uar->index); + MLX5_SET(qpc, qpc, log_page_size, + qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(mdev)); + if (MLX5_CAP_GEN(mdev, cqe_version) == 1) + MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); + MLX5_SET(qpc, qpc, no_sq, 1); + if (max_recv_wr) { + MLX5_SET(qpc, qpc, cqn_rcv, tracker->cq.mcq.cqn); + MLX5_SET(qpc, qpc, log_rq_stride, log_rq_stride - 4); + MLX5_SET(qpc, qpc, log_rq_size, log_rq_sz); + MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); + MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma); + mlx5_fill_page_frag_array(&qp->buf, + (__be64 *)MLX5_ADDR_OF(create_qp_in, + in, pas)); + } else { + MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ); + } + + MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP); + err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out)); + kvfree(in); + if (err) + goto err_in; + + qp->qpn = MLX5_GET(create_qp_out, out, qpn); + return qp; + +err_in: + if (max_recv_wr) + mlx5_frag_buf_free(mdev, &qp->buf); +err_db_free: + mlx5_db_free(mdev, &qp->db); +err_free: + kfree(qp); + return ERR_PTR(err); +} + +static void mlx5vf_post_recv(struct mlx5_vhca_qp *qp) +{ + struct mlx5_wqe_data_seg *data; + unsigned int ix; + + WARN_ON(qp->rq.pc - qp->rq.cc >= qp->rq.wqe_cnt); + ix = qp->rq.pc & (qp->rq.wqe_cnt - 1); + data = mlx5_frag_buf_get_wqe(&qp->rq.fbc, ix); + data->byte_count = cpu_to_be32(qp->max_msg_size); + data->lkey = cpu_to_be32(qp->recv_buf.mkey); + data->addr = cpu_to_be64(qp->recv_buf.next_rq_offset); + qp->rq.pc++; + /* Make sure that descriptors are written before doorbell record. */ + dma_wmb(); + *qp->rq.db = cpu_to_be32(qp->rq.pc & 0xffff); +} + +static int mlx5vf_activate_qp(struct mlx5_core_dev *mdev, + struct mlx5_vhca_qp *qp, u32 remote_qpn, + bool host_qp) +{ + u32 init_in[MLX5_ST_SZ_DW(rst2init_qp_in)] = {}; + u32 rtr_in[MLX5_ST_SZ_DW(init2rtr_qp_in)] = {}; + u32 rts_in[MLX5_ST_SZ_DW(rtr2rts_qp_in)] = {}; + void *qpc; + int ret; + + /* Init */ + qpc = MLX5_ADDR_OF(rst2init_qp_in, init_in, qpc); + MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); + MLX5_SET(qpc, qpc, pm_state, MLX5_QPC_PM_STATE_MIGRATED); + MLX5_SET(qpc, qpc, rre, 1); + MLX5_SET(qpc, qpc, rwe, 1); + MLX5_SET(rst2init_qp_in, init_in, opcode, MLX5_CMD_OP_RST2INIT_QP); + MLX5_SET(rst2init_qp_in, init_in, qpn, qp->qpn); + ret = mlx5_cmd_exec_in(mdev, rst2init_qp, init_in); + if (ret) + return ret; + + if (host_qp) { + struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf; + int i; + + for (i = 0; i < qp->rq.wqe_cnt; i++) { + mlx5vf_post_recv(qp); + recv_buf->next_rq_offset += qp->max_msg_size; + } + } + + /* RTR */ + qpc = MLX5_ADDR_OF(init2rtr_qp_in, rtr_in, qpc); + MLX5_SET(init2rtr_qp_in, rtr_in, qpn, qp->qpn); + MLX5_SET(qpc, qpc, mtu, IB_MTU_4096); + MLX5_SET(qpc, qpc, log_msg_max, MLX5_CAP_GEN(mdev, log_max_msg)); + MLX5_SET(qpc, qpc, remote_qpn, remote_qpn); + MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1); + MLX5_SET(qpc, qpc, primary_address_path.fl, 1); + MLX5_SET(qpc, qpc, min_rnr_nak, 1); + MLX5_SET(init2rtr_qp_in, rtr_in, opcode, MLX5_CMD_OP_INIT2RTR_QP); + MLX5_SET(init2rtr_qp_in, rtr_in, qpn, qp->qpn); + ret = mlx5_cmd_exec_in(mdev, init2rtr_qp, rtr_in); + if (ret || host_qp) + return ret; + + /* RTS */ + qpc = MLX5_ADDR_OF(rtr2rts_qp_in, rts_in, qpc); + MLX5_SET(rtr2rts_qp_in, rts_in, qpn, qp->qpn); + MLX5_SET(qpc, qpc, retry_count, 7); + MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */ + MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x8); /* ~1ms */ + MLX5_SET(rtr2rts_qp_in, rts_in, opcode, MLX5_CMD_OP_RTR2RTS_QP); + MLX5_SET(rtr2rts_qp_in, rts_in, qpn, qp->qpn); + + return mlx5_cmd_exec_in(mdev, rtr2rts_qp, rts_in); +} + +static void mlx5vf_destroy_qp(struct mlx5_core_dev *mdev, + struct mlx5_vhca_qp *qp) +{ + u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {}; + + MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP); + MLX5_SET(destroy_qp_in, in, qpn, qp->qpn); + mlx5_cmd_exec_in(mdev, destroy_qp, in); + + mlx5_frag_buf_free(mdev, &qp->buf); + mlx5_db_free(mdev, &qp->db); + kfree(qp); +} + +static void free_recv_pages(struct mlx5_vhca_recv_buf *recv_buf) +{ + int i; + + /* Undo alloc_pages_bulk_array() */ + for (i = 0; i < recv_buf->npages; i++) + __free_page(recv_buf->page_list[i]); + + kvfree(recv_buf->page_list); +} + +static int alloc_recv_pages(struct mlx5_vhca_recv_buf *recv_buf, + unsigned int npages) +{ + unsigned int filled = 0, done = 0; + int i; + + recv_buf->page_list = kvcalloc(npages, sizeof(*recv_buf->page_list), + GFP_KERNEL); + if (!recv_buf->page_list) + return -ENOMEM; + + for (;;) { + filled = alloc_pages_bulk_array(GFP_KERNEL, npages - done, + recv_buf->page_list + done); + if (!filled) + goto err; + + done += filled; + if (done == npages) + break; + } + + recv_buf->npages = npages; + return 0; + +err: + for (i = 0; i < npages; i++) { + if (recv_buf->page_list[i]) + __free_page(recv_buf->page_list[i]); + } + + kvfree(recv_buf->page_list); + return -ENOMEM; +} + +static int register_dma_recv_pages(struct mlx5_core_dev *mdev, + struct mlx5_vhca_recv_buf *recv_buf) +{ + int i, j; + + recv_buf->dma_addrs = kvcalloc(recv_buf->npages, + sizeof(*recv_buf->dma_addrs), + GFP_KERNEL); + if (!recv_buf->dma_addrs) + return -ENOMEM; + + for (i = 0; i < recv_buf->npages; i++) { + recv_buf->dma_addrs[i] = dma_map_page(mdev->device, + recv_buf->page_list[i], + 0, PAGE_SIZE, + DMA_FROM_DEVICE); + if (dma_mapping_error(mdev->device, recv_buf->dma_addrs[i])) + goto error; + } + return 0; + +error: + for (j = 0; j < i; j++) + dma_unmap_single(mdev->device, recv_buf->dma_addrs[j], + PAGE_SIZE, DMA_FROM_DEVICE); + + kvfree(recv_buf->dma_addrs); + return -ENOMEM; +} + +static void unregister_dma_recv_pages(struct mlx5_core_dev *mdev, + struct mlx5_vhca_recv_buf *recv_buf) +{ + int i; + + for (i = 0; i < recv_buf->npages; i++) + dma_unmap_single(mdev->device, recv_buf->dma_addrs[i], + PAGE_SIZE, DMA_FROM_DEVICE); + + kvfree(recv_buf->dma_addrs); +} + +static void mlx5vf_free_qp_recv_resources(struct mlx5_core_dev *mdev, + struct mlx5_vhca_qp *qp) +{ + struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf; + + mlx5_core_destroy_mkey(mdev, recv_buf->mkey); + unregister_dma_recv_pages(mdev, recv_buf); + free_recv_pages(&qp->recv_buf); +} + +static int mlx5vf_alloc_qp_recv_resources(struct mlx5_core_dev *mdev, + struct mlx5_vhca_qp *qp, u32 pdn, + u64 rq_size) +{ + unsigned int npages = DIV_ROUND_UP_ULL(rq_size, PAGE_SIZE); + struct mlx5_vhca_recv_buf *recv_buf = &qp->recv_buf; + int err; + + err = alloc_recv_pages(recv_buf, npages); + if (err < 0) + return err; + + err = register_dma_recv_pages(mdev, recv_buf); + if (err) + goto end; + + err = _create_mkey(mdev, pdn, NULL, recv_buf, &recv_buf->mkey); + if (err) + goto err_create_mkey; + + return 0; + +err_create_mkey: + unregister_dma_recv_pages(mdev, recv_buf); +end: + free_recv_pages(recv_buf); + return err; +} + +static void +_mlx5vf_free_page_tracker_resources(struct mlx5vf_pci_core_device *mvdev) +{ + struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker; + struct mlx5_core_dev *mdev = mvdev->mdev; + + lockdep_assert_held(&mvdev->state_mutex); + + if (!mvdev->log_active) + return; + + WARN_ON(mvdev->mdev_detach); + + mlx5_eq_notifier_unregister(mdev, &tracker->nb); + mlx5vf_cmd_destroy_tracker(mdev, tracker->id); + mlx5vf_destroy_qp(mdev, tracker->fw_qp); + mlx5vf_free_qp_recv_resources(mdev, tracker->host_qp); + mlx5vf_destroy_qp(mdev, tracker->host_qp); + mlx5vf_destroy_cq(mdev, &tracker->cq); + mlx5_core_dealloc_pd(mdev, tracker->pdn); + mlx5_put_uars_page(mdev, tracker->uar); + mvdev->log_active = false; +} + +int mlx5vf_stop_page_tracker(struct vfio_device *vdev) +{ + struct mlx5vf_pci_core_device *mvdev = container_of( + vdev, struct mlx5vf_pci_core_device, core_device.vdev); + + mutex_lock(&mvdev->state_mutex); + if (!mvdev->log_active) + goto end; + + _mlx5vf_free_page_tracker_resources(mvdev); + mvdev->log_active = false; +end: + mlx5vf_state_mutex_unlock(mvdev); + return 0; +} + +int mlx5vf_start_page_tracker(struct vfio_device *vdev, + struct rb_root_cached *ranges, u32 nnodes, + u64 *page_size) +{ + struct mlx5vf_pci_core_device *mvdev = container_of( + vdev, struct mlx5vf_pci_core_device, core_device.vdev); + struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker; + u8 log_tracked_page = ilog2(*page_size); + struct mlx5_vhca_qp *host_qp; + struct mlx5_vhca_qp *fw_qp; + struct mlx5_core_dev *mdev; + u32 max_msg_size = PAGE_SIZE; + u64 rq_size = SZ_2M; + u32 max_recv_wr; + int err; + + mutex_lock(&mvdev->state_mutex); + if (mvdev->mdev_detach) { + err = -ENOTCONN; + goto end; + } + + if (mvdev->log_active) { + err = -EINVAL; + goto end; + } + + mdev = mvdev->mdev; + memset(tracker, 0, sizeof(*tracker)); + tracker->uar = mlx5_get_uars_page(mdev); + if (IS_ERR(tracker->uar)) { + err = PTR_ERR(tracker->uar); + goto end; + } + + err = mlx5_core_alloc_pd(mdev, &tracker->pdn); + if (err) + goto err_uar; + + max_recv_wr = DIV_ROUND_UP_ULL(rq_size, max_msg_size); + err = mlx5vf_create_cq(mdev, tracker, max_recv_wr); + if (err) + goto err_dealloc_pd; + + host_qp = mlx5vf_create_rc_qp(mdev, tracker, max_recv_wr); + if (IS_ERR(host_qp)) { + err = PTR_ERR(host_qp); + goto err_cq; + } + + host_qp->max_msg_size = max_msg_size; + if (log_tracked_page < MLX5_CAP_ADV_VIRTUALIZATION(mdev, + pg_track_log_min_page_size)) { + log_tracked_page = MLX5_CAP_ADV_VIRTUALIZATION(mdev, + pg_track_log_min_page_size); + } else if (log_tracked_page > MLX5_CAP_ADV_VIRTUALIZATION(mdev, + pg_track_log_max_page_size)) { + log_tracked_page = MLX5_CAP_ADV_VIRTUALIZATION(mdev, + pg_track_log_max_page_size); + } + + host_qp->tracked_page_size = (1ULL << log_tracked_page); + err = mlx5vf_alloc_qp_recv_resources(mdev, host_qp, tracker->pdn, + rq_size); + if (err) + goto err_host_qp; + + fw_qp = mlx5vf_create_rc_qp(mdev, tracker, 0); + if (IS_ERR(fw_qp)) { + err = PTR_ERR(fw_qp); + goto err_recv_resources; + } + + err = mlx5vf_activate_qp(mdev, host_qp, fw_qp->qpn, true); + if (err) + goto err_activate; + + err = mlx5vf_activate_qp(mdev, fw_qp, host_qp->qpn, false); + if (err) + goto err_activate; + + tracker->host_qp = host_qp; + tracker->fw_qp = fw_qp; + err = mlx5vf_create_tracker(mdev, mvdev, ranges, nnodes); + if (err) + goto err_activate; + + MLX5_NB_INIT(&tracker->nb, mlx5vf_event_notifier, NOTIFY_ANY); + mlx5_eq_notifier_register(mdev, &tracker->nb); + *page_size = host_qp->tracked_page_size; + mvdev->log_active = true; + mlx5vf_state_mutex_unlock(mvdev); + return 0; + +err_activate: + mlx5vf_destroy_qp(mdev, fw_qp); +err_recv_resources: + mlx5vf_free_qp_recv_resources(mdev, host_qp); +err_host_qp: + mlx5vf_destroy_qp(mdev, host_qp); +err_cq: + mlx5vf_destroy_cq(mdev, &tracker->cq); +err_dealloc_pd: + mlx5_core_dealloc_pd(mdev, tracker->pdn); +err_uar: + mlx5_put_uars_page(mdev, tracker->uar); +end: + mlx5vf_state_mutex_unlock(mvdev); + return err; +} + +static void +set_report_output(u32 size, int index, struct mlx5_vhca_qp *qp, + struct iova_bitmap *dirty) +{ + u32 entry_size = MLX5_ST_SZ_BYTES(page_track_report_entry); + u32 nent = size / entry_size; + struct page *page; + u64 addr; + u64 *buf; + int i; + + if (WARN_ON(index >= qp->recv_buf.npages || + (nent > qp->max_msg_size / entry_size))) + return; + + page = qp->recv_buf.page_list[index]; + buf = kmap_local_page(page); + for (i = 0; i < nent; i++) { + addr = MLX5_GET(page_track_report_entry, buf + i, + dirty_address_low); + addr |= (u64)MLX5_GET(page_track_report_entry, buf + i, + dirty_address_high) << 32; + iova_bitmap_set(dirty, addr, qp->tracked_page_size); + } + kunmap_local(buf); +} + +static void +mlx5vf_rq_cqe(struct mlx5_vhca_qp *qp, struct mlx5_cqe64 *cqe, + struct iova_bitmap *dirty, int *tracker_status) +{ + u32 size; + int ix; + + qp->rq.cc++; + *tracker_status = be32_to_cpu(cqe->immediate) >> 28; + size = be32_to_cpu(cqe->byte_cnt); + ix = be16_to_cpu(cqe->wqe_counter) & (qp->rq.wqe_cnt - 1); + + /* zero length CQE, no data */ + WARN_ON(!size && *tracker_status == MLX5_PAGE_TRACK_STATE_REPORTING); + if (size) + set_report_output(size, ix, qp, dirty); + + qp->recv_buf.next_rq_offset = ix * qp->max_msg_size; + mlx5vf_post_recv(qp); +} + +static void *get_cqe(struct mlx5_vhca_cq *cq, int n) +{ + return mlx5_frag_buf_get_wqe(&cq->buf.fbc, n); +} + +static struct mlx5_cqe64 *get_sw_cqe(struct mlx5_vhca_cq *cq, int n) +{ + void *cqe = get_cqe(cq, n & (cq->ncqe - 1)); + struct mlx5_cqe64 *cqe64; + + cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; + + if (likely(get_cqe_opcode(cqe64) != MLX5_CQE_INVALID) && + !((cqe64->op_own & MLX5_CQE_OWNER_MASK) ^ !!(n & (cq->ncqe)))) { + return cqe64; + } else { + return NULL; + } +} + +static int +mlx5vf_cq_poll_one(struct mlx5_vhca_cq *cq, struct mlx5_vhca_qp *qp, + struct iova_bitmap *dirty, int *tracker_status) +{ + struct mlx5_cqe64 *cqe; + u8 opcode; + + cqe = get_sw_cqe(cq, cq->mcq.cons_index); + if (!cqe) + return CQ_EMPTY; + + ++cq->mcq.cons_index; + /* + * Make sure we read CQ entry contents after we've checked the + * ownership bit. + */ + rmb(); + opcode = get_cqe_opcode(cqe); + switch (opcode) { + case MLX5_CQE_RESP_SEND_IMM: + mlx5vf_rq_cqe(qp, cqe, dirty, tracker_status); + return CQ_OK; + default: + return CQ_POLL_ERR; + } +} + +int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, + unsigned long length, + struct iova_bitmap *dirty) +{ + struct mlx5vf_pci_core_device *mvdev = container_of( + vdev, struct mlx5vf_pci_core_device, core_device.vdev); + struct mlx5_vhca_page_tracker *tracker = &mvdev->tracker; + struct mlx5_vhca_cq *cq = &tracker->cq; + struct mlx5_core_dev *mdev; + int poll_err, err; + + mutex_lock(&mvdev->state_mutex); + if (!mvdev->log_active) { + err = -EINVAL; + goto end; + } + + if (mvdev->mdev_detach) { + err = -ENOTCONN; + goto end; + } + + mdev = mvdev->mdev; + err = mlx5vf_cmd_modify_tracker(mdev, tracker->id, iova, length, + MLX5_PAGE_TRACK_STATE_REPORTING); + if (err) + goto end; + + tracker->status = MLX5_PAGE_TRACK_STATE_REPORTING; + while (tracker->status == MLX5_PAGE_TRACK_STATE_REPORTING && + !tracker->is_err) { + poll_err = mlx5vf_cq_poll_one(cq, tracker->host_qp, dirty, + &tracker->status); + if (poll_err == CQ_EMPTY) { + mlx5_cq_arm(&cq->mcq, MLX5_CQ_DB_REQ_NOT, tracker->uar->map, + cq->mcq.cons_index); + poll_err = mlx5vf_cq_poll_one(cq, tracker->host_qp, + dirty, &tracker->status); + if (poll_err == CQ_EMPTY) { + wait_for_completion(&mvdev->tracker_comp); + continue; + } + } + if (poll_err == CQ_POLL_ERR) { + err = -EIO; + goto end; + } + mlx5_cq_set_ci(&cq->mcq); + } + + if (tracker->status == MLX5_PAGE_TRACK_STATE_ERROR) + tracker->is_err = true; + + if (tracker->is_err) + err = -EIO; +end: + mlx5vf_state_mutex_unlock(mvdev); + return err; +} diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h index 8208f4701a90800f33c45c7e468a2ea4d652f54c..921d5720a1e57b87a901b8374ac6f4870bb1bafb 100644 --- a/drivers/vfio/pci/mlx5/cmd.h +++ b/drivers/vfio/pci/mlx5/cmd.h @@ -9,6 +9,8 @@ #include #include #include +#include +#include struct mlx5vf_async_data { struct mlx5_async_work cb_work; @@ -39,6 +41,56 @@ struct mlx5_vf_migration_file { struct mlx5vf_async_data async_data; }; +struct mlx5_vhca_cq_buf { + struct mlx5_frag_buf_ctrl fbc; + struct mlx5_frag_buf frag_buf; + int cqe_size; + int nent; +}; + +struct mlx5_vhca_cq { + struct mlx5_vhca_cq_buf buf; + struct mlx5_db db; + struct mlx5_core_cq mcq; + size_t ncqe; +}; + +struct mlx5_vhca_recv_buf { + u32 npages; + struct page **page_list; + dma_addr_t *dma_addrs; + u32 next_rq_offset; + u32 mkey; +}; + +struct mlx5_vhca_qp { + struct mlx5_frag_buf buf; + struct mlx5_db db; + struct mlx5_vhca_recv_buf recv_buf; + u32 tracked_page_size; + u32 max_msg_size; + u32 qpn; + struct { + unsigned int pc; + unsigned int cc; + unsigned int wqe_cnt; + __be32 *db; + struct mlx5_frag_buf_ctrl fbc; + } rq; +}; + +struct mlx5_vhca_page_tracker { + u32 id; + u32 pdn; + u8 is_err:1; + struct mlx5_uars_page *uar; + struct mlx5_vhca_cq cq; + struct mlx5_vhca_qp *host_qp; + struct mlx5_vhca_qp *fw_qp; + struct mlx5_nb nb; + int status; +}; + struct mlx5vf_pci_core_device { struct vfio_pci_core_device core_device; int vf_id; @@ -46,6 +98,8 @@ struct mlx5vf_pci_core_device { u8 migrate_cap:1; u8 deferred_reset:1; u8 mdev_detach:1; + u8 log_active:1; + struct completion tracker_comp; /* protect migration state */ struct mutex state_mutex; enum vfio_device_mig_state mig_state; @@ -53,6 +107,7 @@ struct mlx5vf_pci_core_device { spinlock_t reset_lock; struct mlx5_vf_migration_file *resuming_migf; struct mlx5_vf_migration_file *saving_migf; + struct mlx5_vhca_page_tracker tracker; struct workqueue_struct *cb_wq; struct notifier_block nb; struct mlx5_core_dev *mdev; @@ -63,7 +118,8 @@ int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, size_t *state_size); void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, - const struct vfio_migration_ops *mig_ops); + const struct vfio_migration_ops *mig_ops, + const struct vfio_log_ops *log_ops); void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, @@ -73,4 +129,9 @@ int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev); void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); +int mlx5vf_start_page_tracker(struct vfio_device *vdev, + struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); +int mlx5vf_stop_page_tracker(struct vfio_device *vdev); +int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, + unsigned long length, struct iova_bitmap *dirty); #endif /* MLX5_VFIO_CMD_H */ diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c index a9b63d15c5d3490353653490ade5f8c97092fc96..fd6ccb8454a24a496c351fb415f96ee2fd9361a5 100644 --- a/drivers/vfio/pci/mlx5/main.c +++ b/drivers/vfio/pci/mlx5/main.c @@ -579,8 +579,41 @@ static const struct vfio_migration_ops mlx5vf_pci_mig_ops = { .migration_get_state = mlx5vf_pci_get_device_state, }; +static const struct vfio_log_ops mlx5vf_pci_log_ops = { + .log_start = mlx5vf_start_page_tracker, + .log_stop = mlx5vf_stop_page_tracker, + .log_read_and_clear = mlx5vf_tracker_read_and_clear, +}; + +static int mlx5vf_pci_init_dev(struct vfio_device *core_vdev) +{ + struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev, + struct mlx5vf_pci_core_device, core_device.vdev); + int ret; + + ret = vfio_pci_core_init_dev(core_vdev); + if (ret) + return ret; + + mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops, + &mlx5vf_pci_log_ops); + + return 0; +} + +static void mlx5vf_pci_release_dev(struct vfio_device *core_vdev) +{ + struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev, + struct mlx5vf_pci_core_device, core_device.vdev); + + mlx5vf_cmd_remove_migratable(mvdev); + vfio_pci_core_release_dev(core_vdev); +} + static const struct vfio_device_ops mlx5vf_pci_ops = { .name = "mlx5-vfio-pci", + .init = mlx5vf_pci_init_dev, + .release = mlx5vf_pci_release_dev, .open_device = mlx5vf_pci_open_device, .close_device = mlx5vf_pci_close_device, .ioctl = vfio_pci_core_ioctl, @@ -598,21 +631,19 @@ static int mlx5vf_pci_probe(struct pci_dev *pdev, struct mlx5vf_pci_core_device *mvdev; int ret; - mvdev = kzalloc(sizeof(*mvdev), GFP_KERNEL); - if (!mvdev) - return -ENOMEM; - vfio_pci_core_init_device(&mvdev->core_device, pdev, &mlx5vf_pci_ops); - mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops); + mvdev = vfio_alloc_device(mlx5vf_pci_core_device, core_device.vdev, + &pdev->dev, &mlx5vf_pci_ops); + if (IS_ERR(mvdev)) + return PTR_ERR(mvdev); + dev_set_drvdata(&pdev->dev, &mvdev->core_device); ret = vfio_pci_core_register_device(&mvdev->core_device); if (ret) - goto out_free; + goto out_put_vdev; return 0; -out_free: - mlx5vf_cmd_remove_migratable(mvdev); - vfio_pci_core_uninit_device(&mvdev->core_device); - kfree(mvdev); +out_put_vdev: + vfio_put_device(&mvdev->core_device.vdev); return ret; } @@ -621,9 +652,7 @@ static void mlx5vf_pci_remove(struct pci_dev *pdev) struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev); vfio_pci_core_unregister_device(&mvdev->core_device); - mlx5vf_cmd_remove_migratable(mvdev); - vfio_pci_core_uninit_device(&mvdev->core_device); - kfree(mvdev); + vfio_put_device(&mvdev->core_device.vdev); } static const struct pci_device_id mlx5vf_pci_table[] = { diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index 4d1a97415a27bf38eface45e2452b6277ddb2065..1d4919edfbde488918327ca894529c291310f7cf 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -25,7 +25,7 @@ #include #include -#include +#include "vfio_pci_priv.h" #define DRIVER_AUTHOR "Alex Williamson " #define DRIVER_DESC "VFIO PCI - User Level meta-driver" @@ -127,6 +127,8 @@ static int vfio_pci_open_device(struct vfio_device *core_vdev) static const struct vfio_device_ops vfio_pci_ops = { .name = "vfio-pci", + .init = vfio_pci_core_init_dev, + .release = vfio_pci_core_release_dev, .open_device = vfio_pci_open_device, .close_device = vfio_pci_core_close_device, .ioctl = vfio_pci_core_ioctl, @@ -146,20 +148,19 @@ static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (vfio_pci_is_denylisted(pdev)) return -EINVAL; - vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); - if (!vdev) - return -ENOMEM; - vfio_pci_core_init_device(vdev, pdev, &vfio_pci_ops); + vdev = vfio_alloc_device(vfio_pci_core_device, vdev, &pdev->dev, + &vfio_pci_ops); + if (IS_ERR(vdev)) + return PTR_ERR(vdev); dev_set_drvdata(&pdev->dev, vdev); ret = vfio_pci_core_register_device(vdev); if (ret) - goto out_free; + goto out_put_vdev; return 0; -out_free: - vfio_pci_core_uninit_device(vdev); - kfree(vdev); +out_put_vdev: + vfio_put_device(&vdev->vdev); return ret; } @@ -168,8 +169,7 @@ static void vfio_pci_remove(struct pci_dev *pdev) struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev); vfio_pci_core_unregister_device(vdev); - vfio_pci_core_uninit_device(vdev); - kfree(vdev); + vfio_put_device(&vdev->vdev); } static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index 442d3ba4122b22de9293ceeec9f64240ba090557..4a350421c5f62a7a215b690d6b16d7a9dca7c83a 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -26,7 +26,7 @@ #include #include -#include +#include "vfio_pci_priv.h" /* Fake capability ID for standard config space */ #define PCI_CAP_ID_BASIC 0 @@ -1166,7 +1166,7 @@ static int vfio_msi_config_write(struct vfio_pci_core_device *vdev, int pos, flags = le16_to_cpu(*pflags); /* MSI is enabled via ioctl */ - if (!is_msi(vdev)) + if (vdev->irq_type != VFIO_PCI_MSI_IRQ_INDEX) flags &= ~PCI_MSI_FLAGS_ENABLE; /* Check queue size */ diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index c8d3b0450fb35b5a1f615545ca72d5264833fc0f..badc9d828cac201c3d2a0a9c28ebf971056c89c7 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -28,7 +28,7 @@ #include #include -#include +#include "vfio_pci_priv.h" #define DRIVER_AUTHOR "Alex Williamson " #define DRIVER_DESC "core driver for VFIO based PCI devices" @@ -41,6 +41,23 @@ static bool disable_idle_d3; static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex); static LIST_HEAD(vfio_pci_sriov_pfs); +struct vfio_pci_dummy_resource { + struct resource resource; + int index; + struct list_head res_next; +}; + +struct vfio_pci_vf_token { + struct mutex lock; + uuid_t uuid; + int users; +}; + +struct vfio_pci_mmap_vma { + struct vm_area_struct *vma; + struct list_head vma_next; +}; + static inline bool vfio_vga_disabled(void) { #ifdef CONFIG_VFIO_PCI_VGA @@ -260,16 +277,189 @@ int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t stat return ret; } +static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev, + struct eventfd_ctx *efdctx) +{ + /* + * The vdev power related flags are protected with 'memory_lock' + * semaphore. + */ + vfio_pci_zap_and_down_write_memory_lock(vdev); + if (vdev->pm_runtime_engaged) { + up_write(&vdev->memory_lock); + return -EINVAL; + } + + vdev->pm_runtime_engaged = true; + vdev->pm_wake_eventfd_ctx = efdctx; + pm_runtime_put_noidle(&vdev->pdev->dev); + up_write(&vdev->memory_lock); + + return 0; +} + +static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags, + void __user *arg, size_t argsz) +{ + struct vfio_pci_core_device *vdev = + container_of(device, struct vfio_pci_core_device, vdev); + int ret; + + ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); + if (ret != 1) + return ret; + + /* + * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count + * will be decremented. The pm_runtime_put() will be invoked again + * while returning from the ioctl and then the device can go into + * runtime suspended state. + */ + return vfio_pci_runtime_pm_entry(vdev, NULL); +} + +static int vfio_pci_core_pm_entry_with_wakeup( + struct vfio_device *device, u32 flags, + struct vfio_device_low_power_entry_with_wakeup __user *arg, + size_t argsz) +{ + struct vfio_pci_core_device *vdev = + container_of(device, struct vfio_pci_core_device, vdev); + struct vfio_device_low_power_entry_with_wakeup entry; + struct eventfd_ctx *efdctx; + int ret; + + ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, + sizeof(entry)); + if (ret != 1) + return ret; + + if (copy_from_user(&entry, arg, sizeof(entry))) + return -EFAULT; + + if (entry.wakeup_eventfd < 0) + return -EINVAL; + + efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd); + if (IS_ERR(efdctx)) + return PTR_ERR(efdctx); + + ret = vfio_pci_runtime_pm_entry(vdev, efdctx); + if (ret) + eventfd_ctx_put(efdctx); + + return ret; +} + +static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) +{ + if (vdev->pm_runtime_engaged) { + vdev->pm_runtime_engaged = false; + pm_runtime_get_noresume(&vdev->pdev->dev); + + if (vdev->pm_wake_eventfd_ctx) { + eventfd_ctx_put(vdev->pm_wake_eventfd_ctx); + vdev->pm_wake_eventfd_ctx = NULL; + } + } +} + +static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev) +{ + /* + * The vdev power related flags are protected with 'memory_lock' + * semaphore. + */ + down_write(&vdev->memory_lock); + __vfio_pci_runtime_pm_exit(vdev); + up_write(&vdev->memory_lock); +} + +static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags, + void __user *arg, size_t argsz) +{ + struct vfio_pci_core_device *vdev = + container_of(device, struct vfio_pci_core_device, vdev); + int ret; + + ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0); + if (ret != 1) + return ret; + + /* + * The device is always in the active state here due to pm wrappers + * around ioctls. If the device had entered a low power state and + * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has + * already signaled the eventfd and exited low power mode itself. + * pm_runtime_engaged protects the redundant call here. + */ + vfio_pci_runtime_pm_exit(vdev); + return 0; +} + +#ifdef CONFIG_PM +static int vfio_pci_core_runtime_suspend(struct device *dev) +{ + struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); + + down_write(&vdev->memory_lock); + /* + * The user can move the device into D3hot state before invoking + * power management IOCTL. Move the device into D0 state here and then + * the pci-driver core runtime PM suspend function will move the device + * into the low power state. Also, for the devices which have + * NoSoftRst-, it will help in restoring the original state + * (saved locally in 'vdev->pm_save'). + */ + vfio_pci_set_power_state(vdev, PCI_D0); + up_write(&vdev->memory_lock); + + /* + * If INTx is enabled, then mask INTx before going into the runtime + * suspended state and unmask the same in the runtime resume. + * If INTx has already been masked by the user, then + * vfio_pci_intx_mask() will return false and in that case, INTx + * should not be unmasked in the runtime resume. + */ + vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) && + vfio_pci_intx_mask(vdev)); + + return 0; +} + +static int vfio_pci_core_runtime_resume(struct device *dev) +{ + struct vfio_pci_core_device *vdev = dev_get_drvdata(dev); + + /* + * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit + * low power mode. + */ + down_write(&vdev->memory_lock); + if (vdev->pm_wake_eventfd_ctx) { + eventfd_signal(vdev->pm_wake_eventfd_ctx, 1); + __vfio_pci_runtime_pm_exit(vdev); + } + up_write(&vdev->memory_lock); + + if (vdev->pm_intx_masked) + vfio_pci_intx_unmask(vdev); + + return 0; +} +#endif /* CONFIG_PM */ + /* - * The dev_pm_ops needs to be provided to make pci-driver runtime PM working, - * so use structure without any callbacks. - * * The pci-driver core runtime PM routines always save the device state * before going into suspended state. If the device is going into low power * state with only with runtime PM ops, then no explicit handling is needed * for the devices which have NoSoftRst-. */ -static const struct dev_pm_ops vfio_pci_core_pm_ops = { }; +static const struct dev_pm_ops vfio_pci_core_pm_ops = { + SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend, + vfio_pci_core_runtime_resume, + NULL) +}; int vfio_pci_core_enable(struct vfio_pci_core_device *vdev) { @@ -371,6 +561,18 @@ void vfio_pci_core_disable(struct vfio_pci_core_device *vdev) /* * This function can be invoked while the power state is non-D0. + * This non-D0 power state can be with or without runtime PM. + * vfio_pci_runtime_pm_exit() will internally increment the usage + * count corresponding to pm_runtime_put() called during low power + * feature entry and then pm_runtime_resume() will wake up the device, + * if the device has already gone into the suspended state. Otherwise, + * the vfio_pci_set_power_state() will change the device power state + * to D0. + */ + vfio_pci_runtime_pm_exit(vdev); + pm_runtime_resume(&pdev->dev); + + /* * This function calls __pci_reset_function_locked() which internally * can use pci_pm_reset() for the function reset. pci_pm_reset() will * fail if the power state is non-D0. Also, for the devices which @@ -645,10 +847,10 @@ static int msix_mmappable_cap(struct vfio_pci_core_device *vdev, return vfio_info_add_capability(caps, &header, sizeof(header)); } -int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, - unsigned int type, unsigned int subtype, - const struct vfio_pci_regops *ops, - size_t size, u32 flags, void *data) +int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, + unsigned int type, unsigned int subtype, + const struct vfio_pci_regops *ops, + size_t size, u32 flags, void *data) { struct vfio_pci_region *region; @@ -670,508 +872,532 @@ int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, return 0; } -EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region); +EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region); -long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, - unsigned long arg) +static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev, + struct vfio_device_info __user *arg) { - struct vfio_pci_core_device *vdev = - container_of(core_vdev, struct vfio_pci_core_device, vdev); - unsigned long minsz; - - if (cmd == VFIO_DEVICE_GET_INFO) { - struct vfio_device_info info; - struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; - unsigned long capsz; - int ret; - - minsz = offsetofend(struct vfio_device_info, num_irqs); + unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs); + struct vfio_device_info info; + struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; + unsigned long capsz; + int ret; - /* For backward compatibility, cannot require this */ - capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); + /* For backward compatibility, cannot require this */ + capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); - if (copy_from_user(&info, (void __user *)arg, minsz)) - return -EFAULT; + if (copy_from_user(&info, arg, minsz)) + return -EFAULT; - if (info.argsz < minsz) - return -EINVAL; + if (info.argsz < minsz) + return -EINVAL; - if (info.argsz >= capsz) { - minsz = capsz; - info.cap_offset = 0; - } + if (info.argsz >= capsz) { + minsz = capsz; + info.cap_offset = 0; + } - info.flags = VFIO_DEVICE_FLAGS_PCI; + info.flags = VFIO_DEVICE_FLAGS_PCI; - if (vdev->reset_works) - info.flags |= VFIO_DEVICE_FLAGS_RESET; + if (vdev->reset_works) + info.flags |= VFIO_DEVICE_FLAGS_RESET; - info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; - info.num_irqs = VFIO_PCI_NUM_IRQS; + info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; + info.num_irqs = VFIO_PCI_NUM_IRQS; - ret = vfio_pci_info_zdev_add_caps(vdev, &caps); - if (ret && ret != -ENODEV) { - pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); - return ret; - } + ret = vfio_pci_info_zdev_add_caps(vdev, &caps); + if (ret && ret != -ENODEV) { + pci_warn(vdev->pdev, + "Failed to setup zPCI info capabilities\n"); + return ret; + } - if (caps.size) { - info.flags |= VFIO_DEVICE_FLAGS_CAPS; - if (info.argsz < sizeof(info) + caps.size) { - info.argsz = sizeof(info) + caps.size; - } else { - vfio_info_cap_shift(&caps, sizeof(info)); - if (copy_to_user((void __user *)arg + - sizeof(info), caps.buf, - caps.size)) { - kfree(caps.buf); - return -EFAULT; - } - info.cap_offset = sizeof(info); + if (caps.size) { + info.flags |= VFIO_DEVICE_FLAGS_CAPS; + if (info.argsz < sizeof(info) + caps.size) { + info.argsz = sizeof(info) + caps.size; + } else { + vfio_info_cap_shift(&caps, sizeof(info)); + if (copy_to_user(arg + 1, caps.buf, caps.size)) { + kfree(caps.buf); + return -EFAULT; } - - kfree(caps.buf); + info.cap_offset = sizeof(*arg); } - return copy_to_user((void __user *)arg, &info, minsz) ? - -EFAULT : 0; + kfree(caps.buf); + } - } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { - struct pci_dev *pdev = vdev->pdev; - struct vfio_region_info info; - struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; - int i, ret; + return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; +} - minsz = offsetofend(struct vfio_region_info, offset); +static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev, + struct vfio_region_info __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_region_info, offset); + struct pci_dev *pdev = vdev->pdev; + struct vfio_region_info info; + struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; + int i, ret; - if (copy_from_user(&info, (void __user *)arg, minsz)) - return -EFAULT; + if (copy_from_user(&info, arg, minsz)) + return -EFAULT; - if (info.argsz < minsz) - return -EINVAL; + if (info.argsz < minsz) + return -EINVAL; - switch (info.index) { - case VFIO_PCI_CONFIG_REGION_INDEX: - info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); - info.size = pdev->cfg_size; - info.flags = VFIO_REGION_INFO_FLAG_READ | - VFIO_REGION_INFO_FLAG_WRITE; + switch (info.index) { + case VFIO_PCI_CONFIG_REGION_INDEX: + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = pdev->cfg_size; + info.flags = VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; + break; + case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = pci_resource_len(pdev, info.index); + if (!info.size) { + info.flags = 0; break; - case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: - info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); - info.size = pci_resource_len(pdev, info.index); - if (!info.size) { - info.flags = 0; - break; - } + } - info.flags = VFIO_REGION_INFO_FLAG_READ | - VFIO_REGION_INFO_FLAG_WRITE; - if (vdev->bar_mmap_supported[info.index]) { - info.flags |= VFIO_REGION_INFO_FLAG_MMAP; - if (info.index == vdev->msix_bar) { - ret = msix_mmappable_cap(vdev, &caps); - if (ret) - return ret; - } + info.flags = VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; + if (vdev->bar_mmap_supported[info.index]) { + info.flags |= VFIO_REGION_INFO_FLAG_MMAP; + if (info.index == vdev->msix_bar) { + ret = msix_mmappable_cap(vdev, &caps); + if (ret) + return ret; } + } - break; - case VFIO_PCI_ROM_REGION_INDEX: - { - void __iomem *io; - size_t size; - u16 cmd; - - info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); - info.flags = 0; + break; + case VFIO_PCI_ROM_REGION_INDEX: { + void __iomem *io; + size_t size; + u16 cmd; + + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.flags = 0; + + /* Report the BAR size, not the ROM size */ + info.size = pci_resource_len(pdev, info.index); + if (!info.size) { + /* Shadow ROMs appear as PCI option ROMs */ + if (pdev->resource[PCI_ROM_RESOURCE].flags & + IORESOURCE_ROM_SHADOW) + info.size = 0x20000; + else + break; + } - /* Report the BAR size, not the ROM size */ - info.size = pci_resource_len(pdev, info.index); - if (!info.size) { - /* Shadow ROMs appear as PCI option ROMs */ - if (pdev->resource[PCI_ROM_RESOURCE].flags & - IORESOURCE_ROM_SHADOW) - info.size = 0x20000; - else - break; - } + /* + * Is it really there? Enable memory decode for implicit access + * in pci_map_rom(). + */ + cmd = vfio_pci_memory_lock_and_enable(vdev); + io = pci_map_rom(pdev, &size); + if (io) { + info.flags = VFIO_REGION_INFO_FLAG_READ; + pci_unmap_rom(pdev, io); + } else { + info.size = 0; + } + vfio_pci_memory_unlock_and_restore(vdev, cmd); - /* - * Is it really there? Enable memory decode for - * implicit access in pci_map_rom(). - */ - cmd = vfio_pci_memory_lock_and_enable(vdev); - io = pci_map_rom(pdev, &size); - if (io) { - info.flags = VFIO_REGION_INFO_FLAG_READ; - pci_unmap_rom(pdev, io); - } else { - info.size = 0; - } - vfio_pci_memory_unlock_and_restore(vdev, cmd); + break; + } + case VFIO_PCI_VGA_REGION_INDEX: + if (!vdev->has_vga) + return -EINVAL; - break; - } - case VFIO_PCI_VGA_REGION_INDEX: - if (!vdev->has_vga) - return -EINVAL; + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = 0xc0000; + info.flags = VFIO_REGION_INFO_FLAG_READ | + VFIO_REGION_INFO_FLAG_WRITE; - info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); - info.size = 0xc0000; - info.flags = VFIO_REGION_INFO_FLAG_READ | - VFIO_REGION_INFO_FLAG_WRITE; + break; + default: { + struct vfio_region_info_cap_type cap_type = { + .header.id = VFIO_REGION_INFO_CAP_TYPE, + .header.version = 1 + }; - break; - default: - { - struct vfio_region_info_cap_type cap_type = { - .header.id = VFIO_REGION_INFO_CAP_TYPE, - .header.version = 1 }; + if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) + return -EINVAL; + info.index = array_index_nospec( + info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions); - if (info.index >= - VFIO_PCI_NUM_REGIONS + vdev->num_regions) - return -EINVAL; - info.index = array_index_nospec(info.index, - VFIO_PCI_NUM_REGIONS + - vdev->num_regions); + i = info.index - VFIO_PCI_NUM_REGIONS; - i = info.index - VFIO_PCI_NUM_REGIONS; + info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); + info.size = vdev->region[i].size; + info.flags = vdev->region[i].flags; - info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); - info.size = vdev->region[i].size; - info.flags = vdev->region[i].flags; + cap_type.type = vdev->region[i].type; + cap_type.subtype = vdev->region[i].subtype; - cap_type.type = vdev->region[i].type; - cap_type.subtype = vdev->region[i].subtype; + ret = vfio_info_add_capability(&caps, &cap_type.header, + sizeof(cap_type)); + if (ret) + return ret; - ret = vfio_info_add_capability(&caps, &cap_type.header, - sizeof(cap_type)); + if (vdev->region[i].ops->add_capability) { + ret = vdev->region[i].ops->add_capability( + vdev, &vdev->region[i], &caps); if (ret) return ret; - - if (vdev->region[i].ops->add_capability) { - ret = vdev->region[i].ops->add_capability(vdev, - &vdev->region[i], &caps); - if (ret) - return ret; - } - } } + } + } - if (caps.size) { - info.flags |= VFIO_REGION_INFO_FLAG_CAPS; - if (info.argsz < sizeof(info) + caps.size) { - info.argsz = sizeof(info) + caps.size; - info.cap_offset = 0; - } else { - vfio_info_cap_shift(&caps, sizeof(info)); - if (copy_to_user((void __user *)arg + - sizeof(info), caps.buf, - caps.size)) { - kfree(caps.buf); - return -EFAULT; - } - info.cap_offset = sizeof(info); + if (caps.size) { + info.flags |= VFIO_REGION_INFO_FLAG_CAPS; + if (info.argsz < sizeof(info) + caps.size) { + info.argsz = sizeof(info) + caps.size; + info.cap_offset = 0; + } else { + vfio_info_cap_shift(&caps, sizeof(info)); + if (copy_to_user(arg + 1, caps.buf, caps.size)) { + kfree(caps.buf); + return -EFAULT; } - - kfree(caps.buf); + info.cap_offset = sizeof(*arg); } - return copy_to_user((void __user *)arg, &info, minsz) ? - -EFAULT : 0; + kfree(caps.buf); + } - } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { - struct vfio_irq_info info; + return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; +} - minsz = offsetofend(struct vfio_irq_info, count); +static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev, + struct vfio_irq_info __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_irq_info, count); + struct vfio_irq_info info; - if (copy_from_user(&info, (void __user *)arg, minsz)) - return -EFAULT; + if (copy_from_user(&info, arg, minsz)) + return -EFAULT; - if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) - return -EINVAL; + if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) + return -EINVAL; - switch (info.index) { - case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: - case VFIO_PCI_REQ_IRQ_INDEX: + switch (info.index) { + case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: + case VFIO_PCI_REQ_IRQ_INDEX: + break; + case VFIO_PCI_ERR_IRQ_INDEX: + if (pci_is_pcie(vdev->pdev)) break; - case VFIO_PCI_ERR_IRQ_INDEX: - if (pci_is_pcie(vdev->pdev)) - break; - fallthrough; - default: - return -EINVAL; - } - - info.flags = VFIO_IRQ_INFO_EVENTFD; - - info.count = vfio_pci_get_irq_count(vdev, info.index); + fallthrough; + default: + return -EINVAL; + } - if (info.index == VFIO_PCI_INTX_IRQ_INDEX) - info.flags |= (VFIO_IRQ_INFO_MASKABLE | - VFIO_IRQ_INFO_AUTOMASKED); - else - info.flags |= VFIO_IRQ_INFO_NORESIZE; + info.flags = VFIO_IRQ_INFO_EVENTFD; - return copy_to_user((void __user *)arg, &info, minsz) ? - -EFAULT : 0; + info.count = vfio_pci_get_irq_count(vdev, info.index); - } else if (cmd == VFIO_DEVICE_SET_IRQS) { - struct vfio_irq_set hdr; - u8 *data = NULL; - int max, ret = 0; - size_t data_size = 0; + if (info.index == VFIO_PCI_INTX_IRQ_INDEX) + info.flags |= + (VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED); + else + info.flags |= VFIO_IRQ_INFO_NORESIZE; - minsz = offsetofend(struct vfio_irq_set, count); + return copy_to_user(arg, &info, minsz) ? -EFAULT : 0; +} - if (copy_from_user(&hdr, (void __user *)arg, minsz)) - return -EFAULT; +static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev, + struct vfio_irq_set __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_irq_set, count); + struct vfio_irq_set hdr; + u8 *data = NULL; + int max, ret = 0; + size_t data_size = 0; - max = vfio_pci_get_irq_count(vdev, hdr.index); + if (copy_from_user(&hdr, arg, minsz)) + return -EFAULT; - ret = vfio_set_irqs_validate_and_prepare(&hdr, max, - VFIO_PCI_NUM_IRQS, &data_size); - if (ret) - return ret; + max = vfio_pci_get_irq_count(vdev, hdr.index); - if (data_size) { - data = memdup_user((void __user *)(arg + minsz), - data_size); - if (IS_ERR(data)) - return PTR_ERR(data); - } + ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS, + &data_size); + if (ret) + return ret; - mutex_lock(&vdev->igate); + if (data_size) { + data = memdup_user(&arg->data, data_size); + if (IS_ERR(data)) + return PTR_ERR(data); + } - ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, - hdr.start, hdr.count, data); + mutex_lock(&vdev->igate); - mutex_unlock(&vdev->igate); - kfree(data); + ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start, + hdr.count, data); - return ret; + mutex_unlock(&vdev->igate); + kfree(data); - } else if (cmd == VFIO_DEVICE_RESET) { - int ret; + return ret; +} - if (!vdev->reset_works) - return -EINVAL; +static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev, + void __user *arg) +{ + int ret; - vfio_pci_zap_and_down_write_memory_lock(vdev); + if (!vdev->reset_works) + return -EINVAL; - /* - * This function can be invoked while the power state is non-D0. - * If pci_try_reset_function() has been called while the power - * state is non-D0, then pci_try_reset_function() will - * internally set the power state to D0 without vfio driver - * involvement. For the devices which have NoSoftRst-, the - * reset function can cause the PCI config space reset without - * restoring the original state (saved locally in - * 'vdev->pm_save'). - */ - vfio_pci_set_power_state(vdev, PCI_D0); + vfio_pci_zap_and_down_write_memory_lock(vdev); - ret = pci_try_reset_function(vdev->pdev); - up_write(&vdev->memory_lock); + /* + * This function can be invoked while the power state is non-D0. If + * pci_try_reset_function() has been called while the power state is + * non-D0, then pci_try_reset_function() will internally set the power + * state to D0 without vfio driver involvement. For the devices which + * have NoSoftRst-, the reset function can cause the PCI config space + * reset without restoring the original state (saved locally in + * 'vdev->pm_save'). + */ + vfio_pci_set_power_state(vdev, PCI_D0); - return ret; + ret = pci_try_reset_function(vdev->pdev); + up_write(&vdev->memory_lock); - } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { - struct vfio_pci_hot_reset_info hdr; - struct vfio_pci_fill_info fill = { 0 }; - struct vfio_pci_dependent_device *devices = NULL; - bool slot = false; - int ret = 0; + return ret; +} - minsz = offsetofend(struct vfio_pci_hot_reset_info, count); +static int vfio_pci_ioctl_get_pci_hot_reset_info( + struct vfio_pci_core_device *vdev, + struct vfio_pci_hot_reset_info __user *arg) +{ + unsigned long minsz = + offsetofend(struct vfio_pci_hot_reset_info, count); + struct vfio_pci_hot_reset_info hdr; + struct vfio_pci_fill_info fill = { 0 }; + struct vfio_pci_dependent_device *devices = NULL; + bool slot = false; + int ret = 0; - if (copy_from_user(&hdr, (void __user *)arg, minsz)) - return -EFAULT; + if (copy_from_user(&hdr, arg, minsz)) + return -EFAULT; - if (hdr.argsz < minsz) - return -EINVAL; + if (hdr.argsz < minsz) + return -EINVAL; - hdr.flags = 0; + hdr.flags = 0; - /* Can we do a slot or bus reset or neither? */ - if (!pci_probe_reset_slot(vdev->pdev->slot)) - slot = true; - else if (pci_probe_reset_bus(vdev->pdev->bus)) - return -ENODEV; + /* Can we do a slot or bus reset or neither? */ + if (!pci_probe_reset_slot(vdev->pdev->slot)) + slot = true; + else if (pci_probe_reset_bus(vdev->pdev->bus)) + return -ENODEV; - /* How many devices are affected? */ - ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, - vfio_pci_count_devs, - &fill.max, slot); - if (ret) - return ret; + /* How many devices are affected? */ + ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, + &fill.max, slot); + if (ret) + return ret; - WARN_ON(!fill.max); /* Should always be at least one */ + WARN_ON(!fill.max); /* Should always be at least one */ - /* - * If there's enough space, fill it now, otherwise return - * -ENOSPC and the number of devices affected. - */ - if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { - ret = -ENOSPC; - hdr.count = fill.max; - goto reset_info_exit; - } + /* + * If there's enough space, fill it now, otherwise return -ENOSPC and + * the number of devices affected. + */ + if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { + ret = -ENOSPC; + hdr.count = fill.max; + goto reset_info_exit; + } - devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); - if (!devices) - return -ENOMEM; + devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); + if (!devices) + return -ENOMEM; - fill.devices = devices; + fill.devices = devices; - ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, - vfio_pci_fill_devs, - &fill, slot); + ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs, + &fill, slot); - /* - * If a device was removed between counting and filling, - * we may come up short of fill.max. If a device was - * added, we'll have a return of -EAGAIN above. - */ - if (!ret) - hdr.count = fill.cur; + /* + * If a device was removed between counting and filling, we may come up + * short of fill.max. If a device was added, we'll have a return of + * -EAGAIN above. + */ + if (!ret) + hdr.count = fill.cur; reset_info_exit: - if (copy_to_user((void __user *)arg, &hdr, minsz)) + if (copy_to_user(arg, &hdr, minsz)) + ret = -EFAULT; + + if (!ret) { + if (copy_to_user(&arg->devices, devices, + hdr.count * sizeof(*devices))) ret = -EFAULT; + } - if (!ret) { - if (copy_to_user((void __user *)(arg + minsz), devices, - hdr.count * sizeof(*devices))) - ret = -EFAULT; - } + kfree(devices); + return ret; +} - kfree(devices); - return ret; +static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev, + struct vfio_pci_hot_reset __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count); + struct vfio_pci_hot_reset hdr; + int32_t *group_fds; + struct file **files; + struct vfio_pci_group_info info; + bool slot = false; + int file_idx, count = 0, ret = 0; - } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { - struct vfio_pci_hot_reset hdr; - int32_t *group_fds; - struct file **files; - struct vfio_pci_group_info info; - bool slot = false; - int file_idx, count = 0, ret = 0; + if (copy_from_user(&hdr, arg, minsz)) + return -EFAULT; - minsz = offsetofend(struct vfio_pci_hot_reset, count); + if (hdr.argsz < minsz || hdr.flags) + return -EINVAL; - if (copy_from_user(&hdr, (void __user *)arg, minsz)) - return -EFAULT; + /* Can we do a slot or bus reset or neither? */ + if (!pci_probe_reset_slot(vdev->pdev->slot)) + slot = true; + else if (pci_probe_reset_bus(vdev->pdev->bus)) + return -ENODEV; - if (hdr.argsz < minsz || hdr.flags) - return -EINVAL; + /* + * We can't let userspace give us an arbitrarily large buffer to copy, + * so verify how many we think there could be. Note groups can have + * multiple devices so one group per device is the max. + */ + ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, + &count, slot); + if (ret) + return ret; - /* Can we do a slot or bus reset or neither? */ - if (!pci_probe_reset_slot(vdev->pdev->slot)) - slot = true; - else if (pci_probe_reset_bus(vdev->pdev->bus)) - return -ENODEV; + /* Somewhere between 1 and count is OK */ + if (!hdr.count || hdr.count > count) + return -EINVAL; - /* - * We can't let userspace give us an arbitrarily large - * buffer to copy, so verify how many we think there - * could be. Note groups can have multiple devices so - * one group per device is the max. - */ - ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, - vfio_pci_count_devs, - &count, slot); - if (ret) - return ret; + group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); + files = kcalloc(hdr.count, sizeof(*files), GFP_KERNEL); + if (!group_fds || !files) { + kfree(group_fds); + kfree(files); + return -ENOMEM; + } - /* Somewhere between 1 and count is OK */ - if (!hdr.count || hdr.count > count) - return -EINVAL; + if (copy_from_user(group_fds, arg->group_fds, + hdr.count * sizeof(*group_fds))) { + kfree(group_fds); + kfree(files); + return -EFAULT; + } - group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); - files = kcalloc(hdr.count, sizeof(*files), GFP_KERNEL); - if (!group_fds || !files) { - kfree(group_fds); - kfree(files); - return -ENOMEM; - } + /* + * For each group_fd, get the group through the vfio external user + * interface and store the group and iommu ID. This ensures the group + * is held across the reset. + */ + for (file_idx = 0; file_idx < hdr.count; file_idx++) { + struct file *file = fget(group_fds[file_idx]); - if (copy_from_user(group_fds, (void __user *)(arg + minsz), - hdr.count * sizeof(*group_fds))) { - kfree(group_fds); - kfree(files); - return -EFAULT; + if (!file) { + ret = -EBADF; + break; } - /* - * For each group_fd, get the group through the vfio external - * user interface and store the group and iommu ID. This - * ensures the group is held across the reset. - */ - for (file_idx = 0; file_idx < hdr.count; file_idx++) { - struct file *file = fget(group_fds[file_idx]); - - if (!file) { - ret = -EBADF; - break; - } - - /* Ensure the FD is a vfio group FD.*/ - if (!vfio_file_iommu_group(file)) { - fput(file); - ret = -EINVAL; - break; - } - - files[file_idx] = file; + /* Ensure the FD is a vfio group FD.*/ + if (!vfio_file_is_group(file)) { + fput(file); + ret = -EINVAL; + break; } - kfree(group_fds); + files[file_idx] = file; + } - /* release reference to groups on error */ - if (ret) - goto hot_reset_release; + kfree(group_fds); + + /* release reference to groups on error */ + if (ret) + goto hot_reset_release; - info.count = hdr.count; - info.files = files; + info.count = hdr.count; + info.files = files; - ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info); + ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info); hot_reset_release: - for (file_idx--; file_idx >= 0; file_idx--) - fput(files[file_idx]); + for (file_idx--; file_idx >= 0; file_idx--) + fput(files[file_idx]); - kfree(files); - return ret; - } else if (cmd == VFIO_DEVICE_IOEVENTFD) { - struct vfio_device_ioeventfd ioeventfd; - int count; + kfree(files); + return ret; +} - minsz = offsetofend(struct vfio_device_ioeventfd, fd); +static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev, + struct vfio_device_ioeventfd __user *arg) +{ + unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd); + struct vfio_device_ioeventfd ioeventfd; + int count; - if (copy_from_user(&ioeventfd, (void __user *)arg, minsz)) - return -EFAULT; + if (copy_from_user(&ioeventfd, arg, minsz)) + return -EFAULT; - if (ioeventfd.argsz < minsz) - return -EINVAL; + if (ioeventfd.argsz < minsz) + return -EINVAL; - if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) - return -EINVAL; + if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) + return -EINVAL; - count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; + count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; - if (hweight8(count) != 1 || ioeventfd.fd < -1) - return -EINVAL; + if (hweight8(count) != 1 || ioeventfd.fd < -1) + return -EINVAL; - return vfio_pci_ioeventfd(vdev, ioeventfd.offset, - ioeventfd.data, count, ioeventfd.fd); + return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count, + ioeventfd.fd); +} + +long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, + unsigned long arg) +{ + struct vfio_pci_core_device *vdev = + container_of(core_vdev, struct vfio_pci_core_device, vdev); + void __user *uarg = (void __user *)arg; + + switch (cmd) { + case VFIO_DEVICE_GET_INFO: + return vfio_pci_ioctl_get_info(vdev, uarg); + case VFIO_DEVICE_GET_IRQ_INFO: + return vfio_pci_ioctl_get_irq_info(vdev, uarg); + case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO: + return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg); + case VFIO_DEVICE_GET_REGION_INFO: + return vfio_pci_ioctl_get_region_info(vdev, uarg); + case VFIO_DEVICE_IOEVENTFD: + return vfio_pci_ioctl_ioeventfd(vdev, uarg); + case VFIO_DEVICE_PCI_HOT_RESET: + return vfio_pci_ioctl_pci_hot_reset(vdev, uarg); + case VFIO_DEVICE_RESET: + return vfio_pci_ioctl_reset(vdev, uarg); + case VFIO_DEVICE_SET_IRQS: + return vfio_pci_ioctl_set_irqs(vdev, uarg); + default: + return -ENOTTY; } - return -ENOTTY; } EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl); static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags, - void __user *arg, size_t argsz) + uuid_t __user *arg, size_t argsz) { struct vfio_pci_core_device *vdev = container_of(device, struct vfio_pci_core_device, vdev); @@ -1202,6 +1428,13 @@ int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags, void __user *arg, size_t argsz) { switch (flags & VFIO_DEVICE_FEATURE_MASK) { + case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY: + return vfio_pci_core_pm_entry(device, flags, arg, argsz); + case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP: + return vfio_pci_core_pm_entry_with_wakeup(device, flags, + arg, argsz); + case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT: + return vfio_pci_core_pm_exit(device, flags, arg, argsz); case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: return vfio_pci_core_feature_token(device, flags, arg, argsz); default: @@ -1214,31 +1447,47 @@ static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); + int ret; if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) return -EINVAL; + ret = pm_runtime_resume_and_get(&vdev->pdev->dev); + if (ret) { + pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n", + ret); + return -EIO; + } + switch (index) { case VFIO_PCI_CONFIG_REGION_INDEX: - return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); + ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); + break; case VFIO_PCI_ROM_REGION_INDEX: if (iswrite) - return -EINVAL; - return vfio_pci_bar_rw(vdev, buf, count, ppos, false); + ret = -EINVAL; + else + ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false); + break; case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: - return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); + ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); + break; case VFIO_PCI_VGA_REGION_INDEX: - return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); + ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); + break; + default: index -= VFIO_PCI_NUM_REGIONS; - return vdev->region[index].ops->rw(vdev, buf, + ret = vdev->region[index].ops->rw(vdev, buf, count, ppos, iswrite); + break; } - return -EINVAL; + pm_runtime_put(&vdev->pdev->dev); + return ret; } ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf, @@ -1433,7 +1682,11 @@ static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) mutex_lock(&vdev->vma_lock); down_read(&vdev->memory_lock); - if (!__vfio_pci_memory_enabled(vdev)) { + /* + * Memory region cannot be accessed if the low power feature is engaged + * or memory access is disabled. + */ + if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) { ret = VM_FAULT_SIGBUS; goto up_out; } @@ -1825,12 +2078,12 @@ static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev) VGA_RSRC_LEGACY_MEM); } -void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, - struct pci_dev *pdev, - const struct vfio_device_ops *vfio_pci_ops) +int vfio_pci_core_init_dev(struct vfio_device *core_vdev) { - vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops); - vdev->pdev = pdev; + struct vfio_pci_core_device *vdev = + container_of(core_vdev, struct vfio_pci_core_device, vdev); + + vdev->pdev = to_pci_dev(core_vdev->dev); vdev->irq_type = VFIO_PCI_NUM_IRQS; mutex_init(&vdev->igate); spin_lock_init(&vdev->irqlock); @@ -1841,19 +2094,24 @@ void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, INIT_LIST_HEAD(&vdev->vma_list); INIT_LIST_HEAD(&vdev->sriov_pfs_item); init_rwsem(&vdev->memory_lock); + + return 0; } -EXPORT_SYMBOL_GPL(vfio_pci_core_init_device); +EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev); -void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev) +void vfio_pci_core_release_dev(struct vfio_device *core_vdev) { + struct vfio_pci_core_device *vdev = + container_of(core_vdev, struct vfio_pci_core_device, vdev); + mutex_destroy(&vdev->igate); mutex_destroy(&vdev->ioeventfds_lock); mutex_destroy(&vdev->vma_lock); - vfio_uninit_group_dev(&vdev->vdev); kfree(vdev->region); kfree(vdev->pm_save); + vfio_free_device(core_vdev); } -EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device); +EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev); int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) { @@ -1875,6 +2133,11 @@ int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev) return -EINVAL; } + if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start && + vdev->vdev.log_ops->log_stop && + vdev->vdev.log_ops->log_read_and_clear)) + return -EINVAL; + /* * Prevent binding to PFs with VFs enabled, the VFs might be in use * by the host or other users. We cannot capture the VFs if they @@ -2148,6 +2411,15 @@ static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set, goto err_unlock; } + /* + * Some of the devices in the dev_set can be in the runtime suspended + * state. Increment the usage count for all the devices in the dev_set + * before reset and decrement the same after reset. + */ + ret = vfio_pci_dev_set_pm_runtime_get(dev_set); + if (ret) + goto err_unlock; + list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) { /* * Test whether all the affected devices are contained by the @@ -2203,6 +2475,9 @@ err_undo: else mutex_unlock(&cur->vma_lock); } + + list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) + pm_runtime_put(&cur->pdev->dev); err_unlock: mutex_unlock(&dev_set->lock); return ret; diff --git a/drivers/vfio/pci/vfio_pci_igd.c b/drivers/vfio/pci/vfio_pci_igd.c index 352c725ccf1812071f5126887e5cdaabf80a456d..5e6ca592695485c883e9a3402c21445dbd60bbdf 100644 --- a/drivers/vfio/pci/vfio_pci_igd.c +++ b/drivers/vfio/pci/vfio_pci_igd.c @@ -15,7 +15,7 @@ #include #include -#include +#include "vfio_pci_priv.h" #define OPREGION_SIGNATURE "IntelGraphicsMem" #define OPREGION_SIZE (8 * 1024) @@ -257,7 +257,7 @@ static int vfio_pci_igd_opregion_init(struct vfio_pci_core_device *vdev) } } - ret = vfio_pci_register_dev_region(vdev, + ret = vfio_pci_core_register_dev_region(vdev, PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &vfio_pci_igd_regops, size, VFIO_REGION_INFO_FLAG_READ, opregionvbt); @@ -402,7 +402,7 @@ static int vfio_pci_igd_cfg_init(struct vfio_pci_core_device *vdev) return -EINVAL; } - ret = vfio_pci_register_dev_region(vdev, + ret = vfio_pci_core_register_dev_region(vdev, PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &vfio_pci_igd_cfg_regops, host_bridge->cfg_size, @@ -422,7 +422,7 @@ static int vfio_pci_igd_cfg_init(struct vfio_pci_core_device *vdev) return -EINVAL; } - ret = vfio_pci_register_dev_region(vdev, + ret = vfio_pci_core_register_dev_region(vdev, PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE, VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &vfio_pci_igd_cfg_regops, lpc_bridge->cfg_size, diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index 6069a11fb51acf418bc89a787fb96e81325fbedb..40c3d7cf163f69ce0a860e36d583db4b0ea1015b 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -20,7 +20,33 @@ #include #include -#include +#include "vfio_pci_priv.h" + +struct vfio_pci_irq_ctx { + struct eventfd_ctx *trigger; + struct virqfd *unmask; + struct virqfd *mask; + char *name; + bool masked; + struct irq_bypass_producer producer; +}; + +static bool irq_is(struct vfio_pci_core_device *vdev, int type) +{ + return vdev->irq_type == type; +} + +static bool is_intx(struct vfio_pci_core_device *vdev) +{ + return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; +} + +static bool is_irq_none(struct vfio_pci_core_device *vdev) +{ + return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || + vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || + vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); +} /* * INTx @@ -33,10 +59,12 @@ static void vfio_send_intx_eventfd(void *opaque, void *unused) eventfd_signal(vdev->ctx[0].trigger, 1); } -void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) +/* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ +bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) { struct pci_dev *pdev = vdev->pdev; unsigned long flags; + bool masked_changed = false; spin_lock_irqsave(&vdev->irqlock, flags); @@ -60,9 +88,11 @@ void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) disable_irq_nosync(pdev->irq); vdev->ctx[0].masked = true; + masked_changed = true; } spin_unlock_irqrestore(&vdev->irqlock, flags); + return masked_changed; } /* diff --git a/drivers/vfio/pci/vfio_pci_priv.h b/drivers/vfio/pci/vfio_pci_priv.h new file mode 100644 index 0000000000000000000000000000000000000000..5e4fa69aee16c13966391328440e9353b5e33427 --- /dev/null +++ b/drivers/vfio/pci/vfio_pci_priv.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef VFIO_PCI_PRIV_H +#define VFIO_PCI_PRIV_H + +#include + +/* Special capability IDs predefined access */ +#define PCI_CAP_ID_INVALID 0xFF /* default raw access */ +#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */ + +/* Cap maximum number of ioeventfds per device (arbitrary) */ +#define VFIO_PCI_IOEVENTFD_MAX 1000 + +struct vfio_pci_ioeventfd { + struct list_head next; + struct vfio_pci_core_device *vdev; + struct virqfd *virqfd; + void __iomem *addr; + uint64_t data; + loff_t pos; + int bar; + int count; + bool test_mem; +}; + +bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); +void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); + +int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, + unsigned index, unsigned start, unsigned count, + void *data); + +ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); + +ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); + +#ifdef CONFIG_VFIO_PCI_VGA +ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, + size_t count, loff_t *ppos, bool iswrite); +#else +static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, + char __user *buf, size_t count, + loff_t *ppos, bool iswrite) +{ + return -EINVAL; +} +#endif + +int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, + uint64_t data, int count, int fd); + +int vfio_pci_init_perm_bits(void); +void vfio_pci_uninit_perm_bits(void); + +int vfio_config_init(struct vfio_pci_core_device *vdev); +void vfio_config_free(struct vfio_pci_core_device *vdev); + +int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, + pci_power_t state); + +bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); +void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev); +u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); +void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, + u16 cmd); + +#ifdef CONFIG_VFIO_PCI_IGD +int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); +#else +static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) +{ + return -ENODEV; +} +#endif + +#ifdef CONFIG_VFIO_PCI_ZDEV_KVM +int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, + struct vfio_info_cap *caps); +int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev); +void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev); +#else +static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, + struct vfio_info_cap *caps) +{ + return -ENODEV; +} + +static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) +{ + return 0; +} + +static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) +{} +#endif + +static inline bool vfio_pci_is_vga(struct pci_dev *pdev) +{ + return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; +} + +#endif diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index 82ac1569deb0526d582bd6b28b4adf9cfc075fa1..e352a033b4aef770e434a42ebbeece0fbf6eb7a3 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -17,7 +17,7 @@ #include #include -#include +#include "vfio_pci_priv.h" #ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 @@ -412,8 +412,8 @@ static void vfio_pci_ioeventfd_thread(void *opaque, void *unused) vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem); } -long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, - uint64_t data, int count, int fd) +int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, + uint64_t data, int count, int fd) { struct pci_dev *pdev = vdev->pdev; loff_t pos = offset & VFIO_PCI_OFFSET_MASK; diff --git a/drivers/vfio/pci/vfio_pci_zdev.c b/drivers/vfio/pci/vfio_pci_zdev.c index e163aa9f61444f7b6ead297f7149f478772196d0..0990fdb146b7826c02b214b3a383077a40dbaf0b 100644 --- a/drivers/vfio/pci/vfio_pci_zdev.c +++ b/drivers/vfio/pci/vfio_pci_zdev.c @@ -15,7 +15,7 @@ #include #include -#include +#include "vfio_pci_priv.h" /* * Add the Base PCI Function information to the device info region. @@ -151,7 +151,10 @@ int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) if (!vdev->vdev.kvm) return 0; - return kvm_s390_pci_register_kvm(zdev, vdev->vdev.kvm); + if (zpci_kvm_hook.kvm_register) + return zpci_kvm_hook.kvm_register(zdev, vdev->vdev.kvm); + + return -ENOENT; } void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) @@ -161,5 +164,6 @@ void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) if (!zdev || !vdev->vdev.kvm) return; - kvm_s390_pci_unregister_kvm(zdev); + if (zpci_kvm_hook.kvm_unregister) + zpci_kvm_hook.kvm_unregister(zdev); } diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c index 1aaa4f721bd2ce13f0332aa5c72934997182a4fe..eaea63e5294c58d9874d5526d3921d0bfc88874d 100644 --- a/drivers/vfio/platform/vfio_amba.c +++ b/drivers/vfio/platform/vfio_amba.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "vfio_platform_private.h" @@ -40,20 +41,16 @@ static int get_amba_irq(struct vfio_platform_device *vdev, int i) return ret ? ret : -ENXIO; } -static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id) +static int vfio_amba_init_dev(struct vfio_device *core_vdev) { - struct vfio_platform_device *vdev; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); + struct amba_device *adev = to_amba_device(core_vdev->dev); int ret; - vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); - if (!vdev) - return -ENOMEM; - vdev->name = kasprintf(GFP_KERNEL, "vfio-amba-%08x", adev->periphid); - if (!vdev->name) { - kfree(vdev); + if (!vdev->name) return -ENOMEM; - } vdev->opaque = (void *) adev; vdev->flags = VFIO_DEVICE_FLAGS_AMBA; @@ -61,26 +58,67 @@ static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id) vdev->get_irq = get_amba_irq; vdev->reset_required = false; - ret = vfio_platform_probe_common(vdev, &adev->dev); - if (ret) { + ret = vfio_platform_init_common(vdev); + if (ret) kfree(vdev->name); - kfree(vdev); - return ret; - } + return ret; +} + +static const struct vfio_device_ops vfio_amba_ops; +static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id) +{ + struct vfio_platform_device *vdev; + int ret; + + vdev = vfio_alloc_device(vfio_platform_device, vdev, &adev->dev, + &vfio_amba_ops); + if (IS_ERR(vdev)) + return PTR_ERR(vdev); + ret = vfio_register_group_dev(&vdev->vdev); + if (ret) + goto out_put_vdev; + + pm_runtime_enable(&adev->dev); dev_set_drvdata(&adev->dev, vdev); return 0; + +out_put_vdev: + vfio_put_device(&vdev->vdev); + return ret; +} + +static void vfio_amba_release_dev(struct vfio_device *core_vdev) +{ + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); + + vfio_platform_release_common(vdev); + kfree(vdev->name); + vfio_free_device(core_vdev); } static void vfio_amba_remove(struct amba_device *adev) { struct vfio_platform_device *vdev = dev_get_drvdata(&adev->dev); - vfio_platform_remove_common(vdev); - kfree(vdev->name); - kfree(vdev); + vfio_unregister_group_dev(&vdev->vdev); + pm_runtime_disable(vdev->device); + vfio_put_device(&vdev->vdev); } +static const struct vfio_device_ops vfio_amba_ops = { + .name = "vfio-amba", + .init = vfio_amba_init_dev, + .release = vfio_amba_release_dev, + .open_device = vfio_platform_open_device, + .close_device = vfio_platform_close_device, + .ioctl = vfio_platform_ioctl, + .read = vfio_platform_read, + .write = vfio_platform_write, + .mmap = vfio_platform_mmap, +}; + static const struct amba_id pl330_ids[] = { { 0, 0 }, }; diff --git a/drivers/vfio/platform/vfio_platform.c b/drivers/vfio/platform/vfio_platform.c index 04f40c5acfd673d8f087a0c10505ab837c9ffc0b..82cedcebfd9022ca7e08fe920b01ebc5e3cfa62e 100644 --- a/drivers/vfio/platform/vfio_platform.c +++ b/drivers/vfio/platform/vfio_platform.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "vfio_platform_private.h" @@ -36,14 +37,11 @@ static int get_platform_irq(struct vfio_platform_device *vdev, int i) return platform_get_irq_optional(pdev, i); } -static int vfio_platform_probe(struct platform_device *pdev) +static int vfio_platform_init_dev(struct vfio_device *core_vdev) { - struct vfio_platform_device *vdev; - int ret; - - vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); - if (!vdev) - return -ENOMEM; + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); + struct platform_device *pdev = to_platform_device(core_vdev->dev); vdev->opaque = (void *) pdev; vdev->name = pdev->name; @@ -52,24 +50,64 @@ static int vfio_platform_probe(struct platform_device *pdev) vdev->get_irq = get_platform_irq; vdev->reset_required = reset_required; - ret = vfio_platform_probe_common(vdev, &pdev->dev); - if (ret) { - kfree(vdev); - return ret; - } + return vfio_platform_init_common(vdev); +} + +static const struct vfio_device_ops vfio_platform_ops; +static int vfio_platform_probe(struct platform_device *pdev) +{ + struct vfio_platform_device *vdev; + int ret; + + vdev = vfio_alloc_device(vfio_platform_device, vdev, &pdev->dev, + &vfio_platform_ops); + if (IS_ERR(vdev)) + return PTR_ERR(vdev); + + ret = vfio_register_group_dev(&vdev->vdev); + if (ret) + goto out_put_vdev; + + pm_runtime_enable(&pdev->dev); dev_set_drvdata(&pdev->dev, vdev); return 0; + +out_put_vdev: + vfio_put_device(&vdev->vdev); + return ret; +} + +static void vfio_platform_release_dev(struct vfio_device *core_vdev) +{ + struct vfio_platform_device *vdev = + container_of(core_vdev, struct vfio_platform_device, vdev); + + vfio_platform_release_common(vdev); + vfio_free_device(core_vdev); } static int vfio_platform_remove(struct platform_device *pdev) { struct vfio_platform_device *vdev = dev_get_drvdata(&pdev->dev); - vfio_platform_remove_common(vdev); - kfree(vdev); + vfio_unregister_group_dev(&vdev->vdev); + pm_runtime_disable(vdev->device); + vfio_put_device(&vdev->vdev); return 0; } +static const struct vfio_device_ops vfio_platform_ops = { + .name = "vfio-platform", + .init = vfio_platform_init_dev, + .release = vfio_platform_release_dev, + .open_device = vfio_platform_open_device, + .close_device = vfio_platform_close_device, + .ioctl = vfio_platform_ioctl, + .read = vfio_platform_read, + .write = vfio_platform_write, + .mmap = vfio_platform_mmap, +}; + static struct platform_driver vfio_platform_driver = { .probe = vfio_platform_probe, .remove = vfio_platform_remove, diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c index 256f55b84e70a0e887e38fcac29905e448d48f11..55dc4f43c31e3eccbeada369c6de76ed89e5fc61 100644 --- a/drivers/vfio/platform/vfio_platform_common.c +++ b/drivers/vfio/platform/vfio_platform_common.c @@ -218,7 +218,7 @@ static int vfio_platform_call_reset(struct vfio_platform_device *vdev, return -EINVAL; } -static void vfio_platform_close_device(struct vfio_device *core_vdev) +void vfio_platform_close_device(struct vfio_device *core_vdev) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -236,8 +236,9 @@ static void vfio_platform_close_device(struct vfio_device *core_vdev) vfio_platform_regions_cleanup(vdev); vfio_platform_irq_cleanup(vdev); } +EXPORT_SYMBOL_GPL(vfio_platform_close_device); -static int vfio_platform_open_device(struct vfio_device *core_vdev) +int vfio_platform_open_device(struct vfio_device *core_vdev) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -273,9 +274,10 @@ err_irq: vfio_platform_regions_cleanup(vdev); return ret; } +EXPORT_SYMBOL_GPL(vfio_platform_open_device); -static long vfio_platform_ioctl(struct vfio_device *core_vdev, - unsigned int cmd, unsigned long arg) +long vfio_platform_ioctl(struct vfio_device *core_vdev, + unsigned int cmd, unsigned long arg) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -382,6 +384,7 @@ static long vfio_platform_ioctl(struct vfio_device *core_vdev, return -ENOTTY; } +EXPORT_SYMBOL_GPL(vfio_platform_ioctl); static ssize_t vfio_platform_read_mmio(struct vfio_platform_region *reg, char __user *buf, size_t count, @@ -438,8 +441,8 @@ err: return -EFAULT; } -static ssize_t vfio_platform_read(struct vfio_device *core_vdev, - char __user *buf, size_t count, loff_t *ppos) +ssize_t vfio_platform_read(struct vfio_device *core_vdev, + char __user *buf, size_t count, loff_t *ppos) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -460,6 +463,7 @@ static ssize_t vfio_platform_read(struct vfio_device *core_vdev, return -EINVAL; } +EXPORT_SYMBOL_GPL(vfio_platform_read); static ssize_t vfio_platform_write_mmio(struct vfio_platform_region *reg, const char __user *buf, size_t count, @@ -515,8 +519,8 @@ err: return -EFAULT; } -static ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __user *buf, - size_t count, loff_t *ppos) +ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __user *buf, + size_t count, loff_t *ppos) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -537,6 +541,7 @@ static ssize_t vfio_platform_write(struct vfio_device *core_vdev, const char __u return -EINVAL; } +EXPORT_SYMBOL_GPL(vfio_platform_write); static int vfio_platform_mmap_mmio(struct vfio_platform_region region, struct vm_area_struct *vma) @@ -558,7 +563,7 @@ static int vfio_platform_mmap_mmio(struct vfio_platform_region region, req_len, vma->vm_page_prot); } -static int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) +int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma) { struct vfio_platform_device *vdev = container_of(core_vdev, struct vfio_platform_device, vdev); @@ -598,16 +603,7 @@ static int vfio_platform_mmap(struct vfio_device *core_vdev, struct vm_area_stru return -EINVAL; } - -static const struct vfio_device_ops vfio_platform_ops = { - .name = "vfio-platform", - .open_device = vfio_platform_open_device, - .close_device = vfio_platform_close_device, - .ioctl = vfio_platform_ioctl, - .read = vfio_platform_read, - .write = vfio_platform_write, - .mmap = vfio_platform_mmap, -}; +EXPORT_SYMBOL_GPL(vfio_platform_mmap); static int vfio_platform_of_probe(struct vfio_platform_device *vdev, struct device *dev) @@ -639,55 +635,34 @@ static int vfio_platform_of_probe(struct vfio_platform_device *vdev, * If the firmware is ACPI type, then acpi_disabled is 0. All other checks are * valid checks. We cannot claim that this system is DT. */ -int vfio_platform_probe_common(struct vfio_platform_device *vdev, - struct device *dev) +int vfio_platform_init_common(struct vfio_platform_device *vdev) { int ret; - - vfio_init_group_dev(&vdev->vdev, dev, &vfio_platform_ops); + struct device *dev = vdev->vdev.dev; ret = vfio_platform_acpi_probe(vdev, dev); if (ret) ret = vfio_platform_of_probe(vdev, dev); if (ret) - goto out_uninit; + return ret; vdev->device = dev; + mutex_init(&vdev->igate); ret = vfio_platform_get_reset(vdev); - if (ret && vdev->reset_required) { + if (ret && vdev->reset_required) dev_err(dev, "No reset function found for device %s\n", vdev->name); - goto out_uninit; - } - - ret = vfio_register_group_dev(&vdev->vdev); - if (ret) - goto put_reset; - - mutex_init(&vdev->igate); - - pm_runtime_enable(dev); - return 0; - -put_reset: - vfio_platform_put_reset(vdev); -out_uninit: - vfio_uninit_group_dev(&vdev->vdev); return ret; } -EXPORT_SYMBOL_GPL(vfio_platform_probe_common); +EXPORT_SYMBOL_GPL(vfio_platform_init_common); -void vfio_platform_remove_common(struct vfio_platform_device *vdev) +void vfio_platform_release_common(struct vfio_platform_device *vdev) { - vfio_unregister_group_dev(&vdev->vdev); - - pm_runtime_disable(vdev->device); vfio_platform_put_reset(vdev); - vfio_uninit_group_dev(&vdev->vdev); } -EXPORT_SYMBOL_GPL(vfio_platform_remove_common); +EXPORT_SYMBOL_GPL(vfio_platform_release_common); void __vfio_platform_register_reset(struct vfio_platform_reset_node *node) { diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h index 691b43f4b2b2928debf8f7c8ad0b35447fbeada3..8d8fab51684909ea81e15c9cffc1ae581a6040e4 100644 --- a/drivers/vfio/platform/vfio_platform_private.h +++ b/drivers/vfio/platform/vfio_platform_private.h @@ -78,9 +78,21 @@ struct vfio_platform_reset_node { vfio_platform_reset_fn_t of_reset; }; -int vfio_platform_probe_common(struct vfio_platform_device *vdev, - struct device *dev); -void vfio_platform_remove_common(struct vfio_platform_device *vdev); +int vfio_platform_init_common(struct vfio_platform_device *vdev); +void vfio_platform_release_common(struct vfio_platform_device *vdev); + +int vfio_platform_open_device(struct vfio_device *core_vdev); +void vfio_platform_close_device(struct vfio_device *core_vdev); +long vfio_platform_ioctl(struct vfio_device *core_vdev, + unsigned int cmd, unsigned long arg); +ssize_t vfio_platform_read(struct vfio_device *core_vdev, + char __user *buf, size_t count, + loff_t *ppos); +ssize_t vfio_platform_write(struct vfio_device *core_vdev, + const char __user *buf, + size_t count, loff_t *ppos); +int vfio_platform_mmap(struct vfio_device *core_vdev, + struct vm_area_struct *vma); int vfio_platform_irq_init(struct vfio_platform_device *vdev); void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev); diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 503bea6c843d56ec8c151e25503d42275736e2dd..bcad54bbab08c4b8497e5aead5d5576fc45a01e4 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -3,6 +3,16 @@ * Copyright (C) 2012 Red Hat, Inc. All rights reserved. * Author: Alex Williamson */ +#ifndef __VFIO_VFIO_H__ +#define __VFIO_VFIO_H__ + +#include +#include +#include + +struct iommu_group; +struct vfio_device; +struct vfio_container; enum vfio_group_type { /* @@ -28,6 +38,30 @@ enum vfio_group_type { VFIO_NO_IOMMU, }; +struct vfio_group { + struct device dev; + struct cdev cdev; + /* + * When drivers is non-zero a driver is attached to the struct device + * that provided the iommu_group and thus the iommu_group is a valid + * pointer. When drivers is 0 the driver is being detached. Once users + * reaches 0 then the iommu_group is invalid. + */ + refcount_t drivers; + unsigned int container_users; + struct iommu_group *iommu_group; + struct vfio_container *container; + struct list_head device_list; + struct mutex device_lock; + struct list_head vfio_next; + struct list_head container_next; + enum vfio_group_type type; + struct mutex group_lock; + struct kvm *kvm; + struct file *opened_file; + struct blocking_notifier_head notifier; +}; + /* events for the backend driver notify callback */ enum vfio_iommu_notify_type { VFIO_IOMMU_CONTAINER_CLOSE = 0, @@ -67,5 +101,33 @@ struct vfio_iommu_driver_ops { enum vfio_iommu_notify_type event); }; +struct vfio_iommu_driver { + const struct vfio_iommu_driver_ops *ops; + struct list_head vfio_next; +}; + int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops); void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops); + +bool vfio_assert_device_open(struct vfio_device *device); + +struct vfio_container *vfio_container_from_file(struct file *filep); +int vfio_device_assign_container(struct vfio_device *device); +void vfio_device_unassign_container(struct vfio_device *device); +int vfio_container_attach_group(struct vfio_container *container, + struct vfio_group *group); +void vfio_group_detach_container(struct vfio_group *group); +void vfio_device_container_register(struct vfio_device *device); +void vfio_device_container_unregister(struct vfio_device *device); +long vfio_container_ioctl_check_extension(struct vfio_container *container, + unsigned long arg); +int __init vfio_container_init(void); +void vfio_container_cleanup(void); + +#ifdef CONFIG_VFIO_NOIOMMU +extern bool vfio_noiommu __read_mostly; +#else +enum { vfio_noiommu = false }; +#endif + +#endif diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index db516c90a977066fb1de7de3601669019c40fcd1..23c24fe98c00d4a07ef73826a02d81386cf2118d 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include "vfio.h" @@ -558,6 +557,18 @@ static int vaddr_get_pfns(struct mm_struct *mm, unsigned long vaddr, ret = pin_user_pages_remote(mm, vaddr, npages, flags | FOLL_LONGTERM, pages, NULL, NULL); if (ret > 0) { + int i; + + /* + * The zero page is always resident, we don't need to pin it + * and it falls into our invalid/reserved test so we don't + * unpin in put_pfn(). Unpin all zero pages in the batch here. + */ + for (i = 0 ; i < ret; i++) { + if (unlikely(is_zero_pfn(page_to_pfn(pages[i])))) + unpin_user_page(pages[i]); + } + *pfn = page_to_pfn(pages[0]); goto done; } diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 7cb56c382c97a25c8634b44148ea77bbdf228fff..2d168793d4e1ce99b223c5b4c193e49476cf493c 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -32,6 +32,9 @@ #include #include #include +#include +#include +#include #include "vfio.h" #define DRIVER_VERSION "0.3" @@ -40,54 +43,14 @@ static struct vfio { struct class *class; - struct list_head iommu_drivers_list; - struct mutex iommu_drivers_lock; struct list_head group_list; struct mutex group_lock; /* locks group_list */ struct ida group_ida; dev_t group_devt; + struct class *device_class; + struct ida device_ida; } vfio; -struct vfio_iommu_driver { - const struct vfio_iommu_driver_ops *ops; - struct list_head vfio_next; -}; - -struct vfio_container { - struct kref kref; - struct list_head group_list; - struct rw_semaphore group_lock; - struct vfio_iommu_driver *iommu_driver; - void *iommu_data; - bool noiommu; -}; - -struct vfio_group { - struct device dev; - struct cdev cdev; - refcount_t users; - unsigned int container_users; - struct iommu_group *iommu_group; - struct vfio_container *container; - struct list_head device_list; - struct mutex device_lock; - struct list_head vfio_next; - struct list_head container_next; - enum vfio_group_type type; - unsigned int dev_counter; - struct rw_semaphore group_rwsem; - struct kvm *kvm; - struct file *opened_file; - struct blocking_notifier_head notifier; -}; - -#ifdef CONFIG_VFIO_NOIOMMU -static bool noiommu __read_mostly; -module_param_named(enable_unsafe_noiommu_mode, - noiommu, bool, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode. This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel. If you do not know what this is for, step away. (default: false)"); -#endif - static DEFINE_XARRAY(vfio_device_set_xa); static const struct file_operations vfio_group_fops; @@ -162,146 +125,6 @@ static void vfio_release_device_set(struct vfio_device *device) xa_unlock(&vfio_device_set_xa); } -#ifdef CONFIG_VFIO_NOIOMMU -static void *vfio_noiommu_open(unsigned long arg) -{ - if (arg != VFIO_NOIOMMU_IOMMU) - return ERR_PTR(-EINVAL); - if (!capable(CAP_SYS_RAWIO)) - return ERR_PTR(-EPERM); - - return NULL; -} - -static void vfio_noiommu_release(void *iommu_data) -{ -} - -static long vfio_noiommu_ioctl(void *iommu_data, - unsigned int cmd, unsigned long arg) -{ - if (cmd == VFIO_CHECK_EXTENSION) - return noiommu && (arg == VFIO_NOIOMMU_IOMMU) ? 1 : 0; - - return -ENOTTY; -} - -static int vfio_noiommu_attach_group(void *iommu_data, - struct iommu_group *iommu_group, enum vfio_group_type type) -{ - return 0; -} - -static void vfio_noiommu_detach_group(void *iommu_data, - struct iommu_group *iommu_group) -{ -} - -static const struct vfio_iommu_driver_ops vfio_noiommu_ops = { - .name = "vfio-noiommu", - .owner = THIS_MODULE, - .open = vfio_noiommu_open, - .release = vfio_noiommu_release, - .ioctl = vfio_noiommu_ioctl, - .attach_group = vfio_noiommu_attach_group, - .detach_group = vfio_noiommu_detach_group, -}; - -/* - * Only noiommu containers can use vfio-noiommu and noiommu containers can only - * use vfio-noiommu. - */ -static inline bool vfio_iommu_driver_allowed(struct vfio_container *container, - const struct vfio_iommu_driver *driver) -{ - return container->noiommu == (driver->ops == &vfio_noiommu_ops); -} -#else -static inline bool vfio_iommu_driver_allowed(struct vfio_container *container, - const struct vfio_iommu_driver *driver) -{ - return true; -} -#endif /* CONFIG_VFIO_NOIOMMU */ - -/* - * IOMMU driver registration - */ -int vfio_register_iommu_driver(const struct vfio_iommu_driver_ops *ops) -{ - struct vfio_iommu_driver *driver, *tmp; - - if (WARN_ON(!ops->register_device != !ops->unregister_device)) - return -EINVAL; - - driver = kzalloc(sizeof(*driver), GFP_KERNEL); - if (!driver) - return -ENOMEM; - - driver->ops = ops; - - mutex_lock(&vfio.iommu_drivers_lock); - - /* Check for duplicates */ - list_for_each_entry(tmp, &vfio.iommu_drivers_list, vfio_next) { - if (tmp->ops == ops) { - mutex_unlock(&vfio.iommu_drivers_lock); - kfree(driver); - return -EINVAL; - } - } - - list_add(&driver->vfio_next, &vfio.iommu_drivers_list); - - mutex_unlock(&vfio.iommu_drivers_lock); - - return 0; -} -EXPORT_SYMBOL_GPL(vfio_register_iommu_driver); - -void vfio_unregister_iommu_driver(const struct vfio_iommu_driver_ops *ops) -{ - struct vfio_iommu_driver *driver; - - mutex_lock(&vfio.iommu_drivers_lock); - list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) { - if (driver->ops == ops) { - list_del(&driver->vfio_next); - mutex_unlock(&vfio.iommu_drivers_lock); - kfree(driver); - return; - } - } - mutex_unlock(&vfio.iommu_drivers_lock); -} -EXPORT_SYMBOL_GPL(vfio_unregister_iommu_driver); - -static void vfio_group_get(struct vfio_group *group); - -/* - * Container objects - containers are created when /dev/vfio/vfio is - * opened, but their lifecycle extends until the last user is done, so - * it's freed via kref. Must support container/group/device being - * closed in any order. - */ -static void vfio_container_get(struct vfio_container *container) -{ - kref_get(&container->kref); -} - -static void vfio_container_release(struct kref *kref) -{ - struct vfio_container *container; - container = container_of(kref, struct vfio_container, kref); - - kfree(container); -} - -static void vfio_container_put(struct vfio_container *container) -{ - kref_put(&container->kref, vfio_container_release); -} - /* * Group objects - create, release, get, put, search */ @@ -310,9 +133,13 @@ __vfio_group_get_from_iommu(struct iommu_group *iommu_group) { struct vfio_group *group; + /* + * group->iommu_group from the vfio.group_list cannot be NULL + * under the vfio.group_lock. + */ list_for_each_entry(group, &vfio.group_list, vfio_next) { if (group->iommu_group == iommu_group) { - vfio_group_get(group); + refcount_inc(&group->drivers); return group; } } @@ -335,7 +162,8 @@ static void vfio_group_release(struct device *dev) struct vfio_group *group = container_of(dev, struct vfio_group, dev); mutex_destroy(&group->device_lock); - iommu_group_put(group->iommu_group); + mutex_destroy(&group->group_lock); + WARN_ON(group->iommu_group); ida_free(&vfio.group_ida, MINOR(group->dev.devt)); kfree(group); } @@ -363,8 +191,8 @@ static struct vfio_group *vfio_group_alloc(struct iommu_group *iommu_group, cdev_init(&group->cdev, &vfio_group_fops); group->cdev.owner = THIS_MODULE; - refcount_set(&group->users, 1); - init_rwsem(&group->group_rwsem); + refcount_set(&group->drivers, 1); + mutex_init(&group->group_lock); INIT_LIST_HEAD(&group->device_list); mutex_init(&group->device_lock); group->iommu_group = iommu_group; @@ -420,44 +248,64 @@ err_put: return ret; } -static void vfio_group_put(struct vfio_group *group) +static void vfio_device_remove_group(struct vfio_device *device) { - if (!refcount_dec_and_mutex_lock(&group->users, &vfio.group_lock)) + struct vfio_group *group = device->group; + struct iommu_group *iommu_group; + + if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU) + iommu_group_remove_device(device->dev); + + /* Pairs with vfio_create_group() / vfio_group_get_from_iommu() */ + if (!refcount_dec_and_mutex_lock(&group->drivers, &vfio.group_lock)) return; + list_del(&group->vfio_next); + /* + * We could concurrently probe another driver in the group that might + * race vfio_device_remove_group() with vfio_get_group(), so we have to + * ensure that the sysfs is all cleaned up under lock otherwise the + * cdev_device_add() will fail due to the name aready existing. + */ + cdev_device_del(&group->cdev, &group->dev); + + mutex_lock(&group->group_lock); /* * These data structures all have paired operations that can only be - * undone when the caller holds a live reference on the group. Since all - * pairs must be undone these WARN_ON's indicate some caller did not + * undone when the caller holds a live reference on the device. Since + * all pairs must be undone these WARN_ON's indicate some caller did not * properly hold the group reference. */ WARN_ON(!list_empty(&group->device_list)); - WARN_ON(group->container || group->container_users); WARN_ON(group->notifier.head); - list_del(&group->vfio_next); - cdev_device_del(&group->cdev, &group->dev); + /* + * Revoke all users of group->iommu_group. At this point we know there + * are no devices active because we are unplugging the last one. Setting + * iommu_group to NULL blocks all new users. + */ + if (group->container) + vfio_group_detach_container(group); + iommu_group = group->iommu_group; + group->iommu_group = NULL; + mutex_unlock(&group->group_lock); mutex_unlock(&vfio.group_lock); + iommu_group_put(iommu_group); put_device(&group->dev); } -static void vfio_group_get(struct vfio_group *group) -{ - refcount_inc(&group->users); -} - /* * Device objects - create, release, get, put, search */ /* Device reference always implies a group reference */ -static void vfio_device_put(struct vfio_device *device) +static void vfio_device_put_registration(struct vfio_device *device) { if (refcount_dec_and_test(&device->refcount)) complete(&device->comp); } -static bool vfio_device_try_get(struct vfio_device *device) +static bool vfio_device_try_get_registration(struct vfio_device *device) { return refcount_inc_not_zero(&device->refcount); } @@ -469,7 +317,8 @@ static struct vfio_device *vfio_group_get_device(struct vfio_group *group, mutex_lock(&group->device_lock); list_for_each_entry(device, &group->device_list, group_next) { - if (device->dev == dev && vfio_device_try_get(device)) { + if (device->dev == dev && + vfio_device_try_get_registration(device)) { mutex_unlock(&group->device_lock); return device; } @@ -481,20 +330,110 @@ static struct vfio_device *vfio_group_get_device(struct vfio_group *group, /* * VFIO driver API */ -void vfio_init_group_dev(struct vfio_device *device, struct device *dev, - const struct vfio_device_ops *ops) +/* Release helper called by vfio_put_device() */ +static void vfio_device_release(struct device *dev) { + struct vfio_device *device = + container_of(dev, struct vfio_device, device); + + vfio_release_device_set(device); + ida_free(&vfio.device_ida, device->index); + + /* + * kvfree() cannot be done here due to a life cycle mess in + * vfio-ccw. Before the ccw part is fixed all drivers are + * required to support @release and call vfio_free_device() + * from there. + */ + device->ops->release(device); +} + +/* + * Allocate and initialize vfio_device so it can be registered to vfio + * core. + * + * Drivers should use the wrapper vfio_alloc_device() for allocation. + * @size is the size of the structure to be allocated, including any + * private data used by the driver. + * + * Driver may provide an @init callback to cover device private data. + * + * Use vfio_put_device() to release the structure after success return. + */ +struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, + const struct vfio_device_ops *ops) +{ + struct vfio_device *device; + int ret; + + if (WARN_ON(size < sizeof(struct vfio_device))) + return ERR_PTR(-EINVAL); + + device = kvzalloc(size, GFP_KERNEL); + if (!device) + return ERR_PTR(-ENOMEM); + + ret = vfio_init_device(device, dev, ops); + if (ret) + goto out_free; + return device; + +out_free: + kvfree(device); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(_vfio_alloc_device); + +/* + * Initialize a vfio_device so it can be registered to vfio core. + * + * Only vfio-ccw driver should call this interface. + */ +int vfio_init_device(struct vfio_device *device, struct device *dev, + const struct vfio_device_ops *ops) +{ + int ret; + + ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL); + if (ret < 0) { + dev_dbg(dev, "Error to alloc index\n"); + return ret; + } + + device->index = ret; init_completion(&device->comp); device->dev = dev; device->ops = ops; + + if (ops->init) { + ret = ops->init(device); + if (ret) + goto out_uninit; + } + + device_initialize(&device->device); + device->device.release = vfio_device_release; + device->device.class = vfio.device_class; + device->device.parent = device->dev; + return 0; + +out_uninit: + vfio_release_device_set(device); + ida_free(&vfio.device_ida, device->index); + return ret; } -EXPORT_SYMBOL_GPL(vfio_init_group_dev); +EXPORT_SYMBOL_GPL(vfio_init_device); -void vfio_uninit_group_dev(struct vfio_device *device) +/* + * The helper called by driver @release callback to free the device + * structure. Drivers which don't have private data to clean can + * simply use this helper as its @release. + */ +void vfio_free_device(struct vfio_device *device) { - vfio_release_device_set(device); + kvfree(device); } -EXPORT_SYMBOL_GPL(vfio_uninit_group_dev); +EXPORT_SYMBOL_GPL(vfio_free_device); static struct vfio_group *vfio_noiommu_group_alloc(struct device *dev, enum vfio_group_type type) @@ -535,8 +474,7 @@ static struct vfio_group *vfio_group_find_or_alloc(struct device *dev) struct vfio_group *group; iommu_group = iommu_group_get(dev); -#ifdef CONFIG_VFIO_NOIOMMU - if (!iommu_group && noiommu) { + if (!iommu_group && vfio_noiommu) { /* * With noiommu enabled, create an IOMMU group for devices that * don't already have one, implying no IOMMU hardware/driver @@ -550,7 +488,7 @@ static struct vfio_group *vfio_group_find_or_alloc(struct device *dev) } return group; } -#endif + if (!iommu_group) return ERR_PTR(-EINVAL); @@ -577,7 +515,12 @@ static int __vfio_register_dev(struct vfio_device *device, struct vfio_group *group) { struct vfio_device *existing_device; + int ret; + /* + * In all cases group is the output of one of the group allocation + * functions and we have group->drivers incremented for us. + */ if (IS_ERR(group)) return PTR_ERR(group); @@ -590,28 +533,39 @@ static int __vfio_register_dev(struct vfio_device *device, existing_device = vfio_group_get_device(group, device->dev); if (existing_device) { + /* + * group->iommu_group is non-NULL because we hold the drivers + * refcount. + */ dev_WARN(device->dev, "Device already exists on group %d\n", iommu_group_id(group->iommu_group)); - vfio_device_put(existing_device); - if (group->type == VFIO_NO_IOMMU || - group->type == VFIO_EMULATED_IOMMU) - iommu_group_remove_device(device->dev); - vfio_group_put(group); - return -EBUSY; + vfio_device_put_registration(existing_device); + ret = -EBUSY; + goto err_out; } /* Our reference on group is moved to the device */ device->group = group; + ret = dev_set_name(&device->device, "vfio%d", device->index); + if (ret) + goto err_out; + + ret = device_add(&device->device); + if (ret) + goto err_out; + /* Refcounting can't start until the driver calls register */ refcount_set(&device->refcount, 1); mutex_lock(&group->device_lock); list_add(&device->group_next, &group->device_list); - group->dev_counter++; mutex_unlock(&group->device_lock); return 0; +err_out: + vfio_device_remove_group(device); + return ret; } int vfio_register_group_dev(struct vfio_device *device) @@ -651,7 +605,7 @@ static struct vfio_device *vfio_device_get_from_name(struct vfio_group *group, ret = !strcmp(dev_name(it->dev), buf); } - if (ret && vfio_device_try_get(it)) { + if (ret && vfio_device_try_get_registration(it)) { device = it; break; } @@ -671,7 +625,7 @@ void vfio_unregister_group_dev(struct vfio_device *device) bool interrupted = false; long rc; - vfio_device_put(device); + vfio_device_put_registration(device); rc = try_wait_for_completion(&device->comp); while (rc <= 0) { if (device->ops->request) @@ -696,356 +650,78 @@ void vfio_unregister_group_dev(struct vfio_device *device) mutex_lock(&group->device_lock); list_del(&device->group_next); - group->dev_counter--; mutex_unlock(&group->device_lock); - if (group->type == VFIO_NO_IOMMU || group->type == VFIO_EMULATED_IOMMU) - iommu_group_remove_device(device->dev); + /* Balances device_add in register path */ + device_del(&device->device); - /* Matches the get in vfio_register_group_dev() */ - vfio_group_put(group); + vfio_device_remove_group(device); } EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); -/* - * VFIO base fd, /dev/vfio/vfio - */ -static long vfio_ioctl_check_extension(struct vfio_container *container, - unsigned long arg) -{ - struct vfio_iommu_driver *driver; - long ret = 0; - - down_read(&container->group_lock); - - driver = container->iommu_driver; - - switch (arg) { - /* No base extensions yet */ - default: - /* - * If no driver is set, poll all registered drivers for - * extensions and return the first positive result. If - * a driver is already set, further queries will be passed - * only to that driver. - */ - if (!driver) { - mutex_lock(&vfio.iommu_drivers_lock); - list_for_each_entry(driver, &vfio.iommu_drivers_list, - vfio_next) { - - if (!list_empty(&container->group_list) && - !vfio_iommu_driver_allowed(container, - driver)) - continue; - if (!try_module_get(driver->ops->owner)) - continue; - - ret = driver->ops->ioctl(NULL, - VFIO_CHECK_EXTENSION, - arg); - module_put(driver->ops->owner); - if (ret > 0) - break; - } - mutex_unlock(&vfio.iommu_drivers_lock); - } else - ret = driver->ops->ioctl(container->iommu_data, - VFIO_CHECK_EXTENSION, arg); - } - - up_read(&container->group_lock); - - return ret; -} - -/* hold write lock on container->group_lock */ -static int __vfio_container_attach_groups(struct vfio_container *container, - struct vfio_iommu_driver *driver, - void *data) -{ - struct vfio_group *group; - int ret = -ENODEV; - - list_for_each_entry(group, &container->group_list, container_next) { - ret = driver->ops->attach_group(data, group->iommu_group, - group->type); - if (ret) - goto unwind; - } - - return ret; - -unwind: - list_for_each_entry_continue_reverse(group, &container->group_list, - container_next) { - driver->ops->detach_group(data, group->iommu_group); - } - - return ret; -} - -static long vfio_ioctl_set_iommu(struct vfio_container *container, - unsigned long arg) -{ - struct vfio_iommu_driver *driver; - long ret = -ENODEV; - - down_write(&container->group_lock); - - /* - * The container is designed to be an unprivileged interface while - * the group can be assigned to specific users. Therefore, only by - * adding a group to a container does the user get the privilege of - * enabling the iommu, which may allocate finite resources. There - * is no unset_iommu, but by removing all the groups from a container, - * the container is deprivileged and returns to an unset state. - */ - if (list_empty(&container->group_list) || container->iommu_driver) { - up_write(&container->group_lock); - return -EINVAL; - } - - mutex_lock(&vfio.iommu_drivers_lock); - list_for_each_entry(driver, &vfio.iommu_drivers_list, vfio_next) { - void *data; - - if (!vfio_iommu_driver_allowed(container, driver)) - continue; - if (!try_module_get(driver->ops->owner)) - continue; - - /* - * The arg magic for SET_IOMMU is the same as CHECK_EXTENSION, - * so test which iommu driver reported support for this - * extension and call open on them. We also pass them the - * magic, allowing a single driver to support multiple - * interfaces if they'd like. - */ - if (driver->ops->ioctl(NULL, VFIO_CHECK_EXTENSION, arg) <= 0) { - module_put(driver->ops->owner); - continue; - } - - data = driver->ops->open(arg); - if (IS_ERR(data)) { - ret = PTR_ERR(data); - module_put(driver->ops->owner); - continue; - } - - ret = __vfio_container_attach_groups(container, driver, data); - if (ret) { - driver->ops->release(data); - module_put(driver->ops->owner); - continue; - } - - container->iommu_driver = driver; - container->iommu_data = data; - break; - } - - mutex_unlock(&vfio.iommu_drivers_lock); - up_write(&container->group_lock); - - return ret; -} - -static long vfio_fops_unl_ioctl(struct file *filep, - unsigned int cmd, unsigned long arg) -{ - struct vfio_container *container = filep->private_data; - struct vfio_iommu_driver *driver; - void *data; - long ret = -EINVAL; - - if (!container) - return ret; - - switch (cmd) { - case VFIO_GET_API_VERSION: - ret = VFIO_API_VERSION; - break; - case VFIO_CHECK_EXTENSION: - ret = vfio_ioctl_check_extension(container, arg); - break; - case VFIO_SET_IOMMU: - ret = vfio_ioctl_set_iommu(container, arg); - break; - default: - driver = container->iommu_driver; - data = container->iommu_data; - - if (driver) /* passthrough all unrecognized ioctls */ - ret = driver->ops->ioctl(data, cmd, arg); - } - - return ret; -} - -static int vfio_fops_open(struct inode *inode, struct file *filep) -{ - struct vfio_container *container; - - container = kzalloc(sizeof(*container), GFP_KERNEL); - if (!container) - return -ENOMEM; - - INIT_LIST_HEAD(&container->group_list); - init_rwsem(&container->group_lock); - kref_init(&container->kref); - - filep->private_data = container; - - return 0; -} - -static int vfio_fops_release(struct inode *inode, struct file *filep) -{ - struct vfio_container *container = filep->private_data; - struct vfio_iommu_driver *driver = container->iommu_driver; - - if (driver && driver->ops->notify) - driver->ops->notify(container->iommu_data, - VFIO_IOMMU_CONTAINER_CLOSE); - - filep->private_data = NULL; - - vfio_container_put(container); - - return 0; -} - -static const struct file_operations vfio_fops = { - .owner = THIS_MODULE, - .open = vfio_fops_open, - .release = vfio_fops_release, - .unlocked_ioctl = vfio_fops_unl_ioctl, - .compat_ioctl = compat_ptr_ioctl, -}; - /* * VFIO Group fd, /dev/vfio/$GROUP */ -static void __vfio_group_unset_container(struct vfio_group *group) -{ - struct vfio_container *container = group->container; - struct vfio_iommu_driver *driver; - - lockdep_assert_held_write(&group->group_rwsem); - - down_write(&container->group_lock); - - driver = container->iommu_driver; - if (driver) - driver->ops->detach_group(container->iommu_data, - group->iommu_group); - - if (group->type == VFIO_IOMMU) - iommu_group_release_dma_owner(group->iommu_group); - - group->container = NULL; - group->container_users = 0; - list_del(&group->container_next); - - /* Detaching the last group deprivileges a container, remove iommu */ - if (driver && list_empty(&container->group_list)) { - driver->ops->release(container->iommu_data); - module_put(driver->ops->owner); - container->iommu_driver = NULL; - container->iommu_data = NULL; - } - - up_write(&container->group_lock); - - vfio_container_put(container); -} - /* * VFIO_GROUP_UNSET_CONTAINER should fail if there are other users or * if there was no container to unset. Since the ioctl is called on * the group, we know that still exists, therefore the only valid * transition here is 1->0. */ -static int vfio_group_unset_container(struct vfio_group *group) +static int vfio_group_ioctl_unset_container(struct vfio_group *group) { - lockdep_assert_held_write(&group->group_rwsem); + int ret = 0; - if (!group->container) - return -EINVAL; - if (group->container_users != 1) - return -EBUSY; - __vfio_group_unset_container(group); - return 0; + mutex_lock(&group->group_lock); + if (!group->container) { + ret = -EINVAL; + goto out_unlock; + } + if (group->container_users != 1) { + ret = -EBUSY; + goto out_unlock; + } + vfio_group_detach_container(group); + +out_unlock: + mutex_unlock(&group->group_lock); + return ret; } -static int vfio_group_set_container(struct vfio_group *group, int container_fd) +static int vfio_group_ioctl_set_container(struct vfio_group *group, + int __user *arg) { - struct fd f; - struct vfio_container *container; - struct vfio_iommu_driver *driver; - int ret = 0; - - lockdep_assert_held_write(&group->group_rwsem); - - if (group->container || WARN_ON(group->container_users)) - return -EINVAL; + struct vfio_container *container; + struct fd f; + int ret; + int fd; - if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) - return -EPERM; + if (get_user(fd, arg)) + return -EFAULT; - f = fdget(container_fd); + f = fdget(fd); if (!f.file) return -EBADF; - /* Sanity check, is this really our fd? */ - if (f.file->f_op != &vfio_fops) { - fdput(f); - return -EINVAL; - } - - container = f.file->private_data; - WARN_ON(!container); /* fget ensures we don't race vfio_release */ - - down_write(&container->group_lock); - - /* Real groups and fake groups cannot mix */ - if (!list_empty(&container->group_list) && - container->noiommu != (group->type == VFIO_NO_IOMMU)) { - ret = -EPERM; - goto unlock_out; + mutex_lock(&group->group_lock); + if (group->container || WARN_ON(group->container_users)) { + ret = -EINVAL; + goto out_unlock; } - - if (group->type == VFIO_IOMMU) { - ret = iommu_group_claim_dma_owner(group->iommu_group, f.file); - if (ret) - goto unlock_out; + if (!group->iommu_group) { + ret = -ENODEV; + goto out_unlock; } - driver = container->iommu_driver; - if (driver) { - ret = driver->ops->attach_group(container->iommu_data, - group->iommu_group, - group->type); - if (ret) { - if (group->type == VFIO_IOMMU) - iommu_group_release_dma_owner( - group->iommu_group); - goto unlock_out; - } + container = vfio_container_from_file(f.file); + ret = -EINVAL; + if (container) { + ret = vfio_container_attach_group(container, group); + goto out_unlock; } - group->container = container; - group->container_users = 1; - container->noiommu = (group->type == VFIO_NO_IOMMU); - list_add(&group->container_next, &container->group_list); - - /* Get a reference on the container and mark a user within the group */ - vfio_container_get(container); - -unlock_out: - up_write(&container->group_lock); +out_unlock: + mutex_unlock(&group->group_lock); fdput(f); return ret; } @@ -1053,47 +729,19 @@ unlock_out: static const struct file_operations vfio_device_fops; /* true if the vfio_device has open_device() called but not close_device() */ -static bool vfio_assert_device_open(struct vfio_device *device) +bool vfio_assert_device_open(struct vfio_device *device) { return !WARN_ON_ONCE(!READ_ONCE(device->open_count)); } -static int vfio_device_assign_container(struct vfio_device *device) -{ - struct vfio_group *group = device->group; - - lockdep_assert_held_write(&group->group_rwsem); - - if (!group->container || !group->container->iommu_driver || - WARN_ON(!group->container_users)) - return -EINVAL; - - if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) - return -EPERM; - - get_file(group->opened_file); - group->container_users++; - return 0; -} - -static void vfio_device_unassign_container(struct vfio_device *device) -{ - down_write(&device->group->group_rwsem); - WARN_ON(device->group->container_users <= 1); - device->group->container_users--; - fput(device->group->opened_file); - up_write(&device->group->group_rwsem); -} - static struct file *vfio_device_open(struct vfio_device *device) { - struct vfio_iommu_driver *iommu_driver; struct file *filep; int ret; - down_write(&device->group->group_rwsem); + mutex_lock(&device->group->group_lock); ret = vfio_device_assign_container(device); - up_write(&device->group->group_rwsem); + mutex_unlock(&device->group->group_lock); if (ret) return ERR_PTR(ret); @@ -1110,7 +758,7 @@ static struct file *vfio_device_open(struct vfio_device *device) * lock. If the device driver will use it, it must obtain a * reference and release it during close_device. */ - down_read(&device->group->group_rwsem); + mutex_lock(&device->group->group_lock); device->kvm = device->group->kvm; if (device->ops->open_device) { @@ -1118,13 +766,8 @@ static struct file *vfio_device_open(struct vfio_device *device) if (ret) goto err_undo_count; } - - iommu_driver = device->group->container->iommu_driver; - if (iommu_driver && iommu_driver->ops->register_device) - iommu_driver->ops->register_device( - device->group->container->iommu_data, device); - - up_read(&device->group->group_rwsem); + vfio_device_container_register(device); + mutex_unlock(&device->group->group_lock); } mutex_unlock(&device->dev_set->lock); @@ -1157,17 +800,14 @@ static struct file *vfio_device_open(struct vfio_device *device) err_close_device: mutex_lock(&device->dev_set->lock); - down_read(&device->group->group_rwsem); + mutex_lock(&device->group->group_lock); if (device->open_count == 1 && device->ops->close_device) { device->ops->close_device(device); - iommu_driver = device->group->container->iommu_driver; - if (iommu_driver && iommu_driver->ops->unregister_device) - iommu_driver->ops->unregister_device( - device->group->container->iommu_data, device); + vfio_device_container_unregister(device); } err_undo_count: - up_read(&device->group->group_rwsem); + mutex_unlock(&device->group->group_lock); device->open_count--; if (device->open_count == 0 && device->kvm) device->kvm = NULL; @@ -1178,14 +818,21 @@ err_unassign_container: return ERR_PTR(ret); } -static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) +static int vfio_group_ioctl_get_device_fd(struct vfio_group *group, + char __user *arg) { struct vfio_device *device; struct file *filep; + char *buf; int fdno; int ret; + buf = strndup_user(arg, PAGE_SIZE); + if (IS_ERR(buf)) + return PTR_ERR(buf); + device = vfio_device_get_from_name(group, buf); + kfree(buf); if (IS_ERR(device)) return PTR_ERR(device); @@ -1207,81 +854,60 @@ static int vfio_group_get_device_fd(struct vfio_group *group, char *buf) err_put_fdno: put_unused_fd(fdno); err_put_device: - vfio_device_put(device); + vfio_device_put_registration(device); return ret; } -static long vfio_group_fops_unl_ioctl(struct file *filep, - unsigned int cmd, unsigned long arg) +static int vfio_group_ioctl_get_status(struct vfio_group *group, + struct vfio_group_status __user *arg) { - struct vfio_group *group = filep->private_data; - long ret = -ENOTTY; + unsigned long minsz = offsetofend(struct vfio_group_status, flags); + struct vfio_group_status status; - switch (cmd) { - case VFIO_GROUP_GET_STATUS: - { - struct vfio_group_status status; - unsigned long minsz; + if (copy_from_user(&status, arg, minsz)) + return -EFAULT; - minsz = offsetofend(struct vfio_group_status, flags); + if (status.argsz < minsz) + return -EINVAL; - if (copy_from_user(&status, (void __user *)arg, minsz)) - return -EFAULT; + status.flags = 0; - if (status.argsz < minsz) - return -EINVAL; + mutex_lock(&group->group_lock); + if (!group->iommu_group) { + mutex_unlock(&group->group_lock); + return -ENODEV; + } - status.flags = 0; + if (group->container) + status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET | + VFIO_GROUP_FLAGS_VIABLE; + else if (!iommu_group_dma_owner_claimed(group->iommu_group)) + status.flags |= VFIO_GROUP_FLAGS_VIABLE; + mutex_unlock(&group->group_lock); - down_read(&group->group_rwsem); - if (group->container) - status.flags |= VFIO_GROUP_FLAGS_CONTAINER_SET | - VFIO_GROUP_FLAGS_VIABLE; - else if (!iommu_group_dma_owner_claimed(group->iommu_group)) - status.flags |= VFIO_GROUP_FLAGS_VIABLE; - up_read(&group->group_rwsem); + if (copy_to_user(arg, &status, minsz)) + return -EFAULT; + return 0; +} - if (copy_to_user((void __user *)arg, &status, minsz)) - return -EFAULT; +static long vfio_group_fops_unl_ioctl(struct file *filep, + unsigned int cmd, unsigned long arg) +{ + struct vfio_group *group = filep->private_data; + void __user *uarg = (void __user *)arg; - ret = 0; - break; - } + switch (cmd) { + case VFIO_GROUP_GET_DEVICE_FD: + return vfio_group_ioctl_get_device_fd(group, uarg); + case VFIO_GROUP_GET_STATUS: + return vfio_group_ioctl_get_status(group, uarg); case VFIO_GROUP_SET_CONTAINER: - { - int fd; - - if (get_user(fd, (int __user *)arg)) - return -EFAULT; - - if (fd < 0) - return -EINVAL; - - down_write(&group->group_rwsem); - ret = vfio_group_set_container(group, fd); - up_write(&group->group_rwsem); - break; - } + return vfio_group_ioctl_set_container(group, uarg); case VFIO_GROUP_UNSET_CONTAINER: - down_write(&group->group_rwsem); - ret = vfio_group_unset_container(group); - up_write(&group->group_rwsem); - break; - case VFIO_GROUP_GET_DEVICE_FD: - { - char *buf; - - buf = strndup_user((const char __user *)arg, PAGE_SIZE); - if (IS_ERR(buf)) - return PTR_ERR(buf); - - ret = vfio_group_get_device_fd(group, buf); - kfree(buf); - break; - } + return vfio_group_ioctl_unset_container(group); + default: + return -ENOTTY; } - - return ret; } static int vfio_group_fops_open(struct inode *inode, struct file *filep) @@ -1290,17 +916,20 @@ static int vfio_group_fops_open(struct inode *inode, struct file *filep) container_of(inode->i_cdev, struct vfio_group, cdev); int ret; - down_write(&group->group_rwsem); + mutex_lock(&group->group_lock); - /* users can be zero if this races with vfio_group_put() */ - if (!refcount_inc_not_zero(&group->users)) { + /* + * drivers can be zero if this races with vfio_device_remove_group(), it + * will be stable at 0 under the group rwsem + */ + if (refcount_read(&group->drivers) == 0) { ret = -ENODEV; - goto err_unlock; + goto out_unlock; } if (group->type == VFIO_NO_IOMMU && !capable(CAP_SYS_RAWIO)) { ret = -EPERM; - goto err_put; + goto out_unlock; } /* @@ -1308,17 +937,13 @@ static int vfio_group_fops_open(struct inode *inode, struct file *filep) */ if (group->opened_file) { ret = -EBUSY; - goto err_put; + goto out_unlock; } group->opened_file = filep; filep->private_data = group; - - up_write(&group->group_rwsem); - return 0; -err_put: - vfio_group_put(group); -err_unlock: - up_write(&group->group_rwsem); + ret = 0; +out_unlock: + mutex_unlock(&group->group_lock); return ret; } @@ -1328,21 +953,16 @@ static int vfio_group_fops_release(struct inode *inode, struct file *filep) filep->private_data = NULL; - down_write(&group->group_rwsem); + mutex_lock(&group->group_lock); /* * Device FDs hold a group file reference, therefore the group release * is only called when there are no open devices. */ WARN_ON(group->notifier.head); - if (group->container) { - WARN_ON(group->container_users != 1); - __vfio_group_unset_container(group); - } + if (group->container) + vfio_group_detach_container(group); group->opened_file = NULL; - up_write(&group->group_rwsem); - - vfio_group_put(group); - + mutex_unlock(&group->group_lock); return 0; } @@ -1354,25 +974,54 @@ static const struct file_operations vfio_group_fops = { .release = vfio_group_fops_release, }; +/* + * Wrapper around pm_runtime_resume_and_get(). + * Return error code on failure or 0 on success. + */ +static inline int vfio_device_pm_runtime_get(struct vfio_device *device) +{ + struct device *dev = device->dev; + + if (dev->driver && dev->driver->pm) { + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret) { + dev_info_ratelimited(dev, + "vfio: runtime resume failed %d\n", ret); + return -EIO; + } + } + + return 0; +} + +/* + * Wrapper around pm_runtime_put(). + */ +static inline void vfio_device_pm_runtime_put(struct vfio_device *device) +{ + struct device *dev = device->dev; + + if (dev->driver && dev->driver->pm) + pm_runtime_put(dev); +} + /* * VFIO Device fd */ static int vfio_device_fops_release(struct inode *inode, struct file *filep) { struct vfio_device *device = filep->private_data; - struct vfio_iommu_driver *iommu_driver; mutex_lock(&device->dev_set->lock); vfio_assert_device_open(device); - down_read(&device->group->group_rwsem); + mutex_lock(&device->group->group_lock); if (device->open_count == 1 && device->ops->close_device) device->ops->close_device(device); - iommu_driver = device->group->container->iommu_driver; - if (iommu_driver && iommu_driver->ops->unregister_device) - iommu_driver->ops->unregister_device( - device->group->container->iommu_data, device); - up_read(&device->group->group_rwsem); + vfio_device_container_unregister(device); + mutex_unlock(&device->group->group_lock); device->open_count--; if (device->open_count == 0) device->kvm = NULL; @@ -1382,7 +1031,7 @@ static int vfio_device_fops_release(struct inode *inode, struct file *filep) vfio_device_unassign_container(device); - vfio_device_put(device); + vfio_device_put_registration(device); return 0; } @@ -1628,6 +1277,167 @@ static int vfio_ioctl_device_feature_migration(struct vfio_device *device, return 0; } +/* Ranges should fit into a single kernel page */ +#define LOG_MAX_RANGES \ + (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range)) + +static int +vfio_ioctl_device_feature_logging_start(struct vfio_device *device, + u32 flags, void __user *arg, + size_t argsz) +{ + size_t minsz = + offsetofend(struct vfio_device_feature_dma_logging_control, + ranges); + struct vfio_device_feature_dma_logging_range __user *ranges; + struct vfio_device_feature_dma_logging_control control; + struct vfio_device_feature_dma_logging_range range; + struct rb_root_cached root = RB_ROOT_CACHED; + struct interval_tree_node *nodes; + u64 iova_end; + u32 nnodes; + int i, ret; + + if (!device->log_ops) + return -ENOTTY; + + ret = vfio_check_feature(flags, argsz, + VFIO_DEVICE_FEATURE_SET, + sizeof(control)); + if (ret != 1) + return ret; + + if (copy_from_user(&control, arg, minsz)) + return -EFAULT; + + nnodes = control.num_ranges; + if (!nnodes) + return -EINVAL; + + if (nnodes > LOG_MAX_RANGES) + return -E2BIG; + + ranges = u64_to_user_ptr(control.ranges); + nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node), + GFP_KERNEL); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nnodes; i++) { + if (copy_from_user(&range, &ranges[i], sizeof(range))) { + ret = -EFAULT; + goto end; + } + if (!IS_ALIGNED(range.iova, control.page_size) || + !IS_ALIGNED(range.length, control.page_size)) { + ret = -EINVAL; + goto end; + } + + if (check_add_overflow(range.iova, range.length, &iova_end) || + iova_end > ULONG_MAX) { + ret = -EOVERFLOW; + goto end; + } + + nodes[i].start = range.iova; + nodes[i].last = range.iova + range.length - 1; + if (interval_tree_iter_first(&root, nodes[i].start, + nodes[i].last)) { + /* Range overlapping */ + ret = -EINVAL; + goto end; + } + interval_tree_insert(nodes + i, &root); + } + + ret = device->log_ops->log_start(device, &root, nnodes, + &control.page_size); + if (ret) + goto end; + + if (copy_to_user(arg, &control, sizeof(control))) { + ret = -EFAULT; + device->log_ops->log_stop(device); + } + +end: + kfree(nodes); + return ret; +} + +static int +vfio_ioctl_device_feature_logging_stop(struct vfio_device *device, + u32 flags, void __user *arg, + size_t argsz) +{ + int ret; + + if (!device->log_ops) + return -ENOTTY; + + ret = vfio_check_feature(flags, argsz, + VFIO_DEVICE_FEATURE_SET, 0); + if (ret != 1) + return ret; + + return device->log_ops->log_stop(device); +} + +static int vfio_device_log_read_and_clear(struct iova_bitmap *iter, + unsigned long iova, size_t length, + void *opaque) +{ + struct vfio_device *device = opaque; + + return device->log_ops->log_read_and_clear(device, iova, length, iter); +} + +static int +vfio_ioctl_device_feature_logging_report(struct vfio_device *device, + u32 flags, void __user *arg, + size_t argsz) +{ + size_t minsz = + offsetofend(struct vfio_device_feature_dma_logging_report, + bitmap); + struct vfio_device_feature_dma_logging_report report; + struct iova_bitmap *iter; + u64 iova_end; + int ret; + + if (!device->log_ops) + return -ENOTTY; + + ret = vfio_check_feature(flags, argsz, + VFIO_DEVICE_FEATURE_GET, + sizeof(report)); + if (ret != 1) + return ret; + + if (copy_from_user(&report, arg, minsz)) + return -EFAULT; + + if (report.page_size < SZ_4K || !is_power_of_2(report.page_size)) + return -EINVAL; + + if (check_add_overflow(report.iova, report.length, &iova_end) || + iova_end > ULONG_MAX) + return -EOVERFLOW; + + iter = iova_bitmap_alloc(report.iova, report.length, + report.page_size, + u64_to_user_ptr(report.bitmap)); + if (IS_ERR(iter)) + return PTR_ERR(iter); + + ret = iova_bitmap_for_each(iter, device, + vfio_device_log_read_and_clear); + + iova_bitmap_free(iter); + return ret; +} + static int vfio_ioctl_device_feature(struct vfio_device *device, struct vfio_device_feature __user *arg) { @@ -1661,6 +1471,18 @@ static int vfio_ioctl_device_feature(struct vfio_device *device, return vfio_ioctl_device_feature_mig_device_state( device, feature.flags, arg->data, feature.argsz - minsz); + case VFIO_DEVICE_FEATURE_DMA_LOGGING_START: + return vfio_ioctl_device_feature_logging_start( + device, feature.flags, arg->data, + feature.argsz - minsz); + case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP: + return vfio_ioctl_device_feature_logging_stop( + device, feature.flags, arg->data, + feature.argsz - minsz); + case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT: + return vfio_ioctl_device_feature_logging_report( + device, feature.flags, arg->data, + feature.argsz - minsz); default: if (unlikely(!device->ops->device_feature)) return -EINVAL; @@ -1674,15 +1496,27 @@ static long vfio_device_fops_unl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct vfio_device *device = filep->private_data; + int ret; + + ret = vfio_device_pm_runtime_get(device); + if (ret) + return ret; switch (cmd) { case VFIO_DEVICE_FEATURE: - return vfio_ioctl_device_feature(device, (void __user *)arg); + ret = vfio_ioctl_device_feature(device, (void __user *)arg); + break; + default: if (unlikely(!device->ops->ioctl)) - return -EINVAL; - return device->ops->ioctl(device, cmd, arg); + ret = -EINVAL; + else + ret = device->ops->ioctl(device, cmd, arg); + break; } + + vfio_device_pm_runtime_put(device); + return ret; } static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf, @@ -1732,18 +1566,41 @@ static const struct file_operations vfio_device_fops = { * vfio_file_iommu_group - Return the struct iommu_group for the vfio group file * @file: VFIO group file * - * The returned iommu_group is valid as long as a ref is held on the file. + * The returned iommu_group is valid as long as a ref is held on the file. This + * returns a reference on the group. This function is deprecated, only the SPAPR + * path in kvm should call it. */ struct iommu_group *vfio_file_iommu_group(struct file *file) { struct vfio_group *group = file->private_data; + struct iommu_group *iommu_group = NULL; - if (file->f_op != &vfio_group_fops) + if (!IS_ENABLED(CONFIG_SPAPR_TCE_IOMMU)) return NULL; - return group->iommu_group; + + if (!vfio_file_is_group(file)) + return NULL; + + mutex_lock(&group->group_lock); + if (group->iommu_group) { + iommu_group = group->iommu_group; + iommu_group_ref_get(iommu_group); + } + mutex_unlock(&group->group_lock); + return iommu_group; } EXPORT_SYMBOL_GPL(vfio_file_iommu_group); +/** + * vfio_file_is_group - True if the file is usable with VFIO aPIS + * @file: VFIO group file + */ +bool vfio_file_is_group(struct file *file) +{ + return file->f_op == &vfio_group_fops; +} +EXPORT_SYMBOL_GPL(vfio_file_is_group); + /** * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file * is always CPU cache coherent @@ -1758,13 +1615,13 @@ bool vfio_file_enforced_coherent(struct file *file) struct vfio_group *group = file->private_data; bool ret; - if (file->f_op != &vfio_group_fops) + if (!vfio_file_is_group(file)) return true; - down_read(&group->group_rwsem); + mutex_lock(&group->group_lock); if (group->container) { - ret = vfio_ioctl_check_extension(group->container, - VFIO_DMA_CC_IOMMU); + ret = vfio_container_ioctl_check_extension(group->container, + VFIO_DMA_CC_IOMMU); } else { /* * Since the coherency state is determined only once a container @@ -1773,7 +1630,7 @@ bool vfio_file_enforced_coherent(struct file *file) */ ret = true; } - up_read(&group->group_rwsem); + mutex_unlock(&group->group_lock); return ret; } EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); @@ -1790,12 +1647,12 @@ void vfio_file_set_kvm(struct file *file, struct kvm *kvm) { struct vfio_group *group = file->private_data; - if (file->f_op != &vfio_group_fops) + if (!vfio_file_is_group(file)) return; - down_write(&group->group_rwsem); + mutex_lock(&group->group_lock); group->kvm = kvm; - up_write(&group->group_rwsem); + mutex_unlock(&group->group_lock); } EXPORT_SYMBOL_GPL(vfio_file_set_kvm); @@ -1810,7 +1667,7 @@ bool vfio_file_has_dev(struct file *file, struct vfio_device *device) { struct vfio_group *group = file->private_data; - if (file->f_op != &vfio_group_fops) + if (!vfio_file_is_group(file)) return false; return group == device->group; @@ -1936,114 +1793,6 @@ int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs, } EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare); -/* - * Pin contiguous user pages and return their associated host pages for local - * domain only. - * @device [in] : device - * @iova [in] : starting IOVA of user pages to be pinned. - * @npage [in] : count of pages to be pinned. This count should not - * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. - * @prot [in] : protection flags - * @pages[out] : array of host pages - * Return error or number of pages pinned. - */ -int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova, - int npage, int prot, struct page **pages) -{ - struct vfio_container *container; - struct vfio_group *group = device->group; - struct vfio_iommu_driver *driver; - int ret; - - if (!pages || !npage || !vfio_assert_device_open(device)) - return -EINVAL; - - if (npage > VFIO_PIN_PAGES_MAX_ENTRIES) - return -E2BIG; - - if (group->dev_counter > 1) - return -EINVAL; - - /* group->container cannot change while a vfio device is open */ - container = group->container; - driver = container->iommu_driver; - if (likely(driver && driver->ops->pin_pages)) - ret = driver->ops->pin_pages(container->iommu_data, - group->iommu_group, iova, - npage, prot, pages); - else - ret = -ENOTTY; - - return ret; -} -EXPORT_SYMBOL(vfio_pin_pages); - -/* - * Unpin contiguous host pages for local domain only. - * @device [in] : device - * @iova [in] : starting address of user pages to be unpinned. - * @npage [in] : count of pages to be unpinned. This count should not - * be greater than VFIO_PIN_PAGES_MAX_ENTRIES. - */ -void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage) -{ - struct vfio_container *container; - struct vfio_iommu_driver *driver; - - if (WARN_ON(npage <= 0 || npage > VFIO_PIN_PAGES_MAX_ENTRIES)) - return; - - if (WARN_ON(!vfio_assert_device_open(device))) - return; - - /* group->container cannot change while a vfio device is open */ - container = device->group->container; - driver = container->iommu_driver; - - driver->ops->unpin_pages(container->iommu_data, iova, npage); -} -EXPORT_SYMBOL(vfio_unpin_pages); - -/* - * This interface allows the CPUs to perform some sort of virtual DMA on - * behalf of the device. - * - * CPUs read/write from/into a range of IOVAs pointing to user space memory - * into/from a kernel buffer. - * - * As the read/write of user space memory is conducted via the CPUs and is - * not a real device DMA, it is not necessary to pin the user space memory. - * - * @device [in] : VFIO device - * @iova [in] : base IOVA of a user space buffer - * @data [in] : pointer to kernel buffer - * @len [in] : kernel buffer length - * @write : indicate read or write - * Return error code on failure or 0 on success. - */ -int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data, - size_t len, bool write) -{ - struct vfio_container *container; - struct vfio_iommu_driver *driver; - int ret = 0; - - if (!data || len <= 0 || !vfio_assert_device_open(device)) - return -EINVAL; - - /* group->container cannot change while a vfio device is open */ - container = device->group->container; - driver = container->iommu_driver; - - if (likely(driver && driver->ops->dma_rw)) - ret = driver->ops->dma_rw(container->iommu_data, - iova, data, len, write); - else - ret = -ENOTTY; - return ret; -} -EXPORT_SYMBOL(vfio_dma_rw); - /* * Module/class support */ @@ -2052,59 +1801,50 @@ static char *vfio_devnode(struct device *dev, umode_t *mode) return kasprintf(GFP_KERNEL, "vfio/%s", dev_name(dev)); } -static struct miscdevice vfio_dev = { - .minor = VFIO_MINOR, - .name = "vfio", - .fops = &vfio_fops, - .nodename = "vfio/vfio", - .mode = S_IRUGO | S_IWUGO, -}; - static int __init vfio_init(void) { int ret; ida_init(&vfio.group_ida); + ida_init(&vfio.device_ida); mutex_init(&vfio.group_lock); - mutex_init(&vfio.iommu_drivers_lock); INIT_LIST_HEAD(&vfio.group_list); - INIT_LIST_HEAD(&vfio.iommu_drivers_list); - ret = misc_register(&vfio_dev); - if (ret) { - pr_err("vfio: misc device register failed\n"); + ret = vfio_container_init(); + if (ret) return ret; - } /* /dev/vfio/$GROUP */ vfio.class = class_create(THIS_MODULE, "vfio"); if (IS_ERR(vfio.class)) { ret = PTR_ERR(vfio.class); - goto err_class; + goto err_group_class; } vfio.class->devnode = vfio_devnode; + /* /sys/class/vfio-dev/vfioX */ + vfio.device_class = class_create(THIS_MODULE, "vfio-dev"); + if (IS_ERR(vfio.device_class)) { + ret = PTR_ERR(vfio.device_class); + goto err_dev_class; + } + ret = alloc_chrdev_region(&vfio.group_devt, 0, MINORMASK + 1, "vfio"); if (ret) goto err_alloc_chrdev; -#ifdef CONFIG_VFIO_NOIOMMU - ret = vfio_register_iommu_driver(&vfio_noiommu_ops); -#endif - if (ret) - goto err_driver_register; - pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n"); return 0; -err_driver_register: - unregister_chrdev_region(vfio.group_devt, MINORMASK + 1); err_alloc_chrdev: + class_destroy(vfio.device_class); + vfio.device_class = NULL; +err_dev_class: class_destroy(vfio.class); vfio.class = NULL; -err_class: - misc_deregister(&vfio_dev); +err_group_class: + vfio_container_cleanup(); return ret; } @@ -2112,14 +1852,14 @@ static void __exit vfio_cleanup(void) { WARN_ON(!list_empty(&vfio.group_list)); -#ifdef CONFIG_VFIO_NOIOMMU - vfio_unregister_iommu_driver(&vfio_noiommu_ops); -#endif + ida_destroy(&vfio.device_ida); ida_destroy(&vfio.group_ida); unregister_chrdev_region(vfio.group_devt, MINORMASK + 1); + class_destroy(vfio.device_class); + vfio.device_class = NULL; class_destroy(vfio.class); + vfio_container_cleanup(); vfio.class = NULL; - misc_deregister(&vfio_dev); xa_destroy(&vfio_device_set_xa); } diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 68e4ecd1cc0e844957814c2f98d5245ba20bd177..20265393aee7cff9cd6560afa437925395b460e8 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -118,7 +118,7 @@ struct vhost_net_virtqueue { /* Number of XDP frames batched */ int batched_xdp; /* an array of userspace buffers info */ - struct ubuf_info *ubuf_info; + struct ubuf_info_msgzc *ubuf_info; /* Reference counting for outstanding ubufs. * Protected by vq mutex. Writers must also take device mutex. */ struct vhost_net_ubuf_ref *ubufs; @@ -382,8 +382,9 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net, } static void vhost_zerocopy_callback(struct sk_buff *skb, - struct ubuf_info *ubuf, bool success) + struct ubuf_info *ubuf_base, bool success) { + struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base); struct vhost_net_ubuf_ref *ubufs = ubuf->ctx; struct vhost_virtqueue *vq = ubufs->vq; int cnt; @@ -871,7 +872,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) size_t len, total_len = 0; int err; struct vhost_net_ubuf_ref *ubufs; - struct ubuf_info *ubuf; + struct ubuf_info_msgzc *ubuf; bool zcopy_used; int sent_pkts = 0; @@ -907,14 +908,14 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock) ubuf = nvq->ubuf_info + nvq->upend_idx; vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head); vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS; - ubuf->callback = vhost_zerocopy_callback; ubuf->ctx = nvq->ubufs; ubuf->desc = nvq->upend_idx; - ubuf->flags = SKBFL_ZEROCOPY_FRAG; - refcount_set(&ubuf->refcnt, 1); + ubuf->ubuf.callback = vhost_zerocopy_callback; + ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG; + refcount_set(&ubuf->ubuf.refcnt, 1); msg.msg_control = &ctl; ctl.type = TUN_MSG_UBUF; - ctl.ptr = ubuf; + ctl.ptr = &ubuf->ubuf; msg.msg_controllen = sizeof(ctl); ubufs = nvq->ubufs; atomic_inc(&ubufs->refcount); @@ -1781,7 +1782,7 @@ static struct miscdevice vhost_net_misc = { .fops = &vhost_net_fops, }; -static int vhost_net_init(void) +static int __init vhost_net_init(void) { if (experimental_zcopytx) vhost_net_enable_zcopy(VHOST_NET_VQ_TX); @@ -1789,7 +1790,7 @@ static int vhost_net_init(void) } module_init(vhost_net_init); -static void vhost_net_exit(void) +static void __exit vhost_net_exit(void) { misc_deregister(&vhost_net_misc); } diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index 368330417bde2910174c81c56edf4297f2c7c0fb..5703775af12976ab6d5a7286d8d28811936a7295 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -393,7 +393,7 @@ vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq, return NULL; } - pkt->buf = kmalloc(pkt->len, GFP_KERNEL); + pkt->buf = kvmalloc(pkt->len, GFP_KERNEL); if (!pkt->buf) { kfree(pkt); return NULL; diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig index a003e02e13ce3e1355e56ed416fd739306497219..936ba1e4d35ec5f207f44305d779c18db7247f64 100644 --- a/drivers/video/backlight/Kconfig +++ b/drivers/video/backlight/Kconfig @@ -268,6 +268,19 @@ config BACKLIGHT_MAX8925 If you have a LCD backlight connected to the WLED output of MAX8925 WLED output, say Y here to enable this driver. +config BACKLIGHT_MT6370 + tristate "MediaTek MT6370 Backlight Driver" + depends on MFD_MT6370 + help + This enables support for Mediatek MT6370 Backlight driver. + It's commonly used to drive the display WLED. There are 4 channels + inside, and each channel supports up to 30mA of current capability + with 2048 current steps (only for MT6370/MT6371) or 16384 current + steps (only for MT6372) in exponential or linear mapping curves. + + This driver can also be built as a module. If so, the module + will be called "mt6370-backlight". + config BACKLIGHT_APPLE tristate "Apple Backlight Driver" depends on X86 && ACPI diff --git a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile index cae2c83422ae3dd786d7a69867217b9d1de46f29..e815f3f1deff4e464331d777aeeb8435f9a69f0f 100644 --- a/drivers/video/backlight/Makefile +++ b/drivers/video/backlight/Makefile @@ -44,6 +44,7 @@ obj-$(CONFIG_BACKLIGHT_LP855X) += lp855x_bl.o obj-$(CONFIG_BACKLIGHT_LP8788) += lp8788_bl.o obj-$(CONFIG_BACKLIGHT_LV5207LP) += lv5207lp.o obj-$(CONFIG_BACKLIGHT_MAX8925) += max8925_bl.o +obj-$(CONFIG_BACKLIGHT_MT6370) += mt6370-backlight.o obj-$(CONFIG_BACKLIGHT_OMAP1) += omap1_bl.o obj-$(CONFIG_BACKLIGHT_PANDORA) += pandora_bl.o obj-$(CONFIG_BACKLIGHT_PCF50633) += pcf50633-backlight.o diff --git a/drivers/video/backlight/adp8860_bl.c b/drivers/video/backlight/adp8860_bl.c index 8ec19425671fb9d7ae941030e86b6ea3d036df53..b0fe02273e875ce2e65fc4f6ce4d923f130e46ee 100644 --- a/drivers/video/backlight/adp8860_bl.c +++ b/drivers/video/backlight/adp8860_bl.c @@ -753,7 +753,7 @@ out: return ret; } -static int adp8860_remove(struct i2c_client *client) +static void adp8860_remove(struct i2c_client *client) { struct adp8860_bl *data = i2c_get_clientdata(client); @@ -765,8 +765,6 @@ static int adp8860_remove(struct i2c_client *client) if (data->en_ambl_sens) sysfs_remove_group(&data->bl->dev.kobj, &adp8860_bl_attr_group); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/video/backlight/adp8870_bl.c b/drivers/video/backlight/adp8870_bl.c index 8b5213a39527968edad9bb984e9ef025d6edd806..5becace3fd0f569d94be2c13269567ee5b75b8dd 100644 --- a/drivers/video/backlight/adp8870_bl.c +++ b/drivers/video/backlight/adp8870_bl.c @@ -925,7 +925,7 @@ out: return ret; } -static int adp8870_remove(struct i2c_client *client) +static void adp8870_remove(struct i2c_client *client) { struct adp8870_bl *data = i2c_get_clientdata(client); @@ -937,8 +937,6 @@ static int adp8870_remove(struct i2c_client *client) if (data->pdata->en_ambl_sens) sysfs_remove_group(&data->bl->dev.kobj, &adp8870_bl_attr_group); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/video/backlight/arcxcnn_bl.c b/drivers/video/backlight/arcxcnn_bl.c index 7b1c0a0e6cad23be1dc32fe57a564e360ffcbe49..060c0eef6a52d98335ea6903c4d7b0c2eb4cb129 100644 --- a/drivers/video/backlight/arcxcnn_bl.c +++ b/drivers/video/backlight/arcxcnn_bl.c @@ -362,7 +362,7 @@ probe_err: return ret; } -static int arcxcnn_remove(struct i2c_client *cl) +static void arcxcnn_remove(struct i2c_client *cl) { struct arcxcnn *lp = i2c_get_clientdata(cl); @@ -376,8 +376,6 @@ static int arcxcnn_remove(struct i2c_client *cl) lp->bl->props.brightness = 0; backlight_update_status(lp->bl); - - return 0; } static const struct of_device_id arcxcnn_dt_ids[] = { diff --git a/drivers/video/backlight/bd6107.c b/drivers/video/backlight/bd6107.c index 515184fbe33a938ee0ae9645e0a9240de0a9563f..a506872d43963b0a17562aff7afe3991f87af72f 100644 --- a/drivers/video/backlight/bd6107.c +++ b/drivers/video/backlight/bd6107.c @@ -175,14 +175,12 @@ static int bd6107_probe(struct i2c_client *client, return 0; } -static int bd6107_remove(struct i2c_client *client) +static void bd6107_remove(struct i2c_client *client) { struct backlight_device *backlight = i2c_get_clientdata(client); backlight->props.brightness = 0; backlight_update_status(backlight); - - return 0; } static const struct i2c_device_id bd6107_ids[] = { diff --git a/drivers/video/backlight/lm3630a_bl.c b/drivers/video/backlight/lm3630a_bl.c index 1d17c439430ef79443b43997abfc11ca88ed5a6e..475f35635bf671e81f181419ed62a0384ba1d89e 100644 --- a/drivers/video/backlight/lm3630a_bl.c +++ b/drivers/video/backlight/lm3630a_bl.c @@ -579,7 +579,7 @@ static int lm3630a_probe(struct i2c_client *client, return 0; } -static int lm3630a_remove(struct i2c_client *client) +static void lm3630a_remove(struct i2c_client *client) { int rval; struct lm3630a_chip *pchip = i2c_get_clientdata(client); @@ -596,7 +596,6 @@ static int lm3630a_remove(struct i2c_client *client) free_irq(pchip->irq, pchip); destroy_workqueue(pchip->irqthread); } - return 0; } static const struct i2c_device_id lm3630a_id[] = { diff --git a/drivers/video/backlight/lm3639_bl.c b/drivers/video/backlight/lm3639_bl.c index 48c04155a5f9db7688391777b914a18f5323818e..6580911671a3e774310b2a84d596037693f5092d 100644 --- a/drivers/video/backlight/lm3639_bl.c +++ b/drivers/video/backlight/lm3639_bl.c @@ -390,7 +390,7 @@ err_out: return ret; } -static int lm3639_remove(struct i2c_client *client) +static void lm3639_remove(struct i2c_client *client) { struct lm3639_chip_data *pchip = i2c_get_clientdata(client); @@ -400,7 +400,6 @@ static int lm3639_remove(struct i2c_client *client) led_classdev_unregister(&pchip->cdev_flash); if (pchip->bled) device_remove_file(&(pchip->bled->dev), &dev_attr_bled_mode); - return 0; } static const struct i2c_device_id lm3639_id[] = { diff --git a/drivers/video/backlight/lp855x_bl.c b/drivers/video/backlight/lp855x_bl.c index fc02c5c16055ee4f23e4f213c78070153186ac21..bd0bdeae23a4f569c4e6932f87471cdb7c583fc1 100644 --- a/drivers/video/backlight/lp855x_bl.c +++ b/drivers/video/backlight/lp855x_bl.c @@ -534,7 +534,7 @@ disable_supply: return ret; } -static int lp855x_remove(struct i2c_client *cl) +static void lp855x_remove(struct i2c_client *cl) { struct lp855x *lp = i2c_get_clientdata(cl); @@ -545,8 +545,6 @@ static int lp855x_remove(struct i2c_client *cl) if (lp->supply) regulator_disable(lp->supply); sysfs_remove_group(&lp->dev->kobj, &lp855x_attr_group); - - return 0; } static const struct of_device_id lp855x_dt_ids[] = { diff --git a/drivers/video/backlight/lv5207lp.c b/drivers/video/backlight/lv5207lp.c index 1842ae9a55f8b68fb0719796a71cd08dccec23c8..767b800d79fafbc907c0d34c437a1e3e4918570a 100644 --- a/drivers/video/backlight/lv5207lp.c +++ b/drivers/video/backlight/lv5207lp.c @@ -124,14 +124,12 @@ static int lv5207lp_probe(struct i2c_client *client, return 0; } -static int lv5207lp_remove(struct i2c_client *client) +static void lv5207lp_remove(struct i2c_client *client) { struct backlight_device *backlight = i2c_get_clientdata(client); backlight->props.brightness = 0; backlight_update_status(backlight); - - return 0; } static const struct i2c_device_id lv5207lp_ids[] = { diff --git a/drivers/video/backlight/mt6370-backlight.c b/drivers/video/backlight/mt6370-backlight.c new file mode 100644 index 0000000000000000000000000000000000000000..623d4f2baca2132cbe9c6fcbb8952495da24d873 --- /dev/null +++ b/drivers/video/backlight/mt6370-backlight.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Richtek Technology Corp. + * + * Author: ChiaEn Wu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MT6370_REG_DEV_INFO 0x100 +#define MT6370_REG_BL_EN 0x1A0 +#define MT6370_REG_BL_BSTCTRL 0x1A1 +#define MT6370_REG_BL_PWM 0x1A2 +#define MT6370_REG_BL_DIM2 0x1A4 + +#define MT6370_VENID_MASK GENMASK(7, 4) +#define MT6370_BL_EXT_EN_MASK BIT(7) +#define MT6370_BL_EN_MASK BIT(6) +#define MT6370_BL_CODE_MASK BIT(0) +#define MT6370_BL_CH_MASK GENMASK(5, 2) +#define MT6370_BL_CH_SHIFT 2 +#define MT6370_BL_DIM2_COMMON_MASK GENMASK(2, 0) +#define MT6370_BL_DIM2_COMMON_SHIFT 3 +#define MT6370_BL_DIM2_6372_MASK GENMASK(5, 0) +#define MT6370_BL_DIM2_6372_SHIFT 6 +#define MT6370_BL_PWM_EN_MASK BIT(7) +#define MT6370_BL_PWM_HYS_EN_MASK BIT(2) +#define MT6370_BL_PWM_HYS_SEL_MASK GENMASK(1, 0) +#define MT6370_BL_OVP_EN_MASK BIT(7) +#define MT6370_BL_OVP_SEL_MASK GENMASK(6, 5) +#define MT6370_BL_OVP_SEL_SHIFT 5 +#define MT6370_BL_OC_EN_MASK BIT(3) +#define MT6370_BL_OC_SEL_MASK GENMASK(2, 1) +#define MT6370_BL_OC_SEL_SHIFT 1 + +#define MT6370_BL_PWM_HYS_TH_MIN_STEP 1 +#define MT6370_BL_PWM_HYS_TH_MAX_STEP 64 +#define MT6370_BL_OVP_MIN_UV 17000000 +#define MT6370_BL_OVP_MAX_UV 29000000 +#define MT6370_BL_OVP_STEP_UV 4000000 +#define MT6370_BL_OCP_MIN_UA 900000 +#define MT6370_BL_OCP_MAX_UA 1800000 +#define MT6370_BL_OCP_STEP_UA 300000 +#define MT6370_BL_MAX_COMMON_BRIGHTNESS 2048 +#define MT6370_BL_MAX_6372_BRIGHTNESS 16384 +#define MT6370_BL_MAX_CH 15 + +enum { + MT6370_VID_COMMON = 1, + MT6370_VID_6372, +}; + +struct mt6370_priv { + u8 dim2_mask; + u8 dim2_shift; + int def_max_brightness; + struct backlight_device *bl; + struct device *dev; + struct gpio_desc *enable_gpio; + struct regmap *regmap; +}; + +static int mt6370_bl_update_status(struct backlight_device *bl_dev) +{ + struct mt6370_priv *priv = bl_get_data(bl_dev); + int brightness = backlight_get_brightness(bl_dev); + unsigned int enable_val; + u8 brightness_val[2]; + int ret; + + if (brightness) { + brightness_val[0] = (brightness - 1) & priv->dim2_mask; + brightness_val[1] = (brightness - 1) >> priv->dim2_shift; + + ret = regmap_raw_write(priv->regmap, MT6370_REG_BL_DIM2, + brightness_val, sizeof(brightness_val)); + if (ret) + return ret; + } + + gpiod_set_value(priv->enable_gpio, !!brightness); + + enable_val = brightness ? MT6370_BL_EN_MASK : 0; + return regmap_update_bits(priv->regmap, MT6370_REG_BL_EN, + MT6370_BL_EN_MASK, enable_val); +} + +static int mt6370_bl_get_brightness(struct backlight_device *bl_dev) +{ + struct mt6370_priv *priv = bl_get_data(bl_dev); + unsigned int enable; + u8 brightness_val[2]; + int brightness, ret; + + ret = regmap_read(priv->regmap, MT6370_REG_BL_EN, &enable); + if (ret) + return ret; + + if (!(enable & MT6370_BL_EN_MASK)) + return 0; + + ret = regmap_raw_read(priv->regmap, MT6370_REG_BL_DIM2, + brightness_val, sizeof(brightness_val)); + if (ret) + return ret; + + brightness = brightness_val[1] << priv->dim2_shift; + brightness += brightness_val[0] & priv->dim2_mask; + + return brightness + 1; +} + +static const struct backlight_ops mt6370_bl_ops = { + .options = BL_CORE_SUSPENDRESUME, + .update_status = mt6370_bl_update_status, + .get_brightness = mt6370_bl_get_brightness, +}; + +static int mt6370_init_backlight_properties(struct mt6370_priv *priv, + struct backlight_properties *props) +{ + struct device *dev = priv->dev; + u8 prop_val; + u32 brightness, ovp_uV, ocp_uA; + unsigned int mask, val; + int ret; + + /* Vendor optional properties */ + val = 0; + if (device_property_read_bool(dev, "mediatek,bled-pwm-enable")) + val |= MT6370_BL_PWM_EN_MASK; + + if (device_property_read_bool(dev, "mediatek,bled-pwm-hys-enable")) + val |= MT6370_BL_PWM_HYS_EN_MASK; + + ret = device_property_read_u8(dev, + "mediatek,bled-pwm-hys-input-th-steps", + &prop_val); + if (!ret) { + prop_val = clamp_val(prop_val, + MT6370_BL_PWM_HYS_TH_MIN_STEP, + MT6370_BL_PWM_HYS_TH_MAX_STEP); + prop_val = prop_val <= 1 ? 0 : + prop_val <= 4 ? 1 : + prop_val <= 16 ? 2 : 3; + val |= prop_val; + } + + ret = regmap_update_bits(priv->regmap, MT6370_REG_BL_PWM, + val, val); + if (ret) + return ret; + + val = 0; + if (device_property_read_bool(dev, "mediatek,bled-ovp-shutdown")) + val |= MT6370_BL_OVP_EN_MASK; + + ret = device_property_read_u32(dev, "mediatek,bled-ovp-microvolt", + &ovp_uV); + if (!ret) { + ovp_uV = clamp_val(ovp_uV, MT6370_BL_OVP_MIN_UV, + MT6370_BL_OVP_MAX_UV); + ovp_uV = DIV_ROUND_UP(ovp_uV - MT6370_BL_OVP_MIN_UV, + MT6370_BL_OVP_STEP_UV); + val |= ovp_uV << MT6370_BL_OVP_SEL_SHIFT; + } + + if (device_property_read_bool(dev, "mediatek,bled-ocp-shutdown")) + val |= MT6370_BL_OC_EN_MASK; + + ret = device_property_read_u32(dev, "mediatek,bled-ocp-microamp", + &ocp_uA); + if (!ret) { + ocp_uA = clamp_val(ocp_uA, MT6370_BL_OCP_MIN_UA, + MT6370_BL_OCP_MAX_UA); + ocp_uA = DIV_ROUND_UP(ocp_uA - MT6370_BL_OCP_MIN_UA, + MT6370_BL_OCP_STEP_UA); + val |= ocp_uA << MT6370_BL_OC_SEL_SHIFT; + } + + ret = regmap_update_bits(priv->regmap, MT6370_REG_BL_BSTCTRL, + val, val); + if (ret) + return ret; + + /* Common properties */ + ret = device_property_read_u32(dev, "max-brightness", &brightness); + if (ret) + brightness = priv->def_max_brightness; + + props->max_brightness = min_t(u32, brightness, priv->def_max_brightness); + + ret = device_property_read_u32(dev, "default-brightness", &brightness); + if (ret) + brightness = props->max_brightness; + + props->brightness = min_t(u32, brightness, props->max_brightness); + + val = 0; + if (device_property_read_bool(dev, "mediatek,bled-exponential-mode-enable")) { + val |= MT6370_BL_CODE_MASK; + props->scale = BACKLIGHT_SCALE_NON_LINEAR; + } else + props->scale = BACKLIGHT_SCALE_LINEAR; + + ret = device_property_read_u8(dev, "mediatek,bled-channel-use", + &prop_val); + if (ret) { + dev_err(dev, "mediatek,bled-channel-use DT property missing\n"); + return ret; + } + + if (!prop_val || prop_val > MT6370_BL_MAX_CH) { + dev_err(dev, + "No channel specified or over than upper bound (%d)\n", + prop_val); + return -EINVAL; + } + + mask = MT6370_BL_EXT_EN_MASK | MT6370_BL_CH_MASK; + val |= prop_val << MT6370_BL_CH_SHIFT; + + if (priv->enable_gpio) + val |= MT6370_BL_EXT_EN_MASK; + + return regmap_update_bits(priv->regmap, MT6370_REG_BL_EN, mask, val); +} + +static int mt6370_check_vendor_info(struct mt6370_priv *priv) +{ + /* + * Because MT6372 uses 14 bits to control the brightness, + * MT6370 and MT6371 use 11 bits. This function is used + * to check the vendor's ID and set the relative hardware + * mask, shift and default maximum brightness value that + * should be used. + */ + unsigned int dev_info, hw_vid, of_vid; + int ret; + + ret = regmap_read(priv->regmap, MT6370_REG_DEV_INFO, &dev_info); + if (ret) + return ret; + + of_vid = (uintptr_t)device_get_match_data(priv->dev); + hw_vid = FIELD_GET(MT6370_VENID_MASK, dev_info); + hw_vid = (hw_vid == 0x9 || hw_vid == 0xb) ? MT6370_VID_6372 : MT6370_VID_COMMON; + if (hw_vid != of_vid) + return dev_err_probe(priv->dev, -EINVAL, + "Buggy DT, wrong compatible string\n"); + + if (hw_vid == MT6370_VID_6372) { + priv->dim2_mask = MT6370_BL_DIM2_6372_MASK; + priv->dim2_shift = MT6370_BL_DIM2_6372_SHIFT; + priv->def_max_brightness = MT6370_BL_MAX_6372_BRIGHTNESS; + } else { + priv->dim2_mask = MT6370_BL_DIM2_COMMON_MASK; + priv->dim2_shift = MT6370_BL_DIM2_COMMON_SHIFT; + priv->def_max_brightness = MT6370_BL_MAX_COMMON_BRIGHTNESS; + } + + return 0; +} + +static int mt6370_bl_probe(struct platform_device *pdev) +{ + struct backlight_properties props = { + .type = BACKLIGHT_RAW, + }; + struct device *dev = &pdev->dev; + struct mt6370_priv *priv; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + + priv->regmap = dev_get_regmap(dev->parent, NULL); + if (!priv->regmap) + return dev_err_probe(dev, -ENODEV, "Failed to get regmap\n"); + + ret = mt6370_check_vendor_info(priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to check vendor info\n"); + + priv->enable_gpio = devm_gpiod_get_optional(dev, "enable", + GPIOD_OUT_HIGH); + if (IS_ERR(priv->enable_gpio)) + return dev_err_probe(dev, PTR_ERR(priv->enable_gpio), + "Failed to get 'enable' gpio\n"); + + ret = mt6370_init_backlight_properties(priv, &props); + if (ret) + return dev_err_probe(dev, ret, + "Failed to init backlight properties\n"); + + priv->bl = devm_backlight_device_register(dev, pdev->name, dev, priv, + &mt6370_bl_ops, &props); + if (IS_ERR(priv->bl)) + return dev_err_probe(dev, PTR_ERR(priv->bl), + "Failed to register backlight\n"); + + backlight_update_status(priv->bl); + platform_set_drvdata(pdev, priv); + + return 0; +} + +static int mt6370_bl_remove(struct platform_device *pdev) +{ + struct mt6370_priv *priv = platform_get_drvdata(pdev); + struct backlight_device *bl_dev = priv->bl; + + bl_dev->props.brightness = 0; + backlight_update_status(priv->bl); + + return 0; +} + +static const struct of_device_id mt6370_bl_of_match[] = { + { .compatible = "mediatek,mt6370-backlight", .data = (void *)MT6370_VID_COMMON }, + { .compatible = "mediatek,mt6372-backlight", .data = (void *)MT6370_VID_6372 }, + {} +}; +MODULE_DEVICE_TABLE(of, mt6370_bl_of_match); + +static struct platform_driver mt6370_bl_driver = { + .driver = { + .name = "mt6370-backlight", + .of_match_table = mt6370_bl_of_match, + }, + .probe = mt6370_bl_probe, + .remove = mt6370_bl_remove, +}; +module_platform_driver(mt6370_bl_driver); + +MODULE_AUTHOR("ChiaEn Wu "); +MODULE_DESCRIPTION("MediaTek MT6370 Backlight Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/video/backlight/tosa_bl.c b/drivers/video/backlight/tosa_bl.c index 6df6fcd132e36794094c117fe496e9da32b0ae7b..f55b3d616a8716333dd92fd03a4fe735ac216e87 100644 --- a/drivers/video/backlight/tosa_bl.c +++ b/drivers/video/backlight/tosa_bl.c @@ -121,12 +121,11 @@ err_reg: return ret; } -static int tosa_bl_remove(struct i2c_client *client) +static void tosa_bl_remove(struct i2c_client *client) { struct tosa_bl_data *data = i2c_get_clientdata(client); data->bl = NULL; - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/video/console/sticore.c b/drivers/video/console/sticore.c index bd4dc97d4d340245bc60acb4b33f0d85f3355418..db568f67e4dc2b191ad58e1b4296deb0971c4f60 100644 --- a/drivers/video/console/sticore.c +++ b/drivers/video/console/sticore.c @@ -290,7 +290,7 @@ static char default_sti_path[21] __read_mostly; static int __init sti_setup(char *str) { if (str) - strlcpy (default_sti_path, str, sizeof (default_sti_path)); + strscpy(default_sti_path, str, sizeof(default_sti_path)); return 1; } diff --git a/drivers/video/fbdev/arkfb.c b/drivers/video/fbdev/arkfb.c index a317d9fe1d67de4864af27bc928126335e547e68..5f8fec9e5fd4da339d3788ad7a2fd22a4e74f6bb 100644 --- a/drivers/video/fbdev/arkfb.c +++ b/drivers/video/fbdev/arkfb.c @@ -318,14 +318,6 @@ struct dac_info void *data; }; - -static inline u8 dac_read_reg(struct dac_info *info, u8 reg) -{ - u8 code[2] = {reg, 0}; - info->dac_read_regs(info->data, code, 1); - return code[1]; -} - static inline void dac_read_regs(struct dac_info *info, u8 *code, int count) { info->dac_read_regs(info->data, code, count); diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c index 4804b6e9f3f4873c4dd2dd1b9d892e95aad64964..b3463d137152015aacaa94032d3eab5207589e9f 100644 --- a/drivers/video/fbdev/aty/atyfb_base.c +++ b/drivers/video/fbdev/aty/atyfb_base.c @@ -3896,7 +3896,7 @@ static int __init atyfb_setup(char *options) && (!strncmp(this_opt, "Mach64:", 7))) { static unsigned char m64_num; static char mach64_str[80]; - strlcpy(mach64_str, this_opt + 7, sizeof(mach64_str)); + strscpy(mach64_str, this_opt + 7, sizeof(mach64_str)); if (!store_video_par(mach64_str, m64_num)) { m64_num++; mach64_count = m64_num; diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c index 0a8199985d52faaf1b452934e7b3593551eaa128..8b28c9bddd974301597565bbd977ce1ef7280d54 100644 --- a/drivers/video/fbdev/aty/radeon_base.c +++ b/drivers/video/fbdev/aty/radeon_base.c @@ -1981,7 +1981,7 @@ static int radeon_set_fbinfo(struct radeonfb_info *rinfo) info->screen_base = rinfo->fb_base; info->screen_size = rinfo->mapped_vram; /* Fill fix common fields */ - strlcpy(info->fix.id, rinfo->name, sizeof(info->fix.id)); + strscpy(info->fix.id, rinfo->name, sizeof(info->fix.id)); info->fix.smem_start = rinfo->fb_base_phys; info->fix.smem_len = rinfo->video_ram; info->fix.type = FB_TYPE_PACKED_PIXELS; @@ -2095,34 +2095,34 @@ static void radeon_identify_vram(struct radeonfb_info *rinfo) u32 tmp; /* framebuffer size */ - if ((rinfo->family == CHIP_FAMILY_RS100) || + if ((rinfo->family == CHIP_FAMILY_RS100) || (rinfo->family == CHIP_FAMILY_RS200) || (rinfo->family == CHIP_FAMILY_RS300) || (rinfo->family == CHIP_FAMILY_RC410) || (rinfo->family == CHIP_FAMILY_RS400) || (rinfo->family == CHIP_FAMILY_RS480) ) { - u32 tom = INREG(NB_TOM); - tmp = ((((tom >> 16) - (tom & 0xffff) + 1) << 6) * 1024); - - radeon_fifo_wait(6); - OUTREG(MC_FB_LOCATION, tom); - OUTREG(DISPLAY_BASE_ADDR, (tom & 0xffff) << 16); - OUTREG(CRTC2_DISPLAY_BASE_ADDR, (tom & 0xffff) << 16); - OUTREG(OV0_BASE_ADDR, (tom & 0xffff) << 16); - - /* This is supposed to fix the crtc2 noise problem. */ - OUTREG(GRPH2_BUFFER_CNTL, INREG(GRPH2_BUFFER_CNTL) & ~0x7f0000); - - if ((rinfo->family == CHIP_FAMILY_RS100) || - (rinfo->family == CHIP_FAMILY_RS200)) { - /* This is to workaround the asic bug for RMX, some versions - of BIOS doesn't have this register initialized correctly. - */ - OUTREGP(CRTC_MORE_CNTL, CRTC_H_CUTOFF_ACTIVE_EN, - ~CRTC_H_CUTOFF_ACTIVE_EN); - } - } else { - tmp = INREG(CNFG_MEMSIZE); + u32 tom = INREG(NB_TOM); + + tmp = ((((tom >> 16) - (tom & 0xffff) + 1) << 6) * 1024); + radeon_fifo_wait(6); + OUTREG(MC_FB_LOCATION, tom); + OUTREG(DISPLAY_BASE_ADDR, (tom & 0xffff) << 16); + OUTREG(CRTC2_DISPLAY_BASE_ADDR, (tom & 0xffff) << 16); + OUTREG(OV0_BASE_ADDR, (tom & 0xffff) << 16); + + /* This is supposed to fix the crtc2 noise problem. */ + OUTREG(GRPH2_BUFFER_CNTL, INREG(GRPH2_BUFFER_CNTL) & ~0x7f0000); + + if ((rinfo->family == CHIP_FAMILY_RS100) || + (rinfo->family == CHIP_FAMILY_RS200)) { + /* This is to workaround the asic bug for RMX, some versions + * of BIOS doesn't have this register initialized correctly. + */ + OUTREGP(CRTC_MORE_CNTL, CRTC_H_CUTOFF_ACTIVE_EN, + ~CRTC_H_CUTOFF_ACTIVE_EN); + } + } else { + tmp = INREG(CNFG_MEMSIZE); } /* mem size is bits [28:0], mask off the rest */ diff --git a/drivers/video/fbdev/bw2.c b/drivers/video/fbdev/bw2.c index e7702fe1fe7d76eca07d9e62738c385c03ac9382..6403ae07970d6cbd7110f7b781627d628e69fd83 100644 --- a/drivers/video/fbdev/bw2.c +++ b/drivers/video/fbdev/bw2.c @@ -182,7 +182,7 @@ static int bw2_ioctl(struct fb_info *info, unsigned int cmd, unsigned long arg) static void bw2_init_fix(struct fb_info *info, int linebytes) { - strlcpy(info->fix.id, "bwtwo", sizeof(info->fix.id)); + strscpy(info->fix.id, "bwtwo", sizeof(info->fix.id)); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.visual = FB_VISUAL_MONO01; diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c index 5ad64714d39eae2b32b09951ddf522aaef722307..f1c1c95c1fdf05e52b70075a18f4ed253cbe8d4c 100644 --- a/drivers/video/fbdev/chipsfb.c +++ b/drivers/video/fbdev/chipsfb.c @@ -435,6 +435,7 @@ static int chipsfb_pci_init(struct pci_dev *dp, const struct pci_device_id *ent) err_release_fb: framebuffer_release(p); err_disable: + pci_disable_device(dp); err_out: return rc; } diff --git a/drivers/video/fbdev/cirrusfb.c b/drivers/video/fbdev/cirrusfb.c index 851367e159c022ac3bbdabe3ddea2eebdebe4b12..b08bee43779ae074379bdcdaa3e3b7815d152dff 100644 --- a/drivers/video/fbdev/cirrusfb.c +++ b/drivers/video/fbdev/cirrusfb.c @@ -2000,7 +2000,7 @@ static int cirrusfb_set_fbinfo(struct fb_info *info) } /* Fill fix common fields */ - strlcpy(info->fix.id, cirrusfb_board_info[cinfo->btype].name, + strscpy(info->fix.id, cirrusfb_board_info[cinfo->btype].name, sizeof(info->fix.id)); /* monochrome: only 1 memory plane */ diff --git a/drivers/video/fbdev/clps711x-fb.c b/drivers/video/fbdev/clps711x-fb.c index 771ce1f769515554ee65604cd63e5fc66410916d..a1061c2f16406e2e610bd3991731fa4967962b18 100644 --- a/drivers/video/fbdev/clps711x-fb.c +++ b/drivers/video/fbdev/clps711x-fb.c @@ -326,7 +326,7 @@ static int clps711x_fb_probe(struct platform_device *pdev) info->var.vmode = FB_VMODE_NONINTERLACED; info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.accel = FB_ACCEL_NONE; - strlcpy(info->fix.id, CLPS711X_FB_NAME, sizeof(info->fix.id)); + strscpy(info->fix.id, CLPS711X_FB_NAME, sizeof(info->fix.id)); fb_videomode_to_var(&info->var, &cfb->mode); ret = fb_alloc_cmap(&info->cmap, BIT(CLPS711X_FB_BPP_MAX), 0); diff --git a/drivers/video/fbdev/controlfb.c b/drivers/video/fbdev/controlfb.c index aba46118b208be14b26ea6b8d1b5e7c4cc26b69e..6bbcd9fc864e9eba589dcb2a08aab0e127e51e8c 100644 --- a/drivers/video/fbdev/controlfb.c +++ b/drivers/video/fbdev/controlfb.c @@ -108,13 +108,6 @@ static inline int PAR_EQUAL(struct fb_par_control *x, struct fb_par_control *y) return (!DIRTY(cmode) && !DIRTY(xres) && !DIRTY(yres) && !DIRTY(vxres) && !DIRTY(vyres)); } -static inline int VAR_MATCH(struct fb_var_screeninfo *x, struct fb_var_screeninfo *y) -{ - return (!DIRTY(bits_per_pixel) && !DIRTY(xres) - && !DIRTY(yres) && !DIRTY(xres_virtual) - && !DIRTY(yres_virtual) - && !DIRTY_CMAP(red) && !DIRTY_CMAP(green) && !DIRTY_CMAP(blue)); -} struct fb_info_control { struct fb_info info; diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index cf9ac4da0a82ceb70a2c171328b42f3f00f934ba..098b62f7b701e5815c67994424550416418e98be 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -412,7 +412,7 @@ static int __init fb_console_setup(char *this_opt) while ((options = strsep(&this_opt, ",")) != NULL) { if (!strncmp(options, "font:", 5)) { - strlcpy(fontname, options + 5, sizeof(fontname)); + strscpy(fontname, options + 5, sizeof(fontname)); continue; } @@ -2401,15 +2401,21 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount, struct fb_info *info = fbcon_info_from_console(vc->vc_num); struct fbcon_ops *ops = info->fbcon_par; struct fbcon_display *p = &fb_display[vc->vc_num]; - int resize; + int resize, ret, old_userfont, old_width, old_height, old_charcount; char *old_data = NULL; resize = (w != vc->vc_font.width) || (h != vc->vc_font.height); if (p->userfont) old_data = vc->vc_font.data; vc->vc_font.data = (void *)(p->fontdata = data); + old_userfont = p->userfont; if ((p->userfont = userfont)) REFCOUNT(data)++; + + old_width = vc->vc_font.width; + old_height = vc->vc_font.height; + old_charcount = vc->vc_font.charcount; + vc->vc_font.width = w; vc->vc_font.height = h; vc->vc_font.charcount = charcount; @@ -2425,7 +2431,9 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount, rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres); cols /= w; rows /= h; - vc_resize(vc, cols, rows); + ret = vc_resize(vc, cols, rows); + if (ret) + goto err_out; } else if (con_is_visible(vc) && vc->vc_mode == KD_TEXT) { fbcon_clear_margins(vc, 0); @@ -2435,6 +2443,21 @@ static int fbcon_do_set_font(struct vc_data *vc, int w, int h, int charcount, if (old_data && (--REFCOUNT(old_data) == 0)) kfree(old_data - FONT_EXTRA_WORDS * sizeof(int)); return 0; + +err_out: + p->fontdata = old_data; + vc->vc_font.data = (void *)old_data; + + if (userfont) { + p->userfont = old_userfont; + REFCOUNT(data)--; + } + + vc->vc_font.width = old_width; + vc->vc_font.height = old_height; + vc->vc_font.charcount = old_charcount; + + return ret; } /* diff --git a/drivers/video/fbdev/core/fbsysfs.c b/drivers/video/fbdev/core/fbsysfs.c index c2a60b187467e5f3a1bda2892ed61903f31984ce..4d7f63892dcc4381a78433577cb62a06f305c06c 100644 --- a/drivers/video/fbdev/core/fbsysfs.c +++ b/drivers/video/fbdev/core/fbsysfs.c @@ -84,6 +84,10 @@ void framebuffer_release(struct fb_info *info) if (WARN_ON(refcount_read(&info->count))) return; +#if IS_ENABLED(CONFIG_FB_BACKLIGHT) + mutex_destroy(&info->bl_curve_mutex); +#endif + kfree(info->apertures); kfree(info); } diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c index be7bcf95c96a5c6dc3b73674720d47262a9f3405..585af90a68a5f1c0f06c9101f0bcddf6938bc1ff 100644 --- a/drivers/video/fbdev/cyber2000fb.c +++ b/drivers/video/fbdev/cyber2000fb.c @@ -1135,7 +1135,7 @@ int cyber2000fb_attach(struct cyberpro_info *info, int idx) info->fb_size = int_cfb_info->fb.fix.smem_len; info->info = int_cfb_info; - strlcpy(info->dev_name, int_cfb_info->fb.fix.id, + strscpy(info->dev_name, int_cfb_info->fb.fix.id, sizeof(info->dev_name)); } @@ -1230,7 +1230,7 @@ static int cyber2000fb_ddc_getsda(void *data) static int cyber2000fb_setup_ddc_bus(struct cfb_info *cfb) { - strlcpy(cfb->ddc_adapter.name, cfb->fb.fix.id, + strscpy(cfb->ddc_adapter.name, cfb->fb.fix.id, sizeof(cfb->ddc_adapter.name)); cfb->ddc_adapter.owner = THIS_MODULE; cfb->ddc_adapter.class = I2C_CLASS_DDC; @@ -1305,7 +1305,7 @@ static int cyber2000fb_i2c_getscl(void *data) static int cyber2000fb_i2c_register(struct cfb_info *cfb) { - strlcpy(cfb->i2c_adapter.name, cfb->fb.fix.id, + strscpy(cfb->i2c_adapter.name, cfb->fb.fix.id, sizeof(cfb->i2c_adapter.name)); cfb->i2c_adapter.owner = THIS_MODULE; cfb->i2c_adapter.algo_data = &cfb->i2c_algo; @@ -1501,7 +1501,7 @@ static int cyber2000fb_setup(char *options) if (strncmp(opt, "font:", 5) == 0) { static char default_font_storage[40]; - strlcpy(default_font_storage, opt + 5, + strscpy(default_font_storage, opt + 5, sizeof(default_font_storage)); default_font = default_font_storage; continue; diff --git a/drivers/video/fbdev/ffb.c b/drivers/video/fbdev/ffb.c index b3d580e57221ebedd14e65bf9f68d1b84cc67a0a..7cba3969a9702ca9f82eef36fd7e12e176cc3164 100644 --- a/drivers/video/fbdev/ffb.c +++ b/drivers/video/fbdev/ffb.c @@ -883,7 +883,7 @@ static void ffb_init_fix(struct fb_info *info) } else ffb_type_name = "Elite 3D"; - strlcpy(info->fix.id, ffb_type_name, sizeof(info->fix.id)); + strscpy(info->fix.id, ffb_type_name, sizeof(info->fix.id)); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.visual = FB_VISUAL_TRUECOLOR; diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c index 6b4d5a7f3e152b57cef93bb3f5f78966478d0741..1582c718329c7f1164540349a0fbf59af255058c 100644 --- a/drivers/video/fbdev/gbefb.c +++ b/drivers/video/fbdev/gbefb.c @@ -1072,17 +1072,12 @@ static ssize_t gbefb_show_rev(struct device *device, struct device_attribute *at static DEVICE_ATTR(revision, S_IRUGO, gbefb_show_rev, NULL); -static void gbefb_remove_sysfs(struct device *dev) -{ - device_remove_file(dev, &dev_attr_size); - device_remove_file(dev, &dev_attr_revision); -} - -static void gbefb_create_sysfs(struct device *dev) -{ - device_create_file(dev, &dev_attr_size); - device_create_file(dev, &dev_attr_revision); -} +static struct attribute *gbefb_attrs[] = { + &dev_attr_size.attr, + &dev_attr_revision.attr, + NULL, +}; +ATTRIBUTE_GROUPS(gbefb); /* * Initialization @@ -1221,7 +1216,6 @@ static int gbefb_probe(struct platform_device *p_dev) } platform_set_drvdata(p_dev, info); - gbefb_create_sysfs(&p_dev->dev); fb_info(info, "%s rev %d @ 0x%08x using %dkB memory\n", info->fix.id, gbe_revision, (unsigned)GBE_BASE, @@ -1248,7 +1242,6 @@ static int gbefb_remove(struct platform_device* p_dev) gbe_turn_off(); arch_phys_wc_del(par->wc_cookie); release_mem_region(GBE_BASE, sizeof(struct sgi_gbe)); - gbefb_remove_sysfs(&p_dev->dev); framebuffer_release(info); return 0; @@ -1259,6 +1252,7 @@ static struct platform_driver gbefb_driver = { .remove = gbefb_remove, .driver = { .name = "gbefb", + .dev_groups = gbefb_groups, }, }; diff --git a/drivers/video/fbdev/geode/gx1fb_core.c b/drivers/video/fbdev/geode/gx1fb_core.c index 4cac7e3bb1a0cc2445ed3232d233ef6bc363b618..1514c653a84fb30de0bb730f4560edcd12cdce35 100644 --- a/drivers/video/fbdev/geode/gx1fb_core.c +++ b/drivers/video/fbdev/geode/gx1fb_core.c @@ -415,13 +415,13 @@ static void __init gx1fb_setup(char *options) continue; if (!strncmp(this_opt, "mode:", 5)) - strlcpy(mode_option, this_opt + 5, sizeof(mode_option)); + strscpy(mode_option, this_opt + 5, sizeof(mode_option)); else if (!strncmp(this_opt, "crt:", 4)) crt_option = !!simple_strtoul(this_opt + 4, NULL, 0); else if (!strncmp(this_opt, "panel:", 6)) - strlcpy(panel_option, this_opt + 6, sizeof(panel_option)); + strscpy(panel_option, this_opt + 6, sizeof(panel_option)); else - strlcpy(mode_option, this_opt, sizeof(mode_option)); + strscpy(mode_option, this_opt, sizeof(mode_option)); } } #endif diff --git a/drivers/video/fbdev/gxt4500.c b/drivers/video/fbdev/gxt4500.c index f0d36a4fdaefd159f6babdfc4d0a52ff169963db..0dcef4bec8d7cd871098ed1cbdd784e11df9a59b 100644 --- a/drivers/video/fbdev/gxt4500.c +++ b/drivers/video/fbdev/gxt4500.c @@ -655,7 +655,7 @@ static int gxt4500_probe(struct pci_dev *pdev, const struct pci_device_id *ent) cardtype = ent->driver_data; par->refclk_ps = cardinfo[cardtype].refclk_ps; info->fix = gxt4500_fix; - strlcpy(info->fix.id, cardinfo[cardtype].cardname, + strscpy(info->fix.id, cardinfo[cardtype].cardname, sizeof(info->fix.id)); info->pseudo_palette = par->pseudo_palette; diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c index a0e1d70b90d75f9803796b3d82f232f2ccf342f6..072ce07ba9e0586506fe4914d01cd81d511f804a 100644 --- a/drivers/video/fbdev/hyperv_fb.c +++ b/drivers/video/fbdev/hyperv_fb.c @@ -75,10 +75,6 @@ #define SYNTHVID_DEPTH_WIN8 32 #define SYNTHVID_FB_SIZE_WIN8 (8 * 1024 * 1024) -#define PCI_VENDOR_ID_MICROSOFT 0x1414 -#define PCI_DEVICE_ID_HYPERV_VIDEO 0x5353 - - enum pipe_msg_type { PIPE_MSG_INVALID, PIPE_MSG_DATA, diff --git a/drivers/video/fbdev/i740fb.c b/drivers/video/fbdev/i740fb.c index 199f786f9eedec1d7ec185963c48301bda2ad485..b795f6503cb671466d97bdc9e2470e261fb06695 100644 --- a/drivers/video/fbdev/i740fb.c +++ b/drivers/video/fbdev/i740fb.c @@ -160,7 +160,7 @@ static int i740fb_setup_ddc_bus(struct fb_info *info) { struct i740fb_par *par = info->par; - strlcpy(par->ddc_adapter.name, info->fix.id, + strscpy(par->ddc_adapter.name, info->fix.id, sizeof(par->ddc_adapter.name)); par->ddc_adapter.owner = THIS_MODULE; par->ddc_adapter.class = I2C_CLASS_DDC; diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c index d97d7456d15a0b6eafd92be10787d5d5da436873..51fde1b2a7938889d1a6dcfc1a5cd02d0f5863da 100644 --- a/drivers/video/fbdev/imxfb.c +++ b/drivers/video/fbdev/imxfb.c @@ -681,7 +681,7 @@ static int imxfb_init_fbinfo(struct platform_device *pdev) fbi->devtype = pdev->id_entry->driver_data; - strlcpy(info->fix.id, IMX_NAME, sizeof(info->fix.id)); + strscpy(info->fix.id, IMX_NAME, sizeof(info->fix.id)); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.type_aux = 0; @@ -972,7 +972,6 @@ static int imxfb_probe(struct platform_device *pdev) fbi->regs = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(fbi->regs)) { - dev_err(&pdev->dev, "Cannot map frame buffer registers\n"); ret = PTR_ERR(fbi->regs); goto failed_ioremap; } diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c index 3e26346c05a2852beeba6c25027081749e3b97de..775d34115e2dfdd59f76acbc34322e0cd700861e 100644 --- a/drivers/video/fbdev/matrox/matroxfb_base.c +++ b/drivers/video/fbdev/matrox/matroxfb_base.c @@ -2388,9 +2388,9 @@ static int __init matroxfb_setup(char *options) { else if (!strncmp(this_opt, "mem:", 4)) mem = simple_strtoul(this_opt+4, NULL, 0); else if (!strncmp(this_opt, "mode:", 5)) - strlcpy(videomode, this_opt+5, sizeof(videomode)); + strscpy(videomode, this_opt + 5, sizeof(videomode)); else if (!strncmp(this_opt, "outputs:", 8)) - strlcpy(outputs, this_opt+8, sizeof(outputs)); + strscpy(outputs, this_opt + 8, sizeof(outputs)); else if (!strncmp(this_opt, "dfp:", 4)) { dfp_type = simple_strtoul(this_opt+4, NULL, 0); dfp = 1; @@ -2460,7 +2460,7 @@ static int __init matroxfb_setup(char *options) { else if (!strcmp(this_opt, "dfp")) dfp = value; else { - strlcpy(videomode, this_opt, sizeof(videomode)); + strscpy(videomode, this_opt, sizeof(videomode)); } } } diff --git a/drivers/video/fbdev/matrox/matroxfb_maven.c b/drivers/video/fbdev/matrox/matroxfb_maven.c index 9a98c4a6ba3313e64f38ea385de774610b94b438..f2e02958673dfd608f5c54230a74088870b4e978 100644 --- a/drivers/video/fbdev/matrox/matroxfb_maven.c +++ b/drivers/video/fbdev/matrox/matroxfb_maven.c @@ -1276,11 +1276,10 @@ ERROR0:; return err; } -static int maven_remove(struct i2c_client *client) +static void maven_remove(struct i2c_client *client) { maven_shutdown_client(client); kfree(i2c_get_clientdata(client)); - return 0; } static const struct i2c_device_id maven_id[] = { diff --git a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c index 96800c9c9cd9ee75783daac06433af9b95b699ae..90c79e8c11570399b2fdaa5cbf1298d57a3601fd 100644 --- a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c +++ b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c @@ -693,7 +693,7 @@ static int of_platform_mb862xx_probe(struct platform_device *ofdev) par->dev = dev; par->irq = irq_of_parse_and_map(np, 0); - if (par->irq == NO_IRQ) { + if (!par->irq) { dev_err(dev, "failed to map irq\n"); ret = -ENODEV; goto fbrel; diff --git a/drivers/video/fbdev/omap/omapfb_main.c b/drivers/video/fbdev/omap/omapfb_main.c index dfb4ddc45701eb0136086e9ec986db1ba20ac6e3..17cda576568382b041f379f91df5cb12504a0881 100644 --- a/drivers/video/fbdev/omap/omapfb_main.c +++ b/drivers/video/fbdev/omap/omapfb_main.c @@ -1642,15 +1642,13 @@ static int omapfb_do_probe(struct platform_device *pdev, goto cleanup; } fbdev->int_irq = platform_get_irq(pdev, 0); - if (!fbdev->int_irq) { - dev_err(&pdev->dev, "unable to get irq\n"); + if (fbdev->int_irq < 0) { r = ENXIO; goto cleanup; } fbdev->ext_irq = platform_get_irq(pdev, 1); - if (!fbdev->ext_irq) { - dev_err(&pdev->dev, "unable to get irq\n"); + if (fbdev->ext_irq < 0) { r = ENXIO; goto cleanup; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c index b2d6e6df2161563667a5d5369cf6f3885ebc56fe..92fb6b7e1f6818ec8cb7d76418e6abcc7c31bd27 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c @@ -519,11 +519,9 @@ int dispc_runtime_get(void) DSSDBG("dispc_runtime_get\n"); - r = pm_runtime_get_sync(&dispc.pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&dispc.pdev->dev); + r = pm_runtime_resume_and_get(&dispc.pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } EXPORT_SYMBOL(dispc_runtime_get); diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c index d43b081d592f0101f396c8a157c1cc69f1dca455..54b0f034c2edfdf0c0fa2a50a911646eeefd70e9 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c @@ -1136,11 +1136,9 @@ static int dsi_runtime_get(struct platform_device *dsidev) DSSDBG("dsi_runtime_get\n"); - r = pm_runtime_get_sync(&dsi->pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&dsi->pdev->dev); + r = pm_runtime_resume_and_get(&dsi->pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.c b/drivers/video/fbdev/omap2/omapfb/dss/dss.c index 45b9d3cf3860218c9193f2eb7f5adbc42baad8d2..335e0af4eec1a0bc05d4d6bb621a52a9249612d8 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/dss.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.c @@ -767,11 +767,9 @@ int dss_runtime_get(void) DSSDBG("dss_runtime_get\n"); - r = pm_runtime_get_sync(&dss.pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&dss.pdev->dev); + r = pm_runtime_resume_and_get(&dss.pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c index 800bd108e834d4b721983f80cb753451a4c8caa2..0f39612e002e8aeebdb1cf2bff7d5f2012c96811 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c @@ -38,11 +38,9 @@ static int hdmi_runtime_get(void) DSSDBG("hdmi_runtime_get\n"); - r = pm_runtime_get_sync(&hdmi.pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&hdmi.pdev->dev); + r = pm_runtime_resume_and_get(&hdmi.pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c index 2c03608addcd788f6742244b34dd0bd3ca89b5e2..bfccc2cb917af583608f9393ae9763bb521bc855 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c @@ -42,11 +42,9 @@ static int hdmi_runtime_get(void) DSSDBG("hdmi_runtime_get\n"); - r = pm_runtime_get_sync(&hdmi.pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&hdmi.pdev->dev); + r = pm_runtime_resume_and_get(&hdmi.pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/dss/venc.c b/drivers/video/fbdev/omap2/omapfb/dss/venc.c index 905d642ff9ed70afaa08b4e0dc0d599e00b4c120..78a7309d25dd3e1815b3c7ca55464bbb6481ca6d 100644 --- a/drivers/video/fbdev/omap2/omapfb/dss/venc.c +++ b/drivers/video/fbdev/omap2/omapfb/dss/venc.c @@ -347,11 +347,9 @@ static int venc_runtime_get(void) DSSDBG("venc_runtime_get\n"); - r = pm_runtime_get_sync(&venc.pdev->dev); - if (WARN_ON(r < 0)) { - pm_runtime_put_sync(&venc.pdev->dev); + r = pm_runtime_resume_and_get(&venc.pdev->dev); + if (WARN_ON(r < 0)) return r; - } return 0; } diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c index afa688e754b9592278728aa2c3bf7921bca64d5c..5ccddcfce7228c5c87b93d8903822abb1c0974e2 100644 --- a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c +++ b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c @@ -1331,7 +1331,7 @@ static void clear_fb_info(struct fb_info *fbi) { memset(&fbi->var, 0, sizeof(fbi->var)); memset(&fbi->fix, 0, sizeof(fbi->fix)); - strlcpy(fbi->fix.id, MODULE_NAME, sizeof(fbi->fix.id)); + strscpy(fbi->fix.id, MODULE_NAME, sizeof(fbi->fix.id)); } static int omapfb_free_all_fbmem(struct omapfb2_device *fbdev) diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c index bc80d8498aeb0b60c70a1cafe343f85966fad02c..7da715d31a933b8a2b97a0b643d909a9a4e6abb4 100644 --- a/drivers/video/fbdev/pm2fb.c +++ b/drivers/video/fbdev/pm2fb.c @@ -618,6 +618,11 @@ static int pm2fb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) return -EINVAL; } + if (!var->pixclock) { + DPRINTK("pixclock is zero\n"); + return -EINVAL; + } + if (PICOS2KHZ(var->pixclock) > PM2_MAX_PIXCLOCK) { DPRINTK("pixclock too high (%ldKHz)\n", PICOS2KHZ(var->pixclock)); diff --git a/drivers/video/fbdev/pxa168fb.c b/drivers/video/fbdev/pxa168fb.c index e943300d23e8ebd94eb09e39cde5d6df85a4c157..d5d0bbd39213bcd4c628d1bd6bec337b137a4841 100644 --- a/drivers/video/fbdev/pxa168fb.c +++ b/drivers/video/fbdev/pxa168fb.c @@ -640,7 +640,7 @@ static int pxa168fb_probe(struct platform_device *pdev) info->flags = FBINFO_DEFAULT | FBINFO_PARTIAL_PAN_OK | FBINFO_HWACCEL_XPAN | FBINFO_HWACCEL_YPAN; info->node = -1; - strlcpy(info->fix.id, mi->id, 16); + strscpy(info->fix.id, mi->id, 16); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.type_aux = 0; info->fix.xpanstep = 0; diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c index 66cfc3e9d3cfd73120b407e8dad6e13d3554834f..696ac54311809b2a2bbf50a06bba08db8b46e702 100644 --- a/drivers/video/fbdev/pxafb.c +++ b/drivers/video/fbdev/pxafb.c @@ -2042,7 +2042,7 @@ static int __init pxafb_setup_options(void) return -ENODEV; if (options) - strlcpy(g_options, options, sizeof(g_options)); + strscpy(g_options, options, sizeof(g_options)); return 0; } diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c index 1882408b2d13707c68650a379392a39195267820..7713274bd04c22a53141ff999579b471987e9844 100644 --- a/drivers/video/fbdev/s3fb.c +++ b/drivers/video/fbdev/s3fb.c @@ -249,7 +249,7 @@ static int s3fb_setup_ddc_bus(struct fb_info *info) { struct s3fb_info *par = info->par; - strlcpy(par->ddc_adapter.name, info->fix.id, + strscpy(par->ddc_adapter.name, info->fix.id, sizeof(par->ddc_adapter.name)); par->ddc_adapter.owner = THIS_MODULE; par->ddc_adapter.class = I2C_CLASS_DDC; diff --git a/drivers/video/fbdev/simplefb.c b/drivers/video/fbdev/simplefb.c index cf2a90ecd64e0a8b4cd718e3068fe4b88deccbfa..e770b4a356b57cb0dd4cff8981ebca0f4d1ee332 100644 --- a/drivers/video/fbdev/simplefb.c +++ b/drivers/video/fbdev/simplefb.c @@ -355,7 +355,7 @@ static int simplefb_regulators_get(struct simplefb_par *par, if (!p || p == prop->name) continue; - strlcpy(name, prop->name, + strscpy(name, prop->name, strlen(prop->name) - strlen(SUPPLY_SUFFIX) + 1); regulator = devm_regulator_get_optional(&pdev->dev, name); if (IS_ERR(regulator)) { diff --git a/drivers/video/fbdev/sis/sis_main.c b/drivers/video/fbdev/sis/sis_main.c index 7114c5c17c9121d3ead5516d9fcfa0fc52d9c4cd..1c197c3f95381e531d45209419f72aa172e172c8 100644 --- a/drivers/video/fbdev/sis/sis_main.c +++ b/drivers/video/fbdev/sis/sis_main.c @@ -650,37 +650,37 @@ sisfb_validate_mode(struct sis_video_info *ivideo, int myindex, u32 vbflags) u16 xres=0, yres, myres; #ifdef CONFIG_FB_SIS_300 - if(ivideo->sisvga_engine == SIS_300_VGA) { - if(!(sisbios_mode[myindex].chipset & MD_SIS300)) + if (ivideo->sisvga_engine == SIS_300_VGA) { + if (!(sisbios_mode[myindex].chipset & MD_SIS300)) return -1 ; } #endif #ifdef CONFIG_FB_SIS_315 - if(ivideo->sisvga_engine == SIS_315_VGA) { - if(!(sisbios_mode[myindex].chipset & MD_SIS315)) + if (ivideo->sisvga_engine == SIS_315_VGA) { + if (!(sisbios_mode[myindex].chipset & MD_SIS315)) return -1; } #endif myres = sisbios_mode[myindex].yres; - switch(vbflags & VB_DISPTYPE_DISP2) { + switch (vbflags & VB_DISPTYPE_DISP2) { case CRT2_LCD: xres = ivideo->lcdxres; yres = ivideo->lcdyres; - if((ivideo->SiS_Pr.SiS_CustomT != CUT_PANEL848) && - (ivideo->SiS_Pr.SiS_CustomT != CUT_PANEL856)) { - if(sisbios_mode[myindex].xres > xres) + if ((ivideo->SiS_Pr.SiS_CustomT != CUT_PANEL848) && + (ivideo->SiS_Pr.SiS_CustomT != CUT_PANEL856)) { + if (sisbios_mode[myindex].xres > xres) return -1; - if(myres > yres) + if (myres > yres) return -1; } - if(ivideo->sisfb_fstn) { - if(sisbios_mode[myindex].xres == 320) { - if(myres == 240) { - switch(sisbios_mode[myindex].mode_no[1]) { + if (ivideo->sisfb_fstn) { + if (sisbios_mode[myindex].xres == 320) { + if (myres == 240) { + switch (sisbios_mode[myindex].mode_no[1]) { case 0x50: myindex = MODE_FSTN_8; break; case 0x56: myindex = MODE_FSTN_16; break; case 0x53: return -1; @@ -689,7 +689,7 @@ sisfb_validate_mode(struct sis_video_info *ivideo, int myindex, u32 vbflags) } } - if(SiS_GetModeID_LCD(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, + if (SiS_GetModeID_LCD(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, sisbios_mode[myindex].yres, 0, ivideo->sisfb_fstn, ivideo->SiS_Pr.SiS_CustomT, xres, yres, ivideo->vbflags2) < 0x14) { return -1; @@ -697,14 +697,14 @@ sisfb_validate_mode(struct sis_video_info *ivideo, int myindex, u32 vbflags) break; case CRT2_TV: - if(SiS_GetModeID_TV(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, + if (SiS_GetModeID_TV(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, sisbios_mode[myindex].yres, 0, ivideo->vbflags2) < 0x14) { return -1; } break; case CRT2_VGA: - if(SiS_GetModeID_VGA2(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, + if (SiS_GetModeID_VGA2(ivideo->sisvga_engine, vbflags, sisbios_mode[myindex].xres, sisbios_mode[myindex].yres, 0, ivideo->vbflags2) < 0x14) { return -1; } @@ -1873,7 +1873,7 @@ sisfb_get_fix(struct fb_fix_screeninfo *fix, int con, struct fb_info *info) memset(fix, 0, sizeof(struct fb_fix_screeninfo)); - strlcpy(fix->id, ivideo->myid, sizeof(fix->id)); + strscpy(fix->id, ivideo->myid, sizeof(fix->id)); mutex_lock(&info->mm_lock); fix->smem_start = ivideo->video_base + ivideo->video_offset; @@ -2205,82 +2205,88 @@ static bool sisfb_test_DDC1(struct sis_video_info *ivideo) static void sisfb_sense_crt1(struct sis_video_info *ivideo) { - bool mustwait = false; - u8 sr1F, cr17; + bool mustwait = false; + u8 sr1F, cr17; #ifdef CONFIG_FB_SIS_315 - u8 cr63=0; + u8 cr63 = 0; #endif - u16 temp = 0xffff; - int i; + u16 temp = 0xffff; + int i; + + sr1F = SiS_GetReg(SISSR, 0x1F); + SiS_SetRegOR(SISSR, 0x1F, 0x04); + SiS_SetRegAND(SISSR, 0x1F, 0x3F); - sr1F = SiS_GetReg(SISSR, 0x1F); - SiS_SetRegOR(SISSR, 0x1F, 0x04); - SiS_SetRegAND(SISSR, 0x1F, 0x3F); - if(sr1F & 0xc0) mustwait = true; + if (sr1F & 0xc0) + mustwait = true; #ifdef CONFIG_FB_SIS_315 - if(ivideo->sisvga_engine == SIS_315_VGA) { - cr63 = SiS_GetReg(SISCR, ivideo->SiS_Pr.SiS_MyCR63); - cr63 &= 0x40; - SiS_SetRegAND(SISCR, ivideo->SiS_Pr.SiS_MyCR63, 0xBF); - } + if (ivideo->sisvga_engine == SIS_315_VGA) { + cr63 = SiS_GetReg(SISCR, ivideo->SiS_Pr.SiS_MyCR63); + cr63 &= 0x40; + SiS_SetRegAND(SISCR, ivideo->SiS_Pr.SiS_MyCR63, 0xBF); + } #endif - cr17 = SiS_GetReg(SISCR, 0x17); - cr17 &= 0x80; - if(!cr17) { - SiS_SetRegOR(SISCR, 0x17, 0x80); - mustwait = true; - SiS_SetReg(SISSR, 0x00, 0x01); - SiS_SetReg(SISSR, 0x00, 0x03); - } + cr17 = SiS_GetReg(SISCR, 0x17); + cr17 &= 0x80; - if(mustwait) { - for(i=0; i < 10; i++) sisfbwaitretracecrt1(ivideo); - } + if (!cr17) { + SiS_SetRegOR(SISCR, 0x17, 0x80); + mustwait = true; + SiS_SetReg(SISSR, 0x00, 0x01); + SiS_SetReg(SISSR, 0x00, 0x03); + } + if (mustwait) { + for (i = 0; i < 10; i++) + sisfbwaitretracecrt1(ivideo); + } #ifdef CONFIG_FB_SIS_315 - if(ivideo->chip >= SIS_330) { - SiS_SetRegAND(SISCR, 0x32, ~0x20); - if(ivideo->chip >= SIS_340) { - SiS_SetReg(SISCR, 0x57, 0x4a); - } else { - SiS_SetReg(SISCR, 0x57, 0x5f); - } - SiS_SetRegOR(SISCR, 0x53, 0x02); - while ((SiS_GetRegByte(SISINPSTAT)) & 0x01) break; - while (!((SiS_GetRegByte(SISINPSTAT)) & 0x01)) break; - if ((SiS_GetRegByte(SISMISCW)) & 0x10) temp = 1; - SiS_SetRegAND(SISCR, 0x53, 0xfd); - SiS_SetRegAND(SISCR, 0x57, 0x00); - } + if (ivideo->chip >= SIS_330) { + SiS_SetRegAND(SISCR, 0x32, ~0x20); + if (ivideo->chip >= SIS_340) + SiS_SetReg(SISCR, 0x57, 0x4a); + else + SiS_SetReg(SISCR, 0x57, 0x5f); + + SiS_SetRegOR(SISCR, 0x53, 0x02); + while ((SiS_GetRegByte(SISINPSTAT)) & 0x01) + break; + while (!((SiS_GetRegByte(SISINPSTAT)) & 0x01)) + break; + if ((SiS_GetRegByte(SISMISCW)) & 0x10) + temp = 1; + + SiS_SetRegAND(SISCR, 0x53, 0xfd); + SiS_SetRegAND(SISCR, 0x57, 0x00); + } #endif - if(temp == 0xffff) { - i = 3; - do { - temp = SiS_HandleDDC(&ivideo->SiS_Pr, ivideo->vbflags, - ivideo->sisvga_engine, 0, 0, NULL, ivideo->vbflags2); - } while(((temp == 0) || (temp == 0xffff)) && i--); + if (temp == 0xffff) { + i = 3; - if((temp == 0) || (temp == 0xffff)) { - if(sisfb_test_DDC1(ivideo)) temp = 1; - } - } + do { + temp = SiS_HandleDDC(&ivideo->SiS_Pr, ivideo->vbflags, + ivideo->sisvga_engine, 0, 0, NULL, ivideo->vbflags2); + } while (((temp == 0) || (temp == 0xffff)) && i--); - if((temp) && (temp != 0xffff)) { - SiS_SetRegOR(SISCR, 0x32, 0x20); - } + if ((temp == 0) || (temp == 0xffff)) { + if (sisfb_test_DDC1(ivideo)) + temp = 1; + } + } + + if ((temp) && (temp != 0xffff)) + SiS_SetRegOR(SISCR, 0x32, 0x20); #ifdef CONFIG_FB_SIS_315 - if(ivideo->sisvga_engine == SIS_315_VGA) { - SiS_SetRegANDOR(SISCR, ivideo->SiS_Pr.SiS_MyCR63, 0xBF, cr63); - } + if (ivideo->sisvga_engine == SIS_315_VGA) + SiS_SetRegANDOR(SISCR, ivideo->SiS_Pr.SiS_MyCR63, 0xBF, cr63); #endif - SiS_SetRegANDOR(SISCR, 0x17, 0x7F, cr17); - - SiS_SetReg(SISSR, 0x1F, sr1F); + SiS_SetRegANDOR(SISCR, 0x17, 0x7F, cr17); + SiS_SetReg(SISSR, 0x1F, sr1F); } /* Determine and detect attached devices on SiS30x */ @@ -2294,25 +2300,25 @@ static void SiS_SenseLCD(struct sis_video_info *ivideo) ivideo->SiS_Pr.PanelSelfDetected = false; /* LCD detection only for TMDS bridges */ - if(!(ivideo->vbflags2 & VB2_SISTMDSBRIDGE)) + if (!(ivideo->vbflags2 & VB2_SISTMDSBRIDGE)) return; - if(ivideo->vbflags2 & VB2_30xBDH) + if (ivideo->vbflags2 & VB2_30xBDH) return; /* If LCD already set up by BIOS, skip it */ reg = SiS_GetReg(SISCR, 0x32); - if(reg & 0x08) + if (reg & 0x08) return; realcrtno = 1; - if(ivideo->SiS_Pr.DDCPortMixup) + if (ivideo->SiS_Pr.DDCPortMixup) realcrtno = 0; /* Check DDC capabilities */ temp = SiS_HandleDDC(&ivideo->SiS_Pr, ivideo->vbflags, ivideo->sisvga_engine, realcrtno, 0, &buffer[0], ivideo->vbflags2); - if((!temp) || (temp == 0xffff) || (!(temp & 0x02))) + if ((!temp) || (temp == 0xffff) || (!(temp & 0x02))) return; /* Read DDC data */ @@ -2321,17 +2327,17 @@ static void SiS_SenseLCD(struct sis_video_info *ivideo) temp = SiS_HandleDDC(&ivideo->SiS_Pr, ivideo->vbflags, ivideo->sisvga_engine, realcrtno, 1, &buffer[0], ivideo->vbflags2); - } while((temp) && i--); + } while ((temp) && i--); - if(temp) + if (temp) return; /* No digital device */ - if(!(buffer[0x14] & 0x80)) + if (!(buffer[0x14] & 0x80)) return; /* First detailed timing preferred timing? */ - if(!(buffer[0x18] & 0x02)) + if (!(buffer[0x18] & 0x02)) return; xres = buffer[0x38] | ((buffer[0x3a] & 0xf0) << 4); @@ -2339,26 +2345,26 @@ static void SiS_SenseLCD(struct sis_video_info *ivideo) switch(xres) { case 1024: - if(yres == 768) + if (yres == 768) paneltype = 0x02; break; case 1280: - if(yres == 1024) + if (yres == 1024) paneltype = 0x03; break; case 1600: - if((yres == 1200) && (ivideo->vbflags2 & VB2_30xC)) + if ((yres == 1200) && (ivideo->vbflags2 & VB2_30xC)) paneltype = 0x0b; break; } - if(!paneltype) + if (!paneltype) return; - if(buffer[0x23]) + if (buffer[0x23]) cr37 |= 0x10; - if((buffer[0x47] & 0x18) == 0x18) + if ((buffer[0x47] & 0x18) == 0x18) cr37 |= ((((buffer[0x47] & 0x06) ^ 0x06) << 5) | 0x20); else cr37 |= 0xc0; @@ -2373,31 +2379,34 @@ static void SiS_SenseLCD(struct sis_video_info *ivideo) static int SISDoSense(struct sis_video_info *ivideo, u16 type, u16 test) { - int temp, mytest, result, i, j; - - for(j = 0; j < 10; j++) { - result = 0; - for(i = 0; i < 3; i++) { - mytest = test; - SiS_SetReg(SISPART4, 0x11, (type & 0x00ff)); - temp = (type >> 8) | (mytest & 0x00ff); - SiS_SetRegANDOR(SISPART4, 0x10, 0xe0, temp); - SiS_DDC2Delay(&ivideo->SiS_Pr, 0x1500); - mytest >>= 8; - mytest &= 0x7f; - temp = SiS_GetReg(SISPART4, 0x03); - temp ^= 0x0e; - temp &= mytest; - if(temp == mytest) result++; + int temp, mytest, result, i, j; + + for (j = 0; j < 10; j++) { + result = 0; + for (i = 0; i < 3; i++) { + mytest = test; + SiS_SetReg(SISPART4, 0x11, (type & 0x00ff)); + temp = (type >> 8) | (mytest & 0x00ff); + SiS_SetRegANDOR(SISPART4, 0x10, 0xe0, temp); + SiS_DDC2Delay(&ivideo->SiS_Pr, 0x1500); + mytest >>= 8; + mytest &= 0x7f; + temp = SiS_GetReg(SISPART4, 0x03); + temp ^= 0x0e; + temp &= mytest; + if (temp == mytest) + result++; #if 1 - SiS_SetReg(SISPART4, 0x11, 0x00); - SiS_SetRegAND(SISPART4, 0x10, 0xe0); - SiS_DDC2Delay(&ivideo->SiS_Pr, 0x1000); + SiS_SetReg(SISPART4, 0x11, 0x00); + SiS_SetRegAND(SISPART4, 0x10, 0xe0); + SiS_DDC2Delay(&ivideo->SiS_Pr, 0x1000); #endif - } - if((result == 0) || (result >= 2)) break; - } - return result; + } + + if ((result == 0) || (result >= 2)) + break; + } + return result; } static void SiS_Sense30x(struct sis_video_info *ivideo) @@ -4263,18 +4272,17 @@ static int sisfb_post_300_rwtest(struct sis_video_info *ivideo, int iteration, unsigned int k, RankCapacity, PageCapacity, BankNumHigh, BankNumMid; unsigned int PhysicalAdrOtherPage, PhysicalAdrHigh, PhysicalAdrHalfPage; - for(k = 0; k < ARRAY_SIZE(SiS_DRAMType); k++) { - + for (k = 0; k < ARRAY_SIZE(SiS_DRAMType); k++) { RankCapacity = buswidth * SiS_DRAMType[k][3]; - if(RankCapacity != PseudoRankCapacity) + if (RankCapacity != PseudoRankCapacity) continue; - if((SiS_DRAMType[k][2] + SiS_DRAMType[k][0]) > PseudoAdrPinCount) + if ((SiS_DRAMType[k][2] + SiS_DRAMType[k][0]) > PseudoAdrPinCount) continue; BankNumHigh = RankCapacity * 16 * iteration - 1; - if(iteration == 3) { /* Rank No */ + if (iteration == 3) { /* Rank No */ BankNumMid = RankCapacity * 16 - 1; } else { BankNumMid = RankCapacity * 16 * iteration / 2 - 1; @@ -4288,18 +4296,22 @@ static int sisfb_post_300_rwtest(struct sis_video_info *ivideo, int iteration, SiS_SetRegAND(SISSR, 0x15, 0xFB); /* Test */ SiS_SetRegOR(SISSR, 0x15, 0x04); /* Test */ sr14 = (SiS_DRAMType[k][3] * buswidth) - 1; - if(buswidth == 4) sr14 |= 0x80; - else if(buswidth == 2) sr14 |= 0x40; + + if (buswidth == 4) + sr14 |= 0x80; + else if (buswidth == 2) + sr14 |= 0x40; + SiS_SetReg(SISSR, 0x13, SiS_DRAMType[k][4]); SiS_SetReg(SISSR, 0x14, sr14); BankNumHigh <<= 16; BankNumMid <<= 16; - if((BankNumHigh + PhysicalAdrHigh >= mapsize) || - (BankNumMid + PhysicalAdrHigh >= mapsize) || - (BankNumHigh + PhysicalAdrHalfPage >= mapsize) || - (BankNumHigh + PhysicalAdrOtherPage >= mapsize)) + if ((BankNumHigh + PhysicalAdrHigh >= mapsize) || + (BankNumMid + PhysicalAdrHigh >= mapsize) || + (BankNumHigh + PhysicalAdrHalfPage >= mapsize) || + (BankNumHigh + PhysicalAdrOtherPage >= mapsize)) continue; /* Write data */ @@ -4313,7 +4325,7 @@ static int sisfb_post_300_rwtest(struct sis_video_info *ivideo, int iteration, (FBAddr + BankNumHigh + PhysicalAdrOtherPage)); /* Read data */ - if(readw(FBAddr + BankNumHigh + PhysicalAdrHigh) == PhysicalAdrHigh) + if (readw(FBAddr + BankNumHigh + PhysicalAdrHigh) == PhysicalAdrHigh) return 1; } @@ -5872,7 +5884,7 @@ static int sisfb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ivideo->cardnumber++; } - strlcpy(ivideo->myid, chipinfo->chip_name, sizeof(ivideo->myid)); + strscpy(ivideo->myid, chipinfo->chip_name, sizeof(ivideo->myid)); ivideo->warncount = 0; ivideo->chip_id = pdev->device; @@ -6155,24 +6167,20 @@ static int sisfb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif #ifdef CONFIG_FB_SIS_315 - if(ivideo->sisvga_engine == SIS_315_VGA) { + if (ivideo->sisvga_engine == SIS_315_VGA) { int result = 1; - /* if((ivideo->chip == SIS_315H) || - (ivideo->chip == SIS_315) || - (ivideo->chip == SIS_315PRO) || - (ivideo->chip == SIS_330)) { - sisfb_post_sis315330(pdev); - } else */ if(ivideo->chip == XGI_20) { + + if (ivideo->chip == XGI_20) { result = sisfb_post_xgi(pdev); ivideo->sisfb_can_post = 1; - } else if((ivideo->chip == XGI_40) && ivideo->haveXGIROM) { + } else if ((ivideo->chip == XGI_40) && ivideo->haveXGIROM) { result = sisfb_post_xgi(pdev); ivideo->sisfb_can_post = 1; } else { printk(KERN_INFO "sisfb: Card is not " "POSTed and sisfb can't do this either.\n"); } - if(!result) { + if (!result) { printk(KERN_ERR "sisfb: Failed to POST card\n"); ret = -ENODEV; goto error_3; diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c index 6a52eba645596a4b3abab5f16340d3be5e0c8ef1..fce6cfbadfd60e30e7c28c4afe13267e866c116f 100644 --- a/drivers/video/fbdev/sm501fb.c +++ b/drivers/video/fbdev/sm501fb.c @@ -1719,7 +1719,7 @@ static int sm501fb_init_fb(struct fb_info *fb, enum sm501_controller head, enable = 0; } - strlcpy(fb->fix.id, fbname, sizeof(fb->fix.id)); + strscpy(fb->fix.id, fbname, sizeof(fb->fix.id)); memcpy(&par->ops, (head == HEAD_CRT) ? &sm501fb_ops_crt : &sm501fb_ops_pnl, diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index d7aa5511c3617a07fe9e003ef4e5b539d58d2548..e65bdc499c2365979d41076fd928768e4b4bdb34 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -137,6 +137,8 @@ static int ufx_submit_urb(struct ufx_data *dev, struct urb * urb, size_t len); static int ufx_alloc_urb_list(struct ufx_data *dev, int count, size_t size); static void ufx_free_urb_list(struct ufx_data *dev); +static DEFINE_MUTEX(disconnect_mutex); + /* reads a control register */ static int ufx_reg_read(struct ufx_data *dev, u32 index, u32 *data) { @@ -1071,9 +1073,13 @@ static int ufx_ops_open(struct fb_info *info, int user) if (user == 0 && !console) return -EBUSY; + mutex_lock(&disconnect_mutex); + /* If the USB device is gone, we don't accept new opens */ - if (dev->virtualized) + if (dev->virtualized) { + mutex_unlock(&disconnect_mutex); return -ENODEV; + } dev->fb_count++; @@ -1097,6 +1103,8 @@ static int ufx_ops_open(struct fb_info *info, int user) pr_debug("open /dev/fb%d user=%d fb_info=%p count=%d", info->node, user, info, dev->fb_count); + mutex_unlock(&disconnect_mutex); + return 0; } @@ -1741,6 +1749,8 @@ static void ufx_usb_disconnect(struct usb_interface *interface) { struct ufx_data *dev; + mutex_lock(&disconnect_mutex); + dev = usb_get_intfdata(interface); pr_debug("USB disconnect starting\n"); @@ -1761,6 +1771,8 @@ static void ufx_usb_disconnect(struct usb_interface *interface) kref_put(&dev->kref, ufx_free); /* consider ufx_data freed */ + + mutex_unlock(&disconnect_mutex); } static struct usb_driver ufx_driver = { diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index 5c765655d000a79fdf502a114798834d12e27dde..5c891aa00d5965a91a19e4d3ce139e279053033d 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -450,7 +450,7 @@ static int ssd1307fb_init(struct ssd1307fb_par *par) if (ret < 0) return ret; - /* Set Set Area Color Mode ON/OFF & Low Power Display Mode */ + /* Set Area Color Mode ON/OFF & Low Power Display Mode */ if (par->area_color_enable || par->low_power) { u32 mode; @@ -817,7 +817,7 @@ fb_alloc_error: return ret; } -static int ssd1307fb_remove(struct i2c_client *client) +static void ssd1307fb_remove(struct i2c_client *client) { struct fb_info *info = i2c_get_clientdata(client); struct ssd1307fb_par *par = info->par; @@ -836,8 +836,6 @@ static int ssd1307fb_remove(struct i2c_client *client) fb_deferred_io_cleanup(info); __free_pages(__va(info->fix.smem_start), get_order(info->fix.smem_len)); framebuffer_release(info); - - return 0; } static const struct i2c_device_id ssd1307fb_i2c_id[] = { diff --git a/drivers/video/fbdev/sstfb.c b/drivers/video/fbdev/sstfb.c index 73ca2782ebfc8c534067a30dfd27fbf09eb544f0..a56b24288566b6c19b539246cafb9d45b20130d6 100644 --- a/drivers/video/fbdev/sstfb.c +++ b/drivers/video/fbdev/sstfb.c @@ -1387,7 +1387,7 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto fail; } sst_get_memsize(info, &fix->smem_len); - strlcpy(fix->id, spec->name, sizeof(fix->id)); + strscpy(fix->id, spec->name, sizeof(fix->id)); printk(KERN_INFO "%s (revision %d) with %s dac\n", fix->id, par->revision, par->dac_sw.name); diff --git a/drivers/video/fbdev/stifb.c b/drivers/video/fbdev/stifb.c index 38a861e22c339e1102e4504095f0dad804415e73..7753e586e65a0359cbe4463f0c10cf5a7d84d3e0 100644 --- a/drivers/video/fbdev/stifb.c +++ b/drivers/video/fbdev/stifb.c @@ -1298,7 +1298,7 @@ static int __init stifb_init_fb(struct sti_struct *sti, int bpp_pref) /* limit fbsize to max visible screen size */ if (fix->smem_len > yres*fix->line_length) - fix->smem_len = yres*fix->line_length; + fix->smem_len = ALIGN(yres*fix->line_length, 4*1024*1024); fix->accel = FB_ACCEL_NONE; diff --git a/drivers/video/fbdev/sunxvr1000.c b/drivers/video/fbdev/sunxvr1000.c index 15b079505a0006409bffd8d950b0a0ddf47f41f2..490bd9a147638e6514cdc5fe23a8afacce2a5d3d 100644 --- a/drivers/video/fbdev/sunxvr1000.c +++ b/drivers/video/fbdev/sunxvr1000.c @@ -80,7 +80,7 @@ static int gfb_set_fbinfo(struct gfb_info *gp) info->pseudo_palette = gp->pseudo_palette; /* Fill fix common fields */ - strlcpy(info->fix.id, "gfb", sizeof(info->fix.id)); + strscpy(info->fix.id, "gfb", sizeof(info->fix.id)); info->fix.smem_start = gp->fb_base_phys; info->fix.smem_len = gp->fb_size; info->fix.type = FB_TYPE_PACKED_PIXELS; diff --git a/drivers/video/fbdev/sunxvr2500.c b/drivers/video/fbdev/sunxvr2500.c index 81d59613ea1f67ae66cca76cef16f62af7c84dd9..f4059529c60249a8f669e35c805c235b97e43359 100644 --- a/drivers/video/fbdev/sunxvr2500.c +++ b/drivers/video/fbdev/sunxvr2500.c @@ -85,7 +85,7 @@ static int s3d_set_fbinfo(struct s3d_info *sp) info->pseudo_palette = sp->pseudo_palette; /* Fill fix common fields */ - strlcpy(info->fix.id, "s3d", sizeof(info->fix.id)); + strscpy(info->fix.id, "s3d", sizeof(info->fix.id)); info->fix.smem_start = sp->fb_base_phys; info->fix.smem_len = sp->fb_size; info->fix.type = FB_TYPE_PACKED_PIXELS; diff --git a/drivers/video/fbdev/sunxvr500.c b/drivers/video/fbdev/sunxvr500.c index 3a51b2a1480c1fd9eebfd4f034b1b69cabe90cfb..b0c8cf0c535a4b8399b1df21937aab945a11655a 100644 --- a/drivers/video/fbdev/sunxvr500.c +++ b/drivers/video/fbdev/sunxvr500.c @@ -208,7 +208,7 @@ static int e3d_set_fbinfo(struct e3d_info *ep) info->pseudo_palette = ep->pseudo_palette; /* Fill fix common fields */ - strlcpy(info->fix.id, "e3d", sizeof(info->fix.id)); + strscpy(info->fix.id, "e3d", sizeof(info->fix.id)); info->fix.smem_start = ep->fb_base_phys; info->fix.smem_len = ep->fb_size; info->fix.type = FB_TYPE_PACKED_PIXELS; diff --git a/drivers/video/fbdev/tcx.c b/drivers/video/fbdev/tcx.c index 1638a40fed2254c74b2807812317942c182dcf2f..01d87f53324d985452afb511f343d16ab67ec0dd 100644 --- a/drivers/video/fbdev/tcx.c +++ b/drivers/video/fbdev/tcx.c @@ -333,7 +333,7 @@ tcx_init_fix(struct fb_info *info, int linebytes) else tcx_name = "TCX24"; - strlcpy(info->fix.id, tcx_name, sizeof(info->fix.id)); + strscpy(info->fix.id, tcx_name, sizeof(info->fix.id)); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.visual = FB_VISUAL_PSEUDOCOLOR; diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index 059e0174e139757260fbe3b807545ea2bd724aff..592a913d07189da0a72c049500b830a66a0f7269 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1265,7 +1265,7 @@ static int tdfxfb_setup_ddc_bus(struct tdfxfb_i2c_chan *chan, const char *name, { int rc; - strlcpy(chan->adapter.name, name, sizeof(chan->adapter.name)); + strscpy(chan->adapter.name, name, sizeof(chan->adapter.name)); chan->adapter.owner = THIS_MODULE; chan->adapter.class = I2C_CLASS_DDC; chan->adapter.algo_data = &chan->algo; @@ -1294,7 +1294,7 @@ static int tdfxfb_setup_i2c_bus(struct tdfxfb_i2c_chan *chan, const char *name, { int rc; - strlcpy(chan->adapter.name, name, sizeof(chan->adapter.name)); + strscpy(chan->adapter.name, name, sizeof(chan->adapter.name)); chan->adapter.owner = THIS_MODULE; chan->adapter.algo_data = &chan->algo; chan->adapter.dev.parent = dev; diff --git a/drivers/video/fbdev/tgafb.c b/drivers/video/fbdev/tgafb.c index 4600138e3bef9f88c9a49cf9ef9f82afe68f45b4..251dbd282f5eddeac5ddd44924a770350024e89b 100644 --- a/drivers/video/fbdev/tgafb.c +++ b/drivers/video/fbdev/tgafb.c @@ -1351,7 +1351,7 @@ tgafb_init_fix(struct fb_info *info) memory_size = 16777216; } - strlcpy(info->fix.id, tga_type_name, sizeof(info->fix.id)); + strscpy(info->fix.id, tga_type_name, sizeof(info->fix.id)); info->fix.type = FB_TYPE_PACKED_PIXELS; info->fix.type_aux = 0; diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c index 6813df793c494c9753b94446f908739a9e0d3bf6..219ce7292337034271dc17aabf1105e2a51d2cc8 100644 --- a/drivers/video/fbdev/tridentfb.c +++ b/drivers/video/fbdev/tridentfb.c @@ -271,7 +271,7 @@ static int tridentfb_setup_ddc_bus(struct fb_info *info) { struct tridentfb_par *par = info->par; - strlcpy(par->ddc_adapter.name, info->fix.id, + strscpy(par->ddc_adapter.name, info->fix.id, sizeof(par->ddc_adapter.name)); par->ddc_adapter.owner = THIS_MODULE; par->ddc_adapter.class = I2C_CLASS_DDC; @@ -1128,11 +1128,6 @@ static inline void shadowmode_on(struct tridentfb_par *par) write3CE(par, CyberControl, read3CE(par, CyberControl) | 0x81); } -static inline void shadowmode_off(struct tridentfb_par *par) -{ - write3CE(par, CyberControl, read3CE(par, CyberControl) & 0x7E); -} - /* Set the hardware to the requested video mode */ static int tridentfb_set_par(struct fb_info *info) { @@ -1475,7 +1470,7 @@ static int trident_pci_probe(struct pci_dev *dev, if (err) return err; - err = pci_enable_device(dev); + err = pcim_enable_device(dev); if (err) return err; @@ -1715,12 +1710,10 @@ out_unmap2: kfree(info->pixmap.addr); if (info->screen_base) iounmap(info->screen_base); - release_mem_region(tridentfb_fix.smem_start, tridentfb_fix.smem_len); disable_mmio(info->par); out_unmap1: if (default_par->io_virt) iounmap(default_par->io_virt); - release_mem_region(tridentfb_fix.mmio_start, tridentfb_fix.mmio_len); framebuffer_release(info); return err; } @@ -1735,8 +1728,6 @@ static void trident_pci_remove(struct pci_dev *dev) i2c_del_adapter(&par->ddc_adapter); iounmap(par->io_virt); iounmap(info->screen_base); - release_mem_region(tridentfb_fix.smem_start, tridentfb_fix.smem_len); - release_mem_region(tridentfb_fix.mmio_start, tridentfb_fix.mmio_len); kfree(info->pixmap.addr); fb_dealloc_cmap(&info->cmap); framebuffer_release(info); diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c index c863244ef12cb8dde40d9f4e61eee03430c45da5..216d49c9d47e5c57f11e93c5c01a56b371cb39b1 100644 --- a/drivers/video/fbdev/udlfb.c +++ b/drivers/video/fbdev/udlfb.c @@ -370,7 +370,7 @@ static int dlfb_trim_hline(const u8 *bback, const u8 **bfront, int *width_bytes) const unsigned long *back = (const unsigned long *) bback; const unsigned long *front = (const unsigned long *) *bfront; const int width = *width_bytes / sizeof(unsigned long); - int identical = width; + int identical; int start = width; int end = width; diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c index 4df6772802d78b1bd2ea1650f7ee6b16a719af4f..00d789b6c0faf022ec824d3db9685893867afe6f 100644 --- a/drivers/video/fbdev/uvesafb.c +++ b/drivers/video/fbdev/uvesafb.c @@ -167,7 +167,7 @@ static int uvesafb_exec(struct uvesafb_ktask *task) memcpy(&m->id, &uvesafb_cn_id, sizeof(m->id)); m->seq = seq; m->len = len; - m->ack = prandom_u32(); + m->ack = get_random_u32(); /* uvesafb_task structure */ memcpy(m + 1, &task->t, sizeof(task->t)); @@ -1580,7 +1580,7 @@ static ssize_t uvesafb_show_vendor(struct device *dev, struct uvesafb_par *par = info->par; if (par->vbe_ib.oem_vendor_name_ptr) - return snprintf(buf, PAGE_SIZE, "%s\n", (char *) + return scnprintf(buf, PAGE_SIZE, "%s\n", (char *) (&par->vbe_ib) + par->vbe_ib.oem_vendor_name_ptr); else return 0; @@ -1595,7 +1595,7 @@ static ssize_t uvesafb_show_product_name(struct device *dev, struct uvesafb_par *par = info->par; if (par->vbe_ib.oem_product_name_ptr) - return snprintf(buf, PAGE_SIZE, "%s\n", (char *) + return scnprintf(buf, PAGE_SIZE, "%s\n", (char *) (&par->vbe_ib) + par->vbe_ib.oem_product_name_ptr); else return 0; @@ -1610,7 +1610,7 @@ static ssize_t uvesafb_show_product_rev(struct device *dev, struct uvesafb_par *par = info->par; if (par->vbe_ib.oem_product_rev_ptr) - return snprintf(buf, PAGE_SIZE, "%s\n", (char *) + return scnprintf(buf, PAGE_SIZE, "%s\n", (char *) (&par->vbe_ib) + par->vbe_ib.oem_product_rev_ptr); else return 0; @@ -1625,7 +1625,7 @@ static ssize_t uvesafb_show_oem_string(struct device *dev, struct uvesafb_par *par = info->par; if (par->vbe_ib.oem_string_ptr) - return snprintf(buf, PAGE_SIZE, "%s\n", + return scnprintf(buf, PAGE_SIZE, "%s\n", (char *)(&par->vbe_ib) + par->vbe_ib.oem_string_ptr); else return 0; @@ -1639,7 +1639,7 @@ static ssize_t uvesafb_show_nocrtc(struct device *dev, struct fb_info *info = dev_get_drvdata(dev); struct uvesafb_par *par = info->par; - return snprintf(buf, PAGE_SIZE, "%d\n", par->nocrtc); + return scnprintf(buf, PAGE_SIZE, "%d\n", par->nocrtc); } static ssize_t uvesafb_store_nocrtc(struct device *dev, diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c index 35cf51ae3292917386d8c12c175e7ab9bb5d29e0..af47f82170956740292e935e55e5829fc03787d6 100644 --- a/drivers/video/fbdev/vga16fb.c +++ b/drivers/video/fbdev/vga16fb.c @@ -1421,6 +1421,7 @@ static const struct platform_device_id vga16fb_driver_id_table[] = { {"vga-framebuffer", 0}, { } }; +MODULE_DEVICE_TABLE(platform, vga16fb_driver_id_table); static struct platform_driver vga16fb_driver = { .probe = vga16fb_probe, diff --git a/drivers/virt/nitro_enclaves/Kconfig b/drivers/virt/nitro_enclaves/Kconfig index ce91add81401a22805d253a585d2b15b11572a31..dc4d25c26256e53a341ca881cbf9fa2561794de0 100644 --- a/drivers/virt/nitro_enclaves/Kconfig +++ b/drivers/virt/nitro_enclaves/Kconfig @@ -17,7 +17,7 @@ config NITRO_ENCLAVES config NITRO_ENCLAVES_MISC_DEV_TEST bool "Tests for the misc device functionality of the Nitro Enclaves" if !KUNIT_ALL_TESTS - depends on NITRO_ENCLAVES && KUNIT + depends on NITRO_ENCLAVES && KUNIT=y default KUNIT_ALL_TESTS help Enable KUnit tests for the misc device functionality of the Nitro diff --git a/drivers/virt/vboxguest/vboxguest_core.c b/drivers/virt/vboxguest/vboxguest_core.c index 0b43efddea22ea3c976a3f2b9e014f832834674a..dfd69bd77f531bb624b60378e47091e76ac917ee 100644 --- a/drivers/virt/vboxguest/vboxguest_core.c +++ b/drivers/virt/vboxguest/vboxguest_core.c @@ -198,7 +198,7 @@ static int vbg_report_guest_info(struct vbg_dev *gdev) req2->additions_revision = VBG_SVN_REV; req2->additions_features = VMMDEV_GUEST_INFO2_ADDITIONS_FEATURES_REQUESTOR_INFO; - strlcpy(req2->name, VBG_VERSION_STRING, + strscpy(req2->name, VBG_VERSION_STRING, sizeof(req2->name)); /* diff --git a/drivers/virt/vboxguest/vboxguest_linux.c b/drivers/virt/vboxguest/vboxguest_linux.c index 4ccfd30c2a304bb24b8bccc19302b1b477e7d9e1..c47e62dc55da80f35bc11298fed353b01dec4d3a 100644 --- a/drivers/virt/vboxguest/vboxguest_linux.c +++ b/drivers/virt/vboxguest/vboxguest_linux.c @@ -270,6 +270,13 @@ static ssize_t host_features_show(struct device *dev, static DEVICE_ATTR_RO(host_version); static DEVICE_ATTR_RO(host_features); +static struct attribute *vbg_pci_attrs[] = { + &dev_attr_host_version.attr, + &dev_attr_host_features.attr, + NULL, +}; +ATTRIBUTE_GROUPS(vbg_pci); + /** * Does the PCI detection and init of the device. * @@ -390,12 +397,6 @@ static int vbg_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) } pci_set_drvdata(pci, gdev); - device_create_file(dev, &dev_attr_host_version); - device_create_file(dev, &dev_attr_host_features); - - vbg_info("vboxguest: misc device minor %d, IRQ %d, I/O port %x, MMIO at %pap (size %pap)\n", - gdev->misc_device.minor, pci->irq, gdev->io_port, - &mmio, &mmio_len); return 0; @@ -422,8 +423,6 @@ static void vbg_pci_remove(struct pci_dev *pci) mutex_unlock(&vbg_gdev_mutex); free_irq(pci->irq, gdev); - device_remove_file(gdev->dev, &dev_attr_host_features); - device_remove_file(gdev->dev, &dev_attr_host_version); misc_deregister(&gdev->misc_device_user); misc_deregister(&gdev->misc_device); vbg_core_exit(gdev); @@ -488,6 +487,7 @@ MODULE_DEVICE_TABLE(pci, vbg_pci_ids); static struct pci_driver vbg_pci_driver = { .name = DEVICE_NAME, + .dev_groups = vbg_pci_groups, .id_table = vbg_pci_ids, .probe = vbg_pci_probe, .remove = vbg_pci_remove, diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ad258a9d3b9f453862e454741e099b6ae750a8a3..a6c86f916dbdf5d87eb9dbaa277bfa60a6ea6a5b 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -409,6 +409,9 @@ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc); if (!err) return 0; + /* Is there an interrupt? If not give up. */ + if (!(to_vp_device(vdev)->pci_dev->irq)) + return err; /* Finally fall back to regular interrupts. */ return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx); } diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 4620e9d79dde8cebff063b473cd59d2641f9f15b..2e7689bb933b8e96322d5ca8d0a0f6162e0b2eae 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -352,8 +353,15 @@ static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg, enum dma_data_direction direction) { - if (!vq->use_dma_api) + if (!vq->use_dma_api) { + /* + * If DMA is not used, KMSAN doesn't know that the scatterlist + * is initialized by the hardware. Explicitly check/unpoison it + * depending on the direction. + */ + kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction); return (dma_addr_t)sg_phys(sg); + } /* * We can't use dma_map_sg, because we don't use scatterlists in @@ -1066,7 +1074,7 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, if (!queue) { /* Try to get a single page. You are my only hope! */ queue = vring_alloc_queue(vdev, vring_size(num, vring_align), - &dma_addr, GFP_KERNEL|__GFP_ZERO); + &dma_addr, GFP_KERNEL | __GFP_ZERO); } if (!queue) return -ENOMEM; @@ -1867,7 +1875,7 @@ static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, ring = vring_alloc_queue(vdev, ring_size_in_bytes, &ring_dma_addr, - GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); + GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); if (!ring) goto err; @@ -1879,7 +1887,7 @@ static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, driver = vring_alloc_queue(vdev, event_size_in_bytes, &driver_event_dma_addr, - GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); + GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); if (!driver) goto err; @@ -1889,7 +1897,7 @@ static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, device = vring_alloc_queue(vdev, event_size_in_bytes, &device_event_dma_addr, - GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); + GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO); if (!device) goto err; diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c index 6c962e88501c0931c405b18dae8a47283fd43c5e..62c44616d8a922637e92f89d79d0fd7f8a8858ea 100644 --- a/drivers/w1/masters/ds2482.c +++ b/drivers/w1/masters/ds2482.c @@ -525,7 +525,7 @@ exit: return err; } -static int ds2482_remove(struct i2c_client *client) +static void ds2482_remove(struct i2c_client *client) { struct ds2482_data *data = i2c_get_clientdata(client); int idx; @@ -538,7 +538,6 @@ static int ds2482_remove(struct i2c_client *client) /* Free the memory */ kfree(data); - return 0; } /* diff --git a/drivers/w1/w1_netlink.c b/drivers/w1/w1_netlink.c index fa490aa4407c03e90dbdfa8fadd2664fd95004da..db110cc442b1307d9ca6164eed9ae0e2c0bf8755 100644 --- a/drivers/w1/w1_netlink.c +++ b/drivers/w1/w1_netlink.c @@ -611,7 +611,8 @@ static void w1_cn_callback(struct cn_msg *cn, struct netlink_skb_parms *nsp) } atomic_set(&block->refcnt, 1); block->portid = nsp->portid; - memcpy(&block->request_cn, cn, sizeof(*cn) + cn->len); + block->request_cn = *cn; + memcpy(block->request_cn.data, cn->data, cn->len); node = (struct w1_cb_node *)(block->request_cn.data + cn->len); /* Sneeky, when not bundling, reply_size is the allocated space diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 9295492d24f74e52ad4642774fea57e7a7795fe2..b64bc49c7f30edda4e422519971430ffe2d7a0c7 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -1089,6 +1089,17 @@ config EBC_C384_WDT WinSystems EBC-C384 motherboard. The timeout may be configured via the timeout module parameter. +config EXAR_WDT + tristate "Exar Watchdog Timer" + depends on X86 + select WATCHDOG_CORE + help + Enables watchdog timer support for the watchdog timer present + in some Exar/MaxLinear UART chips like the XR28V38x. + + To compile this driver as a module, choose M here: the + module will be called exar_wdt. + config F71808E_WDT tristate "Fintek F718xx, F818xx Super I/O Watchdog" depends on X86 @@ -1315,7 +1326,7 @@ config IT87_WDT config HP_WATCHDOG tristate "HP ProLiant iLO2+ Hardware Watchdog Timer" select WATCHDOG_CORE - depends on X86 && PCI + depends on (ARM64 || X86) && PCI help A software monitoring watchdog and NMI handling driver. This driver will detect lockups and provide a stack trace. This is a driver that @@ -1325,7 +1336,7 @@ config HP_WATCHDOG config HPWDT_NMI_DECODING bool "NMI support for the HP ProLiant iLO2+ Hardware Watchdog Timer" - depends on HP_WATCHDOG + depends on X86 && HP_WATCHDOG default y help Enables the NMI handler for the watchdog pretimeout NMI and the iLO @@ -1799,7 +1810,7 @@ config BCM7038_WDT tristate "BCM63xx/BCM7038 Watchdog" select WATCHDOG_CORE depends on HAS_IOMEM - depends on ARCH_BCM4908 || ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST + depends on ARCH_BCMBCA || ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST help Watchdog driver for the built-in hardware in Broadcom 7038 and later SoCs used in set-top boxes. BCM7038 was made public @@ -1935,10 +1946,10 @@ config BOOKE_WDT config BOOKE_WDT_DEFAULT_TIMEOUT int "PowerPC Book-E Watchdog Timer Default Timeout" depends on BOOKE_WDT - default 38 if PPC_FSL_BOOK3E - range 0 63 if PPC_FSL_BOOK3E - default 3 if !PPC_FSL_BOOK3E - range 0 3 if !PPC_FSL_BOOK3E + default 38 if PPC_E500 + range 0 63 if PPC_E500 + default 3 if !PPC_E500 + range 0 3 if !PPC_E500 help Select the default watchdog timer period to be used by the PowerPC Book-E watchdog driver. A watchdog "event" occurs when the bit diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index cdeb119e6e61a0a8a9786893745f9b3ce8485963..d41e5f830ae7f85060835ce790d24a8a0dea8789 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -105,6 +105,7 @@ obj-$(CONFIG_ADVANTECH_WDT) += advantechwdt.o obj-$(CONFIG_ALIM1535_WDT) += alim1535_wdt.o obj-$(CONFIG_ALIM7101_WDT) += alim7101_wdt.o obj-$(CONFIG_EBC_C384_WDT) += ebc-c384_wdt.o +obj-$(CONFIG_EXAR_WDT) += exar_wdt.o obj-$(CONFIG_F71808E_WDT) += f71808e_wdt.o obj-$(CONFIG_SP5100_TCO) += sp5100_tco.o obj-$(CONFIG_GEODE_WDT) += geodewdt.o diff --git a/drivers/watchdog/armada_37xx_wdt.c b/drivers/watchdog/armada_37xx_wdt.c index 854b1cc723cb602a070c903e8a2dc13d76cc979c..ac9fed1ef681b39ea0c4bd53cc08927cc88716a8 100644 --- a/drivers/watchdog/armada_37xx_wdt.c +++ b/drivers/watchdog/armada_37xx_wdt.c @@ -179,6 +179,8 @@ static int armada_37xx_wdt_set_timeout(struct watchdog_device *wdt, dev->timeout = (u64)dev->clk_rate * timeout; do_div(dev->timeout, CNTR_CTRL_PRESCALE_MIN); + set_counter_value(dev, CNTR_ID_WDOG, dev->timeout); + return 0; } diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c index bd06622813eb4ba5d7ef360f93b4dc954c260472..0cff2adfbfc966355630cb3bb7e97f476b1eaa39 100644 --- a/drivers/watchdog/aspeed_wdt.c +++ b/drivers/watchdog/aspeed_wdt.c @@ -332,18 +332,18 @@ static int aspeed_wdt_probe(struct platform_device *pdev) u32 reg = readl(wdt->base + WDT_RESET_WIDTH); reg &= config->ext_pulse_width_mask; - if (of_property_read_bool(np, "aspeed,ext-push-pull")) - reg |= WDT_PUSH_PULL_MAGIC; + if (of_property_read_bool(np, "aspeed,ext-active-high")) + reg |= WDT_ACTIVE_HIGH_MAGIC; else - reg |= WDT_OPEN_DRAIN_MAGIC; + reg |= WDT_ACTIVE_LOW_MAGIC; writel(reg, wdt->base + WDT_RESET_WIDTH); reg &= config->ext_pulse_width_mask; - if (of_property_read_bool(np, "aspeed,ext-active-high")) - reg |= WDT_ACTIVE_HIGH_MAGIC; + if (of_property_read_bool(np, "aspeed,ext-push-pull")) + reg |= WDT_PUSH_PULL_MAGIC; else - reg |= WDT_ACTIVE_LOW_MAGIC; + reg |= WDT_OPEN_DRAIN_MAGIC; writel(reg, wdt->base + WDT_RESET_WIDTH); } diff --git a/drivers/watchdog/bd9576_wdt.c b/drivers/watchdog/bd9576_wdt.c index 0b6999f3b6e83be7f323dcc25e4f7b3992c0d50b..4a20e07fbb699b816237ef40634e3c78e97bb73d 100644 --- a/drivers/watchdog/bd9576_wdt.c +++ b/drivers/watchdog/bd9576_wdt.c @@ -9,8 +9,8 @@ #include #include #include -#include #include +#include #include #include @@ -202,10 +202,10 @@ static int bd957x_set_wdt_mode(struct bd9576_wdt_priv *priv, int hw_margin, static int bd9576_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->parent->of_node; struct bd9576_wdt_priv *priv; u32 hw_margin[2]; u32 hw_margin_max = BD957X_WDT_DEFAULT_MARGIN, hw_margin_min = 0; + int count; int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -221,40 +221,51 @@ static int bd9576_wdt_probe(struct platform_device *pdev) return -ENODEV; } - priv->gpiod_en = devm_gpiod_get_from_of_node(dev, dev->parent->of_node, - "rohm,watchdog-enable-gpios", - 0, GPIOD_OUT_LOW, - "watchdog-enable"); + priv->gpiod_en = devm_fwnode_gpiod_get(dev, dev_fwnode(dev->parent), + "rohm,watchdog-enable", + GPIOD_OUT_LOW, + "watchdog-enable"); if (IS_ERR(priv->gpiod_en)) return dev_err_probe(dev, PTR_ERR(priv->gpiod_en), "getting watchdog-enable GPIO failed\n"); - priv->gpiod_ping = devm_gpiod_get_from_of_node(dev, dev->parent->of_node, - "rohm,watchdog-ping-gpios", - 0, GPIOD_OUT_LOW, - "watchdog-ping"); + priv->gpiod_ping = devm_fwnode_gpiod_get(dev, dev_fwnode(dev->parent), + "rohm,watchdog-ping", + GPIOD_OUT_LOW, + "watchdog-ping"); if (IS_ERR(priv->gpiod_ping)) return dev_err_probe(dev, PTR_ERR(priv->gpiod_ping), "getting watchdog-ping GPIO failed\n"); - ret = of_property_read_variable_u32_array(np, "rohm,hw-timeout-ms", - &hw_margin[0], 1, 2); - if (ret < 0 && ret != -EINVAL) - return ret; + count = device_property_count_u32(dev->parent, "rohm,hw-timeout-ms"); + if (count < 0 && count != -EINVAL) + return count; + + if (count > 0) { + if (count > ARRAY_SIZE(hw_margin)) + return -EINVAL; - if (ret == 1) - hw_margin_max = hw_margin[0]; + ret = device_property_read_u32_array(dev->parent, + "rohm,hw-timeout-ms", + hw_margin, count); + if (ret < 0) + return ret; - if (ret == 2) { - hw_margin_max = hw_margin[1]; - hw_margin_min = hw_margin[0]; + if (count == 1) + hw_margin_max = hw_margin[0]; + + if (count == 2) { + hw_margin_max = hw_margin[1]; + hw_margin_min = hw_margin[0]; + } } ret = bd957x_set_wdt_mode(priv, hw_margin_max, hw_margin_min); if (ret) return ret; - priv->always_running = of_property_read_bool(np, "always-running"); + priv->always_running = device_property_read_bool(dev->parent, + "always-running"); watchdog_set_drvdata(&priv->wdd, priv); diff --git a/drivers/watchdog/booke_wdt.c b/drivers/watchdog/booke_wdt.c index 75da5cd0261544fd68bd30c20fc8147c7799240d..932a03f4436a35fbcfffffa49c232c3cd7db27ec 100644 --- a/drivers/watchdog/booke_wdt.c +++ b/drivers/watchdog/booke_wdt.c @@ -27,7 +27,7 @@ */ -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 #define WDTP(x) ((((x)&0x3)<<30)|(((x)&0x3c)<<15)) #define WDTP_MASK (WDTP(0x3f)) #else @@ -45,7 +45,7 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); -#ifdef CONFIG_PPC_FSL_BOOK3E +#ifdef CONFIG_PPC_E500 /* For the specified period, determine the number of seconds * corresponding to the reset time. There will be a watchdog @@ -88,7 +88,7 @@ static unsigned int sec_to_period(unsigned int secs) #define MAX_WDT_TIMEOUT period_to_sec(1) -#else /* CONFIG_PPC_FSL_BOOK3E */ +#else /* CONFIG_PPC_E500 */ static unsigned long long period_to_sec(unsigned int period) { @@ -102,7 +102,7 @@ static unsigned int sec_to_period(unsigned int secs) #define MAX_WDT_TIMEOUT 3 /* from Kconfig */ -#endif /* !CONFIG_PPC_FSL_BOOK3E */ +#endif /* !CONFIG_PPC_E500 */ static void __booke_wdt_set(void *data) { diff --git a/drivers/watchdog/eurotechwdt.c b/drivers/watchdog/eurotechwdt.c index ce682942662cd05970aff6a78f582c4df0f22d48..e26609ad4c17c0df917f84d27cbd2d817ec7318e 100644 --- a/drivers/watchdog/eurotechwdt.c +++ b/drivers/watchdog/eurotechwdt.c @@ -192,7 +192,7 @@ static void eurwdt_ping(void) * @ppos: pointer to the position to write. No seeks allowed * * A write to a watchdog device is defined as a keepalive signal. Any - * write of data will do, as we we don't define content meaning. + * write of data will do, as we don't define content meaning. */ static ssize_t eurwdt_write(struct file *file, const char __user *buf, diff --git a/drivers/watchdog/exar_wdt.c b/drivers/watchdog/exar_wdt.c new file mode 100644 index 0000000000000000000000000000000000000000..35058d8b21bc71334822bd9491b0a9f327835a53 --- /dev/null +++ b/drivers/watchdog/exar_wdt.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * exar_wdt.c - Driver for the watchdog present in some + * Exar/MaxLinear UART chips like the XR28V38x. + * + * (c) Copyright 2022 D. Müller . + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "exar_wdt" + +static const unsigned short sio_config_ports[] = { 0x2e, 0x4e }; +static const unsigned char sio_enter_keys[] = { 0x67, 0x77, 0x87, 0xA0 }; +#define EXAR_EXIT_KEY 0xAA + +#define EXAR_LDN 0x07 +#define EXAR_DID 0x20 +#define EXAR_VID 0x23 +#define EXAR_WDT 0x26 +#define EXAR_ACT 0x30 +#define EXAR_RTBASE 0x60 + +#define EXAR_WDT_LDEV 0x08 + +#define EXAR_VEN_ID 0x13A8 +#define EXAR_DEV_382 0x0382 +#define EXAR_DEV_384 0x0384 + +/* WDT runtime registers */ +#define WDT_CTRL 0x00 +#define WDT_VAL 0x01 + +#define WDT_UNITS_10MS 0x0 /* the 10 millisec unit of the HW is not used */ +#define WDT_UNITS_SEC 0x2 +#define WDT_UNITS_MIN 0x4 + +/* default WDT control for WDTOUT signal activ / rearm by read */ +#define EXAR_WDT_DEF_CONF 0 + +struct wdt_pdev_node { + struct list_head list; + struct platform_device *pdev; + const char name[16]; +}; + +struct wdt_priv { + /* the lock for WDT io operations */ + spinlock_t io_lock; + struct resource wdt_res; + struct watchdog_device wdt_dev; + unsigned short did; + unsigned short config_port; + unsigned char enter_key; + unsigned char unit; + unsigned char timeout; +}; + +#define WATCHDOG_TIMEOUT 60 + +static int timeout = WATCHDOG_TIMEOUT; +module_param(timeout, int, 0); +MODULE_PARM_DESC(timeout, + "Watchdog timeout in seconds. 1<=timeout<=15300, default=" + __MODULE_STRING(WATCHDOG_TIMEOUT) "."); + +static bool nowayout = WATCHDOG_NOWAYOUT; +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, + "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); + +static int exar_sio_enter(const unsigned short config_port, + const unsigned char key) +{ + if (!request_muxed_region(config_port, 2, DRV_NAME)) + return -EBUSY; + + /* write the ENTER-KEY twice */ + outb(key, config_port); + outb(key, config_port); + + return 0; +} + +static void exar_sio_exit(const unsigned short config_port) +{ + outb(EXAR_EXIT_KEY, config_port); + release_region(config_port, 2); +} + +static unsigned char exar_sio_read(const unsigned short config_port, + const unsigned char reg) +{ + outb(reg, config_port); + return inb(config_port + 1); +} + +static void exar_sio_write(const unsigned short config_port, + const unsigned char reg, const unsigned char val) +{ + outb(reg, config_port); + outb(val, config_port + 1); +} + +static unsigned short exar_sio_read16(const unsigned short config_port, + const unsigned char reg) +{ + unsigned char msb, lsb; + + msb = exar_sio_read(config_port, reg); + lsb = exar_sio_read(config_port, reg + 1); + + return (msb << 8) | lsb; +} + +static void exar_sio_select_wdt(const unsigned short config_port) +{ + exar_sio_write(config_port, EXAR_LDN, EXAR_WDT_LDEV); +} + +static void exar_wdt_arm(const struct wdt_priv *priv) +{ + unsigned short rt_base = priv->wdt_res.start; + + /* write timeout value twice to arm watchdog */ + outb(priv->timeout, rt_base + WDT_VAL); + outb(priv->timeout, rt_base + WDT_VAL); +} + +static void exar_wdt_disarm(const struct wdt_priv *priv) +{ + unsigned short rt_base = priv->wdt_res.start; + + /* + * use two accesses with different values to make sure + * that a combination of a previous single access and + * the ones below with the same value are not falsely + * interpreted as "arm watchdog" + */ + outb(0xFF, rt_base + WDT_VAL); + outb(0, rt_base + WDT_VAL); +} + +static int exar_wdt_start(struct watchdog_device *wdog) +{ + struct wdt_priv *priv = watchdog_get_drvdata(wdog); + unsigned short rt_base = priv->wdt_res.start; + + spin_lock(&priv->io_lock); + + exar_wdt_disarm(priv); + outb(priv->unit, rt_base + WDT_CTRL); + exar_wdt_arm(priv); + + spin_unlock(&priv->io_lock); + return 0; +} + +static int exar_wdt_stop(struct watchdog_device *wdog) +{ + struct wdt_priv *priv = watchdog_get_drvdata(wdog); + + spin_lock(&priv->io_lock); + + exar_wdt_disarm(priv); + + spin_unlock(&priv->io_lock); + return 0; +} + +static int exar_wdt_keepalive(struct watchdog_device *wdog) +{ + struct wdt_priv *priv = watchdog_get_drvdata(wdog); + unsigned short rt_base = priv->wdt_res.start; + + spin_lock(&priv->io_lock); + + /* reading the WDT_VAL reg will feed the watchdog */ + inb(rt_base + WDT_VAL); + + spin_unlock(&priv->io_lock); + return 0; +} + +static int exar_wdt_set_timeout(struct watchdog_device *wdog, unsigned int t) +{ + struct wdt_priv *priv = watchdog_get_drvdata(wdog); + bool unit_min = false; + + /* + * if new timeout is bigger then 255 seconds, change the + * unit to minutes and round the timeout up to the next whole minute + */ + if (t > 255) { + unit_min = true; + t = DIV_ROUND_UP(t, 60); + } + + /* save for later use in exar_wdt_start() */ + priv->unit = unit_min ? WDT_UNITS_MIN : WDT_UNITS_SEC; + priv->timeout = t; + + wdog->timeout = unit_min ? t * 60 : t; + + if (watchdog_hw_running(wdog)) + exar_wdt_start(wdog); + + return 0; +} + +static const struct watchdog_info exar_wdt_info = { + .options = WDIOF_KEEPALIVEPING | + WDIOF_SETTIMEOUT | + WDIOF_MAGICCLOSE, + .identity = "Exar/MaxLinear XR28V38x Watchdog", +}; + +static const struct watchdog_ops exar_wdt_ops = { + .owner = THIS_MODULE, + .start = exar_wdt_start, + .stop = exar_wdt_stop, + .ping = exar_wdt_keepalive, + .set_timeout = exar_wdt_set_timeout, +}; + +static int exar_wdt_config(struct watchdog_device *wdog, + const unsigned char conf) +{ + struct wdt_priv *priv = watchdog_get_drvdata(wdog); + int ret; + + ret = exar_sio_enter(priv->config_port, priv->enter_key); + if (ret) + return ret; + + exar_sio_select_wdt(priv->config_port); + exar_sio_write(priv->config_port, EXAR_WDT, conf); + + exar_sio_exit(priv->config_port); + + return 0; +} + +static int __init exar_wdt_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct wdt_priv *priv = dev->platform_data; + struct watchdog_device *wdt_dev = &priv->wdt_dev; + struct resource *res; + int ret; + + res = platform_get_resource(pdev, IORESOURCE_IO, 0); + if (!res) + return -ENXIO; + + spin_lock_init(&priv->io_lock); + + wdt_dev->info = &exar_wdt_info; + wdt_dev->ops = &exar_wdt_ops; + wdt_dev->min_timeout = 1; + wdt_dev->max_timeout = 255 * 60; + + watchdog_init_timeout(wdt_dev, timeout, NULL); + watchdog_set_nowayout(wdt_dev, nowayout); + watchdog_stop_on_reboot(wdt_dev); + watchdog_stop_on_unregister(wdt_dev); + watchdog_set_drvdata(wdt_dev, priv); + + ret = exar_wdt_config(wdt_dev, EXAR_WDT_DEF_CONF); + if (ret) + return ret; + + exar_wdt_set_timeout(wdt_dev, timeout); + /* Make sure that the watchdog is not running */ + exar_wdt_stop(wdt_dev); + + ret = devm_watchdog_register_device(dev, wdt_dev); + if (ret) + return ret; + + dev_info(dev, "XR28V%X WDT initialized. timeout=%d sec (nowayout=%d)\n", + priv->did, timeout, nowayout); + + return 0; +} + +static unsigned short __init exar_detect(const unsigned short config_port, + const unsigned char key, + unsigned short *rt_base) +{ + int ret; + unsigned short base = 0; + unsigned short vid, did; + + ret = exar_sio_enter(config_port, key); + if (ret) + return 0; + + vid = exar_sio_read16(config_port, EXAR_VID); + did = exar_sio_read16(config_port, EXAR_DID); + + /* check for the vendor and device IDs we currently know about */ + if (vid == EXAR_VEN_ID && + (did == EXAR_DEV_382 || + did == EXAR_DEV_384)) { + exar_sio_select_wdt(config_port); + /* is device active? */ + if (exar_sio_read(config_port, EXAR_ACT) == 0x01) + base = exar_sio_read16(config_port, EXAR_RTBASE); + } + + exar_sio_exit(config_port); + + if (base) { + pr_debug("Found a XR28V%X WDT (conf: 0x%x / rt: 0x%04x)\n", + did, config_port, base); + *rt_base = base; + return did; + } + + return 0; +} + +static struct platform_driver exar_wdt_driver = { + .driver = { + .name = DRV_NAME, + }, +}; + +static LIST_HEAD(pdev_list); + +static int __init exar_wdt_register(struct wdt_priv *priv, const int idx) +{ + struct wdt_pdev_node *n; + + n = kzalloc(sizeof(*n), GFP_KERNEL); + if (!n) + return -ENOMEM; + + INIT_LIST_HEAD(&n->list); + + scnprintf((char *)n->name, sizeof(n->name), DRV_NAME ".%d", idx); + priv->wdt_res.name = n->name; + + n->pdev = platform_device_register_resndata(NULL, DRV_NAME, idx, + &priv->wdt_res, 1, + priv, sizeof(*priv)); + if (IS_ERR(n->pdev)) { + kfree(n); + return PTR_ERR(n->pdev); + } + + list_add_tail(&n->list, &pdev_list); + + return 0; +} + +static void exar_wdt_unregister(void) +{ + struct wdt_pdev_node *n, *t; + + list_for_each_entry_safe(n, t, &pdev_list, list) { + platform_device_unregister(n->pdev); + list_del(&n->list); + kfree(n); + } +} + +static int __init exar_wdt_init(void) +{ + int ret, i, j, idx = 0; + + /* search for active Exar watchdogs on all possible locations */ + for (i = 0; i < ARRAY_SIZE(sio_config_ports); i++) { + for (j = 0; j < ARRAY_SIZE(sio_enter_keys); j++) { + unsigned short did, rt_base = 0; + + did = exar_detect(sio_config_ports[i], + sio_enter_keys[j], + &rt_base); + + if (did) { + struct wdt_priv priv = { + .wdt_res = DEFINE_RES_IO(rt_base, 2), + .did = did, + .config_port = sio_config_ports[i], + .enter_key = sio_enter_keys[j], + }; + + ret = exar_wdt_register(&priv, idx); + if (!ret) + idx++; + } + } + } + + if (!idx) + return -ENODEV; + + ret = platform_driver_probe(&exar_wdt_driver, exar_wdt_probe); + if (ret) + exar_wdt_unregister(); + + return ret; +} + +static void __exit exar_wdt_exit(void) +{ + exar_wdt_unregister(); + platform_driver_unregister(&exar_wdt_driver); +} + +module_init(exar_wdt_init); +module_exit(exar_wdt_exit); + +MODULE_AUTHOR("David Müller "); +MODULE_DESCRIPTION("Exar/MaxLinear Watchdog Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/watchdog/ftwdt010_wdt.c b/drivers/watchdog/ftwdt010_wdt.c index 21dcc7765688a3ceea01315fd0f9586d1ac2956e..442c5bf63ff4d95723998c854b57609d04b3a558 100644 --- a/drivers/watchdog/ftwdt010_wdt.c +++ b/drivers/watchdog/ftwdt010_wdt.c @@ -47,21 +47,28 @@ struct ftwdt010_wdt *to_ftwdt010_wdt(struct watchdog_device *wdd) return container_of(wdd, struct ftwdt010_wdt, wdd); } -static int ftwdt010_wdt_start(struct watchdog_device *wdd) +static void ftwdt010_enable(struct ftwdt010_wdt *gwdt, + unsigned int timeout, + bool need_irq) { - struct ftwdt010_wdt *gwdt = to_ftwdt010_wdt(wdd); u32 enable; - writel(wdd->timeout * WDT_CLOCK, gwdt->base + FTWDT010_WDLOAD); + writel(timeout * WDT_CLOCK, gwdt->base + FTWDT010_WDLOAD); writel(WDRESTART_MAGIC, gwdt->base + FTWDT010_WDRESTART); /* set clock before enabling */ enable = WDCR_CLOCK_5MHZ | WDCR_SYS_RST; writel(enable, gwdt->base + FTWDT010_WDCR); - if (gwdt->has_irq) + if (need_irq) enable |= WDCR_WDINTR; enable |= WDCR_ENABLE; writel(enable, gwdt->base + FTWDT010_WDCR); +} +static int ftwdt010_wdt_start(struct watchdog_device *wdd) +{ + struct ftwdt010_wdt *gwdt = to_ftwdt010_wdt(wdd); + + ftwdt010_enable(gwdt, wdd->timeout, gwdt->has_irq); return 0; } @@ -93,6 +100,13 @@ static int ftwdt010_wdt_set_timeout(struct watchdog_device *wdd, return 0; } +static int ftwdt010_wdt_restart(struct watchdog_device *wdd, + unsigned long action, void *data) +{ + ftwdt010_enable(to_ftwdt010_wdt(wdd), 0, false); + return 0; +} + static irqreturn_t ftwdt010_wdt_interrupt(int irq, void *data) { struct ftwdt010_wdt *gwdt = data; @@ -107,6 +121,7 @@ static const struct watchdog_ops ftwdt010_wdt_ops = { .stop = ftwdt010_wdt_stop, .ping = ftwdt010_wdt_ping, .set_timeout = ftwdt010_wdt_set_timeout, + .restart = ftwdt010_wdt_restart, .owner = THIS_MODULE, }; @@ -156,7 +171,7 @@ static int ftwdt010_wdt_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (irq) { + if (irq > 0) { ret = devm_request_irq(dev, irq, ftwdt010_wdt_interrupt, 0, "watchdog bark", gwdt); if (ret) diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c index a5006a58e0dbb680167197c0c9466e6982dc9c2e..f79f932bca14898fc70ca2dd58fc3f5b4320ec78 100644 --- a/drivers/watchdog/hpwdt.c +++ b/drivers/watchdog/hpwdt.c @@ -20,7 +20,9 @@ #include #include #include +#ifdef CONFIG_HPWDT_NMI_DECODING #include +#endif #include #define HPWDT_VERSION "2.0.4" diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 922b603742952c216939a224f2c5774bebba1f87..2897902090b3971d1fbc2bf0abc72d868a68dbd2 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -9,12 +9,15 @@ #include #include #include +#include #include #include #include #define WDOG_CS 0x0 +#define WDOG_CS_FLG BIT(14) #define WDOG_CS_CMD32EN BIT(13) +#define WDOG_CS_PRES BIT(12) #define WDOG_CS_ULK BIT(11) #define WDOG_CS_RCS BIT(10) #define LPO_CLK 0x1 @@ -39,60 +42,105 @@ #define DEFAULT_TIMEOUT 60 #define MAX_TIMEOUT 128 #define WDOG_CLOCK_RATE 1000 -#define WDOG_WAIT_TIMEOUT 20 +#define WDOG_ULK_WAIT_TIMEOUT 1000 +#define WDOG_RCS_WAIT_TIMEOUT 10000 +#define WDOG_RCS_POST_WAIT 3000 + +#define RETRY_MAX 5 static bool nowayout = WATCHDOG_NOWAYOUT; module_param(nowayout, bool, 0000); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +struct imx_wdt_hw_feature { + bool prescaler_enable; + u32 wdog_clock_rate; +}; + struct imx7ulp_wdt_device { struct watchdog_device wdd; void __iomem *base; struct clk *clk; + bool post_rcs_wait; + const struct imx_wdt_hw_feature *hw; }; -static int imx7ulp_wdt_wait(void __iomem *base, u32 mask) +static int imx7ulp_wdt_wait_ulk(void __iomem *base) { u32 val = readl(base + WDOG_CS); - if (!(val & mask) && readl_poll_timeout_atomic(base + WDOG_CS, val, - val & mask, 0, - WDOG_WAIT_TIMEOUT)) + if (!(val & WDOG_CS_ULK) && + readl_poll_timeout_atomic(base + WDOG_CS, val, + val & WDOG_CS_ULK, 0, + WDOG_ULK_WAIT_TIMEOUT)) return -ETIMEDOUT; return 0; } -static int imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable) +static int imx7ulp_wdt_wait_rcs(struct imx7ulp_wdt_device *wdt) { - struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); + int ret = 0; + u32 val = readl(wdt->base + WDOG_CS); + u64 timeout = (val & WDOG_CS_PRES) ? + WDOG_RCS_WAIT_TIMEOUT * 256 : WDOG_RCS_WAIT_TIMEOUT; + unsigned long wait_min = (val & WDOG_CS_PRES) ? + WDOG_RCS_POST_WAIT * 256 : WDOG_RCS_POST_WAIT; + + if (!(val & WDOG_CS_RCS) && + readl_poll_timeout(wdt->base + WDOG_CS, val, val & WDOG_CS_RCS, 100, + timeout)) + ret = -ETIMEDOUT; + + /* Wait 2.5 clocks after RCS done */ + if (wdt->post_rcs_wait) + usleep_range(wait_min, wait_min + 2000); + + return ret; +} +static int _imx7ulp_wdt_enable(struct imx7ulp_wdt_device *wdt, bool enable) +{ u32 val = readl(wdt->base + WDOG_CS); int ret; local_irq_disable(); writel(UNLOCK, wdt->base + WDOG_CNT); - ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK); + ret = imx7ulp_wdt_wait_ulk(wdt->base); if (ret) goto enable_out; if (enable) writel(val | WDOG_CS_EN, wdt->base + WDOG_CS); else writel(val & ~WDOG_CS_EN, wdt->base + WDOG_CS); - imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS); -enable_out: local_irq_enable(); + ret = imx7ulp_wdt_wait_rcs(wdt); return ret; + +enable_out: + local_irq_enable(); + return ret; } -static bool imx7ulp_wdt_is_enabled(void __iomem *base) +static int imx7ulp_wdt_enable(struct watchdog_device *wdog, bool enable) { - u32 val = readl(base + WDOG_CS); + struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); + int ret; + u32 val; + u32 loop = RETRY_MAX; + + do { + ret = _imx7ulp_wdt_enable(wdt, enable); + val = readl(wdt->base + WDOG_CS); + } while (--loop > 0 && ((!!(val & WDOG_CS_EN)) != enable || ret)); - return val & WDOG_CS_EN; + if (loop == 0) + return -EBUSY; + + return ret; } static int imx7ulp_wdt_ping(struct watchdog_device *wdog) @@ -114,26 +162,44 @@ static int imx7ulp_wdt_stop(struct watchdog_device *wdog) return imx7ulp_wdt_enable(wdog, false); } -static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog, - unsigned int timeout) +static int _imx7ulp_wdt_set_timeout(struct imx7ulp_wdt_device *wdt, + unsigned int toval) { - struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); - u32 val = WDOG_CLOCK_RATE * timeout; int ret; local_irq_disable(); writel(UNLOCK, wdt->base + WDOG_CNT); - ret = imx7ulp_wdt_wait(wdt->base, WDOG_CS_ULK); + ret = imx7ulp_wdt_wait_ulk(wdt->base); if (ret) goto timeout_out; - writel(val, wdt->base + WDOG_TOVAL); - imx7ulp_wdt_wait(wdt->base, WDOG_CS_RCS); - - wdog->timeout = timeout; + writel(toval, wdt->base + WDOG_TOVAL); + local_irq_enable(); + ret = imx7ulp_wdt_wait_rcs(wdt); + return ret; timeout_out: local_irq_enable(); + return ret; +} +static int imx7ulp_wdt_set_timeout(struct watchdog_device *wdog, + unsigned int timeout) +{ + struct imx7ulp_wdt_device *wdt = watchdog_get_drvdata(wdog); + u32 toval = wdt->hw->wdog_clock_rate * timeout; + u32 val; + int ret; + u32 loop = RETRY_MAX; + + do { + ret = _imx7ulp_wdt_set_timeout(wdt, toval); + val = readl(wdt->base + WDOG_TOVAL); + } while (--loop > 0 && (val != toval || ret)); + + if (loop == 0) + return -EBUSY; + + wdog->timeout = timeout; return ret; } @@ -173,29 +239,62 @@ static const struct watchdog_info imx7ulp_wdt_info = { WDIOF_MAGICCLOSE, }; -static int imx7ulp_wdt_init(void __iomem *base, unsigned int timeout) +static int _imx7ulp_wdt_init(struct imx7ulp_wdt_device *wdt, unsigned int timeout, unsigned int cs) { u32 val; int ret; local_irq_disable(); - /* unlock the wdog for reconfiguration */ - writel_relaxed(UNLOCK_SEQ0, base + WDOG_CNT); - writel_relaxed(UNLOCK_SEQ1, base + WDOG_CNT); - ret = imx7ulp_wdt_wait(base, WDOG_CS_ULK); + + val = readl(wdt->base + WDOG_CS); + if (val & WDOG_CS_CMD32EN) { + writel(UNLOCK, wdt->base + WDOG_CNT); + } else { + mb(); + /* unlock the wdog for reconfiguration */ + writel_relaxed(UNLOCK_SEQ0, wdt->base + WDOG_CNT); + writel_relaxed(UNLOCK_SEQ1, wdt->base + WDOG_CNT); + mb(); + } + + ret = imx7ulp_wdt_wait_ulk(wdt->base); if (ret) goto init_out; /* set an initial timeout value in TOVAL */ - writel(timeout, base + WDOG_TOVAL); - /* enable 32bit command sequence and reconfigure */ - val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE | - WDOG_CS_WAIT | WDOG_CS_STOP; - writel(val, base + WDOG_CS); - imx7ulp_wdt_wait(base, WDOG_CS_RCS); + writel(timeout, wdt->base + WDOG_TOVAL); + writel(cs, wdt->base + WDOG_CS); + local_irq_enable(); + ret = imx7ulp_wdt_wait_rcs(wdt); + + return ret; init_out: local_irq_enable(); + return ret; +} + +static int imx7ulp_wdt_init(struct imx7ulp_wdt_device *wdt, unsigned int timeout) +{ + /* enable 32bit command sequence and reconfigure */ + u32 val = WDOG_CS_CMD32EN | WDOG_CS_CLK | WDOG_CS_UPDATE | + WDOG_CS_WAIT | WDOG_CS_STOP; + u32 cs, toval; + int ret; + u32 loop = RETRY_MAX; + + if (wdt->hw->prescaler_enable) + val |= WDOG_CS_PRES; + + do { + ret = _imx7ulp_wdt_init(wdt, timeout, val); + toval = readl(wdt->base + WDOG_TOVAL); + cs = readl(wdt->base + WDOG_CS); + cs &= ~(WDOG_CS_FLG | WDOG_CS_ULK | WDOG_CS_RCS); + } while (--loop > 0 && (cs != val || toval != timeout || ret)); + + if (loop == 0) + return -EBUSY; return ret; } @@ -228,6 +327,15 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) return PTR_ERR(imx7ulp_wdt->clk); } + imx7ulp_wdt->post_rcs_wait = true; + if (of_device_is_compatible(dev->of_node, + "fsl,imx8ulp-wdt")) { + dev_info(dev, "imx8ulp wdt probe\n"); + imx7ulp_wdt->post_rcs_wait = false; + } else { + dev_info(dev, "imx7ulp wdt probe\n"); + } + ret = clk_prepare_enable(imx7ulp_wdt->clk); if (ret) return ret; @@ -248,14 +356,16 @@ static int imx7ulp_wdt_probe(struct platform_device *pdev) watchdog_stop_on_reboot(wdog); watchdog_stop_on_unregister(wdog); watchdog_set_drvdata(wdog, imx7ulp_wdt); - ret = imx7ulp_wdt_init(imx7ulp_wdt->base, wdog->timeout * WDOG_CLOCK_RATE); + + imx7ulp_wdt->hw = of_device_get_match_data(dev); + ret = imx7ulp_wdt_init(imx7ulp_wdt, wdog->timeout * imx7ulp_wdt->hw->wdog_clock_rate); if (ret) return ret; return devm_watchdog_register_device(dev, wdog); } -static int __maybe_unused imx7ulp_wdt_suspend(struct device *dev) +static int __maybe_unused imx7ulp_wdt_suspend_noirq(struct device *dev) { struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev); @@ -267,30 +377,44 @@ static int __maybe_unused imx7ulp_wdt_suspend(struct device *dev) return 0; } -static int __maybe_unused imx7ulp_wdt_resume(struct device *dev) +static int __maybe_unused imx7ulp_wdt_resume_noirq(struct device *dev) { struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev); - u32 timeout = imx7ulp_wdt->wdd.timeout * WDOG_CLOCK_RATE; + u32 timeout = imx7ulp_wdt->wdd.timeout * imx7ulp_wdt->hw->wdog_clock_rate; int ret; ret = clk_prepare_enable(imx7ulp_wdt->clk); if (ret) return ret; - if (imx7ulp_wdt_is_enabled(imx7ulp_wdt->base)) - imx7ulp_wdt_init(imx7ulp_wdt->base, timeout); - - if (watchdog_active(&imx7ulp_wdt->wdd)) + if (watchdog_active(&imx7ulp_wdt->wdd)) { + imx7ulp_wdt_init(imx7ulp_wdt, timeout); imx7ulp_wdt_start(&imx7ulp_wdt->wdd); + imx7ulp_wdt_ping(&imx7ulp_wdt->wdd); + } return 0; } -static SIMPLE_DEV_PM_OPS(imx7ulp_wdt_pm_ops, imx7ulp_wdt_suspend, - imx7ulp_wdt_resume); +static const struct dev_pm_ops imx7ulp_wdt_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx7ulp_wdt_suspend_noirq, + imx7ulp_wdt_resume_noirq) +}; + +static const struct imx_wdt_hw_feature imx7ulp_wdt_hw = { + .prescaler_enable = false, + .wdog_clock_rate = 1000, +}; + +static const struct imx_wdt_hw_feature imx93_wdt_hw = { + .prescaler_enable = true, + .wdog_clock_rate = 125, +}; static const struct of_device_id imx7ulp_wdt_dt_ids[] = { - { .compatible = "fsl,imx7ulp-wdt", }, + { .compatible = "fsl,imx8ulp-wdt", .data = &imx7ulp_wdt_hw, }, + { .compatible = "fsl,imx7ulp-wdt", .data = &imx7ulp_wdt_hw, }, + { .compatible = "fsl,imx93-wdt", .data = &imx93_wdt_hw, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, imx7ulp_wdt_dt_ids); diff --git a/drivers/watchdog/meson_gxbb_wdt.c b/drivers/watchdog/meson_gxbb_wdt.c index d3c9e2f6e63b9956f51fd2ae4d61d7e267ebfc31..981a2f7c3bec2641c9f551e11c9767149215f363 100644 --- a/drivers/watchdog/meson_gxbb_wdt.c +++ b/drivers/watchdog/meson_gxbb_wdt.c @@ -156,6 +156,7 @@ static int meson_gxbb_wdt_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct meson_gxbb_wdt *data; int ret; + u32 ctrl_reg; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -189,13 +190,26 @@ static int meson_gxbb_wdt_probe(struct platform_device *pdev) watchdog_set_nowayout(&data->wdt_dev, nowayout); watchdog_set_drvdata(&data->wdt_dev, data); + ctrl_reg = readl(data->reg_base + GXBB_WDT_CTRL_REG) & + GXBB_WDT_CTRL_EN; + + if (ctrl_reg) { + /* Watchdog is running - keep it running but extend timeout + * to the maximum while setting the timebase + */ + set_bit(WDOG_HW_RUNNING, &data->wdt_dev.status); + meson_gxbb_wdt_set_timeout(&data->wdt_dev, + GXBB_WDT_TCNT_SETUP_MASK / 1000); + } + /* Setup with 1ms timebase */ - writel(((clk_get_rate(data->clk) / 1000) & GXBB_WDT_CTRL_DIV_MASK) | - GXBB_WDT_CTRL_EE_RESET | - GXBB_WDT_CTRL_CLK_EN | - GXBB_WDT_CTRL_CLKDIV_EN, - data->reg_base + GXBB_WDT_CTRL_REG); + ctrl_reg |= ((clk_get_rate(data->clk) / 1000) & + GXBB_WDT_CTRL_DIV_MASK) | + GXBB_WDT_CTRL_EE_RESET | + GXBB_WDT_CTRL_CLK_EN | + GXBB_WDT_CTRL_CLKDIV_EN; + writel(ctrl_reg, data->reg_base + GXBB_WDT_CTRL_REG); meson_gxbb_wdt_set_timeout(&data->wdt_dev, data->wdt_dev.timeout); return devm_watchdog_register_device(dev, &data->wdt_dev); diff --git a/drivers/watchdog/npcm_wdt.c b/drivers/watchdog/npcm_wdt.c index 28a24caa2627c924416ef628cc548fa6383d2da2..a5dd1c2301374402ae4d28265798438ddd8cba34 100644 --- a/drivers/watchdog/npcm_wdt.c +++ b/drivers/watchdog/npcm_wdt.c @@ -3,6 +3,7 @@ // Copyright (c) 2018 IBM Corp. #include +#include #include #include #include @@ -43,6 +44,7 @@ struct npcm_wdt { struct watchdog_device wdd; void __iomem *reg; + struct clk *clk; }; static inline struct npcm_wdt *to_npcm_wdt(struct watchdog_device *wdd) @@ -66,6 +68,9 @@ static int npcm_wdt_start(struct watchdog_device *wdd) struct npcm_wdt *wdt = to_npcm_wdt(wdd); u32 val; + if (wdt->clk) + clk_prepare_enable(wdt->clk); + if (wdd->timeout < 2) val = 0x800; else if (wdd->timeout < 3) @@ -100,6 +105,9 @@ static int npcm_wdt_stop(struct watchdog_device *wdd) writel(0, wdt->reg); + if (wdt->clk) + clk_disable_unprepare(wdt->clk); + return 0; } @@ -147,6 +155,10 @@ static int npcm_wdt_restart(struct watchdog_device *wdd, { struct npcm_wdt *wdt = to_npcm_wdt(wdd); + /* For reset, we start the WDT clock and leave it running. */ + if (wdt->clk) + clk_prepare_enable(wdt->clk); + writel(NPCM_WTR | NPCM_WTRE | NPCM_WTE, wdt->reg); udelay(1000); @@ -191,6 +203,10 @@ static int npcm_wdt_probe(struct platform_device *pdev) if (IS_ERR(wdt->reg)) return PTR_ERR(wdt->reg); + wdt->clk = devm_clk_get_optional(&pdev->dev, NULL); + if (IS_ERR(wdt->clk)) + return PTR_ERR(wdt->clk); + irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c index 053ef3bde12d41bd873e432c2c0e1631c2bd064e..6e9253761fc1032bdedbc09dd94e99479a00d87c 100644 --- a/drivers/watchdog/rti_wdt.c +++ b/drivers/watchdog/rti_wdt.c @@ -225,9 +225,8 @@ static int rti_wdt_probe(struct platform_device *pdev) wdt->freq = wdt->freq * 9 / 10; pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { - pm_runtime_put_noidle(dev); pm_runtime_disable(&pdev->dev); return dev_err_probe(dev, ret, "runtime pm failed\n"); } diff --git a/drivers/watchdog/rzg2l_wdt.c b/drivers/watchdog/rzg2l_wdt.c index 6eea0ee4af49eae9b9a98c2c9a8878118bf89214..974a4194a8fd67b940ef47263e4cb190b5a0e863 100644 --- a/drivers/watchdog/rzg2l_wdt.c +++ b/drivers/watchdog/rzg2l_wdt.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -40,6 +40,11 @@ module_param(nowayout, bool, 0); MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); +enum rz_wdt_type { + WDT_RZG2L, + WDT_RZV2M, +}; + struct rzg2l_wdt_priv { void __iomem *base; struct watchdog_device wdev; @@ -48,6 +53,7 @@ struct rzg2l_wdt_priv { unsigned long delay; struct clk *pclk; struct clk *osc_clk; + enum rz_wdt_type devtype; }; static void rzg2l_wdt_wait_delay(struct rzg2l_wdt_priv *priv) @@ -142,11 +148,29 @@ static int rzg2l_wdt_restart(struct watchdog_device *wdev, clk_prepare_enable(priv->pclk); clk_prepare_enable(priv->osc_clk); - /* Generate Reset (WDTRSTB) Signal on parity error */ - rzg2l_wdt_write(priv, 0, PECR); + if (priv->devtype == WDT_RZG2L) { + /* Generate Reset (WDTRSTB) Signal on parity error */ + rzg2l_wdt_write(priv, 0, PECR); + + /* Force parity error */ + rzg2l_wdt_write(priv, PEEN_FORCE, PEEN); + } else { + /* RZ/V2M doesn't have parity error registers */ + + wdev->timeout = 0; + + /* Initialize time out */ + rzg2l_wdt_init_timeout(wdev); - /* Force parity error */ - rzg2l_wdt_write(priv, PEEN_FORCE, PEEN); + /* Initialize watchdog counter register */ + rzg2l_wdt_write(priv, 0, WDTTIM); + + /* Enable watchdog timer*/ + rzg2l_wdt_write(priv, WDTCNT_WDTEN, WDTCNT); + + /* Wait 2 consecutive overflow cycles for reset */ + mdelay(DIV_ROUND_UP(2 * 0xFFFFF * 1000, priv->osc_clk_rate)); + } return 0; } @@ -227,6 +251,8 @@ static int rzg2l_wdt_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to deassert"); + priv->devtype = (uintptr_t)of_device_get_match_data(dev); + pm_runtime_enable(&pdev->dev); priv->wdev.info = &rzg2l_wdt_ident; @@ -255,7 +281,8 @@ static int rzg2l_wdt_probe(struct platform_device *pdev) } static const struct of_device_id rzg2l_wdt_ids[] = { - { .compatible = "renesas,rzg2l-wdt", }, + { .compatible = "renesas,rzg2l-wdt", .data = (void *)WDT_RZG2L }, + { .compatible = "renesas,rzv2m-wdt", .data = (void *)WDT_RZV2M }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, rzg2l_wdt_ids); diff --git a/drivers/watchdog/s3c2410_wdt.c b/drivers/watchdog/s3c2410_wdt.c index 95919392927fcda0b013a317ef1ddc4267a20753..d3fc8ed886fff40ebfa30ffcf39fc287e33dbbdc 100644 --- a/drivers/watchdog/s3c2410_wdt.c +++ b/drivers/watchdog/s3c2410_wdt.c @@ -60,9 +60,13 @@ #define EXYNOS850_CLUSTER0_NONCPU_INT_EN 0x1244 #define EXYNOS850_CLUSTER1_NONCPU_OUT 0x1620 #define EXYNOS850_CLUSTER1_NONCPU_INT_EN 0x1644 +#define EXYNOSAUTOV9_CLUSTER1_NONCPU_OUT 0x1520 +#define EXYNOSAUTOV9_CLUSTER1_NONCPU_INT_EN 0x1544 #define EXYNOS850_CLUSTER0_WDTRESET_BIT 24 #define EXYNOS850_CLUSTER1_WDTRESET_BIT 23 +#define EXYNOSAUTOV9_CLUSTER0_WDTRESET_BIT 25 +#define EXYNOSAUTOV9_CLUSTER1_WDTRESET_BIT 24 /** * DOC: Quirk flags for different Samsung watchdog IP-cores @@ -236,6 +240,30 @@ static const struct s3c2410_wdt_variant drv_data_exynos850_cl1 = { QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN, }; +static const struct s3c2410_wdt_variant drv_data_exynosautov9_cl0 = { + .mask_reset_reg = EXYNOS850_CLUSTER0_NONCPU_INT_EN, + .mask_bit = 2, + .mask_reset_inv = true, + .rst_stat_reg = EXYNOS5_RST_STAT_REG_OFFSET, + .rst_stat_bit = EXYNOSAUTOV9_CLUSTER0_WDTRESET_BIT, + .cnt_en_reg = EXYNOS850_CLUSTER0_NONCPU_OUT, + .cnt_en_bit = 7, + .quirks = QUIRK_HAS_WTCLRINT_REG | QUIRK_HAS_PMU_MASK_RESET | + QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN, +}; + +static const struct s3c2410_wdt_variant drv_data_exynosautov9_cl1 = { + .mask_reset_reg = EXYNOSAUTOV9_CLUSTER1_NONCPU_INT_EN, + .mask_bit = 2, + .mask_reset_inv = true, + .rst_stat_reg = EXYNOS5_RST_STAT_REG_OFFSET, + .rst_stat_bit = EXYNOSAUTOV9_CLUSTER1_WDTRESET_BIT, + .cnt_en_reg = EXYNOSAUTOV9_CLUSTER1_NONCPU_OUT, + .cnt_en_bit = 7, + .quirks = QUIRK_HAS_WTCLRINT_REG | QUIRK_HAS_PMU_MASK_RESET | + QUIRK_HAS_PMU_RST_STAT | QUIRK_HAS_PMU_CNT_EN, +}; + static const struct of_device_id s3c2410_wdt_match[] = { { .compatible = "samsung,s3c2410-wdt", .data = &drv_data_s3c2410 }, @@ -249,6 +277,8 @@ static const struct of_device_id s3c2410_wdt_match[] = { .data = &drv_data_exynos7 }, { .compatible = "samsung,exynos850-wdt", .data = &drv_data_exynos850_cl0 }, + { .compatible = "samsung,exynosautov9-wdt", + .data = &drv_data_exynosautov9_cl0 }, {}, }; MODULE_DEVICE_TABLE(of, s3c2410_wdt_match); @@ -630,8 +660,9 @@ s3c2410_get_wdt_drv_data(struct platform_device *pdev) } #ifdef CONFIG_OF - /* Choose Exynos850 driver data w.r.t. cluster index */ - if (variant == &drv_data_exynos850_cl0) { + /* Choose Exynos850/ExynosAutov9 driver data w.r.t. cluster index */ + if (variant == &drv_data_exynos850_cl0 || + variant == &drv_data_exynosautov9_cl0) { u32 index; int err; @@ -644,9 +675,11 @@ s3c2410_get_wdt_drv_data(struct platform_device *pdev) switch (index) { case 0: - return &drv_data_exynos850_cl0; + return variant; case 1: - return &drv_data_exynos850_cl1; + return (variant == &drv_data_exynos850_cl0) ? + &drv_data_exynos850_cl1 : + &drv_data_exynosautov9_cl1; default: dev_err(dev, "wrong cluster index: %u\n", index); return NULL; diff --git a/drivers/watchdog/sa1100_wdt.c b/drivers/watchdog/sa1100_wdt.c index 2d0a06a158a85117db18827dbd93a7d835f1defe..82ac5d19f519e281a4289aca02c5161844808233 100644 --- a/drivers/watchdog/sa1100_wdt.c +++ b/drivers/watchdog/sa1100_wdt.c @@ -238,7 +238,7 @@ static int sa1100dog_remove(struct platform_device *pdev) return 0; } -struct platform_driver sa1100dog_driver = { +static struct platform_driver sa1100dog_driver = { .driver.name = "sa1100_wdt", .probe = sa1100dog_probe, .remove = sa1100dog_remove, diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c index ae54dd33e23362b249d2feb5136029d49b4f834f..fb426b7d81dac78221286d0545428d982300880b 100644 --- a/drivers/watchdog/sp5100_tco.c +++ b/drivers/watchdog/sp5100_tco.c @@ -65,6 +65,12 @@ static struct pci_dev *sp5100_tco_pci; /* module parameters */ +#define WATCHDOG_ACTION 0 +static bool action = WATCHDOG_ACTION; +module_param(action, bool, 0); +MODULE_PARM_DESC(action, "Action taken when watchdog expires, 0 to reset, 1 to poweroff (default=" + __MODULE_STRING(WATCHDOG_ACTION) ")"); + #define WATCHDOG_HEARTBEAT 60 /* 60 sec default heartbeat. */ static int heartbeat = WATCHDOG_HEARTBEAT; /* in seconds */ module_param(heartbeat, int, 0); @@ -297,8 +303,11 @@ static int sp5100_tco_timer_init(struct sp5100_tco *tco) if (val & SP5100_WDT_FIRED) wdd->bootstatus = WDIOF_CARDRESET; - /* Set watchdog action to reset the system */ - val &= ~SP5100_WDT_ACTION_RESET; + /* Set watchdog action */ + if (action) + val |= SP5100_WDT_ACTION_RESET; + else + val &= ~SP5100_WDT_ACTION_RESET; writel(val, SP5100_WDT_CONTROL(tco->tcobase)); /* Set a reasonable heartbeat before we stop the timer */ diff --git a/drivers/watchdog/twl4030_wdt.c b/drivers/watchdog/twl4030_wdt.c index 355e428c0b99f1b82b15d05a5c13a1f85c02c5bd..36b4a660928d3ddee4356f8f079d0d9e2347afbe 100644 --- a/drivers/watchdog/twl4030_wdt.c +++ b/drivers/watchdog/twl4030_wdt.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/watchdog/w83627hf_wdt.c b/drivers/watchdog/w83627hf_wdt.c index 56a4a4030ca9650b902f8ac55511b3c71ff4c6a5..bc33b63c5a5df2f2d061c7dd2cd61e50a80a35b6 100644 --- a/drivers/watchdog/w83627hf_wdt.c +++ b/drivers/watchdog/w83627hf_wdt.c @@ -113,6 +113,10 @@ MODULE_PARM_DESC(early_disable, "Disable watchdog at boot time (default=0)"); #define W836X7HF_WDT_CSR 0xf7 #define NCT6102D_WDT_CSR 0xf2 +#define WDT_CSR_STATUS 0x10 +#define WDT_CSR_KBD 0x40 +#define WDT_CSR_MOUSE 0x80 + static void superio_outb(int reg, int val) { outb(reg, WDT_EFER); @@ -244,8 +248,12 @@ static int w83627hf_init(struct watchdog_device *wdog, enum chips chip) t = superio_inb(cr_wdt_control) & ~0x0C; superio_outb(cr_wdt_control, t); - /* reset trigger, disable keyboard & mouse turning off watchdog */ - t = superio_inb(cr_wdt_csr) & ~0xD0; + t = superio_inb(cr_wdt_csr); + if (t & WDT_CSR_STATUS) + wdog->bootstatus |= WDIOF_CARDRESET; + + /* reset status, disable keyboard & mouse turning off watchdog */ + t &= ~(WDT_CSR_STATUS | WDT_CSR_KBD | WDT_CSR_MOUSE); superio_outb(cr_wdt_csr, t); superio_exit(); diff --git a/drivers/watchdog/w83977f_wdt.c b/drivers/watchdog/w83977f_wdt.c index fd64ae77780a62b6c72fd69fa1b91c63ba1bce28..31bf21ceaf48d57cbecc129584667c00f0dc3a4d 100644 --- a/drivers/watchdog/w83977f_wdt.c +++ b/drivers/watchdog/w83977f_wdt.c @@ -321,7 +321,7 @@ static int wdt_release(struct inode *inode, struct file *file) * @ppos: pointer to the position to write. No seeks allowed * * A write to a watchdog device is defined as a keepalive signal. Any - * write of data will do, as we we don't define content meaning. + * write of data will do, as we don't define content meaning. */ static ssize_t wdt_write(struct file *file, const char __user *buf, diff --git a/drivers/watchdog/watchdog_dev.c b/drivers/watchdog/watchdog_dev.c index 54903f3c851eb953d379161e225f0adbba2dd217..744b2ab75288d4ac58055b0311950c902b80db8d 100644 --- a/drivers/watchdog/watchdog_dev.c +++ b/drivers/watchdog/watchdog_dev.c @@ -1015,7 +1015,11 @@ static int watchdog_cdev_register(struct watchdog_device *wdd) wd_data->dev.groups = wdd->groups; wd_data->dev.release = watchdog_core_data_release; dev_set_drvdata(&wd_data->dev, wdd); - dev_set_name(&wd_data->dev, "watchdog%d", wdd->id); + err = dev_set_name(&wd_data->dev, "watchdog%d", wdd->id); + if (err) { + put_device(&wd_data->dev); + return err; + } kthread_init_work(&wd_data->work, watchdog_ping_work); hrtimer_init(&wd_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); diff --git a/drivers/watchdog/wdat_wdt.c b/drivers/watchdog/wdat_wdt.c index aeadaa07c891deb7c3b272f6ab4f885db333c5f9..ce7a4a9e4b03ca8d2ae4d70390190d4780c62a76 100644 --- a/drivers/watchdog/wdat_wdt.c +++ b/drivers/watchdog/wdat_wdt.c @@ -342,9 +342,8 @@ static int wdat_wdt_probe(struct platform_device *pdev) return -EINVAL; wdat->period = tbl->timer_period; - wdat->wdd.min_hw_heartbeat_ms = wdat->period * tbl->min_count; - wdat->wdd.max_hw_heartbeat_ms = wdat->period * tbl->max_count; - wdat->wdd.min_timeout = 1; + wdat->wdd.min_timeout = DIV_ROUND_UP(wdat->period * tbl->min_count, 1000); + wdat->wdd.max_timeout = wdat->period * tbl->max_count / 1000; wdat->stopped_in_sleep = tbl->flags & ACPI_WDAT_STOPPED; wdat->wdd.info = &wdat_wdt_info; wdat->wdd.ops = &wdat_wdt_ops; diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c index c5a9b820d43a282e19a50461b2a35373374e7f9c..d0e88875443ae9a495d84bde3fdb439a7659b409 100644 --- a/drivers/watchdog/ziirave_wdt.c +++ b/drivers/watchdog/ziirave_wdt.c @@ -708,13 +708,11 @@ static int ziirave_wdt_probe(struct i2c_client *client, return ret; } -static int ziirave_wdt_remove(struct i2c_client *client) +static void ziirave_wdt_remove(struct i2c_client *client) { struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client); watchdog_unregister_device(&w_priv->wdd); - - return 0; } static const struct i2c_device_id ziirave_wdt_id[] = { diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index a65bd92121a5d8df1badcfa5588cbedd40ea4425..d5d7c402b65112b8592ba10bd3fd1732c26b771e 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -56,7 +56,7 @@ config XEN_MEMORY_HOTPLUG_LIMIT depends on XEN_HAVE_PVMMU depends on MEMORY_HOTPLUG help - Maxmium amount of memory (in GiB) that a PV guest can be + Maximum amount of memory (in GiB) that a PV guest can be expanded to when using memory hotplug. A PV guest can have more memory than this limit if is diff --git a/drivers/xen/gntdev-common.h b/drivers/xen/gntdev-common.h index 40ef379c28ab01a3a11008cb0f85a1b3fde134ca..9c286b2a19001616ae0e9986be13905891b5f5ff 100644 --- a/drivers/xen/gntdev-common.h +++ b/drivers/xen/gntdev-common.h @@ -44,9 +44,10 @@ struct gntdev_unmap_notify { }; struct gntdev_grant_map { + atomic_t in_use; struct mmu_interval_notifier notifier; + bool notifier_init; struct list_head next; - struct vm_area_struct *vma; int index; int count; int flags; diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 84b143eef395b1585f3a8c0fdcb301ce9fbc52ec..4d9a3050de6a3f6e3e2d41abfcefd41564dd3125 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -286,6 +286,9 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map) */ } + if (use_ptemod && map->notifier_init) + mmu_interval_notifier_remove(&map->notifier); + if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { notify_remote_via_evtchn(map->notify.event); evtchn_put(map->notify.event); @@ -298,7 +301,7 @@ void gntdev_put_map(struct gntdev_priv *priv, struct gntdev_grant_map *map) static int find_grant_ptes(pte_t *pte, unsigned long addr, void *data) { struct gntdev_grant_map *map = data; - unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT; + unsigned int pgnr = (addr - map->pages_vm_start) >> PAGE_SHIFT; int flags = map->flags | GNTMAP_application_map | GNTMAP_contains_pte | (1 << _GNTMAP_guest_avail0); u64 pte_maddr; @@ -367,8 +370,7 @@ int gntdev_map_grant_pages(struct gntdev_grant_map *map) for (i = 0; i < map->count; i++) { if (map->map_ops[i].status == GNTST_okay) { map->unmap_ops[i].handle = map->map_ops[i].handle; - if (!use_ptemod) - alloced++; + alloced++; } else if (!err) err = -EINVAL; @@ -377,8 +379,7 @@ int gntdev_map_grant_pages(struct gntdev_grant_map *map) if (use_ptemod) { if (map->kmap_ops[i].status == GNTST_okay) { - if (map->map_ops[i].status == GNTST_okay) - alloced++; + alloced++; map->kunmap_ops[i].handle = map->kmap_ops[i].handle; } else if (!err) err = -EINVAL; @@ -394,8 +395,14 @@ static void __unmap_grant_pages_done(int result, unsigned int i; struct gntdev_grant_map *map = data->data; unsigned int offset = data->unmap_ops - map->unmap_ops; + int successful_unmaps = 0; + int live_grants; for (i = 0; i < data->count; i++) { + if (map->unmap_ops[offset + i].status == GNTST_okay && + map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE) + successful_unmaps++; + WARN_ON(map->unmap_ops[offset + i].status != GNTST_okay && map->unmap_ops[offset + i].handle != INVALID_GRANT_HANDLE); pr_debug("unmap handle=%d st=%d\n", @@ -403,6 +410,10 @@ static void __unmap_grant_pages_done(int result, map->unmap_ops[offset+i].status); map->unmap_ops[offset+i].handle = INVALID_GRANT_HANDLE; if (use_ptemod) { + if (map->kunmap_ops[offset + i].status == GNTST_okay && + map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE) + successful_unmaps++; + WARN_ON(map->kunmap_ops[offset + i].status != GNTST_okay && map->kunmap_ops[offset + i].handle != INVALID_GRANT_HANDLE); pr_debug("kunmap handle=%u st=%d\n", @@ -411,11 +422,15 @@ static void __unmap_grant_pages_done(int result, map->kunmap_ops[offset+i].handle = INVALID_GRANT_HANDLE; } } + /* * Decrease the live-grant counter. This must happen after the loop to * prevent premature reuse of the grants by gnttab_mmap(). */ - atomic_sub(data->count, &map->live_grants); + live_grants = atomic_sub_return(successful_unmaps, &map->live_grants); + if (WARN_ON(live_grants < 0)) + pr_err("%s: live_grants became negative (%d) after unmapping %d pages!\n", + __func__, live_grants, successful_unmaps); /* Release reference taken by __unmap_grant_pages */ gntdev_put_map(NULL, map); @@ -496,11 +511,7 @@ static void gntdev_vma_close(struct vm_area_struct *vma) struct gntdev_priv *priv = file->private_data; pr_debug("gntdev_vma_close %p\n", vma); - if (use_ptemod) { - WARN_ON(map->vma != vma); - mmu_interval_notifier_remove(&map->notifier); - map->vma = NULL; - } + vma->vm_private_data = NULL; gntdev_put_map(priv, map); } @@ -528,29 +539,30 @@ static bool gntdev_invalidate(struct mmu_interval_notifier *mn, struct gntdev_grant_map *map = container_of(mn, struct gntdev_grant_map, notifier); unsigned long mstart, mend; + unsigned long map_start, map_end; if (!mmu_notifier_range_blockable(range)) return false; + map_start = map->pages_vm_start; + map_end = map->pages_vm_start + (map->count << PAGE_SHIFT); + /* * If the VMA is split or otherwise changed the notifier is not * updated, but we don't want to process VA's outside the modified * VMA. FIXME: It would be much more understandable to just prevent * modifying the VMA in the first place. */ - if (map->vma->vm_start >= range->end || - map->vma->vm_end <= range->start) + if (map_start >= range->end || map_end <= range->start) return true; - mstart = max(range->start, map->vma->vm_start); - mend = min(range->end, map->vma->vm_end); + mstart = max(range->start, map_start); + mend = min(range->end, map_end); pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n", - map->index, map->count, - map->vma->vm_start, map->vma->vm_end, - range->start, range->end, mstart, mend); - unmap_grant_pages(map, - (mstart - map->vma->vm_start) >> PAGE_SHIFT, - (mend - mstart) >> PAGE_SHIFT); + map->index, map->count, map_start, map_end, + range->start, range->end, mstart, mend); + unmap_grant_pages(map, (mstart - map_start) >> PAGE_SHIFT, + (mend - mstart) >> PAGE_SHIFT); return true; } @@ -1030,18 +1042,15 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) return -EINVAL; pr_debug("map %d+%d at %lx (pgoff %lx)\n", - index, count, vma->vm_start, vma->vm_pgoff); + index, count, vma->vm_start, vma->vm_pgoff); mutex_lock(&priv->lock); map = gntdev_find_map_index(priv, index, count); if (!map) goto unlock_out; - if (use_ptemod && map->vma) + if (!atomic_add_unless(&map->in_use, 1, 1)) goto unlock_out; - if (atomic_read(&map->live_grants)) { - err = -EAGAIN; - goto unlock_out; - } + refcount_inc(&map->users); vma->vm_ops = &gntdev_vmops; @@ -1062,15 +1071,16 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) map->flags |= GNTMAP_readonly; } + map->pages_vm_start = vma->vm_start; + if (use_ptemod) { - map->vma = vma; err = mmu_interval_notifier_insert_locked( &map->notifier, vma->vm_mm, vma->vm_start, vma->vm_end - vma->vm_start, &gntdev_mmu_ops); - if (err) { - map->vma = NULL; + if (err) goto out_unlock_put; - } + + map->notifier_init = true; } mutex_unlock(&priv->lock); @@ -1087,7 +1097,6 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) */ mmu_interval_read_begin(&map->notifier); - map->pages_vm_start = vma->vm_start; err = apply_to_page_range(vma->vm_mm, vma->vm_start, vma->vm_end - vma->vm_start, find_grant_ptes, map); @@ -1116,13 +1125,8 @@ unlock_out: out_unlock_put: mutex_unlock(&priv->lock); out_put_map: - if (use_ptemod) { + if (use_ptemod) unmap_grant_pages(map, 0, map->count); - if (map->vma) { - mmu_interval_notifier_remove(&map->notifier); - map->vma = NULL; - } - } gntdev_put_map(priv, map); return err; } diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 8973fc1e9cccd2129678ca4e52d3a5ca695cc277..860f37c93af410591e10b75e8f0374fc591a9bd6 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -25,7 +25,7 @@ struct xen_grant_dma_data { bool broken; }; -static DEFINE_XARRAY(xen_grant_dma_devices); +static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ); #define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63) @@ -42,14 +42,29 @@ static inline grant_ref_t dma_to_grant(dma_addr_t dma) static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev) { struct xen_grant_dma_data *data; + unsigned long flags; - xa_lock(&xen_grant_dma_devices); + xa_lock_irqsave(&xen_grant_dma_devices, flags); data = xa_load(&xen_grant_dma_devices, (unsigned long)dev); - xa_unlock(&xen_grant_dma_devices); + xa_unlock_irqrestore(&xen_grant_dma_devices, flags); return data; } +static int store_xen_grant_dma_data(struct device *dev, + struct xen_grant_dma_data *data) +{ + unsigned long flags; + int ret; + + xa_lock_irqsave(&xen_grant_dma_devices, flags); + ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data, + GFP_ATOMIC)); + xa_unlock_irqrestore(&xen_grant_dma_devices, flags); + + return ret; +} + /* * DMA ops for Xen frontends (e.g. virtio). * @@ -153,7 +168,7 @@ static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, unsigned long attrs) { struct xen_grant_dma_data *data; - unsigned int i, n_pages = PFN_UP(size); + unsigned int i, n_pages = PFN_UP(offset + size); grant_ref_t grant; dma_addr_t dma_handle; @@ -185,7 +200,8 @@ static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, unsigned long attrs) { struct xen_grant_dma_data *data; - unsigned int i, n_pages = PFN_UP(size); + unsigned long offset = dma_handle & (PAGE_SIZE - 1); + unsigned int i, n_pages = PFN_UP(offset + size); grant_ref_t grant; if (WARN_ON(dir == DMA_NONE)) @@ -273,72 +289,91 @@ static const struct dma_map_ops xen_grant_dma_ops = { .dma_supported = xen_grant_dma_supported, }; -bool xen_is_grant_dma_device(struct device *dev) +static bool xen_is_dt_grant_dma_device(struct device *dev) { struct device_node *iommu_np; bool has_iommu; - /* XXX Handle only DT devices for now */ - if (!dev->of_node) - return false; - iommu_np = of_parse_phandle(dev->of_node, "iommus", 0); - has_iommu = iommu_np && of_device_is_compatible(iommu_np, "xen,grant-dma"); + has_iommu = iommu_np && + of_device_is_compatible(iommu_np, "xen,grant-dma"); of_node_put(iommu_np); return has_iommu; } +bool xen_is_grant_dma_device(struct device *dev) +{ + /* XXX Handle only DT devices for now */ + if (dev->of_node) + return xen_is_dt_grant_dma_device(dev); + + return false; +} + bool xen_virtio_mem_acc(struct virtio_device *dev) { - if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT)) + if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) return true; return xen_is_grant_dma_device(dev->dev.parent); } -void xen_grant_setup_dma_ops(struct device *dev) +static int xen_dt_grant_init_backend_domid(struct device *dev, + struct xen_grant_dma_data *data) { - struct xen_grant_dma_data *data; struct of_phandle_args iommu_spec; - data = find_xen_grant_dma_data(dev); - if (data) { - dev_err(dev, "Xen grant DMA data is already created\n"); - return; - } - - /* XXX ACPI device unsupported for now */ - if (!dev->of_node) - goto err; - if (of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells", 0, &iommu_spec)) { dev_err(dev, "Cannot parse iommus property\n"); - goto err; + return -ESRCH; } if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") || iommu_spec.args_count != 1) { dev_err(dev, "Incompatible IOMMU node\n"); of_node_put(iommu_spec.np); - goto err; + return -ESRCH; } of_node_put(iommu_spec.np); + /* + * The endpoint ID here means the ID of the domain where the + * corresponding backend is running + */ + data->backend_domid = iommu_spec.args[0]; + + return 0; +} + +void xen_grant_setup_dma_ops(struct device *dev) +{ + struct xen_grant_dma_data *data; + + data = find_xen_grant_dma_data(dev); + if (data) { + dev_err(dev, "Xen grant DMA data is already created\n"); + return; + } + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) goto err; - /* - * The endpoint ID here means the ID of the domain where the corresponding - * backend is running - */ - data->backend_domid = iommu_spec.args[0]; + if (dev->of_node) { + if (xen_dt_grant_init_backend_domid(dev, data)) + goto err; + } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT)) { + dev_info(dev, "Using dom0 as backend\n"); + data->backend_domid = 0; + } else { + /* XXX ACPI device unsupported for now */ + goto err; + } - if (xa_err(xa_store(&xen_grant_dma_devices, (unsigned long)dev, data, - GFP_KERNEL))) { + if (store_xen_grant_dma_data(dev, data)) { dev_err(dev, "Cannot store Xen grant DMA data\n"); goto err; } @@ -348,9 +383,20 @@ void xen_grant_setup_dma_ops(struct device *dev) return; err: + devm_kfree(dev, data); dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n"); } +bool xen_virtio_restricted_mem_acc(struct virtio_device *dev) +{ + bool ret = xen_virtio_mem_acc(dev); + + if (ret) + xen_grant_setup_dma_ops(dev->dev.parent); + + return ret; +} + MODULE_DESCRIPTION("Xen grant DMA-mapping layer"); MODULE_AUTHOR("Juergen Gross "); MODULE_LICENSE("GPL"); diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 738029de3c6722598ba3c96441f0109159b6fed3..e1ec725c2819d4d5dede063eb00d86a6d52944c0 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -1047,6 +1047,9 @@ int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args) size_t size; int i, ret; + if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT)) + return -ENOMEM; + size = args->nr_pages << PAGE_SHIFT; if (args->coherent) args->vaddr = dma_alloc_coherent(args->dev, size, diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 3369734108af23724e728531ce4024ea752b7fbc..fae50a24630bd0e517c5512f327fd9d55d244221 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -282,7 +282,7 @@ static long privcmd_ioctl_mmap(struct file *file, void __user *udata) struct page, lru); struct privcmd_mmap_entry *msg = page_address(page); - vma = find_vma(mm, msg->va); + vma = vma_lookup(mm, msg->va); rc = -EINVAL; if (!vma || (msg->va != vma->vm_start) || vma->vm_private_data) @@ -581,27 +581,30 @@ static int lock_pages( struct privcmd_dm_op_buf kbufs[], unsigned int num, struct page *pages[], unsigned int nr_pages, unsigned int *pinned) { - unsigned int i; + unsigned int i, off = 0; - for (i = 0; i < num; i++) { + for (i = 0; i < num; ) { unsigned int requested; int page_count; requested = DIV_ROUND_UP( offset_in_page(kbufs[i].uptr) + kbufs[i].size, - PAGE_SIZE); + PAGE_SIZE) - off; if (requested > nr_pages) return -ENOSPC; page_count = pin_user_pages_fast( - (unsigned long) kbufs[i].uptr, + (unsigned long)kbufs[i].uptr + off * PAGE_SIZE, requested, FOLL_WRITE, pages); - if (page_count < 0) - return page_count; + if (page_count <= 0) + return page_count ? : -EFAULT; *pinned += page_count; nr_pages -= page_count; pages += page_count; + + off = (requested == page_count) ? 0 : off + page_count; + i += !off; } return 0; @@ -677,10 +680,8 @@ static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) } rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned); - if (rc < 0) { - nr_pages = pinned; + if (rc < 0) goto out; - } for (i = 0; i < kdata.num; i++) { set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr); @@ -692,7 +693,7 @@ static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) xen_preemptible_hcall_end(); out: - unlock_pages(pages, nr_pages); + unlock_pages(pages, pinned); kfree(xbufs); kfree(pages); kfree(kbufs); diff --git a/drivers/xen/xen-pciback/xenbus.c b/drivers/xen/xen-pciback/xenbus.c index bde63ef677b8fa325e1057f7b9ea59e0be2aa872..d171091eec123dda417e3b2b32c5aa70184826d3 100644 --- a/drivers/xen/xen-pciback/xenbus.c +++ b/drivers/xen/xen-pciback/xenbus.c @@ -31,7 +31,7 @@ MODULE_PARM_DESC(passthrough, " frontend (for example, a device at 06:01.b will still appear at\n"\ " 06:01.b to the frontend). This is similar to how Xen 2.0.x\n"\ " exposed PCI devices to its driver domains. This may be required\n"\ - " for drivers which depend on finding their hardward in certain\n"\ + " for drivers which depend on finding their hardware in certain\n"\ " bus/slot locations."); static struct xen_pcibk_device *alloc_pdev(struct xenbus_device *xdev) diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index 7a0c93acc2c5760335c072d929c7d59d4102b357..6106ed93817d670be7001dec34146a3bdb74f013 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c @@ -333,18 +333,6 @@ static int32_t scsiback_result(int32_t result) case DID_TRANSPORT_FAILFAST: host_status = XEN_VSCSIIF_RSLT_HOST_TRANSPORT_FAILFAST; break; - case DID_TARGET_FAILURE: - host_status = XEN_VSCSIIF_RSLT_HOST_TARGET_FAILURE; - break; - case DID_NEXUS_FAILURE: - host_status = XEN_VSCSIIF_RSLT_HOST_NEXUS_FAILURE; - break; - case DID_ALLOC_FAILURE: - host_status = XEN_VSCSIIF_RSLT_HOST_ALLOC_FAILURE; - break; - case DID_MEDIUM_ERROR: - host_status = XEN_VSCSIIF_RSLT_HOST_MEDIUM_ERROR; - break; case DID_TRANSPORT_MARGINAL: host_status = XEN_VSCSIIF_RSLT_HOST_TRANSPORT_MARGINAL; break; @@ -1121,7 +1109,7 @@ static void scsiback_do_1lun_hotplug(struct vscsibk_info *info, int op, "%s: writing %s", __func__, state); return; } - strlcpy(phy, val, VSCSI_NAMELEN); + strscpy(phy, val, VSCSI_NAMELEN); kfree(val); /* virtual SCSI device */ diff --git a/drivers/xen/xenbus/xenbus_client.c b/drivers/xen/xenbus/xenbus_client.c index d5f3f763717eaee68181a66ec6c7eb19289d2c81..d4b2519257962de62ebb83ac52327e41dcbf6cdb 100644 --- a/drivers/xen/xenbus/xenbus_client.c +++ b/drivers/xen/xenbus/xenbus_client.c @@ -382,9 +382,10 @@ int xenbus_setup_ring(struct xenbus_device *dev, gfp_t gfp, void **vaddr, unsigned long ring_size = nr_pages * XEN_PAGE_SIZE; grant_ref_t gref_head; unsigned int i; + void *addr; int ret; - *vaddr = alloc_pages_exact(ring_size, gfp | __GFP_ZERO); + addr = *vaddr = alloc_pages_exact(ring_size, gfp | __GFP_ZERO); if (!*vaddr) { ret = -ENOMEM; goto err; @@ -401,13 +402,15 @@ int xenbus_setup_ring(struct xenbus_device *dev, gfp_t gfp, void **vaddr, unsigned long gfn; if (is_vmalloc_addr(*vaddr)) - gfn = pfn_to_gfn(vmalloc_to_pfn(vaddr[i])); + gfn = pfn_to_gfn(vmalloc_to_pfn(addr)); else - gfn = virt_to_gfn(vaddr[i]); + gfn = virt_to_gfn(addr); grefs[i] = gnttab_claim_grant_reference(&gref_head); gnttab_grant_foreign_access_ref(grefs[i], dev->otherend_id, gfn, 0); + + addr += XEN_PAGE_SIZE; } return 0; diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c index 07b010a68fcf9cd5e6bca1e4ec47d8487013e227..f44d5a64351e4ab0d081234b94c7a288b2f4bd58 100644 --- a/drivers/xen/xenbus/xenbus_probe_frontend.c +++ b/drivers/xen/xenbus/xenbus_probe_frontend.c @@ -40,7 +40,7 @@ static int frontend_bus_id(char bus_id[XEN_BUS_ID_SIZE], const char *nodename) return -EINVAL; } - strlcpy(bus_id, nodename + 1, XEN_BUS_ID_SIZE); + strscpy(bus_id, nodename + 1, XEN_BUS_ID_SIZE); if (!strchr(bus_id, '/')) { pr_warn("bus_id %s no slash\n", bus_id); return -EINVAL; diff --git a/fs/Kconfig b/fs/Kconfig index a547307c1ae824a9283fb1f79c721585fa967323..2685a4d0d353188b6dbf2a30fb6650231e184636 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -235,6 +235,7 @@ config ARCH_SUPPORTS_HUGETLBFS config HUGETLBFS bool "HugeTLB file system support" depends on X86 || IA64 || SPARC64 || ARCH_SUPPORTS_HUGETLBFS || BROKEN + depends on (SYSFS || SYSCTL) help hugetlbfs is a filesystem backing for HugeTLB pages, based on ramfs. For architectures that support it, say Y here and read diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt index 21e154516bf297e35bfbd9884731110609315698..93539aac0e5b2e8a87444742ec267b3e0d8fe3fc 100644 --- a/fs/Kconfig.binfmt +++ b/fs/Kconfig.binfmt @@ -58,7 +58,7 @@ config ARCH_USE_GNU_PROPERTY config BINFMT_ELF_FDPIC bool "Kernel support for FDPIC ELF binaries" default y if !BINFMT_ELF - depends on ARM || ((M68K || SUPERH) && !MMU) + depends on ARM || ((M68K || SUPERH || XTENSA) && !MMU) select ELFCORE help ELF FDPIC binaries are based on ELF, but allow the individual load @@ -142,39 +142,6 @@ config BINFMT_ZFLAT help Support FLAT format compressed binaries -config HAVE_AOUT - def_bool n - -config BINFMT_AOUT - tristate "Kernel support for a.out and ECOFF binaries" - depends on HAVE_AOUT - help - A.out (Assembler.OUTput) is a set of formats for libraries and - executables used in the earliest versions of UNIX. Linux used - the a.out formats QMAGIC and ZMAGIC until they were replaced - with the ELF format. - - The conversion to ELF started in 1995. This option is primarily - provided for historical interest and for the benefit of those - who need to run binaries from that era. - - Most people should answer N here. If you think you may have - occasional use for this format, enable module support above - and answer M here to compile this support as a module called - binfmt_aout. - - If any crucial components of your system (such as /sbin/init - or /lib/ld.so) are still in a.out format, you will have to - say Y here. - -config OSF4_COMPAT - bool "OSF/1 v4 readv/writev compatibility" - depends on ALPHA && BINFMT_AOUT - help - Say Y if you are using OSF/1 binaries (like Netscape and Acrobat) - with v4 shared libraries freely available from Compaq. If you're - going to use shared libraries from Tru64 version 5.0 or later, say N. - config BINFMT_MISC tristate "Kernel support for MISC binaries" help diff --git a/fs/Makefile b/fs/Makefile index 93b80529f8e827177602910de61c82d395a008c6..4dea17840761a0586f7237932167ecbc6399c412 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -38,7 +38,6 @@ obj-$(CONFIG_FS_DAX) += dax.o obj-$(CONFIG_FS_ENCRYPTION) += crypto/ obj-$(CONFIG_FS_VERITY) += verity/ obj-$(CONFIG_FILE_LOCKING) += locks.o -obj-$(CONFIG_BINFMT_AOUT) += binfmt_aout.o obj-$(CONFIG_BINFMT_MISC) += binfmt_misc.o obj-$(CONFIG_BINFMT_SCRIPT) += binfmt_script.o obj-$(CONFIG_BINFMT_ELF) += binfmt_elf.o diff --git a/fs/affs/super.c b/fs/affs/super.c index 4c5f30a833362a67d8fbec022746ba06b87390fb..58b391446ae1fd97e48891c82ec8d88f32314303 100644 --- a/fs/affs/super.c +++ b/fs/affs/super.c @@ -276,7 +276,7 @@ parse_options(char *options, kuid_t *uid, kgid_t *gid, int *mode, int *reserved, char *vol = match_strdup(&args[0]); if (!vol) return 0; - strlcpy(volume, vol, 32); + strscpy(volume, vol, 32); kfree(vol); break; } diff --git a/fs/afs/dir.c b/fs/afs/dir.c index 56ae5cd5184f08c4f5757face83a8cb870d7cd7d..230c2d19116d9daf79840db2e67beaadf127c4d6 100644 --- a/fs/afs/dir.c +++ b/fs/afs/dir.c @@ -24,9 +24,9 @@ static int afs_readdir(struct file *file, struct dir_context *ctx); static int afs_d_revalidate(struct dentry *dentry, unsigned int flags); static int afs_d_delete(const struct dentry *dentry); static void afs_d_iput(struct dentry *dentry, struct inode *inode); -static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, +static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, loff_t fpos, u64 ino, unsigned dtype); -static int afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, +static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, loff_t fpos, u64 ino, unsigned dtype); static int afs_create(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl); @@ -568,7 +568,7 @@ static int afs_readdir(struct file *file, struct dir_context *ctx) * - if afs_dir_iterate_block() spots this function, it'll pass the FID * uniquifier through dtype */ -static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name, +static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, loff_t fpos, u64 ino, unsigned dtype) { struct afs_lookup_one_cookie *cookie = @@ -584,16 +584,16 @@ static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name, if (cookie->name.len != nlen || memcmp(cookie->name.name, name, nlen) != 0) { - _leave(" = 0 [no]"); - return 0; + _leave(" = true [keep looking]"); + return true; } cookie->fid.vnode = ino; cookie->fid.unique = dtype; cookie->found = 1; - _leave(" = -1 [found]"); - return -1; + _leave(" = false [found]"); + return false; } /* @@ -636,12 +636,11 @@ static int afs_do_lookup_one(struct inode *dir, struct dentry *dentry, * - if afs_dir_iterate_block() spots this function, it'll pass the FID * uniquifier through dtype */ -static int afs_lookup_filldir(struct dir_context *ctx, const char *name, +static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, loff_t fpos, u64 ino, unsigned dtype) { struct afs_lookup_cookie *cookie = container_of(ctx, struct afs_lookup_cookie, ctx); - int ret; _enter("{%s,%u},%s,%u,,%llu,%u", cookie->name.name, cookie->name.len, name, nlen, @@ -663,12 +662,10 @@ static int afs_lookup_filldir(struct dir_context *ctx, const char *name, cookie->fids[1].unique = dtype; cookie->found = 1; if (cookie->one_only) - return -1; + return false; } - ret = cookie->nr_fids >= 50 ? -1 : 0; - _leave(" = %d", ret); - return ret; + return cookie->nr_fids < 50; } /* diff --git a/fs/afs/flock.c b/fs/afs/flock.c index c4210a3964d8b3df6b5ed38feb2270e5387d95ad..bbcc5afd15760a998aa13172c51dfc986ac63ff8 100644 --- a/fs/afs/flock.c +++ b/fs/afs/flock.c @@ -76,7 +76,7 @@ void afs_lock_op_done(struct afs_call *call) if (call->error == 0) { spin_lock(&vnode->lock); trace_afs_flock_ev(vnode, NULL, afs_flock_timestamp, 0); - vnode->locked_at = call->reply_time; + vnode->locked_at = call->issue_time; afs_schedule_lock_extension(vnode); spin_unlock(&vnode->lock); } diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c index 4943413d9c5f7ff3011ea2fad230e5a84a6e48e5..7d37f63ef0f092079c207b97cfe059cd77a2ab97 100644 --- a/fs/afs/fsclient.c +++ b/fs/afs/fsclient.c @@ -131,7 +131,7 @@ bad: static time64_t xdr_decode_expiry(struct afs_call *call, u32 expiry) { - return ktime_divns(call->reply_time, NSEC_PER_SEC) + expiry; + return ktime_divns(call->issue_time, NSEC_PER_SEC) + expiry; } static void xdr_decode_AFSCallBack(const __be32 **_bp, diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 64ad55494349bf6b19a6083ab88a7a26eb8fd0c9..723d162078a3c0d7a7df789f60703758be9f9f6e 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -137,7 +137,6 @@ struct afs_call { bool need_attention; /* T if RxRPC poked us */ bool async; /* T if asynchronous */ bool upgrade; /* T to request service upgrade */ - bool have_reply_time; /* T if have got reply_time */ bool intr; /* T if interruptible */ bool unmarshalling_error; /* T if an unmarshalling error occurred */ u16 service_id; /* Actual service ID (after upgrade) */ @@ -151,7 +150,7 @@ struct afs_call { } __attribute__((packed)); __be64 tmp64; }; - ktime_t reply_time; /* Time of first reply packet */ + ktime_t issue_time; /* Time of issue of operation */ }; struct afs_call_type { diff --git a/fs/afs/misc.c b/fs/afs/misc.c index 933e67fcdab1a722c4954e18bed11dd08fc6b8e1..805328ca54284286e2e95608b41d841a70b05caa 100644 --- a/fs/afs/misc.c +++ b/fs/afs/misc.c @@ -69,6 +69,7 @@ int afs_abort_to_error(u32 abort_code) /* Unified AFS error table */ case UAEPERM: return -EPERM; case UAENOENT: return -ENOENT; + case UAEAGAIN: return -EAGAIN; case UAEACCES: return -EACCES; case UAEBUSY: return -EBUSY; case UAEEXIST: return -EEXIST; diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c index d5c4785c862d994835a455d9cfc94a55ce38518f..eccc3cd0cb70047508fa88ea3d723b2e2f1f7f85 100644 --- a/fs/afs/rxrpc.c +++ b/fs/afs/rxrpc.c @@ -351,6 +351,7 @@ void afs_make_call(struct afs_addr_cursor *ac, struct afs_call *call, gfp_t gfp) if (call->max_lifespan) rxrpc_kernel_set_max_life(call->net->socket, rxcall, call->max_lifespan); + call->issue_time = ktime_get_real(); /* send the request */ iov[0].iov_base = call->request; @@ -501,12 +502,6 @@ static void afs_deliver_to_call(struct afs_call *call) return; } - if (!call->have_reply_time && - rxrpc_kernel_get_reply_time(call->net->socket, - call->rxcall, - &call->reply_time)) - call->have_reply_time = true; - ret = call->type->deliver(call); state = READ_ONCE(call->state); if (ret == 0 && call->unmarshalling_error) diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c index fdc7d675b4b0cc7c4c3af854fba4cbd11bd6961c..11571cca86c19a2256209f8e15d6827d21aeda0c 100644 --- a/fs/afs/yfsclient.c +++ b/fs/afs/yfsclient.c @@ -232,8 +232,7 @@ static void xdr_decode_YFSCallBack(const __be32 **_bp, struct afs_callback *cb = &scb->callback; ktime_t cb_expiry; - cb_expiry = call->reply_time; - cb_expiry = ktime_add(cb_expiry, xdr_to_u64(x->expiration_time) * 100); + cb_expiry = ktime_add(call->issue_time, xdr_to_u64(x->expiration_time) * 100); cb->expires_at = ktime_divns(cb_expiry, NSEC_PER_SEC); scb->have_cb = true; *_bp += xdr_size(x); diff --git a/fs/aio.c b/fs/aio.c index 606613e9d1f4fd09899f2237fa26ee4b3848e01e..5b2ff20ad32298e827b8cb68da24160e8b91ae41 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -951,16 +951,13 @@ static bool __get_reqs_available(struct kioctx *ctx) local_irq_save(flags); kcpu = this_cpu_ptr(ctx->cpu); if (!kcpu->reqs_available) { - int old, avail = atomic_read(&ctx->reqs_available); + int avail = atomic_read(&ctx->reqs_available); do { if (avail < ctx->req_batch) goto out; - - old = avail; - avail = atomic_cmpxchg(&ctx->reqs_available, - avail, avail - ctx->req_batch); - } while (avail != old); + } while (!atomic_try_cmpxchg(&ctx->reqs_available, + &avail, avail - ctx->req_batch)); kcpu->reqs_available += ctx->req_batch; } diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index e0c3e33c4177b909551be06137e77c46ef5fa5fd..24192a7667edf766e6932e12459c93379875e7e5 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -32,7 +32,7 @@ static struct inode *anon_inode_inode; */ static char *anon_inodefs_dname(struct dentry *dentry, char *buffer, int buflen) { - return dynamic_dname(dentry, buffer, buflen, "anon_inode:%s", + return dynamic_dname(buffer, buflen, "anon_inode:%s", dentry->d_name.name); } diff --git a/fs/bad_inode.c b/fs/bad_inode.c index 12b8fdcc445bbd115bf34390b51c7efc83eb0455..9d1cde8066cf8bb87b276ac05220357ae26cd78b 100644 --- a/fs/bad_inode.c +++ b/fs/bad_inode.c @@ -147,7 +147,7 @@ static int bad_inode_atomic_open(struct inode *inode, struct dentry *dentry, } static int bad_inode_tmpfile(struct user_namespace *mnt_userns, - struct inode *inode, struct dentry *dentry, + struct inode *inode, struct file *file, umode_t mode) { return -EIO; diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c deleted file mode 100644 index 0dcfc691e7e218bc8db705a4ca1f7f69e445a697..0000000000000000000000000000000000000000 --- a/fs/binfmt_aout.c +++ /dev/null @@ -1,342 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * linux/fs/binfmt_aout.c - * - * Copyright (C) 1991, 1992, 1996 Linus Torvalds - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -static int load_aout_binary(struct linux_binprm *); -static int load_aout_library(struct file*); - -static struct linux_binfmt aout_format = { - .module = THIS_MODULE, - .load_binary = load_aout_binary, - .load_shlib = load_aout_library, -}; - -#define BAD_ADDR(x) ((unsigned long)(x) >= TASK_SIZE) - -static int set_brk(unsigned long start, unsigned long end) -{ - start = PAGE_ALIGN(start); - end = PAGE_ALIGN(end); - if (end > start) - return vm_brk(start, end - start); - return 0; -} - -/* - * create_aout_tables() parses the env- and arg-strings in new user - * memory and creates the pointer tables from them, and puts their - * addresses on the "stack", returning the new stack pointer value. - */ -static unsigned long __user *create_aout_tables(char __user *p, struct linux_binprm * bprm) -{ - char __user * __user *argv; - char __user * __user *envp; - unsigned long __user *sp; - int argc = bprm->argc; - int envc = bprm->envc; - - sp = (void __user *)((-(unsigned long)sizeof(char *)) & (unsigned long) p); -#ifdef __alpha__ -/* whee.. test-programs are so much fun. */ - put_user(0, --sp); - put_user(0, --sp); - if (bprm->loader) { - put_user(0, --sp); - put_user(1003, --sp); - put_user(bprm->loader, --sp); - put_user(1002, --sp); - } - put_user(bprm->exec, --sp); - put_user(1001, --sp); -#endif - sp -= envc+1; - envp = (char __user * __user *) sp; - sp -= argc+1; - argv = (char __user * __user *) sp; -#ifndef __alpha__ - put_user((unsigned long) envp,--sp); - put_user((unsigned long) argv,--sp); -#endif - put_user(argc,--sp); - current->mm->arg_start = (unsigned long) p; - while (argc-->0) { - char c; - put_user(p,argv++); - do { - get_user(c,p++); - } while (c); - } - put_user(NULL,argv); - current->mm->arg_end = current->mm->env_start = (unsigned long) p; - while (envc-->0) { - char c; - put_user(p,envp++); - do { - get_user(c,p++); - } while (c); - } - put_user(NULL,envp); - current->mm->env_end = (unsigned long) p; - return sp; -} - -/* - * These are the functions used to load a.out style executables and shared - * libraries. There is no binary dependent code anywhere else. - */ - -static int load_aout_binary(struct linux_binprm * bprm) -{ - struct pt_regs *regs = current_pt_regs(); - struct exec ex; - unsigned long error; - unsigned long fd_offset; - unsigned long rlim; - int retval; - - ex = *((struct exec *) bprm->buf); /* exec-header */ - if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != OMAGIC && - N_MAGIC(ex) != QMAGIC && N_MAGIC(ex) != NMAGIC) || - N_TRSIZE(ex) || N_DRSIZE(ex) || - i_size_read(file_inode(bprm->file)) < ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) { - return -ENOEXEC; - } - - /* - * Requires a mmap handler. This prevents people from using a.out - * as part of an exploit attack against /proc-related vulnerabilities. - */ - if (!bprm->file->f_op->mmap) - return -ENOEXEC; - - fd_offset = N_TXTOFF(ex); - - /* Check initial limits. This avoids letting people circumvent - * size limits imposed on them by creating programs with large - * arrays in the data or bss. - */ - rlim = rlimit(RLIMIT_DATA); - if (rlim >= RLIM_INFINITY) - rlim = ~0; - if (ex.a_data + ex.a_bss > rlim) - return -ENOMEM; - - /* Flush all traces of the currently running executable */ - retval = begin_new_exec(bprm); - if (retval) - return retval; - - /* OK, This is the point of no return */ -#ifdef __alpha__ - SET_AOUT_PERSONALITY(bprm, ex); -#else - set_personality(PER_LINUX); -#endif - setup_new_exec(bprm); - - current->mm->end_code = ex.a_text + - (current->mm->start_code = N_TXTADDR(ex)); - current->mm->end_data = ex.a_data + - (current->mm->start_data = N_DATADDR(ex)); - current->mm->brk = ex.a_bss + - (current->mm->start_brk = N_BSSADDR(ex)); - - retval = setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT); - if (retval < 0) - return retval; - - - if (N_MAGIC(ex) == OMAGIC) { - unsigned long text_addr, map_size; - loff_t pos; - - text_addr = N_TXTADDR(ex); - -#ifdef __alpha__ - pos = fd_offset; - map_size = ex.a_text+ex.a_data + PAGE_SIZE - 1; -#else - pos = 32; - map_size = ex.a_text+ex.a_data; -#endif - error = vm_brk(text_addr & PAGE_MASK, map_size); - if (error) - return error; - - error = read_code(bprm->file, text_addr, pos, - ex.a_text+ex.a_data); - if ((signed long)error < 0) - return error; - } else { - if ((ex.a_text & 0xfff || ex.a_data & 0xfff) && - (N_MAGIC(ex) != NMAGIC) && printk_ratelimit()) - { - printk(KERN_NOTICE "executable not page aligned\n"); - } - - if ((fd_offset & ~PAGE_MASK) != 0 && printk_ratelimit()) - { - printk(KERN_WARNING - "fd_offset is not page aligned. Please convert program: %pD\n", - bprm->file); - } - - if (!bprm->file->f_op->mmap||((fd_offset & ~PAGE_MASK) != 0)) { - error = vm_brk(N_TXTADDR(ex), ex.a_text+ex.a_data); - if (error) - return error; - - read_code(bprm->file, N_TXTADDR(ex), fd_offset, - ex.a_text + ex.a_data); - goto beyond_if; - } - - error = vm_mmap(bprm->file, N_TXTADDR(ex), ex.a_text, - PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, - fd_offset); - - if (error != N_TXTADDR(ex)) - return error; - - error = vm_mmap(bprm->file, N_DATADDR(ex), ex.a_data, - PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_FIXED | MAP_PRIVATE, - fd_offset + ex.a_text); - if (error != N_DATADDR(ex)) - return error; - } -beyond_if: - set_binfmt(&aout_format); - - retval = set_brk(current->mm->start_brk, current->mm->brk); - if (retval < 0) - return retval; - - current->mm->start_stack = - (unsigned long) create_aout_tables((char __user *) bprm->p, bprm); -#ifdef __alpha__ - regs->gp = ex.a_gpvalue; -#endif - finalize_exec(bprm); - start_thread(regs, ex.a_entry, current->mm->start_stack); - return 0; -} - -static int load_aout_library(struct file *file) -{ - struct inode * inode; - unsigned long bss, start_addr, len; - unsigned long error; - int retval; - struct exec ex; - loff_t pos = 0; - - inode = file_inode(file); - - retval = -ENOEXEC; - error = kernel_read(file, &ex, sizeof(ex), &pos); - if (error != sizeof(ex)) - goto out; - - /* We come in here for the regular a.out style of shared libraries */ - if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != QMAGIC) || N_TRSIZE(ex) || - N_DRSIZE(ex) || ((ex.a_entry & 0xfff) && N_MAGIC(ex) == ZMAGIC) || - i_size_read(inode) < ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) { - goto out; - } - - /* - * Requires a mmap handler. This prevents people from using a.out - * as part of an exploit attack against /proc-related vulnerabilities. - */ - if (!file->f_op->mmap) - goto out; - - if (N_FLAGS(ex)) - goto out; - - /* For QMAGIC, the starting address is 0x20 into the page. We mask - this off to get the starting address for the page */ - - start_addr = ex.a_entry & 0xfffff000; - - if ((N_TXTOFF(ex) & ~PAGE_MASK) != 0) { - if (printk_ratelimit()) - { - printk(KERN_WARNING - "N_TXTOFF is not page aligned. Please convert library: %pD\n", - file); - } - retval = vm_brk(start_addr, ex.a_text + ex.a_data + ex.a_bss); - if (retval) - goto out; - - read_code(file, start_addr, N_TXTOFF(ex), - ex.a_text + ex.a_data); - retval = 0; - goto out; - } - /* Now use mmap to map the library into memory. */ - error = vm_mmap(file, start_addr, ex.a_text + ex.a_data, - PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_FIXED | MAP_PRIVATE, - N_TXTOFF(ex)); - retval = error; - if (error != start_addr) - goto out; - - len = PAGE_ALIGN(ex.a_text + ex.a_data); - bss = ex.a_text + ex.a_data + ex.a_bss; - if (bss > len) { - retval = vm_brk(start_addr + len, bss - len); - if (retval) - goto out; - } - retval = 0; -out: - return retval; -} - -static int __init init_aout_binfmt(void) -{ - register_binfmt(&aout_format); - return 0; -} - -static void __exit exit_aout_binfmt(void) -{ - unregister_binfmt(&aout_format); -} - -core_initcall(init_aout_binfmt); -module_exit(exit_aout_binfmt); -MODULE_LICENSE("GPL"); diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile index 99f9995670ea3be174e250e45714d776c21d2337..fa9ddcc9eb0bce968f79b980e2a21ba12c16cd2d 100644 --- a/fs/btrfs/Makefile +++ b/fs/btrfs/Makefile @@ -31,7 +31,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ uuid-tree.o props.o free-space-tree.o tree-checker.o space-info.o \ block-rsv.o delalloc-space.o block-group.o discard.o reflink.o \ - subpage.o tree-mod-log.o + subpage.o tree-mod-log.o extent-io-tree.o btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o btrfs-$(CONFIG_BTRFS_FS_CHECK_INTEGRITY) += check-integrity.o diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index d385357e19b6136772117b5a987f55fe89e71eb3..dce3a16996b951d3aed5ebe789f727d4d7289149 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1511,16 +1511,118 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, return ret; } -/** - * Check if an extent is shared or not +/* + * The caller has joined a transaction or is holding a read lock on the + * fs_info->commit_root_sem semaphore, so no need to worry about the root's last + * snapshot field changing while updating or checking the cache. + */ +static bool lookup_backref_shared_cache(struct btrfs_backref_shared_cache *cache, + struct btrfs_root *root, + u64 bytenr, int level, bool *is_shared) +{ + struct btrfs_backref_shared_cache_entry *entry; + + if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) + return false; + + /* + * Level -1 is used for the data extent, which is not reliable to cache + * because its reference count can increase or decrease without us + * realizing. We cache results only for extent buffers that lead from + * the root node down to the leaf with the file extent item. + */ + ASSERT(level >= 0); + + entry = &cache->entries[level]; + + /* Unused cache entry or being used for some other extent buffer. */ + if (entry->bytenr != bytenr) + return false; + + /* + * We cached a false result, but the last snapshot generation of the + * root changed, so we now have a snapshot. Don't trust the result. + */ + if (!entry->is_shared && + entry->gen != btrfs_root_last_snapshot(&root->root_item)) + return false; + + /* + * If we cached a true result and the last generation used for dropping + * a root changed, we can not trust the result, because the dropped root + * could be a snapshot sharing this extent buffer. + */ + if (entry->is_shared && + entry->gen != btrfs_get_last_root_drop_gen(root->fs_info)) + return false; + + *is_shared = entry->is_shared; + + return true; +} + +/* + * The caller has joined a transaction or is holding a read lock on the + * fs_info->commit_root_sem semaphore, so no need to worry about the root's last + * snapshot field changing while updating or checking the cache. + */ +static void store_backref_shared_cache(struct btrfs_backref_shared_cache *cache, + struct btrfs_root *root, + u64 bytenr, int level, bool is_shared) +{ + struct btrfs_backref_shared_cache_entry *entry; + u64 gen; + + if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) + return; + + /* + * Level -1 is used for the data extent, which is not reliable to cache + * because its reference count can increase or decrease without us + * realizing. We cache results only for extent buffers that lead from + * the root node down to the leaf with the file extent item. + */ + ASSERT(level >= 0); + + if (is_shared) + gen = btrfs_get_last_root_drop_gen(root->fs_info); + else + gen = btrfs_root_last_snapshot(&root->root_item); + + entry = &cache->entries[level]; + entry->bytenr = bytenr; + entry->is_shared = is_shared; + entry->gen = gen; + + /* + * If we found an extent buffer is shared, set the cache result for all + * extent buffers below it to true. As nodes in the path are COWed, + * their sharedness is moved to their children, and if a leaf is COWed, + * then the sharedness of a data extent becomes direct, the refcount of + * data extent is increased in the extent item at the extent tree. + */ + if (is_shared) { + for (int i = 0; i < level; i++) { + entry = &cache->entries[i]; + entry->is_shared = is_shared; + entry->gen = gen; + } + } +} + +/* + * Check if a data extent is shared or not. * - * @root: root inode belongs to - * @inum: inode number of the inode whose extent we are checking - * @bytenr: logical bytenr of the extent we are checking - * @roots: list of roots this extent is shared among - * @tmp: temporary list used for iteration + * @root: The root the inode belongs to. + * @inum: Number of the inode whose extent we are checking. + * @bytenr: Logical bytenr of the extent we are checking. + * @extent_gen: Generation of the extent (file extent item) or 0 if it is + * not known. + * @roots: List of roots this extent is shared among. + * @tmp: Temporary list used for iteration. + * @cache: A backref lookup result cache. * - * btrfs_check_shared uses the backref walking code but will short + * btrfs_is_data_extent_shared uses the backref walking code but will short * circuit as soon as it finds a root or inode that doesn't match the * one passed in. This provides a significant performance benefit for * callers (such as fiemap) which want to know whether the extent is @@ -1531,8 +1633,10 @@ int btrfs_find_all_roots(struct btrfs_trans_handle *trans, * * Return: 0 if extent is not shared, 1 if it is shared, < 0 on error. */ -int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, - struct ulist *roots, struct ulist *tmp) +int btrfs_is_data_extent_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + u64 extent_gen, + struct ulist *roots, struct ulist *tmp, + struct btrfs_backref_shared_cache *cache) { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_trans_handle *trans; @@ -1545,6 +1649,7 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, .inum = inum, .share_count = 0, }; + int level; ulist_init(roots); ulist_init(tmp); @@ -1561,22 +1666,52 @@ int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, btrfs_get_tree_mod_seq(fs_info, &elem); } + /* -1 means we are in the bytenr of the data extent. */ + level = -1; ULIST_ITER_INIT(&uiter); while (1) { + bool is_shared; + bool cached; + ret = find_parent_nodes(trans, fs_info, bytenr, elem.seq, tmp, roots, NULL, &shared, false); if (ret == BACKREF_FOUND_SHARED) { /* this is the only condition under which we return 1 */ ret = 1; + if (level >= 0) + store_backref_shared_cache(cache, root, bytenr, + level, true); break; } if (ret < 0 && ret != -ENOENT) break; ret = 0; + /* + * If our data extent is not shared through reflinks and it was + * created in a generation after the last one used to create a + * snapshot of the inode's root, then it can not be shared + * indirectly through subtrees, as that can only happen with + * snapshots. In this case bail out, no need to check for the + * sharedness of extent buffers. + */ + if (level == -1 && + extent_gen > btrfs_root_last_snapshot(&root->root_item)) + break; + + if (level >= 0) + store_backref_shared_cache(cache, root, bytenr, + level, false); node = ulist_next(tmp, &uiter); if (!node) break; bytenr = node->val; + level++; + cached = lookup_backref_shared_cache(cache, root, bytenr, level, + &is_shared); + if (cached) { + ret = (is_shared ? 1 : 0); + break; + } shared.share_count = 0; cond_resched(); } diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 2759de7d324c87ad80f520ea164bf66821ff1cd1..52ae6957b414249d494b3cd90ede82970aa7bf1f 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -17,6 +17,20 @@ struct inode_fs_paths { struct btrfs_data_container *fspath; }; +struct btrfs_backref_shared_cache_entry { + u64 bytenr; + u64 gen; + bool is_shared; +}; + +struct btrfs_backref_shared_cache { + /* + * A path from a root to a leaf that has a file extent item pointing to + * a given data extent should never exceed the maximum b+tree height. + */ + struct btrfs_backref_shared_cache_entry entries[BTRFS_MAX_LEVEL]; +}; + typedef int (iterate_extent_inodes_t)(u64 inum, u64 offset, u64 root, void *ctx); @@ -62,8 +76,10 @@ int btrfs_find_one_extref(struct btrfs_root *root, u64 inode_objectid, u64 start_off, struct btrfs_path *path, struct btrfs_inode_extref **ret_extref, u64 *found_off); -int btrfs_check_shared(struct btrfs_root *root, u64 inum, u64 bytenr, - struct ulist *roots, struct ulist *tmp_ulist); +int btrfs_is_data_extent_shared(struct btrfs_root *root, u64 inum, u64 bytenr, + u64 extent_gen, + struct ulist *roots, struct ulist *tmp, + struct btrfs_backref_shared_cache *cache); int __init btrfs_prelim_ref_init(void); void __cold btrfs_prelim_ref_exit(void); diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 993aca2f1e1812d4f7849bbe01357a32c38ab057..32c415cfbdfe7548566763e98a86f0eeff430764 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -440,39 +440,26 @@ void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache, btrfs_put_caching_control(caching_ctl); } -int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache) +static int btrfs_caching_ctl_wait_done(struct btrfs_block_group *cache, + struct btrfs_caching_control *caching_ctl) +{ + wait_event(caching_ctl->wait, btrfs_block_group_done(cache)); + return cache->cached == BTRFS_CACHE_ERROR ? -EIO : 0; +} + +static int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache) { struct btrfs_caching_control *caching_ctl; - int ret = 0; + int ret; caching_ctl = btrfs_get_caching_control(cache); if (!caching_ctl) return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0; - - wait_event(caching_ctl->wait, btrfs_block_group_done(cache)); - if (cache->cached == BTRFS_CACHE_ERROR) - ret = -EIO; + ret = btrfs_caching_ctl_wait_done(cache, caching_ctl); btrfs_put_caching_control(caching_ctl); return ret; } -static bool space_cache_v1_done(struct btrfs_block_group *cache) -{ - bool ret; - - spin_lock(&cache->lock); - ret = cache->cached != BTRFS_CACHE_FAST; - spin_unlock(&cache->lock); - - return ret; -} - -void btrfs_wait_space_cache_v1_finished(struct btrfs_block_group *cache, - struct btrfs_caching_control *caching_ctl) -{ - wait_event(caching_ctl->wait, space_cache_v1_done(cache)); -} - #ifdef CONFIG_BTRFS_DEBUG static void fragment_free_space(struct btrfs_block_group *block_group) { @@ -606,8 +593,6 @@ next: if (need_resched() || rwsem_is_contended(&fs_info->commit_root_sem)) { - if (wakeup) - caching_ctl->progress = last; btrfs_release_path(path); up_read(&fs_info->commit_root_sem); mutex_unlock(&caching_ctl->mutex); @@ -631,9 +616,6 @@ next: key.objectid = last; key.offset = 0; key.type = BTRFS_EXTENT_ITEM_KEY; - - if (wakeup) - caching_ctl->progress = last; btrfs_release_path(path); goto next; } @@ -668,7 +650,6 @@ next: total_found += add_new_free_space(block_group, last, block_group->start + block_group->length); - caching_ctl->progress = (u64)-1; out: btrfs_free_path(path); @@ -738,8 +719,6 @@ done: } #endif - caching_ctl->progress = (u64)-1; - up_read(&fs_info->commit_root_sem); btrfs_free_excluded_extents(block_group); mutex_unlock(&caching_ctl->mutex); @@ -750,9 +729,8 @@ done: btrfs_put_block_group(block_group); } -int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only) +int btrfs_cache_block_group(struct btrfs_block_group *cache, bool wait) { - DEFINE_WAIT(wait); struct btrfs_fs_info *fs_info = cache->fs_info; struct btrfs_caching_control *caching_ctl = NULL; int ret = 0; @@ -769,7 +747,6 @@ int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only mutex_init(&caching_ctl->mutex); init_waitqueue_head(&caching_ctl->wait); caching_ctl->block_group = cache; - caching_ctl->progress = cache->start; refcount_set(&caching_ctl->count, 2); btrfs_init_work(&caching_ctl->work, caching_thread, NULL, NULL); @@ -785,11 +762,7 @@ int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only } WARN_ON(cache->caching_ctl); cache->caching_ctl = caching_ctl; - if (btrfs_test_opt(fs_info, SPACE_CACHE)) - cache->cached = BTRFS_CACHE_FAST; - else - cache->cached = BTRFS_CACHE_STARTED; - cache->has_caching_ctl = 1; + cache->cached = BTRFS_CACHE_STARTED; spin_unlock(&cache->lock); write_lock(&fs_info->block_group_cache_lock); @@ -801,8 +774,10 @@ int btrfs_cache_block_group(struct btrfs_block_group *cache, int load_cache_only btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); out: - if (load_cache_only && caching_ctl) - btrfs_wait_space_cache_v1_finished(cache, caching_ctl); + /* REVIEW */ + if (wait && caching_ctl) + ret = btrfs_caching_ctl_wait_done(cache, caching_ctl); + /* wait_event(caching_ctl->wait, space_cache_v1_done(cache)); */ if (caching_ctl) btrfs_put_caching_control(caching_ctl); @@ -1005,32 +980,31 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, kobject_put(kobj); } - if (block_group->has_caching_ctl) - caching_ctl = btrfs_get_caching_control(block_group); if (block_group->cached == BTRFS_CACHE_STARTED) btrfs_wait_block_group_cache_done(block_group); - if (block_group->has_caching_ctl) { - write_lock(&fs_info->block_group_cache_lock); - if (!caching_ctl) { - struct btrfs_caching_control *ctl; - - list_for_each_entry(ctl, - &fs_info->caching_block_groups, list) - if (ctl->block_group == block_group) { - caching_ctl = ctl; - refcount_inc(&caching_ctl->count); - break; - } - } - if (caching_ctl) - list_del_init(&caching_ctl->list); - write_unlock(&fs_info->block_group_cache_lock); - if (caching_ctl) { - /* Once for the caching bgs list and once for us. */ - btrfs_put_caching_control(caching_ctl); - btrfs_put_caching_control(caching_ctl); + + write_lock(&fs_info->block_group_cache_lock); + caching_ctl = btrfs_get_caching_control(block_group); + if (!caching_ctl) { + struct btrfs_caching_control *ctl; + + list_for_each_entry(ctl, &fs_info->caching_block_groups, list) { + if (ctl->block_group == block_group) { + caching_ctl = ctl; + refcount_inc(&caching_ctl->count); + break; + } } } + if (caching_ctl) + list_del_init(&caching_ctl->list); + write_unlock(&fs_info->block_group_cache_lock); + + if (caching_ctl) { + /* Once for the caching bgs list and once for us. */ + btrfs_put_caching_control(caching_ctl); + btrfs_put_caching_control(caching_ctl); + } spin_lock(&trans->transaction->dirty_bgs_lock); WARN_ON(!list_empty(&block_group->dirty_list)); @@ -1051,12 +1025,13 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, < block_group->zone_unusable); WARN_ON(block_group->space_info->disk_total < block_group->length * factor); - WARN_ON(block_group->zone_is_active && + WARN_ON(test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &block_group->runtime_flags) && block_group->space_info->active_total_bytes < block_group->length); } block_group->space_info->total_bytes -= block_group->length; - if (block_group->zone_is_active) + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) block_group->space_info->active_total_bytes -= block_group->length; block_group->space_info->bytes_readonly -= (block_group->length - block_group->zone_unusable); @@ -1086,7 +1061,8 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, goto out; spin_lock(&block_group->lock); - block_group->removed = 1; + set_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags); + /* * At this point trimming or scrub can't start on this block group, * because we removed the block group from the rbtree @@ -1321,6 +1297,9 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) return; + if (btrfs_fs_closing(fs_info)) + return; + /* * Long running balances can keep us blocked here for eternity, so * simply skip deletion if we're unable to get the mutex. @@ -1560,6 +1539,9 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) return; + if (btrfs_fs_closing(fs_info)) + return; + if (!btrfs_should_reclaim(fs_info)) return; @@ -1907,16 +1889,6 @@ static int exclude_super_stripes(struct btrfs_block_group *cache) return 0; } -static void link_block_group(struct btrfs_block_group *cache) -{ - struct btrfs_space_info *space_info = cache->space_info; - int index = btrfs_bg_flags_to_raid_index(cache->flags); - - down_write(&space_info->groups_sem); - list_add_tail(&cache->list, &space_info->block_groups[index]); - up_write(&space_info->groups_sem); -} - static struct btrfs_block_group *btrfs_create_block_group_cache( struct btrfs_fs_info *fs_info, u64 start) { @@ -1954,7 +1926,8 @@ static struct btrfs_block_group *btrfs_create_block_group_cache( btrfs_init_free_space_ctl(cache, cache->free_space_ctl); atomic_set(&cache->frozen, 0); mutex_init(&cache->free_space_lock); - btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root); + cache->full_stripe_locks_root.root = RB_ROOT; + mutex_init(&cache->full_stripe_locks_root.lock); return cache; } @@ -2019,7 +1992,6 @@ static int read_one_block_group(struct btrfs_fs_info *info, int need_clear) { struct btrfs_block_group *cache; - struct btrfs_space_info *space_info; const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); int ret; @@ -2095,11 +2067,9 @@ static int read_one_block_group(struct btrfs_fs_info *info, /* Should not have any excluded extents. Just in case, though. */ btrfs_free_excluded_extents(cache); } else if (cache->length == cache->used) { - cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; btrfs_free_excluded_extents(cache); } else if (cache->used == 0) { - cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; add_new_free_space(cache, cache->start, cache->start + cache->length); @@ -2112,14 +2082,7 @@ static int read_one_block_group(struct btrfs_fs_info *info, goto error; } trace_btrfs_add_block_group(info, cache, 0); - btrfs_update_space_info(info, cache->flags, cache->length, - cache->used, cache->bytes_super, - cache->zone_unusable, cache->zone_is_active, - &space_info); - - cache->space_info = space_info; - - link_block_group(cache); + btrfs_add_bg_to_space_info(info, cache); set_avail_alloc_bits(info, cache->flags); if (btrfs_chunk_writeable(info, cache->start)) { @@ -2143,7 +2106,6 @@ error: static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) { struct extent_map_tree *em_tree = &fs_info->mapping_tree; - struct btrfs_space_info *space_info; struct rb_node *node; int ret = 0; @@ -2163,7 +2125,6 @@ static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) /* Fill dummy cache as FULL */ bg->length = em->len; bg->flags = map->type; - bg->last_byte_to_unpin = (u64)-1; bg->cached = BTRFS_CACHE_FINISHED; bg->used = em->len; bg->flags = map->type; @@ -2184,10 +2145,7 @@ static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) break; } - btrfs_update_space_info(fs_info, bg->flags, em->len, em->len, - 0, 0, false, &space_info); - bg->space_info = space_info; - link_block_group(bg); + btrfs_add_bg_to_space_info(fs_info, bg); set_avail_alloc_bits(fs_info, bg->flags); } @@ -2207,7 +2165,16 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) int need_clear = 0; u64 cache_gen; - if (!root) + /* + * Either no extent root (with ibadroots rescue option) or we have + * unsupported RO options. The fs can never be mounted read-write, so no + * need to waste time searching block group items. + * + * This also allows new extent tree related changes to be RO compat, + * no need for a full incompat flag. + */ + if (!root || (btrfs_super_compat_ro_flags(info->super_copy) & + ~BTRFS_FEATURE_COMPAT_RO_SUPP)) return fill_dummy_bgs(info); key.objectid = 0; @@ -2442,7 +2409,8 @@ void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) ret = insert_block_group_item(trans, block_group); if (ret) btrfs_abort_transaction(trans, ret); - if (!block_group->chunk_item_inserted) { + if (!test_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, + &block_group->runtime_flags)) { mutex_lock(&fs_info->chunk_mutex); ret = btrfs_chunk_alloc_add_chunk_item(trans, block_group); mutex_unlock(&fs_info->chunk_mutex); @@ -2511,7 +2479,6 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran set_free_space_tree_thresholds(cache); cache->used = bytes_used; cache->flags = type; - cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; cache->global_root_id = calculate_global_root_id(fs_info, cache->start); @@ -2536,14 +2503,6 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran btrfs_free_excluded_extents(cache); -#ifdef CONFIG_BTRFS_DEBUG - if (btrfs_should_fragment_free_space(cache)) { - u64 new_bytes_used = size - bytes_used; - - bytes_used += new_bytes_used >> 1; - fragment_free_space(cache); - } -#endif /* * Ensure the corresponding space_info object is created and * assigned to our block group. We want our bg to be added to the rbtree @@ -2564,12 +2523,17 @@ struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *tran * the rbtree, update the space info's counters. */ trace_btrfs_add_block_group(fs_info, cache, 1); - btrfs_update_space_info(fs_info, cache->flags, size, bytes_used, - cache->bytes_super, cache->zone_unusable, - cache->zone_is_active, &cache->space_info); + btrfs_add_bg_to_space_info(fs_info, cache); btrfs_update_global_block_rsv(fs_info); - link_block_group(cache); +#ifdef CONFIG_BTRFS_DEBUG + if (btrfs_should_fragment_free_space(cache)) { + u64 new_bytes_used = size - bytes_used; + + cache->space_info->bytes_used += new_bytes_used >> 1; + fragment_free_space(cache); + } +#endif list_add_tail(&cache->bg_list, &trans->new_bgs); trans->delayed_ref_updates++; @@ -2886,7 +2850,7 @@ again: cache_size *= fs_info->sectorsize; ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0, - cache_size); + cache_size, false); if (ret) goto out_put; @@ -3312,7 +3276,7 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, * space back to the block group, otherwise we will leak space. */ if (!alloc && !btrfs_block_group_done(cache)) - btrfs_cache_block_group(cache, 1); + btrfs_cache_block_group(cache, true); byte_in_group = bytenr - cache->start; WARN_ON(byte_in_group > cache->length); @@ -3982,35 +3946,24 @@ void btrfs_reserve_chunk_metadata(struct btrfs_trans_handle *trans, void btrfs_put_block_group_cache(struct btrfs_fs_info *info) { struct btrfs_block_group *block_group; - u64 last = 0; - while (1) { - struct inode *inode; + block_group = btrfs_lookup_first_block_group(info, 0); + while (block_group) { + btrfs_wait_block_group_cache_done(block_group); + spin_lock(&block_group->lock); + if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, + &block_group->runtime_flags)) { + struct inode *inode = block_group->inode; - block_group = btrfs_lookup_first_block_group(info, last); - while (block_group) { - btrfs_wait_block_group_cache_done(block_group); - spin_lock(&block_group->lock); - if (block_group->iref) - break; + block_group->inode = NULL; spin_unlock(&block_group->lock); - block_group = btrfs_next_block_group(block_group); - } - if (!block_group) { - if (last == 0) - break; - last = 0; - continue; - } - inode = block_group->inode; - block_group->iref = 0; - block_group->inode = NULL; - spin_unlock(&block_group->lock); - ASSERT(block_group->io_ctl.inode == NULL); - iput(inode); - last = block_group->start + block_group->length; - btrfs_put_block_group(block_group); + ASSERT(block_group->io_ctl.inode == NULL); + iput(inode); + } else { + spin_unlock(&block_group->lock); + } + block_group = btrfs_next_block_group(block_group); } } @@ -4146,7 +4099,7 @@ void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) spin_lock(&block_group->lock); cleanup = (atomic_dec_and_test(&block_group->frozen) && - block_group->removed); + test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)); spin_unlock(&block_group->lock); if (cleanup) { @@ -4167,7 +4120,7 @@ void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) * tasks trimming this block group have left 1 entry each one. * Free them if any. */ - __btrfs_remove_free_space_cache(block_group->free_space_ctl); + btrfs_remove_free_space_cache(block_group); } } diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 35e0e860cc0bf161b630043444cea767decc2bcf..8fb14b99a1d1fdf4e170df449f05c1ec83f94533 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -46,19 +46,44 @@ enum btrfs_chunk_alloc_enum { CHUNK_ALLOC_FORCE_FOR_EXTENT, }; +/* Block group flags set at runtime */ +enum btrfs_block_group_flags { + BLOCK_GROUP_FLAG_IREF, + BLOCK_GROUP_FLAG_REMOVED, + BLOCK_GROUP_FLAG_TO_COPY, + BLOCK_GROUP_FLAG_RELOCATING_REPAIR, + BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, + BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, +}; + +enum btrfs_caching_type { + BTRFS_CACHE_NO, + BTRFS_CACHE_STARTED, + BTRFS_CACHE_FINISHED, + BTRFS_CACHE_ERROR, +}; + struct btrfs_caching_control { struct list_head list; struct mutex mutex; wait_queue_head_t wait; struct btrfs_work work; struct btrfs_block_group *block_group; - u64 progress; refcount_t count; }; /* Once caching_thread() finds this much free space, it will wake up waiters. */ #define CACHING_CTL_WAKE_UP SZ_2M +/* + * Tree to record all locked full stripes of a RAID5/6 block group + */ +struct btrfs_full_stripe_locks_tree { + struct rb_root root; + struct mutex lock; +}; + struct btrfs_block_group { struct btrfs_fs_info *fs_info; struct inode *inode; @@ -95,23 +120,15 @@ struct btrfs_block_group { /* For raid56, this is a full stripe, without parity */ unsigned long full_stripe_len; + unsigned long runtime_flags; unsigned int ro; - unsigned int iref:1; - unsigned int has_caching_ctl:1; - unsigned int removed:1; - unsigned int to_copy:1; - unsigned int relocating_repair:1; - unsigned int chunk_item_inserted:1; - unsigned int zone_is_active:1; - unsigned int zoned_data_reloc_ongoing:1; int disk_cache_state; /* Cache tracking stuff */ int cached; struct btrfs_caching_control *caching_ctl; - u64 last_byte_to_unpin; struct btrfs_space_info *space_info; @@ -263,9 +280,7 @@ void btrfs_dec_nocow_writers(struct btrfs_block_group *bg); void btrfs_wait_nocow_writers(struct btrfs_block_group *bg); void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache, u64 num_bytes); -int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache); -int btrfs_cache_block_group(struct btrfs_block_group *cache, - int load_cache_only); +int btrfs_cache_block_group(struct btrfs_block_group *cache, bool wait); void btrfs_put_caching_control(struct btrfs_caching_control *ctl); struct btrfs_caching_control *btrfs_get_caching_control( struct btrfs_block_group *cache); @@ -307,8 +322,6 @@ void btrfs_reserve_chunk_metadata(struct btrfs_trans_handle *trans, u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags); void btrfs_put_block_group_cache(struct btrfs_fs_info *info); int btrfs_free_block_groups(struct btrfs_fs_info *info); -void btrfs_wait_space_cache_v1_finished(struct btrfs_block_group *cache, - struct btrfs_caching_control *caching_ctl); int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, struct block_device *bdev, u64 physical, u64 **logical, int *naddrs, int *stripe_len); diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 06be0644dd37651ca3afcfe93d7586f4bfc094ae..ec96285357e0a64c4abacf4650764b1c6dcba1c2 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -286,7 +286,7 @@ u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, */ if (block_rsv == delayed_rsv) target = global_rsv; - else if (block_rsv != global_rsv && !delayed_rsv->full) + else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) target = delayed_rsv; if (target && block_rsv->space_info != target->space_info) @@ -424,6 +424,7 @@ void btrfs_init_root_block_rsv(struct btrfs_root *root) case BTRFS_CSUM_TREE_OBJECTID: case BTRFS_EXTENT_TREE_OBJECTID: case BTRFS_FREE_SPACE_TREE_OBJECTID: + case BTRFS_BLOCK_GROUP_TREE_OBJECTID: root->block_rsv = &fs_info->delayed_refs_rsv; break; case BTRFS_ROOT_TREE_OBJECTID: diff --git a/fs/btrfs/block-rsv.h b/fs/btrfs/block-rsv.h index 0c183709be0084f15cc7b1175d3350876c337cc3..578c3497a455c54f6f3e0ab125d29a584b8fb9f4 100644 --- a/fs/btrfs/block-rsv.h +++ b/fs/btrfs/block-rsv.h @@ -92,4 +92,13 @@ static inline void btrfs_unuse_block_rsv(struct btrfs_fs_info *fs_info, btrfs_block_rsv_release(fs_info, block_rsv, 0, NULL); } +/* + * Fast path to check if the reserve is full, may be carefully used outside of + * locks. + */ +static inline bool btrfs_block_rsv_full(const struct btrfs_block_rsv *rsv) +{ + return data_race(rsv->full); +} + #endif /* BTRFS_BLOCK_RSV_H */ diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index b160b8e124e01152ae345297dfa1083d1c0d45f6..54c2ccb36b61253f4a9f4745ebd257da0a3b5e87 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -65,6 +65,8 @@ enum { * on the same file. */ BTRFS_INODE_VERITY_IN_PROGRESS, + /* Set when this inode is a free space inode. */ + BTRFS_INODE_FREE_SPACE_INODE, }; /* in memory btrfs inode */ @@ -94,7 +96,8 @@ struct btrfs_inode { /* special utility tree used to record which mirrors have already been * tried when checksums fail for a given block */ - struct extent_io_tree io_failure_tree; + struct rb_root io_failure_tree; + spinlock_t io_failure_lock; /* * Keep track of where the inode has extent items mapped in order to @@ -250,11 +253,6 @@ struct btrfs_inode { struct inode vfs_inode; }; -static inline u32 btrfs_inode_sectorsize(const struct btrfs_inode *inode) -{ - return inode->root->fs_info->sectorsize; -} - static inline struct btrfs_inode *BTRFS_I(const struct inode *inode) { return container_of(inode, struct btrfs_inode, vfs_inode); @@ -272,13 +270,6 @@ static inline unsigned long btrfs_inode_hash(u64 objectid, return (unsigned long)h; } -static inline void btrfs_insert_inode_hash(struct inode *inode) -{ - unsigned long h = btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root); - - __insert_inode_hash(inode, h); -} - #if BITS_PER_LONG == 32 /* @@ -312,13 +303,7 @@ static inline void btrfs_i_size_write(struct btrfs_inode *inode, u64 size) static inline bool btrfs_is_free_space_inode(struct btrfs_inode *inode) { - struct btrfs_root *root = inode->root; - - if (root == root->fs_info->tree_root && - btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID) - return true; - - return false; + return test_bit(BTRFS_INODE_FREE_SPACE_INODE, &inode->runtime_flags); } static inline bool is_data_inode(struct inode *inode) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index e84d22c5c6a833fa111e77ac6057513f0f243bd7..f1f051ad31474c738bd430ef7db662c398353afb 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -152,9 +154,7 @@ static void finish_compressed_bio_read(struct compressed_bio *cb) } /* Do io completion on the original bio */ - if (cb->status != BLK_STS_OK) - cb->orig_bio->bi_status = cb->status; - bio_endio(cb->orig_bio); + btrfs_bio_end_io(btrfs_bio(cb->orig_bio), cb->status); /* Finally free the cb struct */ kfree(cb->compressed_pages); @@ -166,16 +166,15 @@ static void finish_compressed_bio_read(struct compressed_bio *cb) * before decompressing it into the original bio and freeing the uncompressed * pages. */ -static void end_compressed_bio_read(struct bio *bio) +static void end_compressed_bio_read(struct btrfs_bio *bbio) { - struct compressed_bio *cb = bio->bi_private; + struct compressed_bio *cb = bbio->private; struct inode *inode = cb->inode; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_inode *bi = BTRFS_I(inode); bool csum = !(bi->flags & BTRFS_INODE_NODATASUM) && !test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state); - blk_status_t status = bio->bi_status; - struct btrfs_bio *bbio = btrfs_bio(bio); + blk_status_t status = bbio->bio.bi_status; struct bvec_iter iter; struct bio_vec bv; u32 offset; @@ -186,9 +185,8 @@ static void end_compressed_bio_read(struct bio *bio) if (!status && (!csum || !btrfs_check_data_csum(inode, bbio, offset, bv.bv_page, bv.bv_offset))) { - clean_io_failure(fs_info, &bi->io_failure_tree, - &bi->io_tree, start, bv.bv_page, - btrfs_ino(bi), bv.bv_offset); + btrfs_clean_io_failure(bi, start, bv.bv_page, + bv.bv_offset); } else { int ret; @@ -209,7 +207,7 @@ static void end_compressed_bio_read(struct bio *bio) if (refcount_dec_and_test(&cb->pending_ios)) finish_compressed_bio_read(cb); btrfs_bio_free_csum(bbio); - bio_put(bio); + bio_put(&bbio->bio); } /* @@ -222,8 +220,7 @@ static noinline void end_compressed_writeback(struct inode *inode, struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); unsigned long index = cb->start >> PAGE_SHIFT; unsigned long end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT; - struct page *pages[16]; - unsigned long nr_pages = end_index - index + 1; + struct folio_batch fbatch; const int errno = blk_status_to_errno(cb->status); int i; int ret; @@ -231,24 +228,23 @@ static noinline void end_compressed_writeback(struct inode *inode, if (errno) mapping_set_error(inode->i_mapping, errno); - while (nr_pages > 0) { - ret = find_get_pages_contig(inode->i_mapping, index, - min_t(unsigned long, - nr_pages, ARRAY_SIZE(pages)), pages); - if (ret == 0) { - nr_pages -= 1; - index += 1; - continue; - } + folio_batch_init(&fbatch); + while (index <= end_index) { + ret = filemap_get_folios(inode->i_mapping, &index, end_index, + &fbatch); + + if (ret == 0) + return; + for (i = 0; i < ret; i++) { + struct folio *folio = fbatch.folios[i]; + if (errno) - SetPageError(pages[i]); - btrfs_page_clamp_clear_writeback(fs_info, pages[i], + folio_set_error(folio); + btrfs_page_clamp_clear_writeback(fs_info, &folio->page, cb->start, cb->len); - put_page(pages[i]); } - nr_pages -= ret; - index += ret; + folio_batch_release(&fbatch); } /* the inode may be gone now */ } @@ -301,20 +297,20 @@ static void btrfs_finish_compressed_write_work(struct work_struct *work) * This also calls the writeback end hooks for the file pages so that metadata * and checksums can be updated in the file. */ -static void end_compressed_bio_write(struct bio *bio) +static void end_compressed_bio_write(struct btrfs_bio *bbio) { - struct compressed_bio *cb = bio->bi_private; + struct compressed_bio *cb = bbio->private; - if (bio->bi_status) - cb->status = bio->bi_status; + if (bbio->bio.bi_status) + cb->status = bbio->bio.bi_status; if (refcount_dec_and_test(&cb->pending_ios)) { struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); - btrfs_record_physical_zoned(cb->inode, cb->start, bio); + btrfs_record_physical_zoned(cb->inode, cb->start, &bbio->bio); queue_work(fs_info->compressed_write_workers, &cb->write_end_work); } - bio_put(bio); + bio_put(&bbio->bio); } /* @@ -335,7 +331,8 @@ static void end_compressed_bio_write(struct bio *bio) static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_bytenr, - blk_opf_t opf, bio_end_io_t endio_func, + blk_opf_t opf, + btrfs_bio_end_io_t endio_func, u64 *next_stripe_start) { struct btrfs_fs_info *fs_info = btrfs_sb(cb->inode->i_sb); @@ -344,12 +341,8 @@ static struct bio *alloc_compressed_bio(struct compressed_bio *cb, u64 disk_byte struct bio *bio; int ret; - bio = btrfs_bio_alloc(BIO_MAX_VECS); - + bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, endio_func, cb); bio->bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; - bio->bi_opf = opf; - bio->bi_private = cb; - bio->bi_end_io = endio_func; em = btrfs_get_chunk_map(fs_info, disk_bytenr, fs_info->sectorsize); if (IS_ERR(em)) { @@ -478,8 +471,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, if (!skip_sum) { ret = btrfs_csum_one_bio(inode, bio, start, true); if (ret) { - bio->bi_status = ret; - bio_endio(bio); + btrfs_bio_end_io(btrfs_bio(bio), ret); break; } } @@ -519,7 +511,8 @@ static u64 bio_end_offset(struct bio *bio) */ static noinline int add_ra_bio_pages(struct inode *inode, u64 compressed_end, - struct compressed_bio *cb) + struct compressed_bio *cb, + unsigned long *pflags) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); unsigned long end_index; @@ -588,6 +581,9 @@ static noinline int add_ra_bio_pages(struct inode *inode, continue; } + if (PageWorkingset(page)) + psi_memstall_enter(pflags); + ret = set_page_extent_mapped(page); if (ret < 0) { unlock_page(page); @@ -596,7 +592,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, } page_end = (pg_index << PAGE_SHIFT) + PAGE_SIZE - 1; - lock_extent(tree, cur, page_end); + lock_extent(tree, cur, page_end, NULL); read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, cur, page_end + 1 - cur); read_unlock(&em_tree->lock); @@ -610,7 +606,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, (cur + fs_info->sectorsize > extent_map_end(em)) || (em->block_start >> 9) != cb->orig_bio->bi_iter.bi_sector) { free_extent_map(em); - unlock_extent(tree, cur, page_end); + unlock_extent(tree, cur, page_end, NULL); unlock_page(page); put_page(page); break; @@ -630,7 +626,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, add_size = min(em->start + em->len, page_end + 1) - cur; ret = bio_add_page(cb->orig_bio, page, add_size, offset_in_page(cur)); if (ret != add_size) { - unlock_extent(tree, cur, page_end); + unlock_extent(tree, cur, page_end, NULL); unlock_page(page); put_page(page); break; @@ -674,6 +670,8 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, u64 em_len; u64 em_start; struct extent_map *em; + /* Initialize to 1 to make skip psi_memstall_leave unless needed */ + unsigned long pflags = 1; blk_status_t ret; int ret2; int i; @@ -729,7 +727,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, goto fail; } - add_ra_bio_pages(inode, em_start + em_len, cb); + add_ra_bio_pages(inode, em_start + em_len, cb, &pflags); /* include any pages we added in add_ra-bio_pages */ cb->len = bio->bi_iter.bi_size; @@ -799,8 +797,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, ret = btrfs_lookup_bio_sums(inode, comp_bio, NULL); if (ret) { - comp_bio->bi_status = ret; - bio_endio(comp_bio); + btrfs_bio_end_io(btrfs_bio(comp_bio), ret); break; } @@ -810,6 +807,9 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, } } + if (!pflags) + psi_memstall_leave(&pflags); + if (refcount_dec_and_test(&cb->pending_ios)) finish_compressed_bio_read(cb); return; @@ -826,8 +826,7 @@ fail: kfree(cb); out: free_extent_map(em); - bio->bi_status = ret; - bio_endio(bio); + btrfs_bio_end_io(btrfs_bio(bio), ret); return; } diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index ebfa35fe1c38b0fe6194ee43abc0e79c5cd89978..b39b339fbf96ce31f4c3e858e8aef52710b6fd72 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1447,6 +1447,11 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, return 0; } + if (p->nowait) { + free_extent_buffer(tmp); + return -EAGAIN; + } + if (unlock_up) btrfs_unlock_up_safe(p, level + 1); @@ -1467,6 +1472,8 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, ret = -EAGAIN; goto out; + } else if (p->nowait) { + return -EAGAIN; } if (unlock_up) { @@ -1634,7 +1641,13 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, * We don't know the level of the root node until we actually * have it read locked */ - b = btrfs_read_lock_root_node(root); + if (p->nowait) { + b = btrfs_try_read_lock_root_node(root); + if (IS_ERR(b)) + return b; + } else { + b = btrfs_read_lock_root_node(root); + } level = btrfs_header_level(b); if (level > write_lock_level) goto out; @@ -1910,6 +1923,13 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, WARN_ON(p->nodes[0] != NULL); BUG_ON(!cow && ins_len); + /* + * For now only allow nowait for read only operations. There's no + * strict reason why we can't, we just only need it for reads so it's + * only implemented for reads. + */ + ASSERT(!p->nowait || !cow); + if (ins_len < 0) { lowest_unlock = 2; @@ -1936,7 +1956,12 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, if (p->need_commit_sem) { ASSERT(p->search_commit_root); - down_read(&fs_info->commit_root_sem); + if (p->nowait) { + if (!down_read_trylock(&fs_info->commit_root_sem)) + return -EAGAIN; + } else { + down_read(&fs_info->commit_root_sem); + } } again: @@ -2082,7 +2107,15 @@ cow_done: btrfs_tree_lock(b); p->locks[level] = BTRFS_WRITE_LOCK; } else { - btrfs_tree_read_lock(b); + if (p->nowait) { + if (!btrfs_try_tree_read_lock(b)) { + free_extent_buffer(b); + ret = -EAGAIN; + goto done; + } + } else { + btrfs_tree_read_lock(b); + } p->locks[level] = BTRFS_READ_LOCK; } p->nodes[level] = b; @@ -2131,6 +2164,7 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, lowest_level = p->lowest_level; WARN_ON(p->nodes[0] != NULL); + ASSERT(!p->nowait); if (p->search_commit_root) { BUG_ON(time_seq); @@ -4432,6 +4466,7 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, int ret = 1; int keep_locks = path->keep_locks; + ASSERT(!path->nowait); path->keep_locks = 1; again: cur = btrfs_read_lock_root_node(root); @@ -4612,6 +4647,8 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, int ret; int i; + ASSERT(!path->nowait); + nritems = btrfs_header_nritems(path->nodes[0]); if (nritems == 0) return 1; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 4edb4bfb21664f911a8fd3f4df9c3a59db3d41d1..727595eee9732c0f240c62a5b6cdd66d4f0a04dd 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -42,7 +42,6 @@ struct btrfs_delayed_ref_root; struct btrfs_space_info; struct btrfs_block_group; extern struct kmem_cache *btrfs_trans_handle_cachep; -extern struct kmem_cache *btrfs_bit_radix_cachep; extern struct kmem_cache *btrfs_path_cachep; extern struct kmem_cache *btrfs_free_space_cachep; extern struct kmem_cache *btrfs_free_space_bitmap_cachep; @@ -50,6 +49,11 @@ struct btrfs_ordered_sum; struct btrfs_ref; struct btrfs_bio; struct btrfs_ioctl_encoded_io_args; +struct btrfs_device; +struct btrfs_fs_devices; +struct btrfs_balance_control; +struct btrfs_delayed_root; +struct reloc_control; #define BTRFS_MAGIC 0x4D5F53665248425FULL /* ascii _BHRfS_M, no null */ @@ -280,14 +284,9 @@ struct btrfs_super_block { /* the UUID written into btree blocks */ u8 metadata_uuid[BTRFS_FSID_SIZE]; - /* Extent tree v2 */ - __le64 block_group_root; - __le64 block_group_root_generation; - u8 block_group_root_level; - /* future expansion */ - u8 reserved8[7]; - __le64 reserved[25]; + u8 reserved8[8]; + __le64 reserved[27]; u8 sys_chunk_array[BTRFS_SYSTEM_CHUNK_ARRAY_SIZE]; struct btrfs_root_backup super_roots[BTRFS_NUM_BACKUP_ROOTS]; @@ -307,7 +306,8 @@ static_assert(sizeof(struct btrfs_super_block) == BTRFS_SUPER_INFO_SIZE); #define BTRFS_FEATURE_COMPAT_RO_SUPP \ (BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE | \ BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE_VALID | \ - BTRFS_FEATURE_COMPAT_RO_VERITY) + BTRFS_FEATURE_COMPAT_RO_VERITY | \ + BTRFS_FEATURE_COMPAT_RO_BLOCK_GROUP_TREE) #define BTRFS_FEATURE_COMPAT_RO_SAFE_SET 0ULL #define BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR 0ULL @@ -443,9 +443,10 @@ struct btrfs_path { * header (ie. sizeof(struct btrfs_item) is not included). */ unsigned int search_for_extension:1; + /* Stop search if any locks need to be taken (for read) */ + unsigned int nowait:1; }; -#define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \ - sizeof(struct btrfs_item)) + struct btrfs_dev_replace { u64 replace_state; /* see #define above */ time64_t time_started; /* seconds since 1-Jan-1970 */ @@ -502,22 +503,6 @@ struct btrfs_free_cluster { struct list_head block_group_list; }; -enum btrfs_caching_type { - BTRFS_CACHE_NO, - BTRFS_CACHE_STARTED, - BTRFS_CACHE_FAST, - BTRFS_CACHE_FINISHED, - BTRFS_CACHE_ERROR, -}; - -/* - * Tree to record all locked full stripes of a RAID5/6 block group - */ -struct btrfs_full_stripe_locks_tree { - struct rb_root root; - struct mutex lock; -}; - /* Discard control. */ /* * Async discard uses multiple lists to differentiate the discard filter @@ -549,42 +534,6 @@ struct btrfs_discard_ctl { atomic64_t discard_bytes_saved; }; -void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info); - -/* fs_info */ -struct reloc_control; -struct btrfs_device; -struct btrfs_fs_devices; -struct btrfs_balance_control; -struct btrfs_delayed_root; - -/* - * Block group or device which contains an active swapfile. Used for preventing - * unsafe operations while a swapfile is active. - * - * These are sorted on (ptr, inode) (note that a block group or device can - * contain more than one swapfile). We compare the pointer values because we - * don't actually care what the object is, we just need a quick check whether - * the object exists in the rbtree. - */ -struct btrfs_swapfile_pin { - struct rb_node node; - void *ptr; - struct inode *inode; - /* - * If true, ptr points to a struct btrfs_block_group. Otherwise, ptr - * points to a struct btrfs_device. - */ - bool is_block_group; - /* - * Only used when 'is_block_group' is true and it is the number of - * extents used by a swapfile for this block group ('ptr' field). - */ - int bg_extent_count; -}; - -bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr); - enum { BTRFS_FS_CLOSING_START, BTRFS_FS_CLOSING_DONE, @@ -891,6 +840,7 @@ struct btrfs_fs_info { struct kobject *space_info_kobj; struct kobject *qgroups_kobj; + struct kobject *discard_kobj; /* used to keep from writing metadata until there is a nice batch */ struct percpu_counter dirty_metadata_bytes; @@ -1006,6 +956,7 @@ struct btrfs_fs_info { struct completion qgroup_rescan_completion; struct btrfs_work qgroup_rescan_work; bool qgroup_rescan_running; /* protected by qgroup_rescan_lock */ + u8 qgroup_drop_subtree_thres; /* filesystem state */ unsigned long fs_state; @@ -1089,12 +1040,27 @@ struct btrfs_fs_info { spinlock_t zone_active_bgs_lock; struct list_head zone_active_bgs; - /* Waiters when BTRFS_FS_NEED_ZONE_FINISH is set */ - wait_queue_head_t zone_finish_wait; /* Updates are not protected by any lock */ struct btrfs_commit_stats commit_stats; + /* + * Last generation where we dropped a non-relocation root. + * Use btrfs_set_last_root_drop_gen() and btrfs_get_last_root_drop_gen() + * to change it and to read it, respectively. + */ + u64 last_root_drop_gen; + + /* + * Annotations for transaction events (structures are empty when + * compiled without lockdep). + */ + struct lockdep_map btrfs_trans_num_writers_map; + struct lockdep_map btrfs_trans_num_extwriters_map; + struct lockdep_map btrfs_state_change_map[4]; + struct lockdep_map btrfs_trans_pending_ordered_map; + struct lockdep_map btrfs_ordered_extent_map; + #ifdef CONFIG_BTRFS_FS_REF_VERIFY spinlock_t ref_verify_lock; struct rb_root block_tree; @@ -1102,7 +1068,6 @@ struct btrfs_fs_info { #ifdef CONFIG_BTRFS_DEBUG struct kobject *debug_kobj; - struct kobject *discard_debug_kobj; struct list_head allocated_roots; spinlock_t eb_leak_lock; @@ -1110,11 +1075,84 @@ struct btrfs_fs_info { #endif }; +static inline void btrfs_set_last_root_drop_gen(struct btrfs_fs_info *fs_info, + u64 gen) +{ + WRITE_ONCE(fs_info->last_root_drop_gen, gen); +} + +static inline u64 btrfs_get_last_root_drop_gen(const struct btrfs_fs_info *fs_info) +{ + return READ_ONCE(fs_info->last_root_drop_gen); +} + static inline struct btrfs_fs_info *btrfs_sb(struct super_block *sb) { return sb->s_fs_info; } +/* + * Take the number of bytes to be checksummed and figure out how many leaves + * it would require to store the csums for that many bytes. + */ +static inline u64 btrfs_csum_bytes_to_leaves( + const struct btrfs_fs_info *fs_info, u64 csum_bytes) +{ + const u64 num_csums = csum_bytes >> fs_info->sectorsize_bits; + + return DIV_ROUND_UP_ULL(num_csums, fs_info->csums_per_leaf); +} + +/* + * Use this if we would be adding new items, as we could split nodes as we cow + * down the tree. + */ +static inline u64 btrfs_calc_insert_metadata_size(struct btrfs_fs_info *fs_info, + unsigned num_items) +{ + return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * 2 * num_items; +} + +/* + * Doing a truncate or a modification won't result in new nodes or leaves, just + * what we need for COW. + */ +static inline u64 btrfs_calc_metadata_size(struct btrfs_fs_info *fs_info, + unsigned num_items) +{ + return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * num_items; +} + +#define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \ + sizeof(struct btrfs_item)) + +static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info) +{ + return fs_info->zone_size > 0; +} + +/* + * Count how many fs_info->max_extent_size cover the @size + */ +static inline u32 count_max_extents(struct btrfs_fs_info *fs_info, u64 size) +{ +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + if (!fs_info) + return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE); +#endif + + return div_u64(size + fs_info->max_extent_size - 1, fs_info->max_extent_size); +} + +bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type); +bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type); +void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info); +void btrfs_exclop_finish(struct btrfs_fs_info *fs_info); +void btrfs_exclop_balance(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation op); + /* * The state of btrfs root */ @@ -1177,6 +1215,82 @@ enum { BTRFS_ROOT_RESET_LOCKDEP_CLASS, }; +enum btrfs_lockdep_trans_states { + BTRFS_LOCKDEP_TRANS_COMMIT_START, + BTRFS_LOCKDEP_TRANS_UNBLOCKED, + BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED, + BTRFS_LOCKDEP_TRANS_COMPLETED, +}; + +/* + * Lockdep annotation for wait events. + * + * @owner: The struct where the lockdep map is defined + * @lock: The lockdep map corresponding to a wait event + * + * This macro is used to annotate a wait event. In this case a thread acquires + * the lockdep map as writer (exclusive lock) because it has to block until all + * the threads that hold the lock as readers signal the condition for the wait + * event and release their locks. + */ +#define btrfs_might_wait_for_event(owner, lock) \ + do { \ + rwsem_acquire(&owner->lock##_map, 0, 0, _THIS_IP_); \ + rwsem_release(&owner->lock##_map, _THIS_IP_); \ + } while (0) + +/* + * Protection for the resource/condition of a wait event. + * + * @owner: The struct where the lockdep map is defined + * @lock: The lockdep map corresponding to a wait event + * + * Many threads can modify the condition for the wait event at the same time + * and signal the threads that block on the wait event. The threads that modify + * the condition and do the signaling acquire the lock as readers (shared + * lock). + */ +#define btrfs_lockdep_acquire(owner, lock) \ + rwsem_acquire_read(&owner->lock##_map, 0, 0, _THIS_IP_) + +/* + * Used after signaling the condition for a wait event to release the lockdep + * map held by a reader thread. + */ +#define btrfs_lockdep_release(owner, lock) \ + rwsem_release(&owner->lock##_map, _THIS_IP_) + +/* + * Macros for the transaction states wait events, similar to the generic wait + * event macros. + */ +#define btrfs_might_wait_for_state(owner, i) \ + do { \ + rwsem_acquire(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_); \ + rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_); \ + } while (0) + +#define btrfs_trans_state_lockdep_acquire(owner, i) \ + rwsem_acquire_read(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_) + +#define btrfs_trans_state_lockdep_release(owner, i) \ + rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_) + +/* Initialization of the lockdep map */ +#define btrfs_lockdep_init_map(owner, lock) \ + do { \ + static struct lock_class_key lock##_key; \ + lockdep_init_map(&owner->lock##_map, #lock, &lock##_key, 0); \ + } while (0) + +/* Initialization of the transaction states lockdep maps. */ +#define btrfs_state_lockdep_init_map(owner, lock, state) \ + do { \ + static struct lock_class_key lock##_key; \ + lockdep_init_map(&owner->btrfs_state_change_map[state], #lock, \ + &lock##_key, 0); \ + } while (0) + static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info) { clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags); @@ -2394,17 +2508,6 @@ BTRFS_SETGET_STACK_FUNCS(backup_bytes_used, struct btrfs_root_backup, BTRFS_SETGET_STACK_FUNCS(backup_num_devices, struct btrfs_root_backup, num_devices, 64); -/* - * For extent tree v2 we overload the extent root with the block group root, as - * we will have multiple extent roots. - */ -BTRFS_SETGET_STACK_FUNCS(backup_block_group_root, struct btrfs_root_backup, - extent_root, 64); -BTRFS_SETGET_STACK_FUNCS(backup_block_group_root_gen, struct btrfs_root_backup, - extent_root_gen, 64); -BTRFS_SETGET_STACK_FUNCS(backup_block_group_root_level, - struct btrfs_root_backup, extent_root_level, 8); - /* struct btrfs_balance_item */ BTRFS_SETGET_FUNCS(balance_flags, struct btrfs_balance_item, flags, 64); @@ -2537,13 +2640,6 @@ BTRFS_SETGET_STACK_FUNCS(super_cache_generation, struct btrfs_super_block, BTRFS_SETGET_STACK_FUNCS(super_magic, struct btrfs_super_block, magic, 64); BTRFS_SETGET_STACK_FUNCS(super_uuid_tree_generation, struct btrfs_super_block, uuid_tree_generation, 64); -BTRFS_SETGET_STACK_FUNCS(super_block_group_root, struct btrfs_super_block, - block_group_root, 64); -BTRFS_SETGET_STACK_FUNCS(super_block_group_root_generation, - struct btrfs_super_block, - block_group_root_generation, 64); -BTRFS_SETGET_STACK_FUNCS(super_block_group_root_level, struct btrfs_super_block, - block_group_root_level, 8); int btrfs_super_csum_size(const struct btrfs_super_block *s); const char *btrfs_super_csum_name(u16 csum_type); @@ -2764,45 +2860,6 @@ int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, enum btrfs_inline_ref_type is_data); u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset); -static inline u8 *btrfs_csum_ptr(const struct btrfs_fs_info *fs_info, u8 *csums, - u64 offset) -{ - u64 offset_in_sectors = offset >> fs_info->sectorsize_bits; - - return csums + offset_in_sectors * fs_info->csum_size; -} - -/* - * Take the number of bytes to be checksummed and figure out how many leaves - * it would require to store the csums for that many bytes. - */ -static inline u64 btrfs_csum_bytes_to_leaves( - const struct btrfs_fs_info *fs_info, u64 csum_bytes) -{ - const u64 num_csums = csum_bytes >> fs_info->sectorsize_bits; - - return DIV_ROUND_UP_ULL(num_csums, fs_info->csums_per_leaf); -} - -/* - * Use this if we would be adding new items, as we could split nodes as we cow - * down the tree. - */ -static inline u64 btrfs_calc_insert_metadata_size(struct btrfs_fs_info *fs_info, - unsigned num_items) -{ - return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * 2 * num_items; -} - -/* - * Doing a truncate or a modification won't result in new nodes or leaves, just - * what we need for COW. - */ -static inline u64 btrfs_calc_metadata_size(struct btrfs_fs_info *fs_info, - unsigned num_items) -{ - return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * num_items; -} int btrfs_add_excluded_extent(struct btrfs_fs_info *fs_info, u64 start, u64 num_bytes); @@ -3260,12 +3317,9 @@ int btrfs_find_orphan_item(struct btrfs_root *root, u64 offset); int btrfs_del_csums(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 bytenr, u64 len); blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst); -int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - u64 objectid, u64 pos, - u64 disk_offset, u64 disk_num_bytes, - u64 num_bytes, u64 offset, u64 ram_bytes, - u8 compression, u8 encryption, u16 other_encoding); +int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 objectid, u64 pos, + u64 num_bytes); int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, u64 objectid, @@ -3276,7 +3330,8 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, u64 offset, bool one_ordered); int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, - struct list_head *list, int search_commit); + struct list_head *list, int search_commit, + bool nowait); void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, const struct btrfs_path *path, struct btrfs_file_extent_item *fi, @@ -3302,11 +3357,9 @@ unsigned int btrfs_verify_data_csum(struct btrfs_bio *bbio, u64 start, u64 end); int btrfs_check_data_csum(struct inode *inode, struct btrfs_bio *bbio, u32 bio_offset, struct page *page, u32 pgoff); -struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, - u64 start, u64 len); noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, u64 *orig_start, u64 *orig_block_len, - u64 *ram_bytes, bool strict); + u64 *ram_bytes, bool nowait, bool strict); void __btrfs_del_delalloc_inode(struct btrfs_root *root, struct btrfs_inode *inode); @@ -3361,7 +3414,6 @@ void btrfs_split_delalloc_extent(struct inode *inode, void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end); vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf); void btrfs_evict_inode(struct inode *inode); -int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc); struct inode *btrfs_alloc_inode(struct super_block *sb); void btrfs_destroy_inode(struct inode *inode); void btrfs_free_inode(struct inode *inode); @@ -3442,15 +3494,6 @@ void btrfs_get_block_group_info(struct list_head *groups_list, struct btrfs_ioctl_space_info *space); void btrfs_update_ioctl_balance_args(struct btrfs_fs_info *fs_info, struct btrfs_ioctl_balance_args *bargs); -bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, - enum btrfs_exclusive_operation type); -bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, - enum btrfs_exclusive_operation type); -void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info); -void btrfs_exclop_finish(struct btrfs_fs_info *fs_info); -void btrfs_exclop_balance(struct btrfs_fs_info *fs_info, - enum btrfs_exclusive_operation op); - /* file.c */ int __init btrfs_auto_defrag_init(void); @@ -3460,8 +3503,6 @@ int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans, int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info); void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info); int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync); -void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end, - int skip_pinned); extern const struct file_operations btrfs_file_operations; int btrfs_drop_extents(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_inode *inode, @@ -3481,8 +3522,10 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, struct extent_state **cached, bool noreserve); int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end); int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, - size_t *write_bytes); + size_t *write_bytes, bool nowait); void btrfs_check_nocow_unlock(struct btrfs_inode *inode); +bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, + u64 *delalloc_start_ret, u64 *delalloc_end_ret); /* tree-defrag.c */ int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, @@ -3748,7 +3791,7 @@ const char * __attribute_const__ btrfs_decode_error(int errno); __cold void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int errno); + unsigned int line, int errno, bool first_hit); /* * Call btrfs_abort_transaction as early as possible when an error condition is @@ -3756,9 +3799,11 @@ void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, */ #define btrfs_abort_transaction(trans, errno) \ do { \ + bool first = false; \ /* Report first abort since mount */ \ if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ &((trans)->fs_info->fs_state))) { \ + first = true; \ if ((errno) != -EIO && (errno) != -EROFS) { \ WARN(1, KERN_DEBUG \ "BTRFS: Transaction aborted (error %d)\n", \ @@ -3770,7 +3815,7 @@ do { \ } \ } \ __btrfs_abort_transaction((trans), __func__, \ - __LINE__, (errno)); \ + __LINE__, (errno), first); \ } while (0) #ifdef CONFIG_PRINTK_INDEX @@ -3987,16 +4032,9 @@ int btrfs_scrub_cancel(struct btrfs_fs_info *info); int btrfs_scrub_cancel_dev(struct btrfs_device *dev); int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, struct btrfs_scrub_progress *progress); -static inline void btrfs_init_full_stripe_locks_tree( - struct btrfs_full_stripe_locks_tree *locks_root) -{ - locks_root->root = RB_ROOT; - mutex_init(&locks_root->lock); -} /* dev-replace.c */ void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info); -void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info); void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount); static inline void btrfs_bio_counter_dec(struct btrfs_fs_info *fs_info) @@ -4023,6 +4061,7 @@ static inline int btrfs_defrag_cancelled(struct btrfs_fs_info *fs_info) extern const struct fsverity_operations btrfs_verityops; int btrfs_drop_verity_items(struct btrfs_inode *inode); +int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size); BTRFS_SETGET_FUNCS(verity_descriptor_encryption, struct btrfs_verity_descriptor_item, encryption, 8); @@ -4040,6 +4079,12 @@ static inline int btrfs_drop_verity_items(struct btrfs_inode *inode) return 0; } +static inline int btrfs_get_verity_descriptor(struct inode *inode, void *buf, + size_t buf_size) +{ + return -EPERM; +} + #endif /* Sanity test specific functions */ @@ -4056,24 +4101,6 @@ static inline int btrfs_is_testing(struct btrfs_fs_info *fs_info) } #endif -static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info) -{ - return fs_info->zone_size > 0; -} - -/* - * Count how many fs_info->max_extent_size cover the @size - */ -static inline u32 count_max_extents(struct btrfs_fs_info *fs_info, u64 size) -{ -#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS - if (!fs_info) - return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE); -#endif - - return div_u64(size + fs_info->max_extent_size - 1, fs_info->max_extent_size); -} - static inline bool btrfs_is_data_reloc_root(const struct btrfs_root *root) { return root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID; diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 1e8f17ff829e3508b9438b5d49666bde40c5582d..118b2e20b2e192b12f5a6fb9ca5848e826ef3b3c 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -127,9 +127,11 @@ int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes) } int btrfs_check_data_free_space(struct btrfs_inode *inode, - struct extent_changeset **reserved, u64 start, u64 len) + struct extent_changeset **reserved, u64 start, + u64 len, bool noflush) { struct btrfs_fs_info *fs_info = inode->root->fs_info; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA; int ret; /* align the range */ @@ -137,7 +139,12 @@ int btrfs_check_data_free_space(struct btrfs_inode *inode, round_down(start, fs_info->sectorsize); start = round_down(start, fs_info->sectorsize); - ret = btrfs_alloc_data_chunk_ondemand(inode, len); + if (noflush) + flush = BTRFS_RESERVE_NO_FLUSH; + else if (btrfs_is_free_space_inode(inode)) + flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + + ret = btrfs_reserve_data_bytes(fs_info, len, flush); if (ret < 0) return ret; @@ -454,7 +461,7 @@ int btrfs_delalloc_reserve_space(struct btrfs_inode *inode, { int ret; - ret = btrfs_check_data_free_space(inode, reserved, start, len); + ret = btrfs_check_data_free_space(inode, reserved, start, len, false); if (ret < 0) return ret; ret = btrfs_delalloc_reserve_metadata(inode, len, len, false); diff --git a/fs/btrfs/delalloc-space.h b/fs/btrfs/delalloc-space.h index 28bf5c3ef43053dd0395a8714e565dce184f254c..e07d46043455aa654117b6456e2515ca9f4d2858 100644 --- a/fs/btrfs/delalloc-space.h +++ b/fs/btrfs/delalloc-space.h @@ -7,7 +7,8 @@ struct extent_changeset; int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes); int btrfs_check_data_free_space(struct btrfs_inode *inode, - struct extent_changeset **reserved, u64 start, u64 len); + struct extent_changeset **reserved, u64 start, u64 len, + bool noflush); void btrfs_free_reserved_data_space(struct btrfs_inode *inode, struct extent_changeset *reserved, u64 start, u64 len); void btrfs_delalloc_release_space(struct btrfs_inode *inode, diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index e7f34871a132898a64ceb5e4d16ad468b8602913..cac5169eaf8dea6256bae0165d757cbc51f60f23 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -302,15 +302,21 @@ static inline void btrfs_release_prepared_delayed_node( __btrfs_release_delayed_node(node, 1); } -static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u32 data_len) +static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u16 data_len, + struct btrfs_delayed_node *node, + enum btrfs_delayed_item_type type) { struct btrfs_delayed_item *item; + item = kmalloc(sizeof(*item) + data_len, GFP_NOFS); if (item) { item->data_len = data_len; - item->ins_or_del = 0; + item->type = type; item->bytes_reserved = 0; - item->delayed_node = NULL; + item->delayed_node = node; + RB_CLEAR_NODE(&item->rb_node); + INIT_LIST_HEAD(&item->log_list); + item->logged = false; refcount_set(&item->refs, 1); } return item; @@ -319,72 +325,32 @@ static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u32 data_len) /* * __btrfs_lookup_delayed_item - look up the delayed item by key * @delayed_node: pointer to the delayed node - * @key: the key to look up - * @prev: used to store the prev item if the right item isn't found - * @next: used to store the next item if the right item isn't found + * @index: the dir index value to lookup (offset of a dir index key) * * Note: if we don't find the right item, we will return the prev item and * the next item. */ static struct btrfs_delayed_item *__btrfs_lookup_delayed_item( struct rb_root *root, - struct btrfs_key *key, - struct btrfs_delayed_item **prev, - struct btrfs_delayed_item **next) + u64 index) { - struct rb_node *node, *prev_node = NULL; + struct rb_node *node = root->rb_node; struct btrfs_delayed_item *delayed_item = NULL; - int ret = 0; - - node = root->rb_node; while (node) { delayed_item = rb_entry(node, struct btrfs_delayed_item, rb_node); - prev_node = node; - ret = btrfs_comp_cpu_keys(&delayed_item->key, key); - if (ret < 0) + if (delayed_item->index < index) node = node->rb_right; - else if (ret > 0) + else if (delayed_item->index > index) node = node->rb_left; else return delayed_item; } - if (prev) { - if (!prev_node) - *prev = NULL; - else if (ret < 0) - *prev = delayed_item; - else if ((node = rb_prev(prev_node)) != NULL) { - *prev = rb_entry(node, struct btrfs_delayed_item, - rb_node); - } else - *prev = NULL; - } - - if (next) { - if (!prev_node) - *next = NULL; - else if (ret > 0) - *next = delayed_item; - else if ((node = rb_next(prev_node)) != NULL) { - *next = rb_entry(node, struct btrfs_delayed_item, - rb_node); - } else - *next = NULL; - } return NULL; } -static struct btrfs_delayed_item *__btrfs_lookup_delayed_insertion_item( - struct btrfs_delayed_node *delayed_node, - struct btrfs_key *key) -{ - return __btrfs_lookup_delayed_item(&delayed_node->ins_root.rb_root, key, - NULL, NULL); -} - static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, struct btrfs_delayed_item *ins) { @@ -392,15 +358,13 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, struct rb_node *parent_node = NULL; struct rb_root_cached *root; struct btrfs_delayed_item *item; - int cmp; bool leftmost = true; - if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM) + if (ins->type == BTRFS_DELAYED_INSERTION_ITEM) root = &delayed_node->ins_root; - else if (ins->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) - root = &delayed_node->del_root; else - BUG(); + root = &delayed_node->del_root; + p = &root->rb_root.rb_node; node = &ins->rb_node; @@ -409,11 +373,10 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, item = rb_entry(parent_node, struct btrfs_delayed_item, rb_node); - cmp = btrfs_comp_cpu_keys(&item->key, &ins->key); - if (cmp < 0) { + if (item->index < ins->index) { p = &(*p)->rb_right; leftmost = false; - } else if (cmp > 0) { + } else if (item->index > ins->index) { p = &(*p)->rb_left; } else { return -EEXIST; @@ -422,14 +385,10 @@ static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, rb_link_node(node, parent_node, p); rb_insert_color_cached(node, root, leftmost); - ins->delayed_node = delayed_node; - - /* Delayed items are always for dir index items. */ - ASSERT(ins->key.type == BTRFS_DIR_INDEX_KEY); - if (ins->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM && - ins->key.offset >= delayed_node->index_cnt) - delayed_node->index_cnt = ins->key.offset + 1; + if (ins->type == BTRFS_DELAYED_INSERTION_ITEM && + ins->index >= delayed_node->index_cnt) + delayed_node->index_cnt = ins->index + 1; delayed_node->count++; atomic_inc(&delayed_node->root->fs_info->delayed_root->items); @@ -451,21 +410,21 @@ static void __btrfs_remove_delayed_item(struct btrfs_delayed_item *delayed_item) struct rb_root_cached *root; struct btrfs_delayed_root *delayed_root; - /* Not associated with any delayed_node */ - if (!delayed_item->delayed_node) + /* Not inserted, ignore it. */ + if (RB_EMPTY_NODE(&delayed_item->rb_node)) return; + delayed_root = delayed_item->delayed_node->root->fs_info->delayed_root; BUG_ON(!delayed_root); - BUG_ON(delayed_item->ins_or_del != BTRFS_DELAYED_DELETION_ITEM && - delayed_item->ins_or_del != BTRFS_DELAYED_INSERTION_ITEM); - if (delayed_item->ins_or_del == BTRFS_DELAYED_INSERTION_ITEM) + if (delayed_item->type == BTRFS_DELAYED_INSERTION_ITEM) root = &delayed_item->delayed_node->ins_root; else root = &delayed_item->delayed_node->del_root; rb_erase_cached(&delayed_item->rb_node, root); + RB_CLEAR_NODE(&delayed_item->rb_node); delayed_item->delayed_node->count--; finish_one_item(delayed_root); @@ -520,12 +479,11 @@ static struct btrfs_delayed_item *__btrfs_next_delayed_item( } static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, - struct btrfs_root *root, struct btrfs_delayed_item *item) { struct btrfs_block_rsv *src_rsv; struct btrfs_block_rsv *dst_rsv; - struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_fs_info *fs_info = trans->fs_info; u64 num_bytes; int ret; @@ -545,14 +503,14 @@ static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, true); if (!ret) { trace_btrfs_space_reservation(fs_info, "delayed_item", - item->key.objectid, + item->delayed_node->inode_id, num_bytes, 1); /* * For insertions we track reserved metadata space by accounting * for the number of leaves that will be used, based on the delayed * node's index_items_size field. */ - if (item->ins_or_del == BTRFS_DELAYED_DELETION_ITEM) + if (item->type == BTRFS_DELAYED_DELETION_ITEM) item->bytes_reserved = num_bytes; } @@ -574,8 +532,8 @@ static void btrfs_delayed_item_release_metadata(struct btrfs_root *root, * to release/reserve qgroup space. */ trace_btrfs_space_reservation(fs_info, "delayed_item", - item->key.objectid, item->bytes_reserved, - 0); + item->delayed_node->inode_id, + item->bytes_reserved, 0); btrfs_block_rsv_release(fs_info, rsv, item->bytes_reserved, NULL); } @@ -688,6 +646,8 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, struct btrfs_delayed_item *next; const int max_size = BTRFS_LEAF_DATA_SIZE(fs_info); struct btrfs_item_batch batch; + struct btrfs_key first_key; + const u32 first_data_size = first_item->data_len; int total_size; char *ins_data = NULL; int ret; @@ -716,9 +676,9 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, ASSERT(first_item->bytes_reserved == 0); list_add_tail(&first_item->tree_list, &item_list); - batch.total_data_size = first_item->data_len; + batch.total_data_size = first_data_size; batch.nr = 1; - total_size = first_item->data_len + sizeof(struct btrfs_item); + total_size = first_data_size + sizeof(struct btrfs_item); curr = first_item; while (true) { @@ -732,8 +692,7 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, * We cannot allow gaps in the key space if we're doing log * replay. */ - if (continuous_keys_only && - (next->key.offset != curr->key.offset + 1)) + if (continuous_keys_only && (next->index != curr->index + 1)) break; ASSERT(next->bytes_reserved == 0); @@ -750,8 +709,11 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, } if (batch.nr == 1) { - batch.keys = &first_item->key; - batch.data_sizes = &first_item->data_len; + first_key.objectid = node->inode_id; + first_key.type = BTRFS_DIR_INDEX_KEY; + first_key.offset = first_item->index; + batch.keys = &first_key; + batch.data_sizes = &first_data_size; } else { struct btrfs_key *ins_keys; u32 *ins_sizes; @@ -768,7 +730,9 @@ static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, batch.keys = ins_keys; batch.data_sizes = ins_sizes; list_for_each_entry(curr, &item_list, tree_list) { - ins_keys[i] = curr->key; + ins_keys[i].objectid = node->inode_id; + ins_keys[i].type = BTRFS_DIR_INDEX_KEY; + ins_keys[i].offset = curr->index; ins_sizes[i] = curr->data_len; i++; } @@ -864,6 +828,7 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_delayed_item *item) { + const u64 ino = item->delayed_node->inode_id; struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_delayed_item *curr, *next; struct extent_buffer *leaf = path->nodes[0]; @@ -902,7 +867,9 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, slot++; btrfs_item_key_to_cpu(leaf, &key, slot); - if (btrfs_comp_cpu_keys(&next->key, &key) != 0) + if (key.objectid != ino || + key.type != BTRFS_DIR_INDEX_KEY || + key.offset != next->index) break; nitems++; curr = next; @@ -920,9 +887,8 @@ static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, * Check btrfs_delayed_item_reserve_metadata() to see why we * don't need to release/reserve qgroup space. */ - trace_btrfs_space_reservation(fs_info, "delayed_item", - item->key.objectid, total_reserved_size, - 0); + trace_btrfs_space_reservation(fs_info, "delayed_item", ino, + total_reserved_size, 0); btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, total_reserved_size, NULL); } @@ -940,8 +906,12 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_delayed_node *node) { + struct btrfs_key key; int ret = 0; + key.objectid = node->inode_id; + key.type = BTRFS_DIR_INDEX_KEY; + while (ret == 0) { struct btrfs_delayed_item *item; @@ -952,7 +922,8 @@ static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, break; } - ret = btrfs_search_slot(trans, root, &item->key, path, -1, 1); + key.offset = item->index; + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); if (ret > 0) { /* * There's no matching item in the leaf. This means we @@ -1457,16 +1428,15 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, if (IS_ERR(delayed_node)) return PTR_ERR(delayed_node); - delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len); + delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len, + delayed_node, + BTRFS_DELAYED_INSERTION_ITEM); if (!delayed_item) { ret = -ENOMEM; goto release_node; } - delayed_item->key.objectid = btrfs_ino(dir); - delayed_item->key.type = BTRFS_DIR_INDEX_KEY; - delayed_item->key.offset = index; - delayed_item->ins_or_del = BTRFS_DELAYED_INSERTION_ITEM; + delayed_item->index = index; dir_item = (struct btrfs_dir_item *)delayed_item->data; dir_item->location = *disk_key; @@ -1490,8 +1460,7 @@ int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, } if (reserve_leaf_space) { - ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, - delayed_item); + ret = btrfs_delayed_item_reserve_metadata(trans, delayed_item); /* * Space was reserved for a dir index item insertion when we * started the transaction, so getting a failure here should be @@ -1538,12 +1507,12 @@ release_node: static int btrfs_delete_delayed_insertion_item(struct btrfs_fs_info *fs_info, struct btrfs_delayed_node *node, - struct btrfs_key *key) + u64 index) { struct btrfs_delayed_item *item; mutex_lock(&node->mutex); - item = __btrfs_lookup_delayed_insertion_item(node, key); + item = __btrfs_lookup_delayed_item(&node->ins_root.rb_root, index); if (!item) { mutex_unlock(&node->mutex); return 1; @@ -1589,32 +1558,25 @@ int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, { struct btrfs_delayed_node *node; struct btrfs_delayed_item *item; - struct btrfs_key item_key; int ret; node = btrfs_get_or_create_delayed_node(dir); if (IS_ERR(node)) return PTR_ERR(node); - item_key.objectid = btrfs_ino(dir); - item_key.type = BTRFS_DIR_INDEX_KEY; - item_key.offset = index; - - ret = btrfs_delete_delayed_insertion_item(trans->fs_info, node, - &item_key); + ret = btrfs_delete_delayed_insertion_item(trans->fs_info, node, index); if (!ret) goto end; - item = btrfs_alloc_delayed_item(0); + item = btrfs_alloc_delayed_item(0, node, BTRFS_DELAYED_DELETION_ITEM); if (!item) { ret = -ENOMEM; goto end; } - item->key = item_key; - item->ins_or_del = BTRFS_DELAYED_DELETION_ITEM; + item->index = index; - ret = btrfs_delayed_item_reserve_metadata(trans, dir->root, item); + ret = btrfs_delayed_item_reserve_metadata(trans, item); /* * we have reserved enough space when we start a new transaction, * so reserving metadata failure is impossible. @@ -1743,9 +1705,9 @@ int btrfs_should_delete_dir_index(struct list_head *del_list, int ret = 0; list_for_each_entry(curr, del_list, readdir_list) { - if (curr->key.offset > index) + if (curr->index > index) break; - if (curr->key.offset == index) { + if (curr->index == index) { ret = 1; break; } @@ -1779,13 +1741,13 @@ int btrfs_readdir_delayed_dir_index(struct dir_context *ctx, list_for_each_entry_safe(curr, next, ins_list, readdir_list) { list_del(&curr->readdir_list); - if (curr->key.offset < ctx->pos) { + if (curr->index < ctx->pos) { if (refcount_dec_and_test(&curr->refs)) kfree(curr); continue; } - ctx->pos = curr->key.offset; + ctx->pos = curr->index; di = (struct btrfs_dir_item *)curr->data; name = (char *)(di + 1); @@ -2085,3 +2047,113 @@ void btrfs_destroy_delayed_inodes(struct btrfs_fs_info *fs_info) } } +void btrfs_log_get_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_node *node; + struct btrfs_delayed_item *item; + + node = btrfs_get_delayed_node(inode); + if (!node) + return; + + mutex_lock(&node->mutex); + item = __btrfs_first_delayed_insertion_item(node); + while (item) { + /* + * It's possible that the item is already in a log list. This + * can happen in case two tasks are trying to log the same + * directory. For example if we have tasks A and task B: + * + * Task A collected the delayed items into a log list while + * under the inode's log_mutex (at btrfs_log_inode()), but it + * only releases the items after logging the inodes they point + * to (if they are new inodes), which happens after unlocking + * the log mutex; + * + * Task B enters btrfs_log_inode() and acquires the log_mutex + * of the same directory inode, before task B releases the + * delayed items. This can happen for example when logging some + * inode we need to trigger logging of its parent directory, so + * logging two files that have the same parent directory can + * lead to this. + * + * If this happens, just ignore delayed items already in a log + * list. All the tasks logging the directory are under a log + * transaction and whichever finishes first can not sync the log + * before the other completes and leaves the log transaction. + */ + if (!item->logged && list_empty(&item->log_list)) { + refcount_inc(&item->refs); + list_add_tail(&item->log_list, ins_list); + } + item = __btrfs_next_delayed_item(item); + } + + item = __btrfs_first_delayed_deletion_item(node); + while (item) { + /* It may be non-empty, for the same reason mentioned above. */ + if (!item->logged && list_empty(&item->log_list)) { + refcount_inc(&item->refs); + list_add_tail(&item->log_list, del_list); + } + item = __btrfs_next_delayed_item(item); + } + mutex_unlock(&node->mutex); + + /* + * We are called during inode logging, which means the inode is in use + * and can not be evicted before we finish logging the inode. So we never + * have the last reference on the delayed inode. + * Also, we don't use btrfs_release_delayed_node() because that would + * requeue the delayed inode (change its order in the list of prepared + * nodes) and we don't want to do such change because we don't create or + * delete delayed items. + */ + ASSERT(refcount_read(&node->refs) > 1); + refcount_dec(&node->refs); +} + +void btrfs_log_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_node *node; + struct btrfs_delayed_item *item; + struct btrfs_delayed_item *next; + + node = btrfs_get_delayed_node(inode); + if (!node) + return; + + mutex_lock(&node->mutex); + + list_for_each_entry_safe(item, next, ins_list, log_list) { + item->logged = true; + list_del_init(&item->log_list); + if (refcount_dec_and_test(&item->refs)) + kfree(item); + } + + list_for_each_entry_safe(item, next, del_list, log_list) { + item->logged = true; + list_del_init(&item->log_list); + if (refcount_dec_and_test(&item->refs)) + kfree(item); + } + + mutex_unlock(&node->mutex); + + /* + * We are called during inode logging, which means the inode is in use + * and can not be evicted before we finish logging the inode. So we never + * have the last reference on the delayed inode. + * Also, we don't use btrfs_release_delayed_node() because that would + * requeue the delayed inode (change its order in the list of prepared + * nodes) and we don't want to do such change because we don't create or + * delete delayed items. + */ + ASSERT(refcount_read(&node->refs) > 1); + refcount_dec(&node->refs); +} diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h index 9795dc295a183ca658d92cfdbb87c32771c873b0..0163ca637a96fa5af2b42e541946dae67cb9b4d3 100644 --- a/fs/btrfs/delayed-inode.h +++ b/fs/btrfs/delayed-inode.h @@ -16,9 +16,10 @@ #include #include "ctree.h" -/* types of the delayed item */ -#define BTRFS_DELAYED_INSERTION_ITEM 1 -#define BTRFS_DELAYED_DELETION_ITEM 2 +enum btrfs_delayed_item_type { + BTRFS_DELAYED_INSERTION_ITEM, + BTRFS_DELAYED_DELETION_ITEM +}; struct btrfs_delayed_root { spinlock_t lock; @@ -73,14 +74,27 @@ struct btrfs_delayed_node { struct btrfs_delayed_item { struct rb_node rb_node; - struct btrfs_key key; + /* Offset value of the corresponding dir index key. */ + u64 index; struct list_head tree_list; /* used for batch insert/delete items */ struct list_head readdir_list; /* used for readdir items */ + /* + * Used when logging a directory. + * Insertions and deletions to this list are protected by the parent + * delayed node's mutex. + */ + struct list_head log_list; u64 bytes_reserved; struct btrfs_delayed_node *delayed_node; refcount_t refs; - int ins_or_del; - u32 data_len; + enum btrfs_delayed_item_type type:8; + /* + * Track if this delayed item was already logged. + * Protected by the mutex of the parent delayed inode. + */ + bool logged; + /* The maximum leaf size is 64K, so u16 is more than enough. */ + u16 data_len; char data[]; }; @@ -144,6 +158,14 @@ int btrfs_should_delete_dir_index(struct list_head *del_list, int btrfs_readdir_delayed_dir_index(struct dir_context *ctx, struct list_head *ins_list); +/* Used during directory logging. */ +void btrfs_log_get_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list); +void btrfs_log_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list); + /* for init */ int __init btrfs_delayed_inode_init(void); void __cold btrfs_delayed_inode_exit(void); diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index f43196a893ca3c44797d364a74538b5895cb3b91..61e58066b5fd2bbd6216e84b58896216d042135f 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -165,7 +165,7 @@ no_valid_dev_replace_entry_found: */ if (btrfs_find_device(fs_info->fs_devices, &args)) { btrfs_err(fs_info, - "replace devid present without an active replace item"); +"replace without active item, run 'device scan --forget' on the target device"); ret = -EUCLEAN; } else { dev_replace->srcdev = NULL; @@ -545,10 +545,7 @@ static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info, if (!cache) continue; - spin_lock(&cache->lock); - cache->to_copy = 1; - spin_unlock(&cache->lock); - + set_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); btrfs_put_block_group(cache); } if (iter_ret < 0) @@ -577,7 +574,7 @@ bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, return true; spin_lock(&cache->lock); - if (cache->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) { spin_unlock(&cache->lock); return true; } @@ -610,9 +607,7 @@ bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, } /* Last stripe on this device */ - spin_lock(&cache->lock); - cache->to_copy = 0; - spin_unlock(&cache->lock); + clear_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); return true; } @@ -1129,8 +1124,7 @@ int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info) up_write(&dev_replace->rwsem); /* Scrub for replace must not be running in suspended state */ - ret = btrfs_scrub_cancel(fs_info); - ASSERT(ret != -ENOTCONN); + btrfs_scrub_cancel(fs_info); trans = btrfs_start_transaction(root, 0); if (IS_ERR(trans)) { @@ -1289,11 +1283,6 @@ int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace) return 1; } -void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info) -{ - percpu_counter_inc(&fs_info->dev_replace.bio_counter); -} - void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount) { percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount); diff --git a/fs/btrfs/dev-replace.h b/fs/btrfs/dev-replace.h index 3911049a5f231b24e5aa65319568c4d238968e0e..6084b313056a75a9881c8e494c7c611c0673433f 100644 --- a/fs/btrfs/dev-replace.h +++ b/fs/btrfs/dev-replace.h @@ -7,6 +7,10 @@ #define BTRFS_DEV_REPLACE_H struct btrfs_ioctl_dev_replace_args; +struct btrfs_fs_info; +struct btrfs_trans_handle; +struct btrfs_dev_replace; +struct btrfs_block_group; int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info); int btrfs_run_dev_replace(struct btrfs_trans_handle *trans); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 820b1f1e6b6723dbd6ffcb4da6255331c319a26e..a2da9313c6947c5111c38d2eaa73f14bf5075e9f 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -131,8 +131,7 @@ static int verify_parent_transid(struct extent_io_tree *io_tree, if (atomic) return -EAGAIN; - lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1, - &cached_state); + lock_extent(io_tree, eb->start, eb->start + eb->len - 1, &cached_state); if (extent_buffer_uptodate(eb) && btrfs_header_generation(eb) == parent_transid) { ret = 0; @@ -145,8 +144,8 @@ static int verify_parent_transid(struct extent_io_tree *io_tree, ret = 1; clear_extent_buffer_uptodate(eb); out: - unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1, - &cached_state); + unlock_extent(io_tree, eb->start, eb->start + eb->len - 1, + &cached_state); return ret; } @@ -647,16 +646,14 @@ static void run_one_async_start(struct btrfs_work *work) */ static void run_one_async_done(struct btrfs_work *work) { - struct async_submit_bio *async; - struct inode *inode; - - async = container_of(work, struct async_submit_bio, work); - inode = async->inode; + struct async_submit_bio *async = + container_of(work, struct async_submit_bio, work); + struct inode *inode = async->inode; + struct btrfs_bio *bbio = btrfs_bio(async->bio); /* If an error occurred we just want to clean up the bio and move on */ if (async->status) { - async->bio->bi_status = async->status; - bio_endio(async->bio); + btrfs_bio_end_io(bbio, async->status); return; } @@ -757,6 +754,7 @@ static bool should_async_write(struct btrfs_fs_info *fs_info, void btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_num) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + struct btrfs_bio *bbio = btrfs_bio(bio); blk_status_t ret; bio->bi_opf |= REQ_META; @@ -776,8 +774,7 @@ void btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_ ret = btree_csum_one_bio(bio); if (ret) { - bio->bi_status = ret; - bio_endio(bio); + btrfs_bio_end_io(bbio, ret); return; } @@ -1524,6 +1521,9 @@ static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info, if (objectid == BTRFS_UUID_TREE_OBJECTID) return btrfs_grab_root(fs_info->uuid_root) ? fs_info->uuid_root : ERR_PTR(-ENOENT); + if (objectid == BTRFS_BLOCK_GROUP_TREE_OBJECTID) + return btrfs_grab_root(fs_info->block_group_root) ? + fs_info->block_group_root : ERR_PTR(-ENOENT); if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) { struct btrfs_root *root = btrfs_global_root(fs_info, &key); @@ -1980,14 +1980,7 @@ static void backup_super_roots(struct btrfs_fs_info *info) btrfs_set_backup_chunk_root_level(root_backup, btrfs_header_level(info->chunk_root->node)); - if (btrfs_fs_incompat(info, EXTENT_TREE_V2)) { - btrfs_set_backup_block_group_root(root_backup, - info->block_group_root->node->start); - btrfs_set_backup_block_group_root_gen(root_backup, - btrfs_header_generation(info->block_group_root->node)); - btrfs_set_backup_block_group_root_level(root_backup, - btrfs_header_level(info->block_group_root->node)); - } else { + if (!btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE)) { struct btrfs_root *extent_root = btrfs_extent_root(info, 0); struct btrfs_root *csum_root = btrfs_csum_root(info, 0); @@ -2225,6 +2218,8 @@ static void btrfs_init_balance(struct btrfs_fs_info *fs_info) static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info) { struct inode *inode = fs_info->btree_inode; + unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID, + fs_info->tree_root); inode->i_ino = BTRFS_BTREE_INODE_OBJECTID; set_nlink(inode, 1); @@ -2238,8 +2233,7 @@ static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info) RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node); extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree, - IO_TREE_BTREE_INODE_IO, inode); - BTRFS_I(inode)->io_tree.track_uptodate = false; + IO_TREE_BTREE_INODE_IO, NULL); extent_map_tree_init(&BTRFS_I(inode)->extent_tree); BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root); @@ -2247,7 +2241,7 @@ static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info) BTRFS_I(inode)->location.type = 0; BTRFS_I(inode)->location.offset = 0; set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); - btrfs_insert_inode_hash(inode); + __insert_inode_hash(inode, hash); } static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info) @@ -2266,6 +2260,7 @@ static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info) fs_info->qgroup_seq = 1; fs_info->qgroup_ulist = NULL; fs_info->qgroup_rescan_running = false; + fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL; mutex_init(&fs_info->qgroup_rescan_lock); } @@ -2529,10 +2524,24 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) if (ret) return ret; - location.objectid = BTRFS_DEV_TREE_OBJECTID; location.type = BTRFS_ROOT_ITEM_KEY; location.offset = 0; + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) { + location.objectid = BTRFS_BLOCK_GROUP_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + ret = PTR_ERR(root); + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->block_group_root = root; + } + } + + location.objectid = BTRFS_DEV_TREE_OBJECTID; root = btrfs_read_tree_root(tree_root, &location); if (IS_ERR(root)) { if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { @@ -2600,8 +2609,8 @@ out: * 1, 2 2nd and 3rd backup copy * -1 skip bytenr check */ -static int validate_super(struct btrfs_fs_info *fs_info, - struct btrfs_super_block *sb, int mirror_num) +int btrfs_validate_super(struct btrfs_fs_info *fs_info, + struct btrfs_super_block *sb, int mirror_num) { u64 nodesize = btrfs_super_nodesize(sb); u64 sectorsize = btrfs_super_sectorsize(sb); @@ -2703,6 +2712,18 @@ static int validate_super(struct btrfs_fs_info *fs_info, ret = -EINVAL; } + /* + * Artificial requirement for block-group-tree to force newer features + * (free-space-tree, no-holes) so the test matrix is smaller. + */ + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) && + (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) || + !btrfs_fs_incompat(fs_info, NO_HOLES))) { + btrfs_err(fs_info, + "block-group-tree feature requires fres-space-tree and no-holes"); + ret = -EINVAL; + } + if (memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid, BTRFS_FSID_SIZE) != 0) { btrfs_err(fs_info, @@ -2785,7 +2806,7 @@ static int validate_super(struct btrfs_fs_info *fs_info, */ static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info) { - return validate_super(fs_info, fs_info->super_copy, 0); + return btrfs_validate_super(fs_info, fs_info->super_copy, 0); } /* @@ -2799,7 +2820,7 @@ static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info, { int ret; - ret = validate_super(fs_info, sb, -1); + ret = btrfs_validate_super(fs_info, sb, -1); if (ret < 0) goto out; if (!btrfs_supported_super_csum(btrfs_super_csum_type(sb))) { @@ -2860,17 +2881,7 @@ static int load_important_roots(struct btrfs_fs_info *fs_info) btrfs_warn(fs_info, "couldn't read tree root"); return ret; } - - if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) - return 0; - - bytenr = btrfs_super_block_group_root(sb); - gen = btrfs_super_block_group_root_generation(sb); - level = btrfs_super_block_group_root_level(sb); - ret = load_super_root(fs_info->block_group_root, bytenr, gen, level); - if (ret) - btrfs_warn(fs_info, "couldn't read block group root"); - return ret; + return 0; } static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) @@ -2882,16 +2893,6 @@ static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) int ret = 0; int i; - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - struct btrfs_root *root; - - root = btrfs_alloc_root(fs_info, BTRFS_BLOCK_GROUP_TREE_OBJECTID, - GFP_KERNEL); - if (!root) - return -ENOMEM; - fs_info->block_group_root = root; - } - for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) { if (handle_error) { if (!IS_ERR(tree_root->node)) @@ -2990,6 +2991,19 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) mutex_init(&fs_info->zoned_data_reloc_io_lock); seqlock_init(&fs_info->profiles_lock); + btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered); + btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_start, + BTRFS_LOCKDEP_TRANS_COMMIT_START); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked, + BTRFS_LOCKDEP_TRANS_UNBLOCKED); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed, + BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed, + BTRFS_LOCKDEP_TRANS_COMPLETED); + INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); INIT_LIST_HEAD(&fs_info->space_info); INIT_LIST_HEAD(&fs_info->tree_mod_seq_list); @@ -3068,7 +3082,6 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) init_waitqueue_head(&fs_info->transaction_blocked_wait); init_waitqueue_head(&fs_info->async_submit_wait); init_waitqueue_head(&fs_info->delayed_iputs_wait); - init_waitqueue_head(&fs_info->zone_finish_wait); /* Usable values until the real ones are cached from the superblock */ fs_info->nodesize = 4096; @@ -3280,6 +3293,112 @@ out: return ret; } +/* + * Do various sanity and dependency checks of different features. + * + * This is the place for less strict checks (like for subpage or artificial + * feature dependencies). + * + * For strict checks or possible corruption detection, see + * btrfs_validate_super(). + * + * This should be called after btrfs_parse_options(), as some mount options + * (space cache related) can modify on-disk format like free space tree and + * screw up certain feature dependencies. + */ +int btrfs_check_features(struct btrfs_fs_info *fs_info, struct super_block *sb) +{ + struct btrfs_super_block *disk_super = fs_info->super_copy; + u64 incompat = btrfs_super_incompat_flags(disk_super); + const u64 compat_ro = btrfs_super_compat_ro_flags(disk_super); + const u64 compat_ro_unsupp = (compat_ro & ~BTRFS_FEATURE_COMPAT_RO_SUPP); + + if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) { + btrfs_err(fs_info, + "cannot mount because of unknown incompat features (0x%llx)", + incompat); + return -EINVAL; + } + + /* Runtime limitation for mixed block groups. */ + if ((incompat & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) && + (fs_info->sectorsize != fs_info->nodesize)) { + btrfs_err(fs_info, +"unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups", + fs_info->nodesize, fs_info->sectorsize); + return -EINVAL; + } + + /* Mixed backref is an always-enabled feature. */ + incompat |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; + + /* Set compression related flags just in case. */ + if (fs_info->compress_type == BTRFS_COMPRESS_LZO) + incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; + else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD) + incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; + + /* + * An ancient flag, which should really be marked deprecated. + * Such runtime limitation doesn't really need a incompat flag. + */ + if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) + incompat |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA; + + if (compat_ro_unsupp && !sb_rdonly(sb)) { + btrfs_err(fs_info, + "cannot mount read-write because of unknown compat_ro features (0x%llx)", + compat_ro); + return -EINVAL; + } + + /* + * We have unsupported RO compat features, although RO mounted, we + * should not cause any metadata writes, including log replay. + * Or we could screw up whatever the new feature requires. + */ + if (compat_ro_unsupp && btrfs_super_log_root(disk_super) && + !btrfs_test_opt(fs_info, NOLOGREPLAY)) { + btrfs_err(fs_info, +"cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", + compat_ro); + return -EINVAL; + } + + /* + * Artificial limitations for block group tree, to force + * block-group-tree to rely on no-holes and free-space-tree. + */ + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) && + (!btrfs_fs_incompat(fs_info, NO_HOLES) || + !btrfs_test_opt(fs_info, FREE_SPACE_TREE))) { + btrfs_err(fs_info, +"block-group-tree feature requires no-holes and free-space-tree features"); + return -EINVAL; + } + + /* + * Subpage runtime limitation on v1 cache. + * + * V1 space cache still has some hard codeed PAGE_SIZE usage, while + * we're already defaulting to v2 cache, no need to bother v1 as it's + * going to be deprecated anyway. + */ + if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) { + btrfs_warn(fs_info, + "v1 space cache is not supported for page size %lu with sectorsize %u", + PAGE_SIZE, fs_info->sectorsize); + return -EINVAL; + } + + /* This can be called by remount, we need to protect the super block. */ + spin_lock(&fs_info->super_lock); + btrfs_set_super_incompat_flags(disk_super, incompat); + spin_unlock(&fs_info->super_lock); + + return 0; +} + int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices, char *options) { @@ -3429,72 +3548,12 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device goto fail_alloc; } - features = btrfs_super_incompat_flags(disk_super) & - ~BTRFS_FEATURE_INCOMPAT_SUPP; - if (features) { - btrfs_err(fs_info, - "cannot mount because of unsupported optional features (0x%llx)", - features); - err = -EINVAL; - goto fail_alloc; - } - - features = btrfs_super_incompat_flags(disk_super); - features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; - if (fs_info->compress_type == BTRFS_COMPRESS_LZO) - features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; - else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD) - features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; - - /* - * Flag our filesystem as having big metadata blocks if they are bigger - * than the page size. - */ - if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) - features |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA; - - /* - * mixed block groups end up with duplicate but slightly offset - * extent buffers for the same range. It leads to corruptions - */ - if ((features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) && - (sectorsize != nodesize)) { - btrfs_err(fs_info, -"unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups", - nodesize, sectorsize); - goto fail_alloc; - } - - /* - * Needn't use the lock because there is no other task which will - * update the flag. - */ - btrfs_set_super_incompat_flags(disk_super, features); - - features = btrfs_super_compat_ro_flags(disk_super) & - ~BTRFS_FEATURE_COMPAT_RO_SUPP; - if (!sb_rdonly(sb) && features) { - btrfs_err(fs_info, - "cannot mount read-write because of unsupported optional features (0x%llx)", - features); - err = -EINVAL; - goto fail_alloc; - } - /* - * We have unsupported RO compat features, although RO mounted, we - * should not cause any metadata write, including log replay. - * Or we could screw up whatever the new feature requires. - */ - if (unlikely(features && btrfs_super_log_root(disk_super) && - !btrfs_test_opt(fs_info, NOLOGREPLAY))) { - btrfs_err(fs_info, -"cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", - features); - err = -EINVAL; + ret = btrfs_check_features(fs_info, sb); + if (ret < 0) { + err = ret; goto fail_alloc; } - if (sectorsize < PAGE_SIZE) { struct btrfs_subpage_info *subpage_info; @@ -3834,7 +3893,7 @@ static void btrfs_end_super_write(struct bio *bio) } struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev, - int copy_num) + int copy_num, bool drop_cache) { struct btrfs_super_block *super; struct page *page; @@ -3852,6 +3911,19 @@ struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev, if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev)) return ERR_PTR(-EINVAL); + if (drop_cache) { + /* This should only be called with the primary sb. */ + ASSERT(copy_num == 0); + + /* + * Drop the page of the primary superblock, so later read will + * always read from the device. + */ + invalidate_inode_pages2_range(mapping, + bytenr >> PAGE_SHIFT, + (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT); + } + page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); if (IS_ERR(page)) return ERR_CAST(page); @@ -3883,7 +3955,7 @@ struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev) * later supers, using BTRFS_SUPER_MIRROR_MAX instead */ for (i = 0; i < 1; i++) { - super = btrfs_read_dev_one_super(bdev, i); + super = btrfs_read_dev_one_super(bdev, i, false); if (IS_ERR(super)) continue; @@ -4475,6 +4547,17 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags); + /* + * If we had UNFINISHED_DROPS we could still be processing them, so + * clear that bit and wake up relocation so it can stop. + * We must do this before stopping the block group reclaim task, because + * at btrfs_relocate_block_group() we wait for this bit, and after the + * wait we stop with -EINTR if btrfs_fs_closing() returns non-zero - we + * have just set BTRFS_FS_CLOSING_START, so btrfs_fs_closing() will + * return 1. + */ + btrfs_wake_unfinished_drop(fs_info); + /* * We may have the reclaim task running and relocating a data block group, * in which case it may create delayed iputs. So stop it before we park @@ -4493,12 +4576,6 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) */ kthread_park(fs_info->cleaner_kthread); - /* - * If we had UNFINISHED_DROPS we could still be processing them, so - * clear that bit and wake up relocation so it can stop. - */ - btrfs_wake_unfinished_drop(fs_info); - /* wait for the qgroup rescan worker to stop */ btrfs_qgroup_wait_for_completion(fs_info, false); @@ -4521,6 +4598,31 @@ void __cold close_ctree(struct btrfs_fs_info *fs_info) /* clear out the rbtree of defraggable inodes */ btrfs_cleanup_defrag_inodes(fs_info); + /* + * After we parked the cleaner kthread, ordered extents may have + * completed and created new delayed iputs. If one of the async reclaim + * tasks is running and in the RUN_DELAYED_IPUTS flush state, then we + * can hang forever trying to stop it, because if a delayed iput is + * added after it ran btrfs_run_delayed_iputs() and before it called + * btrfs_wait_on_delayed_iputs(), it will hang forever since there is + * no one else to run iputs. + * + * So wait for all ongoing ordered extents to complete and then run + * delayed iputs. This works because once we reach this point no one + * can either create new ordered extents nor create delayed iputs + * through some other means. + * + * Also note that btrfs_wait_ordered_roots() is not safe here, because + * it waits for BTRFS_ORDERED_COMPLETE to be set on an ordered extent, + * but the delayed iput for the respective inode is made only when doing + * the final btrfs_put_ordered_extent() (which must happen at + * btrfs_finish_ordered_io() when we are unmounting). + */ + btrfs_flush_workqueue(fs_info->endio_write_workers); + /* Ordered extents for free space inodes. */ + btrfs_flush_workqueue(fs_info->endio_freespace_worker); + btrfs_run_delayed_iputs(fs_info); + cancel_work_sync(&fs_info->async_reclaim_work); cancel_work_sync(&fs_info->async_data_reclaim_work); cancel_work_sync(&fs_info->preempt_reclaim_work); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 47ad8e0a2d33f6accdb0fa9ddff54218a8ac92de..c67c15d4d20be0a5f52843821a138bf58ad01f2a 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -46,10 +46,13 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices, char *options); void __cold close_ctree(struct btrfs_fs_info *fs_info); +int btrfs_validate_super(struct btrfs_fs_info *fs_info, + struct btrfs_super_block *sb, int mirror_num); +int btrfs_check_features(struct btrfs_fs_info *fs_info, struct super_block *sb); int write_all_supers(struct btrfs_fs_info *fs_info, int max_mirrors); struct btrfs_super_block *btrfs_read_dev_super(struct block_device *bdev); struct btrfs_super_block *btrfs_read_dev_one_super(struct block_device *bdev, - int copy_num); + int copy_num, bool drop_cache); int btrfs_commit_super(struct btrfs_fs_info *fs_info); struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root, struct btrfs_key *key); @@ -103,7 +106,7 @@ static inline struct btrfs_root *btrfs_grab_root(struct btrfs_root *root) static inline struct btrfs_root *btrfs_block_group_root(struct btrfs_fs_info *fs_info) { - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) return fs_info->block_group_root; return btrfs_extent_root(fs_info, 0); } diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c new file mode 100644 index 0000000000000000000000000000000000000000..618275af19c49da1a75f8b7ca3e08405ec19fe23 --- /dev/null +++ b/fs/btrfs/extent-io-tree.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "ctree.h" +#include "extent-io-tree.h" +#include "btrfs_inode.h" +#include "misc.h" + +static struct kmem_cache *extent_state_cache; + +static inline bool extent_state_in_tree(const struct extent_state *state) +{ + return !RB_EMPTY_NODE(&state->rb_node); +} + +#ifdef CONFIG_BTRFS_DEBUG +static LIST_HEAD(states); +static DEFINE_SPINLOCK(leak_lock); + +static inline void btrfs_leak_debug_add_state(struct extent_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&leak_lock, flags); + list_add(&state->leak_list, &states); + spin_unlock_irqrestore(&leak_lock, flags); +} + +static inline void btrfs_leak_debug_del_state(struct extent_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&leak_lock, flags); + list_del(&state->leak_list); + spin_unlock_irqrestore(&leak_lock, flags); +} + +static inline void btrfs_extent_state_leak_debug_check(void) +{ + struct extent_state *state; + + while (!list_empty(&states)) { + state = list_entry(states.next, struct extent_state, leak_list); + pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n", + state->start, state->end, state->state, + extent_state_in_tree(state), + refcount_read(&state->refs)); + list_del(&state->leak_list); + kmem_cache_free(extent_state_cache, state); + } +} + +#define btrfs_debug_check_extent_io_range(tree, start, end) \ + __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end)) +static inline void __btrfs_debug_check_extent_io_range(const char *caller, + struct extent_io_tree *tree, + u64 start, u64 end) +{ + struct inode *inode = tree->private_data; + u64 isize; + + if (!inode) + return; + + isize = i_size_read(inode); + if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { + btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, + "%s: ino %llu isize %llu odd range [%llu,%llu]", + caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); + } +} +#else +#define btrfs_leak_debug_add_state(state) do {} while (0) +#define btrfs_leak_debug_del_state(state) do {} while (0) +#define btrfs_extent_state_leak_debug_check() do {} while (0) +#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0) +#endif + +/* + * For the file_extent_tree, we want to hold the inode lock when we lookup and + * update the disk_i_size, but lockdep will complain because our io_tree we hold + * the tree lock and get the inode lock when setting delalloc. These two things + * are unrelated, so make a class for the file_extent_tree so we don't get the + * two locking patterns mixed up. + */ +static struct lock_class_key file_extent_tree_class; + +struct tree_entry { + u64 start; + u64 end; + struct rb_node rb_node; +}; + +void extent_io_tree_init(struct btrfs_fs_info *fs_info, + struct extent_io_tree *tree, unsigned int owner, + void *private_data) +{ + tree->fs_info = fs_info; + tree->state = RB_ROOT; + spin_lock_init(&tree->lock); + tree->private_data = private_data; + tree->owner = owner; + if (owner == IO_TREE_INODE_FILE_EXTENT) + lockdep_set_class(&tree->lock, &file_extent_tree_class); +} + +void extent_io_tree_release(struct extent_io_tree *tree) +{ + spin_lock(&tree->lock); + /* + * Do a single barrier for the waitqueue_active check here, the state + * of the waitqueue should not change once extent_io_tree_release is + * called. + */ + smp_mb(); + while (!RB_EMPTY_ROOT(&tree->state)) { + struct rb_node *node; + struct extent_state *state; + + node = rb_first(&tree->state); + state = rb_entry(node, struct extent_state, rb_node); + rb_erase(&state->rb_node, &tree->state); + RB_CLEAR_NODE(&state->rb_node); + /* + * btree io trees aren't supposed to have tasks waiting for + * changes in the flags of extent states ever. + */ + ASSERT(!waitqueue_active(&state->wq)); + free_extent_state(state); + + cond_resched_lock(&tree->lock); + } + spin_unlock(&tree->lock); +} + +static struct extent_state *alloc_extent_state(gfp_t mask) +{ + struct extent_state *state; + + /* + * The given mask might be not appropriate for the slab allocator, + * drop the unsupported bits + */ + mask &= ~(__GFP_DMA32|__GFP_HIGHMEM); + state = kmem_cache_alloc(extent_state_cache, mask); + if (!state) + return state; + state->state = 0; + RB_CLEAR_NODE(&state->rb_node); + btrfs_leak_debug_add_state(state); + refcount_set(&state->refs, 1); + init_waitqueue_head(&state->wq); + trace_alloc_extent_state(state, mask, _RET_IP_); + return state; +} + +static struct extent_state *alloc_extent_state_atomic(struct extent_state *prealloc) +{ + if (!prealloc) + prealloc = alloc_extent_state(GFP_ATOMIC); + + return prealloc; +} + +void free_extent_state(struct extent_state *state) +{ + if (!state) + return; + if (refcount_dec_and_test(&state->refs)) { + WARN_ON(extent_state_in_tree(state)); + btrfs_leak_debug_del_state(state); + trace_free_extent_state(state, _RET_IP_); + kmem_cache_free(extent_state_cache, state); + } +} + +static int add_extent_changeset(struct extent_state *state, u32 bits, + struct extent_changeset *changeset, + int set) +{ + int ret; + + if (!changeset) + return 0; + if (set && (state->state & bits) == bits) + return 0; + if (!set && (state->state & bits) == 0) + return 0; + changeset->bytes_changed += state->end - state->start + 1; + ret = ulist_add(&changeset->range_changed, state->start, state->end, + GFP_ATOMIC); + return ret; +} + +static inline struct extent_state *next_state(struct extent_state *state) +{ + struct rb_node *next = rb_next(&state->rb_node); + + if (next) + return rb_entry(next, struct extent_state, rb_node); + else + return NULL; +} + +static inline struct extent_state *prev_state(struct extent_state *state) +{ + struct rb_node *next = rb_prev(&state->rb_node); + + if (next) + return rb_entry(next, struct extent_state, rb_node); + else + return NULL; +} + +/* + * Search @tree for an entry that contains @offset. Such entry would have + * entry->start <= offset && entry->end >= offset. + * + * @tree: the tree to search + * @offset: offset that should fall within an entry in @tree + * @node_ret: pointer where new node should be anchored (used when inserting an + * entry in the tree) + * @parent_ret: points to entry which would have been the parent of the entry, + * containing @offset + * + * Return a pointer to the entry that contains @offset byte address and don't change + * @node_ret and @parent_ret. + * + * If no such entry exists, return pointer to entry that ends before @offset + * and fill parameters @node_ret and @parent_ret, ie. does not return NULL. + */ +static inline struct extent_state *tree_search_for_insert(struct extent_io_tree *tree, + u64 offset, + struct rb_node ***node_ret, + struct rb_node **parent_ret) +{ + struct rb_root *root = &tree->state; + struct rb_node **node = &root->rb_node; + struct rb_node *prev = NULL; + struct extent_state *entry = NULL; + + while (*node) { + prev = *node; + entry = rb_entry(prev, struct extent_state, rb_node); + + if (offset < entry->start) + node = &(*node)->rb_left; + else if (offset > entry->end) + node = &(*node)->rb_right; + else + return entry; + } + + if (node_ret) + *node_ret = node; + if (parent_ret) + *parent_ret = prev; + + /* Search neighbors until we find the first one past the end */ + while (entry && offset > entry->end) + entry = next_state(entry); + + return entry; +} + +/* + * Search offset in the tree or fill neighbor rbtree node pointers. + * + * @tree: the tree to search + * @offset: offset that should fall within an entry in @tree + * @next_ret: pointer to the first entry whose range ends after @offset + * @prev_ret: pointer to the first entry whose range begins before @offset + * + * Return a pointer to the entry that contains @offset byte address. If no + * such entry exists, then return NULL and fill @prev_ret and @next_ret. + * Otherwise return the found entry and other pointers are left untouched. + */ +static struct extent_state *tree_search_prev_next(struct extent_io_tree *tree, + u64 offset, + struct extent_state **prev_ret, + struct extent_state **next_ret) +{ + struct rb_root *root = &tree->state; + struct rb_node **node = &root->rb_node; + struct extent_state *orig_prev; + struct extent_state *entry = NULL; + + ASSERT(prev_ret); + ASSERT(next_ret); + + while (*node) { + entry = rb_entry(*node, struct extent_state, rb_node); + + if (offset < entry->start) + node = &(*node)->rb_left; + else if (offset > entry->end) + node = &(*node)->rb_right; + else + return entry; + } + + orig_prev = entry; + while (entry && offset > entry->end) + entry = next_state(entry); + *next_ret = entry; + entry = orig_prev; + + while (entry && offset < entry->start) + entry = prev_state(entry); + *prev_ret = entry; + + return NULL; +} + +/* + * Inexact rb-tree search, return the next entry if @offset is not found + */ +static inline struct extent_state *tree_search(struct extent_io_tree *tree, u64 offset) +{ + return tree_search_for_insert(tree, offset, NULL, NULL); +} + +static void extent_io_tree_panic(struct extent_io_tree *tree, int err) +{ + btrfs_panic(tree->fs_info, err, + "locking error: extent tree was modified by another thread while locked"); +} + +/* + * Utility function to look for merge candidates inside a given range. Any + * extents with matching state are merged together into a single extent in the + * tree. Extents with EXTENT_IO in their state field are not merged because + * the end_io handlers need to be able to do operations on them without + * sleeping (or doing allocations/splits). + * + * This should be called with the tree lock held. + */ +static void merge_state(struct extent_io_tree *tree, struct extent_state *state) +{ + struct extent_state *other; + + if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY)) + return; + + other = prev_state(state); + if (other && other->end == state->start - 1 && + other->state == state->state) { + if (tree->private_data) + btrfs_merge_delalloc_extent(tree->private_data, + state, other); + state->start = other->start; + rb_erase(&other->rb_node, &tree->state); + RB_CLEAR_NODE(&other->rb_node); + free_extent_state(other); + } + other = next_state(state); + if (other && other->start == state->end + 1 && + other->state == state->state) { + if (tree->private_data) + btrfs_merge_delalloc_extent(tree->private_data, state, + other); + state->end = other->end; + rb_erase(&other->rb_node, &tree->state); + RB_CLEAR_NODE(&other->rb_node); + free_extent_state(other); + } +} + +static void set_state_bits(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, struct extent_changeset *changeset) +{ + u32 bits_to_set = bits & ~EXTENT_CTLBITS; + int ret; + + if (tree->private_data) + btrfs_set_delalloc_extent(tree->private_data, state, bits); + + ret = add_extent_changeset(state, bits_to_set, changeset, 1); + BUG_ON(ret < 0); + state->state |= bits_to_set; +} + +/* + * Insert an extent_state struct into the tree. 'bits' are set on the + * struct before it is inserted. + * + * This may return -EEXIST if the extent is already there, in which case the + * state struct is freed. + * + * The tree lock is not taken internally. This is a utility function and + * probably isn't what you want to call (see set/clear_extent_bit). + */ +static int insert_state(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, struct extent_changeset *changeset) +{ + struct rb_node **node; + struct rb_node *parent; + const u64 end = state->end; + + set_state_bits(tree, state, bits, changeset); + + node = &tree->state.rb_node; + while (*node) { + struct extent_state *entry; + + parent = *node; + entry = rb_entry(parent, struct extent_state, rb_node); + + if (end < entry->start) { + node = &(*node)->rb_left; + } else if (end > entry->end) { + node = &(*node)->rb_right; + } else { + btrfs_err(tree->fs_info, + "found node %llu %llu on insert of %llu %llu", + entry->start, entry->end, state->start, end); + return -EEXIST; + } + } + + rb_link_node(&state->rb_node, parent, node); + rb_insert_color(&state->rb_node, &tree->state); + + merge_state(tree, state); + return 0; +} + +/* + * Insert state to @tree to the location given by @node and @parent. + */ +static void insert_state_fast(struct extent_io_tree *tree, + struct extent_state *state, struct rb_node **node, + struct rb_node *parent, unsigned bits, + struct extent_changeset *changeset) +{ + set_state_bits(tree, state, bits, changeset); + rb_link_node(&state->rb_node, parent, node); + rb_insert_color(&state->rb_node, &tree->state); + merge_state(tree, state); +} + +/* + * Split a given extent state struct in two, inserting the preallocated + * struct 'prealloc' as the newly created second half. 'split' indicates an + * offset inside 'orig' where it should be split. + * + * Before calling, + * the tree has 'orig' at [orig->start, orig->end]. After calling, there + * are two extent state structs in the tree: + * prealloc: [orig->start, split - 1] + * orig: [ split, orig->end ] + * + * The tree locks are not taken by this function. They need to be held + * by the caller. + */ +static int split_state(struct extent_io_tree *tree, struct extent_state *orig, + struct extent_state *prealloc, u64 split) +{ + struct rb_node *parent = NULL; + struct rb_node **node; + + if (tree->private_data) + btrfs_split_delalloc_extent(tree->private_data, orig, split); + + prealloc->start = orig->start; + prealloc->end = split - 1; + prealloc->state = orig->state; + orig->start = split; + + parent = &orig->rb_node; + node = &parent; + while (*node) { + struct extent_state *entry; + + parent = *node; + entry = rb_entry(parent, struct extent_state, rb_node); + + if (prealloc->end < entry->start) { + node = &(*node)->rb_left; + } else if (prealloc->end > entry->end) { + node = &(*node)->rb_right; + } else { + free_extent_state(prealloc); + return -EEXIST; + } + } + + rb_link_node(&prealloc->rb_node, parent, node); + rb_insert_color(&prealloc->rb_node, &tree->state); + + return 0; +} + +/* + * Utility function to clear some bits in an extent state struct. It will + * optionally wake up anyone waiting on this state (wake == 1). + * + * If no bits are set on the state struct after clearing things, the + * struct is freed and removed from the tree + */ +static struct extent_state *clear_state_bit(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, int wake, + struct extent_changeset *changeset) +{ + struct extent_state *next; + u32 bits_to_clear = bits & ~EXTENT_CTLBITS; + int ret; + + if (tree->private_data) + btrfs_clear_delalloc_extent(tree->private_data, state, bits); + + ret = add_extent_changeset(state, bits_to_clear, changeset, 0); + BUG_ON(ret < 0); + state->state &= ~bits_to_clear; + if (wake) + wake_up(&state->wq); + if (state->state == 0) { + next = next_state(state); + if (extent_state_in_tree(state)) { + rb_erase(&state->rb_node, &tree->state); + RB_CLEAR_NODE(&state->rb_node); + free_extent_state(state); + } else { + WARN_ON(1); + } + } else { + merge_state(tree, state); + next = next_state(state); + } + return next; +} + +/* + * Clear some bits on a range in the tree. This may require splitting or + * inserting elements in the tree, so the gfp mask is used to indicate which + * allocations or sleeping are allowed. + * + * Pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove the given + * range from the tree regardless of state (ie for truncate). + * + * The range [start, end] is inclusive. + * + * This takes the tree lock, and returns 0 on success and < 0 on error. + */ +int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state, + gfp_t mask, struct extent_changeset *changeset) +{ + struct extent_state *state; + struct extent_state *cached; + struct extent_state *prealloc = NULL; + u64 last_end; + int err; + int clear = 0; + int wake; + int delete = (bits & EXTENT_CLEAR_ALL_BITS); + + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits); + + if (delete) + bits |= ~EXTENT_CTLBITS; + + if (bits & EXTENT_DELALLOC) + bits |= EXTENT_NORESERVE; + + wake = (bits & EXTENT_LOCKED) ? 1 : 0; + if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY)) + clear = 1; +again: + if (!prealloc && gfpflags_allow_blocking(mask)) { + /* + * Don't care for allocation failure here because we might end + * up not needing the pre-allocated extent state at all, which + * is the case if we only have in the tree extent states that + * cover our input range and don't cover too any other range. + * If we end up needing a new extent state we allocate it later. + */ + prealloc = alloc_extent_state(mask); + } + + spin_lock(&tree->lock); + if (cached_state) { + cached = *cached_state; + + if (clear) { + *cached_state = NULL; + cached_state = NULL; + } + + if (cached && extent_state_in_tree(cached) && + cached->start <= start && cached->end > start) { + if (clear) + refcount_dec(&cached->refs); + state = cached; + goto hit_next; + } + if (clear) + free_extent_state(cached); + } + + /* This search will find the extents that end after our range starts. */ + state = tree_search(tree, start); + if (!state) + goto out; +hit_next: + if (state->start > end) + goto out; + WARN_ON(state->end < start); + last_end = state->end; + + /* The state doesn't have the wanted bits, go ahead. */ + if (!(state->state & bits)) { + state = next_state(state); + goto next; + } + + /* + * | ---- desired range ---- | + * | state | or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we clear the desired bit + * on it. + */ + + if (state->start < start) { + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + err = split_state(tree, state, prealloc, start); + if (err) + extent_io_tree_panic(tree, err); + + prealloc = NULL; + if (err) + goto out; + if (state->end <= end) { + state = clear_state_bit(tree, state, bits, wake, changeset); + goto next; + } + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | + * We need to split the extent, and clear the bit on the first half. + */ + if (state->start <= end && state->end > end) { + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + err = split_state(tree, state, prealloc, end + 1); + if (err) + extent_io_tree_panic(tree, err); + + if (wake) + wake_up(&state->wq); + + clear_state_bit(tree, prealloc, bits, wake, changeset); + + prealloc = NULL; + goto out; + } + + state = clear_state_bit(tree, state, bits, wake, changeset); +next: + if (last_end == (u64)-1) + goto out; + start = last_end + 1; + if (start <= end && state && !need_resched()) + goto hit_next; + +search_again: + if (start > end) + goto out; + spin_unlock(&tree->lock); + if (gfpflags_allow_blocking(mask)) + cond_resched(); + goto again; + +out: + spin_unlock(&tree->lock); + if (prealloc) + free_extent_state(prealloc); + + return 0; + +} + +static void wait_on_state(struct extent_io_tree *tree, + struct extent_state *state) + __releases(tree->lock) + __acquires(tree->lock) +{ + DEFINE_WAIT(wait); + prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); + spin_unlock(&tree->lock); + schedule(); + spin_lock(&tree->lock); + finish_wait(&state->wq, &wait); +} + +/* + * Wait for one or more bits to clear on a range in the state tree. + * The range [start, end] is inclusive. + * The tree lock is taken by this function + */ +void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits) +{ + struct extent_state *state; + + btrfs_debug_check_extent_io_range(tree, start, end); + + spin_lock(&tree->lock); +again: + while (1) { + /* + * This search will find all the extents that end after our + * range starts. + */ + state = tree_search(tree, start); +process_node: + if (!state) + break; + if (state->start > end) + goto out; + + if (state->state & bits) { + start = state->start; + refcount_inc(&state->refs); + wait_on_state(tree, state); + free_extent_state(state); + goto again; + } + start = state->end + 1; + + if (start > end) + break; + + if (!cond_resched_lock(&tree->lock)) { + state = next_state(state); + goto process_node; + } + } +out: + spin_unlock(&tree->lock); +} + +static void cache_state_if_flags(struct extent_state *state, + struct extent_state **cached_ptr, + unsigned flags) +{ + if (cached_ptr && !(*cached_ptr)) { + if (!flags || (state->state & flags)) { + *cached_ptr = state; + refcount_inc(&state->refs); + } + } +} + +static void cache_state(struct extent_state *state, + struct extent_state **cached_ptr) +{ + return cache_state_if_flags(state, cached_ptr, + EXTENT_LOCKED | EXTENT_BOUNDARY); +} + +/* + * Find the first state struct with 'bits' set after 'start', and return it. + * tree->lock must be held. NULL will returned if nothing was found after + * 'start'. + */ +static struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree, + u64 start, u32 bits) +{ + struct extent_state *state; + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search(tree, start); + while (state) { + if (state->end >= start && (state->state & bits)) + return state; + state = next_state(state); + } + return NULL; +} + +/* + * Find the first offset in the io tree with one or more @bits set. + * + * Note: If there are multiple bits set in @bits, any of them will match. + * + * Return 0 if we find something, and update @start_ret and @end_ret. + * Return 1 if we found nothing. + */ +int find_first_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits, + struct extent_state **cached_state) +{ + struct extent_state *state; + int ret = 1; + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->end == start - 1 && extent_state_in_tree(state)) { + while ((state = next_state(state)) != NULL) { + if (state->state & bits) + goto got_it; + } + free_extent_state(*cached_state); + *cached_state = NULL; + goto out; + } + free_extent_state(*cached_state); + *cached_state = NULL; + } + + state = find_first_extent_bit_state(tree, start, bits); +got_it: + if (state) { + cache_state_if_flags(state, cached_state, 0); + *start_ret = state->start; + *end_ret = state->end; + ret = 0; + } +out: + spin_unlock(&tree->lock); + return ret; +} + +/* + * Find a contiguous area of bits + * + * @tree: io tree to check + * @start: offset to start the search from + * @start_ret: the first offset we found with the bits set + * @end_ret: the final contiguous range of the bits that were set + * @bits: bits to look for + * + * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges + * to set bits appropriately, and then merge them again. During this time it + * will drop the tree->lock, so use this helper if you want to find the actual + * contiguous area for given bits. We will search to the first bit we find, and + * then walk down the tree until we find a non-contiguous area. The area + * returned will be the full contiguous area with the bits set. + */ +int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits) +{ + struct extent_state *state; + int ret = 1; + + spin_lock(&tree->lock); + state = find_first_extent_bit_state(tree, start, bits); + if (state) { + *start_ret = state->start; + *end_ret = state->end; + while ((state = next_state(state)) != NULL) { + if (state->start > (*end_ret + 1)) + break; + *end_ret = state->end; + } + ret = 0; + } + spin_unlock(&tree->lock); + return ret; +} + +/* + * Find a contiguous range of bytes in the file marked as delalloc, not more + * than 'max_bytes'. start and end are used to return the range, + * + * True is returned if we find something, false if nothing was in the tree. + */ +bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, + u64 *end, u64 max_bytes, + struct extent_state **cached_state) +{ + struct extent_state *state; + u64 cur_start = *start; + bool found = false; + u64 total_bytes = 0; + + spin_lock(&tree->lock); + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search(tree, cur_start); + if (!state) { + *end = (u64)-1; + goto out; + } + + while (state) { + if (found && (state->start != cur_start || + (state->state & EXTENT_BOUNDARY))) { + goto out; + } + if (!(state->state & EXTENT_DELALLOC)) { + if (!found) + *end = state->end; + goto out; + } + if (!found) { + *start = state->start; + *cached_state = state; + refcount_inc(&state->refs); + } + found = true; + *end = state->end; + cur_start = state->end + 1; + total_bytes += state->end - state->start + 1; + if (total_bytes >= max_bytes) + break; + state = next_state(state); + } +out: + spin_unlock(&tree->lock); + return found; +} + +/* + * Set some bits on a range in the tree. This may require allocations or + * sleeping, so the gfp mask is used to indicate what is allowed. + * + * If any of the exclusive bits are set, this will fail with -EEXIST if some + * part of the range already has the desired bits set. The start of the + * existing range is returned in failed_start in this case. + * + * [start, end] is inclusive This takes the tree lock. + */ +static int __set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, u64 *failed_start, + struct extent_state **cached_state, + struct extent_changeset *changeset, gfp_t mask) +{ + struct extent_state *state; + struct extent_state *prealloc = NULL; + struct rb_node **p; + struct rb_node *parent; + int err = 0; + u64 last_start; + u64 last_end; + u32 exclusive_bits = (bits & EXTENT_LOCKED); + + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits); + + if (exclusive_bits) + ASSERT(failed_start); + else + ASSERT(failed_start == NULL); +again: + if (!prealloc && gfpflags_allow_blocking(mask)) { + /* + * Don't care for allocation failure here because we might end + * up not needing the pre-allocated extent state at all, which + * is the case if we only have in the tree extent states that + * cover our input range and don't cover too any other range. + * If we end up needing a new extent state we allocate it later. + */ + prealloc = alloc_extent_state(mask); + } + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->start <= start && state->end > start && + extent_state_in_tree(state)) + goto hit_next; + } + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search_for_insert(tree, start, &p, &parent); + if (!state) { + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + prealloc->start = start; + prealloc->end = end; + insert_state_fast(tree, prealloc, p, parent, bits, changeset); + cache_state(prealloc, cached_state); + prealloc = NULL; + goto out; + } +hit_next: + last_start = state->start; + last_end = state->end; + + /* + * | ---- desired range ---- | + * | state | + * + * Just lock what we found and keep going + */ + if (state->start == start && state->end <= end) { + if (state->state & exclusive_bits) { + *failed_start = state->start; + err = -EEXIST; + goto out; + } + + set_state_bits(tree, state, bits, changeset); + cache_state(state, cached_state); + merge_state(tree, state); + if (last_end == (u64)-1) + goto out; + start = last_end + 1; + state = next_state(state); + if (start < end && state && state->start == start && + !need_resched()) + goto hit_next; + goto search_again; + } + + /* + * | ---- desired range ---- | + * | state | + * or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we set the desired bit + * on it. + */ + if (state->start < start) { + if (state->state & exclusive_bits) { + *failed_start = start; + err = -EEXIST; + goto out; + } + + /* + * If this extent already has all the bits we want set, then + * skip it, not necessary to split it or do anything with it. + */ + if ((state->state & bits) == bits) { + start = state->end + 1; + cache_state(state, cached_state); + goto search_again; + } + + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + err = split_state(tree, state, prealloc, start); + if (err) + extent_io_tree_panic(tree, err); + + prealloc = NULL; + if (err) + goto out; + if (state->end <= end) { + set_state_bits(tree, state, bits, changeset); + cache_state(state, cached_state); + merge_state(tree, state); + if (last_end == (u64)-1) + goto out; + start = last_end + 1; + state = next_state(state); + if (start < end && state && state->start == start && + !need_resched()) + goto hit_next; + } + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | or | state | + * + * There's a hole, we need to insert something in it and ignore the + * extent we found. + */ + if (state->start > start) { + u64 this_end; + if (end < last_start) + this_end = end; + else + this_end = last_start - 1; + + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + + /* + * Avoid to free 'prealloc' if it can be merged with the later + * extent. + */ + prealloc->start = start; + prealloc->end = this_end; + err = insert_state(tree, prealloc, bits, changeset); + if (err) + extent_io_tree_panic(tree, err); + + cache_state(prealloc, cached_state); + prealloc = NULL; + start = this_end + 1; + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | + * + * We need to split the extent, and set the bit on the first half + */ + if (state->start <= end && state->end > end) { + if (state->state & exclusive_bits) { + *failed_start = start; + err = -EEXIST; + goto out; + } + + prealloc = alloc_extent_state_atomic(prealloc); + BUG_ON(!prealloc); + err = split_state(tree, state, prealloc, end + 1); + if (err) + extent_io_tree_panic(tree, err); + + set_state_bits(tree, prealloc, bits, changeset); + cache_state(prealloc, cached_state); + merge_state(tree, prealloc); + prealloc = NULL; + goto out; + } + +search_again: + if (start > end) + goto out; + spin_unlock(&tree->lock); + if (gfpflags_allow_blocking(mask)) + cond_resched(); + goto again; + +out: + spin_unlock(&tree->lock); + if (prealloc) + free_extent_state(prealloc); + + return err; + +} + +int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state, gfp_t mask) +{ + return __set_extent_bit(tree, start, end, bits, NULL, cached_state, + NULL, mask); +} + +/* + * Convert all bits in a given range from one bit to another + * + * @tree: the io tree to search + * @start: the start offset in bytes + * @end: the end offset in bytes (inclusive) + * @bits: the bits to set in this range + * @clear_bits: the bits to clear in this range + * @cached_state: state that we're going to cache + * + * This will go through and set bits for the given range. If any states exist + * already in this range they are set with the given bit and cleared of the + * clear_bits. This is only meant to be used by things that are mergeable, ie. + * converting from say DELALLOC to DIRTY. This is not meant to be used with + * boundary bits like LOCK. + * + * All allocations are done with GFP_NOFS. + */ +int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, u32 clear_bits, + struct extent_state **cached_state) +{ + struct extent_state *state; + struct extent_state *prealloc = NULL; + struct rb_node **p; + struct rb_node *parent; + int err = 0; + u64 last_start; + u64 last_end; + bool first_iteration = true; + + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits, + clear_bits); + +again: + if (!prealloc) { + /* + * Best effort, don't worry if extent state allocation fails + * here for the first iteration. We might have a cached state + * that matches exactly the target range, in which case no + * extent state allocations are needed. We'll only know this + * after locking the tree. + */ + prealloc = alloc_extent_state(GFP_NOFS); + if (!prealloc && !first_iteration) + return -ENOMEM; + } + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->start <= start && state->end > start && + extent_state_in_tree(state)) + goto hit_next; + } + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search_for_insert(tree, start, &p, &parent); + if (!state) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + err = -ENOMEM; + goto out; + } + prealloc->start = start; + prealloc->end = end; + insert_state_fast(tree, prealloc, p, parent, bits, NULL); + cache_state(prealloc, cached_state); + prealloc = NULL; + goto out; + } +hit_next: + last_start = state->start; + last_end = state->end; + + /* + * | ---- desired range ---- | + * | state | + * + * Just lock what we found and keep going. + */ + if (state->start == start && state->end <= end) { + set_state_bits(tree, state, bits, NULL); + cache_state(state, cached_state); + state = clear_state_bit(tree, state, clear_bits, 0, NULL); + if (last_end == (u64)-1) + goto out; + start = last_end + 1; + if (start < end && state && state->start == start && + !need_resched()) + goto hit_next; + goto search_again; + } + + /* + * | ---- desired range ---- | + * | state | + * or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we set the desired bit + * on it. + */ + if (state->start < start) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + err = -ENOMEM; + goto out; + } + err = split_state(tree, state, prealloc, start); + if (err) + extent_io_tree_panic(tree, err); + prealloc = NULL; + if (err) + goto out; + if (state->end <= end) { + set_state_bits(tree, state, bits, NULL); + cache_state(state, cached_state); + state = clear_state_bit(tree, state, clear_bits, 0, NULL); + if (last_end == (u64)-1) + goto out; + start = last_end + 1; + if (start < end && state && state->start == start && + !need_resched()) + goto hit_next; + } + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | or | state | + * + * There's a hole, we need to insert something in it and ignore the + * extent we found. + */ + if (state->start > start) { + u64 this_end; + if (end < last_start) + this_end = end; + else + this_end = last_start - 1; + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + err = -ENOMEM; + goto out; + } + + /* + * Avoid to free 'prealloc' if it can be merged with the later + * extent. + */ + prealloc->start = start; + prealloc->end = this_end; + err = insert_state(tree, prealloc, bits, NULL); + if (err) + extent_io_tree_panic(tree, err); + cache_state(prealloc, cached_state); + prealloc = NULL; + start = this_end + 1; + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | + * + * We need to split the extent, and set the bit on the first half. + */ + if (state->start <= end && state->end > end) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + err = -ENOMEM; + goto out; + } + + err = split_state(tree, state, prealloc, end + 1); + if (err) + extent_io_tree_panic(tree, err); + + set_state_bits(tree, prealloc, bits, NULL); + cache_state(prealloc, cached_state); + clear_state_bit(tree, prealloc, clear_bits, 0, NULL); + prealloc = NULL; + goto out; + } + +search_again: + if (start > end) + goto out; + spin_unlock(&tree->lock); + cond_resched(); + first_iteration = false; + goto again; + +out: + spin_unlock(&tree->lock); + if (prealloc) + free_extent_state(prealloc); + + return err; +} + +/* + * Find the first range that has @bits not set. This range could start before + * @start. + * + * @tree: the tree to search + * @start: offset at/after which the found extent should start + * @start_ret: records the beginning of the range + * @end_ret: records the end of the range (inclusive) + * @bits: the set of bits which must be unset + * + * Since unallocated range is also considered one which doesn't have the bits + * set it's possible that @end_ret contains -1, this happens in case the range + * spans (last_range_end, end of device]. In this case it's up to the caller to + * trim @end_ret to the appropriate size. + */ +void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits) +{ + struct extent_state *state; + struct extent_state *prev = NULL, *next; + + spin_lock(&tree->lock); + + /* Find first extent with bits cleared */ + while (1) { + state = tree_search_prev_next(tree, start, &prev, &next); + if (!state && !next && !prev) { + /* + * Tree is completely empty, send full range and let + * caller deal with it + */ + *start_ret = 0; + *end_ret = -1; + goto out; + } else if (!state && !next) { + /* + * We are past the last allocated chunk, set start at + * the end of the last extent. + */ + *start_ret = prev->end + 1; + *end_ret = -1; + goto out; + } else if (!state) { + state = next; + } + + /* + * At this point 'state' either contains 'start' or start is + * before 'state' + */ + if (in_range(start, state->start, state->end - state->start + 1)) { + if (state->state & bits) { + /* + * |--range with bits sets--| + * | + * start + */ + start = state->end + 1; + } else { + /* + * 'start' falls within a range that doesn't + * have the bits set, so take its start as the + * beginning of the desired range + * + * |--range with bits cleared----| + * | + * start + */ + *start_ret = state->start; + break; + } + } else { + /* + * |---prev range---|---hole/unset---|---node range---| + * | + * start + * + * or + * + * |---hole/unset--||--first node--| + * 0 | + * start + */ + if (prev) + *start_ret = prev->end + 1; + else + *start_ret = 0; + break; + } + } + + /* + * Find the longest stretch from start until an entry which has the + * bits set + */ + while (state) { + if (state->end >= start && !(state->state & bits)) { + *end_ret = state->end; + } else { + *end_ret = state->start - 1; + break; + } + state = next_state(state); + } +out: + spin_unlock(&tree->lock); +} + +/* + * Count the number of bytes in the tree that have a given bit(s) set. This + * can be fairly slow, except for EXTENT_DIRTY which is cached. The total + * number found is returned. + */ +u64 count_range_bits(struct extent_io_tree *tree, + u64 *start, u64 search_end, u64 max_bytes, + u32 bits, int contig) +{ + struct extent_state *state; + u64 cur_start = *start; + u64 total_bytes = 0; + u64 last = 0; + int found = 0; + + if (WARN_ON(search_end <= cur_start)) + return 0; + + spin_lock(&tree->lock); + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search(tree, cur_start); + while (state) { + if (state->start > search_end) + break; + if (contig && found && state->start > last + 1) + break; + if (state->end >= cur_start && (state->state & bits) == bits) { + total_bytes += min(search_end, state->end) + 1 - + max(cur_start, state->start); + if (total_bytes >= max_bytes) + break; + if (!found) { + *start = max(cur_start, state->start); + found = 1; + } + last = state->end; + } else if (contig && found) { + break; + } + state = next_state(state); + } + spin_unlock(&tree->lock); + return total_bytes; +} + +/* + * Searche a range in the state tree for a given mask. If 'filled' == 1, this + * returns 1 only if every extent in the tree has the bits set. Otherwise, 1 + * is returned if any bit in the range is found set. + */ +int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, int filled, struct extent_state *cached) +{ + struct extent_state *state = NULL; + int bitset = 0; + + spin_lock(&tree->lock); + if (cached && extent_state_in_tree(cached) && cached->start <= start && + cached->end > start) + state = cached; + else + state = tree_search(tree, start); + while (state && start <= end) { + if (filled && state->start > start) { + bitset = 0; + break; + } + + if (state->start > end) + break; + + if (state->state & bits) { + bitset = 1; + if (!filled) + break; + } else if (filled) { + bitset = 0; + break; + } + + if (state->end == (u64)-1) + break; + + start = state->end + 1; + if (start > end) + break; + state = next_state(state); + } + + /* We ran out of states and were still inside of our range. */ + if (filled && !state) + bitset = 0; + spin_unlock(&tree->lock); + return bitset; +} + +/* Wrappers around set/clear extent bit */ +int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset) +{ + /* + * We don't support EXTENT_LOCKED yet, as current changeset will + * record any bits changed, so for EXTENT_LOCKED case, it will + * either fail with -EEXIST or changeset will record the whole + * range. + */ + ASSERT(!(bits & EXTENT_LOCKED)); + + return __set_extent_bit(tree, start, end, bits, NULL, NULL, changeset, + GFP_NOFS); +} + +int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset) +{ + /* + * Don't support EXTENT_LOCKED case, same reason as + * set_record_extent_bits(). + */ + ASSERT(!(bits & EXTENT_LOCKED)); + + return __clear_extent_bit(tree, start, end, bits, NULL, GFP_NOFS, + changeset); +} + +int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end) +{ + int err; + u64 failed_start; + + err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, &failed_start, + NULL, NULL, GFP_NOFS); + if (err == -EEXIST) { + if (failed_start > start) + clear_extent_bit(tree, start, failed_start - 1, + EXTENT_LOCKED, NULL); + return 0; + } + return 1; +} + +/* + * Either insert or lock state struct between start and end use mask to tell + * us if waiting is desired. + */ +int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, + struct extent_state **cached_state) +{ + int err; + u64 failed_start; + + while (1) { + err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, + &failed_start, cached_state, NULL, + GFP_NOFS); + if (err == -EEXIST) { + wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); + start = failed_start; + } else + break; + WARN_ON(start > end); + } + return err; +} + +void __cold extent_state_free_cachep(void) +{ + btrfs_extent_state_leak_debug_check(); + kmem_cache_destroy(extent_state_cache); +} + +int __init extent_state_init_cachep(void) +{ + extent_state_cache = kmem_cache_create("btrfs_extent_state", + sizeof(struct extent_state), 0, + SLAB_MEM_SPREAD, NULL); + if (!extent_state_cache) + return -ENOMEM; + + return 0; +} diff --git a/fs/btrfs/extent-io-tree.h b/fs/btrfs/extent-io-tree.h index c3eb52dbe61cc16a702c8eb0a983454fe0e0eab9..a855f40dd61d49804cd2f13f174c66a32e02a2f8 100644 --- a/fs/btrfs/extent-io-tree.h +++ b/fs/btrfs/extent-io-tree.h @@ -17,7 +17,6 @@ struct io_failure_record; #define EXTENT_NODATASUM (1U << 7) #define EXTENT_CLEAR_META_RESV (1U << 8) #define EXTENT_NEED_WAIT (1U << 9) -#define EXTENT_DAMAGED (1U << 10) #define EXTENT_NORESERVE (1U << 11) #define EXTENT_QGROUP_RESERVED (1U << 12) #define EXTENT_CLEAR_DATA_RESV (1U << 13) @@ -35,10 +34,18 @@ struct io_failure_record; * delalloc bytes decremented, in an atomic way to prevent races with stat(2). */ #define EXTENT_ADD_INODE_BYTES (1U << 15) + +/* + * Set during truncate when we're clearing an entire range and we just want the + * extent states to go away. + */ +#define EXTENT_CLEAR_ALL_BITS (1U << 16) + #define EXTENT_DO_ACCOUNTING (EXTENT_CLEAR_META_RESV | \ EXTENT_CLEAR_DATA_RESV) #define EXTENT_CTLBITS (EXTENT_DO_ACCOUNTING | \ - EXTENT_ADD_INODE_BYTES) + EXTENT_ADD_INODE_BYTES | \ + EXTENT_CLEAR_ALL_BITS) /* * Redefined bits above which are used only in the device allocation tree, @@ -56,7 +63,6 @@ enum { IO_TREE_FS_EXCLUDED_EXTENTS, IO_TREE_BTREE_INODE_IO, IO_TREE_INODE_IO, - IO_TREE_INODE_IO_FAILURE, IO_TREE_RELOC_BLOCKS, IO_TREE_TRANS_DIRTY_PAGES, IO_TREE_ROOT_DIRTY_LOG_PAGES, @@ -70,8 +76,6 @@ struct extent_io_tree { struct rb_root state; struct btrfs_fs_info *fs_info; void *private_data; - u64 dirty_bytes; - bool track_uptodate; /* Who owns this io tree, should be one of IO_TREE_* */ u8 owner; @@ -89,33 +93,23 @@ struct extent_state { refcount_t refs; u32 state; - struct io_failure_record *failrec; - #ifdef CONFIG_BTRFS_DEBUG struct list_head leak_list; #endif }; -int __init extent_state_cache_init(void); -void __cold extent_state_cache_exit(void); - void extent_io_tree_init(struct btrfs_fs_info *fs_info, struct extent_io_tree *tree, unsigned int owner, void *private_data); void extent_io_tree_release(struct extent_io_tree *tree); -int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, - struct extent_state **cached); - -static inline int lock_extent(struct extent_io_tree *tree, u64 start, u64 end) -{ - return lock_extent_bits(tree, start, end, NULL); -} +int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, + struct extent_state **cached); int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end); -int __init extent_io_init(void); -void __cold extent_io_exit(void); +int __init extent_state_init_cachep(void); +void __cold extent_state_free_cachep(void); u64 count_range_bits(struct extent_io_tree *tree, u64 *start, u64 search_end, @@ -126,72 +120,66 @@ int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, int filled, struct extent_state *cached_state); int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, struct extent_changeset *changeset); -int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, int wake, int delete, - struct extent_state **cached); int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, int wake, int delete, - struct extent_state **cached, gfp_t mask, - struct extent_changeset *changeset); + u32 bits, struct extent_state **cached, gfp_t mask, + struct extent_changeset *changeset); -static inline int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end) +static inline int clear_extent_bit(struct extent_io_tree *tree, u64 start, + u64 end, u32 bits, + struct extent_state **cached) { - return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL); + return __clear_extent_bit(tree, start, end, bits, cached, + GFP_NOFS, NULL); } -static inline int unlock_extent_cached(struct extent_io_tree *tree, u64 start, - u64 end, struct extent_state **cached) +static inline int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, + struct extent_state **cached) { - return __clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached, - GFP_NOFS, NULL); + return __clear_extent_bit(tree, start, end, EXTENT_LOCKED, cached, + GFP_NOFS, NULL); } -static inline int unlock_extent_cached_atomic(struct extent_io_tree *tree, - u64 start, u64 end, struct extent_state **cached) +static inline int unlock_extent_atomic(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) { - return __clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached, - GFP_ATOMIC, NULL); + return __clear_extent_bit(tree, start, end, EXTENT_LOCKED, cached, + GFP_ATOMIC, NULL); } static inline int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits) { - int wake = 0; - - if (bits & EXTENT_LOCKED) - wake = 1; - - return clear_extent_bit(tree, start, end, bits, wake, 0, NULL); + return clear_extent_bit(tree, start, end, bits, NULL); } int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, struct extent_changeset *changeset); int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, unsigned exclusive_bits, u64 *failed_start, - struct extent_state **cached_state, gfp_t mask, - struct extent_changeset *changeset); -int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits); + u32 bits, struct extent_state **cached_state, gfp_t mask); + +static inline int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, + u64 end, u32 bits) +{ + return set_extent_bit(tree, start, end, bits, NULL, GFP_NOWAIT); +} static inline int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits) { - return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS, - NULL); + return set_extent_bit(tree, start, end, bits, NULL, GFP_NOFS); } static inline int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, struct extent_state **cached_state) { - return __clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, - cached_state, GFP_NOFS, NULL); + return __clear_extent_bit(tree, start, end, EXTENT_UPTODATE, + cached_state, GFP_NOFS, NULL); } static inline int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask) { - return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL, NULL, - mask, NULL); + return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL, mask); } static inline int clear_extent_dirty(struct extent_io_tree *tree, u64 start, @@ -199,7 +187,7 @@ static inline int clear_extent_dirty(struct extent_io_tree *tree, u64 start, { return clear_extent_bit(tree, start, end, EXTENT_DIRTY | EXTENT_DELALLOC | - EXTENT_DO_ACCOUNTING, 0, 0, cached); + EXTENT_DO_ACCOUNTING, cached); } int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, @@ -211,30 +199,29 @@ static inline int set_extent_delalloc(struct extent_io_tree *tree, u64 start, struct extent_state **cached_state) { return set_extent_bit(tree, start, end, - EXTENT_DELALLOC | EXTENT_UPTODATE | extra_bits, - 0, NULL, cached_state, GFP_NOFS, NULL); + EXTENT_DELALLOC | extra_bits, + cached_state, GFP_NOFS); } static inline int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end, struct extent_state **cached_state) { return set_extent_bit(tree, start, end, - EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG, - 0, NULL, cached_state, GFP_NOFS, NULL); + EXTENT_DELALLOC | EXTENT_DEFRAG, + cached_state, GFP_NOFS); } static inline int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end) { - return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL, NULL, - GFP_NOFS, NULL); + return set_extent_bit(tree, start, end, EXTENT_NEW, NULL, GFP_NOFS); } static inline int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, struct extent_state **cached_state, gfp_t mask) { - return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL, - cached_state, mask, NULL); + return set_extent_bit(tree, start, end, EXTENT_UPTODATE, + cached_state, mask); } int find_first_extent_bit(struct extent_io_tree *tree, u64 start, @@ -244,24 +231,9 @@ void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 *start_ret, u64 *end_ret, u32 bits); int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, u64 *start_ret, u64 *end_ret, u32 bits); -int extent_invalidate_folio(struct extent_io_tree *tree, - struct folio *folio, size_t offset); bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, u64 *end, u64 max_bytes, struct extent_state **cached_state); - -/* This should be reworked in the future and put elsewhere. */ -struct io_failure_record *get_state_failrec(struct extent_io_tree *tree, u64 start); -int set_state_failrec(struct extent_io_tree *tree, u64 start, - struct io_failure_record *failrec); -void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, - u64 end); -int free_io_failure(struct extent_io_tree *failure_tree, - struct extent_io_tree *io_tree, - struct io_failure_record *rec); -int clean_io_failure(struct btrfs_fs_info *fs_info, - struct extent_io_tree *failure_tree, - struct extent_io_tree *io_tree, u64 start, - struct page *page, u64 ino, unsigned int pg_offset); +void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits); #endif /* BTRFS_EXTENT_IO_TREE_H */ diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index ab944d1f94ef0e1974584d7f5b3f9347673dd274..cd2d36580f1ac169fa0724ba2db732f03bfcfe5d 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2220,6 +2220,12 @@ static noinline int check_delayed_ref(struct btrfs_root *root, } if (!mutex_trylock(&head->mutex)) { + if (path->nowait) { + spin_unlock(&delayed_refs->lock); + btrfs_put_transaction(cur_trans); + return -EAGAIN; + } + refcount_inc(&head->refs); spin_unlock(&delayed_refs->lock); @@ -2551,17 +2557,10 @@ int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans, return -EINVAL; /* - * pull in the free space cache (if any) so that our pin - * removes the free space from the cache. We have load_only set - * to one because the slow code to read in the free extents does check - * the pinned extents. - */ - btrfs_cache_block_group(cache, 1); - /* - * Make sure we wait until the cache is completely built in case it is - * missing or is invalid and therefore needs to be rebuilt. + * Fully cache the free space first so that our pin removes the free space + * from the cache. */ - ret = btrfs_wait_block_group_cache_done(cache); + ret = btrfs_cache_block_group(cache, true); if (ret) goto out; @@ -2584,12 +2583,7 @@ static int __exclude_logged_extent(struct btrfs_fs_info *fs_info, if (!block_group) return -EINVAL; - btrfs_cache_block_group(block_group, 1); - /* - * Make sure we wait until the cache is completely built in case it is - * missing or is invalid and therefore needs to be rebuilt. - */ - ret = btrfs_wait_block_group_cache_done(block_group); + ret = btrfs_cache_block_group(block_group, true); if (ret) goto out; @@ -2698,13 +2692,8 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, len = cache->start + cache->length - start; len = min(len, end + 1 - start); - down_read(&fs_info->commit_root_sem); - if (start < cache->last_byte_to_unpin && return_free_space) { - u64 add_len = min(len, cache->last_byte_to_unpin - start); - - btrfs_add_free_space(cache, start, add_len); - } - up_read(&fs_info->commit_root_sem); + if (return_free_space) + btrfs_add_free_space(cache, start, len); start += len; total_unpinned += len; @@ -3816,7 +3805,8 @@ static int do_allocation_zoned(struct btrfs_block_group *block_group, block_group->start == fs_info->data_reloc_bg || fs_info->data_reloc_bg == 0); - if (block_group->ro || block_group->zoned_data_reloc_ongoing) { + if (block_group->ro || + test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { ret = 1; goto out; } @@ -3893,7 +3883,7 @@ out: * regular extents) at the same time to the same zone, which * easily break the write pointer. */ - block_group->zoned_data_reloc_ongoing = 1; + set_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags); fs_info->data_reloc_bg = 0; } spin_unlock(&fs_info->relocation_bg_lock); @@ -4399,7 +4389,7 @@ have_block_group: ffe_ctl->cached = btrfs_block_group_done(block_group); if (unlikely(!ffe_ctl->cached)) { ffe_ctl->have_caching_bg = true; - ret = btrfs_cache_block_group(block_group, 0); + ret = btrfs_cache_block_group(block_group, false); /* * If we get ENOMEM here or something else we want to @@ -4900,6 +4890,9 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, !test_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &root->state)) lockdep_owner = BTRFS_FS_TREE_OBJECTID; + /* btrfs_clean_tree_block() accesses generation field. */ + btrfs_set_header_generation(buf, trans->transid); + /* * This needs to stay, because we could allocate a freed block from an * old tree into a new tree, so we need to make sure this new block is @@ -5651,6 +5644,8 @@ static noinline int walk_up_tree(struct btrfs_trans_handle *trans, */ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) { + const bool is_reloc_root = (root->root_key.objectid == + BTRFS_TREE_RELOC_OBJECTID); struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_path *path; struct btrfs_trans_handle *trans; @@ -5810,6 +5805,9 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) goto out_end_trans; } + if (!is_reloc_root) + btrfs_set_last_root_drop_gen(fs_info, trans->transid); + btrfs_end_transaction_throttle(trans); if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) { btrfs_debug(fs_info, @@ -5844,7 +5842,7 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) goto out_end_trans; } - if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) { + if (!is_reloc_root) { ret = btrfs_find_root(tree_root, &root->root_key, path, NULL, NULL); if (ret < 0) { @@ -5876,6 +5874,9 @@ int btrfs_drop_snapshot(struct btrfs_root *root, int update_ref, int for_reloc) btrfs_put_root(root); root_dropped = true; out_end_trans: + if (!is_reloc_root) + btrfs_set_last_root_drop_gen(fs_info, trans->transid); + btrfs_end_transaction_throttle(trans); out_free: kfree(wc); @@ -6169,13 +6170,7 @@ int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) if (end - start >= range->minlen) { if (!btrfs_block_group_done(cache)) { - ret = btrfs_cache_block_group(cache, 0); - if (ret) { - bg_failed++; - bg_ret = ret; - continue; - } - ret = btrfs_wait_block_group_cache_done(cache); + ret = btrfs_cache_block_group(cache, true); if (ret) { bg_failed++; bg_ret = ret; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index eed81a7e36a4d0dd1cd1fbd05b4b6383f0fc32f8..4dcf22e051ff88665573bd957c841f195b166f41 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -31,38 +31,27 @@ #include "block-group.h" #include "compression.h" -static struct kmem_cache *extent_state_cache; static struct kmem_cache *extent_buffer_cache; -static struct bio_set btrfs_bioset; - -static inline bool extent_state_in_tree(const struct extent_state *state) -{ - return !RB_EMPTY_NODE(&state->rb_node); -} #ifdef CONFIG_BTRFS_DEBUG -static LIST_HEAD(states); -static DEFINE_SPINLOCK(leak_lock); - -static inline void btrfs_leak_debug_add(spinlock_t *lock, - struct list_head *new, - struct list_head *head) +static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb) { + struct btrfs_fs_info *fs_info = eb->fs_info; unsigned long flags; - spin_lock_irqsave(lock, flags); - list_add(new, head); - spin_unlock_irqrestore(lock, flags); + spin_lock_irqsave(&fs_info->eb_leak_lock, flags); + list_add(&eb->leak_list, &fs_info->allocated_ebs); + spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); } -static inline void btrfs_leak_debug_del(spinlock_t *lock, - struct list_head *entry) +static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb) { + struct btrfs_fs_info *fs_info = eb->fs_info; unsigned long flags; - spin_lock_irqsave(lock, flags); - list_del(entry); - spin_unlock_irqrestore(lock, flags); + spin_lock_irqsave(&fs_info->eb_leak_lock, flags); + list_del(&eb->leak_list); + spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); } void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info) @@ -91,53 +80,11 @@ void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info) } spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); } - -static inline void btrfs_extent_state_leak_debug_check(void) -{ - struct extent_state *state; - - while (!list_empty(&states)) { - state = list_entry(states.next, struct extent_state, leak_list); - pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n", - state->start, state->end, state->state, - extent_state_in_tree(state), - refcount_read(&state->refs)); - list_del(&state->leak_list); - kmem_cache_free(extent_state_cache, state); - } -} - -#define btrfs_debug_check_extent_io_range(tree, start, end) \ - __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end)) -static inline void __btrfs_debug_check_extent_io_range(const char *caller, - struct extent_io_tree *tree, u64 start, u64 end) -{ - struct inode *inode = tree->private_data; - u64 isize; - - if (!inode || !is_data_inode(inode)) - return; - - isize = i_size_read(inode); - if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { - btrfs_debug_rl(BTRFS_I(inode)->root->fs_info, - "%s: ino %llu isize %llu odd range [%llu,%llu]", - caller, btrfs_ino(BTRFS_I(inode)), isize, start, end); - } -} #else -#define btrfs_leak_debug_add(lock, new, head) do {} while (0) -#define btrfs_leak_debug_del(lock, entry) do {} while (0) -#define btrfs_extent_state_leak_debug_check() do {} while (0) -#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0) +#define btrfs_leak_debug_add_eb(eb) do {} while (0) +#define btrfs_leak_debug_del_eb(eb) do {} while (0) #endif -struct tree_entry { - u64 start; - u64 end; - struct rb_node rb_node; -}; - /* * Structure to record info about the bio being assembled, and other info like * how many bytes are there before stripe/ordered extent boundary. @@ -148,6 +95,7 @@ struct btrfs_bio_ctrl { enum btrfs_compression_type compress_type; u32 len_to_stripe_boundary; u32 len_to_oe_boundary; + btrfs_bio_end_io_t end_io_func; }; struct extent_page_data { @@ -161,24 +109,6 @@ struct extent_page_data { unsigned int sync_io:1; }; -static int add_extent_changeset(struct extent_state *state, u32 bits, - struct extent_changeset *changeset, - int set) -{ - int ret; - - if (!changeset) - return 0; - if (set && (state->state & bits) == bits) - return 0; - if (!set && (state->state & bits) == 0) - return 0; - changeset->bytes_changed += state->end - state->start + 1; - ret = ulist_add(&changeset->range_changed, state->start, state->end, - GFP_ATOMIC); - return ret; -} - static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) { struct bio *bio; @@ -207,7 +137,7 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) btrfs_submit_data_read_bio(inode, bio, mirror_num, bio_ctrl->compress_type); - /* The bio is owned by the bi_end_io handler now */ + /* The bio is owned by the end_io handler now */ bio_ctrl->bio = NULL; } @@ -223,1304 +153,33 @@ static void submit_write_bio(struct extent_page_data *epd, int ret) if (ret) { ASSERT(ret < 0); - bio->bi_status = errno_to_blk_status(ret); - bio_endio(bio); - /* The bio is owned by the bi_end_io handler now */ + btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret)); + /* The bio is owned by the end_io handler now */ epd->bio_ctrl.bio = NULL; } else { submit_one_bio(&epd->bio_ctrl); } } -int __init extent_state_cache_init(void) -{ - extent_state_cache = kmem_cache_create("btrfs_extent_state", - sizeof(struct extent_state), 0, - SLAB_MEM_SPREAD, NULL); - if (!extent_state_cache) - return -ENOMEM; - return 0; -} - -int __init extent_io_init(void) -{ - extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer", - sizeof(struct extent_buffer), 0, - SLAB_MEM_SPREAD, NULL); - if (!extent_buffer_cache) - return -ENOMEM; - - if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE, - offsetof(struct btrfs_bio, bio), - BIOSET_NEED_BVECS)) - goto free_buffer_cache; - - if (bioset_integrity_create(&btrfs_bioset, BIO_POOL_SIZE)) - goto free_bioset; - - return 0; - -free_bioset: - bioset_exit(&btrfs_bioset); - -free_buffer_cache: - kmem_cache_destroy(extent_buffer_cache); - extent_buffer_cache = NULL; - return -ENOMEM; -} - -void __cold extent_state_cache_exit(void) -{ - btrfs_extent_state_leak_debug_check(); - kmem_cache_destroy(extent_state_cache); -} - -void __cold extent_io_exit(void) -{ - /* - * Make sure all delayed rcu free are flushed before we - * destroy caches. - */ - rcu_barrier(); - kmem_cache_destroy(extent_buffer_cache); - bioset_exit(&btrfs_bioset); -} - -/* - * For the file_extent_tree, we want to hold the inode lock when we lookup and - * update the disk_i_size, but lockdep will complain because our io_tree we hold - * the tree lock and get the inode lock when setting delalloc. These two things - * are unrelated, so make a class for the file_extent_tree so we don't get the - * two locking patterns mixed up. - */ -static struct lock_class_key file_extent_tree_class; - -void extent_io_tree_init(struct btrfs_fs_info *fs_info, - struct extent_io_tree *tree, unsigned int owner, - void *private_data) -{ - tree->fs_info = fs_info; - tree->state = RB_ROOT; - tree->dirty_bytes = 0; - spin_lock_init(&tree->lock); - tree->private_data = private_data; - tree->owner = owner; - if (owner == IO_TREE_INODE_FILE_EXTENT) - lockdep_set_class(&tree->lock, &file_extent_tree_class); -} - -void extent_io_tree_release(struct extent_io_tree *tree) -{ - spin_lock(&tree->lock); - /* - * Do a single barrier for the waitqueue_active check here, the state - * of the waitqueue should not change once extent_io_tree_release is - * called. - */ - smp_mb(); - while (!RB_EMPTY_ROOT(&tree->state)) { - struct rb_node *node; - struct extent_state *state; - - node = rb_first(&tree->state); - state = rb_entry(node, struct extent_state, rb_node); - rb_erase(&state->rb_node, &tree->state); - RB_CLEAR_NODE(&state->rb_node); - /* - * btree io trees aren't supposed to have tasks waiting for - * changes in the flags of extent states ever. - */ - ASSERT(!waitqueue_active(&state->wq)); - free_extent_state(state); - - cond_resched_lock(&tree->lock); - } - spin_unlock(&tree->lock); -} - -static struct extent_state *alloc_extent_state(gfp_t mask) -{ - struct extent_state *state; - - /* - * The given mask might be not appropriate for the slab allocator, - * drop the unsupported bits - */ - mask &= ~(__GFP_DMA32|__GFP_HIGHMEM); - state = kmem_cache_alloc(extent_state_cache, mask); - if (!state) - return state; - state->state = 0; - state->failrec = NULL; - RB_CLEAR_NODE(&state->rb_node); - btrfs_leak_debug_add(&leak_lock, &state->leak_list, &states); - refcount_set(&state->refs, 1); - init_waitqueue_head(&state->wq); - trace_alloc_extent_state(state, mask, _RET_IP_); - return state; -} - -void free_extent_state(struct extent_state *state) -{ - if (!state) - return; - if (refcount_dec_and_test(&state->refs)) { - WARN_ON(extent_state_in_tree(state)); - btrfs_leak_debug_del(&leak_lock, &state->leak_list); - trace_free_extent_state(state, _RET_IP_); - kmem_cache_free(extent_state_cache, state); - } -} - -/** - * Search @tree for an entry that contains @offset. Such entry would have - * entry->start <= offset && entry->end >= offset. - * - * @tree: the tree to search - * @offset: offset that should fall within an entry in @tree - * @node_ret: pointer where new node should be anchored (used when inserting an - * entry in the tree) - * @parent_ret: points to entry which would have been the parent of the entry, - * containing @offset - * - * Return a pointer to the entry that contains @offset byte address and don't change - * @node_ret and @parent_ret. - * - * If no such entry exists, return pointer to entry that ends before @offset - * and fill parameters @node_ret and @parent_ret, ie. does not return NULL. - */ -static inline struct rb_node *tree_search_for_insert(struct extent_io_tree *tree, - u64 offset, - struct rb_node ***node_ret, - struct rb_node **parent_ret) -{ - struct rb_root *root = &tree->state; - struct rb_node **node = &root->rb_node; - struct rb_node *prev = NULL; - struct tree_entry *entry; - - while (*node) { - prev = *node; - entry = rb_entry(prev, struct tree_entry, rb_node); - - if (offset < entry->start) - node = &(*node)->rb_left; - else if (offset > entry->end) - node = &(*node)->rb_right; - else - return *node; - } - - if (node_ret) - *node_ret = node; - if (parent_ret) - *parent_ret = prev; - - /* Search neighbors until we find the first one past the end */ - while (prev && offset > entry->end) { - prev = rb_next(prev); - entry = rb_entry(prev, struct tree_entry, rb_node); - } - - return prev; -} - -/* - * Inexact rb-tree search, return the next entry if @offset is not found - */ -static inline struct rb_node *tree_search(struct extent_io_tree *tree, u64 offset) -{ - return tree_search_for_insert(tree, offset, NULL, NULL); -} - -/** - * Search offset in the tree or fill neighbor rbtree node pointers. - * - * @tree: the tree to search - * @offset: offset that should fall within an entry in @tree - * @next_ret: pointer to the first entry whose range ends after @offset - * @prev_ret: pointer to the first entry whose range begins before @offset - * - * Return a pointer to the entry that contains @offset byte address. If no - * such entry exists, then return NULL and fill @prev_ret and @next_ret. - * Otherwise return the found entry and other pointers are left untouched. - */ -static struct rb_node *tree_search_prev_next(struct extent_io_tree *tree, - u64 offset, - struct rb_node **prev_ret, - struct rb_node **next_ret) -{ - struct rb_root *root = &tree->state; - struct rb_node **node = &root->rb_node; - struct rb_node *prev = NULL; - struct rb_node *orig_prev = NULL; - struct tree_entry *entry; - - ASSERT(prev_ret); - ASSERT(next_ret); - - while (*node) { - prev = *node; - entry = rb_entry(prev, struct tree_entry, rb_node); - - if (offset < entry->start) - node = &(*node)->rb_left; - else if (offset > entry->end) - node = &(*node)->rb_right; - else - return *node; - } - - orig_prev = prev; - while (prev && offset > entry->end) { - prev = rb_next(prev); - entry = rb_entry(prev, struct tree_entry, rb_node); - } - *next_ret = prev; - prev = orig_prev; - - entry = rb_entry(prev, struct tree_entry, rb_node); - while (prev && offset < entry->start) { - prev = rb_prev(prev); - entry = rb_entry(prev, struct tree_entry, rb_node); - } - *prev_ret = prev; - - return NULL; -} - -/* - * utility function to look for merge candidates inside a given range. - * Any extents with matching state are merged together into a single - * extent in the tree. Extents with EXTENT_IO in their state field - * are not merged because the end_io handlers need to be able to do - * operations on them without sleeping (or doing allocations/splits). - * - * This should be called with the tree lock held. - */ -static void merge_state(struct extent_io_tree *tree, - struct extent_state *state) -{ - struct extent_state *other; - struct rb_node *other_node; - - if (state->state & (EXTENT_LOCKED | EXTENT_BOUNDARY)) - return; - - other_node = rb_prev(&state->rb_node); - if (other_node) { - other = rb_entry(other_node, struct extent_state, rb_node); - if (other->end == state->start - 1 && - other->state == state->state) { - if (tree->private_data && - is_data_inode(tree->private_data)) - btrfs_merge_delalloc_extent(tree->private_data, - state, other); - state->start = other->start; - rb_erase(&other->rb_node, &tree->state); - RB_CLEAR_NODE(&other->rb_node); - free_extent_state(other); - } - } - other_node = rb_next(&state->rb_node); - if (other_node) { - other = rb_entry(other_node, struct extent_state, rb_node); - if (other->start == state->end + 1 && - other->state == state->state) { - if (tree->private_data && - is_data_inode(tree->private_data)) - btrfs_merge_delalloc_extent(tree->private_data, - state, other); - state->end = other->end; - rb_erase(&other->rb_node, &tree->state); - RB_CLEAR_NODE(&other->rb_node); - free_extent_state(other); - } - } -} - -static void set_state_bits(struct extent_io_tree *tree, - struct extent_state *state, u32 bits, - struct extent_changeset *changeset); - -/* - * insert an extent_state struct into the tree. 'bits' are set on the - * struct before it is inserted. - * - * This may return -EEXIST if the extent is already there, in which case the - * state struct is freed. - * - * The tree lock is not taken internally. This is a utility function and - * probably isn't what you want to call (see set/clear_extent_bit). - */ -static int insert_state(struct extent_io_tree *tree, - struct extent_state *state, - u32 bits, struct extent_changeset *changeset) -{ - struct rb_node **node; - struct rb_node *parent; - const u64 end = state->end; - - set_state_bits(tree, state, bits, changeset); - - node = &tree->state.rb_node; - while (*node) { - struct tree_entry *entry; - - parent = *node; - entry = rb_entry(parent, struct tree_entry, rb_node); - - if (end < entry->start) { - node = &(*node)->rb_left; - } else if (end > entry->end) { - node = &(*node)->rb_right; - } else { - btrfs_err(tree->fs_info, - "found node %llu %llu on insert of %llu %llu", - entry->start, entry->end, state->start, end); - return -EEXIST; - } - } - - rb_link_node(&state->rb_node, parent, node); - rb_insert_color(&state->rb_node, &tree->state); - - merge_state(tree, state); - return 0; -} - -/* - * Insert state to @tree to the location given by @node and @parent. - */ -static void insert_state_fast(struct extent_io_tree *tree, - struct extent_state *state, struct rb_node **node, - struct rb_node *parent, unsigned bits, - struct extent_changeset *changeset) -{ - set_state_bits(tree, state, bits, changeset); - rb_link_node(&state->rb_node, parent, node); - rb_insert_color(&state->rb_node, &tree->state); - merge_state(tree, state); -} - -/* - * split a given extent state struct in two, inserting the preallocated - * struct 'prealloc' as the newly created second half. 'split' indicates an - * offset inside 'orig' where it should be split. - * - * Before calling, - * the tree has 'orig' at [orig->start, orig->end]. After calling, there - * are two extent state structs in the tree: - * prealloc: [orig->start, split - 1] - * orig: [ split, orig->end ] - * - * The tree locks are not taken by this function. They need to be held - * by the caller. - */ -static int split_state(struct extent_io_tree *tree, struct extent_state *orig, - struct extent_state *prealloc, u64 split) -{ - struct rb_node *parent = NULL; - struct rb_node **node; - - if (tree->private_data && is_data_inode(tree->private_data)) - btrfs_split_delalloc_extent(tree->private_data, orig, split); - - prealloc->start = orig->start; - prealloc->end = split - 1; - prealloc->state = orig->state; - orig->start = split; - - parent = &orig->rb_node; - node = &parent; - while (*node) { - struct tree_entry *entry; - - parent = *node; - entry = rb_entry(parent, struct tree_entry, rb_node); - - if (prealloc->end < entry->start) { - node = &(*node)->rb_left; - } else if (prealloc->end > entry->end) { - node = &(*node)->rb_right; - } else { - free_extent_state(prealloc); - return -EEXIST; - } - } - - rb_link_node(&prealloc->rb_node, parent, node); - rb_insert_color(&prealloc->rb_node, &tree->state); - - return 0; -} - -static struct extent_state *next_state(struct extent_state *state) +int __init extent_buffer_init_cachep(void) { - struct rb_node *next = rb_next(&state->rb_node); - if (next) - return rb_entry(next, struct extent_state, rb_node); - else - return NULL; -} - -/* - * utility function to clear some bits in an extent state struct. - * it will optionally wake up anyone waiting on this state (wake == 1). - * - * If no bits are set on the state struct after clearing things, the - * struct is freed and removed from the tree - */ -static struct extent_state *clear_state_bit(struct extent_io_tree *tree, - struct extent_state *state, - u32 bits, int wake, - struct extent_changeset *changeset) -{ - struct extent_state *next; - u32 bits_to_clear = bits & ~EXTENT_CTLBITS; - int ret; - - if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) { - u64 range = state->end - state->start + 1; - WARN_ON(range > tree->dirty_bytes); - tree->dirty_bytes -= range; - } - - if (tree->private_data && is_data_inode(tree->private_data)) - btrfs_clear_delalloc_extent(tree->private_data, state, bits); - - ret = add_extent_changeset(state, bits_to_clear, changeset, 0); - BUG_ON(ret < 0); - state->state &= ~bits_to_clear; - if (wake) - wake_up(&state->wq); - if (state->state == 0) { - next = next_state(state); - if (extent_state_in_tree(state)) { - rb_erase(&state->rb_node, &tree->state); - RB_CLEAR_NODE(&state->rb_node); - free_extent_state(state); - } else { - WARN_ON(1); - } - } else { - merge_state(tree, state); - next = next_state(state); - } - return next; -} - -static struct extent_state * -alloc_extent_state_atomic(struct extent_state *prealloc) -{ - if (!prealloc) - prealloc = alloc_extent_state(GFP_ATOMIC); - - return prealloc; -} - -static void extent_io_tree_panic(struct extent_io_tree *tree, int err) -{ - btrfs_panic(tree->fs_info, err, - "locking error: extent tree was modified by another thread while locked"); -} - -/* - * clear some bits on a range in the tree. This may require splitting - * or inserting elements in the tree, so the gfp mask is used to - * indicate which allocations or sleeping are allowed. - * - * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove - * the given range from the tree regardless of state (ie for truncate). - * - * the range [start, end] is inclusive. - * - * This takes the tree lock, and returns 0 on success and < 0 on error. - */ -int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, int wake, int delete, - struct extent_state **cached_state, - gfp_t mask, struct extent_changeset *changeset) -{ - struct extent_state *state; - struct extent_state *cached; - struct extent_state *prealloc = NULL; - struct rb_node *node; - u64 last_end; - int err; - int clear = 0; - - btrfs_debug_check_extent_io_range(tree, start, end); - trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits); - - if (bits & EXTENT_DELALLOC) - bits |= EXTENT_NORESERVE; - - if (delete) - bits |= ~EXTENT_CTLBITS; - - if (bits & (EXTENT_LOCKED | EXTENT_BOUNDARY)) - clear = 1; -again: - if (!prealloc && gfpflags_allow_blocking(mask)) { - /* - * Don't care for allocation failure here because we might end - * up not needing the pre-allocated extent state at all, which - * is the case if we only have in the tree extent states that - * cover our input range and don't cover too any other range. - * If we end up needing a new extent state we allocate it later. - */ - prealloc = alloc_extent_state(mask); - } - - spin_lock(&tree->lock); - if (cached_state) { - cached = *cached_state; - - if (clear) { - *cached_state = NULL; - cached_state = NULL; - } - - if (cached && extent_state_in_tree(cached) && - cached->start <= start && cached->end > start) { - if (clear) - refcount_dec(&cached->refs); - state = cached; - goto hit_next; - } - if (clear) - free_extent_state(cached); - } - /* - * this search will find the extents that end after - * our range starts - */ - node = tree_search(tree, start); - if (!node) - goto out; - state = rb_entry(node, struct extent_state, rb_node); -hit_next: - if (state->start > end) - goto out; - WARN_ON(state->end < start); - last_end = state->end; - - /* the state doesn't have the wanted bits, go ahead */ - if (!(state->state & bits)) { - state = next_state(state); - goto next; - } - - /* - * | ---- desired range ---- | - * | state | or - * | ------------- state -------------- | - * - * We need to split the extent we found, and may flip - * bits on second half. - * - * If the extent we found extends past our range, we - * just split and search again. It'll get split again - * the next time though. - * - * If the extent we found is inside our range, we clear - * the desired bit on it. - */ - - if (state->start < start) { - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - err = split_state(tree, state, prealloc, start); - if (err) - extent_io_tree_panic(tree, err); - - prealloc = NULL; - if (err) - goto out; - if (state->end <= end) { - state = clear_state_bit(tree, state, bits, wake, changeset); - goto next; - } - goto search_again; - } - /* - * | ---- desired range ---- | - * | state | - * We need to split the extent, and clear the bit - * on the first half - */ - if (state->start <= end && state->end > end) { - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - err = split_state(tree, state, prealloc, end + 1); - if (err) - extent_io_tree_panic(tree, err); - - if (wake) - wake_up(&state->wq); - - clear_state_bit(tree, prealloc, bits, wake, changeset); - - prealloc = NULL; - goto out; - } - - state = clear_state_bit(tree, state, bits, wake, changeset); -next: - if (last_end == (u64)-1) - goto out; - start = last_end + 1; - if (start <= end && state && !need_resched()) - goto hit_next; - -search_again: - if (start > end) - goto out; - spin_unlock(&tree->lock); - if (gfpflags_allow_blocking(mask)) - cond_resched(); - goto again; - -out: - spin_unlock(&tree->lock); - if (prealloc) - free_extent_state(prealloc); - - return 0; - -} - -static void wait_on_state(struct extent_io_tree *tree, - struct extent_state *state) - __releases(tree->lock) - __acquires(tree->lock) -{ - DEFINE_WAIT(wait); - prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); - spin_unlock(&tree->lock); - schedule(); - spin_lock(&tree->lock); - finish_wait(&state->wq, &wait); -} - -/* - * waits for one or more bits to clear on a range in the state tree. - * The range [start, end] is inclusive. - * The tree lock is taken by this function - */ -static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits) -{ - struct extent_state *state; - struct rb_node *node; - - btrfs_debug_check_extent_io_range(tree, start, end); - - spin_lock(&tree->lock); -again: - while (1) { - /* - * this search will find all the extents that end after - * our range starts - */ - node = tree_search(tree, start); -process_node: - if (!node) - break; - - state = rb_entry(node, struct extent_state, rb_node); - - if (state->start > end) - goto out; - - if (state->state & bits) { - start = state->start; - refcount_inc(&state->refs); - wait_on_state(tree, state); - free_extent_state(state); - goto again; - } - start = state->end + 1; - - if (start > end) - break; - - if (!cond_resched_lock(&tree->lock)) { - node = rb_next(node); - goto process_node; - } - } -out: - spin_unlock(&tree->lock); -} - -static void set_state_bits(struct extent_io_tree *tree, - struct extent_state *state, - u32 bits, struct extent_changeset *changeset) -{ - u32 bits_to_set = bits & ~EXTENT_CTLBITS; - int ret; - - if (tree->private_data && is_data_inode(tree->private_data)) - btrfs_set_delalloc_extent(tree->private_data, state, bits); - - if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) { - u64 range = state->end - state->start + 1; - tree->dirty_bytes += range; - } - ret = add_extent_changeset(state, bits_to_set, changeset, 1); - BUG_ON(ret < 0); - state->state |= bits_to_set; -} - -static void cache_state_if_flags(struct extent_state *state, - struct extent_state **cached_ptr, - unsigned flags) -{ - if (cached_ptr && !(*cached_ptr)) { - if (!flags || (state->state & flags)) { - *cached_ptr = state; - refcount_inc(&state->refs); - } - } -} - -static void cache_state(struct extent_state *state, - struct extent_state **cached_ptr) -{ - return cache_state_if_flags(state, cached_ptr, - EXTENT_LOCKED | EXTENT_BOUNDARY); -} - -/* - * set some bits on a range in the tree. This may require allocations or - * sleeping, so the gfp mask is used to indicate what is allowed. - * - * If any of the exclusive bits are set, this will fail with -EEXIST if some - * part of the range already has the desired bits set. The start of the - * existing range is returned in failed_start in this case. - * - * [start, end] is inclusive This takes the tree lock. - */ -int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, - u32 exclusive_bits, u64 *failed_start, - struct extent_state **cached_state, gfp_t mask, - struct extent_changeset *changeset) -{ - struct extent_state *state; - struct extent_state *prealloc = NULL; - struct rb_node *node; - struct rb_node **p; - struct rb_node *parent; - int err = 0; - u64 last_start; - u64 last_end; - - btrfs_debug_check_extent_io_range(tree, start, end); - trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits); - - if (exclusive_bits) - ASSERT(failed_start); - else - ASSERT(failed_start == NULL); -again: - if (!prealloc && gfpflags_allow_blocking(mask)) { - /* - * Don't care for allocation failure here because we might end - * up not needing the pre-allocated extent state at all, which - * is the case if we only have in the tree extent states that - * cover our input range and don't cover too any other range. - * If we end up needing a new extent state we allocate it later. - */ - prealloc = alloc_extent_state(mask); - } - - spin_lock(&tree->lock); - if (cached_state && *cached_state) { - state = *cached_state; - if (state->start <= start && state->end > start && - extent_state_in_tree(state)) { - node = &state->rb_node; - goto hit_next; - } - } - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search_for_insert(tree, start, &p, &parent); - if (!node) { - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - prealloc->start = start; - prealloc->end = end; - insert_state_fast(tree, prealloc, p, parent, bits, changeset); - cache_state(prealloc, cached_state); - prealloc = NULL; - goto out; - } - state = rb_entry(node, struct extent_state, rb_node); -hit_next: - last_start = state->start; - last_end = state->end; - - /* - * | ---- desired range ---- | - * | state | - * - * Just lock what we found and keep going - */ - if (state->start == start && state->end <= end) { - if (state->state & exclusive_bits) { - *failed_start = state->start; - err = -EEXIST; - goto out; - } - - set_state_bits(tree, state, bits, changeset); - cache_state(state, cached_state); - merge_state(tree, state); - if (last_end == (u64)-1) - goto out; - start = last_end + 1; - state = next_state(state); - if (start < end && state && state->start == start && - !need_resched()) - goto hit_next; - goto search_again; - } - - /* - * | ---- desired range ---- | - * | state | - * or - * | ------------- state -------------- | - * - * We need to split the extent we found, and may flip bits on - * second half. - * - * If the extent we found extends past our - * range, we just split and search again. It'll get split - * again the next time though. - * - * If the extent we found is inside our range, we set the - * desired bit on it. - */ - if (state->start < start) { - if (state->state & exclusive_bits) { - *failed_start = start; - err = -EEXIST; - goto out; - } - - /* - * If this extent already has all the bits we want set, then - * skip it, not necessary to split it or do anything with it. - */ - if ((state->state & bits) == bits) { - start = state->end + 1; - cache_state(state, cached_state); - goto search_again; - } - - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - err = split_state(tree, state, prealloc, start); - if (err) - extent_io_tree_panic(tree, err); - - prealloc = NULL; - if (err) - goto out; - if (state->end <= end) { - set_state_bits(tree, state, bits, changeset); - cache_state(state, cached_state); - merge_state(tree, state); - if (last_end == (u64)-1) - goto out; - start = last_end + 1; - state = next_state(state); - if (start < end && state && state->start == start && - !need_resched()) - goto hit_next; - } - goto search_again; - } - /* - * | ---- desired range ---- | - * | state | or | state | - * - * There's a hole, we need to insert something in it and - * ignore the extent we found. - */ - if (state->start > start) { - u64 this_end; - if (end < last_start) - this_end = end; - else - this_end = last_start - 1; - - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - - /* - * Avoid to free 'prealloc' if it can be merged with - * the later extent. - */ - prealloc->start = start; - prealloc->end = this_end; - err = insert_state(tree, prealloc, bits, changeset); - if (err) - extent_io_tree_panic(tree, err); - - cache_state(prealloc, cached_state); - prealloc = NULL; - start = this_end + 1; - goto search_again; - } - /* - * | ---- desired range ---- | - * | state | - * We need to split the extent, and set the bit - * on the first half - */ - if (state->start <= end && state->end > end) { - if (state->state & exclusive_bits) { - *failed_start = start; - err = -EEXIST; - goto out; - } - - prealloc = alloc_extent_state_atomic(prealloc); - BUG_ON(!prealloc); - err = split_state(tree, state, prealloc, end + 1); - if (err) - extent_io_tree_panic(tree, err); - - set_state_bits(tree, prealloc, bits, changeset); - cache_state(prealloc, cached_state); - merge_state(tree, prealloc); - prealloc = NULL; - goto out; - } - -search_again: - if (start > end) - goto out; - spin_unlock(&tree->lock); - if (gfpflags_allow_blocking(mask)) - cond_resched(); - goto again; - -out: - spin_unlock(&tree->lock); - if (prealloc) - free_extent_state(prealloc); - - return err; - -} - -/** - * convert_extent_bit - convert all bits in a given range from one bit to - * another - * @tree: the io tree to search - * @start: the start offset in bytes - * @end: the end offset in bytes (inclusive) - * @bits: the bits to set in this range - * @clear_bits: the bits to clear in this range - * @cached_state: state that we're going to cache - * - * This will go through and set bits for the given range. If any states exist - * already in this range they are set with the given bit and cleared of the - * clear_bits. This is only meant to be used by things that are mergeable, ie - * converting from say DELALLOC to DIRTY. This is not meant to be used with - * boundary bits like LOCK. - * - * All allocations are done with GFP_NOFS. - */ -int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, u32 clear_bits, - struct extent_state **cached_state) -{ - struct extent_state *state; - struct extent_state *prealloc = NULL; - struct rb_node *node; - struct rb_node **p; - struct rb_node *parent; - int err = 0; - u64 last_start; - u64 last_end; - bool first_iteration = true; - - btrfs_debug_check_extent_io_range(tree, start, end); - trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits, - clear_bits); - -again: - if (!prealloc) { - /* - * Best effort, don't worry if extent state allocation fails - * here for the first iteration. We might have a cached state - * that matches exactly the target range, in which case no - * extent state allocations are needed. We'll only know this - * after locking the tree. - */ - prealloc = alloc_extent_state(GFP_NOFS); - if (!prealloc && !first_iteration) - return -ENOMEM; - } - - spin_lock(&tree->lock); - if (cached_state && *cached_state) { - state = *cached_state; - if (state->start <= start && state->end > start && - extent_state_in_tree(state)) { - node = &state->rb_node; - goto hit_next; - } - } - - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search_for_insert(tree, start, &p, &parent); - if (!node) { - prealloc = alloc_extent_state_atomic(prealloc); - if (!prealloc) { - err = -ENOMEM; - goto out; - } - prealloc->start = start; - prealloc->end = end; - insert_state_fast(tree, prealloc, p, parent, bits, NULL); - cache_state(prealloc, cached_state); - prealloc = NULL; - goto out; - } - state = rb_entry(node, struct extent_state, rb_node); -hit_next: - last_start = state->start; - last_end = state->end; - - /* - * | ---- desired range ---- | - * | state | - * - * Just lock what we found and keep going - */ - if (state->start == start && state->end <= end) { - set_state_bits(tree, state, bits, NULL); - cache_state(state, cached_state); - state = clear_state_bit(tree, state, clear_bits, 0, NULL); - if (last_end == (u64)-1) - goto out; - start = last_end + 1; - if (start < end && state && state->start == start && - !need_resched()) - goto hit_next; - goto search_again; - } - - /* - * | ---- desired range ---- | - * | state | - * or - * | ------------- state -------------- | - * - * We need to split the extent we found, and may flip bits on - * second half. - * - * If the extent we found extends past our - * range, we just split and search again. It'll get split - * again the next time though. - * - * If the extent we found is inside our range, we set the - * desired bit on it. - */ - if (state->start < start) { - prealloc = alloc_extent_state_atomic(prealloc); - if (!prealloc) { - err = -ENOMEM; - goto out; - } - err = split_state(tree, state, prealloc, start); - if (err) - extent_io_tree_panic(tree, err); - prealloc = NULL; - if (err) - goto out; - if (state->end <= end) { - set_state_bits(tree, state, bits, NULL); - cache_state(state, cached_state); - state = clear_state_bit(tree, state, clear_bits, 0, NULL); - if (last_end == (u64)-1) - goto out; - start = last_end + 1; - if (start < end && state && state->start == start && - !need_resched()) - goto hit_next; - } - goto search_again; - } - /* - * | ---- desired range ---- | - * | state | or | state | - * - * There's a hole, we need to insert something in it and - * ignore the extent we found. - */ - if (state->start > start) { - u64 this_end; - if (end < last_start) - this_end = end; - else - this_end = last_start - 1; - - prealloc = alloc_extent_state_atomic(prealloc); - if (!prealloc) { - err = -ENOMEM; - goto out; - } - - /* - * Avoid to free 'prealloc' if it can be merged with - * the later extent. - */ - prealloc->start = start; - prealloc->end = this_end; - err = insert_state(tree, prealloc, bits, NULL); - if (err) - extent_io_tree_panic(tree, err); - cache_state(prealloc, cached_state); - prealloc = NULL; - start = this_end + 1; - goto search_again; - } - /* - * | ---- desired range ---- | - * | state | - * We need to split the extent, and set the bit - * on the first half - */ - if (state->start <= end && state->end > end) { - prealloc = alloc_extent_state_atomic(prealloc); - if (!prealloc) { - err = -ENOMEM; - goto out; - } - - err = split_state(tree, state, prealloc, end + 1); - if (err) - extent_io_tree_panic(tree, err); - - set_state_bits(tree, prealloc, bits, NULL); - cache_state(prealloc, cached_state); - clear_state_bit(tree, prealloc, clear_bits, 0, NULL); - prealloc = NULL; - goto out; - } - -search_again: - if (start > end) - goto out; - spin_unlock(&tree->lock); - cond_resched(); - first_iteration = false; - goto again; - -out: - spin_unlock(&tree->lock); - if (prealloc) - free_extent_state(prealloc); - - return err; -} - -/* wrappers around set/clear extent bit */ -int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, struct extent_changeset *changeset) -{ - /* - * We don't support EXTENT_LOCKED yet, as current changeset will - * record any bits changed, so for EXTENT_LOCKED case, it will - * either fail with -EEXIST or changeset will record the whole - * range. - */ - BUG_ON(bits & EXTENT_LOCKED); - - return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, GFP_NOFS, - changeset); -} - -int set_extent_bits_nowait(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits) -{ - return set_extent_bit(tree, start, end, bits, 0, NULL, NULL, - GFP_NOWAIT, NULL); -} - -int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, int wake, int delete, - struct extent_state **cached) -{ - return __clear_extent_bit(tree, start, end, bits, wake, delete, - cached, GFP_NOFS, NULL); -} - -int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, struct extent_changeset *changeset) -{ - /* - * Don't support EXTENT_LOCKED case, same reason as - * set_record_extent_bits(). - */ - BUG_ON(bits & EXTENT_LOCKED); - - return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, GFP_NOFS, - changeset); -} - -/* - * either insert or lock state struct between start and end use mask to tell - * us if waiting is desired. - */ -int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, - struct extent_state **cached_state) -{ - int err; - u64 failed_start; + extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer", + sizeof(struct extent_buffer), 0, + SLAB_MEM_SPREAD, NULL); + if (!extent_buffer_cache) + return -ENOMEM; - while (1) { - err = set_extent_bit(tree, start, end, EXTENT_LOCKED, - EXTENT_LOCKED, &failed_start, - cached_state, GFP_NOFS, NULL); - if (err == -EEXIST) { - wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); - start = failed_start; - } else - break; - WARN_ON(start > end); - } - return err; + return 0; } -int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end) +void __cold extent_buffer_free_cachep(void) { - int err; - u64 failed_start; - - err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED, - &failed_start, NULL, GFP_NOFS, NULL); - if (err == -EEXIST) { - if (failed_start > start) - clear_extent_bit(tree, start, failed_start - 1, - EXTENT_LOCKED, 1, 0, NULL); - return 0; - } - return 1; + /* + * Make sure all delayed rcu free are flushed before we + * destroy caches. + */ + rcu_barrier(); + kmem_cache_destroy(extent_buffer_cache); } void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end) @@ -1554,295 +213,6 @@ void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end) } } -/* find the first state struct with 'bits' set after 'start', and - * return it. tree->lock must be held. NULL will returned if - * nothing was found after 'start' - */ -static struct extent_state * -find_first_extent_bit_state(struct extent_io_tree *tree, u64 start, u32 bits) -{ - struct rb_node *node; - struct extent_state *state; - - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search(tree, start); - if (!node) - goto out; - - while (1) { - state = rb_entry(node, struct extent_state, rb_node); - if (state->end >= start && (state->state & bits)) - return state; - - node = rb_next(node); - if (!node) - break; - } -out: - return NULL; -} - -/* - * Find the first offset in the io tree with one or more @bits set. - * - * Note: If there are multiple bits set in @bits, any of them will match. - * - * Return 0 if we find something, and update @start_ret and @end_ret. - * Return 1 if we found nothing. - */ -int find_first_extent_bit(struct extent_io_tree *tree, u64 start, - u64 *start_ret, u64 *end_ret, u32 bits, - struct extent_state **cached_state) -{ - struct extent_state *state; - int ret = 1; - - spin_lock(&tree->lock); - if (cached_state && *cached_state) { - state = *cached_state; - if (state->end == start - 1 && extent_state_in_tree(state)) { - while ((state = next_state(state)) != NULL) { - if (state->state & bits) - goto got_it; - } - free_extent_state(*cached_state); - *cached_state = NULL; - goto out; - } - free_extent_state(*cached_state); - *cached_state = NULL; - } - - state = find_first_extent_bit_state(tree, start, bits); -got_it: - if (state) { - cache_state_if_flags(state, cached_state, 0); - *start_ret = state->start; - *end_ret = state->end; - ret = 0; - } -out: - spin_unlock(&tree->lock); - return ret; -} - -/** - * Find a contiguous area of bits - * - * @tree: io tree to check - * @start: offset to start the search from - * @start_ret: the first offset we found with the bits set - * @end_ret: the final contiguous range of the bits that were set - * @bits: bits to look for - * - * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges - * to set bits appropriately, and then merge them again. During this time it - * will drop the tree->lock, so use this helper if you want to find the actual - * contiguous area for given bits. We will search to the first bit we find, and - * then walk down the tree until we find a non-contiguous area. The area - * returned will be the full contiguous area with the bits set. - */ -int find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, - u64 *start_ret, u64 *end_ret, u32 bits) -{ - struct extent_state *state; - int ret = 1; - - spin_lock(&tree->lock); - state = find_first_extent_bit_state(tree, start, bits); - if (state) { - *start_ret = state->start; - *end_ret = state->end; - while ((state = next_state(state)) != NULL) { - if (state->start > (*end_ret + 1)) - break; - *end_ret = state->end; - } - ret = 0; - } - spin_unlock(&tree->lock); - return ret; -} - -/** - * Find the first range that has @bits not set. This range could start before - * @start. - * - * @tree: the tree to search - * @start: offset at/after which the found extent should start - * @start_ret: records the beginning of the range - * @end_ret: records the end of the range (inclusive) - * @bits: the set of bits which must be unset - * - * Since unallocated range is also considered one which doesn't have the bits - * set it's possible that @end_ret contains -1, this happens in case the range - * spans (last_range_end, end of device]. In this case it's up to the caller to - * trim @end_ret to the appropriate size. - */ -void find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, - u64 *start_ret, u64 *end_ret, u32 bits) -{ - struct extent_state *state; - struct rb_node *node, *prev = NULL, *next; - - spin_lock(&tree->lock); - - /* Find first extent with bits cleared */ - while (1) { - node = tree_search_prev_next(tree, start, &prev, &next); - if (!node && !next && !prev) { - /* - * Tree is completely empty, send full range and let - * caller deal with it - */ - *start_ret = 0; - *end_ret = -1; - goto out; - } else if (!node && !next) { - /* - * We are past the last allocated chunk, set start at - * the end of the last extent. - */ - state = rb_entry(prev, struct extent_state, rb_node); - *start_ret = state->end + 1; - *end_ret = -1; - goto out; - } else if (!node) { - node = next; - } - /* - * At this point 'node' either contains 'start' or start is - * before 'node' - */ - state = rb_entry(node, struct extent_state, rb_node); - - if (in_range(start, state->start, state->end - state->start + 1)) { - if (state->state & bits) { - /* - * |--range with bits sets--| - * | - * start - */ - start = state->end + 1; - } else { - /* - * 'start' falls within a range that doesn't - * have the bits set, so take its start as - * the beginning of the desired range - * - * |--range with bits cleared----| - * | - * start - */ - *start_ret = state->start; - break; - } - } else { - /* - * |---prev range---|---hole/unset---|---node range---| - * | - * start - * - * or - * - * |---hole/unset--||--first node--| - * 0 | - * start - */ - if (prev) { - state = rb_entry(prev, struct extent_state, - rb_node); - *start_ret = state->end + 1; - } else { - *start_ret = 0; - } - break; - } - } - - /* - * Find the longest stretch from start until an entry which has the - * bits set - */ - while (1) { - state = rb_entry(node, struct extent_state, rb_node); - if (state->end >= start && !(state->state & bits)) { - *end_ret = state->end; - } else { - *end_ret = state->start - 1; - break; - } - - node = rb_next(node); - if (!node) - break; - } -out: - spin_unlock(&tree->lock); -} - -/* - * find a contiguous range of bytes in the file marked as delalloc, not - * more than 'max_bytes'. start and end are used to return the range, - * - * true is returned if we find something, false if nothing was in the tree - */ -bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, - u64 *end, u64 max_bytes, - struct extent_state **cached_state) -{ - struct rb_node *node; - struct extent_state *state; - u64 cur_start = *start; - bool found = false; - u64 total_bytes = 0; - - spin_lock(&tree->lock); - - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search(tree, cur_start); - if (!node) { - *end = (u64)-1; - goto out; - } - - while (1) { - state = rb_entry(node, struct extent_state, rb_node); - if (found && (state->start != cur_start || - (state->state & EXTENT_BOUNDARY))) { - goto out; - } - if (!(state->state & EXTENT_DELALLOC)) { - if (!found) - *end = state->end; - goto out; - } - if (!found) { - *start = state->start; - *cached_state = state; - refcount_inc(&state->refs); - } - found = true; - *end = state->end; - cur_start = state->end + 1; - node = rb_next(node); - total_bytes += state->end - state->start + 1; - if (total_bytes >= max_bytes) - break; - if (!node) - break; - } -out: - spin_unlock(&tree->lock); - return found; -} - /* * Process one page for __process_pages_contig(). * @@ -1900,9 +270,8 @@ static int __process_pages_contig(struct address_space *mapping, pgoff_t start_index = start >> PAGE_SHIFT; pgoff_t end_index = end >> PAGE_SHIFT; pgoff_t index = start_index; - unsigned long nr_pages = end_index - start_index + 1; unsigned long pages_processed = 0; - struct page *pages[16]; + struct folio_batch fbatch; int err = 0; int i; @@ -1911,16 +280,17 @@ static int __process_pages_contig(struct address_space *mapping, ASSERT(processed_end && *processed_end == start); } - if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0) + if ((page_ops & PAGE_SET_ERROR) && start_index <= end_index) mapping_set_error(mapping, -EIO); - while (nr_pages > 0) { - int found_pages; + folio_batch_init(&fbatch); + while (index <= end_index) { + int found_folios; + + found_folios = filemap_get_folios_contig(mapping, &index, + end_index, &fbatch); - found_pages = find_get_pages_contig(mapping, index, - min_t(unsigned long, - nr_pages, ARRAY_SIZE(pages)), pages); - if (found_pages == 0) { + if (found_folios == 0) { /* * Only if we're going to lock these pages, we can find * nothing at @index. @@ -1930,23 +300,20 @@ static int __process_pages_contig(struct address_space *mapping, goto out; } - for (i = 0; i < found_pages; i++) { + for (i = 0; i < found_folios; i++) { int process_ret; - + struct folio *folio = fbatch.folios[i]; process_ret = process_one_page(fs_info, mapping, - pages[i], locked_page, page_ops, + &folio->page, locked_page, page_ops, start, end); if (process_ret < 0) { - for (; i < found_pages; i++) - put_page(pages[i]); err = -EAGAIN; + folio_batch_release(&fbatch); goto out; } - put_page(pages[i]); - pages_processed++; + pages_processed += folio_nr_pages(folio); } - nr_pages -= found_pages; - index += found_pages; + folio_batch_release(&fbatch); cond_resched(); } out: @@ -2094,14 +461,14 @@ again: } /* step three, lock the state bits for the whole range */ - lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state); + lock_extent(tree, delalloc_start, delalloc_end, &cached_state); /* then test to make sure it is all still delalloc */ ret = test_range_bit(tree, delalloc_start, delalloc_end, EXTENT_DELALLOC, 1, cached_state); if (!ret) { - unlock_extent_cached(tree, delalloc_start, delalloc_end, - &cached_state); + unlock_extent(tree, delalloc_start, delalloc_end, + &cached_state); __unlock_for_delalloc(inode, locked_page, delalloc_start, delalloc_end); cond_resched(); @@ -2118,210 +485,46 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, struct page *locked_page, u32 clear_bits, unsigned long page_ops) { - clear_extent_bit(&inode->io_tree, start, end, clear_bits, 1, 0, NULL); + clear_extent_bit(&inode->io_tree, start, end, clear_bits, NULL); __process_pages_contig(inode->vfs_inode.i_mapping, locked_page, start, end, page_ops, NULL); } -/* - * count the number of bytes in the tree that have a given bit(s) - * set. This can be fairly slow, except for EXTENT_DIRTY which is - * cached. The total number found is returned. - */ -u64 count_range_bits(struct extent_io_tree *tree, - u64 *start, u64 search_end, u64 max_bytes, - u32 bits, int contig) +static int insert_failrec(struct btrfs_inode *inode, + struct io_failure_record *failrec) { - struct rb_node *node; - struct extent_state *state; - u64 cur_start = *start; - u64 total_bytes = 0; - u64 last = 0; - int found = 0; - - if (WARN_ON(search_end <= cur_start)) - return 0; - - spin_lock(&tree->lock); - if (cur_start == 0 && bits == EXTENT_DIRTY) { - total_bytes = tree->dirty_bytes; - goto out; - } - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search(tree, cur_start); - if (!node) - goto out; - - while (1) { - state = rb_entry(node, struct extent_state, rb_node); - if (state->start > search_end) - break; - if (contig && found && state->start > last + 1) - break; - if (state->end >= cur_start && (state->state & bits) == bits) { - total_bytes += min(search_end, state->end) + 1 - - max(cur_start, state->start); - if (total_bytes >= max_bytes) - break; - if (!found) { - *start = max(cur_start, state->start); - found = 1; - } - last = state->end; - } else if (contig && found) { - break; - } - node = rb_next(node); - if (!node) - break; - } -out: - spin_unlock(&tree->lock); - return total_bytes; -} + struct rb_node *exist; -/* - * set the private field for a given byte offset in the tree. If there isn't - * an extent_state there already, this does nothing. - */ -int set_state_failrec(struct extent_io_tree *tree, u64 start, - struct io_failure_record *failrec) -{ - struct rb_node *node; - struct extent_state *state; - int ret = 0; + spin_lock(&inode->io_failure_lock); + exist = rb_simple_insert(&inode->io_failure_tree, failrec->bytenr, + &failrec->rb_node); + spin_unlock(&inode->io_failure_lock); - spin_lock(&tree->lock); - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search(tree, start); - if (!node) { - ret = -ENOENT; - goto out; - } - state = rb_entry(node, struct extent_state, rb_node); - if (state->start != start) { - ret = -ENOENT; - goto out; - } - state->failrec = failrec; -out: - spin_unlock(&tree->lock); - return ret; + return (exist == NULL) ? 0 : -EEXIST; } -struct io_failure_record *get_state_failrec(struct extent_io_tree *tree, u64 start) +static struct io_failure_record *get_failrec(struct btrfs_inode *inode, u64 start) { struct rb_node *node; - struct extent_state *state; - struct io_failure_record *failrec; - - spin_lock(&tree->lock); - /* - * this search will find all the extents that end after - * our range starts. - */ - node = tree_search(tree, start); - if (!node) { - failrec = ERR_PTR(-ENOENT); - goto out; - } - state = rb_entry(node, struct extent_state, rb_node); - if (state->start != start) { - failrec = ERR_PTR(-ENOENT); - goto out; - } + struct io_failure_record *failrec = ERR_PTR(-ENOENT); - failrec = state->failrec; -out: - spin_unlock(&tree->lock); + spin_lock(&inode->io_failure_lock); + node = rb_simple_search(&inode->io_failure_tree, start); + if (node) + failrec = rb_entry(node, struct io_failure_record, rb_node); + spin_unlock(&inode->io_failure_lock); return failrec; } -/* - * searches a range in the state tree for a given mask. - * If 'filled' == 1, this returns 1 only if every extent in the tree - * has the bits set. Otherwise, 1 is returned if any bit in the - * range is found set. - */ -int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, - u32 bits, int filled, struct extent_state *cached) -{ - struct extent_state *state = NULL; - struct rb_node *node; - int bitset = 0; - - spin_lock(&tree->lock); - if (cached && extent_state_in_tree(cached) && cached->start <= start && - cached->end > start) - node = &cached->rb_node; - else - node = tree_search(tree, start); - while (node && start <= end) { - state = rb_entry(node, struct extent_state, rb_node); - - if (filled && state->start > start) { - bitset = 0; - break; - } - - if (state->start > end) - break; - - if (state->state & bits) { - bitset = 1; - if (!filled) - break; - } else if (filled) { - bitset = 0; - break; - } - - if (state->end == (u64)-1) - break; - - start = state->end + 1; - if (start > end) - break; - node = rb_next(node); - if (!node) { - if (filled) - bitset = 0; - break; - } - } - spin_unlock(&tree->lock); - return bitset; -} - -int free_io_failure(struct extent_io_tree *failure_tree, - struct extent_io_tree *io_tree, - struct io_failure_record *rec) +static void free_io_failure(struct btrfs_inode *inode, + struct io_failure_record *rec) { - int ret; - int err = 0; - - set_state_failrec(failure_tree, rec->start, NULL); - ret = clear_extent_bits(failure_tree, rec->start, - rec->start + rec->len - 1, - EXTENT_LOCKED | EXTENT_DIRTY); - if (ret) - err = ret; - - ret = clear_extent_bits(io_tree, rec->start, - rec->start + rec->len - 1, - EXTENT_DAMAGED); - if (ret && !err) - err = ret; + spin_lock(&inode->io_failure_lock); + rb_erase(&rec->rb_node, &inode->io_failure_tree); + spin_unlock(&inode->io_failure_lock); kfree(rec); - return err; } /* @@ -2456,24 +659,18 @@ static int prev_mirror(const struct io_failure_record *failrec, int cur_mirror) * each time an IO finishes, we do a fast check in the IO failure tree * to see if we need to process or clean up an io_failure_record */ -int clean_io_failure(struct btrfs_fs_info *fs_info, - struct extent_io_tree *failure_tree, - struct extent_io_tree *io_tree, u64 start, - struct page *page, u64 ino, unsigned int pg_offset) +int btrfs_clean_io_failure(struct btrfs_inode *inode, u64 start, + struct page *page, unsigned int pg_offset) { - u64 private; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + u64 ino = btrfs_ino(inode); + u64 locked_start, locked_end; struct io_failure_record *failrec; - struct extent_state *state; int mirror; int ret; - private = 0; - ret = count_range_bits(failure_tree, &private, (u64)-1, 1, - EXTENT_DIRTY, 0); - if (!ret) - return 0; - - failrec = get_state_failrec(failure_tree, start); + failrec = get_failrec(inode, start); if (IS_ERR(failrec)) return 0; @@ -2482,14 +679,10 @@ int clean_io_failure(struct btrfs_fs_info *fs_info, if (sb_rdonly(fs_info->sb)) goto out; - spin_lock(&io_tree->lock); - state = find_first_extent_bit_state(io_tree, - failrec->start, - EXTENT_LOCKED); - spin_unlock(&io_tree->lock); - - if (!state || state->start > failrec->start || - state->end < failrec->start + failrec->len - 1) + ret = find_first_extent_bit(io_tree, failrec->bytenr, &locked_start, + &locked_end, EXTENT_LOCKED, NULL); + if (ret || locked_start > failrec->bytenr || + locked_end < failrec->bytenr + failrec->len - 1) goto out; mirror = failrec->this_mirror; @@ -2500,7 +693,7 @@ int clean_io_failure(struct btrfs_fs_info *fs_info, } while (mirror != failrec->failed_mirror); out: - free_io_failure(failure_tree, io_tree, failrec); + free_io_failure(inode, failrec); return 0; } @@ -2512,30 +705,26 @@ out: */ void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end) { - struct extent_io_tree *failure_tree = &inode->io_failure_tree; struct io_failure_record *failrec; - struct extent_state *state, *next; + struct rb_node *node, *next; - if (RB_EMPTY_ROOT(&failure_tree->state)) + if (RB_EMPTY_ROOT(&inode->io_failure_tree)) return; - spin_lock(&failure_tree->lock); - state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY); - while (state) { - if (state->start > end) + spin_lock(&inode->io_failure_lock); + node = rb_simple_search_first(&inode->io_failure_tree, start); + while (node) { + failrec = rb_entry(node, struct io_failure_record, rb_node); + if (failrec->bytenr > end) break; - ASSERT(state->end <= end); - - next = next_state(state); - - failrec = state->failrec; - free_extent_state(state); + next = rb_next(node); + rb_erase(&failrec->rb_node, &inode->io_failure_tree); kfree(failrec); - state = next; + node = next; } - spin_unlock(&failure_tree->lock); + spin_unlock(&inode->io_failure_lock); } static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode, @@ -2545,16 +734,14 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); u64 start = bbio->file_offset + bio_offset; struct io_failure_record *failrec; - struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; - struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; const u32 sectorsize = fs_info->sectorsize; int ret; - failrec = get_state_failrec(failure_tree, start); + failrec = get_failrec(BTRFS_I(inode), start); if (!IS_ERR(failrec)) { btrfs_debug(fs_info, "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu", - failrec->logical, failrec->start, failrec->len); + failrec->logical, failrec->bytenr, failrec->len); /* * when data can be on disk more than twice, add to failrec here * (e.g. with a list for failed_mirror) to make @@ -2569,7 +756,8 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode if (!failrec) return ERR_PTR(-ENOMEM); - failrec->start = start; + RB_CLEAR_NODE(&failrec->rb_node); + failrec->bytenr = start; failrec->len = sectorsize; failrec->failed_mirror = bbio->mirror_num; failrec->this_mirror = bbio->mirror_num; @@ -2594,14 +782,8 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode } /* Set the bits in the private failure tree */ - ret = set_extent_bits(failure_tree, start, start + sectorsize - 1, - EXTENT_LOCKED | EXTENT_DIRTY); - if (ret >= 0) { - ret = set_state_failrec(failure_tree, start, failrec); - /* Set the bits in the inode's tree */ - ret = set_extent_bits(tree, start, start + sectorsize - 1, - EXTENT_DAMAGED); - } else if (ret < 0) { + ret = insert_failrec(BTRFS_I(inode), failrec); + if (ret) { kfree(failrec); return ERR_PTR(ret); } @@ -2616,8 +798,6 @@ int btrfs_repair_one_sector(struct inode *inode, struct btrfs_bio *failed_bbio, u64 start = failed_bbio->file_offset + bio_offset; struct io_failure_record *failrec; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; - struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; struct bio *failed_bio = &failed_bbio->bio; const int icsum = bio_offset >> fs_info->sectorsize_bits; struct bio *repair_bio; @@ -2646,17 +826,15 @@ int btrfs_repair_one_sector(struct inode *inode, struct btrfs_bio *failed_bbio, btrfs_debug(fs_info, "failed to repair num_copies %d this_mirror %d failed_mirror %d", failrec->num_copies, failrec->this_mirror, failrec->failed_mirror); - free_io_failure(failure_tree, tree, failrec); + free_io_failure(BTRFS_I(inode), failrec); return -EIO; } - repair_bio = btrfs_bio_alloc(1); + repair_bio = btrfs_bio_alloc(1, REQ_OP_READ, failed_bbio->end_io, + failed_bbio->private); repair_bbio = btrfs_bio(repair_bio); repair_bbio->file_offset = start; - repair_bio->bi_opf = REQ_OP_READ; - repair_bio->bi_end_io = failed_bio->bi_end_io; repair_bio->bi_iter.bi_sector = failrec->logical >> 9; - repair_bio->bi_private = failed_bio->bi_private; if (failed_bbio->csum) { const u32 csum_size = fs_info->csum_size; @@ -2720,8 +898,8 @@ static void end_sector_io(struct page *page, u64 offset, bool uptodate) if (uptodate) set_extent_uptodate(&inode->io_tree, offset, offset + sectorsize - 1, &cached, GFP_ATOMIC); - unlock_extent_cached_atomic(&inode->io_tree, offset, - offset + sectorsize - 1, &cached); + unlock_extent_atomic(&inode->io_tree, offset, offset + sectorsize - 1, + &cached); } static void submit_data_read_repair(struct inode *inode, @@ -2823,8 +1001,9 @@ void end_extent_writepage(struct page *page, int err, u64 start, u64 end) * Scheduling is not allowed, so the extent state tree is expected * to have one and only one object corresponding to this IO. */ -static void end_bio_extent_writepage(struct bio *bio) +static void end_bio_extent_writepage(struct btrfs_bio *bbio) { + struct bio *bio = &bbio->bio; int error = blk_status_to_errno(bio->bi_status); struct bio_vec *bvec; u64 start; @@ -2924,11 +1103,7 @@ static void endio_readpage_release_extent(struct processed_extent *processed, * Now we don't have range contiguous to the processed range, release * the processed range now. */ - if (processed->uptodate && tree->track_uptodate) - set_extent_uptodate(tree, processed->start, processed->end, - &cached, GFP_ATOMIC); - unlock_extent_cached_atomic(tree, processed->start, processed->end, - &cached); + unlock_extent_atomic(tree, processed->start, processed->end, &cached); update: /* Update processed to current range */ @@ -2988,11 +1163,10 @@ static struct extent_buffer *find_extent_buffer_readpage( * Scheduling is not allowed, so the extent state tree is expected * to have one and only one object corresponding to this IO. */ -static void end_bio_extent_readpage(struct bio *bio) +static void end_bio_extent_readpage(struct btrfs_bio *bbio) { + struct bio *bio = &bbio->bio; struct bio_vec *bvec; - struct btrfs_bio *bbio = btrfs_bio(bio); - struct extent_io_tree *tree, *failure_tree; struct processed_extent processed = { 0 }; /* * The offset to the beginning of a bio, since one bio can never be @@ -3019,8 +1193,6 @@ static void end_bio_extent_readpage(struct bio *bio) "end_bio_extent_readpage: bi_sector=%llu, err=%d, mirror=%u", bio->bi_iter.bi_sector, bio->bi_status, bbio->mirror_num); - tree = &BTRFS_I(inode)->io_tree; - failure_tree = &BTRFS_I(inode)->io_failure_tree; /* * We always issue full-sector reads, but if some block in a @@ -3061,9 +1233,7 @@ static void end_bio_extent_readpage(struct bio *bio) loff_t i_size = i_size_read(inode); pgoff_t end_index = i_size >> PAGE_SHIFT; - clean_io_failure(BTRFS_I(inode)->root->fs_info, - failure_tree, tree, start, page, - btrfs_ino(BTRFS_I(inode)), 0); + btrfs_clean_io_failure(BTRFS_I(inode), start, page, 0); /* * Zero out the remaining part if this range straddles @@ -3155,55 +1325,11 @@ int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array) * if it could not bulk-allocate. So we must be out of memory. */ if (allocated == last) - return -ENOMEM; - - memalloc_retry_wait(GFP_NOFS); - } - return 0; -} - -/* - * Initialize the members up to but not including 'bio'. Use after allocating a - * new bio by bio_alloc_bioset as it does not initialize the bytes outside of - * 'bio' because use of __GFP_ZERO is not supported. - */ -static inline void btrfs_bio_init(struct btrfs_bio *bbio) -{ - memset(bbio, 0, offsetof(struct btrfs_bio, bio)); -} - -/* - * Allocate a btrfs_io_bio, with @nr_iovecs as maximum number of iovecs. - * - * The bio allocation is backed by bioset and does not fail. - */ -struct bio *btrfs_bio_alloc(unsigned int nr_iovecs) -{ - struct bio *bio; - - ASSERT(0 < nr_iovecs && nr_iovecs <= BIO_MAX_VECS); - bio = bio_alloc_bioset(NULL, nr_iovecs, 0, GFP_NOFS, &btrfs_bioset); - btrfs_bio_init(btrfs_bio(bio)); - return bio; -} - -struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size) -{ - struct bio *bio; - struct btrfs_bio *bbio; - - ASSERT(offset <= UINT_MAX && size <= UINT_MAX); - - /* this will never fail when it's backed by a bioset */ - bio = bio_alloc_clone(orig->bi_bdev, orig, GFP_NOFS, &btrfs_bioset); - ASSERT(bio); - - bbio = btrfs_bio(bio); - btrfs_bio_init(bbio); + return -ENOMEM; - bio_trim(bio, offset >> 9, size >> 9); - bbio->iter = bio->bi_iter; - return bio; + memalloc_retry_wait(GFP_NOFS); + } + return 0; } /** @@ -3233,7 +1359,7 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, u32 bio_size = bio->bi_iter.bi_size; u32 real_size; const sector_t sector = disk_bytenr >> SECTOR_SHIFT; - bool contig; + bool contig = false; int ret; ASSERT(bio); @@ -3242,10 +1368,35 @@ static int btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, if (bio_ctrl->compress_type != compress_type) return 0; - if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) + + if (bio->bi_iter.bi_size == 0) { + /* We can always add a page into an empty bio. */ + contig = true; + } else if (bio_ctrl->compress_type == BTRFS_COMPRESS_NONE) { + struct bio_vec *bvec = bio_last_bvec_all(bio); + + /* + * The contig check requires the following conditions to be met: + * 1) The pages are belonging to the same inode + * This is implied by the call chain. + * + * 2) The range has adjacent logical bytenr + * + * 3) The range has adjacent file offset + * This is required for the usage of btrfs_bio->file_offset. + */ + if (bio_end_sector(bio) == sector && + page_offset(bvec->bv_page) + bvec->bv_offset + + bvec->bv_len == page_offset(page) + pg_offset) + contig = true; + } else { + /* + * For compression, all IO should have its logical bytenr + * set to the starting bytenr of the compressed extent. + */ contig = bio->bi_iter.bi_sector == sector; - else - contig = bio_end_sector(bio) == sector; + } + if (!contig) return 0; @@ -3326,7 +1477,6 @@ static int alloc_new_bio(struct btrfs_inode *inode, struct btrfs_bio_ctrl *bio_ctrl, struct writeback_control *wbc, blk_opf_t opf, - bio_end_io_t end_io_func, u64 disk_bytenr, u32 offset, u64 file_offset, enum btrfs_compression_type compress_type) { @@ -3334,7 +1484,9 @@ static int alloc_new_bio(struct btrfs_inode *inode, struct bio *bio; int ret; - bio = btrfs_bio_alloc(BIO_MAX_VECS); + ASSERT(bio_ctrl->end_io_func); + + bio = btrfs_bio_alloc(BIO_MAX_VECS, opf, bio_ctrl->end_io_func, NULL); /* * For compressed page range, its disk_bytenr is always @disk_bytenr * passed in, no matter if we have added any range into previous bio. @@ -3345,8 +1497,6 @@ static int alloc_new_bio(struct btrfs_inode *inode, bio->bi_iter.bi_sector = (disk_bytenr + offset) >> SECTOR_SHIFT; bio_ctrl->bio = bio; bio_ctrl->compress_type = compress_type; - bio->bi_end_io = end_io_func; - bio->bi_opf = opf; ret = calc_bio_boundaries(bio_ctrl, inode, file_offset); if (ret < 0) goto error; @@ -3385,31 +1535,30 @@ static int alloc_new_bio(struct btrfs_inode *inode, return 0; error: bio_ctrl->bio = NULL; - bio->bi_status = errno_to_blk_status(ret); - bio_endio(bio); + btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret)); return ret; } /* * @opf: bio REQ_OP_* and REQ_* flags as one value * @wbc: optional writeback control for io accounting - * @page: page to add to the bio * @disk_bytenr: logical bytenr where the write will be + * @page: page to add to the bio * @size: portion of page that we want to write to * @pg_offset: offset of the new bio or to check whether we are adding * a contiguous page to the previous one - * @bio_ret: must be valid pointer, newly allocated bio will be stored there - * @end_io_func: end_io callback for new bio - * @mirror_num: desired mirror to read/write - * @prev_bio_flags: flags of previous bio to see if we can merge the current one * @compress_type: compress type for current bio + * + * The will either add the page into the existing @bio_ctrl->bio, or allocate a + * new one in @bio_ctrl->bio. + * The mirror number for this IO should already be initizlied in + * @bio_ctrl->mirror_num. */ static int submit_extent_page(blk_opf_t opf, struct writeback_control *wbc, struct btrfs_bio_ctrl *bio_ctrl, - struct page *page, u64 disk_bytenr, + u64 disk_bytenr, struct page *page, size_t size, unsigned long pg_offset, - bio_end_io_t end_io_func, enum btrfs_compression_type compress_type, bool force_bio_submit) { @@ -3421,6 +1570,9 @@ static int submit_extent_page(blk_opf_t opf, ASSERT(pg_offset < PAGE_SIZE && size <= PAGE_SIZE && pg_offset + size <= PAGE_SIZE); + + ASSERT(bio_ctrl->end_io_func); + if (force_bio_submit) submit_one_bio(bio_ctrl); @@ -3431,7 +1583,7 @@ static int submit_extent_page(blk_opf_t opf, /* Allocate new bio if needed */ if (!bio_ctrl->bio) { ret = alloc_new_bio(inode, bio_ctrl, wbc, opf, - end_io_func, disk_bytenr, offset, + disk_bytenr, offset, page_offset(page) + cur, compress_type); if (ret < 0) @@ -3588,7 +1740,6 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, u64 extent_offset; u64 last_byte = i_size_read(inode); u64 block_start; - u64 cur_end; struct extent_map *em; int ret = 0; size_t pg_offset = 0; @@ -3598,7 +1749,7 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, ret = set_page_extent_mapped(page); if (ret < 0) { - unlock_extent(tree, start, end); + unlock_extent(tree, start, end, NULL); btrfs_page_set_error(fs_info, page, start, PAGE_SIZE); unlock_page(page); goto out; @@ -3612,6 +1763,7 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, memzero_page(page, zero_offset, iosize); } } + bio_ctrl->end_io_func = end_bio_extent_readpage; begin_page_read(fs_info, page); while (cur <= end) { unsigned long this_bio_flag = 0; @@ -3626,15 +1778,14 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, memzero_page(page, pg_offset, iosize); set_extent_uptodate(tree, cur, cur + iosize - 1, &cached, GFP_NOFS); - unlock_extent_cached(tree, cur, - cur + iosize - 1, &cached); + unlock_extent(tree, cur, cur + iosize - 1, &cached); end_page_read(page, true, cur, iosize); break; } em = __get_extent_map(inode, page, pg_offset, cur, end - cur + 1, em_cached); if (IS_ERR(em)) { - unlock_extent(tree, cur, end); + unlock_extent(tree, cur, end, NULL); end_page_read(page, false, cur, end + 1 - cur); ret = PTR_ERR(em); break; @@ -3647,7 +1798,6 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, this_bio_flag = em->compress_type; iosize = min(extent_map_end(em) - cur, end - cur + 1); - cur_end = min(extent_map_end(em) - 1, end); iosize = ALIGN(iosize, blocksize); if (this_bio_flag != BTRFS_COMPRESS_NONE) disk_bytenr = em->block_start; @@ -3710,43 +1860,31 @@ static int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, set_extent_uptodate(tree, cur, cur + iosize - 1, &cached, GFP_NOFS); - unlock_extent_cached(tree, cur, - cur + iosize - 1, &cached); + unlock_extent(tree, cur, cur + iosize - 1, &cached); end_page_read(page, true, cur, iosize); cur = cur + iosize; pg_offset += iosize; continue; } /* the get_extent function already copied into the page */ - if (test_range_bit(tree, cur, cur_end, - EXTENT_UPTODATE, 1, NULL)) { - unlock_extent(tree, cur, cur + iosize - 1); - end_page_read(page, true, cur, iosize); - cur = cur + iosize; - pg_offset += iosize; - continue; - } - /* we have an inline extent but it didn't get marked up - * to date. Error out - */ if (block_start == EXTENT_MAP_INLINE) { - unlock_extent(tree, cur, cur + iosize - 1); - end_page_read(page, false, cur, iosize); + unlock_extent(tree, cur, cur + iosize - 1, NULL); + end_page_read(page, true, cur, iosize); cur = cur + iosize; pg_offset += iosize; continue; } ret = submit_extent_page(REQ_OP_READ | read_flags, NULL, - bio_ctrl, page, disk_bytenr, iosize, - pg_offset, end_bio_extent_readpage, - this_bio_flag, force_bio_submit); + bio_ctrl, disk_bytenr, page, iosize, + pg_offset, this_bio_flag, + force_bio_submit); if (ret) { /* * We have to unlock the remaining range, or the page * will never be unlocked. */ - unlock_extent(tree, cur, end); + unlock_extent(tree, cur, end, NULL); end_page_read(page, false, cur, end + 1 - cur); goto out; } @@ -3959,6 +2097,7 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, */ wbc->nr_to_write--; + epd->bio_ctrl.end_io_func = end_bio_extent_writepage; while (cur <= end) { u64 disk_bytenr; u64 em_end; @@ -4052,10 +2191,9 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, btrfs_page_clear_dirty(fs_info, page, cur, iosize); ret = submit_extent_page(op | write_flags, wbc, - &epd->bio_ctrl, page, - disk_bytenr, iosize, + &epd->bio_ctrl, disk_bytenr, + page, iosize, cur - page_offset(page), - end_bio_extent_writepage, 0, false); if (ret) { has_error = true; @@ -4406,8 +2544,9 @@ static struct extent_buffer *find_extent_buffer_nolock( * Unlike end_bio_extent_buffer_writepage(), we only call end_page_writeback() * after all extent buffers in the page has finished their writeback. */ -static void end_bio_subpage_eb_writepage(struct bio *bio) +static void end_bio_subpage_eb_writepage(struct btrfs_bio *bbio) { + struct bio *bio = &bbio->bio; struct btrfs_fs_info *fs_info; struct bio_vec *bvec; struct bvec_iter_all iter_all; @@ -4463,8 +2602,9 @@ static void end_bio_subpage_eb_writepage(struct bio *bio) bio_put(bio); } -static void end_bio_extent_buffer_writepage(struct bio *bio) +static void end_bio_extent_buffer_writepage(struct btrfs_bio *bbio) { + struct bio *bio = &bbio->bio; struct bio_vec *bvec; struct extent_buffer *eb; int done; @@ -4546,10 +2686,11 @@ static int write_one_subpage_eb(struct extent_buffer *eb, if (no_dirty_ebs) clear_page_dirty_for_io(page); + epd->bio_ctrl.end_io_func = end_bio_subpage_eb_writepage; + ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - &epd->bio_ctrl, page, eb->start, eb->len, - eb->start - page_offset(page), - end_bio_subpage_eb_writepage, 0, false); + &epd->bio_ctrl, eb->start, page, eb->len, + eb->start - page_offset(page), 0, false); if (ret) { btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len); set_btree_ioerr(page, eb); @@ -4580,6 +2721,8 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, prepare_eb_write(eb); + epd->bio_ctrl.end_io_func = end_bio_extent_buffer_writepage; + num_pages = num_extent_pages(eb); for (i = 0; i < num_pages; i++) { struct page *p = eb->pages[i]; @@ -4587,10 +2730,8 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, clear_page_dirty_for_io(p); set_page_writeback(p); ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - &epd->bio_ctrl, p, disk_bytenr, - PAGE_SIZE, 0, - end_bio_extent_buffer_writepage, - 0, false); + &epd->bio_ctrl, disk_bytenr, p, + PAGE_SIZE, 0, 0, false); if (ret) { set_btree_ioerr(p, eb); if (PageWriteback(p)) @@ -5211,7 +3352,7 @@ int extent_invalidate_folio(struct extent_io_tree *tree, if (start > end) return 0; - lock_extent_bits(tree, start, end, &cached_state); + lock_extent(tree, start, end, &cached_state); folio_wait_writeback(folio); /* @@ -5219,7 +3360,7 @@ int extent_invalidate_folio(struct extent_io_tree *tree, * so here we only need to unlock the extent range to free any * existing extent state. */ - unlock_extent_cached(tree, start, end, &cached_state); + unlock_extent(tree, start, end, &cached_state); return 0; } @@ -5238,15 +3379,17 @@ static int try_release_extent_state(struct extent_io_tree *tree, if (test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL)) { ret = 0; } else { + u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM | + EXTENT_DELALLOC_NEW | EXTENT_CTLBITS); + /* * At this point we can safely clear everything except the * locked bit, the nodatasum bit and the delalloc new bit. * The delalloc new bit will be cleared by ordered extent * completion. */ - ret = __clear_extent_bit(tree, start, end, - ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW), - 0, 0, NULL, mask, NULL); + ret = __clear_extent_bit(tree, start, end, clear_bits, NULL, + mask, NULL); /* if clear_extent_bit failed for enomem reasons, * we can't allow the release to continue. @@ -5344,42 +3487,6 @@ next: return try_release_extent_state(tree, page, mask); } -/* - * helper function for fiemap, which doesn't want to see any holes. - * This maps until we find something past 'last' - */ -static struct extent_map *get_extent_skip_holes(struct btrfs_inode *inode, - u64 offset, u64 last) -{ - u64 sectorsize = btrfs_inode_sectorsize(inode); - struct extent_map *em; - u64 len; - - if (offset >= last) - return NULL; - - while (1) { - len = last - offset; - if (len == 0) - break; - len = ALIGN(len, sectorsize); - em = btrfs_get_extent_fiemap(inode, offset, len); - if (IS_ERR(em)) - return em; - - /* if this isn't a hole return it */ - if (em->block_start != EXTENT_MAP_HOLE) - return em; - - /* this is a hole, advance to the next extent */ - offset = extent_map_end(em); - free_extent_map(em); - if (offset >= last) - break; - } - return NULL; -} - /* * To cache previous fiemap extent * @@ -5409,6 +3516,9 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, { int ret = 0; + /* Set at the end of extent_fiemap(). */ + ASSERT((flags & FIEMAP_EXTENT_LAST) == 0); + if (!cache->cached) goto assign; @@ -5432,16 +3542,13 @@ static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, * So truly compressed (physical size smaller than logical size) * extents won't get merged with each other * - * 3) Share same flags except FIEMAP_EXTENT_LAST - * So regular extent won't get merged with prealloc extent + * 3) Share same flags */ if (cache->offset + cache->len == offset && cache->phys + cache->len == phys && - (cache->flags & ~FIEMAP_EXTENT_LAST) == - (flags & ~FIEMAP_EXTENT_LAST)) { + cache->flags == flags) { cache->len += len; - cache->flags |= flags; - goto try_submit_last; + return 0; } /* Not mergeable, need to submit cached one */ @@ -5456,13 +3563,8 @@ assign: cache->phys = phys; cache->len = len; cache->flags = flags; -try_submit_last: - if (cache->flags & FIEMAP_EXTENT_LAST) { - ret = fiemap_fill_next_extent(fieinfo, cache->offset, - cache->phys, cache->len, cache->flags); - cache->cached = false; - } - return ret; + + return 0; } /* @@ -5492,215 +3594,534 @@ static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo, return ret; } -int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, - u64 start, u64 len) +static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path) { - int ret = 0; - u64 off; - u64 max = start + len; - u32 flags = 0; - u32 found_type; - u64 last; - u64 last_for_get_extent = 0; - u64 disko = 0; - u64 isize = i_size_read(&inode->vfs_inode); - struct btrfs_key found_key; - struct extent_map *em = NULL; - struct extent_state *cached_state = NULL; - struct btrfs_path *path; - struct btrfs_root *root = inode->root; - struct fiemap_cache cache = { 0 }; - struct ulist *roots; - struct ulist *tmp_ulist; - int end = 0; - u64 em_start = 0; - u64 em_len = 0; - u64 em_end = 0; + struct extent_buffer *clone; + struct btrfs_key key; + int slot; + int ret; - if (len == 0) - return -EINVAL; + path->slots[0]++; + if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) + return 0; - path = btrfs_alloc_path(); - if (!path) + ret = btrfs_next_leaf(inode->root, path); + if (ret != 0) + return ret; + + /* + * Don't bother with cloning if there are no more file extent items for + * our inode. + */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY) + return 1; + + /* See the comment at fiemap_search_slot() about why we clone. */ + clone = btrfs_clone_extent_buffer(path->nodes[0]); + if (!clone) return -ENOMEM; - roots = ulist_alloc(GFP_KERNEL); - tmp_ulist = ulist_alloc(GFP_KERNEL); - if (!roots || !tmp_ulist) { - ret = -ENOMEM; - goto out_free_ulist; + slot = path->slots[0]; + btrfs_release_path(path); + path->nodes[0] = clone; + path->slots[0] = slot; + + return 0; +} + +/* + * Search for the first file extent item that starts at a given file offset or + * the one that starts immediately before that offset. + * Returns: 0 on success, < 0 on error, 1 if not found. + */ +static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path, + u64 file_offset) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct extent_buffer *clone; + struct btrfs_key key; + int slot; + int ret; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = file_offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret != 0) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + return 1; } /* - * We can't initialize that to 'start' as this could miss extents due - * to extent item merging + * We clone the leaf and use it during fiemap. This is because while + * using the leaf we do expensive things like checking if an extent is + * shared, which can take a long time. In order to prevent blocking + * other tasks for too long, we use a clone of the leaf. We have locked + * the file range in the inode's io tree, so we know none of our file + * extent items can change. This way we avoid blocking other tasks that + * want to insert items for other inodes in the same leaf or b+tree + * rebalance operations (triggered for example when someone is trying + * to push items into this leaf when trying to insert an item in a + * neighbour leaf). + * We also need the private clone because holding a read lock on an + * extent buffer of the subvolume's b+tree will make lockdep unhappy + * when we call fiemap_fill_next_extent(), because that may cause a page + * fault when filling the user space buffer with fiemap data. */ - off = 0; - start = round_down(start, btrfs_inode_sectorsize(inode)); - len = round_up(max, btrfs_inode_sectorsize(inode)) - start; + clone = btrfs_clone_extent_buffer(path->nodes[0]); + if (!clone) + return -ENOMEM; + + slot = path->slots[0]; + btrfs_release_path(path); + path->nodes[0] = clone; + path->slots[0] = slot; + + return 0; +} + +/* + * Process a range which is a hole or a prealloc extent in the inode's subvolume + * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc + * extent. The end offset (@end) is inclusive. + */ +static int fiemap_process_hole(struct btrfs_inode *inode, + struct fiemap_extent_info *fieinfo, + struct fiemap_cache *cache, + struct btrfs_backref_shared_cache *backref_cache, + u64 disk_bytenr, u64 extent_offset, + u64 extent_gen, + struct ulist *roots, struct ulist *tmp_ulist, + u64 start, u64 end) +{ + const u64 i_size = i_size_read(&inode->vfs_inode); + const u64 ino = btrfs_ino(inode); + u64 cur_offset = start; + u64 last_delalloc_end = 0; + u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN; + bool checked_extent_shared = false; + int ret; /* - * lookup the last file extent. We're not using i_size here - * because there might be preallocation past i_size + * There can be no delalloc past i_size, so don't waste time looking for + * it beyond i_size. */ - ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1, - 0); - if (ret < 0) { - goto out_free_ulist; - } else { - WARN_ON(!ret); - if (ret == 1) - ret = 0; - } + while (cur_offset < end && cur_offset < i_size) { + u64 delalloc_start; + u64 delalloc_end; + u64 prealloc_start; + u64 prealloc_len = 0; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end, + &delalloc_start, + &delalloc_end); + if (!delalloc) + break; - path->slots[0]--; - btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); - found_type = found_key.type; - - /* No extents, but there might be delalloc bits */ - if (found_key.objectid != btrfs_ino(inode) || - found_type != BTRFS_EXTENT_DATA_KEY) { - /* have to trust i_size as the end */ - last = (u64)-1; - last_for_get_extent = isize; - } else { /* - * remember the start of the last extent. There are a - * bunch of different factors that go into the length of the - * extent, so its much less complex to remember where it started + * If this is a prealloc extent we have to report every section + * of it that has no delalloc. */ - last = found_key.offset; - last_for_get_extent = last + 1; + if (disk_bytenr != 0) { + if (last_delalloc_end == 0) { + prealloc_start = start; + prealloc_len = delalloc_start - start; + } else { + prealloc_start = last_delalloc_end + 1; + prealloc_len = delalloc_start - prealloc_start; + } + } + + if (prealloc_len > 0) { + if (!checked_extent_shared && fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(inode->root, + ino, disk_bytenr, + extent_gen, roots, + tmp_ulist, + backref_cache); + if (ret < 0) + return ret; + else if (ret > 0) + prealloc_flags |= FIEMAP_EXTENT_SHARED; + + checked_extent_shared = true; + } + ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, + disk_bytenr + extent_offset, + prealloc_len, prealloc_flags); + if (ret) + return ret; + extent_offset += prealloc_len; + } + + ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0, + delalloc_end + 1 - delalloc_start, + FIEMAP_EXTENT_DELALLOC | + FIEMAP_EXTENT_UNKNOWN); + if (ret) + return ret; + + last_delalloc_end = delalloc_end; + cur_offset = delalloc_end + 1; + extent_offset += cur_offset - delalloc_start; + cond_resched(); } - btrfs_release_path(path); /* - * we might have some extents allocated but more delalloc past those - * extents. so, we trust isize unless the start of the last extent is - * beyond isize + * Either we found no delalloc for the whole prealloc extent or we have + * a prealloc extent that spans i_size or starts at or after i_size. */ - if (last < isize) { - last = (u64)-1; - last_for_get_extent = isize; + if (disk_bytenr != 0 && last_delalloc_end < end) { + u64 prealloc_start; + u64 prealloc_len; + + if (last_delalloc_end == 0) { + prealloc_start = start; + prealloc_len = end + 1 - start; + } else { + prealloc_start = last_delalloc_end + 1; + prealloc_len = end + 1 - prealloc_start; + } + + if (!checked_extent_shared && fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(inode->root, + ino, disk_bytenr, + extent_gen, roots, + tmp_ulist, + backref_cache); + if (ret < 0) + return ret; + else if (ret > 0) + prealloc_flags |= FIEMAP_EXTENT_SHARED; + } + ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, + disk_bytenr + extent_offset, + prealloc_len, prealloc_flags); + if (ret) + return ret; } - lock_extent_bits(&inode->io_tree, start, start + len - 1, - &cached_state); + return 0; +} + +static int fiemap_find_last_extent_offset(struct btrfs_inode *inode, + struct btrfs_path *path, + u64 *last_extent_end_ret) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *ei; + struct btrfs_key key; + u64 disk_bytenr; + int ret; - em = get_extent_skip_holes(inode, start, last_for_get_extent); - if (!em) - goto out; - if (IS_ERR(em)) { - ret = PTR_ERR(em); + /* + * Lookup the last file extent. We're not using i_size here because + * there might be preallocation past i_size. + */ + ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0); + /* There can't be a file extent item at offset (u64)-1 */ + ASSERT(ret != 0); + if (ret < 0) + return ret; + + /* + * For a non-existing key, btrfs_search_slot() always leaves us at a + * slot > 0, except if the btree is empty, which is impossible because + * at least it has the inode item for this inode and all the items for + * the root inode 256. + */ + ASSERT(path->slots[0] > 0); + path->slots[0]--; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { + /* No file extent items in the subvolume tree. */ + *last_extent_end_ret = 0; + return 0; + } + + /* + * For an inline extent, the disk_bytenr is where inline data starts at, + * so first check if we have an inline extent item before checking if we + * have an implicit hole (disk_bytenr == 0). + */ + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) { + *last_extent_end_ret = btrfs_file_extent_end(path); + return 0; + } + + /* + * Find the last file extent item that is not a hole (when NO_HOLES is + * not enabled). This should take at most 2 iterations in the worst + * case: we have one hole file extent item at slot 0 of a leaf and + * another hole file extent item as the last item in the previous leaf. + * This is because we merge file extent items that represent holes. + */ + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + while (disk_bytenr == 0) { + ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY); + if (ret < 0) { + return ret; + } else if (ret > 0) { + /* No file extent items that are not holes. */ + *last_extent_end_ret = 0; + return 0; + } + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + } + + *last_extent_end_ret = btrfs_file_extent_end(path); + return 0; +} + +int extent_fiemap(struct btrfs_inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + const u64 ino = btrfs_ino(inode); + struct extent_state *cached_state = NULL; + struct btrfs_path *path; + struct btrfs_root *root = inode->root; + struct fiemap_cache cache = { 0 }; + struct btrfs_backref_shared_cache *backref_cache; + struct ulist *roots; + struct ulist *tmp_ulist; + u64 last_extent_end; + u64 prev_extent_end; + u64 lockstart; + u64 lockend; + bool stopped = false; + int ret; + + backref_cache = kzalloc(sizeof(*backref_cache), GFP_KERNEL); + path = btrfs_alloc_path(); + roots = ulist_alloc(GFP_KERNEL); + tmp_ulist = ulist_alloc(GFP_KERNEL); + if (!backref_cache || !path || !roots || !tmp_ulist) { + ret = -ENOMEM; goto out; } - while (!end) { - u64 offset_in_extent = 0; + lockstart = round_down(start, root->fs_info->sectorsize); + lockend = round_up(start + len, root->fs_info->sectorsize); + prev_extent_end = lockstart; - /* break if the extent we found is outside the range */ - if (em->start >= max || extent_map_end(em) < off) - break; + lock_extent(&inode->io_tree, lockstart, lockend, &cached_state); - /* - * get_extent may return an extent that starts before our - * requested range. We have to make sure the ranges - * we return to fiemap always move forward and don't - * overlap, so adjust the offsets here - */ - em_start = max(em->start, off); + ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); + if (ret < 0) + goto out_unlock; + btrfs_release_path(path); + path->reada = READA_FORWARD; + ret = fiemap_search_slot(inode, path, lockstart); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { /* - * record the offset from the start of the extent - * for adjusting the disk offset below. Only do this if the - * extent isn't compressed since our in ram offset may be past - * what we have actually allocated on disk. + * No file extent item found, but we may have delalloc between + * the current offset and i_size. So check for that. */ - if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) - offset_in_extent = em_start - em->start; - em_end = extent_map_end(em); - em_len = em_end - em_start; - flags = 0; - if (em->block_start < EXTENT_MAP_LAST_BYTE) - disko = em->block_start + offset_in_extent; - else - disko = 0; + ret = 0; + goto check_eof_delalloc; + } + + while (prev_extent_end < lockend) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_file_extent_item *ei; + struct btrfs_key key; + u64 extent_end; + u64 extent_len; + u64 extent_offset = 0; + u64 extent_gen; + u64 disk_bytenr = 0; + u64 flags = 0; + int extent_type; + u8 compression; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + break; + + extent_end = btrfs_file_extent_end(path); /* - * bump off for our next call to get_extent + * The first iteration can leave us at an extent item that ends + * before our range's start. Move to the next item. */ - off = extent_map_end(em); - if (off >= max) - end = 1; - - if (em->block_start == EXTENT_MAP_LAST_BYTE) { - end = 1; - flags |= FIEMAP_EXTENT_LAST; - } else if (em->block_start == EXTENT_MAP_INLINE) { - flags |= (FIEMAP_EXTENT_DATA_INLINE | - FIEMAP_EXTENT_NOT_ALIGNED); - } else if (em->block_start == EXTENT_MAP_DELALLOC) { - flags |= (FIEMAP_EXTENT_DELALLOC | - FIEMAP_EXTENT_UNKNOWN); - } else if (fieinfo->fi_extents_max) { - u64 bytenr = em->block_start - - (em->start - em->orig_start); + if (extent_end <= lockstart) + goto next_item; - /* - * As btrfs supports shared space, this information - * can be exported to userspace tools via - * flag FIEMAP_EXTENT_SHARED. If fi_extents_max == 0 - * then we're just getting a count and we can skip the - * lookup stuff. - */ - ret = btrfs_check_shared(root, btrfs_ino(inode), - bytenr, roots, tmp_ulist); - if (ret < 0) - goto out_free; - if (ret) - flags |= FIEMAP_EXTENT_SHARED; - ret = 0; + /* We have in implicit hole (NO_HOLES feature enabled). */ + if (prev_extent_end < key.offset) { + const u64 range_end = min(key.offset, lockend) - 1; + + ret = fiemap_process_hole(inode, fieinfo, &cache, + backref_cache, 0, 0, 0, + roots, tmp_ulist, + prev_extent_end, range_end); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* fiemap_fill_next_extent() told us to stop. */ + stopped = true; + break; + } + + /* We've reached the end of the fiemap range, stop. */ + if (key.offset >= lockend) { + stopped = true; + break; + } } - if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) + + extent_len = extent_end - key.offset; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + compression = btrfs_file_extent_compression(leaf, ei); + extent_type = btrfs_file_extent_type(leaf, ei); + extent_gen = btrfs_file_extent_generation(leaf, ei); + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (compression == BTRFS_COMPRESS_NONE) + extent_offset = btrfs_file_extent_offset(leaf, ei); + } + + if (compression != BTRFS_COMPRESS_NONE) flags |= FIEMAP_EXTENT_ENCODED; - if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) - flags |= FIEMAP_EXTENT_UNWRITTEN; - free_extent_map(em); - em = NULL; - if ((em_start >= last) || em_len == (u64)-1 || - (last == (u64)-1 && isize <= em_end)) { - flags |= FIEMAP_EXTENT_LAST; - end = 1; + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + flags |= FIEMAP_EXTENT_DATA_INLINE; + flags |= FIEMAP_EXTENT_NOT_ALIGNED; + ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0, + extent_len, flags); + } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + ret = fiemap_process_hole(inode, fieinfo, &cache, + backref_cache, + disk_bytenr, extent_offset, + extent_gen, roots, tmp_ulist, + key.offset, extent_end - 1); + } else if (disk_bytenr == 0) { + /* We have an explicit hole. */ + ret = fiemap_process_hole(inode, fieinfo, &cache, + backref_cache, 0, 0, 0, + roots, tmp_ulist, + key.offset, extent_end - 1); + } else { + /* We have a regular extent. */ + if (fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(root, ino, + disk_bytenr, + extent_gen, + roots, + tmp_ulist, + backref_cache); + if (ret < 0) + goto out_unlock; + else if (ret > 0) + flags |= FIEMAP_EXTENT_SHARED; + } + + ret = emit_fiemap_extent(fieinfo, &cache, key.offset, + disk_bytenr + extent_offset, + extent_len, flags); } - /* now scan forward to see if this is really the last extent. */ - em = get_extent_skip_holes(inode, off, last_for_get_extent); - if (IS_ERR(em)) { - ret = PTR_ERR(em); - goto out; + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* fiemap_fill_next_extent() told us to stop. */ + stopped = true; + break; } - if (!em) { - flags |= FIEMAP_EXTENT_LAST; - end = 1; + + prev_extent_end = extent_end; +next_item: + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out_unlock; } - ret = emit_fiemap_extent(fieinfo, &cache, em_start, disko, - em_len, flags); - if (ret) { - if (ret == 1) - ret = 0; - goto out_free; + + ret = fiemap_next_leaf_item(inode, path); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* No more file extent items for this inode. */ + break; } + cond_resched(); } -out_free: - if (!ret) - ret = emit_last_fiemap_cache(fieinfo, &cache); - free_extent_map(em); -out: - unlock_extent_cached(&inode->io_tree, start, start + len - 1, - &cached_state); -out_free_ulist: +check_eof_delalloc: + /* + * Release (and free) the path before emitting any final entries to + * fiemap_fill_next_extent() to keep lockdep happy. This is because + * once we find no more file extent items exist, we may have a + * non-cloned leaf, and fiemap_fill_next_extent() can trigger page + * faults when copying data to the user space buffer. + */ + btrfs_free_path(path); + path = NULL; + + if (!stopped && prev_extent_end < lockend) { + ret = fiemap_process_hole(inode, fieinfo, &cache, backref_cache, + 0, 0, 0, roots, tmp_ulist, + prev_extent_end, lockend - 1); + if (ret < 0) + goto out_unlock; + prev_extent_end = lockend; + } + + if (cache.cached && cache.offset + cache.len >= last_extent_end) { + const u64 i_size = i_size_read(&inode->vfs_inode); + + if (prev_extent_end < i_size) { + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, + prev_extent_end, + i_size - 1, + &delalloc_start, + &delalloc_end); + if (!delalloc) + cache.flags |= FIEMAP_EXTENT_LAST; + } else { + cache.flags |= FIEMAP_EXTENT_LAST; + } + } + + ret = emit_last_fiemap_cache(fieinfo, &cache); + +out_unlock: + unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); +out: + kfree(backref_cache); btrfs_free_path(path); ulist_free(roots); ulist_free(tmp_ulist); @@ -5831,7 +4252,7 @@ static void btrfs_release_extent_buffer_pages(struct extent_buffer *eb) static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) { btrfs_release_extent_buffer_pages(eb); - btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); + btrfs_leak_debug_del_eb(eb); __free_extent_buffer(eb); } @@ -5848,8 +4269,7 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, eb->bflags = 0; init_rwsem(&eb->lock); - btrfs_leak_debug_add(&fs_info->eb_leak_lock, &eb->leak_list, - &fs_info->allocated_ebs); + btrfs_leak_debug_add_eb(eb); INIT_LIST_HEAD(&eb->release_list); spin_lock_init(&eb->refs_lock); @@ -6317,7 +4737,7 @@ static int release_extent_buffer(struct extent_buffer *eb) spin_unlock(&eb->refs_lock); } - btrfs_leak_debug_del(&eb->fs_info->eb_leak_lock, &eb->leak_list); + btrfs_leak_debug_del_eb(eb); /* Should be safe to release our pages at this point */ btrfs_release_extent_buffer_pages(eb); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS @@ -6337,18 +4757,16 @@ static int release_extent_buffer(struct extent_buffer *eb) void free_extent_buffer(struct extent_buffer *eb) { int refs; - int old; if (!eb) return; + refs = atomic_read(&eb->refs); while (1) { - refs = atomic_read(&eb->refs); if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3) || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs == 1)) break; - old = atomic_cmpxchg(&eb->refs, refs, refs - 1); - if (old == refs) + if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1)) return; } @@ -6544,7 +4962,7 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, if (!try_lock_extent(io_tree, eb->start, eb->start + eb->len - 1)) return -EAGAIN; } else { - ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1); + ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1, NULL); if (ret < 0) return ret; } @@ -6554,7 +4972,7 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, PageUptodate(page) || btrfs_subpage_test_uptodate(fs_info, page, eb->start, eb->len)) { set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); - unlock_extent(io_tree, eb->start, eb->start + eb->len - 1); + unlock_extent(io_tree, eb->start, eb->start + eb->len - 1, NULL); return ret; } @@ -6562,13 +4980,14 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, eb->read_mirror = 0; atomic_set(&eb->io_pages, 1); check_buffer_tree_ref(eb); + bio_ctrl.end_io_func = end_bio_extent_readpage; + btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len); btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len); ret = submit_extent_page(REQ_OP_READ, NULL, &bio_ctrl, - page, eb->start, eb->len, - eb->start - page_offset(page), - end_bio_extent_readpage, 0, true); + eb->start, page, eb->len, + eb->start - page_offset(page), 0, true); if (ret) { /* * In the endio function, if we hit something wrong we will @@ -6659,6 +5078,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) * set io_pages. See check_buffer_tree_ref for a more detailed comment. */ check_buffer_tree_ref(eb); + bio_ctrl.end_io_func = end_bio_extent_readpage; for (i = 0; i < num_pages; i++) { page = eb->pages[i]; @@ -6671,9 +5091,8 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) ClearPageError(page); err = submit_extent_page(REQ_OP_READ, NULL, - &bio_ctrl, page, page_offset(page), - PAGE_SIZE, 0, end_bio_extent_readpage, - 0, false); + &bio_ctrl, page_offset(page), page, + PAGE_SIZE, 0, 0, false); if (err) { /* * We failed to submit the bio so it's the diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 4bc72a87b9a99ce768f18f24d4b5ee0380590831..7929f054dda3ce2ddc048f6db5af23f61300575b 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -60,11 +60,13 @@ enum { struct btrfs_bio; struct btrfs_root; struct btrfs_inode; -struct btrfs_io_bio; struct btrfs_fs_info; struct io_failure_record; struct extent_io_tree; +int __init extent_buffer_init_cachep(void); +void __cold extent_buffer_free_cachep(void); + typedef void (submit_bio_hook_t)(struct inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type); @@ -240,10 +242,10 @@ void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end); void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, struct page *locked_page, u32 bits_to_clear, unsigned long page_ops); +int extent_invalidate_folio(struct extent_io_tree *tree, + struct folio *folio, size_t offset); int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array); -struct bio *btrfs_bio_alloc(unsigned int nr_iovecs); -struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size); void end_extent_writepage(struct page *page, int err, u64 start, u64 end); int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num); @@ -257,8 +259,12 @@ int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num); * bio end_io callback is called to indicate things have failed. */ struct io_failure_record { + /* Use rb_simple_node for search/insert */ + struct { + struct rb_node rb_node; + u64 bytenr; + }; struct page *page; - u64 start; u64 len; u64 logical; int this_mirror; @@ -269,6 +275,9 @@ struct io_failure_record { int btrfs_repair_one_sector(struct inode *inode, struct btrfs_bio *failed_bbio, u32 bio_offset, struct page *page, unsigned int pgoff, submit_bio_hook_t *submit_bio_hook); +void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end); +int btrfs_clean_io_failure(struct btrfs_inode *inode, u64 start, + struct page *page, unsigned int pg_offset); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index 6fee14ce2e6b1c7b902f376941836d5d843b9023..6092a4eedc923608509adc81fc062c7d1f447062 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -7,6 +7,7 @@ #include "volumes.h" #include "extent_map.h" #include "compression.h" +#include "btrfs_inode.h" static struct kmem_cache *extent_map_cache; @@ -54,9 +55,7 @@ struct extent_map *alloc_extent_map(void) if (!em) return NULL; RB_CLEAR_NODE(&em->rb_node); - em->flags = 0; em->compress_type = BTRFS_COMPRESS_NONE; - em->generation = 0; refcount_set(&em->refs, 1); INIT_LIST_HEAD(&em->list); return em; @@ -73,7 +72,6 @@ void free_extent_map(struct extent_map *em) { if (!em) return; - WARN_ON(refcount_read(&em->refs) == 0); if (refcount_dec_and_test(&em->refs)) { WARN_ON(extent_map_in_tree(em)); WARN_ON(!list_empty(&em->list)); @@ -143,8 +141,7 @@ static int tree_insert(struct rb_root_cached *root, struct extent_map *em) * it can't be found, try to find some neighboring extents */ static struct rb_node *__tree_search(struct rb_root *root, u64 offset, - struct rb_node **prev_ret, - struct rb_node **next_ret) + struct rb_node **prev_or_next_ret) { struct rb_node *n = root->rb_node; struct rb_node *prev = NULL; @@ -152,6 +149,8 @@ static struct rb_node *__tree_search(struct rb_root *root, u64 offset, struct extent_map *entry; struct extent_map *prev_entry = NULL; + ASSERT(prev_or_next_ret); + while (n) { entry = rb_entry(n, struct extent_map, rb_node); prev = n; @@ -165,24 +164,29 @@ static struct rb_node *__tree_search(struct rb_root *root, u64 offset, return n; } - if (prev_ret) { - orig_prev = prev; - while (prev && offset >= extent_map_end(prev_entry)) { - prev = rb_next(prev); - prev_entry = rb_entry(prev, struct extent_map, rb_node); - } - *prev_ret = prev; - prev = orig_prev; + orig_prev = prev; + while (prev && offset >= extent_map_end(prev_entry)) { + prev = rb_next(prev); + prev_entry = rb_entry(prev, struct extent_map, rb_node); + } + + /* + * Previous extent map found, return as in this case the caller does not + * care about the next one. + */ + if (prev) { + *prev_or_next_ret = prev; + return NULL; } - if (next_ret) { + prev = orig_prev; + prev_entry = rb_entry(prev, struct extent_map, rb_node); + while (prev && offset < prev_entry->start) { + prev = rb_prev(prev); prev_entry = rb_entry(prev, struct extent_map, rb_node); - while (prev && offset < prev_entry->start) { - prev = rb_prev(prev); - prev_entry = rb_entry(prev, struct extent_map, rb_node); - } - *next_ret = prev; } + *prev_or_next_ret = prev; + return NULL; } @@ -336,6 +340,8 @@ out: void clear_em_logging(struct extent_map_tree *tree, struct extent_map *em) { + lockdep_assert_held_write(&tree->lock); + clear_bit(EXTENT_FLAG_LOGGING, &em->flags); if (extent_map_in_tree(em)) try_merge_map(tree, em); @@ -382,7 +388,7 @@ static void extent_map_device_clear_bits(struct extent_map *em, unsigned bits) __clear_extent_bit(&device->alloc_state, stripe->physical, stripe->physical + stripe_size - 1, bits, - 0, 0, NULL, GFP_NOWAIT, NULL); + NULL, GFP_NOWAIT, NULL); } } @@ -425,16 +431,13 @@ __lookup_extent_mapping(struct extent_map_tree *tree, { struct extent_map *em; struct rb_node *rb_node; - struct rb_node *prev = NULL; - struct rb_node *next = NULL; + struct rb_node *prev_or_next = NULL; u64 end = range_end(start, len); - rb_node = __tree_search(&tree->map.rb_root, start, &prev, &next); + rb_node = __tree_search(&tree->map.rb_root, start, &prev_or_next); if (!rb_node) { - if (prev) - rb_node = prev; - else if (next) - rb_node = next; + if (prev_or_next) + rb_node = prev_or_next; else return NULL; } @@ -658,3 +661,293 @@ int btrfs_add_extent_mapping(struct btrfs_fs_info *fs_info, ASSERT(ret == 0 || ret == -EEXIST); return ret; } + +/* + * Drop all extent maps from a tree in the fastest possible way, rescheduling + * if needed. This avoids searching the tree, from the root down to the first + * extent map, before each deletion. + */ +static void drop_all_extent_maps_fast(struct extent_map_tree *tree) +{ + write_lock(&tree->lock); + while (!RB_EMPTY_ROOT(&tree->map.rb_root)) { + struct extent_map *em; + struct rb_node *node; + + node = rb_first_cached(&tree->map); + em = rb_entry(node, struct extent_map, rb_node); + clear_bit(EXTENT_FLAG_PINNED, &em->flags); + clear_bit(EXTENT_FLAG_LOGGING, &em->flags); + remove_extent_mapping(tree, em); + free_extent_map(em); + cond_resched_rwlock_write(&tree->lock); + } + write_unlock(&tree->lock); +} + +/* + * Drop all extent maps in a given range. + * + * @inode: The target inode. + * @start: Start offset of the range. + * @end: End offset of the range (inclusive value). + * @skip_pinned: Indicate if pinned extent maps should be ignored or not. + * + * This drops all the extent maps that intersect the given range [@start, @end]. + * Extent maps that partially overlap the range and extend behind or beyond it, + * are split. + * The caller should have locked an appropriate file range in the inode's io + * tree before calling this function. + */ +void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end, + bool skip_pinned) +{ + struct extent_map *split; + struct extent_map *split2; + struct extent_map *em; + struct extent_map_tree *em_tree = &inode->extent_tree; + u64 len = end - start + 1; + + WARN_ON(end < start); + if (end == (u64)-1) { + if (start == 0 && !skip_pinned) { + drop_all_extent_maps_fast(em_tree); + return; + } + len = (u64)-1; + } else { + /* Make end offset exclusive for use in the loop below. */ + end++; + } + + /* + * It's ok if we fail to allocate the extent maps, see the comment near + * the bottom of the loop below. We only need two spare extent maps in + * the worst case, where the first extent map that intersects our range + * starts before the range and the last extent map that intersects our + * range ends after our range (and they might be the same extent map), + * because we need to split those two extent maps at the boundaries. + */ + split = alloc_extent_map(); + split2 = alloc_extent_map(); + + write_lock(&em_tree->lock); + em = lookup_extent_mapping(em_tree, start, len); + + while (em) { + /* extent_map_end() returns exclusive value (last byte + 1). */ + const u64 em_end = extent_map_end(em); + struct extent_map *next_em = NULL; + u64 gen; + unsigned long flags; + bool modified; + bool compressed; + + if (em_end < end) { + next_em = next_extent_map(em); + if (next_em) { + if (next_em->start < end) + refcount_inc(&next_em->refs); + else + next_em = NULL; + } + } + + if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) { + start = em_end; + if (end != (u64)-1) + len = start + len - em_end; + goto next; + } + + clear_bit(EXTENT_FLAG_PINNED, &em->flags); + clear_bit(EXTENT_FLAG_LOGGING, &flags); + modified = !list_empty(&em->list); + + /* + * The extent map does not cross our target range, so no need to + * split it, we can remove it directly. + */ + if (em->start >= start && em_end <= end) + goto remove_em; + + flags = em->flags; + gen = em->generation; + compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); + + if (em->start < start) { + if (!split) { + split = split2; + split2 = NULL; + if (!split) + goto remove_em; + } + split->start = em->start; + split->len = start - em->start; + + if (em->block_start < EXTENT_MAP_LAST_BYTE) { + split->orig_start = em->orig_start; + split->block_start = em->block_start; + + if (compressed) + split->block_len = em->block_len; + else + split->block_len = split->len; + split->orig_block_len = max(split->block_len, + em->orig_block_len); + split->ram_bytes = em->ram_bytes; + } else { + split->orig_start = split->start; + split->block_len = 0; + split->block_start = em->block_start; + split->orig_block_len = 0; + split->ram_bytes = split->len; + } + + split->generation = gen; + split->flags = flags; + split->compress_type = em->compress_type; + replace_extent_mapping(em_tree, em, split, modified); + free_extent_map(split); + split = split2; + split2 = NULL; + } + if (em_end > end) { + if (!split) { + split = split2; + split2 = NULL; + if (!split) + goto remove_em; + } + split->start = start + len; + split->len = em_end - (start + len); + split->block_start = em->block_start; + split->flags = flags; + split->compress_type = em->compress_type; + split->generation = gen; + + if (em->block_start < EXTENT_MAP_LAST_BYTE) { + split->orig_block_len = max(em->block_len, + em->orig_block_len); + + split->ram_bytes = em->ram_bytes; + if (compressed) { + split->block_len = em->block_len; + split->orig_start = em->orig_start; + } else { + const u64 diff = start + len - em->start; + + split->block_len = split->len; + split->block_start += diff; + split->orig_start = em->orig_start; + } + } else { + split->ram_bytes = split->len; + split->orig_start = split->start; + split->block_len = 0; + split->orig_block_len = 0; + } + + if (extent_map_in_tree(em)) { + replace_extent_mapping(em_tree, em, split, + modified); + } else { + int ret; + + ret = add_extent_mapping(em_tree, split, + modified); + /* Logic error, shouldn't happen. */ + ASSERT(ret == 0); + if (WARN_ON(ret != 0) && modified) + btrfs_set_inode_full_sync(inode); + } + free_extent_map(split); + split = NULL; + } +remove_em: + if (extent_map_in_tree(em)) { + /* + * If the extent map is still in the tree it means that + * either of the following is true: + * + * 1) It fits entirely in our range (doesn't end beyond + * it or starts before it); + * + * 2) It starts before our range and/or ends after our + * range, and we were not able to allocate the extent + * maps for split operations, @split and @split2. + * + * If we are at case 2) then we just remove the entire + * extent map - this is fine since if anyone needs it to + * access the subranges outside our range, will just + * load it again from the subvolume tree's file extent + * item. However if the extent map was in the list of + * modified extents, then we must mark the inode for a + * full fsync, otherwise a fast fsync will miss this + * extent if it's new and needs to be logged. + */ + if ((em->start < start || em_end > end) && modified) { + ASSERT(!split); + btrfs_set_inode_full_sync(inode); + } + remove_extent_mapping(em_tree, em); + } + + /* + * Once for the tree reference (we replaced or removed the + * extent map from the tree). + */ + free_extent_map(em); +next: + /* Once for us (for our lookup reference). */ + free_extent_map(em); + + em = next_em; + } + + write_unlock(&em_tree->lock); + + free_extent_map(split); + free_extent_map(split2); +} + +/* + * Replace a range in the inode's extent map tree with a new extent map. + * + * @inode: The target inode. + * @new_em: The new extent map to add to the inode's extent map tree. + * @modified: Indicate if the new extent map should be added to the list of + * modified extents (for fast fsync tracking). + * + * Drops all the extent maps in the inode's extent map tree that intersect the + * range of the new extent map and adds the new extent map to the tree. + * The caller should have locked an appropriate file range in the inode's io + * tree before calling this function. + */ +int btrfs_replace_extent_map_range(struct btrfs_inode *inode, + struct extent_map *new_em, + bool modified) +{ + const u64 end = new_em->start + new_em->len - 1; + struct extent_map_tree *tree = &inode->extent_tree; + int ret; + + ASSERT(!extent_map_in_tree(new_em)); + + /* + * The caller has locked an appropriate file range in the inode's io + * tree, but getting -EEXIST when adding the new extent map can still + * happen in case there are extents that partially cover the range, and + * this is due to two tasks operating on different parts of the extent. + * See commit 18e83ac75bfe67 ("Btrfs: fix unexpected EEXIST from + * btrfs_get_extent") for an example and details. + */ + do { + btrfs_drop_extent_map_range(inode, new_em->start, end, false); + write_lock(&tree->lock); + ret = add_extent_mapping(tree, new_em, modified); + write_unlock(&tree->lock); + } while (ret == -EEXIST); + + return ret; +} diff --git a/fs/btrfs/extent_map.h b/fs/btrfs/extent_map.h index d2fa32ffe3045545bf4b8de0803e7c77222b9122..ad311864272a00f67e971d5a84a599851dc355a7 100644 --- a/fs/btrfs/extent_map.h +++ b/fs/btrfs/extent_map.h @@ -63,6 +63,8 @@ struct extent_map_tree { rwlock_t lock; }; +struct btrfs_inode; + static inline int extent_map_in_tree(const struct extent_map *em) { return !RB_EMPTY_NODE(&em->rb_node); @@ -104,5 +106,11 @@ struct extent_map *search_extent_mapping(struct extent_map_tree *tree, int btrfs_add_extent_mapping(struct btrfs_fs_info *fs_info, struct extent_map_tree *em_tree, struct extent_map **em_in, u64 start, u64 len); +void btrfs_drop_extent_map_range(struct btrfs_inode *inode, + u64 start, u64 end, + bool skip_pinned); +int btrfs_replace_extent_map_range(struct btrfs_inode *inode, + struct extent_map *new_em, + bool modified); #endif diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index c828f971a3462b8debc07f62bda5c05e4cb27561..6bb9fa961a6a10347e69cb643154855274395f41 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -118,7 +118,7 @@ int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES)) return 0; return clear_extent_bit(&inode->file_extent_tree, start, - start + len - 1, EXTENT_DIRTY, 0, 0, NULL); + start + len - 1, EXTENT_DIRTY, NULL); } static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info, @@ -129,12 +129,20 @@ static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info, return ncsums * fs_info->sectorsize; } -int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, +/* + * Calculate the total size needed to allocate for an ordered sum structure + * spanning @bytes in the file. + */ +static int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, unsigned long bytes) +{ + int num_sectors = (int)DIV_ROUND_UP(bytes, fs_info->sectorsize); + + return sizeof(struct btrfs_ordered_sum) + num_sectors * fs_info->csum_size; +} + +int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root, - u64 objectid, u64 pos, - u64 disk_offset, u64 disk_num_bytes, - u64 num_bytes, u64 offset, u64 ram_bytes, - u8 compression, u8 encryption, u16 other_encoding) + u64 objectid, u64 pos, u64 num_bytes) { int ret = 0; struct btrfs_file_extent_item *item; @@ -157,16 +165,16 @@ int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, leaf = path->nodes[0]; item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - btrfs_set_file_extent_disk_bytenr(leaf, item, disk_offset); - btrfs_set_file_extent_disk_num_bytes(leaf, item, disk_num_bytes); - btrfs_set_file_extent_offset(leaf, item, offset); + btrfs_set_file_extent_disk_bytenr(leaf, item, 0); + btrfs_set_file_extent_disk_num_bytes(leaf, item, 0); + btrfs_set_file_extent_offset(leaf, item, 0); btrfs_set_file_extent_num_bytes(leaf, item, num_bytes); - btrfs_set_file_extent_ram_bytes(leaf, item, ram_bytes); + btrfs_set_file_extent_ram_bytes(leaf, item, num_bytes); btrfs_set_file_extent_generation(leaf, item, trans->transid); btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); - btrfs_set_file_extent_compression(leaf, item, compression); - btrfs_set_file_extent_encryption(leaf, item, encryption); - btrfs_set_file_extent_other_encoding(leaf, item, other_encoding); + btrfs_set_file_extent_compression(leaf, item, 0); + btrfs_set_file_extent_encryption(leaf, item, 0); + btrfs_set_file_extent_other_encoding(leaf, item, 0); btrfs_mark_buffer_dirty(leaf); out: @@ -503,7 +511,8 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst } int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, - struct list_head *list, int search_commit) + struct list_head *list, int search_commit, + bool nowait) { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_key key; @@ -525,6 +534,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, if (!path) return -ENOMEM; + path->nowait = nowait; if (search_commit) { path->skip_locking = 1; path->reada = READA_FORWARD; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 66c822182ecce7899c0786a42969c0fc893de4e2..176b432035aeaaa3f543b899b246e56dee11e7f3 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -473,7 +473,7 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, */ clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - 0, 0, cached); + cached); err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block, extra_bits, cached); @@ -498,159 +498,6 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, return 0; } -/* - * this drops all the extents in the cache that intersect the range - * [start, end]. Existing extents are split as required. - */ -void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end, - int skip_pinned) -{ - struct extent_map *em; - struct extent_map *split = NULL; - struct extent_map *split2 = NULL; - struct extent_map_tree *em_tree = &inode->extent_tree; - u64 len = end - start + 1; - u64 gen; - int ret; - int testend = 1; - unsigned long flags; - int compressed = 0; - bool modified; - - WARN_ON(end < start); - if (end == (u64)-1) { - len = (u64)-1; - testend = 0; - } - while (1) { - int no_splits = 0; - - modified = false; - if (!split) - split = alloc_extent_map(); - if (!split2) - split2 = alloc_extent_map(); - if (!split || !split2) - no_splits = 1; - - write_lock(&em_tree->lock); - em = lookup_extent_mapping(em_tree, start, len); - if (!em) { - write_unlock(&em_tree->lock); - break; - } - flags = em->flags; - gen = em->generation; - if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) { - if (testend && em->start + em->len >= start + len) { - free_extent_map(em); - write_unlock(&em_tree->lock); - break; - } - start = em->start + em->len; - if (testend) - len = start + len - (em->start + em->len); - free_extent_map(em); - write_unlock(&em_tree->lock); - continue; - } - compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); - clear_bit(EXTENT_FLAG_PINNED, &em->flags); - clear_bit(EXTENT_FLAG_LOGGING, &flags); - modified = !list_empty(&em->list); - if (no_splits) - goto next; - - if (em->start < start) { - split->start = em->start; - split->len = start - em->start; - - if (em->block_start < EXTENT_MAP_LAST_BYTE) { - split->orig_start = em->orig_start; - split->block_start = em->block_start; - - if (compressed) - split->block_len = em->block_len; - else - split->block_len = split->len; - split->orig_block_len = max(split->block_len, - em->orig_block_len); - split->ram_bytes = em->ram_bytes; - } else { - split->orig_start = split->start; - split->block_len = 0; - split->block_start = em->block_start; - split->orig_block_len = 0; - split->ram_bytes = split->len; - } - - split->generation = gen; - split->flags = flags; - split->compress_type = em->compress_type; - replace_extent_mapping(em_tree, em, split, modified); - free_extent_map(split); - split = split2; - split2 = NULL; - } - if (testend && em->start + em->len > start + len) { - u64 diff = start + len - em->start; - - split->start = start + len; - split->len = em->start + em->len - (start + len); - split->flags = flags; - split->compress_type = em->compress_type; - split->generation = gen; - - if (em->block_start < EXTENT_MAP_LAST_BYTE) { - split->orig_block_len = max(em->block_len, - em->orig_block_len); - - split->ram_bytes = em->ram_bytes; - if (compressed) { - split->block_len = em->block_len; - split->block_start = em->block_start; - split->orig_start = em->orig_start; - } else { - split->block_len = split->len; - split->block_start = em->block_start - + diff; - split->orig_start = em->orig_start; - } - } else { - split->ram_bytes = split->len; - split->orig_start = split->start; - split->block_len = 0; - split->block_start = em->block_start; - split->orig_block_len = 0; - } - - if (extent_map_in_tree(em)) { - replace_extent_mapping(em_tree, em, split, - modified); - } else { - ret = add_extent_mapping(em_tree, split, - modified); - ASSERT(ret == 0); /* Logic error */ - } - free_extent_map(split); - split = NULL; - } -next: - if (extent_map_in_tree(em)) - remove_extent_mapping(em_tree, em); - write_unlock(&em_tree->lock); - - /* once for us */ - free_extent_map(em); - /* once for the tree*/ - free_extent_map(em); - } - if (split) - free_extent_map(split); - if (split2) - free_extent_map(split2); -} - /* * this is very complex, but the basic idea is to drop all extents * in the range start - end. hint_block is filled in with a block number @@ -708,7 +555,7 @@ int btrfs_drop_extents(struct btrfs_trans_handle *trans, } if (args->drop_cache) - btrfs_drop_extent_cache(inode, args->start, args->end - 1, 0); + btrfs_drop_extent_map_range(inode, args->start, args->end - 1, false); if (args->start >= inode->disk_i_size && !args->replace_extent) modify_tree = 0; @@ -1339,26 +1186,54 @@ static int prepare_uptodate_page(struct inode *inode, return 0; } +static unsigned int get_prepare_fgp_flags(bool nowait) +{ + unsigned int fgp_flags = FGP_LOCK | FGP_ACCESSED | FGP_CREAT; + + if (nowait) + fgp_flags |= FGP_NOWAIT; + + return fgp_flags; +} + +static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait) +{ + gfp_t gfp; + + gfp = btrfs_alloc_write_mask(inode->i_mapping); + if (nowait) { + gfp &= ~__GFP_DIRECT_RECLAIM; + gfp |= GFP_NOWAIT; + } + + return gfp; +} + /* * this just gets pages into the page cache and locks them down. */ static noinline int prepare_pages(struct inode *inode, struct page **pages, size_t num_pages, loff_t pos, - size_t write_bytes, bool force_uptodate) + size_t write_bytes, bool force_uptodate, + bool nowait) { int i; unsigned long index = pos >> PAGE_SHIFT; - gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping); + gfp_t mask = get_prepare_gfp_flags(inode, nowait); + unsigned int fgp_flags = get_prepare_fgp_flags(nowait); int err = 0; int faili; for (i = 0; i < num_pages; i++) { again: - pages[i] = find_or_create_page(inode->i_mapping, index + i, - mask | __GFP_WRITE); + pages[i] = pagecache_get_page(inode->i_mapping, index + i, + fgp_flags, mask | __GFP_WRITE); if (!pages[i]) { faili = i - 1; - err = -ENOMEM; + if (nowait) + err = -EAGAIN; + else + err = -ENOMEM; goto fail; } @@ -1376,7 +1251,7 @@ again: pos + write_bytes, false); if (err) { put_page(pages[i]); - if (err == -EAGAIN) { + if (!nowait && err == -EAGAIN) { err = 0; goto again; } @@ -1411,7 +1286,7 @@ static noinline int lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, size_t num_pages, loff_t pos, size_t write_bytes, - u64 *lockstart, u64 *lockend, + u64 *lockstart, u64 *lockend, bool nowait, struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; @@ -1426,15 +1301,27 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, if (start_pos < inode->vfs_inode.i_size) { struct btrfs_ordered_extent *ordered; - lock_extent_bits(&inode->io_tree, start_pos, last_pos, - cached_state); + if (nowait) { + if (!try_lock_extent(&inode->io_tree, start_pos, last_pos)) { + for (i = 0; i < num_pages; i++) { + unlock_page(pages[i]); + put_page(pages[i]); + pages[i] = NULL; + } + + return -EAGAIN; + } + } else { + lock_extent(&inode->io_tree, start_pos, last_pos, cached_state); + } + ordered = btrfs_lookup_ordered_range(inode, start_pos, last_pos - start_pos + 1); if (ordered && ordered->file_offset + ordered->num_bytes > start_pos && ordered->file_offset <= last_pos) { - unlock_extent_cached(&inode->io_tree, start_pos, - last_pos, cached_state); + unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); for (i = 0; i < num_pages; i++) { unlock_page(pages[i]); put_page(pages[i]); @@ -1481,7 +1368,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages, * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0. */ int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, - size_t *write_bytes) + size_t *write_bytes, bool nowait) { struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_root *root = inode->root; @@ -1500,17 +1387,22 @@ int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, fs_info->sectorsize) - 1; num_bytes = lockend - lockstart + 1; - btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend, NULL); + if (nowait) { + if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend)) { + btrfs_drew_write_unlock(&root->snapshot_lock); + return -EAGAIN; + } + } else { + btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend, NULL); + } ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes, - NULL, NULL, NULL, false); - if (ret <= 0) { - ret = 0; + NULL, NULL, NULL, nowait, false); + if (ret <= 0) btrfs_drew_write_unlock(&root->snapshot_lock); - } else { + else *write_bytes = min_t(size_t, *write_bytes , num_bytes - pos + lockstart); - } - unlock_extent(&inode->io_tree, lockstart, lockend); + unlock_extent(&inode->io_tree, lockstart, lockend, NULL); return ret; } @@ -1607,8 +1499,10 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, bool force_page_uptodate = false; loff_t old_isize = i_size_read(inode); unsigned int ilock_flags = 0; + const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0); - if (iocb->ki_flags & IOCB_NOWAIT) + if (nowait) ilock_flags |= BTRFS_ILOCK_TRY; ret = btrfs_inode_lock(inode, ilock_flags); @@ -1664,18 +1558,29 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, extent_changeset_release(data_reserved); ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, pos, - write_bytes); + write_bytes, nowait); if (ret < 0) { + int can_nocow; + + if (nowait && (ret == -ENOSPC || ret == -EAGAIN)) { + ret = -EAGAIN; + break; + } + /* * If we don't have to COW at the offset, reserve * metadata only. write_bytes may get smaller than * requested here. */ - if (btrfs_check_nocow_lock(BTRFS_I(inode), pos, - &write_bytes) > 0) - only_release_metadata = true; - else + can_nocow = btrfs_check_nocow_lock(BTRFS_I(inode), pos, + &write_bytes, nowait); + if (can_nocow < 0) + ret = can_nocow; + if (can_nocow > 0) + ret = 0; + if (ret) break; + only_release_metadata = true; } num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE); @@ -1685,7 +1590,7 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, WARN_ON(reserve_bytes == 0); ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), reserve_bytes, - reserve_bytes, false); + reserve_bytes, nowait); if (ret) { if (!only_release_metadata) btrfs_free_reserved_data_space(BTRFS_I(inode), @@ -1698,14 +1603,17 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, release_bytes = reserve_bytes; again: + ret = balance_dirty_pages_ratelimited_flags(inode->i_mapping, bdp_flags); + if (ret) + break; + /* * This is going to setup the pages array with the number of * pages we want, so we don't really need to worry about the * contents of pages from loop to loop */ ret = prepare_pages(inode, pages, num_pages, - pos, write_bytes, - force_page_uptodate); + pos, write_bytes, force_page_uptodate, false); if (ret) { btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes); @@ -1715,10 +1623,11 @@ again: extents_locked = lock_and_cleanup_extent_if_need( BTRFS_I(inode), pages, num_pages, pos, write_bytes, &lockstart, - &lockend, &cached_state); + &lockend, nowait, &cached_state); if (extents_locked < 0) { - if (extents_locked == -EAGAIN) + if (!nowait && extents_locked == -EAGAIN) goto again; + btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes); ret = extents_locked; @@ -1782,8 +1691,8 @@ again: * possible cached extent state to avoid a memory leak. */ if (extents_locked) - unlock_extent_cached(&BTRFS_I(inode)->io_tree, - lockstart, lockend, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, + lockend, &cached_state); else free_extent_state(cached_state); @@ -1801,8 +1710,6 @@ again: cond_resched(); - balance_dirty_pages_ratelimited(inode->i_mapping); - pos += copied; num_written += copied; } @@ -2045,7 +1952,7 @@ ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, if (BTRFS_FS_ERROR(inode->root->fs_info)) return -EROFS; - if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) + if (encoded && (iocb->ki_flags & IOCB_NOWAIT)) return -EOPNOTSUPP; if (sync) @@ -2200,14 +2107,6 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) atomic_inc(&root->log_batch); - /* - * Always check for the full sync flag while holding the inode's lock, - * to avoid races with other tasks. The flag must be either set all the - * time during logging or always off all the time while logging. - */ - full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, - &BTRFS_I(inode)->runtime_flags); - /* * Before we acquired the inode's lock and the mmap lock, someone may * have dirtied more pages in the target range. We need to make sure @@ -2232,6 +2131,17 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) goto out; } + /* + * Always check for the full sync flag while holding the inode's lock, + * to avoid races with other tasks. The flag must be either set all the + * time during logging or always off all the time while logging. + * We check the flag here after starting delalloc above, because when + * running delalloc the full sync flag may be set if we need to drop + * extra extent map ranges due to temporary memory allocation failures. + */ + full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, + &BTRFS_I(inode)->runtime_flags); + /* * We have to do this here to avoid the priority inversion of waiting on * IO of a lower priority task while holding a transaction open. @@ -2380,6 +2290,7 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) ret = btrfs_commit_transaction(trans); out: ASSERT(list_empty(&ctx.list)); + ASSERT(list_empty(&ctx.conflict_inodes)); err = file_check_and_advance_wb_err(file); if (!ret) ret = err; @@ -2448,7 +2359,6 @@ static int fill_holes(struct btrfs_trans_handle *trans, struct extent_buffer *leaf; struct btrfs_file_extent_item *fi; struct extent_map *hole_em; - struct extent_map_tree *em_tree = &inode->extent_tree; struct btrfs_key key; int ret; @@ -2482,6 +2392,7 @@ static int fill_holes(struct btrfs_trans_handle *trans, btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_offset(leaf, fi, 0); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); btrfs_mark_buffer_dirty(leaf); goto out; } @@ -2498,13 +2409,14 @@ static int fill_holes(struct btrfs_trans_handle *trans, btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); btrfs_set_file_extent_offset(leaf, fi, 0); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); btrfs_mark_buffer_dirty(leaf); goto out; } btrfs_release_path(path); - ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), - offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0); + ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, + end - offset); if (ret) return ret; @@ -2513,7 +2425,7 @@ out: hole_em = alloc_extent_map(); if (!hole_em) { - btrfs_drop_extent_cache(inode, offset, end - 1, 0); + btrfs_drop_extent_map_range(inode, offset, end - 1, false); btrfs_set_inode_full_sync(inode); } else { hole_em->start = offset; @@ -2527,12 +2439,7 @@ out: hole_em->compress_type = BTRFS_COMPRESS_NONE; hole_em->generation = trans->transid; - do { - btrfs_drop_extent_cache(inode, offset, end - 1, 0); - write_lock(&em_tree->lock); - ret = add_extent_mapping(em_tree, hole_em, 1); - write_unlock(&em_tree->lock); - } while (ret == -EEXIST); + ret = btrfs_replace_extent_map_range(inode, hole_em, true); free_extent_map(hole_em); if (ret) btrfs_set_inode_full_sync(inode); @@ -2589,8 +2496,8 @@ static void btrfs_punch_hole_lock_range(struct inode *inode, while (1) { truncate_pagecache_range(inode, lockstart, lockend); - lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend, - cached_state); + lock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + cached_state); /* * We can't have ordered extents in the range, nor dirty/writeback * pages, because we have locked the inode's VFS lock in exclusive @@ -2605,8 +2512,8 @@ static void btrfs_punch_hole_lock_range(struct inode *inode, page_lockend)) break; - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, - lockend, cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + cached_state); } btrfs_assert_inode_range_clean(BTRFS_I(inode), lockstart, lockend); @@ -3006,9 +2913,8 @@ static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len) if (ret) goto out_only_mutex; - lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode))); - lockend = round_down(offset + len, - btrfs_inode_sectorsize(BTRFS_I(inode))) - 1; + lockstart = round_up(offset, fs_info->sectorsize); + lockend = round_down(offset + len, fs_info->sectorsize) - 1; same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset)) == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)); /* @@ -3106,8 +3012,8 @@ static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len) btrfs_end_transaction(trans); btrfs_btree_balance_dirty(fs_info); out: - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, - &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); out_only_mutex: if (!updated_inode && truncated_block && !ret) { /* @@ -3210,7 +3116,7 @@ enum { static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode, u64 offset) { - const u64 sectorsize = btrfs_inode_sectorsize(inode); + const u64 sectorsize = inode->root->fs_info->sectorsize; struct extent_map *em; int ret; @@ -3240,7 +3146,7 @@ static int btrfs_zero_range(struct inode *inode, struct extent_changeset *data_reserved = NULL; int ret; u64 alloc_hint = 0; - const u64 sectorsize = btrfs_inode_sectorsize(BTRFS_I(inode)); + const u64 sectorsize = fs_info->sectorsize; u64 alloc_start = round_down(offset, sectorsize); u64 alloc_end = round_up(offset + len, sectorsize); u64 bytes_to_reserve = 0; @@ -3380,16 +3286,16 @@ reserve_space: ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved, alloc_start, bytes_to_reserve); if (ret) { - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, - lockend, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, + lockend, &cached_state); goto out; } ret = btrfs_prealloc_file_range(inode, mode, alloc_start, alloc_end - alloc_start, i_blocksize(inode), offset + len, &alloc_hint); - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, - lockend, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); /* btrfs_prealloc_file_range releases reserved space on error */ if (ret) { space_reserved = false; @@ -3426,7 +3332,7 @@ static long btrfs_fallocate(struct file *file, int mode, u64 data_space_reserved = 0; u64 qgroup_reserved = 0; struct extent_map *em; - int blocksize = btrfs_inode_sectorsize(BTRFS_I(inode)); + int blocksize = BTRFS_I(inode)->root->fs_info->sectorsize; int ret; /* Do not allow fallocate in ZONED mode */ @@ -3500,8 +3406,8 @@ static long btrfs_fallocate(struct file *file, int mode, } locked_end = alloc_end - 1; - lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, - &cached_state); + lock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, + &cached_state); btrfs_assert_inode_range_clean(BTRFS_I(inode), alloc_start, locked_end); @@ -3590,30 +3496,289 @@ static long btrfs_fallocate(struct file *file, int mode, */ ret = btrfs_fallocate_update_isize(inode, actual_end, mode); out_unlock: - unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, - &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, + &cached_state); out: btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP); extent_changeset_free(data_reserved); return ret; } +/* + * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range + * that has unflushed and/or flushing delalloc. There might be other adjacent + * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps + * looping while it gets adjacent subranges, and merging them together. + */ +static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end, + u64 *delalloc_start_ret, u64 *delalloc_end_ret) +{ + const u64 len = end + 1 - start; + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + u64 em_end; + u64 delalloc_len; + + /* + * Search the io tree first for EXTENT_DELALLOC. If we find any, it + * means we have delalloc (dirty pages) for which writeback has not + * started yet. + */ + *delalloc_start_ret = start; + delalloc_len = count_range_bits(&inode->io_tree, delalloc_start_ret, end, + len, EXTENT_DELALLOC, 1); + /* + * If delalloc was found then *delalloc_start_ret has a sector size + * aligned value (rounded down). + */ + if (delalloc_len > 0) + *delalloc_end_ret = *delalloc_start_ret + delalloc_len - 1; + + /* + * Now also check if there's any extent map in the range that does not + * map to a hole or prealloc extent. We do this because: + * + * 1) When delalloc is flushed, the file range is locked, we clear the + * EXTENT_DELALLOC bit from the io tree and create an extent map for + * an allocated extent. So we might just have been called after + * delalloc is flushed and before the ordered extent completes and + * inserts the new file extent item in the subvolume's btree; + * + * 2) We may have an extent map created by flushing delalloc for a + * subrange that starts before the subrange we found marked with + * EXTENT_DELALLOC in the io tree. + */ + read_lock(&em_tree->lock); + em = lookup_extent_mapping(em_tree, start, len); + read_unlock(&em_tree->lock); + + /* extent_map_end() returns a non-inclusive end offset. */ + em_end = em ? extent_map_end(em) : 0; + + /* + * If we have a hole/prealloc extent map, check the next one if this one + * ends before our range's end. + */ + if (em && (em->block_start == EXTENT_MAP_HOLE || + test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) && em_end < end) { + struct extent_map *next_em; + + read_lock(&em_tree->lock); + next_em = lookup_extent_mapping(em_tree, em_end, len - em_end); + read_unlock(&em_tree->lock); + + free_extent_map(em); + em_end = next_em ? extent_map_end(next_em) : 0; + em = next_em; + } + + if (em && (em->block_start == EXTENT_MAP_HOLE || + test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) { + free_extent_map(em); + em = NULL; + } + + /* + * No extent map or one for a hole or prealloc extent. Use the delalloc + * range we found in the io tree if we have one. + */ + if (!em) + return (delalloc_len > 0); + + /* + * We don't have any range as EXTENT_DELALLOC in the io tree, so the + * extent map is the only subrange representing delalloc. + */ + if (delalloc_len == 0) { + *delalloc_start_ret = em->start; + *delalloc_end_ret = min(end, em_end - 1); + free_extent_map(em); + return true; + } + + /* + * The extent map represents a delalloc range that starts before the + * delalloc range we found in the io tree. + */ + if (em->start < *delalloc_start_ret) { + *delalloc_start_ret = em->start; + /* + * If the ranges are adjacent, return a combined range. + * Otherwise return the extent map's range. + */ + if (em_end < *delalloc_start_ret) + *delalloc_end_ret = min(end, em_end - 1); + + free_extent_map(em); + return true; + } + + /* + * The extent map starts after the delalloc range we found in the io + * tree. If it's adjacent, return a combined range, otherwise return + * the range found in the io tree. + */ + if (*delalloc_end_ret + 1 == em->start) + *delalloc_end_ret = min(end, em_end - 1); + + free_extent_map(em); + return true; +} + +/* + * Check if there's delalloc in a given range. + * + * @inode: The inode. + * @start: The start offset of the range. It does not need to be + * sector size aligned. + * @end: The end offset (inclusive value) of the search range. + * It does not need to be sector size aligned. + * @delalloc_start_ret: Output argument, set to the start offset of the + * subrange found with delalloc (may not be sector size + * aligned). + * @delalloc_end_ret: Output argument, set to he end offset (inclusive value) + * of the subrange found with delalloc. + * + * Returns true if a subrange with delalloc is found within the given range, and + * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and + * end offsets of the subrange. + */ +bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, + u64 *delalloc_start_ret, u64 *delalloc_end_ret) +{ + u64 cur_offset = round_down(start, inode->root->fs_info->sectorsize); + u64 prev_delalloc_end = 0; + bool ret = false; + + while (cur_offset < end) { + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = find_delalloc_subrange(inode, cur_offset, end, + &delalloc_start, + &delalloc_end); + if (!delalloc) + break; + + if (prev_delalloc_end == 0) { + /* First subrange found. */ + *delalloc_start_ret = max(delalloc_start, start); + *delalloc_end_ret = delalloc_end; + ret = true; + } else if (delalloc_start == prev_delalloc_end + 1) { + /* Subrange adjacent to the previous one, merge them. */ + *delalloc_end_ret = delalloc_end; + } else { + /* Subrange not adjacent to the previous one, exit. */ + break; + } + + prev_delalloc_end = delalloc_end; + cur_offset = delalloc_end + 1; + cond_resched(); + } + + return ret; +} + +/* + * Check if there's a hole or delalloc range in a range representing a hole (or + * prealloc extent) found in the inode's subvolume btree. + * + * @inode: The inode. + * @whence: Seek mode (SEEK_DATA or SEEK_HOLE). + * @start: Start offset of the hole region. It does not need to be sector + * size aligned. + * @end: End offset (inclusive value) of the hole region. It does not + * need to be sector size aligned. + * @start_ret: Return parameter, used to set the start of the subrange in the + * hole that matches the search criteria (seek mode), if such + * subrange is found (return value of the function is true). + * The value returned here may not be sector size aligned. + * + * Returns true if a subrange matching the given seek mode is found, and if one + * is found, it updates @start_ret with the start of the subrange. + */ +static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence, + u64 start, u64 end, u64 *start_ret) +{ + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, start, end, + &delalloc_start, &delalloc_end); + if (delalloc && whence == SEEK_DATA) { + *start_ret = delalloc_start; + return true; + } + + if (delalloc && whence == SEEK_HOLE) { + /* + * We found delalloc but it starts after out start offset. So we + * have a hole between our start offset and the delalloc start. + */ + if (start < delalloc_start) { + *start_ret = start; + return true; + } + /* + * Delalloc range starts at our start offset. + * If the delalloc range's length is smaller than our range, + * then it means we have a hole that starts where the delalloc + * subrange ends. + */ + if (delalloc_end < end) { + *start_ret = delalloc_end + 1; + return true; + } + + /* There's delalloc for the whole range. */ + return false; + } + + if (!delalloc && whence == SEEK_HOLE) { + *start_ret = start; + return true; + } + + /* + * No delalloc in the range and we are seeking for data. The caller has + * to iterate to the next extent item in the subvolume btree. + */ + return false; +} + static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset, int whence) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_map *em = NULL; struct extent_state *cached_state = NULL; - loff_t i_size = inode->vfs_inode.i_size; + const loff_t i_size = i_size_read(&inode->vfs_inode); + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct btrfs_path *path; + struct btrfs_key key; + u64 last_extent_end; u64 lockstart; u64 lockend; u64 start; - u64 len; - int ret = 0; + int ret; + bool found = false; if (i_size == 0 || offset >= i_size) return -ENXIO; + /* + * Quick path. If the inode has no prealloc extents and its number of + * bytes used matches its i_size, then it can not have holes. + */ + if (whence == SEEK_HOLE && + !(inode->flags & BTRFS_INODE_PREALLOC) && + inode_get_bytes(&inode->vfs_inode) == i_size) + return i_size; + /* * offset can be negative, in this case we start finding DATA/HOLE from * the very start of the file. @@ -3625,45 +3790,164 @@ static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset, if (lockend <= lockstart) lockend = lockstart + fs_info->sectorsize; lockend--; - len = lockend - lockstart + 1; - lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state); + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = start; + + last_extent_end = lockstart; + + lock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } while (start < i_size) { - em = btrfs_get_extent_fiemap(inode, start, len); - if (IS_ERR(em)) { - ret = PTR_ERR(em); - em = NULL; - break; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_file_extent_item *extent; + u64 extent_end; + + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + else if (ret > 0) + break; + + leaf = path->nodes[0]; } - if (whence == SEEK_HOLE && - (em->block_start == EXTENT_MAP_HOLE || - test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) - break; - else if (whence == SEEK_DATA && - (em->block_start != EXTENT_MAP_HOLE && - !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) break; - start = em->start + em->len; - free_extent_map(em); - em = NULL; + extent_end = btrfs_file_extent_end(path); + + /* + * In the first iteration we may have a slot that points to an + * extent that ends before our start offset, so skip it. + */ + if (extent_end <= start) { + path->slots[0]++; + continue; + } + + /* We have an implicit hole, NO_HOLES feature is likely set. */ + if (last_extent_end < key.offset) { + u64 search_start = last_extent_end; + u64 found_start; + + /* + * First iteration, @start matches @offset and it's + * within the hole. + */ + if (start == offset) + search_start = offset; + + found = find_desired_extent_in_hole(inode, whence, + search_start, + key.offset - 1, + &found_start); + if (found) { + start = found_start; + break; + } + /* + * Didn't find data or a hole (due to delalloc) in the + * implicit hole range, so need to analyze the extent. + */ + } + + extent = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + + if (btrfs_file_extent_disk_bytenr(leaf, extent) == 0 || + btrfs_file_extent_type(leaf, extent) == + BTRFS_FILE_EXTENT_PREALLOC) { + /* + * Explicit hole or prealloc extent, search for delalloc. + * A prealloc extent is treated like a hole. + */ + u64 search_start = key.offset; + u64 found_start; + + /* + * First iteration, @start matches @offset and it's + * within the hole. + */ + if (start == offset) + search_start = offset; + + found = find_desired_extent_in_hole(inode, whence, + search_start, + extent_end - 1, + &found_start); + if (found) { + start = found_start; + break; + } + /* + * Didn't find data or a hole (due to delalloc) in the + * implicit hole range, so need to analyze the next + * extent item. + */ + } else { + /* + * Found a regular or inline extent. + * If we are seeking for data, adjust the start offset + * and stop, we're done. + */ + if (whence == SEEK_DATA) { + start = max_t(u64, key.offset, offset); + found = true; + break; + } + /* + * Else, we are seeking for a hole, check the next file + * extent item. + */ + } + + start = extent_end; + last_extent_end = extent_end; + path->slots[0]++; + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } cond_resched(); } - free_extent_map(em); - unlock_extent_cached(&inode->io_tree, lockstart, lockend, - &cached_state); - if (ret) { - offset = ret; - } else { - if (whence == SEEK_DATA && start >= i_size) - offset = -ENXIO; - else - offset = min_t(loff_t, start, i_size); + + /* We have an implicit hole from the last extent found up to i_size. */ + if (!found && start < i_size) { + found = find_desired_extent_in_hole(inode, whence, start, + i_size - 1, &start); + if (!found) + start = i_size; } - return offset; +out: + unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + btrfs_free_path(path); + + if (ret < 0) + return ret; + + if (whence == SEEK_DATA && start >= i_size) + return -ENXIO; + + return min_t(loff_t, start, i_size); } static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence) @@ -3691,7 +3975,7 @@ static int btrfs_file_open(struct inode *inode, struct file *filp) { int ret; - filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC; + filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC | FMODE_BUF_WASYNC; ret = fsverity_file_open(inode, filp); if (ret) @@ -3808,6 +4092,7 @@ const struct file_operations btrfs_file_operations = { .mmap = btrfs_file_mmap, .open = btrfs_file_open, .release = btrfs_release_file, + .get_unmapped_area = thp_get_unmapped_area, .fsync = btrfs_sync_file, .fallocate = btrfs_fallocate, .unlocked_ioctl = btrfs_ioctl, diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 996da650ecdc33a1f90ab40e12a5a79c15830712..f4023651dd68bd704a8efceb893649919573eb28 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -48,6 +48,24 @@ static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, struct btrfs_free_space *info, u64 offset, u64 bytes, bool update_stats); +static void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) +{ + struct btrfs_free_space *info; + struct rb_node *node; + + while ((node = rb_last(&ctl->free_space_offset)) != NULL) { + info = rb_entry(node, struct btrfs_free_space, offset_index); + if (!info->bitmap) { + unlink_free_space(ctl, info, true); + kmem_cache_free(btrfs_free_space_cachep, info); + } else { + free_bitmap(ctl, info); + } + + cond_resched_lock(&ctl->tree_lock); + } +} + static struct inode *__lookup_free_space_inode(struct btrfs_root *root, struct btrfs_path *path, u64 offset) @@ -126,10 +144,8 @@ struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group, block_group->disk_cache_state = BTRFS_DC_CLEAR; } - if (!block_group->iref) { + if (!test_and_set_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) block_group->inode = igrab(inode); - block_group->iref = 1; - } spin_unlock(&block_group->lock); return inode; @@ -241,8 +257,7 @@ int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans, clear_nlink(inode); /* One for the block groups ref */ spin_lock(&block_group->lock); - if (block_group->iref) { - block_group->iref = 0; + if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) { block_group->inode = NULL; spin_unlock(&block_group->lock); iput(inode); @@ -333,8 +348,8 @@ int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans, btrfs_i_size_write(inode, 0); truncate_pagecache(vfs_inode, 0); - lock_extent_bits(&inode->io_tree, 0, (u64)-1, &cached_state); - btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); + lock_extent(&inode->io_tree, 0, (u64)-1, &cached_state); + btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); /* * We skip the throttling logic for free space cache inodes, so we don't @@ -345,7 +360,7 @@ int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans, inode_sub_bytes(&inode->vfs_inode, control.sub_bytes); btrfs_inode_safe_disk_i_size_write(inode, control.last_size); - unlock_extent_cached(&inode->io_tree, 0, (u64)-1, &cached_state); + unlock_extent(&inode->io_tree, 0, (u64)-1, &cached_state); if (ret) goto fail; @@ -693,6 +708,12 @@ static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) max_bitmaps = max_t(u64, max_bitmaps, 1); + if (ctl->total_bitmaps > max_bitmaps) + btrfs_err(block_group->fs_info, +"invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu", + block_group->start, block_group->length, + ctl->total_bitmaps, ctl->unit, max_bitmaps, + bytes_per_bg); ASSERT(ctl->total_bitmaps <= max_bitmaps); /* @@ -875,7 +896,10 @@ out: return ret; free_cache: io_ctl_drop_pages(&io_ctl); + + spin_lock(&ctl->tree_lock); __btrfs_remove_free_space_cache(ctl); + spin_unlock(&ctl->tree_lock); goto out; } @@ -914,6 +938,8 @@ static int copy_free_space_cache(struct btrfs_block_group *block_group, return ret; } +static struct lock_class_key btrfs_free_space_inode_key; + int load_free_space_cache(struct btrfs_block_group *block_group) { struct btrfs_fs_info *fs_info = block_group->fs_info; @@ -983,6 +1009,14 @@ int load_free_space_cache(struct btrfs_block_group *block_group) } spin_unlock(&block_group->lock); + /* + * Reinitialize the class of struct inode's mapping->invalidate_lock for + * free space inodes to prevent false positives related to locks for normal + * inodes. + */ + lockdep_set_class(&(&inode->i_data)->invalidate_lock, + &btrfs_free_space_inode_key); + ret = __load_free_space_cache(fs_info->tree_root, inode, &tmp_ctl, path, block_group->start); btrfs_free_path(path); @@ -1001,7 +1035,13 @@ int load_free_space_cache(struct btrfs_block_group *block_group) if (ret == 0) ret = 1; } else { + /* + * We need to call the _locked variant so we don't try to update + * the discard counters. + */ + spin_lock(&tmp_ctl.tree_lock); __btrfs_remove_free_space_cache(&tmp_ctl); + spin_unlock(&tmp_ctl.tree_lock); btrfs_warn(fs_info, "block group %llu has wrong amount of free space", block_group->start); @@ -1123,7 +1163,7 @@ update_cache_item(struct btrfs_trans_handle *trans, ret = btrfs_search_slot(trans, root, &key, path, 0, 1); if (ret < 0) { clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1, - EXTENT_DELALLOC, 0, 0, NULL); + EXTENT_DELALLOC, NULL); goto fail; } leaf = path->nodes[0]; @@ -1135,8 +1175,8 @@ update_cache_item(struct btrfs_trans_handle *trans, if (found_key.objectid != BTRFS_FREE_SPACE_OBJECTID || found_key.offset != offset) { clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, - inode->i_size - 1, EXTENT_DELALLOC, 0, - 0, NULL); + inode->i_size - 1, EXTENT_DELALLOC, + NULL); btrfs_release_path(path); goto fail; } @@ -1232,7 +1272,7 @@ static int flush_dirty_cache(struct inode *inode) ret = btrfs_wait_ordered_range(inode, 0, (u64)-1); if (ret) clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1, - EXTENT_DELALLOC, 0, 0, NULL); + EXTENT_DELALLOC, NULL); return ret; } @@ -1252,8 +1292,8 @@ cleanup_write_cache_enospc(struct inode *inode, struct extent_state **cached_state) { io_ctl_drop_pages(io_ctl); - unlock_extent_cached(&BTRFS_I(inode)->io_tree, 0, - i_size_read(inode) - 1, cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + cached_state); } static int __btrfs_wait_cache_io(struct btrfs_root *root, @@ -1378,8 +1418,8 @@ static int __btrfs_write_out_cache(struct btrfs_root *root, struct inode *inode, if (ret) goto out_unlock; - lock_extent_bits(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, - &cached_state); + lock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + &cached_state); io_ctl_set_generation(io_ctl, trans->transid); @@ -1434,8 +1474,8 @@ static int __btrfs_write_out_cache(struct btrfs_root *root, struct inode *inode, io_ctl_drop_pages(io_ctl); io_ctl_free(io_ctl); - unlock_extent_cached(&BTRFS_I(inode)->io_tree, 0, - i_size_read(inode) - 1, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + &cached_state); /* * at this point the pages are under IO and we're happy, @@ -2860,7 +2900,8 @@ void btrfs_dump_free_space(struct btrfs_block_group *block_group, if (btrfs_is_zoned(fs_info)) { btrfs_info(fs_info, "free space %llu active %d", block_group->zone_capacity - block_group->alloc_offset, - block_group->zone_is_active); + test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &block_group->runtime_flags)); return; } @@ -2964,34 +3005,6 @@ static void __btrfs_return_cluster_to_free_space( btrfs_put_block_group(block_group); } -static void __btrfs_remove_free_space_cache_locked( - struct btrfs_free_space_ctl *ctl) -{ - struct btrfs_free_space *info; - struct rb_node *node; - - while ((node = rb_last(&ctl->free_space_offset)) != NULL) { - info = rb_entry(node, struct btrfs_free_space, offset_index); - if (!info->bitmap) { - unlink_free_space(ctl, info, true); - kmem_cache_free(btrfs_free_space_cachep, info); - } else { - free_bitmap(ctl, info); - } - - cond_resched_lock(&ctl->tree_lock); - } -} - -void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) -{ - spin_lock(&ctl->tree_lock); - __btrfs_remove_free_space_cache_locked(ctl); - if (ctl->block_group) - btrfs_discard_update_discardable(ctl->block_group); - spin_unlock(&ctl->tree_lock); -} - void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group) { struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; @@ -3009,7 +3022,7 @@ void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group) cond_resched_lock(&ctl->tree_lock); } - __btrfs_remove_free_space_cache_locked(ctl); + __btrfs_remove_free_space_cache(ctl); btrfs_discard_update_discardable(block_group); spin_unlock(&ctl->tree_lock); @@ -3992,7 +4005,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, *trimmed = 0; spin_lock(&block_group->lock); - if (block_group->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { spin_unlock(&block_group->lock); return 0; } @@ -4022,7 +4035,7 @@ int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group, *trimmed = 0; spin_lock(&block_group->lock); - if (block_group->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { spin_unlock(&block_group->lock); return 0; } @@ -4044,7 +4057,7 @@ int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group, *trimmed = 0; spin_lock(&block_group->lock); - if (block_group->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { spin_unlock(&block_group->lock); return 0; } diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h index 15591b2998956cd7c819cfe8d29a0e611ceb341c..6d419ba53e954ffef3c70f31d03919ce29eb760e 100644 --- a/fs/btrfs/free-space-cache.h +++ b/fs/btrfs/free-space-cache.h @@ -113,7 +113,6 @@ int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group, u64 bytenr, u64 size); int btrfs_remove_free_space(struct btrfs_block_group *block_group, u64 bytenr, u64 size); -void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl); void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group); bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group); u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group, diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index 1bf89aa672160b264469c59e486bbcaf3a7768b5..367bcfcf68f51eb085f669d989ba98b7885cdd26 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1453,8 +1453,6 @@ static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, ASSERT(key.type == BTRFS_FREE_SPACE_BITMAP_KEY); ASSERT(key.objectid < end && key.objectid + key.offset <= end); - caching_ctl->progress = key.objectid; - offset = key.objectid; while (offset < key.objectid + key.offset) { bit = free_space_test_bit(block_group, path, offset); @@ -1490,8 +1488,6 @@ static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, goto out; } - caching_ctl->progress = (u64)-1; - ret = 0; out: return ret; @@ -1531,8 +1527,6 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, ASSERT(key.type == BTRFS_FREE_SPACE_EXTENT_KEY); ASSERT(key.objectid < end && key.objectid + key.offset <= end); - caching_ctl->progress = key.objectid; - total_found += add_new_free_space(block_group, key.objectid, key.objectid + key.offset); if (total_found > CACHING_CTL_WAKE_UP) { @@ -1552,8 +1546,6 @@ static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, goto out; } - caching_ctl->progress = (u64)-1; - ret = 0; out: return ret; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index f0c97d25b4a0e762849ac63c3fa7f40d8bf38f2f..b0807c59e32107087998fc7a59450bcffcb1c3fe 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -977,7 +977,7 @@ static int submit_one_async_extent(struct btrfs_inode *inode, if (!(start >= locked_page_end || end <= locked_page_start)) locked_page = async_chunk->locked_page; } - lock_extent(io_tree, start, end); + lock_extent(io_tree, start, end, NULL); /* We have fall back to uncompressed write */ if (!async_extent->pages) @@ -1024,7 +1024,7 @@ static int submit_one_async_extent(struct btrfs_inode *inode, 1 << BTRFS_ORDERED_COMPRESSED, async_extent->compress_type); if (ret) { - btrfs_drop_extent_cache(inode, start, end, 0); + btrfs_drop_extent_map_range(inode, start, end, false); goto out_free_reserve; } btrfs_dec_block_group_reservations(fs_info, ins.objectid); @@ -1254,7 +1254,6 @@ static noinline int cow_file_range(struct btrfs_inode *inode, } alloc_hint = get_extent_allocation_hint(inode, start, num_bytes); - btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0); /* * Relocation relies on the relocated extents to have exactly the same @@ -1319,8 +1318,9 @@ static noinline int cow_file_range(struct btrfs_inode *inode, * skip current ordered extent. */ if (ret) - btrfs_drop_extent_cache(inode, start, - start + ram_size - 1, 0); + btrfs_drop_extent_map_range(inode, start, + start + ram_size - 1, + false); } btrfs_dec_block_group_reservations(fs_info, ins.objectid); @@ -1360,7 +1360,7 @@ out: return ret; out_drop_extent_cache: - btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0); + btrfs_drop_extent_map_range(inode, start, start + ram_size - 1, false); out_reserve: btrfs_dec_block_group_reservations(fs_info, ins.objectid); btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1); @@ -1524,7 +1524,7 @@ static int cow_file_range_async(struct btrfs_inode *inode, unsigned nofs_flag; const blk_opf_t write_flags = wbc_to_write_flags(wbc); - unlock_extent(&inode->io_tree, start, end); + unlock_extent(&inode->io_tree, start, end, NULL); if (inode->flags & BTRFS_INODE_NOCOMPRESS && !btrfs_test_opt(fs_info, FORCE_COMPRESS)) { @@ -1644,10 +1644,9 @@ static noinline int run_delalloc_zoned(struct btrfs_inode *inode, done_offset = end; if (done_offset == start) { - struct btrfs_fs_info *info = inode->root->fs_info; - - wait_var_event(&info->zone_finish_wait, - !test_bit(BTRFS_FS_NEED_ZONE_FINISH, &info->flags)); + wait_on_bit_io(&inode->root->fs_info->flags, + BTRFS_FS_NEED_ZONE_FINISH, + TASK_UNINTERRUPTIBLE); continue; } @@ -1667,7 +1666,7 @@ static noinline int run_delalloc_zoned(struct btrfs_inode *inode, } static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, - u64 bytenr, u64 num_bytes) + u64 bytenr, u64 num_bytes, bool nowait) { struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bytenr); struct btrfs_ordered_sum *sums; @@ -1675,7 +1674,8 @@ static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, LIST_HEAD(list); ret = btrfs_lookup_csums_range(csum_root, bytenr, - bytenr + num_bytes - 1, &list, 0); + bytenr + num_bytes - 1, &list, 0, + nowait); if (ret == 0 && list_empty(&list)) return 0; @@ -1748,7 +1748,7 @@ static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page, if (count > 0) clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, - 0, 0, NULL); + NULL); } return cow_file_range(inode, locked_page, start, end, page_started, @@ -1801,6 +1801,7 @@ static int can_nocow_file_extent(struct btrfs_path *path, u8 extent_type; int can_nocow = 0; int ret = 0; + bool nowait = path->nowait; fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); extent_type = btrfs_file_extent_type(leaf, fi); @@ -1877,7 +1878,8 @@ static int can_nocow_file_extent(struct btrfs_path *path, * Force COW if csums exist in the range. This ensures that csums for a * given extent are either valid or do not exist. */ - ret = csum_exist_in_range(root->fs_info, args->disk_bytenr, args->num_bytes); + ret = csum_exist_in_range(root->fs_info, args->disk_bytenr, args->num_bytes, + nowait); WARN_ON_ONCE(ret > 0 && is_freespace_inode); if (ret != 0) goto out; @@ -2100,8 +2102,8 @@ out_check: 1 << BTRFS_ORDERED_PREALLOC, BTRFS_COMPRESS_NONE); if (ret) { - btrfs_drop_extent_cache(inode, cur_offset, - nocow_end, 0); + btrfs_drop_extent_map_range(inode, cur_offset, + nocow_end, false); goto error; } } else { @@ -2549,7 +2551,7 @@ static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len, ASSERT(pre + post < len); - lock_extent(&inode->io_tree, start, start + len - 1); + lock_extent(&inode->io_tree, start, start + len - 1, NULL); write_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, start, len); if (!em) { @@ -2623,7 +2625,7 @@ static int split_zoned_em(struct btrfs_inode *inode, u64 start, u64 len, out_unlock: write_unlock(&em_tree->lock); - unlock_extent(&inode->io_tree, start, start + len - 1); + unlock_extent(&inode->io_tree, start, start + len - 1, NULL); out: free_extent_map(split_pre); free_extent_map(split_mid); @@ -2701,8 +2703,10 @@ void btrfs_submit_data_write_bio(struct inode *inode, struct bio *bio, int mirro if (bio_op(bio) == REQ_OP_ZONE_APPEND) { ret = extract_ordered_extent(bi, bio, page_offset(bio_first_bvec_all(bio)->bv_page)); - if (ret) - goto out; + if (ret) { + btrfs_bio_end_io(btrfs_bio(bio), ret); + return; + } } /* @@ -2722,16 +2726,12 @@ void btrfs_submit_data_write_bio(struct inode *inode, struct bio *bio, int mirro return; ret = btrfs_csum_one_bio(bi, bio, (u64)-1, false); - if (ret) - goto out; + if (ret) { + btrfs_bio_end_io(btrfs_bio(bio), ret); + return; + } } btrfs_submit_bio(fs_info, bio, mirror_num); - return; -out: - if (ret) { - bio->bi_status = ret; - bio_endio(bio); - } } void btrfs_submit_data_read_bio(struct inode *inode, struct bio *bio, @@ -2758,8 +2758,7 @@ void btrfs_submit_data_read_bio(struct inode *inode, struct bio *bio, */ ret = btrfs_lookup_bio_sums(inode, bio, NULL); if (ret) { - bio->bi_status = ret; - bio_endio(bio); + btrfs_bio_end_io(btrfs_bio(bio), ret); return; } @@ -2819,8 +2818,8 @@ static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode, ret = set_extent_bit(&inode->io_tree, search_start, search_start + em_len - 1, - EXTENT_DELALLOC_NEW, 0, NULL, cached_state, - GFP_NOFS, NULL); + EXTENT_DELALLOC_NEW, cached_state, + GFP_NOFS); next: search_start = extent_map_end(em); free_extent_map(em); @@ -2932,7 +2931,7 @@ again: if (ret) goto out_page; - lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state); + lock_extent(&inode->io_tree, page_start, page_end, &cached_state); /* already ordered? We're done */ if (PageOrdered(page)) @@ -2940,8 +2939,8 @@ again: ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); if (ordered) { - unlock_extent_cached(&inode->io_tree, page_start, page_end, - &cached_state); + unlock_extent(&inode->io_tree, page_start, page_end, + &cached_state); unlock_page(page); btrfs_start_ordered_extent(ordered, 1); btrfs_put_ordered_extent(ordered); @@ -2967,8 +2966,7 @@ out_reserved: if (free_delalloc_space) btrfs_delalloc_release_space(inode, data_reserved, page_start, PAGE_SIZE, true); - unlock_extent_cached(&inode->io_tree, page_start, page_end, - &cached_state); + unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); out_page: if (ret) { /* @@ -3226,6 +3224,8 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) clear_bits |= EXTENT_DELALLOC_NEW; freespace_inode = btrfs_is_free_space_inode(inode); + if (!freespace_inode) + btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent); if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) { ret = -EIO; @@ -3270,7 +3270,7 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) } clear_bits |= EXTENT_LOCKED; - lock_extent_bits(io_tree, start, end, &cached_state); + lock_extent(io_tree, start, end, &cached_state); if (freespace_inode) trans = btrfs_join_transaction_spacecache(root); @@ -3326,7 +3326,7 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) clear_extent_bit(&inode->io_tree, start, end, EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES, - 0, 0, &cached_state); + &cached_state); btrfs_inode_safe_disk_i_size_write(inode, 0); ret = btrfs_update_inode_fallback(trans, root, inode); @@ -3337,7 +3337,6 @@ int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent) ret = 0; out: clear_extent_bit(&inode->io_tree, start, end, clear_bits, - (clear_bits & EXTENT_LOCKED) ? 1 : 0, 0, &cached_state); if (trans) @@ -3362,8 +3361,8 @@ out: unwritten_start += logical_len; clear_extent_uptodate(io_tree, unwritten_start, end, NULL); - /* Drop the cache for the part of the extent we didn't write. */ - btrfs_drop_extent_cache(inode, unwritten_start, end, 0); + /* Drop extent maps for the part of the extent we didn't write. */ + btrfs_drop_extent_map_range(inode, unwritten_start, end, false); /* * If the ordered extent had an IOERR or something else went @@ -3440,6 +3439,13 @@ int btrfs_check_sector_csum(struct btrfs_fs_info *fs_info, struct page *page, return 0; } +static u8 *btrfs_csum_ptr(const struct btrfs_fs_info *fs_info, u8 *csums, u64 offset) +{ + u64 offset_in_sectors = offset >> fs_info->sectorsize_bits; + + return csums + offset_in_sectors * fs_info->csum_size; +} + /* * check_data_csum - verify checksum of one sector of uncompressed data * @inode: inode @@ -4879,9 +4885,9 @@ int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, block_end = block_start + blocksize - 1; ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, - blocksize); + blocksize, false); if (ret < 0) { - if (btrfs_check_nocow_lock(inode, block_start, &write_bytes) > 0) { + if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) { /* For nocow case, no need to reserve data space */ only_release_metadata = true; } else { @@ -4923,12 +4929,11 @@ again: } wait_on_page_writeback(page); - lock_extent_bits(io_tree, block_start, block_end, &cached_state); + lock_extent(io_tree, block_start, block_end, &cached_state); ordered = btrfs_lookup_ordered_extent(inode, block_start); if (ordered) { - unlock_extent_cached(io_tree, block_start, block_end, - &cached_state); + unlock_extent(io_tree, block_start, block_end, &cached_state); unlock_page(page); put_page(page); btrfs_start_ordered_extent(ordered, 1); @@ -4938,13 +4943,12 @@ again: clear_extent_bit(&inode->io_tree, block_start, block_end, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - 0, 0, &cached_state); + &cached_state); ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0, &cached_state); if (ret) { - unlock_extent_cached(io_tree, block_start, block_end, - &cached_state); + unlock_extent(io_tree, block_start, block_end, &cached_state); goto out_unlock; } @@ -4961,11 +4965,11 @@ again: btrfs_page_clear_checked(fs_info, page, block_start, block_end + 1 - block_start); btrfs_page_set_dirty(fs_info, page, block_start, block_end + 1 - block_start); - unlock_extent_cached(io_tree, block_start, block_end, &cached_state); + unlock_extent(io_tree, block_start, block_end, &cached_state); if (only_release_metadata) set_extent_bit(&inode->io_tree, block_start, block_end, - EXTENT_NORESERVE, 0, NULL, NULL, GFP_NOFS, NULL); + EXTENT_NORESERVE, NULL, GFP_NOFS); out_unlock: if (ret) { @@ -5022,8 +5026,7 @@ static int maybe_insert_hole(struct btrfs_root *root, struct btrfs_inode *inode, return ret; } - ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode), - offset, 0, 0, len, 0, len, 0, 0, 0); + ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len); if (ret) { btrfs_abort_transaction(trans, ret); } else { @@ -5047,7 +5050,6 @@ int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) struct extent_io_tree *io_tree = &inode->io_tree; struct extent_map *em = NULL; struct extent_state *cached_state = NULL; - struct extent_map_tree *em_tree = &inode->extent_tree; u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); u64 block_end = ALIGN(size, fs_info->sectorsize); u64 last_byte; @@ -5095,10 +5097,11 @@ int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) if (err) break; - btrfs_drop_extent_cache(inode, cur_offset, - cur_offset + hole_size - 1, 0); hole_em = alloc_extent_map(); if (!hole_em) { + btrfs_drop_extent_map_range(inode, cur_offset, + cur_offset + hole_size - 1, + false); btrfs_set_inode_full_sync(inode); goto next; } @@ -5113,16 +5116,7 @@ int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) hole_em->compress_type = BTRFS_COMPRESS_NONE; hole_em->generation = fs_info->generation; - while (1) { - write_lock(&em_tree->lock); - err = add_extent_mapping(em_tree, hole_em, 1); - write_unlock(&em_tree->lock); - if (err != -EEXIST) - break; - btrfs_drop_extent_cache(inode, cur_offset, - cur_offset + - hole_size - 1, 0); - } + err = btrfs_replace_extent_map_range(inode, hole_em, true); free_extent_map(hole_em); } else { err = btrfs_inode_set_file_extent_range(inode, @@ -5138,7 +5132,7 @@ next: break; } free_extent_map(em); - unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state); + unlock_extent(io_tree, hole_start, block_end - 1, &cached_state); return err; } @@ -5272,7 +5266,7 @@ static int btrfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentr * While truncating the inode pages during eviction, we get the VFS * calling btrfs_invalidate_folio() against each folio of the inode. This * is slow because the calls to btrfs_invalidate_folio() result in a - * huge amount of calls to lock_extent_bits() and clear_extent_bit(), + * huge amount of calls to lock_extent() and clear_extent_bit(), * which keep merging and splitting extent_state structures over and over, * wasting lots of time. * @@ -5284,29 +5278,12 @@ static int btrfs_setattr(struct user_namespace *mnt_userns, struct dentry *dentr static void evict_inode_truncate_pages(struct inode *inode) { struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; - struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree; struct rb_node *node; ASSERT(inode->i_state & I_FREEING); truncate_inode_pages_final(&inode->i_data); - write_lock(&map_tree->lock); - while (!RB_EMPTY_ROOT(&map_tree->map.rb_root)) { - struct extent_map *em; - - node = rb_first_cached(&map_tree->map); - em = rb_entry(node, struct extent_map, rb_node); - clear_bit(EXTENT_FLAG_PINNED, &em->flags); - clear_bit(EXTENT_FLAG_LOGGING, &em->flags); - remove_extent_mapping(map_tree, em); - free_extent_map(em); - if (need_resched()) { - write_unlock(&map_tree->lock); - cond_resched(); - write_lock(&map_tree->lock); - } - } - write_unlock(&map_tree->lock); + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); /* * Keep looping until we have no more ranges in the io tree. @@ -5339,7 +5316,7 @@ static void evict_inode_truncate_pages(struct inode *inode) state_flags = state->state; spin_unlock(&io_tree->lock); - lock_extent_bits(io_tree, start, end, &cached_state); + lock_extent(io_tree, start, end, &cached_state); /* * If still has DELALLOC flag, the extent didn't reach disk, @@ -5354,8 +5331,7 @@ static void evict_inode_truncate_pages(struct inode *inode) end - start + 1); clear_extent_bit(io_tree, start, end, - EXTENT_LOCKED | EXTENT_DELALLOC | - EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1, + EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING, &cached_state); cond_resched(); @@ -5708,6 +5684,11 @@ static int btrfs_init_locked_inode(struct inode *inode, void *p) BTRFS_I(inode)->location.offset = 0; BTRFS_I(inode)->root = btrfs_grab_root(args->root); BUG_ON(args->root && !BTRFS_I(inode)->root); + + if (args->root && args->root == args->root->fs_info->tree_root && + args->ino != BTRFS_BTREE_INODE_OBJECTID) + set_bit(BTRFS_INODE_FREE_SPACE_INODE, + &BTRFS_I(inode)->runtime_flags); return 0; } @@ -6868,7 +6849,6 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, struct btrfs_key found_key; struct extent_map *em = NULL; struct extent_map_tree *em_tree = &inode->extent_tree; - struct extent_io_tree *io_tree = &inode->io_tree; read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, start, len); @@ -7031,8 +7011,6 @@ next: } flush_dcache_page(page); } - set_extent_uptodate(io_tree, em->start, - extent_map_end(em) - 1, NULL, GFP_NOFS); goto insert; } not_found: @@ -7066,133 +7044,6 @@ out: return em; } -struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, - u64 start, u64 len) -{ - struct extent_map *em; - struct extent_map *hole_em = NULL; - u64 delalloc_start = start; - u64 end; - u64 delalloc_len; - u64 delalloc_end; - int err = 0; - - em = btrfs_get_extent(inode, NULL, 0, start, len); - if (IS_ERR(em)) - return em; - /* - * If our em maps to: - * - a hole or - * - a pre-alloc extent, - * there might actually be delalloc bytes behind it. - */ - if (em->block_start != EXTENT_MAP_HOLE && - !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) - return em; - else - hole_em = em; - - /* check to see if we've wrapped (len == -1 or similar) */ - end = start + len; - if (end < start) - end = (u64)-1; - else - end -= 1; - - em = NULL; - - /* ok, we didn't find anything, lets look for delalloc */ - delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start, - end, len, EXTENT_DELALLOC, 1); - delalloc_end = delalloc_start + delalloc_len; - if (delalloc_end < delalloc_start) - delalloc_end = (u64)-1; - - /* - * We didn't find anything useful, return the original results from - * get_extent() - */ - if (delalloc_start > end || delalloc_end <= start) { - em = hole_em; - hole_em = NULL; - goto out; - } - - /* - * Adjust the delalloc_start to make sure it doesn't go backwards from - * the start they passed in - */ - delalloc_start = max(start, delalloc_start); - delalloc_len = delalloc_end - delalloc_start; - - if (delalloc_len > 0) { - u64 hole_start; - u64 hole_len; - const u64 hole_end = extent_map_end(hole_em); - - em = alloc_extent_map(); - if (!em) { - err = -ENOMEM; - goto out; - } - - ASSERT(hole_em); - /* - * When btrfs_get_extent can't find anything it returns one - * huge hole - * - * Make sure what it found really fits our range, and adjust to - * make sure it is based on the start from the caller - */ - if (hole_end <= start || hole_em->start > end) { - free_extent_map(hole_em); - hole_em = NULL; - } else { - hole_start = max(hole_em->start, start); - hole_len = hole_end - hole_start; - } - - if (hole_em && delalloc_start > hole_start) { - /* - * Our hole starts before our delalloc, so we have to - * return just the parts of the hole that go until the - * delalloc starts - */ - em->len = min(hole_len, delalloc_start - hole_start); - em->start = hole_start; - em->orig_start = hole_start; - /* - * Don't adjust block start at all, it is fixed at - * EXTENT_MAP_HOLE - */ - em->block_start = hole_em->block_start; - em->block_len = hole_len; - if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags)) - set_bit(EXTENT_FLAG_PREALLOC, &em->flags); - } else { - /* - * Hole is out of passed range or it starts after - * delalloc range - */ - em->start = delalloc_start; - em->len = delalloc_len; - em->orig_start = delalloc_start; - em->block_start = EXTENT_MAP_DELALLOC; - em->block_len = delalloc_len; - } - } else { - return hole_em; - } -out: - - free_extent_map(hole_em); - if (err) { - free_extent_map(em); - return ERR_PTR(err); - } - return em; -} - static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, const u64 start, const u64 len, @@ -7222,7 +7073,8 @@ static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, if (ret) { if (em) { free_extent_map(em); - btrfs_drop_extent_cache(inode, start, start + len - 1, 0); + btrfs_drop_extent_map_range(inode, start, + start + len - 1, false); } em = ERR_PTR(ret); } @@ -7293,7 +7145,7 @@ static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) */ noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, u64 *orig_start, u64 *orig_block_len, - u64 *ram_bytes, bool strict) + u64 *ram_bytes, bool nowait, bool strict) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct can_nocow_file_extent_args nocow_args = { 0 }; @@ -7309,6 +7161,7 @@ noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, path = btrfs_alloc_path(); if (!path) return -ENOMEM; + path->nowait = nowait; ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(BTRFS_I(inode)), offset, 0); @@ -7405,7 +7258,7 @@ static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, if (!try_lock_extent(io_tree, lockstart, lockend)) return -EAGAIN; } else { - lock_extent_bits(io_tree, lockstart, lockend, cached_state); + lock_extent(io_tree, lockstart, lockend, cached_state); } /* * We're concerned with the entire range that we're going to be @@ -7427,7 +7280,7 @@ static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, lockstart, lockend))) break; - unlock_extent_cached(io_tree, lockstart, lockend, cached_state); + unlock_extent(io_tree, lockstart, lockend, cached_state); if (ordered) { if (nowait) { @@ -7489,7 +7342,6 @@ static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, u64 ram_bytes, int compress_type, int type) { - struct extent_map_tree *em_tree; struct extent_map *em; int ret; @@ -7498,7 +7350,6 @@ static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, type == BTRFS_ORDERED_NOCOW || type == BTRFS_ORDERED_REGULAR); - em_tree = &inode->extent_tree; em = alloc_extent_map(); if (!em) return ERR_PTR(-ENOMEM); @@ -7519,18 +7370,7 @@ static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start, em->compress_type = compress_type; } - do { - btrfs_drop_extent_cache(inode, em->start, - em->start + em->len - 1, 0); - write_lock(&em_tree->lock); - ret = add_extent_mapping(em_tree, em, 1); - write_unlock(&em_tree->lock); - /* - * The caller has taken lock_extent(), who could race with us - * to add em? - */ - } while (ret == -EEXIST); - + ret = btrfs_replace_extent_map_range(inode, em, true); if (ret) { free_extent_map(em); return ERR_PTR(ret); @@ -7578,7 +7418,7 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, block_start = em->block_start + (start - em->start); if (can_nocow_extent(inode, start, &len, &orig_start, - &orig_block_len, &ram_bytes, false) == 1) { + &orig_block_len, &ram_bytes, false, false) == 1) { bg = btrfs_inc_nocow_writers(fs_info, block_start); if (bg) can_nocow = true; @@ -7693,6 +7533,20 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, const u64 data_alloc_len = length; bool unlock_extents = false; + /* + * We could potentially fault if we have a buffer > PAGE_SIZE, and if + * we're NOWAIT we may submit a bio for a partial range and return + * EIOCBQUEUED, which would result in an errant short read. + * + * The best way to handle this would be to allow for partial completions + * of iocb's, so we could submit the partial bio, return and fault in + * the rest of the pages, and then submit the io for the rest of the + * range. However we don't have that currently, so simply return + * -EAGAIN at this point so that the normal path is used. + */ + if (!write && (flags & IOMAP_NOWAIT) && length > PAGE_SIZE) + return -EAGAIN; + /* * Cap the size of reads to that usually seen in buffered I/O as we need * to allocate a contiguous array for the checksums. @@ -7749,7 +7603,7 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, if (write && !(flags & IOMAP_NOWAIT)) { ret = btrfs_check_data_free_space(BTRFS_I(inode), &dio_data->data_reserved, - start, data_alloc_len); + start, data_alloc_len, false); if (!ret) dio_data->data_space_reserved = true; else if (ret && !(BTRFS_I(inode)->flags & @@ -7871,8 +7725,8 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, } if (unlock_extents) - unlock_extent_cached(&BTRFS_I(inode)->io_tree, - lockstart, lockend, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); else free_extent_state(cached_state); @@ -7901,8 +7755,8 @@ static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, return 0; unlock_err: - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, - &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); err: if (dio_data->data_space_reserved) { btrfs_free_reserved_data_space(BTRFS_I(inode), @@ -7925,7 +7779,8 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, if (!write && (iomap->type == IOMAP_HOLE)) { /* If reading from a hole, unlock and return */ - unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1); + unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1, + NULL); return 0; } @@ -7937,7 +7792,7 @@ static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, pos, length, false); else unlock_extent(&BTRFS_I(inode)->io_tree, pos, - pos + length - 1); + pos + length - 1, NULL); ret = -ENOTBLK; } @@ -7962,7 +7817,7 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) } else { unlock_extent(&BTRFS_I(dip->inode)->io_tree, dip->file_offset, - dip->file_offset + dip->bytes - 1); + dip->file_offset + dip->bytes - 1, NULL); } kfree(dip->csums); @@ -7973,7 +7828,7 @@ static void submit_dio_repair_bio(struct inode *inode, struct bio *bio, int mirror_num, enum btrfs_compression_type compress_type) { - struct btrfs_dio_private *dip = bio->bi_private; + struct btrfs_dio_private *dip = btrfs_bio(bio)->private; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); BUG_ON(bio_op(bio) == REQ_OP_WRITE); @@ -7988,8 +7843,6 @@ static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip, { struct inode *inode = dip->inode; struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; - struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; - struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); blk_status_t err = BLK_STS_OK; struct bvec_iter iter; @@ -8002,9 +7855,8 @@ static blk_status_t btrfs_check_read_dio_bio(struct btrfs_dio_private *dip, if (uptodate && (!csum || !btrfs_check_data_csum(inode, bbio, offset, bv.bv_page, bv.bv_offset))) { - clean_io_failure(fs_info, failure_tree, io_tree, start, - bv.bv_page, btrfs_ino(BTRFS_I(inode)), - bv.bv_offset); + btrfs_clean_io_failure(BTRFS_I(inode), start, + bv.bv_page, bv.bv_offset); } else { int ret; @@ -8026,10 +7878,10 @@ static blk_status_t btrfs_submit_bio_start_direct_io(struct inode *inode, return btrfs_csum_one_bio(BTRFS_I(inode), bio, dio_file_offset, false); } -static void btrfs_end_dio_bio(struct bio *bio) +static void btrfs_end_dio_bio(struct btrfs_bio *bbio) { - struct btrfs_dio_private *dip = bio->bi_private; - struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_dio_private *dip = bbio->private; + struct bio *bio = &bbio->bio; blk_status_t err = bio->bi_status; if (err) @@ -8055,7 +7907,7 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, u64 file_offset, int async_submit) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct btrfs_dio_private *dip = bio->bi_private; + struct btrfs_dio_private *dip = btrfs_bio(bio)->private; blk_status_t ret; /* Save the original iter for read repair */ @@ -8078,8 +7930,7 @@ static void btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, */ ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, false); if (ret) { - bio->bi_status = ret; - bio_endio(bio); + btrfs_bio_end_io(btrfs_bio(bio), ret); return; } } else { @@ -8162,9 +8013,8 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, * This will never fail as it's passing GPF_NOFS and * the allocation is backed by btrfs_bioset. */ - bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len); - bio->bi_private = dip; - bio->bi_end_io = btrfs_end_dio_bio; + bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len, + btrfs_end_dio_bio, dip); btrfs_bio(bio)->file_offset = file_offset; if (bio_op(bio) == REQ_OP_ZONE_APPEND) { @@ -8246,6 +8096,25 @@ static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, if (ret) return ret; + /* + * fiemap_prep() called filemap_write_and_wait() for the whole possible + * file range (0 to LLONG_MAX), but that is not enough if we have + * compression enabled. The first filemap_fdatawrite_range() only kicks + * in the compression of data (in an async thread) and will return + * before the compression is done and writeback is started. A second + * filemap_fdatawrite_range() is needed to wait for the compression to + * complete and writeback to start. We also need to wait for ordered + * extents to complete, because our fiemap implementation uses mainly + * file extent items to list the extents, searching for extent maps + * only for file ranges with holes or prealloc extents to figure out + * if we have delalloc in those ranges. + */ + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { + ret = btrfs_wait_ordered_range(inode, 0, LLONG_MAX); + if (ret) + return ret; + } + return extent_fiemap(BTRFS_I(inode), fieinfo, start, len); } @@ -8378,14 +8247,14 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, } if (!inode_evicting) - lock_extent_bits(tree, page_start, page_end, &cached_state); + lock_extent(tree, page_start, page_end, &cached_state); cur = page_start; while (cur < page_end) { struct btrfs_ordered_extent *ordered; - bool delete_states; u64 range_end; u32 range_len; + u32 extra_flags = 0; ordered = btrfs_lookup_first_ordered_range(inode, cur, page_end + 1 - cur); @@ -8395,7 +8264,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * No ordered extent covering this range, we are safe * to delete all extent states in the range. */ - delete_states = true; + extra_flags = EXTENT_CLEAR_ALL_BITS; goto next; } if (ordered->file_offset > cur) { @@ -8406,7 +8275,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * the ordered extent in the next iteration. */ range_end = ordered->file_offset - 1; - delete_states = true; + extra_flags = EXTENT_CLEAR_ALL_BITS; goto next; } @@ -8421,7 +8290,6 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * We can't delete the extent states as * btrfs_finish_ordered_io() may still use some of them. */ - delete_states = false; goto next; } btrfs_page_clear_ordered(fs_info, &folio->page, cur, range_len); @@ -8438,7 +8306,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, clear_extent_bit(tree, cur, range_end, EXTENT_DELALLOC | EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, 1, 0, &cached_state); + EXTENT_DEFRAG, &cached_state); spin_lock_irq(&inode->ordered_tree.lock); set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); @@ -8446,6 +8314,12 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, cur - ordered->file_offset); spin_unlock_irq(&inode->ordered_tree.lock); + /* + * If the ordered extent has finished, we're safe to delete all + * the extent states of the range, otherwise + * btrfs_finish_ordered_io() will get executed by endio for + * other pages, so we can't delete extent states. + */ if (btrfs_dec_test_ordered_pending(inode, &ordered, cur, range_end + 1 - cur)) { btrfs_finish_ordered_io(ordered); @@ -8453,14 +8327,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * The ordered extent has finished, now we're again * safe to delete all extent states of the range. */ - delete_states = true; - } else { - /* - * btrfs_finish_ordered_io() will get executed by endio - * of other pages, thus we can't delete extent states - * anymore - */ - delete_states = false; + extra_flags = EXTENT_CLEAR_ALL_BITS; } next: if (ordered) @@ -8484,8 +8351,8 @@ next: if (!inode_evicting) { clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_UPTODATE | - EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, - delete_states, &cached_state); + EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG | + extra_flags, &cached_state); } cur = range_end + 1; } @@ -8576,11 +8443,11 @@ again: } wait_on_page_writeback(page); - lock_extent_bits(io_tree, page_start, page_end, &cached_state); + lock_extent(io_tree, page_start, page_end, &cached_state); ret2 = set_page_extent_mapped(page); if (ret2 < 0) { ret = vmf_error(ret2); - unlock_extent_cached(io_tree, page_start, page_end, &cached_state); + unlock_extent(io_tree, page_start, page_end, &cached_state); goto out_unlock; } @@ -8591,8 +8458,7 @@ again: ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start, PAGE_SIZE); if (ordered) { - unlock_extent_cached(io_tree, page_start, page_end, - &cached_state); + unlock_extent(io_tree, page_start, page_end, &cached_state); unlock_page(page); up_read(&BTRFS_I(inode)->i_mmap_lock); btrfs_start_ordered_extent(ordered, 1); @@ -8620,13 +8486,12 @@ again: */ clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, 0, 0, &cached_state); + EXTENT_DEFRAG, &cached_state); ret2 = btrfs_set_extent_delalloc(BTRFS_I(inode), page_start, end, 0, &cached_state); if (ret2) { - unlock_extent_cached(io_tree, page_start, page_end, - &cached_state); + unlock_extent(io_tree, page_start, page_end, &cached_state); ret = VM_FAULT_SIGBUS; goto out_unlock; } @@ -8646,7 +8511,7 @@ again: btrfs_set_inode_last_sub_trans(BTRFS_I(inode)); - unlock_extent_cached(io_tree, page_start, page_end, &cached_state); + unlock_extent(io_tree, page_start, page_end, &cached_state); up_read(&BTRFS_I(inode)->i_mmap_lock); btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE); @@ -8747,24 +8612,24 @@ static int btrfs_truncate(struct inode *inode, bool skip_writeback) const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize); control.new_size = new_size; - lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1, + lock_extent(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1, &cached_state); /* * We want to drop from the next block forward in case this new * size is not block aligned since we will be keeping the last * block of the extent just the way it is. */ - btrfs_drop_extent_cache(BTRFS_I(inode), - ALIGN(new_size, fs_info->sectorsize), - (u64)-1, 0); + btrfs_drop_extent_map_range(BTRFS_I(inode), + ALIGN(new_size, fs_info->sectorsize), + (u64)-1, false); ret = btrfs_truncate_inode_items(trans, root, &control); inode_sub_bytes(inode, control.sub_bytes); btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), control.last_size); - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, - (u64)-1, &cached_state); + unlock_extent(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1, + &cached_state); trans->block_rsv = &fs_info->trans_block_rsv; if (ret != -ENOSPC && ret != -EAGAIN) @@ -8895,6 +8760,7 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) ei->last_log_commit = 0; spin_lock_init(&ei->lock); + spin_lock_init(&ei->io_failure_lock); ei->outstanding_extents = 0; if (sb->s_magic != BTRFS_TEST_MAGIC) btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, @@ -8911,12 +8777,9 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) inode = &ei->vfs_inode; extent_map_tree_init(&ei->extent_tree); extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO, inode); - extent_io_tree_init(fs_info, &ei->io_failure_tree, - IO_TREE_INODE_IO_FAILURE, inode); extent_io_tree_init(fs_info, &ei->file_extent_tree, - IO_TREE_INODE_FILE_EXTENT, inode); - ei->io_tree.track_uptodate = true; - ei->io_failure_tree.track_uptodate = true; + IO_TREE_INODE_FILE_EXTENT, NULL); + ei->io_failure_tree = RB_ROOT; atomic_set(&ei->sync_writers, 0); mutex_init(&ei->log_mutex); btrfs_ordered_inode_tree_init(&ei->ordered_tree); @@ -8931,7 +8794,7 @@ struct inode *btrfs_alloc_inode(struct super_block *sb) #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS void btrfs_test_destroy_inode(struct inode *inode) { - btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); } #endif @@ -8946,6 +8809,7 @@ void btrfs_destroy_inode(struct inode *vfs_inode) struct btrfs_ordered_extent *ordered; struct btrfs_inode *inode = BTRFS_I(vfs_inode); struct btrfs_root *root = inode->root; + bool freespace_inode; WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); WARN_ON(vfs_inode->i_data.nrpages); @@ -8967,6 +8831,12 @@ void btrfs_destroy_inode(struct inode *vfs_inode) if (!root) return; + /* + * If this is a free space inode do not take the ordered extents lockdep + * map. + */ + freespace_inode = btrfs_is_free_space_inode(inode); + while (1) { ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); if (!ordered) @@ -8975,6 +8845,10 @@ void btrfs_destroy_inode(struct inode *vfs_inode) btrfs_err(root->fs_info, "found ordered extent %llu %llu on inode cleanup", ordered->file_offset, ordered->num_bytes); + + if (!freespace_inode) + btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent); + btrfs_remove_ordered_extent(inode, ordered); btrfs_put_ordered_extent(ordered); btrfs_put_ordered_extent(ordered); @@ -8982,7 +8856,7 @@ void btrfs_destroy_inode(struct inode *vfs_inode) } btrfs_qgroup_check_reserved_leak(inode); inode_tree_del(inode); - btrfs_drop_extent_cache(inode, 0, (u64)-1, 0); + btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1); btrfs_put_root(inode->root); } @@ -9995,7 +9869,6 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode, struct btrfs_trans_handle *trans) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; struct extent_map *em; struct btrfs_root *root = BTRFS_I(inode)->root; struct btrfs_key ins; @@ -10051,11 +9924,10 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode, break; } - btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, - cur_offset + ins.offset -1, 0); - em = alloc_extent_map(); if (!em) { + btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset, + cur_offset + ins.offset - 1, false); btrfs_set_inode_full_sync(BTRFS_I(inode)); goto next; } @@ -10070,16 +9942,7 @@ static int __btrfs_prealloc_file_range(struct inode *inode, int mode, set_bit(EXTENT_FLAG_PREALLOC, &em->flags); em->generation = trans->transid; - while (1) { - write_lock(&em_tree->lock); - ret = add_extent_mapping(em_tree, em, 1); - write_unlock(&em_tree->lock); - if (ret != -EEXIST) - break; - btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset, - cur_offset + ins.offset - 1, - 0); - } + ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true); free_extent_map(em); next: num_bytes -= ins.offset; @@ -10155,7 +10018,7 @@ static int btrfs_permission(struct user_namespace *mnt_userns, } static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb); struct btrfs_trans_handle *trans; @@ -10163,7 +10026,7 @@ static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, struct inode *inode; struct btrfs_new_inode_args new_inode_args = { .dir = dir, - .dentry = dentry, + .dentry = file->f_path.dentry, .orphan = true, }; unsigned int trans_num_items; @@ -10200,7 +10063,7 @@ static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, set_nlink(inode, 1); if (!ret) { - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); unlock_new_inode(inode); mark_inode_dirty(inode); } @@ -10212,7 +10075,7 @@ out_new_inode_args: out_inode: if (ret) iput(inode); - return ret; + return finish_open_simple(file, ret); } void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end) @@ -10333,7 +10196,7 @@ static ssize_t btrfs_encoded_read_inline( } read_extent_buffer(leaf, tmp, ptr, count); btrfs_release_path(path); - unlock_extent_cached(io_tree, start, lockend, cached_state); + unlock_extent(io_tree, start, lockend, cached_state); btrfs_inode_unlock(&inode->vfs_inode, BTRFS_ILOCK_SHARED); *unlocked = true; @@ -10358,7 +10221,7 @@ struct btrfs_encoded_read_private { static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, struct bio *bio, int mirror_num) { - struct btrfs_encoded_read_private *priv = bio->bi_private; + struct btrfs_encoded_read_private *priv = btrfs_bio(bio)->private; struct btrfs_fs_info *fs_info = inode->root->fs_info; blk_status_t ret; @@ -10376,7 +10239,7 @@ static blk_status_t submit_encoded_read_bio(struct btrfs_inode *inode, static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio) { const bool uptodate = (bbio->bio.bi_status == BLK_STS_OK); - struct btrfs_encoded_read_private *priv = bbio->bio.bi_private; + struct btrfs_encoded_read_private *priv = bbio->private; struct btrfs_inode *inode = priv->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; u32 sectorsize = fs_info->sectorsize; @@ -10404,10 +10267,9 @@ static blk_status_t btrfs_encoded_read_verify_csum(struct btrfs_bio *bbio) return BLK_STS_OK; } -static void btrfs_encoded_read_endio(struct bio *bio) +static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) { - struct btrfs_encoded_read_private *priv = bio->bi_private; - struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_encoded_read_private *priv = bbio->private; blk_status_t status; status = btrfs_encoded_read_verify_csum(bbio); @@ -10425,7 +10287,7 @@ static void btrfs_encoded_read_endio(struct bio *bio) if (!atomic_dec_return(&priv->pending)) wake_up(&priv->wait); btrfs_bio_free_csum(bbio); - bio_put(bio); + bio_put(&bbio->bio); } int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, @@ -10472,12 +10334,11 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, size_t bytes = min_t(u64, remaining, PAGE_SIZE); if (!bio) { - bio = btrfs_bio_alloc(BIO_MAX_VECS); + bio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, + btrfs_encoded_read_endio, + &priv); bio->bi_iter.bi_sector = (disk_bytenr + cur) >> SECTOR_SHIFT; - bio->bi_end_io = btrfs_encoded_read_endio; - bio->bi_private = &priv; - bio->bi_opf = REQ_OP_READ; } if (!bytes || @@ -10538,7 +10399,7 @@ static ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, if (ret) goto out; - unlock_extent_cached(io_tree, start, lockend, cached_state); + unlock_extent(io_tree, start, lockend, cached_state); btrfs_inode_unlock(&inode->vfs_inode, BTRFS_ILOCK_SHARED); *unlocked = true; @@ -10608,13 +10469,13 @@ ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, lockend - start + 1); if (ret) goto out_unlock_inode; - lock_extent_bits(io_tree, start, lockend, &cached_state); + lock_extent(io_tree, start, lockend, &cached_state); ordered = btrfs_lookup_ordered_range(inode, start, lockend - start + 1); if (!ordered) break; btrfs_put_ordered_extent(ordered); - unlock_extent_cached(io_tree, start, lockend, &cached_state); + unlock_extent(io_tree, start, lockend, &cached_state); cond_resched(); } @@ -10688,7 +10549,7 @@ ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, em = NULL; if (disk_bytenr == EXTENT_MAP_HOLE) { - unlock_extent_cached(io_tree, start, lockend, &cached_state); + unlock_extent(io_tree, start, lockend, &cached_state); btrfs_inode_unlock(&inode->vfs_inode, BTRFS_ILOCK_SHARED); unlocked = true; ret = iov_iter_zero(count, iter); @@ -10709,7 +10570,7 @@ out_em: free_extent_map(em); out_unlock_extent: if (!unlocked) - unlock_extent_cached(io_tree, start, lockend, &cached_state); + unlock_extent(io_tree, start, lockend, &cached_state); out_unlock_inode: if (!unlocked) btrfs_inode_unlock(&inode->vfs_inode, BTRFS_ILOCK_SHARED); @@ -10847,14 +10708,14 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, end >> PAGE_SHIFT); if (ret) goto out_pages; - lock_extent_bits(io_tree, start, end, &cached_state); + lock_extent(io_tree, start, end, &cached_state); ordered = btrfs_lookup_ordered_range(inode, start, num_bytes); if (!ordered && !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end)) break; if (ordered) btrfs_put_ordered_extent(ordered); - unlock_extent_cached(io_tree, start, end, &cached_state); + unlock_extent(io_tree, start, end, &cached_state); cond_resched(); } @@ -10908,7 +10769,7 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, (1 << BTRFS_ORDERED_COMPRESSED), compression); if (ret) { - btrfs_drop_extent_cache(inode, start, end, 0); + btrfs_drop_extent_map_range(inode, start, end, false); goto out_free_reserved; } btrfs_dec_block_group_reservations(fs_info, ins.objectid); @@ -10916,7 +10777,7 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, if (start + encoded->len > inode->vfs_inode.i_size) i_size_write(&inode->vfs_inode, start + encoded->len); - unlock_extent_cached(io_tree, start, end, &cached_state); + unlock_extent(io_tree, start, end, &cached_state); btrfs_delalloc_release_extents(inode, num_bytes); @@ -10947,7 +10808,7 @@ out_free_data_space: if (!extent_reserved) btrfs_free_reserved_data_space_noquota(fs_info, disk_num_bytes); out_unlock: - unlock_extent_cached(io_tree, start, end, &cached_state); + unlock_extent(io_tree, start, end, &cached_state); out_pages: for (i = 0; i < nr_pages; i++) { if (pages[i]) @@ -11188,7 +11049,7 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); - lock_extent_bits(io_tree, 0, isize - 1, &cached_state); + lock_extent(io_tree, 0, isize - 1, &cached_state); start = 0; while (start < isize) { u64 logical_block_start, physical_block_start; @@ -11229,7 +11090,7 @@ static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, free_extent_map(em); em = NULL; - ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, true); + ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, false, true); if (ret < 0) { goto out; } else if (ret) { @@ -11325,7 +11186,7 @@ out: if (!IS_ERR_OR_NULL(em)) free_extent_map(em); - unlock_extent_cached(io_tree, 0, isize - 1, &cached_state); + unlock_extent(io_tree, 0, isize - 1, &cached_state); if (ret) btrfs_swap_deactivate(file); diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index fe0cc816b4eba2654470cb26c768a031f04c438c..d5dd8bed1488a807d9ec4e5bb0834b8b6c0b4f1e 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -1218,10 +1218,10 @@ static struct extent_map *defrag_lookup_extent(struct inode *inode, u64 start, /* get the big lock and read metadata off disk */ if (!locked) - lock_extent_bits(io_tree, start, end, &cached); + lock_extent(io_tree, start, end, &cached); em = defrag_get_extent(BTRFS_I(inode), start, newer_than); if (!locked) - unlock_extent_cached(io_tree, start, end, &cached); + unlock_extent(io_tree, start, end, &cached); if (IS_ERR(em)) return NULL; @@ -1333,10 +1333,10 @@ again: while (1) { struct btrfs_ordered_extent *ordered; - lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state); + lock_extent(&inode->io_tree, page_start, page_end, &cached_state); ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); - unlock_extent_cached(&inode->io_tree, page_start, page_end, - &cached_state); + unlock_extent(&inode->io_tree, page_start, page_end, + &cached_state); if (!ordered) break; @@ -1616,7 +1616,7 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, return ret; clear_extent_bit(&inode->io_tree, start, start + len - 1, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | - EXTENT_DEFRAG, 0, 0, cached_state); + EXTENT_DEFRAG, cached_state); set_extent_defrag(&inode->io_tree, start, start + len - 1, cached_state); /* Update the page status */ @@ -1666,9 +1666,9 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, wait_on_page_writeback(pages[i]); /* Lock the pages range */ - lock_extent_bits(&inode->io_tree, start_index << PAGE_SHIFT, - (last_index << PAGE_SHIFT) + PAGE_SIZE - 1, - &cached_state); + lock_extent(&inode->io_tree, start_index << PAGE_SHIFT, + (last_index << PAGE_SHIFT) + PAGE_SIZE - 1, + &cached_state); /* * Now we have a consistent view about the extent map, re-check * which range really needs to be defragged. @@ -1694,9 +1694,9 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, kfree(entry); } unlock_extent: - unlock_extent_cached(&inode->io_tree, start_index << PAGE_SHIFT, - (last_index << PAGE_SHIFT) + PAGE_SIZE - 1, - &cached_state); + unlock_extent(&inode->io_tree, start_index << PAGE_SHIFT, + (last_index << PAGE_SHIFT) + PAGE_SIZE - 1, + &cached_state); free_pages: for (i = 0; i < nr_pages; i++) { if (pages[i]) { diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index 9063072b399bd833423b7faeb84f523a54ef0f88..0eab3cb274a183862c2dc4771ef3ee556f0c59f4 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -285,6 +285,31 @@ struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) return eb; } +/* + * Loop around taking references on and locking the root node of the tree in + * nowait mode until we end up with a lock on the root node or returning to + * avoid blocking. + * + * Return: root extent buffer with read lock held or -EAGAIN. + */ +struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root) +{ + struct extent_buffer *eb; + + while (1) { + eb = btrfs_root_node(root); + if (!btrfs_try_tree_read_lock(eb)) { + free_extent_buffer(eb); + return ERR_PTR(-EAGAIN); + } + if (eb == root->node) + break; + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + } + return eb; +} + /* * DREW locks * ========== diff --git a/fs/btrfs/locking.h b/fs/btrfs/locking.h index ab268be09bb542fe5df6aa53fd59c9f6977f68a6..490c7a79e9959871c4c5e277fec02e955e87e824 100644 --- a/fs/btrfs/locking.h +++ b/fs/btrfs/locking.h @@ -94,6 +94,7 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb); int btrfs_try_tree_write_lock(struct extent_buffer *eb); struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root); struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root); +struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root); #ifdef CONFIG_BTRFS_DEBUG static inline void btrfs_assert_tree_write_locked(struct extent_buffer *eb) diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h index 340f995652f2dac699fcaadaa4b2c0c7346327b3..f9850edfd726793344f2bafb557338858de0ba75 100644 --- a/fs/btrfs/misc.h +++ b/fs/btrfs/misc.h @@ -88,6 +88,41 @@ static inline struct rb_node *rb_simple_search(struct rb_root *root, u64 bytenr) return NULL; } +/* + * Search @root from an entry that starts or comes after @bytenr. + * + * @root: the root to search. + * @bytenr: bytenr to search from. + * + * Return the rb_node that start at or after @bytenr. If there is no entry at + * or after @bytner return NULL. + */ +static inline struct rb_node *rb_simple_search_first(struct rb_root *root, + u64 bytenr) +{ + struct rb_node *node = root->rb_node, *ret = NULL; + struct rb_simple_node *entry, *ret_entry = NULL; + + while (node) { + entry = rb_entry(node, struct rb_simple_node, rb_node); + + if (bytenr < entry->bytenr) { + if (!ret || entry->bytenr < ret_entry->bytenr) { + ret = node; + ret_entry = entry; + } + + node = node->rb_left; + } else if (bytenr > entry->bytenr) { + node = node->rb_right; + } else { + return node; + } + } + + return ret; +} + static inline struct rb_node *rb_simple_insert(struct rb_root *root, u64 bytenr, struct rb_node *node) { diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 1952ac85222c0bb429fda21162f4d7827149ec12..e54f8280031fa14e29ae276e73b0eaf5b431cd3b 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -524,7 +524,15 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *node; bool pending; + bool freespace_inode; + /* + * If this is a free space inode the thread has not acquired the ordered + * extents lockdep map. + */ + freespace_inode = btrfs_is_free_space_inode(btrfs_inode); + + btrfs_lockdep_acquire(fs_info, btrfs_trans_pending_ordered); /* This is paired with btrfs_add_ordered_extent. */ spin_lock(&btrfs_inode->lock); btrfs_mod_outstanding_extents(btrfs_inode, -1); @@ -580,6 +588,8 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, } } + btrfs_lockdep_release(fs_info, btrfs_trans_pending_ordered); + spin_lock(&root->ordered_extent_lock); list_del_init(&entry->root_extent_list); root->nr_ordered_extents--; @@ -594,6 +604,8 @@ void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, } spin_unlock(&root->ordered_extent_lock); wake_up(&entry->wait); + if (!freespace_inode) + btrfs_lockdep_release(fs_info, btrfs_ordered_extent); } static void btrfs_run_ordered_extent_work(struct btrfs_work *work) @@ -712,9 +724,16 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) u64 start = entry->file_offset; u64 end = start + entry->num_bytes - 1; struct btrfs_inode *inode = BTRFS_I(entry->inode); + bool freespace_inode; trace_btrfs_ordered_extent_start(inode, entry); + /* + * If this is a free space inode do not take the ordered extents lockdep + * map. + */ + freespace_inode = btrfs_is_free_space_inode(inode); + /* * pages in the range can be dirty, clean or writeback. We * start IO on any dirty ones so the wait doesn't stall waiting @@ -723,6 +742,8 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait) if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags)) filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, end); if (wait) { + if (!freespace_inode) + btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)); } @@ -1022,7 +1043,7 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, cachedp = cached_state; while (1) { - lock_extent_bits(&inode->io_tree, start, end, cachedp); + lock_extent(&inode->io_tree, start, end, cachedp); ordered = btrfs_lookup_ordered_range(inode, start, end - start + 1); if (!ordered) { @@ -1035,12 +1056,37 @@ void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, refcount_dec(&cache->refs); break; } - unlock_extent_cached(&inode->io_tree, start, end, cachedp); + unlock_extent(&inode->io_tree, start, end, cachedp); btrfs_start_ordered_extent(ordered, 1); btrfs_put_ordered_extent(ordered); } } +/* + * Lock the passed range and ensure all pending ordered extents in it are run + * to completion in nowait mode. + * + * Return true if btrfs_lock_ordered_range does not return any extents, + * otherwise false. + */ +bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end) +{ + struct btrfs_ordered_extent *ordered; + + if (!try_lock_extent(&inode->io_tree, start, end)) + return false; + + ordered = btrfs_lookup_ordered_range(inode, start, end - start + 1); + if (!ordered) + return true; + + btrfs_put_ordered_extent(ordered); + unlock_extent(&inode->io_tree, start, end, NULL); + + return false; +} + + static int clone_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pos, u64 len) { diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index 87792f85e2c4ae376bf66b35ac1940ffcee3bc90..f59f2dbdb25edeed2c0efd9318c8645d6ba8be8b 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -160,18 +160,6 @@ struct btrfs_ordered_extent { struct block_device *bdev; }; -/* - * calculates the total size you need to allocate for an ordered sum - * structure spanning 'bytes' in the file - */ -static inline int btrfs_ordered_sum_size(struct btrfs_fs_info *fs_info, - unsigned long bytes) -{ - int num_sectors = (int)DIV_ROUND_UP(bytes, fs_info->sectorsize); - - return sizeof(struct btrfs_ordered_sum) + num_sectors * fs_info->csum_size; -} - static inline void btrfs_ordered_inode_tree_init(struct btrfs_ordered_inode_tree *t) { @@ -218,6 +206,7 @@ void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, struct extent_state **cached_state); +bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end); int btrfs_split_ordered_extent(struct btrfs_ordered_extent *ordered, u64 pre, u64 post); int __init ordered_data_init(void); diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index a2ec8ecae8dedeb7fba51c18e9c79394fb060275..055a631276ce110300cfbbb5c911e9dc9368fb70 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -270,11 +270,8 @@ int btrfs_load_inode_props(struct inode *inode, struct btrfs_path *path) { struct btrfs_root *root = BTRFS_I(inode)->root; u64 ino = btrfs_ino(BTRFS_I(inode)); - int ret; - - ret = iterate_object_props(root, path, ino, inode_prop_iterator, inode); - return ret; + return iterate_object_props(root, path, ino, inode_prop_iterator, inode); } static int prop_compression_validate(const struct btrfs_inode *inode, diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index db723c0026bd2ef011b071b17c6e16ac5d2e8dbd..9334c3157c22e89ca242daad82705e5c5229d8c3 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -275,7 +275,7 @@ static int __add_relation_rb(struct btrfs_qgroup *member, struct btrfs_qgroup *p } /* - * Add relation specified by two qgoup ids. + * Add relation specified by two qgroup ids. * * Must be called with qgroup_lock held. * @@ -333,6 +333,13 @@ int btrfs_verify_qgroup_counts(struct btrfs_fs_info *fs_info, u64 qgroupid, } #endif +static void qgroup_mark_inconsistent(struct btrfs_fs_info *fs_info) +{ + fs_info->qgroup_flags |= (BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT | + BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN | + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING); +} + /* * The full config is read in one go, only called from open_ctree() * It doesn't use any locking, as at this point we're still single-threaded @@ -401,7 +408,7 @@ int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info) } if (btrfs_qgroup_status_generation(l, ptr) != fs_info->generation) { - flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); btrfs_err(fs_info, "qgroup generation mismatch, marked as inconsistent"); } @@ -419,7 +426,7 @@ int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info) if ((qgroup && found_key.type == BTRFS_QGROUP_INFO_KEY) || (!qgroup && found_key.type == BTRFS_QGROUP_LIMIT_KEY)) { btrfs_err(fs_info, "inconsistent qgroup config"); - flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); } if (!qgroup) { qgroup = add_qgroup_rb(fs_info, found_key.offset); @@ -878,7 +885,8 @@ static int update_qgroup_status_item(struct btrfs_trans_handle *trans) l = path->nodes[0]; slot = path->slots[0]; ptr = btrfs_item_ptr(l, slot, struct btrfs_qgroup_status_item); - btrfs_set_qgroup_status_flags(l, ptr, fs_info->qgroup_flags); + btrfs_set_qgroup_status_flags(l, ptr, fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAGS_MASK); btrfs_set_qgroup_status_generation(l, ptr, trans->transid); btrfs_set_qgroup_status_rescan(l, ptr, fs_info->qgroup_rescan_progress.objectid); @@ -1052,7 +1060,8 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info) btrfs_set_qgroup_status_version(leaf, ptr, BTRFS_QGROUP_STATUS_VERSION); fs_info->qgroup_flags = BTRFS_QGROUP_STATUS_FLAG_ON | BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; - btrfs_set_qgroup_status_flags(leaf, ptr, fs_info->qgroup_flags); + btrfs_set_qgroup_status_flags(leaf, ptr, fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAGS_MASK); btrfs_set_qgroup_status_rescan(leaf, ptr, 0); btrfs_mark_buffer_dirty(leaf); @@ -1174,6 +1183,21 @@ out_add_root: fs_info->qgroup_rescan_running = true; btrfs_queue_work(fs_info->qgroup_rescan_workers, &fs_info->qgroup_rescan_work); + } else { + /* + * We have set both BTRFS_FS_QUOTA_ENABLED and + * BTRFS_QGROUP_STATUS_FLAG_ON, so we can only fail with + * -EINPROGRESS. That can happen because someone started the + * rescan worker by calling quota rescan ioctl before we + * attempted to initialize the rescan worker. Failure due to + * quotas disabled in the meanwhile is not possible, because + * we are holding a write lock on fs_info->subvol_sem, which + * is also acquired when disabling quotas. + * Ignore such error, and any other error would need to undo + * everything we did in the transaction we just committed. + */ + ASSERT(ret == -EINPROGRESS); + ret = 0; } out_free_path: @@ -1255,6 +1279,7 @@ int btrfs_quota_disable(struct btrfs_fs_info *fs_info) quota_root = fs_info->quota_root; fs_info->quota_root = NULL; fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON; + fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL; spin_unlock(&fs_info->qgroup_lock); btrfs_free_qgroup_config(fs_info); @@ -1717,7 +1742,7 @@ int btrfs_limit_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid, ret = update_qgroup_limit_item(trans, qgroup); if (ret) { - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); btrfs_info(fs_info, "unable to update quota limit for %llu", qgroupid); } @@ -1790,10 +1815,13 @@ int btrfs_qgroup_trace_extent_post(struct btrfs_trans_handle *trans, */ ASSERT(trans != NULL); + if (trans->fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING) + return 0; + ret = btrfs_find_all_roots(NULL, trans->fs_info, bytenr, 0, &old_root, true); if (ret < 0) { - trans->fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(trans->fs_info); btrfs_warn(trans->fs_info, "error accounting new delayed refs extent (err code: %d), quota inconsistent", ret); @@ -2269,7 +2297,7 @@ static int qgroup_trace_subtree_swap(struct btrfs_trans_handle *trans, out: btrfs_free_path(dst_path); if (ret < 0) - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); return ret; } @@ -2280,6 +2308,7 @@ int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = trans->fs_info; int ret = 0; int level; + u8 drop_subptree_thres; struct extent_buffer *eb = root_eb; struct btrfs_path *path = NULL; @@ -2289,6 +2318,23 @@ int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans, if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) return 0; + spin_lock(&fs_info->qgroup_lock); + drop_subptree_thres = fs_info->qgroup_drop_subtree_thres; + spin_unlock(&fs_info->qgroup_lock); + + /* + * This function only gets called for snapshot drop, if we hit a high + * node here, it means we are going to change ownership for quite a lot + * of extents, which will greatly slow down btrfs_commit_transaction(). + * + * So here if we find a high tree here, we just skip the accounting and + * mark qgroup inconsistent. + */ + if (root_level >= drop_subptree_thres) { + qgroup_mark_inconsistent(fs_info); + return 0; + } + if (!extent_buffer_uptodate(root_eb)) { ret = btrfs_read_extent_buffer(root_eb, root_gen, root_level, NULL); if (ret) @@ -2604,7 +2650,8 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr, * If quotas get disabled meanwhile, the resources need to be freed and * we can't just exit here. */ - if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) + if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags) || + fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING) goto out_free; if (new_roots) { @@ -2700,7 +2747,8 @@ int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans) num_dirty_extents++; trace_btrfs_qgroup_account_extents(fs_info, record); - if (!ret) { + if (!ret && !(fs_info->qgroup_flags & + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)) { /* * Old roots should be searched when inserting qgroup * extent record @@ -2773,12 +2821,10 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans) spin_unlock(&fs_info->qgroup_lock); ret = update_qgroup_info_item(trans, qgroup); if (ret) - fs_info->qgroup_flags |= - BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); ret = update_qgroup_limit_item(trans, qgroup); if (ret) - fs_info->qgroup_flags |= - BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); spin_lock(&fs_info->qgroup_lock); } if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) @@ -2789,7 +2835,7 @@ int btrfs_run_qgroups(struct btrfs_trans_handle *trans) ret = update_qgroup_status_item(trans); if (ret) - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); return ret; } @@ -2907,7 +2953,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, ret = update_qgroup_limit_item(trans, dstgroup); if (ret) { - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); btrfs_info(fs_info, "unable to update quota limit for %llu", dstgroup->qgroupid); @@ -3013,7 +3059,7 @@ out: if (!committing) mutex_unlock(&fs_info->qgroup_ioctl_lock); if (need_rescan) - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); return ret; } @@ -3286,7 +3332,8 @@ static bool rescan_should_stop(struct btrfs_fs_info *fs_info) { return btrfs_fs_closing(fs_info) || test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state) || - !test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + !test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags) || + fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN; } static void btrfs_qgroup_rescan_worker(struct btrfs_work *work) @@ -3351,7 +3398,8 @@ out: } mutex_lock(&fs_info->qgroup_rescan_lock); - if (!stopped) + if (!stopped || + fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN; if (trans) { ret = update_qgroup_status_item(trans); @@ -3362,6 +3410,7 @@ out: } } fs_info->qgroup_rescan_running = false; + fs_info->qgroup_flags &= ~BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN; complete_all(&fs_info->qgroup_rescan_completion); mutex_unlock(&fs_info->qgroup_rescan_lock); @@ -3372,6 +3421,8 @@ out: if (stopped) { btrfs_info(fs_info, "qgroup scan paused"); + } else if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) { + btrfs_info(fs_info, "qgroup scan cancelled"); } else if (err >= 0) { btrfs_info(fs_info, "qgroup scan completed%s", err > 0 ? " (inconsistency flag cleared)" : ""); @@ -3434,6 +3485,8 @@ qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, memset(&fs_info->qgroup_rescan_progress, 0, sizeof(fs_info->qgroup_rescan_progress)); + fs_info->qgroup_flags &= ~(BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN | + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING); fs_info->qgroup_rescan_progress.objectid = progress_objectid; init_completion(&fs_info->qgroup_rescan_completion); mutex_unlock(&fs_info->qgroup_rescan_lock); @@ -4231,8 +4284,7 @@ out_unlock: spin_unlock(&blocks->lock); out: if (ret < 0) - fs_info->qgroup_flags |= - BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); return ret; } @@ -4319,7 +4371,7 @@ out: btrfs_err_rl(fs_info, "failed to account subtree at bytenr %llu: %d", subvol_eb->start, ret); - fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + qgroup_mark_inconsistent(fs_info); } return ret; } diff --git a/fs/btrfs/qgroup.h b/fs/btrfs/qgroup.h index 0c4dd2a9af96b07ec93f445a7ca60a2a4ce854f7..578c77e94200f580918149609673588444be9a2f 100644 --- a/fs/btrfs/qgroup.h +++ b/fs/btrfs/qgroup.h @@ -100,6 +100,9 @@ * subtree rescan for them. */ +#define BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN (1UL << 3) +#define BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING (1UL << 4) + /* * Record a dirty extent, and info qgroup to update quota on it * TODO: Use kmem cache to alloc it. diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 2feb5c20641aecc0107fbe5a3c9ae35cdfc35bcc..f6395e8288d69b5dc95ca6d24dbc994bac9bed45 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -275,7 +275,6 @@ static void merge_rbio(struct btrfs_raid_bio *dest, /* Also inherit the bitmaps from @victim. */ bitmap_or(&dest->dbitmap, &victim->dbitmap, &dest->dbitmap, dest->stripe_nsectors); - dest->generic_bio_cnt += victim->generic_bio_cnt; bio_list_init(&victim->bio_list); } @@ -814,8 +813,6 @@ static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err) struct bio *cur = bio_list_get(&rbio->bio_list); struct bio *extra; - if (rbio->generic_bio_cnt) - btrfs_bio_counter_sub(rbio->bioc->fs_info, rbio->generic_bio_cnt); /* * Clear the data bitmap, as the rbio may be cached for later usage. * do this before before unlock_stripe() so there will be no new bio @@ -946,6 +943,7 @@ static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info, spin_lock_init(&rbio->bio_list_lock); INIT_LIST_HEAD(&rbio->stripe_cache); INIT_LIST_HEAD(&rbio->hash_list); + btrfs_get_bioc(bioc); rbio->bioc = bioc; rbio->nr_pages = num_pages; rbio->nr_sectors = num_sectors; @@ -1813,15 +1811,12 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) rbio = alloc_rbio(fs_info, bioc); if (IS_ERR(rbio)) { - btrfs_put_bioc(bioc); ret = PTR_ERR(rbio); - goto out_dec_counter; + goto fail; } rbio->operation = BTRFS_RBIO_WRITE; rbio_add_bio(rbio, bio); - rbio->generic_bio_cnt = 1; - /* * don't plug on full rbios, just get them out the door * as quickly as we can @@ -1829,7 +1824,7 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) if (rbio_is_full(rbio)) { ret = full_stripe_write(rbio); if (ret) - goto out_dec_counter; + goto fail; return; } @@ -1844,13 +1839,12 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) } else { ret = __raid56_parity_write(rbio); if (ret) - goto out_dec_counter; + goto fail; } return; -out_dec_counter: - btrfs_bio_counter_dec(fs_info); +fail: bio->bi_status = errno_to_blk_status(ret); bio_endio(bio); } @@ -2198,18 +2192,11 @@ cleanup: * of the drive. */ void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, - int mirror_num, bool generic_io) + int mirror_num) { struct btrfs_fs_info *fs_info = bioc->fs_info; struct btrfs_raid_bio *rbio; - if (generic_io) { - ASSERT(bioc->mirror_num == mirror_num); - btrfs_bio(bio)->mirror_num = mirror_num; - } else { - btrfs_get_bioc(bioc); - } - rbio = alloc_rbio(fs_info, bioc); if (IS_ERR(rbio)) { bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); @@ -2225,14 +2212,11 @@ void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, "%s could not find the bad stripe in raid56 so that we cannot recover any more (bio has logical %llu len %llu, bioc has map_type %llu)", __func__, bio->bi_iter.bi_sector << 9, (u64)bio->bi_iter.bi_size, bioc->map_type); - kfree(rbio); + __free_raid_bio(rbio); bio->bi_status = BLK_STS_IOERR; goto out_end_bio; } - if (generic_io) - rbio->generic_bio_cnt = 1; - /* * Loop retry: * for 'mirror == 2', reconstruct from all other stripes. @@ -2261,8 +2245,6 @@ void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, return; out_end_bio: - btrfs_bio_counter_dec(fs_info); - btrfs_put_bioc(bioc); bio_endio(bio); } @@ -2326,13 +2308,6 @@ struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio, ASSERT(i < rbio->real_stripes); bitmap_copy(&rbio->dbitmap, dbitmap, stripe_nsectors); - - /* - * We have already increased bio_counter when getting bioc, record it - * so we can free it at rbio_orig_end_io(). - */ - rbio->generic_bio_cnt = 1; - return rbio; } @@ -2772,12 +2747,6 @@ raid56_alloc_missing_rbio(struct bio *bio, struct btrfs_io_context *bioc) return NULL; } - /* - * When we get bioc, we have already increased bio_counter, record it - * so we can free it at rbio_orig_end_io() - */ - rbio->generic_bio_cnt = 1; - return rbio; } diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h index 6f48f9e4c869416feef97d38ad7601c33155f4e8..91d5c0adad151edcb378e895d5381e7e44b7aab6 100644 --- a/fs/btrfs/raid56.h +++ b/fs/btrfs/raid56.h @@ -89,8 +89,6 @@ struct btrfs_raid_bio { */ int bio_list_bytes; - int generic_bio_cnt; - refcount_t refs; atomic_t stripes_pending; @@ -166,7 +164,7 @@ static inline int nr_data_stripes(const struct map_lookup *map) struct btrfs_device; void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, - int mirror_num, bool generic_io); + int mirror_num); void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc); void raid56_add_scrub_pages(struct btrfs_raid_bio *rbio, struct page *page, diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 9acf47b11fe63e473a00b3c98ed89e4231a6a5f8..f50586ff85c84258c74d31aabc73fe24270854b9 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -92,7 +92,7 @@ static int copy_inline_to_page(struct btrfs_inode *inode, clear_extent_bit(&inode->io_tree, file_offset, range_end, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, - 0, 0, NULL); + NULL); ret = btrfs_set_extent_delalloc(inode, file_offset, range_end, 0, NULL); if (ret) goto out_unlock; @@ -615,8 +615,8 @@ out: static void btrfs_double_extent_unlock(struct inode *inode1, u64 loff1, struct inode *inode2, u64 loff2, u64 len) { - unlock_extent(&BTRFS_I(inode1)->io_tree, loff1, loff1 + len - 1); - unlock_extent(&BTRFS_I(inode2)->io_tree, loff2, loff2 + len - 1); + unlock_extent(&BTRFS_I(inode1)->io_tree, loff1, loff1 + len - 1, NULL); + unlock_extent(&BTRFS_I(inode2)->io_tree, loff2, loff2 + len - 1, NULL); } static void btrfs_double_extent_lock(struct inode *inode1, u64 loff1, @@ -634,8 +634,8 @@ static void btrfs_double_extent_lock(struct inode *inode1, u64 loff1, swap(range1_end, range2_end); } - lock_extent(&BTRFS_I(inode1)->io_tree, loff1, range1_end); - lock_extent(&BTRFS_I(inode2)->io_tree, loff2, range2_end); + lock_extent(&BTRFS_I(inode1)->io_tree, loff1, range1_end, NULL); + lock_extent(&BTRFS_I(inode2)->io_tree, loff2, range2_end, NULL); btrfs_assert_inode_range_clean(BTRFS_I(inode1), loff1, range1_end); btrfs_assert_inode_range_clean(BTRFS_I(inode2), loff2, range2_end); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 45c02aba2492b3f014dc70712fa6fda829d7ee06..666a37a0ee89786ed45ec596f252d79e78d86cc5 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1124,10 +1124,10 @@ int replace_file_extents(struct btrfs_trans_handle *trans, if (!ret) continue; - btrfs_drop_extent_cache(BTRFS_I(inode), - key.offset, end, 1); + btrfs_drop_extent_map_range(BTRFS_I(inode), + key.offset, end, true); unlock_extent(&BTRFS_I(inode)->io_tree, - key.offset, end); + key.offset, end, NULL); } } @@ -1566,9 +1566,9 @@ static int invalidate_extent_cache(struct btrfs_root *root, } /* the lock_extent waits for read_folio to complete */ - lock_extent(&BTRFS_I(inode)->io_tree, start, end); - btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 1); - unlock_extent(&BTRFS_I(inode)->io_tree, start, end); + lock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); + btrfs_drop_extent_map_range(BTRFS_I(inode), start, end, true); + unlock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); } return 0; } @@ -2869,13 +2869,13 @@ static noinline_for_stack int prealloc_file_extent_cluster( else end = cluster->end - offset; - lock_extent(&inode->io_tree, start, end); + lock_extent(&inode->io_tree, start, end, NULL); num_bytes = end + 1 - start; ret = btrfs_prealloc_file_range(&inode->vfs_inode, 0, start, num_bytes, num_bytes, end + 1, &alloc_hint); cur_offset = end + 1; - unlock_extent(&inode->io_tree, start, end); + unlock_extent(&inode->io_tree, start, end, NULL); if (ret) break; } @@ -2890,7 +2890,6 @@ static noinline_for_stack int prealloc_file_extent_cluster( static noinline_for_stack int setup_relocation_extent_mapping(struct inode *inode, u64 start, u64 end, u64 block_start) { - struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; struct extent_map *em; int ret = 0; @@ -2904,18 +2903,11 @@ static noinline_for_stack int setup_relocation_extent_mapping(struct inode *inod em->block_start = block_start; set_bit(EXTENT_FLAG_PINNED, &em->flags); - lock_extent(&BTRFS_I(inode)->io_tree, start, end); - while (1) { - write_lock(&em_tree->lock); - ret = add_extent_mapping(em_tree, em, 0); - write_unlock(&em_tree->lock); - if (ret != -EEXIST) { - free_extent_map(em); - break; - } - btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0); - } - unlock_extent(&BTRFS_I(inode)->io_tree, start, end); + lock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); + ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, false); + unlock_extent(&BTRFS_I(inode)->io_tree, start, end, NULL); + free_extent_map(em); + return ret; } @@ -3006,7 +2998,7 @@ static int relocate_one_page(struct inode *inode, struct file_ra_state *ra, goto release_page; /* Mark the range delalloc and dirty for later writeback */ - lock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end); + lock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end, NULL); ret = btrfs_set_extent_delalloc(BTRFS_I(inode), clamped_start, clamped_end, 0, NULL); if (ret) { @@ -3039,7 +3031,7 @@ static int relocate_one_page(struct inode *inode, struct file_ra_state *ra, boundary_start, boundary_end, EXTENT_BOUNDARY); } - unlock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end); + unlock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end, NULL); btrfs_delalloc_release_extents(BTRFS_I(inode), clamped_len); cur += clamped_len; @@ -4339,7 +4331,7 @@ int btrfs_reloc_clone_csums(struct btrfs_inode *inode, u64 file_pos, u64 len) disk_bytenr = file_pos + inode->index_cnt; csum_root = btrfs_csum_root(fs_info, disk_bytenr); ret = btrfs_lookup_csums_range(csum_root, disk_bytenr, - disk_bytenr + len - 1, &list, 0); + disk_bytenr + len - 1, &list, 0, false); if (ret) goto out; diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index a64b26b169040100b94f45acfefb050a5a5defba..e1f599d7a9164b7973515afc9de35a9df524040c 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -337,7 +337,6 @@ int btrfs_del_root_ref(struct btrfs_trans_handle *trans, u64 root_id, struct extent_buffer *leaf; struct btrfs_key key; unsigned long ptr; - int err = 0; int ret; path = btrfs_alloc_path(); @@ -349,9 +348,9 @@ int btrfs_del_root_ref(struct btrfs_trans_handle *trans, u64 root_id, key.offset = ref_id; again: ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1); - if (ret < 0) + if (ret < 0) { goto out; - if (ret == 0) { + } else if (ret == 0) { leaf = path->nodes[0]; ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); @@ -359,18 +358,18 @@ again: if ((btrfs_root_ref_dirid(leaf, ref) != dirid) || (btrfs_root_ref_name_len(leaf, ref) != name_len) || memcmp_extent_buffer(leaf, name, ptr, name_len)) { - err = -ENOENT; + ret = -ENOENT; goto out; } *sequence = btrfs_root_ref_sequence(leaf, ref); ret = btrfs_del_item(trans, tree_root, path); - if (ret) { - err = ret; + if (ret) goto out; - } - } else - err = -ENOENT; + } else { + ret = -ENOENT; + goto out; + } if (key.type == BTRFS_ROOT_BACKREF_KEY) { btrfs_release_path(path); @@ -382,7 +381,7 @@ again: out: btrfs_free_path(path); - return err; + return ret; } /* diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 3afe5fa50a631f9f55286cb24e89c9c06ea1c830..f260c53829e5efe07a76c0ec8eed8bf7994ae623 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -54,6 +54,8 @@ struct scrub_ctx; */ #define SCRUB_MAX_SECTORS_PER_BLOCK (BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K) +#define SCRUB_MAX_PAGES (DIV_ROUND_UP(BTRFS_MAX_METADATA_BLOCKSIZE, PAGE_SIZE)) + struct scrub_recover { refcount_t refs; struct btrfs_io_context *bioc; @@ -62,16 +64,12 @@ struct scrub_recover { struct scrub_sector { struct scrub_block *sblock; - struct page *page; - struct btrfs_device *dev; struct list_head list; u64 flags; /* extent flags */ u64 generation; - u64 logical; - u64 physical; - u64 physical_for_dev_replace; + /* Offset in bytes to @sblock. */ + u32 offset; atomic_t refs; - u8 mirror_num; unsigned int have_csum:1; unsigned int io_error:1; u8 csum[BTRFS_CSUM_SIZE]; @@ -94,8 +92,22 @@ struct scrub_bio { }; struct scrub_block { + /* + * Each page will have its page::private used to record the logical + * bytenr. + */ + struct page *pages[SCRUB_MAX_PAGES]; struct scrub_sector *sectors[SCRUB_MAX_SECTORS_PER_BLOCK]; + struct btrfs_device *dev; + /* Logical bytenr of the sblock */ + u64 logical; + u64 physical; + u64 physical_for_dev_replace; + /* Length of sblock in bytes */ + u32 len; int sector_count; + int mirror_num; + atomic_t outstanding_sectors; refcount_t refs; /* free mem on transition to zero */ struct scrub_ctx *sctx; @@ -202,8 +214,174 @@ struct full_stripe_lock { struct mutex mutex; }; +#ifndef CONFIG_64BIT +/* This structure is for archtectures whose (void *) is smaller than u64 */ +struct scrub_page_private { + u64 logical; +}; +#endif + +static int attach_scrub_page_private(struct page *page, u64 logical) +{ +#ifdef CONFIG_64BIT + attach_page_private(page, (void *)logical); + return 0; +#else + struct scrub_page_private *spp; + + spp = kmalloc(sizeof(*spp), GFP_KERNEL); + if (!spp) + return -ENOMEM; + spp->logical = logical; + attach_page_private(page, (void *)spp); + return 0; +#endif +} + +static void detach_scrub_page_private(struct page *page) +{ +#ifdef CONFIG_64BIT + detach_page_private(page); + return; +#else + struct scrub_page_private *spp; + + spp = detach_page_private(page); + kfree(spp); + return; +#endif +} + +static struct scrub_block *alloc_scrub_block(struct scrub_ctx *sctx, + struct btrfs_device *dev, + u64 logical, u64 physical, + u64 physical_for_dev_replace, + int mirror_num) +{ + struct scrub_block *sblock; + + sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); + if (!sblock) + return NULL; + refcount_set(&sblock->refs, 1); + sblock->sctx = sctx; + sblock->logical = logical; + sblock->physical = physical; + sblock->physical_for_dev_replace = physical_for_dev_replace; + sblock->dev = dev; + sblock->mirror_num = mirror_num; + sblock->no_io_error_seen = 1; + /* + * Scrub_block::pages will be allocated at alloc_scrub_sector() when + * the corresponding page is not allocated. + */ + return sblock; +} + +/* + * Allocate a new scrub sector and attach it to @sblock. + * + * Will also allocate new pages for @sblock if needed. + */ +static struct scrub_sector *alloc_scrub_sector(struct scrub_block *sblock, + u64 logical, gfp_t gfp) +{ + const pgoff_t page_index = (logical - sblock->logical) >> PAGE_SHIFT; + struct scrub_sector *ssector; + + /* We must never have scrub_block exceed U32_MAX in size. */ + ASSERT(logical - sblock->logical < U32_MAX); + + ssector = kzalloc(sizeof(*ssector), gfp); + if (!ssector) + return NULL; + + /* Allocate a new page if the slot is not allocated */ + if (!sblock->pages[page_index]) { + int ret; + + sblock->pages[page_index] = alloc_page(gfp); + if (!sblock->pages[page_index]) { + kfree(ssector); + return NULL; + } + ret = attach_scrub_page_private(sblock->pages[page_index], + sblock->logical + (page_index << PAGE_SHIFT)); + if (ret < 0) { + kfree(ssector); + __free_page(sblock->pages[page_index]); + sblock->pages[page_index] = NULL; + return NULL; + } + } + + atomic_set(&ssector->refs, 1); + ssector->sblock = sblock; + /* The sector to be added should not be used */ + ASSERT(sblock->sectors[sblock->sector_count] == NULL); + ssector->offset = logical - sblock->logical; + + /* The sector count must be smaller than the limit */ + ASSERT(sblock->sector_count < SCRUB_MAX_SECTORS_PER_BLOCK); + + sblock->sectors[sblock->sector_count] = ssector; + sblock->sector_count++; + sblock->len += sblock->sctx->fs_info->sectorsize; + + return ssector; +} + +static struct page *scrub_sector_get_page(struct scrub_sector *ssector) +{ + struct scrub_block *sblock = ssector->sblock; + pgoff_t index; + /* + * When calling this function, ssector must be alreaday attached to the + * parent sblock. + */ + ASSERT(sblock); + + /* The range should be inside the sblock range */ + ASSERT(ssector->offset < sblock->len); + + index = ssector->offset >> PAGE_SHIFT; + ASSERT(index < SCRUB_MAX_PAGES); + ASSERT(sblock->pages[index]); + ASSERT(PagePrivate(sblock->pages[index])); + return sblock->pages[index]; +} + +static unsigned int scrub_sector_get_page_offset(struct scrub_sector *ssector) +{ + struct scrub_block *sblock = ssector->sblock; + + /* + * When calling this function, ssector must be already attached to the + * parent sblock. + */ + ASSERT(sblock); + + /* The range should be inside the sblock range */ + ASSERT(ssector->offset < sblock->len); + + return offset_in_page(ssector->offset); +} + +static char *scrub_sector_get_kaddr(struct scrub_sector *ssector) +{ + return page_address(scrub_sector_get_page(ssector)) + + scrub_sector_get_page_offset(ssector); +} + +static int bio_add_scrub_sector(struct bio *bio, struct scrub_sector *ssector, + unsigned int len) +{ + return bio_add_page(bio, scrub_sector_get_page(ssector), len, + scrub_sector_get_page_offset(ssector)); +} + static int scrub_setup_recheck_block(struct scrub_block *original_sblock, - struct scrub_block *sblocks_for_recheck); + struct scrub_block *sblocks_for_recheck[]); static void scrub_recheck_block(struct btrfs_fs_info *fs_info, struct scrub_block *sblock, int retry_failed_mirror); @@ -533,10 +711,8 @@ static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) if (sctx->curr != -1) { struct scrub_bio *sbio = sctx->bios[sctx->curr]; - for (i = 0; i < sbio->sector_count; i++) { - WARN_ON(!sbio->sectors[i]->page); + for (i = 0; i < sbio->sector_count; i++) scrub_block_put(sbio->sectors[i]->sblock); - } bio_put(sbio->bio); } @@ -726,15 +902,22 @@ static void scrub_print_warning(const char *errstr, struct scrub_block *sblock) int ret; WARN_ON(sblock->sector_count < 1); - dev = sblock->sectors[0]->dev; + dev = sblock->dev; fs_info = sblock->sctx->fs_info; + /* Super block error, no need to search extent tree. */ + if (sblock->sectors[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { + btrfs_warn_in_rcu(fs_info, "%s on device %s, physical %llu", + errstr, rcu_str_deref(dev->name), + sblock->physical); + return; + } path = btrfs_alloc_path(); if (!path) return; - swarn.physical = sblock->sectors[0]->physical; - swarn.logical = sblock->sectors[0]->logical; + swarn.physical = sblock->physical; + swarn.logical = sblock->logical; swarn.errstr = errstr; swarn.dev = NULL; @@ -804,13 +987,14 @@ static inline void scrub_put_recover(struct btrfs_fs_info *fs_info, static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) { struct scrub_ctx *sctx = sblock_to_check->sctx; - struct btrfs_device *dev; + struct btrfs_device *dev = sblock_to_check->dev; struct btrfs_fs_info *fs_info; u64 logical; unsigned int failed_mirror_index; unsigned int is_metadata; unsigned int have_csum; - struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */ + /* One scrub_block for each mirror */ + struct scrub_block *sblocks_for_recheck[BTRFS_MAX_MIRRORS] = { 0 }; struct scrub_block *sblock_bad; int ret; int mirror_index; @@ -825,22 +1009,23 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) fs_info = sctx->fs_info; if (sblock_to_check->sectors[0]->flags & BTRFS_EXTENT_FLAG_SUPER) { /* - * if we find an error in a super block, we just report it. + * If we find an error in a super block, we just report it. * They will get written with the next transaction commit * anyway */ + scrub_print_warning("super block error", sblock_to_check); spin_lock(&sctx->stat_lock); ++sctx->stat.super_errors; spin_unlock(&sctx->stat_lock); + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); return 0; } - logical = sblock_to_check->sectors[0]->logical; - BUG_ON(sblock_to_check->sectors[0]->mirror_num < 1); - failed_mirror_index = sblock_to_check->sectors[0]->mirror_num - 1; + logical = sblock_to_check->logical; + ASSERT(sblock_to_check->mirror_num); + failed_mirror_index = sblock_to_check->mirror_num - 1; is_metadata = !(sblock_to_check->sectors[0]->flags & BTRFS_EXTENT_FLAG_DATA); have_csum = sblock_to_check->sectors[0]->have_csum; - dev = sblock_to_check->sectors[0]->dev; if (!sctx->is_dev_replace && btrfs_repair_one_zone(fs_info, logical)) return 0; @@ -902,17 +1087,28 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) * repaired area is verified in order to correctly maintain * the statistics. */ - - sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS, - sizeof(*sblocks_for_recheck), GFP_KERNEL); - if (!sblocks_for_recheck) { - spin_lock(&sctx->stat_lock); - sctx->stat.malloc_errors++; - sctx->stat.read_errors++; - sctx->stat.uncorrectable_errors++; - spin_unlock(&sctx->stat_lock); - btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); - goto out; + for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS; mirror_index++) { + /* + * Note: the two members refs and outstanding_sectors are not + * used in the blocks that are used for the recheck procedure. + * + * But alloc_scrub_block() will initialize sblock::ref anyway, + * so we can use scrub_block_put() to clean them up. + * + * And here we don't setup the physical/dev for the sblock yet, + * they will be correctly initialized in scrub_setup_recheck_block(). + */ + sblocks_for_recheck[mirror_index] = alloc_scrub_block(sctx, NULL, + logical, 0, 0, mirror_index); + if (!sblocks_for_recheck[mirror_index]) { + spin_lock(&sctx->stat_lock); + sctx->stat.malloc_errors++; + sctx->stat.read_errors++; + sctx->stat.uncorrectable_errors++; + spin_unlock(&sctx->stat_lock); + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); + goto out; + } } /* Setup the context, map the logical blocks and alloc the sectors */ @@ -926,7 +1122,7 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) goto out; } BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS); - sblock_bad = sblocks_for_recheck + failed_mirror_index; + sblock_bad = sblocks_for_recheck[failed_mirror_index]; /* build and submit the bios for the failed mirror, check checksums */ scrub_recheck_block(fs_info, sblock_bad, 1); @@ -1011,22 +1207,22 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) if (!scrub_is_page_on_raid56(sblock_bad->sectors[0])) { if (mirror_index >= BTRFS_MAX_MIRRORS) break; - if (!sblocks_for_recheck[mirror_index].sector_count) + if (!sblocks_for_recheck[mirror_index]->sector_count) break; - sblock_other = sblocks_for_recheck + mirror_index; + sblock_other = sblocks_for_recheck[mirror_index]; } else { struct scrub_recover *r = sblock_bad->sectors[0]->recover; int max_allowed = r->bioc->num_stripes - r->bioc->num_tgtdevs; if (mirror_index >= max_allowed) break; - if (!sblocks_for_recheck[1].sector_count) + if (!sblocks_for_recheck[1]->sector_count) break; ASSERT(failed_mirror_index == 0); - sblock_other = sblocks_for_recheck + 1; - sblock_other->sectors[0]->mirror_num = 1 + mirror_index; + sblock_other = sblocks_for_recheck[1]; + sblock_other->mirror_num = 1 + mirror_index; } /* build and submit the bios, check checksums */ @@ -1097,12 +1293,11 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) /* Try to find no-io-error sector in mirrors */ for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS && - sblocks_for_recheck[mirror_index].sector_count > 0; + sblocks_for_recheck[mirror_index]->sector_count > 0; mirror_index++) { - if (!sblocks_for_recheck[mirror_index]. + if (!sblocks_for_recheck[mirror_index]-> sectors[sector_num]->io_error) { - sblock_other = sblocks_for_recheck + - mirror_index; + sblock_other = sblocks_for_recheck[mirror_index]; break; } } @@ -1176,25 +1371,28 @@ did_not_correct_error: } out: - if (sblocks_for_recheck) { - for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS; - mirror_index++) { - struct scrub_block *sblock = sblocks_for_recheck + - mirror_index; - struct scrub_recover *recover; - int i; - - for (i = 0; i < sblock->sector_count; i++) { - sblock->sectors[i]->sblock = NULL; - recover = sblock->sectors[i]->recover; - if (recover) { - scrub_put_recover(fs_info, recover); - sblock->sectors[i]->recover = NULL; - } - scrub_sector_put(sblock->sectors[i]); + for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS; mirror_index++) { + struct scrub_block *sblock = sblocks_for_recheck[mirror_index]; + struct scrub_recover *recover; + int sector_index; + + /* Not allocated, continue checking the next mirror */ + if (!sblock) + continue; + + for (sector_index = 0; sector_index < sblock->sector_count; + sector_index++) { + /* + * Here we just cleanup the recover, each sector will be + * properly cleaned up by later scrub_block_put() + */ + recover = sblock->sectors[sector_index]->recover; + if (recover) { + scrub_put_recover(fs_info, recover); + sblock->sectors[sector_index]->recover = NULL; } } - kfree(sblocks_for_recheck); + scrub_block_put(sblock); } ret = unlock_full_stripe(fs_info, logical, full_stripe_locked); @@ -1244,12 +1442,12 @@ static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type, } static int scrub_setup_recheck_block(struct scrub_block *original_sblock, - struct scrub_block *sblocks_for_recheck) + struct scrub_block *sblocks_for_recheck[]) { struct scrub_ctx *sctx = original_sblock->sctx; struct btrfs_fs_info *fs_info = sctx->fs_info; + u64 logical = original_sblock->logical; u64 length = original_sblock->sector_count << fs_info->sectorsize_bits; - u64 logical = original_sblock->sectors[0]->logical; u64 generation = original_sblock->sectors[0]->generation; u64 flags = original_sblock->sectors[0]->flags; u64 have_csum = original_sblock->sectors[0]->have_csum; @@ -1264,11 +1462,6 @@ static int scrub_setup_recheck_block(struct scrub_block *original_sblock, int nmirrors; int ret; - /* - * Note: the two members refs and outstanding_sectors are not used (and - * not set) in the blocks that are used for the recheck procedure. - */ - while (length > 0) { sublen = min_t(u64, length, fs_info->sectorsize); mapped_length = sublen; @@ -1307,24 +1500,19 @@ static int scrub_setup_recheck_block(struct scrub_block *original_sblock, struct scrub_block *sblock; struct scrub_sector *sector; - sblock = sblocks_for_recheck + mirror_index; + sblock = sblocks_for_recheck[mirror_index]; sblock->sctx = sctx; - sector = kzalloc(sizeof(*sector), GFP_NOFS); + sector = alloc_scrub_sector(sblock, logical, GFP_NOFS); if (!sector) { -leave_nomem: spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; spin_unlock(&sctx->stat_lock); scrub_put_recover(fs_info, recover); return -ENOMEM; } - scrub_sector_get(sector); - sblock->sectors[sector_index] = sector; - sector->sblock = sblock; sector->flags = flags; sector->generation = generation; - sector->logical = logical; sector->have_csum = have_csum; if (have_csum) memcpy(sector->csum, @@ -1339,21 +1527,20 @@ leave_nomem: mirror_index, &stripe_index, &stripe_offset); - sector->physical = bioc->stripes[stripe_index].physical + - stripe_offset; - sector->dev = bioc->stripes[stripe_index].dev; + /* + * We're at the first sector, also populate @sblock + * physical and dev. + */ + if (sector_index == 0) { + sblock->physical = + bioc->stripes[stripe_index].physical + + stripe_offset; + sblock->dev = bioc->stripes[stripe_index].dev; + sblock->physical_for_dev_replace = + original_sblock->physical_for_dev_replace; + } BUG_ON(sector_index >= original_sblock->sector_count); - sector->physical_for_dev_replace = - original_sblock->sectors[sector_index]-> - physical_for_dev_replace; - /* For missing devices, dev->bdev is NULL */ - sector->mirror_num = mirror_index + 1; - sblock->sector_count++; - sector->page = alloc_page(GFP_NOFS); - if (!sector->page) - goto leave_nomem; - scrub_get_recover(recover); sector->recover = recover; } @@ -1377,11 +1564,11 @@ static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info, { DECLARE_COMPLETION_ONSTACK(done); - bio->bi_iter.bi_sector = sector->logical >> 9; + bio->bi_iter.bi_sector = (sector->offset + sector->sblock->logical) >> + SECTOR_SHIFT; bio->bi_private = &done; bio->bi_end_io = scrub_bio_wait_endio; - raid56_parity_recover(bio, sector->recover->bioc, - sector->sblock->sectors[0]->mirror_num, false); + raid56_parity_recover(bio, sector->recover->bioc, sector->sblock->mirror_num); wait_for_completion_io(&done); return blk_status_to_errno(bio->bi_status); @@ -1395,17 +1582,16 @@ static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info, int i; /* All sectors in sblock belong to the same stripe on the same device. */ - ASSERT(first_sector->dev); - if (!first_sector->dev->bdev) + ASSERT(sblock->dev); + if (!sblock->dev->bdev) goto out; - bio = bio_alloc(first_sector->dev->bdev, BIO_MAX_VECS, REQ_OP_READ, GFP_NOFS); + bio = bio_alloc(sblock->dev->bdev, BIO_MAX_VECS, REQ_OP_READ, GFP_NOFS); for (i = 0; i < sblock->sector_count; i++) { struct scrub_sector *sector = sblock->sectors[i]; - WARN_ON(!sector->page); - bio_add_page(bio, sector->page, PAGE_SIZE, 0); + bio_add_scrub_sector(bio, sector, fs_info->sectorsize); } if (scrub_submit_raid56_bio_wait(fs_info, bio, first_sector)) { @@ -1449,16 +1635,16 @@ static void scrub_recheck_block(struct btrfs_fs_info *fs_info, struct bio bio; struct bio_vec bvec; - if (sector->dev->bdev == NULL) { + if (sblock->dev->bdev == NULL) { sector->io_error = 1; sblock->no_io_error_seen = 0; continue; } - WARN_ON(!sector->page); - bio_init(&bio, sector->dev->bdev, &bvec, 1, REQ_OP_READ); - bio_add_page(&bio, sector->page, fs_info->sectorsize, 0); - bio.bi_iter.bi_sector = sector->physical >> 9; + bio_init(&bio, sblock->dev->bdev, &bvec, 1, REQ_OP_READ); + bio_add_scrub_sector(&bio, sector, fs_info->sectorsize); + bio.bi_iter.bi_sector = (sblock->physical + sector->offset) >> + SECTOR_SHIFT; btrfsic_check_bio(&bio); if (submit_bio_wait(&bio)) { @@ -1475,7 +1661,7 @@ static void scrub_recheck_block(struct btrfs_fs_info *fs_info, static inline int scrub_check_fsid(u8 fsid[], struct scrub_sector *sector) { - struct btrfs_fs_devices *fs_devices = sector->dev->fs_devices; + struct btrfs_fs_devices *fs_devices = sector->sblock->dev->fs_devices; int ret; ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE); @@ -1521,30 +1707,29 @@ static int scrub_repair_sector_from_good_copy(struct scrub_block *sblock_bad, struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; const u32 sectorsize = fs_info->sectorsize; - BUG_ON(sector_bad->page == NULL); - BUG_ON(sector_good->page == NULL); if (force_write || sblock_bad->header_error || sblock_bad->checksum_error || sector_bad->io_error) { struct bio bio; struct bio_vec bvec; int ret; - if (!sector_bad->dev->bdev) { + if (!sblock_bad->dev->bdev) { btrfs_warn_rl(fs_info, "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected"); return -EIO; } - bio_init(&bio, sector_bad->dev->bdev, &bvec, 1, REQ_OP_WRITE); - bio.bi_iter.bi_sector = sector_bad->physical >> 9; - __bio_add_page(&bio, sector_good->page, sectorsize, 0); + bio_init(&bio, sblock_bad->dev->bdev, &bvec, 1, REQ_OP_WRITE); + bio.bi_iter.bi_sector = (sblock_bad->physical + + sector_bad->offset) >> SECTOR_SHIFT; + ret = bio_add_scrub_sector(&bio, sector_good, sectorsize); btrfsic_check_bio(&bio); ret = submit_bio_wait(&bio); bio_uninit(&bio); if (ret) { - btrfs_dev_stat_inc_and_print(sector_bad->dev, + btrfs_dev_stat_inc_and_print(sblock_bad->dev, BTRFS_DEV_STAT_WRITE_ERRS); atomic64_inc(&fs_info->dev_replace.num_write_errors); return -EIO; @@ -1577,11 +1762,11 @@ static void scrub_write_block_to_dev_replace(struct scrub_block *sblock) static int scrub_write_sector_to_dev_replace(struct scrub_block *sblock, int sector_num) { + const u32 sectorsize = sblock->sctx->fs_info->sectorsize; struct scrub_sector *sector = sblock->sectors[sector_num]; - BUG_ON(sector->page == NULL); if (sector->io_error) - clear_page(page_address(sector->page)); + memset(scrub_sector_get_kaddr(sector), 0, sectorsize); return scrub_add_sector_to_wr_bio(sblock->sctx, sector); } @@ -1608,9 +1793,15 @@ static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) return ret; } +static void scrub_block_get(struct scrub_block *sblock) +{ + refcount_inc(&sblock->refs); +} + static int scrub_add_sector_to_wr_bio(struct scrub_ctx *sctx, struct scrub_sector *sector) { + struct scrub_block *sblock = sector->sblock; struct scrub_bio *sbio; int ret; const u32 sectorsize = sctx->fs_info->sectorsize; @@ -1629,14 +1820,15 @@ again: } sbio = sctx->wr_curr_bio; if (sbio->sector_count == 0) { - ret = fill_writer_pointer_gap(sctx, sector->physical_for_dev_replace); + ret = fill_writer_pointer_gap(sctx, sector->offset + + sblock->physical_for_dev_replace); if (ret) { mutex_unlock(&sctx->wr_lock); return ret; } - sbio->physical = sector->physical_for_dev_replace; - sbio->logical = sector->logical; + sbio->physical = sblock->physical_for_dev_replace + sector->offset; + sbio->logical = sblock->logical + sector->offset; sbio->dev = sctx->wr_tgtdev; if (!sbio->bio) { sbio->bio = bio_alloc(sbio->dev->bdev, sctx->sectors_per_bio, @@ -1647,14 +1839,14 @@ again: sbio->bio->bi_iter.bi_sector = sbio->physical >> 9; sbio->status = 0; } else if (sbio->physical + sbio->sector_count * sectorsize != - sector->physical_for_dev_replace || + sblock->physical_for_dev_replace + sector->offset || sbio->logical + sbio->sector_count * sectorsize != - sector->logical) { + sblock->logical + sector->offset) { scrub_wr_submit(sctx); goto again; } - ret = bio_add_page(sbio->bio, sector->page, sectorsize, 0); + ret = bio_add_scrub_sector(sbio->bio, sector, sectorsize); if (ret != sectorsize) { if (sbio->sector_count < 1) { bio_put(sbio->bio); @@ -1668,6 +1860,13 @@ again: sbio->sectors[sbio->sector_count] = sector; scrub_sector_get(sector); + /* + * Since ssector no longer holds a page, but uses sblock::pages, we + * have to ensure the sblock had not been freed before our write bio + * finished. + */ + scrub_block_get(sector->sblock); + sbio->sector_count++; if (sbio->sector_count == sctx->sectors_per_bio) scrub_wr_submit(sctx); @@ -1729,8 +1928,14 @@ static void scrub_wr_bio_end_io_worker(struct work_struct *work) } } - for (i = 0; i < sbio->sector_count; i++) + /* + * In scrub_add_sector_to_wr_bio() we grab extra ref for sblock, now in + * endio we should put the sblock. + */ + for (i = 0; i < sbio->sector_count; i++) { + scrub_block_put(sbio->sectors[i]->sblock); scrub_sector_put(sbio->sectors[i]); + } bio_put(sbio->bio); kfree(sbio); @@ -1762,7 +1967,7 @@ static int scrub_checksum(struct scrub_block *sblock) else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) ret = scrub_checksum_tree_block(sblock); else if (flags & BTRFS_EXTENT_FLAG_SUPER) - (void)scrub_checksum_super(sblock); + ret = scrub_checksum_super(sblock); else WARN_ON(1); if (ret) @@ -1785,15 +1990,11 @@ static int scrub_checksum_data(struct scrub_block *sblock) if (!sector->have_csum) return 0; - kaddr = page_address(sector->page); + kaddr = scrub_sector_get_kaddr(sector); shash->tfm = fs_info->csum_shash; crypto_shash_init(shash); - /* - * In scrub_sectors() and scrub_sectors_for_parity() we ensure each sector - * only contains one sector of data. - */ crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum); if (memcmp(csum, sector->csum, fs_info->csum_size)) @@ -1826,7 +2027,7 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) ASSERT(sblock->sector_count == num_sectors); sector = sblock->sectors[0]; - kaddr = page_address(sector->page); + kaddr = scrub_sector_get_kaddr(sector); h = (struct btrfs_header *)kaddr; memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size); @@ -1835,7 +2036,7 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) * a) don't have an extent buffer and * b) the page is already kmapped */ - if (sector->logical != btrfs_stack_header_bytenr(h)) + if (sblock->logical != btrfs_stack_header_bytenr(h)) sblock->header_error = 1; if (sector->generation != btrfs_stack_header_generation(h)) { @@ -1856,7 +2057,7 @@ static int scrub_checksum_tree_block(struct scrub_block *sblock) sectorsize - BTRFS_CSUM_SIZE); for (i = 1; i < num_sectors; i++) { - kaddr = page_address(sblock->sectors[i]->page); + kaddr = scrub_sector_get_kaddr(sblock->sectors[i]); crypto_shash_update(shash, kaddr, sectorsize); } @@ -1881,10 +2082,10 @@ static int scrub_checksum_super(struct scrub_block *sblock) BUG_ON(sblock->sector_count < 1); sector = sblock->sectors[0]; - kaddr = page_address(sector->page); + kaddr = scrub_sector_get_kaddr(sector); s = (struct btrfs_super_block *)kaddr; - if (sector->logical != btrfs_super_bytenr(s)) + if (sblock->logical != btrfs_super_bytenr(s)) ++fail_cor; if (sector->generation != btrfs_super_generation(s)) @@ -1901,31 +2102,9 @@ static int scrub_checksum_super(struct scrub_block *sblock) if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size)) ++fail_cor; - if (fail_cor + fail_gen) { - /* - * if we find an error in a super block, we just report it. - * They will get written with the next transaction commit - * anyway - */ - spin_lock(&sctx->stat_lock); - ++sctx->stat.super_errors; - spin_unlock(&sctx->stat_lock); - if (fail_cor) - btrfs_dev_stat_inc_and_print(sector->dev, - BTRFS_DEV_STAT_CORRUPTION_ERRS); - else - btrfs_dev_stat_inc_and_print(sector->dev, - BTRFS_DEV_STAT_GENERATION_ERRS); - } - return fail_cor + fail_gen; } -static void scrub_block_get(struct scrub_block *sblock) -{ - refcount_inc(&sblock->refs); -} - static void scrub_block_put(struct scrub_block *sblock) { if (refcount_dec_and_test(&sblock->refs)) { @@ -1936,6 +2115,12 @@ static void scrub_block_put(struct scrub_block *sblock) for (i = 0; i < sblock->sector_count; i++) scrub_sector_put(sblock->sectors[i]); + for (i = 0; i < DIV_ROUND_UP(sblock->len, PAGE_SIZE); i++) { + if (sblock->pages[i]) { + detach_scrub_page_private(sblock->pages[i]); + __free_page(sblock->pages[i]); + } + } kfree(sblock); } } @@ -1947,11 +2132,8 @@ static void scrub_sector_get(struct scrub_sector *sector) static void scrub_sector_put(struct scrub_sector *sector) { - if (atomic_dec_and_test(§or->refs)) { - if (sector->page) - __free_page(sector->page); + if (atomic_dec_and_test(§or->refs)) kfree(sector); - } } /* @@ -2056,9 +2238,9 @@ again: } sbio = sctx->bios[sctx->curr]; if (sbio->sector_count == 0) { - sbio->physical = sector->physical; - sbio->logical = sector->logical; - sbio->dev = sector->dev; + sbio->physical = sblock->physical + sector->offset; + sbio->logical = sblock->logical + sector->offset; + sbio->dev = sblock->dev; if (!sbio->bio) { sbio->bio = bio_alloc(sbio->dev->bdev, sctx->sectors_per_bio, REQ_OP_READ, GFP_NOFS); @@ -2068,16 +2250,16 @@ again: sbio->bio->bi_iter.bi_sector = sbio->physical >> 9; sbio->status = 0; } else if (sbio->physical + sbio->sector_count * sectorsize != - sector->physical || + sblock->physical + sector->offset || sbio->logical + sbio->sector_count * sectorsize != - sector->logical || - sbio->dev != sector->dev) { + sblock->logical + sector->offset || + sbio->dev != sblock->dev) { scrub_submit(sctx); goto again; } sbio->sectors[sbio->sector_count] = sector; - ret = bio_add_page(sbio->bio, sector->page, sectorsize, 0); + ret = bio_add_scrub_sector(sbio->bio, sector, sectorsize); if (ret != sectorsize) { if (sbio->sector_count < 1) { bio_put(sbio->bio); @@ -2102,6 +2284,7 @@ static void scrub_missing_raid56_end_io(struct bio *bio) struct scrub_block *sblock = bio->bi_private; struct btrfs_fs_info *fs_info = sblock->sctx->fs_info; + btrfs_bio_counter_dec(fs_info); if (bio->bi_status) sblock->no_io_error_seen = 0; @@ -2118,8 +2301,8 @@ static void scrub_missing_raid56_worker(struct work_struct *work) u64 logical; struct btrfs_device *dev; - logical = sblock->sectors[0]->logical; - dev = sblock->sectors[0]->dev; + logical = sblock->logical; + dev = sblock->dev; if (sblock->no_io_error_seen) scrub_recheck_block_checksum(sblock); @@ -2157,7 +2340,7 @@ static void scrub_missing_raid56_pages(struct scrub_block *sblock) struct scrub_ctx *sctx = sblock->sctx; struct btrfs_fs_info *fs_info = sctx->fs_info; u64 length = sblock->sector_count << fs_info->sectorsize_bits; - u64 logical = sblock->sectors[0]->logical; + u64 logical = sblock->logical; struct btrfs_io_context *bioc = NULL; struct bio *bio; struct btrfs_raid_bio *rbio; @@ -2193,17 +2376,16 @@ static void scrub_missing_raid56_pages(struct scrub_block *sblock) for (i = 0; i < sblock->sector_count; i++) { struct scrub_sector *sector = sblock->sectors[i]; - /* - * For now, our scrub is still one page per sector, so pgoff - * is always 0. - */ - raid56_add_scrub_pages(rbio, sector->page, 0, sector->logical); + raid56_add_scrub_pages(rbio, scrub_sector_get_page(sector), + scrub_sector_get_page_offset(sector), + sector->offset + sector->sblock->logical); } INIT_WORK(&sblock->work, scrub_missing_raid56_worker); scrub_block_get(sblock); scrub_pending_bio_inc(sctx); raid56_submit_missing_rbio(rbio); + btrfs_put_bioc(bioc); return; rbio_out: @@ -2225,7 +2407,8 @@ static int scrub_sectors(struct scrub_ctx *sctx, u64 logical, u32 len, const u32 sectorsize = sctx->fs_info->sectorsize; int index; - sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); + sblock = alloc_scrub_block(sctx, dev, logical, physical, + physical_for_dev_replace, mirror_num); if (!sblock) { spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; @@ -2233,12 +2416,6 @@ static int scrub_sectors(struct scrub_ctx *sctx, u64 logical, u32 len, return -ENOMEM; } - /* one ref inside this function, plus one for each page added to - * a bio later on */ - refcount_set(&sblock->refs, 1); - sblock->sctx = sctx; - sblock->no_io_error_seen = 1; - for (index = 0; len > 0; index++) { struct scrub_sector *sector; /* @@ -2248,36 +2425,22 @@ static int scrub_sectors(struct scrub_ctx *sctx, u64 logical, u32 len, */ u32 l = min(sectorsize, len); - sector = kzalloc(sizeof(*sector), GFP_KERNEL); + sector = alloc_scrub_sector(sblock, logical, GFP_KERNEL); if (!sector) { -leave_nomem: spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; spin_unlock(&sctx->stat_lock); scrub_block_put(sblock); return -ENOMEM; } - ASSERT(index < SCRUB_MAX_SECTORS_PER_BLOCK); - scrub_sector_get(sector); - sblock->sectors[index] = sector; - sector->sblock = sblock; - sector->dev = dev; sector->flags = flags; sector->generation = gen; - sector->logical = logical; - sector->physical = physical; - sector->physical_for_dev_replace = physical_for_dev_replace; - sector->mirror_num = mirror_num; if (csum) { sector->have_csum = 1; memcpy(sector->csum, csum, sctx->fs_info->csum_size); } else { sector->have_csum = 0; } - sblock->sector_count++; - sector->page = alloc_page(GFP_KERNEL); - if (!sector->page) - goto leave_nomem; len -= l; logical += l; physical += l; @@ -2423,8 +2586,9 @@ static void scrub_block_complete(struct scrub_block *sblock) } if (sblock->sparity && corrupted && !sblock->data_corrected) { - u64 start = sblock->sectors[0]->logical; - u64 end = sblock->sectors[sblock->sector_count - 1]->logical + + u64 start = sblock->logical; + u64 end = sblock->logical + + sblock->sectors[sblock->sector_count - 1]->offset + sblock->sctx->fs_info->sectorsize; ASSERT(end - start <= U32_MAX); @@ -2508,11 +2672,17 @@ static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map, u8 csum[BTRFS_CSUM_SIZE]; u32 blocksize; + /* + * Block size determines how many scrub_block will be allocated. Here + * we use BTRFS_STRIPE_LEN (64KiB) as default limit, so we won't + * allocate too many scrub_block, while still won't cause too large + * bios for large extents. + */ if (flags & BTRFS_EXTENT_FLAG_DATA) { if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) blocksize = map->stripe_len; else - blocksize = sctx->fs_info->sectorsize; + blocksize = BTRFS_STRIPE_LEN; spin_lock(&sctx->stat_lock); sctx->stat.data_extents_scrubbed++; sctx->stat.data_bytes_scrubbed += len; @@ -2578,7 +2748,7 @@ static int scrub_sectors_for_parity(struct scrub_parity *sparity, ASSERT(IS_ALIGNED(len, sectorsize)); - sblock = kzalloc(sizeof(*sblock), GFP_KERNEL); + sblock = alloc_scrub_block(sctx, dev, logical, physical, physical, mirror_num); if (!sblock) { spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; @@ -2586,51 +2756,32 @@ static int scrub_sectors_for_parity(struct scrub_parity *sparity, return -ENOMEM; } - /* one ref inside this function, plus one for each page added to - * a bio later on */ - refcount_set(&sblock->refs, 1); - sblock->sctx = sctx; - sblock->no_io_error_seen = 1; sblock->sparity = sparity; scrub_parity_get(sparity); for (index = 0; len > 0; index++) { struct scrub_sector *sector; - sector = kzalloc(sizeof(*sector), GFP_KERNEL); + sector = alloc_scrub_sector(sblock, logical, GFP_KERNEL); if (!sector) { -leave_nomem: spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; spin_unlock(&sctx->stat_lock); scrub_block_put(sblock); return -ENOMEM; } - ASSERT(index < SCRUB_MAX_SECTORS_PER_BLOCK); - /* For scrub block */ - scrub_sector_get(sector); sblock->sectors[index] = sector; /* For scrub parity */ scrub_sector_get(sector); list_add_tail(§or->list, &sparity->sectors_list); - sector->sblock = sblock; - sector->dev = dev; sector->flags = flags; sector->generation = gen; - sector->logical = logical; - sector->physical = physical; - sector->mirror_num = mirror_num; if (csum) { sector->have_csum = 1; memcpy(sector->csum, csum, sctx->fs_info->csum_size); } else { sector->have_csum = 0; } - sblock->sector_count++; - sector->page = alloc_page(GFP_KERNEL); - if (!sector->page) - goto leave_nomem; - /* Iterate over the stripe range in sectorsize steps */ len -= sectorsize; @@ -2774,6 +2925,7 @@ static void scrub_parity_bio_endio_worker(struct work_struct *work) work); struct scrub_ctx *sctx = sparity->sctx; + btrfs_bio_counter_dec(sctx->fs_info); scrub_free_parity(sparity); scrub_pending_bio_dec(sctx); } @@ -2824,6 +2976,7 @@ static void scrub_parity_check_and_repair(struct scrub_parity *sparity) sparity->scrub_dev, &sparity->dbitmap, sparity->nsectors); + btrfs_put_bioc(bioc); if (!rbio) goto rbio_out; @@ -2835,7 +2988,6 @@ rbio_out: bio_put(bio); bioc_out: btrfs_bio_counter_dec(fs_info); - btrfs_put_bioc(bioc); bitmap_or(&sparity->ebitmap, &sparity->ebitmap, &sparity->dbitmap, sparity->nsectors); spin_lock(&sctx->stat_lock); @@ -3077,7 +3229,7 @@ static int scrub_raid56_data_stripe_for_parity(struct scrub_ctx *sctx, ret = btrfs_lookup_csums_range(csum_root, extent_start, extent_start + extent_size - 1, - &sctx->csum_list, 1); + &sctx->csum_list, 1, false); if (ret) { scrub_parity_mark_sectors_error(sparity, extent_start, extent_size); @@ -3266,7 +3418,7 @@ static int scrub_simple_mirror(struct scrub_ctx *sctx, } /* Block group removed? */ spin_lock(&bg->lock); - if (bg->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) { spin_unlock(&bg->lock); ret = 0; break; @@ -3303,7 +3455,7 @@ static int scrub_simple_mirror(struct scrub_ctx *sctx, if (extent_flags & BTRFS_EXTENT_FLAG_DATA) { ret = btrfs_lookup_csums_range(csum_root, cur_logical, cur_logical + scrub_len - 1, - &sctx->csum_list, 1); + &sctx->csum_list, 1, false); if (ret) break; } @@ -3606,7 +3758,7 @@ static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, * kthread or relocation. */ spin_lock(&bg->lock); - if (!bg->removed) + if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) ret = -EINVAL; spin_unlock(&bg->lock); @@ -3764,13 +3916,11 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, } if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { - spin_lock(&cache->lock); - if (!cache->to_copy) { + if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) { spin_unlock(&cache->lock); btrfs_put_block_group(cache); goto skip; } - spin_unlock(&cache->lock); } /* @@ -3782,7 +3932,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, * repair extents. */ spin_lock(&cache->lock); - if (cache->removed) { + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) { spin_unlock(&cache->lock); btrfs_put_block_group(cache); goto skip; @@ -3942,8 +4092,8 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, * balance is triggered or it becomes used and unused again. */ spin_lock(&cache->lock); - if (!cache->removed && !cache->ro && cache->reserved == 0 && - cache->used == 0) { + if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags) && + !cache->ro && cache->reserved == 0 && cache->used == 0) { spin_unlock(&cache->lock); if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) btrfs_discard_queue_work(&fs_info->discard_ctl, @@ -4102,36 +4252,21 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, int ret; struct btrfs_device *dev; unsigned int nofs_flag; + bool need_commit = false; if (btrfs_fs_closing(fs_info)) return -EAGAIN; - if (fs_info->nodesize > BTRFS_STRIPE_LEN) { - /* - * in this case scrub is unable to calculate the checksum - * the way scrub is implemented. Do not handle this - * situation at all because it won't ever happen. - */ - btrfs_err(fs_info, - "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails", - fs_info->nodesize, - BTRFS_STRIPE_LEN); - return -EINVAL; - } + /* At mount time we have ensured nodesize is in the range of [4K, 64K]. */ + ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN); - if (fs_info->nodesize > - SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits || - fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_SECTORS_PER_BLOCK) { - /* - * Would exhaust the array bounds of sectorv member in - * struct scrub_block - */ - btrfs_err(fs_info, -"scrub: nodesize and sectorsize <= SCRUB_MAX_SECTORS_PER_BLOCK (%d <= %d && %d <= %d) fails", - fs_info->nodesize, SCRUB_MAX_SECTORS_PER_BLOCK, - fs_info->sectorsize, SCRUB_MAX_SECTORS_PER_BLOCK); - return -EINVAL; - } + /* + * SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible + * value (max nodesize / min sectorsize), thus nodesize should always + * be fine. + */ + ASSERT(fs_info->nodesize <= + SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits); /* Allocate outside of device_list_mutex */ sctx = scrub_setup_ctx(fs_info, is_dev_replace); @@ -4205,6 +4340,12 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, */ nofs_flag = memalloc_nofs_save(); if (!is_dev_replace) { + u64 old_super_errors; + + spin_lock(&sctx->stat_lock); + old_super_errors = sctx->stat.super_errors; + spin_unlock(&sctx->stat_lock); + btrfs_info(fs_info, "scrub: started on devid %llu", devid); /* * by holding device list mutex, we can @@ -4213,6 +4354,16 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, mutex_lock(&fs_info->fs_devices->device_list_mutex); ret = scrub_supers(sctx, dev); mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + spin_lock(&sctx->stat_lock); + /* + * Super block errors found, but we can not commit transaction + * at current context, since btrfs_commit_transaction() needs + * to pause the current running scrub (hold by ourselves). + */ + if (sctx->stat.super_errors > old_super_errors && !sctx->readonly) + need_commit = true; + spin_unlock(&sctx->stat_lock); } if (!ret) @@ -4239,6 +4390,25 @@ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, scrub_workers_put(fs_info); scrub_put_ctx(sctx); + /* + * We found some super block errors before, now try to force a + * transaction commit, as scrub has finished. + */ + if (need_commit) { + struct btrfs_trans_handle *trans; + + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_err(fs_info, + "scrub: failed to start transaction to fix super block errors: %d", ret); + return ret; + } + ret = btrfs_commit_transaction(trans); + if (ret < 0) + btrfs_err(fs_info, + "scrub: failed to commit transaction to fix super block errors: %d", ret); + } return ret; out: scrub_workers_put(fs_info); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index e7671afcee4f0e90d6ce9114edb5d41f2ea8be34..4ef4167072b8920b562b2846d471efaaba2dabb8 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "send.h" #include "ctree.h" @@ -127,6 +128,8 @@ struct send_ctx { bool cur_inode_new_gen; bool cur_inode_deleted; bool ignore_cur_inode; + bool cur_inode_needs_verity; + void *verity_descriptor; u64 send_progress; @@ -624,6 +627,7 @@ static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len) return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \ } +TLV_PUT_DEFINE_INT(8) TLV_PUT_DEFINE_INT(32) TLV_PUT_DEFINE_INT(64) @@ -842,17 +846,32 @@ out: return ret; } +struct btrfs_inode_info { + u64 size; + u64 gen; + u64 mode; + u64 uid; + u64 gid; + u64 rdev; + u64 fileattr; + u64 nlink; +}; + /* * Helper function to retrieve some fields from an inode item. */ -static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, - u64 ino, u64 *size, u64 *gen, u64 *mode, u64 *uid, - u64 *gid, u64 *rdev, u64 *fileattr) +static int get_inode_info(struct btrfs_root *root, u64 ino, + struct btrfs_inode_info *info) { int ret; + struct btrfs_path *path; struct btrfs_inode_item *ii; struct btrfs_key key; + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + key.objectid = ino; key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; @@ -860,47 +879,43 @@ static int __get_inode_info(struct btrfs_root *root, struct btrfs_path *path, if (ret) { if (ret > 0) ret = -ENOENT; - return ret; + goto out; } + if (!info) + goto out; + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_inode_item); - if (size) - *size = btrfs_inode_size(path->nodes[0], ii); - if (gen) - *gen = btrfs_inode_generation(path->nodes[0], ii); - if (mode) - *mode = btrfs_inode_mode(path->nodes[0], ii); - if (uid) - *uid = btrfs_inode_uid(path->nodes[0], ii); - if (gid) - *gid = btrfs_inode_gid(path->nodes[0], ii); - if (rdev) - *rdev = btrfs_inode_rdev(path->nodes[0], ii); + info->size = btrfs_inode_size(path->nodes[0], ii); + info->gen = btrfs_inode_generation(path->nodes[0], ii); + info->mode = btrfs_inode_mode(path->nodes[0], ii); + info->uid = btrfs_inode_uid(path->nodes[0], ii); + info->gid = btrfs_inode_gid(path->nodes[0], ii); + info->rdev = btrfs_inode_rdev(path->nodes[0], ii); + info->nlink = btrfs_inode_nlink(path->nodes[0], ii); /* * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's * otherwise logically split to 32/32 parts. */ - if (fileattr) - *fileattr = btrfs_inode_flags(path->nodes[0], ii); + info->fileattr = btrfs_inode_flags(path->nodes[0], ii); +out: + btrfs_free_path(path); return ret; } -static int get_inode_info(struct btrfs_root *root, - u64 ino, u64 *size, u64 *gen, - u64 *mode, u64 *uid, u64 *gid, - u64 *rdev, u64 *fileattr) +static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen) { - struct btrfs_path *path; int ret; + struct btrfs_inode_info info; - path = alloc_path_for_send(); - if (!path) - return -ENOMEM; - ret = __get_inode_info(root, path, ino, size, gen, mode, uid, gid, - rdev, fileattr); - btrfs_free_path(path); + if (!gen) + return -EPERM; + + ret = get_inode_info(root, ino, &info); + if (!ret) + *gen = info.gen; return ret; } @@ -1643,21 +1658,22 @@ static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen) int right_ret; u64 left_gen; u64 right_gen; + struct btrfs_inode_info info; - ret = get_inode_info(sctx->send_root, ino, NULL, &left_gen, NULL, NULL, - NULL, NULL, NULL); + ret = get_inode_info(sctx->send_root, ino, &info); if (ret < 0 && ret != -ENOENT) goto out; - left_ret = ret; + left_ret = (info.nlink == 0) ? -ENOENT : ret; + left_gen = info.gen; if (!sctx->parent_root) { right_ret = -ENOENT; } else { - ret = get_inode_info(sctx->parent_root, ino, NULL, &right_gen, - NULL, NULL, NULL, NULL, NULL); + ret = get_inode_info(sctx->parent_root, ino, &info); if (ret < 0 && ret != -ENOENT) goto out; - right_ret = ret; + right_ret = (info.nlink == 0) ? -ENOENT : ret; + right_gen = info.gen; } if (!left_ret && !right_ret) { @@ -1816,8 +1832,7 @@ static int get_first_ref(struct btrfs_root *root, u64 ino, btrfs_release_path(path); if (dir_gen) { - ret = get_inode_info(root, parent_dir, NULL, dir_gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(root, parent_dir, dir_gen); if (ret < 0) goto out; } @@ -1874,6 +1889,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, int ret = 0; u64 gen; u64 other_inode = 0; + struct btrfs_inode_info info; if (!sctx->parent_root) goto out; @@ -1888,8 +1904,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, * and we can just unlink this entry. */ if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID) { - ret = get_inode_info(sctx->parent_root, dir, NULL, &gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->parent_root, dir, &gen); if (ret < 0 && ret != -ENOENT) goto out; if (ret) { @@ -1916,13 +1931,14 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, */ if (other_inode > sctx->send_progress || is_waiting_for_move(sctx, other_inode)) { - ret = get_inode_info(sctx->parent_root, other_inode, NULL, - who_gen, who_mode, NULL, NULL, NULL, NULL); + ret = get_inode_info(sctx->parent_root, other_inode, &info); if (ret < 0) goto out; ret = 1; *who_ino = other_inode; + *who_gen = info.gen; + *who_mode = info.mode; } else { ret = 0; } @@ -1955,8 +1971,7 @@ static int did_overwrite_ref(struct send_ctx *sctx, goto out; if (dir != BTRFS_FIRST_FREE_OBJECTID) { - ret = get_inode_info(sctx->send_root, dir, NULL, &gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, dir, &gen); if (ret < 0 && ret != -ENOENT) goto out; if (ret) { @@ -1978,8 +1993,7 @@ static int did_overwrite_ref(struct send_ctx *sctx, goto out; } - ret = get_inode_info(sctx->send_root, ow_inode, NULL, &gen, NULL, NULL, - NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, ow_inode, &gen); if (ret < 0) goto out; @@ -2645,6 +2659,7 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) int ret = 0; struct fs_path *p; int cmd; + struct btrfs_inode_info info; u64 gen; u64 mode; u64 rdev; @@ -2656,10 +2671,12 @@ static int send_create_inode(struct send_ctx *sctx, u64 ino) return -ENOMEM; if (ino != sctx->cur_ino) { - ret = get_inode_info(sctx->send_root, ino, NULL, &gen, &mode, - NULL, NULL, &rdev, NULL); + ret = get_inode_info(sctx->send_root, ino, &info); if (ret < 0) goto out; + gen = info.gen; + mode = info.mode; + rdev = info.rdev; } else { gen = sctx->cur_inode_gen; mode = sctx->cur_inode_mode; @@ -3359,8 +3376,7 @@ finish: /* * The parent inode might have been deleted in the send snapshot */ - ret = get_inode_info(sctx->send_root, cur->dir, NULL, - NULL, NULL, NULL, NULL, NULL, NULL); + ret = get_inode_info(sctx->send_root, cur->dir, NULL); if (ret == -ENOENT) { ret = 0; continue; @@ -3534,12 +3550,10 @@ static int wait_for_dest_dir_move(struct send_ctx *sctx, goto out; } - ret = get_inode_info(sctx->parent_root, di_key.objectid, NULL, - &left_gen, NULL, NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen); if (ret < 0) goto out; - ret = get_inode_info(sctx->send_root, di_key.objectid, NULL, - &right_gen, NULL, NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen); if (ret < 0) { if (ret == -ENOENT) ret = 0; @@ -3669,8 +3683,7 @@ static int is_ancestor(struct btrfs_root *root, cur_offset = item_size; } - ret = get_inode_info(root, parent, NULL, &parent_gen, - NULL, NULL, NULL, NULL, NULL); + ret = get_inode_gen(root, parent, &parent_gen); if (ret < 0) goto out; ret = check_ino_in_path(root, ino1, ino1_gen, @@ -3760,9 +3773,7 @@ static int wait_for_parent_move(struct send_ctx *sctx, memcmp(path_before->start, path_after->start, len1))) { u64 parent_ino_gen; - ret = get_inode_info(sctx->parent_root, ino, NULL, - &parent_ino_gen, NULL, NULL, NULL, - NULL, NULL); + ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen); if (ret < 0) goto out; if (ino_gen == parent_ino_gen) { @@ -4441,8 +4452,7 @@ static int record_new_ref_if_needed(int num, u64 dir, int index, struct recorded_ref *ref; u64 dir_gen; - ret = get_inode_info(sctx->send_root, dir, NULL, &dir_gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, dir, &dir_gen); if (ret < 0) goto out; @@ -4472,8 +4482,7 @@ static int record_deleted_ref_if_needed(int num, u64 dir, int index, struct recorded_ref *ref; u64 dir_gen; - ret = get_inode_info(sctx->parent_root, dir, NULL, &dir_gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->parent_root, dir, &dir_gen); if (ret < 0) goto out; @@ -4886,6 +4895,84 @@ static int process_all_new_xattrs(struct send_ctx *sctx) return ret; } +static int send_verity(struct send_ctx *sctx, struct fs_path *path, + struct fsverity_descriptor *desc) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM, + le8_to_cpu(desc->hash_algorithm)); + TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE, + 1U << le8_to_cpu(desc->log_blocksize)); + TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt, + le8_to_cpu(desc->salt_size)); + TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature, + le32_to_cpu(desc->sig_size)); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + return ret; +} + +static int process_verity(struct send_ctx *sctx) +{ + int ret = 0; + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + struct inode *inode; + struct fs_path *p; + + inode = btrfs_iget(fs_info->sb, sctx->cur_ino, sctx->send_root); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ret = btrfs_get_verity_descriptor(inode, NULL, 0); + if (ret < 0) + goto iput; + + if (ret > FS_VERITY_MAX_DESCRIPTOR_SIZE) { + ret = -EMSGSIZE; + goto iput; + } + if (!sctx->verity_descriptor) { + sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE, + GFP_KERNEL); + if (!sctx->verity_descriptor) { + ret = -ENOMEM; + goto iput; + } + } + + ret = btrfs_get_verity_descriptor(inode, sctx->verity_descriptor, ret); + if (ret < 0) + goto iput; + + p = fs_path_alloc(); + if (!p) { + ret = -ENOMEM; + goto iput; + } + ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, p); + if (ret < 0) + goto free_path; + + ret = send_verity(sctx, p, sctx->verity_descriptor); + if (ret < 0) + goto free_path; + +free_path: + fs_path_free(p); +iput: + iput(inode); + return ret; +} + static inline u64 max_send_read_size(const struct send_ctx *sctx) { return sctx->send_max_size - SZ_16K; @@ -5056,8 +5143,7 @@ static int send_clone(struct send_ctx *sctx, TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); if (clone_root->root == sctx->send_root) { - ret = get_inode_info(sctx->send_root, clone_root->ino, NULL, - &gen, NULL, NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen); if (ret < 0) goto out; ret = get_cur_path(sctx, clone_root->ino, gen, p); @@ -5536,6 +5622,7 @@ static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path, struct btrfs_path *path; struct btrfs_key key; int ret; + struct btrfs_inode_info info; u64 clone_src_i_size = 0; /* @@ -5565,12 +5652,11 @@ static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path, * There are inodes that have extents that lie behind its i_size. Don't * accept clones from these extents. */ - ret = __get_inode_info(clone_root->root, path, clone_root->ino, - &clone_src_i_size, NULL, NULL, NULL, NULL, NULL, - NULL); + ret = get_inode_info(clone_root->root, clone_root->ino, &info); btrfs_release_path(path); if (ret < 0) goto out; + clone_src_i_size = info.size; /* * We can't send a clone operation for the entire range if we find @@ -6259,6 +6345,7 @@ out: static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) { int ret = 0; + struct btrfs_inode_info info; u64 left_mode; u64 left_uid; u64 left_gid; @@ -6301,11 +6388,13 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) goto out; if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino) goto out; - - ret = get_inode_info(sctx->send_root, sctx->cur_ino, NULL, NULL, - &left_mode, &left_uid, &left_gid, NULL, &left_fileattr); + ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info); if (ret < 0) goto out; + left_mode = info.mode; + left_uid = info.uid; + left_gid = info.gid; + left_fileattr = info.fileattr; if (!sctx->parent_root || sctx->cur_inode_new) { need_chown = 1; @@ -6316,11 +6405,14 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) } else { u64 old_size; - ret = get_inode_info(sctx->parent_root, sctx->cur_ino, - &old_size, NULL, &right_mode, &right_uid, - &right_gid, NULL, &right_fileattr); + ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info); if (ret < 0) goto out; + old_size = info.size; + right_mode = info.mode; + right_uid = info.uid; + right_gid = info.gid; + right_fileattr = info.fileattr; if (left_uid != right_uid || left_gid != right_gid) need_chown = 1; @@ -6377,6 +6469,11 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) if (ret < 0) goto out; } + if (sctx->cur_inode_needs_verity) { + ret = process_verity(sctx); + if (ret < 0) + goto out; + } ret = send_capabilities(sctx); if (ret < 0) @@ -6407,86 +6504,6 @@ out: return ret; } -struct parent_paths_ctx { - struct list_head *refs; - struct send_ctx *sctx; -}; - -static int record_parent_ref(int num, u64 dir, int index, struct fs_path *name, - void *ctx) -{ - struct parent_paths_ctx *ppctx = ctx; - - /* - * Pass 0 as the generation for the directory, we don't care about it - * here as we have no new references to add, we just want to delete all - * references for an inode. - */ - return record_ref_in_tree(&ppctx->sctx->rbtree_deleted_refs, ppctx->refs, - name, dir, 0, ppctx->sctx); -} - -/* - * Issue unlink operations for all paths of the current inode found in the - * parent snapshot. - */ -static int btrfs_unlink_all_paths(struct send_ctx *sctx) -{ - LIST_HEAD(deleted_refs); - struct btrfs_path *path; - struct btrfs_root *root = sctx->parent_root; - struct btrfs_key key; - struct btrfs_key found_key; - struct parent_paths_ctx ctx; - int iter_ret = 0; - int ret; - - path = alloc_path_for_send(); - if (!path) - return -ENOMEM; - - key.objectid = sctx->cur_ino; - key.type = BTRFS_INODE_REF_KEY; - key.offset = 0; - - ctx.refs = &deleted_refs; - ctx.sctx = sctx; - - btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { - if (found_key.objectid != key.objectid) - break; - if (found_key.type != key.type && - found_key.type != BTRFS_INODE_EXTREF_KEY) - break; - - ret = iterate_inode_ref(root, path, &found_key, 1, - record_parent_ref, &ctx); - if (ret < 0) - goto out; - } - /* Catch error found during iteration */ - if (iter_ret < 0) { - ret = iter_ret; - goto out; - } - - while (!list_empty(&deleted_refs)) { - struct recorded_ref *ref; - - ref = list_first_entry(&deleted_refs, struct recorded_ref, list); - ret = send_unlink(sctx, ref->full_path); - if (ret < 0) - goto out; - recorded_ref_free(ref); - } - ret = 0; -out: - btrfs_free_path(path); - if (ret) - __free_recorded_refs(&deleted_refs); - return ret; -} - static void close_current_inode(struct send_ctx *sctx) { u64 i_size; @@ -6577,25 +6594,36 @@ static int changed_inode(struct send_ctx *sctx, * file descriptor against it or turning a RO snapshot into RW mode, * keep an open file descriptor against a file, delete it and then * turn the snapshot back to RO mode before using it for a send - * operation. So if we find such cases, ignore the inode and all its - * items completely if it's a new inode, or if it's a changed inode - * make sure all its previous paths (from the parent snapshot) are all - * unlinked and all other the inode items are ignored. + * operation. The former is what the receiver operation does. + * Therefore, if we want to send these snapshots soon after they're + * received, we need to handle orphan inodes as well. Moreover, orphans + * can appear not only in the send snapshot but also in the parent + * snapshot. Here are several cases: + * + * Case 1: BTRFS_COMPARE_TREE_NEW + * | send snapshot | action + * -------------------------------- + * nlink | 0 | ignore + * + * Case 2: BTRFS_COMPARE_TREE_DELETED + * | parent snapshot | action + * ---------------------------------- + * nlink | 0 | as usual + * Note: No unlinks will be sent because there're no paths for it. + * + * Case 3: BTRFS_COMPARE_TREE_CHANGED + * | | parent snapshot | send snapshot | action + * ----------------------------------------------------------------------- + * subcase 1 | nlink | 0 | 0 | ignore + * subcase 2 | nlink | >0 | 0 | new_gen(deletion) + * subcase 3 | nlink | 0 | >0 | new_gen(creation) + * */ - if (result == BTRFS_COMPARE_TREE_NEW || - result == BTRFS_COMPARE_TREE_CHANGED) { - u32 nlinks; - - nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii); - if (nlinks == 0) { + if (result == BTRFS_COMPARE_TREE_NEW) { + if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) { sctx->ignore_cur_inode = true; - if (result == BTRFS_COMPARE_TREE_CHANGED) - ret = btrfs_unlink_all_paths(sctx); goto out; } - } - - if (result == BTRFS_COMPARE_TREE_NEW) { sctx->cur_inode_gen = left_gen; sctx->cur_inode_new = true; sctx->cur_inode_deleted = false; @@ -6616,6 +6644,16 @@ static int changed_inode(struct send_ctx *sctx, sctx->cur_inode_mode = btrfs_inode_mode( sctx->right_path->nodes[0], right_ii); } else if (result == BTRFS_COMPARE_TREE_CHANGED) { + u32 new_nlinks, old_nlinks; + + new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii); + old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii); + if (new_nlinks == 0 && old_nlinks == 0) { + sctx->ignore_cur_inode = true; + goto out; + } else if (new_nlinks == 0 || old_nlinks == 0) { + sctx->cur_inode_new_gen = 1; + } /* * We need to do some special handling in case the inode was * reported as changed with a changed generation number. This @@ -6642,38 +6680,44 @@ static int changed_inode(struct send_ctx *sctx, /* * Now process the inode as if it was new. */ - sctx->cur_inode_gen = left_gen; - sctx->cur_inode_new = true; - sctx->cur_inode_deleted = false; - sctx->cur_inode_size = btrfs_inode_size( - sctx->left_path->nodes[0], left_ii); - sctx->cur_inode_mode = btrfs_inode_mode( - sctx->left_path->nodes[0], left_ii); - sctx->cur_inode_rdev = btrfs_inode_rdev( - sctx->left_path->nodes[0], left_ii); - ret = send_create_inode_if_needed(sctx); - if (ret < 0) - goto out; + if (new_nlinks > 0) { + sctx->cur_inode_gen = left_gen; + sctx->cur_inode_new = true; + sctx->cur_inode_deleted = false; + sctx->cur_inode_size = btrfs_inode_size( + sctx->left_path->nodes[0], + left_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->left_path->nodes[0], + left_ii); + sctx->cur_inode_rdev = btrfs_inode_rdev( + sctx->left_path->nodes[0], + left_ii); + ret = send_create_inode_if_needed(sctx); + if (ret < 0) + goto out; - ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW); - if (ret < 0) - goto out; - /* - * Advance send_progress now as we did not get into - * process_recorded_refs_if_needed in the new_gen case. - */ - sctx->send_progress = sctx->cur_ino + 1; + ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW); + if (ret < 0) + goto out; + /* + * Advance send_progress now as we did not get + * into process_recorded_refs_if_needed in the + * new_gen case. + */ + sctx->send_progress = sctx->cur_ino + 1; - /* - * Now process all extents and xattrs of the inode as if - * they were all new. - */ - ret = process_all_extents(sctx); - if (ret < 0) - goto out; - ret = process_all_new_xattrs(sctx); - if (ret < 0) - goto out; + /* + * Now process all extents and xattrs of the + * inode as if they were all new. + */ + ret = process_all_extents(sctx); + if (ret < 0) + goto out; + ret = process_all_new_xattrs(sctx); + if (ret < 0) + goto out; + } } else { sctx->cur_inode_gen = left_gen; sctx->cur_inode_new = false; @@ -6785,18 +6829,27 @@ static int changed_extent(struct send_ctx *sctx, return ret; } +static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result) +{ + int ret = 0; + + if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { + if (result == BTRFS_COMPARE_TREE_NEW) + sctx->cur_inode_needs_verity = true; + } + return ret; +} + static int dir_changed(struct send_ctx *sctx, u64 dir) { u64 orig_gen, new_gen; int ret; - ret = get_inode_info(sctx->send_root, dir, NULL, &new_gen, NULL, NULL, - NULL, NULL, NULL); + ret = get_inode_gen(sctx->send_root, dir, &new_gen); if (ret) return ret; - ret = get_inode_info(sctx->parent_root, dir, NULL, &orig_gen, NULL, - NULL, NULL, NULL, NULL); + ret = get_inode_gen(sctx->parent_root, dir, &orig_gen); if (ret) return ret; @@ -6939,6 +6992,9 @@ static int changed_cb(struct btrfs_path *left_path, ret = changed_xattr(sctx, result); else if (key->type == BTRFS_EXTENT_DATA_KEY) ret = changed_extent(sctx, result); + else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY && + key->offset == 0) + ret = changed_verity(sctx, result); } out: @@ -8036,6 +8092,7 @@ out: kvfree(sctx->clone_roots); kfree(sctx->send_buf_pages); kvfree(sctx->send_buf); + kvfree(sctx->verity_descriptor); name_cache_free(sctx); diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h index 4bb4e6a638cb4a4d50a30591704821a203fdbc62..0a4537775e0c375c1139fea75d8bb22aa7373609 100644 --- a/fs/btrfs/send.h +++ b/fs/btrfs/send.h @@ -92,8 +92,11 @@ enum btrfs_send_cmd { BTRFS_SEND_C_ENCODED_WRITE = 25, BTRFS_SEND_C_MAX_V2 = 25, + /* Version 3 */ + BTRFS_SEND_C_ENABLE_VERITY = 26, + BTRFS_SEND_C_MAX_V3 = 26, /* End */ - BTRFS_SEND_C_MAX = 25, + BTRFS_SEND_C_MAX = 26, }; /* attributes in send stream */ @@ -160,8 +163,14 @@ enum { BTRFS_SEND_A_ENCRYPTION = 31, BTRFS_SEND_A_MAX_V2 = 31, - /* End */ - BTRFS_SEND_A_MAX = 31, + /* Version 3 */ + BTRFS_SEND_A_VERITY_ALGORITHM = 32, + BTRFS_SEND_A_VERITY_BLOCK_SIZE = 33, + BTRFS_SEND_A_VERITY_SALT_DATA = 34, + BTRFS_SEND_A_VERITY_SIG_DATA = 35, + BTRFS_SEND_A_MAX_V3 = 35, + + __BTRFS_SEND_A_MAX = 35, }; long btrfs_ioctl_send(struct inode *inode, struct btrfs_ioctl_send_args *arg); diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index d0cbeb7ae81c12ba4bf69fe9cb5c1669b56b3890..f171bf8756336453118155f6948adfaf152d8365 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -199,7 +199,7 @@ static u64 calc_chunk_size(const struct btrfs_fs_info *fs_info, u64 flags) ASSERT(flags & BTRFS_BLOCK_GROUP_TYPE_MASK); if (flags & BTRFS_BLOCK_GROUP_DATA) - return SZ_1G; + return BTRFS_MAX_DATA_CHUNK_SIZE; else if (flags & BTRFS_BLOCK_GROUP_SYSTEM) return SZ_32M; @@ -293,32 +293,36 @@ out: return ret; } -void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, - u64 total_bytes, u64 bytes_used, - u64 bytes_readonly, u64 bytes_zone_unusable, - bool active, struct btrfs_space_info **space_info) +void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, + struct btrfs_block_group *block_group) { struct btrfs_space_info *found; - int factor; + int factor, index; - factor = btrfs_bg_type_to_factor(flags); + factor = btrfs_bg_type_to_factor(block_group->flags); - found = btrfs_find_space_info(info, flags); + found = btrfs_find_space_info(info, block_group->flags); ASSERT(found); spin_lock(&found->lock); - found->total_bytes += total_bytes; - if (active) - found->active_total_bytes += total_bytes; - found->disk_total += total_bytes * factor; - found->bytes_used += bytes_used; - found->disk_used += bytes_used * factor; - found->bytes_readonly += bytes_readonly; - found->bytes_zone_unusable += bytes_zone_unusable; - if (total_bytes > 0) + found->total_bytes += block_group->length; + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) + found->active_total_bytes += block_group->length; + found->disk_total += block_group->length * factor; + found->bytes_used += block_group->used; + found->disk_used += block_group->used * factor; + found->bytes_readonly += block_group->bytes_super; + found->bytes_zone_unusable += block_group->zone_unusable; + if (block_group->length > 0) found->full = 0; btrfs_try_granting_tickets(info, found); spin_unlock(&found->lock); - *space_info = found; + + block_group->space_info = found; + + index = btrfs_bg_flags_to_raid_index(block_group->flags); + down_write(&found->groups_sem); + list_add_tail(&block_group->list, &found->block_groups[index]); + up_write(&found->groups_sem); } struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, @@ -472,28 +476,47 @@ do { \ spin_unlock(&__rsv->lock); \ } while (0) +static const char *space_info_flag_to_str(const struct btrfs_space_info *space_info) +{ + switch (space_info->flags) { + case BTRFS_BLOCK_GROUP_SYSTEM: + return "SYSTEM"; + case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA: + return "DATA+METADATA"; + case BTRFS_BLOCK_GROUP_DATA: + return "DATA"; + case BTRFS_BLOCK_GROUP_METADATA: + return "METADATA"; + default: + return "UNKNOWN"; + } +} + +static void dump_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + DUMP_BLOCK_RSV(fs_info, global_block_rsv); + DUMP_BLOCK_RSV(fs_info, trans_block_rsv); + DUMP_BLOCK_RSV(fs_info, chunk_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); +} + static void __btrfs_dump_space_info(struct btrfs_fs_info *fs_info, struct btrfs_space_info *info) { + const char *flag_str = space_info_flag_to_str(info); lockdep_assert_held(&info->lock); /* The free space could be negative in case of overcommit */ - btrfs_info(fs_info, "space_info %llu has %lld free, is %sfull", - info->flags, + btrfs_info(fs_info, "space_info %s has %lld free, is %sfull", + flag_str, (s64)(info->total_bytes - btrfs_space_info_used(info, true)), info->full ? "" : "not "); btrfs_info(fs_info, - "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu", +"space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu", info->total_bytes, info->bytes_used, info->bytes_pinned, info->bytes_reserved, info->bytes_may_use, info->bytes_readonly, info->bytes_zone_unusable); - - DUMP_BLOCK_RSV(fs_info, global_block_rsv); - DUMP_BLOCK_RSV(fs_info, trans_block_rsv); - DUMP_BLOCK_RSV(fs_info, chunk_block_rsv); - DUMP_BLOCK_RSV(fs_info, delayed_block_rsv); - DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); - } void btrfs_dump_space_info(struct btrfs_fs_info *fs_info, @@ -505,6 +528,7 @@ void btrfs_dump_space_info(struct btrfs_fs_info *fs_info, spin_lock(&info->lock); __btrfs_dump_space_info(fs_info, info); + dump_global_block_rsv(fs_info); spin_unlock(&info->lock); if (!dump_block_groups) @@ -1662,7 +1686,6 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, &space_info->priority_tickets); } } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { - used += orig_bytes; /* * We will do the space reservation dance during log replay, * which means we won't have fs_info->fs_root set, so don't do @@ -1737,7 +1760,8 @@ int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, int ret; ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || - flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE); + flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || + flush == BTRFS_RESERVE_NO_FLUSH); ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA); ret = __reserve_bytes(fs_info, data_sinfo, bytes, flush); @@ -1749,3 +1773,17 @@ int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, } return ret; } + +/* Dump all the space infos when we abort a transaction due to ENOSPC. */ +__cold void btrfs_dump_space_info_for_trans_abort(struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *space_info; + + btrfs_info(fs_info, "dumping space info:"); + list_for_each_entry(space_info, &fs_info->space_info, list) { + spin_lock(&space_info->lock); + __btrfs_dump_space_info(fs_info, space_info); + spin_unlock(&space_info->lock); + } + dump_global_block_rsv(fs_info); +} diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 12fd6147f92d60461bc42418d5fb883f7ab6d221..ce66023a9eb8b0e2e7e4901fdd93b3eaf2d3aa7d 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -123,10 +123,8 @@ DECLARE_SPACE_INFO_UPDATE(bytes_may_use, "space_info"); DECLARE_SPACE_INFO_UPDATE(bytes_pinned, "pinned"); int btrfs_init_space_info(struct btrfs_fs_info *fs_info); -void btrfs_update_space_info(struct btrfs_fs_info *info, u64 flags, - u64 total_bytes, u64 bytes_used, - u64 bytes_readonly, u64 bytes_zone_unusable, - bool active, struct btrfs_space_info **space_info); +void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, + struct btrfs_block_group *block_group); void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info, u64 chunk_size); struct btrfs_space_info *btrfs_find_space_info(struct btrfs_fs_info *info, @@ -159,4 +157,7 @@ static inline void btrfs_space_info_free_bytes_may_use( } int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, enum btrfs_reserve_flush_enum flush); +void btrfs_dump_space_info_for_trans_abort(struct btrfs_fs_info *fs_info); +void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info); + #endif /* BTRFS_SPACE_INFO_H */ diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 6fc2b77ae5c345e4df4180d0b6c6026aa8edb0f8..9a176af847d7f013ad031a03bfcd063b16962f6b 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -337,7 +337,7 @@ bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, * * Even with 0 returned, the page still need extra check to make sure * it's really the correct page, as the caller is using - * find_get_pages_contig(), which can race with page invalidating. + * filemap_get_folios_contig(), which can race with page invalidating. */ int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, struct page *page, u64 start, u32 len) diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index f89beac3c665630982bff57c8a09d3706e32e4b3..9be4fd2db0f44e01b6e4a16157acc66b1871ad8c 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -346,12 +346,14 @@ void __cold btrfs_err_32bit_limit(struct btrfs_fs_info *fs_info) __cold void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, - unsigned int line, int errno) + unsigned int line, int errno, bool first_hit) { struct btrfs_fs_info *fs_info = trans->fs_info; WRITE_ONCE(trans->aborted, errno); WRITE_ONCE(trans->transaction->aborted, errno); + if (first_hit && errno == -ENOSPC) + btrfs_dump_space_info_for_trans_abort(fs_info); /* Wake up anybody who may be waiting on this transaction */ wake_up(&fs_info->transaction_wait); wake_up(&fs_info->transaction_blocked_wait); @@ -626,6 +628,7 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, int saved_compress_level; bool saved_compress_force; int no_compress = 0; + const bool remounting = test_bit(BTRFS_FS_STATE_REMOUNTING, &info->fs_state); if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE)) btrfs_set_opt(info->mount_opt, FREE_SPACE_TREE); @@ -1137,10 +1140,12 @@ out: } if (!ret) ret = btrfs_check_mountopts_zoned(info); - if (!ret && btrfs_test_opt(info, SPACE_CACHE)) - btrfs_info(info, "disk space caching is enabled"); - if (!ret && btrfs_test_opt(info, FREE_SPACE_TREE)) - btrfs_info(info, "using free space tree"); + if (!ret && !remounting) { + if (btrfs_test_opt(info, SPACE_CACHE)) + btrfs_info(info, "disk space caching is enabled"); + if (btrfs_test_opt(info, FREE_SPACE_TREE)) + btrfs_info(info, "using free space tree"); + } return ret; } @@ -2009,14 +2014,10 @@ static int btrfs_remount(struct super_block *sb, int *flags, char *data) if (ret) goto restore; - /* V1 cache is not supported for subpage mount. */ - if (fs_info->sectorsize < PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) { - btrfs_warn(fs_info, - "v1 space cache is not supported for page size %lu with sectorsize %u", - PAGE_SIZE, fs_info->sectorsize); - ret = -EINVAL; + ret = btrfs_check_features(fs_info, sb); + if (ret < 0) goto restore; - } + btrfs_remount_begin(fs_info, old_opts, *flags); btrfs_resize_thread_pool(fs_info, fs_info->thread_pool_size, old_thread_pool_size); @@ -2550,11 +2551,71 @@ static int btrfs_freeze(struct super_block *sb) return btrfs_commit_transaction(trans); } +static int check_dev_super(struct btrfs_device *dev) +{ + struct btrfs_fs_info *fs_info = dev->fs_info; + struct btrfs_super_block *sb; + int ret = 0; + + /* This should be called with fs still frozen. */ + ASSERT(test_bit(BTRFS_FS_FROZEN, &fs_info->flags)); + + /* Missing dev, no need to check. */ + if (!dev->bdev) + return 0; + + /* Only need to check the primary super block. */ + sb = btrfs_read_dev_one_super(dev->bdev, 0, true); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + /* Btrfs_validate_super() includes fsid check against super->fsid. */ + ret = btrfs_validate_super(fs_info, sb, 0); + if (ret < 0) + goto out; + + if (btrfs_super_generation(sb) != fs_info->last_trans_committed) { + btrfs_err(fs_info, "transid mismatch, has %llu expect %llu", + btrfs_super_generation(sb), + fs_info->last_trans_committed); + ret = -EUCLEAN; + goto out; + } +out: + btrfs_release_disk_super(sb); + return ret; +} + static int btrfs_unfreeze(struct super_block *sb) { struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_device *device; + int ret = 0; + /* + * Make sure the fs is not changed by accident (like hibernation then + * modified by other OS). + * If we found anything wrong, we mark the fs error immediately. + * + * And since the fs is frozen, no one can modify the fs yet, thus + * we don't need to hold device_list_mutex. + */ + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + ret = check_dev_super(device); + if (ret < 0) { + btrfs_handle_fs_error(fs_info, ret, + "super block on devid %llu got modified unexpectedly", + device->devid); + break; + } + } clear_bit(BTRFS_FS_FROZEN, &fs_info->flags); + + /* + * We still return 0, to allow VFS layer to unfreeze the fs even the + * above checks failed. Since the fs is either fine or read-only, we're + * safe to continue, without causing further damage. + */ return 0; } @@ -2662,17 +2723,21 @@ static int __init init_btrfs_fs(void) if (err) goto free_compress; - err = extent_io_init(); + err = extent_state_init_cachep(); if (err) goto free_cachep; - err = extent_state_cache_init(); + err = extent_buffer_init_cachep(); + if (err) + goto free_extent_cachep; + + err = btrfs_bioset_init(); if (err) - goto free_extent_io; + goto free_eb_cachep; err = extent_map_init(); if (err) - goto free_extent_state_cache; + goto free_bioset; err = ordered_data_init(); if (err) @@ -2724,10 +2789,12 @@ free_ordered_data: ordered_data_exit(); free_extent_map: extent_map_exit(); -free_extent_state_cache: - extent_state_cache_exit(); -free_extent_io: - extent_io_exit(); +free_bioset: + btrfs_bioset_exit(); +free_eb_cachep: + extent_buffer_free_cachep(); +free_extent_cachep: + extent_state_free_cachep(); free_cachep: btrfs_destroy_cachep(); free_compress: @@ -2746,8 +2813,9 @@ static void __exit exit_btrfs_fs(void) btrfs_prelim_ref_exit(); ordered_data_exit(); extent_map_exit(); - extent_state_cache_exit(); - extent_io_exit(); + btrfs_bioset_exit(); + extent_state_free_cachep(); + extent_buffer_free_cachep(); btrfs_interface_exit(); unregister_filesystem(&btrfs_fs_type); btrfs_exit_sysfs(); diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index d5d0717fd09a353e9f2c60c922c69abbe61aea4a..699b54b3acaae0b6e8e31f69e34066631b1aaa9c 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -35,12 +35,12 @@ * qgroup_attrs /sys/fs/btrfs//qgroups/_ * space_info_attrs /sys/fs/btrfs//allocation/ * raid_attrs /sys/fs/btrfs//allocation// + * discard_attrs /sys/fs/btrfs//discard * * When built with BTRFS_CONFIG_DEBUG: * * btrfs_debug_feature_attrs /sys/fs/btrfs/debug * btrfs_debug_mount_attrs /sys/fs/btrfs//debug - * discard_debug_attrs /sys/fs/btrfs//debug/discard */ struct btrfs_feature_attr { @@ -286,6 +286,7 @@ BTRFS_FEAT_ATTR_INCOMPAT(skinny_metadata, SKINNY_METADATA); BTRFS_FEAT_ATTR_INCOMPAT(no_holes, NO_HOLES); BTRFS_FEAT_ATTR_INCOMPAT(metadata_uuid, METADATA_UUID); BTRFS_FEAT_ATTR_COMPAT_RO(free_space_tree, FREE_SPACE_TREE); +BTRFS_FEAT_ATTR_COMPAT_RO(block_group_tree, BLOCK_GROUP_TREE); BTRFS_FEAT_ATTR_INCOMPAT(raid1c34, RAID1C34); #ifdef CONFIG_BLK_DEV_ZONED BTRFS_FEAT_ATTR_INCOMPAT(zoned, ZONED); @@ -317,6 +318,7 @@ static struct attribute *btrfs_supported_feature_attrs[] = { BTRFS_FEAT_ATTR_PTR(metadata_uuid), BTRFS_FEAT_ATTR_PTR(free_space_tree), BTRFS_FEAT_ATTR_PTR(raid1c34), + BTRFS_FEAT_ATTR_PTR(block_group_tree), #ifdef CONFIG_BLK_DEV_ZONED BTRFS_FEAT_ATTR_PTR(zoned), #endif @@ -429,12 +431,10 @@ static const struct attribute_group btrfs_static_feature_attr_group = { .attrs = btrfs_supported_static_feature_attrs, }; -#ifdef CONFIG_BTRFS_DEBUG - /* * Discard statistics and tunables */ -#define discard_to_fs_info(_kobj) to_fs_info((_kobj)->parent->parent) +#define discard_to_fs_info(_kobj) to_fs_info(get_btrfs_kobj(_kobj)) static ssize_t btrfs_discardable_bytes_show(struct kobject *kobj, struct kobj_attribute *a, @@ -583,11 +583,11 @@ BTRFS_ATTR_RW(discard, max_discard_size, btrfs_discard_max_discard_size_show, btrfs_discard_max_discard_size_store); /* - * Per-filesystem debugging of discard (when mounted with discard=async). + * Per-filesystem stats for discard (when mounted with discard=async). * - * Path: /sys/fs/btrfs//debug/discard/ + * Path: /sys/fs/btrfs//discard/ */ -static const struct attribute *discard_debug_attrs[] = { +static const struct attribute *discard_attrs[] = { BTRFS_ATTR_PTR(discard, discardable_bytes), BTRFS_ATTR_PTR(discard, discardable_extents), BTRFS_ATTR_PTR(discard, discard_bitmap_bytes), @@ -599,6 +599,8 @@ static const struct attribute *discard_debug_attrs[] = { NULL, }; +#ifdef CONFIG_BTRFS_DEBUG + /* * Per-filesystem runtime debugging exported via sysfs. * @@ -837,11 +839,8 @@ static ssize_t btrfs_sinfo_bg_reclaim_threshold_show(struct kobject *kobj, char *buf) { struct btrfs_space_info *space_info = to_space_info(kobj); - ssize_t ret; - ret = sysfs_emit(buf, "%d\n", READ_ONCE(space_info->bg_reclaim_threshold)); - - return ret; + return sysfs_emit(buf, "%d\n", READ_ONCE(space_info->bg_reclaim_threshold)); } static ssize_t btrfs_sinfo_bg_reclaim_threshold_store(struct kobject *kobj, @@ -1150,25 +1149,6 @@ static ssize_t btrfs_generation_show(struct kobject *kobj, } BTRFS_ATTR(, generation, btrfs_generation_show); -/* - * Look for an exact string @string in @buffer with possible leading or - * trailing whitespace - */ -static bool strmatch(const char *buffer, const char *string) -{ - const size_t len = strlen(string); - - /* Skip leading whitespace */ - buffer = skip_spaces(buffer); - - /* Match entire string, check if the rest is whitespace or empty */ - if (strncmp(string, buffer, len) == 0 && - strlen(skip_spaces(buffer + len)) == 0) - return true; - - return false; -} - static const char * const btrfs_read_policy_name[] = { "pid" }; static ssize_t btrfs_read_policy_show(struct kobject *kobj, @@ -1202,7 +1182,7 @@ static ssize_t btrfs_read_policy_store(struct kobject *kobj, int i; for (i = 0; i < BTRFS_NR_READ_POLICY; i++) { - if (strmatch(buf, btrfs_read_policy_name[i])) { + if (sysfs_streq(buf, btrfs_read_policy_name[i])) { if (i != fs_devices->read_policy) { fs_devices->read_policy = i; btrfs_info(fs_devices->fs_info, @@ -1222,11 +1202,8 @@ static ssize_t btrfs_bg_reclaim_threshold_show(struct kobject *kobj, char *buf) { struct btrfs_fs_info *fs_info = to_fs_info(kobj); - ssize_t ret; - ret = sysfs_emit(buf, "%d\n", READ_ONCE(fs_info->bg_reclaim_threshold)); - - return ret; + return sysfs_emit(buf, "%d\n", READ_ONCE(fs_info->bg_reclaim_threshold)); } static ssize_t btrfs_bg_reclaim_threshold_store(struct kobject *kobj, @@ -1427,13 +1404,12 @@ void btrfs_sysfs_remove_mounted(struct btrfs_fs_info *fs_info) kobject_del(fs_info->space_info_kobj); kobject_put(fs_info->space_info_kobj); } -#ifdef CONFIG_BTRFS_DEBUG - if (fs_info->discard_debug_kobj) { - sysfs_remove_files(fs_info->discard_debug_kobj, - discard_debug_attrs); - kobject_del(fs_info->discard_debug_kobj); - kobject_put(fs_info->discard_debug_kobj); + if (fs_info->discard_kobj) { + sysfs_remove_files(fs_info->discard_kobj, discard_attrs); + kobject_del(fs_info->discard_kobj); + kobject_put(fs_info->discard_kobj); } +#ifdef CONFIG_BTRFS_DEBUG if (fs_info->debug_kobj) { sysfs_remove_files(fs_info->debug_kobj, btrfs_debug_mount_attrs); kobject_del(fs_info->debug_kobj); @@ -2001,20 +1977,18 @@ int btrfs_sysfs_add_mounted(struct btrfs_fs_info *fs_info) error = sysfs_create_files(fs_info->debug_kobj, btrfs_debug_mount_attrs); if (error) goto failure; +#endif /* Discard directory */ - fs_info->discard_debug_kobj = kobject_create_and_add("discard", - fs_info->debug_kobj); - if (!fs_info->discard_debug_kobj) { + fs_info->discard_kobj = kobject_create_and_add("discard", fsid_kobj); + if (!fs_info->discard_kobj) { error = -ENOMEM; goto failure; } - error = sysfs_create_files(fs_info->discard_debug_kobj, - discard_debug_attrs); + error = sysfs_create_files(fs_info->discard_kobj, discard_attrs); if (error) goto failure; -#endif error = addrm_unknown_feature_attrs(fs_info, true); if (error) @@ -2041,6 +2015,98 @@ failure: return error; } +static ssize_t qgroup_enabled_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + bool enabled; + + spin_lock(&fs_info->qgroup_lock); + enabled = fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON; + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", enabled); +} +BTRFS_ATTR(qgroups, enabled, qgroup_enabled_show); + +static ssize_t qgroup_inconsistent_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + bool inconsistent; + + spin_lock(&fs_info->qgroup_lock); + inconsistent = (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT); + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", inconsistent); +} +BTRFS_ATTR(qgroups, inconsistent, qgroup_inconsistent_show); + +static ssize_t qgroup_drop_subtree_thres_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + u8 result; + + spin_lock(&fs_info->qgroup_lock); + result = fs_info->qgroup_drop_subtree_thres; + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", result); +} + +static ssize_t qgroup_drop_subtree_thres_store(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + u8 new_thres; + int ret; + + ret = kstrtou8(buf, 10, &new_thres); + if (ret) + return -EINVAL; + + if (new_thres > BTRFS_MAX_LEVEL) + return -EINVAL; + + spin_lock(&fs_info->qgroup_lock); + fs_info->qgroup_drop_subtree_thres = new_thres; + spin_unlock(&fs_info->qgroup_lock); + + return len; +} +BTRFS_ATTR_RW(qgroups, drop_subtree_threshold, qgroup_drop_subtree_thres_show, + qgroup_drop_subtree_thres_store); + +/* + * Qgroups global info + * + * Path: /sys/fs/btrfs//qgroups/ + */ +static struct attribute *qgroups_attrs[] = { + BTRFS_ATTR_PTR(qgroups, enabled), + BTRFS_ATTR_PTR(qgroups, inconsistent), + BTRFS_ATTR_PTR(qgroups, drop_subtree_threshold), + NULL +}; +ATTRIBUTE_GROUPS(qgroups); + +static void qgroups_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static struct kobj_type qgroups_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = qgroups_groups, + .release = qgroups_release, +}; + static inline struct btrfs_fs_info *qgroup_kobj_to_fs_info(struct kobject *kobj) { return to_fs_info(kobj->parent->parent); @@ -2166,11 +2232,15 @@ int btrfs_sysfs_add_qgroups(struct btrfs_fs_info *fs_info) if (fs_info->qgroups_kobj) return 0; - fs_info->qgroups_kobj = kobject_create_and_add("qgroups", fsid_kobj); - if (!fs_info->qgroups_kobj) { - ret = -ENOMEM; + fs_info->qgroups_kobj = kzalloc(sizeof(struct kobject), GFP_KERNEL); + if (!fs_info->qgroups_kobj) + return -ENOMEM; + + ret = kobject_init_and_add(fs_info->qgroups_kobj, &qgroups_ktype, + fsid_kobj, "qgroups"); + if (ret < 0) goto out; - } + rbtree_postorder_for_each_entry_safe(qgroup, next, &fs_info->qgroup_tree, node) { ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index cc9377cf56a33ff3375373d351f89327d194042e..9c478fa256f65cdaaf1771c10367b520d20478b8 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -243,7 +243,7 @@ void btrfs_free_dummy_block_group(struct btrfs_block_group *cache) { if (!cache) return; - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); kfree(cache->free_space_ctl); kfree(cache); } diff --git a/fs/btrfs/tests/extent-io-tests.c b/fs/btrfs/tests/extent-io-tests.c index a232b15b8021f8aa44fe791ccee44878e18a1657..350da449db084a23ef553c715102ffa8cfde5ac3 100644 --- a/fs/btrfs/tests/extent-io-tests.c +++ b/fs/btrfs/tests/extent-io-tests.c @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -20,39 +21,40 @@ static noinline int process_page_range(struct inode *inode, u64 start, u64 end, unsigned long flags) { int ret; - struct page *pages[16]; + struct folio_batch fbatch; unsigned long index = start >> PAGE_SHIFT; unsigned long end_index = end >> PAGE_SHIFT; - unsigned long nr_pages = end_index - index + 1; int i; int count = 0; int loops = 0; - while (nr_pages > 0) { - ret = find_get_pages_contig(inode->i_mapping, index, - min_t(unsigned long, nr_pages, - ARRAY_SIZE(pages)), pages); + folio_batch_init(&fbatch); + + while (index <= end_index) { + ret = filemap_get_folios_contig(inode->i_mapping, &index, + end_index, &fbatch); for (i = 0; i < ret; i++) { + struct folio *folio = fbatch.folios[i]; + if (flags & PROCESS_TEST_LOCKED && - !PageLocked(pages[i])) + !folio_test_locked(folio)) count++; - if (flags & PROCESS_UNLOCK && PageLocked(pages[i])) - unlock_page(pages[i]); - put_page(pages[i]); + if (flags & PROCESS_UNLOCK && folio_test_locked(folio)) + folio_unlock(folio); if (flags & PROCESS_RELEASE) - put_page(pages[i]); + folio_put(folio); } - nr_pages -= ret; - index += ret; + folio_batch_release(&fbatch); cond_resched(); loops++; if (loops > 100000) { printk(KERN_ERR - "stuck in a loop, start %llu, end %llu, nr_pages %lu, ret %d\n", - start, end, nr_pages, ret); + "stuck in a loop, start %llu, end %llu, ret %d\n", + start, end, ret); break; } } + return count; } @@ -80,7 +82,6 @@ static void extent_flag_to_str(const struct extent_state *state, char *dest) PRINT_ONE_FLAG(state, dest, cur, NODATASUM); PRINT_ONE_FLAG(state, dest, cur, CLEAR_META_RESV); PRINT_ONE_FLAG(state, dest, cur, NEED_WAIT); - PRINT_ONE_FLAG(state, dest, cur, DAMAGED); PRINT_ONE_FLAG(state, dest, cur, NORESERVE); PRINT_ONE_FLAG(state, dest, cur, QGROUP_RESERVED); PRINT_ONE_FLAG(state, dest, cur, CLEAR_DATA_RESV); @@ -172,7 +173,7 @@ static int test_find_delalloc(u32 sectorsize) sectorsize - 1, start, end); goto out_bits; } - unlock_extent(tmp, start, end); + unlock_extent(tmp, start, end, NULL); unlock_page(locked_page); put_page(locked_page); @@ -208,7 +209,7 @@ static int test_find_delalloc(u32 sectorsize) test_err("there were unlocked pages in the range"); goto out_bits; } - unlock_extent(tmp, start, end); + unlock_extent(tmp, start, end, NULL); /* locked_page was unlocked above */ put_page(locked_page); @@ -263,7 +264,7 @@ static int test_find_delalloc(u32 sectorsize) test_err("pages in range were not all locked"); goto out_bits; } - unlock_extent(tmp, start, end); + unlock_extent(tmp, start, end, NULL); /* * Now to test where we run into a page that is no longer dirty in the diff --git a/fs/btrfs/tests/free-space-tests.c b/fs/btrfs/tests/free-space-tests.c index 5930cdcae5cb699af626c48f9cea4d46b04e8a28..ebf68fcd2149de122f25013d44c45705257bd023 100644 --- a/fs/btrfs/tests/free-space-tests.c +++ b/fs/btrfs/tests/free-space-tests.c @@ -82,7 +82,7 @@ static int test_extents(struct btrfs_block_group *cache) } /* Cleanup */ - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); return 0; } @@ -149,7 +149,7 @@ static int test_bitmaps(struct btrfs_block_group *cache, u32 sectorsize) return -1; } - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); return 0; } @@ -230,7 +230,7 @@ static int test_bitmaps_and_extents(struct btrfs_block_group *cache, return -1; } - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); /* Now with the extent entry offset into the bitmap */ ret = test_add_free_space_entry(cache, SZ_4M, SZ_4M, 1); @@ -266,7 +266,7 @@ static int test_bitmaps_and_extents(struct btrfs_block_group *cache, * [ bitmap ] * [ del ] */ - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); ret = test_add_free_space_entry(cache, bitmap_offset + SZ_4M, SZ_4M, 1); if (ret) { test_err("couldn't add bitmap %d", ret); @@ -291,7 +291,7 @@ static int test_bitmaps_and_extents(struct btrfs_block_group *cache, return -1; } - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); /* * This blew up before, we have part of the free space in a bitmap and @@ -317,7 +317,7 @@ static int test_bitmaps_and_extents(struct btrfs_block_group *cache, return ret; } - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); return 0; } @@ -629,7 +629,7 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, if (ret) return ret; - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); /* * Now test a similar scenario, but where our extent entry is located @@ -819,7 +819,7 @@ test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, return ret; cache->free_space_ctl->op = orig_free_space_ops; - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); return 0; } @@ -868,7 +868,7 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) } /* Now validate bitmaps do the correct thing. */ - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); for (i = 0; i < 2; i++) { offset = i * BITS_PER_BITMAP * sectorsize; bytes = (i + 1) * SZ_1M; @@ -891,7 +891,7 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) } /* Now validate bitmaps with different ->max_extent_size. */ - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); orig_free_space_ops = cache->free_space_ctl->op; cache->free_space_ctl->op = &test_free_space_ops; @@ -998,7 +998,7 @@ static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) } cache->free_space_ctl->op = orig_free_space_ops; - __btrfs_remove_free_space_cache(cache->free_space_ctl); + btrfs_remove_free_space_cache(cache); return 0; } diff --git a/fs/btrfs/tests/inode-tests.c b/fs/btrfs/tests/inode-tests.c index cac89c38813119221b08b1106357c8d1341eaba7..625f7d398368d949bffb4fd3033b1449f4743daa 100644 --- a/fs/btrfs/tests/inode-tests.c +++ b/fs/btrfs/tests/inode-tests.c @@ -267,7 +267,7 @@ static noinline int test_btrfs_get_extent(u32 sectorsize, u32 nodesize) goto out; } free_extent_map(em); - btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0); + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); /* * All of the magic numbers are based on the mapping setup in @@ -975,7 +975,7 @@ static int test_extent_accounting(u32 sectorsize, u32 nodesize) BTRFS_MAX_EXTENT_SIZE >> 1, (BTRFS_MAX_EXTENT_SIZE >> 1) + sectorsize - 1, EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | - EXTENT_UPTODATE, 0, 0, NULL); + EXTENT_UPTODATE, NULL); if (ret) { test_err("clear_extent_bit returned %d", ret); goto out; @@ -1043,7 +1043,7 @@ static int test_extent_accounting(u32 sectorsize, u32 nodesize) BTRFS_MAX_EXTENT_SIZE + sectorsize, BTRFS_MAX_EXTENT_SIZE + 2 * sectorsize - 1, EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | - EXTENT_UPTODATE, 0, 0, NULL); + EXTENT_UPTODATE, NULL); if (ret) { test_err("clear_extent_bit returned %d", ret); goto out; @@ -1076,7 +1076,7 @@ static int test_extent_accounting(u32 sectorsize, u32 nodesize) /* Empty */ ret = clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, (u64)-1, EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | - EXTENT_UPTODATE, 0, 0, NULL); + EXTENT_UPTODATE, NULL); if (ret) { test_err("clear_extent_bit returned %d", ret); goto out; @@ -1092,7 +1092,7 @@ out: if (ret) clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, (u64)-1, EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | - EXTENT_UPTODATE, 0, 0, NULL); + EXTENT_UPTODATE, NULL); iput(inode); btrfs_free_dummy_root(root); btrfs_free_dummy_fs_info(fs_info); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 0bec10740ad3921ec878ed0368d9425a32843eac..d1f1da6820fb0d51ee8deb349ffb42a3e57e59fe 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -161,7 +161,6 @@ static noinline void switch_commit_roots(struct btrfs_trans_handle *trans) struct btrfs_transaction *cur_trans = trans->transaction; struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_root *root, *tmp; - struct btrfs_caching_control *caching_ctl, *next; /* * At this point no one can be using this transaction to modify any tree @@ -196,46 +195,6 @@ static noinline void switch_commit_roots(struct btrfs_trans_handle *trans) } spin_unlock(&cur_trans->dropped_roots_lock); - /* - * We have to update the last_byte_to_unpin under the commit_root_sem, - * at the same time we swap out the commit roots. - * - * This is because we must have a real view of the last spot the caching - * kthreads were while caching. Consider the following views of the - * extent tree for a block group - * - * commit root - * +----+----+----+----+----+----+----+ - * |\\\\| |\\\\|\\\\| |\\\\|\\\\| - * +----+----+----+----+----+----+----+ - * 0 1 2 3 4 5 6 7 - * - * new commit root - * +----+----+----+----+----+----+----+ - * | | | |\\\\| | |\\\\| - * +----+----+----+----+----+----+----+ - * 0 1 2 3 4 5 6 7 - * - * If the cache_ctl->progress was at 3, then we are only allowed to - * unpin [0,1) and [2,3], because the caching thread has already - * processed those extents. We are not allowed to unpin [5,6), because - * the caching thread will re-start it's search from 3, and thus find - * the hole from [4,6) to add to the free space cache. - */ - write_lock(&fs_info->block_group_cache_lock); - list_for_each_entry_safe(caching_ctl, next, - &fs_info->caching_block_groups, list) { - struct btrfs_block_group *cache = caching_ctl->block_group; - - if (btrfs_block_group_done(cache)) { - cache->last_byte_to_unpin = (u64)-1; - list_del_init(&caching_ctl->list); - btrfs_put_caching_control(caching_ctl); - } else { - cache->last_byte_to_unpin = caching_ctl->progress; - } - } - write_unlock(&fs_info->block_group_cache_lock); up_write(&fs_info->commit_root_sem); } @@ -313,6 +272,8 @@ loop: atomic_inc(&cur_trans->num_writers); extwriter_counter_inc(cur_trans, type); spin_unlock(&fs_info->trans_lock); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); return 0; } spin_unlock(&fs_info->trans_lock); @@ -334,16 +295,23 @@ loop: if (!cur_trans) return -ENOMEM; + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); + spin_lock(&fs_info->trans_lock); if (fs_info->running_transaction) { /* * someone started a transaction after we unlocked. Make sure * to redo the checks above */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); kfree(cur_trans); goto loop; } else if (BTRFS_FS_ERROR(fs_info)) { spin_unlock(&fs_info->trans_lock); + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); kfree(cur_trans); return -EROFS; } @@ -397,7 +365,7 @@ loop: spin_lock_init(&cur_trans->releasing_ebs_lock); list_add_tail(&cur_trans->list, &fs_info->trans_list); extent_io_tree_init(fs_info, &cur_trans->dirty_pages, - IO_TREE_TRANS_DIRTY_PAGES, fs_info->btree_inode); + IO_TREE_TRANS_DIRTY_PAGES, NULL); extent_io_tree_init(fs_info, &cur_trans->pinned_extents, IO_TREE_FS_PINNED_EXTENTS, NULL); fs_info->generation++; @@ -541,6 +509,7 @@ static void wait_current_trans(struct btrfs_fs_info *fs_info) refcount_inc(&cur_trans->use_count); spin_unlock(&fs_info->trans_lock); + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); wait_event(fs_info->transaction_wait, cur_trans->state >= TRANS_STATE_UNBLOCKED || TRANS_ABORTED(cur_trans)); @@ -625,7 +594,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, */ num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items); if (flush == BTRFS_RESERVE_FLUSH_ALL && - delayed_refs_rsv->full == 0) { + btrfs_block_rsv_full(delayed_refs_rsv) == 0) { delayed_refs_bytes = num_bytes; num_bytes <<= 1; } @@ -650,7 +619,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, if (rsv->space_info->force_alloc) do_chunk_alloc = true; } else if (num_items == 0 && flush == BTRFS_RESERVE_FLUSH_ALL && - !delayed_refs_rsv->full) { + !btrfs_block_rsv_full(delayed_refs_rsv)) { /* * Some people call with btrfs_start_transaction(root, 0) * because they can be throttled, but have some other mechanism @@ -859,6 +828,15 @@ static noinline void wait_for_commit(struct btrfs_transaction *commit, u64 transid = commit->transid; bool put = false; + /* + * At the moment this function is called with min_state either being + * TRANS_STATE_COMPLETED or TRANS_STATE_SUPER_COMMITTED. + */ + if (min_state == TRANS_STATE_COMPLETED) + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + else + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + while (1) { wait_event(commit->commit_wait, commit->state >= min_state); if (put) @@ -1022,6 +1000,10 @@ static int __btrfs_end_transaction(struct btrfs_trans_handle *trans, extwriter_counter_dec(cur_trans, trans->type); cond_wake_up(&cur_trans->writer_wait); + + btrfs_lockdep_release(info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(info, btrfs_trans_num_writers); + btrfs_put_transaction(cur_trans); if (current->journal_info == trans) @@ -1134,7 +1116,7 @@ static int __btrfs_wait_marked_extents(struct btrfs_fs_info *fs_info, * it's safe to do it (through extent_io_tree_release()). */ err = clear_extent_bit(dirty_pages, start, end, - EXTENT_NEED_WAIT, 0, 0, &cached_state); + EXTENT_NEED_WAIT, &cached_state); if (err == -ENOMEM) err = 0; if (!err) @@ -1912,14 +1894,6 @@ static void update_super_roots(struct btrfs_fs_info *fs_info) super->cache_generation = 0; if (test_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags)) super->uuid_tree_generation = root_item->generation; - - if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { - root_item = &fs_info->block_group_root->root_item; - - super->block_group_root = root_item->bytenr; - super->block_group_root_generation = root_item->generation; - super->block_group_root_level = root_item->level; - } } int btrfs_transaction_in_commit(struct btrfs_fs_info *info) @@ -1967,6 +1941,7 @@ void btrfs_commit_transaction_async(struct btrfs_trans_handle *trans) * Wait for the current transaction commit to start and block * subsequent transaction joins */ + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); wait_event(fs_info->transaction_blocked_wait, cur_trans->state >= TRANS_STATE_COMMIT_START || TRANS_ABORTED(cur_trans)); @@ -1994,6 +1969,12 @@ static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) if (cur_trans == fs_info->running_transaction) { cur_trans->state = TRANS_STATE_COMMIT_DOING; spin_unlock(&fs_info->trans_lock); + + /* + * The thread has already released the lockdep map as reader + * already in btrfs_commit_transaction(). + */ + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); wait_event(cur_trans->writer_wait, atomic_read(&cur_trans->num_writers) == 1); @@ -2118,12 +2099,12 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ktime_t interval; ASSERT(refcount_read(&trans->use_count) == 1); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); /* Stop the commit early if ->aborted is set */ if (TRANS_ABORTED(cur_trans)) { ret = cur_trans->aborted; - btrfs_end_transaction(trans); - return ret; + goto lockdep_trans_commit_start_release; } btrfs_trans_release_metadata(trans); @@ -2140,10 +2121,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * Any running threads may add more while we are here. */ ret = btrfs_run_delayed_refs(trans, 0); - if (ret) { - btrfs_end_transaction(trans); - return ret; - } + if (ret) + goto lockdep_trans_commit_start_release; } btrfs_create_pending_block_groups(trans); @@ -2172,10 +2151,8 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (run_it) { ret = btrfs_start_dirty_block_groups(trans); - if (ret) { - btrfs_end_transaction(trans); - return ret; - } + if (ret) + goto lockdep_trans_commit_start_release; } } @@ -2190,6 +2167,9 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (trans->in_fsync) want_state = TRANS_STATE_SUPER_COMMITTED; + + btrfs_trans_state_lockdep_release(fs_info, + BTRFS_LOCKDEP_TRANS_COMMIT_START); ret = btrfs_end_transaction(trans); wait_for_commit(cur_trans, want_state); @@ -2203,6 +2183,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) cur_trans->state = TRANS_STATE_COMMIT_START; wake_up(&fs_info->transaction_blocked_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); if (cur_trans->list.prev != &fs_info->trans_list) { enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; @@ -2222,7 +2203,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) btrfs_put_transaction(prev_trans); if (ret) - goto cleanup_transaction; + goto lockdep_release; } else { spin_unlock(&fs_info->trans_lock); } @@ -2236,7 +2217,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) */ if (BTRFS_FS_ERROR(fs_info)) { ret = -EROFS; - goto cleanup_transaction; + goto lockdep_release; } } @@ -2250,19 +2231,28 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ret = btrfs_start_delalloc_flush(fs_info); if (ret) - goto cleanup_transaction; + goto lockdep_release; ret = btrfs_run_delayed_items(trans); if (ret) - goto cleanup_transaction; + goto lockdep_release; + /* + * The thread has started/joined the transaction thus it holds the + * lockdep map as a reader. It has to release it before acquiring the + * lockdep map as a writer. + */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_extwriters); wait_event(cur_trans->writer_wait, extwriter_counter_read(cur_trans) == 0); /* some pending stuffs might be added after the previous flush. */ ret = btrfs_run_delayed_items(trans); - if (ret) + if (ret) { + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); goto cleanup_transaction; + } btrfs_wait_delalloc_flush(fs_info); @@ -2271,6 +2261,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) * transaction. Otherwise if this transaction commits before the ordered * extents complete we lose logged data after a power failure. */ + btrfs_might_wait_for_event(fs_info, btrfs_trans_pending_ordered); wait_event(cur_trans->pending_wait, atomic_read(&cur_trans->pending_ordered) == 0); @@ -2284,9 +2275,27 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) add_pending_snapshot(trans); cur_trans->state = TRANS_STATE_COMMIT_DOING; spin_unlock(&fs_info->trans_lock); + + /* + * The thread has started/joined the transaction thus it holds the + * lockdep map as a reader. It has to release it before acquiring the + * lockdep map as a writer. + */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); wait_event(cur_trans->writer_wait, atomic_read(&cur_trans->num_writers) == 1); + /* + * Make lockdep happy by acquiring the state locks after + * btrfs_trans_num_writers is released. If we acquired the state locks + * before releasing the btrfs_trans_num_writers lock then lockdep would + * complain because we did not follow the reverse order unlocking rule. + */ + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + /* * We've started the commit, clear the flag in case we were triggered to * do an async commit but somebody else started before the transaction @@ -2296,6 +2305,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) if (TRANS_ABORTED(cur_trans)) { ret = cur_trans->aborted; + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); goto scrub_continue; } /* @@ -2430,6 +2440,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) mutex_unlock(&fs_info->reloc_mutex); wake_up(&fs_info->transaction_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); ret = btrfs_write_and_wait_transaction(trans); if (ret) { @@ -2461,6 +2472,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) */ cur_trans->state = TRANS_STATE_SUPER_COMMITTED; wake_up(&cur_trans->commit_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); btrfs_finish_extent_commit(trans); @@ -2474,6 +2486,7 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) */ cur_trans->state = TRANS_STATE_COMPLETED; wake_up(&cur_trans->commit_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); spin_lock(&fs_info->trans_lock); list_del_init(&cur_trans->list); @@ -2502,7 +2515,10 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) unlock_reloc: mutex_unlock(&fs_info->reloc_mutex); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); scrub_continue: + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); btrfs_scrub_continue(fs_info); cleanup_transaction: btrfs_trans_release_metadata(trans); @@ -2515,6 +2531,16 @@ cleanup_transaction: cleanup_transaction(trans, ret); return ret; + +lockdep_release: + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + goto cleanup_transaction; + +lockdep_trans_commit_start_release: + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_START); + btrfs_end_transaction(trans); + return ret; } /* diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 9205c4a5ca81dd4ea4854ba2b1a29d149121a0e3..813986e38258b78ceed6f4207dd1e38b3e66a8b2 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -22,6 +22,8 @@ #include "zoned.h" #include "inode-item.h" +#define MAX_CONFLICT_INODES 10 + /* magic values for the inode_only field in btrfs_log_inode: * * LOG_INODE_ALL means to log everything @@ -31,8 +33,6 @@ enum { LOG_INODE_ALL, LOG_INODE_EXISTS, - LOG_OTHER_INODE, - LOG_OTHER_INODE_ALL, }; /* @@ -801,7 +801,7 @@ static noinline int replay_one_extent(struct btrfs_trans_handle *trans, ret = btrfs_lookup_csums_range(root->log_root, csum_start, csum_end - 1, - &ordered_sums, 0); + &ordered_sums, 0, false); if (ret) goto out; /* @@ -1063,8 +1063,7 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans, struct btrfs_inode *dir, struct btrfs_inode *inode, u64 inode_objectid, u64 parent_objectid, - u64 ref_index, char *name, int namelen, - int *search_done) + u64 ref_index, char *name, int namelen) { int ret; char *victim_name; @@ -1126,19 +1125,12 @@ again: kfree(victim_name); if (ret) return ret; - *search_done = 1; goto again; } kfree(victim_name); ptr = (unsigned long)(victim_ref + 1) + victim_name_len; } - - /* - * NOTE: we have searched root tree and checked the - * corresponding ref, it does not need to check again. - */ - *search_done = 1; } btrfs_release_path(path); @@ -1202,14 +1194,12 @@ again: kfree(victim_name); if (ret) return ret; - *search_done = 1; goto again; } kfree(victim_name); next: cur_offset += victim_name_len + sizeof(*extref); } - *search_done = 1; } btrfs_release_path(path); @@ -1373,103 +1363,6 @@ again: return ret; } -static int btrfs_inode_ref_exists(struct inode *inode, struct inode *dir, - const u8 ref_type, const char *name, - const int namelen) -{ - struct btrfs_key key; - struct btrfs_path *path; - const u64 parent_id = btrfs_ino(BTRFS_I(dir)); - int ret; - - path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; - - key.objectid = btrfs_ino(BTRFS_I(inode)); - key.type = ref_type; - if (key.type == BTRFS_INODE_REF_KEY) - key.offset = parent_id; - else - key.offset = btrfs_extref_hash(parent_id, name, namelen); - - ret = btrfs_search_slot(NULL, BTRFS_I(inode)->root, &key, path, 0, 0); - if (ret < 0) - goto out; - if (ret > 0) { - ret = 0; - goto out; - } - if (key.type == BTRFS_INODE_EXTREF_KEY) - ret = !!btrfs_find_name_in_ext_backref(path->nodes[0], - path->slots[0], parent_id, name, namelen); - else - ret = !!btrfs_find_name_in_backref(path->nodes[0], path->slots[0], - name, namelen); - -out: - btrfs_free_path(path); - return ret; -} - -static int add_link(struct btrfs_trans_handle *trans, - struct inode *dir, struct inode *inode, const char *name, - int namelen, u64 ref_index) -{ - struct btrfs_root *root = BTRFS_I(dir)->root; - struct btrfs_dir_item *dir_item; - struct btrfs_key key; - struct btrfs_path *path; - struct inode *other_inode = NULL; - int ret; - - path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; - - dir_item = btrfs_lookup_dir_item(NULL, root, path, - btrfs_ino(BTRFS_I(dir)), - name, namelen, 0); - if (!dir_item) { - btrfs_release_path(path); - goto add_link; - } else if (IS_ERR(dir_item)) { - ret = PTR_ERR(dir_item); - goto out; - } - - /* - * Our inode's dentry collides with the dentry of another inode which is - * in the log but not yet processed since it has a higher inode number. - * So delete that other dentry. - */ - btrfs_dir_item_key_to_cpu(path->nodes[0], dir_item, &key); - btrfs_release_path(path); - other_inode = read_one_inode(root, key.objectid); - if (!other_inode) { - ret = -ENOENT; - goto out; - } - ret = unlink_inode_for_log_replay(trans, BTRFS_I(dir), BTRFS_I(other_inode), - name, namelen); - if (ret) - goto out; - /* - * If we dropped the link count to 0, bump it so that later the iput() - * on the inode will not free it. We will fixup the link count later. - */ - if (other_inode->i_nlink == 0) - set_nlink(other_inode, 1); -add_link: - ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), - name, namelen, 0, ref_index); -out: - iput(other_inode); - btrfs_free_path(path); - - return ret; -} - /* * replay one inode back reference item found in the log tree. * eb, slot and key refer to the buffer and key found in the log tree. @@ -1490,7 +1383,6 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans, char *name = NULL; int namelen; int ret; - int search_done = 0; int log_ref_ver = 0; u64 parent_objectid; u64 inode_objectid; @@ -1565,51 +1457,19 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans, * overwrite any existing back reference, and we don't * want to create dangling pointers in the directory. */ - - if (!search_done) { - ret = __add_inode_ref(trans, root, path, log, - BTRFS_I(dir), - BTRFS_I(inode), - inode_objectid, - parent_objectid, - ref_index, name, namelen, - &search_done); - if (ret) { - if (ret == 1) - ret = 0; - goto out; - } - } - - /* - * If a reference item already exists for this inode - * with the same parent and name, but different index, - * drop it and the corresponding directory index entries - * from the parent before adding the new reference item - * and dir index entries, otherwise we would fail with - * -EEXIST returned from btrfs_add_link() below. - */ - ret = btrfs_inode_ref_exists(inode, dir, key->type, - name, namelen); - if (ret > 0) { - ret = unlink_inode_for_log_replay(trans, - BTRFS_I(dir), - BTRFS_I(inode), - name, namelen); - /* - * If we dropped the link count to 0, bump it so - * that later the iput() on the inode will not - * free it. We will fixup the link count later. - */ - if (!ret && inode->i_nlink == 0) - set_nlink(inode, 1); - } - if (ret < 0) + ret = __add_inode_ref(trans, root, path, log, + BTRFS_I(dir), BTRFS_I(inode), + inode_objectid, parent_objectid, + ref_index, name, namelen); + if (ret) { + if (ret == 1) + ret = 0; goto out; + } /* insert our name */ - ret = add_link(trans, dir, inode, name, namelen, - ref_index); + ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), + name, namelen, 0, ref_index); if (ret) goto out; @@ -3875,6 +3735,11 @@ static int process_dir_items_leaf(struct btrfs_trans_handle *trans, *last_old_dentry_offset = key.offset; continue; } + + /* If we logged this dir index item before, we can skip it. */ + if (key.offset <= inode->last_dir_index_offset) + continue; + /* * We must make sure that when we log a directory entry, the * corresponding inode, after log replay, has a matching link @@ -3905,51 +3770,6 @@ static int process_dir_items_leaf(struct btrfs_trans_handle *trans, ctx->log_new_dentries = true; } - if (!ctx->logged_before) - goto add_to_batch; - - /* - * If we were logged before and have logged dir items, we can skip - * checking if any item with a key offset larger than the last one - * we logged is in the log tree, saving time and avoiding adding - * contention on the log tree. We can only rely on the value of - * last_dir_index_offset when we know for sure that the inode was - * previously logged in the current transaction. - */ - if (key.offset > inode->last_dir_index_offset) - goto add_to_batch; - /* - * Check if the key was already logged before. If not we can add - * it to a batch for bulk insertion. - */ - ret = btrfs_search_slot(NULL, log, &key, dst_path, 0, 0); - if (ret < 0) { - return ret; - } else if (ret > 0) { - btrfs_release_path(dst_path); - goto add_to_batch; - } - - /* - * Item exists in the log. Overwrite the item in the log if it - * has different content or do nothing if it has exactly the same - * content. And then flush the current batch if any - do it after - * overwriting the current item, or we would deadlock otherwise, - * since we are holding a path for the existing item. - */ - ret = do_overwrite_item(trans, log, dst_path, src, i, &key); - if (ret < 0) - return ret; - - if (batch_size > 0) { - ret = flush_dir_items_batch(trans, log, src, dst_path, - batch_start, batch_size); - if (ret < 0) - return ret; - batch_size = 0; - } - continue; -add_to_batch: if (batch_size == 0) batch_start = i; batch_size++; @@ -4135,6 +3955,71 @@ done: return err; } +/* + * If the inode was logged before and it was evicted, then its + * last_dir_index_offset is (u64)-1, so we don't the value of the last index + * key offset. If that's the case, search for it and update the inode. This + * is to avoid lookups in the log tree every time we try to insert a dir index + * key from a leaf changed in the current transaction, and to allow us to always + * do batch insertions of dir index keys. + */ +static int update_last_dir_index_offset(struct btrfs_inode *inode, + struct btrfs_path *path, + const struct btrfs_log_ctx *ctx) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_key key; + int ret; + + lockdep_assert_held(&inode->log_mutex); + + if (inode->last_dir_index_offset != (u64)-1) + return 0; + + if (!ctx->logged_before) { + inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; + return 0; + } + + key.objectid = ino; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + /* + * An error happened or we actually have an index key with an offset + * value of (u64)-1. Bail out, we're done. + */ + if (ret <= 0) + goto out; + + ret = 0; + inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; + + /* + * No dir index items, bail out and leave last_dir_index_offset with + * the value right before the first valid index value. + */ + if (path->slots[0] == 0) + goto out; + + /* + * btrfs_search_slot() left us at one slot beyond the slot with the last + * index key, or beyond the last key of the directory that is not an + * index key. If we have an index key before, set last_dir_index_offset + * to its offset value, otherwise leave it with a value right before the + * first valid index value, as it means we have an empty directory. + */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_DIR_INDEX_KEY) + inode->last_dir_index_offset = key.offset; + +out: + btrfs_release_path(path); + + return ret; +} + /* * logging directories is very similar to logging inodes, We find all the items * from the current transaction and write them to the log. @@ -4157,6 +4042,10 @@ static noinline int log_directory_changes(struct btrfs_trans_handle *trans, u64 max_key; int ret; + ret = update_last_dir_index_offset(inode, path, ctx); + if (ret) + return ret; + min_key = BTRFS_DIR_START_INDEX; max_key = 0; ctx->last_dir_item_offset = inode->last_dir_index_offset; @@ -4382,8 +4271,8 @@ static int log_csums(struct btrfs_trans_handle *trans, * file which happens to refer to the same extent as well. Such races * can leave checksum items in the log with overlapping ranges. */ - ret = lock_extent_bits(&log_root->log_csum_range, sums->bytenr, - lock_end, &cached_state); + ret = lock_extent(&log_root->log_csum_range, sums->bytenr, lock_end, + &cached_state); if (ret) return ret; /* @@ -4399,8 +4288,8 @@ static int log_csums(struct btrfs_trans_handle *trans, if (!ret) ret = btrfs_csum_file_blocks(trans, log_root, sums); - unlock_extent_cached(&log_root->log_csum_range, sums->bytenr, lock_end, - &cached_state); + unlock_extent(&log_root->log_csum_range, sums->bytenr, lock_end, + &cached_state); return ret; } @@ -4513,7 +4402,7 @@ static noinline int copy_items(struct btrfs_trans_handle *trans, disk_bytenr += extent_offset; ret = btrfs_lookup_csums_range(csum_root, disk_bytenr, disk_bytenr + extent_num_bytes - 1, - &ordered_sums, 0); + &ordered_sums, 0, false); if (ret) goto out; @@ -4709,7 +4598,7 @@ static int log_extent_csums(struct btrfs_trans_handle *trans, ret = btrfs_lookup_csums_range(csum_root, em->block_start + csum_offset, em->block_start + csum_offset + - csum_len - 1, &ordered_sums, 0); + csum_len - 1, &ordered_sums, 0, false); if (ret) return ret; @@ -5221,10 +5110,9 @@ static int btrfs_log_holes(struct btrfs_trans_handle *trans, * leafs from the log root. */ btrfs_release_path(path); - ret = btrfs_insert_file_extent(trans, root->log_root, - ino, prev_extent_end, 0, - 0, hole_len, 0, hole_len, - 0, 0, 0); + ret = btrfs_insert_hole_extent(trans, root->log_root, + ino, prev_extent_end, + hole_len); if (ret < 0) return ret; @@ -5253,10 +5141,8 @@ static int btrfs_log_holes(struct btrfs_trans_handle *trans, btrfs_release_path(path); hole_len = ALIGN(i_size - prev_extent_end, fs_info->sectorsize); - ret = btrfs_insert_file_extent(trans, root->log_root, - ino, prev_extent_end, 0, 0, - hole_len, 0, hole_len, - 0, 0, 0); + ret = btrfs_insert_hole_extent(trans, root->log_root, ino, + prev_extent_end, hole_len); if (ret < 0) return ret; } @@ -5399,111 +5285,461 @@ out: return ret; } -struct btrfs_ino_list { +/* + * Check if we need to log an inode. This is used in contexts where while + * logging an inode we need to log another inode (either that it exists or in + * full mode). This is used instead of btrfs_inode_in_log() because the later + * requires the inode to be in the log and have the log transaction committed, + * while here we do not care if the log transaction was already committed - our + * caller will commit the log later - and we want to avoid logging an inode + * multiple times when multiple tasks have joined the same log transaction. + */ +static bool need_log_inode(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode) +{ + /* + * If a directory was not modified, no dentries added or removed, we can + * and should avoid logging it. + */ + if (S_ISDIR(inode->vfs_inode.i_mode) && inode->last_trans < trans->transid) + return false; + + /* + * If this inode does not have new/updated/deleted xattrs since the last + * time it was logged and is flagged as logged in the current transaction, + * we can skip logging it. As for new/deleted names, those are updated in + * the log by link/unlink/rename operations. + * In case the inode was logged and then evicted and reloaded, its + * logged_trans will be 0, in which case we have to fully log it since + * logged_trans is a transient field, not persisted. + */ + if (inode->logged_trans == trans->transid && + !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) + return false; + + return true; +} + +struct btrfs_dir_list { u64 ino; - u64 parent; struct list_head list; }; -static int log_conflicting_inodes(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_path *path, - struct btrfs_log_ctx *ctx, - u64 ino, u64 parent) +/* + * Log the inodes of the new dentries of a directory. + * See process_dir_items_leaf() for details about why it is needed. + * This is a recursive operation - if an existing dentry corresponds to a + * directory, that directory's new entries are logged too (same behaviour as + * ext3/4, xfs, f2fs, reiserfs, nilfs2). Note that when logging the inodes + * the dentries point to we do not acquire their VFS lock, otherwise lockdep + * complains about the following circular lock dependency / possible deadlock: + * + * CPU0 CPU1 + * ---- ---- + * lock(&type->i_mutex_dir_key#3/2); + * lock(sb_internal#2); + * lock(&type->i_mutex_dir_key#3/2); + * lock(&sb->s_type->i_mutex_key#14); + * + * Where sb_internal is the lock (a counter that works as a lock) acquired by + * sb_start_intwrite() in btrfs_start_transaction(). + * Not acquiring the VFS lock of the inodes is still safe because: + * + * 1) For regular files we log with a mode of LOG_INODE_EXISTS. It's possible + * that while logging the inode new references (names) are added or removed + * from the inode, leaving the logged inode item with a link count that does + * not match the number of logged inode reference items. This is fine because + * at log replay time we compute the real number of links and correct the + * link count in the inode item (see replay_one_buffer() and + * link_to_fixup_dir()); + * + * 2) For directories we log with a mode of LOG_INODE_ALL. It's possible that + * while logging the inode's items new index items (key type + * BTRFS_DIR_INDEX_KEY) are added to fs/subvol tree and the logged inode item + * has a size that doesn't match the sum of the lengths of all the logged + * names - this is ok, not a problem, because at log replay time we set the + * directory's i_size to the correct value (see replay_one_name() and + * do_overwrite_item()). + */ +static int log_new_dir_dentries(struct btrfs_trans_handle *trans, + struct btrfs_inode *start_inode, + struct btrfs_log_ctx *ctx) { - struct btrfs_ino_list *ino_elem; - LIST_HEAD(inode_list); + struct btrfs_root *root = start_inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + LIST_HEAD(dir_list); + struct btrfs_dir_list *dir_elem; + u64 ino = btrfs_ino(start_inode); int ret = 0; - ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); - if (!ino_elem) - return -ENOMEM; - ino_elem->ino = ino; - ino_elem->parent = parent; - list_add_tail(&ino_elem->list, &inode_list); + /* + * If we are logging a new name, as part of a link or rename operation, + * don't bother logging new dentries, as we just want to log the names + * of an inode and that any new parents exist. + */ + if (ctx->logging_new_name) + return 0; - while (!list_empty(&inode_list)) { - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_key key; - struct inode *inode; + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; - ino_elem = list_first_entry(&inode_list, struct btrfs_ino_list, - list); - ino = ino_elem->ino; - parent = ino_elem->parent; - list_del(&ino_elem->list); - kfree(ino_elem); - if (ret) - continue; + while (true) { + struct extent_buffer *leaf; + struct btrfs_key min_key; + bool continue_curr_inode = true; + int nritems; + int i; + min_key.objectid = ino; + min_key.type = BTRFS_DIR_INDEX_KEY; + min_key.offset = 0; +again: btrfs_release_path(path); - - inode = btrfs_iget(fs_info->sb, ino, root); - /* - * If the other inode that had a conflicting dir entry was + ret = btrfs_search_forward(root, &min_key, path, trans->transid); + if (ret < 0) { + break; + } else if (ret > 0) { + ret = 0; + goto next; + } + + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + for (i = path->slots[0]; i < nritems; i++) { + struct btrfs_dir_item *di; + struct btrfs_key di_key; + struct inode *di_inode; + int log_mode = LOG_INODE_EXISTS; + int type; + + btrfs_item_key_to_cpu(leaf, &min_key, i); + if (min_key.objectid != ino || + min_key.type != BTRFS_DIR_INDEX_KEY) { + continue_curr_inode = false; + break; + } + + di = btrfs_item_ptr(leaf, i, struct btrfs_dir_item); + type = btrfs_dir_type(leaf, di); + if (btrfs_dir_transid(leaf, di) < trans->transid) + continue; + btrfs_dir_item_key_to_cpu(leaf, di, &di_key); + if (di_key.type == BTRFS_ROOT_ITEM_KEY) + continue; + + btrfs_release_path(path); + di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root); + if (IS_ERR(di_inode)) { + ret = PTR_ERR(di_inode); + goto out; + } + + if (!need_log_inode(trans, BTRFS_I(di_inode))) { + btrfs_add_delayed_iput(di_inode); + break; + } + + ctx->log_new_dentries = false; + if (type == BTRFS_FT_DIR) + log_mode = LOG_INODE_ALL; + ret = btrfs_log_inode(trans, BTRFS_I(di_inode), + log_mode, ctx); + btrfs_add_delayed_iput(di_inode); + if (ret) + goto out; + if (ctx->log_new_dentries) { + dir_elem = kmalloc(sizeof(*dir_elem), GFP_NOFS); + if (!dir_elem) { + ret = -ENOMEM; + goto out; + } + dir_elem->ino = di_key.objectid; + list_add_tail(&dir_elem->list, &dir_list); + } + break; + } + + if (continue_curr_inode && min_key.offset < (u64)-1) { + min_key.offset++; + goto again; + } + +next: + if (list_empty(&dir_list)) + break; + + dir_elem = list_first_entry(&dir_list, struct btrfs_dir_list, list); + ino = dir_elem->ino; + list_del(&dir_elem->list); + kfree(dir_elem); + } +out: + btrfs_free_path(path); + if (ret) { + struct btrfs_dir_list *next; + + list_for_each_entry_safe(dir_elem, next, &dir_list, list) + kfree(dir_elem); + } + + return ret; +} + +struct btrfs_ino_list { + u64 ino; + u64 parent; + struct list_head list; +}; + +static void free_conflicting_inodes(struct btrfs_log_ctx *ctx) +{ + struct btrfs_ino_list *curr; + struct btrfs_ino_list *next; + + list_for_each_entry_safe(curr, next, &ctx->conflict_inodes, list) { + list_del(&curr->list); + kfree(curr); + } +} + +static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, + struct btrfs_path *path) +{ + struct btrfs_key key; + int ret; + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + path->search_commit_root = 1; + path->skip_locking = 1; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (WARN_ON_ONCE(ret > 0)) { + /* + * We have previously found the inode through the commit root + * so this should not happen. If it does, just error out and + * fallback to a transaction commit. + */ + ret = -ENOENT; + } else if (ret == 0) { + struct btrfs_inode_item *item; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + if (S_ISDIR(btrfs_inode_mode(path->nodes[0], item))) + ret = 1; + } + + btrfs_release_path(path); + path->search_commit_root = 0; + path->skip_locking = 0; + + return ret; +} + +static int add_conflicting_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + u64 ino, u64 parent, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_ino_list *ino_elem; + struct inode *inode; + + /* + * It's rare to have a lot of conflicting inodes, in practice it is not + * common to have more than 1 or 2. We don't want to collect too many, + * as we could end up logging too many inodes (even if only in + * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction + * commits. + */ + if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) + return BTRFS_LOG_FORCE_COMMIT; + + inode = btrfs_iget(root->fs_info->sb, ino, root); + /* + * If the other inode that had a conflicting dir entry was deleted in + * the current transaction then we either: + * + * 1) Log the parent directory (later after adding it to the list) if + * the inode is a directory. This is because it may be a deleted + * subvolume/snapshot or it may be a regular directory that had + * deleted subvolumes/snapshots (or subdirectories that had them), + * and at the moment we can't deal with dropping subvolumes/snapshots + * during log replay. So we just log the parent, which will result in + * a fallback to a transaction commit if we are dealing with those + * cases (last_unlink_trans will match the current transaction); + * + * 2) Do nothing if it's not a directory. During log replay we simply + * unlink the conflicting dentry from the parent directory and then + * add the dentry for our inode. Like this we can avoid logging the + * parent directory (and maybe fallback to a transaction commit in + * case it has a last_unlink_trans == trans->transid, due to moving + * some inode from it to some other directory). + */ + if (IS_ERR(inode)) { + int ret = PTR_ERR(inode); + + if (ret != -ENOENT) + return ret; + + ret = conflicting_inode_is_dir(root, ino, path); + /* Not a directory or we got an error. */ + if (ret <= 0) + return ret; + + /* Conflicting inode is a directory, so we'll log its parent. */ + ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); + if (!ino_elem) + return -ENOMEM; + ino_elem->ino = ino; + ino_elem->parent = parent; + list_add_tail(&ino_elem->list, &ctx->conflict_inodes); + ctx->num_conflict_inodes++; + + return 0; + } + + /* + * If the inode was already logged skip it - otherwise we can hit an + * infinite loop. Example: + * + * From the commit root (previous transaction) we have the following + * inodes: + * + * inode 257 a directory + * inode 258 with references "zz" and "zz_link" on inode 257 + * inode 259 with reference "a" on inode 257 + * + * And in the current (uncommitted) transaction we have: + * + * inode 257 a directory, unchanged + * inode 258 with references "a" and "a2" on inode 257 + * inode 259 with reference "zz_link" on inode 257 + * inode 261 with reference "zz" on inode 257 + * + * When logging inode 261 the following infinite loop could + * happen if we don't skip already logged inodes: + * + * - we detect inode 258 as a conflicting inode, with inode 261 + * on reference "zz", and log it; + * + * - we detect inode 259 as a conflicting inode, with inode 258 + * on reference "a", and log it; + * + * - we detect inode 258 as a conflicting inode, with inode 259 + * on reference "zz_link", and log it - again! After this we + * repeat the above steps forever. + * + * Here we can use need_log_inode() because we only need to log the + * inode in LOG_INODE_EXISTS mode and rename operations update the log, + * so that the log ends up with the new name and without the old name. + */ + if (!need_log_inode(trans, BTRFS_I(inode))) { + btrfs_add_delayed_iput(inode); + return 0; + } + + btrfs_add_delayed_iput(inode); + + ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); + if (!ino_elem) + return -ENOMEM; + ino_elem->ino = ino; + ino_elem->parent = parent; + list_add_tail(&ino_elem->list, &ctx->conflict_inodes); + ctx->num_conflict_inodes++; + + return 0; +} + +static int log_conflicting_inodes(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + + /* + * Conflicting inodes are logged by the first call to btrfs_log_inode(), + * otherwise we could have unbounded recursion of btrfs_log_inode() + * calls. This check guarantees we can have only 1 level of recursion. + */ + if (ctx->logging_conflict_inodes) + return 0; + + ctx->logging_conflict_inodes = true; + + /* + * New conflicting inodes may be found and added to the list while we + * are logging a conflicting inode, so keep iterating while the list is + * not empty. + */ + while (!list_empty(&ctx->conflict_inodes)) { + struct btrfs_ino_list *curr; + struct inode *inode; + u64 ino; + u64 parent; + + curr = list_first_entry(&ctx->conflict_inodes, + struct btrfs_ino_list, list); + ino = curr->ino; + parent = curr->parent; + list_del(&curr->list); + kfree(curr); + + inode = btrfs_iget(fs_info->sb, ino, root); + /* + * If the other inode that had a conflicting dir entry was * deleted in the current transaction, we need to log its parent - * directory. + * directory. See the comment at add_conflicting_inode(). */ if (IS_ERR(inode)) { ret = PTR_ERR(inode); - if (ret == -ENOENT) { - inode = btrfs_iget(fs_info->sb, parent, root); - if (IS_ERR(inode)) { - ret = PTR_ERR(inode); - } else { - ret = btrfs_log_inode(trans, - BTRFS_I(inode), - LOG_OTHER_INODE_ALL, - ctx); - btrfs_add_delayed_iput(inode); - } + if (ret != -ENOENT) + break; + + inode = btrfs_iget(fs_info->sb, parent, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + break; } + + /* + * Always log the directory, we cannot make this + * conditional on need_log_inode() because the directory + * might have been logged in LOG_INODE_EXISTS mode or + * the dir index of the conflicting inode is not in a + * dir index key range logged for the directory. So we + * must make sure the deletion is recorded. + */ + ret = btrfs_log_inode(trans, BTRFS_I(inode), + LOG_INODE_ALL, ctx); + btrfs_add_delayed_iput(inode); + if (ret) + break; continue; } + /* - * If the inode was already logged skip it - otherwise we can - * hit an infinite loop. Example: - * - * From the commit root (previous transaction) we have the - * following inodes: - * - * inode 257 a directory - * inode 258 with references "zz" and "zz_link" on inode 257 - * inode 259 with reference "a" on inode 257 - * - * And in the current (uncommitted) transaction we have: - * - * inode 257 a directory, unchanged - * inode 258 with references "a" and "a2" on inode 257 - * inode 259 with reference "zz_link" on inode 257 - * inode 261 with reference "zz" on inode 257 + * Here we can use need_log_inode() because we only need to log + * the inode in LOG_INODE_EXISTS mode and rename operations + * update the log, so that the log ends up with the new name and + * without the old name. * - * When logging inode 261 the following infinite loop could - * happen if we don't skip already logged inodes: - * - * - we detect inode 258 as a conflicting inode, with inode 261 - * on reference "zz", and log it; - * - * - we detect inode 259 as a conflicting inode, with inode 258 - * on reference "a", and log it; - * - * - we detect inode 258 as a conflicting inode, with inode 259 - * on reference "zz_link", and log it - again! After this we - * repeat the above steps forever. + * We did this check at add_conflicting_inode(), but here we do + * it again because if some other task logged the inode after + * that, we can avoid doing it again. */ - spin_lock(&BTRFS_I(inode)->lock); - /* - * Check the inode's logged_trans only instead of - * btrfs_inode_in_log(). This is because the last_log_commit of - * the inode is not updated when we only log that it exists (see - * btrfs_log_inode()). - */ - if (BTRFS_I(inode)->logged_trans == trans->transid) { - spin_unlock(&BTRFS_I(inode)->lock); + if (!need_log_inode(trans, BTRFS_I(inode))) { btrfs_add_delayed_iput(inode); continue; } - spin_unlock(&BTRFS_I(inode)->lock); + /* * We are safe logging the other inode without acquiring its * lock as long as we log with the LOG_INODE_EXISTS mode. We @@ -5511,67 +5747,16 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, * well because during a rename we pin the log and update the * log with the new name before we unpin it. */ - ret = btrfs_log_inode(trans, BTRFS_I(inode), LOG_OTHER_INODE, ctx); - if (ret) { - btrfs_add_delayed_iput(inode); - continue; - } - - key.objectid = ino; - key.type = BTRFS_INODE_REF_KEY; - key.offset = 0; - ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); - if (ret < 0) { - btrfs_add_delayed_iput(inode); - continue; - } - - while (true) { - struct extent_buffer *leaf = path->nodes[0]; - int slot = path->slots[0]; - u64 other_ino = 0; - u64 other_parent = 0; - - if (slot >= btrfs_header_nritems(leaf)) { - ret = btrfs_next_leaf(root, path); - if (ret < 0) { - break; - } else if (ret > 0) { - ret = 0; - break; - } - continue; - } - - btrfs_item_key_to_cpu(leaf, &key, slot); - if (key.objectid != ino || - (key.type != BTRFS_INODE_REF_KEY && - key.type != BTRFS_INODE_EXTREF_KEY)) { - ret = 0; - break; - } - - ret = btrfs_check_ref_name_override(leaf, slot, &key, - BTRFS_I(inode), &other_ino, - &other_parent); - if (ret < 0) - break; - if (ret > 0) { - ino_elem = kmalloc(sizeof(*ino_elem), GFP_NOFS); - if (!ino_elem) { - ret = -ENOMEM; - break; - } - ino_elem->ino = other_ino; - ino_elem->parent = other_parent; - list_add_tail(&ino_elem->list, &inode_list); - ret = 0; - } - path->slots[0]++; - } + ret = btrfs_log_inode(trans, BTRFS_I(inode), LOG_INODE_EXISTS, ctx); btrfs_add_delayed_iput(inode); + if (ret) + break; } + ctx->logging_conflict_inodes = false; + if (ret) + free_conflicting_inodes(ctx); + return ret; } @@ -5582,7 +5767,6 @@ static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_path *dst_path, const u64 logged_isize, - const bool recursive_logging, const int inode_only, struct btrfs_log_ctx *ctx, bool *need_log_inode_item) @@ -5621,8 +5805,8 @@ again: break; } else if ((min_key->type == BTRFS_INODE_REF_KEY || min_key->type == BTRFS_INODE_EXTREF_KEY) && - inode->generation == trans->transid && - !recursive_logging) { + (inode->generation == trans->transid || + ctx->logging_conflict_inodes)) { u64 other_ino = 0; u64 other_parent = 0; @@ -5646,11 +5830,12 @@ again: return ret; ins_nr = 0; - ret = log_conflicting_inodes(trans, root, path, - ctx, other_ino, other_parent); + btrfs_release_path(path); + ret = add_conflicting_inode(trans, root, path, + other_ino, + other_parent, ctx); if (ret) return ret; - btrfs_release_path(path); goto next_key; } } else if (min_key->type == BTRFS_XATTR_ITEM_KEY) { @@ -5708,28 +5893,393 @@ next_key: } /* - * We may process many leaves full of items for our inode, so - * avoid monopolizing a cpu for too long by rescheduling while - * not holding locks on any tree. + * We may process many leaves full of items for our inode, so + * avoid monopolizing a cpu for too long by rescheduling while + * not holding locks on any tree. + */ + cond_resched(); + } + if (ins_nr) { + ret = copy_items(trans, inode, dst_path, path, ins_start_slot, + ins_nr, inode_only, logged_isize); + if (ret) + return ret; + } + + if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { + /* + * Release the path because otherwise we might attempt to double + * lock the same leaf with btrfs_log_prealloc_extents() below. + */ + btrfs_release_path(path); + ret = btrfs_log_prealloc_extents(trans, inode, dst_path); + } + + return ret; +} + +static int insert_delayed_items_batch(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_path *path, + const struct btrfs_item_batch *batch, + const struct btrfs_delayed_item *first_item) +{ + const struct btrfs_delayed_item *curr = first_item; + int ret; + + ret = btrfs_insert_empty_items(trans, log, path, batch); + if (ret) + return ret; + + for (int i = 0; i < batch->nr; i++) { + char *data_ptr; + + data_ptr = btrfs_item_ptr(path->nodes[0], path->slots[0], char); + write_extent_buffer(path->nodes[0], &curr->data, + (unsigned long)data_ptr, curr->data_len); + curr = list_next_entry(curr, log_list); + path->slots[0]++; + } + + btrfs_release_path(path); + + return 0; +} + +static int log_delayed_insertion_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_ins_list, + struct btrfs_log_ctx *ctx) +{ + /* 195 (4095 bytes of keys and sizes) fits in a single 4K page. */ + const int max_batch_size = 195; + const int leaf_data_size = BTRFS_LEAF_DATA_SIZE(trans->fs_info); + const u64 ino = btrfs_ino(inode); + struct btrfs_root *log = inode->root->log_root; + struct btrfs_item_batch batch = { + .nr = 0, + .total_data_size = 0, + }; + const struct btrfs_delayed_item *first = NULL; + const struct btrfs_delayed_item *curr; + char *ins_data; + struct btrfs_key *ins_keys; + u32 *ins_sizes; + u64 curr_batch_size = 0; + int batch_idx = 0; + int ret; + + /* We are adding dir index items to the log tree. */ + lockdep_assert_held(&inode->log_mutex); + + /* + * We collect delayed items before copying index keys from the subvolume + * to the log tree. However just after we collected them, they may have + * been flushed (all of them or just some of them), and therefore we + * could have copied them from the subvolume tree to the log tree. + * So find the first delayed item that was not yet logged (they are + * sorted by index number). + */ + list_for_each_entry(curr, delayed_ins_list, log_list) { + if (curr->index > inode->last_dir_index_offset) { + first = curr; + break; + } + } + + /* Empty list or all delayed items were already logged. */ + if (!first) + return 0; + + ins_data = kmalloc(max_batch_size * sizeof(u32) + + max_batch_size * sizeof(struct btrfs_key), GFP_NOFS); + if (!ins_data) + return -ENOMEM; + ins_sizes = (u32 *)ins_data; + batch.data_sizes = ins_sizes; + ins_keys = (struct btrfs_key *)(ins_data + max_batch_size * sizeof(u32)); + batch.keys = ins_keys; + + curr = first; + while (!list_entry_is_head(curr, delayed_ins_list, log_list)) { + const u32 curr_size = curr->data_len + sizeof(struct btrfs_item); + + if (curr_batch_size + curr_size > leaf_data_size || + batch.nr == max_batch_size) { + ret = insert_delayed_items_batch(trans, log, path, + &batch, first); + if (ret) + goto out; + batch_idx = 0; + batch.nr = 0; + batch.total_data_size = 0; + curr_batch_size = 0; + first = curr; + } + + ins_sizes[batch_idx] = curr->data_len; + ins_keys[batch_idx].objectid = ino; + ins_keys[batch_idx].type = BTRFS_DIR_INDEX_KEY; + ins_keys[batch_idx].offset = curr->index; + curr_batch_size += curr_size; + batch.total_data_size += curr->data_len; + batch.nr++; + batch_idx++; + curr = list_next_entry(curr, log_list); + } + + ASSERT(batch.nr >= 1); + ret = insert_delayed_items_batch(trans, log, path, &batch, first); + + curr = list_last_entry(delayed_ins_list, struct btrfs_delayed_item, + log_list); + inode->last_dir_index_offset = curr->index; +out: + kfree(ins_data); + + return ret; +} + +static int log_delayed_deletions_full(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + const u64 ino = btrfs_ino(inode); + const struct btrfs_delayed_item *curr; + + curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, + log_list); + + while (!list_entry_is_head(curr, delayed_del_list, log_list)) { + u64 first_dir_index = curr->index; + u64 last_dir_index; + const struct btrfs_delayed_item *next; + int ret; + + /* + * Find a range of consecutive dir index items to delete. Like + * this we log a single dir range item spanning several contiguous + * dir items instead of logging one range item per dir index item. + */ + next = list_next_entry(curr, log_list); + while (!list_entry_is_head(next, delayed_del_list, log_list)) { + if (next->index != curr->index + 1) + break; + curr = next; + next = list_next_entry(next, log_list); + } + + last_dir_index = curr->index; + ASSERT(last_dir_index >= first_dir_index); + + ret = insert_dir_log_key(trans, inode->root->log_root, path, + ino, first_dir_index, last_dir_index); + if (ret) + return ret; + curr = list_next_entry(curr, log_list); + } + + return 0; +} + +static int batch_delete_dir_index_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_log_ctx *ctx, + const struct list_head *delayed_del_list, + const struct btrfs_delayed_item *first, + const struct btrfs_delayed_item **last_ret) +{ + const struct btrfs_delayed_item *next; + struct extent_buffer *leaf = path->nodes[0]; + const int last_slot = btrfs_header_nritems(leaf) - 1; + int slot = path->slots[0] + 1; + const u64 ino = btrfs_ino(inode); + + next = list_next_entry(first, log_list); + + while (slot < last_slot && + !list_entry_is_head(next, delayed_del_list, log_list)) { + struct btrfs_key key; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != ino || + key.type != BTRFS_DIR_INDEX_KEY || + key.offset != next->index) + break; + + slot++; + *last_ret = next; + next = list_next_entry(next, log_list); + } + + return btrfs_del_items(trans, inode->root->log_root, path, + path->slots[0], slot - path->slots[0]); +} + +static int log_delayed_deletions_incremental(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *log = inode->root->log_root; + const struct btrfs_delayed_item *curr; + u64 last_range_start; + u64 last_range_end = 0; + struct btrfs_key key; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_DIR_INDEX_KEY; + curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, + log_list); + + while (!list_entry_is_head(curr, delayed_del_list, log_list)) { + const struct btrfs_delayed_item *last = curr; + u64 first_dir_index = curr->index; + u64 last_dir_index; + bool deleted_items = false; + int ret; + + key.offset = curr->index; + ret = btrfs_search_slot(trans, log, &key, path, -1, 1); + if (ret < 0) { + return ret; + } else if (ret == 0) { + ret = batch_delete_dir_index_items(trans, inode, path, ctx, + delayed_del_list, curr, + &last); + if (ret) + return ret; + deleted_items = true; + } + + btrfs_release_path(path); + + /* + * If we deleted items from the leaf, it means we have a range + * item logging their range, so no need to add one or update an + * existing one. Otherwise we have to log a dir range item. */ - cond_resched(); - } - if (ins_nr) { - ret = copy_items(trans, inode, dst_path, path, ins_start_slot, - ins_nr, inode_only, logged_isize); + if (deleted_items) + goto next_batch; + + last_dir_index = last->index; + ASSERT(last_dir_index >= first_dir_index); + /* + * If this range starts right after where the previous one ends, + * then we want to reuse the previous range item and change its + * end offset to the end of this range. This is just to minimize + * leaf space usage, by avoiding adding a new range item. + */ + if (last_range_end != 0 && first_dir_index == last_range_end + 1) + first_dir_index = last_range_start; + + ret = insert_dir_log_key(trans, log, path, key.objectid, + first_dir_index, last_dir_index); if (ret) return ret; + + last_range_start = first_dir_index; + last_range_end = last_dir_index; +next_batch: + curr = list_next_entry(last, log_list); } - if (inode_only == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { - /* - * Release the path because otherwise we might attempt to double - * lock the same leaf with btrfs_log_prealloc_extents() below. - */ - btrfs_release_path(path); - ret = btrfs_log_prealloc_extents(trans, inode, dst_path); + return 0; +} + +static int log_delayed_deletion_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + /* + * We are deleting dir index items from the log tree or adding range + * items to it. + */ + lockdep_assert_held(&inode->log_mutex); + + if (list_empty(delayed_del_list)) + return 0; + + if (ctx->logged_before) + return log_delayed_deletions_incremental(trans, inode, path, + delayed_del_list, ctx); + + return log_delayed_deletions_full(trans, inode, path, delayed_del_list, + ctx); +} + +/* + * Similar logic as for log_new_dir_dentries(), but it iterates over the delayed + * items instead of the subvolume tree. + */ +static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + const struct list_head *delayed_ins_list, + struct btrfs_log_ctx *ctx) +{ + const bool orig_log_new_dentries = ctx->log_new_dentries; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_item *item; + int ret = 0; + + /* + * No need for the log mutex, plus to avoid potential deadlocks or + * lockdep annotations due to nesting of delayed inode mutexes and log + * mutexes. + */ + lockdep_assert_not_held(&inode->log_mutex); + + ASSERT(!ctx->logging_new_delayed_dentries); + ctx->logging_new_delayed_dentries = true; + + list_for_each_entry(item, delayed_ins_list, log_list) { + struct btrfs_dir_item *dir_item; + struct inode *di_inode; + struct btrfs_key key; + int log_mode = LOG_INODE_EXISTS; + + dir_item = (struct btrfs_dir_item *)item->data; + btrfs_disk_key_to_cpu(&key, &dir_item->location); + + if (key.type == BTRFS_ROOT_ITEM_KEY) + continue; + + di_inode = btrfs_iget(fs_info->sb, key.objectid, inode->root); + if (IS_ERR(di_inode)) { + ret = PTR_ERR(di_inode); + break; + } + + if (!need_log_inode(trans, BTRFS_I(di_inode))) { + btrfs_add_delayed_iput(di_inode); + continue; + } + + if (btrfs_stack_dir_type(dir_item) == BTRFS_FT_DIR) + log_mode = LOG_INODE_ALL; + + ctx->log_new_dentries = false; + ret = btrfs_log_inode(trans, BTRFS_I(di_inode), log_mode, ctx); + + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, BTRFS_I(di_inode), ctx); + + btrfs_add_delayed_iput(di_inode); + + if (ret) + break; } + ctx->log_new_dentries = orig_log_new_dentries; + ctx->logging_new_delayed_dentries = false; + return ret; } @@ -5764,9 +6314,10 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, u64 logged_isize = 0; bool need_log_inode_item = true; bool xattrs_logged = false; - bool recursive_logging = false; bool inode_item_dropped = true; - const bool orig_logged_before = ctx->logged_before; + bool full_dir_logging = false; + LIST_HEAD(delayed_ins_list); + LIST_HEAD(delayed_del_list); path = btrfs_alloc_path(); if (!path) @@ -5794,27 +6345,46 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, max_key.type = (u8)-1; max_key.offset = (u64)-1; + if (S_ISDIR(inode->vfs_inode.i_mode) && inode_only == LOG_INODE_ALL) + full_dir_logging = true; + /* - * Only run delayed items if we are a directory. We want to make sure - * all directory indexes hit the fs/subvolume tree so we can find them - * and figure out which index ranges have to be logged. + * If we are logging a directory while we are logging dentries of the + * delayed items of some other inode, then we need to flush the delayed + * items of this directory and not log the delayed items directly. This + * is to prevent more than one level of recursion into btrfs_log_inode() + * by having something like this: + * + * $ mkdir -p a/b/c/d/e/f/g/h/... + * $ xfs_io -c "fsync" a + * + * Where all directories in the path did not exist before and are + * created in the current transaction. + * So in such a case we directly log the delayed items of the main + * directory ("a") without flushing them first, while for each of its + * subdirectories we flush their delayed items before logging them. + * This prevents a potential unbounded recursion like this: + * + * btrfs_log_inode() + * log_new_delayed_dentries() + * btrfs_log_inode() + * log_new_delayed_dentries() + * btrfs_log_inode() + * log_new_delayed_dentries() + * (...) + * + * We have thresholds for the maximum number of delayed items to have in + * memory, and once they are hit, the items are flushed asynchronously. + * However the limit is quite high, so lets prevent deep levels of + * recursion to happen by limiting the maximum depth to be 1. */ - if (S_ISDIR(inode->vfs_inode.i_mode)) { + if (full_dir_logging && ctx->logging_new_delayed_dentries) { ret = btrfs_commit_inode_delayed_items(trans, inode); if (ret) goto out; } - if (inode_only == LOG_OTHER_INODE || inode_only == LOG_OTHER_INODE_ALL) { - recursive_logging = true; - if (inode_only == LOG_OTHER_INODE) - inode_only = LOG_INODE_EXISTS; - else - inode_only = LOG_INODE_ALL; - mutex_lock_nested(&inode->log_mutex, SINGLE_DEPTH_NESTING); - } else { - mutex_lock(&inode->log_mutex); - } + mutex_lock(&inode->log_mutex); /* * For symlinks, we must always log their content, which is stored in an @@ -5846,9 +6416,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * to known the file was moved from A to B, so logging just A would * result in losing the file after a log replay. */ - if (S_ISDIR(inode->vfs_inode.i_mode) && - inode_only == LOG_INODE_ALL && - inode->last_unlink_trans >= trans->transid) { + if (full_dir_logging && inode->last_unlink_trans >= trans->transid) { btrfs_set_log_full_commit(trans); ret = BTRFS_LOG_FORCE_COMMIT; goto out_unlock; @@ -5859,14 +6427,10 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, * copies of everything. */ if (S_ISDIR(inode->vfs_inode.i_mode)) { - int max_key_type = BTRFS_DIR_LOG_INDEX_KEY; - clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); - if (inode_only == LOG_INODE_EXISTS) - max_key_type = BTRFS_XATTR_ITEM_KEY; if (ctx->logged_before) ret = drop_inode_items(trans, log, path, inode, - max_key_type); + BTRFS_XATTR_ITEM_KEY); } else { if (inode_only == LOG_INODE_EXISTS && ctx->logged_before) { /* @@ -5922,9 +6486,19 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, if (ret) goto out_unlock; + /* + * If we are logging a directory in full mode, collect the delayed items + * before iterating the subvolume tree, so that we don't miss any new + * dir index items in case they get flushed while or right after we are + * iterating the subvolume tree. + */ + if (full_dir_logging && !ctx->logging_new_delayed_dentries) + btrfs_log_get_delayed_items(inode, &delayed_ins_list, + &delayed_del_list); + ret = copy_inode_items_to_log(trans, inode, &min_key, &max_key, path, dst_path, logged_isize, - recursive_logging, inode_only, ctx, + inode_only, ctx, &need_log_inode_item); if (ret) goto out_unlock; @@ -5977,10 +6551,18 @@ log_extents: write_unlock(&em_tree->lock); } - if (inode_only == LOG_INODE_ALL && S_ISDIR(inode->vfs_inode.i_mode)) { + if (full_dir_logging) { ret = log_directory_changes(trans, inode, path, dst_path, ctx); if (ret) goto out_unlock; + ret = log_delayed_insertion_items(trans, inode, path, + &delayed_ins_list, ctx); + if (ret) + goto out_unlock; + ret = log_delayed_deletion_items(trans, inode, path, + &delayed_del_list, ctx); + if (ret) + goto out_unlock; } spin_lock(&inode->lock); @@ -6033,208 +6615,20 @@ out: btrfs_free_path(path); btrfs_free_path(dst_path); - if (recursive_logging) - ctx->logged_before = orig_logged_before; - - return ret; -} - -/* - * Check if we need to log an inode. This is used in contexts where while - * logging an inode we need to log another inode (either that it exists or in - * full mode). This is used instead of btrfs_inode_in_log() because the later - * requires the inode to be in the log and have the log transaction committed, - * while here we do not care if the log transaction was already committed - our - * caller will commit the log later - and we want to avoid logging an inode - * multiple times when multiple tasks have joined the same log transaction. - */ -static bool need_log_inode(struct btrfs_trans_handle *trans, - struct btrfs_inode *inode) -{ - /* - * If a directory was not modified, no dentries added or removed, we can - * and should avoid logging it. - */ - if (S_ISDIR(inode->vfs_inode.i_mode) && inode->last_trans < trans->transid) - return false; - - /* - * If this inode does not have new/updated/deleted xattrs since the last - * time it was logged and is flagged as logged in the current transaction, - * we can skip logging it. As for new/deleted names, those are updated in - * the log by link/unlink/rename operations. - * In case the inode was logged and then evicted and reloaded, its - * logged_trans will be 0, in which case we have to fully log it since - * logged_trans is a transient field, not persisted. - */ - if (inode->logged_trans == trans->transid && - !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) - return false; - - return true; -} - -struct btrfs_dir_list { - u64 ino; - struct list_head list; -}; - -/* - * Log the inodes of the new dentries of a directory. See log_dir_items() for - * details about the why it is needed. - * This is a recursive operation - if an existing dentry corresponds to a - * directory, that directory's new entries are logged too (same behaviour as - * ext3/4, xfs, f2fs, reiserfs, nilfs2). Note that when logging the inodes - * the dentries point to we do not lock their i_mutex, otherwise lockdep - * complains about the following circular lock dependency / possible deadlock: - * - * CPU0 CPU1 - * ---- ---- - * lock(&type->i_mutex_dir_key#3/2); - * lock(sb_internal#2); - * lock(&type->i_mutex_dir_key#3/2); - * lock(&sb->s_type->i_mutex_key#14); - * - * Where sb_internal is the lock (a counter that works as a lock) acquired by - * sb_start_intwrite() in btrfs_start_transaction(). - * Not locking i_mutex of the inodes is still safe because: - * - * 1) For regular files we log with a mode of LOG_INODE_EXISTS. It's possible - * that while logging the inode new references (names) are added or removed - * from the inode, leaving the logged inode item with a link count that does - * not match the number of logged inode reference items. This is fine because - * at log replay time we compute the real number of links and correct the - * link count in the inode item (see replay_one_buffer() and - * link_to_fixup_dir()); - * - * 2) For directories we log with a mode of LOG_INODE_ALL. It's possible that - * while logging the inode's items new index items (key type - * BTRFS_DIR_INDEX_KEY) are added to fs/subvol tree and the logged inode item - * has a size that doesn't match the sum of the lengths of all the logged - * names - this is ok, not a problem, because at log replay time we set the - * directory's i_size to the correct value (see replay_one_name() and - * do_overwrite_item()). - */ -static int log_new_dir_dentries(struct btrfs_trans_handle *trans, - struct btrfs_root *root, - struct btrfs_inode *start_inode, - struct btrfs_log_ctx *ctx) -{ - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_path *path; - LIST_HEAD(dir_list); - struct btrfs_dir_list *dir_elem; - int ret = 0; - - /* - * If we are logging a new name, as part of a link or rename operation, - * don't bother logging new dentries, as we just want to log the names - * of an inode and that any new parents exist. - */ - if (ctx->logging_new_name) - return 0; - - path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; - - dir_elem = kmalloc(sizeof(*dir_elem), GFP_NOFS); - if (!dir_elem) { - btrfs_free_path(path); - return -ENOMEM; - } - dir_elem->ino = btrfs_ino(start_inode); - list_add_tail(&dir_elem->list, &dir_list); - - while (!list_empty(&dir_list)) { - struct extent_buffer *leaf; - struct btrfs_key min_key; - int nritems; - int i; - - dir_elem = list_first_entry(&dir_list, struct btrfs_dir_list, - list); - if (ret) - goto next_dir_inode; - - min_key.objectid = dir_elem->ino; - min_key.type = BTRFS_DIR_INDEX_KEY; - min_key.offset = 0; -again: - btrfs_release_path(path); - ret = btrfs_search_forward(root, &min_key, path, trans->transid); - if (ret < 0) { - goto next_dir_inode; - } else if (ret > 0) { - ret = 0; - goto next_dir_inode; - } - - leaf = path->nodes[0]; - nritems = btrfs_header_nritems(leaf); - for (i = path->slots[0]; i < nritems; i++) { - struct btrfs_dir_item *di; - struct btrfs_key di_key; - struct inode *di_inode; - struct btrfs_dir_list *new_dir_elem; - int log_mode = LOG_INODE_EXISTS; - int type; - - btrfs_item_key_to_cpu(leaf, &min_key, i); - if (min_key.objectid != dir_elem->ino || - min_key.type != BTRFS_DIR_INDEX_KEY) - goto next_dir_inode; - - di = btrfs_item_ptr(leaf, i, struct btrfs_dir_item); - type = btrfs_dir_type(leaf, di); - if (btrfs_dir_transid(leaf, di) < trans->transid) - continue; - btrfs_dir_item_key_to_cpu(leaf, di, &di_key); - if (di_key.type == BTRFS_ROOT_ITEM_KEY) - continue; - - btrfs_release_path(path); - di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root); - if (IS_ERR(di_inode)) { - ret = PTR_ERR(di_inode); - goto next_dir_inode; - } + if (ret) + free_conflicting_inodes(ctx); + else + ret = log_conflicting_inodes(trans, inode->root, ctx); - if (!need_log_inode(trans, BTRFS_I(di_inode))) { - btrfs_add_delayed_iput(di_inode); - break; - } + if (full_dir_logging && !ctx->logging_new_delayed_dentries) { + if (!ret) + ret = log_new_delayed_dentries(trans, inode, + &delayed_ins_list, ctx); - ctx->log_new_dentries = false; - if (type == BTRFS_FT_DIR) - log_mode = LOG_INODE_ALL; - ret = btrfs_log_inode(trans, BTRFS_I(di_inode), - log_mode, ctx); - btrfs_add_delayed_iput(di_inode); - if (ret) - goto next_dir_inode; - if (ctx->log_new_dentries) { - new_dir_elem = kmalloc(sizeof(*new_dir_elem), - GFP_NOFS); - if (!new_dir_elem) { - ret = -ENOMEM; - goto next_dir_inode; - } - new_dir_elem->ino = di_key.objectid; - list_add_tail(&new_dir_elem->list, &dir_list); - } - break; - } - if (min_key.offset < (u64)-1) { - min_key.offset++; - goto again; - } -next_dir_inode: - list_del(&dir_elem->list); - kfree(dir_elem); + btrfs_log_put_delayed_items(inode, &delayed_ins_list, + &delayed_del_list); } - btrfs_free_path(path); return ret; } @@ -6346,7 +6740,7 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, ret = btrfs_log_inode(trans, BTRFS_I(dir_inode), LOG_INODE_ALL, ctx); if (!ret && ctx->log_new_dentries) - ret = log_new_dir_dentries(trans, root, + ret = log_new_dir_dentries(trans, BTRFS_I(dir_inode), ctx); btrfs_add_delayed_iput(dir_inode); if (ret) @@ -6661,7 +7055,7 @@ static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, goto end_trans; if (log_dentries) - ret = log_new_dir_dentries(trans, root, inode, ctx); + ret = log_new_dir_dentries(trans, inode, ctx); else ret = 0; end_trans: @@ -7088,6 +7482,7 @@ void btrfs_log_new_name(struct btrfs_trans_handle *trans, * inconsistent state after a rename operation. */ btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx); + ASSERT(list_empty(&ctx.conflict_inodes)); out: /* * If an error happened mark the log for a full commit because it's not diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h index 57ab5f3b8dc77e5415d63ef006a588112d5630f5..aed1e05e9879e6cae9d0d1fb0d8cfaea4118cf5f 100644 --- a/fs/btrfs/tree-log.h +++ b/fs/btrfs/tree-log.h @@ -20,6 +20,7 @@ struct btrfs_log_ctx { int log_transid; bool log_new_dentries; bool logging_new_name; + bool logging_new_delayed_dentries; /* Indicate if the inode being logged was logged before. */ bool logged_before; /* Tracks the last logged dir item/index key offset. */ @@ -28,6 +29,9 @@ struct btrfs_log_ctx { struct list_head list; /* Only used for fast fsyncs. */ struct list_head ordered_extents; + struct list_head conflict_inodes; + int num_conflict_inodes; + bool logging_conflict_inodes; }; static inline void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, @@ -37,10 +41,14 @@ static inline void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, ctx->log_transid = 0; ctx->log_new_dentries = false; ctx->logging_new_name = false; + ctx->logging_new_delayed_dentries = false; ctx->logged_before = false; ctx->inode = inode; INIT_LIST_HEAD(&ctx->list); INIT_LIST_HEAD(&ctx->ordered_extents); + INIT_LIST_HEAD(&ctx->conflict_inodes); + ctx->num_conflict_inodes = 0; + ctx->logging_conflict_inodes = false; } static inline void btrfs_release_log_ctx_extents(struct btrfs_log_ctx *ctx) diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 90eb5c2830a9273c9e3c3c717b5243b79224343f..ee00e33c309e0f821fe4bd0602b3c867c4504ccf 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -659,8 +659,7 @@ rollback: * * Returns the size on success or a negative error code on failure. */ -static int btrfs_get_verity_descriptor(struct inode *inode, void *buf, - size_t buf_size) +int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) { u64 true_size; int ret = 0; diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 272901514b0c14dc7074e31907df1f46caf3705e..94ba46d5792053553936b55518a002237e5c5fef 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -34,6 +34,8 @@ #include "discard.h" #include "zoned.h" +static struct bio_set btrfs_bioset; + #define BTRFS_BLOCK_GROUP_STRIPE_MASK (BTRFS_BLOCK_GROUP_RAID0 | \ BTRFS_BLOCK_GROUP_RAID10 | \ BTRFS_BLOCK_GROUP_RAID56_MASK) @@ -247,10 +249,10 @@ static int init_first_rw_device(struct btrfs_trans_handle *trans); static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info); static void btrfs_dev_stat_print_on_load(struct btrfs_device *device); static int __btrfs_map_block(struct btrfs_fs_info *fs_info, - enum btrfs_map_op op, - u64 logical, u64 *length, + enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_io_context **bioc_ret, - int mirror_num, int need_raid_map); + struct btrfs_io_stripe *smap, + int *mirror_num_ret, int need_raid_map); /* * Device locking @@ -2017,7 +2019,7 @@ void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct page *page; int ret; - disk_super = btrfs_read_dev_one_super(bdev, copy_num); + disk_super = btrfs_read_dev_one_super(bdev, copy_num, false); if (IS_ERR(disk_super)) continue; @@ -2345,8 +2347,11 @@ int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info, ret = btrfs_get_bdev_and_sb(path, FMODE_READ, fs_info->bdev_holder, 0, &bdev, &disk_super); - if (ret) + if (ret) { + btrfs_put_dev_args_from_path(args); return ret; + } + args->devid = btrfs_stack_device_id(&disk_super->dev_item); memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE); if (btrfs_fs_incompat(fs_info, METADATA_UUID)) @@ -5264,6 +5269,9 @@ static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl, ctl->stripe_size); } + /* Stripe size should not go beyond 1G. */ + ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G); + /* Align to BTRFS_STRIPE_LEN */ ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN); ctl->chunk_size = ctl->stripe_size * data_stripes; @@ -5589,7 +5597,7 @@ int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans, if (ret) goto out; - bg->chunk_item_inserted = 1; + set_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, &bg->runtime_flags); if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size); @@ -5890,7 +5898,6 @@ static struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_ sizeof(u64) * (total_stripes), GFP_NOFS|__GFP_NOFAIL); - atomic_set(&bioc->error, 0); refcount_set(&bioc->refs, 1); bioc->fs_info = fs_info; @@ -6086,7 +6093,7 @@ static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info, int ret = 0; ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, - logical, &length, &bioc, 0, 0); + logical, &length, &bioc, NULL, NULL, 0); if (ret) { ASSERT(bioc == NULL); return ret; @@ -6147,9 +6154,7 @@ static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical) cache = btrfs_lookup_block_group(fs_info, logical); - spin_lock(&cache->lock); - ret = cache->to_copy; - spin_unlock(&cache->lock); + ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); btrfs_put_block_group(cache); return ret; @@ -6345,11 +6350,19 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, return 0; } +static void set_io_stripe(struct btrfs_io_stripe *dst, const struct map_lookup *map, + u32 stripe_index, u64 stripe_offset, u64 stripe_nr) +{ + dst->dev = map->stripes[stripe_index].dev; + dst->physical = map->stripes[stripe_index].physical + + stripe_offset + stripe_nr * map->stripe_len; +} + static int __btrfs_map_block(struct btrfs_fs_info *fs_info, - enum btrfs_map_op op, - u64 logical, u64 *length, + enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_io_context **bioc_ret, - int mirror_num, int need_raid_map) + struct btrfs_io_stripe *smap, + int *mirror_num_ret, int need_raid_map) { struct extent_map *em; struct map_lookup *map; @@ -6360,6 +6373,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, int data_stripes; int i; int ret = 0; + int mirror_num = (mirror_num_ret ? *mirror_num_ret : 0); int num_stripes; int max_errors = 0; int tgtdev_indexes = 0; @@ -6520,6 +6534,29 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, tgtdev_indexes = num_stripes; } + /* + * If this I/O maps to a single device, try to return the device and + * physical block information on the stack instead of allocating an + * I/O context structure. + */ + if (smap && num_alloc_stripes == 1 && + !((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && mirror_num > 1) && + (!need_full_stripe(op) || !dev_replace_is_ongoing || + !dev_replace->tgtdev)) { + if (patch_the_first_stripe_for_dev_replace) { + smap->dev = dev_replace->tgtdev; + smap->physical = physical_to_patch_in_first_stripe; + *mirror_num_ret = map->num_stripes + 1; + } else { + set_io_stripe(smap, map, stripe_index, stripe_offset, + stripe_nr); + *mirror_num_ret = mirror_num; + } + *bioc_ret = NULL; + ret = 0; + goto out; + } + bioc = alloc_btrfs_io_context(fs_info, num_alloc_stripes, tgtdev_indexes); if (!bioc) { ret = -ENOMEM; @@ -6527,9 +6564,8 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, } for (i = 0; i < num_stripes; i++) { - bioc->stripes[i].physical = map->stripes[stripe_index].physical + - stripe_offset + stripe_nr * map->stripe_len; - bioc->stripes[i].dev = map->stripes[stripe_index].dev; + set_io_stripe(&bioc->stripes[i], map, stripe_index, stripe_offset, + stripe_nr); stripe_index++; } @@ -6597,7 +6633,7 @@ int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, struct btrfs_io_context **bioc_ret, int mirror_num) { return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, - mirror_num, 0); + NULL, &mirror_num, 0); } /* For Scrub/replace */ @@ -6605,14 +6641,77 @@ int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_io_context **bioc_ret) { - return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, 0, 1); + return __btrfs_map_block(fs_info, op, logical, length, bioc_ret, + NULL, NULL, 1); +} + +/* + * Initialize a btrfs_bio structure. This skips the embedded bio itself as it + * is already initialized by the block layer. + */ +static inline void btrfs_bio_init(struct btrfs_bio *bbio, + btrfs_bio_end_io_t end_io, void *private) +{ + memset(bbio, 0, offsetof(struct btrfs_bio, bio)); + bbio->end_io = end_io; + bbio->private = private; +} + +/* + * Allocate a btrfs_bio structure. The btrfs_bio is the main I/O container for + * btrfs, and is used for all I/O submitted through btrfs_submit_bio. + * + * Just like the underlying bio_alloc_bioset it will not fail as it is backed by + * a mempool. + */ +struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + btrfs_bio_end_io_t end_io, void *private) +{ + struct bio *bio; + + bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); + btrfs_bio_init(btrfs_bio(bio), end_io, private); + return bio; +} + +struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, + btrfs_bio_end_io_t end_io, void *private) +{ + struct bio *bio; + struct btrfs_bio *bbio; + + ASSERT(offset <= UINT_MAX && size <= UINT_MAX); + + bio = bio_alloc_clone(orig->bi_bdev, orig, GFP_NOFS, &btrfs_bioset); + bbio = btrfs_bio(bio); + btrfs_bio_init(bbio, end_io, private); + + bio_trim(bio, offset >> 9, size >> 9); + bbio->iter = bio->bi_iter; + return bio; +} + +static void btrfs_log_dev_io_error(struct bio *bio, struct btrfs_device *dev) +{ + if (!dev || !dev->bdev) + return; + if (bio->bi_status != BLK_STS_IOERR && bio->bi_status != BLK_STS_TARGET) + return; + + if (btrfs_op(bio) == BTRFS_MAP_WRITE) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); + if (!(bio->bi_opf & REQ_RAHEAD)) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); + if (bio->bi_opf & REQ_PREFLUSH) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_FLUSH_ERRS); } -static struct workqueue_struct *btrfs_end_io_wq(struct btrfs_io_context *bioc) +static struct workqueue_struct *btrfs_end_io_wq(struct btrfs_fs_info *fs_info, + struct bio *bio) { - if (bioc->orig_bio->bi_opf & REQ_META) - return bioc->fs_info->endio_meta_workers; - return bioc->fs_info->endio_workers; + if (bio->bi_opf & REQ_META) + return fs_info->endio_meta_workers; + return fs_info->endio_workers; } static void btrfs_end_bio_work(struct work_struct *work) @@ -6620,103 +6719,101 @@ static void btrfs_end_bio_work(struct work_struct *work) struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); - bio_endio(&bbio->bio); + bbio->end_io(bbio); } -static void btrfs_end_bioc(struct btrfs_io_context *bioc, bool async) +static void btrfs_simple_end_io(struct bio *bio) { - struct bio *orig_bio = bioc->orig_bio; - struct btrfs_bio *bbio = btrfs_bio(orig_bio); + struct btrfs_fs_info *fs_info = bio->bi_private; + struct btrfs_bio *bbio = btrfs_bio(bio); - bbio->mirror_num = bioc->mirror_num; - orig_bio->bi_private = bioc->private; - orig_bio->bi_end_io = bioc->end_io; + btrfs_bio_counter_dec(fs_info); - /* - * Only send an error to the higher layers if it is beyond the tolerance - * threshold. - */ - if (atomic_read(&bioc->error) > bioc->max_errors) - orig_bio->bi_status = BLK_STS_IOERR; - else - orig_bio->bi_status = BLK_STS_OK; + if (bio->bi_status) + btrfs_log_dev_io_error(bio, bbio->device); - if (btrfs_op(orig_bio) == BTRFS_MAP_READ && async) { + if (bio_op(bio) == REQ_OP_READ) { INIT_WORK(&bbio->end_io_work, btrfs_end_bio_work); - queue_work(btrfs_end_io_wq(bioc), &bbio->end_io_work); + queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); } else { - bio_endio(orig_bio); + bbio->end_io(bbio); } +} + +static void btrfs_raid56_end_io(struct bio *bio) +{ + struct btrfs_io_context *bioc = bio->bi_private; + struct btrfs_bio *bbio = btrfs_bio(bio); + + btrfs_bio_counter_dec(bioc->fs_info); + bbio->mirror_num = bioc->mirror_num; + bbio->end_io(bbio); btrfs_put_bioc(bioc); } -static void btrfs_end_bio(struct bio *bio) +static void btrfs_orig_write_end_io(struct bio *bio) { struct btrfs_io_stripe *stripe = bio->bi_private; struct btrfs_io_context *bioc = stripe->bioc; + struct btrfs_bio *bbio = btrfs_bio(bio); + + btrfs_bio_counter_dec(bioc->fs_info); if (bio->bi_status) { atomic_inc(&bioc->error); - if (bio->bi_status == BLK_STS_IOERR || - bio->bi_status == BLK_STS_TARGET) { - if (btrfs_op(bio) == BTRFS_MAP_WRITE) - btrfs_dev_stat_inc_and_print(stripe->dev, - BTRFS_DEV_STAT_WRITE_ERRS); - else if (!(bio->bi_opf & REQ_RAHEAD)) - btrfs_dev_stat_inc_and_print(stripe->dev, - BTRFS_DEV_STAT_READ_ERRS); - if (bio->bi_opf & REQ_PREFLUSH) - btrfs_dev_stat_inc_and_print(stripe->dev, - BTRFS_DEV_STAT_FLUSH_ERRS); - } + btrfs_log_dev_io_error(bio, stripe->dev); } - if (bio != bioc->orig_bio) - bio_put(bio); + /* + * Only send an error to the higher layers if it is beyond the tolerance + * threshold. + */ + if (atomic_read(&bioc->error) > bioc->max_errors) + bio->bi_status = BLK_STS_IOERR; + else + bio->bi_status = BLK_STS_OK; - btrfs_bio_counter_dec(bioc->fs_info); - if (atomic_dec_and_test(&bioc->stripes_pending)) - btrfs_end_bioc(bioc, true); + bbio->end_io(bbio); + btrfs_put_bioc(bioc); } -static void submit_stripe_bio(struct btrfs_io_context *bioc, - struct bio *orig_bio, int dev_nr, bool clone) +static void btrfs_clone_write_end_io(struct bio *bio) { - struct btrfs_fs_info *fs_info = bioc->fs_info; - struct btrfs_device *dev = bioc->stripes[dev_nr].dev; - u64 physical = bioc->stripes[dev_nr].physical; - struct bio *bio; + struct btrfs_io_stripe *stripe = bio->bi_private; + + if (bio->bi_status) { + atomic_inc(&stripe->bioc->error); + btrfs_log_dev_io_error(bio, stripe->dev); + } + + /* Pass on control to the original bio this one was cloned from */ + bio_endio(stripe->bioc->orig_bio); + bio_put(bio); +} +static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) +{ if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || - (btrfs_op(orig_bio) == BTRFS_MAP_WRITE && + (btrfs_op(bio) == BTRFS_MAP_WRITE && !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) { - atomic_inc(&bioc->error); - if (atomic_dec_and_test(&bioc->stripes_pending)) - btrfs_end_bioc(bioc, false); + bio_io_error(bio); return; } - if (clone) { - bio = bio_alloc_clone(dev->bdev, orig_bio, GFP_NOFS, &fs_bio_set); - } else { - bio = orig_bio; - bio_set_dev(bio, dev->bdev); - btrfs_bio(bio)->device = dev; - } + bio_set_dev(bio, dev->bdev); - bioc->stripes[dev_nr].bioc = bioc; - bio->bi_private = &bioc->stripes[dev_nr]; - bio->bi_end_io = btrfs_end_bio; - bio->bi_iter.bi_sector = physical >> 9; /* * For zone append writing, bi_sector must point the beginning of the * zone */ if (bio_op(bio) == REQ_OP_ZONE_APPEND) { + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + if (btrfs_dev_is_sequential(dev, physical)) { - u64 zone_start = round_down(physical, fs_info->zone_size); + u64 zone_start = round_down(physical, + dev->fs_info->zone_size); bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; } else { @@ -6724,50 +6821,53 @@ static void submit_stripe_bio(struct btrfs_io_context *bioc, bio->bi_opf |= REQ_OP_WRITE; } } - btrfs_debug_in_rcu(fs_info, + btrfs_debug_in_rcu(dev->fs_info, "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", __func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector, (unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name), dev->devid, bio->bi_iter.bi_size); - btrfs_bio_counter_inc_noblocked(fs_info); - btrfsic_check_bio(bio); submit_bio(bio); } +static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) +{ + struct bio *orig_bio = bioc->orig_bio, *bio; + + ASSERT(bio_op(orig_bio) != REQ_OP_READ); + + /* Reuse the bio embedded into the btrfs_bio for the last mirror */ + if (dev_nr == bioc->num_stripes - 1) { + bio = orig_bio; + bio->bi_end_io = btrfs_orig_write_end_io; + } else { + bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &fs_bio_set); + bio_inc_remaining(orig_bio); + bio->bi_end_io = btrfs_clone_write_end_io; + } + + bio->bi_private = &bioc->stripes[dev_nr]; + bio->bi_iter.bi_sector = bioc->stripes[dev_nr].physical >> SECTOR_SHIFT; + bioc->stripes[dev_nr].bioc = bioc; + btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio); +} + void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror_num) { u64 logical = bio->bi_iter.bi_sector << 9; u64 length = bio->bi_iter.bi_size; u64 map_length = length; - int ret; - int dev_nr; - int total_devs; struct btrfs_io_context *bioc = NULL; + struct btrfs_io_stripe smap; + int ret; btrfs_bio_counter_inc_blocked(fs_info); - ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical, - &map_length, &bioc, mirror_num, 1); + ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, + &bioc, &smap, &mirror_num, 1); if (ret) { btrfs_bio_counter_dec(fs_info); - bio->bi_status = errno_to_blk_status(ret); - bio_endio(bio); - return; - } - - total_devs = bioc->num_stripes; - bioc->orig_bio = bio; - bioc->private = bio->bi_private; - bioc->end_io = bio->bi_end_io; - atomic_set(&bioc->stripes_pending, total_devs); - - if ((bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) && - ((btrfs_op(bio) == BTRFS_MAP_WRITE) || (mirror_num > 1))) { - if (btrfs_op(bio) == BTRFS_MAP_WRITE) - raid56_parity_write(bio, bioc); - else - raid56_parity_recover(bio, bioc, mirror_num, true); + btrfs_bio_end_io(btrfs_bio(bio), errno_to_blk_status(ret)); return; } @@ -6778,12 +6878,31 @@ void btrfs_submit_bio(struct btrfs_fs_info *fs_info, struct bio *bio, int mirror BUG(); } - for (dev_nr = 0; dev_nr < total_devs; dev_nr++) { - const bool should_clone = (dev_nr < total_devs - 1); + if (!bioc) { + /* Single mirror read/write fast path */ + btrfs_bio(bio)->mirror_num = mirror_num; + btrfs_bio(bio)->device = smap.dev; + bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; + bio->bi_private = fs_info; + bio->bi_end_io = btrfs_simple_end_io; + btrfs_submit_dev_bio(smap.dev, bio); + } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + /* Parity RAID write or read recovery */ + bio->bi_private = bioc; + bio->bi_end_io = btrfs_raid56_end_io; + if (bio_op(bio) == REQ_OP_READ) + raid56_parity_recover(bio, bioc, mirror_num); + else + raid56_parity_write(bio, bioc); + } else { + /* Write to multiple mirrors */ + int total_devs = bioc->num_stripes; + int dev_nr; - submit_stripe_bio(bioc, bio, dev_nr, should_clone); + bioc->orig_bio = bio; + for (dev_nr = 0; dev_nr < total_devs; dev_nr++) + btrfs_submit_mirrored_bio(bioc, dev_nr); } - btrfs_bio_counter_dec(fs_info); } static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args, @@ -8238,7 +8357,7 @@ static int relocating_repair_kthread(void *data) if (!cache) goto out; - if (!cache->relocating_repair) + if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) goto out; ret = btrfs_may_alloc_data_chunk(fs_info, target); @@ -8275,17 +8394,27 @@ bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical) if (!cache) return true; - spin_lock(&cache->lock); - if (cache->relocating_repair) { - spin_unlock(&cache->lock); + if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) { btrfs_put_block_group(cache); return true; } - cache->relocating_repair = 1; - spin_unlock(&cache->lock); kthread_run(relocating_repair_kthread, cache, "btrfs-relocating-repair"); return true; } + +int __init btrfs_bioset_init(void) +{ + if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), + BIOSET_NEED_BVECS)) + return -ENOMEM; + return 0; +} + +void __cold btrfs_bioset_exit(void) +{ + bioset_exit(&btrfs_bioset); +} diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 5639961b3626f76772eee1c5574f362a21c5e29d..599b9d5af349f10fabb30b1da8b35f4b4f1bf4e5 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -180,6 +180,31 @@ struct btrfs_device { u64 scrub_speed_max; }; +/* + * Block group or device which contains an active swapfile. Used for preventing + * unsafe operations while a swapfile is active. + * + * These are sorted on (ptr, inode) (note that a block group or device can + * contain more than one swapfile). We compare the pointer values because we + * don't actually care what the object is, we just need a quick check whether + * the object exists in the rbtree. + */ +struct btrfs_swapfile_pin { + struct rb_node node; + void *ptr; + struct inode *inode; + /* + * If true, ptr points to a struct btrfs_block_group. Otherwise, ptr + * points to a struct btrfs_device. + */ + bool is_block_group; + /* + * Only used when 'is_block_group' is true and it is the number of + * extents used by a swapfile for this block group ('ptr' field). + */ + int bg_extent_count; +}; + /* * If we read those variants at the context of their own lock, we needn't * use the following helpers, reading them directly is safe. @@ -361,6 +386,8 @@ struct btrfs_fs_devices { */ #define BTRFS_MAX_BIO_SECTORS (256) +typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); + /* * Additional info to pass along bio. * @@ -378,6 +405,10 @@ struct btrfs_bio { u8 csum_inline[BTRFS_BIO_INLINE_CSUM_SIZE]; struct bvec_iter iter; + /* End I/O information supplied to btrfs_bio_alloc */ + btrfs_bio_end_io_t end_io; + void *private; + /* For read end I/O handling */ struct work_struct end_io_work; @@ -393,6 +424,20 @@ static inline struct btrfs_bio *btrfs_bio(struct bio *bio) return container_of(bio, struct btrfs_bio, bio); } +int __init btrfs_bioset_init(void); +void __cold btrfs_bioset_exit(void); + +struct bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + btrfs_bio_end_io_t end_io, void *private); +struct bio *btrfs_bio_clone_partial(struct bio *orig, u64 offset, u64 size, + btrfs_bio_end_io_t end_io, void *private); + +static inline void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) +{ + bbio->bio.bi_status = status; + bbio->end_io(bbio); +} + static inline void btrfs_bio_free_csum(struct btrfs_bio *bbio) { if (bbio->csum != bbio->csum_inline) { @@ -451,12 +496,9 @@ struct btrfs_discard_stripe { */ struct btrfs_io_context { refcount_t refs; - atomic_t stripes_pending; struct btrfs_fs_info *fs_info; u64 map_type; /* get from map_lookup->type */ - bio_end_io_t *end_io; struct bio *orig_bio; - void *private; atomic_t error; int max_errors; int num_stripes; @@ -714,4 +756,6 @@ const char *btrfs_bg_type_to_raid_name(u64 flags); int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info); bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical); +bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr); + #endif diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c index 7421abcf325a54ececa7f9c1ad7105921ce2f59d..5bb8d8c86311902ea8065862e66b35ec652f6c85 100644 --- a/fs/btrfs/xattr.c +++ b/fs/btrfs/xattr.c @@ -371,6 +371,9 @@ static int btrfs_xattr_handler_set(const struct xattr_handler *handler, const char *name, const void *buffer, size_t size, int flags) { + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + name = xattr_full_name(handler, name); return btrfs_setxattr_trans(inode, name, buffer, size, flags); } diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index b150b07ba1a7663c9b04c85307324d6819bd1b73..e2d073b08a7d28cbff66b78cac3133bc11a51bf2 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -421,10 +421,19 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) * since btrfs adds the pages one by one to a bio, and btrfs cannot * increase the metadata reservation even if it increases the number of * extents, it is safe to stick with the limit. + * + * With the zoned emulation, we can have non-zoned device on the zoned + * mode. In this case, we don't have a valid max zone append size. So, + * use max_segments * PAGE_SIZE as the pseudo max_zone_append_size. */ - zone_info->max_zone_append_size = - min_t(u64, (u64)bdev_max_zone_append_sectors(bdev) << SECTOR_SHIFT, - (u64)bdev_max_segments(bdev) << PAGE_SHIFT); + if (bdev_is_zoned(bdev)) { + zone_info->max_zone_append_size = min_t(u64, + (u64)bdev_max_zone_append_sectors(bdev) << SECTOR_SHIFT, + (u64)bdev_max_segments(bdev) << PAGE_SHIFT); + } else { + zone_info->max_zone_append_size = + (u64)bdev_max_segments(bdev) << PAGE_SHIFT; + } if (!IS_ALIGNED(nr_sectors, zone_sectors)) zone_info->nr_zones++; @@ -643,80 +652,55 @@ int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, return 0; } +static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *device; + + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + if (device->bdev && + bdev_zoned_model(device->bdev) == BLK_ZONED_HM) { + btrfs_err(fs_info, + "zoned: mode not enabled but zoned device found: %pg", + device->bdev); + return -EINVAL; + } + } + + return 0; +} + int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) { - struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; struct btrfs_device *device; - u64 zoned_devices = 0; - u64 nr_devices = 0; u64 zone_size = 0; u64 max_zone_append_size = 0; - const bool incompat_zoned = btrfs_fs_incompat(fs_info, ZONED); - int ret = 0; + int ret; - /* Count zoned devices */ - list_for_each_entry(device, &fs_devices->devices, dev_list) { - enum blk_zoned_model model; + /* + * Host-Managed devices can't be used without the ZONED flag. With the + * ZONED all devices can be used, using zone emulation if required. + */ + if (!btrfs_fs_incompat(fs_info, ZONED)) + return btrfs_check_for_zoned_device(fs_info); + + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + struct btrfs_zoned_device_info *zone_info = device->zone_info; if (!device->bdev) continue; - model = bdev_zoned_model(device->bdev); - /* - * A Host-Managed zoned device must be used as a zoned device. - * A Host-Aware zoned device and a non-zoned devices can be - * treated as a zoned device, if ZONED flag is enabled in the - * superblock. - */ - if (model == BLK_ZONED_HM || - (model == BLK_ZONED_HA && incompat_zoned) || - (model == BLK_ZONED_NONE && incompat_zoned)) { - struct btrfs_zoned_device_info *zone_info; - - zone_info = device->zone_info; - zoned_devices++; - if (!zone_size) { - zone_size = zone_info->zone_size; - } else if (zone_info->zone_size != zone_size) { - btrfs_err(fs_info, + if (!zone_size) { + zone_size = zone_info->zone_size; + } else if (zone_info->zone_size != zone_size) { + btrfs_err(fs_info, "zoned: unequal block device zone sizes: have %llu found %llu", - device->zone_info->zone_size, - zone_size); - ret = -EINVAL; - goto out; - } - if (!max_zone_append_size || - (zone_info->max_zone_append_size && - zone_info->max_zone_append_size < max_zone_append_size)) - max_zone_append_size = - zone_info->max_zone_append_size; + zone_info->zone_size, zone_size); + return -EINVAL; } - nr_devices++; - } - - if (!zoned_devices && !incompat_zoned) - goto out; - - if (!zoned_devices && incompat_zoned) { - /* No zoned block device found on ZONED filesystem */ - btrfs_err(fs_info, - "zoned: no zoned devices found on a zoned filesystem"); - ret = -EINVAL; - goto out; - } - - if (zoned_devices && !incompat_zoned) { - btrfs_err(fs_info, - "zoned: mode not enabled but zoned device found"); - ret = -EINVAL; - goto out; - } - - if (zoned_devices != nr_devices) { - btrfs_err(fs_info, - "zoned: cannot mix zoned and regular devices"); - ret = -EINVAL; - goto out; + if (!max_zone_append_size || + (zone_info->max_zone_append_size && + zone_info->max_zone_append_size < max_zone_append_size)) + max_zone_append_size = zone_info->max_zone_append_size; } /* @@ -728,14 +712,12 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) btrfs_err(fs_info, "zoned: zone size %llu not aligned to stripe %u", zone_size, BTRFS_STRIPE_LEN); - ret = -EINVAL; - goto out; + return -EINVAL; } if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { btrfs_err(fs_info, "zoned: mixed block groups not supported"); - ret = -EINVAL; - goto out; + return -EINVAL; } fs_info->zone_size = zone_size; @@ -751,11 +733,10 @@ int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) */ ret = btrfs_check_mountopts_zoned(fs_info); if (ret) - goto out; + return ret; btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size); -out: - return ret; + return 0; } int btrfs_check_mountopts_zoned(struct btrfs_fs_info *info) @@ -1178,7 +1159,7 @@ int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) * offset. */ static int calculate_alloc_pointer(struct btrfs_block_group *cache, - u64 *offset_ret) + u64 *offset_ret, bool new) { struct btrfs_fs_info *fs_info = cache->fs_info; struct btrfs_root *root; @@ -1188,6 +1169,21 @@ static int calculate_alloc_pointer(struct btrfs_block_group *cache, int ret; u64 length; + /* + * Avoid tree lookups for a new block group, there's no use for it. + * It must always be 0. + * + * Also, we have a lock chain of extent buffer lock -> chunk mutex. + * For new a block group, this function is called from + * btrfs_make_block_group() which is already taking the chunk mutex. + * Thus, we cannot call calculate_alloc_pointer() which takes extent + * buffer locks to avoid deadlock. + */ + if (new) { + *offset_ret = 0; + return 0; + } + path = btrfs_alloc_path(); if (!path) return -ENOMEM; @@ -1323,6 +1319,13 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) else num_conventional++; + /* + * Consider a zone as active if we can allow any number of + * active zones. + */ + if (!device->zone_info->max_active_zones) + __set_bit(i, active); + if (!is_sequential) { alloc_offsets[i] = WP_CONVENTIONAL; continue; @@ -1389,45 +1392,23 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) __set_bit(i, active); break; } - - /* - * Consider a zone as active if we can allow any number of - * active zones. - */ - if (!device->zone_info->max_active_zones) - __set_bit(i, active); } if (num_sequential > 0) cache->seq_zone = true; if (num_conventional > 0) { - /* - * Avoid calling calculate_alloc_pointer() for new BG. It - * is no use for new BG. It must be always 0. - * - * Also, we have a lock chain of extent buffer lock -> - * chunk mutex. For new BG, this function is called from - * btrfs_make_block_group() which is already taking the - * chunk mutex. Thus, we cannot call - * calculate_alloc_pointer() which takes extent buffer - * locks to avoid deadlock. - */ - /* Zone capacity is always zone size in emulation */ cache->zone_capacity = cache->length; - if (new) { - cache->alloc_offset = 0; - goto out; - } - ret = calculate_alloc_pointer(cache, &last_alloc); - if (ret || map->num_stripes == num_conventional) { - if (!ret) - cache->alloc_offset = last_alloc; - else - btrfs_err(fs_info, + ret = calculate_alloc_pointer(cache, &last_alloc, new); + if (ret) { + btrfs_err(fs_info, "zoned: failed to determine allocation offset of bg %llu", - cache->start); + cache->start); + goto out; + } else if (map->num_stripes == num_conventional) { + cache->alloc_offset = last_alloc; + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); goto out; } } @@ -1443,7 +1424,8 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) } cache->alloc_offset = alloc_offsets[0]; cache->zone_capacity = caps[0]; - cache->zone_is_active = test_bit(0, active); + if (test_bit(0, active)) + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); break; case BTRFS_BLOCK_GROUP_DUP: if (map->type & BTRFS_BLOCK_GROUP_DATA) { @@ -1477,7 +1459,9 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) goto out; } } else { - cache->zone_is_active = test_bit(0, active); + if (test_bit(0, active)) + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &cache->runtime_flags); } cache->alloc_offset = alloc_offsets[0]; cache->zone_capacity = min(caps[0], caps[1]); @@ -1495,13 +1479,6 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) goto out; } - if (cache->zone_is_active) { - btrfs_get_block_group(cache); - spin_lock(&fs_info->zone_active_bgs_lock); - list_add_tail(&cache->active_bg_list, &fs_info->zone_active_bgs); - spin_unlock(&fs_info->zone_active_bgs_lock); - } - out: if (cache->alloc_offset > fs_info->zone_size) { btrfs_err(fs_info, @@ -1526,10 +1503,16 @@ out: ret = -EIO; } - if (!ret) + if (!ret) { cache->meta_write_pointer = cache->alloc_offset + cache->start; - - if (ret) { + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags)) { + btrfs_get_block_group(cache); + spin_lock(&fs_info->zone_active_bgs_lock); + list_add_tail(&cache->active_bg_list, + &fs_info->zone_active_bgs); + spin_unlock(&fs_info->zone_active_bgs_lock); + } + } else { kfree(cache->physical_map); cache->physical_map = NULL; } @@ -1555,7 +1538,6 @@ void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) free = cache->zone_capacity - cache->alloc_offset; /* We only need ->free_space in ALLOC_SEQ block groups */ - cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; cache->free_space_ctl->free_space = free; cache->zone_unusable = unusable; @@ -1863,7 +1845,7 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) spin_lock(&space_info->lock); spin_lock(&block_group->lock); - if (block_group->zone_is_active) { + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { ret = true; goto out_unlock; } @@ -1889,7 +1871,7 @@ bool btrfs_zone_activate(struct btrfs_block_group *block_group) } /* Successfully activated all the zones */ - block_group->zone_is_active = 1; + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); space_info->active_total_bytes += block_group->length; spin_unlock(&block_group->lock); btrfs_try_granting_tickets(fs_info, space_info); @@ -1910,22 +1892,55 @@ out_unlock: return ret; } +static void wait_eb_writebacks(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + const u64 end = block_group->start + block_group->length; + struct radix_tree_iter iter; + struct extent_buffer *eb; + void __rcu **slot; + + rcu_read_lock(); + radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter, + block_group->start >> fs_info->sectorsize_bits) { + eb = radix_tree_deref_slot(slot); + if (!eb) + continue; + if (radix_tree_deref_retry(eb)) { + slot = radix_tree_iter_retry(&iter); + continue; + } + + if (eb->start < block_group->start) + continue; + if (eb->start >= end) + break; + + slot = radix_tree_iter_resume(slot, &iter); + rcu_read_unlock(); + wait_on_extent_buffer_writeback(eb); + rcu_read_lock(); + } + rcu_read_unlock(); +} + static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written) { struct btrfs_fs_info *fs_info = block_group->fs_info; struct map_lookup *map; + const bool is_metadata = (block_group->flags & + (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)); int ret = 0; int i; spin_lock(&block_group->lock); - if (!block_group->zone_is_active) { + if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { spin_unlock(&block_group->lock); return 0; } /* Check if we have unwritten allocated space */ - if ((block_group->flags & - (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) && + if (is_metadata && block_group->start + block_group->alloc_offset > block_group->meta_write_pointer) { spin_unlock(&block_group->lock); return -EAGAIN; @@ -1950,6 +1965,9 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ /* No need to wait for NOCOW writers. Zoned mode does not allow that */ btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group->start, block_group->length); + /* Wait for extent buffers to be written. */ + if (is_metadata) + wait_eb_writebacks(block_group); spin_lock(&block_group->lock); @@ -1957,7 +1975,8 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ * Bail out if someone already deactivated the block group, or * allocated space is left in the block group. */ - if (!block_group->zone_is_active) { + if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &block_group->runtime_flags)) { spin_unlock(&block_group->lock); btrfs_dec_block_group_ro(block_group); return 0; @@ -1970,7 +1989,7 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ } } - block_group->zone_is_active = 0; + clear_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); block_group->alloc_offset = block_group->zone_capacity; block_group->free_space_ctl->free_space = 0; btrfs_clear_treelog_bg(block_group); @@ -2007,8 +2026,7 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ /* For active_bg_list */ btrfs_put_block_group(block_group); - clear_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); - wake_up_all(&fs_info->zone_finish_wait); + clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); return 0; } @@ -2179,13 +2197,14 @@ void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logica ASSERT(block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)); spin_lock(&block_group->lock); - if (!block_group->zoned_data_reloc_ongoing) + if (!test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) goto out; /* All relocation extents are written. */ if (block_group->start + block_group->alloc_offset == logical + length) { /* Now, release this block group for further allocations. */ - block_group->zoned_data_reloc_ongoing = 0; + clear_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, + &block_group->runtime_flags); } out: @@ -2257,7 +2276,9 @@ int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, list) { if (!spin_trylock(&bg->lock)) continue; - if (btrfs_zoned_bg_is_full(bg) || bg->zone_is_active) { + if (btrfs_zoned_bg_is_full(bg) || + test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &bg->runtime_flags)) { spin_unlock(&bg->lock); continue; } diff --git a/fs/buffer.c b/fs/buffer.c index 55e762a58eb651af2e579cfe2a12ff0d11a58b6d..d9c6d1fbb6dde5d6574f63e5b3596e756ee36c5d 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -52,8 +52,8 @@ #include "internal.h" static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); -static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - struct writeback_control *wbc); +static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + struct writeback_control *wbc); #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) @@ -152,7 +152,7 @@ static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate) /* * Default synchronous end-of-IO handler.. Just mark it up-to-date and - * unlock the buffer. This is what ll_rw_block uses too. + * unlock the buffer. */ void end_buffer_read_sync(struct buffer_head *bh, int uptodate) { @@ -491,8 +491,8 @@ int inode_has_buffers(struct inode *inode) * all already-submitted IO to complete, but does not queue any new * writes to the disk. * - * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as - * you dirty the buffers, and then use osync_inode_buffers to wait for + * To do O_SYNC writes, just queue the buffer writes with write_dirty_buffer + * as you dirty the buffers, and then use osync_inode_buffers to wait for * completion. Any other dirty buffers which are not yet queued for * write will not be flushed to disk by the osync. */ @@ -562,7 +562,7 @@ void write_boundary_block(struct block_device *bdev, struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize); if (bh) { if (buffer_dirty(bh)) - ll_rw_block(REQ_OP_WRITE, 1, &bh); + write_dirty_buffer(bh, 0); put_bh(bh); } } @@ -1342,23 +1342,12 @@ void __breadahead(struct block_device *bdev, sector_t block, unsigned size) { struct buffer_head *bh = __getblk(bdev, block, size); if (likely(bh)) { - ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); + bh_readahead(bh, REQ_RAHEAD); brelse(bh); } } EXPORT_SYMBOL(__breadahead); -void __breadahead_gfp(struct block_device *bdev, sector_t block, unsigned size, - gfp_t gfp) -{ - struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp); - if (likely(bh)) { - ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, &bh); - brelse(bh); - } -} -EXPORT_SYMBOL(__breadahead_gfp); - /** * __bread_gfp() - reads a specified block and returns the bh * @bdev: the block_device to read from @@ -1464,19 +1453,15 @@ EXPORT_SYMBOL(set_bh_page); static void discard_buffer(struct buffer_head * bh) { - unsigned long b_state, b_state_old; + unsigned long b_state; lock_buffer(bh); clear_buffer_dirty(bh); bh->b_bdev = NULL; - b_state = bh->b_state; - for (;;) { - b_state_old = cmpxchg(&bh->b_state, b_state, - (b_state & ~BUFFER_FLAGS_DISCARD)); - if (b_state_old == b_state) - break; - b_state = b_state_old; - } + b_state = READ_ONCE(bh->b_state); + do { + } while (!try_cmpxchg(&bh->b_state, &b_state, + b_state & ~BUFFER_FLAGS_DISCARD)); unlock_buffer(bh); } @@ -1817,7 +1802,7 @@ done: /* * The page was marked dirty, but the buffers were * clean. Someone wrote them back by hand with - * ll_rw_block/submit_bh. A rare case. + * write_dirty_buffer/submit_bh. A rare case. */ end_page_writeback(page); @@ -2033,7 +2018,7 @@ int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh) && (block_start < from || block_end > to)) { - ll_rw_block(REQ_OP_READ, 1, &bh); + bh_read_nowait(bh, 0); *wait_bh++=bh; } } @@ -2352,7 +2337,7 @@ int generic_cont_expand_simple(struct inode *inode, loff_t size) struct address_space *mapping = inode->i_mapping; const struct address_space_operations *aops = mapping->a_ops; struct page *page; - void *fsdata; + void *fsdata = NULL; int err; err = inode_newsize_ok(inode, size); @@ -2378,7 +2363,7 @@ static int cont_expand_zero(struct file *file, struct address_space *mapping, const struct address_space_operations *aops = mapping->a_ops; unsigned int blocksize = i_blocksize(inode); struct page *page; - void *fsdata; + void *fsdata = NULL; pgoff_t index, curidx; loff_t curpos; unsigned zerofrom, offset, len; @@ -2593,11 +2578,9 @@ int block_truncate_page(struct address_space *mapping, set_buffer_uptodate(bh); if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { - err = -EIO; - ll_rw_block(REQ_OP_READ, 1, &bh); - wait_on_buffer(bh); + err = bh_read(bh, 0); /* Uhhuh. Read error. Complain and punt. */ - if (!buffer_uptodate(bh)) + if (err < 0) goto unlock; } @@ -2673,8 +2656,8 @@ static void end_bio_bh_io_sync(struct bio *bio) bio_put(bio); } -static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, - struct writeback_control *wbc) +static void submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, + struct writeback_control *wbc) { const enum req_op op = opf & REQ_OP_MASK; struct bio *bio; @@ -2717,70 +2700,14 @@ static int submit_bh_wbc(blk_opf_t opf, struct buffer_head *bh, } submit_bio(bio); - return 0; } -int submit_bh(blk_opf_t opf, struct buffer_head *bh) +void submit_bh(blk_opf_t opf, struct buffer_head *bh) { - return submit_bh_wbc(opf, bh, NULL); + submit_bh_wbc(opf, bh, NULL); } EXPORT_SYMBOL(submit_bh); -/** - * ll_rw_block: low-level access to block devices (DEPRECATED) - * @opf: block layer request operation and flags. - * @nr: number of &struct buffer_heads in the array - * @bhs: array of pointers to &struct buffer_head - * - * ll_rw_block() takes an array of pointers to &struct buffer_heads, and - * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE. - * @opf contains flags modifying the detailed I/O behavior, most notably - * %REQ_RAHEAD. - * - * This function drops any buffer that it cannot get a lock on (with the - * BH_Lock state bit), any buffer that appears to be clean when doing a write - * request, and any buffer that appears to be up-to-date when doing read - * request. Further it marks as clean buffers that are processed for - * writing (the buffer cache won't assume that they are actually clean - * until the buffer gets unlocked). - * - * ll_rw_block sets b_end_io to simple completion handler that marks - * the buffer up-to-date (if appropriate), unlocks the buffer and wakes - * any waiters. - * - * All of the buffers must be for the same device, and must also be a - * multiple of the current approved size for the device. - */ -void ll_rw_block(const blk_opf_t opf, int nr, struct buffer_head *bhs[]) -{ - const enum req_op op = opf & REQ_OP_MASK; - int i; - - for (i = 0; i < nr; i++) { - struct buffer_head *bh = bhs[i]; - - if (!trylock_buffer(bh)) - continue; - if (op == REQ_OP_WRITE) { - if (test_clear_buffer_dirty(bh)) { - bh->b_end_io = end_buffer_write_sync; - get_bh(bh); - submit_bh(opf, bh); - continue; - } - } else { - if (!buffer_uptodate(bh)) { - bh->b_end_io = end_buffer_read_sync; - get_bh(bh); - submit_bh(opf, bh); - continue; - } - } - unlock_buffer(bh); - } -} -EXPORT_SYMBOL(ll_rw_block); - void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { lock_buffer(bh); @@ -2801,8 +2728,6 @@ EXPORT_SYMBOL(write_dirty_buffer); */ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) { - int ret = 0; - WARN_ON(atomic_read(&bh->b_count) < 1); lock_buffer(bh); if (test_clear_buffer_dirty(bh)) { @@ -2817,14 +2742,14 @@ int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) get_bh(bh); bh->b_end_io = end_buffer_write_sync; - ret = submit_bh(REQ_OP_WRITE | op_flags, bh); + submit_bh(REQ_OP_WRITE | op_flags, bh); wait_on_buffer(bh); - if (!ret && !buffer_uptodate(bh)) - ret = -EIO; + if (!buffer_uptodate(bh)) + return -EIO; } else { unlock_buffer(bh); } - return ret; + return 0; } EXPORT_SYMBOL(__sync_dirty_buffer); @@ -3029,29 +2954,69 @@ int bh_uptodate_or_lock(struct buffer_head *bh) EXPORT_SYMBOL(bh_uptodate_or_lock); /** - * bh_submit_read - Submit a locked buffer for reading + * __bh_read - Submit read for a locked buffer * @bh: struct buffer_head + * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ + * @wait: wait until reading finish * - * Returns zero on success and -EIO on error. + * Returns zero on success or don't wait, and -EIO on error. */ -int bh_submit_read(struct buffer_head *bh) +int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) { - BUG_ON(!buffer_locked(bh)); + int ret = 0; - if (buffer_uptodate(bh)) { - unlock_buffer(bh); - return 0; - } + BUG_ON(!buffer_locked(bh)); get_bh(bh); bh->b_end_io = end_buffer_read_sync; - submit_bh(REQ_OP_READ, bh); - wait_on_buffer(bh); - if (buffer_uptodate(bh)) - return 0; - return -EIO; + submit_bh(REQ_OP_READ | op_flags, bh); + if (wait) { + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + } + return ret; +} +EXPORT_SYMBOL(__bh_read); + +/** + * __bh_read_batch - Submit read for a batch of unlocked buffers + * @nr: entry number of the buffer batch + * @bhs: a batch of struct buffer_head + * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ + * @force_lock: force to get a lock on the buffer if set, otherwise drops any + * buffer that cannot lock. + * + * Returns zero on success or don't wait, and -EIO on error. + */ +void __bh_read_batch(int nr, struct buffer_head *bhs[], + blk_opf_t op_flags, bool force_lock) +{ + int i; + + for (i = 0; i < nr; i++) { + struct buffer_head *bh = bhs[i]; + + if (buffer_uptodate(bh)) + continue; + + if (force_lock) + lock_buffer(bh); + else + if (!trylock_buffer(bh)) + continue; + + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + continue; + } + + bh->b_end_io = end_buffer_read_sync; + get_bh(bh); + submit_bh(REQ_OP_READ | op_flags, bh); + } } -EXPORT_SYMBOL(bh_submit_read); +EXPORT_SYMBOL(__bh_read_batch); void __init buffer_init(void) { diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index 6cba2c6de2f963d762da94aea13f37dedd33349a..2ad58c465208480e42106393ae01fd2f317389ee 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -111,6 +111,7 @@ struct cachefiles_cache { char *tag; /* cache binding tag */ refcount_t unbind_pincount;/* refcount to do daemon unbind */ struct xarray reqs; /* xarray of pending on-demand requests */ + unsigned long req_id_next; struct xarray ondemand_ids; /* xarray for ondemand_id allocation */ u32 ondemand_id_next; }; diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index facf2ebe464b35f736e2fc179e6b7bc0c6bafcbf..03ca8f2f657ab8d6ade31a45841cdde9cb0d18ed 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -15,9 +15,8 @@ * file or directory. The caller must hold the inode lock. */ static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, - struct dentry *dentry) + struct inode *inode) { - struct inode *inode = d_backing_inode(dentry); bool can_use = false; if (!(inode->i_flags & S_KERNEL_FILE)) { @@ -26,21 +25,18 @@ static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, can_use = true; } else { trace_cachefiles_mark_failed(object, inode); - pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n", - dentry, inode->i_ino); } return can_use; } static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object, - struct dentry *dentry) + struct inode *inode) { - struct inode *inode = d_backing_inode(dentry); bool can_use; inode_lock(inode); - can_use = __cachefiles_mark_inode_in_use(object, dentry); + can_use = __cachefiles_mark_inode_in_use(object, inode); inode_unlock(inode); return can_use; } @@ -49,21 +45,17 @@ static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object, * Unmark a backing inode. The caller must hold the inode lock. */ static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object, - struct dentry *dentry) + struct inode *inode) { - struct inode *inode = d_backing_inode(dentry); - inode->i_flags &= ~S_KERNEL_FILE; trace_cachefiles_mark_inactive(object, inode); } static void cachefiles_do_unmark_inode_in_use(struct cachefiles_object *object, - struct dentry *dentry) + struct inode *inode) { - struct inode *inode = d_backing_inode(dentry); - inode_lock(inode); - __cachefiles_unmark_inode_in_use(object, dentry); + __cachefiles_unmark_inode_in_use(object, inode); inode_unlock(inode); } @@ -77,14 +69,12 @@ void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, struct cachefiles_cache *cache = object->volume->cache; struct inode *inode = file_inode(file); - if (inode) { - cachefiles_do_unmark_inode_in_use(object, file->f_path.dentry); + cachefiles_do_unmark_inode_in_use(object, inode); - if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { - atomic_long_add(inode->i_blocks, &cache->b_released); - if (atomic_inc_return(&cache->f_released)) - cachefiles_state_changed(cache); - } + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + atomic_long_add(inode->i_blocks, &cache->b_released); + if (atomic_inc_return(&cache->f_released)) + cachefiles_state_changed(cache); } } @@ -164,8 +154,11 @@ retry: inode_lock(d_inode(subdir)); inode_unlock(d_inode(dir)); - if (!__cachefiles_mark_inode_in_use(NULL, subdir)) + if (!__cachefiles_mark_inode_in_use(NULL, d_inode(subdir))) { + pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n", + subdir, d_inode(subdir)->i_ino); goto mark_error; + } inode_unlock(d_inode(subdir)); @@ -224,9 +217,7 @@ nomem_d_alloc: void cachefiles_put_directory(struct dentry *dir) { if (dir) { - inode_lock(dir->d_inode); - __cachefiles_unmark_inode_in_use(NULL, dir); - inode_unlock(dir->d_inode); + cachefiles_do_unmark_inode_in_use(NULL, d_inode(dir)); dput(dir); } } @@ -410,7 +401,7 @@ try_again: "Rename failed with error %d", ret); } - __cachefiles_unmark_inode_in_use(object, rep); + __cachefiles_unmark_inode_in_use(object, d_inode(rep)); unlock_rename(cache->graveyard, dir); dput(grave); _leave(" = 0"); @@ -451,84 +442,72 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) const struct cred *saved_cred; struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; struct file *file; - struct path path; + const struct path parentpath = { .mnt = cache->mnt, .dentry = fan }; uint64_t ni_size; long ret; cachefiles_begin_secure(cache, &saved_cred); - path.mnt = cache->mnt; ret = cachefiles_inject_write_error(); - if (ret == 0) - path.dentry = vfs_tmpfile(&init_user_ns, fan, S_IFREG, O_RDWR); - else - path.dentry = ERR_PTR(ret); - if (IS_ERR(path.dentry)) { - trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(path.dentry), + if (ret == 0) { + file = vfs_tmpfile_open(&init_user_ns, &parentpath, S_IFREG, + O_RDWR | O_LARGEFILE | O_DIRECT, + cache->cache_cred); + ret = PTR_ERR_OR_ZERO(file); + } + if (ret) { + trace_cachefiles_vfs_error(object, d_inode(fan), ret, cachefiles_trace_tmpfile_error); - if (PTR_ERR(path.dentry) == -EIO) + if (ret == -EIO) cachefiles_io_error_obj(object, "Failed to create tmpfile"); - file = ERR_CAST(path.dentry); - goto out; + goto err; } - trace_cachefiles_tmpfile(object, d_backing_inode(path.dentry)); + trace_cachefiles_tmpfile(object, file_inode(file)); - if (!cachefiles_mark_inode_in_use(object, path.dentry)) { - file = ERR_PTR(-EBUSY); - goto out_dput; - } + /* This is a newly created file with no other possible user */ + if (!cachefiles_mark_inode_in_use(object, file_inode(file))) + WARN_ON(1); ret = cachefiles_ondemand_init_object(object); - if (ret < 0) { - file = ERR_PTR(ret); - goto out_unuse; - } + if (ret < 0) + goto err_unuse; ni_size = object->cookie->object_size; ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); if (ni_size > 0) { - trace_cachefiles_trunc(object, d_backing_inode(path.dentry), 0, ni_size, + trace_cachefiles_trunc(object, file_inode(file), 0, ni_size, cachefiles_trunc_expand_tmpfile); ret = cachefiles_inject_write_error(); if (ret == 0) - ret = vfs_truncate(&path, ni_size); + ret = vfs_truncate(&file->f_path, ni_size); if (ret < 0) { trace_cachefiles_vfs_error( - object, d_backing_inode(path.dentry), ret, + object, file_inode(file), ret, cachefiles_trace_trunc_error); - file = ERR_PTR(ret); - goto out_unuse; + goto err_unuse; } } - file = open_with_fake_path(&path, O_RDWR | O_LARGEFILE | O_DIRECT, - d_backing_inode(path.dentry), cache->cache_cred); - if (IS_ERR(file)) { - trace_cachefiles_vfs_error(object, d_backing_inode(path.dentry), - PTR_ERR(file), - cachefiles_trace_open_error); - goto out_unuse; - } + ret = -EINVAL; if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { fput(file); pr_notice("Cache does not support read_iter and write_iter\n"); - file = ERR_PTR(-EINVAL); - goto out_unuse; + goto err_unuse; } - - goto out_dput; - -out_unuse: - cachefiles_do_unmark_inode_in_use(object, path.dentry); -out_dput: - dput(path.dentry); out: cachefiles_end_secure(cache, saved_cred); return file; + +err_unuse: + cachefiles_do_unmark_inode_in_use(object, file_inode(file)); + fput(file); +err: + file = ERR_PTR(ret); + goto out; } /* @@ -569,8 +548,11 @@ static bool cachefiles_open_file(struct cachefiles_object *object, _enter("%pd", dentry); - if (!cachefiles_mark_inode_in_use(object, dentry)) + if (!cachefiles_mark_inode_in_use(object, d_inode(dentry))) { + pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n", + dentry, d_inode(dentry)->i_ino); return false; + } /* We need to open a file interface onto a data file now as we can't do * it on demand because writeback called from do_exit() sees @@ -624,7 +606,7 @@ check_failed: error_fput: fput(file); error: - cachefiles_do_unmark_inode_in_use(object, dentry); + cachefiles_do_unmark_inode_in_use(object, d_inode(dentry)); dput(dentry); return false; } diff --git a/fs/cachefiles/ondemand.c b/fs/cachefiles/ondemand.c index 1fee702d55293e9a25b23fe83dc9bb07dbe96ddb..0254ed39f68ceb75dcb2ec840af88b136ac21fe3 100644 --- a/fs/cachefiles/ondemand.c +++ b/fs/cachefiles/ondemand.c @@ -158,9 +158,13 @@ int cachefiles_ondemand_copen(struct cachefiles_cache *cache, char *args) /* fail OPEN request if daemon reports an error */ if (size < 0) { - if (!IS_ERR_VALUE(size)) - size = -EINVAL; - req->error = size; + if (!IS_ERR_VALUE(size)) { + req->error = -EINVAL; + ret = -EINVAL; + } else { + req->error = size; + ret = 0; + } goto out; } @@ -238,14 +242,19 @@ ssize_t cachefiles_ondemand_daemon_read(struct cachefiles_cache *cache, unsigned long id = 0; size_t n; int ret = 0; - XA_STATE(xas, &cache->reqs, 0); + XA_STATE(xas, &cache->reqs, cache->req_id_next); /* - * Search for a request that has not ever been processed, to prevent - * requests from being processed repeatedly. + * Cyclically search for a request that has not ever been processed, + * to prevent requests from being processed repeatedly, and make + * request distribution fair. */ xa_lock(&cache->reqs); req = xas_find_marked(&xas, UINT_MAX, CACHEFILES_REQ_NEW); + if (!req && cache->req_id_next > 0) { + xas_set(&xas, 0); + req = xas_find_marked(&xas, cache->req_id_next - 1, CACHEFILES_REQ_NEW); + } if (!req) { xa_unlock(&cache->reqs); return 0; @@ -260,6 +269,7 @@ ssize_t cachefiles_ondemand_daemon_read(struct cachefiles_cache *cache, } xas_clear_mark(&xas, CACHEFILES_REQ_NEW); + cache->req_id_next = xas.xa_index + 1; xa_unlock(&cache->reqs); id = xas.xa_index; diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 53cfe026b3ea5353172c354279d9772e449fbe01..fb023f9fafcbe38dedb2246ade02f8fddece19bf 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -754,6 +754,7 @@ void ceph_add_cap(struct inode *inode, cap->issue_seq = seq; cap->mseq = mseq; cap->cap_gen = gen; + wake_up_all(&ci->i_cap_wq); } /* @@ -2285,7 +2286,7 @@ retry: struct ceph_mds_request *req; int i; - sessions = kzalloc(max_sessions * sizeof(s), GFP_KERNEL); + sessions = kcalloc(max_sessions, sizeof(s), GFP_KERNEL); if (!sessions) { err = -ENOMEM; goto out; @@ -2759,13 +2760,17 @@ again: * on transition from wanted -> needed caps. This is needed * for WRBUFFER|WR -> WR to avoid a new WR sync write from * going before a prior buffered writeback happens. + * + * For RDCACHE|RD -> RD, there is not need to wait and we can + * just exclude the revoking caps and force to sync read. */ int not = want & ~(have & need); int revoking = implemented & ~have; + int exclude = revoking & not; dout("get_cap_refs %p have %s but not %s (revoking %s)\n", inode, ceph_cap_string(have), ceph_cap_string(not), ceph_cap_string(revoking)); - if ((revoking & not) == 0) { + if (!exclude || !(exclude & CEPH_CAP_FILE_BUFFER)) { if (!snap_rwsem_locked && !ci->i_head_snapc && (need & CEPH_CAP_FILE_WR)) { @@ -2787,7 +2792,7 @@ again: snap_rwsem_locked = true; } if ((have & want) == want) - *got = need | want; + *got = need | (want & ~exclude); else *got = need; ceph_take_cap_refs(ci, *got, true); @@ -3550,6 +3555,9 @@ static void handle_cap_grant(struct inode *inode, check_caps = 1; /* check auth cap only */ else check_caps = 2; /* check all caps */ + /* If there is new caps, try to wake up the waiters */ + if (~cap->issued & newcaps) + wake = true; cap->issued = newcaps; cap->implemented |= newcaps; } else if (cap->issued == newcaps) { diff --git a/fs/ceph/export.c b/fs/ceph/export.c index e0fa66ac8b9faa612969e0463ade52bc4b60d341..f780e4e0d06295cad7bf8721b6ca2aaa701d454e 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -181,6 +181,7 @@ struct inode *ceph_lookup_inode(struct super_block *sb, u64 ino) static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino) { struct inode *inode = __lookup_inode(sb, ino); + struct ceph_inode_info *ci = ceph_inode(inode); int err; if (IS_ERR(inode)) @@ -192,7 +193,7 @@ static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino) return ERR_PTR(err); } /* -ESTALE if inode as been unlinked and no file is open */ - if ((inode->i_nlink == 0) && (atomic_read(&inode->i_count) == 1)) { + if ((inode->i_nlink == 0) && !__ceph_is_file_opened(ci)) { iput(inode); return ERR_PTR(-ESTALE); } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index 42351d7a0dd6b74b3a246ad472db6f35c6742c11..4af5e55abc1586748882e54e3da3660d8444aaab 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -362,7 +362,7 @@ static int ceph_fill_fragtree(struct inode *inode, if (nsplits != ci->i_fragtree_nsplits) { update = true; } else if (nsplits) { - i = prandom_u32() % nsplits; + i = prandom_u32_max(nsplits); id = le32_to_cpu(fragtree->splits[i].frag); if (!__ceph_find_frag(ci, id)) update = true; @@ -2192,6 +2192,7 @@ int __ceph_setattr(struct inode *inode, struct iattr *attr) inode_dirty_flags = __ceph_mark_dirty_caps(ci, dirtied, &prealloc_cf); inode->i_ctime = attr->ia_ctime; + inode_inc_iversion_raw(inode); } release &= issued; @@ -2356,6 +2357,7 @@ int ceph_do_getvxattr(struct inode *inode, const char *name, void *value, goto out; } + req->r_feature_needed = CEPHFS_FEATURE_OP_GETVXATTR; req->r_path2 = kstrdup(name, GFP_NOFS); if (!req->r_path2) { err = -ENOMEM; @@ -2447,6 +2449,7 @@ int ceph_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags) { struct inode *inode = d_inode(path->dentry); + struct super_block *sb = inode->i_sb; struct ceph_inode_info *ci = ceph_inode(inode); u32 valid_mask = STATX_BASIC_STATS; int err = 0; @@ -2476,16 +2479,34 @@ int ceph_getattr(struct user_namespace *mnt_userns, const struct path *path, } if (ceph_snap(inode) == CEPH_NOSNAP) - stat->dev = inode->i_sb->s_dev; + stat->dev = sb->s_dev; else stat->dev = ci->i_snapid_map ? ci->i_snapid_map->dev : 0; if (S_ISDIR(inode->i_mode)) { - if (ceph_test_mount_opt(ceph_sb_to_client(inode->i_sb), - RBYTES)) + if (ceph_test_mount_opt(ceph_sb_to_client(sb), RBYTES)) { stat->size = ci->i_rbytes; - else + } else if (ceph_snap(inode) == CEPH_SNAPDIR) { + struct ceph_inode_info *pci; + struct ceph_snap_realm *realm; + struct inode *parent; + + parent = ceph_lookup_inode(sb, ceph_ino(inode)); + if (!parent) + return PTR_ERR(parent); + + pci = ceph_inode(parent); + spin_lock(&pci->i_ceph_lock); + realm = pci->i_snap_realm; + if (realm) + stat->size = realm->num_snaps; + else + stat->size = 0; + spin_unlock(&pci->i_ceph_lock); + iput(parent); + } else { stat->size = ci->i_files + ci->i_subdirs; + } stat->blocks = 0; stat->blksize = 65536; /* diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index 80f8b9ec1a312d839111370a0ad5e1e15bdd5f27..26a0a8b9975ef3a019b1036f7cb2ff7f47378282 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2318,6 +2318,7 @@ ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode) INIT_LIST_HEAD(&req->r_unsafe_dir_item); INIT_LIST_HEAD(&req->r_unsafe_target_item); req->r_fmode = -1; + req->r_feature_needed = -1; kref_init(&req->r_kref); RB_CLEAR_NODE(&req->r_node); INIT_LIST_HEAD(&req->r_wait); @@ -2916,6 +2917,16 @@ static void __do_request(struct ceph_mds_client *mdsc, dout("do_request mds%d session %p state %s\n", mds, session, ceph_session_state_name(session->s_state)); + + /* + * The old ceph will crash the MDSs when see unknown OPs + */ + if (req->r_feature_needed > 0 && + !test_bit(req->r_feature_needed, &session->s_features)) { + err = -EOPNOTSUPP; + goto out_session; + } + if (session->s_state != CEPH_MDS_SESSION_OPEN && session->s_state != CEPH_MDS_SESSION_HUNG) { /* diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 256e3eada6c12600b35f9c7bec5a3fabe7b14c98..0598faa50e2e0e6ebf51cd96b71c66e574168307 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -31,8 +31,9 @@ enum ceph_feature_type { CEPHFS_FEATURE_METRIC_COLLECT, CEPHFS_FEATURE_ALTERNATE_NAME, CEPHFS_FEATURE_NOTIFY_SESSION_STATE, + CEPHFS_FEATURE_OP_GETVXATTR, - CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_NOTIFY_SESSION_STATE, + CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_OP_GETVXATTR, }; #define CEPHFS_FEATURES_CLIENT_SUPPORTED { \ @@ -44,6 +45,7 @@ enum ceph_feature_type { CEPHFS_FEATURE_DELEG_INO, \ CEPHFS_FEATURE_METRIC_COLLECT, \ CEPHFS_FEATURE_NOTIFY_SESSION_STATE, \ + CEPHFS_FEATURE_OP_GETVXATTR, \ } /* @@ -336,6 +338,8 @@ struct ceph_mds_request { long long r_dir_ordered_cnt; int r_readdir_cache_idx; + int r_feature_needed; + struct ceph_cap_reservation r_caps_reservation; }; diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c index 8d0a6d2c2da4332c007fc0386d8912a5fda29c65..3fbabc98e1f702a90ea75ce6d4f7c5f9fb7b4a16 100644 --- a/fs/ceph/mdsmap.c +++ b/fs/ceph/mdsmap.c @@ -29,7 +29,7 @@ static int __mdsmap_get_random_mds(struct ceph_mdsmap *m, bool ignore_laggy) return -1; /* pick */ - n = prandom_u32() % n; + n = prandom_u32_max(n); for (j = 0, i = 0; i < m->possible_max_rank; i++) { if (CEPH_MDS_IS_READY(i, ignore_laggy)) j++; diff --git a/fs/cifs/cached_dir.c b/fs/cifs/cached_dir.c index b401339f6e738a390891ec99ace9047400bce649..fe88b67c863fe82be03e584e607dc5b599bd899c 100644 --- a/fs/cifs/cached_dir.c +++ b/fs/cifs/cached_dir.c @@ -5,12 +5,99 @@ * Copyright (c) 2022, Ronnie Sahlberg */ +#include #include "cifsglob.h" #include "cifsproto.h" #include "cifs_debug.h" #include "smb2proto.h" #include "cached_dir.h" +static struct cached_fid *init_cached_dir(const char *path); +static void free_cached_dir(struct cached_fid *cfid); + +static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids, + const char *path, + bool lookup_only) +{ + struct cached_fid *cfid; + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + if (!strcmp(cfid->path, path)) { + /* + * If it doesn't have a lease it is either not yet + * fully cached or it may be in the process of + * being deleted due to a lease break. + */ + if (!cfid->has_lease) { + spin_unlock(&cfids->cfid_list_lock); + return NULL; + } + kref_get(&cfid->refcount); + spin_unlock(&cfids->cfid_list_lock); + return cfid; + } + } + if (lookup_only) { + spin_unlock(&cfids->cfid_list_lock); + return NULL; + } + if (cfids->num_entries >= MAX_CACHED_FIDS) { + spin_unlock(&cfids->cfid_list_lock); + return NULL; + } + cfid = init_cached_dir(path); + if (cfid == NULL) { + spin_unlock(&cfids->cfid_list_lock); + return NULL; + } + cfid->cfids = cfids; + cfids->num_entries++; + list_add(&cfid->entry, &cfids->entries); + cfid->on_list = true; + kref_get(&cfid->refcount); + spin_unlock(&cfids->cfid_list_lock); + return cfid; +} + +static struct dentry * +path_to_dentry(struct cifs_sb_info *cifs_sb, const char *path) +{ + struct dentry *dentry; + const char *s, *p; + char sep; + + sep = CIFS_DIR_SEP(cifs_sb); + dentry = dget(cifs_sb->root); + s = path; + + do { + struct inode *dir = d_inode(dentry); + struct dentry *child; + + if (!S_ISDIR(dir->i_mode)) { + dput(dentry); + dentry = ERR_PTR(-ENOTDIR); + break; + } + + /* skip separators */ + while (*s == sep) + s++; + if (!*s) + break; + p = s++; + /* next separator */ + while (*s && *s != sep) + s++; + + child = lookup_positive_unlocked(p, dentry, s - p); + dput(dentry); + dentry = child; + } while (!IS_ERR(dentry)); + return dentry; +} + /* * Open the and cache a directory handle. * If error then *cfid is not initialized. @@ -31,54 +118,57 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; struct kvec qi_iov[1]; int rc, flags = 0; - __le16 utf16_path = 0; /* Null - since an open of top of share */ + __le16 *utf16_path = NULL; u8 oplock = SMB2_OPLOCK_LEVEL_II; struct cifs_fid *pfid; - struct dentry *dentry; + struct dentry *dentry = NULL; struct cached_fid *cfid; + struct cached_fids *cfids; - if (tcon == NULL || tcon->nohandlecache || + if (tcon == NULL || tcon->cfids == NULL || tcon->nohandlecache || is_smb1_server(tcon->ses->server)) return -EOPNOTSUPP; ses = tcon->ses; server = ses->server; + cfids = tcon->cfids; - if (cifs_sb->root == NULL) - return -ENOENT; + if (!server->ops->new_lease_key) + return -EIO; - if (strlen(path)) + if (cifs_sb->root == NULL) return -ENOENT; - dentry = cifs_sb->root; + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; - cfid = tcon->cfid; - mutex_lock(&cfid->fid_mutex); - if (cfid->is_valid) { - cifs_dbg(FYI, "found a cached root file handle\n"); + cfid = find_or_create_cached_dir(cfids, path, lookup_only); + if (cfid == NULL) { + kfree(utf16_path); + return -ENOENT; + } + /* + * At this point we either have a lease already and we can just + * return it. If not we are guaranteed to be the only thread accessing + * this cfid. + */ + if (cfid->has_lease) { *ret_cfid = cfid; - kref_get(&cfid->refcount); - mutex_unlock(&cfid->fid_mutex); + kfree(utf16_path); return 0; } /* * We do not hold the lock for the open because in case - * SMB2_open needs to reconnect, it will end up calling - * cifs_mark_open_files_invalid() which takes the lock again - * thus causing a deadlock + * SMB2_open needs to reconnect. + * This is safe because no other thread will be able to get a ref + * to the cfid until we have finished opening the file and (possibly) + * acquired a lease. */ - mutex_unlock(&cfid->fid_mutex); - - if (lookup_only) - return -ENOENT; - if (smb3_encryption_required(tcon)) flags |= CIFS_TRANSFORM_REQ; - if (!server->ops->new_lease_key) - return -EIO; - pfid = &cfid->fid; server->ops->new_lease_key(pfid); @@ -99,7 +189,7 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, oparms.reconnect = false; rc = SMB2_open_init(tcon, server, - &rqst[0], &oplock, &oparms, &utf16_path); + &rqst[0], &oplock, &oparms, utf16_path); if (rc) goto oshr_free; smb2_set_next_command(tcon, &rqst[0]); @@ -122,47 +212,13 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, rc = compound_send_recv(xid, ses, server, flags, 2, rqst, resp_buftype, rsp_iov); - mutex_lock(&cfid->fid_mutex); - - /* - * Now we need to check again as the cached root might have - * been successfully re-opened from a concurrent process - */ - - if (cfid->is_valid) { - /* work was already done */ - - /* stash fids for close() later */ - struct cifs_fid fid = { - .persistent_fid = pfid->persistent_fid, - .volatile_fid = pfid->volatile_fid, - }; - - /* - * caller expects this func to set the fid in cfid to valid - * cached root, so increment the refcount. - */ - kref_get(&cfid->refcount); - - mutex_unlock(&cfid->fid_mutex); - - if (rc == 0) { - /* close extra handle outside of crit sec */ - SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); - } - rc = 0; - goto oshr_free; - } - - /* Cached root is still invalid, continue normaly */ - if (rc) { if (rc == -EREMCHG) { tcon->need_reconnect = true; pr_warn_once("server share %s deleted\n", - tcon->treeName); + tcon->tree_name); } - goto oshr_exit; + goto oshr_free; } atomic_inc(&tcon->num_remote_opens); @@ -174,30 +230,18 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId); #endif /* CIFS_DEBUG2 */ - cfid->tcon = tcon; - cfid->is_valid = true; - cfid->dentry = dentry; - dget(dentry); - kref_init(&cfid->refcount); + if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) + goto oshr_free; - /* BB TBD check to see if oplock level check can be removed below */ - if (o_rsp->OplockLevel == SMB2_OPLOCK_LEVEL_LEASE) { - /* - * See commit 2f94a3125b87. Increment the refcount when we - * get a lease for root, release it if lease break occurs - */ - kref_get(&cfid->refcount); - cfid->has_lease = true; - smb2_parse_contexts(server, o_rsp, - &oparms.fid->epoch, - oparms.fid->lease_key, &oplock, - NULL, NULL); - } else - goto oshr_exit; + + smb2_parse_contexts(server, o_rsp, + &oparms.fid->epoch, + oparms.fid->lease_key, &oplock, + NULL, NULL); qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; if (le32_to_cpu(qi_rsp->OutputBufferLength) < sizeof(struct smb2_file_all_info)) - goto oshr_exit; + goto oshr_free; if (!smb2_validate_and_copy_iov( le16_to_cpu(qi_rsp->OutputBufferOffset), sizeof(struct smb2_file_all_info), @@ -205,15 +249,40 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, (char *)&cfid->file_all_info)) cfid->file_all_info_is_valid = true; + if (!path[0]) + dentry = dget(cifs_sb->root); + else { + dentry = path_to_dentry(cifs_sb, path); + if (IS_ERR(dentry)) + goto oshr_free; + } + cfid->dentry = dentry; + cfid->tcon = tcon; cfid->time = jiffies; + cfid->is_open = true; + cfid->has_lease = true; -oshr_exit: - mutex_unlock(&cfid->fid_mutex); oshr_free: + kfree(utf16_path); SMB2_open_free(&rqst[0]); SMB2_query_info_free(&rqst[1]); free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + spin_lock(&cfids->cfid_list_lock); + if (!cfid->has_lease) { + if (cfid->on_list) { + list_del(&cfid->entry); + cfid->on_list = false; + cfids->num_entries--; + } + rc = -ENOENT; + } + spin_unlock(&cfids->cfid_list_lock); + if (rc) { + free_cached_dir(cfid); + cfid = NULL; + } + if (rc == 0) *ret_cfid = cfid; @@ -225,18 +294,22 @@ int open_cached_dir_by_dentry(struct cifs_tcon *tcon, struct cached_fid **ret_cfid) { struct cached_fid *cfid; + struct cached_fids *cfids = tcon->cfids; - cfid = tcon->cfid; + if (cfids == NULL) + return -ENOENT; - mutex_lock(&cfid->fid_mutex); - if (cfid->dentry == dentry) { - cifs_dbg(FYI, "found a cached root file handle by dentry\n"); - *ret_cfid = cfid; - kref_get(&cfid->refcount); - mutex_unlock(&cfid->fid_mutex); - return 0; + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + if (dentry && cfid->dentry == dentry) { + cifs_dbg(FYI, "found a cached root file handle by dentry\n"); + kref_get(&cfid->refcount); + *ret_cfid = cfid; + spin_unlock(&cfids->cfid_list_lock); + return 0; + } } - mutex_unlock(&cfid->fid_mutex); + spin_unlock(&cfids->cfid_list_lock); return -ENOENT; } @@ -245,63 +318,29 @@ smb2_close_cached_fid(struct kref *ref) { struct cached_fid *cfid = container_of(ref, struct cached_fid, refcount); - struct cached_dirent *dirent, *q; - if (cfid->is_valid) { - cifs_dbg(FYI, "clear cached root file handle\n"); - SMB2_close(0, cfid->tcon, cfid->fid.persistent_fid, - cfid->fid.volatile_fid); + spin_lock(&cfid->cfids->cfid_list_lock); + if (cfid->on_list) { + list_del(&cfid->entry); + cfid->on_list = false; + cfid->cfids->num_entries--; } + spin_unlock(&cfid->cfids->cfid_list_lock); - /* - * We only check validity above to send SMB2_close, - * but we still need to invalidate these entries - * when this function is called - */ - cfid->is_valid = false; - cfid->file_all_info_is_valid = false; - cfid->has_lease = false; - if (cfid->dentry) { - dput(cfid->dentry); - cfid->dentry = NULL; - } - /* - * Delete all cached dirent names - */ - mutex_lock(&cfid->dirents.de_mutex); - list_for_each_entry_safe(dirent, q, &cfid->dirents.entries, entry) { - list_del(&dirent->entry); - kfree(dirent->name); - kfree(dirent); + dput(cfid->dentry); + cfid->dentry = NULL; + + if (cfid->is_open) { + SMB2_close(0, cfid->tcon, cfid->fid.persistent_fid, + cfid->fid.volatile_fid); } - cfid->dirents.is_valid = 0; - cfid->dirents.is_failed = 0; - cfid->dirents.ctx = NULL; - cfid->dirents.pos = 0; - mutex_unlock(&cfid->dirents.de_mutex); + free_cached_dir(cfid); } void close_cached_dir(struct cached_fid *cfid) { - mutex_lock(&cfid->fid_mutex); kref_put(&cfid->refcount, smb2_close_cached_fid); - mutex_unlock(&cfid->fid_mutex); -} - -void close_cached_dir_lease_locked(struct cached_fid *cfid) -{ - if (cfid->has_lease) { - cfid->has_lease = false; - kref_put(&cfid->refcount, smb2_close_cached_fid); - } -} - -void close_cached_dir_lease(struct cached_fid *cfid) -{ - mutex_lock(&cfid->fid_mutex); - close_cached_dir_lease_locked(cfid); - mutex_unlock(&cfid->fid_mutex); } /* @@ -314,34 +353,62 @@ void close_all_cached_dirs(struct cifs_sb_info *cifs_sb) struct cached_fid *cfid; struct cifs_tcon *tcon; struct tcon_link *tlink; + struct cached_fids *cfids; for (node = rb_first(root); node; node = rb_next(node)) { tlink = rb_entry(node, struct tcon_link, tl_rbnode); tcon = tlink_tcon(tlink); if (IS_ERR(tcon)) continue; - cfid = tcon->cfid; - mutex_lock(&cfid->fid_mutex); - if (cfid->dentry) { + cfids = tcon->cfids; + if (cfids == NULL) + continue; + list_for_each_entry(cfid, &cfids->entries, entry) { dput(cfid->dentry); cfid->dentry = NULL; } - mutex_unlock(&cfid->fid_mutex); } } /* - * Invalidate and close all cached dirs when a TCON has been reset + * Invalidate all cached dirs when a TCON has been reset * due to a session loss. */ void invalidate_all_cached_dirs(struct cifs_tcon *tcon) { - mutex_lock(&tcon->cfid->fid_mutex); - tcon->cfid->is_valid = false; - /* cached handle is not valid, so SMB2_CLOSE won't be sent below */ - close_cached_dir_lease_locked(tcon->cfid); - memset(&tcon->cfid->fid, 0, sizeof(struct cifs_fid)); - mutex_unlock(&tcon->cfid->fid_mutex); + struct cached_fids *cfids = tcon->cfids; + struct cached_fid *cfid, *q; + struct list_head entry; + + INIT_LIST_HEAD(&entry); + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry_safe(cfid, q, &cfids->entries, entry) { + list_del(&cfid->entry); + list_add(&cfid->entry, &entry); + cfids->num_entries--; + cfid->is_open = false; + /* To prevent race with smb2_cached_lease_break() */ + kref_get(&cfid->refcount); + } + spin_unlock(&cfids->cfid_list_lock); + + list_for_each_entry_safe(cfid, q, &entry, entry) { + cfid->on_list = false; + list_del(&cfid->entry); + cancel_work_sync(&cfid->lease_break); + if (cfid->has_lease) { + /* + * We lease was never cancelled from the server so we + * need to drop the reference. + */ + spin_lock(&cfids->cfid_list_lock); + cfid->has_lease = false; + spin_unlock(&cfids->cfid_list_lock); + kref_put(&cfid->refcount, smb2_close_cached_fid); + } + /* Drop the extra reference opened above*/ + kref_put(&cfid->refcount, smb2_close_cached_fid); + } } static void @@ -350,39 +417,123 @@ smb2_cached_lease_break(struct work_struct *work) struct cached_fid *cfid = container_of(work, struct cached_fid, lease_break); - close_cached_dir_lease(cfid); + spin_lock(&cfid->cfids->cfid_list_lock); + cfid->has_lease = false; + spin_unlock(&cfid->cfids->cfid_list_lock); + kref_put(&cfid->refcount, smb2_close_cached_fid); } int cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16]) { - if (tcon->cfid->is_valid && - !memcmp(lease_key, - tcon->cfid->fid.lease_key, - SMB2_LEASE_KEY_SIZE)) { - tcon->cfid->time = 0; - INIT_WORK(&tcon->cfid->lease_break, - smb2_cached_lease_break); - queue_work(cifsiod_wq, - &tcon->cfid->lease_break); - return true; + struct cached_fids *cfids = tcon->cfids; + struct cached_fid *cfid; + + if (cfids == NULL) + return false; + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + if (cfid->has_lease && + !memcmp(lease_key, + cfid->fid.lease_key, + SMB2_LEASE_KEY_SIZE)) { + cfid->time = 0; + /* + * We found a lease remove it from the list + * so no threads can access it. + */ + list_del(&cfid->entry); + cfid->on_list = false; + cfids->num_entries--; + + queue_work(cifsiod_wq, + &cfid->lease_break); + spin_unlock(&cfids->cfid_list_lock); + return true; + } } + spin_unlock(&cfids->cfid_list_lock); return false; } -struct cached_fid *init_cached_dir(void) +static struct cached_fid *init_cached_dir(const char *path) { struct cached_fid *cfid; - cfid = kzalloc(sizeof(*cfid), GFP_KERNEL); + cfid = kzalloc(sizeof(*cfid), GFP_ATOMIC); if (!cfid) return NULL; + cfid->path = kstrdup(path, GFP_ATOMIC); + if (!cfid->path) { + kfree(cfid); + return NULL; + } + + INIT_WORK(&cfid->lease_break, smb2_cached_lease_break); + INIT_LIST_HEAD(&cfid->entry); INIT_LIST_HEAD(&cfid->dirents.entries); mutex_init(&cfid->dirents.de_mutex); - mutex_init(&cfid->fid_mutex); + spin_lock_init(&cfid->fid_lock); + kref_init(&cfid->refcount); return cfid; } -void free_cached_dir(struct cifs_tcon *tcon) +static void free_cached_dir(struct cached_fid *cfid) +{ + struct cached_dirent *dirent, *q; + + dput(cfid->dentry); + cfid->dentry = NULL; + + /* + * Delete all cached dirent names + */ + list_for_each_entry_safe(dirent, q, &cfid->dirents.entries, entry) { + list_del(&dirent->entry); + kfree(dirent->name); + kfree(dirent); + } + + kfree(cfid->path); + cfid->path = NULL; + kfree(cfid); +} + +struct cached_fids *init_cached_dirs(void) +{ + struct cached_fids *cfids; + + cfids = kzalloc(sizeof(*cfids), GFP_KERNEL); + if (!cfids) + return NULL; + spin_lock_init(&cfids->cfid_list_lock); + INIT_LIST_HEAD(&cfids->entries); + return cfids; +} + +/* + * Called from tconInfoFree when we are tearing down the tcon. + * There are no active users or open files/directories at this point. + */ +void free_cached_dirs(struct cached_fids *cfids) { - kfree(tcon->cfid); + struct cached_fid *cfid, *q; + struct list_head entry; + + INIT_LIST_HEAD(&entry); + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry_safe(cfid, q, &cfids->entries, entry) { + cfid->on_list = false; + cfid->is_open = false; + list_del(&cfid->entry); + list_add(&cfid->entry, &entry); + } + spin_unlock(&cfids->cfid_list_lock); + + list_for_each_entry_safe(cfid, q, &entry, entry) { + list_del(&cfid->entry); + free_cached_dir(cfid); + } + + kfree(cfids); } diff --git a/fs/cifs/cached_dir.h b/fs/cifs/cached_dir.h index bd262dc8b179a42ca575b80434cd40863e8b3f5e..e536304ca2ce4bc6a52ede0b8f78b3ab0efc1f03 100644 --- a/fs/cifs/cached_dir.h +++ b/fs/cifs/cached_dir.h @@ -31,13 +31,17 @@ struct cached_dirents { }; struct cached_fid { - bool is_valid:1; /* Do we have a useable root fid */ - bool file_all_info_is_valid:1; + struct list_head entry; + struct cached_fids *cfids; + const char *path; bool has_lease:1; + bool is_open:1; + bool on_list:1; + bool file_all_info_is_valid:1; unsigned long time; /* jiffies of when lease was taken */ struct kref refcount; struct cifs_fid fid; - struct mutex fid_mutex; + spinlock_t fid_lock; struct cifs_tcon *tcon; struct dentry *dentry; struct work_struct lease_break; @@ -45,8 +49,18 @@ struct cached_fid { struct cached_dirents dirents; }; -extern struct cached_fid *init_cached_dir(void); -extern void free_cached_dir(struct cifs_tcon *tcon); +#define MAX_CACHED_FIDS 16 +struct cached_fids { + /* Must be held when: + * - accessing the cfids->entries list + */ + spinlock_t cfid_list_lock; + int num_entries; + struct list_head entries; +}; + +extern struct cached_fids *init_cached_dirs(void); +extern void free_cached_dirs(struct cached_fids *cfids); extern int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, const char *path, struct cifs_sb_info *cifs_sb, @@ -55,8 +69,6 @@ extern int open_cached_dir_by_dentry(struct cifs_tcon *tcon, struct dentry *dentry, struct cached_fid **cfid); extern void close_cached_dir(struct cached_fid *cfid); -extern void close_cached_dir_lease(struct cached_fid *cfid); -extern void close_cached_dir_lease_locked(struct cached_fid *cfid); extern void close_all_cached_dirs(struct cifs_sb_info *cifs_sb); extern void invalidate_all_cached_dirs(struct cifs_tcon *tcon); extern int cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16]); diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c index c05477e28cffa698868655e24142aaf0e52f83a6..90850da390aebe4858df736b7f61b4a664e4d0ff 100644 --- a/fs/cifs/cifs_debug.c +++ b/fs/cifs/cifs_debug.c @@ -87,7 +87,7 @@ static void cifs_debug_tcon(struct seq_file *m, struct cifs_tcon *tcon) { __u32 dev_type = le32_to_cpu(tcon->fsDevInfo.DeviceType); - seq_printf(m, "%s Mounts: %d ", tcon->treeName, tcon->tc_count); + seq_printf(m, "%s Mounts: %d ", tcon->tree_name, tcon->tc_count); if (tcon->nativeFileSystem) seq_printf(m, "Type: %s ", tcon->nativeFileSystem); seq_printf(m, "DevInfo: 0x%x Attributes: 0x%x\n\tPathComponentMax: %d Status: %d", @@ -601,7 +601,7 @@ static int cifs_stats_proc_show(struct seq_file *m, void *v) list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { i++; - seq_printf(m, "\n%d) %s", i, tcon->treeName); + seq_printf(m, "\n%d) %s", i, tcon->tree_name); if (tcon->need_reconnect) seq_puts(m, "\tDISCONNECTED "); seq_printf(m, "\nSMBs: %d", diff --git a/fs/cifs/cifs_debug.h b/fs/cifs/cifs_debug.h index ee4ea2b60c0fba8930b9edf62377ea098437b2fa..d44808263cfba74fb8aa299f9682e902e19c6191 100644 --- a/fs/cifs/cifs_debug.h +++ b/fs/cifs/cifs_debug.h @@ -108,8 +108,8 @@ do { \ #define cifs_tcon_dbg_func(ratefunc, type, fmt, ...) \ do { \ const char *tn = ""; \ - if (tcon && tcon->treeName) \ - tn = tcon->treeName; \ + if (tcon && tcon->tree_name) \ + tn = tcon->tree_name; \ if ((type) & FYI && cifsFYI & CIFS_INFO) { \ pr_debug_ ## ratefunc("%s: %s " fmt, \ __FILE__, tn, ##__VA_ARGS__); \ @@ -150,7 +150,7 @@ do { \ #define cifs_tcon_dbg(type, fmt, ...) \ do { \ if (0) \ - pr_debug("%s " fmt, tcon->treeName, ##__VA_ARGS__); \ + pr_debug("%s " fmt, tcon->tree_name, ##__VA_ARGS__); \ } while (0) #define cifs_info(fmt, ...) \ diff --git a/fs/cifs/cifs_ioctl.h b/fs/cifs/cifs_ioctl.h index b87cbbe6d2d4bfdf1a2c932b5ff1adcfb4985678..d86d78d5bfdc1fea4d12b521cc6e37a3428b9009 100644 --- a/fs/cifs/cifs_ioctl.h +++ b/fs/cifs/cifs_ioctl.h @@ -91,6 +91,13 @@ struct smb3_notify { bool watch_tree; } __packed; +struct smb3_notify_info { + __u32 completion_filter; + bool watch_tree; + __u32 data_len; /* size of notify data below */ + __u8 notify_data[]; +} __packed; + #define CIFS_IOCTL_MAGIC 0xCF #define CIFS_IOC_COPYCHUNK_FILE _IOW(CIFS_IOCTL_MAGIC, 3, int) #define CIFS_IOC_SET_INTEGRITY _IO(CIFS_IOCTL_MAGIC, 4) @@ -100,6 +107,7 @@ struct smb3_notify { #define CIFS_DUMP_KEY _IOWR(CIFS_IOCTL_MAGIC, 8, struct smb3_key_debug_info) #define CIFS_IOC_NOTIFY _IOW(CIFS_IOCTL_MAGIC, 9, struct smb3_notify) #define CIFS_DUMP_FULL_KEY _IOWR(CIFS_IOCTL_MAGIC, 10, struct smb3_full_key_debug_info) +#define CIFS_IOC_NOTIFY_INFO _IOWR(CIFS_IOCTL_MAGIC, 11, struct smb3_notify_info) #define CIFS_IOC_SHUTDOWN _IOR ('X', 125, __u32) /* diff --git a/fs/cifs/cifs_swn.c b/fs/cifs/cifs_swn.c index 1e4c7cc5287f0317933fbf5ff49d5eb7864e7029..7233c6a7e6d70b03336737d8acc5bb302d3067d2 100644 --- a/fs/cifs/cifs_swn.c +++ b/fs/cifs/cifs_swn.c @@ -256,23 +256,23 @@ static struct cifs_swn_reg *cifs_find_swn_reg(struct cifs_tcon *tcon) const char *share_name; const char *net_name; - net_name = extract_hostname(tcon->treeName); + net_name = extract_hostname(tcon->tree_name); if (IS_ERR(net_name)) { int ret; ret = PTR_ERR(net_name); cifs_dbg(VFS, "%s: failed to extract host name from target '%s': %d\n", - __func__, tcon->treeName, ret); + __func__, tcon->tree_name, ret); return ERR_PTR(-EINVAL); } - share_name = extract_sharename(tcon->treeName); + share_name = extract_sharename(tcon->tree_name); if (IS_ERR(share_name)) { int ret; ret = PTR_ERR(share_name); cifs_dbg(VFS, "%s: failed to extract share name from target '%s': %d\n", - __func__, tcon->treeName, ret); + __func__, tcon->tree_name, ret); kfree(net_name); return ERR_PTR(-EINVAL); } @@ -335,14 +335,14 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon) goto fail; } - reg->net_name = extract_hostname(tcon->treeName); + reg->net_name = extract_hostname(tcon->tree_name); if (IS_ERR(reg->net_name)) { ret = PTR_ERR(reg->net_name); cifs_dbg(VFS, "%s: failed to extract host name from target: %d\n", __func__, ret); goto fail_idr; } - reg->share_name = extract_sharename(tcon->treeName); + reg->share_name = extract_sharename(tcon->tree_name); if (IS_ERR(reg->share_name)) { ret = PTR_ERR(reg->share_name); cifs_dbg(VFS, "%s: failed to extract share name from target: %d\n", __func__, ret); diff --git a/fs/cifs/cifsencrypt.c b/fs/cifs/cifsencrypt.c index 8f7835ccbca16a880c0b1083cf5fb7bd14497765..5db73c0f792a5565f6fdca2c445d3d210a76a4de 100644 --- a/fs/cifs/cifsencrypt.c +++ b/fs/cifs/cifsencrypt.c @@ -32,10 +32,9 @@ int __cifs_calc_signature(struct smb_rqst *rqst, int rc; struct kvec *iov = rqst->rq_iov; int n_vec = rqst->rq_nvec; - int is_smb2 = server->vals->header_preamble_size == 0; /* iov[0] is actual data and not the rfc1002 length for SMB2+ */ - if (is_smb2) { + if (!is_smb1(server)) { if (iov[0].iov_len <= 4) return -EIO; i = 0; @@ -104,26 +103,24 @@ static int cifs_calc_signature(struct smb_rqst *rqst, if (!rqst->rq_iov || !signature || !server) return -EINVAL; - rc = cifs_alloc_hash("md5", &server->secmech.md5, - &server->secmech.sdescmd5); + rc = cifs_alloc_hash("md5", &server->secmech.md5); if (rc) return -1; - rc = crypto_shash_init(&server->secmech.sdescmd5->shash); + rc = crypto_shash_init(server->secmech.md5); if (rc) { cifs_dbg(VFS, "%s: Could not init md5\n", __func__); return rc; } - rc = crypto_shash_update(&server->secmech.sdescmd5->shash, + rc = crypto_shash_update(server->secmech.md5, server->session_key.response, server->session_key.len); if (rc) { cifs_dbg(VFS, "%s: Could not update with response\n", __func__); return rc; } - return __cifs_calc_signature(rqst, server, signature, - &server->secmech.sdescmd5->shash); + return __cifs_calc_signature(rqst, server, signature, server->secmech.md5); } /* must be called with server->srv_mutex held */ @@ -413,7 +410,7 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, wchar_t *domain; wchar_t *server; - if (!ses->server->secmech.sdeschmacmd5) { + if (!ses->server->secmech.hmacmd5) { cifs_dbg(VFS, "%s: can't generate ntlmv2 hash\n", __func__); return -1; } @@ -421,14 +418,14 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, /* calculate md4 hash of password */ E_md4hash(ses->password, nt_hash, nls_cp); - rc = crypto_shash_setkey(ses->server->secmech.hmacmd5, nt_hash, + rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm, nt_hash, CIFS_NTHASH_SIZE); if (rc) { cifs_dbg(VFS, "%s: Could not set NT Hash as a key\n", __func__); return rc; } - rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash); + rc = crypto_shash_init(ses->server->secmech.hmacmd5); if (rc) { cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__); return rc; @@ -449,7 +446,7 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, memset(user, '\0', 2); } - rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_update(ses->server->secmech.hmacmd5, (char *)user, 2 * len); kfree(user); if (rc) { @@ -469,7 +466,7 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, len = cifs_strtoUTF16((__le16 *)domain, ses->domainName, len, nls_cp); rc = - crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, + crypto_shash_update(ses->server->secmech.hmacmd5, (char *)domain, 2 * len); kfree(domain); if (rc) { @@ -489,7 +486,7 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, len = cifs_strtoUTF16((__le16 *)server, ses->ip_addr, len, nls_cp); rc = - crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, + crypto_shash_update(ses->server->secmech.hmacmd5, (char *)server, 2 * len); kfree(server); if (rc) { @@ -499,7 +496,7 @@ static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, } } - rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_final(ses->server->secmech.hmacmd5, ntlmv2_hash); if (rc) cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__); @@ -519,12 +516,12 @@ CalcNTLMv2_response(const struct cifs_ses *ses, char *ntlmv2_hash) hash_len = ses->auth_key.len - (CIFS_SESS_KEY_SIZE + offsetof(struct ntlmv2_resp, challenge.key[0])); - if (!ses->server->secmech.sdeschmacmd5) { + if (!ses->server->secmech.hmacmd5) { cifs_dbg(VFS, "%s: can't generate ntlmv2 hash\n", __func__); return -1; } - rc = crypto_shash_setkey(ses->server->secmech.hmacmd5, + rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm, ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE); if (rc) { cifs_dbg(VFS, "%s: Could not set NTLMV2 Hash as a key\n", @@ -532,7 +529,7 @@ CalcNTLMv2_response(const struct cifs_ses *ses, char *ntlmv2_hash) return rc; } - rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash); + rc = crypto_shash_init(ses->server->secmech.hmacmd5); if (rc) { cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__); return rc; @@ -544,7 +541,7 @@ CalcNTLMv2_response(const struct cifs_ses *ses, char *ntlmv2_hash) else memcpy(ntlmv2->challenge.key, ses->server->cryptkey, CIFS_SERVER_CHALLENGE_SIZE); - rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_update(ses->server->secmech.hmacmd5, ntlmv2->challenge.key, hash_len); if (rc) { cifs_dbg(VFS, "%s: Could not update with response\n", __func__); @@ -552,7 +549,7 @@ CalcNTLMv2_response(const struct cifs_ses *ses, char *ntlmv2_hash) } /* Note that the MD5 digest over writes anon.challenge_key.key */ - rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_final(ses->server->secmech.hmacmd5, ntlmv2->ntlmv2_hash); if (rc) cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__); @@ -628,9 +625,7 @@ setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) cifs_server_lock(ses->server); - rc = cifs_alloc_hash("hmac(md5)", - &ses->server->secmech.hmacmd5, - &ses->server->secmech.sdeschmacmd5); + rc = cifs_alloc_hash("hmac(md5)", &ses->server->secmech.hmacmd5); if (rc) { goto unlock; } @@ -650,7 +645,7 @@ setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) } /* now calculate the session key for NTLMv2 */ - rc = crypto_shash_setkey(ses->server->secmech.hmacmd5, + rc = crypto_shash_setkey(ses->server->secmech.hmacmd5->tfm, ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE); if (rc) { cifs_dbg(VFS, "%s: Could not set NTLMV2 Hash as a key\n", @@ -658,13 +653,13 @@ setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) goto unlock; } - rc = crypto_shash_init(&ses->server->secmech.sdeschmacmd5->shash); + rc = crypto_shash_init(ses->server->secmech.hmacmd5); if (rc) { cifs_dbg(VFS, "%s: Could not init hmacmd5\n", __func__); goto unlock; } - rc = crypto_shash_update(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_update(ses->server->secmech.hmacmd5, ntlmv2->ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE); if (rc) { @@ -672,7 +667,7 @@ setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) goto unlock; } - rc = crypto_shash_final(&ses->server->secmech.sdeschmacmd5->shash, + rc = crypto_shash_final(ses->server->secmech.hmacmd5, ses->auth_key.response); if (rc) cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__); @@ -680,7 +675,7 @@ setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) unlock: cifs_server_unlock(ses->server); setup_ntlmv2_rsp_ret: - kfree(tiblob); + kfree_sensitive(tiblob); return rc; } @@ -719,49 +714,19 @@ calc_seckey(struct cifs_ses *ses) void cifs_crypto_secmech_release(struct TCP_Server_Info *server) { - if (server->secmech.cmacaes) { - crypto_free_shash(server->secmech.cmacaes); - server->secmech.cmacaes = NULL; - } - - if (server->secmech.hmacsha256) { - crypto_free_shash(server->secmech.hmacsha256); - server->secmech.hmacsha256 = NULL; - } - - if (server->secmech.md5) { - crypto_free_shash(server->secmech.md5); - server->secmech.md5 = NULL; - } + cifs_free_hash(&server->secmech.aes_cmac); + cifs_free_hash(&server->secmech.hmacsha256); + cifs_free_hash(&server->secmech.md5); + cifs_free_hash(&server->secmech.sha512); + cifs_free_hash(&server->secmech.hmacmd5); - if (server->secmech.sha512) { - crypto_free_shash(server->secmech.sha512); - server->secmech.sha512 = NULL; + if (server->secmech.enc) { + crypto_free_aead(server->secmech.enc); + server->secmech.enc = NULL; } - if (server->secmech.hmacmd5) { - crypto_free_shash(server->secmech.hmacmd5); - server->secmech.hmacmd5 = NULL; + if (server->secmech.dec) { + crypto_free_aead(server->secmech.dec); + server->secmech.dec = NULL; } - - if (server->secmech.ccmaesencrypt) { - crypto_free_aead(server->secmech.ccmaesencrypt); - server->secmech.ccmaesencrypt = NULL; - } - - if (server->secmech.ccmaesdecrypt) { - crypto_free_aead(server->secmech.ccmaesdecrypt); - server->secmech.ccmaesdecrypt = NULL; - } - - kfree(server->secmech.sdesccmacaes); - server->secmech.sdesccmacaes = NULL; - kfree(server->secmech.sdeschmacsha256); - server->secmech.sdeschmacsha256 = NULL; - kfree(server->secmech.sdeschmacmd5); - server->secmech.sdeschmacmd5 = NULL; - kfree(server->secmech.sdescmd5); - server->secmech.sdescmd5 = NULL; - kfree(server->secmech.sdescsha512); - server->secmech.sdescsha512 = NULL; } diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index f54d8bf2732a50d361b057beca89434440db4c6c..c6ac19223ddc0d37c2eb6cc7b821b333ca17292a 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -396,6 +396,7 @@ cifs_alloc_inode(struct super_block *sb) cifs_inode->epoch = 0; spin_lock_init(&cifs_inode->open_file_lock); generate_random_uuid(cifs_inode->lease_key); + cifs_inode->symlink_target = NULL; /* * Can not set i_flags here - they get immediately overwritten to zero @@ -412,7 +413,11 @@ cifs_alloc_inode(struct super_block *sb) static void cifs_free_inode(struct inode *inode) { - kmem_cache_free(cifs_inode_cachep, CIFS_I(inode)); + struct cifsInodeInfo *cinode = CIFS_I(inode); + + if (S_ISLNK(inode->i_mode)) + kfree(cinode->symlink_target); + kmem_cache_free(cifs_inode_cachep, cinode); } static void @@ -1139,7 +1144,7 @@ const struct inode_operations cifs_file_inode_ops = { }; const struct inode_operations cifs_symlink_inode_ops = { - .get_link = cifs_get_link, + .get_link = simple_get_link, .permission = cifs_permission, .listxattr = cifs_listxattr, }; @@ -1248,6 +1253,12 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, lock_two_nondirectories(target_inode, src_inode); cifs_dbg(FYI, "about to flush pages\n"); + + rc = filemap_write_and_wait_range(src_inode->i_mapping, off, + off + len - 1); + if (rc) + goto out; + /* should we flush first and last page first */ truncate_inode_pages(&target_inode->i_data, 0); diff --git a/fs/cifs/cifsfs.h b/fs/cifs/cifsfs.h index 81f4c15936d050273b9f1ef0a49aa72e86efaf2f..5b4a7a32bdc58c65d71e514dfb0fde0ea2dbac55 100644 --- a/fs/cifs/cifsfs.h +++ b/fs/cifs/cifsfs.h @@ -153,6 +153,6 @@ extern const struct export_operations cifs_export_ops; #endif /* CONFIG_CIFS_NFSD_EXPORT */ /* when changing internal version - update following two lines at same time */ -#define SMB3_PRODUCT_BUILD 38 -#define CIFS_VERSION "2.38" +#define SMB3_PRODUCT_BUILD 39 +#define CIFS_VERSION "2.39" #endif /* _CIFSFS_H */ diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h index f15d7b0c123d7c63166b944266389e735eb57d49..1420acf987f03d3ff1ab63a16b15e260fe87f37f 100644 --- a/fs/cifs/cifsglob.h +++ b/fs/cifs/cifsglob.h @@ -153,26 +153,16 @@ struct session_key { char *response; }; -/* crypto security descriptor definition */ -struct sdesc { - struct shash_desc shash; - char ctx[]; -}; - /* crypto hashing related structure/fields, not specific to a sec mech */ struct cifs_secmech { - struct crypto_shash *hmacmd5; /* hmac-md5 hash function */ - struct crypto_shash *md5; /* md5 hash function */ - struct crypto_shash *hmacsha256; /* hmac-sha256 hash function */ - struct crypto_shash *cmacaes; /* block-cipher based MAC function */ - struct crypto_shash *sha512; /* sha512 hash function */ - struct sdesc *sdeschmacmd5; /* ctxt to generate ntlmv2 hash, CR1 */ - struct sdesc *sdescmd5; /* ctxt to generate cifs/smb signature */ - struct sdesc *sdeschmacsha256; /* ctxt to generate smb2 signature */ - struct sdesc *sdesccmacaes; /* ctxt to generate smb3 signature */ - struct sdesc *sdescsha512; /* ctxt to generate smb3.11 signing key */ - struct crypto_aead *ccmaesencrypt; /* smb3 encryption aead */ - struct crypto_aead *ccmaesdecrypt; /* smb3 decryption aead */ + struct shash_desc *hmacmd5; /* hmacmd5 hash function, for NTLMv2/CR1 hashes */ + struct shash_desc *md5; /* md5 hash function, for CIFS/SMB1 signatures */ + struct shash_desc *hmacsha256; /* hmac-sha256 hash function, for SMB2 signatures */ + struct shash_desc *sha512; /* sha512 hash function, for SMB3.1.1 preauth hash */ + struct shash_desc *aes_cmac; /* block-cipher based MAC function, for SMB3 signatures */ + + struct crypto_aead *enc; /* smb3 encryption AEAD TFM (AES-CCM and AES-GCM) */ + struct crypto_aead *dec; /* smb3 decryption AEAD TFM (AES-CCM and AES-GCM) */ }; /* per smb session structure/fields */ @@ -195,6 +185,19 @@ struct cifs_cred { struct cifs_ace *aces; }; +struct cifs_open_info_data { + char *symlink_target; + union { + struct smb2_file_all_info fi; + struct smb311_posix_qinfo posix_fi; + }; +}; + +static inline void cifs_free_open_info(struct cifs_open_info_data *data) +{ + kfree(data->symlink_target); +} + /* ***************************************************************** * Except the CIFS PDUs themselves all the @@ -317,20 +320,20 @@ struct smb_version_operations { int (*is_path_accessible)(const unsigned int, struct cifs_tcon *, struct cifs_sb_info *, const char *); /* query path data from the server */ - int (*query_path_info)(const unsigned int, struct cifs_tcon *, - struct cifs_sb_info *, const char *, - FILE_ALL_INFO *, bool *, bool *); + int (*query_path_info)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse); /* query file data from the server */ - int (*query_file_info)(const unsigned int, struct cifs_tcon *, - struct cifs_fid *, FILE_ALL_INFO *); + int (*query_file_info)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data); /* query reparse tag from srv to determine which type of special file */ int (*query_reparse_tag)(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const char *path, __u32 *reparse_tag); /* get server index number */ - int (*get_srv_inum)(const unsigned int, struct cifs_tcon *, - struct cifs_sb_info *, const char *, - u64 *uniqueid, FILE_ALL_INFO *); + int (*get_srv_inum)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, u64 *uniqueid, + struct cifs_open_info_data *data); /* set size by path */ int (*set_path_size)(const unsigned int, struct cifs_tcon *, const char *, __u64, struct cifs_sb_info *, bool); @@ -379,8 +382,8 @@ struct smb_version_operations { struct cifs_sb_info *, const char *, char **, bool); /* open a file for non-posix mounts */ - int (*open)(const unsigned int, struct cifs_open_parms *, - __u32 *, FILE_ALL_INFO *); + int (*open)(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, + void *buf); /* set fid protocol-specific info */ void (*set_fid)(struct cifsFileInfo *, struct cifs_fid *, __u32); /* close a file */ @@ -451,7 +454,7 @@ struct smb_version_operations { int (*enum_snapshots)(const unsigned int xid, struct cifs_tcon *tcon, struct cifsFileInfo *src_file, void __user *); int (*notify)(const unsigned int xid, struct file *pfile, - void __user *pbuf); + void __user *pbuf, bool return_changes); int (*query_mf_symlink)(unsigned int, struct cifs_tcon *, struct cifs_sb_info *, const unsigned char *, char *, unsigned int *); @@ -557,6 +560,8 @@ struct smb_version_values { #define HEADER_SIZE(server) (server->vals->header_size) #define MAX_HEADER_SIZE(server) (server->vals->max_header_size) +#define HEADER_PREAMBLE_SIZE(server) (server->vals->header_preamble_size) +#define MID_HEADER_SIZE(server) (HEADER_SIZE(server) - 1 - HEADER_PREAMBLE_SIZE(server)) /** * CIFS superblock mount flags (mnt_cifs_flags) to consider when @@ -750,6 +755,11 @@ struct TCP_Server_Info { #endif }; +static inline bool is_smb1(struct TCP_Server_Info *server) +{ + return HEADER_PREAMBLE_SIZE(server) != 0; +} + static inline void cifs_server_lock(struct TCP_Server_Info *server) { unsigned int nofs_flag = memalloc_nofs_save(); @@ -1126,6 +1136,7 @@ struct cifs_fattr { struct timespec64 cf_mtime; struct timespec64 cf_ctime; u32 cf_cifstag; + char *cf_symlink_target; }; /* @@ -1142,7 +1153,7 @@ struct cifs_tcon { struct list_head openFileList; spinlock_t open_file_lock; /* protects list above */ struct cifs_ses *ses; /* pointer to session associated with */ - char treeName[MAX_TREE_SIZE + 1]; /* UNC name of resource in ASCII */ + char tree_name[MAX_TREE_SIZE + 1]; /* UNC name of resource in ASCII */ char *nativeFileSystem; char *password; /* for share-level security */ __u32 tid; /* The 4 byte tree id */ @@ -1221,7 +1232,7 @@ struct cifs_tcon { struct fscache_volume *fscache; /* cookie for share */ #endif struct list_head pending_opens; /* list of incomplete opens */ - struct cached_fid *cfid; /* Cached root fid */ + struct cached_fids *cfids; /* BB add field for back pointer to sb struct(s)? */ #ifdef CONFIG_CIFS_DFS_UPCALL struct list_head ulist; /* cache update list */ @@ -1388,6 +1399,7 @@ struct cifsFileInfo { struct work_struct put; /* work for the final part of _put */ struct delayed_work deferred; bool deferred_close_scheduled; /* Flag to indicate close is scheduled */ + char *symlink_target; }; struct cifs_io_parms { @@ -1546,6 +1558,7 @@ struct cifsInodeInfo { struct list_head deferred_closes; /* list of deferred closes */ spinlock_t deferred_lock; /* protection on deferred list */ bool lease_granted; /* Flag to indicate whether lease or oplock is granted. */ + char *symlink_target; }; static inline struct cifsInodeInfo * @@ -2114,4 +2127,14 @@ static inline size_t ntlmssp_workstation_name_size(const struct cifs_ses *ses) return sizeof(ses->workstation_name); } +static inline void move_cifs_info_to_smb2(struct smb2_file_all_info *dst, const FILE_ALL_INFO *src) +{ + memcpy(dst, src, (size_t)((u8 *)&src->AccessFlags - (u8 *)src)); + dst->AccessFlags = src->AccessFlags; + dst->CurrentByteOffset = src->CurrentByteOffset; + dst->Mode = src->Mode; + dst->AlignmentRequirement = src->AlignmentRequirement; + dst->FileNameLength = src->FileNameLength; +} + #endif /* _CIFS_GLOB_H */ diff --git a/fs/cifs/cifspdu.h b/fs/cifs/cifspdu.h index aeba371c4c707b5164ce1fdbf7657eee3c8444e9..d1abaeea974a952e8a21b5807df08baa4862d3c6 100644 --- a/fs/cifs/cifspdu.h +++ b/fs/cifs/cifspdu.h @@ -483,7 +483,7 @@ put_bcc(__u16 count, struct smb_hdr *hdr) typedef struct negotiate_req { struct smb_hdr hdr; /* wct = 0 */ __le16 ByteCount; - unsigned char DialectsArray[1]; + unsigned char DialectsArray[]; } __attribute__((packed)) NEGOTIATE_REQ; #define MIN_TZ_ADJ (15 * 60) /* minimum grid for timezones in seconds */ @@ -508,13 +508,14 @@ typedef struct negotiate_rsp { __u8 EncryptionKeyLength; __u16 ByteCount; union { - unsigned char EncryptionKey[1]; /* cap extended security off */ + /* cap extended security off */ + DECLARE_FLEX_ARRAY(unsigned char, EncryptionKey); /* followed by Domain name - if extended security is off */ /* followed by 16 bytes of server GUID */ /* then security blob if cap_extended_security negotiated */ struct { unsigned char GUID[SMB1_CLIENT_GUID_SIZE]; - unsigned char SecurityBlob[1]; + unsigned char SecurityBlob[]; } __attribute__((packed)) extended_response; } __attribute__((packed)) u; } __attribute__((packed)) NEGOTIATE_RSP; diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h index 3bc94bcc7177eb7471b35606e4961bfc554aa001..83e83d8beabba3aff6813c8c6860a26d0fcde061 100644 --- a/fs/cifs/cifsproto.h +++ b/fs/cifs/cifsproto.h @@ -182,10 +182,9 @@ extern int cifs_unlock_range(struct cifsFileInfo *cfile, extern int cifs_push_mandatory_locks(struct cifsFileInfo *cfile); extern void cifs_down_write(struct rw_semaphore *sem); -extern struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, - struct file *file, - struct tcon_link *tlink, - __u32 oplock); +struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, + struct tcon_link *tlink, __u32 oplock, + const char *symlink_target); extern int cifs_posix_open(const char *full_path, struct inode **inode, struct super_block *sb, int mode, unsigned int f_flags, __u32 *oplock, __u16 *netfid, @@ -200,9 +199,9 @@ extern int cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr); extern struct inode *cifs_iget(struct super_block *sb, struct cifs_fattr *fattr); -extern int cifs_get_inode_info(struct inode **inode, const char *full_path, - FILE_ALL_INFO *data, struct super_block *sb, - int xid, const struct cifs_fid *fid); +int cifs_get_inode_info(struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, struct super_block *sb, int xid, + const struct cifs_fid *fid); extern int smb311_posix_get_inode_info(struct inode **pinode, const char *search_path, struct super_block *sb, unsigned int xid); extern int cifs_get_inode_info_unix(struct inode **pinode, @@ -598,9 +597,8 @@ struct cifs_aio_ctx *cifs_aio_ctx_alloc(void); void cifs_aio_ctx_release(struct kref *refcount); int setup_aio_ctx_iter(struct cifs_aio_ctx *ctx, struct iov_iter *iter, int rw); -int cifs_alloc_hash(const char *name, struct crypto_shash **shash, - struct sdesc **sdesc); -void cifs_free_hash(struct crypto_shash **shash, struct sdesc **sdesc); +int cifs_alloc_hash(const char *name, struct shash_desc **sdesc); +void cifs_free_hash(struct shash_desc **sdesc); extern void rqst_page_get_length(struct smb_rqst *rqst, unsigned int page, unsigned int *len, unsigned int *offset); @@ -639,7 +637,7 @@ cifs_chan_is_iface_active(struct cifs_ses *ses, int cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server); int -SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon); +SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_mount); void extract_unc_hostname(const char *unc, const char **h, size_t *len); int copy_path_name(char *dst, const char *src); diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c index 7aa91e2720274a14d97b1c7dfba2010b4d400e60..1724066c15365edce643285d96e875c8ee154273 100644 --- a/fs/cifs/cifssmb.c +++ b/fs/cifs/cifssmb.c @@ -465,7 +465,7 @@ CIFSSMBNegotiate(const unsigned int xid, for (i = 0; i < CIFS_NUM_PROT; i++) { size_t len = strlen(protocols[i].name) + 1; - memcpy(pSMB->DialectsArray+count, protocols[i].name, len); + memcpy(&pSMB->DialectsArray[count], protocols[i].name, len); count += len; } inc_rfc1001_len(pSMB, count); @@ -2305,7 +2305,7 @@ int CIFSSMBRenameOpenFile(const unsigned int xid, struct cifs_tcon *pTcon, remap); } rename_info->target_name_len = cpu_to_le32(2 * len_of_str); - count = 12 /* sizeof(struct set_file_rename) */ + (2 * len_of_str); + count = sizeof(struct set_file_rename) + (2 * len_of_str); byte_count += count; pSMB->DataCount = cpu_to_le16(count); pSMB->TotalDataCount = pSMB->DataCount; diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index 3da5da9f16b0c7949c7ab0c24686bba151189637..ffb291579bb9d9d4000a26997c95c3f886ef1024 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -155,7 +155,7 @@ static void smb2_query_server_interfaces(struct work_struct *work) /* * query server network interfaces, in case they change */ - rc = SMB3_request_interfaces(0, tcon); + rc = SMB3_request_interfaces(0, tcon, false); if (rc) { cifs_dbg(FYI, "%s: failed to query server interfaces: %d\n", __func__, rc); @@ -311,7 +311,7 @@ cifs_abort_connection(struct TCP_Server_Info *server) } server->sequence_number = 0; server->session_estab = false; - kfree(server->session_key.response); + kfree_sensitive(server->session_key.response); server->session_key.response = NULL; server->session_key.len = 0; server->lstrp = jiffies; @@ -702,9 +702,6 @@ cifs_readv_from_socket(struct TCP_Server_Info *server, struct msghdr *smb_msg) int length = 0; int total_read; - smb_msg->msg_control = NULL; - smb_msg->msg_controllen = 0; - for (total_read = 0; msg_data_left(smb_msg); total_read += length) { try_to_freeze(); @@ -760,7 +757,7 @@ int cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, unsigned int to_read) { - struct msghdr smb_msg; + struct msghdr smb_msg = {}; struct kvec iov = {.iov_base = buf, .iov_len = to_read}; iov_iter_kvec(&smb_msg.msg_iter, READ, &iov, 1, to_read); @@ -770,15 +767,13 @@ cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, ssize_t cifs_discard_from_socket(struct TCP_Server_Info *server, size_t to_read) { - struct msghdr smb_msg; + struct msghdr smb_msg = {}; /* * iov_iter_discard already sets smb_msg.type and count and iov_offset * and cifs_readv_from_socket sets msg_control and msg_controllen * so little to initialize in struct msghdr */ - smb_msg.msg_name = NULL; - smb_msg.msg_namelen = 0; iov_iter_discard(&smb_msg.msg_iter, READ, to_read); return cifs_readv_from_socket(server, &smb_msg); @@ -788,7 +783,7 @@ int cifs_read_page_from_socket(struct TCP_Server_Info *server, struct page *page, unsigned int page_offset, unsigned int to_read) { - struct msghdr smb_msg; + struct msghdr smb_msg = {}; struct bio_vec bv = { .bv_page = page, .bv_len = to_read, .bv_offset = page_offset}; iov_iter_bvec(&smb_msg.msg_iter, READ, &bv, 1, to_read); @@ -871,7 +866,7 @@ smb2_get_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) /* * SMB1 does not use credits. */ - if (server->vals->header_preamble_size) + if (is_smb1(server)) return 0; return le16_to_cpu(shdr->CreditRequest); @@ -1050,7 +1045,7 @@ standard_receive3(struct TCP_Server_Info *server, struct mid_q_entry *mid) /* make sure this will fit in a large buffer */ if (pdu_length > CIFSMaxBufSize + MAX_HEADER_SIZE(server) - - server->vals->header_preamble_size) { + HEADER_PREAMBLE_SIZE(server)) { cifs_server_dbg(VFS, "SMB response too long (%u bytes)\n", pdu_length); cifs_reconnect(server, true); return -ECONNABORTED; @@ -1065,8 +1060,7 @@ standard_receive3(struct TCP_Server_Info *server, struct mid_q_entry *mid) /* now read the rest */ length = cifs_read_from_socket(server, buf + HEADER_SIZE(server) - 1, - pdu_length - HEADER_SIZE(server) + 1 - + server->vals->header_preamble_size); + pdu_length - MID_HEADER_SIZE(server)); if (length < 0) return length; @@ -1122,7 +1116,7 @@ smb2_add_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) /* * SMB1 does not use credits. */ - if (server->vals->header_preamble_size) + if (is_smb1(server)) return; if (shdr->CreditRequest) { @@ -1180,10 +1174,10 @@ cifs_demultiplex_thread(void *p) if (length < 0) continue; - if (server->vals->header_preamble_size == 0) - server->total_read = 0; - else + if (is_smb1(server)) server->total_read = length; + else + server->total_read = 0; /* * The right amount was read from socket - 4 bytes, @@ -1198,8 +1192,7 @@ next_pdu: server->pdu_size = pdu_length; /* make sure we have enough to get to the MID */ - if (server->pdu_size < HEADER_SIZE(server) - 1 - - server->vals->header_preamble_size) { + if (server->pdu_size < MID_HEADER_SIZE(server)) { cifs_server_dbg(VFS, "SMB response too short (%u bytes)\n", server->pdu_size); cifs_reconnect(server, true); @@ -1208,9 +1201,8 @@ next_pdu: /* read down to the MID */ length = cifs_read_from_socket(server, - buf + server->vals->header_preamble_size, - HEADER_SIZE(server) - 1 - - server->vals->header_preamble_size); + buf + HEADER_PREAMBLE_SIZE(server), + MID_HEADER_SIZE(server)); if (length < 0) continue; server->total_read += length; @@ -1588,7 +1580,7 @@ cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect) cifs_crypto_secmech_release(server); - kfree(server->session_key.response); + kfree_sensitive(server->session_key.response); server->session_key.response = NULL; server->session_key.len = 0; kfree(server->hostname); @@ -1948,7 +1940,8 @@ void cifs_put_smb_ses(struct cifs_ses *ses) spin_unlock(&ses->ses_lock); cifs_dbg(FYI, "%s: ses_count=%d\n", __func__, ses->ses_count); - cifs_dbg(FYI, "%s: ses ipc: %s\n", __func__, ses->tcon_ipc ? ses->tcon_ipc->treeName : "NONE"); + cifs_dbg(FYI, + "%s: ses ipc: %s\n", __func__, ses->tcon_ipc ? ses->tcon_ipc->tree_name : "NONE"); spin_lock(&cifs_tcp_ses_lock); if (--ses->ses_count > 0) { @@ -2301,7 +2294,7 @@ static int match_tcon(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) { if (tcon->status == TID_EXITING) return 0; - if (strncmp(tcon->treeName, ctx->UNC, MAX_TREE_SIZE)) + if (strncmp(tcon->tree_name, ctx->UNC, MAX_TREE_SIZE)) return 0; if (tcon->seal != ctx->seal) return 0; @@ -2353,7 +2346,9 @@ cifs_put_tcon(struct cifs_tcon *tcon) ses = tcon->ses; cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); if (--tcon->tc_count > 0) { + spin_unlock(&tcon->tc_lock); spin_unlock(&cifs_tcp_ses_lock); return; } @@ -2362,6 +2357,7 @@ cifs_put_tcon(struct cifs_tcon *tcon) WARN_ON(tcon->tc_count < 0); list_del_init(&tcon->tcon_list); + spin_unlock(&tcon->tc_lock); spin_unlock(&cifs_tcp_ses_lock); /* cancel polling of interfaces */ @@ -2836,9 +2832,12 @@ ip_rfc1001_connect(struct TCP_Server_Info *server) * sessinit is sent but no second negprot */ struct rfc1002_session_packet *ses_init_buf; + unsigned int req_noscope_len; struct smb_hdr *smb_buf; + ses_init_buf = kzalloc(sizeof(struct rfc1002_session_packet), GFP_KERNEL); + if (ses_init_buf) { ses_init_buf->trailer.session_req.called_len = 32; @@ -2874,8 +2873,12 @@ ip_rfc1001_connect(struct TCP_Server_Info *server) ses_init_buf->trailer.session_req.scope2 = 0; smb_buf = (struct smb_hdr *)ses_init_buf; - /* sizeof RFC1002_SESSION_REQUEST with no scope */ - smb_buf->smb_buf_length = cpu_to_be32(0x81000044); + /* sizeof RFC1002_SESSION_REQUEST with no scopes */ + req_noscope_len = sizeof(struct rfc1002_session_packet) - 2; + + /* == cpu_to_be32(0x81000044) */ + smb_buf->smb_buf_length = + cpu_to_be32((RFC1002_SESSION_REQUEST << 24) | req_noscope_len); rc = smb_send(server, smb_buf, 0x44); kfree(ses_init_buf); /* @@ -3926,12 +3929,11 @@ CIFSTCon(const unsigned int xid, struct cifs_ses *ses, pSMB->AndXCommand = 0xFF; pSMB->Flags = cpu_to_le16(TCON_EXTENDED_SECINFO); bcc_ptr = &pSMB->Password[0]; - if (tcon->pipe || (ses->server->sec_mode & SECMODE_USER)) { - pSMB->PasswordLength = cpu_to_le16(1); /* minimum */ - *bcc_ptr = 0; /* password is null byte */ - bcc_ptr++; /* skip password */ - /* already aligned so no need to do it below */ - } + + pSMB->PasswordLength = cpu_to_le16(1); /* minimum */ + *bcc_ptr = 0; /* password is null byte */ + bcc_ptr++; /* skip password */ + /* already aligned so no need to do it below */ if (ses->server->sign) smb_buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; @@ -3994,7 +3996,7 @@ CIFSTCon(const unsigned int xid, struct cifs_ses *ses, } bcc_ptr += length + 1; bytes_left -= (length + 1); - strscpy(tcon->treeName, tree, sizeof(tcon->treeName)); + strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); /* mostly informational -- no need to fail on error here */ kfree(tcon->nativeFileSystem); @@ -4139,7 +4141,7 @@ cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, if (ses->auth_key.response) { cifs_dbg(FYI, "Free previous auth_key.response = %p\n", ses->auth_key.response); - kfree(ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); ses->auth_key.response = NULL; ses->auth_key.len = 0; } @@ -4202,7 +4204,7 @@ cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) ctx->local_nls = cifs_sb->local_nls; ctx->linux_uid = fsuid; ctx->cred_uid = fsuid; - ctx->UNC = master_tcon->treeName; + ctx->UNC = master_tcon->tree_name; ctx->retry = master_tcon->retry; ctx->nocase = master_tcon->nocase; ctx->nohandlecache = master_tcon->nohandlecache; @@ -4668,7 +4670,7 @@ int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const stru /* If it is not dfs or there was no cached dfs referral, then reconnect to same share */ if (!server->current_fullpath || dfs_cache_noreq_find(server->current_fullpath + 1, &ref, &tl)) { - rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, cifs_sb->local_nls); + rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, tcon, cifs_sb->local_nls); goto out; } @@ -4712,7 +4714,7 @@ int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon, const stru tcon->status = TID_IN_TCON; spin_unlock(&tcon->tc_lock); - rc = ops->tree_connect(xid, tcon->ses, tcon->treeName, tcon, nlsc); + rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, tcon, nlsc); if (rc) { spin_lock(&tcon->tc_lock); if (tcon->status == TID_IN_TCON) diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c index a9b6c3eba6de505ab1786a10c8f4d5383f020a40..e70915ad754100364c62e509357f412d1b49f9e9 100644 --- a/fs/cifs/dfs_cache.c +++ b/fs/cifs/dfs_cache.c @@ -98,7 +98,7 @@ static struct cifs_ses *find_ipc_from_server_path(struct cifs_ses **ses, const c get_ipc_unc(path, unc, sizeof(unc)); for (; *ses; ses++) { - if (!strcasecmp(unc, (*ses)->tcon_ipc->treeName)) + if (!strcasecmp(unc, (*ses)->tcon_ipc->tree_name)) return *ses; } return ERR_PTR(-ENOENT); diff --git a/fs/cifs/dir.c b/fs/cifs/dir.c index 08f7392716e2f31dd1941a15e24be4c74d13c548..a5c73c2af3a264dca2515695a99ffa7c725d9314 100644 --- a/fs/cifs/dir.c +++ b/fs/cifs/dir.c @@ -50,7 +50,7 @@ cifs_build_path_to_root(struct smb3_fs_context *ctx, struct cifs_sb_info *cifs_s } if (add_treename) - dfsplen = strnlen(tcon->treeName, MAX_TREE_SIZE + 1); + dfsplen = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1); else dfsplen = 0; @@ -59,7 +59,7 @@ cifs_build_path_to_root(struct smb3_fs_context *ctx, struct cifs_sb_info *cifs_s return full_path; if (dfsplen) - memcpy(full_path, tcon->treeName, dfsplen); + memcpy(full_path, tcon->tree_name, dfsplen); full_path[dfsplen] = CIFS_DIR_SEP(cifs_sb); memcpy(full_path + dfsplen + 1, ctx->prepath, pplen); convert_delimiter(full_path, CIFS_DIR_SEP(cifs_sb)); @@ -93,7 +93,7 @@ build_path_from_dentry_optional_prefix(struct dentry *direntry, void *page, return ERR_PTR(-ENOMEM); if (prefix) - dfsplen = strnlen(tcon->treeName, MAX_TREE_SIZE + 1); + dfsplen = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1); else dfsplen = 0; @@ -123,7 +123,7 @@ build_path_from_dentry_optional_prefix(struct dentry *direntry, void *page, } if (dfsplen) { s -= dfsplen; - memcpy(s, tcon->treeName, dfsplen); + memcpy(s, tcon->tree_name, dfsplen); if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) { int i; for (i = 0; i < dfsplen; i++) { @@ -165,10 +165,9 @@ check_name(struct dentry *direntry, struct cifs_tcon *tcon) /* Inode operations in similar order to how they appear in Linux file fs.h */ -static int -cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid, - struct tcon_link *tlink, unsigned oflags, umode_t mode, - __u32 *oplock, struct cifs_fid *fid) +static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid, + struct tcon_link *tlink, unsigned int oflags, umode_t mode, __u32 *oplock, + struct cifs_fid *fid, struct cifs_open_info_data *buf) { int rc = -ENOENT; int create_options = CREATE_NOT_DIR; @@ -177,7 +176,6 @@ cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid, struct cifs_tcon *tcon = tlink_tcon(tlink); const char *full_path; void *page = alloc_dentry_path(); - FILE_ALL_INFO *buf = NULL; struct inode *newinode = NULL; int disposition; struct TCP_Server_Info *server = tcon->ses->server; @@ -290,12 +288,6 @@ cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int xid, goto out; } - buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); - if (buf == NULL) { - rc = -ENOMEM; - goto out; - } - /* * if we're not using unix extensions, see if we need to set * ATTR_READONLY on the create call @@ -364,8 +356,7 @@ cifs_create_get_file_info: { #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ /* TODO: Add support for calling POSIX query info here, but passing in fid */ - rc = cifs_get_inode_info(&newinode, full_path, buf, inode->i_sb, - xid, fid); + rc = cifs_get_inode_info(&newinode, full_path, buf, inode->i_sb, xid, fid); if (newinode) { if (server->ops->set_lease_key) server->ops->set_lease_key(newinode, fid); @@ -402,7 +393,6 @@ cifs_create_set_dentry: d_add(direntry, newinode); out: - kfree(buf); free_dentry_path(page); return rc; @@ -423,10 +413,11 @@ cifs_atomic_open(struct inode *inode, struct dentry *direntry, struct tcon_link *tlink; struct cifs_tcon *tcon; struct TCP_Server_Info *server; - struct cifs_fid fid; + struct cifs_fid fid = {}; struct cifs_pending_open open; __u32 oplock; struct cifsFileInfo *file_info; + struct cifs_open_info_data buf = {}; if (unlikely(cifs_forced_shutdown(CIFS_SB(inode->i_sb)))) return -EIO; @@ -484,8 +475,7 @@ cifs_atomic_open(struct inode *inode, struct dentry *direntry, cifs_add_pending_open(&fid, tlink, &open); rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode, - &oplock, &fid); - + &oplock, &fid, &buf); if (rc) { cifs_del_pending_open(&open); goto out; @@ -510,7 +500,7 @@ cifs_atomic_open(struct inode *inode, struct dentry *direntry, file->f_op = &cifs_file_direct_ops; } - file_info = cifs_new_fileinfo(&fid, file, tlink, oplock); + file_info = cifs_new_fileinfo(&fid, file, tlink, oplock, buf.symlink_target); if (file_info == NULL) { if (server->ops->close) server->ops->close(xid, tcon, &fid); @@ -526,6 +516,7 @@ out: cifs_put_tlink(tlink); out_free_xid: free_xid(xid); + cifs_free_open_info(&buf); return rc; } @@ -547,6 +538,7 @@ int cifs_create(struct user_namespace *mnt_userns, struct inode *inode, struct TCP_Server_Info *server; struct cifs_fid fid; __u32 oplock; + struct cifs_open_info_data buf = {}; cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n", inode, direntry, direntry); @@ -565,11 +557,11 @@ int cifs_create(struct user_namespace *mnt_userns, struct inode *inode, if (server->ops->new_lease_key) server->ops->new_lease_key(&fid); - rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode, - &oplock, &fid); + rc = cifs_do_create(inode, direntry, xid, tlink, oflags, mode, &oplock, &fid, &buf); if (!rc && server->ops->close) server->ops->close(xid, tcon, &fid); + cifs_free_open_info(&buf); cifs_put_tlink(tlink); out_free_xid: free_xid(xid); diff --git a/fs/cifs/file.c b/fs/cifs/file.c index fa738adc031f728d2cd553f0dcaababf2fc275ac..f6ffee514c345cef00fab4cbe930a0cb44676e65 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -209,16 +209,14 @@ posix_open_ret: } #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ -static int -cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *cifs_sb, - struct cifs_tcon *tcon, unsigned int f_flags, __u32 *oplock, - struct cifs_fid *fid, unsigned int xid) +static int cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, unsigned int f_flags, __u32 *oplock, + struct cifs_fid *fid, unsigned int xid, struct cifs_open_info_data *buf) { int rc; int desired_access; int disposition; int create_options = CREATE_NOT_DIR; - FILE_ALL_INFO *buf; struct TCP_Server_Info *server = tcon->ses->server; struct cifs_open_parms oparms; @@ -255,10 +253,6 @@ cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *ci /* BB pass O_SYNC flag through on file attributes .. BB */ - buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); - if (!buf) - return -ENOMEM; - /* O_SYNC also has bit for O_DSYNC so following check picks up either */ if (f_flags & O_SYNC) create_options |= CREATE_WRITE_THROUGH; @@ -276,9 +270,8 @@ cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *ci oparms.reconnect = false; rc = server->ops->open(xid, &oparms, oplock, buf); - if (rc) - goto out; + return rc; /* TODO: Add support for calling posix query info but with passing in fid */ if (tcon->unix_ext) @@ -294,8 +287,6 @@ cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *ci rc = -EOPENSTALE; } -out: - kfree(buf); return rc; } @@ -325,9 +316,9 @@ cifs_down_write(struct rw_semaphore *sem) static void cifsFileInfo_put_work(struct work_struct *work); -struct cifsFileInfo * -cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, - struct tcon_link *tlink, __u32 oplock) +struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, + struct tcon_link *tlink, __u32 oplock, + const char *symlink_target) { struct dentry *dentry = file_dentry(file); struct inode *inode = d_inode(dentry); @@ -347,6 +338,15 @@ cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, return NULL; } + if (symlink_target) { + cfile->symlink_target = kstrdup(symlink_target, GFP_KERNEL); + if (!cfile->symlink_target) { + kfree(fdlocks); + kfree(cfile); + return NULL; + } + } + INIT_LIST_HEAD(&fdlocks->locks); fdlocks->cfile = cfile; cfile->llist = fdlocks; @@ -440,6 +440,7 @@ static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) cifs_put_tlink(cifs_file->tlink); dput(cifs_file->dentry); cifs_sb_deactive(sb); + kfree(cifs_file->symlink_target); kfree(cifs_file); } @@ -488,7 +489,7 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, struct cifsInodeInfo *cifsi = CIFS_I(inode); struct super_block *sb = inode->i_sb; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); - struct cifs_fid fid; + struct cifs_fid fid = {}; struct cifs_pending_open open; bool oplock_break_cancelled; @@ -570,8 +571,9 @@ int cifs_open(struct inode *inode, struct file *file) void *page; const char *full_path; bool posix_open_ok = false; - struct cifs_fid fid; + struct cifs_fid fid = {}; struct cifs_pending_open open; + struct cifs_open_info_data data = {}; xid = get_xid(); @@ -662,15 +664,15 @@ int cifs_open(struct inode *inode, struct file *file) if (server->ops->get_lease_key) server->ops->get_lease_key(inode, &fid); - rc = cifs_nt_open(full_path, inode, cifs_sb, tcon, - file->f_flags, &oplock, &fid, xid); + rc = cifs_nt_open(full_path, inode, cifs_sb, tcon, file->f_flags, &oplock, &fid, + xid, &data); if (rc) { cifs_del_pending_open(&open); goto out; } } - cfile = cifs_new_fileinfo(&fid, file, tlink, oplock); + cfile = cifs_new_fileinfo(&fid, file, tlink, oplock, data.symlink_target); if (cfile == NULL) { if (server->ops->close) server->ops->close(xid, tcon, &fid); @@ -712,6 +714,7 @@ out: free_dentry_path(page); free_xid(xid); cifs_put_tlink(tlink); + cifs_free_open_info(&data); return rc; } @@ -3575,6 +3578,9 @@ static ssize_t __cifs_writev( ssize_t cifs_direct_writev(struct kiocb *iocb, struct iov_iter *from) { + struct file *file = iocb->ki_filp; + + cifs_revalidate_mapping(file->f_inode); return __cifs_writev(iocb, from, true); } @@ -4268,6 +4274,15 @@ static ssize_t __cifs_readv( len = ctx->len; } + if (direct) { + rc = filemap_write_and_wait_range(file->f_inode->i_mapping, + offset, offset + len - 1); + if (rc) { + kref_put(&ctx->refcount, cifs_aio_ctx_release); + return -EAGAIN; + } + } + /* grab a lock here due to read response handlers can access ctx */ mutex_lock(&ctx->aio_mutex); diff --git a/fs/cifs/fs_context.c b/fs/cifs/fs_context.c index 0e13dec86b252736047b5017437b00e56dfe2803..45119597c7655b60150011145b13e00f12e4f270 100644 --- a/fs/cifs/fs_context.c +++ b/fs/cifs/fs_context.c @@ -791,6 +791,13 @@ do { \ cifs_sb->ctx->field = NULL; \ } while (0) +#define STEAL_STRING_SENSITIVE(cifs_sb, ctx, field) \ +do { \ + kfree_sensitive(ctx->field); \ + ctx->field = cifs_sb->ctx->field; \ + cifs_sb->ctx->field = NULL; \ +} while (0) + static int smb3_reconfigure(struct fs_context *fc) { struct smb3_fs_context *ctx = smb3_fc2context(fc); @@ -811,7 +818,7 @@ static int smb3_reconfigure(struct fs_context *fc) STEAL_STRING(cifs_sb, ctx, UNC); STEAL_STRING(cifs_sb, ctx, source); STEAL_STRING(cifs_sb, ctx, username); - STEAL_STRING(cifs_sb, ctx, password); + STEAL_STRING_SENSITIVE(cifs_sb, ctx, password); STEAL_STRING(cifs_sb, ctx, domainname); STEAL_STRING(cifs_sb, ctx, nodename); STEAL_STRING(cifs_sb, ctx, iocharset); @@ -1162,7 +1169,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, } break; case Opt_pass: - kfree(ctx->password); + kfree_sensitive(ctx->password); ctx->password = NULL; if (strlen(param->string) == 0) break; @@ -1470,6 +1477,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, return 0; cifs_parse_mount_err: + kfree_sensitive(ctx->password); return -EINVAL; } diff --git a/fs/cifs/fscache.c b/fs/cifs/fscache.c index 23ef56f55ce509d149923ae218d1f6dabd8f1da1..a1751b95631845151b1e0b8663238eaadc97b56b 100644 --- a/fs/cifs/fscache.c +++ b/fs/cifs/fscache.c @@ -45,7 +45,7 @@ int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon) memset(&key, 0, sizeof(key)); - sharename = extract_sharename(tcon->treeName); + sharename = extract_sharename(tcon->tree_name); if (IS_ERR(sharename)) { cifs_dbg(FYI, "%s: couldn't extract sharename\n", __func__); return -EINVAL; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index bac08c20f559bccbba51f3ae960c9fedd92a1271..7cf96e581d2437aff53124e7ae6157ad1d5602e6 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -210,6 +210,17 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr) */ inode->i_blocks = (512 - 1 + fattr->cf_bytes) >> 9; } + + if (S_ISLNK(fattr->cf_mode)) { + kfree(cifs_i->symlink_target); + cifs_i->symlink_target = fattr->cf_symlink_target; + fattr->cf_symlink_target = NULL; + + if (unlikely(!cifs_i->symlink_target)) + inode->i_link = ERR_PTR(-EOPNOTSUPP); + else + inode->i_link = cifs_i->symlink_target; + } spin_unlock(&inode->i_lock); if (fattr->cf_flags & CIFS_FATTR_DFS_REFERRAL) @@ -347,13 +358,20 @@ cifs_get_file_info_unix(struct file *filp) int rc; unsigned int xid; FILE_UNIX_BASIC_INFO find_data; - struct cifs_fattr fattr; + struct cifs_fattr fattr = {}; struct inode *inode = file_inode(filp); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); struct cifsFileInfo *cfile = filp->private_data; struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); xid = get_xid(); + + if (cfile->symlink_target) { + fattr.cf_symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!fattr.cf_symlink_target) + return -ENOMEM; + } + rc = CIFSSMBUnixQFileInfo(xid, tcon, cfile->fid.netfid, &find_data); if (!rc) { cifs_unix_basic_to_fattr(&fattr, &find_data, cifs_sb); @@ -378,6 +396,7 @@ int cifs_get_inode_info_unix(struct inode **pinode, FILE_UNIX_BASIC_INFO find_data; struct cifs_fattr fattr; struct cifs_tcon *tcon; + struct TCP_Server_Info *server; struct tcon_link *tlink; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); @@ -387,10 +406,12 @@ int cifs_get_inode_info_unix(struct inode **pinode, if (IS_ERR(tlink)) return PTR_ERR(tlink); tcon = tlink_tcon(tlink); + server = tcon->ses->server; /* could have done a find first instead but this returns more info */ rc = CIFSSMBUnixQPathInfo(xid, tcon, full_path, &find_data, cifs_sb->local_nls, cifs_remap(cifs_sb)); + cifs_dbg(FYI, "%s: query path info: rc = %d\n", __func__, rc); cifs_put_tlink(tlink); if (!rc) { @@ -410,6 +431,17 @@ int cifs_get_inode_info_unix(struct inode **pinode, cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); } + if (S_ISLNK(fattr.cf_mode) && !fattr.cf_symlink_target) { + if (!server->ops->query_symlink) + return -EOPNOTSUPP; + rc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path, + &fattr.cf_symlink_target, false); + if (rc) { + cifs_dbg(FYI, "%s: query_symlink: %d\n", __func__, rc); + goto cgiiu_exit; + } + } + if (*pinode == NULL) { /* get new inode */ cifs_fill_uniqueid(sb, &fattr); @@ -432,6 +464,7 @@ int cifs_get_inode_info_unix(struct inode **pinode, } cgiiu_exit: + kfree(fattr.cf_symlink_target); return rc; } #else @@ -601,10 +634,10 @@ static int cifs_sfu_mode(struct cifs_fattr *fattr, const unsigned char *path, } /* Fill a cifs_fattr struct with info from POSIX info struct */ -static void -smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct smb311_posix_qinfo *info, - struct super_block *sb, bool adjust_tz, bool symlink) +static void smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct cifs_open_info_data *data, + struct super_block *sb, bool adjust_tz, bool symlink) { + struct smb311_posix_qinfo *info = &data->posix_fi; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); @@ -639,6 +672,8 @@ smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct smb311_posix_qinfo * if (symlink) { fattr->cf_mode |= S_IFLNK; fattr->cf_dtype = DT_LNK; + fattr->cf_symlink_target = data->symlink_target; + data->symlink_target = NULL; } else if (fattr->cf_cifsattrs & ATTR_DIRECTORY) { fattr->cf_mode |= S_IFDIR; fattr->cf_dtype = DT_DIR; @@ -655,13 +690,11 @@ smb311_posix_info_to_fattr(struct cifs_fattr *fattr, struct smb311_posix_qinfo * fattr->cf_mode, fattr->cf_uniqueid, fattr->cf_nlink); } - -/* Fill a cifs_fattr struct with info from FILE_ALL_INFO */ -static void -cifs_all_info_to_fattr(struct cifs_fattr *fattr, FILE_ALL_INFO *info, - struct super_block *sb, bool adjust_tz, - bool symlink, u32 reparse_tag) +static void cifs_open_info_to_fattr(struct cifs_fattr *fattr, struct cifs_open_info_data *data, + struct super_block *sb, bool adjust_tz, bool symlink, + u32 reparse_tag) { + struct smb2_file_all_info *info = &data->fi; struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); @@ -703,7 +736,8 @@ cifs_all_info_to_fattr(struct cifs_fattr *fattr, FILE_ALL_INFO *info, } else if (reparse_tag == IO_REPARSE_TAG_LX_BLK) { fattr->cf_mode |= S_IFBLK | cifs_sb->ctx->file_mode; fattr->cf_dtype = DT_BLK; - } else if (symlink) { /* TODO add more reparse tag checks */ + } else if (symlink || reparse_tag == IO_REPARSE_TAG_SYMLINK || + reparse_tag == IO_REPARSE_TAG_NFS) { fattr->cf_mode = S_IFLNK; fattr->cf_dtype = DT_LNK; } else if (fattr->cf_cifsattrs & ATTR_DIRECTORY) { @@ -735,6 +769,11 @@ cifs_all_info_to_fattr(struct cifs_fattr *fattr, FILE_ALL_INFO *info, } } + if (S_ISLNK(fattr->cf_mode)) { + fattr->cf_symlink_target = data->symlink_target; + data->symlink_target = NULL; + } + fattr->cf_uid = cifs_sb->ctx->linux_uid; fattr->cf_gid = cifs_sb->ctx->linux_gid; } @@ -744,23 +783,28 @@ cifs_get_file_info(struct file *filp) { int rc; unsigned int xid; - FILE_ALL_INFO find_data; + struct cifs_open_info_data data = {}; struct cifs_fattr fattr; struct inode *inode = file_inode(filp); struct cifsFileInfo *cfile = filp->private_data; struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); struct TCP_Server_Info *server = tcon->ses->server; + bool symlink = false; + u32 tag = 0; if (!server->ops->query_file_info) return -ENOSYS; xid = get_xid(); - rc = server->ops->query_file_info(xid, tcon, &cfile->fid, &find_data); + rc = server->ops->query_file_info(xid, tcon, cfile, &data); switch (rc) { case 0: /* TODO: add support to query reparse tag */ - cifs_all_info_to_fattr(&fattr, &find_data, inode->i_sb, false, - false, 0 /* no reparse tag */); + if (data.symlink_target) { + symlink = true; + tag = IO_REPARSE_TAG_SYMLINK; + } + cifs_open_info_to_fattr(&fattr, &data, inode->i_sb, false, symlink, tag); break; case -EREMOTE: cifs_create_dfs_fattr(&fattr, inode->i_sb); @@ -789,6 +833,7 @@ cifs_get_file_info(struct file *filp) /* if filetype is different, return error */ rc = cifs_fattr_to_inode(inode, &fattr); cgfi_exit: + cifs_free_open_info(&data); free_xid(xid); return rc; } @@ -860,14 +905,9 @@ cifs_backup_query_path_info(int xid, } #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ -static void -cifs_set_fattr_ino(int xid, - struct cifs_tcon *tcon, - struct super_block *sb, - struct inode **inode, - const char *full_path, - FILE_ALL_INFO *data, - struct cifs_fattr *fattr) +static void cifs_set_fattr_ino(int xid, struct cifs_tcon *tcon, struct super_block *sb, + struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, struct cifs_fattr *fattr) { struct cifs_sb_info *cifs_sb = CIFS_SB(sb); struct TCP_Server_Info *server = tcon->ses->server; @@ -885,11 +925,8 @@ cifs_set_fattr_ino(int xid, * If we have an inode pass a NULL tcon to ensure we don't * make a round trip to the server. This only works for SMB2+. */ - rc = server->ops->get_srv_inum(xid, - *inode ? NULL : tcon, - cifs_sb, full_path, - &fattr->cf_uniqueid, - data); + rc = server->ops->get_srv_inum(xid, *inode ? NULL : tcon, cifs_sb, full_path, + &fattr->cf_uniqueid, data); if (rc) { /* * If that fails reuse existing ino or generate one @@ -913,7 +950,7 @@ cifs_set_fattr_ino(int xid, } else { /* make an ino by hashing the UNC */ fattr->cf_flags |= CIFS_FATTR_FAKE_ROOT_INO; - fattr->cf_uniqueid = simple_hashstr(tcon->treeName); + fattr->cf_uniqueid = simple_hashstr(tcon->tree_name); } } } @@ -923,14 +960,10 @@ static inline bool is_inode_cache_good(struct inode *ino) return ino && CIFS_CACHE_READ(CIFS_I(ino)) && CIFS_I(ino)->time != 0; } -int -cifs_get_inode_info(struct inode **inode, - const char *full_path, - FILE_ALL_INFO *in_data, - struct super_block *sb, int xid, - const struct cifs_fid *fid) +int cifs_get_inode_info(struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, struct super_block *sb, int xid, + const struct cifs_fid *fid) { - struct cifs_tcon *tcon; struct TCP_Server_Info *server; struct tcon_link *tlink; @@ -938,8 +971,7 @@ cifs_get_inode_info(struct inode **inode, bool adjust_tz = false; struct cifs_fattr fattr = {0}; bool is_reparse_point = false; - FILE_ALL_INFO *data = in_data; - FILE_ALL_INFO *tmp_data = NULL; + struct cifs_open_info_data tmp_data = {}; void *smb1_backup_rsp_buf = NULL; int rc = 0; int tmprc = 0; @@ -960,21 +992,15 @@ cifs_get_inode_info(struct inode **inode, cifs_dbg(FYI, "No need to revalidate cached inode sizes\n"); goto out; } - tmp_data = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); - if (!tmp_data) { - rc = -ENOMEM; - goto out; - } - rc = server->ops->query_path_info(xid, tcon, cifs_sb, - full_path, tmp_data, - &adjust_tz, &is_reparse_point); + rc = server->ops->query_path_info(xid, tcon, cifs_sb, full_path, &tmp_data, + &adjust_tz, &is_reparse_point); #ifdef CONFIG_CIFS_DFS_UPCALL if (rc == -ENOENT && is_tcon_dfs(tcon)) rc = cifs_dfs_query_info_nonascii_quirk(xid, tcon, cifs_sb, full_path); #endif - data = tmp_data; + data = &tmp_data; } /* @@ -988,14 +1014,24 @@ cifs_get_inode_info(struct inode **inode, * since we have to check if its reparse tag matches a known * special file type e.g. symlink or fifo or char etc. */ - if ((le32_to_cpu(data->Attributes) & ATTR_REPARSE) && - server->ops->query_reparse_tag) { - rc = server->ops->query_reparse_tag(xid, tcon, cifs_sb, - full_path, &reparse_tag); - cifs_dbg(FYI, "reparse tag 0x%x\n", reparse_tag); + if (is_reparse_point && data->symlink_target) { + reparse_tag = IO_REPARSE_TAG_SYMLINK; + } else if ((le32_to_cpu(data->fi.Attributes) & ATTR_REPARSE) && + server->ops->query_reparse_tag) { + tmprc = server->ops->query_reparse_tag(xid, tcon, cifs_sb, full_path, + &reparse_tag); + if (tmprc) + cifs_dbg(FYI, "%s: query_reparse_tag: rc = %d\n", __func__, tmprc); + if (server->ops->query_symlink) { + tmprc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path, + &data->symlink_target, + is_reparse_point); + if (tmprc) + cifs_dbg(FYI, "%s: query_symlink: rc = %d\n", __func__, + tmprc); + } } - cifs_all_info_to_fattr(&fattr, data, sb, adjust_tz, - is_reparse_point, reparse_tag); + cifs_open_info_to_fattr(&fattr, data, sb, adjust_tz, is_reparse_point, reparse_tag); break; case -EREMOTE: /* DFS link, no metadata available on this server */ @@ -1014,18 +1050,20 @@ cifs_get_inode_info(struct inode **inode, */ if (backup_cred(cifs_sb) && is_smb1_server(server)) { /* for easier reading */ + FILE_ALL_INFO *fi; FILE_DIRECTORY_INFO *fdi; SEARCH_ID_FULL_DIR_INFO *si; rc = cifs_backup_query_path_info(xid, tcon, sb, full_path, &smb1_backup_rsp_buf, - &data); + &fi); if (rc) goto out; - fdi = (FILE_DIRECTORY_INFO *)data; - si = (SEARCH_ID_FULL_DIR_INFO *)data; + move_cifs_info_to_smb2(&data->fi, fi); + fdi = (FILE_DIRECTORY_INFO *)fi; + si = (SEARCH_ID_FULL_DIR_INFO *)fi; cifs_dir_info_to_fattr(&fattr, fdi, cifs_sb); fattr.cf_uniqueid = le64_to_cpu(si->UniqueId); @@ -1123,7 +1161,8 @@ handle_mnt_opt: out: cifs_buf_release(smb1_backup_rsp_buf); cifs_put_tlink(tlink); - kfree(tmp_data); + cifs_free_open_info(&tmp_data); + kfree(fattr.cf_symlink_target); return rc; } @@ -1138,7 +1177,7 @@ smb311_posix_get_inode_info(struct inode **inode, bool adjust_tz = false; struct cifs_fattr fattr = {0}; bool symlink = false; - struct smb311_posix_qinfo *data = NULL; + struct cifs_open_info_data data = {}; int rc = 0; int tmprc = 0; @@ -1155,15 +1194,9 @@ smb311_posix_get_inode_info(struct inode **inode, cifs_dbg(FYI, "No need to revalidate cached inode sizes\n"); goto out; } - data = kmalloc(sizeof(struct smb311_posix_qinfo), GFP_KERNEL); - if (!data) { - rc = -ENOMEM; - goto out; - } - rc = smb311_posix_query_path_info(xid, tcon, cifs_sb, - full_path, data, - &adjust_tz, &symlink); + rc = smb311_posix_query_path_info(xid, tcon, cifs_sb, full_path, &data, &adjust_tz, + &symlink); /* * 2. Convert it to internal cifs metadata (fattr) @@ -1171,7 +1204,7 @@ smb311_posix_get_inode_info(struct inode **inode, switch (rc) { case 0: - smb311_posix_info_to_fattr(&fattr, data, sb, adjust_tz, symlink); + smb311_posix_info_to_fattr(&fattr, &data, sb, adjust_tz, symlink); break; case -EREMOTE: /* DFS link, no metadata available on this server */ @@ -1228,7 +1261,8 @@ smb311_posix_get_inode_info(struct inode **inode, } out: cifs_put_tlink(tlink); - kfree(data); + cifs_free_open_info(&data); + kfree(fattr.cf_symlink_target); return rc; } @@ -2265,13 +2299,13 @@ cifs_dentry_needs_reval(struct dentry *dentry) return true; if (!open_cached_dir_by_dentry(tcon, dentry->d_parent, &cfid)) { - mutex_lock(&cfid->fid_mutex); + spin_lock(&cfid->fid_lock); if (cfid->time && cifs_i->time > cfid->time) { - mutex_unlock(&cfid->fid_mutex); + spin_unlock(&cfid->fid_lock); close_cached_dir(cfid); return false; } - mutex_unlock(&cfid->fid_mutex); + spin_unlock(&cfid->fid_lock); close_cached_dir(cfid); } /* @@ -2327,7 +2361,7 @@ cifs_invalidate_mapping(struct inode *inode) static int cifs_wait_bit_killable(struct wait_bit_key *key, int mode) { - freezable_schedule_unsafe(); + schedule(); if (signal_pending_state(mode, current)) return -ERESTARTSYS; return 0; @@ -2345,7 +2379,7 @@ cifs_revalidate_mapping(struct inode *inode) return 0; rc = wait_on_bit_lock_action(flags, CIFS_INO_LOCK, cifs_wait_bit_killable, - TASK_KILLABLE); + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); if (rc) return rc; diff --git a/fs/cifs/ioctl.c b/fs/cifs/ioctl.c index b6e6e5d6c8dd69c87b0b38ea8c5328c27322d54c..89d5fa8873649ce4bea3c6c2393293a7c9bdb139 100644 --- a/fs/cifs/ioctl.c +++ b/fs/cifs/ioctl.c @@ -484,12 +484,35 @@ long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) tcon = tlink_tcon(tlink); if (tcon && tcon->ses->server->ops->notify) { rc = tcon->ses->server->ops->notify(xid, - filep, (void __user *)arg); + filep, (void __user *)arg, + false /* no ret data */); cifs_dbg(FYI, "ioctl notify rc %d\n", rc); } else rc = -EOPNOTSUPP; cifs_put_tlink(tlink); break; + case CIFS_IOC_NOTIFY_INFO: + if (!S_ISDIR(inode->i_mode)) { + /* Notify can only be done on directories */ + rc = -EOPNOTSUPP; + break; + } + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + tcon = tlink_tcon(tlink); + if (tcon && tcon->ses->server->ops->notify) { + rc = tcon->ses->server->ops->notify(xid, + filep, (void __user *)arg, + true /* return details */); + cifs_dbg(FYI, "ioctl notify info rc %d\n", rc); + } else + rc = -EOPNOTSUPP; + cifs_put_tlink(tlink); + break; case CIFS_IOC_SHUTDOWN: rc = cifs_shutdown(inode->i_sb, arg); break; diff --git a/fs/cifs/link.c b/fs/cifs/link.c index 6803cb27eecc33fd2b669e4efa9494f35ac82fe6..bd374feeccaa19f021fdd03a4adf8d8e31f7d9fe 100644 --- a/fs/cifs/link.c +++ b/fs/cifs/link.c @@ -38,29 +38,28 @@ static int symlink_hash(unsigned int link_len, const char *link_str, u8 *md5_hash) { int rc; - struct crypto_shash *md5 = NULL; - struct sdesc *sdescmd5 = NULL; + struct shash_desc *md5 = NULL; - rc = cifs_alloc_hash("md5", &md5, &sdescmd5); + rc = cifs_alloc_hash("md5", &md5); if (rc) goto symlink_hash_err; - rc = crypto_shash_init(&sdescmd5->shash); + rc = crypto_shash_init(md5); if (rc) { cifs_dbg(VFS, "%s: Could not init md5 shash\n", __func__); goto symlink_hash_err; } - rc = crypto_shash_update(&sdescmd5->shash, link_str, link_len); + rc = crypto_shash_update(md5, link_str, link_len); if (rc) { cifs_dbg(VFS, "%s: Could not update with link_str\n", __func__); goto symlink_hash_err; } - rc = crypto_shash_final(&sdescmd5->shash, md5_hash); + rc = crypto_shash_final(md5, md5_hash); if (rc) cifs_dbg(VFS, "%s: Could not generate md5 hash\n", __func__); symlink_hash_err: - cifs_free_hash(&md5, &sdescmd5); + cifs_free_hash(&md5); return rc; } @@ -202,40 +201,6 @@ out: return rc; } -static int -query_mf_symlink(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const unsigned char *path, - char **symlinkinfo) -{ - int rc; - u8 *buf = NULL; - unsigned int link_len = 0; - unsigned int bytes_read = 0; - - buf = kmalloc(CIFS_MF_SYMLINK_FILE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - if (tcon->ses->server->ops->query_mf_symlink) - rc = tcon->ses->server->ops->query_mf_symlink(xid, tcon, - cifs_sb, path, buf, &bytes_read); - else - rc = -ENOSYS; - - if (rc) - goto out; - - if (bytes_read == 0) { /* not a symlink */ - rc = -EINVAL; - goto out; - } - - rc = parse_mf_symlink(buf, bytes_read, &link_len, symlinkinfo); -out: - kfree(buf); - return rc; -} - int check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, @@ -245,6 +210,7 @@ check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, u8 *buf = NULL; unsigned int link_len = 0; unsigned int bytes_read = 0; + char *symlink = NULL; if (!couldbe_mf_symlink(fattr)) /* it's not a symlink */ @@ -266,7 +232,7 @@ check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, if (bytes_read == 0) /* not a symlink */ goto out; - rc = parse_mf_symlink(buf, bytes_read, &link_len, NULL); + rc = parse_mf_symlink(buf, bytes_read, &link_len, &symlink); if (rc == -EINVAL) { /* it's not a symlink */ rc = 0; @@ -281,6 +247,7 @@ check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, fattr->cf_mode &= ~S_IFMT; fattr->cf_mode |= S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO; fattr->cf_dtype = DT_LNK; + fattr->cf_symlink_target = symlink; out: kfree(buf); return rc; @@ -600,75 +567,6 @@ cifs_hl_exit: return rc; } -const char * -cifs_get_link(struct dentry *direntry, struct inode *inode, - struct delayed_call *done) -{ - int rc = -ENOMEM; - unsigned int xid; - const char *full_path; - void *page; - char *target_path = NULL; - struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); - struct tcon_link *tlink = NULL; - struct cifs_tcon *tcon; - struct TCP_Server_Info *server; - - if (!direntry) - return ERR_PTR(-ECHILD); - - xid = get_xid(); - - tlink = cifs_sb_tlink(cifs_sb); - if (IS_ERR(tlink)) { - free_xid(xid); - return ERR_CAST(tlink); - } - tcon = tlink_tcon(tlink); - server = tcon->ses->server; - - page = alloc_dentry_path(); - full_path = build_path_from_dentry(direntry, page); - if (IS_ERR(full_path)) { - free_xid(xid); - cifs_put_tlink(tlink); - free_dentry_path(page); - return ERR_CAST(full_path); - } - - cifs_dbg(FYI, "Full path: %s inode = 0x%p\n", full_path, inode); - - rc = -EACCES; - /* - * First try Minshall+French Symlinks, if configured - * and fallback to UNIX Extensions Symlinks. - */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) - rc = query_mf_symlink(xid, tcon, cifs_sb, full_path, - &target_path); - - if (rc != 0 && server->ops->query_symlink) { - struct cifsInodeInfo *cifsi = CIFS_I(inode); - bool reparse_point = false; - - if (cifsi->cifsAttrs & ATTR_REPARSE) - reparse_point = true; - - rc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path, - &target_path, reparse_point); - } - - free_dentry_path(page); - free_xid(xid); - cifs_put_tlink(tlink); - if (rc != 0) { - kfree(target_path); - return ERR_PTR(rc); - } - set_delayed_call(done, kfree_link, target_path); - return target_path; -} - int cifs_symlink(struct user_namespace *mnt_userns, struct inode *inode, struct dentry *direntry, const char *symname) diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c index 87f60f7367315635949a924de6081ab336715e4c..da51ffd029280ec7c4444f246427254894adfdb3 100644 --- a/fs/cifs/misc.c +++ b/fs/cifs/misc.c @@ -117,8 +117,8 @@ tconInfoAlloc(void) ret_buf = kzalloc(sizeof(*ret_buf), GFP_KERNEL); if (!ret_buf) return NULL; - ret_buf->cfid = init_cached_dir(); - if (!ret_buf->cfid) { + ret_buf->cfids = init_cached_dirs(); + if (!ret_buf->cfids) { kfree(ret_buf); return NULL; } @@ -144,7 +144,7 @@ tconInfoFree(struct cifs_tcon *tcon) cifs_dbg(FYI, "Null buffer passed to tconInfoFree\n"); return; } - free_cached_dir(tcon); + free_cached_dirs(tcon->cfids); atomic_dec(&tconInfoAllocCount); kfree(tcon->nativeFileSystem); kfree_sensitive(tcon->password); @@ -525,7 +525,7 @@ cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb) cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SERVER_INUM; cifs_sb->mnt_cifs_serverino_autodisabled = true; cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s\n", - tcon ? tcon->treeName : "new server"); + tcon ? tcon->tree_name : "new server"); cifs_dbg(VFS, "The server doesn't seem to support them properly or the files might be on different servers (DFS)\n"); cifs_dbg(VFS, "Hardlinks will not be recognized on this mount. Consider mounting with the \"noserverino\" option to silence this message.\n"); @@ -824,7 +824,7 @@ cifs_close_deferred_file_under_dentry(struct cifs_tcon *tcon, const char *path) free_dentry_path(page); } -/* parses DFS refferal V3 structure +/* parses DFS referral V3 structure * caller is responsible for freeing target_nodes * returns: * - on success - 0 @@ -1071,59 +1071,58 @@ setup_aio_ctx_iter(struct cifs_aio_ctx *ctx, struct iov_iter *iter, int rw) /** * cifs_alloc_hash - allocate hash and hash context together * @name: The name of the crypto hash algo - * @shash: Where to put the pointer to the hash algo - * @sdesc: Where to put the pointer to the hash descriptor + * @sdesc: SHASH descriptor where to put the pointer to the hash TFM * * The caller has to make sure @sdesc is initialized to either NULL or - * a valid context. Both can be freed via cifs_free_hash(). + * a valid context. It can be freed via cifs_free_hash(). */ int -cifs_alloc_hash(const char *name, - struct crypto_shash **shash, struct sdesc **sdesc) +cifs_alloc_hash(const char *name, struct shash_desc **sdesc) { int rc = 0; - size_t size; + struct crypto_shash *alg = NULL; - if (*sdesc != NULL) + if (*sdesc) return 0; - *shash = crypto_alloc_shash(name, 0, 0); - if (IS_ERR(*shash)) { - cifs_dbg(VFS, "Could not allocate crypto %s\n", name); - rc = PTR_ERR(*shash); - *shash = NULL; + alg = crypto_alloc_shash(name, 0, 0); + if (IS_ERR(alg)) { + cifs_dbg(VFS, "Could not allocate shash TFM '%s'\n", name); + rc = PTR_ERR(alg); *sdesc = NULL; return rc; } - size = sizeof(struct shash_desc) + crypto_shash_descsize(*shash); - *sdesc = kmalloc(size, GFP_KERNEL); + *sdesc = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(alg), GFP_KERNEL); if (*sdesc == NULL) { - cifs_dbg(VFS, "no memory left to allocate crypto %s\n", name); - crypto_free_shash(*shash); - *shash = NULL; + cifs_dbg(VFS, "no memory left to allocate shash TFM '%s'\n", name); + crypto_free_shash(alg); return -ENOMEM; } - (*sdesc)->shash.tfm = *shash; + (*sdesc)->tfm = alg; return 0; } /** * cifs_free_hash - free hash and hash context together - * @shash: Where to find the pointer to the hash algo - * @sdesc: Where to find the pointer to the hash descriptor + * @sdesc: Where to find the pointer to the hash TFM * - * Freeing a NULL hash or context is safe. + * Freeing a NULL descriptor is safe. */ void -cifs_free_hash(struct crypto_shash **shash, struct sdesc **sdesc) +cifs_free_hash(struct shash_desc **sdesc) { - kfree(*sdesc); + if (unlikely(!sdesc) || !*sdesc) + return; + + if ((*sdesc)->tfm) { + crypto_free_shash((*sdesc)->tfm); + (*sdesc)->tfm = NULL; + } + + kfree_sensitive(*sdesc); *sdesc = NULL; - if (*shash) - crypto_free_shash(*shash); - *shash = NULL; } /** @@ -1328,7 +1327,7 @@ int cifs_dfs_query_info_nonascii_quirk(const unsigned int xid, char *treename, *dfspath, sep; int treenamelen, linkpathlen, rc; - treename = tcon->treeName; + treename = tcon->tree_name; /* MS-DFSC: All paths in REQ_GET_DFS_REFERRAL and RESP_GET_DFS_REFERRAL * messages MUST be encoded with exactly one leading backslash, not two * leading backslashes. diff --git a/fs/cifs/netlink.c b/fs/cifs/netlink.c index 291cb606f149679298d46cafe1aa7cee9da16ae1..147d9409252cd3b22bbd0e5436d97867b6309413 100644 --- a/fs/cifs/netlink.c +++ b/fs/cifs/netlink.c @@ -51,6 +51,7 @@ struct genl_family cifs_genl_family = { .policy = cifs_genl_policy, .ops = cifs_genl_ops, .n_ops = ARRAY_SIZE(cifs_genl_ops), + .resv_start_op = CIFS_GENL_CMD_SWN_NOTIFY + 1, .mcgrps = cifs_genl_mcgrps, .n_mcgrps = ARRAY_SIZE(cifs_genl_mcgrps), }; diff --git a/fs/cifs/readdir.c b/fs/cifs/readdir.c index 8e060c00c969011bd2c6a8f5415c672d0dca8cfb..2d75ba5aaa8adfedc432230e1102bd1561db7d4f 100644 --- a/fs/cifs/readdir.c +++ b/fs/cifs/readdir.c @@ -844,17 +844,34 @@ static bool emit_cached_dirents(struct cached_dirents *cde, struct dir_context *ctx) { struct cached_dirent *dirent; - int rc; + bool rc; list_for_each_entry(dirent, &cde->entries, entry) { - if (ctx->pos >= dirent->pos) + /* + * Skip all early entries prior to the current lseek() + * position. + */ + if (ctx->pos > dirent->pos) continue; + /* + * We recorded the current ->pos value for the dirent + * when we stored it in the cache. + * However, this sequence of ->pos values may have holes + * in it, for example dot-dirs returned from the server + * are suppressed. + * Handle this bu forcing ctx->pos to be the same as the + * ->pos of the current dirent we emit from the cache. + * This means that when we emit these entries from the cache + * we now emit them with the same ->pos value as in the + * initial scan. + */ ctx->pos = dirent->pos; rc = dir_emit(ctx, dirent->name, dirent->namelen, dirent->fattr.cf_uniqueid, dirent->fattr.cf_dtype); if (!rc) return rc; + ctx->pos++; } return true; } @@ -994,6 +1011,8 @@ static int cifs_filldir(char *find_entry, struct file *file, cifs_unix_basic_to_fattr(&fattr, &((FILE_UNIX_INFO *)find_entry)->basic, cifs_sb); + if (S_ISLNK(fattr.cf_mode)) + fattr.cf_flags |= CIFS_FATTR_NEED_REVAL; break; case SMB_FIND_FILE_INFO_STANDARD: cifs_std_info_to_fattr(&fattr, @@ -1202,10 +1221,10 @@ int cifs_readdir(struct file *file, struct dir_context *ctx) ctx->pos, tmp_buf); cifs_save_resume_key(current_entry, cifsFile); break; - } else - current_entry = - nxt_dir_entry(current_entry, end_of_smb, - cifsFile->srch_inf.info_level); + } + current_entry = + nxt_dir_entry(current_entry, end_of_smb, + cifsFile->srch_inf.info_level); } kfree(tmp_buf); diff --git a/fs/cifs/sess.c b/fs/cifs/sess.c index 3af3b05b6c740aa87d48a066d0049f81badf5e93..0435d1dfa9e11fb998d140d11ea88a6fdd76f4d6 100644 --- a/fs/cifs/sess.c +++ b/fs/cifs/sess.c @@ -601,11 +601,6 @@ static void unicode_ssetup_strings(char **pbcc_area, struct cifs_ses *ses, /* BB FIXME add check that strings total less than 335 or will need to send them as arrays */ - /* unicode strings, must be word aligned before the call */ -/* if ((long) bcc_ptr % 2) { - *bcc_ptr = 0; - bcc_ptr++; - } */ /* copy user */ if (ses->user_name == NULL) { /* null user mount */ @@ -1213,10 +1208,18 @@ out_free_smb_buf: static void sess_free_buffer(struct sess_data *sess_data) { + struct kvec *iov = sess_data->iov; + + /* + * Zero the session data before freeing, as it might contain sensitive info (keys, etc). + * Note that iov[1] is already freed by caller. + */ + if (sess_data->buf0_type != CIFS_NO_BUFFER && iov[0].iov_base) + memzero_explicit(iov[0].iov_base, iov[0].iov_len); - free_rsp_buf(sess_data->buf0_type, sess_data->iov[0].iov_base); + free_rsp_buf(sess_data->buf0_type, iov[0].iov_base); sess_data->buf0_type = CIFS_NO_BUFFER; - kfree(sess_data->iov[2].iov_base); + kfree_sensitive(iov[2].iov_base); } static int @@ -1318,7 +1321,7 @@ sess_auth_ntlmv2(struct sess_data *sess_data) } if (ses->capabilities & CAP_UNICODE) { - if (sess_data->iov[0].iov_len % 2) { + if (!IS_ALIGNED(sess_data->iov[0].iov_len, 2)) { *bcc_ptr = 0; bcc_ptr++; } @@ -1358,7 +1361,7 @@ sess_auth_ntlmv2(struct sess_data *sess_data) /* no string area to decode, do nothing */ } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { /* unicode string area must be word-aligned */ - if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) { + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { ++bcc_ptr; --bytes_remaining; } @@ -1374,7 +1377,7 @@ out: sess_data->result = rc; sess_data->func = NULL; sess_free_buffer(sess_data); - kfree(ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); ses->auth_key.response = NULL; } @@ -1442,8 +1445,7 @@ sess_auth_kerberos(struct sess_data *sess_data) if (ses->capabilities & CAP_UNICODE) { /* unicode strings must be word aligned */ - if ((sess_data->iov[0].iov_len - + sess_data->iov[1].iov_len) % 2) { + if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { *bcc_ptr = 0; bcc_ptr++; } @@ -1494,7 +1496,7 @@ sess_auth_kerberos(struct sess_data *sess_data) /* no string area to decode, do nothing */ } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { /* unicode string area must be word-aligned */ - if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) { + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { ++bcc_ptr; --bytes_remaining; } @@ -1513,7 +1515,7 @@ out: sess_data->result = rc; sess_data->func = NULL; sess_free_buffer(sess_data); - kfree(ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); ses->auth_key.response = NULL; } @@ -1546,7 +1548,7 @@ _sess_auth_rawntlmssp_assemble_req(struct sess_data *sess_data) bcc_ptr = sess_data->iov[2].iov_base; /* unicode strings must be word aligned */ - if ((sess_data->iov[0].iov_len + sess_data->iov[1].iov_len) % 2) { + if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { *bcc_ptr = 0; bcc_ptr++; } @@ -1648,7 +1650,7 @@ sess_auth_rawntlmssp_negotiate(struct sess_data *sess_data) rc = decode_ntlmssp_challenge(bcc_ptr, blob_len, ses); out_free_ntlmsspblob: - kfree(ntlmsspblob); + kfree_sensitive(ntlmsspblob); out: sess_free_buffer(sess_data); @@ -1658,9 +1660,9 @@ out: } /* Else error. Cleanup */ - kfree(ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); ses->auth_key.response = NULL; - kfree(ses->ntlmssp); + kfree_sensitive(ses->ntlmssp); ses->ntlmssp = NULL; sess_data->func = NULL; @@ -1747,7 +1749,7 @@ sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data) /* no string area to decode, do nothing */ } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { /* unicode string area must be word-aligned */ - if (((unsigned long) bcc_ptr - (unsigned long) smb_buf) % 2) { + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { ++bcc_ptr; --bytes_remaining; } @@ -1759,7 +1761,7 @@ sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data) } out_free_ntlmsspblob: - kfree(ntlmsspblob); + kfree_sensitive(ntlmsspblob); out: sess_free_buffer(sess_data); @@ -1767,9 +1769,9 @@ out: rc = sess_establish_session(sess_data); /* Cleanup */ - kfree(ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); ses->auth_key.response = NULL; - kfree(ses->ntlmssp); + kfree_sensitive(ses->ntlmssp); ses->ntlmssp = NULL; sess_data->func = NULL; @@ -1845,7 +1847,7 @@ int CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses, rc = sess_data->result; out: - kfree(sess_data); + kfree_sensitive(sess_data); return rc; } #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c index f36b2d2d40ca3785020a048eae13734a26e56306..50480751e521ca752e784bf1f4190f33ca9a8e67 100644 --- a/fs/cifs/smb1ops.c +++ b/fs/cifs/smb1ops.c @@ -542,31 +542,32 @@ cifs_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, return rc; } -static int -cifs_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - FILE_ALL_INFO *data, bool *adjustTZ, bool *symlink) +static int cifs_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjustTZ, bool *symlink) { int rc; + FILE_ALL_INFO fi = {}; *symlink = false; /* could do find first instead but this returns more info */ - rc = CIFSSMBQPathInfo(xid, tcon, full_path, data, 0 /* not legacy */, - cifs_sb->local_nls, cifs_remap(cifs_sb)); + rc = CIFSSMBQPathInfo(xid, tcon, full_path, &fi, 0 /* not legacy */, cifs_sb->local_nls, + cifs_remap(cifs_sb)); /* * BB optimize code so we do not make the above call when server claims * no NT SMB support and the above call failed at least once - set flag * in tcon or mount. */ if ((rc == -EOPNOTSUPP) || (rc == -EINVAL)) { - rc = SMBQueryInformation(xid, tcon, full_path, data, - cifs_sb->local_nls, + rc = SMBQueryInformation(xid, tcon, full_path, &fi, cifs_sb->local_nls, cifs_remap(cifs_sb)); + if (!rc) + move_cifs_info_to_smb2(&data->fi, &fi); *adjustTZ = true; } - if (!rc && (le32_to_cpu(data->Attributes) & ATTR_REPARSE)) { + if (!rc && (le32_to_cpu(fi.Attributes) & ATTR_REPARSE)) { int tmprc; int oplock = 0; struct cifs_fid fid; @@ -592,10 +593,9 @@ cifs_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, return rc; } -static int -cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - u64 *uniqueid, FILE_ALL_INFO *data) +static int cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + u64 *uniqueid, struct cifs_open_info_data *unused) { /* * We can not use the IndexNumber field by default from Windows or @@ -613,11 +613,22 @@ cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, cifs_remap(cifs_sb)); } -static int -cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_fid *fid, FILE_ALL_INFO *data) +static int cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data) { - return CIFSSMBQFileInfo(xid, tcon, fid->netfid, data); + int rc; + FILE_ALL_INFO fi = {}; + + if (cfile->symlink_target) { + data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!data->symlink_target) + return -ENOMEM; + } + + rc = CIFSSMBQFileInfo(xid, tcon, cfile->fid.netfid, &fi); + if (!rc) + move_cifs_info_to_smb2(&data->fi, &fi); + return rc; } static void @@ -702,19 +713,20 @@ cifs_mkdir_setinfo(struct inode *inode, const char *full_path, cifsInode->cifsAttrs = dosattrs; } -static int -cifs_open_file(const unsigned int xid, struct cifs_open_parms *oparms, - __u32 *oplock, FILE_ALL_INFO *buf) +static int cifs_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, + void *buf) { + FILE_ALL_INFO *fi = buf; + if (!(oparms->tcon->ses->capabilities & CAP_NT_SMBS)) return SMBLegacyOpen(xid, oparms->tcon, oparms->path, oparms->disposition, oparms->desired_access, oparms->create_options, - &oparms->fid->netfid, oplock, buf, + &oparms->fid->netfid, oplock, fi, oparms->cifs_sb->local_nls, cifs_remap(oparms->cifs_sb)); - return CIFS_open(xid, oparms, oplock, buf); + return CIFS_open(xid, oparms, oplock, fi); } static void diff --git a/fs/cifs/smb2file.c b/fs/cifs/smb2file.c index 9dfd2dd612c25cca19dd3459fad9164859970c22..ffbd9a99fc128634c6812651f1b9776a9727aeb4 100644 --- a/fs/cifs/smb2file.c +++ b/fs/cifs/smb2file.c @@ -20,40 +20,125 @@ #include "cifs_unicode.h" #include "fscache.h" #include "smb2proto.h" +#include "smb2status.h" -int -smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, - __u32 *oplock, FILE_ALL_INFO *buf) +static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) +{ + struct smb2_err_rsp *err = iov->iov_base; + struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL); + u32 len; + + if (err->ErrorContextCount) { + struct smb2_error_context_rsp *p, *end; + + len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp, + ErrorContextData) + + sizeof(struct smb2_symlink_err_rsp)); + if (le32_to_cpu(err->ByteCount) < len || iov->iov_len < len + sizeof(*err)) + return ERR_PTR(-EINVAL); + + p = (struct smb2_error_context_rsp *)err->ErrorData; + end = (struct smb2_error_context_rsp *)((u8 *)err + iov->iov_len); + do { + if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) { + sym = (struct smb2_symlink_err_rsp *)&p->ErrorContextData; + break; + } + cifs_dbg(FYI, "%s: skipping unhandled error context: 0x%x\n", + __func__, le32_to_cpu(p->ErrorId)); + + len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + p = (struct smb2_error_context_rsp *)((u8 *)&p->ErrorContextData + len); + } while (p < end); + } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && + iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) { + sym = (struct smb2_symlink_err_rsp *)err->ErrorData; + } + + if (!IS_ERR(sym) && (le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) + sym = ERR_PTR(-EINVAL); + + return sym; +} + +int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec *iov, char **path) +{ + struct smb2_symlink_err_rsp *sym; + unsigned int sub_offs, sub_len; + unsigned int print_offs, print_len; + char *s; + + if (!cifs_sb || !iov || !iov->iov_base || !iov->iov_len || !path) + return -EINVAL; + + sym = symlink_data(iov); + if (IS_ERR(sym)) + return PTR_ERR(sym); + + sub_len = le16_to_cpu(sym->SubstituteNameLength); + sub_offs = le16_to_cpu(sym->SubstituteNameOffset); + print_len = le16_to_cpu(sym->PrintNameLength); + print_offs = le16_to_cpu(sym->PrintNameOffset); + + if (iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offs + sub_len || + iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offs + print_len) + return -EINVAL; + + s = cifs_strndup_from_utf16((char *)sym->PathBuffer + sub_offs, sub_len, true, + cifs_sb->local_nls); + if (!s) + return -ENOMEM; + convert_delimiter(s, '/'); + cifs_dbg(FYI, "%s: symlink target: %s\n", __func__, s); + + *path = s; + return 0; +} + +int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, void *buf) { int rc; __le16 *smb2_path; - struct smb2_file_all_info *smb2_data = NULL; __u8 smb2_oplock; + struct cifs_open_info_data *data = buf; + struct smb2_file_all_info file_info = {}; + struct smb2_file_all_info *smb2_data = data ? &file_info : NULL; + struct kvec err_iov = {}; + int err_buftype = CIFS_NO_BUFFER; struct cifs_fid *fid = oparms->fid; struct network_resiliency_req nr_ioctl_req; smb2_path = cifs_convert_path_to_utf16(oparms->path, oparms->cifs_sb); - if (smb2_path == NULL) { - rc = -ENOMEM; - goto out; - } - - smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2, - GFP_KERNEL); - if (smb2_data == NULL) { - rc = -ENOMEM; - goto out; - } + if (smb2_path == NULL) + return -ENOMEM; oparms->desired_access |= FILE_READ_ATTRIBUTES; smb2_oplock = SMB2_OPLOCK_LEVEL_BATCH; - rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, - NULL, NULL); + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, NULL, &err_iov, + &err_buftype); + if (rc && data) { + struct smb2_hdr *hdr = err_iov.iov_base; + + if (unlikely(!err_iov.iov_base || err_buftype == CIFS_NO_BUFFER)) + rc = -ENOMEM; + else if (hdr->Status == STATUS_STOPPED_ON_SYMLINK) { + rc = smb2_parse_symlink_response(oparms->cifs_sb, &err_iov, + &data->symlink_target); + if (!rc) { + memset(smb2_data, 0, sizeof(*smb2_data)); + oparms->create_options |= OPEN_REPARSE_POINT; + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, smb2_data, + NULL, NULL, NULL); + oparms->create_options &= ~OPEN_REPARSE_POINT; + } + } + } + if (rc) goto out; - if (oparms->tcon->use_resilient) { /* default timeout is 0, servers pick default (120 seconds) */ nr_ioctl_req.Timeout = @@ -73,7 +158,7 @@ smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, rc = 0; } - if (buf) { + if (smb2_data) { /* if open response does not have IndexNumber field - get it */ if (smb2_data->IndexNumber == 0) { rc = SMB2_get_srv_num(xid, oparms->tcon, @@ -89,12 +174,12 @@ smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, rc = 0; } } - move_smb2_info_to_cifs(buf, smb2_data); + memcpy(&data->fi, smb2_data, sizeof(data->fi)); } *oplock = smb2_oplock; out: - kfree(smb2_data); + free_rsp_buf(err_buftype, err_iov.iov_base); kfree(smb2_path); return rc; } diff --git a/fs/cifs/smb2inode.c b/fs/cifs/smb2inode.c index b83f59051b26f984f742338d6746f97865e4b9f7..a6640e6ea58bc175bd2a02d5540a9efd9771dbce 100644 --- a/fs/cifs/smb2inode.c +++ b/fs/cifs/smb2inode.c @@ -24,6 +24,7 @@ #include "smb2pdu.h" #include "smb2proto.h" #include "cached_dir.h" +#include "smb2status.h" static void free_set_inf_compound(struct smb_rqst *rqst) @@ -50,13 +51,15 @@ struct cop_vars { /* * note: If cfile is passed, the reference to it is dropped here. * So make sure that you do not reuse cfile after return from this func. + * + * If passing @err_iov and @err_buftype, ensure to make them both large enough (>= 3) to hold all + * error responses. Caller is also responsible for freeing them up. */ -static int -smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - __u32 desired_access, __u32 create_disposition, - __u32 create_options, umode_t mode, void *ptr, int command, - struct cifsFileInfo *cfile) +static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + __u32 desired_access, __u32 create_disposition, __u32 create_options, + umode_t mode, void *ptr, int command, struct cifsFileInfo *cfile, + struct kvec *err_iov, int *err_buftype) { struct cop_vars *vars = NULL; struct kvec *rsp_iov; @@ -70,6 +73,7 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, int num_rqst = 0; int resp_buftype[3]; struct smb2_query_info_rsp *qi_rsp = NULL; + struct cifs_open_info_data *idata; int flags = 0; __u8 delete_pending[8] = {1, 0, 0, 0, 0, 0, 0, 0}; unsigned int size[2]; @@ -379,20 +383,25 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, SMB2_open_free(&rqst[0]); if (rc == -EREMCHG) { - pr_warn_once("server share %s deleted\n", tcon->treeName); + pr_warn_once("server share %s deleted\n", tcon->tree_name); tcon->need_reconnect = true; } switch (command) { case SMB2_OP_QUERY_INFO: + idata = ptr; + if (rc == 0 && cfile && cfile->symlink_target) { + idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!idata->symlink_target) + rc = -ENOMEM; + } if (rc == 0) { qi_rsp = (struct smb2_query_info_rsp *) rsp_iov[1].iov_base; rc = smb2_validate_and_copy_iov( le16_to_cpu(qi_rsp->OutputBufferOffset), le32_to_cpu(qi_rsp->OutputBufferLength), - &rsp_iov[1], sizeof(struct smb2_file_all_info), - ptr); + &rsp_iov[1], sizeof(idata->fi), (char *)&idata->fi); } if (rqst[1].rq_iov) SMB2_query_info_free(&rqst[1]); @@ -406,13 +415,20 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, tcon->tid); break; case SMB2_OP_POSIX_QUERY_INFO: + idata = ptr; + if (rc == 0 && cfile && cfile->symlink_target) { + idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!idata->symlink_target) + rc = -ENOMEM; + } if (rc == 0) { qi_rsp = (struct smb2_query_info_rsp *) rsp_iov[1].iov_base; rc = smb2_validate_and_copy_iov( le16_to_cpu(qi_rsp->OutputBufferOffset), le32_to_cpu(qi_rsp->OutputBufferLength), - &rsp_iov[1], sizeof(struct smb311_posix_qinfo) /* add SIDs */, ptr); + &rsp_iov[1], sizeof(idata->posix_fi) /* add SIDs */, + (char *)&idata->posix_fi); } if (rqst[1].rq_iov) SMB2_query_info_free(&rqst[1]); @@ -477,42 +493,33 @@ smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, free_set_inf_compound(rqst); break; } - free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); - free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); - free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + + if (rc && err_iov && err_buftype) { + memcpy(err_iov, rsp_iov, 3 * sizeof(*err_iov)); + memcpy(err_buftype, resp_buftype, 3 * sizeof(*err_buftype)); + } else { + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + } kfree(vars); return rc; } -void -move_smb2_info_to_cifs(FILE_ALL_INFO *dst, struct smb2_file_all_info *src) -{ - memcpy(dst, src, (size_t)(&src->CurrentByteOffset) - (size_t)src); - dst->CurrentByteOffset = src->CurrentByteOffset; - dst->Mode = src->Mode; - dst->AlignmentRequirement = src->AlignmentRequirement; - dst->IndexNumber1 = 0; /* we don't use it */ -} - -int -smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - FILE_ALL_INFO *data, bool *adjust_tz, bool *reparse) +int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse) { int rc; - struct smb2_file_all_info *smb2_data; __u32 create_options = 0; struct cifsFileInfo *cfile; struct cached_fid *cfid = NULL; + struct kvec err_iov[3] = {}; + int err_buftype[3] = {}; *adjust_tz = false; *reparse = false; - smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2, - GFP_KERNEL); - if (smb2_data == NULL) - return -ENOMEM; - if (strcmp(full_path, "")) rc = -ENOENT; else @@ -520,63 +527,58 @@ smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, /* If it is a root and its handle is cached then use it */ if (!rc) { if (cfid->file_all_info_is_valid) { - move_smb2_info_to_cifs(data, - &cfid->file_all_info); + memcpy(&data->fi, &cfid->file_all_info, sizeof(data->fi)); } else { - rc = SMB2_query_info(xid, tcon, - cfid->fid.persistent_fid, - cfid->fid.volatile_fid, smb2_data); - if (!rc) - move_smb2_info_to_cifs(data, smb2_data); + rc = SMB2_query_info(xid, tcon, cfid->fid.persistent_fid, + cfid->fid.volatile_fid, &data->fi); } close_cached_dir(cfid); - goto out; + return rc; } cifs_get_readable_path(tcon, full_path, &cfile); - rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, - FILE_READ_ATTRIBUTES, FILE_OPEN, create_options, - ACL_NO_MODE, smb2_data, SMB2_OP_QUERY_INFO, cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, FILE_OPEN, + create_options, ACL_NO_MODE, data, SMB2_OP_QUERY_INFO, cfile, + err_iov, err_buftype); if (rc == -EOPNOTSUPP) { + if (err_iov[0].iov_base && err_buftype[0] != CIFS_NO_BUFFER && + ((struct smb2_hdr *)err_iov[0].iov_base)->Command == SMB2_CREATE && + ((struct smb2_hdr *)err_iov[0].iov_base)->Status == STATUS_STOPPED_ON_SYMLINK) { + rc = smb2_parse_symlink_response(cifs_sb, err_iov, &data->symlink_target); + if (rc) + goto out; + } *reparse = true; create_options |= OPEN_REPARSE_POINT; /* Failed on a symbolic link - query a reparse point info */ cifs_get_readable_path(tcon, full_path, &cfile); - rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, - FILE_READ_ATTRIBUTES, FILE_OPEN, - create_options, ACL_NO_MODE, - smb2_data, SMB2_OP_QUERY_INFO, cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, + FILE_OPEN, create_options, ACL_NO_MODE, data, + SMB2_OP_QUERY_INFO, cfile, NULL, NULL); } - if (rc) - goto out; - move_smb2_info_to_cifs(data, smb2_data); out: - kfree(smb2_data); + free_rsp_buf(err_buftype[0], err_iov[0].iov_base); + free_rsp_buf(err_buftype[1], err_iov[1].iov_base); + free_rsp_buf(err_buftype[2], err_iov[2].iov_base); return rc; } -int -smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - struct smb311_posix_qinfo *data, bool *adjust_tz, bool *reparse) +int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse) { int rc; __u32 create_options = 0; struct cifsFileInfo *cfile; - struct smb311_posix_qinfo *smb2_data; + struct kvec err_iov[3] = {}; + int err_buftype[3] = {}; *adjust_tz = false; *reparse = false; - /* BB TODO: Make struct larger when add support for parsing owner SIDs */ - smb2_data = kzalloc(sizeof(struct smb311_posix_qinfo), - GFP_KERNEL); - if (smb2_data == NULL) - return -ENOMEM; - /* * BB TODO: Add support for using the cached root handle. * Create SMB2_query_posix_info worker function to do non-compounded query @@ -585,29 +587,32 @@ smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, */ cifs_get_readable_path(tcon, full_path, &cfile); - rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, - FILE_READ_ATTRIBUTES, FILE_OPEN, create_options, - ACL_NO_MODE, smb2_data, SMB2_OP_POSIX_QUERY_INFO, cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, FILE_OPEN, + create_options, ACL_NO_MODE, data, SMB2_OP_POSIX_QUERY_INFO, cfile, + err_iov, err_buftype); if (rc == -EOPNOTSUPP) { /* BB TODO: When support for special files added to Samba re-verify this path */ + if (err_iov[0].iov_base && err_buftype[0] != CIFS_NO_BUFFER && + ((struct smb2_hdr *)err_iov[0].iov_base)->Command == SMB2_CREATE && + ((struct smb2_hdr *)err_iov[0].iov_base)->Status == STATUS_STOPPED_ON_SYMLINK) { + rc = smb2_parse_symlink_response(cifs_sb, err_iov, &data->symlink_target); + if (rc) + goto out; + } *reparse = true; create_options |= OPEN_REPARSE_POINT; /* Failed on a symbolic link - query a reparse point info */ cifs_get_readable_path(tcon, full_path, &cfile); - rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, - FILE_READ_ATTRIBUTES, FILE_OPEN, - create_options, ACL_NO_MODE, - smb2_data, SMB2_OP_POSIX_QUERY_INFO, cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_READ_ATTRIBUTES, + FILE_OPEN, create_options, ACL_NO_MODE, data, + SMB2_OP_POSIX_QUERY_INFO, cfile, NULL, NULL); } - if (rc) - goto out; - - /* TODO: will need to allow for the 2 SIDs when add support for getting owner UID/GID */ - memcpy(data, smb2_data, sizeof(struct smb311_posix_qinfo)); out: - kfree(smb2_data); + free_rsp_buf(err_buftype[0], err_iov[0].iov_base); + free_rsp_buf(err_buftype[1], err_iov[1].iov_base); + free_rsp_buf(err_buftype[2], err_iov[2].iov_base); return rc; } @@ -619,7 +624,7 @@ smb2_mkdir(const unsigned int xid, struct inode *parent_inode, umode_t mode, return smb2_compound_op(xid, tcon, cifs_sb, name, FILE_WRITE_ATTRIBUTES, FILE_CREATE, CREATE_NOT_FILE, mode, NULL, SMB2_OP_MKDIR, - NULL); + NULL, NULL, NULL); } void @@ -641,7 +646,7 @@ smb2_mkdir_setinfo(struct inode *inode, const char *name, tmprc = smb2_compound_op(xid, tcon, cifs_sb, name, FILE_WRITE_ATTRIBUTES, FILE_CREATE, CREATE_NOT_FILE, ACL_NO_MODE, - &data, SMB2_OP_SET_INFO, cfile); + &data, SMB2_OP_SET_INFO, cfile, NULL, NULL); if (tmprc == 0) cifs_i->cifsAttrs = dosattrs; } @@ -652,7 +657,7 @@ smb2_rmdir(const unsigned int xid, struct cifs_tcon *tcon, const char *name, { return smb2_compound_op(xid, tcon, cifs_sb, name, DELETE, FILE_OPEN, CREATE_NOT_FILE, ACL_NO_MODE, - NULL, SMB2_OP_RMDIR, NULL); + NULL, SMB2_OP_RMDIR, NULL, NULL, NULL); } int @@ -661,7 +666,7 @@ smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, { return smb2_compound_op(xid, tcon, cifs_sb, name, DELETE, FILE_OPEN, CREATE_DELETE_ON_CLOSE | OPEN_REPARSE_POINT, - ACL_NO_MODE, NULL, SMB2_OP_DELETE, NULL); + ACL_NO_MODE, NULL, SMB2_OP_DELETE, NULL, NULL, NULL); } static int @@ -680,7 +685,7 @@ smb2_set_path_attr(const unsigned int xid, struct cifs_tcon *tcon, } rc = smb2_compound_op(xid, tcon, cifs_sb, from_name, access, FILE_OPEN, 0, ACL_NO_MODE, smb2_to_name, - command, cfile); + command, cfile, NULL, NULL); smb2_rename_path: kfree(smb2_to_name); return rc; @@ -720,7 +725,7 @@ smb2_set_path_size(const unsigned int xid, struct cifs_tcon *tcon, cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile); return smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_WRITE_DATA, FILE_OPEN, 0, ACL_NO_MODE, - &eof, SMB2_OP_SET_EOF, cfile); + &eof, SMB2_OP_SET_EOF, cfile, NULL, NULL); } int @@ -746,7 +751,8 @@ smb2_set_file_info(struct inode *inode, const char *full_path, cifs_get_writable_path(tcon, full_path, FIND_WR_ANY, &cfile); rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, FILE_WRITE_ATTRIBUTES, FILE_OPEN, - 0, ACL_NO_MODE, buf, SMB2_OP_SET_INFO, cfile); + 0, ACL_NO_MODE, buf, SMB2_OP_SET_INFO, cfile, + NULL, NULL); cifs_put_tlink(tlink); return rc; } diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index d73e5672aac493b8e1694250c9b8b650d9e6e314..a387204779660f64d47069eb8ae50da9c00b9a44 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -248,7 +248,7 @@ smb2_check_message(char *buf, unsigned int len, struct TCP_Server_Info *server) * Some windows servers (win2016) will pad also the final * PDU in a compound to 8 bytes. */ - if (((calc_len + 7) & ~7) == len) + if (ALIGN(calc_len, 8) == len) return 0; /* @@ -870,8 +870,8 @@ smb311_update_preauth_hash(struct cifs_ses *ses, struct TCP_Server_Info *server, struct kvec *iov, int nvec) { int i, rc; - struct sdesc *d; struct smb2_hdr *hdr; + struct shash_desc *sha512 = NULL; hdr = (struct smb2_hdr *)iov[0].iov_base; /* neg prot are always taken */ @@ -901,14 +901,14 @@ ok: if (rc) return rc; - d = server->secmech.sdescsha512; - rc = crypto_shash_init(&d->shash); + sha512 = server->secmech.sha512; + rc = crypto_shash_init(sha512); if (rc) { cifs_dbg(VFS, "%s: Could not init sha512 shash\n", __func__); return rc; } - rc = crypto_shash_update(&d->shash, ses->preauth_sha_hash, + rc = crypto_shash_update(sha512, ses->preauth_sha_hash, SMB2_PREAUTH_HASH_SIZE); if (rc) { cifs_dbg(VFS, "%s: Could not update sha512 shash\n", __func__); @@ -916,8 +916,7 @@ ok: } for (i = 0; i < nvec; i++) { - rc = crypto_shash_update(&d->shash, - iov[i].iov_base, iov[i].iov_len); + rc = crypto_shash_update(sha512, iov[i].iov_base, iov[i].iov_len); if (rc) { cifs_dbg(VFS, "%s: Could not update sha512 shash\n", __func__); @@ -925,7 +924,7 @@ ok: } } - rc = crypto_shash_final(&d->shash, ses->preauth_sha_hash); + rc = crypto_shash_final(sha512, ses->preauth_sha_hash); if (rc) { cifs_dbg(VFS, "%s: Could not finalize sha512 shash\n", __func__); diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 96f3b0573606ecebc11fc7b0c7d15350c1155eb8..17b25153cb6897d98d135cc910de9cf0559715f8 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -512,8 +512,7 @@ smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) static int parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, - size_t buf_len, - struct cifs_ses *ses) + size_t buf_len, struct cifs_ses *ses, bool in_mount) { struct network_interface_info_ioctl_rsp *p; struct sockaddr_in *addr4; @@ -543,6 +542,21 @@ parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, } spin_unlock(&ses->iface_lock); + /* + * Samba server e.g. can return an empty interface list in some cases, + * which would only be a problem if we were requesting multichannel + */ + if (bytes_left == 0) { + /* avoid spamming logs every 10 minutes, so log only in mount */ + if ((ses->chan_max > 1) && in_mount) + cifs_dbg(VFS, + "multichannel not available\n" + "Empty network interface list returned by server %s\n", + ses->server->hostname); + rc = -EINVAL; + goto out; + } + while (bytes_left >= sizeof(*p)) { memset(&tmp_iface, 0, sizeof(tmp_iface)); tmp_iface.speed = le64_to_cpu(p->LinkSpeed); @@ -673,7 +687,7 @@ out: } int -SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon) +SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_mount) { int rc; unsigned int ret_data_len = 0; @@ -693,7 +707,7 @@ SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon) goto out; } - rc = parse_server_interfaces(out_buf, ret_data_len, ses); + rc = parse_server_interfaces(out_buf, ret_data_len, ses, in_mount); if (rc) goto out; @@ -729,7 +743,7 @@ smb3_qfs_tcon(const unsigned int xid, struct cifs_tcon *tcon, if (rc) return; - SMB3_request_interfaces(xid, tcon); + SMB3_request_interfaces(xid, tcon, true /* called during mount */); SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, FS_ATTRIBUTE_INFORMATION); @@ -787,7 +801,7 @@ smb2_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, rc = open_cached_dir(xid, tcon, full_path, cifs_sb, true, &cfid); if (!rc) { - if (cfid->is_valid) { + if (cfid->has_lease) { close_cached_dir(cfid); return 0; } @@ -817,33 +831,25 @@ smb2_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, return rc; } -static int -smb2_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *full_path, - u64 *uniqueid, FILE_ALL_INFO *data) +static int smb2_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + u64 *uniqueid, struct cifs_open_info_data *data) { - *uniqueid = le64_to_cpu(data->IndexNumber); + *uniqueid = le64_to_cpu(data->fi.IndexNumber); return 0; } -static int -smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_fid *fid, FILE_ALL_INFO *data) +static int smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data) { - int rc; - struct smb2_file_all_info *smb2_data; - - smb2_data = kzalloc(sizeof(struct smb2_file_all_info) + PATH_MAX * 2, - GFP_KERNEL); - if (smb2_data == NULL) - return -ENOMEM; + struct cifs_fid *fid = &cfile->fid; - rc = SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid, - smb2_data); - if (!rc) - move_smb2_info_to_cifs(data, smb2_data); - kfree(smb2_data); - return rc; + if (cfile->symlink_target) { + data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!data->symlink_target) + return -ENOMEM; + } + return SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid, &data->fi); } #ifdef CONFIG_CIFS_XATTR @@ -1327,7 +1333,7 @@ SMB2_request_res_key(const unsigned int xid, struct cifs_tcon *tcon, CIFSMaxBufSize, (char **)&res_key, &ret_data_len); if (rc == -EOPNOTSUPP) { - pr_warn_once("Server share %s does not support copy range\n", tcon->treeName); + pr_warn_once("Server share %s does not support copy range\n", tcon->tree_name); goto req_res_key_exit; } else if (rc) { cifs_tcon_dbg(VFS, "refcpy ioctl error %d getting resume key\n", rc); @@ -1600,17 +1606,8 @@ smb2_copychunk_range(const unsigned int xid, int chunks_copied = 0; bool chunk_sizes_updated = false; ssize_t bytes_written, total_bytes_written = 0; - struct inode *inode; pcchunk = kmalloc(sizeof(struct copychunk_ioctl), GFP_KERNEL); - - /* - * We need to flush all unwritten data before we can send the - * copychunk ioctl to the server. - */ - inode = d_inode(trgtfile->dentry); - filemap_write_and_wait(inode->i_mapping); - if (pcchunk == NULL) return -ENOMEM; @@ -2021,9 +2018,10 @@ smb3_enum_snapshots(const unsigned int xid, struct cifs_tcon *tcon, static int smb3_notify(const unsigned int xid, struct file *pfile, - void __user *ioc_buf) + void __user *ioc_buf, bool return_changes) { - struct smb3_notify notify; + struct smb3_notify_info notify; + struct smb3_notify_info __user *pnotify_buf; struct dentry *dentry = pfile->f_path.dentry; struct inode *inode = file_inode(pfile); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); @@ -2031,10 +2029,12 @@ smb3_notify(const unsigned int xid, struct file *pfile, struct cifs_fid fid; struct cifs_tcon *tcon; const unsigned char *path; + char *returned_ioctl_info = NULL; void *page = alloc_dentry_path(); __le16 *utf16_path = NULL; u8 oplock = SMB2_OPLOCK_LEVEL_NONE; int rc = 0; + __u32 ret_len = 0; path = build_path_from_dentry(dentry, page); if (IS_ERR(path)) { @@ -2048,9 +2048,17 @@ smb3_notify(const unsigned int xid, struct file *pfile, goto notify_exit; } - if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify))) { - rc = -EFAULT; - goto notify_exit; + if (return_changes) { + if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify_info))) { + rc = -EFAULT; + goto notify_exit; + } + } else { + if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify))) { + rc = -EFAULT; + goto notify_exit; + } + notify.data_len = 0; } tcon = cifs_sb_master_tcon(cifs_sb); @@ -2067,12 +2075,22 @@ smb3_notify(const unsigned int xid, struct file *pfile, goto notify_exit; rc = SMB2_change_notify(xid, tcon, fid.persistent_fid, fid.volatile_fid, - notify.watch_tree, notify.completion_filter); + notify.watch_tree, notify.completion_filter, + notify.data_len, &returned_ioctl_info, &ret_len); SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); cifs_dbg(FYI, "change notify for path %s rc %d\n", path, rc); - + if (return_changes && (ret_len > 0) && (notify.data_len > 0)) { + if (ret_len > notify.data_len) + ret_len = notify.data_len; + pnotify_buf = (struct smb3_notify_info __user *)ioc_buf; + if (copy_to_user(pnotify_buf->notify_data, returned_ioctl_info, ret_len)) + rc = -EFAULT; + else if (copy_to_user(&pnotify_buf->data_len, &ret_len, sizeof(ret_len))) + rc = -EFAULT; + } + kfree(returned_ioctl_info); notify_exit: free_dentry_path(page); kfree(utf16_path); @@ -2298,7 +2316,7 @@ smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) spin_unlock(&tcon->tc_lock); spin_unlock(&cifs_tcp_ses_lock); pr_warn_once("Server share %s deleted.\n", - tcon->treeName); + tcon->tree_name); return; } } @@ -2507,7 +2525,7 @@ smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, if (rc == -EREMCHG) { tcon->need_reconnect = true; pr_warn_once("server share %s deleted\n", - tcon->treeName); + tcon->tree_name); } goto qic_exit; } @@ -2823,9 +2841,6 @@ parse_reparse_point(struct reparse_data_buffer *buf, } } -#define SMB2_SYMLINK_STRUCT_SIZE \ - (sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp)) - static int smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const char *full_path, @@ -2837,13 +2852,7 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_fid fid; struct kvec err_iov = {NULL, 0}; - struct smb2_err_rsp *err_buf = NULL; - struct smb2_symlink_err_rsp *symlink; struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses); - unsigned int sub_len; - unsigned int sub_offset; - unsigned int print_len; - unsigned int print_offset; int flags = CIFS_CP_CREATE_CLOSE_OP; struct smb_rqst rqst[3]; int resp_buftype[3]; @@ -2960,47 +2969,7 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon, goto querty_exit; } - err_buf = err_iov.iov_base; - if (le32_to_cpu(err_buf->ByteCount) < sizeof(struct smb2_symlink_err_rsp) || - err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE) { - rc = -EINVAL; - goto querty_exit; - } - - symlink = (struct smb2_symlink_err_rsp *)err_buf->ErrorData; - if (le32_to_cpu(symlink->SymLinkErrorTag) != SYMLINK_ERROR_TAG || - le32_to_cpu(symlink->ReparseTag) != IO_REPARSE_TAG_SYMLINK) { - rc = -EINVAL; - goto querty_exit; - } - - /* open must fail on symlink - reset rc */ - rc = 0; - sub_len = le16_to_cpu(symlink->SubstituteNameLength); - sub_offset = le16_to_cpu(symlink->SubstituteNameOffset); - print_len = le16_to_cpu(symlink->PrintNameLength); - print_offset = le16_to_cpu(symlink->PrintNameOffset); - - if (err_iov.iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offset + sub_len) { - rc = -EINVAL; - goto querty_exit; - } - - if (err_iov.iov_len < - SMB2_SYMLINK_STRUCT_SIZE + print_offset + print_len) { - rc = -EINVAL; - goto querty_exit; - } - - *target_path = cifs_strndup_from_utf16( - (char *)symlink->PathBuffer + sub_offset, - sub_len, true, cifs_sb->local_nls); - if (!(*target_path)) { - rc = -ENOMEM; - goto querty_exit; - } - convert_delimiter(*target_path, '/'); - cifs_dbg(FYI, "%s: target path: %s\n", __func__, *target_path); + rc = smb2_parse_symlink_response(cifs_sb, &err_iov, target_path); querty_exit: cifs_dbg(FYI, "query symlink rc %d\n", rc); @@ -3307,26 +3276,43 @@ get_smb2_acl(struct cifs_sb_info *cifs_sb, return pntsd; } +static long smb3_zero_data(struct file *file, struct cifs_tcon *tcon, + loff_t offset, loff_t len, unsigned int xid) +{ + struct cifsFileInfo *cfile = file->private_data; + struct file_zero_data_information fsctl_buf; + + cifs_dbg(FYI, "Offset %lld len %lld\n", offset, len); + + fsctl_buf.FileOffset = cpu_to_le64(offset); + fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len); + + return SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, + (char *)&fsctl_buf, + sizeof(struct file_zero_data_information), + 0, NULL, NULL); +} + static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, loff_t offset, loff_t len, bool keep_size) { struct cifs_ses *ses = tcon->ses; - struct inode *inode; - struct cifsInodeInfo *cifsi; + struct inode *inode = file_inode(file); + struct cifsInodeInfo *cifsi = CIFS_I(inode); struct cifsFileInfo *cfile = file->private_data; - struct file_zero_data_information fsctl_buf; long rc; unsigned int xid; __le64 eof; xid = get_xid(); - inode = d_inode(cfile->dentry); - cifsi = CIFS_I(inode); - trace_smb3_zero_enter(xid, cfile->fid.persistent_fid, tcon->tid, ses->Suid, offset, len); + inode_lock(inode); + filemap_invalidate_lock(inode->i_mapping); + /* * We zero the range through ioctl, so we need remove the page caches * first, otherwise the data may be inconsistent with the server. @@ -3334,26 +3320,12 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, truncate_pagecache_range(inode, offset, offset + len - 1); /* if file not oplocked can't be sure whether asking to extend size */ - if (!CIFS_CACHE_READ(cifsi)) - if (keep_size == false) { - rc = -EOPNOTSUPP; - trace_smb3_zero_err(xid, cfile->fid.persistent_fid, - tcon->tid, ses->Suid, offset, len, rc); - free_xid(xid); - return rc; - } - - cifs_dbg(FYI, "Offset %lld len %lld\n", offset, len); - - fsctl_buf.FileOffset = cpu_to_le64(offset); - fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len); + rc = -EOPNOTSUPP; + if (keep_size == false && !CIFS_CACHE_READ(cifsi)) + goto zero_range_exit; - rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, - cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, - (char *)&fsctl_buf, - sizeof(struct file_zero_data_information), - 0, NULL, NULL); - if (rc) + rc = smb3_zero_data(file, tcon, offset, len, xid); + if (rc < 0) goto zero_range_exit; /* @@ -3366,6 +3338,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, } zero_range_exit: + filemap_invalidate_unlock(inode->i_mapping); + inode_unlock(inode); free_xid(xid); if (rc) trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, @@ -3379,7 +3353,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, loff_t offset, loff_t len) { - struct inode *inode; + struct inode *inode = file_inode(file); struct cifsFileInfo *cfile = file->private_data; struct file_zero_data_information fsctl_buf; long rc; @@ -3388,14 +3362,12 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, xid = get_xid(); - inode = d_inode(cfile->dentry); - + inode_lock(inode); /* Need to make file sparse, if not already, before freeing range. */ /* Consider adding equivalent for compressed since it could also work */ if (!smb2_set_sparse(xid, tcon, cfile, inode, set_sparse)) { rc = -EOPNOTSUPP; - free_xid(xid); - return rc; + goto out; } filemap_invalidate_lock(inode->i_mapping); @@ -3415,8 +3387,10 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, (char *)&fsctl_buf, sizeof(struct file_zero_data_information), CIFSMaxBufSize, NULL, NULL); - free_xid(xid); filemap_invalidate_unlock(inode->i_mapping); +out: + inode_unlock(inode); + free_xid(xid); return rc; } @@ -3673,39 +3647,50 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, { int rc; unsigned int xid; - struct inode *inode; + struct inode *inode = file_inode(file); struct cifsFileInfo *cfile = file->private_data; - struct cifsInodeInfo *cifsi; + struct cifsInodeInfo *cifsi = CIFS_I(inode); __le64 eof; + loff_t old_eof; xid = get_xid(); - inode = d_inode(cfile->dentry); - cifsi = CIFS_I(inode); + inode_lock(inode); - if (off >= i_size_read(inode) || - off + len >= i_size_read(inode)) { + old_eof = i_size_read(inode); + if ((off >= old_eof) || + off + len >= old_eof) { rc = -EINVAL; goto out; } + filemap_invalidate_lock(inode->i_mapping); + rc = filemap_write_and_wait_range(inode->i_mapping, off, old_eof - 1); + if (rc < 0) + goto out_2; + + truncate_pagecache_range(inode, off, old_eof); + rc = smb2_copychunk_range(xid, cfile, cfile, off + len, - i_size_read(inode) - off - len, off); + old_eof - off - len, off); if (rc < 0) - goto out; + goto out_2; - eof = cpu_to_le64(i_size_read(inode) - len); + eof = cpu_to_le64(old_eof - len); rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, &eof); if (rc < 0) - goto out; + goto out_2; rc = 0; cifsi->server_eof = i_size_read(inode) - len; truncate_setsize(inode, cifsi->server_eof); fscache_resize_cookie(cifs_inode_cookie(inode), cifsi->server_eof); +out_2: + filemap_invalidate_unlock(inode->i_mapping); out: + inode_unlock(inode); free_xid(xid); return rc; } @@ -3716,34 +3701,47 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, int rc; unsigned int xid; struct cifsFileInfo *cfile = file->private_data; + struct inode *inode = file_inode(file); __le64 eof; - __u64 count; + __u64 count, old_eof; xid = get_xid(); - if (off >= i_size_read(file->f_inode)) { + inode_lock(inode); + + old_eof = i_size_read(inode); + if (off >= old_eof) { rc = -EINVAL; goto out; } - count = i_size_read(file->f_inode) - off; - eof = cpu_to_le64(i_size_read(file->f_inode) + len); + count = old_eof - off; + eof = cpu_to_le64(old_eof + len); + + filemap_invalidate_lock(inode->i_mapping); + rc = filemap_write_and_wait_range(inode->i_mapping, off, old_eof + len - 1); + if (rc < 0) + goto out_2; + truncate_pagecache_range(inode, off, old_eof); rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, &eof); if (rc < 0) - goto out; + goto out_2; rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len); if (rc < 0) - goto out; + goto out_2; - rc = smb3_zero_range(file, tcon, off, len, 1); + rc = smb3_zero_data(file, tcon, off, len, xid); if (rc < 0) - goto out; + goto out_2; rc = 0; +out_2: + filemap_invalidate_unlock(inode->i_mapping); out: + inode_unlock(inode); free_xid(xid); return rc; } @@ -4324,8 +4322,7 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, return rc; } - tfm = enc ? server->secmech.ccmaesencrypt : - server->secmech.ccmaesdecrypt; + tfm = enc ? server->secmech.enc : server->secmech.dec; if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) @@ -4390,11 +4387,11 @@ crypt_message(struct TCP_Server_Info *server, int num_rqst, if (!rc && enc) memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE); - kfree(iv); + kfree_sensitive(iv); free_sg: - kfree(sg); + kfree_sensitive(sg); free_req: - kfree(req); + kfree_sensitive(req); return rc; } @@ -5082,7 +5079,7 @@ smb2_make_node(unsigned int xid, struct inode *inode, { struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); int rc = -EPERM; - FILE_ALL_INFO *buf = NULL; + struct cifs_open_info_data buf = {}; struct cifs_io_parms io_parms = {0}; __u32 oplock = 0; struct cifs_fid fid; @@ -5098,7 +5095,7 @@ smb2_make_node(unsigned int xid, struct inode *inode, * and was used by default in earlier versions of Windows */ if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL)) - goto out; + return rc; /* * TODO: Add ability to create instead via reparse point. Windows (e.g. @@ -5107,16 +5104,10 @@ smb2_make_node(unsigned int xid, struct inode *inode, */ if (!S_ISCHR(mode) && !S_ISBLK(mode)) - goto out; + return rc; cifs_dbg(FYI, "sfu compat create special file\n"); - buf = kmalloc(sizeof(FILE_ALL_INFO), GFP_KERNEL); - if (buf == NULL) { - rc = -ENOMEM; - goto out; - } - oparms.tcon = tcon; oparms.cifs_sb = cifs_sb; oparms.desired_access = GENERIC_WRITE; @@ -5131,21 +5122,21 @@ smb2_make_node(unsigned int xid, struct inode *inode, oplock = REQ_OPLOCK; else oplock = 0; - rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, buf); + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, &buf); if (rc) - goto out; + return rc; /* * BB Do not bother to decode buf since no local inode yet to put * timestamps in, but we can reuse it safely. */ - pdev = (struct win_dev *)buf; + pdev = (struct win_dev *)&buf.fi; io_parms.pid = current->tgid; io_parms.tcon = tcon; io_parms.offset = 0; io_parms.length = sizeof(struct win_dev); - iov[1].iov_base = buf; + iov[1].iov_base = &buf.fi; iov[1].iov_len = sizeof(struct win_dev); if (S_ISCHR(mode)) { memcpy(pdev->type, "IntxCHR", 8); @@ -5164,8 +5155,8 @@ smb2_make_node(unsigned int xid, struct inode *inode, d_drop(dentry); /* FIXME: add code here to set EAs */ -out: - kfree(buf); + + cifs_free_open_info(&buf); return rc; } diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c index 91cfc5b47ac7c4a5cced3501bfca93bb734f9269..a2384509ea84bda9b6fb6da380d4dea2c3f4ab03 100644 --- a/fs/cifs/smb2pdu.c +++ b/fs/cifs/smb2pdu.c @@ -466,15 +466,14 @@ build_signing_ctxt(struct smb2_signing_capabilities *pneg_ctxt) /* * Context Data length must be rounded to multiple of 8 for some servers */ - pneg_ctxt->DataLength = cpu_to_le16(DIV_ROUND_UP( - sizeof(struct smb2_signing_capabilities) - - sizeof(struct smb2_neg_context) + - (num_algs * 2 /* sizeof u16 */), 8) * 8); + pneg_ctxt->DataLength = cpu_to_le16(ALIGN(sizeof(struct smb2_signing_capabilities) - + sizeof(struct smb2_neg_context) + + (num_algs * sizeof(u16)), 8)); pneg_ctxt->SigningAlgorithmCount = cpu_to_le16(num_algs); pneg_ctxt->SigningAlgorithms[0] = cpu_to_le16(SIGNING_ALG_AES_CMAC); - ctxt_len += 2 /* sizeof le16 */ * num_algs; - ctxt_len = DIV_ROUND_UP(ctxt_len, 8) * 8; + ctxt_len += sizeof(__le16) * num_algs; + ctxt_len = ALIGN(ctxt_len, 8); return ctxt_len; /* TBD add SIGNING_ALG_AES_GMAC and/or SIGNING_ALG_HMAC_SHA256 */ } @@ -511,8 +510,7 @@ build_netname_ctxt(struct smb2_netname_neg_context *pneg_ctxt, char *hostname) /* copy up to max of first 100 bytes of server name to NetName field */ pneg_ctxt->DataLength = cpu_to_le16(2 * cifs_strtoUTF16(pneg_ctxt->NetName, hostname, 100, cp)); /* context size is DataLength + minimal smb2_neg_context */ - return DIV_ROUND_UP(le16_to_cpu(pneg_ctxt->DataLength) + - sizeof(struct smb2_neg_context), 8) * 8; + return ALIGN(le16_to_cpu(pneg_ctxt->DataLength) + sizeof(struct smb2_neg_context), 8); } static void @@ -557,18 +555,18 @@ assemble_neg_contexts(struct smb2_negotiate_req *req, * round up total_len of fixed part of SMB3 negotiate request to 8 * byte boundary before adding negotiate contexts */ - *total_len = roundup(*total_len, 8); + *total_len = ALIGN(*total_len, 8); pneg_ctxt = (*total_len) + (char *)req; req->NegotiateContextOffset = cpu_to_le32(*total_len); build_preauth_ctxt((struct smb2_preauth_neg_context *)pneg_ctxt); - ctxt_len = DIV_ROUND_UP(sizeof(struct smb2_preauth_neg_context), 8) * 8; + ctxt_len = ALIGN(sizeof(struct smb2_preauth_neg_context), 8); *total_len += ctxt_len; pneg_ctxt += ctxt_len; build_encrypt_ctxt((struct smb2_encryption_neg_context *)pneg_ctxt); - ctxt_len = DIV_ROUND_UP(sizeof(struct smb2_encryption_neg_context), 8) * 8; + ctxt_len = ALIGN(sizeof(struct smb2_encryption_neg_context), 8); *total_len += ctxt_len; pneg_ctxt += ctxt_len; @@ -595,9 +593,7 @@ assemble_neg_contexts(struct smb2_negotiate_req *req, if (server->compress_algorithm) { build_compression_ctxt((struct smb2_compression_capabilities_context *) pneg_ctxt); - ctxt_len = DIV_ROUND_UP( - sizeof(struct smb2_compression_capabilities_context), - 8) * 8; + ctxt_len = ALIGN(sizeof(struct smb2_compression_capabilities_context), 8); *total_len += ctxt_len; pneg_ctxt += ctxt_len; neg_context_count++; @@ -780,7 +776,7 @@ static int smb311_decode_neg_context(struct smb2_negotiate_rsp *rsp, if (rc) break; /* offsets must be 8 byte aligned */ - clen = (clen + 7) & ~0x7; + clen = ALIGN(clen, 8); offset += clen + sizeof(struct smb2_neg_context); len_of_ctxts -= clen; } @@ -873,7 +869,7 @@ SMB2_negotiate(const unsigned int xid, struct smb2_negotiate_rsp *rsp; struct kvec iov[1]; struct kvec rsp_iov; - int rc = 0; + int rc; int resp_buftype; int blob_offset, blob_length; char *security_blob; @@ -965,16 +961,17 @@ SMB2_negotiate(const unsigned int xid, } else if (rc != 0) goto neg_exit; + rc = -EIO; if (strcmp(server->vals->version_string, SMB3ANY_VERSION_STRING) == 0) { if (rsp->DialectRevision == cpu_to_le16(SMB20_PROT_ID)) { cifs_server_dbg(VFS, "SMB2 dialect returned but not requested\n"); - return -EIO; + goto neg_exit; } else if (rsp->DialectRevision == cpu_to_le16(SMB21_PROT_ID)) { cifs_server_dbg(VFS, "SMB2.1 dialect returned but not requested\n"); - return -EIO; + goto neg_exit; } else if (rsp->DialectRevision == cpu_to_le16(SMB311_PROT_ID)) { /* ops set to 3.0 by default for default so update */ server->ops = &smb311_operations; @@ -985,7 +982,7 @@ SMB2_negotiate(const unsigned int xid, if (rsp->DialectRevision == cpu_to_le16(SMB20_PROT_ID)) { cifs_server_dbg(VFS, "SMB2 dialect returned but not requested\n"); - return -EIO; + goto neg_exit; } else if (rsp->DialectRevision == cpu_to_le16(SMB21_PROT_ID)) { /* ops set to 3.0 by default for default so update */ server->ops = &smb21_operations; @@ -999,7 +996,7 @@ SMB2_negotiate(const unsigned int xid, /* if requested single dialect ensure returned dialect matched */ cifs_server_dbg(VFS, "Invalid 0x%x dialect returned: not requested\n", le16_to_cpu(rsp->DialectRevision)); - return -EIO; + goto neg_exit; } cifs_dbg(FYI, "mode 0x%x\n", rsp->SecurityMode); @@ -1017,9 +1014,10 @@ SMB2_negotiate(const unsigned int xid, else { cifs_server_dbg(VFS, "Invalid dialect returned by server 0x%x\n", le16_to_cpu(rsp->DialectRevision)); - rc = -EIO; goto neg_exit; } + + rc = 0; server->dialect = le16_to_cpu(rsp->DialectRevision); /* @@ -1167,9 +1165,9 @@ int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon) pneg_inbuf->Dialects[0] = cpu_to_le16(server->vals->protocol_id); pneg_inbuf->DialectCount = cpu_to_le16(1); - /* structure is big enough for 3 dialects, sending only 1 */ + /* structure is big enough for 4 dialects, sending only 1 */ inbuflen = sizeof(*pneg_inbuf) - - sizeof(pneg_inbuf->Dialects[0]) * 2; + sizeof(pneg_inbuf->Dialects[0]) * 3; } rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, @@ -1343,6 +1341,13 @@ SMB2_sess_alloc_buffer(struct SMB2_sess_data *sess_data) static void SMB2_sess_free_buffer(struct SMB2_sess_data *sess_data) { + int i; + + /* zero the session data before freeing, as it might contain sensitive info (keys, etc) */ + for (i = 0; i < 2; i++) + if (sess_data->iov[i].iov_base) + memzero_explicit(sess_data->iov[i].iov_base, sess_data->iov[i].iov_len); + free_rsp_buf(sess_data->buf0_type, sess_data->iov[0].iov_base); sess_data->buf0_type = CIFS_NO_BUFFER; } @@ -1475,6 +1480,8 @@ SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) out_put_spnego_key: key_invalidate(spnego_key); key_put(spnego_key); + if (rc) + kfree_sensitive(ses->auth_key.response); out: sess_data->result = rc; sess_data->func = NULL; @@ -1571,7 +1578,7 @@ SMB2_sess_auth_rawntlmssp_negotiate(struct SMB2_sess_data *sess_data) } out: - kfree(ntlmssp_blob); + memzero_explicit(ntlmssp_blob, blob_length); SMB2_sess_free_buffer(sess_data); if (!rc) { sess_data->result = 0; @@ -1579,7 +1586,7 @@ out: return; } out_err: - kfree(ses->ntlmssp); + kfree_sensitive(ses->ntlmssp); ses->ntlmssp = NULL; sess_data->result = rc; sess_data->func = NULL; @@ -1655,9 +1662,9 @@ SMB2_sess_auth_rawntlmssp_authenticate(struct SMB2_sess_data *sess_data) } #endif out: - kfree(ntlmssp_blob); + memzero_explicit(ntlmssp_blob, blob_length); SMB2_sess_free_buffer(sess_data); - kfree(ses->ntlmssp); + kfree_sensitive(ses->ntlmssp); ses->ntlmssp = NULL; sess_data->result = rc; sess_data->func = NULL; @@ -1735,7 +1742,7 @@ SMB2_sess_setup(const unsigned int xid, struct cifs_ses *ses, cifs_server_dbg(VFS, "signing requested but authenticated as guest\n"); rc = sess_data->result; out: - kfree(sess_data); + kfree_sensitive(sess_data); return rc; } @@ -1928,7 +1935,7 @@ SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, tcon->capabilities = rsp->Capabilities; /* we keep caps little endian */ tcon->maximal_access = le32_to_cpu(rsp->MaximalAccess); tcon->tid = le32_to_cpu(rsp->hdr.Id.SyncId.TreeId); - strscpy(tcon->treeName, tree, sizeof(tcon->treeName)); + strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); if ((rsp->Capabilities & SMB2_SHARE_CAP_DFS) && ((tcon->share_flags & SHI1005_FLAGS_DFS) == 0)) @@ -1971,6 +1978,7 @@ SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon) if (!ses || !(ses->server)) return -EIO; + trace_smb3_tdis_enter(xid, tcon->tid, ses->Suid, tcon->tree_name); spin_lock(&ses->chan_lock); if ((tcon->need_reconnect) || (CIFS_ALL_CHANS_NEED_RECONNECT(tcon->ses))) { @@ -2002,8 +2010,11 @@ SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon) rc = cifs_send_recv(xid, ses, ses->server, &rqst, &resp_buf_type, flags, &rsp_iov); cifs_small_buf_release(req); - if (rc) + if (rc) { cifs_stats_fail_inc(tcon, SMB2_TREE_DISCONNECT_HE); + trace_smb3_tdis_err(xid, tcon->tid, ses->Suid, rc); + } + trace_smb3_tdis_done(xid, tcon->tid, ses->Suid); return rc; } @@ -2409,9 +2420,9 @@ create_sd_buf(umode_t mode, bool set_owner, unsigned int *len) unsigned int acelen, acl_size, ace_count; unsigned int owner_offset = 0; unsigned int group_offset = 0; - struct smb3_acl acl; + struct smb3_acl acl = {}; - *len = roundup(sizeof(struct crt_sd_ctxt) + (sizeof(struct cifs_ace) * 4), 8); + *len = round_up(sizeof(struct crt_sd_ctxt) + (sizeof(struct cifs_ace) * 4), 8); if (set_owner) { /* sizeof(struct owner_group_sids) is already multiple of 8 so no need to round */ @@ -2482,10 +2493,11 @@ create_sd_buf(umode_t mode, bool set_owner, unsigned int *len) acl.AclRevision = ACL_REVISION; /* See 2.4.4.1 of MS-DTYP */ acl.AclSize = cpu_to_le16(acl_size); acl.AceCount = cpu_to_le16(ace_count); + /* acl.Sbz1 and Sbz2 MBZ so are not set here, but initialized above */ memcpy(aclptr, &acl, sizeof(struct smb3_acl)); buf->ccontext.DataLength = cpu_to_le32(ptr - (__u8 *)&buf->sd); - *len = roundup(ptr - (__u8 *)buf, 8); + *len = round_up((unsigned int)(ptr - (__u8 *)buf), 8); return buf; } @@ -2572,19 +2584,15 @@ alloc_path_with_tree_prefix(__le16 **out_path, int *out_size, int *out_len, path_len = UniStrnlen((wchar_t *)path, PATH_MAX); - /* - * make room for one path separator between the treename and - * path - */ - *out_len = treename_len + 1 + path_len; + /* make room for one path separator only if @path isn't empty */ + *out_len = treename_len + (path[0] ? 1 : 0) + path_len; /* - * final path needs to be null-terminated UTF16 with a - * size aligned to 8 + * final path needs to be 8-byte aligned as specified in + * MS-SMB2 2.2.13 SMB2 CREATE Request. */ - - *out_size = roundup((*out_len+1)*2, 8); - *out_path = kzalloc(*out_size, GFP_KERNEL); + *out_size = round_up(*out_len * sizeof(__le16), 8); + *out_path = kzalloc(*out_size + sizeof(__le16) /* null */, GFP_KERNEL); if (!*out_path) return -ENOMEM; @@ -2676,7 +2684,7 @@ int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS; rc = alloc_path_with_tree_prefix(©_path, ©_size, &name_len, - tcon->treeName, utf16_path); + tcon->tree_name, utf16_path); if (rc) goto err_free_req; @@ -2818,7 +2826,7 @@ SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS; rc = alloc_path_with_tree_prefix(©_path, ©_size, &name_len, - tcon->treeName, path); + tcon->tree_name, path); if (rc) return rc; req->NameLength = cpu_to_le16(name_len * 2); @@ -2828,9 +2836,7 @@ SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, uni_path_len = (2 * UniStrnlen((wchar_t *)path, PATH_MAX)) + 2; /* MUST set path len (NameLength) to 0 opening root of share */ req->NameLength = cpu_to_le16(uni_path_len - 2); - copy_size = uni_path_len; - if (copy_size % 8 != 0) - copy_size = roundup(copy_size, 8); + copy_size = round_up(uni_path_len, 8); copy_path = kzalloc(copy_size, GFP_KERNEL); if (!copy_path) return -ENOMEM; @@ -3013,7 +3019,7 @@ SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, oparms->create_options, oparms->desired_access, rc); if (rc == -EREMCHG) { pr_warn_once("server share %s deleted\n", - tcon->treeName); + tcon->tree_name); tcon->need_reconnect = true; } goto creat_exit; @@ -3474,7 +3480,7 @@ smb2_validate_and_copy_iov(unsigned int offset, unsigned int buffer_length, if (rc) return rc; - memcpy(data, begin_of_buf, buffer_length); + memcpy(data, begin_of_buf, minbufsize); return 0; } @@ -3598,7 +3604,7 @@ query_info(const unsigned int xid, struct cifs_tcon *tcon, rc = smb2_validate_and_copy_iov(le16_to_cpu(rsp->OutputBufferOffset), le32_to_cpu(rsp->OutputBufferLength), - &rsp_iov, min_len, *data); + &rsp_iov, dlen ? *dlen : min_len, *data); if (rc && allocated) { kfree(*data); *data = NULL; @@ -3704,11 +3710,13 @@ SMB2_notify_init(const unsigned int xid, struct smb_rqst *rqst, int SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, bool watch_tree, - u32 completion_filter) + u32 completion_filter, u32 max_out_data_len, char **out_data, + u32 *plen /* returned data len */) { struct cifs_ses *ses = tcon->ses; struct TCP_Server_Info *server = cifs_pick_channel(ses); struct smb_rqst rqst; + struct smb2_change_notify_rsp *smb_rsp; struct kvec iov[1]; struct kvec rsp_iov = {NULL, 0}; int resp_buftype = CIFS_NO_BUFFER; @@ -3724,6 +3732,9 @@ SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, memset(&rqst, 0, sizeof(struct smb_rqst)); memset(&iov, 0, sizeof(iov)); + if (plen) + *plen = 0; + rqst.rq_iov = iov; rqst.rq_nvec = 1; @@ -3742,9 +3753,28 @@ SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, cifs_stats_fail_inc(tcon, SMB2_CHANGE_NOTIFY_HE); trace_smb3_notify_err(xid, persistent_fid, tcon->tid, ses->Suid, (u8)watch_tree, completion_filter, rc); - } else + } else { trace_smb3_notify_done(xid, persistent_fid, tcon->tid, - ses->Suid, (u8)watch_tree, completion_filter); + ses->Suid, (u8)watch_tree, completion_filter); + /* validate that notify information is plausible */ + if ((rsp_iov.iov_base == NULL) || + (rsp_iov.iov_len < sizeof(struct smb2_change_notify_rsp))) + goto cnotify_exit; + + smb_rsp = (struct smb2_change_notify_rsp *)rsp_iov.iov_base; + + smb2_validate_iov(le16_to_cpu(smb_rsp->OutputBufferOffset), + le32_to_cpu(smb_rsp->OutputBufferLength), &rsp_iov, + sizeof(struct file_notify_information)); + + *out_data = kmemdup((char *)smb_rsp + le16_to_cpu(smb_rsp->OutputBufferOffset), + le32_to_cpu(smb_rsp->OutputBufferLength), GFP_KERNEL); + if (*out_data == NULL) { + rc = -ENOMEM; + goto cnotify_exit; + } else + *plen = le32_to_cpu(smb_rsp->OutputBufferLength); + } cnotify_exit: if (rqst.rq_iov) @@ -4092,7 +4122,7 @@ smb2_new_read_req(void **buf, unsigned int *total_len, if (request_type & CHAINED_REQUEST) { if (!(request_type & END_OF_CHAIN)) { /* next 8-byte aligned request */ - *total_len = DIV_ROUND_UP(*total_len, 8) * 8; + *total_len = ALIGN(*total_len, 8); shdr->NextCommand = cpu_to_le32(*total_len); } else /* END_OF_CHAIN */ shdr->NextCommand = 0; @@ -4431,7 +4461,7 @@ smb2_writev_callback(struct mid_q_entry *mid) wdata->bytes, wdata->result); if (wdata->result == -ENOSPC) pr_warn_once("Out of space writing to %s\n", - tcon->treeName); + tcon->tree_name); } else trace_smb3_write_done(0 /* no xid */, wdata->cfile->fid.persistent_fid, diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h index f57881b8464fb372509b5a97b2c479c1fae52aac..1237bb86e93a83995b806c0b5a073beef99c8aaa 100644 --- a/fs/cifs/smb2pdu.h +++ b/fs/cifs/smb2pdu.h @@ -56,6 +56,9 @@ struct smb2_rdma_crypto_transform { #define COMPOUND_FID 0xFFFFFFFFFFFFFFFFULL +#define SMB2_SYMLINK_STRUCT_SIZE \ + (sizeof(struct smb2_err_rsp) - 1 + sizeof(struct smb2_symlink_err_rsp)) + #define SYMLINK_ERROR_TAG 0x4c4d5953 struct smb2_symlink_err_rsp { diff --git a/fs/cifs/smb2proto.h b/fs/cifs/smb2proto.h index 3f740f24b96a7750be18f3b7a2bb02da46b031d0..be21b5d26f67efa9f4f3130d9f94efb542b4629f 100644 --- a/fs/cifs/smb2proto.h +++ b/fs/cifs/smb2proto.h @@ -53,16 +53,12 @@ extern bool smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv); extern int smb3_handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid); - -extern void move_smb2_info_to_cifs(FILE_ALL_INFO *dst, - struct smb2_file_all_info *src); extern int smb2_query_reparse_tag(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const char *path, __u32 *reparse_tag); -extern int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, - const char *full_path, FILE_ALL_INFO *data, - bool *adjust_tz, bool *symlink); +int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse); extern int smb2_set_path_size(const unsigned int xid, struct cifs_tcon *tcon, const char *full_path, __u64 size, struct cifs_sb_info *cifs_sb, bool set_alloc); @@ -95,9 +91,9 @@ extern int smb3_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const unsigned char *path, char *pbuf, unsigned int *pbytes_read); -extern int smb2_open_file(const unsigned int xid, - struct cifs_open_parms *oparms, - __u32 *oplock, FILE_ALL_INFO *buf); +int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec *iov, char **path); +int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, + void *buf); extern int smb2_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, const unsigned int xid); extern int smb2_push_mandatory_locks(struct cifsFileInfo *cfile); @@ -148,7 +144,8 @@ extern int SMB2_ioctl_init(struct cifs_tcon *tcon, extern void SMB2_ioctl_free(struct smb_rqst *rqst); extern int SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, bool watch_tree, - u32 completion_filter); + u32 completion_filter, u32 max_out_data_len, + char **out_data, u32 *plen /* returned data len */); extern int __SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, @@ -278,9 +275,9 @@ extern int smb2_query_info_compound(const unsigned int xid, struct kvec *rsp, int *buftype, struct cifs_sb_info *cifs_sb); /* query path info from the server using SMB311 POSIX extensions*/ -extern int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *sb, const char *path, struct smb311_posix_qinfo *qinf, - bool *adjust_tx, bool *symlink); +int smb311_posix_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data, bool *adjust_tz, bool *reparse); int posix_info_parse(const void *beg, const void *end, struct smb2_posix_info_parsed *out); int posix_info_sid_size(const void *beg, const void *end); diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c index 1a5fc3314dbf5790dcf14de8e8039a9d95f92fcd..8e3f26e6f6b9b46727c623012c51427e6b597e28 100644 --- a/fs/cifs/smb2transport.c +++ b/fs/cifs/smb2transport.c @@ -32,19 +32,17 @@ smb3_crypto_shash_allocate(struct TCP_Server_Info *server) struct cifs_secmech *p = &server->secmech; int rc; - rc = cifs_alloc_hash("hmac(sha256)", - &p->hmacsha256, - &p->sdeschmacsha256); + rc = cifs_alloc_hash("hmac(sha256)", &p->hmacsha256); if (rc) goto err; - rc = cifs_alloc_hash("cmac(aes)", &p->cmacaes, &p->sdesccmacaes); + rc = cifs_alloc_hash("cmac(aes)", &p->aes_cmac); if (rc) goto err; return 0; err: - cifs_free_hash(&p->hmacsha256, &p->sdeschmacsha256); + cifs_free_hash(&p->hmacsha256); return rc; } @@ -54,25 +52,23 @@ smb311_crypto_shash_allocate(struct TCP_Server_Info *server) struct cifs_secmech *p = &server->secmech; int rc = 0; - rc = cifs_alloc_hash("hmac(sha256)", - &p->hmacsha256, - &p->sdeschmacsha256); + rc = cifs_alloc_hash("hmac(sha256)", &p->hmacsha256); if (rc) return rc; - rc = cifs_alloc_hash("cmac(aes)", &p->cmacaes, &p->sdesccmacaes); + rc = cifs_alloc_hash("cmac(aes)", &p->aes_cmac); if (rc) goto err; - rc = cifs_alloc_hash("sha512", &p->sha512, &p->sdescsha512); + rc = cifs_alloc_hash("sha512", &p->sha512); if (rc) goto err; return 0; err: - cifs_free_hash(&p->cmacaes, &p->sdesccmacaes); - cifs_free_hash(&p->hmacsha256, &p->sdeschmacsha256); + cifs_free_hash(&p->aes_cmac); + cifs_free_hash(&p->hmacsha256); return rc; } @@ -219,34 +215,30 @@ smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, struct kvec *iov = rqst->rq_iov; struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base; struct cifs_ses *ses; - struct shash_desc *shash; - struct crypto_shash *hash; - struct sdesc *sdesc = NULL; + struct shash_desc *shash = NULL; struct smb_rqst drqst; ses = smb2_find_smb_ses(server, le64_to_cpu(shdr->SessionId)); - if (!ses) { + if (unlikely(!ses)) { cifs_server_dbg(VFS, "%s: Could not find session\n", __func__); - return 0; + return -ENOENT; } memset(smb2_signature, 0x0, SMB2_HMACSHA256_SIZE); memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE); if (allocate_crypto) { - rc = cifs_alloc_hash("hmac(sha256)", &hash, &sdesc); + rc = cifs_alloc_hash("hmac(sha256)", &shash); if (rc) { cifs_server_dbg(VFS, "%s: sha256 alloc failed\n", __func__); goto out; } - shash = &sdesc->shash; } else { - hash = server->secmech.hmacsha256; - shash = &server->secmech.sdeschmacsha256->shash; + shash = server->secmech.hmacsha256; } - rc = crypto_shash_setkey(hash, ses->auth_key.response, + rc = crypto_shash_setkey(shash->tfm, ses->auth_key.response, SMB2_NTLMV2_SESSKEY_SIZE); if (rc) { cifs_server_dbg(VFS, @@ -288,7 +280,7 @@ smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, out: if (allocate_crypto) - cifs_free_hash(&hash, &sdesc); + cifs_free_hash(&shash); if (ses) cifs_put_smb_ses(ses); return rc; @@ -315,42 +307,38 @@ static int generate_key(struct cifs_ses *ses, struct kvec label, goto smb3signkey_ret; } - rc = crypto_shash_setkey(server->secmech.hmacsha256, + rc = crypto_shash_setkey(server->secmech.hmacsha256->tfm, ses->auth_key.response, SMB2_NTLMV2_SESSKEY_SIZE); if (rc) { cifs_server_dbg(VFS, "%s: Could not set with session key\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_init(&server->secmech.sdeschmacsha256->shash); + rc = crypto_shash_init(server->secmech.hmacsha256); if (rc) { cifs_server_dbg(VFS, "%s: Could not init sign hmac\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - i, 4); + rc = crypto_shash_update(server->secmech.hmacsha256, i, 4); if (rc) { cifs_server_dbg(VFS, "%s: Could not update with n\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - label.iov_base, label.iov_len); + rc = crypto_shash_update(server->secmech.hmacsha256, label.iov_base, label.iov_len); if (rc) { cifs_server_dbg(VFS, "%s: Could not update with label\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - &zero, 1); + rc = crypto_shash_update(server->secmech.hmacsha256, &zero, 1); if (rc) { cifs_server_dbg(VFS, "%s: Could not update with zero\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - context.iov_base, context.iov_len); + rc = crypto_shash_update(server->secmech.hmacsha256, context.iov_base, context.iov_len); if (rc) { cifs_server_dbg(VFS, "%s: Could not update with context\n", __func__); goto smb3signkey_ret; @@ -358,19 +346,16 @@ static int generate_key(struct cifs_ses *ses, struct kvec label, if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) { - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - L256, 4); + rc = crypto_shash_update(server->secmech.hmacsha256, L256, 4); } else { - rc = crypto_shash_update(&server->secmech.sdeschmacsha256->shash, - L128, 4); + rc = crypto_shash_update(server->secmech.hmacsha256, L128, 4); } if (rc) { cifs_server_dbg(VFS, "%s: Could not update with L\n", __func__); goto smb3signkey_ret; } - rc = crypto_shash_final(&server->secmech.sdeschmacsha256->shash, - hashptr); + rc = crypto_shash_final(server->secmech.hmacsha256, hashptr); if (rc) { cifs_server_dbg(VFS, "%s: Could not generate sha256 hash\n", __func__); goto smb3signkey_ret; @@ -550,38 +535,35 @@ smb3_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, unsigned char *sigptr = smb3_signature; struct kvec *iov = rqst->rq_iov; struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base; - struct shash_desc *shash; - struct crypto_shash *hash; - struct sdesc *sdesc = NULL; + struct shash_desc *shash = NULL; struct smb_rqst drqst; u8 key[SMB3_SIGN_KEY_SIZE]; rc = smb2_get_sign_key(le64_to_cpu(shdr->SessionId), server, key); - if (rc) - return 0; + if (unlikely(rc)) { + cifs_server_dbg(VFS, "%s: Could not get signing key\n", __func__); + return rc; + } if (allocate_crypto) { - rc = cifs_alloc_hash("cmac(aes)", &hash, &sdesc); + rc = cifs_alloc_hash("cmac(aes)", &shash); if (rc) return rc; - - shash = &sdesc->shash; } else { - hash = server->secmech.cmacaes; - shash = &server->secmech.sdesccmacaes->shash; + shash = server->secmech.aes_cmac; } memset(smb3_signature, 0x0, SMB2_CMACAES_SIZE); memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE); - rc = crypto_shash_setkey(hash, key, SMB2_CMACAES_SIZE); + rc = crypto_shash_setkey(shash->tfm, key, SMB2_CMACAES_SIZE); if (rc) { cifs_server_dbg(VFS, "%s: Could not set key for cmac aes\n", __func__); goto out; } /* - * we already allocate sdesccmacaes when we init smb3 signing key, + * we already allocate aes_cmac when we init smb3 signing key, * so unlike smb2 case we do not have to check here if secmech are * initialized */ @@ -617,7 +599,7 @@ smb3_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, out: if (allocate_crypto) - cifs_free_hash(&hash, &sdesc); + cifs_free_hash(&shash); return rc; } @@ -902,7 +884,7 @@ smb3_crypto_aead_allocate(struct TCP_Server_Info *server) { struct crypto_aead *tfm; - if (!server->secmech.ccmaesencrypt) { + if (!server->secmech.enc) { if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) tfm = crypto_alloc_aead("gcm(aes)", 0, 0); @@ -913,23 +895,23 @@ smb3_crypto_aead_allocate(struct TCP_Server_Info *server) __func__); return PTR_ERR(tfm); } - server->secmech.ccmaesencrypt = tfm; + server->secmech.enc = tfm; } - if (!server->secmech.ccmaesdecrypt) { + if (!server->secmech.dec) { if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) tfm = crypto_alloc_aead("gcm(aes)", 0, 0); else tfm = crypto_alloc_aead("ccm(aes)", 0, 0); if (IS_ERR(tfm)) { - crypto_free_aead(server->secmech.ccmaesencrypt); - server->secmech.ccmaesencrypt = NULL; + crypto_free_aead(server->secmech.enc); + server->secmech.enc = NULL; cifs_server_dbg(VFS, "%s: Failed to alloc decrypt aead\n", __func__); return PTR_ERR(tfm); } - server->secmech.ccmaesdecrypt = tfm; + server->secmech.dec = tfm; } return 0; diff --git a/fs/cifs/smbdirect.c b/fs/cifs/smbdirect.c index 5fbbec22bcc8ba4d33441cd7a256b483ec23e995..90789aaa6567e67c67476a99f6b9c7580be87af1 100644 --- a/fs/cifs/smbdirect.c +++ b/fs/cifs/smbdirect.c @@ -90,7 +90,7 @@ int smbd_max_send_size = 1364; int smbd_max_fragmented_recv_size = 1024 * 1024; /* The maximum single-message size which can be received */ -int smbd_max_receive_size = 8192; +int smbd_max_receive_size = 1364; /* The timeout to initiate send of a keepalive message on idle */ int smbd_keep_alive_interval = 120; @@ -99,7 +99,7 @@ int smbd_keep_alive_interval = 120; * User configurable initial values for RDMA transport * The actual values used may be lower and are limited to hardware capabilities */ -/* Default maximum number of SGEs in a RDMA write/read */ +/* Default maximum number of pages in a single RDMA write/read */ int smbd_max_frmr_depth = 2048; /* If payload is less than this byte, use RDMA send/recv not read/write */ @@ -270,7 +270,7 @@ static void send_done(struct ib_cq *cq, struct ib_wc *wc) struct smbd_request *request = container_of(wc->wr_cqe, struct smbd_request, cqe); - log_rdma_send(INFO, "smbd_request %p completed wc->status=%d\n", + log_rdma_send(INFO, "smbd_request 0x%p completed wc->status=%d\n", request, wc->status); if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { @@ -448,7 +448,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) struct smbd_connection *info = response->info; int data_length = 0; - log_rdma_recv(INFO, "response=%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%x\n", + log_rdma_recv(INFO, "response=0x%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%u\n", response, response->type, wc->status, wc->opcode, wc->byte_len, wc->pkey_index); @@ -723,7 +723,7 @@ static int smbd_post_send_negotiate_req(struct smbd_connection *info) send_wr.opcode = IB_WR_SEND; send_wr.send_flags = IB_SEND_SIGNALED; - log_rdma_send(INFO, "sge addr=%llx length=%x lkey=%x\n", + log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n", request->sge[0].addr, request->sge[0].length, request->sge[0].lkey); @@ -792,7 +792,7 @@ static int smbd_post_send(struct smbd_connection *info, for (i = 0; i < request->num_sge; i++) { log_rdma_send(INFO, - "rdma_request sge[%d] addr=%llu length=%u\n", + "rdma_request sge[%d] addr=0x%llx length=%u\n", i, request->sge[i].addr, request->sge[i].length); ib_dma_sync_single_for_device( info->id->device, @@ -1017,9 +1017,9 @@ static int smbd_post_send_data( { int i; u32 data_length = 0; - struct scatterlist sgl[SMBDIRECT_MAX_SGE]; + struct scatterlist sgl[SMBDIRECT_MAX_SEND_SGE - 1]; - if (n_vec > SMBDIRECT_MAX_SGE) { + if (n_vec > SMBDIRECT_MAX_SEND_SGE - 1) { cifs_dbg(VFS, "Can't fit data to SGL, n_vec=%d\n", n_vec); return -EINVAL; } @@ -1079,7 +1079,7 @@ static int smbd_negotiate(struct smbd_connection *info) response->type = SMBD_NEGOTIATE_RESP; rc = smbd_post_recv(info, response); - log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=%llx iov.length=%x iov.lkey=%x\n", + log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n", rc, response->sge.addr, response->sge.length, response->sge.lkey); if (rc) @@ -1539,7 +1539,7 @@ static struct smbd_connection *_smbd_get_connection( if (smbd_send_credit_target > info->id->device->attrs.max_cqe || smbd_send_credit_target > info->id->device->attrs.max_qp_wr) { - log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", + log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", smbd_send_credit_target, info->id->device->attrs.max_cqe, info->id->device->attrs.max_qp_wr); @@ -1548,7 +1548,7 @@ static struct smbd_connection *_smbd_get_connection( if (smbd_receive_credit_max > info->id->device->attrs.max_cqe || smbd_receive_credit_max > info->id->device->attrs.max_qp_wr) { - log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n", + log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", smbd_receive_credit_max, info->id->device->attrs.max_cqe, info->id->device->attrs.max_qp_wr); @@ -1562,17 +1562,15 @@ static struct smbd_connection *_smbd_get_connection( info->max_receive_size = smbd_max_receive_size; info->keep_alive_interval = smbd_keep_alive_interval; - if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SGE) { + if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE || + info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) { log_rdma_event(ERR, - "warning: device max_send_sge = %d too small\n", - info->id->device->attrs.max_send_sge); - log_rdma_event(ERR, "Queue Pair creation may fail\n"); - } - if (info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_SGE) { - log_rdma_event(ERR, - "warning: device max_recv_sge = %d too small\n", + "device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", + IB_DEVICE_NAME_MAX, + info->id->device->name, + info->id->device->attrs.max_send_sge, info->id->device->attrs.max_recv_sge); - log_rdma_event(ERR, "Queue Pair creation may fail\n"); + goto config_failed; } info->send_cq = NULL; @@ -1598,8 +1596,8 @@ static struct smbd_connection *_smbd_get_connection( qp_attr.qp_context = info; qp_attr.cap.max_send_wr = info->send_credit_target; qp_attr.cap.max_recv_wr = info->receive_credit_max; - qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SGE; - qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_SGE; + qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SEND_SGE; + qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_RECV_SGE; qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC; @@ -1986,10 +1984,11 @@ int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst_array) { struct smbd_connection *info = server->smbd_conn; - struct kvec vec; + struct kvec vecs[SMBDIRECT_MAX_SEND_SGE - 1]; int nvecs; int size; unsigned int buflen, remaining_data_length; + unsigned int offset, remaining_vec_data_length; int start, i, j; int max_iov_size = info->max_send_size - sizeof(struct smbd_data_transfer); @@ -1998,10 +1997,8 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst; int rqst_idx; - if (info->transport_status != SMBD_CONNECTED) { - rc = -EAGAIN; - goto done; - } + if (info->transport_status != SMBD_CONNECTED) + return -EAGAIN; /* * Add in the page array if there is one. The caller needs to set @@ -2012,125 +2009,95 @@ int smbd_send(struct TCP_Server_Info *server, for (i = 0; i < num_rqst; i++) remaining_data_length += smb_rqst_len(server, &rqst_array[i]); - if (remaining_data_length > info->max_fragmented_send_size) { + if (unlikely(remaining_data_length > info->max_fragmented_send_size)) { + /* assertion: payload never exceeds negotiated maximum */ log_write(ERR, "payload size %d > max size %d\n", remaining_data_length, info->max_fragmented_send_size); - rc = -EINVAL; - goto done; + return -EINVAL; } log_write(INFO, "num_rqst=%d total length=%u\n", num_rqst, remaining_data_length); rqst_idx = 0; -next_rqst: - rqst = &rqst_array[rqst_idx]; - iov = rqst->rq_iov; - - cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n", - rqst_idx, smb_rqst_len(server, rqst)); - for (i = 0; i < rqst->rq_nvec; i++) - dump_smb(iov[i].iov_base, iov[i].iov_len); - - - log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d rq_tailsz=%d buflen=%lu\n", - rqst_idx, rqst->rq_nvec, rqst->rq_npages, rqst->rq_pagesz, - rqst->rq_tailsz, smb_rqst_len(server, rqst)); - - start = i = 0; - buflen = 0; - while (true) { - buflen += iov[i].iov_len; - if (buflen > max_iov_size) { - if (i > start) { - remaining_data_length -= - (buflen-iov[i].iov_len); - log_write(INFO, "sending iov[] from start=%d i=%d nvecs=%d remaining_data_length=%d\n", - start, i, i - start, - remaining_data_length); - rc = smbd_post_send_data( - info, &iov[start], i-start, - remaining_data_length); - if (rc) - goto done; - } else { - /* iov[start] is too big, break it */ - nvecs = (buflen+max_iov_size-1)/max_iov_size; - log_write(INFO, "iov[%d] iov_base=%p buflen=%d break to %d vectors\n", - start, iov[start].iov_base, - buflen, nvecs); - for (j = 0; j < nvecs; j++) { - vec.iov_base = - (char *)iov[start].iov_base + - j*max_iov_size; - vec.iov_len = max_iov_size; - if (j == nvecs-1) - vec.iov_len = - buflen - - max_iov_size*(nvecs-1); - remaining_data_length -= vec.iov_len; - log_write(INFO, - "sending vec j=%d iov_base=%p iov_len=%zu remaining_data_length=%d\n", - j, vec.iov_base, vec.iov_len, - remaining_data_length); - rc = smbd_post_send_data( - info, &vec, 1, - remaining_data_length); - if (rc) - goto done; + do { + rqst = &rqst_array[rqst_idx]; + iov = rqst->rq_iov; + + cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n", + rqst_idx, smb_rqst_len(server, rqst)); + remaining_vec_data_length = 0; + for (i = 0; i < rqst->rq_nvec; i++) { + remaining_vec_data_length += iov[i].iov_len; + dump_smb(iov[i].iov_base, iov[i].iov_len); + } + + log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d rq_tailsz=%d buflen=%lu\n", + rqst_idx, rqst->rq_nvec, + rqst->rq_npages, rqst->rq_pagesz, + rqst->rq_tailsz, smb_rqst_len(server, rqst)); + + start = 0; + offset = 0; + do { + buflen = 0; + i = start; + j = 0; + while (i < rqst->rq_nvec && + j < SMBDIRECT_MAX_SEND_SGE - 1 && + buflen < max_iov_size) { + + vecs[j].iov_base = iov[i].iov_base + offset; + if (buflen + iov[i].iov_len > max_iov_size) { + vecs[j].iov_len = + max_iov_size - iov[i].iov_len; + buflen = max_iov_size; + offset = vecs[j].iov_len; + } else { + vecs[j].iov_len = + iov[i].iov_len - offset; + buflen += vecs[j].iov_len; + offset = 0; + ++i; } - i++; - if (i == rqst->rq_nvec) - break; + ++j; } + + remaining_vec_data_length -= buflen; + remaining_data_length -= buflen; + log_write(INFO, "sending %s iov[%d] from start=%d nvecs=%d remaining_data_length=%d\n", + remaining_vec_data_length > 0 ? + "partial" : "complete", + rqst->rq_nvec, start, j, + remaining_data_length); + start = i; - buflen = 0; - } else { - i++; - if (i == rqst->rq_nvec) { - /* send out all remaining vecs */ - remaining_data_length -= buflen; - log_write(INFO, "sending iov[] from start=%d i=%d nvecs=%d remaining_data_length=%d\n", - start, i, i - start, + rc = smbd_post_send_data(info, vecs, j, remaining_data_length); + if (rc) + goto done; + } while (remaining_vec_data_length > 0); + + /* now sending pages if there are any */ + for (i = 0; i < rqst->rq_npages; i++) { + rqst_page_get_length(rqst, i, &buflen, &offset); + nvecs = (buflen + max_iov_size - 1) / max_iov_size; + log_write(INFO, "sending pages buflen=%d nvecs=%d\n", + buflen, nvecs); + for (j = 0; j < nvecs; j++) { + size = min_t(unsigned int, max_iov_size, remaining_data_length); + remaining_data_length -= size; + log_write(INFO, "sending pages i=%d offset=%d size=%d remaining_data_length=%d\n", + i, j * max_iov_size + offset, size, remaining_data_length); - rc = smbd_post_send_data(info, &iov[start], - i-start, remaining_data_length); + rc = smbd_post_send_page( + info, rqst->rq_pages[i], + j*max_iov_size + offset, + size, remaining_data_length); if (rc) goto done; - break; } } - log_write(INFO, "looping i=%d buflen=%d\n", i, buflen); - } - - /* now sending pages if there are any */ - for (i = 0; i < rqst->rq_npages; i++) { - unsigned int offset; - - rqst_page_get_length(rqst, i, &buflen, &offset); - nvecs = (buflen + max_iov_size - 1) / max_iov_size; - log_write(INFO, "sending pages buflen=%d nvecs=%d\n", - buflen, nvecs); - for (j = 0; j < nvecs; j++) { - size = max_iov_size; - if (j == nvecs-1) - size = buflen - j*max_iov_size; - remaining_data_length -= size; - log_write(INFO, "sending pages i=%d offset=%d size=%d remaining_data_length=%d\n", - i, j * max_iov_size + offset, size, - remaining_data_length); - rc = smbd_post_send_page( - info, rqst->rq_pages[i], - j*max_iov_size + offset, - size, remaining_data_length); - if (rc) - goto done; - } - } - - rqst_idx++; - if (rqst_idx < num_rqst) - goto next_rqst; + } while (++rqst_idx < num_rqst); done: /* diff --git a/fs/cifs/smbdirect.h b/fs/cifs/smbdirect.h index a87fca82a7963c56f3514256c0dda2f334bc6e7b..207ef979cd51c109b1e0c10640cf14b94d2f436e 100644 --- a/fs/cifs/smbdirect.h +++ b/fs/cifs/smbdirect.h @@ -91,7 +91,7 @@ struct smbd_connection { /* Memory registrations */ /* Maximum number of RDMA read/write outstanding on this connection */ int responder_resources; - /* Maximum number of SGEs in a RDMA write/read */ + /* Maximum number of pages in a single RDMA write/read on this connection */ int max_frmr_depth; /* * If payload is less than or equal to the threshold, @@ -225,21 +225,25 @@ struct smbd_buffer_descriptor_v1 { __le32 length; } __packed; -/* Default maximum number of SGEs in a RDMA send/recv */ -#define SMBDIRECT_MAX_SGE 16 +/* Maximum number of SGEs used by smbdirect.c in any send work request */ +#define SMBDIRECT_MAX_SEND_SGE 6 + /* The context for a SMBD request */ struct smbd_request { struct smbd_connection *info; struct ib_cqe cqe; - /* the SGE entries for this packet */ - struct ib_sge sge[SMBDIRECT_MAX_SGE]; + /* the SGE entries for this work request */ + struct ib_sge sge[SMBDIRECT_MAX_SEND_SGE]; int num_sge; /* SMBD packet header follows this structure */ u8 packet[]; }; +/* Maximum number of SGEs used by smbdirect.c in any receive work request */ +#define SMBDIRECT_MAX_RECV_SGE 1 + /* The context for a SMBD response */ struct smbd_response { struct smbd_connection *info; diff --git a/fs/cifs/trace.h b/fs/cifs/trace.h index 6b88dc2e364f54672ad66282f657019e5d2e7b35..110070ba8b04e3b3e4aa2315d1cfce1c933ea809 100644 --- a/fs/cifs/trace.h +++ b/fs/cifs/trace.h @@ -372,6 +372,7 @@ DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_eof_enter); DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_info_compound_enter); DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(delete_enter); DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(mkdir_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(tdis_enter); DECLARE_EVENT_CLASS(smb3_inf_compound_done_class, @@ -409,6 +410,7 @@ DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_eof_done); DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_info_compound_done); DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(delete_done); DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(mkdir_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(tdis_done); DECLARE_EVENT_CLASS(smb3_inf_compound_err_class, @@ -451,6 +453,7 @@ DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_eof_err); DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_info_compound_err); DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(mkdir_err); DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(delete_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(tdis_err); /* * For logging SMB3 Status code and Command for responses which return errors diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c index de7aeced7e16bbe952dd3f9aeb16385bcf40de8d..575fa8f5834223f373a1e6ac7d558c309f0cfa40 100644 --- a/fs/cifs/transport.c +++ b/fs/cifs/transport.c @@ -194,10 +194,6 @@ smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg, *sent = 0; - smb_msg->msg_name = (struct sockaddr *) &server->dstaddr; - smb_msg->msg_namelen = sizeof(struct sockaddr); - smb_msg->msg_control = NULL; - smb_msg->msg_controllen = 0; if (server->noblocksnd) smb_msg->msg_flags = MSG_DONTWAIT + MSG_NOSIGNAL; else @@ -261,8 +257,8 @@ smb_rqst_len(struct TCP_Server_Info *server, struct smb_rqst *rqst) int nvec; unsigned long buflen = 0; - if (server->vals->header_preamble_size == 0 && - rqst->rq_nvec >= 2 && rqst->rq_iov[0].iov_len == 4) { + if (!is_smb1(server) && rqst->rq_nvec >= 2 && + rqst->rq_iov[0].iov_len == 4) { iov = &rqst->rq_iov[1]; nvec = rqst->rq_nvec - 1; } else { @@ -309,7 +305,7 @@ __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, sigset_t mask, oldmask; size_t total_len = 0, sent, size; struct socket *ssocket = server->ssocket; - struct msghdr smb_msg; + struct msghdr smb_msg = {}; __be32 rfc1002_marker; if (cifs_rdma_enabled(server)) { @@ -346,7 +342,7 @@ __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, sigprocmask(SIG_BLOCK, &mask, &oldmask); /* Generate a rfc1002 marker for SMB2+ */ - if (server->vals->header_preamble_size == 0) { + if (!is_smb1(server)) { struct kvec hiov = { .iov_base = &rfc1002_marker, .iov_len = 4 @@ -757,8 +753,9 @@ wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *midQ) { int error; - error = wait_event_freezekillable_unsafe(server->response_q, - midQ->mid_state != MID_REQUEST_SUBMITTED); + error = wait_event_state(server->response_q, + midQ->mid_state != MID_REQUEST_SUBMITTED, + (TASK_KILLABLE|TASK_FREEZABLE_UNSAFE)); if (error < 0) return -ERESTARTSYS; @@ -1238,7 +1235,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, buf = (char *)midQ[i]->resp_buf; resp_iov[i].iov_base = buf; resp_iov[i].iov_len = midQ[i]->resp_buf_size + - server->vals->header_preamble_size; + HEADER_PREAMBLE_SIZE(server); if (midQ[i]->large_buf) resp_buf_type[i] = CIFS_LARGE_BUFFER; @@ -1643,7 +1640,7 @@ int cifs_discard_remaining_data(struct TCP_Server_Info *server) { unsigned int rfclen = server->pdu_size; - int remaining = rfclen + server->vals->header_preamble_size - + int remaining = rfclen + HEADER_PREAMBLE_SIZE(server) - server->total_read; while (remaining > 0) { @@ -1689,8 +1686,7 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) unsigned int data_offset, data_len; struct cifs_readdata *rdata = mid->callback_data; char *buf = server->smallbuf; - unsigned int buflen = server->pdu_size + - server->vals->header_preamble_size; + unsigned int buflen = server->pdu_size + HEADER_PREAMBLE_SIZE(server); bool use_rdma_mr = false; cifs_dbg(FYI, "%s: mid=%llu offset=%llu bytes=%u\n", @@ -1724,10 +1720,10 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) /* set up first two iov for signature check and to get credits */ rdata->iov[0].iov_base = buf; - rdata->iov[0].iov_len = server->vals->header_preamble_size; - rdata->iov[1].iov_base = buf + server->vals->header_preamble_size; + rdata->iov[0].iov_len = HEADER_PREAMBLE_SIZE(server); + rdata->iov[1].iov_base = buf + HEADER_PREAMBLE_SIZE(server); rdata->iov[1].iov_len = - server->total_read - server->vals->header_preamble_size; + server->total_read - HEADER_PREAMBLE_SIZE(server); cifs_dbg(FYI, "0: iov_base=%p iov_len=%zu\n", rdata->iov[0].iov_base, rdata->iov[0].iov_len); cifs_dbg(FYI, "1: iov_base=%p iov_len=%zu\n", @@ -1752,7 +1748,7 @@ cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) } data_offset = server->ops->read_data_offset(buf) + - server->vals->header_preamble_size; + HEADER_PREAMBLE_SIZE(server); if (data_offset < server->total_read) { /* * win2k8 sometimes sends an offset of 0 when the read diff --git a/fs/coredump.c b/fs/coredump.c index 9f4aae2021093f9de95208973b4856d4947adc2d..7bad7785e8e675ac92d5ba6f55bec90658e4ca51 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -354,7 +354,7 @@ static int zap_process(struct task_struct *start, int exit_code) struct task_struct *t; int nr = 0; - /* ignore all signals except SIGKILL, see prepare_signal() */ + /* Allow SIGKILL, see prepare_signal() */ start->signal->flags = SIGNAL_GROUP_EXIT; start->signal->group_exit_code = exit_code; start->signal->group_stop_count = 0; @@ -402,9 +402,8 @@ static int coredump_wait(int exit_code, struct core_state *core_state) if (core_waiters > 0) { struct core_thread *ptr; - freezer_do_not_count(); - wait_for_completion(&core_state->startup); - freezer_count(); + wait_for_completion_state(&core_state->startup, + TASK_UNINTERRUPTIBLE|TASK_FREEZABLE); /* * Wait for all the threads to become inactive, so that * all the thread context (extended register state, like @@ -412,7 +411,7 @@ static int coredump_wait(int exit_code, struct core_state *core_state) */ ptr = core_state->dumper.next; while (ptr != NULL) { - wait_task_inactive(ptr->task, 0); + wait_task_inactive(ptr->task, TASK_ANY); ptr = ptr->next; } } @@ -832,6 +831,39 @@ static int __dump_skip(struct coredump_params *cprm, size_t nr) } } +static int dump_emit_page(struct coredump_params *cprm, struct page *page) +{ + struct bio_vec bvec = { + .bv_page = page, + .bv_offset = 0, + .bv_len = PAGE_SIZE, + }; + struct iov_iter iter; + struct file *file = cprm->file; + loff_t pos; + ssize_t n; + + if (cprm->to_skip) { + if (!__dump_skip(cprm, cprm->to_skip)) + return 0; + cprm->to_skip = 0; + } + if (cprm->written + PAGE_SIZE > cprm->limit) + return 0; + if (dump_interrupted()) + return 0; + pos = file->f_pos; + iov_iter_bvec(&iter, WRITE, &bvec, 1, PAGE_SIZE); + n = __kernel_write_iter(cprm->file, &iter, &pos); + if (n != PAGE_SIZE) + return 0; + file->f_pos = pos; + cprm->written += PAGE_SIZE; + cprm->pos += PAGE_SIZE; + + return 1; +} + int dump_emit(struct coredump_params *cprm, const void *addr, int nr) { if (cprm->to_skip) { @@ -863,7 +895,6 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, for (addr = start; addr < start + len; addr += PAGE_SIZE) { struct page *page; - int stop; /* * To avoid having to allocate page tables for virtual address @@ -874,10 +905,7 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, */ page = get_dump_page(addr); if (page) { - void *kaddr = kmap_local_page(page); - - stop = !dump_emit(cprm, kaddr, PAGE_SIZE); - kunmap_local(kaddr); + int stop = !dump_emit_page(cprm, page); put_page(page); if (stop) return 0; @@ -1072,30 +1100,20 @@ whole: return vma->vm_end - vma->vm_start; } -static struct vm_area_struct *first_vma(struct task_struct *tsk, - struct vm_area_struct *gate_vma) -{ - struct vm_area_struct *ret = tsk->mm->mmap; - - if (ret) - return ret; - return gate_vma; -} - /* * Helper function for iterating across a vma list. It ensures that the caller * will visit `gate_vma' prior to terminating the search. */ -static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma, +static struct vm_area_struct *coredump_next_vma(struct ma_state *mas, + struct vm_area_struct *vma, struct vm_area_struct *gate_vma) { - struct vm_area_struct *ret; - - ret = this_vma->vm_next; - if (ret) - return ret; - if (this_vma == gate_vma) + if (gate_vma && (vma == gate_vma)) return NULL; + + vma = mas_next(mas, ULONG_MAX); + if (vma) + return vma; return gate_vma; } @@ -1119,9 +1137,10 @@ static void free_vma_snapshot(struct coredump_params *cprm) */ static bool dump_vma_snapshot(struct coredump_params *cprm) { - struct vm_area_struct *vma, *gate_vma; + struct vm_area_struct *gate_vma, *vma = NULL; struct mm_struct *mm = current->mm; - int i; + MA_STATE(mas, &mm->mm_mt, 0, 0); + int i = 0; /* * Once the stack expansion code is fixed to not change VMA bounds @@ -1141,8 +1160,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) return false; } - for (i = 0, vma = first_vma(current, gate_vma); vma != NULL; - vma = next_vma(vma, gate_vma), i++) { + while ((vma = coredump_next_vma(&mas, vma, gate_vma)) != NULL) { struct core_vma_metadata *m = cprm->vma_meta + i; m->start = vma->vm_start; @@ -1150,10 +1168,10 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); m->pgoff = vma->vm_pgoff; - m->file = vma->vm_file; if (m->file) get_file(m->file); + i++; } mmap_write_unlock(mm); diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c index 2217fe5ece6f9b0f3e2368939949811a4aad0cbf..1b4403136d05c0aa91ebbca8c3b2c96210916d1f 100644 --- a/fs/crypto/bio.c +++ b/fs/crypto/bio.c @@ -25,21 +25,25 @@ * then this function isn't applicable. This function may sleep, so it must be * called from a workqueue rather than from the bio's bi_end_io callback. * - * This function sets PG_error on any pages that contain any blocks that failed - * to be decrypted. The filesystem must not mark such pages uptodate. + * Return: %true on success; %false on failure. On failure, bio->bi_status is + * also set to an error status. */ -void fscrypt_decrypt_bio(struct bio *bio) +bool fscrypt_decrypt_bio(struct bio *bio) { struct bio_vec *bv; struct bvec_iter_all iter_all; bio_for_each_segment_all(bv, bio, iter_all) { struct page *page = bv->bv_page; - int ret = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len, + int err = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len, bv->bv_offset); - if (ret) - SetPageError(page); + + if (err) { + bio->bi_status = errno_to_blk_status(err); + return false; + } } + return true; } EXPORT_SYMBOL(fscrypt_decrypt_bio); diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h index 3afdaa0847736b1d28b50cc4417db864c787d4ca..d5f68a0c5d1546403775e4544cf2f56a353c24f2 100644 --- a/fs/crypto/fscrypt_private.h +++ b/fs/crypto/fscrypt_private.h @@ -184,7 +184,7 @@ struct fscrypt_symlink_data { struct fscrypt_prepared_key { struct crypto_skcipher *tfm; #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT - struct fscrypt_blk_crypto_key *blk_key; + struct blk_crypto_key *blk_key; #endif }; @@ -225,7 +225,7 @@ struct fscrypt_info { * will be NULL if the master key was found in a process-subscribed * keyring rather than in the filesystem-level keyring. */ - struct key *ci_master_key; + struct fscrypt_master_key *ci_master_key; /* * Link in list of inodes that were unlocked with the master key. @@ -344,7 +344,8 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, const u8 *raw_key, const struct fscrypt_info *ci); -void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key); +void fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key); /* * Check whether the crypto transform or blk-crypto key has been allocated in @@ -390,7 +391,8 @@ fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, } static inline void -fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) +fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) { } @@ -436,6 +438,40 @@ struct fscrypt_master_key_secret { */ struct fscrypt_master_key { + /* + * Back-pointer to the super_block of the filesystem to which this + * master key has been added. Only valid if ->mk_active_refs > 0. + */ + struct super_block *mk_sb; + + /* + * Link in ->mk_sb->s_master_keys->key_hashtable. + * Only valid if ->mk_active_refs > 0. + */ + struct hlist_node mk_node; + + /* Semaphore that protects ->mk_secret and ->mk_users */ + struct rw_semaphore mk_sem; + + /* + * Active and structural reference counts. An active ref guarantees + * that the struct continues to exist, continues to be in the keyring + * ->mk_sb->s_master_keys, and that any embedded subkeys (e.g. + * ->mk_direct_keys) that have been prepared continue to exist. + * A structural ref only guarantees that the struct continues to exist. + * + * There is one active ref associated with ->mk_secret being present, + * and one active ref for each inode in ->mk_decrypted_inodes. + * + * There is one structural ref associated with the active refcount being + * nonzero. Finding a key in the keyring also takes a structural ref, + * which is then held temporarily while the key is operated on. + */ + refcount_t mk_active_refs; + refcount_t mk_struct_refs; + + struct rcu_head mk_rcu_head; + /* * The secret key material. After FS_IOC_REMOVE_ENCRYPTION_KEY is * executed, this is wiped and no new inodes can be unlocked with this @@ -444,7 +480,10 @@ struct fscrypt_master_key { * FS_IOC_REMOVE_ENCRYPTION_KEY can be retried, or * FS_IOC_ADD_ENCRYPTION_KEY can add the secret again. * - * Locking: protected by this master key's key->sem. + * While ->mk_secret is present, one ref in ->mk_active_refs is held. + * + * Locking: protected by ->mk_sem. The manipulation of ->mk_active_refs + * associated with this field is protected by ->mk_sem as well. */ struct fscrypt_master_key_secret mk_secret; @@ -465,22 +504,12 @@ struct fscrypt_master_key { * * This is NULL for v1 policy keys; those can only be added by root. * - * Locking: in addition to this keyring's own semaphore, this is - * protected by this master key's key->sem, so we can do atomic - * search+insert. It can also be searched without taking any locks, but - * in that case the returned key may have already been removed. + * Locking: protected by ->mk_sem. (We don't just rely on the keyrings + * subsystem semaphore ->mk_users->sem, as we need support for atomic + * search+insert along with proper synchronization with ->mk_secret.) */ struct key *mk_users; - /* - * Length of ->mk_decrypted_inodes, plus one if mk_secret is present. - * Once this goes to 0, the master key is removed from ->s_master_keys. - * The 'struct fscrypt_master_key' will continue to live as long as the - * 'struct key' whose payload it is, but we won't let this reference - * count rise again. - */ - refcount_t mk_refcount; - /* * List of inodes that were unlocked using this key. This allows the * inodes to be evicted efficiently if the key is removed. @@ -506,10 +535,10 @@ static inline bool is_master_key_secret_present(const struct fscrypt_master_key_secret *secret) { /* - * The READ_ONCE() is only necessary for fscrypt_drop_inode() and - * fscrypt_key_describe(). These run in atomic context, so they can't - * take the key semaphore and thus 'secret' can change concurrently - * which would be a data race. But they only need to know whether the + * The READ_ONCE() is only necessary for fscrypt_drop_inode(). + * fscrypt_drop_inode() runs in atomic context, so it can't take the key + * semaphore and thus 'secret' can change concurrently which would be a + * data race. But fscrypt_drop_inode() only need to know whether the * secret *was* present at the time of check, so READ_ONCE() suffices. */ return READ_ONCE(secret->size) != 0; @@ -538,7 +567,11 @@ static inline int master_key_spec_len(const struct fscrypt_key_specifier *spec) return 0; } -struct key * +void fscrypt_put_master_key(struct fscrypt_master_key *mk); + +void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk); + +struct fscrypt_master_key * fscrypt_find_master_key(struct super_block *sb, const struct fscrypt_key_specifier *mk_spec); @@ -569,7 +602,8 @@ extern struct fscrypt_mode fscrypt_modes[]; int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, const u8 *raw_key, const struct fscrypt_info *ci); -void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key); +void fscrypt_destroy_prepared_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key); int fscrypt_set_per_file_enc_key(struct fscrypt_info *ci, const u8 *raw_key); diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c index 7c01025879b38fb9bc22b56163f28375d2119d99..7b8c5a1104b5867e292c4c2a31d11c93d1925d30 100644 --- a/fs/crypto/hooks.c +++ b/fs/crypto/hooks.c @@ -5,8 +5,6 @@ * Encryption hooks for higher-level filesystem operations. */ -#include - #include "fscrypt_private.h" /** @@ -142,7 +140,6 @@ int fscrypt_prepare_setflags(struct inode *inode, unsigned int oldflags, unsigned int flags) { struct fscrypt_info *ci; - struct key *key; struct fscrypt_master_key *mk; int err; @@ -158,14 +155,13 @@ int fscrypt_prepare_setflags(struct inode *inode, ci = inode->i_crypt_info; if (ci->ci_policy.version != FSCRYPT_POLICY_V2) return -EINVAL; - key = ci->ci_master_key; - mk = key->payload.data[0]; - down_read(&key->sem); + mk = ci->ci_master_key; + down_read(&mk->mk_sem); if (is_master_key_secret_present(&mk->mk_secret)) err = fscrypt_derive_dirhash_key(ci, mk); else err = -ENOKEY; - up_read(&key->sem); + up_read(&mk->mk_sem); return err; } return 0; diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index 90f3e68f166e393fe6b14f5e54554abd20ee0f52..cea8b14007e6ac2011d18b434194de74f29194be 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -21,26 +21,22 @@ #include "fscrypt_private.h" -struct fscrypt_blk_crypto_key { - struct blk_crypto_key base; - int num_devs; - struct request_queue *devs[]; -}; - -static int fscrypt_get_num_devices(struct super_block *sb) +static struct block_device **fscrypt_get_devices(struct super_block *sb, + unsigned int *num_devs) { - if (sb->s_cop->get_num_devices) - return sb->s_cop->get_num_devices(sb); - return 1; -} + struct block_device **devs; -static void fscrypt_get_devices(struct super_block *sb, int num_devs, - struct request_queue **devs) -{ - if (num_devs == 1) - devs[0] = bdev_get_queue(sb->s_bdev); - else - sb->s_cop->get_devices(sb, devs); + if (sb->s_cop->get_devices) { + devs = sb->s_cop->get_devices(sb, num_devs); + if (devs) + return devs; + } + devs = kmalloc(sizeof(*devs), GFP_KERNEL); + if (!devs) + return ERR_PTR(-ENOMEM); + devs[0] = sb->s_bdev; + *num_devs = 1; + return devs; } static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci) @@ -74,15 +70,17 @@ static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci) * helpful for debugging problems where the "wrong" implementation is used. */ static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode, - struct request_queue **devs, - int num_devs, + struct block_device **devs, + unsigned int num_devs, const struct blk_crypto_config *cfg) { - int i; + unsigned int i; for (i = 0; i < num_devs; i++) { + struct request_queue *q = bdev_get_queue(devs[i]); + if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) || - __blk_crypto_cfg_supported(devs[i]->crypto_profile, cfg)) { + __blk_crypto_cfg_supported(q->crypto_profile, cfg)) { if (!xchg(&mode->logged_blk_crypto_native, 1)) pr_info("fscrypt: %s using blk-crypto (native)\n", mode->friendly_name); @@ -99,9 +97,9 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci) const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; struct blk_crypto_config crypto_cfg; - int num_devs; - struct request_queue **devs; - int i; + struct block_device **devs; + unsigned int num_devs; + unsigned int i; /* The file must need contents encryption, not filenames encryption */ if (!S_ISREG(inode->i_mode)) @@ -129,20 +127,20 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci) return 0; /* - * On all the filesystem's devices, blk-crypto must support the crypto - * configuration that the file would use. + * On all the filesystem's block devices, blk-crypto must support the + * crypto configuration that the file would use. */ crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode; crypto_cfg.data_unit_size = sb->s_blocksize; crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci); - num_devs = fscrypt_get_num_devices(sb); - devs = kmalloc_array(num_devs, sizeof(*devs), GFP_KERNEL); - if (!devs) - return -ENOMEM; - fscrypt_get_devices(sb, num_devs, devs); + + devs = fscrypt_get_devices(sb, &num_devs); + if (IS_ERR(devs)) + return PTR_ERR(devs); for (i = 0; i < num_devs; i++) { - if (!blk_crypto_config_supported(devs[i], &crypto_cfg)) + if (!blk_crypto_config_supported(bdev_get_queue(devs[i]), + &crypto_cfg)) goto out_free_devs; } @@ -162,49 +160,41 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; - int num_devs = fscrypt_get_num_devices(sb); - int queue_refs = 0; - struct fscrypt_blk_crypto_key *blk_key; + struct blk_crypto_key *blk_key; + struct block_device **devs; + unsigned int num_devs; + unsigned int i; int err; - int i; - blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_KERNEL); + blk_key = kmalloc(sizeof(*blk_key), GFP_KERNEL); if (!blk_key) return -ENOMEM; - blk_key->num_devs = num_devs; - fscrypt_get_devices(sb, num_devs, blk_key->devs); - - err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode, + err = blk_crypto_init_key(blk_key, raw_key, crypto_mode, fscrypt_get_dun_bytes(ci), sb->s_blocksize); if (err) { fscrypt_err(inode, "error %d initializing blk-crypto key", err); goto fail; } - /* - * We have to start using blk-crypto on all the filesystem's devices. - * We also have to save all the request_queue's for later so that the - * key can be evicted from them. This is needed because some keys - * aren't destroyed until after the filesystem was already unmounted - * (namely, the per-mode keys in struct fscrypt_master_key). - */ + /* Start using blk-crypto on all the filesystem's block devices. */ + devs = fscrypt_get_devices(sb, &num_devs); + if (IS_ERR(devs)) { + err = PTR_ERR(devs); + goto fail; + } for (i = 0; i < num_devs; i++) { - if (!blk_get_queue(blk_key->devs[i])) { - fscrypt_err(inode, "couldn't get request_queue"); - err = -EAGAIN; - goto fail; - } - queue_refs++; - - err = blk_crypto_start_using_key(&blk_key->base, - blk_key->devs[i]); - if (err) { - fscrypt_err(inode, - "error %d starting to use blk-crypto", err); - goto fail; - } + err = blk_crypto_start_using_key(blk_key, + bdev_get_queue(devs[i])); + if (err) + break; } + kfree(devs); + if (err) { + fscrypt_err(inode, "error %d starting to use blk-crypto", err); + goto fail; + } + /* * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). * I.e., here we publish ->blk_key with a RELEASE barrier so that @@ -215,24 +205,29 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, return 0; fail: - for (i = 0; i < queue_refs; i++) - blk_put_queue(blk_key->devs[i]); kfree_sensitive(blk_key); return err; } -void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) +void fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) { - struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key; - int i; + struct blk_crypto_key *blk_key = prep_key->blk_key; + struct block_device **devs; + unsigned int num_devs; + unsigned int i; - if (blk_key) { - for (i = 0; i < blk_key->num_devs; i++) { - blk_crypto_evict_key(blk_key->devs[i], &blk_key->base); - blk_put_queue(blk_key->devs[i]); - } - kfree_sensitive(blk_key); + if (!blk_key) + return; + + /* Evict the key from all the filesystem's block devices. */ + devs = fscrypt_get_devices(sb, &num_devs); + if (!IS_ERR(devs)) { + for (i = 0; i < num_devs; i++) + blk_crypto_evict_key(bdev_get_queue(devs[i]), blk_key); + kfree(devs); } + kfree_sensitive(blk_key); } bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode) @@ -282,7 +277,7 @@ void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, ci = inode->i_crypt_info; fscrypt_generate_dun(ci, first_lblk, dun); - bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask); + bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask); } EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); @@ -369,7 +364,7 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, * uses the same pointer. I.e., there's currently no need to support * merging requests where the keys are the same but the pointers differ. */ - if (bc->bc_key != &inode->i_crypt_info->ci_enc_key.blk_key->base) + if (bc->bc_key != inode->i_crypt_info->ci_enc_key.blk_key) return false; fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun); @@ -401,46 +396,45 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio, EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh); /** - * fscrypt_dio_supported() - check whether a DIO (direct I/O) request is - * supported as far as encryption is concerned - * @iocb: the file and position the I/O is targeting - * @iter: the I/O data segment(s) + * fscrypt_dio_supported() - check whether DIO (direct I/O) is supported on an + * inode, as far as encryption is concerned + * @inode: the inode in question * * Return: %true if there are no encryption constraints that prevent DIO from * being supported; %false if DIO is unsupported. (Note that in the * %true case, the filesystem might have other, non-encryption-related - * constraints that prevent DIO from actually being supported.) + * constraints that prevent DIO from actually being supported. Also, on + * encrypted files the filesystem is still responsible for only allowing + * DIO when requests are filesystem-block-aligned.) */ -bool fscrypt_dio_supported(struct kiocb *iocb, struct iov_iter *iter) +bool fscrypt_dio_supported(struct inode *inode) { - const struct inode *inode = file_inode(iocb->ki_filp); - const unsigned int blocksize = i_blocksize(inode); + int err; /* If the file is unencrypted, no veto from us. */ if (!fscrypt_needs_contents_encryption(inode)) return true; - /* We only support DIO with inline crypto, not fs-layer crypto. */ - if (!fscrypt_inode_uses_inline_crypto(inode)) - return false; - /* - * Since the granularity of encryption is filesystem blocks, the file - * position and total I/O length must be aligned to the filesystem block - * size -- not just to the block device's logical block size as is - * traditionally the case for DIO on many filesystems. + * We only support DIO with inline crypto, not fs-layer crypto. * - * We require that the user-provided memory buffers be filesystem block - * aligned too. It is simpler to have a single alignment value required - * for all properties of the I/O, as is normally the case for DIO. - * Also, allowing less aligned buffers would imply that data units could - * cross bvecs, which would greatly complicate the I/O stack, which - * assumes that bios can be split at any bvec boundary. + * To determine whether the inode is using inline crypto, we have to set + * up the key if it wasn't already done. This is because in the current + * design of fscrypt, the decision of whether to use inline crypto or + * not isn't made until the inode's encryption key is being set up. In + * the DIO read/write case, the key will always be set up already, since + * the file will be open. But in the case of statx(), the key might not + * be set up yet, as the file might not have been opened yet. */ - if (!IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), blocksize)) + err = fscrypt_require_key(inode); + if (err) { + /* + * Key unavailable or couldn't be set up. This edge case isn't + * worth worrying about; just report that DIO is unsupported. + */ return false; - - return true; + } + return fscrypt_inode_uses_inline_crypto(inode); } EXPORT_SYMBOL_GPL(fscrypt_dio_supported); diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index caee9f8620dd9bb16f9a426d2c38c3c98471a854..1cca09aa43f8b38fdce79258be9b29e9f0986b37 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -18,6 +18,7 @@ * information about these ioctls. */ +#include #include #include #include @@ -25,6 +26,18 @@ #include "fscrypt_private.h" +/* The master encryption keys for a filesystem (->s_master_keys) */ +struct fscrypt_keyring { + /* + * Lock that protects ->key_hashtable. It does *not* protect the + * fscrypt_master_key structs themselves. + */ + spinlock_t lock; + + /* Hash table that maps fscrypt_key_specifier to fscrypt_master_key */ + struct hlist_head key_hashtable[128]; +}; + static void wipe_master_key_secret(struct fscrypt_master_key_secret *secret) { fscrypt_destroy_hkdf(&secret->hkdf); @@ -38,66 +51,81 @@ static void move_master_key_secret(struct fscrypt_master_key_secret *dst, memzero_explicit(src, sizeof(*src)); } -static void free_master_key(struct fscrypt_master_key *mk) +static void fscrypt_free_master_key(struct rcu_head *head) { - size_t i; - - wipe_master_key_secret(&mk->mk_secret); - - for (i = 0; i <= FSCRYPT_MODE_MAX; i++) { - fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]); - fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]); - fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]); - } - - key_put(mk->mk_users); + struct fscrypt_master_key *mk = + container_of(head, struct fscrypt_master_key, mk_rcu_head); + /* + * The master key secret and any embedded subkeys should have already + * been wiped when the last active reference to the fscrypt_master_key + * struct was dropped; doing it here would be unnecessarily late. + * Nevertheless, use kfree_sensitive() in case anything was missed. + */ kfree_sensitive(mk); } -static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec) +void fscrypt_put_master_key(struct fscrypt_master_key *mk) { - if (spec->__reserved) - return false; - return master_key_spec_len(spec) != 0; + if (!refcount_dec_and_test(&mk->mk_struct_refs)) + return; + /* + * No structural references left, so free ->mk_users, and also free the + * fscrypt_master_key struct itself after an RCU grace period ensures + * that concurrent keyring lookups can no longer find it. + */ + WARN_ON(refcount_read(&mk->mk_active_refs) != 0); + key_put(mk->mk_users); + mk->mk_users = NULL; + call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key); } -static int fscrypt_key_instantiate(struct key *key, - struct key_preparsed_payload *prep) +void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk) { - key->payload.data[0] = (struct fscrypt_master_key *)prep->data; - return 0; -} + struct super_block *sb = mk->mk_sb; + struct fscrypt_keyring *keyring = sb->s_master_keys; + size_t i; -static void fscrypt_key_destroy(struct key *key) -{ - free_master_key(key->payload.data[0]); -} + if (!refcount_dec_and_test(&mk->mk_active_refs)) + return; + /* + * No active references left, so complete the full removal of this + * fscrypt_master_key struct by removing it from the keyring and + * destroying any subkeys embedded in it. + */ -static void fscrypt_key_describe(const struct key *key, struct seq_file *m) -{ - seq_puts(m, key->description); + spin_lock(&keyring->lock); + hlist_del_rcu(&mk->mk_node); + spin_unlock(&keyring->lock); - if (key_is_positive(key)) { - const struct fscrypt_master_key *mk = key->payload.data[0]; + /* + * ->mk_active_refs == 0 implies that ->mk_secret is not present and + * that ->mk_decrypted_inodes is empty. + */ + WARN_ON(is_master_key_secret_present(&mk->mk_secret)); + WARN_ON(!list_empty(&mk->mk_decrypted_inodes)); - if (!is_master_key_secret_present(&mk->mk_secret)) - seq_puts(m, ": secret removed"); + for (i = 0; i <= FSCRYPT_MODE_MAX; i++) { + fscrypt_destroy_prepared_key( + sb, &mk->mk_direct_keys[i]); + fscrypt_destroy_prepared_key( + sb, &mk->mk_iv_ino_lblk_64_keys[i]); + fscrypt_destroy_prepared_key( + sb, &mk->mk_iv_ino_lblk_32_keys[i]); } + memzero_explicit(&mk->mk_ino_hash_key, + sizeof(mk->mk_ino_hash_key)); + mk->mk_ino_hash_key_initialized = false; + + /* Drop the structural ref associated with the active refs. */ + fscrypt_put_master_key(mk); } -/* - * Type of key in ->s_master_keys. Each key of this type represents a master - * key which has been added to the filesystem. Its payload is a - * 'struct fscrypt_master_key'. The "." prefix in the key type name prevents - * users from adding keys of this type via the keyrings syscalls rather than via - * the intended method of FS_IOC_ADD_ENCRYPTION_KEY. - */ -static struct key_type key_type_fscrypt = { - .name = "._fscrypt", - .instantiate = fscrypt_key_instantiate, - .destroy = fscrypt_key_destroy, - .describe = fscrypt_key_describe, -}; +static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec) +{ + if (spec->__reserved) + return false; + return master_key_spec_len(spec) != 0; +} static int fscrypt_user_key_instantiate(struct key *key, struct key_preparsed_payload *prep) @@ -131,32 +159,6 @@ static struct key_type key_type_fscrypt_user = { .describe = fscrypt_user_key_describe, }; -/* Search ->s_master_keys or ->mk_users */ -static struct key *search_fscrypt_keyring(struct key *keyring, - struct key_type *type, - const char *description) -{ - /* - * We need to mark the keyring reference as "possessed" so that we - * acquire permission to search it, via the KEY_POS_SEARCH permission. - */ - key_ref_t keyref = make_key_ref(keyring, true /* possessed */); - - keyref = keyring_search(keyref, type, description, false); - if (IS_ERR(keyref)) { - if (PTR_ERR(keyref) == -EAGAIN || /* not found */ - PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ - keyref = ERR_PTR(-ENOKEY); - return ERR_CAST(keyref); - } - return key_ref_to_ptr(keyref); -} - -#define FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE \ - (CONST_STRLEN("fscrypt-") + sizeof_field(struct super_block, s_id)) - -#define FSCRYPT_MK_DESCRIPTION_SIZE (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + 1) - #define FSCRYPT_MK_USERS_DESCRIPTION_SIZE \ (CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \ CONST_STRLEN("-users") + 1) @@ -164,21 +166,6 @@ static struct key *search_fscrypt_keyring(struct key *keyring, #define FSCRYPT_MK_USER_DESCRIPTION_SIZE \ (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1) -static void format_fs_keyring_description( - char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE], - const struct super_block *sb) -{ - sprintf(description, "fscrypt-%s", sb->s_id); -} - -static void format_mk_description( - char description[FSCRYPT_MK_DESCRIPTION_SIZE], - const struct fscrypt_key_specifier *mk_spec) -{ - sprintf(description, "%*phN", - master_key_spec_len(mk_spec), (u8 *)&mk_spec->u); -} - static void format_mk_users_keyring_description( char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE], const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) @@ -199,20 +186,15 @@ static void format_mk_user_description( /* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */ static int allocate_filesystem_keyring(struct super_block *sb) { - char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE]; - struct key *keyring; + struct fscrypt_keyring *keyring; if (sb->s_master_keys) return 0; - format_fs_keyring_description(description, sb); - keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, - current_cred(), KEY_POS_SEARCH | - KEY_USR_SEARCH | KEY_USR_READ | KEY_USR_VIEW, - KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); - if (IS_ERR(keyring)) - return PTR_ERR(keyring); - + keyring = kzalloc(sizeof(*keyring), GFP_KERNEL); + if (!keyring) + return -ENOMEM; + spin_lock_init(&keyring->lock); /* * Pairs with the smp_load_acquire() in fscrypt_find_master_key(). * I.e., here we publish ->s_master_keys with a RELEASE barrier so that @@ -222,21 +204,75 @@ static int allocate_filesystem_keyring(struct super_block *sb) return 0; } -void fscrypt_sb_free(struct super_block *sb) +/* + * This is called at unmount time to release all encryption keys that have been + * added to the filesystem, along with the keyring that contains them. + * + * Note that besides clearing and freeing memory, this might need to evict keys + * from the keyslots of an inline crypto engine. Therefore, this must be called + * while the filesystem's underlying block device(s) are still available. + */ +void fscrypt_sb_delete(struct super_block *sb) { - key_put(sb->s_master_keys); + struct fscrypt_keyring *keyring = sb->s_master_keys; + size_t i; + + if (!keyring) + return; + + for (i = 0; i < ARRAY_SIZE(keyring->key_hashtable); i++) { + struct hlist_head *bucket = &keyring->key_hashtable[i]; + struct fscrypt_master_key *mk; + struct hlist_node *tmp; + + hlist_for_each_entry_safe(mk, tmp, bucket, mk_node) { + /* + * Since all inodes were already evicted, every key + * remaining in the keyring should have an empty inode + * list, and should only still be in the keyring due to + * the single active ref associated with ->mk_secret. + * There should be no structural refs beyond the one + * associated with the active ref. + */ + WARN_ON(refcount_read(&mk->mk_active_refs) != 1); + WARN_ON(refcount_read(&mk->mk_struct_refs) != 1); + WARN_ON(!is_master_key_secret_present(&mk->mk_secret)); + wipe_master_key_secret(&mk->mk_secret); + fscrypt_put_master_key_activeref(mk); + } + } + kfree_sensitive(keyring); sb->s_master_keys = NULL; } +static struct hlist_head * +fscrypt_mk_hash_bucket(struct fscrypt_keyring *keyring, + const struct fscrypt_key_specifier *mk_spec) +{ + /* + * Since key specifiers should be "random" values, it is sufficient to + * use a trivial hash function that just takes the first several bits of + * the key specifier. + */ + unsigned long i = get_unaligned((unsigned long *)&mk_spec->u); + + return &keyring->key_hashtable[i % ARRAY_SIZE(keyring->key_hashtable)]; +} + /* - * Find the specified master key in ->s_master_keys. - * Returns ERR_PTR(-ENOKEY) if not found. + * Find the specified master key struct in ->s_master_keys and take a structural + * ref to it. The structural ref guarantees that the key struct continues to + * exist, but it does *not* guarantee that ->s_master_keys continues to contain + * the key struct. The structural ref needs to be dropped by + * fscrypt_put_master_key(). Returns NULL if the key struct is not found. */ -struct key *fscrypt_find_master_key(struct super_block *sb, - const struct fscrypt_key_specifier *mk_spec) +struct fscrypt_master_key * +fscrypt_find_master_key(struct super_block *sb, + const struct fscrypt_key_specifier *mk_spec) { - struct key *keyring; - char description[FSCRYPT_MK_DESCRIPTION_SIZE]; + struct fscrypt_keyring *keyring; + struct hlist_head *bucket; + struct fscrypt_master_key *mk; /* * Pairs with the smp_store_release() in allocate_filesystem_keyring(). @@ -246,10 +282,38 @@ struct key *fscrypt_find_master_key(struct super_block *sb, */ keyring = smp_load_acquire(&sb->s_master_keys); if (keyring == NULL) - return ERR_PTR(-ENOKEY); /* No keyring yet, so no keys yet. */ - - format_mk_description(description, mk_spec); - return search_fscrypt_keyring(keyring, &key_type_fscrypt, description); + return NULL; /* No keyring yet, so no keys yet. */ + + bucket = fscrypt_mk_hash_bucket(keyring, mk_spec); + rcu_read_lock(); + switch (mk_spec->type) { + case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR: + hlist_for_each_entry_rcu(mk, bucket, mk_node) { + if (mk->mk_spec.type == + FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && + memcmp(mk->mk_spec.u.descriptor, + mk_spec->u.descriptor, + FSCRYPT_KEY_DESCRIPTOR_SIZE) == 0 && + refcount_inc_not_zero(&mk->mk_struct_refs)) + goto out; + } + break; + case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER: + hlist_for_each_entry_rcu(mk, bucket, mk_node) { + if (mk->mk_spec.type == + FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER && + memcmp(mk->mk_spec.u.identifier, + mk_spec->u.identifier, + FSCRYPT_KEY_IDENTIFIER_SIZE) == 0 && + refcount_inc_not_zero(&mk->mk_struct_refs)) + goto out; + } + break; + } + mk = NULL; +out: + rcu_read_unlock(); + return mk; } static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk) @@ -277,17 +341,30 @@ static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk) static struct key *find_master_key_user(struct fscrypt_master_key *mk) { char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; + key_ref_t keyref; format_mk_user_description(description, mk->mk_spec.u.identifier); - return search_fscrypt_keyring(mk->mk_users, &key_type_fscrypt_user, - description); + + /* + * We need to mark the keyring reference as "possessed" so that we + * acquire permission to search it, via the KEY_POS_SEARCH permission. + */ + keyref = keyring_search(make_key_ref(mk->mk_users, true /*possessed*/), + &key_type_fscrypt_user, description, false); + if (IS_ERR(keyref)) { + if (PTR_ERR(keyref) == -EAGAIN || /* not found */ + PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ + keyref = ERR_PTR(-ENOKEY); + return ERR_CAST(keyref); + } + return key_ref_to_ptr(keyref); } /* * Give the current user a "key" in ->mk_users. This charges the user's quota * and marks the master key as added by the current user, so that it cannot be - * removed by another user with the key. Either the master key's key->sem must - * be held for write, or the master key must be still undergoing initialization. + * removed by another user with the key. Either ->mk_sem must be held for + * write, or the master key must be still undergoing initialization. */ static int add_master_key_user(struct fscrypt_master_key *mk) { @@ -309,7 +386,7 @@ static int add_master_key_user(struct fscrypt_master_key *mk) /* * Remove the current user's "key" from ->mk_users. - * The master key's key->sem must be held for write. + * ->mk_sem must be held for write. * * Returns 0 if removed, -ENOKEY if not found, or another -errno code. */ @@ -327,63 +404,49 @@ static int remove_master_key_user(struct fscrypt_master_key *mk) } /* - * Allocate a new fscrypt_master_key which contains the given secret, set it as - * the payload of a new 'struct key' of type fscrypt, and link the 'struct key' - * into the given keyring. Synchronized by fscrypt_add_key_mutex. + * Allocate a new fscrypt_master_key, transfer the given secret over to it, and + * insert it into sb->s_master_keys. */ -static int add_new_master_key(struct fscrypt_master_key_secret *secret, - const struct fscrypt_key_specifier *mk_spec, - struct key *keyring) +static int add_new_master_key(struct super_block *sb, + struct fscrypt_master_key_secret *secret, + const struct fscrypt_key_specifier *mk_spec) { + struct fscrypt_keyring *keyring = sb->s_master_keys; struct fscrypt_master_key *mk; - char description[FSCRYPT_MK_DESCRIPTION_SIZE]; - struct key *key; int err; mk = kzalloc(sizeof(*mk), GFP_KERNEL); if (!mk) return -ENOMEM; + mk->mk_sb = sb; + init_rwsem(&mk->mk_sem); + refcount_set(&mk->mk_struct_refs, 1); mk->mk_spec = *mk_spec; - move_master_key_secret(&mk->mk_secret, secret); - - refcount_set(&mk->mk_refcount, 1); /* secret is present */ INIT_LIST_HEAD(&mk->mk_decrypted_inodes); spin_lock_init(&mk->mk_decrypted_inodes_lock); if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { err = allocate_master_key_users_keyring(mk); if (err) - goto out_free_mk; + goto out_put; err = add_master_key_user(mk); if (err) - goto out_free_mk; + goto out_put; } - /* - * Note that we don't charge this key to anyone's quota, since when - * ->mk_users is in use those keys are charged instead, and otherwise - * (when ->mk_users isn't in use) only root can add these keys. - */ - format_mk_description(description, mk_spec); - key = key_alloc(&key_type_fscrypt, description, - GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), - KEY_POS_SEARCH | KEY_USR_SEARCH | KEY_USR_VIEW, - KEY_ALLOC_NOT_IN_QUOTA, NULL); - if (IS_ERR(key)) { - err = PTR_ERR(key); - goto out_free_mk; - } - err = key_instantiate_and_link(key, mk, sizeof(*mk), keyring, NULL); - key_put(key); - if (err) - goto out_free_mk; + move_master_key_secret(&mk->mk_secret, secret); + refcount_set(&mk->mk_active_refs, 1); /* ->mk_secret is present */ + spin_lock(&keyring->lock); + hlist_add_head_rcu(&mk->mk_node, + fscrypt_mk_hash_bucket(keyring, mk_spec)); + spin_unlock(&keyring->lock); return 0; -out_free_mk: - free_master_key(mk); +out_put: + fscrypt_put_master_key(mk); return err; } @@ -392,42 +455,34 @@ out_free_mk: static int add_existing_master_key(struct fscrypt_master_key *mk, struct fscrypt_master_key_secret *secret) { - struct key *mk_user; - bool rekey; int err; /* * If the current user is already in ->mk_users, then there's nothing to - * do. (Not applicable for v1 policy keys, which have NULL ->mk_users.) + * do. Otherwise, we need to add the user to ->mk_users. (Neither is + * applicable for v1 policy keys, which have NULL ->mk_users.) */ if (mk->mk_users) { - mk_user = find_master_key_user(mk); + struct key *mk_user = find_master_key_user(mk); + if (mk_user != ERR_PTR(-ENOKEY)) { if (IS_ERR(mk_user)) return PTR_ERR(mk_user); key_put(mk_user); return 0; } - } - - /* If we'll be re-adding ->mk_secret, try to take the reference. */ - rekey = !is_master_key_secret_present(&mk->mk_secret); - if (rekey && !refcount_inc_not_zero(&mk->mk_refcount)) - return KEY_DEAD; - - /* Add the current user to ->mk_users, if applicable. */ - if (mk->mk_users) { err = add_master_key_user(mk); - if (err) { - if (rekey && refcount_dec_and_test(&mk->mk_refcount)) - return KEY_DEAD; + if (err) return err; - } } /* Re-add the secret if needed. */ - if (rekey) + if (!is_master_key_secret_present(&mk->mk_secret)) { + if (!refcount_inc_not_zero(&mk->mk_active_refs)) + return KEY_DEAD; move_master_key_secret(&mk->mk_secret, secret); + } + return 0; } @@ -436,38 +491,36 @@ static int do_add_master_key(struct super_block *sb, const struct fscrypt_key_specifier *mk_spec) { static DEFINE_MUTEX(fscrypt_add_key_mutex); - struct key *key; + struct fscrypt_master_key *mk; int err; mutex_lock(&fscrypt_add_key_mutex); /* serialize find + link */ -retry: - key = fscrypt_find_master_key(sb, mk_spec); - if (IS_ERR(key)) { - err = PTR_ERR(key); - if (err != -ENOKEY) - goto out_unlock; + + mk = fscrypt_find_master_key(sb, mk_spec); + if (!mk) { /* Didn't find the key in ->s_master_keys. Add it. */ err = allocate_filesystem_keyring(sb); - if (err) - goto out_unlock; - err = add_new_master_key(secret, mk_spec, sb->s_master_keys); + if (!err) + err = add_new_master_key(sb, secret, mk_spec); } else { /* * Found the key in ->s_master_keys. Re-add the secret if * needed, and add the user to ->mk_users if needed. */ - down_write(&key->sem); - err = add_existing_master_key(key->payload.data[0], secret); - up_write(&key->sem); + down_write(&mk->mk_sem); + err = add_existing_master_key(mk, secret); + up_write(&mk->mk_sem); if (err == KEY_DEAD) { - /* Key being removed or needs to be removed */ - key_invalidate(key); - key_put(key); - goto retry; + /* + * We found a key struct, but it's already been fully + * removed. Ignore the old struct and add a new one. + * fscrypt_add_key_mutex means we don't need to worry + * about concurrent adds. + */ + err = add_new_master_key(sb, secret, mk_spec); } - key_put(key); + fscrypt_put_master_key(mk); } -out_unlock: mutex_unlock(&fscrypt_add_key_mutex); return err; } @@ -771,19 +824,19 @@ int fscrypt_verify_key_added(struct super_block *sb, const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) { struct fscrypt_key_specifier mk_spec; - struct key *key, *mk_user; struct fscrypt_master_key *mk; + struct key *mk_user; int err; mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; memcpy(mk_spec.u.identifier, identifier, FSCRYPT_KEY_IDENTIFIER_SIZE); - key = fscrypt_find_master_key(sb, &mk_spec); - if (IS_ERR(key)) { - err = PTR_ERR(key); + mk = fscrypt_find_master_key(sb, &mk_spec); + if (!mk) { + err = -ENOKEY; goto out; } - mk = key->payload.data[0]; + down_read(&mk->mk_sem); mk_user = find_master_key_user(mk); if (IS_ERR(mk_user)) { err = PTR_ERR(mk_user); @@ -791,7 +844,8 @@ int fscrypt_verify_key_added(struct super_block *sb, key_put(mk_user); err = 0; } - key_put(key); + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); out: if (err == -ENOKEY && capable(CAP_FOWNER)) err = 0; @@ -953,11 +1007,10 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) struct super_block *sb = file_inode(filp)->i_sb; struct fscrypt_remove_key_arg __user *uarg = _uarg; struct fscrypt_remove_key_arg arg; - struct key *key; struct fscrypt_master_key *mk; u32 status_flags = 0; int err; - bool dead; + bool inodes_remain; if (copy_from_user(&arg, uarg, sizeof(arg))) return -EFAULT; @@ -977,12 +1030,10 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) return -EACCES; /* Find the key being removed. */ - key = fscrypt_find_master_key(sb, &arg.key_spec); - if (IS_ERR(key)) - return PTR_ERR(key); - mk = key->payload.data[0]; - - down_write(&key->sem); + mk = fscrypt_find_master_key(sb, &arg.key_spec); + if (!mk) + return -ENOKEY; + down_write(&mk->mk_sem); /* If relevant, remove current user's (or all users) claim to the key */ if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { @@ -991,7 +1042,7 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) else err = remove_master_key_user(mk); if (err) { - up_write(&key->sem); + up_write(&mk->mk_sem); goto out_put_key; } if (mk->mk_users->keys.nr_leaves_on_tree != 0) { @@ -1003,26 +1054,22 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) status_flags |= FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS; err = 0; - up_write(&key->sem); + up_write(&mk->mk_sem); goto out_put_key; } } /* No user claims remaining. Go ahead and wipe the secret. */ - dead = false; + err = -ENOKEY; if (is_master_key_secret_present(&mk->mk_secret)) { wipe_master_key_secret(&mk->mk_secret); - dead = refcount_dec_and_test(&mk->mk_refcount); - } - up_write(&key->sem); - if (dead) { - /* - * No inodes reference the key, and we wiped the secret, so the - * key object is free to be removed from the keyring. - */ - key_invalidate(key); + fscrypt_put_master_key_activeref(mk); err = 0; - } else { + } + inodes_remain = refcount_read(&mk->mk_active_refs) > 0; + up_write(&mk->mk_sem); + + if (inodes_remain) { /* Some inodes still reference this key; try to evict them. */ err = try_to_lock_encrypted_files(sb, mk); if (err == -EBUSY) { @@ -1038,7 +1085,7 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) * has been fully removed including all files locked. */ out_put_key: - key_put(key); + fscrypt_put_master_key(mk); if (err == 0) err = put_user(status_flags, &uarg->removal_status_flags); return err; @@ -1085,7 +1132,6 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) { struct super_block *sb = file_inode(filp)->i_sb; struct fscrypt_get_key_status_arg arg; - struct key *key; struct fscrypt_master_key *mk; int err; @@ -1102,19 +1148,18 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) arg.user_count = 0; memset(arg.__out_reserved, 0, sizeof(arg.__out_reserved)); - key = fscrypt_find_master_key(sb, &arg.key_spec); - if (IS_ERR(key)) { - if (key != ERR_PTR(-ENOKEY)) - return PTR_ERR(key); + mk = fscrypt_find_master_key(sb, &arg.key_spec); + if (!mk) { arg.status = FSCRYPT_KEY_STATUS_ABSENT; err = 0; goto out; } - mk = key->payload.data[0]; - down_read(&key->sem); + down_read(&mk->mk_sem); if (!is_master_key_secret_present(&mk->mk_secret)) { - arg.status = FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED; + arg.status = refcount_read(&mk->mk_active_refs) > 0 ? + FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED : + FSCRYPT_KEY_STATUS_ABSENT /* raced with full removal */; err = 0; goto out_release_key; } @@ -1136,8 +1181,8 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) } err = 0; out_release_key: - up_read(&key->sem); - key_put(key); + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); out: if (!err && copy_to_user(uarg, &arg, sizeof(arg))) err = -EFAULT; @@ -1149,13 +1194,9 @@ int __init fscrypt_init_keyring(void) { int err; - err = register_key_type(&key_type_fscrypt); - if (err) - return err; - err = register_key_type(&key_type_fscrypt_user); if (err) - goto err_unregister_fscrypt; + return err; err = register_key_type(&key_type_fscrypt_provisioning); if (err) @@ -1165,7 +1206,5 @@ int __init fscrypt_init_keyring(void) err_unregister_fscrypt_user: unregister_key_type(&key_type_fscrypt_user); -err_unregister_fscrypt: - unregister_key_type(&key_type_fscrypt); return err; } diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index fbc71abdabe32435ed9bb40c079851519c8bc082..f7407071a952424ed236a67cb0465edbdeaa2d22 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -9,7 +9,6 @@ */ #include -#include #include #include "fscrypt_private.h" @@ -155,10 +154,12 @@ int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, } /* Destroy a crypto transform object and/or blk-crypto key. */ -void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key) +void fscrypt_destroy_prepared_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) { crypto_free_skcipher(prep_key->tfm); - fscrypt_destroy_inline_crypt_key(prep_key); + fscrypt_destroy_inline_crypt_key(sb, prep_key); + memzero_explicit(prep_key, sizeof(*prep_key)); } /* Given a per-file encryption key, set up the file's crypto transform object */ @@ -412,20 +413,18 @@ static bool fscrypt_valid_master_key_size(const struct fscrypt_master_key *mk, /* * Find the master key, then set up the inode's actual encryption key. * - * If the master key is found in the filesystem-level keyring, then the - * corresponding 'struct key' is returned in *master_key_ret with its semaphore - * read-locked. This is needed to ensure that only one task links the - * fscrypt_info into ->mk_decrypted_inodes (as multiple tasks may race to create - * an fscrypt_info for the same inode), and to synchronize the master key being - * removed with a new inode starting to use it. + * If the master key is found in the filesystem-level keyring, then it is + * returned in *mk_ret with its semaphore read-locked. This is needed to ensure + * that only one task links the fscrypt_info into ->mk_decrypted_inodes (as + * multiple tasks may race to create an fscrypt_info for the same inode), and to + * synchronize the master key being removed with a new inode starting to use it. */ static int setup_file_encryption_key(struct fscrypt_info *ci, bool need_dirhash_key, - struct key **master_key_ret) + struct fscrypt_master_key **mk_ret) { - struct key *key; - struct fscrypt_master_key *mk = NULL; struct fscrypt_key_specifier mk_spec; + struct fscrypt_master_key *mk; int err; err = fscrypt_select_encryption_impl(ci); @@ -436,11 +435,10 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, if (err) return err; - key = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec); - if (IS_ERR(key)) { - if (key != ERR_PTR(-ENOKEY) || - ci->ci_policy.version != FSCRYPT_POLICY_V1) - return PTR_ERR(key); + mk = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec); + if (!mk) { + if (ci->ci_policy.version != FSCRYPT_POLICY_V1) + return -ENOKEY; /* * As a legacy fallback for v1 policies, search for the key in @@ -450,9 +448,7 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, */ return fscrypt_setup_v1_file_key_via_subscribed_keyrings(ci); } - - mk = key->payload.data[0]; - down_read(&key->sem); + down_read(&mk->mk_sem); /* Has the secret been removed (via FS_IOC_REMOVE_ENCRYPTION_KEY)? */ if (!is_master_key_secret_present(&mk->mk_secret)) { @@ -480,18 +476,18 @@ static int setup_file_encryption_key(struct fscrypt_info *ci, if (err) goto out_release_key; - *master_key_ret = key; + *mk_ret = mk; return 0; out_release_key: - up_read(&key->sem); - key_put(key); + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); return err; } static void put_crypt_info(struct fscrypt_info *ci) { - struct key *key; + struct fscrypt_master_key *mk; if (!ci) return; @@ -499,26 +495,21 @@ static void put_crypt_info(struct fscrypt_info *ci) if (ci->ci_direct_key) fscrypt_put_direct_key(ci->ci_direct_key); else if (ci->ci_owns_key) - fscrypt_destroy_prepared_key(&ci->ci_enc_key); - - key = ci->ci_master_key; - if (key) { - struct fscrypt_master_key *mk = key->payload.data[0]; + fscrypt_destroy_prepared_key(ci->ci_inode->i_sb, + &ci->ci_enc_key); + mk = ci->ci_master_key; + if (mk) { /* * Remove this inode from the list of inodes that were unlocked - * with the master key. - * - * In addition, if we're removing the last inode from a key that - * already had its secret removed, invalidate the key so that it - * gets removed from ->s_master_keys. + * with the master key. In addition, if we're removing the last + * inode from a master key struct that already had its secret + * removed, then complete the full removal of the struct. */ spin_lock(&mk->mk_decrypted_inodes_lock); list_del(&ci->ci_master_key_link); spin_unlock(&mk->mk_decrypted_inodes_lock); - if (refcount_dec_and_test(&mk->mk_refcount)) - key_invalidate(key); - key_put(key); + fscrypt_put_master_key_activeref(mk); } memzero_explicit(ci, sizeof(*ci)); kmem_cache_free(fscrypt_info_cachep, ci); @@ -532,7 +523,7 @@ fscrypt_setup_encryption_info(struct inode *inode, { struct fscrypt_info *crypt_info; struct fscrypt_mode *mode; - struct key *master_key = NULL; + struct fscrypt_master_key *mk = NULL; int res; res = fscrypt_initialize(inode->i_sb->s_cop->flags); @@ -555,8 +546,7 @@ fscrypt_setup_encryption_info(struct inode *inode, WARN_ON(mode->ivsize > FSCRYPT_MAX_IV_SIZE); crypt_info->ci_mode = mode; - res = setup_file_encryption_key(crypt_info, need_dirhash_key, - &master_key); + res = setup_file_encryption_key(crypt_info, need_dirhash_key, &mk); if (res) goto out; @@ -571,12 +561,9 @@ fscrypt_setup_encryption_info(struct inode *inode, * We won the race and set ->i_crypt_info to our crypt_info. * Now link it into the master key's inode list. */ - if (master_key) { - struct fscrypt_master_key *mk = - master_key->payload.data[0]; - - refcount_inc(&mk->mk_refcount); - crypt_info->ci_master_key = key_get(master_key); + if (mk) { + crypt_info->ci_master_key = mk; + refcount_inc(&mk->mk_active_refs); spin_lock(&mk->mk_decrypted_inodes_lock); list_add(&crypt_info->ci_master_key_link, &mk->mk_decrypted_inodes); @@ -586,9 +573,9 @@ fscrypt_setup_encryption_info(struct inode *inode, } res = 0; out: - if (master_key) { - up_read(&master_key->sem); - key_put(master_key); + if (mk) { + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); } put_crypt_info(crypt_info); return res; @@ -753,7 +740,6 @@ EXPORT_SYMBOL(fscrypt_free_inode); int fscrypt_drop_inode(struct inode *inode) { const struct fscrypt_info *ci = fscrypt_get_info(inode); - const struct fscrypt_master_key *mk; /* * If ci is NULL, then the inode doesn't have an encryption key set up @@ -763,7 +749,6 @@ int fscrypt_drop_inode(struct inode *inode) */ if (!ci || !ci->ci_master_key) return 0; - mk = ci->ci_master_key->payload.data[0]; /* * With proper, non-racy use of FS_IOC_REMOVE_ENCRYPTION_KEY, all inodes @@ -782,6 +767,6 @@ int fscrypt_drop_inode(struct inode *inode) * then the thread removing the key will either evict the inode itself * or will correctly detect that it wasn't evicted due to the race. */ - return !is_master_key_secret_present(&mk->mk_secret); + return !is_master_key_secret_present(&ci->ci_master_key->mk_secret); } EXPORT_SYMBOL_GPL(fscrypt_drop_inode); diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index 2762c53504323fff6ed531c32dfffeae46f11eed..75dabd9b27f9b6ce6c8f78d49e46ae76c22337c9 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -143,6 +143,7 @@ invalid: /* Master key referenced by DIRECT_KEY policy */ struct fscrypt_direct_key { + struct super_block *dk_sb; struct hlist_node dk_node; refcount_t dk_refcount; const struct fscrypt_mode *dk_mode; @@ -154,7 +155,7 @@ struct fscrypt_direct_key { static void free_direct_key(struct fscrypt_direct_key *dk) { if (dk) { - fscrypt_destroy_prepared_key(&dk->dk_key); + fscrypt_destroy_prepared_key(dk->dk_sb, &dk->dk_key); kfree_sensitive(dk); } } @@ -231,6 +232,7 @@ fscrypt_get_direct_key(const struct fscrypt_info *ci, const u8 *raw_key) dk = kzalloc(sizeof(*dk), GFP_KERNEL); if (!dk) return ERR_PTR(-ENOMEM); + dk->dk_sb = ci->ci_inode->i_sb; refcount_set(&dk->dk_refcount, 1); dk->dk_mode = ci->ci_mode; err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci); diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index 80b8ca0f340b29db71f62cbb2fec85b96618e451..46757c3052ef9590e5c0802dd9d32cf675965f2d 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -744,12 +744,8 @@ int fscrypt_set_context(struct inode *inode, void *fs_data) * delayed key setup that requires the inode number. */ if (ci->ci_policy.version == FSCRYPT_POLICY_V2 && - (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) { - const struct fscrypt_master_key *mk = - ci->ci_master_key->payload.data[0]; - - fscrypt_hash_inode_number(ci, mk); - } + (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) + fscrypt_hash_inode_number(ci, ci->ci_master_key); return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data); } @@ -833,19 +829,6 @@ bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1, } EXPORT_SYMBOL_GPL(fscrypt_dummy_policies_equal); -/* Deprecated, do not use */ -int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg, - struct fscrypt_dummy_policy *dummy_policy) -{ - struct fs_parameter param = { - .type = fs_value_is_string, - .string = arg ? (char *)arg : "", - }; - return fscrypt_parse_test_dummy_encryption(¶m, dummy_policy) ?: - fscrypt_add_test_dummy_key(sb, dummy_policy); -} -EXPORT_SYMBOL_GPL(fscrypt_set_test_dummy_encryption); - /** * fscrypt_show_test_dummy_encryption() - show '-o test_dummy_encryption' * @seq: the seq_file to print the option to diff --git a/fs/d_path.c b/fs/d_path.c index e4e0ebad1f1538e1a0d0b3f4f8431ffb5e6b7168..56a6ee4c6331cafef375beb11f4723e850582c7f 100644 --- a/fs/d_path.c +++ b/fs/d_path.c @@ -34,7 +34,7 @@ static bool prepend_char(struct prepend_buffer *p, unsigned char c) } /* - * The source of the prepend data can be an optimistoc load + * The source of the prepend data can be an optimistic load * of a dentry name and length. And because we don't hold any * locks, the length and the pointer to the name may not be * in sync if a concurrent rename happens, and the kernel @@ -297,8 +297,7 @@ EXPORT_SYMBOL(d_path); /* * Helper function for dentry_operations.d_dname() members */ -char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen, - const char *fmt, ...) +char *dynamic_dname(char *buffer, int buflen, const char *fmt, ...) { va_list args; char temp[64]; diff --git a/fs/dax.c b/fs/dax.c index c440dcef4b1bf1bfb5f9907e9e6163d40022f2de..1c6867810cbd67883074ce7ca9c84d77dd991b17 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1445,6 +1445,9 @@ dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, loff_t done = 0; int ret; + if (!iomi.len) + return 0; + if (iov_iter_rw(iter) == WRITE) { lockdep_assert_held_write(&iomi.inode->i_rwsem); iomi.flags |= IOMAP_WRITE; diff --git a/fs/dcache.c b/fs/dcache.c index bb0c4d0038dbdc5a715edeab6e825c4bfe8b6b72..52e6d5fdab6bdf868f63e61c69c84ee74aa98f72 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -2597,15 +2597,7 @@ EXPORT_SYMBOL(d_rehash); static inline unsigned start_dir_add(struct inode *dir) { - /* - * The caller holds a spinlock (dentry::d_lock). On !PREEMPT_RT - * kernels spin_lock() implicitly disables preemption, but not on - * PREEMPT_RT. So for RT it has to be done explicitly to protect - * the sequence count write side critical section against a reader - * or another writer preempting, which would result in a live lock. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); for (;;) { unsigned n = dir->i_dir_seq; if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n) @@ -2618,8 +2610,7 @@ static inline void end_dir_add(struct inode *dir, unsigned int n, wait_queue_head_t *d_wait) { smp_store_release(&dir->i_dir_seq, n + 2); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); wake_up_all(d_wait); } @@ -3258,8 +3249,10 @@ void d_genocide(struct dentry *parent) EXPORT_SYMBOL(d_genocide); -void d_tmpfile(struct dentry *dentry, struct inode *inode) +void d_tmpfile(struct file *file, struct inode *inode) { + struct dentry *dentry = file->f_path.dentry; + inode_dec_link_count(inode); BUG_ON(dentry->d_name.name != dentry->d_iname || !hlist_unhashed(&dentry->d_u.d_alias) || diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c index 950c63fa4d0b26d24e26d754101c4cbd485ad102..ddb3fc258df94eace1fac2db0bbdd81ff684f108 100644 --- a/fs/debugfs/file.c +++ b/fs/debugfs/file.c @@ -1121,7 +1121,7 @@ void debugfs_print_regs32(struct seq_file *s, const struct debugfs_reg32 *regs, } EXPORT_SYMBOL_GPL(debugfs_print_regs32); -static int debugfs_show_regset32(struct seq_file *s, void *data) +static int debugfs_regset32_show(struct seq_file *s, void *data) { struct debugfs_regset32 *regset = s->private; @@ -1136,17 +1136,7 @@ static int debugfs_show_regset32(struct seq_file *s, void *data) return 0; } -static int debugfs_open_regset32(struct inode *inode, struct file *file) -{ - return single_open(file, debugfs_show_regset32, inode->i_private); -} - -static const struct file_operations fops_regset32 = { - .open = debugfs_open_regset32, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(debugfs_regset32); /** * debugfs_create_regset32 - create a debugfs file that returns register values @@ -1167,7 +1157,7 @@ void debugfs_create_regset32(const char *name, umode_t mode, struct dentry *parent, struct debugfs_regset32 *regset) { - debugfs_create_file(name, mode, parent, regset, &fops_regset32); + debugfs_create_file(name, mode, parent, regset, &debugfs_regset32_fops); } EXPORT_SYMBOL_GPL(debugfs_create_regset32); diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c index 3dcf0b8b4e932dba4dc3f85d3c9e2336e8b7e541..2e8e112b19930dcfd8a308645692258d96879de9 100644 --- a/fs/debugfs/inode.c +++ b/fs/debugfs/inode.c @@ -82,6 +82,8 @@ struct debugfs_mount_opts { kuid_t uid; kgid_t gid; umode_t mode; + /* Opt_* bitfield. */ + unsigned int opts; }; enum { @@ -111,6 +113,7 @@ static int debugfs_parse_options(char *data, struct debugfs_mount_opts *opts) kgid_t gid; char *p; + opts->opts = 0; opts->mode = DEBUGFS_DEFAULT_MODE; while ((p = strsep(&data, ",")) != NULL) { @@ -145,24 +148,44 @@ static int debugfs_parse_options(char *data, struct debugfs_mount_opts *opts) * but traditionally debugfs has ignored all mount options */ } + + opts->opts |= BIT(token); } return 0; } -static int debugfs_apply_options(struct super_block *sb) +static void _debugfs_apply_options(struct super_block *sb, bool remount) { struct debugfs_fs_info *fsi = sb->s_fs_info; struct inode *inode = d_inode(sb->s_root); struct debugfs_mount_opts *opts = &fsi->mount_opts; - inode->i_mode &= ~S_IALLUGO; - inode->i_mode |= opts->mode; + /* + * On remount, only reset mode/uid/gid if they were provided as mount + * options. + */ + + if (!remount || opts->opts & BIT(Opt_mode)) { + inode->i_mode &= ~S_IALLUGO; + inode->i_mode |= opts->mode; + } - inode->i_uid = opts->uid; - inode->i_gid = opts->gid; + if (!remount || opts->opts & BIT(Opt_uid)) + inode->i_uid = opts->uid; - return 0; + if (!remount || opts->opts & BIT(Opt_gid)) + inode->i_gid = opts->gid; +} + +static void debugfs_apply_options(struct super_block *sb) +{ + _debugfs_apply_options(sb, false); +} + +static void debugfs_apply_options_remount(struct super_block *sb) +{ + _debugfs_apply_options(sb, true); } static int debugfs_remount(struct super_block *sb, int *flags, char *data) @@ -175,7 +198,7 @@ static int debugfs_remount(struct super_block *sb, int *flags, char *data) if (err) goto fail; - debugfs_apply_options(sb); + debugfs_apply_options_remount(sb); fail: return err; @@ -744,6 +767,28 @@ void debugfs_remove(struct dentry *dentry) } EXPORT_SYMBOL_GPL(debugfs_remove); +/** + * debugfs_lookup_and_remove - lookup a directory or file and recursively remove it + * @name: a pointer to a string containing the name of the item to look up. + * @parent: a pointer to the parent dentry of the item. + * + * This is the equlivant of doing something like + * debugfs_remove(debugfs_lookup(..)) but with the proper reference counting + * handled for the directory being looked up. + */ +void debugfs_lookup_and_remove(const char *name, struct dentry *parent) +{ + struct dentry *dentry; + + dentry = debugfs_lookup(name, parent); + if (!dentry) + return; + + debugfs_remove(dentry); + dput(dentry); +} +EXPORT_SYMBOL_GPL(debugfs_lookup_and_remove); + /** * debugfs_rename - rename a file/directory in the debugfs filesystem * @old_dir: a pointer to the parent dentry for the renamed object. This diff --git a/fs/direct-io.c b/fs/direct-io.c index f669163d5860f90f3f4543618160f49e1e12af52..03d381377ae10a87f99d43a5e80bac5687cf349e 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -421,8 +421,6 @@ static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) unsigned long flags; bio->bi_private = dio; - /* don't account direct I/O as memory stall */ - bio_clear_flag(bio, BIO_WORKINGSET); spin_lock_irqsave(&dio->bio_lock, flags); dio->refcount++; diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c index 19ef136f9e4fc32f8a5881f621b491eb526f24c2..d60a8d8f109df81f6fa701d28d7543420f13c44e 100644 --- a/fs/dlm/ast.c +++ b/fs/dlm/ast.c @@ -200,13 +200,13 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, if (!prev_seq) { kref_get(&lkb->lkb_ref); + mutex_lock(&ls->ls_cb_mutex); if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) { - mutex_lock(&ls->ls_cb_mutex); list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay); - mutex_unlock(&ls->ls_cb_mutex); } else { queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work); } + mutex_unlock(&ls->ls_cb_mutex); } out: mutex_unlock(&lkb->lkb_cb_mutex); @@ -288,10 +288,13 @@ void dlm_callback_stop(struct dlm_ls *ls) void dlm_callback_suspend(struct dlm_ls *ls) { - set_bit(LSFL_CB_DELAY, &ls->ls_flags); + if (ls->ls_callback_wq) { + mutex_lock(&ls->ls_cb_mutex); + set_bit(LSFL_CB_DELAY, &ls->ls_flags); + mutex_unlock(&ls->ls_cb_mutex); - if (ls->ls_callback_wq) flush_workqueue(ls->ls_callback_wq); + } } #define MAX_CB_QUEUE 25 @@ -302,11 +305,11 @@ void dlm_callback_resume(struct dlm_ls *ls) int count = 0, sum = 0; bool empty; - clear_bit(LSFL_CB_DELAY, &ls->ls_flags); - if (!ls->ls_callback_wq) return; + clear_bit(LSFL_CB_DELAY, &ls->ls_flags); + more: mutex_lock(&ls->ls_cb_mutex); list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) { diff --git a/fs/dlm/ast.h b/fs/dlm/ast.h index 181ad7d20c4d3232e14ae7ef773b91ee22a075f9..e5e05fcc58138d71f21c690943c90201055a4253 100644 --- a/fs/dlm/ast.h +++ b/fs/dlm/ast.h @@ -11,7 +11,6 @@ #ifndef __ASTD_DOT_H__ #define __ASTD_DOT_H__ -void dlm_del_ast(struct dlm_lkb *lkb); int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, uint32_t sbflags, uint64_t seq); int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb, diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h index 8aca8085d24e72e50bba324c94e532968e55b05a..e34c3d2639a5a94ed48947581c56e6a3a20ebadf 100644 --- a/fs/dlm/dlm_internal.h +++ b/fs/dlm/dlm_internal.h @@ -661,7 +661,7 @@ struct dlm_ls { spinlock_t ls_recover_idr_lock; wait_queue_head_t ls_wait_general; wait_queue_head_t ls_recover_lock_wait; - struct mutex ls_clear_proc_locks; + spinlock_t ls_clear_proc_locks; struct list_head ls_root_list; /* root resources */ struct rw_semaphore ls_root_sem; /* protect root_list */ diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index dac7eb75dba95c17a26ee13153dd076e7f6e1a50..94a72ede57646915d0cce7253522626df718c29a 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -401,7 +401,7 @@ static int pre_rsb_struct(struct dlm_ls *ls) unlock any spinlocks, go back and call pre_rsb_struct again. Otherwise, take an rsb off the list and return it. */ -static int get_rsb_struct(struct dlm_ls *ls, char *name, int len, +static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len, struct dlm_rsb **r_ret) { struct dlm_rsb *r; @@ -412,7 +412,8 @@ static int get_rsb_struct(struct dlm_ls *ls, char *name, int len, count = ls->ls_new_rsb_count; spin_unlock(&ls->ls_new_rsb_spin); log_debug(ls, "find_rsb retry %d %d %s", - count, dlm_config.ci_new_rsb_count, name); + count, dlm_config.ci_new_rsb_count, + (const char *)name); return -EAGAIN; } @@ -448,7 +449,7 @@ static int rsb_cmp(struct dlm_rsb *r, const char *name, int nlen) return memcmp(r->res_name, maxname, DLM_RESNAME_MAXLEN); } -int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, +int dlm_search_rsb_tree(struct rb_root *tree, const void *name, int len, struct dlm_rsb **r_ret) { struct rb_node *node = tree->rb_node; @@ -546,7 +547,7 @@ static int rsb_insert(struct dlm_rsb *rsb, struct rb_root *tree) * while that rsb has a potentially stale master.) */ -static int find_rsb_dir(struct dlm_ls *ls, char *name, int len, +static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, uint32_t hash, uint32_t b, int dir_nodeid, int from_nodeid, unsigned int flags, struct dlm_rsb **r_ret) @@ -724,7 +725,7 @@ static int find_rsb_dir(struct dlm_ls *ls, char *name, int len, dlm_recover_locks) before we've made ourself master (in dlm_recover_masters). */ -static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len, +static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, uint32_t hash, uint32_t b, int dir_nodeid, int from_nodeid, unsigned int flags, struct dlm_rsb **r_ret) @@ -818,8 +819,9 @@ static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len, return error; } -static int find_rsb(struct dlm_ls *ls, char *name, int len, int from_nodeid, - unsigned int flags, struct dlm_rsb **r_ret) +static int find_rsb(struct dlm_ls *ls, const void *name, int len, + int from_nodeid, unsigned int flags, + struct dlm_rsb **r_ret) { uint32_t hash, b; int dir_nodeid; @@ -2864,17 +2866,9 @@ static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args) static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, struct dlm_args *args) { - int rv = -EINVAL; + int rv = -EBUSY; if (args->flags & DLM_LKF_CONVERT) { - if (lkb->lkb_flags & DLM_IFL_MSTCPY) - goto out; - - if (args->flags & DLM_LKF_QUECVT && - !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1]) - goto out; - - rv = -EBUSY; if (lkb->lkb_status != DLM_LKSTS_GRANTED) goto out; @@ -2884,6 +2878,14 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, if (is_overlap(lkb)) goto out; + + rv = -EINVAL; + if (lkb->lkb_flags & DLM_IFL_MSTCPY) + goto out; + + if (args->flags & DLM_LKF_QUECVT && + !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1]) + goto out; } lkb->lkb_exflags = args->flags; @@ -2900,11 +2902,25 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, #endif rv = 0; out: - if (rv) - log_debug(ls, "validate_lock_args %d %x %x %x %d %d %s", + switch (rv) { + case 0: + break; + case -EINVAL: + /* annoy the user because dlm usage is wrong */ + WARN_ON(1); + log_error(ls, "%s %d %x %x %x %d %d %s", __func__, + rv, lkb->lkb_id, lkb->lkb_flags, args->flags, + lkb->lkb_status, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + break; + default: + log_debug(ls, "%s %d %x %x %x %d %d %s", __func__, rv, lkb->lkb_id, lkb->lkb_flags, args->flags, lkb->lkb_status, lkb->lkb_wait_type, lkb->lkb_resource->res_name); + break; + } + return rv; } @@ -2918,23 +2934,12 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) { struct dlm_ls *ls = lkb->lkb_resource->res_ls; - int rv = -EINVAL; - - if (lkb->lkb_flags & DLM_IFL_MSTCPY) { - log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id); - dlm_print_lkb(lkb); - goto out; - } - - /* an lkb may still exist even though the lock is EOL'ed due to a - cancel, unlock or failed noqueue request; an app can't use these - locks; return same error as if the lkid had not been found at all */ + int rv = -EBUSY; - if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { - log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id); - rv = -ENOENT; + /* normal unlock not allowed if there's any op in progress */ + if (!(args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) && + (lkb->lkb_wait_type || lkb->lkb_wait_count)) goto out; - } /* an lkb may be waiting for an rsb lookup to complete where the lookup was initiated by another lock */ @@ -2949,7 +2954,24 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) unhold_lkb(lkb); /* undoes create_lkb() */ } /* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */ - rv = -EBUSY; + goto out; + } + + rv = -EINVAL; + if (lkb->lkb_flags & DLM_IFL_MSTCPY) { + log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id); + dlm_print_lkb(lkb); + goto out; + } + + /* an lkb may still exist even though the lock is EOL'ed due to a + * cancel, unlock or failed noqueue request; an app can't use these + * locks; return same error as if the lkid had not been found at all + */ + + if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) { + log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id); + rv = -ENOENT; goto out; } @@ -3022,14 +3044,8 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) goto out; } /* add_to_waiters() will set OVERLAP_UNLOCK */ - goto out_ok; } - /* normal unlock not allowed if there's any op in progress */ - rv = -EBUSY; - if (lkb->lkb_wait_type || lkb->lkb_wait_count) - goto out; - out_ok: /* an overlapping op shouldn't blow away exflags from other op */ lkb->lkb_exflags |= args->flags; @@ -3037,11 +3053,25 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) lkb->lkb_astparam = args->astparam; rv = 0; out: - if (rv) - log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", rv, + switch (rv) { + case 0: + break; + case -EINVAL: + /* annoy the user because dlm usage is wrong */ + WARN_ON(1); + log_error(ls, "%s %d %x %x %x %x %d %s", __func__, rv, + lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, + args->flags, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + break; + default: + log_debug(ls, "%s %d %x %x %x %x %d %s", __func__, rv, lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags, args->flags, lkb->lkb_wait_type, lkb->lkb_resource->res_name); + break; + } + return rv; } @@ -3292,8 +3322,9 @@ static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) * request_lock(), convert_lock(), unlock_lock(), cancel_lock() */ -static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name, - int len, struct dlm_args *args) +static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + const void *name, int len, + struct dlm_args *args) { struct dlm_rsb *r; int error; @@ -3392,7 +3423,7 @@ int dlm_lock(dlm_lockspace_t *lockspace, int mode, struct dlm_lksb *lksb, uint32_t flags, - void *name, + const void *name, unsigned int namelen, uint32_t parent_lkid, void (*ast) (void *astarg), @@ -3438,7 +3469,7 @@ int dlm_lock(dlm_lockspace_t *lockspace, if (error == -EINPROGRESS) error = 0; out_put: - trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error); + trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, true); if (convert || error) __put_lkb(ls, lkb); @@ -3623,7 +3654,7 @@ static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb, case cpu_to_le32(DLM_MSG_REQUEST_REPLY): case cpu_to_le32(DLM_MSG_CONVERT_REPLY): case cpu_to_le32(DLM_MSG_GRANT): - if (!lkb->lkb_lvbptr) + if (!lkb->lkb_lvbptr || !(lkb->lkb_exflags & DLM_LKF_VALBLK)) break; memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); break; @@ -5080,8 +5111,11 @@ void dlm_receive_buffer(union dlm_packet *p, int nodeid) down_read(&ls->ls_recv_active); if (hd->h_cmd == DLM_MSG) dlm_receive_message(ls, &p->message, nodeid); - else + else if (hd->h_cmd == DLM_RCOM) dlm_receive_rcom(ls, &p->rcom, nodeid); + else + log_error(ls, "invalid h_cmd %d from %d lockspace %x", + hd->h_cmd, nodeid, le32_to_cpu(hd->u.h_lockspace)); up_read(&ls->ls_recv_active); dlm_put_lockspace(ls); @@ -5801,6 +5835,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, { struct dlm_lkb *lkb; struct dlm_args args; + bool do_put = true; int error; dlm_lock_recovery(ls); @@ -5811,13 +5846,14 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, goto out; } + trace_dlm_lock_start(ls, lkb, name, namelen, mode, flags); + if (flags & DLM_LKF_VALBLK) { ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); if (!ua->lksb.sb_lvbptr) { kfree(ua); - __put_lkb(ls, lkb); error = -ENOMEM; - goto out; + goto out_put; } } #ifdef CONFIG_DLM_DEPRECATED_API @@ -5831,8 +5867,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, kfree(ua->lksb.sb_lvbptr); ua->lksb.sb_lvbptr = NULL; kfree(ua); - __put_lkb(ls, lkb); - goto out; + goto out_put; } /* After ua is attached to lkb it will be freed by dlm_free_lkb(). @@ -5851,8 +5886,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, error = 0; fallthrough; default: - __put_lkb(ls, lkb); - goto out; + goto out_put; } /* add this new lkb to the per-process list of locks */ @@ -5860,6 +5894,11 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, hold_lkb(lkb); list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); spin_unlock(&ua->proc->locks_spin); + do_put = false; + out_put: + trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, false); + if (do_put) + __put_lkb(ls, lkb); out: dlm_unlock_recovery(ls); return error; @@ -5885,6 +5924,8 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, if (error) goto out; + trace_dlm_lock_start(ls, lkb, NULL, 0, mode, flags); + /* user can change the params on its lock when it converts it, or add an lvb that didn't exist before */ @@ -5922,6 +5963,7 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK) error = 0; out_put: + trace_dlm_lock_end(ls, lkb, NULL, 0, mode, flags, error, false); dlm_put_lkb(lkb); out: dlm_unlock_recovery(ls); @@ -6014,6 +6056,8 @@ int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, if (error) goto out; + trace_dlm_unlock_start(ls, lkb, flags); + ua = lkb->lkb_ua; if (lvb_in && ua->lksb.sb_lvbptr) @@ -6042,6 +6086,7 @@ int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking); spin_unlock(&ua->proc->locks_spin); out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); dlm_put_lkb(lkb); out: dlm_unlock_recovery(ls); @@ -6063,6 +6108,8 @@ int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, if (error) goto out; + trace_dlm_unlock_start(ls, lkb, flags); + ua = lkb->lkb_ua; if (ua_tmp->castparam) ua->castparam = ua_tmp->castparam; @@ -6080,6 +6127,7 @@ int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, if (error == -EBUSY) error = 0; out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); dlm_put_lkb(lkb); out: dlm_unlock_recovery(ls); @@ -6101,6 +6149,8 @@ int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid) if (error) goto out; + trace_dlm_unlock_start(ls, lkb, flags); + ua = lkb->lkb_ua; error = set_unlock_args(flags, ua, &args); @@ -6129,6 +6179,7 @@ int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid) if (error == -EBUSY) error = 0; out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); dlm_put_lkb(lkb); out: dlm_unlock_recovery(ls); @@ -6184,7 +6235,7 @@ static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls, { struct dlm_lkb *lkb = NULL; - mutex_lock(&ls->ls_clear_proc_locks); + spin_lock(&ls->ls_clear_proc_locks); if (list_empty(&proc->locks)) goto out; @@ -6196,7 +6247,7 @@ static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls, else lkb->lkb_flags |= DLM_IFL_DEAD; out: - mutex_unlock(&ls->ls_clear_proc_locks); + spin_unlock(&ls->ls_clear_proc_locks); return lkb; } @@ -6233,7 +6284,7 @@ void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) dlm_put_lkb(lkb); } - mutex_lock(&ls->ls_clear_proc_locks); + spin_lock(&ls->ls_clear_proc_locks); /* in-progress unlocks */ list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { @@ -6249,7 +6300,7 @@ void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) dlm_put_lkb(lkb); } - mutex_unlock(&ls->ls_clear_proc_locks); + spin_unlock(&ls->ls_clear_proc_locks); dlm_unlock_recovery(ls); } diff --git a/fs/dlm/lock.h b/fs/dlm/lock.h index a7b6474f009dcd1b3472fa345ee8947f32d4ba95..40c76b5544da54eb6f00529ce7bee7f9878bd6c8 100644 --- a/fs/dlm/lock.h +++ b/fs/dlm/lock.h @@ -36,7 +36,7 @@ static inline void dlm_adjust_timeouts(struct dlm_ls *ls) { } int dlm_master_lookup(struct dlm_ls *ls, int nodeid, char *name, int len, unsigned int flags, int *r_nodeid, int *result); -int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len, +int dlm_search_rsb_tree(struct rb_root *tree, const void *name, int len, struct dlm_rsb **r_ret); void dlm_recover_purge(struct dlm_ls *ls); diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index 3972f4d86c7551c6739fefa8573f956a279cb110..bae050df7abff63d62073b6df50a56a1e976f64f 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -416,7 +416,7 @@ static int new_lockspace(const char *name, const char *cluster, if (namelen > DLM_LOCKSPACE_LEN || namelen == 0) return -EINVAL; - if (!lvblen || (lvblen % 8)) + if (lvblen % 8) return -EINVAL; if (!try_module_get(THIS_MODULE)) @@ -584,7 +584,7 @@ static int new_lockspace(const char *name, const char *cluster, atomic_set(&ls->ls_requestqueue_cnt, 0); init_waitqueue_head(&ls->ls_requestqueue_wait); mutex_init(&ls->ls_requestqueue_mutex); - mutex_init(&ls->ls_clear_proc_locks); + spin_lock_init(&ls->ls_clear_proc_locks); /* Due backwards compatibility with 3.1 we need to use maximum * possible dlm message size to be sure the message will fit and @@ -703,10 +703,11 @@ static int new_lockspace(const char *name, const char *cluster, return error; } -int dlm_new_lockspace(const char *name, const char *cluster, - uint32_t flags, int lvblen, - const struct dlm_lockspace_ops *ops, void *ops_arg, - int *ops_result, dlm_lockspace_t **lockspace) +static int __dlm_new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) { int error = 0; @@ -732,6 +733,25 @@ int dlm_new_lockspace(const char *name, const char *cluster, return error; } +int dlm_new_lockspace(const char *name, const char *cluster, uint32_t flags, + int lvblen, const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) +{ + return __dlm_new_lockspace(name, cluster, flags | DLM_LSFL_FS, lvblen, + ops, ops_arg, ops_result, lockspace); +} + +int dlm_new_user_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) +{ + return __dlm_new_lockspace(name, cluster, flags, lvblen, ops, + ops_arg, ops_result, lockspace); +} + static int lkb_idr_is_local(int id, void *p, void *data) { struct dlm_lkb *lkb = p; diff --git a/fs/dlm/lockspace.h b/fs/dlm/lockspace.h index 306fc4f4ea15dbd88eb161b248b3d0f3c56611ce..03f4a4a3a871c6bab0cc92847db45048899d0fc5 100644 --- a/fs/dlm/lockspace.h +++ b/fs/dlm/lockspace.h @@ -12,6 +12,14 @@ #ifndef __LOCKSPACE_DOT_H__ #define __LOCKSPACE_DOT_H__ +/* DLM_LSFL_FS + * The lockspace user is in the kernel (i.e. filesystem). Enables + * direct bast/cast callbacks. + * + * internal lockspace flag - will be removed in future + */ +#define DLM_LSFL_FS 0x00000004 + int dlm_lockspace_init(void); void dlm_lockspace_exit(void); struct dlm_ls *dlm_find_lockspace_global(uint32_t id); @@ -20,6 +28,11 @@ struct dlm_ls *dlm_find_lockspace_device(int minor); void dlm_put_lockspace(struct dlm_ls *ls); void dlm_stop_lockspaces(void); void dlm_stop_lockspaces_check(void); +int dlm_new_user_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace); #endif /* __LOCKSPACE_DOT_H__ */ diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c index a4e84e8d94c873c95a2f6281e20c7cc87c613e8f..59f64c596233b822f2b14c48c42800201e5771cf 100644 --- a/fs/dlm/lowcomms.c +++ b/fs/dlm/lowcomms.c @@ -1336,6 +1336,8 @@ struct dlm_msg *dlm_lowcomms_new_msg(int nodeid, int len, gfp_t allocation, return NULL; } + /* for dlm_lowcomms_commit_msg() */ + kref_get(&msg->ref); /* we assume if successful commit must called */ msg->idx = idx; return msg; @@ -1375,6 +1377,8 @@ void dlm_lowcomms_commit_msg(struct dlm_msg *msg) { _dlm_lowcomms_commit_msg(msg); srcu_read_unlock(&connections_srcu, msg->idx); + /* because dlm_lowcomms_new_msg() */ + kref_put(&msg->ref, dlm_msg_release); } #endif diff --git a/fs/dlm/netlink.c b/fs/dlm/netlink.c index 67f68d48d60ceaa470d1154deb43f2991511fc8c..4de4b8651c6cc290c249b9fa8160361a677c1089 100644 --- a/fs/dlm/netlink.c +++ b/fs/dlm/netlink.c @@ -75,6 +75,7 @@ static struct genl_family family __ro_after_init = { .version = DLM_GENL_VERSION, .small_ops = dlm_nl_ops, .n_small_ops = ARRAY_SIZE(dlm_nl_ops), + .resv_start_op = DLM_CMD_HELLO + 1, .module = THIS_MODULE, }; diff --git a/fs/dlm/user.c b/fs/dlm/user.c index 99e8f0744513c0272628ff2cba615b0689469c88..c5d27bccc3dc44fb0c14da9514ffa30c51bfb2b9 100644 --- a/fs/dlm/user.c +++ b/fs/dlm/user.c @@ -16,6 +16,8 @@ #include #include +#include + #include "dlm_internal.h" #include "lockspace.h" #include "lock.h" @@ -184,7 +186,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, return; ls = lkb->lkb_resource->res_ls; - mutex_lock(&ls->ls_clear_proc_locks); + spin_lock(&ls->ls_clear_proc_locks); /* If ORPHAN/DEAD flag is set, it means the process is dead so an ast can't be delivered. For ORPHAN's, dlm_clear_proc_locks() freed @@ -230,7 +232,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, spin_unlock(&proc->locks_spin); } out: - mutex_unlock(&ls->ls_clear_proc_locks); + spin_unlock(&ls->ls_clear_proc_locks); } static int device_user_lock(struct dlm_user_proc *proc, @@ -421,9 +423,9 @@ static int device_create_lockspace(struct dlm_lspace_params *params) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - error = dlm_new_lockspace(params->name, dlm_config.ci_cluster_name, params->flags, - DLM_USER_LVB_LEN, NULL, NULL, NULL, - &lockspace); + error = dlm_new_user_lockspace(params->name, dlm_config.ci_cluster_name, + params->flags, DLM_USER_LVB_LEN, NULL, + NULL, NULL, &lockspace); if (error) return error; @@ -882,7 +884,9 @@ static ssize_t device_read(struct file *file, char __user *buf, size_t count, goto try_another; } - if (cb.flags & DLM_CB_CAST) { + if (cb.flags & DLM_CB_BAST) { + trace_dlm_bast(lkb->lkb_resource->res_ls, lkb, cb.mode); + } else if (cb.flags & DLM_CB_CAST) { new_mode = cb.mode; if (!cb.sb_status && lkb->lkb_lksb->sb_lvbptr && @@ -891,6 +895,7 @@ static ssize_t device_read(struct file *file, char __user *buf, size_t count, lkb->lkb_lksb->sb_status = cb.sb_status; lkb->lkb_lksb->sb_flags = cb.sb_flags; + trace_dlm_ast(lkb->lkb_resource->res_ls, lkb); } rv = copy_result_to_user(lkb->lkb_ua, diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h index 5f2b49e13731a2919ca0a168790e235b16c48935..f2ed0c0266cb7d455b95f9d453737bc4222f1804 100644 --- a/fs/ecryptfs/ecryptfs_kernel.h +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -506,7 +506,7 @@ ecryptfs_dentry_to_lower(struct dentry *dentry) return ((struct ecryptfs_dentry_info *)dentry->d_fsdata)->lower_path.dentry; } -static inline struct path * +static inline const struct path * ecryptfs_dentry_to_lower_path(struct dentry *dentry) { return &((struct ecryptfs_dentry_info *)dentry->d_fsdata)->lower_path; diff --git a/fs/ecryptfs/file.c b/fs/ecryptfs/file.c index 18d5b91cb573edc34e4e3e8876911b14c6c2fe76..268b74499c28c7c76f8e3230b66d86fec2394ee6 100644 --- a/fs/ecryptfs/file.c +++ b/fs/ecryptfs/file.c @@ -33,7 +33,7 @@ static ssize_t ecryptfs_read_update_atime(struct kiocb *iocb, struct iov_iter *to) { ssize_t rc; - struct path *path; + const struct path *path; struct file *file = iocb->ki_filp; rc = generic_file_read_iter(iocb, to); @@ -53,7 +53,7 @@ struct ecryptfs_getdents_callback { }; /* Inspired by generic filldir in fs/readdir.c */ -static int +static bool ecryptfs_filldir(struct dir_context *ctx, const char *lower_name, int lower_namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -61,18 +61,19 @@ ecryptfs_filldir(struct dir_context *ctx, const char *lower_name, container_of(ctx, struct ecryptfs_getdents_callback, ctx); size_t name_size; char *name; - int rc; + int err; + bool res; buf->filldir_called++; - rc = ecryptfs_decode_and_decrypt_filename(&name, &name_size, - buf->sb, lower_name, - lower_namelen); - if (rc) { - if (rc != -EINVAL) { + err = ecryptfs_decode_and_decrypt_filename(&name, &name_size, + buf->sb, lower_name, + lower_namelen); + if (err) { + if (err != -EINVAL) { ecryptfs_printk(KERN_DEBUG, "%s: Error attempting to decode and decrypt filename [%s]; rc = [%d]\n", - __func__, lower_name, rc); - return rc; + __func__, lower_name, err); + return false; } /* Mask -EINVAL errors as these are most likely due a plaintext @@ -81,16 +82,15 @@ ecryptfs_filldir(struct dir_context *ctx, const char *lower_name, * the "lost+found" dentry in the root directory of an Ext4 * filesystem. */ - return 0; + return true; } buf->caller->pos = buf->ctx.pos; - rc = !dir_emit(buf->caller, name, name_size, ino, d_type); + res = dir_emit(buf->caller, name, name_size, ino, d_type); kfree(name); - if (!rc) + if (res) buf->entries_written++; - - return rc; + return res; } /** @@ -111,14 +111,8 @@ static int ecryptfs_readdir(struct file *file, struct dir_context *ctx) lower_file = ecryptfs_file_to_lower(file); rc = iterate_dir(lower_file, &buf.ctx); ctx->pos = buf.ctx.pos; - if (rc < 0) - goto out; - if (buf.filldir_called && !buf.entries_written) - goto out; - if (rc >= 0) - fsstack_copy_attr_atime(inode, - file_inode(lower_file)); -out: + if (rc >= 0 && (buf.entries_written || !buf.filldir_called)) + fsstack_copy_attr_atime(inode, file_inode(lower_file)); return rc; } diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c index 16d50dface59abfc03ed0899f91a1b40ac8dd5d2..c214fe0981bdc333f1fdd75a47ef7454d971d2c7 100644 --- a/fs/ecryptfs/inode.c +++ b/fs/ecryptfs/inode.c @@ -317,7 +317,7 @@ static int ecryptfs_i_size_read(struct dentry *dentry, struct inode *inode) static struct dentry *ecryptfs_lookup_interpose(struct dentry *dentry, struct dentry *lower_dentry) { - struct path *path = ecryptfs_dentry_to_lower_path(dentry->d_parent); + const struct path *path = ecryptfs_dentry_to_lower_path(dentry->d_parent); struct inode *inode, *lower_inode; struct ecryptfs_dentry_info *dentry_info; int rc = 0; diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c index 2dd23a82e0de56165f10a51e990ffd2a1d2321d6..2dc927ba067fec4f05b5a78f08fa702457d2ef75 100644 --- a/fs/ecryptfs/main.c +++ b/fs/ecryptfs/main.c @@ -105,7 +105,7 @@ static int ecryptfs_init_lower_file(struct dentry *dentry, struct file **lower_file) { const struct cred *cred = current_cred(); - struct path *path = ecryptfs_dentry_to_lower_path(dentry); + const struct path *path = ecryptfs_dentry_to_lower_path(dentry); int rc; rc = ecryptfs_privileged_open(lower_file, path->dentry, path->mnt, diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 2d55569f96ace320b29edd1a7721c4e4ca94d7e4..51b7ac7166d9656c05ef437e680f92a57b09d0ba 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -317,52 +317,61 @@ dstmap_out: return ret; } -static int z_erofs_shifted_transform(struct z_erofs_decompress_req *rq, - struct page **pagepool) +static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq, + struct page **pagepool) { - const unsigned int nrpages_out = + const unsigned int inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; + const unsigned int outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; const unsigned int righthalf = min_t(unsigned int, rq->outputsize, PAGE_SIZE - rq->pageofs_out); const unsigned int lefthalf = rq->outputsize - righthalf; + const unsigned int interlaced_offset = + rq->alg == Z_EROFS_COMPRESSION_SHIFTED ? 0 : rq->pageofs_out; unsigned char *src, *dst; - if (nrpages_out > 2) { + if (outpages > 2 && rq->alg == Z_EROFS_COMPRESSION_SHIFTED) { DBG_BUGON(1); - return -EIO; + return -EFSCORRUPTED; } if (rq->out[0] == *rq->in) { - DBG_BUGON(nrpages_out != 1); + DBG_BUGON(rq->pageofs_out); return 0; } - src = kmap_atomic(*rq->in) + rq->pageofs_in; + src = kmap_local_page(rq->in[inpages - 1]) + rq->pageofs_in; if (rq->out[0]) { - dst = kmap_atomic(rq->out[0]); - memcpy(dst + rq->pageofs_out, src, righthalf); - kunmap_atomic(dst); + dst = kmap_local_page(rq->out[0]); + memcpy(dst + rq->pageofs_out, src + interlaced_offset, + righthalf); + kunmap_local(dst); } - if (nrpages_out == 2) { - DBG_BUGON(!rq->out[1]); - if (rq->out[1] == *rq->in) { + if (outpages > inpages) { + DBG_BUGON(!rq->out[outpages - 1]); + if (rq->out[outpages - 1] != rq->in[inpages - 1]) { + dst = kmap_local_page(rq->out[outpages - 1]); + memcpy(dst, interlaced_offset ? src : + (src + righthalf), lefthalf); + kunmap_local(dst); + } else if (!interlaced_offset) { memmove(src, src + righthalf, lefthalf); - } else { - dst = kmap_atomic(rq->out[1]); - memcpy(dst, src + righthalf, lefthalf); - kunmap_atomic(dst); } } - kunmap_atomic(src); + kunmap_local(src); return 0; } static struct z_erofs_decompressor decompressors[] = { [Z_EROFS_COMPRESSION_SHIFTED] = { - .decompress = z_erofs_shifted_transform, + .decompress = z_erofs_transform_plain, .name = "shifted" }, + [Z_EROFS_COMPRESSION_INTERLACED] = { + .decompress = z_erofs_transform_plain, + .name = "interlaced" + }, [Z_EROFS_COMPRESSION_LZ4] = { .decompress = z_erofs_lz4_decompress, .name = "lz4" diff --git a/fs/erofs/decompressor_lzma.c b/fs/erofs/decompressor_lzma.c index 5e59b3f523eb62f85e1061f51dfb1d000557b2ca..091fd5adf818f0026f67f70e323fe7b2a7531062 100644 --- a/fs/erofs/decompressor_lzma.c +++ b/fs/erofs/decompressor_lzma.c @@ -217,6 +217,9 @@ again: strm->buf.out_size = min_t(u32, outlen, PAGE_SIZE - pageofs); outlen -= strm->buf.out_size; + if (!rq->out[no] && rq->fillgaps) /* deduped */ + rq->out[no] = erofs_allocpage(pagepool, + GFP_KERNEL | __GFP_NOFAIL); if (rq->out[no]) strm->buf.out = kmap(rq->out[no]) + pageofs; pageofs = 0; diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 2b48373f690b65f93550e916d06d7e61f70487f6..dbcd24371002bbebefed025ba15415e3193101ed 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -25,6 +25,8 @@ #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 #define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 #define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 #define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_ZERO_PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ @@ -32,7 +34,9 @@ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \ - EROFS_FEATURE_INCOMPAT_ZTAILPACKING) + EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \ + EROFS_FEATURE_INCOMPAT_FRAGMENTS | \ + EROFS_FEATURE_INCOMPAT_DEDUPE) #define EROFS_SB_EXTSLOT_SIZE 16 @@ -71,7 +75,9 @@ struct erofs_super_block { } __packed u1; __le16 extra_devices; /* # of devices besides the primary device */ __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */ - __u8 reserved2[38]; + __u8 reserved[6]; + __le64 packed_nid; /* nid of the special packed inode */ + __u8 reserved2[24]; }; /* @@ -295,16 +301,27 @@ struct z_erofs_lzma_cfgs { * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) * bit 3 : tailpacking inline pcluster (0 - off; 1 - on) + * bit 4 : interlaced plain pcluster (0 - off; 1 - on) + * bit 5 : fragment pcluster (0 - off; 1 - on) */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 #define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +#define Z_EROFS_FRAGMENT_INODE_BIT 7 struct z_erofs_map_header { - __le16 h_reserved1; - /* indicates the encoded size of tailpacking data */ - __le16 h_idata_size; + union { + /* fragment data offset in the packed inode */ + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + /* indicates the encoded size of tailpacking data */ + __le16 h_idata_size; + }; + }; __le16 h_advise; /* * bit 0-3 : algorithm type of head 1 (logical cluster type 01); @@ -313,7 +330,8 @@ struct z_erofs_map_header { __u8 h_algorithmtype; /* * bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096; - * bit 3-7 : reserved. + * bit 3-6 : reserved; + * bit 7 : move the whole file into packed inode or not. */ __u8 h_clusterbits; }; @@ -355,6 +373,9 @@ enum { #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2 #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0 +/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ +#define Z_EROFS_VLE_DI_PARTIAL_REF (1 << 15) + /* * D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the * compressed block count of a compressed extent (in logical clusters, aka. @@ -402,6 +423,10 @@ struct erofs_dirent { /* check the EROFS on-disk layout strictly at compile time */ static inline void erofs_check_ondisk_layout_definitions(void) { + const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) { + .h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT + }; + BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128); BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32); BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64); @@ -419,6 +444,9 @@ static inline void erofs_check_ondisk_layout_definitions(void) BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) < Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1); + /* exclude old compiler versions like gcc 7.5.0 */ + BUILD_BUG_ON(__builtin_constant_p(fmh) ? + fmh != cpu_to_le64(1ULL << 63) : 0); } #endif diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index 8e01d89c3319e993293ba8099f5556dd7bfe6edc..998cd26a1b3b113106e6ba9c0aef898b29342750 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -1,10 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2022, Alibaba Cloud + * Copyright (C) 2022, Bytedance Inc. All rights reserved. */ #include #include "internal.h" +static DEFINE_MUTEX(erofs_domain_list_lock); +static DEFINE_MUTEX(erofs_domain_cookies_lock); +static LIST_HEAD(erofs_domain_list); +static struct vfsmount *erofs_pseudo_mnt; + static struct netfs_io_request *erofs_fscache_alloc_request(struct address_space *mapping, loff_t start, size_t len) { @@ -222,8 +228,10 @@ static int erofs_fscache_meta_read_folio(struct file *data, struct folio *folio) rreq = erofs_fscache_alloc_request(folio_mapping(folio), folio_pos(folio), folio_size(folio)); - if (IS_ERR(rreq)) + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); goto out; + } return erofs_fscache_read_folios_async(mdev.m_fscache->cookie, rreq, mdev.m_pa); @@ -232,111 +240,111 @@ out: return ret; } -static int erofs_fscache_read_folio_inline(struct folio *folio, - struct erofs_map_blocks *map) -{ - struct super_block *sb = folio_mapping(folio)->host->i_sb; - struct erofs_buf buf = __EROFS_BUF_INITIALIZER; - erofs_blk_t blknr; - size_t offset, len; - void *src, *dst; - - /* For tail packing layout, the offset may be non-zero. */ - offset = erofs_blkoff(map->m_pa); - blknr = erofs_blknr(map->m_pa); - len = map->m_llen; - - src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP); - if (IS_ERR(src)) - return PTR_ERR(src); - - dst = kmap_local_folio(folio, 0); - memcpy(dst, src + offset, len); - memset(dst + len, 0, PAGE_SIZE - len); - kunmap_local(dst); - - erofs_put_metabuf(&buf); - return 0; -} - -static int erofs_fscache_read_folio(struct file *file, struct folio *folio) +/* + * Read into page cache in the range described by (@pos, @len). + * + * On return, the caller is responsible for page unlocking if the output @unlock + * is true, or the callee will take this responsibility through netfs_io_request + * interface. + * + * The return value is the number of bytes successfully handled, or negative + * error code on failure. The only exception is that, the length of the range + * instead of the error code is returned on failure after netfs_io_request is + * allocated, so that .readahead() could advance rac accordingly. + */ +static int erofs_fscache_data_read(struct address_space *mapping, + loff_t pos, size_t len, bool *unlock) { - struct inode *inode = folio_mapping(folio)->host; + struct inode *inode = mapping->host; struct super_block *sb = inode->i_sb; + struct netfs_io_request *rreq; struct erofs_map_blocks map; struct erofs_map_dev mdev; - struct netfs_io_request *rreq; - erofs_off_t pos; - loff_t pstart; + struct iov_iter iter; + size_t count; int ret; - DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ); + *unlock = true; - pos = folio_pos(folio); map.m_la = pos; - ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); if (ret) - goto out_unlock; + return ret; - if (!(map.m_flags & EROFS_MAP_MAPPED)) { - folio_zero_range(folio, 0, folio_size(folio)); - goto out_uptodate; + if (map.m_flags & EROFS_MAP_META) { + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + erofs_blk_t blknr; + size_t offset, size; + void *src; + + /* For tail packing layout, the offset may be non-zero. */ + offset = erofs_blkoff(map.m_pa); + blknr = erofs_blknr(map.m_pa); + size = map.m_llen; + + src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP); + if (IS_ERR(src)) + return PTR_ERR(src); + + iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, PAGE_SIZE); + if (copy_to_iter(src + offset, size, &iter) != size) + return -EFAULT; + iov_iter_zero(PAGE_SIZE - size, &iter); + erofs_put_metabuf(&buf); + return PAGE_SIZE; } - if (map.m_flags & EROFS_MAP_META) { - ret = erofs_fscache_read_folio_inline(folio, &map); - goto out_uptodate; + count = min_t(size_t, map.m_llen - (pos - map.m_la), len); + DBG_BUGON(!count || count % PAGE_SIZE); + + if (!(map.m_flags & EROFS_MAP_MAPPED)) { + iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, count); + iov_iter_zero(count, &iter); + return count; } mdev = (struct erofs_map_dev) { .m_deviceid = map.m_deviceid, .m_pa = map.m_pa, }; - ret = erofs_map_dev(sb, &mdev); if (ret) - goto out_unlock; - + return ret; - rreq = erofs_fscache_alloc_request(folio_mapping(folio), - folio_pos(folio), folio_size(folio)); + rreq = erofs_fscache_alloc_request(mapping, pos, count); if (IS_ERR(rreq)) - goto out_unlock; - - pstart = mdev.m_pa + (pos - map.m_la); - return erofs_fscache_read_folios_async(mdev.m_fscache->cookie, - rreq, pstart); + return PTR_ERR(rreq); -out_uptodate: - if (!ret) - folio_mark_uptodate(folio); -out_unlock: - folio_unlock(folio); - return ret; + *unlock = false; + erofs_fscache_read_folios_async(mdev.m_fscache->cookie, + rreq, mdev.m_pa + (pos - map.m_la)); + return count; } -static void erofs_fscache_advance_folios(struct readahead_control *rac, - size_t len, bool unlock) +static int erofs_fscache_read_folio(struct file *file, struct folio *folio) { - while (len) { - struct folio *folio = readahead_folio(rac); - len -= folio_size(folio); - if (unlock) { + bool unlock; + int ret; + + DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ); + + ret = erofs_fscache_data_read(folio_mapping(folio), folio_pos(folio), + folio_size(folio), &unlock); + if (unlock) { + if (ret > 0) folio_mark_uptodate(folio); - folio_unlock(folio); - } + folio_unlock(folio); } + return ret < 0 ? ret : 0; } static void erofs_fscache_readahead(struct readahead_control *rac) { - struct inode *inode = rac->mapping->host; - struct super_block *sb = inode->i_sb; - size_t len, count, done = 0; - erofs_off_t pos; - loff_t start, offset; - int ret; + struct folio *folio; + size_t len, done = 0; + loff_t start, pos; + bool unlock; + int ret, size; if (!readahead_count(rac)) return; @@ -345,67 +353,22 @@ static void erofs_fscache_readahead(struct readahead_control *rac) len = readahead_length(rac); do { - struct erofs_map_blocks map; - struct erofs_map_dev mdev; - struct netfs_io_request *rreq; - pos = start + done; - map.m_la = pos; - - ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); - if (ret) + ret = erofs_fscache_data_read(rac->mapping, pos, + len - done, &unlock); + if (ret <= 0) return; - offset = start + done; - count = min_t(size_t, map.m_llen - (pos - map.m_la), - len - done); - - if (!(map.m_flags & EROFS_MAP_MAPPED)) { - struct iov_iter iter; - - iov_iter_xarray(&iter, READ, &rac->mapping->i_pages, - offset, count); - iov_iter_zero(count, &iter); - - erofs_fscache_advance_folios(rac, count, true); - ret = count; - continue; - } - - if (map.m_flags & EROFS_MAP_META) { - struct folio *folio = readahead_folio(rac); - - ret = erofs_fscache_read_folio_inline(folio, &map); - if (!ret) { + size = ret; + while (size) { + folio = readahead_folio(rac); + size -= folio_size(folio); + if (unlock) { folio_mark_uptodate(folio); - ret = folio_size(folio); + folio_unlock(folio); } - - folio_unlock(folio); - continue; } - - mdev = (struct erofs_map_dev) { - .m_deviceid = map.m_deviceid, - .m_pa = map.m_pa, - }; - ret = erofs_map_dev(sb, &mdev); - if (ret) - return; - - rreq = erofs_fscache_alloc_request(rac->mapping, offset, count); - if (IS_ERR(rreq)) - return; - /* - * Drop the ref of folios here. Unlock them in - * rreq_unlock_folios() when rreq complete. - */ - erofs_fscache_advance_folios(rac, count, false); - ret = erofs_fscache_read_folios_async(mdev.m_fscache->cookie, - rreq, mdev.m_pa + (pos - map.m_la)); - if (!ret) - ret = count; - } while (ret > 0 && ((done += ret) < len)); + } while ((done += ret) < len); } static const struct address_space_operations erofs_fscache_meta_aops = { @@ -417,9 +380,114 @@ const struct address_space_operations erofs_fscache_access_aops = { .readahead = erofs_fscache_readahead, }; -int erofs_fscache_register_cookie(struct super_block *sb, - struct erofs_fscache **fscache, - char *name, bool need_inode) +static void erofs_fscache_domain_put(struct erofs_domain *domain) +{ + if (!domain) + return; + mutex_lock(&erofs_domain_list_lock); + if (refcount_dec_and_test(&domain->ref)) { + list_del(&domain->list); + if (list_empty(&erofs_domain_list)) { + kern_unmount(erofs_pseudo_mnt); + erofs_pseudo_mnt = NULL; + } + mutex_unlock(&erofs_domain_list_lock); + fscache_relinquish_volume(domain->volume, NULL, false); + kfree(domain->domain_id); + kfree(domain); + return; + } + mutex_unlock(&erofs_domain_list_lock); +} + +static int erofs_fscache_register_volume(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + char *domain_id = sbi->opt.domain_id; + struct fscache_volume *volume; + char *name; + int ret = 0; + + name = kasprintf(GFP_KERNEL, "erofs,%s", + domain_id ? domain_id : sbi->opt.fsid); + if (!name) + return -ENOMEM; + + volume = fscache_acquire_volume(name, NULL, NULL, 0); + if (IS_ERR_OR_NULL(volume)) { + erofs_err(sb, "failed to register volume for %s", name); + ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP; + volume = NULL; + } + + sbi->volume = volume; + kfree(name); + return ret; +} + +static int erofs_fscache_init_domain(struct super_block *sb) +{ + int err; + struct erofs_domain *domain; + struct erofs_sb_info *sbi = EROFS_SB(sb); + + domain = kzalloc(sizeof(struct erofs_domain), GFP_KERNEL); + if (!domain) + return -ENOMEM; + + domain->domain_id = kstrdup(sbi->opt.domain_id, GFP_KERNEL); + if (!domain->domain_id) { + kfree(domain); + return -ENOMEM; + } + + err = erofs_fscache_register_volume(sb); + if (err) + goto out; + + if (!erofs_pseudo_mnt) { + erofs_pseudo_mnt = kern_mount(&erofs_fs_type); + if (IS_ERR(erofs_pseudo_mnt)) { + err = PTR_ERR(erofs_pseudo_mnt); + goto out; + } + } + + domain->volume = sbi->volume; + refcount_set(&domain->ref, 1); + list_add(&domain->list, &erofs_domain_list); + sbi->domain = domain; + return 0; +out: + kfree(domain->domain_id); + kfree(domain); + return err; +} + +static int erofs_fscache_register_domain(struct super_block *sb) +{ + int err; + struct erofs_domain *domain; + struct erofs_sb_info *sbi = EROFS_SB(sb); + + mutex_lock(&erofs_domain_list_lock); + list_for_each_entry(domain, &erofs_domain_list, list) { + if (!strcmp(domain->domain_id, sbi->opt.domain_id)) { + sbi->domain = domain; + sbi->volume = domain->volume; + refcount_inc(&domain->ref); + mutex_unlock(&erofs_domain_list_lock); + return 0; + } + } + err = erofs_fscache_init_domain(sb); + mutex_unlock(&erofs_domain_list_lock); + return err; +} + +static +struct erofs_fscache *erofs_fscache_acquire_cookie(struct super_block *sb, + char *name, bool need_inode) { struct fscache_volume *volume = EROFS_SB(sb)->volume; struct erofs_fscache *ctx; @@ -428,7 +496,7 @@ int erofs_fscache_register_cookie(struct super_block *sb, ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) - return -ENOMEM; + return ERR_PTR(-ENOMEM); cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE, name, strlen(name), NULL, 0, 0); @@ -458,63 +526,146 @@ int erofs_fscache_register_cookie(struct super_block *sb, ctx->inode = inode; } - *fscache = ctx; - return 0; + return ctx; err_cookie: fscache_unuse_cookie(ctx->cookie, NULL, NULL); fscache_relinquish_cookie(ctx->cookie, false); - ctx->cookie = NULL; err: kfree(ctx); - return ret; + return ERR_PTR(ret); } -void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache) +static void erofs_fscache_relinquish_cookie(struct erofs_fscache *ctx) { - struct erofs_fscache *ctx = *fscache; - - if (!ctx) - return; - fscache_unuse_cookie(ctx->cookie, NULL, NULL); fscache_relinquish_cookie(ctx->cookie, false); - ctx->cookie = NULL; - iput(ctx->inode); - ctx->inode = NULL; - + kfree(ctx->name); kfree(ctx); - *fscache = NULL; +} + +static +struct erofs_fscache *erofs_fscache_domain_init_cookie(struct super_block *sb, + char *name, bool need_inode) +{ + int err; + struct inode *inode; + struct erofs_fscache *ctx; + struct erofs_domain *domain = EROFS_SB(sb)->domain; + + ctx = erofs_fscache_acquire_cookie(sb, name, need_inode); + if (IS_ERR(ctx)) + return ctx; + + ctx->name = kstrdup(name, GFP_KERNEL); + if (!ctx->name) { + err = -ENOMEM; + goto out; + } + + inode = new_inode(erofs_pseudo_mnt->mnt_sb); + if (!inode) { + err = -ENOMEM; + goto out; + } + + ctx->domain = domain; + ctx->anon_inode = inode; + inode->i_private = ctx; + refcount_inc(&domain->ref); + return ctx; +out: + erofs_fscache_relinquish_cookie(ctx); + return ERR_PTR(err); +} + +static +struct erofs_fscache *erofs_domain_register_cookie(struct super_block *sb, + char *name, bool need_inode) +{ + struct inode *inode; + struct erofs_fscache *ctx; + struct erofs_domain *domain = EROFS_SB(sb)->domain; + struct super_block *psb = erofs_pseudo_mnt->mnt_sb; + + mutex_lock(&erofs_domain_cookies_lock); + list_for_each_entry(inode, &psb->s_inodes, i_sb_list) { + ctx = inode->i_private; + if (!ctx || ctx->domain != domain || strcmp(ctx->name, name)) + continue; + igrab(inode); + mutex_unlock(&erofs_domain_cookies_lock); + return ctx; + } + ctx = erofs_fscache_domain_init_cookie(sb, name, need_inode); + mutex_unlock(&erofs_domain_cookies_lock); + return ctx; +} + +struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, + char *name, bool need_inode) +{ + if (EROFS_SB(sb)->opt.domain_id) + return erofs_domain_register_cookie(sb, name, need_inode); + return erofs_fscache_acquire_cookie(sb, name, need_inode); +} + +void erofs_fscache_unregister_cookie(struct erofs_fscache *ctx) +{ + bool drop; + struct erofs_domain *domain; + + if (!ctx) + return; + domain = ctx->domain; + if (domain) { + mutex_lock(&erofs_domain_cookies_lock); + drop = atomic_read(&ctx->anon_inode->i_count) == 1; + iput(ctx->anon_inode); + mutex_unlock(&erofs_domain_cookies_lock); + if (!drop) + return; + } + + erofs_fscache_relinquish_cookie(ctx); + erofs_fscache_domain_put(domain); } int erofs_fscache_register_fs(struct super_block *sb) { + int ret; struct erofs_sb_info *sbi = EROFS_SB(sb); - struct fscache_volume *volume; - char *name; - int ret = 0; + struct erofs_fscache *fscache; - name = kasprintf(GFP_KERNEL, "erofs,%s", sbi->opt.fsid); - if (!name) - return -ENOMEM; + if (sbi->opt.domain_id) + ret = erofs_fscache_register_domain(sb); + else + ret = erofs_fscache_register_volume(sb); + if (ret) + return ret; - volume = fscache_acquire_volume(name, NULL, NULL, 0); - if (IS_ERR_OR_NULL(volume)) { - erofs_err(sb, "failed to register volume for %s", name); - ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP; - volume = NULL; - } + /* acquired domain/volume will be relinquished in kill_sb() on error */ + fscache = erofs_fscache_register_cookie(sb, sbi->opt.fsid, true); + if (IS_ERR(fscache)) + return PTR_ERR(fscache); - sbi->volume = volume; - kfree(name); - return ret; + sbi->s_fscache = fscache; + return 0; } void erofs_fscache_unregister_fs(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); - fscache_relinquish_volume(sbi->volume, NULL, false); + erofs_fscache_unregister_cookie(sbi->s_fscache); + + if (sbi->domain) + erofs_fscache_domain_put(sbi->domain); + else + fscache_relinquish_volume(sbi->volume, NULL, false); + + sbi->s_fscache = NULL; sbi->volume = NULL; + sbi->domain = NULL; } diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 95a403720e8c7b8dd99ec92e7501d0a1899f984b..ad2a82f2eb4cdd0084ef659d1f6522a8a5cc2258 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -214,7 +214,7 @@ static int erofs_fill_symlink(struct inode *inode, void *kaddr, /* if it cannot be handled with fast symlink scheme */ if (vi->datalayout != EROFS_INODE_FLAT_INLINE || - inode->i_size >= EROFS_BLKSIZ) { + inode->i_size >= EROFS_BLKSIZ || inode->i_size < 0) { inode->i_op = &erofs_symlink_iops; return 0; } @@ -241,7 +241,7 @@ static int erofs_fill_symlink(struct inode *inode, void *kaddr, return 0; } -static int erofs_fill_inode(struct inode *inode, int isdir) +static int erofs_fill_inode(struct inode *inode) { struct erofs_inode *vi = EROFS_I(inode); struct erofs_buf buf = __EROFS_BUF_INITIALIZER; @@ -249,7 +249,7 @@ static int erofs_fill_inode(struct inode *inode, int isdir) unsigned int ofs; int err = 0; - trace_erofs_fill_inode(inode, isdir); + trace_erofs_fill_inode(inode); /* read inode base data from disk */ kaddr = erofs_read_inode(&buf, inode, &ofs); @@ -324,21 +324,13 @@ static int erofs_iget_set_actor(struct inode *inode, void *opaque) return 0; } -static inline struct inode *erofs_iget_locked(struct super_block *sb, - erofs_nid_t nid) +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid) { const unsigned long hashval = erofs_inode_hash(nid); + struct inode *inode; - return iget5_locked(sb, hashval, erofs_ilookup_test_actor, + inode = iget5_locked(sb, hashval, erofs_ilookup_test_actor, erofs_iget_set_actor, &nid); -} - -struct inode *erofs_iget(struct super_block *sb, - erofs_nid_t nid, - bool isdir) -{ - struct inode *inode = erofs_iget_locked(sb, nid); - if (!inode) return ERR_PTR(-ENOMEM); @@ -348,10 +340,10 @@ struct inode *erofs_iget(struct super_block *sb, vi->nid = nid; - err = erofs_fill_inode(inode, isdir); - if (!err) + err = erofs_fill_inode(inode); + if (!err) { unlock_new_inode(inode); - else { + } else { iget_failed(inode); inode = ERR_PTR(err); } diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index cfee49d33b95a0b5ce4665af3320a90cc32df45f..1701df48c4464977d8e0524024ce42ff55985914 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -76,6 +76,7 @@ struct erofs_mount_opts { #endif unsigned int mount_opt; char *fsid; + char *domain_id; }; struct erofs_dev_context { @@ -98,9 +99,19 @@ struct erofs_sb_lz4_info { u16 max_pclusterblks; }; +struct erofs_domain { + refcount_t ref; + struct list_head list; + struct fscache_volume *volume; + char *domain_id; +}; + struct erofs_fscache { struct fscache_cookie *cookie; struct inode *inode; + struct inode *anon_inode; + struct erofs_domain *domain; + char *name; }; struct erofs_sb_info { @@ -120,6 +131,7 @@ struct erofs_sb_info { struct inode *managed_cache; struct erofs_sb_lz4_info lz4; + struct inode *packed_inode; #endif /* CONFIG_EROFS_FS_ZIP */ struct erofs_dev_context *devs; struct dax_device *dax_dev; @@ -157,6 +169,7 @@ struct erofs_sb_info { /* fscache support */ struct fscache_volume *volume; struct erofs_fscache *s_fscache; + struct erofs_domain *domain; }; #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) @@ -183,7 +196,6 @@ enum { EROFS_ZIP_CACHE_READAROUND }; -#ifdef CONFIG_EROFS_FS_ZIP #define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL) /* basic unit of the workstation of a super_block */ @@ -195,7 +207,6 @@ struct erofs_workgroup { atomic_t refcount; }; -#if defined(CONFIG_SMP) static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, int val) { @@ -224,35 +235,6 @@ static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) return atomic_cond_read_relaxed(&grp->refcount, VAL != EROFS_LOCKED_MAGIC); } -#else -static inline bool erofs_workgroup_try_to_freeze(struct erofs_workgroup *grp, - int val) -{ - preempt_disable(); - /* no need to spin on UP platforms, let's just disable preemption. */ - if (val != atomic_read(&grp->refcount)) { - preempt_enable(); - return false; - } - return true; -} - -static inline void erofs_workgroup_unfreeze(struct erofs_workgroup *grp, - int orig_val) -{ - preempt_enable(); -} - -static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp) -{ - int v = atomic_read(&grp->refcount); - - /* workgroup is never freezed on uniprocessor systems */ - DBG_BUGON(v == EROFS_LOCKED_MAGIC); - return v; -} -#endif /* !CONFIG_SMP */ -#endif /* !CONFIG_EROFS_FS_ZIP */ /* we strictly follow PAGE_SIZE and no buffer head yet */ #define LOG_BLOCK_SIZE PAGE_SHIFT @@ -306,6 +288,8 @@ EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) /* atomic flag definitions */ @@ -341,8 +325,13 @@ struct erofs_inode { unsigned char z_algorithmtype[2]; unsigned char z_logical_clusterbits; unsigned long z_tailextent_headlcn; - erofs_off_t z_idataoff; - unsigned short z_idata_size; + union { + struct { + erofs_off_t z_idataoff; + unsigned short z_idata_size; + }; + erofs_off_t z_fragmentoff; + }; }; #endif /* CONFIG_EROFS_FS_ZIP */ }; @@ -393,6 +382,7 @@ struct page *erofs_grab_cache_page_nowait(struct address_space *mapping, } extern const struct super_operations erofs_sops; +extern struct file_system_type erofs_fs_type; extern const struct address_space_operations erofs_raw_access_aops; extern const struct address_space_operations z_erofs_aops; @@ -400,6 +390,8 @@ extern const struct address_space_operations z_erofs_aops; enum { BH_Encoded = BH_PrivateStart, BH_FullMapped, + BH_Fragment, + BH_Partialref, }; /* Has a disk mapping */ @@ -410,6 +402,10 @@ enum { #define EROFS_MAP_ENCODED (1 << BH_Encoded) /* The length of extent is full */ #define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped) +/* Located in the special packed inode */ +#define EROFS_MAP_FRAGMENT (1 << BH_Fragment) +/* The extent refers to partial decompressed data */ +#define EROFS_MAP_PARTIAL_REF (1 << BH_Partialref) struct erofs_map_blocks { struct erofs_buf buf; @@ -431,11 +427,12 @@ struct erofs_map_blocks { #define EROFS_GET_BLOCKS_FIEMAP 0x0002 /* Used to map the whole extent if non-negligible data is requested for LZMA */ #define EROFS_GET_BLOCKS_READMORE 0x0004 -/* Used to map tail extent for tailpacking inline pcluster */ +/* Used to map tail extent for tailpacking inline or fragment pcluster */ #define EROFS_GET_BLOCKS_FINDTAIL 0x0008 enum { Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, Z_EROFS_COMPRESSION_RUNTIME_MAX }; @@ -495,7 +492,7 @@ extern const struct inode_operations erofs_generic_iops; extern const struct inode_operations erofs_symlink_iops; extern const struct inode_operations erofs_fast_symlink_iops; -struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir); +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid); int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags); @@ -610,27 +607,26 @@ static inline int z_erofs_load_lzma_config(struct super_block *sb, int erofs_fscache_register_fs(struct super_block *sb); void erofs_fscache_unregister_fs(struct super_block *sb); -int erofs_fscache_register_cookie(struct super_block *sb, - struct erofs_fscache **fscache, - char *name, bool need_inode); -void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache); +struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, + char *name, bool need_inode); +void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache); extern const struct address_space_operations erofs_fscache_access_aops; #else static inline int erofs_fscache_register_fs(struct super_block *sb) { - return 0; + return -EOPNOTSUPP; } static inline void erofs_fscache_unregister_fs(struct super_block *sb) {} -static inline int erofs_fscache_register_cookie(struct super_block *sb, - struct erofs_fscache **fscache, - char *name, bool need_inode) +static inline +struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, + char *name, bool need_inode) { - return -EOPNOTSUPP; + return ERR_PTR(-EOPNOTSUPP); } -static inline void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache) +static inline void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache) { } #endif diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c index fd75506799c4037a6743c4e89cb0321d76754c72..0dc34721080c7f5b312f92d4e1cc3dde6ecb98cc 100644 --- a/fs/erofs/namei.c +++ b/fs/erofs/namei.c @@ -185,7 +185,6 @@ int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid, if (IS_ERR(de)) return PTR_ERR(de); - /* the target page has been mapped */ if (ndirents) de = find_target_dirent(&qn, (u8 *)de, EROFS_BLKSIZ, ndirents); @@ -197,9 +196,7 @@ int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid, return PTR_ERR_OR_ZERO(de); } -/* NOTE: i_mutex is already held by vfs */ -static struct dentry *erofs_lookup(struct inode *dir, - struct dentry *dentry, +static struct dentry *erofs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { int err; @@ -207,17 +204,11 @@ static struct dentry *erofs_lookup(struct inode *dir, unsigned int d_type; struct inode *inode; - DBG_BUGON(!d_really_is_negative(dentry)); - /* dentry must be unhashed in lookup, no need to worry about */ - DBG_BUGON(!d_unhashed(dentry)); - trace_erofs_lookup(dir, dentry, flags); - /* file name exceeds fs limit */ if (dentry->d_name.len > EROFS_NAME_LEN) return ERR_PTR(-ENAMETOOLONG); - /* false uninitialized warnings on gcc 4.8.x */ err = erofs_namei(dir, &dentry->d_name, &nid, &d_type); if (err == -ENOENT) { @@ -228,7 +219,7 @@ static struct dentry *erofs_lookup(struct inode *dir, } else { erofs_dbg("%s, %pd (nid %llu) found, d_type %u", __func__, dentry, nid, d_type); - inode = erofs_iget(dir->i_sb, nid, d_type == FT_DIR); + inode = erofs_iget(dir->i_sb, nid); } return d_splice_alias(inode, dentry); } diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 3173debeaa5a187362b87cc042d3eabe6fa92181..2cf96ce1c32eed444fbf89f4aa90e4f79fbee42c 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -224,10 +224,10 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, struct erofs_device_info *dif, erofs_off_t *pos) { struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_fscache *fscache; struct erofs_deviceslot *dis; struct block_device *bdev; void *ptr; - int ret; ptr = erofs_read_metabuf(buf, sb, erofs_blknr(*pos), EROFS_KMAP); if (IS_ERR(ptr)) @@ -245,10 +245,10 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, } if (erofs_is_fscache_mode(sb)) { - ret = erofs_fscache_register_cookie(sb, &dif->fscache, - dif->path, false); - if (ret) - return ret; + fscache = erofs_fscache_register_cookie(sb, dif->path, false); + if (IS_ERR(fscache)) + return PTR_ERR(fscache); + dif->fscache = fscache; } else { bdev = blkdev_get_by_path(dif->path, FMODE_READ | FMODE_EXCL, sb->s_type); @@ -381,6 +381,17 @@ static int erofs_read_superblock(struct super_block *sb) #endif sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact)); sbi->root_nid = le16_to_cpu(dsb->root_nid); +#ifdef CONFIG_EROFS_FS_ZIP + sbi->packed_inode = NULL; + if (erofs_sb_has_fragments(sbi) && dsb->packed_nid) { + sbi->packed_inode = + erofs_iget(sb, le64_to_cpu(dsb->packed_nid)); + if (IS_ERR(sbi->packed_inode)) { + ret = PTR_ERR(sbi->packed_inode); + goto out; + } + } +#endif sbi->inos = le64_to_cpu(dsb->inos); sbi->build_time = le64_to_cpu(dsb->build_time); @@ -411,6 +422,10 @@ static int erofs_read_superblock(struct super_block *sb) erofs_info(sb, "EXPERIMENTAL compressed inline data feature in use. Use at your own risk!"); if (erofs_is_fscache_mode(sb)) erofs_info(sb, "EXPERIMENTAL fscache-based on-demand read feature in use. Use at your own risk!"); + if (erofs_sb_has_fragments(sbi)) + erofs_info(sb, "EXPERIMENTAL compressed fragments feature in use. Use at your own risk!"); + if (erofs_sb_has_dedupe(sbi)) + erofs_info(sb, "EXPERIMENTAL global deduplication feature in use. Use at your own risk!"); out: erofs_put_metabuf(&buf); return ret; @@ -440,6 +455,7 @@ enum { Opt_dax_enum, Opt_device, Opt_fsid, + Opt_domain_id, Opt_err }; @@ -465,6 +481,7 @@ static const struct fs_parameter_spec erofs_fs_parameters[] = { fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), fsparam_string("device", Opt_device), fsparam_string("fsid", Opt_fsid), + fsparam_string("domain_id", Opt_domain_id), {} }; @@ -568,6 +585,16 @@ static int erofs_fc_parse_param(struct fs_context *fc, return -ENOMEM; #else errorfc(fc, "fsid option not supported"); +#endif + break; + case Opt_domain_id: +#ifdef CONFIG_EROFS_FS_ONDEMAND + kfree(ctx->opt.domain_id); + ctx->opt.domain_id = kstrdup(param->string, GFP_KERNEL); + if (!ctx->opt.domain_id) + return -ENOMEM; +#else + errorfc(fc, "domain_id option not supported"); #endif break; default: @@ -641,7 +668,7 @@ static int erofs_init_managed_cache(struct super_block *sb) { return 0; } static struct inode *erofs_nfs_get_inode(struct super_block *sb, u64 ino, u32 generation) { - return erofs_iget(sb, ino, false); + return erofs_iget(sb, ino); } static struct dentry *erofs_fh_to_dentry(struct super_block *sb, @@ -667,7 +694,7 @@ static struct dentry *erofs_get_parent(struct dentry *child) err = erofs_namei(d_inode(child), &dotdot_name, &nid, &d_type); if (err) return ERR_PTR(err); - return d_obtain_alias(erofs_iget(child->d_sb, nid, d_type == FT_DIR)); + return d_obtain_alias(erofs_iget(child->d_sb, nid)); } static const struct export_operations erofs_export_ops = { @@ -676,6 +703,13 @@ static const struct export_operations erofs_export_ops = { .get_parent = erofs_get_parent, }; +static int erofs_fc_fill_pseudo_super(struct super_block *sb, struct fs_context *fc) +{ + static const struct tree_descr empty_descr = {""}; + + return simple_fill_super(sb, EROFS_SUPER_MAGIC, &empty_descr); +} + static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) { struct inode *inode; @@ -695,6 +729,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_fs_info = sbi; sbi->opt = ctx->opt; ctx->opt.fsid = NULL; + ctx->opt.domain_id = NULL; sbi->devs = ctx->devs; ctx->devs = NULL; @@ -706,11 +741,6 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) if (err) return err; - err = erofs_fscache_register_cookie(sb, &sbi->s_fscache, - sbi->opt.fsid, true); - if (err) - return err; - err = super_setup_bdi(sb); if (err) return err; @@ -752,7 +782,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) #endif /* get the root inode */ - inode = erofs_iget(sb, ROOT_NID(sbi), true); + inode = erofs_iget(sb, ROOT_NID(sbi)); if (IS_ERR(inode)) return PTR_ERR(inode); @@ -781,6 +811,11 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) return 0; } +static int erofs_fc_anon_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, erofs_fc_fill_pseudo_super); +} + static int erofs_fc_get_tree(struct fs_context *fc) { struct erofs_fs_context *ctx = fc->fs_private; @@ -817,7 +852,8 @@ static int erofs_release_device_info(int id, void *ptr, void *data) fs_put_dax(dif->dax_dev, NULL); if (dif->bdev) blkdev_put(dif->bdev, FMODE_READ | FMODE_EXCL); - erofs_fscache_unregister_cookie(&dif->fscache); + erofs_fscache_unregister_cookie(dif->fscache); + dif->fscache = NULL; kfree(dif->path); kfree(dif); return 0; @@ -838,6 +874,7 @@ static void erofs_fc_free(struct fs_context *fc) erofs_free_dev_context(ctx->devs); kfree(ctx->opt.fsid); + kfree(ctx->opt.domain_id); kfree(ctx); } @@ -848,10 +885,21 @@ static const struct fs_context_operations erofs_context_ops = { .free = erofs_fc_free, }; +static const struct fs_context_operations erofs_anon_context_ops = { + .get_tree = erofs_fc_anon_get_tree, +}; + static int erofs_init_fs_context(struct fs_context *fc) { - struct erofs_fs_context *ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + struct erofs_fs_context *ctx; + + /* pseudo mount for anon inodes */ + if (fc->sb_flags & SB_KERNMOUNT) { + fc->ops = &erofs_anon_context_ops; + return 0; + } + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; ctx->devs = kzalloc(sizeof(struct erofs_dev_context), GFP_KERNEL); @@ -878,8 +926,14 @@ static void erofs_kill_sb(struct super_block *sb) WARN_ON(sb->s_magic != EROFS_SUPER_MAGIC); + /* pseudo mount for anon inodes */ + if (sb->s_flags & SB_KERNMOUNT) { + kill_anon_super(sb); + return; + } + if (erofs_is_fscache_mode(sb)) - generic_shutdown_super(sb); + kill_anon_super(sb); else kill_block_super(sb); @@ -889,9 +943,9 @@ static void erofs_kill_sb(struct super_block *sb) erofs_free_dev_context(sbi->devs); fs_put_dax(sbi->dax_dev, NULL); - erofs_fscache_unregister_cookie(&sbi->s_fscache); erofs_fscache_unregister_fs(sb); kfree(sbi->opt.fsid); + kfree(sbi->opt.domain_id); kfree(sbi); sb->s_fs_info = NULL; } @@ -908,11 +962,13 @@ static void erofs_put_super(struct super_block *sb) #ifdef CONFIG_EROFS_FS_ZIP iput(sbi->managed_cache); sbi->managed_cache = NULL; + iput(sbi->packed_inode); + sbi->packed_inode = NULL; #endif - erofs_fscache_unregister_cookie(&sbi->s_fscache); + erofs_fscache_unregister_fs(sb); } -static struct file_system_type erofs_fs_type = { +struct file_system_type erofs_fs_type = { .owner = THIS_MODULE, .name = "erofs", .init_fs_context = erofs_init_fs_context, @@ -1044,6 +1100,8 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root) #ifdef CONFIG_EROFS_FS_ONDEMAND if (opt->fsid) seq_printf(seq, ",fsid=%s", opt->fsid); + if (opt->domain_id) + seq_printf(seq, ",domain_id=%s", opt->domain_id); #endif return 0; } diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c index c1383e508bbecf50b877ac1a4b529f0c4807e146..783bb7b21b514a67a9da748c786e59c8ab6522db 100644 --- a/fs/erofs/sysfs.c +++ b/fs/erofs/sysfs.c @@ -76,6 +76,8 @@ EROFS_ATTR_FEATURE(device_table); EROFS_ATTR_FEATURE(compr_head2); EROFS_ATTR_FEATURE(sb_chksum); EROFS_ATTR_FEATURE(ztailpacking); +EROFS_ATTR_FEATURE(fragments); +EROFS_ATTR_FEATURE(dedupe); static struct attribute *erofs_feat_attrs[] = { ATTR_LIST(zero_padding), @@ -86,6 +88,8 @@ static struct attribute *erofs_feat_attrs[] = { ATTR_LIST(compr_head2), ATTR_LIST(sb_chksum), ATTR_LIST(ztailpacking), + ATTR_LIST(fragments), + ATTR_LIST(dedupe), NULL, }; ATTRIBUTE_GROUPS(erofs_feat); @@ -201,12 +205,27 @@ static struct kobject erofs_feat = { int erofs_register_sysfs(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); + char *name; + char *str = NULL; int err; + if (erofs_is_fscache_mode(sb)) { + if (sbi->opt.domain_id) { + str = kasprintf(GFP_KERNEL, "%s,%s", sbi->opt.domain_id, + sbi->opt.fsid); + if (!str) + return -ENOMEM; + name = str; + } else { + name = sbi->opt.fsid; + } + } else { + name = sb->s_id; + } sbi->s_kobj.kset = &erofs_root; init_completion(&sbi->s_kobj_unregister); - err = kobject_init_and_add(&sbi->s_kobj, &erofs_sb_ktype, NULL, "%s", - erofs_is_fscache_mode(sb) ? sbi->opt.fsid : sb->s_id); + err = kobject_init_and_add(&sbi->s_kobj, &erofs_sb_ktype, NULL, "%s", name); + kfree(str); if (err) goto put_sb_kobj; return 0; diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h index 332462c59f115916d6bc7109774897bf5a038bcf..0a43c9ee9f8f3ba8d7cf999d90d39f435c78cfbf 100644 --- a/fs/erofs/xattr.h +++ b/fs/erofs/xattr.h @@ -39,9 +39,7 @@ static inline unsigned int xattrblock_offset(struct erofs_sb_info *sbi, #ifdef CONFIG_EROFS_FS_XATTR extern const struct xattr_handler erofs_xattr_user_handler; extern const struct xattr_handler erofs_xattr_trusted_handler; -#ifdef CONFIG_EROFS_FS_SECURITY extern const struct xattr_handler erofs_xattr_security_handler; -#endif static inline const struct xattr_handler *erofs_xattr_handler(unsigned int idx) { diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 5792ca9e0d5efaf59a4b9abb65a1e1dd1c503078..559380a535afffda7735fdb80ac09aebc130d8e1 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -7,6 +7,7 @@ #include "zdata.h" #include "compress.h" #include +#include #include @@ -650,6 +651,35 @@ static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, la < fe->headoffset; } +static int z_erofs_read_fragment(struct inode *inode, erofs_off_t pos, + struct page *page, unsigned int pageofs, + unsigned int len) +{ + struct inode *packed_inode = EROFS_I_SB(inode)->packed_inode; + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + u8 *src, *dst; + unsigned int i, cnt; + + pos += EROFS_I(inode)->z_fragmentoff; + for (i = 0; i < len; i += cnt) { + cnt = min_t(unsigned int, len - i, + EROFS_BLKSIZ - erofs_blkoff(pos)); + src = erofs_bread(&buf, packed_inode, + erofs_blknr(pos), EROFS_KMAP); + if (IS_ERR(src)) { + erofs_put_metabuf(&buf); + return PTR_ERR(src); + } + + dst = kmap_local_page(page); + memcpy(dst + pageofs + i, src + erofs_blkoff(pos), cnt); + kunmap_local(dst); + pos += cnt; + } + erofs_put_metabuf(&buf); + return 0; +} + static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, struct page *page, struct page **pagepool) { @@ -688,7 +718,8 @@ repeat: /* didn't get a valid pcluster previously (very rare) */ } - if (!(map->m_flags & EROFS_MAP_MAPPED)) + if (!(map->m_flags & EROFS_MAP_MAPPED) || + map->m_flags & EROFS_MAP_FRAGMENT) goto hitted; err = z_erofs_collector_begin(fe); @@ -735,6 +766,24 @@ hitted: zero_user_segment(page, cur, end); goto next_part; } + if (map->m_flags & EROFS_MAP_FRAGMENT) { + unsigned int pageofs, skip, len; + + if (offset > map->m_la) { + pageofs = 0; + skip = offset - map->m_la; + } else { + pageofs = map->m_la & ~PAGE_MASK; + skip = 0; + } + len = min_t(unsigned int, map->m_llen - skip, end - cur); + err = z_erofs_read_fragment(inode, skip, page, pageofs, len); + if (err) + goto out; + ++spiltted; + tight = false; + goto next_part; + } exclusive = (!cur && (!spiltted || tight)); if (cur) @@ -766,6 +815,7 @@ retry: fe->pcl->multibases = true; if ((map->m_flags & EROFS_MAP_FULL_MAPPED) && + !(map->m_flags & EROFS_MAP_PARTIAL_REF) && fe->pcl->length == map->m_llen) fe->pcl->partial = false; if (fe->pcl->length < offset + end - map->m_la) { @@ -1365,6 +1415,8 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, struct block_device *last_bdev; unsigned int nr_bios = 0; struct bio *bio = NULL; + /* initialize to 1 to make skip psi_memstall_leave unless needed */ + unsigned long pflags = 1; bi_private = jobqueueset_init(sb, q, fgq, force_fg); qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; @@ -1414,10 +1466,15 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, if (bio && (cur != last_index + 1 || last_bdev != mdev.m_bdev)) { submit_bio_retry: + if (!pflags) + psi_memstall_leave(&pflags); submit_bio(bio); bio = NULL; } + if (unlikely(PageWorkingset(page))) + psi_memstall_enter(&pflags); + if (!bio) { bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, REQ_OP_READ, GFP_NOIO); @@ -1445,8 +1502,11 @@ submit_bio_retry: move_to_bypass_jobqueue(pcl, qtail, owned_head); } while (owned_head != Z_EROFS_PCLUSTER_TAIL); - if (bio) + if (bio) { + if (!pflags) + psi_memstall_leave(&pflags); submit_bio(bio); + } /* * although background is preferred, no one is pending for submission. diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 572f0b8151ba84c57bb7b23536e96c01ce58a4af..44c27ef39c436d2362f530798d259de9c7ec499e 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -17,7 +17,7 @@ int z_erofs_fill_inode(struct inode *inode) struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); if (!erofs_sb_has_big_pcluster(sbi) && - !erofs_sb_has_ztailpacking(sbi) && + !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) && vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { vi->z_advise = 0; vi->z_algorithmtype[0] = 0; @@ -55,10 +55,6 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) goto out_unlock; - DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && - !erofs_sb_has_ztailpacking(EROFS_SB(sb)) && - vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); - pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + vi->xattr_isize, 8); kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), @@ -69,6 +65,16 @@ static int z_erofs_fill_inode_lazy(struct inode *inode) } h = kaddr + erofs_blkoff(pos); + /* + * if the highest bit of the 8-byte map header is set, the whole file + * is stored in the packed inode. The rest bits keeps z_fragmentoff. + */ + if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + goto unmap_done; + } vi->z_advise = le16_to_cpu(h->h_advise); vi->z_algorithmtype[0] = h->h_algorithmtype & 15; vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; @@ -123,6 +129,20 @@ unmap_done: if (err < 0) goto out_unlock; } + + if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && + !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { + struct erofs_map_blocks map = { + .buf = __EROFS_BUF_INITIALIZER + }; + + vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); + err = z_erofs_do_map_blocks(inode, &map, + EROFS_GET_BLOCKS_FINDTAIL); + erofs_put_metabuf(&map.buf); + if (err < 0) + goto out_unlock; + } /* paired with smp_mb() at the beginning of the function */ smp_mb(); set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); @@ -141,22 +161,11 @@ struct z_erofs_maprecorder { u8 type, headtype; u16 clusterofs; u16 delta[2]; - erofs_blk_t pblk, compressedlcs; + erofs_blk_t pblk, compressedblks; erofs_off_t nextpackoff; + bool partialref; }; -static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, - erofs_blk_t eblk) -{ - struct super_block *const sb = m->inode->i_sb; - - m->kaddr = erofs_read_metabuf(&m->map->buf, sb, eblk, - EROFS_KMAP_ATOMIC); - if (IS_ERR(m->kaddr)) - return PTR_ERR(m->kaddr); - return 0; -} - static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, unsigned long lcn) { @@ -169,11 +178,11 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, lcn * sizeof(struct z_erofs_vle_decompressed_index); struct z_erofs_vle_decompressed_index *di; unsigned int advise, type; - int err; - err = z_erofs_reload_indexes(m, erofs_blknr(pos)); - if (err) - return err; + m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, + erofs_blknr(pos), EROFS_KMAP_ATOMIC); + if (IS_ERR(m->kaddr)) + return PTR_ERR(m->kaddr); m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); m->lcn = lcn; @@ -192,7 +201,7 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, DBG_BUGON(1); return -EFSCORRUPTED; } - m->compressedlcs = m->delta[0] & + m->compressedblks = m->delta[0] & ~Z_EROFS_VLE_DI_D0_CBLKCNT; m->delta[0] = 1; } @@ -201,6 +210,8 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: + if (advise & Z_EROFS_VLE_DI_PARTIAL_REF) + m->partialref = true; m->clusterofs = le16_to_cpu(di->di_clusterofs); m->pblk = le32_to_cpu(di->di_u.blkaddr); break; @@ -293,7 +304,7 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, DBG_BUGON(1); return -EFSCORRUPTED; } - m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; + m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; m->delta[0] = 1; return 0; } else if (i + 1 != (int)vcnt) { @@ -370,7 +381,6 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, unsigned int compacted_4b_initial, compacted_2b; unsigned int amortizedshift; erofs_off_t pos; - int err; if (lclusterbits != 12) return -EOPNOTSUPP; @@ -407,9 +417,10 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, amortizedshift = 2; out: pos += lcn * (1 << amortizedshift); - err = z_erofs_reload_indexes(m, erofs_blknr(pos)); - if (err) - return err; + m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, + erofs_blknr(pos), EROFS_KMAP_ATOMIC); + if (IS_ERR(m->kaddr)) + return PTR_ERR(m->kaddr); return unpack_compacted_index(m, amortizedshift, pos, lookahead); } @@ -497,7 +508,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, return 0; } lcn = m->lcn + 1; - if (m->compressedlcs) + if (m->compressedblks) goto out; err = z_erofs_load_cluster_from_disk(m, lcn, false); @@ -506,7 +517,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, /* * If the 1st NONHEAD lcluster has already been handled initially w/o - * valid compressedlcs, which means at least it mustn't be CBLKCNT, or + * valid compressedblks, which means at least it mustn't be CBLKCNT, or * an internal implemenatation error is detected. * * The following code can also handle it properly anyway, but let's @@ -523,12 +534,12 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. */ - m->compressedlcs = 1; + m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE); break; case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: if (m->delta[0] != 1) goto err_bonus_cblkcnt; - if (m->compressedlcs) + if (m->compressedblks) break; fallthrough; default: @@ -539,7 +550,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, return -EFSCORRUPTED; } out: - map->m_plen = (u64)m->compressedlcs << lclusterbits; + map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE; return 0; err_bonus_cblkcnt: erofs_err(m->inode->i_sb, @@ -598,6 +609,7 @@ static int z_erofs_do_map_blocks(struct inode *inode, { struct erofs_inode *const vi = EROFS_I(inode); bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; + bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; struct z_erofs_maprecorder m = { .inode = inode, .map = map, @@ -663,15 +675,23 @@ static int z_erofs_do_map_blocks(struct inode *inode, err = -EOPNOTSUPP; goto unmap_out; } - + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_llen = end - map->m_la; - if (flags & EROFS_GET_BLOCKS_FINDTAIL) + if (flags & EROFS_GET_BLOCKS_FINDTAIL) { vi->z_tailextent_headlcn = m.lcn; + /* for non-compact indexes, fragmentoff is 64 bits */ + if (fragment && + vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) + vi->z_fragmentoff |= (u64)m.pblk << 32; + } if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { map->m_flags |= EROFS_MAP_META; map->m_pa = vi->z_idataoff; map->m_plen = vi->z_idata_size; + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_FRAGMENT; } else { map->m_pa = blknr_to_addr(m.pblk); err = z_erofs_get_extent_compressedlen(&m, initial_lcn); @@ -679,12 +699,18 @@ static int z_erofs_do_map_blocks(struct inode *inode, goto out; } - if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) - map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) + if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) { + if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { map->m_algorithmformat = vi->z_algorithmtype[1]; - else + } else { map->m_algorithmformat = vi->z_algorithmtype[0]; + } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || ((flags & EROFS_GET_BLOCKS_READMORE) && @@ -705,10 +731,10 @@ out: return err; } -int z_erofs_map_blocks_iter(struct inode *inode, - struct erofs_map_blocks *map, +int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, int flags) { + struct erofs_inode *const vi = EROFS_I(inode); int err = 0; trace_z_erofs_map_blocks_iter_enter(inode, map, flags); @@ -725,6 +751,15 @@ int z_erofs_map_blocks_iter(struct inode *inode, if (err) goto out; + if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && + !vi->z_tailextent_headlcn) { + map->m_la = 0; + map->m_llen = inode->i_size; + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | + EROFS_MAP_FRAGMENT; + goto out; + } + err = z_erofs_do_map_blocks(inode, map, flags); out: trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); @@ -751,7 +786,8 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, iomap->length = map.m_llen; if (map.m_flags & EROFS_MAP_MAPPED) { iomap->type = IOMAP_MAPPED; - iomap->addr = map.m_pa; + iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? + IOMAP_NULL_ADDR : map.m_pa; } else { iomap->type = IOMAP_HOLE; iomap->addr = IOMAP_NULL_ADDR; diff --git a/fs/eventfd.c b/fs/eventfd.c index 3627dd7d25db80c3f7e65afb8a5cda2e977f5e84..c0ffee99ad238aea86eb9da4bb4bb504274da9d1 100644 --- a/fs/eventfd.c +++ b/fs/eventfd.c @@ -69,17 +69,17 @@ __u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n) * it returns false, the eventfd_signal() call should be deferred to a * safe context. */ - if (WARN_ON_ONCE(current->in_eventfd_signal)) + if (WARN_ON_ONCE(current->in_eventfd)) return 0; spin_lock_irqsave(&ctx->wqh.lock, flags); - current->in_eventfd_signal = 1; + current->in_eventfd = 1; if (ULLONG_MAX - ctx->count < n) n = ULLONG_MAX - ctx->count; ctx->count += n; if (waitqueue_active(&ctx->wqh)) wake_up_locked_poll(&ctx->wqh, EPOLLIN); - current->in_eventfd_signal = 0; + current->in_eventfd = 0; spin_unlock_irqrestore(&ctx->wqh.lock, flags); return n; @@ -253,8 +253,10 @@ static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to) __set_current_state(TASK_RUNNING); } eventfd_ctx_do_read(ctx, &ucnt); + current->in_eventfd = 1; if (waitqueue_active(&ctx->wqh)) wake_up_locked_poll(&ctx->wqh, EPOLLOUT); + current->in_eventfd = 0; spin_unlock_irq(&ctx->wqh.lock); if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt))) return -EFAULT; @@ -301,8 +303,10 @@ static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t c } if (likely(res > 0)) { ctx->count += ucnt; + current->in_eventfd = 1; if (waitqueue_active(&ctx->wqh)) wake_up_locked_poll(&ctx->wqh, EPOLLIN); + current->in_eventfd = 0; } spin_unlock_irq(&ctx->wqh.lock); diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 8b56b94e2f56f80afff003741c6e78bf4f7c8639..52954d4637b546870eb500c14167be8164522742 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -1065,7 +1065,7 @@ static inline bool list_add_tail_lockless(struct list_head *new, * added to the list from another CPU: the winner observes * new->next == new. */ - if (cmpxchg(&new->next, new, head) != new) + if (!try_cmpxchg(&new->next, &new, head)) return false; /* diff --git a/fs/exec.c b/fs/exec.c index 9a5ca7b82bfc5ee62e1533c35fbf7e587b077572..349a5da91efe8e0801999b41c41b96e4ce8af4f5 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -65,7 +64,6 @@ #include #include #include -#include #include #include @@ -684,6 +682,8 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) unsigned long length = old_end - old_start; unsigned long new_start = old_start - shift; unsigned long new_end = old_end - shift; + VMA_ITERATOR(vmi, mm, new_start); + struct vm_area_struct *next; struct mmu_gather tlb; BUG_ON(new_start > new_end); @@ -692,7 +692,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) * ensure there are no vmas between where we want to go * and where we are */ - if (vma != find_vma(mm, new_start)) + if (vma != vma_next(&vmi)) return -EFAULT; /* @@ -711,12 +711,13 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) lru_add_drain(); tlb_gather_mmu(&tlb, mm); + next = vma_next(&vmi); if (new_end > old_start) { /* * when the old and new regions overlap clear from new_end. */ free_pgd_range(&tlb, new_end, old_end, new_end, - vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); + next ? next->vm_start : USER_PGTABLES_CEILING); } else { /* * otherwise, clean from old_start; this is done to not touch @@ -725,7 +726,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) * for the others its just a little faster. */ free_pgd_range(&tlb, old_start, old_end, new_end, - vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); + next ? next->vm_start : USER_PGTABLES_CEILING); } tlb_finish_mmu(&tlb); @@ -958,8 +959,7 @@ struct file *open_exec(const char *name) } EXPORT_SYMBOL(open_exec); -#if defined(CONFIG_HAVE_AOUT) || defined(CONFIG_BINFMT_FLAT) || \ - defined(CONFIG_BINFMT_ELF_FDPIC) +#if defined(CONFIG_BINFMT_FLAT) || defined(CONFIG_BINFMT_ELF_FDPIC) ssize_t read_code(struct file *file, unsigned long addr, loff_t pos, size_t len) { ssize_t res = vfs_read(file, (void __user *)addr, len, &pos); @@ -979,12 +979,10 @@ static int exec_mmap(struct mm_struct *mm) { struct task_struct *tsk; struct mm_struct *old_mm, *active_mm; - bool vfork; int ret; /* Notify parent that we're no longer interested in the old VM */ tsk = current; - vfork = !!tsk->vfork_done; old_mm = current->mm; exec_mm_release(tsk, old_mm); if (old_mm) @@ -1014,6 +1012,7 @@ static int exec_mmap(struct mm_struct *mm) active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; + lru_gen_add_mm(mm); /* * This prevents preemption while active_mm is being loaded and * it and mm are being updated, which could cause problems for @@ -1026,13 +1025,8 @@ static int exec_mmap(struct mm_struct *mm) activate_mm(active_mm, mm); if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); - tsk->mm->vmacache_seqnum = 0; - vmacache_flush(tsk); task_unlock(tsk); - - if (vfork) - timens_on_fork(tsk->nsproxy, tsk); - + lru_gen_use_mm(mm); if (old_mm) { mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); @@ -1595,7 +1589,7 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) { /* Handle suid and sgid on files */ struct user_namespace *mnt_userns; - struct inode *inode; + struct inode *inode = file_inode(file); unsigned int mode; kuid_t uid; kgid_t gid; @@ -1606,7 +1600,6 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) if (task_no_new_privs(current)) return; - inode = file->f_path.dentry->d_inode; mode = READ_ONCE(inode->i_mode); if (!(mode & (S_ISUID|S_ISGID))) return; @@ -1888,7 +1881,7 @@ static int do_execveat_common(int fd, struct filename *filename, * whether NPROC limit is still exceeded. */ if ((current->flags & PF_NPROC_EXCEEDED) && - is_ucounts_overlimit(current_ucounts(), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { + is_rlimit_overlimit(current_ucounts(), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { retval = -EAGAIN; goto out_ret; } diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c index a27b55ec060a08e2c83e3299d9d3496747047972..0fc08fdcba7337d0c4040bce47c4730445036d41 100644 --- a/fs/exfat/dir.c +++ b/fs/exfat/dir.c @@ -212,9 +212,9 @@ static void exfat_free_namebuf(struct exfat_dentry_namebuf *nb) /* skip iterating emit_dots when dir is empty */ #define ITER_POS_FILLED_DOTS (2) -static int exfat_iterate(struct file *filp, struct dir_context *ctx) +static int exfat_iterate(struct file *file, struct dir_context *ctx) { - struct inode *inode = filp->f_path.dentry->d_inode; + struct inode *inode = file_inode(file); struct super_block *sb = inode->i_sb; struct inode *tmp; struct exfat_dir_entry de; @@ -228,7 +228,7 @@ static int exfat_iterate(struct file *filp, struct dir_context *ctx) mutex_lock(&EXFAT_SB(sb)->s_lock); cpos = ctx->pos; - if (!dir_emit_dots(filp, ctx)) + if (!dir_emit_dots(file, ctx)) goto unlock; if (ctx->pos == ITER_POS_FILLED_DOTS) { diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c index ee0b7cf51157059c6fee3c6fc6edd67560862c88..41ae4cce1f4203c1d8ecd6e4bd92e23fc42f7366 100644 --- a/fs/exfat/fatent.c +++ b/fs/exfat/fatent.c @@ -270,8 +270,7 @@ int exfat_zeroed_cluster(struct inode *dir, unsigned int clu) struct super_block *sb = dir->i_sb; struct exfat_sb_info *sbi = EXFAT_SB(sb); struct buffer_head *bh; - sector_t blknr, last_blknr; - int i; + sector_t blknr, last_blknr, i; blknr = exfat_cluster_to_sector(sbi, clu); last_blknr = blknr + sbi->sect_per_clus; diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c index a795437b86d0670adb4d58e854296e474d313647..5590a1e83126c99a2957af22af99a9d28b9747fb 100644 --- a/fs/exfat/inode.c +++ b/fs/exfat/inode.c @@ -552,7 +552,7 @@ static int exfat_fill_inode(struct inode *inode, struct exfat_dir_entry *info) inode->i_uid = sbi->options.fs_uid; inode->i_gid = sbi->options.fs_gid; inode_inc_iversion(inode); - inode->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); if (info->attr & ATTR_SUBDIR) { /* directory */ inode->i_generation &= ~1; diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c index 3ef80d000e13d5a46e886492583322e8344f182e..c648a493faf23865780924ece6dceda506d75152 100644 --- a/fs/exportfs/expfs.c +++ b/fs/exportfs/expfs.c @@ -248,21 +248,20 @@ struct getdents_callback { * A rather strange filldir function to capture * the name matching the specified inode number. */ -static int filldir_one(struct dir_context *ctx, const char *name, int len, +static bool filldir_one(struct dir_context *ctx, const char *name, int len, loff_t pos, u64 ino, unsigned int d_type) { struct getdents_callback *buf = container_of(ctx, struct getdents_callback, ctx); - int result = 0; buf->sequence++; if (buf->ino == ino && len <= NAME_MAX) { memcpy(buf->name, name, len); buf->name[len] = '\0'; buf->found = 1; - result = -1; + return false; // no more } - return result; + return true; } /** diff --git a/fs/ext2/balloc.c b/fs/ext2/balloc.c index c17ccc19b938e22d996476bc8567c5df22716daa..5dc0a31f4a087fb6f3c4dbc469af056b2a1c061d 100644 --- a/fs/ext2/balloc.c +++ b/fs/ext2/balloc.c @@ -126,6 +126,7 @@ read_block_bitmap(struct super_block *sb, unsigned int block_group) struct ext2_group_desc * desc; struct buffer_head * bh = NULL; ext2_fsblk_t bitmap_blk; + int ret; desc = ext2_get_group_desc(sb, block_group, NULL); if (!desc) @@ -139,10 +140,10 @@ read_block_bitmap(struct super_block *sb, unsigned int block_group) block_group, le32_to_cpu(desc->bg_block_bitmap)); return NULL; } - if (likely(bh_uptodate_or_lock(bh))) + ret = bh_read(bh, 0); + if (ret > 0) return bh; - - if (bh_submit_read(bh) < 0) { + if (ret < 0) { brelse(bh); ext2_error(sb, __func__, "Cannot read block bitmap - " diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c index 998dd2ac8008966f10995824fb8e91cb5d47c995..f4944c4dee607f2597f7c258868ebad604f509c6 100644 --- a/fs/ext2/ialloc.c +++ b/fs/ext2/ialloc.c @@ -277,8 +277,7 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent) int best_ndir = inodes_per_group; int best_group = -1; - group = prandom_u32(); - parent_group = (unsigned)group % ngroups; + parent_group = prandom_u32_max(ngroups); for (i = 0; i < ngroups; i++) { group = (parent_group + i) % ngroups; desc = ext2_get_group_desc (sb, group, NULL); diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c index 5fd9a22d2b70c738daa5b42f91d6ebd5b335c394..9125eab85146a51d4d323d8f7d5ab5689e42484c 100644 --- a/fs/ext2/namei.c +++ b/fs/ext2/namei.c @@ -120,7 +120,7 @@ static int ext2_create (struct user_namespace * mnt_userns, } static int ext2_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { struct inode *inode = ext2_new_inode(dir, mode, NULL); if (IS_ERR(inode)) @@ -128,9 +128,9 @@ static int ext2_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, ext2_set_file_ops(inode); mark_inode_dirty(inode); - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); unlock_new_inode(inode); - return 0; + return finish_open_simple(file, 0); } static int ext2_mknod (struct user_namespace * mnt_userns, struct inode * dir, diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 252c742379cfbcad78f898a022dc96fc890d93c6..03f2af98b1b48b40a3efcd16506912d05ae54a07 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -163,7 +163,7 @@ static void ext2_put_super (struct super_block * sb) db_count = sbi->s_gdb_count; for (i = 0; i < db_count; i++) brelse(sbi->s_group_desc[i]); - kfree(sbi->s_group_desc); + kvfree(sbi->s_group_desc); kfree(sbi->s_debts); percpu_counter_destroy(&sbi->s_freeblocks_counter); percpu_counter_destroy(&sbi->s_freeinodes_counter); @@ -1052,6 +1052,13 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) sbi->s_blocks_per_group); goto failed_mount; } + /* At least inode table, bitmaps, and sb have to fit in one group */ + if (sbi->s_blocks_per_group <= sbi->s_itb_per_group + 3) { + ext2_msg(sb, KERN_ERR, + "error: #blocks per group smaller than metadata size: %lu <= %lu", + sbi->s_blocks_per_group, sbi->s_inodes_per_group + 3); + goto failed_mount; + } if (sbi->s_frags_per_group > sb->s_blocksize * 8) { ext2_msg(sb, KERN_ERR, "error: #fragments per group too big: %lu", @@ -1065,9 +1072,14 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) sbi->s_inodes_per_group); goto failed_mount; } + if (sb_bdev_nr_blocks(sb) < le32_to_cpu(es->s_blocks_count)) { + ext2_msg(sb, KERN_ERR, + "bad geometry: block count %u exceeds size of device (%u blocks)", + le32_to_cpu(es->s_blocks_count), + (unsigned)sb_bdev_nr_blocks(sb)); + goto failed_mount; + } - if (EXT2_BLOCKS_PER_GROUP(sb) == 0) - goto cantfind_ext2; sbi->s_groups_count = ((le32_to_cpu(es->s_blocks_count) - le32_to_cpu(es->s_first_data_block) - 1) / EXT2_BLOCKS_PER_GROUP(sb)) + 1; @@ -1080,7 +1092,7 @@ static int ext2_fill_super(struct super_block *sb, void *data, int silent) } db_count = (sbi->s_groups_count + EXT2_DESC_PER_BLOCK(sb) - 1) / EXT2_DESC_PER_BLOCK(sb); - sbi->s_group_desc = kmalloc_array(db_count, + sbi->s_group_desc = kvmalloc_array(db_count, sizeof(struct buffer_head *), GFP_KERNEL); if (sbi->s_group_desc == NULL) { @@ -1206,7 +1218,7 @@ failed_mount2: for (i = 0; i < db_count; i++) brelse(sbi->s_group_desc[i]); failed_mount_group_desc: - kfree(sbi->s_group_desc); + kvfree(sbi->s_group_desc); kfree(sbi->s_debts); failed_mount: brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 9bca5565547baeb0bc5e77e94082b2b5d81b4a75..8d5453852f98ec06c9ed3a49b2c76f7605f8c3f4 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -167,8 +167,6 @@ enum SHIFT_DIRECTION { #define EXT4_MB_CR0_OPTIMIZED 0x8000 /* Avg fragment size rb tree lookup succeeded at least once for cr = 1 */ #define EXT4_MB_CR1_OPTIMIZED 0x00010000 -/* Perform linear traversal for one group */ -#define EXT4_MB_SEARCH_NEXT_LINEAR 0x00020000 struct ext4_allocation_request { /* target inode for block we're allocating */ struct inode *inode; @@ -1600,8 +1598,8 @@ struct ext4_sb_info { struct list_head s_discard_list; struct work_struct s_discard_work; atomic_t s_retry_alloc_pending; - struct rb_root s_mb_avg_fragment_size_root; - rwlock_t s_mb_rb_lock; + struct list_head *s_mb_avg_fragment_size; + rwlock_t *s_mb_avg_fragment_size_locks; struct list_head *s_mb_largest_free_orders; rwlock_t *s_mb_largest_free_orders_locks; @@ -2979,6 +2977,7 @@ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, extern int ext4_write_inode(struct inode *, struct writeback_control *); extern int ext4_setattr(struct user_namespace *, struct dentry *, struct iattr *); +extern u32 ext4_dio_alignment(struct inode *inode); extern int ext4_getattr(struct user_namespace *, const struct path *, struct kstat *, u32, unsigned int); extern void ext4_evict_inode(struct inode *); @@ -3413,6 +3412,8 @@ struct ext4_group_info { ext4_grpblk_t bb_first_free; /* first free block */ ext4_grpblk_t bb_free; /* total free blocks */ ext4_grpblk_t bb_fragments; /* nr of freespace fragments */ + int bb_avg_fragment_size_order; /* order of average + fragment in BG */ ext4_grpblk_t bb_largest_free_order;/* order of largest frag in BG */ ext4_group_t bb_group; /* Group number */ struct list_head bb_prealloc_list; @@ -3420,7 +3421,7 @@ struct ext4_group_info { void *bb_bitmap; #endif struct rw_semaphore alloc_sem; - struct rb_node bb_avg_fragment_size_rb; + struct list_head bb_avg_fragment_size_node; struct list_head bb_largest_free_order_node; ext4_grpblk_t bb_counters[]; /* Nr of free power-of-two-block * regions, index is order. @@ -3591,9 +3592,6 @@ extern bool empty_inline_dir(struct inode *dir, int *has_inline_data); extern struct buffer_head *ext4_get_first_inline_block(struct inode *inode, struct ext4_dir_entry_2 **parent_de, int *retval); -extern int ext4_inline_data_fiemap(struct inode *inode, - struct fiemap_extent_info *fieinfo, - int *has_inline, __u64 start, __u64 len); extern void *ext4_read_inline_link(struct inode *inode); struct iomap; @@ -3712,7 +3710,7 @@ extern int ext4_ext_insert_extent(handle_t *, struct inode *, extern struct ext4_ext_path *ext4_find_extent(struct inode *, ext4_lblk_t, struct ext4_ext_path **, int flags); -extern void ext4_ext_drop_refs(struct ext4_ext_path *); +extern void ext4_free_ext_path(struct ext4_ext_path *); extern int ext4_ext_check_inode(struct inode *inode); extern ext4_lblk_t ext4_ext_next_allocated_block(struct ext4_ext_path *path); extern int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c index c148bb97b527359bf854428f659d7da87584009a..f1956288307f39f497cc088b77bdc4363b3f0fed 100644 --- a/fs/ext4/extents.c +++ b/fs/ext4/extents.c @@ -106,6 +106,25 @@ static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped) return 0; } +static void ext4_ext_drop_refs(struct ext4_ext_path *path) +{ + int depth, i; + + if (!path) + return; + depth = path->p_depth; + for (i = 0; i <= depth; i++, path++) { + brelse(path->p_bh); + path->p_bh = NULL; + } +} + +void ext4_free_ext_path(struct ext4_ext_path *path) +{ + ext4_ext_drop_refs(path); + kfree(path); +} + /* * Make sure 'handle' has at least 'check_cred' credits. If not, restart * transaction with 'restart_cred' credits. The function drops i_data_sem @@ -460,6 +479,10 @@ static int __ext4_ext_check(const char *function, unsigned int line, error_msg = "invalid eh_entries"; goto corrupted; } + if (unlikely((eh->eh_entries == 0) && (depth > 0))) { + error_msg = "eh_entries is 0 but eh_depth is > 0"; + goto corrupted; + } if (!ext4_valid_extent_entries(inode, eh, lblk, &pblk, depth)) { error_msg = "invalid extent entries"; goto corrupted; @@ -632,8 +655,7 @@ int ext4_ext_precache(struct inode *inode) ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED); out: up_read(&ei->i_data_sem); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return ret; } @@ -720,19 +742,6 @@ static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path, #define ext4_ext_show_move(inode, path, newblock, level) #endif -void ext4_ext_drop_refs(struct ext4_ext_path *path) -{ - int depth, i; - - if (!path) - return; - depth = path->p_depth; - for (i = 0; i <= depth; i++, path++) { - brelse(path->p_bh); - path->p_bh = NULL; - } -} - /* * ext4_ext_binsearch_idx: * binary search for the closest index of the given block @@ -951,8 +960,7 @@ ext4_find_extent(struct inode *inode, ext4_lblk_t block, return path; err: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); if (orig_path) *orig_path = NULL; return ERR_PTR(ret); @@ -2170,8 +2178,7 @@ merge: err = ext4_ext_dirty(handle, inode, path + path->p_depth); cleanup: - ext4_ext_drop_refs(npath); - kfree(npath); + ext4_free_ext_path(npath); return err; } @@ -3057,8 +3064,7 @@ again: } } out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); path = NULL; if (err == -EAGAIN) goto again; @@ -4371,8 +4377,7 @@ got_allocated_blocks: allocated = map->m_len; ext4_ext_show_leaf(inode, path); out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); trace_ext4_ext_map_blocks_exit(inode, flags, map, err ? err : allocated); @@ -5241,8 +5246,7 @@ again: break; } out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return ret; } @@ -5534,15 +5538,13 @@ static int ext4_insert_range(struct file *file, loff_t offset, loff_t len) EXT4_GET_BLOCKS_METADATA_NOFAIL); } - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); if (ret < 0) { up_write(&EXT4_I(inode)->i_data_sem); goto out_stop; } } else { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); } ret = ext4_es_remove_extent(inode, offset_lblk, @@ -5762,10 +5764,8 @@ ext4_swap_extents(handle_t *handle, struct inode *inode1, count -= len; repeat: - ext4_ext_drop_refs(path1); - kfree(path1); - ext4_ext_drop_refs(path2); - kfree(path2); + ext4_free_ext_path(path1); + ext4_free_ext_path(path2); path1 = path2 = NULL; } return replaced_count; @@ -5844,8 +5844,7 @@ int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu) } out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return err ? err : mapped; } @@ -5912,8 +5911,7 @@ int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start, ret = ext4_ext_dirty(NULL, inode, &path[path->p_depth]); up_write(&EXT4_I(inode)->i_data_sem); out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); ext4_mark_inode_dirty(NULL, inode); return ret; } @@ -5931,8 +5929,7 @@ void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end) return; ex = path[path->p_depth].p_ext; if (!ex) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); ext4_mark_inode_dirty(NULL, inode); return; } @@ -5945,8 +5942,7 @@ void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end) ext4_ext_dirty(NULL, inode, &path[path->p_depth]); up_write(&EXT4_I(inode)->i_data_sem); ext4_mark_inode_dirty(NULL, inode); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); } } @@ -5985,13 +5981,11 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) return PTR_ERR(path); ex = path[path->p_depth].p_ext; if (!ex) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); goto out; } end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); /* Count the number of data blocks */ cur = 0; @@ -6021,30 +6015,26 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) if (IS_ERR(path)) goto out; numblks += path->p_depth; - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); while (cur < end) { path = ext4_find_extent(inode, cur, NULL, 0); if (IS_ERR(path)) break; ex = path[path->p_depth].p_ext; if (!ex) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return 0; } cur = max(cur + 1, le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex)); ret = skip_hole(inode, &cur); if (ret < 0) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); break; } path2 = ext4_find_extent(inode, cur, NULL, 0); if (IS_ERR(path2)) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); break; } for (i = 0; i <= max(path->p_depth, path2->p_depth); i++) { @@ -6058,10 +6048,8 @@ int ext4_ext_replay_set_iblocks(struct inode *inode) if (cmp1 != cmp2 && cmp2 != 0) numblks++; } - ext4_ext_drop_refs(path); - ext4_ext_drop_refs(path2); - kfree(path); - kfree(path2); + ext4_free_ext_path(path); + ext4_free_ext_path(path2); } out: @@ -6088,13 +6076,11 @@ int ext4_ext_clear_bb(struct inode *inode) return PTR_ERR(path); ex = path[path->p_depth].p_ext; if (!ex) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return 0; } end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); cur = 0; while (cur < end) { @@ -6113,8 +6099,7 @@ int ext4_ext_clear_bb(struct inode *inode) ext4_fc_record_regions(inode->i_sb, inode->i_ino, 0, path[j].p_block, 1, 1); } - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); } ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0); ext4_fc_record_regions(inode->i_sb, inode->i_ino, diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c index 23167efda95ee14659f70ec91a429ebd0a32a984..cd0a861853e3f66268b56420616a4031fefbada9 100644 --- a/fs/ext4/extents_status.c +++ b/fs/ext4/extents_status.c @@ -667,8 +667,7 @@ static void ext4_es_insert_extent_ext_check(struct inode *inode, } } out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); } static void ext4_es_insert_extent_ind_check(struct inode *inode, diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 2af962cbb835ff8919d5217e82270b1daf351b42..ef05bfa87798c12f2ebba8ff56ee58a5ab5343bf 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -229,6 +229,12 @@ __releases(&EXT4_SB(inode->i_sb)->s_fc_lock) finish_wait(wq, &wait.wq_entry); } +static bool ext4_fc_disabled(struct super_block *sb) +{ + return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || + (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); +} + /* * Inform Ext4's fast about start of an inode update * @@ -240,8 +246,7 @@ void ext4_fc_start_update(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; restart: @@ -265,8 +270,7 @@ void ext4_fc_stop_update(struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; if (atomic_dec_and_test(&ei->i_fc_updates)) @@ -283,8 +287,7 @@ void ext4_fc_del(struct inode *inode) struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct ext4_fc_dentry_update *fc_dentry; - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; restart: @@ -337,8 +340,7 @@ void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handl struct ext4_sb_info *sbi = EXT4_SB(sb); tid_t tid; - if (!test_opt2(sb, JOURNAL_FAST_COMMIT) || - (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(sb)) return; ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); @@ -493,10 +495,8 @@ void __ext4_fc_track_unlink(handle_t *handle, void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry) { struct inode *inode = d_inode(dentry); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (sbi->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -522,10 +522,8 @@ void __ext4_fc_track_link(handle_t *handle, void ext4_fc_track_link(handle_t *handle, struct dentry *dentry) { struct inode *inode = d_inode(dentry); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (sbi->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -551,10 +549,8 @@ void __ext4_fc_track_create(handle_t *handle, struct inode *inode, void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) { struct inode *inode = d_inode(dentry); - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (sbi->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -576,22 +572,20 @@ static int __track_inode(struct inode *inode, void *arg, bool update) void ext4_fc_track_inode(handle_t *handle, struct inode *inode) { - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); int ret; if (S_ISDIR(inode->i_mode)) return; + if (ext4_fc_disabled(inode->i_sb)) + return; + if (ext4_should_journal_data(inode)) { ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); return; } - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (sbi->s_mount_state & EXT4_FC_REPLAY)) - return; - if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) return; @@ -634,15 +628,13 @@ static int __track_range(struct inode *inode, void *arg, bool update) void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, ext4_lblk_t end) { - struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); struct __track_range_args args; int ret; if (S_ISDIR(inode->i_mode)) return; - if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || - (sbi->s_mount_state & EXT4_FC_REPLAY)) + if (ext4_fc_disabled(inode->i_sb)) return; if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) @@ -710,10 +702,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) * After allocating len, we should have space at least for a 0 byte * padding. */ - if (len + sizeof(struct ext4_fc_tl) > bsize) + if (len + EXT4_FC_TAG_BASE_LEN > bsize) return NULL; - if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) { + if (bsize - off - 1 > len + EXT4_FC_TAG_BASE_LEN) { /* * Only allocate from current buffer if we have enough space for * this request AND we have space to add a zero byte padding. @@ -730,10 +722,10 @@ static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) /* Need to add PAD tag */ tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off); tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); - pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl); + pad_len = bsize - off - 1 - EXT4_FC_TAG_BASE_LEN; tl->fc_len = cpu_to_le16(pad_len); if (crc) - *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl)); + *crc = ext4_chksum(sbi, *crc, tl, EXT4_FC_TAG_BASE_LEN); if (pad_len > 0) ext4_fc_memzero(sb, tl + 1, pad_len, crc); ext4_fc_submit_bh(sb, false); @@ -775,7 +767,7 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc) * ext4_fc_reserve_space takes care of allocating an extra block if * there's no enough space on this block for accommodating this tail. */ - dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc); + dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc); if (!dst) return -ENOSPC; @@ -785,8 +777,8 @@ static int ext4_fc_write_tail(struct super_block *sb, u32 crc) tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail)); sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); - ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc); - dst += sizeof(tl); + ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, &crc); + dst += EXT4_FC_TAG_BASE_LEN; tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc); dst += sizeof(tail.fc_tid); @@ -808,15 +800,15 @@ static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, struct ext4_fc_tl tl; u8 *dst; - dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc); + dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc); if (!dst) return false; tl.fc_tag = cpu_to_le16(tag); tl.fc_len = cpu_to_le16(len); - ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); - ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc); + ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); + ext4_fc_memcpy(sb, dst + EXT4_FC_TAG_BASE_LEN, val, len, crc); return true; } @@ -828,8 +820,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, struct ext4_fc_dentry_info fcd; struct ext4_fc_tl tl; int dlen = fc_dentry->fcd_name.len; - u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen, - crc); + u8 *dst = ext4_fc_reserve_space(sb, + EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc); if (!dst) return false; @@ -838,8 +830,8 @@ static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); - ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc); - dst += sizeof(tl); + ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc); + dst += EXT4_FC_TAG_BASE_LEN; ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc); dst += sizeof(fcd); ext4_fc_memcpy(sb, dst, fc_dentry->fcd_name.name, dlen, crc); @@ -874,22 +866,25 @@ static int ext4_fc_write_inode(struct inode *inode, u32 *crc) tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); + ret = -ECANCELED; dst = ext4_fc_reserve_space(inode->i_sb, - sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc); + EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc); if (!dst) - return -ECANCELED; + goto err; - if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc)) - return -ECANCELED; - dst += sizeof(tl); + if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc)) + goto err; + dst += EXT4_FC_TAG_BASE_LEN; if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc)) - return -ECANCELED; + goto err; dst += sizeof(fc_inode); if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc), inode_len, crc)) - return -ECANCELED; - - return 0; + goto err; + ret = 0; +err: + brelse(iloc.bh); + return ret; } /* @@ -1343,7 +1338,7 @@ struct dentry_info_args { }; static inline void tl_to_darg(struct dentry_info_args *darg, - struct ext4_fc_tl *tl, u8 *val) + struct ext4_fc_tl *tl, u8 *val) { struct ext4_fc_dentry_info fcd; @@ -1352,8 +1347,14 @@ static inline void tl_to_darg(struct dentry_info_args *darg, darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); darg->ino = le32_to_cpu(fcd.fc_ino); darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); - darg->dname_len = le16_to_cpu(tl->fc_len) - - sizeof(struct ext4_fc_dentry_info); + darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); +} + +static inline void ext4_fc_get_tl(struct ext4_fc_tl *tl, u8 *val) +{ + memcpy(tl, val, EXT4_FC_TAG_BASE_LEN); + tl->fc_len = le16_to_cpu(tl->fc_len); + tl->fc_tag = le16_to_cpu(tl->fc_tag); } /* Unlink replay function */ @@ -1491,13 +1492,15 @@ static int ext4_fc_record_modified_inode(struct super_block *sb, int ino) if (state->fc_modified_inodes[i] == ino) return 0; if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { - state->fc_modified_inodes = krealloc( - state->fc_modified_inodes, + int *fc_modified_inodes; + + fc_modified_inodes = krealloc(state->fc_modified_inodes, sizeof(int) * (state->fc_modified_inodes_size + EXT4_FC_REPLAY_REALLOC_INCREMENT), GFP_KERNEL); - if (!state->fc_modified_inodes) + if (!fc_modified_inodes) return -ENOMEM; + state->fc_modified_inodes = fc_modified_inodes; state->fc_modified_inodes_size += EXT4_FC_REPLAY_REALLOC_INCREMENT; } @@ -1516,7 +1519,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, struct ext4_inode *raw_fc_inode; struct inode *inode = NULL; struct ext4_iloc iloc; - int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag); + int inode_len, ino, ret, tag = tl->fc_tag; struct ext4_extent_header *eh; memcpy(&fc_inode, val, sizeof(fc_inode)); @@ -1541,7 +1544,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, if (ret) goto out; - inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode); + inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); raw_inode = ext4_raw_inode(&iloc); memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); @@ -1682,15 +1685,18 @@ int ext4_fc_record_regions(struct super_block *sb, int ino, if (replay && state->fc_regions_used != state->fc_regions_valid) state->fc_regions_used = state->fc_regions_valid; if (state->fc_regions_used == state->fc_regions_size) { + struct ext4_fc_alloc_region *fc_regions; + + fc_regions = krealloc(state->fc_regions, + sizeof(struct ext4_fc_alloc_region) * + (state->fc_regions_size + + EXT4_FC_REPLAY_REALLOC_INCREMENT), + GFP_KERNEL); + if (!fc_regions) + return -ENOMEM; state->fc_regions_size += EXT4_FC_REPLAY_REALLOC_INCREMENT; - state->fc_regions = krealloc( - state->fc_regions, - state->fc_regions_size * - sizeof(struct ext4_fc_alloc_region), - GFP_KERNEL); - if (!state->fc_regions) - return -ENOMEM; + state->fc_regions = fc_regions; } region = &state->fc_regions[state->fc_regions_used++]; region->ino = ino; @@ -1770,8 +1776,7 @@ static int ext4_fc_replay_add_range(struct super_block *sb, ret = ext4_ext_insert_extent( NULL, inode, &path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); if (ret) goto out; goto next; @@ -1926,8 +1931,7 @@ static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) for (j = 0; j < path->p_depth; j++) ext4_mb_mark_bb(inode->i_sb, path[j].p_block, 1, 1); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); } cur += ret; ext4_mb_mark_bb(inode->i_sb, map.m_pblk, @@ -1972,6 +1976,34 @@ void ext4_fc_replay_cleanup(struct super_block *sb) kfree(sbi->s_fc_replay_state.fc_modified_inodes); } +static inline bool ext4_fc_tag_len_isvalid(struct ext4_fc_tl *tl, + u8 *val, u8 *end) +{ + if (val + tl->fc_len > end) + return false; + + /* Here only check ADD_RANGE/TAIL/HEAD which will read data when do + * journal rescan before do CRC check. Other tags length check will + * rely on CRC check. + */ + switch (tl->fc_tag) { + case EXT4_FC_TAG_ADD_RANGE: + return (sizeof(struct ext4_fc_add_range) == tl->fc_len); + case EXT4_FC_TAG_TAIL: + return (sizeof(struct ext4_fc_tail) <= tl->fc_len); + case EXT4_FC_TAG_HEAD: + return (sizeof(struct ext4_fc_head) == tl->fc_len); + case EXT4_FC_TAG_DEL_RANGE: + case EXT4_FC_TAG_LINK: + case EXT4_FC_TAG_UNLINK: + case EXT4_FC_TAG_CREAT: + case EXT4_FC_TAG_INODE: + case EXT4_FC_TAG_PAD: + default: + return true; + } +} + /* * Recovery Scan phase handler * @@ -2028,12 +2060,18 @@ static int ext4_fc_replay_scan(journal_t *journal, } state->fc_replay_expected_off++; - for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { - memcpy(&tl, cur, sizeof(tl)); - val = cur + sizeof(tl); + for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; + cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { + ext4_fc_get_tl(&tl, cur); + val = cur + EXT4_FC_TAG_BASE_LEN; + if (!ext4_fc_tag_len_isvalid(&tl, val, end)) { + ret = state->fc_replay_num_tags ? + JBD2_FC_REPLAY_STOP : -ECANCELED; + goto out_err; + } ext4_debug("Scan phase, tag:%s, blk %lld\n", - tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr); - switch (le16_to_cpu(tl.fc_tag)) { + tag2str(tl.fc_tag), bh->b_blocknr); + switch (tl.fc_tag) { case EXT4_FC_TAG_ADD_RANGE: memcpy(&ext, val, sizeof(ext)); ex = (struct ext4_extent *)&ext.fc_ex; @@ -2053,13 +2091,13 @@ static int ext4_fc_replay_scan(journal_t *journal, case EXT4_FC_TAG_PAD: state->fc_cur_tag++; state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, - sizeof(tl) + le16_to_cpu(tl.fc_len)); + EXT4_FC_TAG_BASE_LEN + tl.fc_len); break; case EXT4_FC_TAG_TAIL: state->fc_cur_tag++; memcpy(&tail, val, sizeof(tail)); state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, - sizeof(tl) + + EXT4_FC_TAG_BASE_LEN + offsetof(struct ext4_fc_tail, fc_crc)); if (le32_to_cpu(tail.fc_tid) == expected_tid && @@ -2086,7 +2124,7 @@ static int ext4_fc_replay_scan(journal_t *journal, } state->fc_cur_tag++; state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, - sizeof(tl) + le16_to_cpu(tl.fc_len)); + EXT4_FC_TAG_BASE_LEN + tl.fc_len); break; default: ret = state->fc_replay_num_tags ? @@ -2141,19 +2179,20 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, start = (u8 *)bh->b_data; end = (__u8 *)bh->b_data + journal->j_blocksize - 1; - for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) { - memcpy(&tl, cur, sizeof(tl)); - val = cur + sizeof(tl); + for (cur = start; cur < end - EXT4_FC_TAG_BASE_LEN; + cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { + ext4_fc_get_tl(&tl, cur); + val = cur + EXT4_FC_TAG_BASE_LEN; if (state->fc_replay_num_tags == 0) { ret = JBD2_FC_REPLAY_STOP; ext4_fc_set_bitmaps_and_counters(sb); break; } - ext4_debug("Replay phase, tag:%s\n", - tag2str(le16_to_cpu(tl.fc_tag))); + + ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag)); state->fc_replay_num_tags--; - switch (le16_to_cpu(tl.fc_tag)) { + switch (tl.fc_tag) { case EXT4_FC_TAG_LINK: ret = ext4_fc_replay_link(sb, &tl, val); break; @@ -2174,19 +2213,18 @@ static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, break; case EXT4_FC_TAG_PAD: trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, - le16_to_cpu(tl.fc_len), 0); + tl.fc_len, 0); break; case EXT4_FC_TAG_TAIL: - trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0, - le16_to_cpu(tl.fc_len), 0); + trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, + 0, tl.fc_len, 0); memcpy(&tail, val, sizeof(tail)); WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid); break; case EXT4_FC_TAG_HEAD: break; default: - trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0, - le16_to_cpu(tl.fc_len), 0); + trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0); ret = -ECANCELED; break; } diff --git a/fs/ext4/fast_commit.h b/fs/ext4/fast_commit.h index 1db12847a83b643a230cea68d2dd370dd33358a1..a6154c3ed1357cbe4fcf12c7ea0e511723ce4505 100644 --- a/fs/ext4/fast_commit.h +++ b/fs/ext4/fast_commit.h @@ -70,6 +70,9 @@ struct ext4_fc_tail { __le32 fc_crc; }; +/* Tag base length */ +#define EXT4_FC_TAG_BASE_LEN (sizeof(struct ext4_fc_tl)) + /* * Fast commit status codes */ diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 109d07629f81fb9e1d255f95a4965b7b0caf266d..a7a597c727e638dff296d7d7b5b663c63a9d3051 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -36,19 +36,34 @@ #include "acl.h" #include "truncate.h" -static bool ext4_dio_supported(struct kiocb *iocb, struct iov_iter *iter) +/* + * Returns %true if the given DIO request should be attempted with DIO, or + * %false if it should fall back to buffered I/O. + * + * DIO isn't well specified; when it's unsupported (either due to the request + * being misaligned, or due to the file not supporting DIO at all), filesystems + * either fall back to buffered I/O or return EINVAL. For files that don't use + * any special features like encryption or verity, ext4 has traditionally + * returned EINVAL for misaligned DIO. iomap_dio_rw() uses this convention too. + * In this case, we should attempt the DIO, *not* fall back to buffered I/O. + * + * In contrast, in cases where DIO is unsupported due to ext4 features, ext4 + * traditionally falls back to buffered I/O. + * + * This function implements the traditional ext4 behavior in all these cases. + */ +static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) { struct inode *inode = file_inode(iocb->ki_filp); + u32 dio_align = ext4_dio_alignment(inode); - if (!fscrypt_dio_supported(iocb, iter)) - return false; - if (fsverity_active(inode)) - return false; - if (ext4_should_journal_data(inode)) + if (dio_align == 0) return false; - if (ext4_has_inline_data(inode)) - return false; - return true; + + if (dio_align == 1) + return true; + + return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align); } static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) @@ -63,7 +78,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) inode_lock_shared(inode); } - if (!ext4_dio_supported(iocb, to)) { + if (!ext4_should_use_dio(iocb, to)) { inode_unlock_shared(inode); /* * Fallback to buffered I/O if the operation being performed on @@ -511,7 +526,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) } /* Fallback to buffered I/O if the inode does not support direct I/O. */ - if (!ext4_dio_supported(iocb, from)) { + if (!ext4_should_use_dio(iocb, from)) { if (ilock_shared) inode_unlock_shared(inode); else @@ -528,6 +543,12 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) ret = -EAGAIN; goto out; } + /* + * Make sure inline data cannot be created anymore since we are going + * to allocate blocks for DIO. We know the inode does not have any + * inline data now because ext4_dio_supported() checked for that. + */ + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); offset = iocb->ki_pos; count = ret; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index f73e5eb43eae1dace29cd0082500be7ebbe71e06..e9bc46684106b5cb2c3922711f6008584535f882 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -463,10 +463,9 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent, hinfo.hash_version = DX_HASH_HALF_MD4; hinfo.seed = sbi->s_hash_seed; ext4fs_dirhash(parent, qstr->name, qstr->len, &hinfo); - grp = hinfo.hash; + parent_group = hinfo.hash % ngroups; } else - grp = prandom_u32(); - parent_group = (unsigned)grp % ngroups; + parent_group = prandom_u32_max(ngroups); for (i = 0; i < ngroups; i++) { g = (parent_group + i) % ngroups; get_orlov_stats(sb, g, flex_size, &stats); @@ -510,7 +509,7 @@ static int find_group_orlov(struct super_block *sb, struct inode *parent, goto fallback; } - max_dirs = ndirs / ngroups + inodes_per_group / 16; + max_dirs = ndirs / ngroups + inodes_per_group*flex_size / 16; min_inodes = avefreei - inodes_per_group*flex_size / 4; if (min_inodes < 1) min_inodes = 1; @@ -1280,7 +1279,7 @@ got: EXT4_GROUP_INFO_IBITMAP_CORRUPT); goto out; } - inode->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); /* Precompute checksum seed for inode metadata */ if (ext4_has_metadata_csum(sb)) { diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 601214453c3aec9163289c70c58426a40cdfb024..2b5ef1b6424992b43a834de801eec226a5e2900b 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1188,6 +1188,13 @@ retry_grab: page = grab_cache_page_write_begin(mapping, index); if (!page) return -ENOMEM; + /* + * The same as page allocation, we prealloc buffer heads before + * starting the handle. + */ + if (!page_has_buffers(page)) + create_empty_buffers(page, inode->i_sb->s_blocksize, 0); + unlock_page(page); retry_journal: @@ -5342,6 +5349,7 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, int error, rc = 0; int orphan = 0; const unsigned int ia_valid = attr->ia_valid; + bool inc_ivers = true; if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) return -EIO; @@ -5425,8 +5433,8 @@ int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, return -EINVAL; } - if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size) - inode_inc_iversion(inode); + if (attr->ia_size == inode->i_size) + inc_ivers = false; if (shrink) { if (ext4_should_order_data(inode)) { @@ -5528,6 +5536,8 @@ out_mmap_sem: } if (!error) { + if (inc_ivers) + inode_inc_iversion(inode); setattr_copy(mnt_userns, inode, attr); mark_inode_dirty(inode); } @@ -5550,6 +5560,22 @@ err_out: return error; } +u32 ext4_dio_alignment(struct inode *inode) +{ + if (fsverity_active(inode)) + return 0; + if (ext4_should_journal_data(inode)) + return 0; + if (ext4_has_inline_data(inode)) + return 0; + if (IS_ENCRYPTED(inode)) { + if (!fscrypt_dio_supported(inode)) + return 0; + return i_blocksize(inode); + } + return 1; /* use the iomap defaults */ +} + int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { @@ -5565,6 +5591,27 @@ int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path, stat->btime.tv_nsec = ei->i_crtime.tv_nsec; } + /* + * Return the DIO alignment restrictions if requested. We only return + * this information when requested, since on encrypted files it might + * take a fair bit of work to get if the file wasn't opened recently. + */ + if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { + u32 dio_align = ext4_dio_alignment(inode); + + stat->result_mask |= STATX_DIOALIGN; + if (dio_align == 1) { + struct block_device *bdev = inode->i_sb->s_bdev; + + /* iomap defaults */ + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + stat->dio_offset_align = bdev_logical_block_size(bdev); + } else { + stat->dio_mem_align = dio_align; + stat->dio_offset_align = dio_align; + } + } + flags = ei->i_flags & EXT4_FL_USER_VISIBLE; if (flags & EXT4_APPEND_FL) stat->attributes |= STATX_ATTR_APPEND; @@ -5731,9 +5778,6 @@ int ext4_mark_iloc_dirty(handle_t *handle, } ext4_fc_track_inode(handle, inode); - if (IS_I_VERSION(inode)) - inode_inc_iversion(inode); - /* the do_update_inode consumes one bh->b_count */ get_bh(iloc->bh); diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 3cf3ec4b1c2141b181afaecc34ada8ca0a76943c..ded535535b27bc394a56e4d6b9d2fd52e5839812 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -452,9 +452,10 @@ static long swap_inode_boot_loader(struct super_block *sb, swap_inode_data(inode, inode_bl); inode->i_ctime = inode_bl->i_ctime = current_time(inode); + inode_inc_iversion(inode); - inode->i_generation = prandom_u32(); - inode_bl->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); + inode_bl->i_generation = get_random_u32(); ext4_reset_inode_seed(inode); ext4_reset_inode_seed(inode_bl); @@ -665,6 +666,7 @@ static int ext4_ioctl_setflags(struct inode *inode, ext4_set_inode_flags(inode, false); inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); err = ext4_mark_iloc_dirty(handle, inode, &iloc); flags_err: @@ -775,6 +777,7 @@ static int ext4_ioctl_setproject(struct inode *inode, __u32 projid) EXT4_I(inode)->i_projid = kprojid; inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); out_dirty: rc = ext4_mark_iloc_dirty(handle, inode, &iloc); if (!err) @@ -1060,9 +1063,6 @@ static int ext4_ioctl_checkpoint(struct file *filp, unsigned long arg) if (!EXT4_SB(sb)->s_journal) return -ENODEV; - if (flags & ~EXT4_IOC_CHECKPOINT_FLAG_VALID) - return -EINVAL; - if ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && !bdev_max_discard_sectors(EXT4_SB(sb)->s_journal->j_dev)) return -EOPNOTSUPP; @@ -1257,6 +1257,7 @@ static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) err = ext4_reserve_inode_write(handle, inode, &iloc); if (err == 0) { inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); inode->i_generation = generation; err = ext4_mark_iloc_dirty(handle, inode, &iloc); } diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c index bd8f8b5c3d30b0b06fc56103748ae5f05530bd4a..9dad93059945b8e577987d88efc20d6455585f31 100644 --- a/fs/ext4/mballoc.c +++ b/fs/ext4/mballoc.c @@ -140,13 +140,15 @@ * number of buddy bitmap orders possible) number of lists. Group-infos are * placed in appropriate lists. * - * 2) Average fragment size rb tree (sbi->s_mb_avg_fragment_size_root) + * 2) Average fragment size lists (sbi->s_mb_avg_fragment_size) * - * Locking: sbi->s_mb_rb_lock (rwlock) + * Locking: sbi->s_mb_avg_fragment_size_locks(array of rw locks) * - * This is a red black tree consisting of group infos and the tree is sorted - * by average fragment sizes (which is calculated as ext4_group_info->bb_free - * / ext4_group_info->bb_fragments). + * This is an array of lists where in the i-th list there are groups with + * average fragment size >= 2^i and < 2^(i+1). The average fragment size + * is computed as ext4_group_info->bb_free / ext4_group_info->bb_fragments. + * Note that we don't bother with a special list for completely empty groups + * so we only have MB_NUM_ORDERS(sb) lists. * * When "mb_optimize_scan" mount option is set, mballoc consults the above data * structures to decide the order in which groups are to be traversed for @@ -160,7 +162,8 @@ * * At CR = 1, we only consider groups where average fragment size > request * size. So, we lookup a group which has average fragment size just above or - * equal to request size using our rb tree (data structure 2) in O(log N) time. + * equal to request size using our average fragment size group lists (data + * structure 2) in O(1) time. * * If "mb_optimize_scan" mount option is not set, mballoc traverses groups in * linear order which requires O(N) search time for each CR 0 and CR 1 phase. @@ -802,65 +805,51 @@ static void ext4_mb_mark_free_simple(struct super_block *sb, } } -static void ext4_mb_rb_insert(struct rb_root *root, struct rb_node *new, - int (*cmp)(struct rb_node *, struct rb_node *)) +static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len) { - struct rb_node **iter = &root->rb_node, *parent = NULL; + int order; - while (*iter) { - parent = *iter; - if (cmp(new, *iter) > 0) - iter = &((*iter)->rb_left); - else - iter = &((*iter)->rb_right); - } - - rb_link_node(new, parent, iter); - rb_insert_color(new, root); -} - -static int -ext4_mb_avg_fragment_size_cmp(struct rb_node *rb1, struct rb_node *rb2) -{ - struct ext4_group_info *grp1 = rb_entry(rb1, - struct ext4_group_info, - bb_avg_fragment_size_rb); - struct ext4_group_info *grp2 = rb_entry(rb2, - struct ext4_group_info, - bb_avg_fragment_size_rb); - int num_frags_1, num_frags_2; - - num_frags_1 = grp1->bb_fragments ? - grp1->bb_free / grp1->bb_fragments : 0; - num_frags_2 = grp2->bb_fragments ? - grp2->bb_free / grp2->bb_fragments : 0; - - return (num_frags_2 - num_frags_1); + /* + * We don't bother with a special lists groups with only 1 block free + * extents and for completely empty groups. + */ + order = fls(len) - 2; + if (order < 0) + return 0; + if (order == MB_NUM_ORDERS(sb)) + order--; + return order; } -/* - * Reinsert grpinfo into the avg_fragment_size tree with new average - * fragment size. - */ +/* Move group to appropriate avg_fragment_size list */ static void mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) { struct ext4_sb_info *sbi = EXT4_SB(sb); + int new_order; if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_free == 0) return; - write_lock(&sbi->s_mb_rb_lock); - if (!RB_EMPTY_NODE(&grp->bb_avg_fragment_size_rb)) { - rb_erase(&grp->bb_avg_fragment_size_rb, - &sbi->s_mb_avg_fragment_size_root); - RB_CLEAR_NODE(&grp->bb_avg_fragment_size_rb); - } + new_order = mb_avg_fragment_size_order(sb, + grp->bb_free / grp->bb_fragments); + if (new_order == grp->bb_avg_fragment_size_order) + return; - ext4_mb_rb_insert(&sbi->s_mb_avg_fragment_size_root, - &grp->bb_avg_fragment_size_rb, - ext4_mb_avg_fragment_size_cmp); - write_unlock(&sbi->s_mb_rb_lock); + if (grp->bb_avg_fragment_size_order != -1) { + write_lock(&sbi->s_mb_avg_fragment_size_locks[ + grp->bb_avg_fragment_size_order]); + list_del(&grp->bb_avg_fragment_size_node); + write_unlock(&sbi->s_mb_avg_fragment_size_locks[ + grp->bb_avg_fragment_size_order]); + } + grp->bb_avg_fragment_size_order = new_order; + write_lock(&sbi->s_mb_avg_fragment_size_locks[ + grp->bb_avg_fragment_size_order]); + list_add_tail(&grp->bb_avg_fragment_size_node, + &sbi->s_mb_avg_fragment_size[grp->bb_avg_fragment_size_order]); + write_unlock(&sbi->s_mb_avg_fragment_size_locks[ + grp->bb_avg_fragment_size_order]); } /* @@ -909,86 +898,55 @@ static void ext4_mb_choose_next_group_cr0(struct ext4_allocation_context *ac, *new_cr = 1; } else { *group = grp->bb_group; - ac->ac_last_optimal_group = *group; ac->ac_flags |= EXT4_MB_CR0_OPTIMIZED; } } /* - * Choose next group by traversing average fragment size tree. Updates *new_cr - * if cr lvel needs an update. Sets EXT4_MB_SEARCH_NEXT_LINEAR to indicate that - * the linear search should continue for one iteration since there's lock - * contention on the rb tree lock. + * Choose next group by traversing average fragment size list of suitable + * order. Updates *new_cr if cr level needs an update. */ static void ext4_mb_choose_next_group_cr1(struct ext4_allocation_context *ac, int *new_cr, ext4_group_t *group, ext4_group_t ngroups) { struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); - int avg_fragment_size, best_so_far; - struct rb_node *node, *found; - struct ext4_group_info *grp; - - /* - * If there is contention on the lock, instead of waiting for the lock - * to become available, just continue searching lineraly. We'll resume - * our rb tree search later starting at ac->ac_last_optimal_group. - */ - if (!read_trylock(&sbi->s_mb_rb_lock)) { - ac->ac_flags |= EXT4_MB_SEARCH_NEXT_LINEAR; - return; - } + struct ext4_group_info *grp = NULL, *iter; + int i; if (unlikely(ac->ac_flags & EXT4_MB_CR1_OPTIMIZED)) { if (sbi->s_mb_stats) atomic_inc(&sbi->s_bal_cr1_bad_suggestions); - /* We have found something at CR 1 in the past */ - grp = ext4_get_group_info(ac->ac_sb, ac->ac_last_optimal_group); - for (found = rb_next(&grp->bb_avg_fragment_size_rb); found != NULL; - found = rb_next(found)) { - grp = rb_entry(found, struct ext4_group_info, - bb_avg_fragment_size_rb); + } + + for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); + i < MB_NUM_ORDERS(ac->ac_sb); i++) { + if (list_empty(&sbi->s_mb_avg_fragment_size[i])) + continue; + read_lock(&sbi->s_mb_avg_fragment_size_locks[i]); + if (list_empty(&sbi->s_mb_avg_fragment_size[i])) { + read_unlock(&sbi->s_mb_avg_fragment_size_locks[i]); + continue; + } + list_for_each_entry(iter, &sbi->s_mb_avg_fragment_size[i], + bb_avg_fragment_size_node) { if (sbi->s_mb_stats) atomic64_inc(&sbi->s_bal_cX_groups_considered[1]); - if (likely(ext4_mb_good_group(ac, grp->bb_group, 1))) + if (likely(ext4_mb_good_group(ac, iter->bb_group, 1))) { + grp = iter; break; - } - goto done; - } - - node = sbi->s_mb_avg_fragment_size_root.rb_node; - best_so_far = 0; - found = NULL; - - while (node) { - grp = rb_entry(node, struct ext4_group_info, - bb_avg_fragment_size_rb); - avg_fragment_size = 0; - if (ext4_mb_good_group(ac, grp->bb_group, 1)) { - avg_fragment_size = grp->bb_fragments ? - grp->bb_free / grp->bb_fragments : 0; - if (!best_so_far || avg_fragment_size < best_so_far) { - best_so_far = avg_fragment_size; - found = node; } } - if (avg_fragment_size > ac->ac_g_ex.fe_len) - node = node->rb_right; - else - node = node->rb_left; + read_unlock(&sbi->s_mb_avg_fragment_size_locks[i]); + if (grp) + break; } -done: - if (found) { - grp = rb_entry(found, struct ext4_group_info, - bb_avg_fragment_size_rb); + if (grp) { *group = grp->bb_group; ac->ac_flags |= EXT4_MB_CR1_OPTIMIZED; } else { *new_cr = 2; } - - read_unlock(&sbi->s_mb_rb_lock); - ac->ac_last_optimal_group = *group; } static inline int should_optimize_scan(struct ext4_allocation_context *ac) @@ -1017,11 +975,6 @@ next_linear_group(struct ext4_allocation_context *ac, int group, int ngroups) goto inc_and_return; } - if (ac->ac_flags & EXT4_MB_SEARCH_NEXT_LINEAR) { - ac->ac_flags &= ~EXT4_MB_SEARCH_NEXT_LINEAR; - goto inc_and_return; - } - return group; inc_and_return: /* @@ -1049,8 +1002,10 @@ static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, { *new_cr = ac->ac_criteria; - if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) + if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) { + *group = next_linear_group(ac, *group, ngroups); return; + } if (*new_cr == 0) { ext4_mb_choose_next_group_cr0(ac, new_cr, group, ngroups); @@ -1075,23 +1030,25 @@ mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) struct ext4_sb_info *sbi = EXT4_SB(sb); int i; - if (test_opt2(sb, MB_OPTIMIZE_SCAN) && grp->bb_largest_free_order >= 0) { + for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) + if (grp->bb_counters[i] > 0) + break; + /* No need to move between order lists? */ + if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || + i == grp->bb_largest_free_order) { + grp->bb_largest_free_order = i; + return; + } + + if (grp->bb_largest_free_order >= 0) { write_lock(&sbi->s_mb_largest_free_orders_locks[ grp->bb_largest_free_order]); list_del_init(&grp->bb_largest_free_order_node); write_unlock(&sbi->s_mb_largest_free_orders_locks[ grp->bb_largest_free_order]); } - grp->bb_largest_free_order = -1; /* uninit */ - - for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) { - if (grp->bb_counters[i] > 0) { - grp->bb_largest_free_order = i; - break; - } - } - if (test_opt2(sb, MB_OPTIMIZE_SCAN) && - grp->bb_largest_free_order >= 0 && grp->bb_free) { + grp->bb_largest_free_order = i; + if (grp->bb_largest_free_order >= 0 && grp->bb_free) { write_lock(&sbi->s_mb_largest_free_orders_locks[ grp->bb_largest_free_order]); list_add_tail(&grp->bb_largest_free_order_node, @@ -1148,13 +1105,13 @@ void ext4_mb_generate_buddy(struct super_block *sb, EXT4_GROUP_INFO_BBITMAP_CORRUPT); } mb_set_largest_free_order(sb, grp); + mb_update_avg_fragment_size(sb, grp); clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); period = get_cycles() - period; atomic_inc(&sbi->s_mb_buddies_generated); atomic64_add(period, &sbi->s_mb_generation_time); - mb_update_avg_fragment_size(sb, grp); } /* The buddy information is attached the buddy cache inode @@ -2636,7 +2593,7 @@ static noinline_for_stack int ext4_mb_regular_allocator(struct ext4_allocation_context *ac) { ext4_group_t prefetch_grp = 0, ngroups, group, i; - int cr = -1; + int cr = -1, new_cr; int err = 0, first_err = 0; unsigned int nr = 0, prefetch_ios = 0; struct ext4_sb_info *sbi; @@ -2707,17 +2664,14 @@ repeat: * from the goal value specified */ group = ac->ac_g_ex.fe_group; - ac->ac_last_optimal_group = group; ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; prefetch_grp = group; - for (i = 0; i < ngroups; group = next_linear_group(ac, group, ngroups), - i++) { - int ret = 0, new_cr; + for (i = 0, new_cr = cr; i < ngroups; i++, + ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) { + int ret = 0; cond_resched(); - - ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups); if (new_cr != cr) { cr = new_cr; goto repeat; @@ -2991,9 +2945,7 @@ __acquires(&EXT4_SB(sb)->s_mb_rb_lock) struct super_block *sb = pde_data(file_inode(seq->file)); unsigned long position; - read_lock(&EXT4_SB(sb)->s_mb_rb_lock); - - if (*pos < 0 || *pos >= MB_NUM_ORDERS(sb) + 1) + if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) return NULL; position = *pos + 1; return (void *) ((unsigned long) position); @@ -3005,7 +2957,7 @@ static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, lof unsigned long position; ++*pos; - if (*pos < 0 || *pos >= MB_NUM_ORDERS(sb) + 1) + if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) return NULL; position = *pos + 1; return (void *) ((unsigned long) position); @@ -3017,29 +2969,22 @@ static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) struct ext4_sb_info *sbi = EXT4_SB(sb); unsigned long position = ((unsigned long) v); struct ext4_group_info *grp; - struct rb_node *n; - unsigned int count, min, max; + unsigned int count; position--; if (position >= MB_NUM_ORDERS(sb)) { - seq_puts(seq, "fragment_size_tree:\n"); - n = rb_first(&sbi->s_mb_avg_fragment_size_root); - if (!n) { - seq_puts(seq, "\ttree_min: 0\n\ttree_max: 0\n\ttree_nodes: 0\n"); - return 0; - } - grp = rb_entry(n, struct ext4_group_info, bb_avg_fragment_size_rb); - min = grp->bb_fragments ? grp->bb_free / grp->bb_fragments : 0; - count = 1; - while (rb_next(n)) { - count++; - n = rb_next(n); - } - grp = rb_entry(n, struct ext4_group_info, bb_avg_fragment_size_rb); - max = grp->bb_fragments ? grp->bb_free / grp->bb_fragments : 0; + position -= MB_NUM_ORDERS(sb); + if (position == 0) + seq_puts(seq, "avg_fragment_size_lists:\n"); - seq_printf(seq, "\ttree_min: %u\n\ttree_max: %u\n\ttree_nodes: %u\n", - min, max, count); + count = 0; + read_lock(&sbi->s_mb_avg_fragment_size_locks[position]); + list_for_each_entry(grp, &sbi->s_mb_avg_fragment_size[position], + bb_avg_fragment_size_node) + count++; + read_unlock(&sbi->s_mb_avg_fragment_size_locks[position]); + seq_printf(seq, "\tlist_order_%u_groups: %u\n", + (unsigned int)position, count); return 0; } @@ -3049,9 +2994,11 @@ static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) seq_puts(seq, "max_free_order_lists:\n"); } count = 0; + read_lock(&sbi->s_mb_largest_free_orders_locks[position]); list_for_each_entry(grp, &sbi->s_mb_largest_free_orders[position], bb_largest_free_order_node) count++; + read_unlock(&sbi->s_mb_largest_free_orders_locks[position]); seq_printf(seq, "\tlist_order_%u_groups: %u\n", (unsigned int)position, count); @@ -3059,11 +3006,7 @@ static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) } static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v) -__releases(&EXT4_SB(sb)->s_mb_rb_lock) { - struct super_block *sb = pde_data(file_inode(seq->file)); - - read_unlock(&EXT4_SB(sb)->s_mb_rb_lock); } const struct seq_operations ext4_mb_seq_structs_summary_ops = { @@ -3176,8 +3119,9 @@ int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, init_rwsem(&meta_group_info[i]->alloc_sem); meta_group_info[i]->bb_free_root = RB_ROOT; INIT_LIST_HEAD(&meta_group_info[i]->bb_largest_free_order_node); - RB_CLEAR_NODE(&meta_group_info[i]->bb_avg_fragment_size_rb); + INIT_LIST_HEAD(&meta_group_info[i]->bb_avg_fragment_size_node); meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ + meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */ meta_group_info[i]->bb_group = group; mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group); @@ -3426,7 +3370,24 @@ int ext4_mb_init(struct super_block *sb) i++; } while (i < MB_NUM_ORDERS(sb)); - sbi->s_mb_avg_fragment_size_root = RB_ROOT; + sbi->s_mb_avg_fragment_size = + kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head), + GFP_KERNEL); + if (!sbi->s_mb_avg_fragment_size) { + ret = -ENOMEM; + goto out; + } + sbi->s_mb_avg_fragment_size_locks = + kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t), + GFP_KERNEL); + if (!sbi->s_mb_avg_fragment_size_locks) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < MB_NUM_ORDERS(sb); i++) { + INIT_LIST_HEAD(&sbi->s_mb_avg_fragment_size[i]); + rwlock_init(&sbi->s_mb_avg_fragment_size_locks[i]); + } sbi->s_mb_largest_free_orders = kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head), GFP_KERNEL); @@ -3445,7 +3406,6 @@ int ext4_mb_init(struct super_block *sb) INIT_LIST_HEAD(&sbi->s_mb_largest_free_orders[i]); rwlock_init(&sbi->s_mb_largest_free_orders_locks[i]); } - rwlock_init(&sbi->s_mb_rb_lock); spin_lock_init(&sbi->s_md_lock); sbi->s_mb_free_pending = 0; @@ -3516,6 +3476,8 @@ out_free_locality_groups: free_percpu(sbi->s_locality_groups); sbi->s_locality_groups = NULL; out: + kfree(sbi->s_mb_avg_fragment_size); + kfree(sbi->s_mb_avg_fragment_size_locks); kfree(sbi->s_mb_largest_free_orders); kfree(sbi->s_mb_largest_free_orders_locks); kfree(sbi->s_mb_offsets); @@ -3582,6 +3544,8 @@ int ext4_mb_release(struct super_block *sb) kvfree(group_info); rcu_read_unlock(); } + kfree(sbi->s_mb_avg_fragment_size); + kfree(sbi->s_mb_avg_fragment_size_locks); kfree(sbi->s_mb_largest_free_orders); kfree(sbi->s_mb_largest_free_orders_locks); kfree(sbi->s_mb_offsets); @@ -5193,6 +5157,7 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); int bsbits = ac->ac_sb->s_blocksize_bits; loff_t size, isize; + bool inode_pa_eligible, group_pa_eligible; if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) return; @@ -5200,25 +5165,27 @@ static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) return; + group_pa_eligible = sbi->s_mb_group_prealloc > 0; + inode_pa_eligible = true; size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) >> bsbits; + /* No point in using inode preallocation for closed files */ if ((size == isize) && !ext4_fs_is_busy(sbi) && - !inode_is_open_for_write(ac->ac_inode)) { - ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; - return; - } + !inode_is_open_for_write(ac->ac_inode)) + inode_pa_eligible = false; - if (sbi->s_mb_group_prealloc <= 0) { - ac->ac_flags |= EXT4_MB_STREAM_ALLOC; - return; - } - - /* don't use group allocation for large files */ size = max(size, isize); - if (size > sbi->s_mb_stream_request) { - ac->ac_flags |= EXT4_MB_STREAM_ALLOC; + /* Don't use group allocation for large files */ + if (size > sbi->s_mb_stream_request) + group_pa_eligible = false; + + if (!group_pa_eligible) { + if (inode_pa_eligible) + ac->ac_flags |= EXT4_MB_STREAM_ALLOC; + else + ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; return; } @@ -5565,6 +5532,7 @@ ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, ext4_fsblk_t block = 0; unsigned int inquota = 0; unsigned int reserv_clstrs = 0; + int retries = 0; u64 seq; might_sleep(); @@ -5667,7 +5635,8 @@ repeat: ar->len = ac->ac_b_ex.fe_len; } } else { - if (ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) + if (++retries < 3 && + ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) goto repeat; /* * If block allocation fails then the pa allocated above diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h index 39da92ceabf88f9041f4dbcd8c348309436abbdf..dcda2a943cee00aed644575c7bd231ba1883c987 100644 --- a/fs/ext4/mballoc.h +++ b/fs/ext4/mballoc.h @@ -178,7 +178,6 @@ struct ext4_allocation_context { /* copy of the best found extent taken before preallocation efforts */ struct ext4_free_extent ac_f_ex; - ext4_group_t ac_last_optimal_group; __u32 ac_groups_considered; __u32 ac_flags; /* allocation hints */ __u16 ac_groups_scanned; diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 54e7d3c95fd716b6857c0e53ae3314f142fbe51c..0a220ec9862de3ca520d609ec2803008b0412556 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -56,8 +56,7 @@ static int finish_range(handle_t *handle, struct inode *inode, retval = ext4_ext_insert_extent(handle, inode, &path, &newext, 0); err_out: up_write((&EXT4_I(inode)->i_data_sem)); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); lb->first_pblock = 0; return retval; } diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c index 9af68a7ecdcf3aa8c12ee793902a0d27c17a4bf8..588cb09c5291fcd1f0ea6d087634df097b4d8637 100644 --- a/fs/ext4/mmp.c +++ b/fs/ext4/mmp.c @@ -265,7 +265,7 @@ static unsigned int mmp_new_seq(void) u32 new_seq; do { - new_seq = prandom_u32(); + new_seq = get_random_u32(); } while (new_seq > EXT4_MMP_SEQ_MAX); return new_seq; diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c index 701f1d6a217f79fcc6986d5da9ed77343701a581..044e34cd835c125d3b487414a6ae7b0bb6f570ca 100644 --- a/fs/ext4/move_extent.c +++ b/fs/ext4/move_extent.c @@ -32,8 +32,7 @@ get_ext_path(struct inode *inode, ext4_lblk_t lblock, if (IS_ERR(path)) return PTR_ERR(path); if (path[ext_depth(inode)].p_ext == NULL) { - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); *ppath = NULL; return -ENODATA; } @@ -103,12 +102,10 @@ mext_check_coverage(struct inode *inode, ext4_lblk_t from, ext4_lblk_t count, if (unwritten != ext4_ext_is_unwritten(ext)) goto out; from += ext4_ext_get_actual_len(ext); - ext4_ext_drop_refs(path); } ret = 1; out: - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return ret; } @@ -472,19 +469,17 @@ mext_check_arguments(struct inode *orig_inode, if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) return -EPERM; - /* Ext4 move extent does not support swapfile */ + /* Ext4 move extent does not support swap files */ if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) { - ext4_debug("ext4 move extent: The argument files should " - "not be swapfile [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %lu, donor %lu]\n", orig_inode->i_ino, donor_inode->i_ino); - return -EBUSY; + return -ETXTBSY; } if (ext4_is_quota_file(orig_inode) && ext4_is_quota_file(donor_inode)) { - ext4_debug("ext4 move extent: The argument files should " - "not be quota files [ino:orig %lu, donor %lu]\n", + ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %lu, donor %lu]\n", orig_inode->i_ino, donor_inode->i_ino); - return -EBUSY; + return -EOPNOTSUPP; } /* Ext4 move extent supports only extent based file */ @@ -631,11 +626,11 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, if (ret) goto out; ex = path[path->p_depth].p_ext; - next_blk = ext4_ext_next_allocated_block(path); cur_blk = le32_to_cpu(ex->ee_block); cur_len = ext4_ext_get_actual_len(ex); /* Check hole before the start pos */ if (cur_blk + cur_len - 1 < o_start) { + next_blk = ext4_ext_next_allocated_block(path); if (next_blk == EXT_MAX_BLOCKS) { ret = -ENODATA; goto out; @@ -663,7 +658,7 @@ ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, donor_page_index = d_start >> (PAGE_SHIFT - donor_inode->i_blkbits); offset_in_page = o_start % blocks_per_page; - if (cur_len > blocks_per_page- offset_in_page) + if (cur_len > blocks_per_page - offset_in_page) cur_len = blocks_per_page - offset_in_page; /* * Up semaphore to avoid following problems: @@ -694,8 +689,7 @@ out: ext4_discard_preallocations(donor_inode, 0); } - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); ext4_double_up_write_data_sem(orig_inode, donor_inode); unlock_two_nondirectories(orig_inode, donor_inode); diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 3a31b662f6619a57d7f82ce7db16dabdcb449ace..d5daaf41e1fc9e982c432801d1d5053ca269b08d 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -85,15 +85,20 @@ static struct buffer_head *ext4_append(handle_t *handle, return bh; inode->i_size += inode->i_sb->s_blocksize; EXT4_I(inode)->i_disksize = inode->i_size; + err = ext4_mark_inode_dirty(handle, inode); + if (err) + goto out; BUFFER_TRACE(bh, "get_write_access"); err = ext4_journal_get_write_access(handle, inode->i_sb, bh, EXT4_JTR_NONE); - if (err) { - brelse(bh); - ext4_std_error(inode->i_sb, err); - return ERR_PTR(err); - } + if (err) + goto out; return bh; + +out: + brelse(bh); + ext4_std_error(inode->i_sb, err); + return ERR_PTR(err); } static int ext4_dx_csum_verify(struct inode *inode, @@ -126,7 +131,7 @@ static struct buffer_head *__ext4_read_dirblock(struct inode *inode, struct ext4_dir_entry *dirent; int is_dx_block = 0; - if (block >= inode->i_size) { + if (block >= inode->i_size >> inode->i_blkbits) { ext4_error_inode(inode, func, line, block, "Attempting to read directory block (%u) that is past i_size (%llu)", block, inode->i_size); @@ -2849,7 +2854,7 @@ retry: } static int ext4_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { handle_t *handle; struct inode *inode; @@ -2871,7 +2876,7 @@ retry: inode->i_op = &ext4_file_inode_operations; inode->i_fop = &ext4_file_operations; ext4_set_aops(inode); - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); err = ext4_orphan_add(handle, inode); if (err) goto err_unlock_inode; @@ -2882,7 +2887,7 @@ retry: ext4_journal_stop(handle); if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) goto retry; - return err; + return finish_open_simple(file, err); err_unlock_inode: ext4_journal_stop(handle); unlock_new_inode(inode); diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index e02a5f14e0211e7e404be31064559124c923373c..3d21eae267fcaba2126febe294eb157148c226cc 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -75,7 +75,7 @@ static void __read_end_io(struct bio *bio) bio_for_each_segment_all(bv, bio, iter_all) { page = bv->bv_page; - /* PG_error was set if any post_read step failed */ + /* PG_error was set if verity failed. */ if (bio->bi_status || PageError(page)) { ClearPageUptodate(page); /* will re-read again later */ @@ -96,10 +96,12 @@ static void decrypt_work(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); + struct bio *bio = ctx->bio; - fscrypt_decrypt_bio(ctx->bio); - - bio_post_read_processing(ctx); + if (fscrypt_decrypt_bio(bio)) + bio_post_read_processing(ctx); + else + __read_end_io(bio); } static void verity_work(struct work_struct *work) diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c index fea2a68d067b09e2059d869eddf16cc2e004dee4..6dfe9ccae0c50c2ba92cc24e9b7489072f4e42a4 100644 --- a/fs/ext4/resize.c +++ b/fs/ext4/resize.c @@ -2122,7 +2122,7 @@ retry: goto out; } - if (ext4_blocks_count(es) == n_blocks_count) + if (ext4_blocks_count(es) == n_blocks_count && n_blocks_count_retry == 0) goto out; err = ext4_alloc_flex_bg_array(sb, n_group + 1); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 9a66abcca1a852ec49df449bec345103de40d54a..989365b878a67f5271176bd40f5a9cf97fd17224 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -205,19 +205,12 @@ int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, bh_end_io_t *end_io int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait) { - if (trylock_buffer(bh)) { - if (wait) - return ext4_read_bh(bh, op_flags, NULL); + lock_buffer(bh); + if (!wait) { ext4_read_bh_nowait(bh, op_flags, NULL); return 0; } - if (wait) { - wait_on_buffer(bh); - if (buffer_uptodate(bh)) - return 0; - return -EIO; - } - return 0; + return ext4_read_bh(bh, op_flags, NULL); } /* @@ -264,7 +257,8 @@ void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block) struct buffer_head *bh = sb_getblk_gfp(sb, block, 0); if (likely(bh)) { - ext4_read_bh_lock(bh, REQ_RAHEAD, false); + if (trylock_buffer(bh)) + ext4_read_bh_nowait(bh, REQ_RAHEAD, NULL); brelse(bh); } } @@ -1576,7 +1570,7 @@ enum { Opt_bsd_df, Opt_minix_df, Opt_grpid, Opt_nogrpid, Opt_resgid, Opt_resuid, Opt_sb, Opt_nouid32, Opt_debug, Opt_removed, - Opt_user_xattr, Opt_nouser_xattr, Opt_acl, Opt_noacl, + Opt_user_xattr, Opt_acl, Opt_auto_da_alloc, Opt_noauto_da_alloc, Opt_noload, Opt_commit, Opt_min_batch_time, Opt_max_batch_time, Opt_journal_dev, Opt_journal_path, Opt_journal_checksum, Opt_journal_async_commit, @@ -1585,7 +1579,7 @@ enum { Opt_inlinecrypt, Opt_usrjquota, Opt_grpjquota, Opt_quota, Opt_noquota, Opt_barrier, Opt_nobarrier, Opt_err, - Opt_usrquota, Opt_grpquota, Opt_prjquota, Opt_i_version, + Opt_usrquota, Opt_grpquota, Opt_prjquota, Opt_dax, Opt_dax_always, Opt_dax_inode, Opt_dax_never, Opt_stripe, Opt_delalloc, Opt_nodelalloc, Opt_warn_on_error, Opt_nowarn_on_error, Opt_mblk_io_submit, Opt_debug_want_extra_isize, @@ -1662,9 +1656,7 @@ static const struct fs_parameter_spec ext4_param_specs[] = { fsparam_flag ("oldalloc", Opt_removed), fsparam_flag ("orlov", Opt_removed), fsparam_flag ("user_xattr", Opt_user_xattr), - fsparam_flag ("nouser_xattr", Opt_nouser_xattr), fsparam_flag ("acl", Opt_acl), - fsparam_flag ("noacl", Opt_noacl), fsparam_flag ("norecovery", Opt_noload), fsparam_flag ("noload", Opt_noload), fsparam_flag ("bh", Opt_removed), @@ -1694,7 +1686,7 @@ static const struct fs_parameter_spec ext4_param_specs[] = { fsparam_flag ("barrier", Opt_barrier), fsparam_u32 ("barrier", Opt_barrier), fsparam_flag ("nobarrier", Opt_nobarrier), - fsparam_flag ("i_version", Opt_i_version), + fsparam_flag ("i_version", Opt_removed), fsparam_flag ("dax", Opt_dax), fsparam_enum ("dax", Opt_dax_type, ext4_param_dax), fsparam_u32 ("stripe", Opt_stripe), @@ -1814,13 +1806,10 @@ static const struct mount_opts { {Opt_journal_ioprio, 0, MOPT_NO_EXT2}, {Opt_data, 0, MOPT_NO_EXT2}, {Opt_user_xattr, EXT4_MOUNT_XATTR_USER, MOPT_SET}, - {Opt_nouser_xattr, EXT4_MOUNT_XATTR_USER, MOPT_CLEAR}, #ifdef CONFIG_EXT4_FS_POSIX_ACL {Opt_acl, EXT4_MOUNT_POSIX_ACL, MOPT_SET}, - {Opt_noacl, EXT4_MOUNT_POSIX_ACL, MOPT_CLEAR}, #else {Opt_acl, 0, MOPT_NOSUPPORT}, - {Opt_noacl, 0, MOPT_NOSUPPORT}, #endif {Opt_nouid32, EXT4_MOUNT_NO_UID32, MOPT_SET}, {Opt_debug, EXT4_MOUNT_DEBUG, MOPT_SET}, @@ -2120,10 +2109,6 @@ static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param) else return note_qf_name(fc, GRPQUOTA, param); #endif - case Opt_noacl: - case Opt_nouser_xattr: - ext4_msg(NULL, KERN_WARNING, deprecated_msg, param->key, "3.5"); - break; case Opt_sb: if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { ext4_msg(NULL, KERN_WARNING, @@ -2140,11 +2125,6 @@ static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param) case Opt_abort: ctx_set_mount_flag(ctx, EXT4_MF_FS_ABORTED); return 0; - case Opt_i_version: - ext4_msg(NULL, KERN_WARNING, deprecated_msg, param->key, "5.20"); - ext4_msg(NULL, KERN_WARNING, "Use iversion instead\n"); - ctx_set_flags(ctx, SB_I_VERSION); - return 0; case Opt_inlinecrypt: #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT ctx_set_flags(ctx, SB_INLINECRYPT); @@ -2814,14 +2794,6 @@ static void ext4_apply_options(struct fs_context *fc, struct super_block *sb) sb->s_flags &= ~ctx->mask_s_flags; sb->s_flags |= ctx->vals_s_flags; - /* - * i_version differs from common mount option iversion so we have - * to let vfs know that it was set, otherwise it would get cleared - * on remount - */ - if (ctx->mask_s_flags & SB_I_VERSION) - fc->sb_flags |= SB_I_VERSION; - #define APPLY(X) ({ if (ctx->spec & EXT4_SPEC_##X) sbi->X = ctx->X; }) APPLY(s_commit_interval); APPLY(s_stripe); @@ -2970,8 +2942,6 @@ static int _ext4_show_options(struct seq_file *seq, struct super_block *sb, SEQ_OPTS_PRINT("min_batch_time=%u", sbi->s_min_batch_time); if (nodefs || sbi->s_max_batch_time != EXT4_DEF_MAX_BATCH_TIME) SEQ_OPTS_PRINT("max_batch_time=%u", sbi->s_max_batch_time); - if (sb->s_flags & SB_I_VERSION) - SEQ_OPTS_PUTS("i_version"); if (nodefs || sbi->s_stripe) SEQ_OPTS_PRINT("stripe=%lu", sbi->s_stripe); if (nodefs || EXT4_MOUNT_DATA_FLAGS & @@ -3767,6 +3737,7 @@ static int ext4_lazyinit_thread(void *arg) unsigned long next_wakeup, cur; BUG_ON(NULL == eli); + set_freezable(); cont_thread: while (true) { @@ -3811,8 +3782,7 @@ cont_thread: } if (!progress) { elr->lr_next_sched = jiffies + - (prandom_u32() - % (EXT4_DEF_LI_MAX_START_DELAY * HZ)); + prandom_u32_max(EXT4_DEF_LI_MAX_START_DELAY * HZ); } if (time_before(elr->lr_next_sched, next_wakeup)) next_wakeup = elr->lr_next_sched; @@ -3959,8 +3929,8 @@ static struct ext4_li_request *ext4_li_request_new(struct super_block *sb, * spread the inode table initialization requests * better. */ - elr->lr_next_sched = jiffies + (prandom_u32() % - (EXT4_DEF_LI_MAX_START_DELAY * HZ)); + elr->lr_next_sched = jiffies + prandom_u32_max( + EXT4_DEF_LI_MAX_START_DELAY * HZ); return elr; } @@ -3982,9 +3952,9 @@ int ext4_register_li_request(struct super_block *sb, goto out; } - if (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS) && - (first_not_zeroed == ngroups || sb_rdonly(sb) || - !test_opt(sb, INIT_INODE_TABLE))) + if (sb_rdonly(sb) || + (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS) && + (first_not_zeroed == ngroups || !test_opt(sb, INIT_INODE_TABLE)))) goto out; elr = ext4_li_request_new(sb, first_not_zeroed); @@ -4311,112 +4281,10 @@ err_out: return NULL; } -static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) +static void ext4_set_def_opts(struct super_block *sb, + struct ext4_super_block *es) { - struct buffer_head *bh, **group_desc; - struct ext4_super_block *es = NULL; - struct ext4_sb_info *sbi = EXT4_SB(sb); - struct flex_groups **flex_groups; - ext4_fsblk_t block; - ext4_fsblk_t logical_sb_block; - unsigned long offset = 0; unsigned long def_mount_opts; - struct inode *root; - int ret = -ENOMEM; - int blocksize, clustersize; - unsigned int db_count; - unsigned int i; - int needs_recovery, has_huge_files; - __u64 blocks_count; - int err = 0; - ext4_group_t first_not_zeroed; - struct ext4_fs_context *ctx = fc->fs_private; - int silent = fc->sb_flags & SB_SILENT; - - /* Set defaults for the variables that will be set during parsing */ - if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) - ctx->journal_ioprio = DEFAULT_JOURNAL_IOPRIO; - - sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; - sbi->s_sectors_written_start = - part_stat_read(sb->s_bdev, sectors[STAT_WRITE]); - - /* -EINVAL is default */ - ret = -EINVAL; - blocksize = sb_min_blocksize(sb, EXT4_MIN_BLOCK_SIZE); - if (!blocksize) { - ext4_msg(sb, KERN_ERR, "unable to set blocksize"); - goto out_fail; - } - - /* - * The ext4 superblock will not be buffer aligned for other than 1kB - * block sizes. We need to calculate the offset from buffer start. - */ - if (blocksize != EXT4_MIN_BLOCK_SIZE) { - logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; - offset = do_div(logical_sb_block, blocksize); - } else { - logical_sb_block = sbi->s_sb_block; - } - - bh = ext4_sb_bread_unmovable(sb, logical_sb_block); - if (IS_ERR(bh)) { - ext4_msg(sb, KERN_ERR, "unable to read superblock"); - ret = PTR_ERR(bh); - goto out_fail; - } - /* - * Note: s_es must be initialized as soon as possible because - * some ext4 macro-instructions depend on its value - */ - es = (struct ext4_super_block *) (bh->b_data + offset); - sbi->s_es = es; - sb->s_magic = le16_to_cpu(es->s_magic); - if (sb->s_magic != EXT4_SUPER_MAGIC) - goto cantfind_ext4; - sbi->s_kbytes_written = le64_to_cpu(es->s_kbytes_written); - - /* Warn if metadata_csum and gdt_csum are both set. */ - if (ext4_has_feature_metadata_csum(sb) && - ext4_has_feature_gdt_csum(sb)) - ext4_warning(sb, "metadata_csum and uninit_bg are " - "redundant flags; please run fsck."); - - /* Check for a known checksum algorithm */ - if (!ext4_verify_csum_type(sb, es)) { - ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " - "unknown checksum algorithm."); - silent = 1; - goto cantfind_ext4; - } - ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE, - ext4_orphan_file_block_trigger); - - /* Load the checksum driver */ - sbi->s_chksum_driver = crypto_alloc_shash("crc32c", 0, 0); - if (IS_ERR(sbi->s_chksum_driver)) { - ext4_msg(sb, KERN_ERR, "Cannot load crc32c driver."); - ret = PTR_ERR(sbi->s_chksum_driver); - sbi->s_chksum_driver = NULL; - goto failed_mount; - } - - /* Check superblock checksum */ - if (!ext4_superblock_csum_verify(sb, es)) { - ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " - "invalid superblock checksum. Run e2fsck?"); - silent = 1; - ret = -EFSBADCRC; - goto cantfind_ext4; - } - - /* Precompute checksum seed for all metadata */ - if (ext4_has_feature_csum_seed(sb)) - sbi->s_csum_seed = le32_to_cpu(es->s_checksum_seed); - else if (ext4_has_metadata_csum(sb) || ext4_has_feature_ea_inode(sb)) - sbi->s_csum_seed = ext4_chksum(sbi, ~0, es->s_uuid, - sizeof(es->s_uuid)); /* Set defaults before we parse the mount options */ def_mount_opts = le32_to_cpu(es->s_default_mount_opts); @@ -4445,9 +4313,9 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) else if ((def_mount_opts & EXT4_DEFM_JMODE) == EXT4_DEFM_JMODE_WBACK) set_opt(sb, WRITEBACK_DATA); - if (le16_to_cpu(sbi->s_es->s_errors) == EXT4_ERRORS_PANIC) + if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_PANIC) set_opt(sb, ERRORS_PANIC); - else if (le16_to_cpu(sbi->s_es->s_errors) == EXT4_ERRORS_CONTINUE) + else if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_CONTINUE) set_opt(sb, ERRORS_CONT); else set_opt(sb, ERRORS_RO); @@ -4456,12 +4324,6 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (def_mount_opts & EXT4_DEFM_DISCARD) set_opt(sb, DISCARD); - sbi->s_resuid = make_kuid(&init_user_ns, le16_to_cpu(es->s_def_resuid)); - sbi->s_resgid = make_kgid(&init_user_ns, le16_to_cpu(es->s_def_resgid)); - sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; - sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; - sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; - if ((def_mount_opts & EXT4_DEFM_NOBARRIER) == 0) set_opt(sb, BARRIER); @@ -4473,31 +4335,96 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ((def_mount_opts & EXT4_DEFM_NODELALLOC) == 0)) set_opt(sb, DELALLOC); - /* - * set default s_li_wait_mult for lazyinit, for the case there is - * no mount option specified. - */ - sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; + if (sb->s_blocksize == PAGE_SIZE) + set_opt(sb, DIOREAD_NOLOCK); +} - if (le32_to_cpu(es->s_log_block_size) > - (EXT4_MAX_BLOCK_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { - ext4_msg(sb, KERN_ERR, - "Invalid log block size: %u", - le32_to_cpu(es->s_log_block_size)); - goto failed_mount; - } - if (le32_to_cpu(es->s_log_cluster_size) > - (EXT4_MAX_CLUSTER_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { - ext4_msg(sb, KERN_ERR, - "Invalid log cluster size: %u", - le32_to_cpu(es->s_log_cluster_size)); - goto failed_mount; +static int ext4_handle_clustersize(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int clustersize; + + /* Handle clustersize */ + clustersize = BLOCK_SIZE << le32_to_cpu(es->s_log_cluster_size); + if (ext4_has_feature_bigalloc(sb)) { + if (clustersize < sb->s_blocksize) { + ext4_msg(sb, KERN_ERR, + "cluster size (%d) smaller than " + "block size (%lu)", clustersize, sb->s_blocksize); + return -EINVAL; + } + sbi->s_cluster_bits = le32_to_cpu(es->s_log_cluster_size) - + le32_to_cpu(es->s_log_block_size); + sbi->s_clusters_per_group = + le32_to_cpu(es->s_clusters_per_group); + if (sbi->s_clusters_per_group > sb->s_blocksize * 8) { + ext4_msg(sb, KERN_ERR, + "#clusters per group too big: %lu", + sbi->s_clusters_per_group); + return -EINVAL; + } + if (sbi->s_blocks_per_group != + (sbi->s_clusters_per_group * (clustersize / sb->s_blocksize))) { + ext4_msg(sb, KERN_ERR, "blocks per group (%lu) and " + "clusters per group (%lu) inconsistent", + sbi->s_blocks_per_group, + sbi->s_clusters_per_group); + return -EINVAL; + } + } else { + if (clustersize != sb->s_blocksize) { + ext4_msg(sb, KERN_ERR, + "fragment/cluster size (%d) != " + "block size (%lu)", clustersize, sb->s_blocksize); + return -EINVAL; + } + if (sbi->s_blocks_per_group > sb->s_blocksize * 8) { + ext4_msg(sb, KERN_ERR, + "#blocks per group too big: %lu", + sbi->s_blocks_per_group); + return -EINVAL; + } + sbi->s_clusters_per_group = sbi->s_blocks_per_group; + sbi->s_cluster_bits = 0; } + sbi->s_cluster_ratio = clustersize / sb->s_blocksize; - blocksize = EXT4_MIN_BLOCK_SIZE << le32_to_cpu(es->s_log_block_size); + /* Do we have standard group size of clustersize * 8 blocks ? */ + if (sbi->s_blocks_per_group == clustersize << 3) + set_opt2(sb, STD_GROUP_SIZE); - if (blocksize == PAGE_SIZE) - set_opt(sb, DIOREAD_NOLOCK); + return 0; +} + +static void ext4_fast_commit_init(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* Initialize fast commit stuff */ + atomic_set(&sbi->s_fc_subtid, 0); + INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]); + INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]); + INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]); + INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]); + sbi->s_fc_bytes = 0; + ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); + sbi->s_fc_ineligible_tid = 0; + spin_lock_init(&sbi->s_fc_lock); + memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); + sbi->s_fc_replay_state.fc_regions = NULL; + sbi->s_fc_replay_state.fc_regions_size = 0; + sbi->s_fc_replay_state.fc_regions_used = 0; + sbi->s_fc_replay_state.fc_regions_valid = 0; + sbi->s_fc_replay_state.fc_modified_inodes = NULL; + sbi->s_fc_replay_state.fc_modified_inodes_size = 0; + sbi->s_fc_replay_state.fc_modified_inodes_used = 0; +} + +static int ext4_inode_info_init(struct super_block *sb, + struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV) { sbi->s_inode_size = EXT4_GOOD_OLD_INODE_SIZE; @@ -4508,16 +4435,16 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (sbi->s_first_ino < EXT4_GOOD_OLD_FIRST_INO) { ext4_msg(sb, KERN_ERR, "invalid first ino: %u", sbi->s_first_ino); - goto failed_mount; + return -EINVAL; } if ((sbi->s_inode_size < EXT4_GOOD_OLD_INODE_SIZE) || (!is_power_of_2(sbi->s_inode_size)) || - (sbi->s_inode_size > blocksize)) { + (sbi->s_inode_size > sb->s_blocksize)) { ext4_msg(sb, KERN_ERR, "unsupported inode size: %d", sbi->s_inode_size); - ext4_msg(sb, KERN_ERR, "blocksize: %d", blocksize); - goto failed_mount; + ext4_msg(sb, KERN_ERR, "blocksize: %lu", sb->s_blocksize); + return -EINVAL; } /* * i_atime_extra is the last extra field available for @@ -4535,6 +4462,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) } sb->s_time_min = EXT4_TIMESTAMP_MIN; } + if (sbi->s_inode_size > EXT4_GOOD_OLD_INODE_SIZE) { sbi->s_want_extra_isize = sizeof(struct ext4_inode) - EXT4_GOOD_OLD_INODE_SIZE; @@ -4546,7 +4474,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (v > max) { ext4_msg(sb, KERN_ERR, "bad s_want_extra_isize: %d", v); - goto failed_mount; + return -EINVAL; } if (sbi->s_want_extra_isize < v) sbi->s_want_extra_isize = v; @@ -4555,91 +4483,112 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (v > max) { ext4_msg(sb, KERN_ERR, "bad s_min_extra_isize: %d", v); - goto failed_mount; + return -EINVAL; } if (sbi->s_want_extra_isize < v) sbi->s_want_extra_isize = v; } } - err = parse_apply_sb_mount_options(sb, ctx); - if (err < 0) - goto failed_mount; - - sbi->s_def_mount_opt = sbi->s_mount_opt; - - err = ext4_check_opt_consistency(fc, sb); - if (err < 0) - goto failed_mount; - - ext4_apply_options(fc, sb); + return 0; +} #if IS_ENABLED(CONFIG_UNICODE) - if (ext4_has_feature_casefold(sb) && !sb->s_encoding) { - const struct ext4_sb_encodings *encoding_info; - struct unicode_map *encoding; - __u16 encoding_flags = le16_to_cpu(es->s_encoding_flags); +static int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) +{ + const struct ext4_sb_encodings *encoding_info; + struct unicode_map *encoding; + __u16 encoding_flags = le16_to_cpu(es->s_encoding_flags); - encoding_info = ext4_sb_read_encoding(es); - if (!encoding_info) { - ext4_msg(sb, KERN_ERR, - "Encoding requested by superblock is unknown"); - goto failed_mount; - } + if (!ext4_has_feature_casefold(sb) || sb->s_encoding) + return 0; - encoding = utf8_load(encoding_info->version); - if (IS_ERR(encoding)) { - ext4_msg(sb, KERN_ERR, - "can't mount with superblock charset: %s-%u.%u.%u " - "not supported by the kernel. flags: 0x%x.", - encoding_info->name, - unicode_major(encoding_info->version), - unicode_minor(encoding_info->version), - unicode_rev(encoding_info->version), - encoding_flags); - goto failed_mount; - } - ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " - "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, - unicode_major(encoding_info->version), - unicode_minor(encoding_info->version), - unicode_rev(encoding_info->version), - encoding_flags); + encoding_info = ext4_sb_read_encoding(es); + if (!encoding_info) { + ext4_msg(sb, KERN_ERR, + "Encoding requested by superblock is unknown"); + return -EINVAL; + } - sb->s_encoding = encoding; - sb->s_encoding_flags = encoding_flags; + encoding = utf8_load(encoding_info->version); + if (IS_ERR(encoding)) { + ext4_msg(sb, KERN_ERR, + "can't mount with superblock charset: %s-%u.%u.%u " + "not supported by the kernel. flags: 0x%x.", + encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + return -EINVAL; } + ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " + "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + + sb->s_encoding = encoding; + sb->s_encoding_flags = encoding_flags; + + return 0; +} +#else +static inline int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) +{ + return 0; +} #endif - if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { - printk_once(KERN_WARNING "EXT4-fs: Warning: mounting with data=journal disables delayed allocation, dioread_nolock, O_DIRECT and fast_commit support!\n"); - /* can't mount with both data=journal and dioread_nolock. */ - clear_opt(sb, DIOREAD_NOLOCK); - clear_opt2(sb, JOURNAL_FAST_COMMIT); - if (test_opt2(sb, EXPLICIT_DELALLOC)) { - ext4_msg(sb, KERN_ERR, "can't mount with " - "both data=journal and delalloc"); - goto failed_mount; - } - if (test_opt(sb, DAX_ALWAYS)) { - ext4_msg(sb, KERN_ERR, "can't mount with " - "both data=journal and dax"); - goto failed_mount; - } - if (ext4_has_feature_encrypt(sb)) { - ext4_msg(sb, KERN_WARNING, - "encrypted files will use data=ordered " - "instead of data journaling mode"); - } - if (test_opt(sb, DELALLOC)) - clear_opt(sb, DELALLOC); - } else { - sb->s_iflags |= SB_I_CGROUPWB; +static int ext4_init_metadata_csum(struct super_block *sb, struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* Warn if metadata_csum and gdt_csum are both set. */ + if (ext4_has_feature_metadata_csum(sb) && + ext4_has_feature_gdt_csum(sb)) + ext4_warning(sb, "metadata_csum and uninit_bg are " + "redundant flags; please run fsck."); + + /* Check for a known checksum algorithm */ + if (!ext4_verify_csum_type(sb, es)) { + ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " + "unknown checksum algorithm."); + return -EINVAL; } + ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE, + ext4_orphan_file_block_trigger); - sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | - (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + /* Load the checksum driver */ + sbi->s_chksum_driver = crypto_alloc_shash("crc32c", 0, 0); + if (IS_ERR(sbi->s_chksum_driver)) { + int ret = PTR_ERR(sbi->s_chksum_driver); + ext4_msg(sb, KERN_ERR, "Cannot load crc32c driver."); + sbi->s_chksum_driver = NULL; + return ret; + } + + /* Check superblock checksum */ + if (!ext4_superblock_csum_verify(sb, es)) { + ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " + "invalid superblock checksum. Run e2fsck?"); + return -EFSBADCRC; + } + + /* Precompute checksum seed for all metadata */ + if (ext4_has_feature_csum_seed(sb)) + sbi->s_csum_seed = le32_to_cpu(es->s_checksum_seed); + else if (ext4_has_metadata_csum(sb) || ext4_has_feature_ea_inode(sb)) + sbi->s_csum_seed = ext4_chksum(sbi, ~0, es->s_uuid, + sizeof(es->s_uuid)); + return 0; +} +static int ext4_check_feature_compatibility(struct super_block *sb, + struct ext4_super_block *es, + int silent) +{ if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV && (ext4_has_compat_features(sb) || ext4_has_ro_compat_features(sb) || @@ -4653,7 +4602,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (ext4_has_feature_64bit(sb)) { ext4_msg(sb, KERN_ERR, "The Hurd can't support 64-bit file systems"); - goto failed_mount; + return -EINVAL; } /* @@ -4663,7 +4612,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) if (ext4_has_feature_ea_inode(sb)) { ext4_msg(sb, KERN_ERR, "ea_inode feature is not supported for Hurd"); - goto failed_mount; + return -EINVAL; } } @@ -4677,10 +4626,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) * it's actually an ext[34] filesystem. */ if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) - goto failed_mount; + return -EINVAL; ext4_msg(sb, KERN_ERR, "couldn't mount as ext2 due " "to feature incompatibilities"); - goto failed_mount; + return -EINVAL; } } @@ -4694,10 +4643,10 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) * it's actually an ext4 filesystem. */ if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) - goto failed_mount; + return -EINVAL; ext4_msg(sb, KERN_ERR, "couldn't mount as ext3 due " "to feature incompatibilities"); - goto failed_mount; + return -EINVAL; } } @@ -4707,75 +4656,495 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) * so there is a chance incompat flags are set on a rev 0 filesystem. */ if (!ext4_feature_set_ok(sb, (sb_rdonly(sb)))) - goto failed_mount; + return -EINVAL; - if (le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) > (blocksize / 4)) { - ext4_msg(sb, KERN_ERR, - "Number of reserved GDT blocks insanely large: %d", - le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks)); - goto failed_mount; + return 0; +} + +static int ext4_geometry_check(struct super_block *sb, + struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + __u64 blocks_count; + + /* check blocks count against device size */ + blocks_count = sb_bdev_nr_blocks(sb); + if (blocks_count && ext4_blocks_count(es) > blocks_count) { + ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " + "exceeds size of device (%llu blocks)", + ext4_blocks_count(es), blocks_count); + return -EINVAL; } - if (sbi->s_daxdev) { - if (blocksize == PAGE_SIZE) - set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); - else - ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); + /* + * It makes no sense for the first data block to be beyond the end + * of the filesystem. + */ + if (le32_to_cpu(es->s_first_data_block) >= ext4_blocks_count(es)) { + ext4_msg(sb, KERN_WARNING, "bad geometry: first data " + "block %u is beyond end of filesystem (%llu)", + le32_to_cpu(es->s_first_data_block), + ext4_blocks_count(es)); + return -EINVAL; + } + if ((es->s_first_data_block == 0) && (es->s_log_block_size == 0) && + (sbi->s_cluster_ratio == 1)) { + ext4_msg(sb, KERN_WARNING, "bad geometry: first data " + "block is 0 with a 1k block and cluster size"); + return -EINVAL; } - if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { - if (ext4_has_feature_inline_data(sb)) { - ext4_msg(sb, KERN_ERR, "Cannot use DAX on a filesystem" - " that may contain inline data"); - goto failed_mount; + blocks_count = (ext4_blocks_count(es) - + le32_to_cpu(es->s_first_data_block) + + EXT4_BLOCKS_PER_GROUP(sb) - 1); + do_div(blocks_count, EXT4_BLOCKS_PER_GROUP(sb)); + if (blocks_count > ((uint64_t)1<<32) - EXT4_DESC_PER_BLOCK(sb)) { + ext4_msg(sb, KERN_WARNING, "groups count too large: %llu " + "(block count %llu, first data block %u, " + "blocks per group %lu)", blocks_count, + ext4_blocks_count(es), + le32_to_cpu(es->s_first_data_block), + EXT4_BLOCKS_PER_GROUP(sb)); + return -EINVAL; + } + sbi->s_groups_count = blocks_count; + sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count, + (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); + if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) != + le32_to_cpu(es->s_inodes_count)) { + ext4_msg(sb, KERN_ERR, "inodes count not valid: %u vs %llu", + le32_to_cpu(es->s_inodes_count), + ((u64)sbi->s_groups_count * sbi->s_inodes_per_group)); + return -EINVAL; + } + + return 0; +} + +static void ext4_group_desc_free(struct ext4_sb_info *sbi) +{ + struct buffer_head **group_desc; + int i; + + rcu_read_lock(); + group_desc = rcu_dereference(sbi->s_group_desc); + for (i = 0; i < sbi->s_gdb_count; i++) + brelse(group_desc[i]); + kvfree(group_desc); + rcu_read_unlock(); +} + +static int ext4_group_desc_init(struct super_block *sb, + struct ext4_super_block *es, + ext4_fsblk_t logical_sb_block, + ext4_group_t *first_not_zeroed) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned int db_count; + ext4_fsblk_t block; + int ret; + int i; + + db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / + EXT4_DESC_PER_BLOCK(sb); + if (ext4_has_feature_meta_bg(sb)) { + if (le32_to_cpu(es->s_first_meta_bg) > db_count) { + ext4_msg(sb, KERN_WARNING, + "first meta block group too large: %u " + "(group descriptor block count %u)", + le32_to_cpu(es->s_first_meta_bg), db_count); + return -EINVAL; } - if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) { + } + rcu_assign_pointer(sbi->s_group_desc, + kvmalloc_array(db_count, + sizeof(struct buffer_head *), + GFP_KERNEL)); + if (sbi->s_group_desc == NULL) { + ext4_msg(sb, KERN_ERR, "not enough memory"); + return -ENOMEM; + } + + bgl_lock_init(sbi->s_blockgroup_lock); + + /* Pre-read the descriptors into the buffer cache */ + for (i = 0; i < db_count; i++) { + block = descriptor_loc(sb, logical_sb_block, i); + ext4_sb_breadahead_unmovable(sb, block); + } + + for (i = 0; i < db_count; i++) { + struct buffer_head *bh; + + block = descriptor_loc(sb, logical_sb_block, i); + bh = ext4_sb_bread_unmovable(sb, block); + if (IS_ERR(bh)) { ext4_msg(sb, KERN_ERR, - "DAX unsupported by block device."); - goto failed_mount; + "can't read group descriptor %d", i); + sbi->s_gdb_count = i; + ret = PTR_ERR(bh); + goto out; } + rcu_read_lock(); + rcu_dereference(sbi->s_group_desc)[i] = bh; + rcu_read_unlock(); + } + sbi->s_gdb_count = db_count; + if (!ext4_check_descriptors(sb, logical_sb_block, first_not_zeroed)) { + ext4_msg(sb, KERN_ERR, "group descriptors corrupted!"); + ret = -EFSCORRUPTED; + goto out; } + return 0; +out: + ext4_group_desc_free(sbi); + return ret; +} - if (ext4_has_feature_encrypt(sb) && es->s_encryption_level) { - ext4_msg(sb, KERN_ERR, "Unsupported encryption level %d", - es->s_encryption_level); - goto failed_mount; +static int ext4_load_and_init_journal(struct super_block *sb, + struct ext4_super_block *es, + struct ext4_fs_context *ctx) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err; + + err = ext4_load_journal(sb, es, ctx->journal_devnum); + if (err) + return err; + + if (ext4_has_feature_64bit(sb) && + !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_64BIT)) { + ext4_msg(sb, KERN_ERR, "Failed to set 64-bit journal feature"); + goto out; } - if (sb->s_blocksize != blocksize) { - /* - * bh must be released before kill_bdev(), otherwise - * it won't be freed and its page also. kill_bdev() - * is called by sb_set_blocksize(). + if (!set_journal_csum_feature_set(sb)) { + ext4_msg(sb, KERN_ERR, "Failed to set journal checksum " + "feature set"); + goto out; + } + + if (test_opt2(sb, JOURNAL_FAST_COMMIT) && + !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_FAST_COMMIT)) { + ext4_msg(sb, KERN_ERR, + "Failed to set fast commit journal feature"); + goto out; + } + + /* We have now updated the journal if required, so we can + * validate the data journaling mode. */ + switch (test_opt(sb, DATA_FLAGS)) { + case 0: + /* No mode set, assume a default based on the journal + * capabilities: ORDERED_DATA if the journal can + * cope, else JOURNAL_DATA */ - brelse(bh); - /* Validate the filesystem blocksize */ - if (!sb_set_blocksize(sb, blocksize)) { - ext4_msg(sb, KERN_ERR, "bad block size %d", - blocksize); - bh = NULL; - goto failed_mount; + if (jbd2_journal_check_available_features + (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { + set_opt(sb, ORDERED_DATA); + sbi->s_def_mount_opt |= EXT4_MOUNT_ORDERED_DATA; + } else { + set_opt(sb, JOURNAL_DATA); + sbi->s_def_mount_opt |= EXT4_MOUNT_JOURNAL_DATA; + } + break; + + case EXT4_MOUNT_ORDERED_DATA: + case EXT4_MOUNT_WRITEBACK_DATA: + if (!jbd2_journal_check_available_features + (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { + ext4_msg(sb, KERN_ERR, "Journal does not support " + "requested data journaling mode"); + goto out; } + break; + default: + break; + } + + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && + test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit in data=ordered mode"); + goto out; + } + set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); + + sbi->s_journal->j_submit_inode_data_buffers = + ext4_journal_submit_inode_data_buffers; + sbi->s_journal->j_finish_inode_data_buffers = + ext4_journal_finish_inode_data_buffers; + + return 0; + +out: + /* flush s_error_work before journal destroy. */ + flush_work(&sbi->s_error_work); + jbd2_journal_destroy(sbi->s_journal); + sbi->s_journal = NULL; + return err; +} + +static int ext4_journal_data_mode_check(struct super_block *sb) +{ + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { + printk_once(KERN_WARNING "EXT4-fs: Warning: mounting with " + "data=journal disables delayed allocation, " + "dioread_nolock, O_DIRECT and fast_commit support!\n"); + /* can't mount with both data=journal and dioread_nolock. */ + clear_opt(sb, DIOREAD_NOLOCK); + clear_opt2(sb, JOURNAL_FAST_COMMIT); + if (test_opt2(sb, EXPLICIT_DELALLOC)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and delalloc"); + return -EINVAL; + } + if (test_opt(sb, DAX_ALWAYS)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and dax"); + return -EINVAL; + } + if (ext4_has_feature_encrypt(sb)) { + ext4_msg(sb, KERN_WARNING, + "encrypted files will use data=ordered " + "instead of data journaling mode"); + } + if (test_opt(sb, DELALLOC)) + clear_opt(sb, DELALLOC); + } else { + sb->s_iflags |= SB_I_CGROUPWB; + } + + return 0; +} + +static int ext4_load_super(struct super_block *sb, ext4_fsblk_t *lsb, + int silent) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es; + ext4_fsblk_t logical_sb_block; + unsigned long offset = 0; + struct buffer_head *bh; + int ret = -EINVAL; + int blocksize; + + blocksize = sb_min_blocksize(sb, EXT4_MIN_BLOCK_SIZE); + if (!blocksize) { + ext4_msg(sb, KERN_ERR, "unable to set blocksize"); + return -EINVAL; + } + + /* + * The ext4 superblock will not be buffer aligned for other than 1kB + * block sizes. We need to calculate the offset from buffer start. + */ + if (blocksize != EXT4_MIN_BLOCK_SIZE) { logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; offset = do_div(logical_sb_block, blocksize); - bh = ext4_sb_bread_unmovable(sb, logical_sb_block); - if (IS_ERR(bh)) { - ext4_msg(sb, KERN_ERR, - "Can't read superblock on 2nd try"); - ret = PTR_ERR(bh); - bh = NULL; + } else { + logical_sb_block = sbi->s_sb_block; + } + + bh = ext4_sb_bread_unmovable(sb, logical_sb_block); + if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, "unable to read superblock"); + return PTR_ERR(bh); + } + /* + * Note: s_es must be initialized as soon as possible because + * some ext4 macro-instructions depend on its value + */ + es = (struct ext4_super_block *) (bh->b_data + offset); + sbi->s_es = es; + sb->s_magic = le16_to_cpu(es->s_magic); + if (sb->s_magic != EXT4_SUPER_MAGIC) { + if (!silent) + ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); + goto out; + } + + if (le32_to_cpu(es->s_log_block_size) > + (EXT4_MAX_BLOCK_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { + ext4_msg(sb, KERN_ERR, + "Invalid log block size: %u", + le32_to_cpu(es->s_log_block_size)); + goto out; + } + if (le32_to_cpu(es->s_log_cluster_size) > + (EXT4_MAX_CLUSTER_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { + ext4_msg(sb, KERN_ERR, + "Invalid log cluster size: %u", + le32_to_cpu(es->s_log_cluster_size)); + goto out; + } + + blocksize = EXT4_MIN_BLOCK_SIZE << le32_to_cpu(es->s_log_block_size); + + /* + * If the default block size is not the same as the real block size, + * we need to reload it. + */ + if (sb->s_blocksize == blocksize) { + *lsb = logical_sb_block; + sbi->s_sbh = bh; + return 0; + } + + /* + * bh must be released before kill_bdev(), otherwise + * it won't be freed and its page also. kill_bdev() + * is called by sb_set_blocksize(). + */ + brelse(bh); + /* Validate the filesystem blocksize */ + if (!sb_set_blocksize(sb, blocksize)) { + ext4_msg(sb, KERN_ERR, "bad block size %d", + blocksize); + bh = NULL; + goto out; + } + + logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; + offset = do_div(logical_sb_block, blocksize); + bh = ext4_sb_bread_unmovable(sb, logical_sb_block); + if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, "Can't read superblock on 2nd try"); + ret = PTR_ERR(bh); + bh = NULL; + goto out; + } + es = (struct ext4_super_block *)(bh->b_data + offset); + sbi->s_es = es; + if (es->s_magic != cpu_to_le16(EXT4_SUPER_MAGIC)) { + ext4_msg(sb, KERN_ERR, "Magic mismatch, very weird!"); + goto out; + } + *lsb = logical_sb_block; + sbi->s_sbh = bh; + return 0; +out: + brelse(bh); + return ret; +} + +static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) +{ + struct ext4_super_block *es = NULL; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct flex_groups **flex_groups; + ext4_fsblk_t block; + ext4_fsblk_t logical_sb_block; + struct inode *root; + int ret = -ENOMEM; + unsigned int i; + int needs_recovery, has_huge_files; + int err = 0; + ext4_group_t first_not_zeroed; + struct ext4_fs_context *ctx = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + + /* Set defaults for the variables that will be set during parsing */ + if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) + ctx->journal_ioprio = DEFAULT_JOURNAL_IOPRIO; + + sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; + sbi->s_sectors_written_start = + part_stat_read(sb->s_bdev, sectors[STAT_WRITE]); + + /* -EINVAL is default */ + ret = -EINVAL; + err = ext4_load_super(sb, &logical_sb_block, silent); + if (err) + goto out_fail; + + es = sbi->s_es; + sbi->s_kbytes_written = le64_to_cpu(es->s_kbytes_written); + + err = ext4_init_metadata_csum(sb, es); + if (err) + goto failed_mount; + + ext4_set_def_opts(sb, es); + + sbi->s_resuid = make_kuid(&init_user_ns, le16_to_cpu(es->s_def_resuid)); + sbi->s_resgid = make_kgid(&init_user_ns, le16_to_cpu(es->s_def_resgid)); + sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; + sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; + sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; + + /* + * set default s_li_wait_mult for lazyinit, for the case there is + * no mount option specified. + */ + sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; + + if (ext4_inode_info_init(sb, es)) + goto failed_mount; + + err = parse_apply_sb_mount_options(sb, ctx); + if (err < 0) + goto failed_mount; + + sbi->s_def_mount_opt = sbi->s_mount_opt; + + err = ext4_check_opt_consistency(fc, sb); + if (err < 0) + goto failed_mount; + + ext4_apply_options(fc, sb); + + if (ext4_encoding_init(sb, es)) + goto failed_mount; + + if (ext4_journal_data_mode_check(sb)) + goto failed_mount; + + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + + /* i_version is always enabled now */ + sb->s_flags |= SB_I_VERSION; + + if (ext4_check_feature_compatibility(sb, es, silent)) + goto failed_mount; + + if (le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) > (sb->s_blocksize / 4)) { + ext4_msg(sb, KERN_ERR, + "Number of reserved GDT blocks insanely large: %d", + le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks)); + goto failed_mount; + } + + if (sbi->s_daxdev) { + if (sb->s_blocksize == PAGE_SIZE) + set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); + else + ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); + } + + if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { + if (ext4_has_feature_inline_data(sb)) { + ext4_msg(sb, KERN_ERR, "Cannot use DAX on a filesystem" + " that may contain inline data"); goto failed_mount; } - es = (struct ext4_super_block *)(bh->b_data + offset); - sbi->s_es = es; - if (es->s_magic != cpu_to_le16(EXT4_SUPER_MAGIC)) { + if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) { ext4_msg(sb, KERN_ERR, - "Magic mismatch, very weird!"); + "DAX unsupported by block device."); goto failed_mount; } } + if (ext4_has_feature_encrypt(sb) && es->s_encryption_level) { + ext4_msg(sb, KERN_ERR, "Unsupported encryption level %d", + es->s_encryption_level); + goto failed_mount; + } + has_huge_files = ext4_has_feature_huge_file(sb); sbi->s_bitmap_maxbytes = ext4_max_bitmap_size(sb->s_blocksize_bits, has_huge_files); @@ -4797,19 +5166,21 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); - sbi->s_inodes_per_block = blocksize / EXT4_INODE_SIZE(sb); - if (sbi->s_inodes_per_block == 0) - goto cantfind_ext4; + sbi->s_inodes_per_block = sb->s_blocksize / EXT4_INODE_SIZE(sb); + if (sbi->s_inodes_per_block == 0 || sbi->s_blocks_per_group == 0) { + if (!silent) + ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); + goto failed_mount; + } if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || - sbi->s_inodes_per_group > blocksize * 8) { + sbi->s_inodes_per_group > sb->s_blocksize * 8) { ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu\n", sbi->s_inodes_per_group); goto failed_mount; } sbi->s_itb_per_group = sbi->s_inodes_per_group / sbi->s_inodes_per_block; - sbi->s_desc_per_block = blocksize / EXT4_DESC_SIZE(sb); - sbi->s_sbh = bh; + sbi->s_desc_per_block = sb->s_blocksize / EXT4_DESC_SIZE(sb); sbi->s_mount_state = le16_to_cpu(es->s_state) & ~EXT4_FC_REPLAY; sbi->s_addr_per_block_bits = ilog2(EXT4_ADDR_PER_BLOCK(sb)); sbi->s_desc_per_block_bits = ilog2(EXT4_DESC_PER_BLOCK(sb)); @@ -4835,54 +5206,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) } } - /* Handle clustersize */ - clustersize = BLOCK_SIZE << le32_to_cpu(es->s_log_cluster_size); - if (ext4_has_feature_bigalloc(sb)) { - if (clustersize < blocksize) { - ext4_msg(sb, KERN_ERR, - "cluster size (%d) smaller than " - "block size (%d)", clustersize, blocksize); - goto failed_mount; - } - sbi->s_cluster_bits = le32_to_cpu(es->s_log_cluster_size) - - le32_to_cpu(es->s_log_block_size); - sbi->s_clusters_per_group = - le32_to_cpu(es->s_clusters_per_group); - if (sbi->s_clusters_per_group > blocksize * 8) { - ext4_msg(sb, KERN_ERR, - "#clusters per group too big: %lu", - sbi->s_clusters_per_group); - goto failed_mount; - } - if (sbi->s_blocks_per_group != - (sbi->s_clusters_per_group * (clustersize / blocksize))) { - ext4_msg(sb, KERN_ERR, "blocks per group (%lu) and " - "clusters per group (%lu) inconsistent", - sbi->s_blocks_per_group, - sbi->s_clusters_per_group); - goto failed_mount; - } - } else { - if (clustersize != blocksize) { - ext4_msg(sb, KERN_ERR, - "fragment/cluster size (%d) != " - "block size (%d)", clustersize, blocksize); - goto failed_mount; - } - if (sbi->s_blocks_per_group > blocksize * 8) { - ext4_msg(sb, KERN_ERR, - "#blocks per group too big: %lu", - sbi->s_blocks_per_group); - goto failed_mount; - } - sbi->s_clusters_per_group = sbi->s_blocks_per_group; - sbi->s_cluster_bits = 0; - } - sbi->s_cluster_ratio = clustersize / blocksize; - - /* Do we have standard group size of clustersize * 8 blocks ? */ - if (sbi->s_blocks_per_group == clustersize << 3) - set_opt2(sb, STD_GROUP_SIZE); + if (ext4_handle_clustersize(sb)) + goto failed_mount; /* * Test whether we have more sectors than will fit in sector_t, @@ -4896,111 +5221,12 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) goto failed_mount; } - if (EXT4_BLOCKS_PER_GROUP(sb) == 0) - goto cantfind_ext4; - - /* check blocks count against device size */ - blocks_count = sb_bdev_nr_blocks(sb); - if (blocks_count && ext4_blocks_count(es) > blocks_count) { - ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " - "exceeds size of device (%llu blocks)", - ext4_blocks_count(es), blocks_count); - goto failed_mount; - } - - /* - * It makes no sense for the first data block to be beyond the end - * of the filesystem. - */ - if (le32_to_cpu(es->s_first_data_block) >= ext4_blocks_count(es)) { - ext4_msg(sb, KERN_WARNING, "bad geometry: first data " - "block %u is beyond end of filesystem (%llu)", - le32_to_cpu(es->s_first_data_block), - ext4_blocks_count(es)); - goto failed_mount; - } - if ((es->s_first_data_block == 0) && (es->s_log_block_size == 0) && - (sbi->s_cluster_ratio == 1)) { - ext4_msg(sb, KERN_WARNING, "bad geometry: first data " - "block is 0 with a 1k block and cluster size"); + if (ext4_geometry_check(sb, es)) goto failed_mount; - } - blocks_count = (ext4_blocks_count(es) - - le32_to_cpu(es->s_first_data_block) + - EXT4_BLOCKS_PER_GROUP(sb) - 1); - do_div(blocks_count, EXT4_BLOCKS_PER_GROUP(sb)); - if (blocks_count > ((uint64_t)1<<32) - EXT4_DESC_PER_BLOCK(sb)) { - ext4_msg(sb, KERN_WARNING, "groups count too large: %llu " - "(block count %llu, first data block %u, " - "blocks per group %lu)", blocks_count, - ext4_blocks_count(es), - le32_to_cpu(es->s_first_data_block), - EXT4_BLOCKS_PER_GROUP(sb)); - goto failed_mount; - } - sbi->s_groups_count = blocks_count; - sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count, - (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); - if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) != - le32_to_cpu(es->s_inodes_count)) { - ext4_msg(sb, KERN_ERR, "inodes count not valid: %u vs %llu", - le32_to_cpu(es->s_inodes_count), - ((u64)sbi->s_groups_count * sbi->s_inodes_per_group)); - ret = -EINVAL; - goto failed_mount; - } - db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / - EXT4_DESC_PER_BLOCK(sb); - if (ext4_has_feature_meta_bg(sb)) { - if (le32_to_cpu(es->s_first_meta_bg) > db_count) { - ext4_msg(sb, KERN_WARNING, - "first meta block group too large: %u " - "(group descriptor block count %u)", - le32_to_cpu(es->s_first_meta_bg), db_count); - goto failed_mount; - } - } - rcu_assign_pointer(sbi->s_group_desc, - kvmalloc_array(db_count, - sizeof(struct buffer_head *), - GFP_KERNEL)); - if (sbi->s_group_desc == NULL) { - ext4_msg(sb, KERN_ERR, "not enough memory"); - ret = -ENOMEM; + err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); + if (err) goto failed_mount; - } - - bgl_lock_init(sbi->s_blockgroup_lock); - - /* Pre-read the descriptors into the buffer cache */ - for (i = 0; i < db_count; i++) { - block = descriptor_loc(sb, logical_sb_block, i); - ext4_sb_breadahead_unmovable(sb, block); - } - - for (i = 0; i < db_count; i++) { - struct buffer_head *bh; - - block = descriptor_loc(sb, logical_sb_block, i); - bh = ext4_sb_bread_unmovable(sb, block); - if (IS_ERR(bh)) { - ext4_msg(sb, KERN_ERR, - "can't read group descriptor %d", i); - db_count = i; - ret = PTR_ERR(bh); - goto failed_mount2; - } - rcu_read_lock(); - rcu_dereference(sbi->s_group_desc)[i] = bh; - rcu_read_unlock(); - } - sbi->s_gdb_count = db_count; - if (!ext4_check_descriptors(sb, logical_sb_block, &first_not_zeroed)) { - ext4_msg(sb, KERN_ERR, "group descriptors corrupted!"); - ret = -EFSCORRUPTED; - goto failed_mount2; - } timer_setup(&sbi->s_err_report, print_daily_error_info, 0); spin_lock_init(&sbi->s_error_lock); @@ -5038,24 +5264,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) INIT_LIST_HEAD(&sbi->s_orphan); /* unlinked but open files */ mutex_init(&sbi->s_orphan_lock); - /* Initialize fast commit stuff */ - atomic_set(&sbi->s_fc_subtid, 0); - INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]); - INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]); - INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]); - INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]); - sbi->s_fc_bytes = 0; - ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); - sbi->s_fc_ineligible_tid = 0; - spin_lock_init(&sbi->s_fc_lock); - memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); - sbi->s_fc_replay_state.fc_regions = NULL; - sbi->s_fc_replay_state.fc_regions_size = 0; - sbi->s_fc_replay_state.fc_regions_used = 0; - sbi->s_fc_replay_state.fc_regions_valid = 0; - sbi->s_fc_replay_state.fc_modified_inodes = NULL; - sbi->s_fc_replay_state.fc_modified_inodes_size = 0; - sbi->s_fc_replay_state.fc_modified_inodes_used = 0; + ext4_fast_commit_init(sb); sb->s_root = NULL; @@ -5072,37 +5281,37 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) * root first: it may be modified in the journal! */ if (!test_opt(sb, NOLOAD) && ext4_has_feature_journal(sb)) { - err = ext4_load_journal(sb, es, ctx->journal_devnum); + err = ext4_load_and_init_journal(sb, es, ctx); if (err) goto failed_mount3a; } else if (test_opt(sb, NOLOAD) && !sb_rdonly(sb) && ext4_has_feature_journal_needs_recovery(sb)) { ext4_msg(sb, KERN_ERR, "required journal recovery " "suppressed and not mounted read-only"); - goto failed_mount_wq; + goto failed_mount3a; } else { /* Nojournal mode, all journal mount options are illegal */ if (test_opt2(sb, EXPLICIT_JOURNAL_CHECKSUM)) { ext4_msg(sb, KERN_ERR, "can't mount with " "journal_checksum, fs mounted w/o journal"); - goto failed_mount_wq; + goto failed_mount3a; } if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) { ext4_msg(sb, KERN_ERR, "can't mount with " "journal_async_commit, fs mounted w/o journal"); - goto failed_mount_wq; + goto failed_mount3a; } if (sbi->s_commit_interval != JBD2_DEFAULT_MAX_COMMIT_AGE*HZ) { ext4_msg(sb, KERN_ERR, "can't mount with " "commit=%lu, fs mounted w/o journal", sbi->s_commit_interval / HZ); - goto failed_mount_wq; + goto failed_mount3a; } if (EXT4_MOUNT_DATA_FLAGS & (sbi->s_mount_opt ^ sbi->s_def_mount_opt)) { ext4_msg(sb, KERN_ERR, "can't mount with " "data=, fs mounted w/o journal"); - goto failed_mount_wq; + goto failed_mount3a; } sbi->s_def_mount_opt &= ~EXT4_MOUNT_JOURNAL_CHECKSUM; clear_opt(sb, JOURNAL_CHECKSUM); @@ -5110,76 +5319,8 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) clear_opt2(sb, JOURNAL_FAST_COMMIT); sbi->s_journal = NULL; needs_recovery = 0; - goto no_journal; - } - - if (ext4_has_feature_64bit(sb) && - !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, - JBD2_FEATURE_INCOMPAT_64BIT)) { - ext4_msg(sb, KERN_ERR, "Failed to set 64-bit journal feature"); - goto failed_mount_wq; - } - - if (!set_journal_csum_feature_set(sb)) { - ext4_msg(sb, KERN_ERR, "Failed to set journal checksum " - "feature set"); - goto failed_mount_wq; - } - - if (test_opt2(sb, JOURNAL_FAST_COMMIT) && - !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, - JBD2_FEATURE_INCOMPAT_FAST_COMMIT)) { - ext4_msg(sb, KERN_ERR, - "Failed to set fast commit journal feature"); - goto failed_mount_wq; - } - - /* We have now updated the journal if required, so we can - * validate the data journaling mode. */ - switch (test_opt(sb, DATA_FLAGS)) { - case 0: - /* No mode set, assume a default based on the journal - * capabilities: ORDERED_DATA if the journal can - * cope, else JOURNAL_DATA - */ - if (jbd2_journal_check_available_features - (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { - set_opt(sb, ORDERED_DATA); - sbi->s_def_mount_opt |= EXT4_MOUNT_ORDERED_DATA; - } else { - set_opt(sb, JOURNAL_DATA); - sbi->s_def_mount_opt |= EXT4_MOUNT_JOURNAL_DATA; - } - break; - - case EXT4_MOUNT_ORDERED_DATA: - case EXT4_MOUNT_WRITEBACK_DATA: - if (!jbd2_journal_check_available_features - (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { - ext4_msg(sb, KERN_ERR, "Journal does not support " - "requested data journaling mode"); - goto failed_mount_wq; - } - break; - default: - break; - } - - if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && - test_opt(sb, JOURNAL_ASYNC_COMMIT)) { - ext4_msg(sb, KERN_ERR, "can't mount with " - "journal_async_commit in data=ordered mode"); - goto failed_mount_wq; } - set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); - - sbi->s_journal->j_submit_inode_data_buffers = - ext4_journal_submit_inode_data_buffers; - sbi->s_journal->j_finish_inode_data_buffers = - ext4_journal_finish_inode_data_buffers; - -no_journal: if (!test_opt(sb, NO_MBCACHE)) { sbi->s_ea_block_cache = ext4_xattr_create_cache(); if (!sbi->s_ea_block_cache) { @@ -5198,7 +5339,7 @@ no_journal: } } - if (ext4_has_feature_verity(sb) && blocksize != PAGE_SIZE) { + if (ext4_has_feature_verity(sb) && sb->s_blocksize != PAGE_SIZE) { ext4_msg(sb, KERN_ERR, "Unsupported blocksize for fs-verity"); goto failed_mount_wq; } @@ -5408,11 +5549,6 @@ no_journal: return 0; -cantfind_ext4: - if (!silent) - ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); - goto failed_mount; - failed_mount9: ext4_release_orphan_info(sb); failed_mount8: @@ -5466,13 +5602,7 @@ failed_mount3: flush_work(&sbi->s_error_work); del_timer_sync(&sbi->s_err_report); ext4_stop_mmpd(sbi); -failed_mount2: - rcu_read_lock(); - group_desc = rcu_dereference(sbi->s_group_desc); - for (i = 0; i < db_count; i++) - brelse(group_desc[i]); - kvfree(group_desc); - rcu_read_unlock(); + ext4_group_desc_free(sbi); failed_mount: if (sbi->s_chksum_driver) crypto_free_shash(sbi->s_chksum_driver); @@ -5487,7 +5617,7 @@ failed_mount: #endif fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy); /* ext4_blkdev_remove() calls kill_bdev(), release bh before it. */ - brelse(bh); + brelse(sbi->s_sbh); ext4_blkdev_remove(sbi); out_fail: sb->s_fs_info = NULL; @@ -6653,7 +6783,7 @@ static int ext4_write_info(struct super_block *sb, int type) handle_t *handle; /* Data block + inode block */ - handle = ext4_journal_start(d_inode(sb->s_root), EXT4_HT_QUOTA, 2); + handle = ext4_journal_start_sb(sb, EXT4_HT_QUOTA, 2); if (IS_ERR(handle)) return PTR_ERR(handle); ret = dquot_commit_info(sb, type); diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c index b051d19b5c8a04cf313b440782c396f482140011..3c640bd7ecaeb4e80d9e16e06cd08922f1d14a3d 100644 --- a/fs/ext4/verity.c +++ b/fs/ext4/verity.c @@ -298,16 +298,14 @@ static int ext4_get_verity_descriptor_location(struct inode *inode, last_extent = path[path->p_depth].p_ext; if (!last_extent) { EXT4_ERROR_INODE(inode, "verity file has no extents"); - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); return -EFSCORRUPTED; } end_lblk = le32_to_cpu(last_extent->ee_block) + ext4_ext_get_actual_len(last_extent); desc_size_pos = (u64)end_lblk << inode->i_blkbits; - ext4_ext_drop_refs(path); - kfree(path); + ext4_free_ext_path(path); if (desc_size_pos < sizeof(desc_size_disk)) goto bad; @@ -365,13 +363,14 @@ static struct page *ext4_read_merkle_tree_page(struct inode *inode, pgoff_t index, unsigned long num_ra_pages) { - DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); struct page *page; index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED); if (!page || !PageUptodate(page)) { + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); + if (page) put_page(page); else if (num_ra_pages > 1) diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 533216e80fa2b023a5aeb5060961706a5b23b4d7..36d6ba7190b6db6d650ad2ac85c6313aae2b3ae1 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -2412,6 +2412,7 @@ retry_inode: if (!error) { ext4_xattr_update_super_block(handle, inode->i_sb); inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); if (!value) no_expand = 0; error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c index eaa240b21f0715589fb1cbce935449e09e8717ff..5bbc44a5216e670bbea72e113efdf52ef7997416 100644 --- a/fs/f2fs/acl.c +++ b/fs/f2fs/acl.c @@ -219,7 +219,7 @@ static int f2fs_acl_update_mode(struct user_namespace *mnt_userns, return error; if (error == 0) *acl = NULL; - if (!in_group_p(i_gid_into_mnt(mnt_userns, inode)) && + if (!vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode)) && !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) mode &= ~S_ISGID; *mode_p = mode; diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index 8259e0fa97e1fcda7dd6e8ff10044a2ca50ab2f4..0c82dae082aa9e4ee3e37b758cb046fc43766110 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -26,12 +26,16 @@ static struct kmem_cache *ino_entry_slab; struct kmem_cache *f2fs_inode_entry_slab; -void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io) +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason) { f2fs_build_fault_attr(sbi, 0, 0); set_ckpt_flags(sbi, CP_ERROR_FLAG); - if (!end_io) + if (!end_io) { f2fs_flush_merged_writes(sbi); + + f2fs_handle_stop(sbi, reason); + } } /* @@ -89,7 +93,7 @@ repeat: return ERR_PTR(err); } - f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE); lock_page(page); if (unlikely(page->mapping != mapping)) { @@ -122,7 +126,7 @@ retry: if (PTR_ERR(page) == -EIO && ++count <= DEFAULT_RETRY_IO_COUNT) goto retry; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE); } return page; } @@ -140,7 +144,7 @@ static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr, unsigned int segno, offset; bool exist; - if (type != DATA_GENERIC_ENHANCE && type != DATA_GENERIC_ENHANCE_READ) + if (type == DATA_GENERIC) return true; segno = GET_SEGNO(sbi, blkaddr); @@ -148,6 +152,13 @@ static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr, se = get_seg_entry(sbi, segno); exist = f2fs_test_bit(offset, se->cur_valid_map); + if (exist && type == DATA_GENERIC_ENHANCE_UPDATE) { + f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d", + blkaddr, exist); + set_sbi_flag(sbi, SBI_NEED_FSCK); + return exist; + } + if (!exist && type == DATA_GENERIC_ENHANCE) { f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d", blkaddr, exist); @@ -185,6 +196,7 @@ bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, case DATA_GENERIC: case DATA_GENERIC_ENHANCE: case DATA_GENERIC_ENHANCE_READ: + case DATA_GENERIC_ENHANCE_UPDATE: if (unlikely(blkaddr >= MAX_BLKADDR(sbi) || blkaddr < MAIN_BLKADDR(sbi))) { f2fs_warn(sbi, "access invalid blkaddr:%u", @@ -276,7 +288,8 @@ int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, f2fs_put_page(page, err ? 1 : 0); if (!err) - f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, + F2FS_BLKSIZE); } out: blk_finish_plug(&plug); @@ -448,8 +461,7 @@ static bool f2fs_dirty_meta_folio(struct address_space *mapping, if (!folio_test_uptodate(folio)) folio_mark_uptodate(folio); - if (!folio_test_dirty(folio)) { - filemap_dirty_folio(mapping, folio); + if (filemap_dirty_folio(mapping, folio)) { inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META); set_page_private_reference(&folio->page); return true; @@ -1053,7 +1065,8 @@ void f2fs_remove_dirty_inode(struct inode *inode) spin_unlock(&sbi->inode_lock[type]); } -int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type) +int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, + bool from_cp) { struct list_head *head; struct inode *inode; @@ -1088,11 +1101,15 @@ retry: if (inode) { unsigned long cur_ino = inode->i_ino; - F2FS_I(inode)->cp_task = current; + if (from_cp) + F2FS_I(inode)->cp_task = current; + F2FS_I(inode)->wb_task = current; filemap_fdatawrite(inode->i_mapping); - F2FS_I(inode)->cp_task = NULL; + F2FS_I(inode)->wb_task = NULL; + if (from_cp) + F2FS_I(inode)->cp_task = NULL; iput(inode); /* We need to give cpu to another writers. */ @@ -1221,7 +1238,7 @@ retry_flush_dents: /* write all the dirty dentry pages */ if (get_pages(sbi, F2FS_DIRTY_DENTS)) { f2fs_unlock_all(sbi); - err = f2fs_sync_dirty_inodes(sbi, DIR_INODE); + err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true); if (err) return err; cond_resched(); @@ -1892,15 +1909,27 @@ int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi) void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi) { struct ckpt_req_control *cprc = &sbi->cprc_info; + struct task_struct *ckpt_task; + + if (!cprc->f2fs_issue_ckpt) + return; - if (cprc->f2fs_issue_ckpt) { - struct task_struct *ckpt_task = cprc->f2fs_issue_ckpt; + ckpt_task = cprc->f2fs_issue_ckpt; + cprc->f2fs_issue_ckpt = NULL; + kthread_stop(ckpt_task); - cprc->f2fs_issue_ckpt = NULL; - kthread_stop(ckpt_task); + f2fs_flush_ckpt_thread(sbi); +} - flush_remained_ckpt_reqs(sbi, NULL); - } +void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + flush_remained_ckpt_reqs(sbi, NULL); + + /* Let's wait for the previous dispatched checkpoint. */ + while (atomic_read(&cprc->queued_ckpt)) + io_schedule_timeout(DEFAULT_IO_TIMEOUT); } void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi) diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 70e97075e535e53b7710ffe3a3c27116e2fc82c7..d315c2de136f26f544a5518f49cdade81e693898 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -762,6 +762,7 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { ret = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); goto out_release; } @@ -912,17 +913,15 @@ bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) reason = "[C|*|C|*]"; goto out; } - if (compressed) { - if (!__is_valid_data_blkaddr(blkaddr)) { - if (!cluster_end) - cluster_end = i; - continue; - } - /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ - if (cluster_end) { - reason = "[C|N|N|V]"; - goto out; - } + if (!__is_valid_data_blkaddr(blkaddr)) { + if (!cluster_end) + cluster_end = i; + continue; + } + /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ + if (cluster_end) { + reason = "[C|N|N|V]"; + goto out; } } return false; @@ -952,6 +951,7 @@ static int __f2fs_cluster_blocks(struct inode *inode, if (f2fs_sanity_check_cluster(&dn)) { ret = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), ERROR_CORRUPTED_CLUSTER); goto fail; } @@ -1568,12 +1568,8 @@ static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, if (!dic->cbuf) return -ENOMEM; - if (cops->init_decompress_ctx) { - int ret = cops->init_decompress_ctx(dic); - - if (ret) - return ret; - } + if (cops->init_decompress_ctx) + return cops->init_decompress_ctx(dic); return 0; } @@ -1905,7 +1901,7 @@ bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page, void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) { - struct address_space *mapping = sbi->compress_inode->i_mapping; + struct address_space *mapping = COMPRESS_MAPPING(sbi); struct folio_batch fbatch; pgoff_t index = 0; pgoff_t end = MAX_BLKADDR(sbi); diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index aa3ccddfa037612cf0c63721e1e5f975e5598947..a71e818cd67b43eec47c768bc772c93652646479 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -139,7 +139,7 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) continue; } - /* PG_error was set if decryption or verity failed. */ + /* PG_error was set if verity failed. */ if (bio->bi_status || PageError(page)) { ClearPageUptodate(page); /* will re-read again later */ @@ -185,7 +185,7 @@ static void f2fs_verify_bio(struct work_struct *work) struct page *page = bv->bv_page; if (!f2fs_is_compressed_page(page) && - !PageError(page) && !fsverity_verify_page(page)) + !fsverity_verify_page(page)) SetPageError(page); } } else { @@ -236,10 +236,9 @@ static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, bio_for_each_segment_all(bv, ctx->bio, iter_all) { struct page *page = bv->bv_page; - /* PG_error was set if decryption failed. */ if (f2fs_is_compressed_page(page)) - f2fs_end_read_compressed_page(page, PageError(page), - blkaddr, in_task); + f2fs_end_read_compressed_page(page, false, blkaddr, + in_task); else all_compressed = false; @@ -259,14 +258,17 @@ static void f2fs_post_read_work(struct work_struct *work) { struct bio_post_read_ctx *ctx = container_of(work, struct bio_post_read_ctx, work); + struct bio *bio = ctx->bio; - if (ctx->enabled_steps & STEP_DECRYPT) - fscrypt_decrypt_bio(ctx->bio); + if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { + f2fs_finish_read_bio(bio, true); + return; + } if (ctx->enabled_steps & STEP_DECOMPRESS) f2fs_handle_step_decompress(ctx, true); - f2fs_verify_and_finish_bio(ctx->bio, true); + f2fs_verify_and_finish_bio(bio, true); } static void f2fs_read_end_io(struct bio *bio) @@ -333,7 +335,8 @@ static void f2fs_write_end_io(struct bio *bio) mempool_free(page, sbi->write_io_dummy); if (unlikely(bio->bi_status)) - f2fs_stop_checkpoint(sbi, true); + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); continue; } @@ -349,7 +352,8 @@ static void f2fs_write_end_io(struct bio *bio) if (unlikely(bio->bi_status)) { mapping_set_error(page->mapping, -EIO); if (type == F2FS_WB_CP_DATA) - f2fs_stop_checkpoint(sbi, true); + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); } f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && @@ -703,8 +707,10 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, fio->is_por ? META_POR : (__is_meta_io(fio) ? - META_GENERIC : DATA_GENERIC_ENHANCE))) + META_GENERIC : DATA_GENERIC_ENHANCE))) { + f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } trace_f2fs_submit_page_bio(page, fio); @@ -723,7 +729,7 @@ int f2fs_submit_page_bio(struct f2fs_io_info *fio) wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); inc_page_count(fio->sbi, is_read_io(fio->op) ? - __read_io_type(page): WB_DATA_TYPE(fio->page)); + __read_io_type(page) : WB_DATA_TYPE(fio->page)); __submit_bio(fio->sbi, bio, fio->type); return 0; @@ -904,8 +910,10 @@ int f2fs_merge_page_bio(struct f2fs_io_info *fio) fio->encrypted_page : fio->page; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, - __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) + __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) { + f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } trace_f2fs_submit_page_bio(page, fio); @@ -1083,7 +1091,7 @@ static int f2fs_submit_page_read(struct inode *inode, struct page *page, } ClearPageError(page); inc_page_count(sbi, F2FS_RD_DATA); - f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); __submit_bio(sbi, bio, DATA); return 0; } @@ -1215,6 +1223,8 @@ struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, DATA_GENERIC_ENHANCE_READ)) { err = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_INVALID_BLKADDR); goto put_err; } goto got_it; @@ -1235,6 +1245,8 @@ struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, dn.data_blkaddr, DATA_GENERIC_ENHANCE)) { err = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_INVALID_BLKADDR); goto put_err; } got_it: @@ -1548,6 +1560,7 @@ next_block: if (__is_valid_data_blkaddr(blkaddr) && !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto sync_out; } @@ -1593,6 +1606,8 @@ next_block: (flag != F2FS_GET_BLOCK_FIEMAP || IS_ENABLED(CONFIG_F2FS_CHECK_FS))) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, + ERROR_CORRUPTED_CLUSTER); goto sync_out; } if (flag == F2FS_GET_BLOCK_BMAP) { @@ -1816,7 +1831,7 @@ static int f2fs_xattr_fiemap(struct inode *inode, err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); trace_f2fs_fiemap(inode, 0, phys, len, flags, err); - if (err || err == 1) + if (err) return err; } @@ -2074,6 +2089,8 @@ got_it: if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, DATA_GENERIC_ENHANCE_READ)) { ret = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_INVALID_BLKADDR); goto out; } } else { @@ -2122,7 +2139,8 @@ submit_and_realloc: goto submit_and_realloc; inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); - f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, + F2FS_BLKSIZE); ClearPageError(page); *last_block_in_bio = block_nr; goto out; @@ -2270,8 +2288,7 @@ submit_and_realloc: refcount_inc(&dic->refcnt); inc_page_count(sbi, F2FS_RD_DATA); - f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); - f2fs_update_iostat(sbi, FS_CDATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); ClearPageError(page); *last_block_in_bio = blkaddr; } @@ -2543,7 +2560,7 @@ bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) return true; /* if this is cold file, we should overwrite to avoid fragmentation */ - if (file_is_cold(inode)) + if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE)) return true; return check_inplace_update_policy(inode, fio); @@ -2617,8 +2634,11 @@ int f2fs_do_write_data_page(struct f2fs_io_info *fio) fio->old_blkaddr = ei.blk + page->index - ei.fofs; if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, - DATA_GENERIC_ENHANCE)) + DATA_GENERIC_ENHANCE)) { + f2fs_handle_error(fio->sbi, + ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } ipu_force = true; fio->need_lock = LOCK_DONE; @@ -2646,6 +2666,7 @@ got_it: !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, DATA_GENERIC_ENHANCE)) { err = -EFSCORRUPTED; + f2fs_handle_error(fio->sbi, ERROR_INVALID_BLKADDR); goto out_writepage; } @@ -2856,7 +2877,7 @@ out: } unlock_page(page); if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && - !F2FS_I(inode)->cp_task && allow_balance) + !F2FS_I(inode)->wb_task && allow_balance) f2fs_balance_fs(sbi, need_balance_fs); if (unlikely(f2fs_cp_error(sbi))) { @@ -3156,7 +3177,7 @@ static inline bool __should_serialize_io(struct inode *inode, struct writeback_control *wbc) { /* to avoid deadlock in path of data flush */ - if (F2FS_I(inode)->cp_task) + if (F2FS_I(inode)->wb_task) return false; if (!S_ISREG(inode->i_mode)) @@ -3559,6 +3580,7 @@ repeat: if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto fail; } err = f2fs_submit_page_read(inode, page, blkaddr, 0, true); @@ -3697,8 +3719,7 @@ static bool f2fs_dirty_data_folio(struct address_space *mapping, folio_mark_uptodate(folio); BUG_ON(folio_test_swapcache(folio)); - if (!folio_test_dirty(folio)) { - filemap_dirty_folio(mapping, folio); + if (filemap_dirty_folio(mapping, folio)) { f2fs_update_dirty_folio(inode, folio); return true; } @@ -3970,6 +3991,7 @@ static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, if (ret < 0) return ret; + stat_inc_swapfile_inode(inode); set_inode_flag(inode, FI_PIN_FILE); f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); return ret; @@ -3979,6 +4001,7 @@ static void f2fs_swap_deactivate(struct file *file) { struct inode *inode = file_inode(file); + stat_dec_swapfile_inode(inode); clear_inode_flag(inode, FI_PIN_FILE); } #else diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index c01471573977ac7dba2d0d72bf68b91b0de3d51e..a216dcdf69418e7583f48e92db963ed1ab4d0980 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -91,7 +91,7 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->ndirty_files = sbi->ndirty_inode[FILE_INODE]; si->nquota_files = sbi->nquota_files; si->ndirty_all = sbi->ndirty_inode[DIRTY_META]; - si->aw_cnt = sbi->atomic_files; + si->aw_cnt = atomic_read(&sbi->atomic_files); si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt); si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ); si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE); @@ -135,6 +135,7 @@ static void update_general_status(struct f2fs_sb_info *sbi) si->inline_inode = atomic_read(&sbi->inline_inode); si->inline_dir = atomic_read(&sbi->inline_dir); si->compr_inode = atomic_read(&sbi->compr_inode); + si->swapfile_inode = atomic_read(&sbi->swapfile_inode); si->compr_blocks = atomic64_read(&sbi->compr_blocks); si->append = sbi->im[APPEND_INO].ino_num; si->update = sbi->im[UPDATE_INO].ino_num; @@ -347,7 +348,7 @@ static int stat_show(struct seq_file *s, void *v) seq_printf(s, "\n=====[ partition info(%pg). #%d, %s, CP: %s]=====\n", si->sbi->sb->s_bdev, i++, - f2fs_readonly(si->sbi->sb) ? "RO": "RW", + f2fs_readonly(si->sbi->sb) ? "RO" : "RW", is_set_ckpt_flags(si->sbi, CP_DISABLED_FLAG) ? "Disabled" : (f2fs_cp_error(si->sbi) ? "Error" : "Good")); if (si->sbi->s_flag) { @@ -385,6 +386,8 @@ static int stat_show(struct seq_file *s, void *v) si->inline_dir); seq_printf(s, " - Compressed Inode: %u, Blocks: %llu\n", si->compr_inode, si->compr_blocks); + seq_printf(s, " - Swapfile Inode: %u\n", + si->swapfile_inode); seq_printf(s, " - Orphan/Append/Update Inode: %u, %u, %u\n", si->orphans, si->append, si->update); seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n", @@ -607,6 +610,8 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi) atomic_set(&sbi->inline_dir, 0); atomic_set(&sbi->compr_inode, 0); atomic64_set(&sbi->compr_blocks, 0); + atomic_set(&sbi->swapfile_inode, 0); + atomic_set(&sbi->atomic_files, 0); atomic_set(&sbi->inplace_count, 0); for (i = META_CP; i < META_MAX; i++) atomic_set(&sbi->meta_count[i], 0); diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index d5bd7932fb642c7e5fa74ac0d77dc0d80c0206f8..21960a899b6adaf9adb26d8e07ac59d27cfdc330 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -1041,6 +1041,7 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, __func__, le16_to_cpu(de->name_len)); set_sbi_flag(sbi, SBI_NEED_FSCK); err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_CORRUPTED_DIRENT); goto out; } diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 866e72b29bd5ac507bbad6cbf098b2030971d94d..932c070173b976f8c320200e99e73920f7b16bff 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -544,7 +544,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode, if (!et) return; - trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len); + trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, len, 0); write_lock(&et->lock); @@ -583,7 +583,7 @@ static void f2fs_update_extent_tree_range(struct inode *inode, org_end = dei.fofs + dei.len; f2fs_bug_on(sbi, pos >= org_end); - if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) { + if (pos > dei.fofs && pos - dei.fofs >= F2FS_MIN_EXTENT_LEN) { en->ei.len = pos - en->ei.fofs; prev_en = en; parts = 1; @@ -675,7 +675,7 @@ void f2fs_update_extent_tree_range_compressed(struct inode *inode, struct rb_node **insert_p = NULL, *insert_parent = NULL; bool leftmost = false; - trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, llen); + trace_f2fs_update_extent_tree_range(inode, fofs, blkaddr, llen, c_len); /* it is safe here to check FI_NO_EXTENT w/o et->lock in ro image */ if (is_inode_flag_set(inode, FI_NO_EXTENT)) @@ -804,9 +804,8 @@ void f2fs_drop_extent_tree(struct inode *inode) if (!f2fs_may_extent_tree(inode)) return; - set_inode_flag(inode, FI_NO_EXTENT); - write_lock(&et->lock); + set_inode_flag(inode, FI_NO_EXTENT); __free_extent_tree(sbi, et); if (et->largest.len) { et->largest.len = 0; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index 3c7cdb70fe2efc841379cee0e4968c6b60b6f706..e6355a5683b75cc1dd1941bdec48566efa612f66 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -266,6 +266,10 @@ enum { * condition of read on truncated area * by extent_cache */ + DATA_GENERIC_ENHANCE_UPDATE, /* + * strong check on range and segment + * bitmap for update case + */ META_GENERIC, }; @@ -274,7 +278,7 @@ enum { ORPHAN_INO, /* for orphan ino list */ APPEND_INO, /* for append ino list */ UPDATE_INO, /* for update ino list */ - TRANS_DIR_INO, /* for trasactions dir ino list */ + TRANS_DIR_INO, /* for transactions dir ino list */ FLUSH_INO, /* for multiple device flushing */ MAX_INO_ENTRY, /* max. list */ }; @@ -782,6 +786,7 @@ struct f2fs_inode_info { unsigned int clevel; /* maximum level of given file name */ struct task_struct *task; /* lookup and create consistency */ struct task_struct *cp_task; /* separate cp/wb IO stats*/ + struct task_struct *wb_task; /* indicate inode is in context of writeback */ nid_t i_xattr_nid; /* node id that contains xattrs */ loff_t last_disk_size; /* lastly written file size */ spinlock_t i_size_lock; /* protect last_disk_size */ @@ -1158,7 +1163,10 @@ enum iostat_type { APP_BUFFERED_IO, /* app buffered write IOs */ APP_WRITE_IO, /* app write IOs */ APP_MAPPED_IO, /* app mapped IOs */ + APP_BUFFERED_CDATA_IO, /* app buffered write IOs on compressed file */ + APP_MAPPED_CDATA_IO, /* app mapped write IOs on compressed file */ FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */ + FS_CDATA_IO, /* data IOs from kworker/fsync/reclaimer on compressed file */ FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */ FS_META_IO, /* meta IOs from kworker/reclaimer */ FS_GC_DATA_IO, /* data IOs from forground gc */ @@ -1172,6 +1180,8 @@ enum iostat_type { APP_BUFFERED_READ_IO, /* app buffered read IOs */ APP_READ_IO, /* app read IOs */ APP_MAPPED_READ_IO, /* app mapped read IOs */ + APP_BUFFERED_CDATA_READ_IO, /* app buffered read IOs on compressed file */ + APP_MAPPED_CDATA_READ_IO, /* app mapped read IOs on compressed file */ FS_DATA_READ_IO, /* data read IOs */ FS_GDATA_READ_IO, /* data read IOs from background gc */ FS_CDATA_READ_IO, /* compressed data read IOs */ @@ -1247,7 +1257,6 @@ enum inode_type { DIR_INODE, /* for dirty dir inode */ FILE_INODE, /* for dirty regular/symlink inode */ DIRTY_META, /* for all dirtied inode metadata */ - ATOMIC_FILE, /* for all atomic files */ NR_INODE_TYPE, }; @@ -1726,11 +1735,9 @@ struct f2fs_sb_info { unsigned int gc_mode; /* current GC state */ unsigned int next_victim_seg[2]; /* next segment in victim section */ spinlock_t gc_urgent_high_lock; - bool gc_urgent_high_limited; /* indicates having limited trial count */ unsigned int gc_urgent_high_remaining; /* remaining trial count for GC_URGENT_HIGH */ /* for skip statistic */ - unsigned int atomic_files; /* # of opened atomic file */ unsigned long long skipped_gc_rwsem; /* FG_GC only */ /* threshold for gc trials on pinned files */ @@ -1761,6 +1768,8 @@ struct f2fs_sb_info { atomic_t inline_dir; /* # of inline_dentry inodes */ atomic_t compr_inode; /* # of compressed inodes */ atomic64_t compr_blocks; /* # of compressed blocks */ + atomic_t swapfile_inode; /* # of swapfile inodes */ + atomic_t atomic_files; /* # of opened atomic file */ atomic_t max_aw_cnt; /* max # of atomic writes */ unsigned int io_skip_bggc; /* skip background gc for in-flight IO */ unsigned int other_skip_bggc; /* skip background gc for other reasons */ @@ -1806,6 +1815,10 @@ struct f2fs_sb_info { struct workqueue_struct *post_read_wq; /* post read workqueue */ + unsigned char errors[MAX_F2FS_ERRORS]; /* error flags */ + spinlock_t error_lock; /* protect errors array */ + bool error_dirty; /* errors of sb is dirty */ + struct kmem_cache *inline_xattr_slab; /* inline xattr entry */ unsigned int inline_xattr_slab_size; /* default inline xattr slab size */ @@ -2525,7 +2538,7 @@ static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) if (__cp_payload(sbi) > 0) { if (flag == NAT_BITMAP) - return &ckpt->sit_nat_version_bitmap; + return tmp_ptr; else return (unsigned char *)ckpt + F2FS_BLKSIZE; } else { @@ -3547,6 +3560,8 @@ int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); int f2fs_quota_sync(struct super_block *sb, int type); loff_t max_file_blocks(struct inode *inode); void f2fs_quota_off_umount(struct super_block *sb); +void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason); +void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error); int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); int f2fs_sync_fs(struct super_block *sb, int sync); int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi); @@ -3706,7 +3721,9 @@ static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) /* * checkpoint.c */ -void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io); +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason); +void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi); struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index); struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index); struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index); @@ -3736,7 +3753,8 @@ int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi); int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi); void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio); void f2fs_remove_dirty_inode(struct inode *inode); -int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type); +int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, + bool from_cp); void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type); u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi); int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc); @@ -3858,7 +3876,7 @@ struct f2fs_stat_info { int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt; unsigned int cur_ckpt_time, peak_ckpt_time; int inline_xattr, inline_inode, inline_dir, append, update, orphans; - int compr_inode; + int compr_inode, swapfile_inode; unsigned long long compr_blocks; int aw_cnt, max_aw_cnt; unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks; @@ -3947,6 +3965,14 @@ static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) (atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks)) #define stat_sub_compr_blocks(inode, blocks) \ (atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks)) +#define stat_inc_swapfile_inode(inode) \ + (atomic_inc(&F2FS_I_SB(inode)->swapfile_inode)) +#define stat_dec_swapfile_inode(inode) \ + (atomic_dec(&F2FS_I_SB(inode)->swapfile_inode)) +#define stat_inc_atomic_inode(inode) \ + (atomic_inc(&F2FS_I_SB(inode)->atomic_files)) +#define stat_dec_atomic_inode(inode) \ + (atomic_dec(&F2FS_I_SB(inode)->atomic_files)) #define stat_inc_meta_count(sbi, blkaddr) \ do { \ if (blkaddr < SIT_I(sbi)->sit_base_addr) \ @@ -3966,7 +3992,7 @@ static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) (atomic_inc(&(sbi)->inplace_count)) #define stat_update_max_atomic_write(inode) \ do { \ - int cur = F2FS_I_SB(inode)->atomic_files; \ + int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files); \ int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \ if (cur > max) \ atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \ @@ -4031,6 +4057,10 @@ void f2fs_update_sit_info(struct f2fs_sb_info *sbi); #define stat_dec_compr_inode(inode) do { } while (0) #define stat_add_compr_blocks(inode, blocks) do { } while (0) #define stat_sub_compr_blocks(inode, blocks) do { } while (0) +#define stat_inc_swapfile_inode(inode) do { } while (0) +#define stat_dec_swapfile_inode(inode) do { } while (0) +#define stat_inc_atomic_inode(inode) do { } while (0) +#define stat_dec_atomic_inode(inode) do { } while (0) #define stat_update_max_atomic_write(inode) do { } while (0) #define stat_inc_meta_count(sbi, blkaddr) do { } while (0) #define stat_inc_seg_type(sbi, curseg) do { } while (0) @@ -4471,17 +4501,6 @@ static inline void f2fs_i_compr_blocks_update(struct inode *inode, f2fs_mark_inode_dirty_sync(inode, true); } -static inline int block_unaligned_IO(struct inode *inode, - struct kiocb *iocb, struct iov_iter *iter) -{ - unsigned int i_blkbits = READ_ONCE(inode->i_blkbits); - unsigned int blocksize_mask = (1 << i_blkbits) - 1; - loff_t offset = iocb->ki_pos; - unsigned long align = offset | iov_iter_alignment(iter); - - return align & blocksize_mask; -} - static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi, int flag) { @@ -4492,35 +4511,6 @@ static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi, return sbi->aligned_blksize; } -static inline bool f2fs_force_buffered_io(struct inode *inode, - struct kiocb *iocb, struct iov_iter *iter) -{ - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); - int rw = iov_iter_rw(iter); - - if (!fscrypt_dio_supported(iocb, iter)) - return true; - if (fsverity_active(inode)) - return true; - if (f2fs_compressed_file(inode)) - return true; - - /* disallow direct IO if any of devices has unaligned blksize */ - if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize) - return true; - - if (f2fs_lfs_mode(sbi) && (rw == WRITE)) { - if (block_unaligned_IO(inode, iocb, iter)) - return true; - if (F2FS_IO_ALIGNED(sbi)) - return true; - } - if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED)) - return true; - - return false; -} - static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) { return fsverity_active(inode) && diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index ce4905a073b3c413a751efc0dec5069a1a5e32c7..82cda12582272a0040fe168beb454d787b0e2af1 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -43,8 +43,8 @@ static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); if (!ret) - f2fs_update_iostat(F2FS_I_SB(inode), APP_MAPPED_READ_IO, - F2FS_BLKSIZE); + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_MAPPED_READ_IO, F2FS_BLKSIZE); trace_f2fs_filemap_fault(inode, vmf->pgoff, (unsigned long)ret); @@ -154,7 +154,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) if (!PageUptodate(page)) SetPageUptodate(page); - f2fs_update_iostat(sbi, APP_MAPPED_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE); f2fs_update_time(sbi, REQ_TIME); trace_f2fs_vm_page_mkwrite(page, DATA); @@ -808,6 +808,34 @@ int f2fs_truncate(struct inode *inode) return 0; } +static bool f2fs_force_buffered_io(struct inode *inode, int rw) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (!fscrypt_dio_supported(inode)) + return true; + if (fsverity_active(inode)) + return true; + if (f2fs_compressed_file(inode)) + return true; + + /* disallow direct IO if any of devices has unaligned blksize */ + if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize) + return true; + /* + * for blkzoned device, fallback direct IO to buffered IO, so + * all IOs can be serialized by log-structured write. + */ + if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE)) + return true; + if (f2fs_lfs_mode(sbi) && rw == WRITE && F2FS_IO_ALIGNED(sbi)) + return true; + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED)) + return true; + + return false; +} + int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { @@ -824,6 +852,24 @@ int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path, stat->btime.tv_nsec = fi->i_crtime.tv_nsec; } + /* + * Return the DIO alignment restrictions if requested. We only return + * this information when requested, since on encrypted files it might + * take a fair bit of work to get if the file wasn't opened recently. + * + * f2fs sometimes supports DIO reads but not DIO writes. STATX_DIOALIGN + * cannot represent that, so in that case we report no DIO support. + */ + if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { + unsigned int bsize = i_blocksize(inode); + + stat->result_mask |= STATX_DIOALIGN; + if (!f2fs_force_buffered_io(inode, WRITE)) { + stat->dio_mem_align = bsize; + stat->dio_offset_align = bsize; + } + } + flags = fi->i_flags; if (flags & F2FS_COMPR_FL) stat->attributes |= STATX_ATTR_COMPRESSED; @@ -871,9 +917,10 @@ static void __setattr_copy(struct user_namespace *mnt_userns, inode->i_ctime = attr->ia_ctime; if (ia_valid & ATTR_MODE) { umode_t mode = attr->ia_mode; - kgid_t kgid = i_gid_into_mnt(mnt_userns, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode); - if (!in_group_p(kgid) && !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) + if (!vfsgid_in_group_p(vfsgid) && + !capable_wrt_inode_uidgid(mnt_userns, inode, CAP_FSETID)) mode &= ~S_ISGID; set_acl_inode(inode, mode); } @@ -1155,6 +1202,7 @@ next_dnode: !f2fs_is_valid_blkaddr(sbi, *blkaddr, DATA_GENERIC_ENHANCE)) { f2fs_put_dnode(&dn); + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; } @@ -1439,6 +1487,7 @@ static int f2fs_do_zero_range(struct dnode_of_data *dn, pgoff_t start, if (!f2fs_is_valid_blkaddr(sbi, dn->data_blkaddr, DATA_GENERIC_ENHANCE)) { ret = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); break; } @@ -2048,9 +2097,7 @@ static int f2fs_ioc_start_atomic_write(struct file *filp) } f2fs_i_size_write(fi->cow_inode, i_size_read(inode)); - spin_lock(&sbi->inode_lock[ATOMIC_FILE]); - sbi->atomic_files++; - spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); + stat_inc_atomic_inode(inode); set_inode_flag(inode, FI_ATOMIC_FILE); set_inode_flag(fi->cow_inode, FI_COW_FILE); @@ -2144,7 +2191,8 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) if (ret) { if (ret == -EROFS) { ret = 0; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); trace_f2fs_shutdown(sbi, in, ret); } @@ -2157,7 +2205,7 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) ret = freeze_bdev(sb->s_bdev); if (ret) goto out; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); thaw_bdev(sb->s_bdev); break; @@ -2166,16 +2214,16 @@ static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) ret = f2fs_sync_fs(sb, 1); if (ret) goto out; - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_NOSYNC: - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_METAFLUSH: f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_META_IO); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); set_sbi_flag(sbi, SBI_IS_SHUTDOWN); break; case F2FS_GOING_DOWN_NEED_FSCK: @@ -3321,8 +3369,10 @@ static int release_compress_blocks(struct dnode_of_data *dn, pgoff_t count) if (!__is_valid_data_blkaddr(blkaddr)) continue; if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr, - DATA_GENERIC_ENHANCE))) + DATA_GENERIC_ENHANCE))) { + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } } while (count) { @@ -3483,8 +3533,10 @@ static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count) if (!__is_valid_data_blkaddr(blkaddr)) continue; if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr, - DATA_GENERIC_ENHANCE))) + DATA_GENERIC_ENHANCE))) { + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } } while (count) { @@ -3756,6 +3808,8 @@ static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) DATA_GENERIC_ENHANCE)) { ret = -EFSCORRUPTED; f2fs_put_dnode(&dn); + f2fs_handle_error(sbi, + ERROR_INVALID_BLKADDR); goto out; } @@ -4182,7 +4236,7 @@ static bool f2fs_should_use_dio(struct inode *inode, struct kiocb *iocb, if (!(iocb->ki_flags & IOCB_DIRECT)) return false; - if (f2fs_force_buffered_io(inode, iocb, iter)) + if (f2fs_force_buffered_io(inode, iov_iter_rw(iter))) return false; /* @@ -4212,7 +4266,7 @@ static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error, dec_page_count(sbi, F2FS_DIO_READ); if (error) return error; - f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, size); + f2fs_update_iostat(sbi, NULL, APP_DIRECT_READ_IO, size); return 0; } @@ -4301,7 +4355,8 @@ skip_read_trace: } else { ret = filemap_read(iocb, to, 0); if (ret > 0) - f2fs_update_iostat(F2FS_I_SB(inode), APP_BUFFERED_READ_IO, ret); + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_BUFFERED_READ_IO, ret); } if (trace_f2fs_dataread_end_enabled()) trace_f2fs_dataread_end(inode, pos, ret); @@ -4418,7 +4473,8 @@ static ssize_t f2fs_buffered_write_iter(struct kiocb *iocb, if (ret > 0) { iocb->ki_pos += ret; - f2fs_update_iostat(F2FS_I_SB(inode), APP_BUFFERED_IO, ret); + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_BUFFERED_IO, ret); } return ret; } @@ -4431,7 +4487,7 @@ static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, dec_page_count(sbi, F2FS_DIO_WRITE); if (error) return error; - f2fs_update_iostat(sbi, APP_DIRECT_IO, size); + f2fs_update_iostat(sbi, NULL, APP_DIRECT_IO, size); return 0; } @@ -4619,7 +4675,7 @@ static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) skip_write_trace: /* Do the actual write. */ ret = dio ? - f2fs_dio_write_iter(iocb, from, &may_need_sync): + f2fs_dio_write_iter(iocb, from, &may_need_sync) : f2fs_buffered_write_iter(iocb, from); if (trace_f2fs_datawrite_end_enabled()) diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 6da21d405ce1eff931581bc98a592866c02ebd2e..4546e01b2ee082a9eca2a02e4f87d380e274bf40 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -74,7 +74,8 @@ static int gc_thread_func(void *data) if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FAULT_INJECT); } if (!sb_start_write_trylock(sbi->sb)) { @@ -97,14 +98,10 @@ static int gc_thread_func(void *data) */ if (sbi->gc_mode == GC_URGENT_HIGH) { spin_lock(&sbi->gc_urgent_high_lock); - if (sbi->gc_urgent_high_limited) { - if (!sbi->gc_urgent_high_remaining) { - sbi->gc_urgent_high_limited = false; - spin_unlock(&sbi->gc_urgent_high_lock); - sbi->gc_mode = GC_NORMAL; - continue; - } + if (sbi->gc_urgent_high_remaining) { sbi->gc_urgent_high_remaining--; + if (!sbi->gc_urgent_high_remaining) + sbi->gc_mode = GC_NORMAL; } spin_unlock(&sbi->gc_urgent_high_lock); } @@ -285,7 +282,7 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type, /* let's select beginning hot/small space first in no_heap mode*/ if (f2fs_need_rand_seg(sbi)) - p->offset = prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec); + p->offset = prandom_u32_max(MAIN_SECS(sbi) * sbi->segs_per_sec); else if (test_opt(sbi, NOHEAP) && (type == CURSEG_HOT_DATA || IS_NODESEG(type))) p->offset = 0; @@ -1082,7 +1079,7 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, { struct page *node_page; nid_t nid; - unsigned int ofs_in_node; + unsigned int ofs_in_node, max_addrs; block_t source_blkaddr; nid = le32_to_cpu(sum->nid); @@ -1108,6 +1105,14 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, return false; } + max_addrs = IS_INODE(node_page) ? DEF_ADDRS_PER_INODE : + DEF_ADDRS_PER_BLOCK; + if (ofs_in_node >= max_addrs) { + f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%u, nid:%u, max:%u", + ofs_in_node, dni->ino, dni->nid, max_addrs); + return false; + } + *nofs = ofs_of_node(node_page); source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node); f2fs_put_page(node_page, 1); @@ -1159,6 +1164,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, DATA_GENERIC_ENHANCE_READ))) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto put_page; } goto got_it; @@ -1177,6 +1183,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, DATA_GENERIC_ENHANCE))) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto put_page; } got_it: @@ -1206,8 +1213,8 @@ got_it: f2fs_put_page(fio.encrypted_page, 0); f2fs_put_page(page, 1); - f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); - f2fs_update_iostat(sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE); return 0; put_encrypted_page: @@ -1307,8 +1314,10 @@ static int move_data_block(struct inode *inode, block_t bidx, goto up_out; } - f2fs_update_iostat(fio.sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); - f2fs_update_iostat(fio.sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO, + F2FS_BLKSIZE); + f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO, + F2FS_BLKSIZE); lock_page(mpage); if (unlikely(mpage->mapping != META_MAPPING(fio.sbi) || @@ -1360,7 +1369,7 @@ static int move_data_block(struct inode *inode, block_t bidx, goto put_page_out; } - f2fs_update_iostat(fio.sbi, FS_GC_DATA_IO, F2FS_BLKSIZE); + f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE); f2fs_update_data_blkaddr(&dn, newaddr); set_inode_flag(inode, FI_APPEND_WRITE); @@ -1706,7 +1715,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT", segno, type, GET_SUM_TYPE((&sum->footer))); set_sbi_flag(sbi, SBI_NEED_FSCK); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_CORRUPTED_SUMMARY); goto skip; } diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index bf46a7dfbea2fc282229d78c094e45ab16a5f362..21a495234ffd7f22bc63f8bc05b02acd6e4ee4f7 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -64,7 +64,6 @@ bool f2fs_may_inline_dentry(struct inode *inode) void f2fs_do_read_inline_data(struct page *page, struct page *ipage) { struct inode *inode = page->mapping->host; - void *src_addr, *dst_addr; if (PageUptodate(page)) return; @@ -74,11 +73,8 @@ void f2fs_do_read_inline_data(struct page *page, struct page *ipage) zero_user_segment(page, MAX_INLINE_DATA(inode), PAGE_SIZE); /* Copy the whole inline data block */ - src_addr = inline_data_addr(inode, ipage); - dst_addr = kmap_atomic(page); - memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode)); - flush_dcache_page(page); - kunmap_atomic(dst_addr); + memcpy_to_page(page, 0, inline_data_addr(inode, ipage), + MAX_INLINE_DATA(inode)); if (!PageUptodate(page)) SetPageUptodate(page); } @@ -164,6 +160,7 @@ int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) set_sbi_flag(fio.sbi, SBI_NEED_FSCK); f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.", __func__, dn->inode->i_ino, dn->data_blkaddr); + f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; } @@ -246,7 +243,6 @@ out: int f2fs_write_inline_data(struct inode *inode, struct page *page) { - void *src_addr, *dst_addr; struct dnode_of_data dn; int err; @@ -263,10 +259,8 @@ int f2fs_write_inline_data(struct inode *inode, struct page *page) f2fs_bug_on(F2FS_I_SB(inode), page->index); f2fs_wait_on_page_writeback(dn.inode_page, NODE, true, true); - src_addr = kmap_atomic(page); - dst_addr = inline_data_addr(inode, dn.inode_page); - memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode)); - kunmap_atomic(src_addr); + memcpy_from_page(inline_data_addr(inode, dn.inode_page), + page, 0, MAX_INLINE_DATA(inode)); set_page_dirty(dn.inode_page); f2fs_clear_page_cache_dirty_tag(page); @@ -419,6 +413,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct page *ipage, set_sbi_flag(F2FS_P_SB(page), SBI_NEED_FSCK); f2fs_warn(F2FS_P_SB(page), "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.", __func__, dir->i_ino, dn.data_blkaddr); + f2fs_handle_error(F2FS_P_SB(page), ERROR_INVALID_BLKADDR); err = -EFSCORRUPTED; goto out; } diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index 6d11c365d7b4ee14b3486c59a2129b929ccd8c34..9f0d3864d9f13a619285332de1aa307c3776a691 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -81,8 +81,10 @@ static int __written_first_block(struct f2fs_sb_info *sbi, if (!__is_valid_data_blkaddr(addr)) return 1; - if (!f2fs_is_valid_blkaddr(sbi, addr, DATA_GENERIC_ENHANCE)) + if (!f2fs_is_valid_blkaddr(sbi, addr, DATA_GENERIC_ENHANCE)) { + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); return -EFSCORRUPTED; + } return 0; } @@ -333,6 +335,16 @@ static bool sanity_check_inode(struct inode *inode, struct page *node_page) return true; } +static void init_idisk_time(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + fi->i_disk_time[0] = inode->i_atime; + fi->i_disk_time[1] = inode->i_ctime; + fi->i_disk_time[2] = inode->i_mtime; + fi->i_disk_time[3] = fi->i_crtime; +} + static int do_read_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); @@ -405,6 +417,7 @@ static int do_read_inode(struct inode *inode) if (!sanity_check_inode(inode, node_page)) { f2fs_put_page(node_page, 1); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); return -EFSCORRUPTED; } @@ -465,10 +478,7 @@ static int do_read_inode(struct inode *inode) } } - fi->i_disk_time[0] = inode->i_atime; - fi->i_disk_time[1] = inode->i_ctime; - fi->i_disk_time[2] = inode->i_mtime; - fi->i_disk_time[3] = fi->i_crtime; + init_idisk_time(inode); f2fs_put_page(node_page, 1); stat_inc_inline_xattr(inode); @@ -480,6 +490,12 @@ static int do_read_inode(struct inode *inode) return 0; } +static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino) +{ + return ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi) || + ino == F2FS_COMPRESS_INO(sbi); +} + struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) { struct f2fs_sb_info *sbi = F2FS_SB(sb); @@ -491,16 +507,22 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) return ERR_PTR(-ENOMEM); if (!(inode->i_state & I_NEW)) { + if (is_meta_ino(sbi, ino)) { + f2fs_err(sbi, "inaccessible inode: %lu, run fsck to repair", ino); + set_sbi_flag(sbi, SBI_NEED_FSCK); + ret = -EFSCORRUPTED; + trace_f2fs_iget_exit(inode, ret); + iput(inode); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + return ERR_PTR(ret); + } + trace_f2fs_iget(inode); return inode; } - if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) - goto make_now; -#ifdef CONFIG_F2FS_FS_COMPRESSION - if (ino == F2FS_COMPRESS_INO(sbi)) + if (is_meta_ino(sbi, ino)) goto make_now; -#endif ret = do_read_inode(inode); if (ret) @@ -676,11 +698,7 @@ void f2fs_update_inode(struct inode *inode, struct page *node_page) if (inode->i_nlink == 0) clear_page_private_inline(node_page); - F2FS_I(inode)->i_disk_time[0] = inode->i_atime; - F2FS_I(inode)->i_disk_time[1] = inode->i_ctime; - F2FS_I(inode)->i_disk_time[2] = inode->i_mtime; - F2FS_I(inode)->i_disk_time[3] = F2FS_I(inode)->i_crtime; - + init_idisk_time(inode); #ifdef CONFIG_F2FS_CHECK_FS f2fs_inode_chksum_set(F2FS_I_SB(inode), node_page); #endif @@ -699,7 +717,8 @@ retry: cond_resched(); goto retry; } else if (err != -ENOENT) { - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_UPDATE_INODE); } return; } diff --git a/fs/f2fs/iostat.c b/fs/f2fs/iostat.c index d84c5f6cc09d74230c42c04ea7efbd03e2af8665..3166a8939ed4f4f3ffae033adc4d462ca9cf26b6 100644 --- a/fs/f2fs/iostat.c +++ b/fs/f2fs/iostat.c @@ -31,55 +31,65 @@ int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) /* print app write IOs */ seq_puts(seq, "[WRITE]\n"); - seq_printf(seq, "app buffered: %-16llu\n", + seq_printf(seq, "app buffered data: %-16llu\n", sbi->rw_iostat[APP_BUFFERED_IO]); - seq_printf(seq, "app direct: %-16llu\n", + seq_printf(seq, "app direct data: %-16llu\n", sbi->rw_iostat[APP_DIRECT_IO]); - seq_printf(seq, "app mapped: %-16llu\n", + seq_printf(seq, "app mapped data: %-16llu\n", sbi->rw_iostat[APP_MAPPED_IO]); + seq_printf(seq, "app buffered cdata: %-16llu\n", + sbi->rw_iostat[APP_BUFFERED_CDATA_IO]); + seq_printf(seq, "app mapped cdata: %-16llu\n", + sbi->rw_iostat[APP_MAPPED_CDATA_IO]); /* print fs write IOs */ - seq_printf(seq, "fs data: %-16llu\n", + seq_printf(seq, "fs data: %-16llu\n", sbi->rw_iostat[FS_DATA_IO]); - seq_printf(seq, "fs node: %-16llu\n", + seq_printf(seq, "fs cdata: %-16llu\n", + sbi->rw_iostat[FS_CDATA_IO]); + seq_printf(seq, "fs node: %-16llu\n", sbi->rw_iostat[FS_NODE_IO]); - seq_printf(seq, "fs meta: %-16llu\n", + seq_printf(seq, "fs meta: %-16llu\n", sbi->rw_iostat[FS_META_IO]); - seq_printf(seq, "fs gc data: %-16llu\n", + seq_printf(seq, "fs gc data: %-16llu\n", sbi->rw_iostat[FS_GC_DATA_IO]); - seq_printf(seq, "fs gc node: %-16llu\n", + seq_printf(seq, "fs gc node: %-16llu\n", sbi->rw_iostat[FS_GC_NODE_IO]); - seq_printf(seq, "fs cp data: %-16llu\n", + seq_printf(seq, "fs cp data: %-16llu\n", sbi->rw_iostat[FS_CP_DATA_IO]); - seq_printf(seq, "fs cp node: %-16llu\n", + seq_printf(seq, "fs cp node: %-16llu\n", sbi->rw_iostat[FS_CP_NODE_IO]); - seq_printf(seq, "fs cp meta: %-16llu\n", + seq_printf(seq, "fs cp meta: %-16llu\n", sbi->rw_iostat[FS_CP_META_IO]); /* print app read IOs */ seq_puts(seq, "[READ]\n"); - seq_printf(seq, "app buffered: %-16llu\n", + seq_printf(seq, "app buffered data: %-16llu\n", sbi->rw_iostat[APP_BUFFERED_READ_IO]); - seq_printf(seq, "app direct: %-16llu\n", + seq_printf(seq, "app direct data: %-16llu\n", sbi->rw_iostat[APP_DIRECT_READ_IO]); - seq_printf(seq, "app mapped: %-16llu\n", + seq_printf(seq, "app mapped data: %-16llu\n", sbi->rw_iostat[APP_MAPPED_READ_IO]); + seq_printf(seq, "app buffered cdata: %-16llu\n", + sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO]); + seq_printf(seq, "app mapped cdata: %-16llu\n", + sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO]); /* print fs read IOs */ - seq_printf(seq, "fs data: %-16llu\n", + seq_printf(seq, "fs data: %-16llu\n", sbi->rw_iostat[FS_DATA_READ_IO]); - seq_printf(seq, "fs gc data: %-16llu\n", + seq_printf(seq, "fs gc data: %-16llu\n", sbi->rw_iostat[FS_GDATA_READ_IO]); - seq_printf(seq, "fs compr_data: %-16llu\n", + seq_printf(seq, "fs cdata: %-16llu\n", sbi->rw_iostat[FS_CDATA_READ_IO]); - seq_printf(seq, "fs node: %-16llu\n", + seq_printf(seq, "fs node: %-16llu\n", sbi->rw_iostat[FS_NODE_READ_IO]); - seq_printf(seq, "fs meta: %-16llu\n", + seq_printf(seq, "fs meta: %-16llu\n", sbi->rw_iostat[FS_META_READ_IO]); /* print other IOs */ seq_puts(seq, "[OTHER]\n"); - seq_printf(seq, "fs discard: %-16llu\n", + seq_printf(seq, "fs discard: %-16llu\n", sbi->rw_iostat[FS_DISCARD]); return 0; @@ -159,7 +169,7 @@ void f2fs_reset_iostat(struct f2fs_sb_info *sbi) spin_unlock_irq(&sbi->iostat_lat_lock); } -void f2fs_update_iostat(struct f2fs_sb_info *sbi, +void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, enum iostat_type type, unsigned long long io_bytes) { unsigned long flags; @@ -176,6 +186,28 @@ void f2fs_update_iostat(struct f2fs_sb_info *sbi, if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) sbi->rw_iostat[APP_READ_IO] += io_bytes; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (inode && f2fs_compressed_file(inode)) { + if (type == APP_BUFFERED_IO) + sbi->rw_iostat[APP_BUFFERED_CDATA_IO] += io_bytes; + + if (type == APP_BUFFERED_READ_IO) + sbi->rw_iostat[APP_BUFFERED_CDATA_READ_IO] += io_bytes; + + if (type == APP_MAPPED_READ_IO) + sbi->rw_iostat[APP_MAPPED_CDATA_READ_IO] += io_bytes; + + if (type == APP_MAPPED_IO) + sbi->rw_iostat[APP_MAPPED_CDATA_IO] += io_bytes; + + if (type == FS_DATA_READ_IO) + sbi->rw_iostat[FS_CDATA_READ_IO] += io_bytes; + + if (type == FS_DATA_IO) + sbi->rw_iostat[FS_CDATA_IO] += io_bytes; + } +#endif + spin_unlock_irqrestore(&sbi->iostat_lock, flags); f2fs_record_iostat(sbi); diff --git a/fs/f2fs/iostat.h b/fs/f2fs/iostat.h index 22a2d01f57ef3b6116491502ef01a9c15dc167af..2c048307b6e0b5ebba099872a046f81c0f8fe87a 100644 --- a/fs/f2fs/iostat.h +++ b/fs/f2fs/iostat.h @@ -31,7 +31,7 @@ struct iostat_lat_info { extern int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset); extern void f2fs_reset_iostat(struct f2fs_sb_info *sbi); -extern void f2fs_update_iostat(struct f2fs_sb_info *sbi, +extern void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, enum iostat_type type, unsigned long long io_bytes); struct bio_iostat_ctx { @@ -65,7 +65,7 @@ extern void f2fs_destroy_iostat_processing(void); extern int f2fs_init_iostat(struct f2fs_sb_info *sbi); extern void f2fs_destroy_iostat(struct f2fs_sb_info *sbi); #else -static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, +static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, enum iostat_type type, unsigned long long io_bytes) {} static inline void iostat_update_and_unbind_ctx(struct bio *bio, int rw) {} static inline void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index bf00d5057abb8713fdd4ef401f3bb90b897ac3f1..a389772fd212acb93aa2b65e55090ddfd2b8d042 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -50,7 +50,7 @@ static struct inode *f2fs_new_inode(struct user_namespace *mnt_userns, inode->i_blocks = 0; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); F2FS_I(inode)->i_crtime = inode->i_mtime; - inode->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); if (S_ISDIR(inode->i_mode)) F2FS_I(inode)->i_current_depth = 1; @@ -845,7 +845,7 @@ out: } static int __f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode, bool is_whiteout, + struct file *file, umode_t mode, bool is_whiteout, struct inode **new_inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); @@ -892,8 +892,8 @@ static int __f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } else { - if (dentry) - d_tmpfile(dentry, inode); + if (file) + d_tmpfile(file, inode); else f2fs_i_links_write(inode, false); } @@ -915,16 +915,19 @@ out: } static int f2fs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; - return __f2fs_tmpfile(mnt_userns, dir, dentry, mode, false, NULL); + err = __f2fs_tmpfile(mnt_userns, dir, file, mode, false, NULL); + + return finish_open_simple(file, err); } static int f2fs_create_whiteout(struct user_namespace *mnt_userns, diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index e06a0c478b39a287c7a7b303a466d59f0fb87bfd..983572f23896901b50fda587aa8782c99e61fdcb 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -36,6 +36,7 @@ int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid) set_sbi_flag(sbi, SBI_NEED_FSCK); f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.", __func__, nid); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); return -EFSCORRUPTED; } return 0; @@ -585,7 +586,7 @@ retry: ne = nat_in_journal(journal, i); node_info_from_raw_nat(ni, &ne); } - up_read(&curseg->journal_rwsem); + up_read(&curseg->journal_rwsem); if (i >= 0) { f2fs_up_read(&nm_i->nat_tree_lock); goto cache; @@ -1295,6 +1296,7 @@ struct page *f2fs_new_node_page(struct dnode_of_data *dn, unsigned int ofs) if (unlikely(new_ni.blk_addr != NULL_ADDR)) { err = -EFSCORRUPTED; set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto fail; } #endif @@ -1369,7 +1371,7 @@ static int read_node_page(struct page *page, blk_opf_t op_flags) err = f2fs_submit_page_bio(&fio); if (!err) - f2fs_update_iostat(sbi, FS_NODE_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE); return err; } @@ -2147,8 +2149,7 @@ static bool f2fs_dirty_node_folio(struct address_space *mapping, if (IS_INODE(&folio->page)) f2fs_inode_chksum_set(F2FS_M_SB(mapping), &folio->page); #endif - if (!folio_test_dirty(folio)) { - filemap_dirty_folio(mapping, folio); + if (filemap_dirty_folio(mapping, folio)) { inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); set_page_private_reference(&folio->page); return true; diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index dcd0a1e35095175178dff56c7b1f48e9d2fce680..dea95b48b647d61888217a54ad08372d96377fd0 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -474,7 +474,7 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, struct dnode_of_data tdn = *dn; nid_t ino, nid; struct inode *inode; - unsigned int offset; + unsigned int offset, ofs_in_node, max_addrs; block_t bidx; int i; @@ -501,15 +501,25 @@ static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, got_it: /* Use the locked dnode page and inode */ nid = le32_to_cpu(sum.nid); + ofs_in_node = le16_to_cpu(sum.ofs_in_node); + + max_addrs = ADDRS_PER_PAGE(dn->node_page, dn->inode); + if (ofs_in_node >= max_addrs) { + f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%lu, nid:%u, max:%u", + ofs_in_node, dn->inode->i_ino, nid, max_addrs); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUMMARY); + return -EFSCORRUPTED; + } + if (dn->inode->i_ino == nid) { tdn.nid = nid; if (!dn->inode_page_locked) lock_page(dn->inode_page); tdn.node_page = dn->inode_page; - tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node); + tdn.ofs_in_node = ofs_in_node; goto truncate_out; } else if (dn->nid == nid) { - tdn.ofs_in_node = le16_to_cpu(sum.ofs_in_node); + tdn.ofs_in_node = ofs_in_node; goto truncate_out; } @@ -628,6 +638,7 @@ retry_dn: inode->i_ino, ofs_of_node(dn.node_page), ofs_of_node(page)); err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); goto err; } @@ -640,12 +651,14 @@ retry_dn: if (__is_valid_data_blkaddr(src) && !f2fs_is_valid_blkaddr(sbi, src, META_POR)) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto err; } if (__is_valid_data_blkaddr(dest) && !f2fs_is_valid_blkaddr(sbi, dest, META_POR)) { err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); goto err; } @@ -698,6 +711,16 @@ retry_prev: goto err; } + if (f2fs_is_valid_blkaddr(sbi, dest, + DATA_GENERIC_ENHANCE_UPDATE)) { + f2fs_err(sbi, "Inconsistent dest blkaddr:%u, ino:%lu, ofs:%u", + dest, inode->i_ino, dn.ofs_in_node); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, + ERROR_INVALID_BLKADDR); + goto err; + } + /* write dummy data page */ f2fs_replace_block(sbi, &dn, src, dest, ni.version, false, false); diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 0de21f82d7bc835c6df05e1ba8a6b8b6d7918e25..acf3d3fa4363578fac53964314361f4c97ac1a52 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -187,7 +187,6 @@ bool f2fs_need_SSR(struct f2fs_sb_info *sbi) void f2fs_abort_atomic_write(struct inode *inode, bool clean) { - struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct f2fs_inode_info *fi = F2FS_I(inode); if (!f2fs_is_atomic_file(inode)) @@ -200,10 +199,7 @@ void f2fs_abort_atomic_write(struct inode *inode, bool clean) fi->cow_inode = NULL; release_atomic_write_cnt(inode); clear_inode_flag(inode, FI_ATOMIC_FILE); - - spin_lock(&sbi->inode_lock[ATOMIC_FILE]); - sbi->atomic_files--; - spin_unlock(&sbi->inode_lock[ATOMIC_FILE]); + stat_dec_atomic_inode(inode); } static int __replace_atomic_write_block(struct inode *inode, pgoff_t index, @@ -312,6 +308,8 @@ static int __f2fs_commit_atomic_write(struct inode *inode) DATA_GENERIC_ENHANCE)) { f2fs_put_dnode(&dn); ret = -EFSCORRUPTED; + f2fs_handle_error(sbi, + ERROR_INVALID_BLKADDR); goto out; } @@ -376,7 +374,7 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) { if (time_to_inject(sbi, FAULT_CHECKPOINT)) { f2fs_show_injection_info(sbi, FAULT_CHECKPOINT); - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); } /* balance_fs_bg is able to be pending */ @@ -476,12 +474,12 @@ do_sync: mutex_lock(&sbi->flush_lock); blk_start_plug(&plug); - f2fs_sync_dirty_inodes(sbi, FILE_INODE); + f2fs_sync_dirty_inodes(sbi, FILE_INODE, false); blk_finish_plug(&plug); mutex_unlock(&sbi->flush_lock); } - f2fs_sync_fs(sbi->sb, true); + f2fs_sync_fs(sbi->sb, 1); stat_inc_bg_cp_count(sbi->stat_info); } @@ -694,7 +692,8 @@ int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) } while (ret && --count); if (ret) { - f2fs_stop_checkpoint(sbi, false); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FLUSH_FAIL); break; } @@ -1171,7 +1170,7 @@ submit: atomic_inc(&dcc->issued_discard); - f2fs_update_iostat(sbi, FS_DISCARD, 1); + f2fs_update_iostat(sbi, NULL, FS_DISCARD, 1); lstart += len; start += len; @@ -2535,7 +2534,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) sanity_check_seg_type(sbi, seg_type); if (f2fs_need_rand_seg(sbi)) - return prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec); + return prandom_u32_max(MAIN_SECS(sbi) * sbi->segs_per_sec); /* if segs_per_sec is large than 1, we need to keep original policy. */ if (__is_large_section(sbi)) @@ -2589,7 +2588,7 @@ static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) curseg->alloc_type = LFS; if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) curseg->fragment_remained_chunk = - prandom_u32() % sbi->max_fragment_chunk + 1; + prandom_u32_max(sbi->max_fragment_chunk) + 1; } static int __next_free_blkoff(struct f2fs_sb_info *sbi, @@ -2626,9 +2625,9 @@ static void __refresh_next_blkoff(struct f2fs_sb_info *sbi, /* To allocate block chunks in different sizes, use random number */ if (--seg->fragment_remained_chunk <= 0) { seg->fragment_remained_chunk = - prandom_u32() % sbi->max_fragment_chunk + 1; + prandom_u32_max(sbi->max_fragment_chunk) + 1; seg->next_blkoff += - prandom_u32() % sbi->max_fragment_hole + 1; + prandom_u32_max(sbi->max_fragment_hole) + 1; } } } @@ -3388,7 +3387,7 @@ void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page, f2fs_submit_page_write(&fio); stat_inc_meta_count(sbi, page->index); - f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE); } void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) @@ -3398,7 +3397,7 @@ void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) set_summary(&sum, nid, 0, 0); do_write_page(&sum, fio); - f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); + f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE); } void f2fs_outplace_write_data(struct dnode_of_data *dn, @@ -3412,7 +3411,7 @@ void f2fs_outplace_write_data(struct dnode_of_data *dn, do_write_page(&sum, fio); f2fs_update_data_blkaddr(dn, fio->new_blkaddr); - f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE); } int f2fs_inplace_write_data(struct f2fs_io_info *fio) @@ -3432,6 +3431,7 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", __func__, segno); err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); goto drop_bio; } @@ -3453,7 +3453,8 @@ int f2fs_inplace_write_data(struct f2fs_io_info *fio) if (!err) { f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); - f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE); + f2fs_update_iostat(fio->sbi, fio->page->mapping->host, + fio->io_type, F2FS_BLKSIZE); } return err; @@ -4379,6 +4380,8 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) if (se->type >= NR_PERSISTENT_LOG) { f2fs_err(sbi, "Invalid segment type: %u, segno: %u", se->type, start); + f2fs_handle_error(sbi, + ERROR_INCONSISTENT_SUM_TYPE); return -EFSCORRUPTED; } @@ -4415,6 +4418,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) f2fs_err(sbi, "Wrong journal entry on segno %u", start); err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL); break; } @@ -4434,6 +4438,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) f2fs_err(sbi, "Invalid segment type: %u, segno: %u", se->type, start); err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); break; } @@ -4465,6 +4470,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) if (sit_valid_blocks[NODE] != valid_node_count(sbi)) { f2fs_err(sbi, "SIT is corrupted node# %u vs %u", sit_valid_blocks[NODE], valid_node_count(sbi)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT); return -EFSCORRUPTED; } @@ -4473,6 +4479,7 @@ static int build_sit_entries(struct f2fs_sb_info *sbi) f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u", sit_valid_blocks[DATA], sit_valid_blocks[NODE], valid_user_blocks(sbi)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT); return -EFSCORRUPTED; } @@ -4623,6 +4630,7 @@ static int sanity_check_curseg(struct f2fs_sb_info *sbi) f2fs_err(sbi, "Current segment has invalid alloc_type:%d", curseg->alloc_type); + f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); return -EFSCORRUPTED; } @@ -4640,6 +4648,7 @@ out: "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", i, curseg->segno, curseg->alloc_type, curseg->next_blkoff, blkofs); + f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); return -EFSCORRUPTED; } } diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index d1d63766f2c7e57cded48edcb67798c9217dd69d..be8f2d7d007b9eb54cf1b7fabd60ffc3696cab96 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -753,6 +753,7 @@ static inline int check_block_count(struct f2fs_sb_info *sbi, f2fs_err(sbi, "Mismatch valid blocks %d vs. %d", GET_SIT_VBLOCKS(raw_sit), valid_blocks); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT); return -EFSCORRUPTED; } @@ -767,6 +768,7 @@ static inline int check_block_count(struct f2fs_sb_info *sbi, f2fs_err(sbi, "Wrong valid blocks %d or segno %u", GET_SIT_VBLOCKS(raw_sit), segno); set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT); return -EFSCORRUPTED; } return 0; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 2451623c05a7a855879e53cf9f4210545678efd1..3834ead046200ac428b36135f5bcf1f17d735a78 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -301,10 +301,10 @@ static void f2fs_destroy_casefold_cache(void) { } static inline void limit_reserve_root(struct f2fs_sb_info *sbi) { - block_t limit = min((sbi->user_block_count << 1) / 1000, + block_t limit = min((sbi->user_block_count >> 3), sbi->user_block_count - sbi->reserved_blocks); - /* limit is 0.2% */ + /* limit is 12.5% */ if (test_opt(sbi, RESERVE_ROOT) && F2FS_OPTION(sbi).root_reserved_blocks > limit) { F2FS_OPTION(sbi).root_reserved_blocks = limit; @@ -1342,6 +1342,11 @@ default_check: return -EINVAL; } + if (test_opt(sbi, ATGC) && f2fs_lfs_mode(sbi)) { + f2fs_err(sbi, "LFS not compatible with ATGC"); + return -EINVAL; + } + if (f2fs_sb_has_readonly(sbi) && !f2fs_readonly(sbi->sb)) { f2fs_err(sbi, "Allow to mount readonly mode only"); return -EROFS; @@ -1666,9 +1671,8 @@ static int f2fs_freeze(struct super_block *sb) if (is_sbi_flag_set(F2FS_SB(sb), SBI_IS_DIRTY)) return -EINVAL; - /* ensure no checkpoint required */ - if (!llist_empty(&F2FS_SB(sb)->cprc_info.issue_list)) - return -EINVAL; + /* Let's flush checkpoints and stop the thread. */ + f2fs_flush_ckpt_thread(F2FS_SB(sb)); /* to avoid deadlock on f2fs_evict_inode->SB_FREEZE_FS */ set_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); @@ -2181,6 +2185,9 @@ static void f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) f2fs_up_write(&sbi->gc_lock); f2fs_sync_fs(sbi->sb, 1); + + /* Let's ensure there's no pending checkpoint anymore */ + f2fs_flush_ckpt_thread(sbi); } static int f2fs_remount(struct super_block *sb, int *flags, char *data) @@ -2346,6 +2353,9 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data) f2fs_stop_ckpt_thread(sbi); need_restart_ckpt = true; } else { + /* Flush if the prevous checkpoint, if exists. */ + f2fs_flush_ckpt_thread(sbi); + err = f2fs_start_ckpt_thread(sbi); if (err) { f2fs_err(sbi, @@ -2465,7 +2475,6 @@ static ssize_t f2fs_quota_read(struct super_block *sb, int type, char *data, size_t toread; loff_t i_size = i_size_read(inode); struct page *page; - char *kaddr; if (off > i_size) return 0; @@ -2498,9 +2507,7 @@ repeat: return -EIO; } - kaddr = kmap_atomic(page); - memcpy(data, kaddr + offset, tocopy); - kunmap_atomic(kaddr); + memcpy_from_page(data, page, offset, tocopy); f2fs_put_page(page, 1); offset = 0; @@ -2522,7 +2529,6 @@ static ssize_t f2fs_quota_write(struct super_block *sb, int type, size_t towrite = len; struct page *page; void *fsdata = NULL; - char *kaddr; int err = 0; int tocopy; @@ -2541,10 +2547,7 @@ retry: break; } - kaddr = kmap_atomic(page); - memcpy(kaddr + offset, data, tocopy); - kunmap_atomic(kaddr); - flush_dcache_page(page); + memcpy_to_page(page, offset, data, tocopy); a_ops->write_end(NULL, mapping, off, tocopy, tocopy, page, fsdata); @@ -3039,23 +3042,24 @@ static void f2fs_get_ino_and_lblk_bits(struct super_block *sb, *lblk_bits_ret = 8 * sizeof(block_t); } -static int f2fs_get_num_devices(struct super_block *sb) +static struct block_device **f2fs_get_devices(struct super_block *sb, + unsigned int *num_devs) { struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct block_device **devs; + int i; - if (f2fs_is_multi_device(sbi)) - return sbi->s_ndevs; - return 1; -} + if (!f2fs_is_multi_device(sbi)) + return NULL; -static void f2fs_get_devices(struct super_block *sb, - struct request_queue **devs) -{ - struct f2fs_sb_info *sbi = F2FS_SB(sb); - int i; + devs = kmalloc_array(sbi->s_ndevs, sizeof(*devs), GFP_KERNEL); + if (!devs) + return ERR_PTR(-ENOMEM); for (i = 0; i < sbi->s_ndevs; i++) - devs[i] = bdev_get_queue(FDEV(i).bdev); + devs[i] = FDEV(i).bdev; + *num_devs = sbi->s_ndevs; + return devs; } static const struct fscrypt_operations f2fs_cryptops = { @@ -3066,7 +3070,6 @@ static const struct fscrypt_operations f2fs_cryptops = { .empty_dir = f2fs_empty_dir, .has_stable_inodes = f2fs_has_stable_inodes, .get_ino_and_lblk_bits = f2fs_get_ino_and_lblk_bits, - .get_num_devices = f2fs_get_num_devices, .get_devices = f2fs_get_devices, }; #endif @@ -3843,6 +3846,68 @@ int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover) return err; } +void f2fs_handle_stop(struct f2fs_sb_info *sbi, unsigned char reason) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + int err; + + f2fs_down_write(&sbi->sb_lock); + + if (raw_super->s_stop_reason[reason] < ((1 << BITS_PER_BYTE) - 1)) + raw_super->s_stop_reason[reason]++; + + err = f2fs_commit_super(sbi, false); + if (err) + f2fs_err(sbi, "f2fs_commit_super fails to record reason:%u err:%d", + reason, err); + f2fs_up_write(&sbi->sb_lock); +} + +static void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag) +{ + spin_lock(&sbi->error_lock); + if (!test_bit(flag, (unsigned long *)sbi->errors)) { + set_bit(flag, (unsigned long *)sbi->errors); + sbi->error_dirty = true; + } + spin_unlock(&sbi->error_lock); +} + +static bool f2fs_update_errors(struct f2fs_sb_info *sbi) +{ + bool need_update = false; + + spin_lock(&sbi->error_lock); + if (sbi->error_dirty) { + memcpy(F2FS_RAW_SUPER(sbi)->s_errors, sbi->errors, + MAX_F2FS_ERRORS); + sbi->error_dirty = false; + need_update = true; + } + spin_unlock(&sbi->error_lock); + + return need_update; +} + +void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error) +{ + int err; + + f2fs_save_errors(sbi, error); + + f2fs_down_write(&sbi->sb_lock); + + if (!f2fs_update_errors(sbi)) + goto out_unlock; + + err = f2fs_commit_super(sbi, false); + if (err) + f2fs_err(sbi, "f2fs_commit_super fails to record errors:%u, err:%d", + error, err); +out_unlock: + f2fs_up_write(&sbi->sb_lock); +} + static int f2fs_scan_devices(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); @@ -4190,6 +4255,9 @@ try_onemore: goto free_devices; } + spin_lock_init(&sbi->error_lock); + memcpy(sbi->errors, raw_super->s_errors, MAX_F2FS_ERRORS); + sbi->total_valid_node_count = le32_to_cpu(sbi->ckpt->valid_node_count); percpu_counter_set(&sbi->total_valid_inode_count, diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index eba5fb1629d7145479601bcc6f94e4016bc2df6d..df27afd71ef48ae954d23ee320481f93e1245842 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -128,6 +128,12 @@ static ssize_t sb_status_show(struct f2fs_attr *a, return sprintf(buf, "%lx\n", sbi->s_flag); } +static ssize_t cp_status_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sprintf(buf, "%x\n", le32_to_cpu(F2FS_CKPT(sbi)->ckpt_flags)); +} + static ssize_t pending_discard_show(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf) { @@ -527,7 +533,6 @@ out: if (!strcmp(a->attr.name, "gc_urgent_high_remaining")) { spin_lock(&sbi->gc_urgent_high_lock); - sbi->gc_urgent_high_limited = t != 0; sbi->gc_urgent_high_remaining = t; spin_unlock(&sbi->gc_urgent_high_lock); @@ -1030,8 +1035,10 @@ static struct attribute *f2fs_feat_attrs[] = { ATTRIBUTE_GROUPS(f2fs_feat); F2FS_GENERAL_RO_ATTR(sb_status); +F2FS_GENERAL_RO_ATTR(cp_status); static struct attribute *f2fs_stat_attrs[] = { ATTR_LIST(sb_status), + ATTR_LIST(cp_status), NULL, }; ATTRIBUTE_GROUPS(f2fs_stat); diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c index 7b8f2b41c29b12fb79056c325a2ca5a933930f57..c352fff88a5e65e65dbb37f77201197c84161767 100644 --- a/fs/f2fs/verity.c +++ b/fs/f2fs/verity.c @@ -47,16 +47,13 @@ static int pagecache_read(struct inode *inode, void *buf, size_t count, size_t n = min_t(size_t, count, PAGE_SIZE - offset_in_page(pos)); struct page *page; - void *addr; page = read_mapping_page(inode->i_mapping, pos >> PAGE_SHIFT, NULL); if (IS_ERR(page)) return PTR_ERR(page); - addr = kmap_atomic(page); - memcpy(buf, addr + offset_in_page(pos), n); - kunmap_atomic(addr); + memcpy_from_page(buf, page, offset_in_page(pos), n); put_page(page); @@ -85,16 +82,13 @@ static int pagecache_write(struct inode *inode, const void *buf, size_t count, PAGE_SIZE - offset_in_page(pos)); struct page *page; void *fsdata; - void *addr; int res; res = aops->write_begin(NULL, mapping, pos, n, &page, &fsdata); if (res) return res; - addr = kmap_atomic(page); - memcpy(addr + offset_in_page(pos), buf, n); - kunmap_atomic(addr); + memcpy_to_page(page, offset_in_page(pos), buf, n); res = aops->write_end(NULL, mapping, pos, n, n, page, fsdata); if (res < 0) @@ -246,6 +240,8 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, if (pos + size < pos || pos + size > inode->i_sb->s_maxbytes || pos < f2fs_verity_metadata_pos(inode) || size > INT_MAX) { f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr"); + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_VERITY_XATTR); return -EFSCORRUPTED; } if (buf_size) { @@ -262,13 +258,14 @@ static struct page *f2fs_read_merkle_tree_page(struct inode *inode, pgoff_t index, unsigned long num_ra_pages) { - DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); struct page *page; index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; page = find_get_page_flags(inode->i_mapping, index, FGP_ACCESSED); if (!page || !PageUptodate(page)) { + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); + if (page) put_page(page); else if (num_ra_pages > 1) diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index c76c15086e5f5bdf98227c9421e43a6eb684d394..dc2e8637189e2e2541507888a7ea2858ed4289d5 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -367,6 +367,8 @@ static int lookup_all_xattrs(struct inode *inode, struct page *ipage, inode->i_ino); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); err = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); goto out; } check: @@ -583,6 +585,8 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) inode->i_ino); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); error = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); goto cleanup; } @@ -658,6 +662,8 @@ static int __f2fs_setxattr(struct inode *inode, int index, inode->i_ino); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); error = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); goto exit; } @@ -684,6 +690,8 @@ static int __f2fs_setxattr(struct inode *inode, int index, inode->i_ino, ENTRY_SIZE(last)); set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); error = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); goto exit; } last = XATTR_NEXT_ENTRY(last); diff --git a/fs/fat/dir.c b/fs/fat/dir.c index 249825017da759a338fcd575cce47fab7e6b1c3b..00235b8a1823af6b7d6a54cdb12439d2f3365e04 100644 --- a/fs/fat/dir.c +++ b/fs/fat/dir.c @@ -705,7 +705,7 @@ static int fat_readdir(struct file *file, struct dir_context *ctx) } #define FAT_IOCTL_FILLDIR_FUNC(func, dirent_type) \ -static int func(struct dir_context *ctx, const char *name, int name_len, \ +static bool func(struct dir_context *ctx, const char *name, int name_len, \ loff_t offset, u64 ino, unsigned int d_type) \ { \ struct fat_ioctl_filldir_callback *buf = \ @@ -714,7 +714,7 @@ static int func(struct dir_context *ctx, const char *name, int name_len, \ struct dirent_type __user *d2 = d1 + 1; \ \ if (buf->result) \ - return -EINVAL; \ + return false; \ buf->result++; \ \ if (name != NULL) { \ @@ -750,10 +750,10 @@ static int func(struct dir_context *ctx, const char *name, int name_len, \ put_user(short_len, &d1->d_reclen)) \ goto efault; \ } \ - return 0; \ + return true; \ efault: \ buf->result = -EFAULT; \ - return -EFAULT; \ + return false; \ } FAT_IOCTL_FILLDIR_FUNC(fat_ioctl_filldir, __fat_dirent) diff --git a/fs/fat/file.c b/fs/fat/file.c index 3e4eb3467cb44beb276b91f82dded1fe1d0ad331..8a6b493b5b5f64f7c22ae54ddb92addeb1605ce8 100644 --- a/fs/fat/file.c +++ b/fs/fat/file.c @@ -461,8 +461,9 @@ static int fat_allow_set_time(struct user_namespace *mnt_userns, { umode_t allow_utime = sbi->options.allow_utime; - if (!uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode))) { - if (in_group_p(i_gid_into_mnt(mnt_userns, inode))) + if (!vfsuid_eq_kuid(i_uid_into_vfsuid(mnt_userns, inode), + current_fsuid())) { + if (vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode))) allow_utime >>= 3; if (allow_utime & MAY_WRITE) return 1; diff --git a/fs/fat/inode.c b/fs/fat/inode.c index a38238d75c08ed1b415efd7b1de3061cf3bbde22..1cbcc4608dc78f02c9cb8e7919a0f43a214b2f73 100644 --- a/fs/fat/inode.c +++ b/fs/fat/inode.c @@ -523,7 +523,7 @@ int fat_fill_inode(struct inode *inode, struct msdos_dir_entry *de) inode->i_uid = sbi->options.fs_uid; inode->i_gid = sbi->options.fs_gid; inode_inc_iversion(inode); - inode->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); if ((de->attr & ATTR_DIR) && !IS_FREE(de->name)) { inode->i_generation &= ~1; diff --git a/fs/fhandle.c b/fs/fhandle.c index 6630c69c23a2a357fe21947a207afaaac77072bf..f2bc27d1975e14bd45da242fe25b083b32b2e015 100644 --- a/fs/fhandle.c +++ b/fs/fhandle.c @@ -14,7 +14,7 @@ #include "internal.h" #include "mount.h" -static long do_sys_name_to_handle(struct path *path, +static long do_sys_name_to_handle(const struct path *path, struct file_handle __user *ufh, int __user *mnt_id) { diff --git a/fs/file.c b/fs/file.c index 3bcc1ecc314a78e90daedecf6a7ec575c642504a..5f9c802a5d8d34454f2382196ef9d81b43e76bd9 100644 --- a/fs/file.c +++ b/fs/file.c @@ -980,6 +980,7 @@ struct file *task_lookup_next_fd_rcu(struct task_struct *task, unsigned int *ret *ret_fd = fd; return file; } +EXPORT_SYMBOL(task_lookup_next_fd_rcu); /* * Lightweight file lookup - no refcnt increment if fd table isn't shared. diff --git a/fs/file_table.c b/fs/file_table.c index 99c6796c9f28a67eb897611832584d77b95021b9..dd88701e54a93aa33958d03b1dbca5715e290c92 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -324,12 +324,7 @@ static void __fput(struct file *file) } fops_put(file->f_op); put_pid(file->f_owner.pid); - if ((mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) - i_readcount_dec(inode); - if (mode & FMODE_WRITER) { - put_write_access(inode); - __mnt_drop_write(mnt); - } + put_file_access(file); dput(dentry); if (unlikely(mode & FMODE_NEED_UNMOUNT)) dissolve_on_fput(mnt); diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 05221366a16dc913961f3e3f7b4a703477bc7bb9..443f83382b9bdda308451ab285dccec25b4e9edc 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -134,10 +134,10 @@ static bool inode_io_list_move_locked(struct inode *inode, static void wb_wakeup(struct bdi_writeback *wb) { - spin_lock_bh(&wb->work_lock); + spin_lock_irq(&wb->work_lock); if (test_bit(WB_registered, &wb->state)) mod_delayed_work(bdi_wq, &wb->dwork, 0); - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); } static void finish_writeback_work(struct bdi_writeback *wb, @@ -164,7 +164,7 @@ static void wb_queue_work(struct bdi_writeback *wb, if (work->done) atomic_inc(&work->done->cnt); - spin_lock_bh(&wb->work_lock); + spin_lock_irq(&wb->work_lock); if (test_bit(WB_registered, &wb->state)) { list_add_tail(&work->list, &wb->work_list); @@ -172,7 +172,7 @@ static void wb_queue_work(struct bdi_writeback *wb, } else finish_writeback_work(wb, work); - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); } /** @@ -1718,9 +1718,14 @@ static int writeback_single_inode(struct inode *inode, */ if (!(inode->i_state & I_DIRTY_ALL)) inode_cgwb_move_to_attached(inode, wb); - else if (!(inode->i_state & I_SYNC_QUEUED) && - (inode->i_state & I_DIRTY)) - redirty_tail_locked(inode, wb); + else if (!(inode->i_state & I_SYNC_QUEUED)) { + if ((inode->i_state & I_DIRTY)) + redirty_tail_locked(inode, wb); + else if (inode->i_state & I_DIRTY_TIME) { + inode->dirtied_when = jiffies; + inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); + } + } spin_unlock(&wb->list_lock); inode_sync_complete(inode); @@ -2082,13 +2087,13 @@ static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb) { struct wb_writeback_work *work = NULL; - spin_lock_bh(&wb->work_lock); + spin_lock_irq(&wb->work_lock); if (!list_empty(&wb->work_list)) { work = list_entry(wb->work_list.next, struct wb_writeback_work, list); list_del_init(&work->list); } - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); return work; } @@ -2369,6 +2374,20 @@ void __mark_inode_dirty(struct inode *inode, int flags) trace_writeback_mark_inode_dirty(inode, flags); if (flags & I_DIRTY_INODE) { + /* + * Inode timestamp update will piggback on this dirtying. + * We tell ->dirty_inode callback that timestamps need to + * be updated by setting I_DIRTY_TIME in flags. + */ + if (inode->i_state & I_DIRTY_TIME) { + spin_lock(&inode->i_lock); + if (inode->i_state & I_DIRTY_TIME) { + inode->i_state &= ~I_DIRTY_TIME; + flags |= I_DIRTY_TIME; + } + spin_unlock(&inode->i_lock); + } + /* * Notify the filesystem about the inode being dirtied, so that * (if needed) it can update on-disk fields and journal the @@ -2378,7 +2397,8 @@ void __mark_inode_dirty(struct inode *inode, int flags) */ trace_writeback_dirty_inode_start(inode, flags); if (sb->s_op->dirty_inode) - sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE); + sb->s_op->dirty_inode(inode, + flags & (I_DIRTY_INODE | I_DIRTY_TIME)); trace_writeback_dirty_inode(inode, flags); /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ @@ -2399,21 +2419,15 @@ void __mark_inode_dirty(struct inode *inode, int flags) */ smp_mb(); - if (((inode->i_state & flags) == flags) || - (dirtytime && (inode->i_state & I_DIRTY_INODE))) + if ((inode->i_state & flags) == flags) return; spin_lock(&inode->i_lock); - if (dirtytime && (inode->i_state & I_DIRTY_INODE)) - goto out_unlock_inode; if ((inode->i_state & flags) != flags) { const int was_dirty = inode->i_state & I_DIRTY; inode_attach_wb(inode, NULL); - /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ - if (flags & I_DIRTY_INODE) - inode->i_state &= ~I_DIRTY_TIME; inode->i_state |= flags; /* @@ -2486,7 +2500,6 @@ void __mark_inode_dirty(struct inode *inode, int flags) out_unlock: if (wb) spin_unlock(&wb->list_lock); -out_unlock_inode: spin_unlock(&inode->i_lock); } EXPORT_SYMBOL(__mark_inode_dirty); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 51897427a5346ed2377ec92857154601c24917e3..b4a6e0a1b945aaf82eb3f141a2b188307379da7e 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -776,7 +776,8 @@ static int fuse_check_page(struct page *page) 1 << PG_active | 1 << PG_workingset | 1 << PG_reclaim | - 1 << PG_waiters))) { + 1 << PG_waiters | + LRU_GEN_MASK | LRU_REFS_MASK))) { dump_page(page, "fuse: trying to steal weird page"); return 1; } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index b585b04e815e0fa74276c315e941c74a4eb9c8f5..bb97a384dc5dd8b8caeba91c3e9bc6f21fefc342 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -529,7 +529,7 @@ out_err: */ static int fuse_create_open(struct inode *dir, struct dentry *entry, struct file *file, unsigned int flags, - umode_t mode) + umode_t mode, u32 opcode) { int err; struct inode *inode; @@ -573,7 +573,7 @@ static int fuse_create_open(struct inode *dir, struct dentry *entry, inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; } - args.opcode = FUSE_CREATE; + args.opcode = opcode; args.nodeid = get_node_id(dir); args.in_numargs = 2; args.in_args[0].size = sizeof(inarg); @@ -676,7 +676,7 @@ static int fuse_atomic_open(struct inode *dir, struct dentry *entry, if (fc->no_create) goto mknod; - err = fuse_create_open(dir, entry, file, flags, mode); + err = fuse_create_open(dir, entry, file, flags, mode, FUSE_CREATE); if (err == -ENOSYS) { fc->no_create = 1; goto mknod; @@ -802,6 +802,23 @@ static int fuse_create(struct user_namespace *mnt_userns, struct inode *dir, return fuse_mknod(&init_user_ns, dir, entry, mode, 0); } +static int fuse_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, + struct file *file, umode_t mode) +{ + struct fuse_conn *fc = get_fuse_conn(dir); + int err; + + if (fc->no_tmpfile) + return -EOPNOTSUPP; + + err = fuse_create_open(dir, file->f_path.dentry, file, file->f_flags, mode, FUSE_TMPFILE); + if (err == -ENOSYS) { + fc->no_tmpfile = 1; + err = -EOPNOTSUPP; + } + return err; +} + static int fuse_mkdir(struct user_namespace *mnt_userns, struct inode *dir, struct dentry *entry, umode_t mode) { @@ -1913,6 +1930,7 @@ static const struct inode_operations fuse_dir_inode_operations = { .setattr = fuse_setattr, .create = fuse_create, .atomic_open = fuse_atomic_open, + .tmpfile = fuse_tmpfile, .mknod = fuse_mknod, .permission = fuse_permission, .getattr = fuse_getattr, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 488b460e046f4c679cf7b9c328bca6d9e9528774..98a9cf53187311e3ceb1c90b0db516bd73389485 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -784,6 +784,9 @@ struct fuse_conn { /* Does the filesystem support per inode DAX? */ unsigned int inode_dax:1; + /* Is tmpfile not implemented by fs? */ + unsigned int no_tmpfile:1; + /** The number of requests waiting for completion */ atomic_t num_waiting; diff --git a/fs/gfs2/export.c b/fs/gfs2/export.c index 756d05779200dcdac43f67daa0fea9e9ccd3ff6d..cf40895233f5a88147d4dff2caf79b38794e40d9 100644 --- a/fs/gfs2/export.c +++ b/fs/gfs2/export.c @@ -66,7 +66,7 @@ struct get_name_filldir { char *name; }; -static int get_name_filldir(struct dir_context *ctx, const char *name, +static bool get_name_filldir(struct dir_context *ctx, const char *name, int length, loff_t offset, u64 inum, unsigned int type) { @@ -74,12 +74,12 @@ static int get_name_filldir(struct dir_context *ctx, const char *name, container_of(ctx, struct get_name_filldir, ctx); if (inum != gnfd->inum.no_addr) - return 0; + return true; memcpy(gnfd->name, name, length); gnfd->name[length] = 0; - return 1; + return false; } static int gfs2_get_name(struct dentry *parent, char *name, diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index 892006fbbb09f636e632bce3f5b1b4d911eb8e8a..60c6fb91fb589d4f26ada676a31a883a5145484e 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -1443,6 +1443,22 @@ static int gfs2_lock(struct file *file, int cmd, struct file_lock *fl) return dlm_posix_lock(ls->ls_dlm, ip->i_no_addr, file, cmd, fl); } +static void __flock_holder_uninit(struct file *file, struct gfs2_holder *fl_gh) +{ + struct gfs2_glock *gl = fl_gh->gh_gl; + + /* + * Make sure gfs2_glock_put() won't sleep under the file->f_lock + * spinlock. + */ + + gfs2_glock_hold(gl); + spin_lock(&file->f_lock); + gfs2_holder_uninit(fl_gh); + spin_unlock(&file->f_lock); + gfs2_glock_put(gl); +} + static int do_flock(struct file *file, int cmd, struct file_lock *fl) { struct gfs2_file *fp = file->private_data; @@ -1455,7 +1471,9 @@ static int do_flock(struct file *file, int cmd, struct file_lock *fl) int sleeptime; state = (fl->fl_type == F_WRLCK) ? LM_ST_EXCLUSIVE : LM_ST_SHARED; - flags = (IS_SETLKW(cmd) ? 0 : LM_FLAG_TRY_1CB) | GL_EXACT; + flags = GL_EXACT | GL_NOPID; + if (!IS_SETLKW(cmd)) + flags |= LM_FLAG_TRY_1CB; mutex_lock(&fp->f_fl_mutex); @@ -1474,18 +1492,21 @@ static int do_flock(struct file *file, int cmd, struct file_lock *fl) &gfs2_flock_glops, CREATE, &gl); if (error) goto out; + spin_lock(&file->f_lock); gfs2_holder_init(gl, state, flags, fl_gh); + spin_unlock(&file->f_lock); gfs2_glock_put(gl); } for (sleeptime = 1; sleeptime <= 4; sleeptime <<= 1) { error = gfs2_glock_nq(fl_gh); if (error != GLR_TRYFAILED) break; - fl_gh->gh_flags = LM_FLAG_TRY | GL_EXACT; + fl_gh->gh_flags &= ~LM_FLAG_TRY_1CB; + fl_gh->gh_flags |= LM_FLAG_TRY; msleep(sleeptime); } if (error) { - gfs2_holder_uninit(fl_gh); + __flock_holder_uninit(file, fl_gh); if (error == GLR_TRYFAILED) error = -EAGAIN; } else { @@ -1507,7 +1528,7 @@ static void do_unflock(struct file *file, struct file_lock *fl) locks_lock_file_wait(file, fl); if (gfs2_holder_initialized(fl_gh)) { gfs2_glock_dq(fl_gh); - gfs2_holder_uninit(fl_gh); + __flock_holder_uninit(file, fl_gh); } mutex_unlock(&fp->f_fl_mutex); } diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c index 41b6c89e4bf7dfd39dbe26a15a3bc119e4d38e1b..df335c258eb0832c6bbcfbda2db4e9dd1aa4161d 100644 --- a/fs/gfs2/glock.c +++ b/fs/gfs2/glock.c @@ -33,6 +33,9 @@ #include #include #include +#include +#include +#include #include "gfs2.h" #include "incore.h" @@ -59,6 +62,8 @@ typedef void (*glock_examiner) (struct gfs2_glock * gl); static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target); static void __gfs2_glock_dq(struct gfs2_holder *gh); +static void handle_callback(struct gfs2_glock *gl, unsigned int state, + unsigned long delay, bool remote); static struct dentry *gfs2_root; static struct workqueue_struct *glock_workqueue; @@ -730,7 +735,8 @@ static bool is_system_glock(struct gfs2_glock *gl) * */ -static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target) +static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, + unsigned int target) __releases(&gl->gl_lockref.lock) __acquires(&gl->gl_lockref.lock) { @@ -741,7 +747,8 @@ __acquires(&gl->gl_lockref.lock) if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) && gh && !(gh->gh_flags & LM_FLAG_NOEXP)) - return; + goto skip_inval; + lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP | LM_FLAG_PRIORITY); GLOCK_BUG_ON(gl, gl->gl_state == target); @@ -826,6 +833,20 @@ skip_inval: (target != LM_ST_UNLOCKED || test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) { if (!is_system_glock(gl)) { + handle_callback(gl, LM_ST_UNLOCKED, 0, false); /* sets demote */ + /* + * Ordinarily, we would call dlm and its callback would call + * finish_xmote, which would call state_change() to the new state. + * Since we withdrew, we won't call dlm, so call state_change + * manually, but to the UNLOCKED state we desire. + */ + state_change(gl, LM_ST_UNLOCKED); + /* + * We skip telling dlm to do the locking, so we won't get a + * reply that would otherwise clear GLF_LOCK. So we clear it here. + */ + clear_bit(GLF_LOCK, &gl->gl_flags); + clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD); goto out; } else { @@ -1018,16 +1039,18 @@ static void delete_work_func(struct work_struct *work) if (gfs2_queue_delete_work(gl, 5 * HZ)) return; } - goto out; } inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino, GFS2_BLKST_UNLINKED); - if (!IS_ERR_OR_NULL(inode)) { + if (IS_ERR(inode)) { + if (PTR_ERR(inode) == -EAGAIN && + (gfs2_queue_delete_work(gl, 5 * HZ))) + return; + } else { d_prune_aliases(inode); iput(inode); } -out: gfs2_glock_put(gl); } @@ -1436,6 +1459,15 @@ void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...) va_end(args); } +static inline bool pid_is_meaningful(const struct gfs2_holder *gh) +{ + if (!(gh->gh_flags & GL_NOPID)) + return true; + if (gh->gh_state == LM_ST_UNLOCKED) + return true; + return false; +} + /** * add_to_queue - Add a holder to the wait queue (but look for recursion) * @gh: the holder structure to add @@ -1472,10 +1504,17 @@ __acquires(&gl->gl_lockref.lock) } list_for_each_entry(gh2, &gl->gl_holders, gh_list) { - if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid && - (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK) && - !test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags))) - goto trap_recursive; + if (likely(gh2->gh_owner_pid != gh->gh_owner_pid)) + continue; + if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK) + continue; + if (test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags)) + continue; + if (!pid_is_meaningful(gh2)) + continue; + goto trap_recursive; + } + list_for_each_entry(gh2, &gl->gl_holders, gh_list) { if (try_futile && !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) { fail: @@ -2194,6 +2233,20 @@ static void dump_glock_func(struct gfs2_glock *gl) dump_glock(NULL, gl, true); } +static void withdraw_dq(struct gfs2_glock *gl) +{ + spin_lock(&gl->gl_lockref.lock); + if (!__lockref_is_dead(&gl->gl_lockref) && + glock_blocked_by_withdraw(gl)) + do_error(gl, LM_OUT_ERROR); /* remove pending waiters */ + spin_unlock(&gl->gl_lockref.lock); +} + +void gfs2_gl_dq_holders(struct gfs2_sbd *sdp) +{ + glock_hash_walk(withdraw_dq, sdp); +} + /** * gfs2_gl_hash_clear - Empty out the glock hash table * @sdp: the filesystem @@ -2272,19 +2325,24 @@ static const char *hflags2str(char *buf, u16 flags, unsigned long iflags) static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh, const char *fs_id_buf) { - struct task_struct *gh_owner = NULL; + const char *comm = "(none)"; + pid_t owner_pid = 0; char flags_buf[32]; rcu_read_lock(); - if (gh->gh_owner_pid) + if (pid_is_meaningful(gh)) { + struct task_struct *gh_owner; + + comm = "(ended)"; + owner_pid = pid_nr(gh->gh_owner_pid); gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID); + if (gh_owner) + comm = gh_owner->comm; + } gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n", fs_id_buf, state2str(gh->gh_state), hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags), - gh->gh_error, - gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1, - gh_owner ? gh_owner->comm : "(ended)", - (void *)gh->gh_ip); + gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip); rcu_read_unlock(); } @@ -2699,6 +2757,172 @@ static const struct file_operations gfs2_glstats_fops = { .release = gfs2_glocks_release, }; +struct gfs2_glockfd_iter { + struct super_block *sb; + unsigned int tgid; + struct task_struct *task; + unsigned int fd; + struct file *file; +}; + +static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i) +{ + struct pid_namespace *ns = task_active_pid_ns(current); + struct pid *pid; + + if (i->task) + put_task_struct(i->task); + + rcu_read_lock(); +retry: + i->task = NULL; + pid = find_ge_pid(i->tgid, ns); + if (pid) { + i->tgid = pid_nr_ns(pid, ns); + i->task = pid_task(pid, PIDTYPE_TGID); + if (!i->task) { + i->tgid++; + goto retry; + } + get_task_struct(i->task); + } + rcu_read_unlock(); + return i->task; +} + +static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i) +{ + if (i->file) { + fput(i->file); + i->file = NULL; + } + + rcu_read_lock(); + for(;; i->fd++) { + struct inode *inode; + + i->file = task_lookup_next_fd_rcu(i->task, &i->fd); + if (!i->file) { + i->fd = 0; + break; + } + inode = file_inode(i->file); + if (inode->i_sb != i->sb) + continue; + if (get_file_rcu(i->file)) + break; + } + rcu_read_unlock(); + return i->file; +} + +static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct gfs2_glockfd_iter *i = seq->private; + + if (*pos) + return NULL; + while (gfs2_glockfd_next_task(i)) { + if (gfs2_glockfd_next_file(i)) + return i; + i->tgid++; + } + return NULL; +} + +static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr, + loff_t *pos) +{ + struct gfs2_glockfd_iter *i = seq->private; + + (*pos)++; + i->fd++; + do { + if (gfs2_glockfd_next_file(i)) + return i; + i->tgid++; + } while (gfs2_glockfd_next_task(i)); + return NULL; +} + +static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_glockfd_iter *i = seq->private; + + if (i->file) + fput(i->file); + if (i->task) + put_task_struct(i->task); +} + +static void gfs2_glockfd_seq_show_flock(struct seq_file *seq, + struct gfs2_glockfd_iter *i) +{ + struct gfs2_file *fp = i->file->private_data; + struct gfs2_holder *fl_gh = &fp->f_fl_gh; + struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED }; + + if (!READ_ONCE(fl_gh->gh_gl)) + return; + + spin_lock(&i->file->f_lock); + if (gfs2_holder_initialized(fl_gh)) + gl_name = fl_gh->gh_gl->gl_name; + spin_unlock(&i->file->f_lock); + + if (gl_name.ln_type != LM_TYPE_RESERVED) { + seq_printf(seq, "%d %u %u/%llx\n", + i->tgid, i->fd, gl_name.ln_type, + (unsigned long long)gl_name.ln_number); + } +} + +static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_glockfd_iter *i = seq->private; + struct inode *inode = file_inode(i->file); + struct gfs2_glock *gl; + + inode_lock_shared(inode); + gl = GFS2_I(inode)->i_iopen_gh.gh_gl; + if (gl) { + seq_printf(seq, "%d %u %u/%llx\n", + i->tgid, i->fd, gl->gl_name.ln_type, + (unsigned long long)gl->gl_name.ln_number); + } + gfs2_glockfd_seq_show_flock(seq, i); + inode_unlock_shared(inode); + return 0; +} + +static const struct seq_operations gfs2_glockfd_seq_ops = { + .start = gfs2_glockfd_seq_start, + .next = gfs2_glockfd_seq_next, + .stop = gfs2_glockfd_seq_stop, + .show = gfs2_glockfd_seq_show, +}; + +static int gfs2_glockfd_open(struct inode *inode, struct file *file) +{ + struct gfs2_glockfd_iter *i; + struct gfs2_sbd *sdp = inode->i_private; + + i = __seq_open_private(file, &gfs2_glockfd_seq_ops, + sizeof(struct gfs2_glockfd_iter)); + if (!i) + return -ENOMEM; + i->sb = sdp->sd_vfs; + return 0; +} + +static const struct file_operations gfs2_glockfd_fops = { + .owner = THIS_MODULE, + .open = gfs2_glockfd_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats); void gfs2_create_debugfs_file(struct gfs2_sbd *sdp) @@ -2708,6 +2932,9 @@ void gfs2_create_debugfs_file(struct gfs2_sbd *sdp) debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, &gfs2_glocks_fops); + debugfs_create_file("glockfd", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, + &gfs2_glockfd_fops); + debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, &gfs2_glstats_fops); diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h index 5aed8b500cf5ab88015b394d26a147afb1e17dc9..0d068f4fd7d6737c85ea35a0ed22a46c22bc9801 100644 --- a/fs/gfs2/glock.h +++ b/fs/gfs2/glock.h @@ -91,6 +91,7 @@ enum { #define GL_ASYNC 0x0040 #define GL_EXACT 0x0080 #define GL_SKIP 0x0100 +#define GL_NOPID 0x0200 #define GL_NOCACHE 0x0400 /* @@ -274,6 +275,7 @@ extern void gfs2_cancel_delete_work(struct gfs2_glock *gl); extern bool gfs2_delete_work_queued(const struct gfs2_glock *gl); extern void gfs2_flush_delete_work(struct gfs2_sbd *sdp); extern void gfs2_gl_hash_clear(struct gfs2_sbd *sdp); +extern void gfs2_gl_dq_holders(struct gfs2_sbd *sdp); extern void gfs2_glock_thaw(struct gfs2_sbd *sdp); extern void gfs2_glock_add_to_lru(struct gfs2_glock *gl); extern void gfs2_glock_free(struct gfs2_glock *gl); diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index c8ec876f33ea35787d05d2d69c81ca91854ab685..04a201584fa7c70d6403257f2ef17e282d86827d 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -130,6 +130,7 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, if (inode->i_state & I_NEW) { struct gfs2_sbd *sdp = GFS2_SB(inode); struct gfs2_glock *io_gl; + int extra_flags = 0; error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE, &ip->i_gl); @@ -141,9 +142,12 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, if (unlikely(error)) goto fail; - if (blktype != GFS2_BLKST_UNLINKED) + if (blktype == GFS2_BLKST_UNLINKED) + extra_flags |= LM_FLAG_TRY; + else gfs2_cancel_delete_work(io_gl); - error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, + GL_EXACT | GL_NOPID | extra_flags, &ip->i_iopen_gh); gfs2_glock_put(io_gl); if (unlikely(error)) @@ -210,6 +214,8 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, return inode; fail: + if (error == GLR_TRYFAILED) + error = -EAGAIN; if (gfs2_holder_initialized(&ip->i_iopen_gh)) gfs2_glock_dq_uninit(&ip->i_iopen_gh); if (gfs2_holder_initialized(&i_gh)) @@ -720,7 +726,8 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, error = insert_inode_locked4(inode, ip->i_no_addr, iget_test, &ip->i_no_addr); BUG_ON(error); - error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT, &ip->i_iopen_gh); + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT | GL_NOPID, + &ip->i_iopen_gh); if (error) goto fail_gunlock2; diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c index 6ce369b096d4b633964d617d22fa02163c142617..71911bf9ab34e6e972937e521fa137ca229abd30 100644 --- a/fs/gfs2/lock_dlm.c +++ b/fs/gfs2/lock_dlm.c @@ -1302,7 +1302,7 @@ static int gdlm_mount(struct gfs2_sbd *sdp, const char *table) memcpy(cluster, table, strlen(table) - strlen(fsname)); fsname++; - flags = DLM_LSFL_FS | DLM_LSFL_NEWEXCL; + flags = DLM_LSFL_NEWEXCL; /* * create/join lockspace diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c index 14ae9de7627726d515f2419a82dc4e109cd49689..afcb32854f14224f1a07d977faa6c8202baeb38f 100644 --- a/fs/gfs2/main.c +++ b/fs/gfs2/main.c @@ -151,14 +151,6 @@ static int __init init_gfs2_fs(void) if (error) goto fail_shrinker; - error = register_filesystem(&gfs2_fs_type); - if (error) - goto fail_fs1; - - error = register_filesystem(&gfs2meta_fs_type); - if (error) - goto fail_fs2; - error = -ENOMEM; gfs_recovery_wq = alloc_workqueue("gfs_recovery", WQ_MEM_RECLAIM | WQ_FREEZABLE, 0); @@ -180,11 +172,23 @@ static int __init init_gfs2_fs(void) goto fail_mempool; gfs2_register_debugfs(); + error = register_filesystem(&gfs2_fs_type); + if (error) + goto fail_fs1; + + error = register_filesystem(&gfs2meta_fs_type); + if (error) + goto fail_fs2; + pr_info("GFS2 installed\n"); return 0; +fail_fs2: + unregister_filesystem(&gfs2_fs_type); +fail_fs1: + mempool_destroy(gfs2_page_pool); fail_mempool: destroy_workqueue(gfs2_freeze_wq); fail_wq3: @@ -192,10 +196,6 @@ fail_wq3: fail_wq2: destroy_workqueue(gfs_recovery_wq); fail_wq1: - unregister_filesystem(&gfs2meta_fs_type); -fail_fs2: - unregister_filesystem(&gfs2_fs_type); -fail_fs1: unregister_shrinker(&gfs2_qd_shrinker); fail_shrinker: kmem_cache_destroy(gfs2_trans_cachep); diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c index 7e70e0ba5a6c0a09d4b44f2e2c70483571e6411e..6ed728aae9a53957d9fc57a7a4bc156eb1e25e33 100644 --- a/fs/gfs2/meta_io.c +++ b/fs/gfs2/meta_io.c @@ -525,8 +525,7 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) if (buffer_uptodate(first_bh)) goto out; - if (!buffer_locked(first_bh)) - ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &first_bh); + bh_read_nowait(first_bh, REQ_META | REQ_PRIO); dblock++; extlen--; @@ -534,9 +533,7 @@ struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) while (extlen) { bh = gfs2_getbuf(gl, dblock, CREATE); - if (!buffer_uptodate(bh) && !buffer_locked(bh)) - ll_rw_block(REQ_OP_READ | REQ_RAHEAD | REQ_META | - REQ_PRIO, 1, &bh); + bh_readahead(bh, REQ_RAHEAD | REQ_META | REQ_PRIO); brelse(bh); dblock++; extlen--; diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c index 549879929c847c143905d03bfac6bbc1b9564f4b..c0cf1d2d0ef5b3fa6a311775e471189fb9693f68 100644 --- a/fs/gfs2/ops_fstype.c +++ b/fs/gfs2/ops_fstype.c @@ -178,7 +178,10 @@ static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent) pr_warn("Invalid block size\n"); return -EINVAL; } - + if (sb->sb_bsize_shift != ffs(sb->sb_bsize) - 1) { + pr_warn("Invalid block size shift\n"); + return -EINVAL; + } return 0; } @@ -381,8 +384,10 @@ static int init_names(struct gfs2_sbd *sdp, int silent) if (!table[0]) table = sdp->sd_vfs->s_id; - strlcpy(sdp->sd_proto_name, proto, GFS2_FSNAME_LEN); - strlcpy(sdp->sd_table_name, table, GFS2_FSNAME_LEN); + BUILD_BUG_ON(GFS2_LOCKNAME_LEN > GFS2_FSNAME_LEN); + + strscpy(sdp->sd_proto_name, proto, GFS2_LOCKNAME_LEN); + strscpy(sdp->sd_table_name, table, GFS2_LOCKNAME_LEN); table = sdp->sd_table_name; while ((table = strchr(table, '/'))) @@ -401,7 +406,8 @@ static int init_locking(struct gfs2_sbd *sdp, struct gfs2_holder *mount_gh, error = gfs2_glock_nq_num(sdp, GFS2_MOUNT_LOCK, &gfs2_nondisk_glops, - LM_ST_EXCLUSIVE, LM_FLAG_NOEXP | GL_NOCACHE, + LM_ST_EXCLUSIVE, + LM_FLAG_NOEXP | GL_NOCACHE | GL_NOPID, mount_gh); if (error) { fs_err(sdp, "can't acquire mount glock: %d\n", error); @@ -411,7 +417,7 @@ static int init_locking(struct gfs2_sbd *sdp, struct gfs2_holder *mount_gh, error = gfs2_glock_nq_num(sdp, GFS2_LIVE_LOCK, &gfs2_nondisk_glops, LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT, + LM_FLAG_NOEXP | GL_EXACT | GL_NOPID, &sdp->sd_live_gh); if (error) { fs_err(sdp, "can't acquire live glock: %d\n", error); @@ -687,7 +693,7 @@ static int init_statfs(struct gfs2_sbd *sdp) iput(pn); pn = NULL; ip = GFS2_I(sdp->sd_sc_inode); - error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID, &sdp->sd_sc_gh); if (error) { fs_err(sdp, "can't lock local \"sc\" file: %d\n", error); @@ -776,7 +782,7 @@ static int init_journal(struct gfs2_sbd *sdp, int undo) error = gfs2_glock_nq_num(sdp, sdp->sd_lockstruct.ls_jid, &gfs2_journal_glops, LM_ST_EXCLUSIVE, - LM_FLAG_NOEXP | GL_NOCACHE, + LM_FLAG_NOEXP | GL_NOCACHE | GL_NOPID, &sdp->sd_journal_gh); if (error) { fs_err(sdp, "can't acquire journal glock: %d\n", error); @@ -786,7 +792,8 @@ static int init_journal(struct gfs2_sbd *sdp, int undo) ip = GFS2_I(sdp->sd_jdesc->jd_inode); sdp->sd_jinode_gl = ip->i_gl; error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, - LM_FLAG_NOEXP | GL_EXACT | GL_NOCACHE, + LM_FLAG_NOEXP | GL_EXACT | + GL_NOCACHE | GL_NOPID, &sdp->sd_jinode_gh); if (error) { fs_err(sdp, "can't acquire journal inode glock: %d\n", @@ -957,7 +964,7 @@ static int init_per_node(struct gfs2_sbd *sdp, int undo) pn = NULL; ip = GFS2_I(sdp->sd_qc_inode); - error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID, &sdp->sd_qc_gh); if (error) { fs_err(sdp, "can't lock local \"qc\" file: %d\n", error); @@ -1439,13 +1446,13 @@ static int gfs2_parse_param(struct fs_context *fc, struct fs_parameter *param) switch (o) { case Opt_lockproto: - strlcpy(args->ar_lockproto, param->string, GFS2_LOCKNAME_LEN); + strscpy(args->ar_lockproto, param->string, GFS2_LOCKNAME_LEN); break; case Opt_locktable: - strlcpy(args->ar_locktable, param->string, GFS2_LOCKNAME_LEN); + strscpy(args->ar_locktable, param->string, GFS2_LOCKNAME_LEN); break; case Opt_hostdata: - strlcpy(args->ar_hostdata, param->string, GFS2_LOCKNAME_LEN); + strscpy(args->ar_hostdata, param->string, GFS2_LOCKNAME_LEN); break; case Opt_spectator: args->ar_spectator = 1; diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index f201eaf59d0da55cb83190cd2cd41d3b2237d845..1ed17226d9ede985565357b514d32084da890972 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -745,12 +745,8 @@ static int gfs2_write_buf_to_page(struct gfs2_inode *ip, unsigned long index, } if (PageUptodate(page)) set_buffer_uptodate(bh); - if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ | REQ_META | REQ_PRIO, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) - goto unlock_out; - } + if (bh_read(bh, REQ_META | REQ_PRIO) < 0) + goto unlock_out; if (gfs2_is_jdata(ip)) gfs2_trans_add_data(ip->i_gl, bh); else diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c index b5b0f285b27f8f85fb3997e9b4932c6eb6081ba4..b018957a1bb2449307417e3a1dd6411245d62e9c 100644 --- a/fs/gfs2/super.c +++ b/fs/gfs2/super.c @@ -346,7 +346,8 @@ static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp) } error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_EXCLUSIVE, - LM_FLAG_NOEXP, &sdp->sd_freeze_gh); + LM_FLAG_NOEXP | GL_NOPID, + &sdp->sd_freeze_gh); if (error) goto out; diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c index 8241029a2a5d25a18239171bafa11edfb4982ff4..7a6aeffcdf5cae60b6b1926b02e17753b557611a 100644 --- a/fs/gfs2/util.c +++ b/fs/gfs2/util.c @@ -164,6 +164,11 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) } if (!ret) gfs2_make_fs_ro(sdp); + /* + * Dequeue any pending non-system glock holders that can no + * longer be granted because the file system is withdrawn. + */ + gfs2_gl_dq_holders(sdp); gfs2_freeze_unlock(&freeze_gh); } @@ -204,6 +209,7 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) * exception code in glock_dq. */ iput(inode); + sdp->sd_jdesc->jd_inode = NULL; /* * Wait until the journal inode's glock is freed. This allows try locks * on other nodes to be successful, otherwise we remain the owner of @@ -226,7 +232,8 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) */ fs_warn(sdp, "Requesting recovery of jid %d.\n", sdp->sd_lockstruct.ls_jid); - gfs2_holder_reinit(LM_ST_EXCLUSIVE, LM_FLAG_TRY_1CB | LM_FLAG_NOEXP, + gfs2_holder_reinit(LM_ST_EXCLUSIVE, + LM_FLAG_TRY_1CB | LM_FLAG_NOEXP | GL_NOPID, &sdp->sd_live_gh); msleep(GL_GLOCK_MAX_HOLD); /* @@ -251,7 +258,8 @@ static void signal_our_withdraw(struct gfs2_sbd *sdp) fs_warn(sdp, "Unable to recover our journal jid %d.\n", sdp->sd_lockstruct.ls_jid); gfs2_glock_dq_wait(&sdp->sd_live_gh); - gfs2_holder_reinit(LM_ST_SHARED, LM_FLAG_NOEXP | GL_EXACT, + gfs2_holder_reinit(LM_ST_SHARED, + LM_FLAG_NOEXP | GL_EXACT | GL_NOPID, &sdp->sd_live_gh); gfs2_glock_nq(&sdp->sd_live_gh); } diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c index c83fd0e8404d351ee5ada8abc8cde614a0c8c566..2015e42e752a6bb55fe32478b185d204e459a600 100644 --- a/fs/hfs/bnode.c +++ b/fs/hfs/bnode.c @@ -21,7 +21,6 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) int pagenum; int bytes_read; int bytes_to_read; - void *vaddr; off += node->page_offset; pagenum = off >> PAGE_SHIFT; @@ -33,9 +32,7 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) page = node->page[pagenum]; bytes_to_read = min_t(int, len - bytes_read, PAGE_SIZE - off); - vaddr = kmap_atomic(page); - memcpy(buf + bytes_read, vaddr + off, bytes_to_read); - kunmap_atomic(vaddr); + memcpy_from_page(buf + bytes_read, page, off, bytes_to_read); pagenum++; off = 0; /* page offset only applies to the first page */ @@ -80,8 +77,7 @@ void hfs_bnode_write(struct hfs_bnode *node, void *buf, int off, int len) off += node->page_offset; page = node->page[0]; - memcpy(kmap(page) + off, buf, len); - kunmap(page); + memcpy_to_page(page, off, buf, len); set_page_dirty(page); } @@ -105,8 +101,7 @@ void hfs_bnode_clear(struct hfs_bnode *node, int off, int len) off += node->page_offset; page = node->page[0]; - memset(kmap(page) + off, 0, len); - kunmap(page); + memzero_page(page, off, len); set_page_dirty(page); } @@ -123,9 +118,7 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, src_page = src_node->page[0]; dst_page = dst_node->page[0]; - memcpy(kmap(dst_page) + dst, kmap(src_page) + src, len); - kunmap(src_page); - kunmap(dst_page); + memcpy_page(dst_page, dst, src_page, src, len); set_page_dirty(dst_page); } @@ -140,9 +133,9 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) src += node->page_offset; dst += node->page_offset; page = node->page[0]; - ptr = kmap(page); + ptr = kmap_local_page(page); memmove(ptr + dst, ptr + src, len); - kunmap(page); + kunmap_local(ptr); set_page_dirty(page); } @@ -346,13 +339,14 @@ struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num) if (!test_bit(HFS_BNODE_NEW, &node->flags)) return node; - desc = (struct hfs_bnode_desc *)(kmap(node->page[0]) + node->page_offset); + desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) + + node->page_offset); node->prev = be32_to_cpu(desc->prev); node->next = be32_to_cpu(desc->next); node->num_recs = be16_to_cpu(desc->num_recs); node->type = desc->type; node->height = desc->height; - kunmap(node->page[0]); + kunmap_local(desc); switch (node->type) { case HFS_NODE_HEADER: @@ -436,14 +430,12 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) } pagep = node->page; - memset(kmap(*pagep) + node->page_offset, 0, - min((int)PAGE_SIZE, (int)tree->node_size)); + memzero_page(*pagep, node->page_offset, + min((int)PAGE_SIZE, (int)tree->node_size)); set_page_dirty(*pagep); - kunmap(*pagep); for (i = 1; i < tree->pages_per_bnode; i++) { - memset(kmap(*++pagep), 0, PAGE_SIZE); + memzero_page(*++pagep, 0, PAGE_SIZE); set_page_dirty(*pagep); - kunmap(*pagep); } clear_bit(HFS_BNODE_NEW, &node->flags); wake_up(&node->lock_wq); diff --git a/fs/hfs/btree.c b/fs/hfs/btree.c index 19017d2961734fd7701aa1ac31c6a94d8a65c71b..2fa4b1f8cc7fb098a9406d600ded87922c2ba9ea 100644 --- a/fs/hfs/btree.c +++ b/fs/hfs/btree.c @@ -80,7 +80,8 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke goto free_inode; /* Load the header */ - head = (struct hfs_btree_header_rec *)(kmap(page) + sizeof(struct hfs_bnode_desc)); + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + sizeof(struct hfs_bnode_desc)); tree->root = be32_to_cpu(head->root); tree->leaf_count = be32_to_cpu(head->leaf_count); tree->leaf_head = be32_to_cpu(head->leaf_head); @@ -119,11 +120,12 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp ke tree->node_size_shift = ffs(size) - 1; tree->pages_per_bnode = (tree->node_size + PAGE_SIZE - 1) >> PAGE_SHIFT; - kunmap(page); + kunmap_local(head); put_page(page); return tree; fail_page: + kunmap_local(head); put_page(page); free_inode: tree->inode->i_mapping->a_ops = &hfs_aops; @@ -169,7 +171,8 @@ void hfs_btree_write(struct hfs_btree *tree) return; /* Load the header */ page = node->page[0]; - head = (struct hfs_btree_header_rec *)(kmap(page) + sizeof(struct hfs_bnode_desc)); + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + sizeof(struct hfs_bnode_desc)); head->root = cpu_to_be32(tree->root); head->leaf_count = cpu_to_be32(tree->leaf_count); @@ -180,7 +183,7 @@ void hfs_btree_write(struct hfs_btree *tree) head->attributes = cpu_to_be32(tree->attributes); head->depth = cpu_to_be16(tree->depth); - kunmap(page); + kunmap_local(head); set_page_dirty(page); hfs_bnode_put(node); } @@ -268,7 +271,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_local_page(*pagep); off &= ~PAGE_MASK; idx = 0; @@ -281,7 +284,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) idx += i; data[off] |= m; set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_local(data); tree->free_nodes--; mark_inode_dirty(tree->inode); hfs_bnode_put(node); @@ -290,14 +293,14 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) } } if (++off >= PAGE_SIZE) { - kunmap(*pagep); - data = kmap(*++pagep); + kunmap_local(data); + data = kmap_local_page(*++pagep); off = 0; } idx += 8; len--; } - kunmap(*pagep); + kunmap_local(data); nidx = node->next; if (!nidx) { printk(KERN_DEBUG "create new bmap node...\n"); @@ -313,7 +316,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) off = off16; off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_local_page(*pagep); off &= ~PAGE_MASK; } } @@ -360,20 +363,20 @@ void hfs_bmap_free(struct hfs_bnode *node) } off += node->page_offset + nidx / 8; page = node->page[off >> PAGE_SHIFT]; - data = kmap(page); + data = kmap_local_page(page); off &= ~PAGE_MASK; m = 1 << (~nidx & 7); byte = data[off]; if (!(byte & m)) { pr_crit("trying to free free bnode %u(%d)\n", node->this, node->type); - kunmap(page); + kunmap_local(data); hfs_bnode_put(node); return; } data[off] = byte & ~m; set_page_dirty(page); - kunmap(page); + kunmap_local(data); hfs_bnode_put(node); tree->free_nodes++; mark_inode_dirty(tree->inode); diff --git a/fs/hfsplus/bitmap.c b/fs/hfsplus/bitmap.c index cebce0cfe340589533cb4e2d40b065a876dc4b3c..bd8dcea8558800e0362d3f4b5bcad0ba60dca67d 100644 --- a/fs/hfsplus/bitmap.c +++ b/fs/hfsplus/bitmap.c @@ -39,7 +39,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, start = size; goto out; } - pptr = kmap(page); + pptr = kmap_local_page(page); curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; i = offset % 32; offset &= ~(PAGE_CACHE_BITS - 1); @@ -74,7 +74,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, } curr++; } - kunmap(page); + kunmap_local(pptr); offset += PAGE_CACHE_BITS; if (offset >= size) break; @@ -84,7 +84,7 @@ int hfsplus_block_allocate(struct super_block *sb, u32 size, start = size; goto out; } - curr = pptr = kmap(page); + curr = pptr = kmap_local_page(page); if ((size ^ offset) / PAGE_CACHE_BITS) end = pptr + PAGE_CACHE_BITS / 32; else @@ -127,7 +127,7 @@ found: len -= 32; } set_page_dirty(page); - kunmap(page); + kunmap_local(pptr); offset += PAGE_CACHE_BITS; page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS, NULL); @@ -135,7 +135,7 @@ found: start = size; goto out; } - pptr = kmap(page); + pptr = kmap_local_page(page); curr = pptr; end = pptr + PAGE_CACHE_BITS / 32; } @@ -151,7 +151,7 @@ last: done: *curr = cpu_to_be32(n); set_page_dirty(page); - kunmap(page); + kunmap_local(pptr); *max = offset + (curr - pptr) * 32 + i - start; sbi->free_blocks -= *max; hfsplus_mark_mdb_dirty(sb); @@ -185,7 +185,7 @@ int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) page = read_mapping_page(mapping, pnr, NULL); if (IS_ERR(page)) goto kaboom; - pptr = kmap(page); + pptr = kmap_local_page(page); curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; end = pptr + PAGE_CACHE_BITS / 32; len = count; @@ -215,11 +215,11 @@ int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) if (!count) break; set_page_dirty(page); - kunmap(page); + kunmap_local(pptr); page = read_mapping_page(mapping, ++pnr, NULL); if (IS_ERR(page)) goto kaboom; - pptr = kmap(page); + pptr = kmap_local_page(page); curr = pptr; end = pptr + PAGE_CACHE_BITS / 32; } @@ -231,7 +231,7 @@ done: } out: set_page_dirty(page); - kunmap(page); + kunmap_local(pptr); sbi->free_blocks += len; hfsplus_mark_mdb_dirty(sb); mutex_unlock(&sbi->alloc_mutex); diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c index a5ab00e5422030364c4aae4b229ac92928445dd9..87974d5e679156c63809dbae93ddcf32e1e9ab18 100644 --- a/fs/hfsplus/bnode.c +++ b/fs/hfsplus/bnode.c @@ -29,14 +29,12 @@ void hfs_bnode_read(struct hfs_bnode *node, void *buf, int off, int len) off &= ~PAGE_MASK; l = min_t(int, len, PAGE_SIZE - off); - memcpy(buf, kmap(*pagep) + off, l); - kunmap(*pagep); + memcpy_from_page(buf, *pagep, off, l); while ((len -= l) != 0) { buf += l; l = min_t(int, len, PAGE_SIZE); - memcpy(buf, kmap(*++pagep), l); - kunmap(*pagep); + memcpy_from_page(buf, *++pagep, 0, l); } } @@ -82,16 +80,14 @@ void hfs_bnode_write(struct hfs_bnode *node, void *buf, int off, int len) off &= ~PAGE_MASK; l = min_t(int, len, PAGE_SIZE - off); - memcpy(kmap(*pagep) + off, buf, l); + memcpy_to_page(*pagep, off, buf, l); set_page_dirty(*pagep); - kunmap(*pagep); while ((len -= l) != 0) { buf += l; l = min_t(int, len, PAGE_SIZE); - memcpy(kmap(*++pagep), buf, l); + memcpy_to_page(*++pagep, 0, buf, l); set_page_dirty(*pagep); - kunmap(*pagep); } } @@ -112,15 +108,13 @@ void hfs_bnode_clear(struct hfs_bnode *node, int off, int len) off &= ~PAGE_MASK; l = min_t(int, len, PAGE_SIZE - off); - memset(kmap(*pagep) + off, 0, l); + memzero_page(*pagep, off, l); set_page_dirty(*pagep); - kunmap(*pagep); while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memset(kmap(*++pagep), 0, l); + memzero_page(*++pagep, 0, l); set_page_dirty(*pagep); - kunmap(*pagep); } } @@ -142,24 +136,20 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, if (src == dst) { l = min_t(int, len, PAGE_SIZE - src); - memcpy(kmap(*dst_page) + src, kmap(*src_page) + src, l); - kunmap(*src_page); + memcpy_page(*dst_page, src, *src_page, src, l); set_page_dirty(*dst_page); - kunmap(*dst_page); while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memcpy(kmap(*++dst_page), kmap(*++src_page), l); - kunmap(*src_page); + memcpy_page(*++dst_page, 0, *++src_page, 0, l); set_page_dirty(*dst_page); - kunmap(*dst_page); } } else { void *src_ptr, *dst_ptr; do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; if (PAGE_SIZE - src < PAGE_SIZE - dst) { l = PAGE_SIZE - src; src = 0; @@ -171,9 +161,9 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, } l = min(len, l); memcpy(dst_ptr, src_ptr, l); - kunmap(*src_page); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); if (!dst) dst_page++; else @@ -185,6 +175,7 @@ void hfs_bnode_copy(struct hfs_bnode *dst_node, int dst, void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) { struct page **src_page, **dst_page; + void *src_ptr, *dst_ptr; int l; hfs_dbg(BNODE_MOD, "movebytes: %u,%u,%u\n", dst, src, len); @@ -202,27 +193,28 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) if (src == dst) { while (src < len) { - memmove(kmap(*dst_page), kmap(*src_page), src); - kunmap(*src_page); + dst_ptr = kmap_local_page(*dst_page); + src_ptr = kmap_local_page(*src_page); + memmove(dst_ptr, src_ptr, src); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); len -= src; src = PAGE_SIZE; src_page--; dst_page--; } src -= len; - memmove(kmap(*dst_page) + src, - kmap(*src_page) + src, len); - kunmap(*src_page); + dst_ptr = kmap_local_page(*dst_page); + src_ptr = kmap_local_page(*src_page); + memmove(dst_ptr + src, src_ptr + src, len); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); } else { - void *src_ptr, *dst_ptr; - do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; if (src < dst) { l = src; src = PAGE_SIZE; @@ -234,9 +226,9 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) } l = min(len, l); memmove(dst_ptr - l, src_ptr - l, l); - kunmap(*src_page); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); if (dst == PAGE_SIZE) dst_page--; else @@ -251,26 +243,27 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) if (src == dst) { l = min_t(int, len, PAGE_SIZE - src); - memmove(kmap(*dst_page) + src, - kmap(*src_page) + src, l); - kunmap(*src_page); + + dst_ptr = kmap_local_page(*dst_page) + src; + src_ptr = kmap_local_page(*src_page) + src; + memmove(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); while ((len -= l) != 0) { l = min_t(int, len, PAGE_SIZE); - memmove(kmap(*++dst_page), - kmap(*++src_page), l); - kunmap(*src_page); + dst_ptr = kmap_local_page(*++dst_page); + src_ptr = kmap_local_page(*++src_page); + memmove(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); } } else { - void *src_ptr, *dst_ptr; - do { - src_ptr = kmap(*src_page) + src; - dst_ptr = kmap(*dst_page) + dst; + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; if (PAGE_SIZE - src < PAGE_SIZE - dst) { l = PAGE_SIZE - src; @@ -283,9 +276,9 @@ void hfs_bnode_move(struct hfs_bnode *node, int dst, int src, int len) } l = min(len, l); memmove(dst_ptr, src_ptr, l); - kunmap(*src_page); + kunmap_local(src_ptr); set_page_dirty(*dst_page); - kunmap(*dst_page); + kunmap_local(dst_ptr); if (!dst) dst_page++; else @@ -498,14 +491,14 @@ struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num) if (!test_bit(HFS_BNODE_NEW, &node->flags)) return node; - desc = (struct hfs_bnode_desc *)(kmap(node->page[0]) + - node->page_offset); + desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) + + node->page_offset); node->prev = be32_to_cpu(desc->prev); node->next = be32_to_cpu(desc->next); node->num_recs = be16_to_cpu(desc->num_recs); node->type = desc->type; node->height = desc->height; - kunmap(node->page[0]); + kunmap_local(desc); switch (node->type) { case HFS_NODE_HEADER: @@ -589,14 +582,12 @@ struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) } pagep = node->page; - memset(kmap(*pagep) + node->page_offset, 0, - min_t(int, PAGE_SIZE, tree->node_size)); + memzero_page(*pagep, node->page_offset, + min_t(int, PAGE_SIZE, tree->node_size)); set_page_dirty(*pagep); - kunmap(*pagep); for (i = 1; i < tree->pages_per_bnode; i++) { - memset(kmap(*++pagep), 0, PAGE_SIZE); + memzero_page(*++pagep, 0, PAGE_SIZE); set_page_dirty(*pagep); - kunmap(*pagep); } clear_bit(HFS_BNODE_NEW, &node->flags); wake_up(&node->lock_wq); diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c index 66774f4cb4fd5e34ae98de0d9a8ef9528bcfb47c..9e1732a2b92a8c046e828aa130a9d252e81de13e 100644 --- a/fs/hfsplus/btree.c +++ b/fs/hfsplus/btree.c @@ -163,7 +163,7 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) goto free_inode; /* Load the header */ - head = (struct hfs_btree_header_rec *)(kmap(page) + + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + sizeof(struct hfs_bnode_desc)); tree->root = be32_to_cpu(head->root); tree->leaf_count = be32_to_cpu(head->leaf_count); @@ -240,11 +240,12 @@ struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) (tree->node_size + PAGE_SIZE - 1) >> PAGE_SHIFT; - kunmap(page); + kunmap_local(head); put_page(page); return tree; fail_page: + kunmap_local(head); put_page(page); free_inode: tree->inode->i_mapping->a_ops = &hfsplus_aops; @@ -291,7 +292,7 @@ int hfs_btree_write(struct hfs_btree *tree) return -EIO; /* Load the header */ page = node->page[0]; - head = (struct hfs_btree_header_rec *)(kmap(page) + + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + sizeof(struct hfs_bnode_desc)); head->root = cpu_to_be32(tree->root); @@ -303,7 +304,7 @@ int hfs_btree_write(struct hfs_btree *tree) head->attributes = cpu_to_be32(tree->attributes); head->depth = cpu_to_be16(tree->depth); - kunmap(page); + kunmap_local(head); set_page_dirty(page); hfs_bnode_put(node); return 0; @@ -394,7 +395,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_local_page(*pagep); off &= ~PAGE_MASK; idx = 0; @@ -407,7 +408,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) idx += i; data[off] |= m; set_page_dirty(*pagep); - kunmap(*pagep); + kunmap_local(data); tree->free_nodes--; mark_inode_dirty(tree->inode); hfs_bnode_put(node); @@ -417,14 +418,14 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) } } if (++off >= PAGE_SIZE) { - kunmap(*pagep); - data = kmap(*++pagep); + kunmap_local(data); + data = kmap_local_page(*++pagep); off = 0; } idx += 8; len--; } - kunmap(*pagep); + kunmap_local(data); nidx = node->next; if (!nidx) { hfs_dbg(BNODE_MOD, "create new bmap node\n"); @@ -440,7 +441,7 @@ struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) off = off16; off += node->page_offset; pagep = node->page + (off >> PAGE_SHIFT); - data = kmap(*pagep); + data = kmap_local_page(*pagep); off &= ~PAGE_MASK; } } @@ -490,7 +491,7 @@ void hfs_bmap_free(struct hfs_bnode *node) } off += node->page_offset + nidx / 8; page = node->page[off >> PAGE_SHIFT]; - data = kmap(page); + data = kmap_local_page(page); off &= ~PAGE_MASK; m = 1 << (~nidx & 7); byte = data[off]; @@ -498,13 +499,13 @@ void hfs_bmap_free(struct hfs_bnode *node) pr_crit("trying to free free bnode " "%u(%d)\n", node->this, node->type); - kunmap(page); + kunmap_local(data); hfs_bnode_put(node); return; } data[off] = byte & ~m; set_page_dirty(page); - kunmap(page); + kunmap_local(data); hfs_bnode_put(node); tree->free_nodes++; mark_inode_dirty(tree->inode); diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c index 07881b76d42f99070c146c4e28972d94f61856af..277468783feeecbe8199da3d4ab7df924ff3e2e6 100644 --- a/fs/hostfs/hostfs_kern.c +++ b/fs/hostfs/hostfs_kern.c @@ -103,7 +103,7 @@ static char *__dentry_name(struct dentry *dentry, char *name) */ BUG_ON(p + strlen(p) + 1 != name + PATH_MAX); - strlcpy(name, root, PATH_MAX); + strscpy(name, root, PATH_MAX); if (len > p - name) { __putname(name); return NULL; diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index f7a5b5124d8a92412778b2706838d599dfef0fee..dd54f67e47fdf145ab602571f821daee713eaf9c 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -364,13 +364,155 @@ static int hugetlbfs_write_end(struct file *file, struct address_space *mapping, return -EINVAL; } -static void remove_huge_page(struct page *page) +static void hugetlb_delete_from_page_cache(struct page *page) { ClearPageDirty(page); ClearPageUptodate(page); delete_from_page_cache(page); } +/* + * Called with i_mmap_rwsem held for inode based vma maps. This makes + * sure vma (and vm_mm) will not go away. We also hold the hugetlb fault + * mutex for the page in the mapping. So, we can not race with page being + * faulted into the vma. + */ +static bool hugetlb_vma_maps_page(struct vm_area_struct *vma, + unsigned long addr, struct page *page) +{ + pte_t *ptep, pte; + + ptep = huge_pte_offset(vma->vm_mm, addr, + huge_page_size(hstate_vma(vma))); + + if (!ptep) + return false; + + pte = huge_ptep_get(ptep); + if (huge_pte_none(pte) || !pte_present(pte)) + return false; + + if (pte_page(pte) == page) + return true; + + return false; +} + +/* + * Can vma_offset_start/vma_offset_end overflow on 32-bit arches? + * No, because the interval tree returns us only those vmas + * which overlap the truncated area starting at pgoff, + * and no vma on a 32-bit arch can span beyond the 4GB. + */ +static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start) +{ + if (vma->vm_pgoff < start) + return (start - vma->vm_pgoff) << PAGE_SHIFT; + else + return 0; +} + +static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end) +{ + unsigned long t_end; + + if (!end) + return vma->vm_end; + + t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start; + if (t_end > vma->vm_end) + t_end = vma->vm_end; + return t_end; +} + +/* + * Called with hugetlb fault mutex held. Therefore, no more mappings to + * this folio can be created while executing the routine. + */ +static void hugetlb_unmap_file_folio(struct hstate *h, + struct address_space *mapping, + struct folio *folio, pgoff_t index) +{ + struct rb_root_cached *root = &mapping->i_mmap; + struct hugetlb_vma_lock *vma_lock; + struct page *page = &folio->page; + struct vm_area_struct *vma; + unsigned long v_start; + unsigned long v_end; + pgoff_t start, end; + + start = index * pages_per_huge_page(h); + end = (index + 1) * pages_per_huge_page(h); + + i_mmap_lock_write(mapping); +retry: + vma_lock = NULL; + vma_interval_tree_foreach(vma, root, start, end - 1) { + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + + if (!hugetlb_vma_maps_page(vma, vma->vm_start + v_start, page)) + continue; + + if (!hugetlb_vma_trylock_write(vma)) { + vma_lock = vma->vm_private_data; + /* + * If we can not get vma lock, we need to drop + * immap_sema and take locks in order. First, + * take a ref on the vma_lock structure so that + * we can be guaranteed it will not go away when + * dropping immap_sema. + */ + kref_get(&vma_lock->refs); + break; + } + + unmap_hugepage_range(vma, vma->vm_start + v_start, v_end, + NULL, ZAP_FLAG_DROP_MARKER); + hugetlb_vma_unlock_write(vma); + } + + i_mmap_unlock_write(mapping); + + if (vma_lock) { + /* + * Wait on vma_lock. We know it is still valid as we have + * a reference. We must 'open code' vma locking as we do + * not know if vma_lock is still attached to vma. + */ + down_write(&vma_lock->rw_sema); + i_mmap_lock_write(mapping); + + vma = vma_lock->vma; + if (!vma) { + /* + * If lock is no longer attached to vma, then just + * unlock, drop our reference and retry looking for + * other vmas. + */ + up_write(&vma_lock->rw_sema); + kref_put(&vma_lock->refs, hugetlb_vma_lock_release); + goto retry; + } + + /* + * vma_lock is still attached to vma. Check to see if vma + * still maps page and if so, unmap. + */ + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + if (hugetlb_vma_maps_page(vma, vma->vm_start + v_start, page)) + unmap_hugepage_range(vma, vma->vm_start + v_start, + v_end, NULL, + ZAP_FLAG_DROP_MARKER); + + kref_put(&vma_lock->refs, hugetlb_vma_lock_release); + hugetlb_vma_unlock_write(vma); + + goto retry; + } +} + static void hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, zap_flags_t zap_flags) @@ -383,32 +525,66 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, * an inclusive "last". */ vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) { - unsigned long v_offset; + unsigned long v_start; unsigned long v_end; + if (!hugetlb_vma_trylock_write(vma)) + continue; + + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + + unmap_hugepage_range(vma, vma->vm_start + v_start, v_end, + NULL, zap_flags); + /* - * Can the expression below overflow on 32-bit arches? - * No, because the interval tree returns us only those vmas - * which overlap the truncated area starting at pgoff, - * and no vma on a 32-bit arch can span beyond the 4GB. + * Note that vma lock only exists for shared/non-private + * vmas. Therefore, lock is not held when calling + * unmap_hugepage_range for private vmas. */ - if (vma->vm_pgoff < start) - v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT; - else - v_offset = 0; - - if (!end) - v_end = vma->vm_end; - else { - v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) - + vma->vm_start; - if (v_end > vma->vm_end) - v_end = vma->vm_end; - } + hugetlb_vma_unlock_write(vma); + } +} - unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end, - NULL, zap_flags); +/* + * Called with hugetlb fault mutex held. + * Returns true if page was actually removed, false otherwise. + */ +static bool remove_inode_single_folio(struct hstate *h, struct inode *inode, + struct address_space *mapping, + struct folio *folio, pgoff_t index, + bool truncate_op) +{ + bool ret = false; + + /* + * If folio is mapped, it was faulted in after being + * unmapped in caller. Unmap (again) while holding + * the fault mutex. The mutex will prevent faults + * until we finish removing the folio. + */ + if (unlikely(folio_mapped(folio))) + hugetlb_unmap_file_folio(h, mapping, folio, index); + + folio_lock(folio); + /* + * We must remove the folio from page cache before removing + * the region/ reserve map (hugetlb_unreserve_pages). In + * rare out of memory conditions, removal of the region/reserve + * map could fail. Correspondingly, the subpool and global + * reserve usage count can need to be adjusted. + */ + VM_BUG_ON(HPageRestoreReserve(&folio->page)); + hugetlb_delete_from_page_cache(&folio->page); + ret = true; + if (!truncate_op) { + if (unlikely(hugetlb_unreserve_pages(inode, index, + index + 1, 1))) + hugetlb_fix_reserve_counts(inode); } + + folio_unlock(folio); + return ret; } /* @@ -418,10 +594,10 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, * truncation is indicated by end of range being LLONG_MAX * In this case, we first scan the range and release found pages. * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve - * maps and global counts. Page faults can not race with truncation - * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents - * page faults in the truncated range by checking i_size. i_size is - * modified while holding i_mmap_rwsem. + * maps and global counts. Page faults can race with truncation. + * During faults, hugetlb_no_page() checks i_size before page allocation, + * and again after obtaining page table lock. It will 'back out' + * allocations in the truncated range. * hole punch is indicated if end is not LLONG_MAX * In the hole punch case we scan the range and release found pages. * Only when releasing a page is the associated region/reserve map @@ -451,61 +627,17 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, u32 hash = 0; index = folio->index; - if (!truncate_op) { - /* - * Only need to hold the fault mutex in the - * hole punch case. This prevents races with - * page faults. Races are not possible in the - * case of truncation. - */ - hash = hugetlb_fault_mutex_hash(mapping, index); - mutex_lock(&hugetlb_fault_mutex_table[hash]); - } + hash = hugetlb_fault_mutex_hash(mapping, index); + mutex_lock(&hugetlb_fault_mutex_table[hash]); /* - * If folio is mapped, it was faulted in after being - * unmapped in caller. Unmap (again) now after taking - * the fault mutex. The mutex will prevent faults - * until we finish removing the folio. - * - * This race can only happen in the hole punch case. - * Getting here in a truncate operation is a bug. + * Remove folio that was part of folio_batch. */ - if (unlikely(folio_mapped(folio))) { - BUG_ON(truncate_op); - - mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_lock_write(mapping); - mutex_lock(&hugetlb_fault_mutex_table[hash]); - hugetlb_vmdelete_list(&mapping->i_mmap, - index * pages_per_huge_page(h), - (index + 1) * pages_per_huge_page(h), - ZAP_FLAG_DROP_MARKER); - i_mmap_unlock_write(mapping); - } - - folio_lock(folio); - /* - * We must free the huge page and remove from page - * cache (remove_huge_page) BEFORE removing the - * region/reserve map (hugetlb_unreserve_pages). In - * rare out of memory conditions, removal of the - * region/reserve map could fail. Correspondingly, - * the subpool and global reserve usage count can need - * to be adjusted. - */ - VM_BUG_ON(HPageRestoreReserve(&folio->page)); - remove_huge_page(&folio->page); - freed++; - if (!truncate_op) { - if (unlikely(hugetlb_unreserve_pages(inode, - index, index + 1, 1))) - hugetlb_fix_reserve_counts(inode); - } - - folio_unlock(folio); - if (!truncate_op) - mutex_unlock(&hugetlb_fault_mutex_table[hash]); + if (remove_inode_single_folio(h, inode, mapping, folio, + index, truncate_op)) + freed++; + + mutex_unlock(&hugetlb_fault_mutex_table[hash]); } folio_batch_release(&fbatch); cond_resched(); @@ -543,8 +675,8 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) BUG_ON(offset & ~huge_page_mask(h)); pgoff = offset >> PAGE_SHIFT; - i_mmap_lock_write(mapping); i_size_write(inode, offset); + i_mmap_lock_write(mapping); if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0, ZAP_FLAG_DROP_MARKER); @@ -703,11 +835,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, /* addr is the offset within the file (zero based) */ addr = index * hpage_size; - /* - * fault mutex taken here, protects against fault path - * and hole punch. inode_lock previously taken protects - * against truncation. - */ + /* mutex taken here, fault path and hole punch */ hash = hugetlb_fault_mutex_hash(mapping, index); mutex_lock(&hugetlb_fault_mutex_table[hash]); @@ -737,7 +865,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, } clear_huge_page(page, addr, pages_per_huge_page(h)); __SetPageUptodate(page); - error = huge_add_to_page_cache(page, mapping, index); + error = hugetlb_add_to_page_cache(page, mapping, index); if (unlikely(error)) { restore_reserve_on_error(h, &pseudo_vma, addr, page); put_page(page); @@ -749,7 +877,7 @@ static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, SetHPageMigratable(page); /* - * unlock_page because locked by huge_add_to_page_cache() + * unlock_page because locked by hugetlb_add_to_page_cache() * put_page() due to reference from alloc_huge_page() */ unlock_page(page); @@ -885,33 +1013,18 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb, /* * File creation. Allocate an inode, and we're done.. */ -static int do_hugetlbfs_mknod(struct inode *dir, - struct dentry *dentry, - umode_t mode, - dev_t dev, - bool tmpfile) +static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) { struct inode *inode; - int error = -ENOSPC; inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev); - if (inode) { - dir->i_ctime = dir->i_mtime = current_time(dir); - if (tmpfile) { - d_tmpfile(dentry, inode); - } else { - d_instantiate(dentry, inode); - dget(dentry);/* Extra count - pin the dentry in core */ - } - error = 0; - } - return error; -} - -static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode, dev_t dev) -{ - return do_hugetlbfs_mknod(dir, dentry, mode, dev, false); + if (!inode) + return -ENOSPC; + dir->i_ctime = dir->i_mtime = current_time(dir); + d_instantiate(dentry, inode); + dget(dentry);/* Extra count - pin the dentry in core */ + return 0; } static int hugetlbfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, @@ -932,10 +1045,17 @@ static int hugetlbfs_create(struct user_namespace *mnt_userns, } static int hugetlbfs_tmpfile(struct user_namespace *mnt_userns, - struct inode *dir, struct dentry *dentry, + struct inode *dir, struct file *file, umode_t mode) { - return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true); + struct inode *inode; + + inode = hugetlbfs_get_inode(dir->i_sb, dir, mode | S_IFREG, 0); + if (!inode) + return -ENOSPC; + dir->i_ctime = dir->i_mtime = current_time(dir); + d_tmpfile(file, inode); + return finish_open_simple(file, 0); } static int hugetlbfs_symlink(struct user_namespace *mnt_userns, @@ -994,7 +1114,7 @@ static int hugetlbfs_error_remove_page(struct address_space *mapping, struct inode *inode = mapping->host; pgoff_t index = page->index; - remove_huge_page(page); + hugetlb_delete_from_page_cache(page); if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1))) hugetlb_fix_reserve_counts(inode); diff --git a/fs/inode.c b/fs/inode.c index 6462276dfdf04d9ce648b7a2f4a06faa727d9de0..b608528efd3a463d885a7c1205575cf4409ce497 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -192,8 +192,6 @@ int inode_init_always(struct super_block *sb, struct inode *inode) inode->i_wb_frn_history = 0; #endif - if (security_inode_alloc(inode)) - goto out; spin_lock_init(&inode->i_lock); lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key); @@ -228,11 +226,12 @@ int inode_init_always(struct super_block *sb, struct inode *inode) inode->i_fsnotify_mask = 0; #endif inode->i_flctx = NULL; + + if (unlikely(security_inode_alloc(inode))) + return -ENOMEM; this_cpu_inc(nr_inodes); return 0; -out: - return -ENOMEM; } EXPORT_SYMBOL(inode_init_always); @@ -2018,23 +2017,25 @@ static int __file_remove_privs(struct file *file, unsigned int flags) { struct dentry *dentry = file_dentry(file); struct inode *inode = file_inode(file); - int error; + int error = 0; int kill; if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode)) return 0; kill = dentry_needs_remove_privs(dentry); - if (kill <= 0) + if (kill < 0) return kill; - if (flags & IOCB_NOWAIT) - return -EAGAIN; + if (kill) { + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + error = __remove_privs(file_mnt_user_ns(file), dentry, kill); + } - error = __remove_privs(file_mnt_user_ns(file), dentry, kill); if (!error) inode_has_no_xattr(inode); - return error; } diff --git a/fs/internal.h b/fs/internal.h index 87e96b9024ce1db0ee8b6b53a45e0122d7a5eb71..6f0386b34faeca2656575ebd176777239eefa488 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -16,6 +16,7 @@ struct shrink_control; struct fs_context; struct user_namespace; struct pipe_inode_info; +struct iov_iter; /* * block/bdev.c @@ -62,7 +63,7 @@ extern int vfs_path_lookup(struct dentry *, struct vfsmount *, const char *, unsigned int, struct path *); int do_rmdir(int dfd, struct filename *name); int do_unlinkat(int dfd, struct filename *name); -int may_linkat(struct user_namespace *mnt_userns, struct path *link); +int may_linkat(struct user_namespace *mnt_userns, const struct path *link); int do_renameat2(int olddfd, struct filename *oldname, int newdfd, struct filename *newname, unsigned int flags); int do_mkdirat(int dfd, struct filename *name, umode_t mode); @@ -101,6 +102,16 @@ extern void chroot_fs_refs(const struct path *, const struct path *); extern struct file *alloc_empty_file(int, const struct cred *); extern struct file *alloc_empty_file_noaccount(int, const struct cred *); +static inline void put_file_access(struct file *file) +{ + if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) { + i_readcount_dec(file->f_inode); + } else if (file->f_mode & FMODE_WRITER) { + put_write_access(file->f_inode); + __mnt_drop_write(file->f_path.mnt); + } +} + /* * super.c */ @@ -221,3 +232,5 @@ ssize_t do_getxattr(struct user_namespace *mnt_userns, int setxattr_copy(const char __user *name, struct xattr_ctx *ctx); int do_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, struct xattr_ctx *ctx); + +ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos); diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index ca5c62901541e42ca1bfdb88c180f1a1486901bd..91ee0b308e13d3e55368747f81fa19124baf1b18 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -1360,6 +1360,7 @@ iomap_writepage_map(struct iomap_writepage_ctx *wpc, error = wpc->ops->map_blocks(wpc, inode, pos); if (error) break; + trace_iomap_writepage_map(inode, &wpc->iomap); if (WARN_ON_ONCE(wpc->iomap.type == IOMAP_INLINE)) continue; if (wpc->iomap.type == IOMAP_HOLE) @@ -1421,7 +1422,7 @@ iomap_writepage_map(struct iomap_writepage_ctx *wpc, if (!count) folio_end_writeback(folio); done: - mapping_set_error(folio->mapping, error); + mapping_set_error(inode->i_mapping, error); return error; } diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h index d48868fc40d78f37273365e458a8dd9ea0791423..f6ea9540d082d4eddd37d4cb8d3cc95d07a0dd18 100644 --- a/fs/iomap/trace.h +++ b/fs/iomap/trace.h @@ -148,6 +148,7 @@ DEFINE_EVENT(iomap_class, name, \ TP_ARGS(inode, iomap)) DEFINE_IOMAP_EVENT(iomap_iter_dstmap); DEFINE_IOMAP_EVENT(iomap_iter_srcmap); +DEFINE_IOMAP_EVENT(iomap_writepage_map); TRACE_EVENT(iomap_iter, TP_PROTO(struct iomap_iter *iter, const void *ops, diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index b466172eec25b19b3dabc0387136fbf69088b59d..c4da3f634b92e23ca5ade630b8eb38eb8730b814 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -67,8 +67,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, for ( i = 0 ; i < pcount ; i++ ) { if (!pages[i]) continue; - memset(page_address(pages[i]), 0, PAGE_SIZE); - flush_dcache_page(pages[i]); + memzero_page(pages[i], 0, PAGE_SIZE); SetPageUptodate(pages[i]); } return ((loff_t)pcount) << PAGE_SHIFT; @@ -82,7 +81,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, return 0; } haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks); - ll_rw_block(REQ_OP_READ, haveblocks, bhs); + bh_read_batch(haveblocks, bhs); curbh = 0; curpage = 0; @@ -120,7 +119,7 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, zerr != Z_STREAM_END) { if (!stream.avail_out) { if (pages[curpage]) { - stream.next_out = page_address(pages[curpage]) + stream.next_out = kmap_local_page(pages[curpage]) + poffset; stream.avail_out = PAGE_SIZE - poffset; poffset = 0; @@ -176,6 +175,10 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, flush_dcache_page(pages[curpage]); SetPageUptodate(pages[curpage]); } + if (stream.next_out != (unsigned char *)zisofs_sink_page) { + kunmap_local(stream.next_out); + stream.next_out = NULL; + } curpage++; } if (!stream.avail_in) @@ -183,6 +186,8 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, } inflate_out: zlib_inflateEnd(&stream); + if (stream.next_out && stream.next_out != (unsigned char *)zisofs_sink_page) + kunmap_local(stream.next_out); z_eio: mutex_unlock(&zisofs_zlib_lock); @@ -283,9 +288,7 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, } if (poffset && *pages) { - memset(page_address(*pages) + poffset, 0, - PAGE_SIZE - poffset); - flush_dcache_page(*pages); + memzero_page(*pages, poffset, PAGE_SIZE - poffset); SetPageUptodate(*pages); } return 0; @@ -343,10 +346,8 @@ static int zisofs_read_folio(struct file *file, struct folio *folio) for (i = 0; i < pcount; i++, index++) { if (i != full_page) pages[i] = grab_cache_page_nowait(mapping, index); - if (pages[i]) { + if (pages[i]) ClearPageError(pages[i]); - kmap(pages[i]); - } } err = zisofs_fill_pages(inode, full_page, pcount, pages); @@ -357,7 +358,6 @@ static int zisofs_read_folio(struct file *file, struct folio *folio) flush_dcache_page(pages[i]); if (i == full_page && err) SetPageError(pages[i]); - kunmap(pages[i]); unlock_page(pages[i]); if (i != full_page) put_page(pages[i]); diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index 88bf2030346603cb978a3cec49e72a210ada1996..df9d70588b600b382e8075bc3898f0d488fc086a 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -1277,13 +1277,11 @@ static int isofs_read_level3_size(struct inode *inode) } while (more_entries); out: kfree(tmpde); - if (bh) - brelse(bh); + brelse(bh); return 0; out_nomem: - if (bh) - brelse(bh); + brelse(bh); return -ENOMEM; out_noread: @@ -1486,8 +1484,7 @@ static int isofs_read_inode(struct inode *inode, int relocated) ret = 0; out: kfree(tmpde); - if (bh) - brelse(bh); + brelse(bh); return ret; out_badread: diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c index b2b2bc9b88d9d1fbe825d045c2161024db3ca8fc..885a7a6cc53e6df5315adfa6419a11e0c4ce536f 100644 --- a/fs/jbd2/commit.c +++ b/fs/jbd2/commit.c @@ -122,8 +122,8 @@ static int journal_submit_commit_record(journal_t *journal, { struct commit_header *tmp; struct buffer_head *bh; - int ret; struct timespec64 now; + blk_opf_t write_flags = REQ_OP_WRITE | REQ_SYNC; *cbh = NULL; @@ -155,13 +155,11 @@ static int journal_submit_commit_record(journal_t *journal, if (journal->j_flags & JBD2_BARRIER && !jbd2_has_feature_async_commit(journal)) - ret = submit_bh(REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH | - REQ_FUA, bh); - else - ret = submit_bh(REQ_OP_WRITE | REQ_SYNC, bh); + write_flags |= REQ_PREFLUSH | REQ_FUA; + submit_bh(write_flags, bh); *cbh = bh; - return ret; + return 0; } /* @@ -570,7 +568,7 @@ void jbd2_journal_commit_transaction(journal_t *journal) journal->j_running_transaction = NULL; start_time = ktime_get(); commit_transaction->t_log_start = journal->j_head; - wake_up(&journal->j_wait_transaction_locked); + wake_up_all(&journal->j_wait_transaction_locked); write_unlock(&journal->j_state_lock); jbd2_debug(3, "JBD2: commit phase 2a\n"); diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c index 6350d3857c89606ffa6ac7f58f64df71b454f037..2696f43e7239f809bcc1ff7b784ea51d94fa6f99 100644 --- a/fs/jbd2/journal.c +++ b/fs/jbd2/journal.c @@ -923,10 +923,16 @@ int jbd2_fc_wait_bufs(journal_t *journal, int num_blks) for (i = j_fc_off - 1; i >= j_fc_off - num_blks; i--) { bh = journal->j_fc_wbuf[i]; wait_on_buffer(bh); + /* + * Update j_fc_off so jbd2_fc_release_bufs can release remain + * buffer head. + */ + if (unlikely(!buffer_uptodate(bh))) { + journal->j_fc_off = i + 1; + return -EIO; + } put_bh(bh); journal->j_fc_wbuf[i] = NULL; - if (unlikely(!buffer_uptodate(bh))) - return -EIO; } return 0; @@ -1606,7 +1612,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) { struct buffer_head *bh = journal->j_sb_buffer; journal_superblock_t *sb = journal->j_superblock; - int ret; + int ret = 0; /* Buffer got discarded which means block device got invalidated */ if (!buffer_mapped(bh)) { @@ -1636,7 +1642,7 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) sb->s_checksum = jbd2_superblock_csum(journal, sb); get_bh(bh); bh->b_end_io = end_buffer_write_sync; - ret = submit_bh(REQ_OP_WRITE | write_flags, bh); + submit_bh(REQ_OP_WRITE | write_flags, bh); wait_on_buffer(bh); if (buffer_write_io_error(bh)) { clear_buffer_write_io_error(bh); @@ -1644,9 +1650,8 @@ static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) ret = -EIO; } if (ret) { - printk(KERN_ERR "JBD2: Error %d detected when updating " - "journal superblock for %s.\n", ret, - journal->j_devname); + printk(KERN_ERR "JBD2: I/O error when updating journal superblock for %s.\n", + journal->j_devname); if (!is_journal_aborted(journal)) jbd2_journal_abort(journal, ret); } @@ -1893,19 +1898,16 @@ static int journal_get_superblock(journal_t *journal) { struct buffer_head *bh; journal_superblock_t *sb; - int err = -EIO; + int err; bh = journal->j_sb_buffer; J_ASSERT(bh != NULL); - if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { - printk(KERN_ERR - "JBD2: IO error reading journal superblock\n"); - goto out; - } + err = bh_read(bh, 0); + if (err < 0) { + printk(KERN_ERR + "JBD2: IO error reading journal superblock\n"); + goto out; } if (buffer_verified(bh)) diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c index f548479615c69ccfe2dbd2733bc3d6feafd39290..8286a9ec122feb209bdae1c211d803024c5f0f02 100644 --- a/fs/jbd2/recovery.c +++ b/fs/jbd2/recovery.c @@ -100,7 +100,7 @@ static int do_readahead(journal_t *journal, unsigned int start) if (!buffer_uptodate(bh) && !buffer_locked(bh)) { bufs[nbufs++] = bh; if (nbufs == MAXBUF) { - ll_rw_block(REQ_OP_READ, nbufs, bufs); + bh_readahead_batch(nbufs, bufs, 0); journal_brelse_array(bufs, nbufs); nbufs = 0; } @@ -109,7 +109,7 @@ static int do_readahead(journal_t *journal, unsigned int start) } if (nbufs) - ll_rw_block(REQ_OP_READ, nbufs, bufs); + bh_readahead_batch(nbufs, bufs, 0); err = 0; failed: @@ -152,9 +152,14 @@ static int jread(struct buffer_head **bhp, journal_t *journal, return -ENOMEM; if (!buffer_uptodate(bh)) { - /* If this is a brand new buffer, start readahead. - Otherwise, we assume we are already reading it. */ - if (!buffer_req(bh)) + /* + * If this is a brand new buffer, start readahead. + * Otherwise, we assume we are already reading it. + */ + bool need_readahead = !buffer_req(bh); + + bh_read_nowait(bh, 0); + if (need_readahead) do_readahead(journal, offset); wait_on_buffer(bh); } @@ -256,6 +261,7 @@ static int fc_do_one_pass(journal_t *journal, err = journal->j_fc_replay_callback(journal, bh, pass, next_fc_block - journal->j_fc_first, expected_commit_id); + brelse(bh); next_fc_block++; if (err < 0 || err == JBD2_FC_REPLAY_STOP) break; @@ -687,7 +693,6 @@ static int do_one_pass(journal_t *journal, mark_buffer_dirty(nbh); BUFFER_TRACE(nbh, "marking uptodate"); ++info->nr_replays; - /* ll_rw_block(WRITE, 1, &nbh); */ unlock_buffer(nbh); brelse(obh); brelse(nbh); diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c index e1be93ccd81cb43d259b5265ce88b473a7220aee..6a404ac1c178f03652d7c8f8acc1b1191441c9b3 100644 --- a/fs/jbd2/transaction.c +++ b/fs/jbd2/transaction.c @@ -168,7 +168,7 @@ static void wait_transaction_locked(journal_t *journal) int need_to_start; tid_t tid = journal->j_running_transaction->t_tid; - prepare_to_wait(&journal->j_wait_transaction_locked, &wait, + prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, TASK_UNINTERRUPTIBLE); need_to_start = !tid_geq(journal->j_commit_request, tid); read_unlock(&journal->j_state_lock); @@ -194,7 +194,7 @@ static void wait_transaction_switching(journal_t *journal) read_unlock(&journal->j_state_lock); return; } - prepare_to_wait(&journal->j_wait_transaction_locked, &wait, + prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, TASK_UNINTERRUPTIBLE); read_unlock(&journal->j_state_lock); /* @@ -920,7 +920,7 @@ void jbd2_journal_unlock_updates (journal_t *journal) write_lock(&journal->j_state_lock); --journal->j_barrier_count; write_unlock(&journal->j_state_lock); - wake_up(&journal->j_wait_transaction_locked); + wake_up_all(&journal->j_wait_transaction_locked); } static void warn_dirty_buffer(struct buffer_head *bh) diff --git a/fs/jffs2/wbuf.c b/fs/jffs2/wbuf.c index c6821a5094818a48030697b95178241304905d28..4061e0ba70103c3c2f78c112a0336718bdd6e03d 100644 --- a/fs/jffs2/wbuf.c +++ b/fs/jffs2/wbuf.c @@ -1035,7 +1035,7 @@ int jffs2_check_oob_empty(struct jffs2_sb_info *c, { int i, ret; int cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; ops.mode = MTD_OPS_AUTO_OOB; ops.ooblen = NR_OOB_SCAN_PAGES * c->oobavail; @@ -1076,7 +1076,7 @@ int jffs2_check_oob_empty(struct jffs2_sb_info *c, int jffs2_check_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) { - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int ret, cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); ops.mode = MTD_OPS_AUTO_OOB; @@ -1101,7 +1101,7 @@ int jffs2_write_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) { int ret; - struct mtd_oob_ops ops; + struct mtd_oob_ops ops = { }; int cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); ops.mode = MTD_OPS_AUTO_OOB; diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 1cc88ba6de9077c8250f4edf09d3279b507e289b..3990f3e270cb67707c019703ac9868b0b7523617 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -472,6 +472,16 @@ static void kernfs_drain(struct kernfs_node *kn) lockdep_assert_held_write(&root->kernfs_rwsem); WARN_ON_ONCE(kernfs_active(kn)); + /* + * Skip draining if already fully drained. This avoids draining and its + * lockdep annotations for nodes which have never been activated + * allowing embedding kernfs_remove() in create error paths without + * worrying about draining. + */ + if (atomic_read(&kn->active) == KN_DEACTIVATED_BIAS && + !kernfs_should_drain_open_files(kn)) + return; + up_write(&root->kernfs_rwsem); if (kernfs_lockdep(kn)) { @@ -480,7 +490,6 @@ static void kernfs_drain(struct kernfs_node *kn) lock_contended(&kn->dep_map, _RET_IP_); } - /* but everyone should wait for draining */ wait_event(root->deactivate_waitq, atomic_read(&kn->active) == KN_DEACTIVATED_BIAS); @@ -489,7 +498,8 @@ static void kernfs_drain(struct kernfs_node *kn) rwsem_release(&kn->dep_map, _RET_IP_); } - kernfs_drain_open_files(kn); + if (kernfs_should_drain_open_files(kn)) + kernfs_drain_open_files(kn); down_write(&root->kernfs_rwsem); } @@ -695,13 +705,7 @@ struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, goto err_unlock; } - /* - * ACTIVATED is protected with kernfs_mutex but it was clear when - * @kn was added to idr and we just wanna see it set. No need to - * grab kernfs_mutex. - */ - if (unlikely(!(kn->flags & KERNFS_ACTIVATED) || - !atomic_inc_not_zero(&kn->count))) + if (unlikely(!kernfs_active(kn) || !atomic_inc_not_zero(&kn->count))) goto err_unlock; spin_unlock(&kernfs_idr_lock); @@ -743,10 +747,7 @@ int kernfs_add_one(struct kernfs_node *kn) goto out_unlock; ret = -ENOENT; - if (parent->flags & KERNFS_EMPTY_DIR) - goto out_unlock; - - if ((parent->flags & KERNFS_ACTIVATED) && !kernfs_active(parent)) + if (parent->flags & (KERNFS_REMOVING | KERNFS_EMPTY_DIR)) goto out_unlock; kn->hash = kernfs_name_hash(kn->name, kn->ns); @@ -1304,6 +1305,21 @@ static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos, return pos->parent; } +static void kernfs_activate_one(struct kernfs_node *kn) +{ + lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem); + + kn->flags |= KERNFS_ACTIVATED; + + if (kernfs_active(kn) || (kn->flags & (KERNFS_HIDDEN | KERNFS_REMOVING))) + return; + + WARN_ON_ONCE(kn->parent && RB_EMPTY_NODE(&kn->rb)); + WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); + + atomic_sub(KN_DEACTIVATED_BIAS, &kn->active); +} + /** * kernfs_activate - activate a node which started deactivated * @kn: kernfs_node whose subtree is to be activated @@ -1325,15 +1341,42 @@ void kernfs_activate(struct kernfs_node *kn) down_write(&root->kernfs_rwsem); pos = NULL; - while ((pos = kernfs_next_descendant_post(pos, kn))) { - if (pos->flags & KERNFS_ACTIVATED) - continue; + while ((pos = kernfs_next_descendant_post(pos, kn))) + kernfs_activate_one(pos); - WARN_ON_ONCE(pos->parent && RB_EMPTY_NODE(&pos->rb)); - WARN_ON_ONCE(atomic_read(&pos->active) != KN_DEACTIVATED_BIAS); + up_write(&root->kernfs_rwsem); +} - atomic_sub(KN_DEACTIVATED_BIAS, &pos->active); - pos->flags |= KERNFS_ACTIVATED; +/** + * kernfs_show - show or hide a node + * @kn: kernfs_node to show or hide + * @show: whether to show or hide + * + * If @show is %false, @kn is marked hidden and deactivated. A hidden node is + * ignored in future activaitons. If %true, the mark is removed and activation + * state is restored. This function won't implicitly activate a new node in a + * %KERNFS_ROOT_CREATE_DEACTIVATED root which hasn't been activated yet. + * + * To avoid recursion complexities, directories aren't supported for now. + */ +void kernfs_show(struct kernfs_node *kn, bool show) +{ + struct kernfs_root *root = kernfs_root(kn); + + if (WARN_ON_ONCE(kernfs_type(kn) == KERNFS_DIR)) + return; + + down_write(&root->kernfs_rwsem); + + if (show) { + kn->flags &= ~KERNFS_HIDDEN; + if (kn->flags & KERNFS_ACTIVATED) + kernfs_activate_one(kn); + } else { + kn->flags |= KERNFS_HIDDEN; + if (kernfs_active(kn)) + atomic_add(KN_DEACTIVATED_BIAS, &kn->active); + kernfs_drain(kn); } up_write(&root->kernfs_rwsem); @@ -1358,34 +1401,27 @@ static void __kernfs_remove(struct kernfs_node *kn) pr_debug("kernfs %s: removing\n", kn->name); - /* prevent any new usage under @kn by deactivating all nodes */ + /* prevent new usage by marking all nodes removing and deactivating */ pos = NULL; - while ((pos = kernfs_next_descendant_post(pos, kn))) + while ((pos = kernfs_next_descendant_post(pos, kn))) { + pos->flags |= KERNFS_REMOVING; if (kernfs_active(pos)) atomic_add(KN_DEACTIVATED_BIAS, &pos->active); + } /* deactivate and unlink the subtree node-by-node */ do { pos = kernfs_leftmost_descendant(kn); /* - * kernfs_drain() drops kernfs_rwsem temporarily and @pos's + * kernfs_drain() may drop kernfs_rwsem temporarily and @pos's * base ref could have been put by someone else by the time * the function returns. Make sure it doesn't go away * underneath us. */ kernfs_get(pos); - /* - * Drain iff @kn was activated. This avoids draining and - * its lockdep annotations for nodes which have never been - * activated and allows embedding kernfs_remove() in create - * error paths without worrying about draining. - */ - if (kn->flags & KERNFS_ACTIVATED) - kernfs_drain(pos); - else - WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); + kernfs_drain(pos); /* * kernfs_unlink_sibling() succeeds once per node. Use it @@ -1585,8 +1621,11 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name, down_write(&root->kernfs_rwsem); kn = kernfs_find_ns(parent, name, ns); - if (kn) + if (kn) { + kernfs_get(kn); __kernfs_remove(kn); + kernfs_put(kn); + } up_write(&root->kernfs_rwsem); diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index b3ec34386b43c0d6432155926a8e0e9dfe1d0030..9ab6c92e02dab7489adb232563e55fea8e48b4bc 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -23,6 +23,8 @@ struct kernfs_open_node { atomic_t event; wait_queue_head_t poll; struct list_head files; /* goes through kernfs_open_file.list */ + unsigned int nr_mmapped; + unsigned int nr_to_release; }; /* @@ -57,31 +59,17 @@ static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn) } /** - * kernfs_deref_open_node - Get kernfs_open_node corresponding to @kn. - * - * @of: associated kernfs_open_file instance. - * @kn: target kernfs_node. - * - * Fetch and return ->attr.open of @kn if @of->list is non empty. - * If @of->list is not empty we can safely assume that @of is on - * @kn->attr.open->files list and this guarantees that @kn->attr.open - * will not vanish i.e. dereferencing outside RCU read-side critical - * section is safe here. - * - * The caller needs to make sure that @of->list is not empty. + * of_on - Return the kernfs_open_node of the specified kernfs_open_file + * @of: taret kernfs_open_file */ -static struct kernfs_open_node * -kernfs_deref_open_node(struct kernfs_open_file *of, struct kernfs_node *kn) +static struct kernfs_open_node *of_on(struct kernfs_open_file *of) { - struct kernfs_open_node *on; - - on = rcu_dereference_check(kn->attr.open, !list_empty(&of->list)); - - return on; + return rcu_dereference_protected(of->kn->attr.open, + !list_empty(&of->list)); } /** - * kernfs_deref_open_node_protected - Get kernfs_open_node corresponding to @kn + * kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn * * @kn: target kernfs_node. * @@ -96,7 +84,7 @@ kernfs_deref_open_node(struct kernfs_open_file *of, struct kernfs_node *kn) * The caller needs to make sure that kernfs_open_file_mutex is held. */ static struct kernfs_open_node * -kernfs_deref_open_node_protected(struct kernfs_node *kn) +kernfs_deref_open_node_locked(struct kernfs_node *kn) { return rcu_dereference_protected(kn->attr.open, lockdep_is_held(kernfs_open_file_mutex_ptr(kn))); @@ -207,12 +195,8 @@ static void kernfs_seq_stop(struct seq_file *sf, void *v) static int kernfs_seq_show(struct seq_file *sf, void *v) { struct kernfs_open_file *of = sf->private; - struct kernfs_open_node *on = kernfs_deref_open_node(of, of->kn); - - if (!on) - return -EINVAL; - of->event = atomic_read(&on->event); + of->event = atomic_read(&of_on(of)->event); return of->kn->attr.ops->seq_show(sf, v); } @@ -235,7 +219,6 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) struct kernfs_open_file *of = kernfs_of(iocb->ki_filp); ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE); const struct kernfs_ops *ops; - struct kernfs_open_node *on; char *buf; buf = of->prealloc_buf; @@ -257,14 +240,7 @@ static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) goto out_free; } - on = kernfs_deref_open_node(of, of->kn); - if (!on) { - len = -EINVAL; - mutex_unlock(&of->mutex); - goto out_free; - } - - of->event = atomic_read(&on->event); + of->event = atomic_read(&of_on(of)->event); ops = kernfs_ops(of->kn); if (ops->read) @@ -553,6 +529,7 @@ static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma) rc = 0; of->mmapped = true; + of_on(of)->nr_mmapped++; of->vm_ops = vma->vm_ops; vma->vm_ops = &kernfs_vm_ops; out_put: @@ -580,31 +557,30 @@ out_unlock: static int kernfs_get_open_node(struct kernfs_node *kn, struct kernfs_open_file *of) { - struct kernfs_open_node *on, *new_on = NULL; - struct mutex *mutex = NULL; + struct kernfs_open_node *on; + struct mutex *mutex; mutex = kernfs_open_file_mutex_lock(kn); - on = kernfs_deref_open_node_protected(kn); + on = kernfs_deref_open_node_locked(kn); - if (on) { - list_add_tail(&of->list, &on->files); - mutex_unlock(mutex); - return 0; - } else { + if (!on) { /* not there, initialize a new one */ - new_on = kmalloc(sizeof(*new_on), GFP_KERNEL); - if (!new_on) { + on = kzalloc(sizeof(*on), GFP_KERNEL); + if (!on) { mutex_unlock(mutex); return -ENOMEM; } - atomic_set(&new_on->event, 1); - init_waitqueue_head(&new_on->poll); - INIT_LIST_HEAD(&new_on->files); - list_add_tail(&of->list, &new_on->files); - rcu_assign_pointer(kn->attr.open, new_on); + atomic_set(&on->event, 1); + init_waitqueue_head(&on->poll); + INIT_LIST_HEAD(&on->files); + rcu_assign_pointer(kn->attr.open, on); } - mutex_unlock(mutex); + list_add_tail(&of->list, &on->files); + if (kn->flags & KERNFS_HAS_RELEASE) + on->nr_to_release++; + + mutex_unlock(mutex); return 0; } @@ -613,6 +589,7 @@ static int kernfs_get_open_node(struct kernfs_node *kn, * * @kn: target kernfs_node * @of: associated kernfs_open_file + * @open_failed: ->open() failed, cancel ->release() * * Unlink @of from list of @kn's associated open files. If list of * associated open files becomes empty, disassociate and free @@ -622,21 +599,30 @@ static int kernfs_get_open_node(struct kernfs_node *kn, * None. */ static void kernfs_unlink_open_file(struct kernfs_node *kn, - struct kernfs_open_file *of) + struct kernfs_open_file *of, + bool open_failed) { struct kernfs_open_node *on; - struct mutex *mutex = NULL; + struct mutex *mutex; mutex = kernfs_open_file_mutex_lock(kn); - on = kernfs_deref_open_node_protected(kn); + on = kernfs_deref_open_node_locked(kn); if (!on) { mutex_unlock(mutex); return; } - if (of) + if (of) { + if (kn->flags & KERNFS_HAS_RELEASE) { + WARN_ON_ONCE(of->released == open_failed); + if (open_failed) + on->nr_to_release--; + } + if (of->mmapped) + on->nr_mmapped--; list_del(&of->list); + } if (list_empty(&on->files)) { rcu_assign_pointer(kn->attr.open, NULL); @@ -763,7 +749,7 @@ static int kernfs_fop_open(struct inode *inode, struct file *file) return 0; err_put_node: - kernfs_unlink_open_file(kn, of); + kernfs_unlink_open_file(kn, of, true); err_seq_release: seq_release(inode, file); err_free: @@ -795,6 +781,7 @@ static void kernfs_release_file(struct kernfs_node *kn, */ kn->attr.ops->release(of); of->released = true; + of_on(of)->nr_to_release--; } } @@ -802,15 +789,16 @@ static int kernfs_fop_release(struct inode *inode, struct file *filp) { struct kernfs_node *kn = inode->i_private; struct kernfs_open_file *of = kernfs_of(filp); - struct mutex *mutex = NULL; if (kn->flags & KERNFS_HAS_RELEASE) { + struct mutex *mutex; + mutex = kernfs_open_file_mutex_lock(kn); kernfs_release_file(kn, of); mutex_unlock(mutex); } - kernfs_unlink_open_file(kn, of); + kernfs_unlink_open_file(kn, of, false); seq_release(inode, filp); kfree(of->prealloc_buf); kfree(of); @@ -818,28 +806,33 @@ static int kernfs_fop_release(struct inode *inode, struct file *filp) return 0; } -void kernfs_drain_open_files(struct kernfs_node *kn) +bool kernfs_should_drain_open_files(struct kernfs_node *kn) { struct kernfs_open_node *on; - struct kernfs_open_file *of; - struct mutex *mutex = NULL; - - if (!(kn->flags & (KERNFS_HAS_MMAP | KERNFS_HAS_RELEASE))) - return; + bool ret; /* - * lockless opportunistic check is safe below because no one is adding to - * ->attr.open at this point of time. This check allows early bail out - * if ->attr.open is already NULL. kernfs_unlink_open_file makes - * ->attr.open NULL only while holding kernfs_open_file_mutex so below - * check under kernfs_open_file_mutex_ptr(kn) will ensure bailing out if - * ->attr.open became NULL while waiting for the mutex. + * @kn being deactivated guarantees that @kn->attr.open can't change + * beneath us making the lockless test below safe. */ - if (!rcu_access_pointer(kn->attr.open)) - return; + WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); + + rcu_read_lock(); + on = rcu_dereference(kn->attr.open); + ret = on && (on->nr_mmapped || on->nr_to_release); + rcu_read_unlock(); + + return ret; +} + +void kernfs_drain_open_files(struct kernfs_node *kn) +{ + struct kernfs_open_node *on; + struct kernfs_open_file *of; + struct mutex *mutex; mutex = kernfs_open_file_mutex_lock(kn); - on = kernfs_deref_open_node_protected(kn); + on = kernfs_deref_open_node_locked(kn); if (!on) { mutex_unlock(mutex); return; @@ -848,13 +841,17 @@ void kernfs_drain_open_files(struct kernfs_node *kn) list_for_each_entry(of, &on->files, list) { struct inode *inode = file_inode(of->file); - if (kn->flags & KERNFS_HAS_MMAP) + if (of->mmapped) { unmap_mapping_range(inode->i_mapping, 0, 0, 1); + of->mmapped = false; + on->nr_mmapped--; + } if (kn->flags & KERNFS_HAS_RELEASE) kernfs_release_file(kn, of); } + WARN_ON_ONCE(on->nr_mmapped || on->nr_to_release); mutex_unlock(mutex); } @@ -874,11 +871,7 @@ void kernfs_drain_open_files(struct kernfs_node *kn) */ __poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait) { - struct kernfs_node *kn = kernfs_dentry_node(of->file->f_path.dentry); - struct kernfs_open_node *on = kernfs_deref_open_node(of, kn); - - if (!on) - return EPOLLERR; + struct kernfs_open_node *on = of_on(of); poll_wait(of->file, &on->poll, wait); diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h index 3ae214d02d4417ad63c729cad989f1160cc37350..fc5821effd97dc8667ebcb542cef24a789dd8f0c 100644 --- a/fs/kernfs/kernfs-internal.h +++ b/fs/kernfs/kernfs-internal.h @@ -157,6 +157,7 @@ struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, */ extern const struct file_operations kernfs_file_fops; +bool kernfs_should_drain_open_files(struct kernfs_node *kn); void kernfs_drain_open_files(struct kernfs_node *kn); /* diff --git a/fs/ksmbd/auth.c b/fs/ksmbd/auth.c index c5a5c7b90d727135d8f91bc892ebda26526dd25c..2a39ffb8423b75dfc205215d4df85b09a1859aa4 100644 --- a/fs/ksmbd/auth.c +++ b/fs/ksmbd/auth.c @@ -424,6 +424,9 @@ ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, NTLMSSP_NEGOTIATE_56); } + if (cflags & NTLMSSP_NEGOTIATE_SEAL && smb3_encryption_negotiated(conn)) + flags |= NTLMSSP_NEGOTIATE_SEAL; + if (cflags & NTLMSSP_NEGOTIATE_ALWAYS_SIGN) flags |= NTLMSSP_NEGOTIATE_ALWAYS_SIGN; @@ -984,13 +987,16 @@ out: return rc; } -static int ksmbd_get_encryption_key(struct ksmbd_conn *conn, __u64 ses_id, +static int ksmbd_get_encryption_key(struct ksmbd_work *work, __u64 ses_id, int enc, u8 *key) { struct ksmbd_session *sess; u8 *ses_enc_key; - sess = ksmbd_session_lookup_all(conn, ses_id); + if (enc) + sess = work->sess; + else + sess = ksmbd_session_lookup_all(work->conn, ses_id); if (!sess) return -EINVAL; @@ -1078,9 +1084,10 @@ static struct scatterlist *ksmbd_init_sg(struct kvec *iov, unsigned int nvec, return sg; } -int ksmbd_crypt_message(struct ksmbd_conn *conn, struct kvec *iov, +int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, unsigned int nvec, int enc) { + struct ksmbd_conn *conn = work->conn; struct smb2_transform_hdr *tr_hdr = smb2_get_msg(iov[0].iov_base); unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; int rc; @@ -1094,7 +1101,7 @@ int ksmbd_crypt_message(struct ksmbd_conn *conn, struct kvec *iov, unsigned int crypt_len = le32_to_cpu(tr_hdr->OriginalMessageSize); struct ksmbd_crypto_ctx *ctx; - rc = ksmbd_get_encryption_key(conn, + rc = ksmbd_get_encryption_key(work, le64_to_cpu(tr_hdr->SessionId), enc, key); diff --git a/fs/ksmbd/auth.h b/fs/ksmbd/auth.h index 25b772653de0a5668764e0977ec7a5299937a327..362b6159a6cffbd1c38361e6f695211199866644 100644 --- a/fs/ksmbd/auth.h +++ b/fs/ksmbd/auth.h @@ -33,9 +33,10 @@ struct ksmbd_session; struct ksmbd_conn; +struct ksmbd_work; struct kvec; -int ksmbd_crypt_message(struct ksmbd_conn *conn, struct kvec *iov, +int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, unsigned int nvec, int enc); void ksmbd_copy_gss_neg_header(void *buf); int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, diff --git a/fs/ksmbd/connection.c b/fs/ksmbd/connection.c index 756ad631c019a00612acd5e762c2ceef5c67dc7b..12be8386446a393aced9947013f410700bacebb0 100644 --- a/fs/ksmbd/connection.c +++ b/fs/ksmbd/connection.c @@ -60,6 +60,12 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) conn->local_nls = load_nls("utf8"); if (!conn->local_nls) conn->local_nls = load_nls_default(); + if (IS_ENABLED(CONFIG_UNICODE)) + conn->um = utf8_load(UNICODE_AGE(12, 1, 0)); + else + conn->um = ERR_PTR(-EOPNOTSUPP); + if (IS_ERR(conn->um)) + conn->um = NULL; atomic_set(&conn->req_running, 0); atomic_set(&conn->r_count, 0); conn->total_credits = 1; @@ -350,6 +356,8 @@ out: wait_event(conn->r_count_q, atomic_read(&conn->r_count) == 0); + if (IS_ENABLED(CONFIG_UNICODE)) + utf8_unload(conn->um); unload_nls(conn->local_nls); if (default_conn_ops.terminate_fn) default_conn_ops.terminate_fn(conn); diff --git a/fs/ksmbd/connection.h b/fs/ksmbd/connection.h index e7f7d570795196fb064f50d139bee3712c1c9c85..3643354a3fa79c2b351c581691eff7ee74fe761b 100644 --- a/fs/ksmbd/connection.h +++ b/fs/ksmbd/connection.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "smb_common.h" #include "ksmbd_work.h" @@ -46,6 +47,7 @@ struct ksmbd_conn { char *request_buf; struct ksmbd_transport *transport; struct nls_table *local_nls; + struct unicode_map *um; struct list_head conns_list; /* smb session 1 per user */ struct xarray sessions; diff --git a/fs/ksmbd/ksmbd_netlink.h b/fs/ksmbd/ksmbd_netlink.h index e0cbcfa98c7ebb64e506404d9c476afcdede782f..ff07c67f4565e6c353727eeb438b6f3cbe929db0 100644 --- a/fs/ksmbd/ksmbd_netlink.h +++ b/fs/ksmbd/ksmbd_netlink.h @@ -163,7 +163,8 @@ struct ksmbd_share_config_response { __u16 force_directory_mode; __u16 force_uid; __u16 force_gid; - __u32 reserved[128]; /* Reserved room */ + __s8 share_name[KSMBD_REQ_MAX_SHARE_NAME]; + __u32 reserved[112]; /* Reserved room */ __u32 veto_list_sz; __s8 ____payload[]; }; diff --git a/fs/ksmbd/mgmt/share_config.c b/fs/ksmbd/mgmt/share_config.c index c9bca1c2c83490ebcd1b51de7b96f1126c46b23a..328a412259dc1b935eb2ce3eee2977d39155157b 100644 --- a/fs/ksmbd/mgmt/share_config.c +++ b/fs/ksmbd/mgmt/share_config.c @@ -16,6 +16,7 @@ #include "user_config.h" #include "user_session.h" #include "../transport_ipc.h" +#include "../misc.h" #define SHARE_HASH_BITS 3 static DEFINE_HASHTABLE(shares_table, SHARE_HASH_BITS); @@ -26,7 +27,7 @@ struct ksmbd_veto_pattern { struct list_head list; }; -static unsigned int share_name_hash(char *name) +static unsigned int share_name_hash(const char *name) { return jhash(name, strlen(name), 0); } @@ -72,7 +73,7 @@ __get_share_config(struct ksmbd_share_config *share) return share; } -static struct ksmbd_share_config *__share_lookup(char *name) +static struct ksmbd_share_config *__share_lookup(const char *name) { struct ksmbd_share_config *share; unsigned int key = share_name_hash(name); @@ -119,7 +120,8 @@ static int parse_veto_list(struct ksmbd_share_config *share, return 0; } -static struct ksmbd_share_config *share_config_request(char *name) +static struct ksmbd_share_config *share_config_request(struct unicode_map *um, + const char *name) { struct ksmbd_share_config_response *resp; struct ksmbd_share_config *share = NULL; @@ -133,6 +135,19 @@ static struct ksmbd_share_config *share_config_request(char *name) if (resp->flags == KSMBD_SHARE_FLAG_INVALID) goto out; + if (*resp->share_name) { + char *cf_resp_name; + bool equal; + + cf_resp_name = ksmbd_casefold_sharename(um, resp->share_name); + if (IS_ERR(cf_resp_name)) + goto out; + equal = !strcmp(cf_resp_name, name); + kfree(cf_resp_name); + if (!equal) + goto out; + } + share = kzalloc(sizeof(struct ksmbd_share_config), GFP_KERNEL); if (!share) goto out; @@ -190,20 +205,11 @@ out: return share; } -static void strtolower(char *share_name) -{ - while (*share_name) { - *share_name = tolower(*share_name); - share_name++; - } -} - -struct ksmbd_share_config *ksmbd_share_config_get(char *name) +struct ksmbd_share_config *ksmbd_share_config_get(struct unicode_map *um, + const char *name) { struct ksmbd_share_config *share; - strtolower(name); - down_read(&shares_table_lock); share = __share_lookup(name); if (share) @@ -212,7 +218,7 @@ struct ksmbd_share_config *ksmbd_share_config_get(char *name) if (share) return share; - return share_config_request(name); + return share_config_request(um, name); } bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, diff --git a/fs/ksmbd/mgmt/share_config.h b/fs/ksmbd/mgmt/share_config.h index 902f2cb1963a93b9f442341afd7a7d453fd81efc..3fd33829394219bbbbe2203c02f770b7e9912c7a 100644 --- a/fs/ksmbd/mgmt/share_config.h +++ b/fs/ksmbd/mgmt/share_config.h @@ -9,6 +9,7 @@ #include #include #include +#include struct ksmbd_share_config { char *name; @@ -74,7 +75,8 @@ static inline void ksmbd_share_config_put(struct ksmbd_share_config *share) __ksmbd_share_config_put(share); } -struct ksmbd_share_config *ksmbd_share_config_get(char *name); +struct ksmbd_share_config *ksmbd_share_config_get(struct unicode_map *um, + const char *name); bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, const char *filename); #endif /* __SHARE_CONFIG_MANAGEMENT_H__ */ diff --git a/fs/ksmbd/mgmt/tree_connect.c b/fs/ksmbd/mgmt/tree_connect.c index 97ab7987df6ebfa8b004b924513f560bedaaae6a..8ce17b3fb8dad4b7e0f8a8fa03d11c4b82294397 100644 --- a/fs/ksmbd/mgmt/tree_connect.c +++ b/fs/ksmbd/mgmt/tree_connect.c @@ -17,7 +17,7 @@ struct ksmbd_tree_conn_status ksmbd_tree_conn_connect(struct ksmbd_conn *conn, struct ksmbd_session *sess, - char *share_name) + const char *share_name) { struct ksmbd_tree_conn_status status = {-ENOENT, NULL}; struct ksmbd_tree_connect_response *resp = NULL; @@ -26,7 +26,7 @@ ksmbd_tree_conn_connect(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct sockaddr *peer_addr; int ret; - sc = ksmbd_share_config_get(share_name); + sc = ksmbd_share_config_get(conn->um, share_name); if (!sc) return status; @@ -61,7 +61,7 @@ ksmbd_tree_conn_connect(struct ksmbd_conn *conn, struct ksmbd_session *sess, struct ksmbd_share_config *new_sc; ksmbd_share_config_del(sc); - new_sc = ksmbd_share_config_get(share_name); + new_sc = ksmbd_share_config_get(conn->um, share_name); if (!new_sc) { pr_err("Failed to update stale share config\n"); status.ret = -ESTALE; diff --git a/fs/ksmbd/mgmt/tree_connect.h b/fs/ksmbd/mgmt/tree_connect.h index 71e50271dccf0bdfbfce797f90815f7c175f14ff..0f97ddc1e39c07919ed5bc153a41cb77ea716316 100644 --- a/fs/ksmbd/mgmt/tree_connect.h +++ b/fs/ksmbd/mgmt/tree_connect.h @@ -42,7 +42,7 @@ struct ksmbd_session; struct ksmbd_tree_conn_status ksmbd_tree_conn_connect(struct ksmbd_conn *conn, struct ksmbd_session *sess, - char *share_name); + const char *share_name); int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, struct ksmbd_tree_connect *tree_conn); diff --git a/fs/ksmbd/misc.c b/fs/ksmbd/misc.c index df991107ad2c79f853bf21726512f29751838618..9e8afaa686e3aa8c12e908348aa38b34168ec367 100644 --- a/fs/ksmbd/misc.c +++ b/fs/ksmbd/misc.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "misc.h" #include "smb_common.h" @@ -159,7 +160,7 @@ out: */ char *convert_to_nt_pathname(struct ksmbd_share_config *share, - struct path *path) + const struct path *path) { char *pathname, *ab_pathname, *nt_pathname; int share_path_len = share->path_sz; @@ -226,26 +227,53 @@ void ksmbd_conv_path_to_windows(char *path) strreplace(path, '/', '\\'); } +char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name) +{ + char *cf_name; + int cf_len; + + cf_name = kzalloc(KSMBD_REQ_MAX_SHARE_NAME, GFP_KERNEL); + if (!cf_name) + return ERR_PTR(-ENOMEM); + + if (IS_ENABLED(CONFIG_UNICODE) && um) { + const struct qstr q_name = {.name = name, .len = strlen(name)}; + + cf_len = utf8_casefold(um, &q_name, cf_name, + KSMBD_REQ_MAX_SHARE_NAME); + if (cf_len < 0) + goto out_ascii; + + return cf_name; + } + +out_ascii: + cf_len = strscpy(cf_name, name, KSMBD_REQ_MAX_SHARE_NAME); + if (cf_len < 0) { + kfree(cf_name); + return ERR_PTR(-E2BIG); + } + + for (; *cf_name; ++cf_name) + *cf_name = isascii(*cf_name) ? tolower(*cf_name) : *cf_name; + return cf_name - cf_len; +} + /** * ksmbd_extract_sharename() - get share name from tree connect request * @treename: buffer containing tree name and share name * * Return: share name on success, otherwise error */ -char *ksmbd_extract_sharename(char *treename) +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename) { - char *name = treename; - char *dst; - char *pos = strrchr(name, '\\'); + const char *name = treename, *pos = strrchr(name, '\\'); if (pos) name = (pos + 1); /* caller has to free the memory */ - dst = kstrdup(name, GFP_KERNEL); - if (!dst) - return ERR_PTR(-ENOMEM); - return dst; + return ksmbd_casefold_sharename(um, name); } /** diff --git a/fs/ksmbd/misc.h b/fs/ksmbd/misc.h index aae2a252945f871d80b1db861d711871fd05c860..1facfcd21200fd1af3f4c062da4bfb9d0b3e69c4 100644 --- a/fs/ksmbd/misc.h +++ b/fs/ksmbd/misc.h @@ -15,12 +15,13 @@ int match_pattern(const char *str, size_t len, const char *pattern); int ksmbd_validate_filename(char *filename); int parse_stream_name(char *filename, char **stream_name, int *s_type); char *convert_to_nt_pathname(struct ksmbd_share_config *share, - struct path *path); + const struct path *path); int get_nlink(struct kstat *st); void ksmbd_conv_path_to_unix(char *path); void ksmbd_strip_last_slash(char *path); void ksmbd_conv_path_to_windows(char *path); -char *ksmbd_extract_sharename(char *treename); +char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name); +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); char *convert_to_unix_name(struct ksmbd_share_config *share, const char *name); #define KSMBD_DIR_INFO_ALIGNMENT 8 diff --git a/fs/ksmbd/ndr.c b/fs/ksmbd/ndr.c index 5052be9261d91efb810757256efb913eb97ebc7b..0ae8d08d85a87da22995083e87ebada3118db7d3 100644 --- a/fs/ksmbd/ndr.c +++ b/fs/ksmbd/ndr.c @@ -345,6 +345,8 @@ int ndr_encode_posix_acl(struct ndr *n, { unsigned int ref_id = 0x00020000; int ret; + vfsuid_t vfsuid; + vfsgid_t vfsgid; n->offset = 0; n->length = 1024; @@ -372,10 +374,12 @@ int ndr_encode_posix_acl(struct ndr *n, if (ret) return ret; - ret = ndr_write_int64(n, from_kuid(&init_user_ns, i_uid_into_mnt(user_ns, inode))); + vfsuid = i_uid_into_vfsuid(user_ns, inode); + ret = ndr_write_int64(n, from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid))); if (ret) return ret; - ret = ndr_write_int64(n, from_kgid(&init_user_ns, i_gid_into_mnt(user_ns, inode))); + vfsgid = i_gid_into_vfsgid(user_ns, inode); + ret = ndr_write_int64(n, from_kgid(&init_user_ns, vfsgid_into_kgid(vfsgid))); if (ret) return ret; ret = ndr_write_int32(n, inode->i_mode); diff --git a/fs/ksmbd/oplock.c b/fs/ksmbd/oplock.c index 9046cff4374b2c7505a20c11d1626df36e34ae58..d7d47b82451db1610888f7138958d9db907e4333 100644 --- a/fs/ksmbd/oplock.c +++ b/fs/ksmbd/oplock.c @@ -1609,12 +1609,18 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp) struct create_posix_rsp *buf; struct inode *inode = file_inode(fp->filp); struct user_namespace *user_ns = file_mnt_user_ns(fp->filp); + vfsuid_t vfsuid = i_uid_into_vfsuid(user_ns, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(user_ns, inode); buf = (struct create_posix_rsp *)cc; memset(buf, 0, sizeof(struct create_posix_rsp)); buf->ccontext.DataOffset = cpu_to_le16(offsetof (struct create_posix_rsp, nlink)); - buf->ccontext.DataLength = cpu_to_le32(52); + /* + * DataLength = nlink(4) + reparse_tag(4) + mode(4) + + * domain sid(28) + unix group sid(16). + */ + buf->ccontext.DataLength = cpu_to_le32(56); buf->ccontext.NameOffset = cpu_to_le16(offsetof (struct create_posix_rsp, Name)); buf->ccontext.NameLength = cpu_to_le16(POSIX_CTXT_DATA_LEN); @@ -1638,13 +1644,18 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp) buf->nlink = cpu_to_le32(inode->i_nlink); buf->reparse_tag = cpu_to_le32(fp->volatile_id); - buf->mode = cpu_to_le32(inode->i_mode); - id_to_sid(from_kuid_munged(&init_user_ns, - i_uid_into_mnt(user_ns, inode)), - SIDNFS_USER, (struct smb_sid *)&buf->SidBuffer[0]); - id_to_sid(from_kgid_munged(&init_user_ns, - i_gid_into_mnt(user_ns, inode)), - SIDNFS_GROUP, (struct smb_sid *)&buf->SidBuffer[20]); + buf->mode = cpu_to_le32(inode->i_mode & 0777); + /* + * SidBuffer(44) contain two sids(Domain sid(28), UNIX group sid(16)). + * Domain sid(28) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 4(num_subauth)) + RID(4). + * UNIX group id(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ + id_to_sid(from_kuid_munged(&init_user_ns, vfsuid_into_kuid(vfsuid)), + SIDOWNER, (struct smb_sid *)&buf->SidBuffer[0]); + id_to_sid(from_kgid_munged(&init_user_ns, vfsgid_into_kgid(vfsgid)), + SIDUNIX_GROUP, (struct smb_sid *)&buf->SidBuffer[28]); } /* diff --git a/fs/ksmbd/server.c b/fs/ksmbd/server.c index ce42bff42ef9fad4ebb89a05b8fe33445d38c5fc..a0d635304754a4be9462e8ae1290e42c771697e0 100644 --- a/fs/ksmbd/server.c +++ b/fs/ksmbd/server.c @@ -235,10 +235,8 @@ send: if (work->sess && work->sess->enc && work->encrypted && conn->ops->encrypt_resp) { rc = conn->ops->encrypt_resp(work); - if (rc < 0) { + if (rc < 0) conn->ops->set_rsp_status(work, STATUS_DATA_ERROR); - goto send; - } } ksmbd_conn_write(work); diff --git a/fs/ksmbd/smb2pdu.c b/fs/ksmbd/smb2pdu.c index 19412ac701a65cbfcf25a9a0f1a1b8a234c54cb7..b2fc85d440d037550a43315b968afeef0cbe7b36 100644 --- a/fs/ksmbd/smb2pdu.c +++ b/fs/ksmbd/smb2pdu.c @@ -925,7 +925,7 @@ static void decode_encrypt_ctxt(struct ksmbd_conn *conn, * * Return: true if connection should be encrypted, else false */ -static bool smb3_encryption_negotiated(struct ksmbd_conn *conn) +bool smb3_encryption_negotiated(struct ksmbd_conn *conn) { if (!conn->ops->generate_encryptionkey) return false; @@ -1883,7 +1883,7 @@ int smb2_tree_connect(struct ksmbd_work *work) goto out_err1; } - name = ksmbd_extract_sharename(treename); + name = ksmbd_extract_sharename(conn->um, treename); if (IS_ERR(name)) { status.ret = KSMBD_TREE_CONN_STATUS_ERROR; goto out_err1; @@ -2185,7 +2185,7 @@ out: * Return: 0 on success, otherwise error */ static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len, - struct path *path) + const struct path *path) { struct user_namespace *user_ns = mnt_user_ns(path->mnt); char *attr_name = NULL, *value; @@ -2272,7 +2272,7 @@ next: return rc; } -static noinline int smb2_set_stream_name_xattr(struct path *path, +static noinline int smb2_set_stream_name_xattr(const struct path *path, struct ksmbd_file *fp, char *stream_name, int s_type) { @@ -2311,7 +2311,7 @@ static noinline int smb2_set_stream_name_xattr(struct path *path, return 0; } -static int smb2_remove_smb_xattrs(struct path *path) +static int smb2_remove_smb_xattrs(const struct path *path) { struct user_namespace *user_ns = mnt_user_ns(path->mnt); char *name, *xattr_list = NULL; @@ -2345,7 +2345,7 @@ out: return err; } -static int smb2_create_truncate(struct path *path) +static int smb2_create_truncate(const struct path *path) { int rc = vfs_truncate(path, 0); @@ -2364,7 +2364,7 @@ static int smb2_create_truncate(struct path *path) return rc; } -static void smb2_new_xattrs(struct ksmbd_tree_connect *tcon, struct path *path, +static void smb2_new_xattrs(struct ksmbd_tree_connect *tcon, const struct path *path, struct ksmbd_file *fp) { struct xattr_dos_attrib da = {0}; @@ -2387,7 +2387,7 @@ static void smb2_new_xattrs(struct ksmbd_tree_connect *tcon, struct path *path, } static void smb2_update_xattrs(struct ksmbd_tree_connect *tcon, - struct path *path, struct ksmbd_file *fp) + const struct path *path, struct ksmbd_file *fp) { struct xattr_dos_attrib da; int rc; @@ -2447,7 +2447,7 @@ static int smb2_creat(struct ksmbd_work *work, struct path *path, char *name, static int smb2_create_sd_buffer(struct ksmbd_work *work, struct smb2_create_req *req, - struct path *path) + const struct path *path) { struct create_context *context; struct create_sd_buf_req *sd_buf; @@ -2477,8 +2477,11 @@ static void ksmbd_acls_fattr(struct smb_fattr *fattr, struct user_namespace *mnt_userns, struct inode *inode) { - fattr->cf_uid = i_uid_into_mnt(mnt_userns, inode); - fattr->cf_gid = i_gid_into_mnt(mnt_userns, inode); + vfsuid_t vfsuid = i_uid_into_vfsuid(mnt_userns, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode); + + fattr->cf_uid = vfsuid_into_kuid(vfsuid); + fattr->cf_gid = vfsgid_into_kgid(vfsgid); fattr->cf_mode = inode->i_mode; fattr->cf_acls = NULL; fattr->cf_dacls = NULL; @@ -2761,7 +2764,6 @@ int smb2_open(struct ksmbd_work *work) } else { file_present = true; user_ns = mnt_user_ns(path.mnt); - generic_fillattr(user_ns, d_inode(path.dentry), &stat); } if (stream_name) { if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { @@ -2770,7 +2772,8 @@ int smb2_open(struct ksmbd_work *work) rsp->hdr.Status = STATUS_NOT_A_DIRECTORY; } } else { - if (S_ISDIR(stat.mode) && s_type == DATA_STREAM) { + if (file_present && S_ISDIR(d_inode(path.dentry)->i_mode) && + s_type == DATA_STREAM) { rc = -EIO; rsp->hdr.Status = STATUS_FILE_IS_A_DIRECTORY; } @@ -2787,7 +2790,8 @@ int smb2_open(struct ksmbd_work *work) } if (file_present && req->CreateOptions & FILE_NON_DIRECTORY_FILE_LE && - S_ISDIR(stat.mode) && !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { + S_ISDIR(d_inode(path.dentry)->i_mode) && + !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { ksmbd_debug(SMB, "open() argument is a directory: %s, %x\n", name, req->CreateOptions); rsp->hdr.Status = STATUS_FILE_IS_A_DIRECTORY; @@ -2797,7 +2801,7 @@ int smb2_open(struct ksmbd_work *work) if (file_present && (req->CreateOptions & FILE_DIRECTORY_FILE_LE) && !(req->CreateDisposition == FILE_CREATE_LE) && - !S_ISDIR(stat.mode)) { + !S_ISDIR(d_inode(path.dentry)->i_mode)) { rsp->hdr.Status = STATUS_NOT_A_DIRECTORY; rc = -EIO; goto err_out; @@ -3561,17 +3565,22 @@ static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level, posix_info->AllocationSize = cpu_to_le64(ksmbd_kstat->kstat->blocks << 9); posix_info->DeviceId = cpu_to_le32(ksmbd_kstat->kstat->rdev); posix_info->HardLinks = cpu_to_le32(ksmbd_kstat->kstat->nlink); - posix_info->Mode = cpu_to_le32(ksmbd_kstat->kstat->mode); + posix_info->Mode = cpu_to_le32(ksmbd_kstat->kstat->mode & 0777); posix_info->Inode = cpu_to_le64(ksmbd_kstat->kstat->ino); posix_info->DosAttributes = S_ISDIR(ksmbd_kstat->kstat->mode) ? FILE_ATTRIBUTE_DIRECTORY_LE : FILE_ATTRIBUTE_ARCHIVE_LE; if (d_info->hide_dot_file && d_info->name[0] == '.') posix_info->DosAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + /* + * SidBuffer(32) contain two sids(Domain sid(16), UNIX group sid(16)). + * UNIX sid(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ id_to_sid(from_kuid_munged(&init_user_ns, ksmbd_kstat->kstat->uid), - SIDNFS_USER, (struct smb_sid *)&posix_info->SidBuffer[0]); + SIDUNIX_USER, (struct smb_sid *)&posix_info->SidBuffer[0]); id_to_sid(from_kgid_munged(&init_user_ns, ksmbd_kstat->kstat->gid), - SIDNFS_GROUP, (struct smb_sid *)&posix_info->SidBuffer[20]); + SIDUNIX_GROUP, (struct smb_sid *)&posix_info->SidBuffer[16]); memcpy(posix_info->name, conv_name, conv_len); posix_info->name_len = cpu_to_le32(conv_len); posix_info->NextEntryOffset = cpu_to_le32(next_entry_offset); @@ -3776,7 +3785,7 @@ static int reserve_populate_dentry(struct ksmbd_dir_info *d_info, return 0; } -static int __query_dir(struct dir_context *ctx, const char *name, int namlen, +static bool __query_dir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct ksmbd_readdir_data *buf; @@ -3790,27 +3799,20 @@ static int __query_dir(struct dir_context *ctx, const char *name, int namlen, /* dot and dotdot entries are already reserved */ if (!strcmp(".", name) || !strcmp("..", name)) - return 0; + return true; if (ksmbd_share_veto_filename(priv->work->tcon->share_conf, name)) - return 0; + return true; if (!match_pattern(name, namlen, priv->search_pattern)) - return 0; + return true; d_info->name = name; d_info->name_len = namlen; rc = reserve_populate_dentry(d_info, priv->info_level); if (rc) - return rc; - if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY) { + return false; + if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY) d_info->out_buf_len = 0; - return 0; - } - return 0; -} - -static void restart_ctx(struct dir_context *ctx) -{ - ctx->pos = 0; + return true; } static int verify_info_level(int info_level) @@ -3894,8 +3896,7 @@ int smb2_query_dir(struct ksmbd_work *work) inode_permission(file_mnt_user_ns(dir_fp->filp), file_inode(dir_fp->filp), MAY_READ | MAY_EXEC)) { - pr_err("no right to enumerate directory (%pd)\n", - dir_fp->filp->f_path.dentry); + pr_err("no right to enumerate directory (%pD)\n", dir_fp->filp); rc = -EACCES; goto err_out2; } @@ -3921,7 +3922,6 @@ int smb2_query_dir(struct ksmbd_work *work) if (srch_flag & SMB2_REOPEN || srch_flag & SMB2_RESTART_SCANS) { ksmbd_debug(SMB, "Restart directory scan\n"); generic_file_llseek(dir_fp->filp, 0, SEEK_SET); - restart_ctx(&dir_fp->readdir_data.ctx); } memset(&d_info, 0, sizeof(struct ksmbd_dir_info)); @@ -3968,11 +3968,9 @@ int smb2_query_dir(struct ksmbd_work *work) */ if (!d_info.out_buf_len && !d_info.num_entry) goto no_buf_len; - if (rc == 0) - restart_ctx(&dir_fp->readdir_data.ctx); - if (rc == -ENOSPC) + if (rc > 0 || rc == -ENOSPC) rc = 0; - if (rc) + else if (rc) goto err_out; d_info.wptr = d_info.rptr; @@ -4029,6 +4027,8 @@ err_out2: rsp->hdr.Status = STATUS_NO_MEMORY; else if (rc == -EFAULT) rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; + else if (rc == -EIO) + rsp->hdr.Status = STATUS_FILE_CORRUPT_ERROR; if (!rsp->hdr.Status) rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; @@ -4158,7 +4158,7 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, int rc, name_len, value_len, xattr_list_len, idx; ssize_t buf_free_len, alignment_bytes, next_offset, rsp_data_cnt = 0; struct smb2_ea_info_req *ea_req = NULL; - struct path *path; + const struct path *path; struct user_namespace *user_ns = file_mnt_user_ns(fp->filp); if (!(fp->daccess & FILE_READ_EA_LE)) { @@ -4495,7 +4495,7 @@ static void get_file_stream_info(struct ksmbd_work *work, struct smb2_file_stream_info *file_info; char *stream_name, *xattr_list = NULL, *stream_buf; struct kstat stat; - struct path *path = &fp->filp->f_path; + const struct path *path = &fp->filp->f_path; ssize_t xattr_list_len; int nbytes = 0, streamlen, stream_name_len, next, idx = 0; int buf_free_len; @@ -4720,7 +4720,11 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, { struct smb311_posix_qinfo *file_info; struct inode *inode = file_inode(fp->filp); + struct user_namespace *user_ns = file_mnt_user_ns(fp->filp); + vfsuid_t vfsuid = i_uid_into_vfsuid(user_ns, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(user_ns, inode); u64 time; + int out_buf_len = sizeof(struct smb311_posix_qinfo) + 32; file_info = (struct smb311_posix_qinfo *)rsp->Buffer; file_info->CreationTime = cpu_to_le64(fp->create_time); @@ -4735,12 +4739,22 @@ static int find_file_posix_info(struct smb2_query_info_rsp *rsp, file_info->EndOfFile = cpu_to_le64(inode->i_size); file_info->AllocationSize = cpu_to_le64(inode->i_blocks << 9); file_info->HardLinks = cpu_to_le32(inode->i_nlink); - file_info->Mode = cpu_to_le32(inode->i_mode); + file_info->Mode = cpu_to_le32(inode->i_mode & 0777); file_info->DeviceId = cpu_to_le32(inode->i_rdev); - rsp->OutputBufferLength = - cpu_to_le32(sizeof(struct smb311_posix_qinfo)); - inc_rfc1001_len(rsp_org, sizeof(struct smb311_posix_qinfo)); - return 0; + + /* + * Sids(32) contain two sids(Domain sid(16), UNIX group sid(16)). + * UNIX sid(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ + id_to_sid(from_kuid_munged(&init_user_ns, vfsuid_into_kuid(vfsuid)), + SIDUNIX_USER, (struct smb_sid *)&file_info->Sids[0]); + id_to_sid(from_kgid_munged(&init_user_ns, vfsgid_into_kgid(vfsgid)), + SIDUNIX_GROUP, (struct smb_sid *)&file_info->Sids[16]); + + rsp->OutputBufferLength = cpu_to_le32(out_buf_len); + inc_rfc1001_len(rsp_org, out_buf_len); + return out_buf_len; } static int smb2_get_info_file(struct ksmbd_work *work, @@ -4860,8 +4874,8 @@ static int smb2_get_info_file(struct ksmbd_work *work, pr_err("client doesn't negotiate with SMB3.1.1 POSIX Extensions\n"); rc = -EOPNOTSUPP; } else { - rc = find_file_posix_info(rsp, fp, work->response_buf); - file_infoclass_size = sizeof(struct smb311_posix_qinfo); + file_infoclass_size = find_file_posix_info(rsp, fp, + work->response_buf); } break; default: @@ -5413,7 +5427,7 @@ static int smb2_rename(struct ksmbd_work *work, if (!pathname) return -ENOMEM; - abs_oldname = d_path(&fp->filp->f_path, pathname, PATH_MAX); + abs_oldname = file_path(fp->filp, pathname, PATH_MAX); if (IS_ERR(abs_oldname)) { rc = -EINVAL; goto out; @@ -5548,7 +5562,7 @@ static int smb2_create_link(struct ksmbd_work *work, } ksmbd_debug(SMB, "link name is %s\n", link_name); - target_name = d_path(&filp->f_path, pathname, PATH_MAX); + target_name = file_path(filp, pathname, PATH_MAX); if (IS_ERR(target_name)) { rc = -EINVAL; goto out; @@ -6266,8 +6280,8 @@ int smb2_read(struct ksmbd_work *work) goto out; } - ksmbd_debug(SMB, "filename %pd, offset %lld, len %zu\n", - fp->filp->f_path.dentry, offset, length); + ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", + fp->filp, offset, length); work->aux_payload_buf = kvmalloc(length, GFP_KERNEL | __GFP_ZERO); if (!work->aux_payload_buf) { @@ -6531,8 +6545,8 @@ int smb2_write(struct ksmbd_work *work) data_buf = (char *)(((char *)&req->hdr.ProtocolId) + le16_to_cpu(req->DataOffset)); - ksmbd_debug(SMB, "filename %pd, offset %lld, len %zu\n", - fp->filp->f_path.dentry, offset, length); + ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", + fp->filp, offset, length); err = ksmbd_vfs_write(work, fp, data_buf, length, &offset, writethrough, &nbytes); if (err < 0) @@ -7643,11 +7657,16 @@ int smb2_ioctl(struct ksmbd_work *work) goto out; } - if (in_buf_len < sizeof(struct validate_negotiate_info_req)) - return -EINVAL; + if (in_buf_len < offsetof(struct validate_negotiate_info_req, + Dialects)) { + ret = -EINVAL; + goto out; + } - if (out_buf_len < sizeof(struct validate_negotiate_info_rsp)) - return -EINVAL; + if (out_buf_len < sizeof(struct validate_negotiate_info_rsp)) { + ret = -EINVAL; + goto out; + } ret = fsctl_validate_negotiate_info(conn, (struct validate_negotiate_info_req *)&req->Buffer[0], @@ -8573,7 +8592,7 @@ int smb3_encrypt_resp(struct ksmbd_work *work) buf_size += iov[1].iov_len; work->resp_hdr_sz = iov[1].iov_len; - rc = ksmbd_crypt_message(work->conn, iov, rq_nvec, 1); + rc = ksmbd_crypt_message(work, iov, rq_nvec, 1); if (rc) return rc; @@ -8592,7 +8611,6 @@ bool smb3_is_transform_hdr(void *buf) int smb3_decrypt_req(struct ksmbd_work *work) { - struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess; char *buf = work->request_buf; unsigned int pdu_length = get_rfc1002_len(buf); @@ -8612,7 +8630,7 @@ int smb3_decrypt_req(struct ksmbd_work *work) return -ECONNABORTED; } - sess = ksmbd_session_lookup_all(conn, le64_to_cpu(tr_hdr->SessionId)); + sess = ksmbd_session_lookup_all(work->conn, le64_to_cpu(tr_hdr->SessionId)); if (!sess) { pr_err("invalid session id(%llx) in transform header\n", le64_to_cpu(tr_hdr->SessionId)); @@ -8623,7 +8641,7 @@ int smb3_decrypt_req(struct ksmbd_work *work) iov[0].iov_len = sizeof(struct smb2_transform_hdr) + 4; iov[1].iov_base = buf + sizeof(struct smb2_transform_hdr) + 4; iov[1].iov_len = buf_data_size; - rc = ksmbd_crypt_message(conn, iov, 2, 0); + rc = ksmbd_crypt_message(work, iov, 2, 0); if (rc) return rc; diff --git a/fs/ksmbd/smb2pdu.h b/fs/ksmbd/smb2pdu.h index af455278d00591fcd9673b245924b536f266b794..092fdd3f8750514e0144116a284148129c8876b7 100644 --- a/fs/ksmbd/smb2pdu.h +++ b/fs/ksmbd/smb2pdu.h @@ -158,7 +158,8 @@ struct create_posix_rsp { __le32 nlink; __le32 reparse_tag; __le32 mode; - u8 SidBuffer[40]; + /* SidBuffer contain two sids(Domain sid(28), UNIX group sid(16)) */ + u8 SidBuffer[44]; } __packed; struct smb2_buffer_desc_v1 { @@ -439,7 +440,8 @@ struct smb2_posix_info { __le32 HardLinks; __le32 ReparseTag; __le32 Mode; - u8 SidBuffer[40]; + /* SidBuffer contain two sids (UNIX user sid(16), UNIX group sid(16)) */ + u8 SidBuffer[32]; __le32 name_len; u8 name[1]; /* @@ -492,6 +494,7 @@ int smb3_decrypt_req(struct ksmbd_work *work); int smb3_encrypt_resp(struct ksmbd_work *work); bool smb3_11_final_sess_setup_resp(struct ksmbd_work *work); int smb2_set_rsp_credits(struct ksmbd_work *work); +bool smb3_encryption_negotiated(struct ksmbd_conn *conn); /* smb2 misc functions */ int ksmbd_smb2_check_message(struct ksmbd_work *work); diff --git a/fs/ksmbd/smb_common.c b/fs/ksmbd/smb_common.c index 7f8ab14fb8ec147d2212948d2fd1770d31b2e67a..d96da872d70a1701b2e99b962f82ac6b83fd6351 100644 --- a/fs/ksmbd/smb_common.c +++ b/fs/ksmbd/smb_common.c @@ -4,6 +4,8 @@ * Copyright (C) 2018 Namjae Jeon */ +#include + #include "smb_common.h" #include "server.h" #include "misc.h" @@ -625,8 +627,8 @@ int ksmbd_override_fsids(struct ksmbd_work *work) if (!cred) return -ENOMEM; - cred->fsuid = make_kuid(current_user_ns(), uid); - cred->fsgid = make_kgid(current_user_ns(), gid); + cred->fsuid = make_kuid(&init_user_ns, uid); + cred->fsgid = make_kgid(&init_user_ns, gid); gi = groups_alloc(0); if (!gi) { diff --git a/fs/ksmbd/smbacl.c b/fs/ksmbd/smbacl.c index 3781bca2c8fc470507d5239f6b7912604e4fa79a..b05ff9b146b550881b3bcd427351edcaca34f996 100644 --- a/fs/ksmbd/smbacl.c +++ b/fs/ksmbd/smbacl.c @@ -275,7 +275,8 @@ static int sid_to_id(struct user_namespace *user_ns, uid_t id; id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); - uid = mapped_kuid_user(user_ns, &init_user_ns, KUIDT_INIT(id)); + uid = KUIDT_INIT(id); + uid = from_vfsuid(user_ns, &init_user_ns, VFSUIDT_INIT(uid)); if (uid_valid(uid)) { fattr->cf_uid = uid; rc = 0; @@ -285,7 +286,8 @@ static int sid_to_id(struct user_namespace *user_ns, gid_t id; id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); - gid = mapped_kgid_user(user_ns, &init_user_ns, KGIDT_INIT(id)); + gid = KGIDT_INIT(id); + gid = from_vfsgid(user_ns, &init_user_ns, VFSGIDT_INIT(gid)); if (gid_valid(gid)) { fattr->cf_gid = gid; rc = 0; @@ -991,7 +993,7 @@ static void smb_set_ace(struct smb_ace *ace, const struct smb_sid *sid, u8 type, } int smb_inherit_dacl(struct ksmbd_conn *conn, - struct path *path, + const struct path *path, unsigned int uid, unsigned int gid) { const struct smb_sid *psid, *creator = NULL; @@ -1185,7 +1187,7 @@ bool smb_inherit_flags(int flags, bool is_dir) return false; } -int smb_check_perm_dacl(struct ksmbd_conn *conn, struct path *path, +int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, __le32 *pdaccess, int uid) { struct user_namespace *user_ns = mnt_user_ns(path->mnt); @@ -1352,7 +1354,7 @@ err_out: } int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, - struct path *path, struct smb_ntsd *pntsd, int ntsd_len, + const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, bool type_check) { int rc; diff --git a/fs/ksmbd/smbacl.h b/fs/ksmbd/smbacl.h index fcb2c83f29928e9c55bf5f74d6de68dce1925ad9..618f2e0236b31315bcf07c27166dfdc2bf449401 100644 --- a/fs/ksmbd/smbacl.h +++ b/fs/ksmbd/smbacl.h @@ -201,12 +201,12 @@ void posix_state_to_acl(struct posix_acl_state *state, struct posix_acl_entry *pace); int compare_sids(const struct smb_sid *ctsid, const struct smb_sid *cwsid); bool smb_inherit_flags(int flags, bool is_dir); -int smb_inherit_dacl(struct ksmbd_conn *conn, struct path *path, +int smb_inherit_dacl(struct ksmbd_conn *conn, const struct path *path, unsigned int uid, unsigned int gid); -int smb_check_perm_dacl(struct ksmbd_conn *conn, struct path *path, +int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, __le32 *pdaccess, int uid); int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, - struct path *path, struct smb_ntsd *pntsd, int ntsd_len, + const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, bool type_check); void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid); void ksmbd_init_domain(u32 *sub_auth); @@ -214,25 +214,25 @@ void ksmbd_init_domain(u32 *sub_auth); static inline uid_t posix_acl_uid_translate(struct user_namespace *mnt_userns, struct posix_acl_entry *pace) { - kuid_t kuid; + vfsuid_t vfsuid; /* If this is an idmapped mount, apply the idmapping. */ - kuid = mapped_kuid_fs(mnt_userns, &init_user_ns, pace->e_uid); + vfsuid = make_vfsuid(mnt_userns, &init_user_ns, pace->e_uid); /* Translate the kuid into a userspace id ksmbd would see. */ - return from_kuid(&init_user_ns, kuid); + return from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid)); } static inline gid_t posix_acl_gid_translate(struct user_namespace *mnt_userns, struct posix_acl_entry *pace) { - kgid_t kgid; + vfsgid_t vfsgid; /* If this is an idmapped mount, apply the idmapping. */ - kgid = mapped_kgid_fs(mnt_userns, &init_user_ns, pace->e_gid); + vfsgid = make_vfsgid(mnt_userns, &init_user_ns, pace->e_gid); /* Translate the kgid into a userspace id ksmbd would see. */ - return from_kgid(&init_user_ns, kgid); + return from_kgid(&init_user_ns, vfsgid_into_kgid(vfsgid)); } #endif /* _SMBACL_H */ diff --git a/fs/ksmbd/transport_ipc.c b/fs/ksmbd/transport_ipc.c index 7cb0eeb07c80876c3504edb88a8550ffb434a533..c9aca21637d5b688470338046cebd542ea5c91a2 100644 --- a/fs/ksmbd/transport_ipc.c +++ b/fs/ksmbd/transport_ipc.c @@ -197,6 +197,7 @@ static struct genl_family ksmbd_genl_family = { .module = THIS_MODULE, .ops = ksmbd_genl_ops, .n_ops = ARRAY_SIZE(ksmbd_genl_ops), + .resv_start_op = KSMBD_EVENT_SPNEGO_AUTHEN_RESPONSE + 1, }; static void ksmbd_nl_init_fixup(void) diff --git a/fs/ksmbd/transport_rdma.c b/fs/ksmbd/transport_rdma.c index 35b55ee94fe5496c18a8bb55503c2a2a2acd9cc1..096eda9ef873b670b46e735178bccb15742c4268 100644 --- a/fs/ksmbd/transport_rdma.c +++ b/fs/ksmbd/transport_rdma.c @@ -32,7 +32,7 @@ /* SMB_DIRECT negotiation timeout in seconds */ #define SMB_DIRECT_NEGOTIATE_TIMEOUT 120 -#define SMB_DIRECT_MAX_SEND_SGES 8 +#define SMB_DIRECT_MAX_SEND_SGES 6 #define SMB_DIRECT_MAX_RECV_SGES 1 /* @@ -62,13 +62,13 @@ static int smb_direct_receive_credit_max = 255; static int smb_direct_send_credit_target = 255; /* The maximum single message size can be sent to remote peer */ -static int smb_direct_max_send_size = 8192; +static int smb_direct_max_send_size = 1364; /* The maximum fragmented upper-layer payload receive size supported */ static int smb_direct_max_fragmented_recv_size = 1024 * 1024; /* The maximum single-message size which can be received */ -static int smb_direct_max_receive_size = 8192; +static int smb_direct_max_receive_size = 1364; static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE; @@ -1527,6 +1527,8 @@ static int smb_direct_cm_handler(struct rdma_cm_id *cm_id, } case RDMA_CM_EVENT_DEVICE_REMOVAL: case RDMA_CM_EVENT_DISCONNECTED: { + ib_drain_qp(t->qp); + t->status = SMB_DIRECT_CS_DISCONNECTED; wake_up_interruptible(&t->wait_status); wake_up_interruptible(&t->wait_reassembly_queue); diff --git a/fs/ksmbd/transport_tcp.c b/fs/ksmbd/transport_tcp.c index 143bba4e4db814a6f44cb55db0fe40fd9042a8a3..63d55f543bd2e4f9ee09a35e59a585685431b1ea 100644 --- a/fs/ksmbd/transport_tcp.c +++ b/fs/ksmbd/transport_tcp.c @@ -399,7 +399,8 @@ static int create_socket(struct interface *iface) ret = sock_create(PF_INET6, SOCK_STREAM, IPPROTO_TCP, &ksmbd_socket); if (ret) { - pr_err("Can't create socket for ipv6, try ipv4: %d\n", ret); + if (ret != -EAFNOSUPPORT) + pr_err("Can't create socket for ipv6, fallback to ipv4: %d\n", ret); ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, &ksmbd_socket); if (ret) { diff --git a/fs/ksmbd/unicode.h b/fs/ksmbd/unicode.h index 5593024230ae29577576e213214b8906c3eac595..076f6034a7899b170a5a8231d402d0b9a7e334a2 100644 --- a/fs/ksmbd/unicode.h +++ b/fs/ksmbd/unicode.h @@ -24,6 +24,7 @@ #include #include #include +#include #define UNIUPR_NOLOWER /* Example to not expand lower case tables */ @@ -69,7 +70,7 @@ char *smb_strndup_from_utf16(const char *src, const int maxlen, const struct nls_table *codepage); int smbConvertToUTF16(__le16 *target, const char *source, int srclen, const struct nls_table *cp, int mapchars); -char *ksmbd_extract_sharename(char *treename); +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); #endif /* diff --git a/fs/ksmbd/vfs.c b/fs/ksmbd/vfs.c index 78d01033604c6b469f88f1fab62921732c81e6e2..8de970d6146f2b09e7977450b4c56abe3585f6db 100644 --- a/fs/ksmbd/vfs.c +++ b/fs/ksmbd/vfs.c @@ -377,8 +377,7 @@ int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count, if (work->conn->connection_type) { if (!(fp->daccess & (FILE_READ_DATA_LE | FILE_EXECUTE_LE))) { - pr_err("no right to read(%pd)\n", - fp->filp->f_path.dentry); + pr_err("no right to read(%pD)\n", fp->filp); return -EACCES; } } @@ -487,8 +486,7 @@ int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp, if (work->conn->connection_type) { if (!(fp->daccess & FILE_WRITE_DATA_LE)) { - pr_err("no right to write(%pd)\n", - fp->filp->f_path.dentry); + pr_err("no right to write(%pD)\n", fp->filp); err = -EACCES; goto out; } @@ -527,8 +525,8 @@ int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp, if (sync) { err = vfs_fsync_range(filp, offset, offset + *written, 0); if (err < 0) - pr_err("fsync failed for filename = %pd, err = %d\n", - fp->filp->f_path.dentry, err); + pr_err("fsync failed for filename = %pD, err = %d\n", + fp->filp, err); } out: @@ -543,7 +541,7 @@ out: * * Return: 0 on success, otherwise error */ -int ksmbd_vfs_getattr(struct path *path, struct kstat *stat) +int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat) { int err; @@ -1105,7 +1103,7 @@ int ksmbd_vfs_unlink(struct user_namespace *user_ns, return err; } -static int __dir_empty(struct dir_context *ctx, const char *name, int namlen, +static bool __dir_empty(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct ksmbd_readdir_data *buf; @@ -1113,9 +1111,7 @@ static int __dir_empty(struct dir_context *ctx, const char *name, int namlen, buf = container_of(ctx, struct ksmbd_readdir_data, ctx); buf->dirent_count++; - if (buf->dirent_count > 2) - return -ENOTEMPTY; - return 0; + return buf->dirent_count <= 2; } /** @@ -1142,22 +1138,33 @@ int ksmbd_vfs_empty_dir(struct ksmbd_file *fp) return err; } -static int __caseless_lookup(struct dir_context *ctx, const char *name, +static bool __caseless_lookup(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct ksmbd_readdir_data *buf; + int cmp = -EINVAL; buf = container_of(ctx, struct ksmbd_readdir_data, ctx); if (buf->used != namlen) - return 0; - if (!strncasecmp((char *)buf->private, name, namlen)) { + return true; + if (IS_ENABLED(CONFIG_UNICODE) && buf->um) { + const struct qstr q_buf = {.name = buf->private, + .len = buf->used}; + const struct qstr q_name = {.name = name, + .len = namlen}; + + cmp = utf8_strncasecmp(buf->um, &q_buf, &q_name); + } + if (cmp < 0) + cmp = strncasecmp((char *)buf->private, name, namlen); + if (!cmp) { memcpy((char *)buf->private, name, namlen); buf->dirent_count = 1; - return -EEXIST; + return false; } - return 0; + return true; } /** @@ -1168,7 +1175,8 @@ static int __caseless_lookup(struct dir_context *ctx, const char *name, * * Return: 0 on success, otherwise error */ -static int ksmbd_vfs_lookup_in_dir(struct path *dir, char *name, size_t namelen) +static int ksmbd_vfs_lookup_in_dir(const struct path *dir, char *name, + size_t namelen, struct unicode_map *um) { int ret; struct file *dfilp; @@ -1178,6 +1186,7 @@ static int ksmbd_vfs_lookup_in_dir(struct path *dir, char *name, size_t namelen) .private = name, .used = namelen, .dirent_count = 0, + .um = um, }; dfilp = dentry_open(dir, flags, current_cred()); @@ -1240,7 +1249,8 @@ int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, break; err = ksmbd_vfs_lookup_in_dir(&parent, filename, - filename_len); + filename_len, + work->conn->um); path_put(&parent); if (err) goto out; @@ -1743,11 +1753,11 @@ int ksmbd_vfs_copy_file_ranges(struct ksmbd_work *work, *total_size_written = 0; if (!(src_fp->daccess & (FILE_READ_DATA_LE | FILE_EXECUTE_LE))) { - pr_err("no right to read(%pd)\n", src_fp->filp->f_path.dentry); + pr_err("no right to read(%pD)\n", src_fp->filp); return -EACCES; } if (!(dst_fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE))) { - pr_err("no right to write(%pd)\n", dst_fp->filp->f_path.dentry); + pr_err("no right to write(%pD)\n", dst_fp->filp); return -EACCES; } diff --git a/fs/ksmbd/vfs.h b/fs/ksmbd/vfs.h index 70da4c0ba7adae399ec02ac3dff1fadda2bd143e..593059ca8511228e18a1c5c75a777b9fa7b40732 100644 --- a/fs/ksmbd/vfs.h +++ b/fs/ksmbd/vfs.h @@ -12,6 +12,7 @@ #include #include #include +#include #include "smbacl.h" #include "xattr.h" @@ -60,6 +61,7 @@ struct ksmbd_readdir_data { unsigned int used; unsigned int dirent_count; unsigned int file_attr; + struct unicode_map *um; }; /* ksmbd kstat wrapper to get valid create time when reading dir entry */ @@ -85,7 +87,7 @@ int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id); int ksmbd_vfs_remove_file(struct ksmbd_work *work, char *name); int ksmbd_vfs_link(struct ksmbd_work *work, const char *oldname, const char *newname); -int ksmbd_vfs_getattr(struct path *path, struct kstat *stat); +int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat); int ksmbd_vfs_fp_rename(struct ksmbd_work *work, struct ksmbd_file *fp, char *newname); int ksmbd_vfs_truncate(struct ksmbd_work *work, diff --git a/fs/libfs.c b/fs/libfs.c index 31b0ddf01c31daca6a946c48c78b5d821034e0c0..682d56345a1cfeb3977e9d66b00017540f7fb492 100644 --- a/fs/libfs.c +++ b/fs/libfs.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include /* sync_mapping_buffers */ #include @@ -1520,3 +1521,48 @@ void generic_set_encrypted_ci_d_ops(struct dentry *dentry) #endif } EXPORT_SYMBOL(generic_set_encrypted_ci_d_ops); + +/** + * inode_maybe_inc_iversion - increments i_version + * @inode: inode with the i_version that should be updated + * @force: increment the counter even if it's not necessary? + * + * Every time the inode is modified, the i_version field must be seen to have + * changed by any observer. + * + * If "force" is set or the QUERIED flag is set, then ensure that we increment + * the value, and clear the queried flag. + * + * In the common case where neither is set, then we can return "false" without + * updating i_version. + * + * If this function returns false, and no other metadata has changed, then we + * can avoid logging the metadata. + */ +bool inode_maybe_inc_iversion(struct inode *inode, bool force) +{ + u64 cur, new; + + /* + * The i_version field is not strictly ordered with any other inode + * information, but the legacy inode_inc_iversion code used a spinlock + * to serialize increments. + * + * Here, we add full memory barriers to ensure that any de-facto + * ordering with other info is preserved. + * + * This barrier pairs with the barrier in inode_query_iversion() + */ + smp_mb(); + cur = inode_peek_iversion_raw(inode); + do { + /* If flag is clear then we needn't do anything */ + if (!force && !(cur & I_VERSION_QUERIED)) + return false; + + /* Since lowest bit is flag, add 2 to avoid it */ + new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT; + } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new)); + return true; +} +EXPORT_SYMBOL(inode_maybe_inc_iversion); diff --git a/fs/lockd/host.c b/fs/lockd/host.c index f802223e71abe530b4234ac2fa403e5a038e8978..cdc8e12cdac44e87642e24866a6ae71ceaf3ef7a 100644 --- a/fs/lockd/host.c +++ b/fs/lockd/host.c @@ -164,7 +164,7 @@ static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni, host->h_addrbuf = nsm->sm_addrbuf; host->net = ni->net; host->h_cred = get_cred(ni->cred); - strlcpy(host->nodename, utsname()->nodename, sizeof(host->nodename)); + strscpy(host->nodename, utsname()->nodename, sizeof(host->nodename)); out: return host; diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c index bf274f23969b302ade131711f354fb69a596adc2..284b019cb6529b999c9a2dbdd21f06f7c0325c38 100644 --- a/fs/lockd/svc4proc.c +++ b/fs/lockd/svc4proc.c @@ -521,6 +521,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "NULL", @@ -530,6 +531,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_testargs, .pc_encode = nlm4svc_encode_testres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, .pc_name = "TEST", @@ -539,6 +541,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_lockargs, .pc_encode = nlm4svc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "LOCK", @@ -548,6 +551,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_cancargs, .pc_encode = nlm4svc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "CANCEL", @@ -557,6 +561,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_unlockargs, .pc_encode = nlm4svc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "UNLOCK", @@ -566,6 +571,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_testargs, .pc_encode = nlm4svc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "GRANTED", @@ -575,6 +581,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_testargs, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "TEST_MSG", @@ -584,6 +591,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_lockargs, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "LOCK_MSG", @@ -593,6 +601,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_cancargs, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "CANCEL_MSG", @@ -602,6 +611,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_unlockargs, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNLOCK_MSG", @@ -611,6 +621,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_testargs, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "GRANTED_MSG", @@ -620,6 +631,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "TEST_RES", @@ -629,6 +641,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "LOCK_RES", @@ -638,6 +651,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "CANCEL_RES", @@ -647,6 +661,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNLOCK_RES", @@ -656,6 +671,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_res, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "GRANTED_RES", @@ -665,6 +681,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_reboot, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_reboot), + .pc_argzero = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "SM_NOTIFY", @@ -674,6 +691,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, .pc_name = "UNUSED", @@ -683,6 +701,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, .pc_name = "UNUSED", @@ -692,6 +711,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_void, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, .pc_name = "UNUSED", @@ -701,6 +721,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_shareargs, .pc_encode = nlm4svc_encode_shareres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, .pc_name = "SHARE", @@ -710,6 +731,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_shareargs, .pc_encode = nlm4svc_encode_shareres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, .pc_name = "UNSHARE", @@ -719,6 +741,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_lockargs, .pc_encode = nlm4svc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "NM_LOCK", @@ -728,6 +751,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = { .pc_decode = nlm4svc_decode_notify, .pc_encode = nlm4svc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "FREE_ALL", diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c index b09ca35b527cc006c57f4cd8d365d38f5f96e61a..e35c05e2780618bcf62504b79f41f8f12a4f292b 100644 --- a/fs/lockd/svcproc.c +++ b/fs/lockd/svcproc.c @@ -555,6 +555,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "NULL", @@ -564,6 +565,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_testargs, .pc_encode = nlmsvc_encode_testres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+2+No+Rg, .pc_name = "TEST", @@ -573,6 +575,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_lockargs, .pc_encode = nlmsvc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "LOCK", @@ -582,6 +585,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_cancargs, .pc_encode = nlmsvc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "CANCEL", @@ -591,6 +595,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_unlockargs, .pc_encode = nlmsvc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "UNLOCK", @@ -600,6 +605,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_testargs, .pc_encode = nlmsvc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "GRANTED", @@ -609,6 +615,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_testargs, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "TEST_MSG", @@ -618,6 +625,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_lockargs, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "LOCK_MSG", @@ -627,6 +635,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_cancargs, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "CANCEL_MSG", @@ -636,6 +645,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_unlockargs, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNLOCK_MSG", @@ -645,6 +655,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_testargs, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "GRANTED_MSG", @@ -654,6 +665,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "TEST_RES", @@ -663,6 +675,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "LOCK_RES", @@ -672,6 +685,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "CANCEL_RES", @@ -681,6 +695,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNLOCK_RES", @@ -690,6 +705,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_res, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = sizeof(struct nlm_res), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "GRANTED_RES", @@ -699,6 +715,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_reboot, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_reboot), + .pc_argzero = sizeof(struct nlm_reboot), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "SM_NOTIFY", @@ -708,6 +725,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNUSED", @@ -717,6 +735,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNUSED", @@ -726,6 +745,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_void, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_void), + .pc_argzero = sizeof(struct nlm_void), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = St, .pc_name = "UNUSED", @@ -735,6 +755,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_shareargs, .pc_encode = nlmsvc_encode_shareres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, .pc_name = "SHARE", @@ -744,6 +765,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_shareargs, .pc_encode = nlmsvc_encode_shareres, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St+1, .pc_name = "UNSHARE", @@ -753,6 +775,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_lockargs, .pc_encode = nlmsvc_encode_res, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_res), .pc_xdrressize = Ck+St, .pc_name = "NM_LOCK", @@ -762,6 +785,7 @@ const struct svc_procedure nlmsvc_procedures[24] = { .pc_decode = nlmsvc_decode_notify, .pc_encode = nlmsvc_encode_void, .pc_argsize = sizeof(struct nlm_args), + .pc_argzero = sizeof(struct nlm_args), .pc_ressize = sizeof(struct nlm_void), .pc_xdrressize = 0, .pc_name = "FREE_ALL", diff --git a/fs/locks.c b/fs/locks.c index c266cfdc3291f92902481f8d9ed289ae08fe839c..607f94a0e789fc71407780e12580ca89891a1dad 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -2129,6 +2129,7 @@ SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd) else error = locks_lock_file_wait(f.file, &fl); + locks_release_private(&fl); out_putf: fdput(f); diff --git a/fs/mbcache.c b/fs/mbcache.c index 47ccfcbe0a22ec2e65ec47b5e2da35aeed2d6adb..e272ad738faff04b7d24cf86dd115f60182c289c 100644 --- a/fs/mbcache.c +++ b/fs/mbcache.c @@ -90,8 +90,14 @@ int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key, return -ENOMEM; INIT_LIST_HEAD(&entry->e_list); - /* Initial hash reference */ - atomic_set(&entry->e_refcnt, 1); + /* + * We create entry with two references. One reference is kept by the + * hash table, the other reference is used to protect us from + * mb_cache_entry_delete_or_get() until the entry is fully setup. This + * avoids nesting of cache->c_list_lock into hash table bit locks which + * is problematic for RT. + */ + atomic_set(&entry->e_refcnt, 2); entry->e_key = key; entry->e_value = value; entry->e_reusable = reusable; @@ -106,15 +112,12 @@ int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key, } } hlist_bl_add_head(&entry->e_hash_list, head); - /* - * Add entry to LRU list before it can be found by - * mb_cache_entry_delete() to avoid races - */ + hlist_bl_unlock(head); spin_lock(&cache->c_list_lock); list_add_tail(&entry->e_list, &cache->c_list); cache->c_entry_count++; spin_unlock(&cache->c_list_lock); - hlist_bl_unlock(head); + mb_cache_entry_put(cache, entry); return 0; } diff --git a/fs/minix/namei.c b/fs/minix/namei.c index 937fa5fae2b8e2ffdddcc120a9cebfff46bf81d8..8afdc408ca4fd5f893959a007cebae6f443691cd 100644 --- a/fs/minix/namei.c +++ b/fs/minix/namei.c @@ -53,16 +53,16 @@ static int minix_mknod(struct user_namespace *mnt_userns, struct inode *dir, } static int minix_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { int error; struct inode *inode = minix_new_inode(dir, mode, &error); if (inode) { minix_set_inode(inode, 0); mark_inode_dirty(inode); - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); } - return error; + return finish_open_simple(file, error); } static int minix_create(struct user_namespace *mnt_userns, struct inode *dir, diff --git a/fs/namei.c b/fs/namei.c index 53b4bc094db23a7be059f4b27aa1e5750db733ed..578c2110df0223fbdb952e3e002de5feaa2d1446 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -986,7 +986,7 @@ static int nd_jump_root(struct nameidata *nd) * Helper to directly jump to a known parsed path from ->get_link, * caller must have taken a reference to path beforehand. */ -int nd_jump_link(struct path *path) +int nd_jump_link(const struct path *path) { int error = -ELOOP; struct nameidata *nd = current->nameidata; @@ -1178,7 +1178,7 @@ static bool safe_hardlink_source(struct user_namespace *mnt_userns, * * Returns 0 if successful, -ve on error. */ -int may_linkat(struct user_namespace *mnt_userns, struct path *link) +int may_linkat(struct user_namespace *mnt_userns, const struct path *link) { struct inode *inode = link->dentry->d_inode; @@ -3583,72 +3583,94 @@ static int do_open(struct nameidata *nd, * On non-idmapped mounts or if permission checking is to be performed on the * raw inode simply passs init_user_ns. */ -struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, - struct dentry *dentry, umode_t mode, int open_flag) +static int vfs_tmpfile(struct user_namespace *mnt_userns, + const struct path *parentpath, + struct file *file, umode_t mode) { - struct dentry *child = NULL; - struct inode *dir = dentry->d_inode; + struct dentry *child; + struct inode *dir = d_inode(parentpath->dentry); struct inode *inode; int error; /* we want directory to be writable */ error = inode_permission(mnt_userns, dir, MAY_WRITE | MAY_EXEC); if (error) - goto out_err; - error = -EOPNOTSUPP; + return error; if (!dir->i_op->tmpfile) - goto out_err; - error = -ENOMEM; - child = d_alloc(dentry, &slash_name); + return -EOPNOTSUPP; + child = d_alloc(parentpath->dentry, &slash_name); if (unlikely(!child)) - goto out_err; + return -ENOMEM; + file->f_path.mnt = parentpath->mnt; + file->f_path.dentry = child; mode = vfs_prepare_mode(mnt_userns, dir, mode, mode, mode); - error = dir->i_op->tmpfile(mnt_userns, dir, child, mode); + error = dir->i_op->tmpfile(mnt_userns, dir, file, mode); + dput(child); if (error) - goto out_err; - error = -ENOENT; - inode = child->d_inode; - if (unlikely(!inode)) - goto out_err; - if (!(open_flag & O_EXCL)) { + return error; + /* Don't check for other permissions, the inode was just created */ + error = may_open(mnt_userns, &file->f_path, 0, file->f_flags); + if (error) + return error; + inode = file_inode(file); + if (!(file->f_flags & O_EXCL)) { spin_lock(&inode->i_lock); inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } ima_post_create_tmpfile(mnt_userns, inode); - return child; + return 0; +} -out_err: - dput(child); - return ERR_PTR(error); +/** + * vfs_tmpfile_open - open a tmpfile for kernel internal use + * @mnt_userns: user namespace of the mount the inode was found from + * @parentpath: path of the base directory + * @mode: mode of the new tmpfile + * @open_flag: flags + * @cred: credentials for open + * + * Create and open a temporary file. The file is not accounted in nr_files, + * hence this is only for kernel internal use, and must not be installed into + * file tables or such. + */ +struct file *vfs_tmpfile_open(struct user_namespace *mnt_userns, + const struct path *parentpath, + umode_t mode, int open_flag, const struct cred *cred) +{ + struct file *file; + int error; + + file = alloc_empty_file_noaccount(open_flag, cred); + if (!IS_ERR(file)) { + error = vfs_tmpfile(mnt_userns, parentpath, file, mode); + if (error) { + fput(file); + file = ERR_PTR(error); + } + } + return file; } -EXPORT_SYMBOL(vfs_tmpfile); +EXPORT_SYMBOL(vfs_tmpfile_open); static int do_tmpfile(struct nameidata *nd, unsigned flags, const struct open_flags *op, struct file *file) { struct user_namespace *mnt_userns; - struct dentry *child; struct path path; int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path); + if (unlikely(error)) return error; error = mnt_want_write(path.mnt); if (unlikely(error)) goto out; mnt_userns = mnt_user_ns(path.mnt); - child = vfs_tmpfile(mnt_userns, path.dentry, op->mode, op->open_flag); - error = PTR_ERR(child); - if (IS_ERR(child)) + error = vfs_tmpfile(mnt_userns, &path, file, op->mode); + if (error) goto out2; - dput(path.dentry); - path.dentry = child; - audit_inode(nd->name, child, 0); - /* Don't check for other permissions, the inode was just created */ - error = may_open(mnt_userns, &path, 0, op->open_flag); - if (!error) - error = vfs_open(&path, file); + audit_inode(nd->name, file->f_path.dentry, 0); out2: mnt_drop_write(path.mnt); out: @@ -5088,7 +5110,7 @@ int page_symlink(struct inode *inode, const char *symname, int len) const struct address_space_operations *aops = mapping->a_ops; bool nofs = !mapping_gfp_constraint(mapping, __GFP_FS); struct page *page; - void *fsdata; + void *fsdata = NULL; int err; unsigned int flags; diff --git a/fs/namespace.c b/fs/namespace.c index 68789f896f0819e0abeb509e90aea20dbc0fc7c7..df137ba19d3756bd08d544f0d7e95eb52cb4869c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -4238,6 +4238,13 @@ static int build_mount_idmapped(const struct mount_attr *attr, size_t usize, err = -EPERM; goto out_fput; } + + /* We're not controlling the target namespace. */ + if (!ns_capable(mnt_userns, CAP_SYS_ADMIN)) { + err = -EPERM; + goto out_fput; + } + kattr->mnt_userns = get_user_ns(mnt_userns); out_fput: diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c index 8dcb08e1a885d623f82427cc3bc261949c043957..d0cccddb7d0885fe98f477324fdd50012faf3cb7 100644 --- a/fs/nfs/callback_xdr.c +++ b/fs/nfs/callback_xdr.c @@ -1065,6 +1065,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = { .pc_func = nfs4_callback_compound, .pc_encode = nfs4_encode_void, .pc_argsize = 256, + .pc_argzero = 256, .pc_ressize = 256, .pc_xdrressize = NFS4_CALLBACK_BUFSIZE, .pc_name = "COMPOUND", diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index dbab3caa15ed505fa88c10b22af5eefbd7884771..58036f657126878e54eca3448f10032b26b7320c 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2022,7 +2022,7 @@ static int nfs_finish_open(struct nfs_open_context *ctx, err = finish_open(file, dentry, do_open); if (err) goto out; - if (S_ISREG(file->f_path.dentry->d_inode->i_mode)) + if (S_ISREG(file_inode(file)->i_mode)) nfs_file_set_open_context(file, ctx); else err = -EOPENSTALE; @@ -2382,7 +2382,8 @@ static void nfs_dentry_remove_handle_error(struct inode *dir, { switch (error) { case -ENOENT: - d_delete(dentry); + if (d_really_is_positive(dentry)) + d_delete(dentry); nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); break; case 0: @@ -2484,8 +2485,10 @@ int nfs_unlink(struct inode *dir, struct dentry *dentry) */ error = -ETXTBSY; if (WARN_ON(dentry->d_flags & DCACHE_NFSFS_RENAMED) || - WARN_ON(dentry->d_fsdata == NFS_FSDATA_BLOCKED)) + WARN_ON(dentry->d_fsdata == NFS_FSDATA_BLOCKED)) { + spin_unlock(&dentry->d_lock); goto out; + } if (dentry->d_fsdata) /* old devname */ kfree(dentry->d_fsdata); diff --git a/fs/nfs/file.c b/fs/nfs/file.c index d2bcd4834c0e6857713aae58c3910d800594dd64..d8ec889a4b3f761e9378adf325a2846f76555b73 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -221,8 +221,10 @@ nfs_file_fsync_commit(struct file *file, int datasync) int nfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) { - struct nfs_open_context *ctx = nfs_file_open_context(file); struct inode *inode = file_inode(file); + struct nfs_inode *nfsi = NFS_I(inode); + long save_nredirtied = atomic_long_read(&nfsi->redirtied_pages); + long nredirtied; int ret; trace_nfs_fsync_enter(inode); @@ -237,15 +239,10 @@ nfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) ret = pnfs_sync_inode(inode, !!datasync); if (ret != 0) break; - if (!test_and_clear_bit(NFS_CONTEXT_RESEND_WRITES, &ctx->flags)) + nredirtied = atomic_long_read(&nfsi->redirtied_pages); + if (nredirtied == save_nredirtied) break; - /* - * If nfs_file_fsync_commit detected a server reboot, then - * resend all dirty pages that might have been covered by - * the NFS_CONTEXT_RESEND_WRITES flag - */ - start = 0; - end = LLONG_MAX; + save_nredirtied = nredirtied; } trace_nfs_fsync_exit(inode, ret); @@ -570,7 +567,8 @@ static vm_fault_t nfs_vm_page_mkwrite(struct vm_fault *vmf) } wait_on_bit_action(&NFS_I(inode)->flags, NFS_INO_INVALIDATING, - nfs_wait_bit_killable, TASK_KILLABLE); + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); lock_page(page); mapping = page_file_mapping(page); @@ -658,9 +656,9 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) goto out; } if (mntflags & NFS_MOUNT_WRITE_WAIT) { - result = filemap_fdatawait_range(file->f_mapping, - iocb->ki_pos - written, - iocb->ki_pos - 1); + filemap_fdatawait_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); } result = generic_write_sync(iocb, written); if (result < 0) diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 7d285561e59f673ecfc1e34d2c45e9695ef7422a..1ec79ccf89ad241d2d8ffca51550143eb81464ee 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -30,14 +30,20 @@ #define FF_LAYOUT_POLL_RETRY_MAX (15*HZ) #define FF_LAYOUTRETURN_MAXERR 20 +enum nfs4_ff_op_type { + NFS4_FF_OP_LAYOUTSTATS, + NFS4_FF_OP_LAYOUTRETURN, +}; + static unsigned short io_maxretrans; static const struct pnfs_commit_ops ff_layout_commit_ops; static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, struct nfs_pgio_header *hdr); -static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, +static int +ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, struct nfs42_layoutstat_devinfo *devinfo, - int dev_limit); + int dev_limit, enum nfs4_ff_op_type type); static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr, const struct nfs42_layoutstat_devinfo *devinfo, struct nfs4_ff_layout_mirror *mirror); @@ -1373,6 +1379,11 @@ static int ff_layout_read_prepare_common(struct rpc_task *task, return -EIO; } + if (!pnfs_is_valid_lseg(hdr->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + ff_layout_read_record_layoutstats_start(task, hdr); return 0; } @@ -1553,6 +1564,11 @@ static int ff_layout_write_prepare_common(struct rpc_task *task, return -EIO; } + if (!pnfs_is_valid_lseg(hdr->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + ff_layout_write_record_layoutstats_start(task, hdr); return 0; } @@ -1645,15 +1661,23 @@ static void ff_layout_commit_record_layoutstats_done(struct rpc_task *task, set_bit(NFS_LSEG_LAYOUTRETURN, &cdata->lseg->pls_flags); } -static void ff_layout_commit_prepare_common(struct rpc_task *task, - struct nfs_commit_data *cdata) +static int ff_layout_commit_prepare_common(struct rpc_task *task, + struct nfs_commit_data *cdata) { + if (!pnfs_is_valid_lseg(cdata->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + ff_layout_commit_record_layoutstats_start(task, cdata); + return 0; } static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data) { - ff_layout_commit_prepare_common(task, data); + if (ff_layout_commit_prepare_common(task, data)) + return; + rpc_call_start(task); } @@ -1949,6 +1973,65 @@ ff_layout_commit_pagelist(struct inode *inode, struct list_head *mds_pages, ff_layout_initiate_commit); } +static bool ff_layout_match_rw(const struct rpc_task *task, + const struct nfs_pgio_header *hdr, + const struct pnfs_layout_segment *lseg) +{ + return hdr->lseg == lseg; +} + +static bool ff_layout_match_commit(const struct rpc_task *task, + const struct nfs_commit_data *cdata, + const struct pnfs_layout_segment *lseg) +{ + return cdata->lseg == lseg; +} + +static bool ff_layout_match_io(const struct rpc_task *task, const void *data) +{ + const struct rpc_call_ops *ops = task->tk_ops; + + if (ops == &ff_layout_read_call_ops_v3 || + ops == &ff_layout_read_call_ops_v4 || + ops == &ff_layout_write_call_ops_v3 || + ops == &ff_layout_write_call_ops_v4) + return ff_layout_match_rw(task, task->tk_calldata, data); + if (ops == &ff_layout_commit_call_ops_v3 || + ops == &ff_layout_commit_call_ops_v4) + return ff_layout_match_commit(task, task->tk_calldata, data); + return false; +} + +static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds; + struct nfs4_pnfs_ds *ds; + struct nfs_client *ds_clp; + struct rpc_clnt *clnt; + u32 idx; + + for (idx = 0; idx < flseg->mirror_array_cnt; idx++) { + mirror = flseg->mirror_array[idx]; + mirror_ds = mirror->mirror_ds; + if (!mirror_ds) + continue; + ds = mirror->mirror_ds->ds; + if (!ds) + continue; + ds_clp = ds->ds_clp; + if (!ds_clp) + continue; + clnt = ds_clp->cl_rpcclient; + if (!clnt) + continue; + if (!rpc_cancel_tasks(clnt, -EAGAIN, ff_layout_match_io, lseg)) + continue; + rpc_clnt_disconnect(clnt); + } +} + static struct pnfs_ds_commit_info * ff_layout_get_ds_info(struct inode *inode) { @@ -2161,8 +2244,9 @@ ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args) FF_LAYOUTRETURN_MAXERR); spin_lock(&args->inode->i_lock); - ff_args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr, - &ff_args->devinfo[0], ARRAY_SIZE(ff_args->devinfo)); + ff_args->num_dev = ff_layout_mirror_prepare_stats( + &ff_layout->generic_hdr, &ff_args->devinfo[0], + ARRAY_SIZE(ff_args->devinfo), NFS4_FF_OP_LAYOUTRETURN); spin_unlock(&args->inode->i_lock); args->ld_private->ops = &layoutreturn_ops; @@ -2396,7 +2480,7 @@ static const struct nfs4_xdr_opaque_ops layoutstat_ops = { static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, struct nfs42_layoutstat_devinfo *devinfo, - int dev_limit) + int dev_limit, enum nfs4_ff_op_type type) { struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); struct nfs4_ff_layout_mirror *mirror; @@ -2408,7 +2492,9 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, break; if (IS_ERR_OR_NULL(mirror->mirror_ds)) continue; - if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags)) + if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, + &mirror->flags) && + type != NFS4_FF_OP_LAYOUTRETURN) continue; /* mirror refcount put in cleanup_layoutstats */ if (!refcount_inc_not_zero(&mirror->ref)) @@ -2448,7 +2534,9 @@ ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args) spin_lock(&args->inode->i_lock); ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout); args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr, - &args->devinfo[0], dev_count); + &args->devinfo[0], + dev_count, + NFS4_FF_OP_LAYOUTSTATS); spin_unlock(&args->inode->i_lock); if (!args->num_dev) { kfree(args->devinfo); @@ -2501,6 +2589,7 @@ static struct pnfs_layoutdriver_type flexfilelayout_type = { .prepare_layoutreturn = ff_layout_prepare_layoutreturn, .sync = pnfs_nfs_generic_sync, .prepare_layoutstats = ff_layout_prepare_layoutstats, + .cancel_io = ff_layout_cancel_io, }; static int __init nfs4flexfilelayout_init(void) diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index b4e46b0ffa2dc04d268827312286af190fef1ab3..6b2cfa59a1a2b40edc5aecc3417a4a747c0de303 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -72,18 +72,13 @@ nfs_fattr_to_ino_t(struct nfs_fattr *fattr) return nfs_fileid_to_ino_t(fattr->fileid); } -static int nfs_wait_killable(int mode) +int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) { - freezable_schedule_unsafe(); + schedule(); if (signal_pending_state(mode, current)) return -ERESTARTSYS; return 0; } - -int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) -{ - return nfs_wait_killable(mode); -} EXPORT_SYMBOL_GPL(nfs_wait_bit_killable); /** @@ -318,7 +313,7 @@ struct nfs_find_desc { static int nfs_find_actor(struct inode *inode, void *opaque) { - struct nfs_find_desc *desc = (struct nfs_find_desc *)opaque; + struct nfs_find_desc *desc = opaque; struct nfs_fh *fh = desc->fh; struct nfs_fattr *fattr = desc->fattr; @@ -336,7 +331,7 @@ nfs_find_actor(struct inode *inode, void *opaque) static int nfs_init_locked(struct inode *inode, void *opaque) { - struct nfs_find_desc *desc = (struct nfs_find_desc *)opaque; + struct nfs_find_desc *desc = opaque; struct nfs_fattr *fattr = desc->fattr; set_nfs_fileid(inode, fattr->fileid); @@ -426,6 +421,7 @@ nfs_ilookup(struct super_block *sb, struct nfs_fattr *fattr, struct nfs_fh *fh) static void nfs_inode_init_regular(struct nfs_inode *nfsi) { atomic_long_set(&nfsi->nrequests, 0); + atomic_long_set(&nfsi->redirtied_pages, 0); INIT_LIST_HEAD(&nfsi->commit_info.list); atomic_long_set(&nfsi->commit_info.ncommit, 0); atomic_set(&nfsi->commit_info.rpcs_out, 0); @@ -1331,7 +1327,8 @@ int nfs_clear_invalid_mapping(struct address_space *mapping) */ for (;;) { ret = wait_on_bit_action(bitlock, NFS_INO_INVALIDATING, - nfs_wait_bit_killable, TASK_KILLABLE); + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); if (ret) goto out; spin_lock(&inode->i_lock); @@ -2270,7 +2267,7 @@ static inline void nfs4_init_once(struct nfs_inode *nfsi) static void init_once(void *foo) { - struct nfs_inode *nfsi = (struct nfs_inode *) foo; + struct nfs_inode *nfsi = foo; inode_init_once(&nfsi->vfs_inode); INIT_LIST_HEAD(&nfsi->open_files); diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 27c720d71b4e33c65b6cf05d5053acec8a853c4c..d914d609b85b2d67ae315668a3ca59c9afba3bf4 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -435,7 +435,6 @@ extern void nfs_zap_acl_cache(struct inode *inode); extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags); extern bool nfs_check_cache_invalid(struct inode *, unsigned long); extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode); -extern int nfs_wait_atomic_killable(atomic_t *p, unsigned int mode); /* super.c */ extern const struct super_operations nfs_sops; @@ -503,7 +502,6 @@ extern void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, const struct nfs_pgio_completion_ops *compl_ops); extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio); extern void nfs_commit_free(struct nfs_commit_data *p); -extern void nfs_write_prepare(struct rpc_task *task, void *calldata); extern void nfs_commit_prepare(struct rpc_task *task, void *calldata); extern int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, @@ -606,6 +604,31 @@ static inline gfp_t nfs_io_gfp_mask(void) return GFP_KERNEL; } +/* + * Special version of should_remove_suid() that ignores capabilities. + */ +static inline int nfs_should_remove_suid(const struct inode *inode) +{ + umode_t mode = inode->i_mode; + int kill = 0; + + /* suid always must be killed */ + if (unlikely(mode & S_ISUID)) + kill = ATTR_KILL_SUID; + + /* + * sgid without any exec bits is just a mandatory locking mark; leave + * it alone. If some exec bits are set, it's a real sgid; kill it. + */ + if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) + kill |= ATTR_KILL_SGID; + + if (unlikely(kill && S_ISREG(mode))) + return kill; + + return 0; +} + /* unlink.c */ extern struct rpc_task * nfs_async_rename(struct inode *old_dir, struct inode *new_dir, diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index 1597eef40d54f6f7005d9326e154323b7f104fd3..2e7579626cf01dfa66ce8b980357b9b59e0c0db6 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -36,7 +36,8 @@ nfs3_rpc_wrapper(struct rpc_clnt *clnt, struct rpc_message *msg, int flags) res = rpc_call_sync(clnt, msg, flags); if (res != -EJUKEBOX) break; - freezable_schedule_timeout_killable_unsafe(NFS_JUKEBOX_RETRY_TIME); + __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(NFS_JUKEBOX_RETRY_TIME); res = -ERESTARTSYS; } while (!fatal_signal_pending(current)); return res; diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 068c45b3bc1ab19593441cc5b336e5e50a5e4322..13424f0d793b22df9da98d55918d714314f80e9e 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -78,10 +78,15 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); - if (status == 0) + if (status == 0) { + if (nfs_should_remove_suid(inode)) { + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); + spin_unlock(&inode->i_lock); + } status = nfs_post_op_update_inode_force_wcc(inode, res.falloc_fattr); - + } if (msg->rpc_proc == &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE]) trace_nfs4_fallocate(inode, &args, status); else @@ -336,7 +341,7 @@ static ssize_t _nfs42_proc_copy(struct file *src, return status; } } - status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping, + status = nfs_filemap_write_and_wait_range(src->f_mapping, pos_src, pos_src + (loff_t)count - 1); if (status) return status; @@ -1170,6 +1175,7 @@ static int _nfs42_proc_removexattr(struct inode *inode, const char *name) ret = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 1); + trace_nfs4_removexattr(inode, name, ret); if (!ret) nfs4_update_changeattr(inode, &res.cinfo, timestamp, 0); @@ -1209,6 +1215,7 @@ static int _nfs42_proc_setxattr(struct inode *inode, const char *name, ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1); + trace_nfs4_setxattr(inode, name, ret); for (; np > 0; np--) put_page(pages[np - 1]); @@ -1241,6 +1248,7 @@ static ssize_t _nfs42_proc_getxattr(struct inode *inode, const char *name, ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 0); + trace_nfs4_getxattr(inode, name, ret); if (ret < 0) return ret; @@ -1312,6 +1320,7 @@ static ssize_t _nfs42_proc_listxattrs(struct inode *inode, void *buf, ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 0); + trace_nfs4_listxattr(inode, ret); if (ret >= 0) { ret = res.copied; diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c index a9bf09fdf2c3203df1716799502d624cb2ccce58..76ae1183420668b547c161a278a72f3a6e8267c8 100644 --- a/fs/nfs/nfs42xattr.c +++ b/fs/nfs/nfs42xattr.c @@ -981,7 +981,7 @@ nfs4_xattr_entry_count(struct shrinker *shrink, struct shrink_control *sc) static void nfs4_xattr_cache_init_once(void *p) { - struct nfs4_xattr_cache *cache = (struct nfs4_xattr_cache *)p; + struct nfs4_xattr_cache *cache = p; spin_lock_init(&cache->listxattr_lock); atomic_long_set(&cache->nent, 0); diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c index b56f05113d367d6ecfb2f4827e0bac892f8a2193..fe1aeb0f048f2b73bf2fbc92d0771fefac1b1716 100644 --- a/fs/nfs/nfs42xdr.c +++ b/fs/nfs/nfs42xdr.c @@ -569,6 +569,14 @@ static int decode_listxattrs(struct xdr_stream *xdr, */ if (status == -ETOOSMALL) status = -ERANGE; + /* + * Special case: for LISTXATTRS, NFS4ERR_NOXATTR + * should be translated to success with zero-length reply. + */ + if (status == -ENODATA) { + res->eof = true; + status = 0; + } goto out; } diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index 79df6e83881b2154ce30bdc1ff3687fcbc8a9b5f..400a71e75238b55869811dce27238b0d0e88c27e 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -459,7 +459,6 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *); /* nfs4renewd.c */ extern void nfs4_schedule_state_renewal(struct nfs_client *); -extern void nfs4_renewd_prepare_shutdown(struct nfs_server *); extern void nfs4_kill_renewd(struct nfs_client *); extern void nfs4_renew_state(struct work_struct *); extern void nfs4_set_lease_period(struct nfs_client *clp, unsigned long lease); diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 3c5678aec006fce1dc315e7624772f2399172c4a..7a5162afa5c0dc33336a36d51fc2a5f0ca32aebb 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -254,7 +254,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) goto error; ip_addr = (const char *)buf; } - strlcpy(clp->cl_ipaddr, ip_addr, sizeof(clp->cl_ipaddr)); + strscpy(clp->cl_ipaddr, ip_addr, sizeof(clp->cl_ipaddr)); err = nfs_idmap_new(clp); if (err < 0) { diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index e88f6b18445ece3b51ea0781861575908c393893..9eb18128787951218b3f7a898dd9747a112777de 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -340,6 +340,11 @@ static struct file *__nfs42_ssc_open(struct vfsmount *ss_mnt, goto out; } + if (!S_ISREG(fattr->mode)) { + res = ERR_PTR(-EBADF); + goto out; + } + res = ERR_PTR(-ENOMEM); len = strlen(SSC_READ_NAME_BODY) + 16; read_name = kzalloc(len, GFP_KERNEL); @@ -357,6 +362,7 @@ static struct file *__nfs42_ssc_open(struct vfsmount *ss_mnt, r_ino->i_fop); if (IS_ERR(filep)) { res = ERR_CAST(filep); + iput(r_ino); goto out_free_name; } diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c index ec6afd3c4bca6b16c94d73179671409c9d1e6f9d..e3fdd2f45b01f581d9ad165aea047e8d5a2fbf2b 100644 --- a/fs/nfs/nfs4idmap.c +++ b/fs/nfs/nfs4idmap.c @@ -583,7 +583,7 @@ static int nfs_idmap_legacy_upcall(struct key *authkey, void *aux) struct request_key_auth *rka = get_request_key_auth(authkey); struct rpc_pipe_msg *msg; struct idmap_msg *im; - struct idmap *idmap = (struct idmap *)aux; + struct idmap *idmap = aux; struct key *key = rka->target_key; int ret = -ENOKEY; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 3ed14a2a84a4450ab8d4f75fe5b05e204c6bc19f..e2efcd26336c06d0799a53060e789b2162266a92 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -416,8 +416,8 @@ static int nfs4_delay_killable(long *timeout) { might_sleep(); - freezable_schedule_timeout_killable_unsafe( - nfs4_update_delay(timeout)); + __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(nfs4_update_delay(timeout)); if (!__fatal_signal_pending(current)) return 0; return -EINTR; @@ -427,7 +427,8 @@ static int nfs4_delay_interruptible(long *timeout) { might_sleep(); - freezable_schedule_timeout_interruptible_unsafe(nfs4_update_delay(timeout)); + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(nfs4_update_delay(timeout)); if (!signal_pending(current)) return 0; return __fatal_signal_pending(current) ? -EINTR :-ERESTARTSYS; @@ -6607,7 +6608,7 @@ static void nfs4_delegreturn_prepare(struct rpc_task *task, void *data) struct nfs4_delegreturndata *d_data; struct pnfs_layout_hdr *lo; - d_data = (struct nfs4_delegreturndata *)data; + d_data = data; if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task)) { nfs4_sequence_done(task, &d_data->res.seq_res); @@ -7406,7 +7407,8 @@ nfs4_retry_setlk_simple(struct nfs4_state *state, int cmd, status = nfs4_proc_setlk(state, cmd, request); if ((status != -EAGAIN) || IS_SETLK(cmd)) break; - freezable_schedule_timeout_interruptible(timeout); + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); + schedule_timeout(timeout); timeout *= 2; timeout = min_t(unsigned long, NFS4_LOCK_MAXTIMEOUT, timeout); status = -ERESTARTSYS; @@ -7474,10 +7476,8 @@ nfs4_retry_setlk(struct nfs4_state *state, int cmd, struct file_lock *request) break; status = -ERESTARTSYS; - freezer_do_not_count(); - wait_woken(&waiter.wait, TASK_INTERRUPTIBLE, + wait_woken(&waiter.wait, TASK_INTERRUPTIBLE|TASK_FREEZABLE, NFS4_LOCK_MAXTIMEOUT); - freezer_count(); } while (!signalled()); remove_wait_queue(q, &waiter.wait); @@ -8900,7 +8900,7 @@ int nfs4_proc_exchange_id(struct nfs_client *clp, const struct cred *cred) void nfs4_test_session_trunk(struct rpc_clnt *clnt, struct rpc_xprt *xprt, void *data) { - struct nfs4_add_xprt_data *adata = (struct nfs4_add_xprt_data *)data; + struct nfs4_add_xprt_data *adata = data; struct rpc_task *task; int status; diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index 9bab3e9c702a47153703c7f7ae62176fed75fe6f..c3503fb26fa271f1073ede4876331aa2df92cb4e 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -497,8 +497,7 @@ nfs4_alloc_state_owner(struct nfs_server *server, sp = kzalloc(sizeof(*sp), gfp_flags); if (!sp) return NULL; - sp->so_seqid.owner_id = ida_simple_get(&server->openowner_id, 0, 0, - gfp_flags); + sp->so_seqid.owner_id = ida_alloc(&server->openowner_id, gfp_flags); if (sp->so_seqid.owner_id < 0) { kfree(sp); return NULL; @@ -534,7 +533,7 @@ static void nfs4_free_state_owner(struct nfs4_state_owner *sp) { nfs4_destroy_seqid_counter(&sp->so_seqid); put_cred(sp->so_cred); - ida_simple_remove(&sp->so_server->openowner_id, sp->so_seqid.owner_id); + ida_free(&sp->so_server->openowner_id, sp->so_seqid.owner_id); kfree(sp); } @@ -877,8 +876,7 @@ static struct nfs4_lock_state *nfs4_alloc_lock_state(struct nfs4_state *state, f refcount_set(&lsp->ls_count, 1); lsp->ls_state = state; lsp->ls_owner = fl_owner; - lsp->ls_seqid.owner_id = ida_simple_get(&server->lockowner_id, - 0, 0, GFP_KERNEL_ACCOUNT); + lsp->ls_seqid.owner_id = ida_alloc(&server->lockowner_id, GFP_KERNEL_ACCOUNT); if (lsp->ls_seqid.owner_id < 0) goto out_free; INIT_LIST_HEAD(&lsp->ls_locks); @@ -890,7 +888,7 @@ out_free: void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) { - ida_simple_remove(&server->lockowner_id, lsp->ls_seqid.owner_id); + ida_free(&server->lockowner_id, lsp->ls_seqid.owner_id); nfs4_destroy_seqid_counter(&lsp->ls_seqid); kfree(lsp); } @@ -1314,7 +1312,8 @@ int nfs4_wait_clnt_recover(struct nfs_client *clp) refcount_inc(&clp->cl_count); res = wait_on_bit_action(&clp->cl_state, NFS4CLNT_MANAGER_RUNNING, - nfs_wait_bit_killable, TASK_KILLABLE); + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); if (res) goto out; if (clp->cl_cons_state < 0) diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h index 6ee6ad3674a29734bdb7b3bcda5134684634acc8..2cff5901c6894f209035b2bc812476f03954a555 100644 --- a/fs/nfs/nfs4trace.h +++ b/fs/nfs/nfs4trace.h @@ -2097,6 +2097,7 @@ TRACE_EVENT(ff_layout_commit_error, ) ); +#ifdef CONFIG_NFS_V4_2 TRACE_DEFINE_ENUM(NFS4_CONTENT_DATA); TRACE_DEFINE_ENUM(NFS4_CONTENT_HOLE); @@ -2105,7 +2106,6 @@ TRACE_DEFINE_ENUM(NFS4_CONTENT_HOLE); { NFS4_CONTENT_DATA, "DATA" }, \ { NFS4_CONTENT_HOLE, "HOLE" }) -#ifdef CONFIG_NFS_V4_2 TRACE_EVENT(nfs4_llseek, TP_PROTO( const struct inode *inode, @@ -2496,6 +2496,54 @@ TRACE_EVENT(nfs4_offload_cancel, __entry->stateid_seq, __entry->stateid_hash ) ); + +DECLARE_EVENT_CLASS(nfs4_xattr_event, + TP_PROTO( + const struct inode *inode, + const char *name, + int error + ), + + TP_ARGS(inode, name, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __string(name, name) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = NFS_FILEID(inode); + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __assign_str(name, name); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "name=%s", + -__entry->error, show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __get_str(name) + ) +); +#define DEFINE_NFS4_XATTR_EVENT(name) \ + DEFINE_EVENT(nfs4_xattr_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + const char *name, \ + int error \ + ), \ + TP_ARGS(inode, name, error)) +DEFINE_NFS4_XATTR_EVENT(nfs4_getxattr); +DEFINE_NFS4_XATTR_EVENT(nfs4_setxattr); +DEFINE_NFS4_XATTR_EVENT(nfs4_removexattr); + +DEFINE_NFS4_INODE_EVENT(nfs4_listxattr); #endif /* CONFIG_NFS_V4_2 */ #endif /* CONFIG_NFS_V4_1 */ diff --git a/fs/nfs/nfsroot.c b/fs/nfs/nfsroot.c index fa148308822cc8d4b02ee29772c0ba40843f1fe2..620329b7e6aeb18eac2a5512e20592ef344ad506 100644 --- a/fs/nfs/nfsroot.c +++ b/fs/nfs/nfsroot.c @@ -139,7 +139,7 @@ static int __init nfs_root_setup(char *line) ROOT_DEV = Root_NFS; if (line[0] == '/' || line[0] == ',' || (line[0] >= '0' && line[0] <= '9')) { - strlcpy(nfs_root_parms, line, sizeof(nfs_root_parms)); + strscpy(nfs_root_parms, line, sizeof(nfs_root_parms)); } else { size_t n = strlen(line) + sizeof(NFS_ROOT) - 1; if (n >= sizeof(nfs_root_parms)) diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 41a9b6b58fb9fa15c9f162fa8dfa267a1c16e049..a5db5158c63455e1d0424c046ee77cb082a7d731 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -710,6 +710,7 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, u32 seq) { struct pnfs_layout_segment *lseg, *next; + struct nfs_server *server = NFS_SERVER(lo->plh_inode); int remaining = 0; dprintk("%s:Begin lo %p\n", __func__, lo); @@ -722,8 +723,10 @@ pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, "offset %llu length %llu\n", __func__, lseg, lseg->pls_range.iomode, lseg->pls_seq, lseg->pls_range.offset, lseg->pls_range.length); - if (!mark_lseg_invalid(lseg, tmp_list)) - remaining++; + if (mark_lseg_invalid(lseg, tmp_list)) + continue; + remaining++; + pnfs_lseg_cancel_io(server, lseg); } dprintk("%s:Return %i\n", __func__, remaining); return remaining; @@ -1908,7 +1911,7 @@ static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) pnfs_layoutcommit_inode(lo->plh_inode, false); return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, nfs_wait_bit_killable, - TASK_KILLABLE); + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); } static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) @@ -2485,6 +2488,7 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, u32 seq) { struct pnfs_layout_segment *lseg, *next; + struct nfs_server *server = NFS_SERVER(lo->plh_inode); int remaining = 0; dprintk("%s:Begin lo %p\n", __func__, lo); @@ -2507,6 +2511,7 @@ pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, continue; remaining++; set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); + pnfs_lseg_cancel_io(server, lseg); } if (remaining) { @@ -2817,7 +2822,6 @@ int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) /* Resend all requests through the MDS */ nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, hdr->completion_ops); - set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags); return nfs_pageio_resend(&pgio, hdr); } EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); @@ -3193,7 +3197,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) status = wait_on_bit_lock_action(&nfsi->flags, NFS_INO_LAYOUTCOMMITTING, nfs_wait_bit_killable, - TASK_KILLABLE); + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); if (status) goto out; } diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index f331f067691b0854c7806d2099fabf8932c920a9..e3e6a41f19de6d08b5c855f18679b46aa8f37edc 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -169,6 +169,8 @@ struct pnfs_layoutdriver_type { void (*cleanup_layoutcommit) (struct nfs4_layoutcommit_data *data); int (*prepare_layoutcommit) (struct nfs4_layoutcommit_args *args); int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args); + + void (*cancel_io)(struct pnfs_layout_segment *lseg); }; struct pnfs_commit_ops { @@ -685,6 +687,13 @@ pnfs_lseg_request_intersecting(struct pnfs_layout_segment *lseg, struct nfs_page req_offset(req), req_last); } +static inline void pnfs_lseg_cancel_io(struct nfs_server *server, + struct pnfs_layout_segment *lseg) +{ + if (server->pnfs_curr_ld->cancel_io) + server->pnfs_curr_ld->cancel_io(lseg); +} + extern unsigned int layoutstats_timer; #ifdef NFS_DEBUG diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 657c242a18ff1010ff031b7cd0d70a3d6ecc01aa..987c88ddeaf060161e2d1c5fc0a870a94352bac8 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -374,12 +374,12 @@ pnfs_bucket_search_commit_reqs(struct pnfs_commit_bucket *buckets, return NULL; } -/* pnfs_generic_search_commit_reqs - Search lists in @cinfo for the head reqest +/* pnfs_generic_search_commit_reqs - Search lists in @cinfo for the head request * for @page * @cinfo - commit info for current inode * @page - page to search for matching head request * - * Returns a the head request if one is found, otherwise returns NULL. + * Return: the head request if one is found, otherwise %NULL. */ struct nfs_page * pnfs_generic_search_commit_reqs(struct nfs_commit_info *cinfo, struct page *page) diff --git a/fs/nfs/super.c b/fs/nfs/super.c index 82944e14fcea19533f9bb2ea9e1819ea0da784ec..ee66ffdb985e8094131a5226baecb876cb05c4a0 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -1051,22 +1051,31 @@ static void nfs_fill_super(struct super_block *sb, struct nfs_fs_context *ctx) if (ctx->bsize) sb->s_blocksize = nfs_block_size(ctx->bsize, &sb->s_blocksize_bits); - if (server->nfs_client->rpc_ops->version != 2) { - /* The VFS shouldn't apply the umask to mode bits. We will do - * so ourselves when necessary. + switch (server->nfs_client->rpc_ops->version) { + case 2: + sb->s_time_gran = 1000; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + break; + case 3: + /* + * The VFS shouldn't apply the umask to mode bits. + * We will do so ourselves when necessary. */ sb->s_flags |= SB_POSIXACL; sb->s_time_gran = 1; - sb->s_export_op = &nfs_export_ops; - } else - sb->s_time_gran = 1000; - - if (server->nfs_client->rpc_ops->version != 4) { sb->s_time_min = 0; sb->s_time_max = U32_MAX; - } else { + sb->s_export_op = &nfs_export_ops; + break; + case 4: + sb->s_flags |= SB_POSIXACL; + sb->s_time_gran = 1; sb->s_time_min = S64_MIN; sb->s_time_max = S64_MAX; + if (server->caps & NFS_CAP_ATOMIC_OPEN_V1) + sb->s_export_op = &nfs_export_ops; + break; } sb->s_magic = NFS_SUPER_MAGIC; diff --git a/fs/nfs/write.c b/fs/nfs/write.c index 51a7e202d6e542d53382fe2a38b8a7fe2a83fdcd..f41d24b54fd1f8b19a3e34e8cfad1bdb7b5ff206 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -1420,10 +1420,12 @@ static void nfs_initiate_write(struct nfs_pgio_header *hdr, */ static void nfs_redirty_request(struct nfs_page *req) { + struct nfs_inode *nfsi = NFS_I(page_file_mapping(req->wb_page)->host); + /* Bump the transmission count */ req->wb_nio++; nfs_mark_request_dirty(req); - set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); + atomic_long_inc(&nfsi->redirtied_pages); nfs_end_page_writeback(req); nfs_release_request(req); } @@ -1494,31 +1496,6 @@ void nfs_commit_prepare(struct rpc_task *task, void *calldata) NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); } -/* - * Special version of should_remove_suid() that ignores capabilities. - */ -static int nfs_should_remove_suid(const struct inode *inode) -{ - umode_t mode = inode->i_mode; - int kill = 0; - - /* suid always must be killed */ - if (unlikely(mode & S_ISUID)) - kill = ATTR_KILL_SUID; - - /* - * sgid without any exec bits is just a mandatory locking mark; leave - * it alone. If some exec bits are set, it's a real sgid; kill it. - */ - if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) - kill |= ATTR_KILL_SGID; - - if (unlikely(kill && S_ISREG(mode))) - return kill; - - return 0; -} - static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr, struct nfs_fattr *fattr) { @@ -1904,7 +1881,7 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data) /* We have a mismatch. Write the page again */ dprintk_cont(" mismatch\n"); nfs_mark_request_dirty(req); - set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags); + atomic_long_inc(&NFS_I(data->inode)->redirtied_pages); next: nfs_unlock_and_release_request(req); /* Latency breaker */ diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h index 65c331f75e9c7bdb64a222934273d535a0c45a0c..f21259ead64bb3e64471da7eb290557168e81ec8 100644 --- a/fs/nfsd/cache.h +++ b/fs/nfsd/cache.h @@ -84,6 +84,6 @@ int nfsd_reply_cache_init(struct nfsd_net *); void nfsd_reply_cache_shutdown(struct nfsd_net *); int nfsd_cache_lookup(struct svc_rqst *); void nfsd_cache_update(struct svc_rqst *, int, __be32 *); -int nfsd_reply_cache_stats_open(struct inode *, struct file *); +int nfsd_reply_cache_stats_show(struct seq_file *m, void *v); #endif /* NFSCACHE_H */ diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index eeed4ae5b4ad90031a5aab1fe79b6ac0e6c83ccc..29a62db155fbabcc6c8c627d9d2010c9d3e4cbe3 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -405,22 +405,15 @@ nfsd_file_unhash(struct nfsd_file *nf) return false; } -/* - * Return true if the file was unhashed. - */ -static bool +static void nfsd_file_unhash_and_dispose(struct nfsd_file *nf, struct list_head *dispose) { trace_nfsd_file_unhash_and_dispose(nf); - if (!nfsd_file_unhash(nf)) - return false; - /* keep final reference for nfsd_file_lru_dispose */ - if (refcount_dec_not_one(&nf->nf_ref)) - return true; - - nfsd_file_lru_remove(nf); - list_add(&nf->nf_lru, dispose); - return true; + if (nfsd_file_unhash(nf)) { + /* caller must call nfsd_file_dispose_list() later */ + nfsd_file_lru_remove(nf); + list_add(&nf->nf_lru, dispose); + } } static void @@ -562,8 +555,6 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose) * @lock: LRU list lock (unused) * @arg: dispose list * - * Note this can deadlock with nfsd_file_cache_purge. - * * Return values: * %LRU_REMOVED: @item was removed from the LRU * %LRU_ROTATE: @item is to be moved to the LRU tail @@ -748,8 +739,6 @@ nfsd_file_close_inode(struct inode *inode) * * Walk the LRU list and close any entries that have not been used since * the last scan. - * - * Note this can deadlock with nfsd_file_cache_purge. */ static void nfsd_file_delayed_close(struct work_struct *work) @@ -891,16 +880,12 @@ out_err: goto out; } -/* - * Note this can deadlock with nfsd_file_lru_cb. - */ static void __nfsd_file_cache_purge(struct net *net) { struct rhashtable_iter iter; struct nfsd_file *nf; LIST_HEAD(dispose); - bool del; rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter); do { @@ -910,14 +895,7 @@ __nfsd_file_cache_purge(struct net *net) while (!IS_ERR_OR_NULL(nf)) { if (net && nf->nf_net != net) continue; - del = nfsd_file_unhash_and_dispose(nf, &dispose); - - /* - * Deadlock detected! Something marked this entry as - * unhased, but hasn't removed it from the hash list. - */ - WARN_ON_ONCE(!del); - + nfsd_file_unhash_and_dispose(nf, &dispose); nf = rhashtable_walk_next(&iter); } @@ -1064,9 +1042,10 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, .need = may_flags & NFSD_FILE_MAY_MASK, .net = SVC_NET(rqstp), }; - struct nfsd_file *nf, *new; - bool retry = true; + bool open_retry = true; + struct nfsd_file *nf; __be32 status; + int ret; status = fh_verify(rqstp, fhp, S_IFREG, may_flags|NFSD_MAY_OWNER_OVERRIDE); @@ -1076,35 +1055,33 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, key.cred = get_current_cred(); retry: - /* Avoid allocation if the item is already in cache */ - nf = rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key, - nfsd_file_rhash_params); + rcu_read_lock(); + nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key, + nfsd_file_rhash_params); if (nf) nf = nfsd_file_get(nf); + rcu_read_unlock(); if (nf) goto wait_for_construction; - new = nfsd_file_alloc(&key, may_flags); - if (!new) { + nf = nfsd_file_alloc(&key, may_flags); + if (!nf) { status = nfserr_jukebox; goto out_status; } - nf = rhashtable_lookup_get_insert_key(&nfsd_file_rhash_tbl, - &key, &new->nf_rhash, - nfsd_file_rhash_params); - if (!nf) { - nf = new; + ret = rhashtable_lookup_insert_key(&nfsd_file_rhash_tbl, + &key, &nf->nf_rhash, + nfsd_file_rhash_params); + if (likely(ret == 0)) goto open_file; - } - if (IS_ERR(nf)) - goto insert_err; - nf = nfsd_file_get(nf); - if (nf == NULL) { - nf = new; - goto open_file; - } - nfsd_file_slab_free(&new->nf_rcu); + + nfsd_file_slab_free(&nf->nf_rcu); + if (ret == -EEXIST) + goto retry; + trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret); + status = nfserr_jukebox; + goto out_status; wait_for_construction: wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE); @@ -1112,11 +1089,11 @@ wait_for_construction: /* Did construction of this file fail? */ if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf); - if (!retry) { + if (!open_retry) { status = nfserr_jukebox; goto out; } - retry = false; + open_retry = false; nfsd_file_put_noref(nf); goto retry; } @@ -1164,13 +1141,6 @@ open_file: smp_mb__after_atomic(); wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING); goto out; - -insert_err: - nfsd_file_slab_free(&new->nf_rcu); - trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, PTR_ERR(nf)); - nf = NULL; - status = nfserr_jukebox; - goto out_status; } /** @@ -1212,7 +1182,7 @@ nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp, * scraping this file for info should test the labels to ensure they're * getting the correct field. */ -static int nfsd_file_cache_stats_show(struct seq_file *m, void *v) +int nfsd_file_cache_stats_show(struct seq_file *m, void *v) { unsigned long releases = 0, pages_flushed = 0, evictions = 0; unsigned long hits = 0, acquisitions = 0; @@ -1259,8 +1229,3 @@ static int nfsd_file_cache_stats_show(struct seq_file *m, void *v) seq_printf(m, "pages flushed: %lu\n", pages_flushed); return 0; } - -int nfsd_file_cache_stats_open(struct inode *inode, struct file *file) -{ - return single_open(file, nfsd_file_cache_stats_show, NULL); -} diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h index 8e8c0c47d67df489c1d82dc67680adfa05b746ce..357832bac736b89e2ebe5e159e54346e02cf983a 100644 --- a/fs/nfsd/filecache.h +++ b/fs/nfsd/filecache.h @@ -60,5 +60,5 @@ __be32 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct nfsd_file **nfp); __be32 nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned int may_flags, struct nfsd_file **nfp); -int nfsd_file_cache_stats_open(struct inode *, struct file *); +int nfsd_file_cache_stats_show(struct seq_file *m, void *v); #endif /* _FS_NFSD_FILECACHE_H */ diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index ffe17743cc74be62a818481df64550677ed3bc5b..8c854ba3285bbcddfe62010dc6b93018fdc3927c 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -192,6 +192,10 @@ struct nfsd_net { atomic_t nfs4_client_count; int nfs4_max_clients; + + atomic_t nfsd_courtesy_clients; + struct shrinker nfsd_client_shrinker; + struct delayed_work nfsd_shrinker_work; }; /* Simple check to find out if a given net was properly initialized */ diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 9edd3c1a30fb10c14676eda869e9db94e9234804..13e6e6897f6cf332888f44648fc99781ad35fd30 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -331,6 +331,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, @@ -342,6 +343,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_encode = nfsaclsvc_encode_getaclres, .pc_release = nfsaclsvc_release_getacl, .pc_argsize = sizeof(struct nfsd3_getaclargs), + .pc_argzero = sizeof(struct nfsd3_getaclargs), .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), @@ -353,6 +355,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_encode = nfssvc_encode_attrstatres, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd3_setaclargs), + .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, @@ -364,6 +367,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_encode = nfssvc_encode_attrstatres, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, @@ -375,6 +379,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = { .pc_encode = nfsaclsvc_encode_accessres, .pc_release = nfsaclsvc_release_access, .pc_argsize = sizeof(struct nfsd3_accessargs), + .pc_argzero = sizeof(struct nfsd3_accessargs), .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1, diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index 9446c67436649cebfbdca11ea82820b3586c0c13..2fb9ee3564558e11205f34593de438f40824896a 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -252,6 +252,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, @@ -263,6 +264,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_encode = nfs3svc_encode_getaclres, .pc_release = nfs3svc_release_getacl, .pc_argsize = sizeof(struct nfsd3_getaclargs), + .pc_argzero = sizeof(struct nfsd3_getaclargs), .pc_ressize = sizeof(struct nfsd3_getaclres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+2*(1+ACL), @@ -274,6 +276,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = { .pc_encode = nfs3svc_encode_setaclres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_setaclargs), + .pc_argzero = sizeof(struct nfsd3_setaclargs), .pc_ressize = sizeof(struct nfsd3_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT, diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index a41cca619338dba13c600ff49b61e4593b8de798..923d9a80df92cdfc46f5afad010695dae681c603 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -150,7 +150,6 @@ nfsd3_proc_read(struct svc_rqst *rqstp) { struct nfsd3_readargs *argp = rqstp->rq_argp; struct nfsd3_readres *resp = rqstp->rq_resp; - u32 max_blocksize = svc_max_payload(rqstp); unsigned int len; int v; @@ -159,7 +158,8 @@ nfsd3_proc_read(struct svc_rqst *rqstp) (unsigned long) argp->count, (unsigned long long) argp->offset); - argp->count = min_t(u32, argp->count, max_blocksize); + argp->count = min_t(u32, argp->count, svc_max_payload(rqstp)); + argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen); if (argp->offset > (u64)OFFSET_MAX) argp->offset = (u64)OFFSET_MAX; if (argp->offset + argp->count > (u64)OFFSET_MAX) @@ -563,25 +563,18 @@ static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp, { struct xdr_buf *buf = &resp->dirlist; struct xdr_stream *xdr = &resp->xdr; - - count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp)); + unsigned int sendbuf = min_t(unsigned int, rqstp->rq_res.buflen, + svc_max_payload(rqstp)); memset(buf, 0, sizeof(*buf)); /* Reserve room for the NULL ptr & eof flag (-2 words) */ - buf->buflen = count - XDR_UNIT * 2; + buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), sendbuf); + buf->buflen -= XDR_UNIT * 2; buf->pages = rqstp->rq_next_page; rqstp->rq_next_page += (buf->buflen + PAGE_SIZE - 1) >> PAGE_SHIFT; - /* This is xdr_init_encode(), but it assumes that - * the head kvec has already been consumed. */ - xdr_set_scratch_buffer(xdr, NULL, 0); - xdr->buf = buf; - xdr->page_ptr = buf->pages; - xdr->iov = NULL; - xdr->p = page_address(*buf->pages); - xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE); - xdr->rqst = NULL; + xdr_init_encode_pages(xdr, buf, buf->pages, NULL); } /* @@ -808,6 +801,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST, @@ -819,6 +813,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_getattrres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd3_attrstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, @@ -830,6 +825,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_wccstatres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_sattrargs), + .pc_argzero = sizeof(struct nfsd3_sattrargs), .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, @@ -841,6 +837,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_lookupres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), .pc_ressize = sizeof(struct nfsd3_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+pAT+pAT, @@ -852,6 +849,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_accessres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_accessargs), + .pc_argzero = sizeof(struct nfsd3_accessargs), .pc_ressize = sizeof(struct nfsd3_accessres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1, @@ -863,6 +861,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_readlinkres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd3_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4, @@ -874,6 +873,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_readres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_readargs), + .pc_argzero = sizeof(struct nfsd3_readargs), .pc_ressize = sizeof(struct nfsd3_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+4+NFSSVC_MAXBLKSIZE/4, @@ -885,6 +885,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_writeres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_writeargs), + .pc_argzero = sizeof(struct nfsd3_writeargs), .pc_ressize = sizeof(struct nfsd3_writeres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+4, @@ -896,6 +897,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_createres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_createargs), + .pc_argzero = sizeof(struct nfsd3_createargs), .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, @@ -907,6 +909,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_createres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_mkdirargs), + .pc_argzero = sizeof(struct nfsd3_mkdirargs), .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, @@ -918,6 +921,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_createres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_symlinkargs), + .pc_argzero = sizeof(struct nfsd3_symlinkargs), .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, @@ -929,6 +933,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_createres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_mknodargs), + .pc_argzero = sizeof(struct nfsd3_mknodargs), .pc_ressize = sizeof(struct nfsd3_createres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+(1+FH+pAT)+WC, @@ -940,6 +945,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_wccstatres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, @@ -951,6 +957,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_wccstatres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), .pc_ressize = sizeof(struct nfsd3_wccstatres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC, @@ -962,6 +969,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_renameres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_renameargs), + .pc_argzero = sizeof(struct nfsd3_renameargs), .pc_ressize = sizeof(struct nfsd3_renameres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+WC+WC, @@ -973,6 +981,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_linkres, .pc_release = nfs3svc_release_fhandle2, .pc_argsize = sizeof(struct nfsd3_linkargs), + .pc_argzero = sizeof(struct nfsd3_linkargs), .pc_ressize = sizeof(struct nfsd3_linkres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+pAT+WC, @@ -984,6 +993,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_readdirres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_readdirargs), + .pc_argzero = sizeof(struct nfsd3_readdirargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, .pc_name = "READDIR", @@ -994,6 +1004,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_readdirres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_readdirplusargs), + .pc_argzero = sizeof(struct nfsd3_readdirplusargs), .pc_ressize = sizeof(struct nfsd3_readdirres), .pc_cachetype = RC_NOCACHE, .pc_name = "READDIRPLUS", @@ -1003,6 +1014,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_decode = nfs3svc_decode_fhandleargs, .pc_encode = nfs3svc_encode_fsstatres, .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), .pc_ressize = sizeof(struct nfsd3_fsstatres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+2*6+1, @@ -1013,6 +1025,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_decode = nfs3svc_decode_fhandleargs, .pc_encode = nfs3svc_encode_fsinfores, .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), .pc_ressize = sizeof(struct nfsd3_fsinfores), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+12, @@ -1023,6 +1036,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_decode = nfs3svc_decode_fhandleargs, .pc_encode = nfs3svc_encode_pathconfres, .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), .pc_ressize = sizeof(struct nfsd3_pathconfres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+pAT+6, @@ -1034,6 +1048,7 @@ static const struct svc_procedure nfsd_procedures3[22] = { .pc_encode = nfs3svc_encode_commitres, .pc_release = nfs3svc_release_fhandle, .pc_argsize = sizeof(struct nfsd3_commitargs), + .pc_argzero = sizeof(struct nfsd3_commitargs), .pc_ressize = sizeof(struct nfsd3_commitres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+WC+2, diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c index 0293b8d65f10fc948c212f653dd9f3e4121a3b48..3308dd671ef0b87a3016db8468ea33cd3aaaada8 100644 --- a/fs/nfsd/nfs3xdr.c +++ b/fs/nfsd/nfs3xdr.c @@ -571,10 +571,8 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) args->count = max_blocksize; args->len = max_blocksize; } - if (!xdr_stream_subsegment(xdr, &args->payload, args->count)) - return false; - return true; + return xdr_stream_subsegment(xdr, &args->payload, args->count); } bool @@ -616,8 +614,6 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) { struct nfsd3_symlinkargs *args = rqstp->rq_argp; struct kvec *head = rqstp->rq_arg.head; - struct kvec *tail = rqstp->rq_arg.tail; - size_t remaining; if (!svcxdr_decode_diropargs3(xdr, &args->ffh, &args->fname, &args->flen)) return false; @@ -626,16 +622,10 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) if (xdr_stream_decode_u32(xdr, &args->tlen) < 0) return false; - /* request sanity */ - remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len; - remaining -= xdr_stream_pos(xdr); - if (remaining < xdr_align_size(args->tlen)) - return false; - - args->first.iov_base = xdr->p; + /* symlink_data */ args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); - - return true; + args->first.iov_base = xdr_inline_decode(xdr, args->tlen); + return args->first.iov_base != NULL; } bool diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 4ce328209f6140517d0fae5abc6a7c1e60d8e0b0..f0e69edf5f0f1ccabe2a703aca39e0d048bf8ba6 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -1371,11 +1371,21 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, cb->cb_holds_slot = false; } -void nfsd4_run_cb(struct nfsd4_callback *cb) +/** + * nfsd4_run_cb - queue up a callback job to run + * @cb: callback to queue + * + * Kick off a callback to do its thing. Returns false if it was already + * on a queue, true otherwise. + */ +bool nfsd4_run_cb(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; + bool queued; nfsd41_cb_inflight_begin(clp); - if (!nfsd4_queue_cb(cb)) + queued = nfsd4_queue_cb(cb); + if (!queued) nfsd41_cb_inflight_end(clp); + return queued; } diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c index f92161ce1f97dd4412dd59813a5f769dae1b1d54..e70a1a2999b7b4f2943478259c144f14beb0b1b0 100644 --- a/fs/nfsd/nfs4idmap.c +++ b/fs/nfsd/nfs4idmap.c @@ -82,8 +82,8 @@ ent_init(struct cache_head *cnew, struct cache_head *citm) new->id = itm->id; new->type = itm->type; - strlcpy(new->name, itm->name, sizeof(new->name)); - strlcpy(new->authname, itm->authname, sizeof(new->authname)); + strscpy(new->name, itm->name, sizeof(new->name)); + strscpy(new->authname, itm->authname, sizeof(new->authname)); } static void @@ -548,7 +548,7 @@ idmap_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen return nfserr_badowner; memcpy(key.name, name, namelen); key.name[namelen] = '\0'; - strlcpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); + strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); ret = idmap_lookup(rqstp, nametoid_lookup, &key, nn->nametoid_cache, &item); if (ret == -ENOENT) return nfserr_badowner; @@ -584,7 +584,7 @@ static __be32 idmap_id_to_name(struct xdr_stream *xdr, int ret; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); - strlcpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); + strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); ret = idmap_lookup(rqstp, idtoname_lookup, &key, nn->idtoname_cache, &item); if (ret == -ENOENT) return encode_ascii_id(xdr, id); diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 2c05692a9abf1a45944bbe94da635317be24dadb..3564d1c6f61042a71c7200e6f073da1acf7535d1 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -658,7 +658,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task) ktime_t now, cutoff; const struct nfsd4_layout_ops *ops; - + trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task); switch (task->tk_status) { case 0: case -NFS4ERR_DELAY: diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index a72ab97f77efe2b8fcbab183667c77da26909274..8beb2bc4c328f5a908be16c151c013f398cd71ff 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -141,7 +141,6 @@ fh_dup2(struct svc_fh *dst, struct svc_fh *src) static __be32 do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open, int accmode) { - __be32 status; if (open->op_truncate && !(open->op_share_access & NFS4_SHARE_ACCESS_WRITE)) @@ -156,9 +155,7 @@ do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfs if (open->op_share_deny & NFS4_SHARE_DENY_READ) accmode |= NFSD_MAY_WRITE; - status = fh_verify(rqstp, current_fh, S_IFREG, accmode); - - return status; + return fh_verify(rqstp, current_fh, S_IFREG, accmode); } static __be32 nfsd_check_obj_isreg(struct svc_fh *fh) @@ -454,7 +451,6 @@ static __be32 do_open_fhandle(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_open *open) { struct svc_fh *current_fh = &cstate->current_fh; - __be32 status; int accmode = 0; /* We don't know the target directory, and therefore can not @@ -479,9 +475,7 @@ do_open_fhandle(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, str if (open->op_claim_type == NFS4_OPEN_CLAIM_DELEG_CUR_FH) accmode = NFSD_MAY_OWNER_OVERRIDE; - status = do_open_permission(rqstp, current_fh, open, accmode); - - return status; + return do_open_permission(rqstp, current_fh, open, accmode); } static void @@ -668,11 +662,9 @@ static __be32 nfsd4_putrootfh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) { - __be32 status; - fh_put(&cstate->current_fh); - status = exp_pseudoroot(rqstp, &cstate->current_fh); - return status; + + return exp_pseudoroot(rqstp, &cstate->current_fh); } static __be32 @@ -1343,7 +1335,7 @@ try_again: return 0; } if (work) { - strlcpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr) - 1); + strscpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr) - 1); refcount_set(&work->nsui_refcnt, 2); work->nsui_busy = true; list_add_tail(&work->nsui_list, &nn->nfsd_ssc_mount_list); @@ -1621,6 +1613,10 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, struct rpc_task *task) { + struct nfsd4_cb_offload *cbo = + container_of(cb, struct nfsd4_cb_offload, co_cb); + + trace_nfsd_cb_offload_done(&cbo->co_res.cb_stateid, task); return 1; } @@ -1768,7 +1764,13 @@ static int nfsd4_do_async_copy(void *data) filp = nfs42_ssc_open(copy->ss_mnt, ©->c_fh, ©->stateid); if (IS_ERR(filp)) { - nfserr = nfserr_offload_denied; + switch (PTR_ERR(filp)) { + case -EBADF: + nfserr = nfserr_wrong_type; + break; + default: + nfserr = nfserr_offload_denied; + } nfsd4_interssc_disconnect(copy->ss_mnt); goto do_callback; } @@ -1826,7 +1828,7 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (!nfs4_init_copy_state(nn, copy)) goto out_err; refcount_set(&async_copy->refcount, 1); - memcpy(©->cp_res.cb_stateid, ©->cp_stateid.stid, + memcpy(©->cp_res.cb_stateid, ©->cp_stateid.cs_stid, sizeof(copy->cp_res.cb_stateid)); dup_copy_fields(copy, async_copy); async_copy->copy_task = kthread_create(nfsd4_do_async_copy, @@ -1862,7 +1864,7 @@ find_async_copy(struct nfs4_client *clp, stateid_t *stateid) spin_lock(&clp->async_lock); list_for_each_entry(copy, &clp->async_copies, copies) { - if (memcmp(©->cp_stateid.stid, stateid, NFS4_STATEID_SIZE)) + if (memcmp(©->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE)) continue; refcount_inc(©->refcount); spin_unlock(&clp->async_lock); @@ -1916,7 +1918,7 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, cps = nfs4_alloc_init_cpntf_state(nn, stid); if (!cps) goto out; - memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.stid, sizeof(stateid_t)); + memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t)); memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t)); memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t)); @@ -2633,9 +2635,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) status = nfserr_minor_vers_mismatch; if (nfsd_minorversion(nn, args->minorversion, NFSD_TEST) <= 0) goto out; - status = nfserr_resource; - if (args->opcnt > NFSD_MAX_OPS_PER_COMPOUND) - goto out; status = nfs41_check_op_ordering(args); if (status) { @@ -2648,10 +2647,20 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) rqstp->rq_lease_breaker = (void **)&cstate->clp; - trace_nfsd_compound(rqstp, args->opcnt); + trace_nfsd_compound(rqstp, args->tag, args->taglen, args->client_opcnt); while (!status && resp->opcnt < args->opcnt) { op = &args->ops[resp->opcnt++]; + if (unlikely(resp->opcnt == NFSD_MAX_OPS_PER_COMPOUND)) { + /* If there are still more operations to process, + * stop here and report NFS4ERR_RESOURCE. */ + if (cstate->minorversion == 0 && + args->client_opcnt > resp->opcnt) { + op->status = nfserr_resource; + goto encode_op; + } + } + /* * The XDR decode routines may have pre-set op->status; * for example, if there is a miscellaneous XDR error @@ -2727,8 +2736,8 @@ encode_op: status = op->status; } - trace_nfsd_compound_status(args->opcnt, resp->opcnt, status, - nfsd4_op_name(op->opnum)); + trace_nfsd_compound_status(args->client_opcnt, resp->opcnt, + status, nfsd4_op_name(op->opnum)); nfsd4_cstate_clear_replay(cstate); nfsd4_increment_op_stats(op->opnum); @@ -2762,28 +2771,49 @@ out: #define op_encode_channel_attrs_maxsz (6 + 1 + 1) -static inline u32 nfsd4_only_status_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +/* + * The _rsize() helpers are invoked by the NFSv4 COMPOUND decoder, which + * is called before sunrpc sets rq_res.buflen. Thus we have to compute + * the maximum payload size here, based on transport limits and the size + * of the remaining space in the rq_pages array. + */ +static u32 nfsd4_max_payload(const struct svc_rqst *rqstp) +{ + u32 buflen; + + buflen = (rqstp->rq_page_end - rqstp->rq_next_page) * PAGE_SIZE; + buflen -= rqstp->rq_auth_slack; + buflen -= rqstp->rq_res.head[0].iov_len; + return min_t(u32, buflen, svc_max_payload(rqstp)); +} + +static u32 nfsd4_only_status_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size) * sizeof(__be32); } -static inline u32 nfsd4_status_stateid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_status_stateid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_stateid_maxsz)* sizeof(__be32); } -static inline u32 nfsd4_access_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_access_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { /* ac_supported, ac_resp_access */ return (op_encode_hdr_size + 2)* sizeof(__be32); } -static inline u32 nfsd4_commit_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_commit_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_verifier_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_create_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_create_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz + nfs4_fattr_bitmap_maxsz) * sizeof(__be32); @@ -2794,17 +2824,17 @@ static inline u32 nfsd4_create_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op * the op prematurely if the estimate is too large. We may turn off splice * reads unnecessarily. */ -static inline u32 nfsd4_getattr_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_getattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 *bmap = op->u.getattr.ga_bmval; + const u32 *bmap = op->u.getattr.ga_bmval; u32 bmap0 = bmap[0], bmap1 = bmap[1], bmap2 = bmap[2]; u32 ret = 0; if (bmap0 & FATTR4_WORD0_ACL) - return svc_max_payload(rqstp); + return nfsd4_max_payload(rqstp); if (bmap0 & FATTR4_WORD0_FS_LOCATIONS) - return svc_max_payload(rqstp); + return nfsd4_max_payload(rqstp); if (bmap1 & FATTR4_WORD1_OWNER) { ret += IDMAP_NAMESZ + 4; @@ -2832,24 +2862,28 @@ static inline u32 nfsd4_getattr_rsize(struct svc_rqst *rqstp, return ret; } -static inline u32 nfsd4_getfh_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_getfh_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1) * sizeof(__be32) + NFS4_FHSIZE; } -static inline u32 nfsd4_link_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_link_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_lock_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_lock_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_lock_denied_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_open_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_open_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_stateid_maxsz + op_encode_change_info_maxsz + 1 @@ -2857,20 +2891,18 @@ static inline u32 nfsd4_open_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) + op_encode_delegation_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_read_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount = 0, rlen = 0; - - maxcount = svc_max_payload(rqstp); - rlen = min(op->u.read.rd_length, maxcount); + u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp)); return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32); } -static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_read_plus_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount = svc_max_payload(rqstp); - u32 rlen = min(op->u.read.rd_length, maxcount); + u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp)); /* * If we detect that the file changed during hole encoding, then we * recover by encoding the remaining reply as data. This means we need @@ -2881,70 +2913,77 @@ static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32); } -static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_readdir_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount = 0, rlen = 0; - - maxcount = svc_max_payload(rqstp); - rlen = min(op->u.readdir.rd_maxcount, maxcount); + u32 rlen = min(op->u.readdir.rd_maxcount, nfsd4_max_payload(rqstp)); return (op_encode_hdr_size + op_encode_verifier_maxsz + XDR_QUADLEN(rlen)) * sizeof(__be32); } -static inline u32 nfsd4_readlink_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_readlink_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1) * sizeof(__be32) + PAGE_SIZE; } -static inline u32 nfsd4_remove_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_remove_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_rename_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_rename_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz + op_encode_change_info_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_sequence_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_sequence_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5) * sizeof(__be32); } -static inline u32 nfsd4_test_stateid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_test_stateid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1 + op->u.test_stateid.ts_num_ids) * sizeof(__be32); } -static inline u32 nfsd4_setattr_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_setattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + nfs4_fattr_bitmap_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_secinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_secinfo_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + RPC_AUTH_MAXFLAVOR * (4 + XDR_QUADLEN(GSS_OID_MAX_LEN))) * sizeof(__be32); } -static inline u32 nfsd4_setclientid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_setclientid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 2 + XDR_QUADLEN(NFS4_VERIFIER_SIZE)) * sizeof(__be32); } -static inline u32 nfsd4_write_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_write_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 2 + op_encode_verifier_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_exchange_id_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_exchange_id_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 2 + 1 + /* eir_clientid, eir_sequenceid */\ 1 + 1 + /* eir_flags, spr_how */\ @@ -2958,14 +2997,16 @@ static inline u32 nfsd4_exchange_id_rsize(struct svc_rqst *rqstp, struct nfsd4_o 0 /* ignored eir_server_impl_id contents */) * sizeof(__be32); } -static inline u32 nfsd4_bind_conn_to_session_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_bind_conn_to_session_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + \ XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* bctsr_sessid */\ 2 /* bctsr_dir, use_conn_in_rdma_mode */) * sizeof(__be32); } -static inline u32 nfsd4_create_session_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_create_session_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + \ XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* sessionid */\ @@ -2974,7 +3015,8 @@ static inline u32 nfsd4_create_session_rsize(struct svc_rqst *rqstp, struct nfsd op_encode_channel_attrs_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_copy_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1 /* wr_callback */ + @@ -2986,16 +3028,16 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) 1 /* cr_synchronous */) * sizeof(__be32); } -static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_offload_status_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 2 /* osr_count */ + 1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32); } -static inline u32 nfsd4_copy_notify_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_copy_notify_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 3 /* cnr_lease_time */ + @@ -3010,12 +3052,10 @@ static inline u32 nfsd4_copy_notify_rsize(struct svc_rqst *rqstp, } #ifdef CONFIG_NFSD_PNFS -static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_getdeviceinfo_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount = 0, rlen = 0; - - maxcount = svc_max_payload(rqstp); - rlen = min(op->u.getdeviceinfo.gd_maxcount, maxcount); + u32 rlen = min(op->u.getdeviceinfo.gd_maxcount, nfsd4_max_payload(rqstp)); return (op_encode_hdr_size + 1 /* gd_layout_type*/ + @@ -3028,7 +3068,8 @@ static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4 * so we need to define an arbitrary upper bound here. */ #define MAX_LAYOUT_SIZE 128 -static inline u32 nfsd4_layoutget_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_layoutget_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1 /* logr_return_on_close */ + @@ -3037,14 +3078,16 @@ static inline u32 nfsd4_layoutget_rsize(struct svc_rqst *rqstp, struct nfsd4_op MAX_LAYOUT_SIZE) * sizeof(__be32); } -static inline u32 nfsd4_layoutcommit_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_layoutcommit_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1 /* locr_newsize */ + 2 /* ns_size */) * sizeof(__be32); } -static inline u32 nfsd4_layoutreturn_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_layoutreturn_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 1 /* lrs_stateid */ + @@ -3053,41 +3096,36 @@ static inline u32 nfsd4_layoutreturn_rsize(struct svc_rqst *rqstp, struct nfsd4_ #endif /* CONFIG_NFSD_PNFS */ -static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op) +static u32 nfsd4_seek_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + 3) * sizeof(__be32); } -static inline u32 nfsd4_getxattr_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_getxattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount, rlen; - - maxcount = svc_max_payload(rqstp); - rlen = min_t(u32, XATTR_SIZE_MAX, maxcount); + u32 rlen = min_t(u32, XATTR_SIZE_MAX, nfsd4_max_payload(rqstp)); return (op_encode_hdr_size + 1 + XDR_QUADLEN(rlen)) * sizeof(__be32); } -static inline u32 nfsd4_setxattr_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_setxattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz) * sizeof(__be32); } -static inline u32 nfsd4_listxattrs_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_listxattrs_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { - u32 maxcount, rlen; - - maxcount = svc_max_payload(rqstp); - rlen = min(op->u.listxattrs.lsxa_maxcount, maxcount); + u32 rlen = min(op->u.listxattrs.lsxa_maxcount, nfsd4_max_payload(rqstp)); return (op_encode_hdr_size + 4 + XDR_QUADLEN(rlen)) * sizeof(__be32); } -static inline u32 nfsd4_removexattr_rsize(struct svc_rqst *rqstp, - struct nfsd4_op *op) +static u32 nfsd4_removexattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) { return (op_encode_hdr_size + op_encode_change_info_maxsz) * sizeof(__be32); @@ -3576,6 +3614,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 1, @@ -3586,6 +3625,7 @@ static const struct svc_procedure nfsd_procedures4[2] = { .pc_decode = nfs4svc_decode_compoundargs, .pc_encode = nfs4svc_encode_compoundres, .pc_argsize = sizeof(struct nfsd4_compoundargs), + .pc_argzero = offsetof(struct nfsd4_compoundargs, iops), .pc_ressize = sizeof(struct nfsd4_compoundres), .pc_release = nfsd4_release_compoundargs, .pc_cachetype = RC_NOCACHE, diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index c634483d85d2a312db0b69b61294d35bab845d9d..78b8cd9651d5b929cd1352f55ee652c577f61960 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -266,7 +266,7 @@ struct nfs4_dir_ctx { struct list_head names; }; -static int +static bool nfsd4_build_namelist(struct dir_context *__ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { @@ -275,14 +275,14 @@ nfsd4_build_namelist(struct dir_context *__ctx, const char *name, int namlen, struct name_list *entry; if (namlen != HEXDIR_LEN - 1) - return 0; + return true; entry = kmalloc(sizeof(struct name_list), GFP_KERNEL); if (entry == NULL) - return -ENOMEM; + return false; memcpy(entry->name, name, HEXDIR_LEN - 1); entry->name[HEXDIR_LEN - 1] = '\0'; list_add(&entry->list, &ctx->names); - return 0; + return true; } static int @@ -807,16 +807,18 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, if (get_user(namelen, &ci->cc_name.cn_len)) return -EFAULT; name.data = memdup_user(&ci->cc_name.cn_id, namelen); - if (IS_ERR_OR_NULL(name.data)) - return -EFAULT; + if (IS_ERR(name.data)) + return PTR_ERR(name.data); name.len = namelen; get_user(princhashlen, &ci->cc_princhash.cp_len); if (princhashlen > 0) { princhash.data = memdup_user( &ci->cc_princhash.cp_data, princhashlen); - if (IS_ERR_OR_NULL(princhash.data)) - return -EFAULT; + if (IS_ERR(princhash.data)) { + kfree(name.data); + return PTR_ERR(princhash.data); + } princhash.len = princhashlen; } else princhash.len = 0; @@ -827,8 +829,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, if (get_user(namelen, &cnm->cn_len)) return -EFAULT; name.data = memdup_user(&cnm->cn_id, namelen); - if (IS_ERR_OR_NULL(name.data)) - return -EFAULT; + if (IS_ERR(name.data)) + return PTR_ERR(name.data); name.len = namelen; } if (name.len > 5 && memcmp(name.data, "hash:", 5) == 0) { diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index c5d199d7e6b4e2ec8624deab8a8d9ab24519bbeb..4e718500a00c4a1436685abc22760385fbc8e8db 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -160,6 +160,13 @@ static bool is_client_expired(struct nfs4_client *clp) return clp->cl_time == 0; } +static void nfsd4_dec_courtesy_client_count(struct nfsd_net *nn, + struct nfs4_client *clp) +{ + if (clp->cl_state != NFSD4_ACTIVE) + atomic_add_unless(&nn->nfsd_courtesy_clients, -1, 0); +} + static __be32 get_client_locked(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); @@ -169,6 +176,7 @@ static __be32 get_client_locked(struct nfs4_client *clp) if (is_client_expired(clp)) return nfserr_expired; atomic_inc(&clp->cl_rpc_users); + nfsd4_dec_courtesy_client_count(nn, clp); clp->cl_state = NFSD4_ACTIVE; return nfs_ok; } @@ -190,6 +198,7 @@ renew_client_locked(struct nfs4_client *clp) list_move_tail(&clp->cl_lru, &nn->client_lru); clp->cl_time = ktime_get_boottime_seconds(); + nfsd4_dec_courtesy_client_count(nn, clp); clp->cl_state = NFSD4_ACTIVE; } @@ -357,6 +366,8 @@ nfsd4_cb_notify_lock_prepare(struct nfsd4_callback *cb) static int nfsd4_cb_notify_lock_done(struct nfsd4_callback *cb, struct rpc_task *task) { + trace_nfsd_cb_notify_lock_done(&zero_stateid, task); + /* * Since this is just an optimization, we don't try very hard if it * turns out not to succeed. We'll requeue it on NFS4ERR_DELAY, and @@ -963,19 +974,19 @@ out_free: * Create a unique stateid_t to represent each COPY. */ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, - unsigned char sc_type) + unsigned char cs_type) { int new_id; - stid->stid.si_opaque.so_clid.cl_boot = (u32)nn->boot_time; - stid->stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id; - stid->sc_type = sc_type; + stid->cs_stid.si_opaque.so_clid.cl_boot = (u32)nn->boot_time; + stid->cs_stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id; + stid->cs_type = cs_type; idr_preload(GFP_KERNEL); spin_lock(&nn->s2s_cp_lock); new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT); - stid->stid.si_opaque.so_id = new_id; - stid->stid.si_generation = 1; + stid->cs_stid.si_opaque.so_id = new_id; + stid->cs_stid.si_generation = 1; spin_unlock(&nn->s2s_cp_lock); idr_preload_end(); if (new_id < 0) @@ -997,7 +1008,7 @@ struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, if (!cps) return NULL; cps->cpntf_time = ktime_get_boottime_seconds(); - refcount_set(&cps->cp_stateid.sc_count, 1); + refcount_set(&cps->cp_stateid.cs_count, 1); if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID)) goto out_free; spin_lock(&nn->s2s_cp_lock); @@ -1013,11 +1024,11 @@ void nfs4_free_copy_state(struct nfsd4_copy *copy) { struct nfsd_net *nn; - WARN_ON_ONCE(copy->cp_stateid.sc_type != NFS4_COPY_STID); + WARN_ON_ONCE(copy->cp_stateid.cs_type != NFS4_COPY_STID); nn = net_generic(copy->cp_clp->net, nfsd_net_id); spin_lock(&nn->s2s_cp_lock); idr_remove(&nn->s2s_cp_stateids, - copy->cp_stateid.stid.si_opaque.so_id); + copy->cp_stateid.cs_stid.si_opaque.so_id); spin_unlock(&nn->s2s_cp_lock); } @@ -1049,6 +1060,12 @@ static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp) static void nfs4_free_deleg(struct nfs4_stid *stid) { + struct nfs4_delegation *dp = delegstateid(stid); + + WARN_ON_ONCE(!list_empty(&stid->sc_cp_list)); + WARN_ON_ONCE(!list_empty(&dp->dl_perfile)); + WARN_ON_ONCE(!list_empty(&dp->dl_perclnt)); + WARN_ON_ONCE(!list_empty(&dp->dl_recall_lru)); kmem_cache_free(deleg_slab, stid); atomic_long_dec(&num_delegations); } @@ -1462,6 +1479,7 @@ static void nfs4_free_ol_stateid(struct nfs4_stid *stid) release_all_access(stp); if (stp->st_stateowner) nfs4_put_stateowner(stp->st_stateowner); + WARN_ON(!list_empty(&stid->sc_cp_list)); kmem_cache_free(stateid_slab, stid); } @@ -2233,6 +2251,7 @@ __destroy_client(struct nfs4_client *clp) if (clp->cl_cb_conn.cb_xprt) svc_xprt_put(clp->cl_cb_conn.cb_xprt); atomic_add_unless(&nn->nfs4_client_count, -1, 0); + nfsd4_dec_courtesy_client_count(nn, clp); free_client(clp); wake_up_all(&expiry_wq); } @@ -2478,7 +2497,7 @@ static const char *cb_state2str(int state) static int client_info_show(struct seq_file *m, void *v) { - struct inode *inode = m->private; + struct inode *inode = file_inode(m->file); struct nfs4_client *clp; u64 clid; @@ -2518,17 +2537,7 @@ static int client_info_show(struct seq_file *m, void *v) return 0; } -static int client_info_open(struct inode *inode, struct file *file) -{ - return single_open(file, client_info_show, inode); -} - -static const struct file_operations client_info_fops = { - .open = client_info_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(client_info); static void *states_start(struct seq_file *s, loff_t *pos) __acquires(&clp->cl_lock) @@ -4337,7 +4346,27 @@ out: return -ENOMEM; } -void nfsd4_init_leases_net(struct nfsd_net *nn) +static unsigned long +nfsd_courtesy_client_count(struct shrinker *shrink, struct shrink_control *sc) +{ + int cnt; + struct nfsd_net *nn = container_of(shrink, + struct nfsd_net, nfsd_client_shrinker); + + cnt = atomic_read(&nn->nfsd_courtesy_clients); + if (cnt > 0) + mod_delayed_work(laundry_wq, &nn->nfsd_shrinker_work, 0); + return (unsigned long)cnt; +} + +static unsigned long +nfsd_courtesy_client_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + return SHRINK_STOP; +} + +int +nfsd4_init_leases_net(struct nfsd_net *nn) { struct sysinfo si; u64 max_clients; @@ -4346,8 +4375,8 @@ void nfsd4_init_leases_net(struct nfsd_net *nn) nn->nfsd4_grace = 90; nn->somebody_reclaimed = false; nn->track_reclaim_completes = false; - nn->clverifier_counter = prandom_u32(); - nn->clientid_base = prandom_u32(); + nn->clverifier_counter = get_random_u32(); + nn->clientid_base = get_random_u32(); nn->clientid_counter = nn->clientid_base + 1; nn->s2s_cp_cl_id = nn->clientid_counter++; @@ -4356,6 +4385,18 @@ void nfsd4_init_leases_net(struct nfsd_net *nn) max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024); max_clients *= NFS4_CLIENTS_PER_GB; nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB); + + atomic_set(&nn->nfsd_courtesy_clients, 0); + nn->nfsd_client_shrinker.scan_objects = nfsd_courtesy_client_scan; + nn->nfsd_client_shrinker.count_objects = nfsd_courtesy_client_count; + nn->nfsd_client_shrinker.seeks = DEFAULT_SEEKS; + return register_shrinker(&nn->nfsd_client_shrinker, "nfsd-client"); +} + +void +nfsd4_leases_net_shutdown(struct nfsd_net *nn) +{ + unregister_shrinker(&nn->nfsd_client_shrinker); } static void init_nfs4_replay(struct nfs4_replay *rp) @@ -4715,6 +4756,35 @@ nfs4_share_conflict(struct svc_fh *current_fh, unsigned int deny_type) return ret; } +static bool nfsd4_deleg_present(const struct inode *inode) +{ + struct file_lock_context *ctx = smp_load_acquire(&inode->i_flctx); + + return ctx && !list_empty_careful(&ctx->flc_lease); +} + +/** + * nfsd_wait_for_delegreturn - wait for delegations to be returned + * @rqstp: the RPC transaction being executed + * @inode: in-core inode of the file being waited for + * + * The timeout prevents deadlock if all nfsd threads happen to be + * tied up waiting for returning delegations. + * + * Return values: + * %true: delegation was returned + * %false: timed out waiting for delegreturn + */ +bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode) +{ + long __maybe_unused timeo; + + timeo = wait_var_event_timeout(inode, !nfsd4_deleg_present(inode), + NFSD_DELEGRETURN_TIMEOUT); + trace_nfsd_delegret_wakeup(rqstp, inode, timeo); + return timeo > 0; +} + static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) { struct nfs4_delegation *dp = cb_to_delegation(cb); @@ -4743,6 +4813,8 @@ static int nfsd4_cb_recall_done(struct nfsd4_callback *cb, { struct nfs4_delegation *dp = cb_to_delegation(cb); + trace_nfsd_cb_recall_done(&dp->dl_stid.sc_stateid, task); + if (dp->dl_stid.sc_type == NFS4_CLOSED_DELEG_STID || dp->dl_stid.sc_type == NFS4_REVOKED_DELEG_STID) return 1; @@ -4788,18 +4860,17 @@ static void nfsd_break_one_deleg(struct nfs4_delegation *dp) * We're assuming the state code never drops its reference * without first removing the lease. Since we're in this lease * callback (and since the lease code is serialized by the - * i_lock) we know the server hasn't removed the lease yet, and + * flc_lock) we know the server hasn't removed the lease yet, and * we know it's safe to take a reference. */ refcount_inc(&dp->dl_stid.sc_count); - nfsd4_run_cb(&dp->dl_recall); + WARN_ON_ONCE(!nfsd4_run_cb(&dp->dl_recall)); } -/* Called from break_lease() with i_lock held. */ +/* Called from break_lease() with flc_lock held. */ static bool nfsd_break_deleg_cb(struct file_lock *fl) { - bool ret = false; struct nfs4_delegation *dp = (struct nfs4_delegation *)fl->fl_owner; struct nfs4_file *fp = dp->dl_stid.sc_file; struct nfs4_client *clp = dp->dl_stid.sc_client; @@ -4825,7 +4896,7 @@ nfsd_break_deleg_cb(struct file_lock *fl) fp->fi_had_conflict = true; nfsd_break_one_deleg(dp); spin_unlock(&fp->fi_lock); - return ret; + return false; } /** @@ -5878,8 +5949,11 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist, goto exp_client; if (!state_expired(lt, clp->cl_time)) break; - if (!atomic_read(&clp->cl_rpc_users)) + if (!atomic_read(&clp->cl_rpc_users)) { + if (clp->cl_state == NFSD4_ACTIVE) + atomic_inc(&nn->nfsd_courtesy_clients); clp->cl_state = NFSD4_COURTESY; + } if (!client_has_state(clp)) goto exp_client; if (!nfs4_anylock_blockers(clp)) @@ -5894,10 +5968,49 @@ exp_client: spin_unlock(&nn->client_lock); } +static void +nfs4_get_courtesy_client_reaplist(struct nfsd_net *nn, + struct list_head *reaplist) +{ + unsigned int maxreap = 0, reapcnt = 0; + struct list_head *pos, *next; + struct nfs4_client *clp; + + maxreap = NFSD_CLIENT_MAX_TRIM_PER_RUN; + INIT_LIST_HEAD(reaplist); + + spin_lock(&nn->client_lock); + list_for_each_safe(pos, next, &nn->client_lru) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + if (clp->cl_state == NFSD4_ACTIVE) + break; + if (reapcnt >= maxreap) + break; + if (!mark_client_expired_locked(clp)) { + list_add(&clp->cl_lru, reaplist); + reapcnt++; + } + } + spin_unlock(&nn->client_lock); +} + +static void +nfs4_process_client_reaplist(struct list_head *reaplist) +{ + struct list_head *pos, *next; + struct nfs4_client *clp; + + list_for_each_safe(pos, next, reaplist) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + trace_nfsd_clid_purged(&clp->cl_clientid); + list_del_init(&clp->cl_lru); + expire_client(clp); + } +} + static time64_t nfs4_laundromat(struct nfsd_net *nn) { - struct nfs4_client *clp; struct nfs4_openowner *oo; struct nfs4_delegation *dp; struct nfs4_ol_stateid *stp; @@ -5920,18 +6033,14 @@ nfs4_laundromat(struct nfsd_net *nn) spin_lock(&nn->s2s_cp_lock); idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) { cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); - if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID && + if (cps->cp_stateid.cs_type == NFS4_COPYNOTIFY_STID && state_expired(<, cps->cpntf_time)) _free_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); nfs4_get_client_reaplist(nn, &reaplist, <); - list_for_each_safe(pos, next, &reaplist) { - clp = list_entry(pos, struct nfs4_client, cl_lru); - trace_nfsd_clid_purged(&clp->cl_clientid); - list_del_init(&clp->cl_lru); - expire_client(clp); - } + nfs4_process_client_reaplist(&reaplist); + spin_lock(&state_lock); list_for_each_safe(pos, next, &nn->del_recall_lru) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); @@ -6014,6 +6123,18 @@ laundromat_main(struct work_struct *laundry) queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ); } +static void +courtesy_client_reaper(struct work_struct *reaper) +{ + struct list_head reaplist; + struct delayed_work *dwork = to_delayed_work(reaper); + struct nfsd_net *nn = container_of(dwork, struct nfsd_net, + nfsd_shrinker_work); + + nfs4_get_courtesy_client_reaplist(nn, &reaplist); + nfs4_process_client_reaplist(&reaplist); +} + static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp) { if (!fh_match(&fhp->fh_handle, &stp->sc_file->fi_fhandle)) @@ -6149,6 +6270,7 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, struct nfs4_stid **s, struct nfsd_net *nn) { __be32 status; + struct nfs4_stid *stid; bool return_revoked = false; /* @@ -6171,15 +6293,16 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, } if (status) return status; - *s = find_stateid_by_type(cstate->clp, stateid, typemask); - if (!*s) + stid = find_stateid_by_type(cstate->clp, stateid, typemask); + if (!stid) return nfserr_bad_stateid; - if (((*s)->sc_type == NFS4_REVOKED_DELEG_STID) && !return_revoked) { - nfs4_put_stid(*s); + if ((stid->sc_type == NFS4_REVOKED_DELEG_STID) && !return_revoked) { + nfs4_put_stid(stid); if (cstate->minorversion) return nfserr_deleg_revoked; return nfserr_bad_stateid; } + *s = stid; return nfs_ok; } @@ -6244,12 +6367,12 @@ out: static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) { - WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID); - if (!refcount_dec_and_test(&cps->cp_stateid.sc_count)) + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + if (!refcount_dec_and_test(&cps->cp_stateid.cs_count)) return; list_del(&cps->cp_list); idr_remove(&nn->s2s_cp_stateids, - cps->cp_stateid.stid.si_opaque.so_id); + cps->cp_stateid.cs_stid.si_opaque.so_id); kfree(cps); } /* @@ -6271,12 +6394,12 @@ __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, if (cps_t) { state = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); - if (state->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID) { + if (state->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID) { state = NULL; goto unlock; } if (!clp) - refcount_inc(&state->cp_stateid.sc_count); + refcount_inc(&state->cp_stateid.cs_count); else _free_cpntf_state_locked(nn, state); } @@ -6684,6 +6807,7 @@ static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) struct nfs4_client *clp = s->st_stid.sc_client; bool unhashed; LIST_HEAD(reaplist); + struct nfs4_ol_stateid *stp; spin_lock(&clp->cl_lock); unhashed = unhash_open_stateid(s, &reaplist); @@ -6692,6 +6816,8 @@ static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) if (unhashed) put_ol_stateid_locked(s, &reaplist); spin_unlock(&clp->cl_lock); + list_for_each_entry(stp, &reaplist, st_locks) + nfs4_free_cpntf_statelist(clp->net, &stp->st_stid); free_ol_stateid_reaplist(&reaplist); } else { spin_unlock(&clp->cl_lock); @@ -6775,6 +6901,7 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto put_stateid; + wake_up_var(d_inode(cstate->current_fh.fh_dentry)); destroy_delegation(dp); put_stateid: nfs4_put_stid(&dp->dl_stid); @@ -7830,6 +7957,7 @@ static int nfs4_state_create_net(struct net *net) INIT_LIST_HEAD(&nn->blocked_locks_lru); INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main); + INIT_DELAYED_WORK(&nn->nfsd_shrinker_work, courtesy_client_reaper); get_net(net); return 0; diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 1e9690a061eca27a0be7f344cc39df91447a6355..bcfeb1a922c0d334989ac2cf312d1567ad73adc7 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -42,6 +42,8 @@ #include #include #include +#include + #include #include "idmap.h" @@ -791,6 +793,7 @@ nfsd4_decode_commit(struct nfsd4_compoundargs *argp, struct nfsd4_commit *commit return nfserr_bad_xdr; if (xdr_stream_decode_u32(argp->xdr, &commit->co_count) < 0) return nfserr_bad_xdr; + memset(&commit->co_verf, 0, sizeof(commit->co_verf)); return nfs_ok; } @@ -799,6 +802,7 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, struct nfsd4_create *create { __be32 *p, status; + memset(create, 0, sizeof(*create)); if (xdr_stream_decode_u32(argp->xdr, &create->cr_type) < 0) return nfserr_bad_xdr; switch (create->cr_type) { @@ -848,6 +852,7 @@ nfsd4_decode_delegreturn(struct nfsd4_compoundargs *argp, struct nfsd4_delegretu static inline __be32 nfsd4_decode_getattr(struct nfsd4_compoundargs *argp, struct nfsd4_getattr *getattr) { + memset(getattr, 0, sizeof(*getattr)); return nfsd4_decode_bitmap4(argp, getattr->ga_bmval, ARRAY_SIZE(getattr->ga_bmval)); } @@ -855,6 +860,7 @@ nfsd4_decode_getattr(struct nfsd4_compoundargs *argp, struct nfsd4_getattr *geta static __be32 nfsd4_decode_link(struct nfsd4_compoundargs *argp, struct nfsd4_link *link) { + memset(link, 0, sizeof(*link)); return nfsd4_decode_component4(argp, &link->li_name, &link->li_namelen); } @@ -903,6 +909,7 @@ nfsd4_decode_locker4(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock) static __be32 nfsd4_decode_lock(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock) { + memset(lock, 0, sizeof(*lock)); if (xdr_stream_decode_u32(argp->xdr, &lock->lk_type) < 0) return nfserr_bad_xdr; if ((lock->lk_type < NFS4_READ_LT) || (lock->lk_type > NFS4_WRITEW_LT)) @@ -919,6 +926,7 @@ nfsd4_decode_lock(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock) static __be32 nfsd4_decode_lockt(struct nfsd4_compoundargs *argp, struct nfsd4_lockt *lockt) { + memset(lockt, 0, sizeof(*lockt)); if (xdr_stream_decode_u32(argp->xdr, &lockt->lt_type) < 0) return nfserr_bad_xdr; if ((lockt->lt_type < NFS4_READ_LT) || (lockt->lt_type > NFS4_WRITEW_LT)) @@ -1140,11 +1148,8 @@ nfsd4_decode_open(struct nfsd4_compoundargs *argp, struct nfsd4_open *open) __be32 status; u32 dummy; - memset(open->op_bmval, 0, sizeof(open->op_bmval)); - open->op_iattr.ia_valid = 0; - open->op_openowner = NULL; + memset(open, 0, sizeof(*open)); - open->op_xdr_error = 0; if (xdr_stream_decode_u32(argp->xdr, &open->op_seqid) < 0) return nfserr_bad_xdr; /* deleg_want is ignored */ @@ -1179,6 +1184,8 @@ nfsd4_decode_open_confirm(struct nfsd4_compoundargs *argp, struct nfsd4_open_con if (xdr_stream_decode_u32(argp->xdr, &open_conf->oc_seqid) < 0) return nfserr_bad_xdr; + memset(&open_conf->oc_resp_stateid, 0, + sizeof(open_conf->oc_resp_stateid)); return nfs_ok; } @@ -1187,6 +1194,7 @@ nfsd4_decode_open_downgrade(struct nfsd4_compoundargs *argp, struct nfsd4_open_d { __be32 status; + memset(open_down, 0, sizeof(*open_down)); status = nfsd4_decode_stateid4(argp, &open_down->od_stateid); if (status) return status; @@ -1216,6 +1224,7 @@ nfsd4_decode_putfh(struct nfsd4_compoundargs *argp, struct nfsd4_putfh *putfh) if (!putfh->pf_fhval) return nfserr_jukebox; + putfh->no_verify = false; return nfs_ok; } @@ -1232,6 +1241,7 @@ nfsd4_decode_read(struct nfsd4_compoundargs *argp, struct nfsd4_read *read) { __be32 status; + memset(read, 0, sizeof(*read)); status = nfsd4_decode_stateid4(argp, &read->rd_stateid); if (status) return status; @@ -1248,6 +1258,7 @@ nfsd4_decode_readdir(struct nfsd4_compoundargs *argp, struct nfsd4_readdir *read { __be32 status; + memset(readdir, 0, sizeof(*readdir)); if (xdr_stream_decode_u64(argp->xdr, &readdir->rd_cookie) < 0) return nfserr_bad_xdr; status = nfsd4_decode_verifier4(argp, &readdir->rd_verf); @@ -1267,6 +1278,7 @@ nfsd4_decode_readdir(struct nfsd4_compoundargs *argp, struct nfsd4_readdir *read static __be32 nfsd4_decode_remove(struct nfsd4_compoundargs *argp, struct nfsd4_remove *remove) { + memset(&remove->rm_cinfo, 0, sizeof(remove->rm_cinfo)); return nfsd4_decode_component4(argp, &remove->rm_name, &remove->rm_namelen); } @@ -1275,6 +1287,7 @@ nfsd4_decode_rename(struct nfsd4_compoundargs *argp, struct nfsd4_rename *rename { __be32 status; + memset(rename, 0, sizeof(*rename)); status = nfsd4_decode_component4(argp, &rename->rn_sname, &rename->rn_snamelen); if (status) return status; @@ -1291,6 +1304,7 @@ static __be32 nfsd4_decode_secinfo(struct nfsd4_compoundargs *argp, struct nfsd4_secinfo *secinfo) { + secinfo->si_exp = NULL; return nfsd4_decode_component4(argp, &secinfo->si_name, &secinfo->si_namelen); } @@ -1299,6 +1313,7 @@ nfsd4_decode_setattr(struct nfsd4_compoundargs *argp, struct nfsd4_setattr *seta { __be32 status; + memset(setattr, 0, sizeof(*setattr)); status = nfsd4_decode_stateid4(argp, &setattr->sa_stateid); if (status) return status; @@ -1313,6 +1328,8 @@ nfsd4_decode_setclientid(struct nfsd4_compoundargs *argp, struct nfsd4_setclient { __be32 *p, status; + memset(setclientid, 0, sizeof(*setclientid)); + if (argp->minorversion >= 1) return nfserr_notsupp; @@ -1369,6 +1386,8 @@ nfsd4_decode_verify(struct nfsd4_compoundargs *argp, struct nfsd4_verify *verify { __be32 *p, status; + memset(verify, 0, sizeof(*verify)); + status = nfsd4_decode_bitmap4(argp, verify->ve_bmval, ARRAY_SIZE(verify->ve_bmval)); if (status) @@ -1408,6 +1427,9 @@ nfsd4_decode_write(struct nfsd4_compoundargs *argp, struct nfsd4_write *write) if (!xdr_stream_subsegment(argp->xdr, &write->wr_payload, write->wr_buflen)) return nfserr_bad_xdr; + write->wr_bytes_written = 0; + write->wr_how_written = 0; + memset(&write->wr_verifier, 0, sizeof(write->wr_verifier)); return nfs_ok; } @@ -1432,6 +1454,7 @@ nfsd4_decode_release_lockowner(struct nfsd4_compoundargs *argp, struct nfsd4_rel static __be32 nfsd4_decode_backchannel_ctl(struct nfsd4_compoundargs *argp, struct nfsd4_backchannel_ctl *bc) { + memset(bc, 0, sizeof(*bc)); if (xdr_stream_decode_u32(argp->xdr, &bc->bc_cb_program) < 0) return nfserr_bad_xdr; return nfsd4_decode_cb_sec(argp, &bc->bc_cb_sec); @@ -1442,6 +1465,7 @@ static __be32 nfsd4_decode_bind_conn_to_session(struct nfsd4_compoundargs *argp, u32 use_conn_in_rdma_mode; __be32 status; + memset(bcts, 0, sizeof(*bcts)); status = nfsd4_decode_sessionid4(argp, &bcts->sessionid); if (status) return status; @@ -1583,6 +1607,7 @@ nfsd4_decode_exchange_id(struct nfsd4_compoundargs *argp, { __be32 status; + memset(exid, 0, sizeof(*exid)); status = nfsd4_decode_verifier4(argp, &exid->verifier); if (status) return status; @@ -1635,6 +1660,7 @@ nfsd4_decode_create_session(struct nfsd4_compoundargs *argp, { __be32 status; + memset(sess, 0, sizeof(*sess)); status = nfsd4_decode_clientid4(argp, &sess->clientid); if (status) return status; @@ -1650,11 +1676,7 @@ nfsd4_decode_create_session(struct nfsd4_compoundargs *argp, return status; if (xdr_stream_decode_u32(argp->xdr, &sess->callback_prog) < 0) return nfserr_bad_xdr; - status = nfsd4_decode_cb_sec(argp, &sess->cb_sec); - if (status) - return status; - - return nfs_ok; + return nfsd4_decode_cb_sec(argp, &sess->cb_sec); } static __be32 @@ -1678,6 +1700,7 @@ nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp, { __be32 status; + memset(gdev, 0, sizeof(*gdev)); status = nfsd4_decode_deviceid4(argp, &gdev->gd_devid); if (status) return status; @@ -1698,6 +1721,7 @@ nfsd4_decode_layoutcommit(struct nfsd4_compoundargs *argp, { __be32 *p, status; + memset(lcp, 0, sizeof(*lcp)); if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.offset) < 0) return nfserr_bad_xdr; if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.length) < 0) @@ -1733,6 +1757,7 @@ nfsd4_decode_layoutget(struct nfsd4_compoundargs *argp, { __be32 status; + memset(lgp, 0, sizeof(*lgp)); if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_signal) < 0) return nfserr_bad_xdr; if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_layout_type) < 0) @@ -1758,6 +1783,7 @@ static __be32 nfsd4_decode_layoutreturn(struct nfsd4_compoundargs *argp, struct nfsd4_layoutreturn *lrp) { + memset(lrp, 0, sizeof(*lrp)); if (xdr_stream_decode_bool(argp->xdr, &lrp->lr_reclaim) < 0) return nfserr_bad_xdr; if (xdr_stream_decode_u32(argp->xdr, &lrp->lr_layout_type) < 0) @@ -1773,6 +1799,8 @@ static __be32 nfsd4_decode_secinfo_no_name(struct nfsd4_compoundargs *argp, { if (xdr_stream_decode_u32(argp->xdr, &sin->sin_style) < 0) return nfserr_bad_xdr; + + sin->sin_exp = NULL; return nfs_ok; } @@ -1793,6 +1821,7 @@ nfsd4_decode_sequence(struct nfsd4_compoundargs *argp, seq->maxslots = be32_to_cpup(p++); seq->cachethis = be32_to_cpup(p); + seq->status_flags = 0; return nfs_ok; } @@ -1803,6 +1832,7 @@ nfsd4_decode_test_stateid(struct nfsd4_compoundargs *argp, struct nfsd4_test_sta __be32 status; u32 i; + memset(test_stateid, 0, sizeof(*test_stateid)); if (xdr_stream_decode_u32(argp->xdr, &test_stateid->ts_num_ids) < 0) return nfserr_bad_xdr; @@ -1900,6 +1930,7 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy) struct nl4_server *ns_dummy; __be32 status; + memset(copy, 0, sizeof(*copy)); status = nfsd4_decode_stateid4(argp, ©->cp_src_stateid); if (status) return status; @@ -1955,6 +1986,7 @@ nfsd4_decode_copy_notify(struct nfsd4_compoundargs *argp, { __be32 status; + memset(cn, 0, sizeof(*cn)); cn->cpn_src = svcxdr_tmpalloc(argp, sizeof(*cn->cpn_src)); if (cn->cpn_src == NULL) return nfserr_jukebox; @@ -1972,6 +2004,8 @@ static __be32 nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp, struct nfsd4_offload_status *os) { + os->count = 0; + os->status = 0; return nfsd4_decode_stateid4(argp, &os->stateid); } @@ -1988,6 +2022,8 @@ nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek) if (xdr_stream_decode_u32(argp->xdr, &seek->seek_whence) < 0) return nfserr_bad_xdr; + seek->seek_eof = 0; + seek->seek_pos = 0; return nfs_ok; } @@ -2123,6 +2159,7 @@ nfsd4_decode_getxattr(struct nfsd4_compoundargs *argp, __be32 status; u32 maxcount; + memset(getxattr, 0, sizeof(*getxattr)); status = nfsd4_decode_xattr_name(argp, &getxattr->getxa_name); if (status) return status; @@ -2131,8 +2168,7 @@ nfsd4_decode_getxattr(struct nfsd4_compoundargs *argp, maxcount = min_t(u32, XATTR_SIZE_MAX, maxcount); getxattr->getxa_len = maxcount; - - return status; + return nfs_ok; } static __be32 @@ -2142,6 +2178,8 @@ nfsd4_decode_setxattr(struct nfsd4_compoundargs *argp, u32 flags, maxcount, size; __be32 status; + memset(setxattr, 0, sizeof(*setxattr)); + if (xdr_stream_decode_u32(argp->xdr, &flags) < 0) return nfserr_bad_xdr; @@ -2180,6 +2218,8 @@ nfsd4_decode_listxattrs(struct nfsd4_compoundargs *argp, { u32 maxcount; + memset(listxattrs, 0, sizeof(*listxattrs)); + if (xdr_stream_decode_u64(argp->xdr, &listxattrs->lsxa_cookie) < 0) return nfserr_bad_xdr; @@ -2207,6 +2247,7 @@ static __be32 nfsd4_decode_removexattr(struct nfsd4_compoundargs *argp, struct nfsd4_removexattr *removexattr) { + memset(removexattr, 0, sizeof(*removexattr)); return nfsd4_decode_xattr_name(argp, &removexattr->rmxa_name); } @@ -2357,22 +2398,15 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp) if (xdr_stream_decode_u32(argp->xdr, &argp->minorversion) < 0) return false; - if (xdr_stream_decode_u32(argp->xdr, &argp->opcnt) < 0) + if (xdr_stream_decode_u32(argp->xdr, &argp->client_opcnt) < 0) return false; - - /* - * NFS4ERR_RESOURCE is a more helpful error than GARBAGE_ARGS - * here, so we return success at the xdr level so that - * nfsd4_proc can handle this is an NFS-level error. - */ - if (argp->opcnt > NFSD_MAX_OPS_PER_COMPOUND) - return true; + argp->opcnt = min_t(u32, argp->client_opcnt, + NFSD_MAX_OPS_PER_COMPOUND); if (argp->opcnt > ARRAY_SIZE(argp->iops)) { - argp->ops = kzalloc(argp->opcnt * sizeof(*argp->ops), GFP_KERNEL); + argp->ops = vcalloc(argp->opcnt, sizeof(*argp->ops)); if (!argp->ops) { argp->ops = argp->iops; - dprintk("nfsd: couldn't allocate room for COMPOUND\n"); return false; } } @@ -2774,9 +2808,10 @@ static __be32 fattr_handle_absent_fs(u32 *bmval0, u32 *bmval1, u32 *bmval2, u32 } -static int get_parent_attributes(struct svc_export *exp, struct kstat *stat) +static int nfsd4_get_mounted_on_ino(struct svc_export *exp, u64 *pino) { struct path path = exp->ex_path; + struct kstat stat; int err; path_get(&path); @@ -2784,8 +2819,10 @@ static int get_parent_attributes(struct svc_export *exp, struct kstat *stat) if (path.dentry != path.mnt->mnt_root) break; } - err = vfs_getattr(&path, stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); + err = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); path_put(&path); + if (!err) + *pino = stat.ino; return err; } @@ -3282,22 +3319,21 @@ out_acl: *p++ = cpu_to_be32(stat.btime.tv_nsec); } if (bmval1 & FATTR4_WORD1_MOUNTED_ON_FILEID) { - struct kstat parent_stat; u64 ino = stat.ino; p = xdr_reserve_space(xdr, 8); if (!p) goto out_resource; /* - * Get parent's attributes if not ignoring crossmount - * and this is the root of a cross-mounted filesystem. + * Get ino of mountpoint in parent filesystem, if not ignoring + * crossmount and this is the root of a cross-mounted + * filesystem. */ if (ignore_crossmnt == 0 && dentry == exp->ex_path.mnt->mnt_root) { - err = get_parent_attributes(exp, &parent_stat); + err = nfsd4_get_mounted_on_ino(exp, &ino); if (err) goto out_nfserr; - ino = parent_stat.ino; } p = xdr_encode_hyper(p, ino); } @@ -3994,7 +4030,7 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr, } if (resp->xdr->buf->page_len && splice_ok) { WARN_ON_ONCE(1); - return nfserr_resource; + return nfserr_serverfault; } xdr_commit_encode(xdr); @@ -5394,7 +5430,7 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp) struct nfsd4_compoundargs *args = rqstp->rq_argp; if (args->ops != args->iops) { - kfree(args->ops); + vfree(args->ops); args->ops = args->iops; } while (args->to_free) { @@ -5423,12 +5459,8 @@ bool nfs4svc_encode_compoundres(struct svc_rqst *rqstp, struct xdr_stream *xdr) { struct nfsd4_compoundres *resp = rqstp->rq_resp; - struct xdr_buf *buf = xdr->buf; __be32 *p; - WARN_ON_ONCE(buf->len != buf->head[0].iov_len + buf->page_len + - buf->tail[0].iov_len); - /* * Send buffer space for the following items is reserved * at the top of nfsd4_proc_compound(). diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 9b31e1103e7b1d2ae4ded2587b3cff73e79d5fac..3e64a3d50a1c526ab88850da5e460c8d18d8d86a 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -604,9 +604,10 @@ nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *data) * scraping this file for info should test the labels to ensure they're * getting the correct field. */ -static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v) +int nfsd_reply_cache_stats_show(struct seq_file *m, void *v) { - struct nfsd_net *nn = m->private; + struct nfsd_net *nn = net_generic(file_inode(m->file)->i_sb->s_fs_info, + nfsd_net_id); seq_printf(m, "max entries: %u\n", nn->max_drc_entries); seq_printf(m, "num entries: %u\n", @@ -626,11 +627,3 @@ static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v) seq_printf(m, "cachesize at longest: %u\n", nn->longest_chain_cachesize); return 0; } - -int nfsd_reply_cache_stats_open(struct inode *inode, struct file *file) -{ - struct nfsd_net *nn = net_generic(file_inode(file)->i_sb->s_fs_info, - nfsd_net_id); - - return single_open(file, nfsd_reply_cache_stats_show, nn); -} diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 917fa1892fd2d5c12faea564499cbcff2514e4ec..6a29bcfc93909dea11846dfd19bbb86fdb939dd1 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -185,17 +185,7 @@ static int export_features_show(struct seq_file *m, void *v) return 0; } -static int export_features_open(struct inode *inode, struct file *file) -{ - return single_open(file, export_features_show, NULL); -} - -static const struct file_operations export_features_operations = { - .open = export_features_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(export_features); #if defined(CONFIG_SUNRPC_GSS) || defined(CONFIG_SUNRPC_GSS_MODULE) static int supported_enctypes_show(struct seq_file *m, void *v) @@ -204,17 +194,7 @@ static int supported_enctypes_show(struct seq_file *m, void *v) return 0; } -static int supported_enctypes_open(struct inode *inode, struct file *file) -{ - return single_open(file, supported_enctypes_show, NULL); -} - -static const struct file_operations supported_enctypes_ops = { - .open = supported_enctypes_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(supported_enctypes); #endif /* CONFIG_SUNRPC_GSS or CONFIG_SUNRPC_GSS_MODULE */ static const struct file_operations pool_stats_operations = { @@ -224,19 +204,9 @@ static const struct file_operations pool_stats_operations = { .release = nfsd_pool_stats_release, }; -static const struct file_operations reply_cache_stats_operations = { - .open = nfsd_reply_cache_stats_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(nfsd_reply_cache_stats); -static const struct file_operations filecache_ops = { - .open = nfsd_file_cache_stats_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(nfsd_file_cache_stats); /*----------------------------------------------------------------------------*/ /* @@ -1365,7 +1335,7 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc) /* Per-export io stats use same ops as exports file */ [NFSD_Export_Stats] = {"export_stats", &exports_nfsd_operations, S_IRUGO}, [NFSD_Export_features] = {"export_features", - &export_features_operations, S_IRUGO}, + &export_features_fops, S_IRUGO}, [NFSD_FO_UnlockIP] = {"unlock_ip", &transaction_ops, S_IWUSR|S_IRUSR}, [NFSD_FO_UnlockFS] = {"unlock_filesystem", @@ -1374,14 +1344,16 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc) [NFSD_Threads] = {"threads", &transaction_ops, S_IWUSR|S_IRUSR}, [NFSD_Pool_Threads] = {"pool_threads", &transaction_ops, S_IWUSR|S_IRUSR}, [NFSD_Pool_Stats] = {"pool_stats", &pool_stats_operations, S_IRUGO}, - [NFSD_Reply_Cache_Stats] = {"reply_cache_stats", &reply_cache_stats_operations, S_IRUGO}, + [NFSD_Reply_Cache_Stats] = {"reply_cache_stats", + &nfsd_reply_cache_stats_fops, S_IRUGO}, [NFSD_Versions] = {"versions", &transaction_ops, S_IWUSR|S_IRUSR}, [NFSD_Ports] = {"portlist", &transaction_ops, S_IWUSR|S_IRUGO}, [NFSD_MaxBlkSize] = {"max_block_size", &transaction_ops, S_IWUSR|S_IRUGO}, [NFSD_MaxConnections] = {"max_connections", &transaction_ops, S_IWUSR|S_IRUGO}, - [NFSD_Filecache] = {"filecache", &filecache_ops, S_IRUGO}, + [NFSD_Filecache] = {"filecache", &nfsd_file_cache_stats_fops, S_IRUGO}, #if defined(CONFIG_SUNRPC_GSS) || defined(CONFIG_SUNRPC_GSS_MODULE) - [NFSD_SupportedEnctypes] = {"supported_krb5_enctypes", &supported_enctypes_ops, S_IRUGO}, + [NFSD_SupportedEnctypes] = {"supported_krb5_enctypes", + &supported_enctypes_fops, S_IRUGO}, #endif /* CONFIG_SUNRPC_GSS or CONFIG_SUNRPC_GSS_MODULE */ #ifdef CONFIG_NFSD_V4 [NFSD_Leasetime] = {"nfsv4leasetime", &transaction_ops, S_IWUSR|S_IRUSR}, @@ -1481,11 +1453,12 @@ static __net_init int nfsd_init_net(struct net *net) goto out_idmap_error; nn->nfsd_versions = NULL; nn->nfsd4_minorversions = NULL; + retval = nfsd4_init_leases_net(nn); + if (retval) + goto out_drc_error; retval = nfsd_reply_cache_init(nn); if (retval) goto out_drc_error; - nfsd4_init_leases_net(nn); - get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key)); seqlock_init(&nn->writeverf_lock); @@ -1507,6 +1480,7 @@ static __net_exit void nfsd_exit_net(struct net *net) nfsd_idmap_shutdown(net); nfsd_export_shutdown(net); nfsd_netns_free_versions(net_generic(net, nfsd_net_id)); + nfsd4_leases_net_shutdown(nn); } static struct pernet_operations nfsd_net_ops = { diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 57a468ed85c358b377af699a562c0b0fa6e21ed6..09726c5b9a317bbafe823c1c716e963d7a19f238 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -164,6 +164,7 @@ char * nfs4_recoverydir(void); bool nfsd4_spo_must_allow(struct svc_rqst *rqstp); int nfsd4_create_laundry_wq(void); void nfsd4_destroy_laundry_wq(void); +bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode); #else static inline int nfsd4_init_slabs(void) { return 0; } static inline void nfsd4_free_slabs(void) { } @@ -179,6 +180,11 @@ static inline bool nfsd4_spo_must_allow(struct svc_rqst *rqstp) } static inline int nfsd4_create_laundry_wq(void) { return 0; }; static inline void nfsd4_destroy_laundry_wq(void) {}; +static inline bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, + struct inode *inode) +{ + return false; +} #endif /* @@ -343,6 +349,7 @@ void nfsd_lockd_shutdown(void); #define NFSD_COURTESY_CLIENT_TIMEOUT (24 * 60 * 60) /* seconds */ #define NFSD_CLIENT_MAX_TRIM_PER_RUN 128 #define NFS4_CLIENTS_PER_GB 1024 +#define NFSD_DELEGRETURN_TIMEOUT (HZ / 34) /* 30ms */ /* * The following attributes are currently not supported by the NFSv4 server: @@ -498,7 +505,8 @@ extern void unregister_cld_notifier(void); extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn); #endif -extern void nfsd4_init_leases_net(struct nfsd_net *nn); +extern int nfsd4_init_leases_net(struct nfsd_net *nn); +extern void nfsd4_leases_net_shutdown(struct nfsd_net *nn); #else /* CONFIG_NFSD_V4 */ static inline int nfsd4_is_junction(struct dentry *dentry) @@ -506,7 +514,8 @@ static inline int nfsd4_is_junction(struct dentry *dentry) return 0; } -static inline void nfsd4_init_leases_net(struct nfsd_net *nn) {}; +static inline int nfsd4_init_leases_net(struct nfsd_net *nn) { return 0; }; +static inline void nfsd4_leases_net_shutdown(struct nfsd_net *nn) {}; #define register_cld_notifier() 0 #define unregister_cld_notifier() do { } while(0) diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index a5b71526cee0fa779b1302b1a61b9e8c1b8b65aa..d73434200df989572a95181e0bdeb3ae7b9f92c2 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -392,13 +392,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) skip_pseudoflavor_check: /* Finally, check access permissions. */ error = nfsd_permission(rqstp, exp, dentry, access); - - if (error) { - dprintk("fh_verify: %pd2 permission failure, " - "acc=%x, error=%d\n", - dentry, - access, ntohl(error)); - } + trace_nfsd_fh_verify_err(rqstp, fhp, type, access, error); out: if (error == nfserr_stale) nfsd_stats_fh_stale_inc(exp); diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 7381972f16774998a96a178243389c8e0927c97a..82b3ddeacc338a4ffcf85c3440c2f2990e40d002 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -185,6 +185,7 @@ nfsd_proc_read(struct svc_rqst *rqstp) argp->count, argp->offset); argp->count = min_t(u32, argp->count, NFSSVC_MAXBLKSIZE_V2); + argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen); v = 0; len = argp->count; @@ -390,9 +391,8 @@ nfsd_proc_create(struct svc_rqst *rqstp) resp->status = nfs_ok; if (!inode) { /* File doesn't exist. Create it and set attrs */ - resp->status = nfsd_create_locked(rqstp, dirfhp, argp->name, - argp->len, &attrs, type, rdev, - newfhp); + resp->status = nfsd_create_locked(rqstp, dirfhp, &attrs, type, + rdev, newfhp); } else if (type == S_IFREG) { dprintk("nfsd: existing %s, valid=%x, size=%ld\n", argp->name, attr->ia_valid, (long) attr->ia_size); @@ -567,24 +567,15 @@ static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp, struct xdr_buf *buf = &resp->dirlist; struct xdr_stream *xdr = &resp->xdr; - count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp)); - memset(buf, 0, sizeof(*buf)); /* Reserve room for the NULL ptr & eof flag (-2 words) */ - buf->buflen = count - XDR_UNIT * 2; + buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), (u32)PAGE_SIZE); + buf->buflen -= XDR_UNIT * 2; buf->pages = rqstp->rq_next_page; rqstp->rq_next_page++; - /* This is xdr_init_encode(), but it assumes that - * the head kvec has already been consumed. */ - xdr_set_scratch_buffer(xdr, NULL, 0); - xdr->buf = buf; - xdr->page_ptr = buf->pages; - xdr->iov = NULL; - xdr->p = page_address(*buf->pages); - xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE); - xdr->rqst = NULL; + xdr_init_encode_pages(xdr, buf, buf->pages, NULL); } /* @@ -646,6 +637,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, @@ -657,6 +649,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_attrstatres, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT, @@ -668,6 +661,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_attrstatres, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_sattrargs), + .pc_argzero = sizeof(struct nfsd_sattrargs), .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, @@ -678,6 +672,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, @@ -689,6 +684,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_diropres, .pc_release = nfssvc_release_diropres, .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+FH+AT, @@ -699,6 +695,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfssvc_encode_readlinkres, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_readlinkres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+1+NFS_MAXPATHLEN/4, @@ -710,6 +707,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_readres, .pc_release = nfssvc_release_readres, .pc_argsize = sizeof(struct nfsd_readargs), + .pc_argzero = sizeof(struct nfsd_readargs), .pc_ressize = sizeof(struct nfsd_readres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+AT+1+NFSSVC_MAXBLKSIZE_V2/4, @@ -720,6 +718,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_voidarg, .pc_encode = nfssvc_encode_voidres, .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), .pc_ressize = sizeof(struct nfsd_voidres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = 0, @@ -731,6 +730,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_attrstatres, .pc_release = nfssvc_release_attrstat, .pc_argsize = sizeof(struct nfsd_writeargs), + .pc_argzero = sizeof(struct nfsd_writeargs), .pc_ressize = sizeof(struct nfsd_attrstat), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+AT, @@ -742,6 +742,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_diropres, .pc_release = nfssvc_release_diropres, .pc_argsize = sizeof(struct nfsd_createargs), + .pc_argzero = sizeof(struct nfsd_createargs), .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, @@ -752,6 +753,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_diropargs, .pc_encode = nfssvc_encode_statres, .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, @@ -762,6 +764,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_renameargs, .pc_encode = nfssvc_encode_statres, .pc_argsize = sizeof(struct nfsd_renameargs), + .pc_argzero = sizeof(struct nfsd_renameargs), .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, @@ -772,6 +775,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_linkargs, .pc_encode = nfssvc_encode_statres, .pc_argsize = sizeof(struct nfsd_linkargs), + .pc_argzero = sizeof(struct nfsd_linkargs), .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, @@ -782,6 +786,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_symlinkargs, .pc_encode = nfssvc_encode_statres, .pc_argsize = sizeof(struct nfsd_symlinkargs), + .pc_argzero = sizeof(struct nfsd_symlinkargs), .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, @@ -793,6 +798,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_encode = nfssvc_encode_diropres, .pc_release = nfssvc_release_diropres, .pc_argsize = sizeof(struct nfsd_createargs), + .pc_argzero = sizeof(struct nfsd_createargs), .pc_ressize = sizeof(struct nfsd_diropres), .pc_cachetype = RC_REPLBUFF, .pc_xdrressize = ST+FH+AT, @@ -803,6 +809,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_diropargs, .pc_encode = nfssvc_encode_statres, .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), .pc_ressize = sizeof(struct nfsd_stat), .pc_cachetype = RC_REPLSTAT, .pc_xdrressize = ST, @@ -813,6 +820,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_readdirargs, .pc_encode = nfssvc_encode_readdirres, .pc_argsize = sizeof(struct nfsd_readdirargs), + .pc_argzero = sizeof(struct nfsd_readdirargs), .pc_ressize = sizeof(struct nfsd_readdirres), .pc_cachetype = RC_NOCACHE, .pc_name = "READDIR", @@ -822,6 +830,7 @@ static const struct svc_procedure nfsd_procedures2[18] = { .pc_decode = nfssvc_decode_fhandleargs, .pc_encode = nfssvc_encode_statfsres, .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), .pc_ressize = sizeof(struct nfsd_statfsres), .pc_cachetype = RC_NOCACHE, .pc_xdrressize = ST+5, diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 4bb5baa17040fcaae15e80a9e87e977e48986fa7..bfbd9f672f59e289e1c2acb07a1061843d01de86 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -799,7 +799,7 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred) if (nrservs == 0 && nn->nfsd_serv == NULL) goto out; - strlcpy(nn->nfsd_name, utsname()->nodename, + strscpy(nn->nfsd_name, utsname()->nodename, sizeof(nn->nfsd_name)); error = nfsd_create_serv(net); diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index aba8520b4b8b6b18284586604f4231dd266c10d7..caf6355b18fa99769850d41ab88db0f8330848fb 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -338,10 +338,8 @@ nfssvc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) return false; if (args->len > NFSSVC_MAXBLKSIZE_V2) return false; - if (!xdr_stream_subsegment(xdr, &args->payload, args->len)) - return false; - return true; + return xdr_stream_subsegment(xdr, &args->payload, args->len); } bool diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index ae596dbf866755b946cd434877a77275b3ecc594..e2daef3cc0034b5f785958801fb57cb8268e5566 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -57,11 +57,11 @@ typedef struct { } stateid_t; typedef struct { - stateid_t stid; + stateid_t cs_stid; #define NFS4_COPY_STID 1 #define NFS4_COPYNOTIFY_STID 2 - unsigned char sc_type; - refcount_t sc_count; + unsigned char cs_type; + refcount_t cs_count; } copy_stateid_t; struct nfsd4_callback { @@ -175,7 +175,7 @@ static inline struct nfs4_delegation *delegstateid(struct nfs4_stid *s) /* Maximum number of slots per session. 160 is useful for long haul TCP */ #define NFSD_MAX_SLOTS_PER_SESSION 160 /* Maximum number of operations per session compound */ -#define NFSD_MAX_OPS_PER_COMPOUND 16 +#define NFSD_MAX_OPS_PER_COMPOUND 50 /* Maximum session per slot cache size */ #define NFSD_SLOT_CACHE_SIZE 2048 /* Maximum number of NFSD_SLOT_CACHE_SIZE slots per session */ @@ -692,12 +692,11 @@ extern void nfsd4_probe_callback_sync(struct nfs4_client *clp); extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *); extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op); -extern void nfsd4_run_cb(struct nfsd4_callback *cb); +extern bool nfsd4_run_cb(struct nfsd4_callback *cb); extern int nfsd4_create_callback_queue(void); extern void nfsd4_destroy_callback_queue(void); extern void nfsd4_shutdown_callback(struct nfs4_client *); extern void nfsd4_shutdown_copy(struct nfs4_client *clp); -extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, struct nfsd_net *nn); extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c index a8c5a02a84f044a3f856ef6673ab1f02baae2524..777e24e5da33bd456c4ff432c74f0cb0e2c36c6a 100644 --- a/fs/nfsd/stats.c +++ b/fs/nfsd/stats.c @@ -32,7 +32,7 @@ struct svc_stat nfsd_svcstats = { .program = &nfsd_program, }; -static int nfsd_proc_show(struct seq_file *seq, void *v) +static int nfsd_show(struct seq_file *seq, void *v) { int i; @@ -72,17 +72,7 @@ static int nfsd_proc_show(struct seq_file *seq, void *v) return 0; } -static int nfsd_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, nfsd_proc_show, NULL); -} - -static const struct proc_ops nfsd_proc_ops = { - .proc_open = nfsd_proc_open, - .proc_read = seq_read, - .proc_lseek = seq_lseek, - .proc_release = single_release, -}; +DEFINE_PROC_SHOW_ATTRIBUTE(nfsd); int nfsd_percpu_counters_init(struct percpu_counter counters[], int num) { diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 9ebd67d461f9ee51a33c305920a8bddc5f790441..06a96e955bd00dca4cbf3bfd33cd9ba4b165a531 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -84,19 +84,26 @@ DEFINE_NFSD_XDR_ERR_EVENT(cant_encode); { NFSD_MAY_64BIT_COOKIE, "64BIT_COOKIE" }) TRACE_EVENT(nfsd_compound, - TP_PROTO(const struct svc_rqst *rqst, - u32 args_opcnt), - TP_ARGS(rqst, args_opcnt), + TP_PROTO( + const struct svc_rqst *rqst, + const char *tag, + u32 taglen, + u32 opcnt + ), + TP_ARGS(rqst, tag, taglen, opcnt), TP_STRUCT__entry( __field(u32, xid) - __field(u32, args_opcnt) + __field(u32, opcnt) + __string_len(tag, tag, taglen) ), TP_fast_assign( __entry->xid = be32_to_cpu(rqst->rq_xid); - __entry->args_opcnt = args_opcnt; + __entry->opcnt = opcnt; + __assign_str_len(tag, tag, taglen); ), - TP_printk("xid=0x%08x opcnt=%u", - __entry->xid, __entry->args_opcnt) + TP_printk("xid=0x%08x opcnt=%u tag=%s", + __entry->xid, __entry->opcnt, __get_str(tag) + ) ) TRACE_EVENT(nfsd_compound_status, @@ -195,7 +202,7 @@ TRACE_EVENT(nfsd_fh_verify, __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) - __field(void *, inode) + __field(const void *, inode) __field(unsigned long, type) __field(unsigned long, access) ), @@ -211,13 +218,55 @@ TRACE_EVENT(nfsd_fh_verify, __entry->type = type; __entry->access = access; ), - TP_printk("xid=0x%08x fh_hash=0x%08x inode=%p type=%s access=%s", - __entry->xid, __entry->fh_hash, __entry->inode, + TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s", + __entry->xid, __entry->fh_hash, show_fs_file_type(__entry->type), show_nfsd_may_flags(__entry->access) ) ); +TRACE_EVENT_CONDITION(nfsd_fh_verify_err, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + umode_t type, + int access, + __be32 error + ), + TP_ARGS(rqstp, fhp, type, access, error), + TP_CONDITION(error), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) + __field(u32, xid) + __field(u32, fh_hash) + __field(const void *, inode) + __field(unsigned long, type) + __field(unsigned long, access) + __field(int, error) + ), + TP_fast_assign( + __entry->netns_ino = SVC_NET(rqstp)->ns.inum; + __assign_sockaddr(server, &rqstp->rq_xprt->xpt_local, + rqstp->rq_xprt->xpt_locallen); + __assign_sockaddr(client, &rqstp->rq_xprt->xpt_remote, + rqstp->rq_xprt->xpt_remotelen); + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->inode = d_inode(fhp->fh_dentry); + __entry->type = type; + __entry->access = access; + __entry->error = be32_to_cpu(error); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s error=%d", + __entry->xid, __entry->fh_hash, + show_fs_file_type(__entry->type), + show_nfsd_may_flags(__entry->access), + __entry->error + ) +); + DECLARE_EVENT_CLASS(nfsd_fh_err_class, TP_PROTO(struct svc_rqst *rqstp, struct svc_fh *fhp, @@ -489,6 +538,29 @@ DEFINE_NFSD_COPY_ERR_EVENT(clone_file_range_err); #include "filecache.h" #include "vfs.h" +TRACE_EVENT(nfsd_delegret_wakeup, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct inode *inode, + long timeo + ), + TP_ARGS(rqstp, inode, timeo), + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, inode) + __field(long, timeo) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->inode = inode; + __entry->timeo = timeo; + ), + TP_printk("xid=0x%08x inode=%p%s", + __entry->xid, __entry->inode, + __entry->timeo == 0 ? " (timed out)" : "" + ) +); + DECLARE_EVENT_CLASS(nfsd_stateid_class, TP_PROTO(stateid_t *stp), TP_ARGS(stp), @@ -1399,6 +1471,45 @@ TRACE_EVENT(nfsd_cb_offload, __entry->fh_hash, __entry->count, __entry->status) ); +DECLARE_EVENT_CLASS(nfsd_cb_done_class, + TP_PROTO( + const stateid_t *stp, + const struct rpc_task *task + ), + TP_ARGS(stp, task), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + __field(int, status) + ), + TP_fast_assign( + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + __entry->status = task->tk_status; + ), + TP_printk("client %08x:%08x stateid %08x:%08x status=%d", + __entry->cl_boot, __entry->cl_id, __entry->si_id, + __entry->si_generation, __entry->status + ) +); + +#define DEFINE_NFSD_CB_DONE_EVENT(name) \ +DEFINE_EVENT(nfsd_cb_done_class, name, \ + TP_PROTO( \ + const stateid_t *stp, \ + const struct rpc_task *task \ + ), \ + TP_ARGS(stp, task)) + +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_recall_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_notify_lock_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_layout_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_offload_done); + #endif /* _NFSD_TRACE_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 9f486b788ed09fdd007ca342bc4f4e9fe34273cb..f650afedd67fafc5d71bbfe41a845ae4a721afe6 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -300,6 +300,10 @@ commit_metadata(struct svc_fh *fhp) static void nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap) { + /* Ignore mode updates on symlinks */ + if (S_ISLNK(inode->i_mode)) + iap->ia_valid &= ~ATTR_MODE; + /* sanitize the mode change */ if (iap->ia_valid & ATTR_MODE) { iap->ia_mode &= S_IALLUGO; @@ -339,8 +343,61 @@ nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp, return nfserrno(get_write_access(inode)); } -/* - * Set various file attributes. After this call fhp needs an fh_put. +static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap) +{ + int host_err; + + if (iap->ia_valid & ATTR_SIZE) { + /* + * RFC5661, Section 18.30.4: + * Changing the size of a file with SETATTR indirectly + * changes the time_modify and change attributes. + * + * (and similar for the older RFCs) + */ + struct iattr size_attr = { + .ia_valid = ATTR_SIZE | ATTR_CTIME | ATTR_MTIME, + .ia_size = iap->ia_size, + }; + + if (iap->ia_size < 0) + return -EFBIG; + + host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL); + if (host_err) + return host_err; + iap->ia_valid &= ~ATTR_SIZE; + + /* + * Avoid the additional setattr call below if the only other + * attribute that the client sends is the mtime, as we update + * it as part of the size change above. + */ + if ((iap->ia_valid & ~ATTR_MTIME) == 0) + return 0; + } + + if (!iap->ia_valid) + return 0; + + iap->ia_valid |= ATTR_CTIME; + return notify_change(&init_user_ns, dentry, iap, NULL); +} + +/** + * nfsd_setattr - Set various file attributes. + * @rqstp: controlling RPC transaction + * @fhp: filehandle of target + * @attr: attributes to set + * @check_guard: set to 1 if guardtime is a valid timestamp + * @guardtime: do not act if ctime.tv_sec does not match this timestamp + * + * This call may adjust the contents of @attr (in particular, this + * call may change the bits in the na_iattr.ia_valid field). + * + * Returns nfs_ok on success, otherwise an NFS status code is + * returned. Caller must release @fhp by calling fh_put in either + * case. */ __be32 nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, @@ -356,6 +413,7 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, int host_err; bool get_write_count; bool size_change = (iap->ia_valid & ATTR_SIZE); + int retries; if (iap->ia_valid & ATTR_SIZE) { accmode |= NFSD_MAY_WRITE|NFSD_MAY_OWNER_OVERRIDE; @@ -391,13 +449,6 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, dentry = fhp->fh_dentry; inode = d_inode(dentry); - /* Ignore any mode updates on symlinks */ - if (S_ISLNK(inode->i_mode)) - iap->ia_valid &= ~ATTR_MODE; - - if (!iap->ia_valid) - return 0; - nfsd_sanitize_attrs(inode, iap); if (check_guard && guardtime != inode->i_ctime.tv_sec) @@ -417,41 +468,13 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, } inode_lock(inode); - if (size_change) { - /* - * RFC5661, Section 18.30.4: - * Changing the size of a file with SETATTR indirectly - * changes the time_modify and change attributes. - * - * (and similar for the older RFCs) - */ - struct iattr size_attr = { - .ia_valid = ATTR_SIZE | ATTR_CTIME | ATTR_MTIME, - .ia_size = iap->ia_size, - }; - - host_err = -EFBIG; - if (iap->ia_size < 0) - goto out_unlock; - - host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL); - if (host_err) - goto out_unlock; - iap->ia_valid &= ~ATTR_SIZE; - - /* - * Avoid the additional setattr call below if the only other - * attribute that the client sends is the mtime, as we update - * it as part of the size change above. - */ - if ((iap->ia_valid & ~ATTR_MTIME) == 0) - goto out_unlock; + for (retries = 1;;) { + host_err = __nfsd_setattr(dentry, iap); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, inode)) + break; } - - iap->ia_valid |= ATTR_CTIME; - host_err = notify_change(&init_user_ns, dentry, iap, NULL); - -out_unlock: if (attr->na_seclabel && attr->na_seclabel->len) attr->na_labelerr = security_inode_setsecctx(dentry, attr->na_seclabel->data, attr->na_seclabel->len); @@ -846,10 +869,14 @@ nfsd_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf, struct splice_desc *sd) { struct svc_rqst *rqstp = sd->u.data; - - svc_rqst_replace_page(rqstp, buf->page); - if (rqstp->rq_res.page_len == 0) - rqstp->rq_res.page_base = buf->offset; + struct page *page = buf->page; // may be a compound one + unsigned offset = buf->offset; + + page += offset / PAGE_SIZE; + for (int i = sd->len; i > 0; i -= PAGE_SIZE) + svc_rqst_replace_page(rqstp, page++); + if (rqstp->rq_res.page_len == 0) // first call + rqstp->rq_res.page_base = offset % PAGE_SIZE; rqstp->rq_res.page_len += sd->len; return sd->len; } @@ -1252,7 +1279,7 @@ nfsd_check_ignore_resizing(struct iattr *iap) /* The parent directory should already be locked: */ __be32 nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp, - char *fname, int flen, struct nfsd_attrs *attrs, + struct nfsd_attrs *attrs, int type, dev_t rdev, struct svc_fh *resfhp) { struct dentry *dentry, *dchild; @@ -1379,8 +1406,7 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp, if (err) goto out_unlock; fh_fill_pre_attrs(fhp); - err = nfsd_create_locked(rqstp, fhp, fname, flen, attrs, type, - rdev, resfhp); + err = nfsd_create_locked(rqstp, fhp, attrs, type, rdev, resfhp); fh_fill_post_attrs(fhp); out_unlock: inode_unlock(dentry->d_inode); @@ -1670,7 +1696,15 @@ retry: .new_dir = tdir, .new_dentry = ndentry, }; - host_err = vfs_rename(&rd); + int retries; + + for (retries = 1;;) { + host_err = vfs_rename(&rd); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, d_inode(odentry))) + break; + } if (!host_err) { host_err = commit_metadata(tfhp); if (!host_err) @@ -1754,9 +1788,18 @@ nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type, fh_fill_pre_attrs(fhp); if (type != S_IFDIR) { + int retries; + if (rdentry->d_sb->s_export_op->flags & EXPORT_OP_CLOSE_BEFORE_UNLINK) nfsd_close_cached_files(rdentry); - host_err = vfs_unlink(&init_user_ns, dirp, rdentry, NULL); + + for (retries = 1;;) { + host_err = vfs_unlink(&init_user_ns, dirp, rdentry, NULL); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, rinode)) + break; + } } else { host_err = vfs_rmdir(&init_user_ns, dirp, rdentry); } @@ -1811,7 +1854,7 @@ struct readdir_data { int full; }; -static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name, +static bool nfsd_buffered_filldir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { @@ -1823,7 +1866,7 @@ static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name, reclen = ALIGN(sizeof(struct buffered_dirent) + namlen, sizeof(u64)); if (buf->used + reclen > PAGE_SIZE) { buf->full = 1; - return -EINVAL; + return false; } de->namlen = namlen; @@ -1833,7 +1876,7 @@ static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name, memcpy(de->name, name, namlen); buf->used += reclen; - return 0; + return true; } static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp, diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index c95cd414b4bb01d0db40235fbf1eb36c7e14813f..120521bc7b24766842dfa6f143c2129037fc07f2 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -79,8 +79,8 @@ __be32 nfsd4_clone_file_range(struct svc_rqst *rqstp, u64 count, bool sync); #endif /* CONFIG_NFSD_V4 */ __be32 nfsd_create_locked(struct svc_rqst *, struct svc_fh *, - char *name, int len, struct nfsd_attrs *attrs, - int type, dev_t rdev, struct svc_fh *res); + struct nfsd_attrs *attrs, int type, dev_t rdev, + struct svc_fh *res); __be32 nfsd_create(struct svc_rqst *, struct svc_fh *, char *name, int len, struct nfsd_attrs *attrs, int type, dev_t rdev, struct svc_fh *res); diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 96267258e629100943fe980a1f97e056623ba841..0eb00105d845b605ccff6735a362b021c3754ef6 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -717,13 +717,13 @@ struct nfsd4_compoundargs { struct svcxdr_tmpbuf *to_free; struct svc_rqst *rqstp; - u32 taglen; char * tag; + u32 taglen; u32 minorversion; + u32 client_opcnt; u32 opcnt; struct nfsd4_op *ops; struct nfsd4_op iops[8]; - int cachetype; }; struct nfsd4_compoundres { @@ -732,8 +732,8 @@ struct nfsd4_compoundres { struct svc_rqst * rqstp; __be32 *statusp; - u32 taglen; char * tag; + u32 taglen; u32 opcnt; struct nfsd4_compound_state cstate; @@ -888,7 +888,8 @@ struct nfsd4_operation { u32 op_flags; char *op_name; /* Try to get response size before operation */ - u32 (*op_rsize_bop)(struct svc_rqst *, struct nfsd4_op *); + u32 (*op_rsize_bop)(const struct svc_rqst *rqstp, + const struct nfsd4_op *op); void (*op_get_currentstateid)(struct nfsd4_compound_state *, union nfsd4_op_u *); void (*op_set_currentstateid)(struct nfsd4_compound_state *, diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index 9f4d9432d38a196ef4db01996055ee65b1ff3e0f..b9d15c3df3cc193ba10cf88761d45027ee6de346 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -1668,8 +1668,7 @@ static int nilfs_btree_check_delete(struct nilfs_bmap *btree, __u64 key) maxkey = nilfs_btree_node_get_key(node, nchildren - 1); nextmaxkey = (nchildren > 1) ? nilfs_btree_node_get_key(node, nchildren - 2) : 0; - if (bh != NULL) - brelse(bh); + brelse(bh); return (maxkey == key) && (nextmaxkey < NILFS_BMAP_LARGE_LOW); } @@ -1717,8 +1716,7 @@ static int nilfs_btree_gather_data(struct nilfs_bmap *btree, ptrs[i] = le64_to_cpu(dptrs[i]); } - if (bh != NULL) - brelse(bh); + brelse(bh); return nitems; } diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 67f63cfeade5c4867155c9b0f0939b9785d95c99..232dd7b6cca14ba973fce7351d28579801942db9 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -328,6 +328,7 @@ struct inode *nilfs_new_inode(struct inode *dir, umode_t mode) struct inode *inode; struct nilfs_inode_info *ii; struct nilfs_root *root; + struct buffer_head *bh; int err = -ENOMEM; ino_t ino; @@ -343,11 +344,25 @@ struct inode *nilfs_new_inode(struct inode *dir, umode_t mode) ii->i_state = BIT(NILFS_I_NEW); ii->i_root = root; - err = nilfs_ifile_create_inode(root->ifile, &ino, &ii->i_bh); + err = nilfs_ifile_create_inode(root->ifile, &ino, &bh); if (unlikely(err)) goto failed_ifile_create_inode; /* reference count of i_bh inherits from nilfs_mdt_read_block() */ + if (unlikely(ino < NILFS_USER_INO)) { + nilfs_warn(sb, + "inode bitmap is inconsistent for reserved inodes"); + do { + brelse(bh); + err = nilfs_ifile_create_inode(root->ifile, &ino, &bh); + if (unlikely(err)) + goto failed_ifile_create_inode; + } while (ino < NILFS_USER_INO); + + nilfs_info(sb, "repaired inode bitmap for reserved inodes"); + } + ii->i_bh = bh; + atomic64_inc(&root->inodes_count); inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_ino = ino; @@ -440,6 +455,8 @@ int nilfs_read_inode_common(struct inode *inode, inode->i_atime.tv_nsec = le32_to_cpu(raw_inode->i_mtime_nsec); inode->i_ctime.tv_nsec = le32_to_cpu(raw_inode->i_ctime_nsec); inode->i_mtime.tv_nsec = le32_to_cpu(raw_inode->i_mtime_nsec); + if (nilfs_is_metadata_file_inode(inode) && !S_ISREG(inode->i_mode)) + return -EIO; /* this inode is for metadata and corrupted */ if (inode->i_nlink == 0) return -ESTALE; /* this inode is deleted */ diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 3267e96c256cabdf36f45ab1ce91fa57a0a4ddcb..39b7eea2642a09c1c5959428c033c0fac807cbd6 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -480,41 +480,36 @@ unsigned long nilfs_find_uncommitted_extent(struct inode *inode, sector_t start_blk, sector_t *blkoff) { - unsigned int i; + unsigned int i, nr_folios; pgoff_t index; - unsigned int nblocks_in_page; unsigned long length = 0; - sector_t b; - struct pagevec pvec; - struct page *page; + struct folio_batch fbatch; + struct folio *folio; if (inode->i_mapping->nrpages == 0) return 0; index = start_blk >> (PAGE_SHIFT - inode->i_blkbits); - nblocks_in_page = 1U << (PAGE_SHIFT - inode->i_blkbits); - pagevec_init(&pvec); + folio_batch_init(&fbatch); repeat: - pvec.nr = find_get_pages_contig(inode->i_mapping, index, PAGEVEC_SIZE, - pvec.pages); - if (pvec.nr == 0) + nr_folios = filemap_get_folios_contig(inode->i_mapping, &index, ULONG_MAX, + &fbatch); + if (nr_folios == 0) return length; - if (length > 0 && pvec.pages[0]->index > index) - goto out; - - b = pvec.pages[0]->index << (PAGE_SHIFT - inode->i_blkbits); i = 0; do { - page = pvec.pages[i]; + folio = fbatch.folios[i]; - lock_page(page); - if (page_has_buffers(page)) { + folio_lock(folio); + if (folio_buffers(folio)) { struct buffer_head *bh, *head; + sector_t b; - bh = head = page_buffers(page); + b = folio->index << (PAGE_SHIFT - inode->i_blkbits); + bh = head = folio_buffers(folio); do { if (b < start_blk) continue; @@ -529,21 +524,17 @@ repeat: } else { if (length > 0) goto out_locked; - - b += nblocks_in_page; } - unlock_page(page); + folio_unlock(folio); - } while (++i < pagevec_count(&pvec)); + } while (++i < nr_folios); - index = page->index + 1; - pagevec_release(&pvec); + folio_batch_release(&fbatch); cond_resched(); goto repeat; out_locked: - unlock_page(page); -out: - pagevec_release(&pvec); + folio_unlock(folio); + folio_batch_release(&fbatch); return length; } diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 0afe0832c7547174e17b4efa6b39b51d0c131ee6..b4cebad21b48479a39a00af93b5b9c5c62fa0bbf 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -875,9 +875,11 @@ static int nilfs_segctor_create_checkpoint(struct nilfs_sc_info *sci) nilfs_mdt_mark_dirty(nilfs->ns_cpfile); nilfs_cpfile_put_checkpoint( nilfs->ns_cpfile, nilfs->ns_cno, bh_cp); - } else - WARN_ON(err == -EINVAL || err == -ENOENT); - + } else if (err == -EINVAL || err == -ENOENT) { + nilfs_error(sci->sc_super, + "checkpoint creation failed due to metadata corruption."); + err = -EIO; + } return err; } @@ -891,7 +893,11 @@ static int nilfs_segctor_fill_in_checkpoint(struct nilfs_sc_info *sci) err = nilfs_cpfile_get_checkpoint(nilfs->ns_cpfile, nilfs->ns_cno, 0, &raw_cp, &bh_cp); if (unlikely(err)) { - WARN_ON(err == -EINVAL || err == -ENOENT); + if (err == -EINVAL || err == -ENOENT) { + nilfs_error(sci->sc_super, + "checkpoint finalization failed due to metadata corruption."); + err = -EIO; + } goto failed_ibh; } raw_cp->cp_snapshot_list.ssl_next = 0; @@ -2235,7 +2241,6 @@ int nilfs_construct_segment(struct super_block *sb) struct the_nilfs *nilfs = sb->s_fs_info; struct nilfs_sc_info *sci = nilfs->ns_writer; struct nilfs_transaction_info *ti; - int err; if (!sci) return -EROFS; @@ -2243,8 +2248,7 @@ int nilfs_construct_segment(struct super_block *sb) /* A call inside transactions causes a deadlock. */ BUG_ON((ti = current->journal_info) && ti->ti_magic == NILFS_TI_MAGIC); - err = nilfs_segctor_sync(sci); - return err; + return nilfs_segctor_sync(sci); } /** @@ -2786,10 +2790,9 @@ int nilfs_attach_log_writer(struct super_block *sb, struct nilfs_root *root) inode_attach_wb(nilfs->ns_bdev->bd_inode, NULL); err = nilfs_segctor_start_thread(nilfs->ns_writer); - if (err) { - kfree(nilfs->ns_writer); - nilfs->ns_writer = NULL; - } + if (unlikely(err)) + nilfs_detach_log_writer(sb); + return err; } diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index cd7d09a569fff98684698609beaef412bb226754..a2a15bc4df280268ce224fb64b0934513eb2f3e2 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -18,7 +18,7 @@ #include "fanotify.h" -static bool fanotify_path_equal(struct path *p1, struct path *p2) +static bool fanotify_path_equal(const struct path *p1, const struct path *p2) { return p1->mnt == p2->mnt && p1->dentry == p2->dentry; } diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index 1d9f11255c64f5408e6d0ce900b9c8c61867e686..57f51a9a3015d5c2d1c652e95b1216b1462574c0 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -452,13 +452,7 @@ static inline bool fanotify_is_error_event(u32 mask) return mask & FAN_FS_ERROR; } -static inline bool fanotify_event_has_path(struct fanotify_event *event) -{ - return event->type == FANOTIFY_EVENT_TYPE_PATH || - event->type == FANOTIFY_EVENT_TYPE_PATH_PERM; -} - -static inline struct path *fanotify_event_path(struct fanotify_event *event) +static inline const struct path *fanotify_event_path(struct fanotify_event *event) { if (event->type == FANOTIFY_EVENT_TYPE_PATH) return &FANOTIFY_PE(event)->path; diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index f0e49a406ffa820bcca39c2eb7bf99adb880635a..4546da4a54f953cc5a8d0ddc3e2805330fe05006 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -249,7 +249,7 @@ out: return event; } -static int create_fd(struct fsnotify_group *group, struct path *path, +static int create_fd(struct fsnotify_group *group, const struct path *path, struct file **file) { int client_fd; @@ -619,7 +619,7 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group, char __user *buf, size_t count) { struct fanotify_event_metadata metadata; - struct path *path = fanotify_event_path(event); + const struct path *path = fanotify_event_path(event); struct fanotify_info *info = fanotify_event_info(event); unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); unsigned int pidfd_mode = info_mode & FAN_REPORT_PIDFD; @@ -1553,7 +1553,7 @@ static int fanotify_test_fid(struct dentry *dentry) } static int fanotify_events_supported(struct fsnotify_group *group, - struct path *path, __u64 mask, + const struct path *path, __u64 mask, unsigned int flags) { unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS; diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h index 87d8a50ee80387fbf8a23d83923cfb6d4583a960..fde74eb333cc93b9b6e7da341e3d79e0a76bc8b9 100644 --- a/fs/notify/fsnotify.h +++ b/fs/notify/fsnotify.h @@ -76,10 +76,6 @@ static inline void fsnotify_clear_marks_by_sb(struct super_block *sb) */ extern void __fsnotify_update_child_dentry_flags(struct inode *inode); -/* allocate and destroy and event holder to attach events to notification/access queues */ -extern struct fsnotify_event_holder *fsnotify_alloc_event_holder(void); -extern void fsnotify_destroy_event_holder(struct fsnotify_event_holder *holder); - extern struct kmem_cache *fsnotify_mark_connector_cachep; #endif /* __FS_NOTIFY_FSNOTIFY_H_ */ diff --git a/fs/nsfs.c b/fs/nsfs.c index 800c1d0eb0d026e40604e465df209cc11051e79f..3506f6074288a776619a6efc290330db3a5cc36a 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -28,7 +28,7 @@ static char *ns_dname(struct dentry *dentry, char *buffer, int buflen) struct inode *inode = d_inode(dentry); const struct proc_ns_operations *ns_ops = dentry->d_fsdata; - return dynamic_dname(dentry, buffer, buflen, "%s:[%lu]", + return dynamic_dname(buffer, buflen, "%s:[%lu]", ns_ops->name, inode->i_ino); } diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c index 52615e6090e1c80cbc9c326a54cc9003ba9daf1f..a3865bc4a0c650129f78732442e8ba98ad4b1aa2 100644 --- a/fs/ntfs/attrib.c +++ b/fs/ntfs/attrib.c @@ -594,17 +594,37 @@ static int ntfs_attr_find(const ATTR_TYPE type, const ntfschar *name, for (;; a = (ATTR_RECORD*)((u8*)a + le32_to_cpu(a->length))) { u8 *mrec_end = (u8 *)ctx->mrec + le32_to_cpu(ctx->mrec->bytes_allocated); - u8 *name_end = (u8 *)a + le16_to_cpu(a->name_offset) + - a->name_length * sizeof(ntfschar); - if ((u8*)a < (u8*)ctx->mrec || (u8*)a > mrec_end || - name_end > mrec_end) + u8 *name_end; + + /* check whether ATTR_RECORD wrap */ + if ((u8 *)a < (u8 *)ctx->mrec) + break; + + /* check whether Attribute Record Header is within bounds */ + if ((u8 *)a > mrec_end || + (u8 *)a + sizeof(ATTR_RECORD) > mrec_end) + break; + + /* check whether ATTR_RECORD's name is within bounds */ + name_end = (u8 *)a + le16_to_cpu(a->name_offset) + + a->name_length * sizeof(ntfschar); + if (name_end > mrec_end) break; + ctx->attr = a; if (unlikely(le32_to_cpu(a->type) > le32_to_cpu(type) || a->type == AT_END)) return -ENOENT; if (unlikely(!a->length)) break; + + /* check whether ATTR_RECORD's length wrap */ + if ((u8 *)a + le32_to_cpu(a->length) < (u8 *)a) + break; + /* check whether ATTR_RECORD's length is within bounds */ + if ((u8 *)a + le32_to_cpu(a->length) > mrec_end) + break; + if (a->type != type) continue; /* diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 58b660dbbee9b6e0f3d71ce376323fb69200950e..c481b14e4fd989cc9a01278f0f50bc7d44d7b902 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c @@ -527,12 +527,12 @@ err_out: goto out; } -static inline int ntfs_submit_bh_for_read(struct buffer_head *bh) +static inline void ntfs_submit_bh_for_read(struct buffer_head *bh) { lock_buffer(bh); get_bh(bh); bh->b_end_io = end_buffer_read_sync; - return submit_bh(REQ_OP_READ, bh); + submit_bh(REQ_OP_READ, bh); } /** diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c index db0f1995aedd15c9e623dfedc9c3fa34cbb8406d..08c659332e26b0214e27f7753265e481aa6ce107 100644 --- a/fs/ntfs/inode.c +++ b/fs/ntfs/inode.c @@ -1829,6 +1829,13 @@ int ntfs_read_inode_mount(struct inode *vi) goto err_out; } + /* Sanity check offset to the first attribute */ + if (le16_to_cpu(m->attrs_offset) >= le32_to_cpu(m->bytes_allocated)) { + ntfs_error(sb, "Incorrect mft offset to the first attribute %u in superblock.", + le16_to_cpu(m->attrs_offset)); + goto err_out; + } + /* Need this to sanity check attribute list references to $MFT. */ vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number); diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c index 5ae8de09b271b3dfe974bada07ac339fdd788271..001f4e053c85a8c065b14edfd2a28a1686974288 100644 --- a/fs/ntfs/super.c +++ b/fs/ntfs/super.c @@ -2092,7 +2092,8 @@ get_ctx_vol_failed: // TODO: Initialize security. /* Get the extended system files' directory inode. */ vol->extend_ino = ntfs_iget(sb, FILE_Extend); - if (IS_ERR(vol->extend_ino) || is_bad_inode(vol->extend_ino)) { + if (IS_ERR(vol->extend_ino) || is_bad_inode(vol->extend_ino) || + !S_ISDIR(vol->extend_ino->i_mode)) { if (!IS_ERR(vol->extend_ino)) iput(vol->extend_ino); ntfs_error(sb, "Failed to load $Extend."); diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c index 5d44ceac855b7477afc0aa5e8e418bea881fd070..e92bbd754365ed3bc64559cfed0865ef36eec648 100644 --- a/fs/ntfs3/bitmap.c +++ b/fs/ntfs3/bitmap.c @@ -560,7 +560,7 @@ static int wnd_rescan(struct wnd_bitmap *wnd) buf = (ulong *)bh->b_data; - used = __bitmap_weight(buf, wbits); + used = bitmap_weight(buf, wbits); if (used < wbits) { frb = wbits - used; wnd->free_bits[iw] = frb; @@ -1364,7 +1364,7 @@ int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) buf = (ulong *)bh->b_data; __bitmap_clear(buf, b0, blocksize * 8 - b0); - frb = wbits - __bitmap_weight(buf, wbits); + frb = wbits - bitmap_weight(buf, wbits); wnd->total_zeroes += frb - wnd->free_bits[iw]; wnd->free_bits[iw] = frb; diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index e7c494005122c00baf6d48361e1cc85b47d49720..0d611a6c5511f6996372c3cae176f8e0dad85e04 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -3819,7 +3819,7 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) } log_init_pg_hdr(log, page_size, page_size, 1, 1); - log_create(log, l_size, 0, get_random_int(), false, false); + log_create(log, l_size, 0, get_random_u32(), false, false); log->ra = ra; @@ -3893,7 +3893,7 @@ check_restart_area: /* Do some checks based on whether we have a valid log page. */ if (!rst_info.valid_page) { - open_log_count = get_random_int(); + open_log_count = get_random_u32(); goto init_log_instance; } open_log_count = le32_to_cpu(ra2->open_log_count); @@ -4044,7 +4044,7 @@ find_oldest: memcpy(ra->clients, Add2Ptr(ra2, t16), le16_to_cpu(ra2->ra_len) - t16); - log->current_openlog_count = get_random_int(); + log->current_openlog_count = get_random_u32(); ra->open_log_count = cpu_to_le32(log->current_openlog_count); log->ra_size = offsetof(struct RESTART_AREA, clients) + sizeof(struct CLIENT_REC); diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 51363d4e8636b82cab2ed467563dbda3ca6ff9fe..d5a3afbbbfd8cf639fa94d71e19f9e007b69bf68 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -630,12 +630,9 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, bh->b_size = block_size; off = vbo & (PAGE_SIZE - 1); set_bh_page(bh, page, off); - ll_rw_block(REQ_OP_READ, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { - err = -EIO; + err = bh_read(bh, 0); + if (err < 0) goto out; - } zero_user_segment(page, off + voff, off + block_size); } } @@ -1927,8 +1924,6 @@ const struct inode_operations ntfs_link_inode_operations = { .setattr = ntfs3_setattr, .listxattr = ntfs_listxattr, .permission = ntfs_permission, - .get_acl = ntfs_get_acl, - .set_acl = ntfs_set_acl, }; const struct address_space_operations ntfs_aops = { diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index 5bdff12a1232dbcef7ae04e5efd3d91f56034ada..7de8718c68a900a9200f3dfdeb2736a5b153b25f 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -483,8 +483,7 @@ out: } #ifdef CONFIG_NTFS3_FS_POSIX_ACL -static struct posix_acl *ntfs_get_acl_ex(struct user_namespace *mnt_userns, - struct inode *inode, int type, +static struct posix_acl *ntfs_get_acl_ex(struct inode *inode, int type, int locked) { struct ntfs_inode *ni = ntfs_i(inode); @@ -519,7 +518,7 @@ static struct posix_acl *ntfs_get_acl_ex(struct user_namespace *mnt_userns, /* Translate extended attribute to acl. */ if (err >= 0) { - acl = posix_acl_from_xattr(mnt_userns, buf, err); + acl = posix_acl_from_xattr(&init_user_ns, buf, err); } else if (err == -ENODATA) { acl = NULL; } else { @@ -542,8 +541,7 @@ struct posix_acl *ntfs_get_acl(struct inode *inode, int type, bool rcu) if (rcu) return ERR_PTR(-ECHILD); - /* TODO: init_user_ns? */ - return ntfs_get_acl_ex(&init_user_ns, inode, type, 0); + return ntfs_get_acl_ex(inode, type, 0); } static noinline int ntfs_set_acl_ex(struct user_namespace *mnt_userns, @@ -595,7 +593,7 @@ static noinline int ntfs_set_acl_ex(struct user_namespace *mnt_userns, value = kmalloc(size, GFP_NOFS); if (!value) return -ENOMEM; - err = posix_acl_to_xattr(mnt_userns, acl, value, size); + err = posix_acl_to_xattr(&init_user_ns, acl, value, size); if (err < 0) goto out; flags = 0; @@ -627,67 +625,6 @@ int ntfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode, return ntfs_set_acl_ex(mnt_userns, inode, acl, type, false); } -static int ntfs_xattr_get_acl(struct user_namespace *mnt_userns, - struct inode *inode, int type, void *buffer, - size_t size) -{ - struct posix_acl *acl; - int err; - - if (!(inode->i_sb->s_flags & SB_POSIXACL)) { - ntfs_inode_warn(inode, "add mount option \"acl\" to use acl"); - return -EOPNOTSUPP; - } - - acl = ntfs_get_acl(inode, type, false); - if (IS_ERR(acl)) - return PTR_ERR(acl); - - if (!acl) - return -ENODATA; - - err = posix_acl_to_xattr(mnt_userns, acl, buffer, size); - posix_acl_release(acl); - - return err; -} - -static int ntfs_xattr_set_acl(struct user_namespace *mnt_userns, - struct inode *inode, int type, const void *value, - size_t size) -{ - struct posix_acl *acl; - int err; - - if (!(inode->i_sb->s_flags & SB_POSIXACL)) { - ntfs_inode_warn(inode, "add mount option \"acl\" to use acl"); - return -EOPNOTSUPP; - } - - if (!inode_owner_or_capable(mnt_userns, inode)) - return -EPERM; - - if (!value) { - acl = NULL; - } else { - acl = posix_acl_from_xattr(mnt_userns, value, size); - if (IS_ERR(acl)) - return PTR_ERR(acl); - - if (acl) { - err = posix_acl_valid(mnt_userns, acl); - if (err) - goto release_and_out; - } - } - - err = ntfs_set_acl(mnt_userns, inode, acl, type); - -release_and_out: - posix_acl_release(acl); - return err; -} - /* * ntfs_init_acl - Initialize the ACLs of a new inode. * @@ -854,23 +791,6 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de, goto out; } -#ifdef CONFIG_NTFS3_FS_POSIX_ACL - if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 && - !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS, - sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) || - (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 && - !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, - sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) { - /* TODO: init_user_ns? */ - err = ntfs_xattr_get_acl( - &init_user_ns, inode, - name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 - ? ACL_TYPE_ACCESS - : ACL_TYPE_DEFAULT, - buffer, size); - goto out; - } -#endif /* Deal with NTFS extended attribute. */ err = ntfs_get_ea(inode, name, name_len, buffer, size, NULL); @@ -983,22 +903,6 @@ set_new_fa: goto out; } -#ifdef CONFIG_NTFS3_FS_POSIX_ACL - if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 && - !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS, - sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) || - (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 && - !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, - sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) { - err = ntfs_xattr_set_acl( - mnt_userns, inode, - name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 - ? ACL_TYPE_ACCESS - : ACL_TYPE_DEFAULT, - value, size); - goto out; - } -#endif /* Deal with NTFS extended attribute. */ err = ntfs_set_ea(inode, name, name_len, value, size, flags, 0); @@ -1088,7 +992,7 @@ static bool ntfs_xattr_user_list(struct dentry *dentry) } // clang-format off -static const struct xattr_handler ntfs_xattr_handler = { +static const struct xattr_handler ntfs_other_xattr_handler = { .prefix = "", .get = ntfs_getxattr, .set = ntfs_setxattr, @@ -1096,7 +1000,11 @@ static const struct xattr_handler ntfs_xattr_handler = { }; const struct xattr_handler *ntfs_xattr_handlers[] = { - &ntfs_xattr_handler, +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + &posix_acl_access_xattr_handler, + &posix_acl_default_xattr_handler, +#endif + &ntfs_other_xattr_handler, NULL, }; // clang-format on diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index af4157f61927ee19332a4eea4ee455fedadeb902..1d65f6ef00ca8ba7a42eae4c3e87ff885201c43b 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -636,7 +636,7 @@ int ocfs2_map_page_blocks(struct page *page, u64 *p_blkno, !buffer_new(bh) && ocfs2_should_read_blk(inode, page, block_start) && (block_start < from || block_end > to)) { - ll_rw_block(REQ_OP_READ, 1, &bh); + bh_read_nowait(bh, 0); *wait_bh++=bh; } diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c index 81c3d65d68fec7cd4eb7061f1a5cefd81e65d60d..694471fc46b822c30dcec6cce927b278bf72d0c3 100644 --- a/fs/ocfs2/dir.c +++ b/fs/ocfs2/dir.c @@ -2032,7 +2032,7 @@ struct ocfs2_empty_dir_priv { unsigned seen_other; unsigned dx_dir; }; -static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name, +static bool ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name, int name_len, loff_t pos, u64 ino, unsigned type) { @@ -2052,7 +2052,7 @@ static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name, */ if (name_len == 1 && !strncmp(".", name, 1) && pos == 0) { p->seen_dot = 1; - return 0; + return true; } if (name_len == 2 && !strncmp("..", name, 2) && @@ -2060,13 +2060,13 @@ static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name, p->seen_dot_dot = 1; if (p->dx_dir && p->seen_dot) - return 1; + return false; - return 0; + return true; } p->seen_other = 1; - return 1; + return false; } static int ocfs2_empty_dir_dx(struct inode *inode, diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c index 801e60bab95557659c73614e228086df90837303..c28bc983a7b1c05b26b83708179b59974e20f7ac 100644 --- a/fs/ocfs2/dlmglue.c +++ b/fs/ocfs2/dlmglue.c @@ -3403,10 +3403,12 @@ void ocfs2_dlm_shutdown(struct ocfs2_super *osb, ocfs2_lock_res_free(&osb->osb_nfs_sync_lockres); ocfs2_lock_res_free(&osb->osb_orphan_scan.os_lockres); - ocfs2_cluster_disconnect(osb->cconn, hangup_pending); - osb->cconn = NULL; + if (osb->cconn) { + ocfs2_cluster_disconnect(osb->cconn, hangup_pending); + osb->cconn = NULL; - ocfs2_dlm_shutdown_debug(osb); + ocfs2_dlm_shutdown_debug(osb); + } } static int ocfs2_drop_lock(struct ocfs2_super *osb, diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index fa87d89cf75422f1aaac4f95726fb31be1267a3c..126671e6caeda6e777b0861b622f43a23578329a 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -2057,7 +2057,7 @@ struct ocfs2_orphan_filldir_priv { enum ocfs2_orphan_reco_type orphan_reco_type; }; -static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, +static bool ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, int name_len, loff_t pos, u64 ino, unsigned type) { @@ -2066,21 +2066,21 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, struct inode *iter; if (name_len == 1 && !strncmp(".", name, 1)) - return 0; + return true; if (name_len == 2 && !strncmp("..", name, 2)) - return 0; + return true; /* do not include dio entry in case of orphan scan */ if ((p->orphan_reco_type == ORPHAN_NO_NEED_TRUNCATE) && (!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX, OCFS2_DIO_ORPHAN_PREFIX_LEN))) - return 0; + return true; /* Skip bad inodes so that recovery can continue */ iter = ocfs2_iget(p->osb, ino, OCFS2_FI_FLAG_ORPHAN_RECOVERY, 0); if (IS_ERR(iter)) - return 0; + return true; if (!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX, OCFS2_DIO_ORPHAN_PREFIX_LEN)) @@ -2090,7 +2090,7 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, * happen concurrently with unlink/rename */ if (OCFS2_I(iter)->ip_next_orphan) { iput(iter); - return 0; + return true; } trace_ocfs2_orphan_filldir((unsigned long long)OCFS2_I(iter)->ip_blkno); @@ -2099,7 +2099,7 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, OCFS2_I(iter)->ip_next_orphan = p->head; p->head = iter; - return 0; + return true; } static int ocfs2_queue_orphans(struct ocfs2_super *osb, diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h index 638d875eccc7d68df68d7a93a7395b9f520a2b31..7aebdbf5cc0a5678d1a82998f9c213c62a108809 100644 --- a/fs/ocfs2/ocfs2_fs.h +++ b/fs/ocfs2/ocfs2_fs.h @@ -527,7 +527,7 @@ struct ocfs2_extent_block * value -1 (0xFFFF) is OCFS2_INVALID_SLOT. This marks a slot empty. */ struct ocfs2_slot_map { -/*00*/ __le16 sm_slots[0]; +/*00*/ DECLARE_FLEX_ARRAY(__le16, sm_slots); /* * Actual on-disk size is one block. OCFS2_MAX_SLOTS is 255, * 255 * sizeof(__le16) == 512B, within the 512B block minimum blocksize. @@ -548,7 +548,7 @@ struct ocfs2_extended_slot { * i_size. */ struct ocfs2_slot_map_extended { -/*00*/ struct ocfs2_extended_slot se_slots[0]; +/*00*/ DECLARE_FLEX_ARRAY(struct ocfs2_extended_slot, se_slots); /* * Actual size is i_size of the slot_map system file. It should * match s_max_slots * sizeof(struct ocfs2_extended_slot) @@ -727,7 +727,7 @@ struct ocfs2_dinode { struct ocfs2_extent_list i_list; struct ocfs2_truncate_log i_dealloc; struct ocfs2_inline_data i_data; - __u8 i_symlink[0]; + DECLARE_FLEX_ARRAY(__u8, i_symlink); } id2; /* Actual on-disk size is one block */ }; @@ -892,7 +892,7 @@ struct ocfs2_group_desc /*30*/ struct ocfs2_block_check bg_check; /* Error checking */ __le64 bg_reserved2; /*40*/ union { - __u8 bg_bitmap[0]; + DECLARE_FLEX_ARRAY(__u8, bg_bitmap); struct { /* * Block groups may be discontiguous when diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 1358981e80a36597660a54d9ec53edd4433abae7..623db358b1efa81744f36a53d51534e84a54e290 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -2614,7 +2614,7 @@ static inline unsigned int ocfs2_cow_align_length(struct super_block *sb, } /* - * Calculate out the start and number of virtual clusters we need to to CoW. + * Calculate out the start and number of virtual clusters we need to CoW. * * cpos is vitual start cluster position we want to do CoW in a * file and write_len is the cluster length. diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c index a75e2b7d67f56839069368a491773c3710d73f45..64e6ddcfe329a85a329e587b83a7bd610ea18be2 100644 --- a/fs/ocfs2/stack_user.c +++ b/fs/ocfs2/stack_user.c @@ -991,7 +991,7 @@ static int user_cluster_connect(struct ocfs2_cluster_connection *conn) lc->oc_type = NO_CONTROLD; rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name, - DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN, + DLM_LSFL_NEWEXCL, DLM_LVB_LEN, &ocfs2_ls_ops, conn, &ops_rv, &fsdlm); if (rc) { if (rc == -EEXIST || rc == -EPROTO) diff --git a/fs/ocfs2/stackglue.c b/fs/ocfs2/stackglue.c index dd77b7aaabf5c942ce3c43a2cac9e21c1f83aeeb..317126261523b45ad3a03acd8199b7c4db7070c8 100644 --- a/fs/ocfs2/stackglue.c +++ b/fs/ocfs2/stackglue.c @@ -334,10 +334,10 @@ int ocfs2_cluster_connect(const char *stack_name, goto out; } - strlcpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1); + strscpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1); new_conn->cc_namelen = grouplen; if (cluster_name_len) - strlcpy(new_conn->cc_cluster_name, cluster_name, + strscpy(new_conn->cc_cluster_name, cluster_name, CLUSTER_NAME_MAX + 1); new_conn->cc_cluster_name_len = cluster_name_len; new_conn->cc_recovery_handler = recovery_handler; diff --git a/fs/ocfs2/suballoc.h b/fs/ocfs2/suballoc.h index 5805a03d100ba842df00b8abd3b1238f91321e13..9c74eace3adc150761397744db992fc6ee598758 100644 --- a/fs/ocfs2/suballoc.h +++ b/fs/ocfs2/suballoc.h @@ -106,7 +106,7 @@ int ocfs2_claim_clusters(handle_t *handle, u32 *cluster_start, u32 *num_clusters); /* - * Use this variant of ocfs2_claim_clusters to specify a maxiumum + * Use this variant of ocfs2_claim_clusters to specify a maximum * number of clusters smaller than the allocation reserved. */ int __ocfs2_claim_clusters(handle_t *handle, diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index 013a727bd7c82df03b309161f09198108da48373..42c993e53924f97e0626e5befb2757334d0e329f 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1764,9 +1764,7 @@ static int ocfs2_get_sector(struct super_block *sb, if (!buffer_dirty(*bh)) clear_buffer_uptodate(*bh); unlock_buffer(*bh); - ll_rw_block(REQ_OP_READ, 1, bh); - wait_on_buffer(*bh); - if (!buffer_uptodate(*bh)) { + if (bh_read(*bh, 0) < 0) { mlog_errno(-EIO); brelse(*bh); *bh = NULL; @@ -1914,8 +1912,7 @@ static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err) !ocfs2_is_hard_readonly(osb)) hangup_needed = 1; - if (osb->cconn) - ocfs2_dlm_shutdown(osb, hangup_needed); + ocfs2_dlm_shutdown(osb, hangup_needed); ocfs2_blockcheck_stats_debugfs_remove(&osb->osb_ecc_stats); debugfs_remove_recursive(osb->osb_debug_root); @@ -2222,7 +2219,7 @@ static int ocfs2_initialize_super(struct super_block *sb, goto out_journal; } - strlcpy(osb->vol_label, di->id2.i_super.s_label, + strscpy(osb->vol_label, di->id2.i_super.s_label, OCFS2_MAX_VOL_LABEL_LEN); osb->root_blkno = le64_to_cpu(di->id2.i_super.s_root_blkno); osb->system_dir_blkno = le64_to_cpu(di->id2.i_super.s_system_dir_blkno); diff --git a/fs/open.c b/fs/open.c index 8a813fa5ca56511fd946903376d40cfcc2cb8d73..a81319b6177f69a4b905688136c0231a6a3700dc 100644 --- a/fs/open.c +++ b/fs/open.c @@ -716,6 +716,8 @@ int chown_common(const struct path *path, uid_t user, gid_t group) fs_userns = i_user_ns(inode); retry_deleg: + newattrs.ia_vfsuid = INVALID_VFSUID; + newattrs.ia_vfsgid = INVALID_VFSGID; newattrs.ia_valid = ATTR_CTIME; if ((user != (uid_t)-1) && !setattr_vfsuid(&newattrs, uid)) return -EINVAL; @@ -840,7 +842,9 @@ static int do_dentry_open(struct file *f, return 0; } - if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) { + if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) { + i_readcount_inc(inode); + } else if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) { error = get_write_access(inode); if (unlikely(error)) goto cleanup_file; @@ -880,8 +884,6 @@ static int do_dentry_open(struct file *f, goto cleanup_all; } f->f_mode |= FMODE_OPENED; - if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) - i_readcount_inc(inode); if ((f->f_mode & FMODE_READ) && likely(f->f_op->read || f->f_op->read_iter)) f->f_mode |= FMODE_CAN_READ; @@ -935,10 +937,7 @@ cleanup_all: if (WARN_ON_ONCE(error > 0)) error = -EINVAL; fops_put(f->f_op); - if (f->f_mode & FMODE_WRITER) { - put_write_access(inode); - __mnt_drop_write(f->f_path.mnt); - } + put_file_access(f); cleanup_file: path_put(&f->f_path); f->f_path.mnt = NULL; diff --git a/fs/orangefs/dir.c b/fs/orangefs/dir.c index e2c2699d8016274dbe2225b14c0ab8828ceae00f..9cacce5d55c1ba42d9fe7016d63de825ba5cb4f6 100644 --- a/fs/orangefs/dir.c +++ b/fs/orangefs/dir.c @@ -398,7 +398,7 @@ static int orangefs_dir_release(struct inode *inode, struct file *file) const struct file_operations orangefs_dir_operations = { .llseek = orangefs_dir_llseek, .read = generic_read_dir, - .iterate = orangefs_dir_iterate, + .iterate_shared = orangefs_dir_iterate, .open = orangefs_dir_open, .release = orangefs_dir_release }; diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c index 86810e5d79141a65ccfae1be109399052de67ce4..732661aa26804677f82f1c54a18c91057cc315f5 100644 --- a/fs/orangefs/file.c +++ b/fs/orangefs/file.c @@ -417,9 +417,7 @@ static int orangefs_file_release(struct inode *inode, struct file *file) * readahead cache (if any); this forces an expensive refresh of * data for the next caller of mmap (or 'get_block' accesses) */ - if (file_inode(file) && - file_inode(file)->i_mapping && - mapping_nrpages(&file_inode(file)->i_data)) { + if (mapping_nrpages(file->f_mapping)) { if (orangefs_features & ORANGEFS_FEATURE_READAHEAD) { gossip_debug(GOSSIP_INODE_DEBUG, "calling flush_racache on %pU\n", diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index fdde6c56cc3dc86501c5bdeb72cecd499f31a8d5..f436d8847f085a7e981b535e535ccba66b7f10ed 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -44,7 +44,7 @@ static bool ovl_must_copy_xattr(const char *name) !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); } -int ovl_copy_xattr(struct super_block *sb, struct path *oldpath, struct dentry *new) +int ovl_copy_xattr(struct super_block *sb, const struct path *oldpath, struct dentry *new) { struct dentry *old = oldpath->dentry; ssize_t list_size, size, value_size = 0; @@ -132,8 +132,8 @@ out: return error; } -static int ovl_copy_fileattr(struct inode *inode, struct path *old, - struct path *new) +static int ovl_copy_fileattr(struct inode *inode, const struct path *old, + const struct path *new) { struct fileattr oldfa = { .flags_valid = true }; struct fileattr newfa = { .flags_valid = true }; @@ -193,11 +193,11 @@ static int ovl_copy_fileattr(struct inode *inode, struct path *old, return ovl_real_fileattr_set(new, &newfa); } -static int ovl_copy_up_data(struct ovl_fs *ofs, struct path *old, - struct path *new, loff_t len) +static int ovl_copy_up_file(struct ovl_fs *ofs, struct dentry *dentry, + struct file *new_file, loff_t len) { + struct path datapath; struct file *old_file; - struct file *new_file; loff_t old_pos = 0; loff_t new_pos = 0; loff_t cloned; @@ -206,23 +206,18 @@ static int ovl_copy_up_data(struct ovl_fs *ofs, struct path *old, bool skip_hole = false; int error = 0; - if (len == 0) - return 0; + ovl_path_lowerdata(dentry, &datapath); + if (WARN_ON(datapath.dentry == NULL)) + return -EIO; - old_file = ovl_path_open(old, O_LARGEFILE | O_RDONLY); + old_file = ovl_path_open(&datapath, O_LARGEFILE | O_RDONLY); if (IS_ERR(old_file)) return PTR_ERR(old_file); - new_file = ovl_path_open(new, O_LARGEFILE | O_WRONLY); - if (IS_ERR(new_file)) { - error = PTR_ERR(new_file); - goto out_fput; - } - /* Try to use clone_file_range to clone up within the same fs */ cloned = do_clone_file_range(old_file, 0, new_file, 0, len, 0); if (cloned == len) - goto out; + goto out_fput; /* Couldn't clone, so now we try to copy the data */ /* Check if lower fs supports seek operation */ @@ -282,10 +277,8 @@ static int ovl_copy_up_data(struct ovl_fs *ofs, struct path *old, len -= bytes; } -out: if (!error && ovl_should_sync(ofs)) error = vfs_fsync(new_file, 0); - fput(new_file); out_fput: fput(old_file); return error; @@ -556,30 +549,31 @@ static int ovl_link_up(struct ovl_copy_up_ctx *c) return err; } -static int ovl_copy_up_inode(struct ovl_copy_up_ctx *c, struct dentry *temp) +static int ovl_copy_up_data(struct ovl_copy_up_ctx *c, const struct path *temp) { struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); - struct inode *inode = d_inode(c->dentry); - struct path upperpath, datapath; + struct file *new_file; int err; - ovl_path_upper(c->dentry, &upperpath); - if (WARN_ON(upperpath.dentry != NULL)) - return -EIO; + if (!S_ISREG(c->stat.mode) || c->metacopy || !c->stat.size) + return 0; - upperpath.dentry = temp; + new_file = ovl_path_open(temp, O_LARGEFILE | O_WRONLY); + if (IS_ERR(new_file)) + return PTR_ERR(new_file); - /* - * Copy up data first and then xattrs. Writing data after - * xattrs will remove security.capability xattr automatically. - */ - if (S_ISREG(c->stat.mode) && !c->metacopy) { - ovl_path_lowerdata(c->dentry, &datapath); - err = ovl_copy_up_data(ofs, &datapath, &upperpath, - c->stat.size); - if (err) - return err; - } + err = ovl_copy_up_file(ofs, c->dentry, new_file, c->stat.size); + fput(new_file); + + return err; +} + +static int ovl_copy_up_metadata(struct ovl_copy_up_ctx *c, struct dentry *temp) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct inode *inode = d_inode(c->dentry); + struct path upperpath = { .mnt = ovl_upper_mnt(ofs), .dentry = temp }; + int err; err = ovl_copy_xattr(c->dentry->d_sb, &c->lowerpath, temp); if (err) @@ -662,6 +656,7 @@ static int ovl_copy_up_workdir(struct ovl_copy_up_ctx *c) struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); struct inode *inode; struct inode *udir = d_inode(c->destdir), *wdir = d_inode(c->workdir); + struct path path = { .mnt = ovl_upper_mnt(ofs) }; struct dentry *temp, *upper; struct ovl_cu_creds cc; int err; @@ -688,7 +683,16 @@ static int ovl_copy_up_workdir(struct ovl_copy_up_ctx *c) if (IS_ERR(temp)) goto unlock; - err = ovl_copy_up_inode(c, temp); + /* + * Copy up data first and then xattrs. Writing data after + * xattrs will remove security.capability xattr automatically. + */ + path.dentry = temp; + err = ovl_copy_up_data(c, &path); + if (err) + goto cleanup; + + err = ovl_copy_up_metadata(c, temp); if (err) goto cleanup; @@ -732,6 +736,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); struct inode *udir = d_inode(c->destdir); struct dentry *temp, *upper; + struct file *tmpfile; struct ovl_cu_creds cc; int err; @@ -739,15 +744,22 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) if (err) return err; - temp = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode); + tmpfile = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode); ovl_revert_cu_creds(&cc); - if (IS_ERR(temp)) - return PTR_ERR(temp); + if (IS_ERR(tmpfile)) + return PTR_ERR(tmpfile); - err = ovl_copy_up_inode(c, temp); + temp = tmpfile->f_path.dentry; + if (!c->metacopy && c->stat.size) { + err = ovl_copy_up_file(ofs, c->dentry, tmpfile, c->stat.size); + if (err) + return err; + } + + err = ovl_copy_up_metadata(c, temp); if (err) - goto out_dput; + goto out_fput; inode_lock_nested(udir, I_MUTEX_PARENT); @@ -761,16 +773,14 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) inode_unlock(udir); if (err) - goto out_dput; + goto out_fput; if (!c->metacopy) ovl_set_upperdata(d_inode(c->dentry)); - ovl_inode_update(d_inode(c->dentry), temp); + ovl_inode_update(d_inode(c->dentry), dget(temp)); - return 0; - -out_dput: - dput(temp); +out_fput: + fput(tmpfile); return err; } @@ -872,7 +882,7 @@ static bool ovl_need_meta_copy_up(struct dentry *dentry, umode_t mode, return true; } -static ssize_t ovl_getxattr_value(struct path *path, char *name, char **value) +static ssize_t ovl_getxattr_value(const struct path *path, char *name, char **value) { ssize_t res; char *buf; @@ -899,7 +909,7 @@ static ssize_t ovl_getxattr_value(struct path *path, char *name, char **value) static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c) { struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); - struct path upperpath, datapath; + struct path upperpath; int err; char *capability = NULL; ssize_t cap_size; @@ -908,10 +918,6 @@ static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c) if (WARN_ON(upperpath.dentry == NULL)) return -EIO; - ovl_path_lowerdata(c->dentry, &datapath); - if (WARN_ON(datapath.dentry == NULL)) - return -EIO; - if (c->stat.size) { err = cap_size = ovl_getxattr_value(&upperpath, XATTR_NAME_CAPS, &capability); @@ -919,7 +925,7 @@ static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c) goto out; } - err = ovl_copy_up_data(ofs, &datapath, &upperpath, c->stat.size); + err = ovl_copy_up_data(c, &upperpath); if (err) goto out_free; diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index daff601b5c41011c1285f760e0c9b589503d0075..a1a22f58ba183f6d3181ab48fb4c272bc0b489a5 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -38,7 +38,7 @@ static char ovl_whatisit(struct inode *inode, struct inode *realinode) #define OVL_OPEN_FLAGS (O_NOATIME | FMODE_NONOTIFY) static struct file *ovl_open_realfile(const struct file *file, - struct path *realpath) + const struct path *realpath) { struct inode *realinode = d_inode(realpath->dentry); struct inode *inode = file_inode(file); diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index b45fea69fff3fdc26a56cbc46773fdaf5243230f..9e61511de7a7c6c7c0311df10a0e6741623025d6 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -460,9 +460,12 @@ ssize_t ovl_listxattr(struct dentry *dentry, char *list, size_t size) * of the POSIX ACLs retrieved from the lower layer to this function to not * alter the POSIX ACLs for the underlying filesystem. */ -static void ovl_idmap_posix_acl(struct user_namespace *mnt_userns, +static void ovl_idmap_posix_acl(struct inode *realinode, + struct user_namespace *mnt_userns, struct posix_acl *acl) { + struct user_namespace *fs_userns = i_user_ns(realinode); + for (unsigned int i = 0; i < acl->a_count; i++) { vfsuid_t vfsuid; vfsgid_t vfsgid; @@ -470,11 +473,11 @@ static void ovl_idmap_posix_acl(struct user_namespace *mnt_userns, struct posix_acl_entry *e = &acl->a_entries[i]; switch (e->e_tag) { case ACL_USER: - vfsuid = make_vfsuid(mnt_userns, &init_user_ns, e->e_uid); + vfsuid = make_vfsuid(mnt_userns, fs_userns, e->e_uid); e->e_uid = vfsuid_into_kuid(vfsuid); break; case ACL_GROUP: - vfsgid = make_vfsgid(mnt_userns, &init_user_ns, e->e_gid); + vfsgid = make_vfsgid(mnt_userns, fs_userns, e->e_gid); e->e_gid = vfsgid_into_kgid(vfsgid); break; } @@ -536,7 +539,7 @@ struct posix_acl *ovl_get_acl(struct inode *inode, int type, bool rcu) if (!clone) clone = ERR_PTR(-ENOMEM); else - ovl_idmap_posix_acl(mnt_user_ns(realpath.mnt), clone); + ovl_idmap_posix_acl(realinode, mnt_user_ns(realpath.mnt), clone); /* * Since we're not in RCU path walk we always need to release the * original ACLs. @@ -585,7 +588,7 @@ static int ovl_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, * Introducing security_inode_fileattr_get/set() hooks would solve this issue * properly. */ -static int ovl_security_fileattr(struct path *realpath, struct fileattr *fa, +static int ovl_security_fileattr(const struct path *realpath, struct fileattr *fa, bool set) { struct file *file; @@ -607,7 +610,7 @@ static int ovl_security_fileattr(struct path *realpath, struct fileattr *fa, return err; } -int ovl_real_fileattr_set(struct path *realpath, struct fileattr *fa) +int ovl_real_fileattr_set(const struct path *realpath, struct fileattr *fa) { int err; @@ -682,7 +685,7 @@ static void ovl_fileattr_prot_flags(struct inode *inode, struct fileattr *fa) } } -int ovl_real_fileattr_get(struct path *realpath, struct fileattr *fa) +int ovl_real_fileattr_get(const struct path *realpath, struct fileattr *fa) { int err; diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c index 69dc577974f81c3d76fd7d3f25c8e3611afe6858..0fd1d5fdfc728b01bc66ec6f2de39fcc3bff1fe7 100644 --- a/fs/overlayfs/namei.c +++ b/fs/overlayfs/namei.c @@ -26,7 +26,7 @@ struct ovl_lookup_data { bool metacopy; }; -static int ovl_check_redirect(struct path *path, struct ovl_lookup_data *d, +static int ovl_check_redirect(const struct path *path, struct ovl_lookup_data *d, size_t prelen, const char *post) { int res; @@ -194,7 +194,7 @@ struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, return real; } -static bool ovl_is_opaquedir(struct ovl_fs *ofs, struct path *path) +static bool ovl_is_opaquedir(struct ovl_fs *ofs, const struct path *path) { return ovl_path_check_dir_xattr(ofs, path, OVL_XATTR_OPAQUE); } diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index 87759165d32b6b912ae4223e913b020db57e6d51..eee8f08d32b639b1fe69fe28fae89353d060666e 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -208,7 +208,7 @@ static inline int ovl_do_symlink(struct ovl_fs *ofs, return err; } -static inline ssize_t ovl_do_getxattr(struct path *path, const char *name, +static inline ssize_t ovl_do_getxattr(const struct path *path, const char *name, void *value, size_t size) { int err, len; @@ -238,7 +238,7 @@ static inline ssize_t ovl_getxattr_upper(struct ovl_fs *ofs, } static inline ssize_t ovl_path_getxattr(struct ovl_fs *ofs, - struct path *path, + const struct path *path, enum ovl_xattr ox, void *value, size_t size) { @@ -250,7 +250,7 @@ static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry, size_t size, int flags) { int err = vfs_setxattr(ovl_upper_mnt_userns(ofs), dentry, name, - (void *)value, size, flags); + value, size, flags); pr_debug("setxattr(%pd2, \"%s\", \"%*pE\", %zu, %d) = %i\n", dentry, name, min((int)size, 48), value, size, flags, err); @@ -310,14 +310,16 @@ static inline int ovl_do_whiteout(struct ovl_fs *ofs, return err; } -static inline struct dentry *ovl_do_tmpfile(struct ovl_fs *ofs, - struct dentry *dentry, umode_t mode) +static inline struct file *ovl_do_tmpfile(struct ovl_fs *ofs, + struct dentry *dentry, umode_t mode) { - struct dentry *ret = vfs_tmpfile(ovl_upper_mnt_userns(ofs), dentry, mode, 0); - int err = PTR_ERR_OR_ZERO(ret); + struct path path = { .mnt = ovl_upper_mnt(ofs), .dentry = dentry }; + struct file *file = vfs_tmpfile_open(ovl_upper_mnt_userns(ofs), &path, mode, + O_LARGEFILE | O_WRONLY, current_cred()); + int err = PTR_ERR_OR_ZERO(file); pr_debug("tmpfile(%pd2, 0%o) = %i\n", dentry, mode, err); - return ret; + return file; } static inline struct dentry *ovl_lookup_upper(struct ovl_fs *ofs, @@ -401,13 +403,13 @@ void ovl_inode_update(struct inode *inode, struct dentry *upperdentry); void ovl_dir_modified(struct dentry *dentry, bool impurity); u64 ovl_dentry_version_get(struct dentry *dentry); bool ovl_is_whiteout(struct dentry *dentry); -struct file *ovl_path_open(struct path *path, int flags); +struct file *ovl_path_open(const struct path *path, int flags); int ovl_copy_up_start(struct dentry *dentry, int flags); void ovl_copy_up_end(struct dentry *dentry); bool ovl_already_copied_up(struct dentry *dentry, int flags); -bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, struct path *path, +bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, const struct path *path, enum ovl_xattr ox); -bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path); +bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path); static inline bool ovl_check_origin_xattr(struct ovl_fs *ofs, struct dentry *upperdentry) @@ -430,9 +432,9 @@ bool ovl_need_index(struct dentry *dentry); int ovl_nlink_start(struct dentry *dentry); void ovl_nlink_end(struct dentry *dentry); int ovl_lock_rename_workdir(struct dentry *workdir, struct dentry *upperdir); -int ovl_check_metacopy_xattr(struct ovl_fs *ofs, struct path *path); +int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path); bool ovl_is_metacopy_dentry(struct dentry *dentry); -char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct path *path, int padding); +char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding); int ovl_sync_status(struct ovl_fs *ofs); static inline void ovl_set_flag(unsigned long flag, struct inode *inode) @@ -556,7 +558,7 @@ void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper, struct list_head *list); void ovl_cache_free(struct list_head *list); void ovl_dir_cache_free(struct inode *inode); -int ovl_check_d_type_supported(struct path *realpath); +int ovl_check_d_type_supported(const struct path *realpath); int ovl_workdir_cleanup(struct ovl_fs *ofs, struct inode *dir, struct vfsmount *mnt, struct dentry *dentry, int level); int ovl_indexdir_cleanup(struct ovl_fs *ofs); @@ -673,8 +675,8 @@ struct dentry *ovl_create_temp(struct ovl_fs *ofs, struct dentry *workdir, extern const struct file_operations ovl_file_operations; int __init ovl_aio_request_cache_init(void); void ovl_aio_request_cache_destroy(void); -int ovl_real_fileattr_get(struct path *realpath, struct fileattr *fa); -int ovl_real_fileattr_set(struct path *realpath, struct fileattr *fa); +int ovl_real_fileattr_get(const struct path *realpath, struct fileattr *fa); +int ovl_real_fileattr_set(const struct path *realpath, struct fileattr *fa); int ovl_fileattr_get(struct dentry *dentry, struct fileattr *fa); int ovl_fileattr_set(struct user_namespace *mnt_userns, struct dentry *dentry, struct fileattr *fa); @@ -683,7 +685,7 @@ int ovl_fileattr_set(struct user_namespace *mnt_userns, int ovl_copy_up(struct dentry *dentry); int ovl_copy_up_with_data(struct dentry *dentry); int ovl_maybe_copy_up(struct dentry *dentry, int flags); -int ovl_copy_xattr(struct super_block *sb, struct path *path, struct dentry *new); +int ovl_copy_xattr(struct super_block *sb, const struct path *path, struct dentry *new); int ovl_set_attr(struct ovl_fs *ofs, struct dentry *upper, struct kstat *stat); struct ovl_fh *ovl_encode_real_fh(struct ovl_fs *ofs, struct dentry *real, bool is_upper); diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c index 78f62cc1797becb9caac4d1842a5278f81cf2f71..2b210640036c4b49808b29631b7b27f078df5960 100644 --- a/fs/overlayfs/readdir.c +++ b/fs/overlayfs/readdir.c @@ -170,7 +170,7 @@ static struct ovl_cache_entry *ovl_cache_entry_new(struct ovl_readdir_data *rdd, return p; } -static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd, +static bool ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd, const char *name, int len, u64 ino, unsigned int d_type) { @@ -179,22 +179,22 @@ static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd, struct ovl_cache_entry *p; if (ovl_cache_entry_find_link(name, len, &newp, &parent)) - return 0; + return true; p = ovl_cache_entry_new(rdd, name, len, ino, d_type); if (p == NULL) { rdd->err = -ENOMEM; - return -ENOMEM; + return false; } list_add_tail(&p->l_node, rdd->list); rb_link_node(&p->node, parent, newp); rb_insert_color(&p->node, rdd->root); - return 0; + return true; } -static int ovl_fill_lowest(struct ovl_readdir_data *rdd, +static bool ovl_fill_lowest(struct ovl_readdir_data *rdd, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -211,7 +211,7 @@ static int ovl_fill_lowest(struct ovl_readdir_data *rdd, list_add_tail(&p->l_node, &rdd->middle); } - return rdd->err; + return rdd->err == 0; } void ovl_cache_free(struct list_head *list) @@ -250,7 +250,7 @@ static void ovl_cache_put(struct ovl_dir_file *od, struct dentry *dentry) } } -static int ovl_fill_merge(struct dir_context *ctx, const char *name, +static bool ovl_fill_merge(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -264,7 +264,7 @@ static int ovl_fill_merge(struct dir_context *ctx, const char *name, return ovl_fill_lowest(rdd, name, namelen, offset, ino, d_type); } -static int ovl_check_whiteouts(struct path *path, struct ovl_readdir_data *rdd) +static int ovl_check_whiteouts(const struct path *path, struct ovl_readdir_data *rdd) { int err; struct ovl_cache_entry *p; @@ -291,7 +291,7 @@ static int ovl_check_whiteouts(struct path *path, struct ovl_readdir_data *rdd) return err; } -static inline int ovl_dir_read(struct path *realpath, +static inline int ovl_dir_read(const struct path *realpath, struct ovl_readdir_data *rdd) { struct file *realfile; @@ -455,7 +455,7 @@ static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid, * copy up origin, call vfs_getattr() on the overlay entry to make * sure that d_ino will be consistent with st_ino from stat(2). */ -static int ovl_cache_update_ino(struct path *path, struct ovl_cache_entry *p) +static int ovl_cache_update_ino(const struct path *path, struct ovl_cache_entry *p) { struct dentry *dir = path->dentry; @@ -528,7 +528,7 @@ fail: goto out; } -static int ovl_fill_plain(struct dir_context *ctx, const char *name, +static bool ovl_fill_plain(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -540,14 +540,14 @@ static int ovl_fill_plain(struct dir_context *ctx, const char *name, p = ovl_cache_entry_new(rdd, name, namelen, ino, d_type); if (p == NULL) { rdd->err = -ENOMEM; - return -ENOMEM; + return false; } list_add_tail(&p->l_node, rdd->list); - return 0; + return true; } -static int ovl_dir_read_impure(struct path *path, struct list_head *list, +static int ovl_dir_read_impure(const struct path *path, struct list_head *list, struct rb_root *root) { int err; @@ -592,7 +592,7 @@ static int ovl_dir_read_impure(struct path *path, struct list_head *list, return 0; } -static struct ovl_dir_cache *ovl_cache_get_impure(struct path *path) +static struct ovl_dir_cache *ovl_cache_get_impure(const struct path *path) { int res; struct dentry *dentry = path->dentry; @@ -648,7 +648,7 @@ struct ovl_readdir_translate { bool xinowarn; }; -static int ovl_fill_real(struct dir_context *ctx, const char *name, +static bool ovl_fill_real(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -834,7 +834,7 @@ out_unlock: } static struct file *ovl_dir_open_realfile(const struct file *file, - struct path *realpath) + const struct path *realpath) { struct file *res; const struct cred *old_cred; @@ -1027,7 +1027,7 @@ void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper, inode_unlock(upper->d_inode); } -static int ovl_check_d_type(struct dir_context *ctx, const char *name, +static bool ovl_check_d_type(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -1036,19 +1036,19 @@ static int ovl_check_d_type(struct dir_context *ctx, const char *name, /* Even if d_type is not supported, DT_DIR is returned for . and .. */ if (!strncmp(name, ".", namelen) || !strncmp(name, "..", namelen)) - return 0; + return true; if (d_type != DT_UNKNOWN) rdd->d_type_supported = true; - return 0; + return true; } /* * Returns 1 if d_type is supported, 0 not supported/unknown. Negative values * if error is encountered. */ -int ovl_check_d_type_supported(struct path *realpath) +int ovl_check_d_type_supported(const struct path *realpath) { int err; struct ovl_readdir_data rdd = { @@ -1065,7 +1065,7 @@ int ovl_check_d_type_supported(struct path *realpath) #define OVL_INCOMPATDIR_NAME "incompat" -static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, struct path *path, +static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, const struct path *path, int level) { int err; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index ec746d447f1bb147d637ffca6236eac13b348599..a29a8afe9b26226dd94423e75d3287294a18e25b 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "overlayfs.h" MODULE_AUTHOR("Miklos Szeredi "); @@ -908,7 +909,7 @@ static int ovl_mount_dir(const char *name, struct path *path) return err; } -static int ovl_check_namelen(struct path *path, struct ovl_fs *ofs, +static int ovl_check_namelen(const struct path *path, struct ovl_fs *ofs, const char *name) { struct kstatfs statfs; @@ -1022,7 +1023,20 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler, /* Check that everything is OK before copy-up */ if (value) { - acl = posix_acl_from_xattr(&init_user_ns, value, size); + /* The above comment can be understood in two ways: + * + * 1. We just want to check whether the basic POSIX ACL format + * is ok. For example, if the header is correct and the size + * is sane. + * 2. We want to know whether the ACL_{GROUP,USER} entries can + * be mapped according to the underlying filesystem. + * + * Currently, we only check 1. If we wanted to check 2. we + * would need to pass the mnt_userns and the fs_userns of the + * underlying filesystem. But frankly, I think checking 1. is + * enough to start the copy-up. + */ + acl = vfs_set_acl_prepare(&init_user_ns, &init_user_ns, value, size); if (IS_ERR(acl)) return PTR_ERR(acl); } @@ -1353,10 +1367,11 @@ static int ovl_create_volatile_dirty(struct ovl_fs *ofs) } static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, - struct path *workpath) + const struct path *workpath) { struct vfsmount *mnt = ovl_upper_mnt(ofs); - struct dentry *temp, *workdir; + struct dentry *workdir; + struct file *tmpfile; bool rename_whiteout; bool d_type; int fh_type; @@ -1392,10 +1407,10 @@ static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, pr_warn("upper fs needs to support d_type.\n"); /* Check if upper/work fs supports O_TMPFILE */ - temp = ovl_do_tmpfile(ofs, ofs->workdir, S_IFREG | 0); - ofs->tmpfile = !IS_ERR(temp); + tmpfile = ovl_do_tmpfile(ofs, ofs->workdir, S_IFREG | 0); + ofs->tmpfile = !IS_ERR(tmpfile); if (ofs->tmpfile) - dput(temp); + fput(tmpfile); else pr_warn("upper fs does not support tmpfile.\n"); @@ -1482,7 +1497,7 @@ out: } static int ovl_get_workdir(struct super_block *sb, struct ovl_fs *ofs, - struct path *upperpath) + const struct path *upperpath) { int err; struct path workpath = { }; @@ -1525,7 +1540,7 @@ out: } static int ovl_get_indexdir(struct super_block *sb, struct ovl_fs *ofs, - struct ovl_entry *oe, struct path *upperpath) + struct ovl_entry *oe, const struct path *upperpath) { struct vfsmount *mnt = ovl_upper_mnt(ofs); struct dentry *indexdir; diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 87f811c089e4ff395ad8722ad7a0222504bf5b6b..81a57a8d80d9a4d66e885a6ae5b668f3dd3bc4b8 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -490,7 +490,7 @@ bool ovl_is_whiteout(struct dentry *dentry) return inode && IS_WHITEOUT(inode); } -struct file *ovl_path_open(struct path *path, int flags) +struct file *ovl_path_open(const struct path *path, int flags) { struct inode *inode = d_inode(path->dentry); struct user_namespace *real_mnt_userns = mnt_user_ns(path->mnt); @@ -578,7 +578,7 @@ void ovl_copy_up_end(struct dentry *dentry) ovl_inode_unlock(d_inode(dentry)); } -bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path) +bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path) { int res; @@ -591,7 +591,7 @@ bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path) return false; } -bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, struct path *path, +bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, const struct path *path, enum ovl_xattr ox) { int res; @@ -971,7 +971,7 @@ err: } /* err < 0, 0 if no metacopy xattr, 1 if metacopy xattr found */ -int ovl_check_metacopy_xattr(struct ovl_fs *ofs, struct path *path) +int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path) { int res; @@ -1015,7 +1015,7 @@ bool ovl_is_metacopy_dentry(struct dentry *dentry) return (oe->numlower > 1); } -char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct path *path, int padding) +char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding) { int res; char *s, *next, *buf = NULL; diff --git a/fs/pipe.c b/fs/pipe.c index 74ae9fafd25a145de0a1377a2c73940e7a404722..42c7ff41c2dba2904cf6c8f2d10ea6984a402823 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -860,7 +860,7 @@ static struct vfsmount *pipe_mnt __read_mostly; */ static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) { - return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]", + return dynamic_dname(buffer, buflen, "pipe:[%lu]", d_inode(dentry)->i_ino); } diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 1d17d7b13dcd0c2f0d8a26c38f9024eb084b6d39..74dc0f571dc90e943d18e50d4ab8ec960f985493 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -24,6 +24,7 @@ #include #include #include +#include static struct posix_acl **acl_by_type(struct inode *inode, int type) { @@ -361,6 +362,7 @@ posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, const struct posix_acl *acl, int want) { const struct posix_acl_entry *pa, *pe, *mask_obj; + struct user_namespace *fs_userns = i_user_ns(inode); int found = 0; vfsuid_t vfsuid; vfsgid_t vfsgid; @@ -376,7 +378,7 @@ posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, goto check_perm; break; case ACL_USER: - vfsuid = make_vfsuid(mnt_userns, &init_user_ns, + vfsuid = make_vfsuid(mnt_userns, fs_userns, pa->e_uid); if (vfsuid_eq_kuid(vfsuid, current_fsuid())) goto mask; @@ -390,7 +392,7 @@ posix_acl_permission(struct user_namespace *mnt_userns, struct inode *inode, } break; case ACL_GROUP: - vfsgid = make_vfsgid(mnt_userns, &init_user_ns, + vfsgid = make_vfsgid(mnt_userns, fs_userns, pa->e_gid); if (vfsgid_in_group_p(vfsgid)) { found = 1; @@ -709,9 +711,9 @@ EXPORT_SYMBOL(posix_acl_update_mode); /* * Fix up the uids and gids in posix acl extended attributes in place. */ -static int posix_acl_fix_xattr_common(void *value, size_t size) +static int posix_acl_fix_xattr_common(const void *value, size_t size) { - struct posix_acl_xattr_header *header = value; + const struct posix_acl_xattr_header *header = value; int count; if (!header) @@ -719,13 +721,13 @@ static int posix_acl_fix_xattr_common(void *value, size_t size) if (size < sizeof(struct posix_acl_xattr_header)) return -EINVAL; if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION)) - return -EINVAL; + return -EOPNOTSUPP; count = posix_acl_xattr_count(size); if (count < 0) return -EINVAL; if (count == 0) - return -EINVAL; + return 0; return count; } @@ -736,6 +738,7 @@ void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns, { struct posix_acl_xattr_header *header = value; struct posix_acl_xattr_entry *entry = (void *)(header + 1), *end; + struct user_namespace *fs_userns = i_user_ns(inode); int count; vfsuid_t vfsuid; vfsgid_t vfsgid; @@ -746,20 +749,20 @@ void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns, return; count = posix_acl_fix_xattr_common(value, size); - if (count < 0) + if (count <= 0) return; for (end = entry + count; entry != end; entry++) { switch (le16_to_cpu(entry->e_tag)) { case ACL_USER: uid = make_kuid(&init_user_ns, le32_to_cpu(entry->e_id)); - vfsuid = make_vfsuid(mnt_userns, &init_user_ns, uid); + vfsuid = make_vfsuid(mnt_userns, fs_userns, uid); entry->e_id = cpu_to_le32(from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid))); break; case ACL_GROUP: gid = make_kgid(&init_user_ns, le32_to_cpu(entry->e_id)); - vfsgid = make_vfsgid(mnt_userns, &init_user_ns, gid); + vfsgid = make_vfsgid(mnt_userns, fs_userns, gid); entry->e_id = cpu_to_le32(from_kgid(&init_user_ns, vfsgid_into_kgid(vfsgid))); break; @@ -769,45 +772,6 @@ void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns, } } -void posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns, - const struct inode *inode, - void *value, size_t size) -{ - struct posix_acl_xattr_header *header = value; - struct posix_acl_xattr_entry *entry = (void *)(header + 1), *end; - int count; - vfsuid_t vfsuid; - vfsgid_t vfsgid; - kuid_t uid; - kgid_t gid; - - if (no_idmapping(mnt_userns, i_user_ns(inode))) - return; - - count = posix_acl_fix_xattr_common(value, size); - if (count < 0) - return; - - for (end = entry + count; entry != end; entry++) { - switch (le16_to_cpu(entry->e_tag)) { - case ACL_USER: - uid = make_kuid(&init_user_ns, le32_to_cpu(entry->e_id)); - vfsuid = VFSUIDT_INIT(uid); - uid = from_vfsuid(mnt_userns, &init_user_ns, vfsuid); - entry->e_id = cpu_to_le32(from_kuid(&init_user_ns, uid)); - break; - case ACL_GROUP: - gid = make_kgid(&init_user_ns, le32_to_cpu(entry->e_id)); - vfsgid = VFSGIDT_INIT(gid); - gid = from_vfsgid(mnt_userns, &init_user_ns, vfsgid); - entry->e_id = cpu_to_le32(from_kgid(&init_user_ns, gid)); - break; - default: - break; - } - } -} - static void posix_acl_fix_xattr_userns( struct user_namespace *to, struct user_namespace *from, void *value, size_t size) @@ -819,7 +783,7 @@ static void posix_acl_fix_xattr_userns( kgid_t gid; count = posix_acl_fix_xattr_common(value, size); - if (count < 0) + if (count <= 0) return; for (end = entry + count; entry != end; entry++) { @@ -854,12 +818,32 @@ void posix_acl_fix_xattr_to_user(void *value, size_t size) posix_acl_fix_xattr_userns(user_ns, &init_user_ns, value, size); } -/* - * Convert from extended attribute to in-memory representation. +/** + * make_posix_acl - convert POSIX ACLs from uapi to VFS format using the + * provided callbacks to map ACL_{GROUP,USER} entries into the + * appropriate format + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @value: the uapi representation of POSIX ACLs + * @size: the size of @void + * @uid_cb: callback to use for mapping the uid stored in ACL_USER entries + * @gid_cb: callback to use for mapping the gid stored in ACL_GROUP entries + * + * The make_posix_acl() helper is an abstraction to translate from uapi format + * into the VFS format allowing the caller to specific callbacks to map + * ACL_{GROUP,USER} entries into the expected format. This is used in + * posix_acl_from_xattr() and vfs_set_acl_prepare() and avoids pointless code + * duplication. + * + * Return: Allocated struct posix_acl on success, NULL for a valid header but + * without actual POSIX ACL entries, or ERR_PTR() encoded error code. */ -struct posix_acl * -posix_acl_from_xattr(struct user_namespace *user_ns, - const void *value, size_t size) +static struct posix_acl *make_posix_acl(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, const void *value, size_t size, + kuid_t (*uid_cb)(struct user_namespace *, struct user_namespace *, + const struct posix_acl_xattr_entry *), + kgid_t (*gid_cb)(struct user_namespace *, struct user_namespace *, + const struct posix_acl_xattr_entry *)) { const struct posix_acl_xattr_header *header = value; const struct posix_acl_xattr_entry *entry = (const void *)(header + 1), *end; @@ -867,16 +851,9 @@ posix_acl_from_xattr(struct user_namespace *user_ns, struct posix_acl *acl; struct posix_acl_entry *acl_e; - if (!value) - return NULL; - if (size < sizeof(struct posix_acl_xattr_header)) - return ERR_PTR(-EINVAL); - if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION)) - return ERR_PTR(-EOPNOTSUPP); - - count = posix_acl_xattr_count(size); + count = posix_acl_fix_xattr_common(value, size); if (count < 0) - return ERR_PTR(-EINVAL); + return ERR_PTR(count); if (count == 0) return NULL; @@ -897,16 +874,12 @@ posix_acl_from_xattr(struct user_namespace *user_ns, break; case ACL_USER: - acl_e->e_uid = - make_kuid(user_ns, - le32_to_cpu(entry->e_id)); + acl_e->e_uid = uid_cb(mnt_userns, fs_userns, entry); if (!uid_valid(acl_e->e_uid)) goto fail; break; case ACL_GROUP: - acl_e->e_gid = - make_kgid(user_ns, - le32_to_cpu(entry->e_id)); + acl_e->e_gid = gid_cb(mnt_userns, fs_userns, entry); if (!gid_valid(acl_e->e_gid)) goto fail; break; @@ -921,6 +894,181 @@ fail: posix_acl_release(acl); return ERR_PTR(-EINVAL); } + +/** + * vfs_set_acl_prepare_kuid - map ACL_USER uid according to mount- and + * filesystem idmapping + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @e: a ACL_USER entry in POSIX ACL uapi format + * + * The uid stored as ACL_USER entry in @e is a kuid_t stored as a raw {g,u}id + * value. The vfs_set_acl_prepare_kuid() will recover the kuid_t through + * KUIDT_INIT() and then map it according to the idmapped mount. The resulting + * kuid_t is the value which the filesystem can map up into a raw backing store + * id in the filesystem's idmapping. + * + * This is used in vfs_set_acl_prepare() to generate the proper VFS + * representation of POSIX ACLs with ACL_USER entries during setxattr(). + * + * Return: A kuid in @fs_userns for the uid stored in @e. + */ +static inline kuid_t +vfs_set_acl_prepare_kuid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const struct posix_acl_xattr_entry *e) +{ + kuid_t kuid = KUIDT_INIT(le32_to_cpu(e->e_id)); + return from_vfsuid(mnt_userns, fs_userns, VFSUIDT_INIT(kuid)); +} + +/** + * vfs_set_acl_prepare_kgid - map ACL_GROUP gid according to mount- and + * filesystem idmapping + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @e: a ACL_GROUP entry in POSIX ACL uapi format + * + * The gid stored as ACL_GROUP entry in @e is a kgid_t stored as a raw {g,u}id + * value. The vfs_set_acl_prepare_kgid() will recover the kgid_t through + * KGIDT_INIT() and then map it according to the idmapped mount. The resulting + * kgid_t is the value which the filesystem can map up into a raw backing store + * id in the filesystem's idmapping. + * + * This is used in vfs_set_acl_prepare() to generate the proper VFS + * representation of POSIX ACLs with ACL_GROUP entries during setxattr(). + * + * Return: A kgid in @fs_userns for the gid stored in @e. + */ +static inline kgid_t +vfs_set_acl_prepare_kgid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const struct posix_acl_xattr_entry *e) +{ + kgid_t kgid = KGIDT_INIT(le32_to_cpu(e->e_id)); + return from_vfsgid(mnt_userns, fs_userns, VFSGIDT_INIT(kgid)); +} + +/** + * vfs_set_acl_prepare - convert POSIX ACLs from uapi to VFS format taking + * mount and filesystem idmappings into account + * @mnt_userns: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @value: the uapi representation of POSIX ACLs + * @size: the size of @void + * + * When setting POSIX ACLs with ACL_{GROUP,USER} entries they need to be + * mapped according to the relevant mount- and filesystem idmapping. It is + * important that the ACL_{GROUP,USER} entries in struct posix_acl will be + * mapped into k{g,u}id_t that are supposed to be mapped up in the filesystem + * idmapping. This is crucial since the resulting struct posix_acl might be + * cached filesystem wide. The vfs_set_acl_prepare() function will take care to + * perform all necessary idmappings. + * + * Note, that since basically forever the {g,u}id values encoded as + * ACL_{GROUP,USER} entries in the uapi POSIX ACLs passed via @value contain + * values that have been mapped according to the caller's idmapping. In other + * words, POSIX ACLs passed in uapi format as @value during setxattr() contain + * {g,u}id values in their ACL_{GROUP,USER} entries that should actually have + * been stored as k{g,u}id_t. + * + * This means, vfs_set_acl_prepare() needs to first recover the k{g,u}id_t by + * calling K{G,U}IDT_INIT(). Afterwards they can be interpreted as vfs{g,u}id_t + * through from_vfs{g,u}id() to account for any idmapped mounts. The + * vfs_set_acl_prepare_k{g,u}id() helpers will take care to generate the + * correct k{g,u}id_t. + * + * The filesystem will then receive the POSIX ACLs ready to be cached + * filesystem wide and ready to be written to the backing store taking the + * filesystem's idmapping into account. + * + * Return: Allocated struct posix_acl on success, NULL for a valid header but + * without actual POSIX ACL entries, or ERR_PTR() encoded error code. + */ +struct posix_acl *vfs_set_acl_prepare(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const void *value, size_t size) +{ + return make_posix_acl(mnt_userns, fs_userns, value, size, + vfs_set_acl_prepare_kuid, + vfs_set_acl_prepare_kgid); +} +EXPORT_SYMBOL(vfs_set_acl_prepare); + +/** + * posix_acl_from_xattr_kuid - map ACL_USER uid into filesystem idmapping + * @mnt_userns: unused + * @fs_userns: the filesystem's idmapping + * @e: a ACL_USER entry in POSIX ACL uapi format + * + * Map the uid stored as ACL_USER entry in @e into the filesystem's idmapping. + * This is used in posix_acl_from_xattr() to generate the proper VFS + * representation of POSIX ACLs with ACL_USER entries. + * + * Return: A kuid in @fs_userns for the uid stored in @e. + */ +static inline kuid_t +posix_acl_from_xattr_kuid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const struct posix_acl_xattr_entry *e) +{ + return make_kuid(fs_userns, le32_to_cpu(e->e_id)); +} + +/** + * posix_acl_from_xattr_kgid - map ACL_GROUP gid into filesystem idmapping + * @mnt_userns: unused + * @fs_userns: the filesystem's idmapping + * @e: a ACL_GROUP entry in POSIX ACL uapi format + * + * Map the gid stored as ACL_GROUP entry in @e into the filesystem's idmapping. + * This is used in posix_acl_from_xattr() to generate the proper VFS + * representation of POSIX ACLs with ACL_GROUP entries. + * + * Return: A kgid in @fs_userns for the gid stored in @e. + */ +static inline kgid_t +posix_acl_from_xattr_kgid(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const struct posix_acl_xattr_entry *e) +{ + return make_kgid(fs_userns, le32_to_cpu(e->e_id)); +} + +/** + * posix_acl_from_xattr - convert POSIX ACLs from backing store to VFS format + * @fs_userns: the filesystem's idmapping + * @value: the uapi representation of POSIX ACLs + * @size: the size of @void + * + * Filesystems that store POSIX ACLs in the unaltered uapi format should use + * posix_acl_from_xattr() when reading them from the backing store and + * converting them into the struct posix_acl VFS format. The helper is + * specifically intended to be called from the ->get_acl() inode operation. + * + * The posix_acl_from_xattr() function will map the raw {g,u}id values stored + * in ACL_{GROUP,USER} entries into the filesystem idmapping in @fs_userns. The + * posix_acl_from_xattr_k{g,u}id() helpers will take care to generate the + * correct k{g,u}id_t. The returned struct posix_acl can be cached. + * + * Note that posix_acl_from_xattr() does not take idmapped mounts into account. + * If it did it calling is from the ->get_acl() inode operation would return + * POSIX ACLs mapped according to an idmapped mount which would mean that the + * value couldn't be cached for the filesystem. Idmapped mounts are taken into + * account on the fly during permission checking or right at the VFS - + * userspace boundary before reporting them to the user. + * + * Return: Allocated struct posix_acl on success, NULL for a valid header but + * without actual POSIX ACL entries, or ERR_PTR() encoded error code. + */ +struct posix_acl * +posix_acl_from_xattr(struct user_namespace *fs_userns, + const void *value, size_t size) +{ + return make_posix_acl(&init_user_ns, fs_userns, value, size, + posix_acl_from_xattr_kuid, + posix_acl_from_xattr_kgid); +} EXPORT_SYMBOL (posix_acl_from_xattr); /* @@ -1024,7 +1172,17 @@ posix_acl_xattr_set(const struct xattr_handler *handler, int ret; if (value) { - acl = posix_acl_from_xattr(&init_user_ns, value, size); + /* + * By the time we end up here the {g,u}ids stored in + * ACL_{GROUP,USER} have already been mapped according to the + * caller's idmapping. The vfs_set_acl_prepare() helper will + * recover them and take idmapped mounts into account. The + * filesystem will receive the POSIX ACLs in the correct + * format ready to be cached or written to the backing store + * taking the filesystem idmapping into account. + */ + acl = vfs_set_acl_prepare(mnt_userns, i_user_ns(inode), + value, size); if (IS_ERR(acl)) return PTR_ERR(acl); } @@ -1070,6 +1228,8 @@ int simple_set_acl(struct user_namespace *mnt_userns, struct inode *inode, } inode->i_ctime = current_time(inode); + if (IS_I_VERSION(inode)) + inode_inc_iversion(inode); set_cached_acl(inode, type, acl); return 0; } diff --git a/fs/proc/Kconfig b/fs/proc/Kconfig index c930001056f95d6b15f1131fed9b62d1c0999a3e..32b1116ae137c61bc0eb915498dab969b1f4c50c 100644 --- a/fs/proc/Kconfig +++ b/fs/proc/Kconfig @@ -92,6 +92,7 @@ config PROC_PAGE_MONITOR config PROC_CHILDREN bool "Include /proc//task//children file" + depends on PROC_FS default n help Provides a fast way to retrieve first level children pids of a task. See diff --git a/fs/proc/array.c b/fs/proc/array.c index 99fcbfda8e2593596a8d59593a9d06264c892dcb..49283b8103c7ea28809bc1ea4026272a0eebff59 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -279,7 +279,7 @@ static inline void task_sig(struct seq_file *m, struct task_struct *p) collect_sigign_sigcatch(p, &ignored, &caught); num_threads = get_nr_threads(p); rcu_read_lock(); /* FIXME: is this correct? */ - qsize = get_ucounts_value(task_ucounts(p), UCOUNT_RLIMIT_SIGPENDING); + qsize = get_rlimit_value(task_ucounts(p), UCOUNT_RLIMIT_SIGPENDING); rcu_read_unlock(); qlim = task_rlimit(p, RLIMIT_SIGPENDING); unlock_task_sighand(p, &flags); diff --git a/fs/proc/base.c b/fs/proc/base.c index 93f7e3d971e4bbc758165ac09710945a50975af3..9e479d7d202b12c0ddf53e00975858162156db8a 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -1761,7 +1761,7 @@ out: return ERR_PTR(error); } -static int do_proc_readlink(struct path *path, char __user *buffer, int buflen) +static int do_proc_readlink(const struct path *path, char __user *buffer, int buflen) { char *tmp = kmalloc(PATH_MAX, GFP_KERNEL); char *pathname; @@ -2350,6 +2350,7 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) GENRADIX(struct map_files_info) fa; struct map_files_info *p; int ret; + struct vma_iterator vmi; genradix_init(&fa); @@ -2388,7 +2389,9 @@ proc_map_files_readdir(struct file *file, struct dir_context *ctx) * routine might require mmap_lock taken in might_fault(). */ - for (vma = mm->mmap, pos = 2; vma; vma = vma->vm_next) { + pos = 2; + vma_iter_init(&vmi, mm, 0); + for_each_vma(vmi, vma) { if (!vma->vm_file) continue; if (++pos <= ctx->pos) @@ -2728,7 +2731,7 @@ static ssize_t proc_pid_attr_read(struct file * file, char __user * buf, return -ESRCH; length = security_getprocattr(task, PROC_I(inode)->op.lsm, - (char*)file->f_path.dentry->d_name.name, + file->f_path.dentry->d_name.name, &p); put_task_struct(task); if (length > 0) @@ -3196,6 +3199,19 @@ static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace * return 0; } +static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm; + + mm = get_task_mm(task); + if (mm) { + seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); + mmput(mm); + } + + return 0; +} #endif /* CONFIG_KSM */ #ifdef CONFIG_STACKLEAK_METRICS @@ -3331,6 +3347,7 @@ static const struct pid_entry tgid_base_stuff[] = { #endif #ifdef CONFIG_KSM ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), + ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), #endif }; @@ -3668,6 +3685,7 @@ static const struct pid_entry tid_base_stuff[] = { #endif #ifdef CONFIG_KSM ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), + ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), #endif }; diff --git a/fs/proc/devices.c b/fs/proc/devices.c index 837971e741097832d4033c24dc77749ae9bf82ce..fe7bfcb7d0494251de4efa9f9964043623f43b42 100644 --- a/fs/proc/devices.c +++ b/fs/proc/devices.c @@ -4,6 +4,7 @@ #include #include #include +#include "internal.h" static int devinfo_show(struct seq_file *f, void *v) { @@ -54,7 +55,10 @@ static const struct seq_operations devinfo_ops = { static int __init proc_devices_init(void) { - proc_create_seq("devices", 0, NULL, &devinfo_ops); + struct proc_dir_entry *pde; + + pde = proc_create_seq("devices", 0, NULL, &devinfo_ops); + pde_make_permanent(pde); return 0; } fs_initcall(proc_devices_init); diff --git a/fs/proc/internal.h b/fs/proc/internal.h index 06a80f78433d8b40f48aa9419f3b51c9e8f77f65..b701d0207edf098814a70400850e3675aee448f1 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -79,6 +79,11 @@ static inline bool pde_is_permanent(const struct proc_dir_entry *pde) return pde->flags & PROC_ENTRY_PERMANENT; } +static inline void pde_make_permanent(struct proc_dir_entry *pde) +{ + pde->flags |= PROC_ENTRY_PERMANENT; +} + extern struct kmem_cache *proc_dir_entry_cache; void pde_free(struct proc_dir_entry *pde); @@ -285,7 +290,7 @@ struct proc_maps_private { struct task_struct *task; struct mm_struct *mm; #ifdef CONFIG_MMU - struct vm_area_struct *tail_vma; + struct vma_iterator iter; #endif #ifdef CONFIG_NUMA struct mempolicy *task_mempolicy; diff --git a/fs/proc/kmsg.c b/fs/proc/kmsg.c index 592e6dc7c1102b54ea136b21e9b42b711ba09702..2fc92a13f9f8f46a1f51a6882cddbd0c1125e184 100644 --- a/fs/proc/kmsg.c +++ b/fs/proc/kmsg.c @@ -17,8 +17,6 @@ #include -extern wait_queue_head_t log_wait; - static int kmsg_open(struct inode * inode, struct file * file) { return do_syslog(SYSLOG_ACTION_OPEN, NULL, 0, SYSLOG_FROM_PROC); diff --git a/fs/proc/loadavg.c b/fs/proc/loadavg.c index f32878d9a39f394adfbbd2cfcc2b9535ccf20ff0..817981e57223ef62a362f9c57319c125e1e9b6b8 100644 --- a/fs/proc/loadavg.c +++ b/fs/proc/loadavg.c @@ -9,6 +9,7 @@ #include #include #include +#include "internal.h" static int loadavg_proc_show(struct seq_file *m, void *v) { @@ -27,7 +28,10 @@ static int loadavg_proc_show(struct seq_file *m, void *v) static int __init proc_loadavg_init(void) { - proc_create_single("loadavg", 0, NULL, loadavg_proc_show); + struct proc_dir_entry *pde; + + pde = proc_create_single("loadavg", 0, NULL, loadavg_proc_show); + pde_make_permanent(pde); return 0; } fs_initcall(proc_loadavg_init); diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 6e89f0e2fd20f101d84b3c27daf9045ea33af697..5101131e6047736fc3d1f6a532de2ae7a4dbeb22 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c @@ -115,6 +115,8 @@ static int meminfo_proc_show(struct seq_file *m, void *v) #endif show_val_kb(m, "PageTables: ", global_node_page_state(NR_PAGETABLE)); + show_val_kb(m, "SecPageTables: ", + global_node_page_state(NR_SECONDARY_PAGETABLE)); show_val_kb(m, "NFS_Unstable: ", 0); show_val_kb(m, "Bounce: ", @@ -162,7 +164,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v) static int __init proc_meminfo_init(void) { - proc_create_single("meminfo", 0, NULL, meminfo_proc_show); + struct proc_dir_entry *pde; + + pde = proc_create_single("meminfo", 0, NULL, meminfo_proc_show); + pde_make_permanent(pde); return 0; } fs_initcall(proc_meminfo_init); diff --git a/fs/proc/page.c b/fs/proc/page.c index a2873a617ae86232c12376064cbc8ffc4df59ab8..f2273b164535bb33490c021a3bfbbb14a1dc9906 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -91,6 +91,7 @@ static ssize_t kpagecount_read(struct file *file, char __user *buf, } static const struct proc_ops kpagecount_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_lseek = mem_lseek, .proc_read = kpagecount_read, }; @@ -268,6 +269,7 @@ static ssize_t kpageflags_read(struct file *file, char __user *buf, } static const struct proc_ops kpageflags_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_lseek = mem_lseek, .proc_read = kpageflags_read, }; @@ -322,6 +324,7 @@ static ssize_t kpagecgroup_read(struct file *file, char __user *buf, } static const struct proc_ops kpagecgroup_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, .proc_lseek = mem_lseek, .proc_read = kpagecgroup_read, }; diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 021e83fe831f7cc3b90c75b5bc91b0f4e1ea9d9d..48f2d60bd78a2f998ae83975588ca292f14c5428 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -28,13 +28,6 @@ static const struct inode_operations proc_sys_inode_operations; static const struct file_operations proc_sys_dir_file_operations; static const struct inode_operations proc_sys_dir_operations; -/* shared constants to be used in various sysctls */ -const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; -EXPORT_SYMBOL(sysctl_vals); - -const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; -EXPORT_SYMBOL_GPL(sysctl_long_vals); - /* Support for permanently empty directories */ struct ctl_table sysctl_mount_point[] = { @@ -1246,7 +1239,7 @@ static bool get_links(struct ctl_dir *dir, static int insert_links(struct ctl_table_header *head) { struct ctl_table_set *root_set = &sysctl_table_root.default_set; - struct ctl_dir *core_parent = NULL; + struct ctl_dir *core_parent; struct ctl_table_header *links; int err; diff --git a/fs/proc/softirqs.c b/fs/proc/softirqs.c index 12901dcf57e2bdce9610e8979f1f1d3781e55bc1..f4616083faef3bbb9e00e64d212d67cd650d2a1c 100644 --- a/fs/proc/softirqs.c +++ b/fs/proc/softirqs.c @@ -3,6 +3,7 @@ #include #include #include +#include "internal.h" /* * /proc/softirqs ... display the number of softirqs @@ -27,7 +28,10 @@ static int show_softirqs(struct seq_file *p, void *v) static int __init proc_softirqs_init(void) { - proc_create_single("softirqs", 0, NULL, show_softirqs); + struct proc_dir_entry *pde; + + pde = proc_create_single("softirqs", 0, NULL, show_softirqs); + pde_make_permanent(pde); return 0; } fs_initcall(proc_softirqs_init); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index a3398d0f1927f5461b0888fafefa9b02479e7546..8b4f3073f8f55e0bfc920f8c962d0d7dd3932c5c 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include #include #include #include @@ -124,12 +123,26 @@ static void release_task_mempolicy(struct proc_maps_private *priv) } #endif +static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, + loff_t *ppos) +{ + struct vm_area_struct *vma = vma_next(&priv->iter); + + if (vma) { + *ppos = vma->vm_start; + } else { + *ppos = -2UL; + vma = get_gate_vma(priv->mm); + } + + return vma; +} + static void *m_start(struct seq_file *m, loff_t *ppos) { struct proc_maps_private *priv = m->private; unsigned long last_addr = *ppos; struct mm_struct *mm; - struct vm_area_struct *vma; /* See m_next(). Zero at the start or after lseek. */ if (last_addr == -1UL) @@ -153,31 +166,21 @@ static void *m_start(struct seq_file *m, loff_t *ppos) return ERR_PTR(-EINTR); } + vma_iter_init(&priv->iter, mm, last_addr); hold_task_mempolicy(priv); - priv->tail_vma = get_gate_vma(mm); - - vma = find_vma(mm, last_addr); - if (vma) - return vma; + if (last_addr == -2UL) + return get_gate_vma(mm); - return priv->tail_vma; + return proc_get_vma(priv, ppos); } static void *m_next(struct seq_file *m, void *v, loff_t *ppos) { - struct proc_maps_private *priv = m->private; - struct vm_area_struct *next, *vma = v; - - if (vma == priv->tail_vma) - next = NULL; - else if (vma->vm_next) - next = vma->vm_next; - else - next = priv->tail_vma; - - *ppos = next ? next->vm_start : -1UL; - - return next; + if (*ppos == -2UL) { + *ppos = -1UL; + return NULL; + } + return proc_get_vma(m->private, ppos); } static void m_stop(struct seq_file *m, void *v) @@ -527,10 +530,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, struct vm_area_struct *vma = walk->vma; bool locked = !!(vma->vm_flags & VM_LOCKED); struct page *page = NULL; - bool migration = false; + bool migration = false, young = false, dirty = false; if (pte_present(*pte)) { page = vm_normal_page(vma, addr, *pte); + young = pte_young(*pte); + dirty = pte_dirty(*pte); } else if (is_swap_pte(*pte)) { swp_entry_t swpent = pte_to_swp_entry(*pte); @@ -560,8 +565,7 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; - smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), - locked, migration); + smaps_account(mss, page, false, young, dirty, locked, migration); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -863,7 +867,7 @@ static int show_smap(struct seq_file *m, void *v) __show_smap(m, &mss, false); seq_printf(m, "THPeligible: %d\n", - hugepage_vma_check(vma, vma->vm_flags, true, false)); + hugepage_vma_check(vma, vma->vm_flags, true, false, true)); if (arch_pkeys_enabled()) seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); @@ -876,16 +880,16 @@ static int show_smaps_rollup(struct seq_file *m, void *v) { struct proc_maps_private *priv = m->private; struct mem_size_stats mss; - struct mm_struct *mm; + struct mm_struct *mm = priv->mm; struct vm_area_struct *vma; - unsigned long last_vma_end = 0; + unsigned long vma_start = 0, last_vma_end = 0; int ret = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); priv->task = get_proc_task(priv->inode); if (!priv->task) return -ESRCH; - mm = priv->mm; if (!mm || !mmget_not_zero(mm)) { ret = -ESRCH; goto out_put_task; @@ -898,8 +902,13 @@ static int show_smaps_rollup(struct seq_file *m, void *v) goto out_put_mm; hold_task_mempolicy(priv); + vma = mas_find(&mas, 0); - for (vma = priv->mm->mmap; vma;) { + if (unlikely(!vma)) + goto empty_set; + + vma_start = vma->vm_start; + do { smap_gather_stats(vma, &mss, 0); last_vma_end = vma->vm_end; @@ -908,6 +917,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) * access it for write request. */ if (mmap_lock_is_contended(mm)) { + mas_pause(&mas); mmap_read_unlock(mm); ret = mmap_read_lock_killable(mm); if (ret) { @@ -951,7 +961,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) * contains last_vma_end. * Iterate VMA' from last_vma_end. */ - vma = find_vma(mm, last_vma_end - 1); + vma = mas_find(&mas, ULONG_MAX); /* Case 3 above */ if (!vma) break; @@ -965,11 +975,10 @@ static int show_smaps_rollup(struct seq_file *m, void *v) smap_gather_stats(vma, &mss, last_vma_end); } /* Case 2 above */ - vma = vma->vm_next; - } + } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); - show_vma_header_prefix(m, priv->mm->mmap->vm_start, - last_vma_end, 0, 0, 0, 0); +empty_set: + show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); seq_pad(m, ' '); seq_puts(m, "[rollup]\n"); @@ -1262,6 +1271,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, return -ESRCH; mm = get_task_mm(task); if (mm) { + MA_STATE(mas, &mm->mm_mt, 0, 0); struct mmu_notifier_range range; struct clear_refs_private cp = { .type = type, @@ -1281,7 +1291,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, } if (type == CLEAR_REFS_SOFT_DIRTY) { - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (!(vma->vm_flags & VM_SOFTDIRTY)) continue; vma->vm_flags &= ~VM_SOFTDIRTY; @@ -1293,8 +1303,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf, 0, NULL, mm, 0, -1UL); mmu_notifier_invalidate_range_start(&range); } - walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops, - &cp); + walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); if (type == CLEAR_REFS_SOFT_DIRTY) { mmu_notifier_invalidate_range_end(&range); flush_tlb_mm(mm); @@ -1417,9 +1426,19 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_swp_uffd_wp(pte)) flags |= PM_UFFD_WP; entry = pte_to_swp_entry(pte); - if (pm->show_pfn) + if (pm->show_pfn) { + pgoff_t offset; + /* + * For PFN swap offsets, keeping the offset field + * to be PFN only to be compatible with old smaps. + */ + if (is_pfn_swap_entry(entry)) + offset = swp_offset_pfn(entry); + else + offset = swp_offset(entry); frame = swp_type(entry) | - (swp_offset(entry) << MAX_SWAPFILES_SHIFT); + (offset << MAX_SWAPFILES_SHIFT); + } flags |= PM_SWAP; migration = is_migration_entry(entry); if (is_pfn_swap_entry(entry)) @@ -1476,7 +1495,11 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, unsigned long offset; if (pm->show_pfn) { - offset = swp_offset(entry) + + if (is_pfn_swap_entry(entry)) + offset = swp_offset_pfn(entry); + else + offset = swp_offset(entry); + offset = offset + ((addr & ~PMD_MASK) >> PAGE_SHIFT); frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c index a6d21fc0033c6440a48aeb9740cf05a1437eae4f..2fd06f52b6a44825e56082ad6cb8193b4a701aea 100644 --- a/fs/proc/task_nommu.c +++ b/fs/proc/task_nommu.c @@ -20,15 +20,13 @@ */ void task_mem(struct seq_file *m, struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; struct vm_region *region; - struct rb_node *p; unsigned long bytes = 0, sbytes = 0, slack = 0, size; - - mmap_read_lock(mm); - for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { - vma = rb_entry(p, struct vm_area_struct, vm_rb); + mmap_read_lock(mm); + for_each_vma(vmi, vma) { bytes += kobjsize(vma); region = vma->vm_region; @@ -82,15 +80,13 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - struct rb_node *p; unsigned long vsize = 0; mmap_read_lock(mm); - for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { - vma = rb_entry(p, struct vm_area_struct, vm_rb); + for_each_vma(vmi, vma) vsize += vma->vm_end - vma->vm_start; - } mmap_read_unlock(mm); return vsize; } @@ -99,14 +95,13 @@ unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; struct vm_region *region; - struct rb_node *p; unsigned long size = kobjsize(mm); mmap_read_lock(mm); - for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) { - vma = rb_entry(p, struct vm_area_struct, vm_rb); + for_each_vma(vmi, vma) { size += kobjsize(vma); region = vma->vm_region; if (region) { @@ -190,17 +185,19 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) */ static int show_map(struct seq_file *m, void *_p) { - struct rb_node *p = _p; - - return nommu_vma_show(m, rb_entry(p, struct vm_area_struct, vm_rb)); + return nommu_vma_show(m, _p); } static void *m_start(struct seq_file *m, loff_t *pos) { struct proc_maps_private *priv = m->private; struct mm_struct *mm; - struct rb_node *p; - loff_t n = *pos; + struct vm_area_struct *vma; + unsigned long addr = *pos; + + /* See m_next(). Zero at the start or after lseek. */ + if (addr == -1UL) + return NULL; /* pin the task and mm whilst we play with them */ priv->task = get_proc_task(priv->inode); @@ -216,10 +213,10 @@ static void *m_start(struct seq_file *m, loff_t *pos) return ERR_PTR(-EINTR); } - /* start from the Nth VMA */ - for (p = rb_first(&mm->mm_rb); p; p = rb_next(p)) - if (n-- == 0) - return p; + /* start the next element from addr */ + vma = find_vma(mm, addr); + if (vma) + return vma; mmap_read_unlock(mm); mmput(mm); @@ -242,10 +239,10 @@ static void m_stop(struct seq_file *m, void *_vml) static void *m_next(struct seq_file *m, void *_p, loff_t *pos) { - struct rb_node *p = _p; + struct vm_area_struct *vma = _p; - (*pos)++; - return p ? rb_next(p) : NULL; + *pos = vma->vm_end; + return find_vma(vma->vm_mm, vma->vm_end); } static const struct seq_operations proc_pid_maps_ops = { diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c index deb99bc9b7e6b009b8f2210bdf6a0959bfcbe9ed..b5343d209381aae892d2423602607f1ce05e479e 100644 --- a/fs/proc/uptime.c +++ b/fs/proc/uptime.c @@ -7,6 +7,7 @@ #include #include #include +#include "internal.h" static int uptime_proc_show(struct seq_file *m, void *v) { @@ -39,7 +40,10 @@ static int uptime_proc_show(struct seq_file *m, void *v) static int __init proc_uptime_init(void) { - proc_create_single("uptime", 0, NULL, uptime_proc_show); + struct proc_dir_entry *pde; + + pde = proc_create_single("uptime", 0, NULL, uptime_proc_show); + pde_make_permanent(pde); return 0; } fs_initcall(proc_uptime_init); diff --git a/fs/proc/version.c b/fs/proc/version.c index b449f186577f82e0f8e7776d906099ac423d1ed9..02e3c3cd4a9af1e82361e1469ad777e9e53c2f4a 100644 --- a/fs/proc/version.c +++ b/fs/proc/version.c @@ -5,6 +5,7 @@ #include #include #include +#include "internal.h" static int version_proc_show(struct seq_file *m, void *v) { @@ -17,7 +18,10 @@ static int version_proc_show(struct seq_file *m, void *v) static int __init proc_version_init(void) { - proc_create_single("version", 0, NULL, version_proc_show); + struct proc_dir_entry *pde; + + pde = proc_create_single("version", 0, NULL, version_proc_show); + pde_make_permanent(pde); return 0; } fs_initcall(proc_version_init); diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c index b2fd3c20e7c2183f1bcaecee080ecdf0f7efe80d..0c034ea399547b33f5864bde433389b6f012b107 100644 --- a/fs/pstore/platform.c +++ b/fs/pstore/platform.c @@ -28,14 +28,11 @@ #include #include #include -#include #include #include #include #include -#include - #include "internal.h" /* @@ -93,8 +90,7 @@ module_param(compress, charp, 0444); MODULE_PARM_DESC(compress, "compression to use"); /* Compression parameters */ -static struct crypto_acomp *tfm; -static struct acomp_req *creq; +static struct crypto_comp *tfm; struct pstore_zbackend { int (*zbufsize)(size_t size); @@ -272,21 +268,12 @@ static const struct pstore_zbackend zbackends[] = { static int pstore_compress(const void *in, void *out, unsigned int inlen, unsigned int outlen) { - struct scatterlist src, dst; int ret; if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS)) return -EINVAL; - sg_init_table(&src, 1); - sg_set_buf(&src, in, inlen); - - sg_init_table(&dst, 1); - sg_set_buf(&dst, out, outlen); - - acomp_request_set_params(creq, &src, &dst, inlen, outlen); - - ret = crypto_acomp_compress(creq); + ret = crypto_comp_compress(tfm, in, inlen, out, &outlen); if (ret) { pr_err("crypto_comp_compress failed, ret = %d!\n", ret); return ret; @@ -297,7 +284,7 @@ static int pstore_compress(const void *in, void *out, static void allocate_buf_for_compression(void) { - struct crypto_acomp *acomp; + struct crypto_comp *ctx; int size; char *buf; @@ -309,7 +296,7 @@ static void allocate_buf_for_compression(void) if (!psinfo || tfm) return; - if (!crypto_has_acomp(zbackend->name, 0, CRYPTO_ALG_ASYNC)) { + if (!crypto_has_comp(zbackend->name, 0, 0)) { pr_err("Unknown compression: %s\n", zbackend->name); return; } @@ -328,24 +315,16 @@ static void allocate_buf_for_compression(void) return; } - acomp = crypto_alloc_acomp(zbackend->name, 0, CRYPTO_ALG_ASYNC); - if (IS_ERR_OR_NULL(acomp)) { + ctx = crypto_alloc_comp(zbackend->name, 0, 0); + if (IS_ERR_OR_NULL(ctx)) { kfree(buf); pr_err("crypto_alloc_comp('%s') failed: %ld\n", zbackend->name, - PTR_ERR(acomp)); - return; - } - - creq = acomp_request_alloc(acomp); - if (!creq) { - crypto_free_acomp(acomp); - kfree(buf); - pr_err("acomp_request_alloc('%s') failed\n", zbackend->name); + PTR_ERR(ctx)); return; } /* A non-NULL big_oops_buf indicates compression is available. */ - tfm = acomp; + tfm = ctx; big_oops_buf_sz = size; big_oops_buf = buf; @@ -355,8 +334,7 @@ static void allocate_buf_for_compression(void) static void free_buf_for_compression(void) { if (IS_ENABLED(CONFIG_PSTORE_COMPRESS) && tfm) { - acomp_request_free(creq); - crypto_free_acomp(tfm); + crypto_free_comp(tfm); tfm = NULL; } kfree(big_oops_buf); @@ -693,8 +671,6 @@ static void decompress_record(struct pstore_record *record) int ret; int unzipped_len; char *unzipped, *workspace; - struct acomp_req *dreq; - struct scatterlist src, dst; if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS) || !record->compressed) return; @@ -718,30 +694,16 @@ static void decompress_record(struct pstore_record *record) if (!workspace) return; - dreq = acomp_request_alloc(tfm); - if (!dreq) { - kfree(workspace); - return; - } - - sg_init_table(&src, 1); - sg_set_buf(&src, record->buf, record->size); - - sg_init_table(&dst, 1); - sg_set_buf(&dst, workspace, unzipped_len); - - acomp_request_set_params(dreq, &src, &dst, record->size, unzipped_len); - /* After decompression "unzipped_len" is almost certainly smaller. */ - ret = crypto_acomp_decompress(dreq); + ret = crypto_comp_decompress(tfm, record->buf, record->size, + workspace, &unzipped_len); if (ret) { - pr_err("crypto_acomp_decompress failed, ret = %d!\n", ret); + pr_err("crypto_comp_decompress failed, ret = %d!\n", ret); kfree(workspace); return; } /* Append ECC notice to decompressed buffer. */ - unzipped_len = dreq->dlen; memcpy(workspace + unzipped_len, record->buf + record->size, record->ecc_notice_size); @@ -749,7 +711,6 @@ static void decompress_record(struct pstore_record *record) unzipped = kmemdup(workspace, unzipped_len + record->ecc_notice_size, GFP_KERNEL); kfree(workspace); - acomp_request_free(dreq); if (!unzipped) return; diff --git a/fs/qnx6/inode.c b/fs/qnx6/inode.c index b9895afca9d110a38b94d040180114a7e20b449b..85b2fa3b211c95a0febf18b3cdb134b65db8226f 100644 --- a/fs/qnx6/inode.c +++ b/fs/qnx6/inode.c @@ -470,10 +470,8 @@ out2: out1: iput(sbi->inodes); out: - if (bh1) - brelse(bh1); - if (bh2) - brelse(bh2); + brelse(bh1); + brelse(bh2); outnobh: kfree(qs); s->s_fs_info = NULL; diff --git a/fs/quota/quota_tree.c b/fs/quota/quota_tree.c index 5f2405994280a1161796225d34d83c2e46541ee9..0f1493e0f6d059b37032af363f5898415be6d79f 100644 --- a/fs/quota/quota_tree.c +++ b/fs/quota/quota_tree.c @@ -71,6 +71,40 @@ static ssize_t write_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf) return ret; } +static inline int do_check_range(struct super_block *sb, const char *val_name, + uint val, uint min_val, uint max_val) +{ + if (val < min_val || val > max_val) { + quota_error(sb, "Getting %s %u out of range %u-%u", + val_name, val, min_val, max_val); + return -EUCLEAN; + } + + return 0; +} + +static int check_dquot_block_header(struct qtree_mem_dqinfo *info, + struct qt_disk_dqdbheader *dh) +{ + int err = 0; + + err = do_check_range(info->dqi_sb, "dqdh_next_free", + le32_to_cpu(dh->dqdh_next_free), 0, + info->dqi_blocks - 1); + if (err) + return err; + err = do_check_range(info->dqi_sb, "dqdh_prev_free", + le32_to_cpu(dh->dqdh_prev_free), 0, + info->dqi_blocks - 1); + if (err) + return err; + err = do_check_range(info->dqi_sb, "dqdh_entries", + le16_to_cpu(dh->dqdh_entries), 0, + qtree_dqstr_in_blk(info)); + + return err; +} + /* Remove empty block from list and return it */ static int get_free_dqblk(struct qtree_mem_dqinfo *info) { @@ -85,6 +119,9 @@ static int get_free_dqblk(struct qtree_mem_dqinfo *info) ret = read_blk(info, blk, buf); if (ret < 0) goto out_buf; + ret = check_dquot_block_header(info, dh); + if (ret) + goto out_buf; info->dqi_free_blk = le32_to_cpu(dh->dqdh_next_free); } else { @@ -232,6 +269,9 @@ static uint find_free_dqentry(struct qtree_mem_dqinfo *info, *err = read_blk(info, blk, buf); if (*err < 0) goto out_buf; + *err = check_dquot_block_header(info, dh); + if (*err) + goto out_buf; } else { blk = get_free_dqblk(info); if ((int)blk < 0) { @@ -313,6 +353,10 @@ static int do_insert_tree(struct qtree_mem_dqinfo *info, struct dquot *dquot, } ref = (__le32 *)buf; newblk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); + ret = do_check_range(dquot->dq_sb, "block", newblk, 0, + info->dqi_blocks - 1); + if (ret) + goto out_buf; if (!newblk) newson = 1; if (depth == info->dqi_qtree_depth - 1) { @@ -424,6 +468,9 @@ static int free_dqentry(struct qtree_mem_dqinfo *info, struct dquot *dquot, goto out_buf; } dh = (struct qt_disk_dqdbheader *)buf; + ret = check_dquot_block_header(info, dh); + if (ret) + goto out_buf; le16_add_cpu(&dh->dqdh_entries, -1); if (!le16_to_cpu(dh->dqdh_entries)) { /* Block got free? */ ret = remove_free_dqentry(info, buf, blk); @@ -480,12 +527,10 @@ static int remove_tree(struct qtree_mem_dqinfo *info, struct dquot *dquot, goto out_buf; } newblk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); - if (newblk < QT_TREEOFF || newblk >= info->dqi_blocks) { - quota_error(dquot->dq_sb, "Getting block too big (%u >= %u)", - newblk, info->dqi_blocks); - ret = -EUCLEAN; + ret = do_check_range(dquot->dq_sb, "block", newblk, QT_TREEOFF, + info->dqi_blocks - 1); + if (ret) goto out_buf; - } if (depth == info->dqi_qtree_depth - 1) { ret = free_dqentry(info, dquot, newblk); @@ -586,12 +631,10 @@ static loff_t find_tree_dqentry(struct qtree_mem_dqinfo *info, blk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); if (!blk) /* No reference? */ goto out_buf; - if (blk < QT_TREEOFF || blk >= info->dqi_blocks) { - quota_error(dquot->dq_sb, "Getting block too big (%u >= %u)", - blk, info->dqi_blocks); - ret = -EUCLEAN; + ret = do_check_range(dquot->dq_sb, "block", blk, QT_TREEOFF, + info->dqi_blocks - 1); + if (ret) goto out_buf; - } if (depth < info->dqi_qtree_depth - 1) ret = find_tree_dqentry(info, dquot, blk, depth+1); @@ -705,15 +748,21 @@ static int find_next_id(struct qtree_mem_dqinfo *info, qid_t *id, goto out_buf; } for (i = __get_index(info, *id, depth); i < epb; i++) { - if (ref[i] == cpu_to_le32(0)) { + uint blk_no = le32_to_cpu(ref[i]); + + if (blk_no == 0) { *id += level_inc; continue; } + ret = do_check_range(info->dqi_sb, "block", blk_no, 0, + info->dqi_blocks - 1); + if (ret) + goto out_buf; if (depth == info->dqi_qtree_depth - 1) { ret = 0; goto out_buf; } - ret = find_next_id(info, id, le32_to_cpu(ref[i]), depth + 1); + ret = find_next_id(info, id, blk_no, depth + 1); if (ret != -ENOENT) break; } diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c index ba3525ccc27e67a3c36ad322cd347bf43660b135..cb240eac50365b61098151d2df8579a8287c8e74 100644 --- a/fs/ramfs/file-nommu.c +++ b/fs/ramfs/file-nommu.c @@ -203,9 +203,9 @@ static unsigned long ramfs_nommu_get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - unsigned long maxpages, lpages, nr, loop, ret; + unsigned long maxpages, lpages, nr_folios, loop, ret, nr_pages, pfn; struct inode *inode = file_inode(file); - struct page **pages = NULL, **ptr, *page; + struct folio_batch fbatch; loff_t isize; /* the mapping mustn't extend beyond the EOF */ @@ -221,31 +221,39 @@ static unsigned long ramfs_nommu_get_unmapped_area(struct file *file, goto out; /* gang-find the pages */ - pages = kcalloc(lpages, sizeof(struct page *), GFP_KERNEL); - if (!pages) - goto out_free; - - nr = find_get_pages_contig(inode->i_mapping, pgoff, lpages, pages); - if (nr != lpages) - goto out_free_pages; /* leave if some pages were missing */ + folio_batch_init(&fbatch); + nr_pages = 0; +repeat: + nr_folios = filemap_get_folios_contig(inode->i_mapping, &pgoff, + ULONG_MAX, &fbatch); + if (!nr_folios) { + ret = -ENOSYS; + return ret; + } + if (ret == -ENOSYS) { + ret = (unsigned long) folio_address(fbatch.folios[0]); + pfn = folio_pfn(fbatch.folios[0]); + } /* check the pages for physical adjacency */ - ptr = pages; - page = *ptr++; - page++; - for (loop = lpages; loop > 1; loop--) - if (*ptr++ != page++) - goto out_free_pages; + for (loop = 0; loop < nr_folios; loop++) { + if (pfn + nr_pages != folio_pfn(fbatch.folios[loop])) { + ret = -ENOSYS; + goto out_free; /* leave if not physical adjacent */ + } + nr_pages += folio_nr_pages(fbatch.folios[loop]); + if (nr_pages >= lpages) + goto out_free; /* successfully found desired pages*/ + } + if (nr_pages < lpages) { + folio_batch_release(&fbatch); + goto repeat; /* loop if pages are missing */ + } /* okay - all conditions fulfilled */ - ret = (unsigned long) page_address(pages[0]); -out_free_pages: - ptr = pages; - for (loop = nr; loop > 0; loop--) - put_page(*ptr++); out_free: - kfree(pages); + folio_batch_release(&fbatch); out: return ret; } diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c index bc66d0173e3300b29da04ae6a51519097caf0c75..b3257e852820031e4bc90da51aa2fa88260571c6 100644 --- a/fs/ramfs/inode.c +++ b/fs/ramfs/inode.c @@ -146,15 +146,15 @@ static int ramfs_symlink(struct user_namespace *mnt_userns, struct inode *dir, } static int ramfs_tmpfile(struct user_namespace *mnt_userns, - struct inode *dir, struct dentry *dentry, umode_t mode) + struct inode *dir, struct file *file, umode_t mode) { struct inode *inode; inode = ramfs_get_inode(dir->i_sb, dir, mode, 0); if (!inode) return -ENOSPC; - d_tmpfile(dentry, inode); - return 0; + d_tmpfile(file, inode); + return finish_open_simple(file, 0); } static const struct inode_operations ramfs_dir_inode_operations = { diff --git a/fs/read_write.c b/fs/read_write.c index 1a261dcf1778a0862a34600ec3ad3d83c69cfcbc..328ce8cf9a85eef7057ffe0a35902a15c8af8486 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -496,14 +496,9 @@ static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t } /* caller is responsible for file_start_write/file_end_write */ -ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) +ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos) { - struct kvec iov = { - .iov_base = (void *)buf, - .iov_len = min_t(size_t, count, MAX_RW_COUNT), - }; struct kiocb kiocb; - struct iov_iter iter; ssize_t ret; if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE))) @@ -519,8 +514,7 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t init_sync_kiocb(&kiocb, file); kiocb.ki_pos = pos ? *pos : 0; - iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len); - ret = file->f_op->write_iter(&kiocb, &iter); + ret = file->f_op->write_iter(&kiocb, from); if (ret > 0) { if (pos) *pos = kiocb.ki_pos; @@ -530,6 +524,18 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t inc_syscw(current); return ret; } + +/* caller is responsible for file_start_write/file_end_write */ +ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) +{ + struct kvec iov = { + .iov_base = (void *)buf, + .iov_len = min_t(size_t, count, MAX_RW_COUNT), + }; + struct iov_iter iter; + iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len); + return __kernel_write_iter(file, &iter, pos); +} /* * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()", * but autofs is one of the few internal kernel users that actually diff --git a/fs/readdir.c b/fs/readdir.c index 09e8ed7d416141406ed5b2eacef0a9de30120bd3..9c53edb60c03130d119bcc7f2197b30dbc4d690f 100644 --- a/fs/readdir.c +++ b/fs/readdir.c @@ -140,7 +140,7 @@ struct readdir_callback { int result; }; -static int fillonedir(struct dir_context *ctx, const char *name, int namlen, +static bool fillonedir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct readdir_callback *buf = @@ -149,14 +149,14 @@ static int fillonedir(struct dir_context *ctx, const char *name, int namlen, unsigned long d_ino; if (buf->result) - return -EINVAL; + return false; buf->result = verify_dirent_name(name, namlen); - if (buf->result < 0) - return buf->result; + if (buf->result) + return false; d_ino = ino; if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { buf->result = -EOVERFLOW; - return -EOVERFLOW; + return false; } buf->result++; dirent = buf->dirent; @@ -169,12 +169,12 @@ static int fillonedir(struct dir_context *ctx, const char *name, int namlen, unsafe_put_user(namlen, &dirent->d_namlen, efault_end); unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end); user_write_access_end(); - return 0; + return true; efault_end: user_write_access_end(); efault: buf->result = -EFAULT; - return -EFAULT; + return false; } SYSCALL_DEFINE3(old_readdir, unsigned int, fd, @@ -219,7 +219,7 @@ struct getdents_callback { int error; }; -static int filldir(struct dir_context *ctx, const char *name, int namlen, +static bool filldir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct linux_dirent __user *dirent, *prev; @@ -232,18 +232,18 @@ static int filldir(struct dir_context *ctx, const char *name, int namlen, buf->error = verify_dirent_name(name, namlen); if (unlikely(buf->error)) - return buf->error; + return false; buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) - return -EINVAL; + return false; d_ino = ino; if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { buf->error = -EOVERFLOW; - return -EOVERFLOW; + return false; } prev_reclen = buf->prev_reclen; if (prev_reclen && signal_pending(current)) - return -EINTR; + return false; dirent = buf->current_dir; prev = (void __user *) dirent - prev_reclen; if (!user_write_access_begin(prev, reclen + prev_reclen)) @@ -260,12 +260,12 @@ static int filldir(struct dir_context *ctx, const char *name, int namlen, buf->current_dir = (void __user *)dirent + reclen; buf->prev_reclen = reclen; buf->count -= reclen; - return 0; + return true; efault_end: user_write_access_end(); efault: buf->error = -EFAULT; - return -EFAULT; + return false; } SYSCALL_DEFINE3(getdents, unsigned int, fd, @@ -307,7 +307,7 @@ struct getdents_callback64 { int error; }; -static int filldir64(struct dir_context *ctx, const char *name, int namlen, +static bool filldir64(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct linux_dirent64 __user *dirent, *prev; @@ -319,13 +319,13 @@ static int filldir64(struct dir_context *ctx, const char *name, int namlen, buf->error = verify_dirent_name(name, namlen); if (unlikely(buf->error)) - return buf->error; + return false; buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) - return -EINVAL; + return false; prev_reclen = buf->prev_reclen; if (prev_reclen && signal_pending(current)) - return -EINTR; + return false; dirent = buf->current_dir; prev = (void __user *)dirent - prev_reclen; if (!user_write_access_begin(prev, reclen + prev_reclen)) @@ -342,13 +342,13 @@ static int filldir64(struct dir_context *ctx, const char *name, int namlen, buf->prev_reclen = reclen; buf->current_dir = (void __user *)dirent + reclen; buf->count -= reclen; - return 0; + return true; efault_end: user_write_access_end(); efault: buf->error = -EFAULT; - return -EFAULT; + return false; } SYSCALL_DEFINE3(getdents64, unsigned int, fd, @@ -397,7 +397,7 @@ struct compat_readdir_callback { int result; }; -static int compat_fillonedir(struct dir_context *ctx, const char *name, +static bool compat_fillonedir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { @@ -407,14 +407,14 @@ static int compat_fillonedir(struct dir_context *ctx, const char *name, compat_ulong_t d_ino; if (buf->result) - return -EINVAL; + return false; buf->result = verify_dirent_name(name, namlen); - if (buf->result < 0) - return buf->result; + if (buf->result) + return false; d_ino = ino; if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { buf->result = -EOVERFLOW; - return -EOVERFLOW; + return false; } buf->result++; dirent = buf->dirent; @@ -427,12 +427,12 @@ static int compat_fillonedir(struct dir_context *ctx, const char *name, unsafe_put_user(namlen, &dirent->d_namlen, efault_end); unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end); user_write_access_end(); - return 0; + return true; efault_end: user_write_access_end(); efault: buf->result = -EFAULT; - return -EFAULT; + return false; } COMPAT_SYSCALL_DEFINE3(old_readdir, unsigned int, fd, @@ -471,7 +471,7 @@ struct compat_getdents_callback { int error; }; -static int compat_filldir(struct dir_context *ctx, const char *name, int namlen, +static bool compat_filldir(struct dir_context *ctx, const char *name, int namlen, loff_t offset, u64 ino, unsigned int d_type) { struct compat_linux_dirent __user *dirent, *prev; @@ -484,18 +484,18 @@ static int compat_filldir(struct dir_context *ctx, const char *name, int namlen, buf->error = verify_dirent_name(name, namlen); if (unlikely(buf->error)) - return buf->error; + return false; buf->error = -EINVAL; /* only used if we fail.. */ if (reclen > buf->count) - return -EINVAL; + return false; d_ino = ino; if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { buf->error = -EOVERFLOW; - return -EOVERFLOW; + return false; } prev_reclen = buf->prev_reclen; if (prev_reclen && signal_pending(current)) - return -EINTR; + return false; dirent = buf->current_dir; prev = (void __user *) dirent - prev_reclen; if (!user_write_access_begin(prev, reclen + prev_reclen)) @@ -511,12 +511,12 @@ static int compat_filldir(struct dir_context *ctx, const char *name, int namlen, buf->prev_reclen = reclen; buf->current_dir = (void __user *)dirent + reclen; buf->count -= reclen; - return 0; + return true; efault_end: user_write_access_end(); efault: buf->error = -EFAULT; - return -EFAULT; + return false; } COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd, diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c index 94addfcefede47e7a183235a2f37564b16beeef8..9f62da7471c9e1bf75b196196b49056d0dd57576 100644 --- a/fs/reiserfs/journal.c +++ b/fs/reiserfs/journal.c @@ -868,7 +868,7 @@ loop_next: */ if (buffer_dirty(bh) && unlikely(bh->b_page->mapping == NULL)) { spin_unlock(lock); - ll_rw_block(REQ_OP_WRITE, 1, &bh); + write_dirty_buffer(bh, 0); spin_lock(lock); } put_bh(bh); @@ -1054,7 +1054,7 @@ static int flush_commit_list(struct super_block *s, if (tbh) { if (buffer_dirty(tbh)) { depth = reiserfs_write_unlock_nested(s); - ll_rw_block(REQ_OP_WRITE, 1, &tbh); + write_dirty_buffer(tbh, 0); reiserfs_write_lock_nested(s, depth); } put_bh(tbh) ; @@ -2240,7 +2240,7 @@ abort_replay: } } /* read in the log blocks, memcpy to the corresponding real block */ - ll_rw_block(REQ_OP_READ, get_desc_trans_len(desc), log_blocks); + bh_read_batch(get_desc_trans_len(desc), log_blocks); for (i = 0; i < get_desc_trans_len(desc); i++) { wait_on_buffer(log_blocks[i]); @@ -2342,10 +2342,11 @@ static struct buffer_head *reiserfs_breada(struct block_device *dev, } else bhlist[j++] = bh; } - ll_rw_block(REQ_OP_READ, j, bhlist); + bh = bhlist[0]; + bh_read_nowait(bh, 0); + bh_readahead_batch(j - 1, &bhlist[1], 0); for (i = 1; i < j; i++) brelse(bhlist[i]); - bh = bhlist[0]; wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; diff --git a/fs/reiserfs/prints.c b/fs/reiserfs/prints.c index 30319dc33c189616e83686e9bb6fbb1c818d0073..84a194b77f19624bd13e6a7847a6cb5da6c692ad 100644 --- a/fs/reiserfs/prints.c +++ b/fs/reiserfs/prints.c @@ -456,7 +456,7 @@ static int print_internal(struct buffer_head *bh, int first, int last) to = B_NR_ITEMS(bh); } else { from = first; - to = last < B_NR_ITEMS(bh) ? last : B_NR_ITEMS(bh); + to = min_t(int, last, B_NR_ITEMS(bh)); } reiserfs_printk("INTERNAL NODE (%ld) contains %z\n", bh->b_blocknr, bh); diff --git a/fs/reiserfs/procfs.c b/fs/reiserfs/procfs.c index 4a7cb16e9345c03292dc419cff294aa8797fd473..3dba8acf4e832a7592e9a364566db452b6b373eb 100644 --- a/fs/reiserfs/procfs.c +++ b/fs/reiserfs/procfs.c @@ -411,7 +411,7 @@ int reiserfs_proc_info_init(struct super_block *sb) char *s; /* Some block devices use /'s */ - strlcpy(b, sb->s_id, BDEVNAME_SIZE); + strscpy(b, sb->s_id, BDEVNAME_SIZE); s = strchr(b, '/'); if (s) *s = '!'; @@ -441,7 +441,7 @@ int reiserfs_proc_info_done(struct super_block *sb) char *s; /* Some block devices use /'s */ - strlcpy(b, sb->s_id, BDEVNAME_SIZE); + strscpy(b, sb->s_id, BDEVNAME_SIZE); s = strchr(b, '/'); if (s) *s = '!'; diff --git a/fs/reiserfs/resize.c b/fs/reiserfs/resize.c index 8096c74c38ac1d68d08b94e659d1fb10511163c0..7b498a0d060ba9c57249dcdb58444d121e4b802f 100644 --- a/fs/reiserfs/resize.c +++ b/fs/reiserfs/resize.c @@ -97,7 +97,7 @@ int reiserfs_resize(struct super_block *s, unsigned long block_count_new) * using the copy_size var below allows this code to work for * both shrinking and expanding the FS. */ - copy_size = bmap_nr_new < bmap_nr ? bmap_nr_new : bmap_nr; + copy_size = min(bmap_nr_new, bmap_nr); copy_size = copy_size * sizeof(struct reiserfs_list_bitmap_node *); for (i = 0; i < JOURNAL_NUM_BITMAPS; i++) { diff --git a/fs/reiserfs/stree.c b/fs/reiserfs/stree.c index 9a293609a0221c596c6366995dcd284b4a8628e1..84c12a1947b22c5754beeb20ce76a9e6560ca387 100644 --- a/fs/reiserfs/stree.c +++ b/fs/reiserfs/stree.c @@ -579,7 +579,7 @@ static int search_by_key_reada(struct super_block *s, if (!buffer_uptodate(bh[j])) { if (depth == -1) depth = reiserfs_write_unlock_nested(s); - ll_rw_block(REQ_OP_READ | REQ_RAHEAD, 1, bh + j); + bh_readahead(bh[j], REQ_RAHEAD); } brelse(bh[j]); } @@ -685,7 +685,7 @@ int search_by_key(struct super_block *sb, const struct cpu_key *key, if (!buffer_uptodate(bh) && depth == -1) depth = reiserfs_write_unlock_nested(sb); - ll_rw_block(REQ_OP_READ, 1, &bh); + bh_read_nowait(bh, 0); wait_on_buffer(bh); if (depth != -1) diff --git a/fs/reiserfs/super.c b/fs/reiserfs/super.c index c88cd2ce0665d28f151a66722f8f964ef1f88081..929acce6e731745683fc339abb546145d8e38397 100644 --- a/fs/reiserfs/super.c +++ b/fs/reiserfs/super.c @@ -1702,9 +1702,7 @@ static int read_super_block(struct super_block *s, int offset) /* after journal replay, reread all bitmap and super blocks */ static int reread_meta_blocks(struct super_block *s) { - ll_rw_block(REQ_OP_READ, 1, &SB_BUFFER_WITH_SB(s)); - wait_on_buffer(SB_BUFFER_WITH_SB(s)); - if (!buffer_uptodate(SB_BUFFER_WITH_SB(s))) { + if (bh_read(SB_BUFFER_WITH_SB(s), 0) < 0) { reiserfs_warning(s, "reiserfs-2504", "error reading the super"); return 1; } @@ -2504,9 +2502,7 @@ static ssize_t reiserfs_quota_read(struct super_block *sb, int type, char *data, len = i_size - off; toread = len; while (toread > 0) { - tocopy = - sb->s_blocksize - offset < - toread ? sb->s_blocksize - offset : toread; + tocopy = min_t(unsigned long, sb->s_blocksize - offset, toread); tmp_bh.b_state = 0; /* * Quota files are without tails so we can safely @@ -2554,8 +2550,7 @@ static ssize_t reiserfs_quota_write(struct super_block *sb, int type, return -EIO; } while (towrite > 0) { - tocopy = sb->s_blocksize - offset < towrite ? - sb->s_blocksize - offset : towrite; + tocopy = min_t(unsigned long, sb->s_blocksize - offset, towrite); tmp_bh.b_state = 0; reiserfs_write_lock(sb); err = reiserfs_get_block(inode, blk, &tmp_bh, GET_BLOCK_CREATE); diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c index 4366413692836261925eb57edf0f47fd881f4a64..8b2d52443f415ca49160c7032856ed2cc7ae2023 100644 --- a/fs/reiserfs/xattr.c +++ b/fs/reiserfs/xattr.c @@ -189,7 +189,7 @@ struct reiserfs_dentry_buf { struct dentry *dentries[8]; }; -static int +static bool fill_with_dentries(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -200,16 +200,16 @@ fill_with_dentries(struct dir_context *ctx, const char *name, int namelen, WARN_ON_ONCE(!inode_is_locked(d_inode(dbuf->xadir))); if (dbuf->count == ARRAY_SIZE(dbuf->dentries)) - return -ENOSPC; + return false; if (name[0] == '.' && (namelen < 2 || (namelen == 2 && name[1] == '.'))) - return 0; + return true; dentry = lookup_one_len(name, dbuf->xadir, namelen); if (IS_ERR(dentry)) { dbuf->err = PTR_ERR(dentry); - return PTR_ERR(dentry); + return false; } else if (d_really_is_negative(dentry)) { /* A directory entry exists, but no file? */ reiserfs_error(dentry->d_sb, "xattr-20003", @@ -218,11 +218,11 @@ fill_with_dentries(struct dir_context *ctx, const char *name, int namelen, dentry, dbuf->xadir); dput(dentry); dbuf->err = -EIO; - return -EIO; + return false; } dbuf->dentries[dbuf->count++] = dentry; - return 0; + return true; } static void @@ -797,7 +797,7 @@ struct listxattr_buf { struct dentry *dentry; }; -static int listxattr_filler(struct dir_context *ctx, const char *name, +static bool listxattr_filler(struct dir_context *ctx, const char *name, int namelen, loff_t offset, u64 ino, unsigned int d_type) { @@ -813,19 +813,19 @@ static int listxattr_filler(struct dir_context *ctx, const char *name, name); if (!handler /* Unsupported xattr name */ || (handler->list && !handler->list(b->dentry))) - return 0; + return true; size = namelen + 1; if (b->buf) { if (b->pos + size > b->size) { b->pos = -ERANGE; - return -ERANGE; + return false; } memcpy(b->buf + b->pos, name, namelen); b->buf[b->pos + namelen] = 0; } b->pos += size; } - return 0; + return true; } /* diff --git a/fs/smbfs_common/smb2pdu.h b/fs/smbfs_common/smb2pdu.h index 2cab413fffeea4adc732ad5e0ab336694001ecf5..7d605db3bb3b98958547821825f71e337bac0bb3 100644 --- a/fs/smbfs_common/smb2pdu.h +++ b/fs/smbfs_common/smb2pdu.h @@ -1101,7 +1101,11 @@ struct smb2_change_notify_rsp { #define SMB2_CREATE_REQUEST_LEASE "RqLs" #define SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 "DH2Q" #define SMB2_CREATE_DURABLE_HANDLE_RECONNECT_V2 "DH2C" -#define SMB2_CREATE_TAG_POSIX "\x93\xAD\x25\x50\x9C\xB4\x11\xE7\xB4\x23\x83\xDE\x96\x8B\xCD\x7C" +#define SMB2_CREATE_TAG_POSIX "\x93\xAD\x25\x50\x9C\xB4\x11\xE7\xB4\x23\x83\xDE\x96\x8B\xCD\x7C" +#define SMB2_CREATE_APP_INSTANCE_ID "\x45\xBC\xA6\x6A\xEF\xA7\xF7\x4A\x90\x08\xFA\x46\x2E\x14\x4D\x74" +#define SMB2_CREATE_APP_INSTANCE_VERSION "\xB9\x82\xD0\xB7\x3B\x56\x07\x4F\xA0\x7B\x52\x4A\x81\x16\xA0\x10" +#define SVHDX_OPEN_DEVICE_CONTEXT "\x9C\xCB\xCF\x9E\x04\xC1\xE6\x43\x98\x0E\x15\x8D\xA1\xF6\xEC\x83" +#define SMB2_CREATE_TAG_AAPL "AAPL" /* Flag (SMB3 open response) values */ #define SMB2_CREATE_FLAG_REPARSEPOINT 0x01 diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index 98e64fec75b77e09048757fc28ea04f2f81bf53c..e56510964b229e42170304f88ee73515f54f1e6a 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -593,7 +593,7 @@ static void squashfs_readahead(struct readahead_control *ractl) res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); - kfree(actor); + squashfs_page_actor_free(actor); if (res == expected) { int bytes; diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c index be4b12d31e0c36fbd04e2d56f96d13e793371f92..f1ccad519e28ccf890831a36f462f0e3ce5ba3a1 100644 --- a/fs/squashfs/file_direct.c +++ b/fs/squashfs/file_direct.c @@ -74,7 +74,7 @@ int squashfs_readpage_block(struct page *target_page, u64 block, int bsize, /* Decompress directly into the page cache buffers */ res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); - kfree(actor); + squashfs_page_actor_free(actor); if (res < 0) goto mark_errored; diff --git a/fs/squashfs/page_actor.c b/fs/squashfs/page_actor.c index b23b780d8f42ece1639bd4bc21ced7743ffc2643..54b93bf4a25c15b9acb2bf85cbcab6723df89e40 100644 --- a/fs/squashfs/page_actor.c +++ b/fs/squashfs/page_actor.c @@ -52,6 +52,7 @@ struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, actor->buffer = buffer; actor->pages = pages; actor->next_page = 0; + actor->tmp_buffer = NULL; actor->squashfs_first_page = cache_first_page; actor->squashfs_next_page = cache_next_page; actor->squashfs_finish_page = cache_finish_page; @@ -68,20 +69,9 @@ static void *handle_next_page(struct squashfs_page_actor *actor) if ((actor->next_page == actor->pages) || (actor->next_index != actor->page[actor->next_page]->index)) { - if (actor->alloc_buffer) { - void *tmp_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); - - if (tmp_buffer) { - actor->tmp_buffer = tmp_buffer; - actor->next_index++; - actor->returned_pages++; - return tmp_buffer; - } - } - actor->next_index++; actor->returned_pages++; - return ERR_PTR(-ENOMEM); + return actor->alloc_buffer ? actor->tmp_buffer : ERR_PTR(-ENOMEM); } actor->next_index++; @@ -96,11 +86,10 @@ static void *direct_first_page(struct squashfs_page_actor *actor) static void *direct_next_page(struct squashfs_page_actor *actor) { - if (actor->pageaddr) + if (actor->pageaddr) { kunmap_local(actor->pageaddr); - - kfree(actor->tmp_buffer); - actor->pageaddr = actor->tmp_buffer = NULL; + actor->pageaddr = NULL; + } return handle_next_page(actor); } @@ -109,8 +98,6 @@ static void direct_finish_page(struct squashfs_page_actor *actor) { if (actor->pageaddr) kunmap_local(actor->pageaddr); - - kfree(actor->tmp_buffer); } struct squashfs_page_actor *squashfs_page_actor_init_special(struct squashfs_sb_info *msblk, @@ -121,6 +108,16 @@ struct squashfs_page_actor *squashfs_page_actor_init_special(struct squashfs_sb_ if (actor == NULL) return NULL; + if (msblk->decompressor->alloc_buffer) { + actor->tmp_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + + if (actor->tmp_buffer == NULL) { + kfree(actor); + return NULL; + } + } else + actor->tmp_buffer = NULL; + actor->length = length ? : pages * PAGE_SIZE; actor->page = page; actor->pages = pages; @@ -128,7 +125,6 @@ struct squashfs_page_actor *squashfs_page_actor_init_special(struct squashfs_sb_ actor->returned_pages = 0; actor->next_index = page[0]->index & ~((1 << (msblk->block_log - PAGE_SHIFT)) - 1); actor->pageaddr = NULL; - actor->tmp_buffer = NULL; actor->alloc_buffer = msblk->decompressor->alloc_buffer; actor->squashfs_first_page = direct_first_page; actor->squashfs_next_page = direct_next_page; diff --git a/fs/squashfs/page_actor.h b/fs/squashfs/page_actor.h index 24841d28bc0fb85b630535f4c07c6c54b4aa2767..95ffbb543d913b601604ae3cc12f766bb5380295 100644 --- a/fs/squashfs/page_actor.h +++ b/fs/squashfs/page_actor.h @@ -29,6 +29,11 @@ extern struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, extern struct squashfs_page_actor *squashfs_page_actor_init_special( struct squashfs_sb_info *msblk, struct page **page, int pages, int length); +static inline void squashfs_page_actor_free(struct squashfs_page_actor *actor) +{ + kfree(actor->tmp_buffer); + kfree(actor); +} static inline void *squashfs_first_page(struct squashfs_page_actor *actor) { return actor->squashfs_first_page(actor); diff --git a/fs/stat.c b/fs/stat.c index 9ced8860e0f35d305fabe984254d9f746808117a..ef50573c72a2692908d5ad8bcd68e9a09e501c78 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -5,6 +5,7 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ +#include #include #include #include @@ -230,11 +231,22 @@ retry: goto out; error = vfs_getattr(&path, stat, request_mask, flags); + stat->mnt_id = real_mount(path.mnt)->mnt_id; stat->result_mask |= STATX_MNT_ID; + if (path.mnt->mnt_root == path.dentry) stat->attributes |= STATX_ATTR_MOUNT_ROOT; stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT; + + /* Handle STATX_DIOALIGN for block devices. */ + if (request_mask & STATX_DIOALIGN) { + struct inode *inode = d_backing_inode(path.dentry); + + if (S_ISBLK(inode->i_mode)) + bdev_statx_dioalign(inode, stat); + } + path_put(&path); if (retry_estale(error, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; @@ -611,6 +623,8 @@ cp_statx(const struct kstat *stat, struct statx __user *buffer) tmp.stx_dev_major = MAJOR(stat->dev); tmp.stx_dev_minor = MINOR(stat->dev); tmp.stx_mnt_id = stat->mnt_id; + tmp.stx_dio_mem_align = stat->dio_mem_align; + tmp.stx_dio_offset_align = stat->dio_offset_align; return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0; } diff --git a/fs/super.c b/fs/super.c index 734ed584a946e185bacf6d0586e7ab8205fe0435..6a82660e1adba8d6c761095b336db8cb0492ea51 100644 --- a/fs/super.c +++ b/fs/super.c @@ -291,7 +291,6 @@ static void __put_super(struct super_block *s) WARN_ON(s->s_inode_lru.node); WARN_ON(!list_empty(&s->s_mounts)); security_sb_free(s); - fscrypt_sb_free(s); put_user_ns(s->s_user_ns); kfree(s->s_subtype); call_rcu(&s->rcu, destroy_super_rcu); @@ -480,6 +479,7 @@ void generic_shutdown_super(struct super_block *sb) evict_inodes(sb); /* only nonzero refcount inodes can have marks */ fsnotify_sb_delete(sb); + fscrypt_sb_delete(sb); security_sb_delete(sb); if (sb->s_dio_done_wq) { diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 81d26abf486fa7b6b969d21039af2e13e390e600..da85b3979195747d09b788ca7884f7575bf3ff15 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -141,6 +141,8 @@ struct tracefs_mount_opts { kuid_t uid; kgid_t gid; umode_t mode; + /* Opt_* bitfield. */ + unsigned int opts; }; enum { @@ -241,6 +243,7 @@ static int tracefs_parse_options(char *data, struct tracefs_mount_opts *opts) kgid_t gid; char *p; + opts->opts = 0; opts->mode = TRACEFS_DEFAULT_MODE; while ((p = strsep(&data, ",")) != NULL) { @@ -275,24 +278,36 @@ static int tracefs_parse_options(char *data, struct tracefs_mount_opts *opts) * but traditionally tracefs has ignored all mount options */ } + + opts->opts |= BIT(token); } return 0; } -static int tracefs_apply_options(struct super_block *sb) +static int tracefs_apply_options(struct super_block *sb, bool remount) { struct tracefs_fs_info *fsi = sb->s_fs_info; struct inode *inode = d_inode(sb->s_root); struct tracefs_mount_opts *opts = &fsi->mount_opts; - inode->i_mode &= ~S_IALLUGO; - inode->i_mode |= opts->mode; + /* + * On remount, only reset mode/uid/gid if they were provided as mount + * options. + */ + + if (!remount || opts->opts & BIT(Opt_mode)) { + inode->i_mode &= ~S_IALLUGO; + inode->i_mode |= opts->mode; + } - inode->i_uid = opts->uid; + if (!remount || opts->opts & BIT(Opt_uid)) + inode->i_uid = opts->uid; - /* Set all the group ids to the mount option */ - set_gid(sb->s_root, opts->gid); + if (!remount || opts->opts & BIT(Opt_gid)) { + /* Set all the group ids to the mount option */ + set_gid(sb->s_root, opts->gid); + } return 0; } @@ -307,7 +322,7 @@ static int tracefs_remount(struct super_block *sb, int *flags, char *data) if (err) goto fail; - tracefs_apply_options(sb); + tracefs_apply_options(sb, true); fail: return err; @@ -359,7 +374,7 @@ static int trace_fill_super(struct super_block *sb, void *data, int silent) sb->s_op = &tracefs_super_operations; - tracefs_apply_options(sb); + tracefs_apply_options(sb, false); return 0; diff --git a/fs/ubifs/crypto.c b/fs/ubifs/crypto.c index c57b46a352d8fd6637f2e51256e093f364161034..3125e76376ee63428fc68ab38181b9e1d86dae19 100644 --- a/fs/ubifs/crypto.c +++ b/fs/ubifs/crypto.c @@ -24,6 +24,17 @@ static bool ubifs_crypt_empty_dir(struct inode *inode) return ubifs_check_dir_empty(inode) == 0; } +/** + * ubifs_encrypt - Encrypt data. + * @inode: inode which refers to the data node + * @dn: data node to encrypt + * @in_len: length of data to be compressed + * @out_len: allocated memory size for the data area of @dn + * @block: logical block number of the block + * + * This function encrypt a possibly-compressed data in the data node. + * The encrypted data length will store in @out_len. + */ int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn, unsigned int in_len, unsigned int *out_len, int block) { diff --git a/fs/ubifs/debug.c b/fs/ubifs/debug.c index fc718f6178f2553621fa7ed296b08ad23e8fa7cb..3f128b9fdfbb23bd3dd93ea37d54702c59850b0c 100644 --- a/fs/ubifs/debug.c +++ b/fs/ubifs/debug.c @@ -2467,7 +2467,7 @@ error_dump: static inline int chance(unsigned int n, unsigned int out_of) { - return !!((prandom_u32() % out_of) + 1 <= n); + return !!(prandom_u32_max(out_of) + 1 <= n); } @@ -2485,13 +2485,13 @@ static int power_cut_emulated(struct ubifs_info *c, int lnum, int write) if (chance(1, 2)) { d->pc_delay = 1; /* Fail within 1 minute */ - delay = prandom_u32() % 60000; + delay = prandom_u32_max(60000); d->pc_timeout = jiffies; d->pc_timeout += msecs_to_jiffies(delay); ubifs_warn(c, "failing after %lums", delay); } else { d->pc_delay = 2; - delay = prandom_u32() % 10000; + delay = prandom_u32_max(10000); /* Fail within 10000 operations */ d->pc_cnt_max = delay; ubifs_warn(c, "failing after %lu calls", delay); @@ -2571,7 +2571,7 @@ static int corrupt_data(const struct ubifs_info *c, const void *buf, unsigned int from, to, ffs = chance(1, 2); unsigned char *p = (void *)buf; - from = prandom_u32() % len; + from = prandom_u32_max(len); /* Corruption span max to end of write unit */ to = min(len, ALIGN(from + 1, c->max_write_size)); @@ -2581,7 +2581,7 @@ static int corrupt_data(const struct ubifs_info *c, const void *buf, if (ffs) memset(p + from, 0xFF, to - from); else - prandom_bytes(p + from, to - from); + get_random_bytes(p + from, to - from); return to; } diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index 86151889548e3af45a4efc695ab5a6c68d985190..0f29cf2011361c6f9d663dc21f09e363de0c7e0f 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -68,13 +68,14 @@ static int inherit_flags(const struct inode *dir, umode_t mode) * @c: UBIFS file-system description object * @dir: parent directory inode * @mode: inode mode flags + * @is_xattr: whether the inode is xattr inode * * This function finds an unused inode number, allocates new inode and * initializes it. Returns new inode in case of success and an error code in * case of failure. */ struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, - umode_t mode) + umode_t mode, bool is_xattr) { int err; struct inode *inode; @@ -99,10 +100,12 @@ struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, current_time(inode); inode->i_mapping->nrpages = 0; - err = fscrypt_prepare_new_inode(dir, inode, &encrypted); - if (err) { - ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err); - goto out_iput; + if (!is_xattr) { + err = fscrypt_prepare_new_inode(dir, inode, &encrypted); + if (err) { + ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err); + goto out_iput; + } } switch (mode & S_IFMT) { @@ -309,7 +312,7 @@ static int ubifs_create(struct user_namespace *mnt_userns, struct inode *dir, sz_change = CALC_DENT_SIZE(fname_len(&nm)); - inode = ubifs_new_inode(c, dir, mode); + inode = ubifs_new_inode(c, dir, mode, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_fname; @@ -370,7 +373,7 @@ static struct inode *create_whiteout(struct inode *dir, struct dentry *dentry) if (err) return ERR_PTR(err); - inode = ubifs_new_inode(c, dir, mode); + inode = ubifs_new_inode(c, dir, mode, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_free; @@ -424,8 +427,9 @@ static void unlock_2_inodes(struct inode *inode1, struct inode *inode2) } static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { + struct dentry *dentry = file->f_path.dentry; struct inode *inode; struct ubifs_info *c = dir->i_sb->s_fs_info; struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, @@ -462,7 +466,7 @@ static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, return err; } - inode = ubifs_new_inode(c, dir, mode); + inode = ubifs_new_inode(c, dir, mode, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_budg; @@ -475,7 +479,7 @@ static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, mutex_lock(&ui->ui_mutex); insert_inode_hash(inode); - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); ubifs_assert(c, ui->dirty); instantiated = 1; @@ -489,7 +493,7 @@ static int ubifs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, ubifs_release_budget(c, &req); - return 0; + return finish_open_simple(file, 0); out_cancel: unlock_2_inodes(dir, inode); @@ -872,7 +876,7 @@ out_fname: } /** - * check_dir_empty - check if a directory is empty or not. + * ubifs_check_dir_empty - check if a directory is empty or not. * @dir: VFS inode object of the directory to check * * This function checks if directory @dir is empty. Returns zero if the @@ -1004,7 +1008,7 @@ static int ubifs_mkdir(struct user_namespace *mnt_userns, struct inode *dir, sz_change = CALC_DENT_SIZE(fname_len(&nm)); - inode = ubifs_new_inode(c, dir, S_IFDIR | mode); + inode = ubifs_new_inode(c, dir, S_IFDIR | mode, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_fname; @@ -1091,7 +1095,7 @@ static int ubifs_mknod(struct user_namespace *mnt_userns, struct inode *dir, sz_change = CALC_DENT_SIZE(fname_len(&nm)); - inode = ubifs_new_inode(c, dir, mode); + inode = ubifs_new_inode(c, dir, mode, false); if (IS_ERR(inode)) { kfree(dev); err = PTR_ERR(inode); @@ -1173,7 +1177,7 @@ static int ubifs_symlink(struct user_namespace *mnt_userns, struct inode *dir, sz_change = CALC_DENT_SIZE(fname_len(&nm)); - inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO); + inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO, false); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_fname; diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c index 75dab0ae3939d0a9011fad14242fe01b81380a52..d02509920bafadd9897c3c4fece885d3bc1efed9 100644 --- a/fs/ubifs/journal.c +++ b/fs/ubifs/journal.c @@ -503,7 +503,7 @@ static void mark_inode_clean(struct ubifs_info *c, struct ubifs_inode *ui) static void set_dent_cookie(struct ubifs_info *c, struct ubifs_dent_node *dent) { if (c->double_hash) - dent->cookie = (__force __le32) prandom_u32(); + dent->cookie = (__force __le32) get_random_u32(); else dent->cookie = 0; } @@ -1472,23 +1472,25 @@ out_free: * @block: data block number * @dn: data node to re-compress * @new_len: new length + * @dn_size: size of the data node @dn in memory * * This function is used when an inode is truncated and the last data node of * the inode has to be re-compressed/encrypted and re-written. */ static int truncate_data_node(const struct ubifs_info *c, const struct inode *inode, unsigned int block, struct ubifs_data_node *dn, - int *new_len) + int *new_len, int dn_size) { void *buf; - int err, dlen, compr_type, out_len, old_dlen; + int err, dlen, compr_type, out_len, data_size; out_len = le32_to_cpu(dn->size); buf = kmalloc_array(out_len, WORST_COMPR_FACTOR, GFP_NOFS); if (!buf) return -ENOMEM; - dlen = old_dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ; + dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ; + data_size = dn_size - UBIFS_DATA_NODE_SZ; compr_type = le16_to_cpu(dn->compr_type); if (IS_ENCRYPTED(inode)) { @@ -1508,11 +1510,11 @@ static int truncate_data_node(const struct ubifs_info *c, const struct inode *in } if (IS_ENCRYPTED(inode)) { - err = ubifs_encrypt(inode, dn, out_len, &old_dlen, block); + err = ubifs_encrypt(inode, dn, out_len, &data_size, block); if (err) goto out; - out_len = old_dlen; + out_len = data_size; } else { dn->compr_size = 0; } @@ -1550,6 +1552,7 @@ int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode, struct ubifs_trun_node *trun; struct ubifs_data_node *dn; int err, dlen, len, lnum, offs, bit, sz, sync = IS_SYNC(inode); + int dn_size; struct ubifs_inode *ui = ubifs_inode(inode); ino_t inum = inode->i_ino; unsigned int blk; @@ -1562,10 +1565,13 @@ int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode, ubifs_assert(c, S_ISREG(inode->i_mode)); ubifs_assert(c, mutex_is_locked(&ui->ui_mutex)); - sz = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ + - UBIFS_MAX_DATA_NODE_SZ * WORST_COMPR_FACTOR; + dn_size = COMPRESSED_DATA_NODE_BUF_SZ; - sz += ubifs_auth_node_sz(c); + if (IS_ENCRYPTED(inode)) + dn_size += UBIFS_CIPHER_BLOCK_SIZE; + + sz = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ + + dn_size + ubifs_auth_node_sz(c); ino = kmalloc(sz, GFP_NOFS); if (!ino) @@ -1596,15 +1602,15 @@ int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode, if (dn_len <= 0 || dn_len > UBIFS_BLOCK_SIZE) { ubifs_err(c, "bad data node (block %u, inode %lu)", blk, inode->i_ino); - ubifs_dump_node(c, dn, sz - UBIFS_INO_NODE_SZ - - UBIFS_TRUN_NODE_SZ); + ubifs_dump_node(c, dn, dn_size); goto out_free; } if (dn_len <= dlen) dlen = 0; /* Nothing to do */ else { - err = truncate_data_node(c, inode, blk, dn, &dlen); + err = truncate_data_node(c, inode, blk, dn, + &dlen, dn_size); if (err) goto out_free; } diff --git a/fs/ubifs/lpt_commit.c b/fs/ubifs/lpt_commit.c index d76a19e460cd421b863760dfabfa6b4bc0329c9b..cfbc31f709f4b19c53c8c0a413b58f1b1b0a3c5b 100644 --- a/fs/ubifs/lpt_commit.c +++ b/fs/ubifs/lpt_commit.c @@ -1970,28 +1970,28 @@ static int dbg_populate_lsave(struct ubifs_info *c) if (!dbg_is_chk_gen(c)) return 0; - if (prandom_u32() & 3) + if (prandom_u32_max(4)) return 0; for (i = 0; i < c->lsave_cnt; i++) c->lsave[i] = c->main_first; list_for_each_entry(lprops, &c->empty_list, list) - c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum; list_for_each_entry(lprops, &c->freeable_list, list) - c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum; list_for_each_entry(lprops, &c->frdi_idx_list, list) - c->lsave[prandom_u32() % c->lsave_cnt] = lprops->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = lprops->lnum; heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1]; for (i = 0; i < heap->cnt; i++) - c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum; heap = &c->lpt_heap[LPROPS_DIRTY - 1]; for (i = 0; i < heap->cnt; i++) - c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum; heap = &c->lpt_heap[LPROPS_FREE - 1]; for (i = 0; i < heap->cnt; i++) - c->lsave[prandom_u32() % c->lsave_cnt] = heap->arr[i]->lnum; + c->lsave[prandom_u32_max(c->lsave_cnt)] = heap->arr[i]->lnum; return 1; } diff --git a/fs/ubifs/tnc_commit.c b/fs/ubifs/tnc_commit.c index 58c92c96ecef251619ff13e6bbf32931f5f165ec..01362ad5f804ae67b73fba3953695ef52525e2f9 100644 --- a/fs/ubifs/tnc_commit.c +++ b/fs/ubifs/tnc_commit.c @@ -700,7 +700,7 @@ static int alloc_idx_lebs(struct ubifs_info *c, int cnt) c->ilebs[c->ileb_cnt++] = lnum; dbg_cmt("LEB %d", lnum); } - if (dbg_is_chk_index(c) && !(prandom_u32() & 7)) + if (dbg_is_chk_index(c) && !prandom_u32_max(8)) return -ENOSPC; return 0; } diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h index 7d6d2f152e039ccfaf08c01f673dd8624a50cf15..478bbbb5382f88048bb5d5a10077a68ac7135cc6 100644 --- a/fs/ubifs/ubifs.h +++ b/fs/ubifs/ubifs.h @@ -2026,7 +2026,7 @@ int ubifs_update_time(struct inode *inode, struct timespec64 *time, int flags); /* dir.c */ struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, - umode_t mode); + umode_t mode, bool is_xattr); int ubifs_getattr(struct user_namespace *mnt_userns, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int flags); int ubifs_check_dir_empty(struct inode *dir); diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c index e4c4761aff7f828cf01216ffe180439cf8cbe8b0..3db8486e3725e04a8985e60da7814e359df2a56a 100644 --- a/fs/ubifs/xattr.c +++ b/fs/ubifs/xattr.c @@ -110,7 +110,7 @@ static int create_xattr(struct ubifs_info *c, struct inode *host, if (err) return err; - inode = ubifs_new_inode(c, host, S_IFREG | S_IRWXUGO); + inode = ubifs_new_inode(c, host, S_IFREG | S_IRWXUGO, true); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out_budg; diff --git a/fs/udf/dir.c b/fs/udf/dir.c index cad3772f9dbe21faabfe3dbe04bda1901db8b714..be640f4b2f2c8d4c1eaef27e27569bc7806157c5 100644 --- a/fs/udf/dir.c +++ b/fs/udf/dir.c @@ -130,7 +130,7 @@ static int udf_readdir(struct file *file, struct dir_context *ctx) brelse(tmp); } if (num) { - ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha); + bh_readahead_batch(num, bha, REQ_RAHEAD); for (i = 0; i < num; i++) brelse(bha[i]); } diff --git a/fs/udf/directory.c b/fs/udf/directory.c index a2adf62930937448f6e7c80628769c4289008a76..16bcf2c6b8b3bff74d0dc9b2a55632f23f98f743 100644 --- a/fs/udf/directory.c +++ b/fs/udf/directory.c @@ -89,7 +89,7 @@ struct fileIdentDesc *udf_fileident_read(struct inode *dir, loff_t *nf_pos, brelse(tmp); } if (num) { - ll_rw_block(REQ_OP_READ | REQ_RAHEAD, num, bha); + bh_readahead_batch(num, bha, REQ_RAHEAD); for (i = 0; i < num; i++) brelse(bha[i]); } diff --git a/fs/udf/file.c b/fs/udf/file.c index 09aef77269fe4625b48aa558581f518ba93ab12a..5c659e23e578fa33290ad6d72aa7b7fb5ece1fad 100644 --- a/fs/udf/file.c +++ b/fs/udf/file.c @@ -252,6 +252,7 @@ const struct file_operations udf_file_operations = { .release = udf_release_file, .fsync = generic_file_fsync, .splice_read = generic_file_splice_read, + .splice_write = iter_file_splice_write, .llseek = generic_file_llseek, }; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 8d06daed549f992a60cc35cc7f6fa2d1256940ed..dce6ae9ae306c7b0dc80593e8d6fb500d595f9c9 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1211,13 +1211,7 @@ struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, if (!bh) return NULL; - if (buffer_uptodate(bh)) - return bh; - - ll_rw_block(REQ_OP_READ, 1, &bh); - - wait_on_buffer(bh); - if (buffer_uptodate(bh)) + if (bh_read(bh, 0) >= 0) return bh; brelse(bh); diff --git a/fs/udf/namei.c b/fs/udf/namei.c index b3d5f97f16cdb174b2e91ff4514caae2f96dfe26..fb4c30e052453dab74e1f18d4eab764a384d4e9d 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -626,7 +626,7 @@ static int udf_create(struct user_namespace *mnt_userns, struct inode *dir, } static int udf_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { struct inode *inode = udf_new_inode(dir, mode); @@ -640,9 +640,9 @@ static int udf_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, inode->i_op = &udf_file_inode_operations; inode->i_fop = &udf_file_operations; mark_inode_dirty(inode); - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); unlock_new_inode(inode); - return 0; + return finish_open_simple(file, 0); } static int udf_mknod(struct user_namespace *mnt_userns, struct inode *dir, diff --git a/fs/ufs/balloc.c b/fs/ufs/balloc.c index bd810d8239f2ac8f6c51de36d6a8e319f93cf046..2436e3f82147fd4ede1a6b24c845133cc63c0d5e 100644 --- a/fs/ufs/balloc.c +++ b/fs/ufs/balloc.c @@ -295,14 +295,10 @@ static void ufs_change_blocknr(struct inode *inode, sector_t beg, if (!buffer_mapped(bh)) map_bh(bh, inode->i_sb, oldb + pos); - if (!buffer_uptodate(bh)) { - ll_rw_block(REQ_OP_READ, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { - ufs_error(inode->i_sb, __func__, - "read of block failed\n"); - break; - } + if (bh_read(bh, 0) < 0) { + ufs_error(inode->i_sb, __func__, + "read of block failed\n"); + break; } UFSD(" change from %llu to %llu, pos %u\n", diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 1c44bf75f9160cd509f684f0727c8a36be085527..07c81ab3fd4dd301f830180f540db84dc711e00c 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -30,6 +30,7 @@ #include #include #include +#include int sysctl_unprivileged_userfaultfd __read_mostly; @@ -415,13 +416,8 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason) if (ctx->features & UFFD_FEATURE_SIGBUS) goto out; - if ((vmf->flags & FAULT_FLAG_USER) == 0 && - ctx->flags & UFFD_USER_MODE_ONLY) { - printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd " - "sysctl knob to 1 if kernel faults must be handled " - "without obtaining CAP_SYS_PTRACE capability\n"); + if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY)) goto out; - } /* * If it's already released don't get it. This avoids to loop @@ -615,14 +611,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx, if (release_new_ctx) { struct vm_area_struct *vma; struct mm_struct *mm = release_new_ctx->mm; + VMA_ITERATOR(vmi, mm, 0); /* the various vma->vm_userfaultfd_ctx still points to it */ mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) + for_each_vma(vmi, vma) { if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; vma->vm_flags &= ~__VM_UFFD_FLAGS; } + } mmap_write_unlock(mm); userfaultfd_ctx_put(release_new_ctx); @@ -803,11 +801,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps, return false; } -int userfaultfd_unmap_prep(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - struct list_head *unmaps) +int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, + unsigned long end, struct list_head *unmaps) { - for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { + VMA_ITERATOR(vmi, mm, start); + struct vm_area_struct *vma; + + for_each_vma_range(vmi, vma, end) { struct userfaultfd_unmap_ctx *unmap_ctx; struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; @@ -857,6 +857,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) /* len == 0 means wake all */ struct userfaultfd_wake_range range = { .len = 0, }; unsigned long new_flags; + MA_STATE(mas, &mm->mm_mt, 0, 0); WRITE_ONCE(ctx->released, true); @@ -873,7 +874,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file) */ mmap_write_lock(mm); prev = NULL; - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { cond_resched(); BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ !!(vma->vm_flags & __VM_UFFD_FLAGS)); @@ -887,10 +888,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file) vma->vm_file, vma->vm_pgoff, vma_policy(vma), NULL_VM_UFFD_CTX, anon_vma_name(vma)); - if (prev) + if (prev) { + mas_pause(&mas); vma = prev; - else + } else { prev = vma; + } + vma->vm_flags = new_flags; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; } @@ -991,7 +995,7 @@ static int resolve_userfault_fork(struct userfaultfd_ctx *new, int fd; fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new, - O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); + O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode); if (fd < 0) return fd; @@ -1272,6 +1276,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, bool found; bool basic_ioctls; unsigned long start, end, vma_end; + MA_STATE(mas, &mm->mm_mt, 0, 0); user_uffdio_register = (struct uffdio_register __user *) arg; @@ -1314,7 +1319,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, goto out; mmap_write_lock(mm); - vma = find_vma_prev(mm, start, &prev); + mas_set(&mas, start); + vma = mas_find(&mas, ULONG_MAX); if (!vma) goto out_unlock; @@ -1339,7 +1345,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, */ found = false; basic_ioctls = false; - for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { + for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { cond_resched(); BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ @@ -1399,8 +1405,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, } BUG_ON(!found); - if (vma->vm_start < start) - prev = vma; + mas_set(&mas, start); + prev = mas_prev(&mas, 0); + if (prev != vma) + mas_next(&mas, ULONG_MAX); ret = 0; do { @@ -1430,6 +1438,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ((struct vm_userfaultfd_ctx){ ctx }), anon_vma_name(vma)); if (prev) { + /* vma_merge() invalidated the mas */ + mas_pause(&mas); vma = prev; goto next; } @@ -1437,11 +1447,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ret = split_vma(mm, vma, start, 1); if (ret) break; + /* split_vma() invalidated the mas */ + mas_pause(&mas); } if (vma->vm_end > end) { ret = split_vma(mm, vma, end, 0); if (ret) break; + /* split_vma() invalidated the mas */ + mas_pause(&mas); } next: /* @@ -1458,8 +1472,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, skip: prev = vma; start = vma->vm_end; - vma = vma->vm_next; - } while (vma && vma->vm_start < end); + vma = mas_next(&mas, end - 1); + } while (vma); out_unlock: mmap_write_unlock(mm); mmput(mm); @@ -1503,6 +1517,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, bool found; unsigned long start, end, vma_end; const void __user *buf = (void __user *)arg; + MA_STATE(mas, &mm->mm_mt, 0, 0); ret = -EFAULT; if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) @@ -1521,7 +1536,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, goto out; mmap_write_lock(mm); - vma = find_vma_prev(mm, start, &prev); + mas_set(&mas, start); + vma = mas_find(&mas, ULONG_MAX); if (!vma) goto out_unlock; @@ -1546,7 +1562,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, */ found = false; ret = -EINVAL; - for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { + for (cur = vma; cur; cur = mas_next(&mas, end - 1)) { cond_resched(); BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ @@ -1566,8 +1582,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, } BUG_ON(!found); - if (vma->vm_start < start) - prev = vma; + mas_set(&mas, start); + prev = mas_prev(&mas, 0); + if (prev != vma) + mas_next(&mas, ULONG_MAX); ret = 0; do { @@ -1601,6 +1619,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range); } + /* Reset ptes for the whole vma range if wr-protected */ + if (userfaultfd_wp(vma)) + uffd_wp_range(mm, vma, start, vma_end - start, false); + new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS; prev = vma_merge(mm, prev, start, vma_end, new_flags, vma->anon_vma, vma->vm_file, vma->vm_pgoff, @@ -1632,8 +1654,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, skip: prev = vma; start = vma->vm_end; - vma = vma->vm_next; - } while (vma && vma->vm_start < end); + vma = mas_next(&mas, end - 1); + } while (vma); out_unlock: mmap_write_unlock(mm); mmput(mm); @@ -2052,20 +2074,11 @@ static void init_once_userfaultfd_ctx(void *mem) seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock); } -SYSCALL_DEFINE1(userfaultfd, int, flags) +static int new_userfaultfd(int flags) { struct userfaultfd_ctx *ctx; int fd; - if (!sysctl_unprivileged_userfaultfd && - (flags & UFFD_USER_MODE_ONLY) == 0 && - !capable(CAP_SYS_PTRACE)) { - printk_once(KERN_WARNING "uffd: Set unprivileged_userfaultfd " - "sysctl knob to 1 if kernel faults must be handled " - "without obtaining CAP_SYS_PTRACE capability\n"); - return -EPERM; - } - BUG_ON(!current->mm); /* Check the UFFD_* constants for consistency. */ @@ -2090,7 +2103,7 @@ SYSCALL_DEFINE1(userfaultfd, int, flags) mmgrab(ctx->mm); fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx, - O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); + O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL); if (fd < 0) { mmdrop(ctx->mm); kmem_cache_free(userfaultfd_ctx_cachep, ctx); @@ -2098,8 +2111,60 @@ SYSCALL_DEFINE1(userfaultfd, int, flags) return fd; } +static inline bool userfaultfd_syscall_allowed(int flags) +{ + /* Userspace-only page faults are always allowed */ + if (flags & UFFD_USER_MODE_ONLY) + return true; + + /* + * The user is requesting a userfaultfd which can handle kernel faults. + * Privileged users are always allowed to do this. + */ + if (capable(CAP_SYS_PTRACE)) + return true; + + /* Otherwise, access to kernel fault handling is sysctl controlled. */ + return sysctl_unprivileged_userfaultfd; +} + +SYSCALL_DEFINE1(userfaultfd, int, flags) +{ + if (!userfaultfd_syscall_allowed(flags)) + return -EPERM; + + return new_userfaultfd(flags); +} + +static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags) +{ + if (cmd != USERFAULTFD_IOC_NEW) + return -EINVAL; + + return new_userfaultfd(flags); +} + +static const struct file_operations userfaultfd_dev_fops = { + .unlocked_ioctl = userfaultfd_dev_ioctl, + .compat_ioctl = userfaultfd_dev_ioctl, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice userfaultfd_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "userfaultfd", + .fops = &userfaultfd_dev_fops +}; + static int __init userfaultfd_init(void) { + int ret; + + ret = misc_register(&userfaultfd_misc); + if (ret) + return ret; + userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", sizeof(struct userfaultfd_ctx), 0, diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 629785c95007641865a2b32165f382774ac18753..dbe1ce5b450a80873fc751273fef20350510cd79 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -70,8 +70,6 @@ struct fsverity_info { const struct inode *inode; }; -/* Arbitrary limit to bound the kmalloc() size. Can be changed. */ -#define FS_VERITY_MAX_DESCRIPTOR_SIZE 16384 #define FS_VERITY_MAX_SIGNATURE_SIZE (FS_VERITY_MAX_DESCRIPTOR_SIZE - \ sizeof(struct fsverity_descriptor)) diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c index 6ee849dc7bc183c0f55212cb5a5639fe2d134e19..2aefc5565152ad15e3abc46fe8b0cc9045faccaa 100644 --- a/fs/verity/read_metadata.c +++ b/fs/verity/read_metadata.c @@ -53,14 +53,14 @@ static int fsverity_read_merkle_tree(struct inode *inode, break; } - virt = kmap(page); + virt = kmap_local_page(page); if (copy_to_user(buf, virt + offs_in_page, bytes_to_copy)) { - kunmap(page); + kunmap_local(virt); put_page(page); err = -EFAULT; break; } - kunmap(page); + kunmap_local(virt); put_page(page); retval += bytes_to_copy; diff --git a/fs/verity/verify.c b/fs/verity/verify.c index 14e2fb49cff561a9a16ffa71539c0e340fe67e92..bde8c9b7d25f64b9fde4196b02df5679b88ae809 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -39,16 +39,6 @@ static void hash_at_level(const struct merkle_tree_params *params, (params->log_blocksize - params->log_arity); } -/* Extract a hash from a hash page */ -static void extract_hash(struct page *hpage, unsigned int hoffset, - unsigned int hsize, u8 *out) -{ - void *virt = kmap_atomic(hpage); - - memcpy(out, virt + hoffset, hsize); - kunmap_atomic(virt); -} - static inline int cmp_hashes(const struct fsverity_info *vi, const u8 *want_hash, const u8 *real_hash, pgoff_t index, int level) @@ -129,7 +119,7 @@ static bool verify_page(struct inode *inode, const struct fsverity_info *vi, } if (PageChecked(hpage)) { - extract_hash(hpage, hoffset, hsize, _want_hash); + memcpy_from_page(_want_hash, hpage, hoffset, hsize); want_hash = _want_hash; put_page(hpage); pr_debug_ratelimited("Hash page already checked, want %s:%*phN\n", @@ -158,7 +148,7 @@ descend: if (err) goto out; SetPageChecked(hpage); - extract_hash(hpage, hoffset, hsize, _want_hash); + memcpy_from_page(_want_hash, hpage, hoffset, hsize); want_hash = _want_hash; put_page(hpage); pr_debug("Verified hash page at level %d, now want %s:%*phN\n", diff --git a/fs/xattr.c b/fs/xattr.c index a1f4998bc6be30efe98aa47f494a533c25153c7f..61107b6bbed29211786d2eda0318499d95f525fe 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -290,7 +290,7 @@ static inline bool is_posix_acl_xattr(const char *name) int vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, - const char *name, void *value, size_t size, int flags) + const char *name, const void *value, size_t size, int flags) { struct inode *inode = dentry->d_inode; struct inode *delegated_inode = NULL; @@ -298,16 +298,12 @@ vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, int error; if (size && strcmp(name, XATTR_NAME_CAPS) == 0) { - error = cap_convert_nscap(mnt_userns, dentry, - (const void **)&value, size); + error = cap_convert_nscap(mnt_userns, dentry, &value, size); if (error < 0) return error; size = error; } - if (size && is_posix_acl_xattr(name)) - posix_acl_setxattr_idmapped_mnt(mnt_userns, inode, value, size); - retry_deleg: inode_lock(inode); error = __vfs_setxattr_locked(mnt_userns, dentry, name, value, size, @@ -587,9 +583,7 @@ int setxattr_copy(const char __user *name, struct xattr_ctx *ctx) static void setxattr_convert(struct user_namespace *mnt_userns, struct dentry *d, struct xattr_ctx *ctx) { - if (ctx->size && - ((strcmp(ctx->kname->name, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || - (strcmp(ctx->kname->name, XATTR_NAME_POSIX_ACL_DEFAULT) == 0))) + if (ctx->size && is_posix_acl_xattr(ctx->kname->name)) posix_acl_fix_xattr_from_user(ctx->kvalue, ctx->size); } @@ -705,8 +699,7 @@ do_getxattr(struct user_namespace *mnt_userns, struct dentry *d, error = vfs_getxattr(mnt_userns, d, kname, ctx->kvalue, ctx->size); if (error > 0) { - if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) || - (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)) + if (is_posix_acl_xattr(kname)) posix_acl_fix_xattr_to_user(ctx->kvalue, error); if (ctx->size && copy_to_user(ctx->value, ctx->kvalue, error)) error = -EFAULT; diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index e2bdf089c0a31eccc23da94ef68a6b7a2193b605..6261599bb389af955f4dc16fe63b36d1cdf0d8c7 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -1520,7 +1520,7 @@ xfs_alloc_ag_vextent_lastblock( #ifdef DEBUG /* Randomly don't execute the first algorithm. */ - if (prandom_u32() & 1) + if (prandom_u32_max(2)) return 0; #endif diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index e56723dc9cd5b940623aad0b05dc1facdc676903..49d0d4ea63fcdb11f9b1267f2c30e0585e765e70 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -294,7 +294,7 @@ xfs_check_block( else thispa = XFS_BMBT_PTR_ADDR(mp, block, j, dmxr); if (*thispa == *pp) { - xfs_warn(mp, "%s: thispa(%d) == pp(%d) %Ld", + xfs_warn(mp, "%s: thispa(%d) == pp(%d) %lld", __func__, j, i, (unsigned long long)be64_to_cpu(*thispa)); xfs_err(mp, "%s: ptrs are equal in node\n", diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index e7201dc68f4309a422ce611fb53db2d69cd95caf..e576560b46e9789bcd7496de926b840cc46a564b 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -2192,8 +2192,8 @@ xfs_da_grow_inode_int( */ mapp = kmem_alloc(sizeof(*mapp) * count, 0); for (b = *bno, mapi = 0; b < *bno + count; ) { - nmap = min(XFS_BMAP_MAX_NMAP, count); c = (int)(*bno + count - b); + nmap = min(XFS_BMAP_MAX_NMAP, c); error = xfs_bmapi_write(tp, dp, b, c, xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA, args->total, &mapp[mapi], &nmap); diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index 76eedc2756b318cc1967b7699586ad40ec648507..92bac3373f1f5cf55cbdd967db0d3534758518e8 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -261,7 +261,7 @@ xfs_dir_createname( { struct xfs_da_args *args; int rval; - int v; /* type-checking value */ + bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -357,7 +357,7 @@ xfs_dir_lookup( { struct xfs_da_args *args; int rval; - int v; /* type-checking value */ + bool v; int lock_mode; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -435,7 +435,7 @@ xfs_dir_removename( { struct xfs_da_args *args; int rval; - int v; /* type-checking value */ + bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); XFS_STATS_INC(dp->i_mount, xs_dir_remove); @@ -493,7 +493,7 @@ xfs_dir_replace( { struct xfs_da_args *args; int rval; - int v; /* type-checking value */ + bool v; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); @@ -610,19 +610,23 @@ xfs_dir2_grow_inode( int xfs_dir2_isblock( struct xfs_da_args *args, - int *vp) /* out: 1 is block, 0 is not block */ + bool *isblock) { - xfs_fileoff_t last; /* last file offset */ - int rval; + struct xfs_mount *mp = args->dp->i_mount; + xfs_fileoff_t eof; + int error; - if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK))) - return rval; - rval = XFS_FSB_TO_B(args->dp->i_mount, last) == args->geo->blksize; - if (XFS_IS_CORRUPT(args->dp->i_mount, - rval != 0 && - args->dp->i_disk_size != args->geo->blksize)) + error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); + if (error) + return error; + + *isblock = false; + if (XFS_FSB_TO_B(mp, eof) != args->geo->blksize) + return 0; + + *isblock = true; + if (XFS_IS_CORRUPT(mp, args->dp->i_disk_size != args->geo->blksize)) return -EFSCORRUPTED; - *vp = rval; return 0; } @@ -632,14 +636,20 @@ xfs_dir2_isblock( int xfs_dir2_isleaf( struct xfs_da_args *args, - int *vp) /* out: 1 is block, 0 is not block */ + bool *isleaf) { - xfs_fileoff_t last; /* last file offset */ - int rval; + xfs_fileoff_t eof; + int error; - if ((rval = xfs_bmap_last_offset(args->dp, &last, XFS_DATA_FORK))) - return rval; - *vp = last == args->geo->leafblk + args->geo->fsbcount; + error = xfs_bmap_last_offset(args->dp, &eof, XFS_DATA_FORK); + if (error) + return error; + + *isleaf = false; + if (eof != args->geo->leafblk + args->geo->fsbcount) + return 0; + + *isleaf = true; return 0; } diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index b6df3c34b26afb8a703aece814770a283619915b..dd39f17dd9a9cac81fea1f0c76b9545f547f53b7 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -61,8 +61,8 @@ extern int xfs_dir2_sf_to_block(struct xfs_da_args *args); /* * Interface routines used by userspace utilities */ -extern int xfs_dir2_isblock(struct xfs_da_args *args, int *r); -extern int xfs_dir2_isleaf(struct xfs_da_args *args, int *r); +extern int xfs_dir2_isblock(struct xfs_da_args *args, bool *isblock); +extern int xfs_dir2_isleaf(struct xfs_da_args *args, bool *isleaf); extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db, struct xfs_buf *bp); diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c index 003812fd7d3559d13d3fbcbec8be733408ec8ca0..8cd37e6e9d387959133e33c379be16469f512549 100644 --- a/fs/xfs/libxfs/xfs_dir2_sf.c +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -865,7 +865,6 @@ xfs_dir2_sf_lookup( struct xfs_inode *dp = args->dp; struct xfs_mount *mp = dp->i_mount; int i; /* entry index */ - int error; xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ xfs_dir2_sf_hdr_t *sfp; /* shortform structure */ enum xfs_dacmp cmp; /* comparison result */ @@ -929,8 +928,7 @@ xfs_dir2_sf_lookup( if (!ci_sfep) return -ENOENT; /* otherwise process the CI match as required by the caller */ - error = xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen); - return error; + return xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen); } /* diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c index 6cdfd64bc56bdd2132721a38edf5c16a7e9fc64a..94db50eb706ac5e560a65e001142f40442dbd932 100644 --- a/fs/xfs/libxfs/xfs_ialloc.c +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -636,7 +636,7 @@ xfs_ialloc_ag_alloc( /* randomly do sparse inode allocations */ if (xfs_has_sparseinodes(tp->t_mountp) && igeo->ialloc_min_blks < igeo->ialloc_blks) - do_sparse = prandom_u32() & 1; + do_sparse = prandom_u32_max(2); #endif /* @@ -805,7 +805,7 @@ sparse_alloc: * number from being easily guessable. */ error = xfs_ialloc_inode_init(args.mp, tp, NULL, newlen, pag->pag_agno, - args.agbno, args.len, prandom_u32()); + args.agbno, args.len, get_random_u32()); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 9327a4f392065605dc76f8e54131c4dc6b84a681..6b21760184d9ed33620bec3a72e78f49b161fd24 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -78,7 +78,7 @@ xfs_iformat_local( */ if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { xfs_warn(ip->i_mount, - "corrupt inode %Lu (bad size %d for local fork, size = %zd).", + "corrupt inode %llu (bad size %d for local fork, size = %zd).", (unsigned long long) ip->i_ino, size, XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); xfs_inode_verifier_error(ip, -EFSCORRUPTED, @@ -192,7 +192,7 @@ xfs_iformat_btree( XFS_DFORK_SIZE(dip, mp, whichfork) || ifp->if_nextents > ip->i_nblocks) || level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) { - xfs_warn(mp, "corrupt inode %Lu (btree).", + xfs_warn(mp, "corrupt inode %llu (btree).", (unsigned long long) ip->i_ino); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "xfs_iformat_btree", dfp, size, diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 5abb5fdb71d93992e53f040d1552f68ab7a26443..5c87800ab223df02bb959ca021281f44d5e04e1b 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -99,7 +99,7 @@ out: * we check the inode number to make sure it's sane, then we check that * we can look up this filename. Finally, we check the ftype. */ -STATIC int +STATIC bool xchk_dir_actor( struct dir_context *dir_iter, const char *name, @@ -124,7 +124,7 @@ xchk_dir_actor( xfs_dir2_dataptr_to_db(mp->m_dir_geo, pos)); if (xchk_should_terminate(sdc->sc, &error)) - return error; + return !error; /* Does this inode number make sense? */ if (!xfs_verify_dir_ino(mp, ino)) { @@ -191,8 +191,8 @@ out: * and return zero to xchk_directory. */ if (error == 0 && sdc->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) - return -EFSCORRUPTED; - return error; + return false; + return !error; } /* Scrub a directory btree record. */ @@ -676,7 +676,7 @@ xchk_directory_blocks( xfs_dablk_t dabno; xfs_dir2_db_t last_data_db = 0; bool found; - int is_block = 0; + bool is_block = false; int error; /* Ignore local format directories. */ diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index ab182a5cd0c0adca5d987f8bb72b3ca01a3ca0c1..d8dff3fd80530c8ed2f6f6a33b18afbdbfbfc1bf 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -38,7 +38,7 @@ struct xchk_parent_ctx { }; /* Look for a single entry in a directory pointing to an inode. */ -STATIC int +STATIC bool xchk_parent_actor( struct dir_context *dc, const char *name, @@ -62,7 +62,7 @@ xchk_parent_actor( if (xchk_should_terminate(spc->sc, &error)) spc->cancelled = true; - return error; + return !error; } /* Count the number of dentries in the parent dir that point to this inode. */ diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index 5077a7ad56460f78cffb578f3bddc381e2da8cd3..cf5ce607dc051fe15bfb211cff03d0345a73ef04 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -86,8 +86,6 @@ xfs_attri_log_nameval_alloc( */ nv = xlog_kvmalloc(sizeof(struct xfs_attri_log_nameval) + name_len + value_len); - if (!nv) - return nv; nv->name.i_addr = nv + 1; nv->name.i_len = name_len; @@ -441,8 +439,6 @@ xfs_attr_create_intent( attr->xattri_nameval = xfs_attri_log_nameval_alloc(args->name, args->namelen, args->value, args->valuelen); } - if (!attr->xattri_nameval) - return ERR_PTR(-ENOMEM); attrip = xfs_attri_init(mp, attr->xattri_nameval); xfs_trans_add_item(tp, &attrip->attri_item); @@ -762,8 +758,6 @@ xlog_recover_attri_commit_pass2( nv = xfs_attri_log_nameval_alloc(attr_name, attri_formatp->alfi_name_len, attr_value, attri_formatp->alfi_value_len); - if (!nv) - return -ENOMEM; attrip = xfs_attri_init(mp, nv); error = xfs_attri_copy_format(&item->ri_buf[0], &attrip->attri_format); diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index e295fc8062d81e2c5cc5f395fe211c79c7b76d7d..9f3ceb46151566834d63840162b51f4caf6cc8dd 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -512,7 +512,7 @@ xfs_readdir( { struct xfs_da_args args = { NULL }; unsigned int lock_mode; - int isblock; + bool isblock; int error; trace_xfs_readdir(dp); diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 296faa41d81d51ee4d0ebdad3fefbf5d671f92ec..7db588ed0be597b9329e856a726795a1d1da6893 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -274,7 +274,7 @@ xfs_errortag_test( ASSERT(error_tag < XFS_ERRTAG_MAX); randfactor = mp->m_errortag[error_tag]; - if (!randfactor || prandom_u32() % randfactor) + if (!randfactor || prandom_u32_max(randfactor)) return false; xfs_warn_ratelimited(mp, diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 2bbe7916a998d7efbc52aea8b9d90efc897035ed..eae7427062cf9a9448940a84ea4403d76b93fc39 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -596,7 +596,7 @@ xfs_iget_cache_miss( */ if (xfs_has_v3inodes(mp) && (flags & XFS_IGET_CREATE) && !xfs_has_ikeep(mp)) { - VFS_I(ip)->i_generation = prandom_u32(); + VFS_I(ip)->i_generation = get_random_u32(); } else { struct xfs_buf *bp; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 28493c8e9bb231accb53884e1fd126c9d366c407..c000b74dd203582517081fe253d770b8205493fb 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -835,9 +835,8 @@ xfs_init_new_inode( * ID or one of the supplementary group IDs, the S_ISGID bit is cleared * (and only if the irix_sgid_inherit compatibility variable is set). */ - if (irix_sgid_inherit && - (inode->i_mode & S_ISGID) && - !in_group_p(i_gid_into_mnt(mnt_userns, inode))) + if (irix_sgid_inherit && (inode->i_mode & S_ISGID) && + !vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode))) inode->i_mode &= ~S_ISGID; ip->i_disk_size = 0; @@ -3119,7 +3118,7 @@ xfs_iflush( if (XFS_TEST_ERROR(dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC), mp, XFS_ERRTAG_IFLUSH_1)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, - "%s: Bad inode %Lu magic number 0x%x, ptr "PTR_FMT, + "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT, __func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip); goto flush_out; } @@ -3129,7 +3128,7 @@ xfs_iflush( ip->i_df.if_format != XFS_DINODE_FMT_BTREE, mp, XFS_ERRTAG_IFLUSH_3)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, - "%s: Bad regular inode %Lu, ptr "PTR_FMT, + "%s: Bad regular inode %llu, ptr "PTR_FMT, __func__, ip->i_ino, ip); goto flush_out; } @@ -3140,7 +3139,7 @@ xfs_iflush( ip->i_df.if_format != XFS_DINODE_FMT_LOCAL, mp, XFS_ERRTAG_IFLUSH_4)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, - "%s: Bad directory inode %Lu, ptr "PTR_FMT, + "%s: Bad directory inode %llu, ptr "PTR_FMT, __func__, ip->i_ino, ip); goto flush_out; } @@ -3158,7 +3157,7 @@ xfs_iflush( if (XFS_TEST_ERROR(ip->i_forkoff > mp->m_sb.sb_inodesize, mp, XFS_ERRTAG_IFLUSH_6)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, - "%s: bad inode %Lu, forkoff 0x%x, ptr "PTR_FMT, + "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT, __func__, ip->i_ino, ip->i_forkoff, ip); goto flush_out; } diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 6e19ece916bfb5c49713712b5beef37d824a377a..ca2941ab6cbcdd714787dd88040f407a578ab5b3 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -550,7 +550,7 @@ xfs_inode_item_push( if (!bp || (ip->i_flags & XFS_ISTALE)) { /* - * Inode item/buffer is being being aborted due to cluster + * Inode item/buffer is being aborted due to cluster * buffer deletion. Trigger a log force to have that operation * completed and items removed from the AIL before the next push * attempt. diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index d28ffaebd067000efbe0150b47cb217a23ee990d..0e5dba2343ea13e2e0451178097f6ad247371724 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -321,7 +321,7 @@ xlog_recover_inode_commit_pass2( */ if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) { xfs_alert(mp, - "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %Ld", + "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %lld", __func__, dip, bp, in_f->ilf_ino); error = -EFSCORRUPTED; goto out_release; @@ -329,7 +329,7 @@ xlog_recover_inode_commit_pass2( ldip = item->ri_buf[1].i_addr; if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) { xfs_alert(mp, - "%s: Bad inode log record, rec ptr "PTR_FMT", ino %Ld", + "%s: Bad inode log record, rec ptr "PTR_FMT", ino %lld", __func__, item, in_f->ilf_ino); error = -EFSCORRUPTED; goto out_release; diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 45518b8c613c9a97fd364ccc7fead0fcf91f4771..2e10e1c66ad60a6ba0340ffda78b8baab67d6c2a 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -167,7 +167,7 @@ xfs_generic_create( struct dentry *dentry, umode_t mode, dev_t rdev, - bool tmpfile) /* unnamed file */ + struct file *tmpfile) /* unnamed file */ { struct inode *inode; struct xfs_inode *ip = NULL; @@ -234,7 +234,7 @@ xfs_generic_create( * d_tmpfile can immediately set it back to zero. */ set_nlink(inode, 1); - d_tmpfile(dentry, inode); + d_tmpfile(tmpfile, inode); } else d_instantiate(dentry, inode); @@ -261,7 +261,7 @@ xfs_vn_mknod( umode_t mode, dev_t rdev) { - return xfs_generic_create(mnt_userns, dir, dentry, mode, rdev, false); + return xfs_generic_create(mnt_userns, dir, dentry, mode, rdev, NULL); } STATIC int @@ -272,7 +272,7 @@ xfs_vn_create( umode_t mode, bool flags) { - return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, false); + return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, NULL); } STATIC int @@ -283,7 +283,7 @@ xfs_vn_mkdir( umode_t mode) { return xfs_generic_create(mnt_userns, dir, dentry, mode | S_IFDIR, 0, - false); + NULL); } STATIC struct dentry * @@ -558,6 +558,8 @@ xfs_vn_getattr( struct inode *inode = d_inode(path->dentry); struct xfs_inode *ip = XFS_I(inode); struct xfs_mount *mp = ip->i_mount; + vfsuid_t vfsuid = i_uid_into_vfsuid(mnt_userns, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(mnt_userns, inode); trace_xfs_getattr(ip); @@ -568,8 +570,8 @@ xfs_vn_getattr( stat->dev = inode->i_sb->s_dev; stat->mode = inode->i_mode; stat->nlink = inode->i_nlink; - stat->uid = i_uid_into_mnt(mnt_userns, inode); - stat->gid = i_gid_into_mnt(mnt_userns, inode); + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); stat->ino = ip->i_ino; stat->atime = inode->i_atime; stat->mtime = inode->i_mtime; @@ -604,6 +606,16 @@ xfs_vn_getattr( stat->blksize = BLKDEV_IOSIZE; stat->rdev = inode->i_rdev; break; + case S_IFREG: + if (request_mask & STATX_DIOALIGN) { + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + struct block_device *bdev = target->bt_bdev; + + stat->result_mask |= STATX_DIOALIGN; + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + stat->dio_offset_align = bdev_logical_block_size(bdev); + } + fallthrough; default: stat->blksize = xfs_stat_blksize(ip); stat->rdev = 0; @@ -1080,10 +1092,12 @@ STATIC int xfs_vn_tmpfile( struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, + struct file *file, umode_t mode) { - return xfs_generic_create(mnt_userns, dir, dentry, mode, 0, true); + int err = xfs_generic_create(mnt_userns, dir, file->f_path.dentry, mode, 0, file); + + return finish_open_simple(file, err); } static const struct inode_operations xfs_inode_operations = { diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h index cb5fc68c9ea00e75a890ae37dafb03ab217a8afc..e570dcb5df8d52efcc42607173cba782e8a36d4b 100644 --- a/fs/xfs/xfs_iops.h +++ b/fs/xfs/xfs_iops.h @@ -13,7 +13,6 @@ extern const struct file_operations xfs_dir_file_operations; extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size); -extern void xfs_setattr_time(struct xfs_inode *ip, struct iattr *iattr); int xfs_vn_setattr_size(struct user_namespace *mnt_userns, struct dentry *dentry, struct iattr *vap); diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 36312b00b1642e2627f726b70a30d53847b9264d..a1c2bcf65d376fc485294ae296dd56ba31da03f3 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -66,6 +66,8 @@ xfs_bulkstat_one_int( struct xfs_bulkstat *buf = bc->buf; xfs_extnum_t nextents; int error = -EINVAL; + vfsuid_t vfsuid; + vfsgid_t vfsgid; if (xfs_internal_inum(mp, ino)) goto out_advance; @@ -81,14 +83,16 @@ xfs_bulkstat_one_int( ASSERT(ip != NULL); ASSERT(ip->i_imap.im_blkno != 0); inode = VFS_I(ip); + vfsuid = i_uid_into_vfsuid(mnt_userns, inode); + vfsgid = i_gid_into_vfsgid(mnt_userns, inode); /* xfs_iget returns the following without needing * further change. */ buf->bs_projectid = ip->i_projid; buf->bs_ino = ino; - buf->bs_uid = from_kuid(sb_userns, i_uid_into_mnt(mnt_userns, inode)); - buf->bs_gid = from_kgid(sb_userns, i_gid_into_mnt(mnt_userns, inode)); + buf->bs_uid = from_kuid(sb_userns, vfsuid_into_kuid(vfsuid)); + buf->bs_gid = from_kgid(sb_userns, vfsgid_into_kgid(vfsgid)); buf->bs_size = ip->i_disk_size; buf->bs_nlink = inode->i_nlink; diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index 386b0307aed8576a1a833f50aaf7df8b96b57c7d..f02a0dd522b3df1316df282e168c6d429792e5d2 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -226,12 +226,12 @@ xlog_ticket_reservation( if (head == &log->l_write_head) { ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); return tic->t_unit_res; - } else { - if (tic->t_flags & XLOG_TIC_PERM_RESERV) - return tic->t_unit_res * tic->t_cnt; - else - return tic->t_unit_res; } + + if (tic->t_flags & XLOG_TIC_PERM_RESERV) + return tic->t_unit_res * tic->t_cnt; + + return tic->t_unit_res; } STATIC bool @@ -3544,7 +3544,7 @@ xlog_ticket_alloc( tic->t_curr_res = unit_res; tic->t_cnt = cnt; tic->t_ocnt = cnt; - tic->t_tid = prandom_u32(); + tic->t_tid = get_random_u32(); if (permanent) tic->t_flags |= XLOG_TIC_PERM_RESERV; diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index f10c88cee116e17619a9ca03f08f87c998180ddd..e8bb3c2e847e1ad14769ccf2f285df0cb56929db 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -300,26 +300,28 @@ xfs_validate_new_dalign( "alignment check failed: sunit/swidth vs. blocksize(%d)", mp->m_sb.sb_blocksize); return -EINVAL; - } else { - /* - * Convert the stripe unit and width to FSBs. - */ - mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign); - if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) { - xfs_warn(mp, - "alignment check failed: sunit/swidth vs. agsize(%d)", - mp->m_sb.sb_agblocks); - return -EINVAL; - } else if (mp->m_dalign) { - mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth); - } else { - xfs_warn(mp, - "alignment check failed: sunit(%d) less than bsize(%d)", - mp->m_dalign, mp->m_sb.sb_blocksize); - return -EINVAL; - } } + /* + * Convert the stripe unit and width to FSBs. + */ + mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign); + if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) { + xfs_warn(mp, + "alignment check failed: sunit/swidth vs. agsize(%d)", + mp->m_sb.sb_agblocks); + return -EINVAL; + } + + if (!mp->m_dalign) { + xfs_warn(mp, + "alignment check failed: sunit(%d) less than bsize(%d)", + mp->m_dalign, mp->m_sb.sb_blocksize); + return -EINVAL; + } + + mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth); + if (!xfs_has_dalign(mp)) { xfs_warn(mp, "cannot change alignment: superblock does not support data alignment"); diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c index 69d9c83ea4b21ee5f6e91a28741c5acb3a474795..c4078d0ec108f1e64ad476a8b2a5e720c785fcfe 100644 --- a/fs/xfs/xfs_notify_failure.c +++ b/fs/xfs/xfs_notify_failure.c @@ -23,17 +23,18 @@ #include #include -struct failure_info { +struct xfs_failure_info { xfs_agblock_t startblock; xfs_extlen_t blockcount; int mf_flags; + bool want_shutdown; }; static pgoff_t xfs_failure_pgoff( struct xfs_mount *mp, const struct xfs_rmap_irec *rec, - const struct failure_info *notify) + const struct xfs_failure_info *notify) { loff_t pos = XFS_FSB_TO_B(mp, rec->rm_offset); @@ -47,7 +48,7 @@ static unsigned long xfs_failure_pgcnt( struct xfs_mount *mp, const struct xfs_rmap_irec *rec, - const struct failure_info *notify) + const struct xfs_failure_info *notify) { xfs_agblock_t end_rec; xfs_agblock_t end_notify; @@ -71,13 +72,13 @@ xfs_dax_failure_fn( { struct xfs_mount *mp = cur->bc_mp; struct xfs_inode *ip; - struct failure_info *notify = data; + struct xfs_failure_info *notify = data; int error = 0; if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) || (rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))) { - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); - return -EFSCORRUPTED; + notify->want_shutdown = true; + return 0; } /* Get files that incore, filter out others that are not in use. */ @@ -86,8 +87,10 @@ xfs_dax_failure_fn( /* Continue the rmap query if the inode isn't incore */ if (error == -ENODATA) return 0; - if (error) - return error; + if (error) { + notify->want_shutdown = true; + return 0; + } error = mf_dax_kill_procs(VFS_I(ip)->i_mapping, xfs_failure_pgoff(mp, rec, notify), @@ -104,6 +107,7 @@ xfs_dax_notify_ddev_failure( xfs_daddr_t bblen, int mf_flags) { + struct xfs_failure_info notify = { .mf_flags = mf_flags }; struct xfs_trans *tp = NULL; struct xfs_btree_cur *cur = NULL; struct xfs_buf *agf_bp = NULL; @@ -120,7 +124,6 @@ xfs_dax_notify_ddev_failure( for (; agno <= end_agno; agno++) { struct xfs_rmap_irec ri_low = { }; struct xfs_rmap_irec ri_high; - struct failure_info notify; struct xfs_agf *agf; xfs_agblock_t agend; struct xfs_perag *pag; @@ -161,6 +164,11 @@ xfs_dax_notify_ddev_failure( } xfs_trans_cancel(tp); + if (error || notify.want_shutdown) { + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); + if (!error) + error = -EFSCORRUPTED; + } return error; } @@ -175,13 +183,13 @@ xfs_dax_notify_failure( u64 ddev_start; u64 ddev_end; - if (!(mp->m_sb.sb_flags & SB_BORN)) { + if (!(mp->m_super->s_flags & SB_BORN)) { xfs_warn(mp, "filesystem is not ready for notify_failure()!"); return -EIO; } if (mp->m_rtdev_targp && mp->m_rtdev_targp->bt_daxdev == dax_dev) { - xfs_warn(mp, + xfs_debug(mp, "notify_failure() not supported on realtime device!"); return -EOPNOTSUPP; } @@ -194,7 +202,7 @@ xfs_dax_notify_failure( } if (!xfs_has_rmapbt(mp)) { - xfs_warn(mp, "notify_failure() needs rmapbt enabled!"); + xfs_debug(mp, "notify_failure() needs rmapbt enabled!"); return -EOPNOTSUPP; } diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 251f20ddd3683c61e6562b6b4dccfb054b576ab0..93bdd25680bc9a4869f5b8bb2372a37dc5898ace 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -200,7 +200,9 @@ xfs_reflink_trim_around_shared( if (fbno == NULLAGBLOCK) { /* No shared blocks at all. */ return 0; - } else if (fbno == agbno) { + } + + if (fbno == agbno) { /* * The start of this extent is shared. Truncate the * mapping at the end of the shared region so that a @@ -210,16 +212,16 @@ xfs_reflink_trim_around_shared( irec->br_blockcount = flen; *shared = true; return 0; - } else { - /* - * There's a shared extent midway through this extent. - * Truncate the mapping at the start of the shared - * extent so that a subsequent iteration starts at the - * start of the shared region. - */ - irec->br_blockcount = fbno - agbno; - return 0; } + + /* + * There's a shared extent midway through this extent. + * Truncate the mapping at the start of the shared + * extent so that a subsequent iteration starts at the + * start of the shared region. + */ + irec->br_blockcount = fbno - agbno; + return 0; } int diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c index 20e0534a772c92018b2c3a03bf38323dff89a172..90a77cd3ebade814ee72f6b061b5a0a9ac132330 100644 --- a/fs/xfs/xfs_stats.c +++ b/fs/xfs/xfs_stats.c @@ -74,7 +74,7 @@ int xfs_stats_format(struct xfsstats __percpu *stats, char *buf) defer_relog += per_cpu_ptr(stats, i)->s.defer_relog; } - len += scnprintf(buf + len, PATH_MAX-len, "xpc %Lu %Lu %Lu\n", + len += scnprintf(buf + len, PATH_MAX-len, "xpc %llu %llu %llu\n", xs_xstrat_bytes, xs_write_bytes, xs_read_bytes); len += scnprintf(buf + len, PATH_MAX-len, "defer_relog %llu\n", defer_relog); @@ -125,7 +125,7 @@ static int xqmstat_proc_show(struct seq_file *m, void *v) { int j; - seq_printf(m, "qm"); + seq_puts(m, "qm"); for (j = XFSSTAT_START_XQMSTAT; j < XFSSTAT_END_XQMSTAT; j++) seq_printf(m, " %u", counter_val(xfsstats.xs_stats, j)); seq_putc(m, '\n'); diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 9ac59814bbb6c69a2bffd2c2e129728afc467584..f029c6702dda149c025fcc8231a2fc13ac864df9 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -653,7 +653,7 @@ xfs_fs_destroy_inode( static void xfs_fs_dirty_inode( struct inode *inode, - int flag) + int flags) { struct xfs_inode *ip = XFS_I(inode); struct xfs_mount *mp = ip->i_mount; @@ -661,7 +661,13 @@ xfs_fs_dirty_inode( if (!(inode->i_sb->s_flags & SB_LAZYTIME)) return; - if (flag != I_DIRTY_SYNC || !(inode->i_state & I_DIRTY_TIME)) + + /* + * Only do the timestamp update if the inode is dirty (I_DIRTY_SYNC) + * and has dirty timestamp (I_DIRTY_TIME). I_DIRTY_TIME can be passed + * in flags possibly together with I_DIRTY_SYNC. + */ + if ((flags & ~I_DIRTY_TIME) != I_DIRTY_SYNC || !(flags & I_DIRTY_TIME)) return; if (xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp)) diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index f9057af6e0c80d70aa9899c58eb962647e19f7aa..cb7c81ba7fa38bcd4d89dedfc215aa78ba4fd091 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1170,7 +1170,7 @@ DECLARE_EVENT_CLASS(xfs_dqtrx_class, __entry->ino_res_used = qtrx->qt_ino_res_used; __entry->icount_delta = qtrx->qt_icount_delta; ), - TP_printk("dev %d:%d dquot id 0x%x type %s flags %s" + TP_printk("dev %d:%d dquot id 0x%x type %s flags %s " "blk_res %llu bcount_delta %lld delbcnt_delta %lld " "rtblk_res %llu rtblk_res_used %llu rtbcount_delta %lld delrtb_delta %lld " "ino_res %llu ino_res_used %llu icount_delta %lld", @@ -1602,7 +1602,7 @@ TRACE_EVENT(xfs_bunmap, __entry->caller_ip = caller_ip; __entry->flags = flags; ), - TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx" + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx " "flags %s caller %pS", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index d3a97a028560651828a5e73b75f8d3585a5a96eb..16fbf2a1144c17603921cea2af838e2b1c9bdf5b 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -602,9 +602,9 @@ xfsaild( while (1) { if (tout && tout <= 20) - set_current_state(TASK_KILLABLE); + set_current_state(TASK_KILLABLE|TASK_FREEZABLE); else - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); /* * Check kthread_should_stop() after we set the task state to @@ -653,14 +653,14 @@ xfsaild( ailp->ail_target == ailp->ail_target_prev && list_empty(&ailp->ail_buf_list)) { spin_unlock(&ailp->ail_lock); - freezable_schedule(); + schedule(); tout = 0; continue; } spin_unlock(&ailp->ail_lock); if (tout) - freezable_schedule_timeout(msecs_to_jiffies(tout)); + schedule_timeout(msecs_to_jiffies(tout)); __set_current_state(TASK_RUNNING); diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h index e7d27373ff71ff9dc508585eacc118405029fa97..c09d72986968ab224374454efe5cd6ea4d7a372b 100644 --- a/include/acpi/acpi_bus.h +++ b/include/acpi/acpi_bus.h @@ -365,7 +365,6 @@ struct acpi_device { int device_type; acpi_handle handle; /* no handle for fixed hardware */ struct fwnode_handle fwnode; - struct acpi_device *parent; struct list_head wakeup_list; struct list_head del_list; struct acpi_device_status status; @@ -458,6 +457,14 @@ static inline void *acpi_driver_data(struct acpi_device *d) #define to_acpi_device(d) container_of(d, struct acpi_device, dev) #define to_acpi_driver(d) container_of(d, struct acpi_driver, drv) +static inline struct acpi_device *acpi_dev_parent(struct acpi_device *adev) +{ + if (adev->dev.parent) + return to_acpi_device(adev->dev.parent); + + return NULL; +} + static inline void acpi_set_device_status(struct acpi_device *adev, u32 sta) { *((u32 *)&adev->status) = sta; @@ -512,7 +519,6 @@ extern int unregister_acpi_notifier(struct notifier_block *); * External Functions */ -struct acpi_device *acpi_fetch_acpi_dev(acpi_handle handle); acpi_status acpi_bus_get_status_handle(acpi_handle handle, unsigned long long *sta); int acpi_bus_get_status(struct acpi_device *device); @@ -613,8 +619,7 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev); int acpi_iommu_fwspec_init(struct device *dev, u32 id, struct fwnode_handle *fwnode, const struct iommu_ops *ops); -int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset, - u64 *size); +int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map); int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr, const u32 *input_id); static inline int acpi_dma_configure(struct device *dev, @@ -733,10 +738,24 @@ static inline bool acpi_device_can_poweroff(struct acpi_device *adev) } bool acpi_dev_hid_uid_match(struct acpi_device *adev, const char *hid2, const char *uid2); +int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer); void acpi_dev_clear_dependencies(struct acpi_device *supplier); bool acpi_dev_ready_for_enumeration(const struct acpi_device *device); -struct acpi_device *acpi_dev_get_first_consumer_dev(struct acpi_device *supplier); +struct acpi_device *acpi_dev_get_next_consumer_dev(struct acpi_device *supplier, + struct acpi_device *start); + +/** + * for_each_acpi_consumer_dev - iterate over the consumer ACPI devices for a + * given supplier + * @supplier: Pointer to the supplier's ACPI device + * @consumer: Pointer to &struct acpi_device to hold the consumer, initially NULL + */ +#define for_each_acpi_consumer_dev(supplier, consumer) \ + for (consumer = acpi_dev_get_next_consumer_dev(supplier, NULL); \ + consumer; \ + consumer = acpi_dev_get_next_consumer_dev(supplier, consumer)) + struct acpi_device * acpi_dev_get_next_match_dev(struct acpi_device *adev, const char *hid, const char *uid, s64 hrv); struct acpi_device * @@ -767,9 +786,10 @@ static inline void acpi_dev_put(struct acpi_device *adev) put_device(&adev->dev); } -struct acpi_device *acpi_bus_get_acpi_device(acpi_handle handle); +struct acpi_device *acpi_fetch_acpi_dev(acpi_handle handle); +struct acpi_device *acpi_get_acpi_dev(acpi_handle handle); -static inline void acpi_bus_put_acpi_device(struct acpi_device *adev) +static inline void acpi_put_acpi_dev(struct acpi_device *adev) { acpi_dev_put(adev); } diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index f73d357ecdf5ff06a5e3c898df9bfd8182ee2e03..c5614444031ff85f3303f7f09417a442f9f2510c 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -140,6 +140,7 @@ extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs); extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls); extern int cppc_set_enable(int cpu, bool enable); extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps); +extern bool cppc_perf_ctrs_in_pcc(void); extern bool acpi_cpc_valid(void); extern bool cppc_allow_fast_switch(void); extern int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data); @@ -173,6 +174,10 @@ static inline int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps) { return -ENOTSUPP; } +static inline bool cppc_perf_ctrs_in_pcc(void) +{ + return false; +} static inline bool acpi_cpc_valid(void) { return false; diff --git a/include/asm-generic/bitops/generic-non-atomic.h b/include/asm-generic/bitops/generic-non-atomic.h index 3d5ebd24652b9ff2412c45fd829a5fe8b9308e0e..564a8c675d85898a601e889ab471880ef70c8068 100644 --- a/include/asm-generic/bitops/generic-non-atomic.h +++ b/include/asm-generic/bitops/generic-non-atomic.h @@ -4,6 +4,7 @@ #define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H #include +#include #ifndef _LINUX_BITOPS_H #error only can be included directly @@ -127,6 +128,18 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr) return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } +/** + * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); + return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1))); +} + /* * const_*() definitions provide good compile-time optimizations when * the passed arguments can be resolved at compile time. @@ -137,6 +150,7 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr) #define const___test_and_set_bit generic___test_and_set_bit #define const___test_and_clear_bit generic___test_and_clear_bit #define const___test_and_change_bit generic___test_and_change_bit +#define const_test_bit_acquire generic_test_bit_acquire /** * const_test_bit - Determine whether a bit is set diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h index 988a3bbfba34ec9a1b4904520a5cadc469842dbd..2b238b161a6206e6702a34523ee4043d7dd65d96 100644 --- a/include/asm-generic/bitops/instrumented-non-atomic.h +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -142,4 +142,16 @@ _test_bit(unsigned long nr, const volatile unsigned long *addr) return arch_test_bit(nr, addr); } +/** + * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set + * @nr: bit number to test + * @addr: Address to start counting from + */ +static __always_inline bool +_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr) +{ + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); + return arch_test_bit_acquire(nr, addr); +} + #endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bitops/non-atomic.h b/include/asm-generic/bitops/non-atomic.h index 5c37ced343aedc0669347f64d042ffc2c161e5a9..71f8d54a5195e919bb51e3311df3150b087fbacd 100644 --- a/include/asm-generic/bitops/non-atomic.h +++ b/include/asm-generic/bitops/non-atomic.h @@ -13,6 +13,7 @@ #define arch___test_and_change_bit generic___test_and_change_bit #define arch_test_bit generic_test_bit +#define arch_test_bit_acquire generic_test_bit_acquire #include diff --git a/include/asm-generic/bitops/non-instrumented-non-atomic.h b/include/asm-generic/bitops/non-instrumented-non-atomic.h index bdb9b1ffaee90a9444db25575b93a5a0d50e05f9..0ddc78dfc358bece4d4f6a40036e0e11e73684b5 100644 --- a/include/asm-generic/bitops/non-instrumented-non-atomic.h +++ b/include/asm-generic/bitops/non-instrumented-non-atomic.h @@ -12,5 +12,6 @@ #define ___test_and_change_bit arch___test_and_change_bit #define _test_bit arch_test_bit +#define _test_bit_acquire arch_test_bit_acquire #endif /* __ASM_GENERIC_BITOPS_NON_INSTRUMENTED_NON_ATOMIC_H */ diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index ba1f860af38ba2ffd3e87ff778489c8fd116b61b..4050b191e1a97fa33c94d97011949892b06bf3bb 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -220,22 +220,6 @@ extern __printf(1, 2) void __warn_printk(const char *fmt, ...); # define WARN_ON_SMP(x) ({0;}) #endif -/* - * WARN_ON_FUNCTION_MISMATCH() warns if a value doesn't match a - * function address, and can be useful for catching issues with - * callback functions, for example. - * - * With CONFIG_CFI_CLANG, the warning is disabled because the - * compiler replaces function addresses taken in C code with - * local jump table addresses, which breaks cross-module function - * address equality. - */ -#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_MODULES) -# define WARN_ON_FUNCTION_MISMATCH(x, fn) ({ 0; }) -#else -# define WARN_ON_FUNCTION_MISMATCH(x, fn) WARN_ON_ONCE((x) != (fn)) -#endif - #endif /* __ASSEMBLY__ */ #endif diff --git a/include/asm-generic/cacheflush.h b/include/asm-generic/cacheflush.h index 4f07afacbc23953861443be0939fb8d70116396e..f46258d1a080f490d3a88b9c2cf7b6ff1c777e55 100644 --- a/include/asm-generic/cacheflush.h +++ b/include/asm-generic/cacheflush.h @@ -2,6 +2,8 @@ #ifndef _ASM_GENERIC_CACHEFLUSH_H #define _ASM_GENERIC_CACHEFLUSH_H +#include + struct mm_struct; struct vm_area_struct; struct page; @@ -105,14 +107,22 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end) #ifndef copy_to_user_page #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ + instrument_copy_to_user((void __user *)dst, src, len); \ memcpy(dst, src, len); \ flush_icache_user_page(vma, page, vaddr, len); \ } while (0) #endif + #ifndef copy_from_user_page -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - memcpy(dst, src, len) +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + instrument_copy_from_user_before(dst, (void __user *)src, \ + len); \ + memcpy(dst, src, len); \ + instrument_copy_from_user_after(dst, (void __user *)src, len, \ + 0); \ + } while (0) #endif #endif /* _ASM_GENERIC_CACHEFLUSH_H */ diff --git a/include/asm-generic/compat.h b/include/asm-generic/compat.h index d06308a2a7a8d9292dd5d474ea15669455951c37..aeb257ad3d1a6d43d79075d02d4973ef7da2d20b 100644 --- a/include/asm-generic/compat.h +++ b/include/asm-generic/compat.h @@ -14,12 +14,17 @@ #define COMPAT_OFF_T_MAX 0x7fffffff #endif -#if !defined(compat_arg_u64) && !defined(CONFIG_CPU_BIG_ENDIAN) +#ifndef compat_arg_u64 +#ifdef CONFIG_CPU_BIG_ENDIAN #define compat_arg_u64(name) u32 name##_lo, u32 name##_hi #define compat_arg_u64_dual(name) u32, name##_lo, u32, name##_hi +#else +#define compat_arg_u64(name) u32 name##_hi, u32 name##_lo +#define compat_arg_u64_dual(name) u32, name##_hi, u32, name##_lo +#endif #define compat_arg_u64_glue(name) (((u64)name##_lo & 0xffffffffUL) | \ ((u64)name##_hi << 32)) -#endif +#endif /* compat_arg_u64 */ /* These types are common across all compat ABIs */ typedef u32 compat_size_t; diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index c05d2ce9b6cd85bb9c2c439cf35160b4cdbcc838..bfb9eb9d7215b89b8660f2dd0fbf8923262c2da3 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -105,15 +105,12 @@ static inline u64 hv_do_rep_hypercall(u16 code, u16 rep_count, u16 varhead_size, } /* Generate the guest OS identifier as described in the Hyper-V TLFS */ -static inline __u64 generate_guest_id(__u64 d_info1, __u64 kernel_version, - __u64 d_info2) +static inline u64 hv_generate_guest_id(u64 kernel_version) { - __u64 guest_id = 0; + u64 guest_id; - guest_id = (((__u64)HV_LINUX_VENDOR_ID) << 48); - guest_id |= (d_info1 << 48); + guest_id = (((u64)HV_LINUX_VENDOR_ID) << 48); guest_id |= (kernel_version << 16); - guest_id |= d_info2; return guest_id; } diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h index d0f7bdd2fdf235765d53c91bbd78674a19e1168c..db13bb620f527e2a0d8204cfdafe877975c35394 100644 --- a/include/asm-generic/sections.h +++ b/include/asm-generic/sections.h @@ -97,7 +97,7 @@ static inline bool memory_contains(void *begin, void *end, void *virt, /** * memory_intersects - checks if the region occupied by an object intersects * with another memory region - * @begin: virtual address of the beginning of the memory regien + * @begin: virtual address of the beginning of the memory region * @end: virtual address of the end of the memory region * @virt: virtual address of the memory object * @size: size of the memory object @@ -110,7 +110,10 @@ static inline bool memory_intersects(void *begin, void *end, void *virt, { void *vend = virt + size; - return (virt >= begin && virt < end) || (vend >= begin && vend < end); + if (virt < end && vend > begin) + return true; + + return false; } /** diff --git a/include/asm-generic/signal.h b/include/asm-generic/signal.h index c53984fa97614db5aea70411110e9e5573ea4d03..663dd6d0795dcac3a4ffae366c1de24b8f218d48 100644 --- a/include/asm-generic/signal.h +++ b/include/asm-generic/signal.h @@ -5,8 +5,6 @@ #include #ifndef __ASSEMBLY__ -#ifdef SA_RESTORER -#endif #include #undef __HAVE_ARCH_SIG_BITOPS diff --git a/include/asm-generic/softirq_stack.h b/include/asm-generic/softirq_stack.h index d3e2d81656e042f3a74579403470311dc7f3c732..2a67aed9ac5287bc5c6df3bd374b9f2493fd803a 100644 --- a/include/asm-generic/softirq_stack.h +++ b/include/asm-generic/softirq_stack.h @@ -2,7 +2,7 @@ #ifndef __ASM_GENERIC_SOFTIRQ_STACK_H #define __ASM_GENERIC_SOFTIRQ_STACK_H -#if defined(CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK) && !defined(CONFIG_PREEMPT_RT) +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK void do_softirq_own_stack(void); #else static inline void do_softirq_own_stack(void) diff --git a/include/asm-generic/termios-base.h b/include/asm-generic/termios-base.h deleted file mode 100644 index 59c5a3bd4a6e190058e42cb50efc3914f7c5ef42..0000000000000000000000000000000000000000 --- a/include/asm-generic/termios-base.h +++ /dev/null @@ -1,78 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* termios.h: generic termios/termio user copying/translation - */ - -#ifndef _ASM_GENERIC_TERMIOS_BASE_H -#define _ASM_GENERIC_TERMIOS_BASE_H - -#include - -#ifndef __ARCH_TERMIO_GETPUT - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -static inline int user_termio_to_kernel_termios(struct ktermios *termios, - struct termio __user *termio) -{ - unsigned short tmp; - - if (get_user(tmp, &termio->c_iflag) < 0) - goto fault; - termios->c_iflag = (0xffff0000 & termios->c_iflag) | tmp; - - if (get_user(tmp, &termio->c_oflag) < 0) - goto fault; - termios->c_oflag = (0xffff0000 & termios->c_oflag) | tmp; - - if (get_user(tmp, &termio->c_cflag) < 0) - goto fault; - termios->c_cflag = (0xffff0000 & termios->c_cflag) | tmp; - - if (get_user(tmp, &termio->c_lflag) < 0) - goto fault; - termios->c_lflag = (0xffff0000 & termios->c_lflag) | tmp; - - if (get_user(termios->c_line, &termio->c_line) < 0) - goto fault; - - if (copy_from_user(termios->c_cc, termio->c_cc, NCC) != 0) - goto fault; - - return 0; - - fault: - return -EFAULT; -} - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -static inline int kernel_termios_to_user_termio(struct termio __user *termio, - struct ktermios *termios) -{ - if (put_user(termios->c_iflag, &termio->c_iflag) < 0 || - put_user(termios->c_oflag, &termio->c_oflag) < 0 || - put_user(termios->c_cflag, &termio->c_cflag) < 0 || - put_user(termios->c_lflag, &termio->c_lflag) < 0 || - put_user(termios->c_line, &termio->c_line) < 0 || - copy_to_user(termio->c_cc, termios->c_cc, NCC) != 0) - return -EFAULT; - - return 0; -} - -#ifndef user_termios_to_kernel_termios -#define user_termios_to_kernel_termios(k, u) copy_from_user(k, u, sizeof(struct termios)) -#endif - -#ifndef kernel_termios_to_user_termios -#define kernel_termios_to_user_termios(u, k) copy_to_user(u, k, sizeof(struct termios)) -#endif - -#define user_termios_to_kernel_termios_1(k, u) copy_from_user(k, u, sizeof(struct termios)) -#define kernel_termios_to_user_termios_1(u, k) copy_to_user(u, k, sizeof(struct termios)) - -#endif /* __ARCH_TERMIO_GETPUT */ - -#endif /* _ASM_GENERIC_TERMIOS_BASE_H */ diff --git a/include/asm-generic/termios.h b/include/asm-generic/termios.h deleted file mode 100644 index b1398d0d4a1d52e54832fd91645843ad21a34a95..0000000000000000000000000000000000000000 --- a/include/asm-generic/termios.h +++ /dev/null @@ -1,108 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_GENERIC_TERMIOS_H -#define _ASM_GENERIC_TERMIOS_H - - -#include -#include - -/* intr=^C quit=^\ erase=del kill=^U - eof=^D vtime=\0 vmin=\1 sxtc=\0 - start=^Q stop=^S susp=^Z eol=\0 - reprint=^R discard=^U werase=^W lnext=^V - eol2=\0 -*/ -#define INIT_C_CC "\003\034\177\025\004\0\1\0\021\023\032\0\022\017\027\026\0" - -/* - * Translate a "termio" structure into a "termios". Ugh. - */ -static inline int user_termio_to_kernel_termios(struct ktermios *termios, - const struct termio __user *termio) -{ - unsigned short tmp; - - if (get_user(tmp, &termio->c_iflag) < 0) - goto fault; - termios->c_iflag = (0xffff0000 & termios->c_iflag) | tmp; - - if (get_user(tmp, &termio->c_oflag) < 0) - goto fault; - termios->c_oflag = (0xffff0000 & termios->c_oflag) | tmp; - - if (get_user(tmp, &termio->c_cflag) < 0) - goto fault; - termios->c_cflag = (0xffff0000 & termios->c_cflag) | tmp; - - if (get_user(tmp, &termio->c_lflag) < 0) - goto fault; - termios->c_lflag = (0xffff0000 & termios->c_lflag) | tmp; - - if (get_user(termios->c_line, &termio->c_line) < 0) - goto fault; - - if (copy_from_user(termios->c_cc, termio->c_cc, NCC) != 0) - goto fault; - - return 0; - - fault: - return -EFAULT; -} - -/* - * Translate a "termios" structure into a "termio". Ugh. - */ -static inline int kernel_termios_to_user_termio(struct termio __user *termio, - struct ktermios *termios) -{ - if (put_user(termios->c_iflag, &termio->c_iflag) < 0 || - put_user(termios->c_oflag, &termio->c_oflag) < 0 || - put_user(termios->c_cflag, &termio->c_cflag) < 0 || - put_user(termios->c_lflag, &termio->c_lflag) < 0 || - put_user(termios->c_line, &termio->c_line) < 0 || - copy_to_user(termio->c_cc, termios->c_cc, NCC) != 0) - return -EFAULT; - - return 0; -} - -#ifdef TCGETS2 -static inline int user_termios_to_kernel_termios(struct ktermios *k, - struct termios2 __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios2)); -} - -static inline int kernel_termios_to_user_termios(struct termios2 __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios2)); -} - -static inline int user_termios_to_kernel_termios_1(struct ktermios *k, - struct termios __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios)); -} - -static inline int kernel_termios_to_user_termios_1(struct termios __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios)); -} -#else /* TCGETS2 */ -static inline int user_termios_to_kernel_termios(struct ktermios *k, - struct termios __user *u) -{ - return copy_from_user(k, u, sizeof(struct termios)); -} - -static inline int kernel_termios_to_user_termios(struct termios __user *u, - struct ktermios *k) -{ - return copy_to_user(u, k, sizeof(struct termios)); -} -#endif /* TCGETS2 */ - -#endif /* _ASM_GENERIC_TERMIOS_H */ diff --git a/include/asm-generic/unaligned.h b/include/asm-generic/unaligned.h index df30f11b4a46011fdbcefc3d0a9287ebc62e8daa..699650f819706dbada12bf27255ea823c2880efe 100644 --- a/include/asm-generic/unaligned.h +++ b/include/asm-generic/unaligned.h @@ -126,7 +126,7 @@ static inline void put_unaligned_le24(const u32 val, void *p) __put_unaligned_le24(val, p); } -static inline void __put_unaligned_be48(const u64 val, __u8 *p) +static inline void __put_unaligned_be48(const u64 val, u8 *p) { *p++ = val >> 40; *p++ = val >> 32; diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 7515a465ec03a64f1a28db6b917e1d259d2aaade..c15de165ec8ff0df147f0f91f67947e3ebc54aaf 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -154,6 +154,14 @@ #define MEM_DISCARD(sec) *(.mem##sec) #endif +#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_NO_PATCHABLE +#define KEEP_PATCHABLE KEEP(*(__patchable_function_entries)) +#define PATCHABLE_DISCARDS +#else +#define KEEP_PATCHABLE +#define PATCHABLE_DISCARDS *(__patchable_function_entries) +#endif + #ifdef CONFIG_FTRACE_MCOUNT_RECORD /* * The ftrace call sites are logged to a section whose name depends on the @@ -172,7 +180,7 @@ #define MCOUNT_REC() . = ALIGN(8); \ __start_mcount_loc = .; \ KEEP(*(__mcount_loc)) \ - KEEP(*(__patchable_function_entries)) \ + KEEP_PATCHABLE \ __stop_mcount_loc = .; \ ftrace_stub_graph = ftrace_stub; \ ftrace_ops_list_func = arch_ftrace_ops_list_func; @@ -345,6 +353,9 @@ *(__tracepoints) \ /* implement dynamic printk debug */ \ . = ALIGN(8); \ + __start___dyndbg_classes = .; \ + KEEP(*(__dyndbg_classes)) \ + __stop___dyndbg_classes = .; \ __start___dyndbg = .; \ KEEP(*(__dyndbg)) \ __stop___dyndbg = .; \ @@ -421,6 +432,22 @@ __end_ro_after_init = .; #endif +/* + * .kcfi_traps contains a list KCFI trap locations. + */ +#ifndef KCFI_TRAPS +#ifdef CONFIG_ARCH_USES_CFI_TRAPS +#define KCFI_TRAPS \ + __kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) { \ + __start___kcfi_traps = .; \ + KEEP(*(.kcfi_traps)) \ + __stop___kcfi_traps = .; \ + } +#else +#define KCFI_TRAPS +#endif +#endif + /* * Read only Data */ @@ -529,6 +556,8 @@ __stop___modver = .; \ } \ \ + KCFI_TRAPS \ + \ RO_EXCEPTION_TABLE \ NOTES \ BTF \ @@ -537,21 +566,6 @@ __end_rodata = .; -/* - * .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI) - * jump table entries. - */ -#ifdef CONFIG_CFI_CLANG -#define TEXT_CFI_JT \ - . = ALIGN(PMD_SIZE); \ - __cfi_jt_start = .; \ - *(.text..L.cfi.jumptable .text..L.cfi.jumptable.*) \ - . = ALIGN(PMD_SIZE); \ - __cfi_jt_end = .; -#else -#define TEXT_CFI_JT -#endif - /* * Non-instrumentable text section */ @@ -579,7 +593,6 @@ *(.text..refcount) \ *(.ref.text) \ *(.text.asan.* .text.tsan.*) \ - TEXT_CFI_JT \ MEM_KEEP(init.text*) \ MEM_KEEP(exit.text*) \ @@ -1008,8 +1021,7 @@ * keep any .init_array.* sections. * https://bugs.llvm.org/show_bug.cgi?id=46478 */ -#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \ - defined(CONFIG_CFI_CLANG) +#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) # ifdef CONFIG_CONSTRUCTORS # define SANITIZER_DISCARDS \ *(.eh_frame) @@ -1024,6 +1036,7 @@ #define COMMON_DISCARDS \ SANITIZER_DISCARDS \ + PATCHABLE_DISCARDS \ *(.discard) \ *(.discard.*) \ *(.modinfo) \ diff --git a/include/clocksource/timer-ti-dm.h b/include/clocksource/timer-ti-dm.h index b0f80cfd2a263ba04200dd7cc733674d8fb87138..77eceeae708cf2cab40f83105aace0d22022b926 100644 --- a/include/clocksource/timer-ti-dm.h +++ b/include/clocksource/timer-ti-dm.h @@ -52,10 +52,6 @@ #define OMAP_TIMER_TRIGGER_OVERFLOW 0x01 #define OMAP_TIMER_TRIGGER_OVERFLOW_AND_COMPARE 0x02 -/* posted mode types */ -#define OMAP_TIMER_NONPOSTED 0x00 -#define OMAP_TIMER_POSTED 0x01 - /* timer capabilities used in hwmod database */ #define OMAP_TIMER_SECURE 0x80000000 #define OMAP_TIMER_ALWON 0x40000000 @@ -63,73 +59,13 @@ #define OMAP_TIMER_NEEDS_RESET 0x10000000 #define OMAP_TIMER_HAS_DSP_IRQ 0x08000000 -/* - * timer errata flags - * - * Errata i103/i767 impacts all OMAP3/4/5 devices including AM33xx. This - * errata prevents us from using posted mode on these devices, unless the - * timer counter register is never read. For more details please refer to - * the OMAP3/4/5 errata documents. - */ -#define OMAP_TIMER_ERRATA_I103_I767 0x80000000 - -struct timer_regs { - u32 ocp_cfg; - u32 tidr; - u32 tier; - u32 twer; - u32 tclr; - u32 tcrr; - u32 tldr; - u32 ttrg; - u32 twps; - u32 tmar; - u32 tcar1; - u32 tsicr; - u32 tcar2; - u32 tpir; - u32 tnir; - u32 tcvr; - u32 tocr; - u32 towr; -}; - struct omap_dm_timer { - int id; - int irq; - struct clk *fclk; - - void __iomem *io_base; - void __iomem *irq_stat; /* TISR/IRQSTATUS interrupt status */ - void __iomem *irq_ena; /* irq enable */ - void __iomem *irq_dis; /* irq disable, only on v2 ip */ - void __iomem *pend; /* write pending */ - void __iomem *func_base; /* function register base */ - - atomic_t enabled; - unsigned long rate; - unsigned reserved:1; - unsigned posted:1; - struct timer_regs context; - int revision; - u32 capability; - u32 errata; - struct platform_device *pdev; - struct list_head node; - struct notifier_block nb; }; -int omap_dm_timer_reserve_systimer(int id); -struct omap_dm_timer *omap_dm_timer_request_by_cap(u32 cap); - int omap_dm_timer_get_irq(struct omap_dm_timer *timer); u32 omap_dm_timer_modify_idlect_mask(u32 inputmask); -int omap_dm_timer_trigger(struct omap_dm_timer *timer); - -int omap_dm_timers_active(void); - /* * Do not use the defines below, they are not needed. They should be only * used by dmtimer.c and sys_timer related code. @@ -199,52 +135,4 @@ int omap_dm_timers_active(void); #define _OMAP_TIMER_TICK_INT_MASK_SET_OFFSET 0x54 /* TOCR, 34xx only */ #define _OMAP_TIMER_TICK_INT_MASK_COUNT_OFFSET 0x58 /* TOWR, 34xx only */ -/* register offsets with the write pending bit encoded */ -#define WPSHIFT 16 - -#define OMAP_TIMER_WAKEUP_EN_REG (_OMAP_TIMER_WAKEUP_EN_OFFSET \ - | (WP_NONE << WPSHIFT)) - -#define OMAP_TIMER_CTRL_REG (_OMAP_TIMER_CTRL_OFFSET \ - | (WP_TCLR << WPSHIFT)) - -#define OMAP_TIMER_COUNTER_REG (_OMAP_TIMER_COUNTER_OFFSET \ - | (WP_TCRR << WPSHIFT)) - -#define OMAP_TIMER_LOAD_REG (_OMAP_TIMER_LOAD_OFFSET \ - | (WP_TLDR << WPSHIFT)) - -#define OMAP_TIMER_TRIGGER_REG (_OMAP_TIMER_TRIGGER_OFFSET \ - | (WP_TTGR << WPSHIFT)) - -#define OMAP_TIMER_WRITE_PEND_REG (_OMAP_TIMER_WRITE_PEND_OFFSET \ - | (WP_NONE << WPSHIFT)) - -#define OMAP_TIMER_MATCH_REG (_OMAP_TIMER_MATCH_OFFSET \ - | (WP_TMAR << WPSHIFT)) - -#define OMAP_TIMER_CAPTURE_REG (_OMAP_TIMER_CAPTURE_OFFSET \ - | (WP_NONE << WPSHIFT)) - -#define OMAP_TIMER_IF_CTRL_REG (_OMAP_TIMER_IF_CTRL_OFFSET \ - | (WP_NONE << WPSHIFT)) - -#define OMAP_TIMER_CAPTURE2_REG (_OMAP_TIMER_CAPTURE2_OFFSET \ - | (WP_NONE << WPSHIFT)) - -#define OMAP_TIMER_TICK_POS_REG (_OMAP_TIMER_TICK_POS_OFFSET \ - | (WP_TPIR << WPSHIFT)) - -#define OMAP_TIMER_TICK_NEG_REG (_OMAP_TIMER_TICK_NEG_OFFSET \ - | (WP_TNIR << WPSHIFT)) - -#define OMAP_TIMER_TICK_COUNT_REG (_OMAP_TIMER_TICK_COUNT_OFFSET \ - | (WP_TCVR << WPSHIFT)) - -#define OMAP_TIMER_TICK_INT_MASK_SET_REG \ - (_OMAP_TIMER_TICK_INT_MASK_SET_OFFSET | (WP_TOCR << WPSHIFT)) - -#define OMAP_TIMER_TICK_INT_MASK_COUNT_REG \ - (_OMAP_TIMER_TICK_INT_MASK_COUNT_OFFSET | (WP_TOWR << WPSHIFT)) - #endif /* __CLOCKSOURCE_DMTIMER_H */ diff --git a/include/crypto/aria.h b/include/crypto/aria.h index 4a86661788e8656a7dc4bf086aa05057dc56dd68..254da46cc385c230cfbfb4efe78a95336ce0c542 100644 --- a/include/crypto/aria.h +++ b/include/crypto/aria.h @@ -32,18 +32,10 @@ #define ARIA_RD_KEY_WORDS (ARIA_BLOCK_SIZE / sizeof(u32)) struct aria_ctx { - int key_length; - int rounds; u32 enc_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; u32 dec_key[ARIA_MAX_RD_KEYS][ARIA_RD_KEY_WORDS]; -}; - -static const u32 key_rc[5][4] = { - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 }, - { 0xdb92371d, 0x2126e970, 0x03249775, 0x04e8c90e }, - { 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0 }, - { 0x6db14acc, 0x9e21c820, 0xff28b1d5, 0xef5de2b0 } + int rounds; + int key_length; }; static const u32 s1[256] = { @@ -458,4 +450,9 @@ static inline void aria_gsrk(u32 *rk, u32 *x, u32 *y, u32 n) ((y[(q + 2) % 4]) << (32 - r)); } +void aria_encrypt(void *ctx, u8 *out, const u8 *in); +void aria_decrypt(void *ctx, u8 *out, const u8 *in); +int aria_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len); + #endif diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h index 27b7b0224ea6f95330e7276d4f8478d9f016bcc2..d482017f3e203c96068f184ba2e7bad5fd9683d2 100644 --- a/include/crypto/internal/aead.h +++ b/include/crypto/internal/aead.h @@ -114,31 +114,6 @@ static inline void aead_init_queue(struct aead_queue *queue, crypto_init_queue(&queue->base, max_qlen); } -static inline int aead_enqueue_request(struct aead_queue *queue, - struct aead_request *request) -{ - return crypto_enqueue_request(&queue->base, &request->base); -} - -static inline struct aead_request *aead_dequeue_request( - struct aead_queue *queue) -{ - struct crypto_async_request *req; - - req = crypto_dequeue_request(&queue->base); - - return req ? container_of(req, struct aead_request, base) : NULL; -} - -static inline struct aead_request *aead_get_backlog(struct aead_queue *queue) -{ - struct crypto_async_request *req; - - req = crypto_get_backlog(&queue->base); - - return req ? container_of(req, struct aead_request, base) : NULL; -} - static inline unsigned int crypto_aead_alg_chunksize(struct aead_alg *alg) { return alg->chunksize; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 6407b4b61350d9615d1a59723daaf2de7444020e..ccdb05f68a75c3a2e6465ccf21390ac94e886c7f 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -46,12 +46,6 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, walk->offset += nbytes; } -static inline unsigned int scatterwalk_aligned(struct scatter_walk *walk, - unsigned int alignmask) -{ - return !(walk->offset & alignmask); -} - static inline struct page *scatterwalk_page(struct scatter_walk *walk) { return sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index 1d5e3cccb9e3578800875949b2bf2bcbbbe972dd..b1b2df48d42c044bddc713de3e55405d17a85d1e 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -319,8 +319,8 @@ enum drm_panel_orientation { * EDID's detailed monitor range */ struct drm_monitor_range_info { - u8 min_vfreq; - u8 max_vfreq; + u16 min_vfreq; + u16 max_vfreq; }; /** diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h index 28dd80343afa9530a85d1e69feb1ba4019875a5a..429735b91f63239c33c2ce66689002d136e79d2e 100644 --- a/include/drm/drm_edid.h +++ b/include/drm/drm_edid.h @@ -92,6 +92,11 @@ struct detailed_data_string { u8 str[13]; } __attribute__((packed)); +#define DRM_EDID_RANGE_OFFSET_MIN_VFREQ (1 << 0) /* 1.4 */ +#define DRM_EDID_RANGE_OFFSET_MAX_VFREQ (1 << 1) /* 1.4 */ +#define DRM_EDID_RANGE_OFFSET_MIN_HFREQ (1 << 2) /* 1.4 */ +#define DRM_EDID_RANGE_OFFSET_MAX_HFREQ (1 << 3) /* 1.4 */ + #define DRM_EDID_DEFAULT_GTF_SUPPORT_FLAG 0x00 /* 1.3 */ #define DRM_EDID_RANGE_LIMITS_ONLY_FLAG 0x01 /* 1.4 */ #define DRM_EDID_SECONDARY_GTF_SUPPORT_FLAG 0x02 /* 1.3 */ diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h index ed82039bfd5ba5ba412a0ec30eae53638d608d65..a17c2f903f81e6467cadf11b8dcdf229bf16b9a1 100644 --- a/include/drm/drm_gem.h +++ b/include/drm/drm_gem.h @@ -174,6 +174,41 @@ struct drm_gem_object_funcs { const struct vm_operations_struct *vm_ops; }; +/** + * struct drm_gem_lru - A simple LRU helper + * + * A helper for tracking GEM objects in a given state, to aid in + * driver's shrinker implementation. Tracks the count of pages + * for lockless &shrinker.count_objects, and provides + * &drm_gem_lru_scan for driver's &shrinker.scan_objects + * implementation. + */ +struct drm_gem_lru { + /** + * @lock: + * + * Lock protecting movement of GEM objects between LRUs. All + * LRUs that the object can move between should be protected + * by the same lock. + */ + struct mutex *lock; + + /** + * @count: + * + * The total number of backing pages of the GEM objects in + * this LRU. + */ + long count; + + /** + * @list: + * + * The LRU list. + */ + struct list_head list; +}; + /** * struct drm_gem_object - GEM buffer object * @@ -312,6 +347,20 @@ struct drm_gem_object { * */ const struct drm_gem_object_funcs *funcs; + + /** + * @lru_node: + * + * List node in a &drm_gem_lru. + */ + struct list_head lru_node; + + /** + * @lru: + * + * The current LRU list that the GEM object is on. + */ + struct drm_gem_lru *lru; }; /** @@ -423,4 +472,10 @@ void drm_gem_unlock_reservations(struct drm_gem_object **objs, int count, int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, u32 handle, u64 *offset); +void drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock); +void drm_gem_lru_remove(struct drm_gem_object *obj); +void drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj); +unsigned long drm_gem_lru_scan(struct drm_gem_lru *lru, unsigned nr_to_scan, + bool (*shrink)(struct drm_gem_object *obj)); + #endif /* __DRM_GEM_H__ */ diff --git a/include/drm/drm_mipi_dsi.h b/include/drm/drm_mipi_dsi.h index 53e3a8a2f24170fec9ffc8d6391d825721db5188..20b21b577deaaddf373f8749cd90037be6b80b41 100644 --- a/include/drm/drm_mipi_dsi.h +++ b/include/drm/drm_mipi_dsi.h @@ -179,6 +179,7 @@ struct mipi_dsi_device_info { * @lp_rate: maximum lane frequency for low power mode in hertz, this should * be set to the real limits of the hardware, zero is only accepted for * legacy drivers + * @dsc: panel/bridge DSC pps payload to be sent */ struct mipi_dsi_device { struct mipi_dsi_host *host; @@ -191,6 +192,7 @@ struct mipi_dsi_device { unsigned long mode_flags; unsigned long hs_rate; unsigned long lp_rate; + struct drm_dsc_config *dsc; }; #define MIPI_DSI_MODULE_PREFIX "mipi-dsi:" diff --git a/include/drm/drm_panel.h b/include/drm/drm_panel.h index 3a271128c0784d59d47a643856ac24894c0a0881..994bfcdd84c559f88d57496b1117490ec9f3b31e 100644 --- a/include/drm/drm_panel.h +++ b/include/drm/drm_panel.h @@ -188,13 +188,6 @@ struct drm_panel { * Panel entry in registry. */ struct list_head list; - - /** - * @dsc: - * - * Panel DSC pps payload to be sent - */ - struct drm_dsc_config *dsc; }; void drm_panel_init(struct drm_panel *panel, struct device *dev, diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h index 22fabdeed2973d303835ee76d3f574929afc4414..a44fb7ef257f64eb08404d1438dabe77649101c1 100644 --- a/include/drm/drm_print.h +++ b/include/drm/drm_print.h @@ -31,11 +31,12 @@ #include #include #include +#include #include /* Do *not* use outside of drm_print.[ch]! */ -extern unsigned int __drm_debug; +extern unsigned long __drm_debug; /** * DOC: print @@ -275,55 +276,75 @@ static inline struct drm_printer drm_err_printer(const char *prefix) * */ enum drm_debug_category { + /* These names must match those in DYNAMIC_DEBUG_CLASSBITS */ /** * @DRM_UT_CORE: Used in the generic drm code: drm_ioctl.c, drm_mm.c, * drm_memory.c, ... */ - DRM_UT_CORE = 0x01, + DRM_UT_CORE, /** * @DRM_UT_DRIVER: Used in the vendor specific part of the driver: i915, * radeon, ... macro. */ - DRM_UT_DRIVER = 0x02, + DRM_UT_DRIVER, /** * @DRM_UT_KMS: Used in the modesetting code. */ - DRM_UT_KMS = 0x04, + DRM_UT_KMS, /** * @DRM_UT_PRIME: Used in the prime code. */ - DRM_UT_PRIME = 0x08, + DRM_UT_PRIME, /** * @DRM_UT_ATOMIC: Used in the atomic code. */ - DRM_UT_ATOMIC = 0x10, + DRM_UT_ATOMIC, /** * @DRM_UT_VBL: Used for verbose debug message in the vblank code. */ - DRM_UT_VBL = 0x20, + DRM_UT_VBL, /** * @DRM_UT_STATE: Used for verbose atomic state debugging. */ - DRM_UT_STATE = 0x40, + DRM_UT_STATE, /** * @DRM_UT_LEASE: Used in the lease code. */ - DRM_UT_LEASE = 0x80, + DRM_UT_LEASE, /** * @DRM_UT_DP: Used in the DP code. */ - DRM_UT_DP = 0x100, + DRM_UT_DP, /** * @DRM_UT_DRMRES: Used in the drm managed resources code. */ - DRM_UT_DRMRES = 0x200, + DRM_UT_DRMRES }; -static inline bool drm_debug_enabled(enum drm_debug_category category) +static inline bool drm_debug_enabled_raw(enum drm_debug_category category) { - return unlikely(__drm_debug & category); + return unlikely(__drm_debug & BIT(category)); } +#define drm_debug_enabled_instrumented(category) \ + ({ \ + pr_debug("todo: is this frequent enough to optimize ?\n"); \ + drm_debug_enabled_raw(category); \ + }) + +#if defined(CONFIG_DRM_USE_DYNAMIC_DEBUG) +/* + * the drm.debug API uses dyndbg, so each drm_*dbg macro/callsite gets + * a descriptor, and only enabled callsites are reachable. They use + * the private macro to avoid re-testing the enable-bit. + */ +#define __drm_debug_enabled(category) true +#define drm_debug_enabled(category) drm_debug_enabled_instrumented(category) +#else +#define __drm_debug_enabled(category) drm_debug_enabled_raw(category) +#define drm_debug_enabled(category) drm_debug_enabled_raw(category) +#endif + /* * struct device based logging * @@ -333,9 +354,10 @@ static inline bool drm_debug_enabled(enum drm_debug_category category) __printf(3, 4) void drm_dev_printk(const struct device *dev, const char *level, const char *format, ...); -__printf(3, 4) -void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, - const char *format, ...); +struct _ddebug; +__printf(4, 5) +void __drm_dev_dbg(struct _ddebug *desc, const struct device *dev, + enum drm_debug_category category, const char *format, ...); /** * DRM_DEV_ERROR() - Error output. @@ -383,6 +405,15 @@ void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, } \ }) +#if !defined(CONFIG_DRM_USE_DYNAMIC_DEBUG) +#define drm_dev_dbg(dev, cat, fmt, ...) \ + __drm_dev_dbg(NULL, dev, cat, fmt, ##__VA_ARGS__) +#else +#define drm_dev_dbg(dev, cat, fmt, ...) \ + _dynamic_func_call_cls(cat, fmt, __drm_dev_dbg, \ + dev, cat, fmt, ##__VA_ARGS__) +#endif + /** * DRM_DEV_DEBUG() - Debug output for generic drm code * @@ -457,7 +488,7 @@ void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, #define drm_dbg_core(drm, fmt, ...) \ drm_dev_dbg((drm) ? (drm)->dev : NULL, DRM_UT_CORE, fmt, ##__VA_ARGS__) -#define drm_dbg(drm, fmt, ...) \ +#define drm_dbg_driver(drm, fmt, ...) \ drm_dev_dbg((drm) ? (drm)->dev : NULL, DRM_UT_DRIVER, fmt, ##__VA_ARGS__) #define drm_dbg_kms(drm, fmt, ...) \ drm_dev_dbg((drm) ? (drm)->dev : NULL, DRM_UT_KMS, fmt, ##__VA_ARGS__) @@ -476,6 +507,7 @@ void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, #define drm_dbg_drmres(drm, fmt, ...) \ drm_dev_dbg((drm) ? (drm)->dev : NULL, DRM_UT_DRMRES, fmt, ##__VA_ARGS__) +#define drm_dbg(drm, fmt, ...) drm_dbg_driver(drm, fmt, ##__VA_ARGS__) /* * printk based logging @@ -483,11 +515,19 @@ void drm_dev_dbg(const struct device *dev, enum drm_debug_category category, * Prefer drm_device based logging over device or prink based logging. */ -__printf(2, 3) -void __drm_dbg(enum drm_debug_category category, const char *format, ...); +__printf(3, 4) +void ___drm_dbg(struct _ddebug *desc, enum drm_debug_category category, const char *format, ...); __printf(1, 2) void __drm_err(const char *format, ...); +#if !defined(CONFIG_DRM_USE_DYNAMIC_DEBUG) +#define __drm_dbg(fmt, ...) ___drm_dbg(NULL, fmt, ##__VA_ARGS__) +#else +#define __drm_dbg(cat, fmt, ...) \ + _dynamic_func_call_cls(cat, fmt, ___drm_dbg, \ + cat, fmt, ##__VA_ARGS__) +#endif + /* Macros to make printk easier */ #define _DRM_PRINTK(once, level, fmt, ...) \ diff --git a/include/drm/i915_pciids.h b/include/drm/i915_pciids.h index 278031aa2e848377fc32e920b16f06f87ec4f8ac..4a4c190f7698465470ead14c7affc624e01b6dab 100644 --- a/include/drm/i915_pciids.h +++ b/include/drm/i915_pciids.h @@ -641,6 +641,7 @@ INTEL_VGA_DEVICE(0x4682, info), \ INTEL_VGA_DEVICE(0x4688, info), \ INTEL_VGA_DEVICE(0x468A, info), \ + INTEL_VGA_DEVICE(0x468B, info), \ INTEL_VGA_DEVICE(0x4690, info), \ INTEL_VGA_DEVICE(0x4692, info), \ INTEL_VGA_DEVICE(0x4693, info) diff --git a/include/dt-bindings/ata/ahci.h b/include/dt-bindings/ata/ahci.h new file mode 100644 index 0000000000000000000000000000000000000000..77997b35612ca89b8c63fc6a7d83c3a0e09c44ec --- /dev/null +++ b/include/dt-bindings/ata/ahci.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only or BSD-2-Clause */ +/* + * This header provides constants for most AHCI bindings. + */ + +#ifndef _DT_BINDINGS_ATA_AHCI_H +#define _DT_BINDINGS_ATA_AHCI_H + +/* Host Bus Adapter generic platform capabilities */ +#define HBA_SSS (1 << 27) +#define HBA_SMPS (1 << 28) + +/* Host Bus Adapter port-specific platform capabilities */ +#define HBA_PORT_HPCP (1 << 18) +#define HBA_PORT_MPSP (1 << 19) +#define HBA_PORT_CPD (1 << 20) +#define HBA_PORT_ESP (1 << 21) +#define HBA_PORT_FBSCP (1 << 22) + +#endif diff --git a/include/dt-bindings/clock/aspeed-clock.h b/include/dt-bindings/clock/aspeed-clock.h index 9ff4f6e4558c411b2aa0464c4f699fa7c1dc4195..06d568382c779952fee715fe64dd5391cc630713 100644 --- a/include/dt-bindings/clock/aspeed-clock.h +++ b/include/dt-bindings/clock/aspeed-clock.h @@ -52,5 +52,6 @@ #define ASPEED_RESET_I2C 7 #define ASPEED_RESET_AHB 8 #define ASPEED_RESET_CRT1 9 +#define ASPEED_RESET_HACE 10 #endif diff --git a/include/dt-bindings/clock/ast2600-clock.h b/include/dt-bindings/clock/ast2600-clock.h index 62b9520a00fdbbb98b9dde5446b5de3b7f67da01..d8b0db2f7a7d1c2276388e0512b776afe1044618 100644 --- a/include/dt-bindings/clock/ast2600-clock.h +++ b/include/dt-bindings/clock/ast2600-clock.h @@ -111,6 +111,7 @@ #define ASPEED_RESET_PCIE_RC_O 19 #define ASPEED_RESET_PCIE_RC_OEN 18 #define ASPEED_RESET_PCI_DP 5 +#define ASPEED_RESET_HACE 4 #define ASPEED_RESET_AHB 1 #define ASPEED_RESET_SDRAM 0 diff --git a/include/dt-bindings/clock/exynos850.h b/include/dt-bindings/clock/exynos850.h index 0b6a3c6a7c90e2134a8ec8058a1663d4f3aa2023..88d5289883d38361a1250d0856e035705134a890 100644 --- a/include/dt-bindings/clock/exynos850.h +++ b/include/dt-bindings/clock/exynos850.h @@ -58,7 +58,34 @@ #define CLK_MOUT_CLKCMU_APM_BUS 46 #define CLK_DOUT_CLKCMU_APM_BUS 47 #define CLK_GOUT_CLKCMU_APM_BUS 48 -#define TOP_NR_CLK 49 +#define CLK_MOUT_AUD 49 +#define CLK_GOUT_AUD 50 +#define CLK_DOUT_AUD 51 +#define CLK_MOUT_IS_BUS 52 +#define CLK_MOUT_IS_ITP 53 +#define CLK_MOUT_IS_VRA 54 +#define CLK_MOUT_IS_GDC 55 +#define CLK_GOUT_IS_BUS 56 +#define CLK_GOUT_IS_ITP 57 +#define CLK_GOUT_IS_VRA 58 +#define CLK_GOUT_IS_GDC 59 +#define CLK_DOUT_IS_BUS 60 +#define CLK_DOUT_IS_ITP 61 +#define CLK_DOUT_IS_VRA 62 +#define CLK_DOUT_IS_GDC 63 +#define CLK_MOUT_MFCMSCL_MFC 64 +#define CLK_MOUT_MFCMSCL_M2M 65 +#define CLK_MOUT_MFCMSCL_MCSC 66 +#define CLK_MOUT_MFCMSCL_JPEG 67 +#define CLK_GOUT_MFCMSCL_MFC 68 +#define CLK_GOUT_MFCMSCL_M2M 69 +#define CLK_GOUT_MFCMSCL_MCSC 70 +#define CLK_GOUT_MFCMSCL_JPEG 71 +#define CLK_DOUT_MFCMSCL_MFC 72 +#define CLK_DOUT_MFCMSCL_M2M 73 +#define CLK_DOUT_MFCMSCL_MCSC 74 +#define CLK_DOUT_MFCMSCL_JPEG 75 +#define TOP_NR_CLK 76 /* CMU_APM */ #define CLK_RCO_I3C_PMIC 1 @@ -87,6 +114,69 @@ #define CLK_GOUT_SYSREG_APM_PCLK 24 #define APM_NR_CLK 25 +/* CMU_AUD */ +#define CLK_DOUT_AUD_AUDIF 1 +#define CLK_DOUT_AUD_BUSD 2 +#define CLK_DOUT_AUD_BUSP 3 +#define CLK_DOUT_AUD_CNT 4 +#define CLK_DOUT_AUD_CPU 5 +#define CLK_DOUT_AUD_CPU_ACLK 6 +#define CLK_DOUT_AUD_CPU_PCLKDBG 7 +#define CLK_DOUT_AUD_FM 8 +#define CLK_DOUT_AUD_FM_SPDY 9 +#define CLK_DOUT_AUD_MCLK 10 +#define CLK_DOUT_AUD_UAIF0 11 +#define CLK_DOUT_AUD_UAIF1 12 +#define CLK_DOUT_AUD_UAIF2 13 +#define CLK_DOUT_AUD_UAIF3 14 +#define CLK_DOUT_AUD_UAIF4 15 +#define CLK_DOUT_AUD_UAIF5 16 +#define CLK_DOUT_AUD_UAIF6 17 +#define CLK_FOUT_AUD_PLL 18 +#define CLK_GOUT_AUD_ABOX_ACLK 19 +#define CLK_GOUT_AUD_ASB_CCLK 20 +#define CLK_GOUT_AUD_CA32_CCLK 21 +#define CLK_GOUT_AUD_CNT_BCLK 22 +#define CLK_GOUT_AUD_CODEC_MCLK 23 +#define CLK_GOUT_AUD_DAP_CCLK 24 +#define CLK_GOUT_AUD_GPIO_PCLK 25 +#define CLK_GOUT_AUD_PPMU_ACLK 26 +#define CLK_GOUT_AUD_PPMU_PCLK 27 +#define CLK_GOUT_AUD_SPDY_BCLK 28 +#define CLK_GOUT_AUD_SYSMMU_CLK 29 +#define CLK_GOUT_AUD_SYSREG_PCLK 30 +#define CLK_GOUT_AUD_TZPC_PCLK 31 +#define CLK_GOUT_AUD_UAIF0_BCLK 32 +#define CLK_GOUT_AUD_UAIF1_BCLK 33 +#define CLK_GOUT_AUD_UAIF2_BCLK 34 +#define CLK_GOUT_AUD_UAIF3_BCLK 35 +#define CLK_GOUT_AUD_UAIF4_BCLK 36 +#define CLK_GOUT_AUD_UAIF5_BCLK 37 +#define CLK_GOUT_AUD_UAIF6_BCLK 38 +#define CLK_GOUT_AUD_WDT_PCLK 39 +#define CLK_MOUT_AUD_CPU 40 +#define CLK_MOUT_AUD_CPU_HCH 41 +#define CLK_MOUT_AUD_CPU_USER 42 +#define CLK_MOUT_AUD_FM 43 +#define CLK_MOUT_AUD_PLL 44 +#define CLK_MOUT_AUD_TICK_USB_USER 45 +#define CLK_MOUT_AUD_UAIF0 46 +#define CLK_MOUT_AUD_UAIF1 47 +#define CLK_MOUT_AUD_UAIF2 48 +#define CLK_MOUT_AUD_UAIF3 49 +#define CLK_MOUT_AUD_UAIF4 50 +#define CLK_MOUT_AUD_UAIF5 51 +#define CLK_MOUT_AUD_UAIF6 52 +#define IOCLK_AUDIOCDCLK0 53 +#define IOCLK_AUDIOCDCLK1 54 +#define IOCLK_AUDIOCDCLK2 55 +#define IOCLK_AUDIOCDCLK3 56 +#define IOCLK_AUDIOCDCLK4 57 +#define IOCLK_AUDIOCDCLK5 58 +#define IOCLK_AUDIOCDCLK6 59 +#define TICK_USB 60 +#define AUD_NR_CLK 61 + /* CMU_CMGP */ #define CLK_RCO_CMGP 1 #define CLK_MOUT_CMGP_ADC 2 @@ -121,6 +211,50 @@ #define CLK_GOUT_SYSREG_HSI_PCLK 13 #define HSI_NR_CLK 14 +/* CMU_IS */ +#define CLK_MOUT_IS_BUS_USER 1 +#define CLK_MOUT_IS_ITP_USER 2 +#define CLK_MOUT_IS_VRA_USER 3 +#define CLK_MOUT_IS_GDC_USER 4 +#define CLK_DOUT_IS_BUSP 5 +#define CLK_GOUT_IS_CMU_IS_PCLK 6 +#define CLK_GOUT_IS_CSIS0_ACLK 7 +#define CLK_GOUT_IS_CSIS1_ACLK 8 +#define CLK_GOUT_IS_CSIS2_ACLK 9 +#define CLK_GOUT_IS_TZPC_PCLK 10 +#define CLK_GOUT_IS_CSIS_DMA_CLK 11 +#define CLK_GOUT_IS_GDC_CLK 12 +#define CLK_GOUT_IS_IPP_CLK 13 +#define CLK_GOUT_IS_ITP_CLK 14 +#define CLK_GOUT_IS_MCSC_CLK 15 +#define CLK_GOUT_IS_VRA_CLK 16 +#define CLK_GOUT_IS_PPMU_IS0_ACLK 17 +#define CLK_GOUT_IS_PPMU_IS0_PCLK 18 +#define CLK_GOUT_IS_PPMU_IS1_ACLK 19 +#define CLK_GOUT_IS_PPMU_IS1_PCLK 20 +#define CLK_GOUT_IS_SYSMMU_IS0_CLK 21 +#define CLK_GOUT_IS_SYSMMU_IS1_CLK 22 +#define CLK_GOUT_IS_SYSREG_PCLK 23 +#define IS_NR_CLK 24 + +/* CMU_MFCMSCL */ +#define CLK_MOUT_MFCMSCL_MFC_USER 1 +#define CLK_MOUT_MFCMSCL_M2M_USER 2 +#define CLK_MOUT_MFCMSCL_MCSC_USER 3 +#define CLK_MOUT_MFCMSCL_JPEG_USER 4 +#define CLK_DOUT_MFCMSCL_BUSP 5 +#define CLK_GOUT_MFCMSCL_CMU_MFCMSCL_PCLK 6 +#define CLK_GOUT_MFCMSCL_TZPC_PCLK 7 +#define CLK_GOUT_MFCMSCL_JPEG_ACLK 8 +#define CLK_GOUT_MFCMSCL_M2M_ACLK 9 +#define CLK_GOUT_MFCMSCL_MCSC_CLK 10 +#define CLK_GOUT_MFCMSCL_MFC_ACLK 11 +#define CLK_GOUT_MFCMSCL_PPMU_ACLK 12 +#define CLK_GOUT_MFCMSCL_PPMU_PCLK 13 +#define CLK_GOUT_MFCMSCL_SYSMMU_CLK 14 +#define CLK_GOUT_MFCMSCL_SYSREG_PCLK 15 +#define MFCMSCL_NR_CLK 16 + /* CMU_PERI */ #define CLK_MOUT_PERI_BUS_USER 1 #define CLK_MOUT_PERI_UART_USER 2 diff --git a/include/dt-bindings/clock/imx8mm-clock.h b/include/dt-bindings/clock/imx8mm-clock.h index 47c6f7f9582c3b51ab6d99c7652792e6216e9125..1f768b2eeb1a6b631a70739f4f54ec309f5df60b 100644 --- a/include/dt-bindings/clock/imx8mm-clock.h +++ b/include/dt-bindings/clock/imx8mm-clock.h @@ -281,7 +281,6 @@ #define IMX8MM_CLK_CLKOUT2_DIV 256 #define IMX8MM_CLK_CLKOUT2 257 - #define IMX8MM_CLK_END 258 #endif diff --git a/include/dt-bindings/clock/imx93-clock.h b/include/dt-bindings/clock/imx93-clock.h index 21fda9c5cb5e53e62f5260be7166aeda6b59209d..19bc32788d81ef405b38946c9e3166655e3070e0 100644 --- a/include/dt-bindings/clock/imx93-clock.h +++ b/include/dt-bindings/clock/imx93-clock.h @@ -196,6 +196,13 @@ #define IMX93_CLK_TMC_GATE 187 #define IMX93_CLK_PMRO_GATE 188 #define IMX93_CLK_32K 189 -#define IMX93_CLK_END 190 +#define IMX93_CLK_SAI1_IPG 190 +#define IMX93_CLK_SAI2_IPG 191 +#define IMX93_CLK_SAI3_IPG 192 +#define IMX93_CLK_MU1_A_GATE 193 +#define IMX93_CLK_MU1_B_GATE 194 +#define IMX93_CLK_MU2_A_GATE 195 +#define IMX93_CLK_MU2_B_GATE 196 +#define IMX93_CLK_END 197 #endif diff --git a/include/dt-bindings/clk/lochnagar.h b/include/dt-bindings/clock/lochnagar.h similarity index 100% rename from include/dt-bindings/clk/lochnagar.h rename to include/dt-bindings/clock/lochnagar.h diff --git a/include/dt-bindings/clock/marvell,pxa168.h b/include/dt-bindings/clock/marvell,pxa168.h index db2b41f1b127257bb3439a7fde566e4ce2f9a544..c92d969ae941a9af21d23478dbfac5a443c8d6e9 100644 --- a/include/dt-bindings/clock/marvell,pxa168.h +++ b/include/dt-bindings/clock/marvell,pxa168.h @@ -20,8 +20,11 @@ #define PXA168_CLK_PLL1_2_1_5 19 #define PXA168_CLK_PLL1_3_16 20 #define PXA168_CLK_PLL1_192 21 +#define PXA168_CLK_PLL1_2_1_10 22 +#define PXA168_CLK_PLL1_2_3_16 23 #define PXA168_CLK_UART_PLL 27 #define PXA168_CLK_USB_PLL 28 +#define PXA168_CLK_CLK32_2 50 /* apb peripherals */ #define PXA168_CLK_TWSI0 60 @@ -56,6 +59,9 @@ #define PXA168_CLK_CCIC0 107 #define PXA168_CLK_CCIC0_PHY 108 #define PXA168_CLK_CCIC0_SPHY 109 +#define PXA168_CLK_SDH3 110 +#define PXA168_CLK_SDH01_AXI 111 +#define PXA168_CLK_SDH23_AXI 112 #define PXA168_NR_CLKS 200 #endif diff --git a/include/dt-bindings/clock/mediatek,mt6795-clk.h b/include/dt-bindings/clock/mediatek,mt6795-clk.h new file mode 100644 index 0000000000000000000000000000000000000000..9902906ac9025b4b0e919b0fb6f7ea8c24c6a384 --- /dev/null +++ b/include/dt-bindings/clock/mediatek,mt6795-clk.h @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause) */ +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#ifndef _DT_BINDINGS_CLK_MT6795_H +#define _DT_BINDINGS_CLK_MT6795_H + +/* TOPCKGEN */ +#define CLK_TOP_ADSYS_26M 0 +#define CLK_TOP_CLKPH_MCK_O 1 +#define CLK_TOP_USB_SYSPLL_125M 2 +#define CLK_TOP_DSI0_DIG 3 +#define CLK_TOP_DSI1_DIG 4 +#define CLK_TOP_ARMCA53PLL_754M 5 +#define CLK_TOP_ARMCA53PLL_502M 6 +#define CLK_TOP_MAIN_H546M 7 +#define CLK_TOP_MAIN_H364M 8 +#define CLK_TOP_MAIN_H218P4M 9 +#define CLK_TOP_MAIN_H156M 10 +#define CLK_TOP_TVDPLL_445P5M 11 +#define CLK_TOP_TVDPLL_594M 12 +#define CLK_TOP_UNIV_624M 13 +#define CLK_TOP_UNIV_416M 14 +#define CLK_TOP_UNIV_249P6M 15 +#define CLK_TOP_UNIV_178P3M 16 +#define CLK_TOP_UNIV_48M 17 +#define CLK_TOP_CLKRTC_EXT 18 +#define CLK_TOP_CLKRTC_INT 19 +#define CLK_TOP_FPC 20 +#define CLK_TOP_HDMITXPLL_D2 21 +#define CLK_TOP_HDMITXPLL_D3 22 +#define CLK_TOP_ARMCA53PLL_D2 23 +#define CLK_TOP_ARMCA53PLL_D3 24 +#define CLK_TOP_APLL1 25 +#define CLK_TOP_APLL2 26 +#define CLK_TOP_DMPLL 27 +#define CLK_TOP_DMPLL_D2 28 +#define CLK_TOP_DMPLL_D4 29 +#define CLK_TOP_DMPLL_D8 30 +#define CLK_TOP_DMPLL_D16 31 +#define CLK_TOP_MMPLL 32 +#define CLK_TOP_MMPLL_D2 33 +#define CLK_TOP_MSDCPLL 34 +#define CLK_TOP_MSDCPLL_D2 35 +#define CLK_TOP_MSDCPLL_D4 36 +#define CLK_TOP_MSDCPLL2 37 +#define CLK_TOP_MSDCPLL2_D2 38 +#define CLK_TOP_MSDCPLL2_D4 39 +#define CLK_TOP_SYSPLL_D2 40 +#define CLK_TOP_SYSPLL1_D2 41 +#define CLK_TOP_SYSPLL1_D4 42 +#define CLK_TOP_SYSPLL1_D8 43 +#define CLK_TOP_SYSPLL1_D16 44 +#define CLK_TOP_SYSPLL_D3 45 +#define CLK_TOP_SYSPLL2_D2 46 +#define CLK_TOP_SYSPLL2_D4 47 +#define CLK_TOP_SYSPLL_D5 48 +#define CLK_TOP_SYSPLL3_D2 49 +#define CLK_TOP_SYSPLL3_D4 50 +#define CLK_TOP_SYSPLL_D7 51 +#define CLK_TOP_SYSPLL4_D2 52 +#define CLK_TOP_SYSPLL4_D4 53 +#define CLK_TOP_TVDPLL 54 +#define CLK_TOP_TVDPLL_D2 55 +#define CLK_TOP_TVDPLL_D4 56 +#define CLK_TOP_TVDPLL_D8 57 +#define CLK_TOP_TVDPLL_D16 58 +#define CLK_TOP_UNIVPLL_D2 59 +#define CLK_TOP_UNIVPLL1_D2 60 +#define CLK_TOP_UNIVPLL1_D4 61 +#define CLK_TOP_UNIVPLL1_D8 62 +#define CLK_TOP_UNIVPLL_D3 63 +#define CLK_TOP_UNIVPLL2_D2 64 +#define CLK_TOP_UNIVPLL2_D4 65 +#define CLK_TOP_UNIVPLL2_D8 66 +#define CLK_TOP_UNIVPLL_D5 67 +#define CLK_TOP_UNIVPLL3_D2 68 +#define CLK_TOP_UNIVPLL3_D4 69 +#define CLK_TOP_UNIVPLL3_D8 70 +#define CLK_TOP_UNIVPLL_D7 71 +#define CLK_TOP_UNIVPLL_D26 72 +#define CLK_TOP_UNIVPLL_D52 73 +#define CLK_TOP_VCODECPLL 74 +#define CLK_TOP_VCODECPLL_370P5 75 +#define CLK_TOP_VENCPLL 76 +#define CLK_TOP_VENCPLL_D2 77 +#define CLK_TOP_VENCPLL_D4 78 +#define CLK_TOP_AXI_SEL 79 +#define CLK_TOP_MEM_SEL 80 +#define CLK_TOP_DDRPHYCFG_SEL 81 +#define CLK_TOP_MM_SEL 82 +#define CLK_TOP_PWM_SEL 83 +#define CLK_TOP_VDEC_SEL 84 +#define CLK_TOP_VENC_SEL 85 +#define CLK_TOP_MFG_SEL 86 +#define CLK_TOP_CAMTG_SEL 87 +#define CLK_TOP_UART_SEL 88 +#define CLK_TOP_SPI_SEL 89 +#define CLK_TOP_USB20_SEL 90 +#define CLK_TOP_USB30_SEL 91 +#define CLK_TOP_MSDC50_0_H_SEL 92 +#define CLK_TOP_MSDC50_0_SEL 93 +#define CLK_TOP_MSDC30_1_SEL 94 +#define CLK_TOP_MSDC30_2_SEL 95 +#define CLK_TOP_MSDC30_3_SEL 96 +#define CLK_TOP_AUDIO_SEL 97 +#define CLK_TOP_AUD_INTBUS_SEL 98 +#define CLK_TOP_PMICSPI_SEL 99 +#define CLK_TOP_SCP_SEL 100 +#define CLK_TOP_MJC_SEL 101 +#define CLK_TOP_DPI0_SEL 102 +#define CLK_TOP_IRDA_SEL 103 +#define CLK_TOP_CCI400_SEL 104 +#define CLK_TOP_AUD_1_SEL 105 +#define CLK_TOP_AUD_2_SEL 106 +#define CLK_TOP_MEM_MFG_IN_SEL 107 +#define CLK_TOP_AXI_MFG_IN_SEL 108 +#define CLK_TOP_SCAM_SEL 109 +#define CLK_TOP_I2S0_M_SEL 110 +#define CLK_TOP_I2S1_M_SEL 111 +#define CLK_TOP_I2S2_M_SEL 112 +#define CLK_TOP_I2S3_M_SEL 113 +#define CLK_TOP_I2S3_B_SEL 114 +#define CLK_TOP_APLL1_DIV0 115 +#define CLK_TOP_APLL1_DIV1 116 +#define CLK_TOP_APLL1_DIV2 117 +#define CLK_TOP_APLL1_DIV3 118 +#define CLK_TOP_APLL1_DIV4 119 +#define CLK_TOP_APLL1_DIV5 120 +#define CLK_TOP_APLL2_DIV0 121 +#define CLK_TOP_APLL2_DIV1 122 +#define CLK_TOP_APLL2_DIV2 123 +#define CLK_TOP_APLL2_DIV3 124 +#define CLK_TOP_APLL2_DIV4 125 +#define CLK_TOP_APLL2_DIV5 126 +#define CLK_TOP_NR_CLK 127 + +/* APMIXED_SYS */ +#define CLK_APMIXED_ARMCA53PLL 0 +#define CLK_APMIXED_MAINPLL 1 +#define CLK_APMIXED_UNIVPLL 2 +#define CLK_APMIXED_MMPLL 3 +#define CLK_APMIXED_MSDCPLL 4 +#define CLK_APMIXED_VENCPLL 5 +#define CLK_APMIXED_TVDPLL 6 +#define CLK_APMIXED_MPLL 7 +#define CLK_APMIXED_VCODECPLL 8 +#define CLK_APMIXED_APLL1 9 +#define CLK_APMIXED_APLL2 10 +#define CLK_APMIXED_REF2USB_TX 11 +#define CLK_APMIXED_NR_CLK 12 + +/* INFRA_SYS */ +#define CLK_INFRA_DBGCLK 0 +#define CLK_INFRA_SMI 1 +#define CLK_INFRA_AUDIO 2 +#define CLK_INFRA_GCE 3 +#define CLK_INFRA_L2C_SRAM 4 +#define CLK_INFRA_M4U 5 +#define CLK_INFRA_MD1MCU 6 +#define CLK_INFRA_MD1BUS 7 +#define CLK_INFRA_MD1DBB 8 +#define CLK_INFRA_DEVICE_APC 9 +#define CLK_INFRA_TRNG 10 +#define CLK_INFRA_MD1LTE 11 +#define CLK_INFRA_CPUM 12 +#define CLK_INFRA_KP 13 +#define CLK_INFRA_CA53_C0_SEL 14 +#define CLK_INFRA_CA53_C1_SEL 15 +#define CLK_INFRA_NR_CLK 16 + +/* PERI_SYS */ +#define CLK_PERI_NFI 0 +#define CLK_PERI_THERM 1 +#define CLK_PERI_PWM1 2 +#define CLK_PERI_PWM2 3 +#define CLK_PERI_PWM3 4 +#define CLK_PERI_PWM4 5 +#define CLK_PERI_PWM5 6 +#define CLK_PERI_PWM6 7 +#define CLK_PERI_PWM7 8 +#define CLK_PERI_PWM 9 +#define CLK_PERI_USB0 10 +#define CLK_PERI_USB1 11 +#define CLK_PERI_AP_DMA 12 +#define CLK_PERI_MSDC30_0 13 +#define CLK_PERI_MSDC30_1 14 +#define CLK_PERI_MSDC30_2 15 +#define CLK_PERI_MSDC30_3 16 +#define CLK_PERI_NLI_ARB 17 +#define CLK_PERI_IRDA 18 +#define CLK_PERI_UART0 19 +#define CLK_PERI_UART1 20 +#define CLK_PERI_UART2 21 +#define CLK_PERI_UART3 22 +#define CLK_PERI_I2C0 23 +#define CLK_PERI_I2C1 24 +#define CLK_PERI_I2C2 25 +#define CLK_PERI_I2C3 26 +#define CLK_PERI_I2C4 27 +#define CLK_PERI_AUXADC 28 +#define CLK_PERI_SPI0 29 +#define CLK_PERI_UART0_SEL 30 +#define CLK_PERI_UART1_SEL 31 +#define CLK_PERI_UART2_SEL 32 +#define CLK_PERI_UART3_SEL 33 +#define CLK_PERI_NR_CLK 34 + +/* MFG */ +#define CLK_MFG_BAXI 0 +#define CLK_MFG_BMEM 1 +#define CLK_MFG_BG3D 2 +#define CLK_MFG_B26M 3 +#define CLK_MFG_NR_CLK 4 + +/* MM_SYS */ +#define CLK_MM_SMI_COMMON 0 +#define CLK_MM_SMI_LARB0 1 +#define CLK_MM_CAM_MDP 2 +#define CLK_MM_MDP_RDMA0 3 +#define CLK_MM_MDP_RDMA1 4 +#define CLK_MM_MDP_RSZ0 5 +#define CLK_MM_MDP_RSZ1 6 +#define CLK_MM_MDP_RSZ2 7 +#define CLK_MM_MDP_TDSHP0 8 +#define CLK_MM_MDP_TDSHP1 9 +#define CLK_MM_MDP_CROP 10 +#define CLK_MM_MDP_WDMA 11 +#define CLK_MM_MDP_WROT0 12 +#define CLK_MM_MDP_WROT1 13 +#define CLK_MM_FAKE_ENG 14 +#define CLK_MM_MUTEX_32K 15 +#define CLK_MM_DISP_OVL0 16 +#define CLK_MM_DISP_OVL1 17 +#define CLK_MM_DISP_RDMA0 18 +#define CLK_MM_DISP_RDMA1 19 +#define CLK_MM_DISP_RDMA2 20 +#define CLK_MM_DISP_WDMA0 21 +#define CLK_MM_DISP_WDMA1 22 +#define CLK_MM_DISP_COLOR0 23 +#define CLK_MM_DISP_COLOR1 24 +#define CLK_MM_DISP_AAL 25 +#define CLK_MM_DISP_GAMMA 26 +#define CLK_MM_DISP_UFOE 27 +#define CLK_MM_DISP_SPLIT0 28 +#define CLK_MM_DISP_SPLIT1 29 +#define CLK_MM_DISP_MERGE 30 +#define CLK_MM_DISP_OD 31 +#define CLK_MM_DISP_PWM0MM 32 +#define CLK_MM_DISP_PWM026M 33 +#define CLK_MM_DISP_PWM1MM 34 +#define CLK_MM_DISP_PWM126M 35 +#define CLK_MM_DSI0_ENGINE 36 +#define CLK_MM_DSI0_DIGITAL 37 +#define CLK_MM_DSI1_ENGINE 38 +#define CLK_MM_DSI1_DIGITAL 39 +#define CLK_MM_DPI_PIXEL 40 +#define CLK_MM_DPI_ENGINE 41 +#define CLK_MM_NR_CLK 42 + +/* VDEC_SYS */ +#define CLK_VDEC_CKEN 0 +#define CLK_VDEC_LARB_CKEN 1 +#define CLK_VDEC_NR_CLK 2 + +/* VENC_SYS */ +#define CLK_VENC_LARB 0 +#define CLK_VENC_VENC 1 +#define CLK_VENC_JPGENC 2 +#define CLK_VENC_JPGDEC 3 +#define CLK_VENC_NR_CLK 4 + +#endif /* _DT_BINDINGS_CLK_MT6795_H */ diff --git a/include/dt-bindings/clock/mediatek,mt8365-clk.h b/include/dt-bindings/clock/mediatek,mt8365-clk.h new file mode 100644 index 0000000000000000000000000000000000000000..f9aff177581094d1f6094ea29bd6a1f232346472 --- /dev/null +++ b/include/dt-bindings/clock/mediatek,mt8365-clk.h @@ -0,0 +1,373 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) + * + * Copyright (c) 2022 MediaTek Inc. + */ + +#ifndef _DT_BINDINGS_CLK_MT8365_H +#define _DT_BINDINGS_CLK_MT8365_H + +/* TOPCKGEN */ +#define CLK_TOP_CLK_NULL 0 +#define CLK_TOP_I2S0_BCK 1 +#define CLK_TOP_DSI0_LNTC_DSICK 2 +#define CLK_TOP_VPLL_DPIX 3 +#define CLK_TOP_LVDSTX_CLKDIG_CTS 4 +#define CLK_TOP_MFGPLL 5 +#define CLK_TOP_SYSPLL_D2 6 +#define CLK_TOP_SYSPLL1_D2 7 +#define CLK_TOP_SYSPLL1_D4 8 +#define CLK_TOP_SYSPLL1_D8 9 +#define CLK_TOP_SYSPLL1_D16 10 +#define CLK_TOP_SYSPLL_D3 11 +#define CLK_TOP_SYSPLL2_D2 12 +#define CLK_TOP_SYSPLL2_D4 13 +#define CLK_TOP_SYSPLL2_D8 14 +#define CLK_TOP_SYSPLL_D5 15 +#define CLK_TOP_SYSPLL3_D2 16 +#define CLK_TOP_SYSPLL3_D4 17 +#define CLK_TOP_SYSPLL_D7 18 +#define CLK_TOP_SYSPLL4_D2 19 +#define CLK_TOP_SYSPLL4_D4 20 +#define CLK_TOP_UNIVPLL 21 +#define CLK_TOP_UNIVPLL_D2 22 +#define CLK_TOP_UNIVPLL1_D2 23 +#define CLK_TOP_UNIVPLL1_D4 24 +#define CLK_TOP_UNIVPLL_D3 25 +#define CLK_TOP_UNIVPLL2_D2 26 +#define CLK_TOP_UNIVPLL2_D4 27 +#define CLK_TOP_UNIVPLL2_D8 28 +#define CLK_TOP_UNIVPLL2_D32 29 +#define CLK_TOP_UNIVPLL_D5 30 +#define CLK_TOP_UNIVPLL3_D2 31 +#define CLK_TOP_UNIVPLL3_D4 32 +#define CLK_TOP_MMPLL 33 +#define CLK_TOP_MMPLL_D2 34 +#define CLK_TOP_LVDSPLL_D2 35 +#define CLK_TOP_LVDSPLL_D4 36 +#define CLK_TOP_LVDSPLL_D8 37 +#define CLK_TOP_LVDSPLL_D16 38 +#define CLK_TOP_USB20_192M 39 +#define CLK_TOP_USB20_192M_D4 40 +#define CLK_TOP_USB20_192M_D8 41 +#define CLK_TOP_USB20_192M_D16 42 +#define CLK_TOP_USB20_192M_D32 43 +#define CLK_TOP_APLL1 44 +#define CLK_TOP_APLL1_D2 45 +#define CLK_TOP_APLL1_D4 46 +#define CLK_TOP_APLL1_D8 47 +#define CLK_TOP_APLL2 48 +#define CLK_TOP_APLL2_D2 49 +#define CLK_TOP_APLL2_D4 50 +#define CLK_TOP_APLL2_D8 51 +#define CLK_TOP_SYS_26M_D2 52 +#define CLK_TOP_MSDCPLL 53 +#define CLK_TOP_MSDCPLL_D2 54 +#define CLK_TOP_DSPPLL 55 +#define CLK_TOP_DSPPLL_D2 56 +#define CLK_TOP_DSPPLL_D4 57 +#define CLK_TOP_DSPPLL_D8 58 +#define CLK_TOP_APUPLL 59 +#define CLK_TOP_CLK26M_D52 60 +#define CLK_TOP_AXI_SEL 61 +#define CLK_TOP_MEM_SEL 62 +#define CLK_TOP_MM_SEL 63 +#define CLK_TOP_SCP_SEL 64 +#define CLK_TOP_MFG_SEL 65 +#define CLK_TOP_ATB_SEL 66 +#define CLK_TOP_CAMTG_SEL 67 +#define CLK_TOP_CAMTG1_SEL 68 +#define CLK_TOP_UART_SEL 69 +#define CLK_TOP_SPI_SEL 70 +#define CLK_TOP_MSDC50_0_HC_SEL 71 +#define CLK_TOP_MSDC2_2_HC_SEL 72 +#define CLK_TOP_MSDC50_0_SEL 73 +#define CLK_TOP_MSDC50_2_SEL 74 +#define CLK_TOP_MSDC30_1_SEL 75 +#define CLK_TOP_AUDIO_SEL 76 +#define CLK_TOP_AUD_INTBUS_SEL 77 +#define CLK_TOP_AUD_1_SEL 78 +#define CLK_TOP_AUD_2_SEL 79 +#define CLK_TOP_AUD_ENGEN1_SEL 80 +#define CLK_TOP_AUD_ENGEN2_SEL 81 +#define CLK_TOP_AUD_SPDIF_SEL 82 +#define CLK_TOP_DISP_PWM_SEL 83 +#define CLK_TOP_DXCC_SEL 84 +#define CLK_TOP_SSUSB_SYS_SEL 85 +#define CLK_TOP_SSUSB_XHCI_SEL 86 +#define CLK_TOP_SPM_SEL 87 +#define CLK_TOP_I2C_SEL 88 +#define CLK_TOP_PWM_SEL 89 +#define CLK_TOP_SENIF_SEL 90 +#define CLK_TOP_AES_FDE_SEL 91 +#define CLK_TOP_CAMTM_SEL 92 +#define CLK_TOP_DPI0_SEL 93 +#define CLK_TOP_DPI1_SEL 94 +#define CLK_TOP_DSP_SEL 95 +#define CLK_TOP_NFI2X_SEL 96 +#define CLK_TOP_NFIECC_SEL 97 +#define CLK_TOP_ECC_SEL 98 +#define CLK_TOP_ETH_SEL 99 +#define CLK_TOP_GCPU_SEL 100 +#define CLK_TOP_GCPU_CPM_SEL 101 +#define CLK_TOP_APU_SEL 102 +#define CLK_TOP_APU_IF_SEL 103 +#define CLK_TOP_MBIST_DIAG_SEL 104 +#define CLK_TOP_APLL_I2S0_SEL 105 +#define CLK_TOP_APLL_I2S1_SEL 106 +#define CLK_TOP_APLL_I2S2_SEL 107 +#define CLK_TOP_APLL_I2S3_SEL 108 +#define CLK_TOP_APLL_TDMOUT_SEL 109 +#define CLK_TOP_APLL_TDMIN_SEL 110 +#define CLK_TOP_APLL_SPDIF_SEL 111 +#define CLK_TOP_APLL12_CK_DIV0 112 +#define CLK_TOP_APLL12_CK_DIV1 113 +#define CLK_TOP_APLL12_CK_DIV2 114 +#define CLK_TOP_APLL12_CK_DIV3 115 +#define CLK_TOP_APLL12_CK_DIV4 116 +#define CLK_TOP_APLL12_CK_DIV4B 117 +#define CLK_TOP_APLL12_CK_DIV5 118 +#define CLK_TOP_APLL12_CK_DIV5B 119 +#define CLK_TOP_APLL12_CK_DIV6 120 +#define CLK_TOP_AUD_I2S0_M 121 +#define CLK_TOP_AUD_I2S1_M 122 +#define CLK_TOP_AUD_I2S2_M 123 +#define CLK_TOP_AUD_I2S3_M 124 +#define CLK_TOP_AUD_TDMOUT_M 125 +#define CLK_TOP_AUD_TDMOUT_B 126 +#define CLK_TOP_AUD_TDMIN_M 127 +#define CLK_TOP_AUD_TDMIN_B 128 +#define CLK_TOP_AUD_SPDIF_M 129 +#define CLK_TOP_USB20_48M_EN 130 +#define CLK_TOP_UNIVPLL_48M_EN 131 +#define CLK_TOP_LVDSTX_CLKDIG_EN 132 +#define CLK_TOP_VPLL_DPIX_EN 133 +#define CLK_TOP_SSUSB_TOP_CK_EN 134 +#define CLK_TOP_SSUSB_PHY_CK_EN 135 +#define CLK_TOP_CONN_32K 136 +#define CLK_TOP_CONN_26M 137 +#define CLK_TOP_DSP_32K 138 +#define CLK_TOP_DSP_26M 139 +#define CLK_TOP_NR_CLK 140 + +/* INFRACFG */ +#define CLK_IFR_PMIC_TMR 0 +#define CLK_IFR_PMIC_AP 1 +#define CLK_IFR_PMIC_MD 2 +#define CLK_IFR_PMIC_CONN 3 +#define CLK_IFR_ICUSB 4 +#define CLK_IFR_GCE 5 +#define CLK_IFR_THERM 6 +#define CLK_IFR_PWM_HCLK 7 +#define CLK_IFR_PWM1 8 +#define CLK_IFR_PWM2 9 +#define CLK_IFR_PWM3 10 +#define CLK_IFR_PWM4 11 +#define CLK_IFR_PWM5 12 +#define CLK_IFR_PWM 13 +#define CLK_IFR_UART0 14 +#define CLK_IFR_UART1 15 +#define CLK_IFR_UART2 16 +#define CLK_IFR_DSP_UART 17 +#define CLK_IFR_GCE_26M 18 +#define CLK_IFR_CQ_DMA_FPC 19 +#define CLK_IFR_BTIF 20 +#define CLK_IFR_SPI0 21 +#define CLK_IFR_MSDC0_HCLK 22 +#define CLK_IFR_MSDC2_HCLK 23 +#define CLK_IFR_MSDC1_HCLK 24 +#define CLK_IFR_DVFSRC 25 +#define CLK_IFR_GCPU 26 +#define CLK_IFR_TRNG 27 +#define CLK_IFR_AUXADC 28 +#define CLK_IFR_CPUM 29 +#define CLK_IFR_AUXADC_MD 30 +#define CLK_IFR_AP_DMA 31 +#define CLK_IFR_DEBUGSYS 32 +#define CLK_IFR_AUDIO 33 +#define CLK_IFR_PWM_FBCLK6 34 +#define CLK_IFR_DISP_PWM 35 +#define CLK_IFR_AUD_26M_BK 36 +#define CLK_IFR_CQ_DMA 37 +#define CLK_IFR_MSDC0_SF 38 +#define CLK_IFR_MSDC1_SF 39 +#define CLK_IFR_MSDC2_SF 40 +#define CLK_IFR_AP_MSDC0 41 +#define CLK_IFR_MD_MSDC0 42 +#define CLK_IFR_MSDC0_SRC 43 +#define CLK_IFR_MSDC1_SRC 44 +#define CLK_IFR_MSDC2_SRC 45 +#define CLK_IFR_PWRAP_TMR 46 +#define CLK_IFR_PWRAP_SPI 47 +#define CLK_IFR_PWRAP_SYS 48 +#define CLK_IFR_MCU_PM_BK 49 +#define CLK_IFR_IRRX_26M 50 +#define CLK_IFR_IRRX_32K 51 +#define CLK_IFR_I2C0_AXI 52 +#define CLK_IFR_I2C1_AXI 53 +#define CLK_IFR_I2C2_AXI 54 +#define CLK_IFR_I2C3_AXI 55 +#define CLK_IFR_NIC_AXI 56 +#define CLK_IFR_NIC_SLV_AXI 57 +#define CLK_IFR_APU_AXI 58 +#define CLK_IFR_NFIECC 59 +#define CLK_IFR_NFIECC_BK 60 +#define CLK_IFR_NFI1X_BK 61 +#define CLK_IFR_NFI_BK 62 +#define CLK_IFR_MSDC2_AP_BK 63 +#define CLK_IFR_MSDC2_MD_BK 64 +#define CLK_IFR_MSDC2_BK 65 +#define CLK_IFR_SUSB_133_BK 66 +#define CLK_IFR_SUSB_66_BK 67 +#define CLK_IFR_SSUSB_SYS 68 +#define CLK_IFR_SSUSB_REF 69 +#define CLK_IFR_SSUSB_XHCI 70 +#define CLK_IFR_NR_CLK 71 + +/* PERICFG */ +#define CLK_PERIAXI 0 +#define CLK_PERI_NR_CLK 1 + +/* APMIXEDSYS */ +#define CLK_APMIXED_ARMPLL 0 +#define CLK_APMIXED_MAINPLL 1 +#define CLK_APMIXED_UNIVPLL 2 +#define CLK_APMIXED_MFGPLL 3 +#define CLK_APMIXED_MSDCPLL 4 +#define CLK_APMIXED_MMPLL 5 +#define CLK_APMIXED_APLL1 6 +#define CLK_APMIXED_APLL2 7 +#define CLK_APMIXED_LVDSPLL 8 +#define CLK_APMIXED_DSPPLL 9 +#define CLK_APMIXED_APUPLL 10 +#define CLK_APMIXED_UNIV_EN 11 +#define CLK_APMIXED_USB20_EN 12 +#define CLK_APMIXED_NR_CLK 13 + +/* GCE */ +#define CLK_GCE_FAXI 0 +#define CLK_GCE_NR_CLK 1 + +/* AUDIOTOP */ +#define CLK_AUD_AFE 0 +#define CLK_AUD_I2S 1 +#define CLK_AUD_22M 2 +#define CLK_AUD_24M 3 +#define CLK_AUD_INTDIR 4 +#define CLK_AUD_APLL2_TUNER 5 +#define CLK_AUD_APLL_TUNER 6 +#define CLK_AUD_SPDF 7 +#define CLK_AUD_HDMI 8 +#define CLK_AUD_HDMI_IN 9 +#define CLK_AUD_ADC 10 +#define CLK_AUD_DAC 11 +#define CLK_AUD_DAC_PREDIS 12 +#define CLK_AUD_TML 13 +#define CLK_AUD_I2S1_BK 14 +#define CLK_AUD_I2S2_BK 15 +#define CLK_AUD_I2S3_BK 16 +#define CLK_AUD_I2S4_BK 17 +#define CLK_AUD_NR_CLK 18 + +/* MIPI_CSI0A */ +#define CLK_MIPI0A_CSR_CSI_EN_0A 0 +#define CLK_MIPI_RX_ANA_CSI0A_NR_CLK 1 + +/* MIPI_CSI0B */ +#define CLK_MIPI0B_CSR_CSI_EN_0B 0 +#define CLK_MIPI_RX_ANA_CSI0B_NR_CLK 1 + +/* MIPI_CSI1A */ +#define CLK_MIPI1A_CSR_CSI_EN_1A 0 +#define CLK_MIPI_RX_ANA_CSI1A_NR_CLK 1 + +/* MIPI_CSI1B */ +#define CLK_MIPI1B_CSR_CSI_EN_1B 0 +#define CLK_MIPI_RX_ANA_CSI1B_NR_CLK 1 + +/* MIPI_CSI2A */ +#define CLK_MIPI2A_CSR_CSI_EN_2A 0 +#define CLK_MIPI_RX_ANA_CSI2A_NR_CLK 1 + +/* MIPI_CSI2B */ +#define CLK_MIPI2B_CSR_CSI_EN_2B 0 +#define CLK_MIPI_RX_ANA_CSI2B_NR_CLK 1 + +/* MCUCFG */ +#define CLK_MCU_BUS_SEL 0 +#define CLK_MCU_NR_CLK 1 + +/* MFGCFG */ +#define CLK_MFG_BG3D 0 +#define CLK_MFG_MBIST_DIAG 1 +#define CLK_MFG_NR_CLK 2 + +/* MMSYS */ +#define CLK_MM_MM_MDP_RDMA0 0 +#define CLK_MM_MM_MDP_CCORR0 1 +#define CLK_MM_MM_MDP_RSZ0 2 +#define CLK_MM_MM_MDP_RSZ1 3 +#define CLK_MM_MM_MDP_TDSHP0 4 +#define CLK_MM_MM_MDP_WROT0 5 +#define CLK_MM_MM_MDP_WDMA0 6 +#define CLK_MM_MM_DISP_OVL0 7 +#define CLK_MM_MM_DISP_OVL0_2L 8 +#define CLK_MM_MM_DISP_RSZ0 9 +#define CLK_MM_MM_DISP_RDMA0 10 +#define CLK_MM_MM_DISP_WDMA0 11 +#define CLK_MM_MM_DISP_COLOR0 12 +#define CLK_MM_MM_DISP_CCORR0 13 +#define CLK_MM_MM_DISP_AAL0 14 +#define CLK_MM_MM_DISP_GAMMA0 15 +#define CLK_MM_MM_DISP_DITHER0 16 +#define CLK_MM_MM_DSI0 17 +#define CLK_MM_MM_DISP_RDMA1 18 +#define CLK_MM_MM_MDP_RDMA1 19 +#define CLK_MM_DPI0_DPI0 20 +#define CLK_MM_MM_FAKE 21 +#define CLK_MM_MM_SMI_COMMON 22 +#define CLK_MM_MM_SMI_LARB0 23 +#define CLK_MM_MM_SMI_COMM0 24 +#define CLK_MM_MM_SMI_COMM1 25 +#define CLK_MM_MM_CAM_MDP 26 +#define CLK_MM_MM_SMI_IMG 27 +#define CLK_MM_MM_SMI_CAM 28 +#define CLK_MM_IMG_IMG_DL_RELAY 29 +#define CLK_MM_IMG_IMG_DL_ASYNC_TOP 30 +#define CLK_MM_DSI0_DIG_DSI 31 +#define CLK_MM_26M_HRTWT 32 +#define CLK_MM_MM_DPI0 33 +#define CLK_MM_LVDSTX_PXL 34 +#define CLK_MM_LVDSTX_CTS 35 +#define CLK_MM_NR_CLK 36 + +/* IMGSYS */ +#define CLK_CAM_LARB2 0 +#define CLK_CAM 1 +#define CLK_CAMTG 2 +#define CLK_CAM_SENIF 3 +#define CLK_CAMSV0 4 +#define CLK_CAMSV1 5 +#define CLK_CAM_FDVT 6 +#define CLK_CAM_WPE 7 +#define CLK_CAM_NR_CLK 8 + +/* VDECSYS */ +#define CLK_VDEC_VDEC 0 +#define CLK_VDEC_LARB1 1 +#define CLK_VDEC_NR_CLK 2 + +/* VENCSYS */ +#define CLK_VENC 0 +#define CLK_VENC_JPGENC 1 +#define CLK_VENC_NR_CLK 2 + +/* APUSYS */ +#define CLK_APU_IPU_CK 0 +#define CLK_APU_AXI 1 +#define CLK_APU_JTAG 2 +#define CLK_APU_IF_CK 3 +#define CLK_APU_EDMA 4 +#define CLK_APU_AHB 5 +#define CLK_APU_NR_CLK 6 + +#endif /* _DT_BINDINGS_CLK_MT8365_H */ diff --git a/include/dt-bindings/clock/microchip,mpfs-clock.h b/include/dt-bindings/clock/microchip,mpfs-clock.h index 4048669bf756493bbc878163a3cd79cdb57c76cc..79775a5134caa22f46d7b9eb826d3af90bee8708 100644 --- a/include/dt-bindings/clock/microchip,mpfs-clock.h +++ b/include/dt-bindings/clock/microchip,mpfs-clock.h @@ -45,4 +45,27 @@ #define CLK_RTCREF 33 #define CLK_MSSPLL 34 +/* Clock Conditioning Circuitry Clock IDs */ + +#define CLK_CCC_PLL0 0 +#define CLK_CCC_PLL1 1 +#define CLK_CCC_DLL0 2 +#define CLK_CCC_DLL1 3 + +#define CLK_CCC_PLL0_OUT0 4 +#define CLK_CCC_PLL0_OUT1 5 +#define CLK_CCC_PLL0_OUT2 6 +#define CLK_CCC_PLL0_OUT3 7 + +#define CLK_CCC_PLL1_OUT0 8 +#define CLK_CCC_PLL1_OUT1 9 +#define CLK_CCC_PLL1_OUT2 10 +#define CLK_CCC_PLL1_OUT3 11 + +#define CLK_CCC_DLL0_OUT0 12 +#define CLK_CCC_DLL0_OUT1 13 + +#define CLK_CCC_DLL1_OUT0 14 +#define CLK_CCC_DLL1_OUT1 15 + #endif /* _DT_BINDINGS_CLK_MICROCHIP_MPFS_H_ */ diff --git a/include/dt-bindings/clock/mt8195-clk.h b/include/dt-bindings/clock/mt8195-clk.h index 95cf812a0b372e179b8fad7922c56624d7c60ca9..d70d017ad69ca8bcbc15684ced699b977e5fbdd7 100644 --- a/include/dt-bindings/clock/mt8195-clk.h +++ b/include/dt-bindings/clock/mt8195-clk.h @@ -859,6 +859,8 @@ #define CLK_VDO1_DPINTF 47 #define CLK_VDO1_DISP_MONITOR_DPINTF 48 #define CLK_VDO1_26M_SLOW 49 -#define CLK_VDO1_NR_CLK 50 +#define CLK_VDO1_DPI1_HDMI 50 +#define CLK_VDO1_NR_CLK 51 + #endif /* _DT_BINDINGS_CLK_MT8195_H */ diff --git a/include/dt-bindings/clock/qcom,gcc-msm8909.h b/include/dt-bindings/clock/qcom,gcc-msm8909.h new file mode 100644 index 0000000000000000000000000000000000000000..4394ba0034254023a3b6d8772164aa793541e4d5 --- /dev/null +++ b/include/dt-bindings/clock/qcom,gcc-msm8909.h @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (C) 2022 Kernkonzept GmbH. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GCC_8909_H +#define _DT_BINDINGS_CLK_QCOM_GCC_8909_H + +/* PLLs */ +#define GPLL0_EARLY 0 +#define GPLL0 1 +#define GPLL1 2 +#define GPLL1_VOTE 3 +#define GPLL2_EARLY 4 +#define GPLL2 5 +#define BIMC_PLL_EARLY 6 +#define BIMC_PLL 7 + +/* RCGs */ +#define APSS_AHB_CLK_SRC 8 +#define BIMC_DDR_CLK_SRC 9 +#define BIMC_GPU_CLK_SRC 10 +#define BLSP1_QUP1_I2C_APPS_CLK_SRC 11 +#define BLSP1_QUP1_SPI_APPS_CLK_SRC 12 +#define BLSP1_QUP2_I2C_APPS_CLK_SRC 13 +#define BLSP1_QUP2_SPI_APPS_CLK_SRC 14 +#define BLSP1_QUP3_I2C_APPS_CLK_SRC 15 +#define BLSP1_QUP3_SPI_APPS_CLK_SRC 16 +#define BLSP1_QUP4_I2C_APPS_CLK_SRC 17 +#define BLSP1_QUP4_SPI_APPS_CLK_SRC 18 +#define BLSP1_QUP5_I2C_APPS_CLK_SRC 19 +#define BLSP1_QUP5_SPI_APPS_CLK_SRC 20 +#define BLSP1_QUP6_I2C_APPS_CLK_SRC 21 +#define BLSP1_QUP6_SPI_APPS_CLK_SRC 22 +#define BLSP1_UART1_APPS_CLK_SRC 23 +#define BLSP1_UART2_APPS_CLK_SRC 24 +#define BYTE0_CLK_SRC 25 +#define CAMSS_GP0_CLK_SRC 26 +#define CAMSS_GP1_CLK_SRC 27 +#define CAMSS_TOP_AHB_CLK_SRC 28 +#define CODEC_DIGCODEC_CLK_SRC 29 +#define CRYPTO_CLK_SRC 30 +#define CSI0_CLK_SRC 31 +#define CSI0PHYTIMER_CLK_SRC 32 +#define CSI1_CLK_SRC 33 +#define ESC0_CLK_SRC 34 +#define GFX3D_CLK_SRC 35 +#define GP1_CLK_SRC 36 +#define GP2_CLK_SRC 37 +#define GP3_CLK_SRC 38 +#define MCLK0_CLK_SRC 39 +#define MCLK1_CLK_SRC 40 +#define MDP_CLK_SRC 41 +#define PCLK0_CLK_SRC 42 +#define PCNOC_BFDCD_CLK_SRC 43 +#define PDM2_CLK_SRC 44 +#define SDCC1_APPS_CLK_SRC 45 +#define SDCC2_APPS_CLK_SRC 46 +#define SYSTEM_NOC_BFDCD_CLK_SRC 47 +#define ULTAUDIO_AHBFABRIC_CLK_SRC 48 +#define ULTAUDIO_LPAIF_AUX_I2S_CLK_SRC 49 +#define ULTAUDIO_LPAIF_PRI_I2S_CLK_SRC 50 +#define ULTAUDIO_LPAIF_SEC_I2S_CLK_SRC 51 +#define ULTAUDIO_XO_CLK_SRC 52 +#define USB_HS_SYSTEM_CLK_SRC 53 +#define VCODEC0_CLK_SRC 54 +#define VFE0_CLK_SRC 55 +#define VSYNC_CLK_SRC 56 + +/* Voteable Clocks */ +#define GCC_APSS_TCU_CLK 57 +#define GCC_BLSP1_AHB_CLK 58 +#define GCC_BLSP1_SLEEP_CLK 59 +#define GCC_BOOT_ROM_AHB_CLK 60 +#define GCC_CRYPTO_CLK 61 +#define GCC_CRYPTO_AHB_CLK 62 +#define GCC_CRYPTO_AXI_CLK 63 +#define GCC_GFX_TBU_CLK 64 +#define GCC_GFX_TCU_CLK 65 +#define GCC_GTCU_AHB_CLK 66 +#define GCC_MDP_TBU_CLK 67 +#define GCC_PRNG_AHB_CLK 68 +#define GCC_SMMU_CFG_CLK 69 +#define GCC_VENUS_TBU_CLK 70 +#define GCC_VFE_TBU_CLK 71 + +/* Branches */ +#define GCC_BIMC_GFX_CLK 72 +#define GCC_BIMC_GPU_CLK 73 +#define GCC_BLSP1_QUP1_I2C_APPS_CLK 74 +#define GCC_BLSP1_QUP1_SPI_APPS_CLK 75 +#define GCC_BLSP1_QUP2_I2C_APPS_CLK 76 +#define GCC_BLSP1_QUP2_SPI_APPS_CLK 77 +#define GCC_BLSP1_QUP3_I2C_APPS_CLK 78 +#define GCC_BLSP1_QUP3_SPI_APPS_CLK 79 +#define GCC_BLSP1_QUP4_I2C_APPS_CLK 80 +#define GCC_BLSP1_QUP4_SPI_APPS_CLK 81 +#define GCC_BLSP1_QUP5_I2C_APPS_CLK 82 +#define GCC_BLSP1_QUP5_SPI_APPS_CLK 83 +#define GCC_BLSP1_QUP6_I2C_APPS_CLK 84 +#define GCC_BLSP1_QUP6_SPI_APPS_CLK 85 +#define GCC_BLSP1_UART1_APPS_CLK 86 +#define GCC_BLSP1_UART2_APPS_CLK 87 +#define GCC_CAMSS_AHB_CLK 88 +#define GCC_CAMSS_CSI0_CLK 89 +#define GCC_CAMSS_CSI0_AHB_CLK 90 +#define GCC_CAMSS_CSI0PHY_CLK 91 +#define GCC_CAMSS_CSI0PHYTIMER_CLK 92 +#define GCC_CAMSS_CSI0PIX_CLK 93 +#define GCC_CAMSS_CSI0RDI_CLK 94 +#define GCC_CAMSS_CSI1_CLK 95 +#define GCC_CAMSS_CSI1_AHB_CLK 96 +#define GCC_CAMSS_CSI1PHY_CLK 97 +#define GCC_CAMSS_CSI1PIX_CLK 98 +#define GCC_CAMSS_CSI1RDI_CLK 99 +#define GCC_CAMSS_CSI_VFE0_CLK 100 +#define GCC_CAMSS_GP0_CLK 101 +#define GCC_CAMSS_GP1_CLK 102 +#define GCC_CAMSS_ISPIF_AHB_CLK 103 +#define GCC_CAMSS_MCLK0_CLK 104 +#define GCC_CAMSS_MCLK1_CLK 105 +#define GCC_CAMSS_TOP_AHB_CLK 106 +#define GCC_CAMSS_VFE0_CLK 107 +#define GCC_CAMSS_VFE_AHB_CLK 108 +#define GCC_CAMSS_VFE_AXI_CLK 109 +#define GCC_CODEC_DIGCODEC_CLK 110 +#define GCC_GP1_CLK 111 +#define GCC_GP2_CLK 112 +#define GCC_GP3_CLK 113 +#define GCC_MDSS_AHB_CLK 114 +#define GCC_MDSS_AXI_CLK 115 +#define GCC_MDSS_BYTE0_CLK 116 +#define GCC_MDSS_ESC0_CLK 117 +#define GCC_MDSS_MDP_CLK 118 +#define GCC_MDSS_PCLK0_CLK 119 +#define GCC_MDSS_VSYNC_CLK 120 +#define GCC_MSS_CFG_AHB_CLK 121 +#define GCC_MSS_Q6_BIMC_AXI_CLK 122 +#define GCC_OXILI_AHB_CLK 123 +#define GCC_OXILI_GFX3D_CLK 124 +#define GCC_PDM2_CLK 125 +#define GCC_PDM_AHB_CLK 126 +#define GCC_SDCC1_AHB_CLK 127 +#define GCC_SDCC1_APPS_CLK 128 +#define GCC_SDCC2_AHB_CLK 129 +#define GCC_SDCC2_APPS_CLK 130 +#define GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_CLK 131 +#define GCC_ULTAUDIO_AHBFABRIC_IXFABRIC_LPM_CLK 132 +#define GCC_ULTAUDIO_AVSYNC_XO_CLK 133 +#define GCC_ULTAUDIO_LPAIF_AUX_I2S_CLK 134 +#define GCC_ULTAUDIO_LPAIF_PRI_I2S_CLK 135 +#define GCC_ULTAUDIO_LPAIF_SEC_I2S_CLK 136 +#define GCC_ULTAUDIO_PCNOC_MPORT_CLK 137 +#define GCC_ULTAUDIO_PCNOC_SWAY_CLK 138 +#define GCC_ULTAUDIO_STC_XO_CLK 139 +#define GCC_USB2A_PHY_SLEEP_CLK 140 +#define GCC_USB_HS_AHB_CLK 141 +#define GCC_USB_HS_PHY_CFG_AHB_CLK 142 +#define GCC_USB_HS_SYSTEM_CLK 143 +#define GCC_VENUS0_AHB_CLK 144 +#define GCC_VENUS0_AXI_CLK 145 +#define GCC_VENUS0_CORE0_VCODEC0_CLK 146 +#define GCC_VENUS0_VCODEC0_CLK 147 + +/* Resets */ +#define GCC_AUDIO_CORE_BCR 0 +#define GCC_BLSP1_BCR 1 +#define GCC_BLSP1_QUP1_BCR 2 +#define GCC_BLSP1_QUP2_BCR 3 +#define GCC_BLSP1_QUP3_BCR 4 +#define GCC_BLSP1_QUP4_BCR 5 +#define GCC_BLSP1_QUP5_BCR 6 +#define GCC_BLSP1_QUP6_BCR 7 +#define GCC_BLSP1_UART1_BCR 8 +#define GCC_BLSP1_UART2_BCR 9 +#define GCC_CAMSS_CSI0_BCR 10 +#define GCC_CAMSS_CSI0PHY_BCR 11 +#define GCC_CAMSS_CSI0PIX_BCR 12 +#define GCC_CAMSS_CSI0RDI_BCR 13 +#define GCC_CAMSS_CSI1_BCR 14 +#define GCC_CAMSS_CSI1PHY_BCR 15 +#define GCC_CAMSS_CSI1PIX_BCR 16 +#define GCC_CAMSS_CSI1RDI_BCR 17 +#define GCC_CAMSS_CSI_VFE0_BCR 18 +#define GCC_CAMSS_GP0_BCR 19 +#define GCC_CAMSS_GP1_BCR 20 +#define GCC_CAMSS_ISPIF_BCR 21 +#define GCC_CAMSS_MCLK0_BCR 22 +#define GCC_CAMSS_MCLK1_BCR 23 +#define GCC_CAMSS_PHY0_BCR 24 +#define GCC_CAMSS_TOP_BCR 25 +#define GCC_CAMSS_TOP_AHB_BCR 26 +#define GCC_CAMSS_VFE_BCR 27 +#define GCC_CRYPTO_BCR 28 +#define GCC_MDSS_BCR 29 +#define GCC_OXILI_BCR 30 +#define GCC_PDM_BCR 31 +#define GCC_PRNG_BCR 32 +#define GCC_QUSB2_PHY_BCR 33 +#define GCC_SDCC1_BCR 34 +#define GCC_SDCC2_BCR 35 +#define GCC_ULT_AUDIO_BCR 36 +#define GCC_USB2A_PHY_BCR 37 +#define GCC_USB2_HS_PHY_ONLY_BCR 38 +#define GCC_USB_HS_BCR 39 +#define GCC_VENUS0_BCR 40 + +/* Subsystem Restart */ +#define GCC_MSS_RESTART 41 + +/* Power Domains */ +#define MDSS_GDSC 0 +#define OXILI_GDSC 1 +#define VENUS_GDSC 2 +#define VENUS_CORE0_GDSC 3 +#define VFE_GDSC 4 + +#endif diff --git a/include/dt-bindings/clock/qcom,gcc-sdm845.h b/include/dt-bindings/clock/qcom,gcc-sdm845.h index 968fa65b9c4289ff730fcb9c4461b70af936963c..d78b899263a2d93968d61b631b4620658509317f 100644 --- a/include/dt-bindings/clock/qcom,gcc-sdm845.h +++ b/include/dt-bindings/clock/qcom,gcc-sdm845.h @@ -199,6 +199,7 @@ #define GCC_QSPI_CNOC_PERIPH_AHB_CLK 189 #define GCC_LPASS_Q6_AXI_CLK 190 #define GCC_LPASS_SWAY_CLK 191 +#define GPLL6 192 /* GCC Resets */ #define GCC_MMSS_BCR 0 diff --git a/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h b/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h new file mode 100644 index 0000000000000000000000000000000000000000..bb7da46333b0cfb30e9f49616412258f0770d973 --- /dev/null +++ b/include/dt-bindings/clock/qcom,gpucc-sc8280xp.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GPU_CC_SC8280XP_H +#define _DT_BINDINGS_CLK_QCOM_GPU_CC_SC8280XP_H + +/* GPU_CC clocks */ +#define GPU_CC_PLL0 0 +#define GPU_CC_PLL1 1 +#define GPU_CC_AHB_CLK 2 +#define GPU_CC_CB_CLK 3 +#define GPU_CC_CRC_AHB_CLK 4 +#define GPU_CC_CX_GMU_CLK 5 +#define GPU_CC_CX_SNOC_DVM_CLK 6 +#define GPU_CC_CXO_AON_CLK 7 +#define GPU_CC_CXO_CLK 8 +#define GPU_CC_FREQ_MEASURE_CLK 9 +#define GPU_CC_GMU_CLK_SRC 10 +#define GPU_CC_GX_GMU_CLK 11 +#define GPU_CC_GX_VSENSE_CLK 12 +#define GPU_CC_HUB_AHB_DIV_CLK_SRC 13 +#define GPU_CC_HUB_AON_CLK 14 +#define GPU_CC_HUB_CLK_SRC 15 +#define GPU_CC_HUB_CX_INT_CLK 16 +#define GPU_CC_HUB_CX_INT_DIV_CLK_SRC 17 +#define GPU_CC_SLEEP_CLK 18 +#define GPU_CC_HLOS1_VOTE_GPU_SMMU_CLK 19 + +/* GPU_CC power domains */ +#define GPU_CC_CX_GDSC 0 +#define GPU_CC_GX_GDSC 1 + +#endif diff --git a/include/dt-bindings/clock/qcom,lcc-ipq806x.h b/include/dt-bindings/clock/qcom,lcc-ipq806x.h index 25b92bbf0ab4d4f11fce86c8ff3e6705f584e048..e0fb4acf4ba8cca61dfc9a1efea5fd955ebdfe35 100644 --- a/include/dt-bindings/clock/qcom,lcc-ipq806x.h +++ b/include/dt-bindings/clock/qcom,lcc-ipq806x.h @@ -19,4 +19,6 @@ #define SPDIF_CLK 10 #define AHBIX_CLK 11 +#define LCC_PCM_RESET 0 + #endif diff --git a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h index 20ef2ea673f3b47eff21b89aed9c437a0e1b0f74..22dcd47d451399f53c33a162ccec84994e93241a 100644 --- a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h +++ b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h @@ -24,6 +24,11 @@ #define LPASS_AUDIO_CC_RX_MCLK_CLK 14 #define LPASS_AUDIO_CC_RX_MCLK_CLK_SRC 15 +/* LPASS AUDIO CC CSR */ +#define LPASS_AUDIO_SWR_RX_CGCR 0 +#define LPASS_AUDIO_SWR_TX_CGCR 1 +#define LPASS_AUDIO_SWR_WSA_CGCR 2 + /* LPASS_AON_CC clocks */ #define LPASS_AON_CC_PLL 0 #define LPASS_AON_CC_PLL_OUT_EVEN 1 diff --git a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h index 28ed2a07aacc37aec74abec371ffdae4cbffb74c..0324c69ce96869811cf2e4dffd1ece18b7bc16e4 100644 --- a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h +++ b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h @@ -19,6 +19,8 @@ #define LPASS_CORE_CC_LPM_CORE_CLK 9 #define LPASS_CORE_CC_LPM_MEM0_CORE_CLK 10 #define LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK 11 +#define LPASS_CORE_CC_EXT_MCLK0_CLK 12 +#define LPASS_CORE_CC_EXT_MCLK0_CLK_SRC 13 /* LPASS_CORE_CC power domains */ #define LPASS_CORE_CC_LPASS_CORE_HM_GDSC 0 diff --git a/include/dt-bindings/clock/qcom,rpmcc.h b/include/dt-bindings/clock/qcom,rpmcc.h index 015db95303d19f264f9375e3f84eaee9d27e48af..c0ad624e930e92089fc77103530ddd2245d818ff 100644 --- a/include/dt-bindings/clock/qcom,rpmcc.h +++ b/include/dt-bindings/clock/qcom,rpmcc.h @@ -167,5 +167,6 @@ #define RPM_SMD_CPUSS_GNOC_A_CLK 121 #define RPM_SMD_MSS_CFG_AHB_CLK 122 #define RPM_SMD_MSS_CFG_AHB_A_CLK 123 +#define RPM_SMD_BIMC_FREQ_LOG 124 #endif diff --git a/include/dt-bindings/clock/qcom,sm6115-dispcc.h b/include/dt-bindings/clock/qcom,sm6115-dispcc.h new file mode 100644 index 0000000000000000000000000000000000000000..d1a6c45b50290313bf2bc03d593a2948ab5deded --- /dev/null +++ b/include/dt-bindings/clock/qcom,sm6115-dispcc.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2022, The Linux Foundation. All rights reserved. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H +#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM6115_H + +/* DISP_CC clocks */ +#define DISP_CC_PLL0 0 +#define DISP_CC_PLL0_OUT_MAIN 1 +#define DISP_CC_MDSS_AHB_CLK 2 +#define DISP_CC_MDSS_AHB_CLK_SRC 3 +#define DISP_CC_MDSS_BYTE0_CLK 4 +#define DISP_CC_MDSS_BYTE0_CLK_SRC 5 +#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC 6 +#define DISP_CC_MDSS_BYTE0_INTF_CLK 7 +#define DISP_CC_MDSS_ESC0_CLK 8 +#define DISP_CC_MDSS_ESC0_CLK_SRC 9 +#define DISP_CC_MDSS_MDP_CLK 10 +#define DISP_CC_MDSS_MDP_CLK_SRC 11 +#define DISP_CC_MDSS_MDP_LUT_CLK 12 +#define DISP_CC_MDSS_NON_GDSC_AHB_CLK 13 +#define DISP_CC_MDSS_PCLK0_CLK 14 +#define DISP_CC_MDSS_PCLK0_CLK_SRC 15 +#define DISP_CC_MDSS_ROT_CLK 16 +#define DISP_CC_MDSS_ROT_CLK_SRC 17 +#define DISP_CC_MDSS_VSYNC_CLK 18 +#define DISP_CC_MDSS_VSYNC_CLK_SRC 19 +#define DISP_CC_SLEEP_CLK 20 +#define DISP_CC_SLEEP_CLK_SRC 21 + +/* DISP_CC GDSCR */ +#define MDSS_GDSC 0 + +#endif diff --git a/include/dt-bindings/clock/qcom,sm6375-gcc.h b/include/dt-bindings/clock/qcom,sm6375-gcc.h new file mode 100644 index 0000000000000000000000000000000000000000..1e9801e1cedfe236915368ee2e07fa0a8b0ac20d --- /dev/null +++ b/include/dt-bindings/clock/qcom,sm6375-gcc.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, The Linux Foundation. All rights reserved. + * Copyright (c) 2022, Konrad Dybcio + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_GCC_SM6375_H +#define _DT_BINDINGS_CLK_QCOM_GCC_SM6375_H + +/* Clocks */ +#define GPLL0 0 +#define GPLL0_OUT_EVEN 1 +#define GPLL0_OUT_ODD 2 +#define GPLL1 3 +#define GPLL10 4 +#define GPLL11 5 +#define GPLL3 6 +#define GPLL3_OUT_EVEN 7 +#define GPLL4 8 +#define GPLL5 9 +#define GPLL6 10 +#define GPLL6_OUT_EVEN 11 +#define GPLL7 12 +#define GPLL8 13 +#define GPLL8_OUT_EVEN 14 +#define GPLL9 15 +#define GPLL9_OUT_MAIN 16 +#define GCC_AHB2PHY_CSI_CLK 17 +#define GCC_AHB2PHY_USB_CLK 18 +#define GCC_BIMC_GPU_AXI_CLK 19 +#define GCC_BOOT_ROM_AHB_CLK 20 +#define GCC_CAM_THROTTLE_NRT_CLK 21 +#define GCC_CAM_THROTTLE_RT_CLK 22 +#define GCC_CAMERA_AHB_CLK 23 +#define GCC_CAMERA_XO_CLK 24 +#define GCC_CAMSS_AXI_CLK 25 +#define GCC_CAMSS_AXI_CLK_SRC 26 +#define GCC_CAMSS_CAMNOC_ATB_CLK 27 +#define GCC_CAMSS_CAMNOC_NTS_XO_CLK 28 +#define GCC_CAMSS_CCI_0_CLK 29 +#define GCC_CAMSS_CCI_0_CLK_SRC 30 +#define GCC_CAMSS_CCI_1_CLK 31 +#define GCC_CAMSS_CCI_1_CLK_SRC 32 +#define GCC_CAMSS_CPHY_0_CLK 33 +#define GCC_CAMSS_CPHY_1_CLK 34 +#define GCC_CAMSS_CPHY_2_CLK 35 +#define GCC_CAMSS_CPHY_3_CLK 36 +#define GCC_CAMSS_CSI0PHYTIMER_CLK 37 +#define GCC_CAMSS_CSI0PHYTIMER_CLK_SRC 38 +#define GCC_CAMSS_CSI1PHYTIMER_CLK 39 +#define GCC_CAMSS_CSI1PHYTIMER_CLK_SRC 40 +#define GCC_CAMSS_CSI2PHYTIMER_CLK 41 +#define GCC_CAMSS_CSI2PHYTIMER_CLK_SRC 42 +#define GCC_CAMSS_CSI3PHYTIMER_CLK 43 +#define GCC_CAMSS_CSI3PHYTIMER_CLK_SRC 44 +#define GCC_CAMSS_MCLK0_CLK 45 +#define GCC_CAMSS_MCLK0_CLK_SRC 46 +#define GCC_CAMSS_MCLK1_CLK 47 +#define GCC_CAMSS_MCLK1_CLK_SRC 48 +#define GCC_CAMSS_MCLK2_CLK 49 +#define GCC_CAMSS_MCLK2_CLK_SRC 50 +#define GCC_CAMSS_MCLK3_CLK 51 +#define GCC_CAMSS_MCLK3_CLK_SRC 52 +#define GCC_CAMSS_MCLK4_CLK 53 +#define GCC_CAMSS_MCLK4_CLK_SRC 54 +#define GCC_CAMSS_NRT_AXI_CLK 55 +#define GCC_CAMSS_OPE_AHB_CLK 56 +#define GCC_CAMSS_OPE_AHB_CLK_SRC 57 +#define GCC_CAMSS_OPE_CLK 58 +#define GCC_CAMSS_OPE_CLK_SRC 59 +#define GCC_CAMSS_RT_AXI_CLK 60 +#define GCC_CAMSS_TFE_0_CLK 61 +#define GCC_CAMSS_TFE_0_CLK_SRC 62 +#define GCC_CAMSS_TFE_0_CPHY_RX_CLK 63 +#define GCC_CAMSS_TFE_0_CSID_CLK 64 +#define GCC_CAMSS_TFE_0_CSID_CLK_SRC 65 +#define GCC_CAMSS_TFE_1_CLK 66 +#define GCC_CAMSS_TFE_1_CLK_SRC 67 +#define GCC_CAMSS_TFE_1_CPHY_RX_CLK 68 +#define GCC_CAMSS_TFE_1_CSID_CLK 69 +#define GCC_CAMSS_TFE_1_CSID_CLK_SRC 70 +#define GCC_CAMSS_TFE_2_CLK 71 +#define GCC_CAMSS_TFE_2_CLK_SRC 72 +#define GCC_CAMSS_TFE_2_CPHY_RX_CLK 73 +#define GCC_CAMSS_TFE_2_CSID_CLK 74 +#define GCC_CAMSS_TFE_2_CSID_CLK_SRC 75 +#define GCC_CAMSS_TFE_CPHY_RX_CLK_SRC 76 +#define GCC_CAMSS_TOP_AHB_CLK 77 +#define GCC_CAMSS_TOP_AHB_CLK_SRC 78 +#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK 79 +#define GCC_CPUSS_AHB_CLK_SRC 80 +#define GCC_CPUSS_AHB_POSTDIV_CLK_SRC 81 +#define GCC_CPUSS_GNOC_CLK 82 +#define GCC_DISP_AHB_CLK 83 +#define GCC_DISP_GPLL0_CLK_SRC 84 +#define GCC_DISP_GPLL0_DIV_CLK_SRC 85 +#define GCC_DISP_HF_AXI_CLK 86 +#define GCC_DISP_SLEEP_CLK 87 +#define GCC_DISP_THROTTLE_CORE_CLK 88 +#define GCC_DISP_XO_CLK 89 +#define GCC_GP1_CLK 90 +#define GCC_GP1_CLK_SRC 91 +#define GCC_GP2_CLK 92 +#define GCC_GP2_CLK_SRC 93 +#define GCC_GP3_CLK 94 +#define GCC_GP3_CLK_SRC 95 +#define GCC_GPU_CFG_AHB_CLK 96 +#define GCC_GPU_GPLL0_CLK_SRC 97 +#define GCC_GPU_GPLL0_DIV_CLK_SRC 98 +#define GCC_GPU_MEMNOC_GFX_CLK 99 +#define GCC_GPU_SNOC_DVM_GFX_CLK 100 +#define GCC_GPU_THROTTLE_CORE_CLK 101 +#define GCC_PDM2_CLK 102 +#define GCC_PDM2_CLK_SRC 103 +#define GCC_PDM_AHB_CLK 104 +#define GCC_PDM_XO4_CLK 105 +#define GCC_PRNG_AHB_CLK 106 +#define GCC_QMIP_CAMERA_NRT_AHB_CLK 107 +#define GCC_QMIP_CAMERA_RT_AHB_CLK 108 +#define GCC_QMIP_DISP_AHB_CLK 109 +#define GCC_QMIP_GPU_CFG_AHB_CLK 110 +#define GCC_QMIP_VIDEO_VCODEC_AHB_CLK 111 +#define GCC_QUPV3_WRAP0_CORE_2X_CLK 112 +#define GCC_QUPV3_WRAP0_CORE_CLK 113 +#define GCC_QUPV3_WRAP0_S0_CLK 114 +#define GCC_QUPV3_WRAP0_S0_CLK_SRC 115 +#define GCC_QUPV3_WRAP0_S1_CLK 116 +#define GCC_QUPV3_WRAP0_S1_CLK_SRC 117 +#define GCC_QUPV3_WRAP0_S2_CLK 118 +#define GCC_QUPV3_WRAP0_S2_CLK_SRC 119 +#define GCC_QUPV3_WRAP0_S3_CLK 120 +#define GCC_QUPV3_WRAP0_S3_CLK_SRC 121 +#define GCC_QUPV3_WRAP0_S4_CLK 122 +#define GCC_QUPV3_WRAP0_S4_CLK_SRC 123 +#define GCC_QUPV3_WRAP0_S5_CLK 124 +#define GCC_QUPV3_WRAP0_S5_CLK_SRC 125 +#define GCC_QUPV3_WRAP1_CORE_2X_CLK 126 +#define GCC_QUPV3_WRAP1_CORE_CLK 127 +#define GCC_QUPV3_WRAP1_S0_CLK 128 +#define GCC_QUPV3_WRAP1_S0_CLK_SRC 129 +#define GCC_QUPV3_WRAP1_S1_CLK 130 +#define GCC_QUPV3_WRAP1_S1_CLK_SRC 131 +#define GCC_QUPV3_WRAP1_S2_CLK 132 +#define GCC_QUPV3_WRAP1_S2_CLK_SRC 133 +#define GCC_QUPV3_WRAP1_S3_CLK 134 +#define GCC_QUPV3_WRAP1_S3_CLK_SRC 135 +#define GCC_QUPV3_WRAP1_S4_CLK 136 +#define GCC_QUPV3_WRAP1_S4_CLK_SRC 137 +#define GCC_QUPV3_WRAP1_S5_CLK 138 +#define GCC_QUPV3_WRAP1_S5_CLK_SRC 139 +#define GCC_QUPV3_WRAP_0_M_AHB_CLK 140 +#define GCC_QUPV3_WRAP_0_S_AHB_CLK 141 +#define GCC_QUPV3_WRAP_1_M_AHB_CLK 142 +#define GCC_QUPV3_WRAP_1_S_AHB_CLK 143 +#define GCC_RX5_PCIE_CLKREF_EN_CLK 144 +#define GCC_SDCC1_AHB_CLK 145 +#define GCC_SDCC1_APPS_CLK 146 +#define GCC_SDCC1_APPS_CLK_SRC 147 +#define GCC_SDCC1_ICE_CORE_CLK 148 +#define GCC_SDCC1_ICE_CORE_CLK_SRC 149 +#define GCC_SDCC2_AHB_CLK 150 +#define GCC_SDCC2_APPS_CLK 151 +#define GCC_SDCC2_APPS_CLK_SRC 152 +#define GCC_SYS_NOC_CPUSS_AHB_CLK 153 +#define GCC_SYS_NOC_UFS_PHY_AXI_CLK 154 +#define GCC_SYS_NOC_USB3_PRIM_AXI_CLK 155 +#define GCC_UFS_MEM_CLKREF_CLK 156 +#define GCC_UFS_PHY_AHB_CLK 157 +#define GCC_UFS_PHY_AXI_CLK 158 +#define GCC_UFS_PHY_AXI_CLK_SRC 159 +#define GCC_UFS_PHY_ICE_CORE_CLK 160 +#define GCC_UFS_PHY_ICE_CORE_CLK_SRC 161 +#define GCC_UFS_PHY_PHY_AUX_CLK 162 +#define GCC_UFS_PHY_PHY_AUX_CLK_SRC 163 +#define GCC_UFS_PHY_RX_SYMBOL_0_CLK 164 +#define GCC_UFS_PHY_TX_SYMBOL_0_CLK 165 +#define GCC_UFS_PHY_UNIPRO_CORE_CLK 166 +#define GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC 167 +#define GCC_USB30_PRIM_MASTER_CLK 168 +#define GCC_USB30_PRIM_MASTER_CLK_SRC 169 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK 170 +#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC 171 +#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC 172 +#define GCC_USB30_PRIM_SLEEP_CLK 173 +#define GCC_USB3_PRIM_CLKREF_CLK 174 +#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC 175 +#define GCC_USB3_PRIM_PHY_COM_AUX_CLK 176 +#define GCC_USB3_PRIM_PHY_PIPE_CLK 177 +#define GCC_VCODEC0_AXI_CLK 178 +#define GCC_VENUS_AHB_CLK 179 +#define GCC_VENUS_CTL_AXI_CLK 180 +#define GCC_VIDEO_AHB_CLK 181 +#define GCC_VIDEO_AXI0_CLK 182 +#define GCC_VIDEO_THROTTLE_CORE_CLK 183 +#define GCC_VIDEO_VCODEC0_SYS_CLK 184 +#define GCC_VIDEO_VENUS_CLK_SRC 185 +#define GCC_VIDEO_VENUS_CTL_CLK 186 +#define GCC_VIDEO_XO_CLK 187 + +/* Resets */ +#define GCC_CAMSS_OPE_BCR 0 +#define GCC_CAMSS_TFE_BCR 1 +#define GCC_CAMSS_TOP_BCR 2 +#define GCC_GPU_BCR 3 +#define GCC_MMSS_BCR 4 +#define GCC_PDM_BCR 5 +#define GCC_PRNG_BCR 6 +#define GCC_QUPV3_WRAPPER_0_BCR 7 +#define GCC_QUPV3_WRAPPER_1_BCR 8 +#define GCC_QUSB2PHY_PRIM_BCR 9 +#define GCC_QUSB2PHY_SEC_BCR 10 +#define GCC_SDCC1_BCR 11 +#define GCC_SDCC2_BCR 12 +#define GCC_UFS_PHY_BCR 13 +#define GCC_USB30_PRIM_BCR 14 +#define GCC_USB_PHY_CFG_AHB2PHY_BCR 15 +#define GCC_VCODEC0_BCR 16 +#define GCC_VENUS_BCR 17 +#define GCC_VIDEO_INTERFACE_BCR 18 +#define GCC_USB3_DP_PHY_PRIM_BCR 19 +#define GCC_USB3_PHY_PRIM_SP0_BCR 20 + +/* GDSCs */ +#define USB30_PRIM_GDSC 0 +#define UFS_PHY_GDSC 1 +#define CAMSS_TOP_GDSC 2 +#define VENUS_GDSC 3 +#define VCODEC0_GDSC 4 +#define HLOS1_VOTE_MM_SNOC_MMU_TBU_NRT_GDSC 5 +#define HLOS1_VOTE_MM_SNOC_MMU_TBU_RT_GDSC 6 +#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC 7 +#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC 8 + +#endif diff --git a/include/dt-bindings/clock/qcom,sm8450-dispcc.h b/include/dt-bindings/clock/qcom,sm8450-dispcc.h new file mode 100644 index 0000000000000000000000000000000000000000..fd16ca894971487a3621a15c5f0ef7a93bd72fff --- /dev/null +++ b/include/dt-bindings/clock/qcom,sm8450-dispcc.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2022, The Linux Foundation. All rights reserved. + */ + +#ifndef _DT_BINDINGS_CLK_QCOM_DISP_CC_SM8450_H +#define _DT_BINDINGS_CLK_QCOM_DISP_CC_SM8450_H + +/* DISP_CC clocks */ +#define DISP_CC_MDSS_AHB1_CLK 0 +#define DISP_CC_MDSS_AHB_CLK 1 +#define DISP_CC_MDSS_AHB_CLK_SRC 2 +#define DISP_CC_MDSS_BYTE0_CLK 3 +#define DISP_CC_MDSS_BYTE0_CLK_SRC 4 +#define DISP_CC_MDSS_BYTE0_DIV_CLK_SRC 5 +#define DISP_CC_MDSS_BYTE0_INTF_CLK 6 +#define DISP_CC_MDSS_BYTE1_CLK 7 +#define DISP_CC_MDSS_BYTE1_CLK_SRC 8 +#define DISP_CC_MDSS_BYTE1_DIV_CLK_SRC 9 +#define DISP_CC_MDSS_BYTE1_INTF_CLK 10 +#define DISP_CC_MDSS_DPTX0_AUX_CLK 11 +#define DISP_CC_MDSS_DPTX0_AUX_CLK_SRC 12 +#define DISP_CC_MDSS_DPTX0_CRYPTO_CLK 13 +#define DISP_CC_MDSS_DPTX0_LINK_CLK 14 +#define DISP_CC_MDSS_DPTX0_LINK_CLK_SRC 15 +#define DISP_CC_MDSS_DPTX0_LINK_DIV_CLK_SRC 16 +#define DISP_CC_MDSS_DPTX0_LINK_INTF_CLK 17 +#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK 18 +#define DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC 19 +#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK 20 +#define DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC 21 +#define DISP_CC_MDSS_DPTX0_USB_ROUTER_LINK_INTF_CLK 22 +#define DISP_CC_MDSS_DPTX1_AUX_CLK 23 +#define DISP_CC_MDSS_DPTX1_AUX_CLK_SRC 24 +#define DISP_CC_MDSS_DPTX1_CRYPTO_CLK 25 +#define DISP_CC_MDSS_DPTX1_LINK_CLK 26 +#define DISP_CC_MDSS_DPTX1_LINK_CLK_SRC 27 +#define DISP_CC_MDSS_DPTX1_LINK_DIV_CLK_SRC 28 +#define DISP_CC_MDSS_DPTX1_LINK_INTF_CLK 29 +#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK 30 +#define DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC 31 +#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK 32 +#define DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC 33 +#define DISP_CC_MDSS_DPTX1_USB_ROUTER_LINK_INTF_CLK 34 +#define DISP_CC_MDSS_DPTX2_AUX_CLK 35 +#define DISP_CC_MDSS_DPTX2_AUX_CLK_SRC 36 +#define DISP_CC_MDSS_DPTX2_CRYPTO_CLK 37 +#define DISP_CC_MDSS_DPTX2_LINK_CLK 38 +#define DISP_CC_MDSS_DPTX2_LINK_CLK_SRC 39 +#define DISP_CC_MDSS_DPTX2_LINK_DIV_CLK_SRC 40 +#define DISP_CC_MDSS_DPTX2_LINK_INTF_CLK 41 +#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK 42 +#define DISP_CC_MDSS_DPTX2_PIXEL0_CLK_SRC 43 +#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK 44 +#define DISP_CC_MDSS_DPTX2_PIXEL1_CLK_SRC 45 +#define DISP_CC_MDSS_DPTX3_AUX_CLK 46 +#define DISP_CC_MDSS_DPTX3_AUX_CLK_SRC 47 +#define DISP_CC_MDSS_DPTX3_CRYPTO_CLK 48 +#define DISP_CC_MDSS_DPTX3_LINK_CLK 49 +#define DISP_CC_MDSS_DPTX3_LINK_CLK_SRC 50 +#define DISP_CC_MDSS_DPTX3_LINK_DIV_CLK_SRC 51 +#define DISP_CC_MDSS_DPTX3_LINK_INTF_CLK 52 +#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK 53 +#define DISP_CC_MDSS_DPTX3_PIXEL0_CLK_SRC 54 +#define DISP_CC_MDSS_ESC0_CLK 55 +#define DISP_CC_MDSS_ESC0_CLK_SRC 56 +#define DISP_CC_MDSS_ESC1_CLK 57 +#define DISP_CC_MDSS_ESC1_CLK_SRC 58 +#define DISP_CC_MDSS_MDP1_CLK 59 +#define DISP_CC_MDSS_MDP_CLK 60 +#define DISP_CC_MDSS_MDP_CLK_SRC 61 +#define DISP_CC_MDSS_MDP_LUT1_CLK 62 +#define DISP_CC_MDSS_MDP_LUT_CLK 63 +#define DISP_CC_MDSS_NON_GDSC_AHB_CLK 64 +#define DISP_CC_MDSS_PCLK0_CLK 65 +#define DISP_CC_MDSS_PCLK0_CLK_SRC 66 +#define DISP_CC_MDSS_PCLK1_CLK 67 +#define DISP_CC_MDSS_PCLK1_CLK_SRC 68 +#define DISP_CC_MDSS_ROT1_CLK 69 +#define DISP_CC_MDSS_ROT_CLK 70 +#define DISP_CC_MDSS_ROT_CLK_SRC 71 +#define DISP_CC_MDSS_RSCC_AHB_CLK 72 +#define DISP_CC_MDSS_RSCC_VSYNC_CLK 73 +#define DISP_CC_MDSS_VSYNC1_CLK 74 +#define DISP_CC_MDSS_VSYNC_CLK 75 +#define DISP_CC_MDSS_VSYNC_CLK_SRC 76 +#define DISP_CC_PLL0 77 +#define DISP_CC_PLL1 78 +#define DISP_CC_SLEEP_CLK 79 +#define DISP_CC_SLEEP_CLK_SRC 80 +#define DISP_CC_XO_CLK 81 +#define DISP_CC_XO_CLK_SRC 82 + +/* DISP_CC resets */ +#define DISP_CC_MDSS_CORE_BCR 0 +#define DISP_CC_MDSS_CORE_INT2_BCR 1 +#define DISP_CC_MDSS_RSCC_BCR 2 + +/* DISP_CC GDSCR */ +#define MDSS_GDSC 0 +#define MDSS_INT2_GDSC 1 + +#endif diff --git a/include/dt-bindings/clock/rockchip,rv1126-cru.h b/include/dt-bindings/clock/rockchip,rv1126-cru.h new file mode 100644 index 0000000000000000000000000000000000000000..e89a3a5a4a34a37ba1eefbeef8ca527c5ea34d53 --- /dev/null +++ b/include/dt-bindings/clock/rockchip,rv1126-cru.h @@ -0,0 +1,632 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Copyright (c) 2019 Rockchip Electronics Co. Ltd. + * Author: Finley Xiao + */ + +#ifndef _DT_BINDINGS_CLK_ROCKCHIP_RV1126_H +#define _DT_BINDINGS_CLK_ROCKCHIP_RV1126_H + +/* pmucru-clocks indices */ + +/* pll clocks */ +#define PLL_GPLL 1 + +/* sclk (special clocks) */ +#define CLK_OSC0_DIV32K 2 +#define CLK_RTC32K 3 +#define CLK_WIFI_DIV 4 +#define CLK_WIFI_OSC0 5 +#define CLK_WIFI 6 +#define CLK_PMU 7 +#define SCLK_UART1_DIV 8 +#define SCLK_UART1_FRACDIV 9 +#define SCLK_UART1_MUX 10 +#define SCLK_UART1 11 +#define CLK_I2C0 12 +#define CLK_I2C2 13 +#define CLK_CAPTURE_PWM0 14 +#define CLK_PWM0 15 +#define CLK_CAPTURE_PWM1 16 +#define CLK_PWM1 17 +#define CLK_SPI0 18 +#define DBCLK_GPIO0 19 +#define CLK_PMUPVTM 20 +#define CLK_CORE_PMUPVTM 21 +#define CLK_REF12M 22 +#define CLK_USBPHY_OTG_REF 23 +#define CLK_USBPHY_HOST_REF 24 +#define CLK_REF24M 25 +#define CLK_MIPIDSIPHY_REF 26 + +/* pclk */ +#define PCLK_PDPMU 30 +#define PCLK_PMU 31 +#define PCLK_UART1 32 +#define PCLK_I2C0 33 +#define PCLK_I2C2 34 +#define PCLK_PWM0 35 +#define PCLK_PWM1 36 +#define PCLK_SPI0 37 +#define PCLK_GPIO0 38 +#define PCLK_PMUSGRF 39 +#define PCLK_PMUGRF 40 +#define PCLK_PMUCRU 41 +#define PCLK_CHIPVEROTP 42 +#define PCLK_PDPMU_NIU 43 +#define PCLK_PMUPVTM 44 +#define PCLK_SCRKEYGEN 45 + +#define CLKPMU_NR_CLKS (PCLK_SCRKEYGEN + 1) + +/* cru-clocks indices */ + +/* pll clocks */ +#define PLL_APLL 1 +#define PLL_DPLL 2 +#define PLL_CPLL 3 +#define PLL_HPLL 4 + +/* sclk (special clocks) */ +#define ARMCLK 5 +#define USB480M 6 +#define CLK_CORE_CPUPVTM 7 +#define CLK_CPUPVTM 8 +#define CLK_SCR1 9 +#define CLK_SCR1_CORE 10 +#define CLK_SCR1_RTC 11 +#define CLK_SCR1_JTAG 12 +#define SCLK_UART0_DIV 13 +#define SCLK_UART0_FRAC 14 +#define SCLK_UART0_MUX 15 +#define SCLK_UART0 16 +#define SCLK_UART2_DIV 17 +#define SCLK_UART2_FRAC 18 +#define SCLK_UART2_MUX 19 +#define SCLK_UART2 20 +#define SCLK_UART3_DIV 21 +#define SCLK_UART3_FRAC 22 +#define SCLK_UART3_MUX 23 +#define SCLK_UART3 24 +#define SCLK_UART4_DIV 25 +#define SCLK_UART4_FRAC 26 +#define SCLK_UART4_MUX 27 +#define SCLK_UART4 28 +#define SCLK_UART5_DIV 29 +#define SCLK_UART5_FRAC 30 +#define SCLK_UART5_MUX 31 +#define SCLK_UART5 32 +#define CLK_I2C1 33 +#define CLK_I2C3 34 +#define CLK_I2C4 35 +#define CLK_I2C5 36 +#define CLK_SPI1 37 +#define CLK_CAPTURE_PWM2 38 +#define CLK_PWM2 39 +#define DBCLK_GPIO1 40 +#define DBCLK_GPIO2 41 +#define DBCLK_GPIO3 42 +#define DBCLK_GPIO4 43 +#define CLK_SARADC 44 +#define CLK_TIMER0 45 +#define CLK_TIMER1 46 +#define CLK_TIMER2 47 +#define CLK_TIMER3 48 +#define CLK_TIMER4 49 +#define CLK_TIMER5 50 +#define CLK_CAN 51 +#define CLK_NPU_TSADC 52 +#define CLK_NPU_TSADCPHY 53 +#define CLK_CPU_TSADC 54 +#define CLK_CPU_TSADCPHY 55 +#define CLK_CRYPTO_CORE 56 +#define CLK_CRYPTO_PKA 57 +#define MCLK_I2S0_TX_DIV 58 +#define MCLK_I2S0_TX_FRACDIV 59 +#define MCLK_I2S0_TX_MUX 60 +#define MCLK_I2S0_TX 61 +#define MCLK_I2S0_RX_DIV 62 +#define MCLK_I2S0_RX_FRACDIV 63 +#define MCLK_I2S0_RX_MUX 64 +#define MCLK_I2S0_RX 65 +#define MCLK_I2S0_TX_OUT2IO 66 +#define MCLK_I2S0_RX_OUT2IO 67 +#define MCLK_I2S1_DIV 68 +#define MCLK_I2S1_FRACDIV 69 +#define MCLK_I2S1_MUX 70 +#define MCLK_I2S1 71 +#define MCLK_I2S1_OUT2IO 72 +#define MCLK_I2S2_DIV 73 +#define MCLK_I2S2_FRACDIV 74 +#define MCLK_I2S2_MUX 75 +#define MCLK_I2S2 76 +#define MCLK_I2S2_OUT2IO 77 +#define MCLK_PDM 78 +#define SCLK_ADUPWM_DIV 79 +#define SCLK_AUDPWM_FRACDIV 80 +#define SCLK_AUDPWM_MUX 81 +#define SCLK_AUDPWM 82 +#define CLK_ACDCDIG_ADC 83 +#define CLK_ACDCDIG_DAC 84 +#define CLK_ACDCDIG_I2C 85 +#define CLK_VENC_CORE 86 +#define CLK_VDEC_CORE 87 +#define CLK_VDEC_CA 88 +#define CLK_VDEC_HEVC_CA 89 +#define CLK_RGA_CORE 90 +#define CLK_IEP_CORE 91 +#define CLK_ISP_DIV 92 +#define CLK_ISP_NP5 93 +#define CLK_ISP_NUX 94 +#define CLK_ISP 95 +#define CLK_CIF_OUT_DIV 96 +#define CLK_CIF_OUT_FRACDIV 97 +#define CLK_CIF_OUT_MUX 98 +#define CLK_CIF_OUT 99 +#define CLK_MIPICSI_OUT_DIV 100 +#define CLK_MIPICSI_OUT_FRACDIV 101 +#define CLK_MIPICSI_OUT_MUX 102 +#define CLK_MIPICSI_OUT 103 +#define CLK_ISPP_DIV 104 +#define CLK_ISPP_NP5 105 +#define CLK_ISPP_NUX 106 +#define CLK_ISPP 107 +#define CLK_SDMMC 108 +#define SCLK_SDMMC_DRV 109 +#define SCLK_SDMMC_SAMPLE 110 +#define CLK_SDIO 111 +#define SCLK_SDIO_DRV 112 +#define SCLK_SDIO_SAMPLE 113 +#define CLK_EMMC 114 +#define SCLK_EMMC_DRV 115 +#define SCLK_EMMC_SAMPLE 116 +#define CLK_NANDC 117 +#define SCLK_SFC 118 +#define CLK_USBHOST_UTMI_OHCI 119 +#define CLK_USBOTG_REF 120 +#define CLK_GMAC_DIV 121 +#define CLK_GMAC_RGMII_M0 122 +#define CLK_GMAC_SRC_M0 123 +#define CLK_GMAC_RGMII_M1 124 +#define CLK_GMAC_SRC_M1 125 +#define CLK_GMAC_SRC 126 +#define CLK_GMAC_REF 127 +#define CLK_GMAC_TX_SRC 128 +#define CLK_GMAC_TX_DIV5 129 +#define CLK_GMAC_TX_DIV50 130 +#define RGMII_MODE_CLK 131 +#define CLK_GMAC_RX_SRC 132 +#define CLK_GMAC_RX_DIV2 133 +#define CLK_GMAC_RX_DIV20 134 +#define RMII_MODE_CLK 135 +#define CLK_GMAC_TX_RX 136 +#define CLK_GMAC_PTPREF 137 +#define CLK_GMAC_ETHERNET_OUT 138 +#define CLK_DDRPHY 139 +#define CLK_DDR_MON 140 +#define TMCLK_DDR_MON 141 +#define CLK_NPU_DIV 142 +#define CLK_NPU_NP5 143 +#define CLK_CORE_NPU 144 +#define CLK_CORE_NPUPVTM 145 +#define CLK_NPUPVTM 146 +#define SCLK_DDRCLK 147 +#define CLK_OTP 148 + +/* dclk */ +#define DCLK_DECOM 150 +#define DCLK_VOP_DIV 151 +#define DCLK_VOP_FRACDIV 152 +#define DCLK_VOP_MUX 153 +#define DCLK_VOP 154 +#define DCLK_CIF 155 +#define DCLK_CIFLITE 156 + +/* aclk */ +#define ACLK_PDBUS 160 +#define ACLK_DMAC 161 +#define ACLK_DCF 162 +#define ACLK_SPINLOCK 163 +#define ACLK_DECOM 164 +#define ACLK_PDCRYPTO 165 +#define ACLK_CRYPTO 166 +#define ACLK_PDVEPU 167 +#define ACLK_VENC 168 +#define ACLK_PDVDEC 169 +#define ACLK_PDJPEG 170 +#define ACLK_VDEC 171 +#define ACLK_JPEG 172 +#define ACLK_PDVO 173 +#define ACLK_RGA 174 +#define ACLK_VOP 175 +#define ACLK_IEP 176 +#define ACLK_PDVI_DIV 177 +#define ACLK_PDVI_NP5 178 +#define ACLK_PDVI 179 +#define ACLK_ISP 180 +#define ACLK_CIF 181 +#define ACLK_CIFLITE 182 +#define ACLK_PDISPP_DIV 183 +#define ACLK_PDISPP_NP5 184 +#define ACLK_PDISPP 185 +#define ACLK_ISPP 186 +#define ACLK_PDPHP 187 +#define ACLK_PDUSB 188 +#define ACLK_USBOTG 189 +#define ACLK_PDGMAC 190 +#define ACLK_GMAC 191 +#define ACLK_PDNPU_DIV 192 +#define ACLK_PDNPU_NP5 193 +#define ACLK_PDNPU 194 +#define ACLK_NPU 195 + +/* hclk */ +#define HCLK_PDCORE_NIU 200 +#define HCLK_PDUSB 201 +#define HCLK_PDCRYPTO 202 +#define HCLK_CRYPTO 203 +#define HCLK_PDAUDIO 204 +#define HCLK_I2S0 205 +#define HCLK_I2S1 206 +#define HCLK_I2S2 207 +#define HCLK_PDM 208 +#define HCLK_AUDPWM 209 +#define HCLK_PDVEPU 210 +#define HCLK_VENC 211 +#define HCLK_PDVDEC 212 +#define HCLK_PDJPEG 213 +#define HCLK_VDEC 214 +#define HCLK_JPEG 215 +#define HCLK_PDVO 216 +#define HCLK_RGA 217 +#define HCLK_VOP 218 +#define HCLK_IEP 219 +#define HCLK_PDVI 220 +#define HCLK_ISP 221 +#define HCLK_CIF 222 +#define HCLK_CIFLITE 223 +#define HCLK_PDISPP 224 +#define HCLK_ISPP 225 +#define HCLK_PDPHP 226 +#define HCLK_PDSDMMC 227 +#define HCLK_SDMMC 228 +#define HCLK_PDSDIO 229 +#define HCLK_SDIO 230 +#define HCLK_PDNVM 231 +#define HCLK_EMMC 232 +#define HCLK_NANDC 233 +#define HCLK_SFC 234 +#define HCLK_SFCXIP 235 +#define HCLK_PDBUS 236 +#define HCLK_USBHOST 237 +#define HCLK_USBHOST_ARB 238 +#define HCLK_PDNPU 239 +#define HCLK_NPU 240 + +/* pclk */ +#define PCLK_CPUPVTM 245 +#define PCLK_PDBUS 246 +#define PCLK_DCF 247 +#define PCLK_WDT 248 +#define PCLK_MAILBOX 249 +#define PCLK_UART0 250 +#define PCLK_UART2 251 +#define PCLK_UART3 252 +#define PCLK_UART4 253 +#define PCLK_UART5 254 +#define PCLK_I2C1 255 +#define PCLK_I2C3 256 +#define PCLK_I2C4 257 +#define PCLK_I2C5 258 +#define PCLK_SPI1 259 +#define PCLK_PWM2 261 +#define PCLK_GPIO1 262 +#define PCLK_GPIO2 263 +#define PCLK_GPIO3 264 +#define PCLK_GPIO4 265 +#define PCLK_SARADC 266 +#define PCLK_TIMER 267 +#define PCLK_DECOM 268 +#define PCLK_CAN 269 +#define PCLK_NPU_TSADC 270 +#define PCLK_CPU_TSADC 271 +#define PCLK_ACDCDIG 272 +#define PCLK_PDVO 273 +#define PCLK_DSIHOST 274 +#define PCLK_PDVI 275 +#define PCLK_CSIHOST 276 +#define PCLK_PDGMAC 277 +#define PCLK_GMAC 278 +#define PCLK_PDDDR 279 +#define PCLK_DDR_MON 280 +#define PCLK_PDNPU 281 +#define PCLK_NPUPVTM 282 +#define PCLK_PDTOP 283 +#define PCLK_TOPCRU 284 +#define PCLK_TOPGRF 285 +#define PCLK_CPUEMADET 286 +#define PCLK_DDRPHY 287 +#define PCLK_DSIPHY 289 +#define PCLK_CSIPHY0 290 +#define PCLK_CSIPHY1 291 +#define PCLK_USBPHY_HOST 292 +#define PCLK_USBPHY_OTG 293 +#define PCLK_OTP 294 + +#define CLK_NR_CLKS (PCLK_OTP + 1) + +/* pmu soft-reset indices */ + +/* pmu_cru_softrst_con0 */ +#define SRST_PDPMU_NIU_P 0 +#define SRST_PMU_SGRF_P 1 +#define SRST_PMU_SGRF_REMAP_P 2 +#define SRST_I2C0_P 3 +#define SRST_I2C0 4 +#define SRST_I2C2_P 7 +#define SRST_I2C2 8 +#define SRST_UART1_P 9 +#define SRST_UART1 10 +#define SRST_PWM0_P 11 +#define SRST_PWM0 12 +#define SRST_PWM1_P 13 +#define SRST_PWM1 14 +#define SRST_DDR_FAIL_SAFE 15 + +/* pmu_cru_softrst_con1 */ +#define SRST_GPIO0_P 17 +#define SRST_GPIO0_DB 18 +#define SRST_SPI0_P 19 +#define SRST_SPI0 20 +#define SRST_PMUGRF_P 21 +#define SRST_CHIPVEROTP_P 22 +#define SRST_PMUPVTM 24 +#define SRST_PMUPVTM_P 25 +#define SRST_PMUCRU_P 30 + +/* soft-reset indices */ + +/* cru_softrst_con0 */ +#define SRST_CORE0_PO 0 +#define SRST_CORE1_PO 1 +#define SRST_CORE2_PO 2 +#define SRST_CORE3_PO 3 +#define SRST_CORE0 4 +#define SRST_CORE1 5 +#define SRST_CORE2 6 +#define SRST_CORE3 7 +#define SRST_CORE0_DBG 8 +#define SRST_CORE1_DBG 9 +#define SRST_CORE2_DBG 10 +#define SRST_CORE3_DBG 11 +#define SRST_NL2 12 +#define SRST_CORE_NIU_A 13 +#define SRST_DBG_DAPLITE_P 14 +#define SRST_DAPLITE_P 15 + +/* cru_softrst_con1 */ +#define SRST_PDBUS_NIU1_A 16 +#define SRST_PDBUS_NIU1_H 17 +#define SRST_PDBUS_NIU1_P 18 +#define SRST_PDBUS_NIU2_A 19 +#define SRST_PDBUS_NIU2_H 20 +#define SRST_PDBUS_NIU3_A 21 +#define SRST_PDBUS_NIU3_H 22 +#define SRST_PDBUS_HOLD_NIU1_A 23 +#define SRST_DBG_NIU_P 24 +#define SRST_PDCORE_NIIU_H 25 +#define SRST_MUC_NIU 26 +#define SRST_DCF_A 29 +#define SRST_DCF_P 30 +#define SRST_SYSTEM_SRAM_A 31 + +/* cru_softrst_con2 */ +#define SRST_I2C1_P 32 +#define SRST_I2C1 33 +#define SRST_I2C3_P 34 +#define SRST_I2C3 35 +#define SRST_I2C4_P 36 +#define SRST_I2C4 37 +#define SRST_I2C5_P 38 +#define SRST_I2C5 39 +#define SRST_SPI1_P 40 +#define SRST_SPI1 41 +#define SRST_MCU_CORE 42 +#define SRST_PWM2_P 44 +#define SRST_PWM2 45 +#define SRST_SPINLOCK_A 46 + +/* cru_softrst_con3 */ +#define SRST_UART0_P 48 +#define SRST_UART0 49 +#define SRST_UART2_P 50 +#define SRST_UART2 51 +#define SRST_UART3_P 52 +#define SRST_UART3 53 +#define SRST_UART4_P 54 +#define SRST_UART4 55 +#define SRST_UART5_P 56 +#define SRST_UART5 57 +#define SRST_WDT_P 58 +#define SRST_SARADC_P 59 +#define SRST_GRF_P 61 +#define SRST_TIMER_P 62 +#define SRST_MAILBOX_P 63 + +/* cru_softrst_con4 */ +#define SRST_TIMER0 64 +#define SRST_TIMER1 65 +#define SRST_TIMER2 66 +#define SRST_TIMER3 67 +#define SRST_TIMER4 68 +#define SRST_TIMER5 69 +#define SRST_INTMUX_P 70 +#define SRST_GPIO1_P 72 +#define SRST_GPIO1_DB 73 +#define SRST_GPIO2_P 74 +#define SRST_GPIO2_DB 75 +#define SRST_GPIO3_P 76 +#define SRST_GPIO3_DB 77 +#define SRST_GPIO4_P 78 +#define SRST_GPIO4_DB 79 + +/* cru_softrst_con5 */ +#define SRST_CAN_P 80 +#define SRST_CAN 81 +#define SRST_DECOM_A 85 +#define SRST_DECOM_P 86 +#define SRST_DECOM_D 87 +#define SRST_PDCRYPTO_NIU_A 88 +#define SRST_PDCRYPTO_NIU_H 89 +#define SRST_CRYPTO_A 90 +#define SRST_CRYPTO_H 91 +#define SRST_CRYPTO_CORE 92 +#define SRST_CRYPTO_PKA 93 +#define SRST_SGRF_P 95 + +/* cru_softrst_con6 */ +#define SRST_PDAUDIO_NIU_H 96 +#define SRST_PDAUDIO_NIU_P 97 +#define SRST_I2S0_H 98 +#define SRST_I2S0_TX_M 99 +#define SRST_I2S0_RX_M 100 +#define SRST_I2S1_H 101 +#define SRST_I2S1_M 102 +#define SRST_I2S2_H 103 +#define SRST_I2S2_M 104 +#define SRST_PDM_H 105 +#define SRST_PDM_M 106 +#define SRST_AUDPWM_H 107 +#define SRST_AUDPWM 108 +#define SRST_ACDCDIG_P 109 +#define SRST_ACDCDIG 110 + +/* cru_softrst_con7 */ +#define SRST_PDVEPU_NIU_A 112 +#define SRST_PDVEPU_NIU_H 113 +#define SRST_VENC_A 114 +#define SRST_VENC_H 115 +#define SRST_VENC_CORE 116 +#define SRST_PDVDEC_NIU_A 117 +#define SRST_PDVDEC_NIU_H 118 +#define SRST_VDEC_A 119 +#define SRST_VDEC_H 120 +#define SRST_VDEC_CORE 121 +#define SRST_VDEC_CA 122 +#define SRST_VDEC_HEVC_CA 123 +#define SRST_PDJPEG_NIU_A 124 +#define SRST_PDJPEG_NIU_H 125 +#define SRST_JPEG_A 126 +#define SRST_JPEG_H 127 + +/* cru_softrst_con8 */ +#define SRST_PDVO_NIU_A 128 +#define SRST_PDVO_NIU_H 129 +#define SRST_PDVO_NIU_P 130 +#define SRST_RGA_A 131 +#define SRST_RGA_H 132 +#define SRST_RGA_CORE 133 +#define SRST_VOP_A 134 +#define SRST_VOP_H 135 +#define SRST_VOP_D 136 +#define SRST_TXBYTEHS_DSIHOST 137 +#define SRST_DSIHOST_P 138 +#define SRST_IEP_A 139 +#define SRST_IEP_H 140 +#define SRST_IEP_CORE 141 +#define SRST_ISP_RX_P 142 + +/* cru_softrst_con9 */ +#define SRST_PDVI_NIU_A 144 +#define SRST_PDVI_NIU_H 145 +#define SRST_PDVI_NIU_P 146 +#define SRST_ISP 147 +#define SRST_CIF_A 148 +#define SRST_CIF_H 149 +#define SRST_CIF_D 150 +#define SRST_CIF_P 151 +#define SRST_CIF_I 152 +#define SRST_CIF_RX_P 153 +#define SRST_PDISPP_NIU_A 154 +#define SRST_PDISPP_NIU_H 155 +#define SRST_ISPP_A 156 +#define SRST_ISPP_H 157 +#define SRST_ISPP 158 +#define SRST_CSIHOST_P 159 + +/* cru_softrst_con10 */ +#define SRST_PDPHPMID_NIU_A 160 +#define SRST_PDPHPMID_NIU_H 161 +#define SRST_PDNVM_NIU_H 163 +#define SRST_SDMMC_H 164 +#define SRST_SDIO_H 165 +#define SRST_EMMC_H 166 +#define SRST_SFC_H 167 +#define SRST_SFCXIP_H 168 +#define SRST_SFC 169 +#define SRST_NANDC_H 170 +#define SRST_NANDC 171 +#define SRST_PDSDMMC_H 173 +#define SRST_PDSDIO_H 174 + +/* cru_softrst_con11 */ +#define SRST_PDUSB_NIU_A 176 +#define SRST_PDUSB_NIU_H 177 +#define SRST_USBHOST_H 178 +#define SRST_USBHOST_ARB_H 179 +#define SRST_USBHOST_UTMI 180 +#define SRST_USBOTG_A 181 +#define SRST_USBPHY_OTG_P 182 +#define SRST_USBPHY_HOST_P 183 +#define SRST_USBPHYPOR_OTG 184 +#define SRST_USBPHYPOR_HOST 185 +#define SRST_PDGMAC_NIU_A 188 +#define SRST_PDGMAC_NIU_P 189 +#define SRST_GMAC_A 190 + +/* cru_softrst_con12 */ +#define SRST_DDR_DFICTL_P 193 +#define SRST_DDR_MON_P 194 +#define SRST_DDR_STANDBY_P 195 +#define SRST_DDR_GRF_P 196 +#define SRST_DDR_MSCH_P 197 +#define SRST_DDR_SPLIT_A 198 +#define SRST_DDR_MSCH 199 +#define SRST_DDR_DFICTL 202 +#define SRST_DDR_STANDBY 203 +#define SRST_NPUMCU_NIU 205 +#define SRST_DDRPHY_P 206 +#define SRST_DDRPHY 207 + +/* cru_softrst_con13 */ +#define SRST_PDNPU_NIU_A 208 +#define SRST_PDNPU_NIU_H 209 +#define SRST_PDNPU_NIU_P 210 +#define SRST_NPU_A 211 +#define SRST_NPU_H 212 +#define SRST_NPU 213 +#define SRST_NPUPVTM_P 214 +#define SRST_NPUPVTM 215 +#define SRST_NPU_TSADC_P 216 +#define SRST_NPU_TSADC 217 +#define SRST_NPU_TSADCPHY 218 +#define SRST_CIFLITE_A 220 +#define SRST_CIFLITE_H 221 +#define SRST_CIFLITE_D 222 +#define SRST_CIFLITE_RX_P 223 + +/* cru_softrst_con14 */ +#define SRST_TOPNIU_P 224 +#define SRST_TOPCRU_P 225 +#define SRST_TOPGRF_P 226 +#define SRST_CPUEMADET_P 227 +#define SRST_CSIPHY0_P 228 +#define SRST_CSIPHY1_P 229 +#define SRST_DSIPHY_P 230 +#define SRST_CPU_TSADC_P 232 +#define SRST_CPU_TSADC 233 +#define SRST_CPU_TSADCPHY 234 +#define SRST_CPUPVTM_P 235 +#define SRST_CPUPVTM 236 + +#endif diff --git a/include/dt-bindings/clock/samsung,exynosautov9.h b/include/dt-bindings/clock/samsung,exynosautov9.h index ea9f91b4eb1a3586580f22ec074fd1fab045a1a4..42133af6d6b914a1397c35a4d779b16fb8914a5d 100644 --- a/include/dt-bindings/clock/samsung,exynosautov9.h +++ b/include/dt-bindings/clock/samsung,exynosautov9.h @@ -185,6 +185,74 @@ #define CORE_NR_CLK 6 +/* CMU_FSYS0 */ +#define CLK_MOUT_FSYS0_BUS_USER 1 +#define CLK_MOUT_FSYS0_PCIE_USER 2 +#define CLK_GOUT_FSYS0_BUS_PCLK 3 + +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_REFCLK 4 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_REFCLK 5 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_DBI_ACLK 6 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_MSTR_ACLK 7 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_SLV_ACLK 8 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_DBI_ACLK 9 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_MSTR_ACLK 10 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_SLV_ACLK 11 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_PIPE_CLK 12 +#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L0_CLK 13 +#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L0_CLK 14 + +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_REFCLK 15 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_REFCLK 16 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_DBI_ACLK 17 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_MSTR_ACLK 18 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_SLV_ACLK 19 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_DBI_ACLK 20 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_MSTR_ACLK 21 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_SLV_ACLK 22 +#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_PIPE_CLK 23 +#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L1_CLK 24 +#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L1_CLK 25 + +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_REFCLK 26 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_REFCLK 27 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_DBI_ACLK 28 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_MSTR_ACLK 29 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_SLV_ACLK 30 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_DBI_ACLK 31 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_MSTR_ACLK 32 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_SLV_ACLK 33 +#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_PIPE_CLK 34 +#define CLK_GOUT_FSYS0_PCIE_GEN3A_4L_CLK 35 +#define CLK_GOUT_FSYS0_PCIE_GEN3B_4L_CLK 36 + +#define FSYS0_NR_CLK 37 + +/* CMU_FSYS1 */ +#define FOUT_MMC_PLL 1 + +#define CLK_MOUT_FSYS1_BUS_USER 2 +#define CLK_MOUT_FSYS1_MMC_PLL 3 +#define CLK_MOUT_FSYS1_MMC_CARD_USER 4 +#define CLK_MOUT_FSYS1_USBDRD_USER 5 +#define CLK_MOUT_FSYS1_MMC_CARD 6 + +#define CLK_DOUT_FSYS1_MMC_CARD 7 + +#define CLK_GOUT_FSYS1_PCLK 8 +#define CLK_GOUT_FSYS1_MMC_CARD_SDCLKIN 9 +#define CLK_GOUT_FSYS1_MMC_CARD_ACLK 10 +#define CLK_GOUT_FSYS1_USB20DRD_0_REFCLK 11 +#define CLK_GOUT_FSYS1_USB20DRD_1_REFCLK 12 +#define CLK_GOUT_FSYS1_USB30DRD_0_REFCLK 13 +#define CLK_GOUT_FSYS1_USB30DRD_1_REFCLK 14 +#define CLK_GOUT_FSYS1_USB20_0_ACLK 15 +#define CLK_GOUT_FSYS1_USB20_1_ACLK 16 +#define CLK_GOUT_FSYS1_USB30_0_ACLK 17 +#define CLK_GOUT_FSYS1_USB30_1_ACLK 18 + +#define FSYS1_NR_CLK 19 + /* CMU_FSYS2 */ #define CLK_MOUT_FSYS2_BUS_USER 1 #define CLK_MOUT_FSYS2_UFS_EMBD_USER 2 @@ -226,21 +294,21 @@ #define CLK_GOUT_PERIC0_IPCLK_8 28 #define CLK_GOUT_PERIC0_IPCLK_9 29 #define CLK_GOUT_PERIC0_IPCLK_10 30 -#define CLK_GOUT_PERIC0_IPCLK_11 30 -#define CLK_GOUT_PERIC0_PCLK_0 31 -#define CLK_GOUT_PERIC0_PCLK_1 32 -#define CLK_GOUT_PERIC0_PCLK_2 33 -#define CLK_GOUT_PERIC0_PCLK_3 34 -#define CLK_GOUT_PERIC0_PCLK_4 35 -#define CLK_GOUT_PERIC0_PCLK_5 36 -#define CLK_GOUT_PERIC0_PCLK_6 37 -#define CLK_GOUT_PERIC0_PCLK_7 38 -#define CLK_GOUT_PERIC0_PCLK_8 39 -#define CLK_GOUT_PERIC0_PCLK_9 40 -#define CLK_GOUT_PERIC0_PCLK_10 41 -#define CLK_GOUT_PERIC0_PCLK_11 42 - -#define PERIC0_NR_CLK 43 +#define CLK_GOUT_PERIC0_IPCLK_11 31 +#define CLK_GOUT_PERIC0_PCLK_0 32 +#define CLK_GOUT_PERIC0_PCLK_1 33 +#define CLK_GOUT_PERIC0_PCLK_2 34 +#define CLK_GOUT_PERIC0_PCLK_3 35 +#define CLK_GOUT_PERIC0_PCLK_4 36 +#define CLK_GOUT_PERIC0_PCLK_5 37 +#define CLK_GOUT_PERIC0_PCLK_6 38 +#define CLK_GOUT_PERIC0_PCLK_7 39 +#define CLK_GOUT_PERIC0_PCLK_8 40 +#define CLK_GOUT_PERIC0_PCLK_9 41 +#define CLK_GOUT_PERIC0_PCLK_10 42 +#define CLK_GOUT_PERIC0_PCLK_11 43 + +#define PERIC0_NR_CLK 44 /* CMU_PERIC1 */ #define CLK_MOUT_PERIC1_BUS_USER 1 @@ -272,21 +340,21 @@ #define CLK_GOUT_PERIC1_IPCLK_8 28 #define CLK_GOUT_PERIC1_IPCLK_9 29 #define CLK_GOUT_PERIC1_IPCLK_10 30 -#define CLK_GOUT_PERIC1_IPCLK_11 30 -#define CLK_GOUT_PERIC1_PCLK_0 31 -#define CLK_GOUT_PERIC1_PCLK_1 32 -#define CLK_GOUT_PERIC1_PCLK_2 33 -#define CLK_GOUT_PERIC1_PCLK_3 34 -#define CLK_GOUT_PERIC1_PCLK_4 35 -#define CLK_GOUT_PERIC1_PCLK_5 36 -#define CLK_GOUT_PERIC1_PCLK_6 37 -#define CLK_GOUT_PERIC1_PCLK_7 38 -#define CLK_GOUT_PERIC1_PCLK_8 39 -#define CLK_GOUT_PERIC1_PCLK_9 40 -#define CLK_GOUT_PERIC1_PCLK_10 41 -#define CLK_GOUT_PERIC1_PCLK_11 42 - -#define PERIC1_NR_CLK 43 +#define CLK_GOUT_PERIC1_IPCLK_11 31 +#define CLK_GOUT_PERIC1_PCLK_0 32 +#define CLK_GOUT_PERIC1_PCLK_1 33 +#define CLK_GOUT_PERIC1_PCLK_2 34 +#define CLK_GOUT_PERIC1_PCLK_3 35 +#define CLK_GOUT_PERIC1_PCLK_4 36 +#define CLK_GOUT_PERIC1_PCLK_5 37 +#define CLK_GOUT_PERIC1_PCLK_6 38 +#define CLK_GOUT_PERIC1_PCLK_7 39 +#define CLK_GOUT_PERIC1_PCLK_8 40 +#define CLK_GOUT_PERIC1_PCLK_9 41 +#define CLK_GOUT_PERIC1_PCLK_10 42 +#define CLK_GOUT_PERIC1_PCLK_11 43 + +#define PERIC1_NR_CLK 44 /* CMU_PERIS */ #define CLK_MOUT_PERIS_BUS_USER 1 diff --git a/include/dt-bindings/clk/versaclock.h b/include/dt-bindings/clock/versaclock.h similarity index 100% rename from include/dt-bindings/clk/versaclock.h rename to include/dt-bindings/clock/versaclock.h diff --git a/include/dt-bindings/firmware/imx/rsrc.h b/include/dt-bindings/firmware/imx/rsrc.h index 43885056557c4583614b90edc026e8e635d2f6c5..1675de05ad33b941746dc25c00c97304a6156618 100644 --- a/include/dt-bindings/firmware/imx/rsrc.h +++ b/include/dt-bindings/firmware/imx/rsrc.h @@ -37,10 +37,14 @@ #define IMX_SC_R_DC_0_BLIT2 21 #define IMX_SC_R_DC_0_BLIT_OUT 22 #define IMX_SC_R_PERF 23 +#define IMX_SC_R_USB_1_PHY 24 #define IMX_SC_R_DC_0_WARP 25 +#define IMX_SC_R_V2X_MU_0 26 +#define IMX_SC_R_V2X_MU_1 27 #define IMX_SC_R_DC_0_VIDEO0 28 #define IMX_SC_R_DC_0_VIDEO1 29 #define IMX_SC_R_DC_0_FRAC0 30 +#define IMX_SC_R_V2X_MU_2 31 #define IMX_SC_R_DC_0 32 #define IMX_SC_R_GPU_2_PID0 33 #define IMX_SC_R_DC_0_PLL_0 34 @@ -49,7 +53,10 @@ #define IMX_SC_R_DC_1_BLIT1 37 #define IMX_SC_R_DC_1_BLIT2 38 #define IMX_SC_R_DC_1_BLIT_OUT 39 +#define IMX_SC_R_V2X_MU_3 40 +#define IMX_SC_R_V2X_MU_4 41 #define IMX_SC_R_DC_1_WARP 42 +#define IMX_SC_R_SECVIO 44 #define IMX_SC_R_DC_1_VIDEO0 45 #define IMX_SC_R_DC_1_VIDEO1 46 #define IMX_SC_R_DC_1_FRAC0 47 diff --git a/include/dt-bindings/iio/adc/at91-sama5d2_adc.h b/include/dt-bindings/iio/adc/at91-sama5d2_adc.h index 70f99dbdbb42a03c8936a92bbcfc6dcdeb7f12ef..866d36530583292587ee1eda5e900a84b9a428d3 100644 --- a/include/dt-bindings/iio/adc/at91-sama5d2_adc.h +++ b/include/dt-bindings/iio/adc/at91-sama5d2_adc.h @@ -13,4 +13,7 @@ /* pressure channel index */ #define AT91_SAMA5D2_ADC_P_CHANNEL 26 +/* SAMA7G5 Temperature sensor channel index. */ +#define AT91_SAMA7G5_ADC_TEMP_CHANNEL 31 + #endif diff --git a/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h b/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h new file mode 100644 index 0000000000000000000000000000000000000000..6ee7255477632ac544e2043a4f2ce94683fffb29 --- /dev/null +++ b/include/dt-bindings/iio/adc/mediatek,mt6370_adc.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */ + +#ifndef __DT_BINDINGS_MEDIATEK_MT6370_ADC_H__ +#define __DT_BINDINGS_MEDIATEK_MT6370_ADC_H__ + +/* ADC Channel Index */ +#define MT6370_CHAN_VBUSDIV5 0 +#define MT6370_CHAN_VBUSDIV2 1 +#define MT6370_CHAN_VSYS 2 +#define MT6370_CHAN_VBAT 3 +#define MT6370_CHAN_TS_BAT 4 +#define MT6370_CHAN_IBUS 5 +#define MT6370_CHAN_IBAT 6 +#define MT6370_CHAN_CHG_VDDP 7 +#define MT6370_CHAN_TEMP_JC 8 +#define MT6370_CHAN_MAX 9 + +#endif diff --git a/include/dt-bindings/interrupt-controller/irqc-rzg2l.h b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h new file mode 100644 index 0000000000000000000000000000000000000000..34ce778885a12836c319618622a7f9d51e2d9c6b --- /dev/null +++ b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * This header provides constants for Renesas RZ/G2L family IRQC bindings. + * + * Copyright (C) 2022 Renesas Electronics Corp. + * + */ + +#ifndef __DT_BINDINGS_IRQC_RZG2L_H +#define __DT_BINDINGS_IRQC_RZG2L_H + +/* NMI maps to SPI0 */ +#define RZG2L_NMI 0 + +/* IRQ0-7 map to SPI1-8 */ +#define RZG2L_IRQ0 1 +#define RZG2L_IRQ1 2 +#define RZG2L_IRQ2 3 +#define RZG2L_IRQ3 4 +#define RZG2L_IRQ4 5 +#define RZG2L_IRQ5 6 +#define RZG2L_IRQ6 7 +#define RZG2L_IRQ7 8 + +#endif /* __DT_BINDINGS_IRQC_RZG2L_H */ diff --git a/include/dt-bindings/leds/common.h b/include/dt-bindings/leds/common.h index 3be89a7c20a9a45695e65759d01d96191f2f15f1..9a0d33d027fff4ee696d917bb0a8d59b98b2080f 100644 --- a/include/dt-bindings/leds/common.h +++ b/include/dt-bindings/leds/common.h @@ -33,7 +33,12 @@ #define LED_COLOR_ID_MULTI 8 /* For multicolor LEDs */ #define LED_COLOR_ID_RGB 9 /* For multicolor LEDs that can do arbitrary color, so this would include RGBW and similar */ -#define LED_COLOR_ID_MAX 10 +#define LED_COLOR_ID_PURPLE 10 +#define LED_COLOR_ID_ORANGE 11 +#define LED_COLOR_ID_PINK 12 +#define LED_COLOR_ID_CYAN 13 +#define LED_COLOR_ID_LIME 14 +#define LED_COLOR_ID_MAX 15 /* Standard LED functions */ /* Keyboard LEDs, usually it would be input4::capslock etc. */ diff --git a/include/dt-bindings/memory/mt6795-larb-port.h b/include/dt-bindings/memory/mt6795-larb-port.h new file mode 100644 index 0000000000000000000000000000000000000000..58cf6a6b637254c0f9d852e700243ab6474d9a47 --- /dev/null +++ b/include/dt-bindings/memory/mt6795-larb-port.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#ifndef _DT_BINDINGS_MEMORY_MT6795_LARB_PORT_H_ +#define _DT_BINDINGS_MEMORY_MT6795_LARB_PORT_H_ + +#include + +#define M4U_LARB0_ID 0 +#define M4U_LARB1_ID 1 +#define M4U_LARB2_ID 2 +#define M4U_LARB3_ID 3 +#define M4U_LARB4_ID 4 + +/* larb0 */ +#define M4U_PORT_DISP_OVL0 MTK_M4U_ID(M4U_LARB0_ID, 0) +#define M4U_PORT_DISP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 1) +#define M4U_PORT_DISP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 2) +#define M4U_PORT_DISP_WDMA0 MTK_M4U_ID(M4U_LARB0_ID, 3) +#define M4U_PORT_DISP_OVL1 MTK_M4U_ID(M4U_LARB0_ID, 4) +#define M4U_PORT_DISP_RDMA2 MTK_M4U_ID(M4U_LARB0_ID, 5) +#define M4U_PORT_DISP_WDMA1 MTK_M4U_ID(M4U_LARB0_ID, 6) +#define M4U_PORT_DISP_OD_R MTK_M4U_ID(M4U_LARB0_ID, 7) +#define M4U_PORT_DISP_OD_W MTK_M4U_ID(M4U_LARB0_ID, 8) +#define M4U_PORT_MDP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 9) +#define M4U_PORT_MDP_RDMA1 MTK_M4U_ID(M4U_LARB0_ID, 10) +#define M4U_PORT_MDP_WDMA MTK_M4U_ID(M4U_LARB0_ID, 11) +#define M4U_PORT_MDP_WROT0 MTK_M4U_ID(M4U_LARB0_ID, 12) +#define M4U_PORT_MDP_WROT1 MTK_M4U_ID(M4U_LARB0_ID, 13) + +/* larb1 */ +#define M4U_PORT_VDEC_MC MTK_M4U_ID(M4U_LARB1_ID, 0) +#define M4U_PORT_VDEC_PP MTK_M4U_ID(M4U_LARB1_ID, 1) +#define M4U_PORT_VDEC_UFO MTK_M4U_ID(M4U_LARB1_ID, 2) +#define M4U_PORT_VDEC_VLD MTK_M4U_ID(M4U_LARB1_ID, 3) +#define M4U_PORT_VDEC_VLD2 MTK_M4U_ID(M4U_LARB1_ID, 4) +#define M4U_PORT_VDEC_AVC_MV MTK_M4U_ID(M4U_LARB1_ID, 5) +#define M4U_PORT_VDEC_PRED_RD MTK_M4U_ID(M4U_LARB1_ID, 6) +#define M4U_PORT_VDEC_PRED_WR MTK_M4U_ID(M4U_LARB1_ID, 7) +#define M4U_PORT_VDEC_PPWRAP MTK_M4U_ID(M4U_LARB1_ID, 8) + +/* larb2 */ +#define M4U_PORT_CAM_IMGO MTK_M4U_ID(M4U_LARB2_ID, 0) +#define M4U_PORT_CAM_RRZO MTK_M4U_ID(M4U_LARB2_ID, 1) +#define M4U_PORT_CAM_AAO MTK_M4U_ID(M4U_LARB2_ID, 2) +#define M4U_PORT_CAM_LCSO MTK_M4U_ID(M4U_LARB2_ID, 3) +#define M4U_PORT_CAM_ESFKO MTK_M4U_ID(M4U_LARB2_ID, 4) +#define M4U_PORT_CAM_IMGO_S MTK_M4U_ID(M4U_LARB2_ID, 5) +#define M4U_PORT_CAM_LSCI MTK_M4U_ID(M4U_LARB2_ID, 6) +#define M4U_PORT_CAM_LSCI_D MTK_M4U_ID(M4U_LARB2_ID, 7) +#define M4U_PORT_CAM_BPCI MTK_M4U_ID(M4U_LARB2_ID, 8) +#define M4U_PORT_CAM_BPCI_D MTK_M4U_ID(M4U_LARB2_ID, 9) +#define M4U_PORT_CAM_UFDI MTK_M4U_ID(M4U_LARB2_ID, 10) +#define M4U_PORT_CAM_IMGI MTK_M4U_ID(M4U_LARB2_ID, 11) +#define M4U_PORT_CAM_IMG2O MTK_M4U_ID(M4U_LARB2_ID, 12) +#define M4U_PORT_CAM_IMG3O MTK_M4U_ID(M4U_LARB2_ID, 13) +#define M4U_PORT_CAM_VIPI MTK_M4U_ID(M4U_LARB2_ID, 14) +#define M4U_PORT_CAM_VIP2I MTK_M4U_ID(M4U_LARB2_ID, 15) +#define M4U_PORT_CAM_VIP3I MTK_M4U_ID(M4U_LARB2_ID, 16) +#define M4U_PORT_CAM_LCEI MTK_M4U_ID(M4U_LARB2_ID, 17) +#define M4U_PORT_CAM_RB MTK_M4U_ID(M4U_LARB2_ID, 18) +#define M4U_PORT_CAM_RP MTK_M4U_ID(M4U_LARB2_ID, 19) +#define M4U_PORT_CAM_WR MTK_M4U_ID(M4U_LARB2_ID, 20) + +/* larb3 */ +#define M4U_PORT_VENC_RCPU MTK_M4U_ID(M4U_LARB3_ID, 0) +#define M4U_PORT_VENC_REC MTK_M4U_ID(M4U_LARB3_ID, 1) +#define M4U_PORT_VENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 2) +#define M4U_PORT_VENC_SV_COMV MTK_M4U_ID(M4U_LARB3_ID, 3) +#define M4U_PORT_VENC_RD_COMV MTK_M4U_ID(M4U_LARB3_ID, 4) +#define M4U_PORT_JPGENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 5) +#define M4U_PORT_REMDC_SDMA MTK_M4U_ID(M4U_LARB3_ID, 6) +#define M4U_PORT_REMDC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 7) +#define M4U_PORT_JPGENC_RDMA MTK_M4U_ID(M4U_LARB3_ID, 8) +#define M4U_PORT_JPGENC_SDMA MTK_M4U_ID(M4U_LARB3_ID, 9) +#define M4U_PORT_JPGDEC_WDMA MTK_M4U_ID(M4U_LARB3_ID, 10) +#define M4U_PORT_JPGDEC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 11) +#define M4U_PORT_VENC_CUR_LUMA MTK_M4U_ID(M4U_LARB3_ID, 12) +#define M4U_PORT_VENC_CUR_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 13) +#define M4U_PORT_VENC_REF_LUMA MTK_M4U_ID(M4U_LARB3_ID, 14) +#define M4U_PORT_VENC_REF_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 15) +#define M4U_PORT_REMDC_WDMA MTK_M4U_ID(M4U_LARB3_ID, 16) +#define M4U_PORT_VENC_NBM_RDMA MTK_M4U_ID(M4U_LARB3_ID, 17) +#define M4U_PORT_VENC_NBM_WDMA MTK_M4U_ID(M4U_LARB3_ID, 18) + +/* larb4 */ +#define M4U_PORT_MJC_MV_RD MTK_M4U_ID(M4U_LARB4_ID, 0) +#define M4U_PORT_MJC_MV_WR MTK_M4U_ID(M4U_LARB4_ID, 1) +#define M4U_PORT_MJC_DMA_RD MTK_M4U_ID(M4U_LARB4_ID, 2) +#define M4U_PORT_MJC_DMA_WR MTK_M4U_ID(M4U_LARB4_ID, 3) + +#endif diff --git a/include/dt-bindings/memory/tegra234-mc.h b/include/dt-bindings/memory/tegra234-mc.h index 62987b47ce81c150847b4ddf60bc36713b7b8182..bd71cc1d7990f4724aec216c725b4b3871cecdfe 100644 --- a/include/dt-bindings/memory/tegra234-mc.h +++ b/include/dt-bindings/memory/tegra234-mc.h @@ -34,6 +34,16 @@ #define TEGRA234_SID_HOST1X 0x27 #define TEGRA234_SID_VIC 0x34 +/* Shared stream IDs */ +#define TEGRA234_SID_HOST1X_CTX0 0x35 +#define TEGRA234_SID_HOST1X_CTX1 0x36 +#define TEGRA234_SID_HOST1X_CTX2 0x37 +#define TEGRA234_SID_HOST1X_CTX3 0x38 +#define TEGRA234_SID_HOST1X_CTX4 0x39 +#define TEGRA234_SID_HOST1X_CTX5 0x3a +#define TEGRA234_SID_HOST1X_CTX6 0x3b +#define TEGRA234_SID_HOST1X_CTX7 0x3c + /* * memory client IDs */ diff --git a/include/dt-bindings/phy/phy.h b/include/dt-bindings/phy/phy.h index f48c9acf251eb82e354634102c3e4a8542d7bca2..6b901b342348641868335857bbc1aa3f3ae3d817 100644 --- a/include/dt-bindings/phy/phy.h +++ b/include/dt-bindings/phy/phy.h @@ -22,5 +22,6 @@ #define PHY_TYPE_QSGMII 9 #define PHY_TYPE_DPHY 10 #define PHY_TYPE_CPHY 11 +#define PHY_TYPE_USXGMII 12 #endif /* _DT_BINDINGS_PHY */ diff --git a/include/dt-bindings/pinctrl/k3.h b/include/dt-bindings/pinctrl/k3.h index a5204ab91d3ec63d01eeeef33961533cd7048ccf..54df633f9bfefd670ece385b826d76ee6d1afd3a 100644 --- a/include/dt-bindings/pinctrl/k3.h +++ b/include/dt-bindings/pinctrl/k3.h @@ -29,19 +29,22 @@ #define PIN_INPUT_PULLUP (INPUT_EN | PULL_UP) #define PIN_INPUT_PULLDOWN (INPUT_EN | PULL_DOWN) +#define AM62AX_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) +#define AM62AX_MCU_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) + +#define AM62X_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) +#define AM62X_MCU_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) + +#define AM64X_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) +#define AM64X_MCU_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) + #define AM65X_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) #define AM65X_WKUP_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) #define J721E_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) #define J721E_WKUP_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) -#define AM64X_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) -#define AM64X_MCU_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) - #define J721S2_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) #define J721S2_WKUP_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) -#define AM62X_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) -#define AM62X_MCU_IOPAD(pa, val, muxmode) (((pa) & 0x1fff)) ((val) | (muxmode)) - #endif diff --git a/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h b/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h new file mode 100644 index 0000000000000000000000000000000000000000..2688da2f621fddc49d0d8f457b315476ada3b790 --- /dev/null +++ b/include/dt-bindings/pinctrl/mediatek,mt8188-pinfunc.h @@ -0,0 +1,1280 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (C) 2022 MediaTek Inc. + * Author: Hui Liu + */ + +#ifndef __MEDIATEK_MT8188_PINFUNC_H +#define __MEDIATEK_MT8188_PINFUNC_H + +#include "mt65xx.h" + +#define PINMUX_GPIO0__FUNC_B_GPIO0 (MTK_PIN_NO(0) | 0) +#define PINMUX_GPIO0__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(0) | 1) +#define PINMUX_GPIO0__FUNC_O_SPIM5_CSB (MTK_PIN_NO(0) | 2) +#define PINMUX_GPIO0__FUNC_O_UTXD1 (MTK_PIN_NO(0) | 3) +#define PINMUX_GPIO0__FUNC_O_DMIC3_CLK (MTK_PIN_NO(0) | 4) +#define PINMUX_GPIO0__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(0) | 5) +#define PINMUX_GPIO0__FUNC_O_I2SO2_MCK (MTK_PIN_NO(0) | 6) +#define PINMUX_GPIO0__FUNC_B0_DBG_MON_A0 (MTK_PIN_NO(0) | 7) + +#define PINMUX_GPIO1__FUNC_B_GPIO1 (MTK_PIN_NO(1) | 0) +#define PINMUX_GPIO1__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(1) | 1) +#define PINMUX_GPIO1__FUNC_O_SPIM5_CLK (MTK_PIN_NO(1) | 2) +#define PINMUX_GPIO1__FUNC_I1_URXD1 (MTK_PIN_NO(1) | 3) +#define PINMUX_GPIO1__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(1) | 4) +#define PINMUX_GPIO1__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(1) | 5) +#define PINMUX_GPIO1__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(1) | 6) +#define PINMUX_GPIO1__FUNC_B0_DBG_MON_A1 (MTK_PIN_NO(1) | 7) + +#define PINMUX_GPIO2__FUNC_B_GPIO2 (MTK_PIN_NO(2) | 0) +#define PINMUX_GPIO2__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(2) | 1) +#define PINMUX_GPIO2__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(2) | 2) +#define PINMUX_GPIO2__FUNC_O_URTS1 (MTK_PIN_NO(2) | 3) +#define PINMUX_GPIO2__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(2) | 4) +#define PINMUX_GPIO2__FUNC_B0_I2SIN_WS (MTK_PIN_NO(2) | 5) +#define PINMUX_GPIO2__FUNC_B0_I2SO2_WS (MTK_PIN_NO(2) | 6) +#define PINMUX_GPIO2__FUNC_B0_DBG_MON_A2 (MTK_PIN_NO(2) | 7) + +#define PINMUX_GPIO3__FUNC_B_GPIO3 (MTK_PIN_NO(3) | 0) +#define PINMUX_GPIO3__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(3) | 1) +#define PINMUX_GPIO3__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(3) | 2) +#define PINMUX_GPIO3__FUNC_I1_UCTS1 (MTK_PIN_NO(3) | 3) +#define PINMUX_GPIO3__FUNC_O_DMIC4_CLK (MTK_PIN_NO(3) | 4) +#define PINMUX_GPIO3__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(3) | 5) +#define PINMUX_GPIO3__FUNC_O_I2SO2_D0 (MTK_PIN_NO(3) | 6) +#define PINMUX_GPIO3__FUNC_B0_DBG_MON_A3 (MTK_PIN_NO(3) | 7) + +#define PINMUX_GPIO4__FUNC_B_GPIO4 (MTK_PIN_NO(4) | 0) +#define PINMUX_GPIO4__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(4) | 1) +#define PINMUX_GPIO4__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(4) | 2) +#define PINMUX_GPIO4__FUNC_O_I2SO1_MCK (MTK_PIN_NO(4) | 3) +#define PINMUX_GPIO4__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(4) | 4) +#define PINMUX_GPIO4__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(4) | 5) +#define PINMUX_GPIO4__FUNC_O_I2SO2_D1 (MTK_PIN_NO(4) | 6) +#define PINMUX_GPIO4__FUNC_B0_DBG_MON_A4 (MTK_PIN_NO(4) | 7) + +#define PINMUX_GPIO5__FUNC_B_GPIO5 (MTK_PIN_NO(5) | 0) +#define PINMUX_GPIO5__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(5) | 1) +#define PINMUX_GPIO5__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(5) | 2) +#define PINMUX_GPIO5__FUNC_O_I2SO1_BCK (MTK_PIN_NO(5) | 3) +#define PINMUX_GPIO5__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(5) | 4) +#define PINMUX_GPIO5__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(5) | 5) +#define PINMUX_GPIO5__FUNC_O_I2SO2_D2 (MTK_PIN_NO(5) | 6) +#define PINMUX_GPIO5__FUNC_B0_DBG_MON_A5 (MTK_PIN_NO(5) | 7) + +#define PINMUX_GPIO6__FUNC_B_GPIO6 (MTK_PIN_NO(6) | 0) +#define PINMUX_GPIO6__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(6) | 1) +#define PINMUX_GPIO6__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(6) | 2) +#define PINMUX_GPIO6__FUNC_O_I2SO1_WS (MTK_PIN_NO(6) | 3) +#define PINMUX_GPIO6__FUNC_O_DMIC1_CLK (MTK_PIN_NO(6) | 4) +#define PINMUX_GPIO6__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(6) | 5) +#define PINMUX_GPIO6__FUNC_O_I2SO2_D3 (MTK_PIN_NO(6) | 6) +#define PINMUX_GPIO6__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(6) | 7) + +#define PINMUX_GPIO7__FUNC_B_GPIO7 (MTK_PIN_NO(7) | 0) +#define PINMUX_GPIO7__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(7) | 1) +#define PINMUX_GPIO7__FUNC_O_SPIM3_CSB (MTK_PIN_NO(7) | 2) +#define PINMUX_GPIO7__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(7) | 3) +#define PINMUX_GPIO7__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(7) | 4) +#define PINMUX_GPIO7__FUNC_O_CMVREF0 (MTK_PIN_NO(7) | 5) +#define PINMUX_GPIO7__FUNC_O_CLKM0 (MTK_PIN_NO(7) | 6) +#define PINMUX_GPIO7__FUNC_B0_DBG_MON_A6 (MTK_PIN_NO(7) | 7) + +#define PINMUX_GPIO8__FUNC_B_GPIO8 (MTK_PIN_NO(8) | 0) +#define PINMUX_GPIO8__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(8) | 1) +#define PINMUX_GPIO8__FUNC_O_SPIM3_CLK (MTK_PIN_NO(8) | 2) +#define PINMUX_GPIO8__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(8) | 3) +#define PINMUX_GPIO8__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(8) | 4) +#define PINMUX_GPIO8__FUNC_O_CMVREF1 (MTK_PIN_NO(8) | 5) +#define PINMUX_GPIO8__FUNC_O_CLKM1 (MTK_PIN_NO(8) | 6) +#define PINMUX_GPIO8__FUNC_B0_DBG_MON_A7 (MTK_PIN_NO(8) | 7) + +#define PINMUX_GPIO9__FUNC_B_GPIO9 (MTK_PIN_NO(9) | 0) +#define PINMUX_GPIO9__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(9) | 1) +#define PINMUX_GPIO9__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(9) | 2) +#define PINMUX_GPIO9__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(9) | 3) +#define PINMUX_GPIO9__FUNC_O_DMIC2_CLK (MTK_PIN_NO(9) | 4) +#define PINMUX_GPIO9__FUNC_O_CMFLASH0 (MTK_PIN_NO(9) | 5) +#define PINMUX_GPIO9__FUNC_O_PWM_0 (MTK_PIN_NO(9) | 6) +#define PINMUX_GPIO9__FUNC_B0_DBG_MON_A8 (MTK_PIN_NO(9) | 7) + +#define PINMUX_GPIO10__FUNC_B_GPIO10 (MTK_PIN_NO(10) | 0) +#define PINMUX_GPIO10__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(10) | 1) +#define PINMUX_GPIO10__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(10) | 2) +#define PINMUX_GPIO10__FUNC_I0_TDMIN_DI (MTK_PIN_NO(10) | 3) +#define PINMUX_GPIO10__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(10) | 4) +#define PINMUX_GPIO10__FUNC_O_CMFLASH1 (MTK_PIN_NO(10) | 5) +#define PINMUX_GPIO10__FUNC_O_PWM_1 (MTK_PIN_NO(10) | 6) +#define PINMUX_GPIO10__FUNC_B0_DBG_MON_A9 (MTK_PIN_NO(10) | 7) + +#define PINMUX_GPIO11__FUNC_B_GPIO11 (MTK_PIN_NO(11) | 0) +#define PINMUX_GPIO11__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(11) | 1) +#define PINMUX_GPIO11__FUNC_O_SPDIF_OUT (MTK_PIN_NO(11) | 2) +#define PINMUX_GPIO11__FUNC_O_I2SO1_D0 (MTK_PIN_NO(11) | 3) +#define PINMUX_GPIO11__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(11) | 4) +#define PINMUX_GPIO11__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(11) | 5) +#define PINMUX_GPIO11__FUNC_O_CMVREF6 (MTK_PIN_NO(11) | 6) +#define PINMUX_GPIO11__FUNC_B0_DBG_MON_A10 (MTK_PIN_NO(11) | 7) + +#define PINMUX_GPIO12__FUNC_B_GPIO12 (MTK_PIN_NO(12) | 0) +#define PINMUX_GPIO12__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(12) | 1) +#define PINMUX_GPIO12__FUNC_O_SPIM4_CSB (MTK_PIN_NO(12) | 2) +#define PINMUX_GPIO12__FUNC_B1_JTMS_SEL3 (MTK_PIN_NO(12) | 3) +#define PINMUX_GPIO12__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(12) | 4) +#define PINMUX_GPIO12__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(12) | 5) +#define PINMUX_GPIO12__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(12) | 6) +#define PINMUX_GPIO12__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(12) | 7) + +#define PINMUX_GPIO13__FUNC_B_GPIO13 (MTK_PIN_NO(13) | 0) +#define PINMUX_GPIO13__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(13) | 1) +#define PINMUX_GPIO13__FUNC_O_SPIM4_CLK (MTK_PIN_NO(13) | 2) +#define PINMUX_GPIO13__FUNC_I0_JTCK_SEL3 (MTK_PIN_NO(13) | 3) +#define PINMUX_GPIO13__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(13) | 4) +#define PINMUX_GPIO13__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(13) | 5) +#define PINMUX_GPIO13__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(13) | 6) +#define PINMUX_GPIO13__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(13) | 7) + +#define PINMUX_GPIO14__FUNC_B_GPIO14 (MTK_PIN_NO(14) | 0) +#define PINMUX_GPIO14__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(14) | 1) +#define PINMUX_GPIO14__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(14) | 2) +#define PINMUX_GPIO14__FUNC_I1_JTDI_SEL3 (MTK_PIN_NO(14) | 3) +#define PINMUX_GPIO14__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(14) | 4) +#define PINMUX_GPIO14__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(14) | 5) +#define PINMUX_GPIO14__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(14) | 6) +#define PINMUX_GPIO14__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(14) | 7) + +#define PINMUX_GPIO15__FUNC_B_GPIO15 (MTK_PIN_NO(15) | 0) +#define PINMUX_GPIO15__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(15) | 1) +#define PINMUX_GPIO15__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(15) | 2) +#define PINMUX_GPIO15__FUNC_O_JTDO_SEL3 (MTK_PIN_NO(15) | 3) +#define PINMUX_GPIO15__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(15) | 4) +#define PINMUX_GPIO15__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(15) | 5) +#define PINMUX_GPIO15__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(15) | 6) +#define PINMUX_GPIO15__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(15) | 7) + +#define PINMUX_GPIO16__FUNC_B_GPIO16 (MTK_PIN_NO(16) | 0) +#define PINMUX_GPIO16__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(16) | 1) +#define PINMUX_GPIO16__FUNC_O_UTXD3 (MTK_PIN_NO(16) | 2) +#define PINMUX_GPIO16__FUNC_I1_JTRSTn_SEL3 (MTK_PIN_NO(16) | 3) +#define PINMUX_GPIO16__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(16) | 4) +#define PINMUX_GPIO16__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(16) | 5) +#define PINMUX_GPIO16__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(16) | 6) +#define PINMUX_GPIO16__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(16) | 7) + +#define PINMUX_GPIO17__FUNC_B_GPIO17 (MTK_PIN_NO(17) | 0) +#define PINMUX_GPIO17__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(17) | 1) +#define PINMUX_GPIO17__FUNC_I1_URXD3 (MTK_PIN_NO(17) | 2) +#define PINMUX_GPIO17__FUNC_O_CMFLASH2 (MTK_PIN_NO(17) | 3) +#define PINMUX_GPIO17__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(17) | 4) +#define PINMUX_GPIO17__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(17) | 5) +#define PINMUX_GPIO17__FUNC_O_CMVREF7 (MTK_PIN_NO(17) | 6) +#define PINMUX_GPIO17__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(17) | 7) + +#define PINMUX_GPIO18__FUNC_B_GPIO18 (MTK_PIN_NO(18) | 0) +#define PINMUX_GPIO18__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(18) | 1) +#define PINMUX_GPIO18__FUNC_O_CMFLASH0 (MTK_PIN_NO(18) | 2) +#define PINMUX_GPIO18__FUNC_O_CMVREF4 (MTK_PIN_NO(18) | 3) +#define PINMUX_GPIO18__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(18) | 4) +#define PINMUX_GPIO18__FUNC_O_UTXD1 (MTK_PIN_NO(18) | 5) +#define PINMUX_GPIO18__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(18) | 6) +#define PINMUX_GPIO18__FUNC_B0_DBG_MON_A11 (MTK_PIN_NO(18) | 7) + +#define PINMUX_GPIO19__FUNC_B_GPIO19 (MTK_PIN_NO(19) | 0) +#define PINMUX_GPIO19__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(19) | 1) +#define PINMUX_GPIO19__FUNC_O_CMFLASH1 (MTK_PIN_NO(19) | 2) +#define PINMUX_GPIO19__FUNC_O_CMVREF5 (MTK_PIN_NO(19) | 3) +#define PINMUX_GPIO19__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(19) | 4) +#define PINMUX_GPIO19__FUNC_I1_URXD1 (MTK_PIN_NO(19) | 5) +#define PINMUX_GPIO19__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(19) | 6) +#define PINMUX_GPIO19__FUNC_B0_DBG_MON_A12 (MTK_PIN_NO(19) | 7) + +#define PINMUX_GPIO20__FUNC_B_GPIO20 (MTK_PIN_NO(20) | 0) +#define PINMUX_GPIO20__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(20) | 1) +#define PINMUX_GPIO20__FUNC_O_CMFLASH2 (MTK_PIN_NO(20) | 2) +#define PINMUX_GPIO20__FUNC_O_CLKM2 (MTK_PIN_NO(20) | 3) +#define PINMUX_GPIO20__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(20) | 4) +#define PINMUX_GPIO20__FUNC_O_URTS1 (MTK_PIN_NO(20) | 5) +#define PINMUX_GPIO20__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(20) | 6) +#define PINMUX_GPIO20__FUNC_B0_DBG_MON_A13 (MTK_PIN_NO(20) | 7) + +#define PINMUX_GPIO21__FUNC_B_GPIO21 (MTK_PIN_NO(21) | 0) +#define PINMUX_GPIO21__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(21) | 1) +#define PINMUX_GPIO21__FUNC_O_CMFLASH3 (MTK_PIN_NO(21) | 2) +#define PINMUX_GPIO21__FUNC_O_CLKM3 (MTK_PIN_NO(21) | 3) +#define PINMUX_GPIO21__FUNC_I0_TDMIN_DI (MTK_PIN_NO(21) | 4) +#define PINMUX_GPIO21__FUNC_I1_UCTS1 (MTK_PIN_NO(21) | 5) +#define PINMUX_GPIO21__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(21) | 6) +#define PINMUX_GPIO21__FUNC_B0_DBG_MON_A14 (MTK_PIN_NO(21) | 7) + +#define PINMUX_GPIO22__FUNC_B_GPIO22 (MTK_PIN_NO(22) | 0) +#define PINMUX_GPIO22__FUNC_O_CMMCLK0 (MTK_PIN_NO(22) | 1) +#define PINMUX_GPIO22__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(22) | 5) +#define PINMUX_GPIO22__FUNC_B0_DBG_MON_A15 (MTK_PIN_NO(22) | 7) + +#define PINMUX_GPIO23__FUNC_B_GPIO23 (MTK_PIN_NO(23) | 0) +#define PINMUX_GPIO23__FUNC_O_CMMCLK1 (MTK_PIN_NO(23) | 1) +#define PINMUX_GPIO23__FUNC_O_PWM_2 (MTK_PIN_NO(23) | 3) +#define PINMUX_GPIO23__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(23) | 4) +#define PINMUX_GPIO23__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(23) | 5) +#define PINMUX_GPIO23__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(23) | 6) +#define PINMUX_GPIO23__FUNC_B0_DBG_MON_A16 (MTK_PIN_NO(23) | 7) + +#define PINMUX_GPIO24__FUNC_B_GPIO24 (MTK_PIN_NO(24) | 0) +#define PINMUX_GPIO24__FUNC_O_CMMCLK2 (MTK_PIN_NO(24) | 1) +#define PINMUX_GPIO24__FUNC_O_PWM_3 (MTK_PIN_NO(24) | 3) +#define PINMUX_GPIO24__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(24) | 4) +#define PINMUX_GPIO24__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(24) | 5) +#define PINMUX_GPIO24__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(24) | 6) +#define PINMUX_GPIO24__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(24) | 7) + +#define PINMUX_GPIO25__FUNC_B_GPIO25 (MTK_PIN_NO(25) | 0) +#define PINMUX_GPIO25__FUNC_O_LCM_RST (MTK_PIN_NO(25) | 1) +#define PINMUX_GPIO25__FUNC_O_LCM1_RST (MTK_PIN_NO(25) | 2) +#define PINMUX_GPIO25__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(25) | 3) + +#define PINMUX_GPIO26__FUNC_B_GPIO26 (MTK_PIN_NO(26) | 0) +#define PINMUX_GPIO26__FUNC_I0_DSI_TE (MTK_PIN_NO(26) | 1) +#define PINMUX_GPIO26__FUNC_I0_DSI1_TE (MTK_PIN_NO(26) | 2) +#define PINMUX_GPIO26__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(26) | 3) + +#define PINMUX_GPIO27__FUNC_B_GPIO27 (MTK_PIN_NO(27) | 0) +#define PINMUX_GPIO27__FUNC_O_LCM1_RST (MTK_PIN_NO(27) | 1) +#define PINMUX_GPIO27__FUNC_O_LCM_RST (MTK_PIN_NO(27) | 2) +#define PINMUX_GPIO27__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(27) | 3) +#define PINMUX_GPIO27__FUNC_O_CMVREF2 (MTK_PIN_NO(27) | 4) +#define PINMUX_GPIO27__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(27) | 5) +#define PINMUX_GPIO27__FUNC_O_PWM_2 (MTK_PIN_NO(27) | 6) +#define PINMUX_GPIO27__FUNC_B0_DBG_MON_A17 (MTK_PIN_NO(27) | 7) + +#define PINMUX_GPIO28__FUNC_B_GPIO28 (MTK_PIN_NO(28) | 0) +#define PINMUX_GPIO28__FUNC_I0_DSI1_TE (MTK_PIN_NO(28) | 1) +#define PINMUX_GPIO28__FUNC_I0_DSI_TE (MTK_PIN_NO(28) | 2) +#define PINMUX_GPIO28__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(28) | 3) +#define PINMUX_GPIO28__FUNC_O_CMVREF3 (MTK_PIN_NO(28) | 4) +#define PINMUX_GPIO28__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(28) | 5) +#define PINMUX_GPIO28__FUNC_O_PWM_3 (MTK_PIN_NO(28) | 6) +#define PINMUX_GPIO28__FUNC_B0_DBG_MON_A18 (MTK_PIN_NO(28) | 7) + +#define PINMUX_GPIO29__FUNC_B_GPIO29 (MTK_PIN_NO(29) | 0) +#define PINMUX_GPIO29__FUNC_O_DISP_PWM0 (MTK_PIN_NO(29) | 1) +#define PINMUX_GPIO29__FUNC_O_DISP_PWM1 (MTK_PIN_NO(29) | 2) + +#define PINMUX_GPIO30__FUNC_B_GPIO30 (MTK_PIN_NO(30) | 0) +#define PINMUX_GPIO30__FUNC_O_DISP_PWM1 (MTK_PIN_NO(30) | 1) +#define PINMUX_GPIO30__FUNC_O_DISP_PWM0 (MTK_PIN_NO(30) | 2) +#define PINMUX_GPIO30__FUNC_O_CMFLASH3 (MTK_PIN_NO(30) | 3) +#define PINMUX_GPIO30__FUNC_O_PWM_1 (MTK_PIN_NO(30) | 4) +#define PINMUX_GPIO30__FUNC_B0_DBG_MON_A19 (MTK_PIN_NO(30) | 7) + +#define PINMUX_GPIO31__FUNC_B_GPIO31 (MTK_PIN_NO(31) | 0) +#define PINMUX_GPIO31__FUNC_O_UTXD0 (MTK_PIN_NO(31) | 1) +#define PINMUX_GPIO31__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(31) | 2) +#define PINMUX_GPIO31__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(31) | 3) +#define PINMUX_GPIO31__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(31) | 4) +#define PINMUX_GPIO31__FUNC_O_MD32_0_TXD (MTK_PIN_NO(31) | 5) +#define PINMUX_GPIO31__FUNC_O_MD32_1_TXD (MTK_PIN_NO(31) | 6) +#define PINMUX_GPIO31__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(31) | 7) + +#define PINMUX_GPIO32__FUNC_B_GPIO32 (MTK_PIN_NO(32) | 0) +#define PINMUX_GPIO32__FUNC_I1_URXD0 (MTK_PIN_NO(32) | 1) +#define PINMUX_GPIO32__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(32) | 2) +#define PINMUX_GPIO32__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(32) | 3) +#define PINMUX_GPIO32__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(32) | 4) +#define PINMUX_GPIO32__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(32) | 5) +#define PINMUX_GPIO32__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(32) | 6) +#define PINMUX_GPIO32__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(32) | 7) + +#define PINMUX_GPIO33__FUNC_B_GPIO33 (MTK_PIN_NO(33) | 0) +#define PINMUX_GPIO33__FUNC_O_UTXD1 (MTK_PIN_NO(33) | 1) +#define PINMUX_GPIO33__FUNC_O_URTS2 (MTK_PIN_NO(33) | 2) +#define PINMUX_GPIO33__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(33) | 3) +#define PINMUX_GPIO33__FUNC_O_TP_UTXD1_AO (MTK_PIN_NO(33) | 4) +#define PINMUX_GPIO33__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(33) | 5) +#define PINMUX_GPIO33__FUNC_O_MD32_0_TXD (MTK_PIN_NO(33) | 6) +#define PINMUX_GPIO33__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(33) | 7) + +#define PINMUX_GPIO34__FUNC_B_GPIO34 (MTK_PIN_NO(34) | 0) +#define PINMUX_GPIO34__FUNC_I1_URXD1 (MTK_PIN_NO(34) | 1) +#define PINMUX_GPIO34__FUNC_I1_UCTS2 (MTK_PIN_NO(34) | 2) +#define PINMUX_GPIO34__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(34) | 3) +#define PINMUX_GPIO34__FUNC_I1_TP_URXD1_AO (MTK_PIN_NO(34) | 4) +#define PINMUX_GPIO34__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(34) | 5) +#define PINMUX_GPIO34__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(34) | 6) +#define PINMUX_GPIO34__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(34) | 7) + +#define PINMUX_GPIO35__FUNC_B_GPIO35 (MTK_PIN_NO(35) | 0) +#define PINMUX_GPIO35__FUNC_O_UTXD2 (MTK_PIN_NO(35) | 1) +#define PINMUX_GPIO35__FUNC_O_URTS1 (MTK_PIN_NO(35) | 2) +#define PINMUX_GPIO35__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(35) | 3) +#define PINMUX_GPIO35__FUNC_O_TP_URTS1_AO (MTK_PIN_NO(35) | 4) +#define PINMUX_GPIO35__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(35) | 5) +#define PINMUX_GPIO35__FUNC_O_MD32_1_TXD (MTK_PIN_NO(35) | 6) +#define PINMUX_GPIO35__FUNC_B0_DBG_MON_A20 (MTK_PIN_NO(35) | 7) + +#define PINMUX_GPIO36__FUNC_B_GPIO36 (MTK_PIN_NO(36) | 0) +#define PINMUX_GPIO36__FUNC_I1_URXD2 (MTK_PIN_NO(36) | 1) +#define PINMUX_GPIO36__FUNC_I1_UCTS1 (MTK_PIN_NO(36) | 2) +#define PINMUX_GPIO36__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(36) | 3) +#define PINMUX_GPIO36__FUNC_I1_TP_UCTS1_AO (MTK_PIN_NO(36) | 4) +#define PINMUX_GPIO36__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(36) | 5) +#define PINMUX_GPIO36__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(36) | 6) +#define PINMUX_GPIO36__FUNC_B0_DBG_MON_A21 (MTK_PIN_NO(36) | 7) + +#define PINMUX_GPIO37__FUNC_B_GPIO37 (MTK_PIN_NO(37) | 0) +#define PINMUX_GPIO37__FUNC_B1_JTMS_SEL1 (MTK_PIN_NO(37) | 1) +#define PINMUX_GPIO37__FUNC_I0_UDI_TMS (MTK_PIN_NO(37) | 2) +#define PINMUX_GPIO37__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(37) | 3) +#define PINMUX_GPIO37__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(37) | 4) +#define PINMUX_GPIO37__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(37) | 5) +#define PINMUX_GPIO37__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(37) | 6) +#define PINMUX_GPIO37__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(37) | 7) + +#define PINMUX_GPIO38__FUNC_B_GPIO38 (MTK_PIN_NO(38) | 0) +#define PINMUX_GPIO38__FUNC_I0_JTCK_SEL1 (MTK_PIN_NO(38) | 1) +#define PINMUX_GPIO38__FUNC_I0_UDI_TCK (MTK_PIN_NO(38) | 2) +#define PINMUX_GPIO38__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(38) | 3) +#define PINMUX_GPIO38__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(38) | 4) +#define PINMUX_GPIO38__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(38) | 5) +#define PINMUX_GPIO38__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(38) | 6) +#define PINMUX_GPIO38__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(38) | 7) + +#define PINMUX_GPIO39__FUNC_B_GPIO39 (MTK_PIN_NO(39) | 0) +#define PINMUX_GPIO39__FUNC_I1_JTDI_SEL1 (MTK_PIN_NO(39) | 1) +#define PINMUX_GPIO39__FUNC_I0_UDI_TDI (MTK_PIN_NO(39) | 2) +#define PINMUX_GPIO39__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(39) | 3) +#define PINMUX_GPIO39__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(39) | 4) +#define PINMUX_GPIO39__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(39) | 5) +#define PINMUX_GPIO39__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(39) | 6) +#define PINMUX_GPIO39__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(39) | 7) + +#define PINMUX_GPIO40__FUNC_B_GPIO40 (MTK_PIN_NO(40) | 0) +#define PINMUX_GPIO40__FUNC_O_JTDO_SEL1 (MTK_PIN_NO(40) | 1) +#define PINMUX_GPIO40__FUNC_O_UDI_TDO (MTK_PIN_NO(40) | 2) +#define PINMUX_GPIO40__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(40) | 3) +#define PINMUX_GPIO40__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(40) | 4) +#define PINMUX_GPIO40__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(40) | 5) +#define PINMUX_GPIO40__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(40) | 6) +#define PINMUX_GPIO40__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(40) | 7) + +#define PINMUX_GPIO41__FUNC_B_GPIO41 (MTK_PIN_NO(41) | 0) +#define PINMUX_GPIO41__FUNC_I1_JTRSTn_SEL1 (MTK_PIN_NO(41) | 1) +#define PINMUX_GPIO41__FUNC_I0_UDI_NTRST (MTK_PIN_NO(41) | 2) +#define PINMUX_GPIO41__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(41) | 3) +#define PINMUX_GPIO41__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(41) | 4) +#define PINMUX_GPIO41__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(41) | 5) +#define PINMUX_GPIO41__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(41) | 6) +#define PINMUX_GPIO41__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(41) | 7) + +#define PINMUX_GPIO42__FUNC_B_GPIO42 (MTK_PIN_NO(42) | 0) +#define PINMUX_GPIO42__FUNC_B1_KPCOL0 (MTK_PIN_NO(42) | 1) + +#define PINMUX_GPIO43__FUNC_B_GPIO43 (MTK_PIN_NO(43) | 0) +#define PINMUX_GPIO43__FUNC_B1_KPCOL1 (MTK_PIN_NO(43) | 1) +#define PINMUX_GPIO43__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(43) | 2) +#define PINMUX_GPIO43__FUNC_O_CMFLASH2 (MTK_PIN_NO(43) | 3) +#define PINMUX_GPIO43__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(43) | 4) +#define PINMUX_GPIO43__FUNC_O_mbistwriteen_trigger (MTK_PIN_NO(43) | 7) + +#define PINMUX_GPIO44__FUNC_B_GPIO44 (MTK_PIN_NO(44) | 0) +#define PINMUX_GPIO44__FUNC_B1_KPROW0 (MTK_PIN_NO(44) | 1) + +#define PINMUX_GPIO45__FUNC_B_GPIO45 (MTK_PIN_NO(45) | 0) +#define PINMUX_GPIO45__FUNC_B1_KPROW1 (MTK_PIN_NO(45) | 1) +#define PINMUX_GPIO45__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(45) | 2) +#define PINMUX_GPIO45__FUNC_O_CMFLASH3 (MTK_PIN_NO(45) | 3) +#define PINMUX_GPIO45__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(45) | 4) +#define PINMUX_GPIO45__FUNC_O_mbistreaden_trigger (MTK_PIN_NO(45) | 7) + +#define PINMUX_GPIO46__FUNC_B_GPIO46 (MTK_PIN_NO(46) | 0) +#define PINMUX_GPIO46__FUNC_I0_DP_TX_HPD (MTK_PIN_NO(46) | 1) +#define PINMUX_GPIO46__FUNC_O_PWM_0 (MTK_PIN_NO(46) | 2) +#define PINMUX_GPIO46__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(46) | 3) +#define PINMUX_GPIO46__FUNC_B0_DBG_MON_A22 (MTK_PIN_NO(46) | 7) + +#define PINMUX_GPIO47__FUNC_B_GPIO47 (MTK_PIN_NO(47) | 0) +#define PINMUX_GPIO47__FUNC_I1_WAKEN (MTK_PIN_NO(47) | 1) +#define PINMUX_GPIO47__FUNC_O_GDU_TROOPS_DET0 (MTK_PIN_NO(47) | 6) + +#define PINMUX_GPIO48__FUNC_B_GPIO48 (MTK_PIN_NO(48) | 0) +#define PINMUX_GPIO48__FUNC_O_PERSTN (MTK_PIN_NO(48) | 1) +#define PINMUX_GPIO48__FUNC_O_GDU_TROOPS_DET1 (MTK_PIN_NO(48) | 6) + +#define PINMUX_GPIO49__FUNC_B_GPIO49 (MTK_PIN_NO(49) | 0) +#define PINMUX_GPIO49__FUNC_B1_CLKREQN (MTK_PIN_NO(49) | 1) +#define PINMUX_GPIO49__FUNC_O_GDU_TROOPS_DET2 (MTK_PIN_NO(49) | 6) + +#define PINMUX_GPIO50__FUNC_B_GPIO50 (MTK_PIN_NO(50) | 0) +#define PINMUX_GPIO50__FUNC_O_HDMITX20_PWR5V (MTK_PIN_NO(50) | 1) +#define PINMUX_GPIO50__FUNC_I1_IDDIG_1P (MTK_PIN_NO(50) | 3) +#define PINMUX_GPIO50__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(50) | 4) +#define PINMUX_GPIO50__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(50) | 5) +#define PINMUX_GPIO50__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(50) | 6) +#define PINMUX_GPIO50__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(50) | 7) + +#define PINMUX_GPIO51__FUNC_B_GPIO51 (MTK_PIN_NO(51) | 0) +#define PINMUX_GPIO51__FUNC_I0_HDMITX20_HTPLG (MTK_PIN_NO(51) | 1) +#define PINMUX_GPIO51__FUNC_I0_EDP_TX_HPD (MTK_PIN_NO(51) | 2) +#define PINMUX_GPIO51__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(51) | 3) +#define PINMUX_GPIO51__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(51) | 4) +#define PINMUX_GPIO51__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(51) | 5) +#define PINMUX_GPIO51__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(51) | 6) +#define PINMUX_GPIO51__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(51) | 7) + +#define PINMUX_GPIO52__FUNC_B_GPIO52 (MTK_PIN_NO(52) | 0) +#define PINMUX_GPIO52__FUNC_B1_HDMITX20_CEC (MTK_PIN_NO(52) | 1) +#define PINMUX_GPIO52__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(52) | 3) +#define PINMUX_GPIO52__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(52) | 4) +#define PINMUX_GPIO52__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(52) | 5) +#define PINMUX_GPIO52__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(52) | 6) +#define PINMUX_GPIO52__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(52) | 7) + +#define PINMUX_GPIO53__FUNC_B_GPIO53 (MTK_PIN_NO(53) | 0) +#define PINMUX_GPIO53__FUNC_B1_HDMITX20_SCL (MTK_PIN_NO(53) | 1) +#define PINMUX_GPIO53__FUNC_I1_IDDIG_2P (MTK_PIN_NO(53) | 3) +#define PINMUX_GPIO53__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(53) | 4) +#define PINMUX_GPIO53__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(53) | 5) +#define PINMUX_GPIO53__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(53) | 6) +#define PINMUX_GPIO53__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(53) | 7) + +#define PINMUX_GPIO54__FUNC_B_GPIO54 (MTK_PIN_NO(54) | 0) +#define PINMUX_GPIO54__FUNC_B1_HDMITX20_SDA (MTK_PIN_NO(54) | 1) +#define PINMUX_GPIO54__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(54) | 3) +#define PINMUX_GPIO54__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(54) | 4) +#define PINMUX_GPIO54__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(54) | 5) +#define PINMUX_GPIO54__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(54) | 6) +#define PINMUX_GPIO54__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(54) | 7) + +#define PINMUX_GPIO55__FUNC_B_GPIO55 (MTK_PIN_NO(55) | 0) +#define PINMUX_GPIO55__FUNC_B1_SCL0 (MTK_PIN_NO(55) | 1) +#define PINMUX_GPIO55__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(55) | 2) +#define PINMUX_GPIO55__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(55) | 3) +#define PINMUX_GPIO55__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(55) | 4) + +#define PINMUX_GPIO56__FUNC_B_GPIO56 (MTK_PIN_NO(56) | 0) +#define PINMUX_GPIO56__FUNC_B1_SDA0 (MTK_PIN_NO(56) | 1) +#define PINMUX_GPIO56__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(56) | 2) +#define PINMUX_GPIO56__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(56) | 3) +#define PINMUX_GPIO56__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(56) | 4) + +#define PINMUX_GPIO57__FUNC_B_GPIO57 (MTK_PIN_NO(57) | 0) +#define PINMUX_GPIO57__FUNC_B1_SCL1 (MTK_PIN_NO(57) | 1) + +#define PINMUX_GPIO58__FUNC_B_GPIO58 (MTK_PIN_NO(58) | 0) +#define PINMUX_GPIO58__FUNC_B1_SDA1 (MTK_PIN_NO(58) | 1) + +#define PINMUX_GPIO59__FUNC_B_GPIO59 (MTK_PIN_NO(59) | 0) +#define PINMUX_GPIO59__FUNC_B1_SCL2 (MTK_PIN_NO(59) | 1) +#define PINMUX_GPIO59__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(59) | 2) +#define PINMUX_GPIO59__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(59) | 3) + +#define PINMUX_GPIO60__FUNC_B_GPIO60 (MTK_PIN_NO(60) | 0) +#define PINMUX_GPIO60__FUNC_B1_SDA2 (MTK_PIN_NO(60) | 1) +#define PINMUX_GPIO60__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(60) | 2) +#define PINMUX_GPIO60__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(60) | 3) + +#define PINMUX_GPIO61__FUNC_B_GPIO61 (MTK_PIN_NO(61) | 0) +#define PINMUX_GPIO61__FUNC_B1_SCL3 (MTK_PIN_NO(61) | 1) +#define PINMUX_GPIO61__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(61) | 2) +#define PINMUX_GPIO61__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(61) | 3) +#define PINMUX_GPIO61__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(61) | 4) + +#define PINMUX_GPIO62__FUNC_B_GPIO62 (MTK_PIN_NO(62) | 0) +#define PINMUX_GPIO62__FUNC_B1_SDA3 (MTK_PIN_NO(62) | 1) +#define PINMUX_GPIO62__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(62) | 2) +#define PINMUX_GPIO62__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(62) | 3) +#define PINMUX_GPIO62__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(62) | 4) + +#define PINMUX_GPIO63__FUNC_B_GPIO63 (MTK_PIN_NO(63) | 0) +#define PINMUX_GPIO63__FUNC_B1_SCL4 (MTK_PIN_NO(63) | 1) + +#define PINMUX_GPIO64__FUNC_B_GPIO64 (MTK_PIN_NO(64) | 0) +#define PINMUX_GPIO64__FUNC_B1_SDA4 (MTK_PIN_NO(64) | 1) + +#define PINMUX_GPIO65__FUNC_B_GPIO65 (MTK_PIN_NO(65) | 0) +#define PINMUX_GPIO65__FUNC_B1_SCL5 (MTK_PIN_NO(65) | 1) +#define PINMUX_GPIO65__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(65) | 2) +#define PINMUX_GPIO65__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(65) | 3) + +#define PINMUX_GPIO66__FUNC_B_GPIO66 (MTK_PIN_NO(66) | 0) +#define PINMUX_GPIO66__FUNC_B1_SDA5 (MTK_PIN_NO(66) | 1) +#define PINMUX_GPIO66__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(66) | 2) +#define PINMUX_GPIO66__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(66) | 3) + +#define PINMUX_GPIO67__FUNC_B_GPIO67 (MTK_PIN_NO(67) | 0) +#define PINMUX_GPIO67__FUNC_B1_SCL6 (MTK_PIN_NO(67) | 1) +#define PINMUX_GPIO67__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(67) | 2) +#define PINMUX_GPIO67__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(67) | 3) +#define PINMUX_GPIO67__FUNC_B1_PCIE_PHY_I2C_SCL (MTK_PIN_NO(67) | 4) + +#define PINMUX_GPIO68__FUNC_B_GPIO68 (MTK_PIN_NO(68) | 0) +#define PINMUX_GPIO68__FUNC_B1_SDA6 (MTK_PIN_NO(68) | 1) +#define PINMUX_GPIO68__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(68) | 2) +#define PINMUX_GPIO68__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(68) | 3) +#define PINMUX_GPIO68__FUNC_B1_PCIE_PHY_I2C_SDA (MTK_PIN_NO(68) | 4) + +#define PINMUX_GPIO69__FUNC_B_GPIO69 (MTK_PIN_NO(69) | 0) +#define PINMUX_GPIO69__FUNC_O_SPIM0_CSB (MTK_PIN_NO(69) | 1) +#define PINMUX_GPIO69__FUNC_O_SCP_SPI0_CS (MTK_PIN_NO(69) | 2) +#define PINMUX_GPIO69__FUNC_O_DMIC3_CLK (MTK_PIN_NO(69) | 3) +#define PINMUX_GPIO69__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(69) | 4) +#define PINMUX_GPIO69__FUNC_O_CMVREF0 (MTK_PIN_NO(69) | 5) +#define PINMUX_GPIO69__FUNC_O_GDU_SUM_TROOP0_0 (MTK_PIN_NO(69) | 6) +#define PINMUX_GPIO69__FUNC_B0_DBG_MON_A23 (MTK_PIN_NO(69) | 7) + +#define PINMUX_GPIO70__FUNC_B_GPIO70 (MTK_PIN_NO(70) | 0) +#define PINMUX_GPIO70__FUNC_O_SPIM0_CLK (MTK_PIN_NO(70) | 1) +#define PINMUX_GPIO70__FUNC_O_SCP_SPI0_CK (MTK_PIN_NO(70) | 2) +#define PINMUX_GPIO70__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(70) | 3) +#define PINMUX_GPIO70__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(70) | 4) +#define PINMUX_GPIO70__FUNC_O_CMVREF1 (MTK_PIN_NO(70) | 5) +#define PINMUX_GPIO70__FUNC_O_GDU_SUM_TROOP0_1 (MTK_PIN_NO(70) | 6) +#define PINMUX_GPIO70__FUNC_B0_DBG_MON_A24 (MTK_PIN_NO(70) | 7) + +#define PINMUX_GPIO71__FUNC_B_GPIO71 (MTK_PIN_NO(71) | 0) +#define PINMUX_GPIO71__FUNC_B0_SPIM0_MOSI (MTK_PIN_NO(71) | 1) +#define PINMUX_GPIO71__FUNC_O_SCP_SPI0_MO (MTK_PIN_NO(71) | 2) +#define PINMUX_GPIO71__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(71) | 3) +#define PINMUX_GPIO71__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(71) | 4) +#define PINMUX_GPIO71__FUNC_O_CMVREF2 (MTK_PIN_NO(71) | 5) +#define PINMUX_GPIO71__FUNC_O_GDU_SUM_TROOP0_2 (MTK_PIN_NO(71) | 6) +#define PINMUX_GPIO71__FUNC_B0_DBG_MON_A25 (MTK_PIN_NO(71) | 7) + +#define PINMUX_GPIO72__FUNC_B_GPIO72 (MTK_PIN_NO(72) | 0) +#define PINMUX_GPIO72__FUNC_B0_SPIM0_MISO (MTK_PIN_NO(72) | 1) +#define PINMUX_GPIO72__FUNC_I0_SCP_SPI0_MI (MTK_PIN_NO(72) | 2) +#define PINMUX_GPIO72__FUNC_O_DMIC4_CLK (MTK_PIN_NO(72) | 3) +#define PINMUX_GPIO72__FUNC_O_CMVREF3 (MTK_PIN_NO(72) | 5) +#define PINMUX_GPIO72__FUNC_O_GDU_SUM_TROOP1_0 (MTK_PIN_NO(72) | 6) +#define PINMUX_GPIO72__FUNC_B0_DBG_MON_A26 (MTK_PIN_NO(72) | 7) + +#define PINMUX_GPIO73__FUNC_B_GPIO73 (MTK_PIN_NO(73) | 0) +#define PINMUX_GPIO73__FUNC_B0_SPIM0_MIO2 (MTK_PIN_NO(73) | 1) +#define PINMUX_GPIO73__FUNC_O_UTXD3 (MTK_PIN_NO(73) | 2) +#define PINMUX_GPIO73__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(73) | 3) +#define PINMUX_GPIO73__FUNC_O_CLKM0 (MTK_PIN_NO(73) | 4) +#define PINMUX_GPIO73__FUNC_O_CMVREF4 (MTK_PIN_NO(73) | 5) +#define PINMUX_GPIO73__FUNC_O_GDU_SUM_TROOP1_1 (MTK_PIN_NO(73) | 6) +#define PINMUX_GPIO73__FUNC_B0_DBG_MON_A27 (MTK_PIN_NO(73) | 7) + +#define PINMUX_GPIO74__FUNC_B_GPIO74 (MTK_PIN_NO(74) | 0) +#define PINMUX_GPIO74__FUNC_B0_SPIM0_MIO3 (MTK_PIN_NO(74) | 1) +#define PINMUX_GPIO74__FUNC_I1_URXD3 (MTK_PIN_NO(74) | 2) +#define PINMUX_GPIO74__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(74) | 3) +#define PINMUX_GPIO74__FUNC_O_CLKM1 (MTK_PIN_NO(74) | 4) +#define PINMUX_GPIO74__FUNC_O_CMVREF5 (MTK_PIN_NO(74) | 5) +#define PINMUX_GPIO74__FUNC_O_GDU_SUM_TROOP1_2 (MTK_PIN_NO(74) | 6) +#define PINMUX_GPIO74__FUNC_B0_DBG_MON_A28 (MTK_PIN_NO(74) | 7) + +#define PINMUX_GPIO75__FUNC_B_GPIO75 (MTK_PIN_NO(75) | 0) +#define PINMUX_GPIO75__FUNC_O_SPIM1_CSB (MTK_PIN_NO(75) | 1) +#define PINMUX_GPIO75__FUNC_O_SCP_SPI1_A_CS (MTK_PIN_NO(75) | 2) +#define PINMUX_GPIO75__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(75) | 3) +#define PINMUX_GPIO75__FUNC_B1_SCP_SCL0 (MTK_PIN_NO(75) | 4) +#define PINMUX_GPIO75__FUNC_O_CMVREF6 (MTK_PIN_NO(75) | 5) +#define PINMUX_GPIO75__FUNC_O_GDU_SUM_TROOP2_0 (MTK_PIN_NO(75) | 6) +#define PINMUX_GPIO75__FUNC_B0_DBG_MON_A29 (MTK_PIN_NO(75) | 7) + +#define PINMUX_GPIO76__FUNC_B_GPIO76 (MTK_PIN_NO(76) | 0) +#define PINMUX_GPIO76__FUNC_O_SPIM1_CLK (MTK_PIN_NO(76) | 1) +#define PINMUX_GPIO76__FUNC_O_SCP_SPI1_A_CK (MTK_PIN_NO(76) | 2) +#define PINMUX_GPIO76__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(76) | 3) +#define PINMUX_GPIO76__FUNC_B1_SCP_SDA0 (MTK_PIN_NO(76) | 4) +#define PINMUX_GPIO76__FUNC_O_CMVREF7 (MTK_PIN_NO(76) | 5) +#define PINMUX_GPIO76__FUNC_O_GDU_SUM_TROOP2_1 (MTK_PIN_NO(76) | 6) +#define PINMUX_GPIO76__FUNC_B0_DBG_MON_A30 (MTK_PIN_NO(76) | 7) + +#define PINMUX_GPIO77__FUNC_B_GPIO77 (MTK_PIN_NO(77) | 0) +#define PINMUX_GPIO77__FUNC_B0_SPIM1_MOSI (MTK_PIN_NO(77) | 1) +#define PINMUX_GPIO77__FUNC_O_SCP_SPI1_A_MO (MTK_PIN_NO(77) | 2) +#define PINMUX_GPIO77__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(77) | 3) +#define PINMUX_GPIO77__FUNC_B1_SCP_SCL1 (MTK_PIN_NO(77) | 4) +#define PINMUX_GPIO77__FUNC_O_GDU_SUM_TROOP2_2 (MTK_PIN_NO(77) | 6) +#define PINMUX_GPIO77__FUNC_B0_DBG_MON_A31 (MTK_PIN_NO(77) | 7) + +#define PINMUX_GPIO78__FUNC_B_GPIO78 (MTK_PIN_NO(78) | 0) +#define PINMUX_GPIO78__FUNC_B0_SPIM1_MISO (MTK_PIN_NO(78) | 1) +#define PINMUX_GPIO78__FUNC_I0_SCP_SPI1_A_MI (MTK_PIN_NO(78) | 2) +#define PINMUX_GPIO78__FUNC_I0_TDMIN_DI (MTK_PIN_NO(78) | 3) +#define PINMUX_GPIO78__FUNC_B1_SCP_SDA1 (MTK_PIN_NO(78) | 4) +#define PINMUX_GPIO78__FUNC_B0_DBG_MON_A32 (MTK_PIN_NO(78) | 7) + +#define PINMUX_GPIO79__FUNC_B_GPIO79 (MTK_PIN_NO(79) | 0) +#define PINMUX_GPIO79__FUNC_O_SPIM2_CSB (MTK_PIN_NO(79) | 1) +#define PINMUX_GPIO79__FUNC_O_SCP_SPI2_CS (MTK_PIN_NO(79) | 2) +#define PINMUX_GPIO79__FUNC_O_I2SO1_MCK (MTK_PIN_NO(79) | 3) +#define PINMUX_GPIO79__FUNC_O_UTXD2 (MTK_PIN_NO(79) | 4) +#define PINMUX_GPIO79__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(79) | 5) +#define PINMUX_GPIO79__FUNC_B0_PCM_SYNC (MTK_PIN_NO(79) | 6) +#define PINMUX_GPIO79__FUNC_B0_DBG_MON_B0 (MTK_PIN_NO(79) | 7) + +#define PINMUX_GPIO80__FUNC_B_GPIO80 (MTK_PIN_NO(80) | 0) +#define PINMUX_GPIO80__FUNC_O_SPIM2_CLK (MTK_PIN_NO(80) | 1) +#define PINMUX_GPIO80__FUNC_O_SCP_SPI2_CK (MTK_PIN_NO(80) | 2) +#define PINMUX_GPIO80__FUNC_O_I2SO1_BCK (MTK_PIN_NO(80) | 3) +#define PINMUX_GPIO80__FUNC_I1_URXD2 (MTK_PIN_NO(80) | 4) +#define PINMUX_GPIO80__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(80) | 5) +#define PINMUX_GPIO80__FUNC_B0_PCM_CLK (MTK_PIN_NO(80) | 6) +#define PINMUX_GPIO80__FUNC_B0_DBG_MON_B1 (MTK_PIN_NO(80) | 7) + +#define PINMUX_GPIO81__FUNC_B_GPIO81 (MTK_PIN_NO(81) | 0) +#define PINMUX_GPIO81__FUNC_B0_SPIM2_MOSI (MTK_PIN_NO(81) | 1) +#define PINMUX_GPIO81__FUNC_O_SCP_SPI2_MO (MTK_PIN_NO(81) | 2) +#define PINMUX_GPIO81__FUNC_O_I2SO1_WS (MTK_PIN_NO(81) | 3) +#define PINMUX_GPIO81__FUNC_O_URTS2 (MTK_PIN_NO(81) | 4) +#define PINMUX_GPIO81__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(81) | 5) +#define PINMUX_GPIO81__FUNC_O_PCM_DO (MTK_PIN_NO(81) | 6) +#define PINMUX_GPIO81__FUNC_B0_DBG_MON_B2 (MTK_PIN_NO(81) | 7) + +#define PINMUX_GPIO82__FUNC_B_GPIO82 (MTK_PIN_NO(82) | 0) +#define PINMUX_GPIO82__FUNC_B0_SPIM2_MISO (MTK_PIN_NO(82) | 1) +#define PINMUX_GPIO82__FUNC_I0_SCP_SPI2_MI (MTK_PIN_NO(82) | 2) +#define PINMUX_GPIO82__FUNC_O_I2SO1_D0 (MTK_PIN_NO(82) | 3) +#define PINMUX_GPIO82__FUNC_I1_UCTS2 (MTK_PIN_NO(82) | 4) +#define PINMUX_GPIO82__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(82) | 5) +#define PINMUX_GPIO82__FUNC_I0_PCM_DI (MTK_PIN_NO(82) | 6) +#define PINMUX_GPIO82__FUNC_B0_DBG_MON_B3 (MTK_PIN_NO(82) | 7) + +#define PINMUX_GPIO83__FUNC_B_GPIO83 (MTK_PIN_NO(83) | 0) +#define PINMUX_GPIO83__FUNC_I1_IDDIG (MTK_PIN_NO(83) | 1) + +#define PINMUX_GPIO84__FUNC_B_GPIO84 (MTK_PIN_NO(84) | 0) +#define PINMUX_GPIO84__FUNC_O_USB_DRVVBUS (MTK_PIN_NO(84) | 1) + +#define PINMUX_GPIO85__FUNC_B_GPIO85 (MTK_PIN_NO(85) | 0) +#define PINMUX_GPIO85__FUNC_I0_VBUSVALID (MTK_PIN_NO(85) | 1) + +#define PINMUX_GPIO86__FUNC_B_GPIO86 (MTK_PIN_NO(86) | 0) +#define PINMUX_GPIO86__FUNC_I1_IDDIG_1P (MTK_PIN_NO(86) | 1) +#define PINMUX_GPIO86__FUNC_O_UTXD1 (MTK_PIN_NO(86) | 2) +#define PINMUX_GPIO86__FUNC_O_URTS2 (MTK_PIN_NO(86) | 3) +#define PINMUX_GPIO86__FUNC_O_PWM_2 (MTK_PIN_NO(86) | 4) +#define PINMUX_GPIO86__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(86) | 5) +#define PINMUX_GPIO86__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(86) | 6) +#define PINMUX_GPIO86__FUNC_B0_DBG_MON_B4 (MTK_PIN_NO(86) | 7) + +#define PINMUX_GPIO87__FUNC_B_GPIO87 (MTK_PIN_NO(87) | 0) +#define PINMUX_GPIO87__FUNC_O_USB_DRVVBUS_1P (MTK_PIN_NO(87) | 1) +#define PINMUX_GPIO87__FUNC_I1_URXD1 (MTK_PIN_NO(87) | 2) +#define PINMUX_GPIO87__FUNC_I1_UCTS2 (MTK_PIN_NO(87) | 3) +#define PINMUX_GPIO87__FUNC_O_PWM_3 (MTK_PIN_NO(87) | 4) +#define PINMUX_GPIO87__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(87) | 5) +#define PINMUX_GPIO87__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(87) | 6) +#define PINMUX_GPIO87__FUNC_B0_DBG_MON_B5 (MTK_PIN_NO(87) | 7) + +#define PINMUX_GPIO88__FUNC_B_GPIO88 (MTK_PIN_NO(88) | 0) +#define PINMUX_GPIO88__FUNC_I0_VBUSVALID_1P (MTK_PIN_NO(88) | 1) +#define PINMUX_GPIO88__FUNC_O_UTXD2 (MTK_PIN_NO(88) | 2) +#define PINMUX_GPIO88__FUNC_O_URTS1 (MTK_PIN_NO(88) | 3) +#define PINMUX_GPIO88__FUNC_O_CLKM2 (MTK_PIN_NO(88) | 4) +#define PINMUX_GPIO88__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(88) | 5) +#define PINMUX_GPIO88__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(88) | 6) +#define PINMUX_GPIO88__FUNC_B0_DBG_MON_B6 (MTK_PIN_NO(88) | 7) + +#define PINMUX_GPIO89__FUNC_B_GPIO89 (MTK_PIN_NO(89) | 0) +#define PINMUX_GPIO89__FUNC_I1_IDDIG_2P (MTK_PIN_NO(89) | 1) +#define PINMUX_GPIO89__FUNC_I1_URXD2 (MTK_PIN_NO(89) | 2) +#define PINMUX_GPIO89__FUNC_I1_UCTS1 (MTK_PIN_NO(89) | 3) +#define PINMUX_GPIO89__FUNC_O_CLKM3 (MTK_PIN_NO(89) | 4) +#define PINMUX_GPIO89__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(89) | 5) +#define PINMUX_GPIO89__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(89) | 6) +#define PINMUX_GPIO89__FUNC_B0_DBG_MON_B7 (MTK_PIN_NO(89) | 7) + +#define PINMUX_GPIO90__FUNC_B_GPIO90 (MTK_PIN_NO(90) | 0) +#define PINMUX_GPIO90__FUNC_O_USB_DRVVBUS_2P (MTK_PIN_NO(90) | 1) +#define PINMUX_GPIO90__FUNC_O_UTXD3 (MTK_PIN_NO(90) | 2) +#define PINMUX_GPIO90__FUNC_O_ADSP_UTXD0 (MTK_PIN_NO(90) | 3) +#define PINMUX_GPIO90__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(90) | 4) +#define PINMUX_GPIO90__FUNC_O_MD32_0_TXD (MTK_PIN_NO(90) | 5) +#define PINMUX_GPIO90__FUNC_O_MD32_1_TXD (MTK_PIN_NO(90) | 6) +#define PINMUX_GPIO90__FUNC_B0_DBG_MON_B8 (MTK_PIN_NO(90) | 7) + +#define PINMUX_GPIO91__FUNC_B_GPIO91 (MTK_PIN_NO(91) | 0) +#define PINMUX_GPIO91__FUNC_I0_VBUSVALID_2P (MTK_PIN_NO(91) | 1) +#define PINMUX_GPIO91__FUNC_I1_URXD3 (MTK_PIN_NO(91) | 2) +#define PINMUX_GPIO91__FUNC_I1_ADSP_URXD0 (MTK_PIN_NO(91) | 3) +#define PINMUX_GPIO91__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(91) | 4) +#define PINMUX_GPIO91__FUNC_I1_MD32_0_RXD (MTK_PIN_NO(91) | 5) +#define PINMUX_GPIO91__FUNC_I1_MD32_1_RXD (MTK_PIN_NO(91) | 6) +#define PINMUX_GPIO91__FUNC_B0_DBG_MON_B9 (MTK_PIN_NO(91) | 7) + +#define PINMUX_GPIO92__FUNC_B_GPIO92 (MTK_PIN_NO(92) | 0) +#define PINMUX_GPIO92__FUNC_O_PWRAP_SPI0_CSN (MTK_PIN_NO(92) | 1) + +#define PINMUX_GPIO93__FUNC_B_GPIO93 (MTK_PIN_NO(93) | 0) +#define PINMUX_GPIO93__FUNC_O_PWRAP_SPI0_CK (MTK_PIN_NO(93) | 1) + +#define PINMUX_GPIO94__FUNC_B_GPIO94 (MTK_PIN_NO(94) | 0) +#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(94) | 1) +#define PINMUX_GPIO94__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(94) | 2) + +#define PINMUX_GPIO95__FUNC_B_GPIO95 (MTK_PIN_NO(95) | 0) +#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MI (MTK_PIN_NO(95) | 1) +#define PINMUX_GPIO95__FUNC_B0_PWRAP_SPI0_MO (MTK_PIN_NO(95) | 2) + +#define PINMUX_GPIO96__FUNC_B_GPIO96 (MTK_PIN_NO(96) | 0) +#define PINMUX_GPIO96__FUNC_O_SRCLKENA0 (MTK_PIN_NO(96) | 1) + +#define PINMUX_GPIO97__FUNC_B_GPIO97 (MTK_PIN_NO(97) | 0) +#define PINMUX_GPIO97__FUNC_O_SRCLKENA1 (MTK_PIN_NO(97) | 1) + +#define PINMUX_GPIO98__FUNC_B_GPIO98 (MTK_PIN_NO(98) | 0) +#define PINMUX_GPIO98__FUNC_O_SCP_VREQ_VAO (MTK_PIN_NO(98) | 1) +#define PINMUX_GPIO98__FUNC_I0_DVFSRC_EXT_REQ (MTK_PIN_NO(98) | 2) + +#define PINMUX_GPIO99__FUNC_B_GPIO99 (MTK_PIN_NO(99) | 0) +#define PINMUX_GPIO99__FUNC_I0_RTC32K_CK (MTK_PIN_NO(99) | 1) + +#define PINMUX_GPIO100__FUNC_B_GPIO100 (MTK_PIN_NO(100) | 0) +#define PINMUX_GPIO100__FUNC_O_WATCHDOG (MTK_PIN_NO(100) | 1) + +#define PINMUX_GPIO101__FUNC_B_GPIO101 (MTK_PIN_NO(101) | 0) +#define PINMUX_GPIO101__FUNC_O_AUD_CLK_MOSI (MTK_PIN_NO(101) | 1) +#define PINMUX_GPIO101__FUNC_O_I2SO1_MCK (MTK_PIN_NO(101) | 2) +#define PINMUX_GPIO101__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(101) | 3) + +#define PINMUX_GPIO102__FUNC_B_GPIO102 (MTK_PIN_NO(102) | 0) +#define PINMUX_GPIO102__FUNC_O_AUD_SYNC_MOSI (MTK_PIN_NO(102) | 1) +#define PINMUX_GPIO102__FUNC_O_I2SO1_BCK (MTK_PIN_NO(102) | 2) +#define PINMUX_GPIO102__FUNC_B0_I2SIN_WS (MTK_PIN_NO(102) | 3) + +#define PINMUX_GPIO103__FUNC_B_GPIO103 (MTK_PIN_NO(103) | 0) +#define PINMUX_GPIO103__FUNC_O_AUD_DAT_MOSI0 (MTK_PIN_NO(103) | 1) +#define PINMUX_GPIO103__FUNC_O_I2SO1_WS (MTK_PIN_NO(103) | 2) +#define PINMUX_GPIO103__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(103) | 3) + +#define PINMUX_GPIO104__FUNC_B_GPIO104 (MTK_PIN_NO(104) | 0) +#define PINMUX_GPIO104__FUNC_O_AUD_DAT_MOSI1 (MTK_PIN_NO(104) | 1) +#define PINMUX_GPIO104__FUNC_O_I2SO1_D0 (MTK_PIN_NO(104) | 2) +#define PINMUX_GPIO104__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(104) | 3) + +#define PINMUX_GPIO105__FUNC_B_GPIO105 (MTK_PIN_NO(105) | 0) +#define PINMUX_GPIO105__FUNC_I0_AUD_DAT_MISO0 (MTK_PIN_NO(105) | 1) +#define PINMUX_GPIO105__FUNC_I0_VOW_DAT_MISO (MTK_PIN_NO(105) | 2) +#define PINMUX_GPIO105__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(105) | 3) + +#define PINMUX_GPIO106__FUNC_B_GPIO106 (MTK_PIN_NO(106) | 0) +#define PINMUX_GPIO106__FUNC_I0_AUD_DAT_MISO1 (MTK_PIN_NO(106) | 1) +#define PINMUX_GPIO106__FUNC_I0_VOW_CLK_MISO (MTK_PIN_NO(106) | 2) +#define PINMUX_GPIO106__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(106) | 3) + +#define PINMUX_GPIO107__FUNC_B_GPIO107 (MTK_PIN_NO(107) | 0) +#define PINMUX_GPIO107__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(107) | 1) +#define PINMUX_GPIO107__FUNC_I0_SPLIN_MCK (MTK_PIN_NO(107) | 2) +#define PINMUX_GPIO107__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(107) | 3) +#define PINMUX_GPIO107__FUNC_O_CMVREF4 (MTK_PIN_NO(107) | 4) +#define PINMUX_GPIO107__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(107) | 5) +#define PINMUX_GPIO107__FUNC_O_PGD_LV_LSC_PWR0 (MTK_PIN_NO(107) | 6) + +#define PINMUX_GPIO108__FUNC_B_GPIO108 (MTK_PIN_NO(108) | 0) +#define PINMUX_GPIO108__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(108) | 1) +#define PINMUX_GPIO108__FUNC_I0_SPLIN_LRCK (MTK_PIN_NO(108) | 2) +#define PINMUX_GPIO108__FUNC_O_DMIC4_CLK (MTK_PIN_NO(108) | 3) +#define PINMUX_GPIO108__FUNC_O_CMVREF5 (MTK_PIN_NO(108) | 4) +#define PINMUX_GPIO108__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(108) | 5) +#define PINMUX_GPIO108__FUNC_O_PGD_LV_LSC_PWR1 (MTK_PIN_NO(108) | 6) +#define PINMUX_GPIO108__FUNC_B0_DBG_MON_B10 (MTK_PIN_NO(108) | 7) + +#define PINMUX_GPIO109__FUNC_B_GPIO109 (MTK_PIN_NO(109) | 0) +#define PINMUX_GPIO109__FUNC_B0_I2SIN_WS (MTK_PIN_NO(109) | 1) +#define PINMUX_GPIO109__FUNC_I0_SPLIN_BCK (MTK_PIN_NO(109) | 2) +#define PINMUX_GPIO109__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(109) | 3) +#define PINMUX_GPIO109__FUNC_O_CMVREF6 (MTK_PIN_NO(109) | 4) +#define PINMUX_GPIO109__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(109) | 5) +#define PINMUX_GPIO109__FUNC_O_PGD_LV_LSC_PWR2 (MTK_PIN_NO(109) | 6) +#define PINMUX_GPIO109__FUNC_B0_DBG_MON_B11 (MTK_PIN_NO(109) | 7) + +#define PINMUX_GPIO110__FUNC_B_GPIO110 (MTK_PIN_NO(110) | 0) +#define PINMUX_GPIO110__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(110) | 1) +#define PINMUX_GPIO110__FUNC_I0_SPLIN_D0 (MTK_PIN_NO(110) | 2) +#define PINMUX_GPIO110__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(110) | 3) +#define PINMUX_GPIO110__FUNC_O_CMVREF7 (MTK_PIN_NO(110) | 4) +#define PINMUX_GPIO110__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(110) | 5) +#define PINMUX_GPIO110__FUNC_O_PGD_LV_LSC_PWR3 (MTK_PIN_NO(110) | 6) +#define PINMUX_GPIO110__FUNC_B0_DBG_MON_B12 (MTK_PIN_NO(110) | 7) + +#define PINMUX_GPIO111__FUNC_B_GPIO111 (MTK_PIN_NO(111) | 0) +#define PINMUX_GPIO111__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(111) | 1) +#define PINMUX_GPIO111__FUNC_I0_SPLIN_D1 (MTK_PIN_NO(111) | 2) +#define PINMUX_GPIO111__FUNC_O_DMIC3_CLK (MTK_PIN_NO(111) | 3) +#define PINMUX_GPIO111__FUNC_O_SPDIF_OUT (MTK_PIN_NO(111) | 4) +#define PINMUX_GPIO111__FUNC_O_PGD_LV_LSC_PWR4 (MTK_PIN_NO(111) | 6) +#define PINMUX_GPIO111__FUNC_B0_DBG_MON_B13 (MTK_PIN_NO(111) | 7) + +#define PINMUX_GPIO112__FUNC_B_GPIO112 (MTK_PIN_NO(112) | 0) +#define PINMUX_GPIO112__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(112) | 1) +#define PINMUX_GPIO112__FUNC_I0_SPLIN_D2 (MTK_PIN_NO(112) | 2) +#define PINMUX_GPIO112__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(112) | 3) +#define PINMUX_GPIO112__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(112) | 4) +#define PINMUX_GPIO112__FUNC_O_I2SO1_WS (MTK_PIN_NO(112) | 5) +#define PINMUX_GPIO112__FUNC_O_PGD_LV_LSC_PWR5 (MTK_PIN_NO(112) | 6) +#define PINMUX_GPIO112__FUNC_B0_DBG_MON_B14 (MTK_PIN_NO(112) | 7) + +#define PINMUX_GPIO113__FUNC_B_GPIO113 (MTK_PIN_NO(113) | 0) +#define PINMUX_GPIO113__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(113) | 1) +#define PINMUX_GPIO113__FUNC_I0_SPLIN_D3 (MTK_PIN_NO(113) | 2) +#define PINMUX_GPIO113__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(113) | 3) +#define PINMUX_GPIO113__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(113) | 4) +#define PINMUX_GPIO113__FUNC_O_I2SO1_D0 (MTK_PIN_NO(113) | 5) +#define PINMUX_GPIO113__FUNC_B0_DBG_MON_B15 (MTK_PIN_NO(113) | 7) + +#define PINMUX_GPIO114__FUNC_B_GPIO114 (MTK_PIN_NO(114) | 0) +#define PINMUX_GPIO114__FUNC_O_I2SO2_MCK (MTK_PIN_NO(114) | 1) +#define PINMUX_GPIO114__FUNC_B0_I2SIN_MCK (MTK_PIN_NO(114) | 2) +#define PINMUX_GPIO114__FUNC_I1_MCUPM_JTAG_TMS (MTK_PIN_NO(114) | 3) +#define PINMUX_GPIO114__FUNC_B1_APU_JTAG_TMS (MTK_PIN_NO(114) | 4) +#define PINMUX_GPIO114__FUNC_I1_SCP_JTAG1_TMS (MTK_PIN_NO(114) | 5) +#define PINMUX_GPIO114__FUNC_I1_SPM_JTAG_TMS (MTK_PIN_NO(114) | 6) +#define PINMUX_GPIO114__FUNC_B0_DBG_MON_B16 (MTK_PIN_NO(114) | 7) + +#define PINMUX_GPIO115__FUNC_B_GPIO115 (MTK_PIN_NO(115) | 0) +#define PINMUX_GPIO115__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(115) | 1) +#define PINMUX_GPIO115__FUNC_B0_I2SIN_BCK (MTK_PIN_NO(115) | 2) +#define PINMUX_GPIO115__FUNC_I1_MCUPM_JTAG_TCK (MTK_PIN_NO(115) | 3) +#define PINMUX_GPIO115__FUNC_I0_APU_JTAG_TCK (MTK_PIN_NO(115) | 4) +#define PINMUX_GPIO115__FUNC_I1_SCP_JTAG1_TCK (MTK_PIN_NO(115) | 5) +#define PINMUX_GPIO115__FUNC_I1_SPM_JTAG_TCK (MTK_PIN_NO(115) | 6) +#define PINMUX_GPIO115__FUNC_B0_DBG_MON_B17 (MTK_PIN_NO(115) | 7) + +#define PINMUX_GPIO116__FUNC_B_GPIO116 (MTK_PIN_NO(116) | 0) +#define PINMUX_GPIO116__FUNC_B0_I2SO2_WS (MTK_PIN_NO(116) | 1) +#define PINMUX_GPIO116__FUNC_B0_I2SIN_WS (MTK_PIN_NO(116) | 2) +#define PINMUX_GPIO116__FUNC_I1_MCUPM_JTAG_TDI (MTK_PIN_NO(116) | 3) +#define PINMUX_GPIO116__FUNC_I1_APU_JTAG_TDI (MTK_PIN_NO(116) | 4) +#define PINMUX_GPIO116__FUNC_I1_SCP_JTAG1_TDI (MTK_PIN_NO(116) | 5) +#define PINMUX_GPIO116__FUNC_I1_SPM_JTAG_TDI (MTK_PIN_NO(116) | 6) +#define PINMUX_GPIO116__FUNC_B0_DBG_MON_B18 (MTK_PIN_NO(116) | 7) + +#define PINMUX_GPIO117__FUNC_B_GPIO117 (MTK_PIN_NO(117) | 0) +#define PINMUX_GPIO117__FUNC_O_I2SO2_D0 (MTK_PIN_NO(117) | 1) +#define PINMUX_GPIO117__FUNC_I0_I2SIN_D0 (MTK_PIN_NO(117) | 2) +#define PINMUX_GPIO117__FUNC_O_MCUPM_JTAG_TDO (MTK_PIN_NO(117) | 3) +#define PINMUX_GPIO117__FUNC_O_APU_JTAG_TDO (MTK_PIN_NO(117) | 4) +#define PINMUX_GPIO117__FUNC_O_SCP_JTAG1_TDO (MTK_PIN_NO(117) | 5) +#define PINMUX_GPIO117__FUNC_O_SPM_JTAG_TDO (MTK_PIN_NO(117) | 6) +#define PINMUX_GPIO117__FUNC_B0_DBG_MON_B19 (MTK_PIN_NO(117) | 7) + +#define PINMUX_GPIO118__FUNC_B_GPIO118 (MTK_PIN_NO(118) | 0) +#define PINMUX_GPIO118__FUNC_O_I2SO2_D1 (MTK_PIN_NO(118) | 1) +#define PINMUX_GPIO118__FUNC_I0_I2SIN_D1 (MTK_PIN_NO(118) | 2) +#define PINMUX_GPIO118__FUNC_I0_MCUPM_JTAG_TRSTN (MTK_PIN_NO(118) | 3) +#define PINMUX_GPIO118__FUNC_I0_APU_JTAG_TRST (MTK_PIN_NO(118) | 4) +#define PINMUX_GPIO118__FUNC_I0_SCP_JTAG1_TRSTN (MTK_PIN_NO(118) | 5) +#define PINMUX_GPIO118__FUNC_I0_SPM_JTAG_TRSTN (MTK_PIN_NO(118) | 6) +#define PINMUX_GPIO118__FUNC_B0_DBG_MON_B20 (MTK_PIN_NO(118) | 7) + +#define PINMUX_GPIO119__FUNC_B_GPIO119 (MTK_PIN_NO(119) | 0) +#define PINMUX_GPIO119__FUNC_O_I2SO2_D2 (MTK_PIN_NO(119) | 1) +#define PINMUX_GPIO119__FUNC_I0_I2SIN_D2 (MTK_PIN_NO(119) | 2) +#define PINMUX_GPIO119__FUNC_O_UTXD3 (MTK_PIN_NO(119) | 3) +#define PINMUX_GPIO119__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(119) | 4) +#define PINMUX_GPIO119__FUNC_O_I2SO1_MCK (MTK_PIN_NO(119) | 5) +#define PINMUX_GPIO119__FUNC_O_SSPM_UTXD_AO (MTK_PIN_NO(119) | 6) +#define PINMUX_GPIO119__FUNC_B0_DBG_MON_B21 (MTK_PIN_NO(119) | 7) + +#define PINMUX_GPIO120__FUNC_B_GPIO120 (MTK_PIN_NO(120) | 0) +#define PINMUX_GPIO120__FUNC_O_I2SO2_D3 (MTK_PIN_NO(120) | 1) +#define PINMUX_GPIO120__FUNC_I0_I2SIN_D3 (MTK_PIN_NO(120) | 2) +#define PINMUX_GPIO120__FUNC_I1_URXD3 (MTK_PIN_NO(120) | 3) +#define PINMUX_GPIO120__FUNC_I0_TDMIN_DI (MTK_PIN_NO(120) | 4) +#define PINMUX_GPIO120__FUNC_O_I2SO1_BCK (MTK_PIN_NO(120) | 5) +#define PINMUX_GPIO120__FUNC_I1_SSPM_URXD_AO (MTK_PIN_NO(120) | 6) +#define PINMUX_GPIO120__FUNC_B0_DBG_MON_B22 (MTK_PIN_NO(120) | 7) + +#define PINMUX_GPIO121__FUNC_B_GPIO121 (MTK_PIN_NO(121) | 0) +#define PINMUX_GPIO121__FUNC_B0_PCM_CLK (MTK_PIN_NO(121) | 1) +#define PINMUX_GPIO121__FUNC_O_SPIM4_CSB (MTK_PIN_NO(121) | 2) +#define PINMUX_GPIO121__FUNC_O_SCP_SPI1_B_CS (MTK_PIN_NO(121) | 3) +#define PINMUX_GPIO121__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(121) | 4) +#define PINMUX_GPIO121__FUNC_O_AUXIF_ST0 (MTK_PIN_NO(121) | 5) +#define PINMUX_GPIO121__FUNC_O_PGD_DA_EFUSE_RDY (MTK_PIN_NO(121) | 6) +#define PINMUX_GPIO121__FUNC_B0_DBG_MON_B23 (MTK_PIN_NO(121) | 7) + +#define PINMUX_GPIO122__FUNC_B_GPIO122 (MTK_PIN_NO(122) | 0) +#define PINMUX_GPIO122__FUNC_B0_PCM_SYNC (MTK_PIN_NO(122) | 1) +#define PINMUX_GPIO122__FUNC_O_SPIM4_CLK (MTK_PIN_NO(122) | 2) +#define PINMUX_GPIO122__FUNC_O_SCP_SPI1_B_CK (MTK_PIN_NO(122) | 3) +#define PINMUX_GPIO122__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(122) | 4) +#define PINMUX_GPIO122__FUNC_O_AUXIF_CLK0 (MTK_PIN_NO(122) | 5) +#define PINMUX_GPIO122__FUNC_O_PGD_DA_EFUSE_RDY_PRE (MTK_PIN_NO(122) | 6) +#define PINMUX_GPIO122__FUNC_B0_DBG_MON_B24 (MTK_PIN_NO(122) | 7) + +#define PINMUX_GPIO123__FUNC_B_GPIO123 (MTK_PIN_NO(123) | 0) +#define PINMUX_GPIO123__FUNC_O_PCM_DO (MTK_PIN_NO(123) | 1) +#define PINMUX_GPIO123__FUNC_B0_SPIM4_MOSI (MTK_PIN_NO(123) | 2) +#define PINMUX_GPIO123__FUNC_O_SCP_SPI1_B_MO (MTK_PIN_NO(123) | 3) +#define PINMUX_GPIO123__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(123) | 4) +#define PINMUX_GPIO123__FUNC_O_AUXIF_ST1 (MTK_PIN_NO(123) | 5) +#define PINMUX_GPIO123__FUNC_O_PGD_DA_PWRGD_RESET (MTK_PIN_NO(123) | 6) +#define PINMUX_GPIO123__FUNC_B0_DBG_MON_B25 (MTK_PIN_NO(123) | 7) + +#define PINMUX_GPIO124__FUNC_B_GPIO124 (MTK_PIN_NO(124) | 0) +#define PINMUX_GPIO124__FUNC_I0_PCM_DI (MTK_PIN_NO(124) | 1) +#define PINMUX_GPIO124__FUNC_B0_SPIM4_MISO (MTK_PIN_NO(124) | 2) +#define PINMUX_GPIO124__FUNC_I0_SCP_SPI1_B_MI (MTK_PIN_NO(124) | 3) +#define PINMUX_GPIO124__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(124) | 4) +#define PINMUX_GPIO124__FUNC_O_AUXIF_CLK1 (MTK_PIN_NO(124) | 5) +#define PINMUX_GPIO124__FUNC_O_PGD_DA_PWRGD_ENB (MTK_PIN_NO(124) | 6) +#define PINMUX_GPIO124__FUNC_B0_DBG_MON_B26 (MTK_PIN_NO(124) | 7) + +#define PINMUX_GPIO125__FUNC_B_GPIO125 (MTK_PIN_NO(125) | 0) +#define PINMUX_GPIO125__FUNC_O_DMIC1_CLK (MTK_PIN_NO(125) | 1) +#define PINMUX_GPIO125__FUNC_O_SPINOR_CK (MTK_PIN_NO(125) | 2) +#define PINMUX_GPIO125__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(125) | 3) +#define PINMUX_GPIO125__FUNC_O_LVTS_FOUT (MTK_PIN_NO(125) | 6) +#define PINMUX_GPIO125__FUNC_B0_DBG_MON_B27 (MTK_PIN_NO(125) | 7) + +#define PINMUX_GPIO126__FUNC_B_GPIO126 (MTK_PIN_NO(126) | 0) +#define PINMUX_GPIO126__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(126) | 1) +#define PINMUX_GPIO126__FUNC_O_SPINOR_CS (MTK_PIN_NO(126) | 2) +#define PINMUX_GPIO126__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(126) | 3) +#define PINMUX_GPIO126__FUNC_O_LVTS_SDO (MTK_PIN_NO(126) | 6) +#define PINMUX_GPIO126__FUNC_B0_DBG_MON_B28 (MTK_PIN_NO(126) | 7) + +#define PINMUX_GPIO127__FUNC_B_GPIO127 (MTK_PIN_NO(127) | 0) +#define PINMUX_GPIO127__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(127) | 1) +#define PINMUX_GPIO127__FUNC_B0_SPINOR_IO0 (MTK_PIN_NO(127) | 2) +#define PINMUX_GPIO127__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(127) | 3) +#define PINMUX_GPIO127__FUNC_I0_LVTS_26M (MTK_PIN_NO(127) | 6) +#define PINMUX_GPIO127__FUNC_B0_DBG_MON_B29 (MTK_PIN_NO(127) | 7) + +#define PINMUX_GPIO128__FUNC_B_GPIO128 (MTK_PIN_NO(128) | 0) +#define PINMUX_GPIO128__FUNC_O_DMIC2_CLK (MTK_PIN_NO(128) | 1) +#define PINMUX_GPIO128__FUNC_B0_SPINOR_IO1 (MTK_PIN_NO(128) | 2) +#define PINMUX_GPIO128__FUNC_I0_TDMIN_DI (MTK_PIN_NO(128) | 3) +#define PINMUX_GPIO128__FUNC_I0_LVTS_SCF (MTK_PIN_NO(128) | 6) +#define PINMUX_GPIO128__FUNC_B0_DBG_MON_B30 (MTK_PIN_NO(128) | 7) + +#define PINMUX_GPIO129__FUNC_B_GPIO129 (MTK_PIN_NO(129) | 0) +#define PINMUX_GPIO129__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(129) | 1) +#define PINMUX_GPIO129__FUNC_B0_SPINOR_IO2 (MTK_PIN_NO(129) | 2) +#define PINMUX_GPIO129__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(129) | 3) +#define PINMUX_GPIO129__FUNC_I0_LVTS_SCK (MTK_PIN_NO(129) | 6) +#define PINMUX_GPIO129__FUNC_B0_DBG_MON_B31 (MTK_PIN_NO(129) | 7) + +#define PINMUX_GPIO130__FUNC_B_GPIO130 (MTK_PIN_NO(130) | 0) +#define PINMUX_GPIO130__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(130) | 1) +#define PINMUX_GPIO130__FUNC_B0_SPINOR_IO3 (MTK_PIN_NO(130) | 2) +#define PINMUX_GPIO130__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(130) | 3) +#define PINMUX_GPIO130__FUNC_I0_LVTS_SDI (MTK_PIN_NO(130) | 6) +#define PINMUX_GPIO130__FUNC_B0_DBG_MON_B32 (MTK_PIN_NO(130) | 7) + +#define PINMUX_GPIO131__FUNC_B_GPIO131 (MTK_PIN_NO(131) | 0) +#define PINMUX_GPIO131__FUNC_O_DPI_D0 (MTK_PIN_NO(131) | 1) +#define PINMUX_GPIO131__FUNC_O_GBE_TXD3 (MTK_PIN_NO(131) | 2) +#define PINMUX_GPIO131__FUNC_O_DMIC1_CLK (MTK_PIN_NO(131) | 3) +#define PINMUX_GPIO131__FUNC_O_I2SO2_MCK (MTK_PIN_NO(131) | 4) +#define PINMUX_GPIO131__FUNC_B0_TP_GPIO0_AO (MTK_PIN_NO(131) | 5) +#define PINMUX_GPIO131__FUNC_O_SPIM5_CSB (MTK_PIN_NO(131) | 6) +#define PINMUX_GPIO131__FUNC_O_PGD_LV_HSC_PWR0 (MTK_PIN_NO(131) | 7) + +#define PINMUX_GPIO132__FUNC_B_GPIO132 (MTK_PIN_NO(132) | 0) +#define PINMUX_GPIO132__FUNC_O_DPI_D1 (MTK_PIN_NO(132) | 1) +#define PINMUX_GPIO132__FUNC_O_GBE_TXD2 (MTK_PIN_NO(132) | 2) +#define PINMUX_GPIO132__FUNC_I0_DMIC1_DAT (MTK_PIN_NO(132) | 3) +#define PINMUX_GPIO132__FUNC_B0_I2SO2_BCK (MTK_PIN_NO(132) | 4) +#define PINMUX_GPIO132__FUNC_B0_TP_GPIO1_AO (MTK_PIN_NO(132) | 5) +#define PINMUX_GPIO132__FUNC_O_SPIM5_CLK (MTK_PIN_NO(132) | 6) +#define PINMUX_GPIO132__FUNC_O_PGD_LV_HSC_PWR1 (MTK_PIN_NO(132) | 7) + +#define PINMUX_GPIO133__FUNC_B_GPIO133 (MTK_PIN_NO(133) | 0) +#define PINMUX_GPIO133__FUNC_O_DPI_D2 (MTK_PIN_NO(133) | 1) +#define PINMUX_GPIO133__FUNC_O_GBE_TXD1 (MTK_PIN_NO(133) | 2) +#define PINMUX_GPIO133__FUNC_I0_DMIC1_DAT_R (MTK_PIN_NO(133) | 3) +#define PINMUX_GPIO133__FUNC_B0_I2SO2_WS (MTK_PIN_NO(133) | 4) +#define PINMUX_GPIO133__FUNC_B0_TP_GPIO2_AO (MTK_PIN_NO(133) | 5) +#define PINMUX_GPIO133__FUNC_B0_SPIM5_MOSI (MTK_PIN_NO(133) | 6) +#define PINMUX_GPIO133__FUNC_O_PGD_LV_HSC_PWR2 (MTK_PIN_NO(133) | 7) + +#define PINMUX_GPIO134__FUNC_B_GPIO134 (MTK_PIN_NO(134) | 0) +#define PINMUX_GPIO134__FUNC_O_DPI_D3 (MTK_PIN_NO(134) | 1) +#define PINMUX_GPIO134__FUNC_O_GBE_TXD0 (MTK_PIN_NO(134) | 2) +#define PINMUX_GPIO134__FUNC_O_DMIC2_CLK (MTK_PIN_NO(134) | 3) +#define PINMUX_GPIO134__FUNC_O_I2SO2_D0 (MTK_PIN_NO(134) | 4) +#define PINMUX_GPIO134__FUNC_B0_TP_GPIO3_AO (MTK_PIN_NO(134) | 5) +#define PINMUX_GPIO134__FUNC_B0_SPIM5_MISO (MTK_PIN_NO(134) | 6) +#define PINMUX_GPIO134__FUNC_O_PGD_LV_HSC_PWR3 (MTK_PIN_NO(134) | 7) + +#define PINMUX_GPIO135__FUNC_B_GPIO135 (MTK_PIN_NO(135) | 0) +#define PINMUX_GPIO135__FUNC_O_DPI_D4 (MTK_PIN_NO(135) | 1) +#define PINMUX_GPIO135__FUNC_I0_GBE_RXD3 (MTK_PIN_NO(135) | 2) +#define PINMUX_GPIO135__FUNC_I0_DMIC2_DAT (MTK_PIN_NO(135) | 3) +#define PINMUX_GPIO135__FUNC_O_I2SO2_D1 (MTK_PIN_NO(135) | 4) +#define PINMUX_GPIO135__FUNC_B0_TP_GPIO4_AO (MTK_PIN_NO(135) | 5) +#define PINMUX_GPIO135__FUNC_I1_WAKEN (MTK_PIN_NO(135) | 6) +#define PINMUX_GPIO135__FUNC_O_PGD_LV_HSC_PWR4 (MTK_PIN_NO(135) | 7) + +#define PINMUX_GPIO136__FUNC_B_GPIO136 (MTK_PIN_NO(136) | 0) +#define PINMUX_GPIO136__FUNC_O_DPI_D5 (MTK_PIN_NO(136) | 1) +#define PINMUX_GPIO136__FUNC_I0_GBE_RXD2 (MTK_PIN_NO(136) | 2) +#define PINMUX_GPIO136__FUNC_I0_DMIC2_DAT_R (MTK_PIN_NO(136) | 3) +#define PINMUX_GPIO136__FUNC_O_I2SO2_D2 (MTK_PIN_NO(136) | 4) +#define PINMUX_GPIO136__FUNC_B0_TP_GPIO5_AO (MTK_PIN_NO(136) | 5) +#define PINMUX_GPIO136__FUNC_O_PERSTN (MTK_PIN_NO(136) | 6) +#define PINMUX_GPIO136__FUNC_O_PGD_LV_HSC_PWR5 (MTK_PIN_NO(136) | 7) + +#define PINMUX_GPIO137__FUNC_B_GPIO137 (MTK_PIN_NO(137) | 0) +#define PINMUX_GPIO137__FUNC_O_DPI_D6 (MTK_PIN_NO(137) | 1) +#define PINMUX_GPIO137__FUNC_I0_GBE_RXD1 (MTK_PIN_NO(137) | 2) +#define PINMUX_GPIO137__FUNC_O_DMIC3_CLK (MTK_PIN_NO(137) | 3) +#define PINMUX_GPIO137__FUNC_O_I2SO2_D3 (MTK_PIN_NO(137) | 4) +#define PINMUX_GPIO137__FUNC_B0_TP_GPIO6_AO (MTK_PIN_NO(137) | 5) +#define PINMUX_GPIO137__FUNC_B1_CLKREQN (MTK_PIN_NO(137) | 6) +#define PINMUX_GPIO137__FUNC_O_PWM_0 (MTK_PIN_NO(137) | 7) + +#define PINMUX_GPIO138__FUNC_B_GPIO138 (MTK_PIN_NO(138) | 0) +#define PINMUX_GPIO138__FUNC_O_DPI_D7 (MTK_PIN_NO(138) | 1) +#define PINMUX_GPIO138__FUNC_I0_GBE_RXD0 (MTK_PIN_NO(138) | 2) +#define PINMUX_GPIO138__FUNC_I0_DMIC3_DAT (MTK_PIN_NO(138) | 3) +#define PINMUX_GPIO138__FUNC_O_CLKM2 (MTK_PIN_NO(138) | 4) +#define PINMUX_GPIO138__FUNC_B0_TP_GPIO7_AO (MTK_PIN_NO(138) | 5) +#define PINMUX_GPIO138__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(138) | 7) + +#define PINMUX_GPIO139__FUNC_B_GPIO139 (MTK_PIN_NO(139) | 0) +#define PINMUX_GPIO139__FUNC_O_DPI_D8 (MTK_PIN_NO(139) | 1) +#define PINMUX_GPIO139__FUNC_B0_GBE_TXC (MTK_PIN_NO(139) | 2) +#define PINMUX_GPIO139__FUNC_I0_DMIC3_DAT_R (MTK_PIN_NO(139) | 3) +#define PINMUX_GPIO139__FUNC_O_CLKM3 (MTK_PIN_NO(139) | 4) +#define PINMUX_GPIO139__FUNC_O_TP_UTXD2_AO (MTK_PIN_NO(139) | 5) +#define PINMUX_GPIO139__FUNC_O_UTXD2 (MTK_PIN_NO(139) | 6) +#define PINMUX_GPIO139__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(139) | 7) + +#define PINMUX_GPIO140__FUNC_B_GPIO140 (MTK_PIN_NO(140) | 0) +#define PINMUX_GPIO140__FUNC_O_DPI_D9 (MTK_PIN_NO(140) | 1) +#define PINMUX_GPIO140__FUNC_I0_GBE_RXC (MTK_PIN_NO(140) | 2) +#define PINMUX_GPIO140__FUNC_O_DMIC4_CLK (MTK_PIN_NO(140) | 3) +#define PINMUX_GPIO140__FUNC_O_PWM_2 (MTK_PIN_NO(140) | 4) +#define PINMUX_GPIO140__FUNC_I1_TP_URXD2_AO (MTK_PIN_NO(140) | 5) +#define PINMUX_GPIO140__FUNC_I1_URXD2 (MTK_PIN_NO(140) | 6) +#define PINMUX_GPIO140__FUNC_B0_MD32_0_GPIO2 (MTK_PIN_NO(140) | 7) + +#define PINMUX_GPIO141__FUNC_B_GPIO141 (MTK_PIN_NO(141) | 0) +#define PINMUX_GPIO141__FUNC_O_DPI_D10 (MTK_PIN_NO(141) | 1) +#define PINMUX_GPIO141__FUNC_I0_GBE_RXDV (MTK_PIN_NO(141) | 2) +#define PINMUX_GPIO141__FUNC_I0_DMIC4_DAT (MTK_PIN_NO(141) | 3) +#define PINMUX_GPIO141__FUNC_O_PWM_3 (MTK_PIN_NO(141) | 4) +#define PINMUX_GPIO141__FUNC_O_TP_URTS2_AO (MTK_PIN_NO(141) | 5) +#define PINMUX_GPIO141__FUNC_O_URTS2 (MTK_PIN_NO(141) | 6) +#define PINMUX_GPIO141__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(141) | 7) + +#define PINMUX_GPIO142__FUNC_B_GPIO142 (MTK_PIN_NO(142) | 0) +#define PINMUX_GPIO142__FUNC_O_DPI_D11 (MTK_PIN_NO(142) | 1) +#define PINMUX_GPIO142__FUNC_O_GBE_TXEN (MTK_PIN_NO(142) | 2) +#define PINMUX_GPIO142__FUNC_I0_DMIC4_DAT_R (MTK_PIN_NO(142) | 3) +#define PINMUX_GPIO142__FUNC_O_PWM_1 (MTK_PIN_NO(142) | 4) +#define PINMUX_GPIO142__FUNC_I1_TP_UCTS2_AO (MTK_PIN_NO(142) | 5) +#define PINMUX_GPIO142__FUNC_I1_UCTS2 (MTK_PIN_NO(142) | 6) +#define PINMUX_GPIO142__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(142) | 7) + +#define PINMUX_GPIO143__FUNC_B_GPIO143 (MTK_PIN_NO(143) | 0) +#define PINMUX_GPIO143__FUNC_O_DPI_D12 (MTK_PIN_NO(143) | 1) +#define PINMUX_GPIO143__FUNC_O_GBE_MDC (MTK_PIN_NO(143) | 2) +#define PINMUX_GPIO143__FUNC_B0_MD32_0_GPIO0 (MTK_PIN_NO(143) | 3) +#define PINMUX_GPIO143__FUNC_O_CLKM0 (MTK_PIN_NO(143) | 4) +#define PINMUX_GPIO143__FUNC_O_SPIM3_CSB (MTK_PIN_NO(143) | 5) +#define PINMUX_GPIO143__FUNC_O_UTXD1 (MTK_PIN_NO(143) | 6) +#define PINMUX_GPIO143__FUNC_B0_MD32_1_GPIO2 (MTK_PIN_NO(143) | 7) + +#define PINMUX_GPIO144__FUNC_B_GPIO144 (MTK_PIN_NO(144) | 0) +#define PINMUX_GPIO144__FUNC_O_DPI_D13 (MTK_PIN_NO(144) | 1) +#define PINMUX_GPIO144__FUNC_B1_GBE_MDIO (MTK_PIN_NO(144) | 2) +#define PINMUX_GPIO144__FUNC_B0_MD32_0_GPIO1 (MTK_PIN_NO(144) | 3) +#define PINMUX_GPIO144__FUNC_O_CLKM1 (MTK_PIN_NO(144) | 4) +#define PINMUX_GPIO144__FUNC_O_SPIM3_CLK (MTK_PIN_NO(144) | 5) +#define PINMUX_GPIO144__FUNC_I1_URXD1 (MTK_PIN_NO(144) | 6) +#define PINMUX_GPIO144__FUNC_O_PGD_HV_HSC_PWR0 (MTK_PIN_NO(144) | 7) + +#define PINMUX_GPIO145__FUNC_B_GPIO145 (MTK_PIN_NO(145) | 0) +#define PINMUX_GPIO145__FUNC_O_DPI_D14 (MTK_PIN_NO(145) | 1) +#define PINMUX_GPIO145__FUNC_O_GBE_TXER (MTK_PIN_NO(145) | 2) +#define PINMUX_GPIO145__FUNC_B0_MD32_1_GPIO0 (MTK_PIN_NO(145) | 3) +#define PINMUX_GPIO145__FUNC_O_CMFLASH0 (MTK_PIN_NO(145) | 4) +#define PINMUX_GPIO145__FUNC_B0_SPIM3_MOSI (MTK_PIN_NO(145) | 5) +#define PINMUX_GPIO145__FUNC_B0_GBE_AUX_PPS2 (MTK_PIN_NO(145) | 6) +#define PINMUX_GPIO145__FUNC_O_PGD_HV_HSC_PWR1 (MTK_PIN_NO(145) | 7) + +#define PINMUX_GPIO146__FUNC_B_GPIO146 (MTK_PIN_NO(146) | 0) +#define PINMUX_GPIO146__FUNC_O_DPI_D15 (MTK_PIN_NO(146) | 1) +#define PINMUX_GPIO146__FUNC_I0_GBE_RXER (MTK_PIN_NO(146) | 2) +#define PINMUX_GPIO146__FUNC_B0_MD32_1_GPIO1 (MTK_PIN_NO(146) | 3) +#define PINMUX_GPIO146__FUNC_O_CMFLASH1 (MTK_PIN_NO(146) | 4) +#define PINMUX_GPIO146__FUNC_B0_SPIM3_MISO (MTK_PIN_NO(146) | 5) +#define PINMUX_GPIO146__FUNC_B0_GBE_AUX_PPS3 (MTK_PIN_NO(146) | 6) +#define PINMUX_GPIO146__FUNC_O_PGD_HV_HSC_PWR2 (MTK_PIN_NO(146) | 7) + +#define PINMUX_GPIO147__FUNC_B_GPIO147 (MTK_PIN_NO(147) | 0) +#define PINMUX_GPIO147__FUNC_O_DPI_HSYNC (MTK_PIN_NO(147) | 1) +#define PINMUX_GPIO147__FUNC_I0_GBE_COL (MTK_PIN_NO(147) | 2) +#define PINMUX_GPIO147__FUNC_O_I2SO1_MCK (MTK_PIN_NO(147) | 3) +#define PINMUX_GPIO147__FUNC_O_CMVREF0 (MTK_PIN_NO(147) | 4) +#define PINMUX_GPIO147__FUNC_O_SPDIF_OUT (MTK_PIN_NO(147) | 5) +#define PINMUX_GPIO147__FUNC_O_URTS1 (MTK_PIN_NO(147) | 6) +#define PINMUX_GPIO147__FUNC_O_PGD_HV_HSC_PWR3 (MTK_PIN_NO(147) | 7) + +#define PINMUX_GPIO148__FUNC_B_GPIO148 (MTK_PIN_NO(148) | 0) +#define PINMUX_GPIO148__FUNC_O_DPI_VSYNC (MTK_PIN_NO(148) | 1) +#define PINMUX_GPIO148__FUNC_I0_GBE_INTR (MTK_PIN_NO(148) | 2) +#define PINMUX_GPIO148__FUNC_O_I2SO1_BCK (MTK_PIN_NO(148) | 3) +#define PINMUX_GPIO148__FUNC_O_CMVREF1 (MTK_PIN_NO(148) | 4) +#define PINMUX_GPIO148__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(148) | 5) +#define PINMUX_GPIO148__FUNC_I1_UCTS1 (MTK_PIN_NO(148) | 6) +#define PINMUX_GPIO148__FUNC_O_PGD_HV_HSC_PWR4 (MTK_PIN_NO(148) | 7) + +#define PINMUX_GPIO149__FUNC_B_GPIO149 (MTK_PIN_NO(149) | 0) +#define PINMUX_GPIO149__FUNC_O_DPI_DE (MTK_PIN_NO(149) | 1) +#define PINMUX_GPIO149__FUNC_B0_GBE_AUX_PPS0 (MTK_PIN_NO(149) | 2) +#define PINMUX_GPIO149__FUNC_O_I2SO1_WS (MTK_PIN_NO(149) | 3) +#define PINMUX_GPIO149__FUNC_O_CMVREF2 (MTK_PIN_NO(149) | 4) +#define PINMUX_GPIO149__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(149) | 5) +#define PINMUX_GPIO149__FUNC_O_UTXD3 (MTK_PIN_NO(149) | 6) +#define PINMUX_GPIO149__FUNC_O_PGD_HV_HSC_PWR5 (MTK_PIN_NO(149) | 7) + +#define PINMUX_GPIO150__FUNC_B_GPIO150 (MTK_PIN_NO(150) | 0) +#define PINMUX_GPIO150__FUNC_O_DPI_CK (MTK_PIN_NO(150) | 1) +#define PINMUX_GPIO150__FUNC_B0_GBE_AUX_PPS1 (MTK_PIN_NO(150) | 2) +#define PINMUX_GPIO150__FUNC_O_I2SO1_D0 (MTK_PIN_NO(150) | 3) +#define PINMUX_GPIO150__FUNC_O_CMVREF3 (MTK_PIN_NO(150) | 4) +#define PINMUX_GPIO150__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(150) | 5) +#define PINMUX_GPIO150__FUNC_I1_URXD3 (MTK_PIN_NO(150) | 6) + +#define PINMUX_GPIO151__FUNC_B_GPIO151 (MTK_PIN_NO(151) | 0) +#define PINMUX_GPIO151__FUNC_B1_MSDC0_DAT7 (MTK_PIN_NO(151) | 1) + +#define PINMUX_GPIO152__FUNC_B_GPIO152 (MTK_PIN_NO(152) | 0) +#define PINMUX_GPIO152__FUNC_B1_MSDC0_DAT6 (MTK_PIN_NO(152) | 1) + +#define PINMUX_GPIO153__FUNC_B_GPIO153 (MTK_PIN_NO(153) | 0) +#define PINMUX_GPIO153__FUNC_B1_MSDC0_DAT5 (MTK_PIN_NO(153) | 1) + +#define PINMUX_GPIO154__FUNC_B_GPIO154 (MTK_PIN_NO(154) | 0) +#define PINMUX_GPIO154__FUNC_B1_MSDC0_DAT4 (MTK_PIN_NO(154) | 1) + +#define PINMUX_GPIO155__FUNC_B_GPIO155 (MTK_PIN_NO(155) | 0) +#define PINMUX_GPIO155__FUNC_O_MSDC0_RSTB (MTK_PIN_NO(155) | 1) + +#define PINMUX_GPIO156__FUNC_B_GPIO156 (MTK_PIN_NO(156) | 0) +#define PINMUX_GPIO156__FUNC_B1_MSDC0_CMD (MTK_PIN_NO(156) | 1) + +#define PINMUX_GPIO157__FUNC_B_GPIO157 (MTK_PIN_NO(157) | 0) +#define PINMUX_GPIO157__FUNC_B1_MSDC0_CLK (MTK_PIN_NO(157) | 1) + +#define PINMUX_GPIO158__FUNC_B_GPIO158 (MTK_PIN_NO(158) | 0) +#define PINMUX_GPIO158__FUNC_B1_MSDC0_DAT3 (MTK_PIN_NO(158) | 1) + +#define PINMUX_GPIO159__FUNC_B_GPIO159 (MTK_PIN_NO(159) | 0) +#define PINMUX_GPIO159__FUNC_B1_MSDC0_DAT2 (MTK_PIN_NO(159) | 1) + +#define PINMUX_GPIO160__FUNC_B_GPIO160 (MTK_PIN_NO(160) | 0) +#define PINMUX_GPIO160__FUNC_B1_MSDC0_DAT1 (MTK_PIN_NO(160) | 1) + +#define PINMUX_GPIO161__FUNC_B_GPIO161 (MTK_PIN_NO(161) | 0) +#define PINMUX_GPIO161__FUNC_B1_MSDC0_DAT0 (MTK_PIN_NO(161) | 1) + +#define PINMUX_GPIO162__FUNC_B_GPIO162 (MTK_PIN_NO(162) | 0) +#define PINMUX_GPIO162__FUNC_B0_MSDC0_DSL (MTK_PIN_NO(162) | 1) + +#define PINMUX_GPIO163__FUNC_B_GPIO163 (MTK_PIN_NO(163) | 0) +#define PINMUX_GPIO163__FUNC_B1_MSDC1_CMD (MTK_PIN_NO(163) | 1) +#define PINMUX_GPIO163__FUNC_O_SPDIF_OUT (MTK_PIN_NO(163) | 2) +#define PINMUX_GPIO163__FUNC_I1_MD32_0_JTAG_TMS (MTK_PIN_NO(163) | 3) +#define PINMUX_GPIO163__FUNC_I1_ADSP_JTAG0_TMS (MTK_PIN_NO(163) | 4) +#define PINMUX_GPIO163__FUNC_I1_SCP_JTAG0_TMS (MTK_PIN_NO(163) | 5) +#define PINMUX_GPIO163__FUNC_I1_CCU0_JTAG_TMS (MTK_PIN_NO(163) | 6) +#define PINMUX_GPIO163__FUNC_I0_IPU_JTAG_TMS (MTK_PIN_NO(163) | 7) + +#define PINMUX_GPIO164__FUNC_B_GPIO164 (MTK_PIN_NO(164) | 0) +#define PINMUX_GPIO164__FUNC_B1_MSDC1_CLK (MTK_PIN_NO(164) | 1) +#define PINMUX_GPIO164__FUNC_I0_SPDIF_IN0 (MTK_PIN_NO(164) | 2) +#define PINMUX_GPIO164__FUNC_I1_MD32_0_JTAG_TCK (MTK_PIN_NO(164) | 3) +#define PINMUX_GPIO164__FUNC_I0_ADSP_JTAG0_TCK (MTK_PIN_NO(164) | 4) +#define PINMUX_GPIO164__FUNC_I1_SCP_JTAG0_TCK (MTK_PIN_NO(164) | 5) +#define PINMUX_GPIO164__FUNC_I1_CCU0_JTAG_TCK (MTK_PIN_NO(164) | 6) +#define PINMUX_GPIO164__FUNC_I0_IPU_JTAG_TCK (MTK_PIN_NO(164) | 7) + +#define PINMUX_GPIO165__FUNC_B_GPIO165 (MTK_PIN_NO(165) | 0) +#define PINMUX_GPIO165__FUNC_B1_MSDC1_DAT0 (MTK_PIN_NO(165) | 1) +#define PINMUX_GPIO165__FUNC_I0_SPDIF_IN1 (MTK_PIN_NO(165) | 2) +#define PINMUX_GPIO165__FUNC_I1_MD32_0_JTAG_TDI (MTK_PIN_NO(165) | 3) +#define PINMUX_GPIO165__FUNC_I1_ADSP_JTAG0_TDI (MTK_PIN_NO(165) | 4) +#define PINMUX_GPIO165__FUNC_I1_SCP_JTAG0_TDI (MTK_PIN_NO(165) | 5) +#define PINMUX_GPIO165__FUNC_I1_CCU0_JTAG_TDI (MTK_PIN_NO(165) | 6) +#define PINMUX_GPIO165__FUNC_I0_IPU_JTAG_TDI (MTK_PIN_NO(165) | 7) + +#define PINMUX_GPIO166__FUNC_B_GPIO166 (MTK_PIN_NO(166) | 0) +#define PINMUX_GPIO166__FUNC_B1_MSDC1_DAT1 (MTK_PIN_NO(166) | 1) +#define PINMUX_GPIO166__FUNC_I0_SPDIF_IN2 (MTK_PIN_NO(166) | 2) +#define PINMUX_GPIO166__FUNC_O_MD32_0_JTAG_TDO (MTK_PIN_NO(166) | 3) +#define PINMUX_GPIO166__FUNC_O_ADSP_JTAG0_TDO (MTK_PIN_NO(166) | 4) +#define PINMUX_GPIO166__FUNC_O_SCP_JTAG0_TDO (MTK_PIN_NO(166) | 5) +#define PINMUX_GPIO166__FUNC_O_CCU0_JTAG_TDO (MTK_PIN_NO(166) | 6) +#define PINMUX_GPIO166__FUNC_O_IPU_JTAG_TDO (MTK_PIN_NO(166) | 7) + +#define PINMUX_GPIO167__FUNC_B_GPIO167 (MTK_PIN_NO(167) | 0) +#define PINMUX_GPIO167__FUNC_B1_MSDC1_DAT2 (MTK_PIN_NO(167) | 1) +#define PINMUX_GPIO167__FUNC_O_PWM_0 (MTK_PIN_NO(167) | 2) +#define PINMUX_GPIO167__FUNC_I1_MD32_0_JTAG_TRST (MTK_PIN_NO(167) | 3) +#define PINMUX_GPIO167__FUNC_I1_ADSP_JTAG0_TRSTN (MTK_PIN_NO(167) | 4) +#define PINMUX_GPIO167__FUNC_I0_SCP_JTAG0_TRSTN (MTK_PIN_NO(167) | 5) +#define PINMUX_GPIO167__FUNC_I1_CCU0_JTAG_TRST (MTK_PIN_NO(167) | 6) +#define PINMUX_GPIO167__FUNC_I0_IPU_JTAG_TRST (MTK_PIN_NO(167) | 7) + +#define PINMUX_GPIO168__FUNC_B_GPIO168 (MTK_PIN_NO(168) | 0) +#define PINMUX_GPIO168__FUNC_B1_MSDC1_DAT3 (MTK_PIN_NO(168) | 1) +#define PINMUX_GPIO168__FUNC_O_PWM_1 (MTK_PIN_NO(168) | 2) +#define PINMUX_GPIO168__FUNC_O_CLKM0 (MTK_PIN_NO(168) | 3) + +#define PINMUX_GPIO169__FUNC_B_GPIO169 (MTK_PIN_NO(169) | 0) +#define PINMUX_GPIO169__FUNC_B1_MSDC2_CMD (MTK_PIN_NO(169) | 1) +#define PINMUX_GPIO169__FUNC_O_LVTS_FOUT (MTK_PIN_NO(169) | 2) +#define PINMUX_GPIO169__FUNC_I1_MD32_1_JTAG_TMS (MTK_PIN_NO(169) | 3) +#define PINMUX_GPIO169__FUNC_I0_UDI_TMS (MTK_PIN_NO(169) | 4) +#define PINMUX_GPIO169__FUNC_I0_VPU_UDI_TMS (MTK_PIN_NO(169) | 5) +#define PINMUX_GPIO169__FUNC_B0_TDMIN_MCK (MTK_PIN_NO(169) | 6) +#define PINMUX_GPIO169__FUNC_I1_SSPM_JTAG_TMS (MTK_PIN_NO(169) | 7) + +#define PINMUX_GPIO170__FUNC_B_GPIO170 (MTK_PIN_NO(170) | 0) +#define PINMUX_GPIO170__FUNC_B1_MSDC2_CLK (MTK_PIN_NO(170) | 1) +#define PINMUX_GPIO170__FUNC_O_LVTS_SDO (MTK_PIN_NO(170) | 2) +#define PINMUX_GPIO170__FUNC_I1_MD32_1_JTAG_TCK (MTK_PIN_NO(170) | 3) +#define PINMUX_GPIO170__FUNC_I0_UDI_TCK (MTK_PIN_NO(170) | 4) +#define PINMUX_GPIO170__FUNC_I0_VPU_UDI_TCK (MTK_PIN_NO(170) | 5) +#define PINMUX_GPIO170__FUNC_B0_TDMIN_BCK (MTK_PIN_NO(170) | 6) +#define PINMUX_GPIO170__FUNC_I1_SSPM_JTAG_TCK (MTK_PIN_NO(170) | 7) + +#define PINMUX_GPIO171__FUNC_B_GPIO171 (MTK_PIN_NO(171) | 0) +#define PINMUX_GPIO171__FUNC_B1_MSDC2_DAT0 (MTK_PIN_NO(171) | 1) +#define PINMUX_GPIO171__FUNC_I0_LVTS_26M (MTK_PIN_NO(171) | 2) +#define PINMUX_GPIO171__FUNC_I1_MD32_1_JTAG_TDI (MTK_PIN_NO(171) | 3) +#define PINMUX_GPIO171__FUNC_I0_UDI_TDI (MTK_PIN_NO(171) | 4) +#define PINMUX_GPIO171__FUNC_I0_VPU_UDI_TDI (MTK_PIN_NO(171) | 5) +#define PINMUX_GPIO171__FUNC_B0_TDMIN_LRCK (MTK_PIN_NO(171) | 6) +#define PINMUX_GPIO171__FUNC_I1_SSPM_JTAG_TDI (MTK_PIN_NO(171) | 7) + +#define PINMUX_GPIO172__FUNC_B_GPIO172 (MTK_PIN_NO(172) | 0) +#define PINMUX_GPIO172__FUNC_B1_MSDC2_DAT1 (MTK_PIN_NO(172) | 1) +#define PINMUX_GPIO172__FUNC_I0_LVTS_SCF (MTK_PIN_NO(172) | 2) +#define PINMUX_GPIO172__FUNC_O_MD32_1_JTAG_TDO (MTK_PIN_NO(172) | 3) +#define PINMUX_GPIO172__FUNC_O_UDI_TDO (MTK_PIN_NO(172) | 4) +#define PINMUX_GPIO172__FUNC_O_VPU_UDI_TDO (MTK_PIN_NO(172) | 5) +#define PINMUX_GPIO172__FUNC_I0_TDMIN_DI (MTK_PIN_NO(172) | 6) +#define PINMUX_GPIO172__FUNC_O_SSPM_JTAG_TDO (MTK_PIN_NO(172) | 7) + +#define PINMUX_GPIO173__FUNC_B_GPIO173 (MTK_PIN_NO(173) | 0) +#define PINMUX_GPIO173__FUNC_B1_MSDC2_DAT2 (MTK_PIN_NO(173) | 1) +#define PINMUX_GPIO173__FUNC_I0_LVTS_SCK (MTK_PIN_NO(173) | 2) +#define PINMUX_GPIO173__FUNC_I1_MD32_1_JTAG_TRST (MTK_PIN_NO(173) | 3) +#define PINMUX_GPIO173__FUNC_I0_UDI_NTRST (MTK_PIN_NO(173) | 4) +#define PINMUX_GPIO173__FUNC_I0_VPU_UDI_NTRST (MTK_PIN_NO(173) | 5) +#define PINMUX_GPIO173__FUNC_I0_SSPM_JTAG_TRSTN (MTK_PIN_NO(173) | 7) + +#define PINMUX_GPIO174__FUNC_B_GPIO174 (MTK_PIN_NO(174) | 0) +#define PINMUX_GPIO174__FUNC_B1_MSDC2_DAT3 (MTK_PIN_NO(174) | 1) +#define PINMUX_GPIO174__FUNC_I0_LVTS_SDI (MTK_PIN_NO(174) | 2) + +#define PINMUX_GPIO175__FUNC_B_GPIO175 (MTK_PIN_NO(175) | 0) +#define PINMUX_GPIO175__FUNC_B0_SPMI_M_SCL (MTK_PIN_NO(175) | 1) + +#define PINMUX_GPIO176__FUNC_B_GPIO176 (MTK_PIN_NO(176) | 0) +#define PINMUX_GPIO176__FUNC_B0_SPMI_M_SDA (MTK_PIN_NO(176) | 1) + +#endif /* __MEDIATEK_MT8188-PINFUNC_H */ diff --git a/include/dt-bindings/pinctrl/pinctrl-starfive.h b/include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h similarity index 98% rename from include/dt-bindings/pinctrl/pinctrl-starfive.h rename to include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h index de4f75c2c9e85f1d440fcb289f2492b4d7ecc5f6..a200f546d078af6d92c78285f876278723519cde 100644 --- a/include/dt-bindings/pinctrl/pinctrl-starfive.h +++ b/include/dt-bindings/pinctrl/pinctrl-starfive-jh7100.h @@ -3,8 +3,8 @@ * Copyright (C) 2021 Emil Renner Berthing */ -#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_H__ -#define __DT_BINDINGS_PINCTRL_STARFIVE_H__ +#ifndef __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__ +#define __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__ #define PAD_GPIO_OFFSET 0 #define PAD_FUNC_SHARE_OFFSET 64 @@ -272,4 +272,4 @@ #define GPI_NONE 0xff -#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_H__ */ +#endif /* __DT_BINDINGS_PINCTRL_STARFIVE_JH7100_H__ */ diff --git a/include/dt-bindings/pinctrl/samsung.h b/include/dt-bindings/pinctrl/samsung.h index 950970634dfe4be2d513c6a0e961d2197963bfe0..d1da5ff68d0c303f74f88f93b1c32869ab27ff9a 100644 --- a/include/dt-bindings/pinctrl/samsung.h +++ b/include/dt-bindings/pinctrl/samsung.h @@ -10,6 +10,13 @@ #ifndef __DT_BINDINGS_PINCTRL_SAMSUNG_H__ #define __DT_BINDINGS_PINCTRL_SAMSUNG_H__ +/* + * These bindings are deprecated, because they do not match the actual + * concept of bindings but rather contain pure register values. + * Instead include the header in the DTS source directory. + */ +#warning "These bindings are deprecated. Instead use the header in the DTS source directory." + #define EXYNOS_PIN_PULL_NONE 0 #define EXYNOS_PIN_PULL_DOWN 1 #define EXYNOS_PIN_PULL_UP 3 diff --git a/include/dt-bindings/power/fsl,imx93-power.h b/include/dt-bindings/power/fsl,imx93-power.h new file mode 100644 index 0000000000000000000000000000000000000000..17f9f015bf7df56ab49a025adf63b1c296a23f08 --- /dev/null +++ b/include/dt-bindings/power/fsl,imx93-power.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Copyright 2022 NXP + */ + +#ifndef __DT_BINDINGS_IMX93_POWER_H__ +#define __DT_BINDINGS_IMX93_POWER_H__ + +#define IMX93_MEDIABLK_PD_MIPI_DSI 0 +#define IMX93_MEDIABLK_PD_MIPI_CSI 1 +#define IMX93_MEDIABLK_PD_PXP 2 +#define IMX93_MEDIABLK_PD_LCDIF 3 +#define IMX93_MEDIABLK_PD_ISI 4 + +#endif diff --git a/include/dt-bindings/power/imx8mp-power.h b/include/dt-bindings/power/imx8mp-power.h index 7789bcca322313e2f0e19abfda20ca0dceb62468..2fe3c2abad135fde40fea3379135d3345c93f480 100644 --- a/include/dt-bindings/power/imx8mp-power.h +++ b/include/dt-bindings/power/imx8mp-power.h @@ -49,5 +49,11 @@ #define IMX8MP_HDMIBLK_PD_TRNG 4 #define IMX8MP_HDMIBLK_PD_HDMI_TX 5 #define IMX8MP_HDMIBLK_PD_HDMI_TX_PHY 6 +#define IMX8MP_HDMIBLK_PD_HDCP 7 +#define IMX8MP_HDMIBLK_PD_HRV 8 + +#define IMX8MP_VPUBLK_PD_G1 0 +#define IMX8MP_VPUBLK_PD_G2 1 +#define IMX8MP_VPUBLK_PD_VC8000E 2 #endif diff --git a/include/dt-bindings/power/qcom-rpmpd.h b/include/dt-bindings/power/qcom-rpmpd.h index d81de63ae31cf0403158531e7fa5837777b3ecb4..f5f82dde73998ca2358b0134e54560b0e8a9fa96 100644 --- a/include/dt-bindings/power/qcom-rpmpd.h +++ b/include/dt-bindings/power/qcom-rpmpd.h @@ -36,6 +36,18 @@ #define SM6350_MSS 4 #define SM6350_MX 5 +/* SM6350 Power Domain Indexes */ +#define SM6375_VDDCX 0 +#define SM6375_VDDCX_AO 1 +#define SM6375_VDDCX_VFL 2 +#define SM6375_VDDMX 3 +#define SM6375_VDDMX_AO 4 +#define SM6375_VDDMX_VFL 5 +#define SM6375_VDDGX 6 +#define SM6375_VDDGX_AO 7 +#define SM6375_VDD_LPI_CX 8 +#define SM6375_VDD_LPI_MX 9 + /* SM8150 Power Domain Indexes */ #define SM8150_MSS 0 #define SM8150_EBI 1 diff --git a/include/dt-bindings/power/rk3588-power.h b/include/dt-bindings/power/rk3588-power.h new file mode 100644 index 0000000000000000000000000000000000000000..1b92fec013cb617608d59fd3d37591e8d926f6a8 --- /dev/null +++ b/include/dt-bindings/power/rk3588-power.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: (GPL-2.0 or MIT) */ +#ifndef __DT_BINDINGS_POWER_RK3588_POWER_H__ +#define __DT_BINDINGS_POWER_RK3588_POWER_H__ + +/* VD_LITDSU */ +#define RK3588_PD_CPU_0 0 +#define RK3588_PD_CPU_1 1 +#define RK3588_PD_CPU_2 2 +#define RK3588_PD_CPU_3 3 + +/* VD_BIGCORE0 */ +#define RK3588_PD_CPU_4 4 +#define RK3588_PD_CPU_5 5 + +/* VD_BIGCORE1 */ +#define RK3588_PD_CPU_6 6 +#define RK3588_PD_CPU_7 7 + +/* VD_NPU */ +#define RK3588_PD_NPU 8 +#define RK3588_PD_NPUTOP 9 +#define RK3588_PD_NPU1 10 +#define RK3588_PD_NPU2 11 + +/* VD_GPU */ +#define RK3588_PD_GPU 12 + +/* VD_VCODEC */ +#define RK3588_PD_VCODEC 13 +#define RK3588_PD_RKVDEC0 14 +#define RK3588_PD_RKVDEC1 15 +#define RK3588_PD_VENC0 16 +#define RK3588_PD_VENC1 17 + +/* VD_DD01 */ +#define RK3588_PD_DDR01 18 + +/* VD_DD23 */ +#define RK3588_PD_DDR23 19 + +/* VD_LOGIC */ +#define RK3588_PD_CENTER 20 +#define RK3588_PD_VDPU 21 +#define RK3588_PD_RGA30 22 +#define RK3588_PD_AV1 23 +#define RK3588_PD_VOP 24 +#define RK3588_PD_VO0 25 +#define RK3588_PD_VO1 26 +#define RK3588_PD_VI 27 +#define RK3588_PD_ISP1 28 +#define RK3588_PD_FEC 29 +#define RK3588_PD_RGA31 30 +#define RK3588_PD_USB 31 +#define RK3588_PD_PHP 32 +#define RK3588_PD_GMAC 33 +#define RK3588_PD_PCIE 34 +#define RK3588_PD_NVM 35 +#define RK3588_PD_NVM0 36 +#define RK3588_PD_SDIO 37 +#define RK3588_PD_AUDIO 38 +#define RK3588_PD_SECURE 39 +#define RK3588_PD_SDMMC 40 +#define RK3588_PD_CRYPTO 41 +#define RK3588_PD_BUS 42 + +/* VD_PMU */ +#define RK3588_PD_PMU1 43 + +#endif diff --git a/include/dt-bindings/power/rockchip,rv1126-power.h b/include/dt-bindings/power/rockchip,rv1126-power.h new file mode 100644 index 0000000000000000000000000000000000000000..38a68e000d3884efa3ab808c141203abd6850524 --- /dev/null +++ b/include/dt-bindings/power/rockchip,rv1126-power.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __DT_BINDINGS_POWER_RV1126_POWER_H__ +#define __DT_BINDINGS_POWER_RV1126_POWER_H__ + +/* VD_CORE */ +#define RV1126_PD_CPU_0 0 +#define RV1126_PD_CPU_1 1 +#define RV1126_PD_CPU_2 2 +#define RV1126_PD_CPU_3 3 +#define RV1126_PD_CORE_ALIVE 4 + +/* VD_PMU */ +#define RV1126_PD_PMU 5 +#define RV1126_PD_PMU_ALIVE 6 + +/* VD_NPU */ +#define RV1126_PD_NPU 7 + +/* VD_VEPU */ +#define RV1126_PD_VEPU 8 + +/* VD_LOGIC */ +#define RV1126_PD_VI 9 +#define RV1126_PD_VO 10 +#define RV1126_PD_ISPP 11 +#define RV1126_PD_VDPU 12 +#define RV1126_PD_CRYPTO 13 +#define RV1126_PD_DDR 14 +#define RV1126_PD_NVM 15 +#define RV1126_PD_SDIO 16 +#define RV1126_PD_USB 17 +#define RV1126_PD_LOGIC_ALIVE 18 + +#endif diff --git a/include/dt-bindings/reset/bt1-ccu.h b/include/dt-bindings/reset/bt1-ccu.h index 3578e83026bc888f629748e00dd5d82c60d2fe67..c691efaa678f71feddd584b8303cf3df09a58038 100644 --- a/include/dt-bindings/reset/bt1-ccu.h +++ b/include/dt-bindings/reset/bt1-ccu.h @@ -21,5 +21,14 @@ #define CCU_SYS_SATA_REF_RST 0 #define CCU_SYS_APB_RST 1 +#define CCU_SYS_DDR_FULL_RST 2 +#define CCU_SYS_DDR_INIT_RST 3 +#define CCU_SYS_PCIE_PCS_PHY_RST 4 +#define CCU_SYS_PCIE_PIPE0_RST 5 +#define CCU_SYS_PCIE_CORE_RST 6 +#define CCU_SYS_PCIE_PWR_RST 7 +#define CCU_SYS_PCIE_STICKY_RST 8 +#define CCU_SYS_PCIE_NSTICKY_RST 9 +#define CCU_SYS_PCIE_HOT_RST 10 #endif /* __DT_BINDINGS_RESET_BT1_CCU_H */ diff --git a/include/dt-bindings/reset/mediatek,mt6795-resets.h b/include/dt-bindings/reset/mediatek,mt6795-resets.h new file mode 100644 index 0000000000000000000000000000000000000000..5464a4a79a70e203b9e2e6e9f67f9f30f3ace3cb --- /dev/null +++ b/include/dt-bindings/reset/mediatek,mt6795-resets.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause) */ +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#ifndef _DT_BINDINGS_RESET_CONTROLLER_MT6795 +#define _DT_BINDINGS_RESET_CONTROLLER_MT6795 + +/* INFRACFG resets */ +#define MT6795_INFRA_RST0_SCPSYS_RST 0 +#define MT6795_INFRA_RST0_PMIC_WRAP_RST 1 +#define MT6795_INFRA_RST1_MIPI_DSI_RST 2 +#define MT6795_INFRA_RST1_MIPI_CSI_RST 3 +#define MT6795_INFRA_RST1_MM_IOMMU_RST 4 + +/* MMSYS resets */ +#define MT6795_MMSYS_SW0_RST_B_SMI_COMMON 0 +#define MT6795_MMSYS_SW0_RST_B_SMI_LARB 1 +#define MT6795_MMSYS_SW0_RST_B_CAM_MDP 2 +#define MT6795_MMSYS_SW0_RST_B_MDP_RDMA0 3 +#define MT6795_MMSYS_SW0_RST_B_MDP_RDMA1 4 +#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ0 5 +#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ1 6 +#define MT6795_MMSYS_SW0_RST_B_MDP_RSZ2 7 +#define MT6795_MMSYS_SW0_RST_B_MDP_TDSHP0 8 +#define MT6795_MMSYS_SW0_RST_B_MDP_TDSHP1 9 +#define MT6795_MMSYS_SW0_RST_B_MDP_WDMA 10 +#define MT6795_MMSYS_SW0_RST_B_MDP_WROT0 11 +#define MT6795_MMSYS_SW0_RST_B_MDP_WROT1 12 +#define MT6795_MMSYS_SW0_RST_B_MDP_CROP 13 + +/* PERICFG resets */ +#define MT6795_PERI_NFI_SW_RST 0 +#define MT6795_PERI_THERM_SW_RST 1 +#define MT6795_PERI_MSDC1_SW_RST 2 + +/* TOPRGU resets */ +#define MT6795_TOPRGU_INFRA_SW_RST 0 +#define MT6795_TOPRGU_MM_SW_RST 1 +#define MT6795_TOPRGU_MFG_SW_RST 2 +#define MT6795_TOPRGU_VENC_SW_RST 3 +#define MT6795_TOPRGU_VDEC_SW_RST 4 +#define MT6795_TOPRGU_IMG_SW_RST 5 +#define MT6795_TOPRGU_DDRPHY_SW_RST 6 +#define MT6795_TOPRGU_MD_SW_RST 7 +#define MT6795_TOPRGU_INFRA_AO_SW_RST 8 +#define MT6795_TOPRGU_MD_LITE_SW_RST 9 +#define MT6795_TOPRGU_APMIXED_SW_RST 10 +#define MT6795_TOPRGU_PWRAP_SPI_CTL_RST 11 +#define MT6795_TOPRGU_SW_RST_NUM 12 + +#endif /* _DT_BINDINGS_RESET_CONTROLLER_MT6795 */ diff --git a/include/dt-bindings/reset/mt8195-resets.h b/include/dt-bindings/reset/mt8195-resets.h index 0b1937f14b36a2714698c6de9c6be6bfdcbc60af..24ab3631dceaefb0c3028801c0c36240bce1711d 100644 --- a/include/dt-bindings/reset/mt8195-resets.h +++ b/include/dt-bindings/reset/mt8195-resets.h @@ -31,5 +31,8 @@ #define MT8195_INFRA_RST0_THERM_CTRL_SWRST 0 #define MT8195_INFRA_RST3_THERM_CTRL_PTP_SWRST 1 #define MT8195_INFRA_RST4_THERM_CTRL_MCU_SWRST 2 +#define MT8195_INFRA_RST2_PCIE_P0_SWRST 3 +#define MT8195_INFRA_RST2_PCIE_P1_SWRST 4 +#define MT8195_INFRA_RST2_USBSIF_P1_SWRST 5 #endif /* _DT_BINDINGS_RESET_CONTROLLER_MT8195 */ diff --git a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h index 0d3276c8fc11f36cc540f48b88b66b282fac9da7..9f7c5103bc82b6549595c69dd6a47310dfc264d0 100644 --- a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h +++ b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h @@ -193,6 +193,24 @@ #define LPASS_CLK_ID_RX_CORE_MCLK 59 #define LPASS_CLK_ID_RX_CORE_NPL_MCLK 60 #define LPASS_CLK_ID_VA_CORE_2X_MCLK 61 +/* Clock ID for MCLK for WSA2 core */ +#define LPASS_CLK_ID_WSA2_CORE_MCLK 62 +/* Clock ID for NPL MCLK for WSA2 core */ +#define LPASS_CLK_ID_WSA2_CORE_2X_MCLK 63 +/* Clock ID for RX Core TX MCLK */ +#define LPASS_CLK_ID_RX_CORE_TX_MCLK 64 +/* Clock ID for RX CORE TX 2X MCLK */ +#define LPASS_CLK_ID_RX_CORE_TX_2X_MCLK 65 +/* Clock ID for WSA core TX MCLK */ +#define LPASS_CLK_ID_WSA_CORE_TX_MCLK 66 +/* Clock ID for WSA core TX 2X MCLK */ +#define LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK 67 +/* Clock ID for WSA2 core TX MCLK */ +#define LPASS_CLK_ID_WSA2_CORE_TX_MCLK 68 +/* Clock ID for WSA2 core TX 2X MCLK */ +#define LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK 69 +/* Clock ID for RX CORE MCLK2 2X MCLK */ +#define LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK 70 #define LPASS_HW_AVTIMER_VOTE 101 #define LPASS_HW_MACRO_VOTE 102 diff --git a/include/kunit/assert.h b/include/kunit/assert.h index 4b52e12c2ae838b1286e4a775a582b0f95e1caa3..ace3de8d1ee79a82e005a0a13679c6d9517ef529 100644 --- a/include/kunit/assert.h +++ b/include/kunit/assert.h @@ -42,16 +42,15 @@ struct kunit_loc { /** * struct kunit_assert - Data for printing a failed assertion or expectation. - * @format: a function which formats the data in this kunit_assert to a string. * * Represents a failed expectation/assertion. Contains all the data necessary to * format a string to a user reporting the failure. */ -struct kunit_assert { - void (*format)(const struct kunit_assert *assert, - const struct va_format *message, - struct string_stream *stream); -}; +struct kunit_assert {}; + +typedef void (*assert_format_t)(const struct kunit_assert *assert, + const struct va_format *message, + struct string_stream *stream); void kunit_assert_prologue(const struct kunit_loc *loc, enum kunit_assert_type type, @@ -71,16 +70,6 @@ void kunit_fail_assert_format(const struct kunit_assert *assert, const struct va_format *message, struct string_stream *stream); -/** - * KUNIT_INIT_FAIL_ASSERT_STRUCT - Initializer for &struct kunit_fail_assert. - * - * Initializes a &struct kunit_fail_assert. Intended to be used in - * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros. - */ -#define KUNIT_INIT_FAIL_ASSERT_STRUCT { \ - .assert = { .format = kunit_fail_assert_format }, \ -} - /** * struct kunit_unary_assert - Represents a KUNIT_{EXPECT|ASSERT}_{TRUE|FALSE} * @assert: The parent of this type. @@ -110,7 +99,6 @@ void kunit_unary_assert_format(const struct kunit_assert *assert, * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros. */ #define KUNIT_INIT_UNARY_ASSERT_STRUCT(cond, expect_true) { \ - .assert = { .format = kunit_unary_assert_format }, \ .condition = cond, \ .expected_true = expect_true \ } @@ -145,7 +133,6 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, * KUNIT_EXPECT_* and KUNIT_ASSERT_* macros. */ #define KUNIT_INIT_PTR_NOT_ERR_STRUCT(txt, val) { \ - .assert = { .format = kunit_ptr_not_err_assert_format }, \ .text = txt, \ .value = val \ } @@ -190,7 +177,6 @@ void kunit_binary_assert_format(const struct kunit_assert *assert, * KUNIT_INIT_BINARY_ASSERT_STRUCT() - Initializes a binary assert like * kunit_binary_assert, kunit_binary_ptr_assert, etc. * - * @format_func: a function which formats the assert to a string. * @text_: Pointer to a kunit_binary_assert_text. * @left_val: The actual evaluated value of the expression in the left slot. * @right_val: The actual evaluated value of the expression in the right slot. @@ -200,11 +186,9 @@ void kunit_binary_assert_format(const struct kunit_assert *assert, * fields but with different types for left_val/right_val. * This is ultimately used by binary assertion macros like KUNIT_EXPECT_EQ, etc. */ -#define KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func, \ - text_, \ +#define KUNIT_INIT_BINARY_ASSERT_STRUCT(text_, \ left_val, \ right_val) { \ - .assert = { .format = format_func }, \ .text = text_, \ .left_value = left_val, \ .right_value = right_val \ diff --git a/include/kunit/resource.h b/include/kunit/resource.h index 09c2b34d1c6136c0e84bad8106625121a3c962fa..cf6fb8f2ac1bd15d9a8526cf954c04b8c7fde3be 100644 --- a/include/kunit/resource.h +++ b/include/kunit/resource.h @@ -300,22 +300,6 @@ typedef bool (*kunit_resource_match_t)(struct kunit *test, struct kunit_resource *res, void *match_data); -/** - * kunit_resource_instance_match() - Match a resource with the same instance. - * @test: Test case to which the resource belongs. - * @res: The resource. - * @match_data: The resource pointer to match against. - * - * An instance of kunit_resource_match_t that matches a resource whose - * allocation matches @match_data. - */ -static inline bool kunit_resource_instance_match(struct kunit *test, - struct kunit_resource *res, - void *match_data) -{ - return res->data == match_data; -} - /** * kunit_resource_name_match() - Match a resource with the same name. * @test: Test case to which the resource belongs. diff --git a/include/kunit/test.h b/include/kunit/test.h index c958855681ccdd205bc8aa358aa0a9829d0cfede..b1ab6b32216d79cb4c14bf948a6bfd9e8782a5c9 100644 --- a/include/kunit/test.h +++ b/include/kunit/test.h @@ -228,6 +228,8 @@ static inline void kunit_set_failure(struct kunit *test) WRITE_ONCE(test->status, KUNIT_FAILURE); } +bool kunit_enabled(void); + void kunit_init_test(struct kunit *test, const char *name, char *log); int kunit_run_tests(struct kunit_suite *suite); @@ -251,7 +253,6 @@ static inline int kunit_run_all_tests(void) #endif /* IS_BUILTIN(CONFIG_KUNIT) */ #define __kunit_test_suites(unique_array, ...) \ - MODULE_INFO(test, "Y"); \ static struct kunit_suite *unique_array[] \ __aligned(sizeof(struct kunit_suite *)) \ __used __section(".kunit_test_suites") = { __VA_ARGS__ } @@ -472,30 +473,30 @@ void kunit_do_failed_assertion(struct kunit *test, const struct kunit_loc *loc, enum kunit_assert_type type, const struct kunit_assert *assert, + assert_format_t assert_format, const char *fmt, ...); -#define KUNIT_ASSERTION(test, assert_type, pass, assert_class, INITIALIZER, fmt, ...) do { \ - if (unlikely(!(pass))) { \ - static const struct kunit_loc __loc = KUNIT_CURRENT_LOC; \ - struct assert_class __assertion = INITIALIZER; \ - kunit_do_failed_assertion(test, \ - &__loc, \ - assert_type, \ - &__assertion.assert, \ - fmt, \ - ##__VA_ARGS__); \ - } \ +#define _KUNIT_FAILED(test, assert_type, assert_class, assert_format, INITIALIZER, fmt, ...) do { \ + static const struct kunit_loc __loc = KUNIT_CURRENT_LOC; \ + const struct assert_class __assertion = INITIALIZER; \ + kunit_do_failed_assertion(test, \ + &__loc, \ + assert_type, \ + &__assertion.assert, \ + assert_format, \ + fmt, \ + ##__VA_ARGS__); \ } while (0) #define KUNIT_FAIL_ASSERTION(test, assert_type, fmt, ...) \ - KUNIT_ASSERTION(test, \ - assert_type, \ - false, \ - kunit_fail_assert, \ - KUNIT_INIT_FAIL_ASSERT_STRUCT, \ - fmt, \ - ##__VA_ARGS__) + _KUNIT_FAILED(test, \ + assert_type, \ + kunit_fail_assert, \ + kunit_fail_assert_format, \ + {}, \ + fmt, \ + ##__VA_ARGS__) /** * KUNIT_FAIL() - Always causes a test to fail when evaluated. @@ -520,14 +521,19 @@ void kunit_do_failed_assertion(struct kunit *test, expected_true, \ fmt, \ ...) \ - KUNIT_ASSERTION(test, \ - assert_type, \ - !!(condition) == !!expected_true, \ - kunit_unary_assert, \ - KUNIT_INIT_UNARY_ASSERT_STRUCT(#condition, \ - expected_true), \ - fmt, \ - ##__VA_ARGS__) +do { \ + if (likely(!!(condition) == !!expected_true)) \ + break; \ + \ + _KUNIT_FAILED(test, \ + assert_type, \ + kunit_unary_assert, \ + kunit_unary_assert_format, \ + KUNIT_INIT_UNARY_ASSERT_STRUCT(#condition, \ + expected_true), \ + fmt, \ + ##__VA_ARGS__); \ +} while (0) #define KUNIT_TRUE_MSG_ASSERTION(test, assert_type, condition, fmt, ...) \ KUNIT_UNARY_ASSERTION(test, \ @@ -577,16 +583,18 @@ do { \ .right_text = #right, \ }; \ \ - KUNIT_ASSERTION(test, \ - assert_type, \ - __left op __right, \ - assert_class, \ - KUNIT_INIT_BINARY_ASSERT_STRUCT(format_func, \ - &__text, \ - __left, \ - __right), \ - fmt, \ - ##__VA_ARGS__); \ + if (likely(__left op __right)) \ + break; \ + \ + _KUNIT_FAILED(test, \ + assert_type, \ + assert_class, \ + format_func, \ + KUNIT_INIT_BINARY_ASSERT_STRUCT(&__text, \ + __left, \ + __right), \ + fmt, \ + ##__VA_ARGS__); \ } while (0) #define KUNIT_BINARY_INT_ASSERTION(test, \ @@ -635,16 +643,19 @@ do { \ .right_text = #right, \ }; \ \ - KUNIT_ASSERTION(test, \ - assert_type, \ - strcmp(__left, __right) op 0, \ - kunit_binary_str_assert, \ - KUNIT_INIT_BINARY_ASSERT_STRUCT(kunit_binary_str_assert_format,\ - &__text, \ - __left, \ - __right), \ - fmt, \ - ##__VA_ARGS__); \ + if (likely(strcmp(__left, __right) op 0)) \ + break; \ + \ + \ + _KUNIT_FAILED(test, \ + assert_type, \ + kunit_binary_str_assert, \ + kunit_binary_str_assert_format, \ + KUNIT_INIT_BINARY_ASSERT_STRUCT(&__text, \ + __left, \ + __right), \ + fmt, \ + ##__VA_ARGS__); \ } while (0) #define KUNIT_PTR_NOT_ERR_OR_NULL_MSG_ASSERTION(test, \ @@ -655,14 +666,16 @@ do { \ do { \ const typeof(ptr) __ptr = (ptr); \ \ - KUNIT_ASSERTION(test, \ - assert_type, \ - !IS_ERR_OR_NULL(__ptr), \ - kunit_ptr_not_err_assert, \ - KUNIT_INIT_PTR_NOT_ERR_STRUCT(#ptr, \ - __ptr), \ - fmt, \ - ##__VA_ARGS__); \ + if (!IS_ERR_OR_NULL(__ptr)) \ + break; \ + \ + _KUNIT_FAILED(test, \ + assert_type, \ + kunit_ptr_not_err_assert, \ + kunit_ptr_not_err_assert_format, \ + KUNIT_INIT_PTR_NOT_ERR_STRUCT(#ptr, __ptr), \ + fmt, \ + ##__VA_ARGS__); \ } while (0) /** @@ -826,7 +839,7 @@ do { \ #define KUNIT_EXPECT_LE_MSG(test, left, right, fmt, ...) \ KUNIT_BINARY_INT_ASSERTION(test, \ - KUNIT_ASSERTION, \ + KUNIT_EXPECTATION, \ left, <=, right, \ fmt, \ ##__VA_ARGS__) @@ -1116,7 +1129,7 @@ do { \ #define KUNIT_ASSERT_LT_MSG(test, left, right, fmt, ...) \ KUNIT_BINARY_INT_ASSERTION(test, \ - KUNIT_EXPECTATION, \ + KUNIT_ASSERTION, \ left, <, right, \ fmt, \ ##__VA_ARGS__) @@ -1157,7 +1170,7 @@ do { \ #define KUNIT_ASSERT_GT_MSG(test, left, right, fmt, ...) \ KUNIT_BINARY_INT_ASSERTION(test, \ - KUNIT_EXPECTATION, \ + KUNIT_ASSERTION, \ left, >, right, \ fmt, \ ##__VA_ARGS__) diff --git a/include/linux/a.out.h b/include/linux/a.out.h deleted file mode 100644 index 600cf45645c60c0ebe1d6f8016e9c646051c1adf..0000000000000000000000000000000000000000 --- a/include/linux/a.out.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __A_OUT_GNU_H__ -#define __A_OUT_GNU_H__ - -#include - -#ifndef __ASSEMBLY__ -#ifdef linux -#include -#if defined(__i386__) || defined(__mc68000__) -#else -#ifndef SEGMENT_SIZE -#define SEGMENT_SIZE PAGE_SIZE -#endif -#endif -#endif -#endif /*__ASSEMBLY__ */ -#endif /* __A_OUT_GNU_H__ */ diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 6f64b2f3dc547951b2635beb1e6a11dbd033cbee..3015235d65e317b4e53c93030fce27816f902b26 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -279,14 +279,17 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) { } void acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa); +#if defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH) +void acpi_arch_dma_setup(struct device *dev); +#else +static inline void acpi_arch_dma_setup(struct device *dev) { } +#endif + #ifdef CONFIG_ARM64 void acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa); -void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size); #else static inline void acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) { } -static inline void -acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) { } #endif int acpi_numa_memory_affinity_init (struct acpi_srat_mem_affinity *ma); @@ -495,7 +498,7 @@ bool acpi_dev_resource_address_space(struct acpi_resource *ares, struct resource_win *win); bool acpi_dev_resource_ext_address_space(struct acpi_resource *ares, struct resource_win *win); -unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable); +unsigned long acpi_dev_irq_flags(u8 triggering, u8 polarity, u8 shareable, u8 wake_capable); unsigned int acpi_dev_get_irq_type(int triggering, int polarity); bool acpi_dev_resource_interrupt(struct acpi_resource *ares, int index, struct resource *res); @@ -506,6 +509,7 @@ int acpi_dev_get_resources(struct acpi_device *adev, struct list_head *list, void *preproc_data); int acpi_dev_get_dma_resources(struct acpi_device *adev, struct list_head *list); +int acpi_dev_get_memory_resources(struct acpi_device *adev, struct list_head *list); int acpi_dev_filter_resource_type(struct acpi_resource *ares, unsigned long types); @@ -798,6 +802,11 @@ acpi_dev_hid_uid_match(struct acpi_device *adev, const char *hid2, const char *u return false; } +static inline int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer) +{ + return -ENODEV; +} + static inline struct acpi_device * acpi_dev_get_first_match_dev(const char *hid, const char *uid, s64 hrv) { @@ -977,8 +986,7 @@ static inline enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev) return DEV_DMA_NOT_SUPPORTED; } -static inline int acpi_dma_get_range(struct device *dev, u64 *dma_addr, - u64 *offset, u64 *size) +static inline int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map) { return -ENODEV; } @@ -1075,6 +1083,7 @@ acpi_status acpi_os_prepare_extended_sleep(u8 sleep_state, struct acpi_s2idle_dev_ops { struct list_head list_node; void (*prepare)(void); + void (*check)(void); void (*restore)(void); }; int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg); @@ -1202,7 +1211,8 @@ bool acpi_gpio_get_irq_resource(struct acpi_resource *ares, struct acpi_resource_gpio **agpio); bool acpi_gpio_get_io_resource(struct acpi_resource *ares, struct acpi_resource_gpio **agpio); -int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, int index); +int acpi_dev_gpio_irq_wake_get_by(struct acpi_device *adev, const char *name, int index, + bool *wake_capable); #else static inline bool acpi_gpio_get_irq_resource(struct acpi_resource *ares, struct acpi_resource_gpio **agpio) @@ -1214,16 +1224,28 @@ static inline bool acpi_gpio_get_io_resource(struct acpi_resource *ares, { return false; } -static inline int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, - const char *name, int index) +static inline int acpi_dev_gpio_irq_wake_get_by(struct acpi_device *adev, const char *name, + int index, bool *wake_capable) { return -ENXIO; } #endif +static inline int acpi_dev_gpio_irq_wake_get(struct acpi_device *adev, int index, + bool *wake_capable) +{ + return acpi_dev_gpio_irq_wake_get_by(adev, NULL, index, wake_capable); +} + +static inline int acpi_dev_gpio_irq_get_by(struct acpi_device *adev, const char *name, + int index) +{ + return acpi_dev_gpio_irq_wake_get_by(adev, name, index, NULL); +} + static inline int acpi_dev_gpio_irq_get(struct acpi_device *adev, int index) { - return acpi_dev_gpio_irq_get_by(adev, NULL, index); + return acpi_dev_gpio_irq_wake_get_by(adev, NULL, index, NULL); } /* Device properties */ diff --git a/include/linux/ahci_platform.h b/include/linux/ahci_platform.h index 49e5383d4222257bed2e103fa6c2ec29b836bf00..17fa26215292ecee26ab36c6afef8bca15519f93 100644 --- a/include/linux/ahci_platform.h +++ b/include/linux/ahci_platform.h @@ -13,6 +13,7 @@ #include +struct clk; struct device; struct ata_port_info; struct ahci_host_priv; @@ -21,8 +22,12 @@ struct scsi_host_template; int ahci_platform_enable_phys(struct ahci_host_priv *hpriv); void ahci_platform_disable_phys(struct ahci_host_priv *hpriv); +struct clk *ahci_platform_find_clk(struct ahci_host_priv *hpriv, + const char *con_id); int ahci_platform_enable_clks(struct ahci_host_priv *hpriv); void ahci_platform_disable_clks(struct ahci_host_priv *hpriv); +int ahci_platform_deassert_rsts(struct ahci_host_priv *hpriv); +int ahci_platform_assert_rsts(struct ahci_host_priv *hpriv); int ahci_platform_enable_regulators(struct ahci_host_priv *hpriv); void ahci_platform_disable_regulators(struct ahci_host_priv *hpriv); int ahci_platform_enable_resources(struct ahci_host_priv *hpriv); @@ -41,6 +46,7 @@ int ahci_platform_resume_host(struct device *dev); int ahci_platform_suspend(struct device *dev); int ahci_platform_resume(struct device *dev); -#define AHCI_PLATFORM_GET_RESETS 0x01 +#define AHCI_PLATFORM_GET_RESETS BIT(0) +#define AHCI_PLATFORM_RST_TRIGGER BIT(1) #endif /* _AHCI_PLATFORM_H */ diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index e94cdf235f1db986164a2f9e3b16b4c9077cb196..5001e14c5c06ddac13d6cb64d0d50d8e8b4c7b17 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -67,6 +67,7 @@ struct amba_device { struct clk *pclk; struct device_dma_parameters dma_parms; unsigned int periphid; + struct mutex periphid_lock; unsigned int cid; struct amba_cs_uci_id uci; unsigned int irq[AMBA_NR_IRQS]; diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h new file mode 100644 index 0000000000000000000000000000000000000000..1c4b8659f171d3ac9aa80e58402ea92ff0cb9f6a --- /dev/null +++ b/include/linux/amd-pstate.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * linux/include/linux/amd-pstate.h + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. + * + * Author: Meng Li + */ + +#ifndef _LINUX_AMD_PSTATE_H +#define _LINUX_AMD_PSTATE_H + +#include + +/********************************************************************* + * AMD P-state INTERFACE * + *********************************************************************/ +/** + * struct amd_aperf_mperf + * @aperf: actual performance frequency clock count + * @mperf: maximum performance frequency clock count + * @tsc: time stamp counter + */ +struct amd_aperf_mperf { + u64 aperf; + u64 mperf; + u64 tsc; +}; + +/** + * struct amd_cpudata - private CPU data for AMD P-State + * @cpu: CPU number + * @req: constraint request to apply + * @cppc_req_cached: cached performance request hints + * @highest_perf: the maximum performance an individual processor may reach, + * assuming ideal conditions + * @nominal_perf: the maximum sustained performance level of the processor, + * assuming ideal operating conditions + * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power + * savings are achieved + * @lowest_perf: the absolute lowest performance level of the processor + * @max_freq: the frequency that mapped to highest_perf + * @min_freq: the frequency that mapped to lowest_perf + * @nominal_freq: the frequency that mapped to nominal_perf + * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf + * @cur: Difference of Aperf/Mperf/tsc count between last and current sample + * @prev: Last Aperf/Mperf/tsc count value read from register + * @freq: current cpu frequency value + * @boost_supported: check whether the Processor or SBIOS supports boost mode + * + * The amd_cpudata is key private data for each CPU thread in AMD P-State, and + * represents all the attributes and goals that AMD P-State requests at runtime. + */ +struct amd_cpudata { + int cpu; + + struct freq_qos_request req[2]; + u64 cppc_req_cached; + + u32 highest_perf; + u32 nominal_perf; + u32 lowest_nonlinear_perf; + u32 lowest_perf; + + u32 max_freq; + u32 min_freq; + u32 nominal_freq; + u32 lowest_nonlinear_freq; + + struct amd_aperf_mperf cur; + struct amd_aperf_mperf prev; + + u64 freq; + bool boost_supported; +}; + +#endif /* _LINUX_AMD_PSTATE_H */ diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index e5c76c1ef9edfed92f2f1edd4e46b968a47fe7aa..5f02d2e6b9d9d8b06c4faf334312dfc86791255c 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -17,6 +17,7 @@ struct ffa_device { bool mode_32bit; uuid_t uuid; struct device dev; + const struct ffa_ops *ops; }; #define to_ffa_dev(d) container_of(d, struct ffa_device, dev) @@ -47,17 +48,18 @@ static inline void *ffa_dev_get_drvdata(struct ffa_device *fdev) } #if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT) -struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id); +struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id, + const struct ffa_ops *ops); void ffa_device_unregister(struct ffa_device *ffa_dev); int ffa_driver_register(struct ffa_driver *driver, struct module *owner, const char *mod_name); void ffa_driver_unregister(struct ffa_driver *driver); bool ffa_device_is_valid(struct ffa_device *ffa_dev); -const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev); #else static inline -struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id) +struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id, + const struct ffa_ops *ops) { return NULL; } @@ -76,11 +78,6 @@ static inline void ffa_driver_unregister(struct ffa_driver *driver) {} static inline bool ffa_device_is_valid(struct ffa_device *ffa_dev) { return false; } -static inline -const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev) -{ - return NULL; -} #endif /* CONFIG_ARM_FFA_TRANSPORT */ #define ffa_register(driver) \ @@ -109,7 +106,10 @@ struct ffa_partition_info { #define FFA_PARTITION_DIRECT_SEND BIT(1) /* partition can send and receive indirect messages. */ #define FFA_PARTITION_INDIRECT_MSG BIT(2) +/* partition runs in the AArch64 execution state. */ +#define FFA_PARTITION_AARCH64_EXEC BIT(8) u32 properties; + u32 uuid[4]; }; /* For use with FFA_MSG_SEND_DIRECT_{REQ,RESP} which pass data via registers */ @@ -257,18 +257,28 @@ struct ffa_mem_ops_args { struct ffa_mem_region_attributes *attrs; }; -struct ffa_dev_ops { +struct ffa_info_ops { u32 (*api_version_get)(void); int (*partition_info_get)(const char *uuid_str, struct ffa_partition_info *buffer); +}; + +struct ffa_msg_ops { void (*mode_32bit_set)(struct ffa_device *dev); int (*sync_send_receive)(struct ffa_device *dev, struct ffa_send_direct_data *data); +}; + +struct ffa_mem_ops { int (*memory_reclaim)(u64 g_handle, u32 flags); - int (*memory_share)(struct ffa_device *dev, - struct ffa_mem_ops_args *args); - int (*memory_lend)(struct ffa_device *dev, - struct ffa_mem_ops_args *args); + int (*memory_share)(struct ffa_mem_ops_args *args); + int (*memory_lend)(struct ffa_mem_ops_args *args); +}; + +struct ffa_ops { + const struct ffa_info_ops *info_ops; + const struct ffa_msg_ops *msg_ops; + const struct ffa_mem_ops *mem_ops; }; #endif /* _LINUX_ARM_FFA_H */ diff --git a/include/linux/ata.h b/include/linux/ata.h index 21292b5bbb5509787a2f78d551e73b41aa950966..e3050e153a7162652232c97d37b8d3716310368b 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -566,6 +566,18 @@ struct ata_bmdma_prd { ((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \ ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \ ((id)[ATA_ID_FEATURE_SUPP] & (1 << 2))) +#define ata_id_has_devslp(id) \ + ((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \ + ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \ + ((id)[ATA_ID_FEATURE_SUPP] & (1 << 8))) +#define ata_id_has_ncq_autosense(id) \ + ((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \ + ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \ + ((id)[ATA_ID_FEATURE_SUPP] & (1 << 7))) +#define ata_id_has_dipm(id) \ + ((((id)[ATA_ID_SATA_CAPABILITY] != 0x0000) && \ + ((id)[ATA_ID_SATA_CAPABILITY] != 0xffff)) && \ + ((id)[ATA_ID_FEATURE_SUPP] & (1 << 3))) #define ata_id_iordy_disable(id) ((id)[ATA_ID_CAPABILITY] & (1 << 10)) #define ata_id_has_iordy(id) ((id)[ATA_ID_CAPABILITY] & (1 << 11)) #define ata_id_u32(id,n) \ @@ -578,9 +590,6 @@ struct ata_bmdma_prd { #define ata_id_cdb_intr(id) (((id)[ATA_ID_CONFIG] & 0x60) == 0x20) #define ata_id_has_da(id) ((id)[ATA_ID_SATA_CAPABILITY_2] & (1 << 4)) -#define ata_id_has_devslp(id) ((id)[ATA_ID_FEATURE_SUPP] & (1 << 8)) -#define ata_id_has_ncq_autosense(id) \ - ((id)[ATA_ID_FEATURE_SUPP] & (1 << 7)) static inline bool ata_id_has_hipm(const u16 *id) { @@ -592,17 +601,6 @@ static inline bool ata_id_has_hipm(const u16 *id) return val & (1 << 9); } -static inline bool ata_id_has_dipm(const u16 *id) -{ - u16 val = id[ATA_ID_FEATURE_SUPP]; - - if (val == 0 || val == 0xffff) - return false; - - return val & (1 << 3); -} - - static inline bool ata_id_has_fua(const u16 *id) { if ((id[ATA_ID_CFSSE] & 0xC000) != 0x4000) @@ -771,16 +769,21 @@ static inline bool ata_id_has_read_log_dma_ext(const u16 *id) static inline bool ata_id_has_sense_reporting(const u16 *id) { - if (!(id[ATA_ID_CFS_ENABLE_2] & (1 << 15))) + if (!(id[ATA_ID_CFS_ENABLE_2] & BIT(15))) + return false; + if ((id[ATA_ID_COMMAND_SET_3] & (BIT(15) | BIT(14))) != BIT(14)) return false; - return id[ATA_ID_COMMAND_SET_3] & (1 << 6); + return id[ATA_ID_COMMAND_SET_3] & BIT(6); } static inline bool ata_id_sense_reporting_enabled(const u16 *id) { - if (!(id[ATA_ID_CFS_ENABLE_2] & (1 << 15))) + if (!ata_id_has_sense_reporting(id)) + return false; + /* ata_id_has_sense_reporting() == true, word 86 must have bit 15 set */ + if ((id[ATA_ID_COMMAND_SET_4] & (BIT(15) | BIT(14))) != BIT(14)) return false; - return id[ATA_ID_COMMAND_SET_4] & (1 << 6); + return id[ATA_ID_COMMAND_SET_4] & BIT(6); } /** diff --git a/include/linux/bcma/bcma_driver_chipcommon.h b/include/linux/bcma/bcma_driver_chipcommon.h index e3314f746bfa09dd2254e23e31e310f8b1ab68b5..2d94c30ed43979f5181b5acfbf93795e6cfa496c 100644 --- a/include/linux/bcma/bcma_driver_chipcommon.h +++ b/include/linux/bcma/bcma_driver_chipcommon.h @@ -271,6 +271,7 @@ #define BCMA_CC_SROM_CONTROL_OP_WRDIS 0x40000000 #define BCMA_CC_SROM_CONTROL_OP_WREN 0x60000000 #define BCMA_CC_SROM_CONTROL_OTPSEL 0x00000010 +#define BCMA_CC_SROM_CONTROL_OTP_PRESENT 0x00000020 #define BCMA_CC_SROM_CONTROL_LOCK 0x00000008 #define BCMA_CC_SROM_CONTROL_SIZE_MASK 0x00000006 #define BCMA_CC_SROM_CONTROL_SIZE_1K 0x00000000 diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 3dc20c4f394cf9cc0041d0f8d56787662f81845a..8d51f69f9f5ef8adcb05ee9eef0ced6818b76161 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -43,9 +43,6 @@ struct linux_binprm { * original userspace. */ point_of_no_return:1; -#ifdef __alpha__ - unsigned int taso:1; -#endif struct file *executable; /* Executable to pass to the interpreter */ struct file *interpreter; struct file *file; diff --git a/include/linux/bio.h b/include/linux/bio.h index ca22b06700a94d319a443f4d38225f7755e2e958..2c5806997bbf71a5edefd641fbd8e0d05be23111 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -509,7 +509,7 @@ static inline void bio_set_dev(struct bio *bio, struct block_device *bdev) { bio_clear_flag(bio, BIO_REMAPPED); if (bio->bi_bdev != bdev) - bio_clear_flag(bio, BIO_THROTTLED); + bio_clear_flag(bio, BIO_BPS_THROTTLED); bio->bi_bdev = bdev; bio_associate_blkg(bio); } diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index f65410a49fdac03139a11e68eab650f5bd29a197..7d6d73b781472ffbff8d22283ad9b21330960bad 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -51,6 +51,7 @@ struct device; * bitmap_empty(src, nbits) Are all bits zero in *src? * bitmap_full(src, nbits) Are all bits set in *src? * bitmap_weight(src, nbits) Hamming Weight: number set bits + * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap * bitmap_set(dst, pos, nbits) Set specified bit area * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area @@ -164,6 +165,8 @@ bool __bitmap_intersects(const unsigned long *bitmap1, bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); +unsigned int __bitmap_weight_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int nbits); void __bitmap_set(unsigned long *map, unsigned int start, int len); void __bitmap_clear(unsigned long *map, unsigned int start, int len); @@ -222,7 +225,6 @@ void bitmap_copy_le(unsigned long *dst, const unsigned long *src, unsigned int n #else #define bitmap_copy_le bitmap_copy #endif -unsigned int bitmap_ord_to_pos(const unsigned long *bitmap, unsigned int ord, unsigned int nbits); int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, int nmaskbits); @@ -439,6 +441,15 @@ unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits) return __bitmap_weight(src, nbits); } +static __always_inline +unsigned long bitmap_weight_and(const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return hweight_long(*src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)); + return __bitmap_weight_and(src1, src2, nbits); +} + static __always_inline void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) { diff --git a/include/linux/bitops.h b/include/linux/bitops.h index cf9bf65039f22b14b8f810cadc0301397c2303e2..2ba557e067fe69d4d96b5472721001e87a6535cc 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -59,6 +59,7 @@ extern unsigned long __sw_hweight64(__u64 w); #define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) #define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) #define test_bit(nr, addr) bitop(_test_bit, nr, addr) +#define test_bit_acquire(nr, addr) bitop(_test_bit_acquire, nr, addr) /* * Include this here because some architectures need generic_ffs/fls in @@ -246,6 +247,25 @@ static inline unsigned long __ffs64(u64 word) return __ffs((unsigned long)word); } +/** + * fns - find N'th set bit in a word + * @word: The word to search + * @n: Bit to find + */ +static inline unsigned long fns(unsigned long word, unsigned int n) +{ + unsigned int bit; + + while (word) { + bit = __ffs(word); + if (n-- == 0) + return bit; + __clear_bit(bit, &word); + } + + return BITS_PER_LONG; +} + /** * assign_bit - Assign value to a bit in memory * @nr: the bit to set @@ -327,10 +347,10 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr, const typeof(*(ptr)) mask__ = (mask), bits__ = (bits); \ typeof(*(ptr)) old__, new__; \ \ + old__ = READ_ONCE(*(ptr)); \ do { \ - old__ = READ_ONCE(*(ptr)); \ new__ = (old__ & ~mask__) | bits__; \ - } while (cmpxchg(ptr, old__, new__) != old__); \ + } while (!try_cmpxchg(ptr, &old__, new__)); \ \ old__; \ }) @@ -342,11 +362,12 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr, const typeof(*(ptr)) clear__ = (clear), test__ = (test);\ typeof(*(ptr)) old__, new__; \ \ + old__ = READ_ONCE(*(ptr)); \ do { \ - old__ = READ_ONCE(*(ptr)); \ + if (old__ & test__) \ + break; \ new__ = old__ & ~clear__; \ - } while (!(old__ & test__) && \ - cmpxchg(ptr, old__, new__) != old__); \ + } while (!try_cmpxchg(ptr, &old__, new__)); \ \ !(old__ & test__); \ }) diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index 9f40dbc65f82c0e8ad44eaefb7b25a27accc5bf3..dd5841a42c331831ee432ceee8c74078519100fc 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -18,14 +18,14 @@ struct bio; struct cgroup_subsys_state; -struct request_queue; +struct gendisk; #define FC_APPID_LEN 129 #ifdef CONFIG_BLK_CGROUP extern struct cgroup_subsys_state * const blkcg_root_css; -void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay); +void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay); void blkcg_maybe_throttle_current(void); bool blk_cgroup_congested(void); void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css); @@ -39,7 +39,6 @@ struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio); static inline void blkcg_maybe_throttle_current(void) { } static inline bool blk_cgroup_congested(void) { return false; } -static inline void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay) { } static inline struct cgroup_subsys_state *bio_blkcg_css(struct bio *bio) { return NULL; diff --git a/include/linux/blk-mq-pci.h b/include/linux/blk-mq-pci.h index 0b1f45c62623891ac1743a3f8ded7a3da12a80b1..ca544e1d3508f34ab6e198b0bb17efe88de4d14d 100644 --- a/include/linux/blk-mq-pci.h +++ b/include/linux/blk-mq-pci.h @@ -5,7 +5,7 @@ struct blk_mq_queue_map; struct pci_dev; -int blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev, - int offset); +void blk_mq_pci_map_queues(struct blk_mq_queue_map *qmap, struct pci_dev *pdev, + int offset); #endif /* _LINUX_BLK_MQ_PCI_H */ diff --git a/include/linux/blk-mq-rdma.h b/include/linux/blk-mq-rdma.h index 5cc5f0f36218fefabf979bd0bf13005e0a7817af..53b58c610e7671eb37a9925819c890f19677747a 100644 --- a/include/linux/blk-mq-rdma.h +++ b/include/linux/blk-mq-rdma.h @@ -5,7 +5,7 @@ struct blk_mq_tag_set; struct ib_device; -int blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, +void blk_mq_rdma_map_queues(struct blk_mq_queue_map *map, struct ib_device *dev, int first_vec); #endif /* _LINUX_BLK_MQ_RDMA_H */ diff --git a/include/linux/blk-mq-virtio.h b/include/linux/blk-mq-virtio.h index 687ae287e1dc2c447add315f1d1abe8e90e4153b..13226e9b22dd53e4289d506d49c52671de036ee8 100644 --- a/include/linux/blk-mq-virtio.h +++ b/include/linux/blk-mq-virtio.h @@ -5,7 +5,7 @@ struct blk_mq_queue_map; struct virtio_device; -int blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap, +void blk_mq_virtio_map_queues(struct blk_mq_queue_map *qmap, struct virtio_device *vdev, int first_vec); #endif /* _LINUX_BLK_MQ_VIRTIO_H */ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 92294a5fb083612e578532362160c468d0e435c8..ba18e9bdb799b28a3edf9c4aff389bcbf784c5f4 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -14,7 +14,12 @@ struct blk_flush_queue; #define BLKDEV_MIN_RQ 4 #define BLKDEV_DEFAULT_RQ 128 -typedef void (rq_end_io_fn)(struct request *, blk_status_t); +enum rq_end_io_ret { + RQ_END_IO_NONE, + RQ_END_IO_FREE, +}; + +typedef enum rq_end_io_ret (rq_end_io_fn)(struct request *, blk_status_t); /* * request flags */ @@ -268,9 +273,16 @@ static inline void rq_list_move(struct request **src, struct request **dst, rq_list_add(dst, rq); } +/** + * enum blk_eh_timer_return - How the timeout handler should proceed + * @BLK_EH_DONE: The block driver completed the command or will complete it at + * a later time. + * @BLK_EH_RESET_TIMER: Reset the request timer and continue waiting for the + * request to complete. + */ enum blk_eh_timer_return { - BLK_EH_DONE, /* drivers has completed the command */ - BLK_EH_RESET_TIMER, /* reset timer and try again */ + BLK_EH_DONE, + BLK_EH_RESET_TIMER, }; #define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */ @@ -630,7 +642,7 @@ struct blk_mq_ops { * @map_queues: This allows drivers specify their own queue mapping by * overriding the setup-time function that builds the mq_map. */ - int (*map_queues)(struct blk_mq_tag_set *set); + void (*map_queues)(struct blk_mq_tag_set *set); #ifdef CONFIG_BLK_DEBUG_FS /** @@ -841,8 +853,9 @@ static inline bool blk_mq_add_to_batch(struct request *req, struct io_comp_batch *iob, int ioerror, void (*complete)(struct io_comp_batch *)) { - if (!iob || (req->rq_flags & RQF_ELV) || req->end_io || ioerror) + if (!iob || (req->rq_flags & RQF_ELV) || ioerror) return false; + if (!iob->complete) iob->complete = complete; else if (iob->complete != complete) @@ -880,7 +893,7 @@ void blk_mq_freeze_queue_wait(struct request_queue *q); int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, unsigned long timeout); -int blk_mq_map_queues(struct blk_mq_queue_map *qmap); +void blk_mq_map_queues(struct blk_mq_queue_map *qmap); void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues); void blk_mq_quiesce_queue_nowait(struct request_queue *q); @@ -963,15 +976,17 @@ blk_status_t blk_insert_cloned_request(struct request *rq); struct rq_map_data { struct page **pages; - int page_order; - int nr_entries; unsigned long offset; - int null_mapped; - int from_user; + unsigned short page_order; + unsigned short nr_entries; + bool null_mapped; + bool from_user; }; int blk_rq_map_user(struct request_queue *, struct request *, struct rq_map_data *, void __user *, unsigned long, gfp_t); +int blk_rq_map_user_io(struct request *, struct rq_map_data *, + void __user *, unsigned long, gfp_t, bool, int, bool, int); int blk_rq_map_user_iov(struct request_queue *, struct request *, struct rq_map_data *, const struct iov_iter *, gfp_t); int blk_rq_unmap_user(struct bio *); @@ -980,6 +995,7 @@ int blk_rq_map_kern(struct request_queue *, struct request *, void *, int blk_rq_append_bio(struct request *rq, struct bio *bio); void blk_execute_rq_nowait(struct request *rq, bool at_head); blk_status_t blk_execute_rq(struct request *rq, bool at_head); +bool blk_rq_is_poll(struct request *rq); struct req_iterator { struct bvec_iter iter; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 1ef99790f6ed3eee3c11498571ffd6d5c5010e4f..e0b098089ef2b7c818c73fd1219322eaeedd3367 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -321,11 +321,10 @@ enum { BIO_NO_PAGE_REF, /* don't put release vec pages */ BIO_CLONED, /* doesn't own data */ BIO_BOUNCED, /* bio is a bounce bio */ - BIO_WORKINGSET, /* contains userspace workingset pages */ BIO_QUIET, /* Make BIO Quiet */ BIO_CHAIN, /* chained bio, ->bi_remaining in effect */ BIO_REFFED, /* bio has elevated ->bi_cnt */ - BIO_THROTTLED, /* This bio has already been subjected to + BIO_BPS_THROTTLED, /* This bio has already been subjected to * throttling rules. Don't do it again. */ BIO_TRACE_COMPLETION, /* bio_endio() should trace the final completion * of this bio. */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 84b13fdd34a716b539199dec1975c1d17aa309a0..50e358a19d98643b2bc927372cb2c46b6d4ca52c 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -580,9 +580,9 @@ struct request_queue { #define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */ #define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */ -#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ - (1 << QUEUE_FLAG_SAME_COMP) | \ - (1 << QUEUE_FLAG_NOWAIT)) +#define QUEUE_FLAG_MQ_DEFAULT ((1UL << QUEUE_FLAG_IO_STAT) | \ + (1UL << QUEUE_FLAG_SAME_COMP) | \ + (1UL << QUEUE_FLAG_NOWAIT)) void blk_queue_flag_set(unsigned int flag, struct request_queue *q); void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); @@ -618,7 +618,6 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); #define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) #define blk_queue_pm_only(q) atomic_read(&(q)->pm_only) #define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags) -#define blk_queue_nowait(q) test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags) #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) extern void blk_set_pm_only(struct request_queue *q); @@ -1280,6 +1279,11 @@ static inline bool bdev_fua(struct block_device *bdev) return test_bit(QUEUE_FLAG_FUA, &bdev_get_queue(bdev)->queue_flags); } +static inline bool bdev_nowait(struct block_device *bdev) +{ + return test_bit(QUEUE_FLAG_NOWAIT, &bdev_get_queue(bdev)->queue_flags); +} + static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev) { struct request_queue *q = bdev_get_queue(bdev); @@ -1300,6 +1304,15 @@ static inline bool bdev_is_zoned(struct block_device *bdev) return false; } +static inline bool bdev_op_is_zoned_write(struct block_device *bdev, + blk_opf_t op) +{ + if (!bdev_is_zoned(bdev)) + return false; + + return op == REQ_OP_WRITE || op == REQ_OP_WRITE_ZEROES; +} + static inline sector_t bdev_zone_sectors(struct block_device *bdev) { struct request_queue *q = bdev_get_queue(bdev); @@ -1498,6 +1511,7 @@ int sync_blockdev(struct block_device *bdev); int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend); int sync_blockdev_nowait(struct block_device *bdev); void sync_bdevs(bool wait); +void bdev_statx_dioalign(struct inode *inode, struct kstat *stat); void printk_all_partitions(void); #else static inline void invalidate_bdev(struct block_device *bdev) @@ -1514,6 +1528,9 @@ static inline int sync_blockdev_nowait(struct block_device *bdev) static inline void sync_bdevs(bool wait) { } +static inline void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) +{ +} static inline void printk_all_partitions(void) { } diff --git a/include/linux/bma150.h b/include/linux/bma150.h index 31c9e323a3913d76bb6e986e86a268c8a5d2d209..4d4a62d493419486002d3a799c3b59039618f2ac 100644 --- a/include/linux/bma150.h +++ b/include/linux/bma150.h @@ -33,8 +33,8 @@ struct bma150_cfg { unsigned char lg_hyst; /* Low-G hysterisis */ unsigned char lg_dur; /* Low-G duration */ unsigned char lg_thres; /* Low-G threshold */ - unsigned char range; /* one of BMA0150_RANGE_xxx */ - unsigned char bandwidth; /* one of BMA0150_BW_xxx */ + unsigned char range; /* one of BMA150_RANGE_xxx */ + unsigned char bandwidth; /* one of BMA150_BW_xxx */ }; struct bma150_platform_data { diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 2bd1b5f8de9b7c7f3d94f813ba3f6912ce6bf2ed..57e9e109257e5db61c4d83ab37199097010354e9 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -414,6 +414,11 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); int cgroup_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr); + +const struct bpf_func_proto * +cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); +const struct bpf_func_proto * +cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); #else static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; } @@ -444,6 +449,18 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, return -EINVAL; } +static inline const struct bpf_func_proto * +cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + return NULL; +} + +static inline const struct bpf_func_proto * +cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + return NULL; +} + static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map) { return 0; } static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc( diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 20c26aed78962e43ffff7e3d6ec0c8ff99507219..9e7d46d16032f98236dbf35d4390b956a0dbdaa3 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -48,6 +48,7 @@ struct mem_cgroup; struct module; struct bpf_func_state; struct ftrace_ops; +struct cgroup; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; @@ -279,14 +280,33 @@ static inline void check_and_init_map_value(struct bpf_map *map, void *dst) } } -/* copy everything but bpf_spin_lock and bpf_timer. There could be one of each. */ -static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) +/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and + * forced to use 'long' read/writes to try to atomically copy long counters. + * Best-effort only. No barriers here, since it _will_ race with concurrent + * updates from BPF programs. Called from bpf syscall and mostly used with + * size 8 or 16 bytes, so ask compiler to inline it. + */ +static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) +{ + const long *lsrc = src; + long *ldst = dst; + + size /= sizeof(long); + while (size--) + *ldst++ = *lsrc++; +} + +/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */ +static inline void __copy_map_value(struct bpf_map *map, void *dst, void *src, bool long_memcpy) { u32 curr_off = 0; int i; if (likely(!map->off_arr)) { - memcpy(dst, src, map->value_size); + if (long_memcpy) + bpf_long_memcpy(dst, src, round_up(map->value_size, 8)); + else + memcpy(dst, src, map->value_size); return; } @@ -298,6 +318,36 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) } memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off); } + +static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) +{ + __copy_map_value(map, dst, src, false); +} + +static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) +{ + __copy_map_value(map, dst, src, true); +} + +static inline void zero_map_value(struct bpf_map *map, void *dst) +{ + u32 curr_off = 0; + int i; + + if (likely(!map->off_arr)) { + memset(dst, 0, map->value_size); + return; + } + + for (i = 0; i < map->off_arr->cnt; i++) { + u32 next_off = map->off_arr->field_off[i]; + + memset(dst + curr_off, 0, next_off - curr_off); + curr_off += map->off_arr->field_sz[i]; + } + memset(dst + curr_off, 0, map->value_size - curr_off); +} + void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, bool lock_src); void bpf_timer_cancel_and_free(void *timer); @@ -401,7 +451,7 @@ enum bpf_type_flag { /* DYNPTR points to memory local to the bpf program. */ DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS), - /* DYNPTR points to a ringbuf record. */ + /* DYNPTR points to a kernel-produced ringbuf record. */ DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), /* Size is known at compile time. */ @@ -606,6 +656,7 @@ enum bpf_reg_type { PTR_TO_MEM, /* reg points to valid memory region */ PTR_TO_BUF, /* reg points to a read/write buffer */ PTR_TO_FUNC, /* reg points to a bpf program function */ + PTR_TO_DYNPTR, /* reg points to a dynptr */ __BPF_REG_TYPE_MAX, /* Extended reg_types. */ @@ -726,10 +777,14 @@ enum bpf_cgroup_storage_type { */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* The argument is a structure. */ +#define BTF_FMODEL_STRUCT_ARG BIT(0) + struct btf_func_model { u8 ret_size; u8 nr_args; u8 arg_size[MAX_BPF_FUNC_ARGS]; + u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; /* Restore arguments before returning from trampoline to let original function @@ -809,6 +864,10 @@ u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); +u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx); +void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); @@ -891,6 +950,7 @@ struct bpf_dispatcher { struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; int num_progs; void *image; + void *rw_image; u32 image_off; struct bpf_ksym ksym; }; @@ -909,7 +969,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); -int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); +int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \ @@ -923,7 +983,14 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); }, \ } +#ifdef CONFIG_X86_64 +#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5))) +#else +#define BPF_DISPATCHER_ATTRIBUTES +#endif + #define DEFINE_BPF_DISPATCHER(name) \ + notrace BPF_DISPATCHER_ATTRIBUTES \ noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ @@ -945,7 +1012,6 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs); void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, struct bpf_prog *to); /* Called only from JIT-enabled code, so there's no need for stubs. */ -void *bpf_jit_alloc_exec_page(void); void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym); void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); @@ -1333,6 +1399,11 @@ struct bpf_array { #define BPF_MAP_CAN_READ BIT(0) #define BPF_MAP_CAN_WRITE BIT(1) +/* Maximum number of user-producer ring buffer samples that can be drained in + * a call to bpf_user_ringbuf_drain(). + */ +#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) + static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) { u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); @@ -1729,8 +1800,40 @@ int bpf_obj_get_user(const char __user *pathname, int flags); extern int bpf_iter_ ## target(args); \ int __init bpf_iter_ ## target(args) { return 0; } +/* + * The task type of iterators. + * + * For BPF task iterators, they can be parameterized with various + * parameters to visit only some of tasks. + * + * BPF_TASK_ITER_ALL (default) + * Iterate over resources of every task. + * + * BPF_TASK_ITER_TID + * Iterate over resources of a task/tid. + * + * BPF_TASK_ITER_TGID + * Iterate over resources of every task of a process / task group. + */ +enum bpf_iter_task_type { + BPF_TASK_ITER_ALL = 0, + BPF_TASK_ITER_TID, + BPF_TASK_ITER_TGID, +}; + struct bpf_iter_aux_info { + /* for map_elem iter */ struct bpf_map *map; + + /* for cgroup iter */ + struct { + struct cgroup *start; /* starting cgroup */ + enum bpf_cgroup_iter_order order; + } cgroup; + struct { + enum bpf_iter_task_type type; + u32 pid; + } task; }; typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, @@ -1815,22 +1918,6 @@ int bpf_get_file_flag(int flags); int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, size_t actual_size); -/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and - * forced to use 'long' read/writes to try to atomically copy long counters. - * Best-effort only. No barriers here, since it _will_ race with concurrent - * updates from BPF programs. Called from bpf syscall and mostly used with - * size 8 or 16 bytes, so ask compiler to inline it. - */ -static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) -{ - const long *lsrc = src; - long *ldst = dst; - - size /= sizeof(long); - while (size--) - *ldst++ = *lsrc++; -} - /* verify correctness of eBPF program */ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr); @@ -1932,13 +2019,22 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, const char *func_name, struct btf_func_model *m); +struct bpf_kfunc_arg_meta { + u64 r0_size; + bool r0_rdonly; + int ref_obj_id; + u32 flags; +}; + struct bpf_reg_state; int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *regs); +int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, + struct bpf_reg_state *regs); int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, - u32 kfunc_flags); + struct bpf_kfunc_arg_meta *meta); int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, struct bpf_reg_state *reg); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, @@ -1966,6 +2062,17 @@ static inline bool unprivileged_ebpf_enabled(void) return !sysctl_unprivileged_bpf_disabled; } +/* Not all bpf prog type has the bpf_ctx. + * For the bpf prog type that has initialized the bpf_ctx, + * this function can be used to decide if a kernel function + * is called by a bpf program. + */ +static inline bool has_current_bpf_ctx(void) +{ + return !!current->bpf_ctx; +} + +void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog); #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) { @@ -2148,6 +2255,15 @@ static inline struct bpf_prog *bpf_prog_by_id(u32 id) return ERR_PTR(-ENOTSUPP); } +static inline int btf_struct_access(struct bpf_verifier_log *log, + const struct btf *btf, + const struct btf_type *t, int off, int size, + enum bpf_access_type atype, + u32 *next_btf_id, enum bpf_type_flag *flag) +{ + return -EACCES; +} + static inline const struct bpf_func_proto * bpf_base_func_proto(enum bpf_func_id func_id) { @@ -2175,6 +2291,14 @@ static inline bool unprivileged_ebpf_enabled(void) return false; } +static inline bool has_current_bpf_ctx(void) +{ + return false; +} + +static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog) +{ +} #endif /* CONFIG_BPF_SYSCALL */ void __bpf_free_used_btfs(struct bpf_prog_aux *aux, @@ -2349,6 +2473,7 @@ extern const struct bpf_func_proto bpf_get_numa_node_id_proto; extern const struct bpf_func_proto bpf_tail_call_proto; extern const struct bpf_func_proto bpf_ktime_get_ns_proto; extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; +extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto; extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; extern const struct bpf_func_proto bpf_get_current_comm_proto; @@ -2361,6 +2486,7 @@ extern const struct bpf_func_proto bpf_sock_map_update_proto; extern const struct bpf_func_proto bpf_sock_hash_update_proto; extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; +extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto; extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; extern const struct bpf_func_proto bpf_msg_redirect_map_proto; extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; @@ -2410,6 +2536,7 @@ extern const struct bpf_func_proto bpf_loop_proto; extern const struct bpf_func_proto bpf_copy_from_user_task_proto; extern const struct bpf_func_proto bpf_set_retval_proto; extern const struct bpf_func_proto bpf_get_retval_proto; +extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); @@ -2554,7 +2681,7 @@ enum bpf_dynptr_type { BPF_DYNPTR_TYPE_INVALID, /* Points to memory that is local to the bpf program */ BPF_DYNPTR_TYPE_LOCAL, - /* Underlying data is a ringbuf record */ + /* Underlying data is a kernel-produced ringbuf record */ BPF_DYNPTR_TYPE_RINGBUF, }; @@ -2562,6 +2689,7 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, enum bpf_dynptr_type type, u32 offset, u32 size); void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); int bpf_dynptr_check_size(u32 size); +u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr); #ifdef CONFIG_BPF_LSM void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); @@ -2571,4 +2699,12 @@ static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} static inline void bpf_cgroup_atype_put(int cgroup_atype) {} #endif /* CONFIG_BPF_LSM */ +struct key; + +#ifdef CONFIG_KEYS +struct bpf_key { + struct key *key; + bool has_ref; +}; +#endif /* CONFIG_KEYS */ #endif /* _LINUX_BPF_H */ diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h new file mode 100644 index 0000000000000000000000000000000000000000..3e164b8efaa9260dec8d554b327111c6b29856c8 --- /dev/null +++ b/include/linux/bpf_mem_alloc.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ +#ifndef _BPF_MEM_ALLOC_H +#define _BPF_MEM_ALLOC_H +#include +#include + +struct bpf_mem_cache; +struct bpf_mem_caches; + +struct bpf_mem_alloc { + struct bpf_mem_caches __percpu *caches; + struct bpf_mem_cache __percpu *cache; + struct work_struct work; +}; + +int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu); +void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma); + +/* kmalloc/kfree equivalent: */ +void *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size); +void bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr); + +/* kmem_cache_alloc/free equivalent: */ +void *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma); +void bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr); + +#endif /* _BPF_MEM_ALLOC_H */ diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index 2b9112b8017124b1184a4bf7d6b966148b4b25ae..2c6a4f2562a7f8e661aebf932a80e8fa679e2195 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -126,6 +126,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops) #endif BPF_MAP_TYPE(BPF_MAP_TYPE_RINGBUF, ringbuf_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops) +BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops) BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint) BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING, tracing) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 2e3bad8640dc49ccc680bce53ef175cbf7f20c8e..9e1e6965f40747e5097e6e817d4f878582266a25 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -212,6 +212,17 @@ struct bpf_reference_state { * is used purely to inform the user of a reference leak. */ int insn_idx; + /* There can be a case like: + * main (frame 0) + * cb (frame 1) + * func (frame 3) + * cb (frame 4) + * Hence for frame 4, if callback_ref just stored boolean, it would be + * impossible to distinguish nested callback refs. Hence store the + * frameno and compare that to callback_ref in check_reference_leak when + * exiting a callback function. + */ + int callback_ref; }; /* state of the program: @@ -237,6 +248,7 @@ struct bpf_func_state { */ u32 async_entry_cnt; bool in_callback_fn; + struct tnum callback_ret_range; bool in_async_callback_fn; /* The following fields should be last. See copy_func_state() */ @@ -337,6 +349,27 @@ struct bpf_verifier_state { iter < frame->allocated_stack / BPF_REG_SIZE; \ iter++, reg = bpf_get_spilled_reg(iter, frame)) +/* Invoke __expr over regsiters in __vst, setting __state and __reg */ +#define bpf_for_each_reg_in_vstate(__vst, __state, __reg, __expr) \ + ({ \ + struct bpf_verifier_state *___vstate = __vst; \ + int ___i, ___j; \ + for (___i = 0; ___i <= ___vstate->curframe; ___i++) { \ + struct bpf_reg_state *___regs; \ + __state = ___vstate->frame[___i]; \ + ___regs = __state->regs; \ + for (___j = 0; ___j < MAX_BPF_REG; ___j++) { \ + __reg = &___regs[___j]; \ + (void)(__expr); \ + } \ + bpf_for_each_spilled_reg(___j, __state, __reg) { \ + if (!__reg) \ + continue; \ + (void)(__expr); \ + } \ + } \ + }) + /* linked list of verifier states used to prune search */ struct bpf_verifier_state_list { struct bpf_verifier_state state; @@ -560,6 +593,11 @@ int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state u32 regno); int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, u32 regno, u32 mem_size); +bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, + struct bpf_reg_state *reg); +bool is_dynptr_type_expected(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + enum bpf_arg_type arg_type); /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */ static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog, @@ -587,6 +625,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, struct bpf_attach_target_info *tgt_info); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); +int mark_chain_precision(struct bpf_verifier_env *env, int regno); + #define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0) /* extract base type from bpf_{arg, return, reg}_type. */ diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 6ff567ece34a9d6a1fe344faf2b674eec85e0745..9e77165f3ef6d18ded32d2442736148adab16df2 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -293,6 +293,7 @@ #define MII_BRCM_FET_SHDW_MC_FAME 0x4000 /* Force Auto MDIX enable */ #define MII_BRCM_FET_SHDW_AUXMODE4 0x1a /* Auxiliary mode 4 */ +#define MII_BRCM_FET_SHDW_AM4_STANDBY 0x0008 /* Standby enable */ #define MII_BRCM_FET_SHDW_AM4_LED_MASK 0x0003 #define MII_BRCM_FET_SHDW_AM4_LED_MODE1 0x0001 diff --git a/include/linux/btf.h b/include/linux/btf.h index cdb376d53238f48154547ca929f207e2681ffb46..f9aababc5d78c0011ab5a2aabcb4de326bd48c6b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -49,6 +49,17 @@ * for this case. */ #define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */ +#define KF_SLEEPABLE (1 << 5) /* kfunc may sleep */ +#define KF_DESTRUCTIVE (1 << 6) /* kfunc performs destructive actions */ + +/* + * Return the name of the passed struct, if exists, or halt the build if for + * example the structure gets renamed. In this way, developers have to revisit + * the code using that structure name, and update it accordingly. + */ +#define stringify_struct(x) \ + ({ BUILD_BUG_ON(sizeof(struct x) < 0); \ + __stringify(x); }) struct btf; struct btf_member; @@ -439,4 +450,14 @@ static inline int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dt } #endif +static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + t = btf_type_skip_modifiers(btf, t->type, NULL); + + return btf_type_is_struct(t); +} + #endif diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index def8b8d30ccc12563418509ad3368099fa7c404d..33fa5e94aa8071247ced1f32a61a9803e6b409ee 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -137,6 +137,17 @@ BUFFER_FNS(Defer_Completion, defer_completion) static __always_inline void set_buffer_uptodate(struct buffer_head *bh) { + /* + * If somebody else already set this uptodate, they will + * have done the memory barrier, and a reader will thus + * see *some* valid buffer state. + * + * Any other serialization (with IO errors or whatever that + * might clear the bit) has to come from other state (eg BH_Lock). + */ + if (test_bit(BH_Uptodate, &bh->b_state)) + return; + /* * make it consistent with folio_mark_uptodate * pairs with smp_load_acquire in buffer_uptodate @@ -156,7 +167,7 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh) * make it consistent with folio_test_uptodate * pairs with smp_mb__before_atomic in set_buffer_uptodate */ - return (smp_load_acquire(&bh->b_state) & (1UL << BH_Uptodate)) != 0; + return test_bit_acquire(BH_Uptodate, &bh->b_state); } #define bh_offset(bh) ((unsigned long)(bh)->b_data & ~PAGE_MASK) @@ -214,8 +225,6 @@ struct buffer_head *__getblk_gfp(struct block_device *bdev, sector_t block, void __brelse(struct buffer_head *); void __bforget(struct buffer_head *); void __breadahead(struct block_device *, sector_t block, unsigned int size); -void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size, - gfp_t gfp); struct buffer_head *__bread_gfp(struct block_device *, sector_t block, unsigned size, gfp_t gfp); void invalidate_bh_lrus(void); @@ -225,15 +234,16 @@ struct buffer_head *alloc_buffer_head(gfp_t gfp_flags); void free_buffer_head(struct buffer_head * bh); void unlock_buffer(struct buffer_head *bh); void __lock_buffer(struct buffer_head *bh); -void ll_rw_block(blk_opf_t, int, struct buffer_head * bh[]); int sync_dirty_buffer(struct buffer_head *bh); int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); -int submit_bh(blk_opf_t, struct buffer_head *); +void submit_bh(blk_opf_t, struct buffer_head *); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); int bh_uptodate_or_lock(struct buffer_head *bh); -int bh_submit_read(struct buffer_head *bh); +int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait); +void __bh_read_batch(int nr, struct buffer_head *bhs[], + blk_opf_t op_flags, bool force_lock); extern int buffer_heads_over_limit; @@ -340,12 +350,6 @@ sb_breadahead(struct super_block *sb, sector_t block) __breadahead(sb->s_bdev, block, sb->s_blocksize); } -static inline void -sb_breadahead_unmovable(struct super_block *sb, sector_t block) -{ - __breadahead_gfp(sb->s_bdev, block, sb->s_blocksize, 0); -} - static inline struct buffer_head * sb_getblk(struct super_block *sb, sector_t block) { @@ -407,6 +411,41 @@ static inline struct buffer_head *__getblk(struct block_device *bdev, return __getblk_gfp(bdev, block, size, __GFP_MOVABLE); } +static inline void bh_readahead(struct buffer_head *bh, blk_opf_t op_flags) +{ + if (!buffer_uptodate(bh) && trylock_buffer(bh)) { + if (!buffer_uptodate(bh)) + __bh_read(bh, op_flags, false); + else + unlock_buffer(bh); + } +} + +static inline void bh_read_nowait(struct buffer_head *bh, blk_opf_t op_flags) +{ + if (!bh_uptodate_or_lock(bh)) + __bh_read(bh, op_flags, false); +} + +/* Returns 1 if buffer uptodated, 0 on success, and -EIO on error. */ +static inline int bh_read(struct buffer_head *bh, blk_opf_t op_flags) +{ + if (bh_uptodate_or_lock(bh)) + return 1; + return __bh_read(bh, op_flags, true); +} + +static inline void bh_read_batch(int nr, struct buffer_head *bhs[]) +{ + __bh_read_batch(nr, bhs, 0, true); +} + +static inline void bh_readahead_batch(int nr, struct buffer_head *bhs[], + blk_opf_t op_flags) +{ + __bh_read_batch(nr, bhs, op_flags, false); +} + /** * __bread() - reads a specified block and returns the bh * @bdev: the block_device to read from diff --git a/include/linux/cache.h b/include/linux/cache.h index d742c57eaee598cb3213e870108486ecaa1d5367..5da1bbd96154b6840334223e495e5a8837617e49 100644 --- a/include/linux/cache.h +++ b/include/linux/cache.h @@ -85,4 +85,17 @@ #define cache_line_size() L1_CACHE_BYTES #endif +/* + * Helper to add padding within a struct to ensure data fall into separate + * cachelines. + */ +#if defined(CONFIG_SMP) +struct cacheline_padding { + char x[0]; +} ____cacheline_internodealigned_in_smp; +#define CACHELINE_PADDING(name) struct cacheline_padding name +#else +#define CACHELINE_PADDING(name) +#endif + #endif /* __LINUX_CACHE_H */ diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index c3e50e537e397e0ce44aac9cb6641b76d2d8b91d..58f5431a555984a1683f70ddea5c63d40517322a 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -147,6 +147,11 @@ static inline u32 can_get_static_ctrlmode(struct can_priv *priv) return priv->ctrlmode & ~priv->ctrlmode_supported; } +static inline bool can_is_canxl_dev_mtu(unsigned int mtu) +{ + return (mtu >= CANXL_MIN_MTU && mtu <= CANXL_MAX_MTU); +} + void can_setup(struct net_device *dev); struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index 182749e858b3c3018dd89b57bebe66204f0585ab..1abc25a8d144e891c9a54cb8171ec44b25b83602 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -20,7 +20,8 @@ void can_flush_echo_skb(struct net_device *dev); int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev, unsigned int idx, unsigned int frame_len); struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx, - u8 *len_ptr, unsigned int *frame_len_ptr); + unsigned int *len_ptr, + unsigned int *frame_len_ptr); unsigned int __must_check can_get_echo_skb(struct net_device *dev, unsigned int idx, unsigned int *frame_len_ptr); @@ -29,6 +30,9 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx, struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf); struct sk_buff *alloc_canfd_skb(struct net_device *dev, struct canfd_frame **cfd); +struct sk_buff *alloc_canxl_skb(struct net_device *dev, + struct canxl_frame **cxl, + unsigned int data_len); struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf); bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb); @@ -97,10 +101,59 @@ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb) return nskb; } +static inline bool can_is_can_skb(const struct sk_buff *skb) +{ + struct can_frame *cf = (struct can_frame *)skb->data; + + /* the CAN specific type of skb is identified by its data length */ + return (skb->len == CAN_MTU && cf->len <= CAN_MAX_DLEN); +} + static inline bool can_is_canfd_skb(const struct sk_buff *skb) { + struct canfd_frame *cfd = (struct canfd_frame *)skb->data; + /* the CAN specific type of skb is identified by its data length */ - return skb->len == CANFD_MTU; + return (skb->len == CANFD_MTU && cfd->len <= CANFD_MAX_DLEN); +} + +static inline bool can_is_canxl_skb(const struct sk_buff *skb) +{ + const struct canxl_frame *cxl = (struct canxl_frame *)skb->data; + + if (skb->len < CANXL_HDR_SIZE + CANXL_MIN_DLEN || skb->len > CANXL_MTU) + return false; + + /* this also checks valid CAN XL data length boundaries */ + if (skb->len != CANXL_HDR_SIZE + cxl->len) + return false; + + return cxl->flags & CANXL_XLF; +} + +/* get length element value from can[|fd|xl]_frame structure */ +static inline unsigned int can_skb_get_len_val(struct sk_buff *skb) +{ + const struct canxl_frame *cxl = (struct canxl_frame *)skb->data; + const struct canfd_frame *cfd = (struct canfd_frame *)skb->data; + + if (can_is_canxl_skb(skb)) + return cxl->len; + + return cfd->len; +} + +/* get needed data length inside CAN frame for all frame types (RTR aware) */ +static inline unsigned int can_skb_get_data_len(struct sk_buff *skb) +{ + unsigned int len = can_skb_get_len_val(skb); + const struct can_frame *cf = (struct can_frame *)skb->data; + + /* RTR frames have an actual length of zero */ + if (can_is_can_skb(skb) && cf->can_id & CAN_RTR_FLAG) + return 0; + + return len; } #endif /* !_CAN_SKB_H */ diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index e7f2fb2fc2079717142683260332b9c77c3ddcae..99c1726be6ee7e92f6f183614c4b5bc3ab35d6e0 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -207,7 +207,6 @@ struct ceph_msg_data_cursor { struct ceph_msg_data *data; /* current data item */ size_t resid; /* bytes not yet consumed */ - bool last_piece; /* current is last piece */ bool need_crc; /* crc update needed */ union { #ifdef CONFIG_BLOCK @@ -498,8 +497,7 @@ void ceph_con_discard_requeued(struct ceph_connection *con, u64 reconnect_seq); void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor, struct ceph_msg *msg, size_t length); struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor, - size_t *page_offset, size_t *length, - bool *last_piece); + size_t *page_offset, size_t *length); void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes); u32 ceph_crc32c_page(u32 crc, struct page *page, unsigned int page_offset, diff --git a/include/linux/cfi.h b/include/linux/cfi.h index c6dfc1ed06261a3eacd6941ad24371f63d7842fe..5e134f4ce8b75943cb7470ae0490e84dc50daa1a 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -2,49 +2,38 @@ /* * Clang Control Flow Integrity (CFI) support. * - * Copyright (C) 2021 Google LLC + * Copyright (C) 2022 Google LLC */ #ifndef _LINUX_CFI_H #define _LINUX_CFI_H -#ifdef CONFIG_CFI_CLANG -typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag); - -/* Compiler-generated function in each module, and the kernel */ -extern void __cfi_check(uint64_t id, void *ptr, void *diag); - -/* - * Force the compiler to generate a CFI jump table entry for a function - * and store the jump table address to __cfi_jt_. - */ -#define __CFI_ADDRESSABLE(fn, __attr) \ - const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn - -#ifdef CONFIG_CFI_CLANG_SHADOW - -extern void cfi_module_add(struct module *mod, unsigned long base_addr); -extern void cfi_module_remove(struct module *mod, unsigned long base_addr); - -#else +#include +#include -static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {} -static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {} - -#endif /* CONFIG_CFI_CLANG_SHADOW */ - -#else /* !CONFIG_CFI_CLANG */ - -#ifdef CONFIG_X86_KERNEL_IBT - -#define __CFI_ADDRESSABLE(fn, __attr) \ - const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn +#ifdef CONFIG_CFI_CLANG +enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, + unsigned long *target, u32 type); -#endif /* CONFIG_X86_KERNEL_IBT */ +static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs, + unsigned long addr) +{ + return report_cfi_failure(regs, addr, NULL, 0); +} +#ifdef CONFIG_ARCH_USES_CFI_TRAPS +bool is_cfi_trap(unsigned long addr); +#endif #endif /* CONFIG_CFI_CLANG */ -#ifndef __CFI_ADDRESSABLE -#define __CFI_ADDRESSABLE(fn, __attr) -#endif +#ifdef CONFIG_MODULES +#ifdef CONFIG_ARCH_USES_CFI_TRAPS +void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod); +#else +static inline void module_cfi_finalize(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, + struct module *mod) {} +#endif /* CONFIG_ARCH_USES_CFI_TRAPS */ +#endif /* CONFIG_MODULES */ #endif /* _LINUX_CFI_H */ diff --git a/include/linux/cfi_types.h b/include/linux/cfi_types.h new file mode 100644 index 0000000000000000000000000000000000000000..6b87136757655c05d8254b76120f14d32a32f2ac --- /dev/null +++ b/include/linux/cfi_types.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Clang Control Flow Integrity (CFI) type definitions. + */ +#ifndef _LINUX_CFI_TYPES_H +#define _LINUX_CFI_TYPES_H + +#ifdef __ASSEMBLY__ +#include + +#ifdef CONFIG_CFI_CLANG +/* + * Use the __kcfi_typeid_ type identifier symbol to + * annotate indirectly called assembly functions. The compiler emits + * these symbols for all address-taken function declarations in C + * code. + */ +#ifndef __CFI_TYPE +#define __CFI_TYPE(name) \ + .4byte __kcfi_typeid_##name +#endif + +#define SYM_TYPED_ENTRY(name, linkage, align...) \ + linkage(name) ASM_NL \ + align ASM_NL \ + __CFI_TYPE(name) ASM_NL \ + name: + +#define SYM_TYPED_START(name, linkage, align...) \ + SYM_TYPED_ENTRY(name, linkage, align) + +#else /* CONFIG_CFI_CLANG */ + +#define SYM_TYPED_START(name, linkage, align...) \ + SYM_START(name, linkage, align) + +#endif /* CONFIG_CFI_CLANG */ + +#ifndef SYM_TYPED_FUNC_START +#define SYM_TYPED_FUNC_START(name) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + +#endif /* __ASSEMBLY__ */ +#endif /* _LINUX_CFI_TYPES_H */ diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 4bcf56b3491ca00bcc3276ddb2d63d0e28f4ee17..6e01f10f0d88999216270d54031ecb4d7be4d9b5 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -126,11 +126,11 @@ enum { CFTYPE_NO_PREFIX = (1 << 3), /* (DON'T USE FOR NEW FILES) no subsys prefix */ CFTYPE_WORLD_WRITABLE = (1 << 4), /* (DON'T USE FOR NEW FILES) S_IWUGO */ CFTYPE_DEBUG = (1 << 5), /* create when cgroup_debug */ - CFTYPE_PRESSURE = (1 << 6), /* only if pressure feature is enabled */ /* internal flags, do not use outside cgroup core proper */ __CFTYPE_ONLY_ON_DFL = (1 << 16), /* only on default hierarchy */ __CFTYPE_NOT_ON_DFL = (1 << 17), /* not on default hierarchy */ + __CFTYPE_ADDED = (1 << 18), }; /* @@ -384,7 +384,7 @@ struct cgroup { /* * The depth this cgroup is at. The root is at depth zero and each * step down the hierarchy increments the level. This along with - * ancestor_ids[] can determine whether a given cgroup is a + * ancestors[] can determine whether a given cgroup is a * descendant of another without traversing the hierarchy. */ int level; @@ -428,6 +428,9 @@ struct cgroup { struct cgroup_file procs_file; /* handle for "cgroup.procs" */ struct cgroup_file events_file; /* handle for "cgroup.events" */ + /* handles for "{cpu,memory,io,irq}.pressure" */ + struct cgroup_file psi_files[NR_PSI_RESOURCES]; + /* * The bitmask of subsystems enabled on the child cgroups. * ->subtree_control is the one configured through @@ -504,8 +507,8 @@ struct cgroup { /* Used to store internal freezer state */ struct cgroup_freezer_state freezer; - /* ids of the ancestors at each level including self */ - u64 ancestor_ids[]; + /* All ancestors including self */ + struct cgroup *ancestors[]; }; /* @@ -522,11 +525,15 @@ struct cgroup_root { /* Unique id for this hierarchy. */ int hierarchy_id; - /* The root cgroup. Root is destroyed on its release. */ + /* + * The root cgroup. The containing cgroup_root will be destroyed on its + * release. cgrp->ancestors[0] will be used overflowing into the + * following field. cgrp_ancestor_storage must immediately follow. + */ struct cgroup cgrp; - /* for cgrp->ancestor_ids[0] */ - u64 cgrp_ancestor_id_storage; + /* must follow cgrp for cgrp->ancestors[0], see above */ + struct cgroup *cgrp_ancestor_storage; /* Number of cgroups in the hierarchy, used only for /proc/cgroups */ atomic_t nr_cgrps; diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index ed53bfe7c46c4738df18744cc45ff6a204721e11..f2a9f2274c3bb22b78f3e2190b13ee4af0bec1ac 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -114,6 +114,7 @@ int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts); int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts); int cgroup_rm_cftypes(struct cftype *cfts); void cgroup_file_notify(struct cgroup_file *cfile); +void cgroup_file_show(struct cgroup_file *cfile, bool show); int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen); int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry); @@ -432,6 +433,18 @@ static inline void cgroup_put(struct cgroup *cgrp) css_put(&cgrp->self); } +extern struct mutex cgroup_mutex; + +static inline void cgroup_lock(void) +{ + mutex_lock(&cgroup_mutex); +} + +static inline void cgroup_unlock(void) +{ + mutex_unlock(&cgroup_mutex); +} + /** * task_css_set_check - obtain a task's css_set with extra access conditions * @task: the task to obtain css_set for @@ -446,7 +459,6 @@ static inline void cgroup_put(struct cgroup *cgrp) * as locks used during the cgroup_subsys::attach() methods. */ #ifdef CONFIG_PROVE_RCU -extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; #define task_css_set_check(task, __c) \ rcu_dereference_check((task)->cgroups, \ @@ -574,7 +586,7 @@ static inline bool cgroup_is_descendant(struct cgroup *cgrp, { if (cgrp->root != ancestor->root || cgrp->level < ancestor->level) return false; - return cgrp->ancestor_ids[ancestor->level] == cgroup_id(ancestor); + return cgrp->ancestors[ancestor->level] == ancestor; } /** @@ -591,11 +603,9 @@ static inline bool cgroup_is_descendant(struct cgroup *cgrp, static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp, int ancestor_level) { - if (cgrp->level < ancestor_level) + if (ancestor_level < 0 || ancestor_level > cgrp->level) return NULL; - while (cgrp && cgrp->level > ancestor_level) - cgrp = cgroup_parent(cgrp); - return cgrp; + return cgrp->ancestors[ancestor_level]; } /** @@ -672,11 +682,6 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp) pr_cont_kernfs_path(cgrp->kn); } -static inline struct psi_group *cgroup_psi(struct cgroup *cgrp) -{ - return cgrp->psi; -} - bool cgroup_psi_enabled(void); static inline void cgroup_init_kthreadd(void) @@ -708,6 +713,8 @@ struct cgroup; static inline u64 cgroup_id(const struct cgroup *cgrp) { return 1; } static inline void css_get(struct cgroup_subsys_state *css) {} static inline void css_put(struct cgroup_subsys_state *css) {} +static inline void cgroup_lock(void) {} +static inline void cgroup_unlock(void) {} static inline int cgroup_attach_task_all(struct task_struct *from, struct task_struct *t) { return 0; } static inline int cgroupstats_build(struct cgroupstats *stats, @@ -734,11 +741,6 @@ static inline struct cgroup *cgroup_parent(struct cgroup *cgrp) return NULL; } -static inline struct psi_group *cgroup_psi(struct cgroup *cgrp) -{ - return NULL; -} - static inline bool cgroup_psi_enabled(void) { return false; @@ -752,11 +754,6 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task, static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) {} - -static inline struct cgroup *cgroup_get_from_id(u64 id) -{ - return NULL; -} #endif /* !CONFIG_CGROUPS */ #ifdef CONFIG_CGROUPS diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 1615010aa0ecd669305300d63d93aef9d1be3f8d..267cd06b54a0196704e1d395f44ee20c1fed263c 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -42,6 +42,8 @@ struct dentry; * struct clk_rate_request - Structure encoding the clk constraints that * a clock user might require. * + * Should be initialized by calling clk_hw_init_rate_request(). + * * @rate: Requested clock rate. This field will be adjusted by * clock drivers according to hardware capabilities. * @min_rate: Minimum rate imposed by clk users. @@ -60,6 +62,15 @@ struct clk_rate_request { struct clk_hw *best_parent_hw; }; +void clk_hw_init_rate_request(const struct clk_hw *hw, + struct clk_rate_request *req, + unsigned long rate); +void clk_hw_forward_rate_request(const struct clk_hw *core, + const struct clk_rate_request *old_req, + const struct clk_hw *parent, + struct clk_rate_request *req, + unsigned long parent_rate); + /** * struct clk_duty - Struture encoding the duty cycle ratio of a clock * @@ -118,8 +129,9 @@ struct clk_duty { * * @recalc_rate Recalculate the rate of this clock, by querying hardware. The * parent rate is an input parameter. It is up to the caller to - * ensure that the prepare_mutex is held across this call. - * Returns the calculated rate. Optional, but recommended - if + * ensure that the prepare_mutex is held across this call. If the + * driver cannot figure out a rate for this clock, it must return + * 0. Returns the calculated rate. Optional, but recommended - if * this op is not set then clock rate will be initialized to 0. * * @round_rate: Given a target rate as input, returns the closest rate actually @@ -350,7 +362,7 @@ struct clk_hw *__clk_hw_register_fixed_rate(struct device *dev, const char *parent_name, const struct clk_hw *parent_hw, const struct clk_parent_data *parent_data, unsigned long flags, unsigned long fixed_rate, unsigned long fixed_accuracy, - unsigned long clk_fixed_flags); + unsigned long clk_fixed_flags, bool devm); struct clk *clk_register_fixed_rate(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned long fixed_rate); @@ -365,7 +377,20 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, */ #define clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \ - NULL, (flags), (fixed_rate), 0, 0) + NULL, (flags), (fixed_rate), 0, 0, false) + +/** + * devm_clk_hw_register_fixed_rate - register fixed-rate clock with the clock + * framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_name: name of clock's parent + * @flags: framework-specific flags + * @fixed_rate: non-adjustable clock rate + */ +#define devm_clk_hw_register_fixed_rate(dev, name, parent_name, flags, fixed_rate) \ + __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), NULL, \ + NULL, (flags), (fixed_rate), 0, 0, true) /** * clk_hw_register_fixed_rate_parent_hw - register fixed-rate clock with * the clock framework @@ -378,7 +403,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, #define clk_hw_register_fixed_rate_parent_hw(dev, name, parent_hw, flags, \ fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw), \ - NULL, (flags), (fixed_rate), 0, 0) + NULL, (flags), (fixed_rate), 0, 0, false) /** * clk_hw_register_fixed_rate_parent_data - register fixed-rate clock with * the clock framework @@ -392,7 +417,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, fixed_rate) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), (flags), (fixed_rate), 0, \ - 0) + 0, false) /** * clk_hw_register_fixed_rate_with_accuracy - register fixed-rate clock with * the clock framework @@ -408,7 +433,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, fixed_accuracy) \ __clk_hw_register_fixed_rate((dev), NULL, (name), (parent_name), \ NULL, NULL, (flags), (fixed_rate), \ - (fixed_accuracy), 0) + (fixed_accuracy), 0, false) /** * clk_hw_register_fixed_rate_with_accuracy_parent_hw - register fixed-rate * clock with the clock framework @@ -423,7 +448,7 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, parent_hw, flags, fixed_rate, fixed_accuracy) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, (parent_hw) \ NULL, NULL, (flags), (fixed_rate), \ - (fixed_accuracy), 0) + (fixed_accuracy), 0, false) /** * clk_hw_register_fixed_rate_with_accuracy_parent_data - register fixed-rate * clock with the clock framework @@ -438,7 +463,21 @@ struct clk *clk_register_fixed_rate(struct device *dev, const char *name, parent_data, flags, fixed_rate, fixed_accuracy) \ __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ (parent_data), NULL, (flags), \ - (fixed_rate), (fixed_accuracy), 0) + (fixed_rate), (fixed_accuracy), 0, false) +/** + * clk_hw_register_fixed_rate_parent_accuracy - register fixed-rate clock with + * the clock framework + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_name: name of clock's parent + * @flags: framework-specific flags + * @fixed_rate: non-adjustable clock rate + */ +#define clk_hw_register_fixed_rate_parent_accuracy(dev, name, parent_data, \ + flags, fixed_rate) \ + __clk_hw_register_fixed_rate((dev), NULL, (name), NULL, NULL, \ + (parent_data), (flags), (fixed_rate), 0, \ + CLK_FIXED_RATE_PARENT_ACCURACY, false) void clk_unregister_fixed_rate(struct clk *clk); void clk_hw_unregister_fixed_rate(struct clk_hw *hw); @@ -957,6 +996,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name, (parent_names), NULL, NULL, (flags), (reg), \ (shift), (mask), (clk_mux_flags), (table), \ (lock)) +#define clk_hw_register_mux_table_parent_data(dev, name, parent_data, \ + num_parents, flags, reg, shift, mask, \ + clk_mux_flags, table, lock) \ + __clk_hw_register_mux((dev), NULL, (name), (num_parents), \ + NULL, NULL, (parent_data), (flags), (reg), \ + (shift), (mask), (clk_mux_flags), (table), \ + (lock)) #define clk_hw_register_mux(dev, name, parent_names, num_parents, flags, reg, \ shift, width, clk_mux_flags, lock) \ __clk_hw_register_mux((dev), NULL, (name), (num_parents), \ @@ -974,6 +1020,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name, __clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL, NULL, \ (parent_data), (flags), (reg), (shift), \ BIT((width)) - 1, (clk_mux_flags), NULL, (lock)) +#define clk_hw_register_mux_parent_data_table(dev, name, parent_data, \ + num_parents, flags, reg, shift, \ + width, clk_mux_flags, table, \ + lock) \ + __clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL, NULL, \ + (parent_data), (flags), (reg), (shift), \ + BIT((width)) - 1, (clk_mux_flags), table, (lock)) #define devm_clk_hw_register_mux(dev, name, parent_names, num_parents, flags, reg, \ shift, width, clk_mux_flags, lock) \ __devm_clk_hw_register_mux((dev), NULL, (name), (num_parents), \ @@ -987,6 +1040,13 @@ struct clk *clk_register_mux_table(struct device *dev, const char *name, (parent_hws), NULL, (flags), (reg), \ (shift), BIT((width)) - 1, \ (clk_mux_flags), NULL, (lock)) +#define devm_clk_hw_register_mux_parent_data_table(dev, name, parent_data, \ + num_parents, flags, reg, shift, \ + width, clk_mux_flags, table, \ + lock) \ + __devm_clk_hw_register_mux((dev), NULL, (name), (num_parents), NULL, \ + NULL, (parent_data), (flags), (reg), (shift), \ + BIT((width)) - 1, (clk_mux_flags), table, (lock)) int clk_mux_val_to_index(struct clk_hw *hw, const u32 *table, unsigned int flags, unsigned int val); @@ -1255,6 +1315,8 @@ int clk_mux_determine_rate_flags(struct clk_hw *hw, struct clk_rate_request *req, unsigned long flags); void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent); +void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate, + unsigned long *max_rate); void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate, unsigned long max_rate); @@ -1454,7 +1516,7 @@ int devm_of_clk_add_hw_provider(struct device *dev, void *data), void *data); void of_clk_del_provider(struct device_node *np); -void devm_of_clk_del_provider(struct device *dev); + struct clk *of_clk_src_simple_get(struct of_phandle_args *clkspec, void *data); struct clk_hw *of_clk_hw_simple_get(struct of_phandle_args *clkspec, @@ -1491,7 +1553,7 @@ static inline int devm_of_clk_add_hw_provider(struct device *dev, return 0; } static inline void of_clk_del_provider(struct device_node *np) {} -static inline void devm_of_clk_del_provider(struct device *dev) {} + static inline struct clk *of_clk_src_simple_get( struct of_phandle_args *clkspec, void *data) { diff --git a/include/linux/clk.h b/include/linux/clk.h index c13061cabdfc904cb909002824479809fc014ffb..1ef0133242374bab763e97aed82c4ae58b1129ec 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -799,7 +799,7 @@ int clk_set_rate_exclusive(struct clk *clk, unsigned long rate); * * Returns true if @parent is a possible parent for @clk, false otherwise. */ -bool clk_has_parent(struct clk *clk, struct clk *parent); +bool clk_has_parent(const struct clk *clk, const struct clk *parent); /** * clk_set_rate_range - set a rate range for a clock source diff --git a/include/linux/clk/at91_pmc.h b/include/linux/clk/at91_pmc.h index 3484309b59bf1855244b97d7b89863fe4539d273..7af499bdbecb9dedd8309821ffb541ad33bf1d50 100644 --- a/include/linux/clk/at91_pmc.h +++ b/include/linux/clk/at91_pmc.h @@ -12,6 +12,8 @@ #ifndef AT91_PMC_H #define AT91_PMC_H +#include + #define AT91_PMC_V1 (1) /* PMC version 1 */ #define AT91_PMC_V2 (2) /* PMC version 2 [SAM9X60] */ @@ -45,8 +47,8 @@ #define AT91_PMC_PCSR 0x18 /* Peripheral Clock Status Register */ #define AT91_PMC_PLL_ACR 0x18 /* PLL Analog Control Register [for SAM9X60] */ -#define AT91_PMC_PLL_ACR_DEFAULT_UPLL 0x12020010UL /* Default PLL ACR value for UPLL */ -#define AT91_PMC_PLL_ACR_DEFAULT_PLLA 0x00020010UL /* Default PLL ACR value for PLLA */ +#define AT91_PMC_PLL_ACR_DEFAULT_UPLL UL(0x12020010) /* Default PLL ACR value for UPLL */ +#define AT91_PMC_PLL_ACR_DEFAULT_PLLA UL(0x00020010) /* Default PLL ACR value for PLLA */ #define AT91_PMC_PLL_ACR_UTMIVR (1 << 12) /* UPLL Voltage regulator Control */ #define AT91_PMC_PLL_ACR_UTMIBG (1 << 13) /* UPLL Bandgap Control */ diff --git a/include/linux/clk/davinci.h b/include/linux/clk/davinci.h index 8a7b5cd7eac0d666696fc1140ad45a3f87c00035..f6ebab6228c2042ba9f0b5fc6cf8843c1d364ced 100644 --- a/include/linux/clk/davinci.h +++ b/include/linux/clk/davinci.h @@ -28,13 +28,5 @@ int dm365_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgch int dm365_pll2_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); int dm365_psc_init(struct device *dev, void __iomem *base); #endif -#ifdef CONFIG_ARCH_DAVINCI_DM644x -int dm644x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); -int dm644x_psc_init(struct device *dev, void __iomem *base); -#endif -#ifdef CONFIG_ARCH_DAVINCI_DM646x -int dm646x_pll1_init(struct device *dev, void __iomem *base, struct regmap *cfgchip); -int dm646x_psc_init(struct device *dev, void __iomem *base); -#endif #endif /* __LINUX_CLK_DAVINCI_PLL_H___ */ diff --git a/include/linux/clk/spear.h b/include/linux/clk/spear.h index a64d034ceddd21169e012d8b216e2c964c975e4c..eaf95ca656f83bfc524414592a69a507162b1f48 100644 --- a/include/linux/clk/spear.h +++ b/include/linux/clk/spear.h @@ -8,6 +8,20 @@ #ifndef __LINUX_CLK_SPEAR_H #define __LINUX_CLK_SPEAR_H +#ifdef CONFIG_ARCH_SPEAR3XX +void __init spear3xx_clk_init(void __iomem *misc_base, + void __iomem *soc_config_base); +#else +static inline void __init spear3xx_clk_init(void __iomem *misc_base, + void __iomem *soc_config_base) {} +#endif + +#ifdef CONFIG_ARCH_SPEAR6XX +void __init spear6xx_clk_init(void __iomem *misc_base); +#else +static inline void __init spear6xx_clk_init(void __iomem *misc_base) {} +#endif + #ifdef CONFIG_MACH_SPEAR1310 void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base); #else diff --git a/include/linux/clkdev.h b/include/linux/clkdev.h index 8a8423eb8e9ab76724e71a3832ee3209eb90332a..45570bc21a43ae154c307f1aea021a85040f1627 100644 --- a/include/linux/clkdev.h +++ b/include/linux/clkdev.h @@ -46,6 +46,4 @@ int clk_hw_register_clkdev(struct clk_hw *, const char *, const char *); int devm_clk_hw_register_clkdev(struct device *dev, struct clk_hw *hw, const char *con_id, const char *dev_id); -void devm_clk_release_clkdev(struct device *dev, const char *con_id, - const char *dev_id); #endif diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index c84fec767445d69f5fadf1a2687d0c99da2b8ce4..6cfd6902bd5b92ec1a353a2aaa14da27f3932a43 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -51,6 +51,29 @@ #define __no_sanitize_undefined #endif +#if __has_feature(memory_sanitizer) +#define __SANITIZE_MEMORY__ +/* + * Unlike other sanitizers, KMSAN still inserts code into functions marked with + * no_sanitize("kernel-memory"). Using disable_sanitizer_instrumentation + * provides the behavior consistent with other __no_sanitize_ attributes, + * guaranteeing that __no_sanitize_memory functions remain uninstrumented. + */ +#define __no_sanitize_memory __disable_sanitizer_instrumentation + +/* + * The __no_kmsan_checks attribute ensures that a function does not produce + * false positive reports by: + * - initializing all local variables and memory stores in this function; + * - skipping all shadow checks; + * - passing initialized arguments to this function's callees. + */ +#define __no_kmsan_checks __attribute__((no_sanitize("kernel-memory"))) +#else +#define __no_sanitize_memory +#define __no_kmsan_checks +#endif + /* * Support for __has_feature(coverage_sanitizer) was added in Clang 13 together * with no_sanitize("coverage"). Prior versions of Clang support coverage @@ -66,17 +89,9 @@ # define __noscs __attribute__((__no_sanitize__("shadow-call-stack"))) #endif -#define __nocfi __attribute__((__no_sanitize__("cfi"))) -#define __cficanonical __attribute__((__cfi_canonical_jump_table__)) - -#if defined(CONFIG_CFI_CLANG) -/* - * With CONFIG_CFI_CLANG, the compiler replaces function address - * references with the address of the function's CFI jump table - * entry. The function_nocfi macro always returns the address of the - * actual function instead. - */ -#define function_nocfi(x) __builtin_function_start(x) +#if __has_feature(kcfi) +/* Disable CFI checking inside a function. */ +#define __nocfi __attribute__((__no_sanitize__("kcfi"))) #endif /* diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 9b157b71036f1876bc5cd512172dbb3ada69edee..f55a37efdb9748cda549a1b292db66972275bc26 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -114,6 +114,12 @@ #define __SANITIZE_ADDRESS__ #endif +/* + * GCC does not support KMSAN. + */ +#define __no_sanitize_memory +#define __no_kmsan_checks + /* * Turn individual warnings and errors on and off locally, depending * on version. diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 01ce94b58b423e6021bf007edb0fba6bce3cf141..973a1bfd7ef5302a8e3612cb52fdc71bc21764a2 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -203,16 +203,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, __v; \ }) -/* - * With CONFIG_CFI_CLANG, the compiler replaces function addresses in - * instrumented C code with jump table addresses. Architectures that - * support CFI can define this macro to return the actual function address - * when needed. - */ -#ifndef function_nocfi -#define function_nocfi(x) (x) -#endif - #endif /* __KERNEL__ */ /* @@ -221,9 +211,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, * otherwise, or eliminated entirely due to lack of references that are * visible to the compiler. */ -#define __ADDRESSABLE(sym) \ - static void * __section(".discard.addressable") __used \ +#define ___ADDRESSABLE(sym, __attrs) \ + static void * __used __attrs \ __UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym; +#define __ADDRESSABLE(sym) \ + ___ADDRESSABLE(sym, __section(".discard.addressable")) /** * offset_to_ptr - convert a relative memory offset to an absolute pointer @@ -239,6 +231,12 @@ static inline void *offset_to_ptr(const int *off) /* &a[0] degrades to a pointer: a different type from an array */ #define __must_be_array(a) BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0])) +/* + * Whether 'type' is a signed type or an unsigned type. Supports scalar types, + * bool and also pointer types. + */ +#define is_signed_type(type) (((type)(-1)) < (__force type)1) + /* * This is needed in functions which generate the stack canary, see * arch/x86/kernel/smpboot.c::start_secondary() for an example. diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index 445e80517cab689c7ff178f4fa22844c5f14739d..898b3458b24a00dbab43d2deaf89f567ef7939ec 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -35,7 +35,8 @@ /* * Note: do not use this directly. Instead, use __alloc_size() since it is conditionally - * available and includes other attributes. + * available and includes other attributes. For GCC < 9.1, __alloc_size__ gets undefined + * in compiler-gcc.h, due to misbehaviors. * * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-alloc_005fsize-function-attribute * clang: https://clang.llvm.org/docs/AttributeReference.html#alloc-size @@ -371,4 +372,11 @@ */ #define __weak __attribute__((__weak__)) +/* + * Used by functions that use '__builtin_return_address'. These function + * don't want to be splited or made inline, which can make + * the '__builtin_return_address' get unexpected address. + */ +#define __fix_address noinline __noclone + #endif /* __LINUX_COMPILER_ATTRIBUTES_H */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 4f2a819fd60a34ca40067bdc2ceecf68578f893a..eb0466236661f51cb11d9c06416fc11eca35a950 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -4,8 +4,12 @@ #ifndef __ASSEMBLY__ +/* + * Skipped when running bindgen due to a libclang issue; + * see https://github.com/rust-lang/rust-bindgen/issues/2244. + */ #if defined(CONFIG_DEBUG_INFO_BTF) && defined(CONFIG_PAHOLE_HAS_BTF_TAG) && \ - __has_attribute(btf_type_tag) + __has_attribute(btf_type_tag) && !defined(__BINDGEN__) # define BTF_TYPE_TAG(value) __attribute__((btf_type_tag(#value))) #else # define BTF_TYPE_TAG(value) /* nothing */ @@ -229,7 +233,8 @@ struct ftrace_likely_data { /* Section for code which can't be instrumented at all */ #define noinstr \ noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage + __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage \ + __no_sanitize_memory #endif /* __KERNEL__ */ @@ -265,20 +270,18 @@ struct ftrace_likely_data { # define __nocfi #endif -#ifndef __cficanonical -# define __cficanonical -#endif - /* * Any place that could be marked with the "alloc_size" attribute is also - * a place to be marked with the "malloc" attribute. Do this as part of the - * __alloc_size macro to avoid redundant attributes and to avoid missing a - * __malloc marking. + * a place to be marked with the "malloc" attribute, except those that may + * be performing a _reallocation_, as that may alias the existing pointer. + * For these, use __realloc_size(). */ #ifdef __alloc_size__ # define __alloc_size(x, ...) __alloc_size__(x, ## __VA_ARGS__) __malloc +# define __realloc_size(x, ...) __alloc_size__(x, ## __VA_ARGS__) #else # define __alloc_size(x, ...) __malloc +# define __realloc_size(x, ...) #endif #ifndef asm_volatile_goto diff --git a/include/linux/completion.h b/include/linux/completion.h index 51d9ab079629bdbc0dc2af881489ef343bbee674..62b32b19e0a86817cd84f8b311624dc61b3f73d3 100644 --- a/include/linux/completion.h +++ b/include/linux/completion.h @@ -103,6 +103,7 @@ extern void wait_for_completion(struct completion *); extern void wait_for_completion_io(struct completion *); extern int wait_for_completion_interruptible(struct completion *x); extern int wait_for_completion_killable(struct completion *x); +extern int wait_for_completion_state(struct completion *x, unsigned int state); extern unsigned long wait_for_completion_timeout(struct completion *x, unsigned long timeout); extern unsigned long wait_for_completion_io_timeout(struct completion *x, diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 9f445f09fcfeafec7e3f70ef23df9ca83d7bfe5d..1554021231f9c17680284cc93c3881f35189b488 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -372,6 +372,29 @@ static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa, return csa->read(offset, true, false); } +static inline u64 csdev_access_relaxed_read_pair(struct csdev_access *csa, + u32 lo_offset, u32 hi_offset) +{ + if (likely(csa->io_mem)) { + return readl_relaxed(csa->base + lo_offset) | + ((u64)readl_relaxed(csa->base + hi_offset) << 32); + } + + return csa->read(lo_offset, true, false) | (csa->read(hi_offset, true, false) << 32); +} + +static inline void csdev_access_relaxed_write_pair(struct csdev_access *csa, u64 val, + u32 lo_offset, u32 hi_offset) +{ + if (likely(csa->io_mem)) { + writel_relaxed((u32)val, csa->base + lo_offset); + writel_relaxed((u32)(val >> 32), csa->base + hi_offset); + } else { + csa->write((u32)val, lo_offset, true, false); + csa->write((u32)(val >> 32), hi_offset, true, false); + } +} + static inline u32 csdev_access_read32(struct csdev_access *csa, u32 offset) { if (likely(csa->io_mem)) diff --git a/include/linux/counter.h b/include/linux/counter.h index 1fe17f5adb09dcef42b494e6f3e83533a8d82c61..c41fa602ed283875449237af8ceea18f563ab07f 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -31,6 +31,8 @@ enum counter_comp_type { COUNTER_COMP_ENUM, COUNTER_COMP_COUNT_DIRECTION, COUNTER_COMP_COUNT_MODE, + COUNTER_COMP_SIGNAL_POLARITY, + COUNTER_COMP_ARRAY, }; /** @@ -38,66 +40,114 @@ enum counter_comp_type { * @type: Counter component data type * @name: device-specific component name * @priv: component-relevant data - * @action_read: Synapse action mode read callback. The read value of the + * @action_read: Synapse action mode read callback. The read value of the * respective Synapse action mode should be passed back via * the action parameter. - * @device_u8_read: Device u8 component read callback. The read value of the + * @device_u8_read: Device u8 component read callback. The read value of the * respective Device u8 component should be passed back via * the val parameter. - * @count_u8_read: Count u8 component read callback. The read value of the + * @count_u8_read: Count u8 component read callback. The read value of the * respective Count u8 component should be passed back via * the val parameter. - * @signal_u8_read: Signal u8 component read callback. The read value of the + * @signal_u8_read: Signal u8 component read callback. The read value of the * respective Signal u8 component should be passed back via * the val parameter. - * @device_u32_read: Device u32 component read callback. The read value of + * @device_u32_read: Device u32 component read callback. The read value of * the respective Device u32 component should be passed * back via the val parameter. - * @count_u32_read: Count u32 component read callback. The read value of the + * @count_u32_read: Count u32 component read callback. The read value of the * respective Count u32 component should be passed back via * the val parameter. - * @signal_u32_read: Signal u32 component read callback. The read value of + * @signal_u32_read: Signal u32 component read callback. The read value of * the respective Signal u32 component should be passed * back via the val parameter. - * @device_u64_read: Device u64 component read callback. The read value of + * @device_u64_read: Device u64 component read callback. The read value of * the respective Device u64 component should be passed * back via the val parameter. - * @count_u64_read: Count u64 component read callback. The read value of the + * @count_u64_read: Count u64 component read callback. The read value of the * respective Count u64 component should be passed back via * the val parameter. - * @signal_u64_read: Signal u64 component read callback. The read value of + * @signal_u64_read: Signal u64 component read callback. The read value of * the respective Signal u64 component should be passed * back via the val parameter. - * @action_write: Synapse action mode write callback. The write value of + * @signal_array_u32_read: Signal u32 array component read callback. The + * index of the respective Count u32 array + * component element is passed via the idx + * parameter. The read value of the respective + * Count u32 array component element should be + * passed back via the val parameter. + * @device_array_u64_read: Device u64 array component read callback. The + * index of the respective Device u64 array + * component element is passed via the idx + * parameter. The read value of the respective + * Device u64 array component element should be + * passed back via the val parameter. + * @count_array_u64_read: Count u64 array component read callback. The + * index of the respective Count u64 array + * component element is passed via the idx + * parameter. The read value of the respective + * Count u64 array component element should be + * passed back via the val parameter. + * @signal_array_u64_read: Signal u64 array component read callback. The + * index of the respective Count u64 array + * component element is passed via the idx + * parameter. The read value of the respective + * Count u64 array component element should be + * passed back via the val parameter. + * @action_write: Synapse action mode write callback. The write value of * the respective Synapse action mode is passed via the * action parameter. - * @device_u8_write: Device u8 component write callback. The write value of + * @device_u8_write: Device u8 component write callback. The write value of * the respective Device u8 component is passed via the val * parameter. - * @count_u8_write: Count u8 component write callback. The write value of + * @count_u8_write: Count u8 component write callback. The write value of * the respective Count u8 component is passed via the val * parameter. - * @signal_u8_write: Signal u8 component write callback. The write value of + * @signal_u8_write: Signal u8 component write callback. The write value of * the respective Signal u8 component is passed via the val * parameter. - * @device_u32_write: Device u32 component write callback. The write value of + * @device_u32_write: Device u32 component write callback. The write value of * the respective Device u32 component is passed via the * val parameter. - * @count_u32_write: Count u32 component write callback. The write value of + * @count_u32_write: Count u32 component write callback. The write value of * the respective Count u32 component is passed via the val * parameter. - * @signal_u32_write: Signal u32 component write callback. The write value of + * @signal_u32_write: Signal u32 component write callback. The write value of * the respective Signal u32 component is passed via the * val parameter. - * @device_u64_write: Device u64 component write callback. The write value of + * @device_u64_write: Device u64 component write callback. The write value of * the respective Device u64 component is passed via the * val parameter. - * @count_u64_write: Count u64 component write callback. The write value of + * @count_u64_write: Count u64 component write callback. The write value of * the respective Count u64 component is passed via the val * parameter. - * @signal_u64_write: Signal u64 component write callback. The write value of + * @signal_u64_write: Signal u64 component write callback. The write value of * the respective Signal u64 component is passed via the * val parameter. + * @signal_array_u32_write: Signal u32 array component write callback. The + * index of the respective Signal u32 array + * component element is passed via the idx + * parameter. The write value of the respective + * Signal u32 array component element is passed via + * the val parameter. + * @device_array_u64_write: Device u64 array component write callback. The + * index of the respective Device u64 array + * component element is passed via the idx + * parameter. The write value of the respective + * Device u64 array component element is passed via + * the val parameter. + * @count_array_u64_write: Count u64 array component write callback. The + * index of the respective Count u64 array + * component element is passed via the idx + * parameter. The write value of the respective + * Count u64 array component element is passed via + * the val parameter. + * @signal_array_u64_write: Signal u64 array component write callback. The + * index of the respective Signal u64 array + * component element is passed via the idx + * parameter. The write value of the respective + * Signal u64 array component element is passed via + * the val parameter. */ struct counter_comp { enum counter_comp_type type; @@ -125,6 +175,17 @@ struct counter_comp { struct counter_count *count, u64 *val); int (*signal_u64_read)(struct counter_device *counter, struct counter_signal *signal, u64 *val); + int (*signal_array_u32_read)(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, u32 *val); + int (*device_array_u64_read)(struct counter_device *counter, + size_t idx, u64 *val); + int (*count_array_u64_read)(struct counter_device *counter, + struct counter_count *count, + size_t idx, u64 *val); + int (*signal_array_u64_read)(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, u64 *val); }; union { int (*action_write)(struct counter_device *counter, @@ -148,6 +209,17 @@ struct counter_comp { struct counter_count *count, u64 val); int (*signal_u64_write)(struct counter_device *counter, struct counter_signal *signal, u64 val); + int (*signal_array_u32_write)(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, u32 val); + int (*device_array_u64_write)(struct counter_device *counter, + size_t idx, u64 val); + int (*count_array_u64_write)(struct counter_device *counter, + struct counter_count *count, + size_t idx, u64 val); + int (*signal_array_u64_write)(struct counter_device *counter, + struct counter_signal *signal, + size_t idx, u64 val); }; }; @@ -452,6 +524,60 @@ struct counter_available { .priv = &(_available), \ } +struct counter_array { + enum counter_comp_type type; + const struct counter_available *avail; + union { + size_t length; + size_t idx; + }; +}; + +#define DEFINE_COUNTER_ARRAY_U64(_name, _length) \ + struct counter_array _name = { \ + .type = COUNTER_COMP_U64, \ + .length = (_length), \ + } + +#define DEFINE_COUNTER_ARRAY_CAPTURE(_name, _length) \ + DEFINE_COUNTER_ARRAY_U64(_name, _length) + +#define DEFINE_COUNTER_ARRAY_POLARITY(_name, _enums, _length) \ + DEFINE_COUNTER_AVAILABLE(_name##_available, _enums); \ + struct counter_array _name = { \ + .type = COUNTER_COMP_SIGNAL_POLARITY, \ + .avail = &(_name##_available), \ + .length = (_length), \ + } + +#define COUNTER_COMP_DEVICE_ARRAY_U64(_name, _read, _write, _array) \ +{ \ + .type = COUNTER_COMP_ARRAY, \ + .name = (_name), \ + .device_array_u64_read = (_read), \ + .device_array_u64_write = (_write), \ + .priv = &(_array), \ +} +#define COUNTER_COMP_COUNT_ARRAY_U64(_name, _read, _write, _array) \ +{ \ + .type = COUNTER_COMP_ARRAY, \ + .name = (_name), \ + .count_array_u64_read = (_read), \ + .count_array_u64_write = (_write), \ + .priv = &(_array), \ +} +#define COUNTER_COMP_SIGNAL_ARRAY_U64(_name, _read, _write, _array) \ +{ \ + .type = COUNTER_COMP_ARRAY, \ + .name = (_name), \ + .signal_array_u64_read = (_read), \ + .signal_array_u64_write = (_write), \ + .priv = &(_array), \ +} + +#define COUNTER_COMP_CAPTURE(_read, _write) \ + COUNTER_COMP_COUNT_U64("capture", _read, _write) + #define COUNTER_COMP_CEILING(_read, _write) \ COUNTER_COMP_COUNT_U64("ceiling", _read, _write) @@ -477,10 +603,31 @@ struct counter_available { #define COUNTER_COMP_FLOOR(_read, _write) \ COUNTER_COMP_COUNT_U64("floor", _read, _write) +#define COUNTER_COMP_POLARITY(_read, _write, _available) \ +{ \ + .type = COUNTER_COMP_SIGNAL_POLARITY, \ + .name = "polarity", \ + .signal_u32_read = (_read), \ + .signal_u32_write = (_write), \ + .priv = &(_available), \ +} + #define COUNTER_COMP_PRESET(_read, _write) \ COUNTER_COMP_COUNT_U64("preset", _read, _write) #define COUNTER_COMP_PRESET_ENABLE(_read, _write) \ COUNTER_COMP_COUNT_BOOL("preset_enable", _read, _write) +#define COUNTER_COMP_ARRAY_CAPTURE(_read, _write, _array) \ + COUNTER_COMP_COUNT_ARRAY_U64("capture", _read, _write, _array) + +#define COUNTER_COMP_ARRAY_POLARITY(_read, _write, _array) \ +{ \ + .type = COUNTER_COMP_ARRAY, \ + .name = "polarity", \ + .signal_array_u32_read = (_read), \ + .signal_array_u32_write = (_write), \ + .priv = &(_array), \ +} + #endif /* _COUNTER_H_ */ diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index bd047864c7ac7bc58cff53e0ae60f22541af1171..c2aa0aa26b457fd04ad56c3da030a9b788c77b50 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -35,19 +35,23 @@ typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; */ #define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp) -#if NR_CPUS == 1 -#define nr_cpu_ids 1U +#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) +#define nr_cpu_ids ((unsigned int)NR_CPUS) #else extern unsigned int nr_cpu_ids; #endif -#ifdef CONFIG_CPUMASK_OFFSTACK -/* Assuming NR_CPUS is huge, a runtime limit is more efficient. Also, - * not all bits may be allocated. */ -#define nr_cpumask_bits nr_cpu_ids +static inline void set_nr_cpu_ids(unsigned int nr) +{ +#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS) + WARN_ON(nr != nr_cpu_ids); #else -#define nr_cpumask_bits ((unsigned int)NR_CPUS) + nr_cpu_ids = nr; #endif +} + +/* Deprecated. Always use nr_cpu_ids. */ +#define nr_cpumask_bits nr_cpu_ids /* * The following particular system cpumasks and operations manage @@ -67,10 +71,6 @@ extern unsigned int nr_cpu_ids; * cpu_online_mask is the dynamic subset of cpu_present_mask, * indicating those CPUs available for scheduling. * - * If HOTPLUG is enabled, then cpu_possible_mask is forced to have - * all NR_CPUS bits set, otherwise it is just the set of CPUs that - * ACPI reports present at boot. - * * If HOTPLUG is enabled, then cpu_present_mask varies dynamically, * depending on what ACPI reports as currently plugged in, otherwise * cpu_present_mask is just a copy of cpu_possible_mask. @@ -246,9 +246,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, * After the loop, cpu is >= nr_cpu_ids. */ #define for_each_cpu(cpu, mask) \ - for ((cpu) = -1; \ - (cpu) = cpumask_next((cpu), (mask)), \ - (cpu) < nr_cpu_ids;) + for_each_set_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) /** * for_each_cpu_not - iterate over every cpu in a complemented mask @@ -258,9 +256,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p, * After the loop, cpu is >= nr_cpu_ids. */ #define for_each_cpu_not(cpu, mask) \ - for ((cpu) = -1; \ - (cpu) = cpumask_next_zero((cpu), (mask)), \ - (cpu) < nr_cpu_ids;) + for_each_clear_bit(cpu, cpumask_bits(mask), nr_cpumask_bits) #if NR_CPUS == 1 static inline @@ -293,10 +289,8 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta * * After the loop, cpu is >= nr_cpu_ids. */ -#define for_each_cpu_wrap(cpu, mask, start) \ - for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \ - (cpu) < nr_cpumask_bits; \ - (cpu) = cpumask_next_wrap((cpu), (mask), (start), true)) +#define for_each_cpu_wrap(cpu, mask, start) \ + for_each_set_bit_wrap(cpu, cpumask_bits(mask), nr_cpumask_bits, start) /** * for_each_cpu_and - iterate over every cpu in both masks @@ -313,9 +307,25 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta * After the loop, cpu is >= nr_cpu_ids. */ #define for_each_cpu_and(cpu, mask1, mask2) \ - for ((cpu) = -1; \ - (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \ - (cpu) < nr_cpu_ids;) + for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) + +/** + * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding + * those present in another. + * @cpu: the (optionally unsigned) integer iterator + * @mask1: the first cpumask pointer + * @mask2: the second cpumask pointer + * + * This saves a temporary CPU mask in many places. It is equivalent to: + * struct cpumask tmp; + * cpumask_andnot(&tmp, &mask1, &mask2); + * for_each_cpu(cpu, &tmp) + * ... + * + * After the loop, cpu is >= nr_cpu_ids. + */ +#define for_each_cpu_andnot(cpu, mask1, mask2) \ + for_each_andnot_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), nr_cpumask_bits) /** * cpumask_any_but - return a "random" in a cpumask, but not this one. @@ -337,6 +347,50 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) return i; } +/** + * cpumask_nth - get the first cpu in a cpumask + * @srcp: the cpumask pointer + * @cpu: the N'th cpu to find, starting from 0 + * + * Returns >= nr_cpu_ids if such cpu doesn't exist. + */ +static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp) +{ + return find_nth_bit(cpumask_bits(srcp), nr_cpumask_bits, cpumask_check(cpu)); +} + +/** + * cpumask_nth_and - get the first cpu in 2 cpumasks + * @srcp1: the cpumask pointer + * @srcp2: the cpumask pointer + * @cpu: the N'th cpu to find, starting from 0 + * + * Returns >= nr_cpu_ids if such cpu doesn't exist. + */ +static inline +unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return find_nth_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), + nr_cpumask_bits, cpumask_check(cpu)); +} + +/** + * cpumask_nth_andnot - get the first cpu set in 1st cpumask, and clear in 2nd. + * @srcp1: the cpumask pointer + * @srcp2: the cpumask pointer + * @cpu: the N'th cpu to find, starting from 0 + * + * Returns >= nr_cpu_ids if such cpu doesn't exist. + */ +static inline +unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return find_nth_andnot_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), + nr_cpumask_bits, cpumask_check(cpu)); +} + #define CPU_BITS_NONE \ { \ [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \ @@ -586,6 +640,17 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp) return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits); } +/** + * cpumask_weight_and - Count of bits in (*srcp1 & *srcp2) + * @srcp1: the cpumask to count bits (< nr_cpu_ids) in. + * @srcp2: the cpumask to count bits (< nr_cpu_ids) in. + */ +static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1, + const struct cpumask *srcp2) +{ + return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), nr_cpumask_bits); +} + /** * cpumask_shift_right - *dstp = *srcp >> n * @dstp: the cpumask result @@ -1127,9 +1192,10 @@ cpumap_print_list_to_buf(char *buf, const struct cpumask *mask, * cover a worst-case of every other cpu being on one of two nodes for a * very large NR_CPUS. * - * Use PAGE_SIZE as a minimum for smaller configurations. + * Use PAGE_SIZE as a minimum for smaller configurations while avoiding + * unsigned comparison to -1. */ -#define CPUMAP_FILE_MAX_BYTES ((((NR_CPUS * 9)/32 - 1) > PAGE_SIZE) \ +#define CPUMAP_FILE_MAX_BYTES (((NR_CPUS * 9)/32 > PAGE_SIZE) \ ? (NR_CPUS * 9)/32 - 1 : PAGE_SIZE) #define CPULIST_FILE_MAX_BYTES (((NR_CPUS * 7)/2 > PAGE_SIZE) ? (NR_CPUS * 7)/2 : PAGE_SIZE) diff --git a/include/linux/damon.h b/include/linux/damon.h index 7b1f4a4882308cde8c2bed387eaceda05eb01b48..620ada094c3b25b2eca8c0eacc83326742678fb6 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -216,13 +216,26 @@ struct damos_stat { }; /** - * struct damos - Represents a Data Access Monitoring-based Operation Scheme. + * struct damos_access_pattern - Target access pattern of the given scheme. * @min_sz_region: Minimum size of target regions. * @max_sz_region: Maximum size of target regions. * @min_nr_accesses: Minimum ``->nr_accesses`` of target regions. * @max_nr_accesses: Maximum ``->nr_accesses`` of target regions. * @min_age_region: Minimum age of target regions. * @max_age_region: Maximum age of target regions. + */ +struct damos_access_pattern { + unsigned long min_sz_region; + unsigned long max_sz_region; + unsigned int min_nr_accesses; + unsigned int max_nr_accesses; + unsigned int min_age_region; + unsigned int max_age_region; +}; + +/** + * struct damos - Represents a Data Access Monitoring-based Operation Scheme. + * @pattern: Access pattern of target regions. * @action: &damo_action to be applied to the target regions. * @quota: Control the aggressiveness of this scheme. * @wmarks: Watermarks for automated (in)activation of this scheme. @@ -230,10 +243,8 @@ struct damos_stat { * @list: List head for siblings. * * For each aggregation interval, DAMON finds regions which fit in the - * condition (&min_sz_region, &max_sz_region, &min_nr_accesses, - * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to - * those. To avoid consuming too much CPU time or IO resources for the - * &action, "a is used. + * &pattern and applies &action to those. To avoid consuming too much + * CPU time or IO resources for the &action, "a is used. * * To do the work only when needed, schemes can be activated for specific * system situations using &wmarks. If all schemes that registered to the @@ -248,12 +259,7 @@ struct damos_stat { * &action is applied. */ struct damos { - unsigned long min_sz_region; - unsigned long max_sz_region; - unsigned int min_nr_accesses; - unsigned int max_nr_accesses; - unsigned int min_age_region; - unsigned int max_age_region; + struct damos_access_pattern pattern; enum damos_action action; struct damos_quota quota; struct damos_watermarks wmarks; @@ -340,7 +346,7 @@ struct damon_operations { unsigned long (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme); - bool (*target_valid)(void *target); + bool (*target_valid)(struct damon_target *t); void (*cleanup)(struct damon_ctx *context); }; @@ -383,13 +389,15 @@ struct damon_callback { }; /** - * struct damon_ctx - Represents a context for each monitoring. This is the - * main interface that allows users to set the attributes and get the results - * of the monitoring. + * struct damon_attrs - Monitoring attributes for accuracy/overhead control. * * @sample_interval: The time between access samplings. * @aggr_interval: The time between monitor results aggregations. * @ops_update_interval: The time between monitoring operations updates. + * @min_nr_regions: The minimum number of adaptive monitoring + * regions. + * @max_nr_regions: The maximum number of adaptive monitoring + * regions. * * For each @sample_interval, DAMON checks whether each region is accessed or * not. It aggregates and keeps the access information (number of accesses to @@ -399,7 +407,21 @@ struct damon_callback { * @ops_update_interval. All time intervals are in micro-seconds. * Please refer to &struct damon_operations and &struct damon_callback for more * detail. + */ +struct damon_attrs { + unsigned long sample_interval; + unsigned long aggr_interval; + unsigned long ops_update_interval; + unsigned long min_nr_regions; + unsigned long max_nr_regions; +}; + +/** + * struct damon_ctx - Represents a context for each monitoring. This is the + * main interface that allows users to set the attributes and get the results + * of the monitoring. * + * @attrs: Monitoring attributes for accuracy/overhead control. * @kdamond: Kernel thread who does the monitoring. * @kdamond_lock: Mutex for the synchronizations with @kdamond. * @@ -421,15 +443,11 @@ struct damon_callback { * @ops: Set of monitoring operations for given use cases. * @callback: Set of callbacks for monitoring events notifications. * - * @min_nr_regions: The minimum number of adaptive monitoring regions. - * @max_nr_regions: The maximum number of adaptive monitoring regions. * @adaptive_targets: Head of monitoring targets (&damon_target) list. * @schemes: Head of schemes (&damos) list. */ struct damon_ctx { - unsigned long sample_interval; - unsigned long aggr_interval; - unsigned long ops_update_interval; + struct damon_attrs attrs; /* private: internal use only */ struct timespec64 last_aggregation; @@ -442,8 +460,6 @@ struct damon_ctx { struct damon_operations ops; struct damon_callback callback; - unsigned long min_nr_regions; - unsigned long max_nr_regions; struct list_head adaptive_targets; struct list_head schemes; }; @@ -463,9 +479,23 @@ static inline struct damon_region *damon_last_region(struct damon_target *t) return list_last_entry(&t->regions_list, struct damon_region, list); } +static inline struct damon_region *damon_first_region(struct damon_target *t) +{ + return list_first_entry(&t->regions_list, struct damon_region, list); +} + +static inline unsigned long damon_sz_region(struct damon_region *r) +{ + return r->ar.end - r->ar.start; +} + + #define damon_for_each_region(r, t) \ list_for_each_entry(r, &t->regions_list, list) +#define damon_for_each_region_from(r, t) \ + list_for_each_entry_from(r, &t->regions_list, list) + #define damon_for_each_region_safe(r, next, t) \ list_for_each_entry_safe(r, next, &t->regions_list, list) @@ -501,12 +531,9 @@ void damon_destroy_region(struct damon_region *r, struct damon_target *t); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges); -struct damos *damon_new_scheme( - unsigned long min_sz_region, unsigned long max_sz_region, - unsigned int min_nr_accesses, unsigned int max_nr_accesses, - unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action, struct damos_quota *quota, - struct damos_watermarks *wmarks); +struct damos *damon_new_scheme(struct damos_access_pattern *pattern, + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks); void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); void damon_destroy_scheme(struct damos *s); @@ -519,10 +546,8 @@ unsigned int damon_nr_regions(struct damon_target *t); struct damon_ctx *damon_new_ctx(void); void damon_destroy_ctx(struct damon_ctx *ctx); -int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, - unsigned long aggr_int, unsigned long ops_upd_int, - unsigned long min_nr_reg, unsigned long max_nr_reg); -int damon_set_schemes(struct damon_ctx *ctx, +int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); +void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, ssize_t nr_schemes); int damon_nr_running_ctxs(void); bool damon_is_registered_ops(enum damon_ops_id id); @@ -538,6 +563,9 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); +int damon_set_region_biggest_system_ram_default(struct damon_target *t, + unsigned long *start, unsigned long *end); + #endif /* CONFIG_DAMON */ #endif /* _DAMON_H */ diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 92c78ed02b54d5bf59c9f5c7506dcf0c6afc1512..6b351e009f5976db447e8da82e579713d02f20b5 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -16,6 +16,7 @@ #include struct path; +struct file; struct vfsmount; /* @@ -250,7 +251,7 @@ extern struct dentry * d_make_root(struct inode *); /* - the ramfs-type tree */ extern void d_genocide(struct dentry *); -extern void d_tmpfile(struct dentry *, struct inode *); +extern void d_tmpfile(struct file *, struct inode *); extern struct dentry *d_find_alias(struct inode *); extern void d_prune_aliases(struct inode *); @@ -287,8 +288,8 @@ static inline unsigned d_count(const struct dentry *dentry) /* * helper function for dentry_operations.d_dname() members */ -extern __printf(4, 5) -char *dynamic_dname(struct dentry *, char *, int, const char *, ...); +extern __printf(3, 4) +char *dynamic_dname(char *, int, const char *, ...); extern char *__d_path(const struct path *, const struct path *, char *, int); extern char *d_absolute_path(const struct path *, char *, int); diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h index c869f1e73d755832c739e88ed7f73699fbf46c2a..f60674692d36578ae81c0c55aace120f4a444687 100644 --- a/include/linux/debugfs.h +++ b/include/linux/debugfs.h @@ -91,6 +91,8 @@ struct dentry *debugfs_create_automount(const char *name, void debugfs_remove(struct dentry *dentry); #define debugfs_remove_recursive debugfs_remove +void debugfs_lookup_and_remove(const char *name, struct dentry *parent); + const struct file_operations *debugfs_real_fops(const struct file *filp); int debugfs_file_get(struct dentry *dentry); @@ -225,6 +227,10 @@ static inline void debugfs_remove(struct dentry *dentry) static inline void debugfs_remove_recursive(struct dentry *dentry) { } +static inline void debugfs_lookup_and_remove(const char *name, + struct dentry *parent) +{ } + const struct file_operations *debugfs_real_fops(const struct file *filp); static inline int debugfs_file_get(struct dentry *dentry) diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index 58aea2d7385c22c795b801fd06d074aed274f8e0..0da97dba9ef835f5647850d9b27201c84ac8752a 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -73,8 +73,8 @@ extern int delayacct_add_tsk(struct taskstats *, struct task_struct *); extern __u64 __delayacct_blkio_ticks(struct task_struct *); extern void __delayacct_freepages_start(void); extern void __delayacct_freepages_end(void); -extern void __delayacct_thrashing_start(void); -extern void __delayacct_thrashing_end(void); +extern void __delayacct_thrashing_start(bool *in_thrashing); +extern void __delayacct_thrashing_end(bool *in_thrashing); extern void __delayacct_swapin_start(void); extern void __delayacct_swapin_end(void); extern void __delayacct_compact_start(void); @@ -143,22 +143,22 @@ static inline void delayacct_freepages_end(void) __delayacct_freepages_end(); } -static inline void delayacct_thrashing_start(void) +static inline void delayacct_thrashing_start(bool *in_thrashing) { if (!static_branch_unlikely(&delayacct_key)) return; if (current->delays) - __delayacct_thrashing_start(); + __delayacct_thrashing_start(in_thrashing); } -static inline void delayacct_thrashing_end(void) +static inline void delayacct_thrashing_end(bool *in_thrashing) { if (!static_branch_unlikely(&delayacct_key)) return; if (current->delays) - __delayacct_thrashing_end(); + __delayacct_thrashing_end(in_thrashing); } static inline void delayacct_swapin_start(void) @@ -237,9 +237,9 @@ static inline void delayacct_freepages_start(void) {} static inline void delayacct_freepages_end(void) {} -static inline void delayacct_thrashing_start(void) +static inline void delayacct_thrashing_start(bool *in_thrashing) {} -static inline void delayacct_thrashing_end(void) +static inline void delayacct_thrashing_end(bool *in_thrashing) {} static inline void delayacct_swapin_start(void) {} diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 7acaabde5396d8288c5b29864a9b06d8b25ab5c6..2114d65b862fc64e5ed3f8ee1395e140782879dc 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -242,6 +242,7 @@ driver_find_device_by_acpi_dev(struct device_driver *drv, const void *adev) extern int driver_deferred_probe_timeout; void driver_deferred_probe_add(struct device *dev); +int driver_deferred_probe_check_state(struct device *dev); void driver_init(void); /** diff --git a/include/linux/dlm.h b/include/linux/dlm.h index ff951e9f6f2081a2e66eb88d9598155c1a67e38c..c6bc2b5ee7e6af4365ff109e118d7d5af077d9fe 100644 --- a/include/linux/dlm.h +++ b/include/linux/dlm.h @@ -56,9 +56,6 @@ struct dlm_lockspace_ops { * DLM_LSFL_TIMEWARN * The dlm should emit netlink messages if locks have been waiting * for a configurable amount of time. (Unused.) - * DLM_LSFL_FS - * The lockspace user is in the kernel (i.e. filesystem). Enables - * direct bast/cast callbacks. * DLM_LSFL_NEWEXCL * dlm_new_lockspace() should return -EEXIST if the lockspace exists. * @@ -134,7 +131,7 @@ int dlm_lock(dlm_lockspace_t *lockspace, int mode, struct dlm_lksb *lksb, uint32_t flags, - void *name, + const void *name, unsigned int namelen, uint32_t parent_lkid, void (*lockast) (void *astarg), diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h deleted file mode 100644 index 24607dc3c2ac5399139d7f0dec78a3d2b9843c7a..0000000000000000000000000000000000000000 --- a/include/linux/dma-iommu.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2014-2015 ARM Ltd. - */ -#ifndef __DMA_IOMMU_H -#define __DMA_IOMMU_H - -#include -#include - -#ifdef CONFIG_IOMMU_DMA -#include -#include -#include - -/* Domain management interface for IOMMU drivers */ -int iommu_get_dma_cookie(struct iommu_domain *domain); -int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); -void iommu_put_dma_cookie(struct iommu_domain *domain); - -/* Setup call for arch DMA mapping code */ -void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); -int iommu_dma_init_fq(struct iommu_domain *domain); - -/* The DMA API isn't _quite_ the whole story, though... */ -/* - * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU device - * - * The MSI page will be stored in @desc. - * - * Return: 0 on success otherwise an error describing the failure. - */ -int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); - -/* Update the MSI message if required. */ -void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg); - -void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); - -void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, - struct iommu_domain *domain); - -extern bool iommu_dma_forcedac; - -#else /* CONFIG_IOMMU_DMA */ - -struct iommu_domain; -struct msi_desc; -struct msi_msg; -struct device; - -static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, - u64 dma_limit) -{ -} - -static inline int iommu_dma_init_fq(struct iommu_domain *domain) -{ - return -EINVAL; -} - -static inline int iommu_get_dma_cookie(struct iommu_domain *domain) -{ - return -ENODEV; -} - -static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) -{ - return -ENODEV; -} - -static inline void iommu_put_dma_cookie(struct iommu_domain *domain) -{ -} - -static inline int iommu_dma_prepare_msi(struct msi_desc *desc, - phys_addr_t msi_addr) -{ - return 0; -} - -static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg) -{ -} - -static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) -{ -} - -#endif /* CONFIG_IOMMU_DMA */ -#endif /* __DMA_IOMMU_H */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 25a30906289d987e9fc0936132bb526e3bf7a4d0..0ee20b764000cb705a238a3483689be5d4b468fd 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -139,7 +139,6 @@ int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); bool dma_can_mmap(struct device *dev); -int dma_supported(struct device *dev, u64 mask); bool dma_pci_p2pdma_supported(struct device *dev); int dma_set_mask(struct device *dev, u64 mask); int dma_set_coherent_mask(struct device *dev, u64 mask); @@ -248,10 +247,6 @@ static inline bool dma_can_mmap(struct device *dev) { return false; } -static inline int dma_supported(struct device *dev, u64 mask) -{ - return 0; -} static inline bool dma_pci_p2pdma_supported(struct device *dev) { return false; diff --git a/include/linux/dma/hsu.h b/include/linux/dma/hsu.h index a6b7bc70735676b4d8f3f55d95128d511801e4bc..77ea602c287cc45c67288146042896a80eb8c2dc 100644 --- a/include/linux/dma/hsu.h +++ b/include/linux/dma/hsu.h @@ -8,11 +8,13 @@ #ifndef _DMA_HSU_H #define _DMA_HSU_H -#include -#include +#include +#include +#include #include +struct device; struct hsu_dma; /** diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index dce631e678dd6ce0bfd5c9985c0520b3bee78e7c..41682278d2e825b61feb1f632aa370500799a6c4 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -6,6 +6,8 @@ #include #endif +#include + /* * An instance of this structure is created in a special * ELF section at every dynamic debug callsite. At runtime, @@ -21,6 +23,9 @@ struct _ddebug { const char *filename; const char *format; unsigned int lineno:18; +#define CLS_BITS 6 + unsigned int class_id:CLS_BITS; +#define _DPRINTK_CLASS_DFLT ((1 << CLS_BITS) - 1) /* * The flags field controls the behaviour at the callsite. * The bits here are changed dynamically when the user @@ -51,15 +56,82 @@ struct _ddebug { #endif } __attribute__((aligned(8))); - +enum class_map_type { + DD_CLASS_TYPE_DISJOINT_BITS, + /** + * DD_CLASS_TYPE_DISJOINT_BITS: classes are independent, one per bit. + * expecting hex input. Built for drm.debug, basis for other types. + */ + DD_CLASS_TYPE_LEVEL_NUM, + /** + * DD_CLASS_TYPE_LEVEL_NUM: input is numeric level, 0-N. + * N turns on just bits N-1 .. 0, so N=0 turns all bits off. + */ + DD_CLASS_TYPE_DISJOINT_NAMES, + /** + * DD_CLASS_TYPE_DISJOINT_NAMES: input is a CSV of [+-]CLASS_NAMES, + * classes are independent, like _DISJOINT_BITS. + */ + DD_CLASS_TYPE_LEVEL_NAMES, + /** + * DD_CLASS_TYPE_LEVEL_NAMES: input is a CSV of [+-]CLASS_NAMES, + * intended for names like: INFO,DEBUG,TRACE, with a module prefix + * avoid EMERG,ALERT,CRIT,ERR,WARNING: they're not debug + */ +}; + +struct ddebug_class_map { + struct list_head link; + struct module *mod; + const char *mod_name; /* needed for builtins */ + const char **class_names; + const int length; + const int base; /* index of 1st .class_id, allows split/shared space */ + enum class_map_type map_type; +}; + +/** + * DECLARE_DYNDBG_CLASSMAP - declare classnames known by a module + * @_var: a struct ddebug_class_map, passed to module_param_cb + * @_type: enum class_map_type, chooses bits/verbose, numeric/symbolic + * @_base: offset of 1st class-name. splits .class_id space + * @classes: class-names used to control class'd prdbgs + */ +#define DECLARE_DYNDBG_CLASSMAP(_var, _maptype, _base, ...) \ + static const char *_var##_classnames[] = { __VA_ARGS__ }; \ + static struct ddebug_class_map __aligned(8) __used \ + __section("__dyndbg_classes") _var = { \ + .mod = THIS_MODULE, \ + .mod_name = KBUILD_MODNAME, \ + .base = _base, \ + .map_type = _maptype, \ + .length = NUM_TYPE_ARGS(char*, __VA_ARGS__), \ + .class_names = _var##_classnames, \ + } +#define NUM_TYPE_ARGS(eltype, ...) \ + (sizeof((eltype[]){__VA_ARGS__}) / sizeof(eltype)) + +/* encapsulate linker provided built-in (or module) dyndbg data */ +struct _ddebug_info { + struct _ddebug *descs; + struct ddebug_class_map *classes; + unsigned int num_descs; + unsigned int num_classes; +}; + +struct ddebug_class_param { + union { + unsigned long *bits; + unsigned int *lvl; + }; + char flags[8]; + const struct ddebug_class_map *map; +}; #if defined(CONFIG_DYNAMIC_DEBUG_CORE) -/* exported for module authors to exercise >control */ -int dynamic_debug_exec_queries(const char *query, const char *modname); +int ddebug_add_module(struct _ddebug_info *dyndbg, const char *modname); -int ddebug_add_module(struct _ddebug *tab, unsigned int n, - const char *modname); extern int ddebug_remove_module(const char *mod_name); extern __printf(2, 3) void __dynamic_pr_debug(struct _ddebug *descriptor, const char *fmt, ...); @@ -87,7 +159,7 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, const struct ib_device *ibdev, const char *fmt, ...); -#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) \ +#define DEFINE_DYNAMIC_DEBUG_METADATA_CLS(name, cls, fmt) \ static struct _ddebug __aligned(8) \ __section("__dyndbg") name = { \ .modname = KBUILD_MODNAME, \ @@ -96,8 +168,14 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, .format = (fmt), \ .lineno = __LINE__, \ .flags = _DPRINTK_FLAGS_DEFAULT, \ + .class_id = cls, \ _DPRINTK_KEY_INIT \ - } + }; \ + BUILD_BUG_ON_MSG(cls > _DPRINTK_CLASS_DFLT, \ + "classid value overflow") + +#define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) \ + DEFINE_DYNAMIC_DEBUG_METADATA_CLS(name, _DPRINTK_CLASS_DFLT, fmt) #ifdef CONFIG_JUMP_LABEL @@ -128,17 +206,34 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, #endif /* CONFIG_JUMP_LABEL */ -#define __dynamic_func_call(id, fmt, func, ...) do { \ - DEFINE_DYNAMIC_DEBUG_METADATA(id, fmt); \ - if (DYNAMIC_DEBUG_BRANCH(id)) \ - func(&id, ##__VA_ARGS__); \ -} while (0) - -#define __dynamic_func_call_no_desc(id, fmt, func, ...) do { \ - DEFINE_DYNAMIC_DEBUG_METADATA(id, fmt); \ +/* + * Factory macros: ($prefix)dynamic_func_call($suffix) + * + * Lower layer (with __ prefix) gets the callsite metadata, and wraps + * the func inside a debug-branch/static-key construct. Upper layer + * (with _ prefix) does the UNIQUE_ID once, so that lower can ref the + * name/label multiple times, and tie the elements together. + * Multiple flavors: + * (|_cls): adds in _DPRINT_CLASS_DFLT as needed + * (|_no_desc): former gets callsite descriptor as 1st arg (for prdbgs) + */ +#define __dynamic_func_call_cls(id, cls, fmt, func, ...) do { \ + DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt); \ if (DYNAMIC_DEBUG_BRANCH(id)) \ - func(__VA_ARGS__); \ + func(&id, ##__VA_ARGS__); \ +} while (0) +#define __dynamic_func_call(id, fmt, func, ...) \ + __dynamic_func_call_cls(id, _DPRINTK_CLASS_DFLT, fmt, \ + func, ##__VA_ARGS__) + +#define __dynamic_func_call_cls_no_desc(id, cls, fmt, func, ...) do { \ + DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt); \ + if (DYNAMIC_DEBUG_BRANCH(id)) \ + func(__VA_ARGS__); \ } while (0) +#define __dynamic_func_call_no_desc(id, fmt, func, ...) \ + __dynamic_func_call_cls_no_desc(id, _DPRINTK_CLASS_DFLT, \ + fmt, func, ##__VA_ARGS__) /* * "Factory macro" for generating a call to func, guarded by a @@ -148,22 +243,33 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, * the varargs. Note that fmt is repeated in invocations of this * macro. */ +#define _dynamic_func_call_cls(cls, fmt, func, ...) \ + __dynamic_func_call_cls(__UNIQUE_ID(ddebug), cls, fmt, func, ##__VA_ARGS__) #define _dynamic_func_call(fmt, func, ...) \ - __dynamic_func_call(__UNIQUE_ID(ddebug), fmt, func, ##__VA_ARGS__) + _dynamic_func_call_cls(_DPRINTK_CLASS_DFLT, fmt, func, ##__VA_ARGS__) + /* * A variant that does the same, except that the descriptor is not * passed as the first argument to the function; it is only called * with precisely the macro's varargs. */ -#define _dynamic_func_call_no_desc(fmt, func, ...) \ - __dynamic_func_call_no_desc(__UNIQUE_ID(ddebug), fmt, func, ##__VA_ARGS__) +#define _dynamic_func_call_cls_no_desc(cls, fmt, func, ...) \ + __dynamic_func_call_cls_no_desc(__UNIQUE_ID(ddebug), cls, fmt, \ + func, ##__VA_ARGS__) +#define _dynamic_func_call_no_desc(fmt, func, ...) \ + _dynamic_func_call_cls_no_desc(_DPRINTK_CLASS_DFLT, fmt, \ + func, ##__VA_ARGS__) + +#define dynamic_pr_debug_cls(cls, fmt, ...) \ + _dynamic_func_call_cls(cls, fmt, __dynamic_pr_debug, \ + pr_fmt(fmt), ##__VA_ARGS__) #define dynamic_pr_debug(fmt, ...) \ - _dynamic_func_call(fmt, __dynamic_pr_debug, \ + _dynamic_func_call(fmt, __dynamic_pr_debug, \ pr_fmt(fmt), ##__VA_ARGS__) #define dynamic_dev_dbg(dev, fmt, ...) \ - _dynamic_func_call(fmt,__dynamic_dev_dbg, \ + _dynamic_func_call(fmt, __dynamic_dev_dbg, \ dev, fmt, ##__VA_ARGS__) #define dynamic_netdev_dbg(dev, fmt, ...) \ @@ -181,14 +287,24 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, KERN_DEBUG, prefix_str, prefix_type, \ rowsize, groupsize, buf, len, ascii) +struct kernel_param; +int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp); +int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp); + +/* for test only, generally expect drm.debug style macro wrappers */ +#define __pr_debug_cls(cls, fmt, ...) do { \ + BUILD_BUG_ON_MSG(!__builtin_constant_p(cls), \ + "expecting constant class int/enum"); \ + dynamic_pr_debug_cls(cls, fmt, ##__VA_ARGS__); \ + } while (0) + #else /* !CONFIG_DYNAMIC_DEBUG_CORE */ #include #include #include -static inline int ddebug_add_module(struct _ddebug *tab, unsigned int n, - const char *modname) +static inline int ddebug_add_module(struct _ddebug_info *dinfo, const char *modname) { return 0; } @@ -201,7 +317,7 @@ static inline int ddebug_remove_module(const char *mod) static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, const char *modname) { - if (strstr(param, "dyndbg")) { + if (!strcmp(param, "dyndbg")) { /* avoid pr_warn(), which wants pr_fmt() fully defined */ printk(KERN_WARNING "dyndbg param is supported only in " "CONFIG_DYNAMIC_DEBUG builds\n"); @@ -221,12 +337,14 @@ static inline int ddebug_dyndbg_module_param_cb(char *param, char *val, rowsize, groupsize, buf, len, ascii); \ } while (0) -static inline int dynamic_debug_exec_queries(const char *query, const char *modname) -{ - pr_warn("kernel not built with CONFIG_DYNAMIC_DEBUG_CORE\n"); - return 0; -} +struct kernel_param; +static inline int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp) +{ return 0; } +static inline int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp) +{ return 0; } #endif /* !CONFIG_DYNAMIC_DEBUG_CORE */ +extern const struct kernel_param_ops param_ops_dyndbg_classes; + #endif diff --git a/include/linux/edac.h b/include/linux/edac.h index e730b3468719dda3cc4a19fb48a5a8dd0e94f13e..fa4bda2a70f6c61aa7a231f4b4710ffc7b8fe996 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -231,21 +231,21 @@ enum mem_type { #define MEM_FLAG_DDR BIT(MEM_DDR) #define MEM_FLAG_RDDR BIT(MEM_RDDR) #define MEM_FLAG_RMBS BIT(MEM_RMBS) -#define MEM_FLAG_DDR2 BIT(MEM_DDR2) -#define MEM_FLAG_FB_DDR2 BIT(MEM_FB_DDR2) -#define MEM_FLAG_RDDR2 BIT(MEM_RDDR2) -#define MEM_FLAG_XDR BIT(MEM_XDR) -#define MEM_FLAG_DDR3 BIT(MEM_DDR3) -#define MEM_FLAG_RDDR3 BIT(MEM_RDDR3) -#define MEM_FLAG_LPDDR3 BIT(MEM_LPDDR3) -#define MEM_FLAG_DDR4 BIT(MEM_DDR4) -#define MEM_FLAG_RDDR4 BIT(MEM_RDDR4) -#define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) -#define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) -#define MEM_FLAG_DDR5 BIT(MEM_DDR5) -#define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) -#define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) -#define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) +#define MEM_FLAG_DDR2 BIT(MEM_DDR2) +#define MEM_FLAG_FB_DDR2 BIT(MEM_FB_DDR2) +#define MEM_FLAG_RDDR2 BIT(MEM_RDDR2) +#define MEM_FLAG_XDR BIT(MEM_XDR) +#define MEM_FLAG_DDR3 BIT(MEM_DDR3) +#define MEM_FLAG_RDDR3 BIT(MEM_RDDR3) +#define MEM_FLAG_LPDDR3 BIT(MEM_LPDDR3) +#define MEM_FLAG_DDR4 BIT(MEM_DDR4) +#define MEM_FLAG_RDDR4 BIT(MEM_RDDR4) +#define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) +#define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) +#define MEM_FLAG_DDR5 BIT(MEM_DDR5) +#define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) +#define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) +#define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) #define MEM_FLAG_HBM2 BIT(MEM_HBM2) diff --git a/include/linux/efi.h b/include/linux/efi.h index d2b84c2fec39f0268324d1a38a73ed67786973c9..da3974bf05d3e86a6aeaf8f6479719d09a2f407e 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -368,6 +368,9 @@ void efi_native_runtime_setup(void); #define UV_SYSTEM_TABLE_GUID EFI_GUID(0x3b13a7d4, 0x633e, 0x11dd, 0x93, 0xec, 0xda, 0x25, 0x56, 0xd8, 0x95, 0x93) #define LINUX_EFI_CRASH_GUID EFI_GUID(0xcfc8fc79, 0xbe2e, 0x4ddc, 0x97, 0xf0, 0x9f, 0x98, 0xbf, 0xe2, 0x98, 0xa0) #define LOADED_IMAGE_PROTOCOL_GUID EFI_GUID(0x5b1b31a1, 0x9562, 0x11d2, 0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b) +#define LOADED_IMAGE_DEVICE_PATH_PROTOCOL_GUID EFI_GUID(0xbc62157e, 0x3e33, 0x4fec, 0x99, 0x20, 0x2d, 0x3b, 0x36, 0xd7, 0x50, 0xdf) +#define EFI_DEVICE_PATH_PROTOCOL_GUID EFI_GUID(0x09576e91, 0x6d3f, 0x11d2, 0x8e, 0x39, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b) +#define EFI_DEVICE_PATH_TO_TEXT_PROTOCOL_GUID EFI_GUID(0x8b843e20, 0x8132, 0x4852, 0x90, 0xcc, 0x55, 0x1a, 0x4e, 0x4a, 0x7f, 0x1c) #define EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID EFI_GUID(0x9042a9de, 0x23dc, 0x4a38, 0x96, 0xfb, 0x7a, 0xde, 0xd0, 0x80, 0x51, 0x6a) #define EFI_UGA_PROTOCOL_GUID EFI_GUID(0x982c298b, 0xf4fa, 0x41cb, 0xb8, 0x38, 0x77, 0xaa, 0x68, 0x8f, 0xb8, 0x39) #define EFI_PCI_IO_PROTOCOL_GUID EFI_GUID(0x4cf5b200, 0x68b8, 0x4ca5, 0x9e, 0xec, 0xb2, 0x3e, 0x3f, 0x50, 0x02, 0x9a) @@ -408,8 +411,10 @@ void efi_native_runtime_setup(void); #define LINUX_EFI_TPM_FINAL_LOG_GUID EFI_GUID(0x1e2ed096, 0x30e2, 0x4254, 0xbd, 0x89, 0x86, 0x3b, 0xbe, 0xf8, 0x23, 0x25) #define LINUX_EFI_MEMRESERVE_TABLE_GUID EFI_GUID(0x888eb0c6, 0x8ede, 0x4ff5, 0xa8, 0xf0, 0x9a, 0xee, 0x5c, 0xb9, 0x77, 0xc2) #define LINUX_EFI_INITRD_MEDIA_GUID EFI_GUID(0x5568e427, 0x68fc, 0x4f3d, 0xac, 0x74, 0xca, 0x55, 0x52, 0x31, 0xcc, 0x68) +#define LINUX_EFI_ZBOOT_MEDIA_GUID EFI_GUID(0xe565a30d, 0x47da, 0x4dbd, 0xb3, 0x54, 0x9b, 0xb5, 0xc8, 0x4f, 0x8b, 0xe2) #define LINUX_EFI_MOK_VARIABLE_TABLE_GUID EFI_GUID(0xc451ed2b, 0x9694, 0x45d3, 0xba, 0xba, 0xed, 0x9f, 0x89, 0x88, 0xa3, 0x89) #define LINUX_EFI_COCO_SECRET_AREA_GUID EFI_GUID(0xadf956ad, 0xe98c, 0x484c, 0xae, 0x11, 0xb5, 0x1c, 0x7d, 0x33, 0x64, 0x47) +#define LINUX_EFI_BOOT_MEMMAP_GUID EFI_GUID(0x800f683f, 0xd08b, 0x423a, 0xa2, 0x93, 0x96, 0x5c, 0x3c, 0x6f, 0xe2, 0xb4) #define RISCV_EFI_BOOT_PROTOCOL_GUID EFI_GUID(0xccd15fec, 0x6f73, 0x4eec, 0x83, 0x95, 0x3e, 0x69, 0xe4, 0xb9, 0x40, 0xbf) @@ -518,6 +523,15 @@ typedef union { efi_system_table_32_t mixed_mode; } efi_system_table_t; +struct efi_boot_memmap { + unsigned long map_size; + unsigned long desc_size; + u32 desc_ver; + unsigned long map_key; + unsigned long buff_size; + efi_memory_desc_t map[]; +}; + /* * Architecture independent structure for describing a memory map for the * benefit of efi_memmap_init_early(), and for passing context between @@ -952,6 +966,7 @@ extern int efi_status_to_err(efi_status_t status); #define EFI_DEV_MEDIA_VENDOR 3 #define EFI_DEV_MEDIA_FILE 4 #define EFI_DEV_MEDIA_PROTOCOL 5 +#define EFI_DEV_MEDIA_REL_OFFSET 8 #define EFI_DEV_BIOS_BOOT 0x05 #define EFI_DEV_END_PATH 0x7F #define EFI_DEV_END_PATH2 0xFF @@ -982,12 +997,27 @@ struct efi_vendor_dev_path { u8 vendordata[]; } __packed; +struct efi_rel_offset_dev_path { + struct efi_generic_dev_path header; + u32 reserved; + u64 starting_offset; + u64 ending_offset; +} __packed; + +struct efi_mem_mapped_dev_path { + struct efi_generic_dev_path header; + u32 memory_type; + u64 starting_addr; + u64 ending_addr; +} __packed; + struct efi_dev_path { union { struct efi_generic_dev_path header; struct efi_acpi_dev_path acpi; struct efi_pci_dev_path pci; struct efi_vendor_dev_path vendor; + struct efi_rel_offset_dev_path rel_offset; }; } __packed; @@ -1321,6 +1351,11 @@ struct linux_efi_coco_secret_area { u64 size; }; +struct linux_efi_initrd { + unsigned long base; + unsigned long size; +}; + /* Header of a populated EFI secret area */ #define EFI_SECRET_TABLE_HEADER_GUID EFI_GUID(0x1e74f542, 0x71dd, 0x4d66, 0x96, 0x3e, 0xef, 0x42, 0x87, 0xff, 0x17, 0x3b) diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index 84a466b176cf41be0184549824a404c32a8062a7..d95ab85f96ba5a41527722cd9f4a6f6a29554c2e 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -253,7 +253,6 @@ static __always_inline void arch_exit_to_user_mode(void) { } /** * arch_do_signal_or_restart - Architecture specific signal delivery function * @regs: Pointer to currents pt_regs - * @has_signal: actual signal to handle * * Invoked from exit_to_user_mode_loop(). */ diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h index 92b10e67d5f87e7671d8a39eae46677171c0915f..a541f0c4f146c85724b73d2fb38b042bdc9d2154 100644 --- a/include/linux/etherdevice.h +++ b/include/linux/etherdevice.h @@ -428,6 +428,28 @@ static inline bool ether_addr_equal_masked(const u8 *addr1, const u8 *addr2, return true; } +static inline bool ether_addr_is_ipv4_mcast(const u8 *addr) +{ + u8 base[ETH_ALEN] = { 0x01, 0x00, 0x5e, 0x00, 0x00, 0x00 }; + u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0x80, 0x00, 0x00 }; + + return ether_addr_equal_masked(addr, base, mask); +} + +static inline bool ether_addr_is_ipv6_mcast(const u8 *addr) +{ + u8 base[ETH_ALEN] = { 0x33, 0x33, 0x00, 0x00, 0x00, 0x00 }; + u8 mask[ETH_ALEN] = { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }; + + return ether_addr_equal_masked(addr, base, mask); +} + +static inline bool ether_addr_is_ip_mcast(const u8 *addr) +{ + return ether_addr_is_ipv4_mcast(addr) || + ether_addr_is_ipv6_mcast(addr); +} + /** * ether_addr_to_u64 - Convert an Ethernet address into a u64 value. * @addr: Pointer to a six-byte array containing the Ethernet address diff --git a/include/linux/eventfd.h b/include/linux/eventfd.h index 305d5f19093b9239ce454fbcf0eba948ce3ce9a3..30eb30d6909b05f81b838c31cff03495b429f238 100644 --- a/include/linux/eventfd.h +++ b/include/linux/eventfd.h @@ -46,7 +46,7 @@ void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt); static inline bool eventfd_signal_allowed(void) { - return !current->in_eventfd_signal; + return !current->in_eventfd; } #else /* CONFIG_EVENTFD */ diff --git a/include/linux/export-internal.h b/include/linux/export-internal.h index c2b1d4fd5987330021bb4c161fa034b2dc7ec15f..fe7e6ba918f108d0de45193bff987bde136ef340 100644 --- a/include/linux/export-internal.h +++ b/include/linux/export-internal.h @@ -10,8 +10,10 @@ #include #include -/* __used is needed to keep __crc_* for LTO */ #define SYMBOL_CRC(sym, crc, sec) \ - u32 __section("___kcrctab" sec "+" #sym) __used __crc_##sym = crc + asm(".section \"___kcrctab" sec "+" #sym "\",\"a\"" "\n" \ + "__crc_" #sym ":" "\n" \ + ".long " #crc "\n" \ + ".previous" "\n") #endif /* __LINUX_EXPORT_INTERNAL_H__ */ diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index d445150c5350fa4698be486f80fa3205a98ac31d..ee0d75d9a302d47649c79c7dcb4c24abd0bbd547 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -73,6 +73,42 @@ struct f2fs_device { __le32 total_segments; } __packed; +/* reason of stop_checkpoint */ +enum stop_cp_reason { + STOP_CP_REASON_SHUTDOWN, + STOP_CP_REASON_FAULT_INJECT, + STOP_CP_REASON_META_PAGE, + STOP_CP_REASON_WRITE_FAIL, + STOP_CP_REASON_CORRUPTED_SUMMARY, + STOP_CP_REASON_UPDATE_INODE, + STOP_CP_REASON_FLUSH_FAIL, + STOP_CP_REASON_MAX, +}; + +#define MAX_STOP_REASON 32 + +/* detail reason for EFSCORRUPTED */ +enum f2fs_error { + ERROR_CORRUPTED_CLUSTER, + ERROR_FAIL_DECOMPRESSION, + ERROR_INVALID_BLKADDR, + ERROR_CORRUPTED_DIRENT, + ERROR_CORRUPTED_INODE, + ERROR_INCONSISTENT_SUMMARY, + ERROR_INCONSISTENT_FOOTER, + ERROR_INCONSISTENT_SUM_TYPE, + ERROR_CORRUPTED_JOURNAL, + ERROR_INCONSISTENT_NODE_COUNT, + ERROR_INCONSISTENT_BLOCK_COUNT, + ERROR_INVALID_CURSEG, + ERROR_INCONSISTENT_SIT, + ERROR_CORRUPTED_VERITY_XATTR, + ERROR_CORRUPTED_XATTR, + ERROR_MAX, +}; + +#define MAX_F2FS_ERRORS 16 + struct f2fs_super_block { __le32 magic; /* Magic Number */ __le16 major_ver; /* Major Version */ @@ -116,7 +152,9 @@ struct f2fs_super_block { __u8 hot_ext_count; /* # of hot file extension */ __le16 s_encoding; /* Filename charset encoding */ __le16 s_encoding_flags; /* Filename charset encoding flags */ - __u8 reserved[306]; /* valid reserved region */ + __u8 s_stop_reason[MAX_STOP_REASON]; /* stop checkpoint reason */ + __u8 s_errors[MAX_F2FS_ERRORS]; /* reason of image corrupts */ + __u8 reserved[258]; /* valid reserved region */ __le32 crc; /* checksum of superblock */ } __packed; diff --git a/include/linux/filter.h b/include/linux/filter.h index a5f21dc3c4327105d10de6c1665955b3b1bee2f1..efc42a6e3aed0f6605827c5b879e1207fa616608 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -567,6 +567,12 @@ struct sk_filter { DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); +extern struct mutex nf_conn_btf_access_lock; +extern int (*nfct_btf_struct_access)(struct bpf_verifier_log *log, const struct btf *btf, + const struct btf_type *t, int off, int size, + enum bpf_access_type atype, u32 *next_btf_id, + enum bpf_type_flag *flag); + typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx, const struct bpf_insn *insnsi, unsigned int (*bpf_func)(const void *, @@ -900,8 +906,7 @@ int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk); int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk); void sk_reuseport_prog_free(struct bpf_prog *prog); int sk_detach_filter(struct sock *sk); -int sk_get_filter(struct sock *sk, struct sock_filter __user *filter, - unsigned int len); +int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len); bool sk_filter_charge(struct sock *sk, struct sk_filter *fp); void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); @@ -1018,6 +1023,8 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); + struct bpf_binary_header * bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, @@ -1030,6 +1037,9 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void bpf_prog_pack_free(struct bpf_binary_header *hdr); + static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) { return list_empty(&fp->aux->ksym.lnode) || @@ -1100,7 +1110,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog) return false; if (!bpf_jit_harden) return false; - if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN)) + if (bpf_jit_harden == 1 && bpf_capable()) return false; return true; diff --git a/include/linux/find.h b/include/linux/find.h index 424ef67d4a426837eb49524634badbca72b07b3b..ccaf61a0f5fd5a6c89d7507fece5ea3afff56116 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -8,15 +8,33 @@ #include -extern unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le); +unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, + unsigned long start); +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start); +unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start); +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start); extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); +unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n); +unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n); +unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n); extern unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size); extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); +#ifdef __BIG_ENDIAN +unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size); +unsigned long _find_next_zero_bit_le(const unsigned long *addr, unsigned + long size, unsigned long offset); +unsigned long _find_next_bit_le(const unsigned long *addr, unsigned + long size, unsigned long offset); +#endif + #ifndef find_next_bit /** * find_next_bit - find the next set bit in a memory region @@ -41,7 +59,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, return val ? __ffs(val) : size; } - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); + return _find_next_bit(addr, size, offset); } #endif @@ -71,7 +89,38 @@ unsigned long find_next_and_bit(const unsigned long *addr1, return val ? __ffs(val) : size; } - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); + return _find_next_and_bit(addr1, addr2, size, offset); +} +#endif + +#ifndef find_next_andnot_bit +/** + * find_next_andnot_bit - find the next set bit in *addr1 excluding all the bits + * in *addr2 + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit + * If no bits are set, returns @size. + */ +static inline +unsigned long find_next_andnot_bit(const unsigned long *addr1, + const unsigned long *addr2, unsigned long size, + unsigned long offset) +{ + if (small_const_nbits(size)) { + unsigned long val; + + if (unlikely(offset >= size)) + return size; + + val = *addr1 & ~*addr2 & GENMASK(size - 1, offset); + return val ? __ffs(val) : size; + } + + return _find_next_andnot_bit(addr1, addr2, size, offset); } #endif @@ -99,7 +148,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, return val == ~0UL ? size : ffz(val); } - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); + return _find_next_zero_bit(addr, size, offset); } #endif @@ -125,6 +174,87 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) } #endif +/** + * find_nth_bit - find N'th set bit in a memory region + * @addr: The address to start the search at + * @size: The maximum number of bits to search + * @n: The number of set bit, which position is needed, counting from 0 + * + * The following is semantically equivalent: + * idx = find_nth_bit(addr, size, 0); + * idx = find_first_bit(addr, size); + * + * Returns the bit number of the N'th set bit. + * If no such, returns @size. + */ +static inline +unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) +{ + if (n >= size) + return size; + + if (small_const_nbits(size)) { + unsigned long val = *addr & GENMASK(size - 1, 0); + + return val ? fns(val, n) : size; + } + + return __find_nth_bit(addr, size, n); +} + +/** + * find_nth_and_bit - find N'th set bit in 2 memory regions + * @addr1: The 1st address to start the search at + * @addr2: The 2nd address to start the search at + * @size: The maximum number of bits to search + * @n: The number of set bit, which position is needed, counting from 0 + * + * Returns the bit number of the N'th set bit. + * If no such, returns @size. + */ +static inline +unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n) +{ + if (n >= size) + return size; + + if (small_const_nbits(size)) { + unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0); + + return val ? fns(val, n) : size; + } + + return __find_nth_and_bit(addr1, addr2, size, n); +} + +/** + * find_nth_andnot_bit - find N'th set bit in 2 memory regions, + * flipping bits in 2nd region + * @addr1: The 1st address to start the search at + * @addr2: The 2nd address to start the search at + * @size: The maximum number of bits to search + * @n: The number of set bit, which position is needed, counting from 0 + * + * Returns the bit number of the N'th set bit. + * If no such, returns @size. + */ +static inline +unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n) +{ + if (n >= size) + return size; + + if (small_const_nbits(size)) { + unsigned long val = *addr1 & (~*addr2) & GENMASK(size - 1, 0); + + return val ? fns(val, n) : size; + } + + return __find_nth_andnot_bit(addr1, addr2, size, n); +} + #ifndef find_first_and_bit /** * find_first_and_bit - find the first set bit in both memory regions @@ -193,6 +323,78 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) } #endif +/** + * find_next_and_bit_wrap - find the next set bit in both memory regions + * @addr1: The first address to base the search on + * @addr2: The second address to base the search on + * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit, or first set bit up to @offset + * If no bits are set, returns @size. + */ +static inline +unsigned long find_next_and_bit_wrap(const unsigned long *addr1, + const unsigned long *addr2, + unsigned long size, unsigned long offset) +{ + unsigned long bit = find_next_and_bit(addr1, addr2, size, offset); + + if (bit < size) + return bit; + + bit = find_first_and_bit(addr1, addr2, offset); + return bit < offset ? bit : size; +} + +/** + * find_next_bit_wrap - find the next set bit in both memory regions + * @addr: The first address to base the search on + * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at + * + * Returns the bit number for the next set bit, or first set bit up to @offset + * If no bits are set, returns @size. + */ +static inline +unsigned long find_next_bit_wrap(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + unsigned long bit = find_next_bit(addr, size, offset); + + if (bit < size) + return bit; + + bit = find_first_bit(addr, offset); + return bit < offset ? bit : size; +} + +/* + * Helper for for_each_set_bit_wrap(). Make sure you're doing right thing + * before using it alone. + */ +static inline +unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size, + unsigned long start, unsigned long n) +{ + unsigned long bit; + + /* If not wrapped around */ + if (n > start) { + /* and have a bit, just return it. */ + bit = find_next_bit(bitmap, size, n); + if (bit < size) + return bit; + + /* Otherwise, wrap around and ... */ + n = 0; + } + + /* Search the other part. */ + bit = find_next_bit(bitmap, start, n); + return bit < start ? bit : size; +} + /** * find_next_clump8 - find next 8-bit clump with set bits in a memory region * @clump: location to store copy of found clump @@ -247,7 +449,21 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned return val == ~0UL ? size : ffz(val); } - return _find_next_bit(addr, NULL, size, offset, ~0UL, 1); + return _find_next_zero_bit_le(addr, size, offset); +} +#endif + +#ifndef find_first_zero_bit_le +static inline +unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) +{ + if (small_const_nbits(size)) { + unsigned long val = swab(*(const unsigned long *)addr) | ~GENMASK(size - 1, 0); + + return val == ~0UL ? size : ffz(val); + } + + return _find_first_zero_bit_le(addr, size); } #endif @@ -266,40 +482,39 @@ unsigned long find_next_bit_le(const void *addr, unsigned return val ? __ffs(val) : size; } - return _find_next_bit(addr, NULL, size, offset, 0UL, 1); + return _find_next_bit_le(addr, size, offset); } #endif -#ifndef find_first_zero_bit_le -#define find_first_zero_bit_le(addr, size) \ - find_next_zero_bit_le((addr), (size), 0) -#endif - #else #error "Please fix " #endif #define for_each_set_bit(bit, addr, size) \ - for ((bit) = find_next_bit((addr), (size), 0); \ - (bit) < (size); \ - (bit) = find_next_bit((addr), (size), (bit) + 1)) + for ((bit) = 0; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) + +#define for_each_and_bit(bit, addr1, addr2, size) \ + for ((bit) = 0; \ + (bit) = find_next_and_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ + (bit)++) + +#define for_each_andnot_bit(bit, addr1, addr2, size) \ + for ((bit) = 0; \ + (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\ + (bit)++) /* same as for_each_set_bit() but use bit as value to start with */ #define for_each_set_bit_from(bit, addr, size) \ - for ((bit) = find_next_bit((addr), (size), (bit)); \ - (bit) < (size); \ - (bit) = find_next_bit((addr), (size), (bit) + 1)) + for (; (bit) = find_next_bit((addr), (size), (bit)), (bit) < (size); (bit)++) #define for_each_clear_bit(bit, addr, size) \ - for ((bit) = find_next_zero_bit((addr), (size), 0); \ - (bit) < (size); \ - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) + for ((bit) = 0; \ + (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); \ + (bit)++) /* same as for_each_clear_bit() but use bit as value to start with */ #define for_each_clear_bit_from(bit, addr, size) \ - for ((bit) = find_next_zero_bit((addr), (size), (bit)); \ - (bit) < (size); \ - (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) + for (; (bit) = find_next_zero_bit((addr), (size), (bit)), (bit) < (size); (bit)++) /** * for_each_set_bitrange - iterate over all set bit ranges [b; e) @@ -309,11 +524,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned * @size: bitmap size in number of bits */ #define for_each_set_bitrange(b, e, addr, size) \ - for ((b) = find_next_bit((addr), (size), 0), \ - (e) = find_next_zero_bit((addr), (size), (b) + 1); \ + for ((b) = 0; \ + (b) = find_next_bit((addr), (size), b), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1), \ (b) < (size); \ - (b) = find_next_bit((addr), (size), (e) + 1), \ - (e) = find_next_zero_bit((addr), (size), (b) + 1)) + (b) = (e) + 1) /** * for_each_set_bitrange_from - iterate over all set bit ranges [b; e) @@ -323,11 +538,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned * @size: bitmap size in number of bits */ #define for_each_set_bitrange_from(b, e, addr, size) \ - for ((b) = find_next_bit((addr), (size), (b)), \ - (e) = find_next_zero_bit((addr), (size), (b) + 1); \ + for (; \ + (b) = find_next_bit((addr), (size), (b)), \ + (e) = find_next_zero_bit((addr), (size), (b) + 1), \ (b) < (size); \ - (b) = find_next_bit((addr), (size), (e) + 1), \ - (e) = find_next_zero_bit((addr), (size), (b) + 1)) + (b) = (e) + 1) /** * for_each_clear_bitrange - iterate over all unset bit ranges [b; e) @@ -337,11 +552,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned * @size: bitmap size in number of bits */ #define for_each_clear_bitrange(b, e, addr, size) \ - for ((b) = find_next_zero_bit((addr), (size), 0), \ - (e) = find_next_bit((addr), (size), (b) + 1); \ + for ((b) = 0; \ + (b) = find_next_zero_bit((addr), (size), (b)), \ + (e) = find_next_bit((addr), (size), (b) + 1), \ (b) < (size); \ - (b) = find_next_zero_bit((addr), (size), (e) + 1), \ - (e) = find_next_bit((addr), (size), (b) + 1)) + (b) = (e) + 1) /** * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) @@ -351,11 +566,24 @@ unsigned long find_next_bit_le(const void *addr, unsigned * @size: bitmap size in number of bits */ #define for_each_clear_bitrange_from(b, e, addr, size) \ - for ((b) = find_next_zero_bit((addr), (size), (b)), \ - (e) = find_next_bit((addr), (size), (b) + 1); \ + for (; \ + (b) = find_next_zero_bit((addr), (size), (b)), \ + (e) = find_next_bit((addr), (size), (b) + 1), \ (b) < (size); \ - (b) = find_next_zero_bit((addr), (size), (e) + 1), \ - (e) = find_next_bit((addr), (size), (b) + 1)) + (b) = (e) + 1) + +/** + * for_each_set_bit_wrap - iterate over all set bits starting from @start, and + * wrapping around the end of bitmap. + * @bit: offset for current iteration + * @addr: bitmap address to base the search on + * @size: bitmap size in number of bits + * @start: Starting bit for bitmap traversing, wrapping around the bitmap end + */ +#define for_each_set_bit_wrap(bit, addr, size, start) \ + for ((bit) = find_next_bit_wrap((addr), (size), (start)); \ + (bit) < (size); \ + (bit) = __for_each_wrap((addr), (size), (start), (bit) + 1)) /** * for_each_set_clump8 - iterate over bitmap for each 8-bit clump with set bits diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index 9f50dacbf7d637a373eb5b425101b5a447eb55b9..76d2b3ebad8477a962dd2fc37f14509a09b2246a 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -153,6 +153,9 @@ enum pm_ioctl_id { /* Runtime feature configuration */ IOCTL_SET_FEATURE_CONFIG = 26, IOCTL_GET_FEATURE_CONFIG = 27, + /* Dynamic SD/GEM configuration */ + IOCTL_SET_SD_CONFIG = 30, + IOCTL_SET_GEM_CONFIG = 31, }; enum pm_query_id { @@ -399,6 +402,30 @@ enum pm_feature_config_id { PM_FEATURE_EXTWDT_VALUE = 4, }; +/** + * enum pm_sd_config_type - PM SD configuration. + * @SD_CONFIG_EMMC_SEL: To set SD_EMMC_SEL in CTRL_REG_SD and SD_SLOTTYPE + * @SD_CONFIG_BASECLK: To set SD_BASECLK in SD_CONFIG_REG1 + * @SD_CONFIG_8BIT: To set SD_8BIT in SD_CONFIG_REG2 + * @SD_CONFIG_FIXED: To set fixed config registers + */ +enum pm_sd_config_type { + SD_CONFIG_EMMC_SEL = 1, + SD_CONFIG_BASECLK = 2, + SD_CONFIG_8BIT = 3, + SD_CONFIG_FIXED = 4, +}; + +/** + * enum pm_gem_config_type - PM GEM configuration. + * @GEM_CONFIG_SGMII_MODE: To set GEM_SGMII_MODE in GEM_CLK_CTRL register + * @GEM_CONFIG_FIXED: To set fixed config registers + */ +enum pm_gem_config_type { + GEM_CONFIG_SGMII_MODE = 1, + GEM_CONFIG_FIXED = 2, +}; + /** * struct zynqmp_pm_query_data - PM query data * @qid: query ID @@ -475,6 +502,9 @@ int zynqmp_pm_is_function_supported(const u32 api_id, const u32 id); int zynqmp_pm_set_feature_config(enum pm_feature_config_id id, u32 value); int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, u32 *payload); int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset); +int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value); +int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config, + u32 value); #else static inline int zynqmp_pm_get_api_version(u32 *version) { @@ -745,6 +775,21 @@ static inline int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset) { return -ENODEV; } + +static inline int zynqmp_pm_set_sd_config(u32 node, + enum pm_sd_config_type config, + u32 value) +{ + return -ENODEV; +} + +static inline int zynqmp_pm_set_gem_config(u32 node, + enum pm_gem_config_type config, + u32 value) +{ + return -ENODEV; +} + #endif #endif /* __FIRMWARE_ZYNQMP_H__ */ diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 3b401fa0f37462825af2c1d9d371998d52de03ef..4029fe368a4f6ce2305f47ec7fd29a0c2581b0a0 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -2,7 +2,9 @@ #ifndef _LINUX_FORTIFY_STRING_H_ #define _LINUX_FORTIFY_STRING_H_ +#include #include +#include #define __FORTIFY_INLINE extern __always_inline __gnu_inline __overloadable #define __RENAME(x) __asm__(#x) @@ -17,9 +19,10 @@ void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning(" #define __compiletime_strlen(p) \ ({ \ unsigned char *__p = (unsigned char *)(p); \ - size_t __ret = (size_t)-1; \ - size_t __p_size = __builtin_object_size(p, 1); \ - if (__p_size != (size_t)-1) { \ + size_t __ret = SIZE_MAX; \ + size_t __p_size = __member_size(p); \ + if (__p_size != SIZE_MAX && \ + __builtin_constant_p(*__p)) { \ size_t __p_len = __p_size - 1; \ if (__builtin_constant_p(__p[__p_len]) && \ __p[__p_len] == '\0') \ @@ -69,20 +72,59 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __underlying_memcpy(dst, src, bytes) /* - * Clang's use of __builtin_object_size() within inlines needs hinting via - * __pass_object_size(). The preference is to only ever use type 1 (member + * Clang's use of __builtin_*object_size() within inlines needs hinting via + * __pass_*object_size(). The preference is to only ever use type 1 (member * size, rather than struct size), but there remain some stragglers using * type 0 that will be converted in the future. */ -#define POS __pass_object_size(1) -#define POS0 __pass_object_size(0) +#define POS __pass_object_size(1) +#define POS0 __pass_object_size(0) +#define __struct_size(p) __builtin_object_size(p, 0) +#define __member_size(p) __builtin_object_size(p, 1) +#define __compiletime_lessthan(bounds, length) ( \ + __builtin_constant_p((bounds) < (length)) && \ + (bounds) < (length) \ +) + +/** + * strncpy - Copy a string to memory with non-guaranteed NUL padding + * + * @p: pointer to destination of copy + * @q: pointer to NUL-terminated source string to copy + * @size: bytes to write at @p + * + * If strlen(@q) >= @size, the copy of @q will stop after @size bytes, + * and @p will NOT be NUL-terminated + * + * If strlen(@q) < @size, following the copy of @q, trailing NUL bytes + * will be written to @p until @size total bytes have been written. + * + * Do not use this function. While FORTIFY_SOURCE tries to avoid + * over-reads of @q, it cannot defend against writing unterminated + * results to @p. Using strncpy() remains ambiguous and fragile. + * Instead, please choose an alternative, so that the expectation + * of @p's contents is unambiguous: + * + * +--------------------+-----------------+------------+ + * | @p needs to be: | padded to @size | not padded | + * +====================+=================+============+ + * | NUL-terminated | strscpy_pad() | strscpy() | + * +--------------------+-----------------+------------+ + * | not NUL-terminated | strtomem_pad() | strtomem() | + * +--------------------+-----------------+------------+ + * + * Note strscpy*()'s differing return values for detecting truncation, + * and strtomem*()'s expectation that the destination is marked with + * __nonstring when it is a character array. + * + */ __FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3) char *strncpy(char * const POS p, const char *q, __kernel_size_t size) { - size_t p_size = __builtin_object_size(p, 1); + size_t p_size = __member_size(p); - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __write_overflow(); if (p_size < size) fortify_panic(__func__); @@ -92,9 +134,9 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size) __FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2) char *strcat(char * const POS p, const char *q) { - size_t p_size = __builtin_object_size(p, 1); + size_t p_size = __member_size(p); - if (p_size == (size_t)-1) + if (p_size == SIZE_MAX) return __underlying_strcat(p, q); if (strlcat(p, q, p_size) >= p_size) fortify_panic(__func__); @@ -104,12 +146,12 @@ char *strcat(char * const POS p, const char *q) extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen); __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size_t maxlen) { - size_t p_size = __builtin_object_size(p, 1); + size_t p_size = __member_size(p); size_t p_len = __compiletime_strlen(p); size_t ret; /* We can take compile-time actions when maxlen is const. */ - if (__builtin_constant_p(maxlen) && p_len != (size_t)-1) { + if (__builtin_constant_p(maxlen) && p_len != SIZE_MAX) { /* If p is const, we can use its compile-time-known len. */ if (maxlen >= p_size) return p_len; @@ -134,10 +176,10 @@ __FORTIFY_INLINE __diagnose_as(__builtin_strlen, 1) __kernel_size_t __fortify_strlen(const char * const POS p) { __kernel_size_t ret; - size_t p_size = __builtin_object_size(p, 1); + size_t p_size = __member_size(p); /* Give up if we don't know how large p is. */ - if (p_size == (size_t)-1) + if (p_size == SIZE_MAX) return __underlying_strlen(p); ret = strnlen(p, p_size); if (p_size <= ret) @@ -149,12 +191,12 @@ __kernel_size_t __fortify_strlen(const char * const POS p) extern size_t __real_strlcpy(char *, const char *, size_t) __RENAME(strlcpy); __FORTIFY_INLINE size_t strlcpy(char * const POS p, const char * const POS q, size_t size) { - size_t p_size = __builtin_object_size(p, 1); - size_t q_size = __builtin_object_size(q, 1); + size_t p_size = __member_size(p); + size_t q_size = __member_size(q); size_t q_len; /* Full count of source string length. */ size_t len; /* Count of characters going into destination. */ - if (p_size == (size_t)-1 && q_size == (size_t)-1) + if (p_size == SIZE_MAX && q_size == SIZE_MAX) return __real_strlcpy(p, q, size); q_len = strlen(q); len = (q_len >= size) ? size - 1 : q_len; @@ -178,18 +220,18 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s { size_t len; /* Use string size rather than possible enclosing struct size. */ - size_t p_size = __builtin_object_size(p, 1); - size_t q_size = __builtin_object_size(q, 1); + size_t p_size = __member_size(p); + size_t q_size = __member_size(q); /* If we cannot get size of p and q default to call strscpy. */ - if (p_size == (size_t) -1 && q_size == (size_t) -1) + if (p_size == SIZE_MAX && q_size == SIZE_MAX) return __real_strscpy(p, q, size); /* * If size can be known at compile time and is greater than * p_size, generate a compile time write overflow error. */ - if (__builtin_constant_p(size) && size > p_size) + if (__compiletime_lessthan(p_size, size)) __write_overflow(); /* @@ -224,10 +266,10 @@ __FORTIFY_INLINE __diagnose_as(__builtin_strncat, 1, 2, 3) char *strncat(char * const POS p, const char * const POS q, __kernel_size_t count) { size_t p_len, copy_len; - size_t p_size = __builtin_object_size(p, 1); - size_t q_size = __builtin_object_size(q, 1); + size_t p_size = __member_size(p); + size_t q_size = __member_size(q); - if (p_size == (size_t)-1 && q_size == (size_t)-1) + if (p_size == SIZE_MAX && q_size == SIZE_MAX) return __underlying_strncat(p, q, count); p_len = strlen(p); copy_len = strnlen(q, count); @@ -246,15 +288,16 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, /* * Length argument is a constant expression, so we * can perform compile-time bounds checking where - * buffer sizes are known. + * buffer sizes are also known at compile time. */ /* Error when size is larger than enclosing struct. */ - if (p_size > p_size_field && p_size < size) + if (__compiletime_lessthan(p_size_field, p_size) && + __compiletime_lessthan(p_size, size)) __write_overflow(); /* Warn when write size is larger than dest field. */ - if (p_size_field < size) + if (__compiletime_lessthan(p_size_field, size)) __write_overflow_field(p_size_field, size); } /* @@ -268,10 +311,10 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, /* * Always stop accesses beyond the struct that contains the * field, when the buffer's remaining size is known. - * (The -1 test is to optimize away checks where the buffer + * (The SIZE_MAX test is to optimize away checks where the buffer * lengths are unknown.) */ - if (p_size != (size_t)(-1) && p_size < size) + if (p_size != SIZE_MAX && p_size < size) fortify_panic("memset"); } @@ -282,11 +325,13 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, }) /* - * __builtin_object_size() must be captured here to avoid evaluating argument - * side-effects further into the macro layers. + * __struct_size() vs __member_size() must be captured here to avoid + * evaluating argument side-effects further into the macro layers. */ +#ifndef CONFIG_KMSAN #define memset(p, c, s) __fortify_memset_chk(p, c, s, \ - __builtin_object_size(p, 0), __builtin_object_size(p, 1)) + __struct_size(p), __member_size(p)) +#endif /* * To make sure the compiler can enforce protection against buffer overflows, @@ -319,7 +364,7 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size, * V = vulnerable to run-time overflow (will need refactoring to solve) * */ -__FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size, +__FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, const size_t p_size, const size_t q_size, const size_t p_size_field, @@ -330,25 +375,28 @@ __FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size, /* * Length argument is a constant expression, so we * can perform compile-time bounds checking where - * buffer sizes are known. + * buffer sizes are also known at compile time. */ /* Error when size is larger than enclosing struct. */ - if (p_size > p_size_field && p_size < size) + if (__compiletime_lessthan(p_size_field, p_size) && + __compiletime_lessthan(p_size, size)) __write_overflow(); - if (q_size > q_size_field && q_size < size) + if (__compiletime_lessthan(q_size_field, q_size) && + __compiletime_lessthan(q_size, size)) __read_overflow2(); /* Warn when write size argument larger than dest field. */ - if (p_size_field < size) + if (__compiletime_lessthan(p_size_field, size)) __write_overflow_field(p_size_field, size); /* * Warn for source field over-read when building with W=1 * or when an over-write happened, so both can be fixed at * the same time. */ - if ((IS_ENABLED(KBUILD_EXTRA_WARN1) || p_size_field < size) && - q_size_field < size) + if ((IS_ENABLED(KBUILD_EXTRA_WARN1) || + __compiletime_lessthan(p_size_field, size)) && + __compiletime_lessthan(q_size_field, size)) __read_overflow2_field(q_size_field, size); } /* @@ -362,41 +410,104 @@ __FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size, /* * Always stop accesses beyond the struct that contains the * field, when the buffer's remaining size is known. - * (The -1 test is to optimize away checks where the buffer + * (The SIZE_MAX test is to optimize away checks where the buffer * lengths are unknown.) */ - if ((p_size != (size_t)(-1) && p_size < size) || - (q_size != (size_t)(-1) && q_size < size)) + if ((p_size != SIZE_MAX && p_size < size) || + (q_size != SIZE_MAX && q_size < size)) fortify_panic(func); + + /* + * Warn when writing beyond destination field size. + * + * We must ignore p_size_field == 0 for existing 0-element + * fake flexible arrays, until they are all converted to + * proper flexible arrays. + * + * The implementation of __builtin_*object_size() behaves + * like sizeof() when not directly referencing a flexible + * array member, which means there will be many bounds checks + * that will appear at run-time, without a way for them to be + * detected at compile-time (as can be done when the destination + * is specifically the flexible array member). + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101832 + */ + if (p_size_field != 0 && p_size_field != SIZE_MAX && + p_size != p_size_field && p_size_field < size) + return true; + + return false; } #define __fortify_memcpy_chk(p, q, size, p_size, q_size, \ p_size_field, q_size_field, op) ({ \ size_t __fortify_size = (size_t)(size); \ - fortify_memcpy_chk(__fortify_size, p_size, q_size, \ - p_size_field, q_size_field, #op); \ + WARN_ONCE(fortify_memcpy_chk(__fortify_size, p_size, q_size, \ + p_size_field, q_size_field, #op), \ + #op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \ + __fortify_size, \ + "field \"" #p "\" at " __FILE__ ":" __stringify(__LINE__), \ + p_size_field); \ __underlying_##op(p, q, __fortify_size); \ }) /* - * __builtin_object_size() must be captured here to avoid evaluating argument - * side-effects further into the macro layers. + * Notes about compile-time buffer size detection: + * + * With these types... + * + * struct middle { + * u16 a; + * u8 middle_buf[16]; + * int b; + * }; + * struct end { + * u16 a; + * u8 end_buf[16]; + * }; + * struct flex { + * int a; + * u8 flex_buf[]; + * }; + * + * void func(TYPE *ptr) { ... } + * + * Cases where destination size cannot be currently detected: + * - the size of ptr's object (seemingly by design, gcc & clang fail): + * __builtin_object_size(ptr, 1) == SIZE_MAX + * - the size of flexible arrays in ptr's obj (by design, dynamic size): + * __builtin_object_size(ptr->flex_buf, 1) == SIZE_MAX + * - the size of ANY array at the end of ptr's obj (gcc and clang bug): + * __builtin_object_size(ptr->end_buf, 1) == SIZE_MAX + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 + * + * Cases where destination size is currently detected: + * - the size of non-array members within ptr's object: + * __builtin_object_size(ptr->a, 1) == 2 + * - the size of non-flexible-array in the middle of ptr's obj: + * __builtin_object_size(ptr->middle_buf, 1) == 16 + * + */ + +/* + * __struct_size() vs __member_size() must be captured here to avoid + * evaluating argument side-effects further into the macro layers. */ #define memcpy(p, q, s) __fortify_memcpy_chk(p, q, s, \ - __builtin_object_size(p, 0), __builtin_object_size(q, 0), \ - __builtin_object_size(p, 1), __builtin_object_size(q, 1), \ + __struct_size(p), __struct_size(q), \ + __member_size(p), __member_size(q), \ memcpy) #define memmove(p, q, s) __fortify_memcpy_chk(p, q, s, \ - __builtin_object_size(p, 0), __builtin_object_size(q, 0), \ - __builtin_object_size(p, 1), __builtin_object_size(q, 1), \ + __struct_size(p), __struct_size(q), \ + __member_size(p), __member_size(q), \ memmove) extern void *__real_memscan(void *, int, __kernel_size_t) __RENAME(memscan); __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) { - size_t p_size = __builtin_object_size(p, 0); + size_t p_size = __struct_size(p); - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) fortify_panic(__func__); @@ -406,13 +517,13 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size) __FORTIFY_INLINE __diagnose_as(__builtin_memcmp, 1, 2, 3) int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t size) { - size_t p_size = __builtin_object_size(p, 0); - size_t q_size = __builtin_object_size(q, 0); + size_t p_size = __struct_size(p); + size_t q_size = __struct_size(q); if (__builtin_constant_p(size)) { - if (p_size < size) + if (__compiletime_lessthan(p_size, size)) __read_overflow(); - if (q_size < size) + if (__compiletime_lessthan(q_size, size)) __read_overflow2(); } if (p_size < size || q_size < size) @@ -423,9 +534,9 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t __FORTIFY_INLINE __diagnose_as(__builtin_memchr, 1, 2, 3) void *memchr(const void * const POS0 p, int c, __kernel_size_t size) { - size_t p_size = __builtin_object_size(p, 0); + size_t p_size = __struct_size(p); - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) fortify_panic(__func__); @@ -435,9 +546,9 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size) void *__real_memchr_inv(const void *s, int c, size_t n) __RENAME(memchr_inv); __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) { - size_t p_size = __builtin_object_size(p, 0); + size_t p_size = __struct_size(p); - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) fortify_panic(__func__); @@ -447,9 +558,9 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size) extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup); __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp) { - size_t p_size = __builtin_object_size(p, 0); + size_t p_size = __struct_size(p); - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __read_overflow(); if (p_size < size) fortify_panic(__func__); @@ -460,16 +571,18 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp __FORTIFY_INLINE __diagnose_as(__builtin_strcpy, 1, 2) char *strcpy(char * const POS p, const char * const POS q) { - size_t p_size = __builtin_object_size(p, 1); - size_t q_size = __builtin_object_size(q, 1); + size_t p_size = __member_size(p); + size_t q_size = __member_size(q); size_t size; /* If neither buffer size is known, immediately give up. */ - if (p_size == (size_t)-1 && q_size == (size_t)-1) + if (__builtin_constant_p(p_size) && + __builtin_constant_p(q_size) && + p_size == SIZE_MAX && q_size == SIZE_MAX) return __underlying_strcpy(p, q); size = strlen(q) + 1; /* Compile-time check for const size overflow. */ - if (__builtin_constant_p(size) && p_size < size) + if (__compiletime_lessthan(p_size, size)) __write_overflow(); /* Run-time check for dynamic size overflow. */ if (p_size < size) diff --git a/include/linux/freezer.h b/include/linux/freezer.h index 0621c5f86c397e62d549051c19d97b4dff626748..b303472255be4e0886584863e1a462345fcc9fec 100644 --- a/include/linux/freezer.h +++ b/include/linux/freezer.h @@ -8,9 +8,11 @@ #include #include #include +#include #ifdef CONFIG_FREEZER -extern atomic_t system_freezing_cnt; /* nr of freezing conds in effect */ +DECLARE_STATIC_KEY_FALSE(freezer_active); + extern bool pm_freezing; /* PM freezing in effect */ extern bool pm_nosig_freezing; /* PM nosig freezing in effect */ @@ -22,10 +24,7 @@ extern unsigned int freeze_timeout_msecs; /* * Check if a process has been frozen */ -static inline bool frozen(struct task_struct *p) -{ - return p->flags & PF_FROZEN; -} +extern bool frozen(struct task_struct *p); extern bool freezing_slow_path(struct task_struct *p); @@ -34,9 +33,10 @@ extern bool freezing_slow_path(struct task_struct *p); */ static inline bool freezing(struct task_struct *p) { - if (likely(!atomic_read(&system_freezing_cnt))) - return false; - return freezing_slow_path(p); + if (static_branch_unlikely(&freezer_active)) + return freezing_slow_path(p); + + return false; } /* Takes and releases task alloc lock using task_lock() */ @@ -48,23 +48,14 @@ extern int freeze_kernel_threads(void); extern void thaw_processes(void); extern void thaw_kernel_threads(void); -/* - * DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION - * If try_to_freeze causes a lockdep warning it means the caller may deadlock - */ -static inline bool try_to_freeze_unsafe(void) +static inline bool try_to_freeze(void) { might_sleep(); if (likely(!freezing(current))) return false; - return __refrigerator(false); -} - -static inline bool try_to_freeze(void) -{ if (!(current->flags & PF_NOFREEZE)) debug_check_no_locks_held(); - return try_to_freeze_unsafe(); + return __refrigerator(false); } extern bool freeze_task(struct task_struct *p); @@ -79,195 +70,6 @@ static inline bool cgroup_freezing(struct task_struct *task) } #endif /* !CONFIG_CGROUP_FREEZER */ -/* - * The PF_FREEZER_SKIP flag should be set by a vfork parent right before it - * calls wait_for_completion(&vfork) and reset right after it returns from this - * function. Next, the parent should call try_to_freeze() to freeze itself - * appropriately in case the child has exited before the freezing of tasks is - * complete. However, we don't want kernel threads to be frozen in unexpected - * places, so we allow them to block freeze_processes() instead or to set - * PF_NOFREEZE if needed. Fortunately, in the ____call_usermodehelper() case the - * parent won't really block freeze_processes(), since ____call_usermodehelper() - * (the child) does a little before exec/exit and it can't be frozen before - * waking up the parent. - */ - - -/** - * freezer_do_not_count - tell freezer to ignore %current - * - * Tell freezers to ignore the current task when determining whether the - * target frozen state is reached. IOW, the current task will be - * considered frozen enough by freezers. - * - * The caller shouldn't do anything which isn't allowed for a frozen task - * until freezer_cont() is called. Usually, freezer[_do_not]_count() pair - * wrap a scheduling operation and nothing much else. - */ -static inline void freezer_do_not_count(void) -{ - current->flags |= PF_FREEZER_SKIP; -} - -/** - * freezer_count - tell freezer to stop ignoring %current - * - * Undo freezer_do_not_count(). It tells freezers that %current should be - * considered again and tries to freeze if freezing condition is already in - * effect. - */ -static inline void freezer_count(void) -{ - current->flags &= ~PF_FREEZER_SKIP; - /* - * If freezing is in progress, the following paired with smp_mb() - * in freezer_should_skip() ensures that either we see %true - * freezing() or freezer_should_skip() sees !PF_FREEZER_SKIP. - */ - smp_mb(); - try_to_freeze(); -} - -/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */ -static inline void freezer_count_unsafe(void) -{ - current->flags &= ~PF_FREEZER_SKIP; - smp_mb(); - try_to_freeze_unsafe(); -} - -/** - * freezer_should_skip - whether to skip a task when determining frozen - * state is reached - * @p: task in quesion - * - * This function is used by freezers after establishing %true freezing() to - * test whether a task should be skipped when determining the target frozen - * state is reached. IOW, if this function returns %true, @p is considered - * frozen enough. - */ -static inline bool freezer_should_skip(struct task_struct *p) -{ - /* - * The following smp_mb() paired with the one in freezer_count() - * ensures that either freezer_count() sees %true freezing() or we - * see cleared %PF_FREEZER_SKIP and return %false. This makes it - * impossible for a task to slip frozen state testing after - * clearing %PF_FREEZER_SKIP. - */ - smp_mb(); - return p->flags & PF_FREEZER_SKIP; -} - -/* - * These functions are intended to be used whenever you want allow a sleeping - * task to be frozen. Note that neither return any clear indication of - * whether a freeze event happened while in this function. - */ - -/* Like schedule(), but should not block the freezer. */ -static inline void freezable_schedule(void) -{ - freezer_do_not_count(); - schedule(); - freezer_count(); -} - -/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */ -static inline void freezable_schedule_unsafe(void) -{ - freezer_do_not_count(); - schedule(); - freezer_count_unsafe(); -} - -/* - * Like schedule_timeout(), but should not block the freezer. Do not - * call this with locks held. - */ -static inline long freezable_schedule_timeout(long timeout) -{ - long __retval; - freezer_do_not_count(); - __retval = schedule_timeout(timeout); - freezer_count(); - return __retval; -} - -/* - * Like schedule_timeout_interruptible(), but should not block the freezer. Do not - * call this with locks held. - */ -static inline long freezable_schedule_timeout_interruptible(long timeout) -{ - long __retval; - freezer_do_not_count(); - __retval = schedule_timeout_interruptible(timeout); - freezer_count(); - return __retval; -} - -/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */ -static inline long freezable_schedule_timeout_interruptible_unsafe(long timeout) -{ - long __retval; - - freezer_do_not_count(); - __retval = schedule_timeout_interruptible(timeout); - freezer_count_unsafe(); - return __retval; -} - -/* Like schedule_timeout_killable(), but should not block the freezer. */ -static inline long freezable_schedule_timeout_killable(long timeout) -{ - long __retval; - freezer_do_not_count(); - __retval = schedule_timeout_killable(timeout); - freezer_count(); - return __retval; -} - -/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */ -static inline long freezable_schedule_timeout_killable_unsafe(long timeout) -{ - long __retval; - freezer_do_not_count(); - __retval = schedule_timeout_killable(timeout); - freezer_count_unsafe(); - return __retval; -} - -/* - * Like schedule_hrtimeout_range(), but should not block the freezer. Do not - * call this with locks held. - */ -static inline int freezable_schedule_hrtimeout_range(ktime_t *expires, - u64 delta, const enum hrtimer_mode mode) -{ - int __retval; - freezer_do_not_count(); - __retval = schedule_hrtimeout_range(expires, delta, mode); - freezer_count(); - return __retval; -} - -/* - * Freezer-friendly wrappers around wait_event_interruptible(), - * wait_event_killable() and wait_event_interruptible_timeout(), originally - * defined in - */ - -/* DO NOT ADD ANY NEW CALLERS OF THIS FUNCTION */ -#define wait_event_freezekillable_unsafe(wq, condition) \ -({ \ - int __retval; \ - freezer_do_not_count(); \ - __retval = wait_event_killable(wq, (condition)); \ - freezer_count_unsafe(); \ - __retval; \ -}) - #else /* !CONFIG_FREEZER */ static inline bool frozen(struct task_struct *p) { return false; } static inline bool freezing(struct task_struct *p) { return false; } @@ -281,35 +83,8 @@ static inline void thaw_kernel_threads(void) {} static inline bool try_to_freeze(void) { return false; } -static inline void freezer_do_not_count(void) {} -static inline void freezer_count(void) {} -static inline int freezer_should_skip(struct task_struct *p) { return 0; } static inline void set_freezable(void) {} -#define freezable_schedule() schedule() - -#define freezable_schedule_unsafe() schedule() - -#define freezable_schedule_timeout(timeout) schedule_timeout(timeout) - -#define freezable_schedule_timeout_interruptible(timeout) \ - schedule_timeout_interruptible(timeout) - -#define freezable_schedule_timeout_interruptible_unsafe(timeout) \ - schedule_timeout_interruptible(timeout) - -#define freezable_schedule_timeout_killable(timeout) \ - schedule_timeout_killable(timeout) - -#define freezable_schedule_timeout_killable_unsafe(timeout) \ - schedule_timeout_killable(timeout) - -#define freezable_schedule_hrtimeout_range(expires, delta, mode) \ - schedule_hrtimeout_range(expires, delta, mode) - -#define wait_event_freezekillable_unsafe(wq, condition) \ - wait_event_killable(wq, condition) - #endif /* !CONFIG_FREEZER */ #endif /* FREEZER_H_INCLUDED */ diff --git a/include/linux/fs.h b/include/linux/fs.h index 9eced4cc286ee18ad996c337be92ba6fc9f57111..e654435f16512c122f176d0ab447925ae94f75fc 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1472,7 +1472,7 @@ struct super_block { const struct xattr_handler **s_xattr; #ifdef CONFIG_FS_ENCRYPTION const struct fscrypt_operations *s_cop; - struct key *s_master_keys; /* master crypto keys in use */ + struct fscrypt_keyring *s_master_keys; /* master crypto keys in use */ #endif #ifdef CONFIG_FS_VERITY const struct fsverity_operations *s_vop; @@ -2004,8 +2004,9 @@ static inline int vfs_whiteout(struct user_namespace *mnt_userns, WHITEOUT_DEV); } -struct dentry *vfs_tmpfile(struct user_namespace *mnt_userns, - struct dentry *dentry, umode_t mode, int open_flag); +struct file *vfs_tmpfile_open(struct user_namespace *mnt_userns, + const struct path *parentpath, + umode_t mode, int open_flag, const struct cred *cred); int vfs_mkobj(struct dentry *, umode_t, int (*f)(struct dentry *, umode_t, void *), @@ -2038,9 +2039,10 @@ umode_t mode_strip_sgid(struct user_namespace *mnt_userns, * the kernel specify what kind of dirent layout it wants to have. * This allows the kernel to read directories into kernel space or * to have different dirent layouts depending on the binary type. + * Return 'true' to keep going and 'false' if there are no more entries. */ struct dir_context; -typedef int (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64, +typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64, unsigned); struct dir_context { @@ -2132,6 +2134,8 @@ struct file_operations { loff_t len, unsigned int remap_flags); int (*fadvise)(struct file *, loff_t, loff_t, int); int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags); + int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *, + unsigned int poll_flags); } __randomize_layout; struct inode_operations { @@ -2167,7 +2171,7 @@ struct inode_operations { struct file *, unsigned open_flag, umode_t create_mode); int (*tmpfile) (struct user_namespace *, struct inode *, - struct dentry *, umode_t); + struct file *, umode_t); int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int); int (*fileattr_set)(struct user_namespace *mnt_userns, @@ -2371,13 +2375,14 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, * don't have to write inode on fdatasync() when only * e.g. the timestamps have changed. * I_DIRTY_PAGES Inode has dirty pages. Inode itself may be clean. - * I_DIRTY_TIME The inode itself only has dirty timestamps, and the + * I_DIRTY_TIME The inode itself has dirty timestamps, and the * lazytime mount option is enabled. We keep track of this * separately from I_DIRTY_SYNC in order to implement * lazytime. This gets cleared if I_DIRTY_INODE - * (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. I.e. - * either I_DIRTY_TIME *or* I_DIRTY_INODE can be set in - * i_state, but not both. I_DIRTY_PAGES may still be set. + * (I_DIRTY_SYNC and/or I_DIRTY_DATASYNC) gets set. But + * I_DIRTY_TIME can still be set if I_DIRTY_SYNC is already + * in place because writeback might already be in progress + * and we don't want to lose the time update * I_NEW Serves as both a mutex and completion notification. * New inodes set I_NEW. If two processes both create * the same inode, one of them will release its inode and @@ -2779,6 +2784,15 @@ extern int finish_open(struct file *file, struct dentry *dentry, int (*open)(struct inode *, struct file *)); extern int finish_no_open(struct file *file, struct dentry *dentry); +/* Helper for the simple case when original dentry is used */ +static inline int finish_open_simple(struct file *file, int error) +{ + if (error) + return error; + + return finish_open(file, file->f_path.dentry, NULL); +} + /* fs/dcache.c */ extern void __init vfs_caches_init_early(void); extern void __init vfs_caches_init(void); @@ -3540,17 +3554,17 @@ static inline bool dir_emit(struct dir_context *ctx, const char *name, int namelen, u64 ino, unsigned type) { - return ctx->actor(ctx, name, namelen, ctx->pos, ino, type) == 0; + return ctx->actor(ctx, name, namelen, ctx->pos, ino, type); } static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx) { return ctx->actor(ctx, ".", 1, ctx->pos, - file->f_path.dentry->d_inode->i_ino, DT_DIR) == 0; + file->f_path.dentry->d_inode->i_ino, DT_DIR); } static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx) { return ctx->actor(ctx, "..", 2, ctx->pos, - parent_ino(file->f_path.dentry), DT_DIR) == 0; + parent_ino(file->f_path.dentry), DT_DIR); } static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx) { diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 720874e6ee947ec295b618f9709e4bedb414ec08..36e5dd84cf5996b1dba6f47a15e1023c4b64e80f 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -258,7 +258,7 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume, /** * fscache_use_cookie - Request usage of cookie attached to an object - * @object: Object description + * @cookie: The cookie representing the cache object * @will_modify: If cache is expected to be modified locally * * Request usage of the cookie attached to an object. The caller should tell @@ -274,7 +274,7 @@ static inline void fscache_use_cookie(struct fscache_cookie *cookie, /** * fscache_unuse_cookie - Cease usage of cookie attached to an object - * @object: Object description + * @cookie: The cookie representing the cache object * @aux_data: Updated auxiliary data (or NULL) * @object_size: Revised size of the object (or NULL) * diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 7d2f1e0f23b1feae03d9bd8d3702b00c477e5325..cad78b569c7eff712f79698b021efb32c7d5b71e 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -161,24 +161,21 @@ struct fscrypt_operations { int *ino_bits_ret, int *lblk_bits_ret); /* - * Return the number of block devices to which the filesystem may write - * encrypted file contents. + * Return an array of pointers to the block devices to which the + * filesystem may write encrypted file contents, NULL if the filesystem + * only has a single such block device, or an ERR_PTR() on error. + * + * On successful non-NULL return, *num_devs is set to the number of + * devices in the returned array. The caller must free the returned + * array using kfree(). * * If the filesystem can use multiple block devices (other than block * devices that aren't used for encrypted file contents, such as * external journal devices), and wants to support inline encryption, * then it must implement this function. Otherwise it's not needed. */ - int (*get_num_devices)(struct super_block *sb); - - /* - * If ->get_num_devices() returns a value greater than 1, then this - * function is called to get the array of request_queues that the - * filesystem is using -- one per block device. (There may be duplicate - * entries in this array, as block devices can share a request_queue.) - */ - void (*get_devices)(struct super_block *sb, - struct request_queue **devs); + struct block_device **(*get_devices)(struct super_block *sb, + unsigned int *num_devs); }; static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode) @@ -295,8 +292,6 @@ int fscrypt_parse_test_dummy_encryption(const struct fs_parameter *param, struct fscrypt_dummy_policy *dummy_policy); bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1, const struct fscrypt_dummy_policy *p2); -int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg, - struct fscrypt_dummy_policy *dummy_policy); void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep, struct super_block *sb); static inline bool @@ -312,7 +307,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy) } /* keyring.c */ -void fscrypt_sb_free(struct super_block *sb); +void fscrypt_sb_delete(struct super_block *sb); int fscrypt_ioctl_add_key(struct file *filp, void __user *arg); int fscrypt_add_test_dummy_key(struct super_block *sb, const struct fscrypt_dummy_policy *dummy_policy); @@ -353,7 +348,7 @@ u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name); int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags); /* bio.c */ -void fscrypt_decrypt_bio(struct bio *bio); +bool fscrypt_decrypt_bio(struct bio *bio); int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, sector_t pblk, unsigned int len); @@ -526,7 +521,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy) } /* keyring.c */ -static inline void fscrypt_sb_free(struct super_block *sb) +static inline void fscrypt_sb_delete(struct super_block *sb) { } @@ -646,8 +641,9 @@ static inline int fscrypt_d_revalidate(struct dentry *dentry, } /* bio.c */ -static inline void fscrypt_decrypt_bio(struct bio *bio) +static inline bool fscrypt_decrypt_bio(struct bio *bio) { + return true; } static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, @@ -768,7 +764,7 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, bool fscrypt_mergeable_bio_bh(struct bio *bio, const struct buffer_head *next_bh); -bool fscrypt_dio_supported(struct kiocb *iocb, struct iov_iter *iter); +bool fscrypt_dio_supported(struct inode *inode); u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks); @@ -801,11 +797,8 @@ static inline bool fscrypt_mergeable_bio_bh(struct bio *bio, return true; } -static inline bool fscrypt_dio_supported(struct kiocb *iocb, - struct iov_iter *iter) +static inline bool fscrypt_dio_supported(struct inode *inode) { - const struct inode *inode = file_inode(iocb->ki_filp); - return !fscrypt_needs_contents_encryption(inode); } diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index 7af030fa3c366cb18cd1ca794d65d68e95268c9a..40f14e5fed9de902901846a45392a8b9b3b674b2 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -22,6 +22,9 @@ */ #define FS_VERITY_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE +/* Arbitrary limit to bound the kmalloc() size. Can be changed. */ +#define FS_VERITY_MAX_DESCRIPTOR_SIZE 16384 + /* Verity operations for filesystems */ struct fsverity_operations { diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 0b61371e287b9433ff060ef0b74f07387bfc509a..62557d4bffc2b6570ec74c9f224ad877fac25e70 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -1122,47 +1122,6 @@ static inline void unpause_graph_tracing(void) { } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ #ifdef CONFIG_TRACING - -/* flags for current->trace */ -enum { - TSK_TRACE_FL_TRACE_BIT = 0, - TSK_TRACE_FL_GRAPH_BIT = 1, -}; -enum { - TSK_TRACE_FL_TRACE = 1 << TSK_TRACE_FL_TRACE_BIT, - TSK_TRACE_FL_GRAPH = 1 << TSK_TRACE_FL_GRAPH_BIT, -}; - -static inline void set_tsk_trace_trace(struct task_struct *tsk) -{ - set_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace); -} - -static inline void clear_tsk_trace_trace(struct task_struct *tsk) -{ - clear_bit(TSK_TRACE_FL_TRACE_BIT, &tsk->trace); -} - -static inline int test_tsk_trace_trace(struct task_struct *tsk) -{ - return tsk->trace & TSK_TRACE_FL_TRACE; -} - -static inline void set_tsk_trace_graph(struct task_struct *tsk) -{ - set_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace); -} - -static inline void clear_tsk_trace_graph(struct task_struct *tsk) -{ - clear_bit(TSK_TRACE_FL_GRAPH_BIT, &tsk->trace); -} - -static inline int test_tsk_trace_graph(struct task_struct *tsk) -{ - return tsk->trace & TSK_TRACE_FL_GRAPH; -} - enum ftrace_dump_mode; extern enum ftrace_dump_mode ftrace_dump_on_oops; diff --git a/include/linux/gameport.h b/include/linux/gameport.h index 69081d899492e6f79c701d0d3e4aa0d58a6b58c4..8c2f00018e896ddfbb385352c9686cd0df57be67 100644 --- a/include/linux/gameport.h +++ b/include/linux/gameport.h @@ -110,7 +110,7 @@ static inline void gameport_free_port(struct gameport *gameport) static inline void gameport_set_name(struct gameport *gameport, const char *name) { - strlcpy(gameport->name, name, sizeof(gameport->name)); + strscpy(gameport->name, name, sizeof(gameport->name)); } /* diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h index 939b1a8f571b4fa8e02a58156cd51683bd76bf44..4a4b387181ade4e7296e8a1da540a6b8d61818e4 100644 --- a/include/linux/genl_magic_func.h +++ b/include/linux/genl_magic_func.h @@ -294,6 +294,7 @@ static struct genl_family ZZZ_genl_family __ro_after_init = { .ops = ZZZ_genl_ops, .n_ops = ARRAY_SIZE(ZZZ_genl_ops), .mcgrps = ZZZ_genl_mcgrps, + .resv_start_op = 42, /* drbd is currently the only user */ .n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps), .module = THIS_MODULE, }; diff --git a/include/linux/gfp.h b/include/linux/gfp.h index f314be58fa776d3082f4c4de840b4944d1c286b4..ef4aea3b356e76965b3d92fcc59bfcb2744e5c0e 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -18,6 +18,9 @@ static inline int gfp_migratetype(const gfp_t gfp_flags) VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); + BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); + BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> + GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); if (unlikely(page_group_by_mobility_disabled)) return MIGRATE_UNMOVABLE; @@ -33,29 +36,6 @@ static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) return !!(gfp_flags & __GFP_DIRECT_RECLAIM); } -/** - * gfpflags_normal_context - is gfp_flags a normal sleepable context? - * @gfp_flags: gfp_flags to test - * - * Test whether @gfp_flags indicates that the allocation is from the - * %current context and allowed to sleep. - * - * An allocation being allowed to block doesn't mean it owns the %current - * context. When direct reclaim path tries to allocate memory, the - * allocation context is nested inside whatever %current was doing at the - * time of the original allocation. The nested allocation may be allowed - * to block but modifying anything %current owns can corrupt the outer - * context's expectations. - * - * %true result from this function indicates that the allocation context - * can sleep and use anything that's associated with %current. - */ -static inline bool gfpflags_normal_context(const gfp_t gfp_flags) -{ - return (gfp_flags & (__GFP_DIRECT_RECLAIM | __GFP_MEMALLOC)) == - __GFP_DIRECT_RECLAIM; -} - #ifdef CONFIG_HIGHMEM #define OPT_ZONE_HIGHMEM ZONE_HIGHMEM #else diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h index fe0f460d9a3b67c024b79b4b715666782920f78d..36460ced060b0ec1f2a34aa5b167b93d711c8916 100644 --- a/include/linux/gpio/consumer.h +++ b/include/linux/gpio/consumer.h @@ -174,10 +174,6 @@ int desc_to_gpio(const struct gpio_desc *desc); /* Child properties interface */ struct fwnode_handle; -struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, - const char *propname, int index, - enum gpiod_flags dflags, - const char *label); struct gpio_desc *fwnode_gpiod_get_index(struct fwnode_handle *fwnode, const char *con_id, int index, enum gpiod_flags flags, @@ -553,15 +549,6 @@ static inline int desc_to_gpio(const struct gpio_desc *desc) /* Child properties interface */ struct fwnode_handle; -static inline -struct gpio_desc *fwnode_get_named_gpiod(struct fwnode_handle *fwnode, - const char *propname, int index, - enum gpiod_flags dflags, - const char *label) -{ - return ERR_PTR(-ENOSYS); -} - static inline struct gpio_desc *fwnode_gpiod_get_index(struct fwnode_handle *fwnode, const char *con_id, int index, diff --git a/include/linux/hid.h b/include/linux/hid.h index 4363a63b9775ef27e4193cd507b77394fd854b9e..8677ae38599e4390a9278a98d95cb763662189e0 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -314,15 +314,6 @@ struct hid_item { #define HID_BAT_ABSOLUTESTATEOFCHARGE 0x00850065 #define HID_VD_ASUS_CUSTOM_MEDIA_KEYS 0xff310076 -/* - * HID report types --- Ouch! HID spec says 1 2 3! - */ - -#define HID_INPUT_REPORT 0 -#define HID_OUTPUT_REPORT 1 -#define HID_FEATURE_REPORT 2 - -#define HID_REPORT_TYPES 3 /* * HID connect requests @@ -509,7 +500,7 @@ struct hid_report { struct list_head hidinput_list; struct list_head field_entry_list; /* ordered list of input fields */ unsigned int id; /* id of this report */ - unsigned int type; /* report type */ + enum hid_report_type type; /* report type */ unsigned int application; /* application usage for this report */ struct hid_field *field[HID_MAX_FIELDS]; /* fields of the report */ struct hid_field_entry *field_entries; /* allocated memory of input field_entry */ @@ -658,6 +649,8 @@ struct hid_device { /* device report descriptor */ struct list_head debug_list; spinlock_t debug_list_lock; wait_queue_head_t debug_wait; + + unsigned int id; /* system unique id */ }; #define to_hid_device(pdev) \ @@ -924,20 +917,21 @@ extern int hidinput_connect(struct hid_device *hid, unsigned int force); extern void hidinput_disconnect(struct hid_device *); int hid_set_field(struct hid_field *, unsigned, __s32); -int hid_input_report(struct hid_device *, int type, u8 *, u32, int); +int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, + int interrupt); struct hid_field *hidinput_get_led_field(struct hid_device *hid); unsigned int hidinput_count_leds(struct hid_device *hid); __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code); void hid_output_report(struct hid_report *report, __u8 *data); -int __hid_request(struct hid_device *hid, struct hid_report *rep, int reqtype); +int __hid_request(struct hid_device *hid, struct hid_report *rep, enum hid_class_request reqtype); u8 *hid_alloc_report_buf(struct hid_report *report, gfp_t flags); struct hid_device *hid_allocate_device(void); struct hid_report *hid_register_report(struct hid_device *device, - unsigned int type, unsigned int id, + enum hid_report_type type, unsigned int id, unsigned int application); int hid_parse_report(struct hid_device *hid, __u8 *start, unsigned size); struct hid_report *hid_validate_values(struct hid_device *hid, - unsigned int type, unsigned int id, + enum hid_report_type type, unsigned int id, unsigned int field_index, unsigned int report_counts); @@ -1106,10 +1100,11 @@ void hid_hw_stop(struct hid_device *hdev); int __must_check hid_hw_open(struct hid_device *hdev); void hid_hw_close(struct hid_device *hdev); void hid_hw_request(struct hid_device *hdev, - struct hid_report *report, int reqtype); + struct hid_report *report, enum hid_class_request reqtype); int hid_hw_raw_request(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, - size_t len, unsigned char rtype, int reqtype); + size_t len, enum hid_report_type rtype, + enum hid_class_request reqtype); int hid_hw_output_report(struct hid_device *hdev, __u8 *buf, size_t len); /** @@ -1137,7 +1132,7 @@ static inline int hid_hw_power(struct hid_device *hdev, int level) * @reqtype: hid request type */ static inline int hid_hw_idle(struct hid_device *hdev, int report, int idle, - int reqtype) + enum hid_class_request reqtype) { if (hdev->ll_driver->idle) return hdev->ll_driver->idle(hdev, report, idle, reqtype); @@ -1182,8 +1177,8 @@ static inline u32 hid_report_len(struct hid_report *report) return DIV_ROUND_UP(report->size, 8) + (report->id > 0); } -int hid_report_raw_event(struct hid_device *hid, int type, u8 *data, u32 size, - int interrupt); +int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, + int interrupt); /* HID quirks API */ unsigned long hid_lookup_quirk(const struct hid_device *hdev); diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 25679035ca2836b392fb446e347951d22ce96343..e9912da5441b49057a7efaefb3adf351a55ddb47 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -311,6 +312,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from, vfrom = kmap_local_page(from); vto = kmap_local_page(to); copy_user_page(vto, vfrom, vaddr, to); + kmsan_unpoison_memory(page_address(to), PAGE_SIZE); kunmap_local(vto); kunmap_local(vfrom); } @@ -326,6 +328,7 @@ static inline void copy_highpage(struct page *to, struct page *from) vfrom = kmap_local_page(from); vto = kmap_local_page(to); copy_page(vto, vfrom); + kmsan_copy_page_meta(to, from); kunmap_local(vto); kunmap_local(vfrom); } diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 116e8bd68c999cf825c1303c3765590018bcf720..e230c7c46110a5f137128f06b6aff217385f3cfc 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -87,29 +87,6 @@ #define PEH_AXUSER_CFG 0x401001 #define PEH_AXUSER_CFG_ENABLE 0xffffffff -#define QM_AXI_RRESP BIT(0) -#define QM_AXI_BRESP BIT(1) -#define QM_ECC_MBIT BIT(2) -#define QM_ECC_1BIT BIT(3) -#define QM_ACC_GET_TASK_TIMEOUT BIT(4) -#define QM_ACC_DO_TASK_TIMEOUT BIT(5) -#define QM_ACC_WB_NOT_READY_TIMEOUT BIT(6) -#define QM_SQ_CQ_VF_INVALID BIT(7) -#define QM_CQ_VF_INVALID BIT(8) -#define QM_SQ_VF_INVALID BIT(9) -#define QM_DB_TIMEOUT BIT(10) -#define QM_OF_FIFO_OF BIT(11) -#define QM_DB_RANDOM_INVALID BIT(12) -#define QM_MAILBOX_TIMEOUT BIT(13) -#define QM_FLR_TIMEOUT BIT(14) - -#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \ - QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \ - QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID | \ - QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT) -#define QM_BASE_CE QM_ECC_1BIT - -#define QM_Q_DEPTH 1024 #define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 #define QM_SHAPER_CFG 0x100164 @@ -168,6 +145,15 @@ enum qm_vf_state { QM_NOT_READY, }; +enum qm_cap_bits { + QM_SUPPORT_DB_ISOLATION = 0x0, + QM_SUPPORT_FUNC_QOS, + QM_SUPPORT_STOP_QP, + QM_SUPPORT_MB_COMMAND, + QM_SUPPORT_SVA_PREFETCH, + QM_SUPPORT_RPM, +}; + struct dfx_diff_registers { u32 *regs; u32 reg_offset; @@ -232,7 +218,10 @@ struct hisi_qm_err_info { char *acpi_rst; u32 msi_wr_port; u32 ecc_2bits_mask; - u32 dev_ce_mask; + u32 qm_shutdown_mask; + u32 dev_shutdown_mask; + u32 qm_reset_mask; + u32 dev_reset_mask; u32 ce; u32 nfe; u32 fe; @@ -258,6 +247,18 @@ struct hisi_qm_err_ini { void (*err_info_init)(struct hisi_qm *qm); }; +struct hisi_qm_cap_info { + u32 type; + /* Register offset */ + u32 offset; + /* Bit offset in register */ + u32 shift; + u32 mask; + u32 v1_val; + u32 v2_val; + u32 v3_val; +}; + struct hisi_qm_list { struct mutex lock; struct list_head list; @@ -278,6 +279,9 @@ struct hisi_qm { struct pci_dev *pdev; void __iomem *io_base; void __iomem *db_io_base; + + /* Capbility version, 0: not supports */ + u32 cap_ver; u32 sqe_size; u32 qp_base; u32 qp_num; @@ -286,6 +290,8 @@ struct hisi_qm { u32 max_qp_num; u32 vfs_num; u32 db_interval; + u16 eq_depth; + u16 aeq_depth; struct list_head list; struct hisi_qm_list *qm_list; @@ -304,6 +310,8 @@ struct hisi_qm { struct hisi_qm_err_info err_info; struct hisi_qm_err_status err_status; unsigned long misc_ctl; /* driver removing and reset sched */ + /* Device capability bit */ + unsigned long caps; struct rw_semaphore qps_lock; struct idr qp_idr; @@ -326,8 +334,6 @@ struct hisi_qm { bool use_sva; bool is_frozen; - /* doorbell isolation enable */ - bool use_db_isolation; resource_size_t phys_base; resource_size_t db_phys_base; struct uacce_device *uacce; @@ -351,6 +357,8 @@ struct hisi_qp_ops { struct hisi_qp { u32 qp_id; + u16 sq_depth; + u16 cq_depth; u8 alg_type; u8 req_type; @@ -501,6 +509,9 @@ void hisi_qm_pm_init(struct hisi_qm *qm); int hisi_qm_get_dfx_access(struct hisi_qm *qm); void hisi_qm_put_dfx_access(struct hisi_qm *qm); void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset); +u32 hisi_qm_get_hw_info(struct hisi_qm *qm, + const struct hisi_qm_cap_info *info_table, + u32 index, bool is_read); /* Used by VFIO ACC live migration driver */ struct pci_driver *hisi_sec_get_pf_driver(void); diff --git a/include/linux/hp_sdc.h b/include/linux/hp_sdc.h index 6f1dee7e67e068663ab26e288fe1c93c912ecb3c..9be8704e2d38e20e678f62a4c8de365be3497824 100644 --- a/include/linux/hp_sdc.h +++ b/include/linux/hp_sdc.h @@ -180,7 +180,7 @@ switch (val) { \ #define HP_SDC_CMD_SET_IM 0x40 /* 010xxxxx == set irq mask */ -/* The documents provided do not explicitly state that all registers betweem +/* The documents provided do not explicitly state that all registers between * 0x01 and 0x1f inclusive can be read by sending their register index as a * command, but this is implied and appears to be the case. */ diff --git a/include/linux/htcpld.h b/include/linux/htcpld.h index 842fce69ac0637043eeef0f4a35b9fe4702ce3fa..5f8ac9b1d724ce5a4e9615e034800b0eb1ed7064 100644 --- a/include/linux/htcpld.h +++ b/include/linux/htcpld.h @@ -13,8 +13,6 @@ struct htcpld_chip_platform_data { }; struct htcpld_core_platform_data { - unsigned int int_reset_gpio_hi; - unsigned int int_reset_gpio_lo; unsigned int i2c_adapter_id; struct htcpld_chip_platform_data *chip; diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 768e5261fdaed1cf65da6296e99617b750f5940c..a1341fdcf666d04fa05a1c39dc09294dd4f7c906 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -168,9 +168,8 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); } -bool hugepage_vma_check(struct vm_area_struct *vma, - unsigned long vm_flags, - bool smaps, bool in_pf); +bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, + bool smaps, bool in_pf, bool enforce_sysfs); #define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ @@ -219,6 +218,9 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice); +int madvise_collapse(struct vm_area_struct *vma, + struct vm_area_struct **prev, + unsigned long start, unsigned long end); void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next); spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma); @@ -321,8 +323,8 @@ static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, } static inline bool hugepage_vma_check(struct vm_area_struct *vma, - unsigned long vm_flags, - bool smaps, bool in_pf) + unsigned long vm_flags, bool smaps, + bool in_pf, bool enforce_sysfs) { return false; } @@ -362,9 +364,16 @@ static inline void split_huge_pmd_address(struct vm_area_struct *vma, static inline int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice) { - BUG(); - return 0; + return -EINVAL; +} + +static inline int madvise_collapse(struct vm_area_struct *vma, + struct vm_area_struct **prev, + unsigned long start, unsigned long end) +{ + return -EINVAL; } + static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, @@ -435,6 +444,11 @@ static inline int split_folio_to_list(struct folio *folio, return split_huge_page_to_list(&folio->page, list); } +static inline int split_folio(struct folio *folio) +{ + return split_folio_to_list(folio, NULL); +} + /* * archs that select ARCH_WANTS_THP_SWAP but don't support THP_SWP due to * limitations in the implementation like arm64 MTE can override this to diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 3ec981a0d8b3a5b37f5cbd632f618ab139b36f48..8b4f93e848680912894d586d2cd9b98f4f681b1e 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -16,8 +16,9 @@ struct ctl_table; struct user_struct; struct mmu_gather; +struct node; -#ifndef is_hugepd +#ifndef CONFIG_ARCH_HAS_HUGEPD typedef struct { unsigned long pd; } hugepd_t; #define is_hugepd(hugepd) (0) #define __hugepd(x) ((hugepd_t) { (x) }) @@ -114,6 +115,12 @@ struct file_region { #endif }; +struct hugetlb_vma_lock { + struct kref refs; + struct rw_semaphore rw_sema; + struct vm_area_struct *vma; +}; + extern struct resv_map *resv_map_alloc(void); void resv_map_release(struct kref *ref); @@ -126,7 +133,7 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, long min_hpages); void hugepage_put_subpool(struct hugepage_subpool *spool); -void reset_vma_resv_huge_pages(struct vm_area_struct *vma); +void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int hugetlb_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); int hugetlb_overcommit_handler(struct ctl_table *, int, void *, size_t *, @@ -207,13 +214,21 @@ struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, struct page *follow_huge_pd(struct vm_area_struct *vma, unsigned long address, hugepd_t hpd, int flags, int pdshift); -struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, - pmd_t *pmd, int flags); +struct page *follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, + int flags); struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address, pud_t *pud, int flags); struct page *follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags); +void hugetlb_vma_lock_read(struct vm_area_struct *vma); +void hugetlb_vma_unlock_read(struct vm_area_struct *vma); +void hugetlb_vma_lock_write(struct vm_area_struct *vma); +void hugetlb_vma_unlock_write(struct vm_area_struct *vma); +int hugetlb_vma_trylock_write(struct vm_area_struct *vma); +void hugetlb_vma_assert_locked(struct vm_area_struct *vma); +void hugetlb_vma_lock_release(struct kref *kref); + int pmd_huge(pmd_t pmd); int pud_huge(pud_t pud); unsigned long hugetlb_change_protection(struct vm_area_struct *vma, @@ -225,7 +240,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); #else /* !CONFIG_HUGETLB_PAGE */ -static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) +static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma) { } @@ -312,8 +327,8 @@ static inline struct page *follow_huge_pd(struct vm_area_struct *vma, return NULL; } -static inline struct page *follow_huge_pmd(struct mm_struct *mm, - unsigned long address, pmd_t *pmd, int flags) +static inline struct page *follow_huge_pmd_pte(struct vm_area_struct *vma, + unsigned long address, int flags) { return NULL; } @@ -336,6 +351,31 @@ static inline int prepare_hugepage_range(struct file *file, return -EINVAL; } +static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma) +{ +} + +static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma) +{ +} + +static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma) +{ +} + +static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma) +{ +} + +static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma) +{ + return 1; +} + +static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma) +{ +} + static inline int pmd_huge(pmd_t pmd) { return 0; @@ -665,7 +705,7 @@ struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask); struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, unsigned long address); -int huge_add_to_page_cache(struct page *page, struct address_space *mapping, +int hugetlb_add_to_page_cache(struct page *page, struct address_space *mapping, pgoff_t idx); void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct page *page); @@ -935,6 +975,11 @@ static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, } #endif +#ifdef CONFIG_NUMA +void hugetlb_register_node(struct node *node); +void hugetlb_unregister_node(struct node *node); +#endif + #else /* CONFIG_HUGETLB_PAGE */ struct hstate {}; @@ -1109,6 +1154,14 @@ static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { } + +static inline void hugetlb_register_node(struct node *node) +{ +} + +static inline void hugetlb_unregister_node(struct node *node) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, @@ -1123,14 +1176,10 @@ static inline spinlock_t *huge_pte_lock(struct hstate *h, #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA) extern void __init hugetlb_cma_reserve(int order); -extern void __init hugetlb_cma_check(void); #else static inline __init void hugetlb_cma_reserve(int order) { } -static inline __init void hugetlb_cma_check(void) -{ -} #endif bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr); diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index 379344828e781becd6d29e884942a918ffebe696..630cd255d0cfd443c7d2a25f7db79e7790b0ba31 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -90,32 +90,31 @@ hugetlb_cgroup_from_page_rsvd(struct page *page) return __hugetlb_cgroup_from_page(page, true); } -static inline int __set_hugetlb_cgroup(struct page *page, +static inline void __set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg, bool rsvd) { VM_BUG_ON_PAGE(!PageHuge(page), page); if (compound_order(page) < HUGETLB_CGROUP_MIN_ORDER) - return -1; + return; if (rsvd) set_page_private(page + SUBPAGE_INDEX_CGROUP_RSVD, (unsigned long)h_cg); else set_page_private(page + SUBPAGE_INDEX_CGROUP, (unsigned long)h_cg); - return 0; } -static inline int set_hugetlb_cgroup(struct page *page, +static inline void set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) { - return __set_hugetlb_cgroup(page, h_cg, false); + __set_hugetlb_cgroup(page, h_cg, false); } -static inline int set_hugetlb_cgroup_rsvd(struct page *page, +static inline void set_hugetlb_cgroup_rsvd(struct page *page, struct hugetlb_cgroup *h_cg) { - return __set_hugetlb_cgroup(page, h_cg, true); + __set_hugetlb_cgroup(page, h_cg, true); } static inline bool hugetlb_cgroup_disabled(void) @@ -199,16 +198,14 @@ hugetlb_cgroup_from_page_rsvd(struct page *page) return NULL; } -static inline int set_hugetlb_cgroup(struct page *page, +static inline void set_hugetlb_cgroup(struct page *page, struct hugetlb_cgroup *h_cg) { - return 0; } -static inline int set_hugetlb_cgroup_rsvd(struct page *page, +static inline void set_hugetlb_cgroup_rsvd(struct page *page, struct hugetlb_cgroup *h_cg) { - return 0; } static inline bool hugetlb_cgroup_disabled(void) diff --git a/include/linux/hw_breakpoint.h b/include/linux/hw_breakpoint.h index 78dd7035d1e542cac15edb3337951cce8e5d5e2d..f319bd26b0307e1cf9b6c434edf419ef7d33431c 100644 --- a/include/linux/hw_breakpoint.h +++ b/include/linux/hw_breakpoint.h @@ -74,12 +74,12 @@ register_wide_hw_breakpoint(struct perf_event_attr *attr, extern int register_perf_hw_breakpoint(struct perf_event *bp); extern void unregister_hw_breakpoint(struct perf_event *bp); extern void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events); +extern bool hw_breakpoint_is_used(void); extern int dbg_reserve_bp_slot(struct perf_event *bp); extern int dbg_release_bp_slot(struct perf_event *bp); extern int reserve_bp_slot(struct perf_event *bp); extern void release_bp_slot(struct perf_event *bp); -int hw_breakpoint_weight(struct perf_event *bp); int arch_reserve_bp_slot(struct perf_event *bp); void arch_release_bp_slot(struct perf_event *bp); void arch_unregister_hw_breakpoint(struct perf_event *bp); @@ -121,6 +121,8 @@ register_perf_hw_breakpoint(struct perf_event *bp) { return -ENOSYS; } static inline void unregister_hw_breakpoint(struct perf_event *bp) { } static inline void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events) { } +static inline bool hw_breakpoint_is_used(void) { return false; } + static inline int reserve_bp_slot(struct perf_event *bp) {return -ENOSYS; } static inline void release_bp_slot(struct perf_event *bp) { } diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index aa1d4da03538bf4205ee0e6f184e01c75183d62a..77c2885c4c13027f5125ec5f2e3644b28a5adb8c 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -50,6 +50,7 @@ struct hwrng { struct list_head list; struct kref ref; struct completion cleanup_done; + struct completion dying; }; struct device; @@ -61,4 +62,6 @@ extern int devm_hwrng_register(struct device *dev, struct hwrng *rng); extern void hwrng_unregister(struct hwrng *rng); extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng); +extern long hwrng_msleep(struct hwrng *rng, unsigned int msecs); + #endif /* LINUX_HWRANDOM_H_ */ diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 8eab5017bff30b707046cbf8d87709434a8d0b85..f7c49bbdb8a185e166ebb9fdd3482e95427927cb 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -273,7 +273,7 @@ struct i2c_driver { /* Standard driver model interfaces */ int (*probe)(struct i2c_client *client, const struct i2c_device_id *id); - int (*remove)(struct i2c_client *client); + void (*remove)(struct i2c_client *client); /* New driver model interface to aid the seamless removal of the * current probe()'s, more commonly unused than used second parameter. diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 55e6f4ad0ca622af5455c6a2a8d5fde5251c63a6..79690938d9a2d81b23a3116d9ae7ca8f945892f1 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -310,9 +310,11 @@ static inline u16 ieee80211_sn_sub(u16 sn1, u16 sn2) struct ieee80211_hdr { __le16 frame_control; __le16 duration_id; - u8 addr1[ETH_ALEN]; - u8 addr2[ETH_ALEN]; - u8 addr3[ETH_ALEN]; + struct_group(addrs, + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; + u8 addr3[ETH_ALEN]; + ); __le16 seq_ctrl; u8 addr4[ETH_ALEN]; } __packed __aligned(2); @@ -2886,7 +2888,8 @@ ieee80211_he_spr_size(const u8 *he_spr_ie) /* Calculate 802.11be EHT capabilities IE Tx/Rx EHT MCS NSS Support Field size */ static inline u8 ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap, - const struct ieee80211_eht_cap_elem_fixed *eht_cap) + const struct ieee80211_eht_cap_elem_fixed *eht_cap, + bool from_ap) { u8 count = 0; @@ -2907,7 +2910,10 @@ ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap, if (eht_cap->phy_cap_info[0] & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ) count += 3; - return count ? count : 4; + if (count) + return count; + + return from_ap ? 3 : 4; } /* 802.11be EHT PPE Thresholds */ @@ -2943,7 +2949,8 @@ ieee80211_eht_ppe_size(u16 ppe_thres_hdr, const u8 *phy_cap_info) } static inline bool -ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len) +ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len, + bool from_ap) { const struct ieee80211_eht_cap_elem_fixed *elem = (const void *)data; u8 needed = sizeof(struct ieee80211_eht_cap_elem_fixed); @@ -2952,7 +2959,8 @@ ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len) return false; needed += ieee80211_eht_mcs_nss_size((const void *)he_capa, - (const void *)data); + (const void *)data, + from_ap); if (len < needed) return false; diff --git a/include/linux/if_pppol2tp.h b/include/linux/if_pppol2tp.h index 96d40942e5a31efffa3372c00e05bfab023758f2..c87efd333faada7d3c601d8a1c2ea1ecd29f92a5 100644 --- a/include/linux/if_pppol2tp.h +++ b/include/linux/if_pppol2tp.h @@ -4,8 +4,6 @@ * * This file supplies definitions required by the PPP over L2TP driver * (l2tp_ppp.c). All version information wrt this file is located in l2tp_ppp.c - * - * License: */ #ifndef __LINUX_IF_PPPOL2TP_H #define __LINUX_IF_PPPOL2TP_H diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h index 69e813bcb947ef59c1fc23e541a938d5c179d197..ff3beda1312cacd226550336a57ef4a9d1b15bc9 100644 --- a/include/linux/if_pppox.h +++ b/include/linux/if_pppox.h @@ -5,8 +5,6 @@ * * This file supplies definitions required by the PPP over Ethernet driver * (pppox.c). All version information wrt this file is located in pppox.c - * - * License: */ #ifndef __LINUX_IF_PPPOX_H #define __LINUX_IF_PPPOX_H diff --git a/include/linux/igmp.h b/include/linux/igmp.h index 93c262ecbdc92732f902ab0ceb1ceffc63cea1dc..78890143f07901dd19db41172b9f958e2919de7f 100644 --- a/include/linux/igmp.h +++ b/include/linux/igmp.h @@ -118,9 +118,9 @@ extern int ip_mc_source(int add, int omode, struct sock *sk, struct ip_mreq_source *mreqs, int ifindex); extern int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf,int ifindex); extern int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, - struct ip_msfilter __user *optval, int __user *optlen); + sockptr_t optval, sockptr_t optlen); extern int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf, - struct sockaddr_storage __user *p); + sockptr_t optval, size_t offset); extern int ip_mc_sf_allow(struct sock *sk, __be32 local, __be32 rmt, int dif, int sdif); extern void ip_mc_init_dev(struct in_device *); diff --git a/include/linux/iio/consumer.h b/include/linux/iio/consumer.h index 5fa5957586cf4ae040959cce9b31587515ca6b13..6802596b017c9cd57f4b94dc649898bbcdf2db1c 100644 --- a/include/linux/iio/consumer.h +++ b/include/linux/iio/consumer.h @@ -13,7 +13,7 @@ struct iio_dev; struct iio_chan_spec; struct device; -struct device_node; +struct fwnode_handle; /** * struct iio_channel - everything needed for a consumer to use a channel @@ -99,26 +99,20 @@ void iio_channel_release_all(struct iio_channel *chan); struct iio_channel *devm_iio_channel_get_all(struct device *dev); /** - * of_iio_channel_get_by_name() - get description of all that is needed to access channel. - * @np: Pointer to consumer device tree node + * fwnode_iio_channel_get_by_name() - get description of all that is needed to access channel. + * @fwnode: Pointer to consumer Firmware node * @consumer_channel: Unique name to identify the channel on the consumer * side. This typically describes the channels use within * the consumer. E.g. 'battery_voltage' */ -#ifdef CONFIG_OF -struct iio_channel *of_iio_channel_get_by_name(struct device_node *np, const char *name); -#else -static inline struct iio_channel * -of_iio_channel_get_by_name(struct device_node *np, const char *name) -{ - return NULL; -} -#endif +struct iio_channel *fwnode_iio_channel_get_by_name(struct fwnode_handle *fwnode, + const char *name); /** - * devm_of_iio_channel_get_by_name() - Resource managed version of of_iio_channel_get_by_name(). + * devm_fwnode_iio_channel_get_by_name() - Resource managed version of + * fwnode_iio_channel_get_by_name(). * @dev: Pointer to consumer device. - * @np: Pointer to consumer device tree node + * @fwnode: Pointer to consumer Firmware node * @consumer_channel: Unique name to identify the channel on the consumer * side. This typically describes the channels use within * the consumer. E.g. 'battery_voltage' @@ -129,9 +123,9 @@ of_iio_channel_get_by_name(struct device_node *np, const char *name) * The allocated iio channel is automatically released when the device is * unbound. */ -struct iio_channel *devm_of_iio_channel_get_by_name(struct device *dev, - struct device_node *np, - const char *consumer_channel); +struct iio_channel *devm_fwnode_iio_channel_get_by_name(struct device *dev, + struct fwnode_handle *fwnode, + const char *consumer_channel); struct iio_cb_buffer; /** diff --git a/include/linux/iio/iio-opaque.h b/include/linux/iio/iio-opaque.h index 6b3586b3f952f40a74c2d7bf2494c18260e7fb44..d1f8b30a7c8b782c7c7195088fb9a6791502e236 100644 --- a/include/linux/iio/iio-opaque.h +++ b/include/linux/iio/iio-opaque.h @@ -11,6 +11,7 @@ * checked by device drivers but should be considered * read-only as this is a core internal bit * @driver_module: used to make it harder to undercut users + * @mlock_key: lockdep class for iio_dev lock * @info_exist_lock: lock to prevent use during removal * @trig_readonly: mark the current trigger immutable * @event_interface: event chrdevs associated with interrupt lines @@ -42,6 +43,7 @@ struct iio_dev_opaque { int currentmode; int id; struct module *driver_module; + struct lock_class_key mlock_key; struct mutex info_exist_lock; bool trig_readonly; struct iio_event_interface *event_interface; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 5dfbfc991c6967701ac232f2cd4ff7c5c783afb5..f0ec8a5e5a7a9cd7ea1a8ff887460925444e4d39 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -17,7 +17,7 @@ * Currently assumes nano seconds. */ -struct of_phandle_args; +struct fwnode_reference_args; enum iio_shared_by { IIO_SEPARATE, @@ -429,6 +429,8 @@ struct iio_trigger; /* forward declaration */ * provide a custom of_xlate function that reads the * *args* and returns the appropriate index in registered * IIO channels array. + * @fwnode_xlate: fwnode based function pointer to obtain channel specifier index. + * Functionally the same as @of_xlate. * @hwfifo_set_watermark: function pointer to set the current hardware * fifo watermark level; see hwfifo_* entries in * Documentation/ABI/testing/sysfs-bus-iio for details on @@ -508,8 +510,8 @@ struct iio_info { int (*debugfs_reg_access)(struct iio_dev *indio_dev, unsigned reg, unsigned writeval, unsigned *readval); - int (*of_xlate)(struct iio_dev *indio_dev, - const struct of_phandle_args *iiospec); + int (*fwnode_xlate)(struct iio_dev *indio_dev, + const struct fwnode_reference_args *iiospec); int (*hwfifo_set_watermark)(struct iio_dev *indio_dev, unsigned val); int (*hwfifo_flush_to_buffer)(struct iio_dev *indio_dev, unsigned count); diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h index a7aa91f3a8dc7c161093f79e2cce34acacb49751..82faa98c719a00fc638a2830d69760e12e883921 100644 --- a/include/linux/iio/types.h +++ b/include/linux/iio/types.h @@ -17,6 +17,8 @@ enum iio_event_info { IIO_EV_INFO_HIGH_PASS_FILTER_3DB, IIO_EV_INFO_LOW_PASS_FILTER_3DB, IIO_EV_INFO_TIMEOUT, + IIO_EV_INFO_RESET_TIMEOUT, + IIO_EV_INFO_TAP2_MIN_DELAY, }; #define IIO_VAL_INT 1 @@ -63,6 +65,7 @@ enum iio_chan_info_enum { IIO_CHAN_INFO_OVERSAMPLING_RATIO, IIO_CHAN_INFO_THERMOCOUPLE_TYPE, IIO_CHAN_INFO_CALIBAMBIENT, + IIO_CHAN_INFO_ZEROPOINT, }; #endif /* _IIO_TYPES_H_ */ diff --git a/include/linux/init.h b/include/linux/init.h index baf0b29a7010adf7e1779ea935623375e63f019c..077d7f93b402f4e36ad2ed9079f163c0ac7c6d45 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ /* These are for everybody (although not all archs will actually discard it in modules) */ -#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline __nocfi +#define __init __section(".init.text") __cold __latent_entropy __noinitretpoline #define __initdata __section(".init.data") #define __initconst __section(".init.rodata") #define __exitdata __section(".exit.data") @@ -134,7 +134,7 @@ static inline initcall_t initcall_from_entry(initcall_entry_t *entry) extern initcall_entry_t __con_initcall_start[], __con_initcall_end[]; -/* Used for contructor calls. */ +/* Used for constructor calls. */ typedef void (*ctor_fn_t)(void); struct file_system_type; @@ -220,8 +220,8 @@ extern bool initcall_debug; __initcall_name(initstub, __iid, id) #define __define_initcall_stub(__stub, fn) \ - int __init __cficanonical __stub(void); \ - int __init __cficanonical __stub(void) \ + int __init __stub(void); \ + int __init __stub(void) \ { \ return fn(); \ } \ diff --git a/include/linux/input/auo-pixcir-ts.h b/include/linux/input/auo-pixcir-ts.h deleted file mode 100644 index ed0776997a7a50498b57ddaa6fc0e0a2d0334c3a..0000000000000000000000000000000000000000 --- a/include/linux/input/auo-pixcir-ts.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Driver for AUO in-cell touchscreens - * - * Copyright (c) 2011 Heiko Stuebner - * - * based on auo_touch.h from Dell Streak kernel - * - * Copyright (c) 2008 QUALCOMM Incorporated. - * Copyright (c) 2008 QUALCOMM USA, INC. - */ - -#ifndef __AUO_PIXCIR_TS_H__ -#define __AUO_PIXCIR_TS_H__ - -/* - * Interrupt modes: - * periodical: interrupt is asserted periodicaly - * compare coordinates: interrupt is asserted when coordinates change - * indicate touch: interrupt is asserted during touch - */ -#define AUO_PIXCIR_INT_PERIODICAL 0x00 -#define AUO_PIXCIR_INT_COMP_COORD 0x01 -#define AUO_PIXCIR_INT_TOUCH_IND 0x02 - -/* - * @gpio_int interrupt gpio - * @int_setting one of AUO_PIXCIR_INT_* - * @init_hw hardwarespecific init - * @exit_hw hardwarespecific shutdown - * @x_max x-resolution - * @y_max y-resolution - */ -struct auo_pixcir_ts_platdata { - int gpio_int; - int gpio_rst; - - int int_setting; - - unsigned int x_max; - unsigned int y_max; -}; - -#endif diff --git a/include/linux/instrumented.h b/include/linux/instrumented.h index 42faebbaa202a978e2e004ce2d402f26f089ec2b..501fa84867494787e0f7a20e077c1252c5e932d8 100644 --- a/include/linux/instrumented.h +++ b/include/linux/instrumented.h @@ -2,7 +2,7 @@ /* * This header provides generic wrappers for memory access instrumentation that - * the compiler cannot emit for: KASAN, KCSAN. + * the compiler cannot emit for: KASAN, KCSAN, KMSAN. */ #ifndef _LINUX_INSTRUMENTED_H #define _LINUX_INSTRUMENTED_H @@ -10,6 +10,7 @@ #include #include #include +#include #include /** @@ -117,10 +118,11 @@ instrument_copy_to_user(void __user *to, const void *from, unsigned long n) { kasan_check_read(from, n); kcsan_check_read(from, n); + kmsan_copy_to_user(to, from, n, 0); } /** - * instrument_copy_from_user - instrument writes of copy_from_user + * instrument_copy_from_user_before - add instrumentation before copy_from_user * * Instrument writes to kernel memory, that are due to copy_from_user (and * variants). The instrumentation should be inserted before the accesses. @@ -130,10 +132,61 @@ instrument_copy_to_user(void __user *to, const void *from, unsigned long n) * @n number of bytes to copy */ static __always_inline void -instrument_copy_from_user(const void *to, const void __user *from, unsigned long n) +instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n) { kasan_check_write(to, n); kcsan_check_write(to, n); } +/** + * instrument_copy_from_user_after - add instrumentation after copy_from_user + * + * Instrument writes to kernel memory, that are due to copy_from_user (and + * variants). The instrumentation should be inserted after the accesses. + * + * @to destination address + * @from source address + * @n number of bytes to copy + * @left number of bytes not copied (as returned by copy_from_user) + */ +static __always_inline void +instrument_copy_from_user_after(const void *to, const void __user *from, + unsigned long n, unsigned long left) +{ + kmsan_unpoison_memory(to, n - left); +} + +/** + * instrument_get_user() - add instrumentation to get_user()-like macros + * + * get_user() and friends are fragile, so it may depend on the implementation + * whether the instrumentation happens before or after the data is copied from + * the userspace. + * + * @to destination variable, may not be address-taken + */ +#define instrument_get_user(to) \ +({ \ + u64 __tmp = (u64)(to); \ + kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \ + to = __tmp; \ +}) + + +/** + * instrument_put_user() - add instrumentation to put_user()-like macros + * + * put_user() and friends are fragile, so it may depend on the implementation + * whether the instrumentation happens before or after the data is copied from + * the userspace. + * + * @from source address + * @ptr userspace pointer to copy to + * @size number of bytes to copy + */ +#define instrument_put_user(from, ptr, size) \ +({ \ + kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \ +}) + #endif /* _LINUX_INSTRUMENTED_H */ diff --git a/include/linux/interconnect-provider.h b/include/linux/interconnect-provider.h index 6bd01f7159c6c6719d49a92537504d13fef39707..cd5c5a27557f5605b78eed7531b58124b114fbd6 100644 --- a/include/linux/interconnect-provider.h +++ b/include/linux/interconnect-provider.h @@ -123,7 +123,7 @@ void icc_node_add(struct icc_node *node, struct icc_provider *provider); void icc_node_del(struct icc_node *node); int icc_nodes_remove(struct icc_provider *provider); int icc_provider_add(struct icc_provider *provider); -int icc_provider_del(struct icc_provider *provider); +void icc_provider_del(struct icc_provider *provider); struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec); void icc_sync_state(struct device *dev); @@ -172,9 +172,8 @@ static inline int icc_provider_add(struct icc_provider *provider) return -ENOTSUPP; } -static inline int icc_provider_del(struct icc_provider *provider) +static inline void icc_provider_del(struct icc_provider *provider) { - return -ENOTSUPP; } static inline struct icc_node_data *of_icc_get_from_provider(struct of_phandle_args *spec) diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index ca98aeadcc80485d6e005d66ae925c773e792fa1..1f068dfdb140c8674348b58548961bb3ac91a3bf 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -16,7 +16,9 @@ enum io_pgtable_fmt { ARM_V7S, ARM_MALI_LPAE, AMD_IOMMU_V1, + AMD_IOMMU_V2, APPLE_DART, + APPLE_DART2, IO_PGTABLE_NUM_FMTS, }; @@ -260,6 +262,7 @@ extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns; extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns; #endif /* __IO_PGTABLE_H */ diff --git a/include/linux/io.h b/include/linux/io.h index 5fc800390fe4259bc83b07c1e57587c1c122f83f..308f4f0cfb93284143c0b9158fc5e68da1e6b3e7 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -59,8 +59,6 @@ void __iomem *devm_ioremap_uc(struct device *dev, resource_size_t offset, resource_size_t size); void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset, resource_size_t size); -void __iomem *devm_ioremap_np(struct device *dev, resource_size_t offset, - resource_size_t size); void devm_iounmap(struct device *dev, void __iomem *addr); int check_signature(const volatile void __iomem *io_addr, const unsigned char *signature, int length); diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index 4a2f6cc5a4927fdc0280b2b5b86d47938b8bd768..43bc8a2edccf5aa347142fb05029a49241d2b97c 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -4,6 +4,7 @@ #include #include +#include enum io_uring_cmd_flags { IO_URING_F_COMPLETE_DEFER = 1, @@ -20,14 +21,20 @@ enum io_uring_cmd_flags { struct io_uring_cmd { struct file *file; const void *cmd; - /* callback to defer completions to task context */ - void (*task_work_cb)(struct io_uring_cmd *cmd); + union { + /* callback to defer completions to task context */ + void (*task_work_cb)(struct io_uring_cmd *cmd); + /* used for polled completion */ + void *cookie; + }; u32 cmd_op; - u32 pad; + u32 flags; u8 pdu[32]; /* available inline for free use */ }; #if defined(CONFIG_IO_URING) +int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, + struct iov_iter *iter, void *ioucmd); void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2); void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, void (*task_work_cb)(struct io_uring_cmd *)); @@ -55,6 +62,11 @@ static inline void io_uring_free(struct task_struct *tsk) __io_uring_free(tsk); } #else +static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, + struct iov_iter *iter, void *ioucmd) +{ + return -EOPNOTSUPP; +} static inline void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t ret2) { diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 677a25d44d7f3b0574ed926644f20e18b79ec001..f5b687a787a34de90fcdb0219c26ba20adb0a31e 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -34,9 +34,6 @@ struct io_file_table { unsigned int alloc_hint; }; -struct io_notif; -struct io_notif_slot; - struct io_hash_bucket { spinlock_t lock; struct hlist_head list; @@ -184,6 +181,8 @@ struct io_ev_fd { struct eventfd_ctx *cq_ev_fd; unsigned int eventfd_async: 1; struct rcu_head rcu; + atomic_t refs; + atomic_t ops; }; struct io_alloc_cache { @@ -240,8 +239,6 @@ struct io_ring_ctx { unsigned nr_user_files; unsigned nr_user_bufs; struct io_mapped_ubuf **user_bufs; - struct io_notif_slot *notif_slots; - unsigned nr_notif_slots; struct io_submit_state submit_state; @@ -301,6 +298,8 @@ struct io_ring_ctx { struct io_hash_table cancel_table; bool poll_multi_queue; + struct llist_head work_llist; + struct list_head io_buffers_comp; } ____cacheline_aligned_in_smp; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index ea30f00dc1459512878743465cbea9c6afd684e7..a325532aeab58c88c92c315cabfffcafe3d35b52 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -212,7 +212,7 @@ struct iommu_iotlb_gather { * @of_xlate: add OF master IDs to iommu grouping * @is_attach_deferred: Check if domain attach should be deferred from iommu * driver init to device driver init (default no) - * @dev_has/enable/disable_feat: per device entries to check/enable/disable + * @dev_enable/disable_feat: per device entries to enable/disable * iommu specific features. * @sva_bind: Bind process address space to device * @sva_unbind: Unbind process address space from device @@ -227,7 +227,7 @@ struct iommu_iotlb_gather { * @owner: Driver module providing these ops */ struct iommu_ops { - bool (*capable)(enum iommu_cap); + bool (*capable)(struct device *dev, enum iommu_cap); /* Domain allocation and freeing by the iommu driver */ struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); @@ -416,11 +416,9 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev) return dev->iommu->iommu_dev->ops; } -extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); extern int bus_iommu_probe(struct bus_type *bus); extern bool iommu_present(struct bus_type *bus); extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); -extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); extern struct iommu_group *iommu_group_get_by_id(int id); extern void iommu_domain_free(struct iommu_domain *domain); @@ -697,11 +695,6 @@ static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap) return false; } -static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) -{ - return false; -} - static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) { return NULL; @@ -1070,4 +1063,40 @@ void iommu_debugfs_setup(void); static inline void iommu_debugfs_setup(void) {} #endif +#ifdef CONFIG_IOMMU_DMA +#include + +/* Setup call for arch DMA mapping code */ +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); + +int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); + +int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); +void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg); + +#else /* CONFIG_IOMMU_DMA */ + +struct msi_desc; +struct msi_msg; + +static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) +{ +} + +static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) +{ + return -ENODEV; +} + +static inline int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) +{ + return 0; +} + +static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ +} + +#endif /* CONFIG_IOMMU_DMA */ + #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 616b683563a97046cb862ec11238572783579702..27642ca15d932f22823ef7b0a003a2d1ca1333f8 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -79,7 +79,8 @@ struct resource { #define IORESOURCE_IRQ_HIGHLEVEL (1<<2) #define IORESOURCE_IRQ_LOWLEVEL (1<<3) #define IORESOURCE_IRQ_SHAREABLE (1<<4) -#define IORESOURCE_IRQ_OPTIONAL (1<<5) +#define IORESOURCE_IRQ_OPTIONAL (1<<5) +#define IORESOURCE_IRQ_WAKECAPABLE (1<<6) /* PnP DMA specific bits (IORESOURCE_BITS) */ #define IORESOURCE_DMA_TYPE_MASK (3<<0) @@ -172,6 +173,11 @@ enum { #define DEFINE_RES_MEM(_start, _size) \ DEFINE_RES_MEM_NAMED((_start), (_size), NULL) +#define DEFINE_RES_REG_NAMED(_start, _size, _name) \ + DEFINE_RES_NAMED((_start), (_size), (_name), IORESOURCE_REG) +#define DEFINE_RES_REG(_start, _size) \ + DEFINE_RES_REG_NAMED((_start), (_size), NULL) + #define DEFINE_RES_IRQ_NAMED(_irq, _name) \ DEFINE_RES_NAMED((_irq), 1, (_name), IORESOURCE_IRQ) #define DEFINE_RES_IRQ(_irq) \ diff --git a/include/linux/iova.h b/include/linux/iova.h index c6ba6d95d79c2d9ae358dabb2ad15c0620a9c9ee..83c00fac2acb1d952edb3d82963b500f4d803047 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -75,7 +75,7 @@ static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova) return iova >> iova_shift(iovad); } -#if IS_ENABLED(CONFIG_IOMMU_IOVA) +#if IS_REACHABLE(CONFIG_IOMMU_IOVA) int iova_cache_get(void); void iova_cache_put(void); diff --git a/include/linux/iova_bitmap.h b/include/linux/iova_bitmap.h new file mode 100644 index 0000000000000000000000000000000000000000..c006cf0a25f3daac2ccc39c67c9a3193245a4077 --- /dev/null +++ b/include/linux/iova_bitmap.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. + * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ +#ifndef _IOVA_BITMAP_H_ +#define _IOVA_BITMAP_H_ + +#include + +struct iova_bitmap; + +typedef int (*iova_bitmap_fn_t)(struct iova_bitmap *bitmap, + unsigned long iova, size_t length, + void *opaque); + +struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, + unsigned long page_size, + u64 __user *data); +void iova_bitmap_free(struct iova_bitmap *bitmap); +int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, + iova_bitmap_fn_t fn); +void iova_bitmap_set(struct iova_bitmap *bitmap, + unsigned long iova, size_t length); + +#endif diff --git a/include/linux/ipc_namespace.h b/include/linux/ipc_namespace.h index e3e8c8662b490c4de8a9b2e8771f5f48ee1f765d..e8240cf2611ad63f427d8fe90b3fe8cbcb1fb6b0 100644 --- a/include/linux/ipc_namespace.h +++ b/include/linux/ipc_namespace.h @@ -11,6 +11,7 @@ #include #include #include +#include struct user_namespace; @@ -36,8 +37,8 @@ struct ipc_namespace { unsigned int msg_ctlmax; unsigned int msg_ctlmnb; unsigned int msg_ctlmni; - atomic_t msg_bytes; - atomic_t msg_hdrs; + struct percpu_counter percpu_msg_bytes; + struct percpu_counter percpu_msg_hdrs; size_t shm_ctlmax; size_t shm_ctlall; diff --git a/include/linux/irqchip.h b/include/linux/irqchip.h index 3a091d0710ae1fbd8f0ae6e0ab7feac3e5ee9dba..d5e6024cb2a8c11763e191aa1457d02d7cc554e6 100644 --- a/include/linux/irqchip.h +++ b/include/linux/irqchip.h @@ -44,7 +44,8 @@ static const struct of_device_id drv_name##_irqchip_match_table[] = { #define IRQCHIP_MATCH(compat, fn) { .compatible = compat, \ .data = typecheck_irq_init_cb(fn), }, -#define IRQCHIP_PLATFORM_DRIVER_END(drv_name) \ + +#define IRQCHIP_PLATFORM_DRIVER_END(drv_name, ...) \ {}, \ }; \ MODULE_DEVICE_TABLE(of, drv_name##_irqchip_match_table); \ @@ -56,6 +57,7 @@ static struct platform_driver drv_name##_driver = { \ .owner = THIS_MODULE, \ .of_match_table = drv_name##_irqchip_match_table, \ .suppress_bind_attrs = true, \ + __VA_ARGS__ \ }, \ }; \ builtin_platform_driver(drv_name##_driver) diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index 1cd4e36890fbf680859c3daf2d8a9f9295c12b8d..844a8e30e6de5dd0b46d0b45038bd3315f5c0ec9 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -169,6 +169,7 @@ int generic_handle_irq_safe(unsigned int irq); * conversion failed. */ int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq); +int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq); int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq); #endif diff --git a/include/linux/isa.h b/include/linux/isa.h index e30963190968fd0313d8affaff64ecc02d0d9a70..4fbbf5e36e08e07dafd2d7987c98b4a6c3ff01f8 100644 --- a/include/linux/isa.h +++ b/include/linux/isa.h @@ -38,6 +38,32 @@ static inline void isa_unregister_driver(struct isa_driver *d) } #endif +#define module_isa_driver_init(__isa_driver, __num_isa_dev) \ +static int __init __isa_driver##_init(void) \ +{ \ + return isa_register_driver(&(__isa_driver), __num_isa_dev); \ +} \ +module_init(__isa_driver##_init) + +#define module_isa_driver_with_irq_init(__isa_driver, __num_isa_dev, __num_irq) \ +static int __init __isa_driver##_init(void) \ +{ \ + if (__num_irq != __num_isa_dev) { \ + pr_err("%s: Number of irq (%u) does not match number of base (%u)\n", \ + __isa_driver.driver.name, __num_irq, __num_isa_dev); \ + return -EINVAL; \ + } \ + return isa_register_driver(&(__isa_driver), __num_isa_dev); \ +} \ +module_init(__isa_driver##_init) + +#define module_isa_driver_exit(__isa_driver) \ +static void __exit __isa_driver##_exit(void) \ +{ \ + isa_unregister_driver(&(__isa_driver)); \ +} \ +module_exit(__isa_driver##_exit) + /** * module_isa_driver() - Helper macro for registering a ISA driver * @__isa_driver: isa_driver struct @@ -48,16 +74,22 @@ static inline void isa_unregister_driver(struct isa_driver *d) * use this macro once, and calling it replaces module_init and module_exit. */ #define module_isa_driver(__isa_driver, __num_isa_dev) \ -static int __init __isa_driver##_init(void) \ -{ \ - return isa_register_driver(&(__isa_driver), __num_isa_dev); \ -} \ -module_init(__isa_driver##_init); \ -static void __exit __isa_driver##_exit(void) \ -{ \ - isa_unregister_driver(&(__isa_driver)); \ -} \ -module_exit(__isa_driver##_exit); +module_isa_driver_init(__isa_driver, __num_isa_dev); \ +module_isa_driver_exit(__isa_driver) + +/** + * module_isa_driver_with_irq() - Helper macro for registering an ISA driver with irq + * @__isa_driver: isa_driver struct + * @__num_isa_dev: number of devices to register + * @__num_irq: number of IRQ to register + * + * Helper macro for ISA drivers with irq that do not do anything special in + * module init/exit. Each module may only use this macro once, and calling it + * replaces module_init and module_exit. + */ +#define module_isa_driver_with_irq(__isa_driver, __num_isa_dev, __num_irq) \ +module_isa_driver_with_irq_init(__isa_driver, __num_isa_dev, __num_irq); \ +module_isa_driver_exit(__isa_driver) /** * max_num_isa_dev() - Maximum possible number registered of an ISA device diff --git a/include/linux/iversion.h b/include/linux/iversion.h index 3bfebde5a1a6d23587acd695d39f87b6c6d4515c..e27bd4f55d840e656a0aa81bd0240a721f147ad8 100644 --- a/include/linux/iversion.h +++ b/include/linux/iversion.h @@ -123,17 +123,12 @@ inode_peek_iversion_raw(const struct inode *inode) static inline void inode_set_max_iversion_raw(struct inode *inode, u64 val) { - u64 cur, old; + u64 cur = inode_peek_iversion_raw(inode); - cur = inode_peek_iversion_raw(inode); - for (;;) { + do { if (cur > val) break; - old = atomic64_cmpxchg(&inode->i_version, cur, val); - if (likely(old == cur)) - break; - cur = old; - } + } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, val)); } /** @@ -177,56 +172,7 @@ inode_set_iversion_queried(struct inode *inode, u64 val) I_VERSION_QUERIED); } -/** - * inode_maybe_inc_iversion - increments i_version - * @inode: inode with the i_version that should be updated - * @force: increment the counter even if it's not necessary? - * - * Every time the inode is modified, the i_version field must be seen to have - * changed by any observer. - * - * If "force" is set or the QUERIED flag is set, then ensure that we increment - * the value, and clear the queried flag. - * - * In the common case where neither is set, then we can return "false" without - * updating i_version. - * - * If this function returns false, and no other metadata has changed, then we - * can avoid logging the metadata. - */ -static inline bool -inode_maybe_inc_iversion(struct inode *inode, bool force) -{ - u64 cur, old, new; - - /* - * The i_version field is not strictly ordered with any other inode - * information, but the legacy inode_inc_iversion code used a spinlock - * to serialize increments. - * - * Here, we add full memory barriers to ensure that any de-facto - * ordering with other info is preserved. - * - * This barrier pairs with the barrier in inode_query_iversion() - */ - smp_mb(); - cur = inode_peek_iversion_raw(inode); - for (;;) { - /* If flag is clear then we needn't do anything */ - if (!force && !(cur & I_VERSION_QUERIED)) - return false; - - /* Since lowest bit is flag, add 2 to avoid it */ - new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT; - - old = atomic64_cmpxchg(&inode->i_version, cur, new); - if (likely(old == cur)) - break; - cur = old; - } - return true; -} - +bool inode_maybe_inc_iversion(struct inode *inode, bool force); /** * inode_inc_iversion - forcibly increment i_version @@ -304,10 +250,10 @@ inode_peek_iversion(const struct inode *inode) static inline u64 inode_query_iversion(struct inode *inode) { - u64 cur, old, new; + u64 cur, new; cur = inode_peek_iversion_raw(inode); - for (;;) { + do { /* If flag is already set, then no need to swap */ if (cur & I_VERSION_QUERIED) { /* @@ -320,11 +266,7 @@ inode_query_iversion(struct inode *inode) } new = cur | I_VERSION_QUERIED; - old = atomic64_cmpxchg(&inode->i_version, cur, new); - if (likely(old == cur)) - break; - cur = old; - } + } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new)); return cur >> I_VERSION_QUERIED_SHIFT; } diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index ad39636e0c3f1227f85976551a26814e2343e83a..649faac31ddb162701a8b75059775f74aa45dde0 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -15,7 +15,7 @@ #include -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 #define KSYM_SYMBOL_LEN (sizeof("%s+%#lx/%#lx [%s %s]") + \ (KSYM_NAME_LEN - 1) + \ 2*(BITS_PER_LONG*3/10) + (MODULE_NAME_LEN - 1) + \ diff --git a/include/linux/kasan.h b/include/linux/kasan.h index b092277bf48d610aa5acd4de6ae5767529140906..d811b3d7d2a15e5b87b79b0a4e89a5d5794b31fc 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -98,19 +98,13 @@ static inline bool kasan_has_integrated_init(void) #ifdef CONFIG_KASAN struct kasan_cache { +#ifdef CONFIG_KASAN_GENERIC int alloc_meta_offset; int free_meta_offset; +#endif bool is_kmalloc; }; -slab_flags_t __kasan_never_merge(void); -static __always_inline slab_flags_t kasan_never_merge(void) -{ - if (kasan_enabled()) - return __kasan_never_merge(); - return 0; -} - void __kasan_unpoison_range(const void *addr, size_t size); static __always_inline void kasan_unpoison_range(const void *addr, size_t size) { @@ -134,15 +128,6 @@ static __always_inline void kasan_unpoison_pages(struct page *page, __kasan_unpoison_pages(page, order, init); } -void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size, - slab_flags_t *flags); -static __always_inline void kasan_cache_create(struct kmem_cache *cache, - unsigned int *size, slab_flags_t *flags) -{ - if (kasan_enabled()) - __kasan_cache_create(cache, size, flags); -} - void __kasan_cache_create_kmalloc(struct kmem_cache *cache); static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) { @@ -150,14 +135,6 @@ static __always_inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) __kasan_cache_create_kmalloc(cache); } -size_t __kasan_metadata_size(struct kmem_cache *cache); -static __always_inline size_t kasan_metadata_size(struct kmem_cache *cache) -{ - if (kasan_enabled()) - return __kasan_metadata_size(cache); - return 0; -} - void __kasan_poison_slab(struct slab *slab); static __always_inline void kasan_poison_slab(struct slab *slab) { @@ -269,20 +246,12 @@ static __always_inline bool kasan_check_byte(const void *addr) #else /* CONFIG_KASAN */ -static inline slab_flags_t kasan_never_merge(void) -{ - return 0; -} static inline void kasan_unpoison_range(const void *address, size_t size) {} static inline void kasan_poison_pages(struct page *page, unsigned int order, bool init) {} static inline void kasan_unpoison_pages(struct page *page, unsigned int order, bool init) {} -static inline void kasan_cache_create(struct kmem_cache *cache, - unsigned int *size, - slab_flags_t *flags) {} static inline void kasan_cache_create_kmalloc(struct kmem_cache *cache) {} -static inline size_t kasan_metadata_size(struct kmem_cache *cache) { return 0; } static inline void kasan_poison_slab(struct slab *slab) {} static inline void kasan_unpoison_object_data(struct kmem_cache *cache, void *object) {} @@ -333,6 +302,11 @@ static inline void kasan_unpoison_task_stack(struct task_struct *task) {} #ifdef CONFIG_KASAN_GENERIC +size_t kasan_metadata_size(struct kmem_cache *cache); +slab_flags_t kasan_never_merge(void); +void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, + slab_flags_t *flags); + void kasan_cache_shrink(struct kmem_cache *cache); void kasan_cache_shutdown(struct kmem_cache *cache); void kasan_record_aux_stack(void *ptr); @@ -340,6 +314,21 @@ void kasan_record_aux_stack_noalloc(void *ptr); #else /* CONFIG_KASAN_GENERIC */ +/* Tag-based KASAN modes do not use per-object metadata. */ +static inline size_t kasan_metadata_size(struct kmem_cache *cache) +{ + return 0; +} +/* And thus nothing prevents cache merging. */ +static inline slab_flags_t kasan_never_merge(void) +{ + return 0; +} +/* And no cache-related metadata initialization is required. */ +static inline void kasan_cache_create(struct kmem_cache *cache, + unsigned int *size, + slab_flags_t *flags) {} + static inline void kasan_cache_shrink(struct kmem_cache *cache) {} static inline void kasan_cache_shutdown(struct kmem_cache *cache) {} static inline void kasan_record_aux_stack(void *ptr) {} diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 367044d7708c6f32357d82853b48f87499bd7cd1..73f5c120def88bd8d2136c64a2fbbe4d0494b54a 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -108,10 +108,12 @@ enum kernfs_node_flag { KERNFS_HAS_SEQ_SHOW = 0x0040, KERNFS_HAS_MMAP = 0x0080, KERNFS_LOCKDEP = 0x0100, + KERNFS_HIDDEN = 0x0200, KERNFS_SUICIDAL = 0x0400, KERNFS_SUICIDED = 0x0800, KERNFS_EMPTY_DIR = 0x1000, KERNFS_HAS_RELEASE = 0x2000, + KERNFS_REMOVING = 0x4000, }; /* @flags for kernfs_create_root() */ @@ -429,6 +431,7 @@ struct kernfs_node *kernfs_create_link(struct kernfs_node *parent, const char *name, struct kernfs_node *target); void kernfs_activate(struct kernfs_node *kn); +void kernfs_show(struct kernfs_node *kn, bool show); void kernfs_remove(struct kernfs_node *kn); void kernfs_break_active_protection(struct kernfs_node *kn); void kernfs_unbreak_active_protection(struct kernfs_node *kn); diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 13e6c4b58f07d78d30888c36b362e9ff1468c4d4..41a686996aaa32291d0cb4ab22bc556fa223a3b9 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -427,7 +427,7 @@ extern int kexec_load_disabled; extern bool kexec_in_progress; int crash_shrink_memory(unsigned long new_size); -size_t crash_get_memory_size(void); +ssize_t crash_get_memory_size(void); #ifndef arch_kexec_protect_crashkres /* diff --git a/include/linux/key.h b/include/linux/key.h index 7febc4881363ce87c8bfda9e07dc4df877d1c51a..d27477faf00d7d837bdf1dd39d7ff58ec44b3535 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -88,6 +88,12 @@ enum key_need_perm { KEY_DEFER_PERM_CHECK, /* Special: permission check is deferred */ }; +enum key_lookup_flag { + KEY_LOOKUP_CREATE = 0x01, + KEY_LOOKUP_PARTIAL = 0x02, + KEY_LOOKUP_ALL = (KEY_LOOKUP_CREATE | KEY_LOOKUP_PARTIAL), +}; + struct seq_file; struct user_struct; struct signal_struct; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index 384f034ae947ff427eae88c834de1b096c9f3f11..70162d707caf0c3713d03a801a8c5c54b39c64ab 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -16,11 +16,13 @@ extern void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags); extern void khugepaged_min_free_kbytes_update(void); #ifdef CONFIG_SHMEM -extern void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr); +extern int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, + bool install_pmd); #else -static inline void collapse_pte_mapped_thp(struct mm_struct *mm, - unsigned long addr) +static inline int collapse_pte_mapped_thp(struct mm_struct *mm, + unsigned long addr, bool install_pmd) { + return 0; } #endif @@ -46,9 +48,10 @@ static inline void khugepaged_enter_vma(struct vm_area_struct *vma, unsigned long vm_flags) { } -static inline void collapse_pte_mapped_thp(struct mm_struct *mm, - unsigned long addr) +static inline int collapse_pte_mapped_thp(struct mm_struct *mm, + unsigned long addr, bool install_pmd) { + return 0; } static inline void khugepaged_min_free_kbytes_update(void) diff --git a/include/linux/kmsan-checks.h b/include/linux/kmsan-checks.h new file mode 100644 index 0000000000000000000000000000000000000000..c4cae333deec5f24fb231b27c948560a4f0db69c --- /dev/null +++ b/include/linux/kmsan-checks.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KMSAN checks to be used for one-off annotations in subsystems. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#ifndef _LINUX_KMSAN_CHECKS_H +#define _LINUX_KMSAN_CHECKS_H + +#include + +#ifdef CONFIG_KMSAN + +/** + * kmsan_poison_memory() - Mark the memory range as uninitialized. + * @address: address to start with. + * @size: size of buffer to poison. + * @flags: GFP flags for allocations done by this function. + * + * Until other data is written to this range, KMSAN will treat it as + * uninitialized. Error reports for this memory will reference the call site of + * kmsan_poison_memory() as origin. + */ +void kmsan_poison_memory(const void *address, size_t size, gfp_t flags); + +/** + * kmsan_unpoison_memory() - Mark the memory range as initialized. + * @address: address to start with. + * @size: size of buffer to unpoison. + * + * Until other data is written to this range, KMSAN will treat it as + * initialized. + */ +void kmsan_unpoison_memory(const void *address, size_t size); + +/** + * kmsan_check_memory() - Check the memory range for being initialized. + * @address: address to start with. + * @size: size of buffer to check. + * + * If any piece of the given range is marked as uninitialized, KMSAN will report + * an error. + */ +void kmsan_check_memory(const void *address, size_t size); + +/** + * kmsan_copy_to_user() - Notify KMSAN about a data transfer to userspace. + * @to: destination address in the userspace. + * @from: source address in the kernel. + * @to_copy: number of bytes to copy. + * @left: number of bytes not copied. + * + * If this is a real userspace data transfer, KMSAN checks the bytes that were + * actually copied to ensure there was no information leak. If @to belongs to + * the kernel space (which is possible for compat syscalls), KMSAN just copies + * the metadata. + */ +void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy, + size_t left); + +#else + +static inline void kmsan_poison_memory(const void *address, size_t size, + gfp_t flags) +{ +} +static inline void kmsan_unpoison_memory(const void *address, size_t size) +{ +} +static inline void kmsan_check_memory(const void *address, size_t size) +{ +} +static inline void kmsan_copy_to_user(void __user *to, const void *from, + size_t to_copy, size_t left) +{ +} + +#endif + +#endif /* _LINUX_KMSAN_CHECKS_H */ diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h new file mode 100644 index 0000000000000000000000000000000000000000..e38ae3c346184630f1691d6431ffbc3675bd8194 --- /dev/null +++ b/include/linux/kmsan.h @@ -0,0 +1,330 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KMSAN API for subsystems. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ +#ifndef _LINUX_KMSAN_H +#define _LINUX_KMSAN_H + +#include +#include +#include +#include + +struct page; +struct kmem_cache; +struct task_struct; +struct scatterlist; +struct urb; + +#ifdef CONFIG_KMSAN + +/** + * kmsan_task_create() - Initialize KMSAN state for the task. + * @task: task to initialize. + */ +void kmsan_task_create(struct task_struct *task); + +/** + * kmsan_task_exit() - Notify KMSAN that a task has exited. + * @task: task about to finish. + */ +void kmsan_task_exit(struct task_struct *task); + +/** + * kmsan_init_shadow() - Initialize KMSAN shadow at boot time. + * + * Allocate and initialize KMSAN metadata for early allocations. + */ +void __init kmsan_init_shadow(void); + +/** + * kmsan_init_runtime() - Initialize KMSAN state and enable KMSAN. + */ +void __init kmsan_init_runtime(void); + +/** + * kmsan_memblock_free_pages() - handle freeing of memblock pages. + * @page: struct page to free. + * @order: order of @page. + * + * Freed pages are either returned to buddy allocator or held back to be used + * as metadata pages. + */ +bool __init kmsan_memblock_free_pages(struct page *page, unsigned int order); + +/** + * kmsan_alloc_page() - Notify KMSAN about an alloc_pages() call. + * @page: struct page pointer returned by alloc_pages(). + * @order: order of allocated struct page. + * @flags: GFP flags used by alloc_pages() + * + * KMSAN marks 1<<@order pages starting at @page as uninitialized, unless + * @flags contain __GFP_ZERO. + */ +void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags); + +/** + * kmsan_free_page() - Notify KMSAN about a free_pages() call. + * @page: struct page pointer passed to free_pages(). + * @order: order of deallocated struct page. + * + * KMSAN marks freed memory as uninitialized. + */ +void kmsan_free_page(struct page *page, unsigned int order); + +/** + * kmsan_copy_page_meta() - Copy KMSAN metadata between two pages. + * @dst: destination page. + * @src: source page. + * + * KMSAN copies the contents of metadata pages for @src into the metadata pages + * for @dst. If @dst has no associated metadata pages, nothing happens. + * If @src has no associated metadata pages, @dst metadata pages are unpoisoned. + */ +void kmsan_copy_page_meta(struct page *dst, struct page *src); + +/** + * kmsan_slab_alloc() - Notify KMSAN about a slab allocation. + * @s: slab cache the object belongs to. + * @object: object pointer. + * @flags: GFP flags passed to the allocator. + * + * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the + * newly created object, marking it as initialized or uninitialized. + */ +void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags); + +/** + * kmsan_slab_free() - Notify KMSAN about a slab deallocation. + * @s: slab cache the object belongs to. + * @object: object pointer. + * + * KMSAN marks the freed object as uninitialized. + */ +void kmsan_slab_free(struct kmem_cache *s, void *object); + +/** + * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation. + * @ptr: object pointer. + * @size: object size. + * @flags: GFP flags passed to the allocator. + * + * Similar to kmsan_slab_alloc(), but for large allocations. + */ +void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags); + +/** + * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation. + * @ptr: object pointer. + * + * Similar to kmsan_slab_free(), but for large allocations. + */ +void kmsan_kfree_large(const void *ptr); + +/** + * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap. + * @start: start of vmapped range. + * @end: end of vmapped range. + * @prot: page protection flags used for vmap. + * @pages: array of pages. + * @page_shift: page_shift passed to vmap_range_noflush(). + * + * KMSAN maps shadow and origin pages of @pages into contiguous ranges in + * vmalloc metadata address range. + */ +void kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift); + +/** + * kmsan_vunmap_kernel_range_noflush() - Notify KMSAN about a vunmap. + * @start: start of vunmapped range. + * @end: end of vunmapped range. + * + * KMSAN unmaps the contiguous metadata ranges created by + * kmsan_map_kernel_range_noflush(). + */ +void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end); + +/** + * kmsan_ioremap_page_range() - Notify KMSAN about a ioremap_page_range() call. + * @addr: range start. + * @end: range end. + * @phys_addr: physical range start. + * @prot: page protection flags used for ioremap_page_range(). + * @page_shift: page_shift argument passed to vmap_range_noflush(). + * + * KMSAN creates new metadata pages for the physical pages mapped into the + * virtual memory. + */ +void kmsan_ioremap_page_range(unsigned long addr, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot, + unsigned int page_shift); + +/** + * kmsan_iounmap_page_range() - Notify KMSAN about a iounmap_page_range() call. + * @start: range start. + * @end: range end. + * + * KMSAN unmaps the metadata pages for the given range and, unlike for + * vunmap_page_range(), also deallocates them. + */ +void kmsan_iounmap_page_range(unsigned long start, unsigned long end); + +/** + * kmsan_handle_dma() - Handle a DMA data transfer. + * @page: first page of the buffer. + * @offset: offset of the buffer within the first page. + * @size: buffer size. + * @dir: one of possible dma_data_direction values. + * + * Depending on @direction, KMSAN: + * * checks the buffer, if it is copied to device; + * * initializes the buffer, if it is copied from device; + * * does both, if this is a DMA_BIDIRECTIONAL transfer. + */ +void kmsan_handle_dma(struct page *page, size_t offset, size_t size, + enum dma_data_direction dir); + +/** + * kmsan_handle_dma_sg() - Handle a DMA transfer using scatterlist. + * @sg: scatterlist holding DMA buffers. + * @nents: number of scatterlist entries. + * @dir: one of possible dma_data_direction values. + * + * Depending on @direction, KMSAN: + * * checks the buffers in the scatterlist, if they are copied to device; + * * initializes the buffers, if they are copied from device; + * * does both, if this is a DMA_BIDIRECTIONAL transfer. + */ +void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, + enum dma_data_direction dir); + +/** + * kmsan_handle_urb() - Handle a USB data transfer. + * @urb: struct urb pointer. + * @is_out: data transfer direction (true means output to hardware). + * + * If @is_out is true, KMSAN checks the transfer buffer of @urb. Otherwise, + * KMSAN initializes the transfer buffer. + */ +void kmsan_handle_urb(const struct urb *urb, bool is_out); + +/** + * kmsan_unpoison_entry_regs() - Handle pt_regs in low-level entry code. + * @regs: struct pt_regs pointer received from assembly code. + * + * KMSAN unpoisons the contents of the passed pt_regs, preventing potential + * false positive reports. Unlike kmsan_unpoison_memory(), + * kmsan_unpoison_entry_regs() can be called from the regions where + * kmsan_in_runtime() returns true, which is the case in early entry code. + */ +void kmsan_unpoison_entry_regs(const struct pt_regs *regs); + +#else + +static inline void kmsan_init_shadow(void) +{ +} + +static inline void kmsan_init_runtime(void) +{ +} + +static inline bool kmsan_memblock_free_pages(struct page *page, + unsigned int order) +{ + return true; +} + +static inline void kmsan_task_create(struct task_struct *task) +{ +} + +static inline void kmsan_task_exit(struct task_struct *task) +{ +} + +static inline int kmsan_alloc_page(struct page *page, unsigned int order, + gfp_t flags) +{ + return 0; +} + +static inline void kmsan_free_page(struct page *page, unsigned int order) +{ +} + +static inline void kmsan_copy_page_meta(struct page *dst, struct page *src) +{ +} + +static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object, + gfp_t flags) +{ +} + +static inline void kmsan_slab_free(struct kmem_cache *s, void *object) +{ +} + +static inline void kmsan_kmalloc_large(const void *ptr, size_t size, + gfp_t flags) +{ +} + +static inline void kmsan_kfree_large(const void *ptr) +{ +} + +static inline void kmsan_vmap_pages_range_noflush(unsigned long start, + unsigned long end, + pgprot_t prot, + struct page **pages, + unsigned int page_shift) +{ +} + +static inline void kmsan_vunmap_range_noflush(unsigned long start, + unsigned long end) +{ +} + +static inline void kmsan_ioremap_page_range(unsigned long start, + unsigned long end, + phys_addr_t phys_addr, + pgprot_t prot, + unsigned int page_shift) +{ +} + +static inline void kmsan_iounmap_page_range(unsigned long start, + unsigned long end) +{ +} + +static inline void kmsan_handle_dma(struct page *page, size_t offset, + size_t size, enum dma_data_direction dir) +{ +} + +static inline void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, + enum dma_data_direction dir) +{ +} + +static inline void kmsan_handle_urb(const struct urb *urb, bool is_out) +{ +} + +static inline void kmsan_unpoison_entry_regs(const struct pt_regs *regs) +{ +} + +#endif + +#endif /* _LINUX_KMSAN_H */ diff --git a/include/linux/kmsan_types.h b/include/linux/kmsan_types.h new file mode 100644 index 0000000000000000000000000000000000000000..8bfa6c98176d4432950d35e5348e6e1714f88efb --- /dev/null +++ b/include/linux/kmsan_types.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * A minimal header declaring types added by KMSAN to existing kernel structs. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ +#ifndef _LINUX_KMSAN_TYPES_H +#define _LINUX_KMSAN_TYPES_H + +/* These constants are defined in the MSan LLVM instrumentation pass. */ +#define KMSAN_RETVAL_SIZE 800 +#define KMSAN_PARAM_SIZE 800 + +struct kmsan_context_state { + char param_tls[KMSAN_PARAM_SIZE]; + char retval_tls[KMSAN_RETVAL_SIZE]; + char va_arg_tls[KMSAN_PARAM_SIZE]; + char va_arg_origin_tls[KMSAN_PARAM_SIZE]; + u64 va_arg_overflow_size_tls; + char param_origin_tls[KMSAN_PARAM_SIZE]; + u32 retval_origin_tls; +}; + +#undef KMSAN_PARAM_SIZE +#undef KMSAN_RETVAL_SIZE + +struct kmsan_ctx { + struct kmsan_context_state cstate; + int kmsan_in_runtime; + bool allow_reporting; +}; + +#endif /* _LINUX_KMSAN_TYPES_H */ diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 55041d2f884de12bcb58b2ddd8124609195607cc..a0b92be98984ea58b3608b664d8f45d4af60ac32 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -103,6 +103,7 @@ struct kprobe { * this flag is only for optimized_kprobe. */ #define KPROBE_FLAG_FTRACE 8 /* probe is using ftrace */ +#define KPROBE_FLAG_ON_FUNC_ENTRY 16 /* probe is on the function entry */ /* Has this kprobe gone ? */ static inline bool kprobe_gone(struct kprobe *p) diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 0b4f17418f64c11ab7a840b398d8585733715194..7e232ba59b86567c57cee6dbb5d9d6dbf63c31aa 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -15,9 +15,6 @@ #include #include -struct stable_node; -struct mem_cgroup; - #ifdef CONFIG_KSM int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index f4519d3689e102dadb22a212ce994a2f8852acea..32f259fa58013eca0af1ea419b6a6a21d88e8272 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -151,12 +151,11 @@ static inline bool is_error_page(struct page *page) #define KVM_REQUEST_NO_ACTION BIT(10) /* * Architecture-independent vcpu->requests bit members - * Bits 4-7 are reserved for more arch-independent bits. + * Bits 3-7 are reserved for more arch-independent bits. */ #define KVM_REQ_TLB_FLUSH (0 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_VM_DEAD (1 | KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UNBLOCK 2 -#define KVM_REQ_UNHALT 3 #define KVM_REQUEST_ARCH_BASE 8 /* @@ -2247,6 +2246,19 @@ static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) } #endif /* CONFIG_KVM_XFER_TO_GUEST_WORK */ +/* + * If more than one page is being (un)accounted, @virt must be the address of + * the first page of a block of pages what were allocated together (i.e + * accounted together). + * + * kvm_account_pgtable_pages() is thread-safe because mod_lruvec_page_state() + * is thread-safe. + */ +static inline void kvm_account_pgtable_pages(void *virt, int nr) +{ + mod_lruvec_page_state(virt_to_page(virt), NR_SECONDARY_PAGETABLE, nr); +} + /* * This defines how many reserved entries we want to keep before we * kick the vcpu to the userspace to avoid dirty ring full. This diff --git a/include/linux/libata.h b/include/linux/libata.h index 698032e5ef2d3fe0c870c0fda087da4a7eeee2ca..fe990176e6ee993f26ddd07ea2d857c516753b04 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -101,7 +101,7 @@ enum { ATA_DFLAG_UNLOCK_HPA = (1 << 18), /* unlock HPA */ ATA_DFLAG_NCQ_SEND_RECV = (1 << 19), /* device supports NCQ SEND and RECV */ ATA_DFLAG_NCQ_PRIO = (1 << 20), /* device supports NCQ priority */ - ATA_DFLAG_NCQ_PRIO_ENABLE = (1 << 21), /* Priority cmds sent to dev */ + ATA_DFLAG_NCQ_PRIO_ENABLED = (1 << 21), /* Priority cmds sent to dev */ ATA_DFLAG_INIT_MASK = (1 << 24) - 1, ATA_DFLAG_DETACH = (1 << 24), @@ -1136,8 +1136,8 @@ extern int ata_scsi_slave_config(struct scsi_device *sdev); extern void ata_scsi_slave_destroy(struct scsi_device *sdev); extern int ata_scsi_change_queue_depth(struct scsi_device *sdev, int queue_depth); -extern int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev, - int queue_depth); +extern int ata_change_queue_depth(struct ata_port *ap, struct ata_device *dev, + struct scsi_device *sdev, int queue_depth); extern struct ata_device *ata_dev_pair(struct ata_device *adev); extern int ata_do_set_mode(struct ata_link *link, struct ata_device **r_failed_dev); extern void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap); diff --git a/include/linux/linear_range.h b/include/linux/linear_range.h index fd3d0b358f2283b65b51811f422216981c67fb49..2e4f4c3539c075a6caa69f6ae750a556f41fce5d 100644 --- a/include/linux/linear_range.h +++ b/include/linux/linear_range.h @@ -26,6 +26,17 @@ struct linear_range { unsigned int step; }; +#define LINEAR_RANGE(_min, _min_sel, _max_sel, _step) \ + { \ + .min = _min, \ + .min_sel = _min_sel, \ + .max_sel = _max_sel, \ + .step = _step, \ + } + +#define LINEAR_RANGE_IDX(_idx, _min, _min_sel, _max_sel, _step) \ + [_idx] = LINEAR_RANGE(_min, _min_sel, _max_sel, _step) + unsigned int linear_range_values_in_range(const struct linear_range *r); unsigned int linear_range_values_in_range_array(const struct linear_range *r, int ranges); diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 8064481730333deb7ac23e1d39541d138dc93b94..ec119da1d89b407e95bd683919f4b07b239dca08 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -224,6 +224,7 @@ LSM_HOOK(int, -ENOSYS, task_prctl, int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5) LSM_HOOK(void, LSM_RET_VOID, task_to_inode, struct task_struct *p, struct inode *inode) +LSM_HOOK(int, 0, userns_create, const struct cred *cred) LSM_HOOK(int, 0, ipc_permission, struct kern_ipc_perm *ipcp, short flag) LSM_HOOK(void, LSM_RET_VOID, ipc_getsecid, struct kern_ipc_perm *ipcp, u32 *secid) @@ -253,7 +254,7 @@ LSM_HOOK(int, 0, sem_semop, struct kern_ipc_perm *perm, struct sembuf *sops, LSM_HOOK(int, 0, netlink_send, struct sock *sk, struct sk_buff *skb) LSM_HOOK(void, LSM_RET_VOID, d_instantiate, struct dentry *dentry, struct inode *inode) -LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, char *name, +LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, const char *name, char **value) LSM_HOOK(int, -EINVAL, setprocattr, const char *name, void *value, size_t size) LSM_HOOK(int, 0, ismaclabel, const char *name) @@ -407,4 +408,5 @@ LSM_HOOK(int, 0, perf_event_write, struct perf_event *event) #ifdef CONFIG_IO_URING LSM_HOOK(int, 0, uring_override_creds, const struct cred *new) LSM_HOOK(int, 0, uring_sqpoll, void) +LSM_HOOK(int, 0, uring_cmd, struct io_uring_cmd *ioucmd) #endif /* CONFIG_IO_URING */ diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 84a0d7e02176969b2a9549dbd58058ad1862e745..4ec80b96c22e778f951698a0c1110d20e50bc634 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -806,6 +806,10 @@ * security attributes, e.g. for /proc/pid inodes. * @p contains the task_struct for the task. * @inode contains the inode structure for the inode. + * @userns_create: + * Check permission prior to creating a new user namespace. + * @cred points to prepared creds. + * Return 0 if successful, otherwise < 0 error code. * * Security hooks for Netlink messaging. * @@ -1582,6 +1586,9 @@ * Check whether the current task is allowed to spawn a io_uring polling * thread (IORING_SETUP_SQPOLL). * + * @uring_cmd: + * Check whether the file_operations uring_cmd is allowed to run. + * */ union security_list_options { #define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__); diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h new file mode 100644 index 0000000000000000000000000000000000000000..2effab72add10e1030757948328f460cba9389d2 --- /dev/null +++ b/include/linux/maple_tree.h @@ -0,0 +1,685 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef _LINUX_MAPLE_TREE_H +#define _LINUX_MAPLE_TREE_H +/* + * Maple Tree - An RCU-safe adaptive tree for storing ranges + * Copyright (c) 2018-2022 Oracle + * Authors: Liam R. Howlett + * Matthew Wilcox + */ + +#include +#include +#include +/* #define CONFIG_MAPLE_RCU_DISABLED */ +/* #define CONFIG_DEBUG_MAPLE_TREE_VERBOSE */ + +/* + * Allocated nodes are mutable until they have been inserted into the tree, + * at which time they cannot change their type until they have been removed + * from the tree and an RCU grace period has passed. + * + * Removed nodes have their ->parent set to point to themselves. RCU readers + * check ->parent before relying on the value that they loaded from the + * slots array. This lets us reuse the slots array for the RCU head. + * + * Nodes in the tree point to their parent unless bit 0 is set. + */ +#if defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) +/* 64bit sizes */ +#define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ +#define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ +#define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ +#define MAPLE_ARANGE64_META_MAX 15 /* Out of range for metadata */ +#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) +#else +/* 32bit sizes */ +#define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ +#define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ +#define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ +#define MAPLE_ARANGE64_META_MAX 31 /* Out of range for metadata */ +#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) +#endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ + +#define MAPLE_NODE_MASK 255UL + +/* + * The node->parent of the root node has bit 0 set and the rest of the pointer + * is a pointer to the tree itself. No more bits are available in this pointer + * (on m68k, the data structure may only be 2-byte aligned). + * + * Internal non-root nodes can only have maple_range_* nodes as parents. The + * parent pointer is 256B aligned like all other tree nodes. When storing a 32 + * or 64 bit values, the offset can fit into 4 bits. The 16 bit values need an + * extra bit to store the offset. This extra bit comes from a reuse of the last + * bit in the node type. This is possible by using bit 1 to indicate if bit 2 + * is part of the type or the slot. + * + * Once the type is decided, the decision of an allocation range type or a range + * type is done by examining the immutable tree flag for the MAPLE_ALLOC_RANGE + * flag. + * + * Node types: + * 0x??1 = Root + * 0x?00 = 16 bit nodes + * 0x010 = 32 bit nodes + * 0x110 = 64 bit nodes + * + * Slot size and location in the parent pointer: + * type : slot location + * 0x??1 : Root + * 0x?00 : 16 bit values, type in 0-1, slot in 2-6 + * 0x010 : 32 bit values, type in 0-2, slot in 3-6 + * 0x110 : 64 bit values, type in 0-2, slot in 3-6 + */ + +/* + * This metadata is used to optimize the gap updating code and in reverse + * searching for gaps or any other code that needs to find the end of the data. + */ +struct maple_metadata { + unsigned char end; + unsigned char gap; +}; + +/* + * Leaf nodes do not store pointers to nodes, they store user data. Users may + * store almost any bit pattern. As noted above, the optimisation of storing an + * entry at 0 in the root pointer cannot be done for data which have the bottom + * two bits set to '10'. We also reserve values with the bottom two bits set to + * '10' which are below 4096 (ie 2, 6, 10 .. 4094) for internal use. Some APIs + * return errnos as a negative errno shifted right by two bits and the bottom + * two bits set to '10', and while choosing to store these values in the array + * is not an error, it may lead to confusion if you're testing for an error with + * mas_is_err(). + * + * Non-leaf nodes store the type of the node pointed to (enum maple_type in bits + * 3-6), bit 2 is reserved. That leaves bits 0-1 unused for now. + * + * In regular B-Tree terms, pivots are called keys. The term pivot is used to + * indicate that the tree is specifying ranges, Pivots may appear in the + * subtree with an entry attached to the value whereas keys are unique to a + * specific position of a B-tree. Pivot values are inclusive of the slot with + * the same index. + */ + +struct maple_range_64 { + struct maple_pnode *parent; + unsigned long pivot[MAPLE_RANGE64_SLOTS - 1]; + union { + void __rcu *slot[MAPLE_RANGE64_SLOTS]; + struct { + void __rcu *pad[MAPLE_RANGE64_SLOTS - 1]; + struct maple_metadata meta; + }; + }; +}; + +/* + * At tree creation time, the user can specify that they're willing to trade off + * storing fewer entries in a tree in return for storing more information in + * each node. + * + * The maple tree supports recording the largest range of NULL entries available + * in this node, also called gaps. This optimises the tree for allocating a + * range. + */ +struct maple_arange_64 { + struct maple_pnode *parent; + unsigned long pivot[MAPLE_ARANGE64_SLOTS - 1]; + void __rcu *slot[MAPLE_ARANGE64_SLOTS]; + unsigned long gap[MAPLE_ARANGE64_SLOTS]; + struct maple_metadata meta; +}; + +struct maple_alloc { + unsigned long total; + unsigned char node_count; + unsigned int request_count; + struct maple_alloc *slot[MAPLE_ALLOC_SLOTS]; +}; + +struct maple_topiary { + struct maple_pnode *parent; + struct maple_enode *next; /* Overlaps the pivot */ +}; + +enum maple_type { + maple_dense, + maple_leaf_64, + maple_range_64, + maple_arange_64, +}; + + +/** + * DOC: Maple tree flags + * + * * MT_FLAGS_ALLOC_RANGE - Track gaps in this tree + * * MT_FLAGS_USE_RCU - Operate in RCU mode + * * MT_FLAGS_HEIGHT_OFFSET - The position of the tree height in the flags + * * MT_FLAGS_HEIGHT_MASK - The mask for the maple tree height value + * * MT_FLAGS_LOCK_MASK - How the mt_lock is used + * * MT_FLAGS_LOCK_IRQ - Acquired irq-safe + * * MT_FLAGS_LOCK_BH - Acquired bh-safe + * * MT_FLAGS_LOCK_EXTERN - mt_lock is not used + * + * MAPLE_HEIGHT_MAX The largest height that can be stored + */ +#define MT_FLAGS_ALLOC_RANGE 0x01 +#define MT_FLAGS_USE_RCU 0x02 +#define MT_FLAGS_HEIGHT_OFFSET 0x02 +#define MT_FLAGS_HEIGHT_MASK 0x7C +#define MT_FLAGS_LOCK_MASK 0x300 +#define MT_FLAGS_LOCK_IRQ 0x100 +#define MT_FLAGS_LOCK_BH 0x200 +#define MT_FLAGS_LOCK_EXTERN 0x300 + +#define MAPLE_HEIGHT_MAX 31 + + +#define MAPLE_NODE_TYPE_MASK 0x0F +#define MAPLE_NODE_TYPE_SHIFT 0x03 + +#define MAPLE_RESERVED_RANGE 4096 + +#ifdef CONFIG_LOCKDEP +typedef struct lockdep_map *lockdep_map_p; +#define mt_lock_is_held(mt) lock_is_held(mt->ma_external_lock) +#define mt_set_external_lock(mt, lock) \ + (mt)->ma_external_lock = &(lock)->dep_map +#else +typedef struct { /* nothing */ } lockdep_map_p; +#define mt_lock_is_held(mt) 1 +#define mt_set_external_lock(mt, lock) do { } while (0) +#endif + +/* + * If the tree contains a single entry at index 0, it is usually stored in + * tree->ma_root. To optimise for the page cache, an entry which ends in '00', + * '01' or '11' is stored in the root, but an entry which ends in '10' will be + * stored in a node. Bits 3-6 are used to store enum maple_type. + * + * The flags are used both to store some immutable information about this tree + * (set at tree creation time) and dynamic information set under the spinlock. + * + * Another use of flags are to indicate global states of the tree. This is the + * case with the MAPLE_USE_RCU flag, which indicates the tree is currently in + * RCU mode. This mode was added to allow the tree to reuse nodes instead of + * re-allocating and RCU freeing nodes when there is a single user. + */ +struct maple_tree { + union { + spinlock_t ma_lock; + lockdep_map_p ma_external_lock; + }; + void __rcu *ma_root; + unsigned int ma_flags; +}; + +/** + * MTREE_INIT() - Initialize a maple tree + * @name: The maple tree name + * @__flags: The maple tree flags + * + */ +#define MTREE_INIT(name, __flags) { \ + .ma_lock = __SPIN_LOCK_UNLOCKED((name).ma_lock), \ + .ma_flags = __flags, \ + .ma_root = NULL, \ +} + +/** + * MTREE_INIT_EXT() - Initialize a maple tree with an external lock. + * @name: The tree name + * @__flags: The maple tree flags + * @__lock: The external lock + */ +#ifdef CONFIG_LOCKDEP +#define MTREE_INIT_EXT(name, __flags, __lock) { \ + .ma_external_lock = &(__lock).dep_map, \ + .ma_flags = (__flags), \ + .ma_root = NULL, \ +} +#else +#define MTREE_INIT_EXT(name, __flags, __lock) MTREE_INIT(name, __flags) +#endif + +#define DEFINE_MTREE(name) \ + struct maple_tree name = MTREE_INIT(name, 0) + +#define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) +#define mtree_unlock(mt) spin_unlock((&(mt)->ma_lock)) + +/* + * The Maple Tree squeezes various bits in at various points which aren't + * necessarily obvious. Usually, this is done by observing that pointers are + * N-byte aligned and thus the bottom log_2(N) bits are available for use. We + * don't use the high bits of pointers to store additional information because + * we don't know what bits are unused on any given architecture. + * + * Nodes are 256 bytes in size and are also aligned to 256 bytes, giving us 8 + * low bits for our own purposes. Nodes are currently of 4 types: + * 1. Single pointer (Range is 0-0) + * 2. Non-leaf Allocation Range nodes + * 3. Non-leaf Range nodes + * 4. Leaf Range nodes All nodes consist of a number of node slots, + * pivots, and a parent pointer. + */ + +struct maple_node { + union { + struct { + struct maple_pnode *parent; + void __rcu *slot[MAPLE_NODE_SLOTS]; + }; + struct { + void *pad; + struct rcu_head rcu; + struct maple_enode *piv_parent; + unsigned char parent_slot; + enum maple_type type; + unsigned char slot_len; + unsigned int ma_flags; + }; + struct maple_range_64 mr64; + struct maple_arange_64 ma64; + struct maple_alloc alloc; + }; +}; + +/* + * More complicated stores can cause two nodes to become one or three and + * potentially alter the height of the tree. Either half of the tree may need + * to be rebalanced against the other. The ma_topiary struct is used to track + * which nodes have been 'cut' from the tree so that the change can be done + * safely at a later date. This is done to support RCU. + */ +struct ma_topiary { + struct maple_enode *head; + struct maple_enode *tail; + struct maple_tree *mtree; +}; + +void *mtree_load(struct maple_tree *mt, unsigned long index); + +int mtree_insert(struct maple_tree *mt, unsigned long index, + void *entry, gfp_t gfp); +int mtree_insert_range(struct maple_tree *mt, unsigned long first, + unsigned long last, void *entry, gfp_t gfp); +int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, + void *entry, unsigned long size, unsigned long min, + unsigned long max, gfp_t gfp); +int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, + void *entry, unsigned long size, unsigned long min, + unsigned long max, gfp_t gfp); + +int mtree_store_range(struct maple_tree *mt, unsigned long first, + unsigned long last, void *entry, gfp_t gfp); +int mtree_store(struct maple_tree *mt, unsigned long index, + void *entry, gfp_t gfp); +void *mtree_erase(struct maple_tree *mt, unsigned long index); + +void mtree_destroy(struct maple_tree *mt); +void __mt_destroy(struct maple_tree *mt); + +/** + * mtree_empty() - Determine if a tree has any present entries. + * @mt: Maple Tree. + * + * Context: Any context. + * Return: %true if the tree contains only NULL pointers. + */ +static inline bool mtree_empty(const struct maple_tree *mt) +{ + return mt->ma_root == NULL; +} + +/* Advanced API */ + +/* + * The maple state is defined in the struct ma_state and is used to keep track + * of information during operations, and even between operations when using the + * advanced API. + * + * If state->node has bit 0 set then it references a tree location which is not + * a node (eg the root). If bit 1 is set, the rest of the bits are a negative + * errno. Bit 2 (the 'unallocated slots' bit) is clear. Bits 3-6 indicate the + * node type. + * + * state->alloc either has a request number of nodes or an allocated node. If + * stat->alloc has a requested number of nodes, the first bit will be set (0x1) + * and the remaining bits are the value. If state->alloc is a node, then the + * node will be of type maple_alloc. maple_alloc has MAPLE_NODE_SLOTS - 1 for + * storing more allocated nodes, a total number of nodes allocated, and the + * node_count in this node. node_count is the number of allocated nodes in this + * node. The scaling beyond MAPLE_NODE_SLOTS - 1 is handled by storing further + * nodes into state->alloc->slot[0]'s node. Nodes are taken from state->alloc + * by removing a node from the state->alloc node until state->alloc->node_count + * is 1, when state->alloc is returned and the state->alloc->slot[0] is promoted + * to state->alloc. Nodes are pushed onto state->alloc by putting the current + * state->alloc into the pushed node's slot[0]. + * + * The state also contains the implied min/max of the state->node, the depth of + * this search, and the offset. The implied min/max are either from the parent + * node or are 0-oo for the root node. The depth is incremented or decremented + * every time a node is walked down or up. The offset is the slot/pivot of + * interest in the node - either for reading or writing. + * + * When returning a value the maple state index and last respectively contain + * the start and end of the range for the entry. Ranges are inclusive in the + * Maple Tree. + */ +struct ma_state { + struct maple_tree *tree; /* The tree we're operating in */ + unsigned long index; /* The index we're operating on - range start */ + unsigned long last; /* The last index we're operating on - range end */ + struct maple_enode *node; /* The node containing this entry */ + unsigned long min; /* The minimum index of this node - implied pivot min */ + unsigned long max; /* The maximum index of this node - implied pivot max */ + struct maple_alloc *alloc; /* Allocated nodes for this operation */ + unsigned char depth; /* depth of tree descent during write */ + unsigned char offset; + unsigned char mas_flags; +}; + +struct ma_wr_state { + struct ma_state *mas; + struct maple_node *node; /* Decoded mas->node */ + unsigned long r_min; /* range min */ + unsigned long r_max; /* range max */ + enum maple_type type; /* mas->node type */ + unsigned char offset_end; /* The offset where the write ends */ + unsigned char node_end; /* mas->node end */ + unsigned long *pivots; /* mas->node->pivots pointer */ + unsigned long end_piv; /* The pivot at the offset end */ + void __rcu **slots; /* mas->node->slots pointer */ + void *entry; /* The entry to write */ + void *content; /* The existing entry that is being overwritten */ +}; + +#define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) +#define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) + + +/* + * Special values for ma_state.node. + * MAS_START means we have not searched the tree. + * MAS_ROOT means we have searched the tree and the entry we found lives in + * the root of the tree (ie it has index 0, length 1 and is the only entry in + * the tree). + * MAS_NONE means we have searched the tree and there is no node in the + * tree for this entry. For example, we searched for index 1 in an empty + * tree. Or we have a tree which points to a full leaf node and we + * searched for an entry which is larger than can be contained in that + * leaf node. + * MA_ERROR represents an errno. After dropping the lock and attempting + * to resolve the error, the walk would have to be restarted from the + * top of the tree as the tree may have been modified. + */ +#define MAS_START ((struct maple_enode *)1UL) +#define MAS_ROOT ((struct maple_enode *)5UL) +#define MAS_NONE ((struct maple_enode *)9UL) +#define MAS_PAUSE ((struct maple_enode *)17UL) +#define MA_ERROR(err) \ + ((struct maple_enode *)(((unsigned long)err << 2) | 2UL)) + +#define MA_STATE(name, mt, first, end) \ + struct ma_state name = { \ + .tree = mt, \ + .index = first, \ + .last = end, \ + .node = MAS_START, \ + .min = 0, \ + .max = ULONG_MAX, \ + .alloc = NULL, \ + } + +#define MA_WR_STATE(name, ma_state, wr_entry) \ + struct ma_wr_state name = { \ + .mas = ma_state, \ + .content = NULL, \ + .entry = wr_entry, \ + } + +#define MA_TOPIARY(name, tree) \ + struct ma_topiary name = { \ + .head = NULL, \ + .tail = NULL, \ + .mtree = tree, \ + } + +void *mas_walk(struct ma_state *mas); +void *mas_store(struct ma_state *mas, void *entry); +void *mas_erase(struct ma_state *mas); +int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp); +void mas_store_prealloc(struct ma_state *mas, void *entry); +void *mas_find(struct ma_state *mas, unsigned long max); +void *mas_find_rev(struct ma_state *mas, unsigned long min); +int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp); +bool mas_is_err(struct ma_state *mas); + +bool mas_nomem(struct ma_state *mas, gfp_t gfp); +void mas_pause(struct ma_state *mas); +void maple_tree_init(void); +void mas_destroy(struct ma_state *mas); +int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); + +void *mas_prev(struct ma_state *mas, unsigned long min); +void *mas_next(struct ma_state *mas, unsigned long max); + +int mas_empty_area(struct ma_state *mas, unsigned long min, unsigned long max, + unsigned long size); + +/* Checks if a mas has not found anything */ +static inline bool mas_is_none(struct ma_state *mas) +{ + return mas->node == MAS_NONE; +} + +/* Checks if a mas has been paused */ +static inline bool mas_is_paused(struct ma_state *mas) +{ + return mas->node == MAS_PAUSE; +} + +void mas_dup_tree(struct ma_state *oldmas, struct ma_state *mas); +void mas_dup_store(struct ma_state *mas, void *entry); + +/* + * This finds an empty area from the highest address to the lowest. + * AKA "Topdown" version, + */ +int mas_empty_area_rev(struct ma_state *mas, unsigned long min, + unsigned long max, unsigned long size); +/** + * mas_reset() - Reset a Maple Tree operation state. + * @mas: Maple Tree operation state. + * + * Resets the error or walk state of the @mas so future walks of the + * array will start from the root. Use this if you have dropped the + * lock and want to reuse the ma_state. + * + * Context: Any context. + */ +static inline void mas_reset(struct ma_state *mas) +{ + mas->node = MAS_START; +} + +/** + * mas_for_each() - Iterate over a range of the maple tree. + * @__mas: Maple Tree operation state (maple_state) + * @__entry: Entry retrieved from the tree + * @__max: maximum index to retrieve from the tree + * + * When returned, mas->index and mas->last will hold the entire range for the + * entry. + * + * Note: may return the zero entry. + * + */ +#define mas_for_each(__mas, __entry, __max) \ + while (((__entry) = mas_find((__mas), (__max))) != NULL) + + +/** + * mas_set_range() - Set up Maple Tree operation state for a different index. + * @mas: Maple Tree operation state. + * @start: New start of range in the Maple Tree. + * @last: New end of range in the Maple Tree. + * + * Move the operation state to refer to a different range. This will + * have the effect of starting a walk from the top; see mas_next() + * to move to an adjacent index. + */ +static inline +void mas_set_range(struct ma_state *mas, unsigned long start, unsigned long last) +{ + mas->index = start; + mas->last = last; + mas->node = MAS_START; +} + +/** + * mas_set() - Set up Maple Tree operation state for a different index. + * @mas: Maple Tree operation state. + * @index: New index into the Maple Tree. + * + * Move the operation state to refer to a different index. This will + * have the effect of starting a walk from the top; see mas_next() + * to move to an adjacent index. + */ +static inline void mas_set(struct ma_state *mas, unsigned long index) +{ + + mas_set_range(mas, index, index); +} + +static inline bool mt_external_lock(const struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_LOCK_MASK) == MT_FLAGS_LOCK_EXTERN; +} + +/** + * mt_init_flags() - Initialise an empty maple tree with flags. + * @mt: Maple Tree + * @flags: maple tree flags. + * + * If you need to initialise a Maple Tree with special flags (eg, an + * allocation tree), use this function. + * + * Context: Any context. + */ +static inline void mt_init_flags(struct maple_tree *mt, unsigned int flags) +{ + mt->ma_flags = flags; + if (!mt_external_lock(mt)) + spin_lock_init(&mt->ma_lock); + rcu_assign_pointer(mt->ma_root, NULL); +} + +/** + * mt_init() - Initialise an empty maple tree. + * @mt: Maple Tree + * + * An empty Maple Tree. + * + * Context: Any context. + */ +static inline void mt_init(struct maple_tree *mt) +{ + mt_init_flags(mt, 0); +} + +static inline bool mt_in_rcu(struct maple_tree *mt) +{ +#ifdef CONFIG_MAPLE_RCU_DISABLED + return false; +#endif + return mt->ma_flags & MT_FLAGS_USE_RCU; +} + +/** + * mt_clear_in_rcu() - Switch the tree to non-RCU mode. + * @mt: The Maple Tree + */ +static inline void mt_clear_in_rcu(struct maple_tree *mt) +{ + if (!mt_in_rcu(mt)) + return; + + if (mt_external_lock(mt)) { + BUG_ON(!mt_lock_is_held(mt)); + mt->ma_flags &= ~MT_FLAGS_USE_RCU; + } else { + mtree_lock(mt); + mt->ma_flags &= ~MT_FLAGS_USE_RCU; + mtree_unlock(mt); + } +} + +/** + * mt_set_in_rcu() - Switch the tree to RCU safe mode. + * @mt: The Maple Tree + */ +static inline void mt_set_in_rcu(struct maple_tree *mt) +{ + if (mt_in_rcu(mt)) + return; + + if (mt_external_lock(mt)) { + BUG_ON(!mt_lock_is_held(mt)); + mt->ma_flags |= MT_FLAGS_USE_RCU; + } else { + mtree_lock(mt); + mt->ma_flags |= MT_FLAGS_USE_RCU; + mtree_unlock(mt); + } +} + +void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); +void *mt_find_after(struct maple_tree *mt, unsigned long *index, + unsigned long max); +void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min); +void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max); + +/** + * mt_for_each - Iterate over each entry starting at index until max. + * @__tree: The Maple Tree + * @__entry: The current entry + * @__index: The index to update to track the location in the tree + * @__max: The maximum limit for @index + * + * Note: Will not return the zero entry. + */ +#define mt_for_each(__tree, __entry, __index, __max) \ + for (__entry = mt_find(__tree, &(__index), __max); \ + __entry; __entry = mt_find_after(__tree, &(__index), __max)) + + +#ifdef CONFIG_DEBUG_MAPLE_TREE +extern atomic_t maple_tree_tests_run; +extern atomic_t maple_tree_tests_passed; + +void mt_dump(const struct maple_tree *mt); +void mt_validate(struct maple_tree *mt); +#define MT_BUG_ON(__tree, __x) do { \ + atomic_inc(&maple_tree_tests_run); \ + if (__x) { \ + pr_info("BUG at %s:%d (%u)\n", \ + __func__, __LINE__, __x); \ + mt_dump(__tree); \ + pr_info("Pass: %u Run:%u\n", \ + atomic_read(&maple_tree_tests_passed), \ + atomic_read(&maple_tree_tests_run)); \ + dump_stack(); \ + } else { \ + atomic_inc(&maple_tree_tests_passed); \ + } \ +} while (0) +#else +#define MT_BUG_ON(__tree, __x) BUG_ON(__x) +#endif /* CONFIG_DEBUG_MAPLE_TREE */ + +#endif /*_LINUX_MAPLE_TREE_H */ diff --git a/include/linux/mdev.h b/include/linux/mdev.h index 47ad3b104d9e73728b6b69145c8703b2a1c94afa..139d05b26f820aed916fe2be175dd67f5379d02c 100644 --- a/include/linux/mdev.h +++ b/include/linux/mdev.h @@ -10,6 +10,9 @@ #ifndef MDEV_H #define MDEV_H +#include +#include + struct mdev_type; struct mdev_device { @@ -20,67 +23,67 @@ struct mdev_device { bool active; }; -static inline struct mdev_device *to_mdev_device(struct device *dev) -{ - return container_of(dev, struct mdev_device, dev); -} +struct mdev_type { + /* set by the driver before calling mdev_register parent: */ + const char *sysfs_name; + const char *pretty_name; -unsigned int mdev_get_type_group_id(struct mdev_device *mdev); -unsigned int mtype_get_type_group_id(struct mdev_type *mtype); -struct device *mtype_get_parent_dev(struct mdev_type *mtype); + /* set by the core, can be used drivers */ + struct mdev_parent *parent; -/* interface for exporting mdev supported type attributes */ -struct mdev_type_attribute { - struct attribute attr; - ssize_t (*show)(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf); - ssize_t (*store)(struct mdev_type *mtype, - struct mdev_type_attribute *attr, const char *buf, - size_t count); + /* internal only */ + struct kobject kobj; + struct kobject *devices_kobj; }; -#define MDEV_TYPE_ATTR(_name, _mode, _show, _store) \ -struct mdev_type_attribute mdev_type_attr_##_name = \ - __ATTR(_name, _mode, _show, _store) -#define MDEV_TYPE_ATTR_RW(_name) \ - struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RW(_name) -#define MDEV_TYPE_ATTR_RO(_name) \ - struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_RO(_name) -#define MDEV_TYPE_ATTR_WO(_name) \ - struct mdev_type_attribute mdev_type_attr_##_name = __ATTR_WO(_name) +/* embedded into the struct device that the mdev devices hang off */ +struct mdev_parent { + struct device *dev; + struct mdev_driver *mdev_driver; + struct kset *mdev_types_kset; + /* Synchronize device creation/removal with parent unregistration */ + struct rw_semaphore unreg_sem; + struct mdev_type **types; + unsigned int nr_types; + atomic_t available_instances; +}; + +static inline struct mdev_device *to_mdev_device(struct device *dev) +{ + return container_of(dev, struct mdev_device, dev); +} /** * struct mdev_driver - Mediated device driver + * @device_api: string to return for the device_api sysfs + * @max_instances: maximum number of instances supported (optional) * @probe: called when new device created * @remove: called when device removed - * @supported_type_groups: Attributes to define supported types. It is mandatory - * to provide supported types. + * @get_available: Return the max number of instances that can be created + * @show_description: Print a description of the mtype * @driver: device driver structure - * **/ struct mdev_driver { + const char *device_api; + unsigned int max_instances; int (*probe)(struct mdev_device *dev); void (*remove)(struct mdev_device *dev); - struct attribute_group **supported_type_groups; + unsigned int (*get_available)(struct mdev_type *mtype); + ssize_t (*show_description)(struct mdev_type *mtype, char *buf); struct device_driver driver; }; -extern struct bus_type mdev_bus_type; - -int mdev_register_device(struct device *dev, struct mdev_driver *mdev_driver); -void mdev_unregister_device(struct device *dev); +int mdev_register_parent(struct mdev_parent *parent, struct device *dev, + struct mdev_driver *mdev_driver, struct mdev_type **types, + unsigned int nr_types); +void mdev_unregister_parent(struct mdev_parent *parent); int mdev_register_driver(struct mdev_driver *drv); void mdev_unregister_driver(struct mdev_driver *drv); -struct device *mdev_parent_dev(struct mdev_device *mdev); static inline struct device *mdev_dev(struct mdev_device *mdev) { return &mdev->dev; } -static inline struct mdev_device *mdev_from_dev(struct device *dev) -{ - return dev->bus == &mdev_bus_type ? to_mdev_device(dev) : NULL; -} #endif /* MDEV_H */ diff --git a/include/linux/mdio/mdio-i2c.h b/include/linux/mdio/mdio-i2c.h index b1d27f7cd23fb09521c3601cd099f86ca9c4fdb3..65b550a6fc32982e03dd54e41b6ea8ec14941cc2 100644 --- a/include/linux/mdio/mdio-i2c.h +++ b/include/linux/mdio/mdio-i2c.h @@ -11,6 +11,14 @@ struct device; struct i2c_adapter; struct mii_bus; -struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c); +enum mdio_i2c_proto { + MDIO_I2C_NONE, + MDIO_I2C_MARVELL_C22, + MDIO_I2C_C45, + MDIO_I2C_ROLLBALL, +}; + +struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c, + enum mdio_i2c_proto protocol); #endif diff --git a/include/linux/mei_aux.h b/include/linux/mei_aux.h index 587f2512884818f1c5c9845c79bd2bfee3957e3e..506912ad363b38ef490fddb0f314c61ee2126e2d 100644 --- a/include/linux/mei_aux.h +++ b/include/linux/mei_aux.h @@ -7,10 +7,22 @@ #include +/** + * struct mei_aux_device - mei auxiliary device + * @aux_dev: - auxiliary device object + * @irq: interrupt driving the mei auxiliary device + * @bar: mmio resource bar reserved to mei auxiliary device + * @ext_op_mem: resource for extend operational memory + * used in graphics PXP mode. + * @slow_firmware: The device has slow underlying firmware. + * Such firmware will require to use larger operation timeouts. + */ struct mei_aux_device { struct auxiliary_device aux_dev; int irq; struct resource bar; + struct resource ext_op_mem; + bool slow_firmware; }; #define auxiliary_dev_to_mei_aux_dev(auxiliary_dev) \ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 4d31ce55b1c0d67f3b5c903f87be449d9953af35..e1644a24009c8c1199acd8ce4b461aa3617e124d 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -80,29 +80,8 @@ enum mem_cgroup_events_target { MEM_CGROUP_NTARGETS, }; -struct memcg_vmstats_percpu { - /* Local (CPU and cgroup) page state & events */ - long state[MEMCG_NR_STAT]; - unsigned long events[NR_VM_EVENT_ITEMS]; - - /* Delta calculation for lockless upward propagation */ - long state_prev[MEMCG_NR_STAT]; - unsigned long events_prev[NR_VM_EVENT_ITEMS]; - - /* Cgroup1: threshold notifications & softlimit tree updates */ - unsigned long nr_page_events; - unsigned long targets[MEM_CGROUP_NTARGETS]; -}; - -struct memcg_vmstats { - /* Aggregated (CPU and subtree) page state & events */ - long state[MEMCG_NR_STAT]; - unsigned long events[NR_VM_EVENT_ITEMS]; - - /* Pending child counts during tree propagation */ - long state_pending[MEMCG_NR_STAT]; - unsigned long events_pending[NR_VM_EVENT_ITEMS]; -}; +struct memcg_vmstats_percpu; +struct memcg_vmstats; struct mem_cgroup_reclaim_iter { struct mem_cgroup *position; @@ -185,15 +164,6 @@ struct mem_cgroup_thresholds { struct mem_cgroup_threshold_ary *spare; }; -#if defined(CONFIG_SMP) -struct memcg_padding { - char x[0]; -} ____cacheline_internodealigned_in_smp; -#define MEMCG_PADDING(name) struct memcg_padding name -#else -#define MEMCG_PADDING(name) -#endif - /* * Remember four most recent foreign writebacks with dirty pages in this * cgroup. Inode sharing is expected to be uncommon and, even if we miss @@ -304,10 +274,10 @@ struct mem_cgroup { spinlock_t move_lock; unsigned long move_lock_flags; - MEMCG_PADDING(_pad1_); + CACHELINE_PADDING(_pad1_); /* memory.stat */ - struct memcg_vmstats vmstats; + struct memcg_vmstats *vmstats; /* memory.events */ atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; @@ -326,7 +296,7 @@ struct mem_cgroup { struct list_head objcg_list; #endif - MEMCG_PADDING(_pad2_); + CACHELINE_PADDING(_pad2_); /* * set > 0 if pages under this cgroup are moving to other cgroup. @@ -350,14 +320,20 @@ struct mem_cgroup { struct deferred_split deferred_split_queue; #endif +#ifdef CONFIG_LRU_GEN + /* per-memcg mm_struct list */ + struct lru_gen_mm_list mm_list; +#endif + struct mem_cgroup_per_node *nodeinfo[]; }; /* - * size of first charge trial. "32" comes from vmscan.c's magic value. - * TODO: maybe necessary to use big numbers in big irons. + * size of first charge trial. + * TODO: maybe necessary to use big numbers in big irons or dynamic based of the + * workload. */ -#define MEMCG_CHARGE_BATCH 32U +#define MEMCG_CHARGE_BATCH 64U extern struct mem_cgroup *root_mem_cgroup; @@ -444,6 +420,7 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() * * For a kmem folio a caller should hold an rcu read lock to protect memcg * associated with a kmem folio from being released. @@ -505,6 +482,7 @@ static inline struct mem_cgroup *folio_memcg_rcu(struct folio *folio) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() * * For a kmem page a caller should hold an rcu read lock to protect memcg * associated with a kmem page from being released. @@ -689,7 +667,7 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, return __mem_cgroup_charge(folio, mm, gfp); } -int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, +int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, gfp_t gfp, swp_entry_t entry); void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry); @@ -959,6 +937,23 @@ void unlock_page_memcg(struct page *page); void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val); +/* try to stablize folio_memcg() for all the pages in a memcg */ +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) +{ + rcu_read_lock(); + + if (mem_cgroup_disabled() || !atomic_read(&memcg->moving_account)) + return true; + + rcu_read_unlock(); + return false; +} + +static inline void mem_cgroup_unlock_pages(void) +{ + rcu_read_unlock(); +} + /* idx can be of type enum memcg_stat_item or node_stat_item */ static inline void mod_memcg_state(struct mem_cgroup *memcg, int idx, int val) @@ -985,21 +980,24 @@ static inline void mod_memcg_page_state(struct page *page, rcu_read_unlock(); } -static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) -{ - return READ_ONCE(memcg->vmstats.state[idx]); -} +unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); static inline unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) { struct mem_cgroup_per_node *pn; + long x; if (mem_cgroup_disabled()) return node_page_state(lruvec_pgdat(lruvec), idx); pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - return READ_ONCE(pn->lruvec_stats.state[idx]); + x = READ_ONCE(pn->lruvec_stats.state[idx]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; } static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, @@ -1227,7 +1225,7 @@ static inline int mem_cgroup_charge(struct folio *folio, return 0; } -static inline int mem_cgroup_swapin_charge_page(struct page *page, +static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, gfp_t gfp, swp_entry_t entry) { return 0; @@ -1422,6 +1420,18 @@ static inline void folio_memcg_unlock(struct folio *folio) { } +static inline bool mem_cgroup_trylock_pages(struct mem_cgroup *memcg) +{ + /* to match folio_memcg_rcu() */ + rcu_read_lock(); + return true; +} + +static inline void mem_cgroup_unlock_pages(void) +{ + rcu_read_unlock(); +} + static inline void mem_cgroup_handle_over_high(void) { } @@ -1768,7 +1778,7 @@ static inline void count_objcg_event(struct obj_cgroup *objcg, { struct mem_cgroup *memcg; - if (mem_cgroup_kmem_disabled()) + if (!memcg_kmem_enabled()) return; rcu_read_lock(); @@ -1777,42 +1787,6 @@ static inline void count_objcg_event(struct obj_cgroup *objcg, rcu_read_unlock(); } -/** - * get_mem_cgroup_from_obj - get a memcg associated with passed kernel object. - * @p: pointer to object from which memcg should be extracted. It can be NULL. - * - * Retrieves the memory group into which the memory of the pointed kernel - * object is accounted. If memcg is found, its reference is taken. - * If a passed kernel object is uncharged, or if proper memcg cannot be found, - * as well as if mem_cgroup is disabled, NULL is returned. - * - * Return: valid memcg pointer with taken reference or NULL. - */ -static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p) -{ - struct mem_cgroup *memcg; - - rcu_read_lock(); - do { - memcg = mem_cgroup_from_obj(p); - } while (memcg && !css_tryget(&memcg->css)); - rcu_read_unlock(); - return memcg; -} - -/** - * mem_cgroup_or_root - always returns a pointer to a valid memory cgroup. - * @memcg: pointer to a valid memory cgroup or NULL. - * - * If passed argument is not NULL, returns it without any additional checks - * and changes. Otherwise, root_mem_cgroup is returned. - * - * NOTE: root_mem_cgroup can be NULL during early boot. - */ -static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg) -{ - return memcg ? memcg : root_mem_cgroup; -} #else static inline bool mem_cgroup_kmem_disabled(void) { @@ -1869,15 +1843,6 @@ static inline void count_objcg_event(struct obj_cgroup *objcg, { } -static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p) -{ - return NULL; -} - -static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg) -{ - return NULL; -} #endif /* CONFIG_MEMCG_KMEM */ #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h new file mode 100644 index 0000000000000000000000000000000000000000..965009aa01d72b73c66d046a0ff4816e376e852b --- /dev/null +++ b/include/linux/memory-tiers.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_MEMORY_TIERS_H +#define _LINUX_MEMORY_TIERS_H + +#include +#include +#include +#include +/* + * Each tier cover a abstrace distance chunk size of 128 + */ +#define MEMTIER_CHUNK_BITS 7 +#define MEMTIER_CHUNK_SIZE (1 << MEMTIER_CHUNK_BITS) +/* + * Smaller abstract distance values imply faster (higher) memory tiers. Offset + * the DRAM adistance so that we can accommodate devices with a slightly lower + * adistance value (slightly faster) than default DRAM adistance to be part of + * the same memory tier. + */ +#define MEMTIER_ADISTANCE_DRAM ((4 * MEMTIER_CHUNK_SIZE) + (MEMTIER_CHUNK_SIZE >> 1)) +#define MEMTIER_HOTPLUG_PRIO 100 + +struct memory_tier; +struct memory_dev_type { + /* list of memory types that are part of same tier as this type */ + struct list_head tier_sibiling; + /* abstract distance for this specific memory type */ + int adistance; + /* Nodes of same abstract distance */ + nodemask_t nodes; + struct kref kref; +}; + +#ifdef CONFIG_NUMA +extern bool numa_demotion_enabled; +struct memory_dev_type *alloc_memory_type(int adistance); +void destroy_memory_type(struct memory_dev_type *memtype); +void init_node_memory_type(int node, struct memory_dev_type *default_type); +void clear_node_memory_type(int node, struct memory_dev_type *memtype); +#ifdef CONFIG_MIGRATION +int next_demotion_node(int node); +void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); +bool node_is_toptier(int node); +#else +static inline int next_demotion_node(int node) +{ + return NUMA_NO_NODE; +} + +static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) +{ + *targets = NODE_MASK_NONE; +} + +static inline bool node_is_toptier(int node) +{ + return true; +} +#endif + +#else + +#define numa_demotion_enabled false +/* + * CONFIG_NUMA implementation returns non NULL error. + */ +static inline struct memory_dev_type *alloc_memory_type(int adistance) +{ + return NULL; +} + +static inline void destroy_memory_type(struct memory_dev_type *memtype) +{ + +} + +static inline void init_node_memory_type(int node, struct memory_dev_type *default_type) +{ + +} + +static inline void clear_node_memory_type(int node, struct memory_dev_type *memtype) +{ + +} + +static inline int next_demotion_node(int node) +{ + return NUMA_NO_NODE; +} + +static inline void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) +{ + *targets = NODE_MASK_NONE; +} + +static inline bool node_is_toptier(int node) +{ + return true; +} +#endif /* CONFIG_NUMA */ +#endif /* _LINUX_MEMORY_TIERS_H */ diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index e0b2209ab71c2411f15e03824b3b0112a82a02fe..9fcbf57065957244d59e6c52a1f4f906fd48d7bd 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -11,7 +11,6 @@ struct page; struct zone; struct pglist_data; struct mem_section; -struct memory_block; struct memory_group; struct resource; struct vmem_altmap; @@ -44,11 +43,6 @@ extern void arch_refresh_nodedata(int nid, pg_data_t *pgdat); ({ \ memblock_alloc(sizeof(*pgdat), SMP_CACHE_BYTES); \ }) -/* - * This definition is just for error path in node hotadd. - * For node hotremove, we have to replace this. - */ -#define generic_free_nodedata(pgdat) kfree(pgdat) extern pg_data_t *node_data[]; static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) @@ -64,9 +58,6 @@ static inline pg_data_t *generic_alloc_nodedata(int nid) BUG(); return NULL; } -static inline void generic_free_nodedata(pg_data_t *pgdat) -{ -} static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) { } @@ -216,6 +207,22 @@ void put_online_mems(void); void mem_hotplug_begin(void); void mem_hotplug_done(void); +/* See kswapd_is_running() */ +static inline void pgdat_kswapd_lock(pg_data_t *pgdat) +{ + mutex_lock(&pgdat->kswapd_lock); +} + +static inline void pgdat_kswapd_unlock(pg_data_t *pgdat) +{ + mutex_unlock(&pgdat->kswapd_lock); +} + +static inline void pgdat_kswapd_lock_init(pg_data_t *pgdat) +{ + mutex_init(&pgdat->kswapd_lock); +} + #else /* ! CONFIG_MEMORY_HOTPLUG */ #define pfn_to_online_page(pfn) \ ({ \ @@ -252,6 +259,10 @@ static inline bool movable_node_is_enabled(void) { return false; } + +static inline void pgdat_kswapd_lock(pg_data_t *pgdat) {} +static inline void pgdat_kswapd_unlock(pg_data_t *pgdat) {} +static inline void pgdat_kswapd_lock_init(pg_data_t *pgdat) {} #endif /* ! CONFIG_MEMORY_HOTPLUG */ /* @@ -333,7 +344,6 @@ extern void move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, extern void remove_pfn_range_from_zone(struct zone *zone, unsigned long start_pfn, unsigned long nr_pages); -extern bool is_memblock_offlined(struct memory_block *mem); extern int sparse_add_section(int nid, unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 668389b4b53d7a9cf58c239a37583714adbdbac9..d232de7cdc569f88adb5e2fe5d978fcaa68b2cc2 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -151,13 +151,6 @@ extern bool mempolicy_in_oom_domain(struct task_struct *tsk, const nodemask_t *mask); extern nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy); -static inline nodemask_t *policy_nodemask_current(gfp_t gfp) -{ - struct mempolicy *mpol = get_task_policy(current); - - return policy_nodemask(gfp, mpol); -} - extern unsigned int mempolicy_slab_node(void); extern enum zone_type policy_zone; @@ -189,6 +182,7 @@ static inline bool mpol_is_preferred_many(struct mempolicy *pol) return (pol->mode == MPOL_PREFERRED_MANY); } +extern bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone); #else @@ -294,11 +288,6 @@ static inline void mpol_put_task_policy(struct task_struct *task) { } -static inline nodemask_t *policy_nodemask_current(gfp_t gfp) -{ - return NULL; -} - static inline bool mpol_is_preferred_many(struct mempolicy *pol) { return false; diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 19010491a60385f4b8c4952f3a335c631e4c7023..7fcaf3180a5b6268f05451c35083148a4cf2fc02 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -139,6 +139,11 @@ struct dev_pagemap { }; }; +static inline bool pgmap_has_memory_failure(struct dev_pagemap *pgmap) +{ + return pgmap->ops && pgmap->ops->memory_failure; +} + static inline struct vmem_altmap *pgmap_altmap(struct dev_pagemap *pgmap) { if (pgmap->flags & PGMAP_ALTMAP_VALID) @@ -182,6 +187,7 @@ static inline bool folio_is_device_coherent(const struct folio *folio) } #ifdef CONFIG_ZONE_DEVICE +void zone_device_page_init(struct page *page); void *memremap_pages(struct dev_pagemap *pgmap, int nid); void memunmap_pages(struct dev_pagemap *pgmap); void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap); diff --git a/include/linux/mfd/ocelot.h b/include/linux/mfd/ocelot.h new file mode 100644 index 0000000000000000000000000000000000000000..dd72073d2d4f1dc9c018edafb42681709190cab1 --- /dev/null +++ b/include/linux/mfd/ocelot.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* Copyright 2022 Innovative Advantage Inc. */ + +#ifndef _LINUX_MFD_OCELOT_H +#define _LINUX_MFD_OCELOT_H + +#include +#include +#include +#include +#include +#include + +struct resource; + +static inline struct regmap * +ocelot_regmap_from_resource_optional(struct platform_device *pdev, + unsigned int index, + const struct regmap_config *config) +{ + struct device *dev = &pdev->dev; + struct resource *res; + void __iomem *regs; + + /* + * Don't use _get_and_ioremap_resource() here, since that will invoke + * prints of "invalid resource" which will simply add confusion. + */ + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (res) { + regs = devm_ioremap_resource(dev, res); + if (IS_ERR(regs)) + return ERR_CAST(regs); + return devm_regmap_init_mmio(dev, regs, config); + } + + /* + * Fall back to using REG and getting the resource from the parent + * device, which is possible in an MFD configuration + */ + if (dev->parent) { + res = platform_get_resource(pdev, IORESOURCE_REG, index); + if (!res) + return NULL; + + return dev_get_regmap(dev->parent, res->name); + } + + return NULL; +} + +static inline struct regmap * +ocelot_regmap_from_resource(struct platform_device *pdev, unsigned int index, + const struct regmap_config *config) +{ + struct regmap *map; + + map = ocelot_regmap_from_resource_optional(pdev, index, config); + return map ?: ERR_PTR(-ENOENT); +} + +#endif diff --git a/include/linux/mfd/rk808.h b/include/linux/mfd/rk808.h index 58602032e642f96879a1ee4e4314bd71469ddf2b..9af1f3105f80782634651e30949c0898db24ec89 100644 --- a/include/linux/mfd/rk808.h +++ b/include/linux/mfd/rk808.h @@ -519,6 +519,77 @@ enum rk809_reg_id { #define MIC_DIFF_DIS (0x0 << 7) #define MIC_DIFF_EN (0x1 << 7) +/* RK817 Battery Registers */ +#define RK817_GAS_GAUGE_ADC_CONFIG0 0x50 +#define RK817_GG_EN (0x1 << 7) +#define RK817_SYS_VOL_ADC_EN (0x1 << 6) +#define RK817_TS_ADC_EN (0x1 << 5) +#define RK817_USB_VOL_ADC_EN (0x1 << 4) +#define RK817_BAT_VOL_ADC_EN (0x1 << 3) +#define RK817_BAT_CUR_ADC_EN (0x1 << 2) + +#define RK817_GAS_GAUGE_ADC_CONFIG1 0x55 + +#define RK817_VOL_CUR_CALIB_UPD BIT(7) + +#define RK817_GAS_GAUGE_GG_CON 0x56 +#define RK817_GAS_GAUGE_GG_STS 0x57 + +#define RK817_BAT_CON (0x1 << 4) +#define RK817_RELAX_VOL_UPD (0x3 << 2) +#define RK817_RELAX_STS (0x1 << 1) + +#define RK817_GAS_GAUGE_RELAX_THRE_H 0x58 +#define RK817_GAS_GAUGE_RELAX_THRE_L 0x59 +#define RK817_GAS_GAUGE_OCV_THRE_VOL 0x62 +#define RK817_GAS_GAUGE_OCV_VOL_H 0x63 +#define RK817_GAS_GAUGE_OCV_VOL_L 0x64 +#define RK817_GAS_GAUGE_PWRON_VOL_H 0x6b +#define RK817_GAS_GAUGE_PWRON_VOL_L 0x6c +#define RK817_GAS_GAUGE_PWRON_CUR_H 0x6d +#define RK817_GAS_GAUGE_PWRON_CUR_L 0x6e +#define RK817_GAS_GAUGE_OFF_CNT 0x6f +#define RK817_GAS_GAUGE_Q_INIT_H3 0x70 +#define RK817_GAS_GAUGE_Q_INIT_H2 0x71 +#define RK817_GAS_GAUGE_Q_INIT_L1 0x72 +#define RK817_GAS_GAUGE_Q_INIT_L0 0x73 +#define RK817_GAS_GAUGE_Q_PRES_H3 0x74 +#define RK817_GAS_GAUGE_Q_PRES_H2 0x75 +#define RK817_GAS_GAUGE_Q_PRES_L1 0x76 +#define RK817_GAS_GAUGE_Q_PRES_L0 0x77 +#define RK817_GAS_GAUGE_BAT_VOL_H 0x78 +#define RK817_GAS_GAUGE_BAT_VOL_L 0x79 +#define RK817_GAS_GAUGE_BAT_CUR_H 0x7a +#define RK817_GAS_GAUGE_BAT_CUR_L 0x7b +#define RK817_GAS_GAUGE_USB_VOL_H 0x7e +#define RK817_GAS_GAUGE_USB_VOL_L 0x7f +#define RK817_GAS_GAUGE_SYS_VOL_H 0x80 +#define RK817_GAS_GAUGE_SYS_VOL_L 0x81 +#define RK817_GAS_GAUGE_Q_MAX_H3 0x82 +#define RK817_GAS_GAUGE_Q_MAX_H2 0x83 +#define RK817_GAS_GAUGE_Q_MAX_L1 0x84 +#define RK817_GAS_GAUGE_Q_MAX_L0 0x85 +#define RK817_GAS_GAUGE_SLEEP_CON_SAMP_CUR_H 0x8f +#define RK817_GAS_GAUGE_SLEEP_CON_SAMP_CUR_L 0x90 +#define RK817_GAS_GAUGE_CAL_OFFSET_H 0x91 +#define RK817_GAS_GAUGE_CAL_OFFSET_L 0x92 +#define RK817_GAS_GAUGE_VCALIB0_H 0x93 +#define RK817_GAS_GAUGE_VCALIB0_L 0x94 +#define RK817_GAS_GAUGE_VCALIB1_H 0x95 +#define RK817_GAS_GAUGE_VCALIB1_L 0x96 +#define RK817_GAS_GAUGE_IOFFSET_H 0x97 +#define RK817_GAS_GAUGE_IOFFSET_L 0x98 +#define RK817_GAS_GAUGE_BAT_R1 0x9a +#define RK817_GAS_GAUGE_BAT_R2 0x9b +#define RK817_GAS_GAUGE_BAT_R3 0x9c +#define RK817_GAS_GAUGE_DATA0 0x9d +#define RK817_GAS_GAUGE_DATA1 0x9e +#define RK817_GAS_GAUGE_DATA2 0x9f +#define RK817_GAS_GAUGE_DATA3 0xa0 +#define RK817_GAS_GAUGE_DATA4 0xa1 +#define RK817_GAS_GAUGE_DATA5 0xa2 +#define RK817_GAS_GAUGE_CUR_ADC_K0 0xb0 + #define RK817_POWER_EN_REG(i) (0xb1 + (i)) #define RK817_POWER_SLP_EN_REG(i) (0xb5 + (i)) @@ -544,10 +615,30 @@ enum rk809_reg_id { #define RK817_LDO_ON_VSEL_REG(idx) (0xcc + (idx) * 2) #define RK817_BOOST_OTG_CFG (0xde) +#define RK817_PMIC_CHRG_OUT 0xe4 +#define RK817_CHRG_VOL_SEL (0x07 << 4) +#define RK817_CHRG_CUR_SEL (0x07 << 0) + +#define RK817_PMIC_CHRG_IN 0xe5 +#define RK817_USB_VLIM_EN (0x01 << 7) +#define RK817_USB_VLIM_SEL (0x07 << 4) +#define RK817_USB_ILIM_EN (0x01 << 3) +#define RK817_USB_ILIM_SEL (0x07 << 0) +#define RK817_PMIC_CHRG_TERM 0xe6 +#define RK817_CHRG_TERM_ANA_DIG (0x01 << 2) +#define RK817_CHRG_TERM_ANA_SEL (0x03 << 0) +#define RK817_CHRG_EN (0x01 << 6) + +#define RK817_PMIC_CHRG_STS 0xeb +#define RK817_BAT_EXS BIT(7) +#define RK817_CHG_STS (0x07 << 4) + #define RK817_ID_MSB 0xed #define RK817_ID_LSB 0xee #define RK817_SYS_STS 0xf0 +#define RK817_PLUG_IN_STS (0x1 << 6) + #define RK817_SYS_CFG(i) (0xf1 + (i)) #define RK817_ON_SOURCE_REG 0xf5 diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 22c0a0cf5e0c7bb9e533683db323cc2d57be8a2d..3ef77f52a4f042d5c3370b0b7875f3bc30f0dd12 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -62,6 +62,8 @@ extern const char *migrate_reason_names[MR_TYPES]; #ifdef CONFIG_MIGRATION extern void putback_movable_pages(struct list_head *l); +int migrate_folio_extra(struct address_space *mapping, struct folio *dst, + struct folio *src, enum migrate_mode mode, int extra_count); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free, @@ -100,21 +102,6 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping, #endif /* CONFIG_MIGRATION */ -#if defined(CONFIG_MIGRATION) && defined(CONFIG_NUMA) -extern void set_migration_target_nodes(void); -extern void migrate_on_reclaim_init(void); -extern bool numa_demotion_enabled; -extern int next_demotion_node(int node); -#else -static inline void set_migration_target_nodes(void) {} -static inline void migrate_on_reclaim_init(void) {} -static inline int next_demotion_node(int node) -{ - return NUMA_NO_NODE; -} -#define numa_demotion_enabled false -#endif - #ifdef CONFIG_COMPACTION bool PageMovable(struct page *page); void __SetPageMovable(struct page *page, const struct movable_operations *ops); @@ -212,11 +199,24 @@ struct migrate_vma { */ void *pgmap_owner; unsigned long flags; + + /* + * Set to vmf->page if this is being called to migrate a page as part of + * a migrate_to_ram() callback. + */ + struct page *fault_page; }; int migrate_vma_setup(struct migrate_vma *args); void migrate_vma_pages(struct migrate_vma *migrate); void migrate_vma_finalize(struct migrate_vma *migrate); +int migrate_device_range(unsigned long *src_pfns, unsigned long start, + unsigned long npages); +void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns, + unsigned long npages); +void migrate_device_finalize(unsigned long *src_pfns, + unsigned long *dst_pfns, unsigned long npages); + #endif /* CONFIG_MIGRATION */ #endif /* _LINUX_MIGRATE_H */ diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index b5f58fd37a0f75905a86a6253861e1bd05bfb9e7..1ff91cb79ded5be688ab8d67b3e8977ca660fce2 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -325,6 +325,7 @@ enum mlx5_event { MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, MLX5_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, MLX5_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, + MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27, MLX5_EVENT_TYPE_INTERNAL_ERROR = 0x08, MLX5_EVENT_TYPE_PORT_CHANGE = 0x09, @@ -699,6 +700,12 @@ struct mlx5_eqe_temp_warning { __be64 sensor_warning_lsb; } __packed; +struct mlx5_eqe_obj_change { + u8 rsvd0[2]; + __be16 obj_type; + __be32 obj_id; +} __packed; + #define SYNC_RST_STATE_MASK 0xf enum sync_rst_state_type { @@ -737,6 +744,7 @@ union ev_data { struct mlx5_eqe_xrq_err xrq_err; struct mlx5_eqe_sync_fw_update sync_fw_update; struct mlx5_eqe_vhca_state vhca_state; + struct mlx5_eqe_obj_change obj_change; } __packed; struct mlx5_eqe { @@ -874,12 +882,6 @@ static inline u8 get_cqe_opcode(struct mlx5_cqe64 *cqe) return cqe->op_own >> 4; } -static inline u8 get_cqe_enhanced_num_mini_cqes(struct mlx5_cqe64 *cqe) -{ - /* num_of_mini_cqes is zero based */ - return get_cqe_opcode(cqe) + 1; -} - static inline u8 get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe) { return (cqe->lro.tcppsh_abort_dupack >> 6) & 1; @@ -890,11 +892,6 @@ static inline u8 get_cqe_l4_hdr_type(struct mlx5_cqe64 *cqe) return (cqe->l4_l3_hdr_type >> 4) & 0x7; } -static inline u8 get_cqe_l3_hdr_type(struct mlx5_cqe64 *cqe) -{ - return (cqe->l4_l3_hdr_type >> 2) & 0x3; -} - static inline bool cqe_is_tunneled(struct mlx5_cqe64 *cqe) { return cqe->tls_outer_l3_tunneled & 0x1; @@ -1198,8 +1195,10 @@ enum mlx5_cap_type { MLX5_CAP_DEV_EVENT = 0x14, MLX5_CAP_IPSEC, MLX5_CAP_DEV_SHAMPO = 0x1d, + MLX5_CAP_MACSEC = 0x1f, MLX5_CAP_GENERAL_2 = 0x20, MLX5_CAP_PORT_SELECTION = 0x25, + MLX5_CAP_ADV_VIRTUALIZATION = 0x26, /* NUM OF CAP Types */ MLX5_CAP_NUM }; @@ -1365,6 +1364,14 @@ enum mlx5_qcam_feature_groups { MLX5_GET(port_selection_cap, \ mdev->caps.hca[MLX5_CAP_PORT_SELECTION]->max, cap) +#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \ + MLX5_GET(adv_virtualization_cap, \ + mdev->caps.hca[MLX5_CAP_ADV_VIRTUALIZATION]->cur, cap) + +#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \ + MLX5_GET(adv_virtualization_cap, \ + mdev->caps.hca[MLX5_CAP_ADV_VIRTUALIZATION]->max, cap) + #define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \ MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap) @@ -1446,6 +1453,9 @@ enum mlx5_qcam_feature_groups { #define MLX5_CAP_DEV_SHAMPO(mdev, cap)\ MLX5_GET(shampo_cap, mdev->caps.hca_cur[MLX5_CAP_DEV_SHAMPO], cap) +#define MLX5_CAP_MACSEC(mdev, cap)\ + MLX5_GET(macsec_cap, (mdev)->caps.hca[MLX5_CAP_MACSEC]->cur, cap) + enum { MLX5_CMD_STAT_OK = 0x0, MLX5_CMD_STAT_INT_ERR = 0x1, diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 96b16fbe1aa45932b52158da10007956fe081fcc..a12929bc31b2260b10668acb7f198889428c2d90 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -698,6 +698,8 @@ struct mlx5_pps { struct work_struct out_work; u64 start[MAX_PIN_NUM]; u8 enabled; + u64 min_npps_period; + u64 min_out_pulse_duration_ns; }; struct mlx5_timer { @@ -779,6 +781,7 @@ struct mlx5_core_dev { enum mlx5_device_state state; /* sync interface state */ struct mutex intf_state_mutex; + struct lock_class_key lock_key; unsigned long intf_state; struct mlx5_priv priv; struct mlx5_profile profile; @@ -854,11 +857,6 @@ struct mlx5_cmd_work_ent { refcount_t refcnt; }; -struct mlx5_pas { - u64 pa; - u8 log_sz; -}; - enum phy_port_state { MLX5_AAA_111 }; @@ -1015,11 +1013,11 @@ int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size, bool mlx5_cmd_is_down(struct mlx5_core_dev *dev); int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type); -void mlx5_health_flush(struct mlx5_core_dev *dev); void mlx5_health_cleanup(struct mlx5_core_dev *dev); int mlx5_health_init(struct mlx5_core_dev *dev); void mlx5_start_health_poll(struct mlx5_core_dev *dev); void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); +void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev); void mlx5_drain_health_wq(struct mlx5_core_dev *dev); void mlx5_trigger_health_work(struct mlx5_core_dev *dev); int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size, @@ -1084,8 +1082,6 @@ int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num); void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common); int mlx5_query_odp_caps(struct mlx5_core_dev *dev, struct mlx5_odp_caps *odp_caps); -int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev, - u8 port_num, void *out, size_t sz); int mlx5_init_rl_table(struct mlx5_core_dev *dev); void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev); @@ -1152,6 +1148,7 @@ int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev); bool mlx5_lag_is_roce(struct mlx5_core_dev *dev); bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev); bool mlx5_lag_is_active(struct mlx5_core_dev *dev); +bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev); bool mlx5_lag_is_master(struct mlx5_core_dev *dev); bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev); struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev); @@ -1279,16 +1276,21 @@ enum { MLX5_TRIGGERED_CMD_COMP = (u64)1 << 32, }; -static inline bool mlx5_is_roce_init_enabled(struct mlx5_core_dev *dev) +bool mlx5_is_roce_on(struct mlx5_core_dev *dev); + +static inline bool mlx5_get_roce_state(struct mlx5_core_dev *dev) { - struct devlink *devlink = priv_to_devlink(dev); - union devlink_param_value val; - int err; - - err = devlink_param_driverinit_value_get(devlink, - DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, - &val); - return err ? MLX5_CAP_GEN(dev, roce) : val.vbool; + if (MLX5_CAP_GEN(dev, roce_rw_supported)) + return MLX5_CAP_GEN(dev, roce); + + /* If RoCE cap is read-only in FW, get RoCE state from devlink + * in order to support RoCE enable/disable feature + */ + return mlx5_is_roce_on(dev); } +enum { + MLX5_OCTWORD = 16, +}; + #endif /* MLX5_DRIVER_H */ diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 8e73c377da2c698d056a49273427e164f7a10bb1..c7a91981cd5af105481855e27bf07d3b3247c110 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -79,6 +79,7 @@ static inline void build_leftovers_ft_param(int *priority, enum mlx5_flow_namespace_type { MLX5_FLOW_NAMESPACE_BYPASS, + MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, MLX5_FLOW_NAMESPACE_LAG, MLX5_FLOW_NAMESPACE_OFFLOADS, MLX5_FLOW_NAMESPACE_ETHTOOL, @@ -92,7 +93,8 @@ enum mlx5_flow_namespace_type { MLX5_FLOW_NAMESPACE_SNIFFER_RX, MLX5_FLOW_NAMESPACE_SNIFFER_TX, MLX5_FLOW_NAMESPACE_EGRESS, - MLX5_FLOW_NAMESPACE_EGRESS_KERNEL, + MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, + MLX5_FLOW_NAMESPACE_EGRESS_MACSEC, MLX5_FLOW_NAMESPACE_RDMA_RX, MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL, MLX5_FLOW_NAMESPACE_RDMA_TX, @@ -243,10 +245,10 @@ struct mlx5_flow_act { u32 action; struct mlx5_modify_hdr *modify_hdr; struct mlx5_pkt_reformat *pkt_reformat; - union { - u32 ipsec_obj_id; - uintptr_t esp_id; - }; + struct mlx5_flow_act_crypto_params { + u8 type; + u32 obj_id; + } crypto; u32 flags; struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH]; struct ib_counters *counters; diff --git a/include/linux/mlx5/fs_helpers.h b/include/linux/mlx5/fs_helpers.h index 9db21cd0e92c4a1f334f4a8d7c90b4a29ecee7fa..bc5125bc0561176a93308d146ed6e50b5351deed 100644 --- a/include/linux/mlx5/fs_helpers.h +++ b/include/linux/mlx5/fs_helpers.h @@ -38,46 +38,6 @@ #define MLX5_FS_IPV4_VERSION 4 #define MLX5_FS_IPV6_VERSION 6 -static inline bool mlx5_fs_is_ipsec_flow(const u32 *match_c) -{ - void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, - misc_parameters); - - return MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi); -} - -static inline bool _mlx5_fs_is_outer_ipproto_flow(const u32 *match_c, - const u32 *match_v, u8 match) -{ - const void *headers_c = MLX5_ADDR_OF(fte_match_param, match_c, - outer_headers); - const void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, - outer_headers); - - return MLX5_GET(fte_match_set_lyr_2_4, headers_c, ip_protocol) == 0xff && - MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol) == match; -} - -static inline bool mlx5_fs_is_outer_tcp_flow(const u32 *match_c, - const u32 *match_v) -{ - return _mlx5_fs_is_outer_ipproto_flow(match_c, match_v, IPPROTO_TCP); -} - -static inline bool mlx5_fs_is_outer_udp_flow(const u32 *match_c, - const u32 *match_v) -{ - return _mlx5_fs_is_outer_ipproto_flow(match_c, match_v, IPPROTO_UDP); -} - -static inline bool mlx5_fs_is_vxlan_flow(const u32 *match_c) -{ - void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c, - misc_parameters); - - return MLX5_GET(fte_match_set_misc, misc_params_c, vxlan_vni); -} - static inline bool _mlx5_fs_is_outer_ipv_flow(struct mlx5_core_dev *mdev, const u32 *match_c, const u32 *match_v, int version) @@ -131,12 +91,4 @@ mlx5_fs_is_outer_ipv6_flow(struct mlx5_core_dev *mdev, const u32 *match_c, MLX5_FS_IPV6_VERSION); } -static inline bool mlx5_fs_is_outer_ipsec_flow(const u32 *match_c) -{ - void *misc_params_c = - MLX5_ADDR_OF(fte_match_param, match_c, misc_parameters); - - return MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi); -} - #endif diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 4acd5610e96bc092727547fda799125542c4bc0e..5a4e914e2a6ff39b9aa38ec0054dae7186f70fa3 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -68,6 +68,7 @@ enum { MLX5_SET_HCA_CAP_OP_MOD_ODP = 0x2, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC = 0x3, MLX5_SET_HCA_CAP_OP_MOD_ROCE = 0x4, + MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION = 0x25, }; enum { @@ -82,6 +83,7 @@ enum { MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM = (1ULL << MLX5_OBJ_TYPE_SW_ICM), MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT = (1ULL << 11), MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q = (1ULL << 13), + MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD = (1ULL << 39), }; enum { @@ -89,6 +91,7 @@ enum { MLX5_OBJ_TYPE_VIRTIO_NET_Q = 0x000d, MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c, MLX5_OBJ_TYPE_MATCH_DEFINER = 0x0018, + MLX5_OBJ_TYPE_PAGE_TRACK = 0x46, MLX5_OBJ_TYPE_MKEY = 0xff01, MLX5_OBJ_TYPE_QP = 0xff02, MLX5_OBJ_TYPE_PSV = 0xff03, @@ -449,7 +452,12 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 reserved_at_60[0x2]; u8 reformat_insert[0x1]; u8 reformat_remove[0x1]; - u8 reserver_at_64[0x14]; + u8 macsec_encrypt[0x1]; + u8 macsec_decrypt[0x1]; + u8 reserved_at_66[0x2]; + u8 reformat_add_macsec[0x1]; + u8 reformat_remove_macsec[0x1]; + u8 reserved_at_6a[0xe]; u8 log_max_ft_num[0x8]; u8 reserved_at_80[0x10]; @@ -476,6 +484,22 @@ struct mlx5_ifc_odp_per_transport_service_cap_bits { u8 reserved_at_6[0x1a]; }; +struct mlx5_ifc_ipv4_layout_bits { + u8 reserved_at_0[0x60]; + + u8 ipv4[0x20]; +}; + +struct mlx5_ifc_ipv6_layout_bits { + u8 ipv6[16][0x8]; +}; + +union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits { + struct mlx5_ifc_ipv6_layout_bits ipv6_layout; + struct mlx5_ifc_ipv4_layout_bits ipv4_layout; + u8 reserved_at_0[0x80]; +}; + struct mlx5_ifc_fte_match_set_lyr_2_4_bits { u8 smac_47_16[0x20]; @@ -611,7 +635,11 @@ struct mlx5_ifc_fte_match_set_misc2_bits { u8 metadata_reg_a[0x20]; - u8 reserved_at_1a0[0x60]; + u8 reserved_at_1a0[0x8]; + + u8 macsec_syndrome[0x8]; + + u8 reserved_at_1b0[0x50]; }; struct mlx5_ifc_fte_match_set_misc3_bits { @@ -813,7 +841,9 @@ struct mlx5_ifc_flow_table_nic_cap_bits { struct mlx5_ifc_port_selection_cap_bits { u8 reserved_at_0[0x10]; u8 port_select_flow_table[0x1]; - u8 reserved_at_11[0xf]; + u8 reserved_at_11[0x1]; + u8 port_select_flow_table_bypass[0x1]; + u8 reserved_at_13[0xd]; u8 reserved_at_20[0x1e0]; @@ -1276,6 +1306,24 @@ struct mlx5_ifc_ipsec_cap_bits { u8 reserved_at_30[0x7d0]; }; +struct mlx5_ifc_macsec_cap_bits { + u8 macsec_epn[0x1]; + u8 reserved_at_1[0x2]; + u8 macsec_crypto_esp_aes_gcm_256_encrypt[0x1]; + u8 macsec_crypto_esp_aes_gcm_128_encrypt[0x1]; + u8 macsec_crypto_esp_aes_gcm_256_decrypt[0x1]; + u8 macsec_crypto_esp_aes_gcm_128_decrypt[0x1]; + u8 reserved_at_7[0x4]; + u8 log_max_macsec_offload[0x5]; + u8 reserved_at_10[0x10]; + + u8 min_log_macsec_full_replay_window[0x8]; + u8 max_log_macsec_full_replay_window[0x8]; + u8 reserved_at_30[0x10]; + + u8 reserved_at_40[0x7c0]; +}; + enum { MLX5_WQ_TYPE_LINKED_LIST = 0x0, MLX5_WQ_TYPE_CYCLIC = 0x1, @@ -1443,7 +1491,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_120[0xa]; u8 log_max_ra_req_dc[0x6]; - u8 reserved_at_130[0x9]; + u8 reserved_at_130[0x2]; + u8 eth_wqe_too_small[0x1]; + u8 reserved_at_133[0x6]; u8 vnic_env_cq_overrun[0x1]; u8 log_max_ra_res_dc[0x6]; @@ -1707,7 +1757,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 steering_format_version[0x4]; u8 create_qp_start_hint[0x18]; - u8 reserved_at_460[0x3]; + u8 reserved_at_460[0x1]; + u8 ats[0x1]; + u8 reserved_at_462[0x1]; u8 log_max_uctx[0x5]; u8 reserved_at_468[0x2]; u8 ipsec_offload[0x1]; @@ -1733,7 +1785,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 max_geneve_tlv_options[0x8]; u8 reserved_at_568[0x3]; u8 max_geneve_tlv_option_data_len[0x5]; - u8 reserved_at_570[0x10]; + u8 reserved_at_570[0x9]; + u8 adv_virtualization[0x1]; + u8 reserved_at_57a[0x6]; u8 reserved_at_580[0xb]; u8 log_max_dci_stream_channels[0x5]; @@ -1828,7 +1882,13 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 max_reformat_remove_size[0x8]; u8 max_reformat_remove_offset[0x8]; - u8 reserved_at_c0[0x160]; + u8 reserved_at_c0[0xe0]; + + u8 reserved_at_1a0[0xb]; + u8 log_min_mkey_entity_size[0x5]; + u8 reserved_at_1b0[0x10]; + + u8 reserved_at_1c0[0x60]; u8 reserved_at_220[0x1]; u8 sw_vhca_id_valid[0x1]; @@ -3295,6 +3355,7 @@ union mlx5_ifc_hca_cap_union_bits { struct mlx5_ifc_device_mem_cap_bits device_mem_cap; struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap; struct mlx5_ifc_shampo_cap_bits shampo_cap; + struct mlx5_ifc_macsec_cap_bits macsec_cap; u8 reserved_at_0[0x8000]; }; @@ -3310,8 +3371,8 @@ enum { MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH = 0x100, MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2 = 0x400, MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800, - MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT = 0x1000, - MLX5_FLOW_CONTEXT_ACTION_IPSEC_ENCRYPT = 0x2000, + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT = 0x1000, + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT = 0x2000, MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000, }; @@ -3321,6 +3382,11 @@ enum { MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT = 0x2, }; +enum { + MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC = 0x0, + MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC = 0x1, +}; + struct mlx5_ifc_vlan_bits { u8 ethtype[0x10]; u8 prio[0x3]; @@ -3374,7 +3440,7 @@ struct mlx5_ifc_flow_context_bits { u8 extended_destination[0x1]; u8 reserved_at_81[0x1]; u8 flow_source[0x2]; - u8 reserved_at_84[0x4]; + u8 encrypt_decrypt_type[0x4]; u8 destination_list_size[0x18]; u8 reserved_at_a0[0x8]; @@ -3386,7 +3452,7 @@ struct mlx5_ifc_flow_context_bits { struct mlx5_ifc_vlan_bits push_vlan_2; - u8 ipsec_obj_id[0x20]; + u8 encrypt_decrypt_obj_id[0x20]; u8 reserved_at_140[0xc0]; struct mlx5_ifc_fte_match_param_bits match_value; @@ -3475,7 +3541,9 @@ struct mlx5_ifc_vnic_diagnostic_statistics_bits { u8 cq_overrun[0x20]; - u8 reserved_at_220[0xde0]; + u8 eth_wqe_too_small[0x20]; + + u8 reserved_at_220[0xdc0]; }; struct mlx5_ifc_traffic_counter_bits { @@ -3873,7 +3941,9 @@ struct mlx5_ifc_mkc_bits { u8 lw[0x1]; u8 lr[0x1]; u8 access_mode_1_0[0x2]; - u8 reserved_at_18[0x8]; + u8 reserved_at_18[0x2]; + u8 ma_translation_mode[0x2]; + u8 reserved_at_1c[0x4]; u8 qpn[0x18]; u8 mkey_7_0[0x8]; @@ -6316,6 +6386,8 @@ enum mlx5_reformat_ctx_type { MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4, MLX5_REFORMAT_TYPE_INSERT_HDR = 0xf, MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10, + MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11, + MLX5_REFORMAT_TYPE_DEL_MACSEC = 0x12, }; struct mlx5_ifc_alloc_packet_reformat_context_in_bits { @@ -9789,7 +9861,9 @@ struct mlx5_ifc_pcam_reg_bits { struct mlx5_ifc_mcam_enhanced_features_bits { u8 reserved_at_0[0x5d]; u8 mcia_32dwords[0x1]; - u8 reserved_at_5e[0xc]; + u8 out_pulse_duration_ns[0x1]; + u8 npps_period[0x1]; + u8 reserved_at_60[0xa]; u8 reset_state[0x1]; u8 ptpcyc2realtime_modify[0x1]; u8 reserved_at_6c[0x2]; @@ -10289,7 +10363,12 @@ struct mlx5_ifc_mtpps_reg_bits { u8 reserved_at_18[0x4]; u8 cap_max_num_of_pps_out_pins[0x4]; - u8 reserved_at_20[0x24]; + u8 reserved_at_20[0x13]; + u8 cap_log_min_npps_period[0x5]; + u8 reserved_at_38[0x3]; + u8 cap_log_min_out_pulse_duration_ns[0x5]; + + u8 reserved_at_40[0x4]; u8 cap_pin_3_mode[0x4]; u8 reserved_at_48[0x4]; u8 cap_pin_2_mode[0x4]; @@ -10308,7 +10387,9 @@ struct mlx5_ifc_mtpps_reg_bits { u8 cap_pin_4_mode[0x4]; u8 field_select[0x20]; - u8 reserved_at_a0[0x60]; + u8 reserved_at_a0[0x20]; + + u8 npps_period[0x40]; u8 enable[0x1]; u8 reserved_at_101[0xb]; @@ -10317,7 +10398,8 @@ struct mlx5_ifc_mtpps_reg_bits { u8 pin_mode[0x4]; u8 pin[0x8]; - u8 reserved_at_120[0x20]; + u8 reserved_at_120[0x2]; + u8 out_pulse_duration_ns[0x1e]; u8 time_stamp[0x40]; @@ -10920,7 +11002,9 @@ struct mlx5_ifc_lagc_bits { u8 reserved_at_18[0x5]; u8 lag_state[0x3]; - u8 reserved_at_20[0x14]; + u8 reserved_at_20[0xc]; + u8 active_port[0x4]; + u8 reserved_at_30[0x4]; u8 tx_remap_affinity_2[0x4]; u8 reserved_at_38[0x4]; u8 tx_remap_affinity_1[0x4]; @@ -11134,7 +11218,8 @@ struct mlx5_ifc_dealloc_memic_out_bits { struct mlx5_ifc_umem_bits { u8 reserved_at_0[0x80]; - u8 reserved_at_80[0x1b]; + u8 ats[0x1]; + u8 reserved_at_81[0x1a]; u8 log_page_size[0x5]; u8 page_offset[0x20]; @@ -11471,6 +11556,7 @@ enum { MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13, MLX5_GENERAL_OBJECT_TYPES_SAMPLER = 0x20, MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO = 0x24, + MLX5_GENERAL_OBJECT_TYPES_MACSEC = 0x27, }; enum { @@ -11521,6 +11607,96 @@ struct mlx5_ifc_modify_ipsec_obj_in_bits { struct mlx5_ifc_ipsec_obj_bits ipsec_object; }; +enum { + MLX5_MACSEC_ASO_REPLAY_PROTECTION = 0x1, +}; + +enum { + MLX5_MACSEC_ASO_REPLAY_WIN_32BIT = 0x0, + MLX5_MACSEC_ASO_REPLAY_WIN_64BIT = 0x1, + MLX5_MACSEC_ASO_REPLAY_WIN_128BIT = 0x2, + MLX5_MACSEC_ASO_REPLAY_WIN_256BIT = 0x3, +}; + +#define MLX5_MACSEC_ASO_INC_SN 0x2 +#define MLX5_MACSEC_ASO_REG_C_4_5 0x2 + +struct mlx5_ifc_macsec_aso_bits { + u8 valid[0x1]; + u8 reserved_at_1[0x1]; + u8 mode[0x2]; + u8 window_size[0x2]; + u8 soft_lifetime_arm[0x1]; + u8 hard_lifetime_arm[0x1]; + u8 remove_flow_enable[0x1]; + u8 epn_event_arm[0x1]; + u8 reserved_at_a[0x16]; + + u8 remove_flow_packet_count[0x20]; + + u8 remove_flow_soft_lifetime[0x20]; + + u8 reserved_at_60[0x80]; + + u8 mode_parameter[0x20]; + + u8 replay_protection_window[8][0x20]; +}; + +struct mlx5_ifc_macsec_offload_obj_bits { + u8 modify_field_select[0x40]; + + u8 confidentiality_en[0x1]; + u8 reserved_at_41[0x1]; + u8 epn_en[0x1]; + u8 epn_overlap[0x1]; + u8 reserved_at_44[0x2]; + u8 confidentiality_offset[0x2]; + u8 reserved_at_48[0x4]; + u8 aso_return_reg[0x4]; + u8 reserved_at_50[0x10]; + + u8 epn_msb[0x20]; + + u8 reserved_at_80[0x8]; + u8 dekn[0x18]; + + u8 reserved_at_a0[0x20]; + + u8 sci[0x40]; + + u8 reserved_at_100[0x8]; + u8 macsec_aso_access_pd[0x18]; + + u8 reserved_at_120[0x60]; + + u8 salt[3][0x20]; + + u8 reserved_at_1e0[0x20]; + + struct mlx5_ifc_macsec_aso_bits macsec_aso; +}; + +struct mlx5_ifc_create_macsec_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_macsec_offload_obj_bits macsec_object; +}; + +struct mlx5_ifc_modify_macsec_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_macsec_offload_obj_bits macsec_object; +}; + +enum { + MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP = BIT(0), + MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB = BIT(1), +}; + +struct mlx5_ifc_query_macsec_obj_out_bits { + struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr; + struct mlx5_ifc_macsec_offload_obj_bits macsec_object; +}; + struct mlx5_ifc_encryption_key_obj_bits { u8 modify_field_select[0x40]; @@ -11638,6 +11814,7 @@ enum { enum { MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1, MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC = 0x4, }; struct mlx5_ifc_tls_static_params_bits { @@ -11818,4 +11995,82 @@ struct mlx5_ifc_load_vhca_state_out_bits { u8 reserved_at_40[0x40]; }; +struct mlx5_ifc_adv_virtualization_cap_bits { + u8 reserved_at_0[0x3]; + u8 pg_track_log_max_num[0x5]; + u8 pg_track_max_num_range[0x8]; + u8 pg_track_log_min_addr_space[0x8]; + u8 pg_track_log_max_addr_space[0x8]; + + u8 reserved_at_20[0x3]; + u8 pg_track_log_min_msg_size[0x5]; + u8 reserved_at_28[0x3]; + u8 pg_track_log_max_msg_size[0x5]; + u8 reserved_at_30[0x3]; + u8 pg_track_log_min_page_size[0x5]; + u8 reserved_at_38[0x3]; + u8 pg_track_log_max_page_size[0x5]; + + u8 reserved_at_40[0x7c0]; +}; + +struct mlx5_ifc_page_track_report_entry_bits { + u8 dirty_address_high[0x20]; + + u8 dirty_address_low[0x20]; +}; + +enum { + MLX5_PAGE_TRACK_STATE_TRACKING, + MLX5_PAGE_TRACK_STATE_REPORTING, + MLX5_PAGE_TRACK_STATE_ERROR, +}; + +struct mlx5_ifc_page_track_range_bits { + u8 start_address[0x40]; + + u8 length[0x40]; +}; + +struct mlx5_ifc_page_track_bits { + u8 modify_field_select[0x40]; + + u8 reserved_at_40[0x10]; + u8 vhca_id[0x10]; + + u8 reserved_at_60[0x20]; + + u8 state[0x4]; + u8 track_type[0x4]; + u8 log_addr_space_size[0x8]; + u8 reserved_at_90[0x3]; + u8 log_page_size[0x5]; + u8 reserved_at_98[0x3]; + u8 log_msg_size[0x5]; + + u8 reserved_at_a0[0x8]; + u8 reporting_qpn[0x18]; + + u8 reserved_at_c0[0x18]; + u8 num_ranges[0x8]; + + u8 reserved_at_e0[0x20]; + + u8 range_start_address[0x40]; + + u8 length[0x40]; + + struct mlx5_ifc_page_track_range_bits track_range[0]; +}; + +struct mlx5_ifc_create_page_track_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_page_track_bits obj_context; +}; + +struct mlx5_ifc_modify_page_track_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_page_track_bits obj_context; +}; + #endif /* MLX5_IFC_H */ diff --git a/include/linux/mlx5/mlx5_ifc_fpga.h b/include/linux/mlx5/mlx5_ifc_fpga.h index 45c7c0d6763562cddbf57612a384110f1c6682c9..0596472923ad8717ee5d0594102d53e22f62c209 100644 --- a/include/linux/mlx5/mlx5_ifc_fpga.h +++ b/include/linux/mlx5/mlx5_ifc_fpga.h @@ -32,30 +32,6 @@ #ifndef MLX5_IFC_FPGA_H #define MLX5_IFC_FPGA_H -struct mlx5_ifc_ipv4_layout_bits { - u8 reserved_at_0[0x60]; - - u8 ipv4[0x20]; -}; - -struct mlx5_ifc_ipv6_layout_bits { - u8 ipv6[16][0x8]; -}; - -union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits { - struct mlx5_ifc_ipv6_layout_bits ipv6_layout; - struct mlx5_ifc_ipv4_layout_bits ipv4_layout; - u8 reserved_at_0[0x80]; -}; - -enum { - MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX = 0x2c9, -}; - -enum { - MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC = 0x2, -}; - struct mlx5_ifc_fpga_shell_caps_bits { u8 max_num_qps[0x10]; u8 reserved_at_10[0x8]; diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index 8bda3ba5b109c34832e2b1e0d8d7928dd4deaf3d..4657d5c54abefcc58a480c28b34420ad702d31e3 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -162,6 +162,8 @@ enum { MLX5_SEND_WQE_MAX_WQEBBS = 16, }; +#define MLX5_SEND_WQE_MAX_SIZE (MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQE_BB) + enum { MLX5_WQE_FMR_PERM_LOCAL_READ = 1 << 27, MLX5_WQE_FMR_PERM_LOCAL_WRITE = 1 << 28, @@ -252,6 +254,7 @@ enum { enum { MLX5_ETH_WQE_FT_META_IPSEC = BIT(0), + MLX5_ETH_WQE_FT_META_MACSEC = BIT(1), }; struct mlx5_wqe_eth_seg { @@ -475,6 +478,12 @@ struct mlx5_klm { __be64 va; }; +struct mlx5_ksm { + __be32 reserved; + __be32 key; + __be64 va; +}; + struct mlx5_stride_block_entry { __be16 stride; __be16 bcount; diff --git a/include/linux/mm.h b/include/linux/mm.h index 3bedc449c14d8f80530dd7ce6ece6c7c57d6eaa7..8bbcccbc55654341c4d2b361549e1da4da6d65cb 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -661,6 +661,38 @@ static inline bool vma_is_accessible(struct vm_area_struct *vma) return vma->vm_flags & VM_ACCESS_FLAGS; } +static inline +struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) +{ + return mas_find(&vmi->mas, max); +} + +static inline struct vm_area_struct *vma_next(struct vma_iterator *vmi) +{ + /* + * Uses vma_find() to get the first VMA when the iterator starts. + * Calling mas_next() could skip the first entry. + */ + return vma_find(vmi, ULONG_MAX); +} + +static inline struct vm_area_struct *vma_prev(struct vma_iterator *vmi) +{ + return mas_prev(&vmi->mas, 0); +} + +static inline unsigned long vma_iter_addr(struct vma_iterator *vmi) +{ + return vmi->mas.index; +} + +#define for_each_vma(__vmi, __vma) \ + while (((__vma) = vma_next(&(__vmi))) != NULL) + +/* The MM code likes to work with exclusive end addresses */ +#define for_each_vma_range(__vmi, __vma, __end) \ + while (((__vma) = vma_find(&(__vmi), (__end) - 1)) != NULL) + #ifdef CONFIG_SHMEM /* * The vma_is_shmem is not inline because it is used only by slow @@ -697,7 +729,9 @@ static inline unsigned int compound_order(struct page *page) */ static inline unsigned int folio_order(struct folio *folio) { - return compound_order(&folio->page); + if (!folio_test_large(folio)) + return 0; + return folio->_folio_order; } #include @@ -1255,6 +1289,18 @@ static inline int folio_nid(const struct folio *folio) } #ifdef CONFIG_NUMA_BALANCING +/* page access time bits needs to hold at least 4 seconds */ +#define PAGE_ACCESS_TIME_MIN_BITS 12 +#if LAST_CPUPID_SHIFT < PAGE_ACCESS_TIME_MIN_BITS +#define PAGE_ACCESS_TIME_BUCKETS \ + (PAGE_ACCESS_TIME_MIN_BITS - LAST_CPUPID_SHIFT) +#else +#define PAGE_ACCESS_TIME_BUCKETS 0 +#endif + +#define PAGE_ACCESS_TIME_MASK \ + (LAST_CPUPID_MASK << PAGE_ACCESS_TIME_BUCKETS) + static inline int cpu_pid_to_cpupid(int cpu, int pid) { return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); @@ -1318,12 +1364,25 @@ static inline void page_cpupid_reset_last(struct page *page) page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT; } #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ + +static inline int xchg_page_access_time(struct page *page, int time) +{ + int last_time; + + last_time = page_cpupid_xchg_last(page, time >> PAGE_ACCESS_TIME_BUCKETS); + return last_time << PAGE_ACCESS_TIME_BUCKETS; +} #else /* !CONFIG_NUMA_BALANCING */ static inline int page_cpupid_xchg_last(struct page *page, int cpupid) { return page_to_nid(page); /* XXX */ } +static inline int xchg_page_access_time(struct page *page, int time) +{ + return 0; +} + static inline int page_cpupid_last(struct page *page) { return page_to_nid(page); /* XXX */ @@ -1465,6 +1524,11 @@ static inline unsigned long folio_pfn(struct folio *folio) return page_to_pfn(&folio->page); } +static inline struct folio *pfn_folio(unsigned long pfn) +{ + return page_folio(pfn_to_page(pfn)); +} + static inline atomic_t *folio_pincount_ptr(struct folio *folio) { return &folio_page(folio, 1)->compound_pincount; @@ -1544,9 +1608,16 @@ static inline bool is_longterm_pinnable_page(struct page *page) if (mt == MIGRATE_CMA || mt == MIGRATE_ISOLATE) return false; #endif - return !(is_device_coherent_page(page) || - is_zone_movable_page(page) || - is_zero_pfn(page_to_pfn(page))); + /* The zero page may always be pinned */ + if (is_zero_pfn(page_to_pfn(page))) + return true; + + /* Coherent device memory must always allow eviction. */ + if (is_device_coherent_page(page)) + return false; + + /* Otherwise, non-movable zone pages can be pinned. */ + return !is_zone_movable_page(page); } #else static inline bool is_longterm_pinnable_page(struct page *page) @@ -1590,7 +1661,13 @@ static inline void set_page_links(struct page *page, enum zone_type zone, */ static inline long folio_nr_pages(struct folio *folio) { - return compound_nr(&folio->page); + if (!folio_test_large(folio)) + return 1; +#ifdef CONFIG_64BIT + return folio->_folio_nr_pages; +#else + return 1L << folio->_folio_order; +#endif } /** @@ -1769,7 +1846,11 @@ extern void pagefault_out_of_memory(void); */ #define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */ -extern void show_free_areas(unsigned int flags, nodemask_t *nodemask); +extern void __show_free_areas(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +static void __maybe_unused show_free_areas(unsigned int flags, nodemask_t *nodemask) +{ + __show_free_areas(flags, nodemask, MAX_NR_ZONES - 1); +} #ifdef CONFIG_MMU extern bool can_do_mlock(void); @@ -1788,8 +1869,9 @@ void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, unsigned long size); void zap_page_range(struct vm_area_struct *vma, unsigned long address, unsigned long size); -void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, - unsigned long start, unsigned long end); +void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, + struct vm_area_struct *start_vma, unsigned long start, + unsigned long end); struct mmu_notifier_range; @@ -2488,7 +2570,6 @@ extern unsigned long absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn); extern void get_pfn_range_for_nid(unsigned int nid, unsigned long *start_pfn, unsigned long *end_pfn); -extern unsigned long find_min_pfn_with_active_regions(void); #ifndef CONFIG_NUMA static inline int early_pfn_to_nid(unsigned long pfn) @@ -2509,7 +2590,12 @@ extern void calculate_min_free_kbytes(void); extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); extern void __init mmap_init(void); -extern void show_mem(unsigned int flags, nodemask_t *nodemask); + +extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +static inline void show_mem(unsigned int flags, nodemask_t *nodemask) +{ + __show_mem(flags, nodemask, MAX_NR_ZONES - 1); +} extern long si_mem_available(void); extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); @@ -2586,14 +2672,15 @@ extern int __split_vma(struct mm_struct *, struct vm_area_struct *, extern int split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); -extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, - struct rb_node **, struct rb_node *); extern void unlink_file_vma(struct vm_area_struct *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, unsigned long addr, unsigned long len, pgoff_t pgoff, bool *need_rmap_locks); extern void exit_mmap(struct mm_struct *); +void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas); +void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas); + static inline int check_data_rlimit(unsigned long rlim, unsigned long new, unsigned long start, @@ -2641,8 +2728,9 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); -extern int __do_munmap(struct mm_struct *, unsigned long, size_t, - struct list_head *uf, bool downgrade); +extern int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, + unsigned long start, size_t len, struct list_head *uf, + bool downgrade); extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf); extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior); @@ -2709,26 +2797,12 @@ extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long add extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, struct vm_area_struct **pprev); -/** - * find_vma_intersection() - Look up the first VMA which intersects the interval - * @mm: The process address space. - * @start_addr: The inclusive start user address. - * @end_addr: The exclusive end user address. - * - * Returns: The first VMA within the provided range, %NULL otherwise. Assumes - * start_addr < end_addr. +/* + * Look up the first VMA which intersects the interval [start_addr, end_addr) + * NULL if none. Assume start_addr < end_addr. */ -static inline struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, - unsigned long start_addr, - unsigned long end_addr) -{ - struct vm_area_struct *vma = find_vma(mm, start_addr); - - if (vma && end_addr <= vma->vm_start) - vma = NULL; - return vma; -} + unsigned long start_addr, unsigned long end_addr); /** * vma_lookup() - Find a VMA at a specific address @@ -2740,12 +2814,7 @@ struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, static inline struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) { - struct vm_area_struct *vma = find_vma(mm, addr); - - if (vma && addr < vma->vm_start) - vma = NULL; - - return vma; + return mtree_load(&mm->mm_mt, addr); } static inline unsigned long vm_start_gap(struct vm_area_struct *vma) @@ -2781,7 +2850,7 @@ static inline unsigned long vma_pages(struct vm_area_struct *vma) static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end) { - struct vm_area_struct *vma = find_vma(mm, vm_start); + struct vm_area_struct *vma = vma_lookup(mm, vm_start); if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end)) vma = NULL; @@ -2881,11 +2950,9 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, * and return without waiting upon it */ #define FOLL_NOFAULT 0x80 /* do not fault in pages */ #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ -#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */ #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */ #define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */ #define FOLL_REMOTE 0x2000 /* we are working on non-current tsk/mm */ -#define FOLL_COW 0x4000 /* internal GUP flag */ #define FOLL_ANON 0x8000 /* don't do file mappings */ #define FOLL_LONGTERM 0x10000 /* mapping lifetime is indefinite: see below */ #define FOLL_SPLIT_PMD 0x20000 /* split huge pmd before returning */ @@ -2969,8 +3036,8 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) * PageAnonExclusive() has to protect against concurrent GUP: * * Ordinary GUP: Using the PT lock * * GUP-fast and fork(): mm->write_protect_seq - * * GUP-fast and KSM or temporary unmapping (swap, migration): - * clear/invalidate+flush of the page table entry + * * GUP-fast and KSM or temporary unmapping (swap, migration): see + * page_try_share_anon_rmap() * * Must be called with the (sub)page that's actually referenced via the * page table entry, which might not necessarily be the head page for a @@ -2991,6 +3058,11 @@ static inline bool gup_must_unshare(unsigned int flags, struct page *page) */ if (!PageAnon(page)) return false; + + /* Paired with a memory barrier in page_try_share_anon_rmap(). */ + if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + smp_rmb(); + /* * Note that PageKsm() pages cannot be exclusive, and consequently, * cannot get pinned. @@ -2998,6 +3070,21 @@ static inline bool gup_must_unshare(unsigned int flags, struct page *page) return !PageAnonExclusive(page); } +/* + * Indicates whether GUP can follow a PROT_NONE mapped page, or whether + * a (NUMA hinting) fault is required. + */ +static inline bool gup_can_follow_protnone(unsigned int flags) +{ + /* + * FOLL_FORCE has to be able to make progress even if the VMA is + * inaccessible. Further, FOLL_FORCE access usually does not represent + * application behaviour and we should avoid triggering NUMA hinting + * faults. + */ + return flags & FOLL_FORCE; +} + typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data); extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); @@ -3005,7 +3092,7 @@ extern int apply_to_existing_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); -extern void init_mem_debugging_and_hardening(void); +extern void __init init_mem_debugging_and_hardening(void); #ifdef CONFIG_PAGE_POISONING extern void __kernel_poison_pages(struct page *page, int numpages); extern void __kernel_unpoison_pages(struct page *page, int numpages); diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 7b25b53c474a7f17d6ce5cc378a8f1d226afdc00..e8ed225d8f7cac74c5e44ea5417fb757eedcd920 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -34,15 +34,25 @@ static inline int page_is_file_lru(struct page *page) return folio_is_file_lru(page_folio(page)); } -static __always_inline void update_lru_size(struct lruvec *lruvec, +static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) { struct pglist_data *pgdat = lruvec_pgdat(lruvec); + lockdep_assert_held(&lruvec->lru_lock); + WARN_ON_ONCE(nr_pages != (int)nr_pages); + __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages); __mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LRU_BASE + lru, nr_pages); +} + +static __always_inline void update_lru_size(struct lruvec *lruvec, + enum lru_list lru, enum zone_type zid, + long nr_pages) +{ + __update_lru_size(lruvec, lru, zid, nr_pages); #ifdef CONFIG_MEMCG mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages); #endif @@ -66,11 +76,6 @@ static __always_inline void __folio_clear_lru_flags(struct folio *folio) __folio_clear_unevictable(folio); } -static __always_inline void __clear_page_lru_flags(struct page *page) -{ - __folio_clear_lru_flags(page_folio(page)); -} - /** * folio_lru_list - Which LRU list should a folio be on? * @folio: The folio to test. @@ -94,11 +99,224 @@ static __always_inline enum lru_list folio_lru_list(struct folio *folio) return lru; } +#ifdef CONFIG_LRU_GEN + +#ifdef CONFIG_LRU_GEN_ENABLED +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]); +} +#else +static inline bool lru_gen_enabled(void) +{ + DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]); + + return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]); +} +#endif + +static inline bool lru_gen_in_fault(void) +{ + return current->in_lru_fault; +} + +static inline int lru_gen_from_seq(unsigned long seq) +{ + return seq % MAX_NR_GENS; +} + +static inline int lru_hist_from_seq(unsigned long seq) +{ + return seq % NR_HIST_GENS; +} + +static inline int lru_tier_from_refs(int refs) +{ + VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH)); + + /* see the comment in folio_lru_refs() */ + return order_base_2(refs + 1); +} + +static inline int folio_lru_refs(struct folio *folio) +{ + unsigned long flags = READ_ONCE(folio->flags); + bool workingset = flags & BIT(PG_workingset); + + /* + * Return the number of accesses beyond PG_referenced, i.e., N-1 if the + * total number of accesses is N>1, since N=0,1 both map to the first + * tier. lru_tier_from_refs() will account for this off-by-one. Also see + * the comment on MAX_NR_TIERS. + */ + return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + workingset; +} + +static inline int folio_lru_gen(struct folio *folio) +{ + unsigned long flags = READ_ONCE(folio->flags); + + return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + +static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen) +{ + unsigned long max_seq = lruvec->lrugen.max_seq; + + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + + /* see the comment on MIN_NR_GENS */ + return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1); +} + +static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio, + int old_gen, int new_gen) +{ + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + enum lru_list lru = type * LRU_INACTIVE_FILE; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1); + + if (old_gen >= 0) + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + if (new_gen >= 0) + WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone], + lrugen->nr_pages[new_gen][type][zone] + delta); + + /* addition */ + if (old_gen < 0) { + if (lru_gen_is_active(lruvec, new_gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, delta); + return; + } + + /* deletion */ + if (new_gen < 0) { + if (lru_gen_is_active(lruvec, old_gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, -delta); + return; + } + + /* promotion */ + if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) { + __update_lru_size(lruvec, lru, zone, -delta); + __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta); + } + + /* demotion requires isolation, e.g., lru_deactivate_fn() */ + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen)); +} + +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + unsigned long seq; + unsigned long flags; + int gen = folio_lru_gen(folio); + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE_FOLIO(gen != -1, folio); + + if (folio_test_unevictable(folio) || !lrugen->enabled) + return false; + /* + * There are three common cases for this page: + * 1. If it's hot, e.g., freshly faulted in or previously hot and + * migrated, add it to the youngest generation. + * 2. If it's cold but can't be evicted immediately, i.e., an anon page + * not in swapcache or a dirty page pending writeback, add it to the + * second oldest generation. + * 3. Everything else (clean, cold) is added to the oldest generation. + */ + if (folio_test_active(folio)) + seq = lrugen->max_seq; + else if ((type == LRU_GEN_ANON && !folio_test_swapcache(folio)) || + (folio_test_reclaim(folio) && + (folio_test_dirty(folio) || folio_test_writeback(folio)))) + seq = lrugen->min_seq[type] + 1; + else + seq = lrugen->min_seq[type]; + + gen = lru_gen_from_seq(seq); + flags = (gen + 1UL) << LRU_GEN_PGOFF; + /* see the comment on MIN_NR_GENS about PG_active */ + set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags); + + lru_gen_update_size(lruvec, folio, -1, gen); + /* for folio_rotate_reclaimable() */ + if (reclaiming) + list_add_tail(&folio->lru, &lrugen->lists[gen][type][zone]); + else + list_add(&folio->lru, &lrugen->lists[gen][type][zone]); + + return true; +} + +static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + unsigned long flags; + int gen = folio_lru_gen(folio); + + if (gen < 0) + return false; + + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + + /* for folio_migrate_flags() */ + flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; + flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags); + gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + + lru_gen_update_size(lruvec, folio, gen, -1); + list_del(&folio->lru); + + return true; +} + +#else /* !CONFIG_LRU_GEN */ + +static inline bool lru_gen_enabled(void) +{ + return false; +} + +static inline bool lru_gen_in_fault(void) +{ + return false; +} + +static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + return false; +} + +static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + return false; +} + +#endif /* CONFIG_LRU_GEN */ + static __always_inline void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_add_folio(lruvec, folio, false)) + return; + update_lru_size(lruvec, lru, folio_zonenum(folio), folio_nr_pages(folio)); if (lru != LRU_UNEVICTABLE) @@ -116,23 +334,23 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_add_folio(lruvec, folio, true)) + return; + update_lru_size(lruvec, lru, folio_zonenum(folio), folio_nr_pages(folio)); /* This is not expected to be used on LRU_UNEVICTABLE */ list_add_tail(&folio->lru, &lruvec->lists[lru]); } -static __always_inline void add_page_to_lru_list_tail(struct page *page, - struct lruvec *lruvec) -{ - lruvec_add_folio_tail(lruvec, page_folio(page)); -} - static __always_inline void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + if (lru_gen_del_folio(lruvec, folio, false)) + return; + if (lru != LRU_UNEVICTABLE) list_del(&folio->lru); update_lru_size(lruvec, lru, folio_zonenum(folio), diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index cf97f3884fda202d22d41bf5740224cb1617ee5f..500e536796ca4ade1721e1ca82f4b09926e3c8a4 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -223,6 +224,18 @@ struct page { not kmapped, ie. highmem) */ #endif /* WANT_PAGE_VIRTUAL */ +#ifdef CONFIG_KMSAN + /* + * KMSAN metadata for this page: + * - shadow page: every bit indicates whether the corresponding + * bit of the original page is initialized (0) or not (1); + * - origin page: every 4 bytes contain an id of the stack trace + * where the uninitialized value was created. + */ + struct page *kmsan_shadow; + struct page *kmsan_origin; +#endif + #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS int _last_cpupid; #endif @@ -244,6 +257,13 @@ struct page { * @_refcount: Do not access this member directly. Use folio_ref_count() * to find how many references there are to this folio. * @memcg_data: Memory Control Group data. + * @_flags_1: For large folios, additional page flags. + * @__head: Points to the folio. Do not use. + * @_folio_dtor: Which destructor to use for this folio. + * @_folio_order: Do not use directly, call folio_order(). + * @_total_mapcount: Do not use directly, call folio_entire_mapcount(). + * @_pincount: Do not use directly, call folio_maybe_dma_pinned(). + * @_folio_nr_pages: Do not use directly, call folio_nr_pages(). * * A folio is a physically, virtually and logically contiguous set * of bytes. It is a power-of-two in size, and it is aligned to that @@ -282,9 +302,17 @@ struct folio { }; struct page page; }; + unsigned long _flags_1; + unsigned long __head; + unsigned char _folio_dtor; + unsigned char _folio_order; + atomic_t _total_mapcount; + atomic_t _pincount; +#ifdef CONFIG_64BIT + unsigned int _folio_nr_pages; +#endif }; -static_assert(sizeof(struct page) == sizeof(struct folio)); #define FOLIO_MATCH(pg, fl) \ static_assert(offsetof(struct page, pg) == offsetof(struct folio, fl)) FOLIO_MATCH(flags, flags); @@ -299,6 +327,19 @@ FOLIO_MATCH(_refcount, _refcount); FOLIO_MATCH(memcg_data, memcg_data); #endif #undef FOLIO_MATCH +#define FOLIO_MATCH(pg, fl) \ + static_assert(offsetof(struct folio, fl) == \ + offsetof(struct page, pg) + sizeof(struct page)) +FOLIO_MATCH(flags, _flags_1); +FOLIO_MATCH(compound_head, __head); +FOLIO_MATCH(compound_dtor, _folio_dtor); +FOLIO_MATCH(compound_order, _folio_order); +FOLIO_MATCH(compound_mapcount, _total_mapcount); +FOLIO_MATCH(compound_pincount, _pincount); +#ifdef CONFIG_64BIT +FOLIO_MATCH(compound_nr, _folio_nr_pages); +#endif +#undef FOLIO_MATCH static inline atomic_t *folio_mapcount_ptr(struct folio *folio) { @@ -407,21 +448,6 @@ struct vm_area_struct { unsigned long vm_end; /* The first byte after our end address within vm_mm. */ - /* linked list of VM areas per task, sorted by address */ - struct vm_area_struct *vm_next, *vm_prev; - - struct rb_node vm_rb; - - /* - * Largest free memory gap in bytes to the left of this VMA. - * Either between this VMA and vma->vm_prev, or between one of the - * VMAs below us in the VMA rbtree and its ->vm_prev. This helps - * get_unmapped_area find a free area of the right size. - */ - unsigned long rb_subtree_gap; - - /* Second cache line starts here. */ - struct mm_struct *vm_mm; /* The address space we belong to. */ /* @@ -485,9 +511,7 @@ struct vm_area_struct { struct kioctx_table; struct mm_struct { struct { - struct vm_area_struct *mmap; /* list of VMAs */ - struct rb_root mm_rb; - u64 vmacache_seqnum; /* per-thread vmacache */ + struct maple_tree mm_mt; #ifdef CONFIG_MMU unsigned long (*get_unmapped_area) (struct file *filp, unsigned long addr, unsigned long len, @@ -501,7 +525,6 @@ struct mm_struct { unsigned long mmap_compat_legacy_base; #endif unsigned long task_size; /* size of task vm space */ - unsigned long highest_vm_end; /* highest vma end address */ pgd_t * pgd; #ifdef CONFIG_MEMBARRIER @@ -631,22 +654,22 @@ struct mm_struct { #endif #ifdef CONFIG_NUMA_BALANCING /* - * numa_next_scan is the next time that the PTEs will be marked - * pte_numa. NUMA hinting faults will gather statistics and - * migrate pages to new nodes if necessary. + * numa_next_scan is the next time that PTEs will be remapped + * PROT_NONE to trigger NUMA hinting faults; such faults gather + * statistics and migrate pages to new nodes if necessary. */ unsigned long numa_next_scan; - /* Restart point for scanning and setting pte_numa */ + /* Restart point for scanning and remapping PTEs. */ unsigned long numa_scan_offset; - /* numa_scan_seq prevents two threads setting pte_numa */ + /* numa_scan_seq prevents two threads remapping PTEs. */ int numa_scan_seq; #endif /* * An operation with batched TLB flushing is going on. Anything * that can move process memory needs to flush the TLB when - * moving a PROT_NONE or PROT_NUMA mapped page. + * moving a PROT_NONE mapped page. */ atomic_t tlb_flush_pending; #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH @@ -671,7 +694,28 @@ struct mm_struct { * merging. */ unsigned long ksm_merging_pages; + /* + * Represent how many pages are checked for ksm merging + * including merged and not merged. + */ + unsigned long ksm_rmap_items; +#endif +#ifdef CONFIG_LRU_GEN + struct { + /* this mm_struct is on lru_gen_mm_list */ + struct list_head list; + /* + * Set when switching to this mm_struct, as a hint of + * whether it has been used since the last time per-node + * page table walkers cleared the corresponding bits. + */ + unsigned long bitmap; +#ifdef CONFIG_MEMCG + /* points to the memcg of "owner" above */ + struct mem_cgroup *memcg; #endif + } lru_gen; +#endif /* CONFIG_LRU_GEN */ } __randomize_layout; /* @@ -681,6 +725,7 @@ struct mm_struct { unsigned long cpu_bitmap[]; }; +#define MM_MT_FLAGS (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN) extern struct mm_struct init_mm; /* Pointer magic because the dynamic array size confuses some compilers. */ @@ -698,6 +743,87 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) return (struct cpumask *)&mm->cpu_bitmap; } +#ifdef CONFIG_LRU_GEN + +struct lru_gen_mm_list { + /* mm_struct list for page table walkers */ + struct list_head fifo; + /* protects the list above */ + spinlock_t lock; +}; + +void lru_gen_add_mm(struct mm_struct *mm); +void lru_gen_del_mm(struct mm_struct *mm); +#ifdef CONFIG_MEMCG +void lru_gen_migrate_mm(struct mm_struct *mm); +#endif + +static inline void lru_gen_init_mm(struct mm_struct *mm) +{ + INIT_LIST_HEAD(&mm->lru_gen.list); + mm->lru_gen.bitmap = 0; +#ifdef CONFIG_MEMCG + mm->lru_gen.memcg = NULL; +#endif +} + +static inline void lru_gen_use_mm(struct mm_struct *mm) +{ + /* + * When the bitmap is set, page reclaim knows this mm_struct has been + * used since the last time it cleared the bitmap. So it might be worth + * walking the page tables of this mm_struct to clear the accessed bit. + */ + WRITE_ONCE(mm->lru_gen.bitmap, -1); +} + +#else /* !CONFIG_LRU_GEN */ + +static inline void lru_gen_add_mm(struct mm_struct *mm) +{ +} + +static inline void lru_gen_del_mm(struct mm_struct *mm) +{ +} + +#ifdef CONFIG_MEMCG +static inline void lru_gen_migrate_mm(struct mm_struct *mm) +{ +} +#endif + +static inline void lru_gen_init_mm(struct mm_struct *mm) +{ +} + +static inline void lru_gen_use_mm(struct mm_struct *mm) +{ +} + +#endif /* CONFIG_LRU_GEN */ + +struct vma_iterator { + struct ma_state mas; +}; + +#define VMA_ITERATOR(name, __mm, __addr) \ + struct vma_iterator name = { \ + .mas = { \ + .tree = &(__mm)->mm_mt, \ + .index = __addr, \ + .node = MAS_START, \ + }, \ + } + +static inline void vma_iter_init(struct vma_iterator *vmi, + struct mm_struct *mm, unsigned long addr) +{ + vmi->mas.tree = &mm->mm_mt; + vmi->mas.index = addr; + vmi->mas.node = MAS_START; +} + struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); diff --git a/include/linux/mm_types_task.h b/include/linux/mm_types_task.h index c1bc6731125cbbeb99b2bbe0d4757fad6bf5031f..0bb4b6da9993942e320c424990ea8397149b37c8 100644 --- a/include/linux/mm_types_task.h +++ b/include/linux/mm_types_task.h @@ -24,18 +24,6 @@ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) #define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) -/* - * The per task VMA cache array: - */ -#define VMACACHE_BITS 2 -#define VMACACHE_SIZE (1U << VMACACHE_BITS) -#define VMACACHE_MASK (VMACACHE_SIZE - 1) - -struct vmacache { - u64 seqnum; - struct vm_area_struct *vmas[VMACACHE_SIZE]; -}; - /* * When updating this, please also update struct resident_page_types[] in * kernel/fork.c diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index 8a30de08e9139d7b4e4c8cb9a230bc795d95456f..c726ea781255257070197f4802b62c9f3b38fc32 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -293,6 +293,7 @@ struct mmc_card { #define MMC_QUIRK_BROKEN_IRQ_POLLING (1<<11) /* Polling SDIO_CCCR_INTx could create a fake interrupt */ #define MMC_QUIRK_TRIM_BROKEN (1<<12) /* Skip trim */ #define MMC_QUIRK_BROKEN_HPI (1<<13) /* Disable broken HPI support */ +#define MMC_QUIRK_BROKEN_SD_DISCARD (1<<14) /* Disable broken SD discard support */ bool reenable_cmdq; /* Re-enable Command Queue */ diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index eb8bc5b9b0b74d4a4ccfc3efb77e9ec217c7d513..8fdd3cf971a30f59684de944697d97a7a0d013e9 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -476,7 +476,7 @@ struct mmc_host { unsigned int sdio_irqs; struct task_struct *sdio_irq_thread; - struct delayed_work sdio_irq_work; + struct work_struct sdio_irq_work; bool sdio_irq_pending; atomic_t sdio_irq_thread_abort; diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 53f0efa0bccf15fe2fa73fda0c8a41fd4beae418..74f9d9a6d33076dc6c000b32b123355c745896ab 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -74,6 +74,7 @@ #define SDIO_DEVICE_ID_BROADCOM_43362 0xa962 #define SDIO_DEVICE_ID_BROADCOM_43364 0xa9a4 #define SDIO_DEVICE_ID_BROADCOM_43430 0xa9a6 +#define SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439 0xa9af #define SDIO_DEVICE_ID_BROADCOM_43455 0xa9bf #define SDIO_DEVICE_ID_BROADCOM_CYPRESS_43752 0xaae8 diff --git a/include/linux/mmdebug.h b/include/linux/mmdebug.h index 15ae78cd28536eb9107c9ce178273725ea8ab939..b8728d11c94909ac1bed23b3abc87def64229479 100644 --- a/include/linux/mmdebug.h +++ b/include/linux/mmdebug.h @@ -94,6 +94,12 @@ void dump_mm(const struct mm_struct *mm); #define VM_WARN(cond, format...) BUILD_BUG_ON_INVALID(cond) #endif +#ifdef CONFIG_DEBUG_VM_IRQSOFF +#define VM_WARN_ON_IRQS_ENABLED() WARN_ON_ONCE(!irqs_disabled()) +#else +#define VM_WARN_ON_IRQS_ENABLED() do { } while (0) +#endif + #ifdef CONFIG_DEBUG_VIRTUAL #define VIRTUAL_BUG_ON(cond) BUG_ON(cond) #else diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index e24b40c52468a8550eb93c0a0a8c0a95684bea84..5f74891556f33c3df957d744ee15087a0522dc20 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -24,10 +24,10 @@ #include /* Free memory management - zoned buddy allocator. */ -#ifndef CONFIG_FORCE_MAX_ZONEORDER +#ifndef CONFIG_ARCH_FORCE_MAX_ORDER #define MAX_ORDER 11 #else -#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER +#define MAX_ORDER CONFIG_ARCH_FORCE_MAX_ORDER #endif #define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1)) @@ -121,20 +121,6 @@ static inline bool free_area_empty(struct free_area *area, int migratetype) struct pglist_data; -/* - * Add a wild amount of padding here to ensure data fall into separate - * cachelines. There are very few zone structures in the machine, so space - * consumption is not a concern here. - */ -#if defined(CONFIG_SMP) -struct zone_padding { - char x[0]; -} ____cacheline_internodealigned_in_smp; -#define ZONE_PADDING(name) struct zone_padding name; -#else -#define ZONE_PADDING(name) -#endif - #ifdef CONFIG_NUMA enum numa_stat_item { NUMA_HIT, /* allocated in intended node */ @@ -216,11 +202,13 @@ enum node_stat_item { NR_KERNEL_SCS_KB, /* measured in KiB */ #endif NR_PAGETABLE, /* used for pagetables */ + NR_SECONDARY_PAGETABLE, /* secondary pagetables, e.g. KVM pagetables */ #ifdef CONFIG_SWAP NR_SWAPCACHE, #endif #ifdef CONFIG_NUMA_BALANCING PGPROMOTE_SUCCESS, /* promote successfully */ + PGPROMOTE_CANDIDATE, /* candidate pages to promote */ #endif NR_VM_NODE_STAT_ITEMS }; @@ -306,6 +294,8 @@ static inline bool is_active_lru(enum lru_list lru) return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE); } +#define WORKINGSET_ANON 0 +#define WORKINGSET_FILE 1 #define ANON_AND_FILE 2 enum lruvec_flags { @@ -314,6 +304,207 @@ enum lruvec_flags { */ }; +#endif /* !__GENERATING_BOUNDS_H */ + +/* + * Evictable pages are divided into multiple generations. The youngest and the + * oldest generation numbers, max_seq and min_seq, are monotonically increasing. + * They form a sliding window of a variable size [MIN_NR_GENS, MAX_NR_GENS]. An + * offset within MAX_NR_GENS, i.e., gen, indexes the LRU list of the + * corresponding generation. The gen counter in folio->flags stores gen+1 while + * a page is on one of lrugen->lists[]. Otherwise it stores 0. + * + * A page is added to the youngest generation on faulting. The aging needs to + * check the accessed bit at least twice before handing this page over to the + * eviction. The first check takes care of the accessed bit set on the initial + * fault; the second check makes sure this page hasn't been used since then. + * This process, AKA second chance, requires a minimum of two generations, + * hence MIN_NR_GENS. And to maintain ABI compatibility with the active/inactive + * LRU, e.g., /proc/vmstat, these two generations are considered active; the + * rest of generations, if they exist, are considered inactive. See + * lru_gen_is_active(). + * + * PG_active is always cleared while a page is on one of lrugen->lists[] so that + * the aging needs not to worry about it. And it's set again when a page + * considered active is isolated for non-reclaiming purposes, e.g., migration. + * See lru_gen_add_folio() and lru_gen_del_folio(). + * + * MAX_NR_GENS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through page tables. This requires order_base_2(MAX_NR_GENS+1) bits + * in folio->flags. + */ +#define MIN_NR_GENS 2U +#define MAX_NR_GENS 4U + +/* + * Each generation is divided into multiple tiers. A page accessed N times + * through file descriptors is in tier order_base_2(N). A page in the first tier + * (N=0,1) is marked by PG_referenced unless it was faulted in through page + * tables or read ahead. A page in any other tier (N>1) is marked by + * PG_referenced and PG_workingset. This implies a minimum of two tiers is + * supported without using additional bits in folio->flags. + * + * In contrast to moving across generations which requires the LRU lock, moving + * across tiers only involves atomic operations on folio->flags and therefore + * has a negligible cost in the buffered access path. In the eviction path, + * comparisons of refaulted/(evicted+protected) from the first tier and the + * rest infer whether pages accessed multiple times through file descriptors + * are statistically hot and thus worth protecting. + * + * MAX_NR_TIERS is set to 4 so that the multi-gen LRU can support twice the + * number of categories of the active/inactive LRU when keeping track of + * accesses through file descriptors. This uses MAX_NR_TIERS-2 spare bits in + * folio->flags. + */ +#define MAX_NR_TIERS 4U + +#ifndef __GENERATING_BOUNDS_H + +struct lruvec; +struct page_vma_mapped_walk; + +#define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) +#define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) + +#ifdef CONFIG_LRU_GEN + +enum { + LRU_GEN_ANON, + LRU_GEN_FILE, +}; + +enum { + LRU_GEN_CORE, + LRU_GEN_MM_WALK, + LRU_GEN_NONLEAF_YOUNG, + NR_LRU_GEN_CAPS +}; + +#define MIN_LRU_BATCH BITS_PER_LONG +#define MAX_LRU_BATCH (MIN_LRU_BATCH * 64) + +/* whether to keep historical stats from evicted generations */ +#ifdef CONFIG_LRU_GEN_STATS +#define NR_HIST_GENS MAX_NR_GENS +#else +#define NR_HIST_GENS 1U +#endif + +/* + * The youngest generation number is stored in max_seq for both anon and file + * types as they are aged on an equal footing. The oldest generation numbers are + * stored in min_seq[] separately for anon and file types as clean file pages + * can be evicted regardless of swap constraints. + * + * Normally anon and file min_seq are in sync. But if swapping is constrained, + * e.g., out of swap space, file min_seq is allowed to advance and leave anon + * min_seq behind. + * + * The number of pages in each generation is eventually consistent and therefore + * can be transiently negative when reset_batch_size() is pending. + */ +struct lru_gen_struct { + /* the aging increments the youngest generation number */ + unsigned long max_seq; + /* the eviction increments the oldest generation numbers */ + unsigned long min_seq[ANON_AND_FILE]; + /* the birth time of each generation in jiffies */ + unsigned long timestamps[MAX_NR_GENS]; + /* the multi-gen LRU lists, lazily sorted on eviction */ + struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* the multi-gen LRU sizes, eventually consistent */ + long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* the exponential moving average of refaulted */ + unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; + /* the exponential moving average of evicted+protected */ + unsigned long avg_total[ANON_AND_FILE][MAX_NR_TIERS]; + /* the first tier doesn't need protection, hence the minus one */ + unsigned long protected[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS - 1]; + /* can be modified without holding the LRU lock */ + atomic_long_t evicted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; + atomic_long_t refaulted[NR_HIST_GENS][ANON_AND_FILE][MAX_NR_TIERS]; + /* whether the multi-gen LRU is enabled */ + bool enabled; +}; + +enum { + MM_LEAF_TOTAL, /* total leaf entries */ + MM_LEAF_OLD, /* old leaf entries */ + MM_LEAF_YOUNG, /* young leaf entries */ + MM_NONLEAF_TOTAL, /* total non-leaf entries */ + MM_NONLEAF_FOUND, /* non-leaf entries found in Bloom filters */ + MM_NONLEAF_ADDED, /* non-leaf entries added to Bloom filters */ + NR_MM_STATS +}; + +/* double-buffering Bloom filters */ +#define NR_BLOOM_FILTERS 2 + +struct lru_gen_mm_state { + /* set to max_seq after each iteration */ + unsigned long seq; + /* where the current iteration continues (inclusive) */ + struct list_head *head; + /* where the last iteration ended (exclusive) */ + struct list_head *tail; + /* to wait for the last page table walker to finish */ + struct wait_queue_head wait; + /* Bloom filters flip after each iteration */ + unsigned long *filters[NR_BLOOM_FILTERS]; + /* the mm stats for debugging */ + unsigned long stats[NR_HIST_GENS][NR_MM_STATS]; + /* the number of concurrent page table walkers */ + int nr_walkers; +}; + +struct lru_gen_mm_walk { + /* the lruvec under reclaim */ + struct lruvec *lruvec; + /* unstable max_seq from lru_gen_struct */ + unsigned long max_seq; + /* the next address within an mm to scan */ + unsigned long next_addr; + /* to batch promoted pages */ + int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + /* to batch the mm stats */ + int mm_stats[NR_MM_STATS]; + /* total batched items */ + int batched; + bool can_swap; + bool force_scan; +}; + +void lru_gen_init_lruvec(struct lruvec *lruvec); +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw); + +#ifdef CONFIG_MEMCG +void lru_gen_init_memcg(struct mem_cgroup *memcg); +void lru_gen_exit_memcg(struct mem_cgroup *memcg); +#endif + +#else /* !CONFIG_LRU_GEN */ + +static inline void lru_gen_init_lruvec(struct lruvec *lruvec) +{ +} + +static inline void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +{ +} + +#ifdef CONFIG_MEMCG +static inline void lru_gen_init_memcg(struct mem_cgroup *memcg) +{ +} + +static inline void lru_gen_exit_memcg(struct mem_cgroup *memcg) +{ +} +#endif + +#endif /* CONFIG_LRU_GEN */ + struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ @@ -331,6 +522,12 @@ struct lruvec { unsigned long refaults[ANON_AND_FILE]; /* Various lruvec state flags (enum lruvec_flags) */ unsigned long flags; +#ifdef CONFIG_LRU_GEN + /* evictable pages divided into generations */ + struct lru_gen_struct lrugen; + /* to concurrently iterate lru_gen_mm_list */ + struct lru_gen_mm_state mm_state; +#endif #ifdef CONFIG_MEMCG struct pglist_data *pgdat; #endif @@ -368,13 +565,6 @@ enum zone_watermarks { #define NR_LOWORDER_PCP_LISTS (MIGRATE_PCPTYPES * (PAGE_ALLOC_COSTLY_ORDER + 1)) #define NR_PCP_LISTS (NR_LOWORDER_PCP_LISTS + NR_PCP_THP) -/* - * Shift to encode migratetype and order in the same integer, with order - * in the least significant bits. - */ -#define NR_PCP_ORDER_WIDTH 8 -#define NR_PCP_ORDER_MASK ((1<_watermark[WMARK_MIN] + z->watermark_boost) #define low_wmark_pages(z) (z->_watermark[WMARK_LOW] + z->watermark_boost) #define high_wmark_pages(z) (z->_watermark[WMARK_HIGH] + z->watermark_boost) @@ -627,7 +817,7 @@ struct zone { int initialized; /* Write-intensive fields used from the page allocator */ - ZONE_PADDING(_pad1_) + CACHELINE_PADDING(_pad1_); /* free areas of different sizes */ struct free_area free_area[MAX_ORDER]; @@ -639,7 +829,7 @@ struct zone { spinlock_t lock; /* Write-intensive fields used by compaction and vmstats. */ - ZONE_PADDING(_pad2_) + CACHELINE_PADDING(_pad2_); /* * When free pages are below this point, additional steps are taken @@ -676,7 +866,7 @@ struct zone { bool contiguous; - ZONE_PADDING(_pad3_) + CACHELINE_PADDING(_pad3_); /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; @@ -746,6 +936,8 @@ static inline bool zone_is_empty(struct zone *zone) #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) #define KASAN_TAG_PGOFF (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH) +#define LRU_GEN_PGOFF (KASAN_TAG_PGOFF - LRU_GEN_WIDTH) +#define LRU_REFS_PGOFF (LRU_GEN_PGOFF - LRU_REFS_WIDTH) /* * Define the bit shifts to access each section. For non-existent @@ -953,8 +1145,10 @@ typedef struct pglist_data { atomic_t nr_writeback_throttled;/* nr of writeback-throttled tasks */ unsigned long nr_reclaim_start; /* nr pages written while throttled * when throttling started. */ - struct task_struct *kswapd; /* Protected by - mem_hotplug_begin/done() */ +#ifdef CONFIG_MEMORY_HOTPLUG + struct mutex kswapd_lock; +#endif + struct task_struct *kswapd; /* Protected by kswapd_lock */ int kswapd_order; enum zone_type kswapd_highest_zoneidx; @@ -982,7 +1176,7 @@ typedef struct pglist_data { #endif /* CONFIG_NUMA */ /* Write-intensive fields used by page reclaim */ - ZONE_PADDING(_pad1_) + CACHELINE_PADDING(_pad1_); #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT /* @@ -996,6 +1190,21 @@ typedef struct pglist_data { struct deferred_split deferred_split_queue; #endif +#ifdef CONFIG_NUMA_BALANCING + /* start time in ms of current promote rate limit period */ + unsigned int nbp_rl_start; + /* number of promote candidate pages at start time of current rate limit period */ + unsigned long nbp_rl_nr_cand; + /* promote threshold in ms */ + unsigned int nbp_threshold; + /* start time in ms of current promote threshold adjustment period */ + unsigned int nbp_th_start; + /* + * number of promote candidate pages at stat time of current promote + * threshold adjustment period + */ + unsigned long nbp_th_nr_cand; +#endif /* Fields commonly accessed by the page reclaim scanner */ /* @@ -1007,11 +1216,19 @@ typedef struct pglist_data { unsigned long flags; - ZONE_PADDING(_pad2_) +#ifdef CONFIG_LRU_GEN + /* kswap mm walk data */ + struct lru_gen_mm_walk mm_walk; +#endif + + CACHELINE_PADDING(_pad2_); /* Per-node vmstats */ struct per_cpu_nodestat __percpu *per_cpu_nodestats; atomic_long_t vm_stat[NR_VM_NODE_STAT_ITEMS]; +#ifdef CONFIG_NUMA + struct memory_tier __rcu *memtier; +#endif } pg_data_t; #define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages) @@ -1025,11 +1242,6 @@ static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) return pgdat->node_start_pfn + pgdat->node_spanned_pages; } -static inline bool pgdat_is_empty(pg_data_t *pgdat) -{ - return !pgdat->node_start_pfn && !pgdat->node_spanned_pages; -} - #include void build_all_zonelists(pg_data_t *pgdat); diff --git a/include/linux/module.h b/include/linux/module.h index 518296ea7f73af6e4b8fc40f3166ad594ed25a8b..ec61fb53979a92ad164852f50c6c91ffe00deafd 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #include @@ -132,7 +131,7 @@ extern void cleanup_module(void); { return initfn; } \ int init_module(void) __copy(initfn) \ __attribute__((alias(#initfn))); \ - __CFI_ADDRESSABLE(init_module, __initdata); + ___ADDRESSABLE(init_module, __initdata); /* This is only required if you want to be unloadable. */ #define module_exit(exitfn) \ @@ -140,7 +139,7 @@ extern void cleanup_module(void); { return exitfn; } \ void cleanup_module(void) __copy(exitfn) \ __attribute__((alias(#exitfn))); \ - __CFI_ADDRESSABLE(cleanup_module, __exitdata); + ___ADDRESSABLE(cleanup_module, __exitdata); #endif @@ -387,8 +386,9 @@ struct module { const s32 *crcs; unsigned int num_syms; -#ifdef CONFIG_CFI_CLANG - cfi_check_fn cfi_check; +#ifdef CONFIG_ARCH_USES_CFI_TRAPS + s32 *kcfi_traps; + s32 *kcfi_traps_end; #endif /* Kernel parameters. */ diff --git a/include/linux/mroute.h b/include/linux/mroute.h index 6cbbfe94348cee5f4176dec3443bee68fcc7ada0..80b8400ab8b2422ba3625cabcd51c83dbd79710e 100644 --- a/include/linux/mroute.h +++ b/include/linux/mroute.h @@ -17,7 +17,7 @@ static inline int ip_mroute_opt(int opt) } int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int); -int ip_mroute_getsockopt(struct sock *, int, char __user *, int __user *); +int ip_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t); int ipmr_ioctl(struct sock *sk, int cmd, void __user *arg); int ipmr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); int ip_mr_init(void); @@ -29,8 +29,8 @@ static inline int ip_mroute_setsockopt(struct sock *sock, int optname, return -ENOPROTOOPT; } -static inline int ip_mroute_getsockopt(struct sock *sock, int optname, - char __user *optval, int __user *optlen) +static inline int ip_mroute_getsockopt(struct sock *sk, int optname, + sockptr_t optval, sockptr_t optlen) { return -ENOPROTOOPT; } diff --git a/include/linux/mroute6.h b/include/linux/mroute6.h index bc351a85ce9b9cefb04cced1e0a3de42c51e877f..8f2b307fb1241e8a727c12851c3b10a0ddc62bec 100644 --- a/include/linux/mroute6.h +++ b/include/linux/mroute6.h @@ -27,7 +27,7 @@ struct sock; #ifdef CONFIG_IPV6_MROUTE extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int); -extern int ip6_mroute_getsockopt(struct sock *, int, char __user *, int __user *); +extern int ip6_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t); extern int ip6_mr_input(struct sk_buff *skb); extern int ip6mr_ioctl(struct sock *sk, int cmd, void __user *arg); extern int ip6mr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg); @@ -42,7 +42,7 @@ static inline int ip6_mroute_setsockopt(struct sock *sock, int optname, static inline int ip6_mroute_getsockopt(struct sock *sock, - int optname, char __user *optval, int __user *optlen) + int optname, sockptr_t optval, sockptr_t optlen) { return -ENOPROTOOPT; } diff --git a/include/linux/mtd/mtd.h b/include/linux/mtd/mtd.h index 955aee14b0f7e5791ec6348cbe9d6d3d1c1e94d4..7c58c44662b87e363cea26e045f9b636f6ea503c 100644 --- a/include/linux/mtd/mtd.h +++ b/include/linux/mtd/mtd.h @@ -40,6 +40,12 @@ struct mtd_erase_region_info { unsigned long *lockmap; /* If keeping bitmap of locks */ }; +struct mtd_req_stats { + unsigned int uncorrectable_errors; + unsigned int corrected_bitflips; + unsigned int max_bitflips; +}; + /** * struct mtd_oob_ops - oob operation operands * @mode: operation mode @@ -70,6 +76,7 @@ struct mtd_oob_ops { uint32_t ooboffs; uint8_t *datbuf; uint8_t *oobbuf; + struct mtd_req_stats *stats; }; /** @@ -677,6 +684,7 @@ extern int mtd_device_unregister(struct mtd_info *master); extern struct mtd_info *get_mtd_device(struct mtd_info *mtd, int num); extern int __get_mtd_device(struct mtd_info *mtd); extern void __put_mtd_device(struct mtd_info *mtd); +extern struct mtd_info *of_get_mtd_device_by_node(struct device_node *np); extern struct mtd_info *get_mtd_device_nm(const char *name); extern void put_mtd_device(struct mtd_info *mtd); diff --git a/include/linux/namei.h b/include/linux/namei.h index caeb08a98536cc43b53652b4ab70d0adb99252c3..00fee52df84234cfcf7a0bf8bbe327df547b07c4 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -83,7 +83,7 @@ extern int follow_up(struct path *); extern struct dentry *lock_rename(struct dentry *, struct dentry *); extern void unlock_rename(struct dentry *, struct dentry *); -extern int __must_check nd_jump_link(struct path *path); +extern int __must_check nd_jump_link(const struct path *path); static inline void nd_terminate_link(void *name, size_t len, size_t maxlen) { diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 1a3cb93c3dcce129b601240fd502823df048e313..a36edb0ec199393dcb065ff8459d0299688048e5 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -253,11 +253,17 @@ struct netdev_hw_addr_list { #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc) #define netdev_for_each_uc_addr(ha, dev) \ netdev_hw_addr_list_for_each(ha, &(dev)->uc) +#define netdev_for_each_synced_uc_addr(_ha, _dev) \ + netdev_for_each_uc_addr((_ha), (_dev)) \ + if ((_ha)->sync_cnt) #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc) #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc) #define netdev_for_each_mc_addr(ha, dev) \ netdev_hw_addr_list_for_each(ha, &(dev)->mc) +#define netdev_for_each_synced_mc_addr(_ha, _dev) \ + netdev_for_each_mc_addr((_ha), (_dev)) \ + if ((_ha)->sync_cnt) struct hh_cache { unsigned int hh_len; @@ -546,8 +552,8 @@ static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n) { unsigned long val, new; + val = READ_ONCE(n->state); do { - val = READ_ONCE(n->state); if (val & NAPIF_STATE_DISABLE) return true; @@ -555,7 +561,7 @@ static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n) return false; new = val | NAPIF_STATE_MISSED; - } while (cmpxchg(&n->state, val, new) != val); + } while (!try_cmpxchg(&n->state, &val, new)); return true; } @@ -640,9 +646,23 @@ extern int sysctl_devconf_inherit_init_net; */ static inline bool net_has_fallback_tunnels(const struct net *net) { - return !IS_ENABLED(CONFIG_SYSCTL) || - !sysctl_fb_tunnels_only_for_init_net || - (net == &init_net && sysctl_fb_tunnels_only_for_init_net == 1); +#if IS_ENABLED(CONFIG_SYSCTL) + int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net); + + return !fb_tunnels_only_for_init_net || + (net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1); +#else + return true; +#endif +} + +static inline int net_inherit_devconf(void) +{ +#if IS_ENABLED(CONFIG_SYSCTL) + return READ_ONCE(sysctl_devconf_inherit_init_net); +#else + return 0; +#endif } static inline int netdev_queue_numa_node_read(const struct netdev_queue *q) @@ -920,6 +940,7 @@ struct net_device_path_ctx { }; enum tc_setup_type { + TC_QUERY_CAPS, TC_SETUP_QDISC_MQPRIO, TC_SETUP_CLSU32, TC_SETUP_CLSFLOWER, @@ -1837,7 +1858,6 @@ enum netdev_ml_priv_type { * @tipc_ptr: TIPC specific data * @atalk_ptr: AppleTalk link * @ip_ptr: IPv4 specific data - * @dn_ptr: DECnet specific data * @ip6_ptr: IPv6 specific data * @ax25_ptr: AX.25 specific data * @ieee80211_ptr: IEEE 802.11 specific data, assign before registering @@ -2133,9 +2153,6 @@ struct net_device { #if IS_ENABLED(CONFIG_ATALK) void *atalk_ptr; #endif -#if IS_ENABLED(CONFIG_DECNET) - struct dn_dev __rcu *dn_ptr; -#endif #if IS_ENABLED(CONFIG_AX25) void *ax25_ptr; #endif @@ -2537,16 +2554,15 @@ void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi, * @dev: network device * @napi: NAPI context * @poll: polling function - * @weight: default weight * * netif_napi_add() must be used to initialize a NAPI context prior to calling * *any* of the other NAPI-related functions. */ static inline void netif_napi_add(struct net_device *dev, struct napi_struct *napi, - int (*poll)(struct napi_struct *, int), int weight) + int (*poll)(struct napi_struct *, int)) { - netif_napi_add_weight(dev, napi, poll, weight); + netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT); } static inline void @@ -2559,8 +2575,6 @@ netif_napi_add_tx_weight(struct net_device *dev, netif_napi_add_weight(dev, napi, poll, weight); } -#define netif_tx_napi_add netif_napi_add_tx_weight - /** * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only * @dev: network device @@ -3343,6 +3357,16 @@ static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_qu #endif } +/** + * netdev_tx_sent_queue - report the number of bytes queued to a given tx queue + * @dev_queue: network device queue + * @bytes: number of bytes queued to the device queue + * + * Report the number of bytes queued for sending/completion to the network + * device hardware queue. @bytes should be a good approximation and should + * exactly match netdev_completed_queue() @bytes. + * This is typically called once per packet, from ndo_start_xmit(). + */ static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue, unsigned int bytes) { @@ -3388,13 +3412,14 @@ static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue, } /** - * netdev_sent_queue - report the number of bytes queued to hardware - * @dev: network device - * @bytes: number of bytes queued to the hardware device queue + * netdev_sent_queue - report the number of bytes queued to hardware + * @dev: network device + * @bytes: number of bytes queued to the hardware device queue * - * Report the number of bytes queued for sending/completion to the network - * device hardware queue. @bytes should be a good approximation and should - * exactly match netdev_completed_queue() @bytes + * Report the number of bytes queued for sending/completion to the network + * device hardware queue#0. @bytes should be a good approximation and should + * exactly match netdev_completed_queue() @bytes. + * This is typically called once per packet, from ndo_start_xmit(). */ static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes) { @@ -3409,6 +3434,15 @@ static inline bool __netdev_sent_queue(struct net_device *dev, xmit_more); } +/** + * netdev_tx_completed_queue - report number of packets/bytes at TX completion. + * @dev_queue: network device queue + * @pkts: number of packets (currently ignored) + * @bytes: number of bytes dequeued from the device queue + * + * Must be called at most once per TX completion round (and not per + * individual packet), so that BQL can adjust its limits appropriately. + */ static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue, unsigned int pkts, unsigned int bytes) { @@ -3629,9 +3663,8 @@ static inline bool netif_attr_test_online(unsigned long j, static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp, unsigned int nr_bits) { - /* -1 is a legal arg here. */ - if (n != -1) - cpu_max_bits_warn(n, nr_bits); + /* n is a prior cpu */ + cpu_max_bits_warn(n + 1, nr_bits); if (srcp) return find_next_bit(srcp, nr_bits, n + 1); @@ -3652,9 +3685,8 @@ static inline int netif_attrmask_next_and(int n, const unsigned long *src1p, const unsigned long *src2p, unsigned int nr_bits) { - /* -1 is a legal arg here. */ - if (n != -1) - cpu_max_bits_warn(n, nr_bits); + /* n is a prior cpu */ + cpu_max_bits_warn(n + 1, nr_bits); if (src1p && src2p) return find_next_and_bit(src1p, src2p, nr_bits, n + 1); @@ -3788,6 +3820,7 @@ void netif_receive_skb_list(struct list_head *head); gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb); void napi_gro_flush(struct napi_struct *napi, bool flush_old); struct sk_buff *napi_get_frags(struct napi_struct *napi); +void napi_get_frags_check(struct napi_struct *napi); gro_result_t napi_gro_frags(struct napi_struct *napi); struct packet_offload *gro_find_receive_by_type(__be16 type); struct packet_offload *gro_find_complete_by_type(__be16 type); diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index c2c6f332fb90eb3d812ae539e7d3315ad6183426..d8817d381c14bfaecff3fb5a0b0cf487306e173a 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -243,11 +243,6 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net, hook_head = rcu_dereference(net->nf.hooks_bridge[hook]); #endif break; -#if IS_ENABLED(CONFIG_DECNET) - case NFPROTO_DECNET: - hook_head = rcu_dereference(net->nf.hooks_decnet[hook]); - break; -#endif default: WARN_ON_ONCE(1); break; diff --git a/include/linux/netfilter_bridge/ebtables.h b/include/linux/netfilter_bridge/ebtables.h index a13296d6c7ceb2386e3870f6c1671686d990970a..fd533552a062ce0e1219c9225fadf93bc7aacabf 100644 --- a/include/linux/netfilter_bridge/ebtables.h +++ b/include/linux/netfilter_bridge/ebtables.h @@ -94,10 +94,6 @@ struct ebt_table { struct ebt_replace_kernel *table; unsigned int valid_hooks; rwlock_t lock; - /* e.g. could be the table explicitly only allows certain - * matches, targets, ... 0 == let it in */ - int (*check)(const struct ebt_table_info *info, - unsigned int valid_hooks); /* the data used by the kernel */ struct ebt_table_info *private; struct nf_hook_ops *ops; diff --git a/include/linux/netfilter_defs.h b/include/linux/netfilter_defs.h index 8dddfb151f004bb4adf402dcedde3a3f4454b7f0..a5f7bef1b3a47e7082234be8f3816d356cfae3fa 100644 --- a/include/linux/netfilter_defs.h +++ b/include/linux/netfilter_defs.h @@ -7,14 +7,6 @@ /* in/out/forward only */ #define NF_ARP_NUMHOOKS 3 -/* max hook is NF_DN_ROUTE (6), also see uapi/linux/netfilter_decnet.h */ -#define NF_DN_NUMHOOKS 7 - -#if IS_ENABLED(CONFIG_DECNET) -/* Largest hook number + 1, see uapi/linux/netfilter_decnet.h */ -#define NF_MAX_HOOKS NF_DN_NUMHOOKS -#else #define NF_MAX_HOOKS NF_INET_NUMHOOKS -#endif #endif diff --git a/include/linux/netlink.h b/include/linux/netlink.h index bda1c385cffbca2c3728d2692037a655e3995c9a..d51e041d2242681c67cc2499682bc8bfb16dce82 100644 --- a/include/linux/netlink.h +++ b/include/linux/netlink.h @@ -71,6 +71,8 @@ netlink_kernel_create(struct net *net, int unit, struct netlink_kernel_cfg *cfg) * %NL_SET_ERR_MSG * @bad_attr: attribute with error * @policy: policy for a bad attribute + * @miss_type: attribute type which was missing + * @miss_nest: nest missing an attribute (%NULL if missing top level attr) * @cookie: cookie data to return to userspace (for success) * @cookie_len: actual cookie data length */ @@ -78,6 +80,8 @@ struct netlink_ext_ack { const char *_msg; const struct nlattr *bad_attr; const struct nla_policy *policy; + const struct nlattr *miss_nest; + u16 miss_type; u8 cookie[NETLINK_MAX_COOKIE_LEN]; u8 cookie_len; }; @@ -126,6 +130,26 @@ struct netlink_ext_ack { #define NL_SET_ERR_MSG_ATTR(extack, attr, msg) \ NL_SET_ERR_MSG_ATTR_POL(extack, attr, NULL, msg) +#define NL_SET_ERR_ATTR_MISS(extack, nest, type) do { \ + struct netlink_ext_ack *__extack = (extack); \ + \ + if (__extack) { \ + __extack->miss_nest = (nest); \ + __extack->miss_type = (type); \ + } \ +} while (0) + +#define NL_REQ_ATTR_CHECK(extack, nest, tb, type) ({ \ + struct nlattr **__tb = (tb); \ + u32 __attr = (type); \ + int __retval; \ + \ + __retval = !__tb[__attr]; \ + if (__retval) \ + NL_SET_ERR_ATTR_MISS((extack), (nest), __attr); \ + __retval; \ +}) + static inline void nl_set_extack_cookie_u64(struct netlink_ext_ack *extack, u64 cookie) { diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index b32ed68e7dc4971a72d03f1021d5595301cf429f..7931fa47256129e26422531be56b65ec4fb39c50 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -83,7 +83,6 @@ struct nfs_open_context { fmode_t mode; unsigned long flags; -#define NFS_CONTEXT_RESEND_WRITES (1) #define NFS_CONTEXT_BAD (2) #define NFS_CONTEXT_UNLOCK (3) #define NFS_CONTEXT_FILE_OPEN (4) @@ -182,6 +181,7 @@ struct nfs_inode { /* Regular file */ struct { atomic_long_t nrequests; + atomic_long_t redirtied_pages; struct nfs_mds_commit_info commit_info; struct mutex commit_mutex; }; diff --git a/include/linux/node.h b/include/linux/node.h index 40d641a8bfb0d4cfba3cc232f6f837458f7e5c76..427a5975cf405045ded741ed4c1b4b9eaca1ffec 100644 --- a/include/linux/node.h +++ b/include/linux/node.h @@ -2,15 +2,15 @@ /* * include/linux/node.h - generic node definition * - * This is mainly for topological representation. We define the - * basic 'struct node' here, which can be embedded in per-arch + * This is mainly for topological representation. We define the + * basic 'struct node' here, which can be embedded in per-arch * definitions of processors. * * Basic handling of the devices is done in drivers/base/node.c - * and system devices are handled in drivers/base/sys.c. + * and system devices are handled in drivers/base/sys.c. * * Nodes are exported via driverfs in the class/node/devices/ - * directory. + * directory. */ #ifndef _LINUX_NODE_H_ #define _LINUX_NODE_H_ @@ -18,7 +18,6 @@ #include #include #include -#include /** * struct node_hmem_attrs - heterogeneous memory performance attributes @@ -84,10 +83,6 @@ static inline void node_set_perf_attrs(unsigned int nid, struct node { struct device dev; struct list_head access_list; - -#if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_HUGETLBFS) - struct work_struct node_work; -#endif #ifdef CONFIG_HMEM_REPORTING struct list_head cache_attrs; struct device *cache_dev; @@ -96,7 +91,6 @@ struct node { struct memory_block; extern struct node *node_devices[]; -typedef void (*node_registration_func_t)(struct node *); #if defined(CONFIG_MEMORY_HOTPLUG) && defined(CONFIG_NUMA) void register_memory_blocks_under_node(int nid, unsigned long start_pfn, @@ -144,11 +138,6 @@ extern void unregister_memory_block_under_nodes(struct memory_block *mem_blk); extern int register_memory_node_under_compute_node(unsigned int mem_nid, unsigned int cpu_nid, unsigned access); - -#ifdef CONFIG_HUGETLBFS -extern void register_hugetlbfs_with_node(node_registration_func_t doregister, - node_registration_func_t unregister); -#endif #else static inline void node_dev_init(void) { @@ -176,18 +165,8 @@ static inline int unregister_cpu_under_node(unsigned int cpu, unsigned int nid) static inline void unregister_memory_block_under_nodes(struct memory_block *mem_blk) { } - -static inline void register_hugetlbfs_with_node(node_registration_func_t reg, - node_registration_func_t unreg) -{ -} #endif #define to_node(device) container_of(device, struct node, dev) -static inline bool node_is_toptier(int node) -{ - return node_state(node, N_CPU); -} - #endif /* _LINUX_NODE_H_ */ diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 4b71a96190a84ca93912903c08ea470b78b3630c..efef68c9352a0001f9e037dea139d23d80afd269 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -493,6 +493,7 @@ static inline int num_node_state(enum node_states state) #define first_online_node 0 #define first_memory_node 0 #define next_online_node(nid) (MAX_NUMNODES) +#define next_memory_node(nid) (MAX_NUMNODES) #define nr_node_ids 1U #define nr_online_nodes 1U @@ -504,12 +505,20 @@ static inline int num_node_state(enum node_states state) static inline int node_random(const nodemask_t *maskp) { #if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1) - int w, bit = NUMA_NO_NODE; + int w, bit; w = nodes_weight(*maskp); - if (w) - bit = bitmap_ord_to_pos(maskp->bits, - get_random_int() % w, MAX_NUMNODES); + switch (w) { + case 0: + bit = NUMA_NO_NODE; + break; + case 1: + bit = first_node(*maskp); + break; + default: + bit = find_nth_bit(maskp->bits, MAX_NUMNODES, prandom_u32_max(w)); + break; + } return bit; #else return 0; diff --git a/include/linux/nvme.h b/include/linux/nvme.h index ae53d74f3696a8275dd7ba061453779c5f310215..050d7d0cd81b0c8163cfcd6a23eaf414dc26f43d 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -1482,8 +1482,8 @@ struct nvmf_connect_command { }; enum { - NVME_CONNECT_AUTHREQ_ASCR = (1 << 2), - NVME_CONNECT_AUTHREQ_ATR = (1 << 1), + NVME_CONNECT_AUTHREQ_ASCR = (1U << 18), + NVME_CONNECT_AUTHREQ_ATR = (1U << 17), }; struct nvmf_connect_data { diff --git a/include/linux/of.h b/include/linux/of.h index 766d002bddb9a183ba305cdaacf27e1d63c11ab7..6b79ef9a6541f1ffef6088e0936cabc0d0188fdd 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -342,7 +342,7 @@ extern int of_property_read_string_helper(const struct device_node *np, const char **out_strs, size_t sz, int index); extern int of_device_is_compatible(const struct device_node *device, const char *); -extern int of_device_compatible_match(struct device_node *device, +extern int of_device_compatible_match(const struct device_node *device, const char *const *compat); extern bool of_device_is_available(const struct device_node *device); extern bool of_device_is_big_endian(const struct device_node *device); @@ -562,7 +562,7 @@ static inline int of_device_is_compatible(const struct device_node *device, return 0; } -static inline int of_device_compatible_match(struct device_node *device, +static inline int of_device_compatible_match(const struct device_node *device, const char *const *compat) { return 0; diff --git a/include/linux/of_device.h b/include/linux/of_device.h index 1d7992a02e36e283438254ef84ad704e209e2848..1a803e4335d30528aae1ba07674d82d47ba84ff3 100644 --- a/include/linux/of_device.h +++ b/include/linux/of_device.h @@ -101,8 +101,9 @@ static inline struct device_node *of_cpu_device_node_get(int cpu) } static inline int of_dma_configure_id(struct device *dev, - struct device_node *np, - bool force_dma) + struct device_node *np, + bool force_dma, + const u32 *id) { return 0; } diff --git a/include/linux/of_irq.h b/include/linux/of_irq.h index 83fccd0c9bba219192ea30604071287a399898e1..d6d3eae2f1452ff9a85c2fd29b30efb1b91177d4 100644 --- a/include/linux/of_irq.h +++ b/include/linux/of_irq.h @@ -37,9 +37,8 @@ extern unsigned int irq_create_of_mapping(struct of_phandle_args *irq_data); extern int of_irq_to_resource(struct device_node *dev, int index, struct resource *r); -extern void of_irq_init(const struct of_device_id *matches); - #ifdef CONFIG_OF_IRQ +extern void of_irq_init(const struct of_device_id *matches); extern int of_irq_parse_one(struct device_node *device, int index, struct of_phandle_args *out_irq); extern int of_irq_count(struct device_node *dev); @@ -57,6 +56,9 @@ extern struct irq_domain *of_msi_map_get_device_domain(struct device *dev, extern void of_msi_configure(struct device *dev, struct device_node *np); u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in); #else +static inline void of_irq_init(const struct of_device_id *matches) +{ +} static inline int of_irq_parse_one(struct device_node *device, int index, struct of_phandle_args *out_irq) { diff --git a/include/linux/once.h b/include/linux/once.h index b14d8b309d52b198bb144689fe67d9ed235c2b3e..bc714d414448a7ab982f14d4ab76512170a4689f 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -5,10 +5,18 @@ #include #include +/* Helpers used from arbitrary contexts. + * Hard irqs are blocked, be cautious. + */ bool __do_once_start(bool *done, unsigned long *flags); void __do_once_done(bool *done, struct static_key_true *once_key, unsigned long *flags, struct module *mod); +/* Variant for process contexts only. */ +bool __do_once_sleepable_start(bool *done); +void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, + struct module *mod); + /* Call a function exactly once. The idea of DO_ONCE() is to perform * a function call such as initialization of random seeds, etc, only * once, where DO_ONCE() can live in the fast-path. After @func has @@ -52,7 +60,27 @@ void __do_once_done(bool *done, struct static_key_true *once_key, ___ret; \ }) +/* Variant of DO_ONCE() for process/sleepable contexts. */ +#define DO_ONCE_SLEEPABLE(func, ...) \ + ({ \ + bool ___ret = false; \ + static bool __section(".data.once") ___done = false; \ + static DEFINE_STATIC_KEY_TRUE(___once_key); \ + if (static_branch_unlikely(&___once_key)) { \ + ___ret = __do_once_sleepable_start(&___done); \ + if (unlikely(___ret)) { \ + func(__VA_ARGS__); \ + __do_once_sleepable_done(&___done, &___once_key,\ + THIS_MODULE); \ + } \ + } \ + ___ret; \ + }) + #define get_random_once(buf, nbytes) \ DO_ONCE(get_random_bytes, (buf), (nbytes)) +#define get_random_sleepable_once(buf, nbytes) \ + DO_ONCE_SLEEPABLE(get_random_bytes, (buf), (nbytes)) + #endif /* _LINUX_ONCE_H */ diff --git a/include/linux/oom.h b/include/linux/oom.h index 02d1e7bbd8cd5b1f41d661fd11999b822c2485ef..7d0c9c48a0c54e7265e2dc872d7f81c60463788a 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -77,15 +77,6 @@ static inline bool tsk_is_oom_victim(struct task_struct * tsk) return tsk->signal->oom_mm; } -/* - * Use this helper if tsk->mm != mm and the victim mm needs a special - * handling. This is guaranteed to stay true after once set. - */ -static inline bool mm_is_oom_victim(struct mm_struct *mm) -{ - return test_bit(MMF_OOM_VICTIM, &mm->flags); -} - /* * Checks whether a page fault on the given mm is still reliable. * This is no longer true if the oom reaper started to reap the @@ -106,8 +97,6 @@ static inline vm_fault_t check_stable_address_space(struct mm_struct *mm) return 0; } -bool __oom_reap_task_mm(struct mm_struct *mm); - long oom_badness(struct task_struct *p, unsigned long totalpages); diff --git a/include/linux/overflow.h b/include/linux/overflow.h index f1221d11f8e57a7dfe9c33dd6363b5875ef6b796..19dfdd74835eceaf32edd3e1cb4987fe32d8a7db 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -30,7 +30,6 @@ * https://mail-index.netbsd.org/tech-misc/2007/02/05/0000.html - * credit to Christian Biere. */ -#define is_signed_type(type) (((type)(-1)) < (type)1) #define __type_half_max(type) ((type)1 << (8*sizeof(type) - 1 - is_signed_type(type))) #define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T))) #define type_min(T) ((T)((T)-type_max(T)-(T)1)) @@ -52,40 +51,50 @@ static inline bool __must_check __must_check_overflow(bool overflow) return unlikely(overflow); } -/* - * For simplicity and code hygiene, the fallback code below insists on - * a, b and *d having the same type (similar to the min() and max() - * macros), whereas gcc's type-generic overflow checkers accept - * different types. Hence we don't just make check_add_overflow an - * alias for __builtin_add_overflow, but add type checks similar to - * below. +/** check_add_overflow() - Calculate addition with overflow checking + * + * @a: first addend + * @b: second addend + * @d: pointer to store sum + * + * Returns 0 on success. + * + * *@d holds the results of the attempted addition, but is not considered + * "safe for use" on a non-zero return value, which indicates that the + * sum has overflowed or been truncated. */ -#define check_add_overflow(a, b, d) __must_check_overflow(({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - __builtin_add_overflow(__a, __b, __d); \ -})) +#define check_add_overflow(a, b, d) \ + __must_check_overflow(__builtin_add_overflow(a, b, d)) -#define check_sub_overflow(a, b, d) __must_check_overflow(({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - __builtin_sub_overflow(__a, __b, __d); \ -})) +/** check_sub_overflow() - Calculate subtraction with overflow checking + * + * @a: minuend; value to subtract from + * @b: subtrahend; value to subtract from @a + * @d: pointer to store difference + * + * Returns 0 on success. + * + * *@d holds the results of the attempted subtraction, but is not considered + * "safe for use" on a non-zero return value, which indicates that the + * difference has underflowed or been truncated. + */ +#define check_sub_overflow(a, b, d) \ + __must_check_overflow(__builtin_sub_overflow(a, b, d)) -#define check_mul_overflow(a, b, d) __must_check_overflow(({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - typeof(d) __d = (d); \ - (void) (&__a == &__b); \ - (void) (&__a == __d); \ - __builtin_mul_overflow(__a, __b, __d); \ -})) +/** check_mul_overflow() - Calculate multiplication with overflow checking + * + * @a: first factor + * @b: second factor + * @d: pointer to store product + * + * Returns 0 on success. + * + * *@d holds the results of the attempted multiplication, but is not + * considered "safe for use" on a non-zero return value, which indicates + * that the product has overflowed or been truncated. + */ +#define check_mul_overflow(a, b, d) \ + __must_check_overflow(__builtin_mul_overflow(a, b, d)) /** check_shl_overflow() - Calculate a left-shifted value and check overflow * diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h index ef1e3e736e1483e1f6070d99dfac2c163069c902..7d79818dc065130a490a3a23338c5c7019303beb 100644 --- a/include/linux/page-flags-layout.h +++ b/include/linux/page-flags-layout.h @@ -55,7 +55,8 @@ #define SECTIONS_WIDTH 0 #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_SHIFT \ + <= BITS_PER_LONG - NR_PAGEFLAGS #define NODES_WIDTH NODES_SHIFT #elif defined(CONFIG_SPARSEMEM_VMEMMAP) #error "Vmemmap: No space for nodes field in page flags" @@ -89,8 +90,8 @@ #define LAST_CPUPID_SHIFT 0 #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT \ - <= BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ + KASAN_TAG_WIDTH + LAST_CPUPID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS #define LAST_CPUPID_WIDTH LAST_CPUPID_SHIFT #else #define LAST_CPUPID_WIDTH 0 @@ -100,10 +101,15 @@ #define LAST_CPUPID_NOT_IN_PAGE_FLAGS #endif -#if ZONES_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH \ - > BITS_PER_LONG - NR_PAGEFLAGS +#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \ + KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS #error "Not enough bits in page flags" #endif +/* see the comment on MAX_NR_TIERS */ +#define LRU_REFS_WIDTH min(__LRU_REFS_WIDTH, BITS_PER_LONG - NR_PAGEFLAGS - \ + ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \ + NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH) + #endif #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 465ff35a8c00a80c4540cdd7e8a1441b97137fda..0b0ae5084e60c7f8b784a239c718e2a418566f89 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -1058,7 +1058,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) 1UL << PG_private | 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ 1UL << PG_slab | 1UL << PG_active | \ - 1UL << PG_unevictable | __PG_MLOCKED) + 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) /* * Flags checked when a page is prepped for return by the page allocator. @@ -1069,7 +1069,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) * alloc-free cycle to prevent from reusing the page. */ #define PAGE_FLAGS_CHECK_AT_PREP \ - (PAGEFLAGS_MASK & ~__PG_HWPOISON) + ((PAGEFLAGS_MASK & ~__PG_HWPOISON) | LRU_GEN_MASK | LRU_REFS_MASK) #define PAGE_FLAGS_PRIVATE \ (1UL << PG_private | 1UL << PG_private_2) diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index 679591301994d316062f92b275efa2459a8349c9..c141ea9a95ef86c9eea582d1872485e73321914e 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -3,15 +3,17 @@ #define _LINUX_PAGE_COUNTER_H #include +#include #include #include struct page_counter { + /* + * Make sure 'usage' does not share cacheline with any other field. The + * memcg->memory.usage is a hot member of struct mem_cgroup. + */ atomic_long_t usage; - unsigned long min; - unsigned long low; - unsigned long high; - unsigned long max; + CACHELINE_PADDING(_pad1_); /* effective memory.min and memory.min usage tracking */ unsigned long emin; @@ -23,18 +25,18 @@ struct page_counter { atomic_long_t low_usage; atomic_long_t children_low_usage; - /* legacy */ unsigned long watermark; unsigned long failcnt; - /* - * 'parent' is placed here to be far from 'usage' to reduce - * cache false sharing, as 'usage' is written mostly while - * parent is frequently read for cgroup's hierarchical - * counting nature. - */ + /* Keep all the read most fields in a separete cacheline. */ + CACHELINE_PADDING(_pad2_); + + unsigned long min; + unsigned long low; + unsigned long high; + unsigned long max; struct page_counter *parent; -}; +} ____cacheline_internodealigned_in_smp; #if BITS_PER_LONG == 32 #define PAGE_COUNTER_MAX LONG_MAX diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index fabb2e1e087f49923cfce32bcc063284161214c1..22be4582faaeddb6bd3043216a808ece387b5a8c 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -36,9 +36,15 @@ struct page_ext { unsigned long flags; }; +extern bool early_page_ext; extern unsigned long page_ext_size; extern void pgdat_page_ext_init(struct pglist_data *pgdat); +static inline bool early_page_ext_enabled(void) +{ + return early_page_ext; +} + #ifdef CONFIG_SPARSEMEM static inline void page_ext_init_flatmem(void) { @@ -55,7 +61,8 @@ static inline void page_ext_init(void) } #endif -struct page_ext *lookup_page_ext(const struct page *page); +extern struct page_ext *page_ext_get(struct page *page); +extern void page_ext_put(struct page_ext *page_ext); static inline struct page_ext *page_ext_next(struct page_ext *curr) { @@ -67,13 +74,13 @@ static inline struct page_ext *page_ext_next(struct page_ext *curr) #else /* !CONFIG_PAGE_EXTENSION */ struct page_ext; -static inline void pgdat_page_ext_init(struct pglist_data *pgdat) +static inline bool early_page_ext_enabled(void) { + return false; } -static inline struct page_ext *lookup_page_ext(const struct page *page) +static inline void pgdat_page_ext_init(struct pglist_data *pgdat) { - return NULL; } static inline void page_ext_init(void) @@ -87,5 +94,14 @@ static inline void page_ext_init_flatmem_late(void) static inline void page_ext_init_flatmem(void) { } + +static inline struct page_ext *page_ext_get(struct page *page) +{ + return NULL; +} + +static inline void page_ext_put(struct page_ext *page_ext) +{ +} #endif /* CONFIG_PAGE_EXTENSION */ #endif /* __LINUX_PAGE_EXT_H */ diff --git a/include/linux/page_idle.h b/include/linux/page_idle.h index 4663dfed12931a59210cb429f6851f811b66064f..5cb7bd2078ecf8a6bb33549698396cdab85e77dd 100644 --- a/include/linux/page_idle.h +++ b/include/linux/page_idle.h @@ -13,65 +13,79 @@ * If there is not enough space to store Idle and Young bits in page flags, use * page ext flags instead. */ - static inline bool folio_test_young(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); + bool page_young; if (unlikely(!page_ext)) return false; - return test_bit(PAGE_EXT_YOUNG, &page_ext->flags); + page_young = test_bit(PAGE_EXT_YOUNG, &page_ext->flags); + page_ext_put(page_ext); + + return page_young; } static inline void folio_set_young(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); if (unlikely(!page_ext)) return; set_bit(PAGE_EXT_YOUNG, &page_ext->flags); + page_ext_put(page_ext); } static inline bool folio_test_clear_young(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); + bool page_young; if (unlikely(!page_ext)) return false; - return test_and_clear_bit(PAGE_EXT_YOUNG, &page_ext->flags); + page_young = test_and_clear_bit(PAGE_EXT_YOUNG, &page_ext->flags); + page_ext_put(page_ext); + + return page_young; } static inline bool folio_test_idle(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); + bool page_idle; if (unlikely(!page_ext)) return false; - return test_bit(PAGE_EXT_IDLE, &page_ext->flags); + page_idle = test_bit(PAGE_EXT_IDLE, &page_ext->flags); + page_ext_put(page_ext); + + return page_idle; } static inline void folio_set_idle(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); if (unlikely(!page_ext)) return; set_bit(PAGE_EXT_IDLE, &page_ext->flags); + page_ext_put(page_ext); } static inline void folio_clear_idle(struct folio *folio) { - struct page_ext *page_ext = lookup_page_ext(&folio->page); + struct page_ext *page_ext = page_ext_get(&folio->page); if (unlikely(!page_ext)) return; clear_bit(PAGE_EXT_IDLE, &page_ext->flags); + page_ext_put(page_ext); } #endif /* !CONFIG_64BIT */ diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h index 83c7248053a1eb745e0bff410be66d0f04a5e8bb..5f1ae07d724b88ddafadb30c100a6c1a9c2b41ef 100644 --- a/include/linux/pageblock-flags.h +++ b/include/linux/pageblock-flags.h @@ -53,6 +53,10 @@ extern unsigned int pageblock_order; #endif /* CONFIG_HUGETLB_PAGE */ #define pageblock_nr_pages (1UL << pageblock_order) +#define pageblock_align(pfn) ALIGN((pfn), pageblock_nr_pages) +#define pageblock_aligned(pfn) IS_ALIGNED((pfn), pageblock_nr_pages) +#define pageblock_start_pfn(pfn) ALIGN_DOWN((pfn), pageblock_nr_pages) +#define pageblock_end_pfn(pfn) ALIGN((pfn) + 1, pageblock_nr_pages) /* Forward declaration */ struct page; diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0178b2040ea38283fb12c7377759b679545a507f..bbccb404422247ff79d73b67d19cdf6269c0e242 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -718,8 +718,8 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index) unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch); -unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start, - unsigned int nr_pages, struct page **pages); +unsigned filemap_get_folios_contig(struct address_space *mapping, + pgoff_t *start, pgoff_t end, struct folio_batch *fbatch); unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, pgoff_t end, xa_mark_t tag, unsigned int nr_pages, struct page **pages); @@ -989,19 +989,16 @@ static inline int lock_page_killable(struct page *page) } /* - * lock_page_or_retry - Lock the page, unless this would block and the + * folio_lock_or_retry - Lock the folio, unless this would block and the * caller indicated that it can handle a retry. * * Return value and mmap_lock implications depend on flags; see * __folio_lock_or_retry(). */ -static inline bool lock_page_or_retry(struct page *page, struct mm_struct *mm, - unsigned int flags) +static inline bool folio_lock_or_retry(struct folio *folio, + struct mm_struct *mm, unsigned int flags) { - struct folio *folio; might_sleep(); - - folio = page_folio(page); return folio_trylock(folio) || __folio_lock_or_retry(folio, mm, flags); } @@ -1042,7 +1039,6 @@ static inline int wait_on_page_locked_killable(struct page *page) return folio_wait_locked_killable(page_folio(page)); } -int folio_put_wait_locked(struct folio *folio, int state); void wait_on_page_writeback(struct page *page); void folio_wait_writeback(struct folio *folio); int folio_wait_writeback_killable(struct folio *folio); @@ -1173,6 +1169,8 @@ struct readahead_control { pgoff_t _index; unsigned int _nr_pages; unsigned int _batch_count; + bool _workingset; + unsigned long _pflags; }; #define DEFINE_READAHEAD(ractl, f, r, m, i) \ diff --git a/include/linux/pagewalk.h b/include/linux/pagewalk.h index ac7b38ad59036d573ce56e5211e675728af5670a..f3fafb731ffd87b0e1e1a4c8dae65764bd05f6f5 100644 --- a/include/linux/pagewalk.h +++ b/include/linux/pagewalk.h @@ -15,12 +15,12 @@ struct mm_walk; * this handler is required to be able to handle * pmd_trans_huge() pmds. They may simply choose to * split_huge_page() instead of handling it explicitly. - * @pte_entry: if set, called for each non-empty PTE (lowest-level) - * entry + * @pte_entry: if set, called for each PTE (lowest-level) entry, + * including empty ones * @pte_hole: if set, called for each hole at all levels, - * depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD - * 4:PTE. Any folded depths (where PTRS_PER_P?D is equal - * to 1) are skipped. + * depth is -1 if not known, 0:PGD, 1:P4D, 2:PUD, 3:PMD. + * Any folded depths (where PTRS_PER_P?D is equal to 1) + * are skipped. * @hugetlb_entry: if set, called for each hugetlb entry * @test_walk: caller specific callback function to determine whether * we walk over the current vma or not. Returning 0 means diff --git a/include/linux/pci.h b/include/linux/pci.h index 060af91bafcd415411f5d4c23ceb51c75ceed639..2bda4a4e47e815d30d68eb52da827c15e0417cde 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -475,6 +475,7 @@ struct pci_dev { unsigned int broken_cmd_compl:1; /* No compl for some cmds */ #endif #ifdef CONFIG_PCIE_PTM + u16 ptm_cap; /* PTM Capability */ unsigned int ptm_root:1; unsigned int ptm_enabled:1; u8 ptm_granularity; @@ -1677,10 +1678,12 @@ bool pci_ats_disabled(void); #ifdef CONFIG_PCIE_PTM int pci_enable_ptm(struct pci_dev *dev, u8 *granularity); +void pci_disable_ptm(struct pci_dev *dev); bool pcie_ptm_enabled(struct pci_dev *dev); #else static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity) { return -EINVAL; } +static inline void pci_disable_ptm(struct pci_dev *dev) { } static inline bool pcie_ptm_enabled(struct pci_dev *dev) { return false; } #endif @@ -2019,8 +2022,8 @@ enum pci_fixup_pass { #ifdef CONFIG_LTO_CLANG #define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ class_shift, hook, stub) \ - void __cficanonical stub(struct pci_dev *dev); \ - void __cficanonical stub(struct pci_dev *dev) \ + void stub(struct pci_dev *dev); \ + void stub(struct pci_dev *dev) \ { \ hook(dev); \ } \ diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 6feade66efdbdccf28e85ecb9e37bfd8aa80e227..b362d90eb9b0bfe5af5e2a42e249a437a6054fd8 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -75,6 +75,9 @@ #define PCI_CLASS_COMMUNICATION_MODEM 0x0703 #define PCI_CLASS_COMMUNICATION_OTHER 0x0780 +/* Interface for SERIAL/MODEM */ +#define PCI_SERIAL_16550_COMPATIBLE 0x02 + #define PCI_BASE_CLASS_SYSTEM 0x08 #define PCI_CLASS_SYSTEM_PIC 0x0800 #define PCI_CLASS_SYSTEM_PIC_IOAPIC 0x080010 @@ -2079,6 +2082,9 @@ #define PCI_DEVICE_ID_ICE_1712 0x1712 #define PCI_DEVICE_ID_VT1724 0x1724 +#define PCI_VENDOR_ID_MICROSOFT 0x1414 +#define PCI_DEVICE_ID_HYPERV_VIDEO 0x5353 + #define PCI_VENDOR_ID_OXSEMI 0x1415 #define PCI_DEVICE_ID_OXSEMI_12PCI840 0x8403 #define PCI_DEVICE_ID_OXSEMI_PCIe840 0xC000 diff --git a/include/linux/pcs-altera-tse.h b/include/linux/pcs-altera-tse.h new file mode 100644 index 0000000000000000000000000000000000000000..92ab9f08e8359b194641936a239e798b498d238b --- /dev/null +++ b/include/linux/pcs-altera-tse.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 Bootlin + * + * Maxime Chevallier + */ + +#ifndef __LINUX_PCS_ALTERA_TSE_H +#define __LINUX_PCS_ALTERA_TSE_H + +struct phylink_pcs; +struct net_device; + +struct phylink_pcs *alt_tse_pcs_create(struct net_device *ndev, + void __iomem *pcs_base, int reg_width); + +#endif /* __LINUX_PCS_ALTERA_TSE_H */ diff --git a/include/linux/pe.h b/include/linux/pe.h index daf09ffffe38992658a7ec25a5c15a6b376de389..1d3836ef9d92dcd89935faedf76899daf8cdb7e3 100644 --- a/include/linux/pe.h +++ b/include/linux/pe.h @@ -65,6 +65,8 @@ #define IMAGE_FILE_MACHINE_SH5 0x01a8 #define IMAGE_FILE_MACHINE_THUMB 0x01c2 #define IMAGE_FILE_MACHINE_WCEMIPSV2 0x0169 +#define IMAGE_FILE_MACHINE_LOONGARCH32 0x6232 +#define IMAGE_FILE_MACHINE_LOONGARCH64 0x6264 /* flags */ #define IMAGE_FILE_RELOCS_STRIPPED 0x0001 diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index 5fda40f97fe916043b8217e8fd8b6a3ccdb0f26f..36b942b67b7d66ab031d28ad16de10164e8d6a0e 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -121,9 +121,15 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) preempt_enable(); } +extern bool percpu_is_read_locked(struct percpu_rw_semaphore *); extern void percpu_down_write(struct percpu_rw_semaphore *); extern void percpu_up_write(struct percpu_rw_semaphore *); +static inline bool percpu_is_write_locked(struct percpu_rw_semaphore *sem) +{ + return atomic_read(&sem->block); +} + extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, const char *, struct lock_class_key *); diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h index 01861eebed79d9401ff83f997c28f2fb57e983b0..8ed5fba6d156fedd35209d026a692c58afae9d41 100644 --- a/include/linux/percpu_counter.h +++ b/include/linux/percpu_counter.h @@ -15,6 +15,9 @@ #include #include +/* percpu_counter batch for local add or sub */ +#define PERCPU_COUNTER_LOCAL_BATCH INT_MAX + #ifdef CONFIG_SMP struct percpu_counter { @@ -56,6 +59,22 @@ static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) percpu_counter_add_batch(fbc, amount, percpu_counter_batch); } +/* + * With percpu_counter_add_local() and percpu_counter_sub_local(), counts + * are accumulated in local per cpu counter and not in fbc->count until + * local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter + * write efficient. + * But percpu_counter_sum(), instead of percpu_counter_read(), needs to be + * used to add up the counts from each CPU to account for all the local + * counts. So percpu_counter_add_local() and percpu_counter_sub_local() + * should be used when a counter is updated frequently and read rarely. + */ +static inline void +percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) +{ + percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH); +} + static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) { s64 ret = __percpu_counter_sum(fbc); @@ -138,6 +157,13 @@ percpu_counter_add(struct percpu_counter *fbc, s64 amount) preempt_enable(); } +/* non-SMP percpu_counter_add_local is the same with percpu_counter_add */ +static inline void +percpu_counter_add_local(struct percpu_counter *fbc, s64 amount) +{ + percpu_counter_add(fbc, amount); +} + static inline void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch) { @@ -193,4 +219,10 @@ static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) percpu_counter_add(fbc, -amount); } +static inline void +percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount) +{ + percpu_counter_add_local(fbc, -amount); +} + #endif /* _LINUX_PERCPU_COUNTER_H */ diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 0407a38b470ab7ccdb4792957ed179dd8ac15e5d..0356cb6a215d870695f5fbbd8f5b0827bc99e6e5 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -24,10 +24,11 @@ /* * ARM PMU hw_event flags */ -/* Event uses a 64bit counter */ -#define ARMPMU_EVT_64BIT 1 -/* Event uses a 47bit counter */ -#define ARMPMU_EVT_47BIT 2 +#define ARMPMU_EVT_64BIT 0x00001 /* Event uses a 64bit counter */ +#define ARMPMU_EVT_47BIT 0x00002 /* Event uses a 47bit counter */ + +static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_64BIT) == ARMPMU_EVT_64BIT); +static_assert((PERF_EVENT_FLAG_ARCH & ARMPMU_EVT_47BIT) == ARMPMU_EVT_47BIT); #define HW_OP_UNSUPPORTED 0xFFFF #define C(_x) PERF_COUNT_HW_CACHE_##_x diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index bf66fe011fa81d5cf123c31485be2d93bf0cb9f6..e17e86ad6f3a24cb26a125541f7d5787bd66869f 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -45,7 +45,7 @@ struct riscv_pmu { irqreturn_t (*handle_irq)(int irq_num, void *dev); - int num_counters; + unsigned long cmask; u64 (*ctr_read)(struct perf_event *event); int (*ctr_get_idx)(struct perf_event *event); int (*ctr_get_width)(int idx); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index ee8b9ecdc03b752d0aa093f7d5446b01beb4e077..853f64b6c8c2cd9e0d3840a1da96379de466fdf1 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -36,6 +36,7 @@ struct perf_guest_info_callbacks { }; #ifdef CONFIG_HAVE_HW_BREAKPOINT +#include #include #endif @@ -60,6 +61,7 @@ struct perf_guest_info_callbacks { #include #include #include +#include #include struct perf_callchain_entry { @@ -137,9 +139,11 @@ struct hw_perf_event_extra { * PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific * usage. */ -#define PERF_EVENT_FLAG_ARCH 0x0000ffff +#define PERF_EVENT_FLAG_ARCH 0x000fffff #define PERF_EVENT_FLAG_USER_READ_CNT 0x80000000 +static_assert((PERF_EVENT_FLAG_USER_READ_CNT & PERF_EVENT_FLAG_ARCH) == 0); + /** * struct hw_perf_event - performance event hardware details: */ @@ -178,7 +182,7 @@ struct hw_perf_event { * creation and event initalization. */ struct arch_hw_breakpoint info; - struct list_head bp_list; + struct rhlist_head bp_list; }; #endif struct { /* amd_iommu */ @@ -631,7 +635,23 @@ struct pmu_event_list { struct list_head list; }; +/* + * event->sibling_list is modified whole holding both ctx->lock and ctx->mutex + * as such iteration must hold either lock. However, since ctx->lock is an IRQ + * safe lock, and is only held by the CPU doing the modification, having IRQs + * disabled is sufficient since it will hold-off the IPIs. + */ +#ifdef CONFIG_PROVE_LOCKING +#define lockdep_assert_event_ctx(event) \ + WARN_ON_ONCE(__lockdep_enabled && \ + (this_cpu_read(hardirqs_enabled) && \ + lockdep_is_held(&(event)->ctx->mutex) != LOCK_STATE_HELD)) +#else +#define lockdep_assert_event_ctx(event) +#endif + #define for_each_sibling_event(sibling, event) \ + lockdep_assert_event_ctx(event); \ if ((event)->group_leader == (event)) \ list_for_each_entry((sibling), &(event)->sibling_list, sibling_list) @@ -1007,18 +1027,20 @@ struct perf_sample_data { * Fields set by perf_sample_data_init(), group so as to * minimize the cachelines touched. */ - u64 addr; - struct perf_raw_record *raw; - struct perf_branch_stack *br_stack; + u64 sample_flags; u64 period; - union perf_sample_weight weight; - u64 txn; - union perf_mem_data_src data_src; /* * The other fields, optionally {set,used} by * perf_{prepare,output}_sample(). */ + struct perf_branch_stack *br_stack; + union perf_sample_weight weight; + union perf_mem_data_src data_src; + u64 txn; + u64 addr; + struct perf_raw_record *raw; + u64 type; u64 ip; struct { @@ -1056,13 +1078,13 @@ static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr, u64 period) { /* remaining struct members initialized in perf_prepare_sample() */ - data->addr = addr; - data->raw = NULL; - data->br_stack = NULL; + data->sample_flags = PERF_SAMPLE_PERIOD; data->period = period; - data->weight.full = 0; - data->data_src.val = PERF_MEM_NA; - data->txn = 0; + + if (addr) { + data->addr = addr; + data->sample_flags |= PERF_SAMPLE_ADDR; + } } /* @@ -1078,6 +1100,7 @@ static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *b br->abort = 0; br->cycles = 0; br->type = 0; + br->spec = PERF_BR_SPEC_NA; br->reserved = 0; } @@ -1684,4 +1707,30 @@ static inline void perf_lopwr_cb(bool mode) } #endif +#ifdef CONFIG_PERF_EVENTS +static inline bool branch_sample_no_flags(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS; +} + +static inline bool branch_sample_no_cycles(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES; +} + +static inline bool branch_sample_type(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_TYPE_SAVE; +} + +static inline bool branch_sample_hw_index(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; +} + +static inline bool branch_sample_priv(const struct perf_event *event) +{ + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE; +} +#endif /* CONFIG_PERF_EVENTS */ #endif /* _LINUX_PERF_EVENT_H */ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 014ee8f0fbaabc2d7a409abbf0a11be7a604f408..a108b60a6962b343d5569b05388d6e8d5af14254 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -213,7 +213,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, #endif #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) @@ -234,7 +234,7 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, BUILD_BUG(); return 0; } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ #endif #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH @@ -260,6 +260,19 @@ static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif +#ifndef arch_has_hw_pte_young +/* + * Return whether the accessed bit is supported on the local CPU. + * + * This stub assumes accessing through an old PTE triggers a page fault. + * Architectures that automatically set the access bit should overwrite it. + */ +static inline bool arch_has_hw_pte_young(void) +{ + return false; +} +#endif + #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, @@ -1276,8 +1289,7 @@ static inline int pgd_devmap(pgd_t pgd) #endif #if !defined(CONFIG_TRANSPARENT_HUGEPAGE) || \ - (defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ - !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)) + !defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) static inline int pud_trans_huge(pud_t pud) { return 0; @@ -1598,11 +1610,7 @@ typedef unsigned int pgtbl_mod_mask; #endif #ifndef has_transparent_hugepage -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -#define has_transparent_hugepage() 1 -#else -#define has_transparent_hugepage() 0 -#endif +#define has_transparent_hugepage() IS_BUILTIN(CONFIG_TRANSPARENT_HUGEPAGE) #endif /* diff --git a/include/linux/phy.h b/include/linux/phy.h index 87638c55d844209cc895491c606980696accdf0e..ddf66198f75110ed8420c9f9ad21527842d29aaf 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -115,6 +115,8 @@ extern const int phy_10gbit_features_array[1]; * @PHY_INTERFACE_MODE_25GBASER: 25G BaseR * @PHY_INTERFACE_MODE_USXGMII: Universal Serial 10GE MII * @PHY_INTERFACE_MODE_10GKR: 10GBASE-KR - with Clause 73 AN + * @PHY_INTERFACE_MODE_QUSGMII: Quad Universal SGMII + * @PHY_INTERFACE_MODE_1000BASEKX: 1000Base-KX - with Clause 73 AN * @PHY_INTERFACE_MODE_MAX: Book keeping * * Describes the interface between the MAC and PHY. @@ -152,6 +154,8 @@ typedef enum { PHY_INTERFACE_MODE_USXGMII, /* 10GBASE-KR - with Clause 73 AN */ PHY_INTERFACE_MODE_10GKR, + PHY_INTERFACE_MODE_QUSGMII, + PHY_INTERFACE_MODE_1000BASEKX, PHY_INTERFACE_MODE_MAX, } phy_interface_t; @@ -249,6 +253,8 @@ static inline const char *phy_modes(phy_interface_t interface) return "trgmii"; case PHY_INTERFACE_MODE_1000BASEX: return "1000base-x"; + case PHY_INTERFACE_MODE_1000BASEKX: + return "1000base-kx"; case PHY_INTERFACE_MODE_2500BASEX: return "2500base-x"; case PHY_INTERFACE_MODE_5GBASER: @@ -267,12 +273,13 @@ static inline const char *phy_modes(phy_interface_t interface) return "10gbase-kr"; case PHY_INTERFACE_MODE_100BASEX: return "100base-x"; + case PHY_INTERFACE_MODE_QUSGMII: + return "qusgmii"; default: return "unknown"; } } - #define PHY_INIT_TIMEOUT 100000 #define PHY_FORCE_TIMEOUT 10 @@ -564,8 +571,10 @@ struct macsec_ops; * @advertising: Currently advertised linkmodes * @adv_old: Saved advertised while power saving for WoL * @lp_advertising: Current link partner advertised linkmodes + * @host_interfaces: PHY interface modes supported by host * @eee_broken_modes: Energy efficient ethernet modes which should be prohibited * @autoneg: Flag autoneg being used + * @rate_matching: Current rate matching mode * @link: Current link state * @autoneg_complete: Flag auto negotiation of the link has completed * @mdix: Current crossover @@ -588,6 +597,7 @@ struct macsec_ops; * @master_slave_get: Current master/slave advertisement * @master_slave_state: Current master/slave configuration * @mii_ts: Pointer to time stamper callbacks + * @psec: Pointer to Power Sourcing Equipment control struct * @lock: Mutex for serialization access to PHY * @state_queue: Work queue for state machine * @shared: Pointer to private data shared by phys in one package @@ -633,6 +643,8 @@ struct phy_device { unsigned irq_suspended:1; unsigned irq_rerun:1; + int rate_matching; + enum phy_state state; u32 dev_flags; @@ -660,6 +672,9 @@ struct phy_device { /* used with phy_speed_down */ __ETHTOOL_DECLARE_LINK_MODE_MASK(adv_old); + /* Host supported PHY interface types. Should be ignored if empty. */ + DECLARE_PHY_INTERFACE_MASK(host_interfaces); + /* Energy efficient ethernet modes which should be prohibited */ u32 eee_broken_modes; @@ -701,6 +716,7 @@ struct phy_device { struct phylink *phylink; struct net_device *attached_dev; struct mii_timestamper *mii_ts; + struct pse_control *psec; u8 mdix; u8 mdix_ctrl; @@ -797,6 +813,21 @@ struct phy_driver { */ int (*get_features)(struct phy_device *phydev); + /** + * @get_rate_matching: Get the supported type of rate matching for a + * particular phy interface. This is used by phy consumers to determine + * whether to advertise lower-speed modes for that interface. It is + * assumed that if a rate matching mode is supported on an interface, + * then that interface's rate can be adapted to all slower link speeds + * supported by the phy. If iface is %PHY_INTERFACE_MODE_NA, and the phy + * supports any kind of rate matching for any interface, then it must + * return that rate matching mode (preferring %RATE_MATCH_PAUSE to + * %RATE_MATCH_CRS). If the interface is not supported, this should + * return %RATE_MATCH_NONE. + */ + int (*get_rate_matching)(struct phy_device *phydev, + phy_interface_t iface); + /* PHY Power Management */ /** @suspend: Suspend the hardware, saving state if needed */ int (*suspend)(struct phy_device *phydev); @@ -963,6 +994,9 @@ struct phy_fixup { const char *phy_speed_to_str(int speed); const char *phy_duplex_to_str(unsigned int duplex); +const char *phy_rate_matching_to_str(int rate_matching); + +int phy_interface_num_ports(phy_interface_t interface); /* A structure for mapping a particular speed and duplex * combination to a particular SUPPORTED and ADVERTISED value @@ -1677,6 +1711,8 @@ int phy_disable_interrupts(struct phy_device *phydev); void phy_request_interrupt(struct phy_device *phydev); void phy_free_interrupt(struct phy_device *phydev); void phy_print_status(struct phy_device *phydev); +int phy_get_rate_matching(struct phy_device *phydev, + phy_interface_t iface); void phy_set_max_speed(struct phy_device *phydev, u32 max_speed); void phy_remove_link_mode(struct phy_device *phydev, u32 link_mode); void phy_advertise_supported(struct phy_device *phydev); diff --git a/include/linux/phy/pcie.h b/include/linux/phy/pcie.h new file mode 100644 index 0000000000000000000000000000000000000000..e7ac817645764da58f5f5cec9677c9f9c2170d15 --- /dev/null +++ b/include/linux/phy/pcie.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Rockchip Electronics Co., Ltd. + */ +#ifndef __PHY_PCIE_H +#define __PHY_PCIE_H + +#define PHY_MODE_PCIE_RC 20 +#define PHY_MODE_PCIE_EP 21 +#define PHY_MODE_PCIE_BIFURCATION 22 + +#endif diff --git a/include/linux/phy/tegra/xusb.h b/include/linux/phy/tegra/xusb.h index 3a35e74cdc6179473fda48a074f41b218253a321..70998e6dd6fdc9ea5665d8e01615709b042d6b69 100644 --- a/include/linux/phy/tegra/xusb.h +++ b/include/linux/phy/tegra/xusb.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (c) 2016-2020, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved. */ #ifndef PHY_TEGRA_XUSB_H @@ -21,6 +21,8 @@ int tegra_xusb_padctl_usb3_set_lfps_detect(struct tegra_xusb_padctl *padctl, unsigned int port, bool enable); int tegra_xusb_padctl_set_vbus_override(struct tegra_xusb_padctl *padctl, bool val); +void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy); +void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy); int tegra_phy_xusb_utmi_port_reset(struct phy *phy); int tegra_xusb_padctl_get_usb3_companion(struct tegra_xusb_padctl *padctl, unsigned int port); diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 6d06896fc20d8d59c28e4a08052dfea6bb51d2c6..664dd409feb93bfcad54f8ec2f8c449d6e1c21d0 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -21,6 +21,35 @@ enum { MLO_AN_FIXED, /* Fixed-link mode */ MLO_AN_INBAND, /* In-band protocol */ + /* MAC_SYM_PAUSE and MAC_ASYM_PAUSE are used when configuring our + * autonegotiation advertisement. They correspond to the PAUSE and + * ASM_DIR bits defined by 802.3, respectively. + * + * The following table lists the values of tx_pause and rx_pause which + * might be requested in mac_link_up. The exact values depend on either + * the results of autonegotation (if MLO_PAUSE_AN is set) or user + * configuration (if MLO_PAUSE_AN is not set). + * + * MAC_SYM_PAUSE MAC_ASYM_PAUSE MLO_PAUSE_AN tx_pause/rx_pause + * ============= ============== ============ ================== + * 0 0 0 0/0 + * 0 0 1 0/0 + * 0 1 0 0/0, 0/1, 1/0, 1/1 + * 0 1 1 0/0, 1/0 + * 1 0 0 0/0, 1/1 + * 1 0 1 0/0, 1/1 + * 1 1 0 0/0, 0/1, 1/0, 1/1 + * 1 1 1 0/0, 0/1, 1/1 + * + * If you set MAC_ASYM_PAUSE, the user may request any combination of + * tx_pause and rx_pause. You do not have to support these + * combinations. + * + * However, you should support combinations of tx_pause and rx_pause + * which might be the result of autonegotation. For example, don't set + * MAC_SYM_PAUSE unless your device can support tx_pause and rx_pause + * at the same time. + */ MAC_SYM_PAUSE = BIT(0), MAC_ASYM_PAUSE = BIT(1), MAC_10HD = BIT(2), @@ -59,6 +88,10 @@ static inline bool phylink_autoneg_inband(unsigned int mode) * @speed: link speed, one of the SPEED_* constants. * @duplex: link duplex mode, one of DUPLEX_* constants. * @pause: link pause state, described by MLO_PAUSE_* constants. + * @rate_matching: rate matching being performed, one of the RATE_MATCH_* + * constants. If rate matching is taking place, then the speed/duplex of + * the medium link mode (@speed and @duplex) and the speed/duplex of the phy + * interface mode (@interface) are different. * @link: true if the link is up. * @an_enabled: true if autonegotiation is enabled/desired. * @an_complete: true if autonegotiation has completed. @@ -70,6 +103,7 @@ struct phylink_link_state { int speed; int duplex; int pause; + int rate_matching; unsigned int link:1; unsigned int an_enabled:1; unsigned int an_complete:1; @@ -518,8 +552,10 @@ void pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex); #endif -void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, - unsigned long mac_capabilities); +void phylink_caps_to_linkmodes(unsigned long *linkmodes, unsigned long caps); +unsigned long phylink_get_capabilities(phy_interface_t interface, + unsigned long mac_capabilities, + int rate_matching); void phylink_generic_validate(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state); diff --git a/include/linux/platform_data/adp5588.h b/include/linux/platform_data/adp5588.h deleted file mode 100644 index 6d3f7d911a92e9bdfa33a6c893ed8fed88da9d62..0000000000000000000000000000000000000000 --- a/include/linux/platform_data/adp5588.h +++ /dev/null @@ -1,171 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Analog Devices ADP5588 I/O Expander and QWERTY Keypad Controller - * - * Copyright 2009-2010 Analog Devices Inc. - */ - -#ifndef _ADP5588_H -#define _ADP5588_H - -#define DEV_ID 0x00 /* Device ID */ -#define CFG 0x01 /* Configuration Register1 */ -#define INT_STAT 0x02 /* Interrupt Status Register */ -#define KEY_LCK_EC_STAT 0x03 /* Key Lock and Event Counter Register */ -#define Key_EVENTA 0x04 /* Key Event Register A */ -#define Key_EVENTB 0x05 /* Key Event Register B */ -#define Key_EVENTC 0x06 /* Key Event Register C */ -#define Key_EVENTD 0x07 /* Key Event Register D */ -#define Key_EVENTE 0x08 /* Key Event Register E */ -#define Key_EVENTF 0x09 /* Key Event Register F */ -#define Key_EVENTG 0x0A /* Key Event Register G */ -#define Key_EVENTH 0x0B /* Key Event Register H */ -#define Key_EVENTI 0x0C /* Key Event Register I */ -#define Key_EVENTJ 0x0D /* Key Event Register J */ -#define KP_LCK_TMR 0x0E /* Keypad Lock1 to Lock2 Timer */ -#define UNLOCK1 0x0F /* Unlock Key1 */ -#define UNLOCK2 0x10 /* Unlock Key2 */ -#define GPIO_INT_STAT1 0x11 /* GPIO Interrupt Status */ -#define GPIO_INT_STAT2 0x12 /* GPIO Interrupt Status */ -#define GPIO_INT_STAT3 0x13 /* GPIO Interrupt Status */ -#define GPIO_DAT_STAT1 0x14 /* GPIO Data Status, Read twice to clear */ -#define GPIO_DAT_STAT2 0x15 /* GPIO Data Status, Read twice to clear */ -#define GPIO_DAT_STAT3 0x16 /* GPIO Data Status, Read twice to clear */ -#define GPIO_DAT_OUT1 0x17 /* GPIO DATA OUT */ -#define GPIO_DAT_OUT2 0x18 /* GPIO DATA OUT */ -#define GPIO_DAT_OUT3 0x19 /* GPIO DATA OUT */ -#define GPIO_INT_EN1 0x1A /* GPIO Interrupt Enable */ -#define GPIO_INT_EN2 0x1B /* GPIO Interrupt Enable */ -#define GPIO_INT_EN3 0x1C /* GPIO Interrupt Enable */ -#define KP_GPIO1 0x1D /* Keypad or GPIO Selection */ -#define KP_GPIO2 0x1E /* Keypad or GPIO Selection */ -#define KP_GPIO3 0x1F /* Keypad or GPIO Selection */ -#define GPI_EM1 0x20 /* GPI Event Mode 1 */ -#define GPI_EM2 0x21 /* GPI Event Mode 2 */ -#define GPI_EM3 0x22 /* GPI Event Mode 3 */ -#define GPIO_DIR1 0x23 /* GPIO Data Direction */ -#define GPIO_DIR2 0x24 /* GPIO Data Direction */ -#define GPIO_DIR3 0x25 /* GPIO Data Direction */ -#define GPIO_INT_LVL1 0x26 /* GPIO Edge/Level Detect */ -#define GPIO_INT_LVL2 0x27 /* GPIO Edge/Level Detect */ -#define GPIO_INT_LVL3 0x28 /* GPIO Edge/Level Detect */ -#define Debounce_DIS1 0x29 /* Debounce Disable */ -#define Debounce_DIS2 0x2A /* Debounce Disable */ -#define Debounce_DIS3 0x2B /* Debounce Disable */ -#define GPIO_PULL1 0x2C /* GPIO Pull Disable */ -#define GPIO_PULL2 0x2D /* GPIO Pull Disable */ -#define GPIO_PULL3 0x2E /* GPIO Pull Disable */ -#define CMP_CFG_STAT 0x30 /* Comparator Configuration and Status Register */ -#define CMP_CONFG_SENS1 0x31 /* Sensor1 Comparator Configuration Register */ -#define CMP_CONFG_SENS2 0x32 /* L2 Light Sensor Reference Level, Output Falling for Sensor 1 */ -#define CMP1_LVL2_TRIP 0x33 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 1 */ -#define CMP1_LVL2_HYS 0x34 /* L3 Light Sensor Reference Level, Output Falling For Sensor 1 */ -#define CMP1_LVL3_TRIP 0x35 /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 1 */ -#define CMP1_LVL3_HYS 0x36 /* Sensor 2 Comparator Configuration Register */ -#define CMP2_LVL2_TRIP 0x37 /* L2 Light Sensor Reference Level, Output Falling for Sensor 2 */ -#define CMP2_LVL2_HYS 0x38 /* L2 Light Sensor Hysteresis (Active when Output Rising) for Sensor 2 */ -#define CMP2_LVL3_TRIP 0x39 /* L3 Light Sensor Reference Level, Output Falling For Sensor 2 */ -#define CMP2_LVL3_HYS 0x3A /* L3 Light Sensor Hysteresis (Active when Output Rising) For Sensor 2 */ -#define CMP1_ADC_DAT_R1 0x3B /* Comparator 1 ADC data Register1 */ -#define CMP1_ADC_DAT_R2 0x3C /* Comparator 1 ADC data Register2 */ -#define CMP2_ADC_DAT_R1 0x3D /* Comparator 2 ADC data Register1 */ -#define CMP2_ADC_DAT_R2 0x3E /* Comparator 2 ADC data Register2 */ - -#define ADP5588_DEVICE_ID_MASK 0xF - - /* Configuration Register1 */ -#define ADP5588_AUTO_INC (1 << 7) -#define ADP5588_GPIEM_CFG (1 << 6) -#define ADP5588_OVR_FLOW_M (1 << 5) -#define ADP5588_INT_CFG (1 << 4) -#define ADP5588_OVR_FLOW_IEN (1 << 3) -#define ADP5588_K_LCK_IM (1 << 2) -#define ADP5588_GPI_IEN (1 << 1) -#define ADP5588_KE_IEN (1 << 0) - -/* Interrupt Status Register */ -#define ADP5588_CMP2_INT (1 << 5) -#define ADP5588_CMP1_INT (1 << 4) -#define ADP5588_OVR_FLOW_INT (1 << 3) -#define ADP5588_K_LCK_INT (1 << 2) -#define ADP5588_GPI_INT (1 << 1) -#define ADP5588_KE_INT (1 << 0) - -/* Key Lock and Event Counter Register */ -#define ADP5588_K_LCK_EN (1 << 6) -#define ADP5588_LCK21 0x30 -#define ADP5588_KEC 0xF - -#define ADP5588_MAXGPIO 18 -#define ADP5588_BANK(offs) ((offs) >> 3) -#define ADP5588_BIT(offs) (1u << ((offs) & 0x7)) - -/* Put one of these structures in i2c_board_info platform_data */ - -#define ADP5588_KEYMAPSIZE 80 - -#define GPI_PIN_ROW0 97 -#define GPI_PIN_ROW1 98 -#define GPI_PIN_ROW2 99 -#define GPI_PIN_ROW3 100 -#define GPI_PIN_ROW4 101 -#define GPI_PIN_ROW5 102 -#define GPI_PIN_ROW6 103 -#define GPI_PIN_ROW7 104 -#define GPI_PIN_COL0 105 -#define GPI_PIN_COL1 106 -#define GPI_PIN_COL2 107 -#define GPI_PIN_COL3 108 -#define GPI_PIN_COL4 109 -#define GPI_PIN_COL5 110 -#define GPI_PIN_COL6 111 -#define GPI_PIN_COL7 112 -#define GPI_PIN_COL8 113 -#define GPI_PIN_COL9 114 - -#define GPI_PIN_ROW_BASE GPI_PIN_ROW0 -#define GPI_PIN_ROW_END GPI_PIN_ROW7 -#define GPI_PIN_COL_BASE GPI_PIN_COL0 -#define GPI_PIN_COL_END GPI_PIN_COL9 - -#define GPI_PIN_BASE GPI_PIN_ROW_BASE -#define GPI_PIN_END GPI_PIN_COL_END - -#define ADP5588_GPIMAPSIZE_MAX (GPI_PIN_END - GPI_PIN_BASE + 1) - -struct adp5588_gpi_map { - unsigned short pin; - unsigned short sw_evt; -}; - -struct adp5588_kpad_platform_data { - int rows; /* Number of rows */ - int cols; /* Number of columns */ - const unsigned short *keymap; /* Pointer to keymap */ - unsigned short keymapsize; /* Keymap size */ - unsigned repeat:1; /* Enable key repeat */ - unsigned en_keylock:1; /* Enable Key Lock feature */ - unsigned short unlock_key1; /* Unlock Key 1 */ - unsigned short unlock_key2; /* Unlock Key 2 */ - const struct adp5588_gpi_map *gpimap; - unsigned short gpimapsize; - const struct adp5588_gpio_platform_data *gpio_data; -}; - -struct i2c_client; /* forward declaration */ - -struct adp5588_gpio_platform_data { - int gpio_start; /* GPIO Chip base # */ - const char *const *names; - unsigned irq_base; /* interrupt base # */ - unsigned pullup_dis_mask; /* Pull-Up Disable Mask */ - int (*setup)(struct i2c_client *client, - unsigned gpio, unsigned ngpio, - void *context); - int (*teardown)(struct i2c_client *client, - unsigned gpio, unsigned ngpio, - void *context); - void *context; -}; - -#endif diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 8b1b795867a1bd1083f17e3f4744badcf00dc134..5744a2d746aabd6735a81b3ea5cae0e387752e57 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -5724,8 +5724,21 @@ enum typec_control_command { TYPEC_CONTROL_COMMAND_EXIT_MODES, TYPEC_CONTROL_COMMAND_CLEAR_EVENTS, TYPEC_CONTROL_COMMAND_ENTER_MODE, + TYPEC_CONTROL_COMMAND_TBT_UFP_REPLY, + TYPEC_CONTROL_COMMAND_USB_MUX_SET, }; +/* Replies the AP may specify to the TBT EnterMode command as a UFP */ +enum typec_tbt_ufp_reply { + TYPEC_TBT_UFP_REPLY_NAK, + TYPEC_TBT_UFP_REPLY_ACK, +}; + +struct typec_usb_mux_set { + uint8_t mux_index; /* Index of the mux to set in the chain */ + uint8_t mux_flags; /* USB_PD_MUX_*-encoded USB mux state to set */ +} __ec_align1; + struct ec_params_typec_control { uint8_t port; uint8_t command; /* enum typec_control_command */ @@ -5739,6 +5752,8 @@ struct ec_params_typec_control { union { uint32_t clear_events_mask; uint8_t mode_to_enter; /* enum typec_mode */ + uint8_t tbt_ufp_reply; /* enum typec_tbt_ufp_reply */ + struct typec_usb_mux_set mux_params; uint8_t placeholder[128]; }; } __ec_align1; @@ -5817,6 +5832,9 @@ enum tcpc_cc_polarity { #define PD_STATUS_EVENT_SOP_DISC_DONE BIT(0) #define PD_STATUS_EVENT_SOP_PRIME_DISC_DONE BIT(1) #define PD_STATUS_EVENT_HARD_RESET BIT(2) +#define PD_STATUS_EVENT_DISCONNECTED BIT(3) +#define PD_STATUS_EVENT_MUX_0_SET_DONE BIT(4) +#define PD_STATUS_EVENT_MUX_1_SET_DONE BIT(5) struct ec_params_typec_status { uint8_t port; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index 408b29ca4004be01d21c23b68d8221895508d908..e43107e0bee162a736024d67a7b9934d9ec193f2 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -169,6 +169,7 @@ struct cros_ec_device { int event_size; u32 host_event_wake_mask; u32 last_resume_result; + u16 suspend_timeout_ms; ktime_t last_event_time; struct notifier_block notifier_ready; diff --git a/include/linux/platform_data/dma-hsu.h b/include/linux/platform_data/dma-hsu.h index c65b412b2b332d482f63db83c65012bbaded0105..611bae193c1ce8e275e993ad3922eacdfcbda7a4 100644 --- a/include/linux/platform_data/dma-hsu.h +++ b/include/linux/platform_data/dma-hsu.h @@ -8,7 +8,7 @@ #ifndef _PLATFORM_DATA_DMA_HSU_H #define _PLATFORM_DATA_DMA_HSU_H -#include +struct device; struct hsu_dma_slave { struct device *dma_dev; diff --git a/include/linux/platform_data/emc2305.h b/include/linux/platform_data/emc2305.h new file mode 100644 index 0000000000000000000000000000000000000000..54d672dd6f7d1302ac701d36c71e4f1c11447b46 --- /dev/null +++ b/include/linux/platform_data/emc2305.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LINUX_PLATFORM_DATA_EMC2305__ +#define __LINUX_PLATFORM_DATA_EMC2305__ + +#define EMC2305_PWM_MAX 5 + +/** + * struct emc2305_platform_data - EMC2305 driver platform data + * @max_state: maximum cooling state of the cooling device; + * @pwm_num: number of active channels; + * @pwm_separate: separate PWM settings for every channel; + * @pwm_min: array of minimum PWM per channel; + */ +struct emc2305_platform_data { + u8 max_state; + u8 pwm_num; + bool pwm_separate; + u8 pwm_min[EMC2305_PWM_MAX]; +}; + +#endif diff --git a/include/linux/platform_data/pca953x.h b/include/linux/platform_data/pca953x.h index 4eb53e02399786300f8f27f3682504a8696f83a1..96c1a14ab3657aa367ae1e9ba5a79feaa8710430 100644 --- a/include/linux/platform_data/pca953x.h +++ b/include/linux/platform_data/pca953x.h @@ -22,7 +22,7 @@ struct pca953x_platform_data { int (*setup)(struct i2c_client *client, unsigned gpio, unsigned ngpio, void *context); - int (*teardown)(struct i2c_client *client, + void (*teardown)(struct i2c_client *client, unsigned gpio, unsigned ngpio, void *context); const char *const *names; diff --git a/include/linux/platform_data/ssm2518.h b/include/linux/platform_data/ssm2518.h deleted file mode 100644 index 3f9e632d6f6330e05bee893139924f6043e8498b..0000000000000000000000000000000000000000 --- a/include/linux/platform_data/ssm2518.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * SSM2518 amplifier audio driver - * - * Copyright 2013 Analog Devices Inc. - * Author: Lars-Peter Clausen - */ - -#ifndef __LINUX_PLATFORM_DATA_SSM2518_H__ -#define __LINUX_PLATFORM_DATA_SSM2518_H__ - -/** - * struct ssm2518_platform_data - Platform data for the ssm2518 driver - * @enable_gpio: GPIO connected to the nSD pin. Set to -1 if the nSD pin is - * hardwired. - */ -struct ssm2518_platform_data { - int enable_gpio; -}; - -#endif diff --git a/include/linux/platform_data/tps68470.h b/include/linux/platform_data/tps68470.h index 126d082c3f2e191bece97f74c77a062e3bd99d76..e605a2cab07f1b730c886ce0560e170338eccc1f 100644 --- a/include/linux/platform_data/tps68470.h +++ b/include/linux/platform_data/tps68470.h @@ -27,9 +27,14 @@ struct tps68470_regulator_platform_data { const struct regulator_init_data *reg_init_data[TPS68470_NUM_REGULATORS]; }; -struct tps68470_clk_platform_data { +struct tps68470_clk_consumer { const char *consumer_dev_name; const char *consumer_con_id; }; +struct tps68470_clk_platform_data { + unsigned int n_consumers; + struct tps68470_clk_consumer consumers[]; +}; + #endif diff --git a/include/linux/platform_data/usb-s3c2410_udc.h b/include/linux/platform_data/usb-s3c2410_udc.h index 07394819d03b9af2b6d6133a3a8bb5baf37bd9eb..c0fbe1fe3426c1980bb58195e30c64d520c74014 100644 --- a/include/linux/platform_data/usb-s3c2410_udc.h +++ b/include/linux/platform_data/usb-s3c2410_udc.h @@ -22,12 +22,6 @@ enum s3c2410_udc_cmd_e { struct s3c2410_udc_mach_info { void (*udc_command)(enum s3c2410_udc_cmd_e); void (*vbus_draw)(unsigned int ma); - - unsigned int pullup_pin; - unsigned int pullup_pin_inverted; - - unsigned int vbus_pin; - unsigned char vbus_pin_inverted; }; extern void __init s3c24xx_udc_set_platdata(struct s3c2410_udc_mach_info *); diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 98f2b2f20f3e9a60db96329124c31077a166cbc7..28234dc9fa6ad1a7330afe51206b26c4b6dd4673 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -65,6 +65,7 @@ #define ASUS_WMI_DEVID_PANEL_OD 0x00050019 #define ASUS_WMI_DEVID_CAMERA 0x00060013 #define ASUS_WMI_DEVID_LID_FLIP 0x00060062 +#define ASUS_WMI_DEVID_LID_FLIP_ROG 0x00060077 /* Storage */ #define ASUS_WMI_DEVID_CARDREADER 0x00080013 @@ -78,6 +79,7 @@ #define ASUS_WMI_DEVID_THERMAL_CTRL 0x00110011 #define ASUS_WMI_DEVID_FAN_CTRL 0x00110012 /* deprecated */ #define ASUS_WMI_DEVID_CPU_FAN_CTRL 0x00110013 +#define ASUS_WMI_DEVID_GPU_FAN_CTRL 0x00110014 #define ASUS_WMI_DEVID_CPU_FAN_CURVE 0x00110024 #define ASUS_WMI_DEVID_GPU_FAN_CURVE 0x00110025 @@ -99,6 +101,15 @@ /* dgpu on/off */ #define ASUS_WMI_DEVID_DGPU 0x00090020 +/* gpu mux switch, 0 = dGPU, 1 = Optimus */ +#define ASUS_WMI_DEVID_GPU_MUX 0x00090016 + +/* TUF laptop RGB modes/colours */ +#define ASUS_WMI_DEVID_TUF_RGB_MODE 0x00100056 + +/* TUF laptop RGB power/state */ +#define ASUS_WMI_DEVID_TUF_RGB_STATE 0x00100057 + /* DSTS masks */ #define ASUS_WMI_DSTS_STATUS_BIT 0x00000001 #define ASUS_WMI_DSTS_UNKNOWN_BIT 0x00000002 diff --git a/include/linux/platform_data/x86/pmc_atom.h b/include/linux/platform_data/x86/pmc_atom.h index 3edfb6d4e67ab1e1501cfa8af2611cd7bb4a582d..b8a701c77fd00cc2d983ad69cca9ebd5a66acee9 100644 --- a/include/linux/platform_data/x86/pmc_atom.h +++ b/include/linux/platform_data/x86/pmc_atom.h @@ -1,12 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Intel Atom SOC Power Management Controller Header File - * Copyright (c) 2014, Intel Corporation. + * Intel Atom SoC Power Management Controller Header File + * Copyright (c) 2014-2015,2022 Intel Corporation. */ #ifndef PMC_ATOM_H #define PMC_ATOM_H +#include + /* ValleyView Power Control Unit PCI Device ID */ #define PCI_DEVICE_ID_VLV_PMC 0x0F1C /* CherryTrail Power Control Unit PCI Device ID */ @@ -139,9 +141,9 @@ #define ACPI_MMIO_REG_LEN 0x100 #define PM1_CNT 0x4 -#define SLEEP_TYPE_MASK 0xFFFFECFF +#define SLEEP_TYPE_MASK GENMASK(12, 10) #define SLEEP_TYPE_S5 0x1C00 -#define SLEEP_ENABLE 0x2000 +#define SLEEP_ENABLE BIT(13) extern int pmc_atom_read(int offset, u32 *value); diff --git a/include/linux/platform_data/x86/simatic-ipc-base.h b/include/linux/platform_data/x86/simatic-ipc-base.h index 39fefd48cf4d357fe37ddad269200ec6a07649b6..57d6a10dfc9e0ae532ad92979c7b649b386acbaa 100644 --- a/include/linux/platform_data/x86/simatic-ipc-base.h +++ b/include/linux/platform_data/x86/simatic-ipc-base.h @@ -19,6 +19,7 @@ #define SIMATIC_IPC_DEVICE_427E 2 #define SIMATIC_IPC_DEVICE_127E 3 #define SIMATIC_IPC_DEVICE_227E 4 +#define SIMATIC_IPC_DEVICE_227G 5 struct simatic_ipc_platform { u8 devmode; diff --git a/include/linux/platform_data/x86/simatic-ipc.h b/include/linux/platform_data/x86/simatic-ipc.h index f3b76b39776b49dadc1d0613f9c0c786d177767c..632320ec8f0826d6dfa8bafb6402aab65ae4b7b1 100644 --- a/include/linux/platform_data/x86/simatic-ipc.h +++ b/include/linux/platform_data/x86/simatic-ipc.h @@ -31,6 +31,8 @@ enum simatic_ipc_station_ids { SIMATIC_IPC_IPC427E = 0x00000A01, SIMATIC_IPC_IPC477E = 0x00000A02, SIMATIC_IPC_IPC127E = 0x00000D01, + SIMATIC_IPC_IPC227G = 0x00000F01, + SIMATIC_IPC_IPC427G = 0x00001001, }; static inline u32 simatic_ipc_get_station_id(u8 *data, int max_len) diff --git a/include/linux/pm.h b/include/linux/pm.h index 871c9c49ec9d2b08a7cfbeb76be43906719bfbb2..93cd34f00822cabfe72ccc0e25aea710d05403f6 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -375,19 +375,20 @@ const struct dev_pm_ops name = { \ } #ifdef CONFIG_PM -#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ - runtime_resume_fn, idle_fn, sec, ns) \ - _DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ - runtime_resume_fn, idle_fn); \ - __EXPORT_SYMBOL(name, sec, ns) +#define _EXPORT_DEV_PM_OPS(name, sec, ns) \ + const struct dev_pm_ops name; \ + __EXPORT_SYMBOL(name, sec, ns); \ + const struct dev_pm_ops name #else -#define _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, runtime_suspend_fn, \ - runtime_resume_fn, idle_fn, sec, ns) \ -static __maybe_unused _DEFINE_DEV_PM_OPS(__static_##name, suspend_fn, \ - resume_fn, runtime_suspend_fn, \ - runtime_resume_fn, idle_fn) +#define _EXPORT_DEV_PM_OPS(name, sec, ns) \ + static __maybe_unused const struct dev_pm_ops __static_##name #endif +#define EXPORT_DEV_PM_OPS(name) _EXPORT_DEV_PM_OPS(name, "", "") +#define EXPORT_GPL_DEV_PM_OPS(name) _EXPORT_DEV_PM_OPS(name, "_gpl", "") +#define EXPORT_NS_DEV_PM_OPS(name, ns) _EXPORT_DEV_PM_OPS(name, "", #ns) +#define EXPORT_NS_GPL_DEV_PM_OPS(name, ns) _EXPORT_DEV_PM_OPS(name, "_gpl", #ns) + /* * Use this if you want to use the same suspend and resume callbacks for suspend * to RAM and hibernation. @@ -399,13 +400,21 @@ static __maybe_unused _DEFINE_DEV_PM_OPS(__static_##name, suspend_fn, \ _DEFINE_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL) #define EXPORT_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ - _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "", "") + EXPORT_DEV_PM_OPS(name) = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + } #define EXPORT_GPL_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ - _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "_gpl", "") + EXPORT_GPL_DEV_PM_OPS(name) = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + } #define EXPORT_NS_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn, ns) \ - _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "", #ns) + EXPORT_NS_DEV_PM_OPS(name, ns) = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + } #define EXPORT_NS_GPL_SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn, ns) \ - _EXPORT_DEV_PM_OPS(name, suspend_fn, resume_fn, NULL, NULL, NULL, "_gpl", #ns) + EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \ + SYSTEM_SLEEP_PM_OPS(suspend_fn, resume_fn) \ + } /* Deprecated. Use DEFINE_SIMPLE_DEV_PM_OPS() instead. */ #define SIMPLE_DEV_PM_OPS(name, suspend_fn, resume_fn) \ diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 0a41b2dcccad545ecda628951a11713ba671e627..9a8151a2bdea6f714b8290d1e698e3165985aa1d 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -40,17 +40,21 @@ resume_fn, idle_fn) #define EXPORT_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ - _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ - suspend_fn, resume_fn, idle_fn, "", "") + EXPORT_DEV_PM_OPS(name) = { \ + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ + } #define EXPORT_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn) \ - _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ - suspend_fn, resume_fn, idle_fn, "_gpl", "") + EXPORT_GPL_DEV_PM_OPS(name) = { \ + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ + } #define EXPORT_NS_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \ - _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ - suspend_fn, resume_fn, idle_fn, "", #ns) + EXPORT_NS_DEV_PM_OPS(name, ns) = { \ + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ + } #define EXPORT_NS_GPL_RUNTIME_DEV_PM_OPS(name, suspend_fn, resume_fn, idle_fn, ns) \ - _EXPORT_DEV_PM_OPS(name, pm_runtime_force_suspend, pm_runtime_force_resume, \ - suspend_fn, resume_fn, idle_fn, "_gpl", #ns) + EXPORT_NS_GPL_DEV_PM_OPS(name, ns) = { \ + RUNTIME_PM_OPS(suspend_fn, resume_fn, idle_fn) \ + } #ifdef CONFIG_PM extern struct workqueue_struct *pm_wq; diff --git a/include/linux/poison.h b/include/linux/poison.h index d62ef5a6b4e9c624383cd92b3773a9ea44e2c500..2d3249eb0e62d4b29f03e6e03a21934f555278aa 100644 --- a/include/linux/poison.h +++ b/include/linux/poison.h @@ -81,4 +81,7 @@ /********** net/core/page_pool.c **********/ #define PP_SIGNATURE (0x40 + POISON_POINTER_DELTA) +/********** kernel/bpf/ **********/ +#define BPF_PTR_POISON ((void *)(0xeB9FUL + POISON_POINTER_DELTA)) + #endif diff --git a/include/linux/posix_acl_xattr.h b/include/linux/posix_acl_xattr.h index b6bd3eac2bcc885ebbee9f2bbcb525dd81991a17..8163dd48c430189eb6b512aa3db6df07eb0016d5 100644 --- a/include/linux/posix_acl_xattr.h +++ b/include/linux/posix_acl_xattr.h @@ -38,9 +38,6 @@ void posix_acl_fix_xattr_to_user(void *value, size_t size); void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns, const struct inode *inode, void *value, size_t size); -void posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns, - const struct inode *inode, - void *value, size_t size); #else static inline void posix_acl_fix_xattr_from_user(void *value, size_t size) { @@ -54,18 +51,15 @@ posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns, size_t size) { } -static inline void -posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns, - const struct inode *inode, void *value, - size_t size) -{ -} #endif struct posix_acl *posix_acl_from_xattr(struct user_namespace *user_ns, const void *value, size_t size); int posix_acl_to_xattr(struct user_namespace *user_ns, const struct posix_acl *acl, void *buffer, size_t size); +struct posix_acl *vfs_set_acl_prepare(struct user_namespace *mnt_userns, + struct user_namespace *fs_userns, + const void *value, size_t size); extern const struct xattr_handler posix_acl_access_xattr_handler; extern const struct xattr_handler posix_acl_default_xattr_handler; diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index cb380c1d94595231ff20bc73d845946c0bba3b32..aa2c4a7c4826da06d8ebce07a78d6c6fbc858e0a 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -374,9 +374,37 @@ struct power_supply_vbat_ri_table { * These timers should be chosen to align with the typical discharge curve * for the battery. * - * When the main CC/CV charging is complete the battery can optionally be - * maintenance charged at the voltages from this table: a table of settings is - * traversed using a slightly lower current and voltage than what is used for + * Ordinary CC/CV charging will stop charging when the charge current goes + * below charge_term_current_ua, and then restart it (if the device is still + * plugged into the charger) at charge_restart_voltage_uv. This happens in most + * consumer products because the power usage while connected to a charger is + * not zero, and devices are not manufactured to draw power directly from the + * charger: instead they will at all times dissipate the battery a little, like + * the power used in standby mode. This will over time give a charge graph + * such as this: + * + * Energy + * ^ ... ... ... ... ... ... ... + * | . . . . . . . . . . . . . + * | .. . .. . .. . .. . .. . .. . .. + * |. .. .. .. .. .. .. + * +-------------------------------------------------------------------> t + * + * Practically this means that the Li-ions are wandering back and forth in the + * battery and this causes degeneration of the battery anode and cathode. + * To prolong the life of the battery, maintenance charging is applied after + * reaching charge_term_current_ua to hold up the charge in the battery while + * consuming power, thus lowering the wear on the battery: + * + * Energy + * ^ ....................................... + * | . ...................... + * | .. + * |. + * +-------------------------------------------------------------------> t + * + * Maintenance charging uses the voltages from this table: a table of settings + * is traversed using a slightly lower current and voltage than what is used for * CC/CV charging. The maintenance charging will for safety reasons not go on * indefinately: we lower the current and voltage with successive maintenance * settings, then disable charging completely after we reach the last one, @@ -385,14 +413,22 @@ struct power_supply_vbat_ri_table { * ordinary CC/CV charging from there. * * As an example, a Samsung EB425161LA Lithium-Ion battery is CC/CV charged - * at 900mA to 4340mV, then maintenance charged at 600mA and 4150mV for - * 60 hours, then maintenance charged at 600mA and 4100mV for 200 hours. + * at 900mA to 4340mV, then maintenance charged at 600mA and 4150mV for up to + * 60 hours, then maintenance charged at 600mA and 4100mV for up to 200 hours. * After this the charge cycle is restarted waiting for * charge_restart_voltage_uv. * * For most mobile electronics this type of maintenance charging is enough for * the user to disconnect the device and make use of it before both maintenance - * charging cycles are complete. + * charging cycles are complete, if the current and voltage has been chosen + * appropriately. These need to be determined from battery discharge curves + * and expected standby current. + * + * If the voltage anyway drops to charge_restart_voltage_uv during maintenance + * charging, ordinary CC/CV charging is restarted. This can happen if the + * device is e.g. actively used during charging, so more current is drawn than + * the expected stand-by current. Also overvoltage protection will be applied + * as usual. */ struct power_supply_maintenance_charge_table { int charge_current_max_ua; diff --git a/include/linux/prandom.h b/include/linux/prandom.h index deace5fb4e628beccac22d6df991761d35646584..e0a0759dd09c0984482a9f0d208bab874e978dd2 100644 --- a/include/linux/prandom.h +++ b/include/linux/prandom.h @@ -12,16 +12,6 @@ #include #include -static inline u32 prandom_u32(void) -{ - return get_random_u32(); -} - -static inline void prandom_bytes(void *buf, size_t nbytes) -{ - return get_random_bytes(buf, nbytes); -} - struct rnd_state { __u32 s1, s2, s3, s4; }; @@ -37,17 +27,20 @@ void prandom_seed_full_state(struct rnd_state __percpu *pcpu_state); * prandom_u32_max - returns a pseudo-random number in interval [0, ep_ro) * @ep_ro: right open interval endpoint * - * Returns a pseudo-random number that is in interval [0, ep_ro). Note - * that the result depends on PRNG being well distributed in [0, ~0U] - * u32 space. Here we use maximally equidistributed combined Tausworthe - * generator, that is, prandom_u32(). This is useful when requesting a - * random index of an array containing ep_ro elements, for example. + * Returns a pseudo-random number that is in interval [0, ep_ro). This is + * useful when requesting a random index of an array containing ep_ro elements, + * for example. The result is somewhat biased when ep_ro is not a power of 2, + * so do not use this for cryptographic purposes. * * Returns: pseudo-random number in interval [0, ep_ro) */ static inline u32 prandom_u32_max(u32 ep_ro) { - return (u32)(((u64) prandom_u32() * ep_ro) >> 32); + if (__builtin_constant_p(ep_ro <= 1U << 8) && ep_ro <= 1U << 8) + return (get_random_u8() * ep_ro) >> 8; + if (__builtin_constant_p(ep_ro <= 1U << 16) && ep_ro <= 1U << 16) + return (get_random_u16() * ep_ro) >> 16; + return ((u64)get_random_u32() * ep_ro) >> 32; } /* diff --git a/include/linux/preempt.h b/include/linux/preempt.h index b4381f255a5cafe2aca6a1b73286dc62add68934..0df425bf9bd752cee27952d81b7170dfeedb7b53 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -421,4 +421,46 @@ static inline void migrate_enable(void) { } #endif /* CONFIG_SMP */ +/** + * preempt_disable_nested - Disable preemption inside a normally preempt disabled section + * + * Use for code which requires preemption protection inside a critical + * section which has preemption disabled implicitly on non-PREEMPT_RT + * enabled kernels, by e.g.: + * - holding a spinlock/rwlock + * - soft interrupt context + * - regular interrupt handlers + * + * On PREEMPT_RT enabled kernels spinlock/rwlock held sections, soft + * interrupt context and regular interrupt handlers are preemptible and + * only prevent migration. preempt_disable_nested() ensures that preemption + * is disabled for cases which require CPU local serialization even on + * PREEMPT_RT. For non-PREEMPT_RT kernels this is a NOP. + * + * The use cases are code sequences which are not serialized by a + * particular lock instance, e.g.: + * - seqcount write side critical sections where the seqcount is not + * associated to a particular lock and therefore the automatic + * protection mechanism does not work. This prevents a live lock + * against a preempting high priority reader. + * - RMW per CPU variable updates like vmstat. + */ +/* Macro to avoid header recursion hell vs. lockdep */ +#define preempt_disable_nested() \ +do { \ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) \ + preempt_disable(); \ + else \ + lockdep_assert_preemption_disabled(); \ +} while (0) + +/** + * preempt_enable_nested - Undo the effect of preempt_disable_nested() + */ +static __always_inline void preempt_enable_nested(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + preempt_enable(); +} + #endif /* __LINUX_PREEMPT_H */ diff --git a/include/linux/printk.h b/include/linux/printk.h index cf7d666ab1f8e38dec0f2edd8272def486507f01..8c81806c2e99f906184269505fcc2a549dd25a28 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -169,8 +169,6 @@ extern void __printk_safe_exit(void); #define printk_deferred_enter __printk_safe_enter #define printk_deferred_exit __printk_safe_exit -extern bool pr_flush(int timeout_ms, bool reset_on_progress); - /* * Please don't use printk_ratelimit(), because it shares ratelimiting state * with all other unrelated printk_ratelimit() callsites. Instead use @@ -221,11 +219,6 @@ static inline void printk_deferred_exit(void) { } -static inline bool pr_flush(int timeout_ms, bool reset_on_progress) -{ - return true; -} - static inline int printk_ratelimit(void) { return 0; diff --git a/include/linux/property.h b/include/linux/property.h index a5b429d623f6f3b44821c4bde7500a189489323d..117cc200c656da4c1458b882cda2447468bdc963 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -32,7 +32,7 @@ enum dev_dma_attr { DEV_DMA_COHERENT, }; -struct fwnode_handle *dev_fwnode(struct device *dev); +struct fwnode_handle *dev_fwnode(const struct device *dev); bool device_property_present(struct device *dev, const char *propname); int device_property_read_u8_array(struct device *dev, const char *propname, @@ -387,7 +387,7 @@ bool device_dma_supported(struct device *dev); enum dev_dma_attr device_get_dma_attr(struct device *dev); -const void *device_get_match_data(struct device *dev); +const void *device_get_match_data(const struct device *dev); int device_get_phy_mode(struct device *dev); int fwnode_get_phy_mode(struct fwnode_handle *fwnode); diff --git a/include/linux/pse-pd/pse.h b/include/linux/pse-pd/pse.h new file mode 100644 index 0000000000000000000000000000000000000000..fb724c65c77bcf51799a63d917fcadca2ba718ca --- /dev/null +++ b/include/linux/pse-pd/pse.h @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +// Copyright (c) 2022 Pengutronix, Oleksij Rempel + */ +#ifndef _LINUX_PSE_CONTROLLER_H +#define _LINUX_PSE_CONTROLLER_H + +#include +#include +#include + +struct phy_device; +struct pse_controller_dev; + +/** + * struct pse_control_config - PSE control/channel configuration. + * + * @admin_cotrol: set PoDL PSE admin control as described in + * IEEE 802.3-2018 30.15.1.2.1 acPoDLPSEAdminControl + */ +struct pse_control_config { + enum ethtool_podl_pse_admin_state admin_cotrol; +}; + +/** + * struct pse_control_status - PSE control/channel status. + * + * @podl_admin_state: operational state of the PoDL PSE + * functions. IEEE 802.3-2018 30.15.1.1.2 aPoDLPSEAdminState + * @podl_pw_status: power detection status of the PoDL PSE. + * IEEE 802.3-2018 30.15.1.1.3 aPoDLPSEPowerDetectionStatus: + */ +struct pse_control_status { + enum ethtool_podl_pse_admin_state podl_admin_state; + enum ethtool_podl_pse_pw_d_status podl_pw_status; +}; + +/** + * struct pse_controller_ops - PSE controller driver callbacks + * + * @ethtool_get_status: get PSE control status for ethtool interface + * @ethtool_set_config: set PSE control configuration over ethtool interface + */ +struct pse_controller_ops { + int (*ethtool_get_status)(struct pse_controller_dev *pcdev, + unsigned long id, struct netlink_ext_ack *extack, + struct pse_control_status *status); + int (*ethtool_set_config)(struct pse_controller_dev *pcdev, + unsigned long id, struct netlink_ext_ack *extack, + const struct pse_control_config *config); +}; + +struct module; +struct device_node; +struct of_phandle_args; +struct pse_control; + +/** + * struct pse_controller_dev - PSE controller entity that might + * provide multiple PSE controls + * @ops: a pointer to device specific struct pse_controller_ops + * @owner: kernel module of the PSE controller driver + * @list: internal list of PSE controller devices + * @pse_control_head: head of internal list of requested PSE controls + * @dev: corresponding driver model device struct + * @of_pse_n_cells: number of cells in PSE line specifiers + * @of_xlate: translation function to translate from specifier as found in the + * device tree to id as given to the PSE control ops + * @nr_lines: number of PSE controls in this controller device + * @lock: Mutex for serialization access to the PSE controller + */ +struct pse_controller_dev { + const struct pse_controller_ops *ops; + struct module *owner; + struct list_head list; + struct list_head pse_control_head; + struct device *dev; + int of_pse_n_cells; + int (*of_xlate)(struct pse_controller_dev *pcdev, + const struct of_phandle_args *pse_spec); + unsigned int nr_lines; + struct mutex lock; +}; + +#if IS_ENABLED(CONFIG_PSE_CONTROLLER) +int pse_controller_register(struct pse_controller_dev *pcdev); +void pse_controller_unregister(struct pse_controller_dev *pcdev); +struct device; +int devm_pse_controller_register(struct device *dev, + struct pse_controller_dev *pcdev); + +struct pse_control *of_pse_control_get(struct device_node *node); +void pse_control_put(struct pse_control *psec); + +int pse_ethtool_get_status(struct pse_control *psec, + struct netlink_ext_ack *extack, + struct pse_control_status *status); +int pse_ethtool_set_config(struct pse_control *psec, + struct netlink_ext_ack *extack, + const struct pse_control_config *config); + +#else + +static inline struct pse_control *of_pse_control_get(struct device_node *node) +{ + return ERR_PTR(-ENOENT); +} + +static inline void pse_control_put(struct pse_control *psec) +{ +} + +static inline int pse_ethtool_get_status(struct pse_control *psec, + struct netlink_ext_ack *extack, + struct pse_control_status *status) +{ + return -ENOTSUPP; +} + +static inline int pse_ethtool_set_config(struct pse_control *psec, + struct netlink_ext_ack *extack, + const struct pse_control_config *config) +{ + return -ENOTSUPP; +} + +#endif + +#endif diff --git a/include/linux/psi.h b/include/linux/psi.h index 89784763d19e2647bdfa0082652abcccec7cf038..b029a847def1e28d47f35f1cb0a2337b53a77d2c 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -7,6 +7,7 @@ #include #include #include +#include struct seq_file; struct css_set; @@ -18,25 +19,27 @@ extern struct psi_group psi_system; void psi_init(void); -void psi_task_change(struct task_struct *task, int clear, int set); -void psi_task_switch(struct task_struct *prev, struct task_struct *next, - bool sleep); - void psi_memstall_enter(unsigned long *flags); void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, - char *buf, size_t nbytes, enum psi_res res); + char *buf, enum psi_res res); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, poll_table *wait); #ifdef CONFIG_CGROUPS +static inline struct psi_group *cgroup_psi(struct cgroup *cgrp) +{ + return cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; +} + int psi_cgroup_alloc(struct cgroup *cgrp); void psi_cgroup_free(struct cgroup *cgrp); void cgroup_move_task(struct task_struct *p, struct css_set *to); +void psi_cgroup_restart(struct psi_group *group); #endif #else /* CONFIG_PSI */ @@ -58,6 +61,7 @@ static inline void cgroup_move_task(struct task_struct *p, struct css_set *to) { rcu_assign_pointer(p->cgroups, to); } +static inline void psi_cgroup_restart(struct psi_group *group) {} #endif #endif /* CONFIG_PSI */ diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h index c7fe7c08971839b518ba84d8136e3bf238675a1c..6e4372735068925c00d85e4a1ad9c1ca3bdefa6d 100644 --- a/include/linux/psi_types.h +++ b/include/linux/psi_types.h @@ -15,13 +15,6 @@ enum psi_task_count { NR_IOWAIT, NR_MEMSTALL, NR_RUNNING, - /* - * This can't have values other than 0 or 1 and could be - * implemented as a bit flag. But for now we still have room - * in the first cacheline of psi_group_cpu, and this way we - * don't have to special case any state tracking for it. - */ - NR_ONCPU, /* * For IO and CPU stalls the presence of running/oncpu tasks * in the domain means a partial rather than a full stall. @@ -32,22 +25,27 @@ enum psi_task_count { * threads and memstall ones. */ NR_MEMSTALL_RUNNING, - NR_PSI_TASK_COUNTS = 5, + NR_PSI_TASK_COUNTS = 4, }; /* Task state bitmasks */ #define TSK_IOWAIT (1 << NR_IOWAIT) #define TSK_MEMSTALL (1 << NR_MEMSTALL) #define TSK_RUNNING (1 << NR_RUNNING) -#define TSK_ONCPU (1 << NR_ONCPU) #define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING) +/* Only one task can be scheduled, no corresponding task count */ +#define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS) + /* Resources that workloads could be stalled on */ enum psi_res { PSI_IO, PSI_MEM, PSI_CPU, - NR_PSI_RESOURCES = 3, +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + PSI_IRQ, +#endif + NR_PSI_RESOURCES, }; /* @@ -63,11 +61,17 @@ enum psi_states { PSI_MEM_FULL, PSI_CPU_SOME, PSI_CPU_FULL, +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + PSI_IRQ_FULL, +#endif /* Only per-CPU, to weigh the CPU in the global average: */ PSI_NONIDLE, - NR_PSI_STATES = 7, + NR_PSI_STATES, }; +/* Use one bit in the state mask to track TSK_ONCPU */ +#define PSI_ONCPU (1 << NR_PSI_STATES) + enum psi_aggregators { PSI_AVGS = 0, PSI_POLL, @@ -147,6 +151,9 @@ struct psi_trigger { }; struct psi_group { + struct psi_group *parent; + bool enabled; + /* Protects data used by the aggregator */ struct mutex avgs_lock; @@ -188,6 +195,8 @@ struct psi_group { #else /* CONFIG_PSI */ +#define NR_PSI_RESOURCES 0 + struct psi_group { }; #endif /* CONFIG_PSI */ diff --git a/include/linux/pwm.h b/include/linux/pwm.h index 9429930c556696b38e2249200a689e8611b56374..d70c6e5a839d6902cb509966050c5ba85a51c92e 100644 --- a/include/linux/pwm.h +++ b/include/linux/pwm.h @@ -403,13 +403,9 @@ struct pwm_device *of_pwm_single_xlate(struct pwm_chip *pc, const struct of_phandle_args *args); struct pwm_device *pwm_get(struct device *dev, const char *con_id); -struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np, - const char *con_id); void pwm_put(struct pwm_device *pwm); struct pwm_device *devm_pwm_get(struct device *dev, const char *con_id); -struct pwm_device *devm_of_pwm_get(struct device *dev, struct device_node *np, - const char *con_id); struct pwm_device *devm_fwnode_pwm_get(struct device *dev, struct fwnode_handle *fwnode, const char *con_id); @@ -497,14 +493,6 @@ static inline struct pwm_device *pwm_get(struct device *dev, return ERR_PTR(-ENODEV); } -static inline struct pwm_device *of_pwm_get(struct device *dev, - struct device_node *np, - const char *con_id) -{ - might_sleep(); - return ERR_PTR(-ENODEV); -} - static inline void pwm_put(struct pwm_device *pwm) { might_sleep(); @@ -517,14 +505,6 @@ static inline struct pwm_device *devm_pwm_get(struct device *dev, return ERR_PTR(-ENODEV); } -static inline struct pwm_device *devm_of_pwm_get(struct device *dev, - struct device_node *np, - const char *con_id) -{ - might_sleep(); - return ERR_PTR(-ENODEV); -} - static inline struct pwm_device * devm_fwnode_pwm_get(struct device *dev, struct fwnode_handle *fwnode, const char *con_id) diff --git a/include/linux/random.h b/include/linux/random.h index 3fec206487f6661e830bb02a5b04a73ba39dd1e0..147a5e0d0b8ed244bf839c5ccfadb0b9352c8be7 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -38,12 +38,10 @@ static inline int unregister_random_vmfork_notifier(struct notifier_block *nb) { #endif void get_random_bytes(void *buf, size_t len); +u8 get_random_u8(void); +u16 get_random_u16(void); u32 get_random_u32(void); u64 get_random_u64(void); -static inline unsigned int get_random_int(void) -{ - return get_random_u32(); -} static inline unsigned long get_random_long(void) { #if BITS_PER_LONG == 64 @@ -72,7 +70,8 @@ static inline unsigned long get_random_canary(void) return get_random_long() & CANARY_MASK; } -int __init random_init(const char *command_line); +void __init random_init_early(const char *command_line); +void __init random_init(void); bool rng_is_initialized(void); int wait_for_random_bytes(void); @@ -93,9 +92,10 @@ static inline int get_random_bytes_wait(void *buf, size_t nbytes) *out = get_random_ ## name(); \ return 0; \ } +declare_get_random_var_wait(u8, u8) +declare_get_random_var_wait(u16, u16) declare_get_random_var_wait(u32, u32) declare_get_random_var_wait(u64, u32) -declare_get_random_var_wait(int, unsigned int) declare_get_random_var_wait(long, unsigned long) #undef declare_get_random_var diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index f527f27e643878308bc2f288bad0c07226544ffe..08605ce7379d7b0826cadaff8e6c520292a2fdfd 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -42,7 +42,31 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void rcu_barrier_tasks_rude(void); void synchronize_rcu(void); + +struct rcu_gp_oldstate; unsigned long get_completed_synchronize_rcu(void); +void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); + +// Maximum number of unsigned long values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2 + +/** + * same_state_synchronize_rcu - Are two old-state values identical? + * @oldstate1: First old-state value. + * @oldstate2: Second old-state value. + * + * The two old-state values must have been obtained from either + * get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or + * get_completed_synchronize_rcu(). Returns @true if the two values are + * identical and @false otherwise. This allows structures whose lifetimes + * are tracked by old-state values to push these values to a list header, + * allowing those structures to be slightly smaller. + */ +static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2) +{ + return oldstate1 == oldstate2; +} #ifdef CONFIG_PREEMPT_RCU @@ -496,13 +520,21 @@ do { \ * against NULL. Although rcu_access_pointer() may also be used in cases * where update-side locks prevent the value of the pointer from changing, * you should instead use rcu_dereference_protected() for this use case. + * Within an RCU read-side critical section, there is little reason to + * use rcu_access_pointer(). + * + * It is usually best to test the rcu_access_pointer() return value + * directly in order to avoid accidental dereferences being introduced + * by later inattentive changes. In other words, assigning the + * rcu_access_pointer() return value to a local variable results in an + * accident waiting to happen. * * It is also permissible to use rcu_access_pointer() when read-side - * access to the pointer was removed at least one grace period ago, as - * is the case in the context of the RCU callback that is freeing up - * the data, or after a synchronize_rcu() returns. This can be useful - * when tearing down multi-linked structures after a grace period - * has elapsed. + * access to the pointer was removed at least one grace period ago, as is + * the case in the context of the RCU callback that is freeing up the data, + * or after a synchronize_rcu() returns. This can be useful when tearing + * down multi-linked structures after a grace period has elapsed. However, + * rcu_dereference_protected() is normally preferred for this use case. */ #define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu) diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 62815c0a2dcefc9c53cfc8b3e838bcab7a5c3ee3..768196a5f39d695b434ae8c92f5a9665f63fc746 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,25 +14,75 @@ #include /* for HZ */ +struct rcu_gp_oldstate { + unsigned long rgos_norm; +}; + +// Maximum number of rcu_gp_oldstate values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 + +/* + * Are the two oldstate values the same? See the Tree RCU version for + * docbook header. + */ +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, + struct rcu_gp_oldstate *rgosp2) +{ + return rgosp1->rgos_norm == rgosp2->rgos_norm; +} + unsigned long get_state_synchronize_rcu(void); + +static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm = get_state_synchronize_rcu(); +} + unsigned long start_poll_synchronize_rcu(void); + +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm = start_poll_synchronize_rcu(); +} + bool poll_state_synchronize_rcu(unsigned long oldstate); +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + return poll_state_synchronize_rcu(rgosp->rgos_norm); +} + static inline void cond_synchronize_rcu(unsigned long oldstate) { might_sleep(); } +static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + cond_synchronize_rcu(rgosp->rgos_norm); +} + static inline unsigned long start_poll_synchronize_rcu_expedited(void) { return start_poll_synchronize_rcu(); } +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); +} + static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) { cond_synchronize_rcu(oldstate); } +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +{ + cond_synchronize_rcu_expedited(rgosp->rgos_norm); +} + extern void rcu_barrier(void); static inline void synchronize_rcu_expedited(void) diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 47eaa4cb0df729a9b7e4ef242ae5a6d153fda057..5efb51486e8af807ca5241ffc8d377816c72e006 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -40,12 +40,52 @@ bool rcu_eqs_special_set(int cpu); void rcu_momentary_dyntick_idle(void); void kfree_rcu_scheduler_running(void); bool rcu_gp_might_be_stalled(void); + +struct rcu_gp_oldstate { + unsigned long rgos_norm; + unsigned long rgos_exp; +}; + +// Maximum number of rcu_gp_oldstate values corresponding to +// not-yet-completed RCU grace periods. +#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 + +/** + * same_state_synchronize_rcu_full - Are two old-state values identical? + * @rgosp1: First old-state value. + * @rgosp2: Second old-state value. + * + * The two old-state values must have been obtained from either + * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), + * or get_completed_synchronize_rcu_full(). Returns @true if the two + * values are identical and @false otherwise. This allows structures + * whose lifetimes are tracked by old-state values to push these values + * to a list header, allowing those structures to be slightly smaller. + * + * Note that equality is judged on a bitwise basis, so that an + * @rcu_gp_oldstate structure with an already-completed state in one field + * will compare not-equal to a structure with an already-completed state + * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * so how did such a situation come to pass in the first place? + */ +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, + struct rcu_gp_oldstate *rgosp2) +{ + return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; +} + unsigned long start_poll_synchronize_rcu_expedited(void); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); void cond_synchronize_rcu_expedited(unsigned long oldstate); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); unsigned long get_state_synchronize_rcu(void); +void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); unsigned long start_poll_synchronize_rcu(void); +void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); bool poll_state_synchronize_rcu(unsigned long oldstate); +bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); void cond_synchronize_rcu(unsigned long oldstate); +void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); bool rcu_is_idle_cpu(int cpu); diff --git a/include/linux/reboot.h b/include/linux/reboot.h index e5d9ef886179c0924a12f98ca8e378a1462e3606..2b6bb593be5b655b3251506841b227813525454c 100644 --- a/include/linux/reboot.h +++ b/include/linux/reboot.h @@ -105,6 +105,14 @@ enum sys_off_mode { */ SYS_OFF_MODE_POWER_OFF, + /** + * @SYS_OFF_MODE_RESTART_PREPARE: + * + * Handlers prepare system to be restarted. Handlers are + * allowed to sleep. + */ + SYS_OFF_MODE_RESTART_PREPARE, + /** * @SYS_OFF_MODE_RESTART: * diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 7cf2157134acd475e61e1cd3632dc6d2dc356573..ca3434dca3a06a7281dbf39b357961f8ac889a7c 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -311,6 +311,8 @@ typedef void (*regmap_unlock)(void *); * This field is a duplicate of a similar file in * 'struct regmap_bus' and serves exact same purpose. * Use it only for "no-bus" cases. + * @io_port: Support IO port accessors. Makes sense only when MMIO vs. IO port + * access can be distinguished. * @max_register: Optional, specifies the maximum valid register address. * @wr_table: Optional, points to a struct regmap_access_table specifying * valid ranges for write access. @@ -399,6 +401,7 @@ struct regmap_config { size_t max_raw_write; bool fast_io; + bool io_port; unsigned int max_register; const struct regmap_access_table *wr_table; @@ -489,8 +492,12 @@ typedef int (*regmap_hw_read)(void *context, void *val_buf, size_t val_size); typedef int (*regmap_hw_reg_read)(void *context, unsigned int reg, unsigned int *val); +typedef int (*regmap_hw_reg_noinc_read)(void *context, unsigned int reg, + void *val, size_t val_count); typedef int (*regmap_hw_reg_write)(void *context, unsigned int reg, unsigned int val); +typedef int (*regmap_hw_reg_noinc_write)(void *context, unsigned int reg, + const void *val, size_t val_count); typedef int (*regmap_hw_reg_update_bits)(void *context, unsigned int reg, unsigned int mask, unsigned int val); typedef struct regmap_async *(*regmap_hw_async_alloc)(void); @@ -511,6 +518,8 @@ typedef void (*regmap_hw_free_context)(void *context); * must serialise with respect to non-async I/O. * @reg_write: Write a single register value to the given register address. This * write operation has to complete when returning from the function. + * @reg_write_noinc: Write multiple register value to the same register. This + * write operation has to complete when returning from the function. * @reg_update_bits: Update bits operation to be used against volatile * registers, intended for devices supporting some mechanism * for setting clearing bits without having to @@ -538,9 +547,11 @@ struct regmap_bus { regmap_hw_gather_write gather_write; regmap_hw_async_write async_write; regmap_hw_reg_write reg_write; + regmap_hw_reg_noinc_write reg_noinc_write; regmap_hw_reg_update_bits reg_update_bits; regmap_hw_read read; regmap_hw_reg_read reg_read; + regmap_hw_reg_noinc_read reg_noinc_read; regmap_hw_free_context free_context; regmap_hw_async_alloc async_alloc; u8 read_flag_mask; diff --git a/include/linux/regulator/consumer.h b/include/linux/regulator/consumer.h index bc6cda706d1fe108e20b376ad75e66d7cb1633f4..ee3b4a0146119809e17476e7a30c62b7763125ea 100644 --- a/include/linux/regulator/consumer.h +++ b/include/linux/regulator/consumer.h @@ -207,6 +207,8 @@ struct regulator *__must_check regulator_get_optional(struct device *dev, const char *id); struct regulator *__must_check devm_regulator_get_optional(struct device *dev, const char *id); +int devm_regulator_get_enable(struct device *dev, const char *id); +int devm_regulator_get_enable_optional(struct device *dev, const char *id); void regulator_put(struct regulator *regulator); void devm_regulator_put(struct regulator *regulator); @@ -244,12 +246,15 @@ int __must_check regulator_bulk_get(struct device *dev, int num_consumers, struct regulator_bulk_data *consumers); int __must_check devm_regulator_bulk_get(struct device *dev, int num_consumers, struct regulator_bulk_data *consumers); +void devm_regulator_bulk_put(struct regulator_bulk_data *consumers); int __must_check devm_regulator_bulk_get_const( struct device *dev, int num_consumers, const struct regulator_bulk_data *in_consumers, struct regulator_bulk_data **out_consumers); int __must_check regulator_bulk_enable(int num_consumers, struct regulator_bulk_data *consumers); +int devm_regulator_bulk_get_enable(struct device *dev, int num_consumers, + const char * const *id); int regulator_bulk_disable(int num_consumers, struct regulator_bulk_data *consumers); int regulator_bulk_force_disable(int num_consumers, @@ -354,6 +359,17 @@ devm_regulator_get_exclusive(struct device *dev, const char *id) return ERR_PTR(-ENODEV); } +static inline int devm_regulator_get_enable(struct device *dev, const char *id) +{ + return -ENODEV; +} + +static inline int devm_regulator_get_enable_optional(struct device *dev, + const char *id) +{ + return -ENODEV; +} + static inline struct regulator *__must_check regulator_get_optional(struct device *dev, const char *id) { @@ -375,6 +391,10 @@ static inline void devm_regulator_put(struct regulator *regulator) { } +static inline void devm_regulator_bulk_put(struct regulator_bulk_data *consumers) +{ +} + static inline int regulator_register_supply_alias(struct device *dev, const char *id, struct device *alias_dev, @@ -465,6 +485,13 @@ static inline int regulator_bulk_enable(int num_consumers, return 0; } +static inline int devm_regulator_bulk_get_enable(struct device *dev, + int num_consumers, + const char * const *id) +{ + return 0; +} + static inline int regulator_bulk_disable(int num_consumers, struct regulator_bulk_data *consumers) { diff --git a/include/linux/regulator/gpio-regulator.h b/include/linux/regulator/gpio-regulator.h index fdeb312cdabdfdd33699c780d823f9e7979367ac..c223e50ff9f78e1da92f18ab649cb2f6003986c0 100644 --- a/include/linux/regulator/gpio-regulator.h +++ b/include/linux/regulator/gpio-regulator.h @@ -42,6 +42,7 @@ struct gpio_regulator_state { /** * struct gpio_regulator_config - config structure * @supply_name: Name of the regulator supply + * @input_supply: Name of the input regulator supply * @enabled_at_boot: Whether regulator has been enabled at * boot or not. 1 = Yes, 0 = No * This is used to keep the regulator at @@ -62,6 +63,7 @@ struct gpio_regulator_state { */ struct gpio_regulator_config { const char *supply_name; + const char *input_supply; unsigned enabled_at_boot:1; unsigned startup_delay; diff --git a/include/linux/regulator/mt6331-regulator.h b/include/linux/regulator/mt6331-regulator.h new file mode 100644 index 0000000000000000000000000000000000000000..2801a9879c1475ecf764e74a6c386eaf78214853 --- /dev/null +++ b/include/linux/regulator/mt6331-regulator.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#ifndef __LINUX_REGULATOR_MT6331_H +#define __LINUX_REGULATOR_MT6331_H + +enum { + /* BUCK */ + MT6331_ID_VDVFS11 = 0, + MT6331_ID_VDVFS12, + MT6331_ID_VDVFS13, + MT6331_ID_VDVFS14, + MT6331_ID_VCORE2, + MT6331_ID_VIO18, + /* LDO */ + MT6331_ID_VTCXO1, + MT6331_ID_VTCXO2, + MT6331_ID_AVDD32_AUD, + MT6331_ID_VAUXA32, + MT6331_ID_VCAMA, + MT6331_ID_VIO28, + MT6331_ID_VCAM_AF, + MT6331_ID_VMC, + MT6331_ID_VMCH, + MT6331_ID_VEMC33, + MT6331_ID_VGP1, + MT6331_ID_VSIM1, + MT6331_ID_VSIM2, + MT6331_ID_VMIPI, + MT6331_ID_VIBR, + MT6331_ID_VGP4, + MT6331_ID_VCAMD, + MT6331_ID_VUSB10, + MT6331_ID_VCAM_IO, + MT6331_ID_VSRAM_DVFS1, + MT6331_ID_VGP2, + MT6331_ID_VGP3, + MT6331_ID_VRTC, + MT6331_ID_VDIG18, + MT6331_ID_VREG_MAX +}; + +#endif /* __LINUX_REGULATOR_MT6331_H */ diff --git a/include/linux/regulator/mt6332-regulator.h b/include/linux/regulator/mt6332-regulator.h new file mode 100644 index 0000000000000000000000000000000000000000..af5e3ed310296856ec90414ce9efbcf567820042 --- /dev/null +++ b/include/linux/regulator/mt6332-regulator.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2022 Collabora Ltd. + * Author: AngeloGioacchino Del Regno + */ + +#ifndef __LINUX_REGULATOR_MT6332_H +#define __LINUX_REGULATOR_MT6332_H + +enum { + /* BUCK */ + MT6332_ID_VDRAM = 0, + MT6332_ID_VDVFS2, + MT6332_ID_VPA, + MT6332_ID_VRF1, + MT6332_ID_VRF2, + MT6332_ID_VSBST, + /* LDO */ + MT6332_ID_VAUXB32, + MT6332_ID_VBIF28, + MT6332_ID_VDIG18, + MT6332_ID_VSRAM_DVFS2, + MT6332_ID_VUSB33, + MT6332_ID_VREG_MAX +}; + +#endif /* __LINUX_REGULATOR_MT6332_H */ diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index aea79c77db0ff26d9dfe20cd1377b01687a641ca..fe8978eb69f1c7688c85ba4ecd67cf5e8f306a4e 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -489,6 +489,20 @@ struct rproc_dump_segment { loff_t offset; }; +/** + * enum rproc_features - features supported + * + * @RPROC_FEAT_ATTACH_ON_RECOVERY: The remote processor does not need help + * from Linux to recover, such as firmware + * loading. Linux just needs to attach after + * recovery. + */ + +enum rproc_features { + RPROC_FEAT_ATTACH_ON_RECOVERY, + RPROC_MAX_FEATURES, +}; + /** * struct rproc - represents a physical remote processor device * @node: list node of this rproc object @@ -530,6 +544,7 @@ struct rproc_dump_segment { * @elf_machine: firmware ELF machine * @cdev: character device of the rproc * @cdev_put_on_release: flag to indicate if remoteproc should be shutdown on @char_dev release + * @features: indicate remoteproc features */ struct rproc { struct list_head node; @@ -570,6 +585,7 @@ struct rproc { u16 elf_machine; struct cdev cdev; bool cdev_put_on_release; + DECLARE_BITMAP(features, RPROC_MAX_FEATURES); }; /** @@ -616,9 +632,8 @@ struct rproc_vring { /** * struct rproc_vdev - remoteproc state for a supported virtio device - * @refcount: reference counter for the vdev and vring allocations * @subdev: handle for registering the vdev as a rproc subdevice - * @dev: device struct used for reference count semantics + * @pdev: remoteproc virtio platform device * @id: virtio device id (as in virtio_ids.h) * @node: list node * @rproc: the rproc handle @@ -627,10 +642,9 @@ struct rproc_vring { * @index: vdev position versus other vdev declared in resource table */ struct rproc_vdev { - struct kref refcount; struct rproc_subdev subdev; - struct device dev; + struct platform_device *pdev; unsigned int id; struct list_head node; diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 21deb5212bbdd82e58dac7b4764762c66fffba9b..0cf5b20c6ddf8e5187d674b06f224d579d6f7d66 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -15,6 +15,9 @@ int proc_resctrl_show(struct seq_file *m, #endif +/* max value for struct rdt_domain's mbps_val */ +#define MBA_MAX_MBPS U32_MAX + /** * enum resctrl_conf_type - The type of configuration. * @CDP_NONE: No prioritisation, both code and data are controlled or monitored. @@ -29,6 +32,16 @@ enum resctrl_conf_type { #define CDP_NUM_TYPES (CDP_DATA + 1) +/* + * Event IDs, the values match those used to program IA32_QM_EVTSEL before + * reading IA32_QM_CTR on RDT systems. + */ +enum resctrl_event_id { + QOS_L3_OCCUP_EVENT_ID = 0x01, + QOS_L3_MBM_TOTAL_EVENT_ID = 0x02, + QOS_L3_MBM_LOCAL_EVENT_ID = 0x03, +}; + /** * struct resctrl_staged_config - parsed configuration to be applied * @new_ctrl: new ctrl value to be loaded @@ -53,6 +66,9 @@ struct resctrl_staged_config { * @cqm_work_cpu: worker CPU for CQM h/w counters * @plr: pseudo-locked region (if any) associated with domain * @staged_config: parsed configuration to be applied + * @mbps_val: When mba_sc is enabled, this holds the array of user + * specified control values for mba_sc in MBps, indexed + * by closid */ struct rdt_domain { struct list_head list; @@ -67,6 +83,7 @@ struct rdt_domain { int cqm_work_cpu; struct pseudo_lock_region *plr; struct resctrl_staged_config staged_config[CDP_NUM_TYPES]; + u32 *mbps_val; }; /** @@ -130,8 +147,6 @@ struct resctrl_schema; /** * struct rdt_resource - attributes of a resctrl resource * @rid: The index of the resource - * @alloc_enabled: Is allocation enabled on this machine - * @mon_enabled: Is monitoring enabled for this feature * @alloc_capable: Is allocation available on this machine * @mon_capable: Is monitor feature available on this machine * @num_rmid: Number of RMIDs available @@ -150,8 +165,6 @@ struct resctrl_schema; */ struct rdt_resource { int rid; - bool alloc_enabled; - bool mon_enabled; bool alloc_capable; bool mon_capable; int num_rmid; @@ -194,7 +207,50 @@ struct resctrl_schema { /* The number of closid supported by this resource regardless of CDP */ u32 resctrl_arch_get_num_closid(struct rdt_resource *r); int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid); + +/* + * Update the ctrl_val and apply this config right now. + * Must be called on one of the domain's CPUs. + */ +int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d, + u32 closid, enum resctrl_conf_type t, u32 cfg_val); + u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d, u32 closid, enum resctrl_conf_type type); +int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d); +void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d); + +/** + * resctrl_arch_rmid_read() - Read the eventid counter corresponding to rmid + * for this resource and domain. + * @r: resource that the counter should be read from. + * @d: domain that the counter should be read from. + * @rmid: rmid of the counter to read. + * @eventid: eventid to read, e.g. L3 occupancy. + * @val: result of the counter read in bytes. + * + * Call from process context on a CPU that belongs to domain @d. + * + * Return: + * 0 on success, or -EIO, -EINVAL etc on error. + */ +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d, + u32 rmid, enum resctrl_event_id eventid, u64 *val); + +/** + * resctrl_arch_reset_rmid() - Reset any private state associated with rmid + * and eventid. + * @r: The domain's resource. + * @d: The rmid's domain. + * @rmid: The rmid whose counter values should be reset. + * @eventid: The eventid whose counter values should be reset. + * + * This can be called from any CPU. + */ +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d, + u32 rmid, enum resctrl_event_id eventid); + +extern unsigned int resctrl_rmid_realloc_threshold; +extern unsigned int resctrl_rmid_realloc_limit; #endif /* _RESCTRL_H */ diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index dac53fd3afea33575f2addbab4db1382b5a737d3..2504df9a0453e6cfdcec454db7d56728b329cb23 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -101,7 +101,7 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full); __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, struct file *filp, poll_table *poll_table); - +void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu); #define RING_BUFFER_ALL_CPUS -1 diff --git a/include/linux/rmap.h b/include/linux/rmap.h index bf80adca980b96beff8e586b125de015cf2a400e..bd3504d11b15590f0e41232e234eebc1149baad4 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -41,12 +41,15 @@ struct anon_vma { atomic_t refcount; /* - * Count of child anon_vmas and VMAs which points to this anon_vma. + * Count of child anon_vmas. Equals to the count of all anon_vmas that + * have ->parent pointing to this one, including itself. * * This counter is used for making decision about reusing anon_vma * instead of forking new one. See comments in function anon_vma_clone. */ - unsigned degree; + unsigned long num_children; + /* Count of VMAs whose ->anon_vma pointer points to this object. */ + unsigned long num_active_vmas; struct anon_vma *parent; /* Parent of this anon_vma */ @@ -163,7 +166,7 @@ static inline void anon_vma_merge(struct vm_area_struct *vma, unlink_anon_vmas(next); } -struct anon_vma *page_get_anon_vma(struct page *page); +struct anon_vma *folio_get_anon_vma(struct folio *folio); /* RMAP flags, currently only relevant for some anon rmap operations. */ typedef int __bitwise rmap_t; @@ -267,7 +270,7 @@ dup: * @page: the exclusive anonymous page to try marking possibly shared * * The caller needs to hold the PT lock and has to have the page table entry - * cleared/invalidated+flushed, to properly sync against GUP-fast. + * cleared/invalidated. * * This is similar to page_try_dup_anon_rmap(), however, not used during fork() * to duplicate a mapping, but instead to prepare for KSM or temporarily @@ -283,12 +286,68 @@ static inline int page_try_share_anon_rmap(struct page *page) { VM_BUG_ON_PAGE(!PageAnon(page) || !PageAnonExclusive(page), page); - /* See page_try_dup_anon_rmap(). */ - if (likely(!is_device_private_page(page) && - unlikely(page_maybe_dma_pinned(page)))) - return -EBUSY; + /* device private pages cannot get pinned via GUP. */ + if (unlikely(is_device_private_page(page))) { + ClearPageAnonExclusive(page); + return 0; + } + + /* + * We have to make sure that when we clear PageAnonExclusive, that + * the page is not pinned and that concurrent GUP-fast won't succeed in + * concurrently pinning the page. + * + * Conceptually, PageAnonExclusive clearing consists of: + * (A1) Clear PTE + * (A2) Check if the page is pinned; back off if so. + * (A3) Clear PageAnonExclusive + * (A4) Restore PTE (optional, but certainly not writable) + * + * When clearing PageAnonExclusive, we cannot possibly map the page + * writable again, because anon pages that may be shared must never + * be writable. So in any case, if the PTE was writable it cannot + * be writable anymore afterwards and there would be a PTE change. Only + * if the PTE wasn't writable, there might not be a PTE change. + * + * Conceptually, GUP-fast pinning of an anon page consists of: + * (B1) Read the PTE + * (B2) FOLL_WRITE: check if the PTE is not writable; back off if so. + * (B3) Pin the mapped page + * (B4) Check if the PTE changed by re-reading it; back off if so. + * (B5) If the original PTE is not writable, check if + * PageAnonExclusive is not set; back off if so. + * + * If the PTE was writable, we only have to make sure that GUP-fast + * observes a PTE change and properly backs off. + * + * If the PTE was not writable, we have to make sure that GUP-fast either + * detects a (temporary) PTE change or that PageAnonExclusive is cleared + * and properly backs off. + * + * Consequently, when clearing PageAnonExclusive(), we have to make + * sure that (A1), (A2)/(A3) and (A4) happen in the right memory + * order. In GUP-fast pinning code, we have to make sure that (B3),(B4) + * and (B5) happen in the right memory order. + * + * We assume that there might not be a memory barrier after + * clearing/invalidating the PTE (A1) and before restoring the PTE (A4), + * so we use explicit ones here. + */ + /* Paired with the memory barrier in try_grab_folio(). */ + if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + smp_mb(); + + if (unlikely(page_maybe_dma_pinned(page))) + return -EBUSY; ClearPageAnonExclusive(page); + + /* + * This is conceptually a smp_wmb() paired with the smp_rmb() in + * gup_must_unshare(). + */ + if (IS_ENABLED(CONFIG_HAVE_FAST_GUP)) + smp_mb__after_atomic(); return 0; } @@ -402,13 +461,8 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); - -/* - * Called by memory-failure.c to kill processes. - */ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, struct rmap_walk_control *rwc); -void page_unlock_anon_vma_read(struct anon_vma *anon_vma); #else /* !CONFIG_MMU */ diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 8f416c5e929ea865417b426079cc7c76eb47ec42..c0ef596f340b58e0caf76d154bdd16dcc0711eeb 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -1,7 +1,7 @@ #ifndef __LINUX_RWLOCK_H #define __LINUX_RWLOCK_H -#ifndef __LINUX_SPINLOCK_H +#ifndef __LINUX_INSIDE_SPINLOCK_H # error "please don't include this file directly" #endif diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 8f5a86e210b90c43c5246eaa121b2048d0351066..4d2d5205ab5860b03e1b8d61a6e4a7d4d55712a0 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -575,8 +575,9 @@ void sbitmap_queue_wake_all(struct sbitmap_queue *sbq); * sbitmap_queue_wake_up() - Wake up some of waiters in one waitqueue * on a &struct sbitmap_queue. * @sbq: Bitmap queue to wake up. + * @nr: Number of bits cleared. */ -void sbitmap_queue_wake_up(struct sbitmap_queue *sbq); +void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr); /** * sbitmap_queue_show() - Dump &struct sbitmap_queue information to a &struct diff --git a/include/linux/sched.h b/include/linux/sched.h index e7b2f8a5c711c1942abcdc6b2640f66bea868b25..ffb6eb55cd135b26aa8e0564efa843f02fbe86b4 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -81,25 +82,34 @@ struct task_group; */ /* Used in tsk->state: */ -#define TASK_RUNNING 0x0000 -#define TASK_INTERRUPTIBLE 0x0001 -#define TASK_UNINTERRUPTIBLE 0x0002 -#define __TASK_STOPPED 0x0004 -#define __TASK_TRACED 0x0008 +#define TASK_RUNNING 0x00000000 +#define TASK_INTERRUPTIBLE 0x00000001 +#define TASK_UNINTERRUPTIBLE 0x00000002 +#define __TASK_STOPPED 0x00000004 +#define __TASK_TRACED 0x00000008 /* Used in tsk->exit_state: */ -#define EXIT_DEAD 0x0010 -#define EXIT_ZOMBIE 0x0020 +#define EXIT_DEAD 0x00000010 +#define EXIT_ZOMBIE 0x00000020 #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) /* Used in tsk->state again: */ -#define TASK_PARKED 0x0040 -#define TASK_DEAD 0x0080 -#define TASK_WAKEKILL 0x0100 -#define TASK_WAKING 0x0200 -#define TASK_NOLOAD 0x0400 -#define TASK_NEW 0x0800 -/* RT specific auxilliary flag to mark RT lock waiters */ -#define TASK_RTLOCK_WAIT 0x1000 -#define TASK_STATE_MAX 0x2000 +#define TASK_PARKED 0x00000040 +#define TASK_DEAD 0x00000080 +#define TASK_WAKEKILL 0x00000100 +#define TASK_WAKING 0x00000200 +#define TASK_NOLOAD 0x00000400 +#define TASK_NEW 0x00000800 +#define TASK_RTLOCK_WAIT 0x00001000 +#define TASK_FREEZABLE 0x00002000 +#define __TASK_FREEZABLE_UNSAFE (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP)) +#define TASK_FROZEN 0x00008000 +#define TASK_STATE_MAX 0x00010000 + +#define TASK_ANY (TASK_STATE_MAX-1) + +/* + * DO NOT ADD ANY NEW USERS ! + */ +#define TASK_FREEZABLE_UNSAFE (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE) /* Convenience macros for the sake of set_current_state: */ #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) @@ -860,9 +870,6 @@ struct task_struct { struct mm_struct *mm; struct mm_struct *active_mm; - /* Per-thread vma caching: */ - struct vmacache vmacache; - #ifdef SPLIT_RSS_COUNTING struct task_rss_stat rss_stat; #endif @@ -914,6 +921,10 @@ struct task_struct { #ifdef CONFIG_MEMCG unsigned in_user_fault:1; #endif +#ifdef CONFIG_LRU_GEN + /* whether the LRU algorithm may apply to this access */ + unsigned in_lru_fault:1; +#endif #ifdef CONFIG_COMPAT_BRK unsigned brk_randomized:1; #endif @@ -936,7 +947,7 @@ struct task_struct { #endif #ifdef CONFIG_EVENTFD /* Recursion prevention for eventfd_signal() */ - unsigned in_eventfd_signal:1; + unsigned in_eventfd:1; #endif #ifdef CONFIG_IOMMU_SVA unsigned pasid_activated:1; @@ -944,6 +955,10 @@ struct task_struct { #ifdef CONFIG_CPU_SUP_INTEL unsigned reported_split_lock:1; #endif +#ifdef CONFIG_TASK_DELAY_ACCT + /* delay due to memory thrashing */ + unsigned in_thrashing:1; +#endif unsigned long atomic_flags; /* Flags requiring atomic access. */ @@ -1355,6 +1370,10 @@ struct task_struct { #endif #endif +#ifdef CONFIG_KMSAN + struct kmsan_ctx kmsan_ctx; +#endif + #if IS_ENABLED(CONFIG_KUNIT) struct kunit *kunit_test; #endif @@ -1381,9 +1400,6 @@ struct task_struct { #endif #ifdef CONFIG_TRACING - /* State flags for use by tracers: */ - unsigned long trace; - /* Bitmask and counter of trace recursion: */ unsigned long trace_recursion; #endif /* CONFIG_TRACING */ @@ -1713,8 +1729,9 @@ extern struct pid *cad_pid; #define PF_MEMALLOC 0x00000800 /* Allocating memory */ #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */ #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ +#define PF__HOLE__00004000 0x00004000 #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ -#define PF_FROZEN 0x00010000 /* Frozen for system suspend */ +#define PF__HOLE__00010000 0x00010000 #define PF_KSWAPD 0x00020000 /* I am kswapd */ #define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */ #define PF_MEMALLOC_NOIO 0x00080000 /* All allocation requests will inherit GFP_NOIO */ @@ -1722,10 +1739,14 @@ extern struct pid *cad_pid; * I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ +#define PF__HOLE__00800000 0x00800000 +#define PF__HOLE__01000000 0x01000000 +#define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ #define PF_MEMALLOC_PIN 0x10000000 /* Allocation context constrained to zones which allow long term pinning. */ -#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ +#define PF__HOLE__20000000 0x20000000 +#define PF__HOLE__40000000 0x40000000 #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ /* diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index 4d0a5be28b70f11ef5e17acf8eb6033fac9606d4..8270ad7ae14c2ab8f70c9d1b1687476dd8e06d56 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -71,9 +71,8 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_UNSTABLE 22 /* mm is unstable for copy_from_user */ #define MMF_HUGE_ZERO_PAGE 23 /* mm has ever used the global huge zero page */ #define MMF_DISABLE_THP 24 /* disable THP for all VMAs */ -#define MMF_OOM_VICTIM 25 /* mm is the oom victim */ -#define MMF_OOM_REAP_QUEUED 26 /* mm was queued for oom_reaper */ -#define MMF_MULTIPROCESS 27 /* mm is shared between processes */ +#define MMF_OOM_REAP_QUEUED 25 /* mm was queued for oom_reaper */ +#define MMF_MULTIPROCESS 26 /* mm is shared between processes */ /* * MMF_HAS_PINNED: Whether this mm has pinned any pages. This can be either * replaced in the future by mm.pinned_vm when it becomes stable, or grow into @@ -81,7 +80,7 @@ static inline int get_dumpable(struct mm_struct *mm) * pinned pages were unpinned later on, we'll still keep this bit set for the * lifecycle of this mm, just for simplicity. */ -#define MMF_HAS_PINNED 28 /* FOLL_PIN has run, never cleared */ +#define MMF_HAS_PINNED 27 /* FOLL_PIN has run, never cleared */ #define MMF_DISABLE_THP_MASK (1 << MMF_DISABLE_THP) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index cafbe03eed017224b790714dd9c2dfaa3ae16256..20099268fa257f40d8d45e7fa795cb8492b9ca34 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -94,6 +94,7 @@ struct signal_struct { refcount_t sigcnt; atomic_t live; int nr_threads; + int quick_threads; struct list_head thread_head; wait_queue_head_t wait_chldexit; /* for wait4() */ diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index e650946816d008eb77eb72cd7a8c8104d2a85ef3..303ee7dd0c7e2a67d9901755d994003f1ae7fc8b 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -27,6 +27,7 @@ enum sched_tunable_scaling { #ifdef CONFIG_NUMA_BALANCING extern int sysctl_numa_balancing_mode; +extern unsigned int sysctl_numa_balancing_promote_rate_limit; #else #define sysctl_numa_balancing_mode 0 #endif diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 81cab4b01edcb1bb9f40edf2e88de83b207e40f9..d6c48163c6defba399fc5e0588d1ce855418c97e 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -127,6 +127,9 @@ static inline void put_task_struct_many(struct task_struct *t, int nr) void put_task_struct_rcu_user(struct task_struct *task); +/* Free all architecture-specific resources held by a thread. */ +void release_thread(struct task_struct *dead_task); + #ifdef CONFIG_ARCH_WANTS_DYNAMIC_TASK_STRUCT extern int arch_task_struct_size __read_mostly; #else diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index a193884ecf2b19d64ae29eecb114dc6e6008d1a9..4f765bc788ffb7bd1a6d09ab3705a2ae27c19e9a 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -84,7 +84,7 @@ struct scmi_protocol_handle; struct scmi_clk_proto_ops { int (*count_get)(const struct scmi_protocol_handle *ph); - const struct scmi_clock_info *(*info_get) + const struct scmi_clock_info __must_check *(*info_get) (const struct scmi_protocol_handle *ph, u32 clk_id); int (*rate_get)(const struct scmi_protocol_handle *ph, u32 clk_id, u64 *rate); @@ -466,7 +466,7 @@ enum scmi_sensor_class { */ struct scmi_sensor_proto_ops { int (*count_get)(const struct scmi_protocol_handle *ph); - const struct scmi_sensor_info *(*info_get) + const struct scmi_sensor_info __must_check *(*info_get) (const struct scmi_protocol_handle *ph, u32 sensor_id); int (*trip_point_config)(const struct scmi_protocol_handle *ph, u32 sensor_id, u8 trip_id, u64 trip_value); diff --git a/include/linux/security.h b/include/linux/security.h index 1bc362cb413f2ffa4e252d6a0dd75b60b452d0bc..ca1b7109c0dbb820fae784f9760f20942ee0db31 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -114,6 +114,7 @@ enum lockdown_reason { LOCKDOWN_IOPORT, LOCKDOWN_MSR, LOCKDOWN_ACPI_TABLES, + LOCKDOWN_DEVICE_TREE, LOCKDOWN_PCMCIA_CIS, LOCKDOWN_TIOCSSERIAL, LOCKDOWN_MODULE_PARAMETERS, @@ -122,6 +123,7 @@ enum lockdown_reason { LOCKDOWN_XMON_WR, LOCKDOWN_BPF_WRITE_USER, LOCKDOWN_DBG_WRITE_KERNEL, + LOCKDOWN_RTAS_ERROR_INJECTION, LOCKDOWN_INTEGRITY_MAX, LOCKDOWN_KCORE, LOCKDOWN_KPROBES, @@ -437,6 +439,7 @@ int security_task_kill(struct task_struct *p, struct kernel_siginfo *info, int security_task_prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5); void security_task_to_inode(struct task_struct *p, struct inode *inode); +int security_create_user_ns(const struct cred *cred); int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag); void security_ipc_getsecid(struct kern_ipc_perm *ipcp, u32 *secid); int security_msg_msg_alloc(struct msg_msg *msg); @@ -461,7 +464,7 @@ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd); int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops, unsigned nsops, int alter); void security_d_instantiate(struct dentry *dentry, struct inode *inode); -int security_getprocattr(struct task_struct *p, const char *lsm, char *name, +int security_getprocattr(struct task_struct *p, const char *lsm, const char *name, char **value); int security_setprocattr(const char *lsm, const char *name, void *value, size_t size); @@ -1194,6 +1197,11 @@ static inline int security_task_prctl(int option, unsigned long arg2, static inline void security_task_to_inode(struct task_struct *p, struct inode *inode) { } +static inline int security_create_user_ns(const struct cred *cred) +{ + return 0; +} + static inline int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag) { @@ -1301,7 +1309,7 @@ static inline void security_d_instantiate(struct dentry *dentry, { } static inline int security_getprocattr(struct task_struct *p, const char *lsm, - char *name, char **value) + const char *name, char **value) { return -EINVAL; } @@ -2060,6 +2068,7 @@ static inline int security_perf_event_write(struct perf_event *event) #ifdef CONFIG_SECURITY extern int security_uring_override_creds(const struct cred *new); extern int security_uring_sqpoll(void); +extern int security_uring_cmd(struct io_uring_cmd *ioucmd); #else static inline int security_uring_override_creds(const struct cred *new) { @@ -2069,6 +2078,10 @@ static inline int security_uring_sqpoll(void) { return 0; } +static inline int security_uring_cmd(struct io_uring_cmd *ioucmd) +{ + return 0; +} #endif /* CONFIG_SECURITY */ #endif /* CONFIG_IO_URING */ diff --git a/include/linux/sed-opal.h b/include/linux/sed-opal.h index 1ac0d712a9c36f440f4f3db92783cc16e55a7370..6f837bb6c7156aede902c37e84a5016c1376e681 100644 --- a/include/linux/sed-opal.h +++ b/include/linux/sed-opal.h @@ -43,6 +43,7 @@ static inline bool is_sed_ioctl(unsigned int cmd) case IOC_OPAL_MBR_DONE: case IOC_OPAL_WRITE_SHADOW_MBR: case IOC_OPAL_GENERIC_TABLE_RW: + case IOC_OPAL_GET_STATUS: return true; } return false; diff --git a/include/linux/serdev.h b/include/linux/serdev.h index 3368c261ab623a91aa98dd5a85f8312f6664b047..66f624fc618ca3bb4d216a8dba8615af84c911d1 100644 --- a/include/linux/serdev.h +++ b/include/linux/serdev.h @@ -7,6 +7,7 @@ #include #include +#include #include #include diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index 8c7b793aa4d702e8b1ac4b75c15d7fc4f2b6adba..19376bee9667695496fdf5a0c66492d06532545c 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -32,7 +32,7 @@ struct plat_serial8250_port { void (*serial_out)(struct uart_port *, int, int); void (*set_termios)(struct uart_port *, struct ktermios *new, - struct ktermios *old); + const struct ktermios *old); void (*set_ldisc)(struct uart_port *, struct ktermios *); unsigned int (*get_mctrl)(struct uart_port *); @@ -74,6 +74,7 @@ struct uart_8250_port; struct uart_8250_ops { int (*setup_irq)(struct uart_8250_port *); void (*release_irq)(struct uart_8250_port *); + void (*setup_timer)(struct uart_8250_port *); }; struct uart_8250_em485 { @@ -157,7 +158,7 @@ extern int early_serial8250_setup(struct earlycon_device *device, extern void serial8250_update_uartclk(struct uart_port *port, unsigned int uartclk); extern void serial8250_do_set_termios(struct uart_port *port, - struct ktermios *termios, struct ktermios *old); + struct ktermios *termios, const struct ktermios *old); extern void serial8250_do_set_ldisc(struct uart_port *port, struct ktermios *termios); extern unsigned int serial8250_do_get_mctrl(struct uart_port *port); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index aef3145f2032969e671bdd88d54415603a9d8199..d657f2a42a7b8fadd84d88952e5e2a022b36b32c 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -141,6 +141,14 @@ struct gpio_desc; * Locking: none. * Interrupts: caller dependent. * + * @start_rx: ``void ()(struct uart_port *port)`` + * + * Start receiving characters. + * + * Locking: @port->lock taken. + * Interrupts: locally disabled. + * This call must not sleep + * * @stop_rx: ``void ()(struct uart_port *port)`` * * Stop receiving characters; the @port is in the process of being closed. @@ -379,7 +387,7 @@ struct uart_ops { void (*shutdown)(struct uart_port *); void (*flush_buffer)(struct uart_port *); void (*set_termios)(struct uart_port *, struct ktermios *new, - struct ktermios *old); + const struct ktermios *old); void (*set_ldisc)(struct uart_port *, struct ktermios *); void (*pm)(struct uart_port *, unsigned int state, unsigned int oldstate); @@ -414,7 +422,7 @@ struct uart_icount { __u32 buf_overrun; }; -typedef unsigned int __bitwise upf_t; +typedef u64 __bitwise upf_t; typedef unsigned int __bitwise upstat_t; struct uart_port { @@ -425,7 +433,7 @@ struct uart_port { void (*serial_out)(struct uart_port *, int, int); void (*set_termios)(struct uart_port *, struct ktermios *new, - struct ktermios *old); + const struct ktermios *old); void (*set_ldisc)(struct uart_port *, struct ktermios *); unsigned int (*get_mctrl)(struct uart_port *); @@ -505,23 +513,24 @@ struct uart_port { #define UPF_BUGGY_UART ((__force upf_t) ASYNC_BUGGY_UART /* 14 */ ) #define UPF_MAGIC_MULTIPLIER ((__force upf_t) ASYNC_MAGIC_MULTIPLIER /* 16 */ ) -#define UPF_NO_THRE_TEST ((__force upf_t) (1 << 19)) +#define UPF_NO_THRE_TEST ((__force upf_t) BIT_ULL(19)) /* Port has hardware-assisted h/w flow control */ -#define UPF_AUTO_CTS ((__force upf_t) (1 << 20)) -#define UPF_AUTO_RTS ((__force upf_t) (1 << 21)) +#define UPF_AUTO_CTS ((__force upf_t) BIT_ULL(20)) +#define UPF_AUTO_RTS ((__force upf_t) BIT_ULL(21)) #define UPF_HARD_FLOW ((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS)) /* Port has hardware-assisted s/w flow control */ -#define UPF_SOFT_FLOW ((__force upf_t) (1 << 22)) -#define UPF_CONS_FLOW ((__force upf_t) (1 << 23)) -#define UPF_SHARE_IRQ ((__force upf_t) (1 << 24)) -#define UPF_EXAR_EFR ((__force upf_t) (1 << 25)) -#define UPF_BUG_THRE ((__force upf_t) (1 << 26)) +#define UPF_SOFT_FLOW ((__force upf_t) BIT_ULL(22)) +#define UPF_CONS_FLOW ((__force upf_t) BIT_ULL(23)) +#define UPF_SHARE_IRQ ((__force upf_t) BIT_ULL(24)) +#define UPF_EXAR_EFR ((__force upf_t) BIT_ULL(25)) +#define UPF_BUG_THRE ((__force upf_t) BIT_ULL(26)) /* The exact UART type is known and should not be probed. */ -#define UPF_FIXED_TYPE ((__force upf_t) (1 << 27)) -#define UPF_BOOT_AUTOCONF ((__force upf_t) (1 << 28)) -#define UPF_FIXED_PORT ((__force upf_t) (1 << 29)) -#define UPF_DEAD ((__force upf_t) (1 << 30)) -#define UPF_IOREMAP ((__force upf_t) (1 << 31)) +#define UPF_FIXED_TYPE ((__force upf_t) BIT_ULL(27)) +#define UPF_BOOT_AUTOCONF ((__force upf_t) BIT_ULL(28)) +#define UPF_FIXED_PORT ((__force upf_t) BIT_ULL(29)) +#define UPF_DEAD ((__force upf_t) BIT_ULL(30)) +#define UPF_IOREMAP ((__force upf_t) BIT_ULL(31)) +#define UPF_FULL_PROBE ((__force upf_t) BIT_ULL(32)) #define __UPF_CHANGE_MASK 0x17fff #define UPF_CHANGE_MASK ((__force upf_t) __UPF_CHANGE_MASK) @@ -616,6 +625,23 @@ struct uart_state { /* number of characters left in xmit buffer before we ask for more */ #define WAKEUP_CHARS 256 +/** + * uart_xmit_advance - Advance xmit buffer and account Tx'ed chars + * @up: uart_port structure describing the port + * @chars: number of characters sent + * + * This function advances the tail of circular xmit buffer by the number of + * @chars transmitted and handles accounting of transmitted bytes (into + * @up's icount.tx). + */ +static inline void uart_xmit_advance(struct uart_port *up, unsigned int chars) +{ + struct circ_buf *xmit = &up->state->xmit; + + xmit->tail = (xmit->tail + chars) & (UART_XMIT_SIZE - 1); + up->icount.tx += chars; +} + struct module; struct tty_driver; @@ -644,7 +670,7 @@ void uart_write_wakeup(struct uart_port *port); void uart_update_timeout(struct uart_port *port, unsigned int cflag, unsigned int baud); unsigned int uart_get_baud_rate(struct uart_port *port, struct ktermios *termios, - struct ktermios *old, unsigned int min, + const struct ktermios *old, unsigned int min, unsigned int max); unsigned int uart_get_divisor(struct uart_port *port, unsigned int baud); @@ -925,5 +951,4 @@ static inline int uart_handle_break(struct uart_port *port) !((cflag) & CLOCAL)) int uart_get_rs485_mode(struct uart_port *port); -int uart_rs485_config(struct uart_port *port); #endif /* LINUX_SERIAL_CORE_H */ diff --git a/include/linux/sfp.h b/include/linux/sfp.h index 302094b855fba3d57a0efa2f23fafb9a6f3e146e..d1f343853b6c7383e22146e5588367e6d6524dcf 100644 --- a/include/linux/sfp.h +++ b/include/linux/sfp.h @@ -535,7 +535,7 @@ int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id, unsigned long *support); bool sfp_may_have_phy(struct sfp_bus *bus, const struct sfp_eeprom_id *id); void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, - unsigned long *support); + unsigned long *support, unsigned long *interfaces); phy_interface_t sfp_select_interface(struct sfp_bus *bus, unsigned long *link_modes); @@ -568,7 +568,8 @@ static inline bool sfp_may_have_phy(struct sfp_bus *bus, static inline void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id, - unsigned long *support) + unsigned long *support, + unsigned long *interfaces) { } diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 1b6c4013f691b0ee64eade2718e33b9c6bd01c1b..d500ea967dc738ec41be9679d635babf76608c5b 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -29,15 +29,10 @@ struct shmem_inode_info { struct inode vfs_inode; }; -#define SHMEM_FL_USER_VISIBLE FS_FL_USER_VISIBLE -#define SHMEM_FL_USER_MODIFIABLE FS_FL_USER_MODIFIABLE -#define SHMEM_FL_INHERITED FS_FL_USER_MODIFIABLE - -/* Flags that are appropriate for regular files (all but dir-specific ones). */ -#define SHMEM_REG_FLMASK (~(FS_DIRSYNC_FL | FS_TOPDIR_FL)) - -/* Flags that are appropriate for non-directories/regular files. */ -#define SHMEM_OTHER_FLMASK (FS_NODUMP_FL | FS_NOATIME_FL) +#define SHMEM_FL_USER_VISIBLE FS_FL_USER_VISIBLE +#define SHMEM_FL_USER_MODIFIABLE \ + (FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL | FS_NOATIME_FL) +#define SHMEM_FL_INHERITED (FS_NODUMP_FL | FS_NOATIME_FL) struct shmem_sb_info { unsigned long max_blocks; /* How many blocks are allowed */ @@ -97,17 +92,19 @@ extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end); int shmem_unuse(unsigned int type); -extern bool shmem_is_huge(struct vm_area_struct *vma, - struct inode *inode, pgoff_t index); -static inline bool shmem_huge_enabled(struct vm_area_struct *vma) +extern bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, + pgoff_t index, bool shmem_huge_force); +static inline bool shmem_huge_enabled(struct vm_area_struct *vma, + bool shmem_huge_force) { - return shmem_is_huge(vma, file_inode(vma->vm_file), vma->vm_pgoff); + return shmem_is_huge(vma, file_inode(vma->vm_file), vma->vm_pgoff, + shmem_huge_force); } extern unsigned long shmem_swap_usage(struct vm_area_struct *vma); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); -/* Flag allocation requirements to shmem_getpage */ +/* Flag allocation requirements to shmem_get_folio */ enum sgp_type { SGP_READ, /* don't exceed i_size, don't allocate page */ SGP_NOALLOC, /* similar, but fail on hole or use fallocated page */ @@ -116,8 +113,8 @@ enum sgp_type { SGP_FALLOC, /* like SGP_WRITE, but make existing page Uptodate */ }; -extern int shmem_getpage(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp); +int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop, + enum sgp_type sgp); static inline struct page *shmem_read_mapping_page( struct address_space *mapping, pgoff_t index) diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index ca8afa382bf2936f12d34dce1aa1329a640802c5..7be5bb4c94b6d8db1349cf1454e9dbcf5683d294 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -533,6 +533,13 @@ enum { struct ubuf_info { void (*callback)(struct sk_buff *, struct ubuf_info *, bool zerocopy_success); + refcount_t refcnt; + u8 flags; +}; + +struct ubuf_info_msgzc { + struct ubuf_info ubuf; + union { struct { unsigned long desc; @@ -545,8 +552,6 @@ struct ubuf_info { u32 bytelen; }; }; - refcount_t refcnt; - u8 flags; struct mmpin { struct user_struct *user; @@ -555,6 +560,8 @@ struct ubuf_info { }; #define skb_uarg(SKB) ((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg)) +#define uarg_to_msgzc(ubuf_ptr) container_of((ubuf_ptr), struct ubuf_info_msgzc, \ + ubuf) int mm_account_pinned_pages(struct mmpin *mmp, size_t size); void mm_unaccount_pinned_pages(struct mmpin *mmp); @@ -796,6 +803,7 @@ typedef unsigned char *sk_buff_data_t; * @csum_level: indicates the number of consecutive checksums found in * the packet minus one that have been verified as * CHECKSUM_UNNECESSARY (max 3) + * @scm_io_uring: SKB holds io_uring registered files * @dst_pending_confirm: need to confirm neighbour * @decrypted: Decrypted SKB * @slow_gro: state present at GRO time, slower prepare step required @@ -975,6 +983,7 @@ struct sk_buff { #endif __u8 slow_gro:1; __u8 csum_not_inet:1; + __u8 scm_io_uring:1; #ifdef CONFIG_NET_SCHED __u16 tc_index; /* traffic control index */ @@ -1195,7 +1204,8 @@ static inline bool skb_unref(struct sk_buff *skb) return true; } -void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason); +void __fix_address +kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason); /** * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason @@ -1460,8 +1470,8 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector, unsigned int key_count); struct bpf_flow_dissector; -bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, - __be16 proto, int nhoff, int hlen, unsigned int flags); +u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, + __be16 proto, int nhoff, int hlen, unsigned int flags); bool __skb_flow_dissect(const struct net *net, const struct sk_buff *skb, @@ -2444,6 +2454,27 @@ static inline void skb_fill_page_desc(struct sk_buff *skb, int i, skb_shinfo(skb)->nr_frags = i + 1; } +/** + * skb_fill_page_desc_noacc - initialise a paged fragment in an skb + * @skb: buffer containing fragment to be initialised + * @i: paged fragment index to initialise + * @page: the page to use for this fragment + * @off: the offset to the data with @page + * @size: the length of the data + * + * Variant of skb_fill_page_desc() which does not deal with + * pfmemalloc, if page is not owned by us. + */ +static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i, + struct page *page, int off, + int size) +{ + struct skb_shared_info *shinfo = skb_shinfo(skb); + + __skb_fill_page_desc_noacc(shinfo, i, page, off, size); + shinfo->nr_frags = i + 1; +} + void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, int size, unsigned int truesize); @@ -2587,20 +2618,6 @@ void *skb_pull_data(struct sk_buff *skb, size_t len); void *__pskb_pull_tail(struct sk_buff *skb, int delta); -static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len) -{ - if (len > skb_headlen(skb) && - !__pskb_pull_tail(skb, len - skb_headlen(skb))) - return NULL; - skb->len -= len; - return skb->data += len; -} - -static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) -{ - return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len); -} - static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len) { if (likely(len <= skb_headlen(skb))) @@ -2610,6 +2627,15 @@ static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len) return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL; } +static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) +{ + if (!pskb_may_pull(skb, len)) + return NULL; + + skb->len -= len; + return skb->data += len; +} + void skb_condense(struct sk_buff *skb); /** diff --git a/include/linux/slab.h b/include/linux/slab.h index 0fefdf528e0d2e08287fe349dec0aeeed3e5a78b..90877fcde70bd615ab059b57f7ee44a31a67d6f7 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -29,6 +29,8 @@ #define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) /* DEBUG: Poison objects */ #define SLAB_POISON ((slab_flags_t __force)0x00000800U) +/* Indicate a kmalloc slab */ +#define SLAB_KMALLOC ((slab_flags_t __force)0x00001000U) /* Align objs on cache lines */ #define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) /* Use GFP_DMA memory */ @@ -106,7 +108,7 @@ # define SLAB_ACCOUNT 0 #endif -#ifdef CONFIG_KASAN +#ifdef CONFIG_KASAN_GENERIC #define SLAB_KASAN ((slab_flags_t __force)0x08000000U) #else #define SLAB_KASAN 0 @@ -119,6 +121,12 @@ */ #define SLAB_NO_USER_FLAGS ((slab_flags_t __force)0x10000000U) +#ifdef CONFIG_KFENCE +#define SLAB_SKIP_KFENCE ((slab_flags_t __force)0x20000000U) +#else +#define SLAB_SKIP_KFENCE 0 +#endif + /* The following flags affect the page allocator grouping pages by mobility */ /* Objects are reclaimable */ #define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) @@ -184,11 +192,25 @@ int kmem_cache_shrink(struct kmem_cache *s); /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __alloc_size(2); +void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2); void kfree(const void *objp); void kfree_sensitive(const void *objp); size_t __ksize(const void *objp); + +/** + * ksize - Report actual allocation size of associated object + * + * @objp: Pointer returned from a prior kmalloc()-family allocation. + * + * This should not be used for writing beyond the originally requested + * allocation size. Either use krealloc() or round up the allocation size + * with kmalloc_size_roundup() prior to allocation. If this is used to + * access beyond the originally requested allocation size, UBSAN_BOUNDS + * and/or FORTIFY_SOURCE may trip, since they only know about the + * originally allocated size via the __alloc_size attribute. + */ size_t ksize(const void *objp); + #ifdef CONFIG_PRINTK bool kmem_valid_obj(void *object); void kmem_dump_obj(void *object); @@ -243,27 +265,17 @@ static inline unsigned int arch_slab_minalign(void) #ifdef CONFIG_SLAB /* - * The largest kmalloc size supported by the SLAB allocators is - * 32 megabyte (2^25) or the maximum allocatable page order if that is - * less than 32 MB. - * - * WARNING: Its not easy to increase this value since the allocators have - * to do various tricks to work around compiler limitations in order to - * ensure proper constant folding. + * SLAB and SLUB directly allocates requests fitting in to an order-1 page + * (PAGE_SIZE*2). Larger requests are passed to the page allocator. */ -#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ - (MAX_ORDER + PAGE_SHIFT - 1) : 25) -#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) #ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5 #endif #endif #ifdef CONFIG_SLUB -/* - * SLUB directly allocates requests fitting in to an order-1 page - * (PAGE_SIZE*2). Larger requests are passed to the page allocator. - */ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) #ifndef KMALLOC_SHIFT_LOW @@ -415,10 +427,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size, if (size <= 512 * 1024) return 19; if (size <= 1024 * 1024) return 20; if (size <= 2 * 1024 * 1024) return 21; - if (size <= 4 * 1024 * 1024) return 22; - if (size <= 8 * 1024 * 1024) return 23; - if (size <= 16 * 1024 * 1024) return 24; - if (size <= 32 * 1024 * 1024) return 25; if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant) BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()"); @@ -428,6 +436,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size, /* Will never be reached. Needed because the compiler may complain */ return -1; } +static_assert(PAGE_SHIFT <= 20); #define kmalloc_index(s) __kmalloc_index(s, true) #endif /* !CONFIG_SLOB */ @@ -456,42 +465,22 @@ static __always_inline void kfree_bulk(size_t size, void **p) kmem_cache_free_bulk(NULL, size, p); } -#ifdef CONFIG_NUMA void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __alloc_size(1); void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment __malloc; -#else -static __always_inline __alloc_size(1) void *__kmalloc_node(size_t size, gfp_t flags, int node) -{ - return __kmalloc(size, flags); -} - -static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) -{ - return kmem_cache_alloc(s, flags); -} -#endif #ifdef CONFIG_TRACING -extern void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) - __assume_slab_alignment __alloc_size(3); - -#ifdef CONFIG_NUMA -extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) __assume_slab_alignment - __alloc_size(4); -#else -static __always_inline __alloc_size(4) void *kmem_cache_alloc_node_trace(struct kmem_cache *s, - gfp_t gfpflags, int node, size_t size) -{ - return kmem_cache_alloc_trace(s, gfpflags, size); -} -#endif /* CONFIG_NUMA */ +void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) + __assume_kmalloc_alignment __alloc_size(3); +void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) __assume_kmalloc_alignment + __alloc_size(4); #else /* CONFIG_TRACING */ -static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s, - gfp_t flags, size_t size) +/* Save a function call when CONFIG_TRACING=n */ +static __always_inline __alloc_size(3) +void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) { void *ret = kmem_cache_alloc(s, flags); @@ -499,8 +488,9 @@ static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_ return ret; } -static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) +static __always_inline __alloc_size(4) +void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) { void *ret = kmem_cache_alloc_node(s, gfpflags, node); @@ -509,25 +499,11 @@ static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, g } #endif /* CONFIG_TRACING */ -extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment - __alloc_size(1); +void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment + __alloc_size(1); -#ifdef CONFIG_TRACING -extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) - __assume_page_alignment __alloc_size(1); -#else -static __always_inline __alloc_size(1) void *kmalloc_order_trace(size_t size, gfp_t flags, - unsigned int order) -{ - return kmalloc_order(size, flags, order); -} -#endif - -static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags) -{ - unsigned int order = get_order(size); - return kmalloc_order_trace(size, flags, order); -} +void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment + __alloc_size(1); /** * kmalloc - allocate memory @@ -597,7 +573,7 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_trace( + return kmalloc_trace( kmalloc_caches[kmalloc_type(flags)][index], flags, size); #endif @@ -605,23 +581,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) return __kmalloc(size, flags); } +#ifndef CONFIG_SLOB static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) { -#ifndef CONFIG_SLOB - if (__builtin_constant_p(size) && - size <= KMALLOC_MAX_CACHE_SIZE) { - unsigned int i = kmalloc_index(size); + if (__builtin_constant_p(size)) { + unsigned int index; - if (!i) + if (size > KMALLOC_MAX_CACHE_SIZE) + return kmalloc_large_node(size, flags, node); + + index = kmalloc_index(size); + + if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_node_trace( - kmalloc_caches[kmalloc_type(flags)][i], - flags, node, size); + return kmalloc_node_trace( + kmalloc_caches[kmalloc_type(flags)][index], + flags, node, size); } -#endif return __kmalloc_node(size, flags, node); } +#else +static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) +{ + if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE) + return kmalloc_large_node(size, flags, node); + + return __kmalloc_node(size, flags, node); +} +#endif /** * kmalloc_array - allocate memory for an array. @@ -647,10 +635,10 @@ static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_ * @new_size: new size of a single member of the array * @flags: the type of memory to allocate (see kmalloc) */ -static inline __alloc_size(2, 3) void * __must_check krealloc_array(void *p, - size_t new_n, - size_t new_size, - gfp_t flags) +static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, + size_t new_n, + size_t new_size, + gfp_t flags) { size_t bytes; @@ -671,6 +659,12 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag return kmalloc_array(n, size, flags | __GFP_ZERO); } +void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, + unsigned long caller) __alloc_size(1); +#define kmalloc_node_track_caller(size, flags, node) \ + __kmalloc_node_track_caller(size, flags, node, \ + _RET_IP_) + /* * kmalloc_track_caller is a special version of kmalloc that records the * calling function of the routine calling it for slab leak tracking instead @@ -679,9 +673,9 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag * allocator where we care about the real place the memory allocation * request comes from. */ -extern void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller); #define kmalloc_track_caller(size, flags) \ - __kmalloc_track_caller(size, flags, _RET_IP_) + __kmalloc_node_track_caller(size, flags, \ + NUMA_NO_NODE, _RET_IP_) static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, int node) @@ -700,21 +694,6 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); } - -#ifdef CONFIG_NUMA -extern void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, - unsigned long caller) __alloc_size(1); -#define kmalloc_node_track_caller(size, flags, node) \ - __kmalloc_node_track_caller(size, flags, node, \ - _RET_IP_) - -#else /* CONFIG_NUMA */ - -#define kmalloc_node_track_caller(size, flags, node) \ - kmalloc_track_caller(size, flags) - -#endif /* CONFIG_NUMA */ - /* * Shortcuts */ @@ -774,11 +753,28 @@ static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t fla } extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) - __alloc_size(3); + __realloc_size(3); extern void kvfree(const void *addr); extern void kvfree_sensitive(const void *addr, size_t len); unsigned int kmem_cache_size(struct kmem_cache *s); + +/** + * kmalloc_size_roundup - Report allocation bucket size for the given size + * + * @size: Number of bytes to round up from. + * + * This returns the number of bytes that would be available in a kmalloc() + * allocation of @size bytes. For example, a 126 byte request would be + * rounded up to the next sized kmalloc bucket, 128 bytes. (This is strictly + * for the general-purpose kmalloc()-based allocations, and is not for the + * pre-sized kmem_cache_alloc()-based allocations.) + * + * Use this to kmalloc() the full bucket size ahead of time instead of using + * ksize() to query the size after an allocation. + */ +size_t kmalloc_size_roundup(size_t size); + void __init kmem_cache_init_late(void); #if defined(CONFIG_SMP) && defined(CONFIG_SLAB) diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index e24c9aff6fed0cdc8abefabc97f0acbf1bf1f01c..f0ffad6a336531e0383cc2162b481241ef1ad440 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -33,7 +33,6 @@ struct kmem_cache { size_t colour; /* cache colouring range */ unsigned int colour_off; /* colour offset */ - struct kmem_cache *freelist_cache; unsigned int freelist_size; /* constructor func */ diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 88eb832eac7bb8a9a4aded62db736ac7868d95fc..c9cabb679cd1119c376d80be11f5ca9e3a63d054 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -152,4 +152,16 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message, unsigned long timeout, bool atomic); +/* + * Process incoming messages in atomic context. + * This only guarantees that messages arrive as far as the recv_message_early + * callback; drivers expecting to handle incoming messages synchronously + * by calling this function must do it that way. + * Will return 1 if some data was processed, 0 if none was, or a + * negative error code on failure. + * + * @rtk: RTKit reference + */ +int apple_rtkit_poll(struct apple_rtkit *rtk); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/soc/mediatek/mtk-mmsys.h b/include/linux/soc/mediatek/mtk-mmsys.h index 59117d970daf9cbd6f0930134382c2eae7d68278..d2b02bb43768326a20d8c79edc3c0a860f0f9e96 100644 --- a/include/linux/soc/mediatek/mtk-mmsys.h +++ b/include/linux/soc/mediatek/mtk-mmsys.h @@ -65,4 +65,6 @@ void mtk_mmsys_ddp_disconnect(struct device *dev, enum mtk_ddp_comp_id cur, enum mtk_ddp_comp_id next); +void mtk_mmsys_ddp_dpi_fmt_config(struct device *dev, u32 val); + #endif /* __MTK_MMSYS_H */ diff --git a/include/linux/soc/mediatek/mtk-mutex.h b/include/linux/soc/mediatek/mtk-mutex.h index a0f4f51a3b45dc2e7849020b3a37aff9414600ed..b335c2837cd8b074503c3d7f67862f4d546b88b6 100644 --- a/include/linux/soc/mediatek/mtk-mutex.h +++ b/include/linux/soc/mediatek/mtk-mutex.h @@ -20,6 +20,8 @@ enum mtk_mutex_mod_index { MUTEX_MOD_IDX_MDP_WDMA, MUTEX_MOD_IDX_MDP_AAL0, MUTEX_MOD_IDX_MDP_CCORR0, + MUTEX_MOD_IDX_MDP_HDR0, + MUTEX_MOD_IDX_MDP_COLOR0, MUTEX_MOD_IDX_MAX /* ALWAYS keep at the end */ }; diff --git a/include/linux/soc/mediatek/mtk_sip_svc.h b/include/linux/soc/mediatek/mtk_sip_svc.h index 082398e0cfb16e6cbc2cdab4b8f2acd3bc3b5742..0761128b435477826deb150b1e2a4f51ded473ec 100644 --- a/include/linux/soc/mediatek/mtk_sip_svc.h +++ b/include/linux/soc/mediatek/mtk_sip_svc.h @@ -22,4 +22,7 @@ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, MTK_SIP_SMC_CONVENTION, \ ARM_SMCCC_OWNER_SIP, fn_id) +/* IOMMU related SMC call */ +#define MTK_SIP_KERNEL_IOMMU_CONTROL MTK_SIP_SMC_CMD(0x514) + #endif diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h index 7e00cca067090a9d78a5eae612a1e656f14168b8..4450c8b7a1cb3310f75aed9722b4edf5ac583e6a 100644 --- a/include/linux/soc/mediatek/mtk_wed.h +++ b/include/linux/soc/mediatek/mtk_wed.h @@ -11,9 +11,15 @@ struct mtk_wed_hw; struct mtk_wdma_desc; +enum mtk_wed_bus_tye { + MTK_WED_BUS_PCIE, + MTK_WED_BUS_AXI, +}; + struct mtk_wed_ring { struct mtk_wdma_desc *desc; dma_addr_t desc_phys; + u32 desc_size; int size; u32 reg_base; @@ -42,13 +48,24 @@ struct mtk_wed_device { /* filled by driver: */ struct { - struct pci_dev *pci_dev; + union { + struct platform_device *platform_dev; + struct pci_dev *pci_dev; + }; + enum mtk_wed_bus_tye bus_type; u32 wpdma_phys; + u32 wpdma_int; + u32 wpdma_mask; + u32 wpdma_tx; + u32 wpdma_txfree; u16 token_start; unsigned int nbuf; + u8 tx_tbit[MTK_WED_TX_QUEUES]; + u8 txfree_tbit; + u32 (*init_buf)(void *ptr, dma_addr_t phys, int token_id); int (*offload_enable)(struct mtk_wed_device *wed); void (*offload_disable)(struct mtk_wed_device *wed); diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index 9ed5384c5ca12c73e488f7302cea8790832169fa..bc2fb8343a94474e32d0ec2a7008c2f46eba406f 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -78,11 +78,40 @@ struct llcc_edac_reg_data { u8 ways_shift; }; +struct llcc_edac_reg_offset { + /* LLCC TRP registers */ + u32 trp_ecc_error_status0; + u32 trp_ecc_error_status1; + u32 trp_ecc_sb_err_syn0; + u32 trp_ecc_db_err_syn0; + u32 trp_ecc_error_cntr_clear; + u32 trp_interrupt_0_status; + u32 trp_interrupt_0_clear; + u32 trp_interrupt_0_enable; + + /* LLCC Common registers */ + u32 cmn_status0; + u32 cmn_interrupt_0_enable; + u32 cmn_interrupt_2_enable; + + /* LLCC DRP registers */ + u32 drp_ecc_error_cfg; + u32 drp_ecc_error_cntr_clear; + u32 drp_interrupt_status; + u32 drp_interrupt_clear; + u32 drp_interrupt_enable; + u32 drp_ecc_error_status0; + u32 drp_ecc_error_status1; + u32 drp_ecc_sb_err_syn0; + u32 drp_ecc_db_err_syn0; +}; + /** * struct llcc_drv_data - Data associated with the llcc driver * @regmap: regmap associated with the llcc device * @bcast_regmap: regmap associated with llcc broadcast offset * @cfg: pointer to the data structure for slice configuration + * @edac_reg_offset: Offset of the LLCC EDAC registers * @lock: mutex associated with each slice * @cfg_size: size of the config data table * @max_slices: max slices as read from device tree @@ -96,6 +125,7 @@ struct llcc_drv_data { struct regmap *regmap; struct regmap *bcast_regmap; const struct llcc_slice_config *cfg; + const struct llcc_edac_reg_offset *edac_reg_offset; struct mutex lock; u32 cfg_size; u32 max_slices; diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h index b1f80e756d2af55e39a9501f251dea37ba18a4a5..469e02d2aa0d8157ab4732c0960dea73e52590fe 100644 --- a/include/linux/soc/qcom/qmi.h +++ b/include/linux/soc/qcom/qmi.h @@ -75,7 +75,7 @@ struct qmi_elem_info { enum qmi_array_type array_type; u8 tlv_type; u32 offset; - struct qmi_elem_info *ei_array; + const struct qmi_elem_info *ei_array; }; #define QMI_RESULT_SUCCESS_V01 0 @@ -102,7 +102,7 @@ struct qmi_response_type_v01 { u16 error; }; -extern struct qmi_elem_info qmi_response_type_v01_ei[]; +extern const struct qmi_elem_info qmi_response_type_v01_ei[]; /** * struct qmi_service - context to track lookup-results @@ -173,7 +173,7 @@ struct qmi_txn { struct completion completion; int result; - struct qmi_elem_info *ei; + const struct qmi_elem_info *ei; void *dest; }; @@ -189,7 +189,7 @@ struct qmi_msg_handler { unsigned int type; unsigned int msg_id; - struct qmi_elem_info *ei; + const struct qmi_elem_info *ei; size_t decoded_size; void (*fn)(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, @@ -249,23 +249,23 @@ void qmi_handle_release(struct qmi_handle *qmi); ssize_t qmi_send_request(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, struct qmi_txn *txn, int msg_id, size_t len, - struct qmi_elem_info *ei, const void *c_struct); + const struct qmi_elem_info *ei, const void *c_struct); ssize_t qmi_send_response(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, struct qmi_txn *txn, int msg_id, size_t len, - struct qmi_elem_info *ei, const void *c_struct); + const struct qmi_elem_info *ei, const void *c_struct); ssize_t qmi_send_indication(struct qmi_handle *qmi, struct sockaddr_qrtr *sq, - int msg_id, size_t len, struct qmi_elem_info *ei, + int msg_id, size_t len, const struct qmi_elem_info *ei, const void *c_struct); void *qmi_encode_message(int type, unsigned int msg_id, size_t *len, - unsigned int txn_id, struct qmi_elem_info *ei, + unsigned int txn_id, const struct qmi_elem_info *ei, const void *c_struct); int qmi_decode_message(const void *buf, size_t len, - struct qmi_elem_info *ei, void *c_struct); + const struct qmi_elem_info *ei, void *c_struct); int qmi_txn_init(struct qmi_handle *qmi, struct qmi_txn *txn, - struct qmi_elem_info *ei, void *c_struct); + const struct qmi_elem_info *ei, void *c_struct); int qmi_txn_wait(struct qmi_txn *txn, unsigned long timeout); void qmi_txn_cancel(struct qmi_txn *txn); diff --git a/include/linux/soc/qcom/smd-rpm.h b/include/linux/soc/qcom/smd-rpm.h index 82c9d489833a6900214ea0eb239a297418c172b9..3ab8c07f71c0cea1ed8bad65a13a417cdd75eacd 100644 --- a/include/linux/soc/qcom/smd-rpm.h +++ b/include/linux/soc/qcom/smd-rpm.h @@ -41,6 +41,7 @@ struct qcom_smd_rpm; #define QCOM_SMD_RPM_HWKM_CLK 0x6d6b7768 #define QCOM_SMD_RPM_PKA_CLK 0x616b70 #define QCOM_SMD_RPM_MCFG_CLK 0x6766636d +#define QCOM_SMD_RPM_MMXI_CLK 0x69786d6d int qcom_rpm_smd_write(struct qcom_smd_rpm *rpm, int state, diff --git a/include/linux/soc/sunxi/sunxi_sram.h b/include/linux/soc/sunxi/sunxi_sram.h index c5f663bba9c28930ba31db7849d3385b027fee94..60e274d1b821b7bcd6faeceb46579f3609122df3 100644 --- a/include/linux/soc/sunxi/sunxi_sram.h +++ b/include/linux/soc/sunxi/sunxi_sram.h @@ -14,6 +14,6 @@ #define _SUNXI_SRAM_H_ int sunxi_sram_claim(struct device *dev); -int sunxi_sram_release(struct device *dev); +void sunxi_sram_release(struct device *dev); #endif /* _SUNXI_SRAM_H_ */ diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index d45902fb4cad9ca001a4c95ba88f2344cebb4af3..bae5e2369b4f7a441a6f15149097afa3f18c6488 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -64,6 +64,11 @@ static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset, return 0; } +static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size) +{ + return copy_to_sockptr_offset(dst, 0, src, size); +} + static inline void *memdup_sockptr(sockptr_t src, size_t len) { void *p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN); diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index 39058c841469f258646d67b1d807b3165e4ace5d..9e4537f409c295714354e16676223c541834974a 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -839,6 +839,8 @@ struct sdw_defer { * @set_bus_conf: Set the bus configuration * @pre_bank_switch: Callback for pre bank switch * @post_bank_switch: Callback for post bank switch + * @read_ping_status: Read status from PING frames, reported with two bits per Device. + * Bits 31:24 are reserved. */ struct sdw_master_ops { int (*read_prop)(struct sdw_bus *bus); @@ -855,6 +857,7 @@ struct sdw_master_ops { struct sdw_bus_params *params); int (*pre_bank_switch)(struct sdw_bus *bus); int (*post_bank_switch)(struct sdw_bus *bus); + u32 (*read_ping_status)(struct sdw_bus *bus); }; @@ -889,6 +892,9 @@ struct sdw_master_ops { * meaningful if multi_link is set. If set to 1, hardware-based * synchronization will be used even if a stream only uses a single * SoundWire segment. + * @dev_num_ida_min: if set, defines the minimum values for the IDA + * used to allocate system-unique device numbers. This value needs to be + * identical across all SoundWire bus in the system. */ struct sdw_bus { struct device *dev; @@ -913,12 +919,15 @@ struct sdw_bus { u32 bank_switch_timeout; bool multi_link; int hw_sync_min_links; + int dev_num_ida_min; }; int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent, struct fwnode_handle *fwnode); void sdw_bus_master_delete(struct sdw_bus *bus); +void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay); + /** * sdw_port_config: Master or Slave Port configuration * diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index ec16ae49e6a43c3a54cf9c8d2b377a36b5dca12e..2e9fd91572d4cca908a8f4e749f61e20cae93254 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -15,32 +15,21 @@ #define SDW_LINK_SIZE 0x10000 /* Intel SHIM Registers Definition */ +/* LCAP */ #define SDW_SHIM_LCAP 0x0 -#define SDW_SHIM_LCTL 0x4 -#define SDW_SHIM_IPPTR 0x8 -#define SDW_SHIM_SYNC 0xC - -#define SDW_SHIM_CTLSCAP(x) (0x010 + 0x60 * (x)) -#define SDW_SHIM_CTLS0CM(x) (0x012 + 0x60 * (x)) -#define SDW_SHIM_CTLS1CM(x) (0x014 + 0x60 * (x)) -#define SDW_SHIM_CTLS2CM(x) (0x016 + 0x60 * (x)) -#define SDW_SHIM_CTLS3CM(x) (0x018 + 0x60 * (x)) -#define SDW_SHIM_PCMSCAP(x) (0x020 + 0x60 * (x)) +#define SDW_SHIM_LCAP_LCOUNT_MASK GENMASK(2, 0) -#define SDW_SHIM_PCMSYCHM(x, y) (0x022 + (0x60 * (x)) + (0x2 * (y))) -#define SDW_SHIM_PCMSYCHC(x, y) (0x042 + (0x60 * (x)) + (0x2 * (y))) -#define SDW_SHIM_PDMSCAP(x) (0x062 + 0x60 * (x)) -#define SDW_SHIM_IOCTL(x) (0x06C + 0x60 * (x)) -#define SDW_SHIM_CTMCTL(x) (0x06E + 0x60 * (x)) - -#define SDW_SHIM_WAKEEN 0x190 -#define SDW_SHIM_WAKESTS 0x192 +/* LCTL */ +#define SDW_SHIM_LCTL 0x4 #define SDW_SHIM_LCTL_SPA BIT(0) #define SDW_SHIM_LCTL_SPA_MASK GENMASK(3, 0) #define SDW_SHIM_LCTL_CPA BIT(8) #define SDW_SHIM_LCTL_CPA_MASK GENMASK(11, 8) +/* SYNC */ +#define SDW_SHIM_SYNC 0xC + #define SDW_SHIM_SYNC_SYNCPRD_VAL_24 (24000 / SDW_CADENCE_GSYNC_KHZ - 1) #define SDW_SHIM_SYNC_SYNCPRD_VAL_38_4 (38400 / SDW_CADENCE_GSYNC_KHZ - 1) #define SDW_SHIM_SYNC_SYNCPRD GENMASK(14, 0) @@ -49,19 +38,33 @@ #define SDW_SHIM_SYNC_CMDSYNC BIT(16) #define SDW_SHIM_SYNC_SYNCGO BIT(24) +/* Control stream capabililities and channel mask */ +#define SDW_SHIM_CTLSCAP(x) (0x010 + 0x60 * (x)) +#define SDW_SHIM_CTLS0CM(x) (0x012 + 0x60 * (x)) +#define SDW_SHIM_CTLS1CM(x) (0x014 + 0x60 * (x)) +#define SDW_SHIM_CTLS2CM(x) (0x016 + 0x60 * (x)) +#define SDW_SHIM_CTLS3CM(x) (0x018 + 0x60 * (x)) + +/* PCM Stream capabilities */ +#define SDW_SHIM_PCMSCAP(x) (0x020 + 0x60 * (x)) + #define SDW_SHIM_PCMSCAP_ISS GENMASK(3, 0) #define SDW_SHIM_PCMSCAP_OSS GENMASK(7, 4) #define SDW_SHIM_PCMSCAP_BSS GENMASK(12, 8) +/* PCM Stream Channel Map */ +#define SDW_SHIM_PCMSYCHM(x, y) (0x022 + (0x60 * (x)) + (0x2 * (y))) + +/* PCM Stream Channel Count */ +#define SDW_SHIM_PCMSYCHC(x, y) (0x042 + (0x60 * (x)) + (0x2 * (y))) + #define SDW_SHIM_PCMSYCM_LCHN GENMASK(3, 0) #define SDW_SHIM_PCMSYCM_HCHN GENMASK(7, 4) #define SDW_SHIM_PCMSYCM_STREAM GENMASK(13, 8) #define SDW_SHIM_PCMSYCM_DIR BIT(15) -#define SDW_SHIM_PDMSCAP_ISS GENMASK(3, 0) -#define SDW_SHIM_PDMSCAP_OSS GENMASK(7, 4) -#define SDW_SHIM_PDMSCAP_BSS GENMASK(12, 8) -#define SDW_SHIM_PDMSCAP_CPSS GENMASK(15, 13) +/* IO control */ +#define SDW_SHIM_IOCTL(x) (0x06C + 0x60 * (x)) #define SDW_SHIM_IOCTL_MIF BIT(0) #define SDW_SHIM_IOCTL_CO BIT(1) @@ -73,13 +76,23 @@ #define SDW_SHIM_IOCTL_CIBD BIT(8) #define SDW_SHIM_IOCTL_DIBD BIT(9) -#define SDW_SHIM_CTMCTL_DACTQE BIT(0) -#define SDW_SHIM_CTMCTL_DODS BIT(1) -#define SDW_SHIM_CTMCTL_DOAIS GENMASK(4, 3) +/* Wake Enable*/ +#define SDW_SHIM_WAKEEN 0x190 #define SDW_SHIM_WAKEEN_ENABLE BIT(0) + +/* Wake Status */ +#define SDW_SHIM_WAKESTS 0x192 + #define SDW_SHIM_WAKESTS_STATUS BIT(0) +/* AC Timing control */ +#define SDW_SHIM_CTMCTL(x) (0x06E + 0x60 * (x)) + +#define SDW_SHIM_CTMCTL_DACTQE BIT(0) +#define SDW_SHIM_CTMCTL_DODS BIT(1) +#define SDW_SHIM_CTMCTL_DOAIS GENMASK(4, 3) + /* Intel ALH Register definitions */ #define SDW_ALH_STRMZCFG(x) (0x000 + (0x4 * (x))) #define SDW_ALH_NUM_STREAMS 64 diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index e6c73d5ff1a8298dcde9641f187cd6d8806ca20d..fbf8c0d95968e4a909e214130ba2dcc8b655ace6 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -378,6 +378,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @cleanup: frees controller-specific state * @can_dma: determine whether this controller supports DMA * @dma_map_dev: device which can be used for DMA mapping + * @cur_rx_dma_dev: device which is currently used for RX DMA mapping + * @cur_tx_dma_dev: device which is currently used for TX DMA mapping * @queued: whether this controller is providing an internal message queue * @kworker: pointer to thread struct for message pump * @pump_messages: work struct for scheduling work to the message pump @@ -469,6 +471,7 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * SPI_TRANS_FAIL_NO_START. * @queue_empty: signal green light for opportunistically skipping the queue * for spi_sync transfers. + * @must_async: disable all fast paths in the core * * Each SPI controller can communicate with one or more @spi_device * children. These make a small bus, sharing MOSI, MISO and SCK signals @@ -609,6 +612,8 @@ struct spi_controller { struct spi_device *spi, struct spi_transfer *xfer); struct device *dma_map_dev; + struct device *cur_rx_dma_dev; + struct device *cur_tx_dma_dev; /* * These hooks are for drivers that want to use the generic @@ -690,6 +695,7 @@ struct spi_controller { /* Flag for enabling opportunistic skipping of the queue in spi_sync */ bool queue_empty; + bool must_async; }; static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) @@ -846,6 +852,7 @@ struct spi_res { * @bits_per_word: select a bits_per_word other than the device default * for this transfer. If 0 the default (from @spi_device) is used. * @dummy_data: indicates transfer is dummy bytes transfer. + * @cs_off: performs the transfer with chipselect off. * @cs_change: affects chipselect after this transfer completes * @cs_change_delay: delay between cs deassert and assert when * @cs_change is set and @spi_transfer is not the last in @spi_message @@ -956,6 +963,7 @@ struct spi_transfer { struct sg_table rx_sg; unsigned dummy_data:1; + unsigned cs_off:1; unsigned cs_change:1; unsigned tx_nbits:3; unsigned rx_nbits:3; diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 5c0c5174155d05a5ee3228b7fedd25b8c7a96f9d..1341f7d62da44663d48630991e7455e364430b63 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __LINUX_SPINLOCK_H #define __LINUX_SPINLOCK_H +#define __LINUX_INSIDE_SPINLOCK_H /* * include/linux/spinlock.h - generic spinlock/rwlock declarations @@ -492,4 +493,5 @@ int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask, void free_bucket_spinlocks(spinlock_t *locks); +#undef __LINUX_INSIDE_SPINLOCK_H #endif /* __LINUX_SPINLOCK_H */ diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 51fa0dab68c4dc6af3c8922694f257ea9db3e2d2..89eb6f4c659c706c1b7376534a25d53a316316b7 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -1,7 +1,7 @@ #ifndef __LINUX_SPINLOCK_API_SMP_H #define __LINUX_SPINLOCK_API_SMP_H -#ifndef __LINUX_SPINLOCK_H +#ifndef __LINUX_INSIDE_SPINLOCK_H # error "please don't include this file directly" #endif diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index b8ba00ccccdebd44e7ad723c66e772e434e73aa8..819aeba1c87e685bfafaf78e2d781d6e432b9877 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -1,7 +1,7 @@ #ifndef __LINUX_SPINLOCK_API_UP_H #define __LINUX_SPINLOCK_API_UP_H -#ifndef __LINUX_SPINLOCK_H +#ifndef __LINUX_INSIDE_SPINLOCK_H # error "please don't include this file directly" #endif diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 835aedaf68acd5ab8a6109659ffd8227c755ebec..61c49b16f69ab03ba053ace72c421385309ad713 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -2,7 +2,7 @@ #ifndef __LINUX_SPINLOCK_RT_H #define __LINUX_SPINLOCK_RT_H -#ifndef __LINUX_SPINLOCK_H +#ifndef __LINUX_INSIDE_SPINLOCK_H #error Do not include directly. Use spinlock.h #endif diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h index 16521074b6f7c8576339fd516401915685800494..c87204247592fd2a037c071eaf1fbf93abd16aeb 100644 --- a/include/linux/spinlock_up.h +++ b/include/linux/spinlock_up.h @@ -1,7 +1,7 @@ #ifndef __LINUX_SPINLOCK_UP_H #define __LINUX_SPINLOCK_UP_H -#ifndef __LINUX_SPINLOCK_H +#ifndef __LINUX_INSIDE_SPINLOCK_H # error "please don't include this file directly" #endif diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 6cfaa0a9a9b961f467524de59edbf8aedb73f50d..5aa5e0faf6a121a2770e6362cf368ce3efceefc7 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -15,10 +15,10 @@ struct srcu_struct { short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ - unsigned short srcu_idx; /* Current reader array element in bit 0x2. */ - unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */ u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ + unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ + unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ struct swait_queue_head srcu_wq; /* Last srcu_read_unlock() wakes GP. */ struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ @@ -82,10 +82,12 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, int idx; idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1; - pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n", + pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n", tt, tf, idx, data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])), - data_race(READ_ONCE(ssp->srcu_lock_nesting[idx]))); + data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])), + data_race(READ_ONCE(ssp->srcu_idx)), + data_race(READ_ONCE(ssp->srcu_idx_max))); } #endif diff --git a/include/linux/stackdepot.h b/include/linux/stackdepot.h index bc2797955de9020a3ba1296ecfb791f7bf610621..9ca7798d7a318c92522da45488992c2ff81a0d04 100644 --- a/include/linux/stackdepot.h +++ b/include/linux/stackdepot.h @@ -14,9 +14,15 @@ #include typedef u32 depot_stack_handle_t; +/* + * Number of bits in the handle that stack depot doesn't use. Users may store + * information in them. + */ +#define STACK_DEPOT_EXTRA_BITS 5 depot_stack_handle_t __stack_depot_save(unsigned long *entries, unsigned int nr_entries, + unsigned int extra_bits, gfp_t gfp_flags, bool can_alloc); /* @@ -59,6 +65,8 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int stack_depot_fetch(depot_stack_handle_t handle, unsigned long **entries); +unsigned int stack_depot_get_extra_bits(depot_stack_handle_t handle); + int stack_depot_snprint(depot_stack_handle_t handle, char *buf, size_t size, int spaces); diff --git a/include/linux/stat.h b/include/linux/stat.h index 7df06931f25d857e4266d507a7d192bad402fc08..ff277ced50e9fda9f9fdb4c177524f5c7a3e4b94 100644 --- a/include/linux/stat.h +++ b/include/linux/stat.h @@ -50,6 +50,8 @@ struct kstat { struct timespec64 btime; /* File creation time */ u64 blocks; u64 mnt_id; + u32 dio_mem_align; + u32 dio_offset_align; }; #endif diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 8df475db88c040456e887696ac725f1b50451c69..fb2e88614f5d17e77c1b459b7f20b2a0032b5230 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -257,7 +257,6 @@ struct plat_stmmacenet_data { u8 vlan_fail_q; unsigned int eee_usecs_rate; struct pci_dev *pdev; - bool has_crossts; int int_snapshot_num; int ext_snapshot_num; bool int_snapshot_en; diff --git a/include/linux/string.h b/include/linux/string.h index 61ec7e4f6311a8c87b2793eb3e8cdc25a7bf5f6e..cf7607b321027a7c108dacbbf5cb7bf19a6c4978 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -260,6 +260,49 @@ static inline const char *kbasename(const char *path) void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count, int pad); +/** + * strtomem_pad - Copy NUL-terminated string to non-NUL-terminated buffer + * + * @dest: Pointer of destination character array (marked as __nonstring) + * @src: Pointer to NUL-terminated string + * @pad: Padding character to fill any remaining bytes of @dest after copy + * + * This is a replacement for strncpy() uses where the destination is not + * a NUL-terminated string, but with bounds checking on the source size, and + * an explicit padding character. If padding is not required, use strtomem(). + * + * Note that the size of @dest is not an argument, as the length of @dest + * must be discoverable by the compiler. + */ +#define strtomem_pad(dest, src, pad) do { \ + const size_t _dest_len = __builtin_object_size(dest, 1); \ + \ + BUILD_BUG_ON(!__builtin_constant_p(_dest_len) || \ + _dest_len == (size_t)-1); \ + memcpy_and_pad(dest, _dest_len, src, strnlen(src, _dest_len), pad); \ +} while (0) + +/** + * strtomem - Copy NUL-terminated string to non-NUL-terminated buffer + * + * @dest: Pointer of destination character array (marked as __nonstring) + * @src: Pointer to NUL-terminated string + * + * This is a replacement for strncpy() uses where the destination is not + * a NUL-terminated string, but with bounds checking on the source size, and + * without trailing padding. If padding is required, use strtomem_pad(). + * + * Note that the size of @dest is not an argument, as the length of @dest + * must be discoverable by the compiler. + */ +#define strtomem(dest, src) do { \ + const size_t _dest_len = __builtin_object_size(dest, 1); \ + \ + BUILD_BUG_ON(!__builtin_constant_p(_dest_len) || \ + _dest_len == (size_t)-1); \ + memcpy(dest, src, min(_dest_len, strnlen(src, _dest_len))); \ +} while (0) + /** * memset_after - Set a value after a struct member to the end of a struct * diff --git a/include/linux/string_helpers.h b/include/linux/string_helpers.h index 4d72258d42fd9249d6a21021f2e0c5e98298db01..8530c7328269257eb19ffdcd94e35ce7f3e50e07 100644 --- a/include/linux/string_helpers.h +++ b/include/linux/string_helpers.h @@ -21,6 +21,8 @@ enum string_size_units { void string_get_size(u64 size, u64 blk_size, enum string_size_units units, char *buf, int len); +int parse_int_array_user(const char __user *from, size_t count, int **array); + #define UNESCAPE_SPACE BIT(0) #define UNESCAPE_OCTAL BIT(1) #define UNESCAPE_HEX BIT(2) @@ -126,4 +128,9 @@ static inline const char *str_enabled_disabled(bool v) return v ? "enabled" : "disabled"; } +static inline const char *str_read_write(bool v) +{ + return v ? "read" : "write"; +} + #endif diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 75eea5ebb179b0519979f33b4c09869b8ab1ceec..770ef2cb57752fe6fddb71700f913b4148881ad7 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -246,6 +246,7 @@ void rpc_clnt_xprt_switch_remove_xprt(struct rpc_clnt *, struct rpc_xprt *); bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, const struct sockaddr *sap); void rpc_clnt_xprt_set_online(struct rpc_clnt *clnt, struct rpc_xprt *xprt); +void rpc_clnt_disconnect(struct rpc_clnt *clnt); void rpc_cleanup_clids(void); static inline int rpc_reply_expected(struct rpc_task *task) diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index acc62647317c6a9346461963841f6f924258466f..b8ca3ecaf8d76b88b459b806bade1be8ef371f98 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -209,11 +209,17 @@ struct rpc_task *rpc_run_task(const struct rpc_task_setup *); struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req); void rpc_put_task(struct rpc_task *); void rpc_put_task_async(struct rpc_task *); +bool rpc_task_set_rpc_status(struct rpc_task *task, int rpc_status); +void rpc_task_try_cancel(struct rpc_task *task, int error); void rpc_signal_task(struct rpc_task *); void rpc_exit_task(struct rpc_task *); void rpc_exit(struct rpc_task *, int); void rpc_release_calldata(const struct rpc_call_ops *, void *); void rpc_killall_tasks(struct rpc_clnt *); +unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data); void rpc_execute(struct rpc_task *); void rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *); void rpc_init_wait_queue(struct rpc_wait_queue *, const char *); @@ -252,7 +258,7 @@ int rpc_malloc(struct rpc_task *); void rpc_free(struct rpc_task *); int rpciod_up(void); void rpciod_down(void); -int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *); +int rpc_wait_for_completion_task(struct rpc_task *task); #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) struct net; void rpc_show_tasks(struct net *); @@ -264,11 +270,6 @@ extern struct workqueue_struct *xprtiod_workqueue; void rpc_prepare_task(struct rpc_task *task); gfp_t rpc_task_gfp_mask(void); -static inline int rpc_wait_for_completion_task(struct rpc_task *task) -{ - return __rpc_wait_for_completion_task(task, NULL); -} - #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) static inline const char * rpc_qname(const struct rpc_wait_queue *q) { diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index daecb009c05b56f27e7fce6dbd5e3927c360de29..88de45491376a6c236363becd0770c25a36a3de7 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -472,6 +472,7 @@ struct svc_procedure { /* XDR free result: */ void (*pc_release)(struct svc_rqst *); unsigned int pc_argsize; /* argument struct size */ + unsigned int pc_argzero; /* how much of argument to clear */ unsigned int pc_ressize; /* result struct size */ unsigned int pc_cachetype; /* cache info (NFS) */ unsigned int pc_xdrressize; /* maximum size of XDR reply */ @@ -544,16 +545,27 @@ static inline void svc_reserve_auth(struct svc_rqst *rqstp, int space) } /** - * svcxdr_init_decode - Prepare an xdr_stream for svc Call decoding + * svcxdr_init_decode - Prepare an xdr_stream for Call decoding * @rqstp: controlling server RPC transaction context * + * This function currently assumes the RPC header in rq_arg has + * already been decoded. Upon return, xdr->p points to the + * location of the upper layer header. */ static inline void svcxdr_init_decode(struct svc_rqst *rqstp) { struct xdr_stream *xdr = &rqstp->rq_arg_stream; - struct kvec *argv = rqstp->rq_arg.head; + struct xdr_buf *buf = &rqstp->rq_arg; + struct kvec *argv = buf->head; - xdr_init_decode(xdr, &rqstp->rq_arg, argv->iov_base, NULL); + /* + * svc_getnl() and friends do not keep the xdr_buf's ::len + * field up to date. Refresh that field before initializing + * the argument decoding stream. + */ + buf->len = buf->head->iov_len + buf->page_len + buf->tail->iov_len; + + xdr_init_decode(xdr, buf, argv->iov_base, NULL); xdr_set_scratch_page(xdr, rqstp->rq_scratch_page); } @@ -576,7 +588,7 @@ static inline void svcxdr_init_encode(struct svc_rqst *rqstp) xdr->end = resv->iov_base + PAGE_SIZE - rqstp->rq_auth_slack; buf->len = resv->iov_len; xdr->page_ptr = buf->pages - 1; - buf->buflen = PAGE_SIZE * (1 + rqstp->rq_page_end - buf->pages); + buf->buflen = PAGE_SIZE * (rqstp->rq_page_end - buf->pages); buf->buflen -= rqstp->rq_auth_slack; xdr->rqst = NULL; } diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 69175029abbbf59902621cbe89e20839f961154f..f84e2a1358e17042c1f4ecbfe11c89c51940af57 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -240,6 +240,8 @@ typedef int (*kxdrdproc_t)(struct rpc_rqst *rqstp, struct xdr_stream *xdr, extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, struct rpc_rqst *rqst); +extern void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf, + struct page **pages, struct rpc_rqst *rqst); extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes); extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes); diff --git a/include/linux/suspend.h b/include/linux/suspend.h index 70f2921e2e703248944bda50a2e2b5951d743ef7..cfe19a02891855ff9fe15f1b78bcf0ecb2a30f80 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -75,7 +75,7 @@ extern struct suspend_stats suspend_stats; static inline void dpm_save_failed_dev(const char *name) { - strlcpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], + strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev], name, sizeof(suspend_stats.failed_devs[0])); suspend_stats.last_failed_dev++; @@ -191,6 +191,7 @@ struct platform_s2idle_ops { int (*begin)(void); int (*prepare)(void); int (*prepare_late)(void); + void (*check)(void); bool (*wake)(void); void (*restore_early)(void); void (*restore)(void); @@ -510,8 +511,8 @@ extern bool pm_save_wakeup_count(unsigned int count); extern void pm_wakep_autosleep_enabled(bool set); extern void pm_print_active_wakeup_sources(void); -extern void lock_system_sleep(void); -extern void unlock_system_sleep(void); +extern unsigned int lock_system_sleep(void); +extern void unlock_system_sleep(unsigned int); #else /* !CONFIG_PM_SLEEP */ @@ -534,8 +535,8 @@ static inline void pm_system_wakeup(void) {} static inline void pm_wakeup_clear(bool reset) {} static inline void pm_system_irq_wakeup(unsigned int irq_number) {} -static inline void lock_system_sleep(void) {} -static inline void unlock_system_sleep(void) {} +static inline unsigned int lock_system_sleep(void) { return 0; } +static inline void unlock_system_sleep(unsigned int flags) {} #endif /* !CONFIG_PM_SLEEP */ diff --git a/include/linux/swab.h b/include/linux/swab.h index bcff5149861a9eb0dc2fa3b8da4bf8a80e5dc343..9b804dbb0d794dd7a5b230f48b32b96584b537f3 100644 --- a/include/linux/swab.h +++ b/include/linux/swab.h @@ -20,4 +20,29 @@ # define swab64s __swab64s # define swahw32s __swahw32s # define swahb32s __swahb32s + +static inline void swab16_array(u16 *buf, unsigned int words) +{ + while (words--) { + swab16s(buf); + buf++; + } +} + +static inline void swab32_array(u32 *buf, unsigned int words) +{ + while (words--) { + swab32s(buf); + buf++; + } +} + +static inline void swab64_array(u64 *buf, unsigned int words) +{ + while (words--) { + swab64s(buf); + buf++; + } +} + #endif /* _LINUX_SWAB_H */ diff --git a/include/linux/swap.h b/include/linux/swap.h index 43150b9bbc5caff9f8e3b87d7fbd5ee5376c450f..a18cf4b7c724c05a1a499ea2ac2723ec641cb5b2 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -162,6 +162,10 @@ union swap_header { */ struct reclaim_state { unsigned long reclaimed_slab; +#ifdef CONFIG_LRU_GEN + /* per-thread mm walk data */ + struct lru_gen_mm_walk *mm_walk; +#endif }; #ifdef __KERNEL__ @@ -351,6 +355,11 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio) return entry; } +static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry) +{ + folio->private = (void *)entry.val; +} + /* linux/mm/workingset.c */ void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); @@ -375,11 +384,11 @@ extern unsigned long totalreserve_pages; /* linux/mm/swap.c */ -extern void lru_note_cost(struct lruvec *lruvec, bool file, - unsigned int nr_pages); -extern void lru_note_cost_folio(struct folio *); -extern void folio_add_lru(struct folio *); -extern void lru_cache_add(struct page *); +void lru_note_cost(struct lruvec *lruvec, bool file, unsigned int nr_pages); +void lru_note_cost_folio(struct folio *); +void folio_add_lru(struct folio *); +void folio_add_lru_vma(struct folio *, struct vm_area_struct *); +void lru_cache_add(struct page *); void mark_page_accessed(struct page *); void folio_mark_accessed(struct folio *); @@ -481,7 +490,8 @@ static inline long get_nr_swap_pages(void) extern void si_swapinfo(struct sysinfo *); swp_entry_t folio_alloc_swap(struct folio *folio); -extern void put_swap_page(struct page *page, swp_entry_t entry); +bool folio_free_swap(struct folio *folio); +void put_swap_folio(struct folio *folio, swp_entry_t entry); extern swp_entry_t get_swap_page_of_type(int); extern int get_swap_pages(int n, swp_entry_t swp_entries[], int entry_size); extern int add_swap_count_continuation(swp_entry_t, gfp_t); @@ -500,7 +510,6 @@ extern int __swp_swapcount(swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); extern struct swap_info_struct *page_swap_info(struct page *); extern struct swap_info_struct *swp_swap_info(swp_entry_t entry); -extern int try_to_free_swap(struct page *); struct backing_dev_info; extern int init_swap_address_space(unsigned int type, unsigned long nr_pages); extern void exit_swap_address_space(unsigned int type); @@ -566,7 +575,7 @@ static inline void swap_free(swp_entry_t swp) { } -static inline void put_swap_page(struct page *page, swp_entry_t swp) +static inline void put_swap_folio(struct folio *folio, swp_entry_t swp) { } @@ -585,11 +594,6 @@ static inline int swp_swapcount(swp_entry_t entry) return 0; } -static inline int try_to_free_swap(struct page *page) -{ - return 0; -} - static inline swp_entry_t folio_alloc_swap(struct folio *folio) { swp_entry_t entry; @@ -597,6 +601,11 @@ static inline swp_entry_t folio_alloc_swap(struct folio *folio) return entry; } +static inline bool folio_free_swap(struct folio *folio) +{ + return false; +} + static inline int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block) @@ -657,7 +666,7 @@ static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp) cgroup_throttle_swaprate(&folio->page, gfp); } -#ifdef CONFIG_MEMCG_SWAP +#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP) void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry); int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry); static inline int mem_cgroup_try_charge_swap(struct folio *folio, @@ -677,7 +686,7 @@ static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_p } extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); -extern bool mem_cgroup_swap_full(struct page *page); +extern bool mem_cgroup_swap_full(struct folio *folio); #else static inline void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) { @@ -699,7 +708,7 @@ static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) return get_nr_swap_pages(); } -static inline bool mem_cgroup_swap_full(struct page *page) +static inline bool mem_cgroup_swap_full(struct folio *folio) { return vm_swap_full(); } diff --git a/include/linux/swap_cgroup.h b/include/linux/swap_cgroup.h index a12dd1c3966c909be0c15ac9b08245f12f739d36..ae73a87775b3ab3763ac966b19a79dfd597c01f8 100644 --- a/include/linux/swap_cgroup.h +++ b/include/linux/swap_cgroup.h @@ -4,7 +4,7 @@ #include -#ifdef CONFIG_MEMCG_SWAP +#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP) extern unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, unsigned short old, unsigned short new); @@ -40,6 +40,6 @@ static inline void swap_cgroup_swapoff(int type) return; } -#endif /* CONFIG_MEMCG_SWAP */ +#endif #endif /* __LINUX_SWAP_CGROUP_H */ diff --git a/include/linux/swapfile.h b/include/linux/swapfile.h index 54078542134c137c06867a10b14aab6ba40fbe3a..7ed529a77c5b368bb105a0b4c36cdfed3605ea0b 100644 --- a/include/linux/swapfile.h +++ b/include/linux/swapfile.h @@ -8,6 +8,11 @@ */ extern struct swap_info_struct *swap_info[]; extern unsigned long generic_max_swapfile_size(void); -extern unsigned long max_swapfile_size(void); +unsigned long arch_max_swapfile_size(void); + +/* Maximum swapfile size supported for the arch (not inclusive). */ +extern unsigned long swapfile_maximum_size; +/* Whether swap migration entry supports storing A/D bits for the arch */ +extern bool swap_migration_ad_supported; #endif /* _LINUX_SWAPFILE_H */ diff --git a/include/linux/swapops.h b/include/linux/swapops.h index a3d435bf9f972930d1d9ebae20d000cf071623a9..86b95ccb81bb7b07e916725e7dd5509b918336d7 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -8,6 +8,10 @@ #ifdef CONFIG_MMU +#ifdef CONFIG_SWAP +#include +#endif /* CONFIG_SWAP */ + /* * swapcache pages are stored in the swapper_space radix tree. We want to * get good packing density in that tree, so the index should be dense in @@ -23,6 +27,45 @@ #define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) #define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) +/* + * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To + * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries + * can use the extra bits to store other information besides PFN. + */ +#ifdef MAX_PHYSMEM_BITS +#define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) +#else /* MAX_PHYSMEM_BITS */ +#define SWP_PFN_BITS (BITS_PER_LONG - PAGE_SHIFT) +#endif /* MAX_PHYSMEM_BITS */ +#define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) + +/** + * Migration swap entry specific bitfield definitions. Layout: + * + * |----------+--------------------| + * | swp_type | swp_offset | + * |----------+--------+-+-+-------| + * | | resv |D|A| PFN | + * |----------+--------+-+-+-------| + * + * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) + * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) + * + * Note: A/D bits will be stored in migration entries iff there're enough + * free bits in arch specific swp offset. By default we'll ignore A/D bits + * when migrating a page. Please refer to migration_entry_supports_ad() + * for more information. If there're more bits besides PFN and A/D bits, + * they should be reserved and always be zeros. + */ +#define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) +#define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) +#define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) + +#define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) +#define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) + +static inline bool is_pfn_swap_entry(swp_entry_t entry); + /* Clear all flags but only keep swp_entry_t related information */ static inline pte_t pte_swp_clear_flags(pte_t pte) { @@ -64,6 +107,17 @@ static inline pgoff_t swp_offset(swp_entry_t entry) return entry.val & SWP_OFFSET_MASK; } +/* + * This should only be called upon a pfn swap entry to get the PFN stored + * in the swap entry. Please refers to is_pfn_swap_entry() for definition + * of pfn swap entry. + */ +static inline unsigned long swp_offset_pfn(swp_entry_t entry) +{ + VM_BUG_ON(!is_pfn_swap_entry(entry)); + return swp_offset(entry) & SWP_PFN_MASK; +} + /* check whether a pte points to a swap entry */ static inline int is_swap_pte(pte_t pte) { @@ -240,6 +294,52 @@ static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) return swp_entry(SWP_MIGRATION_WRITE, offset); } +/* + * Returns whether the host has large enough swap offset field to support + * carrying over pgtable A/D bits for page migrations. The result is + * pretty much arch specific. + */ +static inline bool migration_entry_supports_ad(void) +{ +#ifdef CONFIG_SWAP + return swap_migration_ad_supported; +#else /* CONFIG_SWAP */ + return false; +#endif /* CONFIG_SWAP */ +} + +static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) +{ + if (migration_entry_supports_ad()) + return swp_entry(swp_type(entry), + swp_offset(entry) | SWP_MIG_YOUNG); + return entry; +} + +static inline bool is_migration_entry_young(swp_entry_t entry) +{ + if (migration_entry_supports_ad()) + return swp_offset(entry) & SWP_MIG_YOUNG; + /* Keep the old behavior of aging page after migration */ + return false; +} + +static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) +{ + if (migration_entry_supports_ad()) + return swp_entry(swp_type(entry), + swp_offset(entry) | SWP_MIG_DIRTY); + return entry; +} + +static inline bool is_migration_entry_dirty(swp_entry_t entry) +{ + if (migration_entry_supports_ad()) + return swp_offset(entry) & SWP_MIG_DIRTY; + /* Keep the old behavior of clean page after migration */ + return false; +} + extern void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl); extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, @@ -247,8 +347,8 @@ extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, #ifdef CONFIG_HUGETLB_PAGE extern void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl); extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); -#endif -#else +#endif /* CONFIG_HUGETLB_PAGE */ +#else /* CONFIG_MIGRATION */ static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) { return swp_entry(0, 0); @@ -276,7 +376,7 @@ static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, #ifdef CONFIG_HUGETLB_PAGE static inline void __migration_entry_wait_huge(pte_t *ptep, spinlock_t *ptl) { } static inline void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte) { } -#endif +#endif /* CONFIG_HUGETLB_PAGE */ static inline int is_writable_migration_entry(swp_entry_t entry) { return 0; @@ -286,7 +386,26 @@ static inline int is_readable_migration_entry(swp_entry_t entry) return 0; } -#endif +static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) +{ + return entry; +} + +static inline bool is_migration_entry_young(swp_entry_t entry) +{ + return false; +} + +static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) +{ + return entry; +} + +static inline bool is_migration_entry_dirty(swp_entry_t entry) +{ + return false; +} +#endif /* CONFIG_MIGRATION */ typedef unsigned long pte_marker; @@ -369,7 +488,7 @@ static inline int pte_none_mostly(pte_t pte) static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) { - struct page *p = pfn_to_page(swp_offset(entry)); + struct page *p = pfn_to_page(swp_offset_pfn(entry)); /* * Any use of migration entries may only occur while the @@ -387,6 +506,9 @@ static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) */ static inline bool is_pfn_swap_entry(swp_entry_t entry) { + /* Make sure the swp offset can always store the needed fields */ + BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS); + return is_migration_entry(entry) || is_device_private_entry(entry) || is_device_exclusive_entry(entry); } @@ -426,7 +548,7 @@ static inline int is_pmd_migration_entry(pmd_t pmd) { return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); } -#else +#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -455,7 +577,7 @@ static inline int is_pmd_migration_entry(pmd_t pmd) { return 0; } -#endif +#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ #ifdef CONFIG_MEMORY_FAILURE @@ -475,27 +597,17 @@ static inline int is_hwpoison_entry(swp_entry_t entry) return swp_type(entry) == SWP_HWPOISON; } -static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry) -{ - return swp_offset(entry); -} - static inline void num_poisoned_pages_inc(void) { atomic_long_inc(&num_poisoned_pages); } -static inline void num_poisoned_pages_dec(void) -{ - atomic_long_dec(&num_poisoned_pages); -} - static inline void num_poisoned_pages_sub(long i) { atomic_long_sub(i, &num_poisoned_pages); } -#else +#else /* CONFIG_MEMORY_FAILURE */ static inline swp_entry_t make_hwpoison_entry(struct page *page) { @@ -514,7 +626,7 @@ static inline void num_poisoned_pages_inc(void) static inline void num_poisoned_pages_sub(long i) { } -#endif +#endif /* CONFIG_MEMORY_FAILURE */ static inline int non_swap_entry(swp_entry_t entry) { diff --git a/include/linux/syslog.h b/include/linux/syslog.h index 86af908e26633f9dda7ab6cf7300badb7a4d2321..955f80e34d4fda361fd120907930d4595842ca8a 100644 --- a/include/linux/syslog.h +++ b/include/linux/syslog.h @@ -8,6 +8,8 @@ #ifndef _LINUX_SYSLOG_H #define _LINUX_SYSLOG_H +#include + /* Close the log. Currently a NOP. */ #define SYSLOG_ACTION_CLOSE 0 /* Open the log. Currently a NOP. */ @@ -35,5 +37,6 @@ #define SYSLOG_FROM_PROC 1 int do_syslog(int type, char __user *buf, int count, int source); +extern wait_queue_head_t log_wait; #endif /* _LINUX_SYSLOG_H */ diff --git a/include/linux/tcp.h b/include/linux/tcp.h index a9fbe22732c3ba67584df13344955b584f242b54..41b1da621a458026caeab976843f42e63b59a6de 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -295,7 +295,7 @@ struct tcp_sock { u32 packets_out; /* Packets which are "in flight" */ u32 retrans_out; /* Retransmitted packets out */ u32 max_packets_out; /* max packets_out in last window */ - u32 max_packets_seq; /* right edge of max_packets_out flight */ + u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */ u16 urg_data; /* Saved octet of OOB data and control flags */ u8 ecn_flags; /* ECN status bits. */ @@ -388,6 +388,12 @@ struct tcp_sock { u8 bpf_sock_ops_cb_flags; /* Control calling BPF programs * values defined in uapi/linux/tcp.h */ + u8 bpf_chg_cc_inprogress:1; /* In the middle of + * bpf_setsockopt(TCP_CONGESTION), + * it is to avoid the bpf_tcp_cc->init() + * to recur itself by calling + * bpf_setsockopt(TCP_CONGESTION, "itself"). + */ #define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) (TP->bpf_sock_ops_cb_flags & ARG) #else #define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) 0 diff --git a/include/linux/termios_internal.h b/include/linux/termios_internal.h new file mode 100644 index 0000000000000000000000000000000000000000..d77f29e5e2b7156fbad1627e463581d81a9adbf0 --- /dev/null +++ b/include/linux/termios_internal.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_TERMIOS_CONV_H +#define _LINUX_TERMIOS_CONV_H + +#include +#include + +/* intr=^C quit=^\ erase=del kill=^U + eof=^D vtime=\0 vmin=\1 sxtc=\0 + start=^Q stop=^S susp=^Z eol=\0 + reprint=^R discard=^O werase=^W lnext=^V + eol2=\0 +*/ + +#ifdef VDSUSP +#define INIT_C_CC_VDSUSP_EXTRA [VDSUSP] = 'Y'-0x40, +#else +#define INIT_C_CC_VDSUSP_EXTRA +#endif + +#define INIT_C_CC { \ + [VINTR] = 'C'-0x40, \ + [VQUIT] = '\\'-0x40, \ + [VERASE] = '\177', \ + [VKILL] = 'U'-0x40, \ + [VEOF] = 'D'-0x40, \ + [VSTART] = 'Q'-0x40, \ + [VSTOP] = 'S'-0x40, \ + [VSUSP] = 'Z'-0x40, \ + [VREPRINT] = 'R'-0x40, \ + [VDISCARD] = 'O'-0x40, \ + [VWERASE] = 'W'-0x40, \ + [VLNEXT] = 'V'-0x40, \ + INIT_C_CC_VDSUSP_EXTRA \ + [VMIN] = 1 } + +int user_termio_to_kernel_termios(struct ktermios *, struct termio __user *); +int kernel_termios_to_user_termio(struct termio __user *, struct ktermios *); +#ifdef TCGETS2 +int user_termios_to_kernel_termios(struct ktermios *, struct termios2 __user *); +int kernel_termios_to_user_termios(struct termios2 __user *, struct ktermios *); +int user_termios_to_kernel_termios_1(struct ktermios *, struct termios __user *); +int kernel_termios_to_user_termios_1(struct termios __user *, struct ktermios *); +#else /* TCGETS2 */ +int user_termios_to_kernel_termios(struct ktermios *, struct termios __user *); +int kernel_termios_to_user_termios(struct termios __user *, struct ktermios *); +#endif /* TCGETS2 */ + +#endif /* _LINUX_TERMIOS_CONV_H */ diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 1386c713885d87ccb5a19d7ff7022042e26b5581..9ecc128944a19ba222cbaf043ec0fa2d9f812788 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -17,8 +17,6 @@ #include #include -#define THERMAL_MAX_TRIPS 12 - /* invalid cooling state */ #define THERMAL_CSTATE_INVALID -1UL @@ -296,82 +294,43 @@ struct thermal_zone_params { int offset; }; -/** - * struct thermal_zone_of_device_ops - callbacks for handling DT based zones - * - * Mandatory: - * @get_temp: a pointer to a function that reads the sensor temperature. - * - * Optional: - * @get_trend: a pointer to a function that reads the sensor temperature trend. - * @set_trips: a pointer to a function that sets a temperature window. When - * this window is left the driver must inform the thermal core via - * thermal_zone_device_update. - * @set_emul_temp: a pointer to a function that sets sensor emulated - * temperature. - * @set_trip_temp: a pointer to a function that sets the trip temperature on - * hardware. - * @change_mode: a pointer to a function that notifies the thermal zone - * mode change. - */ -struct thermal_zone_of_device_ops { - int (*get_temp)(void *, int *); - int (*get_trend)(void *, int, enum thermal_trend *); - int (*set_trips)(void *, int, int); - int (*set_emul_temp)(void *, int); - int (*set_trip_temp)(void *, int, int); - int (*change_mode) (void *, enum thermal_device_mode); -}; - /* Function declarations */ #ifdef CONFIG_THERMAL_OF -int thermal_zone_of_get_sensor_id(struct device_node *tz_np, - struct device_node *sensor_np, - u32 *id); -struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int id, void *data, - const struct thermal_zone_of_device_ops *ops); -void thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tz); -struct thermal_zone_device *devm_thermal_zone_of_sensor_register( - struct device *dev, int id, void *data, - const struct thermal_zone_of_device_ops *ops); -void devm_thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tz); -#else +struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, + const struct thermal_zone_device_ops *ops); -static inline int thermal_zone_of_get_sensor_id(struct device_node *tz_np, - struct device_node *sensor_np, - u32 *id) -{ - return -ENOENT; -} -static inline struct thermal_zone_device * -thermal_zone_of_sensor_register(struct device *dev, int id, void *data, - const struct thermal_zone_of_device_ops *ops) +struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int id, void *data, + const struct thermal_zone_device_ops *ops); + +void thermal_of_zone_unregister(struct thermal_zone_device *tz); + +void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz); + +void thermal_of_zone_unregister(struct thermal_zone_device *tz); + +#else +static inline +struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data, + const struct thermal_zone_device_ops *ops) { - return ERR_PTR(-ENODEV); + return ERR_PTR(-ENOTSUPP); } static inline -void thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tz) +struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int id, void *data, + const struct thermal_zone_device_ops *ops) { + return ERR_PTR(-ENOTSUPP); } -static inline struct thermal_zone_device *devm_thermal_zone_of_sensor_register( - struct device *dev, int id, void *data, - const struct thermal_zone_of_device_ops *ops) +static inline void thermal_of_zone_unregister(struct thermal_zone_device *tz) { - return ERR_PTR(-ENODEV); } -static inline -void devm_thermal_zone_of_sensor_unregister(struct device *dev, - struct thermal_zone_device *tz) +static inline void devm_thermal_of_zone_unregister(struct device *dev, + struct thermal_zone_device *tz) { } - #endif #ifdef CONFIG_THERMAL diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 9f442d73f3df883ad2f1bfb088125bcdd2663e69..90cd08ab2f5d201610de795a985c675c94a623b5 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -187,6 +187,7 @@ void tb_unregister_property_dir(const char *key, struct tb_property_dir *dir); * @device_name: Name of the device (or %NULL if not known) * @link_speed: Speed of the link in Gb/s * @link_width: Width of the link (1 or 2) + * @link_usb4: Downstream link is USB4 * @is_unplugged: The XDomain is unplugged * @needs_uuid: If the XDomain does not have @remote_uuid it will be * queried first @@ -234,6 +235,7 @@ struct tb_xdomain { const char *device_name; unsigned int link_speed; unsigned int link_width; + bool link_usb4; bool is_unplugged; bool needs_uuid; struct ida service_ids; diff --git a/include/linux/tnum.h b/include/linux/tnum.h index 498dbcedb451ad17c9d3df43376fdc1ce0fe9022..1c3948a1d6ad9b8b2fe4870c2d1321ea277adf66 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -21,7 +21,12 @@ struct tnum { struct tnum tnum_const(u64 value); /* A completely unknown value */ extern const struct tnum tnum_unknown; -/* A value that's unknown except that @min <= value <= @max */ +/* An unknown value that is a superset of @min <= value <= @max. + * + * Could include values outside the range of [@min, @max]. + * For example tnum_range(0, 2) is represented by {0, 1, 2, *3*}, + * rather than the intended set of {0, 1, 2}. + */ struct tnum tnum_range(u64 min, u64 max); /* Arithmetic and logical ops */ @@ -73,7 +78,18 @@ static inline bool tnum_is_unknown(struct tnum a) */ bool tnum_is_aligned(struct tnum a, u64 size); -/* Returns true if @b represents a subset of @a. */ +/* Returns true if @b represents a subset of @a. + * + * Note that using tnum_range() as @a requires extra cautions as tnum_in() may + * return true unexpectedly due to tnum limited ability to represent tight + * range, e.g. + * + * tnum_in(tnum_range(0, 2), tnum_const(3)) == true + * + * As a rule of thumb, if @a is explicitly coded rather than coming from + * reg->var_off, it should be in form of tnum_const(), tnum_range(0, 2**n - 1), + * or tnum_range(2**n, 2**(n+1) - 1). + */ bool tnum_in(struct tnum a, struct tnum b); /* Formatting functions. These have snprintf-like semantics: they will write diff --git a/include/linux/trace.h b/include/linux/trace.h index bf169612ffe12837c48d1889290713fe1b557e74..b5e16e438448f26bbba255b11f610835ad9a671b 100644 --- a/include/linux/trace.h +++ b/include/linux/trace.h @@ -2,8 +2,6 @@ #ifndef _LINUX_TRACE_H #define _LINUX_TRACE_H -#ifdef CONFIG_TRACING - #define TRACE_EXPORT_FUNCTION BIT(0) #define TRACE_EXPORT_EVENT BIT(1) #define TRACE_EXPORT_MARKER BIT(2) @@ -28,6 +26,8 @@ struct trace_export { int flags; }; +#ifdef CONFIG_TRACING + int register_ftrace_export(struct trace_export *export); int unregister_ftrace_export(struct trace_export *export); @@ -48,6 +48,38 @@ void osnoise_arch_unregister(void); void osnoise_trace_irq_entry(int id); void osnoise_trace_irq_exit(int id, const char *desc); +#else /* CONFIG_TRACING */ +static inline int register_ftrace_export(struct trace_export *export) +{ + return -EINVAL; +} +static inline int unregister_ftrace_export(struct trace_export *export) +{ + return 0; +} +static inline void trace_printk_init_buffers(void) +{ +} +static inline int trace_array_printk(struct trace_array *tr, unsigned long ip, + const char *fmt, ...) +{ + return 0; +} +static inline int trace_array_init_printk(struct trace_array *tr) +{ + return -EINVAL; +} +static inline void trace_array_put(struct trace_array *tr) +{ +} +static inline struct trace_array *trace_array_get_by_name(const char *name) +{ + return NULL; +} +static inline int trace_array_destroy(struct trace_array *tr) +{ + return 0; +} #endif /* CONFIG_TRACING */ #endif /* _LINUX_TRACE_H */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index b18759a673c667dab46a4640a56c1941ea462cd0..20749bd9db718cbd8ce15e8de2043fb3e873580d 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -92,6 +92,7 @@ struct trace_iterator { unsigned int temp_size; char *fmt; /* modified format holder */ unsigned int fmt_size; + long wait_index; /* trace_seq for __print_flags() and __print_symbolic() etc. */ struct trace_seq tmp_seq; @@ -814,8 +815,6 @@ extern int trace_add_event_call(struct trace_event_call *call); extern int trace_remove_event_call(struct trace_event_call *call); extern int trace_event_get_offsets(struct trace_event_call *call); -#define is_signed_type(type) (((type)(-1)) < (type)1) - int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); int trace_set_clr_event(const char *system, const char *event, int set); int trace_array_set_clr_event(struct trace_array *tr, const char *system, diff --git a/include/linux/tty.h b/include/linux/tty.h index 7b0a5d478ef660e500764d7d8ba2c176bf6d2700..730c3301d7101474f1dac3ca26c5747efcb4687b 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -122,8 +122,6 @@ struct tty_operations; /** * struct tty_struct - state associated with a tty while open * - * @magic: magic value set early in @alloc_tty_struct to %TTY_MAGIC, for - * debugging purposes * @kref: reference counting by tty_kref_get() and tty_kref_put(), reaching zero * frees the structure * @dev: class device or %NULL (e.g. ptys, serdev) @@ -193,7 +191,6 @@ struct tty_operations; * &struct tty_port. */ struct tty_struct { - int magic; struct kref kref; struct device *dev; struct tty_driver *driver; @@ -260,9 +257,6 @@ struct tty_file_private { struct list_head list; }; -/* tty magic number */ -#define TTY_MAGIC 0x5401 - /** * DOC: TTY Struct Flags * @@ -434,7 +428,7 @@ int tty_hung_up_p(struct file *filp); void do_SAK(struct tty_struct *tty); void __do_SAK(struct tty_struct *tty); void no_tty(void); -speed_t tty_termios_baud_rate(struct ktermios *termios); +speed_t tty_termios_baud_rate(const struct ktermios *termios); void tty_termios_encode_baud_rate(struct ktermios *termios, speed_t ibaud, speed_t obaud); void tty_encode_baud_rate(struct tty_struct *tty, speed_t ibaud, @@ -458,7 +452,7 @@ static inline speed_t tty_get_baud_rate(struct tty_struct *tty) unsigned char tty_get_char_size(unsigned int cflag); unsigned char tty_get_frame_size(unsigned int cflag); -void tty_termios_copy_hw(struct ktermios *new, struct ktermios *old); +void tty_termios_copy_hw(struct ktermios *new, const struct ktermios *old); int tty_termios_hw_change(const struct ktermios *a, const struct ktermios *b); int tty_set_termios(struct tty_struct *tty, struct ktermios *kt); diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 4841d8069c072bae22e7936be443b30f64148e1c..e00034118c7bcaee786d638b5f7a8f58c3d812d0 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -141,7 +142,7 @@ struct serial_struct; * * Optional. * - * @set_termios: ``void ()(struct tty_struct *tty, struct ktermios *old)`` + * @set_termios: ``void ()(struct tty_struct *tty, const struct ktermios *old)`` * * This routine allows the @tty driver to be notified when device's * termios settings have changed. New settings are in @tty->termios. @@ -365,7 +366,7 @@ struct tty_operations { unsigned int cmd, unsigned long arg); long (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); - void (*set_termios)(struct tty_struct *tty, struct ktermios * old); + void (*set_termios)(struct tty_struct *tty, const struct ktermios *old); void (*throttle)(struct tty_struct * tty); void (*unthrottle)(struct tty_struct * tty); void (*stop)(struct tty_struct *tty); @@ -396,7 +397,6 @@ struct tty_operations { /** * struct tty_driver -- driver for TTY devices * - * @magic: set to %TTY_DRIVER_MAGIC in __tty_alloc_driver() * @kref: reference counting. Reaching zero frees all the internals and the * driver. * @cdevs: allocated/registered character /dev devices @@ -432,7 +432,6 @@ struct tty_operations { * @driver_name, @name, @type, @subtype, @init_termios, and @ops. */ struct tty_driver { - int magic; struct kref kref; struct cdev **cdevs; struct module *owner; @@ -489,9 +488,6 @@ static inline void tty_set_operations(struct tty_driver *driver, driver->ops = op; } -/* tty driver magic number */ -#define TTY_DRIVER_MAGIC 0x5402 - /** * DOC: TTY Driver Flags * diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index ede6f2157f329fe8dc4b0abd01b972238c117bbe..dcb61ec11424ad46e02188b57720a22ac7b72889 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -130,7 +130,7 @@ int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, * a pointer to wordsize-sensitive structure belongs here, but most of * ldiscs will happily leave it %NULL. * - * @set_termios: [TTY] ``void ()(struct tty_struct *tty, struct ktermios *old)`` + * @set_termios: [TTY] ``void ()(struct tty_struct *tty, const struct ktermios *old)`` * * This function notifies the line discpline that a change has been made * to the termios structure. @@ -227,7 +227,7 @@ struct tty_ldisc_ops { unsigned long arg); int (*compat_ioctl)(struct tty_struct *tty, unsigned int cmd, unsigned long arg); - void (*set_termios)(struct tty_struct *tty, struct ktermios *old); + void (*set_termios)(struct tty_struct *tty, const struct ktermios *old); __poll_t (*poll)(struct tty_struct *tty, struct file *file, struct poll_table_struct *wait); void (*hangup)(struct tty_struct *tty); diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 6ad4e9032d5384a550d75d02b7abdaa81ec3b952..46040d66334a84f5c493417a200789a00dccb9e1 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h @@ -8,7 +8,7 @@ * * Key points : * - * - Use a seqcount on 32-bit SMP, only disable preemption for 32-bit UP. + * - Use a seqcount on 32-bit * - The whole thing is a no-op on 64-bit architectures. * * Usage constraints: @@ -20,7 +20,8 @@ * writer and also spin forever. * * 3) Write side must use the _irqsave() variant if other writers, or a reader, - * can be invoked from an IRQ context. + * can be invoked from an IRQ context. On 64bit systems this variant does not + * disable interrupts. * * 4) If reader fetches several counters, there is no guarantee the whole values * are consistent w.r.t. each other (remember point #2: seqcounts are not @@ -29,11 +30,6 @@ * 5) Readers are allowed to sleep or be preempted/interrupted: they perform * pure reads. * - * 6) Readers must use both u64_stats_fetch_{begin,retry}_irq() if the stats - * might be updated from a hardirq or softirq context (remember point #1: - * seqcounts are not used for UP kernels). 32-bit UP stat readers could read - * corrupted 64-bit values otherwise. - * * Usage : * * Stats producer (writer) should use following template granted it already got @@ -66,7 +62,7 @@ #include struct u64_stats_sync { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) +#if BITS_PER_LONG == 32 seqcount_t seq; #endif }; @@ -98,7 +94,22 @@ static inline void u64_stats_inc(u64_stats_t *p) local64_inc(&p->v); } -#else +static inline void u64_stats_init(struct u64_stats_sync *syncp) { } +static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { } +static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { } +static inline unsigned long __u64_stats_irqsave(void) { return 0; } +static inline void __u64_stats_irqrestore(unsigned long flags) { } +static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) +{ + return 0; +} +static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, + unsigned int start) +{ + return false; +} + +#else /* 64 bit */ typedef struct { u64 v; @@ -123,123 +134,95 @@ static inline void u64_stats_inc(u64_stats_t *p) { p->v++; } -#endif -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) -#define u64_stats_init(syncp) seqcount_init(&(syncp)->seq) -#else static inline void u64_stats_init(struct u64_stats_sync *syncp) { + seqcount_init(&syncp->seq); } -#endif -static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) +static inline void __u64_stats_update_begin(struct u64_stats_sync *syncp) { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); write_seqcount_begin(&syncp->seq); -#endif } -static inline void u64_stats_update_end(struct u64_stats_sync *syncp) +static inline void __u64_stats_update_end(struct u64_stats_sync *syncp) { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) write_seqcount_end(&syncp->seq); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); -#endif + preempt_enable_nested(); } -static inline unsigned long -u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp) +static inline unsigned long __u64_stats_irqsave(void) { - unsigned long flags = 0; + unsigned long flags; -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); - else - local_irq_save(flags); - write_seqcount_begin(&syncp->seq); -#endif + local_irq_save(flags); return flags; } -static inline void -u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp, - unsigned long flags) +static inline void __u64_stats_irqrestore(unsigned long flags) { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) - write_seqcount_end(&syncp->seq); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); - else - local_irq_restore(flags); -#endif + local_irq_restore(flags); } static inline unsigned int __u64_stats_fetch_begin(const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) return read_seqcount_begin(&syncp->seq); -#else - return 0; -#endif } -static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp) +static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, + unsigned int start) { -#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT)) - preempt_disable(); -#endif - return __u64_stats_fetch_begin(syncp); + return read_seqcount_retry(&syncp->seq, start); } +#endif /* !64 bit */ -static inline bool __u64_stats_fetch_retry(const struct u64_stats_sync *syncp, - unsigned int start) +static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) { -#if BITS_PER_LONG == 32 && (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)) - return read_seqcount_retry(&syncp->seq, start); -#else - return false; -#endif + __u64_stats_update_begin(syncp); +} + +static inline void u64_stats_update_end(struct u64_stats_sync *syncp) +{ + __u64_stats_update_end(syncp); +} + +static inline unsigned long u64_stats_update_begin_irqsave(struct u64_stats_sync *syncp) +{ + unsigned long flags = __u64_stats_irqsave(); + + __u64_stats_update_begin(syncp); + return flags; +} + +static inline void u64_stats_update_end_irqrestore(struct u64_stats_sync *syncp, + unsigned long flags) +{ + __u64_stats_update_end(syncp); + __u64_stats_irqrestore(flags); +} + +static inline unsigned int u64_stats_fetch_begin(const struct u64_stats_sync *syncp) +{ + return __u64_stats_fetch_begin(syncp); } static inline bool u64_stats_fetch_retry(const struct u64_stats_sync *syncp, unsigned int start) { -#if BITS_PER_LONG == 32 && (!defined(CONFIG_SMP) && !defined(CONFIG_PREEMPT_RT)) - preempt_enable(); -#endif return __u64_stats_fetch_retry(syncp, start); } -/* - * In case irq handlers can update u64 counters, readers can use following helpers - * - SMP 32bit arches use seqcount protection, irq safe. - * - UP 32bit must disable irqs. - * - 64bit have no problem atomically reading u64 values, irq safe. - */ +/* Obsolete interfaces */ static inline unsigned int u64_stats_fetch_begin_irq(const struct u64_stats_sync *syncp) { -#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) - preempt_disable(); -#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP) - local_irq_disable(); -#endif - return __u64_stats_fetch_begin(syncp); + return u64_stats_fetch_begin(syncp); } static inline bool u64_stats_fetch_retry_irq(const struct u64_stats_sync *syncp, unsigned int start) { -#if BITS_PER_LONG == 32 && defined(CONFIG_PREEMPT_RT) - preempt_enable(); -#elif BITS_PER_LONG == 32 && !defined(CONFIG_SMP) - local_irq_enable(); -#endif - return __u64_stats_fetch_retry(syncp, start); + return u64_stats_fetch_retry(syncp, start); } #endif /* _LINUX_U64_STATS_SYNC_H */ diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 47e5d374c7ebec86306b31d3d46665ca1c39f583..afb18f198843b4a508abda6efe9c575f02085c32 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -58,20 +58,28 @@ static __always_inline __must_check unsigned long __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) { - instrument_copy_from_user(to, from, n); + unsigned long res; + + instrument_copy_from_user_before(to, from, n); check_object_size(to, n, false); - return raw_copy_from_user(to, from, n); + res = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, res); + return res; } static __always_inline __must_check unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { + unsigned long res; + might_fault(); + instrument_copy_from_user_before(to, from, n); if (should_fail_usercopy()) return n; - instrument_copy_from_user(to, from, n); check_object_size(to, n, false); - return raw_copy_from_user(to, from, n); + res = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, res); + return res; } /** @@ -115,8 +123,9 @@ _copy_from_user(void *to, const void __user *from, unsigned long n) unsigned long res = n; might_fault(); if (!should_fail_usercopy() && likely(access_ok(from, n))) { - instrument_copy_from_user(to, from, n); + instrument_copy_from_user_before(to, from, n); res = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, res); } if (unlikely(res)) memset(to + (n - res), 0, res); diff --git a/include/linux/ucb1400.h b/include/linux/ucb1400.h index 22345391350b9869e812ae1becda878843a162b2..2516082cd3a935af86d2b309eed5620c398f020b 100644 --- a/include/linux/ucb1400.h +++ b/include/linux/ucb1400.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include /* * UCB1400 AC-link registers diff --git a/include/linux/udp.h b/include/linux/udp.h index 254a2654400f8adacbb20854c6bd0eeadff5ffba..e96da4157d04daa97cbf064f9b2c013e1ecdb9b9 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -70,6 +70,7 @@ struct udp_sock { * For encapsulation sockets. */ int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); + void (*encap_err_rcv)(struct sock *sk, struct sk_buff *skb, unsigned int udp_offset); int (*encap_err_lookup)(struct sock *sk, struct sk_buff *skb); void (*encap_destroy)(struct sock *sk); diff --git a/include/linux/uio.h b/include/linux/uio.h index 5896af36199cbe3df2185b0072d200db1b5dbc34..2e3134b14ffd98348a361b1d9aec13041338daf1 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -298,7 +298,7 @@ iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes) shorted = iov_iter_count(i) - max_bytes; iov_iter_truncate(i, max_bytes); } - npages = iov_iter_npages(i, INT_MAX); + npages = iov_iter_npages(i, maxpages); if (shorted) iov_iter_reexpand(i, iov_iter_count(i) + shorted); diff --git a/include/linux/umh.h b/include/linux/umh.h index 244aff6382208824e64c6387a38bb60e967f0548..5d1f6129b84703bf4252ab3ac8206f5037cda9f4 100644 --- a/include/linux/umh.h +++ b/include/linux/umh.h @@ -11,10 +11,11 @@ struct cred; struct file; -#define UMH_NO_WAIT 0 /* don't wait at all */ -#define UMH_WAIT_EXEC 1 /* wait for the exec, but not the process */ -#define UMH_WAIT_PROC 2 /* wait for the process to complete */ -#define UMH_KILLABLE 4 /* wait for EXEC/PROC killable */ +#define UMH_NO_WAIT 0x00 /* don't wait at all */ +#define UMH_WAIT_EXEC 0x01 /* wait for the exec, but not the process */ +#define UMH_WAIT_PROC 0x02 /* wait for the process to complete */ +#define UMH_KILLABLE 0x04 /* wait for EXEC/PROC killable */ +#define UMH_FREEZABLE 0x08 /* wait for EXEC/PROC freezable */ struct subprocess_info { struct work_struct work; diff --git a/include/linux/units.h b/include/linux/units.h index 681fc652e3d71318fb457900fdcd5e389e4c5f24..2793a41e73a2b6c36a14bd9a245e582e5c4ae560 100644 --- a/include/linux/units.h +++ b/include/linux/units.h @@ -20,6 +20,9 @@ #define PICO 1000000000000ULL #define FEMTO 1000000000000000ULL +#define NANOHZ_PER_HZ 1000000000UL +#define MICROHZ_PER_HZ 1000000UL +#define MILLIHZ_PER_HZ 1000UL #define HZ_PER_KHZ 1000UL #define KHZ_PER_MHZ 1000UL #define HZ_PER_MHZ 1000000UL diff --git a/include/linux/usb.h b/include/linux/usb.h index f7a9914fc97f2c7f2c29959ccc8cd6306e9c63dd..9ff1ad4dfad12bf7273d273f596cd750a899b023 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -575,6 +575,7 @@ struct usb3_lpm_parameters { * @devaddr: device address, XHCI: assigned by HW, others: same as devnum * @can_submit: URBs may be submitted * @persist_enabled: USB_PERSIST enabled for this device + * @reset_in_progress: the device is being reset * @have_langid: whether string_langid is valid * @authorized: policy has said we can use it; * (user space) policy determines if we authorize this device to be @@ -662,6 +663,7 @@ struct usb_device { unsigned can_submit:1; unsigned persist_enabled:1; + unsigned reset_in_progress:1; unsigned have_langid:1; unsigned authorized:1; unsigned authenticated:1; diff --git a/include/linux/usb/chipidea.h b/include/linux/usb/chipidea.h index edf3342507f16484909cbb5eaffd2283be23eaad..ee38835ed77cc4bd2b43425fffbaff280c3ab881 100644 --- a/include/linux/usb/chipidea.h +++ b/include/linux/usb/chipidea.h @@ -62,6 +62,7 @@ struct ci_hdrc_platform_data { #define CI_HDRC_REQUIRES_ALIGNED_DMA BIT(13) #define CI_HDRC_IMX_IS_HSIC BIT(14) #define CI_HDRC_PMQOS BIT(15) +#define CI_HDRC_PHY_VBUS_CONTROL BIT(16) enum usb_dr_mode dr_mode; #define CI_HDRC_CONTROLLER_RESET_EVENT 0 #define CI_HDRC_CONTROLLER_STOPPED_EVENT 1 diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h index 67f8713d3fa37d74b289ea3db6abe61787452a3d..78cd566ee238007e4113a90997234b7f2e440550 100644 --- a/include/linux/usb/hcd.h +++ b/include/linux/usb/hcd.h @@ -479,7 +479,6 @@ static inline int ehset_single_step_set_feature(struct usb_hcd *hcd, int port) struct pci_dev; struct pci_device_id; extern int usb_hcd_pci_probe(struct pci_dev *dev, - const struct pci_device_id *id, const struct hc_driver *driver); extern void usb_hcd_pci_remove(struct pci_dev *dev); extern void usb_hcd_pci_shutdown(struct pci_dev *dev); diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 8ea319f89e1ff2162ac1eed437d0b0ab7ba647c8..f7bfedb740f56e2d3509956c6d2f8da8b0e32acd 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -276,8 +276,8 @@ struct usb_serial_driver { unsigned int cmd, unsigned long arg); void (*get_serial)(struct tty_struct *tty, struct serial_struct *ss); int (*set_serial)(struct tty_struct *tty, struct serial_struct *ss); - void (*set_termios)(struct tty_struct *tty, - struct usb_serial_port *port, struct ktermios *old); + void (*set_termios)(struct tty_struct *tty, struct usb_serial_port *port, + const struct ktermios *old); void (*break_ctl)(struct tty_struct *tty, int break_state); unsigned int (*chars_in_buffer)(struct tty_struct *tty); void (*wait_until_sent)(struct tty_struct *tty, long timeout); diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index 20c0bedb8ec86ccd647aedffedef33ebe31b8eef..17657451c762cfc912f43775003c46ec0d4da946 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -167,6 +167,11 @@ /* I2C_WRITE_BYTE_COUNT + 1 when TX_BUF_BYTE_x is only accessible I2C_WRITE_BYTE_COUNT */ #define TCPC_TRANSMIT_BUFFER_MAX_LEN 31 +#define tcpc_presenting_rd(reg, cc) \ + (!(TCPC_ROLE_CTRL_DRP & (reg)) && \ + (((reg) & (TCPC_ROLE_CTRL_## cc ##_MASK << TCPC_ROLE_CTRL_## cc ##_SHIFT)) == \ + (TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_## cc ##_SHIFT))) + struct tcpci; /* @@ -207,4 +212,21 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci); struct tcpm_port; struct tcpm_port *tcpci_get_tcpm_port(struct tcpci *tcpci); + +static inline enum typec_cc_status tcpci_to_typec_cc(unsigned int cc, bool sink) +{ + switch (cc) { + case 0x1: + return sink ? TYPEC_CC_RP_DEF : TYPEC_CC_RA; + case 0x2: + return sink ? TYPEC_CC_RP_1_5 : TYPEC_CC_RD; + case 0x3: + if (sink) + return TYPEC_CC_RP_3_0; + fallthrough; + case 0x0: + default: + return TYPEC_CC_OPEN; + } +} #endif /* __LINUX_USB_TCPCI_H */ diff --git a/include/linux/usb/typec_dp.h b/include/linux/usb/typec_dp.h index cfb916cccd316b345a824564600324c65e1f3630..8d09c2f0a9b80717cdda7eba91cf63959d3be158 100644 --- a/include/linux/usb/typec_dp.h +++ b/include/linux/usb/typec_dp.h @@ -73,6 +73,11 @@ enum { #define DP_CAP_USB BIT(7) #define DP_CAP_DFP_D_PIN_ASSIGN(_cap_) (((_cap_) & GENMASK(15, 8)) >> 8) #define DP_CAP_UFP_D_PIN_ASSIGN(_cap_) (((_cap_) & GENMASK(23, 16)) >> 16) +/* Get pin assignment taking plug & receptacle into consideration */ +#define DP_CAP_PIN_ASSIGN_UFP_D(_cap_) ((_cap_ & DP_CAP_RECEPTACLE) ? \ + DP_CAP_UFP_D_PIN_ASSIGN(_cap_) : DP_CAP_DFP_D_PIN_ASSIGN(_cap_)) +#define DP_CAP_PIN_ASSIGN_DFP_D(_cap_) ((_cap_ & DP_CAP_RECEPTACLE) ? \ + DP_CAP_DFP_D_PIN_ASSIGN(_cap_) : DP_CAP_UFP_D_PIN_ASSIGN(_cap_)) /* DisplayPort Status Update VDO bits */ #define DP_STATUS_CONNECTION(_status_) ((_status_) & 3) diff --git a/include/linux/user_events.h b/include/linux/user_events.h index 736e0560346386820886e5073e29dcea772a576d..592a3fbed98e40a6177b31c4229d64b9bf81b59d 100644 --- a/include/linux/user_events.h +++ b/include/linux/user_events.h @@ -20,15 +20,6 @@ #define USER_EVENTS_SYSTEM "user_events" #define USER_EVENTS_PREFIX "u:" -/* Bits 0-6 are for known probe types, Bit 7 is for unknown probes */ -#define EVENT_BIT_FTRACE 0 -#define EVENT_BIT_PERF 1 -#define EVENT_BIT_OTHER 7 - -#define EVENT_STATUS_FTRACE (1 << EVENT_BIT_FTRACE) -#define EVENT_STATUS_PERF (1 << EVENT_BIT_PERF) -#define EVENT_STATUS_OTHER (1 << EVENT_BIT_OTHER) - /* Create dynamic location entry within a 32-bit value */ #define DYN_LOC(offset, size) ((size) << 16 | (offset)) @@ -45,12 +36,12 @@ struct user_reg { /* Input: Pointer to string with event name, description and flags */ __u64 name_args; - /* Output: Byte index of the event within the status page */ - __u32 status_index; + /* Output: Bitwise index of the event within the status page */ + __u32 status_bit; /* Output: Index of the event to use when writing data */ __u32 write_index; -}; +} __attribute__((__packed__)); #define DIAG_IOC_MAGIC '*' diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index 33a4240e6a6f1789f8a3153a264030e62f4645d7..45f09bec02c485dbcf9f9607725a7f839dda01d3 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -54,15 +54,17 @@ enum ucount_type { UCOUNT_FANOTIFY_GROUPS, UCOUNT_FANOTIFY_MARKS, #endif + UCOUNT_COUNTS, +}; + +enum rlimit_type { UCOUNT_RLIMIT_NPROC, UCOUNT_RLIMIT_MSGQUEUE, UCOUNT_RLIMIT_SIGPENDING, UCOUNT_RLIMIT_MEMLOCK, - UCOUNT_COUNTS, + UCOUNT_RLIMIT_COUNTS, }; -#define MAX_PER_NAMESPACE_UCOUNTS UCOUNT_RLIMIT_NPROC - struct user_namespace { struct uid_gid_map uid_map; struct uid_gid_map gid_map; @@ -99,6 +101,7 @@ struct user_namespace { #endif struct ucounts *ucounts; long ucount_max[UCOUNT_COUNTS]; + long rlimit_max[UCOUNT_RLIMIT_COUNTS]; } __randomize_layout; struct ucounts { @@ -107,6 +110,7 @@ struct ucounts { kuid_t uid; atomic_t count; atomic_long_t ucount[UCOUNT_COUNTS]; + atomic_long_t rlimit[UCOUNT_RLIMIT_COUNTS]; }; extern struct user_namespace init_user_ns; @@ -120,21 +124,26 @@ struct ucounts *alloc_ucounts(struct user_namespace *ns, kuid_t uid); struct ucounts * __must_check get_ucounts(struct ucounts *ucounts); void put_ucounts(struct ucounts *ucounts); -static inline long get_ucounts_value(struct ucounts *ucounts, enum ucount_type type) +static inline long get_rlimit_value(struct ucounts *ucounts, enum rlimit_type type) { - return atomic_long_read(&ucounts->ucount[type]); + return atomic_long_read(&ucounts->rlimit[type]); } -long inc_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v); -bool dec_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v); -long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum ucount_type type); -void dec_rlimit_put_ucounts(struct ucounts *ucounts, enum ucount_type type); -bool is_ucounts_overlimit(struct ucounts *ucounts, enum ucount_type type, unsigned long max); +long inc_rlimit_ucounts(struct ucounts *ucounts, enum rlimit_type type, long v); +bool dec_rlimit_ucounts(struct ucounts *ucounts, enum rlimit_type type, long v); +long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum rlimit_type type); +void dec_rlimit_put_ucounts(struct ucounts *ucounts, enum rlimit_type type); +bool is_rlimit_overlimit(struct ucounts *ucounts, enum rlimit_type type, unsigned long max); + +static inline long get_userns_rlimit_max(struct user_namespace *ns, enum rlimit_type type) +{ + return READ_ONCE(ns->rlimit_max[type]); +} -static inline void set_rlimit_ucount_max(struct user_namespace *ns, - enum ucount_type type, unsigned long max) +static inline void set_userns_rlimit_max(struct user_namespace *ns, + enum rlimit_type type, unsigned long max) { - ns->ucount_max[type] = max <= LONG_MAX ? max : LONG_MAX; + ns->rlimit_max[type] = max <= LONG_MAX ? max : LONG_MAX; } #ifdef CONFIG_USER_NS diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index 732b522bacb7e5c15a8d6e00f9f4d4e8c6daa4ea..f07e6998bb68e289cf5e1451e60317e161371680 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -73,6 +73,8 @@ extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, extern int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool enable_wp, atomic_t *mmap_changing); +extern void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *vma, + unsigned long start, unsigned long len, bool enable_wp); /* mm helpers */ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, @@ -173,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma, unsigned long start, unsigned long end); -extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - struct list_head *uf); +extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, + unsigned long end, struct list_head *uf); extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); @@ -256,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma, return true; } -static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, +static inline int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start, unsigned long end, struct list_head *uf) { diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index d282f464d2f1a7168ee18e14ccd7c9d98a0ae195..6d0f5e4e82c252285f68f339bfae43fbe53a91a4 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -104,6 +104,7 @@ struct vdpa_iova_range { }; struct vdpa_dev_set_config { + u64 device_features; struct { u8 mac[ETH_ALEN]; u16 mtu; diff --git a/include/linux/verification.h b/include/linux/verification.h index a655923335aebd587b80c5da9db979958389ca84..f34e50ebcf60a4377cbcafbc8960d898265b9a9c 100644 --- a/include/linux/verification.h +++ b/include/linux/verification.h @@ -17,6 +17,14 @@ #define VERIFY_USE_SECONDARY_KEYRING ((struct key *)1UL) #define VERIFY_USE_PLATFORM_KEYRING ((struct key *)2UL) +static inline int system_keyring_id_check(u64 id) +{ + if (id > (unsigned long)VERIFY_USE_PLATFORM_KEYRING) + return -EINVAL; + + return 0; +} + /* * The use to which an asymmetric key is being put. */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index e05ddc6fe6a55607d1ac65a2a0949ad45c078230..e7cebeb875dd1abdc2a9ed969532f2be5f5b2029 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -14,6 +14,7 @@ #include #include #include +#include struct kvm; @@ -33,10 +34,11 @@ struct vfio_device { struct device *dev; const struct vfio_device_ops *ops; /* - * mig_ops is a static property of the vfio_device which must be set - * prior to registering the vfio_device. + * mig_ops/log_ops is a static property of the vfio_device which must + * be set prior to registering the vfio_device. */ const struct vfio_migration_ops *mig_ops; + const struct vfio_log_ops *log_ops; struct vfio_group *group; struct vfio_device_set *dev_set; struct list_head dev_set_list; @@ -45,7 +47,9 @@ struct vfio_device { struct kvm *kvm; /* Members below here are private, not for driver use */ - refcount_t refcount; + unsigned int index; + struct device device; /* device.kref covers object life circle */ + refcount_t refcount; /* user count on registered device*/ unsigned int open_count; struct completion comp; struct list_head group_next; @@ -55,6 +59,8 @@ struct vfio_device { /** * struct vfio_device_ops - VFIO bus driver device callbacks * + * @init: initialize private fields in device structure + * @release: Reclaim private fields in device structure * @open_device: Called when the first file descriptor is opened for this device * @close_device: Opposite of open_device * @read: Perform read(2) on device file descriptor @@ -72,6 +78,8 @@ struct vfio_device { */ struct vfio_device_ops { char *name; + int (*init)(struct vfio_device *vdev); + void (*release)(struct vfio_device *vdev); int (*open_device)(struct vfio_device *vdev); void (*close_device)(struct vfio_device *vdev); ssize_t (*read)(struct vfio_device *vdev, char __user *buf, @@ -108,6 +116,28 @@ struct vfio_migration_ops { enum vfio_device_mig_state *curr_state); }; +/** + * @log_start: Optional callback to ask the device start DMA logging. + * @log_stop: Optional callback to ask the device stop DMA logging. + * @log_read_and_clear: Optional callback to ask the device read + * and clear the dirty DMAs in some given range. + * + * The vfio core implementation of the DEVICE_FEATURE_DMA_LOGGING_ set + * of features does not track logging state relative to the device, + * therefore the device implementation of vfio_log_ops must handle + * arbitrary user requests. This includes rejecting subsequent calls + * to log_start without an intervening log_stop, as well as graceful + * handling of log_stop and log_read_and_clear from invalid states. + */ +struct vfio_log_ops { + int (*log_start)(struct vfio_device *device, + struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); + int (*log_stop)(struct vfio_device *device); + int (*log_read_and_clear)(struct vfio_device *device, + unsigned long iova, unsigned long length, + struct iova_bitmap *dirty); +}; + /** * vfio_check_feature - Validate user input for the VFIO_DEVICE_FEATURE ioctl * @flags: Arg from the device_feature op @@ -137,9 +167,23 @@ static inline int vfio_check_feature(u32 flags, size_t argsz, u32 supported_ops, return 1; } -void vfio_init_group_dev(struct vfio_device *device, struct device *dev, - const struct vfio_device_ops *ops); -void vfio_uninit_group_dev(struct vfio_device *device); +struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, + const struct vfio_device_ops *ops); +#define vfio_alloc_device(dev_struct, member, dev, ops) \ + container_of(_vfio_alloc_device(sizeof(struct dev_struct) + \ + BUILD_BUG_ON_ZERO(offsetof( \ + struct dev_struct, member)), \ + dev, ops), \ + struct dev_struct, member) + +int vfio_init_device(struct vfio_device *device, struct device *dev, + const struct vfio_device_ops *ops); +void vfio_free_device(struct vfio_device *device); +static inline void vfio_put_device(struct vfio_device *device) +{ + put_device(&device->device); +} + int vfio_register_group_dev(struct vfio_device *device); int vfio_register_emulated_iommu_dev(struct vfio_device *device); void vfio_unregister_group_dev(struct vfio_device *device); @@ -155,6 +199,7 @@ int vfio_mig_get_next_state(struct vfio_device *device, * External user API */ struct iommu_group *vfio_file_iommu_group(struct file *file); +bool vfio_file_is_group(struct file *file); bool vfio_file_enforced_coherent(struct file *file); void vfio_file_set_kvm(struct file *file, struct kvm *kvm); bool vfio_file_has_dev(struct file *file, struct vfio_device *device); diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 5579ece4347bdc5427905027615f570796c9d015..367fd79226a3061518ee9c77187d902c705aa843 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -20,39 +20,10 @@ #define VFIO_PCI_CORE_H #define VFIO_PCI_OFFSET_SHIFT 40 - #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) -/* Special capability IDs predefined access */ -#define PCI_CAP_ID_INVALID 0xFF /* default raw access */ -#define PCI_CAP_ID_INVALID_VIRT 0xFE /* default virt access */ - -/* Cap maximum number of ioeventfds per device (arbitrary) */ -#define VFIO_PCI_IOEVENTFD_MAX 1000 - -struct vfio_pci_ioeventfd { - struct list_head next; - struct vfio_pci_core_device *vdev; - struct virqfd *virqfd; - void __iomem *addr; - uint64_t data; - loff_t pos; - int bar; - int count; - bool test_mem; -}; - -struct vfio_pci_irq_ctx { - struct eventfd_ctx *trigger; - struct virqfd *unmask; - struct virqfd *mask; - char *name; - bool masked; - struct irq_bypass_producer producer; -}; - struct vfio_pci_core_device; struct vfio_pci_region; @@ -78,23 +49,6 @@ struct vfio_pci_region { u32 flags; }; -struct vfio_pci_dummy_resource { - struct resource resource; - int index; - struct list_head res_next; -}; - -struct vfio_pci_vf_token { - struct mutex lock; - uuid_t uuid; - int users; -}; - -struct vfio_pci_mmap_vma { - struct vm_area_struct *vma; - struct list_head vma_next; -}; - struct vfio_pci_core_device { struct vfio_device vdev; struct pci_dev *pdev; @@ -124,11 +78,14 @@ struct vfio_pci_core_device { bool needs_reset; bool nointx; bool needs_pm_restore; + bool pm_intx_masked; + bool pm_runtime_engaged; struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; int ioeventfds_nr; struct eventfd_ctx *err_trigger; struct eventfd_ctx *req_trigger; + struct eventfd_ctx *pm_wake_eventfd_ctx; struct list_head dummy_resources_list; struct mutex ioeventfds_lock; struct list_head ioeventfds_list; @@ -141,100 +98,17 @@ struct vfio_pci_core_device { struct rw_semaphore memory_lock; }; -#define is_intx(vdev) (vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) -#define is_msi(vdev) (vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX) -#define is_msix(vdev) (vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX) -#define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev))) -#define irq_is(vdev, type) (vdev->irq_type == type) - -void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev); -void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev); - -int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, - uint32_t flags, unsigned index, - unsigned start, unsigned count, void *data); - -ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev, - char __user *buf, size_t count, - loff_t *ppos, bool iswrite); - -ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); - -#ifdef CONFIG_VFIO_PCI_VGA -ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, - size_t count, loff_t *ppos, bool iswrite); -#else -static inline ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, - char __user *buf, size_t count, - loff_t *ppos, bool iswrite) -{ - return -EINVAL; -} -#endif - -long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, - uint64_t data, int count, int fd); - -int vfio_pci_init_perm_bits(void); -void vfio_pci_uninit_perm_bits(void); - -int vfio_config_init(struct vfio_pci_core_device *vdev); -void vfio_config_free(struct vfio_pci_core_device *vdev); - -int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev, - unsigned int type, unsigned int subtype, - const struct vfio_pci_regops *ops, - size_t size, u32 flags, void *data); - -int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, - pci_power_t state); - -bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev); -void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev); -u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev); -void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, - u16 cmd); - -#ifdef CONFIG_VFIO_PCI_IGD -int vfio_pci_igd_init(struct vfio_pci_core_device *vdev); -#else -static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev) -{ - return -ENODEV; -} -#endif - -#ifdef CONFIG_VFIO_PCI_ZDEV_KVM -int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, - struct vfio_info_cap *caps); -int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev); -void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev); -#else -static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev, - struct vfio_info_cap *caps) -{ - return -ENODEV; -} - -static inline int vfio_pci_zdev_open_device(struct vfio_pci_core_device *vdev) -{ - return 0; -} - -static inline void vfio_pci_zdev_close_device(struct vfio_pci_core_device *vdev) -{} -#endif - /* Will be exported for vfio pci drivers usage */ +int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, + unsigned int type, unsigned int subtype, + const struct vfio_pci_regops *ops, + size_t size, u32 flags, void *data); void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, bool is_disable_idle_d3); void vfio_pci_core_close_device(struct vfio_device *core_vdev); -void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev, - struct pci_dev *pdev, - const struct vfio_device_ops *vfio_pci_ops); +int vfio_pci_core_init_dev(struct vfio_device *core_vdev); +void vfio_pci_core_release_dev(struct vfio_device *core_vdev); int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); -void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev); void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); extern const struct pci_error_handlers vfio_pci_core_err_handlers; int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, @@ -256,9 +130,4 @@ void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, pci_channel_state_t state); -static inline bool vfio_pci_is_vga(struct pci_dev *pdev) -{ - return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; -} - #endif /* VFIO_PCI_CORE_H */ diff --git a/include/linux/virtio_pci_legacy.h b/include/linux/virtio_pci_legacy.h index e5d665faf00e175f66dca6aa3abfd14f50a4ff9c..a8dc757d0367015fa9056c9052c1cc21f4d2bce0 100644 --- a/include/linux/virtio_pci_legacy.h +++ b/include/linux/virtio_pci_legacy.h @@ -32,8 +32,6 @@ void vp_legacy_set_queue_address(struct virtio_pci_legacy_device *ldev, u16 index, u32 queue_pfn); bool vp_legacy_get_queue_enable(struct virtio_pci_legacy_device *ldev, u16 idx); -void vp_legacy_set_queue_size(struct virtio_pci_legacy_device *ldev, - u16 idx, u16 size); u16 vp_legacy_get_queue_size(struct virtio_pci_legacy_device *ldev, u16 idx); int vp_legacy_probe(struct virtio_pci_legacy_device *ldev); diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 404024486fa5397d1f87b986267e4f21cc8d0560..3518dba1e02f4bdf5f6217a9ea8af2fad38f13af 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -20,12 +20,19 @@ #define HIGHMEM_ZONE(xx) #endif -#define FOR_ALL_ZONES(xx) DMA_ZONE(xx) DMA32_ZONE(xx) xx##_NORMAL, HIGHMEM_ZONE(xx) xx##_MOVABLE +#ifdef CONFIG_ZONE_DEVICE +#define DEVICE_ZONE(xx) xx##_DEVICE, +#else +#define DEVICE_ZONE(xx) +#endif + +#define FOR_ALL_ZONES(xx) DMA_ZONE(xx) DMA32_ZONE(xx) xx##_NORMAL, \ + HIGHMEM_ZONE(xx) xx##_MOVABLE, DEVICE_ZONE(xx) enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, - FOR_ALL_ZONES(PGALLOC), - FOR_ALL_ZONES(ALLOCSTALL), - FOR_ALL_ZONES(PGSCAN_SKIP), + FOR_ALL_ZONES(PGALLOC) + FOR_ALL_ZONES(ALLOCSTALL) + FOR_ALL_ZONES(PGSCAN_SKIP) PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE, PGFAULT, PGMAJFAULT, PGLAZYFREED, @@ -122,10 +129,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, NR_TLB_LOCAL_FLUSH_ALL, NR_TLB_LOCAL_FLUSH_ONE, #endif /* CONFIG_DEBUG_TLBFLUSH */ -#ifdef CONFIG_DEBUG_VM_VMACACHE - VMACACHE_FIND_CALLS, - VMACACHE_FIND_HITS, -#endif #ifdef CONFIG_SWAP SWAP_RA, SWAP_RA_HIT, diff --git a/include/linux/vmacache.h b/include/linux/vmacache.h deleted file mode 100644 index 6fce268a4588e5a064214b84309f4520f8759d65..0000000000000000000000000000000000000000 --- a/include/linux/vmacache.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_VMACACHE_H -#define __LINUX_VMACACHE_H - -#include -#include - -static inline void vmacache_flush(struct task_struct *tsk) -{ - memset(tsk->vmacache.vmas, 0, sizeof(tsk->vmacache.vmas)); -} - -extern void vmacache_update(unsigned long addr, struct vm_area_struct *newvma); -extern struct vm_area_struct *vmacache_find(struct mm_struct *mm, - unsigned long addr); - -#ifndef CONFIG_MMU -extern struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, - unsigned long start, - unsigned long end); -#endif - -static inline void vmacache_invalidate(struct mm_struct *mm) -{ - mm->vmacache_seqnum++; -} - -#endif /* __LINUX_VMACACHE_H */ diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index bfe38869498d7cfa199cfc653d3b4b63f854b371..19cf5b6892ceba12c9f8acfa0a7910b4a2e726d4 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -125,12 +125,6 @@ static inline void vm_events_fold_cpu(int cpu) #define count_vm_tlb_events(x, y) do { (void)(y); } while (0) #endif -#ifdef CONFIG_DEBUG_VM_VMACACHE -#define count_vm_vmacache_event(x) count_vm_event(x) -#else -#define count_vm_vmacache_event(x) do {} while (0) -#endif - #define __count_zid_vm_events(item, zid, delta) \ __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta) diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h index b5ab452fca5b5ceaf3c741a2e0888c2950a61ce4..c1f5aebef1705c15ba03120653d5cfbea104b39c 100644 --- a/include/linux/vt_kern.h +++ b/include/linux/vt_kern.h @@ -30,7 +30,6 @@ struct vc_data *vc_deallocate(unsigned int console); void reset_palette(struct vc_data *vc); void do_blank_screen(int entering_gfx); void do_unblank_screen(int leaving_gfx); -void unblank_screen(void); void poke_blanked_console(void); int con_font_op(struct vc_data *vc, struct console_font_op *op); int con_set_cmap(unsigned char __user *cmap); diff --git a/include/linux/wait.h b/include/linux/wait.h index 58cfbf81447cc10373cf8a90baf84a5ce2592098..7f5a51aae0a73d1d7521660ba1bcd664f36285a6 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -281,7 +281,7 @@ static inline void wake_up_pollfree(struct wait_queue_head *wq_head) #define ___wait_is_interruptible(state) \ (!__builtin_constant_p(state) || \ - state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \ + (state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL))) extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags); @@ -361,8 +361,8 @@ do { \ } while (0) #define __wait_event_freezable(wq_head, condition) \ - ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \ - freezable_schedule()) + ___wait_event(wq_head, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), \ + 0, 0, schedule()) /** * wait_event_freezable - sleep (or freeze) until a condition gets true @@ -420,8 +420,8 @@ do { \ #define __wait_event_freezable_timeout(wq_head, condition, timeout) \ ___wait_event(wq_head, ___wait_cond_timeout(condition), \ - TASK_INTERRUPTIBLE, 0, timeout, \ - __ret = freezable_schedule_timeout(__ret)) + (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 0, timeout, \ + __ret = schedule_timeout(__ret)) /* * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid @@ -642,8 +642,8 @@ do { \ #define __wait_event_freezable_exclusive(wq, condition) \ - ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \ - freezable_schedule()) + ___wait_event(wq, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 1, 0,\ + schedule()) #define wait_event_freezable_exclusive(wq, condition) \ ({ \ @@ -932,6 +932,34 @@ extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *); __ret; \ }) +#define __wait_event_state(wq, condition, state) \ + ___wait_event(wq, condition, state, 0, 0, schedule()) + +/** + * wait_event_state - sleep until a condition gets true + * @wq_head: the waitqueue to wait on + * @condition: a C expression for the event to wait for + * @state: state to sleep in + * + * The process is put to sleep (@state) until the @condition evaluates to true + * or a signal is received (when allowed by @state). The @condition is checked + * each time the waitqueue @wq_head is woken up. + * + * wake_up() has to be called after changing any variable that could + * change the result of the wait condition. + * + * The function will return -ERESTARTSYS if it was interrupted by a signal + * (when allowed by @state) and 0 if @condition evaluated to true. + */ +#define wait_event_state(wq_head, condition, state) \ +({ \ + int __ret = 0; \ + might_sleep(); \ + if (!(condition)) \ + __ret = __wait_event_state(wq_head, condition, state); \ + __ret; \ +}) + #define __wait_event_killable_timeout(wq_head, condition, timeout) \ ___wait_event(wq_head, ___wait_cond_timeout(condition), \ TASK_KILLABLE, 0, timeout, \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 7dec36aecbd9fe239ba1b94f81c729e6665d6bd4..7725b7579b7819d3b1f9267fc42e26bf44ff13c0 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -71,7 +71,7 @@ static inline int wait_on_bit(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait, @@ -96,7 +96,7 @@ static inline int wait_on_bit_io(unsigned long *word, int bit, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, bit_wait_io, @@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode, unsigned long timeout) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit_timeout(word, bit, bit_wait_timeout, @@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action, unsigned mode) { might_sleep(); - if (!test_bit(bit, word)) + if (!test_bit_acquire(bit, word)) return 0; return out_of_line_wait_on_bit(word, bit, action, mode); } diff --git a/include/linux/wireless.h b/include/linux/wireless.h index 2d1b54556eff4cf38448a4371a72f37d0de2eb19..e6e34d74dda044f7ba020d09e65162b01fc703b7 100644 --- a/include/linux/wireless.h +++ b/include/linux/wireless.h @@ -26,7 +26,15 @@ struct compat_iw_point { struct __compat_iw_event { __u16 len; /* Real length of this stuff */ __u16 cmd; /* Wireless IOCTL */ - compat_caddr_t pointer; + + union { + compat_caddr_t pointer; + + /* we need ptr_bytes to make memcpy() run-time destination + * buffer bounds checking happy, nothing special + */ + DECLARE_FLEX_ARRAY(__u8, ptr_bytes); + }; }; #define IW_EV_COMPAT_LCP_LEN offsetof(struct __compat_iw_event, pointer) #define IW_EV_COMPAT_POINT_OFF offsetof(struct compat_iw_point, length) diff --git a/include/linux/writeback.h b/include/linux/writeback.h index 3f045f6d6c4f0b3084fdaa50f0a476479e2f6868..06f9291b6fd512c290d40c28827ed75480a3e264 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -17,20 +17,12 @@ struct bio; DECLARE_PER_CPU(int, dirty_throttle_leaks); /* - * The 1/4 region under the global dirty thresh is for smooth dirty throttling: - * - * (thresh - thresh/DIRTY_FULL_SCOPE, thresh) - * - * Further beyond, all dirtier tasks will enter a loop waiting (possibly long - * time) for the dirty pages to drop, unless written enough pages. - * * The global dirty threshold is normally equal to the global dirty limit, * except when the system suddenly allocates a lot of anonymous memory and * knocks down the global dirty threshold quickly, in which case the global * dirty limit will follow down slowly to prevent livelocking all dirtier tasks. */ #define DIRTY_SCOPE 8 -#define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2) struct backing_dev_info; diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 979a9d3e5bfbfc32c8764a5186efb3f8b6968ee8..4c379d23ec6e73b9590fba7c16e1152fb79376fd 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -61,7 +61,7 @@ int __vfs_setxattr_locked(struct user_namespace *, struct dentry *, const char *, const void *, size_t, int, struct inode **); int vfs_setxattr(struct user_namespace *, struct dentry *, const char *, - void *, size_t, int); + const void *, size_t, int); int __vfs_removexattr(struct user_namespace *, struct dentry *, const char *); int __vfs_removexattr_locked(struct user_namespace *, struct dentry *, const char *, struct inode **); diff --git a/include/math-emu/op-common.h b/include/math-emu/op-common.h index 4b57bbba588afd68d93b9822ada8b7bbb604e35f..8ce066c035cfa797dbfaf7fbf210664ba0628bb9 100644 --- a/include/math-emu/op-common.h +++ b/include/math-emu/op-common.h @@ -662,12 +662,14 @@ do { \ if (X##_e < 0) \ { \ FP_SET_EXCEPTION(FP_EX_INEXACT); \ + fallthrough; \ case FP_CLS_ZERO: \ r = 0; \ } \ else if (X##_e >= rsize - (rsigned > 0 || X##_s) \ || (!rsigned && X##_s)) \ { /* overflow */ \ + fallthrough; \ case FP_CLS_NAN: \ case FP_CLS_INF: \ if (rsigned == 2) \ @@ -767,6 +769,7 @@ do { \ if (X##_e >= rsize - (rsigned > 0 || X##_s) \ || (!rsigned && X##_s)) \ { /* overflow */ \ + fallthrough; \ case FP_CLS_NAN: \ case FP_CLS_INF: \ if (!rsigned) \ diff --git a/include/media/davinci/vpbe_display.h b/include/media/davinci/vpbe_display.h index 6d2a9374013067e40b0b32c0a1ac605713ed3836..d8751ea926a2b32200604dbc08f3322e1f5be410 100644 --- a/include/media/davinci/vpbe_display.h +++ b/include/media/davinci/vpbe_display.h @@ -69,13 +69,13 @@ struct vpbe_layer { struct vpbe_disp_buffer *cur_frm; /* Pointer pointing to next v4l2_buffer */ struct vpbe_disp_buffer *next_frm; - /* videobuf specific parameters - * Buffer queue used in video-buf + /* vb2 specific parameters + * Buffer queue used in vb2 */ struct vb2_queue buffer_queue; /* Queue of filled frames */ struct list_head dma_queue; - /* Used in video-buf */ + /* Used for video buffer handling */ spinlock_t irqlock; /* V4l2 specific parameters */ /* Identifies video device for this layer */ diff --git a/include/media/v4l2-common.h b/include/media/v4l2-common.h index b708d63995f458a8fe21e2910b18a76d08392235..725ff91b26e0635459f7898c9c9060e7a573b352 100644 --- a/include/media/v4l2-common.h +++ b/include/media/v4l2-common.h @@ -540,6 +540,10 @@ int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt, u32 pixelformat, s64 v4l2_get_link_freq(struct v4l2_ctrl_handler *handler, unsigned int mul, unsigned int div); +void v4l2_simplify_fraction(u32 *numerator, u32 *denominator, + unsigned int n_terms, unsigned int threshold); +u32 v4l2_fraction_to_interval(u32 numerator, u32 denominator); + static inline u64 v4l2_buffer_get_timestamp(const struct v4l2_buffer *buf) { /* diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index 00828a4f9404aee934a254ebf3aef570e1dedecf..b76a0714d425426b90f84947dd6ec1d1050efa0f 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -128,13 +128,13 @@ struct v4l2_ctrl_ops { * otherwise. */ struct v4l2_ctrl_type_ops { - bool (*equal)(const struct v4l2_ctrl *ctrl, u32 idx, + bool (*equal)(const struct v4l2_ctrl *ctrl, u32 elems, union v4l2_ctrl_ptr ptr1, union v4l2_ctrl_ptr ptr2); - void (*init)(const struct v4l2_ctrl *ctrl, u32 idx, + void (*init)(const struct v4l2_ctrl *ctrl, u32 from_idx, u32 tot_elems, union v4l2_ctrl_ptr ptr); void (*log)(const struct v4l2_ctrl *ctrl); - int (*validate)(const struct v4l2_ctrl *ctrl, u32 idx, + int (*validate)(const struct v4l2_ctrl *ctrl, u32 elems, union v4l2_ctrl_ptr ptr); }; @@ -203,7 +203,7 @@ typedef void (*v4l2_ctrl_notify_fnc)(struct v4l2_ctrl *ctrl, void *priv); * @elem_size: The size in bytes of the control. * @new_elems: The number of elements in p_new. This is the same as @elems, * except for dynamic arrays. In that case it is in the range of - * 1 to @p_dyn_alloc_elems. + * 1 to @p_array_alloc_elems. * @dims: The size of each dimension. * @nr_of_dims:The number of dimensions in @dims. * @menu_skip_mask: The control's skip mask for menu controls. This makes it @@ -227,12 +227,11 @@ typedef void (*v4l2_ctrl_notify_fnc)(struct v4l2_ctrl *ctrl, void *priv); * not freed when the control is deleted. Should this be needed * then a new internal bitfield can be added to tell the framework * to free this pointer. - * @p_dyn: Pointer to the dynamically allocated array. Only valid if - * @is_dyn_array is true. - * @p_dyn_alloc_elems: The number of elements in the dynamically allocated - * array for both the cur and new values. So @p_dyn is actually - * sized for 2 * @p_dyn_alloc_elems * @elem_size. Only valid if - * @is_dyn_array is true. + * @p_array: Pointer to the allocated array. Only valid if @is_array is true. + * @p_array_alloc_elems: The number of elements in the allocated + * array for both the cur and new values. So @p_array is actually + * sized for 2 * @p_array_alloc_elems * @elem_size. Only valid if + * @is_array is true. * @cur: Structure to store the current value. * @cur.val: The control's current value, if the @type is represented via * a u32 integer (see &enum v4l2_ctrl_type). @@ -291,8 +290,8 @@ struct v4l2_ctrl { }; unsigned long flags; void *priv; - void *p_dyn; - u32 p_dyn_alloc_elems; + void *p_array; + u32 p_array_alloc_elems; s32 val; struct { s32 val; @@ -319,15 +318,15 @@ struct v4l2_ctrl { * from a cluster with multiple controls twice (when the first * control of a cluster is applied, they all are). * @p_req_valid: If set, then p_req contains the control value for the request. - * @p_req_dyn_enomem: If set, then p_req is invalid since allocating space for - * a dynamic array failed. Attempting to read this value shall - * result in ENOMEM. Only valid if ctrl->is_dyn_array is true. - * @p_req_dyn_alloc_elems: The number of elements allocated for the dynamic - * array. Only valid if @p_req_valid and ctrl->is_dyn_array are + * @p_req_array_enomem: If set, then p_req is invalid since allocating space for + * an array failed. Attempting to read this value shall + * result in ENOMEM. Only valid if ctrl->is_array is true. + * @p_req_array_alloc_elems: The number of elements allocated for the + * array. Only valid if @p_req_valid and ctrl->is_array are * true. * @p_req_elems: The number of elements in @p_req. This is the same as * ctrl->elems, except for dynamic arrays. In that case it is in - * the range of 1 to @p_req_dyn_alloc_elems. Only valid if + * the range of 1 to @p_req_array_alloc_elems. Only valid if * @p_req_valid is true. * @p_req: If the control handler containing this control reference * is bound to a media request, then this points to the @@ -349,8 +348,8 @@ struct v4l2_ctrl_ref { bool from_other_dev; bool req_done; bool p_req_valid; - bool p_req_dyn_enomem; - u32 p_req_dyn_alloc_elems; + bool p_req_array_enomem; + u32 p_req_array_alloc_elems; u32 p_req_elems; union v4l2_ctrl_ptr p_req; }; @@ -958,6 +957,59 @@ static inline int v4l2_ctrl_modify_range(struct v4l2_ctrl *ctrl, return rval; } +/** + *__v4l2_ctrl_modify_dimensions() - Unlocked variant of v4l2_ctrl_modify_dimensions() + * + * @ctrl: The control to update. + * @dims: The control's new dimensions. + * + * Update the dimensions of an array control on the fly. The elements of the + * array are reset to their default value, even if the dimensions are + * unchanged. + * + * An error is returned if @dims is invalid for this control. + * + * The caller is responsible for acquiring the control handler mutex on behalf + * of __v4l2_ctrl_modify_dimensions(). + * + * Note: calling this function when the same control is used in pending requests + * is untested. It should work (a request with the wrong size of the control + * will drop that control silently), but it will be very confusing. + */ +int __v4l2_ctrl_modify_dimensions(struct v4l2_ctrl *ctrl, + u32 dims[V4L2_CTRL_MAX_DIMS]); + +/** + * v4l2_ctrl_modify_dimensions() - Update the dimensions of an array control. + * + * @ctrl: The control to update. + * @dims: The control's new dimensions. + * + * Update the dimensions of an array control on the fly. The elements of the + * array are reset to their default value, even if the dimensions are + * unchanged. + * + * An error is returned if @dims is invalid for this control type. + * + * This function assumes that the control handler is not locked and will + * take the lock itself. + * + * Note: calling this function when the same control is used in pending requests + * is untested. It should work (a request with the wrong size of the control + * will drop that control silently), but it will be very confusing. + */ +static inline int v4l2_ctrl_modify_dimensions(struct v4l2_ctrl *ctrl, + u32 dims[V4L2_CTRL_MAX_DIMS]) +{ + int rval; + + v4l2_ctrl_lock(ctrl); + rval = __v4l2_ctrl_modify_dimensions(ctrl, dims); + v4l2_ctrl_unlock(ctrl); + + return rval; +} + /** * v4l2_ctrl_notify() - Function to set a notify callback for a control. * @@ -1486,4 +1538,52 @@ int v4l2_ctrl_subdev_log_status(struct v4l2_subdev *sd); int v4l2_ctrl_new_fwnode_properties(struct v4l2_ctrl_handler *hdl, const struct v4l2_ctrl_ops *ctrl_ops, const struct v4l2_fwnode_device_properties *p); + +/** + * v4l2_ctrl_type_op_equal - Default v4l2_ctrl_type_ops equal callback. + * + * @ctrl: The v4l2_ctrl pointer. + * @elems: The number of elements to compare. + * @ptr1: A v4l2 control value. + * @ptr2: A v4l2 control value. + * + * Return: true if values are equal, otherwise false. + */ +bool v4l2_ctrl_type_op_equal(const struct v4l2_ctrl *ctrl, u32 elems, + union v4l2_ctrl_ptr ptr1, union v4l2_ctrl_ptr ptr2); + +/** + * v4l2_ctrl_type_op_init - Default v4l2_ctrl_type_ops init callback. + * + * @ctrl: The v4l2_ctrl pointer. + * @from_idx: Starting element index. + * @elems: The number of elements to initialize. + * @ptr: The v4l2 control value. + * + * Return: void + */ +void v4l2_ctrl_type_op_init(const struct v4l2_ctrl *ctrl, u32 from_idx, + u32 elems, union v4l2_ctrl_ptr ptr); + +/** + * v4l2_ctrl_type_op_log - Default v4l2_ctrl_type_ops log callback. + * + * @ctrl: The v4l2_ctrl pointer. + * + * Return: void + */ +void v4l2_ctrl_type_op_log(const struct v4l2_ctrl *ctrl); + +/** + * v4l2_ctrl_type_op_validate - Default v4l2_ctrl_type_ops validate callback. + * + * @ctrl: The v4l2_ctrl pointer. + * @elems: The number of elements in the control. + * @ptr: The v4l2 control value. + * + * Return: 0 on success, a negative error code on failure. + */ +int v4l2_ctrl_type_op_validate(const struct v4l2_ctrl *ctrl, u32 elems, + union v4l2_ctrl_ptr ptr); + #endif diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h index fdbd5257e020f72f9c655afc659ea5915ebd94e2..bb9de6a899e07149bbe8affce2af84cf99e08f3d 100644 --- a/include/media/v4l2-mem2mem.h +++ b/include/media/v4l2-mem2mem.h @@ -486,10 +486,10 @@ __poll_t v4l2_m2m_poll(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, * @vma: pointer to struct &vm_area_struct * * Call from driver's mmap() function. Will handle mmap() for both queues - * seamlessly for videobuffer, which will receive normal per-queue offsets and - * proper videobuf queue pointers. The differentiation is made outside videobuf - * by adding a predefined offset to buffers from one of the queues and - * subtracting it before passing it back to videobuf. Only drivers (and + * seamlessly for the video buffer, which will receive normal per-queue offsets + * and proper vb2 queue pointers. The differentiation is made outside + * vb2 by adding a predefined offset to buffers from one of the queues + * and subtracting it before passing it back to vb2. Only drivers (and * thus applications) receive modified offsets. */ int v4l2_m2m_mmap(struct file *file, struct v4l2_m2m_ctx *m2m_ctx, @@ -544,7 +544,7 @@ void v4l2_m2m_release(struct v4l2_m2m_dev *m2m_dev); * @m2m_dev: opaque pointer to the internal data to handle M2M context * @drv_priv: driver's instance private data * @queue_init: a callback for queue type-specific initialization function - * to be used for initializing videobuf_queues + * to be used for initializing vb2_queues * * Usually called from driver's ``open()`` function. */ @@ -579,7 +579,7 @@ void v4l2_m2m_ctx_release(struct v4l2_m2m_ctx *m2m_ctx); * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx * @vbuf: pointer to struct &vb2_v4l2_buffer * - * Call from videobuf_queue_ops->ops->buf_queue, videobuf_queue_ops callback. + * Call from vb2_queue_ops->ops->buf_queue, vb2_queue_ops callback. */ void v4l2_m2m_buf_queue(struct v4l2_m2m_ctx *m2m_ctx, struct vb2_v4l2_buffer *vbuf); diff --git a/include/media/v4l2-uvc.h b/include/media/v4l2-uvc.h new file mode 100644 index 0000000000000000000000000000000000000000..f83e31661333bbd707cf0a37de8557ff85599ef7 --- /dev/null +++ b/include/media/v4l2-uvc.h @@ -0,0 +1,359 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * v4l2 uvc internal API header + * + * Some commonly needed functions for uvc drivers + */ + +#ifndef __LINUX_V4L2_UVC_H +#define __LINUX_V4L2_UVC_H + +/* ------------------------------------------------------------------------ + * GUIDs + */ +#define UVC_GUID_UVC_CAMERA \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01} +#define UVC_GUID_UVC_OUTPUT \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02} +#define UVC_GUID_UVC_MEDIA_TRANSPORT_INPUT \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03} +#define UVC_GUID_UVC_PROCESSING \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01} +#define UVC_GUID_UVC_SELECTOR \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02} +#define UVC_GUID_EXT_GPIO_CONTROLLER \ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03} + +#define UVC_GUID_FORMAT_MJPEG \ + { 'M', 'J', 'P', 'G', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_YUY2 \ + { 'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_YUY2_ISIGHT \ + { 'Y', 'U', 'Y', '2', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0x00, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_NV12 \ + { 'N', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_YV12 \ + { 'Y', 'V', '1', '2', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_I420 \ + { 'I', '4', '2', '0', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_UYVY \ + { 'U', 'Y', 'V', 'Y', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y800 \ + { 'Y', '8', '0', '0', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y8 \ + { 'Y', '8', ' ', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y10 \ + { 'Y', '1', '0', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y12 \ + { 'Y', '1', '2', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y16 \ + { 'Y', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_BY8 \ + { 'B', 'Y', '8', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_BA81 \ + { 'B', 'A', '8', '1', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_GBRG \ + { 'G', 'B', 'R', 'G', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_GRBG \ + { 'G', 'R', 'B', 'G', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_RGGB \ + { 'R', 'G', 'G', 'B', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_BG16 \ + { 'B', 'G', '1', '6', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_GB16 \ + { 'G', 'B', '1', '6', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_RG16 \ + { 'R', 'G', '1', '6', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_GR16 \ + { 'G', 'R', '1', '6', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_RGBP \ + { 'R', 'G', 'B', 'P', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_BGR3 \ + { 0x7d, 0xeb, 0x36, 0xe4, 0x4f, 0x52, 0xce, 0x11, \ + 0x9f, 0x53, 0x00, 0x20, 0xaf, 0x0b, 0xa7, 0x70} +#define UVC_GUID_FORMAT_M420 \ + { 'M', '4', '2', '0', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} + +#define UVC_GUID_FORMAT_H264 \ + { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_H265 \ + { 'H', '2', '6', '5', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y8I \ + { 'Y', '8', 'I', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Y12I \ + { 'Y', '1', '2', 'I', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_Z16 \ + { 'Z', '1', '6', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_RW10 \ + { 'R', 'W', '1', '0', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_INVZ \ + { 'I', 'N', 'V', 'Z', 0x90, 0x2d, 0x58, 0x4a, \ + 0x92, 0x0b, 0x77, 0x3f, 0x1f, 0x2c, 0x55, 0x6b} +#define UVC_GUID_FORMAT_INZI \ + { 'I', 'N', 'Z', 'I', 0x66, 0x1a, 0x42, 0xa2, \ + 0x90, 0x65, 0xd0, 0x18, 0x14, 0xa8, 0xef, 0x8a} +#define UVC_GUID_FORMAT_INVI \ + { 'I', 'N', 'V', 'I', 0xdb, 0x57, 0x49, 0x5e, \ + 0x8e, 0x3f, 0xf4, 0x79, 0x53, 0x2b, 0x94, 0x6f} +#define UVC_GUID_FORMAT_CNF4 \ + { 'C', ' ', ' ', ' ', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} + +#define UVC_GUID_FORMAT_D3DFMT_L8 \ + {0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_KSMEDIA_L8_IR \ + {0x32, 0x00, 0x00, 0x00, 0x02, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} + +#define UVC_GUID_FORMAT_HEVC \ + { 'H', 'E', 'V', 'C', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} + +/* ------------------------------------------------------------------------ + * Video formats + */ + +struct uvc_format_desc { + char *name; + u8 guid[16]; + u32 fcc; +}; + +static struct uvc_format_desc uvc_fmts[] = { + { + .name = "YUV 4:2:2 (YUYV)", + .guid = UVC_GUID_FORMAT_YUY2, + .fcc = V4L2_PIX_FMT_YUYV, + }, + { + .name = "YUV 4:2:2 (YUYV)", + .guid = UVC_GUID_FORMAT_YUY2_ISIGHT, + .fcc = V4L2_PIX_FMT_YUYV, + }, + { + .name = "YUV 4:2:0 (NV12)", + .guid = UVC_GUID_FORMAT_NV12, + .fcc = V4L2_PIX_FMT_NV12, + }, + { + .name = "MJPEG", + .guid = UVC_GUID_FORMAT_MJPEG, + .fcc = V4L2_PIX_FMT_MJPEG, + }, + { + .name = "YVU 4:2:0 (YV12)", + .guid = UVC_GUID_FORMAT_YV12, + .fcc = V4L2_PIX_FMT_YVU420, + }, + { + .name = "YUV 4:2:0 (I420)", + .guid = UVC_GUID_FORMAT_I420, + .fcc = V4L2_PIX_FMT_YUV420, + }, + { + .name = "YUV 4:2:0 (M420)", + .guid = UVC_GUID_FORMAT_M420, + .fcc = V4L2_PIX_FMT_M420, + }, + { + .name = "YUV 4:2:2 (UYVY)", + .guid = UVC_GUID_FORMAT_UYVY, + .fcc = V4L2_PIX_FMT_UYVY, + }, + { + .name = "Greyscale 8-bit (Y800)", + .guid = UVC_GUID_FORMAT_Y800, + .fcc = V4L2_PIX_FMT_GREY, + }, + { + .name = "Greyscale 8-bit (Y8 )", + .guid = UVC_GUID_FORMAT_Y8, + .fcc = V4L2_PIX_FMT_GREY, + }, + { + .name = "Greyscale 8-bit (D3DFMT_L8)", + .guid = UVC_GUID_FORMAT_D3DFMT_L8, + .fcc = V4L2_PIX_FMT_GREY, + }, + { + .name = "IR 8-bit (L8_IR)", + .guid = UVC_GUID_FORMAT_KSMEDIA_L8_IR, + .fcc = V4L2_PIX_FMT_GREY, + }, + { + .name = "Greyscale 10-bit (Y10 )", + .guid = UVC_GUID_FORMAT_Y10, + .fcc = V4L2_PIX_FMT_Y10, + }, + { + .name = "Greyscale 12-bit (Y12 )", + .guid = UVC_GUID_FORMAT_Y12, + .fcc = V4L2_PIX_FMT_Y12, + }, + { + .name = "Greyscale 16-bit (Y16 )", + .guid = UVC_GUID_FORMAT_Y16, + .fcc = V4L2_PIX_FMT_Y16, + }, + { + .name = "BGGR Bayer (BY8 )", + .guid = UVC_GUID_FORMAT_BY8, + .fcc = V4L2_PIX_FMT_SBGGR8, + }, + { + .name = "BGGR Bayer (BA81)", + .guid = UVC_GUID_FORMAT_BA81, + .fcc = V4L2_PIX_FMT_SBGGR8, + }, + { + .name = "GBRG Bayer (GBRG)", + .guid = UVC_GUID_FORMAT_GBRG, + .fcc = V4L2_PIX_FMT_SGBRG8, + }, + { + .name = "GRBG Bayer (GRBG)", + .guid = UVC_GUID_FORMAT_GRBG, + .fcc = V4L2_PIX_FMT_SGRBG8, + }, + { + .name = "RGGB Bayer (RGGB)", + .guid = UVC_GUID_FORMAT_RGGB, + .fcc = V4L2_PIX_FMT_SRGGB8, + }, + { + .name = "RGB565", + .guid = UVC_GUID_FORMAT_RGBP, + .fcc = V4L2_PIX_FMT_RGB565, + }, + { + .name = "BGR 8:8:8 (BGR3)", + .guid = UVC_GUID_FORMAT_BGR3, + .fcc = V4L2_PIX_FMT_BGR24, + }, + { + .name = "H.264", + .guid = UVC_GUID_FORMAT_H264, + .fcc = V4L2_PIX_FMT_H264, + }, + { + .name = "H.265", + .guid = UVC_GUID_FORMAT_H265, + .fcc = V4L2_PIX_FMT_HEVC, + }, + { + .name = "Greyscale 8 L/R (Y8I)", + .guid = UVC_GUID_FORMAT_Y8I, + .fcc = V4L2_PIX_FMT_Y8I, + }, + { + .name = "Greyscale 12 L/R (Y12I)", + .guid = UVC_GUID_FORMAT_Y12I, + .fcc = V4L2_PIX_FMT_Y12I, + }, + { + .name = "Depth data 16-bit (Z16)", + .guid = UVC_GUID_FORMAT_Z16, + .fcc = V4L2_PIX_FMT_Z16, + }, + { + .name = "Bayer 10-bit (SRGGB10P)", + .guid = UVC_GUID_FORMAT_RW10, + .fcc = V4L2_PIX_FMT_SRGGB10P, + }, + { + .name = "Bayer 16-bit (SBGGR16)", + .guid = UVC_GUID_FORMAT_BG16, + .fcc = V4L2_PIX_FMT_SBGGR16, + }, + { + .name = "Bayer 16-bit (SGBRG16)", + .guid = UVC_GUID_FORMAT_GB16, + .fcc = V4L2_PIX_FMT_SGBRG16, + }, + { + .name = "Bayer 16-bit (SRGGB16)", + .guid = UVC_GUID_FORMAT_RG16, + .fcc = V4L2_PIX_FMT_SRGGB16, + }, + { + .name = "Bayer 16-bit (SGRBG16)", + .guid = UVC_GUID_FORMAT_GR16, + .fcc = V4L2_PIX_FMT_SGRBG16, + }, + { + .name = "Depth data 16-bit (Z16)", + .guid = UVC_GUID_FORMAT_INVZ, + .fcc = V4L2_PIX_FMT_Z16, + }, + { + .name = "Greyscale 10-bit (Y10 )", + .guid = UVC_GUID_FORMAT_INVI, + .fcc = V4L2_PIX_FMT_Y10, + }, + { + .name = "IR:Depth 26-bit (INZI)", + .guid = UVC_GUID_FORMAT_INZI, + .fcc = V4L2_PIX_FMT_INZI, + }, + { + .name = "4-bit Depth Confidence (Packed)", + .guid = UVC_GUID_FORMAT_CNF4, + .fcc = V4L2_PIX_FMT_CNF4, + }, + { + .name = "HEVC", + .guid = UVC_GUID_FORMAT_HEVC, + .fcc = V4L2_PIX_FMT_HEVC, + }, +}; + +static inline struct uvc_format_desc *uvc_format_by_guid(const u8 guid[16]) +{ + unsigned int len = ARRAY_SIZE(uvc_fmts); + unsigned int i; + + for (i = 0; i < len; ++i) { + if (memcmp(guid, uvc_fmts[i].guid, 16) == 0) + return &uvc_fmts[i]; + } + + return NULL; +} + +#endif /* __LINUX_V4L2_UVC_H */ diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h index 5468b633b9d2093eb258e2c1c83cf8dd0e31ce8a..3253bd2f6feed5cee61dce8f9e5f6c967c726ceb 100644 --- a/include/media/videobuf2-core.h +++ b/include/media/videobuf2-core.h @@ -65,7 +65,7 @@ struct vb2_buffer; * DMABUF memory types. * @get_userptr: acquire userspace memory for a hardware operation; used for * USERPTR memory types; vaddr is the address passed to the - * videobuf layer when queuing a video buffer of USERPTR type; + * videobuf2 layer when queuing a video buffer of USERPTR type; * should return an allocator private per-buffer structure * associated with the buffer on success, ERR_PTR() on failure; * the returned private structure will then be passed as @buf_priv @@ -97,7 +97,7 @@ struct vb2_buffer; * associated with the passed private structure or NULL if not * available. * @num_users: return the current number of users of a memory buffer; - * return 1 if the videobuf layer (or actually the driver using + * return 1 if the videobuf2 layer (or actually the driver using * it) is the only user. * @mmap: setup a userspace mapping for a given memory buffer under * the provided virtual memory region. @@ -210,11 +210,11 @@ enum vb2_io_modes { * enum vb2_buffer_state - current video buffer state. * @VB2_BUF_STATE_DEQUEUED: buffer under userspace control. * @VB2_BUF_STATE_IN_REQUEST: buffer is queued in media request. - * @VB2_BUF_STATE_PREPARING: buffer is being prepared in videobuf. - * @VB2_BUF_STATE_QUEUED: buffer queued in videobuf, but not in driver. + * @VB2_BUF_STATE_PREPARING: buffer is being prepared in videobuf2. + * @VB2_BUF_STATE_QUEUED: buffer queued in videobuf2, but not in driver. * @VB2_BUF_STATE_ACTIVE: buffer queued in driver and possibly used * in a hardware operation. - * @VB2_BUF_STATE_DONE: buffer returned from driver to videobuf, but + * @VB2_BUF_STATE_DONE: buffer returned from driver to videobuf2, but * not yet dequeued to userspace. * @VB2_BUF_STATE_ERROR: same as above, but the operation on the buffer * has ended with an error, which will be reported @@ -466,7 +466,7 @@ struct vb2_buf_ops { }; /** - * struct vb2_queue - a videobuf queue. + * struct vb2_queue - a videobuf2 queue. * * @type: private buffer type whose content is defined by the vb2-core * caller. For example, for V4L2, it should match @@ -544,7 +544,7 @@ struct vb2_buf_ops { * @mmap_lock: private mutex used when buffers are allocated/freed/mmapped * @memory: current memory type used * @dma_dir: DMA mapping direction. - * @bufs: videobuf buffer structures + * @bufs: videobuf2 buffer structures * @num_buffers: number of allocated/used buffers * @queued_list: list of buffers currently queued from userspace * @queued_count: number of buffers queued and ready for streaming. @@ -683,7 +683,7 @@ void *vb2_plane_vaddr(struct vb2_buffer *vb, unsigned int plane_no); void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no); /** - * vb2_buffer_done() - inform videobuf that an operation on a buffer + * vb2_buffer_done() - inform videobuf2 that an operation on a buffer * is finished. * @vb: pointer to &struct vb2_buffer to be used. * @state: state of the buffer, as defined by &enum vb2_buffer_state. diff --git a/include/media/videobuf2-dvb.h b/include/media/videobuf2-dvb.h index 8605366ec87c8af8efe18bc78c201f75f6dd0dd8..2d577b94563743c490516d924dbd64546580dc8a 100644 --- a/include/media/videobuf2-dvb.h +++ b/include/media/videobuf2-dvb.h @@ -24,7 +24,7 @@ struct vb2_dvb { struct dvb_frontend *frontend; struct vb2_queue dvbq; - /* video-buf-dvb state info */ + /* vb2-dvb state info */ struct mutex lock; int nfeeds; diff --git a/include/media/videobuf2-v4l2.h b/include/media/videobuf2-v4l2.h index 76e405c0b00307ae3c19268078882c30c6caf629..5a845887850b3db8b0fc6f5e5e99728840d7acab 100644 --- a/include/media/videobuf2-v4l2.h +++ b/include/media/videobuf2-v4l2.h @@ -62,22 +62,6 @@ struct vb2_v4l2_buffer { #define to_vb2_v4l2_buffer(vb) \ container_of(vb, struct vb2_v4l2_buffer, vb2_buf) -/** - * vb2_find_timestamp() - Find buffer with given timestamp in the queue - * - * @q: pointer to &struct vb2_queue with videobuf2 queue. - * @timestamp: the timestamp to find. - * @start_idx: the start index (usually 0) in the buffer array to start - * searching from. Note that there may be multiple buffers - * with the same timestamp value, so you can restart the search - * by setting @start_idx to the previously found index + 1. - * - * Returns the buffer index of the buffer with the given @timestamp, or - * -1 if no buffer with @timestamp was found. - */ -int vb2_find_timestamp(const struct vb2_queue *q, u64 timestamp, - unsigned int start_idx); - /** * vb2_find_buffer() - Find a buffer with given timestamp * diff --git a/include/media/vsp1.h b/include/media/vsp1.h index cc1b0d42ce95b080b0d5b08c05ed3ffeaf8945a4..48f4a5023d81512d0bd7a7e0afc409d706ede2e1 100644 --- a/include/media/vsp1.h +++ b/include/media/vsp1.h @@ -51,6 +51,7 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index, * @dst: destination rectangle on the display (integer coordinates) * @alpha: alpha value (0: fully transparent, 255: fully opaque) * @zpos: Z position of the plane (from 0 to number of planes minus 1) + * @premult: true for premultiplied alpha */ struct vsp1_du_atomic_config { u32 pixelformat; @@ -60,6 +61,7 @@ struct vsp1_du_atomic_config { struct v4l2_rect dst; unsigned int alpha; unsigned int zpos; + bool premult; }; /** diff --git a/include/net/9p/9p.h b/include/net/9p/9p.h index 24a509f559ee2b5210724a1117eb6be54679a056..13abe013af21cd331464c23acb5a561224d4244a 100644 --- a/include/net/9p/9p.h +++ b/include/net/9p/9p.h @@ -331,6 +331,9 @@ enum p9_qid_t { /* size of header for zero copy read/write */ #define P9_ZC_HDR_SZ 4096 +/* maximum length of an error string */ +#define P9_ERRMAX 128 + /** * struct p9_qid - file system entity information * @type: 8-bit type &p9_qid_t diff --git a/include/net/9p/transport.h b/include/net/9p/transport.h index ff842f9630718bb8ee6885385775badaef700906..766ec07c95999dcc686432b12d480ff86217984e 100644 --- a/include/net/9p/transport.h +++ b/include/net/9p/transport.h @@ -19,6 +19,10 @@ * @list: used to maintain a list of currently available transports * @name: the human-readable name of the transport * @maxsize: transport provided maximum packet size + * @pooled_rbuffers: currently only set for RDMA transport which pulls the + * response buffers from a shared pool, and accordingly + * we're less flexible when choosing the response message + * size in this case * @def: set if this transport should be considered the default * @create: member function to create a new connection on this transport * @close: member function to discard a connection on this transport @@ -38,6 +42,7 @@ struct p9_trans_module { struct list_head list; char *name; /* name of transport */ int maxsize; /* max message size of transport */ + bool pooled_rbuffers; int def; /* this transport should be default */ struct module *owner; int (*create)(struct p9_client *client, diff --git a/include/net/act_api.h b/include/net/act_api.h index 9cf6870b526eb2a54bd8c8d11706588667e68a20..61f2ceb3939ee2dde7515350de1ad482b00d7fdd 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -111,6 +111,7 @@ struct tc_action_ops { struct list_head head; char kind[IFNAMSIZ]; enum tca_id id; /* identifier should match kind */ + unsigned int net_id; size_t size; struct module *owner; int (*act)(struct sk_buff *, const struct tc_action *, diff --git a/include/net/af_rxrpc.h b/include/net/af_rxrpc.h index cee5f83c0f11a1c581951cd66ca6e69f8e0fb613..b69ca695935ce756e8b1b2e2dc837187f4f088c2 100644 --- a/include/net/af_rxrpc.h +++ b/include/net/af_rxrpc.h @@ -66,8 +66,6 @@ int rxrpc_kernel_charge_accept(struct socket *, rxrpc_notify_rx_t, void rxrpc_kernel_set_tx_length(struct socket *, struct rxrpc_call *, s64); bool rxrpc_kernel_check_life(const struct socket *, const struct rxrpc_call *); u32 rxrpc_kernel_get_epoch(struct socket *, struct rxrpc_call *); -bool rxrpc_kernel_get_reply_time(struct socket *, struct rxrpc_call *, - ktime_t *); bool rxrpc_kernel_call_is_complete(struct rxrpc_call *); void rxrpc_kernel_set_max_life(struct socket *, struct rxrpc_call *, unsigned long); diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index 1c53c4c4d88fecc3e0b8e1632b9132f644fea403..568a87c5e0d0f95622bc153ecb9a39e803479ad7 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -78,6 +78,7 @@ struct vsock_sock { s64 vsock_stream_has_data(struct vsock_sock *vsk); s64 vsock_stream_has_space(struct vsock_sock *vsk); struct sock *vsock_create_connected(struct sock *parent); +void vsock_data_ready(struct sock *sk); /**** TRANSPORT ****/ @@ -135,6 +136,7 @@ struct vsock_transport { u64 (*stream_rcvhiwat)(struct vsock_sock *); bool (*stream_is_active)(struct vsock_sock *); bool (*stream_allow)(u32 cid, u32 port); + int (*set_rcvlowat)(struct vsock_sock *vsk, int val); /* SEQ_PACKET. */ ssize_t (*seqpacket_dequeue)(struct vsock_sock *vsk, struct msghdr *msg, diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index e72f3b247b5ec824fe9b429f031f05fe5c297ded..bcc5a4cd2c17b79b6f43507e22263103fb2da96b 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -627,6 +627,7 @@ static inline bool iso_enabled(void) int mgmt_init(void); void mgmt_exit(void); +void mgmt_cleanup(struct sock *sk); void bt_sock_reclassify_lock(struct sock *sk, int proto); diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index cf29511b25a8d8a8e79823a8e4a990684110b136..e004ba04a9ae16b7cac00f3c6d9f2a8ad9807cd8 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -354,6 +354,10 @@ enum { HCI_LE_SIMULTANEOUS_ROLES, HCI_CMD_DRAIN_WORKQUEUE, + HCI_MESH_EXPERIMENTAL, + HCI_MESH, + HCI_MESH_SENDING, + __HCI_NUM_FLAGS, }; diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index e7862903187d856f501cc5b5abdf8659814d2af2..c54bc71254afab6d401c5bb6071e544dbb8a7585 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -238,6 +238,7 @@ struct adv_info { bool enabled; bool pending; bool periodic; + __u8 mesh; __u8 instance; __u32 flags; __u16 timeout; @@ -372,6 +373,8 @@ struct hci_dev { __u8 le_resolv_list_size; __u8 le_num_of_adv_sets; __u8 le_states[8]; + __u8 mesh_ad_types[16]; + __u8 mesh_send_ref; __u8 commands[64]; __u8 hci_ver; __u16 hci_rev; @@ -511,6 +514,7 @@ struct hci_dev { struct list_head cmd_sync_work_list; struct mutex cmd_sync_work_lock; struct work_struct cmd_sync_cancel_work; + struct work_struct reenable_adv_work; __u16 discov_timeout; struct delayed_work discov_off; @@ -561,6 +565,7 @@ struct hci_dev { struct hci_conn_hash conn_hash; + struct list_head mesh_pending; struct list_head mgmt_pending; struct list_head reject_list; struct list_head accept_list; @@ -614,6 +619,8 @@ struct hci_dev { struct delayed_work rpa_expired; bdaddr_t rpa; + struct delayed_work mesh_send_done; + enum { INTERLEAVE_SCAN_NONE, INTERLEAVE_SCAN_NO_FILTER, @@ -1576,7 +1583,8 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, u32 flags, u16 adv_data_len, u8 *adv_data, u16 scan_rsp_len, u8 *scan_rsp_data, u16 timeout, u16 duration, s8 tx_power, - u32 min_interval, u32 max_interval); + u32 min_interval, u32 max_interval, + u8 mesh_handle); struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance, u32 flags, u8 data_len, u8 *data, u32 min_interval, u32 max_interval); @@ -1997,6 +2005,9 @@ void hci_mgmt_chan_unregister(struct hci_mgmt_chan *c); #define DISCOV_LE_FAST_ADV_INT_MAX 0x00F0 /* 150 msec */ #define DISCOV_LE_PER_ADV_INT_MIN 0x00A0 /* 200 msec */ #define DISCOV_LE_PER_ADV_INT_MAX 0x00A0 /* 200 msec */ +#define DISCOV_LE_ADV_MESH_MIN 0x00A0 /* 100 msec */ +#define DISCOV_LE_ADV_MESH_MAX 0x00A0 /* 100 msec */ +#define INTERVAL_TO_MS(x) (((x) * 10) / 0x10) #define NAME_RESOLVE_DURATION msecs_to_jiffies(10240) /* 10.24 sec */ @@ -2048,7 +2059,8 @@ void mgmt_start_discovery_complete(struct hci_dev *hdev, u8 status); void mgmt_stop_discovery_complete(struct hci_dev *hdev, u8 status); void mgmt_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, u8 *dev_class, s8 rssi, u32 flags, - u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len); + u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len, + u64 instant); void mgmt_remote_name(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, s8 rssi, u8 *name, u8 name_len); void mgmt_discovering(struct hci_dev *hdev, u8 discovering); @@ -2075,6 +2087,7 @@ int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip); void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle, bdaddr_t *bdaddr, u8 addr_type); +int hci_abort_conn(struct hci_conn *conn, u8 reason); u8 hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency, u16 to_multiplier); void hci_le_start_enc(struct hci_conn *conn, __le16 ediv, __le64 rand, diff --git a/include/net/bluetooth/hci_sock.h b/include/net/bluetooth/hci_sock.h index 0520e21ab698349457a7b2861a5eb2d57708419d..9949870f7d780367cd9628522e9a3c00b024493f 100644 --- a/include/net/bluetooth/hci_sock.h +++ b/include/net/bluetooth/hci_sock.h @@ -124,8 +124,6 @@ struct hci_dev_info { __u16 acl_pkts; __u16 sco_mtu; __u16 sco_pkts; - __u16 iso_mtu; - __u16 iso_pkts; struct hci_dev_stats stat; }; diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 3843f5060c73750509ae6566856b5f8a09fd0413..17f5a4c32f36ee1bfeff1b0bf5950e132bf218bb 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -16,6 +16,7 @@ struct hci_cmd_sync_work_entry { hci_cmd_sync_work_destroy_t destroy; }; +struct adv_info; /* Function with sync suffix shall not be called with hdev->lock held as they * wait the command to complete and in the meantime an event could be received * which could attempt to acquire hdev->lock causing a deadlock. @@ -51,11 +52,16 @@ int hci_update_class_sync(struct hci_dev *hdev); int hci_update_name_sync(struct hci_dev *hdev); int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode); +int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, + bool use_rpa, struct adv_info *adv_instance, + u8 *own_addr_type, bdaddr_t *rand_addr); + int hci_update_random_address_sync(struct hci_dev *hdev, bool require_privacy, bool rpa, u8 *own_addr_type); int hci_update_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance); int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance); +int hci_update_adv_data(struct hci_dev *hdev, u8 instance); int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance, bool force); @@ -72,7 +78,8 @@ int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len, int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk, u8 instance, bool force); int hci_disable_advertising_sync(struct hci_dev *hdev); - +int hci_clear_adv_instance_sync(struct hci_dev *hdev, struct sock *sk, + u8 instance, bool force); int hci_update_passive_scan_sync(struct hci_dev *hdev); int hci_update_passive_scan(struct hci_dev *hdev); int hci_read_rssi_sync(struct hci_dev *hdev, __le16 handle); diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h index 7c1ad0f6fcecb05b33b627c5d6a4a636f2cec0ea..743f6f59dff81af5bfa9a8fb9d34c95877a7d0ab 100644 --- a/include/net/bluetooth/mgmt.h +++ b/include/net/bluetooth/mgmt.h @@ -837,6 +837,42 @@ struct mgmt_cp_add_adv_patterns_monitor_rssi { struct mgmt_adv_pattern patterns[]; } __packed; #define MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE 8 +#define MGMT_OP_SET_MESH_RECEIVER 0x0057 +struct mgmt_cp_set_mesh { + __u8 enable; + __le16 window; + __le16 period; + __u8 num_ad_types; + __u8 ad_types[]; +} __packed; +#define MGMT_SET_MESH_RECEIVER_SIZE 6 + +#define MGMT_OP_MESH_READ_FEATURES 0x0058 +#define MGMT_MESH_READ_FEATURES_SIZE 0 +#define MESH_HANDLES_MAX 3 +struct mgmt_rp_mesh_read_features { + __le16 index; + __u8 max_handles; + __u8 used_handles; + __u8 handles[MESH_HANDLES_MAX]; +} __packed; + +#define MGMT_OP_MESH_SEND 0x0059 +struct mgmt_cp_mesh_send { + struct mgmt_addr_info addr; + __le64 instant; + __le16 delay; + __u8 cnt; + __u8 adv_data_len; + __u8 adv_data[]; +} __packed; +#define MGMT_MESH_SEND_SIZE 19 + +#define MGMT_OP_MESH_SEND_CANCEL 0x005A +struct mgmt_cp_mesh_send_cancel { + __u8 handle; +} __packed; +#define MGMT_MESH_SEND_CANCEL_SIZE 1 #define MGMT_EV_CMD_COMPLETE 0x0001 struct mgmt_ev_cmd_complete { @@ -1120,3 +1156,19 @@ struct mgmt_ev_adv_monitor_device_lost { __le16 monitor_handle; struct mgmt_addr_info addr; } __packed; + +#define MGMT_EV_MESH_DEVICE_FOUND 0x0031 +struct mgmt_ev_mesh_device_found { + struct mgmt_addr_info addr; + __s8 rssi; + __le64 instant; + __le32 flags; + __le16 eir_len; + __u8 eir[]; +} __packed; + + +#define MGMT_EV_MESH_PACKET_CMPLT 0x0032 +struct mgmt_ev_mesh_pkt_cmplt { + __u8 handle; +} __packed; diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index 184105d682942c2b2a50b023bf6be3441fbf7a21..a016f275cb0163babd5f2c9243371c35f5db959f 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -15,8 +15,6 @@ #define PKT_TYPE_LACPDU cpu_to_be16(ETH_P_SLOW) #define AD_TIMER_INTERVAL 100 /*msec*/ -#define MULTICAST_LACPDU_ADDR {0x01, 0x80, 0xC2, 0x00, 0x00, 0x02} - #define AD_LACP_SLOW 0 #define AD_LACP_FAST 1 @@ -290,7 +288,7 @@ static inline const char *bond_3ad_churn_desc(churn_state_t state) } /* ========== AD Exported functions to the main bonding code ========== */ -void bond_3ad_initialize(struct bonding *bond, u16 tick_resolution); +void bond_3ad_initialize(struct bonding *bond); void bond_3ad_bind_slave(struct slave *slave); void bond_3ad_unbind_slave(struct slave *slave); void bond_3ad_state_machine_handler(struct work_struct *); diff --git a/include/net/bonding.h b/include/net/bonding.h index afd606df149adc4b1e18a562fa5f9f71f0e85a47..e999f851738bd68b88ffb82daa6960aa5068c229 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -786,6 +786,9 @@ extern struct rtnl_link_ops bond_link_ops; /* exported from bond_sysfs_slave.c */ extern const struct sysfs_ops slave_sysfs_ops; +/* exported from bond_3ad.c */ +extern const u8 lacpdu_mcast_addr[]; + static inline netdev_tx_t bond_tx_drop(struct net_device *dev, struct sk_buff *skb) { dev_core_stats_tx_dropped_inc(dev); diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h index c4898fcbf923bf01f14c6bcc694eb036d75d7195..f90f0021f5f2dd4932c564ce8b8a7333093456c5 100644 --- a/include/net/busy_poll.h +++ b/include/net/busy_poll.h @@ -33,7 +33,7 @@ extern unsigned int sysctl_net_busy_poll __read_mostly; static inline bool net_busy_loop_on(void) { - return sysctl_net_busy_poll; + return READ_ONCE(sysctl_net_busy_poll); } static inline bool sk_can_busy_loop(const struct sock *sk) diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 908d5839348434dfbc1f5f07958ade2aa4351428..e09ff87146c1c56522ef7b5188951b94f60420cd 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -2316,6 +2316,7 @@ struct ocb_setup { * @cwmax: Maximum contention window [a value of the form 2^n-1 in the range * 1..32767] * @aifs: Arbitration interframe space [0..255] + * @link_id: link_id or -1 for non-MLD */ struct ieee80211_txq_params { enum nl80211_ac ac; @@ -2323,6 +2324,7 @@ struct ieee80211_txq_params { u16 cwmin; u16 cwmax; u8 aifs; + int link_id; }; /** @@ -3929,22 +3931,33 @@ struct mgmt_frame_regs { * @del_intf_link: Remove an MLO link from the given interface. * * @add_key: add a key with the given parameters. @mac_addr will be %NULL - * when adding a group key. + * when adding a group key. @link_id will be -1 for non-MLO connection. + * For MLO connection, @link_id will be >= 0 for group key and -1 for + * pairwise key, @mac_addr will be peer's MLD address for MLO pairwise key. * * @get_key: get information about the key with the given parameters. * @mac_addr will be %NULL when requesting information for a group * key. All pointers given to the @callback function need not be valid * after it returns. This function should return an error if it is * not possible to retrieve the key, -ENOENT if it doesn't exist. + * @link_id will be -1 for non-MLO connection. For MLO connection, + * @link_id will be >= 0 for group key and -1 for pairwise key, @mac_addr + * will be peer's MLD address for MLO pairwise key. * * @del_key: remove a key given the @mac_addr (%NULL for a group key) - * and @key_index, return -ENOENT if the key doesn't exist. + * and @key_index, return -ENOENT if the key doesn't exist. @link_id will + * be -1 for non-MLO connection. For MLO connection, @link_id will be >= 0 + * for group key and -1 for pairwise key, @mac_addr will be peer's MLD + * address for MLO pairwise key. * - * @set_default_key: set the default key on an interface + * @set_default_key: set the default key on an interface. @link_id will be >= 0 + * for MLO connection and -1 for non-MLO connection. * - * @set_default_mgmt_key: set the default management frame key on an interface + * @set_default_mgmt_key: set the default management frame key on an interface. + * @link_id will be >= 0 for MLO connection and -1 for non-MLO connection. * - * @set_default_beacon_key: set the default Beacon frame key on an interface + * @set_default_beacon_key: set the default Beacon frame key on an interface. + * @link_id will be >= 0 for MLO connection and -1 for non-MLO connection. * * @set_rekey_data: give the data necessary for GTK rekeying to the driver * @@ -4293,22 +4306,24 @@ struct cfg80211_ops { unsigned int link_id); int (*add_key)(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, - struct key_params *params); + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, struct key_params *params); int (*get_key)(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, - void *cookie, + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params*)); int (*del_key)(struct wiphy *wiphy, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr); + int link_id, u8 key_index, bool pairwise, + const u8 *mac_addr); int (*set_default_key)(struct wiphy *wiphy, - struct net_device *netdev, + struct net_device *netdev, int link_id, u8 key_index, bool unicast, bool multicast); int (*set_default_mgmt_key)(struct wiphy *wiphy, - struct net_device *netdev, + struct net_device *netdev, int link_id, u8 key_index); int (*set_default_beacon_key)(struct wiphy *wiphy, struct net_device *netdev, + int link_id, u8 key_index); int (*start_ap)(struct wiphy *wiphy, struct net_device *dev, @@ -8266,6 +8281,7 @@ void cfg80211_ch_switch_notify(struct net_device *dev, * cfg80211_ch_switch_started_notify - notify channel switch start * @dev: the device on which the channel switch started * @chandef: the future channel definition + * @link_id: the link ID for MLO, must be 0 for non-MLO * @count: the number of TBTTs until the channel switch happens * @quiet: whether or not immediate quiet was requested by the AP * @@ -8275,7 +8291,8 @@ void cfg80211_ch_switch_notify(struct net_device *dev, */ void cfg80211_ch_switch_started_notify(struct net_device *dev, struct cfg80211_chan_def *chandef, - u8 count, bool quiet); + unsigned int link_id, u8 count, + bool quiet); /** * ieee80211_operating_class_to_band - convert operating class to band diff --git a/include/net/devlink.h b/include/net/devlink.h index 119ed1ffb988d8be9b9baf129f131c0a6d7a0f21..ba6b8b0949432a677c85fd3b66c81e609d27edc0 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -118,7 +118,6 @@ struct devlink_rate { struct devlink_port { struct list_head list; - struct list_head param_list; struct list_head region_list; struct devlink *devlink; unsigned int index; @@ -130,7 +129,9 @@ struct devlink_port { void *type_dev; struct devlink_port_attrs attrs; u8 attrs_set:1, - switch_port:1; + switch_port:1, + registered:1, + initialized:1; struct delayed_work type_warn_dw; struct list_head reporter_list; struct mutex reporters_lock; /* Protects reporter_list */ @@ -624,8 +625,7 @@ struct devlink_flash_update_params { u32 overwrite_mask; }; -#define DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT BIT(0) -#define DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK BIT(1) +#define DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK BIT(0) struct devlink_region; struct devlink_info_req; @@ -1564,6 +1564,9 @@ void devlink_set_features(struct devlink *devlink, u64 features); void devlink_register(struct devlink *devlink); void devlink_unregister(struct devlink *devlink); void devlink_free(struct devlink *devlink); +void devlink_port_init(struct devlink *devlink, + struct devlink_port *devlink_port); +void devlink_port_fini(struct devlink_port *devlink_port); int devl_port_register(struct devlink *devlink, struct devlink_port *devlink_port, unsigned int port_index); @@ -1714,15 +1717,31 @@ int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name); int devlink_info_board_serial_number_put(struct devlink_info_req *req, const char *bsn); + +enum devlink_info_version_type { + DEVLINK_INFO_VERSION_TYPE_NONE, + DEVLINK_INFO_VERSION_TYPE_COMPONENT, /* May be used as flash update + * component by name. + */ +}; + int devlink_info_version_fixed_put(struct devlink_info_req *req, const char *version_name, const char *version_value); int devlink_info_version_stored_put(struct devlink_info_req *req, const char *version_name, const char *version_value); +int devlink_info_version_stored_put_ext(struct devlink_info_req *req, + const char *version_name, + const char *version_value, + enum devlink_info_version_type version_type); int devlink_info_version_running_put(struct devlink_info_req *req, const char *version_name, const char *version_value); +int devlink_info_version_running_put_ext(struct devlink_info_req *req, + const char *version_name, + const char *version_value, + enum devlink_info_version_type version_type); int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg); int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg); diff --git a/include/net/dn.h b/include/net/dn.h deleted file mode 100644 index ba9655b0098ae0a9b83ca6402a5f8afeeacd89a4..0000000000000000000000000000000000000000 --- a/include/net/dn.h +++ /dev/null @@ -1,231 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _NET_DN_H -#define _NET_DN_H - -#include -#include -#include -#include -#include - -struct dn_scp /* Session Control Port */ -{ - unsigned char state; -#define DN_O 1 /* Open */ -#define DN_CR 2 /* Connect Receive */ -#define DN_DR 3 /* Disconnect Reject */ -#define DN_DRC 4 /* Discon. Rej. Complete*/ -#define DN_CC 5 /* Connect Confirm */ -#define DN_CI 6 /* Connect Initiate */ -#define DN_NR 7 /* No resources */ -#define DN_NC 8 /* No communication */ -#define DN_CD 9 /* Connect Delivery */ -#define DN_RJ 10 /* Rejected */ -#define DN_RUN 11 /* Running */ -#define DN_DI 12 /* Disconnect Initiate */ -#define DN_DIC 13 /* Disconnect Complete */ -#define DN_DN 14 /* Disconnect Notificat */ -#define DN_CL 15 /* Closed */ -#define DN_CN 16 /* Closed Notification */ - - __le16 addrloc; - __le16 addrrem; - __u16 numdat; - __u16 numoth; - __u16 numoth_rcv; - __u16 numdat_rcv; - __u16 ackxmt_dat; - __u16 ackxmt_oth; - __u16 ackrcv_dat; - __u16 ackrcv_oth; - __u8 flowrem_sw; - __u8 flowloc_sw; -#define DN_SEND 2 -#define DN_DONTSEND 1 -#define DN_NOCHANGE 0 - __u16 flowrem_dat; - __u16 flowrem_oth; - __u16 flowloc_dat; - __u16 flowloc_oth; - __u8 services_rem; - __u8 services_loc; - __u8 info_rem; - __u8 info_loc; - - __u16 segsize_rem; - __u16 segsize_loc; - - __u8 nonagle; - __u8 multi_ireq; - __u8 accept_mode; - unsigned long seg_total; /* Running total of current segment */ - - struct optdata_dn conndata_in; - struct optdata_dn conndata_out; - struct optdata_dn discdata_in; - struct optdata_dn discdata_out; - struct accessdata_dn accessdata; - - struct sockaddr_dn addr; /* Local address */ - struct sockaddr_dn peer; /* Remote address */ - - /* - * In this case the RTT estimation is not specified in the - * docs, nor is any back off algorithm. Here we follow well - * known tcp algorithms with a few small variations. - * - * snd_window: Max number of packets we send before we wait for - * an ack to come back. This will become part of a - * more complicated scheme when we support flow - * control. - * - * nsp_srtt: Round-Trip-Time (x8) in jiffies. This is a rolling - * average. - * nsp_rttvar: Round-Trip-Time-Varience (x4) in jiffies. This is the - * varience of the smoothed average (but calculated in - * a simpler way than for normal statistical varience - * calculations). - * - * nsp_rxtshift: Backoff counter. Value is zero normally, each time - * a packet is lost is increases by one until an ack - * is received. Its used to index an array of backoff - * multipliers. - */ -#define NSP_MIN_WINDOW 1 -#define NSP_MAX_WINDOW (0x07fe) - unsigned long max_window; - unsigned long snd_window; -#define NSP_INITIAL_SRTT (HZ) - unsigned long nsp_srtt; -#define NSP_INITIAL_RTTVAR (HZ*3) - unsigned long nsp_rttvar; -#define NSP_MAXRXTSHIFT 12 - unsigned long nsp_rxtshift; - - /* - * Output queues, one for data, one for otherdata/linkservice - */ - struct sk_buff_head data_xmit_queue; - struct sk_buff_head other_xmit_queue; - - /* - * Input queue for other data - */ - struct sk_buff_head other_receive_queue; - int other_report; - - /* - * Stuff to do with the slow timer - */ - unsigned long stamp; /* time of last transmit */ - unsigned long persist; - int (*persist_fxn)(struct sock *sk); - unsigned long keepalive; - void (*keepalive_fxn)(struct sock *sk); - -}; - -static inline struct dn_scp *DN_SK(struct sock *sk) -{ - return (struct dn_scp *)(sk + 1); -} - -/* - * src,dst : Source and Destination DECnet addresses - * hops : Number of hops through the network - * dst_port, src_port : NSP port numbers - * services, info : Useful data extracted from conninit messages - * rt_flags : Routing flags byte - * nsp_flags : NSP layer flags byte - * segsize : Size of segment - * segnum : Number, for data, otherdata and linkservice - * xmit_count : Number of times we've transmitted this skb - * stamp : Time stamp of most recent transmission, used in RTT calculations - * iif: Input interface number - * - * As a general policy, this structure keeps all addresses in network - * byte order, and all else in host byte order. Thus dst, src, dst_port - * and src_port are in network order. All else is in host order. - * - */ -#define DN_SKB_CB(skb) ((struct dn_skb_cb *)(skb)->cb) -struct dn_skb_cb { - __le16 dst; - __le16 src; - __u16 hops; - __le16 dst_port; - __le16 src_port; - __u8 services; - __u8 info; - __u8 rt_flags; - __u8 nsp_flags; - __u16 segsize; - __u16 segnum; - __u16 xmit_count; - unsigned long stamp; - int iif; -}; - -static inline __le16 dn_eth2dn(const unsigned char *ethaddr) -{ - return get_unaligned((__le16 *)(ethaddr + 4)); -} - -static inline __le16 dn_saddr2dn(struct sockaddr_dn *saddr) -{ - return *(__le16 *)saddr->sdn_nodeaddr; -} - -static inline void dn_dn2eth(unsigned char *ethaddr, __le16 addr) -{ - __u16 a = le16_to_cpu(addr); - ethaddr[0] = 0xAA; - ethaddr[1] = 0x00; - ethaddr[2] = 0x04; - ethaddr[3] = 0x00; - ethaddr[4] = (__u8)(a & 0xff); - ethaddr[5] = (__u8)(a >> 8); -} - -static inline void dn_sk_ports_copy(struct flowidn *fld, struct dn_scp *scp) -{ - fld->fld_sport = scp->addrloc; - fld->fld_dport = scp->addrrem; -} - -unsigned int dn_mss_from_pmtu(struct net_device *dev, int mtu); -void dn_register_sysctl(void); -void dn_unregister_sysctl(void); - -#define DN_MENUVER_ACC 0x01 -#define DN_MENUVER_USR 0x02 -#define DN_MENUVER_PRX 0x04 -#define DN_MENUVER_UIC 0x08 - -struct sock *dn_sklist_find_listener(struct sockaddr_dn *addr); -struct sock *dn_find_by_skb(struct sk_buff *skb); -#define DN_ASCBUF_LEN 9 -char *dn_addr2asc(__u16, char *); -int dn_destroy_timer(struct sock *sk); - -int dn_sockaddr2username(struct sockaddr_dn *addr, unsigned char *buf, - unsigned char type); -int dn_username2sockaddr(unsigned char *data, int len, struct sockaddr_dn *addr, - unsigned char *type); - -void dn_start_slow_timer(struct sock *sk); -void dn_stop_slow_timer(struct sock *sk); - -extern __le16 decnet_address; -extern int decnet_debug_level; -extern int decnet_time_wait; -extern int decnet_dn_count; -extern int decnet_di_count; -extern int decnet_dr_count; -extern int decnet_no_fc_max_cwnd; - -extern long sysctl_decnet_mem[3]; -extern int sysctl_decnet_wmem[3]; -extern int sysctl_decnet_rmem[3]; - -#endif /* _NET_DN_H */ diff --git a/include/net/dn_dev.h b/include/net/dn_dev.h deleted file mode 100644 index bec303ea8367965418c1f5f77f85d3518abc22e9..0000000000000000000000000000000000000000 --- a/include/net/dn_dev.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _NET_DN_DEV_H -#define _NET_DN_DEV_H - -#include - -struct dn_dev; - -struct dn_ifaddr { - struct dn_ifaddr __rcu *ifa_next; - struct dn_dev *ifa_dev; - __le16 ifa_local; - __le16 ifa_address; - __u32 ifa_flags; - __u8 ifa_scope; - char ifa_label[IFNAMSIZ]; - struct rcu_head rcu; -}; - -#define DN_DEV_S_RU 0 /* Run - working normally */ -#define DN_DEV_S_CR 1 /* Circuit Rejected */ -#define DN_DEV_S_DS 2 /* Data Link Start */ -#define DN_DEV_S_RI 3 /* Routing Layer Initialize */ -#define DN_DEV_S_RV 4 /* Routing Layer Verify */ -#define DN_DEV_S_RC 5 /* Routing Layer Complete */ -#define DN_DEV_S_OF 6 /* Off */ -#define DN_DEV_S_HA 7 /* Halt */ - - -/* - * The dn_dev_parms structure contains the set of parameters - * for each device (hence inclusion in the dn_dev structure) - * and an array is used to store the default types of supported - * device (in dn_dev.c). - * - * The type field matches the ARPHRD_ constants and is used in - * searching the list for supported devices when new devices - * come up. - * - * The mode field is used to find out if a device is broadcast, - * multipoint, or pointopoint. Please note that DECnet thinks - * different ways about devices to the rest of the kernel - * so the normal IFF_xxx flags are invalid here. For devices - * which can be any combination of the previously mentioned - * attributes, you can set this on a per device basis by - * installing an up() routine. - * - * The device state field, defines the initial state in which the - * device will come up. In the dn_dev structure, it is the actual - * state. - * - * Things have changed here. I've killed timer1 since it's a user space - * issue for a user space routing deamon to sort out. The kernel does - * not need to be bothered with it. - * - * Timers: - * t2 - Rate limit timer, min time between routing and hello messages - * t3 - Hello timer, send hello messages when it expires - * - * Callbacks: - * up() - Called to initialize device, return value can veto use of - * device with DECnet. - * down() - Called to turn device off when it goes down - * timer3() - Called once for each ifaddr when timer 3 goes off - * - * sysctl - Hook for sysctl things - * - */ -struct dn_dev_parms { - int type; /* ARPHRD_xxx */ - int mode; /* Broadcast, Unicast, Mulitpoint */ -#define DN_DEV_BCAST 1 -#define DN_DEV_UCAST 2 -#define DN_DEV_MPOINT 4 - int state; /* Initial state */ - int forwarding; /* 0=EndNode, 1=L1Router, 2=L2Router */ - unsigned long t2; /* Default value of t2 */ - unsigned long t3; /* Default value of t3 */ - int priority; /* Priority to be a router */ - char *name; /* Name for sysctl */ - int (*up)(struct net_device *); - void (*down)(struct net_device *); - void (*timer3)(struct net_device *, struct dn_ifaddr *ifa); - void *sysctl; -}; - - -struct dn_dev { - struct dn_ifaddr __rcu *ifa_list; - struct net_device *dev; - struct dn_dev_parms parms; - char use_long; - struct timer_list timer; - unsigned long t3; - struct neigh_parms *neigh_parms; - __u8 addr[ETH_ALEN]; - struct neighbour *router; /* Default router on circuit */ - struct neighbour *peer; /* Peer on pointopoint links */ - unsigned long uptime; /* Time device went up in jiffies */ -}; - -struct dn_short_packet { - __u8 msgflg; - __le16 dstnode; - __le16 srcnode; - __u8 forward; -} __packed; - -struct dn_long_packet { - __u8 msgflg; - __u8 d_area; - __u8 d_subarea; - __u8 d_id[6]; - __u8 s_area; - __u8 s_subarea; - __u8 s_id[6]; - __u8 nl2; - __u8 visit_ct; - __u8 s_class; - __u8 pt; -} __packed; - -/*------------------------- DRP - Routing messages ---------------------*/ - -struct endnode_hello_message { - __u8 msgflg; - __u8 tiver[3]; - __u8 id[6]; - __u8 iinfo; - __le16 blksize; - __u8 area; - __u8 seed[8]; - __u8 neighbor[6]; - __le16 timer; - __u8 mpd; - __u8 datalen; - __u8 data[2]; -} __packed; - -struct rtnode_hello_message { - __u8 msgflg; - __u8 tiver[3]; - __u8 id[6]; - __u8 iinfo; - __le16 blksize; - __u8 priority; - __u8 area; - __le16 timer; - __u8 mpd; -} __packed; - - -void dn_dev_init(void); -void dn_dev_cleanup(void); - -int dn_dev_ioctl(unsigned int cmd, void __user *arg); - -void dn_dev_devices_off(void); -void dn_dev_devices_on(void); - -void dn_dev_init_pkt(struct sk_buff *skb); -void dn_dev_veri_pkt(struct sk_buff *skb); -void dn_dev_hello(struct sk_buff *skb); - -void dn_dev_up(struct net_device *); -void dn_dev_down(struct net_device *); - -int dn_dev_set_default(struct net_device *dev, int force); -struct net_device *dn_dev_get_default(void); -int dn_dev_bind_default(__le16 *addr); - -int register_dnaddr_notifier(struct notifier_block *nb); -int unregister_dnaddr_notifier(struct notifier_block *nb); - -static inline int dn_dev_islocal(struct net_device *dev, __le16 addr) -{ - struct dn_dev *dn_db; - struct dn_ifaddr *ifa; - int res = 0; - - rcu_read_lock(); - dn_db = rcu_dereference(dev->dn_ptr); - if (dn_db == NULL) { - printk(KERN_DEBUG "dn_dev_islocal: Called for non DECnet device\n"); - goto out; - } - - for (ifa = rcu_dereference(dn_db->ifa_list); - ifa != NULL; - ifa = rcu_dereference(ifa->ifa_next)) - if ((addr ^ ifa->ifa_local) == 0) { - res = 1; - break; - } -out: - rcu_read_unlock(); - return res; -} - -#endif /* _NET_DN_DEV_H */ diff --git a/include/net/dn_fib.h b/include/net/dn_fib.h deleted file mode 100644 index 1929a3cd5ebe9cabb5f72073d532de7eafa6dfce..0000000000000000000000000000000000000000 --- a/include/net/dn_fib.h +++ /dev/null @@ -1,169 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _NET_DN_FIB_H -#define _NET_DN_FIB_H - -#include -#include -#include -#include - -extern const struct nla_policy rtm_dn_policy[]; - -struct dn_fib_res { - struct fib_rule *r; - struct dn_fib_info *fi; - unsigned char prefixlen; - unsigned char nh_sel; - unsigned char type; - unsigned char scope; -}; - -struct dn_fib_nh { - struct net_device *nh_dev; - unsigned int nh_flags; - unsigned char nh_scope; - int nh_weight; - int nh_power; - int nh_oif; - __le16 nh_gw; -}; - -struct dn_fib_info { - struct dn_fib_info *fib_next; - struct dn_fib_info *fib_prev; - refcount_t fib_treeref; - refcount_t fib_clntref; - int fib_dead; - unsigned int fib_flags; - int fib_protocol; - __le16 fib_prefsrc; - __u32 fib_priority; - __u32 fib_metrics[RTAX_MAX]; - int fib_nhs; - int fib_power; - struct dn_fib_nh fib_nh[0]; -#define dn_fib_dev fib_nh[0].nh_dev -}; - - -#define DN_FIB_RES_RESET(res) ((res).nh_sel = 0) -#define DN_FIB_RES_NH(res) ((res).fi->fib_nh[(res).nh_sel]) - -#define DN_FIB_RES_PREFSRC(res) ((res).fi->fib_prefsrc ? : __dn_fib_res_prefsrc(&res)) -#define DN_FIB_RES_GW(res) (DN_FIB_RES_NH(res).nh_gw) -#define DN_FIB_RES_DEV(res) (DN_FIB_RES_NH(res).nh_dev) -#define DN_FIB_RES_OIF(res) (DN_FIB_RES_NH(res).nh_oif) - -typedef struct { - __le16 datum; -} dn_fib_key_t; - -typedef struct { - __le16 datum; -} dn_fib_hash_t; - -typedef struct { - __u16 datum; -} dn_fib_idx_t; - -struct dn_fib_node { - struct dn_fib_node *fn_next; - struct dn_fib_info *fn_info; -#define DN_FIB_INFO(f) ((f)->fn_info) - dn_fib_key_t fn_key; - u8 fn_type; - u8 fn_scope; - u8 fn_state; -}; - - -struct dn_fib_table { - struct hlist_node hlist; - u32 n; - - int (*insert)(struct dn_fib_table *t, struct rtmsg *r, - struct nlattr *attrs[], struct nlmsghdr *n, - struct netlink_skb_parms *req); - int (*delete)(struct dn_fib_table *t, struct rtmsg *r, - struct nlattr *attrs[], struct nlmsghdr *n, - struct netlink_skb_parms *req); - int (*lookup)(struct dn_fib_table *t, const struct flowidn *fld, - struct dn_fib_res *res); - int (*flush)(struct dn_fib_table *t); - int (*dump)(struct dn_fib_table *t, struct sk_buff *skb, struct netlink_callback *cb); - - unsigned char data[]; -}; - -#ifdef CONFIG_DECNET_ROUTER -/* - * dn_fib.c - */ -void dn_fib_init(void); -void dn_fib_cleanup(void); - -int dn_fib_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg); -struct dn_fib_info *dn_fib_create_info(const struct rtmsg *r, - struct nlattr *attrs[], - const struct nlmsghdr *nlh, int *errp); -int dn_fib_semantic_match(int type, struct dn_fib_info *fi, - const struct flowidn *fld, struct dn_fib_res *res); -void dn_fib_release_info(struct dn_fib_info *fi); -void dn_fib_flush(void); -void dn_fib_select_multipath(const struct flowidn *fld, struct dn_fib_res *res); - -/* - * dn_tables.c - */ -struct dn_fib_table *dn_fib_get_table(u32 n, int creat); -struct dn_fib_table *dn_fib_empty_table(void); -void dn_fib_table_init(void); -void dn_fib_table_cleanup(void); - -/* - * dn_rules.c - */ -void dn_fib_rules_init(void); -void dn_fib_rules_cleanup(void); -unsigned int dnet_addr_type(__le16 addr); -int dn_fib_lookup(struct flowidn *fld, struct dn_fib_res *res); - -int dn_fib_dump(struct sk_buff *skb, struct netlink_callback *cb); - -void dn_fib_free_info(struct dn_fib_info *fi); - -static inline void dn_fib_info_put(struct dn_fib_info *fi) -{ - if (refcount_dec_and_test(&fi->fib_clntref)) - dn_fib_free_info(fi); -} - -static inline void dn_fib_res_put(struct dn_fib_res *res) -{ - if (res->fi) - dn_fib_info_put(res->fi); - if (res->r) - fib_rule_put(res->r); -} - -#else /* Endnode */ - -#define dn_fib_init() do { } while(0) -#define dn_fib_cleanup() do { } while(0) - -#define dn_fib_lookup(fl, res) (-ESRCH) -#define dn_fib_info_put(fi) do { } while(0) -#define dn_fib_select_multipath(fl, res) do { } while(0) -#define dn_fib_rules_policy(saddr,res,flags) (0) -#define dn_fib_res_put(res) do { } while(0) - -#endif /* CONFIG_DECNET_ROUTER */ - -static inline __le16 dnet_make_mask(int n) -{ - if (n) - return cpu_to_le16(~((1 << (16 - n)) - 1)); - return cpu_to_le16(0); -} - -#endif /* _NET_DN_FIB_H */ diff --git a/include/net/dn_neigh.h b/include/net/dn_neigh.h deleted file mode 100644 index 1f7df98bfc33027e3ac1d68c91b241ec4480edca..0000000000000000000000000000000000000000 --- a/include/net/dn_neigh.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _NET_DN_NEIGH_H -#define _NET_DN_NEIGH_H - -#include - -/* - * The position of the first two fields of - * this structure are critical - SJW - */ -struct dn_neigh { - struct neighbour n; - __le16 addr; - unsigned long flags; -#define DN_NDFLAG_R1 0x0001 /* Router L1 */ -#define DN_NDFLAG_R2 0x0002 /* Router L2 */ -#define DN_NDFLAG_P3 0x0004 /* Phase III Node */ - unsigned long blksize; - __u8 priority; -}; - -void dn_neigh_init(void); -void dn_neigh_cleanup(void); -int dn_neigh_router_hello(struct net *net, struct sock *sk, struct sk_buff *skb); -int dn_neigh_endnode_hello(struct net *net, struct sock *sk, struct sk_buff *skb); -void dn_neigh_pointopoint_hello(struct sk_buff *skb); -int dn_neigh_elist(struct net_device *dev, unsigned char *ptr, int n); -int dn_to_neigh_output(struct net *net, struct sock *sk, struct sk_buff *skb); - -extern struct neigh_table dn_neigh_table; - -#endif /* _NET_DN_NEIGH_H */ diff --git a/include/net/dn_nsp.h b/include/net/dn_nsp.h deleted file mode 100644 index a4a18fee0b7c07b1eb6af90fdfb1638f0c5378e7..0000000000000000000000000000000000000000 --- a/include/net/dn_nsp.h +++ /dev/null @@ -1,201 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _NET_DN_NSP_H -#define _NET_DN_NSP_H -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - -*******************************************************************************/ -/* dn_nsp.c functions prototyping */ -#include -#include -#include - -struct sk_buff; -struct sk_buff_head; - -void dn_nsp_send_data_ack(struct sock *sk); -void dn_nsp_send_oth_ack(struct sock *sk); -void dn_send_conn_ack(struct sock *sk); -void dn_send_conn_conf(struct sock *sk, gfp_t gfp); -void dn_nsp_send_disc(struct sock *sk, unsigned char type, - unsigned short reason, gfp_t gfp); -void dn_nsp_return_disc(struct sk_buff *skb, unsigned char type, - unsigned short reason); -void dn_nsp_send_link(struct sock *sk, unsigned char lsflags, char fcval); -void dn_nsp_send_conninit(struct sock *sk, unsigned char flags); - -void dn_nsp_output(struct sock *sk); -int dn_nsp_check_xmit_queue(struct sock *sk, struct sk_buff *skb, - struct sk_buff_head *q, unsigned short acknum); -void dn_nsp_queue_xmit(struct sock *sk, struct sk_buff *skb, gfp_t gfp, - int oob); -unsigned long dn_nsp_persist(struct sock *sk); -int dn_nsp_xmit_timeout(struct sock *sk); - -int dn_nsp_rx(struct sk_buff *); -int dn_nsp_backlog_rcv(struct sock *sk, struct sk_buff *skb); - -struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri); -struct sk_buff *dn_alloc_send_skb(struct sock *sk, size_t *size, int noblock, - long timeo, int *err); - -#define NSP_REASON_OK 0 /* No error */ -#define NSP_REASON_NR 1 /* No resources */ -#define NSP_REASON_UN 2 /* Unrecognised node name */ -#define NSP_REASON_SD 3 /* Node shutting down */ -#define NSP_REASON_ID 4 /* Invalid destination end user */ -#define NSP_REASON_ER 5 /* End user lacks resources */ -#define NSP_REASON_OB 6 /* Object too busy */ -#define NSP_REASON_US 7 /* Unspecified error */ -#define NSP_REASON_TP 8 /* Third-Party abort */ -#define NSP_REASON_EA 9 /* End user has aborted the link */ -#define NSP_REASON_IF 10 /* Invalid node name format */ -#define NSP_REASON_LS 11 /* Local node shutdown */ -#define NSP_REASON_LL 32 /* Node lacks logical-link resources */ -#define NSP_REASON_LE 33 /* End user lacks logical-link resources */ -#define NSP_REASON_UR 34 /* Unacceptable RQSTRID or PASSWORD field */ -#define NSP_REASON_UA 36 /* Unacceptable ACCOUNT field */ -#define NSP_REASON_TM 38 /* End user timed out logical link */ -#define NSP_REASON_NU 39 /* Node unreachable */ -#define NSP_REASON_NL 41 /* No-link message */ -#define NSP_REASON_DC 42 /* Disconnect confirm */ -#define NSP_REASON_IO 43 /* Image data field overflow */ - -#define NSP_DISCINIT 0x38 -#define NSP_DISCCONF 0x48 - -/*------------------------- NSP - messages ------------------------------*/ -/* Data Messages */ -/*---------------*/ - -/* Data Messages (data segment/interrupt/link service) */ - -struct nsp_data_seg_msg { - __u8 msgflg; - __le16 dstaddr; - __le16 srcaddr; -} __packed; - -struct nsp_data_opt_msg { - __le16 acknum; - __le16 segnum; - __le16 lsflgs; -} __packed; - -struct nsp_data_opt_msg1 { - __le16 acknum; - __le16 segnum; -} __packed; - - -/* Acknowledgment Message (data/other data) */ -struct nsp_data_ack_msg { - __u8 msgflg; - __le16 dstaddr; - __le16 srcaddr; - __le16 acknum; -} __packed; - -/* Connect Acknowledgment Message */ -struct nsp_conn_ack_msg { - __u8 msgflg; - __le16 dstaddr; -} __packed; - - -/* Connect Initiate/Retransmit Initiate/Connect Confirm */ -struct nsp_conn_init_msg { - __u8 msgflg; -#define NSP_CI 0x18 /* Connect Initiate */ -#define NSP_RCI 0x68 /* Retrans. Conn Init */ - __le16 dstaddr; - __le16 srcaddr; - __u8 services; -#define NSP_FC_NONE 0x00 /* Flow Control None */ -#define NSP_FC_SRC 0x04 /* Seg Req. Count */ -#define NSP_FC_SCMC 0x08 /* Sess. Control Mess */ -#define NSP_FC_MASK 0x0c /* FC type mask */ - __u8 info; - __le16 segsize; -} __packed; - -/* Disconnect Initiate/Disconnect Confirm */ -struct nsp_disconn_init_msg { - __u8 msgflg; - __le16 dstaddr; - __le16 srcaddr; - __le16 reason; -} __packed; - - - -struct srcobj_fmt { - __u8 format; - __u8 task; - __le16 grpcode; - __le16 usrcode; - __u8 dlen; -} __packed; - -/* - * A collection of functions for manipulating the sequence - * numbers used in NSP. Similar in operation to the functions - * of the same name in TCP. - */ -static __inline__ int dn_before(__u16 seq1, __u16 seq2) -{ - seq1 &= 0x0fff; - seq2 &= 0x0fff; - - return (int)((seq1 - seq2) & 0x0fff) > 2048; -} - - -static __inline__ int dn_after(__u16 seq1, __u16 seq2) -{ - seq1 &= 0x0fff; - seq2 &= 0x0fff; - - return (int)((seq2 - seq1) & 0x0fff) > 2048; -} - -static __inline__ int dn_equal(__u16 seq1, __u16 seq2) -{ - return ((seq1 ^ seq2) & 0x0fff) == 0; -} - -static __inline__ int dn_before_or_equal(__u16 seq1, __u16 seq2) -{ - return (dn_before(seq1, seq2) || dn_equal(seq1, seq2)); -} - -static __inline__ void seq_add(__u16 *seq, __u16 off) -{ - (*seq) += off; - (*seq) &= 0x0fff; -} - -static __inline__ int seq_next(__u16 seq1, __u16 seq2) -{ - return dn_equal(seq1 + 1, seq2); -} - -/* - * Can we delay the ack ? - */ -static __inline__ int sendack(__u16 seq) -{ - return (int)((seq & 0x1000) ? 0 : 1); -} - -/* - * Is socket congested ? - */ -static __inline__ int dn_congested(struct sock *sk) -{ - return atomic_read(&sk->sk_rmem_alloc) > (sk->sk_rcvbuf >> 1); -} - -#define DN_MAX_NSP_DATA_HEADER (11) - -#endif /* _NET_DN_NSP_H */ diff --git a/include/net/dn_route.h b/include/net/dn_route.h deleted file mode 100644 index 88c0300236cc246f2434a1cbd5bdd89d455d4b54..0000000000000000000000000000000000000000 --- a/include/net/dn_route.h +++ /dev/null @@ -1,118 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _NET_DN_ROUTE_H -#define _NET_DN_ROUTE_H - -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - -*******************************************************************************/ - -#include -#include - -struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri); -int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *, - struct sock *sk, int flags); -int dn_cache_dump(struct sk_buff *skb, struct netlink_callback *cb); -void dn_rt_cache_flush(int delay); -int dn_route_rcv(struct sk_buff *skb, struct net_device *dev, - struct packet_type *pt, struct net_device *orig_dev); - -/* Masks for flags field */ -#define DN_RT_F_PID 0x07 /* Mask for packet type */ -#define DN_RT_F_PF 0x80 /* Padding Follows */ -#define DN_RT_F_VER 0x40 /* Version =0 discard packet if ==1 */ -#define DN_RT_F_IE 0x20 /* Intra Ethernet, Reserved in short pkt */ -#define DN_RT_F_RTS 0x10 /* Packet is being returned to sender */ -#define DN_RT_F_RQR 0x08 /* Return packet to sender upon non-delivery */ - -/* Mask for types of routing packets */ -#define DN_RT_PKT_MSK 0x06 -/* Types of routing packets */ -#define DN_RT_PKT_SHORT 0x02 /* Short routing packet */ -#define DN_RT_PKT_LONG 0x06 /* Long routing packet */ - -/* Mask for control/routing selection */ -#define DN_RT_PKT_CNTL 0x01 /* Set to 1 if a control packet */ -/* Types of control packets */ -#define DN_RT_CNTL_MSK 0x0f /* Mask for control packets */ -#define DN_RT_PKT_INIT 0x01 /* Initialisation packet */ -#define DN_RT_PKT_VERI 0x03 /* Verification Message */ -#define DN_RT_PKT_HELO 0x05 /* Hello and Test Message */ -#define DN_RT_PKT_L1RT 0x07 /* Level 1 Routing Message */ -#define DN_RT_PKT_L2RT 0x09 /* Level 2 Routing Message */ -#define DN_RT_PKT_ERTH 0x0b /* Ethernet Router Hello */ -#define DN_RT_PKT_EEDH 0x0d /* Ethernet EndNode Hello */ - -/* Values for info field in hello message */ -#define DN_RT_INFO_TYPE 0x03 /* Type mask */ -#define DN_RT_INFO_L1RT 0x02 /* L1 Router */ -#define DN_RT_INFO_L2RT 0x01 /* L2 Router */ -#define DN_RT_INFO_ENDN 0x03 /* EndNode */ -#define DN_RT_INFO_VERI 0x04 /* Verification Reqd. */ -#define DN_RT_INFO_RJCT 0x08 /* Reject Flag, Reserved */ -#define DN_RT_INFO_VFLD 0x10 /* Verification Failed, Reserved */ -#define DN_RT_INFO_NOML 0x20 /* No Multicast traffic accepted */ -#define DN_RT_INFO_BLKR 0x40 /* Blocking Requested */ - -/* - * The fl structure is what we used to look up the route. - * The rt_saddr & rt_daddr entries are the same as key.saddr & key.daddr - * except for local input routes, where the rt_saddr = fl.fld_dst and - * rt_daddr = fl.fld_src to allow the route to be used for returning - * packets to the originating host. - */ -struct dn_route { - struct dst_entry dst; - struct dn_route __rcu *dn_next; - - struct neighbour *n; - - struct flowidn fld; - - __le16 rt_saddr; - __le16 rt_daddr; - __le16 rt_gateway; - __le16 rt_local_src; /* Source used for forwarding packets */ - __le16 rt_src_map; - __le16 rt_dst_map; - - unsigned int rt_flags; - unsigned int rt_type; -}; - -static inline bool dn_is_input_route(struct dn_route *rt) -{ - return rt->fld.flowidn_iif != 0; -} - -static inline bool dn_is_output_route(struct dn_route *rt) -{ - return rt->fld.flowidn_iif == 0; -} - -void dn_route_init(void); -void dn_route_cleanup(void); - -#include -#include - -static inline void dn_rt_send(struct sk_buff *skb) -{ - dev_queue_xmit(skb); -} - -static inline void dn_rt_finish_output(struct sk_buff *skb, char *dst, char *src) -{ - struct net_device *dev = skb->dev; - - if ((dev->type != ARPHRD_ETHER) && (dev->type != ARPHRD_LOOPBACK)) - dst = NULL; - - if (dev_hard_header(skb, dev, ETH_P_DNA_RT, dst, src, skb->len) >= 0) - dn_rt_send(skb); - else - kfree_skb(skb); -} - -#endif /* _NET_DN_ROUTE_H */ diff --git a/include/net/dropreason.h b/include/net/dropreason.h index fae9b40e54fae5f355ca6751484a767695549561..c1cbcdbaf1492c6bd5a18b9af0a0e0beb071c674 100644 --- a/include/net/dropreason.h +++ b/include/net/dropreason.h @@ -3,6 +3,73 @@ #ifndef _LINUX_DROPREASON_H #define _LINUX_DROPREASON_H +#define DEFINE_DROP_REASON(FN, FNe) \ + FN(NOT_SPECIFIED) \ + FN(NO_SOCKET) \ + FN(PKT_TOO_SMALL) \ + FN(TCP_CSUM) \ + FN(SOCKET_FILTER) \ + FN(UDP_CSUM) \ + FN(NETFILTER_DROP) \ + FN(OTHERHOST) \ + FN(IP_CSUM) \ + FN(IP_INHDR) \ + FN(IP_RPFILTER) \ + FN(UNICAST_IN_L2_MULTICAST) \ + FN(XFRM_POLICY) \ + FN(IP_NOPROTO) \ + FN(SOCKET_RCVBUFF) \ + FN(PROTO_MEM) \ + FN(TCP_MD5NOTFOUND) \ + FN(TCP_MD5UNEXPECTED) \ + FN(TCP_MD5FAILURE) \ + FN(SOCKET_BACKLOG) \ + FN(TCP_FLAGS) \ + FN(TCP_ZEROWINDOW) \ + FN(TCP_OLD_DATA) \ + FN(TCP_OVERWINDOW) \ + FN(TCP_OFOMERGE) \ + FN(TCP_RFC7323_PAWS) \ + FN(TCP_INVALID_SEQUENCE) \ + FN(TCP_RESET) \ + FN(TCP_INVALID_SYN) \ + FN(TCP_CLOSE) \ + FN(TCP_FASTOPEN) \ + FN(TCP_OLD_ACK) \ + FN(TCP_TOO_OLD_ACK) \ + FN(TCP_ACK_UNSENT_DATA) \ + FN(TCP_OFO_QUEUE_PRUNE) \ + FN(TCP_OFO_DROP) \ + FN(IP_OUTNOROUTES) \ + FN(BPF_CGROUP_EGRESS) \ + FN(IPV6DISABLED) \ + FN(NEIGH_CREATEFAIL) \ + FN(NEIGH_FAILED) \ + FN(NEIGH_QUEUEFULL) \ + FN(NEIGH_DEAD) \ + FN(TC_EGRESS) \ + FN(QDISC_DROP) \ + FN(CPU_BACKLOG) \ + FN(XDP) \ + FN(TC_INGRESS) \ + FN(UNHANDLED_PROTO) \ + FN(SKB_CSUM) \ + FN(SKB_GSO_SEG) \ + FN(SKB_UCOPY_FAULT) \ + FN(DEV_HDR) \ + FN(DEV_READY) \ + FN(FULL_RING) \ + FN(NOMEM) \ + FN(HDR_TRUNC) \ + FN(TAP_FILTER) \ + FN(TAP_TXFILTER) \ + FN(ICMP_CSUM) \ + FN(INVALID_PROTO) \ + FN(IP_INADDRERRORS) \ + FN(IP_INNOROUTES) \ + FN(PKT_TOO_BIG) \ + FNe(MAX) + /** * enum skb_drop_reason - the reasons of skb drops * diff --git a/include/net/dsa.h b/include/net/dsa.h index b902b31bebcefe0f3b0bcb4282d9609812e4e65c..ee369670e20e418bd5170b192257acb96ebf2f55 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -294,12 +294,13 @@ struct dsa_port { u8 lag_tx_enabled:1; - u8 devlink_port_setup:1; - /* Master state bits, valid only on CPU ports */ u8 master_admin_up:1; u8 master_oper_up:1; + /* Valid only on user ports */ + u8 cpu_port_in_lag:1; + u8 setup:1; struct device_node *dn; @@ -559,6 +560,14 @@ static inline bool dsa_is_user_port(struct dsa_switch *ds, int p) list_for_each_entry((_dp), &(_dst)->ports, list) \ if (dsa_port_is_user((_dp))) +#define dsa_tree_for_each_user_port_continue_reverse(_dp, _dst) \ + list_for_each_entry_continue_reverse((_dp), &(_dst)->ports, list) \ + if (dsa_port_is_user((_dp))) + +#define dsa_tree_for_each_cpu_port(_dp, _dst) \ + list_for_each_entry((_dp), &(_dst)->ports, list) \ + if (dsa_port_is_cpu((_dp))) + #define dsa_switch_for_each_port(_dp, _ds) \ list_for_each_entry((_dp), &(_ds)->dst->ports, list) \ if ((_dp)->ds == (_ds)) @@ -714,6 +723,14 @@ static inline bool dsa_port_offloads_lag(struct dsa_port *dp, return dsa_port_lag_dev_get(dp) == lag->dev; } +static inline struct net_device *dsa_port_to_master(const struct dsa_port *dp) +{ + if (dp->cpu_port_in_lag) + return dsa_port_lag_dev_get(dp->cpu_dp); + + return dp->cpu_dp->master; +} + static inline struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp) { @@ -798,6 +815,12 @@ dsa_tree_offloads_bridge_dev(struct dsa_switch_tree *dst, return false; } +static inline bool dsa_port_tree_same(const struct dsa_port *a, + const struct dsa_port *b) +{ + return a->ds->dst == b->ds->dst; +} + typedef int dsa_fdb_dump_cb_t(const unsigned char *addr, u16 vid, bool is_static, void *data); struct dsa_switch_ops { @@ -821,6 +844,10 @@ struct dsa_switch_ops { int (*connect_tag_protocol)(struct dsa_switch *ds, enum dsa_tag_protocol proto); + int (*port_change_master)(struct dsa_switch *ds, int port, + struct net_device *master, + struct netlink_ext_ack *extack); + /* Optional switch-wide initialization and destruction methods */ int (*setup)(struct dsa_switch *ds); void (*teardown)(struct dsa_switch *ds); @@ -1077,7 +1104,8 @@ struct dsa_switch_ops { int port); int (*crosschip_lag_join)(struct dsa_switch *ds, int sw_index, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info); + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack); int (*crosschip_lag_leave)(struct dsa_switch *ds, int sw_index, int port, struct dsa_lag lag); @@ -1152,7 +1180,8 @@ struct dsa_switch_ops { int (*port_lag_change)(struct dsa_switch *ds, int port); int (*port_lag_join)(struct dsa_switch *ds, int port, struct dsa_lag lag, - struct netdev_lag_upper_info *info); + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack); int (*port_lag_leave)(struct dsa_switch *ds, int port, struct dsa_lag lag); diff --git a/include/net/dst.h b/include/net/dst.h index 6aa252c3fc55ccaee58faebf265510469e91d780..00b479ce6b99c9b329a30f8201882a51c9a135a9 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -239,12 +239,6 @@ static inline void dst_use_noref(struct dst_entry *dst, unsigned long time) } } -static inline void dst_hold_and_use(struct dst_entry *dst, unsigned long time) -{ - dst_hold(dst); - dst_use_noref(dst, time); -} - static inline struct dst_entry *dst_clone(struct dst_entry *dst) { if (dst) diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h index adab27ba1ecbf054f5284e4342b1551b104d9a2c..a454cf4327feb419100da9284d46515680051a91 100644 --- a/include/net/dst_metadata.h +++ b/include/net/dst_metadata.h @@ -4,11 +4,14 @@ #include #include +#include #include enum metadata_type { METADATA_IP_TUNNEL, METADATA_HW_PORT_MUX, + METADATA_MACSEC, + METADATA_XFRM, }; struct hw_port_info { @@ -16,12 +19,23 @@ struct hw_port_info { u32 port_id; }; +struct macsec_info { + sci_t sci; +}; + +struct xfrm_md_info { + u32 if_id; + int link; +}; + struct metadata_dst { struct dst_entry dst; enum metadata_type type; union { struct ip_tunnel_info tun_info; struct hw_port_info port_info; + struct macsec_info macsec_info; + struct xfrm_md_info xfrm_info; } u; }; @@ -53,6 +67,27 @@ skb_tunnel_info(const struct sk_buff *skb) return NULL; } +static inline struct xfrm_md_info *lwt_xfrm_info(struct lwtunnel_state *lwt) +{ + return (struct xfrm_md_info *)lwt->data; +} + +static inline struct xfrm_md_info *skb_xfrm_md_info(const struct sk_buff *skb) +{ + struct metadata_dst *md_dst = skb_metadata_dst(skb); + struct dst_entry *dst; + + if (md_dst && md_dst->type == METADATA_XFRM) + return &md_dst->u.xfrm_info; + + dst = skb_dst(skb); + if (dst && dst->lwtstate && + dst->lwtstate->type == LWTUNNEL_ENCAP_XFRM) + return lwt_xfrm_info(dst->lwtstate); + + return NULL; +} + static inline bool skb_valid_dst(const struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); @@ -82,6 +117,12 @@ static inline int skb_metadata_dst_cmp(const struct sk_buff *skb_a, return memcmp(&a->u.tun_info, &b->u.tun_info, sizeof(a->u.tun_info) + a->u.tun_info.options_len); + case METADATA_MACSEC: + return memcmp(&a->u.macsec_info, &b->u.macsec_info, + sizeof(a->u.macsec_info)); + case METADATA_XFRM: + return memcmp(&a->u.xfrm_info, &b->u.xfrm_info, + sizeof(a->u.xfrm_info)); default: return 1; } diff --git a/include/net/flow.h b/include/net/flow.h index 987bd511d6524288deaede100cfc155048d6071f..2f0da4f0318b580051c0fec2b98f745190c62cce 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -54,11 +54,6 @@ union flowi_uli { __u8 code; } icmpt; - struct { - __le16 dport; - __le16 sport; - } dnports; - __be32 gre_key; struct { @@ -156,27 +151,11 @@ struct flowi6 { __u32 mp_hash; } __attribute__((__aligned__(BITS_PER_LONG/8))); -struct flowidn { - struct flowi_common __fl_common; -#define flowidn_oif __fl_common.flowic_oif -#define flowidn_iif __fl_common.flowic_iif -#define flowidn_mark __fl_common.flowic_mark -#define flowidn_scope __fl_common.flowic_scope -#define flowidn_proto __fl_common.flowic_proto -#define flowidn_flags __fl_common.flowic_flags - __le16 daddr; - __le16 saddr; - union flowi_uli uli; -#define fld_sport uli.ports.sport -#define fld_dport uli.ports.dport -} __attribute__((__aligned__(BITS_PER_LONG/8))); - struct flowi { union { struct flowi_common __fl_common; struct flowi4 ip4; struct flowi6 ip6; - struct flowidn dn; } u; #define flowi_oif u.__fl_common.flowic_oif #define flowi_iif u.__fl_common.flowic_iif @@ -211,11 +190,6 @@ static inline struct flowi_common *flowi6_to_flowi_common(struct flowi6 *fl6) return &(fl6->__fl_common); } -static inline struct flowi *flowidn_to_flowi(struct flowidn *fldn) -{ - return container_of(fldn, struct flowi, u.dn); -} - __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys); #endif diff --git a/include/net/flow_dissector.h b/include/net/flow_dissector.h index 6c74812d64b2629c3ade83ce520408a78d7e2510..5ccf52ef8809988e39395e9801911764007354ed 100644 --- a/include/net/flow_dissector.h +++ b/include/net/flow_dissector.h @@ -289,6 +289,14 @@ struct flow_dissector_key_pppoe { __be16 type; }; +/** + * struct flow_dissector_key_l2tpv3: + * @session_id: identifier for a l2tp session + */ +struct flow_dissector_key_l2tpv3 { + __be32 session_id; +}; + enum flow_dissector_key_id { FLOW_DISSECTOR_KEY_CONTROL, /* struct flow_dissector_key_control */ FLOW_DISSECTOR_KEY_BASIC, /* struct flow_dissector_key_basic */ @@ -320,6 +328,7 @@ enum flow_dissector_key_id { FLOW_DISSECTOR_KEY_HASH, /* struct flow_dissector_key_hash */ FLOW_DISSECTOR_KEY_NUM_OF_VLANS, /* struct flow_dissector_key_num_of_vlans */ FLOW_DISSECTOR_KEY_PPPOE, /* struct flow_dissector_key_pppoe */ + FLOW_DISSECTOR_KEY_L2TPV3, /* struct flow_dissector_key_l2tpv3 */ FLOW_DISSECTOR_KEY_MAX, }; diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h index 2a9a9e42e7fd347d23fb6e241671c0837e8edfcb..e343f9f8363e3ebcc03074809ff46cb8737f20fb 100644 --- a/include/net/flow_offload.h +++ b/include/net/flow_offload.h @@ -80,6 +80,10 @@ struct flow_match_pppoe { struct flow_dissector_key_pppoe *key, *mask; }; +struct flow_match_l2tpv3 { + struct flow_dissector_key_l2tpv3 *key, *mask; +}; + struct flow_rule; void flow_rule_match_meta(const struct flow_rule *rule, @@ -128,6 +132,8 @@ void flow_rule_match_ct(const struct flow_rule *rule, struct flow_match_ct *out); void flow_rule_match_pppoe(const struct flow_rule *rule, struct flow_match_pppoe *out); +void flow_rule_match_l2tpv3(const struct flow_rule *rule, + struct flow_match_l2tpv3 *out); enum flow_action_id { FLOW_ACTION_ACCEPT = 0, diff --git a/include/net/genetlink.h b/include/net/genetlink.h index 56a50e1c51b97d1b6aebeb1b594df78485f38d8b..8f780170e2f87fea3b5c7deaa48d5008d3f78e7e 100644 --- a/include/net/genetlink.h +++ b/include/net/genetlink.h @@ -39,6 +39,8 @@ struct genl_info; * undo operations done by pre_doit, for example release locks * @mcgrps: multicast groups used by this family * @n_mcgrps: number of multicast groups + * @resv_start_op: first operation for which reserved fields of the header + * can be validated, new families should leave this field at zero * @mcgrp_offset: starting number of multicast group IDs in this family * (private) * @ops: the operations supported by this family @@ -58,6 +60,7 @@ struct genl_family { u8 n_ops; u8 n_small_ops; u8 n_mcgrps; + u8 resv_start_op; const struct nla_policy *policy; int (*pre_doit)(const struct genl_ops *ops, struct sk_buff *skb, @@ -107,6 +110,13 @@ static inline void genl_info_net_set(struct genl_info *info, struct net *net) #define GENL_SET_ERR_MSG(info, msg) NL_SET_ERR_MSG((info)->extack, msg) +/* Report that a root attribute is missing */ +#define GENL_REQ_ATTR_CHECK(info, attr) ({ \ + struct genl_info *__info = (info); \ + \ + NL_REQ_ATTR_CHECK(__info->extack, NULL, __info->attrs, (attr)); \ +}) + enum genl_validate_flags { GENL_DONT_VALIDATE_STRICT = BIT(0), GENL_DONT_VALIDATE_DUMP = BIT(1), diff --git a/include/net/gro.h b/include/net/gro.h index 867656b0739c0985cae50d1aaf2e1d70bb1c80ed..a4fab706240d20aee6f46055d5bbaeb397e6ba5c 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -160,6 +160,17 @@ static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, return skb->data + offset; } +static inline void *skb_gro_header(struct sk_buff *skb, + unsigned int hlen, unsigned int offset) +{ + void *ptr; + + ptr = skb_gro_header_fast(skb, offset); + if (skb_gro_header_hard(skb, hlen)) + ptr = skb_gro_header_slow(skb, hlen, offset); + return ptr; +} + static inline void *skb_gro_network_header(struct sk_buff *skb) { return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + @@ -301,12 +312,9 @@ static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, return ptr; } - ptr = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, off + plen)) { - ptr = skb_gro_header_slow(skb, off + plen, off); - if (!ptr) - return NULL; - } + ptr = skb_gro_header(skb, off + plen, off); + if (!ptr) + return NULL; delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, start, offset); @@ -329,12 +337,9 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, if (!grc->delta) return; - ptr = skb_gro_header_fast(skb, grc->offset); - if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { - ptr = skb_gro_header_slow(skb, plen, grc->offset); - if (!ptr) - return; - } + ptr = skb_gro_header(skb, plen, grc->offset); + if (!ptr) + return; remcsum_unadjust((__sum16 *)ptr, grc->delta); } @@ -405,9 +410,7 @@ static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) off = skb_gro_offset(skb); hlen = off + sizeof(*uh); - uh = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) - uh = skb_gro_header_slow(skb, hlen, off); + uh = skb_gro_header(skb, hlen, off); return uh; } @@ -439,7 +442,7 @@ static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, { list_add_tail(&skb->list, &napi->rx_list); napi->rx_count += segs; - if (napi->rx_count >= gro_normal_batch) + if (napi->rx_count >= READ_ONCE(gro_normal_batch)) gro_normal_list(napi); } diff --git a/include/net/ieee802154_netdev.h b/include/net/ieee802154_netdev.h index d0d188c3294bd68eeaed992f06a5e20ba33a7648..03b64bf876a4654f0466744fac5fb4b0fb52f4cf 100644 --- a/include/net/ieee802154_netdev.h +++ b/include/net/ieee802154_netdev.h @@ -15,6 +15,22 @@ #ifndef IEEE802154_NETDEVICE_H #define IEEE802154_NETDEVICE_H +#define IEEE802154_REQUIRED_SIZE(struct_type, member) \ + (offsetof(typeof(struct_type), member) + \ + sizeof(((typeof(struct_type) *)(NULL))->member)) + +#define IEEE802154_ADDR_OFFSET \ + offsetof(typeof(struct sockaddr_ieee802154), addr) + +#define IEEE802154_MIN_NAMELEN (IEEE802154_ADDR_OFFSET + \ + IEEE802154_REQUIRED_SIZE(struct ieee802154_addr_sa, addr_type)) + +#define IEEE802154_NAMELEN_SHORT (IEEE802154_ADDR_OFFSET + \ + IEEE802154_REQUIRED_SIZE(struct ieee802154_addr_sa, short_addr)) + +#define IEEE802154_NAMELEN_LONG (IEEE802154_ADDR_OFFSET + \ + IEEE802154_REQUIRED_SIZE(struct ieee802154_addr_sa, hwaddr)) + #include #include #include @@ -165,6 +181,33 @@ static inline void ieee802154_devaddr_to_raw(void *raw, __le64 addr) memcpy(raw, &temp, IEEE802154_ADDR_LEN); } +static inline int +ieee802154_sockaddr_check_size(struct sockaddr_ieee802154 *daddr, int len) +{ + struct ieee802154_addr_sa *sa; + int ret = 0; + + sa = &daddr->addr; + if (len < IEEE802154_MIN_NAMELEN) + return -EINVAL; + switch (sa->addr_type) { + case IEEE802154_ADDR_NONE: + break; + case IEEE802154_ADDR_SHORT: + if (len < IEEE802154_NAMELEN_SHORT) + ret = -EINVAL; + break; + case IEEE802154_ADDR_LONG: + if (len < IEEE802154_NAMELEN_LONG) + ret = -EINVAL; + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + static inline void ieee802154_addr_from_sa(struct ieee802154_addr *a, const struct ieee802154_addr_sa *sa) { diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index ee88f0f1350f4e1d8a9c3daea7c7bf8310b2fa62..c2b15f7e551617b06863a3b52056348d9c53bb12 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -25,6 +25,7 @@ #undef INET_CSK_CLEAR_TIMERS struct inet_bind_bucket; +struct inet_bind2_bucket; struct tcp_congestion_ops; /* @@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops { * * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node + * @icsk_bind2_hash: Bind node in the bhash2 table * @icsk_timeout: Timeout * @icsk_retransmit_timer: Resend (no ack) * @icsk_rto: Retransmit timeout @@ -83,6 +85,7 @@ struct inet_connection_sock { struct inet_sock icsk_inet; struct request_sock_queue icsk_accept_queue; struct inet_bind_bucket *icsk_bind_hash; + struct inet_bind2_bucket *icsk_bind2_hash; unsigned long icsk_timeout; struct timer_list icsk_retransmit_timer; struct timer_list icsk_delack_timer; diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index e9cf2157ed8aca069c9f0e9bc82e31a9035d6b06..3af1e927247dbf99651e5fd3b8c0f10a383713a6 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -90,7 +91,31 @@ struct inet_bind_bucket { struct hlist_head owners; }; -static inline struct net *ib_net(struct inet_bind_bucket *ib) +struct inet_bind2_bucket { + possible_net_t ib_net; + int l3mdev; + unsigned short port; +#if IS_ENABLED(CONFIG_IPV6) + unsigned short family; +#endif + union { +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr v6_rcv_saddr; +#endif + __be32 rcv_saddr; + }; + /* Node in the bhash2 inet_bind_hashbucket chain */ + struct hlist_node node; + /* List of sockets hashed to this bucket */ + struct hlist_head owners; +}; + +static inline struct net *ib_net(const struct inet_bind_bucket *ib) +{ + return read_pnet(&ib->ib_net); +} + +static inline struct net *ib2_net(const struct inet_bind2_bucket *ib) { return read_pnet(&ib->ib_net); } @@ -133,14 +158,33 @@ struct inet_hashinfo { * TCP hash as well as the others for fast bind/connect. */ struct kmem_cache *bind_bucket_cachep; + /* This bind table is hashed by local port */ struct inet_bind_hashbucket *bhash; + struct kmem_cache *bind2_bucket_cachep; + /* This bind table is hashed by local port and sk->sk_rcv_saddr (ipv4) + * or sk->sk_v6_rcv_saddr (ipv6). This 2nd bind table is used + * primarily for expediting bind conflict resolution. + */ + struct inet_bind_hashbucket *bhash2; unsigned int bhash_size; /* The 2nd listener table hashed by local port and address */ unsigned int lhash2_mask; struct inet_listen_hashbucket *lhash2; + + bool pernet; }; +static inline struct inet_hashinfo *tcp_or_dccp_get_hashinfo(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IP_DCCP) + return sk->sk_prot->h.hashinfo ? : + sock_net(sk)->ipv4.tcp_death_row.hashinfo; +#else + return sock_net(sk)->ipv4.tcp_death_row.hashinfo; +#endif +} + static inline struct inet_listen_hashbucket * inet_lhash2_bucket(struct inet_hashinfo *h, u32 hash) { @@ -175,6 +219,10 @@ static inline void inet_ehash_locks_free(struct inet_hashinfo *hashinfo) hashinfo->ehash_locks = NULL; } +struct inet_hashinfo *inet_pernet_hashinfo_alloc(struct inet_hashinfo *hashinfo, + unsigned int ehash_entries); +void inet_pernet_hashinfo_free(struct inet_hashinfo *hashinfo); + struct inet_bind_bucket * inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net, struct inet_bind_hashbucket *head, @@ -182,14 +230,61 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net, void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb); +bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, + const struct net *net, unsigned short port, + int l3mdev); + +struct inet_bind2_bucket * +inet_bind2_bucket_create(struct kmem_cache *cachep, struct net *net, + struct inet_bind_hashbucket *head, + unsigned short port, int l3mdev, + const struct sock *sk); + +void inet_bind2_bucket_destroy(struct kmem_cache *cachep, + struct inet_bind2_bucket *tb); + +struct inet_bind2_bucket * +inet_bind2_bucket_find(const struct inet_bind_hashbucket *head, + const struct net *net, + unsigned short port, int l3mdev, + const struct sock *sk); + +bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb, + const struct net *net, unsigned short port, + int l3mdev, const struct sock *sk); + static inline u32 inet_bhashfn(const struct net *net, const __u16 lport, const u32 bhash_size) { return (lport + net_hash_mix(net)) & (bhash_size - 1); } +static inline struct inet_bind_hashbucket * +inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk, + const struct net *net, unsigned short port) +{ + u32 hash; + +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) + hash = ipv6_portaddr_hash(net, &sk->sk_v6_rcv_saddr, port); + else +#endif + hash = ipv4_portaddr_hash(net, sk->sk_rcv_saddr, port); + return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; +} + +struct inet_bind_hashbucket * +inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port); + +/* This should be called whenever a socket's sk_rcv_saddr (ipv4) or + * sk_v6_rcv_saddr (ipv6) changes after it has been binded. The socket's + * rcv_saddr field should already have been updated when this is called. + */ +int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk); + void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, - const unsigned short snum); + struct inet_bind2_bucket *tb2, unsigned short port); /* Caller must disable local BH processing. */ int __inet_inherit_port(const struct sock *sk, struct sock *child); diff --git a/include/net/ip.h b/include/net/ip.h index 1c979fd1904ce371d372ce42f3359fa277202295..038097c2a1520e4ee85a4574c16521460852f940 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -743,8 +743,12 @@ void ip_cmsg_recv_offset(struct msghdr *msg, struct sock *sk, int ip_cmsg_send(struct sock *sk, struct msghdr *msg, struct ipcm_cookie *ipc, bool allow_ipv6); DECLARE_STATIC_KEY_FALSE(ip4_min_ttl); +int do_ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, + unsigned int optlen); int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); +int do_ip_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen); int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen); int ip_ra_control(struct sock *sk, unsigned char on, diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 63fac94f9acedf57beb0bfcee91ee9bef2d9c745..fca3576798166416982ee6a9100b003810c49830 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -246,7 +246,8 @@ static inline void ip_tunnel_init_flow(struct flowi4 *fl4, __be32 daddr, __be32 saddr, __be32 key, __u8 tos, struct net *net, int oif, - __u32 mark, __u32 tun_inner_hash) + __u32 mark, __u32 tun_inner_hash, + __u8 flow_flags) { memset(fl4, 0, sizeof(*fl4)); @@ -263,6 +264,7 @@ static inline void ip_tunnel_init_flow(struct flowi4 *fl4, fl4->fl4_gre_key = key; fl4->flowi4_mark = mark; fl4->flowi4_multipath_hash = tun_inner_hash; + fl4->flowi4_flags = flow_flags; } int ip_tunnel_init(struct net_device *dev); @@ -300,6 +302,12 @@ int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[], struct ip_tunnel_parm *p, __u32 fwmark); void ip_tunnel_setup(struct net_device *dev, unsigned int net_id); +bool ip_tunnel_netlink_encap_parms(struct nlattr *data[], + struct ip_tunnel_encap *encap); + +void ip_tunnel_netlink_parms(struct nlattr *data[], + struct ip_tunnel_parm *parms); + extern const struct header_ops ip_tunnel_header_ops; __be16 ip_tunnel_parse_protocol(const struct sk_buff *skb); diff --git a/include/net/ipcomp.h b/include/net/ipcomp.h index c31108295079d67fa9ad66840baab315314472c2..8660a2a6d1fc76a7df7e90db95908cedb47b55e3 100644 --- a/include/net/ipcomp.h +++ b/include/net/ipcomp.h @@ -22,7 +22,7 @@ struct xfrm_state; int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb); int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb); void ipcomp_destroy(struct xfrm_state *x); -int ipcomp_init_state(struct xfrm_state *x); +int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack); static inline struct ip_comp_hdr *ip_comp_hdr(const struct sk_buff *skb) { diff --git a/include/net/ipv6.h b/include/net/ipv6.h index de9dcc5652c483c0a656b65504a356be9294f7fe..37943ba3a73c5c6a5124fa6cf2199c7987abb11d 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1156,8 +1156,12 @@ struct in6_addr *fl6_update_dst(struct flowi6 *fl6, */ DECLARE_STATIC_KEY_FALSE(ip6_min_hopcount); +int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, + unsigned int optlen); int ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); +int do_ipv6_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen); int ipv6_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen); @@ -1178,6 +1182,8 @@ void ipv6_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port, void ipv6_local_error(struct sock *sk, int err, struct flowi6 *fl6, u32 info); void ipv6_local_rxpmtu(struct sock *sk, struct flowi6 *fl6, u32 mtu); +void inet6_cleanup_sock(struct sock *sk); +void inet6_sock_destruct(struct sock *sk); int inet6_release(struct socket *sock); int inet6_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len); int inet6_getname(struct socket *sock, struct sockaddr *uaddr, @@ -1207,7 +1213,7 @@ int ip6_mc_source(int add, int omode, struct sock *sk, int ip6_mc_msfilter(struct sock *sk, struct group_filter *gsf, struct sockaddr_storage *list); int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf, - struct sockaddr_storage __user *p); + sockptr_t optval, size_t ss_offset); #ifdef CONFIG_PROC_FS int ac6_proc_init(struct net *net); diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h index 45e0339be6fa4aa8e6c3ca074e00aff1c3b97b2f..c48186bf47372761493e9502da90dadcc0c26b01 100644 --- a/include/net/ipv6_stubs.h +++ b/include/net/ipv6_stubs.h @@ -81,6 +81,10 @@ struct ipv6_bpf_stub { const struct in6_addr *daddr, __be16 dport, int dif, int sdif, struct udp_table *tbl, struct sk_buff *skb); + int (*ipv6_setsockopt)(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen); + int (*ipv6_getsockopt)(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen); }; extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; diff --git a/include/net/mac80211.h b/include/net/mac80211.h index f198af600b5ec42e338a3bc67dfd05dd8e88d53a..ac2bad57933f866dfad6c6cace1e5d1d3f08a93f 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1480,6 +1481,10 @@ enum mac80211_rx_encoding { * each A-MPDU but the same for each subframe within one A-MPDU * @ampdu_delimiter_crc: A-MPDU delimiter CRC * @zero_length_psdu_type: radiotap type of the 0-length PSDU + * @link_valid: if the link which is identified by @link_id is valid. This flag + * is set only when connection is MLO. + * @link_id: id of the link used to receive the packet. This is used along with + * @link_valid. */ struct ieee80211_rx_status { u64 mactime; @@ -1504,6 +1509,7 @@ struct ieee80211_rx_status { s8 chain_signal[IEEE80211_MAX_CHAINS]; u8 ampdu_delimiter_crc; u8 zero_length_psdu_type; + u8 link_valid:1, link_id:4; }; static inline u32 @@ -1794,6 +1800,9 @@ struct ieee80211_vif_cfg { * @link_conf: in case of MLD, the per-link BSS configuration, * indexed by link ID * @valid_links: bitmap of valid links, or 0 for non-MLO. + * @active_links: The bitmap of active links, or 0 for non-MLO. + * The driver shouldn't change this directly, but use the + * API calls meant for that purpose. * @addr: address of this interface * @p2p: indicates whether this AP or STA interface is a p2p * interface, i.e. a GO or p2p-sta respectively @@ -1829,7 +1838,7 @@ struct ieee80211_vif { struct ieee80211_vif_cfg cfg; struct ieee80211_bss_conf bss_conf; struct ieee80211_bss_conf __rcu *link_conf[IEEE80211_MLD_MAX_NUM_LINKS]; - u16 valid_links; + u16 valid_links, active_links; u8 addr[ETH_ALEN] __aligned(2); bool p2p; @@ -1856,12 +1865,11 @@ struct ieee80211_vif { u8 drv_priv[] __aligned(sizeof(void *)); }; -/* FIXME: for now loop over all the available links; later will be changed - * to loop only over the active links. - */ -#define for_each_vif_active_link(vif, link, link_id) \ - for (link_id = 0; link_id < ARRAY_SIZE((vif)->link_conf); link_id++) \ - if ((link = rcu_dereference((vif)->link_conf[link_id]))) +#define for_each_vif_active_link(vif, link, link_id) \ + for (link_id = 0; link_id < ARRAY_SIZE((vif)->link_conf); link_id++) \ + if ((!(vif)->active_links || \ + (vif)->active_links & BIT(link_id)) && \ + (link = rcu_dereference((vif)->link_conf[link_id]))) static inline bool ieee80211_vif_is_mesh(struct ieee80211_vif *vif) { @@ -1894,6 +1902,19 @@ struct ieee80211_vif *wdev_to_ieee80211_vif(struct wireless_dev *wdev); */ struct wireless_dev *ieee80211_vif_to_wdev(struct ieee80211_vif *vif); +/** + * lockdep_vif_mutex_held - for lockdep checks on link poiners + * @vif: the interface to check + */ +static inline bool lockdep_vif_mutex_held(struct ieee80211_vif *vif) +{ + return lockdep_is_held(&ieee80211_vif_to_wdev(vif)->mtx); +} + +#define link_conf_dereference_protected(vif, link_id) \ + rcu_dereference_protected((vif)->link_conf[link_id], \ + lockdep_vif_mutex_held(vif)) + /** * enum ieee80211_key_flags - key flags * @@ -1975,6 +1996,7 @@ enum ieee80211_key_flags { * - Temporal Authenticator Rx MIC Key (64 bits) * @icv_len: The ICV length for this key type * @iv_len: The IV length for this key type + * @link_id: the link ID for MLO, or -1 for non-MLO or pairwise keys */ struct ieee80211_key_conf { atomic64_t tx_pn; @@ -1984,6 +2006,7 @@ struct ieee80211_key_conf { u8 hw_key_idx; s8 keyidx; u16 flags; + s8 link_id; u8 keylen; u8 key[]; }; @@ -2120,6 +2143,34 @@ struct ieee80211_sta_txpwr { enum nl80211_tx_power_setting type; }; +/** + * struct ieee80211_sta_aggregates - info that is aggregated from active links + * + * Used for any per-link data that needs to be aggregated and updated in the + * main &struct ieee80211_sta when updated or the active links change. + * + * @max_amsdu_len: indicates the maximal length of an A-MSDU in bytes. + * This field is always valid for packets with a VHT preamble. + * For packets with a HT preamble, additional limits apply: + * + * * If the skb is transmitted as part of a BA agreement, the + * A-MSDU maximal size is min(max_amsdu_len, 4065) bytes. + * * If the skb is not part of a BA agreement, the A-MSDU maximal + * size is min(max_amsdu_len, 7935) bytes. + * + * Both additional HT limits must be enforced by the low level + * driver. This is defined by the spec (IEEE 802.11-2012 section + * 8.3.2.2 NOTE 2). + * @max_rc_amsdu_len: Maximum A-MSDU size in bytes recommended by rate control. + * @max_tid_amsdu_len: Maximum A-MSDU size in bytes for this TID + */ +struct ieee80211_sta_aggregates { + u16 max_amsdu_len; + + u16 max_rc_amsdu_len; + u16 max_tid_amsdu_len[IEEE80211_NUM_TIDS]; +}; + /** * struct ieee80211_link_sta - station Link specific info * All link specific info for a STA link for a non MLD STA(single) @@ -2128,6 +2179,8 @@ struct ieee80211_sta_txpwr { * @addr: MAC address of the Link STA. For non-MLO STA this is same as the addr * in ieee80211_sta. For MLO Link STA this addr can be same or different * from addr in ieee80211_sta (representing MLD STA addr) + * @link_id: the link ID for this link STA (0 for deflink) + * @smps_mode: current SMPS mode (off, static or dynamic) * @supp_rates: Bitmap of supported rates * @ht_cap: HT capabilities of this STA; restricted to our own capabilities * @vht_cap: VHT capabilities of this STA; restricted to our own capabilities @@ -2144,6 +2197,8 @@ struct ieee80211_sta_txpwr { */ struct ieee80211_link_sta { u8 addr[ETH_ALEN]; + u8 link_id; + enum ieee80211_smps_mode smps_mode; u32 supp_rates[NUM_NL80211_BANDS]; struct ieee80211_sta_ht_cap ht_cap; @@ -2152,6 +2207,8 @@ struct ieee80211_link_sta { struct ieee80211_he_6ghz_capa he_6ghz_capa; struct ieee80211_sta_eht_cap eht_cap; + struct ieee80211_sta_aggregates agg; + u8 rx_nss; enum ieee80211_sta_rx_bandwidth bandwidth; struct ieee80211_sta_txpwr txpwr; @@ -2182,7 +2239,6 @@ struct ieee80211_link_sta { * if wme is supported. The bits order is like in * IEEE80211_WMM_IE_STA_QOSINFO_AC_*. * @max_sp: max Service Period. Only valid if wme is supported. - * @smps_mode: current SMPS mode (off, static or dynamic) * @rates: rate control selection table * @tdls: indicates whether the STA is a TDLS peer * @tdls_initiator: indicates the STA is an initiator of the TDLS link. Only @@ -2192,9 +2248,10 @@ struct ieee80211_link_sta { * @max_amsdu_subframes: indicates the maximal number of MSDUs in a single * A-MSDU. Taken from the Extended Capabilities element. 0 means * unlimited. + * @cur: currently valid data as aggregated from the active links + * For non MLO STA it will point to the deflink data. For MLO STA + * ieee80211_sta_recalc_aggregates() must be called to update it. * @support_p2p_ps: indicates whether the STA supports P2P PS mechanism or not. - * @max_rc_amsdu_len: Maximum A-MSDU size in bytes recommended by rate control. - * @max_tid_amsdu_len: Maximum A-MSDU size in bytes for this TID * @txq: per-TID data TX queues (if driver uses the TXQ abstraction); note that * the last entry (%IEEE80211_NUM_TIDS) is used for non-data frames * @deflink: This holds the default link STA information, for non MLO STA all link @@ -2217,7 +2274,6 @@ struct ieee80211_sta { bool wme; u8 uapsd_queues; u8 max_sp; - enum ieee80211_smps_mode smps_mode; struct ieee80211_sta_rates __rcu *rates; bool tdls; bool tdls_initiator; @@ -2225,25 +2281,9 @@ struct ieee80211_sta { bool mlo; u8 max_amsdu_subframes; - /** - * @max_amsdu_len: - * indicates the maximal length of an A-MSDU in bytes. - * This field is always valid for packets with a VHT preamble. - * For packets with a HT preamble, additional limits apply: - * - * * If the skb is transmitted as part of a BA agreement, the - * A-MSDU maximal size is min(max_amsdu_len, 4065) bytes. - * * If the skb is not part of a BA agreement, the A-MSDU maximal - * size is min(max_amsdu_len, 7935) bytes. - * - * Both additional HT limits must be enforced by the low level - * driver. This is defined by the spec (IEEE 802.11-2012 section - * 8.3.2.2 NOTE 2). - */ - u16 max_amsdu_len; + struct ieee80211_sta_aggregates *cur; + bool support_p2p_ps; - u16 max_rc_amsdu_len; - u16 max_tid_amsdu_len[IEEE80211_NUM_TIDS]; struct ieee80211_txq *txq[IEEE80211_NUM_TIDS + 1]; @@ -2255,13 +2295,24 @@ struct ieee80211_sta { u8 drv_priv[] __aligned(sizeof(void *)); }; -/* FIXME: need to loop only over links which are active and check the actual - * lock - */ -#define for_each_sta_active_link(sta, link_sta, link_id) \ - for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++) \ - if (((link_sta) = rcu_dereference_protected((sta)->link[link_id],\ - 1))) \ +#ifdef CONFIG_LOCKDEP +bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta); +#else +static inline bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta) +{ + return true; +} +#endif + +#define link_sta_dereference_protected(sta, link_id) \ + rcu_dereference_protected((sta)->link[link_id], \ + lockdep_sta_mutex_held(sta)) + +#define for_each_sta_active_link(vif, sta, link_sta, link_id) \ + for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++) \ + if ((!(vif)->active_links || \ + (vif)->active_links & BIT(link_id)) && \ + ((link_sta) = link_sta_dereference_protected(sta, link_id))) /** * enum sta_notify_cmd - sta notify command @@ -5287,6 +5338,9 @@ struct sk_buff *ieee80211_pspoll_get(struct ieee80211_hw *hw, * ieee80211_nullfunc_get - retrieve a nullfunc template * @hw: pointer obtained from ieee80211_alloc_hw(). * @vif: &struct ieee80211_vif pointer from the add_interface callback. + * @link_id: If the vif is an MLD, get a frame with the link addresses + * for the given link ID. For a link_id < 0 you get a frame with + * MLD addresses, however useful that might be. * @qos_ok: QoS NDP is acceptable to the caller, this should be set * if at all possible * @@ -5304,7 +5358,7 @@ struct sk_buff *ieee80211_pspoll_get(struct ieee80211_hw *hw, */ struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - bool qos_ok); + int link_id, bool qos_ok); /** * ieee80211_probereq_get - retrieve a Probe Request template @@ -5975,6 +6029,22 @@ struct ieee80211_sta *ieee80211_find_sta_by_ifaddr(struct ieee80211_hw *hw, const u8 *addr, const u8 *localaddr); +/** + * ieee80211_find_sta_by_link_addrs - find STA by link addresses + * @hw: pointer as obtained from ieee80211_alloc_hw() + * @addr: remote station's link address + * @localaddr: local link address, use %NULL for any (but avoid that) + * @link_id: pointer to obtain the link ID if the STA is found, + * may be %NULL if the link ID is not needed + * + * Obtain the STA by link address, must use RCU protection. + */ +struct ieee80211_sta * +ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw, + const u8 *addr, + const u8 *localaddr, + unsigned int *link_id); + /** * ieee80211_sta_block_awake - block station from waking up * @hw: the hardware @@ -6050,6 +6120,19 @@ void ieee80211_sta_eosp(struct ieee80211_sta *pubsta); */ void ieee80211_send_eosp_nullfunc(struct ieee80211_sta *pubsta, int tid); +/** + * ieee80211_sta_recalc_aggregates - recalculate aggregate data after a change + * @pubsta: the station + * + * Call this function after changing a per-link aggregate data as referenced in + * &struct ieee80211_sta_aggregates by accessing the agg field of + * &struct ieee80211_link_sta. + * + * With non MLO the data in deflink will be referenced directly. In that case + * there is no need to call this function. + */ +void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta); + /** * ieee80211_sta_register_airtime - register airtime usage for a sta/tid * @@ -7101,4 +7184,45 @@ static inline bool ieee80211_is_tx_data(struct sk_buff *skb) ieee80211_is_data(hdr->frame_control); } +/** + * ieee80211_set_active_links - set active links in client mode + * @vif: interface to set active links on + * @active_links: the new active links bitmap + * + * This changes the active links on an interface. The interface + * must be in client mode (in AP mode, all links are always active), + * and @active_links must be a subset of the vif's valid_links. + * + * If a link is switched off and another is switched on at the same + * time (e.g. active_links going from 0x1 to 0x10) then you will get + * a sequence of calls like + * - change_vif_links(0x11) + * - unassign_vif_chanctx(link_id=0) + * - change_sta_links(0x11) for each affected STA (the AP) + * (TDLS connections on now inactive links should be torn down) + * - remove group keys on the old link (link_id 0) + * - add new group keys (GTK/IGTK/BIGTK) on the new link (link_id 4) + * - change_sta_links(0x10) for each affected STA (the AP) + * - assign_vif_chanctx(link_id=4) + * - change_vif_links(0x10) + * + * Note: This function acquires some mac80211 locks and must not + * be called with any driver locks held that could cause a + * lock dependency inversion. Best call it without locks. + */ +int ieee80211_set_active_links(struct ieee80211_vif *vif, u16 active_links); + +/** + * ieee80211_set_active_links_async - asynchronously set active links + * @vif: interface to set active links on + * @active_links: the new active links bitmap + * + * See ieee80211_set_active_links() for more information, the only + * difference here is that the link change is triggered async and + * can be called in any context, but the link switch will only be + * completed after it returns. + */ +void ieee80211_set_active_links_async(struct ieee80211_vif *vif, + u16 active_links); + #endif /* MAC80211_H */ diff --git a/include/net/macsec.h b/include/net/macsec.h index d6fa6b97f6efa347ea24653dfced8456576e7102..5b9c61c4d3a625d1596b24b805cc5975d5844e8a 100644 --- a/include/net/macsec.h +++ b/include/net/macsec.h @@ -14,12 +14,27 @@ #define MACSEC_DEFAULT_PN_LEN 4 #define MACSEC_XPN_PN_LEN 8 -#define MACSEC_SALT_LEN 12 #define MACSEC_NUM_AN 4 /* 2 bits for the association number */ +#define MACSEC_SCI_LEN 8 +#define MACSEC_PORT_ES (htons(0x0001)) + +#define MACSEC_TCI_VERSION 0x80 +#define MACSEC_TCI_ES 0x40 /* end station */ +#define MACSEC_TCI_SC 0x20 /* SCI present */ +#define MACSEC_TCI_SCB 0x10 /* epon */ +#define MACSEC_TCI_E 0x08 /* encryption */ +#define MACSEC_TCI_C 0x04 /* changed text */ +#define MACSEC_AN_MASK 0x03 /* association number */ +#define MACSEC_TCI_CONFID (MACSEC_TCI_E | MACSEC_TCI_C) + +#define MACSEC_DEFAULT_ICV_LEN 16 + typedef u64 __bitwise sci_t; typedef u32 __bitwise ssci_t; +struct metadata_dst; + typedef union salt { struct { u32 ssci; @@ -183,6 +198,7 @@ struct macsec_tx_sa { * @scb: single copy broadcast flag * @sa: array of secure associations * @stats: stats for this TXSC + * @md_dst: MACsec offload metadata dst */ struct macsec_tx_sc { bool active; @@ -193,6 +209,7 @@ struct macsec_tx_sc { bool scb; struct macsec_tx_sa __rcu *sa[MACSEC_NUM_AN]; struct pcpu_tx_sc_stats __percpu *stats; + struct metadata_dst *md_dst; }; /** @@ -254,8 +271,6 @@ struct macsec_context { struct macsec_rx_sa_stats *rx_sa_stats; struct macsec_dev_stats *dev_stats; } stats; - - u8 prepare:1; }; /** @@ -289,5 +304,12 @@ struct macsec_ops { }; void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa); +static inline bool macsec_send_sci(const struct macsec_secy *secy) +{ + const struct macsec_tx_sc *tx_sc = &secy->tx_sc; + + return tx_sc->send_sci || + (secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb); +} #endif /* _NET_MACSEC_H_ */ diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 3827a6b395fdb67f391853d528bb6f7ff9169449..20745cf7ae1adcedbaf11521c5e418ae766b131a 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -276,11 +276,6 @@ static inline void *neighbour_priv(const struct neighbour *n) extern const struct nla_policy nda_policy[]; -static inline bool neigh_key_eq16(const struct neighbour *n, const void *pkey) -{ - return *(const u16 *)n->primary_key == *(const u16 *)pkey; -} - static inline bool neigh_key_eq32(const struct neighbour *n, const void *pkey) { return *(const u32 *)n->primary_key == *(const u32 *)pkey; diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index a32be8aa7ed262d71bcd167ceb6a829d04c30e4c..6a2019aaa46448be88b4994e5d41a07ce194a25f 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -53,8 +53,6 @@ struct nf_conntrack_net { /* only used when new connection is allocated: */ atomic_t count; unsigned int expect_count; - u8 sysctl_auto_assign_helper; - bool auto_assign_helper_warned; /* only used from work queues, configuration plane, and so on: */ unsigned int users4; diff --git a/include/net/netfilter/nf_conntrack_bpf.h b/include/net/netfilter/nf_conntrack_bpf.h index a473b56842c5b6866873084c88b6383588d96f55..2d0da478c8e08e35fea347696108bd2b58b2b602 100644 --- a/include/net/netfilter/nf_conntrack_bpf.h +++ b/include/net/netfilter/nf_conntrack_bpf.h @@ -3,13 +3,18 @@ #ifndef _NF_CONNTRACK_BPF_H #define _NF_CONNTRACK_BPF_H -#include #include +#include + +struct nf_conn___init { + struct nf_conn ct; +}; #if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) extern int register_nf_conntrack_bpf(void); +extern void cleanup_nf_conntrack_bpf(void); #else @@ -18,6 +23,24 @@ static inline int register_nf_conntrack_bpf(void) return 0; } +static inline void cleanup_nf_conntrack_bpf(void) +{ +} + +#endif + +#if (IS_BUILTIN(CONFIG_NF_NAT) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ + (IS_MODULE(CONFIG_NF_NAT) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)) + +extern int register_nf_nat_bpf(void); + +#else + +static inline int register_nf_nat_bpf(void) +{ + return 0; +} + #endif #endif /* _NF_CONNTRACK_BPF_H */ diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 3cd3a6e631aa5a17c7ed412a94d07956946934b4..b2b9de70d9f4d4ff3658205fb9c6d7c07a997004 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -86,10 +86,6 @@ extern spinlock_t nf_conntrack_expect_lock; /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ -#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ - (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \ - IS_ENABLED(CONFIG_NF_CT_NETLINK)) - static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) { if (timeout > INT_MAX) @@ -101,6 +97,4 @@ int __nf_ct_change_timeout(struct nf_conn *ct, u64 cta_timeout); void __nf_ct_change_status(struct nf_conn *ct, unsigned long on, unsigned long off); int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status); -#endif - #endif /* _NF_CONNTRACK_CORE_H */ diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index d5326c44b4535bb917ce13c3faddbeceab43aaa7..cd982f4a0f50cac10887e13ba50d180b4f3dc95e 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -270,6 +270,7 @@ void flow_offload_refresh(struct nf_flowtable *flow_table, struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table, struct flow_offload_tuple *tuple); +void nf_flow_table_gc_run(struct nf_flowtable *flow_table); void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable, struct net_device *dev); void nf_flow_table_cleanup(struct net_device *dev); @@ -306,6 +307,8 @@ void nf_flow_offload_stats(struct nf_flowtable *flowtable, struct flow_offload *flow); void nf_flow_table_offload_flush(struct nf_flowtable *flowtable); +void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable); + int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, struct net_device *dev, enum flow_block_command cmd); diff --git a/include/net/netfilter/nf_nat_helper.h b/include/net/netfilter/nf_nat_helper.h index efae84646353ca272f4ef9f16382a2c82c939c0a..44c421b9be853a74a71baa58b3ccca9d83a96944 100644 --- a/include/net/netfilter/nf_nat_helper.h +++ b/include/net/netfilter/nf_nat_helper.h @@ -38,4 +38,5 @@ bool nf_nat_mangle_udp_packet(struct sk_buff *skb, struct nf_conn *ct, * to port ct->master->saved_proto. */ void nf_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this); +u16 nf_nat_exp_find_port(struct nf_conntrack_expect *exp, u16 port); #endif diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 980daa6e1e3aa42d42be855630614afb65a5acd9..c81021ab07aa17e527d644998ab8ebba70eaf7b1 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -43,7 +43,7 @@ void nf_queue_entry_free(struct nf_queue_entry *entry); static inline void init_hashrandom(u32 *jhash_initval) { while (*jhash_initval == 0) - *jhash_initval = prandom_u32(); + *jhash_initval = get_random_u32(); } static inline u32 hash_v4(const struct iphdr *iph, u32 initval) diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 99aae36c04b97532bc6884497db8ad6482fdfdcc..cdb7db9b0e25204fbcafef518a9c36d24da1edff 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -1652,6 +1652,7 @@ struct nftables_pernet { struct list_head module_list; struct list_head notify_list; struct mutex commit_mutex; + u64 table_handle; unsigned int base_seq; u8 validate_state; }; diff --git a/include/net/netlink.h b/include/net/netlink.h index 7a2a9d3144ba6c32777594d5585597948f480404..4418b1981e318d7dd9118091004c1e760b660f7e 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -325,6 +325,7 @@ struct nla_policy { struct netlink_range_validation_signed *range_signed; struct { s16 min, max; + u8 network_byte_order:1; }; int (*validate)(const struct nlattr *attr, struct netlink_ext_ack *extack); @@ -418,6 +419,14 @@ struct nla_policy { .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \ .validation_type = NLA_VALIDATE_MAX, \ .max = _max, \ + .network_byte_order = 0, \ +} + +#define NLA_POLICY_MAX_BE(tp, _max) { \ + .type = NLA_ENSURE_UINT_TYPE(tp), \ + .validation_type = NLA_VALIDATE_MAX, \ + .max = _max, \ + .network_byte_order = 1, \ } #define NLA_POLICY_MASK(tp, _mask) { \ @@ -741,6 +750,7 @@ static inline int __nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen, * @hdrlen: length of family specific header * @tb: destination array with maxtype+1 elements * @maxtype: maximum attribute type to be expected + * @policy: validation policy * @extack: extended ACK report struct * * See nla_parse() @@ -760,6 +770,7 @@ static inline int nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen, * @hdrlen: length of family specific header * @tb: destination array with maxtype+1 elements * @maxtype: maximum attribute type to be expected + * @policy: validation policy * @extack: extended ACK report struct * * See nla_parse_deprecated() @@ -779,6 +790,7 @@ static inline int nlmsg_parse_deprecated(const struct nlmsghdr *nlh, int hdrlen, * @hdrlen: length of family specific header * @tb: destination array with maxtype+1 elements * @maxtype: maximum attribute type to be expected + * @policy: validation policy * @extack: extended ACK report struct * * See nla_parse_deprecated_strict() @@ -814,7 +826,6 @@ static inline struct nlattr *nlmsg_find_attr(const struct nlmsghdr *nlh, * @len: length of attribute stream * @maxtype: maximum attribute type to be expected * @policy: validation policy - * @validate: validation strictness * @extack: extended ACK report struct * * Validates all attributes in the specified attribute stream against the diff --git a/include/net/netns/conntrack.h b/include/net/netns/conntrack.h index c396a3862e80866cefbb62277b7088025e768490..e1290c159184a7b74ed658f2a1fe311c556227ae 100644 --- a/include/net/netns/conntrack.h +++ b/include/net/netns/conntrack.h @@ -101,7 +101,6 @@ struct netns_ct { u8 sysctl_log_invalid; /* Log invalid packets */ u8 sysctl_events; u8 sysctl_acct; - u8 sysctl_auto_assign_helper; u8 sysctl_tstamp; u8 sysctl_checksum; diff --git a/include/net/netns/generic.h b/include/net/netns/generic.h index 7ce68183f6e180cdf1032e6edc60da00b9aed25a..00c399edeed11c79e9640e1c2932bd39f155c10c 100644 --- a/include/net/netns/generic.h +++ b/include/net/netns/generic.h @@ -33,7 +33,7 @@ struct net_generic { struct rcu_head rcu; } s; - void *ptr[0]; + DECLARE_FLEX_ARRAY(void *, ptr); }; }; diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index c7320ef356d940aefba708ea98de502253d2b04d..1b8004679445179aa6c0f4db5649c8a84cbfcdcf 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -34,6 +34,7 @@ struct inet_hashinfo; struct inet_timewait_death_row { refcount_t tw_refcount; + /* Padding to avoid false sharing, tw_refcount can be often written */ struct inet_hashinfo *hashinfo ____cacheline_aligned_in_smp; int sysctl_max_tw_buckets; }; @@ -41,7 +42,7 @@ struct inet_timewait_death_row { struct tcp_fastopen_context; struct netns_ipv4 { - struct inet_timewait_death_row *tcp_death_row; + struct inet_timewait_death_row tcp_death_row; #ifdef CONFIG_SYSCTL struct ctl_table_header *forw_hdr; @@ -170,6 +171,7 @@ struct netns_ipv4 { int sysctl_tcp_pacing_ca_ratio; int sysctl_tcp_wmem[3]; int sysctl_tcp_rmem[3]; + unsigned int sysctl_tcp_child_ehash_entries; unsigned long sysctl_tcp_comp_sack_delay_ns; unsigned long sysctl_tcp_comp_sack_slack_ns; int sysctl_max_syn_backlog; @@ -179,6 +181,8 @@ struct netns_ipv4 { unsigned int sysctl_tcp_fastopen_blackhole_timeout; atomic_t tfo_active_disable_times; unsigned long tfo_active_disable_stamp; + u32 tcp_challenge_timestamp; + u32 tcp_challenge_count; int sysctl_udp_wmem_min; int sysctl_udp_rmem_min; diff --git a/include/net/netns/netfilter.h b/include/net/netns/netfilter.h index b593f95e99913e9946da7bd2d3b28462350217ee..02bbdc577f8e2881c50e7eaccf9f1c4223d9da50 100644 --- a/include/net/netns/netfilter.h +++ b/include/net/netns/netfilter.h @@ -24,9 +24,6 @@ struct netns_nf { #ifdef CONFIG_NETFILTER_FAMILY_BRIDGE struct nf_hook_entries __rcu *hooks_bridge[NF_INET_NUMHOOKS]; #endif -#if IS_ENABLED(CONFIG_DECNET) - struct nf_hook_entries __rcu *hooks_decnet[NF_DN_NUMHOOKS]; -#endif #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4) unsigned int defrag_ipv4_users; #endif diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h index 2adbe2b245df8b548b2aa895d037e7f789878bb9..582212ada3ba7441d34fafdd81ae09dae929e8da 100644 --- a/include/net/netns/smc.h +++ b/include/net/netns/smc.h @@ -19,5 +19,8 @@ struct netns_smc { #endif unsigned int sysctl_autocorking_size; unsigned int sysctl_smcr_buf_type; + int sysctl_smcr_testlink_time; + int sysctl_wmem; + int sysctl_rmem; }; #endif diff --git a/include/net/nl802154.h b/include/net/nl802154.h index 145acb8f250957f5fd7ae682a475034eea6055fc..f5850b569c52cde7e67e1a41a7b390e43a40c98c 100644 --- a/include/net/nl802154.h +++ b/include/net/nl802154.h @@ -58,9 +58,6 @@ enum nl802154_commands { NL802154_CMD_SET_WPAN_PHY_NETNS, - /* add new commands above here */ - -#ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL NL802154_CMD_SET_SEC_PARAMS, NL802154_CMD_GET_SEC_KEY, /* can dump */ NL802154_CMD_NEW_SEC_KEY, @@ -74,7 +71,8 @@ enum nl802154_commands { NL802154_CMD_GET_SEC_LEVEL, /* can dump */ NL802154_CMD_NEW_SEC_LEVEL, NL802154_CMD_DEL_SEC_LEVEL, -#endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */ + + /* add new commands above here */ /* used to define NL802154_CMD_MAX below */ __NL802154_CMD_AFTER_LAST, diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index d9d90e6925e146522bb4cee39caac80499639e3a..4cabb32a2ad944a22c2f19898499649b2d8f9347 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -81,6 +81,19 @@ int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res, bool compat_mode); +static inline bool tc_cls_stats_dump(struct tcf_proto *tp, + struct tcf_walker *arg, + void *filter) +{ + if (arg->count >= arg->skip && arg->fn(tp, filter, arg) < 0) { + arg->stop = 1; + return false; + } + + arg->count++; + return true; +} + #else static inline bool tcf_block_shared(struct tcf_block *block) { @@ -197,6 +210,18 @@ tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r) __tcf_unbind_filter(q, r); } +static inline void tc_cls_bind_class(u32 classid, unsigned long cl, + void *q, struct tcf_result *res, + unsigned long base) +{ + if (res->classid == classid) { + if (cl) + __tcf_bind_filter(q, res, base); + else + __tcf_unbind_filter(q, res); + } +} + struct tcf_exts { #ifdef CONFIG_NET_CLS_ACT __u32 type; /* for backward compat(TCA_OLD_COMPAT) */ diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 3372a1f67cf4eb45827cda62e8b0b6af126792b0..38207873eda69bb708c54822b12e6b4419e3ca64 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -100,7 +100,7 @@ struct Qdisc *fifo_create_dflt(struct Qdisc *sch, struct Qdisc_ops *ops, struct netlink_ext_ack *extack); int register_qdisc(struct Qdisc_ops *qops); -int unregister_qdisc(struct Qdisc_ops *qops); +void unregister_qdisc(struct Qdisc_ops *qops); void qdisc_get_default(char *id, size_t len); int qdisc_set_default(const char *id); @@ -141,6 +141,11 @@ static inline struct net *qdisc_net(struct Qdisc *q) return dev_net(q->dev_queue->dev); } +struct tc_query_caps_base { + enum tc_setup_type type; + void *caps; +}; + struct tc_cbs_qopt_offload { u8 enable; s32 queue; @@ -155,6 +160,10 @@ struct tc_etf_qopt_offload { s32 queue; }; +struct tc_taprio_caps { + bool supports_queue_max_sdu:1; +}; + struct tc_taprio_sched_entry { u8 command; /* TC_TAPRIO_CMD_* */ @@ -168,6 +177,7 @@ struct tc_taprio_qopt_offload { ktime_t base_time; u64 cycle_time; u64 cycle_time_extension; + u32 max_sdu[TC_MAX_QUEUE]; size_t num_entries; struct tc_taprio_sched_entry entries[]; @@ -222,4 +232,17 @@ static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb) return cb; } +static inline bool tc_qdisc_stats_dump(struct Qdisc *sch, + unsigned long cl, + struct qdisc_walker *arg) +{ + if (arg->count >= arg->skip && arg->fn(sch, cl, arg) < 0) { + arg->stop = 1; + return false; + } + + arg->count++; + return true; +} + #endif diff --git a/include/net/red.h b/include/net/red.h index be11dbd26492094e8b477339c66b3a098e06c39f..425364de0df7913cdd6361a0eb8d18b418372787 100644 --- a/include/net/red.h +++ b/include/net/red.h @@ -122,7 +122,6 @@ struct red_stats { u32 forced_drop; /* Forced drops, qavg > max_thresh */ u32 forced_mark; /* Forced marks, qavg > max_thresh */ u32 pdrop; /* Drops due to queue limits */ - u32 other; /* Drops due to drop() calls */ }; struct red_parms { @@ -364,7 +363,7 @@ static inline unsigned long red_calc_qavg(const struct red_parms *p, static inline u32 red_random(const struct red_parms *p) { - return reciprocal_divide(prandom_u32(), p->max_P_reciprocal); + return reciprocal_divide(get_random_u32(), p->max_P_reciprocal); } static inline int red_mark_probability(const struct red_parms *p, diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index ec693fe7c55398bb44977d6d2e54b2d65b97087b..d5517719af4ef22282f0a15b132f8e8a07ae4179 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -326,11 +326,6 @@ struct tcf_result { }; const struct tcf_proto *goto_tp; - /* used in the skb_tc_reinsert function */ - struct { - bool ingress; - struct gnet_stats_queue *qstats; - }; }; }; @@ -682,6 +677,9 @@ qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, { } #endif +void qdisc_offload_query_caps(struct net_device *dev, + enum tc_setup_type type, + void *caps, size_t caps_len); struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack); @@ -940,13 +938,6 @@ static inline void qdisc_purge_queue(struct Qdisc *sch) qdisc_tree_reduce_backlog(sch, qlen, backlog); } -static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh) -{ - qh->head = NULL; - qh->tail = NULL; - qh->qlen = 0; -} - static inline void __qdisc_enqueue_tail(struct sk_buff *skb, struct qdisc_skb_head *qh) { @@ -1137,7 +1128,6 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) static inline void qdisc_reset_queue(struct Qdisc *sch) { __qdisc_reset_queue(&sch->q); - sch->qstats.backlog = 0; } static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, diff --git a/include/net/sock.h b/include/net/sock.h index d08cfe190a78ba309ff31e549a7ba30d570e7105..9e464f6409a7175cef5f8ec22e70cade19df5e60 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -348,6 +348,7 @@ struct sk_filter; * @sk_txtime_report_errors: set report errors mode for SO_TXTIME * @sk_txtime_unused: unused txtime flags * @ns_tracker: tracker for netns reference + * @sk_bind2_node: bind node in the bhash2 table */ struct sock { /* @@ -537,6 +538,7 @@ struct sock { #endif struct rcu_head sk_rcu; netns_tracker ns_tracker; + struct hlist_node sk_bind2_node; }; enum sk_pacing { @@ -742,11 +744,6 @@ static inline void sk_node_init(struct hlist_node *node) node->pprev = NULL; } -static inline void sk_nulls_node_init(struct hlist_nulls_node *node) -{ - node->pprev = NULL; -} - static inline void __sk_del_node(struct sock *sk) { __hlist_del(&sk->sk_node); @@ -870,6 +867,16 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_add_head(&sk->sk_bind_node, list); } +static inline void __sk_del_bind2_node(struct sock *sk) +{ + __hlist_del(&sk->sk_bind2_node); +} + +static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list) +{ + hlist_add_head(&sk->sk_bind2_node, list); +} + #define sk_for_each(__sk, list) \ hlist_for_each_entry(__sk, list, sk_node) #define sk_for_each_rcu(__sk, list) \ @@ -887,6 +894,8 @@ static inline void sk_add_bind_node(struct sock *sk, hlist_for_each_entry_safe(__sk, tmp, list, sk_node) #define sk_for_each_bound(__sk, list) \ hlist_for_each_entry(__sk, list, sk_bind_node) +#define sk_for_each_bound_bhash2(__sk, list) \ + hlist_for_each_entry(__sk, list, sk_bind2_node) /** * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset @@ -1774,6 +1783,11 @@ static inline void unlock_sock_fast(struct sock *sk, bool slow) } } +void sockopt_lock_sock(struct sock *sk); +void sockopt_release_sock(struct sock *sk); +bool sockopt_ns_capable(struct user_namespace *ns, int cap); +bool sockopt_capable(int cap); + /* Used by processes to "lock" a socket state, so that * interrupts and bottom half handlers won't change it * from under us. It essentially blocks any incoming @@ -1848,9 +1862,13 @@ void sock_pfree(struct sk_buff *skb); #define sock_edemux sock_efree #endif +int sk_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen); int sock_setsockopt(struct socket *sock, int level, int op, sockptr_t optval, unsigned int optlen); +int sk_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen); int sock_getsockopt(struct socket *sock, int level, int op, char __user *optval, int __user *optlen); int sock_gettstamp(struct socket *sock, void __user *userstamp, @@ -2091,7 +2109,7 @@ static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk) static inline u32 net_tx_rndhash(void) { - u32 v = prandom_u32(); + u32 v = get_random_u32(); return v ?: 1; } diff --git a/include/net/tcp.h b/include/net/tcp.h index d10962b9f0d0d25f46331b6be3fd7f9c2a136a80..14d45661a84d85816b58bb86a77683e27d4447d8 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -327,6 +327,8 @@ void tcp_remove_empty_skb(struct sock *sk); int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw); int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size); +int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, + size_t size, struct ubuf_info *uarg); int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, int flags); int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset, @@ -346,6 +348,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb); void tcp_rcv_space_adjust(struct sock *sk); int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); void tcp_twsk_destructor(struct sock *sk); +void tcp_twsk_purge(struct list_head *net_exit_list, int family); ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); @@ -402,9 +405,13 @@ void tcp_init_sock(struct sock *sk); void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb); __poll_t tcp_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait); +int do_tcp_getsockopt(struct sock *sk, int level, + int optname, sockptr_t optval, sockptr_t optlen); int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen); bool tcp_bpf_bypass_getsockopt(int level, int optname); +int do_tcp_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen); int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); void tcp_set_keepalive(struct sock *sk, int val); @@ -1295,11 +1302,14 @@ static inline bool tcp_is_cwnd_limited(const struct sock *sk) { const struct tcp_sock *tp = tcp_sk(sk); + if (tp->is_cwnd_limited) + return true; + /* If in slow start, ensure cwnd grows to twice what was ACKed. */ if (tcp_in_slow_start(tp)) return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out; - return tp->is_cwnd_limited; + return false; } /* BBR congestion control needs pacing. diff --git a/include/net/tls.h b/include/net/tls.h index cb205f9d9473b22b93bf221abda90823fed8df40..154949c7b0c88e9bb23231d3de52a389fdfc0ea8 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -51,6 +51,16 @@ struct tls_rec; +struct tls_cipher_size_desc { + unsigned int iv; + unsigned int key; + unsigned int salt; + unsigned int tag; + unsigned int rec_seq; +}; + +extern const struct tls_cipher_size_desc tls_cipher_size_desc[]; + /* Maximum data size carried in a TLS record */ #define TLS_MAX_PAYLOAD_SIZE ((size_t)1 << 14) diff --git a/include/net/udp.h b/include/net/udp.h index 5ee88ddf79c3fe61c007d501dcd5f124ff517642..fee053bcd17c6ddc7cea6c055e587f4321e800e0 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -247,7 +247,7 @@ static inline bool udp_sk_bound_dev_eq(struct net *net, int bound_dev_if, } /* net/ipv4/udp.c */ -void udp_destruct_sock(struct sock *sk); +void udp_destruct_common(struct sock *sk); void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len); int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb); void udp_skb_destructor(struct sock *sk, struct sk_buff *skb); diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index afc7ce713657b7483068d3d0d71c460bdfc3798e..72394f441dad83c1ad144bd6ff566536dbe46773 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -67,6 +67,9 @@ static inline int udp_sock_create(struct net *net, typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); typedef int (*udp_tunnel_encap_err_lookup_t)(struct sock *sk, struct sk_buff *skb); +typedef void (*udp_tunnel_encap_err_rcv_t)(struct sock *sk, + struct sk_buff *skb, + unsigned int udp_offset); typedef void (*udp_tunnel_encap_destroy_t)(struct sock *sk); typedef struct sk_buff *(*udp_tunnel_gro_receive_t)(struct sock *sk, struct list_head *head, @@ -80,6 +83,7 @@ struct udp_tunnel_sock_cfg { __u8 encap_type; udp_tunnel_encap_rcv_t encap_rcv; udp_tunnel_encap_err_lookup_t encap_err_lookup; + udp_tunnel_encap_err_rcv_t encap_err_rcv; udp_tunnel_encap_destroy_t encap_destroy; udp_tunnel_gro_receive_t gro_receive; udp_tunnel_gro_complete_t gro_complete; diff --git a/include/net/udplite.h b/include/net/udplite.h index 0143b373602ec547e5f133f9c02ae4e539ecfcfa..299c14ce2bb949029624710798810919cccb7934 100644 --- a/include/net/udplite.h +++ b/include/net/udplite.h @@ -25,14 +25,6 @@ static __inline__ int udplite_getfrag(void *from, char *to, int offset, return copy_from_iter_full(to, len, &msg->msg_iter) ? 0 : -EFAULT; } -/* Designate sk as UDP-Lite socket */ -static inline int udplite_sk_init(struct sock *sk) -{ - udp_init_sock(sk); - udp_sk(sk)->pcflag = UDPLITE_BIT; - return 0; -} - /* * Checksumming routines */ diff --git a/include/net/xdp.h b/include/net/xdp.h index 04c852c7a77fa240d0fcd98bced49aa971b22623..55dbc68bfffcec972cba05423192a3489544063e 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -164,13 +164,13 @@ struct xdp_frame { void *data; u16 len; u16 headroom; - u32 metasize:8; - u32 frame_sz:24; + u32 metasize; /* uses lower 8-bits */ /* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time, * while mem info is valid on remote CPU. */ struct xdp_mem_info mem; struct net_device *dev_rx; /* used by cpumap */ + u32 frame_sz; u32 flags; /* supported values defined in xdp_buff_flags */ }; diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 0e58c38ce0c193997cb2f393b213d2d46ec2a2dd..9c0d860609ba9479f727b23c7456b45270416659 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -9,6 +9,9 @@ #include #include +#define XDP_UMEM_MIN_CHUNK_SHIFT 11 +#define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT) + #ifdef CONFIG_XDP_SOCKETS void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); @@ -104,13 +107,6 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) xp_free(xskb); } -static inline void xsk_buff_discard(struct xdp_buff *xdp) -{ - struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - - xp_release(xskb); -} - static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size) { xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM; diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 6e8fa98f786fa990d76be3c0a46be18c83d253ff..dbc81f5eb5538ffdb320bac28990fa2050b92ad2 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -312,9 +312,15 @@ struct km_event { struct net *net; }; +struct xfrm_if_decode_session_result { + struct net *net; + u32 if_id; +}; + struct xfrm_if_cb { - struct xfrm_if *(*decode_session)(struct sk_buff *skb, - unsigned short family); + bool (*decode_session)(struct sk_buff *skb, + unsigned short family, + struct xfrm_if_decode_session_result *res); }; void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb); @@ -399,7 +405,8 @@ struct xfrm_type { #define XFRM_TYPE_LOCAL_COADDR 4 #define XFRM_TYPE_REMOTE_COADDR 8 - int (*init_state)(struct xfrm_state *x); + int (*init_state)(struct xfrm_state *x, + struct netlink_ext_ack *extack); void (*destructor)(struct xfrm_state *); int (*input)(struct xfrm_state *, struct sk_buff *skb); int (*output)(struct xfrm_state *, struct sk_buff *pskb); @@ -985,6 +992,7 @@ void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev); struct xfrm_if_parms { int link; /* ifindex of underlying L2 interface */ u32 if_id; /* interface identifyer */ + bool collect_md; }; struct xfrm_if { @@ -1573,9 +1581,10 @@ int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_vali void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si); void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si); u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq); -int xfrm_init_replay(struct xfrm_state *x); +int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack); u32 xfrm_state_mtu(struct xfrm_state *x, int mtu); -int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload); +int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload, + struct netlink_ext_ack *extack); int xfrm_init_state(struct xfrm_state *x); int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type); int xfrm_input_resume(struct sk_buff *skb, int nexthdr); @@ -1879,7 +1888,8 @@ void xfrm_dev_resume(struct sk_buff *skb); void xfrm_dev_backlog(struct softnet_data *sd); struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again); int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, - struct xfrm_user_offload *xuo); + struct xfrm_user_offload *xuo, + struct netlink_ext_ack *extack); bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x); static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x) @@ -1942,7 +1952,7 @@ static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_fea return skb; } -static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo) +static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack) { return 0; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 647722e847b4131bd3cde64e15b2a834ef04a664..f787c3f524b0358d9394d93bdd1c1521f9d8ee35 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -95,7 +95,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, struct xdp_umem *umem); int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev, u16 queue_id, u16 flags); -int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem, +int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, struct net_device *dev, u16 queue_id); int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs); void xp_destroy(struct xsk_buff_pool *pool); diff --git a/include/rdma/ib_cm.h b/include/rdma/ib_cm.h index e23eb357b7613a2eac49f477e790d1afa87c407c..a2ac62b4a6cf1d79c87392c46e8ad5a1476c1e31 100644 --- a/include/rdma/ib_cm.h +++ b/include/rdma/ib_cm.h @@ -294,7 +294,6 @@ struct ib_cm_id { void *context; struct ib_device *device; __be64 service_id; - __be64 service_mask; enum ib_cm_state state; /* internal CM/debug use */ enum ib_cm_lap_state lap_state; /* internal CM/debug use */ __be32 local_id; @@ -340,13 +339,8 @@ void ib_destroy_cm_id(struct ib_cm_id *cm_id); * and service ID resolution requests. The service ID should be specified * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will * assign a service ID to the caller. - * @service_mask: Mask applied to service ID used to listen across a - * range of service IDs. If set to 0, the service ID is matched - * exactly. This parameter is ignored if %service_id is set to - * IB_CM_ASSIGN_SERVICE_ID. */ -int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, - __be64 service_mask); +int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id); struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, ib_cm_handler cm_handler, @@ -354,6 +348,8 @@ struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device, struct ib_cm_req_param { struct sa_path_rec *primary_path; + struct sa_path_rec *primary_path_inbound; + struct sa_path_rec *primary_path_outbound; struct sa_path_rec *alternate_path; const struct ib_gid_attr *ppath_sgid_attr; __be64 service_id; diff --git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h index 3634d4cc7a5638670bada0432ea7ff25d385044d..e930bec33b31a6f2eed6445d105c840bb63681d7 100644 --- a/include/rdma/ib_sa.h +++ b/include/rdma/ib_sa.h @@ -186,6 +186,7 @@ struct sa_path_rec { struct sa_path_rec_opa opa; }; enum sa_path_rec_type rec_type; + u32 flags; }; static inline enum ib_gid_type @@ -413,7 +414,7 @@ int ib_sa_path_rec_get(struct ib_sa_client *client, struct ib_device *device, ib_sa_comp_mask comp_mask, unsigned long timeout_ms, gfp_t gfp_mask, void (*callback)(int status, struct sa_path_rec *resp, - void *context), + int num_prs, void *context), void *context, struct ib_sa_query **query); struct ib_sa_multicast { diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h index 5b18e2e36ee64ef5bef7286d65a5eba6cc457403..cdc7cafab572651e92a3f0e77dd82d0e67642c79 100644 --- a/include/rdma/rdma_cm.h +++ b/include/rdma/rdma_cm.h @@ -49,10 +49,21 @@ struct rdma_addr { struct rdma_dev_addr dev_addr; }; +#define RDMA_PRIMARY_PATH_MAX_REC_NUM 3 struct rdma_route { struct rdma_addr addr; struct sa_path_rec *path_rec; - int num_paths; + + /* Optional path records of primary path */ + struct sa_path_rec *path_rec_inbound; + struct sa_path_rec *path_rec_outbound; + + /* + * 0 - No primary nor alternate path is available + * 1 - Only primary path is available + * 2 - Both primary and alternate path are available + */ + int num_pri_alt_paths; }; struct rdma_conn_param { diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h index 2dafd7dbe89304d4198d194b2f454bfc875cca26..c429d6ddb1292efbb4297085c09fad927095fad0 100644 --- a/include/rdma/rdma_vt.h +++ b/include/rdma/rdma_vt.h @@ -445,7 +445,7 @@ static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi, * to work by setting the name manually here. */ dev_set_name(&rdi->ibdev.dev, fmt, name, unit); - strlcpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX); + strscpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX); } /** diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 23bb404aba12c05dada31f18afc9d922e6bf6f18..9d45a5b203169e1761e9108045202984e12d219a 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -24,6 +24,7 @@ enum uverbs_attr_type { UVERBS_ATTR_TYPE_PTR_OUT, UVERBS_ATTR_TYPE_IDR, UVERBS_ATTR_TYPE_FD, + UVERBS_ATTR_TYPE_RAW_FD, UVERBS_ATTR_TYPE_ENUM_IN, UVERBS_ATTR_TYPE_IDRS_ARRAY, }; @@ -521,6 +522,11 @@ struct uapi_definition { .u.obj.access = _access, \ __VA_ARGS__ } }) +#define UVERBS_ATTR_RAW_FD(_attr_id, ...) \ + (&(const struct uverbs_attr_def){ \ + .id = (_attr_id), \ + .attr = { .type = UVERBS_ATTR_TYPE_RAW_FD, __VA_ARGS__ } }) + #define UVERBS_ATTR_PTR_IN(_attr_id, _type, ...) \ (&(const struct uverbs_attr_def){ \ .id = _attr_id, \ @@ -999,4 +1005,11 @@ _uverbs_get_const_unsigned(u64 *to, uverbs_get_const_default_unsigned(_to, _attrs_bundle, _idx, \ _default)) +static inline int +uverbs_get_raw_fd(int *to, const struct uverbs_attr_bundle *attrs_bundle, + size_t idx) +{ + return uverbs_get_const_signed(to, attrs_bundle, idx); +} + #endif diff --git a/include/scsi/scsi_cmnd.h b/include/scsi/scsi_cmnd.h index bac55decf900a8d6edc7e435d224010cc41c50b6..7d3622db38edccfdce1bf2a90bfedd80f0218abd 100644 --- a/include/scsi/scsi_cmnd.h +++ b/include/scsi/scsi_cmnd.h @@ -201,7 +201,7 @@ static inline unsigned int scsi_get_resid(struct scsi_cmnd *cmd) for_each_sg(scsi_sglist(cmd), sg, nseg, __i) static inline int scsi_sg_copy_from_buffer(struct scsi_cmnd *cmd, - void *buf, int buflen) + const void *buf, int buflen) { return sg_copy_from_buffer(scsi_sglist(cmd), scsi_sg_count(cmd), buf, buflen); diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h index 3113471ca375bb1d05874293d6f17d7aa768e472..c36656d8ac6c758af020f077488a4ab648805ed9 100644 --- a/include/scsi/scsi_device.h +++ b/include/scsi/scsi_device.h @@ -231,6 +231,7 @@ struct scsi_device { atomic_t iorequest_cnt; atomic_t iodone_cnt; atomic_t ioerr_cnt; + atomic_t iotmo_cnt; struct device sdev_gendev, sdev_dev; @@ -309,8 +310,6 @@ struct scsi_target { struct list_head devices; struct device dev; struct kref reap_ref; /* last put renders target invisible */ - atomic_t sdev_count; - wait_queue_head_t sdev_wq; unsigned int channel; unsigned int id; /* target id ... replace * scsi_device.id eventually */ diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index aa7b7496c93aa16a6c1487425ec1de6dc862923b..fcf25f1642a3a977eb60dda92a5bbad506eb30d9 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -276,7 +276,7 @@ struct scsi_host_template { * * Status: OPTIONAL */ - int (* map_queues)(struct Scsi_Host *shost); + void (* map_queues)(struct Scsi_Host *shost); /* * SCSI interface of blk_poll - poll for IO completions. @@ -557,6 +557,8 @@ struct Scsi_Host { struct scsi_host_template *hostt; struct scsi_transport_template *transportt; + struct kref tagset_refcnt; + struct completion tagset_freed; /* Area to keep a shared tag map */ struct blk_mq_tag_set tag_set; @@ -690,9 +692,6 @@ struct Scsi_Host { /* ldm bits */ struct device shost_gendev, shost_dev; - atomic_t target_count; - wait_queue_head_t targets_wq; - /* * Points to the transport data (if any) which is allocated * separately diff --git a/include/scsi/scsi_status.h b/include/scsi/scsi_status.h index 31d30cee18691a5ed9a0218182d763f2a6785ce3..9cb85262de64fafe091f743a5ef3563d271801da 100644 --- a/include/scsi/scsi_status.h +++ b/include/scsi/scsi_status.h @@ -62,12 +62,12 @@ enum scsi_host_status { * recover the link. Transport class will * retry or fail IO */ DID_TRANSPORT_FAILFAST = 0x0f, /* Transport class fastfailed the io */ - DID_TARGET_FAILURE = 0x10, /* Permanent target failure, do not retry on - * other paths */ - DID_NEXUS_FAILURE = 0x11, /* Permanent nexus failure, retry on other - * paths might yield different results */ - DID_ALLOC_FAILURE = 0x12, /* Space allocation on the device failed */ - DID_MEDIUM_ERROR = 0x13, /* Medium error */ + /* + * We used to have DID_TARGET_FAILURE, DID_NEXUS_FAILURE, + * DID_ALLOC_FAILURE and DID_MEDIUM_ERROR at 0x10 - 0x13. For compat + * with userspace apps that parse the host byte for SG IO, we leave + * that block of codes unused and start at 0x14 below. + */ DID_TRANSPORT_MARGINAL = 0x14, /* Transport marginal errors */ }; diff --git a/include/soc/at91/pm.h b/include/soc/at91/pm.h deleted file mode 100644 index 7a41e53a3ffa3e90c7ddee987db670587ec1284a..0000000000000000000000000000000000000000 --- a/include/soc/at91/pm.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Atmel Power Management - * - * Copyright (C) 2020 Atmel - * - * Author: Lee Jones - */ - -#ifndef __SOC_ATMEL_PM_H -#define __SOC_ATMEL_PM_H - -void at91_pinctrl_gpio_suspend(void); -void at91_pinctrl_gpio_resume(void); - -#endif /* __SOC_ATMEL_PM_H */ diff --git a/include/soc/at91/sama7-ddr.h b/include/soc/at91/sama7-ddr.h index 9e17247474fa92db82521d76f8a73c57d254de78..6ce3bd22f6c693240c0592479f4ddc2ab5d1ffc4 100644 --- a/include/soc/at91/sama7-ddr.h +++ b/include/soc/at91/sama7-ddr.h @@ -38,6 +38,14 @@ #define DDR3PHY_DSGCR_ODTPDD_ODT0 (1 << 20) /* ODT[0] Power Down Driver */ #define DDR3PHY_ZQ0SR0 (0x188) /* ZQ status register 0 */ +#define DDR3PHY_ZQ0SR0_PDO_OFF (0) /* Pull-down output impedance select offset */ +#define DDR3PHY_ZQ0SR0_PUO_OFF (5) /* Pull-up output impedance select offset */ +#define DDR3PHY_ZQ0SR0_PDODT_OFF (10) /* Pull-down on-die termination impedance select offset */ +#define DDR3PHY_ZQ0SRO_PUODT_OFF (15) /* Pull-up on-die termination impedance select offset */ + +#define DDR3PHY_DX0DLLCR (0x1CC) /* DDR3PHY DATX8 DLL Control Register */ +#define DDR3PHY_DX1DLLCR (0x20C) /* DDR3PHY DATX8 DLL Control Register */ +#define DDR3PHY_DXDLLCR_DLLDIS (1 << 31) /* DLL Disable */ /* UDDRC */ #define UDDRC_STAT (0x04) /* UDDRC Operating Mode Status Register */ diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h index 59eeba31c19289e260d7529836c08b5c1c75be68..0d3d6beb7fdbdeb918d1db32ee29ddc65526792c 100644 --- a/include/soc/fsl/qman.h +++ b/include/soc/fsl/qman.h @@ -1171,6 +1171,15 @@ int qman_delete_cgr(struct qman_cgr *cgr); */ void qman_delete_cgr_safe(struct qman_cgr *cgr); +/** + * qman_update_cgr_safe - Modifies a congestion group object from any CPU + * @cgr: the 'cgr' object to modify + * @opts: state of the CGR settings + * + * This will select the proper CPU and modify the CGR settings. + */ +int qman_update_cgr_safe(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts); + /** * qman_query_cgr_congested - Queries CGR's congestion status * @cgr: the 'cgr' object to query diff --git a/include/soc/mediatek/smi.h b/include/soc/mediatek/smi.h index 11f7d6b5964269cfe58bfadeb18358586c5b339b..dfd8efca5e606accf947ff3878a28791d4bd58de 100644 --- a/include/soc/mediatek/smi.h +++ b/include/soc/mediatek/smi.h @@ -11,6 +11,11 @@ #if IS_ENABLED(CONFIG_MTK_SMI) +enum iommu_atf_cmd { + IOMMU_ATF_CMD_CONFIG_SMI_LARB, /* For mm master to en/disable iommu */ + IOMMU_ATF_CMD_MAX, +}; + #define MTK_SMI_MMU_EN(port) BIT(port) struct mtk_smi_larb_iommu { diff --git a/include/soc/microchip/mpfs.h b/include/soc/microchip/mpfs.h index 6466515262bdb380ba31000206eeddb018d42a3f..f916dcde457f4cadf5fe75ece9b438d211c0462d 100644 --- a/include/soc/microchip/mpfs.h +++ b/include/soc/microchip/mpfs.h @@ -40,4 +40,12 @@ struct mpfs_sys_controller *mpfs_sys_controller_get(struct device *dev); #endif /* if IS_ENABLED(CONFIG_POLARFIRE_SOC_SYS_CTRL) */ +#if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) + +u32 mpfs_reset_read(struct device *dev); + +void mpfs_reset_write(struct device *dev, u32 val); + +#endif /* if IS_ENABLED(CONFIG_MCHP_CLK_MPFS) */ + #endif /* __SOC_MPFS_H__ */ diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index 2edea901bbd5a89368c7146bbc62e7e723c93b04..967ba30ea63639417e25be5d5bdc382282df0eb9 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -392,7 +392,7 @@ enum ocelot_reg { SYS_COUNT_TX_GREEN_PRIO_5, SYS_COUNT_TX_GREEN_PRIO_6, SYS_COUNT_TX_GREEN_PRIO_7, - SYS_COUNT_TX_AGING, + SYS_COUNT_TX_AGED, SYS_COUNT_DROP_LOCAL, SYS_COUNT_DROP_TAIL, SYS_COUNT_DROP_YELLOW_PRIO_0, @@ -411,6 +411,10 @@ enum ocelot_reg { SYS_COUNT_DROP_GREEN_PRIO_5, SYS_COUNT_DROP_GREEN_PRIO_6, SYS_COUNT_DROP_GREEN_PRIO_7, + SYS_COUNT_SF_MATCHING_FRAMES, + SYS_COUNT_SF_NOT_PASSING_FRAMES, + SYS_COUNT_SF_NOT_PASSING_SDU, + SYS_COUNT_SF_RED_FRAMES, SYS_RESET_CFG, SYS_SR_ETYPE_CFG, SYS_VLAN_ETYPE_CFG, @@ -433,7 +437,6 @@ enum ocelot_reg { SYS_MMGT_FAST, SYS_EVENTS_DIF, SYS_EVENTS_CORE, - SYS_CNT, SYS_PTP_STATUS, SYS_PTP_TXSTAMP, SYS_PTP_NXT, @@ -695,6 +698,112 @@ struct ocelot_stat_layout { char name[ETH_GSTRING_LEN]; }; +/* 32-bit counter checked for wraparound by ocelot_port_update_stats() + * and copied to ocelot->stats. + */ +#define OCELOT_STAT(kind) \ + [OCELOT_STAT_ ## kind] = { .reg = SYS_COUNT_ ## kind } +/* Same as above, except also exported to ethtool -S. Standard counters should + * only be exposed to more specific interfaces rather than by their string name. + */ +#define OCELOT_STAT_ETHTOOL(kind, ethtool_name) \ + [OCELOT_STAT_ ## kind] = { .reg = SYS_COUNT_ ## kind, .name = ethtool_name } + +#define OCELOT_COMMON_STATS \ + OCELOT_STAT_ETHTOOL(RX_OCTETS, "rx_octets"), \ + OCELOT_STAT_ETHTOOL(RX_UNICAST, "rx_unicast"), \ + OCELOT_STAT_ETHTOOL(RX_MULTICAST, "rx_multicast"), \ + OCELOT_STAT_ETHTOOL(RX_BROADCAST, "rx_broadcast"), \ + OCELOT_STAT_ETHTOOL(RX_SHORTS, "rx_shorts"), \ + OCELOT_STAT_ETHTOOL(RX_FRAGMENTS, "rx_fragments"), \ + OCELOT_STAT_ETHTOOL(RX_JABBERS, "rx_jabbers"), \ + OCELOT_STAT_ETHTOOL(RX_CRC_ALIGN_ERRS, "rx_crc_align_errs"), \ + OCELOT_STAT_ETHTOOL(RX_SYM_ERRS, "rx_sym_errs"), \ + OCELOT_STAT_ETHTOOL(RX_64, "rx_frames_below_65_octets"), \ + OCELOT_STAT_ETHTOOL(RX_65_127, "rx_frames_65_to_127_octets"), \ + OCELOT_STAT_ETHTOOL(RX_128_255, "rx_frames_128_to_255_octets"), \ + OCELOT_STAT_ETHTOOL(RX_256_511, "rx_frames_256_to_511_octets"), \ + OCELOT_STAT_ETHTOOL(RX_512_1023, "rx_frames_512_to_1023_octets"), \ + OCELOT_STAT_ETHTOOL(RX_1024_1526, "rx_frames_1024_to_1526_octets"), \ + OCELOT_STAT_ETHTOOL(RX_1527_MAX, "rx_frames_over_1526_octets"), \ + OCELOT_STAT_ETHTOOL(RX_PAUSE, "rx_pause"), \ + OCELOT_STAT_ETHTOOL(RX_CONTROL, "rx_control"), \ + OCELOT_STAT_ETHTOOL(RX_LONGS, "rx_longs"), \ + OCELOT_STAT_ETHTOOL(RX_CLASSIFIED_DROPS, "rx_classified_drops"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_0, "rx_red_prio_0"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_1, "rx_red_prio_1"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_2, "rx_red_prio_2"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_3, "rx_red_prio_3"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_4, "rx_red_prio_4"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_5, "rx_red_prio_5"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_6, "rx_red_prio_6"), \ + OCELOT_STAT_ETHTOOL(RX_RED_PRIO_7, "rx_red_prio_7"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_0, "rx_yellow_prio_0"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_1, "rx_yellow_prio_1"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_2, "rx_yellow_prio_2"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_3, "rx_yellow_prio_3"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_4, "rx_yellow_prio_4"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_5, "rx_yellow_prio_5"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_6, "rx_yellow_prio_6"), \ + OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_7, "rx_yellow_prio_7"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_0, "rx_green_prio_0"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_1, "rx_green_prio_1"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_2, "rx_green_prio_2"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_3, "rx_green_prio_3"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_4, "rx_green_prio_4"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_5, "rx_green_prio_5"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_6, "rx_green_prio_6"), \ + OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_7, "rx_green_prio_7"), \ + OCELOT_STAT_ETHTOOL(TX_OCTETS, "tx_octets"), \ + OCELOT_STAT_ETHTOOL(TX_UNICAST, "tx_unicast"), \ + OCELOT_STAT_ETHTOOL(TX_MULTICAST, "tx_multicast"), \ + OCELOT_STAT_ETHTOOL(TX_BROADCAST, "tx_broadcast"), \ + OCELOT_STAT_ETHTOOL(TX_COLLISION, "tx_collision"), \ + OCELOT_STAT_ETHTOOL(TX_DROPS, "tx_drops"), \ + OCELOT_STAT_ETHTOOL(TX_PAUSE, "tx_pause"), \ + OCELOT_STAT_ETHTOOL(TX_64, "tx_frames_below_65_octets"), \ + OCELOT_STAT_ETHTOOL(TX_65_127, "tx_frames_65_to_127_octets"), \ + OCELOT_STAT_ETHTOOL(TX_128_255, "tx_frames_128_255_octets"), \ + OCELOT_STAT_ETHTOOL(TX_256_511, "tx_frames_256_511_octets"), \ + OCELOT_STAT_ETHTOOL(TX_512_1023, "tx_frames_512_1023_octets"), \ + OCELOT_STAT_ETHTOOL(TX_1024_1526, "tx_frames_1024_1526_octets"), \ + OCELOT_STAT_ETHTOOL(TX_1527_MAX, "tx_frames_over_1526_octets"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_0, "tx_yellow_prio_0"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_1, "tx_yellow_prio_1"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_2, "tx_yellow_prio_2"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_3, "tx_yellow_prio_3"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_4, "tx_yellow_prio_4"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_5, "tx_yellow_prio_5"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_6, "tx_yellow_prio_6"), \ + OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_7, "tx_yellow_prio_7"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_0, "tx_green_prio_0"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_1, "tx_green_prio_1"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_2, "tx_green_prio_2"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_3, "tx_green_prio_3"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_4, "tx_green_prio_4"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_5, "tx_green_prio_5"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_6, "tx_green_prio_6"), \ + OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_7, "tx_green_prio_7"), \ + OCELOT_STAT_ETHTOOL(TX_AGED, "tx_aged"), \ + OCELOT_STAT_ETHTOOL(DROP_LOCAL, "drop_local"), \ + OCELOT_STAT_ETHTOOL(DROP_TAIL, "drop_tail"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_0, "drop_yellow_prio_0"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_1, "drop_yellow_prio_1"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_2, "drop_yellow_prio_2"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_3, "drop_yellow_prio_3"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_4, "drop_yellow_prio_4"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_5, "drop_yellow_prio_5"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_6, "drop_yellow_prio_6"), \ + OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_7, "drop_yellow_prio_7"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_0, "drop_green_prio_0"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_1, "drop_green_prio_1"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_2, "drop_green_prio_2"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_3, "drop_green_prio_3"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_4, "drop_green_prio_4"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_5, "drop_green_prio_5"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_6, "drop_green_prio_6"), \ + OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_7, "drop_green_prio_7") + struct ocelot_stats_region { struct list_head node; u32 base; @@ -726,6 +835,7 @@ struct ocelot_ops { struct flow_stats *stats); void (*cut_through_fwd)(struct ocelot *ocelot); void (*tas_clock_adjust)(struct ocelot *ocelot); + void (*update_stats)(struct ocelot *ocelot); }; struct ocelot_vcap_policer { @@ -763,6 +873,8 @@ struct ocelot_psfp_list { struct list_head stream_list; struct list_head sfi_list; struct list_head sgi_list; + /* Serialize access to the lists */ + struct mutex lock; }; enum ocelot_sb { @@ -898,12 +1010,15 @@ struct ocelot { struct ocelot_psfp_list psfp; - /* Workqueue to check statistics for overflow with its lock */ - spinlock_t stats_lock; - u64 *stats; + /* Workqueue to check statistics for overflow */ struct delayed_work stats_work; struct workqueue_struct *stats_queue; + /* Lock for serializing access to the statistics array */ + spinlock_t stats_lock; + u64 *stats; + /* Lock for serializing indirect access to STAT_VIEW registers */ + struct mutex stat_view_lock; /* Lock for serializing access to the MAC table */ struct mutex mact_lock; /* Lock for serializing forwarding domain changes */ @@ -1024,6 +1139,8 @@ void ocelot_deinit(struct ocelot *ocelot); void ocelot_init_port(struct ocelot *ocelot, int port); void ocelot_deinit_port(struct ocelot *ocelot, int port); +void ocelot_port_setup_dsa_8021q_cpu(struct ocelot *ocelot, int cpu); +void ocelot_port_teardown_dsa_8021q_cpu(struct ocelot *ocelot, int cpu); void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port, int cpu); void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port); u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port); @@ -1032,6 +1149,19 @@ u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port); void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data); void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data); int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset); +void ocelot_port_get_stats64(struct ocelot *ocelot, int port, + struct rtnl_link_stats64 *stats); +void ocelot_port_get_pause_stats(struct ocelot *ocelot, int port, + struct ethtool_pause_stats *pause_stats); +void ocelot_port_get_rmon_stats(struct ocelot *ocelot, int port, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges); +void ocelot_port_get_eth_ctrl_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_ctrl_stats *ctrl_stats); +void ocelot_port_get_eth_mac_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_mac_stats *mac_stats); +void ocelot_port_get_eth_phy_stats(struct ocelot *ocelot, int port, + struct ethtool_eth_phy_stats *phy_stats); int ocelot_get_ts_info(struct ocelot *ocelot, int port, struct ethtool_ts_info *info); void ocelot_set_ageing_time(struct ocelot *ocelot, unsigned int msecs); @@ -1099,10 +1229,12 @@ int ocelot_port_mdb_del(struct ocelot *ocelot, int port, const struct net_device *bridge); int ocelot_port_lag_join(struct ocelot *ocelot, int port, struct net_device *bond, - struct netdev_lag_upper_info *info); + struct netdev_lag_upper_info *info, + struct netlink_ext_ack *extack); void ocelot_port_lag_leave(struct ocelot *ocelot, int port, struct net_device *bond); void ocelot_port_lag_change(struct ocelot *ocelot, int port, bool lag_tx_active); +int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond); int ocelot_devlink_sb_register(struct ocelot *ocelot); void ocelot_devlink_sb_unregister(struct ocelot *ocelot); diff --git a/include/soc/sifive/sifive_ccache.h b/include/soc/sifive/sifive_ccache.h new file mode 100644 index 0000000000000000000000000000000000000000..4d4ed49388a0a1968d84f405b9e9b2a98572bb3c --- /dev/null +++ b/include/soc/sifive/sifive_ccache.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * SiFive Composable Cache Controller header file + * + */ + +#ifndef __SOC_SIFIVE_CCACHE_H +#define __SOC_SIFIVE_CCACHE_H + +extern int register_sifive_ccache_error_notifier(struct notifier_block *nb); +extern int unregister_sifive_ccache_error_notifier(struct notifier_block *nb); + +#define SIFIVE_CCACHE_ERR_TYPE_CE 0 +#define SIFIVE_CCACHE_ERR_TYPE_UE 1 + +#endif /* __SOC_SIFIVE_CCACHE_H */ diff --git a/include/soc/sifive/sifive_l2_cache.h b/include/soc/sifive/sifive_l2_cache.h deleted file mode 100644 index 92ade10ed67e94bac8439512cbe3a628958e7b6b..0000000000000000000000000000000000000000 --- a/include/soc/sifive/sifive_l2_cache.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SiFive L2 Cache Controller header file - * - */ - -#ifndef __SOC_SIFIVE_L2_CACHE_H -#define __SOC_SIFIVE_L2_CACHE_H - -extern int register_sifive_l2_error_notifier(struct notifier_block *nb); -extern int unregister_sifive_l2_error_notifier(struct notifier_block *nb); - -#define SIFIVE_L2_ERR_TYPE_CE 0 -#define SIFIVE_L2_ERR_TYPE_UE 1 - -#endif /* __SOC_SIFIVE_L2_CACHE_H */ diff --git a/include/soc/tegra/fuse.h b/include/soc/tegra/fuse.h index 67d2bc856fbc0854c280b07f96f65905360b41e2..977c334136e91090cf4086a08c28b895ac3109a0 100644 --- a/include/soc/tegra/fuse.h +++ b/include/soc/tegra/fuse.h @@ -58,6 +58,7 @@ u32 tegra_read_chipid(void); u8 tegra_get_chip_id(void); u8 tegra_get_platform(void); bool tegra_is_silicon(void); +int tegra194_miscreg_mask_serror(void); #else static struct tegra_sku_info tegra_sku_info __maybe_unused; @@ -95,6 +96,11 @@ static inline bool tegra_is_silicon(void) { return false; } + +static inline int tegra194_miscreg_mask_serror(void) +{ + return false; +} #endif struct device *tegra_soc_device_register(void); diff --git a/include/soc/tegra/tegra-cbb.h b/include/soc/tegra/tegra-cbb.h new file mode 100644 index 0000000000000000000000000000000000000000..e864c2ebe794e4f8962f0a97ff74e896216de957 --- /dev/null +++ b/include/soc/tegra/tegra-cbb.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved + */ + +#ifndef TEGRA_CBB_H +#define TEGRA_CBB_H + +#include + +struct tegra_cbb_error { + const char *code; + const char *source; + const char *desc; +}; + +struct tegra_cbb { + struct device *dev; + const struct tegra_cbb_ops *ops; + struct list_head node; +}; + +struct tegra_cbb_ops { + int (*debugfs_show)(struct tegra_cbb *cbb, struct seq_file *s, void *v); + int (*interrupt_enable)(struct tegra_cbb *cbb); + void (*error_enable)(struct tegra_cbb *cbb); + void (*fault_enable)(struct tegra_cbb *cbb); + void (*stall_enable)(struct tegra_cbb *cbb); + void (*error_clear)(struct tegra_cbb *cbb); + u32 (*get_status)(struct tegra_cbb *cbb); +}; + +int tegra_cbb_get_irq(struct platform_device *pdev, unsigned int *nonsec_irq, + unsigned int *sec_irq); +__printf(2, 3) +void tegra_cbb_print_err(struct seq_file *file, const char *fmt, ...); + +void tegra_cbb_print_cache(struct seq_file *file, u32 cache); +void tegra_cbb_print_prot(struct seq_file *file, u32 prot); +int tegra_cbb_register(struct tegra_cbb *cbb); + +void tegra_cbb_fault_enable(struct tegra_cbb *cbb); +void tegra_cbb_stall_enable(struct tegra_cbb *cbb); +void tegra_cbb_error_clear(struct tegra_cbb *cbb); +u32 tegra_cbb_get_status(struct tegra_cbb *cbb); + +#endif /* TEGRA_CBB_H */ diff --git a/include/sound/acp62_chip_offset_byte.h b/include/sound/acp62_chip_offset_byte.h new file mode 100644 index 0000000000000000000000000000000000000000..f03992f81168bc64b8225007a235da3f5ee822ee --- /dev/null +++ b/include/sound/acp62_chip_offset_byte.h @@ -0,0 +1,444 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * AMD ACP 6.2 Register Documentation + * + * Copyright 2022 Advanced Micro Devices, Inc. + */ + +#ifndef _acp_ip_OFFSET_HEADER +#define _acp_ip_OFFSET_HEADER + +/* Registers from ACP_DMA block */ +#define ACP_DMA_CNTL_0 0x0000000 +#define ACP_DMA_CNTL_1 0x0000004 +#define ACP_DMA_CNTL_2 0x0000008 +#define ACP_DMA_CNTL_3 0x000000C +#define ACP_DMA_CNTL_4 0x0000010 +#define ACP_DMA_CNTL_5 0x0000014 +#define ACP_DMA_CNTL_6 0x0000018 +#define ACP_DMA_CNTL_7 0x000001C +#define ACP_DMA_DSCR_STRT_IDX_0 0x0000020 +#define ACP_DMA_DSCR_STRT_IDX_1 0x0000024 +#define ACP_DMA_DSCR_STRT_IDX_2 0x0000028 +#define ACP_DMA_DSCR_STRT_IDX_3 0x000002C +#define ACP_DMA_DSCR_STRT_IDX_4 0x0000030 +#define ACP_DMA_DSCR_STRT_IDX_5 0x0000034 +#define ACP_DMA_DSCR_STRT_IDX_6 0x0000038 +#define ACP_DMA_DSCR_STRT_IDX_7 0x000003C +#define ACP_DMA_DSCR_CNT_0 0x0000040 +#define ACP_DMA_DSCR_CNT_1 0x0000044 +#define ACP_DMA_DSCR_CNT_2 0x0000048 +#define ACP_DMA_DSCR_CNT_3 0x000004C +#define ACP_DMA_DSCR_CNT_4 0x0000050 +#define ACP_DMA_DSCR_CNT_5 0x0000054 +#define ACP_DMA_DSCR_CNT_6 0x0000058 +#define ACP_DMA_DSCR_CNT_7 0x000005C +#define ACP_DMA_PRIO_0 0x0000060 +#define ACP_DMA_PRIO_1 0x0000064 +#define ACP_DMA_PRIO_2 0x0000068 +#define ACP_DMA_PRIO_3 0x000006C +#define ACP_DMA_PRIO_4 0x0000070 +#define ACP_DMA_PRIO_5 0x0000074 +#define ACP_DMA_PRIO_6 0x0000078 +#define ACP_DMA_PRIO_7 0x000007C +#define ACP_DMA_CUR_DSCR_0 0x0000080 +#define ACP_DMA_CUR_DSCR_1 0x0000084 +#define ACP_DMA_CUR_DSCR_2 0x0000088 +#define ACP_DMA_CUR_DSCR_3 0x000008C +#define ACP_DMA_CUR_DSCR_4 0x0000090 +#define ACP_DMA_CUR_DSCR_5 0x0000094 +#define ACP_DMA_CUR_DSCR_6 0x0000098 +#define ACP_DMA_CUR_DSCR_7 0x000009C +#define ACP_DMA_CUR_TRANS_CNT_0 0x00000A0 +#define ACP_DMA_CUR_TRANS_CNT_1 0x00000A4 +#define ACP_DMA_CUR_TRANS_CNT_2 0x00000A8 +#define ACP_DMA_CUR_TRANS_CNT_3 0x00000AC +#define ACP_DMA_CUR_TRANS_CNT_4 0x00000B0 +#define ACP_DMA_CUR_TRANS_CNT_5 0x00000B4 +#define ACP_DMA_CUR_TRANS_CNT_6 0x00000B8 +#define ACP_DMA_CUR_TRANS_CNT_7 0x00000BC +#define ACP_DMA_ERR_STS_0 0x00000C0 +#define ACP_DMA_ERR_STS_1 0x00000C4 +#define ACP_DMA_ERR_STS_2 0x00000C8 +#define ACP_DMA_ERR_STS_3 0x00000CC +#define ACP_DMA_ERR_STS_4 0x00000D0 +#define ACP_DMA_ERR_STS_5 0x00000D4 +#define ACP_DMA_ERR_STS_6 0x00000D8 +#define ACP_DMA_ERR_STS_7 0x00000DC +#define ACP_DMA_DESC_BASE_ADDR 0x00000E0 +#define ACP_DMA_DESC_MAX_NUM_DSCR 0x00000E4 +#define ACP_DMA_CH_STS 0x00000E8 +#define ACP_DMA_CH_GROUP 0x00000EC +#define ACP_DMA_CH_RST_STS 0x00000F0 + +/* Registers from ACP_AXI2AXIATU block */ +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1 0x0000C00 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_1 0x0000C04 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_2 0x0000C08 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_2 0x0000C0C +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_3 0x0000C10 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_3 0x0000C14 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_4 0x0000C18 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_4 0x0000C1C +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_5 0x0000C20 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_5 0x0000C24 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_6 0x0000C28 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_6 0x0000C2C +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_7 0x0000C30 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_7 0x0000C34 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_8 0x0000C38 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_8 0x0000C3C +#define ACPAXI2AXI_ATU_CTRL 0x0000C40 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_9 0x0000C44 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_9 0x0000C48 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_10 0x0000C4C +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_10 0x0000C50 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_11 0x0000C54 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_11 0x0000C58 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_12 0x0000C5C +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_12 0x0000C60 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_13 0x0000C64 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_13 0x0000C68 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_14 0x0000C6C +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_14 0x0000C70 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_15 0x0000C74 +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_15 0x0000C78 +#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_16 0x0000C7C +#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_16 0x0000C80 + +/* Registers from ACP_CLKRST block */ +#define ACP_SOFT_RESET 0x0001000 +#define ACP_CONTROL 0x0001004 +#define ACP_STATUS 0x0001008 +#define ACP_DYNAMIC_CG_MASTER_CONTROL 0x0001010 +#define ACP_ZSC_DSP_CTRL 0x0001014 +#define ACP_ZSC_STS 0x0001018 +#define ACP_PGFSM_CONTROL 0x0001024 +#define ACP_PGFSM_STATUS 0x0001028 +#define ACP_CLKMUX_SEL 0x000102C + +/* Registers from ACP_AON block */ +#define ACP_PME_EN 0x0001400 +#define ACP_DEVICE_STATE 0x0001404 +#define AZ_DEVICE_STATE 0x0001408 +#define ACP_PIN_CONFIG 0x0001440 +#define ACP_PAD_PULLUP_CTRL 0x0001444 +#define ACP_PAD_PULLDOWN_CTRL 0x0001448 +#define ACP_PAD_DRIVE_STRENGTH_CTRL 0x000144C +#define ACP_PAD_SCHMEN_CTRL 0x0001450 +#define ACP_SW_PAD_KEEPER_EN 0x0001454 +#define ACP_SW_WAKE_EN 0x0001458 +#define ACP_I2S_WAKE_EN 0x000145C +#define ACP_SW1_WAKE_EN 0x0001460 + +/* Registers from ACP_P1_MISC block */ +#define ACP_EXTERNAL_INTR_ENB 0x0001A00 +#define ACP_EXTERNAL_INTR_CNTL 0x0001A04 +#define ACP_EXTERNAL_INTR_CNTL1 0x0001A08 +#define ACP_EXTERNAL_INTR_STAT 0x0001A0C +#define ACP_EXTERNAL_INTR_STAT1 0x0001A10 +#define ACP_ERROR_STATUS 0x0001A4C +#define ACP_P1_SW_I2S_ERROR_REASON 0x0001A50 +#define ACP_P1_SW_POS_TRACK_I2S_TX_CTRL 0x0001A6C +#define ACP_P1_SW_I2S_TX_DMA_POS 0x0001A70 +#define ACP_P1_SW_POS_TRACK_I2S_RX_CTRL 0x0001A74 +#define ACP_P1_SW_I2S_RX_DMA_POS 0x0001A78 +#define ACP_P1_DMIC_I2S_GPIO_INTR_CTRL 0x0001A7C +#define ACP_P1_DMIC_I2S_GPIO_INTR_STATUS 0x0001A80 +#define ACP_SCRATCH_REG_BASE_ADDR 0x0001A84 +#define ACP_P1_SW_POS_TRACK_BT_TX_CTRL 0x0001A88 +#define ACP_P1_SW_BT_TX_DMA_POS 0x0001A8C +#define ACP_P1_SW_POS_TRACK_HS_TX_CTRL 0x0001A90 +#define ACP_P1_SW_HS_TX_DMA_POS 0x0001A94 +#define ACP_P1_SW_POS_TRACK_BT_RX_CTRL 0x0001A98 +#define ACP_P1_SW_BT_RX_DMA_POS 0x0001A9C +#define ACP_P1_SW_POS_TRACK_HS_RX_CTRL 0x0001AA0 +#define ACP_P1_SW_HS_RX_DMA_POS 0x0001AA4 + +/* Registers from ACP_AUDIO_BUFFERS block */ +#define ACP_I2S_RX_RINGBUFADDR 0x0002000 +#define ACP_I2S_RX_RINGBUFSIZE 0x0002004 +#define ACP_I2S_RX_LINKPOSITIONCNTR 0x0002008 +#define ACP_I2S_RX_FIFOADDR 0x000200C +#define ACP_I2S_RX_FIFOSIZE 0x0002010 +#define ACP_I2S_RX_DMA_SIZE 0x0002014 +#define ACP_I2S_RX_LINEARPOSITIONCNTR_HIGH 0x0002018 +#define ACP_I2S_RX_LINEARPOSITIONCNTR_LOW 0x000201C +#define ACP_I2S_RX_INTR_WATERMARK_SIZE 0x0002020 +#define ACP_I2S_TX_RINGBUFADDR 0x0002024 +#define ACP_I2S_TX_RINGBUFSIZE 0x0002028 +#define ACP_I2S_TX_LINKPOSITIONCNTR 0x000202C +#define ACP_I2S_TX_FIFOADDR 0x0002030 +#define ACP_I2S_TX_FIFOSIZE 0x0002034 +#define ACP_I2S_TX_DMA_SIZE 0x0002038 +#define ACP_I2S_TX_LINEARPOSITIONCNTR_HIGH 0x000203C +#define ACP_I2S_TX_LINEARPOSITIONCNTR_LOW 0x0002040 +#define ACP_I2S_TX_INTR_WATERMARK_SIZE 0x0002044 +#define ACP_BT_RX_RINGBUFADDR 0x0002048 +#define ACP_BT_RX_RINGBUFSIZE 0x000204C +#define ACP_BT_RX_LINKPOSITIONCNTR 0x0002050 +#define ACP_BT_RX_FIFOADDR 0x0002054 +#define ACP_BT_RX_FIFOSIZE 0x0002058 +#define ACP_BT_RX_DMA_SIZE 0x000205C +#define ACP_BT_RX_LINEARPOSITIONCNTR_HIGH 0x0002060 +#define ACP_BT_RX_LINEARPOSITIONCNTR_LOW 0x0002064 +#define ACP_BT_RX_INTR_WATERMARK_SIZE 0x0002068 +#define ACP_BT_TX_RINGBUFADDR 0x000206C +#define ACP_BT_TX_RINGBUFSIZE 0x0002070 +#define ACP_BT_TX_LINKPOSITIONCNTR 0x0002074 +#define ACP_BT_TX_FIFOADDR 0x0002078 +#define ACP_BT_TX_FIFOSIZE 0x000207C +#define ACP_BT_TX_DMA_SIZE 0x0002080 +#define ACP_BT_TX_LINEARPOSITIONCNTR_HIGH 0x0002084 +#define ACP_BT_TX_LINEARPOSITIONCNTR_LOW 0x0002088 +#define ACP_BT_TX_INTR_WATERMARK_SIZE 0x000208C +#define ACP_HS_RX_RINGBUFADDR 0x0002090 +#define ACP_HS_RX_RINGBUFSIZE 0x0002094 +#define ACP_HS_RX_LINKPOSITIONCNTR 0x0002098 +#define ACP_HS_RX_FIFOADDR 0x000209C +#define ACP_HS_RX_FIFOSIZE 0x00020A0 +#define ACP_HS_RX_DMA_SIZE 0x00020A4 +#define ACP_HS_RX_LINEARPOSITIONCNTR_HIGH 0x00020A8 +#define ACP_HS_RX_LINEARPOSITIONCNTR_LOW 0x00020AC +#define ACP_HS_RX_INTR_WATERMARK_SIZE 0x00020B0 +#define ACP_HS_TX_RINGBUFADDR 0x00020B4 +#define ACP_HS_TX_RINGBUFSIZE 0x00020B8 +#define ACP_HS_TX_LINKPOSITIONCNTR 0x00020BC +#define ACP_HS_TX_FIFOADDR 0x00020C0 +#define ACP_HS_TX_FIFOSIZE 0x00020C4 +#define ACP_HS_TX_DMA_SIZE 0x00020C8 +#define ACP_HS_TX_LINEARPOSITIONCNTR_HIGH 0x00020CC +#define ACP_HS_TX_LINEARPOSITIONCNTR_LOW 0x00020D0 +#define ACP_HS_TX_INTR_WATERMARK_SIZE 0x00020D4 + +/* Registers from ACP_I2S_TDM block */ +#define ACP_I2STDM_IER 0x0002400 +#define ACP_I2STDM_IRER 0x0002404 +#define ACP_I2STDM_RXFRMT 0x0002408 +#define ACP_I2STDM_ITER 0x000240C +#define ACP_I2STDM_TXFRMT 0x0002410 +#define ACP_I2STDM0_MSTRCLKGEN 0x0002414 +#define ACP_I2STDM1_MSTRCLKGEN 0x0002418 +#define ACP_I2STDM2_MSTRCLKGEN 0x000241C +#define ACP_I2STDM_REFCLKGEN 0x0002420 + +/* Registers from ACP_BT_TDM block */ +#define ACP_BTTDM_IER 0x0002800 +#define ACP_BTTDM_IRER 0x0002804 +#define ACP_BTTDM_RXFRMT 0x0002808 +#define ACP_BTTDM_ITER 0x000280C +#define ACP_BTTDM_TXFRMT 0x0002810 +#define ACP_HSTDM_IER 0x0002814 +#define ACP_HSTDM_IRER 0x0002818 +#define ACP_HSTDM_RXFRMT 0x000281C +#define ACP_HSTDM_ITER 0x0002820 +#define ACP_HSTDM_TXFRMT 0x0002824 + +/* Registers from ACP_WOV block */ +#define ACP_WOV_PDM_ENABLE 0x0002C04 +#define ACP_WOV_PDM_DMA_ENABLE 0x0002C08 +#define ACP_WOV_RX_RINGBUFADDR 0x0002C0C +#define ACP_WOV_RX_RINGBUFSIZE 0x0002C10 +#define ACP_WOV_RX_LINKPOSITIONCNTR 0x0002C14 +#define ACP_WOV_RX_LINEARPOSITIONCNTR_HIGH 0x0002C18 +#define ACP_WOV_RX_LINEARPOSITIONCNTR_LOW 0x0002C1C +#define ACP_WOV_RX_INTR_WATERMARK_SIZE 0x0002C20 +#define ACP_WOV_PDM_FIFO_FLUSH 0x0002C24 +#define ACP_WOV_PDM_NO_OF_CHANNELS 0x0002C28 +#define ACP_WOV_PDM_DECIMATION_FACTOR 0x0002C2C +#define ACP_WOV_PDM_VAD_CTRL 0x0002C30 +#define ACP_WOV_WAKE 0x0002C54 +#define ACP_WOV_BUFFER_STATUS 0x0002C58 +#define ACP_WOV_MISC_CTRL 0x0002C5C +#define ACP_WOV_CLK_CTRL 0x0002C60 +#define ACP_PDM_VAD_DYNAMIC_CLK_GATING_EN 0x0002C64 +#define ACP_WOV_ERROR_STATUS_REGISTER 0x0002C68 +#define ACP_PDM_CLKDIV 0x0002C6C + +/* Registers from ACP_P1_AUDIO_BUFFERS block */ +#define ACP_P1_I2S_RX_RINGBUFADDR 0x0003A00 +#define ACP_P1_I2S_RX_RINGBUFSIZE 0x0003A04 +#define ACP_P1_I2S_RX_LINKPOSITIONCNTR 0x0003A08 +#define ACP_P1_I2S_RX_FIFOADDR 0x0003A0C +#define ACP_P1_I2S_RX_FIFOSIZE 0x0003A10 +#define ACP_P1_I2S_RX_DMA_SIZE 0x0003A14 +#define ACP_P1_I2S_RX_LINEARPOSITIONCNTR_HIGH 0x0003A18 +#define ACP_P1_I2S_RX_LINEARPOSITIONCNTR_LOW 0x0003A1C +#define ACP_P1_I2S_RX_INTR_WATERMARK_SIZE 0x0003A20 +#define ACP_P1_I2S_TX_RINGBUFADDR 0x0003A24 +#define ACP_P1_I2S_TX_RINGBUFSIZE 0x0003A28 +#define ACP_P1_I2S_TX_LINKPOSITIONCNTR 0x0003A2C +#define ACP_P1_I2S_TX_FIFOADDR 0x0003A30 +#define ACP_P1_I2S_TX_FIFOSIZE 0x0003A34 +#define ACP_P1_I2S_TX_DMA_SIZE 0x0003A38 +#define ACP_P1_I2S_TX_LINEARPOSITIONCNTR_HIGH 0x0003A3C +#define ACP_P1_I2S_TX_LINEARPOSITIONCNTR_LOW 0x0003A40 +#define ACP_P1_I2S_TX_INTR_WATERMARK_SIZE 0x0003A44 +#define ACP_P1_BT_RX_RINGBUFADDR 0x0003A48 +#define ACP_P1_BT_RX_RINGBUFSIZE 0x0003A4C +#define ACP_P1_BT_RX_LINKPOSITIONCNTR 0x0003A50 +#define ACP_P1_BT_RX_FIFOADDR 0x0003A54 +#define ACP_P1_BT_RX_FIFOSIZE 0x0003A58 +#define ACP_P1_BT_RX_DMA_SIZE 0x0003A5C +#define ACP_P1_BT_RX_LINEARPOSITIONCNTR_HIGH 0x0003A60 +#define ACP_P1_BT_RX_LINEARPOSITIONCNTR_LOW 0x0003A64 +#define ACP_P1_BT_RX_INTR_WATERMARK_SIZE 0x0003A68 +#define ACP_P1_BT_TX_RINGBUFADDR 0x0003A6C +#define ACP_P1_BT_TX_RINGBUFSIZE 0x0003A70 +#define ACP_P1_BT_TX_LINKPOSITIONCNTR 0x0003A74 +#define ACP_P1_BT_TX_FIFOADDR 0x0003A78 +#define ACP_P1_BT_TX_FIFOSIZE 0x0003A7C +#define ACP_P1_BT_TX_DMA_SIZE 0x0003A80 +#define ACP_P1_BT_TX_LINEARPOSITIONCNTR_HIGH 0x0003A84 +#define ACP_P1_BT_TX_LINEARPOSITIONCNTR_LOW 0x0003A88 +#define ACP_P1_BT_TX_INTR_WATERMARK_SIZE 0x0003A8C +#define ACP_P1_HS_RX_RINGBUFADDR 0x0003A90 +#define ACP_P1_HS_RX_RINGBUFSIZE 0x0003A94 +#define ACP_P1_HS_RX_LINKPOSITIONCNTR 0x0003A98 +#define ACP_P1_HS_RX_FIFOADDR 0x0003A9C +#define ACP_P1_HS_RX_FIFOSIZE 0x0003AA0 +#define ACP_P1_HS_RX_DMA_SIZE 0x0003AA4 +#define ACP_P1_HS_RX_LINEARPOSITIONCNTR_HIGH 0x0003AA8 +#define ACP_P1_HS_RX_LINEARPOSITIONCNTR_LOW 0x0003AAC +#define ACP_P1_HS_RX_INTR_WATERMARK_SIZE 0x0003AB0 +#define ACP_P1_HS_TX_RINGBUFADDR 0x0003AB4 +#define ACP_P1_HS_TX_RINGBUFSIZE 0x0003AB8 +#define ACP_P1_HS_TX_LINKPOSITIONCNTR 0x0003ABC +#define ACP_P1_HS_TX_FIFOADDR 0x0003AC0 +#define ACP_P1_HS_TX_FIFOSIZE 0x0003AC4 +#define ACP_P1_HS_TX_DMA_SIZE 0x0003AC8 +#define ACP_P1_HS_TX_LINEARPOSITIONCNTR_HIGH 0x0003ACC +#define ACP_P1_HS_TX_LINEARPOSITIONCNTR_LOW 0x0003AD0 +#define ACP_P1_HS_TX_INTR_WATERMARK_SIZE 0x0003AD4 + +/* Registers from ACP_SCRATCH block */ +#define ACP_SCRATCH_REG_0 0x0010000 +#define ACP_SCRATCH_REG_1 0x0010004 +#define ACP_SCRATCH_REG_2 0x0010008 +#define ACP_SCRATCH_REG_3 0x001000C +#define ACP_SCRATCH_REG_4 0x0010010 +#define ACP_SCRATCH_REG_5 0x0010014 +#define ACP_SCRATCH_REG_6 0x0010018 +#define ACP_SCRATCH_REG_7 0x001001C +#define ACP_SCRATCH_REG_8 0x0010020 +#define ACP_SCRATCH_REG_9 0x0010024 +#define ACP_SCRATCH_REG_10 0x0010028 +#define ACP_SCRATCH_REG_11 0x001002C +#define ACP_SCRATCH_REG_12 0x0010030 +#define ACP_SCRATCH_REG_13 0x0010034 +#define ACP_SCRATCH_REG_14 0x0010038 +#define ACP_SCRATCH_REG_15 0x001003C +#define ACP_SCRATCH_REG_16 0x0010040 +#define ACP_SCRATCH_REG_17 0x0010044 +#define ACP_SCRATCH_REG_18 0x0010048 +#define ACP_SCRATCH_REG_19 0x001004C +#define ACP_SCRATCH_REG_20 0x0010050 +#define ACP_SCRATCH_REG_21 0x0010054 +#define ACP_SCRATCH_REG_22 0x0010058 +#define ACP_SCRATCH_REG_23 0x001005C +#define ACP_SCRATCH_REG_24 0x0010060 +#define ACP_SCRATCH_REG_25 0x0010064 +#define ACP_SCRATCH_REG_26 0x0010068 +#define ACP_SCRATCH_REG_27 0x001006C +#define ACP_SCRATCH_REG_28 0x0010070 +#define ACP_SCRATCH_REG_29 0x0010074 +#define ACP_SCRATCH_REG_30 0x0010078 +#define ACP_SCRATCH_REG_31 0x001007C +#define ACP_SCRATCH_REG_32 0x0010080 +#define ACP_SCRATCH_REG_33 0x0010084 +#define ACP_SCRATCH_REG_34 0x0010088 +#define ACP_SCRATCH_REG_35 0x001008C +#define ACP_SCRATCH_REG_36 0x0010090 +#define ACP_SCRATCH_REG_37 0x0010094 +#define ACP_SCRATCH_REG_38 0x0010098 +#define ACP_SCRATCH_REG_39 0x001009C +#define ACP_SCRATCH_REG_40 0x00100A0 +#define ACP_SCRATCH_REG_41 0x00100A4 +#define ACP_SCRATCH_REG_42 0x00100A8 +#define ACP_SCRATCH_REG_43 0x00100AC +#define ACP_SCRATCH_REG_44 0x00100B0 +#define ACP_SCRATCH_REG_45 0x00100B4 +#define ACP_SCRATCH_REG_46 0x00100B8 +#define ACP_SCRATCH_REG_47 0x00100BC +#define ACP_SCRATCH_REG_48 0x00100C0 +#define ACP_SCRATCH_REG_49 0x00100C4 +#define ACP_SCRATCH_REG_50 0x00100C8 +#define ACP_SCRATCH_REG_51 0x00100CC +#define ACP_SCRATCH_REG_52 0x00100D0 +#define ACP_SCRATCH_REG_53 0x00100D4 +#define ACP_SCRATCH_REG_54 0x00100D8 +#define ACP_SCRATCH_REG_55 0x00100DC +#define ACP_SCRATCH_REG_56 0x00100E0 +#define ACP_SCRATCH_REG_57 0x00100E4 +#define ACP_SCRATCH_REG_58 0x00100E8 +#define ACP_SCRATCH_REG_59 0x00100EC +#define ACP_SCRATCH_REG_60 0x00100F0 +#define ACP_SCRATCH_REG_61 0x00100F4 +#define ACP_SCRATCH_REG_62 0x00100F8 +#define ACP_SCRATCH_REG_63 0x00100FC +#define ACP_SCRATCH_REG_64 0x0010100 +#define ACP_SCRATCH_REG_65 0x0010104 +#define ACP_SCRATCH_REG_66 0x0010108 +#define ACP_SCRATCH_REG_67 0x001010C +#define ACP_SCRATCH_REG_68 0x0010110 +#define ACP_SCRATCH_REG_69 0x0010114 +#define ACP_SCRATCH_REG_70 0x0010118 +#define ACP_SCRATCH_REG_71 0x001011C +#define ACP_SCRATCH_REG_72 0x0010120 +#define ACP_SCRATCH_REG_73 0x0010124 +#define ACP_SCRATCH_REG_74 0x0010128 +#define ACP_SCRATCH_REG_75 0x001012C +#define ACP_SCRATCH_REG_76 0x0010130 +#define ACP_SCRATCH_REG_77 0x0010134 +#define ACP_SCRATCH_REG_78 0x0010138 +#define ACP_SCRATCH_REG_79 0x001013C +#define ACP_SCRATCH_REG_80 0x0010140 +#define ACP_SCRATCH_REG_81 0x0010144 +#define ACP_SCRATCH_REG_82 0x0010148 +#define ACP_SCRATCH_REG_83 0x001014C +#define ACP_SCRATCH_REG_84 0x0010150 +#define ACP_SCRATCH_REG_85 0x0010154 +#define ACP_SCRATCH_REG_86 0x0010158 +#define ACP_SCRATCH_REG_87 0x001015C +#define ACP_SCRATCH_REG_88 0x0010160 +#define ACP_SCRATCH_REG_89 0x0010164 +#define ACP_SCRATCH_REG_90 0x0010168 +#define ACP_SCRATCH_REG_91 0x001016C +#define ACP_SCRATCH_REG_92 0x0010170 +#define ACP_SCRATCH_REG_93 0x0010174 +#define ACP_SCRATCH_REG_94 0x0010178 +#define ACP_SCRATCH_REG_95 0x001017C +#define ACP_SCRATCH_REG_96 0x0010180 +#define ACP_SCRATCH_REG_97 0x0010184 +#define ACP_SCRATCH_REG_98 0x0010188 +#define ACP_SCRATCH_REG_99 0x001018C +#define ACP_SCRATCH_REG_100 0x0010190 +#define ACP_SCRATCH_REG_101 0x0010194 +#define ACP_SCRATCH_REG_102 0x0010198 +#define ACP_SCRATCH_REG_103 0x001019C +#define ACP_SCRATCH_REG_104 0x00101A0 +#define ACP_SCRATCH_REG_105 0x00101A4 +#define ACP_SCRATCH_REG_106 0x00101A8 +#define ACP_SCRATCH_REG_107 0x00101AC +#define ACP_SCRATCH_REG_108 0x00101B0 +#define ACP_SCRATCH_REG_109 0x00101B4 +#define ACP_SCRATCH_REG_110 0x00101B8 +#define ACP_SCRATCH_REG_111 0x00101BC +#define ACP_SCRATCH_REG_112 0x00101C0 +#define ACP_SCRATCH_REG_113 0x00101C4 +#define ACP_SCRATCH_REG_114 0x00101C8 +#define ACP_SCRATCH_REG_115 0x00101CC +#define ACP_SCRATCH_REG_116 0x00101D0 +#define ACP_SCRATCH_REG_117 0x00101D4 +#define ACP_SCRATCH_REG_118 0x00101D8 +#define ACP_SCRATCH_REG_119 0x00101DC +#define ACP_SCRATCH_REG_120 0x00101E0 +#define ACP_SCRATCH_REG_121 0x00101E4 +#define ACP_SCRATCH_REG_122 0x00101E8 +#define ACP_SCRATCH_REG_123 0x00101EC +#define ACP_SCRATCH_REG_124 0x00101F0 +#define ACP_SCRATCH_REG_125 0x00101F4 +#define ACP_SCRATCH_REG_126 0x00101F8 +#define ACP_SCRATCH_REG_127 0x00101FC +#define ACP_SCRATCH_REG_128 0x0010200 +#endif diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h index a55d522f177247372abb1e3ba95779883feb01fb..1d1c24fdd0cae54b230415521a43d1591431c5e4 100644 --- a/include/sound/cs42l42.h +++ b/include/sound/cs42l42.h @@ -40,6 +40,7 @@ #define CS42L42_PAGE_30 0x3000 #define CS42L42_CHIP_ID 0x42A42 +#define CS42L83_CHIP_ID 0x42A83 /* Page 0x10 Global Registers */ #define CS42L42_DEVID_AB (CS42L42_PAGE_10 + 0x01) diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h index 6d3c82c4b6acdfea864576a2a3fc237379b6666b..25ec8c181688d722058a5df3804e168dc1e2df6b 100644 --- a/include/sound/hda_codec.h +++ b/include/sound/hda_codec.h @@ -258,7 +258,6 @@ struct hda_codec { unsigned int link_down_at_suspend:1; /* link down at runtime suspend */ unsigned int relaxed_resume:1; /* don't resume forcibly for jack */ unsigned int forced_resume:1; /* forced resume for jack */ - unsigned int mst_no_extra_pcms:1; /* no backup PCMs for DP-MST */ #ifdef CONFIG_PM unsigned long power_on_acct; @@ -293,8 +292,6 @@ struct hda_codec { #define dev_to_hda_codec(_dev) container_of(_dev, struct hda_codec, core.dev) #define hda_codec_dev(_dev) (&(_dev)->core.dev) -#define hdac_to_hda_priv(_hdac) \ - container_of(_hdac, struct hdac_hda_priv, codec.core) #define hdac_to_hda_codec(_hdac) container_of(_hdac, struct hda_codec, core) #define list_for_each_codec(c, bus) \ diff --git a/include/sound/hda_register.h b/include/sound/hda_register.h index ad8b71b1dbb6731ad5657a5e6c4dd8621503ae72..d37cf43546eba946f58c3f487261a41a413cbe57 100644 --- a/include/sound/hda_register.h +++ b/include/sound/hda_register.h @@ -260,7 +260,18 @@ enum { SDI0, SDI1, SDI2, SDI3, SDO0, SDO1, SDO2, SDO3 }; #define AZX_REG_ML_LCAP 0x00 #define AZX_REG_ML_LCTL 0x04 + +#define AZX_ML_LCTL_CPA BIT(23) +#define AZX_ML_LCTL_CPA_SHIFT 23 +#define AZX_ML_LCTL_SPA BIT(16) +#define AZX_ML_LCTL_SPA_SHIFT 16 +#define AZX_ML_LCTL_SCF GENMASK(3, 0) + #define AZX_REG_ML_LOSIDV 0x08 + +/* bit0 is reserved, with BIT(1) mapping to stream1 */ +#define AZX_ML_LOSIDV_STREAM_MASK 0xFFFE + #define AZX_REG_ML_LSDIID 0x0C #define AZX_REG_ML_LPSOO 0x10 #define AZX_REG_ML_LPSIO 0x12 @@ -268,15 +279,6 @@ enum { SDI0, SDI1, SDI2, SDI3, SDO0, SDO1, SDO2, SDO3 }; #define AZX_REG_ML_LOUTPAY 0x20 #define AZX_REG_ML_LINPAY 0x30 -/* bit0 is reserved, with BIT(1) mapping to stream1 */ -#define ML_LOSIDV_STREAM_MASK 0xFFFE - -#define ML_LCTL_SCF_MASK 0xF -#define AZX_MLCTL_SPA (0x1 << 16) -#define AZX_MLCTL_CPA (0x1 << 23) -#define AZX_MLCTL_SPA_SHIFT 16 -#define AZX_MLCTL_CPA_SHIFT 23 - /* registers for DMA Resume Capability Structure */ #define AZX_DRSM_CAP_ID 0x5 #define AZX_REG_DRSM_CTL 0x4 diff --git a/include/sound/hdaudio.h b/include/sound/hdaudio.h index 797bf67a164dbca813cd7045679e38215a7ca761..35778f953a3f91e1fdacc6461198795b253ae2c1 100644 --- a/include/sound/hdaudio.h +++ b/include/sound/hdaudio.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -550,6 +551,7 @@ void snd_hdac_stream_init(struct hdac_bus *bus, struct hdac_stream *azx_dev, int idx, int direction, int tag); struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus, struct snd_pcm_substream *substream); +void snd_hdac_stream_release_locked(struct hdac_stream *azx_dev); void snd_hdac_stream_release(struct hdac_stream *azx_dev); struct hdac_stream *snd_hdac_get_stream(struct hdac_bus *bus, int dir, int stream_tag); @@ -560,8 +562,8 @@ int snd_hdac_stream_setup_periods(struct hdac_stream *azx_dev); int snd_hdac_stream_set_params(struct hdac_stream *azx_dev, unsigned int format_val); void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start); -void snd_hdac_stream_clear(struct hdac_stream *azx_dev); void snd_hdac_stream_stop(struct hdac_stream *azx_dev); +void snd_hdac_stop_streams(struct hdac_bus *bus); void snd_hdac_stop_streams_and_chip(struct hdac_bus *bus); void snd_hdac_stream_reset(struct hdac_stream *azx_dev); void snd_hdac_stream_sync_trigger(struct hdac_stream *azx_dev, bool set, @@ -589,6 +591,12 @@ int snd_hdac_get_stream_stripe_ctl(struct hdac_bus *bus, snd_hdac_reg_readw((dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg) #define snd_hdac_stream_readb(dev, reg) \ snd_hdac_reg_readb((dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg) +#define snd_hdac_stream_readb_poll(dev, reg, val, cond, delay_us, timeout_us) \ + read_poll_timeout_atomic(snd_hdac_reg_readb, val, cond, delay_us, timeout_us, \ + false, (dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg) +#define snd_hdac_stream_readl_poll(dev, reg, val, cond, delay_us, timeout_us) \ + read_poll_timeout_atomic(snd_hdac_reg_readl, val, cond, delay_us, timeout_us, \ + false, (dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg) /* update a register, pass without AZX_REG_ prefix */ #define snd_hdac_stream_updatel(dev, reg, mask, val) \ diff --git a/include/sound/hdaudio_ext.h b/include/sound/hdaudio_ext.h index d26234f9ee4657541ab7e5689486dff9f10fabbe..83aed26ab143392c87796f62ebc859f12fa4d443 100644 --- a/include/sound/hdaudio_ext.h +++ b/include/sound/hdaudio_ext.h @@ -11,9 +11,6 @@ int snd_hdac_ext_bus_init(struct hdac_bus *bus, struct device *dev, const struct hdac_ext_bus_ops *ext_ops); void snd_hdac_ext_bus_exit(struct hdac_bus *bus); -int snd_hdac_ext_bus_device_init(struct hdac_bus *bus, int addr, - struct hdac_device *hdev, int type); -void snd_hdac_ext_bus_device_exit(struct hdac_device *hdev); void snd_hdac_ext_bus_device_remove(struct hdac_bus *bus); #define HDA_CODEC_REV_EXT_ENTRY(_vid, _rev, _name, drv_data) \ @@ -80,12 +77,9 @@ struct hdac_ext_stream { #define stream_to_hdac_ext_stream(s) \ container_of(s, struct hdac_ext_stream, hstream) -void snd_hdac_ext_stream_init(struct hdac_bus *bus, - struct hdac_ext_stream *hext_stream, int idx, - int direction, int tag); int snd_hdac_ext_stream_init_all(struct hdac_bus *bus, int start_idx, int num_stream, int dir); -void snd_hdac_stream_free_all(struct hdac_bus *bus); +void snd_hdac_ext_stream_free_all(struct hdac_bus *bus); void snd_hdac_link_free_all(struct hdac_bus *bus); struct hdac_ext_stream *snd_hdac_ext_stream_assign(struct hdac_bus *bus, struct snd_pcm_substream *substream, @@ -188,12 +182,6 @@ void snd_hdac_ext_bus_link_power(struct hdac_device *codec, bool enable); #define snd_hdac_adsp_readq_poll(chip, reg, val, cond, delay_us, timeout_us) \ readq_poll_timeout((chip)->dsp_ba + (reg), val, cond, \ delay_us, timeout_us) -#define snd_hdac_stream_readb_poll(strm, reg, val, cond, delay_us, timeout_us) \ - readb_poll_timeout((strm)->sd_addr + AZX_REG_ ## reg, val, cond, \ - delay_us, timeout_us) -#define snd_hdac_stream_readl_poll(strm, reg, val, cond, delay_us, timeout_us) \ - readl_poll_timeout((strm)->sd_addr + AZX_REG_ ## reg, val, cond, \ - delay_us, timeout_us) struct hdac_ext_device; diff --git a/include/sound/intel-nhlt.h b/include/sound/intel-nhlt.h index 3d5cf201cd80217158a945dbad2f2ce25198b2a6..53470d6a28d659db09e9e58e85e24ffe5e862db5 100644 --- a/include/sound/intel-nhlt.h +++ b/include/sound/intel-nhlt.h @@ -136,6 +136,8 @@ bool intel_nhlt_has_endpoint_type(struct nhlt_acpi_table *nhlt, u8 link_type); int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8 device_type); +int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num); + struct nhlt_specific_cfg * intel_nhlt_get_endpoint_blob(struct device *dev, struct nhlt_acpi_table *nhlt, u32 bus_id, u8 link_type, u8 vbps, u8 bps, @@ -169,6 +171,11 @@ static inline int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8 return 0; } +static inline int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num) +{ + return 0; +} + static inline struct nhlt_specific_cfg * intel_nhlt_get_endpoint_blob(struct device *dev, struct nhlt_acpi_table *nhlt, u32 bus_id, u8 link_type, u8 vbps, u8 bps, diff --git a/include/sound/memalloc.h b/include/sound/memalloc.h index 8d79cebf95f328357cd83598374d29a85603df9b..43d524580bd2649d34728b99bc47c6bc35165e5b 100644 --- a/include/sound/memalloc.h +++ b/include/sound/memalloc.h @@ -26,9 +26,6 @@ struct snd_dma_device { struct device *dev; /* generic device */ }; -#define snd_dma_continuous_data(x) ((struct device *)(__force unsigned long)(x)) - - /* * buffer types */ diff --git a/include/sound/pcm.h b/include/sound/pcm.h index 8c48a5bce88c166026fdece07e60603eba733ef9..7b1a022910e8ef0b2337e3cce559c728b0cca960 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -346,6 +346,8 @@ static inline void snd_pcm_pack_audio_tstamp_report(__u32 *data, __u32 *accuracy struct snd_pcm_runtime { /* -- Status -- */ + snd_pcm_state_t state; /* stream state */ + snd_pcm_state_t suspended_state; /* suspended stream state */ struct snd_pcm_substream *trigger_master; struct timespec64 trigger_tstamp; /* trigger timestamp */ bool trigger_tstamp_latched; /* trigger timestamp latched in low-level driver/hardware */ @@ -678,11 +680,25 @@ void snd_pcm_stream_unlock_irqrestore(struct snd_pcm_substream *substream, */ static inline int snd_pcm_running(struct snd_pcm_substream *substream) { - return (substream->runtime->status->state == SNDRV_PCM_STATE_RUNNING || - (substream->runtime->status->state == SNDRV_PCM_STATE_DRAINING && + return (substream->runtime->state == SNDRV_PCM_STATE_RUNNING || + (substream->runtime->state == SNDRV_PCM_STATE_DRAINING && substream->stream == SNDRV_PCM_STREAM_PLAYBACK)); } +/** + * __snd_pcm_set_state - Change the current PCM state + * @runtime: PCM runtime to set + * @state: the current state to set + * + * Call within the stream lock + */ +static inline void __snd_pcm_set_state(struct snd_pcm_runtime *runtime, + snd_pcm_state_t state) +{ + runtime->state = state; + runtime->status->state = state; /* copy for mmap */ +} + /** * bytes_to_samples - Unit conversion of the size from bytes to samples * @runtime: PCM runtime instance diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h index ab55f40896e0a48343afa240c0954549841d21d3..a0b827f0c2f60189ceeeed1b2ccce1b570c5441e 100644 --- a/include/sound/simple_card_utils.h +++ b/include/sound/simple_card_utils.h @@ -39,6 +39,7 @@ struct asoc_simple_dai { struct asoc_simple_data { u32 convert_rate; u32 convert_channels; + const char *convert_sample_format; }; struct asoc_simple_jack { diff --git a/include/sound/soc-acpi-intel-match.h b/include/sound/soc-acpi-intel-match.h index bc7fd46ec2bc8c960336a65a35155ba349e465bd..82a7db23db6951fd92b16e5d763f628cdf080bb1 100644 --- a/include/sound/soc-acpi-intel-match.h +++ b/include/sound/soc-acpi-intel-match.h @@ -14,7 +14,6 @@ * these tables are not constants, some fields can be used for * pdata or machine ops */ -extern struct snd_soc_acpi_mach snd_soc_acpi_intel_haswell_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_broadwell_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_baytrail_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cherrytrail_machines[]; @@ -30,6 +29,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_ehl_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_jsl_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_adl_machines[]; +extern struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cnl_sdw_machines[]; @@ -38,6 +38,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cml_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_adl_sdw_machines[]; +extern struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_sdw_machines[]; extern struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_sdw_machines[]; /* diff --git a/include/sound/soc.h b/include/sound/soc.h index aad24a1d32767a3d8a5c082afce63ca88e68c146..37bbfc8b45cb28b9033c823a1db64565e320d4c1 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -31,31 +31,31 @@ #define SOC_DOUBLE_VALUE(xreg, shift_left, shift_right, xmax, xinvert, xautodisable) \ ((unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, .shift = shift_left, \ - .rshift = shift_right, .max = xmax, .platform_max = xmax, \ + .rshift = shift_right, .max = xmax, \ .invert = xinvert, .autodisable = xautodisable}) #define SOC_DOUBLE_S_VALUE(xreg, shift_left, shift_right, xmin, xmax, xsign_bit, xinvert, xautodisable) \ ((unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, .shift = shift_left, \ - .rshift = shift_right, .min = xmin, .max = xmax, .platform_max = xmax, \ + .rshift = shift_right, .min = xmin, .max = xmax, \ .sign_bit = xsign_bit, .invert = xinvert, .autodisable = xautodisable}) #define SOC_SINGLE_VALUE(xreg, xshift, xmax, xinvert, xautodisable) \ SOC_DOUBLE_VALUE(xreg, xshift, xshift, xmax, xinvert, xautodisable) #define SOC_SINGLE_VALUE_EXT(xreg, xmax, xinvert) \ ((unsigned long)&(struct soc_mixer_control) \ - {.reg = xreg, .max = xmax, .platform_max = xmax, .invert = xinvert}) + {.reg = xreg, .max = xmax, .invert = xinvert}) #define SOC_DOUBLE_R_VALUE(xlreg, xrreg, xshift, xmax, xinvert) \ ((unsigned long)&(struct soc_mixer_control) \ {.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \ - .max = xmax, .platform_max = xmax, .invert = xinvert}) + .max = xmax, .invert = xinvert}) #define SOC_DOUBLE_R_S_VALUE(xlreg, xrreg, xshift, xmin, xmax, xsign_bit, xinvert) \ ((unsigned long)&(struct soc_mixer_control) \ {.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \ - .max = xmax, .min = xmin, .platform_max = xmax, .sign_bit = xsign_bit, \ + .max = xmax, .min = xmin, .sign_bit = xsign_bit, \ .invert = xinvert}) #define SOC_DOUBLE_R_RANGE_VALUE(xlreg, xrreg, xshift, xmin, xmax, xinvert) \ ((unsigned long)&(struct soc_mixer_control) \ {.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \ - .min = xmin, .max = xmax, .platform_max = xmax, .invert = xinvert}) + .min = xmin, .max = xmax, .invert = xinvert}) #define SOC_SINGLE(xname, reg, shift, max, invert) \ { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \ .info = snd_soc_info_volsw, .get = snd_soc_get_volsw,\ @@ -68,7 +68,7 @@ .private_value = (unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, .shift = xshift, \ .rshift = xshift, .min = xmin, .max = xmax, \ - .platform_max = xmax, .invert = xinvert} } + .invert = xinvert} } #define SOC_SINGLE_TLV(xname, reg, shift, max, invert, tlv_array) \ { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \ .access = SNDRV_CTL_ELEM_ACCESS_TLV_READ |\ @@ -99,7 +99,7 @@ .private_value = (unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, .shift = xshift, \ .rshift = xshift, .min = xmin, .max = xmax, \ - .platform_max = xmax, .invert = xinvert} } + .invert = xinvert} } #define SOC_DOUBLE(xname, reg, shift_left, shift_right, max, invert) \ { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname),\ .info = snd_soc_info_volsw, .get = snd_soc_get_volsw, \ @@ -199,7 +199,7 @@ .put = snd_soc_put_volsw, \ .private_value = (unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, \ - .min = xmin, .max = xmax, .platform_max = xmax, \ + .min = xmin, .max = xmax, \ .sign_bit = 7,} } #define SOC_DOUBLE_S8_TLV(xname, xreg, xmin, xmax, tlv_array) \ { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \ @@ -273,7 +273,7 @@ .private_value = (unsigned long)&(struct soc_mixer_control) \ {.reg = xreg, .rreg = xreg, .shift = xshift, \ .rshift = xshift, .min = xmin, .max = xmax, \ - .platform_max = xmax, .invert = xinvert} } + .invert = xinvert} } #define SOC_DOUBLE_EXT_TLV(xname, xreg, shift_left, shift_right, xmax, xinvert,\ xhandler_get, xhandler_put, tlv_array) \ { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \ @@ -1062,7 +1062,8 @@ struct snd_soc_pcm_runtime { unsigned int params_select; /* currently selected param for dai link */ /* Dynamic PCM BE runtime data */ - struct snd_soc_dpcm_runtime dpcm[2]; + struct snd_soc_dpcm_runtime dpcm[SNDRV_PCM_STREAM_LAST + 1]; + struct snd_soc_dapm_widget *c2c_widget[SNDRV_PCM_STREAM_LAST + 1]; long pmdown_time; @@ -1078,11 +1079,6 @@ struct snd_soc_pcm_runtime { * asoc_rtd_to_codec() */ struct snd_soc_dai **dais; - unsigned int num_codecs; - unsigned int num_cpus; - - struct snd_soc_dapm_widget *playback_widget; - struct snd_soc_dapm_widget *capture_widget; struct delayed_work delayed_work; void (*close_delayed_work_func)(struct snd_soc_pcm_runtime *rtd); @@ -1108,7 +1104,7 @@ struct snd_soc_pcm_runtime { }; /* see soc_new_pcm_runtime() */ #define asoc_rtd_to_cpu(rtd, n) (rtd)->dais[n] -#define asoc_rtd_to_codec(rtd, n) (rtd)->dais[n + (rtd)->num_cpus] +#define asoc_rtd_to_codec(rtd, n) (rtd)->dais[n + (rtd)->dai_link->num_cpus] #define asoc_substream_to_rtd(substream) \ (struct snd_soc_pcm_runtime *)snd_pcm_substream_chip(substream) @@ -1118,15 +1114,15 @@ struct snd_soc_pcm_runtime { (i)++) #define for_each_rtd_cpu_dais(rtd, i, dai) \ for ((i) = 0; \ - ((i) < rtd->num_cpus) && ((dai) = asoc_rtd_to_cpu(rtd, i)); \ + ((i) < rtd->dai_link->num_cpus) && ((dai) = asoc_rtd_to_cpu(rtd, i)); \ (i)++) #define for_each_rtd_codec_dais(rtd, i, dai) \ for ((i) = 0; \ - ((i) < rtd->num_codecs) && ((dai) = asoc_rtd_to_codec(rtd, i)); \ + ((i) < rtd->dai_link->num_codecs) && ((dai) = asoc_rtd_to_codec(rtd, i)); \ (i)++) #define for_each_rtd_dais(rtd, i, dai) \ for ((i) = 0; \ - ((i) < (rtd)->num_cpus + (rtd)->num_codecs) && \ + ((i) < (rtd)->dai_link->num_cpus + (rtd)->dai_link->num_codecs) && \ ((dai) = (rtd)->dais[i]); \ (i)++) diff --git a/include/sound/sof.h b/include/sound/sof.h index 367dccfea7ade86d47eef46b8cc1cba54c4e3f1a..341fef19e61244496c2f6a45067c1f88b720dd6b 100644 --- a/include/sound/sof.h +++ b/include/sound/sof.h @@ -89,6 +89,7 @@ struct snd_sof_pdata { /* machine */ struct platform_device *pdev_mach; const struct snd_soc_acpi_mach *machine; + const struct snd_sof_of_mach *of_machine; void *hw_pdata; @@ -102,6 +103,7 @@ struct snd_sof_pdata { struct sof_dev_desc { /* list of machines using this configuration */ struct snd_soc_acpi_mach *machines; + struct snd_sof_of_mach *of_machines; /* alternate list of machines using this configuration */ struct snd_soc_acpi_mach *alt_machines; diff --git a/include/sound/sof/control.h b/include/sound/sof/control.h index 7379a33d724710c43194e51201734204b0030714..983d374fe511c1b62914fad26a098d00dc391cda 100644 --- a/include/sound/sof/control.h +++ b/include/sound/sof/control.h @@ -117,11 +117,11 @@ struct sof_ipc_ctrl_data { /* control data - add new types if needed */ union { /* channel values can be used by volume type controls */ - struct sof_ipc_ctrl_value_chan chanv[0]; + DECLARE_FLEX_ARRAY(struct sof_ipc_ctrl_value_chan, chanv); /* component values used by routing controls like mux, mixer */ - struct sof_ipc_ctrl_value_comp compv[0]; + DECLARE_FLEX_ARRAY(struct sof_ipc_ctrl_value_comp, compv); /* data can be used by binary controls */ - struct sof_abi_hdr data[0]; + DECLARE_FLEX_ARRAY(struct sof_abi_hdr, data); }; } __packed; diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h index 21d98f31a9cabf1c563dda807968852220c33065..83fd81c82e4c1c4ea5d36550d8073670609da208 100644 --- a/include/sound/sof/dai.h +++ b/include/sound/sof/dai.h @@ -84,6 +84,7 @@ enum sof_ipc_dai_type { SOF_DAI_AMD_BT, /**< AMD ACP BT*/ SOF_DAI_AMD_SP, /**< AMD ACP SP */ SOF_DAI_AMD_DMIC, /**< AMD ACP DMIC */ + SOF_DAI_AMD_HS, /**< Amd HS */ SOF_DAI_MEDIATEK_AFE, /**< Mediatek AFE */ }; @@ -112,6 +113,7 @@ struct sof_ipc_dai_config { struct sof_ipc_dai_acp_params acpbt; struct sof_ipc_dai_acp_params acpsp; struct sof_ipc_dai_acpdmic_params acpdmic; + struct sof_ipc_dai_acp_params acphs; struct sof_ipc_dai_mtk_afe_params afe; }; } __packed; diff --git a/include/sound/sof/ipc4/header.h b/include/sound/sof/ipc4/header.h index a795deacc2eae482280a4210dc1088b770a731db..99efe0ef1784819dd66d54929622c21bd5096051 100644 --- a/include/sound/sof/ipc4/header.h +++ b/include/sound/sof/ipc4/header.h @@ -427,6 +427,11 @@ struct sof_ipc4_dx_state_info { #define SOF_IPC4_NOTIFICATION_TYPE_GET(x) (((x) & SOF_IPC4_NOTIFICATION_TYPE_MASK) >> \ SOF_IPC4_NOTIFICATION_TYPE_SHIFT) +#define SOF_IPC4_LOG_CORE_SHIFT 12 +#define SOF_IPC4_LOG_CORE_MASK GENMASK(15, 12) +#define SOF_IPC4_LOG_CORE_GET(x) (((x) & SOF_IPC4_LOG_CORE_MASK) >> \ + SOF_IPC4_LOG_CORE_SHIFT) + /* Value of notification type field - must fit into 8 bits */ enum sof_ipc4_notification_type { /* Phrase detected (notification from WoV module) */ diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index 73df80d462dc83ef760270988d71553931418da2..ed50e81174bf43ac92f14be429b316d4845a034c 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -84,7 +84,6 @@ struct raid56_bio_trace_info; EM( IO_TREE_FS_EXCLUDED_EXTENTS, "EXCLUDED_EXTENTS") \ EM( IO_TREE_BTREE_INODE_IO, "BTREE_INODE_IO") \ EM( IO_TREE_INODE_IO, "INODE_IO") \ - EM( IO_TREE_INODE_IO_FAILURE, "INODE_IO_FAILURE") \ EM( IO_TREE_RELOC_BLOCKS, "RELOC_BLOCKS") \ EM( IO_TREE_TRANS_DIRTY_PAGES, "TRANS_DIRTY_PAGES") \ EM( IO_TREE_ROOT_DIRTY_LOG_PAGES, "ROOT_DIRTY_LOG_PAGES") \ @@ -154,7 +153,6 @@ FLUSH_STATES { EXTENT_NODATASUM, "NODATASUM"}, \ { EXTENT_CLEAR_META_RESV, "CLEAR_META_RESV"}, \ { EXTENT_NEED_WAIT, "NEED_WAIT"}, \ - { EXTENT_DAMAGED, "DAMAGED"}, \ { EXTENT_NORESERVE, "NORESERVE"}, \ { EXTENT_QGROUP_RESERVED, "QGROUP_RESERVED"}, \ { EXTENT_CLEAR_DATA_RESV, "CLEAR_DATA_RESV"}, \ diff --git a/include/trace/events/dlm.h b/include/trace/events/dlm.h index bad21222130efed1628eaf9373b7af8cb249ca07..da0eaae98fa34641923fe7af8f0e3e686a092090 100644 --- a/include/trace/events/dlm.h +++ b/include/trace/events/dlm.h @@ -49,7 +49,7 @@ /* note: we begin tracing dlm_lock_start() only if ls and lkb are found */ TRACE_EVENT(dlm_lock_start, - TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, void *name, + TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, const void *name, unsigned int namelen, int mode, __u32 flags), TP_ARGS(ls, lkb, name, namelen, mode, flags), @@ -91,10 +91,11 @@ TRACE_EVENT(dlm_lock_start, TRACE_EVENT(dlm_lock_end, - TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, void *name, - unsigned int namelen, int mode, __u32 flags, int error), + TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, const void *name, + unsigned int namelen, int mode, __u32 flags, int error, + bool kernel_lock), - TP_ARGS(ls, lkb, name, namelen, mode, flags, error), + TP_ARGS(ls, lkb, name, namelen, mode, flags, error, kernel_lock), TP_STRUCT__entry( __field(__u32, ls_id) @@ -113,6 +114,7 @@ TRACE_EVENT(dlm_lock_end, __entry->lkb_id = lkb->lkb_id; __entry->mode = mode; __entry->flags = flags; + __entry->error = error; r = lkb->lkb_resource; if (r) @@ -122,14 +124,14 @@ TRACE_EVENT(dlm_lock_end, memcpy(__get_dynamic_array(res_name), name, __get_dynamic_array_len(res_name)); - /* return value will be zeroed in those cases by dlm_lock() - * we do it here again to not introduce more overhead if - * trace isn't running and error reflects the return value. - */ - if (error == -EAGAIN || error == -EDEADLK) - __entry->error = 0; - else - __entry->error = error; + if (kernel_lock) { + /* return value will be zeroed in those cases by dlm_lock() + * we do it here again to not introduce more overhead if + * trace isn't running and error reflects the return value. + */ + if (error == -EAGAIN || error == -EDEADLK) + __entry->error = 0; + } ), diff --git a/include/trace/events/erofs.h b/include/trace/events/erofs.h index 57de057bd503a6dcc6d995dfa11af98ea43862d6..4f4c44ea3a655293c4b67e1fb19a54e32ccff15c 100644 --- a/include/trace/events/erofs.h +++ b/include/trace/events/erofs.h @@ -53,15 +53,14 @@ TRACE_EVENT(erofs_lookup, ); TRACE_EVENT(erofs_fill_inode, - TP_PROTO(struct inode *inode, int isdir), - TP_ARGS(inode, isdir), + TP_PROTO(struct inode *inode), + TP_ARGS(inode), TP_STRUCT__entry( __field(dev_t, dev ) __field(erofs_nid_t, nid ) __field(erofs_blk_t, blkaddr ) __field(unsigned int, ofs ) - __field(int, isdir ) ), TP_fast_assign( @@ -69,13 +68,11 @@ TRACE_EVENT(erofs_fill_inode, __entry->nid = EROFS_I(inode)->nid; __entry->blkaddr = erofs_blknr(iloc(EROFS_I_SB(inode), __entry->nid)); __entry->ofs = erofs_blkoff(iloc(EROFS_I_SB(inode), __entry->nid)); - __entry->isdir = isdir; ), - TP_printk("dev = (%d,%d), nid = %llu, blkaddr %u ofs %u, isdir %d", + TP_printk("dev = (%d,%d), nid = %llu, blkaddr %u ofs %u", show_dev_nid(__entry), - __entry->blkaddr, __entry->ofs, - __entry->isdir) + __entry->blkaddr, __entry->ofs) ); TRACE_EVENT(erofs_readpage, diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h index f1e922237736807d8d4f03357c9d4a3bcbc89309..c6b372401c27877c63fb7b84b645d9663e9acf17 100644 --- a/include/trace/events/f2fs.h +++ b/include/trace/events/f2fs.h @@ -1578,9 +1578,10 @@ TRACE_EVENT_CONDITION(f2fs_lookup_extent_tree_end, TRACE_EVENT(f2fs_update_extent_tree_range, TP_PROTO(struct inode *inode, unsigned int pgofs, block_t blkaddr, - unsigned int len), + unsigned int len, + unsigned int c_len), - TP_ARGS(inode, pgofs, blkaddr, len), + TP_ARGS(inode, pgofs, blkaddr, len, c_len), TP_STRUCT__entry( __field(dev_t, dev) @@ -1588,6 +1589,7 @@ TRACE_EVENT(f2fs_update_extent_tree_range, __field(unsigned int, pgofs) __field(u32, blk) __field(unsigned int, len) + __field(unsigned int, c_len) ), TP_fast_assign( @@ -1596,14 +1598,17 @@ TRACE_EVENT(f2fs_update_extent_tree_range, __entry->pgofs = pgofs; __entry->blk = blkaddr; __entry->len = len; + __entry->c_len = c_len; ), TP_printk("dev = (%d,%d), ino = %lu, pgofs = %u, " - "blkaddr = %u, len = %u", + "blkaddr = %u, len = %u, " + "c_len = %u", show_dev_ino(__entry), __entry->pgofs, __entry->blk, - __entry->len) + __entry->len, + __entry->c_len) ); TRACE_EVENT(f2fs_shrink_extent_tree, @@ -1823,7 +1828,10 @@ TRACE_EVENT(f2fs_iostat, __field(unsigned long long, app_bio) __field(unsigned long long, app_wio) __field(unsigned long long, app_mio) + __field(unsigned long long, app_bcdio) + __field(unsigned long long, app_mcdio) __field(unsigned long long, fs_dio) + __field(unsigned long long, fs_cdio) __field(unsigned long long, fs_nio) __field(unsigned long long, fs_mio) __field(unsigned long long, fs_gc_dio) @@ -1835,6 +1843,8 @@ TRACE_EVENT(f2fs_iostat, __field(unsigned long long, app_brio) __field(unsigned long long, app_rio) __field(unsigned long long, app_mrio) + __field(unsigned long long, app_bcrio) + __field(unsigned long long, app_mcrio) __field(unsigned long long, fs_drio) __field(unsigned long long, fs_gdrio) __field(unsigned long long, fs_cdrio) @@ -1849,7 +1859,10 @@ TRACE_EVENT(f2fs_iostat, __entry->app_bio = iostat[APP_BUFFERED_IO]; __entry->app_wio = iostat[APP_WRITE_IO]; __entry->app_mio = iostat[APP_MAPPED_IO]; + __entry->app_bcdio = iostat[APP_BUFFERED_CDATA_IO]; + __entry->app_mcdio = iostat[APP_MAPPED_CDATA_IO]; __entry->fs_dio = iostat[FS_DATA_IO]; + __entry->fs_cdio = iostat[FS_CDATA_IO]; __entry->fs_nio = iostat[FS_NODE_IO]; __entry->fs_mio = iostat[FS_META_IO]; __entry->fs_gc_dio = iostat[FS_GC_DATA_IO]; @@ -1861,6 +1874,8 @@ TRACE_EVENT(f2fs_iostat, __entry->app_brio = iostat[APP_BUFFERED_READ_IO]; __entry->app_rio = iostat[APP_READ_IO]; __entry->app_mrio = iostat[APP_MAPPED_READ_IO]; + __entry->app_bcrio = iostat[APP_BUFFERED_CDATA_READ_IO]; + __entry->app_mcrio = iostat[APP_MAPPED_CDATA_READ_IO]; __entry->fs_drio = iostat[FS_DATA_READ_IO]; __entry->fs_gdrio = iostat[FS_GDATA_READ_IO]; __entry->fs_cdrio = iostat[FS_CDATA_READ_IO]; @@ -1870,20 +1885,24 @@ TRACE_EVENT(f2fs_iostat, ), TP_printk("dev = (%d,%d), " - "app [write=%llu (direct=%llu, buffered=%llu), mapped=%llu], " - "fs [data=%llu, node=%llu, meta=%llu, discard=%llu], " + "app [write=%llu (direct=%llu, buffered=%llu), mapped=%llu, " + "compr(buffered=%llu, mapped=%llu)], " + "fs [data=%llu, cdata=%llu, node=%llu, meta=%llu, discard=%llu], " "gc [data=%llu, node=%llu], " "cp [data=%llu, node=%llu, meta=%llu], " "app [read=%llu (direct=%llu, buffered=%llu), mapped=%llu], " - "fs [data=%llu, (gc_data=%llu, compr_data=%llu), " + "compr(buffered=%llu, mapped=%llu)], " + "fs [data=%llu, (gc_data=%llu, cdata=%llu), " "node=%llu, meta=%llu]", show_dev(__entry->dev), __entry->app_wio, __entry->app_dio, - __entry->app_bio, __entry->app_mio, __entry->fs_dio, + __entry->app_bio, __entry->app_mio, __entry->app_bcdio, + __entry->app_mcdio, __entry->fs_dio, __entry->fs_cdio, __entry->fs_nio, __entry->fs_mio, __entry->fs_discard, __entry->fs_gc_dio, __entry->fs_gc_nio, __entry->fs_cp_dio, __entry->fs_cp_nio, __entry->fs_cp_mio, __entry->app_rio, __entry->app_drio, __entry->app_brio, - __entry->app_mrio, __entry->fs_drio, __entry->fs_gdrio, + __entry->app_mrio, __entry->app_bcrio, __entry->app_mcrio, + __entry->fs_drio, __entry->fs_gdrio, __entry->fs_cdrio, __entry->fs_nrio, __entry->fs_mrio) ); diff --git a/include/trace/events/habanalabs.h b/include/trace/events/habanalabs.h new file mode 100644 index 0000000000000000000000000000000000000000..f05c5fa668a26db8a318b63ff59918af98650037 --- /dev/null +++ b/include/trace/events/habanalabs.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2016-2021 HabanaLabs, Ltd. + * All Rights Reserved. + * + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM habanalabs + +#if !defined(_TRACE_HABANALABS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HABANALABS_H + +#include + +DECLARE_EVENT_CLASS(habanalabs_mmu_template, + TP_PROTO(struct device *dev, u64 virt_addr, u64 phys_addr, u32 page_size, bool flush_pte), + + TP_ARGS(dev, virt_addr, phys_addr, page_size, flush_pte), + + TP_STRUCT__entry( + __string(dname, dev_name(dev)) + __field(u64, virt_addr) + __field(u64, phys_addr) + __field(u32, page_size) + __field(u8, flush_pte) + ), + + TP_fast_assign( + __assign_str(dname, dev_name(dev)); + __entry->virt_addr = virt_addr; + __entry->phys_addr = phys_addr; + __entry->page_size = page_size; + __entry->flush_pte = flush_pte; + ), + + TP_printk("%s: vaddr: %#llx, paddr: %#llx, psize: %#x, flush: %s", + __get_str(dname), + __entry->virt_addr, + __entry->phys_addr, + __entry->page_size, + __entry->flush_pte ? "true" : "false") +); + +DEFINE_EVENT(habanalabs_mmu_template, habanalabs_mmu_map, + TP_PROTO(struct device *dev, u64 virt_addr, u64 phys_addr, u32 page_size, bool flush_pte), + TP_ARGS(dev, virt_addr, phys_addr, page_size, flush_pte)); + +DEFINE_EVENT(habanalabs_mmu_template, habanalabs_mmu_unmap, + TP_PROTO(struct device *dev, u64 virt_addr, u64 phys_addr, u32 page_size, bool flush_pte), + TP_ARGS(dev, virt_addr, phys_addr, page_size, flush_pte)); + +DECLARE_EVENT_CLASS(habanalabs_dma_alloc_template, + TP_PROTO(struct device *dev, u64 cpu_addr, u64 dma_addr, size_t size, const char *caller), + + TP_ARGS(dev, cpu_addr, dma_addr, size, caller), + + TP_STRUCT__entry( + __string(dname, dev_name(dev)) + __field(u64, cpu_addr) + __field(u64, dma_addr) + __field(u32, size) + __field(const char *, caller) + ), + + TP_fast_assign( + __assign_str(dname, dev_name(dev)); + __entry->cpu_addr = cpu_addr; + __entry->dma_addr = dma_addr; + __entry->size = size; + __entry->caller = caller; + ), + + TP_printk("%s: cpu_addr: %#llx, dma_addr: %#llx, size: %#x, caller: %s", + __get_str(dname), + __entry->cpu_addr, + __entry->dma_addr, + __entry->size, + __entry->caller) +); + +DEFINE_EVENT(habanalabs_dma_alloc_template, habanalabs_dma_alloc, + TP_PROTO(struct device *dev, u64 cpu_addr, u64 dma_addr, size_t size, const char *caller), + TP_ARGS(dev, cpu_addr, dma_addr, size, caller)); + +DEFINE_EVENT(habanalabs_dma_alloc_template, habanalabs_dma_free, + TP_PROTO(struct device *dev, u64 cpu_addr, u64 dma_addr, size_t size, const char *caller), + TP_ARGS(dev, cpu_addr, dma_addr, size, caller)); + +#endif /* if !defined(_TRACE_HABANALABS_H) || defined(TRACE_HEADER_MULTI_READ) */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h index d651f3437367d0fd28be9fa564f05a59e84f3483..935af4947917345d3651a2771827ec115a64df1c 100644 --- a/include/trace/events/huge_memory.h +++ b/include/trace/events/huge_memory.h @@ -11,11 +11,14 @@ EM( SCAN_FAIL, "failed") \ EM( SCAN_SUCCEED, "succeeded") \ EM( SCAN_PMD_NULL, "pmd_null") \ + EM( SCAN_PMD_NONE, "pmd_none") \ + EM( SCAN_PMD_MAPPED, "page_pmd_mapped") \ EM( SCAN_EXCEED_NONE_PTE, "exceed_none_pte") \ EM( SCAN_EXCEED_SWAP_PTE, "exceed_swap_pte") \ EM( SCAN_EXCEED_SHARED_PTE, "exceed_shared_pte") \ EM( SCAN_PTE_NON_PRESENT, "pte_non_present") \ EM( SCAN_PTE_UFFD_WP, "pte_uffd_wp") \ + EM( SCAN_PTE_MAPPED_HUGEPAGE, "pte_mapped_hugepage") \ EM( SCAN_PAGE_RO, "no_writable_page") \ EM( SCAN_LACK_REFERENCED_PAGE, "lack_referenced_page") \ EM( SCAN_PAGE_NULL, "page_null") \ @@ -166,5 +169,39 @@ TRACE_EVENT(mm_collapse_huge_page_swapin, __entry->ret) ); +TRACE_EVENT(mm_khugepaged_scan_file, + + TP_PROTO(struct mm_struct *mm, struct page *page, const char *filename, + int present, int swap, int result), + + TP_ARGS(mm, page, filename, present, swap, result), + + TP_STRUCT__entry( + __field(struct mm_struct *, mm) + __field(unsigned long, pfn) + __string(filename, filename) + __field(int, present) + __field(int, swap) + __field(int, result) + ), + + TP_fast_assign( + __entry->mm = mm; + __entry->pfn = page ? page_to_pfn(page) : -1; + __assign_str(filename, filename); + __entry->present = present; + __entry->swap = swap; + __entry->result = result; + ), + + TP_printk("mm=%p, scan_pfn=0x%lx, filename=%s, present=%d, swap=%d, result=%s", + __entry->mm, + __entry->pfn, + __get_str(filename), + __entry->present, + __entry->swap, + __print_symbolic(__entry->result, SCAN_STATUS)) +); + #endif /* __HUGE_MEMORY_H */ #include diff --git a/include/trace/events/io_uring.h b/include/trace/events/io_uring.h index c5b21ff0ac8595584c457f9872c9468b1673ffbd..936fd41bf147e1ec61be5f0c2b78c1edf3e47f16 100644 --- a/include/trace/events/io_uring.h +++ b/include/trace/events/io_uring.h @@ -655,6 +655,35 @@ TRACE_EVENT(io_uring_short_write, __entry->wanted, __entry->got) ); +/* + * io_uring_local_work_run - ran ring local task work + * + * @tctx: pointer to a io_uring_ctx + * @count: how many functions it ran + * @loops: how many loops it ran + * + */ +TRACE_EVENT(io_uring_local_work_run, + + TP_PROTO(void *ctx, int count, unsigned int loops), + + TP_ARGS(ctx, count, loops), + + TP_STRUCT__entry ( + __field(void *, ctx ) + __field(int, count ) + __field(unsigned int, loops ) + ), + + TP_fast_assign( + __entry->ctx = ctx; + __entry->count = count; + __entry->loops = loops; + ), + + TP_printk("ring %p, count %d, loops %u", __entry->ctx, __entry->count, __entry->loops) +); + #endif /* _TRACE_IO_URING_H */ /* This part must be outside protection */ diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index 4cb51ace600d41db56b51cb7d8f3b27bcdb5b380..243073cfc29d44a16ee573c05813ebb1e7be2eb3 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -9,16 +9,15 @@ #include #include -DECLARE_EVENT_CLASS(kmem_alloc, +TRACE_EVENT(kmem_cache_alloc, TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s, - size_t bytes_req, - size_t bytes_alloc, - gfp_t gfp_flags), + gfp_t gfp_flags, + int node), - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags), + TP_ARGS(call_site, ptr, s, gfp_flags, node), TP_STRUCT__entry( __field( unsigned long, call_site ) @@ -26,56 +25,42 @@ DECLARE_EVENT_CLASS(kmem_alloc, __field( size_t, bytes_req ) __field( size_t, bytes_alloc ) __field( unsigned long, gfp_flags ) + __field( int, node ) __field( bool, accounted ) ), TP_fast_assign( __entry->call_site = call_site; __entry->ptr = ptr; - __entry->bytes_req = bytes_req; - __entry->bytes_alloc = bytes_alloc; + __entry->bytes_req = s->object_size; + __entry->bytes_alloc = s->size; __entry->gfp_flags = (__force unsigned long)gfp_flags; + __entry->node = node; __entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ? ((gfp_flags & __GFP_ACCOUNT) || - (s && s->flags & SLAB_ACCOUNT)) : false; + (s->flags & SLAB_ACCOUNT)) : false; ), - TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s accounted=%s", + TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s", (void *)__entry->call_site, __entry->ptr, __entry->bytes_req, __entry->bytes_alloc, show_gfp_flags(__entry->gfp_flags), + __entry->node, __entry->accounted ? "true" : "false") ); -DEFINE_EVENT(kmem_alloc, kmalloc, - - TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags) -); - -DEFINE_EVENT(kmem_alloc, kmem_cache_alloc, - - TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s, - size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags), - - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags) -); - -DECLARE_EVENT_CLASS(kmem_alloc_node, +TRACE_EVENT(kmalloc, TP_PROTO(unsigned long call_site, const void *ptr, - struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags, int node), - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node), + TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node), TP_STRUCT__entry( __field( unsigned long, call_site ) @@ -84,7 +69,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __field( size_t, bytes_alloc ) __field( unsigned long, gfp_flags ) __field( int, node ) - __field( bool, accounted ) ), TP_fast_assign( @@ -94,9 +78,6 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->bytes_alloc = bytes_alloc; __entry->gfp_flags = (__force unsigned long)gfp_flags; __entry->node = node; - __entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ? - ((gfp_flags & __GFP_ACCOUNT) || - (s && s->flags & SLAB_ACCOUNT)) : false; ), TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s", @@ -106,25 +87,8 @@ DECLARE_EVENT_CLASS(kmem_alloc_node, __entry->bytes_alloc, show_gfp_flags(__entry->gfp_flags), __entry->node, - __entry->accounted ? "true" : "false") -); - -DEFINE_EVENT(kmem_alloc_node, kmalloc_node, - - TP_PROTO(unsigned long call_site, const void *ptr, - struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc, - gfp_t gfp_flags, int node), - - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node) -); - -DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node, - - TP_PROTO(unsigned long call_site, const void *ptr, - struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc, - gfp_t gfp_flags, int node), - - TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node) + (IS_ENABLED(CONFIG_MEMCG_KMEM) && + (__entry->gfp_flags & (__force unsigned long)__GFP_ACCOUNT)) ? "true" : "false") ); TRACE_EVENT(kfree, @@ -149,20 +113,20 @@ TRACE_EVENT(kfree, TRACE_EVENT(kmem_cache_free, - TP_PROTO(unsigned long call_site, const void *ptr, const char *name), + TP_PROTO(unsigned long call_site, const void *ptr, const struct kmem_cache *s), - TP_ARGS(call_site, ptr, name), + TP_ARGS(call_site, ptr, s), TP_STRUCT__entry( __field( unsigned long, call_site ) __field( const void *, ptr ) - __string( name, name ) + __string( name, s->name ) ), TP_fast_assign( __entry->call_site = call_site; __entry->ptr = ptr; - __assign_str(name, name); + __assign_str(name, s->name); ), TP_printk("call_site=%pS ptr=%p name=%s", diff --git a/include/trace/events/maple_tree.h b/include/trace/events/maple_tree.h new file mode 100644 index 0000000000000000000000000000000000000000..2be403bdc2bd8d89c05e94d80b7ef478afdb17cb --- /dev/null +++ b/include/trace/events/maple_tree.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM maple_tree + +#if !defined(_TRACE_MM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_MM_H + + +#include + +struct ma_state; + +TRACE_EVENT(ma_op, + + TP_PROTO(const char *fn, struct ma_state *mas), + + TP_ARGS(fn, mas), + + TP_STRUCT__entry( + __field(const char *, fn) + __field(unsigned long, min) + __field(unsigned long, max) + __field(unsigned long, index) + __field(unsigned long, last) + __field(void *, node) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->min = mas->min; + __entry->max = mas->max; + __entry->index = mas->index; + __entry->last = mas->last; + __entry->node = mas->node; + ), + + TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", + __entry->fn, + (void *) __entry->node, + (unsigned long) __entry->min, + (unsigned long) __entry->max, + (unsigned long) __entry->index, + (unsigned long) __entry->last + ) +) +TRACE_EVENT(ma_read, + + TP_PROTO(const char *fn, struct ma_state *mas), + + TP_ARGS(fn, mas), + + TP_STRUCT__entry( + __field(const char *, fn) + __field(unsigned long, min) + __field(unsigned long, max) + __field(unsigned long, index) + __field(unsigned long, last) + __field(void *, node) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->min = mas->min; + __entry->max = mas->max; + __entry->index = mas->index; + __entry->last = mas->last; + __entry->node = mas->node; + ), + + TP_printk("%s\tNode: %p (%lu %lu) range: %lu-%lu", + __entry->fn, + (void *) __entry->node, + (unsigned long) __entry->min, + (unsigned long) __entry->max, + (unsigned long) __entry->index, + (unsigned long) __entry->last + ) +) + +TRACE_EVENT(ma_write, + + TP_PROTO(const char *fn, struct ma_state *mas, unsigned long piv, + void *val), + + TP_ARGS(fn, mas, piv, val), + + TP_STRUCT__entry( + __field(const char *, fn) + __field(unsigned long, min) + __field(unsigned long, max) + __field(unsigned long, index) + __field(unsigned long, last) + __field(unsigned long, piv) + __field(void *, val) + __field(void *, node) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->min = mas->min; + __entry->max = mas->max; + __entry->index = mas->index; + __entry->last = mas->last; + __entry->piv = piv; + __entry->val = val; + __entry->node = mas->node; + ), + + TP_printk("%s\tNode %p (%lu %lu) range:%lu-%lu piv (%lu) val %p", + __entry->fn, + (void *) __entry->node, + (unsigned long) __entry->min, + (unsigned long) __entry->max, + (unsigned long) __entry->index, + (unsigned long) __entry->last, + (unsigned long) __entry->piv, + (void *) __entry->val + ) +) +#endif /* _TRACE_MM_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index 4661f7ba07c05e642f5b6cf663c0c8739f994747..216de5f0362101d8adf9f9e231f03c086951df0d 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -42,6 +42,79 @@ TRACE_EVENT(vm_unmapped_area, __entry->low_limit, __entry->high_limit, __entry->align_mask, __entry->align_offset) ); + +TRACE_EVENT(vma_mas_szero, + TP_PROTO(struct maple_tree *mt, unsigned long start, + unsigned long end), + + TP_ARGS(mt, start, end), + + TP_STRUCT__entry( + __field(struct maple_tree *, mt) + __field(unsigned long, start) + __field(unsigned long, end) + ), + + TP_fast_assign( + __entry->mt = mt; + __entry->start = start; + __entry->end = end; + ), + + TP_printk("mt_mod %p, (NULL), SNULL, %lu, %lu,", + __entry->mt, + (unsigned long) __entry->start, + (unsigned long) __entry->end + ) +); + +TRACE_EVENT(vma_store, + TP_PROTO(struct maple_tree *mt, struct vm_area_struct *vma), + + TP_ARGS(mt, vma), + + TP_STRUCT__entry( + __field(struct maple_tree *, mt) + __field(struct vm_area_struct *, vma) + __field(unsigned long, vm_start) + __field(unsigned long, vm_end) + ), + + TP_fast_assign( + __entry->mt = mt; + __entry->vma = vma; + __entry->vm_start = vma->vm_start; + __entry->vm_end = vma->vm_end - 1; + ), + + TP_printk("mt_mod %p, (%p), STORE, %lu, %lu,", + __entry->mt, __entry->vma, + (unsigned long) __entry->vm_start, + (unsigned long) __entry->vm_end + ) +); + + +TRACE_EVENT(exit_mmap, + TP_PROTO(struct mm_struct *mm), + + TP_ARGS(mm), + + TP_STRUCT__entry( + __field(struct mm_struct *, mm) + __field(struct maple_tree *, mt) + ), + + TP_fast_assign( + __entry->mm = mm; + __entry->mt = &mm->mm_mt; + ), + + TP_printk("mt_mod %p, DESTROY\n", + __entry->mt + ) +); + #endif /* This part must be outside protection */ diff --git a/include/trace/events/scmi.h b/include/trace/events/scmi.h index 65016a767b7ae9e8d041ec406a22ebe8e49ade05..f160d68f961d2685204eaaa95f55061d3471c0c5 100644 --- a/include/trace/events/scmi.h +++ b/include/trace/events/scmi.h @@ -27,9 +27,9 @@ TRACE_EVENT(scmi_fc_call, __entry->val2 = val2; ), - TP_printk("[0x%02X]:[0x%02X]:[%08X]:%u:%u", - __entry->protocol_id, __entry->msg_id, - __entry->res_id, __entry->val1, __entry->val2) + TP_printk("pt=%02X msg_id=%02X res_id:%u vals=%u:%u", + __entry->protocol_id, __entry->msg_id, + __entry->res_id, __entry->val1, __entry->val2) ); TRACE_EVENT(scmi_xfer_begin, @@ -53,9 +53,9 @@ TRACE_EVENT(scmi_xfer_begin, __entry->poll = poll; ), - TP_printk("transfer_id=%d msg_id=%u protocol_id=%u seq=%u poll=%u", - __entry->transfer_id, __entry->msg_id, __entry->protocol_id, - __entry->seq, __entry->poll) + TP_printk("pt=%02X msg_id=%02X seq=%04X transfer_id=%X poll=%u", + __entry->protocol_id, __entry->msg_id, __entry->seq, + __entry->transfer_id, __entry->poll) ); TRACE_EVENT(scmi_xfer_response_wait, @@ -81,9 +81,9 @@ TRACE_EVENT(scmi_xfer_response_wait, __entry->poll = poll; ), - TP_printk("transfer_id=%d msg_id=%u protocol_id=%u seq=%u tmo_ms=%u poll=%u", - __entry->transfer_id, __entry->msg_id, __entry->protocol_id, - __entry->seq, __entry->timeout, __entry->poll) + TP_printk("pt=%02X msg_id=%02X seq=%04X transfer_id=%X tmo_ms=%u poll=%u", + __entry->protocol_id, __entry->msg_id, __entry->seq, + __entry->transfer_id, __entry->timeout, __entry->poll) ); TRACE_EVENT(scmi_xfer_end, @@ -107,9 +107,9 @@ TRACE_EVENT(scmi_xfer_end, __entry->status = status; ), - TP_printk("transfer_id=%d msg_id=%u protocol_id=%u seq=%u status=%d", - __entry->transfer_id, __entry->msg_id, __entry->protocol_id, - __entry->seq, __entry->status) + TP_printk("pt=%02X msg_id=%02X seq=%04X transfer_id=%X s=%d", + __entry->protocol_id, __entry->msg_id, __entry->seq, + __entry->transfer_id, __entry->status) ); TRACE_EVENT(scmi_rx_done, @@ -133,9 +133,9 @@ TRACE_EVENT(scmi_rx_done, __entry->msg_type = msg_type; ), - TP_printk("transfer_id=%d msg_id=%u protocol_id=%u seq=%u msg_type=%u", - __entry->transfer_id, __entry->msg_id, __entry->protocol_id, - __entry->seq, __entry->msg_type) + TP_printk("pt=%02X msg_id=%02X seq=%04X transfer_id=%X msg_type=%u", + __entry->protocol_id, __entry->msg_id, __entry->seq, + __entry->transfer_id, __entry->msg_type) ); TRACE_EVENT(scmi_msg_dump, diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h index 45264e4bb254f794e0236fe4fd42f1fbcdef4a30..50a974f7dfb4759db0417bdfb4cc473f912f930c 100644 --- a/include/trace/events/skb.h +++ b/include/trace/events/skb.h @@ -9,6 +9,15 @@ #include #include +#undef FN +#define FN(reason) TRACE_DEFINE_ENUM(SKB_DROP_REASON_##reason); +DEFINE_DROP_REASON(FN, FN) + +#undef FN +#undef FNe +#define FN(reason) { SKB_DROP_REASON_##reason, #reason }, +#define FNe(reason) { SKB_DROP_REASON_##reason, #reason } + /* * Tracepoint for free an sk_buff: */ @@ -35,9 +44,13 @@ TRACE_EVENT(kfree_skb, TP_printk("skbaddr=%p protocol=%u location=%p reason: %s", __entry->skbaddr, __entry->protocol, __entry->location, - drop_reasons[__entry->reason]) + __print_symbolic(__entry->reason, + DEFINE_DROP_REASON(FN, FNe))) ); +#undef FN +#undef FNe + TRACE_EVENT(consume_skb, TP_PROTO(struct sk_buff *skb), diff --git a/include/trace/events/sof.h b/include/trace/events/sof.h new file mode 100644 index 0000000000000000000000000000000000000000..21c2a1efb9f6e6ad8143620a4a0a0ef5a48ea659 --- /dev/null +++ b/include/trace/events/sof.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright(c) 2022 Intel Corporation. All rights reserved. + * + * Author: Noah Klayman + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM sof + +#if !defined(_TRACE_SOF_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_SOF_H +#include +#include +#include +#include "../../../sound/soc/sof/sof-audio.h" + +DECLARE_EVENT_CLASS(sof_widget_template, + TP_PROTO(struct snd_sof_widget *swidget), + TP_ARGS(swidget), + TP_STRUCT__entry( + __string(name, swidget->widget->name) + __field(int, use_count) + ), + TP_fast_assign( + __assign_str(name, swidget->widget->name); + __entry->use_count = swidget->use_count; + ), + TP_printk("name=%s use_count=%d", __get_str(name), __entry->use_count) +); + +DEFINE_EVENT(sof_widget_template, sof_widget_setup, + TP_PROTO(struct snd_sof_widget *swidget), + TP_ARGS(swidget) +); + +DEFINE_EVENT(sof_widget_template, sof_widget_free, + TP_PROTO(struct snd_sof_widget *swidget), + TP_ARGS(swidget) +); + +TRACE_EVENT(sof_ipc3_period_elapsed_position, + TP_PROTO(struct snd_sof_dev *sdev, struct sof_ipc_stream_posn *posn), + TP_ARGS(sdev, posn), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u64, host_posn) + __field(u64, dai_posn) + __field(u64, wallclock) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->host_posn = posn->host_posn; + __entry->dai_posn = posn->dai_posn; + __entry->wallclock = posn->wallclock; + ), + TP_printk("device_name=%s host_posn=%#llx dai_posn=%#llx wallclock=%#llx", + __get_str(device_name), __entry->host_posn, __entry->dai_posn, + __entry->wallclock) +); + +TRACE_EVENT(sof_pcm_pointer_position, + TP_PROTO(struct snd_sof_dev *sdev, + struct snd_sof_pcm *spcm, + struct snd_pcm_substream *substream, + snd_pcm_uframes_t dma_posn, + snd_pcm_uframes_t dai_posn + ), + TP_ARGS(sdev, spcm, substream, dma_posn, dai_posn), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u32, pcm_id) + __field(int, stream) + __field(unsigned long, dma_posn) + __field(unsigned long, dai_posn) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->pcm_id = le32_to_cpu(spcm->pcm.pcm_id); + __entry->stream = substream->stream; + __entry->dma_posn = dma_posn; + __entry->dai_posn = dai_posn; + ), + TP_printk("device_name=%s pcm_id=%d stream=%d dma_posn=%lu dai_posn=%lu", + __get_str(device_name), __entry->pcm_id, __entry->stream, + __entry->dma_posn, __entry->dai_posn) +); + +TRACE_EVENT(sof_stream_position_ipc_rx, + TP_PROTO(struct device *dev), + TP_ARGS(dev), + TP_STRUCT__entry( + __string(device_name, dev_name(dev)) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(dev)); + ), + TP_printk("device_name=%s", __get_str(device_name)) +); + +TRACE_EVENT(sof_ipc4_fw_config, + TP_PROTO(struct snd_sof_dev *sdev, char *key, u32 value), + TP_ARGS(sdev, key, value), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __string(key, key) + __field(u32, value) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __assign_str(key, key); + __entry->value = value; + ), + TP_printk("device_name=%s key=%s value=%d", + __get_str(device_name), __get_str(key), __entry->value) +); + +#endif /* _TRACE_SOF_H */ + +/* This part must be outside protection */ +#include diff --git a/include/trace/events/sof_intel.h b/include/trace/events/sof_intel.h new file mode 100644 index 0000000000000000000000000000000000000000..2a77f9d26c0bf68c9e040da88125ed6192cf5854 --- /dev/null +++ b/include/trace/events/sof_intel.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright(c) 2022 Intel Corporation. All rights reserved. + * + * Author: Noah Klayman + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM sof_intel + +#if !defined(_TRACE_SOF_INTEL_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_SOF_INTEL_H +#include +#include +#include "../../../sound/soc/sof/sof-audio.h" + +TRACE_EVENT(sof_intel_hda_irq, + TP_PROTO(struct snd_sof_dev *sdev, char *source), + TP_ARGS(sdev, source), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __string(source, source) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __assign_str(source, source); + ), + TP_printk("device_name=%s source=%s", + __get_str(device_name), __get_str(source)) +); + +DECLARE_EVENT_CLASS(sof_intel_ipc_firmware_template, + TP_ARGS(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext), + TP_PROTO(sdev, msg, msg_ext), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u32, msg) + __field(u32, msg_ext) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->msg = msg; + __entry->msg_ext = msg_ext; + ), + TP_printk("device_name=%s msg=%#x msg_ext=%#x", + __get_str(device_name), __entry->msg, __entry->msg_ext) +); + +DEFINE_EVENT(sof_intel_ipc_firmware_template, sof_intel_ipc_firmware_response, + TP_PROTO(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext), + TP_ARGS(sdev, msg, msg_ext) +); + +DEFINE_EVENT(sof_intel_ipc_firmware_template, sof_intel_ipc_firmware_initiated, + TP_PROTO(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext), + TP_ARGS(sdev, msg, msg_ext) +); + +TRACE_EVENT(sof_intel_D0I3C_updated, + TP_PROTO(struct snd_sof_dev *sdev, u8 reg), + TP_ARGS(sdev, reg), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u8, reg) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->reg = reg; + ), + TP_printk("device_name=%s register=%#x", + __get_str(device_name), __entry->reg) +); + +TRACE_EVENT(sof_intel_hda_irq_ipc_check, + TP_PROTO(struct snd_sof_dev *sdev, u32 irq_status), + TP_ARGS(sdev, irq_status), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u32, irq_status) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->irq_status = irq_status; + ), + TP_printk("device_name=%s irq_status=%#x", + __get_str(device_name), __entry->irq_status) +); + +TRACE_EVENT(sof_intel_hda_dsp_pcm, + TP_PROTO(struct snd_sof_dev *sdev, + struct hdac_stream *hstream, + struct snd_pcm_substream *substream, + snd_pcm_uframes_t pos + ), + TP_ARGS(sdev, hstream, substream, pos), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u32, hstream_index) + __field(u32, substream) + __field(unsigned long, pos) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->hstream_index = hstream->index; + __entry->substream = substream->stream; + __entry->pos = pos; + ), + TP_printk("device_name=%s hstream_index=%d substream=%d pos=%lu", + __get_str(device_name), __entry->hstream_index, + __entry->substream, __entry->pos) +); + +TRACE_EVENT(sof_intel_hda_dsp_stream_status, + TP_PROTO(struct device *dev, struct hdac_stream *s, u32 status), + TP_ARGS(dev, s, status), + TP_STRUCT__entry( + __string(device_name, dev_name(dev)) + __field(u32, stream) + __field(u32, status) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(dev)); + __entry->stream = s->index; + __entry->status = status; + ), + TP_printk("device_name=%s stream=%d status=%#x", + __get_str(device_name), __entry->stream, __entry->status) +); + +TRACE_EVENT(sof_intel_hda_dsp_check_stream_irq, + TP_PROTO(struct snd_sof_dev *sdev, u32 status), + TP_ARGS(sdev, status), + TP_STRUCT__entry( + __string(device_name, dev_name(sdev->dev)) + __field(u32, status) + ), + TP_fast_assign( + __assign_str(device_name, dev_name(sdev->dev)); + __entry->status = status; + ), + TP_printk("device_name=%s status=%#x", + __get_str(device_name), __entry->status) +); + +#endif /* _TRACE_SOF_INTEL_H */ + +/* This part must be outside protection */ +#include diff --git a/include/uapi/asm-generic/hugetlb_encode.h b/include/uapi/asm-generic/hugetlb_encode.h index 4f3d5aaa11f531164beab5a47bed8478e5f17546..de687009bfe5394f139b57f05562eb327e0101af 100644 --- a/include/uapi/asm-generic/hugetlb_encode.h +++ b/include/uapi/asm-generic/hugetlb_encode.h @@ -20,18 +20,18 @@ #define HUGETLB_FLAG_ENCODE_SHIFT 26 #define HUGETLB_FLAG_ENCODE_MASK 0x3f -#define HUGETLB_FLAG_ENCODE_16KB (14 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_64KB (16 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_512KB (19 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_1MB (20 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_2MB (21 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_8MB (23 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_16MB (24 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_32MB (25 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_256MB (28 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_512MB (29 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_1GB (30 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_2GB (31 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_16GB (34 << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16KB (14U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_64KB (16U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_512KB (19U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_1MB (20U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_2MB (21U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_8MB (23U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16MB (24U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_32MB (25U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_256MB (28U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_512MB (29U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_1GB (30U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_2GB (31U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16GB (34U << HUGETLB_FLAG_ENCODE_SHIFT) #endif /* _ASM_GENERIC_HUGETLB_ENCODE_H_ */ diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index 6c1aa92a92e4411946335cbb9724b75d8efa987a..6ce1f1ceb432c64599f706b86e74a12581c2a54e 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -77,6 +77,8 @@ #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index c2c9c674a2232e76038d925ad7b1b6f58b322215..7ee65c0b4f70a157726c5c6964920f9d3fa6d934 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -755,6 +755,14 @@ struct drm_amdgpu_cs_chunk_data { #define AMDGPU_INFO_FW_TOC 0x15 /* Subquery id: Query CAP firmware version */ #define AMDGPU_INFO_FW_CAP 0x16 + /* Subquery id: Query GFX RLCP firmware version */ + #define AMDGPU_INFO_FW_GFX_RLCP 0x17 + /* Subquery id: Query GFX RLCV firmware version */ + #define AMDGPU_INFO_FW_GFX_RLCV 0x18 + /* Subquery id: Query MES_KIQ firmware version */ + #define AMDGPU_INFO_FW_MES_KIQ 0x19 + /* Subquery id: Query MES firmware version */ + #define AMDGPU_INFO_FW_MES 0x1a /* number of bytes moved for TTM migration */ #define AMDGPU_INFO_NUM_BYTES_MOVED 0x0f diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 7bf9ba1329be939906df35f479990e16bac1e7b7..51b9aa640ad2af58bebe1b846685e75bf6f9011f 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -79,7 +79,7 @@ struct bpf_insn { /* Key of an a BPF_MAP_TYPE_LPM_TRIE entry */ struct bpf_lpm_trie_key { __u32 prefixlen; /* up to 32 for AF_INET, 128 for AF_INET6 */ - __u8 data[]; /* Arbitrary size */ + __u8 data[0]; /* Arbitrary size */ }; struct bpf_cgroup_storage_key { @@ -87,10 +87,35 @@ struct bpf_cgroup_storage_key { __u32 attach_type; /* program attach type (enum bpf_attach_type) */ }; +enum bpf_cgroup_iter_order { + BPF_CGROUP_ITER_ORDER_UNSPEC = 0, + BPF_CGROUP_ITER_SELF_ONLY, /* process only a single object. */ + BPF_CGROUP_ITER_DESCENDANTS_PRE, /* walk descendants in pre-order. */ + BPF_CGROUP_ITER_DESCENDANTS_POST, /* walk descendants in post-order. */ + BPF_CGROUP_ITER_ANCESTORS_UP, /* walk ancestors upward. */ +}; + union bpf_iter_link_info { struct { __u32 map_fd; } map; + struct { + enum bpf_cgroup_iter_order order; + + /* At most one of cgroup_fd and cgroup_id can be non-zero. If + * both are zero, the walk starts from the default cgroup v2 + * root. For walking v1 hierarchy, one should always explicitly + * specify cgroup_fd. + */ + __u32 cgroup_fd; + __u64 cgroup_id; + } cgroup; + /* Parameters of task iterators. */ + struct { + __u32 tid; + __u32 pid; + __u32 pid_fd; + } task; }; /* BPF syscall commands, see bpf(2) man-page for more details. */ @@ -909,6 +934,7 @@ enum bpf_map_type { BPF_MAP_TYPE_INODE_STORAGE, BPF_MAP_TYPE_TASK_STORAGE, BPF_MAP_TYPE_BLOOM_FILTER, + BPF_MAP_TYPE_USER_RINGBUF, }; /* Note that tracing related programs such as @@ -1233,7 +1259,7 @@ enum { /* Query effective (directly attached + inherited from ancestor cgroups) * programs that will be executed for events within a cgroup. - * attach_flags with this flag are returned only for directly attached programs. + * attach_flags with this flag are always returned 0. */ #define BPF_F_QUERY_EFFECTIVE (1U << 0) @@ -1432,7 +1458,10 @@ union bpf_attr { __u32 attach_flags; __aligned_u64 prog_ids; __u32 prog_cnt; - __aligned_u64 prog_attach_flags; /* output: per-program attach_flags */ + /* output: per-program attach_flags. + * not allowed to be set during effective query. + */ + __aligned_u64 prog_attach_flags; } query; struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ @@ -2573,10 +2602,12 @@ union bpf_attr { * There are two supported modes at this time: * * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer - * (room space is added or removed below the layer 2 header). + * (room space is added or removed between the layer 2 and + * layer 3 headers). * * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer - * (room space is added or removed below the layer 3 header). + * (room space is added or removed between the layer 3 and + * layer 4 headers). * * The following flags are supported at this time: * @@ -3008,8 +3039,18 @@ union bpf_attr { * **BPF_F_USER_STACK** * Collect a user space stack instead of a kernel stack. * **BPF_F_USER_BUILD_ID** - * Collect buildid+offset instead of ips for user stack, - * only valid if **BPF_F_USER_STACK** is also specified. + * Collect (build_id, file_offset) instead of ips for user + * stack, only valid if **BPF_F_USER_STACK** is also + * specified. + * + * *file_offset* is an offset relative to the beginning + * of the executable or shared object file backing the vma + * which the *ip* falls in. It is *not* an offset relative + * to that object's base address. Accordingly, it must be + * adjusted by adding (sh_addr - sh_offset), where + * sh_{addr,offset} correspond to the executable section + * containing *file_offset* in the object, for comparisons + * to symbols' st_value to be valid. * * **bpf_get_stack**\ () can collect up to * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject @@ -4425,7 +4466,7 @@ union bpf_attr { * * **-EEXIST** if the option already exists. * - * **-EFAULT** on failrue to parse the existing header options. + * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. @@ -4634,7 +4675,7 @@ union bpf_attr { * a *map* with *task* as the **key**. From this * perspective, the usage is not much different from * **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this - * helper enforces the key must be an task_struct and the map must also + * helper enforces the key must be a task_struct and the map must also * be a **BPF_MAP_TYPE_TASK_STORAGE**. * * Underneath, the value is stored locally at *task* instead of @@ -4692,7 +4733,7 @@ union bpf_attr { * * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size) * Description - * Returns the stored IMA hash of the *inode* (if it's avaialable). + * Returns the stored IMA hash of the *inode* (if it's available). * If the hash is larger than *size*, then only *size* * bytes will be copied to *dst* * Return @@ -4716,12 +4757,12 @@ union bpf_attr { * * The argument *len_diff* can be used for querying with a planned * size change. This allows to check MTU prior to changing packet - * ctx. Providing an *len_diff* adjustment that is larger than the + * ctx. Providing a *len_diff* adjustment that is larger than the * actual packet size (resulting in negative packet size) will in - * principle not exceed the MTU, why it is not considered a - * failure. Other BPF-helpers are needed for performing the - * planned size change, why the responsability for catch a negative - * packet size belong in those helpers. + * principle not exceed the MTU, which is why it is not considered + * a failure. Other BPF helpers are needed for performing the + * planned size change; therefore the responsibility for catching + * a negative packet size belongs in those helpers. * * Specifying *ifindex* zero means the MTU check is performed * against the current net device. This is practical if this isn't @@ -4919,6 +4960,7 @@ union bpf_attr { * Get address of the traced function (for tracing and kprobe programs). * Return * Address of the traced function. + * 0 for kprobes placed within the function (not at the entry). * * u64 bpf_get_attach_cookie(void *ctx) * Description @@ -5048,12 +5090,12 @@ union bpf_attr { * * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) * Description - * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * Get **n**-th argument register (zero based) of the traced function (for tracing programs) * returned in **value**. * * Return * 0 on success. - * **-EINVAL** if n >= arguments count of traced function. + * **-EINVAL** if n >= argument register count of traced function. * * long bpf_get_func_ret(void *ctx, u64 *value) * Description @@ -5066,24 +5108,37 @@ union bpf_attr { * * long bpf_get_func_arg_cnt(void *ctx) * Description - * Get number of arguments of the traced function (for tracing programs). + * Get number of registers of the traced function (for tracing programs) where + * function arguments are stored in these registers. * * Return - * The number of arguments of the traced function. + * The number of argument registers of the traced function. * * int bpf_get_retval(void) * Description - * Get the syscall's return value that will be returned to userspace. + * Get the BPF program's return value that will be returned to the upper layers. * - * This helper is currently supported by cgroup programs only. + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. * Return - * The syscall's return value. + * The BPF program's return value. * * int bpf_set_retval(int retval) * Description - * Set the syscall's return value that will be returned to userspace. + * Set the BPF program's return value that will be returned to the upper layers. + * + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. + * + * Note that there is the following corner case where the program exports an error + * via bpf_set_retval but signals success via 'return 1': + * + * bpf_set_retval(-EPERM); + * return 1; + * + * In this case, the BPF program's return value will use helper's -EPERM. This + * still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case. * - * This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. * @@ -5331,6 +5386,55 @@ union bpf_attr { * **-EACCES** if the SYN cookie is not valid. * * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. + * + * u64 bpf_ktime_get_tai_ns(void) + * Description + * A nonsettable system-wide clock derived from wall-clock time but + * ignoring leap seconds. This clock does not experience + * discontinuities and backwards jumps caused by NTP inserting leap + * seconds as CLOCK_REALTIME does. + * + * See: **clock_gettime**\ (**CLOCK_TAI**) + * Return + * Current *ktime*. + * + * long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags) + * Description + * Drain samples from the specified user ring buffer, and invoke + * the provided callback for each such sample: + * + * long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx); + * + * If **callback_fn** returns 0, the helper will continue to try + * and drain the next sample, up to a maximum of + * BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, + * the helper will skip the rest of the samples and return. Other + * return values are not used now, and will be rejected by the + * verifier. + * Return + * The number of drained samples if no error was encountered while + * draining samples, or 0 if no samples were present in the ring + * buffer. If a user-space producer was epoll-waiting on this map, + * and at least one sample was drained, they will receive an event + * notification notifying them of available space in the ring + * buffer. If the BPF_RB_NO_WAKEUP flag is passed to this + * function, no wakeup notification will be sent. If the + * BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will + * be sent even if no sample was drained. + * + * On failure, the returned value is one of the following: + * + * **-EBUSY** if the ring buffer is contended, and another calling + * context was concurrently draining the ring buffer. + * + * **-EINVAL** if user-space is not properly tracking the ring + * buffer due to the producer position not being aligned to 8 + * bytes, a sample not being aligned to 8 bytes, or the producer + * position not matching the advertised length of a sample. + * + * **-E2BIG** if user-space has tried to publish a sample which is + * larger than the size of the ring buffer, or which cannot fit + * within a struct bpf_dynptr. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5541,6 +5645,8 @@ union bpf_attr { FN(tcp_raw_gen_syncookie_ipv6), \ FN(tcp_raw_check_syncookie_ipv4), \ FN(tcp_raw_check_syncookie_ipv6), \ + FN(ktime_get_tai_ns), \ + FN(user_ringbuf_drain), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -5603,6 +5709,11 @@ enum { BPF_F_SEQ_NUMBER = (1ULL << 3), }; +/* BPF_FUNC_skb_get_tunnel_key flags. */ +enum { + BPF_F_TUNINFO_FLAGS = (1ULL << 4), +}; + /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and * BPF_FUNC_perf_event_read_value flags. */ @@ -5792,7 +5903,10 @@ struct bpf_tunnel_key { }; __u8 tunnel_tos; __u8 tunnel_ttl; - __u16 tunnel_ext; /* Padding, future use. */ + union { + __u16 tunnel_ext; /* compat */ + __be16 tunnel_flags; + }; __u32 tunnel_label; union { __u32 local_ipv4; @@ -5836,6 +5950,11 @@ enum bpf_ret_code { * represented by BPF_REDIRECT above). */ BPF_LWT_REROUTE = 128, + /* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR + * to indicate that no custom dissection was performed, and + * fallback to standard dissector is requested. + */ + BPF_FLOW_DISSECTOR_CONTINUE = 129, }; struct bpf_sock { @@ -6134,11 +6253,26 @@ struct bpf_link_info { struct { __aligned_u64 target_name; /* in/out: target_name buffer ptr */ __u32 target_name_len; /* in/out: target_name buffer len */ + + /* If the iter specific field is 32 bits, it can be put + * in the first or second union. Otherwise it should be + * put in the second union. + */ union { struct { __u32 map_id; } map; }; + union { + struct { + __u64 cgroup_id; + __u32 order; + } cgroup; + struct { + __u32 tid; + __u32 pid; + } task; + }; } iter; struct { __u32 netns_ino; diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index 7ada84e4a3ed11055d0f4497199267a687979abe..5655e89b962be87e5bd6cb1710669cbffad676a0 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -290,6 +290,12 @@ struct btrfs_ioctl_fs_info_args { #define BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE_VALID (1ULL << 1) #define BTRFS_FEATURE_COMPAT_RO_VERITY (1ULL << 2) +/* + * Put all block group items into a dedicated block group tree, greatly + * reducing mount time for large filesystem due to better locality. + */ +#define BTRFS_FEATURE_COMPAT_RO_BLOCK_GROUP_TREE (1ULL << 3) + #define BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF (1ULL << 0) #define BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL (1ULL << 1) #define BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS (1ULL << 2) diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index 5f32a2a495dc32aee9fc4669f0f2d85c34e26f30..1f7a38ec6ac316ca4d2f5fe03b1173aa736cea70 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -965,6 +965,10 @@ static inline __u16 btrfs_qgroup_level(__u64 qgroupid) */ #define BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT (1ULL << 2) +#define BTRFS_QGROUP_STATUS_FLAGS_MASK (BTRFS_QGROUP_STATUS_FLAG_ON | \ + BTRFS_QGROUP_STATUS_FLAG_RESCAN | \ + BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) + #define BTRFS_QGROUP_STATUS_VERSION 1 struct btrfs_qgroup_status_item { diff --git a/include/uapi/linux/can.h b/include/uapi/linux/can.h index 90801ada2bbe3e891f1ea84ad3730dc8b81f90a0..dd645ea72306c3f69ee0a0bd1224acce8e552849 100644 --- a/include/uapi/linux/can.h +++ b/include/uapi/linux/can.h @@ -48,6 +48,7 @@ #include #include +#include /* for offsetof */ /* controller area network (CAN) kernel definitions */ @@ -60,6 +61,7 @@ #define CAN_SFF_MASK 0x000007FFU /* standard frame format (SFF) */ #define CAN_EFF_MASK 0x1FFFFFFFU /* extended frame format (EFF) */ #define CAN_ERR_MASK 0x1FFFFFFFU /* omit EFF, RTR, ERR flags */ +#define CANXL_PRIO_MASK CAN_SFF_MASK /* 11 bit priority mask */ /* * Controller Area Network Identifier structure @@ -73,6 +75,7 @@ typedef __u32 canid_t; #define CAN_SFF_ID_BITS 11 #define CAN_EFF_ID_BITS 29 +#define CANXL_PRIO_BITS CAN_SFF_ID_BITS /* * Controller Area Network Error Message Frame Mask structure @@ -91,6 +94,16 @@ typedef __u32 can_err_mask_t; #define CANFD_MAX_DLC 15 #define CANFD_MAX_DLEN 64 +/* + * CAN XL payload length and DLC definitions according to ISO 11898-1 + * CAN XL DLC ranges from 0 .. 2047 => data length from 1 .. 2048 byte + */ +#define CANXL_MIN_DLC 0 +#define CANXL_MAX_DLC 2047 +#define CANXL_MAX_DLC_MASK 0x07FF +#define CANXL_MIN_DLEN 1 +#define CANXL_MAX_DLEN 2048 + /** * struct can_frame - Classical CAN frame structure (aka CAN 2.0B) * @can_id: CAN ID of the frame and CAN_*_FLAG flags, see canid_t definition @@ -141,8 +154,8 @@ struct can_frame { * When this is done the former differentiation via CAN_MTU / CANFD_MTU gets * lost. CANFD_FDF allows programmers to mark CAN FD frames in the case of * using struct canfd_frame for mixed CAN / CAN FD content (dual use). - * N.B. the Kernel APIs do NOT provide mixed CAN / CAN FD content inside of - * struct canfd_frame therefore the CANFD_FDF flag is disregarded by Linux. + * Since the introduction of CAN XL the CANFD_FDF flag is set in all CAN FD + * frame structures provided by the CAN subsystem of the Linux kernel. */ #define CANFD_BRS 0x01 /* bit rate switch (second bitrate for payload data) */ #define CANFD_ESI 0x02 /* error state indicator of the transmitting node */ @@ -166,8 +179,46 @@ struct canfd_frame { __u8 data[CANFD_MAX_DLEN] __attribute__((aligned(8))); }; +/* + * defined bits for canxl_frame.flags + * + * The canxl_frame.flags element contains two bits CANXL_XLF and CANXL_SEC + * and shares the relative position of the struct can[fd]_frame.len element. + * The CANXL_XLF bit ALWAYS needs to be set to indicate a valid CAN XL frame. + * As a side effect setting this bit intentionally breaks the length checks + * for Classical CAN and CAN FD frames. + * + * Undefined bits in canxl_frame.flags are reserved and shall be set to zero. + */ +#define CANXL_XLF 0x80 /* mandatory CAN XL frame flag (must always be set!) */ +#define CANXL_SEC 0x01 /* Simple Extended Content (security/segmentation) */ + +/** + * struct canxl_frame - CAN with e'X'tended frame 'L'ength frame structure + * @prio: 11 bit arbitration priority with zero'ed CAN_*_FLAG flags + * @flags: additional flags for CAN XL + * @sdt: SDU (service data unit) type + * @len: frame payload length in byte (CANXL_MIN_DLEN .. CANXL_MAX_DLEN) + * @af: acceptance field + * @data: CAN XL frame payload (CANXL_MIN_DLEN .. CANXL_MAX_DLEN byte) + * + * @prio shares the same position as @can_id from struct can[fd]_frame. + */ +struct canxl_frame { + canid_t prio; /* 11 bit priority for arbitration (canid_t) */ + __u8 flags; /* additional flags for CAN XL */ + __u8 sdt; /* SDU (service data unit) type */ + __u16 len; /* frame payload length in byte */ + __u32 af; /* acceptance field */ + __u8 data[CANXL_MAX_DLEN]; +}; + #define CAN_MTU (sizeof(struct can_frame)) #define CANFD_MTU (sizeof(struct canfd_frame)) +#define CANXL_MTU (sizeof(struct canxl_frame)) +#define CANXL_HDR_SIZE (offsetof(struct canxl_frame, data)) +#define CANXL_MIN_MTU (CANXL_HDR_SIZE + 64) +#define CANXL_MAX_MTU CANXL_MTU /* particular protocols of the protocol family PF_CAN */ #define CAN_RAW 1 /* RAW sockets */ diff --git a/include/uapi/linux/can/raw.h b/include/uapi/linux/can/raw.h index 3386aa81fdf2f1f76dbc2789bd9ed2bc43c32fb1..ff12f525c37cebcf6442344b78af91e78b4cf955 100644 --- a/include/uapi/linux/can/raw.h +++ b/include/uapi/linux/can/raw.h @@ -62,6 +62,7 @@ enum { CAN_RAW_RECV_OWN_MSGS, /* receive my own msgs (default:off) */ CAN_RAW_FD_FRAMES, /* allow CAN FD frames (default:off) */ CAN_RAW_JOIN_FILTERS, /* all filters must match to trigger */ + CAN_RAW_XL_FRAMES, /* allow CAN XL frames (default:off) */ }; #endif /* !_UAPI_CAN_RAW_H */ diff --git a/include/uapi/linux/counter.h b/include/uapi/linux/counter.h index 96c5ffd368adc76b360110911486042a85f08e1d..8ab12d731e3bde08ebcae105a3715e11a20b1283 100644 --- a/include/uapi/linux/counter.h +++ b/include/uapi/linux/counter.h @@ -63,6 +63,8 @@ enum counter_event_type { COUNTER_EVENT_INDEX, /* State of counter is changed */ COUNTER_EVENT_CHANGE_OF_STATE, + /* Count value captured */ + COUNTER_EVENT_CAPTURE, }; /** @@ -153,4 +155,10 @@ enum counter_synapse_action { COUNTER_SYNAPSE_ACTION_BOTH_EDGES, }; +/* Signal polarity values */ +enum counter_signal_polarity { + COUNTER_SIGNAL_POLARITY_POSITIVE, + COUNTER_SIGNAL_POLARITY_NEGATIVE, +}; + #endif /* _UAPI_COUNTER_H_ */ diff --git a/include/uapi/linux/dlm.h b/include/uapi/linux/dlm.h index 0d2eca28756762b4b56dc531842ccf8ad137bc85..1923f4f3b05e215af202dc44f223f82fd843b6ed 100644 --- a/include/uapi/linux/dlm.h +++ b/include/uapi/linux/dlm.h @@ -69,7 +69,6 @@ struct dlm_lksb { /* dlm_new_lockspace() flags */ #define DLM_LSFL_TIMEWARN 0x00000002 -#define DLM_LSFL_FS 0x00000004 #define DLM_LSFL_NEWEXCL 0x00000008 diff --git a/include/uapi/linux/dn.h b/include/uapi/linux/dn.h deleted file mode 100644 index 36ca71bd8bbe233ed11fbbc9d76ad387122e1124..0000000000000000000000000000000000000000 --- a/include/uapi/linux/dn.h +++ /dev/null @@ -1,149 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef _LINUX_DN_H -#define _LINUX_DN_H - -#include -#include -#include - -/* - - DECnet Data Structures and Constants - -*/ - -/* - * DNPROTO_NSP can't be the same as SOL_SOCKET, - * so increment each by one (compared to ULTRIX) - */ -#define DNPROTO_NSP 2 /* NSP protocol number */ -#define DNPROTO_ROU 3 /* Routing protocol number */ -#define DNPROTO_NML 4 /* Net mgt protocol number */ -#define DNPROTO_EVL 5 /* Evl protocol number (usr) */ -#define DNPROTO_EVR 6 /* Evl protocol number (evl) */ -#define DNPROTO_NSPT 7 /* NSP trace protocol number */ - - -#define DN_ADDL 2 -#define DN_MAXADDL 2 /* ULTRIX headers have 20 here, but pathworks has 2 */ -#define DN_MAXOPTL 16 -#define DN_MAXOBJL 16 -#define DN_MAXACCL 40 -#define DN_MAXALIASL 128 -#define DN_MAXNODEL 256 -#define DNBUFSIZE 65023 - -/* - * SET/GET Socket options - must match the DSO_ numbers below - */ -#define SO_CONDATA 1 -#define SO_CONACCESS 2 -#define SO_PROXYUSR 3 -#define SO_LINKINFO 7 - -#define DSO_CONDATA 1 /* Set/Get connect data */ -#define DSO_DISDATA 10 /* Set/Get disconnect data */ -#define DSO_CONACCESS 2 /* Set/Get connect access data */ -#define DSO_ACCEPTMODE 4 /* Set/Get accept mode */ -#define DSO_CONACCEPT 5 /* Accept deferred connection */ -#define DSO_CONREJECT 6 /* Reject deferred connection */ -#define DSO_LINKINFO 7 /* Set/Get link information */ -#define DSO_STREAM 8 /* Set socket type to stream */ -#define DSO_SEQPACKET 9 /* Set socket type to sequenced packet */ -#define DSO_MAXWINDOW 11 /* Maximum window size allowed */ -#define DSO_NODELAY 12 /* Turn off nagle */ -#define DSO_CORK 13 /* Wait for more data! */ -#define DSO_SERVICES 14 /* NSP Services field */ -#define DSO_INFO 15 /* NSP Info field */ -#define DSO_MAX 15 /* Maximum option number */ - - -/* LINK States */ -#define LL_INACTIVE 0 -#define LL_CONNECTING 1 -#define LL_RUNNING 2 -#define LL_DISCONNECTING 3 - -#define ACC_IMMED 0 -#define ACC_DEFER 1 - -#define SDF_WILD 1 /* Wild card object */ -#define SDF_PROXY 2 /* Addr eligible for proxy */ -#define SDF_UICPROXY 4 /* Use uic-based proxy */ - -/* Structures */ - - -struct dn_naddr { - __le16 a_len; - __u8 a_addr[DN_MAXADDL]; /* Two bytes little endian */ -}; - -struct sockaddr_dn { - __u16 sdn_family; - __u8 sdn_flags; - __u8 sdn_objnum; - __le16 sdn_objnamel; - __u8 sdn_objname[DN_MAXOBJL]; - struct dn_naddr sdn_add; -}; -#define sdn_nodeaddrl sdn_add.a_len /* Node address length */ -#define sdn_nodeaddr sdn_add.a_addr /* Node address */ - - - -/* - * DECnet set/get DSO_CONDATA, DSO_DISDATA (optional data) structure - */ -struct optdata_dn { - __le16 opt_status; /* Extended status return */ -#define opt_sts opt_status - __le16 opt_optl; /* Length of user data */ - __u8 opt_data[16]; /* User data */ -}; - -struct accessdata_dn { - __u8 acc_accl; - __u8 acc_acc[DN_MAXACCL]; - __u8 acc_passl; - __u8 acc_pass[DN_MAXACCL]; - __u8 acc_userl; - __u8 acc_user[DN_MAXACCL]; -}; - -/* - * DECnet logical link information structure - */ -struct linkinfo_dn { - __u16 idn_segsize; /* Segment size for link */ - __u8 idn_linkstate; /* Logical link state */ -}; - -/* - * Ethernet address format (for DECnet) - */ -union etheraddress { - __u8 dne_addr[ETH_ALEN]; /* Full ethernet address */ - struct { - __u8 dne_hiord[4]; /* DECnet HIORD prefix */ - __u8 dne_nodeaddr[2]; /* DECnet node address */ - } dne_remote; -}; - - -/* - * DECnet physical socket address format - */ -struct dn_addr { - __le16 dna_family; /* AF_DECnet */ - union etheraddress dna_netaddr; /* DECnet ethernet address */ -}; - -#define DECNET_IOCTL_BASE 0x89 /* PROTOPRIVATE range */ - -#define SIOCSNETADDR _IOW(DECNET_IOCTL_BASE, 0xe0, struct dn_naddr) -#define SIOCGNETADDR _IOR(DECNET_IOCTL_BASE, 0xe1, struct dn_naddr) -#define OSIOCSNETADDR _IOW(DECNET_IOCTL_BASE, 0xe0, int) -#define OSIOCGNETADDR _IOR(DECNET_IOCTL_BASE, 0xe1, int) - -#endif /* _LINUX_DN_H */ diff --git a/include/uapi/linux/dw100.h b/include/uapi/linux/dw100.h new file mode 100644 index 0000000000000000000000000000000000000000..3356496edd6b08c5ccb14897bec3b9bebe98d770 --- /dev/null +++ b/include/uapi/linux/dw100.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ +/* Copyright 2022 NXP */ + +#ifndef __UAPI_DW100_H__ +#define __UAPI_DW100_H__ + +#include + +/* + * Check Documentation/userspace-api/media/drivers/dw100.rst for control details. + */ +#define V4L2_CID_DW100_DEWARPING_16x16_VERTEX_MAP (V4L2_CID_USER_DW100_BASE + 1) + +#endif diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index 2d5741fd44bbcd7ba231056abd82b8b5656eec10..dc2aa3d75b3958792a5d592ca1021f3a06673e33 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -736,6 +736,51 @@ enum ethtool_module_power_mode { ETHTOOL_MODULE_POWER_MODE_HIGH, }; +/** + * enum ethtool_podl_pse_admin_state - operational state of the PoDL PSE + * functions. IEEE 802.3-2018 30.15.1.1.2 aPoDLPSEAdminState + * @ETHTOOL_PODL_PSE_ADMIN_STATE_UNKNOWN: state of PoDL PSE functions are + * unknown + * @ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED: PoDL PSE functions are disabled + * @ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED: PoDL PSE functions are enabled + */ +enum ethtool_podl_pse_admin_state { + ETHTOOL_PODL_PSE_ADMIN_STATE_UNKNOWN = 1, + ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED, + ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED, +}; + +/** + * enum ethtool_podl_pse_pw_d_status - power detection status of the PoDL PSE. + * IEEE 802.3-2018 30.15.1.1.3 aPoDLPSEPowerDetectionStatus: + * @ETHTOOL_PODL_PSE_PW_D_STATUS_UNKNOWN: PoDL PSE + * @ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED: "The enumeration “disabled” is + * asserted true when the PoDL PSE state diagram variable mr_pse_enable is + * false" + * @ETHTOOL_PODL_PSE_PW_D_STATUS_SEARCHING: "The enumeration “searching” is + * asserted true when either of the PSE state diagram variables + * pi_detecting or pi_classifying is true." + * @ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING: "The enumeration “deliveringPower” + * is asserted true when the PoDL PSE state diagram variable pi_powered is + * true." + * @ETHTOOL_PODL_PSE_PW_D_STATUS_SLEEP: "The enumeration “sleep” is asserted + * true when the PoDL PSE state diagram variable pi_sleeping is true." + * @ETHTOOL_PODL_PSE_PW_D_STATUS_IDLE: "The enumeration “idle” is asserted true + * when the logical combination of the PoDL PSE state diagram variables + * pi_prebiased*!pi_sleeping is true." + * @ETHTOOL_PODL_PSE_PW_D_STATUS_ERROR: "The enumeration “error” is asserted + * true when the PoDL PSE state diagram variable overload_held is true." + */ +enum ethtool_podl_pse_pw_d_status { + ETHTOOL_PODL_PSE_PW_D_STATUS_UNKNOWN = 1, + ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED, + ETHTOOL_PODL_PSE_PW_D_STATUS_SEARCHING, + ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING, + ETHTOOL_PODL_PSE_PW_D_STATUS_SLEEP, + ETHTOOL_PODL_PSE_PW_D_STATUS_IDLE, + ETHTOOL_PODL_PSE_PW_D_STATUS_ERROR, +}; + /** * struct ethtool_gstrings - string set for data tagging * @cmd: Command number = %ETHTOOL_GSTRINGS @@ -1840,6 +1885,20 @@ static inline int ethtool_validate_duplex(__u8 duplex) #define MASTER_SLAVE_STATE_SLAVE 3 #define MASTER_SLAVE_STATE_ERR 4 +/* These are used to throttle the rate of data on the phy interface when the + * native speed of the interface is higher than the link speed. These should + * not be used for phy interfaces which natively support multiple speeds (e.g. + * MII or SGMII). + */ +/* No rate matching performed. */ +#define RATE_MATCH_NONE 0 +/* The phy sends pause frames to throttle the MAC. */ +#define RATE_MATCH_PAUSE 1 +/* The phy asserts CRS to prevent the MAC from transmitting. */ +#define RATE_MATCH_CRS 2 +/* The MAC is programmed with a sufficiently-large IPG. */ +#define RATE_MATCH_OPEN_LOOP 3 + /* Which connector port. */ #define PORT_TP 0x00 #define PORT_AUI 0x01 @@ -2033,8 +2092,8 @@ enum ethtool_reset_flags { * reported consistently by PHYLIB. Read-only. * @master_slave_cfg: Master/slave port mode. * @master_slave_state: Master/slave port state. + * @rate_matching: Rate adaptation performed by the PHY * @reserved: Reserved for future use; see the note on reserved space. - * @reserved1: Reserved for future use; see the note on reserved space. * @link_mode_masks: Variable length bitmaps. * * If autonegotiation is disabled, the speed and @duplex represent the @@ -2085,7 +2144,7 @@ struct ethtool_link_settings { __u8 transceiver; __u8 master_slave_cfg; __u8 master_slave_state; - __u8 reserved1[1]; + __u8 rate_matching; __u32 reserved[7]; __u32 link_mode_masks[]; /* layout of link_mode_masks fields: diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index d2fb4f7be61b2ab312f7a9e2c8117b5187a8146c..bb57084ac524a1830531eb0cdc38efe28f4d6194 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -49,6 +49,8 @@ enum { ETHTOOL_MSG_PHC_VCLOCKS_GET, ETHTOOL_MSG_MODULE_GET, ETHTOOL_MSG_MODULE_SET, + ETHTOOL_MSG_PSE_GET, + ETHTOOL_MSG_PSE_SET, /* add new constants above here */ __ETHTOOL_MSG_USER_CNT, @@ -94,6 +96,7 @@ enum { ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY, ETHTOOL_MSG_MODULE_GET_REPLY, ETHTOOL_MSG_MODULE_NTF, + ETHTOOL_MSG_PSE_GET_REPLY, /* add new constants above here */ __ETHTOOL_MSG_KERNEL_CNT, @@ -242,6 +245,7 @@ enum { ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG, /* u8 */ ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE, /* u8 */ ETHTOOL_A_LINKMODES_LANES, /* u32 */ + ETHTOOL_A_LINKMODES_RATE_MATCHING, /* u8 */ /* add new constants above here */ __ETHTOOL_A_LINKMODES_CNT, @@ -862,6 +866,19 @@ enum { ETHTOOL_A_MODULE_MAX = (__ETHTOOL_A_MODULE_CNT - 1) }; +/* Power Sourcing Equipment */ +enum { + ETHTOOL_A_PSE_UNSPEC, + ETHTOOL_A_PSE_HEADER, /* nest - _A_HEADER_* */ + ETHTOOL_A_PODL_PSE_ADMIN_STATE, /* u32 */ + ETHTOOL_A_PODL_PSE_ADMIN_CONTROL, /* u32 */ + ETHTOOL_A_PODL_PSE_PW_D_STATUS, /* u32 */ + + /* add new constants above here */ + __ETHTOOL_A_PSE_CNT, + ETHTOOL_A_PSE_MAX = (__ETHTOOL_A_PSE_CNT - 1) +}; + /* generic netlink info */ #define ETHTOOL_GENL_NAME "ethtool" #define ETHTOOL_GENL_VERSION 1 diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index d6ccee9618917f0ef3e9eb5007cabc17c1596f70..76ee8f9e024af596d39665debcb50b509aff5dad 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -194,6 +194,9 @@ * - add FUSE_SECURITY_CTX init flag * - add security context to create, mkdir, symlink, and mknod requests * - add FUSE_HAS_INODE_DAX, FUSE_ATTR_DAX + * + * 7.37 + * - add FUSE_TMPFILE */ #ifndef _LINUX_FUSE_H @@ -229,7 +232,7 @@ #define FUSE_KERNEL_VERSION 7 /** Minor version number of this interface */ -#define FUSE_KERNEL_MINOR_VERSION 36 +#define FUSE_KERNEL_MINOR_VERSION 37 /** The node ID of the root inode */ #define FUSE_ROOT_ID 1 @@ -537,6 +540,7 @@ enum fuse_opcode { FUSE_SETUPMAPPING = 48, FUSE_REMOVEMAPPING = 49, FUSE_SYNCFS = 50, + FUSE_TMPFILE = 51, /* CUSE specific operations */ CUSE_INIT = 4096, diff --git a/include/uapi/linux/hid.h b/include/uapi/linux/hid.h index b34492a87a8a46caa06369d7837c7ee40c651b3f..a4dcb34386e3952e5053a7466d9d4be3bcd642ff 100644 --- a/include/uapi/linux/hid.h +++ b/include/uapi/linux/hid.h @@ -42,16 +42,30 @@ #define USB_INTERFACE_PROTOCOL_KEYBOARD 1 #define USB_INTERFACE_PROTOCOL_MOUSE 2 +/* + * HID report types --- Ouch! HID spec says 1 2 3! + */ + +enum hid_report_type { + HID_INPUT_REPORT = 0, + HID_OUTPUT_REPORT = 1, + HID_FEATURE_REPORT = 2, + + HID_REPORT_TYPES, +}; + /* * HID class requests */ -#define HID_REQ_GET_REPORT 0x01 -#define HID_REQ_GET_IDLE 0x02 -#define HID_REQ_GET_PROTOCOL 0x03 -#define HID_REQ_SET_REPORT 0x09 -#define HID_REQ_SET_IDLE 0x0A -#define HID_REQ_SET_PROTOCOL 0x0B +enum hid_class_request { + HID_REQ_GET_REPORT = 0x01, + HID_REQ_GET_IDLE = 0x02, + HID_REQ_GET_PROTOCOL = 0x03, + HID_REQ_SET_REPORT = 0x09, + HID_REQ_SET_IDLE = 0x0A, + HID_REQ_SET_PROTOCOL = 0x0B, +}; /* * HID class descriptor types diff --git a/include/uapi/linux/if_ether.h b/include/uapi/linux/if_ether.h index d370165bc621e873568c7c215e9e46b773922b3e..69e0457eb2000dfda7b96fd4b1daff93f7970f9f 100644 --- a/include/uapi/linux/if_ether.h +++ b/include/uapi/linux/if_ether.h @@ -138,6 +138,7 @@ #define ETH_P_LOCALTALK 0x0009 /* Localtalk pseudo type */ #define ETH_P_CAN 0x000C /* CAN: Controller Area Network */ #define ETH_P_CANFD 0x000D /* CANFD: CAN flexible data rate*/ +#define ETH_P_CANXL 0x000E /* CANXL: eXtended frame Length */ #define ETH_P_PPPTALK 0x0010 /* Dummy type for Atalk over PPP*/ #define ETH_P_TR_802_2 0x0011 /* 802.2 frames */ #define ETH_P_MOBITEX 0x0015 /* Mobitex (kaz@cafe.net) */ diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index e36d9d2c65a7a0b82debab5a6c03a595e67dafea..5e7a1041df3a4cc4e5840e54ef22e5f83629ac70 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -370,6 +370,7 @@ enum { IFLA_GRO_MAX_SIZE, IFLA_TSO_MAX_SIZE, IFLA_TSO_MAX_SEGS, + IFLA_ALLMULTI, /* Allmulti count: > 0 means acts ALLMULTI */ __IFLA_MAX }; @@ -694,6 +695,7 @@ enum { IFLA_XFRM_UNSPEC, IFLA_XFRM_LINK, IFLA_XFRM_IF_ID, + IFLA_XFRM_COLLECT_METADATA, __IFLA_XFRM_MAX }; @@ -1374,4 +1376,14 @@ enum { #define IFLA_MCTP_MAX (__IFLA_MCTP_MAX - 1) +/* DSA section */ + +enum { + IFLA_DSA_UNSPEC, + IFLA_DSA_MASTER, + __IFLA_DSA_MAX, +}; + +#define IFLA_DSA_MAX (__IFLA_DSA_MAX - 1) + #endif /* _UAPI_LINUX_IF_LINK_H */ diff --git a/include/uapi/linux/if_macsec.h b/include/uapi/linux/if_macsec.h index 3af2aa069a3677ca30691c854268664201f7103f..d5b6d1f37353e788ff8d0fdaf010946986c6a3fd 100644 --- a/include/uapi/linux/if_macsec.h +++ b/include/uapi/linux/if_macsec.h @@ -22,6 +22,8 @@ #define MACSEC_KEYID_LEN 16 +#define MACSEC_SALT_LEN 12 + /* cipher IDs as per IEEE802.1AE-2018 (Table 14-1) */ #define MACSEC_CIPHER_ID_GCM_AES_128 0x0080C20001000001ULL #define MACSEC_CIPHER_ID_GCM_AES_256 0x0080C20001000002ULL diff --git a/include/uapi/linux/if_tun.h b/include/uapi/linux/if_tun.h index 2ec07de1d73be4e5da22b4638f37c082e59de2fd..b6d7b868f2904d78b82259b3a971bc34a74a62f6 100644 --- a/include/uapi/linux/if_tun.h +++ b/include/uapi/linux/if_tun.h @@ -67,6 +67,8 @@ #define IFF_TAP 0x0002 #define IFF_NAPI 0x0010 #define IFF_NAPI_FRAGS 0x0020 +/* Used in TUNSETIFF to bring up tun/tap without carrier */ +#define IFF_NO_CARRIER 0x0040 #define IFF_NO_PI 0x1000 /* This flag has no real effect */ #define IFF_ONE_QUEUE 0x2000 diff --git a/include/uapi/linux/iio/types.h b/include/uapi/linux/iio/types.h index 472cead10d8d63630687d93a49ebdd895add30e6..c79f2f046a0be1c6d71a0bcb9e6378b8bce04b3c 100644 --- a/include/uapi/linux/iio/types.h +++ b/include/uapi/linux/iio/types.h @@ -95,6 +95,12 @@ enum iio_modifier { IIO_MOD_ETHANOL, IIO_MOD_H2, IIO_MOD_O2, + IIO_MOD_LINEAR_X, + IIO_MOD_LINEAR_Y, + IIO_MOD_LINEAR_Z, + IIO_MOD_PITCH, + IIO_MOD_YAW, + IIO_MOD_ROLL, }; enum iio_event_type { @@ -105,6 +111,7 @@ enum iio_event_type { IIO_EV_TYPE_MAG_ADAPTIVE, IIO_EV_TYPE_CHANGE, IIO_EV_TYPE_MAG_REFERENCED, + IIO_EV_TYPE_GESTURE, }; enum iio_event_direction { @@ -112,7 +119,8 @@ enum iio_event_direction { IIO_EV_DIR_RISING, IIO_EV_DIR_FALLING, IIO_EV_DIR_NONE, + IIO_EV_DIR_SINGLETAP, + IIO_EV_DIR_DOUBLETAP, }; #endif /* _UAPI_IIO_TYPES_H_ */ - diff --git a/include/uapi/linux/in.h b/include/uapi/linux/in.h index 14168225cecdc74cb7c46b3abd0c1f0bcf874a7a..f243ce665f74f82d51b14d27df362292844ab9e0 100644 --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -68,6 +68,8 @@ enum { #define IPPROTO_PIM IPPROTO_PIM IPPROTO_COMP = 108, /* Compression Header Protocol */ #define IPPROTO_COMP IPPROTO_COMP + IPPROTO_L2TP = 115, /* Layer 2 Tunnelling Protocol */ +#define IPPROTO_L2TP IPPROTO_L2TP IPPROTO_SCTP = 132, /* Stream Control Transport Protocol */ #define IPPROTO_SCTP IPPROTO_SCTP IPPROTO_UDPLITE = 136, /* UDP-Lite (RFC 3828) */ @@ -188,21 +190,13 @@ struct ip_mreq_source { }; struct ip_msfilter { + __be32 imsf_multiaddr; + __be32 imsf_interface; + __u32 imsf_fmode; + __u32 imsf_numsrc; union { - struct { - __be32 imsf_multiaddr_aux; - __be32 imsf_interface_aux; - __u32 imsf_fmode_aux; - __u32 imsf_numsrc_aux; - __be32 imsf_slist[1]; - }; - struct { - __be32 imsf_multiaddr; - __be32 imsf_interface; - __u32 imsf_fmode; - __u32 imsf_numsrc; - __be32 imsf_slist_flex[]; - }; + __be32 imsf_slist[1]; + __DECLARE_FLEX_ARRAY(__be32, imsf_slist_flex); }; }; diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h index dff8e7f17074851211eba754d7eeb19da806931b..7ad931a329706c7253914b9a04fa5902c3d8b194 100644 --- a/include/uapi/linux/input-event-codes.h +++ b/include/uapi/linux/input-event-codes.h @@ -862,6 +862,7 @@ #define ABS_TOOL_WIDTH 0x1c #define ABS_VOLUME 0x20 +#define ABS_PROFILE 0x21 #define ABS_MISC 0x28 diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 1463cfecb56b03ba42a29f2eb8473f06303abf00..ab7458033ee3b3dd4105d4086ae41e3efec39568 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -12,6 +12,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + /* * IO submission data structure (Submission Queue Entry) */ @@ -52,6 +56,7 @@ struct io_uring_sqe { __u32 hardlink_flags; __u32 xattr_flags; __u32 msg_ring_flags; + __u32 uring_cmd_flags; }; __u64 user_data; /* data to be passed back at completion time */ /* pack this to avoid bogus arm OABI complaints */ @@ -67,8 +72,8 @@ struct io_uring_sqe { __s32 splice_fd_in; __u32 file_index; struct { - __u16 notification_idx; __u16 addr_len; + __u16 __pad3[1]; }; }; union { @@ -153,6 +158,13 @@ enum { */ #define IORING_SETUP_SINGLE_ISSUER (1U << 12) +/* + * Defer running task work to get events. + * Rather than running bits of task work whenever the task transitions + * try to do it just before it is needed. + */ +#define IORING_SETUP_DEFER_TASKRUN (1U << 13) + enum io_uring_op { IORING_OP_NOP, IORING_OP_READV, @@ -174,8 +186,7 @@ enum io_uring_op { IORING_OP_FALLOCATE, IORING_OP_OPENAT, IORING_OP_CLOSE, - IORING_OP_RSRC_UPDATE, - IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE, + IORING_OP_FILES_UPDATE, IORING_OP_STATX, IORING_OP_READ, IORING_OP_WRITE, @@ -202,12 +213,21 @@ enum io_uring_op { IORING_OP_GETXATTR, IORING_OP_SOCKET, IORING_OP_URING_CMD, - IORING_OP_SENDZC_NOTIF, + IORING_OP_SEND_ZC, + IORING_OP_SENDMSG_ZC, /* this goes last, obviously */ IORING_OP_LAST, }; +/* + * sqe->uring_cmd_flags + * IORING_URING_CMD_FIXED use registered buffer; pass thig flag + * along with setting sqe->buf_index. + */ +#define IORING_URING_CMD_FIXED (1U << 0) + + /* * sqe->fsync_flags */ @@ -224,7 +244,6 @@ enum io_uring_op { #define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) #define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) #define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) - /* * sqe->splice_flags * extends splice(2) flags @@ -277,29 +296,16 @@ enum io_uring_op { * * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. - * - * IORING_RECVSEND_NOTIF_FLUSH Flush a notification after a successful - * successful. Only for zerocopy sends. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0) #define IORING_RECV_MULTISHOT (1U << 1) #define IORING_RECVSEND_FIXED_BUF (1U << 2) -#define IORING_RECVSEND_NOTIF_FLUSH (1U << 3) /* * accept flags stored in sqe->ioprio */ #define IORING_ACCEPT_MULTISHOT (1U << 0) - -/* - * IORING_OP_RSRC_UPDATE flags - */ -enum { - IORING_RSRC_UPDATE_FILES, - IORING_RSRC_UPDATE_NOTIF, -}; - /* * IORING_OP_MSG_RING command types, stored in sqe->addr */ @@ -337,10 +343,13 @@ struct io_uring_cqe { * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries * IORING_CQE_F_SOCK_NONEMPTY If set, more data to read after socket recv + * IORING_CQE_F_NOTIF Set for notification CQEs. Can be used to distinct + * them from sends. */ #define IORING_CQE_F_BUFFER (1U << 0) #define IORING_CQE_F_MORE (1U << 1) #define IORING_CQE_F_SOCK_NONEMPTY (1U << 2) +#define IORING_CQE_F_NOTIF (1U << 3) enum { IORING_CQE_BUFFER_SHIFT = 16, @@ -481,10 +490,6 @@ enum { /* register a range of fixed file slots for automatic slot allocation */ IORING_REGISTER_FILE_ALLOC_RANGE = 25, - /* zerocopy notification API */ - IORING_REGISTER_NOTIFIERS = 26, - IORING_UNREGISTER_NOTIFIERS = 27, - /* this goes last */ IORING_REGISTER_LAST }; @@ -661,4 +666,8 @@ struct io_uring_recvmsg_out { __u32 flags; }; +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index eed0315a77a6db675d639e6ed9ef26b5f9081280..0d5d4419139aeca5a932fa9f0ac4e29330f75767 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1177,6 +1177,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_VM_DISABLE_NX_HUGE_PAGES 220 #define KVM_CAP_S390_ZPCI_OP 221 #define KVM_CAP_S390_CPU_TOPOLOGY 222 +#define KVM_CAP_DIRTY_LOG_RING_ACQ_REL 223 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/include/uapi/linux/l2tp.h b/include/uapi/linux/l2tp.h index bab8c9708611139357eb888c9ad367935f380844..7d81c3e1ec29aabd374e94f70ec1161e9f5f78ad 100644 --- a/include/uapi/linux/l2tp.h +++ b/include/uapi/linux/l2tp.h @@ -13,8 +13,6 @@ #include #include -#define IPPROTO_L2TP 115 - /** * struct sockaddr_l2tpip - the sockaddr structure for L2TP-over-IP sockets * @l2tp_family: address family number AF_L2TPIP. diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index 23df4e0e8ace78a6b58d9289fd30ee435fb7f2ff..9c4bcc37a455cf41584d1377706dc85a9ebebcc2 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -26,7 +26,7 @@ struct landlock_ruleset_attr { * Landlock filesystem access rights that are not part of * handled_access_fs are allowed. This is needed for backward * compatibility reasons. One exception is the - * LANDLOCK_ACCESS_FS_REFER access right, which is always implicitly + * %LANDLOCK_ACCESS_FS_REFER access right, which is always implicitly * handled, but must still be explicitly handled to add new rules with * this access right. */ @@ -128,11 +128,11 @@ struct landlock_path_beneath_attr { * hierarchy must also always have the same or a superset of restrictions of * the source hierarchy. If it is not the case, or if the domain doesn't * handle this access right, such actions are denied by default with errno - * set to EXDEV. Linking also requires a LANDLOCK_ACCESS_FS_MAKE_* access - * right on the destination directory, and renaming also requires a - * LANDLOCK_ACCESS_FS_REMOVE_* access right on the source's (file or + * set to ``EXDEV``. Linking also requires a ``LANDLOCK_ACCESS_FS_MAKE_*`` + * access right on the destination directory, and renaming also requires a + * ``LANDLOCK_ACCESS_FS_REMOVE_*`` access right on the source's (file or * directory) parent. Otherwise, such actions are denied with errno set to - * EACCES. The EACCES errno prevails over EXDEV to let user space + * ``EACCES``. The ``EACCES`` errno prevails over ``EXDEV`` to let user space * efficiently deal with an unrecoverable error. * * .. warning:: diff --git a/include/uapi/linux/lwtunnel.h b/include/uapi/linux/lwtunnel.h index 2e206919125c4b5e1e6df366bb8446c829b10b2a..229655ef792f621ce8652af837467e4c6558c68d 100644 --- a/include/uapi/linux/lwtunnel.h +++ b/include/uapi/linux/lwtunnel.h @@ -15,6 +15,7 @@ enum lwtunnel_encap_types { LWTUNNEL_ENCAP_SEG6_LOCAL, LWTUNNEL_ENCAP_RPL, LWTUNNEL_ENCAP_IOAM6, + LWTUNNEL_ENCAP_XFRM, __LWTUNNEL_ENCAP_MAX, }; @@ -111,4 +112,13 @@ enum { #define LWT_BPF_MAX_HEADROOM 256 +enum { + LWT_XFRM_UNSPEC, + LWT_XFRM_IF_ID, + LWT_XFRM_LINK, + __LWT_XFRM_MAX, +}; + +#define LWT_XFRM_MAX (__LWT_XFRM_MAX - 1) + #endif /* _UAPI_LWTUNNEL_H_ */ diff --git a/include/uapi/linux/netfilter.h b/include/uapi/linux/netfilter.h index 53411ccc69db08009608de551a11a1b23100db9c..5a79ccb76701c6c8e7fde72f7702deccb7d2804f 100644 --- a/include/uapi/linux/netfilter.h +++ b/include/uapi/linux/netfilter.h @@ -63,7 +63,9 @@ enum { NFPROTO_NETDEV = 5, NFPROTO_BRIDGE = 7, NFPROTO_IPV6 = 10, +#ifndef __KERNEL__ /* no longer supported by kernel */ NFPROTO_DECNET = 12, +#endif NFPROTO_NUMPROTO, }; diff --git a/include/uapi/linux/netfilter/ipset/ip_set.h b/include/uapi/linux/netfilter/ipset/ip_set.h index 6397d75899bcee4cb1f162c0fc779054c846d667..79e5d68b87affef9653e0baafd62d072f16fff3e 100644 --- a/include/uapi/linux/netfilter/ipset/ip_set.h +++ b/include/uapi/linux/netfilter/ipset/ip_set.h @@ -3,10 +3,6 @@ * Patrick Schaaf * Martin Josefsson * Copyright (C) 2003-2011 Jozsef Kadlecsik - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _UAPI_IP_SET_H #define _UAPI_IP_SET_H diff --git a/include/uapi/linux/netfilter/xt_AUDIT.h b/include/uapi/linux/netfilter/xt_AUDIT.h index 1b314e2f84ac727397fa5bc167cae33e5203c82b..56a3f6092e0c35b466288c20274350820db89bb0 100644 --- a/include/uapi/linux/netfilter/xt_AUDIT.h +++ b/include/uapi/linux/netfilter/xt_AUDIT.h @@ -4,10 +4,6 @@ * * (C) 2010-2011 Thomas Graf * (C) 2010-2011 Red Hat, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _XT_AUDIT_TARGET_H diff --git a/include/uapi/linux/netfilter/xt_connmark.h b/include/uapi/linux/netfilter/xt_connmark.h index f01c19b83a2b11e2fb4065cde521495c9bb9d365..41b578ccd03b847d388d9a69248a542f4824efcc 100644 --- a/include/uapi/linux/netfilter/xt_connmark.h +++ b/include/uapi/linux/netfilter/xt_connmark.h @@ -1,18 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* Copyright (C) 2002,2004 MARA Systems AB + * by Henrik Nordstrom + */ + #ifndef _XT_CONNMARK_H #define _XT_CONNMARK_H #include -/* Copyright (C) 2002,2004 MARA Systems AB - * by Henrik Nordstrom - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - enum { XT_CONNMARK_SET = 0, XT_CONNMARK_SAVE, diff --git a/include/uapi/linux/netfilter/xt_osf.h b/include/uapi/linux/netfilter/xt_osf.h index 6e466236ca4b2a428c17be2d5f7e9e58b5f12425..f1f097896bdf9b76d34a469ed00847232321082c 100644 --- a/include/uapi/linux/netfilter/xt_osf.h +++ b/include/uapi/linux/netfilter/xt_osf.h @@ -1,20 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (c) 2003+ Evgeniy Polyakov - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, see . */ #ifndef _XT_OSF_H diff --git a/include/uapi/linux/netfilter_decnet.h b/include/uapi/linux/netfilter_decnet.h deleted file mode 100644 index 3c77f54560f21ae2da5503a7a0c097fb82c8c7a6..0000000000000000000000000000000000000000 --- a/include/uapi/linux/netfilter_decnet.h +++ /dev/null @@ -1,72 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#ifndef __LINUX_DECNET_NETFILTER_H -#define __LINUX_DECNET_NETFILTER_H - -/* DECnet-specific defines for netfilter. - * This file (C) Steve Whitehouse 1999 derived from the - * ipv4 netfilter header file which is - * (C)1998 Rusty Russell -- This code is GPL. - */ - -#include - -/* only for userspace compatibility */ -#ifndef __KERNEL__ - -#include /* for INT_MIN, INT_MAX */ - -/* kernel define is in netfilter_defs.h */ -#define NF_DN_NUMHOOKS 7 -#endif /* ! __KERNEL__ */ - -/* DECnet Hooks */ -/* After promisc drops, checksum checks. */ -#define NF_DN_PRE_ROUTING 0 -/* If the packet is destined for this box. */ -#define NF_DN_LOCAL_IN 1 -/* If the packet is destined for another interface. */ -#define NF_DN_FORWARD 2 -/* Packets coming from a local process. */ -#define NF_DN_LOCAL_OUT 3 -/* Packets about to hit the wire. */ -#define NF_DN_POST_ROUTING 4 -/* Input Hello Packets */ -#define NF_DN_HELLO 5 -/* Input Routing Packets */ -#define NF_DN_ROUTE 6 - -enum nf_dn_hook_priorities { - NF_DN_PRI_FIRST = INT_MIN, - NF_DN_PRI_CONNTRACK = -200, - NF_DN_PRI_MANGLE = -150, - NF_DN_PRI_NAT_DST = -100, - NF_DN_PRI_FILTER = 0, - NF_DN_PRI_NAT_SRC = 100, - NF_DN_PRI_DNRTMSG = 200, - NF_DN_PRI_LAST = INT_MAX, -}; - -struct nf_dn_rtmsg { - int nfdn_ifindex; -}; - -#define NFDN_RTMSG(r) ((unsigned char *)(r) + NLMSG_ALIGN(sizeof(struct nf_dn_rtmsg))) - -#ifndef __KERNEL__ -/* backwards compatibility for userspace */ -#define DNRMG_L1_GROUP 0x01 -#define DNRMG_L2_GROUP 0x02 -#endif - -enum { - DNRNG_NLGRP_NONE, -#define DNRNG_NLGRP_NONE DNRNG_NLGRP_NONE - DNRNG_NLGRP_L1, -#define DNRNG_NLGRP_L1 DNRNG_NLGRP_L1 - DNRNG_NLGRP_L2, -#define DNRNG_NLGRP_L2 DNRNG_NLGRP_L2 - __DNRNG_NLGRP_MAX -}; -#define DNRNG_NLGRP_MAX (__DNRNG_NLGRP_MAX - 1) - -#endif /*__LINUX_DECNET_NETFILTER_H*/ diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h index 855dffb4c1c3e906882644f4bf4e729190c82913..e2ae82e3f9f7187d12ad6d78b6875cfccce878cd 100644 --- a/include/uapi/linux/netlink.h +++ b/include/uapi/linux/netlink.h @@ -20,7 +20,7 @@ #define NETLINK_CONNECTOR 11 #define NETLINK_NETFILTER 12 /* netfilter subsystem */ #define NETLINK_IP6_FW 13 -#define NETLINK_DNRTMSG 14 /* DECnet routing messages */ +#define NETLINK_DNRTMSG 14 /* DECnet routing messages (obsolete) */ #define NETLINK_KOBJECT_UEVENT 15 /* Kernel messages to userspace */ #define NETLINK_GENERIC 16 /* leave room for NETLINK_DM (DM Events) */ @@ -41,12 +41,20 @@ struct sockaddr_nl { __u32 nl_groups; /* multicast groups mask */ }; +/** + * struct nlmsghdr - fixed format metadata header of Netlink messages + * @nlmsg_len: Length of message including header + * @nlmsg_type: Message content type + * @nlmsg_flags: Additional flags + * @nlmsg_seq: Sequence number + * @nlmsg_pid: Sending process port ID + */ struct nlmsghdr { - __u32 nlmsg_len; /* Length of message including header */ - __u16 nlmsg_type; /* Message content */ - __u16 nlmsg_flags; /* Additional flags */ - __u32 nlmsg_seq; /* Sequence number */ - __u32 nlmsg_pid; /* Sending process port ID */ + __u32 nlmsg_len; + __u16 nlmsg_type; + __u16 nlmsg_flags; + __u32 nlmsg_seq; + __u32 nlmsg_pid; }; /* Flags values */ @@ -54,7 +62,7 @@ struct nlmsghdr { #define NLM_F_REQUEST 0x01 /* It is request message. */ #define NLM_F_MULTI 0x02 /* Multipart message, terminated by NLMSG_DONE */ #define NLM_F_ACK 0x04 /* Reply with ack, with zero or error code */ -#define NLM_F_ECHO 0x08 /* Echo this request */ +#define NLM_F_ECHO 0x08 /* Receive resulting notifications */ #define NLM_F_DUMP_INTR 0x10 /* Dump was inconsistent due to sequence change */ #define NLM_F_DUMP_FILTERED 0x20 /* Dump was filtered as requested */ @@ -132,6 +140,10 @@ struct nlmsgerr { * be used - in the success case - to identify a created * object or operation or similar (binary) * @NLMSGERR_ATTR_POLICY: policy for a rejected attribute + * @NLMSGERR_ATTR_MISS_TYPE: type of a missing required attribute, + * %NLMSGERR_ATTR_MISS_NEST will not be present if the attribute was + * missing at the message level + * @NLMSGERR_ATTR_MISS_NEST: offset of the nest where attribute was missing * @__NLMSGERR_ATTR_MAX: number of attributes * @NLMSGERR_ATTR_MAX: highest attribute number */ @@ -141,6 +153,8 @@ enum nlmsgerr_attrs { NLMSGERR_ATTR_OFFS, NLMSGERR_ATTR_COOKIE, NLMSGERR_ATTR_POLICY, + NLMSGERR_ATTR_MISS_TYPE, + NLMSGERR_ATTR_MISS_NEST, __NLMSGERR_ATTR_MAX, NLMSGERR_ATTR_MAX = __NLMSGERR_ATTR_MAX - 1 @@ -337,6 +351,9 @@ enum netlink_attribute_type { * bitfield32 type (U32) * @NL_POLICY_TYPE_ATTR_MASK: mask of valid bits for unsigned integers (U64) * @NL_POLICY_TYPE_ATTR_PAD: pad attribute for 64-bit alignment + * + * @__NL_POLICY_TYPE_ATTR_MAX: number of attributes + * @NL_POLICY_TYPE_ATTR_MAX: highest attribute number */ enum netlink_policy_type_attr { NL_POLICY_TYPE_ATTR_UNSPEC, diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index ffb7c573e299b92a9d41cd1aabe73d7434249d17..c32e7616a366a04695511679ca9e8e0332662513 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -377,14 +377,22 @@ * the non-transmitting interfaces are deleted as well. * * @NL80211_CMD_GET_KEY: Get sequence counter information for a key specified - * by %NL80211_ATTR_KEY_IDX and/or %NL80211_ATTR_MAC. + * by %NL80211_ATTR_KEY_IDX and/or %NL80211_ATTR_MAC. %NL80211_ATTR_MAC + * represents peer's MLD address for MLO pairwise key. For MLO group key, + * the link is identified by %NL80211_ATTR_MLO_LINK_ID. * @NL80211_CMD_SET_KEY: Set key attributes %NL80211_ATTR_KEY_DEFAULT, * %NL80211_ATTR_KEY_DEFAULT_MGMT, or %NL80211_ATTR_KEY_THRESHOLD. + * For MLO connection, the link to set default key is identified by + * %NL80211_ATTR_MLO_LINK_ID. * @NL80211_CMD_NEW_KEY: add a key with given %NL80211_ATTR_KEY_DATA, * %NL80211_ATTR_KEY_IDX, %NL80211_ATTR_MAC, %NL80211_ATTR_KEY_CIPHER, - * and %NL80211_ATTR_KEY_SEQ attributes. + * and %NL80211_ATTR_KEY_SEQ attributes. %NL80211_ATTR_MAC represents + * peer's MLD address for MLO pairwise key. The link to add MLO + * group key is identified by %NL80211_ATTR_MLO_LINK_ID. * @NL80211_CMD_DEL_KEY: delete a key identified by %NL80211_ATTR_KEY_IDX - * or %NL80211_ATTR_MAC. + * or %NL80211_ATTR_MAC. %NL80211_ATTR_MAC represents peer's MLD address + * for MLO pairwise key. The link to delete group key is identified by + * %NL80211_ATTR_MLO_LINK_ID. * * @NL80211_CMD_GET_BEACON: (not used) * @NL80211_CMD_SET_BEACON: change the beacon on an access point interface @@ -4951,6 +4959,7 @@ enum nl80211_bss_scan_width { * using the nesting index as the antenna number. * @NL80211_BSS_FREQUENCY_OFFSET: frequency offset in KHz * @NL80211_BSS_MLO_LINK_ID: MLO link ID of the BSS (u8). + * @NL80211_BSS_MLD_ADDR: MLD address of this BSS if connected to it. * @__NL80211_BSS_AFTER_LAST: internal * @NL80211_BSS_MAX: highest BSS attribute */ @@ -4977,6 +4986,7 @@ enum nl80211_bss { NL80211_BSS_CHAIN_SIGNAL, NL80211_BSS_FREQUENCY_OFFSET, NL80211_BSS_MLO_LINK_ID, + NL80211_BSS_MLD_ADDR, /* keep last */ __NL80211_BSS_AFTER_LAST, @@ -6273,6 +6283,14 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_RADAR_BACKGROUND: Device supports background radar/CAC * detection. * + * @NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE: Device can perform a MAC address + * change without having to bring the underlying network device down + * first. For example, in station mode this can be used to vary the + * origin MAC address prior to a connection to a new AP for privacy + * or other reasons. Note that certain driver specific restrictions + * might apply, e.g. no scans in progress, no offchannel operations + * in progress, and no active connections. + * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. */ @@ -6340,6 +6358,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_BSS_COLOR, NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD, NL80211_EXT_FEATURE_RADAR_BACKGROUND, + NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/include/uapi/linux/openvswitch.h b/include/uapi/linux/openvswitch.h index ce3e1738d4278d14820bd1e18083c7bf5d30169d..94066f87e9ee6d58f515751a1d87cfda544b501d 100644 --- a/include/uapi/linux/openvswitch.h +++ b/include/uapi/linux/openvswitch.h @@ -76,6 +76,8 @@ enum ovs_datapath_cmd { * datapath. Always present in notifications. * @OVS_DP_ATTR_MEGAFLOW_STATS: Statistics about mega flow masks usage for the * datapath. Always present in notifications. + * @OVS_DP_ATTR_IFINDEX: Interface index for a new datapath netdev. Only + * valid for %OVS_DP_CMD_NEW requests. * * These attributes follow the &struct ovs_header within the Generic Netlink * payload for %OVS_DP_* commands. @@ -92,6 +94,7 @@ enum ovs_datapath_attr { OVS_DP_ATTR_PER_CPU_PIDS, /* Netlink PIDS to receive upcalls in * per-cpu dispatch mode */ + OVS_DP_ATTR_IFINDEX, __OVS_DP_ATTR_MAX }; diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index 03b370062741abd20d900da559b45be82fb2bbd0..85be78e0e7f654c80a773236030564cc57bbb461 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -164,8 +164,6 @@ enum perf_event_sample_format { PERF_SAMPLE_WEIGHT_STRUCT = 1U << 24, PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ - - __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ }; #define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT) @@ -204,6 +202,8 @@ enum perf_branch_sample_type_shift { PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */ + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */ + PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ }; @@ -233,6 +233,8 @@ enum perf_branch_sample_type { PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT, + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT, + PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, }; @@ -253,9 +255,48 @@ enum { PERF_BR_COND_RET = 10, /* conditional function return */ PERF_BR_ERET = 11, /* exception return */ PERF_BR_IRQ = 12, /* irq */ + PERF_BR_SERROR = 13, /* system error */ + PERF_BR_NO_TX = 14, /* not in transaction */ + PERF_BR_EXTEND_ABI = 15, /* extend ABI */ PERF_BR_MAX, }; +/* + * Common branch speculation outcome classification + */ +enum { + PERF_BR_SPEC_NA = 0, /* Not available */ + PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */ + PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */ + PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */ + PERF_BR_SPEC_MAX, +}; + +enum { + PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */ + PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */ + PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */ + PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */ + PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */ + PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */ + PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */ + PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */ + PERF_BR_NEW_MAX, +}; + +enum { + PERF_BR_PRIV_UNKNOWN = 0, + PERF_BR_PRIV_USER = 1, + PERF_BR_PRIV_KERNEL = 2, + PERF_BR_PRIV_HV = 3, +}; + +#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1 +#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2 +#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3 +#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4 +#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5 + #define PERF_SAMPLE_BRANCH_PLM_ALL \ (PERF_SAMPLE_BRANCH_USER|\ PERF_SAMPLE_BRANCH_KERNEL|\ @@ -1295,7 +1336,9 @@ union perf_mem_data_src { #define PERF_MEM_LVLNUM_L2 0x02 /* L2 */ #define PERF_MEM_LVLNUM_L3 0x03 /* L3 */ #define PERF_MEM_LVLNUM_L4 0x04 /* L4 */ -/* 5-0xa available */ +/* 5-0x8 available */ +#define PERF_MEM_LVLNUM_EXTN_MEM 0x09 /* Extension memory */ +#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */ #define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */ #define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */ #define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */ @@ -1313,7 +1356,7 @@ union perf_mem_data_src { #define PERF_MEM_SNOOP_SHIFT 19 #define PERF_MEM_SNOOPX_FWD 0x01 /* forward */ -/* 1 free */ +#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */ #define PERF_MEM_SNOOPX_SHIFT 38 /* locked instruction */ @@ -1363,6 +1406,7 @@ union perf_mem_data_src { * abort: aborting a hardware transaction * cycles: cycles from last branch (or 0 if not supported) * type: branch type + * spec: branch speculation info (or 0 if not supported) */ struct perf_branch_entry { __u64 from; @@ -1373,7 +1417,10 @@ struct perf_branch_entry { abort:1, /* transaction abort */ cycles:16, /* cycle count to last branch */ type:4, /* branch type */ - reserved:40; + spec:2, /* branch speculation info */ + new_type:4, /* additional branch type */ + priv:3, /* privilege level */ + reserved:31; }; union perf_sample_weight { diff --git a/include/uapi/linux/pkt_cls.h b/include/uapi/linux/pkt_cls.h index 877309d6ca3ce14ff1a13b10eed706a43ea2025a..648a82f326666ee971a6e19eda41903db4d6ba65 100644 --- a/include/uapi/linux/pkt_cls.h +++ b/include/uapi/linux/pkt_cls.h @@ -592,6 +592,8 @@ enum { TCA_FLOWER_KEY_PPPOE_SID, /* be16 */ TCA_FLOWER_KEY_PPP_PROTO, /* be16 */ + TCA_FLOWER_KEY_L2TPV3_SID, /* be32 */ + __TCA_FLOWER_MAX, }; diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h index f292b467b27fcc3e9d0320773665b0aab59e5f1b..000eec106856a4860e1b6a73dcfd5f91c0c9aa14 100644 --- a/include/uapi/linux/pkt_sched.h +++ b/include/uapi/linux/pkt_sched.h @@ -1232,6 +1232,16 @@ enum { #define TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST _BITUL(0) #define TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD _BITUL(1) +enum { + TCA_TAPRIO_TC_ENTRY_UNSPEC, + TCA_TAPRIO_TC_ENTRY_INDEX, /* u32 */ + TCA_TAPRIO_TC_ENTRY_MAX_SDU, /* u32 */ + + /* add new constants above here */ + __TCA_TAPRIO_TC_ENTRY_CNT, + TCA_TAPRIO_TC_ENTRY_MAX = (__TCA_TAPRIO_TC_ENTRY_CNT - 1) +}; + enum { TCA_TAPRIO_ATTR_UNSPEC, TCA_TAPRIO_ATTR_PRIOMAP, /* struct tc_mqprio_qopt */ @@ -1245,6 +1255,7 @@ enum { TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION, /* s64 */ TCA_TAPRIO_ATTR_FLAGS, /* u32 */ TCA_TAPRIO_ATTR_TXTIME_DELAY, /* u32 */ + TCA_TAPRIO_ATTR_TC_ENTRY, /* nest */ __TCA_TAPRIO_ATTR_MAX, }; diff --git a/include/uapi/linux/psci.h b/include/uapi/linux/psci.h index 2bf93c0d6354b07ec02fbc4561f5ea0fc223c56b..3511095c2702bef5f4a0f160298e5d255579a3e5 100644 --- a/include/uapi/linux/psci.h +++ b/include/uapi/linux/psci.h @@ -48,12 +48,26 @@ #define PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU PSCI_0_2_FN64(7) #define PSCI_1_0_FN_PSCI_FEATURES PSCI_0_2_FN(10) +#define PSCI_1_0_FN_CPU_FREEZE PSCI_0_2_FN(11) +#define PSCI_1_0_FN_CPU_DEFAULT_SUSPEND PSCI_0_2_FN(12) +#define PSCI_1_0_FN_NODE_HW_STATE PSCI_0_2_FN(13) #define PSCI_1_0_FN_SYSTEM_SUSPEND PSCI_0_2_FN(14) #define PSCI_1_0_FN_SET_SUSPEND_MODE PSCI_0_2_FN(15) +#define PSCI_1_0_FN_STAT_RESIDENCY PSCI_0_2_FN(16) +#define PSCI_1_0_FN_STAT_COUNT PSCI_0_2_FN(17) + #define PSCI_1_1_FN_SYSTEM_RESET2 PSCI_0_2_FN(18) +#define PSCI_1_1_FN_MEM_PROTECT PSCI_0_2_FN(19) +#define PSCI_1_1_FN_MEM_PROTECT_CHECK_RANGE PSCI_0_2_FN(19) +#define PSCI_1_0_FN64_CPU_DEFAULT_SUSPEND PSCI_0_2_FN64(12) +#define PSCI_1_0_FN64_NODE_HW_STATE PSCI_0_2_FN64(13) #define PSCI_1_0_FN64_SYSTEM_SUSPEND PSCI_0_2_FN64(14) +#define PSCI_1_0_FN64_STAT_RESIDENCY PSCI_0_2_FN64(16) +#define PSCI_1_0_FN64_STAT_COUNT PSCI_0_2_FN64(17) + #define PSCI_1_1_FN64_SYSTEM_RESET2 PSCI_0_2_FN64(18) +#define PSCI_1_1_FN64_MEM_PROTECT_CHECK_RANGE PSCI_0_2_FN64(19) /* PSCI v0.2 power state encoding for CPU_SUSPEND function */ #define PSCI_0_2_POWER_STATE_ID_MASK 0xffff diff --git a/include/uapi/linux/sed-opal.h b/include/uapi/linux/sed-opal.h index 6f5af1a84213bb72ac262b9138b087076dd5c5a8..2573772e2fb3e946dcc122da9a75f52b3b2b2fb9 100644 --- a/include/uapi/linux/sed-opal.h +++ b/include/uapi/linux/sed-opal.h @@ -132,6 +132,18 @@ struct opal_read_write_table { __u64 priv; }; +#define OPAL_FL_SUPPORTED 0x00000001 +#define OPAL_FL_LOCKING_SUPPORTED 0x00000002 +#define OPAL_FL_LOCKING_ENABLED 0x00000004 +#define OPAL_FL_LOCKED 0x00000008 +#define OPAL_FL_MBR_ENABLED 0x00000010 +#define OPAL_FL_MBR_DONE 0x00000020 + +struct opal_status { + __u32 flags; + __u32 reserved; +}; + #define IOC_OPAL_SAVE _IOW('p', 220, struct opal_lock_unlock) #define IOC_OPAL_LOCK_UNLOCK _IOW('p', 221, struct opal_lock_unlock) #define IOC_OPAL_TAKE_OWNERSHIP _IOW('p', 222, struct opal_key) @@ -148,5 +160,6 @@ struct opal_read_write_table { #define IOC_OPAL_MBR_DONE _IOW('p', 233, struct opal_mbr_done) #define IOC_OPAL_WRITE_SHADOW_MBR _IOW('p', 234, struct opal_shadow_mbr) #define IOC_OPAL_GENERIC_TABLE_RW _IOW('p', 235, struct opal_read_write_table) +#define IOC_OPAL_GET_STATUS _IOR('p', 236, struct opal_status) #endif /* _UAPI_SED_OPAL_H */ diff --git a/include/uapi/linux/seg6_local.h b/include/uapi/linux/seg6_local.h index 332b18f318f8bbb5cd7e45f2f1784979823e5794..4fdc424c9cb3a115f66e258b9a792e2e2b440f31 100644 --- a/include/uapi/linux/seg6_local.h +++ b/include/uapi/linux/seg6_local.h @@ -28,6 +28,7 @@ enum { SEG6_LOCAL_BPF, SEG6_LOCAL_VRFTABLE, SEG6_LOCAL_COUNTERS, + SEG6_LOCAL_FLAVORS, __SEG6_LOCAL_MAX, }; #define SEG6_LOCAL_MAX (__SEG6_LOCAL_MAX - 1) @@ -110,4 +111,27 @@ enum { #define SEG6_LOCAL_CNT_MAX (__SEG6_LOCAL_CNT_MAX - 1) +/* SRv6 End* Flavor attributes */ +enum { + SEG6_LOCAL_FLV_UNSPEC, + SEG6_LOCAL_FLV_OPERATION, + SEG6_LOCAL_FLV_LCBLOCK_BITS, + SEG6_LOCAL_FLV_LCNODE_FN_BITS, + __SEG6_LOCAL_FLV_MAX, +}; + +#define SEG6_LOCAL_FLV_MAX (__SEG6_LOCAL_FLV_MAX - 1) + +/* Designed flavor operations for SRv6 End* Behavior */ +enum { + SEG6_LOCAL_FLV_OP_UNSPEC, + SEG6_LOCAL_FLV_OP_PSP, + SEG6_LOCAL_FLV_OP_USP, + SEG6_LOCAL_FLV_OP_USD, + SEG6_LOCAL_FLV_OP_NEXT_CSID, + __SEG6_LOCAL_FLV_OP_MAX +}; + +#define SEG6_LOCAL_FLV_OP_MAX (__SEG6_LOCAL_FLV_OP_MAX - 1) + #endif diff --git a/include/uapi/linux/stat.h b/include/uapi/linux/stat.h index 1500a0f58041ae24924cad59dcf0fa4b4d6f68b6..7cab2c65d3d7fce9210d2fb6d02012233b9923cf 100644 --- a/include/uapi/linux/stat.h +++ b/include/uapi/linux/stat.h @@ -124,7 +124,8 @@ struct statx { __u32 stx_dev_minor; /* 0x90 */ __u64 stx_mnt_id; - __u64 __spare2; + __u32 stx_dio_mem_align; /* Memory buffer alignment for direct I/O */ + __u32 stx_dio_offset_align; /* File offset alignment for direct I/O */ /* 0xa0 */ __u64 __spare3[12]; /* Spare space for future expansion */ /* 0x100 */ @@ -152,6 +153,7 @@ struct statx { #define STATX_BASIC_STATS 0x000007ffU /* The stuff in the normal stat struct */ #define STATX_BTIME 0x00000800U /* Want/got stx_btime */ #define STATX_MNT_ID 0x00001000U /* Got stx_mnt_id */ +#define STATX_DIOALIGN 0x00002000U /* Want/got direct I/O alignment info */ #define STATX__RESERVED 0x80000000U /* Reserved for future struct statx expansion */ diff --git a/include/uapi/linux/tc_act/tc_bpf.h b/include/uapi/linux/tc_act/tc_bpf.h index 653c4f94f76e371cf08283010db13615f5d1dfc0..fe6c8f8f3e8c60961135609d5bce024979064ccc 100644 --- a/include/uapi/linux/tc_act/tc_bpf.h +++ b/include/uapi/linux/tc_act/tc_bpf.h @@ -1,11 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (c) 2015 Jiri Pirko - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. */ #ifndef __LINUX_TC_BPF_H diff --git a/include/uapi/linux/tc_act/tc_skbedit.h b/include/uapi/linux/tc_act/tc_skbedit.h index 6cb6101208d0a7f75f4eae9f99d7b3886e8f51a2..64032513cc4cad0aea255adb97e18888cb405527 100644 --- a/include/uapi/linux/tc_act/tc_skbedit.h +++ b/include/uapi/linux/tc_act/tc_skbedit.h @@ -2,19 +2,6 @@ /* * Copyright (c) 2008, Intel Corporation. * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * * Author: Alexander Duyck */ diff --git a/include/uapi/linux/tc_act/tc_skbmod.h b/include/uapi/linux/tc_act/tc_skbmod.h index af6ef2cfbf3db8ef86937c4c9c85bfc7451c9b04..ac62c9a993ea07eb611505ef48a94b8500544bd3 100644 --- a/include/uapi/linux/tc_act/tc_skbmod.h +++ b/include/uapi/linux/tc_act/tc_skbmod.h @@ -1,12 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (c) 2016, Jamal Hadi Salim - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. -*/ + */ #ifndef __LINUX_TC_SKBMOD_H #define __LINUX_TC_SKBMOD_H diff --git a/include/uapi/linux/tc_act/tc_tunnel_key.h b/include/uapi/linux/tc_act/tc_tunnel_key.h index 3f10dc4e7a4bb43f8e0f0f013cdce23b3a3e1e8d..49ad4033951bf0feb3ddebba78a85feda2119234 100644 --- a/include/uapi/linux/tc_act/tc_tunnel_key.h +++ b/include/uapi/linux/tc_act/tc_tunnel_key.h @@ -2,11 +2,6 @@ /* * Copyright (c) 2016, Amir Vadai * Copyright (c) 2016, Mellanox Technologies. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. */ #ifndef __LINUX_TC_TUNNEL_KEY_H diff --git a/include/uapi/linux/tc_act/tc_vlan.h b/include/uapi/linux/tc_act/tc_vlan.h index 5b306fe815ccb0a13a4a9763ed728964d390661c..3e1f8e57cdd26005dd30f43d3ea1e3e7e798db8c 100644 --- a/include/uapi/linux/tc_act/tc_vlan.h +++ b/include/uapi/linux/tc_act/tc_vlan.h @@ -1,11 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (c) 2014 Jiri Pirko - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. */ #ifndef __LINUX_TC_VLAN_H diff --git a/include/uapi/linux/tls.h b/include/uapi/linux/tls.h index f1157d8f4acdbbc56b9a0a88d2040dd3d7d262a7..b66a800389cc0712c12d75e4172003c2d1862089 100644 --- a/include/uapi/linux/tls.h +++ b/include/uapi/linux/tls.h @@ -100,6 +100,20 @@ #define TLS_CIPHER_SM4_CCM_TAG_SIZE 16 #define TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE 8 +#define TLS_CIPHER_ARIA_GCM_128 57 +#define TLS_CIPHER_ARIA_GCM_128_IV_SIZE 8 +#define TLS_CIPHER_ARIA_GCM_128_KEY_SIZE 16 +#define TLS_CIPHER_ARIA_GCM_128_SALT_SIZE 4 +#define TLS_CIPHER_ARIA_GCM_128_TAG_SIZE 16 +#define TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE 8 + +#define TLS_CIPHER_ARIA_GCM_256 58 +#define TLS_CIPHER_ARIA_GCM_256_IV_SIZE 8 +#define TLS_CIPHER_ARIA_GCM_256_KEY_SIZE 32 +#define TLS_CIPHER_ARIA_GCM_256_SALT_SIZE 4 +#define TLS_CIPHER_ARIA_GCM_256_TAG_SIZE 16 +#define TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE 8 + #define TLS_SET_RECORD_TYPE 1 #define TLS_GET_RECORD_TYPE 2 @@ -156,6 +170,22 @@ struct tls12_crypto_info_sm4_ccm { unsigned char rec_seq[TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE]; }; +struct tls12_crypto_info_aria_gcm_128 { + struct tls_crypto_info info; + unsigned char iv[TLS_CIPHER_ARIA_GCM_128_IV_SIZE]; + unsigned char key[TLS_CIPHER_ARIA_GCM_128_KEY_SIZE]; + unsigned char salt[TLS_CIPHER_ARIA_GCM_128_SALT_SIZE]; + unsigned char rec_seq[TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE]; +}; + +struct tls12_crypto_info_aria_gcm_256 { + struct tls_crypto_info info; + unsigned char iv[TLS_CIPHER_ARIA_GCM_256_IV_SIZE]; + unsigned char key[TLS_CIPHER_ARIA_GCM_256_KEY_SIZE]; + unsigned char salt[TLS_CIPHER_ARIA_GCM_256_SALT_SIZE]; + unsigned char rec_seq[TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE]; +}; + enum { TLS_INFO_UNSPEC, TLS_INFO_VERSION, diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h index 677edaab2b66ea856724e3f40dc4d9b990d739f6..8f88e3a29998fb6e79672e3122caa88d11331945 100644 --- a/include/uapi/linux/ublk_cmd.h +++ b/include/uapi/linux/ublk_cmd.h @@ -17,7 +17,8 @@ #define UBLK_CMD_STOP_DEV 0x07 #define UBLK_CMD_SET_PARAMS 0x08 #define UBLK_CMD_GET_PARAMS 0x09 - +#define UBLK_CMD_START_USER_RECOVERY 0x10 +#define UBLK_CMD_END_USER_RECOVERY 0x11 /* * IO commands, issued by ublk server, and handled by ublk driver. * @@ -74,9 +75,14 @@ */ #define UBLK_F_NEED_GET_DATA (1UL << 2) +#define UBLK_F_USER_RECOVERY (1UL << 3) + +#define UBLK_F_USER_RECOVERY_REISSUE (1UL << 4) + /* device state */ #define UBLK_S_DEV_DEAD 0 #define UBLK_S_DEV_LIVE 1 +#define UBLK_S_DEV_QUIESCED 2 /* shipped via sqe->cmd of io_uring command */ struct ublksrv_ctrl_cmd { diff --git a/include/uapi/linux/usbip.h b/include/uapi/linux/usbip.h index fd393d908d8a3170c6ff57a431c924fdfd229b18..e4421ad55b2eedb02df062860818ee1c52f1dd8d 100644 --- a/include/uapi/linux/usbip.h +++ b/include/uapi/linux/usbip.h @@ -24,4 +24,30 @@ enum usbip_device_status { VDEV_ST_USED, VDEV_ST_ERROR }; + +/* USB URB Transfer flags: + * + * USBIP server and client (vchi) pack URBs in TCP packets. The following + * are the transfer type defines used in USBIP protocol. + */ + +#define USBIP_URB_SHORT_NOT_OK 0x0001 +#define USBIP_URB_ISO_ASAP 0x0002 +#define USBIP_URB_NO_TRANSFER_DMA_MAP 0x0004 +#define USBIP_URB_ZERO_PACKET 0x0040 +#define USBIP_URB_NO_INTERRUPT 0x0080 +#define USBIP_URB_FREE_BUFFER 0x0100 +#define USBIP_URB_DIR_IN 0x0200 +#define USBIP_URB_DIR_OUT 0 +#define USBIP_URB_DIR_MASK USBIP_URB_DIR_IN + +#define USBIP_URB_DMA_MAP_SINGLE 0x00010000 +#define USBIP_URB_DMA_MAP_PAGE 0x00020000 +#define USBIP_URB_DMA_MAP_SG 0x00040000 +#define USBIP_URB_MAP_LOCAL 0x00080000 +#define USBIP_URB_SETUP_MAP_SINGLE 0x00100000 +#define USBIP_URB_SETUP_MAP_LOCAL 0x00200000 +#define USBIP_URB_DMA_SG_COMBINED 0x00400000 +#define USBIP_URB_ALIGNED_TEMP_BUFFER 0x00800000 + #endif /* _UAPI_LINUX_USBIP_H */ diff --git a/include/uapi/linux/userfaultfd.h b/include/uapi/linux/userfaultfd.h index 7d32b1e797fb2f3deb67bebe097982256d69135a..005e5e306266454530472c001d5781908048ba9b 100644 --- a/include/uapi/linux/userfaultfd.h +++ b/include/uapi/linux/userfaultfd.h @@ -12,6 +12,10 @@ #include +/* ioctls for /dev/userfaultfd */ +#define USERFAULTFD_IOC 0xAA +#define USERFAULTFD_IOC_NEW _IO(USERFAULTFD_IOC, 0x00) + /* * If the UFFDIO_API is upgraded someday, the UFFDIO_UNREGISTER and * UFFDIO_WAKE ioctls should be defined as _IOW and not as _IOR. In diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h index 5f46bf4a570c135bb2ca729a6840d0e5adea0ede..b5e7d082b8adf65a25d71d520af633d0739763fc 100644 --- a/include/uapi/linux/v4l2-controls.h +++ b/include/uapi/linux/v4l2-controls.h @@ -225,6 +225,12 @@ enum v4l2_colorfx { */ #define V4L2_CID_USER_ISL7998X_BASE (V4L2_CID_USER_BASE + 0x1180) +/* + * The base for DW100 driver controls. + * We reserve 16 controls for this driver. + */ +#define V4L2_CID_USER_DW100_BASE (V4L2_CID_USER_BASE + 0x1190) + /* MPEG-class control IDs */ /* The MPEG controls are applicable to all codec controls * and the 'MPEG' part of the define is historical */ @@ -1730,7 +1736,7 @@ struct v4l2_vp8_segment { * @sharpness_level: matches sharpness_level syntax element. * @level: matches loop_filter_level syntax element. * @padding: padding field. Should be zeroed by applications. - * @flags: see V4L2_VP8_LF_FLAG_{}. + * @flags: see V4L2_VP8_LF_{}. * * This structure contains loop filter related parameters. * See the 'mb_lf_adjustments()' part of the frame header syntax, diff --git a/include/uapi/linux/vdpa.h b/include/uapi/linux/vdpa.h index 25c55cab3d7c7da5dbbc89a94562462cc907cf20..9bd79235c875f25ea6825dccd89858b5973796f2 100644 --- a/include/uapi/linux/vdpa.h +++ b/include/uapi/linux/vdpa.h @@ -46,12 +46,18 @@ enum vdpa_attr { VDPA_ATTR_DEV_NEGOTIATED_FEATURES, /* u64 */ VDPA_ATTR_DEV_MGMTDEV_MAX_VQS, /* u32 */ + /* virtio features that are supported by the vDPA management device */ VDPA_ATTR_DEV_SUPPORTED_FEATURES, /* u64 */ VDPA_ATTR_DEV_QUEUE_INDEX, /* u32 */ VDPA_ATTR_DEV_VENDOR_ATTR_NAME, /* string */ VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, /* u64 */ + VDPA_ATTR_DEV_FEATURES, /* u64 */ + + /* virtio features that are supported by the vDPA device */ + VDPA_ATTR_VDPA_DEV_SUPPORTED_FEATURES, /* u64 */ + /* new attributes must be added above here */ VDPA_ATTR_MAX, }; diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index 733a1cddde30a5a028af55aad33ba4c0b0b59ea8..d7d8e0922376cce4cf4893fee9d4939aa90d1e9e 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -986,6 +986,148 @@ enum vfio_device_mig_state { VFIO_DEVICE_STATE_RUNNING_P2P = 5, }; +/* + * Upon VFIO_DEVICE_FEATURE_SET, allow the device to be moved into a low power + * state with the platform-based power management. Device use of lower power + * states depends on factors managed by the runtime power management core, + * including system level support and coordinating support among dependent + * devices. Enabling device low power entry does not guarantee lower power + * usage by the device, nor is a mechanism provided through this feature to + * know the current power state of the device. If any device access happens + * (either from the host or through the vfio uAPI) when the device is in the + * low power state, then the host will move the device out of the low power + * state as necessary prior to the access. Once the access is completed, the + * device may re-enter the low power state. For single shot low power support + * with wake-up notification, see + * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP below. Access to mmap'd + * device regions is disabled on LOW_POWER_ENTRY and may only be resumed after + * calling LOW_POWER_EXIT. + */ +#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY 3 + +/* + * This device feature has the same behavior as + * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY with the exception that the user + * provides an eventfd for wake-up notification. When the device moves out of + * the low power state for the wake-up, the host will not allow the device to + * re-enter a low power state without a subsequent user call to one of the low + * power entry device feature IOCTLs. Access to mmap'd device regions is + * disabled on LOW_POWER_ENTRY_WITH_WAKEUP and may only be resumed after the + * low power exit. The low power exit can happen either through LOW_POWER_EXIT + * or through any other access (where the wake-up notification has been + * generated). The access to mmap'd device regions will not trigger low power + * exit. + * + * The notification through the provided eventfd will be generated only when + * the device has entered and is resumed from a low power state after + * calling this device feature IOCTL. A device that has not entered low power + * state, as managed through the runtime power management core, will not + * generate a notification through the provided eventfd on access. Calling the + * LOW_POWER_EXIT feature is optional in the case where notification has been + * signaled on the provided eventfd that a resume from low power has occurred. + */ +struct vfio_device_low_power_entry_with_wakeup { + __s32 wakeup_eventfd; + __u32 reserved; +}; + +#define VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP 4 + +/* + * Upon VFIO_DEVICE_FEATURE_SET, disallow use of device low power states as + * previously enabled via VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY or + * VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP device features. + * This device feature IOCTL may itself generate a wakeup eventfd notification + * in the latter case if the device had previously entered a low power state. + */ +#define VFIO_DEVICE_FEATURE_LOW_POWER_EXIT 5 + +/* + * Upon VFIO_DEVICE_FEATURE_SET start/stop device DMA logging. + * VFIO_DEVICE_FEATURE_PROBE can be used to detect if the device supports + * DMA logging. + * + * DMA logging allows a device to internally record what DMAs the device is + * initiating and report them back to userspace. It is part of the VFIO + * migration infrastructure that allows implementing dirty page tracking + * during the pre copy phase of live migration. Only DMA WRITEs are logged, + * and this API is not connected to VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE. + * + * When DMA logging is started a range of IOVAs to monitor is provided and the + * device can optimize its logging to cover only the IOVA range given. Each + * DMA that the device initiates inside the range will be logged by the device + * for later retrieval. + * + * page_size is an input that hints what tracking granularity the device + * should try to achieve. If the device cannot do the hinted page size then + * it's the driver choice which page size to pick based on its support. + * On output the device will return the page size it selected. + * + * ranges is a pointer to an array of + * struct vfio_device_feature_dma_logging_range. + * + * The core kernel code guarantees to support by minimum num_ranges that fit + * into a single kernel page. User space can try higher values but should give + * up if the above can't be achieved as of some driver limitations. + * + * A single call to start device DMA logging can be issued and a matching stop + * should follow at the end. Another start is not allowed in the meantime. + */ +struct vfio_device_feature_dma_logging_control { + __aligned_u64 page_size; + __u32 num_ranges; + __u32 __reserved; + __aligned_u64 ranges; +}; + +struct vfio_device_feature_dma_logging_range { + __aligned_u64 iova; + __aligned_u64 length; +}; + +#define VFIO_DEVICE_FEATURE_DMA_LOGGING_START 6 + +/* + * Upon VFIO_DEVICE_FEATURE_SET stop device DMA logging that was started + * by VFIO_DEVICE_FEATURE_DMA_LOGGING_START + */ +#define VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP 7 + +/* + * Upon VFIO_DEVICE_FEATURE_GET read back and clear the device DMA log + * + * Query the device's DMA log for written pages within the given IOVA range. + * During querying the log is cleared for the IOVA range. + * + * bitmap is a pointer to an array of u64s that will hold the output bitmap + * with 1 bit reporting a page_size unit of IOVA. The mapping of IOVA to bits + * is given by: + * bitmap[(addr - iova)/page_size] & (1ULL << (addr % 64)) + * + * The input page_size can be any power of two value and does not have to + * match the value given to VFIO_DEVICE_FEATURE_DMA_LOGGING_START. The driver + * will format its internal logging to match the reporting page size, possibly + * by replicating bits if the internal page size is lower than requested. + * + * The LOGGING_REPORT will only set bits in the bitmap and never clear or + * perform any initialization of the user provided bitmap. + * + * If any error is returned userspace should assume that the dirty log is + * corrupted. Error recovery is to consider all memory dirty and try to + * restart the dirty tracking, or to abort/restart the whole migration. + * + * If DMA logging is not enabled, an error will be returned. + * + */ +struct vfio_device_feature_dma_logging_report { + __aligned_u64 iova; + __aligned_u64 length; + __aligned_u64 page_size; + __aligned_u64 bitmap; +}; + +#define VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT 8 + /* -------- API for Type1 VFIO IOMMU -------- */ /** diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index 01e630f2ec786938a498f37dbd9cce80b5e3c231..86cae23cc44696fcdc2db1f22d9619701eb855df 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -502,7 +502,6 @@ struct v4l2_capability { #define V4L2_CAP_META_CAPTURE 0x00800000 /* Is a metadata capture device */ #define V4L2_CAP_READWRITE 0x01000000 /* read/write systemcalls */ -#define V4L2_CAP_ASYNCIO 0x02000000 /* async I/O */ #define V4L2_CAP_STREAMING 0x04000000 /* streaming I/O ioctls */ #define V4L2_CAP_META_OUTPUT 0x08000000 /* Is a metadata output device */ @@ -2435,6 +2434,7 @@ struct v4l2_event_vsync { #define V4L2_EVENT_CTRL_CH_VALUE (1 << 0) #define V4L2_EVENT_CTRL_CH_FLAGS (1 << 1) #define V4L2_EVENT_CTRL_CH_RANGE (1 << 2) +#define V4L2_EVENT_CTRL_CH_DIMENSIONS (1 << 3) struct v4l2_event_ctrl { __u32 changes; @@ -2682,6 +2682,11 @@ struct v4l2_create_buffers { #ifndef __KERNEL__ #define V4L2_PIX_FMT_HM12 V4L2_PIX_FMT_NV12_16L16 #define V4L2_PIX_FMT_SUNXI_TILED_NV12 V4L2_PIX_FMT_NV12_32L32 +/* + * This capability was never implemented, anyone using this cap should drop it + * from their code. + */ +#define V4L2_CAP_ASYNCIO 0x02000000 #endif #endif /* _UAPI__LINUX_VIDEODEV2_H */ diff --git a/include/uapi/linux/virtio_blk.h b/include/uapi/linux/virtio_blk.h index d888f013d9ffad273cd48a828f71e6e718568154..58e70b24b504f50758a790dddad4a9d8d3c55e7d 100644 --- a/include/uapi/linux/virtio_blk.h +++ b/include/uapi/linux/virtio_blk.h @@ -40,6 +40,7 @@ #define VIRTIO_BLK_F_MQ 12 /* support more than one vq */ #define VIRTIO_BLK_F_DISCARD 13 /* DISCARD is supported */ #define VIRTIO_BLK_F_WRITE_ZEROES 14 /* WRITE ZEROES is supported */ +#define VIRTIO_BLK_F_SECURE_ERASE 16 /* Secure Erase is supported */ /* Legacy feature bits */ #ifndef VIRTIO_BLK_NO_LEGACY @@ -121,6 +122,21 @@ struct virtio_blk_config { __u8 write_zeroes_may_unmap; __u8 unused1[3]; + + /* the next 3 entries are guarded by VIRTIO_BLK_F_SECURE_ERASE */ + /* + * The maximum secure erase sectors (in 512-byte sectors) for + * one segment. + */ + __virtio32 max_secure_erase_sectors; + /* + * The maximum number of secure erase segments in a + * secure erase command. + */ + __virtio32 max_secure_erase_seg; + /* Secure erase commands must be aligned to this number of sectors. */ + __virtio32 secure_erase_sector_alignment; + } __attribute__((packed)); /* @@ -155,6 +171,9 @@ struct virtio_blk_config { /* Write zeroes command */ #define VIRTIO_BLK_T_WRITE_ZEROES 13 +/* Secure erase command */ +#define VIRTIO_BLK_T_SECURE_ERASE 14 + #ifndef VIRTIO_BLK_NO_LEGACY /* Barrier before this op. */ #define VIRTIO_BLK_T_BARRIER 0x80000000 diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h index 29ced55514d4173114550565ec752b1646920ca8..6cb842ea8979a1cf629a5519495e3432ece37ac4 100644 --- a/include/uapi/linux/virtio_net.h +++ b/include/uapi/linux/virtio_net.h @@ -56,7 +56,7 @@ #define VIRTIO_NET_F_MQ 22 /* Device supports Receive Flow * Steering */ #define VIRTIO_NET_F_CTRL_MAC_ADDR 23 /* Set MAC address */ -#define VIRTIO_NET_F_NOTF_COAL 53 /* Guest can handle notifications coalescing */ +#define VIRTIO_NET_F_NOTF_COAL 53 /* Device supports notifications coalescing */ #define VIRTIO_NET_F_HASH_REPORT 57 /* Supports hash report */ #define VIRTIO_NET_F_RSS 60 /* Supports RSS RX steering */ #define VIRTIO_NET_F_RSC_EXT 61 /* extended coalescing info */ @@ -364,24 +364,24 @@ struct virtio_net_hash_config { */ #define VIRTIO_NET_CTRL_NOTF_COAL 6 /* - * Set the tx-usecs/tx-max-packets patameters. - * tx-usecs - Maximum number of usecs to delay a TX notification. - * tx-max-packets - Maximum number of packets to send before a TX notification. + * Set the tx-usecs/tx-max-packets parameters. */ struct virtio_net_ctrl_coal_tx { + /* Maximum number of packets to send before a TX notification */ __le32 tx_max_packets; + /* Maximum number of usecs to delay a TX notification */ __le32 tx_usecs; }; #define VIRTIO_NET_CTRL_NOTF_COAL_TX_SET 0 /* - * Set the rx-usecs/rx-max-packets patameters. - * rx-usecs - Maximum number of usecs to delay a RX notification. - * rx-max-frames - Maximum number of packets to receive before a RX notification. + * Set the rx-usecs/rx-max-packets parameters. */ struct virtio_net_ctrl_coal_rx { + /* Maximum number of packets to receive before a RX notification */ __le32 rx_max_packets; + /* Maximum number of usecs to delay a RX notification */ __le32 rx_usecs; }; diff --git a/include/uapi/linux/xfrm.h b/include/uapi/linux/xfrm.h index b1f3e6a8f11a16dee683c295350c3aa89be61984..4f84ea7ee14c902ee473a843af97b396009e0a04 100644 --- a/include/uapi/linux/xfrm.h +++ b/include/uapi/linux/xfrm.h @@ -296,7 +296,7 @@ enum xfrm_attr_type_t { XFRMA_ETIMER_THRESH, XFRMA_SRCADDR, /* xfrm_address_t */ XFRMA_COADDR, /* xfrm_address_t */ - XFRMA_LASTUSED, /* unsigned long */ + XFRMA_LASTUSED, /* __u64 */ XFRMA_POLICY_TYPE, /* struct xfrm_userpolicy_type */ XFRMA_MIGRATE, XFRMA_ALG_AEAD, /* struct xfrm_algo_aead */ diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 5d06d5c74dd193bb8860a6c1bb5a7d6830676679..e00ebe05097d7f7d93b5b663d03951b5265123ec 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -707,6 +707,25 @@ enum hl_server_type { HL_SERVER_GAUDI2_HLS2 = 5 }; +/* + * Notifier event values - for the notification mechanism and the HL_INFO_GET_EVENTS command + * + * HL_NOTIFIER_EVENT_TPC_ASSERT - Indicates TPC assert event + * HL_NOTIFIER_EVENT_UNDEFINED_OPCODE - Indicates undefined operation code + * HL_NOTIFIER_EVENT_DEVICE_RESET - Indicates device requires a reset + * HL_NOTIFIER_EVENT_CS_TIMEOUT - Indicates CS timeout error + * HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE - Indicates device is unavailable + * HL_NOTIFIER_EVENT_USER_ENGINE_ERR - Indicates device engine in error state + * HL_NOTIFIER_EVENT_GENERAL_HW_ERR - Indicates device HW error + */ +#define HL_NOTIFIER_EVENT_TPC_ASSERT (1ULL << 0) +#define HL_NOTIFIER_EVENT_UNDEFINED_OPCODE (1ULL << 1) +#define HL_NOTIFIER_EVENT_DEVICE_RESET (1ULL << 2) +#define HL_NOTIFIER_EVENT_CS_TIMEOUT (1ULL << 3) +#define HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE (1ULL << 4) +#define HL_NOTIFIER_EVENT_USER_ENGINE_ERR (1ULL << 5) +#define HL_NOTIFIER_EVENT_GENERAL_HW_ERR (1ULL << 6) + /* Opcode for management ioctl * * HW_IP_INFO - Receive information about different IP blocks in the @@ -754,6 +773,7 @@ enum hl_server_type { * Razwi initiator. * Razwi cause, was it a page fault or MMU access error. * HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES - Retrieve valid page sizes for device memory allocation + * HL_INFO_SECURED_ATTESTATION - Retrieve attestation report of the boot. * HL_INFO_REGISTER_EVENTFD - Register eventfd for event notifications. * HL_INFO_UNREGISTER_EVENTFD - Unregister eventfd * HL_INFO_GET_EVENTS - Retrieve the last occurred events @@ -783,14 +803,19 @@ enum hl_server_type { #define HL_INFO_CS_TIMEOUT_EVENT 24 #define HL_INFO_RAZWI_EVENT 25 #define HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES 26 +#define HL_INFO_SECURED_ATTESTATION 27 #define HL_INFO_REGISTER_EVENTFD 28 #define HL_INFO_UNREGISTER_EVENTFD 29 #define HL_INFO_GET_EVENTS 30 #define HL_INFO_UNDEFINED_OPCODE_EVENT 31 +#define HL_INFO_ENGINE_STATUS 32 #define HL_INFO_VERSION_MAX_LEN 128 #define HL_INFO_CARD_NAME_MAX_LEN 16 +/* Maximum buffer size for retrieving engines status */ +#define HL_ENGINES_DATA_MAX_SIZE SZ_1M + /** * struct hl_info_hw_ip_info - hardware information on various IPs in the ASIC * @sram_base_address: The first SRAM physical base address that is free to be @@ -821,6 +846,7 @@ enum hl_server_type { * @tpc_enabled_mask: Bit-mask that represents which TPCs are enabled. Relevant * for Goya/Gaudi only. * @dram_enabled: Whether the DRAM is enabled. + * @security_enabled: Whether security is enabled on device. * @mme_master_slave_mode: Indicate whether the MME is working in master/slave * configuration. Relevant for Greco and later. * @cpucp_version: The CPUCP f/w version. @@ -852,7 +878,7 @@ struct hl_info_hw_ip_info { __u32 psoc_pci_pll_div_factor; __u8 tpc_enabled_mask; __u8 dram_enabled; - __u8 reserved; + __u8 security_enabled; __u8 mme_master_slave_mode; __u8 cpucp_version[HL_INFO_VERSION_MAX_LEN]; __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN]; @@ -876,13 +902,13 @@ struct hl_info_hw_idle { __u32 is_idle; /* * Bitmask of busy engines. - * Bits definition is according to `enum _enging_id'. + * Bits definition is according to `enum _engine_id'. */ __u32 busy_engines_mask; /* * Extended Bitmask of busy engines. - * Bits definition is according to `enum _enging_id'. + * Bits definition is according to `enum _engine_id'. */ __u64 busy_engines_mask_ext[HL_BUSY_ENGINES_MASK_EXT_SIZE]; }; @@ -1078,12 +1104,12 @@ struct hl_info_razwi_event { * struct hl_info_undefined_opcode_event - info about last undefined opcode error * @timestamp: timestamp of the undefined opcode error * @cb_addr_streams: CB addresses (per stream) that are currently exists in the PQ - * entiers. In case all streams array entries are + * entries. In case all streams array entries are * filled with values, it means the execution was in Lower-CP. * @cq_addr: the address of the current handled command buffer * @cq_size: the size of the current handled command buffer * @cb_addr_streams_len: num of streams - actual len of cb_addr_streams array. - * should be equal to 1 incase of undefined opcode + * should be equal to 1 in case of undefined opcode * in Upper-CP (specific stream) and equal to 4 incase * of undefined opcode in Lower-CP. * @engine_id: engine-id that the error occurred on @@ -1109,6 +1135,45 @@ struct hl_info_dev_memalloc_page_sizes { __u64 page_order_bitmask; }; +#define SEC_PCR_DATA_BUF_SZ 256 +#define SEC_PCR_QUOTE_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */ +#define SEC_SIGNATURE_BUF_SZ 255 /* (256 - 1) 1 byte used for size */ +#define SEC_PUB_DATA_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */ +#define SEC_CERTIFICATE_BUF_SZ 2046 /* (2048 - 2) 2 bytes used for size */ + +/* + * struct hl_info_sec_attest - attestation report of the boot + * @nonce: number only used once. random number provided by host. this also passed to the quote + * command as a qualifying data. + * @pcr_quote_len: length of the attestation quote data (bytes) + * @pub_data_len: length of the public data (bytes) + * @certificate_len: length of the certificate (bytes) + * @pcr_num_reg: number of PCR registers in the pcr_data array + * @pcr_reg_len: length of each PCR register in the pcr_data array (bytes) + * @quote_sig_len: length of the attestation report signature (bytes) + * @pcr_data: raw values of the PCR registers + * @pcr_quote: attestation report data structure + * @quote_sig: signature structure of the attestation report + * @public_data: public key for the signed attestation + * (outPublic + name + qualifiedName) + * @certificate: certificate for the attestation signing key + */ +struct hl_info_sec_attest { + __u32 nonce; + __u16 pcr_quote_len; + __u16 pub_data_len; + __u16 certificate_len; + __u8 pcr_num_reg; + __u8 pcr_reg_len; + __u8 quote_sig_len; + __u8 pcr_data[SEC_PCR_DATA_BUF_SZ]; + __u8 pcr_quote[SEC_PCR_QUOTE_BUF_SZ]; + __u8 quote_sig[SEC_SIGNATURE_BUF_SZ]; + __u8 public_data[SEC_PUB_DATA_BUF_SZ]; + __u8 certificate[SEC_CERTIFICATE_BUF_SZ]; + __u8 pad0[2]; +}; + enum gaudi_dcores { HL_GAUDI_WS_DCORE, HL_GAUDI_WN_DCORE, @@ -1130,6 +1195,11 @@ enum gaudi_dcores { * resolution. Currently not in use. * @pll_index: Index as defined in hl__pll_index enumeration. * @eventfd: event file descriptor for event notifications. + * @user_buffer_actual_size: Actual data size which was copied to user allocated buffer by the + * driver. It is possible for the user to allocate buffer larger than + * needed, hence updating this variable so user will know the exact amount + * of bytes copied by the kernel to the buffer. + * @sec_attest_nonce: Nonce number used for attestation report. * @pad: Padding to 64 bit. */ struct hl_info_args { @@ -1143,6 +1213,8 @@ struct hl_info_args { __u32 period_ms; __u32 pll_index; __u32 eventfd; + __u32 user_buffer_actual_size; + __u32 sec_attest_nonce; }; __u32 pad; @@ -1337,17 +1409,47 @@ struct hl_cs_chunk { #define HL_CS_FLAGS_RESERVE_SIGNALS_ONLY 0x1000 #define HL_CS_FLAGS_UNRESERVE_SIGNALS_ONLY 0x2000 +/* + * The engine cores CS is merged into the existing CS ioctls. + * Use it to control the engine cores mode. + */ +#define HL_CS_FLAGS_ENGINE_CORE_COMMAND 0x4000 + #define HL_CS_STATUS_SUCCESS 0 #define HL_MAX_JOBS_PER_CS 512 +/* HL_ENGINE_CORE_ values + * + * HL_ENGINE_CORE_HALT: engine core halt + * HL_ENGINE_CORE_RUN: engine core run + */ +#define HL_ENGINE_CORE_HALT (1 << 0) +#define HL_ENGINE_CORE_RUN (1 << 1) + struct hl_cs_in { - /* this holds address of array of hl_cs_chunk for restore phase */ - __u64 chunks_restore; + union { + struct { + /* this holds address of array of hl_cs_chunk for restore phase */ + __u64 chunks_restore; + + /* holds address of array of hl_cs_chunk for execution phase */ + __u64 chunks_execute; + }; + + /* Valid only when HL_CS_FLAGS_ENGINE_CORE_COMMAND is set */ + struct { + /* this holds address of array of uint32 for engine_cores */ + __u64 engine_cores; - /* holds address of array of hl_cs_chunk for execution phase */ - __u64 chunks_execute; + /* number of engine cores in engine_cores array */ + __u32 num_engine_cores; + + /* the core command to be sent towards engine cores */ + __u32 core_command; + }; + }; union { /* @@ -1412,7 +1514,7 @@ struct hl_cs_out { /* Valid only when HL_CS_FLAGS_RESERVE_SIGNALS_ONLY is set */ struct { - /* This is the resereved signal handle id */ + /* This is the reserved signal handle id */ __u32 handle_id; /* This is the signals count */ @@ -1874,21 +1976,6 @@ struct hl_debug_args { __u32 ctx_id; }; -/* - * Notifier event values - for the notification mechanism and the HL_INFO_GET_EVENTS command - * - * HL_NOTIFIER_EVENT_TPC_ASSERT - Indicates TPC assert event - * HL_NOTIFIER_EVENT_UNDEFINED_OPCODE - Indicates undefined operation code - * HL_NOTIFIER_EVENT_DEVICE_RESET - Indicates device requires a reset - * HL_NOTIFIER_EVENT_CS_TIMEOUT - Indicates CS timeout error - * HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE - Indicates device is unavailable - */ -#define HL_NOTIFIER_EVENT_TPC_ASSERT (1ULL << 0) -#define HL_NOTIFIER_EVENT_UNDEFINED_OPCODE (1ULL << 1) -#define HL_NOTIFIER_EVENT_DEVICE_RESET (1ULL << 2) -#define HL_NOTIFIER_EVENT_CS_TIMEOUT (1ULL << 3) -#define HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE (1ULL << 4) - /* * Various information operations such as: * - H/W IP information diff --git a/include/uapi/misc/uacce/hisi_qm.h b/include/uapi/misc/uacce/hisi_qm.h index 1faef5ff87ef908102468f8f062929752bdd0c07..3e66dbc2f3236083357ee47ff79401a094430ef3 100644 --- a/include/uapi/misc/uacce/hisi_qm.h +++ b/include/uapi/misc/uacce/hisi_qm.h @@ -14,11 +14,26 @@ struct hisi_qp_ctx { __u16 qc_type; }; +/** + * struct hisi_qp_info - User data for hisi qp. + * @sqe_size: Submission queue element size + * @sq_depth: The number of sqe + * @cq_depth: The number of cqe + * @reserved: Reserved data + */ +struct hisi_qp_info { + __u32 sqe_size; + __u16 sq_depth; + __u16 cq_depth; + __u64 reserved; +}; + #define HISI_QM_API_VER_BASE "hisi_qm_v1" #define HISI_QM_API_VER2_BASE "hisi_qm_v2" #define HISI_QM_API_VER3_BASE "hisi_qm_v3" /* UACCE_CMD_QM_SET_QP_CTX: Set qp algorithm type */ #define UACCE_CMD_QM_SET_QP_CTX _IOWR('H', 10, struct hisi_qp_ctx) - +/* UACCE_CMD_QM_SET_QP_INFO: Set qp depth and BD size */ +#define UACCE_CMD_QM_SET_QP_INFO _IOWR('H', 11, struct hisi_qp_info) #endif diff --git a/include/uapi/mtd/mtd-abi.h b/include/uapi/mtd/mtd-abi.h index 890d9e5b76d77edad99a5772b0151035ed6a3f7a..714d55b49d2ac3ea9379b71319aa8661a12ed2d1 100644 --- a/include/uapi/mtd/mtd-abi.h +++ b/include/uapi/mtd/mtd-abi.h @@ -55,9 +55,9 @@ struct mtd_oob_buf64 { * @MTD_OPS_RAW: data are transferred as-is, with no error correction; * this mode implies %MTD_OPS_PLACE_OOB * - * These modes can be passed to ioctl(MEMWRITE) and are also used internally. - * See notes on "MTD file modes" for discussion on %MTD_OPS_RAW vs. - * %MTD_FILE_MODE_RAW. + * These modes can be passed to ioctl(MEMWRITE) and ioctl(MEMREAD); they are + * also used internally. See notes on "MTD file modes" for discussion on + * %MTD_OPS_RAW vs. %MTD_FILE_MODE_RAW. */ enum { MTD_OPS_PLACE_OOB = 0, @@ -91,6 +91,53 @@ struct mtd_write_req { __u8 padding[7]; }; +/** + * struct mtd_read_req_ecc_stats - ECC statistics for a read operation + * + * @uncorrectable_errors: the number of uncorrectable errors that happened + * during the read operation + * @corrected_bitflips: the number of bitflips corrected during the read + * operation + * @max_bitflips: the maximum number of bitflips detected in any single ECC + * step for the data read during the operation; this information + * can be used to decide whether the data stored in a specific + * region of the MTD device should be moved somewhere else to + * avoid data loss. + */ +struct mtd_read_req_ecc_stats { + __u32 uncorrectable_errors; + __u32 corrected_bitflips; + __u32 max_bitflips; +}; + +/** + * struct mtd_read_req - data structure for requesting a read operation + * + * @start: start address + * @len: length of data buffer (only lower 32 bits are used) + * @ooblen: length of OOB buffer (only lower 32 bits are used) + * @usr_data: user-provided data buffer + * @usr_oob: user-provided OOB buffer + * @mode: MTD mode (see "MTD operation modes") + * @padding: reserved, must be set to 0 + * @ecc_stats: ECC statistics for the read operation + * + * This structure supports ioctl(MEMREAD) operations, allowing data and/or OOB + * reads in various modes. To read from OOB-only, set @usr_data == NULL, and to + * read data-only, set @usr_oob == NULL. However, setting both @usr_data and + * @usr_oob to NULL is not allowed. + */ +struct mtd_read_req { + __u64 start; + __u64 len; + __u64 ooblen; + __u64 usr_data; + __u64 usr_oob; + __u8 mode; + __u8 padding[7]; + struct mtd_read_req_ecc_stats ecc_stats; +}; + #define MTD_ABSENT 0 #define MTD_RAM 1 #define MTD_ROM 2 @@ -207,6 +254,12 @@ struct otp_info { #define MEMWRITE _IOWR('M', 24, struct mtd_write_req) /* Erase a given range of user data (must be in mode %MTD_FILE_MODE_OTP_USER) */ #define OTPERASE _IOW('M', 25, struct otp_info) +/* + * Most generic read interface; can read in-band and/or out-of-band in various + * modes (see "struct mtd_read_req"). This ioctl is not supported for flashes + * without OOB, e.g., NOR flash. + */ +#define MEMREAD _IOWR('M', 26, struct mtd_read_req) /* * Obsolete legacy interface. Keep it in order not to break userspace @@ -270,8 +323,9 @@ struct mtd_ecc_stats { * Note: %MTD_FILE_MODE_RAW provides the same functionality as %MTD_OPS_RAW - * raw access to the flash, without error correction or autoplacement schemes. * Wherever possible, the MTD_OPS_* mode will override the MTD_FILE_MODE_* mode - * (e.g., when using ioctl(MEMWRITE)), but in some cases, the MTD_FILE_MODE is - * used out of necessity (e.g., `write()', ioctl(MEMWRITEOOB64)). + * (e.g., when using ioctl(MEMWRITE) or ioctl(MEMREAD)), but in some cases, the + * MTD_FILE_MODE is used out of necessity (e.g., `write()', + * ioctl(MEMWRITEOOB64)). */ enum mtd_file_modes { MTD_FILE_MODE_NORMAL = MTD_OTP_OFF, diff --git a/include/uapi/mtd/ubi-user.h b/include/uapi/mtd/ubi-user.h index b69e9ba6742b96ebf1988069f18c2143acaeb26a..dcb179de43585f74ad80fc8da96a66e73f5fdf99 100644 --- a/include/uapi/mtd/ubi-user.h +++ b/include/uapi/mtd/ubi-user.h @@ -247,6 +247,7 @@ enum { * @vid_hdr_offset: VID header offset (use defaults if %0) * @max_beb_per1024: maximum expected number of bad PEB per 1024 PEBs * @padding: reserved for future, not used, has to be zeroed + * @disable_fm: whether disable fastmap * * This data structure is used to specify MTD device UBI has to attach and the * parameters it has to use. The number which should be assigned to the new UBI @@ -281,13 +282,18 @@ enum { * eraseblocks for new bad eraseblocks, but attempts to use available * eraseblocks (if any). The accepted range is 0-768. If 0 is given, the * default kernel value of %CONFIG_MTD_UBI_BEB_LIMIT will be used. + * + * If @disable_fm is not zero, ubi doesn't create new fastmap even the module + * param 'fm_autoconvert' is set, and existed old fastmap will be destroyed + * after doing full scanning. */ struct ubi_attach_req { __s32 ubi_num; __s32 mtd_num; __s32 vid_hdr_offset; __s16 max_beb_per1024; - __s8 padding[10]; + __s8 disable_fm; + __s8 padding[9]; }; /* diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h index 08035ccf1fff46eb8748c29ce7395bf108860776..163ac79556d6804de3f9492390bf488259542dcf 100644 --- a/include/uapi/rdma/efa-abi.h +++ b/include/uapi/rdma/efa-abi.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */ /* - * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved. */ #ifndef EFA_ABI_USER_H @@ -54,6 +54,7 @@ struct efa_ibv_alloc_pd_resp { enum { EFA_CREATE_CQ_WITH_COMPLETION_CHANNEL = 1 << 0, + EFA_CREATE_CQ_WITH_SGID = 1 << 1, }; struct efa_ibv_create_cq { @@ -118,6 +119,7 @@ enum { EFA_QUERY_DEVICE_CAPS_RDMA_READ = 1 << 0, EFA_QUERY_DEVICE_CAPS_RNR_RETRY = 1 << 1, EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS = 1 << 2, + EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID = 1 << 3, }; struct efa_ibv_ex_query_device_resp { diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h index 86be4a92b67bfe52ad3b55ee80002bad72ea4025..a96b7d2770e15d10c57860f7b6c470b90b04d24a 100644 --- a/include/uapi/rdma/mlx5-abi.h +++ b/include/uapi/rdma/mlx5-abi.h @@ -104,6 +104,7 @@ enum mlx5_ib_alloc_ucontext_resp_mask { MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE = 1UL << 2, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS = 1UL << 3, MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS = 1UL << 4, + MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG = 1UL << 5, }; enum mlx5_user_cmds_supp_uhw { diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h index 3bee490eb5857ffa4adfb3b52302887e38e2aa2d..595edad03dfe5495addaaf4063f064fc88497ce9 100644 --- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h +++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h @@ -174,6 +174,7 @@ enum mlx5_ib_devx_umem_reg_attrs { MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP, + MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD, }; enum mlx5_ib_devx_umem_dereg_attrs { diff --git a/include/uapi/rdma/rdma_user_rxe.h b/include/uapi/rdma/rdma_user_rxe.h index f09c5c9e3dd5d87108a3cb3967eae8480c0ee315..73f679dfd2dfb126ffeaf9949fd3c871da912ea9 100644 --- a/include/uapi/rdma/rdma_user_rxe.h +++ b/include/uapi/rdma/rdma_user_rxe.h @@ -74,7 +74,7 @@ struct rxe_av { struct rxe_send_wr { __aligned_u64 wr_id; - __u32 num_sge; + __u32 reserved; __u32 opcode; __u32 send_flags; union { @@ -166,7 +166,7 @@ struct rxe_send_wqe { struct rxe_recv_wqe { __aligned_u64 wr_id; - __u32 num_sge; + __u32 reserved; __u32 padding; struct rxe_dma_info dma; }; diff --git a/include/uapi/scsi/scsi_netlink_fc.h b/include/uapi/scsi/scsi_netlink_fc.h index 7535253f1a96d5617560cef3bfe5f10b6fa40ca0..b46e9cbeb001e4911eb4d351b5eea05e39c1ef06 100644 --- a/include/uapi/scsi/scsi_netlink_fc.h +++ b/include/uapi/scsi/scsi_netlink_fc.h @@ -35,7 +35,7 @@ * FC Transport Broadcast Event Message : * FC_NL_ASYNC_EVENT * - * Note: if Vendor Unique message, &event_data will be start of + * Note: if Vendor Unique message, event_data_flex will be start of * vendor unique payload, and the length of the payload is * per event_datalen * @@ -50,7 +50,10 @@ struct fc_nl_event { __u16 event_datalen; __u32 event_num; __u32 event_code; - __u32 event_data; + union { + __u32 event_data; + __DECLARE_FLEX_ARRAY(__u8, event_data_flex); + }; } __attribute__((aligned(sizeof(__u64)))); diff --git a/include/uapi/sound/asequencer.h b/include/uapi/sound/asequencer.h index a75e14edc957e448ffdc5917880aad0ce8117f37..6d4a2c60808dd2a5ed8cd5895cd5e7e69899edb0 100644 --- a/include/uapi/sound/asequencer.h +++ b/include/uapi/sound/asequencer.h @@ -3,22 +3,6 @@ * Main header file for the ALSA sequencer * Copyright (c) 1998-1999 by Frank van de Pol * (c) 1998-1999 by Jaroslav Kysela - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #ifndef _UAPI__SOUND_ASEQUENCER_H #define _UAPI__SOUND_ASEQUENCER_H diff --git a/include/uapi/sound/asoc.h b/include/uapi/sound/asoc.h index 053949287ce893b3ca6a1db422180edf874fc703..9f35bedafcff065006cbcb18d06dc42e45070bd8 100644 --- a/include/uapi/sound/asoc.h +++ b/include/uapi/sound/asoc.h @@ -5,10 +5,6 @@ * Copyright (C) 2012 Texas Instruments Inc. * Copyright (C) 2015 Intel Corporation. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * Simple file API to load FW that includes mixers, coefficients, DAPM graphs, * algorithms, equalisers, DAIs, widgets etc. */ diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h index 3974a2a911cc4993ba340ca594b2440f84a60f1f..de6810e94abed3e91de3ef42d1eaca0353c1b8d3 100644 --- a/include/uapi/sound/asound.h +++ b/include/uapi/sound/asound.h @@ -3,22 +3,6 @@ * Advanced Linux Sound Architecture - ALSA - Driver * Copyright (c) 1994-2003 by Jaroslav Kysela , * Abramo Bagnara - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #ifndef _UAPI__SOUND_ASOUND_H diff --git a/include/uapi/sound/asound_fm.h b/include/uapi/sound/asound_fm.h index 8471f404ff0b061b6132e327b9b74a74df38814c..25ec5e38af5c15c6928159627b758f9264b46397 100644 --- a/include/uapi/sound/asound_fm.h +++ b/include/uapi/sound/asound_fm.h @@ -10,21 +10,6 @@ * 4Front Technologies * * Direct FM control - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #define SNDRV_DM_FM_MODE_OPL2 0x00 diff --git a/include/uapi/sound/compress_offload.h b/include/uapi/sound/compress_offload.h index 3aef123dbd7f61d8dbdaad5fa0d06eb9fd48f4de..d185957f3fe0b7f9d0a526565c00d1845de4ae9a 100644 --- a/include/uapi/sound/compress_offload.h +++ b/include/uapi/sound/compress_offload.h @@ -5,23 +5,6 @@ * Copyright (C) 2011 Intel Corporation * Authors: Vinod Koul * Pierre-Louis Bossart - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * */ #ifndef __COMPRESS_OFFLOAD_H #define __COMPRESS_OFFLOAD_H diff --git a/include/uapi/sound/compress_params.h b/include/uapi/sound/compress_params.h index 7263617169195600b5749a1941356829e81b3b31..ddc77322d571ca4da025165c08293c3856926e12 100644 --- a/include/uapi/sound/compress_params.h +++ b/include/uapi/sound/compress_params.h @@ -7,47 +7,13 @@ * Authors: Pierre-Louis Bossart * Vinod Koul * - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * * The definitions in this file are derived from the OpenMAX AL version 1.1 - * and OpenMAX IL v 1.1.2 header files which contain the copyright notice below. + * and OpenMAX IL v 1.1.2 header files which contain the copyright notice below + * and are licensed under the MIT license. * * Copyright (c) 2007-2010 The Khronos Group Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and/or associated documentation files (the - * "Materials "), to deal in the Materials without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Materials, and to - * permit persons to whom the Materials are furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included - * in all copies or substantial portions of the Materials. - * - * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. - * */ #ifndef __SND_COMPRESS_PARAMS_H #define __SND_COMPRESS_PARAMS_H diff --git a/include/uapi/sound/emu10k1.h b/include/uapi/sound/emu10k1.h index 88609cc0524c9f20acdb24c916a9b3c29779713b..1c1f1dd446118ea457dc155ef6b26f6e2c9162fb 100644 --- a/include/uapi/sound/emu10k1.h +++ b/include/uapi/sound/emu10k1.h @@ -3,22 +3,6 @@ * Copyright (c) by Jaroslav Kysela , * Creative Labs, Inc. * Definitions for EMU10K1 (SB Live!) chips - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #ifndef _UAPI__SOUND_EMU10K1_H #define _UAPI__SOUND_EMU10K1_H diff --git a/include/uapi/sound/hdsp.h b/include/uapi/sound/hdsp.h index b8df62b60f4d021f827b338ba4bcbac219bf1a9a..0961954658d64fd55bedaf98497bb3b412c5b54f 100644 --- a/include/uapi/sound/hdsp.h +++ b/include/uapi/sound/hdsp.h @@ -4,20 +4,6 @@ /* * Copyright (C) 2003 Thomas Charbonnel (thomas@undata.org) - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifdef __linux__ diff --git a/include/uapi/sound/hdspm.h b/include/uapi/sound/hdspm.h index 14af3d00ea3f78c4f03f9682045ba2f3d367c4e8..7043bb3d435a9461148dbd73e49621e6244944cf 100644 --- a/include/uapi/sound/hdspm.h +++ b/include/uapi/sound/hdspm.h @@ -4,21 +4,6 @@ /* * Copyright (C) 2003 Winfried Ritsch (IEM) * based on hdsp.h from Thomas Charbonnel (thomas@undata.org) - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifdef __linux__ diff --git a/include/uapi/sound/sb16_csp.h b/include/uapi/sound/sb16_csp.h index e64851481d88b0764f46c0f2966b15ea5229e146..5a80f5ec02ee156a4648a9509aa369ba040d3197 100644 --- a/include/uapi/sound/sb16_csp.h +++ b/include/uapi/sound/sb16_csp.h @@ -4,21 +4,6 @@ * Takashi Iwai * * SB16ASP/AWE32 CSP control - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #ifndef _UAPI__SOUND_SB16_CSP_H #define _UAPI__SOUND_SB16_CSP_H diff --git a/include/uapi/sound/sfnt_info.h b/include/uapi/sound/sfnt_info.h index c9a810a6ef48d8f310d2cb6e4b74e76ea398b035..f2b5e13fb5a7e01f31f9c0048af285376b4add4c 100644 --- a/include/uapi/sound/sfnt_info.h +++ b/include/uapi/sound/sfnt_info.h @@ -6,21 +6,6 @@ * Patch record compatible with AWE driver on OSS * * Copyright (C) 1999-2000 Takashi Iwai - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #include diff --git a/include/uapi/sound/snd_sst_tokens.h b/include/uapi/sound/snd_sst_tokens.h index ff3748e9308a04a78a1cc388dd86fa01864ca833..defeb0c6ed2084654213974f58150be003121a72 100644 --- a/include/uapi/sound/snd_sst_tokens.h +++ b/include/uapi/sound/snd_sst_tokens.h @@ -4,16 +4,6 @@ * * Copyright (C) 2016 Intel Corp * Author: Shreyas NC - * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #ifndef __SND_SST_TOKENS_H__ #define __SND_SST_TOKENS_H__ diff --git a/include/uapi/sound/tlv.h b/include/uapi/sound/tlv.h index 7d6d65f60a42ea20ce17e9a778621773882a39ba..b99a2414b53dc8b12d014c7a0d835220479993cd 100644 --- a/include/uapi/sound/tlv.h +++ b/include/uapi/sound/tlv.h @@ -1,15 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ #ifndef __UAPI_SOUND_TLV_H #define __UAPI_SOUND_TLV_H diff --git a/include/uapi/sound/usb_stream.h b/include/uapi/sound/usb_stream.h index ffdd3ea1e31de9678eacb4d19cca88ab7ab7dd2c..50609016185aea4e21768e1cfbd98d47a8b635dd 100644 --- a/include/uapi/sound/usb_stream.h +++ b/include/uapi/sound/usb_stream.h @@ -1,20 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (C) 2007, 2008 Karsten Wiese - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #ifndef _UAPI__SOUND_USB_STREAM_H diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index 7fe1a926cd99f3888cafa0ea859cc3461095c195..9f28349ebcff5037218c45e6816d9066d2fd27cf 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -160,8 +160,10 @@ struct ufs_pm_lvl_states { * @task_tag: Task tag of the command * @lun: LUN of the command * @intr_cmd: Interrupt command (doesn't participate in interrupt aggregation) - * @issue_time_stamp: time stamp for debug purposes - * @compl_time_stamp: time stamp for statistics + * @issue_time_stamp: time stamp for debug purposes (CLOCK_MONOTONIC) + * @issue_time_stamp_local_clock: time stamp for debug purposes (local_clock) + * @compl_time_stamp: time stamp for statistics (CLOCK_MONOTONIC) + * @compl_time_stamp_local_clock: time stamp for debug purposes (local_clock) * @crypto_key_slot: the key slot to use for inline crypto (-1 if none) * @data_unit_num: the data unit number for the first block for inline crypto * @req_abort_skip: skip request abort task flag @@ -185,7 +187,9 @@ struct ufshcd_lrb { u8 lun; /* UPIU LUN id field is only 8-bit wide */ bool intr_cmd; ktime_t issue_time_stamp; + u64 issue_time_stamp_local_clock; ktime_t compl_time_stamp; + u64 compl_time_stamp_local_clock; #ifdef CONFIG_SCSI_UFS_CRYPTO int crypto_key_slot; u64 data_unit_num; @@ -430,7 +434,7 @@ struct ufs_clk_scaling { struct ufs_event_hist { int pos; u32 val[UFS_EVENT_HIST_LENGTH]; - ktime_t tstamp[UFS_EVENT_HIST_LENGTH]; + u64 tstamp[UFS_EVENT_HIST_LENGTH]; unsigned long long cnt; }; @@ -446,10 +450,10 @@ struct ufs_event_hist { */ struct ufs_stats { u32 last_intr_status; - ktime_t last_intr_ts; + u64 last_intr_ts; u32 hibern8_exit_cnt; - ktime_t last_hibern8_exit_tstamp; + u64 last_hibern8_exit_tstamp; struct ufs_event_hist event[UFS_EVT_CNT]; }; @@ -660,6 +664,12 @@ enum ufshcd_caps { * notification if it is supported by the UFS device. */ UFSHCD_CAP_TEMP_NOTIF = 1 << 11, + + /* + * Enable WriteBooster when scaling up the clock and disable + * WriteBooster when scaling the clock down. + */ + UFSHCD_CAP_WB_WITH_CLK_SCALING = 1 << 12, }; struct ufs_hba_variant_params { @@ -1017,6 +1027,11 @@ static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba) return hba->caps & UFSHCD_CAP_WB_EN; } +static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba) +{ + return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING; +} + #define ufshcd_writel(hba, val, reg) \ writel((val), (hba)->mmio_base + (reg)) #define ufshcd_readl(hba, reg) \ @@ -1160,26 +1175,6 @@ static inline int ufshcd_disable_host_tx_lcc(struct ufs_hba *hba) return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0); } -/* Expose Query-Request API */ -int ufshcd_query_descriptor_retry(struct ufs_hba *hba, - enum query_opcode opcode, - enum desc_idn idn, u8 index, - u8 selector, - u8 *desc_buf, int *buf_len); -int ufshcd_read_desc_param(struct ufs_hba *hba, - enum desc_idn desc_id, - int desc_index, - u8 param_offset, - u8 *param_read_buf, - u8 param_size); -int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, - enum attr_idn idn, u8 index, u8 selector, - u32 *attr_val); -int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, - enum attr_idn idn, u8 index, u8 selector, u32 *attr_val); -int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, - enum flag_idn idn, u8 index, bool *flag_res); - void ufshcd_auto_hibern8_enable(struct ufs_hba *hba); void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit); void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, @@ -1211,6 +1206,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, enum query_opcode desc_op); int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable); +int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable); int ufshcd_suspend_prepare(struct device *dev); int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm); void ufshcd_resume_complete(struct device *dev); diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index f81aa95ffbc40b490eb3fe3fd64f8b8deb0960c8..f525566a0864de12a3f2e4fdd62b606081a10565 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -135,11 +135,7 @@ static inline u32 ufshci_version(u32 major, u32 minor) #define UFSHCD_UIC_MASK (UIC_COMMAND_COMPL | UFSHCD_UIC_PWR_MASK) -#define UFSHCD_ERROR_MASK (UIC_ERROR |\ - DEVICE_FATAL_ERROR |\ - CONTROLLER_FATAL_ERROR |\ - SYSTEM_BUS_FATAL_ERROR |\ - CRYPTO_ENGINE_FATAL_ERROR) +#define UFSHCD_ERROR_MASK (UIC_ERROR | INT_FATAL_ERRORS) #define INT_FATAL_ERRORS (DEVICE_FATAL_ERROR |\ CONTROLLER_FATAL_ERROR |\ diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index dae0f350c6780bc4e214ddb24a3e12efc0f26a8d..a34f4271a2e9f348f0b99670e920843ffc322626 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -219,6 +219,7 @@ static inline void xen_preemptible_hcall_end(void) { } void xen_grant_setup_dma_ops(struct device *dev); bool xen_is_grant_dma_device(struct device *dev); bool xen_virtio_mem_acc(struct virtio_device *dev); +bool xen_virtio_restricted_mem_acc(struct virtio_device *dev); #else static inline void xen_grant_setup_dma_ops(struct device *dev) { @@ -234,6 +235,11 @@ static inline bool xen_virtio_mem_acc(struct virtio_device *dev) { return false; } + +static inline bool xen_virtio_restricted_mem_acc(struct virtio_device *dev) +{ + return false; +} #endif /* CONFIG_XEN_GRANT_DMA_OPS */ #endif /* INCLUDE_XEN_OPS_H */ diff --git a/init/.gitignore b/init/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..cbbe270cec00617004b3b818e584046255bd90e5 --- /dev/null +++ b/init/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +/utsversion-tmp.h diff --git a/init/Kconfig b/init/Kconfig index 532362fcfe31fd3c58b418ad54f9d42080b1a00e..694f7c160c9c107e513ca69b7efe17b232570405 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -60,6 +60,17 @@ config LLD_VERSION default $(ld-version) if LD_IS_LLD default 0 +config RUST_IS_AVAILABLE + def_bool $(success,$(srctree)/scripts/rust_is_available.sh) + help + This shows whether a suitable Rust toolchain is available (found). + + Please see Documentation/rust/quick-start.rst for instructions on how + to satify the build requirements of Rust support. + + In particular, the Makefile target 'rustavailable' is useful to check + why the Rust toolchain is not being detected. + config CC_CAN_LINK bool default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag)) if 64BIT @@ -147,7 +158,8 @@ config WERROR default COMPILE_TEST help A kernel build should not cause any compiler warnings, and this - enables the '-Werror' flag to enforce that rule by default. + enables the '-Werror' (for C) and '-Dwarnings' (for Rust) flags + to enforce that rule by default. However, if you have a new (or very old) compiler with odd and unusual warnings, or you have some architecture with problems, @@ -461,8 +473,7 @@ config VIRT_CPU_ACCOUNTING choice prompt "Cputime accounting" - default TICK_CPU_ACCOUNTING if !PPC64 - default VIRT_CPU_ACCOUNTING_NATIVE if PPC64 + default TICK_CPU_ACCOUNTING # Kind of a stub config for the pure tick based cputime accounting config TICK_CPU_ACCOUNTING @@ -958,11 +969,6 @@ config MEMCG help Provides control over the memory footprint of tasks in a cgroup. -config MEMCG_SWAP - bool - depends on MEMCG && SWAP - default y - config MEMCG_KMEM bool depends on MEMCG && !SLOB @@ -1261,6 +1267,7 @@ endif # NAMESPACES config CHECKPOINT_RESTORE bool "Checkpoint/restore support" + depends on PROC_FS select PROC_CHILDREN select KCMP default n @@ -1899,6 +1906,38 @@ config PROFILING Say Y here to enable the extended profiling support mechanisms used by profilers. +config RUST + bool "Rust support" + depends on HAVE_RUST + depends on RUST_IS_AVAILABLE + depends on !MODVERSIONS + depends on !GCC_PLUGINS + depends on !RANDSTRUCT + depends on !DEBUG_INFO_BTF + select CONSTRUCTORS + help + Enables Rust support in the kernel. + + This allows other Rust-related options, like drivers written in Rust, + to be selected. + + It is also required to be able to load external kernel modules + written in Rust. + + See Documentation/rust/ for more information. + + If unsure, say N. + +config RUSTC_VERSION_TEXT + string + depends on RUST + default $(shell,command -v $(RUSTC) >/dev/null 2>&1 && $(RUSTC) --version || echo n) + +config BINDGEN_VERSION_TEXT + string + depends on RUST + default $(shell,command -v $(BINDGEN) >/dev/null 2>&1 && $(BINDGEN) --version || echo n) + # # Place an empty function call at each tracepoint site. Can be # dynamically changed for a probe function. diff --git a/init/Makefile b/init/Makefile index d82623d7fc8e8c89d6442dfa15feac384a9ec2c4..8316c23bead26814a724e311c41d1b9c5431c86c 100644 --- a/init/Makefile +++ b/init/Makefile @@ -19,20 +19,43 @@ mounts-y := do_mounts.o mounts-$(CONFIG_BLK_DEV_RAM) += do_mounts_rd.o mounts-$(CONFIG_BLK_DEV_INITRD) += do_mounts_initrd.o -# dependencies on generated files need to be listed explicitly -$(obj)/version.o: include/generated/compile.h - -# compile.h changes depending on hostname, generation number, etc, -# so we regenerate it always. -# mkcompile_h will make sure to only update the -# actual file if its content has changed. - -quiet_cmd_compile.h = CHK $@ - cmd_compile.h = \ - $(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ - "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT_BUILD)" \ - "$(CONFIG_PREEMPT_DYNAMIC)" "$(CONFIG_PREEMPT_RT)" \ - "$(CONFIG_CC_VERSION_TEXT)" "$(LD)" - -include/generated/compile.h: FORCE - $(call cmd,compile.h) +# +# UTS_VERSION +# + +smp-flag-$(CONFIG_SMP) := SMP +preempt-flag-$(CONFIG_PREEMPT_BUILD) := PREEMPT +preempt-flag-$(CONFIG_PREEMPT_DYNAMIC) := PREEMPT_DYNAMIC +preempt-flag-$(CONFIG_PREEMPT_RT) := PREEMPT_RT + +build-version = $(or $(KBUILD_BUILD_VERSION), $(build-version-auto)) +build-timestamp = $(or $(KBUILD_BUILD_TIMESTAMP), $(build-timestamp-auto)) + +# Maximum length of UTS_VERSION is 64 chars +filechk_uts_version = \ + utsver=$$(echo '$(pound)'"$(build-version)" $(smp-flag-y) $(preempt-flag-y) "$(build-timestamp)" | cut -b -64); \ + echo '$(pound)'define UTS_VERSION \""$${utsver}"\" + +# +# Build version.c with temporary UTS_VERSION +# + +$(obj)/utsversion-tmp.h: FORCE + $(call filechk,uts_version) + +clean-files += utsversion-tmp.h + +$(obj)/version.o: $(obj)/utsversion-tmp.h +CFLAGS_version.o := -include $(obj)/utsversion-tmp.h + +# +# Build version-timestamp.c with final UTS_VERSION +# + +include/generated/utsversion.h: build-version-auto = $(shell $(srctree)/$(src)/build-version) +include/generated/utsversion.h: build-timestamp-auto = $(shell LC_ALL=C date) +include/generated/utsversion.h: FORCE + $(call filechk,uts_version) + +$(obj)/version-timestamp.o: include/generated/utsversion.h +CFLAGS_version-timestamp.o := -include include/generated/utsversion.h diff --git a/init/build-version b/init/build-version new file mode 100755 index 0000000000000000000000000000000000000000..537d458150832a66501c74f7d544a9e618d9f7c1 --- /dev/null +++ b/init/build-version @@ -0,0 +1,10 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +prev_ver=$(cat .version 2>/dev/null) && +ver=$(expr ${prev_ver} + 1 2>/dev/null) || +ver=1 + +echo ${ver} > .version + +echo ${ver} diff --git a/init/do_mounts.c b/init/do_mounts.c index 7058e14ad5f70ee9b56d49198b13f00951973490..811e94daf0a84af833eda996af765c1f05926680 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -296,7 +296,7 @@ EXPORT_SYMBOL_GPL(name_to_dev_t); static int __init root_dev_setup(char *line) { - strlcpy(saved_root_name, line, sizeof(saved_root_name)); + strscpy(saved_root_name, line, sizeof(saved_root_name)); return 1; } @@ -343,7 +343,7 @@ static int __init split_fs_names(char *page, size_t size, char *names) int count = 1; char *p = page; - strlcpy(p, root_fs_names, size); + strscpy(p, root_fs_names, size); while (*p++) { if (p[-1] == ',') { p[-1] = '\0'; diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c index 327962ea354c096fd56e4b7aa03cc5928440279f..34731241377d3097826799bd4464bfc91245d340 100644 --- a/init/do_mounts_initrd.c +++ b/init/do_mounts_initrd.c @@ -99,19 +99,11 @@ static void __init handle_initrd(void) init_mkdir("/old", 0700); init_chdir("/old"); - /* - * In case that a resume from disk is carried out by linuxrc or one of - * its children, we need to tell the freezer not to wait for us. - */ - current->flags |= PF_FREEZER_SKIP; - info = call_usermodehelper_setup("/linuxrc", argv, envp_init, GFP_KERNEL, init_linuxrc, NULL, NULL); if (!info) return; - call_usermodehelper_exec(info, UMH_WAIT_PROC); - - current->flags &= ~PF_FREEZER_SKIP; + call_usermodehelper_exec(info, UMH_WAIT_PROC|UMH_FREEZABLE); /* move initrd to rootfs' /old */ init_mount("..", ".", NULL, MS_MOVE, NULL); diff --git a/init/initramfs.c b/init/initramfs.c index 18229cfe8906b7632d7f3f94574c870145f565a6..2f5bfb7d765216c631759c1c37bf14ac90001982 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -482,7 +482,7 @@ static long __init flush_buffer(void *bufv, unsigned long len) return origLen; } -static unsigned long my_inptr; /* index of next byte to be processed in inbuf */ +static unsigned long my_inptr __initdata; /* index of next byte to be processed in inbuf */ #include diff --git a/init/main.c b/init/main.c index 91642a4e69be62b139fd77b6f7d7791d249b54fe..aa21add5f7c54aa64d846b0d9f7fd3299412baf5 100644 --- a/init/main.c +++ b/init/main.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -117,6 +118,7 @@ static int kernel_init(void *); extern void init_IRQ(void); extern void radix_tree_init(void); +extern void maple_tree_init(void); /* * Debug helper: via this flag we know that we are in 'early bootup code' @@ -422,7 +424,7 @@ static void __init setup_boot_config(void) if (!data) data = xbc_get_embedded_bootconfig(&size); - strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); + strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); err = parse_args("bootconfig", tmp_cmdline, NULL, 0, 0, 0, NULL, bootconfig_params); @@ -762,7 +764,7 @@ void __init parse_early_param(void) return; /* All fall through to do_early_param. */ - strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); + strscpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE); parse_early_options(tmp_cmdline); done = 1; } @@ -836,6 +838,7 @@ static void __init mm_init(void) init_mem_debugging_and_hardening(); kfence_alloc_pool(); report_meminit(); + kmsan_init_shadow(); stack_depot_early_init(); mem_init(); mem_init_print_info(); @@ -849,10 +852,14 @@ static void __init mm_init(void) pgtable_init(); debug_objects_mem_init(); vmalloc_init(); + /* Should be run after vmap initialization */ + if (early_page_ext_enabled()) + page_ext_init(); /* Should be run before the first non-init thread is created */ init_espfix_bsp(); /* Should be run after espfix64 is set up. */ pti_init(); + kmsan_init_runtime(); } #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET @@ -976,6 +983,9 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) parse_args("Setting extra init args", extra_init_args, NULL, 0, -1, -1, NULL, set_init_arg); + /* Architectural and non-timekeeping rng init, before allocator init */ + random_init_early(command_line); + /* * These use large bootmem allocations and must precede * kmem_cache_init() @@ -1002,6 +1012,7 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) "Interrupts were enabled *very* early, fixing it\n")) local_irq_disable(); radix_tree_init(); + maple_tree_init(); /* * Set up housekeeping before setting up workqueues to allow the unbound @@ -1035,17 +1046,13 @@ asmlinkage __visible void __init __no_sanitize_address start_kernel(void) hrtimers_init(); softirq_init(); timekeeping_init(); - kfence_init(); time_init(); - /* - * For best initial stack canary entropy, prepare it after: - * - setup_arch() for any UEFI RNG entropy and boot cmdline access - * - timekeeping_init() for ktime entropy used in random_init() - * - time_init() for making random_get_entropy() work on some platforms - * - random_init() to initialize the RNG from from early entropy sources - */ - random_init(command_line); + /* This must be after timekeeping is initialized */ + random_init(); + + /* These make use of the fully initialized rng */ + kfence_init(); boot_init_stack_canary(); perf_event_init(); @@ -1237,7 +1244,7 @@ __setup("initcall_blacklist=", initcall_blacklist); static __init_or_module void trace_initcall_start_cb(void *data, initcall_t fn) { - ktime_t *calltime = (ktime_t *)data; + ktime_t *calltime = data; printk(KERN_DEBUG "calling %pS @ %i\n", fn, task_pid_nr(current)); *calltime = ktime_get(); @@ -1246,7 +1253,7 @@ trace_initcall_start_cb(void *data, initcall_t fn) static __init_or_module void trace_initcall_finish_cb(void *data, initcall_t fn, int ret) { - ktime_t rettime, *calltime = (ktime_t *)data; + ktime_t rettime, *calltime = data; rettime = ktime_get(); printk(KERN_DEBUG "initcall %pS returned %d after %lld usecs\n", @@ -1446,13 +1453,25 @@ static noinline void __init kernel_init_freeable(void); #if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_STRICT_MODULE_RWX) bool rodata_enabled __ro_after_init = true; + +#ifndef arch_parse_debug_rodata +static inline bool arch_parse_debug_rodata(char *str) { return false; } +#endif + static int __init set_debug_rodata(char *str) { - if (strtobool(str, &rodata_enabled)) + if (arch_parse_debug_rodata(str)) + return 0; + + if (str && !strcmp(str, "on")) + rodata_enabled = true; + else if (str && !strcmp(str, "off")) + rodata_enabled = false; + else pr_warn("Invalid option string for rodata: '%s'\n", str); - return 1; + return 0; } -__setup("rodata=", set_debug_rodata); +early_param("rodata", set_debug_rodata); #endif #ifdef CONFIG_STRICT_KERNEL_RWX @@ -1606,7 +1625,8 @@ static noinline void __init kernel_init_freeable(void) padata_init(); page_alloc_init_late(); /* Initialize page ext after all struct pages are initialized. */ - page_ext_init(); + if (!early_page_ext_enabled()) + page_ext_init(); do_basic_setup(); diff --git a/init/version-timestamp.c b/init/version-timestamp.c new file mode 100644 index 0000000000000000000000000000000000000000..179e93bae5390e80cd903c691a303872906db4af --- /dev/null +++ b/init/version-timestamp.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include + +struct uts_namespace init_uts_ns = { + .ns.count = REFCOUNT_INIT(2), + .name = { + .sysname = UTS_SYSNAME, + .nodename = UTS_NODENAME, + .release = UTS_RELEASE, + .version = UTS_VERSION, + .machine = UTS_MACHINE, + .domainname = UTS_DOMAINNAME, + }, + .user_ns = &init_user_ns, + .ns.inum = PROC_UTS_INIT_INO, +#ifdef CONFIG_UTS_NS + .ns.ops = &utsns_operations, +#endif +}; + +/* FIXED STRINGS! Don't touch! */ +const char linux_banner[] = + "Linux version " UTS_RELEASE " (" LINUX_COMPILE_BY "@" + LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION "\n"; diff --git a/init/version.c b/init/version.c index b7f9559d417c74858220ac324b1a1e1ce2b09119..01d4ab05f0bac09c820b2379e5f0abf70c778cfc 100644 --- a/init/version.c +++ b/init/version.c @@ -16,27 +16,8 @@ #include #include #include -#include #include -struct uts_namespace init_uts_ns = { - .ns.count = REFCOUNT_INIT(2), - .name = { - .sysname = UTS_SYSNAME, - .nodename = UTS_NODENAME, - .release = UTS_RELEASE, - .version = UTS_VERSION, - .machine = UTS_MACHINE, - .domainname = UTS_DOMAINNAME, - }, - .user_ns = &init_user_ns, - .ns.inum = PROC_UTS_INIT_INO, -#ifdef CONFIG_UTS_NS - .ns.ops = &utsns_operations, -#endif -}; -EXPORT_SYMBOL_GPL(init_uts_ns); - static int __init early_hostname(char *arg) { size_t bufsize = sizeof(init_uts_ns.name.nodename); @@ -52,11 +33,6 @@ static int __init early_hostname(char *arg) } early_param("hostname", early_hostname); -/* FIXED STRINGS! Don't touch! */ -const char linux_banner[] = - "Linux version " UTS_RELEASE " (" LINUX_COMPILE_BY "@" - LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION "\n"; - const char linux_proc_banner[] = "%s version %s" " (" LINUX_COMPILE_BY "@" LINUX_COMPILE_HOST ")" @@ -64,3 +40,16 @@ const char linux_proc_banner[] = BUILD_SALT; BUILD_LTO_INFO; + +/* + * init_uts_ns and linux_banner contain the build version and timestamp, + * which are really fixed at the very last step of build process. + * They are compiled with __weak first, and without __weak later. + */ + +struct uts_namespace init_uts_ns __weak; +const char linux_banner[] __weak; + +#include "version-timestamp.c" + +EXPORT_SYMBOL_GPL(init_uts_ns); diff --git a/io_uring/cancel.c b/io_uring/cancel.c index e4e1dc0325f0c8db54b858f8af678bdc43cf6c47..2291a53cdabd1191242f53ac220833174fff2def 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -218,7 +218,7 @@ static int __io_sync_cancel(struct io_uring_task *tctx, (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) { unsigned long file_ptr; - if (unlikely(fd > ctx->nr_user_files)) + if (unlikely(fd >= ctx->nr_user_files)) return -EBADF; fd = array_index_nospec(fd, ctx->nr_user_files); file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr; @@ -292,7 +292,7 @@ int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg) break; mutex_unlock(&ctx->uring_lock); - ret = io_run_task_work_sig(); + ret = io_run_task_work_sig(ctx); if (ret < 0) { mutex_lock(&ctx->uring_lock); break; diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c index b29e2d02216f22184582a512f316ddcf798345ba..2e04850a657b075e6acf0e295beba6d2e230a7ff 100644 --- a/io_uring/fdinfo.c +++ b/io_uring/fdinfo.c @@ -60,13 +60,15 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, unsigned int cq_head = READ_ONCE(r->cq.head); unsigned int cq_tail = READ_ONCE(r->cq.tail); unsigned int cq_shift = 0; + unsigned int sq_shift = 0; unsigned int sq_entries, cq_entries; bool has_lock; - bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32); unsigned int i; - if (is_cqe32) + if (ctx->flags & IORING_SETUP_CQE32) cq_shift = 1; + if (ctx->flags & IORING_SETUP_SQE128) + sq_shift = 1; /* * we may get imprecise sqe and cqe info if uring is actively running @@ -82,19 +84,36 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, seq_printf(m, "CqHead:\t%u\n", cq_head); seq_printf(m, "CqTail:\t%u\n", cq_tail); seq_printf(m, "CachedCqTail:\t%u\n", ctx->cached_cq_tail); - seq_printf(m, "SQEs:\t%u\n", sq_tail - ctx->cached_sq_head); + seq_printf(m, "SQEs:\t%u\n", sq_tail - sq_head); sq_entries = min(sq_tail - sq_head, ctx->sq_entries); for (i = 0; i < sq_entries; i++) { unsigned int entry = i + sq_head; - unsigned int sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]); struct io_uring_sqe *sqe; + unsigned int sq_idx; + sq_idx = READ_ONCE(ctx->sq_array[entry & sq_mask]); if (sq_idx > sq_mask) continue; - sqe = &ctx->sq_sqes[sq_idx]; - seq_printf(m, "%5u: opcode:%d, fd:%d, flags:%x, user_data:%llu\n", - sq_idx, sqe->opcode, sqe->fd, sqe->flags, - sqe->user_data); + sqe = &ctx->sq_sqes[sq_idx << sq_shift]; + seq_printf(m, "%5u: opcode:%s, fd:%d, flags:%x, off:%llu, " + "addr:0x%llx, rw_flags:0x%x, buf_index:%d " + "user_data:%llu", + sq_idx, io_uring_get_opcode(sqe->opcode), sqe->fd, + sqe->flags, (unsigned long long) sqe->off, + (unsigned long long) sqe->addr, sqe->rw_flags, + sqe->buf_index, sqe->user_data); + if (sq_shift) { + u64 *sqeb = (void *) (sqe + 1); + int size = sizeof(struct io_uring_sqe) / sizeof(u64); + int j; + + for (j = 0; j < size; j++) { + seq_printf(m, ", e%d:0x%llx", j, + (unsigned long long) *sqeb); + sqeb++; + } + } + seq_printf(m, "\n"); } seq_printf(m, "CQEs:\t%u\n", cq_tail - cq_head); cq_entries = min(cq_tail - cq_head, ctx->cq_entries); @@ -102,16 +121,13 @@ static __cold void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, unsigned int entry = i + cq_head; struct io_uring_cqe *cqe = &r->cqes[(entry & cq_mask) << cq_shift]; - if (!is_cqe32) { - seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x\n", + seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x", entry & cq_mask, cqe->user_data, cqe->res, cqe->flags); - } else { - seq_printf(m, "%5u: user_data:%llu, res:%d, flag:%x, " - "extra1:%llu, extra2:%llu\n", - entry & cq_mask, cqe->user_data, cqe->res, - cqe->flags, cqe->big_cqe[0], cqe->big_cqe[1]); - } + if (cq_shift) + seq_printf(m, ", extra1:%llu, extra2:%llu\n", + cqe->big_cqe[0], cqe->big_cqe[1]); + seq_printf(m, "\n"); } /* diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index ebfdb2212ec2520829c8d1c209970b5869e1b026..de08d9902b30b59efaea8784172ff5a3dd3140ca 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -125,6 +125,11 @@ enum { IO_CHECK_CQ_DROPPED_BIT, }; +enum { + IO_EVENTFD_OP_SIGNAL_BIT, + IO_EVENTFD_OP_FREE_BIT, +}; + struct io_defer_entry { struct list_head list; struct io_kiocb *req; @@ -142,7 +147,7 @@ static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, static void io_dismantle_req(struct io_kiocb *req); static void io_clean_op(struct io_kiocb *req); static void io_queue_sqe(struct io_kiocb *req); - +static void io_move_task_work_from_local(struct io_ring_ctx *ctx); static void __io_submit_flush_completions(struct io_ring_ctx *ctx); static struct kmem_cache *req_cachep; @@ -316,6 +321,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) INIT_LIST_HEAD(&ctx->rsrc_ref_list); INIT_DELAYED_WORK(&ctx->rsrc_put_work, io_rsrc_put_work); init_llist_head(&ctx->rsrc_put_llist); + init_llist_head(&ctx->work_llist); INIT_LIST_HEAD(&ctx->tctx_list); ctx->submit_state.free_list.next = NULL; INIT_WQ_LIST(&ctx->locked_free_list); @@ -477,25 +483,28 @@ static __cold void io_queue_deferred(struct io_ring_ctx *ctx) } } -static void io_eventfd_signal(struct io_ring_ctx *ctx) + +static void io_eventfd_ops(struct rcu_head *rcu) { - struct io_ev_fd *ev_fd; - bool skip; + struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); + int ops = atomic_xchg(&ev_fd->ops, 0); - spin_lock(&ctx->completion_lock); - /* - * Eventfd should only get triggered when at least one event has been - * posted. Some applications rely on the eventfd notification count only - * changing IFF a new CQE has been added to the CQ ring. There's no - * depedency on 1:1 relationship between how many times this function is - * called (and hence the eventfd count) and number of CQEs posted to the - * CQ ring. + if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT)) + eventfd_signal(ev_fd->cq_ev_fd, 1); + + /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback + * ordering in a race but if references are 0 we know we have to free + * it regardless. */ - skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; - ctx->evfd_last_cq_tail = ctx->cached_cq_tail; - spin_unlock(&ctx->completion_lock); - if (skip) - return; + if (atomic_dec_and_test(&ev_fd->refs)) { + eventfd_ctx_put(ev_fd->cq_ev_fd); + kfree(ev_fd); + } +} + +static void io_eventfd_signal(struct io_ring_ctx *ctx) +{ + struct io_ev_fd *ev_fd = NULL; rcu_read_lock(); /* @@ -513,13 +522,46 @@ static void io_eventfd_signal(struct io_ring_ctx *ctx) goto out; if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) goto out; + if (ev_fd->eventfd_async && !io_wq_current_is_worker()) + goto out; - if (!ev_fd->eventfd_async || io_wq_current_is_worker()) + if (likely(eventfd_signal_allowed())) { eventfd_signal(ev_fd->cq_ev_fd, 1); + } else { + atomic_inc(&ev_fd->refs); + if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops)) + call_rcu(&ev_fd->rcu, io_eventfd_ops); + else + atomic_dec(&ev_fd->refs); + } + out: rcu_read_unlock(); } +static void io_eventfd_flush_signal(struct io_ring_ctx *ctx) +{ + bool skip; + + spin_lock(&ctx->completion_lock); + + /* + * Eventfd should only get triggered when at least one event has been + * posted. Some applications rely on the eventfd notification count + * only changing IFF a new CQE has been added to the CQ ring. There's + * no depedency on 1:1 relationship between how many times this + * function is called (and hence the eventfd count) and number of CQEs + * posted to the CQ ring. + */ + skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail; + ctx->evfd_last_cq_tail = ctx->cached_cq_tail; + spin_unlock(&ctx->completion_lock); + if (skip) + return; + + io_eventfd_signal(ctx); +} + void __io_commit_cqring_flush(struct io_ring_ctx *ctx) { if (ctx->off_timeout_used || ctx->drain_active) { @@ -531,7 +573,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) spin_unlock(&ctx->completion_lock); } if (ctx->has_evfd) - io_eventfd_signal(ctx); + io_eventfd_flush_signal(ctx); } static inline void io_cqring_ev_posted(struct io_ring_ctx *ctx) @@ -567,7 +609,7 @@ static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force) io_cq_lock(ctx); while (!list_empty(&ctx->cq_overflow_list)) { - struct io_uring_cqe *cqe = io_get_cqe(ctx); + struct io_uring_cqe *cqe = io_get_cqe_overflow(ctx, true); struct io_overflow_cqe *ocqe; if (!cqe && !force) @@ -694,12 +736,19 @@ bool io_req_cqe_overflow(struct io_kiocb *req) * control dependency is enough as we're using WRITE_ONCE to * fill the cq entry */ -struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx) +struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow) { struct io_rings *rings = ctx->rings; unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1); unsigned int free, queued, len; + /* + * Posting into the CQ when there are pending overflowed CQEs may break + * ordering guarantees, which will affect links, F_MORE users and more. + * Force overflow the completion. + */ + if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))) + return NULL; /* userspace may cheat modifying the tail, be safe and do min */ queued = min(__io_cqring_events(ctx), ctx->cq_entries); @@ -823,8 +872,12 @@ inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) void io_req_complete_failed(struct io_kiocb *req, s32 res) { + const struct io_op_def *def = &io_op_defs[req->opcode]; + req_set_fail(req); io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); + if (def->fail) + def->fail(req); io_req_complete_post(req); } @@ -1047,17 +1100,41 @@ void tctx_task_work(struct callback_head *cb) trace_io_uring_task_work_run(tctx, count, loops); } -void io_req_task_work_add(struct io_kiocb *req) +static void io_req_local_work_add(struct io_kiocb *req) +{ + struct io_ring_ctx *ctx = req->ctx; + + if (!llist_add(&req->io_task_work.node, &ctx->work_llist)) + return; + /* need it for the following io_cqring_wake() */ + smp_mb__after_atomic(); + + if (unlikely(atomic_read(&req->task->io_uring->in_idle))) { + io_move_task_work_from_local(ctx); + return; + } + + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) + atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); + + if (ctx->has_evfd) + io_eventfd_signal(ctx); + __io_cqring_wake(ctx); +} + +static inline void __io_req_task_work_add(struct io_kiocb *req, bool allow_local) { struct io_uring_task *tctx = req->task->io_uring; struct io_ring_ctx *ctx = req->ctx; struct llist_node *node; - bool running; - running = !llist_add(&req->io_task_work.node, &tctx->task_list); + if (allow_local && ctx->flags & IORING_SETUP_DEFER_TASKRUN) { + io_req_local_work_add(req); + return; + } /* task_work already pending, we're done */ - if (running) + if (!llist_add(&req->io_task_work.node, &tctx->task_list)) return; if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) @@ -1077,6 +1154,84 @@ void io_req_task_work_add(struct io_kiocb *req) } } +void io_req_task_work_add(struct io_kiocb *req) +{ + __io_req_task_work_add(req, true); +} + +static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) +{ + struct llist_node *node; + + node = llist_del_all(&ctx->work_llist); + while (node) { + struct io_kiocb *req = container_of(node, struct io_kiocb, + io_task_work.node); + + node = node->next; + __io_req_task_work_add(req, false); + } +} + +int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) +{ + struct llist_node *node; + struct llist_node fake; + struct llist_node *current_final = NULL; + int ret; + unsigned int loops = 1; + + if (unlikely(ctx->submitter_task != current)) + return -EEXIST; + + node = io_llist_xchg(&ctx->work_llist, &fake); + ret = 0; +again: + while (node != current_final) { + struct llist_node *next = node->next; + struct io_kiocb *req = container_of(node, struct io_kiocb, + io_task_work.node); + prefetch(container_of(next, struct io_kiocb, io_task_work.node)); + req->io_task_work.func(req, &locked); + ret++; + node = next; + } + + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) + atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); + + node = io_llist_cmpxchg(&ctx->work_llist, &fake, NULL); + if (node != &fake) { + loops++; + current_final = &fake; + node = io_llist_xchg(&ctx->work_llist, &fake); + goto again; + } + + if (locked) + io_submit_flush_completions(ctx); + trace_io_uring_local_work_run(ctx, ret, loops); + return ret; + +} + +int io_run_local_work(struct io_ring_ctx *ctx) +{ + bool locked; + int ret; + + if (llist_empty(&ctx->work_llist)) + return 0; + + __set_current_state(TASK_RUNNING); + locked = mutex_trylock(&ctx->uring_lock); + ret = __io_run_local_work(ctx, locked); + if (locked) + mutex_unlock(&ctx->uring_lock); + + return ret; +} + static void io_req_tw_post(struct io_kiocb *req, bool *locked) { io_req_complete_post(req); @@ -1183,7 +1338,7 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) struct io_wq_work_node *node, *prev; struct io_submit_state *state = &ctx->submit_state; - spin_lock(&ctx->completion_lock); + io_cq_lock(ctx); wq_list_for_each(node, prev, &state->compl_reqs) { struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); @@ -1254,6 +1409,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) int ret = 0; unsigned long check_cq; + if (!io_allowed_run_tw(ctx)) + return -EEXIST; + check_cq = READ_ONCE(ctx->check_cq); if (unlikely(check_cq)) { if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)) @@ -1284,13 +1442,19 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) * forever, while the workqueue is stuck trying to acquire the * very same mutex. */ - if (wq_list_empty(&ctx->iopoll_list)) { + if (wq_list_empty(&ctx->iopoll_list) || + io_task_work_pending(ctx)) { u32 tail = ctx->cached_cq_tail; - mutex_unlock(&ctx->uring_lock); - io_run_task_work(); - mutex_lock(&ctx->uring_lock); + if (!llist_empty(&ctx->work_llist)) + __io_run_local_work(ctx, true); + if (task_work_pending(current) || + wq_list_empty(&ctx->iopoll_list)) { + mutex_unlock(&ctx->uring_lock); + io_run_task_work(); + mutex_lock(&ctx->uring_lock); + } /* some requests don't go through iopoll_list */ if (tail != ctx->cached_cq_tail || wq_list_empty(&ctx->iopoll_list)) @@ -1377,7 +1541,7 @@ static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags) static bool io_bdev_nowait(struct block_device *bdev) { - return !bdev || blk_queue_nowait(bdev_get_queue(bdev)); + return !bdev || bdev_nowait(bdev); } /* @@ -1450,9 +1614,10 @@ int io_req_prep_async(struct io_kiocb *req) return 0; if (WARN_ON_ONCE(req_has_async_data(req))) return -EFAULT; - if (io_alloc_async_data(req)) - return -EAGAIN; - + if (!io_op_defs[req->opcode].manual_alloc) { + if (io_alloc_async_data(req)) + return -EAGAIN; + } return def->prep_async(req); } @@ -1727,13 +1892,10 @@ static void io_queue_async(struct io_kiocb *req, int ret) switch (io_arm_poll_handler(req, 0)) { case IO_APOLL_READY: + io_kbuf_recycle(req, 0); io_req_task_queue(req); break; case IO_APOLL_ABORTED: - /* - * Queued up for async execution, worker will release - * submit reference when the iocb is actually submitted. - */ io_kbuf_recycle(req, 0); io_queue_iowq(req, NULL); break; @@ -2147,6 +2309,13 @@ struct io_wait_queue { unsigned nr_timeouts; }; +static inline bool io_has_work(struct io_ring_ctx *ctx) +{ + return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || + ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && + !llist_empty(&ctx->work_llist)); +} + static inline bool io_should_wake(struct io_wait_queue *iowq) { struct io_ring_ctx *ctx = iowq->ctx; @@ -2165,20 +2334,20 @@ static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode, { struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, wq); + struct io_ring_ctx *ctx = iowq->ctx; /* * Cannot safely flush overflowed CQEs from here, ensure we wake up * the task, and the next invocation will do it. */ - if (io_should_wake(iowq) || - test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &iowq->ctx->check_cq)) + if (io_should_wake(iowq) || io_has_work(ctx)) return autoremove_wake_function(curr, mode, wake_flags, key); return -1; } -int io_run_task_work_sig(void) +int io_run_task_work_sig(struct io_ring_ctx *ctx) { - if (io_run_task_work()) + if (io_run_task_work_ctx(ctx) > 0) return 1; if (task_sigpending(current)) return -EINTR; @@ -2194,7 +2363,7 @@ static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx, unsigned long check_cq; /* make sure we run task_work before checking for signals */ - ret = io_run_task_work_sig(); + ret = io_run_task_work_sig(ctx); if (ret || io_should_wake(iowq)) return ret; @@ -2224,13 +2393,19 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, ktime_t timeout = KTIME_MAX; int ret; + if (!io_allowed_run_tw(ctx)) + return -EEXIST; + do { + /* always run at least 1 task work to process local work */ + ret = io_run_task_work_ctx(ctx); + if (ret < 0) + return ret; io_cqring_overflow_flush(ctx); + if (io_cqring_events(ctx) >= min_events) return 0; - if (!io_run_task_work()) - break; - } while (1); + } while (ret > 0); if (sig) { #ifdef CONFIG_COMPAT @@ -2364,17 +2539,11 @@ static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, ev_fd->eventfd_async = eventfd_async; ctx->has_evfd = true; rcu_assign_pointer(ctx->io_ev_fd, ev_fd); + atomic_set(&ev_fd->refs, 1); + atomic_set(&ev_fd->ops, 0); return 0; } -static void io_eventfd_put(struct rcu_head *rcu) -{ - struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu); - - eventfd_ctx_put(ev_fd->cq_ev_fd); - kfree(ev_fd); -} - static int io_eventfd_unregister(struct io_ring_ctx *ctx) { struct io_ev_fd *ev_fd; @@ -2384,7 +2553,8 @@ static int io_eventfd_unregister(struct io_ring_ctx *ctx) if (ev_fd) { ctx->has_evfd = false; rcu_assign_pointer(ctx->io_ev_fd, NULL); - call_rcu(&ev_fd->rcu, io_eventfd_put); + if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops)) + call_rcu(&ev_fd->rcu, io_eventfd_ops); return 0; } @@ -2416,12 +2586,6 @@ static void io_req_caches_free(struct io_ring_ctx *ctx) static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) { io_sq_thread_finish(ctx); - - if (ctx->mm_account) { - mmdrop(ctx->mm_account); - ctx->mm_account = NULL; - } - io_rsrc_refs_drop(ctx); /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */ io_wait_rsrc_data(ctx->buf_data); @@ -2462,8 +2626,11 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) } #endif WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list)); - WARN_ON_ONCE(ctx->notif_slots || ctx->nr_notif_slots); + if (ctx->mm_account) { + mmdrop(ctx->mm_account); + ctx->mm_account = NULL; + } io_mem_free(ctx->rings); io_mem_free(ctx->sq_sqes); @@ -2507,8 +2674,8 @@ static __poll_t io_uring_poll(struct file *file, poll_table *wait) * Users may get EPOLLIN meanwhile seeing nothing in cqring, this * pushs them to do the flush. */ - if (io_cqring_events(ctx) || - test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) + + if (io_cqring_events(ctx) || io_has_work(ctx)) mask |= EPOLLIN | EPOLLRDNORM; return mask; @@ -2571,6 +2738,9 @@ static __cold void io_ring_exit_work(struct work_struct *work) * as nobody else will be looking for them. */ do { + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + io_move_task_work_from_local(ctx); + while (io_uring_try_cancel_requests(ctx, NULL, true)) cond_resched(); @@ -2639,7 +2809,6 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_unregister_personality(ctx, index); if (ctx->rings) io_poll_remove_all(ctx, NULL, true); - io_notif_unregister(ctx); mutex_unlock(&ctx->uring_lock); /* failed during ring init, it couldn't have issued any requests */ @@ -2647,6 +2816,9 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) io_kill_timeouts(ctx, NULL, true); /* if we failed setting up the ctx, we might not have any rings */ io_iopoll_try_reap_events(ctx); + /* drop cached put refs after potentially doing completions */ + if (current->io_uring) + io_uring_drop_tctx_refs(current); } INIT_WORK(&ctx->exit_work, io_ring_exit_work); @@ -2766,13 +2938,15 @@ static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx, } } + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + ret |= io_run_local_work(ctx) > 0; ret |= io_cancel_defer_files(ctx, task, cancel_all); mutex_lock(&ctx->uring_lock); ret |= io_poll_remove_all(ctx, task, cancel_all); mutex_unlock(&ctx->uring_lock); ret |= io_kill_timeouts(ctx, task, cancel_all); if (task) - ret |= io_run_task_work(); + ret |= io_run_task_work() > 0; return ret; } @@ -2988,8 +3162,6 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, struct fd f; long ret; - io_run_task_work(); - if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP | IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG | IORING_ENTER_REGISTERED_RING))) @@ -3055,12 +3227,22 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, mutex_unlock(&ctx->uring_lock); goto out; } - if ((flags & IORING_ENTER_GETEVENTS) && ctx->syscall_iopoll) - goto iopoll_locked; + if (flags & IORING_ENTER_GETEVENTS) { + if (ctx->syscall_iopoll) + goto iopoll_locked; + /* + * Ignore errors, we'll soon call io_cqring_wait() and + * it should handle ownership problems if any. + */ + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + (void)io_run_local_work_locked(ctx); + } mutex_unlock(&ctx->uring_lock); } + if (flags & IORING_ENTER_GETEVENTS) { int ret2; + if (ctx->syscall_iopoll) { /* * We disallow the app entering submit/complete with @@ -3179,7 +3361,7 @@ static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file) if (fd < 0) return fd; - ret = __io_uring_add_tctx_node(ctx, false); + ret = __io_uring_add_tctx_node(ctx); if (ret) { put_unused_fd(fd); return ret; @@ -3289,17 +3471,29 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, if (ctx->flags & IORING_SETUP_SQPOLL) { /* IPI related flags don't make sense with SQPOLL */ if (ctx->flags & (IORING_SETUP_COOP_TASKRUN | - IORING_SETUP_TASKRUN_FLAG)) + IORING_SETUP_TASKRUN_FLAG | + IORING_SETUP_DEFER_TASKRUN)) goto err; ctx->notify_method = TWA_SIGNAL_NO_IPI; } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) { ctx->notify_method = TWA_SIGNAL_NO_IPI; } else { - if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) + if (ctx->flags & IORING_SETUP_TASKRUN_FLAG && + !(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) goto err; ctx->notify_method = TWA_SIGNAL; } + /* + * For DEFER_TASKRUN we require the completion task to be the same as the + * submission task. This implies that there is only one submitter, so enforce + * that. + */ + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN && + !(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) { + goto err; + } + /* * This is just grabbed for accounting purposes. When a process exits, * the mm is exited and dropped before the files, hence we need to hang @@ -3353,6 +3547,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, goto err; } + if (ctx->flags & IORING_SETUP_SINGLE_ISSUER + && !(ctx->flags & IORING_SETUP_R_DISABLED)) + ctx->submitter_task = get_task_struct(current); + file = io_uring_get_file(ctx); if (IS_ERR(file)) { ret = PTR_ERR(file); @@ -3400,7 +3598,7 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params) IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL | IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG | IORING_SETUP_SQE128 | IORING_SETUP_CQE32 | - IORING_SETUP_SINGLE_ISSUER)) + IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN)) return -EINVAL; return io_uring_create(entries, &p, params); @@ -3544,6 +3742,9 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx) if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD; + if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) + ctx->submitter_task = get_task_struct(current); + if (ctx->restrictions.registered) ctx->restricted = 1; @@ -3695,6 +3896,9 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs))) return -ENXIO; + if (ctx->submitter_task && ctx->submitter_task != current) + return -EEXIST; + if (ctx->restricted) { if (opcode >= IORING_REGISTER_LAST) return -EINVAL; @@ -3838,15 +4042,6 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, break; ret = io_register_file_alloc_range(ctx, arg); break; - case IORING_REGISTER_NOTIFIERS: - ret = io_notif_register(ctx, arg, nr_args); - break; - case IORING_UNREGISTER_NOTIFIERS: - ret = -EINVAL; - if (arg || nr_args) - break; - ret = io_notif_unregister(ctx); - break; default: ret = -EINVAL; break; @@ -3872,7 +4067,7 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, ctx = f.file->private_data; - io_run_task_work(); + io_run_task_work_ctx(ctx); mutex_lock(&ctx->uring_lock); ret = __io_uring_register(ctx, opcode, arg, nr_args); @@ -3932,8 +4127,8 @@ static int __init io_uring_init(void) BUILD_BUG_SQE_ELEM(42, __u16, personality); BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); BUILD_BUG_SQE_ELEM(44, __u32, file_index); - BUILD_BUG_SQE_ELEM(44, __u16, notification_idx); - BUILD_BUG_SQE_ELEM(46, __u16, addr_len); + BUILD_BUG_SQE_ELEM(44, __u16, addr_len); + BUILD_BUG_SQE_ELEM(46, __u16, __pad3[0]); BUILD_BUG_SQE_ELEM(48, __u64, addr3); BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd); BUILD_BUG_SQE_ELEM(56, __u64, __pad2); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 2f73f83af9604e8c104fb41c2cf07ab22247c8ce..ef77d2aa3172ca470a7364af92abf57232ddbc8b 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -24,9 +24,11 @@ enum { IOU_STOP_MULTISHOT = -ECANCELED, }; -struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx); +struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); bool io_req_cqe_overflow(struct io_kiocb *req); -int io_run_task_work_sig(void); +int io_run_task_work_sig(struct io_ring_ctx *ctx); +int __io_run_local_work(struct io_ring_ctx *ctx, bool locked); +int io_run_local_work(struct io_ring_ctx *ctx); void io_req_complete_failed(struct io_kiocb *req, s32 res); void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); void io_req_complete_post(struct io_kiocb *req); @@ -91,7 +93,8 @@ static inline void io_cq_lock(struct io_ring_ctx *ctx) void io_cq_unlock_post(struct io_ring_ctx *ctx); -static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) +static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx, + bool overflow) { if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) { struct io_uring_cqe *cqe = ctx->cqe_cached; @@ -103,7 +106,12 @@ static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) return cqe; } - return __io_get_cqe(ctx); + return __io_get_cqe(ctx, overflow); +} + +static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx) +{ + return io_get_cqe_overflow(ctx, false); } static inline bool __io_fill_cqe_req(struct io_ring_ctx *ctx, @@ -195,17 +203,24 @@ static inline void io_commit_cqring(struct io_ring_ctx *ctx) smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); } -static inline void io_cqring_wake(struct io_ring_ctx *ctx) +/* requires smb_mb() prior, see wq_has_sleeper() */ +static inline void __io_cqring_wake(struct io_ring_ctx *ctx) { /* * wake_up_all() may seem excessive, but io_wake_function() and * io_should_wake() handle the termination of the loop and only * wake as many waiters as we need to. */ - if (wq_has_sleeper(&ctx->cq_wait)) + if (waitqueue_active(&ctx->cq_wait)) wake_up_all(&ctx->cq_wait); } +static inline void io_cqring_wake(struct io_ring_ctx *ctx) +{ + smp_mb(); + __io_cqring_wake(ctx); +} + static inline bool io_sqring_full(struct io_ring_ctx *ctx) { struct io_rings *r = ctx->rings; @@ -221,17 +236,50 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; } -static inline bool io_run_task_work(void) +static inline int io_run_task_work(void) { - if (test_thread_flag(TIF_NOTIFY_SIGNAL)) { + if (task_work_pending(current)) { + if (test_thread_flag(TIF_NOTIFY_SIGNAL)) + clear_notify_signal(); __set_current_state(TASK_RUNNING); - clear_notify_signal(); - if (task_work_pending(current)) - task_work_run(); - return true; + task_work_run(); + return 1; } - return false; + return 0; +} + +static inline bool io_task_work_pending(struct io_ring_ctx *ctx) +{ + return test_thread_flag(TIF_NOTIFY_SIGNAL) || + !wq_list_empty(&ctx->work_llist); +} + +static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) +{ + int ret = 0; + int ret2; + + if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) + ret = io_run_local_work(ctx); + + /* want to run this after in case more is added */ + ret2 = io_run_task_work(); + + /* Try propagate error in favour of if tasks were run, + * but still make sure to run them if requested + */ + if (ret >= 0) + ret += ret2; + + return ret; +} + +static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) +{ + if (llist_empty(&ctx->work_llist)) + return 0; + return __io_run_local_work(ctx, true); } static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) @@ -301,4 +349,10 @@ static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) return container_of(node, struct io_kiocb, comp_list); } +static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) +{ + return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) || + ctx->submitter_task == current); +} + #endif diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index d6af208d109ffe2956741c42a0a3daeb150745aa..c23e15d7d3cafd81fb0170faf2152dcd23b42682 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -86,14 +86,6 @@ static inline bool io_do_buffer_select(struct io_kiocb *req) static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags) { - /* - * READV uses fields in `struct io_rw` (len/addr) to stash the selected - * buffer data. However if that buffer is recycled the original request - * data stored in addr is lost. Therefore forbid recycling for now. - */ - if (req->opcode == IORING_OP_READV) - return; - if (req->flags & REQ_F_BUFFER_SELECTED) io_kbuf_recycle_legacy(req, issue_flags); if (req->flags & REQ_F_BUFFER_RING) diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 976c4ba68ee7ec07052d0a5473a487ed8a1c9822..4a7e5d030c782f1d175b69afb463b2cc52c7f814 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -165,7 +165,8 @@ done: req_set_fail(req); io_req_set_res(req, ret, 0); /* put file to avoid an attempt to IOPOLL the req */ - io_put_file(req->file); + if (!(req->flags & REQ_F_FIXED_FILE)) + io_put_file(req->file); req->file = NULL; return IOU_OK; } diff --git a/io_uring/net.c b/io_uring/net.c index f8cdf1dc3863be57a5a94211287220086f9d7384..8c7226b5bf41381f3ab720c7ea4bd9d2c2a7ccd3 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -46,6 +46,7 @@ struct io_connect { struct file *file; struct sockaddr __user *addr; int addr_len; + bool in_progress; }; struct io_sr_msg { @@ -55,22 +56,15 @@ struct io_sr_msg { struct user_msghdr __user *umsg; void __user *buf; }; + unsigned len; + unsigned done_io; unsigned msg_flags; - unsigned flags; - size_t len; - size_t done_io; -}; - -struct io_sendzc { - struct file *file; - void __user *buf; - size_t len; - u16 slot_idx; - unsigned msg_flags; - unsigned flags; - unsigned addr_len; + u16 flags; + /* initialised and used only by !msg send variants */ + u16 addr_len; void __user *addr; - size_t done_io; + /* used only for send zerocopy */ + struct io_kiocb *notif; }; #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) @@ -126,28 +120,36 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) } } -static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req, - unsigned int issue_flags) +static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req, + unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; struct io_cache_entry *entry; + struct io_async_msghdr *hdr; if (!(issue_flags & IO_URING_F_UNLOCKED) && (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { - struct io_async_msghdr *hdr; - hdr = container_of(entry, struct io_async_msghdr, cache); + hdr->free_iov = NULL; req->flags |= REQ_F_ASYNC_DATA; req->async_data = hdr; return hdr; } - if (!io_alloc_async_data(req)) - return req->async_data; - + if (!io_alloc_async_data(req)) { + hdr = req->async_data; + hdr->free_iov = NULL; + return hdr; + } return NULL; } +static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req) +{ + /* ->prep_async is always called from the submission context */ + return io_msg_alloc_async(req, 0); +} + static int io_setup_async_msg(struct io_kiocb *req, struct io_async_msghdr *kmsg, unsigned int issue_flags) @@ -156,17 +158,20 @@ static int io_setup_async_msg(struct io_kiocb *req, if (req_has_async_data(req)) return -EAGAIN; - async_msg = io_recvmsg_alloc_async(req, issue_flags); + async_msg = io_msg_alloc_async(req, issue_flags); if (!async_msg) { kfree(kmsg->free_iov); return -ENOMEM; } req->flags |= REQ_F_NEED_CLEANUP; memcpy(async_msg, kmsg, sizeof(*kmsg)); - async_msg->msg.msg_name = &async_msg->addr; + if (async_msg->msg.msg_name) + async_msg->msg.msg_name = &async_msg->addr; /* if were using fast_iov, set it to the new one */ - if (!async_msg->free_iov) - async_msg->msg.msg_iter.iov = async_msg->fast_iov; + if (!kmsg->free_iov) { + size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; + async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; + } return -EAGAIN; } @@ -182,10 +187,43 @@ static int io_sendmsg_copy_hdr(struct io_kiocb *req, &iomsg->free_iov); } +int io_send_prep_async(struct io_kiocb *req) +{ + struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); + struct io_async_msghdr *io; + int ret; + + if (!zc->addr || req_has_async_data(req)) + return 0; + io = io_msg_alloc_async_prep(req); + if (!io) + return -ENOMEM; + ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); + return ret; +} + +static int io_setup_async_addr(struct io_kiocb *req, + struct sockaddr_storage *addr_storage, + unsigned int issue_flags) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + struct io_async_msghdr *io; + + if (!sr->addr || req_has_async_data(req)) + return -EAGAIN; + io = io_msg_alloc_async(req, issue_flags); + if (!io) + return -ENOMEM; + memcpy(&io->addr, addr_storage, sizeof(io->addr)); + return -EAGAIN; +} + int io_sendmsg_prep_async(struct io_kiocb *req) { int ret; + if (!io_msg_alloc_async_prep(req)) + return -ENOMEM; ret = io_sendmsg_copy_hdr(req, req->async_data); if (!ret) req->flags |= REQ_F_NEED_CLEANUP; @@ -203,8 +241,14 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); - if (unlikely(sqe->file_index || sqe->addr2)) + if (req->opcode == IORING_OP_SEND) { + if (READ_ONCE(sqe->__pad3[0])) + return -EINVAL; + sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); + sr->addr_len = READ_ONCE(sqe->addr_len); + } else if (sqe->addr2 || sqe->file_index) { return -EINVAL; + } sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); @@ -260,13 +304,13 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) if (ret < min_ret) { if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) return io_setup_async_msg(req, kmsg, issue_flags); - if (ret == -ERESTARTSYS) - ret = -EINTR; if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; return io_setup_async_msg(req, kmsg, issue_flags); } + if (ret == -ERESTARTSYS) + ret = -EINTR; req_set_fail(req); } /* fast path, check for non-NULL to avoid function call */ @@ -284,6 +328,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) int io_send(struct io_kiocb *req, unsigned int issue_flags) { + struct sockaddr_storage __address; struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct iovec iov; @@ -292,9 +337,29 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) int min_ret = 0; int ret; + msg.msg_name = NULL; + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_namelen = 0; + msg.msg_ubuf = NULL; + + if (sr->addr) { + if (req_has_async_data(req)) { + struct io_async_msghdr *io = req->async_data; + + msg.msg_name = &io->addr; + } else { + ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address); + if (unlikely(ret < 0)) + return ret; + msg.msg_name = (struct sockaddr *)&__address; + } + msg.msg_namelen = sr->addr_len; + } + if (!(req->flags & REQ_F_POLLED) && (sr->flags & IORING_RECVSEND_POLL_FIRST)) - return -EAGAIN; + return io_setup_async_addr(req, &__address, issue_flags); sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -304,12 +369,6 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(ret)) return ret; - msg.msg_name = NULL; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_namelen = 0; - msg.msg_ubuf = NULL; - flags = sr->msg_flags; if (issue_flags & IO_URING_F_NONBLOCK) flags |= MSG_DONTWAIT; @@ -320,16 +379,17 @@ int io_send(struct io_kiocb *req, unsigned int issue_flags) ret = sock_sendmsg(sock, &msg); if (ret < min_ret) { if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) - return -EAGAIN; - if (ret == -ERESTARTSYS) - ret = -EINTR; + return io_setup_async_addr(req, &__address, issue_flags); + if (ret > 0 && io_net_retry(sock, flags)) { sr->len -= ret; sr->buf += ret; sr->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; - return -EAGAIN; + return io_setup_async_addr(req, &__address, issue_flags); } + if (ret == -ERESTARTSYS) + ret = -EINTR; req_set_fail(req); } if (ret >= 0) @@ -423,7 +483,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (msg.msg_iovlen == 0) { sr->len = 0; - iomsg->free_iov = NULL; } else if (msg.msg_iovlen > 1) { return -EINVAL; } else { @@ -434,7 +493,6 @@ static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, if (clen < 0) return -EINVAL; sr->len = clen; - iomsg->free_iov = NULL; } if (req->flags & REQ_F_APOLL_MULTISHOT) { @@ -473,6 +531,8 @@ int io_recvmsg_prep_async(struct io_kiocb *req) { int ret; + if (!io_msg_alloc_async_prep(req)) + return -ENOMEM; ret = io_recvmsg_copy_hdr(req, req->async_data); if (!ret) req->flags |= REQ_F_NEED_CLEANUP; @@ -720,13 +780,13 @@ retry_multishot: } return ret; } - if (ret == -ERESTARTSYS) - ret = -EINTR; if (ret > 0 && io_net_retry(sock, flags)) { sr->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; return io_setup_async_msg(req, kmsg, issue_flags); } + if (ret == -ERESTARTSYS) + ret = -EINTR; req_set_fail(req); } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { req_set_fail(req); @@ -816,8 +876,6 @@ retry_multishot: return -EAGAIN; } - if (ret == -ERESTARTSYS) - ret = -EINTR; if (ret > 0 && io_net_retry(sock, flags)) { sr->len -= ret; sr->buf += ret; @@ -825,6 +883,8 @@ retry_multishot: req->flags |= REQ_F_PARTIAL_IO; return -EAGAIN; } + if (ret == -ERESTARTSYS) + ret = -EINTR; req_set_fail(req); } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { out_free: @@ -848,18 +908,46 @@ out_free: return ret; } -int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +void io_send_zc_cleanup(struct io_kiocb *req) +{ + struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); + struct io_async_msghdr *io; + + if (req_has_async_data(req)) { + io = req->async_data; + /* might be ->fast_iov if *msg_copy_hdr failed */ + if (io->free_iov != io->fast_iov) + kfree(io->free_iov); + } + if (zc->notif) { + io_notif_flush(zc->notif); + zc->notif = NULL; + } +} + +int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); + struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_ring_ctx *ctx = req->ctx; + struct io_kiocb *notif; - if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)) + if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))) + return -EINVAL; + /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ + if (req->flags & REQ_F_CQE_SKIP) return -EINVAL; zc->flags = READ_ONCE(sqe->ioprio); if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | - IORING_RECVSEND_FIXED_BUF | IORING_RECVSEND_NOTIF_FLUSH)) + IORING_RECVSEND_FIXED_BUF)) return -EINVAL; + notif = zc->notif = io_alloc_notif(ctx); + if (!notif) + return -ENOMEM; + notif->cqe.user_data = req->cqe.user_data; + notif->cqe.res = 0; + notif->cqe.flags = IORING_CQE_F_NOTIF; + req->flags |= REQ_F_NEED_CLEANUP; if (zc->flags & IORING_RECVSEND_FIXED_BUF) { unsigned idx = READ_ONCE(sqe->buf_index); @@ -867,18 +955,27 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EFAULT; idx = array_index_nospec(idx, ctx->nr_user_bufs); req->imu = READ_ONCE(ctx->user_bufs[idx]); - io_req_set_rsrc_node(req, ctx, 0); + io_req_set_rsrc_node(notif, ctx, 0); + } + + if (req->opcode == IORING_OP_SEND_ZC) { + if (READ_ONCE(sqe->__pad3[0])) + return -EINVAL; + zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); + zc->addr_len = READ_ONCE(sqe->addr_len); + } else { + if (unlikely(sqe->addr2 || sqe->file_index)) + return -EINVAL; + if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) + return -EINVAL; } zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); zc->len = READ_ONCE(sqe->len); zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; - zc->slot_idx = READ_ONCE(sqe->notification_idx); if (zc->msg_flags & MSG_DONTWAIT) req->flags |= REQ_F_NOWAIT; - zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); - zc->addr_len = READ_ONCE(sqe->addr_len); zc->done_io = 0; #ifdef CONFIG_COMPAT @@ -888,6 +985,13 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } +static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb, + struct iov_iter *from, size_t length) +{ + skb_zcopy_downgrade_managed(skb); + return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); +} + static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, struct iov_iter *from, size_t length) { @@ -898,13 +1002,10 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, ssize_t copied = 0; unsigned long truesize = 0; - if (!shinfo->nr_frags) + if (!frag) shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; - - if (!skb_zcopy_managed(skb) || !iov_iter_is_bvec(from)) { - skb_zcopy_downgrade_managed(skb); + else if (unlikely(!skb_zcopy_managed(skb))) return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); - } bi.bi_size = min(from->count, length); bi.bi_bvec_done = from->iov_offset; @@ -925,7 +1026,7 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, shinfo->nr_frags = frag; from->bvec += bi.bi_idx; from->nr_segs -= bi.bi_idx; - from->count = bi.bi_size; + from->count -= copied; from->iov_offset = bi.bi_bvec_done; skb->data_len += copied; @@ -942,62 +1043,58 @@ static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, return ret; } -int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) +int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) { - struct sockaddr_storage address; - struct io_ring_ctx *ctx = req->ctx; - struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); - struct io_notif_slot *notif_slot; - struct io_kiocb *notif; + struct sockaddr_storage __address; + struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct iovec iov; struct socket *sock; unsigned msg_flags; int ret, min_ret = 0; - if (!(req->flags & REQ_F_POLLED) && - (zc->flags & IORING_RECVSEND_POLL_FIRST)) - return -EAGAIN; - - if (issue_flags & IO_URING_F_UNLOCKED) - return -EAGAIN; sock = sock_from_file(req->file); if (unlikely(!sock)) return -ENOTSOCK; - notif_slot = io_get_notif_slot(ctx, zc->slot_idx); - if (!notif_slot) - return -EINVAL; - notif = io_get_notif(ctx, notif_slot); - if (!notif) - return -ENOMEM; - msg.msg_name = NULL; msg.msg_control = NULL; msg.msg_controllen = 0; msg.msg_namelen = 0; if (zc->addr) { - ret = move_addr_to_kernel(zc->addr, zc->addr_len, &address); - if (unlikely(ret < 0)) - return ret; - msg.msg_name = (struct sockaddr *)&address; + if (req_has_async_data(req)) { + struct io_async_msghdr *io = req->async_data; + + msg.msg_name = &io->addr; + } else { + ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); + if (unlikely(ret < 0)) + return ret; + msg.msg_name = (struct sockaddr *)&__address; + } msg.msg_namelen = zc->addr_len; } + if (!(req->flags & REQ_F_POLLED) && + (zc->flags & IORING_RECVSEND_POLL_FIRST)) + return io_setup_async_addr(req, &__address, issue_flags); + if (zc->flags & IORING_RECVSEND_FIXED_BUF) { ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, (u64)(uintptr_t)zc->buf, zc->len); if (unlikely(ret)) - return ret; + return ret; + msg.sg_from_iter = io_sg_from_iter; } else { ret = import_single_range(WRITE, zc->buf, zc->len, &iov, &msg.msg_iter); if (unlikely(ret)) return ret; - ret = io_notif_account_mem(notif, zc->len); + ret = io_notif_account_mem(zc->notif, zc->len); if (unlikely(ret)) return ret; + msg.sg_from_iter = io_sg_from_iter_iovec; } msg_flags = zc->msg_flags | MSG_ZEROCOPY; @@ -1007,34 +1104,126 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) min_ret = iov_iter_count(&msg.msg_iter); msg.msg_flags = msg_flags; - msg.msg_ubuf = &io_notif_to_data(notif)->uarg; - msg.sg_from_iter = io_sg_from_iter; + msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; ret = sock_sendmsg(sock, &msg); if (unlikely(ret < min_ret)) { if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) - return -EAGAIN; + return io_setup_async_addr(req, &__address, issue_flags); + if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { zc->len -= ret; zc->buf += ret; zc->done_io += ret; req->flags |= REQ_F_PARTIAL_IO; - return -EAGAIN; + return io_setup_async_addr(req, &__address, issue_flags); } if (ret == -ERESTARTSYS) ret = -EINTR; - } else if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) { - io_notif_slot_flush_submit(notif_slot, 0); + req_set_fail(req); } if (ret >= 0) ret += zc->done_io; else if (zc->done_io) ret = zc->done_io; - io_req_set_res(req, ret, 0); + + /* + * If we're in io-wq we can't rely on tw ordering guarantees, defer + * flushing notif to io_send_zc_cleanup() + */ + if (!(issue_flags & IO_URING_F_UNLOCKED)) { + io_notif_flush(zc->notif); + req->flags &= ~REQ_F_NEED_CLEANUP; + } + io_req_set_res(req, ret, IORING_CQE_F_MORE); return IOU_OK; } +int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + struct io_async_msghdr iomsg, *kmsg; + struct socket *sock; + unsigned flags; + int ret, min_ret = 0; + + sock = sock_from_file(req->file); + if (unlikely(!sock)) + return -ENOTSOCK; + + if (req_has_async_data(req)) { + kmsg = req->async_data; + } else { + ret = io_sendmsg_copy_hdr(req, &iomsg); + if (ret) + return ret; + kmsg = &iomsg; + } + + if (!(req->flags & REQ_F_POLLED) && + (sr->flags & IORING_RECVSEND_POLL_FIRST)) + return io_setup_async_msg(req, kmsg, issue_flags); + + flags = sr->msg_flags | MSG_ZEROCOPY; + if (issue_flags & IO_URING_F_NONBLOCK) + flags |= MSG_DONTWAIT; + if (flags & MSG_WAITALL) + min_ret = iov_iter_count(&kmsg->msg.msg_iter); + + kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; + kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; + ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); + + if (unlikely(ret < min_ret)) { + if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) + return io_setup_async_msg(req, kmsg, issue_flags); + + if (ret > 0 && io_net_retry(sock, flags)) { + sr->done_io += ret; + req->flags |= REQ_F_PARTIAL_IO; + return io_setup_async_msg(req, kmsg, issue_flags); + } + if (ret == -ERESTARTSYS) + ret = -EINTR; + req_set_fail(req); + } + /* fast path, check for non-NULL to avoid function call */ + if (kmsg->free_iov) { + kfree(kmsg->free_iov); + kmsg->free_iov = NULL; + } + + io_netmsg_recycle(req, issue_flags); + if (ret >= 0) + ret += sr->done_io; + else if (sr->done_io) + ret = sr->done_io; + + /* + * If we're in io-wq we can't rely on tw ordering guarantees, defer + * flushing notif to io_send_zc_cleanup() + */ + if (!(issue_flags & IO_URING_F_UNLOCKED)) { + io_notif_flush(sr->notif); + req->flags &= ~REQ_F_NEED_CLEANUP; + } + io_req_set_res(req, ret, IORING_CQE_F_MORE); + return IOU_OK; +} + +void io_sendrecv_fail(struct io_kiocb *req) +{ + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + + if (req->flags & REQ_F_PARTIAL_IO) + req->cqe.res = sr->done_io; + + if ((req->flags & REQ_F_NEED_CLEANUP) && + (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) + req->cqe.flags |= IORING_CQE_F_MORE; +} + int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); @@ -1198,6 +1387,7 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); conn->addr_len = READ_ONCE(sqe->addr2); + conn->in_progress = false; return 0; } @@ -1209,6 +1399,16 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) int ret; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; + if (connect->in_progress) { + struct socket *socket; + + ret = -ENOTSOCK; + socket = sock_from_file(req->file); + if (socket) + ret = sock_error(socket->sk); + goto out; + } + if (req_has_async_data(req)) { io = req->async_data; } else { @@ -1225,13 +1425,17 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_connect_file(req->file, &io->address, connect->addr_len, file_flags); if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { - if (req_has_async_data(req)) - return -EAGAIN; - if (io_alloc_async_data(req)) { - ret = -ENOMEM; - goto out; + if (ret == -EINPROGRESS) { + connect->in_progress = true; + } else { + if (req_has_async_data(req)) + return -EAGAIN; + if (io_alloc_async_data(req)) { + ret = -ENOMEM; + goto out; + } + memcpy(req->async_data, &__io, sizeof(__io)); } - memcpy(req->async_data, &__io, sizeof(__io)); return -EAGAIN; } if (ret == -ERESTARTSYS) diff --git a/io_uring/net.h b/io_uring/net.h index 7c438d39c0899a17070bbb01efbbe4f20815aaab..5ffa11bf5d2e71ee9b3e185e3b95d9698394f6c5 100644 --- a/io_uring/net.h +++ b/io_uring/net.h @@ -35,13 +35,17 @@ int io_sendmsg_prep_async(struct io_kiocb *req); void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req); int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags); + int io_send(struct io_kiocb *req, unsigned int issue_flags); +int io_send_prep_async(struct io_kiocb *req); int io_recvmsg_prep_async(struct io_kiocb *req); int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags); int io_recv(struct io_kiocb *req, unsigned int issue_flags); +void io_sendrecv_fail(struct io_kiocb *req); + int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_accept(struct io_kiocb *req, unsigned int issue_flags); @@ -52,8 +56,10 @@ int io_connect_prep_async(struct io_kiocb *req); int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_connect(struct io_kiocb *req, unsigned int issue_flags); -int io_sendzc(struct io_kiocb *req, unsigned int issue_flags); -int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +int io_send_zc(struct io_kiocb *req, unsigned int issue_flags); +int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags); +int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +void io_send_zc_cleanup(struct io_kiocb *req); void io_netmsg_cache_free(struct io_cache_entry *entry); #else diff --git a/io_uring/notif.c b/io_uring/notif.c index 977736e82c1aab5c5870619b7bc18c70bfa94448..e37c6569d82e8da251357e9e30105757037a0b9e 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -21,14 +21,6 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked) io_req_task_complete(notif, locked); } -static inline void io_notif_complete(struct io_kiocb *notif) - __must_hold(¬if->ctx->uring_lock) -{ - bool locked = true; - - __io_notif_complete_tw(notif, &locked); -} - static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, bool success) @@ -42,8 +34,7 @@ static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, } } -struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, - struct io_notif_slot *slot) +struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_kiocb *notif; @@ -59,99 +50,23 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, io_get_task_refs(1); notif->rsrc_node = NULL; io_req_set_rsrc_node(notif, ctx, 0); - notif->cqe.user_data = slot->tag; - notif->cqe.flags = slot->seq++; - notif->cqe.res = 0; nd = io_notif_to_data(notif); nd->account_pages = 0; nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; nd->uarg.callback = io_uring_tx_zerocopy_callback; - /* master ref owned by io_notif_slot, will be dropped on flush */ refcount_set(&nd->uarg.refcnt, 1); return notif; } -void io_notif_slot_flush(struct io_notif_slot *slot) - __must_hold(&ctx->uring_lock) +void io_notif_flush(struct io_kiocb *notif) + __must_hold(&slot->notif->ctx->uring_lock) { - struct io_kiocb *notif = slot->notif; struct io_notif_data *nd = io_notif_to_data(notif); - slot->notif = NULL; - /* drop slot's master ref */ - if (refcount_dec_and_test(&nd->uarg.refcnt)) - io_notif_complete(notif); -} - -__cold int io_notif_unregister(struct io_ring_ctx *ctx) - __must_hold(&ctx->uring_lock) -{ - int i; - - if (!ctx->notif_slots) - return -ENXIO; - - for (i = 0; i < ctx->nr_notif_slots; i++) { - struct io_notif_slot *slot = &ctx->notif_slots[i]; - struct io_kiocb *notif = slot->notif; - struct io_notif_data *nd; - - if (!notif) - continue; - nd = io_notif_to_data(notif); - slot->notif = NULL; - if (!refcount_dec_and_test(&nd->uarg.refcnt)) - continue; + if (refcount_dec_and_test(&nd->uarg.refcnt)) { notif->io_task_work.func = __io_notif_complete_tw; io_req_task_work_add(notif); } - - kvfree(ctx->notif_slots); - ctx->notif_slots = NULL; - ctx->nr_notif_slots = 0; - return 0; -} - -__cold int io_notif_register(struct io_ring_ctx *ctx, - void __user *arg, unsigned int size) - __must_hold(&ctx->uring_lock) -{ - struct io_uring_notification_slot __user *slots; - struct io_uring_notification_slot slot; - struct io_uring_notification_register reg; - unsigned i; - - if (ctx->nr_notif_slots) - return -EBUSY; - if (size != sizeof(reg)) - return -EINVAL; - if (copy_from_user(®, arg, sizeof(reg))) - return -EFAULT; - if (!reg.nr_slots || reg.nr_slots > IORING_MAX_NOTIF_SLOTS) - return -EINVAL; - if (reg.resv || reg.resv2 || reg.resv3) - return -EINVAL; - - slots = u64_to_user_ptr(reg.data); - ctx->notif_slots = kvcalloc(reg.nr_slots, sizeof(ctx->notif_slots[0]), - GFP_KERNEL_ACCOUNT); - if (!ctx->notif_slots) - return -ENOMEM; - - for (i = 0; i < reg.nr_slots; i++, ctx->nr_notif_slots++) { - struct io_notif_slot *notif_slot = &ctx->notif_slots[i]; - - if (copy_from_user(&slot, &slots[i], sizeof(slot))) { - io_notif_unregister(ctx); - return -EFAULT; - } - if (slot.resv[0] | slot.resv[1] | slot.resv[2]) { - io_notif_unregister(ctx); - return -EINVAL; - } - notif_slot->tag = slot.tag; - } - return 0; } diff --git a/io_uring/notif.h b/io_uring/notif.h index 80f6445e0c2ba573d0db9287db49ccf87159cefc..5b4d710c8ca541bcb572dcef330ac16adec19d34 100644 --- a/io_uring/notif.h +++ b/io_uring/notif.h @@ -8,7 +8,6 @@ #include "rsrc.h" #define IO_NOTIF_SPLICE_BATCH 32 -#define IORING_MAX_NOTIF_SLOTS (1U << 15) struct io_notif_data { struct file *file; @@ -16,63 +15,14 @@ struct io_notif_data { unsigned long account_pages; }; -struct io_notif_slot { - /* - * Current/active notifier. A slot holds only one active notifier at a - * time and keeps one reference to it. Flush releases the reference and - * lazily replaces it with a new notifier. - */ - struct io_kiocb *notif; - - /* - * Default ->user_data for this slot notifiers CQEs - */ - u64 tag; - /* - * Notifiers of a slot live in generations, we create a new notifier - * only after flushing the previous one. Track the sequential number - * for all notifiers and copy it into notifiers's cqe->cflags - */ - u32 seq; -}; - -int io_notif_register(struct io_ring_ctx *ctx, - void __user *arg, unsigned int size); -int io_notif_unregister(struct io_ring_ctx *ctx); - -void io_notif_slot_flush(struct io_notif_slot *slot); -struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, - struct io_notif_slot *slot); +void io_notif_flush(struct io_kiocb *notif); +struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx); static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) { return io_kiocb_to_cmd(notif, struct io_notif_data); } -static inline struct io_kiocb *io_get_notif(struct io_ring_ctx *ctx, - struct io_notif_slot *slot) -{ - if (!slot->notif) - slot->notif = io_alloc_notif(ctx, slot); - return slot->notif; -} - -static inline struct io_notif_slot *io_get_notif_slot(struct io_ring_ctx *ctx, - unsigned idx) - __must_hold(&ctx->uring_lock) -{ - if (idx >= ctx->nr_notif_slots) - return NULL; - idx = array_index_nospec(idx, ctx->nr_notif_slots); - return &ctx->notif_slots[idx]; -} - -static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot, - unsigned int issue_flags) -{ - io_notif_slot_flush(slot); -} - static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) { struct io_ring_ctx *ctx = notif->ctx; diff --git a/io_uring/opdef.c b/io_uring/opdef.c index 72dd2b2d8a9df110963bce445bf0e89e6fef2c2e..83dc0f9ad3b2ffde2999e79904eb49b9f25737bd 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -69,6 +69,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_read, .prep_async = io_readv_prep_async, .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, }, [IORING_OP_WRITEV] = { .needs_file = 1, @@ -85,6 +86,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_write, .prep_async = io_writev_prep_async, .cleanup = io_readv_writev_cleanup, + .fail = io_rw_fail, }, [IORING_OP_FSYNC] = { .needs_file = 1, @@ -105,6 +107,7 @@ const struct io_op_def io_op_defs[] = { .name = "READ_FIXED", .prep = io_prep_rw, .issue = io_read, + .fail = io_rw_fail, }, [IORING_OP_WRITE_FIXED] = { .needs_file = 1, @@ -119,6 +122,7 @@ const struct io_op_def io_op_defs[] = { .name = "WRITE_FIXED", .prep = io_prep_rw, .issue = io_write, + .fail = io_rw_fail, }, [IORING_OP_POLL_ADD] = { .needs_file = 1, @@ -146,6 +150,7 @@ const struct io_op_def io_op_defs[] = { .unbound_nonreg_file = 1, .pollout = 1, .ioprio = 1, + .manual_alloc = 1, .name = "SENDMSG", #if defined(CONFIG_NET) .async_size = sizeof(struct io_async_msghdr), @@ -153,6 +158,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_sendmsg, .prep_async = io_sendmsg_prep_async, .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif @@ -163,6 +169,7 @@ const struct io_op_def io_op_defs[] = { .pollin = 1, .buffer_select = 1, .ioprio = 1, + .manual_alloc = 1, .name = "RECVMSG", #if defined(CONFIG_NET) .async_size = sizeof(struct io_async_msghdr), @@ -170,6 +177,7 @@ const struct io_op_def io_op_defs[] = { .issue = io_recvmsg, .prep_async = io_recvmsg_prep_async, .cleanup = io_sendmsg_recvmsg_cleanup, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif @@ -246,13 +254,12 @@ const struct io_op_def io_op_defs[] = { .prep = io_close_prep, .issue = io_close, }, - [IORING_OP_RSRC_UPDATE] = { + [IORING_OP_FILES_UPDATE] = { .audit_skip = 1, .iopoll = 1, - .name = "RSRC_UPDATE", - .prep = io_rsrc_update_prep, - .issue = io_rsrc_update, - .ioprio = 1, + .name = "FILES_UPDATE", + .prep = io_files_update_prep, + .issue = io_files_update, }, [IORING_OP_STATX] = { .audit_skip = 1, @@ -274,6 +281,7 @@ const struct io_op_def io_op_defs[] = { .name = "READ", .prep = io_prep_rw, .issue = io_read, + .fail = io_rw_fail, }, [IORING_OP_WRITE] = { .needs_file = 1, @@ -288,6 +296,7 @@ const struct io_op_def io_op_defs[] = { .name = "WRITE", .prep = io_prep_rw, .issue = io_write, + .fail = io_rw_fail, }, [IORING_OP_FADVISE] = { .needs_file = 1, @@ -307,10 +316,14 @@ const struct io_op_def io_op_defs[] = { .pollout = 1, .audit_skip = 1, .ioprio = 1, + .manual_alloc = 1, .name = "SEND", #if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), .prep = io_sendmsg_prep, .issue = io_send, + .fail = io_sendrecv_fail, + .prep_async = io_send_prep_async, #else .prep = io_eopnotsupp_prep, #endif @@ -326,6 +339,7 @@ const struct io_op_def io_op_defs[] = { #if defined(CONFIG_NET) .prep = io_recvmsg_prep, .issue = io_recv, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif @@ -466,25 +480,48 @@ const struct io_op_def io_op_defs[] = { .needs_file = 1, .plug = 1, .name = "URING_CMD", + .iopoll = 1, .async_size = uring_cmd_pdu_size(1), .prep = io_uring_cmd_prep, .issue = io_uring_cmd, .prep_async = io_uring_cmd_prep_async, }, - [IORING_OP_SENDZC_NOTIF] = { - .name = "SENDZC_NOTIF", + [IORING_OP_SEND_ZC] = { + .name = "SEND_ZC", .needs_file = 1, .unbound_nonreg_file = 1, .pollout = 1, .audit_skip = 1, .ioprio = 1, + .manual_alloc = 1, #if defined(CONFIG_NET) - .prep = io_sendzc_prep, - .issue = io_sendzc, + .async_size = sizeof(struct io_async_msghdr), + .prep = io_send_zc_prep, + .issue = io_send_zc, + .prep_async = io_send_prep_async, + .cleanup = io_send_zc_cleanup, + .fail = io_sendrecv_fail, +#else + .prep = io_eopnotsupp_prep, +#endif + }, + [IORING_OP_SENDMSG_ZC] = { + .name = "SENDMSG_ZC", + .needs_file = 1, + .unbound_nonreg_file = 1, + .pollout = 1, + .ioprio = 1, + .manual_alloc = 1, +#if defined(CONFIG_NET) + .async_size = sizeof(struct io_async_msghdr), + .prep = io_send_zc_prep, + .issue = io_sendmsg_zc, + .prep_async = io_sendmsg_prep_async, + .cleanup = io_send_zc_cleanup, + .fail = io_sendrecv_fail, #else .prep = io_eopnotsupp_prep, #endif - }, }; diff --git a/io_uring/opdef.h b/io_uring/opdef.h index ece8ed4f96c43422d72cb1610a9b8c6d468eb2c8..3efe06d25473ac39508f5651a24d252dd666d02e 100644 --- a/io_uring/opdef.h +++ b/io_uring/opdef.h @@ -25,6 +25,8 @@ struct io_op_def { unsigned ioprio : 1; /* supports iopoll */ unsigned iopoll : 1; + /* opcode specific path will handle ->async_data allocation if needed */ + unsigned manual_alloc : 1; /* size of async data needed, if any */ unsigned short async_size; @@ -34,6 +36,7 @@ struct io_op_def { int (*issue)(struct io_kiocb *, unsigned int); int (*prep_async)(struct io_kiocb *); void (*cleanup)(struct io_kiocb *); + void (*fail)(struct io_kiocb *); }; extern const struct io_op_def io_op_defs[]; diff --git a/io_uring/poll.c b/io_uring/poll.c index d5bad0bea6e4b190e623c24279ef017a05d75323..0d9f49c575e0f5e5e8ae4014884769f26d4e3beb 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -857,7 +857,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (sqe->buf_index || sqe->off || sqe->addr) return -EINVAL; flags = READ_ONCE(sqe->len); - if (flags & ~(IORING_POLL_ADD_MULTI|IORING_POLL_ADD_LEVEL)) + if (flags & ~IORING_POLL_ADD_MULTI) return -EINVAL; if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP)) return -EINVAL; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 71359a4d0bd4e6e90b71baf7bcd14051940337c1..012fdb04ec238ee4e6649b0e25485a9d49d32b4e 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -15,14 +15,12 @@ #include "io_uring.h" #include "openclose.h" #include "rsrc.h" -#include "notif.h" struct io_rsrc_update { struct file *file; u64 arg; u32 nr_args; u32 offset; - int type; }; static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, @@ -343,7 +341,7 @@ __cold static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, flush_delayed_work(&ctx->rsrc_put_work); reinit_completion(&data->done); - ret = io_run_task_work_sig(); + ret = io_run_task_work_sig(ctx); mutex_lock(&ctx->uring_lock); } while (ret >= 0); data->quiesce = false; @@ -655,7 +653,7 @@ __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, return -EINVAL; } -int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); @@ -669,7 +667,6 @@ int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (!up->nr_args) return -EINVAL; up->arg = READ_ONCE(sqe->addr); - up->type = READ_ONCE(sqe->ioprio); return 0; } @@ -712,7 +709,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, return ret; } -static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) +int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); struct io_ring_ctx *ctx = req->ctx; @@ -741,54 +738,6 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) return IOU_OK; } -static int io_notif_update(struct io_kiocb *req, unsigned int issue_flags) -{ - struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); - struct io_ring_ctx *ctx = req->ctx; - unsigned len = up->nr_args; - unsigned idx_end, idx = up->offset; - int ret = 0; - - io_ring_submit_lock(ctx, issue_flags); - if (unlikely(check_add_overflow(idx, len, &idx_end))) { - ret = -EOVERFLOW; - goto out; - } - if (unlikely(idx_end > ctx->nr_notif_slots)) { - ret = -EINVAL; - goto out; - } - - for (; idx < idx_end; idx++) { - struct io_notif_slot *slot = &ctx->notif_slots[idx]; - - if (!slot->notif) - continue; - if (up->arg) - slot->tag = up->arg; - io_notif_slot_flush_submit(slot, issue_flags); - } -out: - io_ring_submit_unlock(ctx, issue_flags); - if (ret < 0) - req_set_fail(req); - io_req_set_res(req, ret, 0); - return IOU_OK; -} - -int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags) -{ - struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); - - switch (up->type) { - case IORING_RSRC_UPDATE_FILES: - return io_files_update(req, issue_flags); - case IORING_RSRC_UPDATE_NOTIF: - return io_notif_update(req, issue_flags); - } - return -EINVAL; -} - int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx, struct io_rsrc_node *node, void *rsrc) { @@ -906,6 +855,7 @@ int __io_scm_file_account(struct io_ring_ctx *ctx, struct file *file) UNIXCB(skb).fp = fpl; skb->sk = sk; + skb->scm_io_uring = 1; skb->destructor = unix_destruct_scm; refcount_add(skb->truesize, &sk->sk_wmem_alloc); } diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index f3a9a177941f8ed4ef0e7bc58fb2ddc07ac28fd7..9bce15665444e99f6c97de8cfac0c4ff4d87946f 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -167,8 +167,8 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx) return &data->tags[table_idx][off]; } -int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags); -int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +int io_files_update(struct io_kiocb *req, unsigned int issue_flags); +int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int __io_account_mem(struct user_struct *user, unsigned long nr_pages); diff --git a/io_uring/rw.c b/io_uring/rw.c index 1babd77da79c7e1b97600f7c5084f9c9bc3fe5fa..100de2626e4788bd2e5ebdd6ee1c8556a6432b44 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -33,6 +33,46 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) return req->flags & REQ_F_SUPPORT_NOWAIT; } +#ifdef CONFIG_COMPAT +static int io_iov_compat_buffer_select_prep(struct io_rw *rw) +{ + struct compat_iovec __user *uiov; + compat_ssize_t clen; + + uiov = u64_to_user_ptr(rw->addr); + if (!access_ok(uiov, sizeof(*uiov))) + return -EFAULT; + if (__get_user(clen, &uiov->iov_len)) + return -EFAULT; + if (clen < 0) + return -EINVAL; + + rw->len = clen; + return 0; +} +#endif + +static int io_iov_buffer_select_prep(struct io_kiocb *req) +{ + struct iovec __user *uiov; + struct iovec iov; + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + if (rw->len != 1) + return -EINVAL; + +#ifdef CONFIG_COMPAT + if (req->ctx->compat) + return io_iov_compat_buffer_select_prep(rw); +#endif + + uiov = u64_to_user_ptr(rw->addr); + if (copy_from_user(&iov, uiov, sizeof(*uiov))) + return -EFAULT; + rw->len = iov.iov_len; + return 0; +} + int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); @@ -69,6 +109,16 @@ int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) rw->addr = READ_ONCE(sqe->addr); rw->len = READ_ONCE(sqe->len); rw->flags = READ_ONCE(sqe->rw_flags); + + /* Have to do this validation here, as this is in io_read() rw->len might + * have chanaged due to buffer selection + */ + if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { + ret = io_iov_buffer_select_prep(req); + if (ret) + return ret; + } + return 0; } @@ -184,19 +234,34 @@ static void kiocb_end_write(struct io_kiocb *req) } } -static bool __io_complete_rw_common(struct io_kiocb *req, long res) +/* + * Trigger the notifications after having done some IO, and finish the write + * accounting, if any. + */ +static void io_req_io_end(struct io_kiocb *req) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + WARN_ON(!in_task()); + if (rw->kiocb.ki_flags & IOCB_WRITE) { kiocb_end_write(req); fsnotify_modify(req->file); } else { fsnotify_access(req->file); } +} + +static bool __io_complete_rw_common(struct io_kiocb *req, long res) +{ if (unlikely(res != req->cqe.res)) { if ((res == -EAGAIN || res == -EOPNOTSUPP) && io_rw_should_reissue(req)) { + /* + * Reissue will start accounting again, finish the + * current cycle. + */ + io_req_io_end(req); req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; return true; } @@ -206,6 +271,26 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res) return false; } +static inline int io_fixup_rw_res(struct io_kiocb *req, long res) +{ + struct io_async_rw *io = req->async_data; + + /* add previously done IO, if any */ + if (req_has_async_data(req) && io->bytes_done > 0) { + if (res < 0) + res = io->bytes_done; + else + res += io->bytes_done; + } + return res; +} + +static void io_req_rw_complete(struct io_kiocb *req, bool *locked) +{ + io_req_io_end(req); + io_req_task_complete(req, locked); +} + static void io_complete_rw(struct kiocb *kiocb, long res) { struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); @@ -213,8 +298,8 @@ static void io_complete_rw(struct kiocb *kiocb, long res) if (__io_complete_rw_common(req, res)) return; - io_req_set_res(req, res, 0); - req->io_task_work.func = io_req_task_complete; + io_req_set_res(req, io_fixup_rw_res(req, res), 0); + req->io_task_work.func = io_req_rw_complete; io_req_task_work_add(req); } @@ -240,22 +325,19 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) static int kiocb_done(struct io_kiocb *req, ssize_t ret, unsigned int issue_flags) { - struct io_async_rw *io = req->async_data; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - - /* add previously done IO, if any */ - if (req_has_async_data(req) && io->bytes_done > 0) { - if (ret < 0) - ret = io->bytes_done; - else - ret += io->bytes_done; - } + unsigned final_ret = io_fixup_rw_res(req, ret); if (req->flags & REQ_F_CUR_POS) req->file->f_pos = rw->kiocb.ki_pos; if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { if (!__io_complete_rw_common(req, ret)) { - io_req_set_res(req, req->cqe.res, + /* + * Safe to call io_end from here as we're inline + * from the submission path. + */ + io_req_io_end(req); + io_req_set_res(req, final_ret, io_put_kbuf(req, issue_flags)); return IOU_OK; } @@ -268,84 +350,11 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, if (io_resubmit_prep(req)) io_req_task_queue_reissue(req); else - io_req_task_queue_fail(req, ret); + io_req_task_queue_fail(req, final_ret); } return IOU_ISSUE_SKIP_COMPLETE; } -#ifdef CONFIG_COMPAT -static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, - unsigned int issue_flags) -{ - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - struct compat_iovec __user *uiov; - compat_ssize_t clen; - void __user *buf; - size_t len; - - uiov = u64_to_user_ptr(rw->addr); - if (!access_ok(uiov, sizeof(*uiov))) - return -EFAULT; - if (__get_user(clen, &uiov->iov_len)) - return -EFAULT; - if (clen < 0) - return -EINVAL; - - len = clen; - buf = io_buffer_select(req, &len, issue_flags); - if (!buf) - return -ENOBUFS; - rw->addr = (unsigned long) buf; - iov[0].iov_base = buf; - rw->len = iov[0].iov_len = (compat_size_t) len; - return 0; -} -#endif - -static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, - unsigned int issue_flags) -{ - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - struct iovec __user *uiov = u64_to_user_ptr(rw->addr); - void __user *buf; - ssize_t len; - - if (copy_from_user(iov, uiov, sizeof(*uiov))) - return -EFAULT; - - len = iov[0].iov_len; - if (len < 0) - return -EINVAL; - buf = io_buffer_select(req, &len, issue_flags); - if (!buf) - return -ENOBUFS; - rw->addr = (unsigned long) buf; - iov[0].iov_base = buf; - rw->len = iov[0].iov_len = len; - return 0; -} - -static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, - unsigned int issue_flags) -{ - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - - if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { - iov[0].iov_base = u64_to_user_ptr(rw->addr); - iov[0].iov_len = rw->len; - return 0; - } - if (rw->len != 1) - return -EINVAL; - -#ifdef CONFIG_COMPAT - if (req->ctx->compat) - return io_compat_import(req, iov, issue_flags); -#endif - - return __io_iov_buffer_select(req, iov, issue_flags); -} - static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, struct io_rw_state *s, unsigned int issue_flags) @@ -368,7 +377,8 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, buf = u64_to_user_ptr(rw->addr); sqe_len = rw->len; - if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE) { + if (opcode == IORING_OP_READ || opcode == IORING_OP_WRITE || + (req->flags & REQ_F_BUFFER_SELECT)) { if (io_do_buffer_select(req)) { buf = io_buffer_select(req, &sqe_len, issue_flags); if (!buf) @@ -384,14 +394,6 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, } iovec = s->fast_iov; - if (req->flags & REQ_F_BUFFER_SELECT) { - ret = io_iov_buffer_select(req, iovec, issue_flags); - if (ret) - return ERR_PTR(ret); - iov_iter_init(iter, ddir, iovec, 1, iovec->iov_len); - return NULL; - } - ret = __import_iovec(ddir, buf, sqe_len, UIO_FASTIOV, &iovec, iter, req->ctx->compat); if (unlikely(ret < 0)) @@ -788,10 +790,12 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) iov_iter_restore(&s->iter, &s->iter_state); ret2 = io_setup_async_rw(req, iovec, s, true); - if (ret2) - return ret2; - iovec = NULL; + if (ret2) { + ret = ret > 0 ? ret : ret2; + goto done; + } + io = req->async_data; s = &io->s; /* @@ -817,6 +821,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) return -EAGAIN; } + req->cqe.res = iov_iter_count(&s->iter); /* * Now retry read with the IOCB_WAITQ parts set in the iocb. If * we get -EIOCBQUEUED, then we'll get a notification when the @@ -931,7 +936,7 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) goto copy_iov; if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { - struct io_async_rw *rw; + struct io_async_rw *io; trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, req->cqe.res, ret2); @@ -944,9 +949,9 @@ int io_write(struct io_kiocb *req, unsigned int issue_flags) iov_iter_save_state(&s->iter, &s->iter_state); ret = io_setup_async_rw(req, iovec, s, true); - rw = req->async_data; - if (rw) - rw->bytes_done += ret2; + io = req->async_data; + if (io) + io->bytes_done += ret2; if (kiocb->ki_flags & IOCB_WRITE) kiocb_end_write(req); @@ -978,6 +983,14 @@ static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx) io_cqring_wake(ctx); } +void io_rw_fail(struct io_kiocb *req) +{ + int res; + + res = io_fixup_rw_res(req, req->cqe.res); + io_req_set_res(req, res, req->cqe.flags); +} + int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) { struct io_wq_work_node *pos, *start, *prev; @@ -994,7 +1007,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) wq_list_for_each(pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + struct file *file = req->file; int ret; /* @@ -1005,7 +1018,17 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) if (READ_ONCE(req->iopoll_completed)) break; - ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob, poll_flags); + if (req->opcode == IORING_OP_URING_CMD) { + struct io_uring_cmd *ioucmd; + + ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); + ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, + poll_flags); + } else { + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); + + ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); + } if (unlikely(ret < 0)) return ret; else if (ret) diff --git a/io_uring/rw.h b/io_uring/rw.h index 0204c3fcafa517d267fb28228fc65048e2f8556e..3b733f4b610ac98baecb635bf3630bcd01ba5205 100644 --- a/io_uring/rw.h +++ b/io_uring/rw.h @@ -21,3 +21,4 @@ int io_readv_prep_async(struct io_kiocb *req); int io_write(struct io_kiocb *req, unsigned int issue_flags); int io_writev_prep_async(struct io_kiocb *req); void io_readv_writev_cleanup(struct io_kiocb *req); +void io_rw_fail(struct io_kiocb *req); diff --git a/io_uring/tctx.c b/io_uring/tctx.c index 7f97d97fef0a9652dd2804f0d6c9c3a9263b97e8..4324b1cf1f6afaf5e6afc740f8e1ee03e2afa719 100644 --- a/io_uring/tctx.c +++ b/io_uring/tctx.c @@ -91,32 +91,12 @@ __cold int io_uring_alloc_task_context(struct task_struct *task, return 0; } -static int io_register_submitter(struct io_ring_ctx *ctx) -{ - int ret = 0; - - mutex_lock(&ctx->uring_lock); - if (!ctx->submitter_task) - ctx->submitter_task = get_task_struct(current); - else if (ctx->submitter_task != current) - ret = -EEXIST; - mutex_unlock(&ctx->uring_lock); - - return ret; -} - -int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter) +int __io_uring_add_tctx_node(struct io_ring_ctx *ctx) { struct io_uring_task *tctx = current->io_uring; struct io_tctx_node *node; int ret; - if ((ctx->flags & IORING_SETUP_SINGLE_ISSUER) && submitter) { - ret = io_register_submitter(ctx); - if (ret) - return ret; - } - if (unlikely(!tctx)) { ret = io_uring_alloc_task_context(current, ctx); if (unlikely(ret)) @@ -150,8 +130,22 @@ int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter) list_add(&node->ctx_node, &ctx->tctx_list); mutex_unlock(&ctx->uring_lock); } - if (submitter) - tctx->last = ctx; + return 0; +} + +int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx) +{ + int ret; + + if (ctx->flags & IORING_SETUP_SINGLE_ISSUER + && ctx->submitter_task != current) + return -EEXIST; + + ret = __io_uring_add_tctx_node(ctx); + if (ret) + return ret; + + current->io_uring->last = ctx; return 0; } @@ -259,7 +253,7 @@ int io_ringfd_register(struct io_ring_ctx *ctx, void __user *__arg, return -EINVAL; mutex_unlock(&ctx->uring_lock); - ret = __io_uring_add_tctx_node(ctx, false); + ret = __io_uring_add_tctx_node(ctx); mutex_lock(&ctx->uring_lock); if (ret) return ret; diff --git a/io_uring/tctx.h b/io_uring/tctx.h index 25974beed4d6be1e14d8dbd674e74bae48e849dc..608e96de70a2c6f8fec3e644733daf4a09742e21 100644 --- a/io_uring/tctx.h +++ b/io_uring/tctx.h @@ -9,7 +9,8 @@ struct io_tctx_node { int io_uring_alloc_task_context(struct task_struct *task, struct io_ring_ctx *ctx); void io_uring_del_tctx_node(unsigned long index); -int __io_uring_add_tctx_node(struct io_ring_ctx *ctx, bool submitter); +int __io_uring_add_tctx_node(struct io_ring_ctx *ctx); +int __io_uring_add_tctx_node_from_submit(struct io_ring_ctx *ctx); void io_uring_clean_tctx(struct io_uring_task *tctx); void io_uring_unreg_ringfd(void); @@ -27,5 +28,6 @@ static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx) if (likely(tctx && tctx->last == ctx)) return 0; - return __io_uring_add_tctx_node(ctx, true); + + return __io_uring_add_tctx_node_from_submit(ctx); } diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 78ea2c64b70e0e11ab95d4cc57073db3c475bd9b..e8a8c20994805ced29c10543b1aa637c8e32c389 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -149,11 +149,10 @@ static inline void io_remove_next_linked(struct io_kiocb *req) nxt->link = NULL; } -bool io_disarm_next(struct io_kiocb *req) +void io_disarm_next(struct io_kiocb *req) __must_hold(&req->ctx->completion_lock) { struct io_kiocb *link = NULL; - bool posted = false; if (req->flags & REQ_F_ARM_LTIMEOUT) { link = req->link; @@ -161,7 +160,6 @@ bool io_disarm_next(struct io_kiocb *req) if (link && link->opcode == IORING_OP_LINK_TIMEOUT) { io_remove_next_linked(req); io_req_tw_post_queue(link, -ECANCELED, 0); - posted = true; } } else if (req->flags & REQ_F_LINK_TIMEOUT) { struct io_ring_ctx *ctx = req->ctx; @@ -169,17 +167,12 @@ bool io_disarm_next(struct io_kiocb *req) spin_lock_irq(&ctx->timeout_lock); link = io_disarm_linked_timeout(req); spin_unlock_irq(&ctx->timeout_lock); - if (link) { - posted = true; + if (link) io_req_tw_post_queue(link, -ECANCELED, 0); - } } if (unlikely((req->flags & REQ_F_FAIL) && - !(req->flags & REQ_F_HARDLINK))) { - posted |= (req->link != NULL); + !(req->flags & REQ_F_HARDLINK))) io_fail_links(req); - } - return posted; } struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, diff --git a/io_uring/timeout.h b/io_uring/timeout.h index 858c62644897a55236e834605d77156acee4c042..a6939f18313e86e66574232c3b2d041a161add60 100644 --- a/io_uring/timeout.h +++ b/io_uring/timeout.h @@ -27,7 +27,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd); __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk, bool cancel_all); void io_queue_linked_timeout(struct io_kiocb *req); -bool io_disarm_next(struct io_kiocb *req); +void io_disarm_next(struct io_kiocb *req); int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 8e0cc2d9205eaeeec162510d805932f176ff1bbc..e50de0b6b9f843f094a3e54923c953d85ee24fc7 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -3,10 +3,13 @@ #include #include #include +#include +#include #include #include "io_uring.h" +#include "rsrc.h" #include "uring_cmd.h" static void io_uring_cmd_work(struct io_kiocb *req, bool *locked) @@ -49,7 +52,11 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) io_req_set_res(req, ret, 0); if (req->ctx->flags & IORING_SETUP_CQE32) io_req_set_cqe32_extra(req, res2, 0); - __io_req_complete(req, 0); + if (req->ctx->flags & IORING_SETUP_IOPOLL) + /* order with io_iopoll_req_issued() checking ->iopoll_complete */ + smp_store_release(&req->iopoll_completed, 1); + else + __io_req_complete(req, 0); } EXPORT_SYMBOL_GPL(io_uring_cmd_done); @@ -71,8 +78,24 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); - if (sqe->rw_flags || sqe->__pad1) + if (sqe->__pad1) return -EINVAL; + + ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags); + if (ioucmd->flags & ~IORING_URING_CMD_FIXED) + return -EINVAL; + + if (ioucmd->flags & IORING_URING_CMD_FIXED) { + struct io_ring_ctx *ctx = req->ctx; + u16 index; + + req->buf_index = READ_ONCE(sqe->buf_index); + if (unlikely(req->buf_index >= ctx->nr_user_bufs)) + return -EFAULT; + index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); + req->imu = ctx->user_bufs[index]; + io_req_set_rsrc_node(req, ctx, 0); + } ioucmd->cmd = sqe->cmd; ioucmd->cmd_op = READ_ONCE(sqe->cmd_op); return 0; @@ -88,12 +111,19 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) if (!req->file->f_op->uring_cmd) return -EOPNOTSUPP; + ret = security_uring_cmd(ioucmd); + if (ret) + return ret; + if (ctx->flags & IORING_SETUP_SQE128) issue_flags |= IO_URING_F_SQE128; if (ctx->flags & IORING_SETUP_CQE32) issue_flags |= IO_URING_F_CQE32; - if (ctx->flags & IORING_SETUP_IOPOLL) + if (ctx->flags & IORING_SETUP_IOPOLL) { issue_flags |= IO_URING_F_IOPOLL; + req->iopoll_completed = 0; + WRITE_ONCE(ioucmd->cookie, NULL); + } if (req_has_async_data(req)) ioucmd->cmd = req->async_data; @@ -112,8 +142,17 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); - return IOU_OK; + return ret; } return IOU_ISSUE_SKIP_COMPLETE; } + +int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, + struct iov_iter *iter, void *ioucmd) +{ + struct io_kiocb *req = cmd_to_io_kiocb(ioucmd); + + return io_import_fixed(rw, iter, req->imu, ubuf, len); +} +EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed); diff --git a/io_uring/xattr.c b/io_uring/xattr.c index 84180afd090b743bcdb23ccd88450ea9aea226a6..99df641594d7455a704a8dea24d8bdbecdcda820 100644 --- a/io_uring/xattr.c +++ b/io_uring/xattr.c @@ -206,7 +206,7 @@ int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) } static int __io_setxattr(struct io_kiocb *req, unsigned int issue_flags, - struct path *path) + const struct path *path) { struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; diff --git a/ipc/mqueue.c b/ipc/mqueue.c index f98de32aeea174c440e084362bcca1c3edfe5e3c..467a194b8a2ec6464f475fca5c5ff569002ed018 100644 --- a/ipc/mqueue.c +++ b/ipc/mqueue.c @@ -986,8 +986,7 @@ SYSCALL_DEFINE1(mq_unlink, const char __user *, u_name) out_unlock: inode_unlock(d_inode(mnt->mnt_root)); - if (inode) - iput(inode); + iput(inode); mnt_drop_write(mnt); out_name: putname(name); @@ -1746,6 +1745,7 @@ out_filesystem: unregister_filesystem(&mqueue_fs_type); out_sysctl: kmem_cache_destroy(mqueue_inode_cachep); + retire_mq_sysctls(&init_ipc_ns); return error; } diff --git a/ipc/msg.c b/ipc/msg.c index a0d05775af2c5a1921282fed41732f571499e0f4..e4e0990e08f754d5f667722753c6d26f9cdfd7e2 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -285,10 +286,10 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) rcu_read_unlock(); list_for_each_entry_safe(msg, t, &msq->q_messages, m_list) { - atomic_dec(&ns->msg_hdrs); + percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1); free_msg(msg); } - atomic_sub(msq->q_cbytes, &ns->msg_bytes); + percpu_counter_sub_local(&ns->percpu_msg_bytes, msq->q_cbytes); ipc_update_pid(&msq->q_lspid, NULL); ipc_update_pid(&msq->q_lrpid, NULL); ipc_rcu_putref(&msq->q_perm, msg_rcu_free); @@ -495,17 +496,22 @@ static int msgctl_info(struct ipc_namespace *ns, int msqid, msginfo->msgssz = MSGSSZ; msginfo->msgseg = MSGSEG; down_read(&msg_ids(ns).rwsem); - if (cmd == MSG_INFO) { + if (cmd == MSG_INFO) msginfo->msgpool = msg_ids(ns).in_use; - msginfo->msgmap = atomic_read(&ns->msg_hdrs); - msginfo->msgtql = atomic_read(&ns->msg_bytes); + max_idx = ipc_get_maxidx(&msg_ids(ns)); + up_read(&msg_ids(ns).rwsem); + if (cmd == MSG_INFO) { + msginfo->msgmap = min_t(int, + percpu_counter_sum(&ns->percpu_msg_hdrs), + INT_MAX); + msginfo->msgtql = min_t(int, + percpu_counter_sum(&ns->percpu_msg_bytes), + INT_MAX); } else { msginfo->msgmap = MSGMAP; msginfo->msgpool = MSGPOOL; msginfo->msgtql = MSGTQL; } - max_idx = ipc_get_maxidx(&msg_ids(ns)); - up_read(&msg_ids(ns).rwsem); return (max_idx < 0) ? 0 : max_idx; } @@ -935,8 +941,8 @@ static long do_msgsnd(int msqid, long mtype, void __user *mtext, list_add_tail(&msg->m_list, &msq->q_messages); msq->q_cbytes += msgsz; msq->q_qnum++; - atomic_add(msgsz, &ns->msg_bytes); - atomic_inc(&ns->msg_hdrs); + percpu_counter_add_local(&ns->percpu_msg_bytes, msgsz); + percpu_counter_add_local(&ns->percpu_msg_hdrs, 1); } err = 0; @@ -1159,8 +1165,8 @@ static long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, in msq->q_rtime = ktime_get_real_seconds(); ipc_update_pid(&msq->q_lrpid, task_tgid(current)); msq->q_cbytes -= msg->m_ts; - atomic_sub(msg->m_ts, &ns->msg_bytes); - atomic_dec(&ns->msg_hdrs); + percpu_counter_sub_local(&ns->percpu_msg_bytes, msg->m_ts); + percpu_counter_sub_local(&ns->percpu_msg_hdrs, 1); ss_wakeup(msq, &wake_q, false); goto out_unlock0; @@ -1297,20 +1303,34 @@ COMPAT_SYSCALL_DEFINE5(msgrcv, int, msqid, compat_uptr_t, msgp, } #endif -void msg_init_ns(struct ipc_namespace *ns) +int msg_init_ns(struct ipc_namespace *ns) { + int ret; + ns->msg_ctlmax = MSGMAX; ns->msg_ctlmnb = MSGMNB; ns->msg_ctlmni = MSGMNI; - atomic_set(&ns->msg_bytes, 0); - atomic_set(&ns->msg_hdrs, 0); + ret = percpu_counter_init(&ns->percpu_msg_bytes, 0, GFP_KERNEL); + if (ret) + goto fail_msg_bytes; + ret = percpu_counter_init(&ns->percpu_msg_hdrs, 0, GFP_KERNEL); + if (ret) + goto fail_msg_hdrs; ipc_init_ids(&ns->ids[IPC_MSG_IDS]); + return 0; + +fail_msg_hdrs: + percpu_counter_destroy(&ns->percpu_msg_bytes); +fail_msg_bytes: + return ret; } #ifdef CONFIG_IPC_NS void msg_exit_ns(struct ipc_namespace *ns) { + percpu_counter_destroy(&ns->percpu_msg_bytes); + percpu_counter_destroy(&ns->percpu_msg_hdrs); free_ipcs(ns, &msg_ids(ns), freeque); idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr); rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht); diff --git a/ipc/namespace.c b/ipc/namespace.c index e1fcaedba4fae0768b5b5379b627d431232e0c72..8316ea5857333d756c4a6741853d92820a0e5d65 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -66,8 +66,11 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns, if (!setup_ipc_sysctls(ns)) goto fail_mq; + err = msg_init_ns(ns); + if (err) + goto fail_put; + sem_init_ns(ns); - msg_init_ns(ns); shm_init_ns(ns); return ns; diff --git a/ipc/shm.c b/ipc/shm.c index b3048ebd5c315c3768e376a87019c85bd8b86c0d..7d86f058fb861b8331faa0edd20fa890424561e7 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -1721,7 +1721,7 @@ long ksys_shmdt(char __user *shmaddr) #ifdef CONFIG_MMU loff_t size = 0; struct file *file; - struct vm_area_struct *next; + VMA_ITERATOR(vmi, mm, addr); #endif if (addr & ~PAGE_MASK) @@ -1751,12 +1751,9 @@ long ksys_shmdt(char __user *shmaddr) * match the usual checks anyway. So assume all vma's are * above the starting address given. */ - vma = find_vma(mm, addr); #ifdef CONFIG_MMU - while (vma) { - next = vma->vm_next; - + for_each_vma(vmi, vma) { /* * Check if the starting address would match, i.e. it's * a fragment created by mprotect() and/or munmap(), or it @@ -1774,6 +1771,7 @@ long ksys_shmdt(char __user *shmaddr) file = vma->vm_file; size = i_size_read(file_inode(vma->vm_file)); do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); + mas_pause(&vmi.mas); /* * We discovered the size of the shm segment, so * break out of here and fall through to the next @@ -1781,10 +1779,9 @@ long ksys_shmdt(char __user *shmaddr) * searching for matching vma's. */ retval = 0; - vma = next; + vma = vma_next(&vmi); break; } - vma = next; } /* @@ -1794,17 +1791,19 @@ long ksys_shmdt(char __user *shmaddr) */ size = PAGE_ALIGN(size); while (vma && (loff_t)(vma->vm_end - addr) <= size) { - next = vma->vm_next; - /* finding a matching vma now does not alter retval */ if ((vma->vm_ops == &shm_vm_ops) && ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) + (vma->vm_file == file)) { do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL); - vma = next; + mas_pause(&vmi.mas); + } + + vma = vma_next(&vmi); } #else /* CONFIG_MMU */ + vma = vma_lookup(mm, addr); /* under NOMMU conditions, the exact address to be destroyed must be * given */ diff --git a/ipc/util.c b/ipc/util.c index a2208d0f26b2df52605ad610758b7b8b6f64a46f..05cb9de66735086b5e09e3705b92e480a99d849e 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -782,28 +782,37 @@ struct pid_namespace *ipc_seq_pid_ns(struct seq_file *s) return iter->pid_ns; } -/* - * This routine locks the ipc structure found at least at position pos. +/** + * sysvipc_find_ipc - Find and lock the ipc structure based on seq pos + * @ids: ipc identifier set + * @pos: expected position + * + * The function finds an ipc structure, based on the sequence file + * position @pos. If there is no ipc structure at position @pos, then + * the successor is selected. + * If a structure is found, then it is locked (both rcu_read_lock() and + * ipc_lock_object()) and @pos is set to the position needed to locate + * the found ipc structure. + * If nothing is found (i.e. EOF), @pos is not modified. + * + * The function returns the found ipc structure, or NULL at EOF. */ -static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t pos, - loff_t *new_pos) +static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t *pos) { - struct kern_ipc_perm *ipc = NULL; - int max_idx = ipc_get_maxidx(ids); + int tmpidx; + struct kern_ipc_perm *ipc; - if (max_idx == -1 || pos > max_idx) - goto out; + /* convert from position to idr index -> "-1" */ + tmpidx = *pos - 1; - for (; pos <= max_idx; pos++) { - ipc = idr_find(&ids->ipcs_idr, pos); - if (ipc != NULL) { - rcu_read_lock(); - ipc_lock_object(ipc); - break; - } + ipc = idr_get_next(&ids->ipcs_idr, &tmpidx); + if (ipc != NULL) { + rcu_read_lock(); + ipc_lock_object(ipc); + + /* convert from idr index to position -> "+1" */ + *pos = tmpidx + 1; } -out: - *new_pos = pos + 1; return ipc; } @@ -817,11 +826,13 @@ static void *sysvipc_proc_next(struct seq_file *s, void *it, loff_t *pos) if (ipc && ipc != SEQ_START_TOKEN) ipc_unlock(ipc); - return sysvipc_find_ipc(&iter->ns->ids[iface->ids], *pos, pos); + /* Next -> search for *pos+1 */ + (*pos)++; + return sysvipc_find_ipc(&iter->ns->ids[iface->ids], pos); } /* - * File positions: pos 0 -> header, pos n -> ipc id = n - 1. + * File positions: pos 0 -> header, pos n -> ipc idx = n - 1. * SeqFile iterator: iterator value locked ipc pointer or SEQ_TOKEN_START. */ static void *sysvipc_proc_start(struct seq_file *s, loff_t *pos) @@ -846,8 +857,8 @@ static void *sysvipc_proc_start(struct seq_file *s, loff_t *pos) if (*pos == 0) return SEQ_START_TOKEN; - /* Find the (pos-1)th ipc */ - return sysvipc_find_ipc(ids, *pos - 1, pos); + /* Otherwise return the correct ipc structure */ + return sysvipc_find_ipc(ids, pos); } static void sysvipc_proc_stop(struct seq_file *s, void *it) diff --git a/ipc/util.h b/ipc/util.h index 2dd7ce0416d8edfe5398333281abea0cdd556068..b2906e3665394210fc27d282d0027324fd38101b 100644 --- a/ipc/util.h +++ b/ipc/util.h @@ -64,7 +64,7 @@ static inline void mq_put_mnt(struct ipc_namespace *ns) { } #ifdef CONFIG_SYSVIPC void sem_init_ns(struct ipc_namespace *ns); -void msg_init_ns(struct ipc_namespace *ns); +int msg_init_ns(struct ipc_namespace *ns); void shm_init_ns(struct ipc_namespace *ns); void sem_exit_ns(struct ipc_namespace *ns); @@ -72,7 +72,7 @@ void msg_exit_ns(struct ipc_namespace *ns); void shm_exit_ns(struct ipc_namespace *ns); #else static inline void sem_init_ns(struct ipc_namespace *ns) { } -static inline void msg_init_ns(struct ipc_namespace *ns) { } +static inline int msg_init_ns(struct ipc_namespace *ns) { return 0; } static inline void shm_init_ns(struct ipc_namespace *ns) { } static inline void sem_exit_ns(struct ipc_namespace *ns) { } diff --git a/kernel/Makefile b/kernel/Makefile index 318789c728d3290d0053f9f1a6bbe5235c969399..d754e0be1176df3677c12fe2a9170acfef3a764a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -38,6 +38,7 @@ KCOV_INSTRUMENT_kcov.o := n KASAN_SANITIZE_kcov.o := n KCSAN_SANITIZE_kcov.o := n UBSAN_SANITIZE_kcov.o := n +KMSAN_SANITIZE_kcov.o := n CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector # Don't instrument error handlers diff --git a/kernel/acct.c b/kernel/acct.c index 13706356ec54d4386eb37d563e3a13d769da63e0..62200d799b9b0066a770ffe308a5eae2250edfd7 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -555,15 +555,14 @@ void acct_collect(long exitcode, int group_dead) unsigned long vsize = 0; if (group_dead && current->mm) { + struct mm_struct *mm = current->mm; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - mmap_read_lock(current->mm); - vma = current->mm->mmap; - while (vma) { + mmap_read_lock(mm); + for_each_vma(vmi, vma) vsize += vma->vm_end - vma->vm_start; - vma = vma->vm_next; - } - mmap_read_unlock(current->mm); + mmap_read_unlock(mm); } spin_lock_irq(¤t->sighand->siglock); diff --git a/kernel/audit.c b/kernel/audit.c index a75978ae38ad7238a6565d3c902009d7bd802c98..9bc0b0301198c170ef794a02c95cfb5c5f576f73 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -321,7 +321,6 @@ static inline int audit_rate_check(void) static DEFINE_SPINLOCK(lock); unsigned long flags; unsigned long now; - unsigned long elapsed; int retval = 0; if (!audit_rate_limit) return 1; @@ -330,9 +329,8 @@ static inline int audit_rate_check(void) if (++messages < audit_rate_limit) { retval = 1; } else { - now = jiffies; - elapsed = now - last_check; - if (elapsed > HZ) { + now = jiffies; + if (time_after(now, last_check + HZ)) { last_check = now; messages = 0; retval = 1; @@ -366,7 +364,7 @@ void audit_log_lost(const char *message) if (!print) { spin_lock_irqsave(&lock, flags); now = jiffies; - if (now - last_msg > HZ) { + if (time_after(now, last_msg + HZ)) { print = 1; last_msg = now; } diff --git a/kernel/audit.h b/kernel/audit.h index 58b66543b4d57e6542f97900cc7809f4aea7de61..c57b008b9914e5616ec47a9af0e7794ff344978a 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -133,7 +133,7 @@ struct audit_context { struct sockaddr_storage *sockaddr; size_t sockaddr_len; /* Save things to print about task_struct */ - pid_t pid, ppid; + pid_t ppid; kuid_t uid, euid, suid, fsuid; kgid_t gid, egid, sgid, fsgid; unsigned long personality; @@ -245,8 +245,6 @@ struct audit_netlink_list { int audit_send_list_thread(void *_dest); -extern int selinux_audit_rule_update(void); - extern struct mutex audit_filter_mutex; extern int audit_del_rule(struct audit_entry *entry); extern void audit_free_rule_rcu(struct rcu_head *head); diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c index 6432a37ac1c94d1757265f6ccdefe0870cdb6c43..c565fbf66ac8767285b3de2fbd44928aaab59b2b 100644 --- a/kernel/audit_fsnotify.c +++ b/kernel/audit_fsnotify.c @@ -102,6 +102,7 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa ret = fsnotify_add_inode_mark(&audit_mark->mark, inode, 0); if (ret < 0) { + audit_mark->path = NULL; fsnotify_put_mark(&audit_mark->mark); audit_mark = ERR_PTR(ret); } diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index 4b0957aa2cd4e62544164d51ad11943fd4ab330d..65075f1e4ac8c8751b639bff2d7b66f79c6a7dec 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -133,7 +133,7 @@ int audit_watch_compare(struct audit_watch *watch, unsigned long ino, dev_t dev) } /* Initialize a parent watch entry. */ -static struct audit_parent *audit_init_parent(struct path *path) +static struct audit_parent *audit_init_parent(const struct path *path) { struct inode *inode = d_backing_inode(path->dentry); struct audit_parent *parent; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index dd8d9ab747c3eee398c21b17a0513db39075cd87..9f8c05228d6d653b89ddcb9c00965f604588a215 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -965,7 +965,7 @@ static void audit_reset_context(struct audit_context *ctx) if (!ctx) return; - /* if ctx is non-null, reset the "ctx->state" regardless */ + /* if ctx is non-null, reset the "ctx->context" regardless */ ctx->context = AUDIT_CTX_UNUSED; if (ctx->dummy) return; @@ -1002,7 +1002,7 @@ static void audit_reset_context(struct audit_context *ctx) kfree(ctx->sockaddr); ctx->sockaddr = NULL; ctx->sockaddr_len = 0; - ctx->pid = ctx->ppid = 0; + ctx->ppid = 0; ctx->uid = ctx->euid = ctx->suid = ctx->fsuid = KUIDT_INIT(0); ctx->gid = ctx->egid = ctx->sgid = ctx->fsgid = KGIDT_INIT(0); ctx->personality = 0; @@ -1016,7 +1016,6 @@ static void audit_reset_context(struct audit_context *ctx) WARN_ON(!list_empty(&ctx->killed_trees)); audit_free_module(ctx); ctx->fds[0] = -1; - audit_proctitle_free(ctx); ctx->type = 0; /* reset last for audit_free_*() */ } @@ -1077,6 +1076,7 @@ static inline void audit_free_context(struct audit_context *context) { /* resetting is extra work, but it is likely just noise */ audit_reset_context(context); + audit_proctitle_free(context); free_tree_refs(context); kfree(context->filterkey); kfree(context); @@ -1833,7 +1833,7 @@ void __audit_free(struct task_struct *tsk) /* We are called either by do_exit() or the fork() error handling code; * in the former case tsk == current and in the latter tsk is a - * random task_struct that doesn't doesn't have any meaningful data we + * random task_struct that doesn't have any meaningful data we * need to log via audit_log_exit(). */ if (tsk == current && !context->dummy) { @@ -1940,6 +1940,7 @@ void __audit_uring_exit(int success, long code) goto out; } + audit_return_fixup(ctx, success, code); if (ctx->context == AUDIT_CTX_SYSCALL) { /* * NOTE: See the note in __audit_uring_entry() about the case @@ -1981,7 +1982,6 @@ void __audit_uring_exit(int success, long code) audit_filter_inodes(current, ctx); if (ctx->current_state != AUDIT_STATE_RECORD) goto out; - audit_return_fixup(ctx, success, code); audit_log_exit(); out: @@ -2065,13 +2065,13 @@ void __audit_syscall_exit(int success, long return_code) if (!list_empty(&context->killed_trees)) audit_kill_trees(context); + audit_return_fixup(context, success, return_code); /* run through both filters to ensure we set the filterkey properly */ audit_filter_syscall(current, context); audit_filter_inodes(current, context); - if (context->current_state < AUDIT_STATE_RECORD) + if (context->current_state != AUDIT_STATE_RECORD) goto out; - audit_return_fixup(context, success, return_code); audit_log_exit(); out: diff --git a/kernel/bounds.c b/kernel/bounds.c index 9795d75b09b2323306ad6a058a6350a87a251443..b529182e8b04fc5e53f106b5b7d0fdeea6c877aa 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -22,6 +22,13 @@ int main(void) DEFINE(NR_CPUS_BITS, ilog2(CONFIG_NR_CPUS)); #endif DEFINE(SPINLOCK_SIZE, sizeof(spinlock_t)); +#ifdef CONFIG_LRU_GEN + DEFINE(LRU_GEN_WIDTH, order_base_2(MAX_NR_GENS + 1)); + DEFINE(__LRU_REFS_WIDTH, MAX_NR_TIERS - 2); +#else + DEFINE(LRU_GEN_WIDTH, 0); + DEFINE(__LRU_REFS_WIDTH, 0); +#endif /* End of constants */ return 0; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 057ba8e01e70face05b27e845e4c7b09b3e82ed5..341c94f208f4cbdd7ecca1169ac8f05656368aad 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -13,7 +13,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o obj-$(CONFIG_BPF_SYSCALL) += disasm.o obj-$(CONFIG_BPF_JIT) += trampoline.o -obj-$(CONFIG_BPF_SYSCALL) += btf.o +obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o obj-$(CONFIG_BPF_JIT) += dispatcher.o ifeq ($(CONFIG_NET),y) obj-$(CONFIG_BPF_SYSCALL) += devmap.o @@ -24,6 +24,9 @@ endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif +ifeq ($(CONFIG_CGROUPS),y) +obj-$(CONFIG_BPF_SYSCALL) += cgroup_iter.o +endif obj-$(CONFIG_CGROUP_BPF) += cgroup.o ifeq ($(CONFIG_INET),y) obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 624527401d4d8f51036e19daf3c9a276864a1430..832b2659e96e2761d57f533a595aedf94d1b2112 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -279,7 +279,8 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value) rcu_read_lock(); pptr = array->pptrs[index & array->index_mask]; for_each_possible_cpu(cpu) { - bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size); + copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); + check_and_init_map_value(map, value + off); off += size; } rcu_read_unlock(); @@ -338,8 +339,9 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value, return -EINVAL; if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { - memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]), - value, map->value_size); + val = this_cpu_ptr(array->pptrs[index & array->index_mask]); + copy_map_value(map, val, value); + check_and_free_fields(array, val); } else { val = array->value + (u64)array->elem_size * (index & array->index_mask); @@ -383,7 +385,8 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, rcu_read_lock(); pptr = array->pptrs[index & array->index_mask]; for_each_possible_cpu(cpu) { - bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size); + copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off); + check_and_free_fields(array, per_cpu_ptr(pptr, cpu)); off += size; } rcu_read_unlock(); @@ -421,8 +424,20 @@ static void array_map_free(struct bpf_map *map) int i; if (map_value_has_kptrs(map)) { - for (i = 0; i < array->map.max_entries; i++) - bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); + if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { + for (i = 0; i < array->map.max_entries; i++) { + void __percpu *pptr = array->pptrs[i & array->index_mask]; + int cpu; + + for_each_possible_cpu(cpu) { + bpf_map_free_kptrs(map, per_cpu_ptr(pptr, cpu)); + cond_resched(); + } + } + } else { + for (i = 0; i < array->map.max_entries; i++) + bpf_map_free_kptrs(map, array_map_elem_ptr(array, i)); + } bpf_map_free_kptr_off_tab(map); } @@ -608,9 +623,9 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v) pptr = v; size = array->elem_size; for_each_possible_cpu(cpu) { - bpf_long_memcpy(info->percpu_value_buf + off, - per_cpu_ptr(pptr, cpu), - size); + copy_map_value_long(map, info->percpu_value_buf + off, + per_cpu_ptr(pptr, cpu)); + check_and_init_map_value(map, info->percpu_value_buf + off); off += size; } ctx.value = info->percpu_value_buf; diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index b9ea539a556141034397bed047f7a3c9fdfeef0d..48ee750849f25772ebe7f5cc325be53c509b33b0 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -158,7 +158,7 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) attr->value_size / sizeof(u32); if (!(attr->map_flags & BPF_F_ZERO_SEED)) - bloom->hash_seed = get_random_int(); + bloom->hash_seed = get_random_u32(); return &bloom->map; } diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 24b755eca0b393320db530bf98ba9158cba5e3b3..5dc307bdeaebc7e9789f7162a1d2243f57747e69 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -202,6 +202,11 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, } stop: offs = seq->count; + if (IS_ERR(p)) { + seq->op->stop(seq, NULL); + err = PTR_ERR(p); + goto done; + } /* bpf program called if !p */ seq->op->stop(seq, p); if (!p) { @@ -689,19 +694,24 @@ struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) { + struct bpf_run_ctx run_ctx, *old_run_ctx; int ret; if (prog->aux->sleepable) { rcu_read_lock_trace(); migrate_disable(); might_fault(); + old_run_ctx = bpf_set_run_ctx(&run_ctx); ret = bpf_prog_run(prog, ctx); + bpf_reset_run_ctx(old_run_ctx); migrate_enable(); rcu_read_unlock_trace(); } else { rcu_read_lock(); migrate_disable(); + old_run_ctx = bpf_set_run_ctx(&run_ctx); ret = bpf_prog_run(prog, ctx); + bpf_reset_run_ctx(old_run_ctx); migrate_enable(); rcu_read_unlock(); } diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 8ce40fd869f6a7c7099c4bc9892fea7b4b382f3b..802fc15b0d73cd9ecfe5bf66f7f4d5a8576006f7 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -555,11 +555,11 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, struct bpf_local_storage_elem, map_node))) { if (busy_counter) { migrate_disable(); - __this_cpu_inc(*busy_counter); + this_cpu_inc(*busy_counter); } bpf_selem_unlink(selem, false); if (busy_counter) { - __this_cpu_dec(*busy_counter); + this_cpu_dec(*busy_counter); migrate_enable(); } cond_resched_rcu(); @@ -582,7 +582,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, synchronize_rcu(); kvfree(smap->buckets); - kfree(smap); + bpf_map_area_free(smap); } int bpf_local_storage_map_alloc_check(union bpf_attr *attr) @@ -610,7 +610,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr) unsigned int i; u32 nbuckets; - smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT); + smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE); if (!smap) return ERR_PTR(-ENOMEM); bpf_map_init_from_attr(&smap->map, attr); @@ -623,7 +623,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr) smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets, GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT); if (!smap->buckets) { - kfree(smap); + bpf_map_area_free(smap); return ERR_PTR(-ENOMEM); } diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index fa71d58b7dedad2352b19f3b420140d0e0b2cb09..d6c9b3705f2423f4ded1b649210dd4fbf68d155a 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -41,17 +41,21 @@ BTF_SET_END(bpf_lsm_hooks) */ BTF_SET_START(bpf_lsm_current_hooks) /* operate on freshly allocated sk without any cgroup association */ +#ifdef CONFIG_SECURITY_NETWORK BTF_ID(func, bpf_lsm_sk_alloc_security) BTF_ID(func, bpf_lsm_sk_free_security) +#endif BTF_SET_END(bpf_lsm_current_hooks) /* List of LSM hooks that trigger while the socket is properly locked. */ BTF_SET_START(bpf_lsm_locked_sockopt_hooks) +#ifdef CONFIG_SECURITY_NETWORK BTF_ID(func, bpf_lsm_socket_sock_rcv_skb) BTF_ID(func, bpf_lsm_sock_graft) BTF_ID(func, bpf_lsm_inet_csk_clone) BTF_ID(func, bpf_lsm_inet_conn_established) +#endif BTF_SET_END(bpf_lsm_locked_sockopt_hooks) /* List of LSM hooks that trigger while the socket is _not_ locked, @@ -59,8 +63,10 @@ BTF_SET_END(bpf_lsm_locked_sockopt_hooks) * in the early init phase. */ BTF_SET_START(bpf_lsm_unlocked_sockopt_hooks) +#ifdef CONFIG_SECURITY_NETWORK BTF_ID(func, bpf_lsm_socket_post_create) BTF_ID(func, bpf_lsm_socket_socketpair) +#endif BTF_SET_END(bpf_lsm_unlocked_sockopt_hooks) #ifdef CONFIG_CGROUP_BPF @@ -189,6 +195,14 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto = { static const struct bpf_func_proto * bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + if (prog->expected_attach_type == BPF_LSM_CGROUP) { + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + } + switch (func_id) { case BPF_FUNC_inode_storage_get: return &bpf_inode_storage_get_proto; @@ -212,15 +226,6 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL; case BPF_FUNC_get_attach_cookie: return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL; - case BPF_FUNC_get_local_storage: - return prog->expected_attach_type == BPF_LSM_CGROUP ? - &bpf_get_local_storage_proto : NULL; - case BPF_FUNC_set_retval: - return prog->expected_attach_type == BPF_LSM_CGROUP ? - &bpf_set_retval_proto : NULL; - case BPF_FUNC_get_retval: - return prog->expected_attach_type == BPF_LSM_CGROUP ? - &bpf_get_retval_proto : NULL; #ifdef CONFIG_NET case BPF_FUNC_setsockopt: if (prog->expected_attach_type != BPF_LSM_CGROUP) @@ -335,6 +340,7 @@ BTF_ID(func, bpf_lsm_task_getsecid_obj) BTF_ID(func, bpf_lsm_task_prctl) BTF_ID(func, bpf_lsm_task_setscheduler) BTF_ID(func, bpf_lsm_task_to_inode) +BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) bool bpf_lsm_is_sleepable_hook(u32 btf_id) diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c index e9014dc626820a3af282e85e6bead87fcb3c3788..6f290623347e01d7bc2984f8ff0e8d7c82e1c840 100644 --- a/kernel/bpf/bpf_task_storage.c +++ b/kernel/bpf/bpf_task_storage.c @@ -26,20 +26,20 @@ static DEFINE_PER_CPU(int, bpf_task_storage_busy); static void bpf_task_storage_lock(void) { migrate_disable(); - __this_cpu_inc(bpf_task_storage_busy); + this_cpu_inc(bpf_task_storage_busy); } static void bpf_task_storage_unlock(void) { - __this_cpu_dec(bpf_task_storage_busy); + this_cpu_dec(bpf_task_storage_busy); migrate_enable(); } static bool bpf_task_storage_trylock(void) { migrate_disable(); - if (unlikely(__this_cpu_inc_return(bpf_task_storage_busy) != 1)) { - __this_cpu_dec(bpf_task_storage_busy); + if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) { + this_cpu_dec(bpf_task_storage_busy); migrate_enable(); return false; } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7e64447659f3db7f6b8c76905cc1a2fe34921e08..eba603cec2c5847e8caa2a16766c7bd2ac167bda 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -208,7 +208,7 @@ enum btf_kfunc_hook { }; enum { - BTF_KFUNC_SET_MAX_CNT = 32, + BTF_KFUNC_SET_MAX_CNT = 256, BTF_DTOR_KFUNC_MAX_CNT = 256, }; @@ -818,6 +818,7 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id) return NULL; return btf->types[type_id]; } +EXPORT_SYMBOL_GPL(btf_type_by_id); /* * Regular int is not a bit field and it must be either @@ -1396,7 +1397,6 @@ __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, const char *fmt, ...) { struct bpf_verifier_log *log = &env->log; - u8 kind = BTF_INFO_KIND(t->info); struct btf *btf = env->btf; va_list args; @@ -1412,7 +1412,7 @@ __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, __btf_verifier_log(log, "[%u] %s %s%s", env->log_type_id, - btf_kind_str[kind], + btf_type_str(t), __btf_name_by_offset(btf, t->name_off), log_details ? " " : ""); @@ -3128,7 +3128,7 @@ static int btf_struct_resolve(struct btf_verifier_env *env, if (v->next_member) { const struct btf_type *last_member_type; const struct btf_member *last_member; - u16 last_member_type_id; + u32 last_member_type_id; last_member = btf_type_member(v->t) + v->next_member - 1; last_member_type_id = last_member->type; @@ -4854,7 +4854,6 @@ static int btf_parse_hdr(struct btf_verifier_env *env) u32 hdr_len, hdr_copy, btf_data_size; const struct btf_header *hdr; struct btf *btf; - int err; btf = env->btf; btf_data_size = btf->data_size; @@ -4911,11 +4910,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env) return -EINVAL; } - err = btf_check_sec_info(env, btf_data_size); - if (err) - return err; - - return 0; + return btf_check_sec_info(env, btf_data_size); } static int btf_check_type_tags(struct btf_verifier_env *env, @@ -5328,6 +5323,34 @@ static bool is_int_ptr(struct btf *btf, const struct btf_type *t) return btf_type_is_int(t); } +static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto, + int off) +{ + const struct btf_param *args; + const struct btf_type *t; + u32 offset = 0, nr_args; + int i; + + if (!func_proto) + return off / 8; + + nr_args = btf_type_vlen(func_proto); + args = (const struct btf_param *)(func_proto + 1); + for (i = 0; i < nr_args; i++) { + t = btf_type_skip_modifiers(btf, args[i].type, NULL); + offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); + if (off < offset) + return i; + } + + t = btf_type_skip_modifiers(btf, func_proto->type, NULL); + offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8); + if (off < offset) + return nr_args; + + return nr_args + 1; +} + bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) @@ -5347,7 +5370,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, tname, off); return false; } - arg = off / 8; + arg = get_ctx_arg_idx(btf, t, off); args = (const struct btf_param *)(t + 1); /* if (t == NULL) Fall back to default BPF prog with * MAX_BPF_FUNC_REG_ARGS u64 arguments. @@ -5398,7 +5421,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, if (!btf_type_is_small_int(t)) { bpf_log(log, "ret type %s not allowed for fmod_ret\n", - btf_kind_str[BTF_INFO_KIND(t->info)]); + btf_type_str(t)); return false; } break; @@ -5417,7 +5440,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, /* skip modifiers */ while (btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); - if (btf_type_is_small_int(t) || btf_is_any_enum(t)) + if (btf_type_is_small_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t)) /* accessing a scalar */ return true; if (!btf_type_is_ptr(t)) { @@ -5425,7 +5448,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, "func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n", tname, arg, __btf_name_by_offset(btf, t->name_off), - btf_kind_str[BTF_INFO_KIND(t->info)]); + btf_type_str(t)); return false; } @@ -5509,11 +5532,11 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, if (!btf_type_is_struct(t)) { bpf_log(log, "func '%s' arg%d type %s is not a struct\n", - tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]); + tname, arg, btf_type_str(t)); return false; } bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n", - tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)], + tname, arg, info->btf_id, btf_type_str(t), __btf_name_by_offset(btf, t->name_off)); return true; } @@ -5864,26 +5887,25 @@ again: } static int __get_type_size(struct btf *btf, u32 btf_id, - const struct btf_type **bad_type) + const struct btf_type **ret_type) { const struct btf_type *t; + *ret_type = btf_type_by_id(btf, 0); if (!btf_id) /* void */ return 0; t = btf_type_by_id(btf, btf_id); while (t && btf_type_is_modifier(t)) t = btf_type_by_id(btf, t->type); - if (!t) { - *bad_type = btf_type_by_id(btf, 0); + if (!t) return -EINVAL; - } + *ret_type = t; if (btf_type_is_ptr(t)) /* kernel size of pointer. Not BPF's size of pointer*/ return sizeof(void *); - if (btf_type_is_int(t) || btf_is_any_enum(t)) + if (btf_type_is_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t)) return t->size; - *bad_type = t; return -EINVAL; } @@ -5902,8 +5924,10 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, /* BTF function prototype doesn't match the verifier types. * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args. */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { m->arg_size[i] = 8; + m->arg_flags[i] = 0; + } m->ret_size = 8; m->nr_args = MAX_BPF_FUNC_REG_ARGS; return 0; @@ -5917,10 +5941,10 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0) { + if (ret < 0 || __btf_type_is_struct(t)) { bpf_log(log, "The function %s return type %s is unsupported.\n", - tname, btf_kind_str[BTF_INFO_KIND(t->info)]); + tname, btf_type_str(t)); return -EINVAL; } m->ret_size = ret; @@ -5933,10 +5957,12 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, args[i].type, &t); - if (ret < 0) { + + /* No support of struct argument size greater than 16 bytes */ + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s arg%d type %s is unsupported.\n", - tname, i, btf_kind_str[BTF_INFO_KIND(t->info)]); + tname, i, btf_type_str(t)); return -EINVAL; } if (ret == 0) { @@ -5946,6 +5972,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } m->arg_size[i] = ret; + m->arg_flags[i] = __btf_type_is_struct(t) ? BTF_FMODEL_STRUCT_ARG : 0; } m->nr_args = nargs; return 0; @@ -6167,14 +6194,41 @@ static bool is_kfunc_arg_mem_size(const struct btf *btf, return true; } +static bool btf_is_kfunc_arg_mem_size(const struct btf *btf, + const struct btf_param *arg, + const struct bpf_reg_state *reg, + const char *name) +{ + int len, target_len = strlen(name); + const struct btf_type *t; + const char *param_name; + + t = btf_type_skip_modifiers(btf, arg->type, NULL); + if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE) + return false; + + param_name = btf_name_by_offset(btf, arg->name_off); + if (str_is_empty(param_name)) + return false; + len = strlen(param_name); + if (len != target_len) + return false; + if (strcmp(param_name, name)) + return false; + + return true; +} + static int btf_check_func_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, bool ptr_to_mem_ok, - u32 kfunc_flags) + struct bpf_kfunc_arg_meta *kfunc_meta, + bool processing_call) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); - bool rel = false, kptr_get = false, trusted_arg = false; + bool rel = false, kptr_get = false, trusted_args = false; + bool sleepable = false; struct bpf_verifier_log *log = &env->log; u32 i, nargs, ref_id, ref_obj_id = 0; bool is_kfunc = btf_is_kernel(btf); @@ -6207,11 +6261,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return -EINVAL; } - if (is_kfunc) { + if (is_kfunc && kfunc_meta) { /* Only kfunc can be release func */ - rel = kfunc_flags & KF_RELEASE; - kptr_get = kfunc_flags & KF_KPTR_GET; - trusted_arg = kfunc_flags & KF_TRUSTED_ARGS; + rel = kfunc_meta->flags & KF_RELEASE; + kptr_get = kfunc_meta->flags & KF_KPTR_GET; + trusted_args = kfunc_meta->flags & KF_TRUSTED_ARGS; + sleepable = kfunc_meta->flags & KF_SLEEPABLE; } /* check that BTF function arguments match actual types that the @@ -6221,9 +6276,42 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, enum bpf_arg_type arg_type = ARG_DONTCARE; u32 regno = i + 1; struct bpf_reg_state *reg = ®s[regno]; + bool obj_ptr = false; t = btf_type_skip_modifiers(btf, args[i].type, NULL); if (btf_type_is_scalar(t)) { + if (is_kfunc && kfunc_meta) { + bool is_buf_size = false; + + /* check for any const scalar parameter of name "rdonly_buf_size" + * or "rdwr_buf_size" + */ + if (btf_is_kfunc_arg_mem_size(btf, &args[i], reg, + "rdonly_buf_size")) { + kfunc_meta->r0_rdonly = true; + is_buf_size = true; + } else if (btf_is_kfunc_arg_mem_size(btf, &args[i], reg, + "rdwr_buf_size")) + is_buf_size = true; + + if (is_buf_size) { + if (kfunc_meta->r0_size) { + bpf_log(log, "2 or more rdonly/rdwr_buf_size parameters for kfunc"); + return -EINVAL; + } + + if (!tnum_is_const(reg->var_off)) { + bpf_log(log, "R%d is not a const\n", regno); + return -EINVAL; + } + + kfunc_meta->r0_size = reg->var_off.value; + ret = mark_chain_precision(env, regno); + if (ret) + return ret; + } + } + if (reg->type == SCALAR_VALUE) continue; bpf_log(log, "R%d is not a scalar\n", regno); @@ -6236,10 +6324,17 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return -EINVAL; } + /* These register types have special constraints wrt ref_obj_id + * and offset checks. The rest of trusted args don't. + */ + obj_ptr = reg->type == PTR_TO_CTX || reg->type == PTR_TO_BTF_ID || + reg2btf_ids[base_type(reg->type)]; + /* Check if argument must be a referenced pointer, args + i has * been verified to be a pointer (after skipping modifiers). + * PTR_TO_CTX is ok without having non-zero ref_obj_id. */ - if (is_kfunc && trusted_arg && !reg->ref_obj_id) { + if (is_kfunc && trusted_args && (obj_ptr && reg->type != PTR_TO_CTX) && !reg->ref_obj_id) { bpf_log(log, "R%d must be referenced\n", regno); return -EINVAL; } @@ -6248,12 +6343,23 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, ref_tname = btf_name_by_offset(btf, ref_t->name_off); /* Trusted args have the same offset checks as release arguments */ - if (trusted_arg || (rel && reg->ref_obj_id)) + if ((trusted_args && obj_ptr) || (rel && reg->ref_obj_id)) arg_type |= OBJ_RELEASE; ret = check_func_arg_reg_off(env, reg, regno, arg_type); if (ret < 0) return ret; + if (is_kfunc && reg->ref_obj_id) { + /* Ensure only one argument is referenced PTR_TO_BTF_ID */ + if (ref_obj_id) { + bpf_log(log, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", + regno, reg->ref_obj_id, ref_obj_id); + return -EFAULT; + } + ref_regno = regno; + ref_obj_id = reg->ref_obj_id; + } + /* kptr_get is only true for kfunc */ if (i == 0 && kptr_get) { struct bpf_map_value_off_desc *off_desc; @@ -6326,16 +6432,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, if (reg->type == PTR_TO_BTF_ID) { reg_btf = reg->btf; reg_ref_id = reg->btf_id; - /* Ensure only one argument is referenced PTR_TO_BTF_ID */ - if (reg->ref_obj_id) { - if (ref_obj_id) { - bpf_log(log, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", - regno, reg->ref_obj_id, ref_obj_id); - return -EFAULT; - } - ref_regno = regno; - ref_obj_id = reg->ref_obj_id; - } } else { reg_btf = btf_vmlinux; reg_ref_id = *reg2btf_ids[base_type(reg->type)]; @@ -6347,7 +6443,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, reg_ref_t->name_off); if (!btf_struct_ids_match(log, reg_btf, reg_ref_id, reg->off, btf, ref_id, - trusted_arg || (rel && reg->ref_obj_id))) { + trusted_args || (rel && reg->ref_obj_id))) { bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n", func_name, i, btf_type_str(ref_t), ref_tname, @@ -6355,21 +6451,26 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, reg_ref_tname); return -EINVAL; } - } else if (ptr_to_mem_ok) { + } else if (ptr_to_mem_ok && processing_call) { const struct btf_type *resolve_ret; u32 type_size; if (is_kfunc) { bool arg_mem_size = i + 1 < nargs && is_kfunc_arg_mem_size(btf, &args[i + 1], ®s[regno + 1]); + bool arg_dynptr = btf_type_is_struct(ref_t) && + !strcmp(ref_tname, + stringify_struct(bpf_dynptr_kern)); /* Permit pointer to mem, but only when argument * type is pointer to scalar, or struct composed * (recursively) of scalars. * When arg_mem_size is true, the pointer can be * void *. + * Also permit initialized local dynamic pointers. */ if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(log, btf, ref_t, 0) && + !arg_dynptr && (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) { bpf_log(log, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n", @@ -6377,6 +6478,34 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, return -EINVAL; } + if (arg_dynptr) { + if (reg->type != PTR_TO_STACK) { + bpf_log(log, "arg#%d pointer type %s %s not to stack\n", + i, btf_type_str(ref_t), + ref_tname); + return -EINVAL; + } + + if (!is_dynptr_reg_valid_init(env, reg)) { + bpf_log(log, + "arg#%d pointer type %s %s must be valid and initialized\n", + i, btf_type_str(ref_t), + ref_tname); + return -EINVAL; + } + + if (!is_dynptr_type_expected(env, reg, + ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL)) { + bpf_log(log, + "arg#%d pointer type %s %s points to unsupported dynamic pointer type\n", + i, btf_type_str(ref_t), + ref_tname); + return -EINVAL; + } + + continue; + } + /* Check for mem, len pair */ if (arg_mem_size) { if (check_kfunc_mem_size_reg(env, ®s[regno + 1], regno + 1)) { @@ -6419,11 +6548,21 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, func_name); return -EINVAL; } + + if (sleepable && !env->prog->aux->sleepable) { + bpf_log(log, "kernel function %s is sleepable but the program is not\n", + func_name); + return -EINVAL; + } + + if (kfunc_meta && ref_obj_id) + kfunc_meta->ref_obj_id = ref_obj_id; + /* returns argument register number > 0 in case of reference release kfunc */ return rel ? ref_regno : 0; } -/* Compare BTF of a function with given bpf_reg_state. +/* Compare BTF of a function declaration with given bpf_reg_state. * Returns: * EFAULT - there is a verifier bug. Abort verification. * EINVAL - there is a type mismatch or BTF is not available. @@ -6450,7 +6589,50 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, return -EINVAL; is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, 0); + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, NULL, false); + + /* Compiler optimizations can remove arguments from static functions + * or mismatched type can be passed into a global function. + * In such cases mark the function as unreliable from BTF point of view. + */ + if (err) + prog->aux->func_info_aux[subprog].unreliable = true; + return err; +} + +/* Compare BTF of a function call with given bpf_reg_state. + * Returns: + * EFAULT - there is a verifier bug. Abort verification. + * EINVAL - there is a type mismatch or BTF is not available. + * 0 - BTF matches with what bpf_reg_state expects. + * Only PTR_TO_CTX and SCALAR_VALUE states are recognized. + * + * NOTE: the code is duplicated from btf_check_subprog_arg_match() + * because btf_check_func_arg_match() is still doing both. Once that + * function is split in 2, we can call from here btf_check_subprog_arg_match() + * first, and then treat the calling part in a new code path. + */ +int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog, + struct bpf_reg_state *regs) +{ + struct bpf_prog *prog = env->prog; + struct btf *btf = prog->aux->btf; + bool is_global; + u32 btf_id; + int err; + + if (!prog->aux->func_info) + return -EINVAL; + + btf_id = prog->aux->func_info[subprog].type_id; + if (!btf_id) + return -EFAULT; + + if (prog->aux->func_info_aux[subprog].unreliable) + return -EINVAL; + + is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; + err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, NULL, true); /* Compiler optimizations can remove arguments from static functions * or mismatched type can be passed into a global function. @@ -6464,9 +6646,9 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog, int btf_check_kfunc_arg_match(struct bpf_verifier_env *env, const struct btf *btf, u32 func_id, struct bpf_reg_state *regs, - u32 kfunc_flags) + struct bpf_kfunc_arg_meta *meta) { - return btf_check_func_arg_match(env, btf, func_id, regs, true, kfunc_flags); + return btf_check_func_arg_match(env, btf, func_id, regs, true, meta, true); } /* Convert BTF of a function into bpf_reg_state if possible @@ -6580,7 +6762,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, continue; } bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n", - i, btf_kind_str[BTF_INFO_KIND(t->info)], tname); + i, btf_type_str(t), tname); return -EINVAL; } return 0; @@ -7235,6 +7417,7 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type) case BPF_PROG_TYPE_STRUCT_OPS: return BTF_KFUNC_HOOK_STRUCT_OPS; case BPF_PROG_TYPE_TRACING: + case BPF_PROG_TYPE_LSM: return BTF_KFUNC_HOOK_TRACING; case BPF_PROG_TYPE_SYSCALL: return BTF_KFUNC_HOOK_SYSCALL; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 59b7eb60d5b46a05dfba0ce598f3dad53aa9863e..bf2fdb33fb3139dc58b312c4a1db4071eaecc600 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -921,8 +921,10 @@ static void purge_effective_progs(struct cgroup *cgrp, struct bpf_prog *prog, pos++; } } + + /* no link or prog match, skip the cgroup of this layer */ + continue; found: - BUG_ON(!cg); progs = rcu_dereference_protected( desc->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); @@ -1018,6 +1020,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, union bpf_attr __user *uattr) { __u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags); + bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE; __u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids); enum bpf_attach_type type = attr->query.attach_type; enum cgroup_bpf_attach_type from_atype, to_atype; @@ -1027,8 +1030,12 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, int total_cnt = 0; u32 flags; + if (effective_query && prog_attach_flags) + return -EINVAL; + if (type == BPF_LSM_CGROUP) { - if (attr->query.prog_cnt && prog_ids && !prog_attach_flags) + if (!effective_query && attr->query.prog_cnt && + prog_ids && !prog_attach_flags) return -EINVAL; from_atype = CGROUP_LSM_START; @@ -1043,7 +1050,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } for (atype = from_atype; atype <= to_atype; atype++) { - if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) { + if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); total_cnt += bpf_prog_array_length(effective); @@ -1052,6 +1059,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } } + /* always output uattr->query.attach_flags as 0 during effective query */ + flags = effective_query ? 0 : flags; if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) return -EFAULT; if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt))) @@ -1066,7 +1075,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, } for (atype = from_atype; atype <= to_atype && total_cnt; atype++) { - if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) { + if (effective_query) { effective = rcu_dereference_protected(cgrp->bpf.effective[atype], lockdep_is_held(&cgroup_mutex)); cnt = min_t(int, bpf_prog_array_length(effective), total_cnt); @@ -1088,15 +1097,16 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, if (++i == cnt) break; } - } - if (prog_attach_flags) { - flags = cgrp->bpf.flags[atype]; + if (prog_attach_flags) { + flags = cgrp->bpf.flags[atype]; - for (i = 0; i < cnt; i++) - if (copy_to_user(prog_attach_flags + i, &flags, sizeof(flags))) - return -EFAULT; - prog_attach_flags += cnt; + for (i = 0; i < cnt; i++) + if (copy_to_user(prog_attach_flags + i, + &flags, sizeof(flags))) + return -EFAULT; + prog_attach_flags += cnt; + } } prog_ids += cnt; @@ -1527,6 +1537,37 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, return ret; } +BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) +{ + /* flags argument is not used now, + * but provides an ability to extend the API. + * verifier checks that its value is correct. + */ + enum bpf_cgroup_storage_type stype = cgroup_storage_type(map); + struct bpf_cgroup_storage *storage; + struct bpf_cg_run_ctx *ctx; + void *ptr; + + /* get current cgroup storage from BPF run context */ + ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx); + storage = ctx->prog_item->cgroup_storage[stype]; + + if (stype == BPF_CGROUP_STORAGE_SHARED) + ptr = &READ_ONCE(storage->buf)->data[0]; + else + ptr = this_cpu_ptr(storage->percpu_buf); + + return (unsigned long)ptr; +} + +const struct bpf_func_proto bpf_get_local_storage_proto = { + .func = bpf_get_local_storage, + .gpl_only = false, + .ret_type = RET_PTR_TO_MAP_VALUE, + .arg1_type = ARG_CONST_MAP_PTR, + .arg2_type = ARG_ANYTHING, +}; + BPF_CALL_0(bpf_get_retval) { struct bpf_cg_run_ctx *ctx = @@ -1558,32 +1599,26 @@ const struct bpf_func_proto bpf_set_retval_proto = { }; static const struct bpf_func_proto * -cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + + func_proto = cgroup_current_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { - case BPF_FUNC_get_current_uid_gid: - return &bpf_get_current_uid_gid_proto; - case BPF_FUNC_get_local_storage: - return &bpf_get_local_storage_proto; - case BPF_FUNC_get_current_cgroup_id: - return &bpf_get_current_cgroup_id_proto; case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; - case BPF_FUNC_get_retval: - return &bpf_get_retval_proto; - case BPF_FUNC_set_retval: - return &bpf_set_retval_proto; default: return bpf_base_func_proto(func_id); } } -static const struct bpf_func_proto * -cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) -{ - return cgroup_base_func_proto(func_id, prog); -} - static bool cgroup_dev_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, @@ -2096,11 +2131,17 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { static const struct bpf_func_proto * sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + + func_proto = cgroup_current_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { - case BPF_FUNC_strtol: - return &bpf_strtol_proto; - case BPF_FUNC_strtoul: - return &bpf_strtoul_proto; case BPF_FUNC_sysctl_get_name: return &bpf_sysctl_get_name_proto; case BPF_FUNC_sysctl_get_current_value: @@ -2111,8 +2152,10 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_sysctl_set_new_value_proto; case BPF_FUNC_ktime_get_coarse_ns: return &bpf_ktime_get_coarse_ns_proto; + case BPF_FUNC_perf_event_output: + return &bpf_event_output_data_proto; default: - return cgroup_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -2233,6 +2276,16 @@ static const struct bpf_func_proto bpf_get_netns_cookie_sockopt_proto = { static const struct bpf_func_proto * cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + + func_proto = cgroup_current_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { #ifdef CONFIG_NET case BPF_FUNC_get_netns_cookie: @@ -2254,8 +2307,10 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_FUNC_tcp_sock: return &bpf_tcp_sock_proto; #endif + case BPF_FUNC_perf_event_output: + return &bpf_event_output_data_proto; default: - return cgroup_base_func_proto(func_id, prog); + return bpf_base_func_proto(func_id); } } @@ -2420,3 +2475,69 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { const struct bpf_prog_ops cg_sockopt_prog_ops = { }; + +/* Common helpers for cgroup hooks. */ +const struct bpf_func_proto * +cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_get_local_storage: + return &bpf_get_local_storage_proto; + case BPF_FUNC_get_retval: + switch (prog->expected_attach_type) { + case BPF_CGROUP_INET_INGRESS: + case BPF_CGROUP_INET_EGRESS: + case BPF_CGROUP_SOCK_OPS: + case BPF_CGROUP_UDP4_RECVMSG: + case BPF_CGROUP_UDP6_RECVMSG: + case BPF_CGROUP_INET4_GETPEERNAME: + case BPF_CGROUP_INET6_GETPEERNAME: + case BPF_CGROUP_INET4_GETSOCKNAME: + case BPF_CGROUP_INET6_GETSOCKNAME: + return NULL; + default: + return &bpf_get_retval_proto; + } + case BPF_FUNC_set_retval: + switch (prog->expected_attach_type) { + case BPF_CGROUP_INET_INGRESS: + case BPF_CGROUP_INET_EGRESS: + case BPF_CGROUP_SOCK_OPS: + case BPF_CGROUP_UDP4_RECVMSG: + case BPF_CGROUP_UDP6_RECVMSG: + case BPF_CGROUP_INET4_GETPEERNAME: + case BPF_CGROUP_INET6_GETPEERNAME: + case BPF_CGROUP_INET4_GETSOCKNAME: + case BPF_CGROUP_INET6_GETSOCKNAME: + return NULL; + default: + return &bpf_set_retval_proto; + } + default: + return NULL; + } +} + +/* Common helpers for cgroup hooks with valid process context. */ +const struct bpf_func_proto * +cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_get_current_uid_gid: + return &bpf_get_current_uid_gid_proto; + case BPF_FUNC_get_current_pid_tgid: + return &bpf_get_current_pid_tgid_proto; + case BPF_FUNC_get_current_comm: + return &bpf_get_current_comm_proto; + case BPF_FUNC_get_current_cgroup_id: + return &bpf_get_current_cgroup_id_proto; + case BPF_FUNC_get_current_ancestor_cgroup_id: + return &bpf_get_current_ancestor_cgroup_id_proto; +#ifdef CONFIG_CGROUP_NET_CLASSID + case BPF_FUNC_get_cgroup_classid: + return &bpf_get_cgroup_classid_curr_proto; +#endif + default: + return NULL; + } +} diff --git a/kernel/bpf/cgroup_iter.c b/kernel/bpf/cgroup_iter.c new file mode 100644 index 0000000000000000000000000000000000000000..0d200a993489cb72545cf325d565685f58e3d54b --- /dev/null +++ b/kernel/bpf/cgroup_iter.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2022 Google */ +#include +#include +#include +#include +#include + +#include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */ + +/* cgroup_iter provides four modes of traversal to the cgroup hierarchy. + * + * 1. Walk the descendants of a cgroup in pre-order. + * 2. Walk the descendants of a cgroup in post-order. + * 3. Walk the ancestors of a cgroup. + * 4. Show the given cgroup only. + * + * For walking descendants, cgroup_iter can walk in either pre-order or + * post-order. For walking ancestors, the iter walks up from a cgroup to + * the root. + * + * The iter program can terminate the walk early by returning 1. Walk + * continues if prog returns 0. + * + * The prog can check (seq->num == 0) to determine whether this is + * the first element. The prog may also be passed a NULL cgroup, + * which means the walk has completed and the prog has a chance to + * do post-processing, such as outputting an epilogue. + * + * Note: the iter_prog is called with cgroup_mutex held. + * + * Currently only one session is supported, which means, depending on the + * volume of data bpf program intends to send to user space, the number + * of cgroups that can be walked is limited. For example, given the current + * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each + * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can + * be walked is 512. This is a limitation of cgroup_iter. If the output data + * is larger than the kernel buffer size, after all data in the kernel buffer + * is consumed by user space, the subsequent read() syscall will signal + * EOPNOTSUPP. In order to work around, the user may have to update their + * program to reduce the volume of data sent to output. For example, skip + * some uninteresting cgroups. + */ + +struct bpf_iter__cgroup { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct cgroup *, cgroup); +}; + +struct cgroup_iter_priv { + struct cgroup_subsys_state *start_css; + bool visited_all; + bool terminate; + int order; +}; + +static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct cgroup_iter_priv *p = seq->private; + + mutex_lock(&cgroup_mutex); + + /* cgroup_iter doesn't support read across multiple sessions. */ + if (*pos > 0) { + if (p->visited_all) + return NULL; + + /* Haven't visited all, but because cgroup_mutex has dropped, + * return -EOPNOTSUPP to indicate incomplete iteration. + */ + return ERR_PTR(-EOPNOTSUPP); + } + + ++*pos; + p->terminate = false; + p->visited_all = false; + if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) + return css_next_descendant_pre(NULL, p->start_css); + else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) + return css_next_descendant_post(NULL, p->start_css); + else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */ + return p->start_css; +} + +static int __cgroup_iter_seq_show(struct seq_file *seq, + struct cgroup_subsys_state *css, int in_stop); + +static void cgroup_iter_seq_stop(struct seq_file *seq, void *v) +{ + struct cgroup_iter_priv *p = seq->private; + + mutex_unlock(&cgroup_mutex); + + /* pass NULL to the prog for post-processing */ + if (!v) { + __cgroup_iter_seq_show(seq, NULL, true); + p->visited_all = true; + } +} + +static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v; + struct cgroup_iter_priv *p = seq->private; + + ++*pos; + if (p->terminate) + return NULL; + + if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE) + return css_next_descendant_pre(curr, p->start_css); + else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST) + return css_next_descendant_post(curr, p->start_css); + else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP) + return curr->parent; + else /* BPF_CGROUP_ITER_SELF_ONLY */ + return NULL; +} + +static int __cgroup_iter_seq_show(struct seq_file *seq, + struct cgroup_subsys_state *css, int in_stop) +{ + struct cgroup_iter_priv *p = seq->private; + struct bpf_iter__cgroup ctx; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + int ret = 0; + + /* cgroup is dead, skip this element */ + if (css && cgroup_is_dead(css->cgroup)) + return 0; + + ctx.meta = &meta; + ctx.cgroup = css ? css->cgroup : NULL; + meta.seq = seq; + prog = bpf_iter_get_info(&meta, in_stop); + if (prog) + ret = bpf_iter_run_prog(prog, &ctx); + + /* if prog returns > 0, terminate after this element. */ + if (ret != 0) + p->terminate = true; + + return 0; +} + +static int cgroup_iter_seq_show(struct seq_file *seq, void *v) +{ + return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v, + false); +} + +static const struct seq_operations cgroup_iter_seq_ops = { + .start = cgroup_iter_seq_start, + .next = cgroup_iter_seq_next, + .stop = cgroup_iter_seq_stop, + .show = cgroup_iter_seq_show, +}; + +BTF_ID_LIST_SINGLE(bpf_cgroup_btf_id, struct, cgroup) + +static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux) +{ + struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv; + struct cgroup *cgrp = aux->cgroup.start; + + p->start_css = &cgrp->self; + p->terminate = false; + p->visited_all = false; + p->order = aux->cgroup.order; + return 0; +} + +static const struct bpf_iter_seq_info cgroup_iter_seq_info = { + .seq_ops = &cgroup_iter_seq_ops, + .init_seq_private = cgroup_iter_seq_init, + .seq_priv_size = sizeof(struct cgroup_iter_priv), +}; + +static int bpf_iter_attach_cgroup(struct bpf_prog *prog, + union bpf_iter_link_info *linfo, + struct bpf_iter_aux_info *aux) +{ + int fd = linfo->cgroup.cgroup_fd; + u64 id = linfo->cgroup.cgroup_id; + int order = linfo->cgroup.order; + struct cgroup *cgrp; + + if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE && + order != BPF_CGROUP_ITER_DESCENDANTS_POST && + order != BPF_CGROUP_ITER_ANCESTORS_UP && + order != BPF_CGROUP_ITER_SELF_ONLY) + return -EINVAL; + + if (fd && id) + return -EINVAL; + + if (fd) + cgrp = cgroup_get_from_fd(fd); + else if (id) + cgrp = cgroup_get_from_id(id); + else /* walk the entire hierarchy by default. */ + cgrp = cgroup_get_from_path("/"); + + if (IS_ERR(cgrp)) + return PTR_ERR(cgrp); + + aux->cgroup.start = cgrp; + aux->cgroup.order = order; + return 0; +} + +static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux) +{ + cgroup_put(aux->cgroup.start); +} + +static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux, + struct seq_file *seq) +{ + char *buf; + + buf = kzalloc(PATH_MAX, GFP_KERNEL); + if (!buf) { + seq_puts(seq, "cgroup_path:\t\n"); + goto show_order; + } + + /* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path + * will print nothing. + * + * Path is in the calling process's cgroup namespace. + */ + cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX, + current->nsproxy->cgroup_ns); + seq_printf(seq, "cgroup_path:\t%s\n", buf); + kfree(buf); + +show_order: + if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE) + seq_puts(seq, "order: descendants_pre\n"); + else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST) + seq_puts(seq, "order: descendants_post\n"); + else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP) + seq_puts(seq, "order: ancestors_up\n"); + else /* BPF_CGROUP_ITER_SELF_ONLY */ + seq_puts(seq, "order: self_only\n"); +} + +static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux, + struct bpf_link_info *info) +{ + info->iter.cgroup.order = aux->cgroup.order; + info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start); + return 0; +} + +DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta, + struct cgroup *cgroup) + +static struct bpf_iter_reg bpf_cgroup_reg_info = { + .target = "cgroup", + .feature = BPF_ITER_RESCHED, + .attach_target = bpf_iter_attach_cgroup, + .detach_target = bpf_iter_detach_cgroup, + .show_fdinfo = bpf_iter_cgroup_show_fdinfo, + .fill_link_info = bpf_iter_cgroup_fill_link_info, + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__cgroup, cgroup), + PTR_TO_BTF_ID_OR_NULL }, + }, + .seq_info = &cgroup_iter_seq_info, +}; + +static int __init bpf_cgroup_iter_init(void) +{ + bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0]; + return bpf_iter_reg_target(&bpf_cgroup_reg_info); +} + +late_initcall(bpf_cgroup_iter_init); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index c1e10d088dbb7388c44b17a01711c0b5d505ddb4..25a54e04560e5370635bd31dd007db754ecfae46 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -825,6 +825,11 @@ struct bpf_prog_pack { unsigned long bitmap[]; }; +void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) +{ + memset(area, 0, size); +} + #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) static DEFINE_MUTEX(pack_mutex); @@ -864,7 +869,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins return pack; } -static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) { unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size); struct bpf_prog_pack *pack; @@ -905,7 +910,7 @@ out: return ptr; } -static void bpf_prog_pack_free(struct bpf_binary_header *hdr) +void bpf_prog_pack_free(struct bpf_binary_header *hdr) { struct bpf_prog_pack *pack = NULL, *tmp; unsigned int nbits; @@ -971,7 +976,7 @@ pure_initcall(bpf_jit_charge_init); int bpf_jit_charge_modmem(u32 size) { - if (atomic_long_add_return(size, &bpf_jit_current) > bpf_jit_limit) { + if (atomic_long_add_return(size, &bpf_jit_current) > READ_ONCE(bpf_jit_limit)) { if (!bpf_capable()) { atomic_long_sub(size, &bpf_jit_current); return -EPERM; @@ -1027,7 +1032,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, hdr->size = size; hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), PAGE_SIZE - sizeof(*hdr)); - start = (get_random_int() % hole) & ~(alignment - 1); + start = prandom_u32_max(hole) & ~(alignment - 1); /* Leave a random number of instructions before BPF code. */ *image_ptr = &hdr->image[start]; @@ -1089,7 +1094,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, hole = min_t(unsigned int, size - (proglen + sizeof(*ro_header)), BPF_PROG_CHUNK_SIZE - sizeof(*ro_header)); - start = (get_random_int() % hole) & ~(alignment - 1); + start = prandom_u32_max(hole) & ~(alignment - 1); *image_ptr = &ro_header->image[start]; *rw_image = &(*rw_header)->image[start]; @@ -1211,7 +1216,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, bool emit_zext) { struct bpf_insn *to = to_buff; - u32 imm_rnd = get_random_int(); + u32 imm_rnd = get_random_u32(); s16 off; BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); @@ -2002,7 +2007,7 @@ out: static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ { \ u64 stack[stack_size / sizeof(u64)]; \ - u64 regs[MAX_BPF_EXT_REG]; \ + u64 regs[MAX_BPF_EXT_REG] = {}; \ \ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ ARG1 = (u64) (unsigned long) ctx; \ @@ -2623,6 +2628,7 @@ const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak; const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak; +const struct bpf_func_proto bpf_ktime_get_tai_ns_proto __weak; const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c index f4860ac756cdd04d421416f1b437e4ae1b3d561e..b5ba34ddd4b64818792919073be7837e69be534e 100644 --- a/kernel/bpf/cpumap.c +++ b/kernel/bpf/cpumap.c @@ -97,7 +97,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) attr->map_flags & ~BPF_F_NUMA_NODE) return ERR_PTR(-EINVAL); - cmap = kzalloc(sizeof(*cmap), GFP_USER | __GFP_ACCOUNT); + cmap = bpf_map_area_alloc(sizeof(*cmap), NUMA_NO_NODE); if (!cmap) return ERR_PTR(-ENOMEM); @@ -118,7 +118,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr) return &cmap->map; free_cmap: - kfree(cmap); + bpf_map_area_free(cmap); return ERR_PTR(err); } @@ -623,7 +623,7 @@ static void cpu_map_free(struct bpf_map *map) __cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */ } bpf_map_area_free(cmap->cpu_map); - kfree(cmap); + bpf_map_area_free(cmap); } /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index a0e02b0094876b40686215fbcfcd7049876246cd..f9a87dcc5535bbfcdc31b2e96843b108b54873d4 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -163,13 +163,13 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr) if (!capable(CAP_NET_ADMIN)) return ERR_PTR(-EPERM); - dtab = kzalloc(sizeof(*dtab), GFP_USER | __GFP_ACCOUNT); + dtab = bpf_map_area_alloc(sizeof(*dtab), NUMA_NO_NODE); if (!dtab) return ERR_PTR(-ENOMEM); err = dev_map_init_map(dtab, attr); if (err) { - kfree(dtab); + bpf_map_area_free(dtab); return ERR_PTR(err); } @@ -240,7 +240,7 @@ static void dev_map_free(struct bpf_map *map) bpf_map_area_free(dtab->netdev_map); } - kfree(dtab); + bpf_map_area_free(dtab); } static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key) diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index 2444bd15cc2d03da179102574720f3ef7a4fffca..fa64b80b8bcab5b4504920f7e48ea65cf86558f3 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -85,12 +85,12 @@ static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d, return false; } -int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs) +int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs) { return -ENOTSUPP; } -static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image) +static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) { s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; int i; @@ -99,12 +99,12 @@ static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image) if (d->progs[i].prog) *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func; } - return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs); + return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs); } static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) { - void *old, *new; + void *old, *new, *tmp; u32 noff; int err; @@ -117,8 +117,14 @@ static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) } new = d->num_progs ? d->image + noff : NULL; + tmp = d->num_progs ? d->rw_image + noff : NULL; if (new) { - if (bpf_dispatcher_prepare(d, new)) + /* Prepare the dispatcher in d->rw_image. Then use + * bpf_arch_text_copy to update d->image, which is RO+X. + */ + if (bpf_dispatcher_prepare(d, new, tmp)) + return; + if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2))) return; } @@ -140,9 +146,18 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, mutex_lock(&d->mutex); if (!d->image) { - d->image = bpf_jit_alloc_exec_page(); + d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); if (!d->image) goto out; + d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); + if (!d->rw_image) { + u32 size = PAGE_SIZE; + + bpf_arch_text_copy(d->image, &size, sizeof(size)); + bpf_prog_pack_free((struct bpf_binary_header *)d->image); + d->image = NULL; + goto out; + } bpf_image_ksym_add(d->image, &d->ksym); } diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 6c530a5e560a4a8945149355cc1c63c788e5fc80..f39ee3e0558976fc593f81b8ad6c221df0ba4806 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -14,6 +14,7 @@ #include "percpu_freelist.h" #include "bpf_lru_list.h" #include "map_in_map.h" +#include #define HTAB_CREATE_FLAG_MASK \ (BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE | \ @@ -67,24 +68,16 @@ * In theory the BPF locks could be converted to regular spinlocks as well, * but the bucket locks and percpu_freelist locks can be taken from * arbitrary contexts (perf, kprobes, tracepoints) which are required to be - * atomic contexts even on RT. These mechanisms require preallocated maps, - * so there is no need to invoke memory allocations within the lock held - * sections. - * - * BPF maps which need dynamic allocation are only used from (forced) - * thread context on RT and can therefore use regular spinlocks which in - * turn allows to invoke memory allocations from the lock held section. - * - * On a non RT kernel this distinction is neither possible nor required. - * spinlock maps to raw_spinlock and the extra code is optimized out by the - * compiler. + * atomic contexts even on RT. Before the introduction of bpf_mem_alloc, + * it is only safe to use raw spinlock for preallocated hash map on a RT kernel, + * because there is no memory allocation within the lock held sections. However + * after hash map was fully converted to use bpf_mem_alloc, there will be + * non-synchronous memory allocation for non-preallocated hash map, so it is + * safe to always use raw spinlock for bucket lock. */ struct bucket { struct hlist_nulls_head head; - union { - raw_spinlock_t raw_lock; - spinlock_t lock; - }; + raw_spinlock_t raw_lock; }; #define HASHTAB_MAP_LOCK_COUNT 8 @@ -92,6 +85,8 @@ struct bucket { struct bpf_htab { struct bpf_map map; + struct bpf_mem_alloc ma; + struct bpf_mem_alloc pcpu_ma; struct bucket *buckets; void *elems; union { @@ -99,7 +94,12 @@ struct bpf_htab { struct bpf_lru lru; }; struct htab_elem *__percpu *extra_elems; - atomic_t count; /* number of elements in this hashtable */ + /* number of elements in non-preallocated hashtable are kept + * in either pcount or count + */ + struct percpu_counter pcount; + atomic_t count; + bool use_percpu_counter; u32 n_buckets; /* number of hash buckets */ u32 elem_size; /* size of each element in bytes */ u32 hashrnd; @@ -114,14 +114,14 @@ struct htab_elem { struct { void *padding; union { - struct bpf_htab *htab; struct pcpu_freelist_node fnode; struct htab_elem *batch_flink; }; }; }; union { - struct rcu_head rcu; + /* pointer to per-cpu pointer */ + void *ptr_to_pptr; struct bpf_lru_node lru_node; }; u32 hash; @@ -133,26 +133,15 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab) return !(htab->map.map_flags & BPF_F_NO_PREALLOC); } -static inline bool htab_use_raw_lock(const struct bpf_htab *htab) -{ - return (!IS_ENABLED(CONFIG_PREEMPT_RT) || htab_is_prealloc(htab)); -} - static void htab_init_buckets(struct bpf_htab *htab) { unsigned int i; for (i = 0; i < htab->n_buckets; i++) { INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i); - if (htab_use_raw_lock(htab)) { - raw_spin_lock_init(&htab->buckets[i].raw_lock); - lockdep_set_class(&htab->buckets[i].raw_lock, - &htab->lockdep_key); - } else { - spin_lock_init(&htab->buckets[i].lock); - lockdep_set_class(&htab->buckets[i].lock, + raw_spin_lock_init(&htab->buckets[i].raw_lock); + lockdep_set_class(&htab->buckets[i].raw_lock, &htab->lockdep_key); - } cond_resched(); } } @@ -165,17 +154,14 @@ static inline int htab_lock_bucket(const struct bpf_htab *htab, hash = hash & HASHTAB_MAP_LOCK_MASK; - migrate_disable(); + preempt_disable(); if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) { __this_cpu_dec(*(htab->map_locked[hash])); - migrate_enable(); + preempt_enable(); return -EBUSY; } - if (htab_use_raw_lock(htab)) - raw_spin_lock_irqsave(&b->raw_lock, flags); - else - spin_lock_irqsave(&b->lock, flags); + raw_spin_lock_irqsave(&b->raw_lock, flags); *pflags = flags; return 0; @@ -186,12 +172,9 @@ static inline void htab_unlock_bucket(const struct bpf_htab *htab, unsigned long flags) { hash = hash & HASHTAB_MAP_LOCK_MASK; - if (htab_use_raw_lock(htab)) - raw_spin_unlock_irqrestore(&b->raw_lock, flags); - else - spin_unlock_irqrestore(&b->lock, flags); + raw_spin_unlock_irqrestore(&b->raw_lock, flags); __this_cpu_dec(*(htab->map_locked[hash])); - migrate_enable(); + preempt_enable(); } static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node); @@ -428,8 +411,6 @@ static int htab_map_alloc_check(union bpf_attr *attr) bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED); int numa_node = bpf_map_attr_numa_node(attr); - BUILD_BUG_ON(offsetof(struct htab_elem, htab) != - offsetof(struct htab_elem, hash_node.pprev)); BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) != offsetof(struct htab_elem, hash_node.pprev)); @@ -491,7 +472,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) struct bpf_htab *htab; int err, i; - htab = kzalloc(sizeof(*htab), GFP_USER | __GFP_ACCOUNT); + htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE); if (!htab) return ERR_PTR(-ENOMEM); @@ -546,10 +527,33 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) if (htab->map.map_flags & BPF_F_ZERO_SEED) htab->hashrnd = 0; else - htab->hashrnd = get_random_int(); + htab->hashrnd = get_random_u32(); htab_init_buckets(htab); +/* compute_batch_value() computes batch value as num_online_cpus() * 2 + * and __percpu_counter_compare() needs + * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus() + * for percpu_counter to be faster than atomic_t. In practice the average bpf + * hash map size is 10k, which means that a system with 64 cpus will fill + * hashmap to 20% of 10k before percpu_counter becomes ineffective. Therefore + * define our own batch count as 32 then 10k hash map can be filled up to 80%: + * 10k - 8k > 32 _batch_ * 64 _cpus_ + * and __percpu_counter_compare() will still be fast. At that point hash map + * collisions will dominate its performance anyway. Assume that hash map filled + * to 50+% isn't going to be O(1) and use the following formula to choose + * between percpu_counter and atomic_t. + */ +#define PERCPU_COUNTER_BATCH 32 + if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH) + htab->use_percpu_counter = true; + + if (htab->use_percpu_counter) { + err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL); + if (err) + goto free_map_locked; + } + if (prealloc) { err = prealloc_init(htab); if (err) @@ -563,6 +567,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) if (err) goto free_prealloc; } + } else { + err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false); + if (err) + goto free_map_locked; + if (percpu) { + err = bpf_mem_alloc_init(&htab->pcpu_ma, + round_up(htab->map.value_size, 8), true); + if (err) + goto free_map_locked; + } } return &htab->map; @@ -570,12 +584,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) free_prealloc: prealloc_destroy(htab); free_map_locked: + if (htab->use_percpu_counter) + percpu_counter_destroy(&htab->pcount); for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++) free_percpu(htab->map_locked[i]); bpf_map_area_free(htab->buckets); + bpf_mem_alloc_destroy(&htab->pcpu_ma); + bpf_mem_alloc_destroy(&htab->ma); free_htab: lockdep_unregister_key(&htab->lockdep_key); - kfree(htab); + bpf_map_area_free(htab); return ERR_PTR(err); } @@ -847,17 +865,9 @@ find_first_elem: static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) { if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) - free_percpu(htab_elem_get_ptr(l, htab->map.key_size)); + bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); check_and_free_fields(htab, l); - kfree(l); -} - -static void htab_elem_free_rcu(struct rcu_head *head) -{ - struct htab_elem *l = container_of(head, struct htab_elem, rcu); - struct bpf_htab *htab = l->htab; - - htab_elem_free(htab, l); + bpf_mem_cache_free(&htab->ma, l); } static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) @@ -871,6 +881,31 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) } } +static bool is_map_full(struct bpf_htab *htab) +{ + if (htab->use_percpu_counter) + return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, + PERCPU_COUNTER_BATCH) >= 0; + return atomic_read(&htab->count) >= htab->map.max_entries; +} + +static void inc_elem_count(struct bpf_htab *htab) +{ + if (htab->use_percpu_counter) + percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); + else + atomic_inc(&htab->count); +} + +static void dec_elem_count(struct bpf_htab *htab) +{ + if (htab->use_percpu_counter) + percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); + else + atomic_dec(&htab->count); +} + + static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) { htab_put_fd_value(htab, l); @@ -879,9 +914,8 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) check_and_free_fields(htab, l); __pcpu_freelist_push(&htab->freelist, &l->fnode); } else { - atomic_dec(&htab->count); - l->htab = htab; - call_rcu(&l->rcu, htab_elem_free_rcu); + dec_elem_count(htab); + htab_elem_free(htab, l); } } @@ -906,13 +940,12 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, void *value, bool onallcpus) { - /* When using prealloc and not setting the initial value on all cpus, - * zero-fill element values for other cpus (just as what happens when - * not using prealloc). Otherwise, bpf program has no way to ensure + /* When not setting the initial value on all cpus, zero-fill element + * values for other cpus. Otherwise, bpf program has no way to ensure * known initial values for cpus other than current one * (onallcpus=false always when coming from bpf prog). */ - if (htab_is_prealloc(htab) && !onallcpus) { + if (!onallcpus) { u32 size = round_up(htab->map.value_size, 8); int current_cpu = raw_smp_processor_id(); int cpu; @@ -963,19 +996,16 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, l_new = container_of(l, struct htab_elem, fnode); } } else { - if (atomic_inc_return(&htab->count) > htab->map.max_entries) - if (!old_elem) { + if (is_map_full(htab)) + if (!old_elem) /* when map is full and update() is replacing * old element, it's ok to allocate, since * old element will be freed immediately. * Otherwise return an error */ - l_new = ERR_PTR(-E2BIG); - goto dec_count; - } - l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size, - GFP_NOWAIT | __GFP_NOWARN, - htab->map.numa_node); + return ERR_PTR(-E2BIG); + inc_elem_count(htab); + l_new = bpf_mem_cache_alloc(&htab->ma); if (!l_new) { l_new = ERR_PTR(-ENOMEM); goto dec_count; @@ -986,18 +1016,18 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, memcpy(l_new->key, key, key_size); if (percpu) { - size = round_up(size, 8); if (prealloc) { pptr = htab_elem_get_ptr(l_new, key_size); } else { /* alloc_percpu zero-fills */ - pptr = bpf_map_alloc_percpu(&htab->map, size, 8, - GFP_NOWAIT | __GFP_NOWARN); + pptr = bpf_mem_cache_alloc(&htab->pcpu_ma); if (!pptr) { - kfree(l_new); + bpf_mem_cache_free(&htab->ma, l_new); l_new = ERR_PTR(-ENOMEM); goto dec_count; } + l_new->ptr_to_pptr = pptr; + pptr = *(void **)pptr; } pcpu_init_value(htab, pptr, value, onallcpus); @@ -1016,7 +1046,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, l_new->hash = hash; return l_new; dec_count: - atomic_dec(&htab->count); + dec_elem_count(htab); return l_new; } @@ -1416,6 +1446,10 @@ static void delete_all_elements(struct bpf_htab *htab) { int i; + /* It's called from a worker thread, so disable migration here, + * since bpf_mem_cache_free() relies on that. + */ + migrate_disable(); for (i = 0; i < htab->n_buckets; i++) { struct hlist_nulls_head *head = select_bucket(htab, i); struct hlist_nulls_node *n; @@ -1426,6 +1460,7 @@ static void delete_all_elements(struct bpf_htab *htab) htab_elem_free(htab, l); } } + migrate_enable(); } static void htab_free_malloced_timers(struct bpf_htab *htab) @@ -1475,10 +1510,10 @@ static void htab_map_free(struct bpf_map *map) * There is no need to synchronize_rcu() here to protect map elements. */ - /* some of free_htab_elem() callbacks for elements of this map may - * not have executed. Wait for them. + /* htab no longer uses call_rcu() directly. bpf_mem_alloc does it + * underneath and is reponsible for waiting for callbacks to finish + * during bpf_mem_alloc_destroy(). */ - rcu_barrier(); if (!htab_is_prealloc(htab)) { delete_all_elements(htab); } else { @@ -1489,10 +1524,14 @@ static void htab_map_free(struct bpf_map *map) bpf_map_free_kptr_off_tab(map); free_percpu(htab->extra_elems); bpf_map_area_free(htab->buckets); + bpf_mem_alloc_destroy(&htab->pcpu_ma); + bpf_mem_alloc_destroy(&htab->ma); + if (htab->use_percpu_counter) + percpu_counter_destroy(&htab->pcount); for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++) free_percpu(htab->map_locked[i]); lockdep_unregister_key(&htab->lockdep_key); - kfree(htab); + bpf_map_area_free(htab); } static void htab_map_seq_show_elem(struct bpf_map *map, void *key, @@ -1691,8 +1730,11 @@ again_nocopy: /* do not grab the lock unless need it (bucket_cnt > 0). */ if (locked) { ret = htab_lock_bucket(htab, b, batch, &flags); - if (ret) - goto next_batch; + if (ret) { + rcu_read_unlock(); + bpf_enable_instrumentation(); + goto after_loop; + } } bucket_cnt = 0; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 1f961f9982d276353035185024901672de569ed9..a6b04faed282bd3d9826a7931362a3aca60cde36 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -198,6 +199,18 @@ const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto = { .ret_type = RET_INTEGER, }; +BPF_CALL_0(bpf_ktime_get_tai_ns) +{ + /* NMI safe access to clock tai */ + return ktime_get_tai_fast_ns(); +} + +const struct bpf_func_proto bpf_ktime_get_tai_ns_proto = { + .func = bpf_ktime_get_tai_ns, + .gpl_only = false, + .ret_type = RET_INTEGER, +}; + BPF_CALL_0(bpf_get_current_pid_tgid) { struct task_struct *task = current; @@ -415,40 +428,7 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, }; - -#ifdef CONFIG_CGROUP_BPF - -BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags) -{ - /* flags argument is not used now, - * but provides an ability to extend the API. - * verifier checks that its value is correct. - */ - enum bpf_cgroup_storage_type stype = cgroup_storage_type(map); - struct bpf_cgroup_storage *storage; - struct bpf_cg_run_ctx *ctx; - void *ptr; - - /* get current cgroup storage from BPF run context */ - ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx); - storage = ctx->prog_item->cgroup_storage[stype]; - - if (stype == BPF_CGROUP_STORAGE_SHARED) - ptr = &READ_ONCE(storage->buf)->data[0]; - else - ptr = this_cpu_ptr(storage->percpu_buf); - - return (unsigned long)ptr; -} - -const struct bpf_func_proto bpf_get_local_storage_proto = { - .func = bpf_get_local_storage, - .gpl_only = false, - .ret_type = RET_PTR_TO_MAP_VALUE, - .arg1_type = ARG_CONST_MAP_PTR, - .arg2_type = ARG_ANYTHING, -}; -#endif +#endif /* CONFIG_CGROUPS */ #define BPF_STRTOX_BASE_MASK 0x1F @@ -577,7 +557,6 @@ const struct bpf_func_proto bpf_strtoul_proto = { .arg3_type = ARG_ANYTHING, .arg4_type = ARG_PTR_TO_LONG, }; -#endif BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2) { @@ -1398,10 +1377,9 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr) } /* Unlike other PTR_TO_BTF_ID helpers the btf_id in bpf_kptr_xchg() - * helper is determined dynamically by the verifier. + * helper is determined dynamically by the verifier. Use BPF_PTR_POISON to + * denote type that verifier will determine. */ -#define BPF_PTR_POISON ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) - static const struct bpf_func_proto bpf_kptr_xchg_proto = { .func = bpf_kptr_xchg, .gpl_only = false, @@ -1430,7 +1408,7 @@ static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_typ ptr->size |= type << DYNPTR_TYPE_SHIFT; } -static u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr) +u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr) { return ptr->size & DYNPTR_SIZE_MASK; } @@ -1468,6 +1446,8 @@ BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_ { int err; + BTF_TYPE_EMIT(struct bpf_dynptr); + err = bpf_dynptr_check_size(size); if (err) goto error; @@ -1617,6 +1597,8 @@ bpf_base_func_proto(enum bpf_func_id func_id) return &bpf_ktime_get_ns_proto; case BPF_FUNC_ktime_get_boot_ns: return &bpf_ktime_get_boot_ns_proto; + case BPF_FUNC_ktime_get_tai_ns: + return &bpf_ktime_get_tai_ns_proto; case BPF_FUNC_ringbuf_output: return &bpf_ringbuf_output_proto; case BPF_FUNC_ringbuf_reserve: @@ -1627,26 +1609,12 @@ bpf_base_func_proto(enum bpf_func_id func_id) return &bpf_ringbuf_discard_proto; case BPF_FUNC_ringbuf_query: return &bpf_ringbuf_query_proto; - case BPF_FUNC_ringbuf_reserve_dynptr: - return &bpf_ringbuf_reserve_dynptr_proto; - case BPF_FUNC_ringbuf_submit_dynptr: - return &bpf_ringbuf_submit_dynptr_proto; - case BPF_FUNC_ringbuf_discard_dynptr: - return &bpf_ringbuf_discard_dynptr_proto; - case BPF_FUNC_for_each_map_elem: - return &bpf_for_each_map_elem_proto; - case BPF_FUNC_loop: - return &bpf_loop_proto; case BPF_FUNC_strncmp: return &bpf_strncmp_proto; - case BPF_FUNC_dynptr_from_mem: - return &bpf_dynptr_from_mem_proto; - case BPF_FUNC_dynptr_read: - return &bpf_dynptr_read_proto; - case BPF_FUNC_dynptr_write: - return &bpf_dynptr_write_proto; - case BPF_FUNC_dynptr_data: - return &bpf_dynptr_data_proto; + case BPF_FUNC_strtol: + return &bpf_strtol_proto; + case BPF_FUNC_strtoul: + return &bpf_strtoul_proto; default: break; } @@ -1675,6 +1643,26 @@ bpf_base_func_proto(enum bpf_func_id func_id) return &bpf_timer_cancel_proto; case BPF_FUNC_kptr_xchg: return &bpf_kptr_xchg_proto; + case BPF_FUNC_for_each_map_elem: + return &bpf_for_each_map_elem_proto; + case BPF_FUNC_loop: + return &bpf_loop_proto; + case BPF_FUNC_user_ringbuf_drain: + return &bpf_user_ringbuf_drain_proto; + case BPF_FUNC_ringbuf_reserve_dynptr: + return &bpf_ringbuf_reserve_dynptr_proto; + case BPF_FUNC_ringbuf_submit_dynptr: + return &bpf_ringbuf_submit_dynptr_proto; + case BPF_FUNC_ringbuf_discard_dynptr: + return &bpf_ringbuf_discard_dynptr_proto; + case BPF_FUNC_dynptr_from_mem: + return &bpf_dynptr_from_mem_proto; + case BPF_FUNC_dynptr_read: + return &bpf_dynptr_read_proto; + case BPF_FUNC_dynptr_write: + return &bpf_dynptr_write_proto; + case BPF_FUNC_dynptr_data: + return &bpf_dynptr_data_proto; default: break; } @@ -1711,3 +1699,21 @@ bpf_base_func_proto(enum bpf_func_id func_id) return NULL; } } + +BTF_SET8_START(tracing_btf_ids) +#ifdef CONFIG_KEXEC_CORE +BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE) +#endif +BTF_SET8_END(tracing_btf_ids) + +static const struct btf_kfunc_id_set tracing_kfunc_set = { + .owner = THIS_MODULE, + .set = &tracing_btf_ids, +}; + +static int __init kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &tracing_kfunc_set); +} + +late_initcall(kfunc_init); diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 49ef0ce040c7a879398be305e802efe51e3d814f..098cf336fae6e1fd628b2c693b7252b19d41d2e6 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -313,8 +313,7 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) /* max_entries is not used and enforced to be 0 */ return ERR_PTR(-EINVAL); - map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map), - __GFP_ZERO | GFP_USER | __GFP_ACCOUNT, numa_node); + map = bpf_map_area_alloc(sizeof(struct bpf_cgroup_storage_map), numa_node); if (!map) return ERR_PTR(-ENOMEM); @@ -346,7 +345,7 @@ static void cgroup_storage_map_free(struct bpf_map *_map) WARN_ON(!RB_EMPTY_ROOT(&map->root)); WARN_ON(!list_empty(&map->list)); - kfree(map); + bpf_map_area_free(map); } static int cgroup_storage_delete_elem(struct bpf_map *map, void *key) diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c index d789e3b831add378268787b1a201bc5a38d24639..d833496e9e42633fb159a044d12ce41f1521fa8d 100644 --- a/kernel/bpf/lpm_trie.c +++ b/kernel/bpf/lpm_trie.c @@ -558,7 +558,7 @@ static struct bpf_map *trie_alloc(union bpf_attr *attr) attr->value_size > LPM_VAL_SIZE_MAX) return ERR_PTR(-EINVAL); - trie = kzalloc(sizeof(*trie), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT); + trie = bpf_map_area_alloc(sizeof(*trie), NUMA_NO_NODE); if (!trie) return ERR_PTR(-ENOMEM); @@ -609,7 +609,7 @@ static void trie_free(struct bpf_map *map) } out: - kfree(trie); + bpf_map_area_free(trie); } static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c new file mode 100644 index 0000000000000000000000000000000000000000..5f83be1d20181b47a30ebd34cba708b51ea48349 --- /dev/null +++ b/kernel/bpf/memalloc.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include +#include +#include +#include + +/* Any context (including NMI) BPF specific memory allocator. + * + * Tracing BPF programs can attach to kprobe and fentry. Hence they + * run in unknown context where calling plain kmalloc() might not be safe. + * + * Front-end kmalloc() with per-cpu per-bucket cache of free elements. + * Refill this cache asynchronously from irq_work. + * + * CPU_0 buckets + * 16 32 64 96 128 196 256 512 1024 2048 4096 + * ... + * CPU_N buckets + * 16 32 64 96 128 196 256 512 1024 2048 4096 + * + * The buckets are prefilled at the start. + * BPF programs always run with migration disabled. + * It's safe to allocate from cache of the current cpu with irqs disabled. + * Free-ing is always done into bucket of the current cpu as well. + * irq_work trims extra free elements from buckets with kfree + * and refills them with kmalloc, so global kmalloc logic takes care + * of freeing objects allocated by one cpu and freed on another. + * + * Every allocated objected is padded with extra 8 bytes that contains + * struct llist_node. + */ +#define LLIST_NODE_SZ sizeof(struct llist_node) + +/* similar to kmalloc, but sizeof == 8 bucket is gone */ +static u8 size_index[24] __ro_after_init = { + 3, /* 8 */ + 3, /* 16 */ + 4, /* 24 */ + 4, /* 32 */ + 5, /* 40 */ + 5, /* 48 */ + 5, /* 56 */ + 5, /* 64 */ + 1, /* 72 */ + 1, /* 80 */ + 1, /* 88 */ + 1, /* 96 */ + 6, /* 104 */ + 6, /* 112 */ + 6, /* 120 */ + 6, /* 128 */ + 2, /* 136 */ + 2, /* 144 */ + 2, /* 152 */ + 2, /* 160 */ + 2, /* 168 */ + 2, /* 176 */ + 2, /* 184 */ + 2 /* 192 */ +}; + +static int bpf_mem_cache_idx(size_t size) +{ + if (!size || size > 4096) + return -1; + + if (size <= 192) + return size_index[(size - 1) / 8] - 1; + + return fls(size - 1) - 1; +} + +#define NUM_CACHES 11 + +struct bpf_mem_cache { + /* per-cpu list of free objects of size 'unit_size'. + * All accesses are done with interrupts disabled and 'active' counter + * protection with __llist_add() and __llist_del_first(). + */ + struct llist_head free_llist; + local_t active; + + /* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill + * are sequenced by per-cpu 'active' counter. But unit_free() cannot + * fail. When 'active' is busy the unit_free() will add an object to + * free_llist_extra. + */ + struct llist_head free_llist_extra; + + struct irq_work refill_work; + struct obj_cgroup *objcg; + int unit_size; + /* count of objects in free_llist */ + int free_cnt; + int low_watermark, high_watermark, batch; + int percpu_size; + + struct rcu_head rcu; + struct llist_head free_by_rcu; + struct llist_head waiting_for_gp; + atomic_t call_rcu_in_progress; +}; + +struct bpf_mem_caches { + struct bpf_mem_cache cache[NUM_CACHES]; +}; + +static struct llist_node notrace *__llist_del_first(struct llist_head *head) +{ + struct llist_node *entry, *next; + + entry = head->first; + if (!entry) + return NULL; + next = entry->next; + head->first = next; + return entry; +} + +static void *__alloc(struct bpf_mem_cache *c, int node) +{ + /* Allocate, but don't deplete atomic reserves that typical + * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc + * will allocate from the current numa node which is what we + * want here. + */ + gfp_t flags = GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT; + + if (c->percpu_size) { + void **obj = kmalloc_node(c->percpu_size, flags, node); + void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags); + + if (!obj || !pptr) { + free_percpu(pptr); + kfree(obj); + return NULL; + } + obj[1] = pptr; + return obj; + } + + return kmalloc_node(c->unit_size, flags, node); +} + +static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c) +{ +#ifdef CONFIG_MEMCG_KMEM + if (c->objcg) + return get_mem_cgroup_from_objcg(c->objcg); +#endif + +#ifdef CONFIG_MEMCG + return root_mem_cgroup; +#else + return NULL; +#endif +} + +/* Mostly runs from irq_work except __init phase. */ +static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node) +{ + struct mem_cgroup *memcg = NULL, *old_memcg; + unsigned long flags; + void *obj; + int i; + + memcg = get_memcg(c); + old_memcg = set_active_memcg(memcg); + for (i = 0; i < cnt; i++) { + obj = __alloc(c, node); + if (!obj) + break; + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + /* In RT irq_work runs in per-cpu kthread, so disable + * interrupts to avoid preemption and interrupts and + * reduce the chance of bpf prog executing on this cpu + * when active counter is busy. + */ + local_irq_save(flags); + /* alloc_bulk runs from irq_work which will not preempt a bpf + * program that does unit_alloc/unit_free since IRQs are + * disabled there. There is no race to increment 'active' + * counter. It protects free_llist from corruption in case NMI + * bpf prog preempted this loop. + */ + WARN_ON_ONCE(local_inc_return(&c->active) != 1); + __llist_add(obj, &c->free_llist); + c->free_cnt++; + local_dec(&c->active); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_restore(flags); + } + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +} + +static void free_one(struct bpf_mem_cache *c, void *obj) +{ + if (c->percpu_size) { + free_percpu(((void **)obj)[1]); + kfree(obj); + return; + } + + kfree(obj); +} + +static void __free_rcu(struct rcu_head *head) +{ + struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu); + struct llist_node *llnode = llist_del_all(&c->waiting_for_gp); + struct llist_node *pos, *t; + + llist_for_each_safe(pos, t, llnode) + free_one(c, pos); + atomic_set(&c->call_rcu_in_progress, 0); +} + +static void __free_rcu_tasks_trace(struct rcu_head *head) +{ + struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu); + + call_rcu(&c->rcu, __free_rcu); +} + +static void enque_to_free(struct bpf_mem_cache *c, void *obj) +{ + struct llist_node *llnode = obj; + + /* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work. + * Nothing races to add to free_by_rcu list. + */ + __llist_add(llnode, &c->free_by_rcu); +} + +static void do_call_rcu(struct bpf_mem_cache *c) +{ + struct llist_node *llnode, *t; + + if (atomic_xchg(&c->call_rcu_in_progress, 1)) + return; + + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp)); + llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu)) + /* There is no concurrent __llist_add(waiting_for_gp) access. + * It doesn't race with llist_del_all either. + * But there could be two concurrent llist_del_all(waiting_for_gp): + * from __free_rcu() and from drain_mem_cache(). + */ + __llist_add(llnode, &c->waiting_for_gp); + /* Use call_rcu_tasks_trace() to wait for sleepable progs to finish. + * Then use call_rcu() to wait for normal progs to finish + * and finally do free_one() on each element. + */ + call_rcu_tasks_trace(&c->rcu, __free_rcu_tasks_trace); +} + +static void free_bulk(struct bpf_mem_cache *c) +{ + struct llist_node *llnode, *t; + unsigned long flags; + int cnt; + + do { + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_save(flags); + WARN_ON_ONCE(local_inc_return(&c->active) != 1); + llnode = __llist_del_first(&c->free_llist); + if (llnode) + cnt = --c->free_cnt; + else + cnt = 0; + local_dec(&c->active); + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + local_irq_restore(flags); + if (llnode) + enque_to_free(c, llnode); + } while (cnt > (c->high_watermark + c->low_watermark) / 2); + + /* and drain free_llist_extra */ + llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) + enque_to_free(c, llnode); + do_call_rcu(c); +} + +static void bpf_mem_refill(struct irq_work *work) +{ + struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work); + int cnt; + + /* Racy access to free_cnt. It doesn't need to be 100% accurate */ + cnt = c->free_cnt; + if (cnt < c->low_watermark) + /* irq_work runs on this cpu and kmalloc will allocate + * from the current numa node which is what we want here. + */ + alloc_bulk(c, c->batch, NUMA_NO_NODE); + else if (cnt > c->high_watermark) + free_bulk(c); +} + +static void notrace irq_work_raise(struct bpf_mem_cache *c) +{ + irq_work_queue(&c->refill_work); +} + +/* For typical bpf map case that uses bpf_mem_cache_alloc and single bucket + * the freelist cache will be elem_size * 64 (or less) on each cpu. + * + * For bpf programs that don't have statically known allocation sizes and + * assuming (low_mark + high_mark) / 2 as an average number of elements per + * bucket and all buckets are used the total amount of memory in freelists + * on each cpu will be: + * 64*16 + 64*32 + 64*64 + 64*96 + 64*128 + 64*196 + 64*256 + 32*512 + 16*1024 + 8*2048 + 4*4096 + * == ~ 116 Kbyte using below heuristic. + * Initialized, but unused bpf allocator (not bpf map specific one) will + * consume ~ 11 Kbyte per cpu. + * Typical case will be between 11K and 116K closer to 11K. + * bpf progs can and should share bpf_mem_cache when possible. + */ + +static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu) +{ + init_irq_work(&c->refill_work, bpf_mem_refill); + if (c->unit_size <= 256) { + c->low_watermark = 32; + c->high_watermark = 96; + } else { + /* When page_size == 4k, order-0 cache will have low_mark == 2 + * and high_mark == 6 with batch alloc of 3 individual pages at + * a time. + * 8k allocs and above low == 1, high == 3, batch == 1. + */ + c->low_watermark = max(32 * 256 / c->unit_size, 1); + c->high_watermark = max(96 * 256 / c->unit_size, 3); + } + c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1); + + /* To avoid consuming memory assume that 1st run of bpf + * prog won't be doing more than 4 map_update_elem from + * irq disabled region + */ + alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu)); +} + +/* When size != 0 bpf_mem_cache for each cpu. + * This is typical bpf hash map use case when all elements have equal size. + * + * When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on + * kmalloc/kfree. Max allocation size is 4096 in this case. + * This is bpf_dynptr and bpf_kptr use case. + */ +int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu) +{ + static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096}; + struct bpf_mem_caches *cc, __percpu *pcc; + struct bpf_mem_cache *c, __percpu *pc; + struct obj_cgroup *objcg = NULL; + int cpu, i, unit_size, percpu_size = 0; + + if (size) { + pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL); + if (!pc) + return -ENOMEM; + + if (percpu) + /* room for llist_node and per-cpu pointer */ + percpu_size = LLIST_NODE_SZ + sizeof(void *); + else + size += LLIST_NODE_SZ; /* room for llist_node */ + unit_size = size; + +#ifdef CONFIG_MEMCG_KMEM + objcg = get_obj_cgroup_from_current(); +#endif + for_each_possible_cpu(cpu) { + c = per_cpu_ptr(pc, cpu); + c->unit_size = unit_size; + c->objcg = objcg; + c->percpu_size = percpu_size; + prefill_mem_cache(c, cpu); + } + ma->cache = pc; + return 0; + } + + /* size == 0 && percpu is an invalid combination */ + if (WARN_ON_ONCE(percpu)) + return -EINVAL; + + pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL); + if (!pcc) + return -ENOMEM; +#ifdef CONFIG_MEMCG_KMEM + objcg = get_obj_cgroup_from_current(); +#endif + for_each_possible_cpu(cpu) { + cc = per_cpu_ptr(pcc, cpu); + for (i = 0; i < NUM_CACHES; i++) { + c = &cc->cache[i]; + c->unit_size = sizes[i]; + c->objcg = objcg; + prefill_mem_cache(c, cpu); + } + } + ma->caches = pcc; + return 0; +} + +static void drain_mem_cache(struct bpf_mem_cache *c) +{ + struct llist_node *llnode, *t; + + /* No progs are using this bpf_mem_cache, but htab_map_free() called + * bpf_mem_cache_free() for all remaining elements and they can be in + * free_by_rcu or in waiting_for_gp lists, so drain those lists now. + */ + llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu)) + free_one(c, llnode); + llist_for_each_safe(llnode, t, llist_del_all(&c->waiting_for_gp)) + free_one(c, llnode); + llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist)) + free_one(c, llnode); + llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) + free_one(c, llnode); +} + +static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma) +{ + free_percpu(ma->cache); + free_percpu(ma->caches); + ma->cache = NULL; + ma->caches = NULL; +} + +static void free_mem_alloc(struct bpf_mem_alloc *ma) +{ + /* waiting_for_gp lists was drained, but __free_rcu might + * still execute. Wait for it now before we freeing percpu caches. + */ + rcu_barrier_tasks_trace(); + rcu_barrier(); + free_mem_alloc_no_barrier(ma); +} + +static void free_mem_alloc_deferred(struct work_struct *work) +{ + struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work); + + free_mem_alloc(ma); + kfree(ma); +} + +static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress) +{ + struct bpf_mem_alloc *copy; + + if (!rcu_in_progress) { + /* Fast path. No callbacks are pending, hence no need to do + * rcu_barrier-s. + */ + free_mem_alloc_no_barrier(ma); + return; + } + + copy = kmalloc(sizeof(*ma), GFP_KERNEL); + if (!copy) { + /* Slow path with inline barrier-s */ + free_mem_alloc(ma); + return; + } + + /* Defer barriers into worker to let the rest of map memory to be freed */ + copy->cache = ma->cache; + ma->cache = NULL; + copy->caches = ma->caches; + ma->caches = NULL; + INIT_WORK(©->work, free_mem_alloc_deferred); + queue_work(system_unbound_wq, ©->work); +} + +void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) +{ + struct bpf_mem_caches *cc; + struct bpf_mem_cache *c; + int cpu, i, rcu_in_progress; + + if (ma->cache) { + rcu_in_progress = 0; + for_each_possible_cpu(cpu) { + c = per_cpu_ptr(ma->cache, cpu); + drain_mem_cache(c); + rcu_in_progress += atomic_read(&c->call_rcu_in_progress); + } + /* objcg is the same across cpus */ + if (c->objcg) + obj_cgroup_put(c->objcg); + destroy_mem_alloc(ma, rcu_in_progress); + } + if (ma->caches) { + rcu_in_progress = 0; + for_each_possible_cpu(cpu) { + cc = per_cpu_ptr(ma->caches, cpu); + for (i = 0; i < NUM_CACHES; i++) { + c = &cc->cache[i]; + drain_mem_cache(c); + rcu_in_progress += atomic_read(&c->call_rcu_in_progress); + } + } + if (c->objcg) + obj_cgroup_put(c->objcg); + destroy_mem_alloc(ma, rcu_in_progress); + } +} + +/* notrace is necessary here and in other functions to make sure + * bpf programs cannot attach to them and cause llist corruptions. + */ +static void notrace *unit_alloc(struct bpf_mem_cache *c) +{ + struct llist_node *llnode = NULL; + unsigned long flags; + int cnt = 0; + + /* Disable irqs to prevent the following race for majority of prog types: + * prog_A + * bpf_mem_alloc + * preemption or irq -> prog_B + * bpf_mem_alloc + * + * but prog_B could be a perf_event NMI prog. + * Use per-cpu 'active' counter to order free_list access between + * unit_alloc/unit_free/bpf_mem_refill. + */ + local_irq_save(flags); + if (local_inc_return(&c->active) == 1) { + llnode = __llist_del_first(&c->free_llist); + if (llnode) + cnt = --c->free_cnt; + } + local_dec(&c->active); + local_irq_restore(flags); + + WARN_ON(cnt < 0); + + if (cnt < c->low_watermark) + irq_work_raise(c); + return llnode; +} + +/* Though 'ptr' object could have been allocated on a different cpu + * add it to the free_llist of the current cpu. + * Let kfree() logic deal with it when it's later called from irq_work. + */ +static void notrace unit_free(struct bpf_mem_cache *c, void *ptr) +{ + struct llist_node *llnode = ptr - LLIST_NODE_SZ; + unsigned long flags; + int cnt = 0; + + BUILD_BUG_ON(LLIST_NODE_SZ > 8); + + local_irq_save(flags); + if (local_inc_return(&c->active) == 1) { + __llist_add(llnode, &c->free_llist); + cnt = ++c->free_cnt; + } else { + /* unit_free() cannot fail. Therefore add an object to atomic + * llist. free_bulk() will drain it. Though free_llist_extra is + * a per-cpu list we have to use atomic llist_add here, since + * it also can be interrupted by bpf nmi prog that does another + * unit_free() into the same free_llist_extra. + */ + llist_add(llnode, &c->free_llist_extra); + } + local_dec(&c->active); + local_irq_restore(flags); + + if (cnt > c->high_watermark) + /* free few objects from current cpu into global kmalloc pool */ + irq_work_raise(c); +} + +/* Called from BPF program or from sys_bpf syscall. + * In both cases migration is disabled. + */ +void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size) +{ + int idx; + void *ret; + + if (!size) + return ZERO_SIZE_PTR; + + idx = bpf_mem_cache_idx(size + LLIST_NODE_SZ); + if (idx < 0) + return NULL; + + ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx); + return !ret ? NULL : ret + LLIST_NODE_SZ; +} + +void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr) +{ + int idx; + + if (!ptr) + return; + + idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ)); + if (idx < 0) + return; + + unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr); +} + +void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma) +{ + void *ret; + + ret = unit_alloc(this_cpu_ptr(ma->cache)); + return !ret ? NULL : ret + LLIST_NODE_SZ; +} + +void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr) +{ + if (!ptr) + return; + + unit_free(this_cpu_ptr(ma->cache), ptr); +} diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index bd09290e364844a35a3c80bc56a63037356c9966..13e4efc971e6d2d84fe62d9196a1f60b9d4b0f9d 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -372,7 +372,7 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) attr->map_type != BPF_MAP_TYPE_HASH) return ERR_PTR(-EINVAL); - offmap = kzalloc(sizeof(*offmap), GFP_USER); + offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE); if (!offmap) return ERR_PTR(-ENOMEM); @@ -404,7 +404,7 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) err_unlock: up_write(&bpf_devs_lock); rtnl_unlock(); - kfree(offmap); + bpf_map_area_free(offmap); return ERR_PTR(err); } @@ -428,7 +428,7 @@ void bpf_map_offload_map_free(struct bpf_map *map) up_write(&bpf_devs_lock); rtnl_unlock(); - kfree(offmap); + bpf_map_area_free(offmap); } int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value) diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c index 00b874c8e889f1a019d553fbca2bff34c0527864..b6e7f5c5b9ab5712783b3c7875f6d78994daa038 100644 --- a/kernel/bpf/percpu_freelist.c +++ b/kernel/bpf/percpu_freelist.c @@ -58,23 +58,21 @@ static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s, { int cpu, orig_cpu; - orig_cpu = cpu = raw_smp_processor_id(); + orig_cpu = raw_smp_processor_id(); while (1) { - struct pcpu_freelist_head *head; + for_each_cpu_wrap(cpu, cpu_possible_mask, orig_cpu) { + struct pcpu_freelist_head *head; - head = per_cpu_ptr(s->freelist, cpu); - if (raw_spin_trylock(&head->lock)) { - pcpu_freelist_push_node(head, node); - raw_spin_unlock(&head->lock); - return; + head = per_cpu_ptr(s->freelist, cpu); + if (raw_spin_trylock(&head->lock)) { + pcpu_freelist_push_node(head, node); + raw_spin_unlock(&head->lock); + return; + } } - cpu = cpumask_next(cpu, cpu_possible_mask); - if (cpu >= nr_cpu_ids) - cpu = 0; /* cannot lock any per cpu lock, try extralist */ - if (cpu == orig_cpu && - pcpu_freelist_try_push_extra(s, node)) + if (pcpu_freelist_try_push_extra(s, node)) return; } } @@ -125,13 +123,12 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) { struct pcpu_freelist_head *head; struct pcpu_freelist_node *node; - int orig_cpu, cpu; + int cpu; - orig_cpu = cpu = raw_smp_processor_id(); - while (1) { + for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { head = per_cpu_ptr(s->freelist, cpu); if (!READ_ONCE(head->first)) - goto next_cpu; + continue; raw_spin_lock(&head->lock); node = head->first; if (node) { @@ -140,12 +137,6 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) return node; } raw_spin_unlock(&head->lock); -next_cpu: - cpu = cpumask_next(cpu, cpu_possible_mask); - if (cpu >= nr_cpu_ids) - cpu = 0; - if (cpu == orig_cpu) - break; } /* per cpu lists are all empty, try extralist */ @@ -164,13 +155,12 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s) { struct pcpu_freelist_head *head; struct pcpu_freelist_node *node; - int orig_cpu, cpu; + int cpu; - orig_cpu = cpu = raw_smp_processor_id(); - while (1) { + for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { head = per_cpu_ptr(s->freelist, cpu); if (!READ_ONCE(head->first)) - goto next_cpu; + continue; if (raw_spin_trylock(&head->lock)) { node = head->first; if (node) { @@ -180,12 +170,6 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s) } raw_spin_unlock(&head->lock); } -next_cpu: - cpu = cpumask_next(cpu, cpu_possible_mask); - if (cpu >= nr_cpu_ids) - cpu = 0; - if (cpu == orig_cpu) - break; } /* cannot pop from per cpu lists, try extralist */ diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index a1c0794ae49def65d7e25d9038b728752c5f9828..8a5e060de63bc834bea5efea1434901ff62a66cc 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -78,8 +78,6 @@ static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr) if (!qs) return ERR_PTR(-ENOMEM); - memset(qs, 0, sizeof(*qs)); - bpf_map_init_from_attr(&qs->map, attr); qs->size = size; diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index ded4faeca19225d49bb37b78817a7fc147aefe33..9e832acf4692574d5e7d774ce4b7b3b64610cbf2 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -38,10 +38,43 @@ struct bpf_ringbuf { struct page **pages; int nr_pages; spinlock_t spinlock ____cacheline_aligned_in_smp; - /* Consumer and producer counters are put into separate pages to allow - * mapping consumer page as r/w, but restrict producer page to r/o. - * This protects producer position from being modified by user-space - * application and ruining in-kernel position tracking. + /* For user-space producer ring buffers, an atomic_t busy bit is used + * to synchronize access to the ring buffers in the kernel, rather than + * the spinlock that is used for kernel-producer ring buffers. This is + * done because the ring buffer must hold a lock across a BPF program's + * callback: + * + * __bpf_user_ringbuf_peek() // lock acquired + * -> program callback_fn() + * -> __bpf_user_ringbuf_sample_release() // lock released + * + * It is unsafe and incorrect to hold an IRQ spinlock across what could + * be a long execution window, so we instead simply disallow concurrent + * access to the ring buffer by kernel consumers, and return -EBUSY from + * __bpf_user_ringbuf_peek() if the busy bit is held by another task. + */ + atomic_t busy ____cacheline_aligned_in_smp; + /* Consumer and producer counters are put into separate pages to + * allow each position to be mapped with different permissions. + * This prevents a user-space application from modifying the + * position and ruining in-kernel tracking. The permissions of the + * pages depend on who is producing samples: user-space or the + * kernel. + * + * Kernel-producer + * --------------- + * The producer position and data pages are mapped as r/o in + * userspace. For this approach, bits in the header of samples are + * used to signal to user-space, and to other producers, whether a + * sample is currently being written. + * + * User-space producer + * ------------------- + * Only the page containing the consumer position is mapped r/o in + * user-space. User-space producers also use bits of the header to + * communicate to the kernel, but the kernel must carefully check and + * validate each sample to ensure that they're correctly formatted, and + * fully contained within the ring buffer. */ unsigned long consumer_pos __aligned(PAGE_SIZE); unsigned long producer_pos __aligned(PAGE_SIZE); @@ -116,7 +149,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node) err_free_pages: for (i = 0; i < nr_pages; i++) __free_page(pages[i]); - kvfree(pages); + bpf_map_area_free(pages); return NULL; } @@ -136,6 +169,7 @@ static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node) return NULL; spin_lock_init(&rb->spinlock); + atomic_set(&rb->busy, 0); init_waitqueue_head(&rb->waitq); init_irq_work(&rb->work, bpf_ringbuf_notify); @@ -164,7 +198,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr) return ERR_PTR(-E2BIG); #endif - rb_map = kzalloc(sizeof(*rb_map), GFP_USER | __GFP_ACCOUNT); + rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE); if (!rb_map) return ERR_PTR(-ENOMEM); @@ -172,7 +206,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr) rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node); if (!rb_map->rb) { - kfree(rb_map); + bpf_map_area_free(rb_map); return ERR_PTR(-ENOMEM); } @@ -190,7 +224,7 @@ static void bpf_ringbuf_free(struct bpf_ringbuf *rb) vunmap(rb); for (i = 0; i < nr_pages; i++) __free_page(pages[i]); - kvfree(pages); + bpf_map_area_free(pages); } static void ringbuf_map_free(struct bpf_map *map) @@ -199,7 +233,7 @@ static void ringbuf_map_free(struct bpf_map *map) rb_map = container_of(map, struct bpf_ringbuf_map, map); bpf_ringbuf_free(rb_map->rb); - kfree(rb_map); + bpf_map_area_free(rb_map); } static void *ringbuf_map_lookup_elem(struct bpf_map *map, void *key) @@ -224,7 +258,7 @@ static int ringbuf_map_get_next_key(struct bpf_map *map, void *key, return -ENOTSUPP; } -static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) +static int ringbuf_map_mmap_kern(struct bpf_map *map, struct vm_area_struct *vma) { struct bpf_ringbuf_map *rb_map; @@ -242,6 +276,26 @@ static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) vma->vm_pgoff + RINGBUF_PGOFF); } +static int ringbuf_map_mmap_user(struct bpf_map *map, struct vm_area_struct *vma) +{ + struct bpf_ringbuf_map *rb_map; + + rb_map = container_of(map, struct bpf_ringbuf_map, map); + + if (vma->vm_flags & VM_WRITE) { + if (vma->vm_pgoff == 0) + /* Disallow writable mappings to the consumer pointer, + * and allow writable mappings to both the producer + * position, and the ring buffer data itself. + */ + return -EPERM; + } else { + vma->vm_flags &= ~VM_MAYWRITE; + } + /* remap_vmalloc_range() checks size and offset constraints */ + return remap_vmalloc_range(vma, rb_map->rb, vma->vm_pgoff + RINGBUF_PGOFF); +} + static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) { unsigned long cons_pos, prod_pos; @@ -251,8 +305,13 @@ static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb) return prod_pos - cons_pos; } -static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp, - struct poll_table_struct *pts) +static u32 ringbuf_total_data_sz(const struct bpf_ringbuf *rb) +{ + return rb->mask + 1; +} + +static __poll_t ringbuf_map_poll_kern(struct bpf_map *map, struct file *filp, + struct poll_table_struct *pts) { struct bpf_ringbuf_map *rb_map; @@ -264,13 +323,26 @@ static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp, return 0; } +static __poll_t ringbuf_map_poll_user(struct bpf_map *map, struct file *filp, + struct poll_table_struct *pts) +{ + struct bpf_ringbuf_map *rb_map; + + rb_map = container_of(map, struct bpf_ringbuf_map, map); + poll_wait(filp, &rb_map->rb->waitq, pts); + + if (ringbuf_avail_data_sz(rb_map->rb) < ringbuf_total_data_sz(rb_map->rb)) + return EPOLLOUT | EPOLLWRNORM; + return 0; +} + BTF_ID_LIST_SINGLE(ringbuf_map_btf_ids, struct, bpf_ringbuf_map) const struct bpf_map_ops ringbuf_map_ops = { .map_meta_equal = bpf_map_meta_equal, .map_alloc = ringbuf_map_alloc, .map_free = ringbuf_map_free, - .map_mmap = ringbuf_map_mmap, - .map_poll = ringbuf_map_poll, + .map_mmap = ringbuf_map_mmap_kern, + .map_poll = ringbuf_map_poll_kern, .map_lookup_elem = ringbuf_map_lookup_elem, .map_update_elem = ringbuf_map_update_elem, .map_delete_elem = ringbuf_map_delete_elem, @@ -278,6 +350,20 @@ const struct bpf_map_ops ringbuf_map_ops = { .map_btf_id = &ringbuf_map_btf_ids[0], }; +BTF_ID_LIST_SINGLE(user_ringbuf_map_btf_ids, struct, bpf_ringbuf_map) +const struct bpf_map_ops user_ringbuf_map_ops = { + .map_meta_equal = bpf_map_meta_equal, + .map_alloc = ringbuf_map_alloc, + .map_free = ringbuf_map_free, + .map_mmap = ringbuf_map_mmap_user, + .map_poll = ringbuf_map_poll_user, + .map_lookup_elem = ringbuf_map_lookup_elem, + .map_update_elem = ringbuf_map_update_elem, + .map_delete_elem = ringbuf_map_delete_elem, + .map_get_next_key = ringbuf_map_get_next_key, + .map_btf_id = &user_ringbuf_map_btf_ids[0], +}; + /* Given pointer to ring buffer record metadata and struct bpf_ringbuf itself, * calculate offset from record metadata to ring buffer in pages, rounded * down. This page offset is stored as part of record metadata and allows to @@ -312,7 +398,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) return NULL; len = round_up(size + BPF_RINGBUF_HDR_SZ, 8); - if (len > rb->mask + 1) + if (len > ringbuf_total_data_sz(rb)) return NULL; cons_pos = smp_load_acquire(&rb->consumer_pos); @@ -459,7 +545,7 @@ BPF_CALL_2(bpf_ringbuf_query, struct bpf_map *, map, u64, flags) case BPF_RB_AVAIL_DATA: return ringbuf_avail_data_sz(rb); case BPF_RB_RING_SIZE: - return rb->mask + 1; + return ringbuf_total_data_sz(rb); case BPF_RB_CONS_POS: return smp_load_acquire(&rb->consumer_pos); case BPF_RB_PROD_POS: @@ -553,3 +639,138 @@ const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = { .arg1_type = ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE, .arg2_type = ARG_ANYTHING, }; + +static int __bpf_user_ringbuf_peek(struct bpf_ringbuf *rb, void **sample, u32 *size) +{ + int err; + u32 hdr_len, sample_len, total_len, flags, *hdr; + u64 cons_pos, prod_pos; + + /* Synchronizes with smp_store_release() in user-space producer. */ + prod_pos = smp_load_acquire(&rb->producer_pos); + if (prod_pos % 8) + return -EINVAL; + + /* Synchronizes with smp_store_release() in __bpf_user_ringbuf_sample_release() */ + cons_pos = smp_load_acquire(&rb->consumer_pos); + if (cons_pos >= prod_pos) + return -ENODATA; + + hdr = (u32 *)((uintptr_t)rb->data + (uintptr_t)(cons_pos & rb->mask)); + /* Synchronizes with smp_store_release() in user-space producer. */ + hdr_len = smp_load_acquire(hdr); + flags = hdr_len & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT); + sample_len = hdr_len & ~flags; + total_len = round_up(sample_len + BPF_RINGBUF_HDR_SZ, 8); + + /* The sample must fit within the region advertised by the producer position. */ + if (total_len > prod_pos - cons_pos) + return -EINVAL; + + /* The sample must fit within the data region of the ring buffer. */ + if (total_len > ringbuf_total_data_sz(rb)) + return -E2BIG; + + /* The sample must fit into a struct bpf_dynptr. */ + err = bpf_dynptr_check_size(sample_len); + if (err) + return -E2BIG; + + if (flags & BPF_RINGBUF_DISCARD_BIT) { + /* If the discard bit is set, the sample should be skipped. + * + * Update the consumer pos, and return -EAGAIN so the caller + * knows to skip this sample and try to read the next one. + */ + smp_store_release(&rb->consumer_pos, cons_pos + total_len); + return -EAGAIN; + } + + if (flags & BPF_RINGBUF_BUSY_BIT) + return -ENODATA; + + *sample = (void *)((uintptr_t)rb->data + + (uintptr_t)((cons_pos + BPF_RINGBUF_HDR_SZ) & rb->mask)); + *size = sample_len; + return 0; +} + +static void __bpf_user_ringbuf_sample_release(struct bpf_ringbuf *rb, size_t size, u64 flags) +{ + u64 consumer_pos; + u32 rounded_size = round_up(size + BPF_RINGBUF_HDR_SZ, 8); + + /* Using smp_load_acquire() is unnecessary here, as the busy-bit + * prevents another task from writing to consumer_pos after it was read + * by this task with smp_load_acquire() in __bpf_user_ringbuf_peek(). + */ + consumer_pos = rb->consumer_pos; + /* Synchronizes with smp_load_acquire() in user-space producer. */ + smp_store_release(&rb->consumer_pos, consumer_pos + rounded_size); +} + +BPF_CALL_4(bpf_user_ringbuf_drain, struct bpf_map *, map, + void *, callback_fn, void *, callback_ctx, u64, flags) +{ + struct bpf_ringbuf *rb; + long samples, discarded_samples = 0, ret = 0; + bpf_callback_t callback = (bpf_callback_t)callback_fn; + u64 wakeup_flags = BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP; + int busy = 0; + + if (unlikely(flags & ~wakeup_flags)) + return -EINVAL; + + rb = container_of(map, struct bpf_ringbuf_map, map)->rb; + + /* If another consumer is already consuming a sample, wait for them to finish. */ + if (!atomic_try_cmpxchg(&rb->busy, &busy, 1)) + return -EBUSY; + + for (samples = 0; samples < BPF_MAX_USER_RINGBUF_SAMPLES && ret == 0; samples++) { + int err; + u32 size; + void *sample; + struct bpf_dynptr_kern dynptr; + + err = __bpf_user_ringbuf_peek(rb, &sample, &size); + if (err) { + if (err == -ENODATA) { + break; + } else if (err == -EAGAIN) { + discarded_samples++; + continue; + } else { + ret = err; + goto schedule_work_return; + } + } + + bpf_dynptr_init(&dynptr, sample, BPF_DYNPTR_TYPE_LOCAL, 0, size); + ret = callback((uintptr_t)&dynptr, (uintptr_t)callback_ctx, 0, 0, 0); + __bpf_user_ringbuf_sample_release(rb, size, flags); + } + ret = samples - discarded_samples; + +schedule_work_return: + /* Prevent the clearing of the busy-bit from being reordered before the + * storing of any rb consumer or producer positions. + */ + smp_mb__before_atomic(); + atomic_set(&rb->busy, 0); + + if (flags & BPF_RB_FORCE_WAKEUP) + irq_work_queue(&rb->work); + else if (!(flags & BPF_RB_NO_WAKEUP) && samples > 0) + irq_work_queue(&rb->work); + return ret; +} + +const struct bpf_func_proto bpf_user_ringbuf_drain_proto = { + .func = bpf_user_ringbuf_drain, + .ret_type = RET_INTEGER, + .arg1_type = ARG_CONST_MAP_PTR, + .arg2_type = ARG_PTR_TO_FUNC, + .arg3_type = ARG_PTR_TO_STACK_OR_NULL, + .arg4_type = ARG_ANYTHING, +}; diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 1adbe67cdb95e1d18d084a57eb61f30991cb63a4..aecea7451b6103785203e32a832f2e9590b4abae 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -338,7 +338,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, int ret; /* perf_sample_data doesn't have callchain, use bpf_get_stackid */ - if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) + if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) return bpf_get_stackid((unsigned long)(ctx->regs), (unsigned long) map, flags, 0, 0); @@ -506,7 +506,7 @@ BPF_CALL_4(bpf_get_stack_pe, struct bpf_perf_event_data_kern *, ctx, int err = -EINVAL; __u64 nr_kernel; - if (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) + if (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)) return __bpf_get_stack(regs, NULL, NULL, buf, size, flags); if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a4d40d98428a3ba220cce2c4c4f313eed66b73b7..7b373a5e861f445ee5d73dad224fc5b38066ecfc 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -598,7 +598,7 @@ void bpf_map_free_kptrs(struct bpf_map *map, void *map_value) if (off_desc->type == BPF_KPTR_UNREF) { u64 *p = (u64 *)btf_id_ptr; - WRITE_ONCE(p, 0); + WRITE_ONCE(*p, 0); continue; } old_ptr = xchg(btf_id_ptr, 0); @@ -638,7 +638,10 @@ static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock) bpf_map_free_id(map, do_idr_lock); btf_put(map->btf); INIT_WORK(&map->work, bpf_map_free_deferred); - schedule_work(&map->work); + /* Avoid spawning kworkers, since they all might contend + * for the same mutex like slab_mutex. + */ + queue_work(system_unbound_wq, &map->work); } } @@ -1046,7 +1049,8 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf, } if (map->map_type != BPF_MAP_TYPE_HASH && map->map_type != BPF_MAP_TYPE_LRU_HASH && - map->map_type != BPF_MAP_TYPE_ARRAY) { + map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) { ret = -EOPNOTSUPP; goto free_map_tab; } @@ -1413,19 +1417,14 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map); - - err = -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); - if (!value) + value = kvmemdup_bpfptr(uvalue, value_size); + if (IS_ERR(value)) { + err = PTR_ERR(value); goto free_key; - - err = -EFAULT; - if (copy_from_bpfptr(value, uvalue, value_size) != 0) - goto free_value; + } err = bpf_map_update_value(map, f, key, value, attr->flags); -free_value: kvfree(value); free_key: kvfree(key); @@ -1437,9 +1436,9 @@ err_put: #define BPF_MAP_DELETE_ELEM_LAST_FIELD key -static int map_delete_elem(union bpf_attr *attr) +static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr) { - void __user *ukey = u64_to_user_ptr(attr->key); + bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); int ufd = attr->map_fd; struct bpf_map *map; struct fd f; @@ -1459,7 +1458,7 @@ static int map_delete_elem(union bpf_attr *attr) goto err_put; } - key = __bpf_copy_key(ukey, map->key_size); + key = ___bpf_copy_key(ukey, map->key_size); if (IS_ERR(key)) { err = PTR_ERR(key); goto err_put; @@ -2094,6 +2093,17 @@ struct bpf_prog_kstats { u64 misses; }; +void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog) +{ + struct bpf_prog_stats *stats; + unsigned int flags; + + stats = this_cpu_ptr(prog->stats); + flags = u64_stats_update_begin_irqsave(&stats->syncp); + u64_stats_inc(&stats->misses); + u64_stats_update_end_irqrestore(&stats->syncp, flags); +} + static void bpf_prog_get_stats(const struct bpf_prog *prog, struct bpf_prog_kstats *stats) { @@ -4395,7 +4405,9 @@ static int bpf_task_fd_query(const union bpf_attr *attr, if (attr->task_fd_query.flags != 0) return -EINVAL; + rcu_read_lock(); task = get_pid_task(find_vpid(pid), PIDTYPE_PID); + rcu_read_unlock(); if (!task) return -ENOENT; @@ -4941,7 +4953,7 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size) err = map_update_elem(&attr, uattr); break; case BPF_MAP_DELETE_ELEM: - err = map_delete_elem(&attr); + err = map_delete_elem(&attr, uattr); break; case BPF_MAP_GET_NEXT_KEY: err = map_get_next_key(&attr); @@ -5073,8 +5085,10 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size) { switch (cmd) { case BPF_MAP_CREATE: + case BPF_MAP_DELETE_ELEM: case BPF_MAP_UPDATE_ELEM: case BPF_MAP_FREEZE: + case BPF_MAP_GET_FD_BY_ID: case BPF_PROG_LOAD: case BPF_BTF_LOAD: case BPF_LINK_CREATE: @@ -5197,7 +5211,7 @@ syscall_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { switch (func_id) { case BPF_FUNC_sys_bpf: - return &bpf_sys_bpf_proto; + return !perfmon_capable() ? NULL : &bpf_sys_bpf_proto; case BPF_FUNC_btf_find_by_name_kind: return &bpf_btf_find_by_name_kind_proto; case BPF_FUNC_sys_close: diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 8c921799def4927dd09d2a799d05c9d4b87fba46..c2a2182ce570219bad9cc5362e15435d4873b6ba 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -10,8 +10,17 @@ #include #include "mmap_unlock_work.h" +static const char * const iter_task_type_names[] = { + "ALL", + "TID", + "PID", +}; + struct bpf_iter_seq_task_common { struct pid_namespace *ns; + enum bpf_iter_task_type type; + u32 pid; + u32 pid_visiting; }; struct bpf_iter_seq_task_info { @@ -22,18 +31,115 @@ struct bpf_iter_seq_task_info { u32 tid; }; -static struct task_struct *task_seq_get_next(struct pid_namespace *ns, +static struct task_struct *task_group_seq_get_next(struct bpf_iter_seq_task_common *common, + u32 *tid, + bool skip_if_dup_files) +{ + struct task_struct *task, *next_task; + struct pid *pid; + u32 saved_tid; + + if (!*tid) { + /* The first time, the iterator calls this function. */ + pid = find_pid_ns(common->pid, common->ns); + if (!pid) + return NULL; + + task = get_pid_task(pid, PIDTYPE_TGID); + if (!task) + return NULL; + + *tid = common->pid; + common->pid_visiting = common->pid; + + return task; + } + + /* If the control returns to user space and comes back to the + * kernel again, *tid and common->pid_visiting should be the + * same for task_seq_start() to pick up the correct task. + */ + if (*tid == common->pid_visiting) { + pid = find_pid_ns(common->pid_visiting, common->ns); + task = get_pid_task(pid, PIDTYPE_PID); + + return task; + } + + pid = find_pid_ns(common->pid_visiting, common->ns); + if (!pid) + return NULL; + + task = get_pid_task(pid, PIDTYPE_PID); + if (!task) + return NULL; + +retry: + if (!pid_alive(task)) { + put_task_struct(task); + return NULL; + } + + next_task = next_thread(task); + put_task_struct(task); + if (!next_task) + return NULL; + + saved_tid = *tid; + *tid = __task_pid_nr_ns(next_task, PIDTYPE_PID, common->ns); + if (!*tid || *tid == common->pid) { + /* Run out of tasks of a process. The tasks of a + * thread_group are linked as circular linked list. + */ + *tid = saved_tid; + return NULL; + } + + get_task_struct(next_task); + common->pid_visiting = *tid; + + if (skip_if_dup_files && task->files == task->group_leader->files) { + task = next_task; + goto retry; + } + + return next_task; +} + +static struct task_struct *task_seq_get_next(struct bpf_iter_seq_task_common *common, u32 *tid, bool skip_if_dup_files) { struct task_struct *task = NULL; struct pid *pid; + if (common->type == BPF_TASK_ITER_TID) { + if (*tid && *tid != common->pid) + return NULL; + rcu_read_lock(); + pid = find_pid_ns(common->pid, common->ns); + if (pid) { + task = get_pid_task(pid, PIDTYPE_TGID); + *tid = common->pid; + } + rcu_read_unlock(); + + return task; + } + + if (common->type == BPF_TASK_ITER_TGID) { + rcu_read_lock(); + task = task_group_seq_get_next(common, tid, skip_if_dup_files); + rcu_read_unlock(); + + return task; + } + rcu_read_lock(); retry: - pid = find_ge_pid(*tid, ns); + pid = find_ge_pid(*tid, common->ns); if (pid) { - *tid = pid_nr_ns(pid, ns); + *tid = pid_nr_ns(pid, common->ns); task = get_pid_task(pid, PIDTYPE_PID); if (!task) { ++*tid; @@ -56,7 +162,7 @@ static void *task_seq_start(struct seq_file *seq, loff_t *pos) struct bpf_iter_seq_task_info *info = seq->private; struct task_struct *task; - task = task_seq_get_next(info->common.ns, &info->tid, false); + task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; @@ -73,7 +179,7 @@ static void *task_seq_next(struct seq_file *seq, void *v, loff_t *pos) ++*pos; ++info->tid; put_task_struct((struct task_struct *)v); - task = task_seq_get_next(info->common.ns, &info->tid, false); + task = task_seq_get_next(&info->common, &info->tid, false); if (!task) return NULL; @@ -117,6 +223,41 @@ static void task_seq_stop(struct seq_file *seq, void *v) put_task_struct((struct task_struct *)v); } +static int bpf_iter_attach_task(struct bpf_prog *prog, + union bpf_iter_link_info *linfo, + struct bpf_iter_aux_info *aux) +{ + unsigned int flags; + struct pid *pid; + pid_t tgid; + + if ((!!linfo->task.tid + !!linfo->task.pid + !!linfo->task.pid_fd) > 1) + return -EINVAL; + + aux->task.type = BPF_TASK_ITER_ALL; + if (linfo->task.tid != 0) { + aux->task.type = BPF_TASK_ITER_TID; + aux->task.pid = linfo->task.tid; + } + if (linfo->task.pid != 0) { + aux->task.type = BPF_TASK_ITER_TGID; + aux->task.pid = linfo->task.pid; + } + if (linfo->task.pid_fd != 0) { + aux->task.type = BPF_TASK_ITER_TGID; + + pid = pidfd_get_pid(linfo->task.pid_fd, &flags); + if (IS_ERR(pid)) + return PTR_ERR(pid); + + tgid = pid_nr_ns(pid, task_active_pid_ns(current)); + aux->task.pid = tgid; + put_pid(pid); + } + + return 0; +} + static const struct seq_operations task_seq_ops = { .start = task_seq_start, .next = task_seq_next, @@ -137,8 +278,7 @@ struct bpf_iter_seq_task_file_info { static struct file * task_file_seq_get_next(struct bpf_iter_seq_task_file_info *info) { - struct pid_namespace *ns = info->common.ns; - u32 curr_tid = info->tid; + u32 saved_tid = info->tid; struct task_struct *curr_task; unsigned int curr_fd = info->fd; @@ -151,21 +291,18 @@ again: curr_task = info->task; curr_fd = info->fd; } else { - curr_task = task_seq_get_next(ns, &curr_tid, true); + curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { info->task = NULL; - info->tid = curr_tid; return NULL; } - /* set info->task and info->tid */ + /* set info->task */ info->task = curr_task; - if (curr_tid == info->tid) { + if (saved_tid == info->tid) curr_fd = info->fd; - } else { - info->tid = curr_tid; + else curr_fd = 0; - } } rcu_read_lock(); @@ -186,9 +323,15 @@ again: /* the current task is done, go to the next task */ rcu_read_unlock(); put_task_struct(curr_task); + + if (info->common.type == BPF_TASK_ITER_TID) { + info->task = NULL; + return NULL; + } + info->task = NULL; info->fd = 0; - curr_tid = ++(info->tid); + saved_tid = ++(info->tid); goto again; } @@ -269,6 +412,9 @@ static int init_seq_pidns(void *priv_data, struct bpf_iter_aux_info *aux) struct bpf_iter_seq_task_common *common = priv_data; common->ns = get_pid_ns(task_active_pid_ns(current)); + common->type = aux->task.type; + common->pid = aux->task.pid; + return 0; } @@ -299,19 +445,18 @@ struct bpf_iter_seq_task_vma_info { }; enum bpf_task_vma_iter_find_op { - task_vma_iter_first_vma, /* use mm->mmap */ - task_vma_iter_next_vma, /* use curr_vma->vm_next */ + task_vma_iter_first_vma, /* use find_vma() with addr 0 */ + task_vma_iter_next_vma, /* use vma_next() with curr_vma */ task_vma_iter_find_vma, /* use find_vma() to find next vma */ }; static struct vm_area_struct * task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) { - struct pid_namespace *ns = info->common.ns; enum bpf_task_vma_iter_find_op op; struct vm_area_struct *curr_vma; struct task_struct *curr_task; - u32 curr_tid = info->tid; + u32 saved_tid = info->tid; /* If this function returns a non-NULL vma, it holds a reference to * the task_struct, and holds read lock on vma->mm->mmap_lock. @@ -371,14 +516,13 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) } } else { again: - curr_task = task_seq_get_next(ns, &curr_tid, true); + curr_task = task_seq_get_next(&info->common, &info->tid, true); if (!curr_task) { - info->tid = curr_tid + 1; + info->tid++; goto finish; } - if (curr_tid != info->tid) { - info->tid = curr_tid; + if (saved_tid != info->tid) { /* new task, process the first vma */ op = task_vma_iter_first_vma; } else { @@ -400,10 +544,10 @@ again: switch (op) { case task_vma_iter_first_vma: - curr_vma = curr_task->mm->mmap; + curr_vma = find_vma(curr_task->mm, 0); break; case task_vma_iter_next_vma: - curr_vma = curr_vma->vm_next; + curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); break; case task_vma_iter_find_vma: /* We dropped mmap_lock so it is necessary to use find_vma @@ -417,7 +561,7 @@ again: if (curr_vma && curr_vma->vm_start == info->prev_vm_start && curr_vma->vm_end == info->prev_vm_end) - curr_vma = curr_vma->vm_next; + curr_vma = find_vma(curr_task->mm, curr_vma->vm_end); break; } if (!curr_vma) { @@ -430,9 +574,12 @@ again: return curr_vma; next_task: + if (info->common.type == BPF_TASK_ITER_TID) + goto finish; + put_task_struct(curr_task); info->task = NULL; - curr_tid++; + info->tid++; goto again; finish: @@ -531,8 +678,33 @@ static const struct bpf_iter_seq_info task_seq_info = { .seq_priv_size = sizeof(struct bpf_iter_seq_task_info), }; +static int bpf_iter_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info) +{ + switch (aux->task.type) { + case BPF_TASK_ITER_TID: + info->iter.task.tid = aux->task.pid; + break; + case BPF_TASK_ITER_TGID: + info->iter.task.pid = aux->task.pid; + break; + default: + break; + } + return 0; +} + +static void bpf_iter_task_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq) +{ + seq_printf(seq, "task_type:\t%s\n", iter_task_type_names[aux->task.type]); + if (aux->task.type == BPF_TASK_ITER_TID) + seq_printf(seq, "tid:\t%u\n", aux->task.pid); + else if (aux->task.type == BPF_TASK_ITER_TGID) + seq_printf(seq, "pid:\t%u\n", aux->task.pid); +} + static struct bpf_iter_reg task_reg_info = { .target = "task", + .attach_target = bpf_iter_attach_task, .feature = BPF_ITER_RESCHED, .ctx_arg_info_size = 1, .ctx_arg_info = { @@ -540,6 +712,8 @@ static struct bpf_iter_reg task_reg_info = { PTR_TO_BTF_ID_OR_NULL }, }, .seq_info = &task_seq_info, + .fill_link_info = bpf_iter_fill_link_info, + .show_fdinfo = bpf_iter_task_show_fdinfo, }; static const struct bpf_iter_seq_info task_file_seq_info = { @@ -551,6 +725,7 @@ static const struct bpf_iter_seq_info task_file_seq_info = { static struct bpf_iter_reg task_file_reg_info = { .target = "task_file", + .attach_target = bpf_iter_attach_task, .feature = BPF_ITER_RESCHED, .ctx_arg_info_size = 2, .ctx_arg_info = { @@ -560,6 +735,8 @@ static struct bpf_iter_reg task_file_reg_info = { PTR_TO_BTF_ID_OR_NULL }, }, .seq_info = &task_file_seq_info, + .fill_link_info = bpf_iter_fill_link_info, + .show_fdinfo = bpf_iter_task_show_fdinfo, }; static const struct bpf_iter_seq_info task_vma_seq_info = { @@ -571,6 +748,7 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", + .attach_target = bpf_iter_attach_task, .feature = BPF_ITER_RESCHED, .ctx_arg_info_size = 2, .ctx_arg_info = { @@ -580,6 +758,8 @@ static struct bpf_iter_reg task_vma_reg_info = { PTR_TO_BTF_ID_OR_NULL }, }, .seq_info = &task_vma_seq_info, + .fill_link_info = bpf_iter_fill_link_info, + .show_fdinfo = bpf_iter_task_show_fdinfo, }; BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index ff87e38af8a7ab140f672a4fb0b5bdd86400f5f7..bf0906e1e2b97ec54893117e4a2dd5883afd7bac 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -116,22 +116,6 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC); } -void *bpf_jit_alloc_exec_page(void) -{ - void *image; - - image = bpf_jit_alloc_exec(PAGE_SIZE); - if (!image) - return NULL; - - set_vm_flush_reset_perms(image); - /* Keep image as writeable. The alternative is to keep flipping ro/rw - * every time new program is attached or detached. - */ - set_memory_x((long)image, 1); - return image; -} - void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym) { ksym->start = (unsigned long) data; @@ -404,9 +388,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx) goto out_free_im; err = -ENOMEM; - im->image = image = bpf_jit_alloc_exec_page(); + im->image = image = bpf_jit_alloc_exec(PAGE_SIZE); if (!image) goto out_uncharge; + set_vm_flush_reset_perms(image); err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL); if (err) @@ -483,6 +468,9 @@ again: if (err < 0) goto out; + set_memory_ro((long)im->image, 1); + set_memory_x((long)im->image, 1); + WARN_ON(tr->cur_image && tr->selector == 0); WARN_ON(!tr->cur_image && tr->selector); if (tr->cur_image) @@ -863,17 +851,6 @@ static __always_inline u64 notrace bpf_prog_start_time(void) return start; } -static void notrace inc_misses_counter(struct bpf_prog *prog) -{ - struct bpf_prog_stats *stats; - unsigned int flags; - - stats = this_cpu_ptr(prog->stats); - flags = u64_stats_update_begin_irqsave(&stats->syncp); - u64_stats_inc(&stats->misses); - u64_stats_update_end_irqrestore(&stats->syncp, flags); -} - /* The logic is similar to bpf_prog_run(), but with an explicit * rcu_read_lock() and migrate_disable() which are required * for the trampoline. The macro is split into @@ -895,8 +872,8 @@ u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *ru run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); - if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { - inc_misses_counter(prog); + if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { + bpf_prog_inc_misses_counter(prog); return 0; } return bpf_prog_start_time(); @@ -930,7 +907,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ bpf_reset_run_ctx(run_ctx->saved_run_ctx); update_prog_stats(prog, start); - __this_cpu_dec(*(prog->active)); + this_cpu_dec(*(prog->active)); migrate_enable(); rcu_read_unlock(); } @@ -966,8 +943,8 @@ u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_r migrate_disable(); might_fault(); - if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) { - inc_misses_counter(prog); + if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { + bpf_prog_inc_misses_counter(prog); return 0; } @@ -982,11 +959,34 @@ void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, bpf_reset_run_ctx(run_ctx->saved_run_ctx); update_prog_stats(prog, start); - __this_cpu_dec(*(prog->active)); + this_cpu_dec(*(prog->active)); migrate_enable(); rcu_read_unlock_trace(); } +u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog, + struct bpf_tramp_run_ctx *run_ctx) + __acquires(RCU) +{ + rcu_read_lock(); + migrate_disable(); + + run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); + + return bpf_prog_start_time(); +} + +void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start, + struct bpf_tramp_run_ctx *run_ctx) + __releases(RCU) +{ + bpf_reset_run_ctx(run_ctx->saved_run_ctx); + + update_prog_stats(prog, start); + migrate_enable(); + rcu_read_unlock(); +} + void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr) { percpu_ref_get(&tr->pcref); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 096fdac701654f7e23a9766ba47115debb0ef7b1..014ee0953dbdea83bec4d794005e5a56e16e4d08 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "disasm.h" @@ -370,6 +371,7 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, bpf_verifier_vlog(log, fmt, args); va_end(args); } +EXPORT_SYMBOL_GPL(bpf_log); static const char *ltrim(const char *s) { @@ -427,6 +429,7 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env, static bool type_is_pkt_pointer(enum bpf_reg_type type) { + type = base_type(type); return type == PTR_TO_PACKET || type == PTR_TO_PACKET_META; } @@ -456,10 +459,9 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) { - return base_type(type) == PTR_TO_SOCKET || - base_type(type) == PTR_TO_TCP_SOCK || - base_type(type) == PTR_TO_MEM || - base_type(type) == PTR_TO_BTF_ID; + type = base_type(type); + return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK || + type == PTR_TO_MEM || type == PTR_TO_BTF_ID; } static bool type_is_rdonly_mem(u32 type) @@ -467,25 +469,11 @@ static bool type_is_rdonly_mem(u32 type) return type & MEM_RDONLY; } -static bool arg_type_may_be_refcounted(enum bpf_arg_type type) -{ - return type == ARG_PTR_TO_SOCK_COMMON; -} - static bool type_may_be_null(u32 type) { return type & PTR_MAYBE_NULL; } -static bool may_be_acquire_function(enum bpf_func_id func_id) -{ - return func_id == BPF_FUNC_sk_lookup_tcp || - func_id == BPF_FUNC_sk_lookup_udp || - func_id == BPF_FUNC_skc_lookup_tcp || - func_id == BPF_FUNC_map_lookup_elem || - func_id == BPF_FUNC_ringbuf_reserve; -} - static bool is_acquire_function(enum bpf_func_id func_id, const struct bpf_map *map) { @@ -518,6 +506,26 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id) func_id == BPF_FUNC_skc_to_tcp_request_sock; } +static bool is_dynptr_ref_function(enum bpf_func_id func_id) +{ + return func_id == BPF_FUNC_dynptr_data; +} + +static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id, + const struct bpf_map *map) +{ + int ref_obj_uses = 0; + + if (is_ptr_cast_function(func_id)) + ref_obj_uses++; + if (is_acquire_function(func_id, map)) + ref_obj_uses++; + if (is_dynptr_ref_function(func_id)) + ref_obj_uses++; + + return ref_obj_uses > 1; +} + static bool is_cmpxchg_insn(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_STX && @@ -555,6 +563,7 @@ static const char *reg_type_str(struct bpf_verifier_env *env, [PTR_TO_BUF] = "buf", [PTR_TO_FUNC] = "func", [PTR_TO_MAP_KEY] = "map_key", + [PTR_TO_DYNPTR] = "dynptr_ptr", }; if (type & PTR_MAYBE_NULL) { @@ -773,8 +782,8 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_ return true; } -static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - enum bpf_arg_type arg_type) +bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, + struct bpf_reg_state *reg) { struct bpf_func_state *state = func(env, reg); int spi = get_spi(reg->off); @@ -790,11 +799,24 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re return false; } + return true; +} + +bool is_dynptr_type_expected(struct bpf_verifier_env *env, + struct bpf_reg_state *reg, + enum bpf_arg_type arg_type) +{ + struct bpf_func_state *state = func(env, reg); + enum bpf_dynptr_type dynptr_type; + int spi = get_spi(reg->off); + /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true; - return state->stack[spi].spilled_ptr.dynptr.type == arg_to_dynptr_type(arg_type); + dynptr_type = arg_to_dynptr_type(arg_type); + + return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; } /* The reg state of a pointer or a bounded scalar was saved when @@ -1086,6 +1108,7 @@ static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) id = ++env->id_gen; state->refs[new_ofs].id = id; state->refs[new_ofs].insn_idx = insn_idx; + state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0; return id; } @@ -1098,6 +1121,9 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id) last_idx = state->acquired_refs - 1; for (i = 0; i < state->acquired_refs; i++) { if (state->refs[i].id == ptr_id) { + /* Cannot release caller references in callbacks */ + if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) + return -EINVAL; if (last_idx && i != last_idx) memcpy(&state->refs[i], &state->refs[last_idx], sizeof(*state->refs)); @@ -1739,6 +1765,7 @@ static void init_func_state(struct bpf_verifier_env *env, state->callsite = callsite; state->frameno = frameno; state->subprogno = subprogno; + state->callback_ret_range = tnum_range(0, 0); init_reg_state(env, state); mark_verifier_state_scratched(env); } @@ -2898,7 +2925,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, return 0; } -static int mark_chain_precision(struct bpf_verifier_env *env, int regno) +int mark_chain_precision(struct bpf_verifier_env *env, int regno) { return __mark_chain_precision(env, regno, -1); } @@ -5223,6 +5250,25 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, env, regno, reg->off, access_size, zero_size_allowed, ACCESS_HELPER, meta); + case PTR_TO_CTX: + /* in case the function doesn't know how to access the context, + * (because we are in a program of type SYSCALL for example), we + * can not statically check its size. + * Dynamically check it now. + */ + if (!env->ops->convert_ctx_access) { + enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ; + int offset = access_size - 1; + + /* Allow zero-byte read from PTR_TO_CTX */ + if (access_size == 0) + return zero_size_allowed ? 0 : -EACCES; + + return check_mem_access(env, env->insn_idx, regno, offset, BPF_B, + atype, -1, false); + } + + fallthrough; default: /* scalar_value or invalid ptr */ /* Allow zero-byte read from NULL, regardless of pointer type */ if (zero_size_allowed && access_size == 0 && @@ -5656,6 +5702,12 @@ static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } }; static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } }; static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } }; +static const struct bpf_reg_types dynptr_types = { + .types = { + PTR_TO_STACK, + PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL, + } +}; static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, @@ -5682,7 +5734,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types, [ARG_PTR_TO_TIMER] = &timer_types, [ARG_PTR_TO_KPTR] = &kptr_types, - [ARG_PTR_TO_DYNPTR] = &stack_ptr_types, + [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; static int check_reg_type(struct bpf_verifier_env *env, u32 regno, @@ -5751,13 +5803,22 @@ found: if (meta->func_id == BPF_FUNC_kptr_xchg) { if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno)) return -EACCES; - } else if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, - btf_vmlinux, *arg_btf_id, - strict_type_match)) { - verbose(env, "R%d is of type %s but %s is expected\n", - regno, kernel_type_name(reg->btf, reg->btf_id), - kernel_type_name(btf_vmlinux, *arg_btf_id)); - return -EACCES; + } else { + if (arg_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n", + regno); + return -EACCES; + } + + if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, + btf_vmlinux, *arg_btf_id, + strict_type_match)) { + verbose(env, "R%d is of type %s but %s is expected\n", + regno, kernel_type_name(reg->btf, reg->btf_id), + kernel_type_name(btf_vmlinux, *arg_btf_id)); + return -EACCES; + } } } @@ -6025,6 +6086,13 @@ skip_type_check: err = check_mem_size_reg(env, reg, regno, true, meta); break; case ARG_PTR_TO_DYNPTR: + /* We only need to check for initialized / uninitialized helper + * dynptr args if the dynptr is not PTR_TO_DYNPTR, as the + * assumption is that if it is, that a helper function + * initialized the dynptr on behalf of the BPF program. + */ + if (base_type(reg->type) == PTR_TO_DYNPTR) + break; if (arg_type & MEM_UNINIT) { if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); @@ -6040,21 +6108,27 @@ skip_type_check: } meta->uninit_dynptr_regno = regno; - } else if (!is_dynptr_reg_valid_init(env, reg, arg_type)) { + } else if (!is_dynptr_reg_valid_init(env, reg)) { + verbose(env, + "Expected an initialized dynptr as arg #%d\n", + arg + 1); + return -EINVAL; + } else if (!is_dynptr_type_expected(env, reg, arg_type)) { const char *err_extra = ""; switch (arg_type & DYNPTR_TYPE_FLAG_MASK) { case DYNPTR_TYPE_LOCAL: - err_extra = "local "; + err_extra = "local"; break; case DYNPTR_TYPE_RINGBUF: - err_extra = "ringbuf "; + err_extra = "ringbuf"; break; default: + err_extra = ""; break; } - - verbose(env, "Expected an initialized %sdynptr as arg #%d\n", + verbose(env, + "Expected a dynptr of type %s as arg #%d\n", err_extra, arg + 1); return -EINVAL; } @@ -6066,6 +6140,9 @@ skip_type_check: return -EACCES; } meta->mem_size = reg->var_off.value; + err = mark_chain_precision(env, regno); + if (err) + return err; break; case ARG_PTR_TO_INT: case ARG_PTR_TO_LONG: @@ -6196,6 +6273,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, func_id != BPF_FUNC_ringbuf_discard_dynptr) goto error; break; + case BPF_MAP_TYPE_USER_RINGBUF: + if (func_id != BPF_FUNC_user_ringbuf_drain) + goto error; + break; case BPF_MAP_TYPE_STACK_TRACE: if (func_id != BPF_FUNC_get_stackid) goto error; @@ -6315,6 +6396,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, if (map->map_type != BPF_MAP_TYPE_RINGBUF) goto error; break; + case BPF_FUNC_user_ringbuf_drain: + if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF) + goto error; + break; case BPF_FUNC_get_stackid: if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) goto error; @@ -6453,33 +6538,6 @@ static bool check_arg_pair_ok(const struct bpf_func_proto *fn) return true; } -static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) -{ - int count = 0; - - if (arg_type_may_be_refcounted(fn->arg1_type)) - count++; - if (arg_type_may_be_refcounted(fn->arg2_type)) - count++; - if (arg_type_may_be_refcounted(fn->arg3_type)) - count++; - if (arg_type_may_be_refcounted(fn->arg4_type)) - count++; - if (arg_type_may_be_refcounted(fn->arg5_type)) - count++; - - /* A reference acquiring function cannot acquire - * another refcounted ptr. - */ - if (may_be_acquire_function(func_id) && count) - return false; - - /* We only support one arg being unreferenced at the moment, - * which is sufficient for the helper functions we have right now. - */ - return count <= 1; -} - static bool check_btf_id_ok(const struct bpf_func_proto *fn) { int i; @@ -6498,43 +6556,25 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) return true; } -static int check_func_proto(const struct bpf_func_proto *fn, int func_id, - struct bpf_call_arg_meta *meta) +static int check_func_proto(const struct bpf_func_proto *fn, int func_id) { return check_raw_mode_ok(fn) && check_arg_pair_ok(fn) && - check_btf_id_ok(fn) && - check_refcount_ok(fn, func_id) ? 0 : -EINVAL; + check_btf_id_ok(fn) ? 0 : -EINVAL; } /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] * are now invalid, so turn them into unknown SCALAR_VALUE. */ -static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, - struct bpf_func_state *state) +static void clear_all_pkt_pointers(struct bpf_verifier_env *env) { - struct bpf_reg_state *regs = state->regs, *reg; - int i; - - for (i = 0; i < MAX_BPF_REG; i++) - if (reg_is_pkt_pointer_any(®s[i])) - mark_reg_unknown(env, regs, i); + struct bpf_func_state *state; + struct bpf_reg_state *reg; - bpf_for_each_spilled_reg(i, state, reg) { - if (!reg) - continue; + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg_is_pkt_pointer_any(reg)) __mark_reg_unknown(env, reg); - } -} - -static void clear_all_pkt_pointers(struct bpf_verifier_env *env) -{ - struct bpf_verifier_state *vstate = env->cur_state; - int i; - - for (i = 0; i <= vstate->curframe; i++) - __clear_all_pkt_pointers(env, vstate->frame[i]); + })); } enum { @@ -6563,41 +6603,24 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static void release_reg_references(struct bpf_verifier_env *env, - struct bpf_func_state *state, - int ref_obj_id) -{ - struct bpf_reg_state *regs = state->regs, *reg; - int i; - - for (i = 0; i < MAX_BPF_REG; i++) - if (regs[i].ref_obj_id == ref_obj_id) - mark_reg_unknown(env, regs, i); - - bpf_for_each_spilled_reg(i, state, reg) { - if (!reg) - continue; - if (reg->ref_obj_id == ref_obj_id) - __mark_reg_unknown(env, reg); - } -} - /* The pointer with the specified id has released its reference to kernel * resources. Identify all copies of the same pointer and clear the reference. */ static int release_reference(struct bpf_verifier_env *env, int ref_obj_id) { - struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state; + struct bpf_reg_state *reg; int err; - int i; err = release_reference_state(cur_func(env), ref_obj_id); if (err) return err; - for (i = 0; i <= vstate->curframe; i++) - release_reg_references(env, vstate->frame[i], ref_obj_id); + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (reg->ref_obj_id == ref_obj_id) + __mark_reg_unknown(env, reg); + })); return 0; } @@ -6645,7 +6668,7 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn func_info_aux = env->prog->aux->func_info_aux; if (func_info_aux) is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; - err = btf_check_subprog_arg_match(env, subprog, caller->regs); + err = btf_check_subprog_call(env, subprog, caller->regs); if (err == -EFAULT) return err; if (is_global) { @@ -6819,6 +6842,7 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, return err; callee->in_callback_fn = true; + callee->callback_ret_range = tnum_range(0, 1); return 0; } @@ -6840,6 +6864,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + callee->callback_ret_range = tnum_range(0, 1); return 0; } @@ -6869,6 +6894,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_async_callback_fn = true; + callee->callback_ret_range = tnum_range(0, 1); return 0; } @@ -6895,6 +6921,30 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, /* unused */ __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_callback_fn = true; + callee->callback_ret_range = tnum_range(0, 1); + return 0; +} + +static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void + * callback_ctx, u64 flags); + * callback_fn(struct bpf_dynptr_t* dynptr, void *callback_ctx); + */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); + callee->regs[BPF_REG_1].type = PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_callback_fn = true; return 0; } @@ -6923,7 +6973,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) caller = state->frame[state->curframe]; if (callee->in_callback_fn) { /* enforce R0 return value range [0, 1]. */ - struct tnum range = tnum_range(0, 1); + struct tnum range = callee->callback_ret_range; if (r0->type != SCALAR_VALUE) { verbose(env, "R0 not a scalar value\n"); @@ -6938,10 +6988,17 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) caller->regs[BPF_REG_0] = *r0; } - /* Transfer references to the caller */ - err = copy_reference_state(caller, callee); - if (err) - return err; + /* callback_fn frame should have released its own additions to parent's + * reference state at this point, or check_reference_leak would + * complain, hence it must be the same as the caller. There is no need + * to copy it back. + */ + if (!callee->in_callback_fn) { + /* Transfer references to the caller */ + err = copy_reference_state(caller, callee); + if (err) + return err; + } *insn_idx = callee->callsite + 1; if (env->log.level & BPF_LOG_LEVEL) { @@ -7030,8 +7087,7 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; struct bpf_reg_state *regs = cur_regs(env), *reg; struct bpf_map *map = meta->map_ptr; - struct tnum range; - u64 val; + u64 val, max; int err; if (func_id != BPF_FUNC_tail_call) @@ -7041,10 +7097,11 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, return -EINVAL; } - range = tnum_range(0, map->max_entries - 1); reg = ®s[BPF_REG_3]; + val = reg->var_off.value; + max = map->max_entries; - if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { + if (!(register_is_const(reg) && val < max)) { bpf_map_key_store(aux, BPF_MAP_KEY_POISON); return 0; } @@ -7052,8 +7109,6 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, err = mark_chain_precision(env, BPF_REG_3); if (err) return err; - - val = reg->var_off.value; if (bpf_map_key_unseen(aux)) bpf_map_key_store(aux, val); else if (!bpf_map_key_poisoned(aux) && @@ -7065,13 +7120,20 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, static int check_reference_leak(struct bpf_verifier_env *env) { struct bpf_func_state *state = cur_func(env); + bool refs_lingering = false; int i; + if (state->frameno && !state->in_callback_fn) + return 0; + for (i = 0; i < state->acquired_refs; i++) { + if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno) + continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); + refs_lingering = true; } - return state->acquired_refs ? -EINVAL : 0; + return refs_lingering ? -EINVAL : 0; } static int check_bpf_snprintf_call(struct bpf_verifier_env *env, @@ -7218,7 +7280,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn memset(&meta, 0, sizeof(meta)); meta.pkt_access = fn->pkt_access; - err = check_func_proto(fn, func_id, &meta); + err = check_func_proto(fn, func_id); if (err) { verbose(env, "kernel subsystem misconfigured func %s#%d\n", func_id_name(func_id), func_id); @@ -7343,6 +7405,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } } break; + case BPF_FUNC_dynptr_data: + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { + if (arg_type_is_dynptr(fn->arg_type[i])) { + struct bpf_reg_state *reg = ®s[BPF_REG_1 + i]; + + if (meta.ref_obj_id) { + verbose(env, "verifier internal error: meta.ref_obj_id already set\n"); + return -EFAULT; + } + + if (base_type(reg->type) != PTR_TO_DYNPTR) + /* Find the id of the dynptr we're + * tracking the reference of + */ + meta.ref_obj_id = stack_slot_get_id(env, reg); + break; + } + } + if (i == MAX_BPF_FUNC_REG_ARGS) { + verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n"); + return -EFAULT; + } + break; + case BPF_FUNC_user_ringbuf_drain: + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_user_ringbuf_callback_state); + break; } if (err) @@ -7359,13 +7448,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* update return register (already marked as written above) */ ret_type = fn->ret_type; - ret_flag = type_flag(fn->ret_type); - if (ret_type == RET_INTEGER) { + ret_flag = type_flag(ret_type); + + switch (base_type(ret_type)) { + case RET_INTEGER: /* sets type to SCALAR_VALUE */ mark_reg_unknown(env, regs, BPF_REG_0); - } else if (ret_type == RET_VOID) { + break; + case RET_VOID: regs[BPF_REG_0].type = NOT_INIT; - } else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) { + break; + case RET_PTR_TO_MAP_VALUE: /* There is no offset yet applied, variable or fixed */ mark_reg_known_zero(env, regs, BPF_REG_0); /* remember map_ptr, so that check_map_access() @@ -7384,20 +7477,26 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn map_value_has_spin_lock(meta.map_ptr)) { regs[BPF_REG_0].id = ++env->id_gen; } - } else if (base_type(ret_type) == RET_PTR_TO_SOCKET) { + break; + case RET_PTR_TO_SOCKET: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag; - } else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) { + break; + case RET_PTR_TO_SOCK_COMMON: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag; - } else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) { + break; + case RET_PTR_TO_TCP_SOCK: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag; - } else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) { + break; + case RET_PTR_TO_ALLOC_MEM: mark_reg_known_zero(env, regs, BPF_REG_0); regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag; regs[BPF_REG_0].mem_size = meta.mem_size; - } else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) { + break; + case RET_PTR_TO_MEM_OR_BTF_ID: + { const struct btf_type *t; mark_reg_known_zero(env, regs, BPF_REG_0); @@ -7429,7 +7528,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].btf = meta.ret_btf; regs[BPF_REG_0].btf_id = meta.ret_btf_id; } - } else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) { + break; + } + case RET_PTR_TO_BTF_ID: + { struct btf *ret_btf; int ret_btf_id; @@ -7439,6 +7541,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn ret_btf = meta.kptr_off_desc->kptr.btf; ret_btf_id = meta.kptr_off_desc->kptr.btf_id; } else { + if (fn->ret_btf_id == BPF_PTR_POISON) { + verbose(env, "verifier internal error:"); + verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n", + func_id_name(func_id)); + return -EINVAL; + } ret_btf = btf_vmlinux; ret_btf_id = *fn->ret_btf_id; } @@ -7450,7 +7558,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn } regs[BPF_REG_0].btf = ret_btf; regs[BPF_REG_0].btf_id = ret_btf_id; - } else { + break; + } + default: verbose(env, "unknown return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id), func_id); return -EINVAL; @@ -7459,7 +7569,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (type_may_be_null(regs[BPF_REG_0].type)) regs[BPF_REG_0].id = ++env->id_gen; - if (is_ptr_cast_function(func_id)) { + if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) { + verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n", + func_id_name(func_id), func_id); + return -EFAULT; + } + + if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) { /* For release_reference() */ regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; } else if (is_acquire_function(func_id, meta.map_ptr)) { @@ -7471,21 +7587,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn regs[BPF_REG_0].id = id; /* For release_reference() */ regs[BPF_REG_0].ref_obj_id = id; - } else if (func_id == BPF_FUNC_dynptr_data) { - int dynptr_id = 0, i; - - /* Find the id of the dynptr we're acquiring a reference to */ - for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) { - if (arg_type_is_dynptr(fn->arg_type[i])) { - if (dynptr_id) { - verbose(env, "verifier internal error: multiple dynptr args in func\n"); - return -EFAULT; - } - dynptr_id = stack_slot_get_id(env, ®s[BPF_REG_1 + i]); - } - } - /* For release_reference() */ - regs[BPF_REG_0].ref_obj_id = dynptr_id; } do_refine_retval_range(regs, fn->ret_type, func_id, &meta); @@ -7557,6 +7658,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, { const struct btf_type *t, *func, *func_proto, *ptr_type; struct bpf_reg_state *regs = cur_regs(env); + struct bpf_kfunc_arg_meta meta = { 0 }; const char *func_name, *ptr_type_name; u32 i, nargs, func_id, ptr_type_id; int err, insn_idx = *insn_idx_p; @@ -7584,10 +7686,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, func_name); return -EACCES; } + if (*kfunc_flags & KF_DESTRUCTIVE && !capable(CAP_SYS_BOOT)) { + verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capabilities\n"); + return -EACCES; + } + acq = *kfunc_flags & KF_ACQUIRE; + meta.flags = *kfunc_flags; + /* Check the arguments */ - err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, *kfunc_flags); + err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, &meta); if (err < 0) return err; /* In case of release function, we get register number of refcounted @@ -7608,7 +7717,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* Check return type */ t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL); - if (acq && !btf_type_is_ptr(t)) { + if (acq && !btf_type_is_struct_ptr(desc_btf, t)) { verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n"); return -EINVAL; } @@ -7620,17 +7729,33 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); if (!btf_type_is_struct(ptr_type)) { - ptr_type_name = btf_name_by_offset(desc_btf, - ptr_type->name_off); - verbose(env, "kernel function %s returns pointer type %s %s is not supported\n", - func_name, btf_type_str(ptr_type), - ptr_type_name); - return -EINVAL; + if (!meta.r0_size) { + ptr_type_name = btf_name_by_offset(desc_btf, + ptr_type->name_off); + verbose(env, + "kernel function %s returns pointer type %s %s is not supported\n", + func_name, + btf_type_str(ptr_type), + ptr_type_name); + return -EINVAL; + } + + mark_reg_known_zero(env, regs, BPF_REG_0); + regs[BPF_REG_0].type = PTR_TO_MEM; + regs[BPF_REG_0].mem_size = meta.r0_size; + + if (meta.r0_rdonly) + regs[BPF_REG_0].type |= MEM_RDONLY; + + /* Ensures we don't access the memory after a release_reference() */ + if (meta.ref_obj_id) + regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; + } else { + mark_reg_known_zero(env, regs, BPF_REG_0); + regs[BPF_REG_0].btf = desc_btf; + regs[BPF_REG_0].type = PTR_TO_BTF_ID; + regs[BPF_REG_0].btf_id = ptr_type_id; } - mark_reg_known_zero(env, regs, BPF_REG_0); - regs[BPF_REG_0].btf = desc_btf; - regs[BPF_REG_0].type = PTR_TO_BTF_ID; - regs[BPF_REG_0].btf_id = ptr_type_id; if (*kfunc_flags & KF_RET_NULL) { regs[BPF_REG_0].type |= PTR_MAYBE_NULL; /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */ @@ -9273,34 +9398,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } -static void __find_good_pkt_pointers(struct bpf_func_state *state, - struct bpf_reg_state *dst_reg, - enum bpf_reg_type type, int new_range) -{ - struct bpf_reg_state *reg; - int i; - - for (i = 0; i < MAX_BPF_REG; i++) { - reg = &state->regs[i]; - if (reg->type == type && reg->id == dst_reg->id) - /* keep the maximum range already checked */ - reg->range = max(reg->range, new_range); - } - - bpf_for_each_spilled_reg(i, state, reg) { - if (!reg) - continue; - if (reg->type == type && reg->id == dst_reg->id) - reg->range = max(reg->range, new_range); - } -} - static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, struct bpf_reg_state *dst_reg, enum bpf_reg_type type, bool range_right_open) { - int new_range, i; + struct bpf_func_state *state; + struct bpf_reg_state *reg; + int new_range; if (dst_reg->off < 0 || (dst_reg->off == 0 && range_right_open)) @@ -9365,9 +9470,11 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, * the range won't allow anything. * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. */ - for (i = 0; i <= vstate->curframe; i++) - __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, - new_range); + bpf_for_each_reg_in_vstate(vstate, state, reg, ({ + if (reg->type == type && reg->id == dst_reg->id) + /* keep the maximum range already checked */ + reg->range = max(reg->range, new_range); + })); } static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) @@ -9856,7 +9963,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, if (!reg_may_point_to_spin_lock(reg)) { /* For not-NULL ptr, reg->ref_obj_id will be reset - * in release_reg_references(). + * in release_reference(). * * reg->id is still used by spin_lock ptr. Other * than spin_lock ptr type, reg->id can be reset. @@ -9866,22 +9973,6 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, } } -static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, - bool is_null) -{ - struct bpf_reg_state *reg; - int i; - - for (i = 0; i < MAX_BPF_REG; i++) - mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); - - bpf_for_each_spilled_reg(i, state, reg) { - if (!reg) - continue; - mark_ptr_or_null_reg(state, reg, id, is_null); - } -} - /* The logic is similar to find_good_pkt_pointers(), both could eventually * be folded together at some point. */ @@ -9889,10 +9980,9 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, bool is_null) { struct bpf_func_state *state = vstate->frame[vstate->curframe]; - struct bpf_reg_state *regs = state->regs; + struct bpf_reg_state *regs = state->regs, *reg; u32 ref_obj_id = regs[regno].ref_obj_id; u32 id = regs[regno].id; - int i; if (ref_obj_id && ref_obj_id == id && is_null) /* regs[regno] is in the " == NULL" branch. @@ -9901,8 +9991,9 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, */ WARN_ON_ONCE(release_reference_state(state, id)); - for (i = 0; i <= vstate->curframe; i++) - __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); + bpf_for_each_reg_in_vstate(vstate, state, reg, ({ + mark_ptr_or_null_reg(state, reg, id, is_null); + })); } static bool try_match_pkt_pointers(const struct bpf_insn *insn, @@ -10015,23 +10106,11 @@ static void find_equal_scalars(struct bpf_verifier_state *vstate, { struct bpf_func_state *state; struct bpf_reg_state *reg; - int i, j; - - for (i = 0; i <= vstate->curframe; i++) { - state = vstate->frame[i]; - for (j = 0; j < MAX_BPF_REG; j++) { - reg = &state->regs[j]; - if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) - *reg = *known_reg; - } - bpf_for_each_spilled_reg(j, state, reg) { - if (!reg) - continue; - if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) - *reg = *known_reg; - } - } + bpf_for_each_reg_in_vstate(vstate, state, reg, ({ + if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) + *reg = *known_reg; + })); } static int check_cond_jmp_op(struct bpf_verifier_env *env, @@ -12332,6 +12411,16 @@ static int do_check(struct bpf_verifier_env *env) return -EINVAL; } + /* We must do check_reference_leak here before + * prepare_func_exit to handle the case when + * state->curframe > 0, it may be a callback + * function, for which reference_state must + * match caller reference state when it exits. + */ + err = check_reference_leak(env); + if (err) + return err; + if (state->curframe) { /* exit from nested function */ err = prepare_func_exit(env, &env->insn_idx); @@ -12341,10 +12430,6 @@ static int do_check(struct bpf_verifier_env *env) continue; } - err = check_reference_leak(env); - if (err) - return err; - err = check_return_code(env); if (err) return err; @@ -12557,14 +12642,6 @@ err_put: return err; } -static int check_map_prealloc(struct bpf_map *map) -{ - return (map->map_type != BPF_MAP_TYPE_HASH && - map->map_type != BPF_MAP_TYPE_PERCPU_HASH && - map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || - !(map->map_flags & BPF_F_NO_PREALLOC); -} - static bool is_tracing_prog_type(enum bpf_prog_type type) { switch (type) { @@ -12579,50 +12656,12 @@ static bool is_tracing_prog_type(enum bpf_prog_type type) } } -static bool is_preallocated_map(struct bpf_map *map) -{ - if (!check_map_prealloc(map)) - return false; - if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) - return false; - return true; -} - static int check_map_prog_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, struct bpf_prog *prog) { enum bpf_prog_type prog_type = resolve_prog_type(prog); - /* - * Validate that trace type programs use preallocated hash maps. - * - * For programs attached to PERF events this is mandatory as the - * perf NMI can hit any arbitrary code sequence. - * - * All other trace types using preallocated hash maps are unsafe as - * well because tracepoint or kprobes can be inside locked regions - * of the memory allocator or at a place where a recursion into the - * memory allocator would see inconsistent state. - * - * On RT enabled kernels run-time allocation of all trace type - * programs is strictly prohibited due to lock type constraints. On - * !RT kernels it is allowed for backwards compatibility reasons for - * now, but warnings are emitted so developers are made aware of - * the unsafety and can fix their programs before this is enforced. - */ - if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { - if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { - verbose(env, "perf_event programs can only use preallocated hash map\n"); - return -EINVAL; - } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) { - verbose(env, "trace type programs can only use preallocated hash map\n"); - return -EINVAL; - } - WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); - verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); - } if (map_value_has_spin_lock(map)) { if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { @@ -12669,13 +12708,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, case BPF_MAP_TYPE_LRU_PERCPU_HASH: case BPF_MAP_TYPE_ARRAY_OF_MAPS: case BPF_MAP_TYPE_HASH_OF_MAPS: - if (!is_preallocated_map(map)) { - verbose(env, - "Sleepable programs can only use preallocated maps\n"); - return -EINVAL; - } - break; case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: case BPF_MAP_TYPE_INODE_STORAGE: case BPF_MAP_TYPE_SK_STORAGE: case BPF_MAP_TYPE_TASK_STORAGE: @@ -13316,7 +13350,7 @@ static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, aux[adj_idx].ptr_type == PTR_TO_CTX) continue; - imm_rnd = get_random_int(); + imm_rnd = get_random_u32(); rnd_hi32_patch[0] = insn; rnd_hi32_patch[1].imm = imm_rnd; rnd_hi32_patch[3].dst_reg = load_reg; @@ -13469,9 +13503,6 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEM | BPF_SIZE((insn)->code); env->prog->aux->num_exentries++; - } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { - verbose(env, "Writes through BTF pointers are not allowed\n"); - return -EINVAL; } continue; default: diff --git a/kernel/cfi.c b/kernel/cfi.c index 2046276ee2348e8cf2317d4f57d831933014f6b5..08caad7767176e2c119d2443f84ab7ee61ac0463 100644 --- a/kernel/cfi.c +++ b/kernel/cfi.c @@ -1,339 +1,101 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Clang Control Flow Integrity (CFI) error and slowpath handling. + * Clang Control Flow Integrity (CFI) error handling. * - * Copyright (C) 2021 Google LLC + * Copyright (C) 2022 Google LLC */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include -/* Compiler-defined handler names */ -#ifdef CONFIG_CFI_PERMISSIVE -#define cfi_failure_handler __ubsan_handle_cfi_check_fail -#else -#define cfi_failure_handler __ubsan_handle_cfi_check_fail_abort -#endif - -static inline void handle_cfi_failure(void *ptr) +enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr, + unsigned long *target, u32 type) { - if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) - WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr); + if (target) + pr_err("CFI failure at %pS (target: %pS; expected type: 0x%08x)\n", + (void *)addr, (void *)*target, type); else - panic("CFI failure (target: %pS)\n", ptr); -} - -#ifdef CONFIG_MODULES -#ifdef CONFIG_CFI_CLANG_SHADOW -/* - * Index type. A 16-bit index can address at most (2^16)-2 pages (taking - * into account SHADOW_INVALID), i.e. ~256M with 4k pages. - */ -typedef u16 shadow_t; -#define SHADOW_INVALID ((shadow_t)~0UL) - -struct cfi_shadow { - /* Page index for the beginning of the shadow */ - unsigned long base; - /* An array of __cfi_check locations (as indices to the shadow) */ - shadow_t shadow[1]; -} __packed; - -/* - * The shadow covers ~128M from the beginning of the module region. If - * the region is larger, we fall back to __module_address for the rest. - */ -#define __SHADOW_RANGE (_UL(SZ_128M) >> PAGE_SHIFT) - -/* The in-memory size of struct cfi_shadow, always at least one page */ -#define __SHADOW_PAGES ((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT) -#define SHADOW_PAGES max(1UL, __SHADOW_PAGES) -#define SHADOW_SIZE (SHADOW_PAGES << PAGE_SHIFT) - -/* The actual size of the shadow array, minus metadata */ -#define SHADOW_ARR_SIZE (SHADOW_SIZE - offsetof(struct cfi_shadow, shadow)) -#define SHADOW_ARR_SLOTS (SHADOW_ARR_SIZE / sizeof(shadow_t)) - -static DEFINE_MUTEX(shadow_update_lock); -static struct cfi_shadow __rcu *cfi_shadow __read_mostly; + pr_err("CFI failure at %pS (no target information)\n", + (void *)addr); -/* Returns the index in the shadow for the given address */ -static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr) -{ - unsigned long index; - unsigned long page = ptr >> PAGE_SHIFT; - - if (unlikely(page < s->base)) - return -1; /* Outside of module area */ - - index = page - s->base; - - if (index >= SHADOW_ARR_SLOTS) - return -1; /* Cannot be addressed with shadow */ - - return (int)index; -} - -/* Returns the page address for an index in the shadow */ -static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s, - int index) -{ - if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS)) - return 0; - - return (s->base + index) << PAGE_SHIFT; -} - -/* Returns the __cfi_check function address for the given shadow location */ -static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s, - int index) -{ - if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS)) - return 0; - - if (unlikely(s->shadow[index] == SHADOW_INVALID)) - return 0; - - /* __cfi_check is always page aligned */ - return (s->base + s->shadow[index]) << PAGE_SHIFT; -} - -static void prepare_next_shadow(const struct cfi_shadow __rcu *prev, - struct cfi_shadow *next) -{ - int i, index, check; - - /* Mark everything invalid */ - memset(next->shadow, 0xFF, SHADOW_ARR_SIZE); - - if (!prev) - return; /* No previous shadow */ - - /* If the base address didn't change, an update is not needed */ - if (prev->base == next->base) { - memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE); - return; - } - - /* Convert the previous shadow to the new address range */ - for (i = 0; i < SHADOW_ARR_SLOTS; ++i) { - if (prev->shadow[i] == SHADOW_INVALID) - continue; - - index = ptr_to_shadow(next, shadow_to_ptr(prev, i)); - if (index < 0) - continue; - - check = ptr_to_shadow(next, - shadow_to_check_fn(prev, prev->shadow[i])); - if (check < 0) - continue; - - next->shadow[index] = (shadow_t)check; - } -} - -static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod, - unsigned long min_addr, unsigned long max_addr) -{ - int check_index; - unsigned long check = (unsigned long)mod->cfi_check; - unsigned long ptr; - - if (unlikely(!PAGE_ALIGNED(check))) { - pr_warn("cfi: not using shadow for module %s\n", mod->name); - return; + if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) { + __warn(NULL, 0, (void *)addr, 0, regs, NULL); + return BUG_TRAP_TYPE_WARN; } - check_index = ptr_to_shadow(s, check); - if (check_index < 0) - return; /* Module not addressable with shadow */ - - /* For each page, store the check function index in the shadow */ - for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) { - int index = ptr_to_shadow(s, ptr); - - if (index >= 0) { - /* Each page must only contain one module */ - WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID); - s->shadow[index] = (shadow_t)check_index; - } - } + return BUG_TRAP_TYPE_BUG; } -static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod, - unsigned long min_addr, unsigned long max_addr) +#ifdef CONFIG_ARCH_USES_CFI_TRAPS +static inline unsigned long trap_address(s32 *p) { - unsigned long ptr; - - for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) { - int index = ptr_to_shadow(s, ptr); - - if (index >= 0) - s->shadow[index] = SHADOW_INVALID; - } + return (unsigned long)((long)p + (long)*p); } -typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *, - unsigned long min_addr, unsigned long max_addr); - -static void update_shadow(struct module *mod, unsigned long base_addr, - update_shadow_fn fn) +static bool is_trap(unsigned long addr, s32 *start, s32 *end) { - struct cfi_shadow *prev; - struct cfi_shadow *next; - unsigned long min_addr, max_addr; - - next = vmalloc(SHADOW_SIZE); - - mutex_lock(&shadow_update_lock); - prev = rcu_dereference_protected(cfi_shadow, - mutex_is_locked(&shadow_update_lock)); - - if (next) { - next->base = base_addr >> PAGE_SHIFT; - prepare_next_shadow(prev, next); + s32 *p; - min_addr = (unsigned long)mod->core_layout.base; - max_addr = min_addr + mod->core_layout.text_size; - fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK); - - set_memory_ro((unsigned long)next, SHADOW_PAGES); - } - - rcu_assign_pointer(cfi_shadow, next); - mutex_unlock(&shadow_update_lock); - synchronize_rcu(); - - if (prev) { - set_memory_rw((unsigned long)prev, SHADOW_PAGES); - vfree(prev); + for (p = start; p < end; ++p) { + if (trap_address(p) == addr) + return true; } -} -void cfi_module_add(struct module *mod, unsigned long base_addr) -{ - update_shadow(mod, base_addr, add_module_to_shadow); + return false; } -void cfi_module_remove(struct module *mod, unsigned long base_addr) -{ - update_shadow(mod, base_addr, remove_module_from_shadow); -} - -static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s, - unsigned long ptr) +#ifdef CONFIG_MODULES +/* Populates `kcfi_trap(_end)?` fields in `struct module`. */ +void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod) { - int index; - - if (unlikely(!s)) - return NULL; /* No shadow available */ - - index = ptr_to_shadow(s, ptr); - if (index < 0) - return NULL; /* Cannot be addressed with shadow */ + char *secstrings; + unsigned int i; - return (cfi_check_fn)shadow_to_check_fn(s, index); -} - -static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr) -{ - cfi_check_fn fn; + mod->kcfi_traps = NULL; + mod->kcfi_traps_end = NULL; - rcu_read_lock_sched_notrace(); - fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr); - rcu_read_unlock_sched_notrace(); + secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; - return fn; -} - -#else /* !CONFIG_CFI_CLANG_SHADOW */ + for (i = 1; i < hdr->e_shnum; i++) { + if (strcmp(secstrings + sechdrs[i].sh_name, "__kcfi_traps")) + continue; -static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr) -{ - return NULL; + mod->kcfi_traps = (s32 *)sechdrs[i].sh_addr; + mod->kcfi_traps_end = (s32 *)(sechdrs[i].sh_addr + sechdrs[i].sh_size); + break; + } } -#endif /* CONFIG_CFI_CLANG_SHADOW */ - -static inline cfi_check_fn find_module_check_fn(unsigned long ptr) +static bool is_module_cfi_trap(unsigned long addr) { - cfi_check_fn fn = NULL; struct module *mod; + bool found = false; rcu_read_lock_sched_notrace(); - mod = __module_address(ptr); - if (mod) - fn = mod->cfi_check; - rcu_read_unlock_sched_notrace(); - - return fn; -} - -static inline cfi_check_fn find_check_fn(unsigned long ptr) -{ - cfi_check_fn fn = NULL; - unsigned long flags; - bool rcu_idle; - - if (is_kernel_text(ptr)) - return __cfi_check; - /* - * Indirect call checks can happen when RCU is not watching. Both - * the shadow and __module_address use RCU, so we need to wake it - * up if necessary. - */ - rcu_idle = !rcu_is_watching(); - if (rcu_idle) { - local_irq_save(flags); - ct_irq_enter(); - } - - if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW)) - fn = find_shadow_check_fn(ptr); - if (!fn) - fn = find_module_check_fn(ptr); + mod = __module_address(addr); + if (mod) + found = is_trap(addr, mod->kcfi_traps, mod->kcfi_traps_end); - if (rcu_idle) { - ct_irq_exit(); - local_irq_restore(flags); - } + rcu_read_unlock_sched_notrace(); - return fn; + return found; } - -void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag) +#else /* CONFIG_MODULES */ +static inline bool is_module_cfi_trap(unsigned long addr) { - cfi_check_fn fn = find_check_fn((unsigned long)ptr); - - if (likely(fn)) - fn(id, ptr, diag); - else /* Don't allow unchecked modules */ - handle_cfi_failure(ptr); + return false; } -EXPORT_SYMBOL(__cfi_slowpath_diag); +#endif /* CONFIG_MODULES */ -#else /* !CONFIG_MODULES */ +extern s32 __start___kcfi_traps[]; +extern s32 __stop___kcfi_traps[]; -void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag) +bool is_cfi_trap(unsigned long addr) { - handle_cfi_failure(ptr); /* No modules */ -} -EXPORT_SYMBOL(__cfi_slowpath_diag); + if (is_trap(addr, __start___kcfi_traps, __stop___kcfi_traps)) + return true; -#endif /* CONFIG_MODULES */ - -void cfi_failure_handler(void *data, void *ptr, void *vtable) -{ - handle_cfi_failure(ptr); + return is_module_cfi_trap(addr); } -EXPORT_SYMBOL(cfi_failure_handler); +#endif /* CONFIG_ARCH_USES_CFI_TRAPS */ diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h index 36b740cb3d59ef1ee111912c516de1c3f281e008..fd4020835ec6ca7be67868f150a15b5529b93a8e 100644 --- a/kernel/cgroup/cgroup-internal.h +++ b/kernel/cgroup/cgroup-internal.h @@ -164,7 +164,6 @@ struct cgroup_mgctx { #define DEFINE_CGROUP_MGCTX(name) \ struct cgroup_mgctx name = CGROUP_MGCTX_INIT(name) -extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct cgroup_subsys *cgroup_subsys[]; extern struct list_head cgroup_roots; @@ -250,6 +249,8 @@ int cgroup_migrate(struct task_struct *leader, bool threadgroup, int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, bool threadgroup); +void cgroup_attach_lock(bool lock_threadgroup); +void cgroup_attach_unlock(bool lock_threadgroup); struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, bool *locked) __acquires(&cgroup_threadgroup_rwsem); diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 2ade21b54dc44150f26479d03dac1ec705ad626c..52bb5a74a23b98f8721466126ec84a8059c6a3f7 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -59,7 +59,7 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) int retval = 0; mutex_lock(&cgroup_mutex); - percpu_down_write(&cgroup_threadgroup_rwsem); + cgroup_attach_lock(true); for_each_root(root) { struct cgroup *from_cgrp; @@ -71,7 +71,7 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) if (retval) break; } - percpu_up_write(&cgroup_threadgroup_rwsem); + cgroup_attach_unlock(true); mutex_unlock(&cgroup_mutex); return retval; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index ffaccd6373f1e786e9c90f8230b16f11ef4b8535..7f486677ab1febcf064735ed47c87075b5118db7 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -217,6 +217,7 @@ struct cgroup_namespace init_cgroup_ns = { static struct file_system_type cgroup2_fs_type; static struct cftype cgroup_base_files[]; +static struct cftype cgroup_psi_files[]; /* cgroup optional features */ enum cgroup_opt_features { @@ -1689,12 +1690,16 @@ static void css_clear_dir(struct cgroup_subsys_state *css) css->flags &= ~CSS_VISIBLE; if (!css->ss) { - if (cgroup_on_dfl(cgrp)) - cfts = cgroup_base_files; - else - cfts = cgroup1_base_files; - - cgroup_addrm_files(css, cgrp, cfts, false); + if (cgroup_on_dfl(cgrp)) { + cgroup_addrm_files(css, cgrp, + cgroup_base_files, false); + if (cgroup_psi_enabled()) + cgroup_addrm_files(css, cgrp, + cgroup_psi_files, false); + } else { + cgroup_addrm_files(css, cgrp, + cgroup1_base_files, false); + } } else { list_for_each_entry(cfts, &css->ss->cfts, node) cgroup_addrm_files(css, cgrp, cfts, false); @@ -1717,14 +1722,22 @@ static int css_populate_dir(struct cgroup_subsys_state *css) return 0; if (!css->ss) { - if (cgroup_on_dfl(cgrp)) - cfts = cgroup_base_files; - else - cfts = cgroup1_base_files; - - ret = cgroup_addrm_files(&cgrp->self, cgrp, cfts, true); - if (ret < 0) - return ret; + if (cgroup_on_dfl(cgrp)) { + ret = cgroup_addrm_files(&cgrp->self, cgrp, + cgroup_base_files, true); + if (ret < 0) + return ret; + + if (cgroup_psi_enabled()) { + ret = cgroup_addrm_files(&cgrp->self, cgrp, + cgroup_psi_files, true); + if (ret < 0) + return ret; + } + } else { + cgroup_addrm_files(css, cgrp, + cgroup1_base_files, true); + } } else { list_for_each_entry(cfts, &css->ss->cfts, node) { ret = cgroup_addrm_files(css, cgrp, cfts, true); @@ -1820,6 +1833,7 @@ int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask) if (ss->css_rstat_flush) { list_del_rcu(&css->rstat_css_node); + synchronize_rcu(); list_add_rcu(&css->rstat_css_node, &dcgrp->rstat_css_list); } @@ -2049,7 +2063,7 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) } root_cgrp->kn = kernfs_root_to_node(root->kf_root); WARN_ON_ONCE(cgroup_ino(root_cgrp) != 1); - root_cgrp->ancestor_ids[0] = cgroup_id(root_cgrp); + root_cgrp->ancestors[0] = root_cgrp; ret = css_populate_dir(&root_cgrp->self); if (ret) @@ -2172,7 +2186,7 @@ static int cgroup_get_tree(struct fs_context *fc) struct cgroup_fs_context *ctx = cgroup_fc2context(fc); int ret; - cgrp_dfl_visible = true; + WRITE_ONCE(cgrp_dfl_visible, true); cgroup_get_live(&cgrp_dfl_root.cgrp); ctx->root = &cgrp_dfl_root; @@ -2360,7 +2374,7 @@ int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns); } else { /* if no hierarchy exists, everyone is in "/" */ - ret = strlcpy(buf, "/", buflen); + ret = strscpy(buf, "/", buflen); } spin_unlock_irq(&css_set_lock); @@ -2369,6 +2383,47 @@ int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) } EXPORT_SYMBOL_GPL(task_cgroup_path); +/** + * cgroup_attach_lock - Lock for ->attach() + * @lock_threadgroup: whether to down_write cgroup_threadgroup_rwsem + * + * cgroup migration sometimes needs to stabilize threadgroups against forks and + * exits by write-locking cgroup_threadgroup_rwsem. However, some ->attach() + * implementations (e.g. cpuset), also need to disable CPU hotplug. + * Unfortunately, letting ->attach() operations acquire cpus_read_lock() can + * lead to deadlocks. + * + * Bringing up a CPU may involve creating and destroying tasks which requires + * read-locking threadgroup_rwsem, so threadgroup_rwsem nests inside + * cpus_read_lock(). If we call an ->attach() which acquires the cpus lock while + * write-locking threadgroup_rwsem, the locking order is reversed and we end up + * waiting for an on-going CPU hotplug operation which in turn is waiting for + * the threadgroup_rwsem to be released to create new tasks. For more details: + * + * http://lkml.kernel.org/r/20220711174629.uehfmqegcwn2lqzu@wubuntu + * + * Resolve the situation by always acquiring cpus_read_lock() before optionally + * write-locking cgroup_threadgroup_rwsem. This allows ->attach() to assume that + * CPU hotplug is disabled on entry. + */ +void cgroup_attach_lock(bool lock_threadgroup) +{ + cpus_read_lock(); + if (lock_threadgroup) + percpu_down_write(&cgroup_threadgroup_rwsem); +} + +/** + * cgroup_attach_unlock - Undo cgroup_attach_lock() + * @lock_threadgroup: whether to up_write cgroup_threadgroup_rwsem + */ +void cgroup_attach_unlock(bool lock_threadgroup) +{ + if (lock_threadgroup) + percpu_up_write(&cgroup_threadgroup_rwsem); + cpus_read_unlock(); +} + /** * cgroup_migrate_add_task - add a migration target task to a migration context * @task: target task @@ -2841,8 +2896,7 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader, } struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, - bool *locked) - __acquires(&cgroup_threadgroup_rwsem) + bool *threadgroup_locked) { struct task_struct *tsk; pid_t pid; @@ -2859,12 +2913,8 @@ struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, * Therefore, we can skip the global lock. */ lockdep_assert_held(&cgroup_mutex); - if (pid || threadgroup) { - percpu_down_write(&cgroup_threadgroup_rwsem); - *locked = true; - } else { - *locked = false; - } + *threadgroup_locked = pid || threadgroup; + cgroup_attach_lock(*threadgroup_locked); rcu_read_lock(); if (pid) { @@ -2895,17 +2945,14 @@ struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup, goto out_unlock_rcu; out_unlock_threadgroup: - if (*locked) { - percpu_up_write(&cgroup_threadgroup_rwsem); - *locked = false; - } + cgroup_attach_unlock(*threadgroup_locked); + *threadgroup_locked = false; out_unlock_rcu: rcu_read_unlock(); return tsk; } -void cgroup_procs_write_finish(struct task_struct *task, bool locked) - __releases(&cgroup_threadgroup_rwsem) +void cgroup_procs_write_finish(struct task_struct *task, bool threadgroup_locked) { struct cgroup_subsys *ss; int ssid; @@ -2913,8 +2960,8 @@ void cgroup_procs_write_finish(struct task_struct *task, bool locked) /* release reference from cgroup_procs_write_start() */ put_task_struct(task); - if (locked) - percpu_up_write(&cgroup_threadgroup_rwsem); + cgroup_attach_unlock(threadgroup_locked); + for_each_subsys(ss, ssid) if (ss->post_attach) ss->post_attach(); @@ -3000,8 +3047,7 @@ static int cgroup_update_dfl_csses(struct cgroup *cgrp) * write-locking can be skipped safely. */ has_tasks = !list_empty(&mgctx.preloaded_src_csets); - if (has_tasks) - percpu_down_write(&cgroup_threadgroup_rwsem); + cgroup_attach_lock(has_tasks); /* NULL dst indicates self on default hierarchy */ ret = cgroup_migrate_prepare_dst(&mgctx); @@ -3022,8 +3068,7 @@ static int cgroup_update_dfl_csses(struct cgroup *cgrp) ret = cgroup_migrate_execute(&mgctx); out_finish: cgroup_migrate_finish(&mgctx); - if (has_tasks) - percpu_up_write(&cgroup_threadgroup_rwsem); + cgroup_attach_unlock(has_tasks); return ret; } @@ -3260,11 +3305,7 @@ static int cgroup_apply_control(struct cgroup *cgrp) * making the following cgroup_update_dfl_csses() properly update * css associations of all tasks in the subtree. */ - ret = cgroup_update_dfl_csses(cgrp); - if (ret) - return ret; - - return 0; + return cgroup_update_dfl_csses(cgrp); } /** @@ -3657,27 +3698,27 @@ static int cpu_stat_show(struct seq_file *seq, void *v) static int cgroup_io_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; + struct psi_group *psi = cgroup_psi(cgrp); return psi_show(seq, psi, PSI_IO); } static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; + struct psi_group *psi = cgroup_psi(cgrp); return psi_show(seq, psi, PSI_MEM); } static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) { struct cgroup *cgrp = seq_css(seq)->cgroup; - struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; + struct psi_group *psi = cgroup_psi(cgrp); return psi_show(seq, psi, PSI_CPU); } -static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, enum psi_res res) +static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, enum psi_res res) { struct cgroup_file_ctx *ctx = of->priv; struct psi_trigger *new; @@ -3697,8 +3738,8 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, return -EBUSY; } - psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi; - new = psi_trigger_create(psi, buf, nbytes, res); + psi = cgroup_psi(cgrp); + new = psi_trigger_create(psi, buf, res); if (IS_ERR(new)) { cgroup_put(cgrp); return PTR_ERR(new); @@ -3714,21 +3755,86 @@ static ssize_t cgroup_io_pressure_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - return cgroup_pressure_write(of, buf, nbytes, PSI_IO); + return pressure_write(of, buf, nbytes, PSI_IO); } static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - return cgroup_pressure_write(of, buf, nbytes, PSI_MEM); + return pressure_write(of, buf, nbytes, PSI_MEM); } static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - return cgroup_pressure_write(of, buf, nbytes, PSI_CPU); + return pressure_write(of, buf, nbytes, PSI_CPU); +} + +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +static int cgroup_irq_pressure_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + struct psi_group *psi = cgroup_psi(cgrp); + + return psi_show(seq, psi, PSI_IRQ); +} + +static ssize_t cgroup_irq_pressure_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + return pressure_write(of, buf, nbytes, PSI_IRQ); +} +#endif + +static int cgroup_pressure_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + struct psi_group *psi = cgroup_psi(cgrp); + + seq_printf(seq, "%d\n", psi->enabled); + + return 0; +} + +static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + ssize_t ret; + int enable; + struct cgroup *cgrp; + struct psi_group *psi; + + ret = kstrtoint(strstrip(buf), 0, &enable); + if (ret) + return ret; + + if (enable < 0 || enable > 1) + return -ERANGE; + + cgrp = cgroup_kn_lock_live(of->kn, false); + if (!cgrp) + return -ENOENT; + + psi = cgroup_psi(cgrp); + if (psi->enabled != enable) { + int i; + + /* show or hide {cpu,memory,io,irq}.pressure files */ + for (i = 0; i < NR_PSI_RESOURCES; i++) + cgroup_file_show(&cgrp->psi_files[i], enable); + + psi->enabled = enable; + if (enable) + psi_cgroup_restart(psi); + } + + cgroup_kn_unlock(of->kn); + + return nbytes; } static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of, @@ -3748,6 +3854,9 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) bool cgroup_psi_enabled(void) { + if (static_branch_likely(&psi_disabled)) + return false; + return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0; } @@ -4100,8 +4209,6 @@ static int cgroup_addrm_files(struct cgroup_subsys_state *css, restart: for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) { /* does cft->flags tell us to skip this file on @cgrp? */ - if ((cft->flags & CFTYPE_PRESSURE) && !cgroup_psi_enabled()) - continue; if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) continue; if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) @@ -4166,21 +4273,25 @@ static void cgroup_exit_cftypes(struct cftype *cfts) cft->ss = NULL; /* revert flags set by cgroup core while adding @cfts */ - cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); + cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL | + __CFTYPE_ADDED); } } static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) { struct cftype *cft; + int ret = 0; for (cft = cfts; cft->name[0] != '\0'; cft++) { struct kernfs_ops *kf_ops; WARN_ON(cft->ss || cft->kf_ops); - if ((cft->flags & CFTYPE_PRESSURE) && !cgroup_psi_enabled()) - continue; + if (cft->flags & __CFTYPE_ADDED) { + ret = -EBUSY; + break; + } if (cft->seq_start) kf_ops = &cgroup_kf_ops; @@ -4194,26 +4305,26 @@ static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); if (!kf_ops) { - cgroup_exit_cftypes(cfts); - return -ENOMEM; + ret = -ENOMEM; + break; } kf_ops->atomic_write_len = cft->max_write_len; } cft->kf_ops = kf_ops; cft->ss = ss; + cft->flags |= __CFTYPE_ADDED; } - return 0; + if (ret) + cgroup_exit_cftypes(cfts); + return ret; } static int cgroup_rm_cftypes_locked(struct cftype *cfts) { lockdep_assert_held(&cgroup_mutex); - if (!cfts || !cfts[0].ss) - return -ENOENT; - list_del(&cfts->node); cgroup_apply_cftypes(cfts, false); cgroup_exit_cftypes(cfts); @@ -4235,6 +4346,12 @@ int cgroup_rm_cftypes(struct cftype *cfts) { int ret; + if (!cfts || cfts[0].name[0] == '\0') + return 0; + + if (!(cfts[0].flags & __CFTYPE_ADDED)) + return -ENOENT; + mutex_lock(&cgroup_mutex); ret = cgroup_rm_cftypes_locked(cfts); mutex_unlock(&cgroup_mutex); @@ -4339,6 +4456,26 @@ void cgroup_file_notify(struct cgroup_file *cfile) spin_unlock_irqrestore(&cgroup_file_kn_lock, flags); } +/** + * cgroup_file_show - show or hide a hidden cgroup file + * @cfile: target cgroup_file obtained by setting cftype->file_offset + * @show: whether to show or hide + */ +void cgroup_file_show(struct cgroup_file *cfile, bool show) +{ + struct kernfs_node *kn; + + spin_lock_irq(&cgroup_file_kn_lock); + kn = cfile->kn; + kernfs_get(kn); + spin_unlock_irq(&cgroup_file_kn_lock); + + if (kn) + kernfs_show(kn, show); + + kernfs_put(kn); +} + /** * css_next_child - find the next child of a given css * @pos: the current position (%NULL to initiate traversal) @@ -4971,13 +5108,13 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, struct task_struct *task; const struct cred *saved_cred; ssize_t ret; - bool locked; + bool threadgroup_locked; dst_cgrp = cgroup_kn_lock_live(of->kn, false); if (!dst_cgrp) return -ENODEV; - task = cgroup_procs_write_start(buf, threadgroup, &locked); + task = cgroup_procs_write_start(buf, threadgroup, &threadgroup_locked); ret = PTR_ERR_OR_ZERO(task); if (ret) goto out_unlock; @@ -5003,7 +5140,7 @@ static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, ret = cgroup_attach_task(dst_cgrp, task, threadgroup); out_finish: - cgroup_procs_write_finish(task, locked); + cgroup_procs_write_finish(task, threadgroup_locked); out_unlock: cgroup_kn_unlock(of->kn); @@ -5099,10 +5236,14 @@ static struct cftype cgroup_base_files[] = { .name = "cpu.stat", .seq_show = cpu_stat_show, }, + { } /* terminate */ +}; + +static struct cftype cgroup_psi_files[] = { #ifdef CONFIG_PSI { .name = "io.pressure", - .flags = CFTYPE_PRESSURE, + .file_offset = offsetof(struct cgroup, psi_files[PSI_IO]), .seq_show = cgroup_io_pressure_show, .write = cgroup_io_pressure_write, .poll = cgroup_pressure_poll, @@ -5110,7 +5251,7 @@ static struct cftype cgroup_base_files[] = { }, { .name = "memory.pressure", - .flags = CFTYPE_PRESSURE, + .file_offset = offsetof(struct cgroup, psi_files[PSI_MEM]), .seq_show = cgroup_memory_pressure_show, .write = cgroup_memory_pressure_write, .poll = cgroup_pressure_poll, @@ -5118,12 +5259,27 @@ static struct cftype cgroup_base_files[] = { }, { .name = "cpu.pressure", - .flags = CFTYPE_PRESSURE, + .file_offset = offsetof(struct cgroup, psi_files[PSI_CPU]), .seq_show = cgroup_cpu_pressure_show, .write = cgroup_cpu_pressure_write, .poll = cgroup_pressure_poll, .release = cgroup_pressure_release, }, +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + { + .name = "irq.pressure", + .file_offset = offsetof(struct cgroup, psi_files[PSI_IRQ]), + .seq_show = cgroup_irq_pressure_show, + .write = cgroup_irq_pressure_write, + .poll = cgroup_pressure_poll, + .release = cgroup_pressure_release, + }, +#endif + { + .name = "cgroup.pressure", + .seq_show = cgroup_pressure_show, + .write = cgroup_pressure_write, + }, #endif /* CONFIG_PSI */ { } /* terminate */ }; @@ -5400,8 +5556,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, int ret; /* allocate the cgroup and its ID, 0 is reserved for the root */ - cgrp = kzalloc(struct_size(cgrp, ancestor_ids, (level + 1)), - GFP_KERNEL); + cgrp = kzalloc(struct_size(cgrp, ancestors, (level + 1)), GFP_KERNEL); if (!cgrp) return ERR_PTR(-ENOMEM); @@ -5453,7 +5608,7 @@ static struct cgroup *cgroup_create(struct cgroup *parent, const char *name, spin_lock_irq(&css_set_lock); for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) { - cgrp->ancestor_ids[tcgrp->level] = cgroup_id(tcgrp); + cgrp->ancestors[tcgrp->level] = tcgrp; if (tcgrp != cgrp) { tcgrp->nr_descendants++; @@ -5886,6 +6041,7 @@ int __init cgroup_init(void) BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); BUG_ON(cgroup_init_cftypes(NULL, cgroup_base_files)); + BUG_ON(cgroup_init_cftypes(NULL, cgroup_psi_files)); BUG_ON(cgroup_init_cftypes(NULL, cgroup1_base_files)); cgroup_rstat_boot(); @@ -6006,16 +6162,22 @@ void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) /* * cgroup_get_from_id : get the cgroup associated with cgroup id * @id: cgroup id - * On success return the cgrp, on failure return NULL + * On success return the cgrp or ERR_PTR on failure + * Only cgroups within current task's cgroup NS are valid. */ struct cgroup *cgroup_get_from_id(u64 id) { struct kernfs_node *kn; - struct cgroup *cgrp = NULL; + struct cgroup *cgrp, *root_cgrp; kn = kernfs_find_and_get_node_by_id(cgrp_dfl_root.kf_root, id); if (!kn) - goto out; + return ERR_PTR(-ENOENT); + + if (kernfs_type(kn) != KERNFS_DIR) { + kernfs_put(kn); + return ERR_PTR(-ENOENT); + } rcu_read_lock(); @@ -6024,9 +6186,19 @@ struct cgroup *cgroup_get_from_id(u64 id) cgrp = NULL; rcu_read_unlock(); - kernfs_put(kn); -out: + + if (!cgrp) + return ERR_PTR(-ENOENT); + + spin_lock_irq(&css_set_lock); + root_cgrp = current_cgns_cgroup_from_root(&cgrp_dfl_root); + spin_unlock_irq(&css_set_lock); + if (!cgroup_is_descendant(cgrp, root_cgrp)) { + cgroup_put(cgrp); + return ERR_PTR(-ENOENT); + } + return cgrp; } EXPORT_SYMBOL_GPL(cgroup_get_from_id); @@ -6056,7 +6228,7 @@ int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, struct cgroup *cgrp; int ssid, count = 0; - if (root == &cgrp_dfl_root && !cgrp_dfl_visible) + if (root == &cgrp_dfl_root && !READ_ONCE(cgrp_dfl_visible)) continue; seq_printf(m, "%d:", root->hierarchy_id); @@ -6132,11 +6304,6 @@ static struct cgroup *cgroup_get_from_file(struct file *f) return ERR_CAST(css); cgrp = css->cgroup; - if (!cgroup_on_dfl(cgrp)) { - cgroup_put(cgrp); - return ERR_PTR(-EBADF); - } - return cgrp; } @@ -6603,8 +6770,12 @@ struct cgroup *cgroup_get_from_path(const char *path) { struct kernfs_node *kn; struct cgroup *cgrp = ERR_PTR(-ENOENT); + struct cgroup *root_cgrp; - kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path); + spin_lock_irq(&css_set_lock); + root_cgrp = current_cgns_cgroup_from_root(&cgrp_dfl_root); + kn = kernfs_walk_and_get(root_cgrp->kn, path); + spin_unlock_irq(&css_set_lock); if (!kn) goto out; @@ -6762,9 +6933,6 @@ static ssize_t show_delegatable_files(struct cftype *files, char *buf, if (!(cft->flags & CFTYPE_NS_DELEGATABLE)) continue; - if ((cft->flags & CFTYPE_PRESSURE) && !cgroup_psi_enabled()) - continue; - if (prefix) ret += snprintf(buf + ret, size - ret, "%s.", prefix); @@ -6784,8 +6952,11 @@ static ssize_t delegate_show(struct kobject *kobj, struct kobj_attribute *attr, int ssid; ssize_t ret = 0; - ret = show_delegatable_files(cgroup_base_files, buf, PAGE_SIZE - ret, - NULL); + ret = show_delegatable_files(cgroup_base_files, buf + ret, + PAGE_SIZE - ret, NULL); + if (cgroup_psi_enabled()) + ret += show_delegatable_files(cgroup_psi_files, buf + ret, + PAGE_SIZE - ret, NULL); for_each_subsys(ss, ssid) ret += show_delegatable_files(ss->dfl_cftypes, buf + ret, diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 58aadfda9b8b3204e251bc86da1353df29df2dee..b474289c15b821c95e8fbacb9961e8c1ac37cd66 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,30 @@ struct fmeter { spinlock_t lock; /* guards read or write of above */ }; +/* + * Invalid partition error code + */ +enum prs_errcode { + PERR_NONE = 0, + PERR_INVCPUS, + PERR_INVPARENT, + PERR_NOTPART, + PERR_NOTEXCL, + PERR_NOCPUS, + PERR_HOTPLUG, + PERR_CPUSEMPTY, +}; + +static const char * const perr_strings[] = { + [PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus", + [PERR_INVPARENT] = "Parent is an invalid partition root", + [PERR_NOTPART] = "Parent is not a partition root", + [PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive", + [PERR_NOCPUS] = "Parent unable to distribute cpu downstream", + [PERR_HOTPLUG] = "No cpu available due to hotplug", + [PERR_CPUSEMPTY] = "cpuset.cpus is empty", +}; + struct cpuset { struct cgroup_subsys_state css; @@ -168,6 +193,9 @@ struct cpuset { int use_parent_ecpus; int child_ecpus_count; + /* Invalid partition error code, not lock protected */ + enum prs_errcode prs_err; + /* Handle for cpuset.cpus.partition */ struct cgroup_file partition_file; }; @@ -175,20 +203,22 @@ struct cpuset { /* * Partition root states: * - * 0 - not a partition root - * + * 0 - member (not a partition root) * 1 - partition root - * + * 2 - partition root without load balancing (isolated) * -1 - invalid partition root - * None of the cpus in cpus_allowed can be put into the parent's - * subparts_cpus. In this case, the cpuset is not a real partition - * root anymore. However, the CPU_EXCLUSIVE bit will still be set - * and the cpuset can be restored back to a partition root if the - * parent cpuset can give more CPUs back to this child cpuset. + * -2 - invalid isolated partition root */ -#define PRS_DISABLED 0 -#define PRS_ENABLED 1 -#define PRS_ERROR -1 +#define PRS_MEMBER 0 +#define PRS_ROOT 1 +#define PRS_ISOLATED 2 +#define PRS_INVALID_ROOT -1 +#define PRS_INVALID_ISOLATED -2 + +static inline bool is_prs_invalid(int prs_state) +{ + return prs_state < 0; +} /* * Temporary cpumasks for working with partitions that are passed among @@ -268,25 +298,43 @@ static inline int is_spread_slab(const struct cpuset *cs) return test_bit(CS_SPREAD_SLAB, &cs->flags); } -static inline int is_partition_root(const struct cpuset *cs) +static inline int is_partition_valid(const struct cpuset *cs) { return cs->partition_root_state > 0; } +static inline int is_partition_invalid(const struct cpuset *cs) +{ + return cs->partition_root_state < 0; +} + +/* + * Callers should hold callback_lock to modify partition_root_state. + */ +static inline void make_partition_invalid(struct cpuset *cs) +{ + if (is_partition_valid(cs)) + cs->partition_root_state = -cs->partition_root_state; +} + /* * Send notification event of whenever partition_root_state changes. */ -static inline void notify_partition_change(struct cpuset *cs, - int old_prs, int new_prs) +static inline void notify_partition_change(struct cpuset *cs, int old_prs) { - if (old_prs != new_prs) - cgroup_file_notify(&cs->partition_file); + if (old_prs == cs->partition_root_state) + return; + cgroup_file_notify(&cs->partition_file); + + /* Reset prs_err if not invalid */ + if (is_partition_valid(cs)) + WRITE_ONCE(cs->prs_err, PERR_NONE); } static struct cpuset top_cpuset = { .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)), - .partition_root_state = PRS_ENABLED, + .partition_root_state = PRS_ROOT, }; /** @@ -404,6 +452,41 @@ static inline bool is_in_v2_mode(void) (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); } +/** + * partition_is_populated - check if partition has tasks + * @cs: partition root to be checked + * @excluded_child: a child cpuset to be excluded in task checking + * Return: true if there are tasks, false otherwise + * + * It is assumed that @cs is a valid partition root. @excluded_child should + * be non-NULL when this cpuset is going to become a partition itself. + */ +static inline bool partition_is_populated(struct cpuset *cs, + struct cpuset *excluded_child) +{ + struct cgroup_subsys_state *css; + struct cpuset *child; + + if (cs->css.cgroup->nr_populated_csets) + return true; + if (!excluded_child && !cs->nr_subparts_cpus) + return cgroup_is_populated(cs->css.cgroup); + + rcu_read_lock(); + cpuset_for_each_child(child, css, cs) { + if (child == excluded_child) + continue; + if (is_partition_valid(child)) + continue; + if (cgroup_is_populated(child->css.cgroup)) { + rcu_read_unlock(); + return true; + } + } + rcu_read_unlock(); + return false; +} + /* * Return in pmask the portion of a task's cpusets's cpus_allowed that * are online and are capable of running the task. If none are found, @@ -658,22 +741,6 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) par = parent_cs(cur); - /* - * If either I or some sibling (!= me) is exclusive, we can't - * overlap - */ - ret = -EINVAL; - cpuset_for_each_child(c, css, par) { - if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && - c != cur && - cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) - goto out; - if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && - c != cur && - nodes_intersects(trial->mems_allowed, c->mems_allowed)) - goto out; - } - /* * Cpusets with tasks - existing or newly being attached - can't * be changed to have empty cpus_allowed or mems_allowed. @@ -698,6 +765,22 @@ static int validate_change(struct cpuset *cur, struct cpuset *trial) trial->cpus_allowed)) goto out; + /* + * If either I or some sibling (!= me) is exclusive, we can't + * overlap + */ + ret = -EINVAL; + cpuset_for_each_child(c, css, par) { + if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && + c != cur && + cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) + goto out; + if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && + c != cur && + nodes_intersects(trial->mems_allowed, c->mems_allowed)) + goto out; + } + ret = 0; out: rcu_read_unlock(); @@ -875,7 +958,7 @@ static int generate_sched_domains(cpumask_var_t **domains, csa[csn++] = cp; /* skip @cp's subtree if not a partition root */ - if (!is_partition_root(cp)) + if (!is_partition_valid(cp)) pos_css = css_rightmost_descendant(pos_css); } rcu_read_unlock(); @@ -1081,7 +1164,7 @@ static void rebuild_sched_domains_locked(void) if (top_cpuset.nr_subparts_cpus) { rcu_read_lock(); cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { - if (!is_partition_root(cs)) { + if (!is_partition_valid(cs)) { pos_css = css_rightmost_descendant(pos_css); continue; } @@ -1127,10 +1210,18 @@ static void update_tasks_cpumask(struct cpuset *cs) { struct css_task_iter it; struct task_struct *task; + bool top_cs = cs == &top_cpuset; css_task_iter_start(&cs->css, 0, &it); - while ((task = css_task_iter_next(&it))) + while ((task = css_task_iter_next(&it))) { + /* + * Percpu kthreads in top_cpuset are ignored + */ + if (top_cs && (task->flags & PF_KTHREAD) && + kthread_is_per_cpu(task)) + continue; set_cpus_allowed_ptr(task, cs->effective_cpus); + } css_task_iter_end(&it); } @@ -1165,15 +1256,18 @@ enum subparts_cmd { partcmd_enable, /* Enable partition root */ partcmd_disable, /* Disable partition root */ partcmd_update, /* Update parent's subparts_cpus */ + partcmd_invalidate, /* Make partition invalid */ }; +static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, + int turning_on); /** * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset * @cpuset: The cpuset that requests change in partition root state * @cmd: Partition root state change command * @newmask: Optional new cpumask for partcmd_update * @tmp: Temporary addmask and delmask - * Return: 0, 1 or an error code + * Return: 0 or a partition root state error code * * For partcmd_enable, the cpuset is being transformed from a non-partition * root to a partition root. The cpus_allowed mask of the given cpuset will @@ -1184,38 +1278,36 @@ enum subparts_cmd { * For partcmd_disable, the cpuset is being transformed from a partition * root back to a non-partition root. Any CPUs in cpus_allowed that are in * parent's subparts_cpus will be taken away from that cpumask and put back - * into parent's effective_cpus. 0 should always be returned. + * into parent's effective_cpus. 0 will always be returned. * - * For partcmd_update, if the optional newmask is specified, the cpu - * list is to be changed from cpus_allowed to newmask. Otherwise, - * cpus_allowed is assumed to remain the same. The cpuset should either - * be a partition root or an invalid partition root. The partition root - * state may change if newmask is NULL and none of the requested CPUs can - * be granted by the parent. The function will return 1 if changes to - * parent's subparts_cpus and effective_cpus happen or 0 otherwise. - * Error code should only be returned when newmask is non-NULL. + * For partcmd_update, if the optional newmask is specified, the cpu list is + * to be changed from cpus_allowed to newmask. Otherwise, cpus_allowed is + * assumed to remain the same. The cpuset should either be a valid or invalid + * partition root. The partition root state may change from valid to invalid + * or vice versa. An error code will only be returned if transitioning from + * invalid to valid violates the exclusivity rule. * - * The partcmd_enable and partcmd_disable commands are used by - * update_prstate(). The partcmd_update command is used by - * update_cpumasks_hier() with newmask NULL and update_cpumask() with - * newmask set. - * - * The checking is more strict when enabling partition root than the - * other two commands. + * For partcmd_invalidate, the current partition will be made invalid. * - * Because of the implicit cpu exclusive nature of a partition root, - * cpumask changes that violates the cpu exclusivity rule will not be - * permitted when checked by validate_change(). + * The partcmd_enable and partcmd_disable commands are used by + * update_prstate(). An error code may be returned and the caller will check + * for error. + * + * The partcmd_update command is used by update_cpumasks_hier() with newmask + * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used + * by update_cpumask() with NULL newmask. In both cases, the callers won't + * check for error and so partition_root_state and prs_error will be updated + * directly. */ -static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd, +static int update_parent_subparts_cpumask(struct cpuset *cs, int cmd, struct cpumask *newmask, struct tmpmasks *tmp) { - struct cpuset *parent = parent_cs(cpuset); + struct cpuset *parent = parent_cs(cs); int adding; /* Moving cpus from effective_cpus to subparts_cpus */ int deleting; /* Moving cpus from subparts_cpus to effective_cpus */ int old_prs, new_prs; - bool part_error = false; /* Partition error? */ + int part_error = PERR_NONE; /* Partition error? */ percpu_rwsem_assert_held(&cpuset_rwsem); @@ -1224,125 +1316,164 @@ static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd, * The new cpumask, if present, or the current cpus_allowed must * not be empty. */ - if (!is_partition_root(parent) || - (newmask && cpumask_empty(newmask)) || - (!newmask && cpumask_empty(cpuset->cpus_allowed))) - return -EINVAL; - - /* - * Enabling/disabling partition root is not allowed if there are - * online children. - */ - if ((cmd != partcmd_update) && css_has_online_children(&cpuset->css)) - return -EBUSY; - - /* - * Enabling partition root is not allowed if not all the CPUs - * can be granted from parent's effective_cpus or at least one - * CPU will be left after that. - */ - if ((cmd == partcmd_enable) && - (!cpumask_subset(cpuset->cpus_allowed, parent->effective_cpus) || - cpumask_equal(cpuset->cpus_allowed, parent->effective_cpus))) - return -EINVAL; + if (!is_partition_valid(parent)) { + return is_partition_invalid(parent) + ? PERR_INVPARENT : PERR_NOTPART; + } + if ((newmask && cpumask_empty(newmask)) || + (!newmask && cpumask_empty(cs->cpus_allowed))) + return PERR_CPUSEMPTY; /* - * A cpumask update cannot make parent's effective_cpus become empty. + * new_prs will only be changed for the partcmd_update and + * partcmd_invalidate commands. */ adding = deleting = false; - old_prs = new_prs = cpuset->partition_root_state; + old_prs = new_prs = cs->partition_root_state; if (cmd == partcmd_enable) { - cpumask_copy(tmp->addmask, cpuset->cpus_allowed); + /* + * Enabling partition root is not allowed if cpus_allowed + * doesn't overlap parent's cpus_allowed. + */ + if (!cpumask_intersects(cs->cpus_allowed, parent->cpus_allowed)) + return PERR_INVCPUS; + + /* + * A parent can be left with no CPU as long as there is no + * task directly associated with the parent partition. + */ + if (!cpumask_intersects(cs->cpus_allowed, parent->effective_cpus) && + partition_is_populated(parent, cs)) + return PERR_NOCPUS; + + cpumask_copy(tmp->addmask, cs->cpus_allowed); adding = true; } else if (cmd == partcmd_disable) { - deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, + /* + * Need to remove cpus from parent's subparts_cpus for valid + * partition root. + */ + deleting = !is_prs_invalid(old_prs) && + cpumask_and(tmp->delmask, cs->cpus_allowed, + parent->subparts_cpus); + } else if (cmd == partcmd_invalidate) { + if (is_prs_invalid(old_prs)) + return 0; + + /* + * Make the current partition invalid. It is assumed that + * invalidation is caused by violating cpu exclusivity rule. + */ + deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, parent->subparts_cpus); + if (old_prs > 0) { + new_prs = -old_prs; + part_error = PERR_NOTEXCL; + } } else if (newmask) { /* * partcmd_update with newmask: * + * Compute add/delete mask to/from subparts_cpus + * * delmask = cpus_allowed & ~newmask & parent->subparts_cpus - * addmask = newmask & parent->effective_cpus + * addmask = newmask & parent->cpus_allowed * & ~parent->subparts_cpus */ - cpumask_andnot(tmp->delmask, cpuset->cpus_allowed, newmask); + cpumask_andnot(tmp->delmask, cs->cpus_allowed, newmask); deleting = cpumask_and(tmp->delmask, tmp->delmask, parent->subparts_cpus); - cpumask_and(tmp->addmask, newmask, parent->effective_cpus); + cpumask_and(tmp->addmask, newmask, parent->cpus_allowed); adding = cpumask_andnot(tmp->addmask, tmp->addmask, parent->subparts_cpus); /* - * Return error if the new effective_cpus could become empty. + * Make partition invalid if parent's effective_cpus could + * become empty and there are tasks in the parent. */ if (adding && - cpumask_equal(parent->effective_cpus, tmp->addmask)) { - if (!deleting) - return -EINVAL; - /* - * As some of the CPUs in subparts_cpus might have - * been offlined, we need to compute the real delmask - * to confirm that. - */ - if (!cpumask_and(tmp->addmask, tmp->delmask, - cpu_active_mask)) - return -EINVAL; - cpumask_copy(tmp->addmask, parent->effective_cpus); + cpumask_subset(parent->effective_cpus, tmp->addmask) && + !cpumask_intersects(tmp->delmask, cpu_active_mask) && + partition_is_populated(parent, cs)) { + part_error = PERR_NOCPUS; + adding = false; + deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, + parent->subparts_cpus); } } else { /* * partcmd_update w/o newmask: * - * addmask = cpus_allowed & parent->effective_cpus + * delmask = cpus_allowed & parent->subparts_cpus + * addmask = cpus_allowed & parent->cpus_allowed + * & ~parent->subparts_cpus * - * Note that parent's subparts_cpus may have been - * pre-shrunk in case there is a change in the cpu list. - * So no deletion is needed. + * This gets invoked either due to a hotplug event or from + * update_cpumasks_hier(). This can cause the state of a + * partition root to transition from valid to invalid or vice + * versa. So we still need to compute the addmask and delmask. + + * A partition error happens when: + * 1) Cpuset is valid partition, but parent does not distribute + * out any CPUs. + * 2) Parent has tasks and all its effective CPUs will have + * to be distributed out. */ - adding = cpumask_and(tmp->addmask, cpuset->cpus_allowed, - parent->effective_cpus); - part_error = cpumask_equal(tmp->addmask, - parent->effective_cpus); + cpumask_and(tmp->addmask, cs->cpus_allowed, + parent->cpus_allowed); + adding = cpumask_andnot(tmp->addmask, tmp->addmask, + parent->subparts_cpus); + + if ((is_partition_valid(cs) && !parent->nr_subparts_cpus) || + (adding && + cpumask_subset(parent->effective_cpus, tmp->addmask) && + partition_is_populated(parent, cs))) { + part_error = PERR_NOCPUS; + adding = false; + } + + if (part_error && is_partition_valid(cs) && + parent->nr_subparts_cpus) + deleting = cpumask_and(tmp->delmask, cs->cpus_allowed, + parent->subparts_cpus); } + if (part_error) + WRITE_ONCE(cs->prs_err, part_error); if (cmd == partcmd_update) { - int prev_prs = cpuset->partition_root_state; - /* - * Check for possible transition between PRS_ENABLED - * and PRS_ERROR. + * Check for possible transition between valid and invalid + * partition root. */ - switch (cpuset->partition_root_state) { - case PRS_ENABLED: + switch (cs->partition_root_state) { + case PRS_ROOT: + case PRS_ISOLATED: if (part_error) - new_prs = PRS_ERROR; + new_prs = -old_prs; break; - case PRS_ERROR: + case PRS_INVALID_ROOT: + case PRS_INVALID_ISOLATED: if (!part_error) - new_prs = PRS_ENABLED; + new_prs = -old_prs; break; } - /* - * Set part_error if previously in invalid state. - */ - part_error = (prev_prs == PRS_ERROR); - } - - if (!part_error && (new_prs == PRS_ERROR)) - return 0; /* Nothing need to be done */ - - if (new_prs == PRS_ERROR) { - /* - * Remove all its cpus from parent's subparts_cpus. - */ - adding = false; - deleting = cpumask_and(tmp->delmask, cpuset->cpus_allowed, - parent->subparts_cpus); } if (!adding && !deleting && (new_prs == old_prs)) return 0; + /* + * Transitioning between invalid to valid or vice versa may require + * changing CS_CPU_EXCLUSIVE and CS_SCHED_LOAD_BALANCE. + */ + if (old_prs != new_prs) { + if (is_prs_invalid(old_prs) && !is_cpu_exclusive(cs) && + (update_flag(CS_CPU_EXCLUSIVE, cs, 1) < 0)) + return PERR_NOTEXCL; + if (is_prs_invalid(new_prs) && is_cpu_exclusive(cs)) + update_flag(CS_CPU_EXCLUSIVE, cs, 0); + } + /* * Change the parent's subparts_cpus. * Newly added CPUs will be removed from effective_cpus and @@ -1369,18 +1500,32 @@ static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd, parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus); if (old_prs != new_prs) - cpuset->partition_root_state = new_prs; + cs->partition_root_state = new_prs; spin_unlock_irq(&callback_lock); - notify_partition_change(cpuset, old_prs, new_prs); - return cmd == partcmd_update; + if (adding || deleting) + update_tasks_cpumask(parent); + + /* + * Set or clear CS_SCHED_LOAD_BALANCE when partcmd_update, if necessary. + * rebuild_sched_domains_locked() may be called. + */ + if (old_prs != new_prs) { + if (old_prs == PRS_ISOLATED) + update_flag(CS_SCHED_LOAD_BALANCE, cs, 1); + else if (new_prs == PRS_ISOLATED) + update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); + } + notify_partition_change(cs, old_prs); + return 0; } /* * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree * @cs: the cpuset to consider * @tmp: temp variables for calculating effective_cpus & partition setup + * @force: don't skip any descendant cpusets if set * * When configured cpumask is changed, the effective cpumasks of this cpuset * and all its descendants need to be updated. @@ -1389,7 +1534,8 @@ static int update_parent_subparts_cpumask(struct cpuset *cpuset, int cmd, * * Called with cpuset_rwsem held */ -static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) +static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, + bool force) { struct cpuset *cp; struct cgroup_subsys_state *pos_css; @@ -1399,14 +1545,21 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) rcu_read_lock(); cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); + bool update_parent = false; compute_effective_cpumask(tmp->new_cpus, cp, parent); /* * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs. + * parent, which is guaranteed to have some CPUs unless + * it is a partition root that has explicitly distributed + * out all its CPUs. */ if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) { + if (is_partition_valid(cp) && + cpumask_equal(cp->cpus_allowed, cp->subparts_cpus)) + goto update_parent_subparts; + cpumask_copy(tmp->new_cpus, parent->effective_cpus); if (!cp->use_parent_ecpus) { cp->use_parent_ecpus = true; @@ -1420,14 +1573,15 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) /* * Skip the whole subtree if the cpumask remains the same - * and has no partition root state. + * and has no partition root state and force flag not set. */ - if (!cp->partition_root_state && + if (!cp->partition_root_state && !force && cpumask_equal(tmp->new_cpus, cp->effective_cpus)) { pos_css = css_rightmost_descendant(pos_css); continue; } +update_parent_subparts: /* * update_parent_subparts_cpumask() should have been called * for cs already in update_cpumask(). We should also call @@ -1437,36 +1591,22 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) old_prs = new_prs = cp->partition_root_state; if ((cp != cs) && old_prs) { switch (parent->partition_root_state) { - case PRS_DISABLED: - /* - * If parent is not a partition root or an - * invalid partition root, clear its state - * and its CS_CPU_EXCLUSIVE flag. - */ - WARN_ON_ONCE(cp->partition_root_state - != PRS_ERROR); - new_prs = PRS_DISABLED; - - /* - * clear_bit() is an atomic operation and - * readers aren't interested in the state - * of CS_CPU_EXCLUSIVE anyway. So we can - * just update the flag without holding - * the callback_lock. - */ - clear_bit(CS_CPU_EXCLUSIVE, &cp->flags); + case PRS_ROOT: + case PRS_ISOLATED: + update_parent = true; break; - case PRS_ENABLED: - if (update_parent_subparts_cpumask(cp, partcmd_update, NULL, tmp)) - update_tasks_cpumask(parent); - break; - - case PRS_ERROR: + default: /* - * When parent is invalid, it has to be too. + * When parent is not a partition root or is + * invalid, child partition roots become + * invalid too. */ - new_prs = PRS_ERROR; + if (is_partition_valid(cp)) + new_prs = -cp->partition_root_state; + WRITE_ONCE(cp->prs_err, + is_partition_invalid(parent) + ? PERR_INVPARENT : PERR_NOTPART); break; } } @@ -1475,42 +1615,44 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) continue; rcu_read_unlock(); + if (update_parent) { + update_parent_subparts_cpumask(cp, partcmd_update, NULL, + tmp); + /* + * The cpuset partition_root_state may become + * invalid. Capture it. + */ + new_prs = cp->partition_root_state; + } + spin_lock_irq(&callback_lock); - cpumask_copy(cp->effective_cpus, tmp->new_cpus); - if (cp->nr_subparts_cpus && (new_prs != PRS_ENABLED)) { + if (cp->nr_subparts_cpus && !is_partition_valid(cp)) { + /* + * Put all active subparts_cpus back to effective_cpus. + */ + cpumask_or(tmp->new_cpus, tmp->new_cpus, + cp->subparts_cpus); + cpumask_and(tmp->new_cpus, tmp->new_cpus, + cpu_active_mask); cp->nr_subparts_cpus = 0; cpumask_clear(cp->subparts_cpus); - } else if (cp->nr_subparts_cpus) { + } + + cpumask_copy(cp->effective_cpus, tmp->new_cpus); + if (cp->nr_subparts_cpus) { /* * Make sure that effective_cpus & subparts_cpus * are mutually exclusive. - * - * In the unlikely event that effective_cpus - * becomes empty. we clear cp->nr_subparts_cpus and - * let its child partition roots to compete for - * CPUs again. */ cpumask_andnot(cp->effective_cpus, cp->effective_cpus, cp->subparts_cpus); - if (cpumask_empty(cp->effective_cpus)) { - cpumask_copy(cp->effective_cpus, tmp->new_cpus); - cpumask_clear(cp->subparts_cpus); - cp->nr_subparts_cpus = 0; - } else if (!cpumask_subset(cp->subparts_cpus, - tmp->new_cpus)) { - cpumask_andnot(cp->subparts_cpus, - cp->subparts_cpus, tmp->new_cpus); - cp->nr_subparts_cpus - = cpumask_weight(cp->subparts_cpus); - } } - if (new_prs != old_prs) - cp->partition_root_state = new_prs; - + cp->partition_root_state = new_prs; spin_unlock_irq(&callback_lock); - notify_partition_change(cp, old_prs, new_prs); + + notify_partition_change(cp, old_prs); WARN_ON(!is_in_v2_mode() && !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); @@ -1526,7 +1668,7 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp) if (!cpumask_empty(cp->cpus_allowed) && is_sched_load_balance(cp) && (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) || - is_partition_root(cp))) + is_partition_valid(cp))) need_rebuild_sched_domains = true; rcu_read_lock(); @@ -1570,7 +1712,7 @@ static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs, continue; rcu_read_unlock(); - update_cpumasks_hier(sibling, tmp); + update_cpumasks_hier(sibling, tmp, false); rcu_read_lock(); css_put(&sibling->css); } @@ -1588,6 +1730,7 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, { int retval; struct tmpmasks tmp; + bool invalidate = false; /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ if (cs == &top_cpuset) @@ -1615,10 +1758,6 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) return 0; - retval = validate_change(cs, trialcs); - if (retval < 0) - return retval; - #ifdef CONFIG_CPUMASK_OFFSTACK /* * Use the cpumasks in trialcs for tmpmasks when they are pointers @@ -1629,28 +1768,70 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, tmp.new_cpus = trialcs->cpus_allowed; #endif + retval = validate_change(cs, trialcs); + + if ((retval == -EINVAL) && cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { + struct cpuset *cp, *parent; + struct cgroup_subsys_state *css; + + /* + * The -EINVAL error code indicates that partition sibling + * CPU exclusivity rule has been violated. We still allow + * the cpumask change to proceed while invalidating the + * partition. However, any conflicting sibling partitions + * have to be marked as invalid too. + */ + invalidate = true; + rcu_read_lock(); + parent = parent_cs(cs); + cpuset_for_each_child(cp, css, parent) + if (is_partition_valid(cp) && + cpumask_intersects(trialcs->cpus_allowed, cp->cpus_allowed)) { + rcu_read_unlock(); + update_parent_subparts_cpumask(cp, partcmd_invalidate, NULL, &tmp); + rcu_read_lock(); + } + rcu_read_unlock(); + retval = 0; + } + if (retval < 0) + return retval; + if (cs->partition_root_state) { - /* Cpumask of a partition root cannot be empty */ - if (cpumask_empty(trialcs->cpus_allowed)) - return -EINVAL; - if (update_parent_subparts_cpumask(cs, partcmd_update, - trialcs->cpus_allowed, &tmp) < 0) - return -EINVAL; + if (invalidate) + update_parent_subparts_cpumask(cs, partcmd_invalidate, + NULL, &tmp); + else + update_parent_subparts_cpumask(cs, partcmd_update, + trialcs->cpus_allowed, &tmp); } + compute_effective_cpumask(trialcs->effective_cpus, trialcs, + parent_cs(cs)); spin_lock_irq(&callback_lock); cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); /* - * Make sure that subparts_cpus is a subset of cpus_allowed. + * Make sure that subparts_cpus, if not empty, is a subset of + * cpus_allowed. Clear subparts_cpus if partition not valid or + * empty effective cpus with tasks. */ if (cs->nr_subparts_cpus) { - cpumask_and(cs->subparts_cpus, cs->subparts_cpus, cs->cpus_allowed); - cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus); + if (!is_partition_valid(cs) || + (cpumask_subset(trialcs->effective_cpus, cs->subparts_cpus) && + partition_is_populated(cs, NULL))) { + cs->nr_subparts_cpus = 0; + cpumask_clear(cs->subparts_cpus); + } else { + cpumask_and(cs->subparts_cpus, cs->subparts_cpus, + cs->cpus_allowed); + cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus); + } } spin_unlock_irq(&callback_lock); - update_cpumasks_hier(cs, &tmp); + /* effective_cpus will be updated here */ + update_cpumasks_hier(cs, &tmp, false); if (cs->partition_root_state) { struct cpuset *parent = parent_cs(cs); @@ -2026,16 +2207,18 @@ out: return err; } -/* +/** * update_prstate - update partition_root_state - * cs: the cpuset to update - * new_prs: new partition root state + * @cs: the cpuset to update + * @new_prs: new partition root state + * Return: 0 if successful, != 0 if error * * Call with cpuset_rwsem held. */ static int update_prstate(struct cpuset *cs, int new_prs) { - int err, old_prs = cs->partition_root_state; + int err = PERR_NONE, old_prs = cs->partition_root_state; + bool sched_domain_rebuilt = false; struct cpuset *parent = parent_cs(cs); struct tmpmasks tmpmask; @@ -2043,28 +2226,33 @@ static int update_prstate(struct cpuset *cs, int new_prs) return 0; /* - * Cannot force a partial or invalid partition root to a full - * partition root. + * For a previously invalid partition root, leave it at being + * invalid if new_prs is not "member". */ - if (new_prs && (old_prs == PRS_ERROR)) - return -EINVAL; + if (new_prs && is_prs_invalid(old_prs)) { + cs->partition_root_state = -new_prs; + return 0; + } if (alloc_cpumasks(NULL, &tmpmask)) return -ENOMEM; - err = -EINVAL; if (!old_prs) { /* * Turning on partition root requires setting the * CS_CPU_EXCLUSIVE bit implicitly as well and cpus_allowed - * cannot be NULL. + * cannot be empty. */ - if (cpumask_empty(cs->cpus_allowed)) + if (cpumask_empty(cs->cpus_allowed)) { + err = PERR_CPUSEMPTY; goto out; + } err = update_flag(CS_CPU_EXCLUSIVE, cs, 1); - if (err) + if (err) { + err = PERR_NOTEXCL; goto out; + } err = update_parent_subparts_cpumask(cs, partcmd_enable, NULL, &tmpmask); @@ -2072,47 +2260,77 @@ static int update_prstate(struct cpuset *cs, int new_prs) update_flag(CS_CPU_EXCLUSIVE, cs, 0); goto out; } + + if (new_prs == PRS_ISOLATED) { + /* + * Disable the load balance flag should not return an + * error unless the system is running out of memory. + */ + update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); + sched_domain_rebuilt = true; + } + } else if (old_prs && new_prs) { + /* + * A change in load balance state only, no change in cpumasks. + */ + update_flag(CS_SCHED_LOAD_BALANCE, cs, (new_prs != PRS_ISOLATED)); + sched_domain_rebuilt = true; + goto out; /* Sched domain is rebuilt in update_flag() */ } else { /* - * Turning off partition root will clear the - * CS_CPU_EXCLUSIVE bit. + * Switching back to member is always allowed even if it + * disables child partitions. */ - if (old_prs == PRS_ERROR) { - update_flag(CS_CPU_EXCLUSIVE, cs, 0); - err = 0; - goto out; - } + update_parent_subparts_cpumask(cs, partcmd_disable, NULL, + &tmpmask); - err = update_parent_subparts_cpumask(cs, partcmd_disable, - NULL, &tmpmask); - if (err) - goto out; + /* + * If there are child partitions, they will all become invalid. + */ + if (unlikely(cs->nr_subparts_cpus)) { + spin_lock_irq(&callback_lock); + cs->nr_subparts_cpus = 0; + cpumask_clear(cs->subparts_cpus); + compute_effective_cpumask(cs->effective_cpus, cs, parent); + spin_unlock_irq(&callback_lock); + } /* Turning off CS_CPU_EXCLUSIVE will not return error */ update_flag(CS_CPU_EXCLUSIVE, cs, 0); + + if (!is_sched_load_balance(cs)) { + /* Make sure load balance is on */ + update_flag(CS_SCHED_LOAD_BALANCE, cs, 1); + sched_domain_rebuilt = true; + } } - /* - * Update cpumask of parent's tasks except when it is the top - * cpuset as some system daemons cannot be mapped to other CPUs. - */ - if (parent != &top_cpuset) - update_tasks_cpumask(parent); + update_tasks_cpumask(parent); if (parent->child_ecpus_count) update_sibling_cpumasks(parent, cs, &tmpmask); - rebuild_sched_domains_locked(); + if (!sched_domain_rebuilt) + rebuild_sched_domains_locked(); out: - if (!err) { - spin_lock_irq(&callback_lock); - cs->partition_root_state = new_prs; - spin_unlock_irq(&callback_lock); - notify_partition_change(cs, old_prs, new_prs); - } + /* + * Make partition invalid if an error happen + */ + if (err) + new_prs = -new_prs; + spin_lock_irq(&callback_lock); + cs->partition_root_state = new_prs; + spin_unlock_irq(&callback_lock); + /* + * Update child cpusets, if present. + * Force update if switching back to member. + */ + if (!list_empty(&cs->css.children)) + update_cpumasks_hier(cs, &tmpmask, !new_prs); + notify_partition_change(cs, old_prs); free_cpumasks(NULL, &tmpmask); - return err; + return 0; } /* @@ -2238,6 +2456,12 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) goto out_unlock; + /* + * Task cannot be moved to a cpuset with empty effective cpus. + */ + if (cpumask_empty(cs->effective_cpus)) + goto out_unlock; + cgroup_taskset_for_each(task, css, tset) { ret = task_can_attach(task, cs->effective_cpus); if (ret) @@ -2289,7 +2513,7 @@ static void cpuset_attach(struct cgroup_taskset *tset) cgroup_taskset_first(tset, &css); cs = css_cs(css); - cpus_read_lock(); + lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ percpu_down_write(&cpuset_rwsem); guarantee_online_mems(cs, &cpuset_attach_nodemask_to); @@ -2343,7 +2567,6 @@ static void cpuset_attach(struct cgroup_taskset *tset) wake_up(&cpuset_attach_wq); percpu_up_write(&cpuset_rwsem); - cpus_read_unlock(); } /* The various types of files and directories in a cpuset file system */ @@ -2599,16 +2822,29 @@ static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) static int sched_partition_show(struct seq_file *seq, void *v) { struct cpuset *cs = css_cs(seq_css(seq)); + const char *err, *type = NULL; switch (cs->partition_root_state) { - case PRS_ENABLED: + case PRS_ROOT: seq_puts(seq, "root\n"); break; - case PRS_DISABLED: + case PRS_ISOLATED: + seq_puts(seq, "isolated\n"); + break; + case PRS_MEMBER: seq_puts(seq, "member\n"); break; - case PRS_ERROR: - seq_puts(seq, "root invalid\n"); + case PRS_INVALID_ROOT: + type = "root"; + fallthrough; + case PRS_INVALID_ISOLATED: + if (!type) + type = "isolated"; + err = perr_strings[READ_ONCE(cs->prs_err)]; + if (err) + seq_printf(seq, "%s invalid (%s)\n", type, err); + else + seq_printf(seq, "%s invalid\n", type); break; } return 0; @@ -2627,9 +2863,11 @@ static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf, * Convert "root" to ENABLED, and convert "member" to DISABLED. */ if (!strcmp(buf, "root")) - val = PRS_ENABLED; + val = PRS_ROOT; else if (!strcmp(buf, "member")) - val = PRS_DISABLED; + val = PRS_MEMBER; + else if (!strcmp(buf, "isolated")) + val = PRS_ISOLATED; else return -EINVAL; @@ -2928,7 +3166,7 @@ static void cpuset_css_offline(struct cgroup_subsys_state *css) cpus_read_lock(); percpu_down_write(&cpuset_rwsem); - if (is_partition_root(cs)) + if (is_partition_valid(cs)) update_prstate(cs, 0); if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && @@ -3104,7 +3342,8 @@ hotplug_update_tasks(struct cpuset *cs, struct cpumask *new_cpus, nodemask_t *new_mems, bool cpus_updated, bool mems_updated) { - if (cpumask_empty(new_cpus)) + /* A partition root is allowed to have empty effective cpus */ + if (cpumask_empty(new_cpus) && !is_partition_valid(cs)) cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); if (nodes_empty(*new_mems)) *new_mems = parent_cs(cs)->effective_mems; @@ -3173,11 +3412,31 @@ retry: /* * In the unlikely event that a partition root has empty - * effective_cpus or its parent becomes erroneous, we have to - * transition it to the erroneous state. + * effective_cpus with tasks, we will have to invalidate child + * partitions, if present, by setting nr_subparts_cpus to 0 to + * reclaim their cpus. */ - if (is_partition_root(cs) && (cpumask_empty(&new_cpus) || - (parent->partition_root_state == PRS_ERROR))) { + if (cs->nr_subparts_cpus && is_partition_valid(cs) && + cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)) { + spin_lock_irq(&callback_lock); + cs->nr_subparts_cpus = 0; + cpumask_clear(cs->subparts_cpus); + spin_unlock_irq(&callback_lock); + compute_effective_cpumask(&new_cpus, cs, parent); + } + + /* + * Force the partition to become invalid if either one of + * the following conditions hold: + * 1) empty effective cpus but not valid empty partition. + * 2) parent is invalid or doesn't grant any cpus to child + * partitions. + */ + if (is_partition_valid(cs) && (!parent->nr_subparts_cpus || + (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { + int old_prs, parent_prs; + + update_parent_subparts_cpumask(cs, partcmd_disable, NULL, tmp); if (cs->nr_subparts_cpus) { spin_lock_irq(&callback_lock); cs->nr_subparts_cpus = 0; @@ -3186,39 +3445,32 @@ retry: compute_effective_cpumask(&new_cpus, cs, parent); } - /* - * If the effective_cpus is empty because the child - * partitions take away all the CPUs, we can keep - * the current partition and let the child partitions - * fight for available CPUs. - */ - if ((parent->partition_root_state == PRS_ERROR) || - cpumask_empty(&new_cpus)) { - int old_prs; - - update_parent_subparts_cpumask(cs, partcmd_disable, - NULL, tmp); - old_prs = cs->partition_root_state; - if (old_prs != PRS_ERROR) { - spin_lock_irq(&callback_lock); - cs->partition_root_state = PRS_ERROR; - spin_unlock_irq(&callback_lock); - notify_partition_change(cs, old_prs, PRS_ERROR); - } + old_prs = cs->partition_root_state; + parent_prs = parent->partition_root_state; + if (is_partition_valid(cs)) { + spin_lock_irq(&callback_lock); + make_partition_invalid(cs); + spin_unlock_irq(&callback_lock); + if (is_prs_invalid(parent_prs)) + WRITE_ONCE(cs->prs_err, PERR_INVPARENT); + else if (!parent_prs) + WRITE_ONCE(cs->prs_err, PERR_NOTPART); + else + WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); + notify_partition_change(cs, old_prs); } cpuset_force_rebuild(); } /* - * On the other hand, an erroneous partition root may be transitioned - * back to a regular one or a partition root with no CPU allocated - * from the parent may change to erroneous. + * On the other hand, an invalid partition root may be transitioned + * back to a regular one. */ - if (is_partition_root(parent) && - ((cs->partition_root_state == PRS_ERROR) || - !cpumask_intersects(&new_cpus, parent->subparts_cpus)) && - update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp)) - cpuset_force_rebuild(); + else if (is_partition_valid(parent) && is_partition_invalid(cs)) { + update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp); + if (is_partition_valid(cs)) + cpuset_force_rebuild(); + } update_tasks: cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); diff --git a/kernel/cgroup/legacy_freezer.c b/kernel/cgroup/legacy_freezer.c index 08236798d17315622d73540d62a89dcbafdc5c77..1b6b21851e9d47daa3456123d604ab4d5e0f3b9c 100644 --- a/kernel/cgroup/legacy_freezer.c +++ b/kernel/cgroup/legacy_freezer.c @@ -113,7 +113,7 @@ static int freezer_css_online(struct cgroup_subsys_state *css) if (parent && (parent->state & CGROUP_FREEZING)) { freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN; - atomic_inc(&system_freezing_cnt); + static_branch_inc(&freezer_active); } mutex_unlock(&freezer_mutex); @@ -134,7 +134,7 @@ static void freezer_css_offline(struct cgroup_subsys_state *css) mutex_lock(&freezer_mutex); if (freezer->state & CGROUP_FREEZING) - atomic_dec(&system_freezing_cnt); + static_branch_dec(&freezer_active); freezer->state = 0; @@ -179,6 +179,7 @@ static void freezer_attach(struct cgroup_taskset *tset) __thaw_task(task); } else { freeze_task(task); + /* clear FROZEN and propagate upwards */ while (freezer && (freezer->state & CGROUP_FROZEN)) { freezer->state &= ~CGROUP_FROZEN; @@ -271,16 +272,8 @@ static void update_if_frozen(struct cgroup_subsys_state *css) css_task_iter_start(css, 0, &it); while ((task = css_task_iter_next(&it))) { - if (freezing(task)) { - /* - * freezer_should_skip() indicates that the task - * should be skipped when determining freezing - * completion. Consider it frozen in addition to - * the usual frozen condition. - */ - if (!frozen(task) && !freezer_should_skip(task)) - goto out_iter_end; - } + if (freezing(task) && !frozen(task)) + goto out_iter_end; } freezer->state |= CGROUP_FROZEN; @@ -357,7 +350,7 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze, if (freeze) { if (!(freezer->state & CGROUP_FREEZING)) - atomic_inc(&system_freezing_cnt); + static_branch_inc(&freezer_active); freezer->state |= state; freeze_cgroup(freezer); } else { @@ -366,9 +359,9 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze, freezer->state &= ~state; if (!(freezer->state & CGROUP_FREEZING)) { - if (was_freezing) - atomic_dec(&system_freezing_cnt); freezer->state &= ~CGROUP_FROZEN; + if (was_freezing) + static_branch_dec(&freezer_active); unfreeze_cgroup(freezer); } } diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c index 511af87f685e8ec6bffc482a1f2941379ce36a05..7695e60bcb4092000f7cb1ed61f6fcd6d5a8c6a0 100644 --- a/kernel/cgroup/pids.c +++ b/kernel/cgroup/pids.c @@ -47,6 +47,7 @@ struct pids_cgroup { */ atomic64_t counter; atomic64_t limit; + int64_t watermark; /* Handle for "pids.events" */ struct cgroup_file events_file; @@ -85,6 +86,16 @@ static void pids_css_free(struct cgroup_subsys_state *css) kfree(css_pids(css)); } +static void pids_update_watermark(struct pids_cgroup *p, int64_t nr_pids) +{ + /* + * This is racy, but we don't need perfectly accurate tallying of + * the watermark, and this lets us avoid extra atomic overhead. + */ + if (nr_pids > READ_ONCE(p->watermark)) + WRITE_ONCE(p->watermark, nr_pids); +} + /** * pids_cancel - uncharge the local pid count * @pids: the pid cgroup state @@ -128,8 +139,11 @@ static void pids_charge(struct pids_cgroup *pids, int num) { struct pids_cgroup *p; - for (p = pids; parent_pids(p); p = parent_pids(p)) - atomic64_add(num, &p->counter); + for (p = pids; parent_pids(p); p = parent_pids(p)) { + int64_t new = atomic64_add_return(num, &p->counter); + + pids_update_watermark(p, new); + } } /** @@ -156,6 +170,12 @@ static int pids_try_charge(struct pids_cgroup *pids, int num) */ if (new > limit) goto revert; + + /* + * Not technically accurate if we go over limit somewhere up + * the hierarchy, but that's tolerable for the watermark. + */ + pids_update_watermark(p, new); } return 0; @@ -311,6 +331,14 @@ static s64 pids_current_read(struct cgroup_subsys_state *css, return atomic64_read(&pids->counter); } +static s64 pids_peak_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct pids_cgroup *pids = css_pids(css); + + return READ_ONCE(pids->watermark); +} + static int pids_events_show(struct seq_file *sf, void *v) { struct pids_cgroup *pids = css_pids(seq_css(sf)); @@ -331,6 +359,11 @@ static struct cftype pids_files[] = { .read_s64 = pids_current_read, .flags = CFTYPE_NOT_ON_ROOT, }, + { + .name = "peak", + .flags = CFTYPE_NOT_ON_ROOT, + .read_s64 = pids_peak_read, + }, { .name = "events", .seq_show = pids_events_show, diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c index feb59380c89627e30dbe197722ff2c5fc2f830ba..793ecff29038514eb50e95d2e89e36b8c2bed7e2 100644 --- a/kernel/cgroup/rstat.c +++ b/kernel/cgroup/rstat.c @@ -3,6 +3,10 @@ #include +#include +#include +#include + static DEFINE_SPINLOCK(cgroup_rstat_lock); static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock); @@ -141,6 +145,31 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos, return pos; } +/* + * A hook for bpf stat collectors to attach to and flush their stats. + * Together with providing bpf kfuncs for cgroup_rstat_updated() and + * cgroup_rstat_flush(), this enables a complete workflow where bpf progs that + * collect cgroup stats can integrate with rstat for efficient flushing. + * + * A static noinline declaration here could cause the compiler to optimize away + * the function. A global noinline declaration will keep the definition, but may + * optimize away the callsite. Therefore, __weak is needed to ensure that the + * call is still emitted, by telling the compiler that we don't know what the + * function might eventually be. + * + * __diag_* below are needed to dismiss the missing prototype warning. + */ +__diag_push(); +__diag_ignore_all("-Wmissing-prototypes", + "kfuncs which will be used in BPF programs"); + +__weak noinline void bpf_rstat_flush(struct cgroup *cgrp, + struct cgroup *parent, int cpu) +{ +} + +__diag_pop(); + /* see cgroup_rstat_flush() */ static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep) __releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock) @@ -168,6 +197,7 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep) struct cgroup_subsys_state *css; cgroup_base_stat_flush(pos, cpu); + bpf_rstat_flush(pos, cgroup_parent(pos), cpu); rcu_read_lock(); list_for_each_entry_rcu(css, &pos->rstat_css_list, @@ -501,3 +531,21 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq) seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time); #endif } + +/* Add bpf kfuncs for cgroup_rstat_updated() and cgroup_rstat_flush() */ +BTF_SET8_START(bpf_rstat_kfunc_ids) +BTF_ID_FLAGS(func, cgroup_rstat_updated) +BTF_ID_FLAGS(func, cgroup_rstat_flush, KF_SLEEPABLE) +BTF_SET8_END(bpf_rstat_kfunc_ids) + +static const struct btf_kfunc_id_set bpf_rstat_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_rstat_kfunc_ids, +}; + +static int __init bpf_rstat_kfunc_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, + &bpf_rstat_kfunc_set); +} +late_initcall(bpf_rstat_kfunc_init); diff --git a/kernel/configs/rust.config b/kernel/configs/rust.config new file mode 100644 index 0000000000000000000000000000000000000000..38a7c5362c9cf1f8b5956e673de7c90359bf7503 --- /dev/null +++ b/kernel/configs/rust.config @@ -0,0 +1 @@ +CONFIG_RUST=y diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 07b26df453a977c39f6771743a6eba9f694ad215..a0eb4d5cf557717b500d69270b4dfb1201cefea8 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -494,6 +494,7 @@ static int __init crash_save_vmcoreinfo_init(void) #ifdef CONFIG_KALLSYMS VMCOREINFO_SYMBOL(kallsyms_names); + VMCOREINFO_SYMBOL(kallsyms_num_syms); VMCOREINFO_SYMBOL(kallsyms_token_table); VMCOREINFO_SYMBOL(kallsyms_token_index); #ifdef CONFIG_KALLSYMS_BASE_RELATIVE diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 7beceb447211d12cc3f62bfb128619507b7ff55d..d5e9ccde3ab8e97f51e93b8fb57ba0e2a101ecfc 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -50,7 +50,6 @@ #include #include #include -#include #include #include #include @@ -283,17 +282,6 @@ static void kgdb_flush_swbreak_addr(unsigned long addr) if (!CACHE_FLUSH_IS_SAFE) return; - if (current->mm) { - int i; - - for (i = 0; i < VMACACHE_SIZE; i++) { - if (!current->vmacache.vmas[i]) - continue; - flush_cache_range(current->vmacache.vmas[i], - addr, addr + BREAK_INSTR_SIZE); - } - } - /* Force flush instruction cache if it was outside the mm */ flush_icache_range(addr, addr + BREAK_INSTR_SIZE); } diff --git a/kernel/delayacct.c b/kernel/delayacct.c index 164ed9ef77a33223d386c475a0a597239d48c5ea..e39cb696cfbd4493ebc61be63c022bc96af47635 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -214,13 +214,22 @@ void __delayacct_freepages_end(void) ¤t->delays->freepages_count); } -void __delayacct_thrashing_start(void) +void __delayacct_thrashing_start(bool *in_thrashing) { + *in_thrashing = !!current->in_thrashing; + if (*in_thrashing) + return; + + current->in_thrashing = 1; current->delays->thrashing_start = local_clock(); } -void __delayacct_thrashing_end(void) +void __delayacct_thrashing_end(bool *in_thrashing) { + if (*in_thrashing) + return; + + current->in_thrashing = 0; delayacct_end(¤t->delays->lock, ¤t->delays->thrashing_start, ¤t->delays->thrashing_delay, diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c index 2caafd13f8aac27f92e0675d94050175c45d585d..18c93c2276cae436c89341e11830cb694fc87e8f 100644 --- a/kernel/dma/debug.c +++ b/kernel/dma/debug.c @@ -350,11 +350,10 @@ static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket, unsigned long *flags) { - unsigned int max_range = dma_get_max_seg_size(ref->dev); struct dma_debug_entry *entry, index = *ref; - unsigned int range = 0; + int limit = min(HASH_SIZE, (index.dev_addr >> HASH_FN_SHIFT) + 1); - while (range <= max_range) { + for (int i = 0; i < limit; i++) { entry = __hash_bucket_find(*bucket, ref, containing_match); if (entry) @@ -364,7 +363,6 @@ static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket, * Nothing found, go back a hash bucket */ put_hash_bucket(*bucket, *flags); - range += (1 << HASH_FN_SHIFT); index.dev_addr -= (1 << HASH_FN_SHIFT); *bucket = get_hash_bucket(&index, flags); } diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 49cbf3e33de71cb8fbff2d67fc4b23e1e7b4c4f5..33437d6206445812b6d4d5b33c77235d18074dec 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -156,6 +157,7 @@ dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); else addr = ops->map_page(dev, page, offset, size, dir, attrs); + kmsan_handle_dma(page, offset, size, dir); debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); return addr; @@ -194,11 +196,13 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, else ents = ops->map_sg(dev, sg, nents, dir, attrs); - if (ents > 0) + if (ents > 0) { + kmsan_handle_dma_sg(sg, nents, dir); debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); - else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && - ents != -EIO && ents != -EREMOTEIO)) + } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && + ents != -EIO && ents != -EREMOTEIO)) { return -EIO; + } return ents; } @@ -707,7 +711,7 @@ int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, } EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); -int dma_supported(struct device *dev, u64 mask) +static int dma_supported(struct device *dev, u64 mask) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -721,7 +725,6 @@ int dma_supported(struct device *dev, u64 mask) return 1; return ops->dma_supported(dev, mask); } -EXPORT_SYMBOL(dma_supported); bool dma_pci_p2pdma_supported(struct device *dev) { diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index c5a9190b218f959d3a050e23fc25472ab8fbfeb5..339a990554e7fed98dd337efe4fb759a98161cdb 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -326,9 +326,6 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, swiotlb_adjust_nareas(num_possible_cpus()); nslabs = default_nslabs; - if (nslabs < IO_TLB_MIN_SLABS) - panic("%s: nslabs = %lu too small\n", __func__, nslabs); - /* * By default allocate the bounce buffer memory from low memory, but * allow to pick a location everywhere for hypervisors with guest @@ -341,8 +338,7 @@ retry: else tlb = memblock_alloc_low(bytes, PAGE_SIZE); if (!tlb) { - pr_warn("%s: Failed to allocate %zu bytes tlb structure\n", - __func__, bytes); + pr_warn("%s: failed to allocate tlb structure\n", __func__); return; } @@ -350,22 +346,27 @@ retry: memblock_free(tlb, PAGE_ALIGN(bytes)); nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); - if (nslabs < IO_TLB_MIN_SLABS) - panic("%s: Failed to remap %zu bytes\n", - __func__, bytes); - goto retry; + if (nslabs >= IO_TLB_MIN_SLABS) + goto retry; + + pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes); + return; } alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs)); mem->slots = memblock_alloc(alloc_size, PAGE_SIZE); - if (!mem->slots) - panic("%s: Failed to allocate %zu bytes align=0x%lx\n", - __func__, alloc_size, PAGE_SIZE); + if (!mem->slots) { + pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n", + __func__, alloc_size, PAGE_SIZE); + return; + } mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area), default_nareas), SMP_CACHE_BYTES); - if (!mem->areas) - panic("%s: Failed to allocate mem->areas.\n", __func__); + if (!mem->areas) { + pr_warn("%s: Failed to allocate mem->areas.\n", __func__); + return; + } swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false, default_nareas); @@ -549,9 +550,8 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size } if (PageHighMem(pfn_to_page(pfn))) { - /* The buffer does not have a mapping. Map it in and copy */ unsigned int offset = orig_addr & ~PAGE_MASK; - char *buffer; + struct page *page; unsigned int sz = 0; unsigned long flags; @@ -559,12 +559,11 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size sz = min_t(size_t, PAGE_SIZE - offset, size); local_irq_save(flags); - buffer = kmap_atomic(pfn_to_page(pfn)); + page = pfn_to_page(pfn); if (dir == DMA_TO_DEVICE) - memcpy(vaddr, buffer + offset, sz); + memcpy_from_page(vaddr, page, offset, sz); else - memcpy(buffer + offset, vaddr, sz); - kunmap_atomic(buffer); + memcpy_to_page(page, offset, vaddr, sz); local_irq_restore(flags); size -= sz; @@ -579,7 +578,10 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size } } -#define slot_addr(start, idx) ((start) + ((idx) << IO_TLB_SHIFT)) +static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx) +{ + return start + (idx << IO_TLB_SHIFT); +} /* * Carefully handle integer overflow which can occur when boundary_mask == ~0UL. @@ -732,8 +734,11 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, int index; phys_addr_t tlb_addr; - if (!mem || !mem->nslabs) - panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer"); + if (!mem || !mem->nslabs) { + dev_warn_ratelimited(dev, + "Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer"); + return (phys_addr_t)DMA_MAPPING_ERROR; + } if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n"); @@ -765,7 +770,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr, /* * When dir == DMA_FROM_DEVICE we could omit the copy from the orig * to the tlb buffer, if we knew for sure the device will - * overwirte the entire current content. But we don't. Thus + * overwrite the entire current content. But we don't. Thus * unconditional bounce may prevent leaking swiotlb content (i.e. * kernel memory) to user-space. */ diff --git a/kernel/entry/common.c b/kernel/entry/common.c index 063068a9ea9b35a54289d361a9429dd660f34b8e..846add8394c4159c8e5accb77ecc10e463519e3b 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ static __always_inline void __enter_from_user_mode(struct pt_regs *regs) user_exit_irqoff(); instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); trace_hardirqs_off_finish(); instrumentation_end(); } @@ -352,6 +354,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) lockdep_hardirqs_off(CALLER_ADDR0); ct_irq_enter(); instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); trace_hardirqs_off_finish(); instrumentation_end(); @@ -367,6 +370,7 @@ noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs) */ lockdep_hardirqs_off(CALLER_ADDR0); instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); rcu_irq_enter_check_tick(); trace_hardirqs_off_finish(); instrumentation_end(); @@ -452,6 +456,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs) ct_nmi_enter(); instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); trace_hardirqs_off_finish(); ftrace_nmi_enter(); instrumentation_end(); diff --git a/kernel/events/Makefile b/kernel/events/Makefile index 8591c180b52b3ab78a5d65de3a06708263d65e6e..91a62f5667434f5514240a597b87709fb8a0dbdc 100644 --- a/kernel/events/Makefile +++ b/kernel/events/Makefile @@ -2,4 +2,5 @@ obj-y := core.o ring_buffer.o callchain.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o +obj-$(CONFIG_HW_BREAKPOINT_KUNIT_TEST) += hw_breakpoint_test.o obj-$(CONFIG_UPROBES) += uprobes.o diff --git a/kernel/events/core.c b/kernel/events/core.c index 2621fd24ad260b891d5ef2ed1571eacb30e5e5ef..aefc1e08e015e495ec17166b7e3b050b676a5cd4 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1468,6 +1468,8 @@ static void __update_context_time(struct perf_event_context *ctx, bool adv) { u64 now = perf_clock(); + lockdep_assert_held(&ctx->lock); + if (adv) ctx->time += now - ctx->timestamp; ctx->timestamp = now; @@ -2224,16 +2226,22 @@ static inline int __pmu_filter_match(struct perf_event *event) static inline int pmu_filter_match(struct perf_event *event) { struct perf_event *sibling; + unsigned long flags; + int ret = 1; if (!__pmu_filter_match(event)) return 0; + local_irq_save(flags); for_each_sibling_event(sibling, event) { - if (!__pmu_filter_match(sibling)) - return 0; + if (!__pmu_filter_match(sibling)) { + ret = 0; + break; + } } + local_irq_restore(flags); - return 1; + return ret; } static inline int @@ -6794,11 +6802,10 @@ out_put: static void __perf_event_header__init_id(struct perf_event_header *header, struct perf_sample_data *data, - struct perf_event *event) + struct perf_event *event, + u64 sample_type) { - u64 sample_type = event->attr.sample_type; - - data->type = sample_type; + data->type = event->attr.sample_type; header->size += event->id_header_size; if (sample_type & PERF_SAMPLE_TID) { @@ -6827,7 +6834,7 @@ void perf_event_header__init_id(struct perf_event_header *header, struct perf_event *event) { if (event->attr.sample_id_all) - __perf_event_header__init_id(header, data, event); + __perf_event_header__init_id(header, data, event, event->attr.sample_type); } static void __perf_event__output_id_sample(struct perf_output_handle *handle, @@ -6893,9 +6900,16 @@ static void perf_output_read_group(struct perf_output_handle *handle, { struct perf_event *leader = event->group_leader, *sub; u64 read_format = event->attr.read_format; + unsigned long flags; u64 values[6]; int n = 0; + /* + * Disabling interrupts avoids all counter scheduling + * (context switches, timer based rotation and IPIs). + */ + local_irq_save(flags); + values[n++] = 1 + leader->nr_siblings; if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) @@ -6931,6 +6945,8 @@ static void perf_output_read_group(struct perf_output_handle *handle, __output_copy(handle, values, n * sizeof(u64)); } + + local_irq_restore(flags); } #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ @@ -6967,11 +6983,6 @@ static void perf_output_read(struct perf_output_handle *handle, perf_output_read_one(handle, event, enabled, running); } -static inline bool perf_sample_save_hw_index(struct perf_event *event) -{ - return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; -} - void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, @@ -7053,14 +7064,14 @@ void perf_output_sample(struct perf_output_handle *handle, } if (sample_type & PERF_SAMPLE_BRANCH_STACK) { - if (data->br_stack) { + if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { size_t size; size = data->br_stack->nr * sizeof(struct perf_branch_entry); perf_output_put(handle, data->br_stack->nr); - if (perf_sample_save_hw_index(event)) + if (branch_sample_hw_index(event)) perf_output_put(handle, data->br_stack->hw_idx); perf_output_copy(handle, data->br_stack->entries, size); } else { @@ -7303,6 +7314,7 @@ void perf_prepare_sample(struct perf_event_header *header, struct pt_regs *regs) { u64 sample_type = event->attr.sample_type; + u64 filtered_sample_type; header->type = PERF_RECORD_SAMPLE; header->size = sizeof(*header) + event->header_size; @@ -7310,7 +7322,12 @@ void perf_prepare_sample(struct perf_event_header *header, header->misc = 0; header->misc |= perf_misc_flags(regs); - __perf_event_header__init_id(header, data, event); + /* + * Clear the sample flags that have already been done by the + * PMU driver. + */ + filtered_sample_type = sample_type & ~data->sample_flags; + __perf_event_header__init_id(header, data, event, filtered_sample_type); if (sample_type & (PERF_SAMPLE_IP | PERF_SAMPLE_CODE_PAGE_SIZE)) data->ip = perf_instruction_pointer(regs); @@ -7318,7 +7335,7 @@ void perf_prepare_sample(struct perf_event_header *header, if (sample_type & PERF_SAMPLE_CALLCHAIN) { int size = 1; - if (!(sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY)) + if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) data->callchain = perf_callchain(event, regs); size += data->callchain->nr; @@ -7330,7 +7347,7 @@ void perf_prepare_sample(struct perf_event_header *header, struct perf_raw_record *raw = data->raw; int size; - if (raw) { + if (raw && (data->sample_flags & PERF_SAMPLE_RAW)) { struct perf_raw_frag *frag = &raw->frag; u32 sum = 0; @@ -7346,6 +7363,7 @@ void perf_prepare_sample(struct perf_event_header *header, frag->pad = raw->size - sum; } else { size = sizeof(u64); + data->raw = NULL; } header->size += size; @@ -7353,8 +7371,8 @@ void perf_prepare_sample(struct perf_event_header *header, if (sample_type & PERF_SAMPLE_BRANCH_STACK) { int size = sizeof(u64); /* nr */ - if (data->br_stack) { - if (perf_sample_save_hw_index(event)) + if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { + if (branch_sample_hw_index(event)) size += sizeof(u64); size += data->br_stack->nr @@ -7403,6 +7421,20 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } + if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + data->weight.full = 0; + + if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) + data->data_src.val = PERF_MEM_NA; + + if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) + data->txn = 0; + + if (sample_type & (PERF_SAMPLE_ADDR | PERF_SAMPLE_PHYS_ADDR | PERF_SAMPLE_DATA_PAGE_SIZE)) { + if (filtered_sample_type & PERF_SAMPLE_ADDR) + data->addr = 0; + } + if (sample_type & PERF_SAMPLE_REGS_INTR) { /* regs dump ABI info */ int size = sizeof(u64); @@ -7418,7 +7450,8 @@ void perf_prepare_sample(struct perf_event_header *header, header->size += size; } - if (sample_type & PERF_SAMPLE_PHYS_ADDR) + if (sample_type & PERF_SAMPLE_PHYS_ADDR && + filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) data->phys_addr = perf_virt_to_phys(data->addr); #ifdef CONFIG_CGROUP_PERF @@ -9989,8 +10022,16 @@ static void bpf_overflow_handler(struct perf_event *event, goto out; rcu_read_lock(); prog = READ_ONCE(event->prog); - if (prog) + if (prog) { + if (prog->call_get_stack && + (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) && + !(data->sample_flags & PERF_SAMPLE_CALLCHAIN)) { + data->callchain = perf_callchain(event, regs); + data->sample_flags |= PERF_SAMPLE_CALLCHAIN; + } + ret = bpf_prog_run(prog, &ctx); + } rcu_read_unlock(); out: __this_cpu_dec(bpf_prog_active); @@ -10016,7 +10057,7 @@ static int perf_event_set_bpf_handler(struct perf_event *event, if (event->attr.precise_ip && prog->call_get_stack && - (!(event->attr.sample_type & __PERF_SAMPLE_CALLCHAIN_EARLY) || + (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) || event->attr.exclude_callchain_kernel || event->attr.exclude_callchain_user)) { /* @@ -10229,8 +10270,9 @@ static void perf_addr_filter_apply(struct perf_addr_filter *filter, struct perf_addr_filter_range *fr) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (!vma->vm_file) continue; @@ -10933,7 +10975,7 @@ static ssize_t nr_addr_filters_show(struct device *dev, { struct pmu *pmu = dev_get_drvdata(dev); - return snprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); + return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); } DEVICE_ATTR_RO(nr_addr_filters); @@ -10944,7 +10986,7 @@ type_show(struct device *dev, struct device_attribute *attr, char *page) { struct pmu *pmu = dev_get_drvdata(dev); - return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); + return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type); } static DEVICE_ATTR_RO(type); @@ -10955,7 +10997,7 @@ perf_event_mux_interval_ms_show(struct device *dev, { struct pmu *pmu = dev_get_drvdata(dev); - return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); + return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms); } static DEFINE_MUTEX(mux_interval_mutex); @@ -11709,11 +11751,9 @@ err_pmu: event->destroy(event); module_put(pmu->module); err_ns: - if (event->ns) - put_pid_ns(event->ns); if (event->hw.target) put_task_struct(event->hw.target); - kmem_cache_free(perf_event_cache, event); + call_rcu(&event->rcu_head, free_event_rcu); return ERR_PTR(err); } diff --git a/kernel/events/hw_breakpoint.c b/kernel/events/hw_breakpoint.c index f32320ac02fd8b79fef8c1333a4e167f27b4200e..c3797701339cbd575ca3c58ed9f0313e16e74cc8 100644 --- a/kernel/events/hw_breakpoint.c +++ b/kernel/events/hw_breakpoint.c @@ -17,61 +17,276 @@ * This file contains the arch-independent routines. */ +#include + +#include +#include +#include +#include +#include #include -#include -#include -#include #include #include -#include +#include +#include +#include #include +#include #include -#include #include -#include -#include -#include -#include -#include /* - * Constraints data + * Datastructure to track the total uses of N slots across tasks or CPUs; + * bp_slots_histogram::count[N] is the number of assigned N+1 breakpoint slots. + */ +struct bp_slots_histogram { +#ifdef hw_breakpoint_slots + atomic_t count[hw_breakpoint_slots(0)]; +#else + atomic_t *count; +#endif +}; + +/* + * Per-CPU constraints data. */ struct bp_cpuinfo { - /* Number of pinned cpu breakpoints in a cpu */ - unsigned int cpu_pinned; - /* tsk_pinned[n] is the number of tasks having n+1 breakpoints */ - unsigned int *tsk_pinned; - /* Number of non-pinned cpu/task breakpoints in a cpu */ - unsigned int flexible; /* XXX: placeholder, see fetch_this_slot() */ + /* Number of pinned CPU breakpoints in a CPU. */ + unsigned int cpu_pinned; + /* Histogram of pinned task breakpoints in a CPU. */ + struct bp_slots_histogram tsk_pinned; }; static DEFINE_PER_CPU(struct bp_cpuinfo, bp_cpuinfo[TYPE_MAX]); -static int nr_slots[TYPE_MAX]; static struct bp_cpuinfo *get_bp_info(int cpu, enum bp_type_idx type) { return per_cpu_ptr(bp_cpuinfo + type, cpu); } +/* Number of pinned CPU breakpoints globally. */ +static struct bp_slots_histogram cpu_pinned[TYPE_MAX]; +/* Number of pinned CPU-independent task breakpoints. */ +static struct bp_slots_histogram tsk_pinned_all[TYPE_MAX]; + /* Keep track of the breakpoints attached to tasks */ -static LIST_HEAD(bp_task_head); +static struct rhltable task_bps_ht; +static const struct rhashtable_params task_bps_ht_params = { + .head_offset = offsetof(struct hw_perf_event, bp_list), + .key_offset = offsetof(struct hw_perf_event, target), + .key_len = sizeof_field(struct hw_perf_event, target), + .automatic_shrinking = true, +}; -static int constraints_initialized; +static bool constraints_initialized __ro_after_init; -/* Gather the number of total pinned and un-pinned bp in a cpuset */ -struct bp_busy_slots { - unsigned int pinned; - unsigned int flexible; -}; +/* + * Synchronizes accesses to the per-CPU constraints; the locking rules are: + * + * 1. Atomic updates to bp_cpuinfo::tsk_pinned only require a held read-lock + * (due to bp_slots_histogram::count being atomic, no update are lost). + * + * 2. Holding a write-lock is required for computations that require a + * stable snapshot of all bp_cpuinfo::tsk_pinned. + * + * 3. In all other cases, non-atomic accesses require the appropriately held + * lock (read-lock for read-only accesses; write-lock for reads/writes). + */ +DEFINE_STATIC_PERCPU_RWSEM(bp_cpuinfo_sem); -/* Serialize accesses to the above constraints */ -static DEFINE_MUTEX(nr_bp_mutex); +/* + * Return mutex to serialize accesses to per-task lists in task_bps_ht. Since + * rhltable synchronizes concurrent insertions/deletions, independent tasks may + * insert/delete concurrently; therefore, a mutex per task is sufficient. + * + * Uses task_struct::perf_event_mutex, to avoid extending task_struct with a + * hw_breakpoint-only mutex, which may be infrequently used. The caveat here is + * that hw_breakpoint may contend with per-task perf event list management. The + * assumption is that perf usecases involving hw_breakpoints are very unlikely + * to result in unnecessary contention. + */ +static inline struct mutex *get_task_bps_mutex(struct perf_event *bp) +{ + struct task_struct *tsk = bp->hw.target; -__weak int hw_breakpoint_weight(struct perf_event *bp) + return tsk ? &tsk->perf_event_mutex : NULL; +} + +static struct mutex *bp_constraints_lock(struct perf_event *bp) +{ + struct mutex *tsk_mtx = get_task_bps_mutex(bp); + + if (tsk_mtx) { + /* + * Fully analogous to the perf_try_init_event() nesting + * argument in the comment near perf_event_ctx_lock_nested(); + * this child->perf_event_mutex cannot ever deadlock against + * the parent->perf_event_mutex usage from + * perf_event_task_{en,dis}able(). + * + * Specifically, inherited events will never occur on + * ->perf_event_list. + */ + mutex_lock_nested(tsk_mtx, SINGLE_DEPTH_NESTING); + percpu_down_read(&bp_cpuinfo_sem); + } else { + percpu_down_write(&bp_cpuinfo_sem); + } + + return tsk_mtx; +} + +static void bp_constraints_unlock(struct mutex *tsk_mtx) +{ + if (tsk_mtx) { + percpu_up_read(&bp_cpuinfo_sem); + mutex_unlock(tsk_mtx); + } else { + percpu_up_write(&bp_cpuinfo_sem); + } +} + +static bool bp_constraints_is_locked(struct perf_event *bp) +{ + struct mutex *tsk_mtx = get_task_bps_mutex(bp); + + return percpu_is_write_locked(&bp_cpuinfo_sem) || + (tsk_mtx ? mutex_is_locked(tsk_mtx) : + percpu_is_read_locked(&bp_cpuinfo_sem)); +} + +static inline void assert_bp_constraints_lock_held(struct perf_event *bp) +{ + struct mutex *tsk_mtx = get_task_bps_mutex(bp); + + if (tsk_mtx) + lockdep_assert_held(tsk_mtx); + lockdep_assert_held(&bp_cpuinfo_sem); +} + +#ifdef hw_breakpoint_slots +/* + * Number of breakpoint slots is constant, and the same for all types. + */ +static_assert(hw_breakpoint_slots(TYPE_INST) == hw_breakpoint_slots(TYPE_DATA)); +static inline int hw_breakpoint_slots_cached(int type) { return hw_breakpoint_slots(type); } +static inline int init_breakpoint_slots(void) { return 0; } +#else +/* + * Dynamic number of breakpoint slots. + */ +static int __nr_bp_slots[TYPE_MAX] __ro_after_init; + +static inline int hw_breakpoint_slots_cached(int type) +{ + return __nr_bp_slots[type]; +} + +static __init bool +bp_slots_histogram_alloc(struct bp_slots_histogram *hist, enum bp_type_idx type) +{ + hist->count = kcalloc(hw_breakpoint_slots_cached(type), sizeof(*hist->count), GFP_KERNEL); + return hist->count; +} + +static __init void bp_slots_histogram_free(struct bp_slots_histogram *hist) +{ + kfree(hist->count); +} + +static __init int init_breakpoint_slots(void) +{ + int i, cpu, err_cpu; + + for (i = 0; i < TYPE_MAX; i++) + __nr_bp_slots[i] = hw_breakpoint_slots(i); + + for_each_possible_cpu(cpu) { + for (i = 0; i < TYPE_MAX; i++) { + struct bp_cpuinfo *info = get_bp_info(cpu, i); + + if (!bp_slots_histogram_alloc(&info->tsk_pinned, i)) + goto err; + } + } + for (i = 0; i < TYPE_MAX; i++) { + if (!bp_slots_histogram_alloc(&cpu_pinned[i], i)) + goto err; + if (!bp_slots_histogram_alloc(&tsk_pinned_all[i], i)) + goto err; + } + + return 0; +err: + for_each_possible_cpu(err_cpu) { + for (i = 0; i < TYPE_MAX; i++) + bp_slots_histogram_free(&get_bp_info(err_cpu, i)->tsk_pinned); + if (err_cpu == cpu) + break; + } + for (i = 0; i < TYPE_MAX; i++) { + bp_slots_histogram_free(&cpu_pinned[i]); + bp_slots_histogram_free(&tsk_pinned_all[i]); + } + + return -ENOMEM; +} +#endif + +static inline void +bp_slots_histogram_add(struct bp_slots_histogram *hist, int old, int val) +{ + const int old_idx = old - 1; + const int new_idx = old_idx + val; + + if (old_idx >= 0) + WARN_ON(atomic_dec_return_relaxed(&hist->count[old_idx]) < 0); + if (new_idx >= 0) + WARN_ON(atomic_inc_return_relaxed(&hist->count[new_idx]) < 0); +} + +static int +bp_slots_histogram_max(struct bp_slots_histogram *hist, enum bp_type_idx type) +{ + for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) { + const int count = atomic_read(&hist->count[i]); + + /* Catch unexpected writers; we want a stable snapshot. */ + ASSERT_EXCLUSIVE_WRITER(hist->count[i]); + if (count > 0) + return i + 1; + WARN(count < 0, "inconsistent breakpoint slots histogram"); + } + + return 0; +} + +static int +bp_slots_histogram_max_merge(struct bp_slots_histogram *hist1, struct bp_slots_histogram *hist2, + enum bp_type_idx type) +{ + for (int i = hw_breakpoint_slots_cached(type) - 1; i >= 0; i--) { + const int count1 = atomic_read(&hist1->count[i]); + const int count2 = atomic_read(&hist2->count[i]); + + /* Catch unexpected writers; we want a stable snapshot. */ + ASSERT_EXCLUSIVE_WRITER(hist1->count[i]); + ASSERT_EXCLUSIVE_WRITER(hist2->count[i]); + if (count1 + count2 > 0) + return i + 1; + WARN(count1 < 0, "inconsistent breakpoint slots histogram"); + WARN(count2 < 0, "inconsistent breakpoint slots histogram"); + } + + return 0; +} + +#ifndef hw_breakpoint_weight +static inline int hw_breakpoint_weight(struct perf_event *bp) { return 1; } +#endif static inline enum bp_type_idx find_slot_idx(u64 bp_type) { @@ -82,39 +297,61 @@ static inline enum bp_type_idx find_slot_idx(u64 bp_type) } /* - * Report the maximum number of pinned breakpoints a task - * have in this cpu + * Return the maximum number of pinned breakpoints a task has in this CPU. */ static unsigned int max_task_bp_pinned(int cpu, enum bp_type_idx type) { - unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned; - int i; + struct bp_slots_histogram *tsk_pinned = &get_bp_info(cpu, type)->tsk_pinned; - for (i = nr_slots[type] - 1; i >= 0; i--) { - if (tsk_pinned[i] > 0) - return i + 1; - } - - return 0; + /* + * At this point we want to have acquired the bp_cpuinfo_sem as a + * writer to ensure that there are no concurrent writers in + * toggle_bp_task_slot() to tsk_pinned, and we get a stable snapshot. + */ + lockdep_assert_held_write(&bp_cpuinfo_sem); + return bp_slots_histogram_max_merge(tsk_pinned, &tsk_pinned_all[type], type); } /* * Count the number of breakpoints of the same type and same task. * The given event must be not on the list. + * + * If @cpu is -1, but the result of task_bp_pinned() is not CPU-independent, + * returns a negative value. */ static int task_bp_pinned(int cpu, struct perf_event *bp, enum bp_type_idx type) { - struct task_struct *tsk = bp->hw.target; + struct rhlist_head *head, *pos; struct perf_event *iter; int count = 0; - list_for_each_entry(iter, &bp_task_head, hw.bp_list) { - if (iter->hw.target == tsk && - find_slot_idx(iter->attr.bp_type) == type && - (iter->cpu < 0 || cpu == iter->cpu)) - count += hw_breakpoint_weight(iter); + /* + * We need a stable snapshot of the per-task breakpoint list. + */ + assert_bp_constraints_lock_held(bp); + + rcu_read_lock(); + head = rhltable_lookup(&task_bps_ht, &bp->hw.target, task_bps_ht_params); + if (!head) + goto out; + + rhl_for_each_entry_rcu(iter, pos, head, hw.bp_list) { + if (find_slot_idx(iter->attr.bp_type) != type) + continue; + + if (iter->cpu >= 0) { + if (cpu == -1) { + count = -1; + goto out; + } else if (cpu != iter->cpu) + continue; + } + + count += hw_breakpoint_weight(iter); } +out: + rcu_read_unlock(); return count; } @@ -126,16 +363,29 @@ static const struct cpumask *cpumask_of_bp(struct perf_event *bp) } /* - * Report the number of pinned/un-pinned breakpoints we have in - * a given cpu (cpu > -1) or in all of them (cpu = -1). + * Returns the max pinned breakpoint slots in a given + * CPU (cpu > -1) or across all of them (cpu = -1). */ -static void -fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp, - enum bp_type_idx type) +static int +max_bp_pinned_slots(struct perf_event *bp, enum bp_type_idx type) { const struct cpumask *cpumask = cpumask_of_bp(bp); + int pinned_slots = 0; int cpu; + if (bp->hw.target && bp->cpu < 0) { + int max_pinned = task_bp_pinned(-1, bp, type); + + if (max_pinned >= 0) { + /* + * Fast path: task_bp_pinned() is CPU-independent and + * returns the same value for any CPU. + */ + max_pinned += bp_slots_histogram_max(&cpu_pinned[type], type); + return max_pinned; + } + } + for_each_cpu(cpu, cpumask) { struct bp_cpuinfo *info = get_bp_info(cpu, type); int nr; @@ -146,71 +396,131 @@ fetch_bp_busy_slots(struct bp_busy_slots *slots, struct perf_event *bp, else nr += task_bp_pinned(cpu, bp, type); - if (nr > slots->pinned) - slots->pinned = nr; - - nr = info->flexible; - if (nr > slots->flexible) - slots->flexible = nr; + pinned_slots = max(nr, pinned_slots); } -} -/* - * For now, continue to consider flexible as pinned, until we can - * ensure no flexible event can ever be scheduled before a pinned event - * in a same cpu. - */ -static void -fetch_this_slot(struct bp_busy_slots *slots, int weight) -{ - slots->pinned += weight; -} - -/* - * Add a pinned breakpoint for the given task in our constraint table - */ -static void toggle_bp_task_slot(struct perf_event *bp, int cpu, - enum bp_type_idx type, int weight) -{ - unsigned int *tsk_pinned = get_bp_info(cpu, type)->tsk_pinned; - int old_idx, new_idx; - - old_idx = task_bp_pinned(cpu, bp, type) - 1; - new_idx = old_idx + weight; - - if (old_idx >= 0) - tsk_pinned[old_idx]--; - if (new_idx >= 0) - tsk_pinned[new_idx]++; + return pinned_slots; } /* * Add/remove the given breakpoint in our constraint table */ -static void -toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type, - int weight) +static int +toggle_bp_slot(struct perf_event *bp, bool enable, enum bp_type_idx type, int weight) { - const struct cpumask *cpumask = cpumask_of_bp(bp); - int cpu; + int cpu, next_tsk_pinned; if (!enable) weight = -weight; - /* Pinned counter cpu profiling */ if (!bp->hw.target) { - get_bp_info(bp->cpu, type)->cpu_pinned += weight; - return; + /* + * Update the pinned CPU slots, in per-CPU bp_cpuinfo and in the + * global histogram. + */ + struct bp_cpuinfo *info = get_bp_info(bp->cpu, type); + + lockdep_assert_held_write(&bp_cpuinfo_sem); + bp_slots_histogram_add(&cpu_pinned[type], info->cpu_pinned, weight); + info->cpu_pinned += weight; + return 0; + } + + /* + * If bp->hw.target, tsk_pinned is only modified, but not used + * otherwise. We can permit concurrent updates as long as there are no + * other uses: having acquired bp_cpuinfo_sem as a reader allows + * concurrent updates here. Uses of tsk_pinned will require acquiring + * bp_cpuinfo_sem as a writer to stabilize tsk_pinned's value. + */ + lockdep_assert_held_read(&bp_cpuinfo_sem); + + /* + * Update the pinned task slots, in per-CPU bp_cpuinfo and in the global + * histogram. We need to take care of 4 cases: + * + * 1. This breakpoint targets all CPUs (cpu < 0), and there may only + * exist other task breakpoints targeting all CPUs. In this case we + * can simply update the global slots histogram. + * + * 2. This breakpoint targets a specific CPU (cpu >= 0), but there may + * only exist other task breakpoints targeting all CPUs. + * + * a. On enable: remove the existing breakpoints from the global + * slots histogram and use the per-CPU histogram. + * + * b. On disable: re-insert the existing breakpoints into the global + * slots histogram and remove from per-CPU histogram. + * + * 3. Some other existing task breakpoints target specific CPUs. Only + * update the per-CPU slots histogram. + */ + + if (!enable) { + /* + * Remove before updating histograms so we can determine if this + * was the last task breakpoint for a specific CPU. + */ + int ret = rhltable_remove(&task_bps_ht, &bp->hw.bp_list, task_bps_ht_params); + + if (ret) + return ret; + } + /* + * Note: If !enable, next_tsk_pinned will not count the to-be-removed breakpoint. + */ + next_tsk_pinned = task_bp_pinned(-1, bp, type); + + if (next_tsk_pinned >= 0) { + if (bp->cpu < 0) { /* Case 1: fast path */ + if (!enable) + next_tsk_pinned += hw_breakpoint_weight(bp); + bp_slots_histogram_add(&tsk_pinned_all[type], next_tsk_pinned, weight); + } else if (enable) { /* Case 2.a: slow path */ + /* Add existing to per-CPU histograms. */ + for_each_possible_cpu(cpu) { + bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned, + 0, next_tsk_pinned); + } + /* Add this first CPU-pinned task breakpoint. */ + bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned, + next_tsk_pinned, weight); + /* Rebalance global task pinned histogram. */ + bp_slots_histogram_add(&tsk_pinned_all[type], next_tsk_pinned, + -next_tsk_pinned); + } else { /* Case 2.b: slow path */ + /* Remove this last CPU-pinned task breakpoint. */ + bp_slots_histogram_add(&get_bp_info(bp->cpu, type)->tsk_pinned, + next_tsk_pinned + hw_breakpoint_weight(bp), weight); + /* Remove all from per-CPU histograms. */ + for_each_possible_cpu(cpu) { + bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned, + next_tsk_pinned, -next_tsk_pinned); + } + /* Rebalance global task pinned histogram. */ + bp_slots_histogram_add(&tsk_pinned_all[type], 0, next_tsk_pinned); + } + } else { /* Case 3: slow path */ + const struct cpumask *cpumask = cpumask_of_bp(bp); + + for_each_cpu(cpu, cpumask) { + next_tsk_pinned = task_bp_pinned(cpu, bp, type); + if (!enable) + next_tsk_pinned += hw_breakpoint_weight(bp); + bp_slots_histogram_add(&get_bp_info(cpu, type)->tsk_pinned, + next_tsk_pinned, weight); + } } - /* Pinned counter task profiling */ - for_each_cpu(cpu, cpumask) - toggle_bp_task_slot(bp, cpu, type, weight); + /* + * Readers want a stable snapshot of the per-task breakpoint list. + */ + assert_bp_constraints_lock_held(bp); if (enable) - list_add_tail(&bp->hw.bp_list, &bp_task_head); - else - list_del(&bp->hw.bp_list); + return rhltable_insert(&task_bps_ht, &bp->hw.bp_list, task_bps_ht_params); + + return 0; } __weak int arch_reserve_bp_slot(struct perf_event *bp) @@ -234,7 +544,12 @@ __weak void arch_unregister_hw_breakpoint(struct perf_event *bp) } /* - * Constraints to check before allowing this new breakpoint counter: + * Constraints to check before allowing this new breakpoint counter. + * + * Note: Flexible breakpoints are currently unimplemented, but outlined in the + * below algorithm for completeness. The implementation treats flexible as + * pinned due to no guarantee that we currently always schedule flexible events + * before a pinned event in a same CPU. * * == Non-pinned counter == (Considered as pinned for now) * @@ -276,8 +591,8 @@ __weak void arch_unregister_hw_breakpoint(struct perf_event *bp) */ static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type) { - struct bp_busy_slots slots = {0}; enum bp_type_idx type; + int max_pinned_slots; int weight; int ret; @@ -293,36 +608,24 @@ static int __reserve_bp_slot(struct perf_event *bp, u64 bp_type) type = find_slot_idx(bp_type); weight = hw_breakpoint_weight(bp); - fetch_bp_busy_slots(&slots, bp, type); - /* - * Simulate the addition of this breakpoint to the constraints - * and see the result. - */ - fetch_this_slot(&slots, weight); - - /* Flexible counters need to keep at least one slot */ - if (slots.pinned + (!!slots.flexible) > nr_slots[type]) + /* Check if this new breakpoint can be satisfied across all CPUs. */ + max_pinned_slots = max_bp_pinned_slots(bp, type) + weight; + if (max_pinned_slots > hw_breakpoint_slots_cached(type)) return -ENOSPC; ret = arch_reserve_bp_slot(bp); if (ret) return ret; - toggle_bp_slot(bp, true, type, weight); - - return 0; + return toggle_bp_slot(bp, true, type, weight); } int reserve_bp_slot(struct perf_event *bp) { - int ret; - - mutex_lock(&nr_bp_mutex); - - ret = __reserve_bp_slot(bp, bp->attr.bp_type); - - mutex_unlock(&nr_bp_mutex); + struct mutex *mtx = bp_constraints_lock(bp); + int ret = __reserve_bp_slot(bp, bp->attr.bp_type); + bp_constraints_unlock(mtx); return ret; } @@ -335,17 +638,16 @@ static void __release_bp_slot(struct perf_event *bp, u64 bp_type) type = find_slot_idx(bp_type); weight = hw_breakpoint_weight(bp); - toggle_bp_slot(bp, false, type, weight); + WARN_ON(toggle_bp_slot(bp, false, type, weight)); } void release_bp_slot(struct perf_event *bp) { - mutex_lock(&nr_bp_mutex); + struct mutex *mtx = bp_constraints_lock(bp); arch_unregister_hw_breakpoint(bp); __release_bp_slot(bp, bp->attr.bp_type); - - mutex_unlock(&nr_bp_mutex); + bp_constraints_unlock(mtx); } static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type) @@ -372,11 +674,10 @@ static int __modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type) static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type) { - int ret; + struct mutex *mtx = bp_constraints_lock(bp); + int ret = __modify_bp_slot(bp, old_type, new_type); - mutex_lock(&nr_bp_mutex); - ret = __modify_bp_slot(bp, old_type, new_type); - mutex_unlock(&nr_bp_mutex); + bp_constraints_unlock(mtx); return ret; } @@ -387,18 +688,28 @@ static int modify_bp_slot(struct perf_event *bp, u64 old_type, u64 new_type) */ int dbg_reserve_bp_slot(struct perf_event *bp) { - if (mutex_is_locked(&nr_bp_mutex)) + int ret; + + if (bp_constraints_is_locked(bp)) return -1; - return __reserve_bp_slot(bp, bp->attr.bp_type); + /* Locks aren't held; disable lockdep assert checking. */ + lockdep_off(); + ret = __reserve_bp_slot(bp, bp->attr.bp_type); + lockdep_on(); + + return ret; } int dbg_release_bp_slot(struct perf_event *bp) { - if (mutex_is_locked(&nr_bp_mutex)) + if (bp_constraints_is_locked(bp)) return -1; + /* Locks aren't held; disable lockdep assert checking. */ + lockdep_off(); __release_bp_slot(bp, bp->attr.bp_type); + lockdep_on(); return 0; } @@ -604,6 +915,50 @@ void unregister_wide_hw_breakpoint(struct perf_event * __percpu *cpu_events) } EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint); +/** + * hw_breakpoint_is_used - check if breakpoints are currently used + * + * Returns: true if breakpoints are used, false otherwise. + */ +bool hw_breakpoint_is_used(void) +{ + int cpu; + + if (!constraints_initialized) + return false; + + for_each_possible_cpu(cpu) { + for (int type = 0; type < TYPE_MAX; ++type) { + struct bp_cpuinfo *info = get_bp_info(cpu, type); + + if (info->cpu_pinned) + return true; + + for (int slot = 0; slot < hw_breakpoint_slots_cached(type); ++slot) { + if (atomic_read(&info->tsk_pinned.count[slot])) + return true; + } + } + } + + for (int type = 0; type < TYPE_MAX; ++type) { + for (int slot = 0; slot < hw_breakpoint_slots_cached(type); ++slot) { + /* + * Warn, because if there are CPU pinned counters, + * should never get here; bp_cpuinfo::cpu_pinned should + * be consistent with the global cpu_pinned histogram. + */ + if (WARN_ON(atomic_read(&cpu_pinned[type].count[slot]))) + return true; + + if (atomic_read(&tsk_pinned_all[type].count[slot])) + return true; + } + } + + return false; +} + static struct notifier_block hw_breakpoint_exceptions_nb = { .notifier_call = hw_breakpoint_exceptions_notify, /* we need to be notified first */ @@ -678,38 +1033,19 @@ static struct pmu perf_breakpoint = { int __init init_hw_breakpoint(void) { - int cpu, err_cpu; - int i; - - for (i = 0; i < TYPE_MAX; i++) - nr_slots[i] = hw_breakpoint_slots(i); + int ret; - for_each_possible_cpu(cpu) { - for (i = 0; i < TYPE_MAX; i++) { - struct bp_cpuinfo *info = get_bp_info(cpu, i); + ret = rhltable_init(&task_bps_ht, &task_bps_ht_params); + if (ret) + return ret; - info->tsk_pinned = kcalloc(nr_slots[i], sizeof(int), - GFP_KERNEL); - if (!info->tsk_pinned) - goto err_alloc; - } - } + ret = init_breakpoint_slots(); + if (ret) + return ret; - constraints_initialized = 1; + constraints_initialized = true; perf_pmu_register(&perf_breakpoint, "breakpoint", PERF_TYPE_BREAKPOINT); return register_die_notifier(&hw_breakpoint_exceptions_nb); - - err_alloc: - for_each_possible_cpu(err_cpu) { - for (i = 0; i < TYPE_MAX; i++) - kfree(get_bp_info(err_cpu, i)->tsk_pinned); - if (err_cpu == cpu) - break; - } - - return -ENOMEM; } - - diff --git a/kernel/events/hw_breakpoint_test.c b/kernel/events/hw_breakpoint_test.c new file mode 100644 index 0000000000000000000000000000000000000000..5ced822df788943ba3c483d24ede8f052f7dde2f --- /dev/null +++ b/kernel/events/hw_breakpoint_test.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for hw_breakpoint constraints accounting logic. + * + * Copyright (C) 2022, Google LLC. + */ + +#include +#include +#include +#include +#include +#include + +#define TEST_REQUIRES_BP_SLOTS(test, slots) \ + do { \ + if ((slots) > get_test_bp_slots()) { \ + kunit_skip((test), "Requires breakpoint slots: %d > %d", slots, \ + get_test_bp_slots()); \ + } \ + } while (0) + +#define TEST_EXPECT_NOSPC(expr) KUNIT_EXPECT_EQ(test, -ENOSPC, PTR_ERR(expr)) + +#define MAX_TEST_BREAKPOINTS 512 + +static char break_vars[MAX_TEST_BREAKPOINTS]; +static struct perf_event *test_bps[MAX_TEST_BREAKPOINTS]; +static struct task_struct *__other_task; + +static struct perf_event *register_test_bp(int cpu, struct task_struct *tsk, int idx) +{ + struct perf_event_attr attr = {}; + + if (WARN_ON(idx < 0 || idx >= MAX_TEST_BREAKPOINTS)) + return NULL; + + hw_breakpoint_init(&attr); + attr.bp_addr = (unsigned long)&break_vars[idx]; + attr.bp_len = HW_BREAKPOINT_LEN_1; + attr.bp_type = HW_BREAKPOINT_RW; + return perf_event_create_kernel_counter(&attr, cpu, tsk, NULL, NULL); +} + +static void unregister_test_bp(struct perf_event **bp) +{ + if (WARN_ON(IS_ERR(*bp))) + return; + if (WARN_ON(!*bp)) + return; + unregister_hw_breakpoint(*bp); + *bp = NULL; +} + +static int get_test_bp_slots(void) +{ + static int slots; + + if (!slots) + slots = hw_breakpoint_slots(TYPE_DATA); + + return slots; +} + +static void fill_one_bp_slot(struct kunit *test, int *id, int cpu, struct task_struct *tsk) +{ + struct perf_event *bp = register_test_bp(cpu, tsk, *id); + + KUNIT_ASSERT_NOT_NULL(test, bp); + KUNIT_ASSERT_FALSE(test, IS_ERR(bp)); + KUNIT_ASSERT_NULL(test, test_bps[*id]); + test_bps[(*id)++] = bp; +} + +/* + * Fills up the given @cpu/@tsk with breakpoints, only leaving @skip slots free. + * + * Returns true if this can be called again, continuing at @id. + */ +static bool fill_bp_slots(struct kunit *test, int *id, int cpu, struct task_struct *tsk, int skip) +{ + for (int i = 0; i < get_test_bp_slots() - skip; ++i) + fill_one_bp_slot(test, id, cpu, tsk); + + return *id + get_test_bp_slots() <= MAX_TEST_BREAKPOINTS; +} + +static int dummy_kthread(void *arg) +{ + return 0; +} + +static struct task_struct *get_other_task(struct kunit *test) +{ + struct task_struct *tsk; + + if (__other_task) + return __other_task; + + tsk = kthread_create(dummy_kthread, NULL, "hw_breakpoint_dummy_task"); + KUNIT_ASSERT_FALSE(test, IS_ERR(tsk)); + __other_task = tsk; + return __other_task; +} + +static int get_test_cpu(int num) +{ + int cpu; + + WARN_ON(num < 0); + + for_each_online_cpu(cpu) { + if (num-- <= 0) + break; + } + + return cpu; +} + +/* ===== Test cases ===== */ + +static void test_one_cpu(struct kunit *test) +{ + int idx = 0; + + fill_bp_slots(test, &idx, get_test_cpu(0), NULL, 0); + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); +} + +static void test_many_cpus(struct kunit *test) +{ + int idx = 0; + int cpu; + + /* Test that CPUs are independent. */ + for_each_online_cpu(cpu) { + bool do_continue = fill_bp_slots(test, &idx, cpu, NULL, 0); + + TEST_EXPECT_NOSPC(register_test_bp(cpu, NULL, idx)); + if (!do_continue) + break; + } +} + +static void test_one_task_on_all_cpus(struct kunit *test) +{ + int idx = 0; + + fill_bp_slots(test, &idx, -1, current, 0); + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + /* Remove one and adding back CPU-target should work. */ + unregister_test_bp(&test_bps[0]); + fill_one_bp_slot(test, &idx, get_test_cpu(0), NULL); +} + +static void test_two_tasks_on_all_cpus(struct kunit *test) +{ + int idx = 0; + + /* Test that tasks are independent. */ + fill_bp_slots(test, &idx, -1, current, 0); + fill_bp_slots(test, &idx, -1, get_other_task(test), 0); + + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(-1, get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + /* Remove one from first task and adding back CPU-target should not work. */ + unregister_test_bp(&test_bps[0]); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); +} + +static void test_one_task_on_one_cpu(struct kunit *test) +{ + int idx = 0; + + fill_bp_slots(test, &idx, get_test_cpu(0), current, 0); + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + /* + * Remove one and adding back CPU-target should work; this case is + * special vs. above because the task's constraints are CPU-dependent. + */ + unregister_test_bp(&test_bps[0]); + fill_one_bp_slot(test, &idx, get_test_cpu(0), NULL); +} + +static void test_one_task_mixed(struct kunit *test) +{ + int idx = 0; + + TEST_REQUIRES_BP_SLOTS(test, 3); + + fill_one_bp_slot(test, &idx, get_test_cpu(0), current); + fill_bp_slots(test, &idx, -1, current, 1); + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + + /* Transition from CPU-dependent pinned count to CPU-independent. */ + unregister_test_bp(&test_bps[0]); + unregister_test_bp(&test_bps[1]); + fill_one_bp_slot(test, &idx, get_test_cpu(0), NULL); + fill_one_bp_slot(test, &idx, get_test_cpu(0), NULL); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); +} + +static void test_two_tasks_on_one_cpu(struct kunit *test) +{ + int idx = 0; + + fill_bp_slots(test, &idx, get_test_cpu(0), current, 0); + fill_bp_slots(test, &idx, get_test_cpu(0), get_other_task(test), 0); + + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(-1, get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + /* Can still create breakpoints on some other CPU. */ + fill_bp_slots(test, &idx, get_test_cpu(1), NULL, 0); +} + +static void test_two_tasks_on_one_all_cpus(struct kunit *test) +{ + int idx = 0; + + fill_bp_slots(test, &idx, get_test_cpu(0), current, 0); + fill_bp_slots(test, &idx, -1, get_other_task(test), 0); + + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(-1, get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), get_other_task(test), idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + /* Cannot create breakpoints on some other CPU either. */ + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(1), NULL, idx)); +} + +static void test_task_on_all_and_one_cpu(struct kunit *test) +{ + int tsk_on_cpu_idx, cpu_idx; + int idx = 0; + + TEST_REQUIRES_BP_SLOTS(test, 3); + + fill_bp_slots(test, &idx, -1, current, 2); + /* Transitioning from only all CPU breakpoints to mixed. */ + tsk_on_cpu_idx = idx; + fill_one_bp_slot(test, &idx, get_test_cpu(0), current); + fill_one_bp_slot(test, &idx, -1, current); + + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + + /* We should still be able to use up another CPU's slots. */ + cpu_idx = idx; + fill_one_bp_slot(test, &idx, get_test_cpu(1), NULL); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(1), NULL, idx)); + + /* Transitioning back to task target on all CPUs. */ + unregister_test_bp(&test_bps[tsk_on_cpu_idx]); + /* Still have a CPU target breakpoint in get_test_cpu(1). */ + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + /* Remove it and try again. */ + unregister_test_bp(&test_bps[cpu_idx]); + fill_one_bp_slot(test, &idx, -1, current); + + TEST_EXPECT_NOSPC(register_test_bp(-1, current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), current, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(0), NULL, idx)); + TEST_EXPECT_NOSPC(register_test_bp(get_test_cpu(1), NULL, idx)); +} + +static struct kunit_case hw_breakpoint_test_cases[] = { + KUNIT_CASE(test_one_cpu), + KUNIT_CASE(test_many_cpus), + KUNIT_CASE(test_one_task_on_all_cpus), + KUNIT_CASE(test_two_tasks_on_all_cpus), + KUNIT_CASE(test_one_task_on_one_cpu), + KUNIT_CASE(test_one_task_mixed), + KUNIT_CASE(test_two_tasks_on_one_cpu), + KUNIT_CASE(test_two_tasks_on_one_all_cpus), + KUNIT_CASE(test_task_on_all_and_one_cpu), + {}, +}; + +static int test_init(struct kunit *test) +{ + /* Most test cases want 2 distinct CPUs. */ + if (num_online_cpus() < 2) + return -EINVAL; + + /* Want the system to not use breakpoints elsewhere. */ + if (hw_breakpoint_is_used()) + return -EBUSY; + + return 0; +} + +static void test_exit(struct kunit *test) +{ + for (int i = 0; i < MAX_TEST_BREAKPOINTS; ++i) { + if (test_bps[i]) + unregister_test_bp(&test_bps[i]); + } + + if (__other_task) { + kthread_stop(__other_task); + __other_task = NULL; + } + + /* Verify that internal state agrees that no breakpoints are in use. */ + KUNIT_EXPECT_FALSE(test, hw_breakpoint_is_used()); +} + +static struct kunit_suite hw_breakpoint_test_suite = { + .name = "hw_breakpoint", + .test_cases = hw_breakpoint_test_cases, + .init = test_init, + .exit = test_exit, +}; + +kunit_test_suites(&hw_breakpoint_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Marco Elver "); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 2eaa327f8158dca5654cd4881058b8648c49be8e..d9e357b7e17c91cfc461bb06b0c3f89a7b649135 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -19,7 +19,7 @@ #include #include /* anon_vma_prepare */ #include /* set_pte_at_notify */ -#include /* try_to_free_swap */ +#include /* folio_free_swap */ #include /* user_enable_single_step */ #include /* notifier mechanism */ #include "../../mm/internal.h" /* munlock_vma_page */ @@ -154,8 +154,10 @@ static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) static int __replace_page(struct vm_area_struct *vma, unsigned long addr, struct page *old_page, struct page *new_page) { + struct folio *old_folio = page_folio(old_page); + struct folio *new_folio; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, page_folio(old_page), vma, addr, 0); + DEFINE_FOLIO_VMA_WALK(pvmw, old_folio, vma, addr, 0); int err; struct mmu_notifier_range range; @@ -163,14 +165,14 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, addr + PAGE_SIZE); if (new_page) { - err = mem_cgroup_charge(page_folio(new_page), vma->vm_mm, - GFP_KERNEL); + new_folio = page_folio(new_page); + err = mem_cgroup_charge(new_folio, vma->vm_mm, GFP_KERNEL); if (err) return err; } - /* For try_to_free_swap() below */ - lock_page(old_page); + /* For folio_free_swap() below */ + folio_lock(old_folio); mmu_notifier_invalidate_range_start(&range); err = -EAGAIN; @@ -179,14 +181,14 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, VM_BUG_ON_PAGE(addr != pvmw.address, old_page); if (new_page) { - get_page(new_page); + folio_get(new_folio); page_add_new_anon_rmap(new_page, vma, addr); - lru_cache_add_inactive_or_unevictable(new_page, vma); + folio_add_lru_vma(new_folio, vma); } else /* no new page, just dec_mm_counter for old_page */ dec_mm_counter(mm, MM_ANONPAGES); - if (!PageAnon(old_page)) { + if (!folio_test_anon(old_folio)) { dec_mm_counter(mm, mm_counter_file(old_page)); inc_mm_counter(mm, MM_ANONPAGES); } @@ -198,15 +200,15 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, mk_pte(new_page, vma->vm_page_prot)); page_remove_rmap(old_page, vma, false); - if (!page_mapped(old_page)) - try_to_free_swap(old_page); + if (!folio_mapped(old_folio)) + folio_free_swap(old_folio); page_vma_mapped_walk_done(&pvmw); - put_page(old_page); + folio_put(old_folio); err = 0; unlock: mmu_notifier_invalidate_range_end(&range); - unlock_page(old_page); + folio_unlock(old_folio); return err; } @@ -349,9 +351,10 @@ static bool valid_ref_ctr_vma(struct uprobe *uprobe, static struct vm_area_struct * find_ref_ctr_vma(struct uprobe *uprobe, struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *tmp; - for (tmp = mm->mmap; tmp; tmp = tmp->vm_next) + for_each_vma(vmi, tmp) if (valid_ref_ctr_vma(uprobe, tmp)) return tmp; @@ -552,7 +555,7 @@ put_old: /* try collapse pmd for compound page */ if (!ret && orig_page_huge) - collapse_pte_mapped_thp(mm, vaddr); + collapse_pte_mapped_thp(mm, vaddr, false); return ret; } @@ -1231,11 +1234,12 @@ int uprobe_apply(struct inode *inode, loff_t offset, static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; int err = 0; mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { unsigned long vaddr; loff_t offset; @@ -1983,9 +1987,10 @@ bool uprobe_deny_signal(void) static void mmf_recalc_uprobes(struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (!valid_vma(vma, false)) continue; /* diff --git a/kernel/exit.c b/kernel/exit.c index 84021b24f79e3d1d2dc11fe70bf69cf620c7fc85..35e0a31a0315c1f4f3aa86c17903d118b16fa1d9 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -183,6 +184,10 @@ void put_task_struct_rcu_user(struct task_struct *task) call_rcu(&task->rcu, delayed_put_task_struct); } +void __weak release_thread(struct task_struct *dead_task) +{ +} + void release_task(struct task_struct *p) { struct task_struct *leader; @@ -374,10 +379,10 @@ static void coredump_task_exit(struct task_struct *tsk) complete(&core_state->startup); for (;;) { - set_current_state(TASK_UNINTERRUPTIBLE); + set_current_state(TASK_UNINTERRUPTIBLE|TASK_FREEZABLE); if (!self.task) /* see coredump_finish() */ break; - freezable_schedule(); + schedule(); } __set_current_state(TASK_RUNNING); } @@ -466,6 +471,7 @@ assign_new_owner: goto retry; } WRITE_ONCE(mm->owner, c); + lru_gen_migrate_mm(mm); task_unlock(c); put_task_struct(c); } @@ -733,14 +739,33 @@ static void check_stack_usage(void) static inline void check_stack_usage(void) {} #endif +static void synchronize_group_exit(struct task_struct *tsk, long code) +{ + struct sighand_struct *sighand = tsk->sighand; + struct signal_struct *signal = tsk->signal; + + spin_lock_irq(&sighand->siglock); + signal->quick_threads--; + if ((signal->quick_threads == 0) && + !(signal->flags & SIGNAL_GROUP_EXIT)) { + signal->flags = SIGNAL_GROUP_EXIT; + signal->group_exit_code = code; + signal->group_stop_count = 0; + } + spin_unlock_irq(&sighand->siglock); +} + void __noreturn do_exit(long code) { struct task_struct *tsk = current; int group_dead; + synchronize_group_exit(tsk, code); + WARN_ON(tsk->plug); kcov_task_exit(tsk); + kmsan_task_exit(tsk); coredump_task_exit(tsk); ptrace_event(PTRACE_EVENT_EXIT, code); @@ -905,7 +930,7 @@ do_group_exit(int exit_code) exit_code = sig->group_exit_code; else if (sig->group_exec_task) exit_code = 0; - else if (!thread_group_empty(current)) { + else { struct sighand_struct *const sighand = current->sighand; spin_lock_irq(&sighand->siglock); diff --git a/kernel/fail_function.c b/kernel/fail_function.c index 60dc825ecc2b31ee3add395282fc79cf1e8c5f3d..a7ccd2930c5f46169f6f7eda93113f9ac8a1568b 100644 --- a/kernel/fail_function.c +++ b/kernel/fail_function.c @@ -247,15 +247,11 @@ static ssize_t fei_write(struct file *file, const char __user *buffer, /* cut off if it is too long */ if (count > KSYM_NAME_LEN) count = KSYM_NAME_LEN; - buf = kmalloc(count + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - if (copy_from_user(buf, buffer, count)) { - ret = -EFAULT; - goto out_free; - } - buf[count] = '\0'; + buf = memdup_user_nul(buffer, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + sym = strstrip(buf); mutex_lock(&fei_lock); @@ -298,17 +294,15 @@ static ssize_t fei_write(struct file *file, const char __user *buffer, } ret = register_kprobe(&attr->kp); - if (!ret) - fei_debugfs_add_attr(attr); - if (ret < 0) - fei_attr_remove(attr); - else { - list_add_tail(&attr->list, &fei_attr_list); - ret = count; + if (ret) { + fei_attr_free(attr); + goto out; } + fei_debugfs_add_attr(attr); + list_add_tail(&attr->list, &fei_attr_list); + ret = count; out: mutex_unlock(&fei_lock); -out_free: kfree(buf); return ret; } diff --git a/kernel/fork.c b/kernel/fork.c index 90c85b17bf698744c12cb165c05396fa55db7c3a..08969f5aa38d5906b2f42f975054c62bc7f7bf80 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -37,13 +37,13 @@ #include #include #include +#include #include #include #include #include #include #include -#include #include #include #include @@ -97,7 +97,6 @@ #include #include #include -#include #include #include @@ -475,7 +474,6 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig) */ *new = data_race(*orig); INIT_LIST_HEAD(&new->anon_vma_chain); - new->vm_next = new->vm_prev = NULL; dup_anon_vma_name(orig, new); } return new; @@ -580,11 +578,12 @@ static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) static __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) { - struct vm_area_struct *mpnt, *tmp, *prev, **pprev; - struct rb_node **rb_link, *rb_parent; + struct vm_area_struct *mpnt, *tmp; int retval; - unsigned long charge; + unsigned long charge = 0; LIST_HEAD(uf); + MA_STATE(old_mas, &oldmm->mm_mt, 0, 0); + MA_STATE(mas, &mm->mm_mt, 0, 0); uprobe_start_dup_mmap(); if (mmap_write_lock_killable(oldmm)) { @@ -606,16 +605,16 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, mm->exec_vm = oldmm->exec_vm; mm->stack_vm = oldmm->stack_vm; - rb_link = &mm->mm_rb.rb_node; - rb_parent = NULL; - pprev = &mm->mmap; retval = ksm_fork(mm, oldmm); if (retval) goto out; khugepaged_fork(mm, oldmm); - prev = NULL; - for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { + retval = mas_expected_entries(&mas, oldmm->map_count); + if (retval) + goto out; + + mas_for_each(&old_mas, mpnt, ULONG_MAX) { struct file *file; if (mpnt->vm_flags & VM_DONTCOPY) { @@ -629,7 +628,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, */ if (fatal_signal_pending(current)) { retval = -EINTR; - goto out; + goto loop_out; } if (mpnt->vm_flags & VM_ACCOUNT) { unsigned long len = vma_pages(mpnt); @@ -675,24 +674,17 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, } /* - * Clear hugetlb-related page reserves for children. This only - * affects MAP_PRIVATE mappings. Faults generated by the child - * are not guaranteed to succeed, even if read-only + * Copy/update hugetlb private vma information. */ if (is_vm_hugetlb_page(tmp)) - reset_vma_resv_huge_pages(tmp); - - /* - * Link in the new vma and copy the page table entries. - */ - *pprev = tmp; - pprev = &tmp->vm_next; - tmp->vm_prev = prev; - prev = tmp; + hugetlb_dup_vma_private(tmp); - __vma_link_rb(mm, tmp, rb_link, rb_parent); - rb_link = &tmp->vm_rb.rb_right; - rb_parent = &tmp->vm_rb; + /* Link the vma into the MT */ + mas.index = tmp->vm_start; + mas.last = tmp->vm_end - 1; + mas_store(&mas, tmp); + if (mas_is_err(&mas)) + goto fail_nomem_mas_store; mm->map_count++; if (!(tmp->vm_flags & VM_WIPEONFORK)) @@ -702,10 +694,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, tmp->vm_ops->open(tmp); if (retval) - goto out; + goto loop_out; } /* a new mm has just been created */ retval = arch_dup_mmap(oldmm, mm); +loop_out: + mas_destroy(&mas); out: mmap_write_unlock(mm); flush_tlb_mm(oldmm); @@ -714,6 +708,9 @@ out: fail_uprobe_end: uprobe_end_dup_mmap(); return retval; + +fail_nomem_mas_store: + unlink_anon_vmas(tmp); fail_nomem_anon_vma_fork: mpol_put(vma_policy(tmp)); fail_nomem_policy: @@ -721,7 +718,7 @@ fail_nomem_policy: fail_nomem: retval = -ENOMEM; vm_unacct_memory(charge); - goto out; + goto loop_out; } static inline int mm_alloc_pgd(struct mm_struct *mm) @@ -925,13 +922,13 @@ void __init fork_init(void) init_task.signal->rlim[RLIMIT_SIGPENDING] = init_task.signal->rlim[RLIMIT_NPROC]; - for (i = 0; i < MAX_PER_NAMESPACE_UCOUNTS; i++) + for (i = 0; i < UCOUNT_COUNTS; i++) init_user_ns.ucount_max[i] = max_threads/2; - set_rlimit_ucount_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY); - set_rlimit_ucount_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY); - set_rlimit_ucount_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY); - set_rlimit_ucount_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); + set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_NPROC, RLIM_INFINITY); + set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MSGQUEUE, RLIM_INFINITY); + set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_SIGPENDING, RLIM_INFINITY); + set_userns_rlimit_max(&init_user_ns, UCOUNT_RLIMIT_MEMLOCK, RLIM_INFINITY); #ifdef CONFIG_VMAP_STACK cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "fork:vm_stack_cache", @@ -1026,6 +1023,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) tsk->worker_private = NULL; kcov_task_init(tsk); + kmsan_task_create(tsk); kmap_local_fork(tsk); #ifdef CONFIG_FAULT_INJECTION @@ -1109,9 +1107,8 @@ static void mm_init_uprobes_state(struct mm_struct *mm) static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, struct user_namespace *user_ns) { - mm->mmap = NULL; - mm->mm_rb = RB_ROOT; - mm->vmacache_seqnum = 0; + mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); + mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); atomic_set(&mm->mm_users, 1); atomic_set(&mm->mm_count, 1); seqcount_init(&mm->write_protect_seq); @@ -1152,6 +1149,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, goto fail_nocontext; mm->user_ns = get_user_ns(user_ns); + lru_gen_init_mm(mm); return mm; fail_nocontext: @@ -1194,6 +1192,7 @@ static inline void __mmput(struct mm_struct *mm) } if (mm->binfmt) module_put(mm->binfmt->module); + lru_gen_del_mm(mm); mmdrop(mm); } @@ -1225,6 +1224,7 @@ void mmput_async(struct mm_struct *mm) schedule_work(&mm->async_put_work); } } +EXPORT_SYMBOL_GPL(mmput_async); #endif /** @@ -1284,13 +1284,16 @@ int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) /* Forbid mm->exe_file change if old file still mapped. */ old_exe_file = get_mm_exe_file(mm); if (old_exe_file) { + VMA_ITERATOR(vmi, mm, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma && !ret; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (!vma->vm_file) continue; if (path_equal(&vma->vm_file->f_path, - &old_exe_file->f_path)) + &old_exe_file->f_path)) { ret = -EBUSY; + break; + } } mmap_read_unlock(mm); fput(old_exe_file); @@ -1420,13 +1423,12 @@ static void complete_vfork_done(struct task_struct *tsk) static int wait_for_vfork_done(struct task_struct *child, struct completion *vfork) { + unsigned int state = TASK_UNINTERRUPTIBLE|TASK_KILLABLE|TASK_FREEZABLE; int killed; - freezer_do_not_count(); cgroup_enter_frozen(); - killed = wait_for_completion_killable(vfork); + killed = wait_for_completion_state(vfork, state); cgroup_leave_frozen(false); - freezer_count(); if (killed) { task_lock(child); @@ -1566,9 +1568,6 @@ static int copy_mm(unsigned long clone_flags, struct task_struct *tsk) if (!oldmm) return 0; - /* initialize the new vmacache entries */ - vmacache_flush(tsk); - if (clone_flags & CLONE_VM) { mmget(oldmm); mm = oldmm; @@ -1692,6 +1691,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) return -ENOMEM; sig->nr_threads = 1; + sig->quick_threads = 1; atomic_set(&sig->live, 1); refcount_set(&sig->sigcnt, 1); @@ -2046,11 +2046,8 @@ static __latent_entropy struct task_struct *copy_process( /* * If the new process will be in a different time namespace * do not allow it to share VM or a thread group with the forking task. - * - * On vfork, the child process enters the target time namespace only - * after exec. */ - if ((clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM) { + if (clone_flags & (CLONE_THREAD | CLONE_VM)) { if (nsp->time_ns != nsp->time_ns_for_children) return ERR_PTR(-EINVAL); } @@ -2118,7 +2115,7 @@ static __latent_entropy struct task_struct *copy_process( goto bad_fork_free; retval = -EAGAIN; - if (is_ucounts_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { + if (is_rlimit_overlimit(task_ucounts(p), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) { if (p->real_cred->user != INIT_USER && !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN)) goto bad_fork_cleanup_count; @@ -2462,6 +2459,7 @@ static __latent_entropy struct task_struct *copy_process( __this_cpu_inc(process_counts); } else { current->signal->nr_threads++; + current->signal->quick_threads++; atomic_inc(¤t->signal->live); refcount_inc(¤t->signal->sigcnt); task_join_group_stop(p); @@ -2694,6 +2692,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) get_task_struct(p); } + if (IS_ENABLED(CONFIG_LRU_GEN) && !(clone_flags & CLONE_VM)) { + /* lock the task to synchronize with memcg migration */ + task_lock(p); + lru_gen_add_mm(p->mm); + task_unlock(p); + } + wake_up_new_task(p); /* forking complete and child started to run, tell ptracer */ diff --git a/kernel/freezer.c b/kernel/freezer.c index 45ab36ffd0e79cf84fe4a47323811e27ae11357d..4fad0e6fca6447d72388e7ea44e90f34924beb03 100644 --- a/kernel/freezer.c +++ b/kernel/freezer.c @@ -13,10 +13,11 @@ #include /* total number of freezing conditions in effect */ -atomic_t system_freezing_cnt = ATOMIC_INIT(0); -EXPORT_SYMBOL(system_freezing_cnt); +DEFINE_STATIC_KEY_FALSE(freezer_active); +EXPORT_SYMBOL(freezer_active); -/* indicate whether PM freezing is in effect, protected by +/* + * indicate whether PM freezing is in effect, protected by * system_transition_mutex */ bool pm_freezing; @@ -29,7 +30,7 @@ static DEFINE_SPINLOCK(freezer_lock); * freezing_slow_path - slow path for testing whether a task needs to be frozen * @p: task to be tested * - * This function is called by freezing() if system_freezing_cnt isn't zero + * This function is called by freezing() if freezer_active isn't zero * and tests whether @p needs to enter and stay in frozen state. Can be * called under any context. The freezers are responsible for ensuring the * target tasks see the updated state. @@ -52,41 +53,40 @@ bool freezing_slow_path(struct task_struct *p) } EXPORT_SYMBOL(freezing_slow_path); +bool frozen(struct task_struct *p) +{ + return READ_ONCE(p->__state) & TASK_FROZEN; +} + /* Refrigerator is place where frozen processes are stored :-). */ bool __refrigerator(bool check_kthr_stop) { - /* Hmm, should we be allowed to suspend when there are realtime - processes around? */ + unsigned int state = get_current_state(); bool was_frozen = false; - unsigned int save = get_current_state(); pr_debug("%s entered refrigerator\n", current->comm); + WARN_ON_ONCE(state && !(state & TASK_NORMAL)); + for (;;) { - set_current_state(TASK_UNINTERRUPTIBLE); + bool freeze; + + set_current_state(TASK_FROZEN); spin_lock_irq(&freezer_lock); - current->flags |= PF_FROZEN; - if (!freezing(current) || - (check_kthr_stop && kthread_should_stop())) - current->flags &= ~PF_FROZEN; + freeze = freezing(current) && !(check_kthr_stop && kthread_should_stop()); spin_unlock_irq(&freezer_lock); - if (!(current->flags & PF_FROZEN)) + if (!freeze) break; + was_frozen = true; schedule(); } + __set_current_state(TASK_RUNNING); pr_debug("%s left refrigerator\n", current->comm); - /* - * Restore saved task state before returning. The mb'd version - * needs to be used; otherwise, it might silently break - * synchronization which depends on ordered task state change. - */ - set_current_state(save); - return was_frozen; } EXPORT_SYMBOL(__refrigerator); @@ -101,6 +101,44 @@ static void fake_signal_wake_up(struct task_struct *p) } } +static int __set_task_frozen(struct task_struct *p, void *arg) +{ + unsigned int state = READ_ONCE(p->__state); + + if (p->on_rq) + return 0; + + if (p != current && task_curr(p)) + return 0; + + if (!(state & (TASK_FREEZABLE | __TASK_STOPPED | __TASK_TRACED))) + return 0; + + /* + * Only TASK_NORMAL can be augmented with TASK_FREEZABLE, since they + * can suffer spurious wakeups. + */ + if (state & TASK_FREEZABLE) + WARN_ON_ONCE(!(state & TASK_NORMAL)); + +#ifdef CONFIG_LOCKDEP + /* + * It's dangerous to freeze with locks held; there be dragons there. + */ + if (!(state & __TASK_FREEZABLE_UNSAFE)) + WARN_ON_ONCE(debug_locks && p->lockdep_depth); +#endif + + WRITE_ONCE(p->__state, TASK_FROZEN); + return TASK_FROZEN; +} + +static bool __freeze_task(struct task_struct *p) +{ + /* TASK_FREEZABLE|TASK_STOPPED|TASK_TRACED -> TASK_FROZEN */ + return task_call_func(p, __set_task_frozen, NULL); +} + /** * freeze_task - send a freeze request to given task * @p: task to send the request to @@ -116,20 +154,8 @@ bool freeze_task(struct task_struct *p) { unsigned long flags; - /* - * This check can race with freezer_do_not_count, but worst case that - * will result in an extra wakeup being sent to the task. It does not - * race with freezer_count(), the barriers in freezer_count() and - * freezer_should_skip() ensure that either freezer_count() sees - * freezing == true in try_to_freeze() and freezes, or - * freezer_should_skip() sees !PF_FREEZE_SKIP and freezes the task - * normally. - */ - if (freezer_should_skip(p)) - return false; - spin_lock_irqsave(&freezer_lock, flags); - if (!freezing(p) || frozen(p)) { + if (!freezing(p) || frozen(p) || __freeze_task(p)) { spin_unlock_irqrestore(&freezer_lock, flags); return false; } @@ -137,19 +163,52 @@ bool freeze_task(struct task_struct *p) if (!(p->flags & PF_KTHREAD)) fake_signal_wake_up(p); else - wake_up_state(p, TASK_INTERRUPTIBLE); + wake_up_state(p, TASK_NORMAL); spin_unlock_irqrestore(&freezer_lock, flags); return true; } +/* + * The special task states (TASK_STOPPED, TASK_TRACED) keep their canonical + * state in p->jobctl. If either of them got a wakeup that was missed because + * TASK_FROZEN, then their canonical state reflects that and the below will + * refuse to restore the special state and instead issue the wakeup. + */ +static int __set_task_special(struct task_struct *p, void *arg) +{ + unsigned int state = 0; + + if (p->jobctl & JOBCTL_TRACED) + state = TASK_TRACED; + + else if (p->jobctl & JOBCTL_STOPPED) + state = TASK_STOPPED; + + if (state) + WRITE_ONCE(p->__state, state); + + return state; +} + void __thaw_task(struct task_struct *p) { - unsigned long flags; + unsigned long flags, flags2; spin_lock_irqsave(&freezer_lock, flags); - if (frozen(p)) - wake_up_process(p); + if (WARN_ON_ONCE(freezing(p))) + goto unlock; + + if (lock_task_sighand(p, &flags2)) { + /* TASK_FROZEN -> TASK_{STOPPED,TRACED} */ + bool ret = task_call_func(p, __set_task_special, NULL); + unlock_task_sighand(p, &flags2); + if (ret) + goto unlock; + } + + wake_up_state(p, TASK_FROZEN); +unlock: spin_unlock_irqrestore(&freezer_lock, flags); } diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 4ce0923f1ce39e4226992efa097cd6ce824570e0..ba01b94082033b18e2c07e7f67599862f5f12929 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -334,7 +334,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, * futex_queue() calls spin_unlock() upon completion, both serializing * access to the hash list and forcing another memory barrier. */ - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); futex_queue(q, hb); /* Arm the timer */ @@ -352,7 +352,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, * is no timeout, or if it has yet to expire. */ if (!timeout || timeout->task) - freezable_schedule(); + schedule(); } __set_current_state(TASK_RUNNING); } @@ -430,7 +430,7 @@ retry: return ret; } - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); for (i = 0; i < count; i++) { u32 __user *uaddr = (u32 __user *)(unsigned long)vs[i].w.uaddr; @@ -504,7 +504,7 @@ static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count, return; } - freezable_schedule(); + schedule(); } /** diff --git a/kernel/gen_kheaders.sh b/kernel/gen_kheaders.sh index 0c78e64f747df590616a2e7a8da4d8a08c379b41..473036b43c832b2a94c8eef8edb9ef82c11b9a6c 100755 --- a/kernel/gen_kheaders.sh +++ b/kernel/gen_kheaders.sh @@ -31,8 +31,8 @@ if [ "$building_out_of_srctree" ]; then fi all_dirs="$all_dirs $dir_list" -# include/generated/compile.h is ignored because it is touched even when none -# of the source files changed. +# include/generated/utsversion.h is ignored because it is generated after this +# script is executed. (utsversion.h is unneeded for kheaders) # # When Kconfig regenerates include/generated/autoconf.h, its timestamp is # updated, but the contents might be still the same. When any CONFIG option is @@ -42,7 +42,7 @@ all_dirs="$all_dirs $dir_list" # # Ignore them for md5 calculation to avoid pointless regeneration. headers_md5="$(find $all_dirs -name "*.h" | - grep -v "include/generated/compile.h" | + grep -v "include/generated/utsversion.h" | grep -v "include/generated/autoconf.h" | xargs ls -l | md5sum | cut -d ' ' -f1)" diff --git a/kernel/hung_task.c b/kernel/hung_task.c index bb2354f73dedcaf2c2d82ebebca2d5b9b86d2bf0..c71889f3f3fc23ca7da3215caa5f5a4e69e7a41c 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c @@ -95,8 +95,8 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout) * Ensure the task is not frozen. * Also, skip vfork and any other user process that freezer should skip. */ - if (unlikely(t->flags & (PF_FROZEN | PF_FREEZER_SKIP))) - return; + if (unlikely(READ_ONCE(t->__state) & TASK_FROZEN)) + return; /* * When a freshly created task is scheduled once, changes its state to @@ -191,6 +191,8 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) hung_task_show_lock = false; rcu_read_lock(); for_each_process_thread(g, t) { + unsigned int state; + if (!max_count--) goto unlock; if (time_after(jiffies, last_break + HUNG_TASK_LOCK_BREAK)) { @@ -198,8 +200,14 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) goto unlock; last_break = jiffies; } - /* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */ - if (READ_ONCE(t->__state) == TASK_UNINTERRUPTIBLE) + /* + * skip the TASK_KILLABLE tasks -- these can be killed + * skip the TASK_IDLE tasks -- those are genuinely idle + */ + state = READ_ONCE(t->__state); + if ((state & TASK_UNINTERRUPTIBLE) && + !(state & TASK_WAKEKILL) && + !(state & TASK_NOLOAD)) check_hung_task(t, timeout); } unlock: diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 5db0230aa6b52fbd945bba99d60b498a08c66277..a91f9001103ceb1b723721ac562074195d84e088 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -705,6 +705,30 @@ int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq) } EXPORT_SYMBOL_GPL(generic_handle_domain_irq); + /** + * generic_handle_irq_safe - Invoke the handler for a HW irq belonging + * to a domain from any context. + * @domain: The domain where to perform the lookup + * @hwirq: The HW irq number to convert to a logical one + * + * Returns: 0 on success, a negative value on error. + * + * This function can be called from any context (IRQ or process + * context). If the interrupt is marked as 'enforce IRQ-context only' then + * the function must be invoked from hard interrupt context. + */ +int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq) +{ + unsigned long flags; + int ret; + + local_irq_save(flags); + ret = handle_irq_desc(irq_resolve_mapping(domain, hwirq)); + local_irq_restore(flags); + return ret; +} +EXPORT_SYMBOL_GPL(generic_handle_domain_irq_safe); + /** * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging * to a domain. diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 3e7e2c2ad2f75ef603c32750e94c7afe5fdef994..60c20f301a6ba2c794c7716c87e7bde61de196e7 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -50,12 +50,20 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, data = &kallsyms_names[off]; len = *data; data++; + off++; + + /* If MSB is 1, it is a "big" symbol, so needs an additional byte. */ + if ((len & 0x80) != 0) { + len = (len & 0x7F) | (*data << 7); + data++; + off++; + } /* * Update the offset to return the offset for the next symbol on * the compressed stream. */ - off += len + 1; + off += len; /* * For every byte on the compressed symbol data, copy the table @@ -108,7 +116,7 @@ static char kallsyms_get_symbol_type(unsigned int off) static unsigned int get_symbol_offset(unsigned long pos) { const u8 *name; - int i; + int i, len; /* * Use the closest marker we have. We have markers every 256 positions, @@ -122,8 +130,18 @@ static unsigned int get_symbol_offset(unsigned long pos) * so we just need to add the len to the current pointer for every * symbol we wish to skip. */ - for (i = 0; i < (pos & 0xFF); i++) - name = name + (*name) + 1; + for (i = 0; i < (pos & 0xFF); i++) { + len = *name; + + /* + * If MSB is 1, it is a "big" symbol, so we need to look into + * the next byte (and skip it, too). + */ + if ((len & 0x80) != 0) + len = ((len & 0x7F) | (name[1] << 7)) + 1; + + name = name + len + 1; + } return name - kallsyms_names; } @@ -159,7 +177,6 @@ static bool cleanup_symbol_name(char *s) * character in an identifier in C. Suffixes observed: * - foo.llvm.[0-9a-f]+ * - foo.[0-9a-f]+ - * - foo.[0-9a-f]+.cfi_jt */ res = strchr(s, '.'); if (res) { @@ -167,22 +184,6 @@ static bool cleanup_symbol_name(char *s) return true; } - if (!IS_ENABLED(CONFIG_CFI_CLANG) || - !IS_ENABLED(CONFIG_LTO_CLANG_THIN) || - CONFIG_CLANG_VERSION >= 130000) - return false; - - /* - * Prior to LLVM 13, the following suffixes were observed when thinLTO - * and CFI are both enabled: - * - foo$[0-9]+ - */ - res = strrchr(s, '$'); - if (res) { - *res = '\0'; - return true; - } - return false; } diff --git a/kernel/kcov.c b/kernel/kcov.c index e19c84b02452eb2d0c0fa46ac3248535c6a81736..e5cd09fd8a05087cac8226ab5fa1fc693927b88e 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -152,6 +153,12 @@ static void kcov_remote_area_put(struct kcov_remote_area *area, INIT_LIST_HEAD(&area->list); area->size = size; list_add(&area->list, &kcov_remote_areas); + /* + * KMSAN doesn't instrument this file, so it may not know area->list + * is initialized. Unpoison it explicitly to avoid reports in + * kcov_remote_area_get(). + */ + kmsan_unpoison_memory(&area->list, sizeof(area->list)); } static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t) diff --git a/kernel/kcsan/selftest.c b/kernel/kcsan/selftest.c index 75712959c84e01db3c86fd2135604c8d76f7a171..00cdf8fa5693623273eb1244706ae3d5f6aeaab8 100644 --- a/kernel/kcsan/selftest.c +++ b/kernel/kcsan/selftest.c @@ -26,7 +26,7 @@ static bool __init test_requires(void) { /* random should be initialized for the below tests */ - return prandom_u32() + prandom_u32() != 0; + return get_random_u32() + get_random_u32() != 0; } /* @@ -46,7 +46,7 @@ static bool __init test_encode_decode(void) unsigned long addr; size_t verif_size; - prandom_bytes(&addr, sizeof(addr)); + get_random_bytes(&addr, sizeof(addr)); if (addr < PAGE_SIZE) addr = PAGE_SIZE; diff --git a/kernel/kexec.c b/kernel/kexec.c index b5e40f06976815e60b96dc8166c333a2562b3565..cb8e6e6f983c7900e9950ed2e0b719ff4a524f03 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -93,13 +93,10 @@ static int do_kexec_load(unsigned long entry, unsigned long nr_segments, /* * Because we write directly to the reserved memory region when loading - * crash kernels we need a mutex here to prevent multiple crash kernels - * from attempting to load simultaneously, and to prevent a crash kernel - * from loading over the top of a in use crash kernel. - * - * KISS: always take the mutex. + * crash kernels we need a serialization here to prevent multiple crash + * kernels from attempting to load simultaneously. */ - if (!mutex_trylock(&kexec_mutex)) + if (!kexec_trylock()) return -EBUSY; if (flags & KEXEC_ON_CRASH) { @@ -165,7 +162,7 @@ out: kimage_free(image); out_unlock: - mutex_unlock(&kexec_mutex); + kexec_unlock(); return ret; } diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index acd029b307e42d32a5143bd03ab38f05dc094236..ca2743f9c634e4c94b4f4f30c19cd33549fb4cdf 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -46,7 +46,7 @@ #include #include "kexec_internal.h" -DEFINE_MUTEX(kexec_mutex); +atomic_t __kexec_lock = ATOMIC_INIT(0); /* Per cpu memory for storing cpu states in case of system crash. */ note_buf_t __percpu *crash_notes; @@ -809,7 +809,7 @@ static int kimage_load_normal_segment(struct kimage *image, if (result < 0) goto out; - ptr = kmap(page); + ptr = kmap_local_page(page); /* Start with a clear page */ clear_page(ptr); ptr += maddr & ~PAGE_MASK; @@ -822,7 +822,7 @@ static int kimage_load_normal_segment(struct kimage *image, memcpy(ptr, kbuf, uchunk); else result = copy_from_user(ptr, buf, uchunk); - kunmap(page); + kunmap_local(ptr); if (result) { result = -EFAULT; goto out; @@ -873,7 +873,7 @@ static int kimage_load_crash_segment(struct kimage *image, goto out; } arch_kexec_post_alloc_pages(page_address(page), 1, 0); - ptr = kmap(page); + ptr = kmap_local_page(page); ptr += maddr & ~PAGE_MASK; mchunk = min_t(size_t, mbytes, PAGE_SIZE - (maddr & ~PAGE_MASK)); @@ -889,7 +889,7 @@ static int kimage_load_crash_segment(struct kimage *image, else result = copy_from_user(ptr, buf, uchunk); kexec_flush_icache_page(page); - kunmap(page); + kunmap_local(ptr); arch_kexec_pre_free_pages(page_address(page), 1); if (result) { result = -EFAULT; @@ -959,7 +959,7 @@ late_initcall(kexec_core_sysctl_init); */ void __noclone __crash_kexec(struct pt_regs *regs) { - /* Take the kexec_mutex here to prevent sys_kexec_load + /* Take the kexec_lock here to prevent sys_kexec_load * running on one cpu from replacing the crash kernel * we are using after a panic on a different cpu. * @@ -967,7 +967,7 @@ void __noclone __crash_kexec(struct pt_regs *regs) * of memory the xchg(&kexec_crash_image) would be * sufficient. But since I reuse the memory... */ - if (mutex_trylock(&kexec_mutex)) { + if (kexec_trylock()) { if (kexec_crash_image) { struct pt_regs fixed_regs; @@ -976,7 +976,7 @@ void __noclone __crash_kexec(struct pt_regs *regs) machine_crash_shutdown(&fixed_regs); machine_kexec(kexec_crash_image); } - mutex_unlock(&kexec_mutex); + kexec_unlock(); } } STACK_FRAME_NON_STANDARD(__crash_kexec); @@ -1004,14 +1004,17 @@ void crash_kexec(struct pt_regs *regs) } } -size_t crash_get_memory_size(void) +ssize_t crash_get_memory_size(void) { - size_t size = 0; + ssize_t size = 0; + + if (!kexec_trylock()) + return -EBUSY; - mutex_lock(&kexec_mutex); if (crashk_res.end != crashk_res.start) size = resource_size(&crashk_res); - mutex_unlock(&kexec_mutex); + + kexec_unlock(); return size; } @@ -1022,7 +1025,8 @@ int crash_shrink_memory(unsigned long new_size) unsigned long old_size; struct resource *ram_res; - mutex_lock(&kexec_mutex); + if (!kexec_trylock()) + return -EBUSY; if (kexec_crash_image) { ret = -ENOENT; @@ -1060,7 +1064,7 @@ int crash_shrink_memory(unsigned long new_size) insert_resource(&iomem_resource, ram_res); unlock: - mutex_unlock(&kexec_mutex); + kexec_unlock(); return ret; } @@ -1132,7 +1136,7 @@ int kernel_kexec(void) { int error = 0; - if (!mutex_trylock(&kexec_mutex)) + if (!kexec_trylock()) return -EBUSY; if (!kexec_image) { error = -EINVAL; @@ -1208,6 +1212,6 @@ int kernel_kexec(void) #endif Unlock: - mutex_unlock(&kexec_mutex); + kexec_unlock(); return error; } diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 1d546dc97c5023a9aa148277acd45ba026609c22..45637511e0de69ee5ef68c7936875efb813a6d06 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -339,7 +339,7 @@ SYSCALL_DEFINE5(kexec_file_load, int, kernel_fd, int, initrd_fd, image = NULL; - if (!mutex_trylock(&kexec_mutex)) + if (!kexec_trylock()) return -EBUSY; dest_image = &kexec_image; @@ -411,7 +411,7 @@ out: if ((flags & KEXEC_FILE_ON_CRASH) && kexec_crash_image) arch_kexec_protect_crashkres(); - mutex_unlock(&kexec_mutex); + kexec_unlock(); kimage_free(image); return ret; } diff --git a/kernel/kexec_internal.h b/kernel/kexec_internal.h index 48aaf2ac0d0d15d7cd1dca3c42fa9b2f00313c9e..74da1409cd14b5db50c60832eb7445a747ae0447 100644 --- a/kernel/kexec_internal.h +++ b/kernel/kexec_internal.h @@ -13,7 +13,20 @@ void kimage_terminate(struct kimage *image); int kimage_is_destination_range(struct kimage *image, unsigned long start, unsigned long end); -extern struct mutex kexec_mutex; +/* + * Whatever is used to serialize accesses to the kexec_crash_image needs to be + * NMI safe, as __crash_kexec() can happen during nmi_panic(), so here we use a + * "simple" atomic variable that is acquired with a cmpxchg(). + */ +extern atomic_t __kexec_lock; +static inline bool kexec_trylock(void) +{ + return atomic_cmpxchg_acquire(&__kexec_lock, 0, 1) == 0; +} +static inline void kexec_unlock(void) +{ + atomic_set_release(&__kexec_lock, 0); +} #ifdef CONFIG_KEXEC_FILE #include diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 80697e5e03e49121211c80c3b8b6fc95bab9b904..3220b0a2fb4a318d4d10d8dda288a082d0499b2f 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1562,6 +1562,7 @@ static int check_kprobe_address_safe(struct kprobe *p, /* Ensure it is not in reserved area nor out of text */ if (!(core_kernel_text((unsigned long) p->addr) || is_module_text_address((unsigned long) p->addr)) || + in_gate_area_no_mm((unsigned long) p->addr) || within_kprobe_blacklist((unsigned long) p->addr) || jump_label_text_reserved(p->addr, p->addr) || static_call_text_reserved(p->addr, p->addr) || @@ -1606,9 +1607,10 @@ int register_kprobe(struct kprobe *p) struct kprobe *old_p; struct module *probed_mod; kprobe_opcode_t *addr; + bool on_func_entry; /* Adjust probe address from symbol */ - addr = kprobe_addr(p); + addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); if (IS_ERR(addr)) return PTR_ERR(addr); p->addr = addr; @@ -1628,6 +1630,9 @@ int register_kprobe(struct kprobe *p) mutex_lock(&kprobe_mutex); + if (on_func_entry) + p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; + old_p = get_kprobe(p->addr); if (old_p) { /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ @@ -1707,11 +1712,12 @@ static struct kprobe *__disable_kprobe(struct kprobe *p) /* Try to disarm and disable this/parent probe */ if (p == orig_p || aggr_kprobe_disabled(orig_p)) { /* - * If 'kprobes_all_disarmed' is set, 'orig_p' - * should have already been disarmed, so - * skip unneed disarming process. + * Don't be lazy here. Even if 'kprobes_all_disarmed' + * is false, 'orig_p' might not have been armed yet. + * Note arm_all_kprobes() __tries__ to arm all kprobes + * on the best effort basis. */ - if (!kprobes_all_disarmed) { + if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) { ret = disarm_kprobe(orig_p, true); if (ret) { p->flags &= ~KPROBE_FLAG_DISABLED; diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index b1292a57c2a5326462e83180ba2aaad677270a79..65dba9076f312d1857441f173381add01f592d9e 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -105,7 +105,12 @@ KERNEL_ATTR_RO(kexec_crash_loaded); static ssize_t kexec_crash_size_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sprintf(buf, "%zu\n", crash_get_memory_size()); + ssize_t size = crash_get_memory_size(); + + if (size < 0) + return size; + + return sprintf(buf, "%zd\n", size); } static ssize_t kexec_crash_size_store(struct kobject *kobj, struct kobj_attribute *attr, diff --git a/kernel/kthread.c b/kernel/kthread.c index 3c677918d8f2fece96a252343139a0671d29e180..f97fd01a29325f6d7f2e6d04ad5f87cb9d3577f3 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -704,6 +704,7 @@ int kthread_stop(struct task_struct *k) kthread = to_kthread(k); set_bit(KTHREAD_SHOULD_STOP, &kthread->flags); kthread_unpark(k); + set_tsk_thread_flag(k, TIF_NOTIFY_SIGNAL); wake_up_process(k); wait_for_completion(&kthread->exited); ret = kthread->result; @@ -1050,8 +1051,7 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker, struct timer_list *timer = &dwork->timer; struct kthread_work *work = &dwork->work; - WARN_ON_FUNCTION_MISMATCH(timer->function, - kthread_delayed_work_timer_fn); + WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn); /* * If @delay is 0, queue @dwork->work immediately. This is for diff --git a/kernel/latencytop.c b/kernel/latencytop.c index 76166df011a4db0aa01c0c3c73c047b0bb6ca1da..781249098cb6e729b77b81658c217097085e6eb6 100644 --- a/kernel/latencytop.c +++ b/kernel/latencytop.c @@ -112,7 +112,7 @@ static void __sched account_global_scheduler_latency(struct task_struct *tsk, struct latency_record *lat) { - int firstnonnull = MAXLR + 1; + int firstnonnull = MAXLR; int i; /* skip kernel threads for now */ @@ -150,7 +150,7 @@ account_global_scheduler_latency(struct task_struct *tsk, } i = firstnonnull; - if (i >= MAXLR - 1) + if (i >= MAXLR) return; /* Allocted a new one: */ diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index bc475e62279d2a39687e5209d6eced904a6c4691..9ada0bc5247be5de41c3c21476b1dd2cd9f20a9a 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -213,7 +213,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, * we use the smallest/strictest upper bound possible (56, based on * the current definition of MODULE_NAME_LEN) to prevent overflows. */ - BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128); + BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 512); relas = (Elf_Rela *) relasec->sh_addr; /* For each rela in this klp relocation section */ @@ -227,7 +227,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab, /* Format: .klp.sym.sym_objname.sym_name,sympos */ cnt = sscanf(strtab + sym->st_name, - ".klp.sym.%55[^.].%127[^,],%lu", + ".klp.sym.%55[^.].%511[^,],%lu", sym_objname, sym_name, &sympos); if (cnt != 3) { pr_err("symbol %s has an incorrectly formatted name\n", @@ -325,6 +325,7 @@ int klp_apply_section_relocs(struct module *pmod, Elf_Shdr *sechdrs, * /sys/kernel/livepatch//transition * /sys/kernel/livepatch//force * /sys/kernel/livepatch// + * /sys/kernel/livepatch///patched * /sys/kernel/livepatch/// */ static int __klp_disable_patch(struct klp_patch *patch); @@ -431,6 +432,22 @@ static struct attribute *klp_patch_attrs[] = { }; ATTRIBUTE_GROUPS(klp_patch); +static ssize_t patched_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct klp_object *obj; + + obj = container_of(kobj, struct klp_object, kobj); + return sysfs_emit(buf, "%d\n", obj->patched); +} + +static struct kobj_attribute patched_kobj_attr = __ATTR_RO(patched); +static struct attribute *klp_object_attrs[] = { + &patched_kobj_attr.attr, + NULL, +}; +ATTRIBUTE_GROUPS(klp_object); + static void klp_free_object_dynamic(struct klp_object *obj) { kfree(obj->name); @@ -576,6 +593,7 @@ static void klp_kobj_release_object(struct kobject *kobj) static struct kobj_type klp_ktype_object = { .release = klp_kobj_release_object, .sysfs_ops = &kobj_sysfs_ops, + .default_groups = klp_object_groups, }; static void klp_kobj_release_func(struct kobject *kobj) @@ -1171,7 +1189,7 @@ int klp_module_coming(struct module *mod) return -EINVAL; if (!strcmp(mod->name, "vmlinux")) { - pr_err("vmlinux.ko: invalid module name"); + pr_err("vmlinux.ko: invalid module name\n"); return -EINVAL; } diff --git a/kernel/livepatch/transition.c b/kernel/livepatch/transition.c index 5d03a2ad106619533609bf08ff40bb2a1c504424..30187b1d8275942d6f45bef46ca00c0a825c219f 100644 --- a/kernel/livepatch/transition.c +++ b/kernel/livepatch/transition.c @@ -610,9 +610,23 @@ void klp_reverse_transition(void) /* Called from copy_process() during fork */ void klp_copy_process(struct task_struct *child) { - child->patch_state = current->patch_state; - /* TIF_PATCH_PENDING gets copied in setup_thread_stack() */ + /* + * The parent process may have gone through a KLP transition since + * the thread flag was copied in setup_thread_stack earlier. Bring + * the task flag up to date with the parent here. + * + * The operation is serialized against all klp_*_transition() + * operations by the tasklist_lock. The only exception is + * klp_update_patch_state(current), but we cannot race with + * that because we are current. + */ + if (test_tsk_thread_flag(current, TIF_PATCH_PENDING)) + set_tsk_thread_flag(child, TIF_PATCH_PENDING); + else + clear_tsk_thread_flag(child, TIF_PATCH_PENDING); + + child->patch_state = current->patch_state; } /* diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile index d51cabf28f382b5ad2c95ab4cc904a3f881ca4bc..ea925731fa40f8833ccefac31f36e46c11056ad8 100644 --- a/kernel/locking/Makefile +++ b/kernel/locking/Makefile @@ -5,8 +5,9 @@ KCOV_INSTRUMENT := n obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o -# Avoid recursion lockdep -> KCSAN -> ... -> lockdep. +# Avoid recursion lockdep -> sanitizer -> ... -> lockdep. KCSAN_SANITIZE_lockdep.o := n +KMSAN_SANITIZE_lockdep.o := n ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 64a13eb560781c84ff5d809c3a7b0276e2152a23..e3375bc40dadc7b56482cd52a04a53f69c764df3 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -934,8 +934,10 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) * Huh! same key, different name? Did someone trample * on some memory? We're most confused. */ - WARN_ON_ONCE(class->name != lock->name && - lock->key != &__lockdep_no_validate__); + WARN_ONCE(class->name != lock->name && + lock->key != &__lockdep_no_validate__, + "Looking for class \"%s\" with key %ps, but found a different class \"%s\" with the same key\n", + lock->name, lock->key, class->name); return class; } } diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 5fe4c5495ba3c87c8505fe0dde278d0112579985..185bd1c906b0113f3921385bdd00d7950100023c 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -192,6 +192,12 @@ EXPORT_SYMBOL_GPL(__percpu_down_read); __sum; \ }) +bool percpu_is_read_locked(struct percpu_rw_semaphore *sem) +{ + return per_cpu_sum(*sem->read_count) != 0 && !atomic_read(&sem->block); +} +EXPORT_SYMBOL_GPL(percpu_is_read_locked); + /* * Return true if the modular sum of the sem->read_count per-CPU variable is * zero. If this sum is zero, then it is stable due to the fact that if any diff --git a/kernel/locking/qrwlock.c b/kernel/locking/qrwlock.c index 2e1600906c9f5cd868415d20e2d7024c5b1e0531..d2ef312a8611ebaae018281b1addf3114845eed8 100644 --- a/kernel/locking/qrwlock.c +++ b/kernel/locking/qrwlock.c @@ -18,7 +18,7 @@ * queued_read_lock_slowpath - acquire read lock of a queued rwlock * @lock: Pointer to queued rwlock structure */ -void queued_read_lock_slowpath(struct qrwlock *lock) +void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock) { /* * Readers come here when they cannot get the lock without waiting @@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath); * queued_write_lock_slowpath - acquire write lock of a queued rwlock * @lock : Pointer to queued rwlock structure */ -void queued_write_lock_slowpath(struct qrwlock *lock) +void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock) { int cnts; diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index 65a9a10caa6f50de9a87d28174c6ce707035b3f2..2b23378775feec6060ba25160d1c57506fcb907d 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : * queue : ^--' : */ -void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) +void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) { struct mcs_spinlock *prev, *next, *node; u32 old, tail; diff --git a/kernel/locking/qspinlock_paravirt.h b/kernel/locking/qspinlock_paravirt.h index e84d21aa0722b48f18e8507ff68b405b2a79356b..6afc249ce697dba8d05374cf8f3637d08a62b6fa 100644 --- a/kernel/locking/qspinlock_paravirt.h +++ b/kernel/locking/qspinlock_paravirt.h @@ -489,7 +489,7 @@ gotlock: * PV versions of the unlock fastpath and slowpath functions to be used * instead of queued_spin_unlock(). */ -__visible void +__visible __lockfunc void __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) { struct pv_node *node; @@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) #include #ifndef __pv_queued_spin_unlock -__visible void __pv_queued_spin_unlock(struct qspinlock *lock) +__visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock) { u8 locked; diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c index 65f0262f635e124e9ee1ad046e2ca034d9c89724..44873594de0316473537f07d4f729852f27e5bde 100644 --- a/kernel/locking/rwsem.c +++ b/kernel/locking/rwsem.c @@ -133,14 +133,19 @@ * the owner value concurrently without lock. Read from owner, however, * may not need READ_ONCE() as long as the pointer value is only used * for comparison and isn't being dereferenced. + * + * Both rwsem_{set,clear}_owner() functions should be in the same + * preempt disable section as the atomic op that changes sem->count. */ static inline void rwsem_set_owner(struct rw_semaphore *sem) { + lockdep_assert_preemption_disabled(); atomic_long_set(&sem->owner, (long)current); } static inline void rwsem_clear_owner(struct rw_semaphore *sem) { + lockdep_assert_preemption_disabled(); atomic_long_set(&sem->owner, 0); } @@ -251,13 +256,16 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) static inline bool rwsem_write_trylock(struct rw_semaphore *sem) { long tmp = RWSEM_UNLOCKED_VALUE; + bool ret = false; + preempt_disable(); if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { rwsem_set_owner(sem); - return true; + ret = true; } - return false; + preempt_enable(); + return ret; } /* @@ -1352,8 +1360,10 @@ static inline void __up_write(struct rw_semaphore *sem) DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); + preempt_disable(); rwsem_clear_owner(sem); tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); + preempt_enable(); if (unlikely(tmp & RWSEM_FLAG_WAITERS)) rwsem_wake(sem); } diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c index f2654d2fe43aa17f56235d4676c4264fb9922fc4..34bfae72f29526a49aaff2c0619d5e34515c0e3c 100644 --- a/kernel/locking/semaphore.c +++ b/kernel/locking/semaphore.c @@ -51,7 +51,7 @@ static noinline void __up(struct semaphore *sem); * Use of this function is deprecated, please use down_interruptible() or * down_killable() instead. */ -void down(struct semaphore *sem) +void __sched down(struct semaphore *sem) { unsigned long flags; @@ -74,7 +74,7 @@ EXPORT_SYMBOL(down); * If the sleep is interrupted by a signal, this function will return -EINTR. * If the semaphore is successfully acquired, this function returns 0. */ -int down_interruptible(struct semaphore *sem) +int __sched down_interruptible(struct semaphore *sem) { unsigned long flags; int result = 0; @@ -101,7 +101,7 @@ EXPORT_SYMBOL(down_interruptible); * -EINTR. If the semaphore is successfully acquired, this function returns * 0. */ -int down_killable(struct semaphore *sem) +int __sched down_killable(struct semaphore *sem) { unsigned long flags; int result = 0; @@ -131,7 +131,7 @@ EXPORT_SYMBOL(down_killable); * Unlike mutex_trylock, this function can be used from interrupt context, * and the semaphore can be released by any task or interrupt. */ -int down_trylock(struct semaphore *sem) +int __sched down_trylock(struct semaphore *sem) { unsigned long flags; int count; @@ -156,7 +156,7 @@ EXPORT_SYMBOL(down_trylock); * If the semaphore is not released within the specified number of jiffies, * this function returns -ETIME. It returns 0 if the semaphore was acquired. */ -int down_timeout(struct semaphore *sem, long timeout) +int __sched down_timeout(struct semaphore *sem, long timeout) { unsigned long flags; int result = 0; @@ -180,7 +180,7 @@ EXPORT_SYMBOL(down_timeout); * Release the semaphore. Unlike mutexes, up() may be called from any * context and even by tasks which have never called down(). */ -void up(struct semaphore *sem) +void __sched up(struct semaphore *sem) { unsigned long flags; diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index 7f49baaa49793abe495bfc7f681c85d6da78c7c0..8475a0794f8c5ad26b90b9109b7c2880d7629da4 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -133,7 +133,7 @@ BUILD_LOCK_OPS(write, rwlock); #endif #ifndef CONFIG_INLINE_SPIN_TRYLOCK -int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) +noinline int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) { return __raw_spin_trylock(lock); } @@ -141,7 +141,7 @@ EXPORT_SYMBOL(_raw_spin_trylock); #endif #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH -int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) +noinline int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) { return __raw_spin_trylock_bh(lock); } @@ -149,7 +149,7 @@ EXPORT_SYMBOL(_raw_spin_trylock_bh); #endif #ifndef CONFIG_INLINE_SPIN_LOCK -void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) { __raw_spin_lock(lock); } @@ -157,7 +157,7 @@ EXPORT_SYMBOL(_raw_spin_lock); #endif #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE -unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) +noinline unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) { return __raw_spin_lock_irqsave(lock); } @@ -165,7 +165,7 @@ EXPORT_SYMBOL(_raw_spin_lock_irqsave); #endif #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ -void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) { __raw_spin_lock_irq(lock); } @@ -173,7 +173,7 @@ EXPORT_SYMBOL(_raw_spin_lock_irq); #endif #ifndef CONFIG_INLINE_SPIN_LOCK_BH -void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { __raw_spin_lock_bh(lock); } @@ -181,7 +181,7 @@ EXPORT_SYMBOL(_raw_spin_lock_bh); #endif #ifdef CONFIG_UNINLINE_SPIN_UNLOCK -void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) { __raw_spin_unlock(lock); } @@ -189,7 +189,7 @@ EXPORT_SYMBOL(_raw_spin_unlock); #endif #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE -void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) +noinline void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) { __raw_spin_unlock_irqrestore(lock, flags); } @@ -197,7 +197,7 @@ EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); #endif #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ -void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) { __raw_spin_unlock_irq(lock); } @@ -205,7 +205,7 @@ EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH -void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) +noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { __raw_spin_unlock_bh(lock); } @@ -215,7 +215,7 @@ EXPORT_SYMBOL(_raw_spin_unlock_bh); #ifndef CONFIG_PREEMPT_RT #ifndef CONFIG_INLINE_READ_TRYLOCK -int __lockfunc _raw_read_trylock(rwlock_t *lock) +noinline int __lockfunc _raw_read_trylock(rwlock_t *lock) { return __raw_read_trylock(lock); } @@ -223,7 +223,7 @@ EXPORT_SYMBOL(_raw_read_trylock); #endif #ifndef CONFIG_INLINE_READ_LOCK -void __lockfunc _raw_read_lock(rwlock_t *lock) +noinline void __lockfunc _raw_read_lock(rwlock_t *lock) { __raw_read_lock(lock); } @@ -231,7 +231,7 @@ EXPORT_SYMBOL(_raw_read_lock); #endif #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE -unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) +noinline unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) { return __raw_read_lock_irqsave(lock); } @@ -239,7 +239,7 @@ EXPORT_SYMBOL(_raw_read_lock_irqsave); #endif #ifndef CONFIG_INLINE_READ_LOCK_IRQ -void __lockfunc _raw_read_lock_irq(rwlock_t *lock) +noinline void __lockfunc _raw_read_lock_irq(rwlock_t *lock) { __raw_read_lock_irq(lock); } @@ -247,7 +247,7 @@ EXPORT_SYMBOL(_raw_read_lock_irq); #endif #ifndef CONFIG_INLINE_READ_LOCK_BH -void __lockfunc _raw_read_lock_bh(rwlock_t *lock) +noinline void __lockfunc _raw_read_lock_bh(rwlock_t *lock) { __raw_read_lock_bh(lock); } @@ -255,7 +255,7 @@ EXPORT_SYMBOL(_raw_read_lock_bh); #endif #ifndef CONFIG_INLINE_READ_UNLOCK -void __lockfunc _raw_read_unlock(rwlock_t *lock) +noinline void __lockfunc _raw_read_unlock(rwlock_t *lock) { __raw_read_unlock(lock); } @@ -263,7 +263,7 @@ EXPORT_SYMBOL(_raw_read_unlock); #endif #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE -void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) +noinline void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { __raw_read_unlock_irqrestore(lock, flags); } @@ -271,7 +271,7 @@ EXPORT_SYMBOL(_raw_read_unlock_irqrestore); #endif #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ -void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) +noinline void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) { __raw_read_unlock_irq(lock); } @@ -279,7 +279,7 @@ EXPORT_SYMBOL(_raw_read_unlock_irq); #endif #ifndef CONFIG_INLINE_READ_UNLOCK_BH -void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) +noinline void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) { __raw_read_unlock_bh(lock); } @@ -287,7 +287,7 @@ EXPORT_SYMBOL(_raw_read_unlock_bh); #endif #ifndef CONFIG_INLINE_WRITE_TRYLOCK -int __lockfunc _raw_write_trylock(rwlock_t *lock) +noinline int __lockfunc _raw_write_trylock(rwlock_t *lock) { return __raw_write_trylock(lock); } @@ -295,7 +295,7 @@ EXPORT_SYMBOL(_raw_write_trylock); #endif #ifndef CONFIG_INLINE_WRITE_LOCK -void __lockfunc _raw_write_lock(rwlock_t *lock) +noinline void __lockfunc _raw_write_lock(rwlock_t *lock) { __raw_write_lock(lock); } @@ -313,7 +313,7 @@ EXPORT_SYMBOL(_raw_write_lock_nested); #endif #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE -unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) +noinline unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) { return __raw_write_lock_irqsave(lock); } @@ -321,7 +321,7 @@ EXPORT_SYMBOL(_raw_write_lock_irqsave); #endif #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ -void __lockfunc _raw_write_lock_irq(rwlock_t *lock) +noinline void __lockfunc _raw_write_lock_irq(rwlock_t *lock) { __raw_write_lock_irq(lock); } @@ -329,7 +329,7 @@ EXPORT_SYMBOL(_raw_write_lock_irq); #endif #ifndef CONFIG_INLINE_WRITE_LOCK_BH -void __lockfunc _raw_write_lock_bh(rwlock_t *lock) +noinline void __lockfunc _raw_write_lock_bh(rwlock_t *lock) { __raw_write_lock_bh(lock); } @@ -337,7 +337,7 @@ EXPORT_SYMBOL(_raw_write_lock_bh); #endif #ifndef CONFIG_INLINE_WRITE_UNLOCK -void __lockfunc _raw_write_unlock(rwlock_t *lock) +noinline void __lockfunc _raw_write_unlock(rwlock_t *lock) { __raw_write_unlock(lock); } @@ -345,7 +345,7 @@ EXPORT_SYMBOL(_raw_write_unlock); #endif #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE -void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) +noinline void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { __raw_write_unlock_irqrestore(lock, flags); } @@ -353,7 +353,7 @@ EXPORT_SYMBOL(_raw_write_unlock_irqrestore); #endif #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ -void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) +noinline void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) { __raw_write_unlock_irq(lock); } @@ -361,7 +361,7 @@ EXPORT_SYMBOL(_raw_write_unlock_irq); #endif #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH -void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) +noinline void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) { __raw_write_unlock_bh(lock); } diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c index 353004155d659fa271d0921bb454a2eb27ad1b8a..43efb2a0416025f9dc9ff4f89bbec416fd79d10a 100644 --- a/kernel/locking/test-ww_mutex.c +++ b/kernel/locking/test-ww_mutex.c @@ -399,7 +399,7 @@ static int *get_random_order(int count) order[n] = n; for (n = count - 1; n > 1; n--) { - r = get_random_int() % (n + 1); + r = prandom_u32_max(n + 1); if (r != n) { tmp = order[n]; order[n] = order[r]; @@ -538,7 +538,7 @@ static void stress_one_work(struct work_struct *work) { struct stress *stress = container_of(work, typeof(*stress), work); const int nlocks = stress->nlocks; - struct ww_mutex *lock = stress->locks + (get_random_int() % nlocks); + struct ww_mutex *lock = stress->locks + prandom_u32_max(nlocks); int err; do { diff --git a/kernel/module/decompress.c b/kernel/module/decompress.c index 4d0bcb3d9e449b780899f4012db9d64e76d3b1b3..c033572d83f0e889d997f66a7269af69a936ec61 100644 --- a/kernel/module/decompress.c +++ b/kernel/module/decompress.c @@ -256,7 +256,7 @@ void module_decompress_cleanup(struct load_info *info) static ssize_t compression_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sysfs_emit(buf, "%s\n", __stringify(MODULE_COMPRESSION)); + return sysfs_emit(buf, __stringify(MODULE_COMPRESSION) "\n"); } static struct kobj_attribute module_compression_attr = __ATTR_RO(compression); diff --git a/kernel/module/internal.h b/kernel/module/internal.h index 680d980a4fb296cc7aecf917791f735a01b7bc19..2e2bf236f558256422cdfe38152649dbe53dce0d 100644 --- a/kernel/module/internal.h +++ b/kernel/module/internal.h @@ -53,6 +53,7 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[]; extern const s32 __start___kcrctab[]; extern const s32 __start___kcrctab_gpl[]; +#include struct load_info { const char *name; /* pointer to module in temporary copy, freed at end of load_module() */ @@ -62,8 +63,7 @@ struct load_info { Elf_Shdr *sechdrs; char *secstrings, *strtab; unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; - struct _ddebug *debug; - unsigned int num_debug; + struct _ddebug_info dyndbg; bool sig_ok; #ifdef CONFIG_KALLSYMS unsigned long mod_kallsyms_init_off; diff --git a/kernel/module/main.c b/kernel/module/main.c index 6a477c622544d4351781870b4350bc32167e10ae..d02d39c7174e141b06a5e766cfa8553a47c6833e 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include "internal.h" @@ -1144,8 +1145,6 @@ void __weak module_arch_freeing_init(struct module *mod) { } -static void cfi_cleanup(struct module *mod); - /* Free a module, remove from lists, etc. */ static void free_module(struct module *mod) { @@ -1190,9 +1189,6 @@ static void free_module(struct module *mod) mod->name); mutex_unlock(&module_mutex); - /* Clean up CFI for the module. */ - cfi_cleanup(mod); - /* This may be empty, but that's OK */ module_arch_freeing_init(mod); module_memfree(mod->init_layout.base); @@ -1598,16 +1594,16 @@ static void free_modinfo(struct module *mod) } } -static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num) +static void dynamic_debug_setup(struct module *mod, struct _ddebug_info *dyndbg) { - if (!debug) + if (!dyndbg->num_descs) return; - ddebug_add_module(debug, num, mod->name); + ddebug_add_module(dyndbg, mod->name); } -static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug) +static void dynamic_debug_remove(struct module *mod, struct _ddebug_info *dyndbg) { - if (debug) + if (dyndbg->num_descs) ddebug_remove_module(mod->name); } @@ -2099,7 +2095,7 @@ static int find_module_sections(struct module *mod, struct load_info *info) sizeof(*mod->static_call_sites), &mod->num_static_call_sites); #endif -#ifdef CONFIG_KUNIT +#if IS_ENABLED(CONFIG_KUNIT) mod->kunit_suites = section_objs(info, ".kunit_test_suites", sizeof(*mod->kunit_suites), &mod->num_kunit_suites); @@ -2111,8 +2107,10 @@ static int find_module_sections(struct module *mod, struct load_info *info) if (section_addr(info, "__obsparm")) pr_warn("%s: Ignoring obsolete parameters\n", mod->name); - info->debug = section_objs(info, "__dyndbg", - sizeof(*info->debug), &info->num_debug); + info->dyndbg.descs = section_objs(info, "__dyndbg", + sizeof(*info->dyndbg.descs), &info->dyndbg.num_descs); + info->dyndbg.classes = section_objs(info, "__dyndbg_classes", + sizeof(*info->dyndbg.classes), &info->dyndbg.num_classes); return 0; } @@ -2602,8 +2600,9 @@ static int complete_formation(struct module *mod, struct load_info *info) if (err < 0) goto out; - /* This relies on module_mutex for list integrity. */ + /* These rely on module_mutex for list integrity. */ module_bug_finalize(info->hdr, info->sechdrs, mod); + module_cfi_finalize(info->hdr, info->sechdrs, mod); if (module_check_misalignment(mod)) goto out_misaligned; @@ -2665,8 +2664,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname, return 0; } -static void cfi_init(struct module *mod); - /* * Allocate and load the module: note that size of section 0 is always * zero, and we rely on this for optional sections. @@ -2796,9 +2793,6 @@ static int load_module(struct load_info *info, const char __user *uargs, flush_module_icache(mod); - /* Setup CFI for the module. */ - cfi_init(mod); - /* Now copy in args */ mod->args = strndup_user(uargs, ~0UL >> 1); if (IS_ERR(mod->args)) { @@ -2807,7 +2801,7 @@ static int load_module(struct load_info *info, const char __user *uargs, } init_build_id(mod, info); - dynamic_debug_setup(mod, info->debug, info->num_debug); + dynamic_debug_setup(mod, &info->dyndbg); /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */ ftrace_module_init(mod); @@ -2871,11 +2865,10 @@ static int load_module(struct load_info *info, const char __user *uargs, ddebug_cleanup: ftrace_release_mod(mod); - dynamic_debug_remove(mod, info->debug); + dynamic_debug_remove(mod, &info->dyndbg); synchronize_rcu(); kfree(mod->args); free_arch_cleanup: - cfi_cleanup(mod); module_arch_cleanup(mod); free_modinfo: free_modinfo(mod); @@ -2961,41 +2954,6 @@ static inline int within(unsigned long addr, void *start, unsigned long size) return ((void *)addr >= start && (void *)addr < start + size); } -static void cfi_init(struct module *mod) -{ -#ifdef CONFIG_CFI_CLANG - initcall_t *init; -#ifdef CONFIG_MODULE_UNLOAD - exitcall_t *exit; -#endif - - rcu_read_lock_sched(); - mod->cfi_check = (cfi_check_fn) - find_kallsyms_symbol_value(mod, "__cfi_check"); - init = (initcall_t *) - find_kallsyms_symbol_value(mod, "__cfi_jt_init_module"); - /* Fix init/exit functions to point to the CFI jump table */ - if (init) - mod->init = *init; -#ifdef CONFIG_MODULE_UNLOAD - exit = (exitcall_t *) - find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module"); - if (exit) - mod->exit = *exit; -#endif - rcu_read_unlock_sched(); - - cfi_module_add(mod, mod_tree.addr_min); -#endif -} - -static void cfi_cleanup(struct module *mod) -{ -#ifdef CONFIG_CFI_CLANG - cfi_module_remove(mod, mod_tree.addr_min); -#endif -} - /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */ char *module_flags(struct module *mod, char *buf, bool show_state) { diff --git a/kernel/module/tracking.c b/kernel/module/tracking.c index 7f8133044d092bf4292732dcf3ef16f8761249aa..26d812e0761526021087937754e38c4e9011b0d7 100644 --- a/kernel/module/tracking.c +++ b/kernel/module/tracking.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "internal.h" @@ -21,6 +22,9 @@ int try_add_tainted_module(struct module *mod) module_assert_mutex_or_preempt(); + if (!mod->taints) + goto out; + list_for_each_entry_rcu(mod_taint, &unloaded_tainted_modules, list, lockdep_is_held(&module_mutex)) { if (!strcmp(mod_taint->name, mod->name) && @@ -59,3 +63,70 @@ void print_unloaded_tainted_modules(void) } } } + +#ifdef CONFIG_DEBUG_FS +static void *unloaded_tainted_modules_seq_start(struct seq_file *m, loff_t *pos) + __acquires(rcu) +{ + rcu_read_lock(); + return seq_list_start_rcu(&unloaded_tainted_modules, *pos); +} + +static void *unloaded_tainted_modules_seq_next(struct seq_file *m, void *p, loff_t *pos) +{ + return seq_list_next_rcu(p, &unloaded_tainted_modules, pos); +} + +static void unloaded_tainted_modules_seq_stop(struct seq_file *m, void *p) + __releases(rcu) +{ + rcu_read_unlock(); +} + +static int unloaded_tainted_modules_seq_show(struct seq_file *m, void *p) +{ + struct mod_unload_taint *mod_taint; + char buf[MODULE_FLAGS_BUF_SIZE]; + size_t l; + + mod_taint = list_entry(p, struct mod_unload_taint, list); + l = module_flags_taint(mod_taint->taints, buf); + buf[l++] = '\0'; + + seq_printf(m, "%s (%s) %llu", mod_taint->name, buf, mod_taint->count); + seq_puts(m, "\n"); + + return 0; +} + +static const struct seq_operations unloaded_tainted_modules_seq_ops = { + .start = unloaded_tainted_modules_seq_start, + .next = unloaded_tainted_modules_seq_next, + .stop = unloaded_tainted_modules_seq_stop, + .show = unloaded_tainted_modules_seq_show, +}; + +static int unloaded_tainted_modules_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &unloaded_tainted_modules_seq_ops); +} + +static const struct file_operations unloaded_tainted_modules_fops = { + .open = unloaded_tainted_modules_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __init unloaded_tainted_modules_init(void) +{ + struct dentry *dir; + + dir = debugfs_create_dir("modules", NULL); + debugfs_create_file("unloaded_tainted", 0444, dir, NULL, + &unloaded_tainted_modules_fops); + + return 0; +} +module_init(unloaded_tainted_modules_init); +#endif /* CONFIG_DEBUG_FS */ diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index b4cbb406bc2840af0029107e3bbbff030081937f..eec72ca962e249c94266192b77a3c1f92ec8e889 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -179,8 +179,7 @@ int copy_namespaces(unsigned long flags, struct task_struct *tsk) if (IS_ERR(new_ns)) return PTR_ERR(new_ns); - if ((flags & CLONE_VM) == 0) - timens_on_fork(new_ns, tsk); + timens_on_fork(new_ns, tsk); tsk->nsproxy = new_ns; return 0; diff --git a/kernel/panic.c b/kernel/panic.c index c6eb8f8db0c059a3b932d5f304aba39f2e260173..da323209f5833183609f08a468b894fca7f6d099 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -329,9 +329,6 @@ void panic(const char *fmt, ...) if (_crash_kexec_post_notifiers) __crash_kexec(NULL); -#ifdef CONFIG_VT - unblank_screen(); -#endif console_unblank(); /* diff --git a/kernel/pid.c b/kernel/pid.c index 2fc0a16ec77b1d9b6efe3b6e098f7f23921abe17..3fbc5e46b72173cf216850ea85afc24ace98fd3c 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -519,6 +519,7 @@ struct pid *find_ge_pid(int nr, struct pid_namespace *ns) { return idr_get_next(&ns->idr, &nr); } +EXPORT_SYMBOL_GPL(find_ge_pid); struct pid *pidfd_get_pid(unsigned int fd, unsigned int *flags) { diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 89c71fce225dd7a70c7fe7b30f81d7e0f644bf3a..f58a0aa92310c723d1bbc97527f9111863648f33 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -92,20 +92,24 @@ bool hibernation_available(void) */ void hibernation_set_ops(const struct platform_hibernation_ops *ops) { + unsigned int sleep_flags; + if (ops && !(ops->begin && ops->end && ops->pre_snapshot && ops->prepare && ops->finish && ops->enter && ops->pre_restore && ops->restore_cleanup && ops->leave)) { WARN_ON(1); return; } - lock_system_sleep(); + + sleep_flags = lock_system_sleep(); + hibernation_ops = ops; if (ops) hibernation_mode = HIBERNATION_PLATFORM; else if (hibernation_mode == HIBERNATION_PLATFORM) hibernation_mode = HIBERNATION_SHUTDOWN; - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); } EXPORT_SYMBOL_GPL(hibernation_set_ops); @@ -713,6 +717,7 @@ static int load_image_and_restore(void) int hibernate(void) { bool snapshot_test = false; + unsigned int sleep_flags; int error; if (!hibernation_available()) { @@ -720,7 +725,7 @@ int hibernate(void) return -EPERM; } - lock_system_sleep(); + sleep_flags = lock_system_sleep(); /* The snapshot device should not be opened while we're running */ if (!hibernate_acquire()) { error = -EBUSY; @@ -794,7 +799,7 @@ int hibernate(void) pm_restore_console(); hibernate_release(); Unlock: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); pr_info("hibernation exit\n"); return error; @@ -809,9 +814,10 @@ int hibernate(void) */ int hibernate_quiet_exec(int (*func)(void *data), void *data) { + unsigned int sleep_flags; int error; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); if (!hibernate_acquire()) { error = -EBUSY; @@ -891,7 +897,7 @@ restore: hibernate_release(); unlock: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return error; } @@ -1100,11 +1106,12 @@ static ssize_t disk_show(struct kobject *kobj, struct kobj_attribute *attr, static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t n) { + int mode = HIBERNATION_INVALID; + unsigned int sleep_flags; int error = 0; - int i; int len; char *p; - int mode = HIBERNATION_INVALID; + int i; if (!hibernation_available()) return -EPERM; @@ -1112,7 +1119,7 @@ static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, p = memchr(buf, '\n', n); len = p ? p - buf : n; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); for (i = HIBERNATION_FIRST; i <= HIBERNATION_MAX; i++) { if (len == strlen(hibernation_modes[i]) && !strncmp(buf, hibernation_modes[i], len)) { @@ -1142,7 +1149,7 @@ static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, if (!error) pm_pr_dbg("Hibernation mode set to '%s'\n", hibernation_modes[mode]); - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return error ? error : n; } @@ -1158,9 +1165,10 @@ static ssize_t resume_show(struct kobject *kobj, struct kobj_attribute *attr, static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t n) { - dev_t res; + unsigned int sleep_flags; int len = n; char *name; + dev_t res; if (len && buf[len-1] == '\n') len--; @@ -1173,9 +1181,10 @@ static ssize_t resume_store(struct kobject *kobj, struct kobj_attribute *attr, if (!res) return -EINVAL; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); swsusp_resume_device = res; - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); + pm_pr_dbg("Configured hibernation resume from disk to %u\n", swsusp_resume_device); noresume = 0; diff --git a/kernel/power/main.c b/kernel/power/main.c index e3694034b753664d91e981b6455e005d36feed16..31ec4a9b9d7045315ed60c26c8b385be74eed652 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -21,14 +21,16 @@ #ifdef CONFIG_PM_SLEEP -void lock_system_sleep(void) +unsigned int lock_system_sleep(void) { - current->flags |= PF_FREEZER_SKIP; + unsigned int flags = current->flags; + current->flags |= PF_NOFREEZE; mutex_lock(&system_transition_mutex); + return flags; } EXPORT_SYMBOL_GPL(lock_system_sleep); -void unlock_system_sleep(void) +void unlock_system_sleep(unsigned int flags) { /* * Don't use freezer_count() because we don't want the call to @@ -46,7 +48,8 @@ void unlock_system_sleep(void) * Which means, if we use try_to_freeze() here, it would make them * enter the refrigerator, thus causing hibernation to lockup. */ - current->flags &= ~PF_FREEZER_SKIP; + if (!(flags & PF_NOFREEZE)) + current->flags &= ~PF_NOFREEZE; mutex_unlock(&system_transition_mutex); } EXPORT_SYMBOL_GPL(unlock_system_sleep); @@ -263,16 +266,17 @@ static ssize_t pm_test_show(struct kobject *kobj, struct kobj_attribute *attr, static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t n) { + unsigned int sleep_flags; const char * const *s; + int error = -EINVAL; int level; char *p; int len; - int error = -EINVAL; p = memchr(buf, '\n', n); len = p ? p - buf : n; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); level = TEST_FIRST; for (s = &pm_tests[level]; level <= TEST_MAX; s++, level++) @@ -282,7 +286,7 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, break; } - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return error ? error : n; } diff --git a/kernel/power/process.c b/kernel/power/process.c index 3068601e585a8fd5066d5b0bfd4c1c1fb7ddaee0..ddd9988327fe2b107046747a32fbe25caf31697a 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c @@ -50,8 +50,7 @@ static int try_to_freeze_tasks(bool user_only) if (p == current || !freeze_task(p)) continue; - if (!freezer_should_skip(p)) - todo++; + todo++; } read_unlock(&tasklist_lock); @@ -96,8 +95,7 @@ static int try_to_freeze_tasks(bool user_only) if (!wakeup || pm_debug_messages_on) { read_lock(&tasklist_lock); for_each_process_thread(g, p) { - if (p != current && !freezer_should_skip(p) - && freezing(p) && !frozen(p)) + if (p != current && freezing(p) && !frozen(p)) sched_show_task(p); } read_unlock(&tasklist_lock); @@ -129,7 +127,7 @@ int freeze_processes(void) current->flags |= PF_SUSPEND_TASK; if (!pm_freezing) - atomic_inc(&system_freezing_cnt); + static_branch_inc(&freezer_active); pm_wakeup_clear(0); pr_info("Freezing user space processes ... "); @@ -190,7 +188,7 @@ void thaw_processes(void) trace_suspend_resume(TPS("thaw_processes"), 0, true); if (pm_freezing) - atomic_dec(&system_freezing_cnt); + static_branch_dec(&freezer_active); pm_freezing = false; pm_nosig_freezing = false; diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 827075944d28357950b3bf55ec22d53987bbd920..fa3bf161d13f79a6d28dc891d3b5708871e4a969 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -75,9 +75,11 @@ EXPORT_SYMBOL_GPL(pm_suspend_default_s2idle); void s2idle_set_ops(const struct platform_s2idle_ops *ops) { - lock_system_sleep(); + unsigned int sleep_flags; + + sleep_flags = lock_system_sleep(); s2idle_ops = ops; - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); } static void s2idle_begin(void) @@ -136,6 +138,9 @@ static void s2idle_loop(void) break; } + if (s2idle_ops && s2idle_ops->check) + s2idle_ops->check(); + s2idle_enter(); } @@ -200,7 +205,9 @@ __setup("mem_sleep_default=", mem_sleep_default_setup); */ void suspend_set_ops(const struct platform_suspend_ops *ops) { - lock_system_sleep(); + unsigned int sleep_flags; + + sleep_flags = lock_system_sleep(); suspend_ops = ops; @@ -216,7 +223,7 @@ void suspend_set_ops(const struct platform_suspend_ops *ops) mem_sleep_current = PM_SUSPEND_MEM; } - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); } EXPORT_SYMBOL_GPL(suspend_set_ops); diff --git a/kernel/power/user.c b/kernel/power/user.c index d43c2aa583b26b66289185b5bc83a185ce326af8..3a4e70366f354c0167c6630c3063af8ffa26bfbf 100644 --- a/kernel/power/user.c +++ b/kernel/power/user.c @@ -47,12 +47,13 @@ int is_hibernate_resume_dev(dev_t dev) static int snapshot_open(struct inode *inode, struct file *filp) { struct snapshot_data *data; + unsigned int sleep_flags; int error; if (!hibernation_available()) return -EPERM; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); if (!hibernate_acquire()) { error = -EBUSY; @@ -98,7 +99,7 @@ static int snapshot_open(struct inode *inode, struct file *filp) data->dev = 0; Unlock: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return error; } @@ -106,8 +107,9 @@ static int snapshot_open(struct inode *inode, struct file *filp) static int snapshot_release(struct inode *inode, struct file *filp) { struct snapshot_data *data; + unsigned int sleep_flags; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); swsusp_free(); data = filp->private_data; @@ -124,7 +126,7 @@ static int snapshot_release(struct inode *inode, struct file *filp) PM_POST_HIBERNATION : PM_POST_RESTORE); hibernate_release(); - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return 0; } @@ -132,11 +134,12 @@ static int snapshot_release(struct inode *inode, struct file *filp) static ssize_t snapshot_read(struct file *filp, char __user *buf, size_t count, loff_t *offp) { + loff_t pg_offp = *offp & ~PAGE_MASK; struct snapshot_data *data; + unsigned int sleep_flags; ssize_t res; - loff_t pg_offp = *offp & ~PAGE_MASK; - lock_system_sleep(); + sleep_flags = lock_system_sleep(); data = filp->private_data; if (!data->ready) { @@ -157,7 +160,7 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf, *offp += res; Unlock: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return res; } @@ -165,16 +168,17 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf, static ssize_t snapshot_write(struct file *filp, const char __user *buf, size_t count, loff_t *offp) { + loff_t pg_offp = *offp & ~PAGE_MASK; struct snapshot_data *data; + unsigned long sleep_flags; ssize_t res; - loff_t pg_offp = *offp & ~PAGE_MASK; if (need_wait) { wait_for_device_probe(); need_wait = false; } - lock_system_sleep(); + sleep_flags = lock_system_sleep(); data = filp->private_data; @@ -196,7 +200,7 @@ static ssize_t snapshot_write(struct file *filp, const char __user *buf, if (res > 0) *offp += res; unlock: - unlock_system_sleep(); + unlock_system_sleep(sleep_flags); return res; } diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index a1a81fd9889bb35d54102ce3cbc84a76cca24020..e4f1e7478b521a7028ac8cd658cc54ae5e917bc0 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -220,9 +220,6 @@ int devkmsg_sysctl_set_loglvl(struct ctl_table *table, int write, } #endif /* CONFIG_PRINTK && CONFIG_SYSCTL */ -/* Number of registered extended console drivers. */ -static int nr_ext_console_drivers; - /* * Helper macros to handle lockdep when locking/unlocking console_sem. We use * macros instead of functions so that _RET_IP_ contains useful information. @@ -433,7 +430,7 @@ static struct printk_ringbuffer *prb = &printk_rb_static; * per_cpu_areas are initialised. This variable is set to true when * it's safe to access per-CPU data. */ -static bool __printk_percpu_data_ready __read_mostly; +static bool __printk_percpu_data_ready __ro_after_init; bool printk_percpu_data_ready(void) { @@ -2296,6 +2293,7 @@ asmlinkage __visible int _printk(const char *fmt, ...) } EXPORT_SYMBOL(_printk); +static bool pr_flush(int timeout_ms, bool reset_on_progress); static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress); #else /* CONFIG_PRINTK */ @@ -2330,6 +2328,7 @@ static void call_console_driver(struct console *con, const char *text, size_t le { } static bool suppress_message_printing(int level) { return false; } +static bool pr_flush(int timeout_ms, bool reset_on_progress) { return true; } static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; } #endif /* CONFIG_PRINTK */ @@ -3186,9 +3185,6 @@ void register_console(struct console *newcon) console_drivers->next = newcon; } - if (newcon->flags & CON_EXTENDED) - nr_ext_console_drivers++; - newcon->dropped = 0; if (newcon->flags & CON_PRINTBUFFER) { /* Get a consistent copy of @syslog_seq. */ @@ -3213,9 +3209,6 @@ void register_console(struct console *newcon) if (bootcon_enabled && ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) && !keep_bootcon) { - /* We need to iterate through all boot consoles, to make - * sure we print everything out, before we unregister them. - */ for_each_console(con) if (con->flags & CON_BOOT) unregister_console(con); @@ -3254,9 +3247,6 @@ int unregister_console(struct console *console) if (res) goto out_disable_unlock; - if (console->flags & CON_EXTENDED) - nr_ext_console_drivers--; - /* * If this isn't the last console and it has CON_CONSDEV set, we * need to set it on the next preferred console. @@ -3438,11 +3428,10 @@ static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progre * Context: Process context. May sleep while acquiring console lock. * Return: true if all enabled printers are caught up. */ -bool pr_flush(int timeout_ms, bool reset_on_progress) +static bool pr_flush(int timeout_ms, bool reset_on_progress) { return __pr_flush(NULL, timeout_ms, reset_on_progress); } -EXPORT_SYMBOL(pr_flush); /* * Delayed printk version, for scheduler-internal messages: diff --git a/kernel/profile.c b/kernel/profile.c index 7ea01ba30e757ef11235ba2702cab5c50751ec01..8a77769bc4b4cbb655244fd163fc72a860b4bcdd 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -59,43 +59,39 @@ int profile_setup(char *str) static const char schedstr[] = "schedule"; static const char sleepstr[] = "sleep"; static const char kvmstr[] = "kvm"; + const char *select = NULL; int par; if (!strncmp(str, sleepstr, strlen(sleepstr))) { #ifdef CONFIG_SCHEDSTATS force_schedstat_enabled(); prof_on = SLEEP_PROFILING; - if (str[strlen(sleepstr)] == ',') - str += strlen(sleepstr) + 1; - if (get_option(&str, &par)) - prof_shift = clamp(par, 0, BITS_PER_LONG - 1); - pr_info("kernel sleep profiling enabled (shift: %u)\n", - prof_shift); + select = sleepstr; #else pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n"); #endif /* CONFIG_SCHEDSTATS */ } else if (!strncmp(str, schedstr, strlen(schedstr))) { prof_on = SCHED_PROFILING; - if (str[strlen(schedstr)] == ',') - str += strlen(schedstr) + 1; - if (get_option(&str, &par)) - prof_shift = clamp(par, 0, BITS_PER_LONG - 1); - pr_info("kernel schedule profiling enabled (shift: %u)\n", - prof_shift); + select = schedstr; } else if (!strncmp(str, kvmstr, strlen(kvmstr))) { prof_on = KVM_PROFILING; - if (str[strlen(kvmstr)] == ',') - str += strlen(kvmstr) + 1; - if (get_option(&str, &par)) - prof_shift = clamp(par, 0, BITS_PER_LONG - 1); - pr_info("kernel KVM profiling enabled (shift: %u)\n", - prof_shift); + select = kvmstr; } else if (get_option(&str, &par)) { prof_shift = clamp(par, 0, BITS_PER_LONG - 1); prof_on = CPU_PROFILING; pr_info("kernel profiling enabled (shift: %u)\n", prof_shift); } + + if (select) { + if (str[strlen(select)] == ',') + str += strlen(select) + 1; + if (get_option(&str, &par)) + prof_shift = clamp(par, 0, BITS_PER_LONG - 1); + pr_info("kernel %s profiling enabled (shift: %u)\n", + select, prof_shift); + } + return 1; } __setup("profile=", profile_setup); diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 1893d909e45ca6c575efb105ce12f35da7eadbb7..54482193e1edc2c665a8f32f08c4d81300757dbc 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -269,7 +269,7 @@ static int ptrace_check_attach(struct task_struct *child, bool ignore_state) read_unlock(&tasklist_lock); if (!ret && !ignore_state && - WARN_ON_ONCE(!wait_task_inactive(child, __TASK_TRACED))) + WARN_ON_ONCE(!wait_task_inactive(child, __TASK_TRACED|TASK_FROZEN))) ret = -ESRCH; return ret; diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index d8e1b270a065f9d5133f2b3a053da5c7f5e0e3fe..503c2aa845a4a607d483045ea3dad0448ff38f9f 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -84,10 +84,15 @@ torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress test torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); +torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); +torture_param(bool, gp_cond_exp_full, false, + "Use conditional/async full-stateexpedited GP wait primitives"); torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); +torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); +torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); @@ -194,16 +199,24 @@ static int rcu_torture_writer_state; #define RTWS_DEF_FREE 3 #define RTWS_EXP_SYNC 4 #define RTWS_COND_GET 5 -#define RTWS_COND_GET_EXP 6 -#define RTWS_COND_SYNC 7 -#define RTWS_COND_SYNC_EXP 8 -#define RTWS_POLL_GET 9 -#define RTWS_POLL_GET_EXP 10 -#define RTWS_POLL_WAIT 11 -#define RTWS_POLL_WAIT_EXP 12 -#define RTWS_SYNC 13 -#define RTWS_STUTTER 14 -#define RTWS_STOPPING 15 +#define RTWS_COND_GET_FULL 6 +#define RTWS_COND_GET_EXP 7 +#define RTWS_COND_GET_EXP_FULL 8 +#define RTWS_COND_SYNC 9 +#define RTWS_COND_SYNC_FULL 10 +#define RTWS_COND_SYNC_EXP 11 +#define RTWS_COND_SYNC_EXP_FULL 12 +#define RTWS_POLL_GET 13 +#define RTWS_POLL_GET_FULL 14 +#define RTWS_POLL_GET_EXP 15 +#define RTWS_POLL_GET_EXP_FULL 16 +#define RTWS_POLL_WAIT 17 +#define RTWS_POLL_WAIT_FULL 18 +#define RTWS_POLL_WAIT_EXP 19 +#define RTWS_POLL_WAIT_EXP_FULL 20 +#define RTWS_SYNC 21 +#define RTWS_STUTTER 22 +#define RTWS_STOPPING 23 static const char * const rcu_torture_writer_state_names[] = { "RTWS_FIXED_DELAY", "RTWS_DELAY", @@ -211,13 +224,21 @@ static const char * const rcu_torture_writer_state_names[] = { "RTWS_DEF_FREE", "RTWS_EXP_SYNC", "RTWS_COND_GET", + "RTWS_COND_GET_FULL", "RTWS_COND_GET_EXP", + "RTWS_COND_GET_EXP_FULL", "RTWS_COND_SYNC", + "RTWS_COND_SYNC_FULL", "RTWS_COND_SYNC_EXP", + "RTWS_COND_SYNC_EXP_FULL", "RTWS_POLL_GET", + "RTWS_POLL_GET_FULL", "RTWS_POLL_GET_EXP", + "RTWS_POLL_GET_EXP_FULL", "RTWS_POLL_WAIT", + "RTWS_POLL_WAIT_FULL", "RTWS_POLL_WAIT_EXP", + "RTWS_POLL_WAIT_EXP_FULL", "RTWS_SYNC", "RTWS_STUTTER", "RTWS_STOPPING", @@ -332,13 +353,21 @@ struct rcu_torture_ops { void (*exp_sync)(void); unsigned long (*get_gp_state_exp)(void); unsigned long (*start_gp_poll_exp)(void); + void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); bool (*poll_gp_state_exp)(unsigned long oldstate); void (*cond_sync_exp)(unsigned long oldstate); + void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); unsigned long (*get_gp_state)(void); + void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); unsigned long (*get_gp_completed)(void); + void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp); unsigned long (*start_gp_poll)(void); + void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); bool (*poll_gp_state)(unsigned long oldstate); + bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); + bool (*poll_need_2gp)(bool poll, bool poll_full); void (*cond_sync)(unsigned long oldstate); + void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); call_rcu_func_t call; void (*cb_barrier)(void); void (*fqs)(void); @@ -489,6 +518,11 @@ static void rcu_sync_torture_init(void) INIT_LIST_HEAD(&rcu_torture_removed); } +static bool rcu_poll_need_2gp(bool poll, bool poll_full) +{ + return poll; +} + static struct rcu_torture_ops rcu_ops = { .ttype = RCU_FLAVOR, .init = rcu_sync_torture_init, @@ -502,12 +536,19 @@ static struct rcu_torture_ops rcu_ops = { .sync = synchronize_rcu, .exp_sync = synchronize_rcu_expedited, .get_gp_state = get_state_synchronize_rcu, + .get_gp_state_full = get_state_synchronize_rcu_full, .get_gp_completed = get_completed_synchronize_rcu, + .get_gp_completed_full = get_completed_synchronize_rcu_full, .start_gp_poll = start_poll_synchronize_rcu, + .start_gp_poll_full = start_poll_synchronize_rcu_full, .poll_gp_state = poll_state_synchronize_rcu, + .poll_gp_state_full = poll_state_synchronize_rcu_full, + .poll_need_2gp = rcu_poll_need_2gp, .cond_sync = cond_synchronize_rcu, + .cond_sync_full = cond_synchronize_rcu_full, .get_gp_state_exp = get_state_synchronize_rcu, .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, + .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, .poll_gp_state_exp = poll_state_synchronize_rcu, .cond_sync_exp = cond_synchronize_rcu_expedited, .call = call_rcu, @@ -709,6 +750,9 @@ static struct rcu_torture_ops srcud_ops = { .deferred_free = srcu_torture_deferred_free, .sync = srcu_torture_synchronize, .exp_sync = srcu_torture_synchronize_expedited, + .get_gp_state = srcu_torture_get_gp_state, + .start_gp_poll = srcu_torture_start_gp_poll, + .poll_gp_state = srcu_torture_poll_gp_state, .call = srcu_torture_call, .cb_barrier = srcu_torture_barrier, .stats = srcu_torture_stats, @@ -1148,15 +1192,35 @@ static int nsynctypes; */ static void rcu_torture_write_types(void) { - bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp; - bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll; - bool gp_sync1 = gp_sync; + bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; + bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; + bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; + bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; /* Initialize synctype[] array. If none set, take default. */ - if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp && - !gp_normal1 && !gp_poll1 && !gp_sync1) - gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 = - gp_normal1 = gp_poll1 = gp_sync1 = true; + if (!gp_cond1 && + !gp_cond_exp1 && + !gp_cond_full1 && + !gp_cond_exp_full1 && + !gp_exp1 && + !gp_poll_exp1 && + !gp_poll_exp_full1 && + !gp_normal1 && + !gp_poll1 && + !gp_poll_full1 && + !gp_sync1) { + gp_cond1 = true; + gp_cond_exp1 = true; + gp_cond_full1 = true; + gp_cond_exp_full1 = true; + gp_exp1 = true; + gp_poll_exp1 = true; + gp_poll_exp_full1 = true; + gp_normal1 = true; + gp_poll1 = true; + gp_poll_full1 = true; + gp_sync1 = true; + } if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { synctype[nsynctypes++] = RTWS_COND_GET; pr_info("%s: Testing conditional GPs.\n", __func__); @@ -1169,6 +1233,19 @@ static void rcu_torture_write_types(void) } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { pr_alert("%s: gp_cond_exp without primitives.\n", __func__); } + if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { + synctype[nsynctypes++] = RTWS_COND_GET_FULL; + pr_info("%s: Testing conditional full-state GPs.\n", __func__); + } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { + pr_alert("%s: gp_cond_full without primitives.\n", __func__); + } + if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { + synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; + pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); + } else if (gp_cond_exp_full && + (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { + pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); + } if (gp_exp1 && cur_ops->exp_sync) { synctype[nsynctypes++] = RTWS_EXP_SYNC; pr_info("%s: Testing expedited GPs.\n", __func__); @@ -1187,12 +1264,25 @@ static void rcu_torture_write_types(void) } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { pr_alert("%s: gp_poll without primitives.\n", __func__); } + if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { + synctype[nsynctypes++] = RTWS_POLL_GET_FULL; + pr_info("%s: Testing polling full-state GPs.\n", __func__); + } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { + pr_alert("%s: gp_poll_full without primitives.\n", __func__); + } if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { synctype[nsynctypes++] = RTWS_POLL_GET_EXP; pr_info("%s: Testing polling expedited GPs.\n", __func__); } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { pr_alert("%s: gp_poll_exp without primitives.\n", __func__); } + if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { + synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; + pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); + } else if (gp_poll_exp_full && + (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { + pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); + } if (gp_sync1 && cur_ops->sync) { synctype[nsynctypes++] = RTWS_SYNC; pr_info("%s: Testing normal GPs.\n", __func__); @@ -1201,6 +1291,40 @@ static void rcu_torture_write_types(void) } } +/* + * Do the specified rcu_torture_writer() synchronous grace period, + * while also testing out the polled APIs. Note well that the single-CPU + * grace-period optimizations must be accounted for. + */ +static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) +{ + unsigned long cookie; + struct rcu_gp_oldstate cookie_full; + bool dopoll; + bool dopoll_full; + unsigned long r = torture_random(trsp); + + dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); + dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); + if (dopoll || dopoll_full) + cpus_read_lock(); + if (dopoll) + cookie = cur_ops->get_gp_state(); + if (dopoll_full) + cur_ops->get_gp_state_full(&cookie_full); + if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) + sync(); + sync(); + WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), + "%s: Cookie check 3 failed %pS() online %*pbl.", + __func__, sync, cpumask_pr_args(cpu_online_mask)); + WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), + "%s: Cookie check 4 failed %pS() online %*pbl", + __func__, sync, cpumask_pr_args(cpu_online_mask)); + if (dopoll || dopoll_full) + cpus_read_unlock(); +} + /* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure @@ -1212,8 +1336,10 @@ rcu_torture_writer(void *arg) bool boot_ended; bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); unsigned long cookie; + struct rcu_gp_oldstate cookie_full; int expediting = 0; unsigned long gp_snap; + struct rcu_gp_oldstate gp_snap_full; int i; int idx; int oldnice = task_nice(current); @@ -1261,11 +1387,12 @@ rcu_torture_writer(void *arg) atomic_inc(&rcu_torture_wcount[i]); WRITE_ONCE(old_rp->rtort_pipe_count, old_rp->rtort_pipe_count + 1); + + // Make sure readers block polled grace periods. if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { idx = cur_ops->readlock(); cookie = cur_ops->get_gp_state(); - WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE && - cur_ops->poll_gp_state(cookie), + WARN_ONCE(cur_ops->poll_gp_state(cookie), "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", __func__, rcu_torture_writer_state_getname(), @@ -1277,6 +1404,21 @@ rcu_torture_writer(void *arg) } cur_ops->readunlock(idx); } + if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { + idx = cur_ops->readlock(); + cur_ops->get_gp_state_full(&cookie_full); + WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), + "%s: Cookie check 5 failed %s(%d) online %*pbl\n", + __func__, + rcu_torture_writer_state_getname(), + rcu_torture_writer_state, + cpumask_pr_args(cpu_online_mask)); + if (cur_ops->get_gp_completed_full) { + cur_ops->get_gp_completed_full(&cookie_full); + WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); + } + cur_ops->readunlock(idx); + } switch (synctype[torture_random(&rand) % nsynctypes]) { case RTWS_DEF_FREE: rcu_torture_writer_state = RTWS_DEF_FREE; @@ -1284,12 +1426,7 @@ rcu_torture_writer(void *arg) break; case RTWS_EXP_SYNC: rcu_torture_writer_state = RTWS_EXP_SYNC; - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state(); - cur_ops->exp_sync(); - cur_ops->exp_sync(); - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); + do_rtws_sync(&rand, cur_ops->exp_sync); rcu_torture_pipe_update(old_rp); break; case RTWS_COND_GET: @@ -1308,6 +1445,22 @@ rcu_torture_writer(void *arg) cur_ops->cond_sync_exp(gp_snap); rcu_torture_pipe_update(old_rp); break; + case RTWS_COND_GET_FULL: + rcu_torture_writer_state = RTWS_COND_GET_FULL; + cur_ops->get_gp_state_full(&gp_snap_full); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + rcu_torture_writer_state = RTWS_COND_SYNC_FULL; + cur_ops->cond_sync_full(&gp_snap_full); + rcu_torture_pipe_update(old_rp); + break; + case RTWS_COND_GET_EXP_FULL: + rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; + cur_ops->get_gp_state_full(&gp_snap_full); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; + cur_ops->cond_sync_exp_full(&gp_snap_full); + rcu_torture_pipe_update(old_rp); + break; case RTWS_POLL_GET: rcu_torture_writer_state = RTWS_POLL_GET; gp_snap = cur_ops->start_gp_poll(); @@ -1317,6 +1470,15 @@ rcu_torture_writer(void *arg) &rand); rcu_torture_pipe_update(old_rp); break; + case RTWS_POLL_GET_FULL: + rcu_torture_writer_state = RTWS_POLL_GET_FULL; + cur_ops->start_gp_poll_full(&gp_snap_full); + rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; + while (!cur_ops->poll_gp_state_full(&gp_snap_full)) + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + rcu_torture_pipe_update(old_rp); + break; case RTWS_POLL_GET_EXP: rcu_torture_writer_state = RTWS_POLL_GET_EXP; gp_snap = cur_ops->start_gp_poll_exp(); @@ -1326,14 +1488,18 @@ rcu_torture_writer(void *arg) &rand); rcu_torture_pipe_update(old_rp); break; + case RTWS_POLL_GET_EXP_FULL: + rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; + cur_ops->start_gp_poll_exp_full(&gp_snap_full); + rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; + while (!cur_ops->poll_gp_state_full(&gp_snap_full)) + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + rcu_torture_pipe_update(old_rp); + break; case RTWS_SYNC: rcu_torture_writer_state = RTWS_SYNC; - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state(); - cur_ops->sync(); - cur_ops->sync(); - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); + do_rtws_sync(&rand, cur_ops->sync); rcu_torture_pipe_update(old_rp); break; default: @@ -1400,6 +1566,7 @@ static int rcu_torture_fakewriter(void *arg) { unsigned long gp_snap; + struct rcu_gp_oldstate gp_snap_full; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); @@ -1438,6 +1605,16 @@ rcu_torture_fakewriter(void *arg) torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); cur_ops->cond_sync_exp(gp_snap); break; + case RTWS_COND_GET_FULL: + cur_ops->get_gp_state_full(&gp_snap_full); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + cur_ops->cond_sync_full(&gp_snap_full); + break; + case RTWS_COND_GET_EXP_FULL: + cur_ops->get_gp_state_full(&gp_snap_full); + torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); + cur_ops->cond_sync_exp_full(&gp_snap_full); + break; case RTWS_POLL_GET: gp_snap = cur_ops->start_gp_poll(); while (!cur_ops->poll_gp_state(gp_snap)) { @@ -1445,6 +1622,13 @@ rcu_torture_fakewriter(void *arg) &rand); } break; + case RTWS_POLL_GET_FULL: + cur_ops->start_gp_poll_full(&gp_snap_full); + while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + } + break; case RTWS_POLL_GET_EXP: gp_snap = cur_ops->start_gp_poll_exp(); while (!cur_ops->poll_gp_state_exp(gp_snap)) { @@ -1452,6 +1636,13 @@ rcu_torture_fakewriter(void *arg) &rand); } break; + case RTWS_POLL_GET_EXP_FULL: + cur_ops->start_gp_poll_exp_full(&gp_snap_full); + while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { + torture_hrtimeout_jiffies(torture_random(&rand) % 16, + &rand); + } + break; case RTWS_SYNC: cur_ops->sync(); break; @@ -1715,7 +1906,9 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { + bool checkpolling = !(torture_random(trsp) & 0xfff); unsigned long cookie; + struct rcu_gp_oldstate cookie_full; int i; unsigned long started; unsigned long completed; @@ -1731,8 +1924,12 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) WARN_ON_ONCE(!rcu_is_watching()); newstate = rcutorture_extend_mask(readstate, trsp); rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - cookie = cur_ops->get_gp_state(); + if (checkpolling) { + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + cookie = cur_ops->get_gp_state(); + if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) + cur_ops->get_gp_state_full(&cookie_full); + } started = cur_ops->get_gp_seq(); ts = rcu_trace_clock_local(); p = rcu_dereference_check(rcu_torture_current, @@ -1766,13 +1963,22 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) } __this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); - if (cur_ops->get_gp_state && cur_ops->poll_gp_state) - WARN_ONCE(cur_ops->poll_gp_state(cookie), - "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", - __func__, - rcu_torture_writer_state_getname(), - rcu_torture_writer_state, - cookie, cur_ops->get_gp_state()); + if (checkpolling) { + if (cur_ops->get_gp_state && cur_ops->poll_gp_state) + WARN_ONCE(cur_ops->poll_gp_state(cookie), + "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", + __func__, + rcu_torture_writer_state_getname(), + rcu_torture_writer_state, + cookie, cur_ops->get_gp_state()); + if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) + WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), + "%s: Cookie check 6 failed %s(%d) online %*pbl\n", + __func__, + rcu_torture_writer_state_getname(), + rcu_torture_writer_state, + cpumask_pr_args(cpu_online_mask)); + } rcutorture_one_extend(&readstate, 0, trsp, rtrsp); WARN_ON_ONCE(readstate); // This next splat is expected behavior if leakpointer, especially @@ -2600,12 +2806,12 @@ static int rcutorture_oom_notify(struct notifier_block *self, for (i = 0; i < fwd_progress; i++) ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); - rcu_barrier(); + cur_ops->cb_barrier(); ncbs = 0; for (i = 0; i < fwd_progress; i++) ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); - rcu_barrier(); + cur_ops->cb_barrier(); ncbs = 0; for (i = 0; i < fwd_progress; i++) ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 92c002d65482872dc536efd16ff2317518a0a34b..33adafdad26138906557fa78ee024d4bef11965e 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp) struct srcu_struct *ssp; ssp = container_of(wp, struct srcu_struct, srcu_work); - if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) + if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) return; /* Already running or nothing to do. */ /* Remove recently arrived callbacks and wait for readers. */ @@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp) * straighten that out. */ WRITE_ONCE(ssp->srcu_gp_running, false); - if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) + if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) schedule_work(&ssp->srcu_work); } EXPORT_SYMBOL_GPL(srcu_drive_gp); static void srcu_gp_start_if_needed(struct srcu_struct *ssp) { - unsigned short cookie; + unsigned long cookie; cookie = get_state_synchronize_srcu(ssp); - if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) + if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) return; WRITE_ONCE(ssp->srcu_idx_max, cookie); if (!READ_ONCE(ssp->srcu_gp_running)) { @@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) barrier(); ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1; barrier(); - return ret & USHRT_MAX; + return ret; } EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); @@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); */ bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) { - bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie); + unsigned long cur_s = READ_ONCE(ssp->srcu_idx); barrier(); - return ret; + return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3); } EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 83c7e6620d40317c4785c3da770dcd82e2eb1069..f5bf6fb430dabf9ec6e2d34cb521dab03e75b2b5 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -560,7 +560,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp) { /* Complain if the scheduler has not started. */ - RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, + WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE, "synchronize_rcu_tasks called too soon"); // If the grace-period kthread is running, use it. @@ -1500,6 +1500,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) if (rcu_tasks_trace_pertask_prep(t, true)) trc_add_holdout(t, hop); rcu_read_unlock(); + cond_resched_tasks_rcu_qs(); } // Only after all running tasks have been accounted for is it @@ -1520,6 +1521,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) raw_spin_lock_irqsave_rcu_node(rtpcp, flags); } raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); + cond_resched_tasks_rcu_qs(); } // Re-enable CPU hotplug now that the holdout list is populated. @@ -1619,6 +1621,7 @@ static void check_all_holdout_tasks_trace(struct list_head *hop, trc_del_holdout(t); else if (needreport) show_stalled_task_trace(t, firstreport); + cond_resched_tasks_rcu_qs(); } // Re-enable CPU hotplug now that the holdout list scan has completed. diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index f0561ee16b9c25bc82ce26cd750a1a70962e058b..a33a8d4942c37eef52f3da2bc4fdc88d49043b16 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -158,6 +158,10 @@ void synchronize_rcu(void) } EXPORT_SYMBOL_GPL(synchronize_rcu); +static void tiny_rcu_leak_callback(struct rcu_head *rhp) +{ +} + /* * Post an RCU callback to be invoked after the end of an RCU grace * period. But since we have but one CPU, that would be after any @@ -165,9 +169,20 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); */ void call_rcu(struct rcu_head *head, rcu_callback_t func) { + static atomic_t doublefrees; unsigned long flags; - debug_rcu_head_queue(head); + if (debug_rcu_head_queue(head)) { + if (atomic_inc_return(&doublefrees) < 4) { + pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); + mem_dump_obj(head); + } + + if (!__is_kvfree_rcu_offset((unsigned long)head->func)) + WRITE_ONCE(head->func, tiny_rcu_leak_callback); + return; + } + head->func = func; head->next = NULL; @@ -183,6 +198,16 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func) } EXPORT_SYMBOL_GPL(call_rcu); +/* + * Store a grace-period-counter "cookie". For more information, + * see the Tree RCU header comment. + */ +void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; +} +EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); + /* * Return a grace-period-counter "cookie". For more information, * see the Tree RCU header comment. diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 79aea7df4345e417ec4ed75646902f9eb2227f1a..6bb8e72bc8151ef2eb4093f2464f32c225672f88 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -76,6 +76,7 @@ /* Data structures. */ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = { + .gpwrap = true, #ifdef CONFIG_RCU_NOCB_CPU .cblist.flags = SEGCBLIST_RCU_CORE, #endif @@ -1755,6 +1756,8 @@ static noinline void rcu_gp_cleanup(void) dump_blkd_tasks(rnp, 10); WARN_ON_ONCE(rnp->qsmask); WRITE_ONCE(rnp->gp_seq, new_gp_seq); + if (!rnp->parent) + smp_mb(); // Order against failing poll_state_synchronize_rcu_full(). rdp = this_cpu_ptr(&rcu_data); if (rnp == rdp->mynode) needgp = __note_gp_changes(rnp, rdp) || needgp; @@ -2341,8 +2344,8 @@ void rcu_sched_clock_irq(int user) rcu_flavor_sched_clock_irq(user); if (rcu_pending(user)) invoke_rcu_core(); - if (user) - rcu_tasks_classic_qs(current, false); + if (user || rcu_is_cpu_rrupt_from_idle()) + rcu_note_voluntary_context_switch(current); lockdep_assert_irqs_disabled(); trace_rcu_utilization(TPS("End scheduler-tick")); @@ -2832,7 +2835,7 @@ EXPORT_SYMBOL_GPL(call_rcu); /* Maximum number of jiffies to wait before draining a batch. */ -#define KFREE_DRAIN_JIFFIES (HZ / 50) +#define KFREE_DRAIN_JIFFIES (5 * HZ) #define KFREE_N_BATCHES 2 #define FREE_N_CHANNELS 2 @@ -3093,6 +3096,21 @@ need_offload_krc(struct kfree_rcu_cpu *krcp) return !!krcp->head; } +static void +schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp) +{ + long delay, delay_left; + + delay = READ_ONCE(krcp->count) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES; + if (delayed_work_pending(&krcp->monitor_work)) { + delay_left = krcp->monitor_work.timer.expires - jiffies; + if (delay < delay_left) + mod_delayed_work(system_wq, &krcp->monitor_work, delay); + return; + } + queue_delayed_work(system_wq, &krcp->monitor_work, delay); +} + /* * This function is invoked after the KFREE_DRAIN_JIFFIES timeout. */ @@ -3150,7 +3168,7 @@ static void kfree_rcu_monitor(struct work_struct *work) // work to repeat an attempt. Because previous batches are // still in progress. if (need_offload_krc(krcp)) - schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); + schedule_delayed_monitor_work(krcp); raw_spin_unlock_irqrestore(&krcp->lock, flags); } @@ -3183,15 +3201,16 @@ static void fill_page_cache_func(struct work_struct *work) bnode = (struct kvfree_rcu_bulk_data *) __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); - if (bnode) { - raw_spin_lock_irqsave(&krcp->lock, flags); - pushed = put_cached_bnode(krcp, bnode); - raw_spin_unlock_irqrestore(&krcp->lock, flags); + if (!bnode) + break; - if (!pushed) { - free_page((unsigned long) bnode); - break; - } + raw_spin_lock_irqsave(&krcp->lock, flags); + pushed = put_cached_bnode(krcp, bnode); + raw_spin_unlock_irqrestore(&krcp->lock, flags); + + if (!pushed) { + free_page((unsigned long) bnode); + break; } } @@ -3338,7 +3357,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) // Set timer to drain after KFREE_DRAIN_JIFFIES. if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING) - schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES); + schedule_delayed_monitor_work(krcp); unlock_return: krc_this_cpu_unlock(krcp, flags); @@ -3371,7 +3390,7 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc) atomic_set(&krcp->backoff_page_cache_fill, 1); } - return count; + return count == 0 ? SHRINK_EMPTY : count; } static unsigned long @@ -3414,49 +3433,27 @@ void __init kfree_rcu_scheduler_running(void) raw_spin_lock_irqsave(&krcp->lock, flags); if (need_offload_krc(krcp)) - schedule_delayed_work_on(cpu, &krcp->monitor_work, KFREE_DRAIN_JIFFIES); + schedule_delayed_monitor_work(krcp); raw_spin_unlock_irqrestore(&krcp->lock, flags); } } /* * During early boot, any blocking grace-period wait automatically - * implies a grace period. Later on, this is never the case for PREEMPTION. + * implies a grace period. * - * However, because a context switch is a grace period for !PREEMPTION, any - * blocking grace-period wait automatically implies a grace period if - * there is only one CPU online at any point time during execution of - * either synchronize_rcu() or synchronize_rcu_expedited(). It is OK to - * occasionally incorrectly indicate that there are multiple CPUs online - * when there was in fact only one the whole time, as this just adds some - * overhead: RCU still operates correctly. + * Later on, this could in theory be the case for kernels built with + * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this + * is not a common case. Furthermore, this optimization would cause + * the rcu_gp_oldstate structure to expand by 50%, so this potential + * grace-period optimization is ignored once the scheduler is running. */ static int rcu_blocking_is_gp(void) { - int ret; - - // Invoking preempt_model_*() too early gets a splat. - if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE || - preempt_model_full() || preempt_model_rt()) - return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE; + if (rcu_scheduler_active != RCU_SCHEDULER_INACTIVE) + return false; might_sleep(); /* Check for RCU read-side critical section. */ - preempt_disable(); - /* - * If the rcu_state.n_online_cpus counter is equal to one, - * there is only one CPU, and that CPU sees all prior accesses - * made by any CPU that was online at the time of its access. - * Furthermore, if this counter is equal to one, its value cannot - * change until after the preempt_enable() below. - * - * Furthermore, if rcu_state.n_online_cpus is equal to one here, - * all later CPUs (both this one and any that come online later - * on) are guaranteed to see all accesses prior to this point - * in the code, without the need for additional memory barriers. - * Those memory barriers are provided by CPU-hotplug code. - */ - ret = READ_ONCE(rcu_state.n_online_cpus) <= 1; - preempt_enable(); - return ret; + return true; } /** @@ -3499,29 +3496,58 @@ static int rcu_blocking_is_gp(void) */ void synchronize_rcu(void) { + unsigned long flags; + struct rcu_node *rnp; + RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_rcu() in RCU read-side critical section"); - if (rcu_blocking_is_gp()) { - // Note well that this code runs with !PREEMPT && !SMP. - // In addition, all code that advances grace periods runs at - // process level. Therefore, this normal GP overlaps with - // other normal GPs only by being fully nested within them, - // which allows reuse of ->gp_seq_polled_snap. - rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); - rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); - if (rcu_init_invoked()) - cond_resched_tasks_rcu_qs(); - return; // Context allows vacuous grace periods. + if (!rcu_blocking_is_gp()) { + if (rcu_gp_is_expedited()) + synchronize_rcu_expedited(); + else + wait_rcu_gp(call_rcu); + return; } - if (rcu_gp_is_expedited()) - synchronize_rcu_expedited(); - else - wait_rcu_gp(call_rcu); + + // Context allows vacuous grace periods. + // Note well that this code runs with !PREEMPT && !SMP. + // In addition, all code that advances grace periods runs at + // process level. Therefore, this normal GP overlaps with other + // normal GPs only by being fully nested within them, which allows + // reuse of ->gp_seq_polled_snap. + rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap); + rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap); + + // Update the normal grace-period counters to record + // this grace period, but only those used by the boot CPU. + // The rcu_scheduler_starting() will take care of the rest of + // these counters. + local_irq_save(flags); + WARN_ON_ONCE(num_online_cpus() > 1); + rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT); + for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent) + rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; + local_irq_restore(flags); } EXPORT_SYMBOL_GPL(synchronize_rcu); +/** + * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie + * @rgosp: Place to put state cookie + * + * Stores into @rgosp a value that will always be treated by functions + * like poll_state_synchronize_rcu_full() as a cookie whose grace period + * has already completed. + */ +void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; + rgosp->rgos_exp = RCU_GET_STATE_COMPLETED; +} +EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); + /** * get_state_synchronize_rcu - Snapshot current RCU state * @@ -3541,21 +3567,42 @@ unsigned long get_state_synchronize_rcu(void) EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); /** - * start_poll_synchronize_rcu - Snapshot and start RCU grace period + * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited + * @rgosp: location to place combined normal/expedited grace-period state * - * Returns a cookie that is used by a later call to cond_synchronize_rcu() - * or poll_state_synchronize_rcu() to determine whether or not a full - * grace period has elapsed in the meantime. If the needed grace period - * is not already slated to start, notifies RCU core of the need for that - * grace period. + * Places the normal and expedited grace-period states in @rgosp. This + * state value can be passed to a later call to cond_synchronize_rcu_full() + * or poll_state_synchronize_rcu_full() to determine whether or not a + * grace period (whether normal or expedited) has elapsed in the meantime. + * The rcu_gp_oldstate structure takes up twice the memory of an unsigned + * long, but is guaranteed to see all grace periods. In contrast, the + * combined state occupies less memory, but can sometimes fail to take + * grace periods into account. * - * Interrupts must be enabled for the case where it is necessary to awaken - * the grace-period kthread. + * This does not guarantee that the needed grace period will actually + * start. */ -unsigned long start_poll_synchronize_rcu(void) +void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + struct rcu_node *rnp = rcu_get_root(); + + /* + * Any prior manipulation of RCU-protected data must happen + * before the loads from ->gp_seq and ->expedited_sequence. + */ + smp_mb(); /* ^^^ */ + rgosp->rgos_norm = rcu_seq_snap(&rnp->gp_seq); + rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); +} +EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full); + +/* + * Helper function for start_poll_synchronize_rcu() and + * start_poll_synchronize_rcu_full(). + */ +static void start_poll_synchronize_rcu_common(void) { unsigned long flags; - unsigned long gp_seq = get_state_synchronize_rcu(); bool needwake; struct rcu_data *rdp; struct rcu_node *rnp; @@ -3575,17 +3622,57 @@ unsigned long start_poll_synchronize_rcu(void) raw_spin_unlock_irqrestore_rcu_node(rnp, flags); if (needwake) rcu_gp_kthread_wake(); +} + +/** + * start_poll_synchronize_rcu - Snapshot and start RCU grace period + * + * Returns a cookie that is used by a later call to cond_synchronize_rcu() + * or poll_state_synchronize_rcu() to determine whether or not a full + * grace period has elapsed in the meantime. If the needed grace period + * is not already slated to start, notifies RCU core of the need for that + * grace period. + * + * Interrupts must be enabled for the case where it is necessary to awaken + * the grace-period kthread. + */ +unsigned long start_poll_synchronize_rcu(void) +{ + unsigned long gp_seq = get_state_synchronize_rcu(); + + start_poll_synchronize_rcu_common(); return gp_seq; } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); /** - * poll_state_synchronize_rcu - Conditionally wait for an RCU grace period + * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period + * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * + * Places the normal and expedited grace-period states in *@rgos. This + * state value can be passed to a later call to cond_synchronize_rcu_full() + * or poll_state_synchronize_rcu_full() to determine whether or not a + * grace period (whether normal or expedited) has elapsed in the meantime. + * If the needed grace period is not already slated to start, notifies + * RCU core of the need for that grace period. + * + * Interrupts must be enabled for the case where it is necessary to awaken + * the grace-period kthread. + */ +void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + get_state_synchronize_rcu_full(rgosp); + + start_poll_synchronize_rcu_common(); +} +EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full); + +/** + * poll_state_synchronize_rcu - Has the specified RCU grace period completed? * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu() * * If a full RCU grace period has elapsed since the earlier call from - * which oldstate was obtained, return @true, otherwise return @false. + * which @oldstate was obtained, return @true, otherwise return @false. * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller * can explicitly wait for a grace period, for example, by passing @oldstate @@ -3594,10 +3681,11 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited for * more than a billion grace periods (and way more on a 64-bit system!). - * Those needing to keep oldstate values for very long time periods - * (many hours even on 32-bit systems) should check them occasionally - * and either refresh them or set a flag indicating that the grace period - * has completed. + * Those needing to keep old state values for very long time periods + * (many hours even on 32-bit systems) should check them occasionally and + * either refresh them or set a flag indicating that the grace period has + * completed. Alternatively, they can use get_completed_synchronize_rcu() + * to get a guaranteed-completed grace-period state. * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call @@ -3616,8 +3704,56 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); /** - * cond_synchronize_rcu - Conditionally wait for an RCU grace period + * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? + * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * + * If a full RCU grace period has elapsed since the earlier call from + * which *rgosp was obtained, return @true, otherwise return @false. + * If @false is returned, it is the caller's responsibility to invoke this + * function later on until it does return @true. Alternatively, the caller + * can explicitly wait for a grace period, for example, by passing @rgosp + * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). + * + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited + * for more than a billion grace periods (and way more on a 64-bit + * system!). Those needing to keep rcu_gp_oldstate values for very + * long time periods (many hours even on 32-bit systems) should check + * them occasionally and either refresh them or set a flag indicating + * that the grace period has completed. Alternatively, they can use + * get_completed_synchronize_rcu_full() to get a guaranteed-completed + * grace-period state. + * + * This function provides the same memory-ordering guarantees that would + * be provided by a synchronize_rcu() that was invoked at the call to + * the function that provided @rgosp, and that returned at the end of this + * function. And this guarantee requires that the root rcu_node structure's + * ->gp_seq field be checked instead of that of the rcu_state structure. + * The problem is that the just-ending grace-period's callbacks can be + * invoked between the time that the root rcu_node structure's ->gp_seq + * field is updated and the time that the rcu_state structure's ->gp_seq + * field is updated. Therefore, if a single synchronize_rcu() is to + * cause a subsequent poll_state_synchronize_rcu_full() to return @true, + * then the root rcu_node structure is the one that needs to be polled. + */ +bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + struct rcu_node *rnp = rcu_get_root(); + + smp_mb(); // Order against root rcu_node structure grace-period cleanup. + if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || + rgosp->rgos_exp == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { + smp_mb(); /* Ensure GP ends before subsequent accesses. */ + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu_full); + +/** + * cond_synchronize_rcu - Conditionally wait for an RCU grace period * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited() * * If a full RCU grace period has elapsed since the earlier call to @@ -3641,6 +3777,33 @@ void cond_synchronize_rcu(unsigned long oldstate) } EXPORT_SYMBOL_GPL(cond_synchronize_rcu); +/** + * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period + * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * + * If a full RCU grace period has elapsed since the call to + * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), + * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * obtained, just return. Otherwise, invoke synchronize_rcu() to wait + * for a full grace period. + * + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for + * more than 2 billion grace periods (and way more on a 64-bit system!), + * so waiting for a couple of additional grace periods should be just fine. + * + * This function provides the same memory-ordering guarantees that + * would be provided by a synchronize_rcu() that was invoked at the call + * to the function that provided @rgosp and that returned at the end of + * this function. + */ +void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +{ + if (!poll_state_synchronize_rcu_full(rgosp)) + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); + /* * Check to see if there is any immediate RCU-related work to be done by * the current CPU, returning 1 if so and zero otherwise. The checks are @@ -4312,9 +4475,20 @@ early_initcall(rcu_spawn_gp_kthread); */ void rcu_scheduler_starting(void) { + unsigned long flags; + struct rcu_node *rnp; + WARN_ON(num_online_cpus() != 1); WARN_ON(nr_context_switches() > 0); rcu_test_sync_prims(); + + // Fix up the ->gp_seq counters. + local_irq_save(flags); + rcu_for_each_node_breadth_first(rnp) + rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq; + local_irq_restore(flags); + + // Switch out of early boot mode. rcu_scheduler_active = RCU_SCHEDULER_INIT; rcu_test_sync_prims(); } diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index be667583a5547e59542ca5ff44ea11c419f9862a..18e9b4cd78ef8294a2e5ab752c981dbaaa1d4a79 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -828,11 +828,13 @@ static void rcu_exp_handler(void *unused) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; + bool preempt_bh_enabled = !(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK)); if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) return; - if (rcu_is_cpu_rrupt_from_idle()) { + if (rcu_is_cpu_rrupt_from_idle() || + (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); return; } @@ -906,6 +908,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp) void synchronize_rcu_expedited(void) { bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT); + unsigned long flags; struct rcu_exp_work rew; struct rcu_node *rnp; unsigned long s; @@ -924,8 +927,11 @@ void synchronize_rcu_expedited(void) // them, which allows reuse of ->gp_seq_polled_exp_snap. rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap); rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap); - if (rcu_init_invoked()) - cond_resched(); + + local_irq_save(flags); + WARN_ON_ONCE(num_online_cpus() > 1); + rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT); + local_irq_restore(flags); return; // Context allows vacuous grace periods. } @@ -1027,6 +1033,24 @@ unsigned long start_poll_synchronize_rcu_expedited(void) } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); +/** + * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period + * @rgosp: Place to put snapshot of grace-period state + * + * Places the normal and expedited grace-period states in rgosp. This + * state value can be passed to a later call to cond_synchronize_rcu_full() + * or poll_state_synchronize_rcu_full() to determine whether or not a + * grace period (whether normal or expedited) has elapsed in the meantime. + * If the needed expedited grace period is not already slated to start, + * initiates that grace period. + */ +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +{ + get_state_synchronize_rcu_full(rgosp); + (void)start_poll_synchronize_rcu_expedited(); +} +EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full); + /** * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period * @@ -1053,3 +1077,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate) synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); + +/** + * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period + * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * + * If a full RCU grace period has elapsed since the call to + * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), + * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * obtained, just return. Otherwise, invoke synchronize_rcu_expedited() + * to wait for a full grace period. + * + * Yes, this function does not take counter wrap into account. + * But counter wrap is harmless. If the counter wraps, we have waited for + * more than 2 billion grace periods (and way more on a 64-bit system!), + * so waiting for a couple of additional grace periods should be just fine. + * + * This function provides the same memory-ordering guarantees that + * would be provided by a synchronize_rcu() that was invoked at the call + * to the function that provided @rgosp and that returned at the end of + * this function. + */ +void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +{ + if (!poll_state_synchronize_rcu_full(rgosp)) + synchronize_rcu_expedited(); +} +EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index a8f574d8850d22db43ad208b9ca295a9dc75735a..0a5f0ef41484518575591f7811eff31d45e751ff 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1111,7 +1111,7 @@ int rcu_nocb_cpu_deoffload(int cpu) if (!ret) cpumask_clear_cpu(cpu, rcu_nocb_mask); } else { - pr_info("NOCB: Can't CB-deoffload an offline CPU\n"); + pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu); ret = -EINVAL; } } @@ -1196,7 +1196,7 @@ int rcu_nocb_cpu_offload(int cpu) if (!ret) cpumask_set_cpu(cpu, rcu_nocb_mask); } else { - pr_info("NOCB: Can't CB-offload an offline CPU\n"); + pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu); ret = -EINVAL; } } @@ -1452,8 +1452,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) (long)rdp->nocb_gp_seq, rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', - rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, - show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); + rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, + show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread)); } /* Dump out nocb kthread state for the specified rcu_data structure. */ @@ -1497,7 +1497,7 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], rcu_segcblist_n_cbs(&rdp->cblist), rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', - rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, + rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1, show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); /* It is OK for GP kthreads to have GP state. */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 438ecae6bd7e7ab8cea93e4ac97b65bb1cff1066..e3142ee35fc6ac80c94e7697795b23bf70b492ac 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -641,7 +641,8 @@ static void rcu_read_unlock_special(struct task_struct *t) expboost = (t->rcu_blocked_node && READ_ONCE(t->rcu_blocked_node->exp_tasks)) || (rdp->grpmask & READ_ONCE(rnp->expmask)) || - IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) || + (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && + ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) || (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled && t->rcu_blocked_node); // Need to defer quiescent state until everything is enabled. @@ -718,9 +719,6 @@ static void rcu_flavor_sched_clock_irq(int user) struct task_struct *t = current; lockdep_assert_irqs_disabled(); - if (user || rcu_is_cpu_rrupt_from_idle()) { - rcu_note_voluntary_context_switch(current); - } if (rcu_preempt_depth() > 0 || (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) { /* No QS, force context switch if deferred. */ @@ -824,6 +822,7 @@ void rcu_read_unlock_strict(void) if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread) return; rdp = this_cpu_ptr(&rcu_data); + rdp->cpu_no_qs.b.norm = false; rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } @@ -869,7 +868,7 @@ void rcu_all_qs(void) if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) return; - preempt_disable(); + preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) { preempt_enable(); @@ -931,10 +930,13 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) return false; } -// Except that we do need to respond to a request by an expedited grace -// period for a quiescent state from this CPU. Note that requests from -// tasks are handled when removing the task from the blocked-tasks list -// below. +// Except that we do need to respond to a request by an expedited +// grace period for a quiescent state from this CPU. Note that in +// non-preemptible kernels, there can be no context switches within RCU +// read-side critical sections, which in turn means that the leaf rcu_node +// structure's blocked-tasks list is always empty. is therefore no need to +// actually check it. Instead, a quiescent state from this CPU suffices, +// and this function is only called from such a quiescent state. notrace void rcu_preempt_deferred_qs(struct task_struct *t) { struct rcu_data *rdp = this_cpu_ptr(&rcu_data); @@ -972,7 +974,6 @@ static void rcu_flavor_sched_clock_irq(int user) * neither access nor modify, at least not while the * corresponding CPU is online. */ - rcu_qs(); } } @@ -1238,8 +1239,11 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) cpu != outgoingcpu) cpumask_set_cpu(cpu, cm); cpumask_and(cm, cm, housekeeping_cpumask(HK_TYPE_RCU)); - if (cpumask_empty(cm)) + if (cpumask_empty(cm)) { cpumask_copy(cm, housekeeping_cpumask(HK_TYPE_RCU)); + if (outgoingcpu >= 0) + cpumask_clear_cpu(outgoingcpu, cm); + } set_cpus_allowed_ptr(t, cm); mutex_unlock(&rnp->boost_kthread_mutex); free_cpumask_var(cm); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index c3fbbcc09327ff348055cebd21003b266fa983bf..5653560573e22d650788b7695fedc85f50568756 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -368,7 +368,7 @@ static void rcu_dump_cpu_stacks(void) if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) { if (cpu_is_offline(cpu)) pr_err("Offline CPU %d blocking current GP.\n", cpu); - else if (!trigger_single_cpu_backtrace(cpu)) + else dump_cpu_task(cpu); } raw_spin_unlock_irqrestore_rcu_node(rnp, flags); @@ -511,8 +511,7 @@ static void rcu_check_gp_kthread_starvation(void) pr_err("RCU GP kthread last ran on offline CPU %d.\n", cpu); } else { pr_err("Stack dump where RCU GP kthread last ran:\n"); - if (!trigger_single_cpu_backtrace(cpu)) - dump_cpu_task(cpu); + dump_cpu_task(cpu); } } wake_up_process(gpk); diff --git a/kernel/reboot.c b/kernel/reboot.c index 3c35445bf5ad34cad207b91e8086e8d8498fa6f6..3bba88c7ffc6be6b86ee69d1b73f64d7beb70542 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -243,6 +243,17 @@ void migrate_to_reboot_cpu(void) set_cpus_allowed_ptr(current, cpumask_of(cpu)); } +/* + * Notifier list for kernel code which wants to be called + * to prepare system for restart. + */ +static BLOCKING_NOTIFIER_HEAD(restart_prep_handler_list); + +static void do_kernel_restart_prepare(void) +{ + blocking_notifier_call_chain(&restart_prep_handler_list, 0, NULL); +} + /** * kernel_restart - reboot the system * @cmd: pointer to buffer containing command to execute for restart @@ -254,6 +265,7 @@ void migrate_to_reboot_cpu(void) void kernel_restart(char *cmd) { kernel_restart_prepare(cmd); + do_kernel_restart_prepare(); migrate_to_reboot_cpu(); syscore_shutdown(); if (!cmd) @@ -396,6 +408,11 @@ register_sys_off_handler(enum sys_off_mode mode, handler->list = &power_off_handler_list; break; + case SYS_OFF_MODE_RESTART_PREPARE: + handler->list = &restart_prep_handler_list; + handler->blocking = true; + break; + case SYS_OFF_MODE_RESTART: handler->list = &restart_handler_list; break; diff --git a/kernel/relay.c b/kernel/relay.c index 6a611e779e9587f6c79f5a72cab91e9708aab327..d7edc934c56d5e498461a078941e80dac4cf8345 100644 --- a/kernel/relay.c +++ b/kernel/relay.c @@ -60,10 +60,7 @@ static const struct vm_operations_struct relay_file_mmap_ops = { */ static struct page **relay_alloc_page_array(unsigned int n_pages) { - const size_t pa_size = n_pages * sizeof(struct page *); - if (pa_size > PAGE_SIZE) - return vzalloc(pa_size); - return kzalloc(pa_size, GFP_KERNEL); + return kvcalloc(n_pages, sizeof(struct page *), GFP_KERNEL); } /* diff --git a/kernel/sched/autogroup.c b/kernel/sched/autogroup.c index 4ebaf97f7bd8576b31a27f3836535e264a42b007..991fc90025357964edf8d64a361a843e926160c1 100644 --- a/kernel/sched/autogroup.c +++ b/kernel/sched/autogroup.c @@ -161,7 +161,8 @@ autogroup_move_group(struct task_struct *p, struct autogroup *ag) struct task_struct *t; unsigned long flags; - BUG_ON(!lock_task_sighand(p, &flags)); + if (WARN_ON_ONCE(!lock_task_sighand(p, &flags))) + return; prev = p->signal->autogroup; if (prev == ag) { diff --git a/kernel/sched/completion.c b/kernel/sched/completion.c index 35f15c26ed54d47ad12bb8b07011f1cf54e5b066..d57a5c1c1cd97ac15bc23a0e1d56f1d3ea7ee83f 100644 --- a/kernel/sched/completion.c +++ b/kernel/sched/completion.c @@ -204,6 +204,7 @@ EXPORT_SYMBOL(wait_for_completion_io_timeout); int __sched wait_for_completion_interruptible(struct completion *x) { long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); + if (t == -ERESTARTSYS) return t; return 0; @@ -241,12 +242,23 @@ EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); int __sched wait_for_completion_killable(struct completion *x) { long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); + if (t == -ERESTARTSYS) return t; return 0; } EXPORT_SYMBOL(wait_for_completion_killable); +int __sched wait_for_completion_state(struct completion *x, unsigned int state) +{ + long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state); + + if (t == -ERESTARTSYS) + return t; + return 0; +} +EXPORT_SYMBOL(wait_for_completion_state); + /** * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) * @x: holds the state of this particular completion diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ee28253c9ac0c2ed4e602a9584b5e1a77c4f2825..5800b0623ff30687cf60b24e5109fc40e5ee9229 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -73,6 +73,7 @@ #include +#include #include #include @@ -142,11 +143,7 @@ __read_mostly int sysctl_resched_latency_warn_once = 1; * Number of tasks to iterate in a single balance run. * Limited because this is done with IRQs disabled. */ -#ifdef CONFIG_PREEMPT_RT -const_debug unsigned int sysctl_sched_nr_migrate = 8; -#else -const_debug unsigned int sysctl_sched_nr_migrate = 32; -#endif +const_debug unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; __read_mostly int scheduler_running; @@ -360,10 +357,7 @@ static void __sched_core_flip(bool enabled) /* * Toggle the offline CPUs. */ - cpumask_copy(&sched_core_mask, cpu_possible_mask); - cpumask_andnot(&sched_core_mask, &sched_core_mask, cpu_online_mask); - - for_each_cpu(cpu, &sched_core_mask) + for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) cpu_rq(cpu)->core_enabled = enabled; cpus_read_unlock(); @@ -481,8 +475,7 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } * p->se.load, p->rt_priority, * p->dl.dl_{runtime, deadline, period, flags, bw, density} * - sched_setnuma(): p->numa_preferred_nid - * - sched_move_task()/ - * cpu_cgroup_fork(): p->sched_task_group + * - sched_move_task(): p->sched_task_group * - uclamp_update_active() p->uclamp* * * p->state <- TASK_*: @@ -708,6 +701,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) rq->prev_irq_time += irq_delta; delta -= irq_delta; + psi_account_irqtime(rq->curr, irq_delta); #endif #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING if (static_key_false((¶virt_steal_rq_enabled))) { @@ -2328,7 +2322,7 @@ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, rq = cpu_rq(new_cpu); rq_lock(rq, rf); - BUG_ON(task_cpu(p) != new_cpu); + WARN_ON_ONCE(task_cpu(p) != new_cpu); activate_task(rq, p, 0); check_preempt_curr(rq, p, 0); @@ -2778,7 +2772,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag return -EINVAL; } - if (task_running(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { + if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { /* * MIGRATE_ENABLE gets here because 'p == current', but for * anything else we cannot do is_migration_disabled(), punt @@ -3254,12 +3248,12 @@ out: /* * wait_task_inactive - wait for a thread to unschedule. * - * If @match_state is nonzero, it's the @p->state value just checked and - * not expected to change. If it changes, i.e. @p might have woken up, - * then return zero. When we succeed in waiting for @p to be off its CPU, - * we return a positive number (its total switch count). If a second call - * a short while later returns the same number, the caller can be sure that - * @p has remained unscheduled the whole time. + * Wait for the thread to block in any of the states set in @match_state. + * If it changes, i.e. @p might have woken up, then return zero. When we + * succeed in waiting for @p to be off its CPU, we return a positive number + * (its total switch count). If a second call a short while later returns the + * same number, the caller can be sure that @p has remained unscheduled the + * whole time. * * The caller must ensure that the task *will* unschedule sometime soon, * else this function might spin for a *long* time. This function can't @@ -3290,12 +3284,12 @@ unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state * * NOTE! Since we don't hold any locks, it's not * even sure that "rq" stays as the right runqueue! - * But we don't care, since "task_running()" will + * But we don't care, since "task_on_cpu()" will * return false if the runqueue has changed and p * is actually now running somewhere else! */ - while (task_running(rq, p)) { - if (match_state && unlikely(READ_ONCE(p->__state) != match_state)) + while (task_on_cpu(rq, p)) { + if (!(READ_ONCE(p->__state) & match_state)) return 0; cpu_relax(); } @@ -3307,10 +3301,10 @@ unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state */ rq = task_rq_lock(p, &rf); trace_sched_wait_task(p); - running = task_running(rq, p); + running = task_on_cpu(rq, p); queued = task_on_rq_queued(p); ncsw = 0; - if (!match_state || READ_ONCE(p->__state) == match_state) + if (READ_ONCE(p->__state) & match_state) ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ task_rq_unlock(rq, p, &rf); @@ -4396,6 +4390,17 @@ void set_numabalancing_state(bool enabled) } #ifdef CONFIG_PROC_SYSCTL +static void reset_memory_tiering(void) +{ + struct pglist_data *pgdat; + + for_each_online_pgdat(pgdat) { + pgdat->nbp_threshold = 0; + pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); + pgdat->nbp_th_start = jiffies_to_msecs(jiffies); + } +} + int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -4412,6 +4417,9 @@ int sysctl_numa_balancing(struct ctl_table *table, int write, if (err < 0) return err; if (write) { + if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && + (state & NUMA_BALANCING_MEMORY_TIERING)) + reset_memory_tiering(); sysctl_numa_balancing_mode = state; __set_numabalancing_state(state); } @@ -5166,6 +5174,7 @@ context_switch(struct rq *rq, struct task_struct *prev, * finish_task_switch()'s mmdrop(). */ switch_mm_irqs_off(prev->active_mm, next->mm, next); + lru_gen_use_mm(next->mm); if (!prev->mm) { // from kernel /* will mmdrop() in finish_task_switch(). */ @@ -6429,7 +6438,7 @@ static void __sched notrace __schedule(unsigned int sched_mode) prev->sched_contributes_to_load = (prev_state & TASK_UNINTERRUPTIBLE) && !(prev_state & TASK_NOLOAD) && - !(prev->flags & PF_FROZEN); + !(prev_state & TASK_FROZEN); if (prev->sched_contributes_to_load) rq->nr_uninterruptible++; @@ -8649,7 +8658,7 @@ again: if (curr->sched_class != p->sched_class) goto out_unlock; - if (task_running(p_rq, p) || !task_is_running(p)) + if (task_on_cpu(p_rq, p) || !task_is_running(p)) goto out_unlock; yielded = curr->sched_class->yield_to_task(rq, p); @@ -8861,7 +8870,7 @@ void sched_show_task(struct task_struct *p) if (pid_alive(p)) ppid = task_pid_nr(rcu_dereference(p->real_parent)); rcu_read_unlock(); - pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n", + pr_cont(" stack:%-5lu pid:%-5d ppid:%-6d flags:0x%08lx\n", free, task_pid_nr(p), ppid, read_task_thread_flags(p)); @@ -8889,7 +8898,7 @@ state_filter_match(unsigned long state_filter, struct task_struct *p) * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows * TASK_KILLABLE). */ - if (state_filter == TASK_UNINTERRUPTIBLE && state == TASK_IDLE) + if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) return false; return true; @@ -9601,9 +9610,6 @@ LIST_HEAD(task_groups); static struct kmem_cache *task_group_cache __read_mostly; #endif -DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); -DECLARE_PER_CPU(cpumask_var_t, select_rq_mask); - void __init sched_init(void) { unsigned long ptr = 0; @@ -9647,14 +9653,6 @@ void __init sched_init(void) #endif /* CONFIG_RT_GROUP_SCHED */ } -#ifdef CONFIG_CPUMASK_OFFSTACK - for_each_possible_cpu(i) { - per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( - cpumask_size(), GFP_KERNEL, cpu_to_node(i)); - per_cpu(select_rq_mask, i) = (cpumask_var_t)kzalloc_node( - cpumask_size(), GFP_KERNEL, cpu_to_node(i)); - } -#endif /* CONFIG_CPUMASK_OFFSTACK */ init_rt_bandwidth(&def_rt_bandwidth, global_rt_period(), global_rt_runtime()); @@ -10163,7 +10161,7 @@ void sched_release_group(struct task_group *tg) spin_unlock_irqrestore(&task_group_lock, flags); } -static void sched_change_group(struct task_struct *tsk, int type) +static void sched_change_group(struct task_struct *tsk) { struct task_group *tg; @@ -10179,7 +10177,7 @@ static void sched_change_group(struct task_struct *tsk, int type) #ifdef CONFIG_FAIR_GROUP_SCHED if (tsk->sched_class->task_change_group) - tsk->sched_class->task_change_group(tsk, type); + tsk->sched_class->task_change_group(tsk); else #endif set_task_rq(tsk, task_cpu(tsk)); @@ -10210,7 +10208,7 @@ void sched_move_task(struct task_struct *tsk) if (running) put_prev_task(rq, tsk); - sched_change_group(tsk, TASK_MOVE_GROUP); + sched_change_group(tsk); if (queued) enqueue_task(rq, tsk, queue_flags); @@ -10288,53 +10286,19 @@ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) sched_unregister_group(tg); } -/* - * This is called before wake_up_new_task(), therefore we really only - * have to set its group bits, all the other stuff does not apply. - */ -static void cpu_cgroup_fork(struct task_struct *task) -{ - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(task, &rf); - - update_rq_clock(rq); - sched_change_group(task, TASK_SET_GROUP); - - task_rq_unlock(rq, task, &rf); -} - +#ifdef CONFIG_RT_GROUP_SCHED static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) { struct task_struct *task; struct cgroup_subsys_state *css; - int ret = 0; cgroup_taskset_for_each(task, css, tset) { -#ifdef CONFIG_RT_GROUP_SCHED if (!sched_rt_can_attach(css_tg(css), task)) return -EINVAL; -#endif - /* - * Serialize against wake_up_new_task() such that if it's - * running, we're sure to observe its full state. - */ - raw_spin_lock_irq(&task->pi_lock); - /* - * Avoid calling sched_move_task() before wake_up_new_task() - * has happened. This would lead to problems with PELT, due to - * move wanting to detach+attach while we're not attached yet. - */ - if (READ_ONCE(task->__state) == TASK_NEW) - ret = -EINVAL; - raw_spin_unlock_irq(&task->pi_lock); - - if (ret) - break; } - return ret; + return 0; } +#endif static void cpu_cgroup_attach(struct cgroup_taskset *tset) { @@ -11170,8 +11134,9 @@ struct cgroup_subsys cpu_cgrp_subsys = { .css_released = cpu_cgroup_css_released, .css_free = cpu_cgroup_css_free, .css_extra_stat_show = cpu_extra_stat_show, - .fork = cpu_cgroup_fork, +#ifdef CONFIG_RT_GROUP_SCHED .can_attach = cpu_cgroup_can_attach, +#endif .attach = cpu_cgroup_attach, .legacy_cftypes = cpu_legacy_files, .dfl_cftypes = cpu_files, @@ -11183,6 +11148,19 @@ struct cgroup_subsys cpu_cgrp_subsys = { void dump_cpu_task(int cpu) { + if (cpu == smp_processor_id() && in_hardirq()) { + struct pt_regs *regs; + + regs = get_irq_regs(); + if (regs) { + show_regs(regs); + return; + } + } + + if (trigger_single_cpu_backtrace(cpu)) + return; + pr_info("Task dump for CPU %d:\n", cpu); sched_show_task(cpu_curr(cpu)); } diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c index 93878cb2a46dcaab88465c2f9c76db353144c275..a57fd8f27498f60eefc9b617a57712e1eaae9c4c 100644 --- a/kernel/sched/core_sched.c +++ b/kernel/sched/core_sched.c @@ -88,7 +88,7 @@ static unsigned long sched_core_update_cookie(struct task_struct *p, * core has now entered/left forced idle state. Defer accounting to the * next scheduling edge, rather than always forcing a reschedule here. */ - if (task_running(rq, p)) + if (task_on_cpu(rq, p)) resched_curr(rq); task_rq_unlock(rq, p, &rf); @@ -205,7 +205,7 @@ int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, default: err = -EINVAL; goto out; - }; + } if (type == PIDTYPE_PID) { __sched_core_set(task, cookie); diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c index 02d970a879edd4e15ba7e1cce67f9b3637d86b2e..57c92d751bcd733053f171bbeee2cd514647e8d8 100644 --- a/kernel/sched/cpudeadline.c +++ b/kernel/sched/cpudeadline.c @@ -123,7 +123,7 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p, unsigned long cap, max_cap = 0; int cpu, max_cpu = -1; - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return 1; /* Ensure the capacity of the CPUs fits the task. */ diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 1207c78f85c11fab5ac211098ae175d14c66526d..9161d1136d01c290c237c1a8f1126deb9c48fb4e 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -25,6 +25,9 @@ struct sugov_policy { unsigned int next_freq; unsigned int cached_raw_freq; + /* max CPU capacity, which is equal for all CPUs in freq. domain */ + unsigned long max; + /* The next fields are only needed if fast switch cannot be used: */ struct irq_work irq_work; struct kthread_work work; @@ -48,7 +51,6 @@ struct sugov_cpu { unsigned long util; unsigned long bw_dl; - unsigned long max; /* The field below is for single-CPU policies only: */ #ifdef CONFIG_NO_HZ_COMMON @@ -158,7 +160,6 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu) { struct rq *rq = cpu_rq(sg_cpu->cpu); - sg_cpu->max = arch_scale_cpu_capacity(sg_cpu->cpu); sg_cpu->bw_dl = cpu_bw_dl(rq); sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu), FREQUENCY_UTIL, NULL); @@ -253,6 +254,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, */ static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time) { + struct sugov_policy *sg_policy = sg_cpu->sg_policy; unsigned long boost; /* No boost currently required */ @@ -280,7 +282,8 @@ static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time) * sg_cpu->util is already in capacity scale; convert iowait_boost * into the same scale so we can compare. */ - boost = (sg_cpu->iowait_boost * sg_cpu->max) >> SCHED_CAPACITY_SHIFT; + boost = sg_cpu->iowait_boost * sg_policy->max; + boost >>= SCHED_CAPACITY_SHIFT; boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL); if (sg_cpu->util < boost) sg_cpu->util = boost; @@ -337,7 +340,7 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time, if (!sugov_update_single_common(sg_cpu, time, flags)) return; - next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max); + next_f = get_next_freq(sg_policy, sg_cpu->util, sg_policy->max); /* * Do not reduce the frequency if the CPU has not been idle * recently, as the reduction is likely to be premature then. @@ -373,6 +376,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, unsigned int flags) { struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); + struct sugov_policy *sg_policy = sg_cpu->sg_policy; unsigned long prev_util = sg_cpu->util; /* @@ -399,7 +403,8 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, sg_cpu->util = prev_util; cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl), - map_util_perf(sg_cpu->util), sg_cpu->max); + map_util_perf(sg_cpu->util), + sg_policy->max); sg_cpu->sg_policy->last_freq_update_time = time; } @@ -408,25 +413,19 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) { struct sugov_policy *sg_policy = sg_cpu->sg_policy; struct cpufreq_policy *policy = sg_policy->policy; - unsigned long util = 0, max = 1; + unsigned long util = 0; unsigned int j; for_each_cpu(j, policy->cpus) { struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j); - unsigned long j_util, j_max; sugov_get_util(j_sg_cpu); sugov_iowait_apply(j_sg_cpu, time); - j_util = j_sg_cpu->util; - j_max = j_sg_cpu->max; - if (j_util * max > j_max * util) { - util = j_util; - max = j_max; - } + util = max(j_sg_cpu->util, util); } - return get_next_freq(sg_policy, util, max); + return get_next_freq(sg_policy, util, sg_policy->max); } static void @@ -752,7 +751,7 @@ static int sugov_start(struct cpufreq_policy *policy) { struct sugov_policy *sg_policy = policy->governor_data; void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); - unsigned int cpu; + unsigned int cpu = cpumask_first(policy->cpus); sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; sg_policy->last_freq_update_time = 0; @@ -760,6 +759,7 @@ static int sugov_start(struct cpufreq_policy *policy) sg_policy->work_in_progress = false; sg_policy->limits_changed = false; sg_policy->cached_raw_freq = 0; + sg_policy->max = arch_scale_cpu_capacity(cpu); sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS); diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index fa9ce9d836839df6cb220d70c7b2dc1f11fb344d..a286e726eb4b857d12f2a03b2cadfa07d0df9e52 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c @@ -147,7 +147,7 @@ int cpupri_find_fitness(struct cpupri *cp, struct task_struct *p, int task_pri = convert_prio(p->prio); int idx, cpu; - BUG_ON(task_pri >= CPUPRI_NR_PRIORITIES); + WARN_ON_ONCE(task_pri >= CPUPRI_NR_PRIORITIES); for (idx = 0; idx < task_pri; idx++) { diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 0ab79d819a0d64f0b533683605b914d8b9398443..86dea6a05267d408e2f46ab189989b6a26c8bafb 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -124,15 +124,12 @@ static inline int dl_bw_cpus(int i) return cpus; } -static inline unsigned long __dl_bw_capacity(int i) +static inline unsigned long __dl_bw_capacity(const struct cpumask *mask) { - struct root_domain *rd = cpu_rq(i)->rd; unsigned long cap = 0; + int i; - RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), - "sched RCU must be held"); - - for_each_cpu_and(i, rd->span, cpu_active_mask) + for_each_cpu_and(i, mask, cpu_active_mask) cap += capacity_orig_of(i); return cap; @@ -144,11 +141,14 @@ static inline unsigned long __dl_bw_capacity(int i) */ static inline unsigned long dl_bw_capacity(int i) { - if (!static_branch_unlikely(&sched_asym_cpucapacity) && + if (!sched_asym_cpucap_active() && capacity_orig_of(i) == SCHED_CAPACITY_SCALE) { return dl_bw_cpus(i) << SCHED_CAPACITY_SHIFT; } else { - return __dl_bw_capacity(i); + RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), + "sched RCU must be held"); + + return __dl_bw_capacity(cpu_rq(i)->rd->span); } } @@ -310,7 +310,7 @@ static void dl_change_utilization(struct task_struct *p, u64 new_bw) { struct rq *rq; - BUG_ON(p->dl.flags & SCHED_FLAG_SUGOV); + WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); if (task_on_rq_queued(p)) return; @@ -431,8 +431,8 @@ static void task_non_contending(struct task_struct *p) sub_rq_bw(&p->dl, &rq->dl); raw_spin_lock(&dl_b->lock); __dl_sub(dl_b, p->dl.dl_bw, dl_bw_cpus(task_cpu(p))); - __dl_clear_params(p); raw_spin_unlock(&dl_b->lock); + __dl_clear_params(p); } return; @@ -607,7 +607,7 @@ static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) { struct rb_node *leftmost; - BUG_ON(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); + WARN_ON_ONCE(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); leftmost = rb_add_cached(&p->pushable_dl_tasks, &rq->dl.pushable_dl_tasks_root, @@ -684,7 +684,7 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p * Failed to find any suitable CPU. * The task will never come back! */ - BUG_ON(dl_bandwidth_enabled()); + WARN_ON_ONCE(dl_bandwidth_enabled()); /* * If admission control is disabled we @@ -770,6 +770,14 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags); static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags); static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, int flags); +static inline void replenish_dl_new_period(struct sched_dl_entity *dl_se, + struct rq *rq) +{ + /* for non-boosted task, pi_of(dl_se) == dl_se */ + dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; + dl_se->runtime = pi_of(dl_se)->dl_runtime; +} + /* * We are being explicitly informed that a new instance is starting, * and this means that: @@ -803,8 +811,7 @@ static inline void setup_new_dl_entity(struct sched_dl_entity *dl_se) * future; in fact, we must consider execution overheads (time * spent on hardirq context, etc.). */ - dl_se->deadline = rq_clock(rq) + dl_se->dl_deadline; - dl_se->runtime = dl_se->dl_runtime; + replenish_dl_new_period(dl_se, rq); } /* @@ -830,16 +837,14 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) struct dl_rq *dl_rq = dl_rq_of_se(dl_se); struct rq *rq = rq_of_dl_rq(dl_rq); - BUG_ON(pi_of(dl_se)->dl_runtime <= 0); + WARN_ON_ONCE(pi_of(dl_se)->dl_runtime <= 0); /* * This could be the case for a !-dl task that is boosted. * Just go with full inherited parameters. */ - if (dl_se->dl_deadline == 0) { - dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; - dl_se->runtime = pi_of(dl_se)->dl_runtime; - } + if (dl_se->dl_deadline == 0) + replenish_dl_new_period(dl_se, rq); if (dl_se->dl_yielded && dl_se->runtime > 0) dl_se->runtime = 0; @@ -866,8 +871,7 @@ static void replenish_dl_entity(struct sched_dl_entity *dl_se) */ if (dl_time_before(dl_se->deadline, rq_clock(rq))) { printk_deferred_once("sched: DL replenish lagged too much\n"); - dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; - dl_se->runtime = pi_of(dl_se)->dl_runtime; + replenish_dl_new_period(dl_se, rq); } if (dl_se->dl_yielded) @@ -1024,8 +1028,7 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) return; } - dl_se->deadline = rq_clock(rq) + pi_of(dl_se)->dl_deadline; - dl_se->runtime = pi_of(dl_se)->dl_runtime; + replenish_dl_new_period(dl_se, rq); } } @@ -1333,11 +1336,7 @@ static void update_curr_dl(struct rq *rq) trace_sched_stat_runtime(curr, delta_exec, 0); - curr->se.sum_exec_runtime += delta_exec; - account_group_exec_runtime(curr, delta_exec); - - curr->se.exec_start = now; - cgroup_account_cputime(curr, delta_exec); + update_current_exec_runtime(curr, now, delta_exec); if (dl_entity_is_special(dl_se)) return; @@ -1616,7 +1615,7 @@ static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) { struct dl_rq *dl_rq = dl_rq_of_se(dl_se); - BUG_ON(!RB_EMPTY_NODE(&dl_se->rb_node)); + WARN_ON_ONCE(!RB_EMPTY_NODE(&dl_se->rb_node)); rb_add_cached(&dl_se->rb_node, &dl_rq->root, __dl_less); @@ -1640,7 +1639,7 @@ static void __dequeue_dl_entity(struct sched_dl_entity *dl_se) static void enqueue_dl_entity(struct sched_dl_entity *dl_se, int flags) { - BUG_ON(on_dl_rq(dl_se)); + WARN_ON_ONCE(on_dl_rq(dl_se)); update_stats_enqueue_dl(dl_rq_of_se(dl_se), dl_se, flags); @@ -1814,6 +1813,14 @@ static void yield_task_dl(struct rq *rq) #ifdef CONFIG_SMP +static inline bool dl_task_is_earliest_deadline(struct task_struct *p, + struct rq *rq) +{ + return (!rq->dl.dl_nr_running || + dl_time_before(p->dl.deadline, + rq->dl.earliest_dl.curr)); +} + static int find_later_rq(struct task_struct *task); static int @@ -1849,16 +1856,14 @@ select_task_rq_dl(struct task_struct *p, int cpu, int flags) * Take the capacity of the CPU into account to * ensure it fits the requirement of the task. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) + if (sched_asym_cpucap_active()) select_rq |= !dl_task_fits_capacity(p, cpu); if (select_rq) { int target = find_later_rq(p); if (target != -1 && - (dl_time_before(p->dl.deadline, - cpu_rq(target)->dl.earliest_dl.curr) || - (cpu_rq(target)->dl.dl_nr_running == 0))) + dl_task_is_earliest_deadline(p, cpu_rq(target))) cpu = target; } rcu_read_unlock(); @@ -2017,7 +2022,7 @@ static struct task_struct *pick_task_dl(struct rq *rq) return NULL; dl_se = pick_next_dl_entity(dl_rq); - BUG_ON(!dl_se); + WARN_ON_ONCE(!dl_se); p = dl_task_of(dl_se); return p; @@ -2087,7 +2092,7 @@ static void task_fork_dl(struct task_struct *p) static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu) { - if (!task_running(rq, p) && + if (!task_on_cpu(rq, p) && cpumask_test_cpu(cpu, &p->cpus_mask)) return 1; return 0; @@ -2225,9 +2230,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) later_rq = cpu_rq(cpu); - if (later_rq->dl.dl_nr_running && - !dl_time_before(task->dl.deadline, - later_rq->dl.earliest_dl.curr)) { + if (!dl_task_is_earliest_deadline(task, later_rq)) { /* * Target rq has tasks of equal or earlier deadline, * retrying does not release any lock and is unlikely @@ -2241,7 +2244,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) if (double_lock_balance(rq, later_rq)) { if (unlikely(task_rq(task) != rq || !cpumask_test_cpu(later_rq->cpu, &task->cpus_mask) || - task_running(rq, task) || + task_on_cpu(rq, task) || !dl_task(task) || !task_on_rq_queued(task))) { double_unlock_balance(rq, later_rq); @@ -2255,9 +2258,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) * its earliest one has a later deadline than our * task, the rq is a good one. */ - if (!later_rq->dl.dl_nr_running || - dl_time_before(task->dl.deadline, - later_rq->dl.earliest_dl.curr)) + if (dl_task_is_earliest_deadline(task, later_rq)) break; /* Otherwise we try again. */ @@ -2277,12 +2278,12 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); - BUG_ON(rq->cpu != task_cpu(p)); - BUG_ON(task_current(rq, p)); - BUG_ON(p->nr_cpus_allowed <= 1); + WARN_ON_ONCE(rq->cpu != task_cpu(p)); + WARN_ON_ONCE(task_current(rq, p)); + WARN_ON_ONCE(p->nr_cpus_allowed <= 1); - BUG_ON(!task_on_rq_queued(p)); - BUG_ON(!dl_task(p)); + WARN_ON_ONCE(!task_on_rq_queued(p)); + WARN_ON_ONCE(!dl_task(p)); return p; } @@ -2428,9 +2429,7 @@ static void pull_dl_task(struct rq *this_rq) * - it will preempt the last one we pulled (if any). */ if (p && dl_time_before(p->dl.deadline, dmin) && - (!this_rq->dl.dl_nr_running || - dl_time_before(p->dl.deadline, - this_rq->dl.earliest_dl.curr))) { + dl_task_is_earliest_deadline(p, this_rq)) { WARN_ON(p == src_rq->curr); WARN_ON(!task_on_rq_queued(p)); @@ -2475,7 +2474,7 @@ skip: */ static void task_woken_dl(struct rq *rq, struct task_struct *p) { - if (!task_running(rq, p) && + if (!task_on_cpu(rq, p) && !test_tsk_need_resched(rq->curr) && p->nr_cpus_allowed > 1 && dl_task(rq->curr) && @@ -2492,7 +2491,7 @@ static void set_cpus_allowed_dl(struct task_struct *p, struct root_domain *src_rd; struct rq *rq; - BUG_ON(!dl_task(p)); + WARN_ON_ONCE(!dl_task(p)); rq = task_rq(p); src_rd = rq->rd; @@ -3007,17 +3006,15 @@ bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr) int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial) { - int ret = 1, trial_cpus; + unsigned long flags, cap; struct dl_bw *cur_dl_b; - unsigned long flags; + int ret = 1; rcu_read_lock_sched(); cur_dl_b = dl_bw_of(cpumask_any(cur)); - trial_cpus = cpumask_weight(trial); - + cap = __dl_bw_capacity(trial); raw_spin_lock_irqsave(&cur_dl_b->lock, flags); - if (cur_dl_b->bw != -1 && - cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw) + if (__dl_overflow(cur_dl_b, cap, 0, 0)) ret = 0; raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); rcu_read_unlock_sched(); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index bb3d63bdf4ae87558782b143a6dc238ad07cfd5a..1637b65ba07aca9ee67c2e23ae36cba1ad144734 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -333,6 +333,7 @@ static __init int sched_init_debug(void) debugfs_create_u32("scan_period_min_ms", 0644, numa, &sysctl_numa_balancing_scan_period_min); debugfs_create_u32("scan_period_max_ms", 0644, numa, &sysctl_numa_balancing_scan_period_max); debugfs_create_u32("scan_size_mb", 0644, numa, &sysctl_numa_balancing_scan_size); + debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); #endif debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); @@ -416,7 +417,7 @@ void update_sched_domain_debugfs(void) char buf[32]; snprintf(buf, sizeof(buf), "cpu%d", cpu); - debugfs_remove(debugfs_lookup(buf, sd_dentry)); + debugfs_lookup_and_remove(buf, sd_dentry); d_cpu = debugfs_create_dir(buf, sd_dentry); i = 0; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 914096c5b1ae1eae09e4fde0b0c859f3dbdf3ae6..e4a0b8bd941c78446353d275d4b22e283acd21f9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -40,6 +40,7 @@ #include #include +#include #include #include #include @@ -799,8 +800,6 @@ void init_entity_runnable_average(struct sched_entity *se) /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */ } -static void attach_entity_cfs_rq(struct sched_entity *se); - /* * With new tasks being created, their initial util_avgs are extrapolated * based on the cfs_rq's current util_avg: @@ -835,20 +834,6 @@ void post_init_entity_util_avg(struct task_struct *p) long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; - if (cap > 0) { - if (cfs_rq->avg.util_avg != 0) { - sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; - sa->util_avg /= (cfs_rq->avg.load_avg + 1); - - if (sa->util_avg > cap) - sa->util_avg = cap; - } else { - sa->util_avg = cap; - } - } - - sa->runnable_avg = sa->util_avg; - if (p->sched_class != &fair_sched_class) { /* * For !fair tasks do: @@ -864,7 +849,19 @@ void post_init_entity_util_avg(struct task_struct *p) return; } - attach_entity_cfs_rq(se); + if (cap > 0) { + if (cfs_rq->avg.util_avg != 0) { + sa->util_avg = cfs_rq->avg.util_avg * se->load.weight; + sa->util_avg /= (cfs_rq->avg.load_avg + 1); + + if (sa->util_avg > cap) + sa->util_avg = cap; + } else { + sa->util_avg = cap; + } + } + + sa->runnable_avg = sa->util_avg; } #else /* !CONFIG_SMP */ @@ -1094,6 +1091,12 @@ unsigned int sysctl_numa_balancing_scan_size = 256; /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ unsigned int sysctl_numa_balancing_scan_delay = 1000; +/* The page with hint page fault latency < threshold in ms is considered hot */ +unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; + +/* Restrict the NUMA promotion throughput (MB/s) for each target node. */ +unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; + struct numa_group { refcount_t refcount; @@ -1436,6 +1439,120 @@ static inline unsigned long group_weight(struct task_struct *p, int nid, return 1000 * faults / total_faults; } +/* + * If memory tiering mode is enabled, cpupid of slow memory page is + * used to record scan time instead of CPU and PID. When tiering mode + * is disabled at run time, the scan time (in cpupid) will be + * interpreted as CPU and PID. So CPU needs to be checked to avoid to + * access out of array bound. + */ +static inline bool cpupid_valid(int cpupid) +{ + return cpupid_to_cpu(cpupid) < nr_cpu_ids; +} + +/* + * For memory tiering mode, if there are enough free pages (more than + * enough watermark defined here) in fast memory node, to take full + * advantage of fast memory capacity, all recently accessed slow + * memory pages will be migrated to fast memory node without + * considering hot threshold. + */ +static bool pgdat_free_space_enough(struct pglist_data *pgdat) +{ + int z; + unsigned long enough_wmark; + + enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, + pgdat->node_present_pages >> 4); + for (z = pgdat->nr_zones - 1; z >= 0; z--) { + struct zone *zone = pgdat->node_zones + z; + + if (!populated_zone(zone)) + continue; + + if (zone_watermark_ok(zone, 0, + wmark_pages(zone, WMARK_PROMO) + enough_wmark, + ZONE_MOVABLE, 0)) + return true; + } + return false; +} + +/* + * For memory tiering mode, when page tables are scanned, the scan + * time will be recorded in struct page in addition to make page + * PROT_NONE for slow memory page. So when the page is accessed, in + * hint page fault handler, the hint page fault latency is calculated + * via, + * + * hint page fault latency = hint page fault time - scan time + * + * The smaller the hint page fault latency, the higher the possibility + * for the page to be hot. + */ +static int numa_hint_fault_latency(struct page *page) +{ + int last_time, time; + + time = jiffies_to_msecs(jiffies); + last_time = xchg_page_access_time(page, time); + + return (time - last_time) & PAGE_ACCESS_TIME_MASK; +} + +/* + * For memory tiering mode, too high promotion/demotion throughput may + * hurt application latency. So we provide a mechanism to rate limit + * the number of pages that are tried to be promoted. + */ +static bool numa_promotion_rate_limit(struct pglist_data *pgdat, + unsigned long rate_limit, int nr) +{ + unsigned long nr_cand; + unsigned int now, start; + + now = jiffies_to_msecs(jiffies); + mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); + nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); + start = pgdat->nbp_rl_start; + if (now - start > MSEC_PER_SEC && + cmpxchg(&pgdat->nbp_rl_start, start, now) == start) + pgdat->nbp_rl_nr_cand = nr_cand; + if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) + return true; + return false; +} + +#define NUMA_MIGRATION_ADJUST_STEPS 16 + +static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, + unsigned long rate_limit, + unsigned int ref_th) +{ + unsigned int now, start, th_period, unit_th, th; + unsigned long nr_cand, ref_cand, diff_cand; + + now = jiffies_to_msecs(jiffies); + th_period = sysctl_numa_balancing_scan_period_max; + start = pgdat->nbp_th_start; + if (now - start > th_period && + cmpxchg(&pgdat->nbp_th_start, start, now) == start) { + ref_cand = rate_limit * + sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; + nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); + diff_cand = nr_cand - pgdat->nbp_th_nr_cand; + unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; + th = pgdat->nbp_threshold ? : ref_th; + if (diff_cand > ref_cand * 11 / 10) + th = max(th - unit_th, unit_th); + else if (diff_cand < ref_cand * 9 / 10) + th = min(th + unit_th, ref_th * 2); + pgdat->nbp_th_nr_cand = nr_cand; + pgdat->nbp_threshold = th; + } +} + bool should_numa_migrate_memory(struct task_struct *p, struct page * page, int src_nid, int dst_cpu) { @@ -1443,9 +1560,44 @@ bool should_numa_migrate_memory(struct task_struct *p, struct page * page, int dst_nid = cpu_to_node(dst_cpu); int last_cpupid, this_cpupid; + /* + * The pages in slow memory node should be migrated according + * to hot/cold instead of private/shared. + */ + if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && + !node_is_toptier(src_nid)) { + struct pglist_data *pgdat; + unsigned long rate_limit; + unsigned int latency, th, def_th; + + pgdat = NODE_DATA(dst_nid); + if (pgdat_free_space_enough(pgdat)) { + /* workload changed, reset hot threshold */ + pgdat->nbp_threshold = 0; + return true; + } + + def_th = sysctl_numa_balancing_hot_threshold; + rate_limit = sysctl_numa_balancing_promote_rate_limit << \ + (20 - PAGE_SHIFT); + numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); + + th = pgdat->nbp_threshold ? : def_th; + latency = numa_hint_fault_latency(page); + if (latency >= th) + return false; + + return !numa_promotion_rate_limit(pgdat, rate_limit, + thp_nr_pages(page)); + } + this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); last_cpupid = page_cpupid_xchg_last(page, this_cpupid); + if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && + !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) + return false; + /* * Allow first faults or private faults to migrate immediately early in * the lifetime of a task. The magic number 4 is based on waiting for @@ -1592,11 +1744,11 @@ numa_type numa_classify(unsigned int imbalance_pct, #ifdef CONFIG_SCHED_SMT /* Forward declarations of select_idle_sibling helpers */ -static inline bool test_idle_cores(int cpu, bool def); +static inline bool test_idle_cores(int cpu); static inline int numa_idle_core(int idle_core, int cpu) { if (!static_branch_likely(&sched_smt_present) || - idle_core >= 0 || !test_idle_cores(cpu, false)) + idle_core >= 0 || !test_idle_cores(cpu)) return idle_core; /* @@ -2600,7 +2752,7 @@ static void task_numa_group(struct task_struct *p, int cpupid, int flags, if (!join) return; - BUG_ON(irqs_disabled()); + WARN_ON_ONCE(irqs_disabled()); double_lock_irq(&my_grp->lock, &grp->lock); for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { @@ -2685,6 +2837,15 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) if (!p->mm) return; + /* + * NUMA faults statistics are unnecessary for the slow memory + * node for memory tiering mode. + */ + if (!node_is_toptier(mem_node) && + (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || + !cpupid_valid(last_cpupid))) + return; + /* Allocate buffer to track faults on a per-node basis */ if (unlikely(!p->numa_faults)) { int size = sizeof(*p->numa_faults) * @@ -2765,6 +2926,7 @@ static void task_numa_work(struct callback_head *work) struct task_struct *p = current; struct mm_struct *mm = p->mm; u64 runtime = p->se.sum_exec_runtime; + MA_STATE(mas, &mm->mm_mt, 0, 0); struct vm_area_struct *vma; unsigned long start, end; unsigned long nr_pte_updates = 0; @@ -2821,13 +2983,16 @@ static void task_numa_work(struct callback_head *work) if (!mmap_read_trylock(mm)) return; - vma = find_vma(mm, start); + mas_set(&mas, start); + vma = mas_find(&mas, ULONG_MAX); if (!vma) { reset_ptenuma_scan(p); start = 0; - vma = mm->mmap; + mas_set(&mas, start); + vma = mas_find(&mas, ULONG_MAX); } - for (; vma; vma = vma->vm_next) { + + for (; vma; vma = mas_find(&mas, ULONG_MAX)) { if (!vma_migratable(vma) || !vma_policy_mof(vma) || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { continue; @@ -3838,8 +4003,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {} * @cfs_rq: cfs_rq to update * * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) - * avg. The immediate corollary is that all (fair) tasks must be attached, see - * post_init_entity_util_avg(). + * avg. The immediate corollary is that all (fair) tasks must be attached. * * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. * @@ -4003,6 +4167,7 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s #define UPDATE_TG 0x1 #define SKIP_AGE_LOAD 0x2 #define DO_ATTACH 0x4 +#define DO_DETACH 0x8 /* Update task and its cfs_rq load average */ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) @@ -4032,6 +4197,13 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s attach_entity_load_avg(cfs_rq, se); update_tg_load_avg(cfs_rq); + } else if (flags & DO_DETACH) { + /* + * DO_DETACH means we're here from dequeue_entity() + * and we are migrating task out of the CPU. + */ + detach_entity_load_avg(cfs_rq, se); + update_tg_load_avg(cfs_rq); } else if (decayed) { cfs_rq_util_change(cfs_rq, 0); @@ -4064,8 +4236,8 @@ static void remove_entity_load_avg(struct sched_entity *se) /* * tasks cannot exit without having gone through wake_up_new_task() -> - * post_init_entity_util_avg() which will have added things to the - * cfs_rq, so we can remove unconditionally. + * enqueue_task_fair() which will have added things to the cfs_rq, + * so we can remove unconditionally. */ sync_entity_load_avg(se); @@ -4262,7 +4434,7 @@ static inline int task_fits_capacity(struct task_struct *p, static inline void update_misfit_status(struct task_struct *p, struct rq *rq) { - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return; if (!p || p->nr_cpus_allowed == 1) { @@ -4292,6 +4464,7 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) #define UPDATE_TG 0x0 #define SKIP_AGE_LOAD 0x0 #define DO_ATTACH 0x0 +#define DO_DETACH 0x0 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1) { @@ -4434,7 +4607,8 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. - * - Add its load to cfs_rq->runnable_avg + * - For group_entity, update its runnable_weight to reflect the new + * h_nr_running of its group cfs_rq. * - For group_entity, update its weight to reflect the new share of * its group cfs_rq * - Add its new weight to cfs_rq->load.weight @@ -4511,6 +4685,11 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { + int action = UPDATE_TG; + + if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) + action |= DO_DETACH; + /* * Update run-time statistics of the 'current'. */ @@ -4519,12 +4698,13 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) /* * When dequeuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. - * - Subtract its load from the cfs_rq->runnable_avg. + * - For group_entity, update its runnable_weight to reflect the new + * h_nr_running of its group cfs_rq. * - Subtract its previous weight from cfs_rq->load.weight. * - For group entity, update its weight to reflect the new share * of its group cfs_rq. */ - update_load_avg(cfs_rq, se, UPDATE_TG); + update_load_avg(cfs_rq, se, action); se_update_runnable(se); update_stats_dequeue_fair(cfs_rq, se, flags); @@ -5893,8 +6073,8 @@ dequeue_throttle: #ifdef CONFIG_SMP /* Working cpumask for: load_balance, load_balance_newidle. */ -DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); -DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); +static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); +static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); #ifdef CONFIG_NO_HZ_COMMON @@ -6260,7 +6440,7 @@ static inline void set_idle_cores(int cpu, int val) WRITE_ONCE(sds->has_idle_cores, val); } -static inline bool test_idle_cores(int cpu, bool def) +static inline bool test_idle_cores(int cpu) { struct sched_domain_shared *sds; @@ -6268,7 +6448,7 @@ static inline bool test_idle_cores(int cpu, bool def) if (sds) return READ_ONCE(sds->has_idle_cores); - return def; + return false; } /* @@ -6284,7 +6464,7 @@ void __update_idle_core(struct rq *rq) int cpu; rcu_read_lock(); - if (test_idle_cores(core, true)) + if (test_idle_cores(core)) goto unlock; for_each_cpu(cpu, cpu_smt_mask(core)) { @@ -6310,9 +6490,6 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu bool idle = true; int cpu; - if (!static_branch_likely(&sched_smt_present)) - return __select_idle_cpu(core, p); - for_each_cpu(cpu, cpu_smt_mask(core)) { if (!available_idle_cpu(cpu)) { idle = false; @@ -6339,13 +6516,12 @@ static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpu /* * Scan the local SMT mask for idle CPUs. */ -static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) +static int select_idle_smt(struct task_struct *p, int target) { int cpu; - for_each_cpu(cpu, cpu_smt_mask(target)) { - if (!cpumask_test_cpu(cpu, p->cpus_ptr) || - !cpumask_test_cpu(cpu, sched_domain_span(sd))) + for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { + if (cpu == target) continue; if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) return cpu; @@ -6360,9 +6536,9 @@ static inline void set_idle_cores(int cpu, int val) { } -static inline bool test_idle_cores(int cpu, bool def) +static inline bool test_idle_cores(int cpu) { - return def; + return false; } static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) @@ -6370,7 +6546,7 @@ static inline int select_idle_core(struct task_struct *p, int core, struct cpuma return __select_idle_cpu(core, p); } -static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) +static inline int select_idle_smt(struct task_struct *p, int target) { return -1; } @@ -6389,19 +6565,19 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool struct sched_domain_shared *sd_share; struct rq *this_rq = this_rq(); int this = smp_processor_id(); - struct sched_domain *this_sd; + struct sched_domain *this_sd = NULL; u64 time = 0; - this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); - if (!this_sd) - return -1; - cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); if (sched_feat(SIS_PROP) && !has_idle_core) { u64 avg_cost, avg_idle, span_avg; unsigned long now = jiffies; + this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); + if (!this_sd) + return -1; + /* * If we're busy, the assumption that the last idle period * predicts the future is flawed; age away the remaining @@ -6455,7 +6631,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool if (has_idle_core) set_idle_cores(target, false); - if (sched_feat(SIS_PROP) && !has_idle_core) { + if (sched_feat(SIS_PROP) && this_sd && !has_idle_core) { time = cpu_clock(this) - time; /* @@ -6506,7 +6682,7 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) static inline bool asym_fits_capacity(unsigned long task_util, int cpu) { - if (static_branch_unlikely(&sched_asym_cpucapacity)) + if (sched_asym_cpucap_active()) return fits_capacity(task_util, capacity_of(cpu)); return true; @@ -6526,7 +6702,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * On asymmetric system, update task utilization because we will check * that the task fits with cpu's capacity. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { sync_entity_load_avg(&p->se); task_util = uclamp_task_util(p); } @@ -6580,7 +6756,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) * For asymmetric CPU capacity systems, our domain of interest is * sd_asym_cpucapacity rather than sd_llc. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); /* * On an asymmetric CPU capacity system where an exclusive @@ -6601,10 +6777,10 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) return target; if (sched_smt_active()) { - has_idle_core = test_idle_cores(target, false); + has_idle_core = test_idle_cores(target); if (!has_idle_core && cpus_share_cache(prev, target)) { - i = select_idle_smt(p, sd, prev); + i = select_idle_smt(p, prev); if ((unsigned int)i < nr_cpumask_bits) return i; } @@ -7076,8 +7252,6 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) return new_cpu; } -static void detach_entity_cfs_rq(struct sched_entity *se); - /* * Called immediately before a task is migrated to a new CPU; task_cpu(p) and * cfs_rq_of(p) references at time of call are still valid and identify the @@ -7099,15 +7273,7 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) se->vruntime -= u64_u32_load(cfs_rq->min_vruntime); } - if (p->on_rq == TASK_ON_RQ_MIGRATING) { - /* - * In case of TASK_ON_RQ_MIGRATING we in fact hold the 'old' - * rq->lock and can modify state directly. - */ - lockdep_assert_rq_held(task_rq(p)); - detach_entity_cfs_rq(se); - - } else { + if (!task_on_rq_migrating(p)) { remove_entity_load_avg(se); /* @@ -7279,7 +7445,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ return; find_matching_se(&se, &pse); - BUG_ON(!pse); + WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); pse_is_idle = se_is_idle(pse); @@ -7938,7 +8104,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) /* Record that we found at least one task that could run on dst_cpu */ env->flags &= ~LBF_ALL_PINNED; - if (task_running(env->src_rq, p)) { + if (task_on_cpu(env->src_rq, p)) { schedstat_inc(p->stats.nr_failed_migrations_running); return 0; } @@ -8012,8 +8178,6 @@ static struct task_struct *detach_one_task(struct lb_env *env) return NULL; } -static const unsigned int sched_nr_migrate_break = 32; - /* * detach_tasks() -- tries to detach up to imbalance load/util/tasks from * busiest_rq, as part of a balancing operation within domain "sd". @@ -8049,20 +8213,24 @@ static int detach_tasks(struct lb_env *env) if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) break; - p = list_last_entry(tasks, struct task_struct, se.group_node); - env->loop++; - /* We've more or less seen every task there is, call it quits */ - if (env->loop > env->loop_max) + /* + * We've more or less seen every task there is, call it quits + * unless we haven't found any movable task yet. + */ + if (env->loop > env->loop_max && + !(env->flags & LBF_ALL_PINNED)) break; /* take a breather every nr_migrate tasks */ if (env->loop > env->loop_break) { - env->loop_break += sched_nr_migrate_break; + env->loop_break += SCHED_NR_MIGRATE_BREAK; env->flags |= LBF_NEED_BREAK; break; } + p = list_last_entry(tasks, struct task_struct, se.group_node); + if (!can_migrate_task(p, env)) goto next; @@ -8159,7 +8327,7 @@ static void attach_task(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); - BUG_ON(task_rq(p) != rq); + WARN_ON_ONCE(task_rq(p) != rq); activate_task(rq, p, ENQUEUE_NOCLOCK); check_preempt_curr(rq, p, 0); } @@ -10099,14 +10267,13 @@ static int load_balance(int this_cpu, struct rq *this_rq, struct rq *busiest; struct rq_flags rf; struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); - struct lb_env env = { .sd = sd, .dst_cpu = this_cpu, .dst_rq = this_rq, .dst_grpmask = sched_group_span(sd->groups), .idle = idle, - .loop_break = sched_nr_migrate_break, + .loop_break = SCHED_NR_MIGRATE_BREAK, .cpus = cpus, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), @@ -10134,7 +10301,7 @@ redo: goto out_balanced; } - BUG_ON(busiest == env.dst_rq); + WARN_ON_ONCE(busiest == env.dst_rq); schedstat_add(sd->lb_imbalance[idle], env.imbalance); @@ -10182,7 +10349,9 @@ more_balance: if (env.flags & LBF_NEED_BREAK) { env.flags &= ~LBF_NEED_BREAK; - goto more_balance; + /* Stop if we tried all running tasks */ + if (env.loop < busiest->nr_running) + goto more_balance; } /* @@ -10213,7 +10382,7 @@ more_balance: env.dst_cpu = env.new_dst_cpu; env.flags &= ~LBF_DST_PINNED; env.loop = 0; - env.loop_break = sched_nr_migrate_break; + env.loop_break = SCHED_NR_MIGRATE_BREAK; /* * Go back to "more_balance" rather than "redo" since we @@ -10245,7 +10414,7 @@ more_balance: */ if (!cpumask_subset(cpus, env.dst_grpmask)) { env.loop = 0; - env.loop_break = sched_nr_migrate_break; + env.loop_break = SCHED_NR_MIGRATE_BREAK; goto redo; } goto out_all_pinned; @@ -10430,7 +10599,7 @@ static int active_load_balance_cpu_stop(void *data) * we need to fix it. Originally reported by * Bjorn Helgaas on a 128-CPU setup. */ - BUG_ON(busiest_rq == target_rq); + WARN_ON_ONCE(busiest_rq == target_rq); /* Search for an sd spanning us and the target CPU. */ rcu_read_lock(); @@ -10916,8 +11085,7 @@ static bool update_nohz_stats(struct rq *rq) * can be a simple update of blocked load or a complete load balance with * tasks movement depending of flags. */ -static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags, - enum cpu_idle_type idle) +static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) { /* Earliest time when we have to do rebalance again */ unsigned long now = jiffies; @@ -11032,7 +11200,7 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) if (idle != CPU_IDLE) return false; - _nohz_idle_balance(this_rq, flags, idle); + _nohz_idle_balance(this_rq, flags); return true; } @@ -11052,7 +11220,7 @@ void nohz_run_idle_balance(int cpu) * (ie NOHZ_STATS_KICK set) and will do the same. */ if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) - _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK, CPU_IDLE); + _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); } static void nohz_newidle_balance(struct rq *this_rq) @@ -11552,6 +11720,17 @@ static void detach_entity_cfs_rq(struct sched_entity *se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); +#ifdef CONFIG_SMP + /* + * In case the task sched_avg hasn't been attached: + * - A forked task which hasn't been woken up by wake_up_new_task(). + * - A task which has been woken up by try_to_wake_up() but is + * waiting for actually being woken up by sched_ttwu_pending(). + */ + if (!se->avg.last_update_time) + return; +#endif + /* Catch up with the cfs_rq and remove our load when we leave */ update_load_avg(cfs_rq, se, 0); detach_entity_load_avg(cfs_rq, se); @@ -11563,14 +11742,6 @@ static void attach_entity_cfs_rq(struct sched_entity *se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); -#ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Since the real-depth could have been changed (only FAIR - * class maintain depth value), reset depth properly. - */ - se->depth = se->parent ? se->parent->depth + 1 : 0; -#endif - /* Synchronize entity with its cfs_rq */ update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); attach_entity_load_avg(cfs_rq, se); @@ -11666,39 +11837,25 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) } #ifdef CONFIG_FAIR_GROUP_SCHED -static void task_set_group_fair(struct task_struct *p) +static void task_change_group_fair(struct task_struct *p) { - struct sched_entity *se = &p->se; - - set_task_rq(p, task_cpu(p)); - se->depth = se->parent ? se->parent->depth + 1 : 0; -} + /* + * We couldn't detach or attach a forked task which + * hasn't been woken up by wake_up_new_task(). + */ + if (READ_ONCE(p->__state) == TASK_NEW) + return; -static void task_move_group_fair(struct task_struct *p) -{ detach_task_cfs_rq(p); - set_task_rq(p, task_cpu(p)); #ifdef CONFIG_SMP /* Tell se's cfs_rq has been changed -- migrated */ p->se.avg.last_update_time = 0; #endif + set_task_rq(p, task_cpu(p)); attach_task_cfs_rq(p); } -static void task_change_group_fair(struct task_struct *p, int type) -{ - switch (type) { - case TASK_SET_GROUP: - task_set_group_fair(p); - break; - - case TASK_MOVE_GROUP: - task_move_group_fair(p); - break; - } -} - void free_fair_sched_group(struct task_group *tg) { int i; @@ -12075,6 +12232,13 @@ void show_numa_stats(struct task_struct *p, struct seq_file *m) __init void init_sched_fair_class(void) { #ifdef CONFIG_SMP + int i; + + for_each_possible_cpu(i) { + zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); + zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); + } + open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index ec66b40bdd403ba2b4e9d5bb8c4459fc2a0f5ef2..ee2ecc081422eab3244ec6f5b6b1f67e13203f03 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -181,6 +181,7 @@ static void group_init(struct psi_group *group) { int cpu; + group->enabled = true; for_each_possible_cpu(cpu) seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq); group->avg_last_update = sched_clock(); @@ -190,12 +191,8 @@ static void group_init(struct psi_group *group) /* Init trigger-related members */ mutex_init(&group->trigger_lock); INIT_LIST_HEAD(&group->triggers); - memset(group->nr_triggers, 0, sizeof(group->nr_triggers)); - group->poll_states = 0; group->poll_min_period = U32_MAX; - memset(group->polling_total, 0, sizeof(group->polling_total)); group->polling_next_update = ULLONG_MAX; - group->polling_until = 0; init_waitqueue_head(&group->poll_wait); timer_setup(&group->poll_timer, poll_timer_fn, 0); rcu_assign_pointer(group->poll_task, NULL); @@ -205,6 +202,7 @@ void __init psi_init(void) { if (!psi_enable) { static_branch_enable(&psi_disabled); + static_branch_disable(&psi_cgroups_enabled); return; } @@ -215,7 +213,7 @@ void __init psi_init(void) group_init(&psi_system); } -static bool test_state(unsigned int *tasks, enum psi_states state) +static bool test_state(unsigned int *tasks, enum psi_states state, bool oncpu) { switch (state) { case PSI_IO_SOME: @@ -228,9 +226,9 @@ static bool test_state(unsigned int *tasks, enum psi_states state) return unlikely(tasks[NR_MEMSTALL] && tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]); case PSI_CPU_SOME: - return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]); + return unlikely(tasks[NR_RUNNING] > oncpu); case PSI_CPU_FULL: - return unlikely(tasks[NR_RUNNING] && !tasks[NR_ONCPU]); + return unlikely(tasks[NR_RUNNING] && !oncpu); case PSI_NONIDLE: return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] || tasks[NR_RUNNING]; @@ -692,35 +690,53 @@ static void psi_group_change(struct psi_group *group, int cpu, bool wake_clock) { struct psi_group_cpu *groupc; - u32 state_mask = 0; unsigned int t, m; enum psi_states s; + u32 state_mask; groupc = per_cpu_ptr(group->pcpu, cpu); /* - * First we assess the aggregate resource states this CPU's - * tasks have been in since the last change, and account any - * SOME and FULL time these may have resulted in. - * - * Then we update the task counts according to the state + * First we update the task counts according to the state * change requested through the @clear and @set bits. + * + * Then if the cgroup PSI stats accounting enabled, we + * assess the aggregate resource states this CPU's tasks + * have been in since the last change, and account any + * SOME and FULL time these may have resulted in. */ write_seqcount_begin(&groupc->seq); - record_times(groupc, now); + /* + * Start with TSK_ONCPU, which doesn't have a corresponding + * task count - it's just a boolean flag directly encoded in + * the state mask. Clear, set, or carry the current state if + * no changes are requested. + */ + if (unlikely(clear & TSK_ONCPU)) { + state_mask = 0; + clear &= ~TSK_ONCPU; + } else if (unlikely(set & TSK_ONCPU)) { + state_mask = PSI_ONCPU; + set &= ~TSK_ONCPU; + } else { + state_mask = groupc->state_mask & PSI_ONCPU; + } + /* + * The rest of the state mask is calculated based on the task + * counts. Update those first, then construct the mask. + */ for (t = 0, m = clear; m; m &= ~(1 << t), t++) { if (!(m & (1 << t))) continue; if (groupc->tasks[t]) { groupc->tasks[t]--; } else if (!psi_bug) { - printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n", + printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n", cpu, t, groupc->tasks[0], groupc->tasks[1], groupc->tasks[2], - groupc->tasks[3], groupc->tasks[4], - clear, set); + groupc->tasks[3], clear, set); psi_bug = 1; } } @@ -729,9 +745,25 @@ static void psi_group_change(struct psi_group *group, int cpu, if (set & (1 << t)) groupc->tasks[t]++; - /* Calculate state mask representing active states */ + if (!group->enabled) { + /* + * On the first group change after disabling PSI, conclude + * the current state and flush its time. This is unlikely + * to matter to the user, but aggregation (get_recent_times) + * may have already incorporated the live state into times_prev; + * avoid a delta sample underflow when PSI is later re-enabled. + */ + if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE))) + record_times(groupc, now); + + groupc->state_mask = state_mask; + + write_seqcount_end(&groupc->seq); + return; + } + for (s = 0; s < NR_PSI_STATES; s++) { - if (test_state(groupc->tasks, s)) + if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU)) state_mask |= (1 << s); } @@ -743,9 +775,11 @@ static void psi_group_change(struct psi_group *group, int cpu, * task in a cgroup is in_memstall, the corresponding groupc * on that cpu is in PSI_MEM_FULL state. */ - if (unlikely(groupc->tasks[NR_ONCPU] && cpu_curr(cpu)->in_memstall)) + if (unlikely((state_mask & PSI_ONCPU) && cpu_curr(cpu)->in_memstall)) state_mask |= (1 << PSI_MEM_FULL); + record_times(groupc, now); + groupc->state_mask = state_mask; write_seqcount_end(&groupc->seq); @@ -757,27 +791,12 @@ static void psi_group_change(struct psi_group *group, int cpu, schedule_delayed_work(&group->avgs_work, PSI_FREQ); } -static struct psi_group *iterate_groups(struct task_struct *task, void **iter) +static inline struct psi_group *task_psi_group(struct task_struct *task) { - if (*iter == &psi_system) - return NULL; - #ifdef CONFIG_CGROUPS - if (static_branch_likely(&psi_cgroups_enabled)) { - struct cgroup *cgroup = NULL; - - if (!*iter) - cgroup = task->cgroups->dfl_cgrp; - else - cgroup = cgroup_parent(*iter); - - if (cgroup && cgroup_parent(cgroup)) { - *iter = cgroup; - return cgroup_psi(cgroup); - } - } + if (static_branch_likely(&psi_cgroups_enabled)) + return cgroup_psi(task_dfl_cgroup(task)); #endif - *iter = &psi_system; return &psi_system; } @@ -800,8 +819,6 @@ void psi_task_change(struct task_struct *task, int clear, int set) { int cpu = task_cpu(task); struct psi_group *group; - bool wake_clock = true; - void *iter = NULL; u64 now; if (!task->pid) @@ -810,19 +827,11 @@ void psi_task_change(struct task_struct *task, int clear, int set) psi_flags_change(task, clear, set); now = cpu_clock(cpu); - /* - * Periodic aggregation shuts off if there is a period of no - * task changes, so we wake it back up if necessary. However, - * don't do this if the task change is the aggregation worker - * itself going to sleep, or we'll ping-pong forever. - */ - if (unlikely((clear & TSK_RUNNING) && - (task->flags & PF_WQ_WORKER) && - wq_worker_last_func(task) == psi_avgs_work)) - wake_clock = false; - while ((group = iterate_groups(task, &iter))) - psi_group_change(group, cpu, clear, set, now, wake_clock); + group = task_psi_group(task); + do { + psi_group_change(group, cpu, clear, set, now, true); + } while ((group = group->parent)); } void psi_task_switch(struct task_struct *prev, struct task_struct *next, @@ -830,34 +839,30 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, { struct psi_group *group, *common = NULL; int cpu = task_cpu(prev); - void *iter; u64 now = cpu_clock(cpu); if (next->pid) { - bool identical_state; - psi_flags_change(next, 0, TSK_ONCPU); /* - * When switching between tasks that have an identical - * runtime state, the cgroup that contains both tasks - * we reach the first common ancestor. Iterate @next's - * ancestors only until we encounter @prev's ONCPU. + * Set TSK_ONCPU on @next's cgroups. If @next shares any + * ancestors with @prev, those will already have @prev's + * TSK_ONCPU bit set, and we can stop the iteration there. */ - identical_state = prev->psi_flags == next->psi_flags; - iter = NULL; - while ((group = iterate_groups(next, &iter))) { - if (identical_state && - per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) { + group = task_psi_group(next); + do { + if (per_cpu_ptr(group->pcpu, cpu)->state_mask & + PSI_ONCPU) { common = group; break; } psi_group_change(group, cpu, 0, TSK_ONCPU, now, true); - } + } while ((group = group->parent)); } if (prev->pid) { int clear = TSK_ONCPU, set = 0; + bool wake_clock = true; /* * When we're going to sleep, psi_dequeue() lets us @@ -871,26 +876,74 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, clear |= TSK_MEMSTALL_RUNNING; if (prev->in_iowait) set |= TSK_IOWAIT; + + /* + * Periodic aggregation shuts off if there is a period of no + * task changes, so we wake it back up if necessary. However, + * don't do this if the task change is the aggregation worker + * itself going to sleep, or we'll ping-pong forever. + */ + if (unlikely((prev->flags & PF_WQ_WORKER) && + wq_worker_last_func(prev) == psi_avgs_work)) + wake_clock = false; } psi_flags_change(prev, clear, set); - iter = NULL; - while ((group = iterate_groups(prev, &iter)) && group != common) - psi_group_change(group, cpu, clear, set, now, true); + group = task_psi_group(prev); + do { + if (group == common) + break; + psi_group_change(group, cpu, clear, set, now, wake_clock); + } while ((group = group->parent)); /* - * TSK_ONCPU is handled up to the common ancestor. If we're tasked - * with dequeuing too, finish that for the rest of the hierarchy. + * TSK_ONCPU is handled up to the common ancestor. If there are + * any other differences between the two tasks (e.g. prev goes + * to sleep, or only one task is memstall), finish propagating + * those differences all the way up to the root. */ - if (sleep) { + if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) { clear &= ~TSK_ONCPU; - for (; group; group = iterate_groups(prev, &iter)) - psi_group_change(group, cpu, clear, set, now, true); + for (; group; group = group->parent) + psi_group_change(group, cpu, clear, set, now, wake_clock); } } } +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +void psi_account_irqtime(struct task_struct *task, u32 delta) +{ + int cpu = task_cpu(task); + struct psi_group *group; + struct psi_group_cpu *groupc; + u64 now; + + if (!task->pid) + return; + + now = cpu_clock(cpu); + + group = task_psi_group(task); + do { + if (!group->enabled) + continue; + + groupc = per_cpu_ptr(group->pcpu, cpu); + + write_seqcount_begin(&groupc->seq); + + record_times(groupc, now); + groupc->times[PSI_IRQ_FULL] += delta; + + write_seqcount_end(&groupc->seq); + + if (group->poll_states & (1 << PSI_IRQ_FULL)) + psi_schedule_poll_work(group, 1); + } while ((group = group->parent)); +} +#endif + /** * psi_memstall_enter - mark the beginning of a memory stall section * @flags: flags to handle nested sections @@ -921,6 +974,7 @@ void psi_memstall_enter(unsigned long *flags) rq_unlock_irq(rq, &rf); } +EXPORT_SYMBOL_GPL(psi_memstall_enter); /** * psi_memstall_leave - mark the end of an memory stall section @@ -950,14 +1004,15 @@ void psi_memstall_leave(unsigned long *flags) rq_unlock_irq(rq, &rf); } +EXPORT_SYMBOL_GPL(psi_memstall_leave); #ifdef CONFIG_CGROUPS int psi_cgroup_alloc(struct cgroup *cgroup) { - if (static_branch_likely(&psi_disabled)) + if (!static_branch_likely(&psi_cgroups_enabled)) return 0; - cgroup->psi = kmalloc(sizeof(struct psi_group), GFP_KERNEL); + cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL); if (!cgroup->psi) return -ENOMEM; @@ -967,12 +1022,13 @@ int psi_cgroup_alloc(struct cgroup *cgroup) return -ENOMEM; } group_init(cgroup->psi); + cgroup->psi->parent = cgroup_psi(cgroup_parent(cgroup)); return 0; } void psi_cgroup_free(struct cgroup *cgroup) { - if (static_branch_likely(&psi_disabled)) + if (!static_branch_likely(&psi_cgroups_enabled)) return; cancel_delayed_work_sync(&cgroup->psi->avgs_work); @@ -1000,7 +1056,7 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) struct rq_flags rf; struct rq *rq; - if (static_branch_likely(&psi_disabled)) { + if (!static_branch_likely(&psi_cgroups_enabled)) { /* * Lame to do this here, but the scheduler cannot be locked * from the outside, so we move cgroups from inside sched/. @@ -1048,10 +1104,45 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) task_rq_unlock(rq, task, &rf); } + +void psi_cgroup_restart(struct psi_group *group) +{ + int cpu; + + /* + * After we disable psi_group->enabled, we don't actually + * stop percpu tasks accounting in each psi_group_cpu, + * instead only stop test_state() loop, record_times() + * and averaging worker, see psi_group_change() for details. + * + * When disable cgroup PSI, this function has nothing to sync + * since cgroup pressure files are hidden and percpu psi_group_cpu + * would see !psi_group->enabled and only do task accounting. + * + * When re-enable cgroup PSI, this function use psi_group_change() + * to get correct state mask from test_state() loop on tasks[], + * and restart groupc->state_start from now, use .clear = .set = 0 + * here since no task status really changed. + */ + if (!group->enabled) + return; + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct rq_flags rf; + u64 now; + + rq_lock_irq(rq, &rf); + now = cpu_clock(cpu); + psi_group_change(group, cpu, 0, 0, now, true); + rq_unlock_irq(rq, &rf); + } +} #endif /* CONFIG_CGROUPS */ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) { + bool only_full = false; int full; u64 now; @@ -1066,7 +1157,11 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) group->avg_next_update = update_averages(group, now); mutex_unlock(&group->avgs_lock); - for (full = 0; full < 2; full++) { +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + only_full = res == PSI_IRQ; +#endif + + for (full = 0; full < 2 - only_full; full++) { unsigned long avg[3] = { 0, }; u64 total = 0; int w; @@ -1080,7 +1175,7 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) } seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n", - full ? "full" : "some", + full || only_full ? "full" : "some", LOAD_INT(avg[0]), LOAD_FRAC(avg[0]), LOAD_INT(avg[1]), LOAD_FRAC(avg[1]), LOAD_INT(avg[2]), LOAD_FRAC(avg[2]), @@ -1091,7 +1186,7 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) } struct psi_trigger *psi_trigger_create(struct psi_group *group, - char *buf, size_t nbytes, enum psi_res res) + char *buf, enum psi_res res) { struct psi_trigger *t; enum psi_states state; @@ -1108,6 +1203,11 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, else return ERR_PTR(-EINVAL); +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + if (res == PSI_IRQ && --state != PSI_IRQ_FULL) + return ERR_PTR(-EINVAL); +#endif + if (state >= PSI_NONIDLE) return ERR_PTR(-EINVAL); @@ -1320,7 +1420,7 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, nbytes, res); + new = psi_trigger_create(&psi_system, buf, res); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); @@ -1392,6 +1492,33 @@ static const struct proc_ops psi_cpu_proc_ops = { .proc_release = psi_fop_release, }; +#ifdef CONFIG_IRQ_TIME_ACCOUNTING +static int psi_irq_show(struct seq_file *m, void *v) +{ + return psi_show(m, &psi_system, PSI_IRQ); +} + +static int psi_irq_open(struct inode *inode, struct file *file) +{ + return psi_open(file, psi_irq_show); +} + +static ssize_t psi_irq_write(struct file *file, const char __user *user_buf, + size_t nbytes, loff_t *ppos) +{ + return psi_write(file, user_buf, nbytes, PSI_IRQ); +} + +static const struct proc_ops psi_irq_proc_ops = { + .proc_open = psi_irq_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_write = psi_irq_write, + .proc_poll = psi_fop_poll, + .proc_release = psi_fop_release, +}; +#endif + static int __init psi_proc_init(void) { if (psi_enable) { @@ -1399,6 +1526,9 @@ static int __init psi_proc_init(void) proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops); proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops); proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops); +#ifdef CONFIG_IRQ_TIME_ACCOUNTING + proc_create("pressure/irq", 0666, NULL, &psi_irq_proc_ops); +#endif } return 0; } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 55f39c8f42032817f8be1c1304c0585903dfbd5b..d869bcf898ccb768eaec98f36fed5990fbd984e9 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -509,7 +509,7 @@ static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu) unsigned int cpu_cap; /* Only heterogeneous systems can benefit from this check */ - if (!static_branch_unlikely(&sched_asym_cpucapacity)) + if (!sched_asym_cpucap_active()) return true; min_cap = uclamp_eff_value(p, UCLAMP_MIN); @@ -843,7 +843,7 @@ static void __disable_runtime(struct rq *rq) * We cannot be left wanting - that would mean some runtime * leaked out of the system. */ - BUG_ON(want); + WARN_ON_ONCE(want); balanced: /* * Disable all the borrow logic by pretending we have inf @@ -1062,11 +1062,7 @@ static void update_curr_rt(struct rq *rq) trace_sched_stat_runtime(curr, delta_exec, 0); - curr->se.sum_exec_runtime += delta_exec; - account_group_exec_runtime(curr, delta_exec); - - curr->se.exec_start = now; - cgroup_account_cputime(curr, delta_exec); + update_current_exec_runtime(curr, now, delta_exec); if (!rt_bandwidth_enabled()) return; @@ -1849,7 +1845,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p) static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) { - if (!task_running(rq, p) && + if (!task_on_cpu(rq, p) && cpumask_test_cpu(cpu, &p->cpus_mask)) return 1; @@ -1897,7 +1893,7 @@ static int find_lowest_rq(struct task_struct *task) * If we're on asym system ensure we consider the different capacities * of the CPUs when searching for the lowest_mask. */ - if (static_branch_unlikely(&sched_asym_cpucapacity)) { + if (sched_asym_cpucap_active()) { ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri, task, lowest_mask, @@ -2004,7 +2000,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq) */ if (unlikely(task_rq(task) != rq || !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) || - task_running(rq, task) || + task_on_cpu(rq, task) || !rt_task(task) || !task_on_rq_queued(task))) { @@ -2462,7 +2458,7 @@ skip: */ static void task_woken_rt(struct rq *rq, struct task_struct *p) { - bool need_to_push = !task_running(rq, p) && + bool need_to_push = !task_on_cpu(rq, p) && !test_tsk_need_resched(rq->curr) && p->nr_cpus_allowed > 1 && (dl_task(rq->curr) || rt_task(rq->curr)) && diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index e26688d387aebe3588b9175de20f3b646c7fd8e3..1644242ecd11af5f1788ac86d0553a1388716ca4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -321,21 +321,6 @@ struct dl_bw { u64 total_bw; }; -/* - * Verify the fitness of task @p to run on @cpu taking into account the - * CPU original capacity and the runtime/deadline ratio of the task. - * - * The function will return true if the CPU original capacity of the - * @cpu scaled by SCHED_CAPACITY_SCALE >= runtime/deadline ratio of the - * task and false otherwise. - */ -static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) -{ - unsigned long cap = arch_scale_cpu_capacity(cpu); - - return cap_scale(p->dl.dl_deadline, cap) >= p->dl.dl_runtime; -} - extern void init_dl_bw(struct dl_bw *dl_b); extern int sched_dl_global_validate(void); extern void sched_dl_do_global(void); @@ -1815,6 +1800,11 @@ DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); extern struct static_key_false sched_asym_cpucapacity; +static __always_inline bool sched_asym_cpucap_active(void) +{ + return static_branch_unlikely(&sched_asym_cpucapacity); +} + struct sched_group_capacity { atomic_t ref; /* @@ -1942,6 +1932,7 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]); p->se.cfs_rq = tg->cfs_rq[cpu]; p->se.parent = tg->se[cpu]; + p->se.depth = tg->se[cpu] ? tg->se[cpu]->depth + 1 : 0; #endif #ifdef CONFIG_RT_GROUP_SCHED @@ -2060,7 +2051,7 @@ static inline int task_current(struct rq *rq, struct task_struct *p) return rq->curr == p; } -static inline int task_running(struct rq *rq, struct task_struct *p) +static inline int task_on_cpu(struct rq *rq, struct task_struct *p) { #ifdef CONFIG_SMP return p->on_cpu; @@ -2204,11 +2195,8 @@ struct sched_class { void (*update_curr)(struct rq *rq); -#define TASK_SET_GROUP 0 -#define TASK_MOVE_GROUP 1 - #ifdef CONFIG_FAIR_GROUP_SCHED - void (*task_change_group)(struct task_struct *p, int type); + void (*task_change_group)(struct task_struct *p); #endif }; @@ -2435,6 +2423,12 @@ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags); extern void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); +#ifdef CONFIG_PREEMPT_RT +#define SCHED_NR_MIGRATE_BREAK 8 +#else +#define SCHED_NR_MIGRATE_BREAK 32 +#endif + extern const_debug unsigned int sysctl_sched_nr_migrate; extern const_debug unsigned int sysctl_sched_migration_cost; @@ -2452,6 +2446,7 @@ extern unsigned int sysctl_numa_balancing_scan_delay; extern unsigned int sysctl_numa_balancing_scan_period_min; extern unsigned int sysctl_numa_balancing_scan_period_max; extern unsigned int sysctl_numa_balancing_scan_size; +extern unsigned int sysctl_numa_balancing_hot_threshold; #endif #ifdef CONFIG_SCHED_HRTICK @@ -2709,8 +2704,8 @@ static inline void double_rq_lock(struct rq *rq1, struct rq *rq2) __acquires(rq1->lock) __acquires(rq2->lock) { - BUG_ON(!irqs_disabled()); - BUG_ON(rq1 != rq2); + WARN_ON_ONCE(!irqs_disabled()); + WARN_ON_ONCE(rq1 != rq2); raw_spin_rq_lock(rq1); __acquire(rq2->lock); /* Fake it out ;) */ double_rq_clock_clear_update(rq1, rq2); @@ -2726,7 +2721,7 @@ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) __releases(rq1->lock) __releases(rq2->lock) { - BUG_ON(rq1 != rq2); + WARN_ON_ONCE(rq1 != rq2); raw_spin_rq_unlock(rq1); __release(rq2->lock); } @@ -2896,6 +2891,21 @@ unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, enum cpu_util_type type, struct task_struct *p); +/* + * Verify the fitness of task @p to run on @cpu taking into account the + * CPU original capacity and the runtime/deadline ratio of the task. + * + * The function will return true if the original capacity of @cpu is + * greater than or equal to task's deadline density right shifted by + * (BW_SHIFT - SCHED_CAPACITY_SHIFT) and false otherwise. + */ +static inline bool dl_task_fits_capacity(struct task_struct *p, int cpu) +{ + unsigned long cap = arch_scale_cpu_capacity(cpu); + + return cap >= p->dl.dl_density >> (BW_SHIFT - SCHED_CAPACITY_SHIFT); +} + static inline unsigned long cpu_bw_dl(struct rq *rq) { return (rq->dl.running_bw * SCHED_CAPACITY_SCALE) >> BW_SHIFT; @@ -3157,4 +3167,14 @@ extern int sched_dynamic_mode(const char *str); extern void sched_dynamic_update(int mode); #endif +static inline void update_current_exec_runtime(struct task_struct *curr, + u64 now, u64 delta_exec) +{ + curr->se.sum_exec_runtime += delta_exec; + account_group_exec_runtime(curr, delta_exec); + + curr->se.exec_start = now; + cgroup_account_cputime(curr, delta_exec); +} + #endif /* _KERNEL_SCHED_SCHED_H */ diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index baa839c1ba96db271a0267f468e3a4a608c2ee3f..84a188913cc9d082faee858777ef81d321fbf9a9 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -107,6 +107,11 @@ __schedstats_from_se(struct sched_entity *se) } #ifdef CONFIG_PSI +void psi_task_change(struct task_struct *task, int clear, int set); +void psi_task_switch(struct task_struct *prev, struct task_struct *next, + bool sleep); +void psi_account_irqtime(struct task_struct *task, u32 delta); + /* * PSI tracks state that persists across sleeps, such as iowaits and * memory stalls. As a result, it has to distinguish between sleeps, @@ -201,6 +206,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) {} static inline void psi_sched_switch(struct task_struct *prev, struct task_struct *next, bool sleep) {} +static inline void psi_account_irqtime(struct task_struct *task, u32 delta) {} #endif /* CONFIG_PSI */ #ifdef CONFIG_SCHED_INFO diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index d04073a93eb4df2bfac25686a236fc2ca977e962..85590599b4d60545b7774222971f8f7baa678dac 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -71,20 +71,17 @@ static void yield_task_stop(struct rq *rq) static void put_prev_task_stop(struct rq *rq, struct task_struct *prev) { struct task_struct *curr = rq->curr; - u64 delta_exec; + u64 now, delta_exec; - delta_exec = rq_clock_task(rq) - curr->se.exec_start; + now = rq_clock_task(rq); + delta_exec = now - curr->se.exec_start; if (unlikely((s64)delta_exec < 0)) delta_exec = 0; schedstat_set(curr->stats.exec_max, max(curr->stats.exec_max, delta_exec)); - curr->se.sum_exec_runtime += delta_exec; - account_group_exec_runtime(curr, delta_exec); - - curr->se.exec_start = rq_clock_task(rq); - cgroup_account_cputime(curr, delta_exec); + update_current_exec_runtime(curr, now, delta_exec); } /* diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index d4788f810b55511d6c88ca626ffddf4b6c7864a3..0b1cd985dc2749a9db3aadf3dc13bd179a7a450f 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -47,7 +47,7 @@ __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_ prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode); if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) ret = (*action)(&wbq_entry->key, mode); - } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); + } while (test_bit_acquire(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret); finish_wait(wq_head, &wbq_entry->wq_entry); diff --git a/kernel/signal.c b/kernel/signal.c index 6f86fda5e432aeb32ae5dbca2c4264a2781b3dde..d140672185a489bff3a44f759497b5f8e61e433b 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -913,8 +913,9 @@ static bool prepare_signal(int sig, struct task_struct *p, bool force) if (signal->core_state) return sig == SIGKILL; /* - * The process is in the middle of dying, nothing to do. + * The process is in the middle of dying, drop the signal. */ + return false; } else if (sig_kernel_stop(sig)) { /* * This is a stop signal. Remove SIGCONT from all queues. @@ -2304,7 +2305,7 @@ static int ptrace_stop(int exit_code, int why, unsigned long message, read_unlock(&tasklist_lock); cgroup_enter_frozen(); preempt_enable_no_resched(); - freezable_schedule(); + schedule(); cgroup_leave_frozen(true); /* @@ -2473,7 +2474,7 @@ static bool do_signal_stop(int signr) /* Now we don't run again until woken by SIGCONT or SIGKILL */ cgroup_enter_frozen(); - freezable_schedule(); + schedule(); return true; } else { /* @@ -2548,11 +2549,11 @@ static void do_freezer_trap(void) * immediately (if there is a non-fatal signal pending), and * put the task into sleep. */ - __set_current_state(TASK_INTERRUPTIBLE); + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); clear_thread_flag(TIF_SIGPENDING); spin_unlock_irq(¤t->sighand->siglock); cgroup_enter_frozen(); - freezable_schedule(); + schedule(); } static int ptrace_signal(int signr, kernel_siginfo_t *info, enum pid_type type) @@ -3600,9 +3601,9 @@ static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info, recalc_sigpending(); spin_unlock_irq(&tsk->sighand->siglock); - __set_current_state(TASK_INTERRUPTIBLE); - ret = freezable_schedule_hrtimeout_range(to, tsk->timer_slack_ns, - HRTIMER_MODE_REL); + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); + ret = schedule_hrtimeout_range(to, tsk->timer_slack_ns, + HRTIMER_MODE_REL); spin_lock_irq(&tsk->sighand->siglock); __set_task_blocked(tsk, &tsk->real_blocked); sigemptyset(&tsk->real_blocked); diff --git a/kernel/smp.c b/kernel/smp.c index 650810a6f29b3a1ef2a2554b6f9272067715b4c8..06a413987a14acbdcccdc6f4e29b104770f2241a 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -370,8 +370,7 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 * if (cpu >= 0) { if (static_branch_unlikely(&csdlock_debug_extended)) csd_lock_print_extended(csd, cpu); - if (!trigger_single_cpu_backtrace(cpu)) - dump_cpu_task(cpu); + dump_cpu_task(cpu); if (!cpu_cur_csd) { pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu); arch_send_call_function_single_ipi(cpu); @@ -1070,7 +1069,7 @@ static int __init nrcpus(char *str) int nr_cpus; if (get_option(&str, &nr_cpus) && nr_cpus > 0 && nr_cpus < nr_cpu_ids) - nr_cpu_ids = nr_cpus; + set_nr_cpu_ids(nr_cpus); return 0; } @@ -1088,14 +1087,16 @@ static int __init maxcpus(char *str) early_param("maxcpus", maxcpus); +#if (NR_CPUS > 1) && !defined(CONFIG_FORCE_NR_CPUS) /* Setup number of possible processor ids */ unsigned int nr_cpu_ids __read_mostly = NR_CPUS; EXPORT_SYMBOL(nr_cpu_ids); +#endif /* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */ void __init setup_nr_cpu_ids(void) { - nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; + set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1); } /* Called by boot processor to activate the rest. */ diff --git a/kernel/smpboot.c b/kernel/smpboot.c index b9f54544e7499bb068a809436dbc1e1913f1a3ce..2c7396da470c5127fb8329d2d4a50d1a819b60cd 100644 --- a/kernel/smpboot.c +++ b/kernel/smpboot.c @@ -433,7 +433,7 @@ bool cpu_wait_death(unsigned int cpu, int seconds) /* The outgoing CPU will normally get done quite quickly. */ if (atomic_read(&per_cpu(cpu_hotplug_state, cpu)) == CPU_DEAD) - goto update_state; + goto update_state_early; udelay(5); /* But if the outgoing CPU dawdles, wait increasingly long times. */ @@ -444,16 +444,17 @@ bool cpu_wait_death(unsigned int cpu, int seconds) break; sleep_jf = DIV_ROUND_UP(sleep_jf * 11, 10); } -update_state: +update_state_early: oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); +update_state: if (oldstate == CPU_DEAD) { /* Outgoing CPU died normally, update state. */ smp_mb(); /* atomic_read() before update. */ atomic_set(&per_cpu(cpu_hotplug_state, cpu), CPU_POST_DEAD); } else { /* Outgoing CPU still hasn't died, set state accordingly. */ - if (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), - oldstate, CPU_BROKEN) != oldstate) + if (!atomic_try_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), + &oldstate, CPU_BROKEN)) goto update_state; ret = false; } @@ -475,14 +476,14 @@ bool cpu_report_death(void) int newstate; int cpu = smp_processor_id(); + oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); do { - oldstate = atomic_read(&per_cpu(cpu_hotplug_state, cpu)); if (oldstate != CPU_BROKEN) newstate = CPU_DEAD; else newstate = CPU_DEAD_FROZEN; - } while (atomic_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), - oldstate, newstate) != oldstate); + } while (!atomic_try_cmpxchg(&per_cpu(cpu_hotplug_state, cpu), + &oldstate, newstate)); return newstate == CPU_DEAD; } diff --git a/kernel/sys.c b/kernel/sys.c index b911fa6d81ab7a19bc3d40dfb796e361adbbcc8d..5fd54bf0e886726dfb9fb955b0099239aedc9464 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -496,7 +497,7 @@ static void flag_nproc_exceeded(struct cred *new) * for programs doing set*uid()+execve() by harmlessly deferring the * failure to the execve() stage. */ - if (is_ucounts_overlimit(new->ucounts, UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC)) && + if (is_rlimit_overlimit(new->ucounts, UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC)) && new->user != INIT_USER) current->flags |= PF_NPROC_EXCEEDED; else @@ -1366,6 +1367,7 @@ SYSCALL_DEFINE2(sethostname, char __user *, name, int, len) if (!copy_from_user(tmp, name, len)) { struct new_utsname *u; + add_device_randomness(tmp, len); down_write(&uts_sem); u = utsname(); memcpy(u->nodename, tmp, len); @@ -1419,6 +1421,7 @@ SYSCALL_DEFINE2(setdomainname, char __user *, name, int, len) if (!copy_from_user(tmp, name, len)) { struct new_utsname *u; + add_device_randomness(tmp, len); down_write(&uts_sem); u = utsname(); memcpy(u->domainname, tmp, len); diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index a492f159624fa2545dfc43b78337a6166bfece18..860b2dcf3ac465973493a911ecf4fedda79ba049 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -277,6 +277,7 @@ COND_SYSCALL(landlock_restrict_self); /* mm/fadvise.c */ COND_SYSCALL(fadvise64_64); +COND_SYSCALL_COMPAT(fadvise64_64); /* mm/, CONFIG_MMU only */ COND_SYSCALL(swapon); diff --git a/kernel/sysctl-test.c b/kernel/sysctl-test.c index 664ded05dd7a1969df37b42a857fd7e178b828ce..6ef887c19c4887d28c5bc7f80c89abc947bd42d9 100644 --- a/kernel/sysctl-test.c +++ b/kernel/sysctl-test.c @@ -9,9 +9,6 @@ #define KUNIT_PROC_READ 0 #define KUNIT_PROC_WRITE 1 -static int i_zero; -static int i_one_hundred = 100; - /* * Test that proc_dointvec will not try to use a NULL .data field even when the * length is non-zero. @@ -29,8 +26,8 @@ static void sysctl_test_api_dointvec_null_tbl_data(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; /* * proc_dointvec expects a buffer in user space, so we allocate one. We @@ -79,8 +76,8 @@ static void sysctl_test_api_dointvec_table_maxlen_unset(struct kunit *test) .maxlen = 0, .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int), GFP_USER); @@ -122,8 +119,8 @@ static void sysctl_test_api_dointvec_table_len_is_zero(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int), GFP_USER); @@ -156,8 +153,8 @@ static void sysctl_test_api_dointvec_table_read_but_position_set( .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; void __user *buffer = (void __user *)kunit_kzalloc(test, sizeof(int), GFP_USER); @@ -191,8 +188,8 @@ static void sysctl_test_dointvec_read_happy_single_positive(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; size_t len = 4; loff_t pos = 0; @@ -222,8 +219,8 @@ static void sysctl_test_dointvec_read_happy_single_negative(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; size_t len = 5; loff_t pos = 0; @@ -251,8 +248,8 @@ static void sysctl_test_dointvec_write_happy_single_positive(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; char input[] = "9"; size_t len = sizeof(input) - 1; @@ -281,8 +278,8 @@ static void sysctl_test_dointvec_write_happy_single_negative(struct kunit *test) .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; char input[] = "-9"; size_t len = sizeof(input) - 1; @@ -313,8 +310,8 @@ static void sysctl_test_api_dointvec_write_single_less_int_min( .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; size_t max_len = 32, len = max_len; loff_t pos = 0; @@ -351,8 +348,8 @@ static void sysctl_test_api_dointvec_write_single_greater_int_max( .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec, - .extra1 = &i_zero, - .extra2 = &i_one_hundred, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, }; size_t max_len = 32, len = max_len; loff_t pos = 0; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 205d605cacc5bb24fd7967168d220c038ec515e8..188c305aeb8b7fd8984721122dcfe11bd658aeaa 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -82,9 +82,16 @@ #include #endif +/* shared constants to be used in various sysctls */ +const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; +EXPORT_SYMBOL(sysctl_vals); + +const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; +EXPORT_SYMBOL_GPL(sysctl_long_vals); + #if defined(CONFIG_SYSCTL) -/* Constants used for minimum and maximum */ +/* Constants used for minimum and maximum */ #ifdef CONFIG_PERF_EVENTS static const int six_hundred_forty_kb = 640 * 1024; @@ -129,11 +136,6 @@ static enum sysctl_writes_mode sysctl_writes_strict = SYSCTL_WRITES_STRICT; int sysctl_legacy_va_layout; #endif -#ifdef CONFIG_COMPACTION -/* min_extfrag_threshold is SYSCTL_ZERO */; -static const int max_extfrag_threshold = 1000; -#endif - #endif /* CONFIG_SYSCTL */ /* @@ -1052,9 +1054,9 @@ static int __do_proc_doulongvec_minmax(void *data, struct ctl_table *table, return 0; } - i = (unsigned long *) data; - min = (unsigned long *) table->extra1; - max = (unsigned long *) table->extra2; + i = data; + min = table->extra1; + max = table->extra2; vleft = table->maxlen / sizeof(unsigned long); left = *lenp; @@ -1641,6 +1643,14 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_FOUR, }, + { + .procname = "numa_balancing_promote_rate_limit_MBps", + .data = &sysctl_numa_balancing_promote_rate_limit, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, #endif /* CONFIG_NUMA_BALANCING */ { .procname = "panic", @@ -2216,7 +2226,7 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = (void *)&max_extfrag_threshold, + .extra2 = SYSCTL_ONE_THOUSAND, }, { .procname = "compact_unevictable_allowed", diff --git a/kernel/task_work.c b/kernel/task_work.c index dff75bcde1514c82fc6580bcf407cd2ff7ae03cb..065e1ef8fc8d72e55c66000b7b25f9ba0189db45 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -47,12 +47,12 @@ int task_work_add(struct task_struct *task, struct callback_head *work, /* record the work call stack in order to print it in KASAN reports */ kasan_record_aux_stack(work); + head = READ_ONCE(task->task_works); do { - head = READ_ONCE(task->task_works); if (unlikely(head == &work_exited)) return -ESRCH; work->next = head; - } while (cmpxchg(&task->task_works, head, work) != head); + } while (!try_cmpxchg(&task->task_works, &head, work)); switch (notify) { case TWA_NONE: @@ -100,10 +100,12 @@ task_work_cancel_match(struct task_struct *task, * we raced with task_work_run(), *pprev == NULL/exited. */ raw_spin_lock_irqsave(&task->pi_lock, flags); - while ((work = READ_ONCE(*pprev))) { - if (!match(work, data)) + work = READ_ONCE(*pprev); + while (work) { + if (!match(work, data)) { pprev = &work->next; - else if (cmpxchg(pprev, work, work->next) == work) + work = READ_ONCE(*pprev); + } else if (try_cmpxchg(pprev, &work, work->next)) break; } raw_spin_unlock_irqrestore(&task->pi_lock, flags); @@ -151,16 +153,16 @@ void task_work_run(void) * work->func() can do task_work_add(), do not set * work_exited unless the list is empty. */ + work = READ_ONCE(task->task_works); do { head = NULL; - work = READ_ONCE(task->task_works); if (!work) { if (task->flags & PF_EXITING) head = &work_exited; else break; } - } while (cmpxchg(&task->task_works, work, head) != work); + } while (!try_cmpxchg(&task->task_works, &work, head)); if (!work) break; diff --git a/kernel/taskstats.c b/kernel/taskstats.c index f7e246336218e559c0e5fa0a9d37cc3778fe40b7..8ce3fa0c19e2d7ae7457469da00d184d5834aadc 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -688,6 +688,7 @@ static struct genl_family family __ro_after_init = { .module = THIS_MODULE, .ops = taskstats_ops, .n_ops = ARRAY_SIZE(taskstats_ops), + .resv_start_op = CGROUPSTATS_CMD_GET + 1, .netnsok = true, }; diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index cee5da1e54c4121d771bf5fa19c07584e000bf49..8058bec87acee919514d13a98be07b37b423b4f4 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -310,7 +310,7 @@ static void clocksource_verify_choose_cpus(void) * CPUs that are currently online. */ for (i = 1; i < n; i++) { - cpu = prandom_u32() % nr_cpu_ids; + cpu = prandom_u32_max(nr_cpu_ids); cpu = cpumask_next(cpu - 1, cpu_online_mask); if (cpu >= nr_cpu_ids) cpu = cpumask_first(cpu_online_mask); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 23af5eca11b14c81eaba6e5c49f791806f110c2e..3ae661ab62603c4f6ad6a162e4d3586346ead934 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2037,11 +2037,11 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod struct restart_block *restart; do { - set_current_state(TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); hrtimer_sleeper_start_expires(t, mode); if (likely(t->task)) - freezable_schedule(); + schedule(); hrtimer_cancel(&t->timer); mode = HRTIMER_MODE_ABS; diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 1052126bdca22bf92bf0333d6fc63aa26900c00d..e9e95c790b8ee769d12327e6328ebf9264bc4895 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -51,6 +51,12 @@ config HAVE_DYNAMIC_FTRACE_WITH_ARGS This allows for use of regs_get_kernel_argument() and kernel_stack_pointer(). +config HAVE_DYNAMIC_FTRACE_NO_PATCHABLE + bool + help + If the architecture generates __patchable_function_entries sections + but does not want them included in the ftrace locations. + config HAVE_FTRACE_MCOUNT_RECORD bool help diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 68e5cdd24cef883f7f87fb76e80af40a0a87fd97..49fb9ec8366de4601467086bcb0c3ce723cc1459 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include @@ -1026,11 +1028,30 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = { .arg1_type = ARG_PTR_TO_CTX, }; +#ifdef CONFIG_X86_KERNEL_IBT +static unsigned long get_entry_ip(unsigned long fentry_ip) +{ + u32 instr; + + /* Being extra safe in here in case entry ip is on the page-edge. */ + if (get_kernel_nofault(instr, (u32 *) fentry_ip - 1)) + return fentry_ip; + if (is_endbr(instr)) + fentry_ip -= ENDBR_INSN_SIZE; + return fentry_ip; +} +#else +#define get_entry_ip(fentry_ip) fentry_ip +#endif + BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs) { struct kprobe *kp = kprobe_running(); - return kp ? (uintptr_t)kp->addr : 0; + if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY)) + return 0; + + return get_entry_ip((uintptr_t)kp->addr); } static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = { @@ -1181,6 +1202,184 @@ static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = { .arg1_type = ARG_PTR_TO_CTX, }; +#ifdef CONFIG_KEYS +__diag_push(); +__diag_ignore_all("-Wmissing-prototypes", + "kfuncs which will be used in BPF programs"); + +/** + * bpf_lookup_user_key - lookup a key by its serial + * @serial: key handle serial number + * @flags: lookup-specific flags + * + * Search a key with a given *serial* and the provided *flags*. + * If found, increment the reference count of the key by one, and + * return it in the bpf_key structure. + * + * The bpf_key structure must be passed to bpf_key_put() when done + * with it, so that the key reference count is decremented and the + * bpf_key structure is freed. + * + * Permission checks are deferred to the time the key is used by + * one of the available key-specific kfuncs. + * + * Set *flags* with KEY_LOOKUP_CREATE, to attempt creating a requested + * special keyring (e.g. session keyring), if it doesn't yet exist. + * Set *flags* with KEY_LOOKUP_PARTIAL, to lookup a key without waiting + * for the key construction, and to retrieve uninstantiated keys (keys + * without data attached to them). + * + * Return: a bpf_key pointer with a valid key pointer if the key is found, a + * NULL pointer otherwise. + */ +struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags) +{ + key_ref_t key_ref; + struct bpf_key *bkey; + + if (flags & ~KEY_LOOKUP_ALL) + return NULL; + + /* + * Permission check is deferred until the key is used, as the + * intent of the caller is unknown here. + */ + key_ref = lookup_user_key(serial, flags, KEY_DEFER_PERM_CHECK); + if (IS_ERR(key_ref)) + return NULL; + + bkey = kmalloc(sizeof(*bkey), GFP_KERNEL); + if (!bkey) { + key_put(key_ref_to_ptr(key_ref)); + return NULL; + } + + bkey->key = key_ref_to_ptr(key_ref); + bkey->has_ref = true; + + return bkey; +} + +/** + * bpf_lookup_system_key - lookup a key by a system-defined ID + * @id: key ID + * + * Obtain a bpf_key structure with a key pointer set to the passed key ID. + * The key pointer is marked as invalid, to prevent bpf_key_put() from + * attempting to decrement the key reference count on that pointer. The key + * pointer set in such way is currently understood only by + * verify_pkcs7_signature(). + * + * Set *id* to one of the values defined in include/linux/verification.h: + * 0 for the primary keyring (immutable keyring of system keys); + * VERIFY_USE_SECONDARY_KEYRING for both the primary and secondary keyring + * (where keys can be added only if they are vouched for by existing keys + * in those keyrings); VERIFY_USE_PLATFORM_KEYRING for the platform + * keyring (primarily used by the integrity subsystem to verify a kexec'ed + * kerned image and, possibly, the initramfs signature). + * + * Return: a bpf_key pointer with an invalid key pointer set from the + * pre-determined ID on success, a NULL pointer otherwise + */ +struct bpf_key *bpf_lookup_system_key(u64 id) +{ + struct bpf_key *bkey; + + if (system_keyring_id_check(id) < 0) + return NULL; + + bkey = kmalloc(sizeof(*bkey), GFP_ATOMIC); + if (!bkey) + return NULL; + + bkey->key = (struct key *)(unsigned long)id; + bkey->has_ref = false; + + return bkey; +} + +/** + * bpf_key_put - decrement key reference count if key is valid and free bpf_key + * @bkey: bpf_key structure + * + * Decrement the reference count of the key inside *bkey*, if the pointer + * is valid, and free *bkey*. + */ +void bpf_key_put(struct bpf_key *bkey) +{ + if (bkey->has_ref) + key_put(bkey->key); + + kfree(bkey); +} + +#ifdef CONFIG_SYSTEM_DATA_VERIFICATION +/** + * bpf_verify_pkcs7_signature - verify a PKCS#7 signature + * @data_ptr: data to verify + * @sig_ptr: signature of the data + * @trusted_keyring: keyring with keys trusted for signature verification + * + * Verify the PKCS#7 signature *sig_ptr* against the supplied *data_ptr* + * with keys in a keyring referenced by *trusted_keyring*. + * + * Return: 0 on success, a negative value on error. + */ +int bpf_verify_pkcs7_signature(struct bpf_dynptr_kern *data_ptr, + struct bpf_dynptr_kern *sig_ptr, + struct bpf_key *trusted_keyring) +{ + int ret; + + if (trusted_keyring->has_ref) { + /* + * Do the permission check deferred in bpf_lookup_user_key(). + * See bpf_lookup_user_key() for more details. + * + * A call to key_task_permission() here would be redundant, as + * it is already done by keyring_search() called by + * find_asymmetric_key(). + */ + ret = key_validate(trusted_keyring->key); + if (ret < 0) + return ret; + } + + return verify_pkcs7_signature(data_ptr->data, + bpf_dynptr_get_size(data_ptr), + sig_ptr->data, + bpf_dynptr_get_size(sig_ptr), + trusted_keyring->key, + VERIFYING_UNSPECIFIED_SIGNATURE, NULL, + NULL); +} +#endif /* CONFIG_SYSTEM_DATA_VERIFICATION */ + +__diag_pop(); + +BTF_SET8_START(key_sig_kfunc_set) +BTF_ID_FLAGS(func, bpf_lookup_user_key, KF_ACQUIRE | KF_RET_NULL | KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_lookup_system_key, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_key_put, KF_RELEASE) +#ifdef CONFIG_SYSTEM_DATA_VERIFICATION +BTF_ID_FLAGS(func, bpf_verify_pkcs7_signature, KF_SLEEPABLE) +#endif +BTF_SET8_END(key_sig_kfunc_set) + +static const struct btf_kfunc_id_set bpf_key_sig_kfunc_set = { + .owner = THIS_MODULE, + .set = &key_sig_kfunc_set, +}; + +static int __init bpf_key_sig_kfuncs_init(void) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, + &bpf_key_sig_kfunc_set); +} + +late_initcall(bpf_key_sig_kfuncs_init); +#endif /* CONFIG_KEYS */ + static const struct bpf_func_proto * bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -1507,6 +1706,9 @@ BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) return -EINVAL; + if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK))) + return -ENOENT; + if (unlikely(!br_stack)) return -ENOENT; @@ -2042,9 +2244,15 @@ static __always_inline void __bpf_trace_run(struct bpf_prog *prog, u64 *args) { cant_sleep(); + if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { + bpf_prog_inc_misses_counter(prog); + goto out; + } rcu_read_lock(); (void) bpf_prog_run(prog, args); rcu_read_unlock(); +out: + this_cpu_dec(*(prog->active)); } #define UNPACK(...) __VA_ARGS__ @@ -2414,13 +2622,13 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, } static void -kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip, +kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip, struct pt_regs *regs) { struct bpf_kprobe_multi_link *link; link = container_of(fp, struct bpf_kprobe_multi_link, fp); - kprobe_multi_link_prog_run(link, entry_ip, regs); + kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs); } static int symbols_cmp_r(const void *a, const void *b, const void *priv) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 126c769d36c3cf48cf46bb495b96cf834662b5cc..fbf2543111c05c2ffcb2be47b8497e723b6724ff 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -1644,6 +1644,18 @@ ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_ex static struct ftrace_ops * ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); +static bool skip_record(struct dyn_ftrace *rec) +{ + /* + * At boot up, weak functions are set to disable. Function tracing + * can be enabled before they are, and they still need to be disabled now. + * If the record is disabled, still continue if it is marked as already + * enabled (this is needed to keep the accounting working). + */ + return rec->flags & FTRACE_FL_DISABLED && + !(rec->flags & FTRACE_FL_ENABLED); +} + static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, int filter_hash, bool inc) @@ -1693,7 +1705,7 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, int in_hash = 0; int match = 0; - if (rec->flags & FTRACE_FL_DISABLED) + if (skip_record(rec)) continue; if (all) { @@ -1861,8 +1873,6 @@ static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, ftrace_hash_rec_update_modify(ops, filter_hash, 1); } -static bool ops_references_ip(struct ftrace_ops *ops, unsigned long ip); - /* * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK * or no-needed to update, -EBUSY if it detects a conflict of the flag @@ -2018,7 +2028,6 @@ static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, static void print_ip_ins(const char *fmt, const unsigned char *p) { char ins[MCOUNT_INSN_SIZE]; - int i; if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) { printk(KERN_CONT "%s[FAULT] %px\n", fmt, p); @@ -2026,9 +2035,7 @@ static void print_ip_ins(const char *fmt, const unsigned char *p) } printk(KERN_CONT "%s", fmt); - - for (i = 0; i < MCOUNT_INSN_SIZE; i++) - printk(KERN_CONT "%s%02x", i ? ":" : "", ins[i]); + pr_cont("%*phC", MCOUNT_INSN_SIZE, ins); } enum ftrace_bug_type ftrace_bug_type; @@ -2128,7 +2135,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) ftrace_bug_type = FTRACE_BUG_UNKNOWN; - if (rec->flags & FTRACE_FL_DISABLED) + if (skip_record(rec)) return FTRACE_UPDATE_IGNORE; /* @@ -2243,7 +2250,7 @@ static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) if (update) { /* If there's no more users, clear all flags */ if (!ftrace_rec_count(rec)) - rec->flags = 0; + rec->flags &= FTRACE_FL_DISABLED; else /* * Just disable the record, but keep the ops TRAMP @@ -2636,7 +2643,7 @@ void __weak ftrace_replace_code(int mod_flags) do_for_each_ftrace_rec(pg, rec) { - if (rec->flags & FTRACE_FL_DISABLED) + if (skip_record(rec)) continue; failed = __ftrace_replace_code(rec, enable); @@ -3118,49 +3125,6 @@ static inline int ops_traces_mod(struct ftrace_ops *ops) ftrace_hash_empty(ops->func_hash->notrace_hash); } -/* - * Check if the current ops references the given ip. - * - * If the ops traces all functions, then it was already accounted for. - * If the ops does not trace the current record function, skip it. - * If the ops ignores the function via notrace filter, skip it. - */ -static bool -ops_references_ip(struct ftrace_ops *ops, unsigned long ip) -{ - /* If ops isn't enabled, ignore it */ - if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) - return false; - - /* If ops traces all then it includes this function */ - if (ops_traces_mod(ops)) - return true; - - /* The function must be in the filter */ - if (!ftrace_hash_empty(ops->func_hash->filter_hash) && - !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) - return false; - - /* If in notrace hash, we ignore it too */ - if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) - return false; - - return true; -} - -/* - * Check if the current ops references the record. - * - * If the ops traces all functions, then it was already accounted for. - * If the ops does not trace the current record function, skip it. - * If the ops ignores the function via notrace filter, skip it. - */ -static bool -ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec) -{ - return ops_references_ip(ops, rec->ip); -} - static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) { bool init_nop = ftrace_need_init_nop(); @@ -5472,6 +5436,8 @@ static struct ftrace_ops stub_ops = { * it is safe to modify the ftrace record, where it should be * currently calling @old_addr directly, to call @new_addr. * + * This is called with direct_mutex locked. + * * Safety checks should be made to make sure that the code at * @rec->ip is currently calling @old_addr. And this must * also update entry->direct to @new_addr. @@ -5484,6 +5450,8 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, unsigned long ip = rec->ip; int ret; + lockdep_assert_held(&direct_mutex); + /* * The ftrace_lock was used to determine if the record * had more than one registered user to it. If it did, @@ -5506,7 +5474,7 @@ int __weak ftrace_modify_direct_caller(struct ftrace_func_entry *entry, if (ret) goto out_lock; - ret = register_ftrace_function(&stub_ops); + ret = register_ftrace_function_nolock(&stub_ops); if (ret) { ftrace_set_filter_ip(&stub_ops, ip, 1, 0); goto out_lock; @@ -6126,8 +6094,12 @@ int ftrace_regex_release(struct inode *inode, struct file *file) if (filter_hash) { orig_hash = &iter->ops->func_hash->filter_hash; - if (iter->tr && !list_empty(&iter->tr->mod_trace)) - iter->hash->flags |= FTRACE_HASH_FL_MOD; + if (iter->tr) { + if (list_empty(&iter->tr->mod_trace)) + iter->hash->flags &= ~FTRACE_HASH_FL_MOD; + else + iter->hash->flags |= FTRACE_HASH_FL_MOD; + } } else orig_hash = &iter->ops->func_hash->notrace_hash; @@ -6822,6 +6794,38 @@ static int ftrace_get_trampoline_kallsym(unsigned int symnum, return -ERANGE; } +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES) +/* + * Check if the current ops references the given ip. + * + * If the ops traces all functions, then it was already accounted for. + * If the ops does not trace the current record function, skip it. + * If the ops ignores the function via notrace filter, skip it. + */ +static bool +ops_references_ip(struct ftrace_ops *ops, unsigned long ip) +{ + /* If ops isn't enabled, ignore it */ + if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) + return false; + + /* If ops traces all then it includes this function */ + if (ops_traces_mod(ops)) + return true; + + /* The function must be in the filter */ + if (!ftrace_hash_empty(ops->func_hash->filter_hash) && + !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) + return false; + + /* If in notrace hash, we ignore it too */ + if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) + return false; + + return true; +} +#endif + #ifdef CONFIG_MODULES #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) @@ -6834,7 +6838,7 @@ static int referenced_filters(struct dyn_ftrace *rec) int cnt = 0; for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { - if (ops_references_rec(ops, rec)) { + if (ops_references_ip(ops, rec->ip)) { if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) continue; if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) @@ -8278,8 +8282,7 @@ static int kallsyms_callback(void *data, const char *name, if (args->addrs[idx]) return 0; - addr = ftrace_location(addr); - if (!addr) + if (!ftrace_location(addr)) return 0; args->addrs[idx] = addr; diff --git a/kernel/trace/kprobe_event_gen_test.c b/kernel/trace/kprobe_event_gen_test.c index 18b0f1cbb947f05fc3add1744c25737970a215b3..80e04a1e19772ae7cd4cecdc544f9ebaa1fd0e6b 100644 --- a/kernel/trace/kprobe_event_gen_test.c +++ b/kernel/trace/kprobe_event_gen_test.c @@ -35,6 +35,45 @@ static struct trace_event_file *gen_kprobe_test; static struct trace_event_file *gen_kretprobe_test; +#define KPROBE_GEN_TEST_FUNC "do_sys_open" + +/* X86 */ +#if defined(CONFIG_X86_64) || defined(CONFIG_X86_32) +#define KPROBE_GEN_TEST_ARG0 "dfd=%ax" +#define KPROBE_GEN_TEST_ARG1 "filename=%dx" +#define KPROBE_GEN_TEST_ARG2 "flags=%cx" +#define KPROBE_GEN_TEST_ARG3 "mode=+4($stack)" + +/* ARM64 */ +#elif defined(CONFIG_ARM64) +#define KPROBE_GEN_TEST_ARG0 "dfd=%x0" +#define KPROBE_GEN_TEST_ARG1 "filename=%x1" +#define KPROBE_GEN_TEST_ARG2 "flags=%x2" +#define KPROBE_GEN_TEST_ARG3 "mode=%x3" + +/* ARM */ +#elif defined(CONFIG_ARM) +#define KPROBE_GEN_TEST_ARG0 "dfd=%r0" +#define KPROBE_GEN_TEST_ARG1 "filename=%r1" +#define KPROBE_GEN_TEST_ARG2 "flags=%r2" +#define KPROBE_GEN_TEST_ARG3 "mode=%r3" + +/* RISCV */ +#elif defined(CONFIG_RISCV) +#define KPROBE_GEN_TEST_ARG0 "dfd=%a0" +#define KPROBE_GEN_TEST_ARG1 "filename=%a1" +#define KPROBE_GEN_TEST_ARG2 "flags=%a2" +#define KPROBE_GEN_TEST_ARG3 "mode=%a3" + +/* others */ +#else +#define KPROBE_GEN_TEST_ARG0 NULL +#define KPROBE_GEN_TEST_ARG1 NULL +#define KPROBE_GEN_TEST_ARG2 NULL +#define KPROBE_GEN_TEST_ARG3 NULL +#endif + + /* * Test to make sure we can create a kprobe event, then add more * fields. @@ -58,14 +97,14 @@ static int __init test_gen_kprobe_cmd(void) * fields. */ ret = kprobe_event_gen_cmd_start(&cmd, "gen_kprobe_test", - "do_sys_open", - "dfd=%ax", "filename=%dx"); + KPROBE_GEN_TEST_FUNC, + KPROBE_GEN_TEST_ARG0, KPROBE_GEN_TEST_ARG1); if (ret) goto free; /* Use kprobe_event_add_fields to add the rest of the fields */ - ret = kprobe_event_add_fields(&cmd, "flags=%cx", "mode=+4($stack)"); + ret = kprobe_event_add_fields(&cmd, KPROBE_GEN_TEST_ARG2, KPROBE_GEN_TEST_ARG3); if (ret) goto free; @@ -128,7 +167,7 @@ static int __init test_gen_kretprobe_cmd(void) * Define the kretprobe event. */ ret = kretprobe_event_gen_cmd_start(&cmd, "gen_kretprobe_test", - "do_sys_open", + KPROBE_GEN_TEST_FUNC, "$retval"); if (ret) goto free; @@ -206,7 +245,7 @@ static void __exit kprobe_event_gen_test_exit(void) WARN_ON(kprobe_event_delete("gen_kprobe_test")); /* Disable the event or you can't remove it */ - WARN_ON(trace_array_set_clr_event(gen_kprobe_test->tr, + WARN_ON(trace_array_set_clr_event(gen_kretprobe_test->tr, "kprobes", "gen_kretprobe_test", false)); diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index d59b6a328b7feca1fa9cb947ed7c7e5d07cd22e2..199759c735196a03086d1a6b3a31d1c5d358f7a6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -413,6 +413,7 @@ struct rb_irq_work { struct irq_work work; wait_queue_head_t waiters; wait_queue_head_t full_waiters; + long wait_index; bool waiters_pending; bool full_waiters_pending; bool wakeup_full; @@ -884,7 +885,7 @@ size_t ring_buffer_nr_pages(struct trace_buffer *buffer, int cpu) } /** - * ring_buffer_nr_pages_dirty - get the number of used pages in the ring buffer + * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer * @buffer: The ring_buffer to get the number of pages from * @cpu: The cpu of the ring_buffer to get the number of pages from * @@ -917,12 +918,44 @@ static void rb_wake_up_waiters(struct irq_work *work) struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work); wake_up_all(&rbwork->waiters); - if (rbwork->wakeup_full) { + if (rbwork->full_waiters_pending || rbwork->wakeup_full) { rbwork->wakeup_full = false; + rbwork->full_waiters_pending = false; wake_up_all(&rbwork->full_waiters); } } +/** + * ring_buffer_wake_waiters - wake up any waiters on this ring buffer + * @buffer: The ring buffer to wake waiters on + * + * In the case of a file that represents a ring buffer is closing, + * it is prudent to wake up any waiters that are on this. + */ +void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) +{ + struct ring_buffer_per_cpu *cpu_buffer; + struct rb_irq_work *rbwork; + + if (cpu == RING_BUFFER_ALL_CPUS) { + + /* Wake up individual ones too. One level recursion */ + for_each_buffer_cpu(buffer, cpu) + ring_buffer_wake_waiters(buffer, cpu); + + rbwork = &buffer->irq_work; + } else { + cpu_buffer = buffer->buffers[cpu]; + rbwork = &cpu_buffer->irq_work; + } + + rbwork->wait_index++; + /* make sure the waiters see the new index */ + smp_wmb(); + + rb_wake_up_waiters(&rbwork->work); +} + /** * ring_buffer_wait - wait for input to the ring buffer * @buffer: buffer to wait on @@ -938,6 +971,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) struct ring_buffer_per_cpu *cpu_buffer; DEFINE_WAIT(wait); struct rb_irq_work *work; + long wait_index; int ret = 0; /* @@ -956,6 +990,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) work = &cpu_buffer->irq_work; } + wait_index = READ_ONCE(work->wait_index); while (true) { if (full) @@ -1011,7 +1046,7 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) nr_pages = cpu_buffer->nr_pages; dirty = ring_buffer_nr_dirty_pages(buffer, cpu); if (!cpu_buffer->shortest_full || - cpu_buffer->shortest_full < full) + cpu_buffer->shortest_full > full) cpu_buffer->shortest_full = full; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); if (!pagebusy && @@ -1020,6 +1055,11 @@ int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full) } schedule(); + + /* Make sure to see the new wait index */ + smp_rmb(); + if (wait_index != work->wait_index) + break; } if (full) @@ -2608,6 +2648,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, /* Mark the rest of the page with padding */ rb_event_set_padding(event); + /* Make sure the padding is visible before the write update */ + smp_wmb(); + /* Set the write back to the previous setting */ local_sub(length, &tail_page->write); return; @@ -2619,6 +2662,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, /* time delta must be non zero */ event->time_delta = 1; + /* Make sure the padding is visible before the tail_page->write update */ + smp_wmb(); + /* Set write to end of buffer */ length = (tail + length) - BUF_PAGE_SIZE; local_sub(length, &tail_page->write); @@ -4587,6 +4633,33 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); + /* + * The writer has preempt disable, wait for it. But not forever + * Although, 1 second is pretty much "forever" + */ +#define USECS_WAIT 1000000 + for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { + /* If the write is past the end of page, a writer is still updating it */ + if (likely(!reader || rb_page_write(reader) <= BUF_PAGE_SIZE)) + break; + + udelay(1); + + /* Get the latest version of the reader write value */ + smp_rmb(); + } + + /* The writer is not moving forward? Something is wrong */ + if (RB_WARN_ON(cpu_buffer, nr_loops == USECS_WAIT)) + reader = NULL; + + /* + * Make sure we see any padding after the write update + * (see rb_reset_tail()) + */ + smp_rmb(); + + return reader; } @@ -5232,7 +5305,7 @@ void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu) EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); /** - * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer + * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer * @buffer: The ring buffer to reset a per cpu buffer of * @cpu: The CPU buffer to be reset */ @@ -5302,7 +5375,7 @@ void ring_buffer_reset(struct trace_buffer *buffer) EXPORT_SYMBOL_GPL(ring_buffer_reset); /** - * rind_buffer_empty - is the ring buffer empty? + * ring_buffer_empty - is the ring buffer empty? * @buffer: The ring buffer to test */ bool ring_buffer_empty(struct trace_buffer *buffer) @@ -5616,7 +5689,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, unsigned int pos = 0; unsigned int size; - if (full) + /* + * If a full page is expected, this can still be returned + * if there's been a previous partial read and the + * rest of the page can be read and the commit page is off + * the reader page. + */ + if (full && + (!read || (len < (commit - read)) || + cpu_buffer->reader_page == cpu_buffer->commit_page)) goto out_unlock; if (len > (commit - read)) diff --git a/kernel/trace/rv/monitors/wip/wip.c b/kernel/trace/rv/monitors/wip/wip.c index 83cace53b9fafecbec80c3bd0ba602fca4515231..b2b49a27e8863bd7558b886ca91cf2394946f15f 100644 --- a/kernel/trace/rv/monitors/wip/wip.c +++ b/kernel/trace/rv/monitors/wip/wip.c @@ -16,7 +16,7 @@ #include "wip.h" -struct rv_monitor rv_wip; +static struct rv_monitor rv_wip; DECLARE_DA_MON_PER_CPU(wip, unsigned char); static void handle_preempt_disable(void *data, unsigned long ip, unsigned long parent_ip) @@ -60,7 +60,7 @@ static void disable_wip(void) da_monitor_destroy_wip(); } -struct rv_monitor rv_wip = { +static struct rv_monitor rv_wip = { .name = "wip", .description = "wakeup in preemptive per-cpu testing monitor.", .enable = enable_wip, @@ -69,13 +69,13 @@ struct rv_monitor rv_wip = { .enabled = 0, }; -static int register_wip(void) +static int __init register_wip(void) { rv_register_monitor(&rv_wip); return 0; } -static void unregister_wip(void) +static void __exit unregister_wip(void) { rv_unregister_monitor(&rv_wip); } diff --git a/kernel/trace/rv/monitors/wip/wip.h b/kernel/trace/rv/monitors/wip/wip.h index c1c47e2305efa2e6c43642536ff99fcebc9cb30e..dacc37b62a2c5d675fef631a92f626189d5e95d9 100644 --- a/kernel/trace/rv/monitors/wip/wip.h +++ b/kernel/trace/rv/monitors/wip/wip.h @@ -27,7 +27,7 @@ struct automaton_wip { bool final_states[state_max_wip]; }; -struct automaton_wip automaton_wip = { +static struct automaton_wip automaton_wip = { .state_names = { "preemptive", "non_preemptive" diff --git a/kernel/trace/rv/monitors/wwnr/wwnr.c b/kernel/trace/rv/monitors/wwnr/wwnr.c index 599225d9cf38242084a20789e880ae69035dbf58..0e43dd2db685d0ae99606e1758d7b390cf8824d8 100644 --- a/kernel/trace/rv/monitors/wwnr/wwnr.c +++ b/kernel/trace/rv/monitors/wwnr/wwnr.c @@ -15,7 +15,7 @@ #include "wwnr.h" -struct rv_monitor rv_wwnr; +static struct rv_monitor rv_wwnr; DECLARE_DA_MON_PER_TASK(wwnr, unsigned char); static void handle_switch(void *data, bool preempt, struct task_struct *p, @@ -59,7 +59,7 @@ static void disable_wwnr(void) da_monitor_destroy_wwnr(); } -struct rv_monitor rv_wwnr = { +static struct rv_monitor rv_wwnr = { .name = "wwnr", .description = "wakeup while not running per-task testing model.", .enable = enable_wwnr, @@ -68,13 +68,13 @@ struct rv_monitor rv_wwnr = { .enabled = 0, }; -static int register_wwnr(void) +static int __init register_wwnr(void) { rv_register_monitor(&rv_wwnr); return 0; } -static void unregister_wwnr(void) +static void __exit unregister_wwnr(void) { rv_unregister_monitor(&rv_wwnr); } diff --git a/kernel/trace/rv/monitors/wwnr/wwnr.h b/kernel/trace/rv/monitors/wwnr/wwnr.h index d1afe55cdd4c11acb152485a6de1fb03ec8891b6..118e576b91b4b1414d39522eb0d8307032379e67 100644 --- a/kernel/trace/rv/monitors/wwnr/wwnr.h +++ b/kernel/trace/rv/monitors/wwnr/wwnr.h @@ -27,7 +27,7 @@ struct automaton_wwnr { bool final_states[state_max_wwnr]; }; -struct automaton_wwnr automaton_wwnr = { +static struct automaton_wwnr automaton_wwnr = { .state_names = { "not_running", "running" diff --git a/kernel/trace/rv/reactor_panic.c b/kernel/trace/rv/reactor_panic.c index b698d05dd06955e883aee394837dd54d2781c5e9..d65f6c25a87cdd31a9079e3217103c72207f1fe7 100644 --- a/kernel/trace/rv/reactor_panic.c +++ b/kernel/trace/rv/reactor_panic.c @@ -24,13 +24,13 @@ static struct rv_reactor rv_panic = { .react = rv_panic_reaction }; -static int register_react_panic(void) +static int __init register_react_panic(void) { rv_register_reactor(&rv_panic); return 0; } -static void unregister_react_panic(void) +static void __exit unregister_react_panic(void) { rv_unregister_reactor(&rv_panic); } diff --git a/kernel/trace/rv/reactor_printk.c b/kernel/trace/rv/reactor_printk.c index 31899f953af4e0f9b7baec97ababa8dbd995e36b..4b6b7106a477c2c105823ed798a05a006513e1ae 100644 --- a/kernel/trace/rv/reactor_printk.c +++ b/kernel/trace/rv/reactor_printk.c @@ -23,13 +23,13 @@ static struct rv_reactor rv_printk = { .react = rv_printk_reaction }; -static int register_react_printk(void) +static int __init register_react_printk(void) { rv_register_reactor(&rv_printk); return 0; } -static void unregister_react_printk(void) +static void __exit unregister_react_printk(void) { rv_unregister_reactor(&rv_printk); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index d3005279165d9c9c2277f2b391bc72ec3fd56115..47a44b055a1d4821e1f738ebe56b4240ae30e157 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1193,12 +1193,14 @@ void *tracing_cond_snapshot_data(struct trace_array *tr) { void *cond_data = NULL; + local_irq_disable(); arch_spin_lock(&tr->max_lock); if (tr->cond_snapshot) cond_data = tr->cond_snapshot->cond_data; arch_spin_unlock(&tr->max_lock); + local_irq_enable(); return cond_data; } @@ -1334,9 +1336,11 @@ int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, goto fail_unlock; } + local_irq_disable(); arch_spin_lock(&tr->max_lock); tr->cond_snapshot = cond_snapshot; arch_spin_unlock(&tr->max_lock); + local_irq_enable(); mutex_unlock(&trace_types_lock); @@ -1363,6 +1367,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) { int ret = 0; + local_irq_disable(); arch_spin_lock(&tr->max_lock); if (!tr->cond_snapshot) @@ -1373,6 +1378,7 @@ int tracing_snapshot_cond_disable(struct trace_array *tr) } arch_spin_unlock(&tr->max_lock); + local_irq_enable(); return ret; } @@ -2200,6 +2206,11 @@ static size_t tgid_map_max; #define SAVED_CMDLINES_DEFAULT 128 #define NO_CMDLINE_MAP UINT_MAX +/* + * Preemption must be disabled before acquiring trace_cmdline_lock. + * The various trace_arrays' max_lock must be acquired in a context + * where interrupt is disabled. + */ static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; struct saved_cmdlines_buffer { unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; @@ -2412,7 +2423,11 @@ static int trace_save_cmdline(struct task_struct *tsk) * the lock, but we also don't want to spin * nor do we want to disable interrupts, * so if we miss here, then better luck next time. + * + * This is called within the scheduler and wake up, so interrupts + * had better been disabled and run queue lock been held. */ + lockdep_assert_preemption_disabled(); if (!arch_spin_trylock(&trace_cmdline_lock)) return 0; @@ -5890,9 +5905,11 @@ tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf, char buf[64]; int r; + preempt_disable(); arch_spin_lock(&trace_cmdline_lock); r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num); arch_spin_unlock(&trace_cmdline_lock); + preempt_enable(); return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } @@ -5917,10 +5934,12 @@ static int tracing_resize_saved_cmdlines(unsigned int val) return -ENOMEM; } + preempt_disable(); arch_spin_lock(&trace_cmdline_lock); savedcmd_temp = savedcmd; savedcmd = s; arch_spin_unlock(&trace_cmdline_lock); + preempt_enable(); free_saved_cmdlines_buffer(savedcmd_temp); return 0; @@ -6373,10 +6392,12 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) #ifdef CONFIG_TRACER_SNAPSHOT if (t->use_max_tr) { + local_irq_disable(); arch_spin_lock(&tr->max_lock); if (tr->cond_snapshot) ret = -EBUSY; arch_spin_unlock(&tr->max_lock); + local_irq_enable(); if (ret) goto out; } @@ -6407,12 +6428,12 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) if (tr->current_trace->reset) tr->current_trace->reset(tr); +#ifdef CONFIG_TRACER_MAX_TRACE + had_max_tr = tr->current_trace->use_max_tr; + /* Current trace needs to be nop_trace before synchronize_rcu */ tr->current_trace = &nop_trace; -#ifdef CONFIG_TRACER_MAX_TRACE - had_max_tr = tr->allocated_snapshot; - if (had_max_tr && !t->use_max_tr) { /* * We need to make sure that the update_max_tr sees that @@ -6425,11 +6446,13 @@ int tracing_set_tracer(struct trace_array *tr, const char *buf) free_snapshot(tr); } - if (t->use_max_tr && !had_max_tr) { + if (t->use_max_tr && !tr->allocated_snapshot) { ret = tracing_alloc_snapshot_instance(tr); if (ret < 0) goto out; } +#else + tr->current_trace = &nop_trace; #endif if (t->init) { @@ -7436,10 +7459,12 @@ tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, goto out; } + local_irq_disable(); arch_spin_lock(&tr->max_lock); if (tr->cond_snapshot) ret = -EBUSY; arch_spin_unlock(&tr->max_lock); + local_irq_enable(); if (ret) goto out; @@ -8137,6 +8162,12 @@ static int tracing_buffers_release(struct inode *inode, struct file *file) __trace_array_put(iter->tr); + iter->wait_index++; + /* Make sure the waiters see the new wait_index */ + smp_wmb(); + + ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); + if (info->spare) ring_buffer_free_read_page(iter->array_buffer->buffer, info->spare_cpu, info->spare); @@ -8290,6 +8321,8 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, /* did we read anything? */ if (!spd.nr_pages) { + long wait_index; + if (ret) goto out; @@ -8297,10 +8330,21 @@ tracing_buffers_splice_read(struct file *file, loff_t *ppos, if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) goto out; + wait_index = READ_ONCE(iter->wait_index); + ret = wait_on_pipe(iter, iter->tr->buffer_percent); if (ret) goto out; + /* No need to wait after waking up when tracing is off */ + if (!tracer_tracing_is_on(iter->tr)) + goto out; + + /* Make sure we see the new wait_index */ + smp_rmb(); + if (wait_index != iter->wait_index) + goto out; + goto again; } @@ -8311,12 +8355,34 @@ out: return ret; } +/* An ioctl call with cmd 0 to the ring buffer file will wake up all waiters */ +static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct ftrace_buffer_info *info = file->private_data; + struct trace_iterator *iter = &info->iter; + + if (cmd) + return -ENOIOCTLCMD; + + mutex_lock(&trace_types_lock); + + iter->wait_index++; + /* Make sure the waiters see the new wait_index */ + smp_wmb(); + + ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); + + mutex_unlock(&trace_types_lock); + return 0; +} + static const struct file_operations tracing_buffers_fops = { .open = tracing_buffers_open, .read = tracing_buffers_read, .poll = tracing_buffers_poll, .release = tracing_buffers_release, .splice_read = tracing_buffers_splice_read, + .unlocked_ioctl = tracing_buffers_ioctl, .llseek = no_llseek, }; @@ -9005,6 +9071,8 @@ rb_simple_write(struct file *filp, const char __user *ubuf, tracer_tracing_off(tr); if (tr->current_trace->stop) tr->current_trace->stop(tr); + /* Wake up any waiters */ + ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS); } mutex_unlock(&trace_types_lock); } @@ -10091,7 +10159,7 @@ __init static int tracer_alloc_buffers(void) * buffer. The memory will be removed once the "instance" is removed. */ ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, - "trace/RB:preapre", trace_rb_cpu_prepare, + "trace/RB:prepare", trace_rb_cpu_prepare, NULL); if (ret < 0) goto out_free_cpumask; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 900e75d96c8466b035a521705826f9270d59d353..54ee5711c72995c9dd1e926c010d30dd90c38f0c 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -1435,8 +1435,6 @@ event_trigger_unlock_commit(struct trace_event_file *file, struct filter_pred; struct regex; -typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event); - typedef int (*regex_match_func)(char *str, struct regex *r, int len); enum regex_type { @@ -1455,17 +1453,6 @@ struct regex { regex_match_func match; }; -struct filter_pred { - filter_pred_fn_t fn; - u64 val; - struct regex regex; - unsigned short *ops; - struct ftrace_event_field *field; - int offset; - int not; - int op; -}; - static inline bool is_string_field(struct ftrace_event_field *field) { return field->filter_type == FILTER_DYN_STRING || diff --git a/kernel/trace/trace_benchmark.c b/kernel/trace/trace_benchmark.c index 801c2a7f76054d7aa9e2230d7986a92e2e0f4acb..54d5fa35c90abf77d2dc268bfaa37c54a8fd90dd 100644 --- a/kernel/trace/trace_benchmark.c +++ b/kernel/trace/trace_benchmark.c @@ -51,7 +51,7 @@ static void trace_do_benchmark(void) local_irq_disable(); start = trace_clock_local(); - trace_benchmark_event(bm_str); + trace_benchmark_event(bm_str, bm_last); stop = trace_clock_local(); local_irq_enable(); diff --git a/kernel/trace/trace_benchmark.h b/kernel/trace/trace_benchmark.h index 79e6fbe5b3658d8cc2fc392e0208c7a773ea4e31..c3e91060dc949d8438d24a46bf9304719cea6857 100644 --- a/kernel/trace/trace_benchmark.h +++ b/kernel/trace/trace_benchmark.h @@ -14,19 +14,21 @@ extern void trace_benchmark_unreg(void); TRACE_EVENT_FN(benchmark_event, - TP_PROTO(const char *str), + TP_PROTO(const char *str, u64 delta), - TP_ARGS(str), + TP_ARGS(str, delta), TP_STRUCT__entry( __array( char, str, BENCHMARK_EVENT_STRLEN ) + __field( u64, delta) ), TP_fast_assign( memcpy(__entry->str, str, BENCHMARK_EVENT_STRLEN); + __entry->delta = delta; ), - TP_printk("%s", __entry->str), + TP_printk("%s delta=%llu", __entry->str, __entry->delta), trace_benchmark_reg, trace_benchmark_unreg ); diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index 1783e3478912499a86710cf0184c5be9250112c4..5dd0617e5df6dbf36798d1c7b243ea52ae9f6ba2 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -16,6 +16,7 @@ #include "trace_dynevent.h" #include "trace_probe.h" #include "trace_probe_tmpl.h" +#include "trace_probe_kernel.h" #define EPROBE_EVENT_SYSTEM "eprobes" @@ -26,6 +27,9 @@ struct trace_eprobe { /* tracepoint event */ const char *event_name; + /* filter string for the tracepoint */ + char *filter_str; + struct trace_event_call *event; struct dyn_event devent; @@ -453,29 +457,14 @@ NOKPROBE_SYMBOL(process_fetch_insn) static nokprobe_inline int fetch_store_strlen_user(unsigned long addr) { - const void __user *uaddr = (__force const void __user *)addr; - - return strnlen_user_nofault(uaddr, MAX_STRING_SIZE); + return kern_fetch_store_strlen_user(addr); } /* Return the length of string -- including null terminal byte */ static nokprobe_inline int fetch_store_strlen(unsigned long addr) { - int ret, len = 0; - u8 c; - -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if (addr < TASK_SIZE) - return fetch_store_strlen_user(addr); -#endif - - do { - ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1); - len++; - } while (c && ret == 0 && len < MAX_STRING_SIZE); - - return (ret < 0) ? ret : len; + return kern_fetch_store_strlen(addr); } /* @@ -485,21 +474,7 @@ fetch_store_strlen(unsigned long addr) static nokprobe_inline int fetch_store_string_user(unsigned long addr, void *dest, void *base) { - const void __user *uaddr = (__force const void __user *)addr; - int maxlen = get_loc_len(*(u32 *)dest); - void *__dest; - long ret; - - if (unlikely(!maxlen)) - return -ENOMEM; - - __dest = get_loc_data(dest, base); - - ret = strncpy_from_user_nofault(__dest, uaddr, maxlen); - if (ret >= 0) - *(u32 *)dest = make_data_loc(ret, __dest - base); - - return ret; + return kern_fetch_store_string_user(addr, dest, base); } /* @@ -509,29 +484,7 @@ fetch_store_string_user(unsigned long addr, void *dest, void *base) static nokprobe_inline int fetch_store_string(unsigned long addr, void *dest, void *base) { - int maxlen = get_loc_len(*(u32 *)dest); - void *__dest; - long ret; - -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if ((unsigned long)addr < TASK_SIZE) - return fetch_store_string_user(addr, dest, base); -#endif - - if (unlikely(!maxlen)) - return -ENOMEM; - - __dest = get_loc_data(dest, base); - - /* - * Try to get string again, since the string can be changed while - * probing. - */ - ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen); - if (ret >= 0) - *(u32 *)dest = make_data_loc(ret, __dest - base); - - return ret; + return kern_fetch_store_string(addr, dest, base); } static nokprobe_inline int @@ -664,14 +617,15 @@ static struct event_trigger_data * new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file) { struct event_trigger_data *trigger; + struct event_filter *filter = NULL; struct eprobe_data *edata; + int ret; edata = kzalloc(sizeof(*edata), GFP_KERNEL); trigger = kzalloc(sizeof(*trigger), GFP_KERNEL); if (!trigger || !edata) { - kfree(edata); - kfree(trigger); - return ERR_PTR(-ENOMEM); + ret = -ENOMEM; + goto error; } trigger->flags = EVENT_TRIGGER_FL_PROBE; @@ -686,13 +640,25 @@ new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file) trigger->cmd_ops = &event_trigger_cmd; INIT_LIST_HEAD(&trigger->list); - RCU_INIT_POINTER(trigger->filter, NULL); + + if (ep->filter_str) { + ret = create_event_filter(file->tr, file->event_call, + ep->filter_str, false, &filter); + if (ret) + goto error; + } + RCU_INIT_POINTER(trigger->filter, filter); edata->file = file; edata->ep = ep; trigger->private_data = edata; return trigger; +error: + free_event_filter(filter); + kfree(edata); + kfree(trigger); + return ERR_PTR(ret); } static int enable_eprobe(struct trace_eprobe *ep, @@ -726,6 +692,7 @@ static int disable_eprobe(struct trace_eprobe *ep, { struct event_trigger_data *trigger = NULL, *iter; struct trace_event_file *file; + struct event_filter *filter; struct eprobe_data *edata; file = find_event_file(tr, ep->event_system, ep->event_name); @@ -752,6 +719,10 @@ static int disable_eprobe(struct trace_eprobe *ep, /* Make sure nothing is using the edata or trigger */ tracepoint_synchronize_unregister(); + filter = rcu_access_pointer(trigger->filter); + + if (filter) + free_event_filter(filter); kfree(edata); kfree(trigger); @@ -927,12 +898,62 @@ static int trace_eprobe_tp_update_arg(struct trace_eprobe *ep, const char *argv[ return ret; } +static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[]) +{ + struct event_filter *dummy; + int i, ret, len = 0; + char *p; + + if (argc == 0) { + trace_probe_log_err(0, NO_EP_FILTER); + return -EINVAL; + } + + /* Recover the filter string */ + for (i = 0; i < argc; i++) + len += strlen(argv[i]) + 1; + + ep->filter_str = kzalloc(len, GFP_KERNEL); + if (!ep->filter_str) + return -ENOMEM; + + p = ep->filter_str; + for (i = 0; i < argc; i++) { + ret = snprintf(p, len, "%s ", argv[i]); + if (ret < 0) + goto error; + if (ret > len) { + ret = -E2BIG; + goto error; + } + p += ret; + len -= ret; + } + p[-1] = '\0'; + + /* + * Ensure the filter string can be parsed correctly. Note, this + * filter string is for the original event, not for the eprobe. + */ + ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str, + true, &dummy); + free_event_filter(dummy); + if (ret) + goto error; + + return 0; +error: + kfree(ep->filter_str); + ep->filter_str = NULL; + return ret; +} + static int __trace_eprobe_create(int argc, const char *argv[]) { /* * Argument syntax: - * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] - * Fetch args: + * e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER] + * Fetch args (no space): * =$[:TYPE] */ const char *event = NULL, *group = EPROBE_EVENT_SYSTEM; @@ -942,8 +963,8 @@ static int __trace_eprobe_create(int argc, const char *argv[]) char buf1[MAX_EVENT_NAME_LEN]; char buf2[MAX_EVENT_NAME_LEN]; char gbuf[MAX_EVENT_NAME_LEN]; - int ret = 0; - int i; + int ret = 0, filter_idx = 0; + int i, filter_cnt; if (argc < 2 || argv[0][0] != 'e') return -ECANCELED; @@ -968,11 +989,19 @@ static int __trace_eprobe_create(int argc, const char *argv[]) } if (!event) { - strscpy(buf1, argv[1], MAX_EVENT_NAME_LEN); - sanitize_event_name(buf1); + strscpy(buf1, sys_event, MAX_EVENT_NAME_LEN); event = buf1; } + for (i = 2; i < argc; i++) { + if (!strcmp(argv[i], "if")) { + filter_idx = i + 1; + filter_cnt = argc - filter_idx; + argc = i; + break; + } + } + mutex_lock(&event_mutex); event_call = find_and_get_event(sys_name, sys_event); ep = alloc_event_probe(group, event, event_call, argc - 2); @@ -988,6 +1017,14 @@ static int __trace_eprobe_create(int argc, const char *argv[]) goto error; } + if (filter_idx) { + trace_probe_log_set_index(filter_idx); + ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx); + if (ret) + goto parse_error; + } else + ep->filter_str = NULL; + argc -= 2; argv += 2; /* parse arguments */ for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 4b1057ab9d96825f42e87c367263bf122db11819..96acc2b71ac747020a44fe22db21d3594e7d4993 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -43,6 +43,42 @@ enum filter_op_ids { OPS }; static const char * ops[] = { OPS }; +enum filter_pred_fn { + FILTER_PRED_FN_NOP, + FILTER_PRED_FN_64, + FILTER_PRED_FN_S64, + FILTER_PRED_FN_U64, + FILTER_PRED_FN_32, + FILTER_PRED_FN_S32, + FILTER_PRED_FN_U32, + FILTER_PRED_FN_16, + FILTER_PRED_FN_S16, + FILTER_PRED_FN_U16, + FILTER_PRED_FN_8, + FILTER_PRED_FN_S8, + FILTER_PRED_FN_U8, + FILTER_PRED_FN_COMM, + FILTER_PRED_FN_STRING, + FILTER_PRED_FN_STRLOC, + FILTER_PRED_FN_STRRELLOC, + FILTER_PRED_FN_PCHAR_USER, + FILTER_PRED_FN_PCHAR, + FILTER_PRED_FN_CPU, + FILTER_PRED_FN_, + FILTER_PRED_TEST_VISITED, +}; + +struct filter_pred { + enum filter_pred_fn fn_num; + u64 val; + struct regex regex; + unsigned short *ops; + struct ftrace_event_field *field; + int offset; + int not; + int op; +}; + /* * pred functions are OP_LE, OP_LT, OP_GE, OP_GT, and OP_BAND * pred_funcs_##type below must match the order of them above. @@ -590,44 +626,48 @@ out_free: return ERR_PTR(ret); } +enum pred_cmp_types { + PRED_CMP_TYPE_NOP, + PRED_CMP_TYPE_LT, + PRED_CMP_TYPE_LE, + PRED_CMP_TYPE_GT, + PRED_CMP_TYPE_GE, + PRED_CMP_TYPE_BAND, +}; + #define DEFINE_COMPARISON_PRED(type) \ -static int filter_pred_LT_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr = (type *)(event + pred->offset); \ - type val = (type)pred->val; \ - return *addr < val; \ -} \ -static int filter_pred_LE_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr = (type *)(event + pred->offset); \ - type val = (type)pred->val; \ - return *addr <= val; \ -} \ -static int filter_pred_GT_##type(struct filter_pred *pred, void *event) \ +static int filter_pred_##type(struct filter_pred *pred, void *event) \ { \ - type *addr = (type *)(event + pred->offset); \ - type val = (type)pred->val; \ - return *addr > val; \ -} \ -static int filter_pred_GE_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr = (type *)(event + pred->offset); \ - type val = (type)pred->val; \ - return *addr >= val; \ -} \ -static int filter_pred_BAND_##type(struct filter_pred *pred, void *event) \ -{ \ - type *addr = (type *)(event + pred->offset); \ - type val = (type)pred->val; \ - return !!(*addr & val); \ -} \ -static const filter_pred_fn_t pred_funcs_##type[] = { \ - filter_pred_LE_##type, \ - filter_pred_LT_##type, \ - filter_pred_GE_##type, \ - filter_pred_GT_##type, \ - filter_pred_BAND_##type, \ -}; + switch (pred->op) { \ + case OP_LT: { \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + return *addr < val; \ + } \ + case OP_LE: { \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + return *addr <= val; \ + } \ + case OP_GT: { \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + return *addr > val; \ + } \ + case OP_GE: { \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + return *addr >= val; \ + } \ + case OP_BAND: { \ + type *addr = (type *)(event + pred->offset); \ + type val = (type)pred->val; \ + return !!(*addr & val); \ + } \ + default: \ + return 0; \ + } \ +} #define DEFINE_EQUALITY_PRED(size) \ static int filter_pred_##size(struct filter_pred *pred, void *event) \ @@ -836,11 +876,6 @@ static int filter_pred_comm(struct filter_pred *pred, void *event) return cmp ^ pred->not; } -static int filter_pred_none(struct filter_pred *pred, void *event) -{ - return 0; -} - /* * regex_match_foo - Basic regex callbacks * @@ -986,6 +1021,19 @@ static void filter_build_regex(struct filter_pred *pred) } } + +#ifdef CONFIG_FTRACE_STARTUP_TEST +static int test_pred_visited_fn(struct filter_pred *pred, void *event); +#else +static int test_pred_visited_fn(struct filter_pred *pred, void *event) +{ + return 0; +} +#endif + + +static int filter_pred_fn_call(struct filter_pred *pred, void *event); + /* return 1 if event matches, 0 otherwise (discard) */ int filter_match_preds(struct event_filter *filter, void *rec) { @@ -1003,7 +1051,7 @@ int filter_match_preds(struct event_filter *filter, void *rec) for (i = 0; prog[i].pred; i++) { struct filter_pred *pred = prog[i].pred; - int match = pred->fn(pred, rec); + int match = filter_pred_fn_call(pred, rec); if (match == prog[i].when_to_branch) i = prog[i].target; } @@ -1189,10 +1237,10 @@ int filter_assign_type(const char *type) return FILTER_OTHER; } -static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, - int field_size, int field_is_signed) +static enum filter_pred_fn select_comparison_fn(enum filter_op_ids op, + int field_size, int field_is_signed) { - filter_pred_fn_t fn = NULL; + enum filter_pred_fn fn = FILTER_PRED_FN_NOP; int pred_func_index = -1; switch (op) { @@ -1201,50 +1249,99 @@ static filter_pred_fn_t select_comparison_fn(enum filter_op_ids op, break; default: if (WARN_ON_ONCE(op < PRED_FUNC_START)) - return NULL; + return fn; pred_func_index = op - PRED_FUNC_START; if (WARN_ON_ONCE(pred_func_index > PRED_FUNC_MAX)) - return NULL; + return fn; } switch (field_size) { case 8: if (pred_func_index < 0) - fn = filter_pred_64; + fn = FILTER_PRED_FN_64; else if (field_is_signed) - fn = pred_funcs_s64[pred_func_index]; + fn = FILTER_PRED_FN_S64; else - fn = pred_funcs_u64[pred_func_index]; + fn = FILTER_PRED_FN_U64; break; case 4: if (pred_func_index < 0) - fn = filter_pred_32; + fn = FILTER_PRED_FN_32; else if (field_is_signed) - fn = pred_funcs_s32[pred_func_index]; + fn = FILTER_PRED_FN_S32; else - fn = pred_funcs_u32[pred_func_index]; + fn = FILTER_PRED_FN_U32; break; case 2: if (pred_func_index < 0) - fn = filter_pred_16; + fn = FILTER_PRED_FN_16; else if (field_is_signed) - fn = pred_funcs_s16[pred_func_index]; + fn = FILTER_PRED_FN_S16; else - fn = pred_funcs_u16[pred_func_index]; + fn = FILTER_PRED_FN_U16; break; case 1: if (pred_func_index < 0) - fn = filter_pred_8; + fn = FILTER_PRED_FN_8; else if (field_is_signed) - fn = pred_funcs_s8[pred_func_index]; + fn = FILTER_PRED_FN_S8; else - fn = pred_funcs_u8[pred_func_index]; + fn = FILTER_PRED_FN_U8; break; } return fn; } + +static int filter_pred_fn_call(struct filter_pred *pred, void *event) +{ + switch (pred->fn_num) { + case FILTER_PRED_FN_64: + return filter_pred_64(pred, event); + case FILTER_PRED_FN_S64: + return filter_pred_s64(pred, event); + case FILTER_PRED_FN_U64: + return filter_pred_u64(pred, event); + case FILTER_PRED_FN_32: + return filter_pred_32(pred, event); + case FILTER_PRED_FN_S32: + return filter_pred_s32(pred, event); + case FILTER_PRED_FN_U32: + return filter_pred_u32(pred, event); + case FILTER_PRED_FN_16: + return filter_pred_16(pred, event); + case FILTER_PRED_FN_S16: + return filter_pred_s16(pred, event); + case FILTER_PRED_FN_U16: + return filter_pred_u16(pred, event); + case FILTER_PRED_FN_8: + return filter_pred_8(pred, event); + case FILTER_PRED_FN_S8: + return filter_pred_s8(pred, event); + case FILTER_PRED_FN_U8: + return filter_pred_u8(pred, event); + case FILTER_PRED_FN_COMM: + return filter_pred_comm(pred, event); + case FILTER_PRED_FN_STRING: + return filter_pred_string(pred, event); + case FILTER_PRED_FN_STRLOC: + return filter_pred_strloc(pred, event); + case FILTER_PRED_FN_STRRELLOC: + return filter_pred_strrelloc(pred, event); + case FILTER_PRED_FN_PCHAR_USER: + return filter_pred_pchar_user(pred, event); + case FILTER_PRED_FN_PCHAR: + return filter_pred_pchar(pred, event); + case FILTER_PRED_FN_CPU: + return filter_pred_cpu(pred, event); + case FILTER_PRED_TEST_VISITED: + return test_pred_visited_fn(pred, event); + default: + return 0; + } +} + /* Called when a predicate is encountered by predicate_parse() */ static int parse_pred(const char *str, void *data, int pos, struct filter_parse_error *pe, @@ -1338,7 +1435,7 @@ static int parse_pred(const char *str, void *data, parse_error(pe, FILT_ERR_IP_FIELD_ONLY, pos + i); goto err_free; } - pred->fn = filter_pred_none; + pred->fn_num = FILTER_PRED_FN_NOP; /* * Quotes are not required, but if they exist then we need @@ -1416,16 +1513,16 @@ static int parse_pred(const char *str, void *data, filter_build_regex(pred); if (field->filter_type == FILTER_COMM) { - pred->fn = filter_pred_comm; + pred->fn_num = FILTER_PRED_FN_COMM; } else if (field->filter_type == FILTER_STATIC_STRING) { - pred->fn = filter_pred_string; + pred->fn_num = FILTER_PRED_FN_STRING; pred->regex.field_len = field->size; } else if (field->filter_type == FILTER_DYN_STRING) { - pred->fn = filter_pred_strloc; + pred->fn_num = FILTER_PRED_FN_STRLOC; } else if (field->filter_type == FILTER_RDYN_STRING) - pred->fn = filter_pred_strrelloc; + pred->fn_num = FILTER_PRED_FN_STRRELLOC; else { if (!ustring_per_cpu) { @@ -1436,9 +1533,9 @@ static int parse_pred(const char *str, void *data, } if (ustring) - pred->fn = filter_pred_pchar_user; + pred->fn_num = FILTER_PRED_FN_PCHAR_USER; else - pred->fn = filter_pred_pchar; + pred->fn_num = FILTER_PRED_FN_PCHAR; } /* go past the last quote */ i++; @@ -1486,10 +1583,10 @@ static int parse_pred(const char *str, void *data, pred->val = val; if (field->filter_type == FILTER_CPU) - pred->fn = filter_pred_cpu; + pred->fn_num = FILTER_PRED_FN_CPU; else { - pred->fn = select_comparison_fn(pred->op, field->size, - field->is_signed); + pred->fn_num = select_comparison_fn(pred->op, field->size, + field->is_signed); if (pred->op == OP_NE) pred->not = 1; } @@ -2296,7 +2393,7 @@ static void update_pred_fn(struct event_filter *filter, char *fields) struct filter_pred *pred = prog[i].pred; struct ftrace_event_field *field = pred->field; - WARN_ON_ONCE(!pred->fn); + WARN_ON_ONCE(pred->fn_num == FILTER_PRED_FN_NOP); if (!field) { WARN_ONCE(1, "all leafs should have field defined %d", i); @@ -2306,7 +2403,7 @@ static void update_pred_fn(struct event_filter *filter, char *fields) if (!strchr(fields, *field->name)) continue; - pred->fn = test_pred_visited_fn; + pred->fn_num = FILTER_PRED_TEST_VISITED; } } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index fdf784620c283a4a01238f52117e4d6164b08e67..48465f7e97b42b215cb3c72d779da6279a3acfe5 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -104,6 +104,38 @@ enum field_op_id { FIELD_OP_MULT, }; +enum hist_field_fn { + HIST_FIELD_FN_NOP, + HIST_FIELD_FN_VAR_REF, + HIST_FIELD_FN_COUNTER, + HIST_FIELD_FN_CONST, + HIST_FIELD_FN_LOG2, + HIST_FIELD_FN_BUCKET, + HIST_FIELD_FN_TIMESTAMP, + HIST_FIELD_FN_CPU, + HIST_FIELD_FN_STRING, + HIST_FIELD_FN_DYNSTRING, + HIST_FIELD_FN_RELDYNSTRING, + HIST_FIELD_FN_PSTRING, + HIST_FIELD_FN_S64, + HIST_FIELD_FN_U64, + HIST_FIELD_FN_S32, + HIST_FIELD_FN_U32, + HIST_FIELD_FN_S16, + HIST_FIELD_FN_U16, + HIST_FIELD_FN_S8, + HIST_FIELD_FN_U8, + HIST_FIELD_FN_UMINUS, + HIST_FIELD_FN_MINUS, + HIST_FIELD_FN_PLUS, + HIST_FIELD_FN_DIV, + HIST_FIELD_FN_MULT, + HIST_FIELD_FN_DIV_POWER2, + HIST_FIELD_FN_DIV_NOT_POWER2, + HIST_FIELD_FN_DIV_MULT_SHIFT, + HIST_FIELD_FN_EXECNAME, +}; + /* * A hist_var (histogram variable) contains variable information for * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF @@ -123,15 +155,15 @@ struct hist_var { struct hist_field { struct ftrace_event_field *field; unsigned long flags; - hist_field_fn_t fn; - unsigned int ref; - unsigned int size; - unsigned int offset; - unsigned int is_signed; unsigned long buckets; const char *type; struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; struct hist_trigger_data *hist_data; + enum hist_field_fn fn_num; + unsigned int ref; + unsigned int size; + unsigned int offset; + unsigned int is_signed; /* * Variable fields contain variable-specific info in var. @@ -166,14 +198,11 @@ struct hist_field { u64 div_multiplier; }; -static u64 hist_field_none(struct hist_field *field, - struct tracing_map_elt *elt, - struct trace_buffer *buffer, - struct ring_buffer_event *rbe, - void *event) -{ - return 0; -} +static u64 hist_fn_call(struct hist_field *hist_field, + struct tracing_map_elt *elt, + struct trace_buffer *buffer, + struct ring_buffer_event *rbe, + void *event); static u64 hist_field_const(struct hist_field *field, struct tracing_map_elt *elt, @@ -250,7 +279,7 @@ static u64 hist_field_log2(struct hist_field *hist_field, { struct hist_field *operand = hist_field->operands[0]; - u64 val = operand->fn(operand, elt, buffer, rbe, event); + u64 val = hist_fn_call(operand, elt, buffer, rbe, event); return (u64) ilog2(roundup_pow_of_two(val)); } @@ -264,7 +293,7 @@ static u64 hist_field_bucket(struct hist_field *hist_field, struct hist_field *operand = hist_field->operands[0]; unsigned long buckets = hist_field->buckets; - u64 val = operand->fn(operand, elt, buffer, rbe, event); + u64 val = hist_fn_call(operand, elt, buffer, rbe, event); if (WARN_ON_ONCE(!buckets)) return val; @@ -285,8 +314,8 @@ static u64 hist_field_plus(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); - u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); + u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); return val1 + val2; } @@ -300,8 +329,8 @@ static u64 hist_field_minus(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); - u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); + u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); return val1 - val2; } @@ -315,8 +344,8 @@ static u64 hist_field_div(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); - u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); + u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); /* Return -1 for the undefined case */ if (!val2) @@ -338,7 +367,7 @@ static u64 div_by_power_of_two(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); return val1 >> __ffs64(operand2->constant); } @@ -352,7 +381,7 @@ static u64 div_by_not_power_of_two(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); return div64_u64(val1, operand2->constant); } @@ -366,7 +395,7 @@ static u64 div_by_mult_and_shift(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); /* * If the divisor is a constant, do a multiplication and shift instead. @@ -400,8 +429,8 @@ static u64 hist_field_mult(struct hist_field *hist_field, struct hist_field *operand1 = hist_field->operands[0]; struct hist_field *operand2 = hist_field->operands[1]; - u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event); - u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event); + u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event); + u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event); return val1 * val2; } @@ -414,7 +443,7 @@ static u64 hist_field_unary_minus(struct hist_field *hist_field, { struct hist_field *operand = hist_field->operands[0]; - s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event); + s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event); u64 val = (u64)-sval; return val; @@ -657,19 +686,19 @@ struct snapshot_context { * Returns the specific division function to use if the divisor * is constant. This avoids extra branches when the trigger is hit. */ -static hist_field_fn_t hist_field_get_div_fn(struct hist_field *divisor) +static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor) { u64 div = divisor->constant; if (!(div & (div - 1))) - return div_by_power_of_two; + return HIST_FIELD_FN_DIV_POWER2; /* If the divisor is too large, do a regular division */ if (div > (1 << HIST_DIV_SHIFT)) - return div_by_not_power_of_two; + return HIST_FIELD_FN_DIV_NOT_POWER2; divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div); - return div_by_mult_and_shift; + return HIST_FIELD_FN_DIV_MULT_SHIFT; } static void track_data_free(struct track_data *track_data) @@ -1334,38 +1363,32 @@ static const char *hist_field_name(struct hist_field *field, return field_name; } -static hist_field_fn_t select_value_fn(int field_size, int field_is_signed) +static enum hist_field_fn select_value_fn(int field_size, int field_is_signed) { - hist_field_fn_t fn = NULL; - switch (field_size) { case 8: if (field_is_signed) - fn = hist_field_s64; + return HIST_FIELD_FN_S64; else - fn = hist_field_u64; - break; + return HIST_FIELD_FN_U64; case 4: if (field_is_signed) - fn = hist_field_s32; + return HIST_FIELD_FN_S32; else - fn = hist_field_u32; - break; + return HIST_FIELD_FN_U32; case 2: if (field_is_signed) - fn = hist_field_s16; + return HIST_FIELD_FN_S16; else - fn = hist_field_u16; - break; + return HIST_FIELD_FN_U16; case 1: if (field_is_signed) - fn = hist_field_s8; + return HIST_FIELD_FN_S8; else - fn = hist_field_u8; - break; + return HIST_FIELD_FN_U8; } - return fn; + return HIST_FIELD_FN_NOP; } static int parse_map_size(char *str) @@ -1922,19 +1945,19 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, goto out; /* caller will populate */ if (flags & HIST_FIELD_FL_VAR_REF) { - hist_field->fn = hist_field_var_ref; + hist_field->fn_num = HIST_FIELD_FN_VAR_REF; goto out; } if (flags & HIST_FIELD_FL_HITCOUNT) { - hist_field->fn = hist_field_counter; + hist_field->fn_num = HIST_FIELD_FN_COUNTER; hist_field->size = sizeof(u64); hist_field->type = "u64"; goto out; } if (flags & HIST_FIELD_FL_CONST) { - hist_field->fn = hist_field_const; + hist_field->fn_num = HIST_FIELD_FN_CONST; hist_field->size = sizeof(u64); hist_field->type = kstrdup("u64", GFP_KERNEL); if (!hist_field->type) @@ -1943,14 +1966,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, } if (flags & HIST_FIELD_FL_STACKTRACE) { - hist_field->fn = hist_field_none; + hist_field->fn_num = HIST_FIELD_FN_NOP; goto out; } if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) { unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET); - hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 : - hist_field_bucket; + hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 : + HIST_FIELD_FN_BUCKET; hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL); hist_field->size = hist_field->operands[0]->size; hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL); @@ -1960,14 +1983,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, } if (flags & HIST_FIELD_FL_TIMESTAMP) { - hist_field->fn = hist_field_timestamp; + hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP; hist_field->size = sizeof(u64); hist_field->type = "u64"; goto out; } if (flags & HIST_FIELD_FL_CPU) { - hist_field->fn = hist_field_cpu; + hist_field->fn_num = HIST_FIELD_FN_CPU; hist_field->size = sizeof(int); hist_field->type = "unsigned int"; goto out; @@ -1987,14 +2010,14 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, goto free; if (field->filter_type == FILTER_STATIC_STRING) { - hist_field->fn = hist_field_string; + hist_field->fn_num = HIST_FIELD_FN_STRING; hist_field->size = field->size; } else if (field->filter_type == FILTER_DYN_STRING) { - hist_field->fn = hist_field_dynstring; + hist_field->fn_num = HIST_FIELD_FN_DYNSTRING; } else if (field->filter_type == FILTER_RDYN_STRING) - hist_field->fn = hist_field_reldynstring; + hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING; else - hist_field->fn = hist_field_pstring; + hist_field->fn_num = HIST_FIELD_FN_PSTRING; } else { hist_field->size = field->size; hist_field->is_signed = field->is_signed; @@ -2002,9 +2025,9 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, if (!hist_field->type) goto free; - hist_field->fn = select_value_fn(field->size, - field->is_signed); - if (!hist_field->fn) { + hist_field->fn_num = select_value_fn(field->size, + field->is_signed); + if (hist_field->fn_num == HIST_FIELD_FN_NOP) { destroy_hist_field(hist_field, 0); return NULL; } @@ -2340,7 +2363,7 @@ static struct hist_field *create_alias(struct hist_trigger_data *hist_data, if (!alias) return NULL; - alias->fn = var_ref->fn; + alias->fn_num = var_ref->fn_num; alias->operands[0] = var_ref; if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) { @@ -2523,7 +2546,7 @@ static struct hist_field *parse_unary(struct hist_trigger_data *hist_data, expr->flags |= operand1->flags & (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS); - expr->fn = hist_field_unary_minus; + expr->fn_num = HIST_FIELD_FN_UMINUS; expr->operands[0] = operand1; expr->size = operand1->size; expr->is_signed = operand1->is_signed; @@ -2595,7 +2618,7 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, unsigned long operand_flags, operand2_flags; int field_op, ret = -EINVAL; char *sep, *operand1_str; - hist_field_fn_t op_fn; + enum hist_field_fn op_fn; bool combine_consts; if (*n_subexprs > 3) { @@ -2654,16 +2677,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, switch (field_op) { case FIELD_OP_MINUS: - op_fn = hist_field_minus; + op_fn = HIST_FIELD_FN_MINUS; break; case FIELD_OP_PLUS: - op_fn = hist_field_plus; + op_fn = HIST_FIELD_FN_PLUS; break; case FIELD_OP_DIV: - op_fn = hist_field_div; + op_fn = HIST_FIELD_FN_DIV; break; case FIELD_OP_MULT: - op_fn = hist_field_mult; + op_fn = HIST_FIELD_FN_MULT; break; default: ret = -EINVAL; @@ -2719,13 +2742,16 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, op_fn = hist_field_get_div_fn(operand2); } + expr->fn_num = op_fn; + if (combine_consts) { if (var1) expr->operands[0] = var1; if (var2) expr->operands[1] = var2; - expr->constant = op_fn(expr, NULL, NULL, NULL, NULL); + expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL); + expr->fn_num = HIST_FIELD_FN_CONST; expr->operands[0] = NULL; expr->operands[1] = NULL; @@ -2739,8 +2765,6 @@ static struct hist_field *parse_expr(struct hist_trigger_data *hist_data, expr->name = expr_str(expr, 0); } else { - expr->fn = op_fn; - /* The operand sizes should be the same, so just pick one */ expr->size = operand1->size; expr->is_signed = operand1->is_signed; @@ -3065,7 +3089,7 @@ static inline void __update_field_vars(struct tracing_map_elt *elt, struct hist_field *var = field_var->var; struct hist_field *val = field_var->val; - var_val = val->fn(val, elt, buffer, rbe, rec); + var_val = hist_fn_call(val, elt, buffer, rbe, rec); var_idx = var->var.idx; if (val->flags & HIST_FIELD_FL_STRING) { @@ -4186,6 +4210,74 @@ static u64 hist_field_execname(struct hist_field *hist_field, return (u64)(unsigned long)(elt_data->comm); } +static u64 hist_fn_call(struct hist_field *hist_field, + struct tracing_map_elt *elt, + struct trace_buffer *buffer, + struct ring_buffer_event *rbe, + void *event) +{ + switch (hist_field->fn_num) { + case HIST_FIELD_FN_VAR_REF: + return hist_field_var_ref(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_COUNTER: + return hist_field_counter(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_CONST: + return hist_field_const(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_LOG2: + return hist_field_log2(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_BUCKET: + return hist_field_bucket(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_TIMESTAMP: + return hist_field_timestamp(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_CPU: + return hist_field_cpu(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_STRING: + return hist_field_string(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_DYNSTRING: + return hist_field_dynstring(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_RELDYNSTRING: + return hist_field_reldynstring(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_PSTRING: + return hist_field_pstring(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_S64: + return hist_field_s64(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_U64: + return hist_field_u64(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_S32: + return hist_field_s32(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_U32: + return hist_field_u32(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_S16: + return hist_field_s16(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_U16: + return hist_field_u16(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_S8: + return hist_field_s8(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_U8: + return hist_field_u8(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_UMINUS: + return hist_field_unary_minus(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_MINUS: + return hist_field_minus(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_PLUS: + return hist_field_plus(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_DIV: + return hist_field_div(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_MULT: + return hist_field_mult(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_DIV_POWER2: + return div_by_power_of_two(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_DIV_NOT_POWER2: + return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_DIV_MULT_SHIFT: + return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event); + case HIST_FIELD_FN_EXECNAME: + return hist_field_execname(hist_field, elt, buffer, rbe, event); + default: + return 0; + } +} + /* Convert a var that points to common_pid.execname to a string */ static void update_var_execname(struct hist_field *hist_field) { @@ -4197,7 +4289,7 @@ static void update_var_execname(struct hist_field *hist_field) kfree_const(hist_field->type); hist_field->type = "char[]"; - hist_field->fn = hist_field_execname; + hist_field->fn_num = HIST_FIELD_FN_EXECNAME; } static int create_var_field(struct hist_trigger_data *hist_data, @@ -4956,7 +5048,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, for_each_hist_val_field(i, hist_data) { hist_field = hist_data->fields[i]; - hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec); + hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); if (hist_field->flags & HIST_FIELD_FL_VAR) { var_idx = hist_field->var.idx; @@ -4987,7 +5079,7 @@ static void hist_trigger_elt_update(struct hist_trigger_data *hist_data, for_each_hist_key_field(i, hist_data) { hist_field = hist_data->fields[i]; if (hist_field->flags & HIST_FIELD_FL_VAR) { - hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec); + hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec); var_idx = hist_field->var.idx; tracing_map_set_var(elt, var_idx, hist_val); } @@ -5062,7 +5154,7 @@ static void event_hist_trigger(struct event_trigger_data *data, HIST_STACKTRACE_SKIP); key = entries; } else { - field_contents = key_field->fn(key_field, elt, buffer, rbe, rec); + field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec); if (key_field->flags & HIST_FIELD_FL_STRING) { key = (void *)(unsigned long)field_contents; use_compound_key = true; diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 5e8c07aef071be51cf3e880068b87ddfa034bcc6..e310052dc83ce4bc680c6d14e799cdc8bbee9019 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -17,6 +17,8 @@ /* for gfp flag names */ #include #include +#include "trace_probe.h" +#include "trace_probe_kernel.h" #include "trace_synth.h" @@ -409,6 +411,7 @@ static unsigned int trace_string(struct synth_trace_event *entry, { unsigned int len = 0; char *str_field; + int ret; if (is_dynamic) { u32 data_offset; @@ -417,19 +420,27 @@ static unsigned int trace_string(struct synth_trace_event *entry, data_offset += event->n_u64 * sizeof(u64); data_offset += data_size; - str_field = (char *)entry + data_offset; - - len = strlen(str_val) + 1; - strscpy(str_field, str_val, len); + len = kern_fetch_store_strlen((unsigned long)str_val); data_offset |= len << 16; *(u32 *)&entry->fields[*n_u64] = data_offset; + ret = kern_fetch_store_string((unsigned long)str_val, &entry->fields[*n_u64], entry); + (*n_u64)++; } else { str_field = (char *)&entry->fields[*n_u64]; - strscpy(str_field, str_val, STR_VAR_LEN_MAX); +#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + if ((unsigned long)str_val < TASK_SIZE) + ret = strncpy_from_user_nofault(str_field, str_val, STR_VAR_LEN_MAX); + else +#endif + ret = strncpy_from_kernel_nofault(str_field, str_val, STR_VAR_LEN_MAX); + + if (ret < 0) + strcpy(str_field, FAULT_STRING); + (*n_u64) += STR_VAR_LEN_MAX / sizeof(u64); } @@ -462,7 +473,7 @@ static notrace void trace_event_raw_event_synth(void *__data, val_idx = var_ref_idx[field_pos]; str_val = (char *)(long)var_ref_vals[val_idx]; - len = strlen(str_val) + 1; + len = kern_fetch_store_strlen((unsigned long)str_val); fields_size += len; } diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index cb866c3141af2323001595386dd22a6eff4e9a35..918730d749325147c3772c2f858cb18e805b1291 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -142,7 +142,8 @@ static bool check_user_trigger(struct trace_event_file *file) { struct event_trigger_data *data; - list_for_each_entry_rcu(data, &file->triggers, list) { + list_for_each_entry_rcu(data, &file->triggers, list, + lockdep_is_held(&event_mutex)) { if (data->flags & EVENT_TRIGGER_FL_PROBE) continue; return true; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index a6621c52ce454fc91a6f82ffdf532e1ef4e6d16b..ae78c2d53c8ad621027f98d057a27248f9ccbe2c 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -39,28 +40,69 @@ */ #define MAX_PAGE_ORDER 0 #define MAX_PAGES (1 << MAX_PAGE_ORDER) -#define MAX_EVENTS (MAX_PAGES * PAGE_SIZE) +#define MAX_BYTES (MAX_PAGES * PAGE_SIZE) +#define MAX_EVENTS (MAX_BYTES * 8) /* Limit how long of an event name plus args within the subsystem. */ #define MAX_EVENT_DESC 512 #define EVENT_NAME(user_event) ((user_event)->tracepoint.name) #define MAX_FIELD_ARRAY_SIZE 1024 -#define MAX_FIELD_ARG_NAME 256 -static char *register_page_data; +/* + * The MAP_STATUS_* macros are used for taking a index and determining the + * appropriate byte and the bit in the byte to set/reset for an event. + * + * The lower 3 bits of the index decide which bit to set. + * The remaining upper bits of the index decide which byte to use for the bit. + * + * This is used when an event has a probe attached/removed to reflect live + * status of the event wanting tracing or not to user-programs via shared + * memory maps. + */ +#define MAP_STATUS_BYTE(index) ((index) >> 3) +#define MAP_STATUS_MASK(index) BIT((index) & 7) + +/* + * Internal bits (kernel side only) to keep track of connected probes: + * These are used when status is requested in text form about an event. These + * bits are compared against an internal byte on the event to determine which + * probes to print out to the user. + * + * These do not reflect the mapped bytes between the user and kernel space. + */ +#define EVENT_STATUS_FTRACE BIT(0) +#define EVENT_STATUS_PERF BIT(1) +#define EVENT_STATUS_OTHER BIT(7) + +/* + * Stores the pages, tables, and locks for a group of events. + * Each logical grouping of events has its own group, with a + * matching page for status checks within user programs. This + * allows for isolation of events to user programs by various + * means. + */ +struct user_event_group { + struct page *pages; + char *register_page_data; + char *system_name; + struct hlist_node node; + struct mutex reg_mutex; + DECLARE_HASHTABLE(register_table, 8); + DECLARE_BITMAP(page_bitmap, MAX_EVENTS); +}; -static DEFINE_MUTEX(reg_mutex); -static DEFINE_HASHTABLE(register_table, 4); -static DECLARE_BITMAP(page_bitmap, MAX_EVENTS); +/* Group for init_user_ns mapping, top-most group */ +static struct user_event_group *init_group; /* * Stores per-event properties, as users register events * within a file a user_event might be created if it does not * already exist. These are globally used and their lifetime * is tied to the refcnt member. These cannot go away until the - * refcnt reaches zero. + * refcnt reaches one. */ struct user_event { + struct user_event_group *group; struct tracepoint tracepoint; struct trace_event_call call; struct trace_event_class class; @@ -68,10 +110,11 @@ struct user_event { struct hlist_node node; struct list_head fields; struct list_head validators; - atomic_t refcnt; + refcount_t refcnt; int index; int flags; int min_size; + char status; }; /* @@ -86,6 +129,11 @@ struct user_event_refs { struct user_event *events[]; }; +struct user_event_file_info { + struct user_event_group *group; + struct user_event_refs *refs; +}; + #define VALIDATOR_ENSURE_NULL (1 << 0) #define VALIDATOR_REL (1 << 1) @@ -98,7 +146,8 @@ struct user_event_validator { typedef void (*user_event_func_t) (struct user_event *user, struct iov_iter *i, void *tpdata, bool *faulted); -static int user_event_parse(char *name, char *args, char *flags, +static int user_event_parse(struct user_event_group *group, char *name, + char *args, char *flags, struct user_event **newuser); static u32 user_event_key(char *name) @@ -106,6 +155,144 @@ static u32 user_event_key(char *name) return jhash(name, strlen(name), 0); } +static void set_page_reservations(char *pages, bool set) +{ + int page; + + for (page = 0; page < MAX_PAGES; ++page) { + void *addr = pages + (PAGE_SIZE * page); + + if (set) + SetPageReserved(virt_to_page(addr)); + else + ClearPageReserved(virt_to_page(addr)); + } +} + +static void user_event_group_destroy(struct user_event_group *group) +{ + if (group->register_page_data) + set_page_reservations(group->register_page_data, false); + + if (group->pages) + __free_pages(group->pages, MAX_PAGE_ORDER); + + kfree(group->system_name); + kfree(group); +} + +static char *user_event_group_system_name(struct user_namespace *user_ns) +{ + char *system_name; + int len = sizeof(USER_EVENTS_SYSTEM) + 1; + + if (user_ns != &init_user_ns) { + /* + * Unexpected at this point: + * We only currently support init_user_ns. + * When we enable more, this will trigger a failure so log. + */ + pr_warn("user_events: Namespace other than init_user_ns!\n"); + return NULL; + } + + system_name = kmalloc(len, GFP_KERNEL); + + if (!system_name) + return NULL; + + snprintf(system_name, len, "%s", USER_EVENTS_SYSTEM); + + return system_name; +} + +static inline struct user_event_group +*user_event_group_from_user_ns(struct user_namespace *user_ns) +{ + if (user_ns == &init_user_ns) + return init_group; + + return NULL; +} + +static struct user_event_group *current_user_event_group(void) +{ + struct user_namespace *user_ns = current_user_ns(); + struct user_event_group *group = NULL; + + while (user_ns) { + group = user_event_group_from_user_ns(user_ns); + + if (group) + break; + + user_ns = user_ns->parent; + } + + return group; +} + +static struct user_event_group +*user_event_group_create(struct user_namespace *user_ns) +{ + struct user_event_group *group; + + group = kzalloc(sizeof(*group), GFP_KERNEL); + + if (!group) + return NULL; + + group->system_name = user_event_group_system_name(user_ns); + + if (!group->system_name) + goto error; + + group->pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER); + + if (!group->pages) + goto error; + + group->register_page_data = page_address(group->pages); + + set_page_reservations(group->register_page_data, true); + + /* Zero all bits beside 0 (which is reserved for failures) */ + bitmap_zero(group->page_bitmap, MAX_EVENTS); + set_bit(0, group->page_bitmap); + + mutex_init(&group->reg_mutex); + hash_init(group->register_table); + + return group; +error: + if (group) + user_event_group_destroy(group); + + return NULL; +}; + +static __always_inline +void user_event_register_set(struct user_event *user) +{ + int i = user->index; + + user->group->register_page_data[MAP_STATUS_BYTE(i)] |= MAP_STATUS_MASK(i); +} + +static __always_inline +void user_event_register_clear(struct user_event *user) +{ + int i = user->index; + + user->group->register_page_data[MAP_STATUS_BYTE(i)] &= ~MAP_STATUS_MASK(i); +} + +static __always_inline __must_check +bool user_event_last_ref(struct user_event *user) +{ + return refcount_read(&user->refcnt) == 1; +} + static __always_inline __must_check size_t copy_nofault(void *addr, size_t bytes, struct iov_iter *i) { @@ -141,7 +328,8 @@ static struct list_head *user_event_get_fields(struct trace_event_call *call) * * Upon success user_event has its ref count increased by 1. */ -static int user_event_parse_cmd(char *raw_command, struct user_event **newuser) +static int user_event_parse_cmd(struct user_event_group *group, + char *raw_command, struct user_event **newuser) { char *name = raw_command; char *args = strpbrk(name, " "); @@ -155,7 +343,7 @@ static int user_event_parse_cmd(char *raw_command, struct user_event **newuser) if (flags) *flags++ = '\0'; - return user_event_parse(name, args, flags, newuser); + return user_event_parse(group, name, args, flags, newuser); } static int user_field_array_size(const char *type) @@ -277,7 +465,7 @@ static int user_event_add_field(struct user_event *user, const char *type, goto add_field; add_validator: - if (strstr(type, "char") != 0) + if (strstr(type, "char") != NULL) validator_flags |= VALIDATOR_ENSURE_NULL; validator = kmalloc(sizeof(*validator), GFP_KERNEL); @@ -458,7 +646,7 @@ static const char *user_field_format(const char *type) return "%d"; if (strcmp(type, "unsigned char") == 0) return "%u"; - if (strstr(type, "char[") != 0) + if (strstr(type, "char[") != NULL) return "%s"; /* Unknown, likely struct, allowed treat as 64-bit */ @@ -479,10 +667,52 @@ static bool user_field_is_dyn_string(const char *type, const char **str_func) return false; check: - return strstr(type, "char") != 0; + return strstr(type, "char") != NULL; } #define LEN_OR_ZERO (len ? len - pos : 0) +static int user_dyn_field_set_string(int argc, const char **argv, int *iout, + char *buf, int len, bool *colon) +{ + int pos = 0, i = *iout; + + *colon = false; + + for (; i < argc; ++i) { + if (i != *iout) + pos += snprintf(buf + pos, LEN_OR_ZERO, " "); + + pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", argv[i]); + + if (strchr(argv[i], ';')) { + ++i; + *colon = true; + break; + } + } + + /* Actual set, advance i */ + if (len != 0) + *iout = i; + + return pos + 1; +} + +static int user_field_set_string(struct ftrace_event_field *field, + char *buf, int len, bool colon) +{ + int pos = 0; + + pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->type); + pos += snprintf(buf + pos, LEN_OR_ZERO, " "); + pos += snprintf(buf + pos, LEN_OR_ZERO, "%s", field->name); + + if (colon) + pos += snprintf(buf + pos, LEN_OR_ZERO, ";"); + + return pos + 1; +} + static int user_event_set_print_fmt(struct user_event *user, char *buf, int len) { struct ftrace_event_field *field, *next; @@ -600,8 +830,8 @@ static int destroy_user_event(struct user_event *user) dyn_event_remove(&user->devent); - register_page_data[user->index] = 0; - clear_bit(user->index, page_bitmap); + user_event_register_clear(user); + clear_bit(user->index, user->group->page_bitmap); hash_del(&user->node); user_event_destroy_validators(user); @@ -612,16 +842,17 @@ static int destroy_user_event(struct user_event *user) return ret; } -static struct user_event *find_user_event(char *name, u32 *outkey) +static struct user_event *find_user_event(struct user_event_group *group, + char *name, u32 *outkey) { struct user_event *user; u32 key = user_event_key(name); *outkey = key; - hash_for_each_possible(register_table, user, node, key) + hash_for_each_possible(group->register_table, user, node, key) if (!strcmp(EVENT_NAME(user), name)) { - atomic_inc(&user->refcnt); + refcount_inc(&user->refcnt); return user; } @@ -779,7 +1010,12 @@ static void update_reg_page_for(struct user_event *user) rcu_read_unlock_sched(); } - register_page_data[user->index] = status; + if (status) + user_event_register_set(user); + else + user_event_register_clear(user); + + user->status = status; } /* @@ -835,17 +1071,18 @@ static int user_event_reg(struct trace_event_call *call, return ret; inc: - atomic_inc(&user->refcnt); + refcount_inc(&user->refcnt); update_reg_page_for(user); return 0; dec: update_reg_page_for(user); - atomic_dec(&user->refcnt); + refcount_dec(&user->refcnt); return 0; } static int user_event_create(const char *raw_command) { + struct user_event_group *group; struct user_event *user; char *name; int ret; @@ -861,14 +1098,19 @@ static int user_event_create(const char *raw_command) if (!name) return -ENOMEM; - mutex_lock(®_mutex); + group = current_user_event_group(); - ret = user_event_parse_cmd(name, &user); + if (!group) + return -ENOENT; + + mutex_lock(&group->reg_mutex); + + ret = user_event_parse_cmd(group, name, &user); if (!ret) - atomic_dec(&user->refcnt); + refcount_dec(&user->refcnt); - mutex_unlock(®_mutex); + mutex_unlock(&group->reg_mutex); if (ret) kfree(name); @@ -910,14 +1152,14 @@ static bool user_event_is_busy(struct dyn_event *ev) { struct user_event *user = container_of(ev, struct user_event, devent); - return atomic_read(&user->refcnt) != 0; + return !user_event_last_ref(user); } static int user_event_free(struct dyn_event *ev) { struct user_event *user = container_of(ev, struct user_event, devent); - if (atomic_read(&user->refcnt) != 0) + if (!user_event_last_ref(user)) return -EBUSY; return destroy_user_event(user); @@ -926,49 +1168,35 @@ static int user_event_free(struct dyn_event *ev) static bool user_field_match(struct ftrace_event_field *field, int argc, const char **argv, int *iout) { - char *field_name, *arg_name; - int len, pos, i = *iout; + char *field_name = NULL, *dyn_field_name = NULL; bool colon = false, match = false; + int dyn_len, len; - if (i >= argc) + if (*iout >= argc) return false; - len = MAX_FIELD_ARG_NAME; - field_name = kmalloc(len, GFP_KERNEL); - arg_name = kmalloc(len, GFP_KERNEL); - - if (!arg_name || !field_name) - goto out; - - pos = 0; - - for (; i < argc; ++i) { - if (i != *iout) - pos += snprintf(arg_name + pos, len - pos, " "); + dyn_len = user_dyn_field_set_string(argc, argv, iout, dyn_field_name, + 0, &colon); - pos += snprintf(arg_name + pos, len - pos, argv[i]); + len = user_field_set_string(field, field_name, 0, colon); - if (strchr(argv[i], ';')) { - ++i; - colon = true; - break; - } - } + if (dyn_len != len) + return false; - pos = 0; + dyn_field_name = kmalloc(dyn_len, GFP_KERNEL); + field_name = kmalloc(len, GFP_KERNEL); - pos += snprintf(field_name + pos, len - pos, field->type); - pos += snprintf(field_name + pos, len - pos, " "); - pos += snprintf(field_name + pos, len - pos, field->name); + if (!dyn_field_name || !field_name) + goto out; - if (colon) - pos += snprintf(field_name + pos, len - pos, ";"); + user_dyn_field_set_string(argc, argv, iout, dyn_field_name, + dyn_len, &colon); - *iout = i; + user_field_set_string(field, field_name, len, colon); - match = strcmp(arg_name, field_name) == 0; + match = strcmp(dyn_field_name, field_name) == 0; out: - kfree(arg_name); + kfree(dyn_field_name); kfree(field_name); return match; @@ -1036,7 +1264,8 @@ static int user_event_trace_register(struct user_event *user) * The name buffer lifetime is owned by this method for success cases only. * Upon success the returned user_event has its ref count increased by 1. */ -static int user_event_parse(char *name, char *args, char *flags, +static int user_event_parse(struct user_event_group *group, char *name, + char *args, char *flags, struct user_event **newuser) { int ret; @@ -1046,7 +1275,7 @@ static int user_event_parse(char *name, char *args, char *flags, /* Prevent dyn_event from racing */ mutex_lock(&event_mutex); - user = find_user_event(name, &key); + user = find_user_event(group, name, &key); mutex_unlock(&event_mutex); if (user) { @@ -1059,7 +1288,7 @@ static int user_event_parse(char *name, char *args, char *flags, return 0; } - index = find_first_zero_bit(page_bitmap, MAX_EVENTS); + index = find_first_zero_bit(group->page_bitmap, MAX_EVENTS); if (index == MAX_EVENTS) return -EMFILE; @@ -1073,6 +1302,7 @@ static int user_event_parse(char *name, char *args, char *flags, INIT_LIST_HEAD(&user->fields); INIT_LIST_HEAD(&user->validators); + user->group = group; user->tracepoint.name = name; ret = user_event_parse_fields(user, args); @@ -1091,8 +1321,8 @@ static int user_event_parse(char *name, char *args, char *flags, user->call.flags = TRACE_EVENT_FL_TRACEPOINT; user->call.tp = &user->tracepoint; user->call.event.funcs = &user_event_funcs; + user->class.system = group->system_name; - user->class.system = USER_EVENTS_SYSTEM; user->class.fields_array = user_event_fields_array; user->class.get_fields = user_event_get_fields; user->class.reg = user_event_reg; @@ -1110,13 +1340,13 @@ static int user_event_parse(char *name, char *args, char *flags, user->index = index; - /* Ensure we track ref */ - atomic_inc(&user->refcnt); + /* Ensure we track self ref and caller ref (2) */ + refcount_set(&user->refcnt, 2); dyn_event_init(&user->devent, &user_event_dops); dyn_event_add(&user->devent, &user->call); - set_bit(user->index, page_bitmap); - hash_add(register_table, &user->node, key); + set_bit(user->index, group->page_bitmap); + hash_add(group->register_table, &user->node, key); mutex_unlock(&event_mutex); @@ -1134,32 +1364,20 @@ put_user: /* * Deletes a previously created event if it is no longer being used. */ -static int delete_user_event(char *name) +static int delete_user_event(struct user_event_group *group, char *name) { u32 key; - int ret; - struct user_event *user = find_user_event(name, &key); + struct user_event *user = find_user_event(group, name, &key); if (!user) return -ENOENT; - /* Ensure we are the last ref */ - if (atomic_read(&user->refcnt) != 1) { - ret = -EBUSY; - goto put_ref; - } - - ret = destroy_user_event(user); - - if (ret) - goto put_ref; + refcount_dec(&user->refcnt); - return ret; -put_ref: - /* No longer have this ref */ - atomic_dec(&user->refcnt); + if (!user_event_last_ref(user)) + return -EBUSY; - return ret; + return destroy_user_event(user); } /* @@ -1167,6 +1385,7 @@ put_ref: */ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) { + struct user_event_file_info *info = file->private_data; struct user_event_refs *refs; struct user_event *user = NULL; struct tracepoint *tp; @@ -1178,7 +1397,7 @@ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) rcu_read_lock_sched(); - refs = rcu_dereference_sched(file->private_data); + refs = rcu_dereference_sched(info->refs); /* * The refs->events array is protected by RCU, and new items may be @@ -1236,6 +1455,28 @@ static ssize_t user_events_write_core(struct file *file, struct iov_iter *i) return ret; } +static int user_events_open(struct inode *node, struct file *file) +{ + struct user_event_group *group; + struct user_event_file_info *info; + + group = current_user_event_group(); + + if (!group) + return -ENOENT; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + + if (!info) + return -ENOMEM; + + info->group = group; + + file->private_data = info; + + return 0; +} + static ssize_t user_events_write(struct file *file, const char __user *ubuf, size_t count, loff_t *ppos) { @@ -1245,7 +1486,8 @@ static ssize_t user_events_write(struct file *file, const char __user *ubuf, if (unlikely(*ppos != 0)) return -EFAULT; - if (unlikely(import_single_range(READ, (char *)ubuf, count, &iov, &i))) + if (unlikely(import_single_range(WRITE, (char __user *)ubuf, + count, &iov, &i))) return -EFAULT; return user_events_write_core(file, &i); @@ -1256,13 +1498,15 @@ static ssize_t user_events_write_iter(struct kiocb *kp, struct iov_iter *i) return user_events_write_core(kp->ki_filp, i); } -static int user_events_ref_add(struct file *file, struct user_event *user) +static int user_events_ref_add(struct user_event_file_info *info, + struct user_event *user) { + struct user_event_group *group = info->group; struct user_event_refs *refs, *new_refs; int i, size, count = 0; - refs = rcu_dereference_protected(file->private_data, - lockdep_is_held(®_mutex)); + refs = rcu_dereference_protected(info->refs, + lockdep_is_held(&group->reg_mutex)); if (refs) { count = refs->count; @@ -1286,9 +1530,9 @@ static int user_events_ref_add(struct file *file, struct user_event *user) new_refs->events[i] = user; - atomic_inc(&user->refcnt); + refcount_inc(&user->refcnt); - rcu_assign_pointer(file->private_data, new_refs); + rcu_assign_pointer(info->refs, new_refs); if (refs) kfree_rcu(refs, rcu); @@ -1309,13 +1553,24 @@ static long user_reg_get(struct user_reg __user *ureg, struct user_reg *kreg) if (size > PAGE_SIZE) return -E2BIG; - return copy_struct_from_user(kreg, sizeof(*kreg), ureg, size); + if (size < offsetofend(struct user_reg, write_index)) + return -EINVAL; + + ret = copy_struct_from_user(kreg, sizeof(*kreg), ureg, size); + + if (ret) + return ret; + + kreg->size = size; + + return 0; } /* * Registers a user_event on behalf of a user process. */ -static long user_events_ioctl_reg(struct file *file, unsigned long uarg) +static long user_events_ioctl_reg(struct user_event_file_info *info, + unsigned long uarg) { struct user_reg __user *ureg = (struct user_reg __user *)uarg; struct user_reg reg; @@ -1336,24 +1591,24 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) return ret; } - ret = user_event_parse_cmd(name, &user); + ret = user_event_parse_cmd(info->group, name, &user); if (ret) { kfree(name); return ret; } - ret = user_events_ref_add(file, user); + ret = user_events_ref_add(info, user); /* No longer need parse ref, ref_add either worked or not */ - atomic_dec(&user->refcnt); + refcount_dec(&user->refcnt); /* Positive number is index and valid */ if (ret < 0) return ret; put_user((u32)ret, &ureg->write_index); - put_user(user->index, &ureg->status_index); + put_user(user->index, &ureg->status_bit); return 0; } @@ -1361,7 +1616,8 @@ static long user_events_ioctl_reg(struct file *file, unsigned long uarg) /* * Deletes a user_event on behalf of a user process. */ -static long user_events_ioctl_del(struct file *file, unsigned long uarg) +static long user_events_ioctl_del(struct user_event_file_info *info, + unsigned long uarg) { void __user *ubuf = (void __user *)uarg; char *name; @@ -1374,7 +1630,7 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) /* event_mutex prevents dyn_event from racing */ mutex_lock(&event_mutex); - ret = delete_user_event(name); + ret = delete_user_event(info->group, name); mutex_unlock(&event_mutex); kfree(name); @@ -1388,19 +1644,21 @@ static long user_events_ioctl_del(struct file *file, unsigned long uarg) static long user_events_ioctl(struct file *file, unsigned int cmd, unsigned long uarg) { + struct user_event_file_info *info = file->private_data; + struct user_event_group *group = info->group; long ret = -ENOTTY; switch (cmd) { case DIAG_IOCSREG: - mutex_lock(®_mutex); - ret = user_events_ioctl_reg(file, uarg); - mutex_unlock(®_mutex); + mutex_lock(&group->reg_mutex); + ret = user_events_ioctl_reg(info, uarg); + mutex_unlock(&group->reg_mutex); break; case DIAG_IOCSDEL: - mutex_lock(®_mutex); - ret = user_events_ioctl_del(file, uarg); - mutex_unlock(®_mutex); + mutex_lock(&group->reg_mutex); + ret = user_events_ioctl_del(info, uarg); + mutex_unlock(&group->reg_mutex); break; } @@ -1412,17 +1670,24 @@ static long user_events_ioctl(struct file *file, unsigned int cmd, */ static int user_events_release(struct inode *node, struct file *file) { + struct user_event_file_info *info = file->private_data; + struct user_event_group *group; struct user_event_refs *refs; struct user_event *user; int i; + if (!info) + return -EINVAL; + + group = info->group; + /* * Ensure refs cannot change under any situation by taking the * register mutex during the final freeing of the references. */ - mutex_lock(®_mutex); + mutex_lock(&group->reg_mutex); - refs = file->private_data; + refs = info->refs; if (!refs) goto out; @@ -1436,37 +1701,56 @@ static int user_events_release(struct inode *node, struct file *file) user = refs->events[i]; if (user) - atomic_dec(&user->refcnt); + refcount_dec(&user->refcnt); } out: file->private_data = NULL; - mutex_unlock(®_mutex); + mutex_unlock(&group->reg_mutex); kfree(refs); + kfree(info); return 0; } static const struct file_operations user_data_fops = { + .open = user_events_open, .write = user_events_write, .write_iter = user_events_write_iter, .unlocked_ioctl = user_events_ioctl, .release = user_events_release, }; +static struct user_event_group *user_status_group(struct file *file) +{ + struct seq_file *m = file->private_data; + + if (!m) + return NULL; + + return m->private; +} + /* * Maps the shared page into the user process for checking if event is enabled. */ static int user_status_mmap(struct file *file, struct vm_area_struct *vma) { + char *pages; + struct user_event_group *group = user_status_group(file); unsigned long size = vma->vm_end - vma->vm_start; - if (size != MAX_EVENTS) + if (size != MAX_BYTES) + return -EINVAL; + + if (!group) return -EINVAL; + pages = group->register_page_data; + return remap_pfn_range(vma, vma->vm_start, - virt_to_phys(register_page_data) >> PAGE_SHIFT, + virt_to_phys(pages) >> PAGE_SHIFT, size, vm_get_page_prot(VM_READ)); } @@ -1490,14 +1774,18 @@ static void user_seq_stop(struct seq_file *m, void *p) static int user_seq_show(struct seq_file *m, void *p) { + struct user_event_group *group = m->private; struct user_event *user; char status; int i, active = 0, busy = 0, flags; - mutex_lock(®_mutex); + if (!group) + return -EINVAL; + + mutex_lock(&group->reg_mutex); - hash_for_each(register_table, i, user, node) { - status = register_page_data[user->index]; + hash_for_each(group->register_table, i, user, node) { + status = user->status; flags = user->flags; seq_printf(m, "%d:%s", user->index, EVENT_NAME(user)); @@ -1520,7 +1808,7 @@ static int user_seq_show(struct seq_file *m, void *p) active++; } - mutex_unlock(®_mutex); + mutex_unlock(&group->reg_mutex); seq_puts(m, "\n"); seq_printf(m, "Active: %d\n", active); @@ -1539,7 +1827,24 @@ static const struct seq_operations user_seq_ops = { static int user_status_open(struct inode *node, struct file *file) { - return seq_open(file, &user_seq_ops); + struct user_event_group *group; + int ret; + + group = current_user_event_group(); + + if (!group) + return -ENOENT; + + ret = seq_open(file, &user_seq_ops); + + if (!ret) { + /* Chain group to seq_file */ + struct seq_file *m = file->private_data; + + m->private = group; + } + + return ret; } static const struct file_operations user_status_fops = { @@ -1580,42 +1885,21 @@ err: return -ENODEV; } -static void set_page_reservations(bool set) -{ - int page; - - for (page = 0; page < MAX_PAGES; ++page) { - void *addr = register_page_data + (PAGE_SIZE * page); - - if (set) - SetPageReserved(virt_to_page(addr)); - else - ClearPageReserved(virt_to_page(addr)); - } -} - static int __init trace_events_user_init(void) { - struct page *pages; int ret; - /* Zero all bits beside 0 (which is reserved for failures) */ - bitmap_zero(page_bitmap, MAX_EVENTS); - set_bit(0, page_bitmap); + init_group = user_event_group_create(&init_user_ns); - pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, MAX_PAGE_ORDER); - if (!pages) + if (!init_group) return -ENOMEM; - register_page_data = page_address(pages); - - set_page_reservations(true); ret = create_user_tracefs(); if (ret) { pr_warn("user_events could not register with tracefs\n"); - set_page_reservations(false); - __free_pages(pages, MAX_PAGE_ORDER); + user_event_group_destroy(init_group); + init_group = NULL; return ret; } diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 23f7f0ec4f4cf372b829a77d971b06a49a8c949d..5a75b039e58602e1a0294479d3b0ceb7b72e3430 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -20,6 +20,7 @@ #include "trace_kprobe_selftest.h" #include "trace_probe.h" #include "trace_probe_tmpl.h" +#include "trace_probe_kernel.h" #define KPROBE_EVENT_SYSTEM "kprobes" #define KRETPROBE_MAXACTIVE_MAX 4096 @@ -1223,29 +1224,14 @@ static const struct file_operations kprobe_profile_ops = { static nokprobe_inline int fetch_store_strlen_user(unsigned long addr) { - const void __user *uaddr = (__force const void __user *)addr; - - return strnlen_user_nofault(uaddr, MAX_STRING_SIZE); + return kern_fetch_store_strlen_user(addr); } /* Return the length of string -- including null terminal byte */ static nokprobe_inline int fetch_store_strlen(unsigned long addr) { - int ret, len = 0; - u8 c; - -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if (addr < TASK_SIZE) - return fetch_store_strlen_user(addr); -#endif - - do { - ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1); - len++; - } while (c && ret == 0 && len < MAX_STRING_SIZE); - - return (ret < 0) ? ret : len; + return kern_fetch_store_strlen(addr); } /* @@ -1255,21 +1241,7 @@ fetch_store_strlen(unsigned long addr) static nokprobe_inline int fetch_store_string_user(unsigned long addr, void *dest, void *base) { - const void __user *uaddr = (__force const void __user *)addr; - int maxlen = get_loc_len(*(u32 *)dest); - void *__dest; - long ret; - - if (unlikely(!maxlen)) - return -ENOMEM; - - __dest = get_loc_data(dest, base); - - ret = strncpy_from_user_nofault(__dest, uaddr, maxlen); - if (ret >= 0) - *(u32 *)dest = make_data_loc(ret, __dest - base); - - return ret; + return kern_fetch_store_string_user(addr, dest, base); } /* @@ -1279,29 +1251,7 @@ fetch_store_string_user(unsigned long addr, void *dest, void *base) static nokprobe_inline int fetch_store_string(unsigned long addr, void *dest, void *base) { - int maxlen = get_loc_len(*(u32 *)dest); - void *__dest; - long ret; - -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if ((unsigned long)addr < TASK_SIZE) - return fetch_store_string_user(addr, dest, base); -#endif - - if (unlikely(!maxlen)) - return -ENOMEM; - - __dest = get_loc_data(dest, base); - - /* - * Try to get string again, since the string can be changed while - * probing. - */ - ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen); - if (ret >= 0) - *(u32 *)dest = make_data_loc(ret, __dest - base); - - return ret; + return kern_fetch_store_string(addr, dest, base); } static nokprobe_inline int diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 313439920a8ce90d27f13eab9931c08e29ebaf2b..78d536d3ff3dbc7737c322cb80371b668d2cf06d 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1786,8 +1786,9 @@ static int start_per_cpu_kthreads(void) for_each_cpu(cpu, current_mask) { retval = start_kthread(cpu); if (retval) { + cpus_read_unlock(); stop_per_cpu_kthreads(); - break; + return retval; } } diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index 95b58bd757ce400f1766e296088650850d57ae30..1e130da1b742c663e6c578ccf581ec05635baa6a 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -95,14 +95,14 @@ __visible void trace_hardirqs_on_caller(unsigned long caller_addr) } lockdep_hardirqs_on_prepare(); - lockdep_hardirqs_on(CALLER_ADDR0); + lockdep_hardirqs_on(caller_addr); } EXPORT_SYMBOL(trace_hardirqs_on_caller); NOKPROBE_SYMBOL(trace_hardirqs_on_caller); __visible void trace_hardirqs_off_caller(unsigned long caller_addr) { - lockdep_hardirqs_off(CALLER_ADDR0); + lockdep_hardirqs_off(caller_addr); if (!this_cpu_read(tracing_irq_cpu)) { this_cpu_write(tracing_irq_cpu, 1); diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 3b3869ae8cfd1542524e7701ce5f44e1e3a912e9..de38f1c037762e92374a6d2b3019010476b508c7 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -445,7 +445,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(SAME_PROBE, "There is already the exact same probe event"),\ C(NO_EVENT_INFO, "This requires both group and event name to attach"),\ C(BAD_ATTACH_EVENT, "Attached event does not exist"),\ - C(BAD_ATTACH_ARG, "Attached event does not have this field"), + C(BAD_ATTACH_ARG, "Attached event does not have this field"),\ + C(NO_EP_FILTER, "No filter rule after 'if'"), #undef C #define C(a, b) TP_ERR_##a diff --git a/kernel/trace/trace_probe_kernel.h b/kernel/trace/trace_probe_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..77dbd9ff97826866e211ec4e5b1867940fc298f4 --- /dev/null +++ b/kernel/trace/trace_probe_kernel.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __TRACE_PROBE_KERNEL_H_ +#define __TRACE_PROBE_KERNEL_H_ + +#define FAULT_STRING "(fault)" + +/* + * This depends on trace_probe.h, but can not include it due to + * the way trace_probe_tmpl.h is used by trace_kprobe.c and trace_eprobe.c. + * Which means that any other user must include trace_probe.h before including + * this file. + */ +/* Return the length of string -- including null terminal byte */ +static nokprobe_inline int +kern_fetch_store_strlen_user(unsigned long addr) +{ + const void __user *uaddr = (__force const void __user *)addr; + int ret; + + ret = strnlen_user_nofault(uaddr, MAX_STRING_SIZE); + /* + * strnlen_user_nofault returns zero on fault, insert the + * FAULT_STRING when that occurs. + */ + if (ret <= 0) + return strlen(FAULT_STRING) + 1; + return ret; +} + +/* Return the length of string -- including null terminal byte */ +static nokprobe_inline int +kern_fetch_store_strlen(unsigned long addr) +{ + int ret, len = 0; + u8 c; + +#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + if (addr < TASK_SIZE) + return kern_fetch_store_strlen_user(addr); +#endif + + do { + ret = copy_from_kernel_nofault(&c, (u8 *)addr + len, 1); + len++; + } while (c && ret == 0 && len < MAX_STRING_SIZE); + + /* For faults, return enough to hold the FAULT_STRING */ + return (ret < 0) ? strlen(FAULT_STRING) + 1 : len; +} + +static nokprobe_inline void set_data_loc(int ret, void *dest, void *__dest, void *base, int len) +{ + if (ret >= 0) { + *(u32 *)dest = make_data_loc(ret, __dest - base); + } else { + strscpy(__dest, FAULT_STRING, len); + ret = strlen(__dest) + 1; + } +} + +/* + * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf + * with max length and relative data location. + */ +static nokprobe_inline int +kern_fetch_store_string_user(unsigned long addr, void *dest, void *base) +{ + const void __user *uaddr = (__force const void __user *)addr; + int maxlen = get_loc_len(*(u32 *)dest); + void *__dest; + long ret; + + if (unlikely(!maxlen)) + return -ENOMEM; + + __dest = get_loc_data(dest, base); + + ret = strncpy_from_user_nofault(__dest, uaddr, maxlen); + set_data_loc(ret, dest, __dest, base, maxlen); + + return ret; +} + +/* + * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max + * length and relative data location. + */ +static nokprobe_inline int +kern_fetch_store_string(unsigned long addr, void *dest, void *base) +{ + int maxlen = get_loc_len(*(u32 *)dest); + void *__dest; + long ret; + +#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + if ((unsigned long)addr < TASK_SIZE) + return kern_fetch_store_string_user(addr, dest, base); +#endif + + if (unlikely(!maxlen)) + return -ENOMEM; + + __dest = get_loc_data(dest, base); + + /* + * Try to get string again, since the string can be changed while + * probing. + */ + ret = strncpy_from_kernel_nofault(__dest, (void *)addr, maxlen); + set_data_loc(ret, dest, __dest, base, maxlen); + + return ret; +} + +#endif /* __TRACE_PROBE_KERNEL_H_ */ diff --git a/kernel/trace/tracing_map.c b/kernel/trace/tracing_map.c index 9901708ce6b8a5333a2ab3e5ff8a172f206cab8c..c774e560f2f957127c7e41b825164a0d102b6fd0 100644 --- a/kernel/trace/tracing_map.c +++ b/kernel/trace/tracing_map.c @@ -961,7 +961,7 @@ create_sort_entry(void *key, struct tracing_map_elt *elt) static void detect_dups(struct tracing_map_sort_entry **sort_entries, int n_entries, unsigned int key_size) { - unsigned int dups = 0, total_dups = 0; + unsigned int total_dups = 0; int i; void *key; @@ -974,11 +974,10 @@ static void detect_dups(struct tracing_map_sort_entry **sort_entries, key = sort_entries[0]->key; for (i = 1; i < n_entries; i++) { if (!memcmp(sort_entries[i]->key, key, key_size)) { - dups++; total_dups++; + total_dups++; continue; } key = sort_entries[i]->key; - dups = 0; } WARN_ONCE(total_dups > 0, diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index 64ea283f2f86dddb19b4c8e2d683544a14b3eb63..f23144af5743048ead3c3c800ce596f4cb050ae9 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -571,7 +571,8 @@ static void for_each_tracepoint_range( bool trace_module_has_bad_taint(struct module *mod) { return mod->taints & ~((1 << TAINT_OOT_MODULE) | (1 << TAINT_CRAP) | - (1 << TAINT_UNSIGNED_MODULE)); + (1 << TAINT_UNSIGNED_MODULE) | + (1 << TAINT_TEST)); } static BLOCKING_NOTIFIER_HEAD(tracepoint_notify_list); @@ -639,7 +640,6 @@ static void tp_module_going_check_quiescent(struct tracepoint *tp, void *priv) static int tracepoint_module_coming(struct module *mod) { struct tp_module *tp_mod; - int ret = 0; if (!mod->num_tracepoints) return 0; @@ -647,23 +647,22 @@ static int tracepoint_module_coming(struct module *mod) /* * We skip modules that taint the kernel, especially those with different * module headers (for forced load), to make sure we don't cause a crash. - * Staging, out-of-tree, and unsigned GPL modules are fine. + * Staging, out-of-tree, unsigned GPL, and test modules are fine. */ if (trace_module_has_bad_taint(mod)) return 0; - mutex_lock(&tracepoint_module_list_mutex); + tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL); - if (!tp_mod) { - ret = -ENOMEM; - goto end; - } + if (!tp_mod) + return -ENOMEM; tp_mod->mod = mod; + + mutex_lock(&tracepoint_module_list_mutex); list_add_tail(&tp_mod->list, &tracepoint_module_list); blocking_notifier_call_chain(&tracepoint_notify_list, MODULE_STATE_COMING, tp_mod); -end: mutex_unlock(&tracepoint_module_list_mutex); - return ret; + return 0; } static void tracepoint_module_going(struct module *mod) diff --git a/kernel/ucount.c b/kernel/ucount.c index 06ea04d4468522aac6526a986d14429490c9bbef..ee8e57fd6f907847d033d2141e5cdaf6e4c18112 100644 --- a/kernel/ucount.c +++ b/kernel/ucount.c @@ -87,10 +87,6 @@ static struct ctl_table user_table[] = { UCOUNT_ENTRY("max_fanotify_groups"), UCOUNT_ENTRY("max_fanotify_marks"), #endif - { }, - { }, - { }, - { }, { } }; #endif /* CONFIG_SYSCTL */ @@ -263,29 +259,29 @@ void dec_ucount(struct ucounts *ucounts, enum ucount_type type) put_ucounts(ucounts); } -long inc_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v) +long inc_rlimit_ucounts(struct ucounts *ucounts, enum rlimit_type type, long v) { struct ucounts *iter; long max = LONG_MAX; long ret = 0; for (iter = ucounts; iter; iter = iter->ns->ucounts) { - long new = atomic_long_add_return(v, &iter->ucount[type]); + long new = atomic_long_add_return(v, &iter->rlimit[type]); if (new < 0 || new > max) ret = LONG_MAX; else if (iter == ucounts) ret = new; - max = READ_ONCE(iter->ns->ucount_max[type]); + max = get_userns_rlimit_max(iter->ns, type); } return ret; } -bool dec_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v) +bool dec_rlimit_ucounts(struct ucounts *ucounts, enum rlimit_type type, long v) { struct ucounts *iter; long new = -1; /* Silence compiler warning */ for (iter = ucounts; iter; iter = iter->ns->ucounts) { - long dec = atomic_long_sub_return(v, &iter->ucount[type]); + long dec = atomic_long_sub_return(v, &iter->rlimit[type]); WARN_ON_ONCE(dec < 0); if (iter == ucounts) new = dec; @@ -294,11 +290,11 @@ bool dec_rlimit_ucounts(struct ucounts *ucounts, enum ucount_type type, long v) } static void do_dec_rlimit_put_ucounts(struct ucounts *ucounts, - struct ucounts *last, enum ucount_type type) + struct ucounts *last, enum rlimit_type type) { struct ucounts *iter, *next; for (iter = ucounts; iter != last; iter = next) { - long dec = atomic_long_sub_return(1, &iter->ucount[type]); + long dec = atomic_long_sub_return(1, &iter->rlimit[type]); WARN_ON_ONCE(dec < 0); next = iter->ns->ucounts; if (dec == 0) @@ -306,12 +302,12 @@ static void do_dec_rlimit_put_ucounts(struct ucounts *ucounts, } } -void dec_rlimit_put_ucounts(struct ucounts *ucounts, enum ucount_type type) +void dec_rlimit_put_ucounts(struct ucounts *ucounts, enum rlimit_type type) { do_dec_rlimit_put_ucounts(ucounts, NULL, type); } -long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum ucount_type type) +long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum rlimit_type type) { /* Caller must hold a reference to ucounts */ struct ucounts *iter; @@ -319,12 +315,12 @@ long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum ucount_type type) long dec, ret = 0; for (iter = ucounts; iter; iter = iter->ns->ucounts) { - long new = atomic_long_add_return(1, &iter->ucount[type]); + long new = atomic_long_add_return(1, &iter->rlimit[type]); if (new < 0 || new > max) goto unwind; if (iter == ucounts) ret = new; - max = READ_ONCE(iter->ns->ucount_max[type]); + max = get_userns_rlimit_max(iter->ns, type); /* * Grab an extra ucount reference for the caller when * the rlimit count was previously 0. @@ -336,24 +332,24 @@ long inc_rlimit_get_ucounts(struct ucounts *ucounts, enum ucount_type type) } return ret; dec_unwind: - dec = atomic_long_sub_return(1, &iter->ucount[type]); + dec = atomic_long_sub_return(1, &iter->rlimit[type]); WARN_ON_ONCE(dec < 0); unwind: do_dec_rlimit_put_ucounts(ucounts, iter, type); return 0; } -bool is_ucounts_overlimit(struct ucounts *ucounts, enum ucount_type type, unsigned long rlimit) +bool is_rlimit_overlimit(struct ucounts *ucounts, enum rlimit_type type, unsigned long rlimit) { struct ucounts *iter; long max = rlimit; if (rlimit > LONG_MAX) max = LONG_MAX; for (iter = ucounts; iter; iter = iter->ns->ucounts) { - long val = get_ucounts_value(iter, type); + long val = get_rlimit_value(iter, type); if (val < 0 || val > max) return true; - max = READ_ONCE(iter->ns->ucount_max[type]); + max = get_userns_rlimit_max(iter->ns, type); } return false; } diff --git a/kernel/umh.c b/kernel/umh.c index b989736e8707473e42541d75ea99e306a21f3f3f..8506315186652dffd12eb395931c270a82463516 100644 --- a/kernel/umh.c +++ b/kernel/umh.c @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -403,6 +404,7 @@ EXPORT_SYMBOL(call_usermodehelper_setup); */ int call_usermodehelper_exec(struct subprocess_info *sub_info, int wait) { + unsigned int state = TASK_UNINTERRUPTIBLE; DECLARE_COMPLETION_ONSTACK(done); int retval = 0; @@ -436,18 +438,22 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info, int wait) if (wait == UMH_NO_WAIT) /* task has freed sub_info */ goto unlock; - if (wait & UMH_KILLABLE) { - retval = wait_for_completion_killable(&done); - if (!retval) - goto wait_done; + if (wait & UMH_KILLABLE) + state |= TASK_KILLABLE; + + if (wait & UMH_FREEZABLE) + state |= TASK_FREEZABLE; + retval = wait_for_completion_state(&done, state); + if (!retval) + goto wait_done; + + if (wait & UMH_KILLABLE) { /* umh_complete() will see NULL and free sub_info */ if (xchg(&sub_info->complete, NULL)) goto unlock; - /* fallthrough, umh_complete() was already called */ } - wait_for_completion(&done); wait_done: retval = sub_info->retval; out: diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 5481ba44a8d68463aa01f529a4886a8905190550..54211dbd516c57e59fe136bfa72ccde212fb04e9 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -113,6 +114,10 @@ int create_user_ns(struct cred *new) !kgid_has_mapping(parent_ns, group)) goto fail_dec; + ret = security_create_user_ns(new); + if (ret < 0) + goto fail_dec; + ret = -ENOMEM; ns = kmem_cache_zalloc(user_ns_cachep, GFP_KERNEL); if (!ns) @@ -131,13 +136,13 @@ int create_user_ns(struct cred *new) ns->owner = owner; ns->group = group; INIT_WORK(&ns->work, free_user_ns); - for (i = 0; i < MAX_PER_NAMESPACE_UCOUNTS; i++) { + for (i = 0; i < UCOUNT_COUNTS; i++) { ns->ucount_max[i] = INT_MAX; } - set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_NPROC, enforced_nproc_rlimit()); - set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_MSGQUEUE, rlimit(RLIMIT_MSGQUEUE)); - set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_SIGPENDING, rlimit(RLIMIT_SIGPENDING)); - set_rlimit_ucount_max(ns, UCOUNT_RLIMIT_MEMLOCK, rlimit(RLIMIT_MEMLOCK)); + set_userns_rlimit_max(ns, UCOUNT_RLIMIT_NPROC, enforced_nproc_rlimit()); + set_userns_rlimit_max(ns, UCOUNT_RLIMIT_MSGQUEUE, rlimit(RLIMIT_MSGQUEUE)); + set_userns_rlimit_max(ns, UCOUNT_RLIMIT_SIGPENDING, rlimit(RLIMIT_SIGPENDING)); + set_userns_rlimit_max(ns, UCOUNT_RLIMIT_MEMLOCK, rlimit(RLIMIT_MEMLOCK)); ns->ucounts = ucounts; /* Inherit USERNS_SETGROUPS_ALLOWED from our parent */ diff --git a/kernel/utsname_sysctl.c b/kernel/utsname_sysctl.c index 4ca61d49885b68187eb6fcb6ad209209b955b400..064072c16e3d918f352fc2948696c42fe8c0b550 100644 --- a/kernel/utsname_sysctl.c +++ b/kernel/utsname_sysctl.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,7 @@ static int proc_do_uts_string(struct ctl_table *table, int write, * theoretically be incorrect if there are two parallel writes * at non-zero offsets to the same sysctl. */ + add_device_randomness(tmp_data, sizeof(tmp_data)); down_write(&uts_sem); memcpy(get_uts(table), tmp_data, sizeof(tmp_data)); up_write(&uts_sem); @@ -73,6 +75,13 @@ static DEFINE_CTL_TABLE_POLL(hostname_poll); static DEFINE_CTL_TABLE_POLL(domainname_poll); static struct ctl_table uts_kern_table[] = { + { + .procname = "arch", + .data = init_uts_ns.name.machine, + .maxlen = sizeof(init_uts_ns.name.machine), + .mode = 0444, + .proc_handler = proc_do_uts_string, + }, { .procname = "ostype", .data = init_uts_ns.name.sysname, diff --git a/kernel/workqueue.c b/kernel/workqueue.c index aeea9731ef80473fde61f107d73b59414eb2ea66..7cd5f5e7e0a1bf3a51c8e070aac09d7dca9864f8 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -1651,7 +1651,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, struct work_struct *work = &dwork->work; WARN_ON_ONCE(!wq); - WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn); + WARN_ON_ONCE(timer->function != delayed_work_timer_fn); WARN_ON_ONCE(timer_pending(timer)); WARN_ON_ONCE(!list_empty(&work->entry)); @@ -3066,10 +3066,8 @@ static bool __flush_work(struct work_struct *work, bool from_cancel) if (WARN_ON(!work->func)) return false; - if (!from_cancel) { - lock_map_acquire(&work->lockdep_map); - lock_map_release(&work->lockdep_map); - } + lock_map_acquire(&work->lockdep_map); + lock_map_release(&work->lockdep_map); if (start_flush_work(work, &barr, from_cancel)) { wait_for_completion(&barr.done); diff --git a/lib/Kconfig b/lib/Kconfig index dc1ab2ed1dc6e7d19b9aa492081da04d6a40fee2..9bbf8a4b2108e6b9e263fd420aaaf3c1a20453ff 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -127,9 +127,6 @@ config TRACE_MMIO_ACCESS source "lib/crypto/Kconfig" -config LIB_MEMNEQ - bool - config CRC_CCITT tristate "CRC-CCITT functions" help @@ -343,12 +340,16 @@ config LZ4HC_COMPRESS config LZ4_DECOMPRESS tristate -config ZSTD_COMPRESS +config ZSTD_COMMON select XXHASH tristate +config ZSTD_COMPRESS + select ZSTD_COMMON + tristate + config ZSTD_DECOMPRESS - select XXHASH + select ZSTD_COMMON tristate source "lib/xz/Kconfig" @@ -527,6 +528,15 @@ config CPUMASK_OFFSTACK them on the stack. This is a bit more expensive, but avoids stack overflow. +config FORCE_NR_CPUS + bool "NR_CPUS is set to an actual number of CPUs" + depends on SMP + help + Say Yes if you have NR_CPUS set to an actual number of possible + CPUs in your system, not to a default value. This forces the core + code to rely on compile-time value and optimize kernel routines + better. + config CPU_RMAP bool depends on SMP diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 072e4b289c13e2b312000092b59e12868c7cf301..3fc7abffc7aa20e0851a50216fb82f7febcbceea 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -231,6 +231,11 @@ config DEBUG_INFO in the "Debug information" choice below, indicating that debug information will be generated for build targets. +# Clang is known to generate .{s,u}leb128 with symbol deltas with DWARF5, which +# some targets may not support: https://sourceware.org/bugzilla/show_bug.cgi?id=27215 +config AS_HAS_NON_CONST_LEB128 + def_bool $(as-instr,.uleb128 .Lexpr_end4 - .Lexpr_start3\n.Lexpr_start3:\n.Lexpr_end4:) + choice prompt "Debug information" depends on DEBUG_KERNEL @@ -253,6 +258,7 @@ config DEBUG_INFO_NONE config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT bool "Rely on the toolchain's implicit default DWARF version" select DEBUG_INFO + depends on !CC_IS_CLANG || AS_IS_LLVM || CLANG_VERSION < 140000 || (AS_IS_GNU && AS_VERSION >= 23502 && AS_HAS_NON_CONST_LEB128) help The implicit default version of DWARF debug info produced by a toolchain changes over time. @@ -264,8 +270,10 @@ config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT config DEBUG_INFO_DWARF4 bool "Generate DWARF Version 4 debuginfo" select DEBUG_INFO + depends on !CC_IS_CLANG || AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502) help - Generate DWARF v4 debug info. This requires gcc 4.5+ and gdb 7.0+. + Generate DWARF v4 debug info. This requires gcc 4.5+, binutils 2.35.2 + if using clang without clang's integrated assembler, and gdb 7.0+. If you have consumers of DWARF debug info that are not ready for newer revisions of DWARF, you may wish to choose this or have your @@ -274,7 +282,7 @@ config DEBUG_INFO_DWARF4 config DEBUG_INFO_DWARF5 bool "Generate DWARF Version 5 debuginfo" select DEBUG_INFO - depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) + depends on !CC_IS_CLANG || AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502 && AS_HAS_NON_CONST_LEB128) help Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc 5.0+ accepts the -gdwarf-5 flag but only had partial support for some @@ -803,6 +811,9 @@ config ARCH_HAS_DEBUG_VM_PGTABLE An architecture should select this when it can successfully build and run DEBUG_VM_PGTABLE. +config DEBUG_VM_IRQSOFF + def_bool DEBUG_VM && !PREEMPT_RT + config DEBUG_VM bool "Debug VM" depends on DEBUG_KERNEL @@ -812,13 +823,12 @@ config DEBUG_VM If unsure, say N. -config DEBUG_VM_VMACACHE - bool "Debug VMA caching" +config DEBUG_VM_MAPLE_TREE + bool "Debug VM maple trees" depends on DEBUG_VM + select DEBUG_MAPLE_TREE help - Enable this to turn on VMA caching debug information. Doing so - can cause significant overhead, so only enable it in non-production - environments. + Enable VM maple tree debugging information and extra validations. If unsure, say N. @@ -971,6 +981,7 @@ config DEBUG_STACKOVERFLOW source "lib/Kconfig.kasan" source "lib/Kconfig.kfence" +source "lib/Kconfig.kmsan" endmenu # "Memory Debugging" @@ -1635,6 +1646,14 @@ config BUG_ON_DATA_CORRUPTION If unsure, say N. +config DEBUG_MAPLE_TREE + bool "Debug maple trees" + depends on DEBUG_KERNEL + help + Enable maple tree debugging information and extra validations. + + If unsure, say N. + endmenu config DEBUG_CREDENTIALS @@ -2029,13 +2048,16 @@ config LKDTM Documentation on how to use the module can be found in Documentation/fault-injection/provoke-crashes.rst -config TEST_CPUMASK - tristate "cpumask tests" if !KUNIT_ALL_TESTS +config CPUMASK_KUNIT_TEST + tristate "KUnit test for cpumask" if !KUNIT_ALL_TESTS depends on KUNIT default KUNIT_ALL_TESTS help Enable to turn on cpumask tests, running at boot or module load time. + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + If unsure, say N. config TEST_LIST_SORT @@ -2506,6 +2528,18 @@ config MEMCPY_KUNIT_TEST If unsure, say N. +config IS_SIGNED_TYPE_KUNIT_TEST + tristate "Test is_signed_type() macro" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Builds unit tests for the is_signed_type() macro. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config OVERFLOW_KUNIT_TEST tristate "Test check_*_overflow() functions at runtime" if !KUNIT_ALL_TESTS depends on KUNIT @@ -2530,6 +2564,25 @@ config STACKINIT_KUNIT_TEST CONFIG_GCC_PLUGIN_STRUCTLEAK, CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF, or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL. +config FORTIFY_KUNIT_TEST + tristate "Test fortified str*() and mem*() function internals at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT && FORTIFY_SOURCE + default KUNIT_ALL_TESTS + help + Builds unit tests for checking internals of FORTIFY_SOURCE as used + by the str*() and mem*() family of functions. For testing runtime + traps of FORTIFY_SOURCE, see LKDTM's "FORTIFY_*" tests. + +config HW_BREAKPOINT_KUNIT_TEST + bool "Test hw_breakpoint constraints accounting" if !KUNIT_ALL_TESTS + depends on HAVE_HW_BREAKPOINT + depends on KUNIT=y + default KUNIT_ALL_TESTS + help + Tests for hw_breakpoint constraints accounting. + + If unsure, say N. + config TEST_UDELAY tristate "udelay test driver" help @@ -2546,6 +2599,16 @@ config TEST_STATIC_KEYS If unsure, say N. +config TEST_DYNAMIC_DEBUG + tristate "Test DYNAMIC_DEBUG" + depends on DYNAMIC_DEBUG + help + This module registers a tracer callback to count enabled + pr_debugs in a 'do_debugging' function, then alters their + enablements, calls the function, and compares counts. + + If unsure, say N. + config TEST_KMOD tristate "kmod stress tester" depends on m @@ -2705,6 +2768,40 @@ config HYPERV_TESTING endmenu # "Kernel Testing and Coverage" +menu "Rust hacking" + +config RUST_DEBUG_ASSERTIONS + bool "Debug assertions" + depends on RUST + help + Enables rustc's `-Cdebug-assertions` codegen option. + + This flag lets you turn `cfg(debug_assertions)` conditional + compilation on or off. This can be used to enable extra debugging + code in development but not in production. For example, it controls + the behavior of the standard library's `debug_assert!` macro. + + Note that this will apply to all Rust code, including `core`. + + If unsure, say N. + +config RUST_OVERFLOW_CHECKS + bool "Overflow checks" + default y + depends on RUST + help + Enables rustc's `-Coverflow-checks` codegen option. + + This flag allows you to control the behavior of runtime integer + overflow. When overflow-checks are enabled, a Rust panic will occur + on overflow. + + Note that this will apply to all Rust code, including `core`. + + If unsure, say Y. + +endmenu # "Rust" + source "Documentation/Kconfig" endmenu # Kernel hacking diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index f0973da583e04078fcb2b9ba18c33ffc275fcba2..ca09b1cf8ee9d3b7993352bc57da5d159bb06c56 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -167,14 +167,6 @@ config KASAN_STACK as well, as it adds inline-style instrumentation that is run unconditionally. -config KASAN_TAGS_IDENTIFY - bool "Memory corruption type identification" - depends on KASAN_SW_TAGS || KASAN_HW_TAGS - help - Enables best-effort identification of the bug types (use-after-free - or out-of-bounds) at the cost of increased memory consumption. - Only applicable for the tag-based KASAN modes. - config KASAN_VMALLOC bool "Check accesses to vmalloc allocations" depends on HAVE_ARCH_KASAN_VMALLOC diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb index 05dae05b6cc9ef84e2443b0ea713ec59940d6963..3b9a44008433201ec86443fe0626967c37a488a3 100644 --- a/lib/Kconfig.kgdb +++ b/lib/Kconfig.kgdb @@ -121,7 +121,7 @@ config KDB_DEFAULT_ENABLE config KDB_KEYBOARD bool "KGDB_KDB: keyboard as input device" - depends on VT && KGDB_KDB + depends on VT && KGDB_KDB && !PARISC default n help KDB can use a PS/2 type keyboard for an input device diff --git a/lib/Kconfig.kmsan b/lib/Kconfig.kmsan new file mode 100644 index 0000000000000000000000000000000000000000..b2489dd6503fac7bd1ee4418004e2747920e527a --- /dev/null +++ b/lib/Kconfig.kmsan @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HAVE_ARCH_KMSAN + bool + +config HAVE_KMSAN_COMPILER + # Clang versions <14.0.0 also support -fsanitize=kernel-memory, but not + # all the features necessary to build the kernel with KMSAN. + depends on CC_IS_CLANG && CLANG_VERSION >= 140000 + def_bool $(cc-option,-fsanitize=kernel-memory -mllvm -msan-disable-checks=1) + +config KMSAN + bool "KMSAN: detector of uninitialized values use" + depends on HAVE_ARCH_KMSAN && HAVE_KMSAN_COMPILER + depends on SLUB && DEBUG_KERNEL && !KASAN && !KCSAN + select STACKDEPOT + select STACKDEPOT_ALWAYS_INIT + help + KernelMemorySanitizer (KMSAN) is a dynamic detector of uses of + uninitialized values in the kernel. It is based on compiler + instrumentation provided by Clang and thus requires Clang to build. + + An important note is that KMSAN is not intended for production use, + because it drastically increases kernel memory footprint and slows + the whole system down. + + See for more details. + +if KMSAN + +config HAVE_KMSAN_PARAM_RETVAL + # -fsanitize-memory-param-retval is supported only by Clang >= 14. + depends on HAVE_KMSAN_COMPILER + def_bool $(cc-option,-fsanitize=kernel-memory -fsanitize-memory-param-retval) + +config KMSAN_CHECK_PARAM_RETVAL + bool "Check for uninitialized values passed to and returned from functions" + default y + depends on HAVE_KMSAN_PARAM_RETVAL + help + If the compiler supports -fsanitize-memory-param-retval, KMSAN will + eagerly check every function parameter passed by value and every + function return value. + + Disabling KMSAN_CHECK_PARAM_RETVAL will result in tracking shadow for + function parameters and return values across function borders. This + is a more relaxed mode, but it generates more instrumentation code and + may potentially report errors in corner cases when non-instrumented + functions call instrumented ones. + +config KMSAN_KUNIT_TEST + tristate "KMSAN integration test suite" if !KUNIT_ALL_TESTS + default KUNIT_ALL_TESTS + depends on TRACEPOINTS && KUNIT + help + Test suite for KMSAN, testing various error detection scenarios, + and checking that reports are correctly output to console. + + Say Y here if you want the test to be built into the kernel and run + during boot; say M if you want the test to build as a module; say N + if you are unsure. + +endif diff --git a/lib/Makefile b/lib/Makefile index 5927d7fa08063dec069e64a7e469e7fc9dcf333c..161d6a724ff710bdb1ac5475354c36c8257660cb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -29,7 +29,7 @@ endif lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o timerqueue.o xarray.o \ - idr.o extable.o irq_regs.o argv_split.o \ + maple_tree.o idr.o extable.o irq_regs.o argv_split.o \ flex_proportions.o ratelimit.o show_mem.o \ is_single_threaded.o plist.o decompress.o kobject_uevent.o \ earlycpio.o seq_buf.o siphash.o dec_and_lock.o \ @@ -60,15 +60,11 @@ obj-$(CONFIG_TEST_BPF) += test_bpf.o obj-$(CONFIG_TEST_FIRMWARE) += test_firmware.o obj-$(CONFIG_TEST_BITOPS) += test_bitops.o CFLAGS_test_bitops.o += -Werror +obj-$(CONFIG_CPUMASK_KUNIT_TEST) += cpumask_kunit.o obj-$(CONFIG_TEST_SYSCTL) += test_sysctl.o obj-$(CONFIG_TEST_SIPHASH) += test_siphash.o obj-$(CONFIG_HASH_KUNIT_TEST) += test_hash.o obj-$(CONFIG_TEST_IDA) += test_ida.o -obj-$(CONFIG_KASAN_KUNIT_TEST) += test_kasan.o -CFLAGS_test_kasan.o += -fno-builtin -CFLAGS_test_kasan.o += $(call cc-disable-warning, vla) -obj-$(CONFIG_KASAN_MODULE_TEST) += test_kasan_module.o -CFLAGS_test_kasan_module.o += -fno-builtin obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o CFLAGS_test_ubsan.o += $(call cc-disable-warning, vla) UBSAN_SANITIZE_test_ubsan.o := y @@ -82,6 +78,7 @@ obj-$(CONFIG_TEST_SORT) += test_sort.o obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_keys.o obj-$(CONFIG_TEST_STATIC_KEYS) += test_static_key_base.o +obj-$(CONFIG_TEST_DYNAMIC_DEBUG) += test_dynamic_debug.o obj-$(CONFIG_TEST_PRINTF) += test_printf.o obj-$(CONFIG_TEST_SCANF) += test_scanf.o obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o @@ -100,7 +97,6 @@ obj-$(CONFIG_TEST_HMM) += test_hmm.o obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o -obj-$(CONFIG_TEST_CPUMASK) += test_cpumask.o CFLAGS_test_fprobe.o += $(CC_FLAGS_FTRACE) obj-$(CONFIG_FPROBE_SANITY_TEST) += test_fprobe.o # @@ -254,7 +250,6 @@ obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o lib-$(CONFIG_CLZ_TAB) += clz_tab.o -lib-$(CONFIG_LIB_MEMNEQ) += memneq.o obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o @@ -275,6 +270,9 @@ obj-$(CONFIG_POLYNOMIAL) += polynomial.o CFLAGS_stackdepot.o += -fno-builtin obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n +# In particular, instrumenting stackdepot.c with KMSAN will result in infinite +# recursion. +KMSAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n obj-$(CONFIG_REF_TRACKER) += ref_tracker.o @@ -377,9 +375,11 @@ obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o +obj-$(CONFIG_IS_SIGNED_TYPE_KUNIT_TEST) += is_signed_type_kunit.o obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o +obj-$(CONFIG_FORTIFY_KUNIT_TEST) += fortify_kunit.o obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o diff --git a/lib/bitmap.c b/lib/bitmap.c index 488e6c3e5acc8f115f1968584db497ea99d9798c..1c81413c51f86db68be63034528efd17f997d6ba 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -333,20 +333,32 @@ bool __bitmap_subset(const unsigned long *bitmap1, } EXPORT_SYMBOL(__bitmap_subset); +#define BITMAP_WEIGHT(FETCH, bits) \ +({ \ + unsigned int __bits = (bits), idx, w = 0; \ + \ + for (idx = 0; idx < __bits / BITS_PER_LONG; idx++) \ + w += hweight_long(FETCH); \ + \ + if (__bits % BITS_PER_LONG) \ + w += hweight_long((FETCH) & BITMAP_LAST_WORD_MASK(__bits)); \ + \ + w; \ +}) + unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) { - unsigned int k, lim = bits/BITS_PER_LONG, w = 0; - - for (k = 0; k < lim; k++) - w += hweight_long(bitmap[k]); - - if (bits % BITS_PER_LONG) - w += hweight_long(bitmap[k] & BITMAP_LAST_WORD_MASK(bits)); - - return w; + return BITMAP_WEIGHT(bitmap[idx], bits); } EXPORT_SYMBOL(__bitmap_weight); +unsigned int __bitmap_weight_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) +{ + return BITMAP_WEIGHT(bitmap1[idx] & bitmap2[idx], bits); +} +EXPORT_SYMBOL(__bitmap_weight_and); + void __bitmap_set(unsigned long *map, unsigned int start, int len) { unsigned long *p = map + BIT_WORD(start); @@ -953,37 +965,7 @@ static int bitmap_pos_to_ord(const unsigned long *buf, unsigned int pos, unsigne if (pos >= nbits || !test_bit(pos, buf)) return -1; - return __bitmap_weight(buf, pos); -} - -/** - * bitmap_ord_to_pos - find position of n-th set bit in bitmap - * @buf: pointer to bitmap - * @ord: ordinal bit position (n-th set bit, n >= 0) - * @nbits: number of valid bit positions in @buf - * - * Map the ordinal offset of bit @ord in @buf to its position in @buf. - * Value of @ord should be in range 0 <= @ord < weight(buf). If @ord - * >= weight(buf), returns @nbits. - * - * If for example, just bits 4 through 7 are set in @buf, then @ord - * values 0 through 3 will get mapped to 4 through 7, respectively, - * and all other @ord values returns @nbits. When @ord value 3 - * gets mapped to (returns) @pos value 7 in this example, that means - * that the 3rd set bit (starting with 0th) is at position 7 in @buf. - * - * The bit positions 0 through @nbits-1 are valid positions in @buf. - */ -unsigned int bitmap_ord_to_pos(const unsigned long *buf, unsigned int ord, unsigned int nbits) -{ - unsigned int pos; - - for (pos = find_first_bit(buf, nbits); - pos < nbits && ord; - pos = find_next_bit(buf, nbits, pos + 1)) - ord--; - - return pos; + return bitmap_weight(buf, pos); } /** @@ -1035,7 +1017,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, if (n < 0 || w == 0) set_bit(oldbit, dst); /* identity map */ else - set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); + set_bit(find_nth_bit(new, nbits, n % w), dst); } } EXPORT_SYMBOL(bitmap_remap); @@ -1074,7 +1056,7 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, if (n < 0 || w == 0) return oldbit; else - return bitmap_ord_to_pos(new, n % w, bits); + return find_nth_bit(new, bits, n % w); } EXPORT_SYMBOL(bitmap_bitremap); @@ -1198,7 +1180,7 @@ void bitmap_onto(unsigned long *dst, const unsigned long *orig, * The following code is a more efficient, but less * obvious, equivalent to the loop: * for (m = 0; m < bitmap_weight(relmap, bits); m++) { - * n = bitmap_ord_to_pos(orig, m, bits); + * n = find_nth_bit(orig, bits, m); * if (test_bit(m, orig)) * set_bit(n, dst); * } diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c index 8be59f84eaeaf495e9dcf6b3ca049ab360f04753..bfd53972a4d895fc8596b15b180722b9feebe72c 100644 --- a/lib/bust_spinlocks.c +++ b/lib/bust_spinlocks.c @@ -22,9 +22,6 @@ void bust_spinlocks(int yes) if (yes) { ++oops_in_progress; } else { -#ifdef CONFIG_VT - unblank_screen(); -#endif console_unblank(); if (--oops_in_progress == 0) wake_up_klogd(); diff --git a/lib/cmdline.c b/lib/cmdline.c index 5546bf58878061863fe4950403168405f1cb2c23..90ed997d9570169ade250a661046b7b18f4b8992 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -260,7 +260,7 @@ char *next_arg(char *args, char **param, char **val) args[i-1] = '\0'; } } - if (quoted && args[i-1] == '"') + if (quoted && i > 0 && args[i-1] == '"') args[i-1] = '\0'; if (args[i]) { diff --git a/lib/cmdline_kunit.c b/lib/cmdline_kunit.c index a72a2c16066ef7cb2efa1e8c1636f4ff8a8a672b..d4572dbc914539a56c8b2a6da5ab101a9e11d80d 100644 --- a/lib/cmdline_kunit.c +++ b/lib/cmdline_kunit.c @@ -76,7 +76,7 @@ static void cmdline_test_lead_int(struct kunit *test) int rc = cmdline_test_values[i]; int offset; - sprintf(in, "%u%s", get_random_int() % 256, str); + sprintf(in, "%u%s", get_random_u8(), str); /* Only first '-' after the number will advance the pointer */ offset = strlen(in) - strlen(str) + !!(rc == 2); cmdline_do_one_test(test, in, rc, offset); @@ -94,7 +94,7 @@ static void cmdline_test_tail_int(struct kunit *test) int rc = strcmp(str, "") ? (strcmp(str, "-") ? 0 : 1) : 1; int offset; - sprintf(in, "%s%u", str, get_random_int() % 256); + sprintf(in, "%s%u", str, get_random_u8()); /* * Only first and leading '-' not followed by integer * will advance the pointer. diff --git a/lib/cpumask.c b/lib/cpumask.c index f0ae119be8c41e3e1f56233ed122f8e2590916b3..c7c392514fd34f3da4415c11dcd560e780941171 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -128,23 +128,21 @@ unsigned int cpumask_local_spread(unsigned int i, int node) i %= num_online_cpus(); if (node == NUMA_NO_NODE) { - for_each_cpu(cpu, cpu_online_mask) - if (i-- == 0) - return cpu; + cpu = cpumask_nth(i, cpu_online_mask); + if (cpu < nr_cpu_ids) + return cpu; } else { /* NUMA first. */ - for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask) - if (i-- == 0) - return cpu; - - for_each_cpu(cpu, cpu_online_mask) { - /* Skip NUMA nodes, done above. */ - if (cpumask_test_cpu(cpu, cpumask_of_node(node))) - continue; - - if (i-- == 0) - return cpu; - } + cpu = cpumask_nth_and(i, cpu_online_mask, cpumask_of_node(node)); + if (cpu < nr_cpu_ids) + return cpu; + + i -= cpumask_weight_and(cpu_online_mask, cpumask_of_node(node)); + + /* Skip NUMA nodes, done above. */ + cpu = cpumask_nth_andnot(i, cpu_online_mask, cpumask_of_node(node)); + if (cpu < nr_cpu_ids) + return cpu; } BUG(); } @@ -168,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); - next = cpumask_next_and(prev, src1p, src2p); - if (next >= nr_cpu_ids) - next = cpumask_first_and(src1p, src2p); - + next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p), + nr_cpumask_bits, prev + 1); if (next < nr_cpu_ids) __this_cpu_write(distribute_cpu_mask_prev, next); @@ -185,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp) /* NOTE: our first selection will skip 0. */ prev = __this_cpu_read(distribute_cpu_mask_prev); - - next = cpumask_next(prev, srcp); - if (next >= nr_cpu_ids) - next = cpumask_first(srcp); - + next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1); if (next < nr_cpu_ids) __this_cpu_write(distribute_cpu_mask_prev, next); diff --git a/lib/cpumask_kunit.c b/lib/cpumask_kunit.c new file mode 100644 index 0000000000000000000000000000000000000000..d1fc6ece21f3704566f9296b47bf47d3f04c8296 --- /dev/null +++ b/lib/cpumask_kunit.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for cpumask. + * + * Author: Sander Vanheule + */ + +#include +#include +#include + +#define MASK_MSG(m) \ + "%s contains %sCPUs %*pbl", #m, (cpumask_weight(m) ? "" : "no "), \ + nr_cpumask_bits, cpumask_bits(m) + +#define EXPECT_FOR_EACH_CPU_EQ(test, mask) \ + do { \ + const cpumask_t *m = (mask); \ + int mask_weight = cpumask_weight(m); \ + int cpu, iter = 0; \ + for_each_cpu(cpu, m) \ + iter++; \ + KUNIT_EXPECT_EQ_MSG((test), mask_weight, iter, MASK_MSG(mask)); \ + } while (0) + +#define EXPECT_FOR_EACH_CPU_NOT_EQ(test, mask) \ + do { \ + const cpumask_t *m = (mask); \ + int mask_weight = cpumask_weight(m); \ + int cpu, iter = 0; \ + for_each_cpu_not(cpu, m) \ + iter++; \ + KUNIT_EXPECT_EQ_MSG((test), nr_cpu_ids - mask_weight, iter, MASK_MSG(mask)); \ + } while (0) + +#define EXPECT_FOR_EACH_CPU_OP_EQ(test, op, mask1, mask2) \ + do { \ + const cpumask_t *m1 = (mask1); \ + const cpumask_t *m2 = (mask2); \ + int weight; \ + int cpu, iter = 0; \ + cpumask_##op(&mask_tmp, m1, m2); \ + weight = cpumask_weight(&mask_tmp); \ + for_each_cpu_##op(cpu, mask1, mask2) \ + iter++; \ + KUNIT_EXPECT_EQ((test), weight, iter); \ + } while (0) + +#define EXPECT_FOR_EACH_CPU_WRAP_EQ(test, mask) \ + do { \ + const cpumask_t *m = (mask); \ + int mask_weight = cpumask_weight(m); \ + int cpu, iter = 0; \ + for_each_cpu_wrap(cpu, m, nr_cpu_ids / 2) \ + iter++; \ + KUNIT_EXPECT_EQ_MSG((test), mask_weight, iter, MASK_MSG(mask)); \ + } while (0) + +#define EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, name) \ + do { \ + int mask_weight = num_##name##_cpus(); \ + int cpu, iter = 0; \ + for_each_##name##_cpu(cpu) \ + iter++; \ + KUNIT_EXPECT_EQ_MSG((test), mask_weight, iter, MASK_MSG(cpu_##name##_mask)); \ + } while (0) + +static cpumask_t mask_empty; +static cpumask_t mask_all; +static cpumask_t mask_tmp; + +static void test_cpumask_weight(struct kunit *test) +{ + KUNIT_EXPECT_TRUE_MSG(test, cpumask_empty(&mask_empty), MASK_MSG(&mask_empty)); + KUNIT_EXPECT_TRUE_MSG(test, cpumask_full(&mask_all), MASK_MSG(&mask_all)); + + KUNIT_EXPECT_EQ_MSG(test, 0, cpumask_weight(&mask_empty), MASK_MSG(&mask_empty)); + KUNIT_EXPECT_EQ_MSG(test, nr_cpu_ids, cpumask_weight(cpu_possible_mask), + MASK_MSG(cpu_possible_mask)); + KUNIT_EXPECT_EQ_MSG(test, nr_cpumask_bits, cpumask_weight(&mask_all), MASK_MSG(&mask_all)); +} + +static void test_cpumask_first(struct kunit *test) +{ + KUNIT_EXPECT_LE_MSG(test, nr_cpu_ids, cpumask_first(&mask_empty), MASK_MSG(&mask_empty)); + KUNIT_EXPECT_EQ_MSG(test, 0, cpumask_first(cpu_possible_mask), MASK_MSG(cpu_possible_mask)); + + KUNIT_EXPECT_EQ_MSG(test, 0, cpumask_first_zero(&mask_empty), MASK_MSG(&mask_empty)); + KUNIT_EXPECT_LE_MSG(test, nr_cpu_ids, cpumask_first_zero(cpu_possible_mask), + MASK_MSG(cpu_possible_mask)); +} + +static void test_cpumask_last(struct kunit *test) +{ + KUNIT_EXPECT_LE_MSG(test, nr_cpumask_bits, cpumask_last(&mask_empty), + MASK_MSG(&mask_empty)); + KUNIT_EXPECT_EQ_MSG(test, nr_cpu_ids - 1, cpumask_last(cpu_possible_mask), + MASK_MSG(cpu_possible_mask)); +} + +static void test_cpumask_next(struct kunit *test) +{ + KUNIT_EXPECT_EQ_MSG(test, 0, cpumask_next_zero(-1, &mask_empty), MASK_MSG(&mask_empty)); + KUNIT_EXPECT_LE_MSG(test, nr_cpu_ids, cpumask_next_zero(-1, cpu_possible_mask), + MASK_MSG(cpu_possible_mask)); + + KUNIT_EXPECT_LE_MSG(test, nr_cpu_ids, cpumask_next(-1, &mask_empty), + MASK_MSG(&mask_empty)); + KUNIT_EXPECT_EQ_MSG(test, 0, cpumask_next(-1, cpu_possible_mask), + MASK_MSG(cpu_possible_mask)); +} + +static void test_cpumask_iterators(struct kunit *test) +{ + EXPECT_FOR_EACH_CPU_EQ(test, &mask_empty); + EXPECT_FOR_EACH_CPU_NOT_EQ(test, &mask_empty); + EXPECT_FOR_EACH_CPU_WRAP_EQ(test, &mask_empty); + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, &mask_empty, &mask_empty); + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, &mask_empty); + EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, &mask_empty, &mask_empty); + + EXPECT_FOR_EACH_CPU_EQ(test, cpu_possible_mask); + EXPECT_FOR_EACH_CPU_NOT_EQ(test, cpu_possible_mask); + EXPECT_FOR_EACH_CPU_WRAP_EQ(test, cpu_possible_mask); + EXPECT_FOR_EACH_CPU_OP_EQ(test, and, cpu_possible_mask, cpu_possible_mask); + EXPECT_FOR_EACH_CPU_OP_EQ(test, andnot, cpu_possible_mask, &mask_empty); +} + +static void test_cpumask_iterators_builtin(struct kunit *test) +{ + EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, possible); + + /* Ensure the dynamic masks are stable while running the tests */ + cpu_hotplug_disable(); + + EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, online); + EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, present); + + cpu_hotplug_enable(); +} + +static int test_cpumask_init(struct kunit *test) +{ + cpumask_clear(&mask_empty); + cpumask_setall(&mask_all); + + return 0; +} + +static struct kunit_case test_cpumask_cases[] = { + KUNIT_CASE(test_cpumask_weight), + KUNIT_CASE(test_cpumask_first), + KUNIT_CASE(test_cpumask_last), + KUNIT_CASE(test_cpumask_next), + KUNIT_CASE(test_cpumask_iterators), + KUNIT_CASE(test_cpumask_iterators_builtin), + {} +}; + +static struct kunit_suite test_cpumask_suite = { + .name = "cpumask", + .init = test_cpumask_init, + .test_cases = test_cpumask_cases, +}; +kunit_test_suite(test_cpumask_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 9ff549f63540fa3a586382f89d8fcf3886c0940b..7e9683e9f5c6364514521d226c4e8429ed88088b 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -2,6 +2,9 @@ menu "Crypto library routines" +config CRYPTO_LIB_UTILS + tristate + config CRYPTO_LIB_AES tristate @@ -33,7 +36,7 @@ config CRYPTO_ARCH_HAVE_LIB_CHACHA config CRYPTO_LIB_CHACHA_GENERIC tristate - select XOR_BLOCKS + select CRYPTO_LIB_UTILS help This symbol can be depended upon by arch implementations of the ChaCha library interface that require the generic code as a @@ -43,7 +46,6 @@ config CRYPTO_LIB_CHACHA_GENERIC config CRYPTO_LIB_CHACHA tristate "ChaCha library interface" - depends on CRYPTO depends on CRYPTO_ARCH_HAVE_LIB_CHACHA || !CRYPTO_ARCH_HAVE_LIB_CHACHA select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n help @@ -71,7 +73,7 @@ config CRYPTO_LIB_CURVE25519 tristate "Curve25519 scalar multiplication library" depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519 select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n - select LIB_MEMNEQ + select CRYPTO_LIB_UTILS help Enable the Curve25519 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 919cbb2c220d617b8dd11deac579c2829e2d5c0c..c852f067ab06019e35205b29ecaeeb6fcc122ee4 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o +libcryptoutils-y := memneq.o utils.o + # chacha is used by the /dev/random driver which is always builtin obj-y += chacha.o obj-$(CONFIG_CRYPTO_LIB_CHACHA_GENERIC) += libchacha.o diff --git a/lib/memneq.c b/lib/crypto/memneq.c similarity index 98% rename from lib/memneq.c rename to lib/crypto/memneq.c index fb11608b1ec1ddaa58f3b6516698261c5395d44e..243d8677cc515da0c3c95c2fefaa95c648c2140a 100644 --- a/lib/memneq.c +++ b/lib/crypto/memneq.c @@ -59,10 +59,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include - -#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ +#include +#include /* Generic path for arbitrary size */ static inline unsigned long @@ -172,5 +171,3 @@ noinline unsigned long __crypto_memneq(const void *a, const void *b, } } EXPORT_SYMBOL(__crypto_memneq); - -#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ diff --git a/lib/crypto/utils.c b/lib/crypto/utils.c new file mode 100644 index 0000000000000000000000000000000000000000..53230ab1b19576382f02f5de75bbe6da9cf859b6 --- /dev/null +++ b/lib/crypto/utils.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Crypto library utility functions + * + * Copyright (c) 2006 Herbert Xu + */ + +#include +#include +#include + +/* + * XOR @len bytes from @src1 and @src2 together, writing the result to @dst + * (which may alias one of the sources). Don't call this directly; call + * crypto_xor() or crypto_xor_cpy() instead. + */ +void __crypto_xor(u8 *dst, const u8 *src1, const u8 *src2, unsigned int len) +{ + int relalign = 0; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + int size = sizeof(unsigned long); + int d = (((unsigned long)dst ^ (unsigned long)src1) | + ((unsigned long)dst ^ (unsigned long)src2)) & + (size - 1); + + relalign = d ? 1 << __ffs(d) : size; + + /* + * If we care about alignment, process as many bytes as + * needed to advance dst and src to values whose alignments + * equal their relative alignment. This will allow us to + * process the remainder of the input using optimal strides. + */ + while (((unsigned long)dst & (relalign - 1)) && len > 0) { + *dst++ = *src1++ ^ *src2++; + len--; + } + } + + while (IS_ENABLED(CONFIG_64BIT) && len >= 8 && !(relalign & 7)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u64 l = get_unaligned((u64 *)src1) ^ + get_unaligned((u64 *)src2); + put_unaligned(l, (u64 *)dst); + } else { + *(u64 *)dst = *(u64 *)src1 ^ *(u64 *)src2; + } + dst += 8; + src1 += 8; + src2 += 8; + len -= 8; + } + + while (len >= 4 && !(relalign & 3)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u32 l = get_unaligned((u32 *)src1) ^ + get_unaligned((u32 *)src2); + put_unaligned(l, (u32 *)dst); + } else { + *(u32 *)dst = *(u32 *)src1 ^ *(u32 *)src2; + } + dst += 4; + src1 += 4; + src2 += 4; + len -= 4; + } + + while (len >= 2 && !(relalign & 1)) { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) { + u16 l = get_unaligned((u16 *)src1) ^ + get_unaligned((u16 *)src2); + put_unaligned(l, (u16 *)dst); + } else { + *(u16 *)dst = *(u16 *)src1 ^ *(u16 *)src2; + } + dst += 2; + src1 += 2; + src2 += 2; + len -= 2; + } + + while (len--) + *dst++ = *src1++ ^ *src2++; +} +EXPORT_SYMBOL_GPL(__crypto_xor); + +MODULE_LICENSE("GPL"); diff --git a/lib/devres.c b/lib/devres.c index 55eb07e80cbb3b31e98c55209012dd2768859b3f..6baf43902eadb45e611bf4c4e19b47bb7f27de78 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -103,21 +103,6 @@ void __iomem *devm_ioremap_wc(struct device *dev, resource_size_t offset, } EXPORT_SYMBOL(devm_ioremap_wc); -/** - * devm_ioremap_np - Managed ioremap_np() - * @dev: Generic device to remap IO address for - * @offset: Resource address to map - * @size: Size of map - * - * Managed ioremap_np(). Map is automatically unmapped on driver detach. - */ -void __iomem *devm_ioremap_np(struct device *dev, resource_size_t offset, - resource_size_t size) -{ - return __devm_ioremap(dev, offset, size, DEVM_IOREMAP_NP); -} -EXPORT_SYMBOL(devm_ioremap_np); - /** * devm_iounmap - Managed iounmap() * @dev: Generic device to unmap for diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index dd7f56af9aed364b70b16ca51fe9377a229d3dcc..009f2ead09c1e0009479f999b36953dee4a02249 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -41,9 +41,11 @@ extern struct _ddebug __start___dyndbg[]; extern struct _ddebug __stop___dyndbg[]; +extern struct ddebug_class_map __start___dyndbg_classes[]; +extern struct ddebug_class_map __stop___dyndbg_classes[]; struct ddebug_table { - struct list_head link; + struct list_head link, maps; const char *mod_name; unsigned int num_ddebugs; struct _ddebug *ddebugs; @@ -54,12 +56,13 @@ struct ddebug_query { const char *module; const char *function; const char *format; + const char *class_string; unsigned int first_lineno, last_lineno; }; struct ddebug_iter { struct ddebug_table *table; - unsigned int idx; + int idx; }; struct flag_settings { @@ -134,15 +137,33 @@ static void vpr_info_dq(const struct ddebug_query *query, const char *msg) fmtlen--; } - v3pr_info("%s: func=\"%s\" file=\"%s\" module=\"%s\" format=\"%.*s\" lineno=%u-%u\n", - msg, - query->function ?: "", - query->filename ?: "", - query->module ?: "", - fmtlen, query->format ?: "", - query->first_lineno, query->last_lineno); + v3pr_info("%s: func=\"%s\" file=\"%s\" module=\"%s\" format=\"%.*s\" lineno=%u-%u class=%s\n", + msg, + query->function ?: "", + query->filename ?: "", + query->module ?: "", + fmtlen, query->format ?: "", + query->first_lineno, query->last_lineno, query->class_string); } +static struct ddebug_class_map *ddebug_find_valid_class(struct ddebug_table const *dt, + const char *class_string, int *class_id) +{ + struct ddebug_class_map *map; + int idx; + + list_for_each_entry(map, &dt->maps, link) { + idx = match_string(map->class_names, map->length, class_string); + if (idx >= 0) { + *class_id = idx + map->base; + return map; + } + } + *class_id = -ENOENT; + return NULL; +} + +#define __outvar /* filled by callee */ /* * Search the tables for _ddebug's which match the given `query' and * apply the `flags' and `mask' to them. Returns number of matching @@ -156,7 +177,9 @@ static int ddebug_change(const struct ddebug_query *query, struct ddebug_table *dt; unsigned int newflags; unsigned int nfound = 0; - struct flagsbuf fbuf; + struct flagsbuf fbuf, nbuf; + struct ddebug_class_map *map = NULL; + int __outvar valid_class; /* search for matching ddebugs */ mutex_lock(&ddebug_lock); @@ -167,9 +190,22 @@ static int ddebug_change(const struct ddebug_query *query, !match_wildcard(query->module, dt->mod_name)) continue; + if (query->class_string) { + map = ddebug_find_valid_class(dt, query->class_string, &valid_class); + if (!map) + continue; + } else { + /* constrain query, do not touch class'd callsites */ + valid_class = _DPRINTK_CLASS_DFLT; + } + for (i = 0; i < dt->num_ddebugs; i++) { struct _ddebug *dp = &dt->ddebugs[i]; + /* match site against query-class */ + if (dp->class_id != valid_class) + continue; + /* match against the source filename */ if (query->filename && !match_wildcard(query->filename, dp->filename) && @@ -211,16 +247,18 @@ static int ddebug_change(const struct ddebug_query *query, continue; #ifdef CONFIG_JUMP_LABEL if (dp->flags & _DPRINTK_FLAGS_PRINT) { - if (!(modifiers->flags & _DPRINTK_FLAGS_PRINT)) + if (!(newflags & _DPRINTK_FLAGS_PRINT)) static_branch_disable(&dp->key.dd_key_true); - } else if (modifiers->flags & _DPRINTK_FLAGS_PRINT) + } else if (newflags & _DPRINTK_FLAGS_PRINT) { static_branch_enable(&dp->key.dd_key_true); + } #endif + v4pr_info("changed %s:%d [%s]%s %s => %s\n", + trim_prefix(dp->filename), dp->lineno, + dt->mod_name, dp->function, + ddebug_describe_flags(dp->flags, &fbuf), + ddebug_describe_flags(newflags, &nbuf)); dp->flags = newflags; - v4pr_info("changed %s:%d [%s]%s =%s\n", - trim_prefix(dp->filename), dp->lineno, - dt->mod_name, dp->function, - ddebug_describe_flags(dp->flags, &fbuf)); } } mutex_unlock(&ddebug_lock); @@ -383,10 +421,6 @@ static int ddebug_parse_query(char *words[], int nwords, return -EINVAL; } - if (modname) - /* support $modname.dyndbg= */ - query->module = modname; - for (i = 0; i < nwords; i += 2) { char *keyword = words[i]; char *arg = words[i+1]; @@ -420,6 +454,8 @@ static int ddebug_parse_query(char *words[], int nwords, } else if (!strcmp(keyword, "line")) { if (parse_linerange(query, arg)) return -EINVAL; + } else if (!strcmp(keyword, "class")) { + rc = check_set(&query->class_string, arg, "class"); } else { pr_err("unknown keyword \"%s\"\n", keyword); return -EINVAL; @@ -427,6 +463,13 @@ static int ddebug_parse_query(char *words[], int nwords, if (rc) return rc; } + if (!query->module && modname) + /* + * support $modname.dyndbg=, when + * not given in the query itself + */ + query->module = modname; + vpr_info_dq(query, "parsed"); return 0; } @@ -553,34 +596,217 @@ static int ddebug_exec_queries(char *query, const char *modname) return nfound; } +/* apply a new bitmap to the sys-knob's current bit-state */ +static int ddebug_apply_class_bitmap(const struct ddebug_class_param *dcp, + unsigned long *new_bits, unsigned long *old_bits) +{ +#define QUERY_SIZE 128 + char query[QUERY_SIZE]; + const struct ddebug_class_map *map = dcp->map; + int matches = 0; + int bi, ct; + + v2pr_info("apply: 0x%lx to: 0x%lx\n", *new_bits, *old_bits); + + for (bi = 0; bi < map->length; bi++) { + if (test_bit(bi, new_bits) == test_bit(bi, old_bits)) + continue; + + snprintf(query, QUERY_SIZE, "class %s %c%s", map->class_names[bi], + test_bit(bi, new_bits) ? '+' : '-', dcp->flags); + + ct = ddebug_exec_queries(query, NULL); + matches += ct; + + v2pr_info("bit_%d: %d matches on class: %s -> 0x%lx\n", bi, + ct, map->class_names[bi], *new_bits); + } + return matches; +} + +/* stub to later conditionally add "$module." prefix where not already done */ +#define KP_NAME(kp) kp->name + +#define CLASSMAP_BITMASK(width) ((1UL << (width)) - 1) + +/* accept comma-separated-list of [+-] classnames */ +static int param_set_dyndbg_classnames(const char *instr, const struct kernel_param *kp) +{ + const struct ddebug_class_param *dcp = kp->arg; + const struct ddebug_class_map *map = dcp->map; + unsigned long curr_bits, old_bits; + char *cl_str, *p, *tmp; + int cls_id, totct = 0; + bool wanted; + + cl_str = tmp = kstrdup(instr, GFP_KERNEL); + p = strchr(cl_str, '\n'); + if (p) + *p = '\0'; + + /* start with previously set state-bits, then modify */ + curr_bits = old_bits = *dcp->bits; + vpr_info("\"%s\" > %s:0x%lx\n", cl_str, KP_NAME(kp), curr_bits); + + for (; cl_str; cl_str = p) { + p = strchr(cl_str, ','); + if (p) + *p++ = '\0'; + + if (*cl_str == '-') { + wanted = false; + cl_str++; + } else { + wanted = true; + if (*cl_str == '+') + cl_str++; + } + cls_id = match_string(map->class_names, map->length, cl_str); + if (cls_id < 0) { + pr_err("%s unknown to %s\n", cl_str, KP_NAME(kp)); + continue; + } + + /* have one or more valid class_ids of one *_NAMES type */ + switch (map->map_type) { + case DD_CLASS_TYPE_DISJOINT_NAMES: + /* the +/- pertains to a single bit */ + if (test_bit(cls_id, &curr_bits) == wanted) { + v3pr_info("no change on %s\n", cl_str); + continue; + } + curr_bits ^= BIT(cls_id); + totct += ddebug_apply_class_bitmap(dcp, &curr_bits, dcp->bits); + *dcp->bits = curr_bits; + v2pr_info("%s: changed bit %d:%s\n", KP_NAME(kp), cls_id, + map->class_names[cls_id]); + break; + case DD_CLASS_TYPE_LEVEL_NAMES: + /* cls_id = N in 0..max. wanted +/- determines N or N-1 */ + old_bits = CLASSMAP_BITMASK(*dcp->lvl); + curr_bits = CLASSMAP_BITMASK(cls_id + (wanted ? 1 : 0 )); + + totct += ddebug_apply_class_bitmap(dcp, &curr_bits, &old_bits); + *dcp->lvl = (cls_id + (wanted ? 1 : 0)); + v2pr_info("%s: changed bit-%d: \"%s\" %lx->%lx\n", KP_NAME(kp), cls_id, + map->class_names[cls_id], old_bits, curr_bits); + break; + default: + pr_err("illegal map-type value %d\n", map->map_type); + } + } + kfree(tmp); + vpr_info("total matches: %d\n", totct); + return 0; +} + /** - * dynamic_debug_exec_queries - select and change dynamic-debug prints - * @query: query-string described in admin-guide/dynamic-debug-howto - * @modname: string containing module name, usually &module.mod_name + * param_set_dyndbg_classes - class FOO >control + * @instr: string echo>d to sysfs, input depends on map_type + * @kp: kp->arg has state: bits/lvl, map, map_type + * + * Enable/disable prdbgs by their class, as given in the arguments to + * DECLARE_DYNDBG_CLASSMAP. For LEVEL map-types, enforce relative + * levels by bitpos. * - * This uses the >/proc/dynamic_debug/control reader, allowing module - * authors to modify their dynamic-debug callsites. The modname is - * canonically struct module.mod_name, but can also be null or a - * module-wildcard, for example: "drm*". + * Returns: 0 or <0 if error. */ -int dynamic_debug_exec_queries(const char *query, const char *modname) +int param_set_dyndbg_classes(const char *instr, const struct kernel_param *kp) { - int rc; - char *qry; /* writable copy of query */ - - if (!query) { - pr_err("non-null query/command string expected\n"); + const struct ddebug_class_param *dcp = kp->arg; + const struct ddebug_class_map *map = dcp->map; + unsigned long inrep, new_bits, old_bits; + int rc, totct = 0; + + switch (map->map_type) { + + case DD_CLASS_TYPE_DISJOINT_NAMES: + case DD_CLASS_TYPE_LEVEL_NAMES: + /* handle [+-]classnames list separately, we are done here */ + return param_set_dyndbg_classnames(instr, kp); + + case DD_CLASS_TYPE_DISJOINT_BITS: + case DD_CLASS_TYPE_LEVEL_NUM: + /* numeric input, accept and fall-thru */ + rc = kstrtoul(instr, 0, &inrep); + if (rc) { + pr_err("expecting numeric input: %s > %s\n", instr, KP_NAME(kp)); + return -EINVAL; + } + break; + default: + pr_err("%s: bad map type: %d\n", KP_NAME(kp), map->map_type); return -EINVAL; } - qry = kstrndup(query, PAGE_SIZE, GFP_KERNEL); - if (!qry) - return -ENOMEM; - rc = ddebug_exec_queries(qry, modname); - kfree(qry); - return rc; + /* only _BITS,_NUM (numeric) map-types get here */ + switch (map->map_type) { + case DD_CLASS_TYPE_DISJOINT_BITS: + /* expect bits. mask and warn if too many */ + if (inrep & ~CLASSMAP_BITMASK(map->length)) { + pr_warn("%s: input: 0x%lx exceeds mask: 0x%lx, masking\n", + KP_NAME(kp), inrep, CLASSMAP_BITMASK(map->length)); + inrep &= CLASSMAP_BITMASK(map->length); + } + v2pr_info("bits:%lx > %s\n", inrep, KP_NAME(kp)); + totct += ddebug_apply_class_bitmap(dcp, &inrep, dcp->bits); + *dcp->bits = inrep; + break; + case DD_CLASS_TYPE_LEVEL_NUM: + /* input is bitpos, of highest verbosity to be enabled */ + if (inrep > map->length) { + pr_warn("%s: level:%ld exceeds max:%d, clamping\n", + KP_NAME(kp), inrep, map->length); + inrep = map->length; + } + old_bits = CLASSMAP_BITMASK(*dcp->lvl); + new_bits = CLASSMAP_BITMASK(inrep); + v2pr_info("lvl:%ld bits:0x%lx > %s\n", inrep, new_bits, KP_NAME(kp)); + totct += ddebug_apply_class_bitmap(dcp, &new_bits, &old_bits); + *dcp->lvl = inrep; + break; + default: + pr_warn("%s: bad map type: %d\n", KP_NAME(kp), map->map_type); + } + vpr_info("%s: total matches: %d\n", KP_NAME(kp), totct); + return 0; +} +EXPORT_SYMBOL(param_set_dyndbg_classes); + +/** + * param_get_dyndbg_classes - classes reader + * @buffer: string description of controlled bits -> classes + * @kp: kp->arg has state: bits, map + * + * Reads last written state, underlying prdbg state may have been + * altered by direct >control. Displays 0x for DISJOINT, 0-N for + * LEVEL Returns: #chars written or <0 on error + */ +int param_get_dyndbg_classes(char *buffer, const struct kernel_param *kp) +{ + const struct ddebug_class_param *dcp = kp->arg; + const struct ddebug_class_map *map = dcp->map; + + switch (map->map_type) { + + case DD_CLASS_TYPE_DISJOINT_NAMES: + case DD_CLASS_TYPE_DISJOINT_BITS: + return scnprintf(buffer, PAGE_SIZE, "0x%lx\n", *dcp->bits); + + case DD_CLASS_TYPE_LEVEL_NAMES: + case DD_CLASS_TYPE_LEVEL_NUM: + return scnprintf(buffer, PAGE_SIZE, "%d\n", *dcp->lvl); + default: + return -1; + } } -EXPORT_SYMBOL_GPL(dynamic_debug_exec_queries); +EXPORT_SYMBOL(param_get_dyndbg_classes); + +const struct kernel_param_ops param_ops_dyndbg_classes = { + .set = param_set_dyndbg_classes, + .get = param_get_dyndbg_classes, +}; +EXPORT_SYMBOL(param_ops_dyndbg_classes); #define PREFIX_SIZE 64 @@ -803,13 +1029,12 @@ static struct _ddebug *ddebug_iter_first(struct ddebug_iter *iter) { if (list_empty(&ddebug_tables)) { iter->table = NULL; - iter->idx = 0; return NULL; } iter->table = list_entry(ddebug_tables.next, struct ddebug_table, link); - iter->idx = 0; - return &iter->table->ddebugs[iter->idx]; + iter->idx = iter->table->num_ddebugs; + return &iter->table->ddebugs[--iter->idx]; } /* @@ -822,15 +1047,16 @@ static struct _ddebug *ddebug_iter_next(struct ddebug_iter *iter) { if (iter->table == NULL) return NULL; - if (++iter->idx == iter->table->num_ddebugs) { + if (--iter->idx < 0) { /* iterate to next table */ - iter->idx = 0; if (list_is_last(&iter->table->link, &ddebug_tables)) { iter->table = NULL; return NULL; } iter->table = list_entry(iter->table->link.next, struct ddebug_table, link); + iter->idx = iter->table->num_ddebugs; + --iter->idx; } return &iter->table->ddebugs[iter->idx]; } @@ -876,6 +1102,20 @@ static void *ddebug_proc_next(struct seq_file *m, void *p, loff_t *pos) return dp; } +#define class_in_range(class_id, map) \ + (class_id >= map->base && class_id < map->base + map->length) + +static const char *ddebug_class_name(struct ddebug_iter *iter, struct _ddebug *dp) +{ + struct ddebug_class_map *map; + + list_for_each_entry(map, &iter->table->maps, link) + if (class_in_range(dp->class_id, map)) + return map->class_names[dp->class_id - map->base]; + + return NULL; +} + /* * Seq_ops show method. Called several times within a read() * call from userspace, with ddebug_lock held. Formats the @@ -887,6 +1127,7 @@ static int ddebug_proc_show(struct seq_file *m, void *p) struct ddebug_iter *iter = m->private; struct _ddebug *dp = p; struct flagsbuf flags; + char const *class; if (p == SEQ_START_TOKEN) { seq_puts(m, @@ -898,8 +1139,17 @@ static int ddebug_proc_show(struct seq_file *m, void *p) trim_prefix(dp->filename), dp->lineno, iter->table->mod_name, dp->function, ddebug_describe_flags(dp->flags, &flags)); - seq_escape(m, dp->format, "\t\r\n\""); - seq_puts(m, "\"\n"); + seq_escape_str(m, dp->format, ESCAPE_SPACE, "\t\r\n\""); + seq_puts(m, "\""); + + if (dp->class_id != _DPRINTK_CLASS_DFLT) { + class = ddebug_class_name(iter, dp); + if (class) + seq_printf(m, " class:%s", class); + else + seq_printf(m, " class unknown, _id:%d", dp->class_id); + } + seq_puts(m, "\n"); return 0; } @@ -943,18 +1193,50 @@ static const struct proc_ops proc_fops = { .proc_write = ddebug_proc_write }; +static void ddebug_attach_module_classes(struct ddebug_table *dt, + struct ddebug_class_map *classes, + int num_classes) +{ + struct ddebug_class_map *cm; + int i, j, ct = 0; + + for (cm = classes, i = 0; i < num_classes; i++, cm++) { + + if (!strcmp(cm->mod_name, dt->mod_name)) { + + v2pr_info("class[%d]: module:%s base:%d len:%d ty:%d\n", i, + cm->mod_name, cm->base, cm->length, cm->map_type); + + for (j = 0; j < cm->length; j++) + v3pr_info(" %d: %d %s\n", j + cm->base, j, + cm->class_names[j]); + + list_add(&cm->link, &dt->maps); + ct++; + } + } + if (ct) + vpr_info("module:%s attached %d classes\n", dt->mod_name, ct); +} + /* * Allocate a new ddebug_table for the given module * and add it to the global list. */ -int ddebug_add_module(struct _ddebug *tab, unsigned int n, - const char *name) +static int __ddebug_add_module(struct _ddebug_info *di, unsigned int base, + const char *modname) { struct ddebug_table *dt; + v3pr_info("add-module: %s.%d sites\n", modname, di->num_descs); + if (!di->num_descs) { + v3pr_info(" skip %s\n", modname); + return 0; + } + dt = kzalloc(sizeof(*dt), GFP_KERNEL); if (dt == NULL) { - pr_err("error adding module: %s\n", name); + pr_err("error adding module: %s\n", modname); return -ENOMEM; } /* @@ -963,18 +1245,29 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n, * member of struct module, which lives at least as long as * this struct ddebug_table. */ - dt->mod_name = name; - dt->num_ddebugs = n; - dt->ddebugs = tab; + dt->mod_name = modname; + dt->ddebugs = di->descs; + dt->num_ddebugs = di->num_descs; + + INIT_LIST_HEAD(&dt->link); + INIT_LIST_HEAD(&dt->maps); + + if (di->classes && di->num_classes) + ddebug_attach_module_classes(dt, di->classes, di->num_classes); mutex_lock(&ddebug_lock); - list_add(&dt->link, &ddebug_tables); + list_add_tail(&dt->link, &ddebug_tables); mutex_unlock(&ddebug_lock); - vpr_info("%3u debug prints in module %s\n", n, dt->mod_name); + vpr_info("%3u debug prints in module %s\n", di->num_descs, modname); return 0; } +int ddebug_add_module(struct _ddebug_info *di, const char *modname) +{ + return __ddebug_add_module(di, 0, modname); +} + /* helper for ddebug_dyndbg_(boot|module)_param_cb */ static int ddebug_dyndbg_param_cb(char *param, char *val, const char *modname, int on_err) @@ -1083,11 +1376,17 @@ static int __init dynamic_debug_init_control(void) static int __init dynamic_debug_init(void) { - struct _ddebug *iter, *iter_start; - const char *modname = NULL; + struct _ddebug *iter, *iter_mod_start; + int ret, i, mod_sites, mod_ct; + const char *modname; char *cmdline; - int ret = 0; - int n = 0, entries = 0, modct = 0; + + struct _ddebug_info di = { + .descs = __start___dyndbg, + .classes = __start___dyndbg_classes, + .num_descs = __stop___dyndbg - __start___dyndbg, + .num_classes = __stop___dyndbg_classes - __start___dyndbg_classes, + }; if (&__start___dyndbg == &__stop___dyndbg) { if (IS_ENABLED(CONFIG_DYNAMIC_DEBUG)) { @@ -1098,30 +1397,39 @@ static int __init dynamic_debug_init(void) ddebug_init_success = 1; return 0; } - iter = __start___dyndbg; + + iter = iter_mod_start = __start___dyndbg; modname = iter->modname; - iter_start = iter; - for (; iter < __stop___dyndbg; iter++) { - entries++; + i = mod_sites = mod_ct = 0; + + for (; iter < __stop___dyndbg; iter++, i++, mod_sites++) { + if (strcmp(modname, iter->modname)) { - modct++; - ret = ddebug_add_module(iter_start, n, modname); + mod_ct++; + di.num_descs = mod_sites; + di.descs = iter_mod_start; + ret = __ddebug_add_module(&di, i - mod_sites, modname); if (ret) goto out_err; - n = 0; + + mod_sites = 0; modname = iter->modname; - iter_start = iter; + iter_mod_start = iter; } - n++; } - ret = ddebug_add_module(iter_start, n, modname); + di.num_descs = mod_sites; + di.descs = iter_mod_start; + ret = __ddebug_add_module(&di, i - mod_sites, modname); if (ret) goto out_err; ddebug_init_success = 1; vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n", - entries, modct, (int)((modct * sizeof(struct ddebug_table)) >> 10), - (int)((entries * sizeof(struct _ddebug)) >> 10)); + i, mod_ct, (int)((mod_ct * sizeof(struct ddebug_table)) >> 10), + (int)((i * sizeof(struct _ddebug)) >> 10)); + + if (di.num_classes) + v2pr_info(" %d builtin ddebug class-maps\n", di.num_classes); /* now that ddebug tables are loaded, process all boot args * again to find and activate queries given in dyndbg params. diff --git a/lib/earlycpio.c b/lib/earlycpio.c index 7921193f042439b783ad4d303040067939d9e6d3..d2c37d64fd0c390311fee22ab2a75de1d4fd5d56 100644 --- a/lib/earlycpio.c +++ b/lib/earlycpio.c @@ -126,7 +126,7 @@ struct cpio_data find_cpio_data(const char *path, void *data, "File %s exceeding MAX_CPIO_FILE_NAME [%d]\n", p, MAX_CPIO_FILE_NAME); } - strlcpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME); + strscpy(cd.name, p + mypathsize, MAX_CPIO_FILE_NAME); cd.data = (void *)dptr; cd.size = ch[C_FILESIZE]; diff --git a/lib/fault-inject.c b/lib/fault-inject.c index 423784d9c058eaf7d8833eade65bd389bd6a993e..96e092de5b72364205085aedba1e30909a3046d6 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -139,7 +139,7 @@ bool should_fail(struct fault_attr *attr, ssize_t size) return false; } - if (attr->probability <= prandom_u32() % 100) + if (attr->probability <= prandom_u32_max(100)) return false; if (!fail_stacktrace(attr)) diff --git a/lib/find_bit.c b/lib/find_bit.c index 1b8e4b2a9cba8298893b6f3f4b4f3ebd928870d4..18bc0a7ac8eed9807962024df4cd421ac4272c7e 100644 --- a/lib/find_bit.c +++ b/lib/find_bit.c @@ -19,57 +19,78 @@ #include #include -#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ - !defined(find_next_bit_le) || !defined(find_next_zero_bit_le) || \ - !defined(find_next_and_bit) /* - * This is a common helper function for find_next_bit, find_next_zero_bit, and - * find_next_and_bit. The differences are: - * - The "invert" argument, which is XORed with each fetched word before - * searching it for one bits. - * - The optional "addr2", which is anded with "addr1" if present. + * Common helper for find_bit() function family + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) + * @size: The bitmap size in bits */ -unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le) -{ - unsigned long tmp, mask; - - if (unlikely(start >= nbits)) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - - /* Handle 1st word. */ - mask = BITMAP_FIRST_WORD_MASK(start); - if (le) - mask = swab(mask); - - tmp &= mask; - - start = round_down(start, BITS_PER_LONG); - - while (!tmp) { - start += BITS_PER_LONG; - if (start >= nbits) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - } +#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ +({ \ + unsigned long idx, val, sz = (size); \ + \ + for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ + val = (FETCH); \ + if (val) { \ + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ + break; \ + } \ + } \ + \ + sz; \ +}) - if (le) - tmp = swab(tmp); - - return min(start + __ffs(tmp), nbits); -} -EXPORT_SYMBOL(_find_next_bit); -#endif +/* + * Common helper for find_next_bit() function family + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) + * @size: The bitmap size in bits + * @start: The bitnumber to start searching at + */ +#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ +({ \ + unsigned long mask, idx, tmp, sz = (size), __start = (start); \ + \ + if (unlikely(__start >= sz)) \ + goto out; \ + \ + mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ + idx = __start / BITS_PER_LONG; \ + \ + for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ + if ((idx + 1) * BITS_PER_LONG >= sz) \ + goto out; \ + idx++; \ + } \ + \ + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ +out: \ + sz; \ +}) + +#define FIND_NTH_BIT(FETCH, size, num) \ +({ \ + unsigned long sz = (size), nr = (num), idx, w, tmp; \ + \ + for (idx = 0; (idx + 1) * BITS_PER_LONG <= sz; idx++) { \ + if (idx * BITS_PER_LONG + nr >= sz) \ + goto out; \ + \ + tmp = (FETCH); \ + w = hweight_long(tmp); \ + if (w > nr) \ + goto found; \ + \ + nr -= w; \ + } \ + \ + if (sz % BITS_PER_LONG) \ + tmp = (FETCH) & BITMAP_LAST_WORD_MASK(sz); \ +found: \ + sz = min(idx * BITS_PER_LONG + fns(tmp, nr), sz); \ +out: \ + sz; \ +}) #ifndef find_first_bit /* @@ -77,14 +98,7 @@ EXPORT_SYMBOL(_find_next_bit); */ unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx]) - return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); - } - - return size; + return FIND_FIRST_BIT(addr[idx], /* nop */, size); } EXPORT_SYMBOL(_find_first_bit); #endif @@ -97,15 +111,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size) { - unsigned long idx, val; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - val = addr1[idx] & addr2[idx]; - if (val) - return min(idx * BITS_PER_LONG + __ffs(val), size); - } - - return size; + return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); } EXPORT_SYMBOL(_find_first_and_bit); #endif @@ -116,16 +122,64 @@ EXPORT_SYMBOL(_find_first_and_bit); */ unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; + return FIND_FIRST_BIT(~addr[idx], /* nop */, size); +} +EXPORT_SYMBOL(_find_first_zero_bit); +#endif - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx] != ~0UL) - return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); - } +#ifndef find_next_bit +unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_bit); +#endif - return size; +unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n) +{ + return FIND_NTH_BIT(addr[idx], size, n); } -EXPORT_SYMBOL(_find_first_zero_bit); +EXPORT_SYMBOL(__find_nth_bit); + +unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n) +{ + return FIND_NTH_BIT(addr1[idx] & addr2[idx], size, n); +} +EXPORT_SYMBOL(__find_nth_and_bit); + +unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long size, unsigned long n) +{ + return FIND_NTH_BIT(addr1[idx] & ~addr2[idx], size, n); +} +EXPORT_SYMBOL(__find_nth_andnot_bit); + +#ifndef find_next_and_bit +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_and_bit); +#endif + +#ifndef find_next_andnot_bit +unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr1[idx] & ~addr2[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_andnot_bit); +#endif + +#ifndef find_next_zero_bit +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start) +{ + return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); +} +EXPORT_SYMBOL(_find_next_zero_bit); #endif #ifndef find_last_bit @@ -161,3 +215,38 @@ unsigned long find_next_clump8(unsigned long *clump, const unsigned long *addr, return offset; } EXPORT_SYMBOL(find_next_clump8); + +#ifdef __BIG_ENDIAN + +#ifndef find_first_zero_bit_le +/* + * Find the first cleared bit in an LE memory region. + */ +unsigned long _find_first_zero_bit_le(const unsigned long *addr, unsigned long size) +{ + return FIND_FIRST_BIT(~addr[idx], swab, size); +} +EXPORT_SYMBOL(_find_first_zero_bit_le); + +#endif + +#ifndef find_next_zero_bit_le +unsigned long _find_next_zero_bit_le(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + return FIND_NEXT_BIT(~addr[idx], swab, size, offset); +} +EXPORT_SYMBOL(_find_next_zero_bit_le); +#endif + +#ifndef find_next_bit_le +unsigned long _find_next_bit_le(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + return FIND_NEXT_BIT(addr[idx], swab, size, offset); +} +EXPORT_SYMBOL(_find_next_bit_le); + +#endif + +#endif /* __BIG_ENDIAN */ diff --git a/lib/find_bit_benchmark.c b/lib/find_bit_benchmark.c index db904b57d4b87ea68beb69c0e218bb512f5df891..7c3c011abd29489d7914f5d3e87aaa90c7c24e14 100644 --- a/lib/find_bit_benchmark.c +++ b/lib/find_bit_benchmark.c @@ -115,6 +115,22 @@ static int __init test_find_last_bit(const void *bitmap, unsigned long len) return 0; } +static int __init test_find_nth_bit(const unsigned long *bitmap, unsigned long len) +{ + unsigned long l, n, w = bitmap_weight(bitmap, len); + ktime_t time; + + time = ktime_get(); + for (n = 0; n < w; n++) { + l = find_nth_bit(bitmap, len, n); + WARN_ON(l >= len); + } + time = ktime_get() - time; + pr_err("find_nth_bit: %18llu ns, %6ld iterations\n", time, w); + + return 0; +} + static int __init test_find_next_and_bit(const void *bitmap, const void *bitmap2, unsigned long len) { @@ -142,6 +158,7 @@ static int __init find_bit_test(void) test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); + test_find_nth_bit(bitmap, BITMAP_LEN / 10); /* * test_find_first_bit() may take some time, so @@ -157,13 +174,14 @@ static int __init find_bit_test(void) bitmap_zero(bitmap2, BITMAP_LEN); while (nbits--) { - __set_bit(prandom_u32() % BITMAP_LEN, bitmap); - __set_bit(prandom_u32() % BITMAP_LEN, bitmap2); + __set_bit(prandom_u32_max(BITMAP_LEN), bitmap); + __set_bit(prandom_u32_max(BITMAP_LEN), bitmap2); } test_find_next_bit(bitmap, BITMAP_LEN); test_find_next_zero_bit(bitmap, BITMAP_LEN); test_find_last_bit(bitmap, BITMAP_LEN); + test_find_nth_bit(bitmap, BITMAP_LEN); test_find_first_bit(bitmap, BITMAP_LEN); test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN); test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN); diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c index 05cccbcf1661a360973bfa4169e57fdb02112bc0..83332fefa6f42e5e57434d214594e7c2306b1a0d 100644 --- a/lib/flex_proportions.c +++ b/lib/flex_proportions.c @@ -70,6 +70,7 @@ bool fprop_new_period(struct fprop_global *p, int periods) */ if (events <= 1) return false; + preempt_disable_nested(); write_seqcount_begin(&p->sequence); if (periods < 64) events -= events >> periods; @@ -77,6 +78,7 @@ bool fprop_new_period(struct fprop_global *p, int periods) percpu_counter_add(&p->events, -events); p->period += periods; write_seqcount_end(&p->sequence); + preempt_enable_nested(); return true; } diff --git a/lib/fortify_kunit.c b/lib/fortify_kunit.c new file mode 100644 index 0000000000000000000000000000000000000000..409af07f340af76352be862d21a2358b27e91619 --- /dev/null +++ b/lib/fortify_kunit.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Runtime test cases for CONFIG_FORTIFY_SOURCE that aren't expected to + * Oops the kernel on success. (For those, see drivers/misc/lkdtm/fortify.c) + * + * For corner cases with UBSAN, try testing with: + * + * ./tools/testing/kunit/kunit.py run --arch=x86_64 \ + * --kconfig_add CONFIG_FORTIFY_SOURCE=y \ + * --kconfig_add CONFIG_UBSAN=y \ + * --kconfig_add CONFIG_UBSAN_TRAP=y \ + * --kconfig_add CONFIG_UBSAN_BOUNDS=y \ + * --kconfig_add CONFIG_UBSAN_LOCAL_BOUNDS=y \ + * --make_options LLVM=1 fortify + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +static const char array_of_10[] = "this is 10"; +static const char *ptr_of_11 = "this is 11!"; +static char array_unknown[] = "compiler thinks I might change"; + +static void known_sizes_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, __compiletime_strlen("88888888"), 8); + KUNIT_EXPECT_EQ(test, __compiletime_strlen(array_of_10), 10); + KUNIT_EXPECT_EQ(test, __compiletime_strlen(ptr_of_11), 11); + + KUNIT_EXPECT_EQ(test, __compiletime_strlen(array_unknown), SIZE_MAX); + /* Externally defined and dynamically sized string pointer: */ + KUNIT_EXPECT_EQ(test, __compiletime_strlen(test->name), SIZE_MAX); +} + +/* This is volatile so the optimizer can't perform DCE below. */ +static volatile int pick; + +/* Not inline to keep optimizer from figuring out which string we want. */ +static noinline size_t want_minus_one(int pick) +{ + const char *str; + + switch (pick) { + case 1: + str = "4444"; + break; + case 2: + str = "333"; + break; + default: + str = "1"; + break; + } + return __compiletime_strlen(str); +} + +static void control_flow_split_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, want_minus_one(pick), SIZE_MAX); +} + +static struct kunit_case fortify_test_cases[] = { + KUNIT_CASE(known_sizes_test), + KUNIT_CASE(control_flow_split_test), + {} +}; + +static struct kunit_suite fortify_test_suite = { + .name = "fortify", + .test_cases = fortify_test_cases, +}; + +kunit_test_suite(fortify_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/iomap.c b/lib/iomap.c index fbaa3e8f19d6cfa54fcd33ca46be643e20d83f4c..4f8b31baa5752aa1f2a58bf422a97bf50a153f60 100644 --- a/lib/iomap.c +++ b/lib/iomap.c @@ -6,6 +6,7 @@ */ #include #include +#include #include @@ -70,26 +71,35 @@ static void bad_io_access(unsigned long port, const char *access) #define mmio_read64be(addr) swab64(readq(addr)) #endif +/* + * Here and below, we apply __no_kmsan_checks to functions reading data from + * hardware, to ensure that KMSAN marks their return values as initialized. + */ +__no_kmsan_checks unsigned int ioread8(const void __iomem *addr) { IO_COND(addr, return inb(port), return readb(addr)); return 0xff; } +__no_kmsan_checks unsigned int ioread16(const void __iomem *addr) { IO_COND(addr, return inw(port), return readw(addr)); return 0xffff; } +__no_kmsan_checks unsigned int ioread16be(const void __iomem *addr) { IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr)); return 0xffff; } +__no_kmsan_checks unsigned int ioread32(const void __iomem *addr) { IO_COND(addr, return inl(port), return readl(addr)); return 0xffffffff; } +__no_kmsan_checks unsigned int ioread32be(const void __iomem *addr) { IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr)); @@ -142,18 +152,21 @@ static u64 pio_read64be_hi_lo(unsigned long port) return lo | (hi << 32); } +__no_kmsan_checks u64 ioread64_lo_hi(const void __iomem *addr) { IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr)); return 0xffffffffffffffffULL; } +__no_kmsan_checks u64 ioread64_hi_lo(const void __iomem *addr) { IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr)); return 0xffffffffffffffffULL; } +__no_kmsan_checks u64 ioread64be_lo_hi(const void __iomem *addr) { IO_COND(addr, return pio_read64be_lo_hi(port), @@ -161,6 +174,7 @@ u64 ioread64be_lo_hi(const void __iomem *addr) return 0xffffffffffffffffULL; } +__no_kmsan_checks u64 ioread64be_hi_lo(const void __iomem *addr) { IO_COND(addr, return pio_read64be_hi_lo(port), @@ -188,22 +202,32 @@ EXPORT_SYMBOL(ioread64be_hi_lo); void iowrite8(u8 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, outb(val,port), writeb(val, addr)); } void iowrite16(u16 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, outw(val,port), writew(val, addr)); } void iowrite16be(u16 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr)); } void iowrite32(u32 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, outl(val,port), writel(val, addr)); } void iowrite32be(u32 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr)); } EXPORT_SYMBOL(iowrite8); @@ -239,24 +263,32 @@ static void pio_write64be_hi_lo(u64 val, unsigned long port) void iowrite64_lo_hi(u64 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write64_lo_hi(val, port), writeq(val, addr)); } void iowrite64_hi_lo(u64 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write64_hi_lo(val, port), writeq(val, addr)); } void iowrite64be_lo_hi(u64 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write64be_lo_hi(val, port), mmio_write64be(val, addr)); } void iowrite64be_hi_lo(u64 val, void __iomem *addr) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(&val, sizeof(val)); IO_COND(addr, pio_write64be_hi_lo(val, port), mmio_write64be(val, addr)); } @@ -328,14 +360,20 @@ static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count) void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count)); + /* KMSAN must treat values read from devices as initialized. */ + kmsan_unpoison_memory(dst, count); } void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count)); + /* KMSAN must treat values read from devices as initialized. */ + kmsan_unpoison_memory(dst, count * 2); } void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count) { IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count)); + /* KMSAN must treat values read from devices as initialized. */ + kmsan_unpoison_memory(dst, count * 4); } EXPORT_SYMBOL(ioread8_rep); EXPORT_SYMBOL(ioread16_rep); @@ -343,14 +381,20 @@ EXPORT_SYMBOL(ioread32_rep); void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(src, count); IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count)); } void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(src, count * 2); IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count)); } void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count) { + /* Make sure uninitialized memory isn't copied to devices. */ + kmsan_check_memory(src, count * 4); IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count)); } EXPORT_SYMBOL(iowrite8_rep); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 4b7fce72e3e521348851c56f4a6335f8728c07b4..c3ca28ca68a65d70865d19accea5f07a9ff39b4b 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -174,13 +174,16 @@ static int copyout(void __user *to, const void *from, size_t n) static int copyin(void *to, const void __user *from, size_t n) { + size_t res = n; + if (should_fail_usercopy()) return n; if (access_ok(from, n)) { - instrument_copy_from_user(to, from, n); - n = raw_copy_from_user(to, from, n); + instrument_copy_from_user_before(to, from, n); + res = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, res); } - return n; + return res; } static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe, diff --git a/lib/is_signed_type_kunit.c b/lib/is_signed_type_kunit.c new file mode 100644 index 0000000000000000000000000000000000000000..207207522925e4a034126e91ae2595ba41c620c4 --- /dev/null +++ b/lib/is_signed_type_kunit.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * ./tools/testing/kunit/kunit.py run is_signed_type [--raw_output] + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +enum unsigned_enum { + constant_a = 3, +}; + +enum signed_enum { + constant_b = -1, + constant_c = 2, +}; + +static void is_signed_type_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, is_signed_type(bool), false); + KUNIT_EXPECT_EQ(test, is_signed_type(signed char), true); + KUNIT_EXPECT_EQ(test, is_signed_type(unsigned char), false); +#ifdef __CHAR_UNSIGNED__ + KUNIT_EXPECT_EQ(test, is_signed_type(char), false); +#else + KUNIT_EXPECT_EQ(test, is_signed_type(char), true); +#endif + KUNIT_EXPECT_EQ(test, is_signed_type(int), true); + KUNIT_EXPECT_EQ(test, is_signed_type(unsigned int), false); + KUNIT_EXPECT_EQ(test, is_signed_type(long), true); + KUNIT_EXPECT_EQ(test, is_signed_type(unsigned long), false); + KUNIT_EXPECT_EQ(test, is_signed_type(long long), true); + KUNIT_EXPECT_EQ(test, is_signed_type(unsigned long long), false); + KUNIT_EXPECT_EQ(test, is_signed_type(enum unsigned_enum), false); + KUNIT_EXPECT_EQ(test, is_signed_type(enum signed_enum), true); + KUNIT_EXPECT_EQ(test, is_signed_type(void *), false); + KUNIT_EXPECT_EQ(test, is_signed_type(const char *), false); +} + +static struct kunit_case is_signed_type_test_cases[] = { + KUNIT_CASE(is_signed_type_test), + {} +}; + +static struct kunit_suite is_signed_type_test_suite = { + .name = "is_signed_type", + .test_cases = is_signed_type_test_cases, +}; + +kunit_test_suite(is_signed_type_test_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/lib/kobject.c b/lib/kobject.c index 5f0e71ab292cb07e3650a09665afe977d149c7df..a0b2dbfcfa23347d6e546fbb79da78ba21e16c79 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -694,7 +694,7 @@ static void kobject_release(struct kref *kref) { struct kobject *kobj = container_of(kref, struct kobject, kref); #ifdef CONFIG_DEBUG_KOBJECT_RELEASE - unsigned long delay = HZ + HZ * (get_random_int() & 0x3); + unsigned long delay = HZ + HZ * prandom_u32_max(4); pr_info("kobject: '%s' (%p): %s, parent %p (delayed %ld)\n", kobject_name(kobj), kobj, __func__, kobj->parent, delay); INIT_DELAYED_WORK(&kobj->release, kobject_delayed_cleanup); diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig index 0b5dfb001bacc1b173f6fb8b2b3f944d6d891b01..626719b95badd59a761082d78676734624fb65b6 100644 --- a/lib/kunit/Kconfig +++ b/lib/kunit/Kconfig @@ -59,4 +59,15 @@ config KUNIT_ALL_TESTS If unsure, say N. +config KUNIT_DEFAULT_ENABLED + bool "Default value of kunit.enable" + default y + help + Sets the default value of kunit.enable. If set to N then KUnit + tests will not execute unless kunit.enable=1 is passed to the + kernel command line. + + In most cases this should be left as Y. Only if additional opt-in + behavior is needed should this be set to N. + endif # KUNIT diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 5e223327196a6d8493c921c040446c76cd313999..9bbc422c284bfb921823d0b45c6639883f394478 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -190,6 +190,10 @@ int kunit_run_all_tests(void) { struct suite_set suite_set = {__kunit_suites_start, __kunit_suites_end}; int err = 0; + if (!kunit_enabled()) { + pr_info("kunit: disabled\n"); + goto out; + } if (filter_glob_param) { suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err); diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c index 13d0bd8b07a98fd5973ecbe34902a03bf8443945..4df0335d0d06edb874820f37128c91013152e2c3 100644 --- a/lib/kunit/kunit-test.c +++ b/lib/kunit/kunit-test.c @@ -161,6 +161,13 @@ static void kunit_resource_test_alloc_resource(struct kunit *test) kunit_put_resource(res); } +static inline bool kunit_resource_instance_match(struct kunit *test, + struct kunit_resource *res, + void *match_data) +{ + return res->data == match_data; +} + /* * Note: tests below use kunit_alloc_and_get_resource(), so as a consequence * they have a reference to the associated resource that they must release diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index 141789ca8949b45d4481ecbaed86c1529f718bde..f5ae79c374003ebcb58cef9a3fc2c2b62320717b 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -12,62 +12,29 @@ #include "string-stream.h" -struct string_stream_fragment_alloc_context { - struct kunit *test; - int len; - gfp_t gfp; -}; -static int string_stream_fragment_init(struct kunit_resource *res, - void *context) +static struct string_stream_fragment *alloc_string_stream_fragment( + struct kunit *test, int len, gfp_t gfp) { - struct string_stream_fragment_alloc_context *ctx = context; struct string_stream_fragment *frag; - frag = kunit_kzalloc(ctx->test, sizeof(*frag), ctx->gfp); + frag = kunit_kzalloc(test, sizeof(*frag), gfp); if (!frag) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - frag->test = ctx->test; - frag->fragment = kunit_kmalloc(ctx->test, ctx->len, ctx->gfp); + frag->fragment = kunit_kmalloc(test, len, gfp); if (!frag->fragment) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - res->data = frag; - - return 0; + return frag; } -static void string_stream_fragment_free(struct kunit_resource *res) +static void string_stream_fragment_destroy(struct kunit *test, + struct string_stream_fragment *frag) { - struct string_stream_fragment *frag = res->data; - list_del(&frag->node); - kunit_kfree(frag->test, frag->fragment); - kunit_kfree(frag->test, frag); -} - -static struct string_stream_fragment *alloc_string_stream_fragment( - struct kunit *test, int len, gfp_t gfp) -{ - struct string_stream_fragment_alloc_context context = { - .test = test, - .len = len, - .gfp = gfp - }; - - return kunit_alloc_resource(test, - string_stream_fragment_init, - string_stream_fragment_free, - gfp, - &context); -} - -static int string_stream_fragment_destroy(struct string_stream_fragment *frag) -{ - return kunit_destroy_resource(frag->test, - kunit_resource_instance_match, - frag); + kunit_kfree(test, frag->fragment); + kunit_kfree(test, frag); } int string_stream_vadd(struct string_stream *stream, @@ -122,7 +89,7 @@ static void string_stream_clear(struct string_stream *stream) frag_container_safe, &stream->fragments, node) { - string_stream_fragment_destroy(frag_container); + string_stream_fragment_destroy(stream->test, frag_container); } stream->length = 0; spin_unlock(&stream->lock); @@ -169,48 +136,23 @@ struct string_stream_alloc_context { gfp_t gfp; }; -static int string_stream_init(struct kunit_resource *res, void *context) +struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp) { struct string_stream *stream; - struct string_stream_alloc_context *ctx = context; - stream = kunit_kzalloc(ctx->test, sizeof(*stream), ctx->gfp); + stream = kunit_kzalloc(test, sizeof(*stream), gfp); if (!stream) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - res->data = stream; - stream->gfp = ctx->gfp; - stream->test = ctx->test; + stream->gfp = gfp; + stream->test = test; INIT_LIST_HEAD(&stream->fragments); spin_lock_init(&stream->lock); - return 0; + return stream; } -static void string_stream_free(struct kunit_resource *res) +void string_stream_destroy(struct string_stream *stream) { - struct string_stream *stream = res->data; - string_stream_clear(stream); } - -struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp) -{ - struct string_stream_alloc_context context = { - .test = test, - .gfp = gfp - }; - - return kunit_alloc_resource(test, - string_stream_init, - string_stream_free, - gfp, - &context); -} - -int string_stream_destroy(struct string_stream *stream) -{ - return kunit_destroy_resource(stream->test, - kunit_resource_instance_match, - stream); -} diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h index 43f9508a55b40f883280ddf0bf63109a86ed236e..b669f9a75a9489237ecd463b9ec356c0eda82d26 100644 --- a/lib/kunit/string-stream.h +++ b/lib/kunit/string-stream.h @@ -14,7 +14,6 @@ #include struct string_stream_fragment { - struct kunit *test; struct list_head node; char *fragment; }; @@ -46,6 +45,6 @@ int string_stream_append(struct string_stream *stream, bool string_stream_is_empty(struct string_stream *stream); -int string_stream_destroy(struct string_stream *stream); +void string_stream_destroy(struct string_stream *stream); #endif /* _KUNIT_STRING_STREAM_H */ diff --git a/lib/kunit/test.c b/lib/kunit/test.c index b73d5bb5c47381e232861a43d28f9e16d727cc13..90640a43cf623eeb711af33aa356beecea8ced72 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -54,6 +54,17 @@ void __kunit_fail_current_test(const char *file, int line, const char *fmt, ...) EXPORT_SYMBOL_GPL(__kunit_fail_current_test); #endif +/* + * Enable KUnit tests to run. + */ +#ifdef CONFIG_KUNIT_DEFAULT_ENABLED +static bool enable_param = true; +#else +static bool enable_param; +#endif +module_param_named(enable, enable_param, bool, 0); +MODULE_PARM_DESC(enable, "Enable KUnit tests"); + /* * KUnit statistic mode: * 0 - disabled @@ -247,7 +258,7 @@ static void kunit_print_string_stream(struct kunit *test, static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, enum kunit_assert_type type, const struct kunit_assert *assert, - const struct va_format *message) + assert_format_t assert_format, const struct va_format *message) { struct string_stream *stream; @@ -263,11 +274,11 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, } kunit_assert_prologue(loc, type, stream); - assert->format(assert, message, stream); + assert_format(assert, message, stream); kunit_print_string_stream(test, stream); - WARN_ON(string_stream_destroy(stream)); + string_stream_destroy(stream); } static void __noreturn kunit_abort(struct kunit *test) @@ -287,6 +298,7 @@ void kunit_do_failed_assertion(struct kunit *test, const struct kunit_loc *loc, enum kunit_assert_type type, const struct kunit_assert *assert, + assert_format_t assert_format, const char *fmt, ...) { va_list args; @@ -296,7 +308,7 @@ void kunit_do_failed_assertion(struct kunit *test, message.fmt = fmt; message.va = &args; - kunit_fail(test, loc, type, assert, &message); + kunit_fail(test, loc, type, assert, assert_format, &message); va_end(args); @@ -586,10 +598,20 @@ static void kunit_init_suite(struct kunit_suite *suite) suite->suite_init_err = 0; } +bool kunit_enabled(void) +{ + return enable_param; +} + int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites) { unsigned int i; + if (!kunit_enabled() && num_suites > 0) { + pr_info("kunit: disabled\n"); + return 0; + } + for (i = 0; i < num_suites; i++) { kunit_init_suite(suites[i]); kunit_run_tests(suites[i]); @@ -607,6 +629,9 @@ void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites) { unsigned int i; + if (!kunit_enabled()) + return; + for (i = 0; i < num_suites; i++) kunit_exit_suite(suites[i]); @@ -689,21 +714,20 @@ void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp) } EXPORT_SYMBOL_GPL(kunit_kmalloc_array); -void kunit_kfree(struct kunit *test, const void *ptr) +static inline bool kunit_kfree_match(struct kunit *test, + struct kunit_resource *res, void *match_data) { - struct kunit_resource *res; - - res = kunit_find_resource(test, kunit_resource_instance_match, - (void *)ptr); - - /* - * Removing the resource from the list of resources drops the - * reference count to 1; the final put will trigger the free. - */ - kunit_remove_resource(test, res); + /* Only match resources allocated with kunit_kmalloc() and friends. */ + return res->free == kunit_kmalloc_array_free && res->data == match_data; +} - kunit_put_resource(res); +void kunit_kfree(struct kunit *test, const void *ptr) +{ + if (!ptr) + return; + if (kunit_destroy_resource(test, kunit_kfree_match, (void *)ptr)) + KUNIT_FAIL(test, "kunit_kfree: %px already freed or not allocated by kunit", ptr); } EXPORT_SYMBOL_GPL(kunit_kfree); diff --git a/lib/llist.c b/lib/llist.c index 611ce4881a875691c1369f125d5a63cdbc60b70a..7d78b736e8afdb16e8060f896ce75f5e9d734ba2 100644 --- a/lib/llist.c +++ b/lib/llist.c @@ -30,7 +30,7 @@ bool llist_add_batch(struct llist_node *new_first, struct llist_node *new_last, do { new_last->next = first = READ_ONCE(head->first); - } while (cmpxchg(&head->first, first, new_first) != first); + } while (!try_cmpxchg(&head->first, &first, new_first)); return !first; } @@ -52,18 +52,14 @@ EXPORT_SYMBOL_GPL(llist_add_batch); */ struct llist_node *llist_del_first(struct llist_head *head) { - struct llist_node *entry, *old_entry, *next; + struct llist_node *entry, *next; entry = smp_load_acquire(&head->first); - for (;;) { + do { if (entry == NULL) return NULL; - old_entry = entry; next = READ_ONCE(entry->next); - entry = cmpxchg(&head->first, old_entry, next); - if (entry == old_entry) - break; - } + } while (!try_cmpxchg(&head->first, &entry, next)); return entry; } diff --git a/lib/maple_tree.c b/lib/maple_tree.c new file mode 100644 index 0000000000000000000000000000000000000000..e1743803c8512519baf7c6511959284a7062837f --- /dev/null +++ b/lib/maple_tree.c @@ -0,0 +1,7130 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Maple Tree implementation + * Copyright (c) 2018-2022 Oracle Corporation + * Authors: Liam R. Howlett + * Matthew Wilcox + */ + +/* + * DOC: Interesting implementation details of the Maple Tree + * + * Each node type has a number of slots for entries and a number of slots for + * pivots. In the case of dense nodes, the pivots are implied by the position + * and are simply the slot index + the minimum of the node. + * + * In regular B-Tree terms, pivots are called keys. The term pivot is used to + * indicate that the tree is specifying ranges, Pivots may appear in the + * subtree with an entry attached to the value where as keys are unique to a + * specific position of a B-tree. Pivot values are inclusive of the slot with + * the same index. + * + * + * The following illustrates the layout of a range64 nodes slots and pivots. + * + * + * Slots -> | 0 | 1 | 2 | ... | 12 | 13 | 14 | 15 | + * ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ ┬ + * │ │ │ │ │ │ │ │ └─ Implied maximum + * │ │ │ │ │ │ │ └─ Pivot 14 + * │ │ │ │ │ │ └─ Pivot 13 + * │ │ │ │ │ └─ Pivot 12 + * │ │ │ │ └─ Pivot 11 + * │ │ │ └─ Pivot 2 + * │ │ └─ Pivot 1 + * │ └─ Pivot 0 + * └─ Implied minimum + * + * Slot contents: + * Internal (non-leaf) nodes contain pointers to other nodes. + * Leaf nodes contain entries. + * + * The location of interest is often referred to as an offset. All offsets have + * a slot, but the last offset has an implied pivot from the node above (or + * UINT_MAX for the root node. + * + * Ranges complicate certain write activities. When modifying any of + * the B-tree variants, it is known that one entry will either be added or + * deleted. When modifying the Maple Tree, one store operation may overwrite + * the entire data set, or one half of the tree, or the middle half of the tree. + * + */ + + +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#define MA_ROOT_PARENT 1 + +/* + * Maple state flags + * * MA_STATE_BULK - Bulk insert mode + * * MA_STATE_REBALANCE - Indicate a rebalance during bulk insert + * * MA_STATE_PREALLOC - Preallocated nodes, WARN_ON allocation + */ +#define MA_STATE_BULK 1 +#define MA_STATE_REBALANCE 2 +#define MA_STATE_PREALLOC 4 + +#define ma_parent_ptr(x) ((struct maple_pnode *)(x)) +#define ma_mnode_ptr(x) ((struct maple_node *)(x)) +#define ma_enode_ptr(x) ((struct maple_enode *)(x)) +static struct kmem_cache *maple_node_cache; + +#ifdef CONFIG_DEBUG_MAPLE_TREE +static const unsigned long mt_max[] = { + [maple_dense] = MAPLE_NODE_SLOTS, + [maple_leaf_64] = ULONG_MAX, + [maple_range_64] = ULONG_MAX, + [maple_arange_64] = ULONG_MAX, +}; +#define mt_node_max(x) mt_max[mte_node_type(x)] +#endif + +static const unsigned char mt_slots[] = { + [maple_dense] = MAPLE_NODE_SLOTS, + [maple_leaf_64] = MAPLE_RANGE64_SLOTS, + [maple_range_64] = MAPLE_RANGE64_SLOTS, + [maple_arange_64] = MAPLE_ARANGE64_SLOTS, +}; +#define mt_slot_count(x) mt_slots[mte_node_type(x)] + +static const unsigned char mt_pivots[] = { + [maple_dense] = 0, + [maple_leaf_64] = MAPLE_RANGE64_SLOTS - 1, + [maple_range_64] = MAPLE_RANGE64_SLOTS - 1, + [maple_arange_64] = MAPLE_ARANGE64_SLOTS - 1, +}; +#define mt_pivot_count(x) mt_pivots[mte_node_type(x)] + +static const unsigned char mt_min_slots[] = { + [maple_dense] = MAPLE_NODE_SLOTS / 2, + [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, + [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2, + [maple_arange_64] = (MAPLE_ARANGE64_SLOTS / 2) - 1, +}; +#define mt_min_slot_count(x) mt_min_slots[mte_node_type(x)] + +#define MAPLE_BIG_NODE_SLOTS (MAPLE_RANGE64_SLOTS * 2 + 2) +#define MAPLE_BIG_NODE_GAPS (MAPLE_ARANGE64_SLOTS * 2 + 1) + +struct maple_big_node { + struct maple_pnode *parent; + unsigned long pivot[MAPLE_BIG_NODE_SLOTS - 1]; + union { + struct maple_enode *slot[MAPLE_BIG_NODE_SLOTS]; + struct { + unsigned long padding[MAPLE_BIG_NODE_GAPS]; + unsigned long gap[MAPLE_BIG_NODE_GAPS]; + }; + }; + unsigned char b_end; + enum maple_type type; +}; + +/* + * The maple_subtree_state is used to build a tree to replace a segment of an + * existing tree in a more atomic way. Any walkers of the older tree will hit a + * dead node and restart on updates. + */ +struct maple_subtree_state { + struct ma_state *orig_l; /* Original left side of subtree */ + struct ma_state *orig_r; /* Original right side of subtree */ + struct ma_state *l; /* New left side of subtree */ + struct ma_state *m; /* New middle of subtree (rare) */ + struct ma_state *r; /* New right side of subtree */ + struct ma_topiary *free; /* nodes to be freed */ + struct ma_topiary *destroy; /* Nodes to be destroyed (walked and freed) */ + struct maple_big_node *bn; +}; + +/* Functions */ +static inline struct maple_node *mt_alloc_one(gfp_t gfp) +{ + return kmem_cache_alloc(maple_node_cache, gfp | __GFP_ZERO); +} + +static inline int mt_alloc_bulk(gfp_t gfp, size_t size, void **nodes) +{ + return kmem_cache_alloc_bulk(maple_node_cache, gfp | __GFP_ZERO, size, + nodes); +} + +static inline void mt_free_bulk(size_t size, void __rcu **nodes) +{ + kmem_cache_free_bulk(maple_node_cache, size, (void **)nodes); +} + +static void mt_free_rcu(struct rcu_head *head) +{ + struct maple_node *node = container_of(head, struct maple_node, rcu); + + kmem_cache_free(maple_node_cache, node); +} + +/* + * ma_free_rcu() - Use rcu callback to free a maple node + * @node: The node to free + * + * The maple tree uses the parent pointer to indicate this node is no longer in + * use and will be freed. + */ +static void ma_free_rcu(struct maple_node *node) +{ + node->parent = ma_parent_ptr(node); + call_rcu(&node->rcu, mt_free_rcu); +} + +static unsigned int mt_height(const struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; +} + +static void mas_set_height(struct ma_state *mas) +{ + unsigned int new_flags = mas->tree->ma_flags; + + new_flags &= ~MT_FLAGS_HEIGHT_MASK; + BUG_ON(mas->depth > MAPLE_HEIGHT_MAX); + new_flags |= mas->depth << MT_FLAGS_HEIGHT_OFFSET; + mas->tree->ma_flags = new_flags; +} + +static unsigned int mas_mt_height(struct ma_state *mas) +{ + return mt_height(mas->tree); +} + +static inline enum maple_type mte_node_type(const struct maple_enode *entry) +{ + return ((unsigned long)entry >> MAPLE_NODE_TYPE_SHIFT) & + MAPLE_NODE_TYPE_MASK; +} + +static inline bool ma_is_dense(const enum maple_type type) +{ + return type < maple_leaf_64; +} + +static inline bool ma_is_leaf(const enum maple_type type) +{ + return type < maple_range_64; +} + +static inline bool mte_is_leaf(const struct maple_enode *entry) +{ + return ma_is_leaf(mte_node_type(entry)); +} + +/* + * We also reserve values with the bottom two bits set to '10' which are + * below 4096 + */ +static inline bool mt_is_reserved(const void *entry) +{ + return ((unsigned long)entry < MAPLE_RESERVED_RANGE) && + xa_is_internal(entry); +} + +static inline void mas_set_err(struct ma_state *mas, long err) +{ + mas->node = MA_ERROR(err); +} + +static inline bool mas_is_ptr(struct ma_state *mas) +{ + return mas->node == MAS_ROOT; +} + +static inline bool mas_is_start(struct ma_state *mas) +{ + return mas->node == MAS_START; +} + +bool mas_is_err(struct ma_state *mas) +{ + return xa_is_err(mas->node); +} + +static inline bool mas_searchable(struct ma_state *mas) +{ + if (mas_is_none(mas)) + return false; + + if (mas_is_ptr(mas)) + return false; + + return true; +} + +static inline struct maple_node *mte_to_node(const struct maple_enode *entry) +{ + return (struct maple_node *)((unsigned long)entry & ~MAPLE_NODE_MASK); +} + +/* + * mte_to_mat() - Convert a maple encoded node to a maple topiary node. + * @entry: The maple encoded node + * + * Return: a maple topiary pointer + */ +static inline struct maple_topiary *mte_to_mat(const struct maple_enode *entry) +{ + return (struct maple_topiary *) + ((unsigned long)entry & ~MAPLE_NODE_MASK); +} + +/* + * mas_mn() - Get the maple state node. + * @mas: The maple state + * + * Return: the maple node (not encoded - bare pointer). + */ +static inline struct maple_node *mas_mn(const struct ma_state *mas) +{ + return mte_to_node(mas->node); +} + +/* + * mte_set_node_dead() - Set a maple encoded node as dead. + * @mn: The maple encoded node. + */ +static inline void mte_set_node_dead(struct maple_enode *mn) +{ + mte_to_node(mn)->parent = ma_parent_ptr(mte_to_node(mn)); + smp_wmb(); /* Needed for RCU */ +} + +/* Bit 1 indicates the root is a node */ +#define MAPLE_ROOT_NODE 0x02 +/* maple_type stored bit 3-6 */ +#define MAPLE_ENODE_TYPE_SHIFT 0x03 +/* Bit 2 means a NULL somewhere below */ +#define MAPLE_ENODE_NULL 0x04 + +static inline struct maple_enode *mt_mk_node(const struct maple_node *node, + enum maple_type type) +{ + return (void *)((unsigned long)node | + (type << MAPLE_ENODE_TYPE_SHIFT) | MAPLE_ENODE_NULL); +} + +static inline void *mte_mk_root(const struct maple_enode *node) +{ + return (void *)((unsigned long)node | MAPLE_ROOT_NODE); +} + +static inline void *mte_safe_root(const struct maple_enode *node) +{ + return (void *)((unsigned long)node & ~MAPLE_ROOT_NODE); +} + +static inline void mte_set_full(const struct maple_enode *node) +{ + node = (void *)((unsigned long)node & ~MAPLE_ENODE_NULL); +} + +static inline void mte_clear_full(const struct maple_enode *node) +{ + node = (void *)((unsigned long)node | MAPLE_ENODE_NULL); +} + +static inline bool ma_is_root(struct maple_node *node) +{ + return ((unsigned long)node->parent & MA_ROOT_PARENT); +} + +static inline bool mte_is_root(const struct maple_enode *node) +{ + return ma_is_root(mte_to_node(node)); +} + +static inline bool mas_is_root_limits(const struct ma_state *mas) +{ + return !mas->min && mas->max == ULONG_MAX; +} + +static inline bool mt_is_alloc(struct maple_tree *mt) +{ + return (mt->ma_flags & MT_FLAGS_ALLOC_RANGE); +} + +/* + * The Parent Pointer + * Excluding root, the parent pointer is 256B aligned like all other tree nodes. + * When storing a 32 or 64 bit values, the offset can fit into 5 bits. The 16 + * bit values need an extra bit to store the offset. This extra bit comes from + * a reuse of the last bit in the node type. This is possible by using bit 1 to + * indicate if bit 2 is part of the type or the slot. + * + * Note types: + * 0x??1 = Root + * 0x?00 = 16 bit nodes + * 0x010 = 32 bit nodes + * 0x110 = 64 bit nodes + * + * Slot size and alignment + * 0b??1 : Root + * 0b?00 : 16 bit values, type in 0-1, slot in 2-7 + * 0b010 : 32 bit values, type in 0-2, slot in 3-7 + * 0b110 : 64 bit values, type in 0-2, slot in 3-7 + */ + +#define MAPLE_PARENT_ROOT 0x01 + +#define MAPLE_PARENT_SLOT_SHIFT 0x03 +#define MAPLE_PARENT_SLOT_MASK 0xF8 + +#define MAPLE_PARENT_16B_SLOT_SHIFT 0x02 +#define MAPLE_PARENT_16B_SLOT_MASK 0xFC + +#define MAPLE_PARENT_RANGE64 0x06 +#define MAPLE_PARENT_RANGE32 0x04 +#define MAPLE_PARENT_NOT_RANGE16 0x02 + +/* + * mte_parent_shift() - Get the parent shift for the slot storage. + * @parent: The parent pointer cast as an unsigned long + * Return: The shift into that pointer to the star to of the slot + */ +static inline unsigned long mte_parent_shift(unsigned long parent) +{ + /* Note bit 1 == 0 means 16B */ + if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) + return MAPLE_PARENT_SLOT_SHIFT; + + return MAPLE_PARENT_16B_SLOT_SHIFT; +} + +/* + * mte_parent_slot_mask() - Get the slot mask for the parent. + * @parent: The parent pointer cast as an unsigned long. + * Return: The slot mask for that parent. + */ +static inline unsigned long mte_parent_slot_mask(unsigned long parent) +{ + /* Note bit 1 == 0 means 16B */ + if (likely(parent & MAPLE_PARENT_NOT_RANGE16)) + return MAPLE_PARENT_SLOT_MASK; + + return MAPLE_PARENT_16B_SLOT_MASK; +} + +/* + * mas_parent_enum() - Return the maple_type of the parent from the stored + * parent type. + * @mas: The maple state + * @node: The maple_enode to extract the parent's enum + * Return: The node->parent maple_type + */ +static inline +enum maple_type mte_parent_enum(struct maple_enode *p_enode, + struct maple_tree *mt) +{ + unsigned long p_type; + + p_type = (unsigned long)p_enode; + if (p_type & MAPLE_PARENT_ROOT) + return 0; /* Validated in the caller. */ + + p_type &= MAPLE_NODE_MASK; + p_type = p_type & ~(MAPLE_PARENT_ROOT | mte_parent_slot_mask(p_type)); + + switch (p_type) { + case MAPLE_PARENT_RANGE64: /* or MAPLE_PARENT_ARANGE64 */ + if (mt_is_alloc(mt)) + return maple_arange_64; + return maple_range_64; + } + + return 0; +} + +static inline +enum maple_type mas_parent_enum(struct ma_state *mas, struct maple_enode *enode) +{ + return mte_parent_enum(ma_enode_ptr(mte_to_node(enode)->parent), mas->tree); +} + +/* + * mte_set_parent() - Set the parent node and encode the slot + * @enode: The encoded maple node. + * @parent: The encoded maple node that is the parent of @enode. + * @slot: The slot that @enode resides in @parent. + * + * Slot number is encoded in the enode->parent bit 3-6 or 2-6, depending on the + * parent type. + */ +static inline +void mte_set_parent(struct maple_enode *enode, const struct maple_enode *parent, + unsigned char slot) +{ + unsigned long val = (unsigned long) parent; + unsigned long shift; + unsigned long type; + enum maple_type p_type = mte_node_type(parent); + + BUG_ON(p_type == maple_dense); + BUG_ON(p_type == maple_leaf_64); + + switch (p_type) { + case maple_range_64: + case maple_arange_64: + shift = MAPLE_PARENT_SLOT_SHIFT; + type = MAPLE_PARENT_RANGE64; + break; + default: + case maple_dense: + case maple_leaf_64: + shift = type = 0; + break; + } + + val &= ~MAPLE_NODE_MASK; /* Clear all node metadata in parent */ + val |= (slot << shift) | type; + mte_to_node(enode)->parent = ma_parent_ptr(val); +} + +/* + * mte_parent_slot() - get the parent slot of @enode. + * @enode: The encoded maple node. + * + * Return: The slot in the parent node where @enode resides. + */ +static inline unsigned int mte_parent_slot(const struct maple_enode *enode) +{ + unsigned long val = (unsigned long) mte_to_node(enode)->parent; + + /* Root. */ + if (val & 1) + return 0; + + /* + * Okay to use MAPLE_PARENT_16B_SLOT_MASK as the last bit will be lost + * by shift if the parent shift is MAPLE_PARENT_SLOT_SHIFT + */ + return (val & MAPLE_PARENT_16B_SLOT_MASK) >> mte_parent_shift(val); +} + +/* + * mte_parent() - Get the parent of @node. + * @node: The encoded maple node. + * + * Return: The parent maple node. + */ +static inline struct maple_node *mte_parent(const struct maple_enode *enode) +{ + return (void *)((unsigned long) + (mte_to_node(enode)->parent) & ~MAPLE_NODE_MASK); +} + +/* + * ma_dead_node() - check if the @enode is dead. + * @enode: The encoded maple node + * + * Return: true if dead, false otherwise. + */ +static inline bool ma_dead_node(const struct maple_node *node) +{ + struct maple_node *parent = (void *)((unsigned long) + node->parent & ~MAPLE_NODE_MASK); + + return (parent == node); +} +/* + * mte_dead_node() - check if the @enode is dead. + * @enode: The encoded maple node + * + * Return: true if dead, false otherwise. + */ +static inline bool mte_dead_node(const struct maple_enode *enode) +{ + struct maple_node *parent, *node; + + node = mte_to_node(enode); + parent = mte_parent(enode); + return (parent == node); +} + +/* + * mas_allocated() - Get the number of nodes allocated in a maple state. + * @mas: The maple state + * + * The ma_state alloc member is overloaded to hold a pointer to the first + * allocated node or to the number of requested nodes to allocate. If bit 0 is + * set, then the alloc contains the number of requested nodes. If there is an + * allocated node, then the total allocated nodes is in that node. + * + * Return: The total number of nodes allocated + */ +static inline unsigned long mas_allocated(const struct ma_state *mas) +{ + if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) + return 0; + + return mas->alloc->total; +} + +/* + * mas_set_alloc_req() - Set the requested number of allocations. + * @mas: the maple state + * @count: the number of allocations. + * + * The requested number of allocations is either in the first allocated node, + * located in @mas->alloc->request_count, or directly in @mas->alloc if there is + * no allocated node. Set the request either in the node or do the necessary + * encoding to store in @mas->alloc directly. + */ +static inline void mas_set_alloc_req(struct ma_state *mas, unsigned long count) +{ + if (!mas->alloc || ((unsigned long)mas->alloc & 0x1)) { + if (!count) + mas->alloc = NULL; + else + mas->alloc = (struct maple_alloc *)(((count) << 1U) | 1U); + return; + } + + mas->alloc->request_count = count; +} + +/* + * mas_alloc_req() - get the requested number of allocations. + * @mas: The maple state + * + * The alloc count is either stored directly in @mas, or in + * @mas->alloc->request_count if there is at least one node allocated. Decode + * the request count if it's stored directly in @mas->alloc. + * + * Return: The allocation request count. + */ +static inline unsigned int mas_alloc_req(const struct ma_state *mas) +{ + if ((unsigned long)mas->alloc & 0x1) + return (unsigned long)(mas->alloc) >> 1; + else if (mas->alloc) + return mas->alloc->request_count; + return 0; +} + +/* + * ma_pivots() - Get a pointer to the maple node pivots. + * @node - the maple node + * @type - the node type + * + * Return: A pointer to the maple node pivots + */ +static inline unsigned long *ma_pivots(struct maple_node *node, + enum maple_type type) +{ + switch (type) { + case maple_arange_64: + return node->ma64.pivot; + case maple_range_64: + case maple_leaf_64: + return node->mr64.pivot; + case maple_dense: + return NULL; + } + return NULL; +} + +/* + * ma_gaps() - Get a pointer to the maple node gaps. + * @node - the maple node + * @type - the node type + * + * Return: A pointer to the maple node gaps + */ +static inline unsigned long *ma_gaps(struct maple_node *node, + enum maple_type type) +{ + switch (type) { + case maple_arange_64: + return node->ma64.gap; + case maple_range_64: + case maple_leaf_64: + case maple_dense: + return NULL; + } + return NULL; +} + +/* + * mte_pivot() - Get the pivot at @piv of the maple encoded node. + * @mn: The maple encoded node. + * @piv: The pivot. + * + * Return: the pivot at @piv of @mn. + */ +static inline unsigned long mte_pivot(const struct maple_enode *mn, + unsigned char piv) +{ + struct maple_node *node = mte_to_node(mn); + + if (piv >= mt_pivots[piv]) { + WARN_ON(1); + return 0; + } + switch (mte_node_type(mn)) { + case maple_arange_64: + return node->ma64.pivot[piv]; + case maple_range_64: + case maple_leaf_64: + return node->mr64.pivot[piv]; + case maple_dense: + return 0; + } + return 0; +} + +/* + * mas_safe_pivot() - get the pivot at @piv or mas->max. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @piv: The pivot to fetch + * @type: The maple node type + * + * Return: The pivot at @piv within the limit of the @pivots array, @mas->max + * otherwise. + */ +static inline unsigned long +mas_safe_pivot(const struct ma_state *mas, unsigned long *pivots, + unsigned char piv, enum maple_type type) +{ + if (piv >= mt_pivots[type]) + return mas->max; + + return pivots[piv]; +} + +/* + * mas_safe_min() - Return the minimum for a given offset. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @offset: The offset into the pivot array + * + * Return: The minimum range value that is contained in @offset. + */ +static inline unsigned long +mas_safe_min(struct ma_state *mas, unsigned long *pivots, unsigned char offset) +{ + if (likely(offset)) + return pivots[offset - 1] + 1; + + return mas->min; +} + +/* + * mas_logical_pivot() - Get the logical pivot of a given offset. + * @mas: The maple state + * @pivots: The pointer to the maple node pivots + * @offset: The offset into the pivot array + * @type: The maple node type + * + * When there is no value at a pivot (beyond the end of the data), then the + * pivot is actually @mas->max. + * + * Return: the logical pivot of a given @offset. + */ +static inline unsigned long +mas_logical_pivot(struct ma_state *mas, unsigned long *pivots, + unsigned char offset, enum maple_type type) +{ + unsigned long lpiv = mas_safe_pivot(mas, pivots, offset, type); + + if (likely(lpiv)) + return lpiv; + + if (likely(offset)) + return mas->max; + + return lpiv; +} + +/* + * mte_set_pivot() - Set a pivot to a value in an encoded maple node. + * @mn: The encoded maple node + * @piv: The pivot offset + * @val: The value of the pivot + */ +static inline void mte_set_pivot(struct maple_enode *mn, unsigned char piv, + unsigned long val) +{ + struct maple_node *node = mte_to_node(mn); + enum maple_type type = mte_node_type(mn); + + BUG_ON(piv >= mt_pivots[type]); + switch (type) { + default: + case maple_range_64: + case maple_leaf_64: + node->mr64.pivot[piv] = val; + break; + case maple_arange_64: + node->ma64.pivot[piv] = val; + break; + case maple_dense: + break; + } + +} + +/* + * ma_slots() - Get a pointer to the maple node slots. + * @mn: The maple node + * @mt: The maple node type + * + * Return: A pointer to the maple node slots + */ +static inline void __rcu **ma_slots(struct maple_node *mn, enum maple_type mt) +{ + switch (mt) { + default: + case maple_arange_64: + return mn->ma64.slot; + case maple_range_64: + case maple_leaf_64: + return mn->mr64.slot; + case maple_dense: + return mn->slot; + } +} + +static inline bool mt_locked(const struct maple_tree *mt) +{ + return mt_external_lock(mt) ? mt_lock_is_held(mt) : + lockdep_is_held(&mt->ma_lock); +} + +static inline void *mt_slot(const struct maple_tree *mt, + void __rcu **slots, unsigned char offset) +{ + return rcu_dereference_check(slots[offset], mt_locked(mt)); +} + +/* + * mas_slot_locked() - Get the slot value when holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset. + */ +static inline void *mas_slot_locked(struct ma_state *mas, void __rcu **slots, + unsigned char offset) +{ + return rcu_dereference_protected(slots[offset], mt_locked(mas->tree)); +} + +/* + * mas_slot() - Get the slot value when not holding the maple tree lock. + * @mas: The maple state + * @slots: The pointer to the slots + * @offset: The offset into the slots array to fetch + * + * Return: The entry stored in @slots at the @offset + */ +static inline void *mas_slot(struct ma_state *mas, void __rcu **slots, + unsigned char offset) +{ + return mt_slot(mas->tree, slots, offset); +} + +/* + * mas_root() - Get the maple tree root. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ +static inline void *mas_root(struct ma_state *mas) +{ + return rcu_dereference_check(mas->tree->ma_root, mt_locked(mas->tree)); +} + +static inline void *mt_root_locked(struct maple_tree *mt) +{ + return rcu_dereference_protected(mt->ma_root, mt_locked(mt)); +} + +/* + * mas_root_locked() - Get the maple tree root when holding the maple tree lock. + * @mas: The maple state. + * + * Return: The pointer to the root of the tree + */ +static inline void *mas_root_locked(struct ma_state *mas) +{ + return mt_root_locked(mas->tree); +} + +static inline struct maple_metadata *ma_meta(struct maple_node *mn, + enum maple_type mt) +{ + switch (mt) { + case maple_arange_64: + return &mn->ma64.meta; + default: + return &mn->mr64.meta; + } +} + +/* + * ma_set_meta() - Set the metadata information of a node. + * @mn: The maple node + * @mt: The maple node type + * @offset: The offset of the highest sub-gap in this node. + * @end: The end of the data in this node. + */ +static inline void ma_set_meta(struct maple_node *mn, enum maple_type mt, + unsigned char offset, unsigned char end) +{ + struct maple_metadata *meta = ma_meta(mn, mt); + + meta->gap = offset; + meta->end = end; +} + +/* + * ma_meta_end() - Get the data end of a node from the metadata + * @mn: The maple node + * @mt: The maple node type + */ +static inline unsigned char ma_meta_end(struct maple_node *mn, + enum maple_type mt) +{ + struct maple_metadata *meta = ma_meta(mn, mt); + + return meta->end; +} + +/* + * ma_meta_gap() - Get the largest gap location of a node from the metadata + * @mn: The maple node + * @mt: The maple node type + */ +static inline unsigned char ma_meta_gap(struct maple_node *mn, + enum maple_type mt) +{ + BUG_ON(mt != maple_arange_64); + + return mn->ma64.meta.gap; +} + +/* + * ma_set_meta_gap() - Set the largest gap location in a nodes metadata + * @mn: The maple node + * @mn: The maple node type + * @offset: The location of the largest gap. + */ +static inline void ma_set_meta_gap(struct maple_node *mn, enum maple_type mt, + unsigned char offset) +{ + + struct maple_metadata *meta = ma_meta(mn, mt); + + meta->gap = offset; +} + +/* + * mat_add() - Add a @dead_enode to the ma_topiary of a list of dead nodes. + * @mat - the ma_topiary, a linked list of dead nodes. + * @dead_enode - the node to be marked as dead and added to the tail of the list + * + * Add the @dead_enode to the linked list in @mat. + */ +static inline void mat_add(struct ma_topiary *mat, + struct maple_enode *dead_enode) +{ + mte_set_node_dead(dead_enode); + mte_to_mat(dead_enode)->next = NULL; + if (!mat->tail) { + mat->tail = mat->head = dead_enode; + return; + } + + mte_to_mat(mat->tail)->next = dead_enode; + mat->tail = dead_enode; +} + +static void mte_destroy_walk(struct maple_enode *, struct maple_tree *); +static inline void mas_free(struct ma_state *mas, struct maple_enode *used); + +/* + * mas_mat_free() - Free all nodes in a dead list. + * @mas - the maple state + * @mat - the ma_topiary linked list of dead nodes to free. + * + * Free walk a dead list. + */ +static void mas_mat_free(struct ma_state *mas, struct ma_topiary *mat) +{ + struct maple_enode *next; + + while (mat->head) { + next = mte_to_mat(mat->head)->next; + mas_free(mas, mat->head); + mat->head = next; + } +} + +/* + * mas_mat_destroy() - Free all nodes and subtrees in a dead list. + * @mas - the maple state + * @mat - the ma_topiary linked list of dead nodes to free. + * + * Destroy walk a dead list. + */ +static void mas_mat_destroy(struct ma_state *mas, struct ma_topiary *mat) +{ + struct maple_enode *next; + + while (mat->head) { + next = mte_to_mat(mat->head)->next; + mte_destroy_walk(mat->head, mat->mtree); + mat->head = next; + } +} +/* + * mas_descend() - Descend into the slot stored in the ma_state. + * @mas - the maple state. + * + * Note: Not RCU safe, only use in write side or debug code. + */ +static inline void mas_descend(struct ma_state *mas) +{ + enum maple_type type; + unsigned long *pivots; + struct maple_node *node; + void __rcu **slots; + + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + + if (mas->offset) + mas->min = pivots[mas->offset - 1] + 1; + mas->max = mas_safe_pivot(mas, pivots, mas->offset, type); + mas->node = mas_slot(mas, slots, mas->offset); +} + +/* + * mte_set_gap() - Set a maple node gap. + * @mn: The encoded maple node + * @gap: The offset of the gap to set + * @val: The gap value + */ +static inline void mte_set_gap(const struct maple_enode *mn, + unsigned char gap, unsigned long val) +{ + switch (mte_node_type(mn)) { + default: + break; + case maple_arange_64: + mte_to_node(mn)->ma64.gap[gap] = val; + break; + } +} + +/* + * mas_ascend() - Walk up a level of the tree. + * @mas: The maple state + * + * Sets the @mas->max and @mas->min to the correct values when walking up. This + * may cause several levels of walking up to find the correct min and max. + * May find a dead node which will cause a premature return. + * Return: 1 on dead node, 0 otherwise + */ +static int mas_ascend(struct ma_state *mas) +{ + struct maple_enode *p_enode; /* parent enode. */ + struct maple_enode *a_enode; /* ancestor enode. */ + struct maple_node *a_node; /* ancestor node. */ + struct maple_node *p_node; /* parent node. */ + unsigned char a_slot; + enum maple_type a_type; + unsigned long min, max; + unsigned long *pivots; + unsigned char offset; + bool set_max = false, set_min = false; + + a_node = mas_mn(mas); + if (ma_is_root(a_node)) { + mas->offset = 0; + return 0; + } + + p_node = mte_parent(mas->node); + if (unlikely(a_node == p_node)) + return 1; + a_type = mas_parent_enum(mas, mas->node); + offset = mte_parent_slot(mas->node); + a_enode = mt_mk_node(p_node, a_type); + + /* Check to make sure all parent information is still accurate */ + if (p_node != mte_parent(mas->node)) + return 1; + + mas->node = a_enode; + mas->offset = offset; + + if (mte_is_root(a_enode)) { + mas->max = ULONG_MAX; + mas->min = 0; + return 0; + } + + min = 0; + max = ULONG_MAX; + do { + p_enode = a_enode; + a_type = mas_parent_enum(mas, p_enode); + a_node = mte_parent(p_enode); + a_slot = mte_parent_slot(p_enode); + pivots = ma_pivots(a_node, a_type); + a_enode = mt_mk_node(a_node, a_type); + + if (!set_min && a_slot) { + set_min = true; + min = pivots[a_slot - 1] + 1; + } + + if (!set_max && a_slot < mt_pivots[a_type]) { + set_max = true; + max = pivots[a_slot]; + } + + if (unlikely(ma_dead_node(a_node))) + return 1; + + if (unlikely(ma_is_root(a_node))) + break; + + } while (!set_min || !set_max); + + mas->max = max; + mas->min = min; + return 0; +} + +/* + * mas_pop_node() - Get a previously allocated maple node from the maple state. + * @mas: The maple state + * + * Return: A pointer to a maple node. + */ +static inline struct maple_node *mas_pop_node(struct ma_state *mas) +{ + struct maple_alloc *ret, *node = mas->alloc; + unsigned long total = mas_allocated(mas); + + /* nothing or a request pending. */ + if (unlikely(!total)) + return NULL; + + if (total == 1) { + /* single allocation in this ma_state */ + mas->alloc = NULL; + ret = node; + goto single_node; + } + + if (!node->node_count) { + /* Single allocation in this node. */ + mas->alloc = node->slot[0]; + node->slot[0] = NULL; + mas->alloc->total = node->total - 1; + ret = node; + goto new_head; + } + + node->total--; + ret = node->slot[node->node_count]; + node->slot[node->node_count--] = NULL; + +single_node: +new_head: + ret->total = 0; + ret->node_count = 0; + if (ret->request_count) { + mas_set_alloc_req(mas, ret->request_count + 1); + ret->request_count = 0; + } + return (struct maple_node *)ret; +} + +/* + * mas_push_node() - Push a node back on the maple state allocation. + * @mas: The maple state + * @used: The used maple node + * + * Stores the maple node back into @mas->alloc for reuse. Updates allocated and + * requested node count as necessary. + */ +static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) +{ + struct maple_alloc *reuse = (struct maple_alloc *)used; + struct maple_alloc *head = mas->alloc; + unsigned long count; + unsigned int requested = mas_alloc_req(mas); + + memset(reuse, 0, sizeof(*reuse)); + count = mas_allocated(mas); + + if (count && (head->node_count < MAPLE_ALLOC_SLOTS - 1)) { + if (head->slot[0]) + head->node_count++; + head->slot[head->node_count] = reuse; + head->total++; + goto done; + } + + reuse->total = 1; + if ((head) && !((unsigned long)head & 0x1)) { + head->request_count = 0; + reuse->slot[0] = head; + reuse->total += head->total; + } + + mas->alloc = reuse; +done: + if (requested > 1) + mas_set_alloc_req(mas, requested - 1); +} + +/* + * mas_alloc_nodes() - Allocate nodes into a maple state + * @mas: The maple state + * @gfp: The GFP Flags + */ +static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) +{ + struct maple_alloc *node; + struct maple_alloc **nodep = &mas->alloc; + unsigned long allocated = mas_allocated(mas); + unsigned long success = allocated; + unsigned int requested = mas_alloc_req(mas); + unsigned int count; + void **slots = NULL; + unsigned int max_req = 0; + + if (!requested) + return; + + mas_set_alloc_req(mas, 0); + if (mas->mas_flags & MA_STATE_PREALLOC) { + if (allocated) + return; + WARN_ON(!allocated); + } + + if (!allocated || mas->alloc->node_count == MAPLE_ALLOC_SLOTS - 1) { + node = (struct maple_alloc *)mt_alloc_one(gfp); + if (!node) + goto nomem_one; + + if (allocated) + node->slot[0] = mas->alloc; + + success++; + mas->alloc = node; + requested--; + } + + node = mas->alloc; + while (requested) { + max_req = MAPLE_ALLOC_SLOTS; + if (node->slot[0]) { + unsigned int offset = node->node_count + 1; + + slots = (void **)&node->slot[offset]; + max_req -= offset; + } else { + slots = (void **)&node->slot; + } + + max_req = min(requested, max_req); + count = mt_alloc_bulk(gfp, max_req, slots); + if (!count) + goto nomem_bulk; + + node->node_count += count; + /* zero indexed. */ + if (slots == (void **)&node->slot) + node->node_count--; + + success += count; + nodep = &node->slot[0]; + node = *nodep; + requested -= count; + } + mas->alloc->total = success; + return; + +nomem_bulk: + /* Clean up potential freed allocations on bulk failure */ + memset(slots, 0, max_req * sizeof(unsigned long)); +nomem_one: + mas_set_alloc_req(mas, requested); + if (mas->alloc && !(((unsigned long)mas->alloc & 0x1))) + mas->alloc->total = success; + mas_set_err(mas, -ENOMEM); + return; + +} + +/* + * mas_free() - Free an encoded maple node + * @mas: The maple state + * @used: The encoded maple node to free. + * + * Uses rcu free if necessary, pushes @used back on the maple state allocations + * otherwise. + */ +static inline void mas_free(struct ma_state *mas, struct maple_enode *used) +{ + struct maple_node *tmp = mte_to_node(used); + + if (mt_in_rcu(mas->tree)) + ma_free_rcu(tmp); + else + mas_push_node(mas, tmp); +} + +/* + * mas_node_count() - Check if enough nodes are allocated and request more if + * there is not enough nodes. + * @mas: The maple state + * @count: The number of nodes needed + * @gfp: the gfp flags + */ +static void mas_node_count_gfp(struct ma_state *mas, int count, gfp_t gfp) +{ + unsigned long allocated = mas_allocated(mas); + + if (allocated < count) { + mas_set_alloc_req(mas, count - allocated); + mas_alloc_nodes(mas, gfp); + } +} + +/* + * mas_node_count() - Check if enough nodes are allocated and request more if + * there is not enough nodes. + * @mas: The maple state + * @count: The number of nodes needed + * + * Note: Uses GFP_NOWAIT | __GFP_NOWARN for gfp flags. + */ +static void mas_node_count(struct ma_state *mas, int count) +{ + return mas_node_count_gfp(mas, count, GFP_NOWAIT | __GFP_NOWARN); +} + +/* + * mas_start() - Sets up maple state for operations. + * @mas: The maple state. + * + * If mas->node == MAS_START, then set the min, max, depth, and offset to + * defaults. + * + * Return: + * - If mas->node is an error or not MAS_START, return NULL. + * - If it's an empty tree: NULL & mas->node == MAS_NONE + * - If it's a single entry: The entry & mas->node == MAS_ROOT + * - If it's a tree: NULL & mas->node == safe root node. + */ +static inline struct maple_enode *mas_start(struct ma_state *mas) +{ + if (likely(mas_is_start(mas))) { + struct maple_enode *root; + + mas->node = MAS_NONE; + mas->min = 0; + mas->max = ULONG_MAX; + mas->depth = 0; + mas->offset = 0; + + root = mas_root(mas); + /* Tree with nodes */ + if (likely(xa_is_node(root))) { + mas->node = mte_safe_root(root); + return NULL; + } + + /* empty tree */ + if (unlikely(!root)) { + mas->offset = MAPLE_NODE_SLOTS; + return NULL; + } + + /* Single entry tree */ + mas->node = MAS_ROOT; + mas->offset = MAPLE_NODE_SLOTS; + + /* Single entry tree. */ + if (mas->index > 0) + return NULL; + + return root; + } + + return NULL; +} + +/* + * ma_data_end() - Find the end of the data in a node. + * @node: The maple node + * @type: The maple node type + * @pivots: The array of pivots in the node + * @max: The maximum value in the node + * + * Uses metadata to find the end of the data when possible. + * Return: The zero indexed last slot with data (may be null). + */ +static inline unsigned char ma_data_end(struct maple_node *node, + enum maple_type type, + unsigned long *pivots, + unsigned long max) +{ + unsigned char offset; + + if (type == maple_arange_64) + return ma_meta_end(node, type); + + offset = mt_pivots[type] - 1; + if (likely(!pivots[offset])) + return ma_meta_end(node, type); + + if (likely(pivots[offset] == max)) + return offset; + + return mt_pivots[type]; +} + +/* + * mas_data_end() - Find the end of the data (slot). + * @mas: the maple state + * + * This method is optimized to check the metadata of a node if the node type + * supports data end metadata. + * + * Return: The zero indexed last slot with data (may be null). + */ +static inline unsigned char mas_data_end(struct ma_state *mas) +{ + enum maple_type type; + struct maple_node *node; + unsigned char offset; + unsigned long *pivots; + + type = mte_node_type(mas->node); + node = mas_mn(mas); + if (type == maple_arange_64) + return ma_meta_end(node, type); + + pivots = ma_pivots(node, type); + offset = mt_pivots[type] - 1; + if (likely(!pivots[offset])) + return ma_meta_end(node, type); + + if (likely(pivots[offset] == mas->max)) + return offset; + + return mt_pivots[type]; +} + +/* + * mas_leaf_max_gap() - Returns the largest gap in a leaf node + * @mas - the maple state + * + * Return: The maximum gap in the leaf. + */ +static unsigned long mas_leaf_max_gap(struct ma_state *mas) +{ + enum maple_type mt; + unsigned long pstart, gap, max_gap; + struct maple_node *mn; + unsigned long *pivots; + void __rcu **slots; + unsigned char i; + unsigned char max_piv; + + mt = mte_node_type(mas->node); + mn = mas_mn(mas); + slots = ma_slots(mn, mt); + max_gap = 0; + if (unlikely(ma_is_dense(mt))) { + gap = 0; + for (i = 0; i < mt_slots[mt]; i++) { + if (slots[i]) { + if (gap > max_gap) + max_gap = gap; + gap = 0; + } else { + gap++; + } + } + if (gap > max_gap) + max_gap = gap; + return max_gap; + } + + /* + * Check the first implied pivot optimizes the loop below and slot 1 may + * be skipped if there is a gap in slot 0. + */ + pivots = ma_pivots(mn, mt); + if (likely(!slots[0])) { + max_gap = pivots[0] - mas->min + 1; + i = 2; + } else { + i = 1; + } + + /* reduce max_piv as the special case is checked before the loop */ + max_piv = ma_data_end(mn, mt, pivots, mas->max) - 1; + /* + * Check end implied pivot which can only be a gap on the right most + * node. + */ + if (unlikely(mas->max == ULONG_MAX) && !slots[max_piv + 1]) { + gap = ULONG_MAX - pivots[max_piv]; + if (gap > max_gap) + max_gap = gap; + } + + for (; i <= max_piv; i++) { + /* data == no gap. */ + if (likely(slots[i])) + continue; + + pstart = pivots[i - 1]; + gap = pivots[i] - pstart; + if (gap > max_gap) + max_gap = gap; + + /* There cannot be two gaps in a row. */ + i++; + } + return max_gap; +} + +/* + * ma_max_gap() - Get the maximum gap in a maple node (non-leaf) + * @node: The maple node + * @gaps: The pointer to the gaps + * @mt: The maple node type + * @*off: Pointer to store the offset location of the gap. + * + * Uses the metadata data end to scan backwards across set gaps. + * + * Return: The maximum gap value + */ +static inline unsigned long +ma_max_gap(struct maple_node *node, unsigned long *gaps, enum maple_type mt, + unsigned char *off) +{ + unsigned char offset, i; + unsigned long max_gap = 0; + + i = offset = ma_meta_end(node, mt); + do { + if (gaps[i] > max_gap) { + max_gap = gaps[i]; + offset = i; + } + } while (i--); + + *off = offset; + return max_gap; +} + +/* + * mas_max_gap() - find the largest gap in a non-leaf node and set the slot. + * @mas: The maple state. + * + * If the metadata gap is set to MAPLE_ARANGE64_META_MAX, there is no gap. + * + * Return: The gap value. + */ +static inline unsigned long mas_max_gap(struct ma_state *mas) +{ + unsigned long *gaps; + unsigned char offset; + enum maple_type mt; + struct maple_node *node; + + mt = mte_node_type(mas->node); + if (ma_is_leaf(mt)) + return mas_leaf_max_gap(mas); + + node = mas_mn(mas); + offset = ma_meta_gap(node, mt); + if (offset == MAPLE_ARANGE64_META_MAX) + return 0; + + gaps = ma_gaps(node, mt); + return gaps[offset]; +} + +/* + * mas_parent_gap() - Set the parent gap and any gaps above, as needed + * @mas: The maple state + * @offset: The gap offset in the parent to set + * @new: The new gap value. + * + * Set the parent gap then continue to set the gap upwards, using the metadata + * of the parent to see if it is necessary to check the node above. + */ +static inline void mas_parent_gap(struct ma_state *mas, unsigned char offset, + unsigned long new) +{ + unsigned long meta_gap = 0; + struct maple_node *pnode; + struct maple_enode *penode; + unsigned long *pgaps; + unsigned char meta_offset; + enum maple_type pmt; + + pnode = mte_parent(mas->node); + pmt = mas_parent_enum(mas, mas->node); + penode = mt_mk_node(pnode, pmt); + pgaps = ma_gaps(pnode, pmt); + +ascend: + meta_offset = ma_meta_gap(pnode, pmt); + if (meta_offset == MAPLE_ARANGE64_META_MAX) + meta_gap = 0; + else + meta_gap = pgaps[meta_offset]; + + pgaps[offset] = new; + + if (meta_gap == new) + return; + + if (offset != meta_offset) { + if (meta_gap > new) + return; + + ma_set_meta_gap(pnode, pmt, offset); + } else if (new < meta_gap) { + meta_offset = 15; + new = ma_max_gap(pnode, pgaps, pmt, &meta_offset); + ma_set_meta_gap(pnode, pmt, meta_offset); + } + + if (ma_is_root(pnode)) + return; + + /* Go to the parent node. */ + pnode = mte_parent(penode); + pmt = mas_parent_enum(mas, penode); + pgaps = ma_gaps(pnode, pmt); + offset = mte_parent_slot(penode); + penode = mt_mk_node(pnode, pmt); + goto ascend; +} + +/* + * mas_update_gap() - Update a nodes gaps and propagate up if necessary. + * @mas - the maple state. + */ +static inline void mas_update_gap(struct ma_state *mas) +{ + unsigned char pslot; + unsigned long p_gap; + unsigned long max_gap; + + if (!mt_is_alloc(mas->tree)) + return; + + if (mte_is_root(mas->node)) + return; + + max_gap = mas_max_gap(mas); + + pslot = mte_parent_slot(mas->node); + p_gap = ma_gaps(mte_parent(mas->node), + mas_parent_enum(mas, mas->node))[pslot]; + + if (p_gap != max_gap) + mas_parent_gap(mas, pslot, max_gap); +} + +/* + * mas_adopt_children() - Set the parent pointer of all nodes in @parent to + * @parent with the slot encoded. + * @mas - the maple state (for the tree) + * @parent - the maple encoded node containing the children. + */ +static inline void mas_adopt_children(struct ma_state *mas, + struct maple_enode *parent) +{ + enum maple_type type = mte_node_type(parent); + struct maple_node *node = mas_mn(mas); + void __rcu **slots = ma_slots(node, type); + unsigned long *pivots = ma_pivots(node, type); + struct maple_enode *child; + unsigned char offset; + + offset = ma_data_end(node, type, pivots, mas->max); + do { + child = mas_slot_locked(mas, slots, offset); + mte_set_parent(child, parent, offset); + } while (offset--); +} + +/* + * mas_replace() - Replace a maple node in the tree with mas->node. Uses the + * parent encoding to locate the maple node in the tree. + * @mas - the ma_state to use for operations. + * @advanced - boolean to adopt the child nodes and free the old node (false) or + * leave the node (true) and handle the adoption and free elsewhere. + */ +static inline void mas_replace(struct ma_state *mas, bool advanced) + __must_hold(mas->tree->lock) +{ + struct maple_node *mn = mas_mn(mas); + struct maple_enode *old_enode; + unsigned char offset = 0; + void __rcu **slots = NULL; + + if (ma_is_root(mn)) { + old_enode = mas_root_locked(mas); + } else { + offset = mte_parent_slot(mas->node); + slots = ma_slots(mte_parent(mas->node), + mas_parent_enum(mas, mas->node)); + old_enode = mas_slot_locked(mas, slots, offset); + } + + if (!advanced && !mte_is_leaf(mas->node)) + mas_adopt_children(mas, mas->node); + + if (mte_is_root(mas->node)) { + mn->parent = ma_parent_ptr( + ((unsigned long)mas->tree | MA_ROOT_PARENT)); + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + mas_set_height(mas); + } else { + rcu_assign_pointer(slots[offset], mas->node); + } + + if (!advanced) + mas_free(mas, old_enode); +} + +/* + * mas_new_child() - Find the new child of a node. + * @mas: the maple state + * @child: the maple state to store the child. + */ +static inline bool mas_new_child(struct ma_state *mas, struct ma_state *child) + __must_hold(mas->tree->lock) +{ + enum maple_type mt; + unsigned char offset; + unsigned char end; + unsigned long *pivots; + struct maple_enode *entry; + struct maple_node *node; + void __rcu **slots; + + mt = mte_node_type(mas->node); + node = mas_mn(mas); + slots = ma_slots(node, mt); + pivots = ma_pivots(node, mt); + end = ma_data_end(node, mt, pivots, mas->max); + for (offset = mas->offset; offset <= end; offset++) { + entry = mas_slot_locked(mas, slots, offset); + if (mte_parent(entry) == node) { + *child = *mas; + mas->offset = offset + 1; + child->offset = offset; + mas_descend(child); + child->offset = 0; + return true; + } + } + return false; +} + +/* + * mab_shift_right() - Shift the data in mab right. Note, does not clean out the + * old data or set b_node->b_end. + * @b_node: the maple_big_node + * @shift: the shift count + */ +static inline void mab_shift_right(struct maple_big_node *b_node, + unsigned char shift) +{ + unsigned long size = b_node->b_end * sizeof(unsigned long); + + memmove(b_node->pivot + shift, b_node->pivot, size); + memmove(b_node->slot + shift, b_node->slot, size); + if (b_node->type == maple_arange_64) + memmove(b_node->gap + shift, b_node->gap, size); +} + +/* + * mab_middle_node() - Check if a middle node is needed (unlikely) + * @b_node: the maple_big_node that contains the data. + * @size: the amount of data in the b_node + * @split: the potential split location + * @slot_count: the size that can be stored in a single node being considered. + * + * Return: true if a middle node is required. + */ +static inline bool mab_middle_node(struct maple_big_node *b_node, int split, + unsigned char slot_count) +{ + unsigned char size = b_node->b_end; + + if (size >= 2 * slot_count) + return true; + + if (!b_node->slot[split] && (size >= 2 * slot_count - 1)) + return true; + + return false; +} + +/* + * mab_no_null_split() - ensure the split doesn't fall on a NULL + * @b_node: the maple_big_node with the data + * @split: the suggested split location + * @slot_count: the number of slots in the node being considered. + * + * Return: the split location. + */ +static inline int mab_no_null_split(struct maple_big_node *b_node, + unsigned char split, unsigned char slot_count) +{ + if (!b_node->slot[split]) { + /* + * If the split is less than the max slot && the right side will + * still be sufficient, then increment the split on NULL. + */ + if ((split < slot_count - 1) && + (b_node->b_end - split) > (mt_min_slots[b_node->type])) + split++; + else + split--; + } + return split; +} + +/* + * mab_calc_split() - Calculate the split location and if there needs to be two + * splits. + * @bn: The maple_big_node with the data + * @mid_split: The second split, if required. 0 otherwise. + * + * Return: The first split location. The middle split is set in @mid_split. + */ +static inline int mab_calc_split(struct ma_state *mas, + struct maple_big_node *bn, unsigned char *mid_split, unsigned long min) +{ + unsigned char b_end = bn->b_end; + int split = b_end / 2; /* Assume equal split. */ + unsigned char slot_min, slot_count = mt_slots[bn->type]; + + /* + * To support gap tracking, all NULL entries are kept together and a node cannot + * end on a NULL entry, with the exception of the left-most leaf. The + * limitation means that the split of a node must be checked for this condition + * and be able to put more data in one direction or the other. + */ + if (unlikely((mas->mas_flags & MA_STATE_BULK))) { + *mid_split = 0; + split = b_end - mt_min_slots[bn->type]; + + if (!ma_is_leaf(bn->type)) + return split; + + mas->mas_flags |= MA_STATE_REBALANCE; + if (!bn->slot[split]) + split--; + return split; + } + + /* + * Although extremely rare, it is possible to enter what is known as the 3-way + * split scenario. The 3-way split comes about by means of a store of a range + * that overwrites the end and beginning of two full nodes. The result is a set + * of entries that cannot be stored in 2 nodes. Sometimes, these two nodes can + * also be located in different parent nodes which are also full. This can + * carry upwards all the way to the root in the worst case. + */ + if (unlikely(mab_middle_node(bn, split, slot_count))) { + split = b_end / 3; + *mid_split = split * 2; + } else { + slot_min = mt_min_slots[bn->type]; + + *mid_split = 0; + /* + * Avoid having a range less than the slot count unless it + * causes one node to be deficient. + * NOTE: mt_min_slots is 1 based, b_end and split are zero. + */ + while (((bn->pivot[split] - min) < slot_count - 1) && + (split < slot_count - 1) && (b_end - split > slot_min)) + split++; + } + + /* Avoid ending a node on a NULL entry */ + split = mab_no_null_split(bn, split, slot_count); + if (!(*mid_split)) + return split; + + *mid_split = mab_no_null_split(bn, *mid_split, slot_count); + + return split; +} + +/* + * mas_mab_cp() - Copy data from a maple state inclusively to a maple_big_node + * and set @b_node->b_end to the next free slot. + * @mas: The maple state + * @mas_start: The starting slot to copy + * @mas_end: The end slot to copy (inclusively) + * @b_node: The maple_big_node to place the data + * @mab_start: The starting location in maple_big_node to store the data. + */ +static inline void mas_mab_cp(struct ma_state *mas, unsigned char mas_start, + unsigned char mas_end, struct maple_big_node *b_node, + unsigned char mab_start) +{ + enum maple_type mt; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots, *gaps; + int i = mas_start, j = mab_start; + unsigned char piv_end; + + node = mas_mn(mas); + mt = mte_node_type(mas->node); + pivots = ma_pivots(node, mt); + if (!i) { + b_node->pivot[j] = pivots[i++]; + if (unlikely(i > mas_end)) + goto complete; + j++; + } + + piv_end = min(mas_end, mt_pivots[mt]); + for (; i < piv_end; i++, j++) { + b_node->pivot[j] = pivots[i]; + if (unlikely(!b_node->pivot[j])) + break; + + if (unlikely(mas->max == b_node->pivot[j])) + goto complete; + } + + if (likely(i <= mas_end)) + b_node->pivot[j] = mas_safe_pivot(mas, pivots, i, mt); + +complete: + b_node->b_end = ++j; + j -= mab_start; + slots = ma_slots(node, mt); + memcpy(b_node->slot + mab_start, slots + mas_start, sizeof(void *) * j); + if (!ma_is_leaf(mt) && mt_is_alloc(mas->tree)) { + gaps = ma_gaps(node, mt); + memcpy(b_node->gap + mab_start, gaps + mas_start, + sizeof(unsigned long) * j); + } +} + +/* + * mas_leaf_set_meta() - Set the metadata of a leaf if possible. + * @mas: The maple state + * @node: The maple node + * @pivots: pointer to the maple node pivots + * @mt: The maple type + * @end: The assumed end + * + * Note, end may be incremented within this function but not modified at the + * source. This is fine since the metadata is the last thing to be stored in a + * node during a write. + */ +static inline void mas_leaf_set_meta(struct ma_state *mas, + struct maple_node *node, unsigned long *pivots, + enum maple_type mt, unsigned char end) +{ + /* There is no room for metadata already */ + if (mt_pivots[mt] <= end) + return; + + if (pivots[end] && pivots[end] < mas->max) + end++; + + if (end < mt_slots[mt] - 1) + ma_set_meta(node, mt, 0, end); +} + +/* + * mab_mas_cp() - Copy data from maple_big_node to a maple encoded node. + * @b_node: the maple_big_node that has the data + * @mab_start: the start location in @b_node. + * @mab_end: The end location in @b_node (inclusively) + * @mas: The maple state with the maple encoded node. + */ +static inline void mab_mas_cp(struct maple_big_node *b_node, + unsigned char mab_start, unsigned char mab_end, + struct ma_state *mas, bool new_max) +{ + int i, j = 0; + enum maple_type mt = mte_node_type(mas->node); + struct maple_node *node = mte_to_node(mas->node); + void __rcu **slots = ma_slots(node, mt); + unsigned long *pivots = ma_pivots(node, mt); + unsigned long *gaps = NULL; + unsigned char end; + + if (mab_end - mab_start > mt_pivots[mt]) + mab_end--; + + if (!pivots[mt_pivots[mt] - 1]) + slots[mt_pivots[mt]] = NULL; + + i = mab_start; + do { + pivots[j++] = b_node->pivot[i++]; + } while (i <= mab_end && likely(b_node->pivot[i])); + + memcpy(slots, b_node->slot + mab_start, + sizeof(void *) * (i - mab_start)); + + if (new_max) + mas->max = b_node->pivot[i - 1]; + + end = j - 1; + if (likely(!ma_is_leaf(mt) && mt_is_alloc(mas->tree))) { + unsigned long max_gap = 0; + unsigned char offset = 15; + + gaps = ma_gaps(node, mt); + do { + gaps[--j] = b_node->gap[--i]; + if (gaps[j] > max_gap) { + offset = j; + max_gap = gaps[j]; + } + } while (j); + + ma_set_meta(node, mt, offset, end); + } else { + mas_leaf_set_meta(mas, node, pivots, mt, end); + } +} + +/* + * mas_descend_adopt() - Descend through a sub-tree and adopt children. + * @mas: the maple state with the maple encoded node of the sub-tree. + * + * Descend through a sub-tree and adopt children who do not have the correct + * parents set. Follow the parents which have the correct parents as they are + * the new entries which need to be followed to find other incorrectly set + * parents. + */ +static inline void mas_descend_adopt(struct ma_state *mas) +{ + struct ma_state list[3], next[3]; + int i, n; + + /* + * At each level there may be up to 3 correct parent pointers which indicates + * the new nodes which need to be walked to find any new nodes at a lower level. + */ + + for (i = 0; i < 3; i++) { + list[i] = *mas; + list[i].offset = 0; + next[i].offset = 0; + } + next[0] = *mas; + + while (!mte_is_leaf(list[0].node)) { + n = 0; + for (i = 0; i < 3; i++) { + if (mas_is_none(&list[i])) + continue; + + if (i && list[i-1].node == list[i].node) + continue; + + while ((n < 3) && (mas_new_child(&list[i], &next[n]))) + n++; + + mas_adopt_children(&list[i], list[i].node); + } + + while (n < 3) + next[n++].node = MAS_NONE; + + /* descend by setting the list to the children */ + for (i = 0; i < 3; i++) + list[i] = next[i]; + } +} + +/* + * mas_bulk_rebalance() - Rebalance the end of a tree after a bulk insert. + * @mas: The maple state + * @end: The maple node end + * @mt: The maple node type + */ +static inline void mas_bulk_rebalance(struct ma_state *mas, unsigned char end, + enum maple_type mt) +{ + if (!(mas->mas_flags & MA_STATE_BULK)) + return; + + if (mte_is_root(mas->node)) + return; + + if (end > mt_min_slots[mt]) { + mas->mas_flags &= ~MA_STATE_REBALANCE; + return; + } +} + +/* + * mas_store_b_node() - Store an @entry into the b_node while also copying the + * data from a maple encoded node. + * @wr_mas: the maple write state + * @b_node: the maple_big_node to fill with data + * @offset_end: the offset to end copying + * + * Return: The actual end of the data stored in @b_node + */ +static inline void mas_store_b_node(struct ma_wr_state *wr_mas, + struct maple_big_node *b_node, unsigned char offset_end) +{ + unsigned char slot; + unsigned char b_end; + /* Possible underflow of piv will wrap back to 0 before use. */ + unsigned long piv; + struct ma_state *mas = wr_mas->mas; + + b_node->type = wr_mas->type; + b_end = 0; + slot = mas->offset; + if (slot) { + /* Copy start data up to insert. */ + mas_mab_cp(mas, 0, slot - 1, b_node, 0); + b_end = b_node->b_end; + piv = b_node->pivot[b_end - 1]; + } else + piv = mas->min - 1; + + if (piv + 1 < mas->index) { + /* Handle range starting after old range */ + b_node->slot[b_end] = wr_mas->content; + if (!wr_mas->content) + b_node->gap[b_end] = mas->index - 1 - piv; + b_node->pivot[b_end++] = mas->index - 1; + } + + /* Store the new entry. */ + mas->offset = b_end; + b_node->slot[b_end] = wr_mas->entry; + b_node->pivot[b_end] = mas->last; + + /* Appended. */ + if (mas->last >= mas->max) + goto b_end; + + /* Handle new range ending before old range ends */ + piv = mas_logical_pivot(mas, wr_mas->pivots, offset_end, wr_mas->type); + if (piv > mas->last) { + if (piv == ULONG_MAX) + mas_bulk_rebalance(mas, b_node->b_end, wr_mas->type); + + if (offset_end != slot) + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, + offset_end); + + b_node->slot[++b_end] = wr_mas->content; + if (!wr_mas->content) + b_node->gap[b_end] = piv - mas->last + 1; + b_node->pivot[b_end] = piv; + } + + slot = offset_end + 1; + if (slot > wr_mas->node_end) + goto b_end; + + /* Copy end data to the end of the node. */ + mas_mab_cp(mas, slot, wr_mas->node_end + 1, b_node, ++b_end); + b_node->b_end--; + return; + +b_end: + b_node->b_end = b_end; +} + +/* + * mas_prev_sibling() - Find the previous node with the same parent. + * @mas: the maple state + * + * Return: True if there is a previous sibling, false otherwise. + */ +static inline bool mas_prev_sibling(struct ma_state *mas) +{ + unsigned int p_slot = mte_parent_slot(mas->node); + + if (mte_is_root(mas->node)) + return false; + + if (!p_slot) + return false; + + mas_ascend(mas); + mas->offset = p_slot - 1; + mas_descend(mas); + return true; +} + +/* + * mas_next_sibling() - Find the next node with the same parent. + * @mas: the maple state + * + * Return: true if there is a next sibling, false otherwise. + */ +static inline bool mas_next_sibling(struct ma_state *mas) +{ + MA_STATE(parent, mas->tree, mas->index, mas->last); + + if (mte_is_root(mas->node)) + return false; + + parent = *mas; + mas_ascend(&parent); + parent.offset = mte_parent_slot(mas->node) + 1; + if (parent.offset > mas_data_end(&parent)) + return false; + + *mas = parent; + mas_descend(mas); + return true; +} + +/* + * mte_node_or_node() - Return the encoded node or MAS_NONE. + * @enode: The encoded maple node. + * + * Shorthand to avoid setting %NULLs in the tree or maple_subtree_state. + * + * Return: @enode or MAS_NONE + */ +static inline struct maple_enode *mte_node_or_none(struct maple_enode *enode) +{ + if (enode) + return enode; + + return ma_enode_ptr(MAS_NONE); +} + +/* + * mas_wr_node_walk() - Find the correct offset for the index in the @mas. + * @wr_mas: The maple write state + * + * Uses mas_slot_locked() and does not need to worry about dead nodes. + */ +static inline void mas_wr_node_walk(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned char count; + unsigned char offset; + unsigned long index, min, max; + + if (unlikely(ma_is_dense(wr_mas->type))) { + wr_mas->r_max = wr_mas->r_min = mas->index; + mas->offset = mas->index = mas->min; + return; + } + + wr_mas->node = mas_mn(wr_mas->mas); + wr_mas->pivots = ma_pivots(wr_mas->node, wr_mas->type); + count = wr_mas->node_end = ma_data_end(wr_mas->node, wr_mas->type, + wr_mas->pivots, mas->max); + offset = mas->offset; + min = mas_safe_min(mas, wr_mas->pivots, offset); + if (unlikely(offset == count)) + goto max; + + max = wr_mas->pivots[offset]; + index = mas->index; + if (unlikely(index <= max)) + goto done; + + if (unlikely(!max && offset)) + goto max; + + min = max + 1; + while (++offset < count) { + max = wr_mas->pivots[offset]; + if (index <= max) + goto done; + else if (unlikely(!max)) + break; + + min = max + 1; + } + +max: + max = mas->max; +done: + wr_mas->r_max = max; + wr_mas->r_min = min; + wr_mas->offset_end = mas->offset = offset; +} + +/* + * mas_topiary_range() - Add a range of slots to the topiary. + * @mas: The maple state + * @destroy: The topiary to add the slots (usually destroy) + * @start: The starting slot inclusively + * @end: The end slot inclusively + */ +static inline void mas_topiary_range(struct ma_state *mas, + struct ma_topiary *destroy, unsigned char start, unsigned char end) +{ + void __rcu **slots; + unsigned char offset; + + MT_BUG_ON(mas->tree, mte_is_leaf(mas->node)); + slots = ma_slots(mas_mn(mas), mte_node_type(mas->node)); + for (offset = start; offset <= end; offset++) { + struct maple_enode *enode = mas_slot_locked(mas, slots, offset); + + if (mte_dead_node(enode)) + continue; + + mat_add(destroy, enode); + } +} + +/* + * mast_topiary() - Add the portions of the tree to the removal list; either to + * be freed or discarded (destroy walk). + * @mast: The maple_subtree_state. + */ +static inline void mast_topiary(struct maple_subtree_state *mast) +{ + MA_WR_STATE(wr_mas, mast->orig_l, NULL); + unsigned char r_start, r_end; + unsigned char l_start, l_end; + void __rcu **l_slots, **r_slots; + + wr_mas.type = mte_node_type(mast->orig_l->node); + mast->orig_l->index = mast->orig_l->last; + mas_wr_node_walk(&wr_mas); + l_start = mast->orig_l->offset + 1; + l_end = mas_data_end(mast->orig_l); + r_start = 0; + r_end = mast->orig_r->offset; + + if (r_end) + r_end--; + + l_slots = ma_slots(mas_mn(mast->orig_l), + mte_node_type(mast->orig_l->node)); + + r_slots = ma_slots(mas_mn(mast->orig_r), + mte_node_type(mast->orig_r->node)); + + if ((l_start < l_end) && + mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_start))) { + l_start++; + } + + if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_end))) { + if (r_end) + r_end--; + } + + if ((l_start > r_end) && (mast->orig_l->node == mast->orig_r->node)) + return; + + /* At the node where left and right sides meet, add the parts between */ + if (mast->orig_l->node == mast->orig_r->node) { + return mas_topiary_range(mast->orig_l, mast->destroy, + l_start, r_end); + } + + /* mast->orig_r is different and consumed. */ + if (mte_is_leaf(mast->orig_r->node)) + return; + + if (mte_dead_node(mas_slot_locked(mast->orig_l, l_slots, l_end))) + l_end--; + + + if (l_start <= l_end) + mas_topiary_range(mast->orig_l, mast->destroy, l_start, l_end); + + if (mte_dead_node(mas_slot_locked(mast->orig_r, r_slots, r_start))) + r_start++; + + if (r_start <= r_end) + mas_topiary_range(mast->orig_r, mast->destroy, 0, r_end); +} + +/* + * mast_rebalance_next() - Rebalance against the next node + * @mast: The maple subtree state + * @old_r: The encoded maple node to the right (next node). + */ +static inline void mast_rebalance_next(struct maple_subtree_state *mast) +{ + unsigned char b_end = mast->bn->b_end; + + mas_mab_cp(mast->orig_r, 0, mt_slot_count(mast->orig_r->node), + mast->bn, b_end); + mast->orig_r->last = mast->orig_r->max; +} + +/* + * mast_rebalance_prev() - Rebalance against the previous node + * @mast: The maple subtree state + * @old_l: The encoded maple node to the left (previous node) + */ +static inline void mast_rebalance_prev(struct maple_subtree_state *mast) +{ + unsigned char end = mas_data_end(mast->orig_l) + 1; + unsigned char b_end = mast->bn->b_end; + + mab_shift_right(mast->bn, end); + mas_mab_cp(mast->orig_l, 0, end - 1, mast->bn, 0); + mast->l->min = mast->orig_l->min; + mast->orig_l->index = mast->orig_l->min; + mast->bn->b_end = end + b_end; + mast->l->offset += end; +} + +/* + * mast_spanning_rebalance() - Rebalance nodes with nearest neighbour favouring + * the node to the right. Checking the nodes to the right then the left at each + * level upwards until root is reached. Free and destroy as needed. + * Data is copied into the @mast->bn. + * @mast: The maple_subtree_state. + */ +static inline +bool mast_spanning_rebalance(struct maple_subtree_state *mast) +{ + struct ma_state r_tmp = *mast->orig_r; + struct ma_state l_tmp = *mast->orig_l; + struct maple_enode *ancestor = NULL; + unsigned char start, end; + unsigned char depth = 0; + + r_tmp = *mast->orig_r; + l_tmp = *mast->orig_l; + do { + mas_ascend(mast->orig_r); + mas_ascend(mast->orig_l); + depth++; + if (!ancestor && + (mast->orig_r->node == mast->orig_l->node)) { + ancestor = mast->orig_r->node; + end = mast->orig_r->offset - 1; + start = mast->orig_l->offset + 1; + } + + if (mast->orig_r->offset < mas_data_end(mast->orig_r)) { + if (!ancestor) { + ancestor = mast->orig_r->node; + start = 0; + } + + mast->orig_r->offset++; + do { + mas_descend(mast->orig_r); + mast->orig_r->offset = 0; + depth--; + } while (depth); + + mast_rebalance_next(mast); + do { + unsigned char l_off = 0; + struct maple_enode *child = r_tmp.node; + + mas_ascend(&r_tmp); + if (ancestor == r_tmp.node) + l_off = start; + + if (r_tmp.offset) + r_tmp.offset--; + + if (l_off < r_tmp.offset) + mas_topiary_range(&r_tmp, mast->destroy, + l_off, r_tmp.offset); + + if (l_tmp.node != child) + mat_add(mast->free, child); + + } while (r_tmp.node != ancestor); + + *mast->orig_l = l_tmp; + return true; + + } else if (mast->orig_l->offset != 0) { + if (!ancestor) { + ancestor = mast->orig_l->node; + end = mas_data_end(mast->orig_l); + } + + mast->orig_l->offset--; + do { + mas_descend(mast->orig_l); + mast->orig_l->offset = + mas_data_end(mast->orig_l); + depth--; + } while (depth); + + mast_rebalance_prev(mast); + do { + unsigned char r_off; + struct maple_enode *child = l_tmp.node; + + mas_ascend(&l_tmp); + if (ancestor == l_tmp.node) + r_off = end; + else + r_off = mas_data_end(&l_tmp); + + if (l_tmp.offset < r_off) + l_tmp.offset++; + + if (l_tmp.offset < r_off) + mas_topiary_range(&l_tmp, mast->destroy, + l_tmp.offset, r_off); + + if (r_tmp.node != child) + mat_add(mast->free, child); + + } while (l_tmp.node != ancestor); + + *mast->orig_r = r_tmp; + return true; + } + } while (!mte_is_root(mast->orig_r->node)); + + *mast->orig_r = r_tmp; + *mast->orig_l = l_tmp; + return false; +} + +/* + * mast_ascend_free() - Add current original maple state nodes to the free list + * and ascend. + * @mast: the maple subtree state. + * + * Ascend the original left and right sides and add the previous nodes to the + * free list. Set the slots to point to the correct location in the new nodes. + */ +static inline void +mast_ascend_free(struct maple_subtree_state *mast) +{ + MA_WR_STATE(wr_mas, mast->orig_r, NULL); + struct maple_enode *left = mast->orig_l->node; + struct maple_enode *right = mast->orig_r->node; + + mas_ascend(mast->orig_l); + mas_ascend(mast->orig_r); + mat_add(mast->free, left); + + if (left != right) + mat_add(mast->free, right); + + mast->orig_r->offset = 0; + mast->orig_r->index = mast->r->max; + /* last should be larger than or equal to index */ + if (mast->orig_r->last < mast->orig_r->index) + mast->orig_r->last = mast->orig_r->index; + /* + * The node may not contain the value so set slot to ensure all + * of the nodes contents are freed or destroyed. + */ + wr_mas.type = mte_node_type(mast->orig_r->node); + mas_wr_node_walk(&wr_mas); + /* Set up the left side of things */ + mast->orig_l->offset = 0; + mast->orig_l->index = mast->l->min; + wr_mas.mas = mast->orig_l; + wr_mas.type = mte_node_type(mast->orig_l->node); + mas_wr_node_walk(&wr_mas); + + mast->bn->type = wr_mas.type; +} + +/* + * mas_new_ma_node() - Create and return a new maple node. Helper function. + * @mas: the maple state with the allocations. + * @b_node: the maple_big_node with the type encoding. + * + * Use the node type from the maple_big_node to allocate a new node from the + * ma_state. This function exists mainly for code readability. + * + * Return: A new maple encoded node + */ +static inline struct maple_enode +*mas_new_ma_node(struct ma_state *mas, struct maple_big_node *b_node) +{ + return mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), b_node->type); +} + +/* + * mas_mab_to_node() - Set up right and middle nodes + * + * @mas: the maple state that contains the allocations. + * @b_node: the node which contains the data. + * @left: The pointer which will have the left node + * @right: The pointer which may have the right node + * @middle: the pointer which may have the middle node (rare) + * @mid_split: the split location for the middle node + * + * Return: the split of left. + */ +static inline unsigned char mas_mab_to_node(struct ma_state *mas, + struct maple_big_node *b_node, struct maple_enode **left, + struct maple_enode **right, struct maple_enode **middle, + unsigned char *mid_split, unsigned long min) +{ + unsigned char split = 0; + unsigned char slot_count = mt_slots[b_node->type]; + + *left = mas_new_ma_node(mas, b_node); + *right = NULL; + *middle = NULL; + *mid_split = 0; + + if (b_node->b_end < slot_count) { + split = b_node->b_end; + } else { + split = mab_calc_split(mas, b_node, mid_split, min); + *right = mas_new_ma_node(mas, b_node); + } + + if (*mid_split) + *middle = mas_new_ma_node(mas, b_node); + + return split; + +} + +/* + * mab_set_b_end() - Add entry to b_node at b_node->b_end and increment the end + * pointer. + * @b_node - the big node to add the entry + * @mas - the maple state to get the pivot (mas->max) + * @entry - the entry to add, if NULL nothing happens. + */ +static inline void mab_set_b_end(struct maple_big_node *b_node, + struct ma_state *mas, + void *entry) +{ + if (!entry) + return; + + b_node->slot[b_node->b_end] = entry; + if (mt_is_alloc(mas->tree)) + b_node->gap[b_node->b_end] = mas_max_gap(mas); + b_node->pivot[b_node->b_end++] = mas->max; +} + +/* + * mas_set_split_parent() - combine_then_separate helper function. Sets the parent + * of @mas->node to either @left or @right, depending on @slot and @split + * + * @mas - the maple state with the node that needs a parent + * @left - possible parent 1 + * @right - possible parent 2 + * @slot - the slot the mas->node was placed + * @split - the split location between @left and @right + */ +static inline void mas_set_split_parent(struct ma_state *mas, + struct maple_enode *left, + struct maple_enode *right, + unsigned char *slot, unsigned char split) +{ + if (mas_is_none(mas)) + return; + + if ((*slot) <= split) + mte_set_parent(mas->node, left, *slot); + else if (right) + mte_set_parent(mas->node, right, (*slot) - split - 1); + + (*slot)++; +} + +/* + * mte_mid_split_check() - Check if the next node passes the mid-split + * @**l: Pointer to left encoded maple node. + * @**m: Pointer to middle encoded maple node. + * @**r: Pointer to right encoded maple node. + * @slot: The offset + * @*split: The split location. + * @mid_split: The middle split. + */ +static inline void mte_mid_split_check(struct maple_enode **l, + struct maple_enode **r, + struct maple_enode *right, + unsigned char slot, + unsigned char *split, + unsigned char mid_split) +{ + if (*r == right) + return; + + if (slot < mid_split) + return; + + *l = *r; + *r = right; + *split = mid_split; +} + +/* + * mast_set_split_parents() - Helper function to set three nodes parents. Slot + * is taken from @mast->l. + * @mast - the maple subtree state + * @left - the left node + * @right - the right node + * @split - the split location. + */ +static inline void mast_set_split_parents(struct maple_subtree_state *mast, + struct maple_enode *left, + struct maple_enode *middle, + struct maple_enode *right, + unsigned char split, + unsigned char mid_split) +{ + unsigned char slot; + struct maple_enode *l = left; + struct maple_enode *r = right; + + if (mas_is_none(mast->l)) + return; + + if (middle) + r = middle; + + slot = mast->l->offset; + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->l, l, r, &slot, split); + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->m, l, r, &slot, split); + + mte_mid_split_check(&l, &r, right, slot, &split, mid_split); + mas_set_split_parent(mast->r, l, r, &slot, split); +} + +/* + * mas_wmb_replace() - Write memory barrier and replace + * @mas: The maple state + * @free: the maple topiary list of nodes to free + * @destroy: The maple topiary list of nodes to destroy (walk and free) + * + * Updates gap as necessary. + */ +static inline void mas_wmb_replace(struct ma_state *mas, + struct ma_topiary *free, + struct ma_topiary *destroy) +{ + /* All nodes must see old data as dead prior to replacing that data */ + smp_wmb(); /* Needed for RCU */ + + /* Insert the new data in the tree */ + mas_replace(mas, true); + + if (!mte_is_leaf(mas->node)) + mas_descend_adopt(mas); + + mas_mat_free(mas, free); + + if (destroy) + mas_mat_destroy(mas, destroy); + + if (mte_is_leaf(mas->node)) + return; + + mas_update_gap(mas); +} + +/* + * mast_new_root() - Set a new tree root during subtree creation + * @mast: The maple subtree state + * @mas: The maple state + */ +static inline void mast_new_root(struct maple_subtree_state *mast, + struct ma_state *mas) +{ + mas_mn(mast->l)->parent = + ma_parent_ptr(((unsigned long)mas->tree | MA_ROOT_PARENT)); + if (!mte_dead_node(mast->orig_l->node) && + !mte_is_root(mast->orig_l->node)) { + do { + mast_ascend_free(mast); + mast_topiary(mast); + } while (!mte_is_root(mast->orig_l->node)); + } + if ((mast->orig_l->node != mas->node) && + (mast->l->depth > mas_mt_height(mas))) { + mat_add(mast->free, mas->node); + } +} + +/* + * mast_cp_to_nodes() - Copy data out to nodes. + * @mast: The maple subtree state + * @left: The left encoded maple node + * @middle: The middle encoded maple node + * @right: The right encoded maple node + * @split: The location to split between left and (middle ? middle : right) + * @mid_split: The location to split between middle and right. + */ +static inline void mast_cp_to_nodes(struct maple_subtree_state *mast, + struct maple_enode *left, struct maple_enode *middle, + struct maple_enode *right, unsigned char split, unsigned char mid_split) +{ + bool new_lmax = true; + + mast->l->node = mte_node_or_none(left); + mast->m->node = mte_node_or_none(middle); + mast->r->node = mte_node_or_none(right); + + mast->l->min = mast->orig_l->min; + if (split == mast->bn->b_end) { + mast->l->max = mast->orig_r->max; + new_lmax = false; + } + + mab_mas_cp(mast->bn, 0, split, mast->l, new_lmax); + + if (middle) { + mab_mas_cp(mast->bn, 1 + split, mid_split, mast->m, true); + mast->m->min = mast->bn->pivot[split] + 1; + split = mid_split; + } + + mast->r->max = mast->orig_r->max; + if (right) { + mab_mas_cp(mast->bn, 1 + split, mast->bn->b_end, mast->r, false); + mast->r->min = mast->bn->pivot[split] + 1; + } +} + +/* + * mast_combine_cp_left - Copy in the original left side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state + */ +static inline void mast_combine_cp_left(struct maple_subtree_state *mast) +{ + unsigned char l_slot = mast->orig_l->offset; + + if (!l_slot) + return; + + mas_mab_cp(mast->orig_l, 0, l_slot - 1, mast->bn, 0); +} + +/* + * mast_combine_cp_right: Copy in the original right side of the tree into the + * combined data set in the maple subtree state big node. + * @mast: The maple subtree state + */ +static inline void mast_combine_cp_right(struct maple_subtree_state *mast) +{ + if (mast->bn->pivot[mast->bn->b_end - 1] >= mast->orig_r->max) + return; + + mas_mab_cp(mast->orig_r, mast->orig_r->offset + 1, + mt_slot_count(mast->orig_r->node), mast->bn, + mast->bn->b_end); + mast->orig_r->last = mast->orig_r->max; +} + +/* + * mast_sufficient: Check if the maple subtree state has enough data in the big + * node to create at least one sufficient node + * @mast: the maple subtree state + */ +static inline bool mast_sufficient(struct maple_subtree_state *mast) +{ + if (mast->bn->b_end > mt_min_slot_count(mast->orig_l->node)) + return true; + + return false; +} + +/* + * mast_overflow: Check if there is too much data in the subtree state for a + * single node. + * @mast: The maple subtree state + */ +static inline bool mast_overflow(struct maple_subtree_state *mast) +{ + if (mast->bn->b_end >= mt_slot_count(mast->orig_l->node)) + return true; + + return false; +} + +static inline void *mtree_range_walk(struct ma_state *mas) +{ + unsigned long *pivots; + unsigned char offset; + struct maple_node *node; + struct maple_enode *next, *last; + enum maple_type type; + void __rcu **slots; + unsigned char end; + unsigned long max, min; + unsigned long prev_max, prev_min; + + last = next = mas->node; + prev_min = min = mas->min; + max = mas->max; + do { + offset = 0; + last = next; + node = mte_to_node(next); + type = mte_node_type(next); + pivots = ma_pivots(node, type); + end = ma_data_end(node, type, pivots, max); + if (unlikely(ma_dead_node(node))) + goto dead_node; + + if (pivots[offset] >= mas->index) { + prev_max = max; + prev_min = min; + max = pivots[offset]; + goto next; + } + + do { + offset++; + } while ((offset < end) && (pivots[offset] < mas->index)); + + prev_min = min; + min = pivots[offset - 1] + 1; + prev_max = max; + if (likely(offset < end && pivots[offset])) + max = pivots[offset]; + +next: + slots = ma_slots(node, type); + next = mt_slot(mas->tree, slots, offset); + if (unlikely(ma_dead_node(node))) + goto dead_node; + } while (!ma_is_leaf(type)); + + mas->offset = offset; + mas->index = min; + mas->last = max; + mas->min = prev_min; + mas->max = prev_max; + mas->node = last; + return (void *) next; + +dead_node: + mas_reset(mas); + return NULL; +} + +/* + * mas_spanning_rebalance() - Rebalance across two nodes which may not be peers. + * @mas: The starting maple state + * @mast: The maple_subtree_state, keeps track of 4 maple states. + * @count: The estimated count of iterations needed. + * + * Follow the tree upwards from @l_mas and @r_mas for @count, or until the root + * is hit. First @b_node is split into two entries which are inserted into the + * next iteration of the loop. @b_node is returned populated with the final + * iteration. @mas is used to obtain allocations. orig_l_mas keeps track of the + * nodes that will remain active by using orig_l_mas->index and orig_l_mas->last + * to account of what has been copied into the new sub-tree. The update of + * orig_l_mas->last is used in mas_consume to find the slots that will need to + * be either freed or destroyed. orig_l_mas->depth keeps track of the height of + * the new sub-tree in case the sub-tree becomes the full tree. + * + * Return: the number of elements in b_node during the last loop. + */ +static int mas_spanning_rebalance(struct ma_state *mas, + struct maple_subtree_state *mast, unsigned char count) +{ + unsigned char split, mid_split; + unsigned char slot = 0; + struct maple_enode *left = NULL, *middle = NULL, *right = NULL; + + MA_STATE(l_mas, mas->tree, mas->index, mas->index); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + MA_STATE(m_mas, mas->tree, mas->index, mas->index); + MA_TOPIARY(free, mas->tree); + MA_TOPIARY(destroy, mas->tree); + + /* + * The tree needs to be rebalanced and leaves need to be kept at the same level. + * Rebalancing is done by use of the ``struct maple_topiary``. + */ + mast->l = &l_mas; + mast->m = &m_mas; + mast->r = &r_mas; + mast->free = &free; + mast->destroy = &destroy; + l_mas.node = r_mas.node = m_mas.node = MAS_NONE; + if (!(mast->orig_l->min && mast->orig_r->max == ULONG_MAX) && + unlikely(mast->bn->b_end <= mt_min_slots[mast->bn->type])) + mast_spanning_rebalance(mast); + + mast->orig_l->depth = 0; + + /* + * Each level of the tree is examined and balanced, pushing data to the left or + * right, or rebalancing against left or right nodes is employed to avoid + * rippling up the tree to limit the amount of churn. Once a new sub-section of + * the tree is created, there may be a mix of new and old nodes. The old nodes + * will have the incorrect parent pointers and currently be in two trees: the + * original tree and the partially new tree. To remedy the parent pointers in + * the old tree, the new data is swapped into the active tree and a walk down + * the tree is performed and the parent pointers are updated. + * See mas_descend_adopt() for more information.. + */ + while (count--) { + mast->bn->b_end--; + mast->bn->type = mte_node_type(mast->orig_l->node); + split = mas_mab_to_node(mas, mast->bn, &left, &right, &middle, + &mid_split, mast->orig_l->min); + mast_set_split_parents(mast, left, middle, right, split, + mid_split); + mast_cp_to_nodes(mast, left, middle, right, split, mid_split); + + /* + * Copy data from next level in the tree to mast->bn from next + * iteration + */ + memset(mast->bn, 0, sizeof(struct maple_big_node)); + mast->bn->type = mte_node_type(left); + mast->orig_l->depth++; + + /* Root already stored in l->node. */ + if (mas_is_root_limits(mast->l)) + goto new_root; + + mast_ascend_free(mast); + mast_combine_cp_left(mast); + l_mas.offset = mast->bn->b_end; + mab_set_b_end(mast->bn, &l_mas, left); + mab_set_b_end(mast->bn, &m_mas, middle); + mab_set_b_end(mast->bn, &r_mas, right); + + /* Copy anything necessary out of the right node. */ + mast_combine_cp_right(mast); + mast_topiary(mast); + mast->orig_l->last = mast->orig_l->max; + + if (mast_sufficient(mast)) + continue; + + if (mast_overflow(mast)) + continue; + + /* May be a new root stored in mast->bn */ + if (mas_is_root_limits(mast->orig_l)) + break; + + mast_spanning_rebalance(mast); + + /* rebalancing from other nodes may require another loop. */ + if (!count) + count++; + } + + l_mas.node = mt_mk_node(ma_mnode_ptr(mas_pop_node(mas)), + mte_node_type(mast->orig_l->node)); + mast->orig_l->depth++; + mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, &l_mas, true); + mte_set_parent(left, l_mas.node, slot); + if (middle) + mte_set_parent(middle, l_mas.node, ++slot); + + if (right) + mte_set_parent(right, l_mas.node, ++slot); + + if (mas_is_root_limits(mast->l)) { +new_root: + mast_new_root(mast, mas); + } else { + mas_mn(&l_mas)->parent = mas_mn(mast->orig_l)->parent; + } + + if (!mte_dead_node(mast->orig_l->node)) + mat_add(&free, mast->orig_l->node); + + mas->depth = mast->orig_l->depth; + *mast->orig_l = l_mas; + mte_set_node_dead(mas->node); + + /* Set up mas for insertion. */ + mast->orig_l->depth = mas->depth; + mast->orig_l->alloc = mas->alloc; + *mas = *mast->orig_l; + mas_wmb_replace(mas, &free, &destroy); + mtree_range_walk(mas); + return mast->bn->b_end; +} + +/* + * mas_rebalance() - Rebalance a given node. + * @mas: The maple state + * @b_node: The big maple node. + * + * Rebalance two nodes into a single node or two new nodes that are sufficient. + * Continue upwards until tree is sufficient. + * + * Return: the number of elements in b_node during the last loop. + */ +static inline int mas_rebalance(struct ma_state *mas, + struct maple_big_node *b_node) +{ + char empty_count = mas_mt_height(mas); + struct maple_subtree_state mast; + unsigned char shift, b_end = ++b_node->b_end; + + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + + trace_ma_op(__func__, mas); + + /* + * Rebalancing occurs if a node is insufficient. Data is rebalanced + * against the node to the right if it exists, otherwise the node to the + * left of this node is rebalanced against this node. If rebalancing + * causes just one node to be produced instead of two, then the parent + * is also examined and rebalanced if it is insufficient. Every level + * tries to combine the data in the same way. If one node contains the + * entire range of the tree, then that node is used as a new root node. + */ + mas_node_count(mas, 1 + empty_count * 3); + if (mas_is_err(mas)) + return 0; + + mast.orig_l = &l_mas; + mast.orig_r = &r_mas; + mast.bn = b_node; + mast.bn->type = mte_node_type(mas->node); + + l_mas = r_mas = *mas; + + if (mas_next_sibling(&r_mas)) { + mas_mab_cp(&r_mas, 0, mt_slot_count(r_mas.node), b_node, b_end); + r_mas.last = r_mas.index = r_mas.max; + } else { + mas_prev_sibling(&l_mas); + shift = mas_data_end(&l_mas) + 1; + mab_shift_right(b_node, shift); + mas->offset += shift; + mas_mab_cp(&l_mas, 0, shift - 1, b_node, 0); + b_node->b_end = shift + b_end; + l_mas.index = l_mas.last = l_mas.min; + } + + return mas_spanning_rebalance(mas, &mast, empty_count); +} + +/* + * mas_destroy_rebalance() - Rebalance left-most node while destroying the maple + * state. + * @mas: The maple state + * @end: The end of the left-most node. + * + * During a mass-insert event (such as forking), it may be necessary to + * rebalance the left-most node when it is not sufficient. + */ +static inline void mas_destroy_rebalance(struct ma_state *mas, unsigned char end) +{ + enum maple_type mt = mte_node_type(mas->node); + struct maple_node reuse, *newnode, *parent, *new_left, *left, *node; + struct maple_enode *eparent; + unsigned char offset, tmp, split = mt_slots[mt] / 2; + void __rcu **l_slots, **slots; + unsigned long *l_pivs, *pivs, gap; + bool in_rcu = mt_in_rcu(mas->tree); + + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + + l_mas = *mas; + mas_prev_sibling(&l_mas); + + /* set up node. */ + if (in_rcu) { + /* Allocate for both left and right as well as parent. */ + mas_node_count(mas, 3); + if (mas_is_err(mas)) + return; + + newnode = mas_pop_node(mas); + } else { + newnode = &reuse; + } + + node = mas_mn(mas); + newnode->parent = node->parent; + slots = ma_slots(newnode, mt); + pivs = ma_pivots(newnode, mt); + left = mas_mn(&l_mas); + l_slots = ma_slots(left, mt); + l_pivs = ma_pivots(left, mt); + if (!l_slots[split]) + split++; + tmp = mas_data_end(&l_mas) - split; + + memcpy(slots, l_slots + split + 1, sizeof(void *) * tmp); + memcpy(pivs, l_pivs + split + 1, sizeof(unsigned long) * tmp); + pivs[tmp] = l_mas.max; + memcpy(slots + tmp, ma_slots(node, mt), sizeof(void *) * end); + memcpy(pivs + tmp, ma_pivots(node, mt), sizeof(unsigned long) * end); + + l_mas.max = l_pivs[split]; + mas->min = l_mas.max + 1; + eparent = mt_mk_node(mte_parent(l_mas.node), + mas_parent_enum(&l_mas, l_mas.node)); + tmp += end; + if (!in_rcu) { + unsigned char max_p = mt_pivots[mt]; + unsigned char max_s = mt_slots[mt]; + + if (tmp < max_p) + memset(pivs + tmp, 0, + sizeof(unsigned long *) * (max_p - tmp)); + + if (tmp < mt_slots[mt]) + memset(slots + tmp, 0, sizeof(void *) * (max_s - tmp)); + + memcpy(node, newnode, sizeof(struct maple_node)); + ma_set_meta(node, mt, 0, tmp - 1); + mte_set_pivot(eparent, mte_parent_slot(l_mas.node), + l_pivs[split]); + + /* Remove data from l_pivs. */ + tmp = split + 1; + memset(l_pivs + tmp, 0, sizeof(unsigned long) * (max_p - tmp)); + memset(l_slots + tmp, 0, sizeof(void *) * (max_s - tmp)); + ma_set_meta(left, mt, 0, split); + + goto done; + } + + /* RCU requires replacing both l_mas, mas, and parent. */ + mas->node = mt_mk_node(newnode, mt); + ma_set_meta(newnode, mt, 0, tmp); + + new_left = mas_pop_node(mas); + new_left->parent = left->parent; + mt = mte_node_type(l_mas.node); + slots = ma_slots(new_left, mt); + pivs = ma_pivots(new_left, mt); + memcpy(slots, l_slots, sizeof(void *) * split); + memcpy(pivs, l_pivs, sizeof(unsigned long) * split); + ma_set_meta(new_left, mt, 0, split); + l_mas.node = mt_mk_node(new_left, mt); + + /* replace parent. */ + offset = mte_parent_slot(mas->node); + mt = mas_parent_enum(&l_mas, l_mas.node); + parent = mas_pop_node(mas); + slots = ma_slots(parent, mt); + pivs = ma_pivots(parent, mt); + memcpy(parent, mte_to_node(eparent), sizeof(struct maple_node)); + rcu_assign_pointer(slots[offset], mas->node); + rcu_assign_pointer(slots[offset - 1], l_mas.node); + pivs[offset - 1] = l_mas.max; + eparent = mt_mk_node(parent, mt); +done: + gap = mas_leaf_max_gap(mas); + mte_set_gap(eparent, mte_parent_slot(mas->node), gap); + gap = mas_leaf_max_gap(&l_mas); + mte_set_gap(eparent, mte_parent_slot(l_mas.node), gap); + mas_ascend(mas); + + if (in_rcu) + mas_replace(mas, false); + + mas_update_gap(mas); +} + +/* + * mas_split_final_node() - Split the final node in a subtree operation. + * @mast: the maple subtree state + * @mas: The maple state + * @height: The height of the tree in case it's a new root. + */ +static inline bool mas_split_final_node(struct maple_subtree_state *mast, + struct ma_state *mas, int height) +{ + struct maple_enode *ancestor; + + if (mte_is_root(mas->node)) { + if (mt_is_alloc(mas->tree)) + mast->bn->type = maple_arange_64; + else + mast->bn->type = maple_range_64; + mas->depth = height; + } + /* + * Only a single node is used here, could be root. + * The Big_node data should just fit in a single node. + */ + ancestor = mas_new_ma_node(mas, mast->bn); + mte_set_parent(mast->l->node, ancestor, mast->l->offset); + mte_set_parent(mast->r->node, ancestor, mast->r->offset); + mte_to_node(ancestor)->parent = mas_mn(mas)->parent; + + mast->l->node = ancestor; + mab_mas_cp(mast->bn, 0, mt_slots[mast->bn->type] - 1, mast->l, true); + mas->offset = mast->bn->b_end - 1; + return true; +} + +/* + * mast_fill_bnode() - Copy data into the big node in the subtree state + * @mast: The maple subtree state + * @mas: the maple state + * @skip: The number of entries to skip for new nodes insertion. + */ +static inline void mast_fill_bnode(struct maple_subtree_state *mast, + struct ma_state *mas, + unsigned char skip) +{ + bool cp = true; + struct maple_enode *old = mas->node; + unsigned char split; + + memset(mast->bn->gap, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->gap)); + memset(mast->bn->slot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->slot)); + memset(mast->bn->pivot, 0, sizeof(unsigned long) * ARRAY_SIZE(mast->bn->pivot)); + mast->bn->b_end = 0; + + if (mte_is_root(mas->node)) { + cp = false; + } else { + mas_ascend(mas); + mat_add(mast->free, old); + mas->offset = mte_parent_slot(mas->node); + } + + if (cp && mast->l->offset) + mas_mab_cp(mas, 0, mast->l->offset - 1, mast->bn, 0); + + split = mast->bn->b_end; + mab_set_b_end(mast->bn, mast->l, mast->l->node); + mast->r->offset = mast->bn->b_end; + mab_set_b_end(mast->bn, mast->r, mast->r->node); + if (mast->bn->pivot[mast->bn->b_end - 1] == mas->max) + cp = false; + + if (cp) + mas_mab_cp(mas, split + skip, mt_slot_count(mas->node) - 1, + mast->bn, mast->bn->b_end); + + mast->bn->b_end--; + mast->bn->type = mte_node_type(mas->node); +} + +/* + * mast_split_data() - Split the data in the subtree state big node into regular + * nodes. + * @mast: The maple subtree state + * @mas: The maple state + * @split: The location to split the big node + */ +static inline void mast_split_data(struct maple_subtree_state *mast, + struct ma_state *mas, unsigned char split) +{ + unsigned char p_slot; + + mab_mas_cp(mast->bn, 0, split, mast->l, true); + mte_set_pivot(mast->r->node, 0, mast->r->max); + mab_mas_cp(mast->bn, split + 1, mast->bn->b_end, mast->r, false); + mast->l->offset = mte_parent_slot(mas->node); + mast->l->max = mast->bn->pivot[split]; + mast->r->min = mast->l->max + 1; + if (mte_is_leaf(mas->node)) + return; + + p_slot = mast->orig_l->offset; + mas_set_split_parent(mast->orig_l, mast->l->node, mast->r->node, + &p_slot, split); + mas_set_split_parent(mast->orig_r, mast->l->node, mast->r->node, + &p_slot, split); +} + +/* + * mas_push_data() - Instead of splitting a node, it is beneficial to push the + * data to the right or left node if there is room. + * @mas: The maple state + * @height: The current height of the maple state + * @mast: The maple subtree state + * @left: Push left or not. + * + * Keeping the height of the tree low means faster lookups. + * + * Return: True if pushed, false otherwise. + */ +static inline bool mas_push_data(struct ma_state *mas, int height, + struct maple_subtree_state *mast, bool left) +{ + unsigned char slot_total = mast->bn->b_end; + unsigned char end, space, split; + + MA_STATE(tmp_mas, mas->tree, mas->index, mas->last); + tmp_mas = *mas; + tmp_mas.depth = mast->l->depth; + + if (left && !mas_prev_sibling(&tmp_mas)) + return false; + else if (!left && !mas_next_sibling(&tmp_mas)) + return false; + + end = mas_data_end(&tmp_mas); + slot_total += end; + space = 2 * mt_slot_count(mas->node) - 2; + /* -2 instead of -1 to ensure there isn't a triple split */ + if (ma_is_leaf(mast->bn->type)) + space--; + + if (mas->max == ULONG_MAX) + space--; + + if (slot_total >= space) + return false; + + /* Get the data; Fill mast->bn */ + mast->bn->b_end++; + if (left) { + mab_shift_right(mast->bn, end + 1); + mas_mab_cp(&tmp_mas, 0, end, mast->bn, 0); + mast->bn->b_end = slot_total + 1; + } else { + mas_mab_cp(&tmp_mas, 0, end, mast->bn, mast->bn->b_end); + } + + /* Configure mast for splitting of mast->bn */ + split = mt_slots[mast->bn->type] - 2; + if (left) { + /* Switch mas to prev node */ + mat_add(mast->free, mas->node); + *mas = tmp_mas; + /* Start using mast->l for the left side. */ + tmp_mas.node = mast->l->node; + *mast->l = tmp_mas; + } else { + mat_add(mast->free, tmp_mas.node); + tmp_mas.node = mast->r->node; + *mast->r = tmp_mas; + split = slot_total - split; + } + split = mab_no_null_split(mast->bn, split, mt_slots[mast->bn->type]); + /* Update parent slot for split calculation. */ + if (left) + mast->orig_l->offset += end + 1; + + mast_split_data(mast, mas, split); + mast_fill_bnode(mast, mas, 2); + mas_split_final_node(mast, mas, height + 1); + return true; +} + +/* + * mas_split() - Split data that is too big for one node into two. + * @mas: The maple state + * @b_node: The maple big node + * Return: 1 on success, 0 on failure. + */ +static int mas_split(struct ma_state *mas, struct maple_big_node *b_node) +{ + + struct maple_subtree_state mast; + int height = 0; + unsigned char mid_split, split = 0; + + /* + * Splitting is handled differently from any other B-tree; the Maple + * Tree splits upwards. Splitting up means that the split operation + * occurs when the walk of the tree hits the leaves and not on the way + * down. The reason for splitting up is that it is impossible to know + * how much space will be needed until the leaf is (or leaves are) + * reached. Since overwriting data is allowed and a range could + * overwrite more than one range or result in changing one entry into 3 + * entries, it is impossible to know if a split is required until the + * data is examined. + * + * Splitting is a balancing act between keeping allocations to a minimum + * and avoiding a 'jitter' event where a tree is expanded to make room + * for an entry followed by a contraction when the entry is removed. To + * accomplish the balance, there are empty slots remaining in both left + * and right nodes after a split. + */ + MA_STATE(l_mas, mas->tree, mas->index, mas->last); + MA_STATE(r_mas, mas->tree, mas->index, mas->last); + MA_STATE(prev_l_mas, mas->tree, mas->index, mas->last); + MA_STATE(prev_r_mas, mas->tree, mas->index, mas->last); + MA_TOPIARY(mat, mas->tree); + + trace_ma_op(__func__, mas); + mas->depth = mas_mt_height(mas); + /* Allocation failures will happen early. */ + mas_node_count(mas, 1 + mas->depth * 2); + if (mas_is_err(mas)) + return 0; + + mast.l = &l_mas; + mast.r = &r_mas; + mast.orig_l = &prev_l_mas; + mast.orig_r = &prev_r_mas; + mast.free = &mat; + mast.bn = b_node; + + while (height++ <= mas->depth) { + if (mt_slots[b_node->type] > b_node->b_end) { + mas_split_final_node(&mast, mas, height); + break; + } + + l_mas = r_mas = *mas; + l_mas.node = mas_new_ma_node(mas, b_node); + r_mas.node = mas_new_ma_node(mas, b_node); + /* + * Another way that 'jitter' is avoided is to terminate a split up early if the + * left or right node has space to spare. This is referred to as "pushing left" + * or "pushing right" and is similar to the B* tree, except the nodes left or + * right can rarely be reused due to RCU, but the ripple upwards is halted which + * is a significant savings. + */ + /* Try to push left. */ + if (mas_push_data(mas, height, &mast, true)) + break; + + /* Try to push right. */ + if (mas_push_data(mas, height, &mast, false)) + break; + + split = mab_calc_split(mas, b_node, &mid_split, prev_l_mas.min); + mast_split_data(&mast, mas, split); + /* + * Usually correct, mab_mas_cp in the above call overwrites + * r->max. + */ + mast.r->max = mas->max; + mast_fill_bnode(&mast, mas, 1); + prev_l_mas = *mast.l; + prev_r_mas = *mast.r; + } + + /* Set the original node as dead */ + mat_add(mast.free, mas->node); + mas->node = l_mas.node; + mas_wmb_replace(mas, mast.free, NULL); + mtree_range_walk(mas); + return 1; +} + +/* + * mas_reuse_node() - Reuse the node to store the data. + * @wr_mas: The maple write state + * @bn: The maple big node + * @end: The end of the data. + * + * Will always return false in RCU mode. + * + * Return: True if node was reused, false otherwise. + */ +static inline bool mas_reuse_node(struct ma_wr_state *wr_mas, + struct maple_big_node *bn, unsigned char end) +{ + /* Need to be rcu safe. */ + if (mt_in_rcu(wr_mas->mas->tree)) + return false; + + if (end > bn->b_end) { + int clear = mt_slots[wr_mas->type] - bn->b_end; + + memset(wr_mas->slots + bn->b_end, 0, sizeof(void *) * clear--); + memset(wr_mas->pivots + bn->b_end, 0, sizeof(void *) * clear); + } + mab_mas_cp(bn, 0, bn->b_end, wr_mas->mas, false); + return true; +} + +/* + * mas_commit_b_node() - Commit the big node into the tree. + * @wr_mas: The maple write state + * @b_node: The maple big node + * @end: The end of the data. + */ +static inline int mas_commit_b_node(struct ma_wr_state *wr_mas, + struct maple_big_node *b_node, unsigned char end) +{ + struct maple_node *node; + unsigned char b_end = b_node->b_end; + enum maple_type b_type = b_node->type; + + if ((b_end < mt_min_slots[b_type]) && + (!mte_is_root(wr_mas->mas->node)) && + (mas_mt_height(wr_mas->mas) > 1)) + return mas_rebalance(wr_mas->mas, b_node); + + if (b_end >= mt_slots[b_type]) + return mas_split(wr_mas->mas, b_node); + + if (mas_reuse_node(wr_mas, b_node, end)) + goto reuse_node; + + mas_node_count(wr_mas->mas, 1); + if (mas_is_err(wr_mas->mas)) + return 0; + + node = mas_pop_node(wr_mas->mas); + node->parent = mas_mn(wr_mas->mas)->parent; + wr_mas->mas->node = mt_mk_node(node, b_type); + mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true); + + mas_replace(wr_mas->mas, false); +reuse_node: + mas_update_gap(wr_mas->mas); + return 1; +} + +/* + * mas_root_expand() - Expand a root to a node + * @mas: The maple state + * @entry: The entry to store into the tree + */ +static inline int mas_root_expand(struct ma_state *mas, void *entry) +{ + void *contents = mas_root_locked(mas); + enum maple_type type = maple_leaf_64; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots; + int slot = 0; + + mas_node_count(mas, 1); + if (unlikely(mas_is_err(mas))) + return 0; + + node = mas_pop_node(mas); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + node->parent = ma_parent_ptr( + ((unsigned long)mas->tree | MA_ROOT_PARENT)); + mas->node = mt_mk_node(node, type); + + if (mas->index) { + if (contents) { + rcu_assign_pointer(slots[slot], contents); + if (likely(mas->index > 1)) + slot++; + } + pivots[slot++] = mas->index - 1; + } + + rcu_assign_pointer(slots[slot], entry); + mas->offset = slot; + pivots[slot] = mas->last; + if (mas->last != ULONG_MAX) + slot++; + mas->depth = 1; + mas_set_height(mas); + + /* swap the new root into the tree */ + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + ma_set_meta(node, maple_leaf_64, 0, slot); + return slot; +} + +static inline void mas_store_root(struct ma_state *mas, void *entry) +{ + if (likely((mas->last != 0) || (mas->index != 0))) + mas_root_expand(mas, entry); + else if (((unsigned long) (entry) & 3) == 2) + mas_root_expand(mas, entry); + else { + rcu_assign_pointer(mas->tree->ma_root, entry); + mas->node = MAS_START; + } +} + +/* + * mas_is_span_wr() - Check if the write needs to be treated as a write that + * spans the node. + * @mas: The maple state + * @piv: The pivot value being written + * @type: The maple node type + * @entry: The data to write + * + * Spanning writes are writes that start in one node and end in another OR if + * the write of a %NULL will cause the node to end with a %NULL. + * + * Return: True if this is a spanning write, false otherwise. + */ +static bool mas_is_span_wr(struct ma_wr_state *wr_mas) +{ + unsigned long max; + unsigned long last = wr_mas->mas->last; + unsigned long piv = wr_mas->r_max; + enum maple_type type = wr_mas->type; + void *entry = wr_mas->entry; + + /* Contained in this pivot */ + if (piv > last) + return false; + + max = wr_mas->mas->max; + if (unlikely(ma_is_leaf(type))) { + /* Fits in the node, but may span slots. */ + if (last < max) + return false; + + /* Writes to the end of the node but not null. */ + if ((last == max) && entry) + return false; + + /* + * Writing ULONG_MAX is not a spanning write regardless of the + * value being written as long as the range fits in the node. + */ + if ((last == ULONG_MAX) && (last == max)) + return false; + } else if (piv == last) { + if (entry) + return false; + + /* Detect spanning store wr walk */ + if (last == ULONG_MAX) + return false; + } + + trace_ma_write(__func__, wr_mas->mas, piv, entry); + + return true; +} + +static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) +{ + wr_mas->mas->depth++; + wr_mas->type = mte_node_type(wr_mas->mas->node); + mas_wr_node_walk(wr_mas); + wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); +} + +static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) +{ + wr_mas->mas->max = wr_mas->r_max; + wr_mas->mas->min = wr_mas->r_min; + wr_mas->mas->node = wr_mas->content; + wr_mas->mas->offset = 0; +} +/* + * mas_wr_walk() - Walk the tree for a write. + * @wr_mas: The maple write state + * + * Uses mas_slot_locked() and does not need to worry about dead nodes. + * + * Return: True if it's contained in a node, false on spanning write. + */ +static bool mas_wr_walk(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + while (true) { + mas_wr_walk_descend(wr_mas); + if (unlikely(mas_is_span_wr(wr_mas))) + return false; + + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, + mas->offset); + if (ma_is_leaf(wr_mas->type)) + return true; + + mas_wr_walk_traverse(wr_mas); + } + + return true; +} + +static bool mas_wr_walk_index(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + while (true) { + mas_wr_walk_descend(wr_mas); + wr_mas->content = mas_slot_locked(mas, wr_mas->slots, + mas->offset); + if (ma_is_leaf(wr_mas->type)) + return true; + mas_wr_walk_traverse(wr_mas); + + } + return true; +} +/* + * mas_extend_spanning_null() - Extend a store of a %NULL to include surrounding %NULLs. + * @l_wr_mas: The left maple write state + * @r_wr_mas: The right maple write state + */ +static inline void mas_extend_spanning_null(struct ma_wr_state *l_wr_mas, + struct ma_wr_state *r_wr_mas) +{ + struct ma_state *r_mas = r_wr_mas->mas; + struct ma_state *l_mas = l_wr_mas->mas; + unsigned char l_slot; + + l_slot = l_mas->offset; + if (!l_wr_mas->content) + l_mas->index = l_wr_mas->r_min; + + if ((l_mas->index == l_wr_mas->r_min) && + (l_slot && + !mas_slot_locked(l_mas, l_wr_mas->slots, l_slot - 1))) { + if (l_slot > 1) + l_mas->index = l_wr_mas->pivots[l_slot - 2] + 1; + else + l_mas->index = l_mas->min; + + l_mas->offset = l_slot - 1; + } + + if (!r_wr_mas->content) { + if (r_mas->last < r_wr_mas->r_max) + r_mas->last = r_wr_mas->r_max; + r_mas->offset++; + } else if ((r_mas->last == r_wr_mas->r_max) && + (r_mas->last < r_mas->max) && + !mas_slot_locked(r_mas, r_wr_mas->slots, r_mas->offset + 1)) { + r_mas->last = mas_safe_pivot(r_mas, r_wr_mas->pivots, + r_wr_mas->type, r_mas->offset + 1); + r_mas->offset++; + } +} + +static inline void *mas_state_walk(struct ma_state *mas) +{ + void *entry; + + entry = mas_start(mas); + if (mas_is_none(mas)) + return NULL; + + if (mas_is_ptr(mas)) + return entry; + + return mtree_range_walk(mas); +} + +/* + * mtree_lookup_walk() - Internal quick lookup that does not keep maple state up + * to date. + * + * @mas: The maple state. + * + * Note: Leaves mas in undesirable state. + * Return: The entry for @mas->index or %NULL on dead node. + */ +static inline void *mtree_lookup_walk(struct ma_state *mas) +{ + unsigned long *pivots; + unsigned char offset; + struct maple_node *node; + struct maple_enode *next; + enum maple_type type; + void __rcu **slots; + unsigned char end; + unsigned long max; + + next = mas->node; + max = ULONG_MAX; + do { + offset = 0; + node = mte_to_node(next); + type = mte_node_type(next); + pivots = ma_pivots(node, type); + end = ma_data_end(node, type, pivots, max); + if (unlikely(ma_dead_node(node))) + goto dead_node; + + if (pivots[offset] >= mas->index) + goto next; + + do { + offset++; + } while ((offset < end) && (pivots[offset] < mas->index)); + + if (likely(offset > end)) + max = pivots[offset]; + +next: + slots = ma_slots(node, type); + next = mt_slot(mas->tree, slots, offset); + if (unlikely(ma_dead_node(node))) + goto dead_node; + } while (!ma_is_leaf(type)); + + return (void *) next; + +dead_node: + mas_reset(mas); + return NULL; +} + +/* + * mas_new_root() - Create a new root node that only contains the entry passed + * in. + * @mas: The maple state + * @entry: The entry to store. + * + * Only valid when the index == 0 and the last == ULONG_MAX + * + * Return 0 on error, 1 on success. + */ +static inline int mas_new_root(struct ma_state *mas, void *entry) +{ + struct maple_enode *root = mas_root_locked(mas); + enum maple_type type = maple_leaf_64; + struct maple_node *node; + void __rcu **slots; + unsigned long *pivots; + + if (!entry && !mas->index && mas->last == ULONG_MAX) { + mas->depth = 0; + mas_set_height(mas); + rcu_assign_pointer(mas->tree->ma_root, entry); + mas->node = MAS_START; + goto done; + } + + mas_node_count(mas, 1); + if (mas_is_err(mas)) + return 0; + + node = mas_pop_node(mas); + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + node->parent = ma_parent_ptr( + ((unsigned long)mas->tree | MA_ROOT_PARENT)); + mas->node = mt_mk_node(node, type); + rcu_assign_pointer(slots[0], entry); + pivots[0] = mas->last; + mas->depth = 1; + mas_set_height(mas); + rcu_assign_pointer(mas->tree->ma_root, mte_mk_root(mas->node)); + +done: + if (xa_is_node(root)) + mte_destroy_walk(root, mas->tree); + + return 1; +} +/* + * mas_wr_spanning_store() - Create a subtree with the store operation completed + * and new nodes where necessary, then place the sub-tree in the actual tree. + * Note that mas is expected to point to the node which caused the store to + * span. + * @wr_mas: The maple write state + * + * Return: 0 on error, positive on success. + */ +static inline int mas_wr_spanning_store(struct ma_wr_state *wr_mas) +{ + struct maple_subtree_state mast; + struct maple_big_node b_node; + struct ma_state *mas; + unsigned char height; + + /* Left and Right side of spanning store */ + MA_STATE(l_mas, NULL, 0, 0); + MA_STATE(r_mas, NULL, 0, 0); + + MA_WR_STATE(r_wr_mas, &r_mas, wr_mas->entry); + MA_WR_STATE(l_wr_mas, &l_mas, wr_mas->entry); + + /* + * A store operation that spans multiple nodes is called a spanning + * store and is handled early in the store call stack by the function + * mas_is_span_wr(). When a spanning store is identified, the maple + * state is duplicated. The first maple state walks the left tree path + * to ``index``, the duplicate walks the right tree path to ``last``. + * The data in the two nodes are combined into a single node, two nodes, + * or possibly three nodes (see the 3-way split above). A ``NULL`` + * written to the last entry of a node is considered a spanning store as + * a rebalance is required for the operation to complete and an overflow + * of data may happen. + */ + mas = wr_mas->mas; + trace_ma_op(__func__, mas); + + if (unlikely(!mas->index && mas->last == ULONG_MAX)) + return mas_new_root(mas, wr_mas->entry); + /* + * Node rebalancing may occur due to this store, so there may be three new + * entries per level plus a new root. + */ + height = mas_mt_height(mas); + mas_node_count(mas, 1 + height * 3); + if (mas_is_err(mas)) + return 0; + + /* + * Set up right side. Need to get to the next offset after the spanning + * store to ensure it's not NULL and to combine both the next node and + * the node with the start together. + */ + r_mas = *mas; + /* Avoid overflow, walk to next slot in the tree. */ + if (r_mas.last + 1) + r_mas.last++; + + r_mas.index = r_mas.last; + mas_wr_walk_index(&r_wr_mas); + r_mas.last = r_mas.index = mas->last; + + /* Set up left side. */ + l_mas = *mas; + mas_wr_walk_index(&l_wr_mas); + + if (!wr_mas->entry) { + mas_extend_spanning_null(&l_wr_mas, &r_wr_mas); + mas->offset = l_mas.offset; + mas->index = l_mas.index; + mas->last = l_mas.last = r_mas.last; + } + + /* expanding NULLs may make this cover the entire range */ + if (!l_mas.index && r_mas.last == ULONG_MAX) { + mas_set_range(mas, 0, ULONG_MAX); + return mas_new_root(mas, wr_mas->entry); + } + + memset(&b_node, 0, sizeof(struct maple_big_node)); + /* Copy l_mas and store the value in b_node. */ + mas_store_b_node(&l_wr_mas, &b_node, l_wr_mas.node_end); + /* Copy r_mas into b_node. */ + if (r_mas.offset <= r_wr_mas.node_end) + mas_mab_cp(&r_mas, r_mas.offset, r_wr_mas.node_end, + &b_node, b_node.b_end + 1); + else + b_node.b_end++; + + /* Stop spanning searches by searching for just index. */ + l_mas.index = l_mas.last = mas->index; + + mast.bn = &b_node; + mast.orig_l = &l_mas; + mast.orig_r = &r_mas; + /* Combine l_mas and r_mas and split them up evenly again. */ + return mas_spanning_rebalance(mas, &mast, height + 1); +} + +/* + * mas_wr_node_store() - Attempt to store the value in a node + * @wr_mas: The maple write state + * + * Attempts to reuse the node, but may allocate. + * + * Return: True if stored, false otherwise + */ +static inline bool mas_wr_node_store(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + void __rcu **dst_slots; + unsigned long *dst_pivots; + unsigned char dst_offset; + unsigned char new_end = wr_mas->node_end; + unsigned char offset; + unsigned char node_slots = mt_slots[wr_mas->type]; + struct maple_node reuse, *newnode; + unsigned char copy_size, max_piv = mt_pivots[wr_mas->type]; + bool in_rcu = mt_in_rcu(mas->tree); + + offset = mas->offset; + if (mas->last == wr_mas->r_max) { + /* runs right to the end of the node */ + if (mas->last == mas->max) + new_end = offset; + /* don't copy this offset */ + wr_mas->offset_end++; + } else if (mas->last < wr_mas->r_max) { + /* new range ends in this range */ + if (unlikely(wr_mas->r_max == ULONG_MAX)) + mas_bulk_rebalance(mas, wr_mas->node_end, wr_mas->type); + + new_end++; + } else { + if (wr_mas->end_piv == mas->last) + wr_mas->offset_end++; + + new_end -= wr_mas->offset_end - offset - 1; + } + + /* new range starts within a range */ + if (wr_mas->r_min < mas->index) + new_end++; + + /* Not enough room */ + if (new_end >= node_slots) + return false; + + /* Not enough data. */ + if (!mte_is_root(mas->node) && (new_end <= mt_min_slots[wr_mas->type]) && + !(mas->mas_flags & MA_STATE_BULK)) + return false; + + /* set up node. */ + if (in_rcu) { + mas_node_count(mas, 1); + if (mas_is_err(mas)) + return false; + + newnode = mas_pop_node(mas); + } else { + memset(&reuse, 0, sizeof(struct maple_node)); + newnode = &reuse; + } + + newnode->parent = mas_mn(mas)->parent; + dst_pivots = ma_pivots(newnode, wr_mas->type); + dst_slots = ma_slots(newnode, wr_mas->type); + /* Copy from start to insert point */ + memcpy(dst_pivots, wr_mas->pivots, sizeof(unsigned long) * (offset + 1)); + memcpy(dst_slots, wr_mas->slots, sizeof(void *) * (offset + 1)); + dst_offset = offset; + + /* Handle insert of new range starting after old range */ + if (wr_mas->r_min < mas->index) { + mas->offset++; + rcu_assign_pointer(dst_slots[dst_offset], wr_mas->content); + dst_pivots[dst_offset++] = mas->index - 1; + } + + /* Store the new entry and range end. */ + if (dst_offset < max_piv) + dst_pivots[dst_offset] = mas->last; + mas->offset = dst_offset; + rcu_assign_pointer(dst_slots[dst_offset], wr_mas->entry); + + /* + * this range wrote to the end of the node or it overwrote the rest of + * the data + */ + if (wr_mas->offset_end > wr_mas->node_end || mas->last >= mas->max) { + new_end = dst_offset; + goto done; + } + + dst_offset++; + /* Copy to the end of node if necessary. */ + copy_size = wr_mas->node_end - wr_mas->offset_end + 1; + memcpy(dst_slots + dst_offset, wr_mas->slots + wr_mas->offset_end, + sizeof(void *) * copy_size); + if (dst_offset < max_piv) { + if (copy_size > max_piv - dst_offset) + copy_size = max_piv - dst_offset; + + memcpy(dst_pivots + dst_offset, + wr_mas->pivots + wr_mas->offset_end, + sizeof(unsigned long) * copy_size); + } + + if ((wr_mas->node_end == node_slots - 1) && (new_end < node_slots - 1)) + dst_pivots[new_end] = mas->max; + +done: + mas_leaf_set_meta(mas, newnode, dst_pivots, maple_leaf_64, new_end); + if (in_rcu) { + mas->node = mt_mk_node(newnode, wr_mas->type); + mas_replace(mas, false); + } else { + memcpy(wr_mas->node, newnode, sizeof(struct maple_node)); + } + trace_ma_write(__func__, mas, 0, wr_mas->entry); + mas_update_gap(mas); + return true; +} + +/* + * mas_wr_slot_store: Attempt to store a value in a slot. + * @wr_mas: the maple write state + * + * Return: True if stored, false otherwise + */ +static inline bool mas_wr_slot_store(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + unsigned long lmax; /* Logical max. */ + unsigned char offset = mas->offset; + + if ((wr_mas->r_max > mas->last) && ((wr_mas->r_min != mas->index) || + (offset != wr_mas->node_end))) + return false; + + if (offset == wr_mas->node_end - 1) + lmax = mas->max; + else + lmax = wr_mas->pivots[offset + 1]; + + /* going to overwrite too many slots. */ + if (lmax < mas->last) + return false; + + if (wr_mas->r_min == mas->index) { + /* overwriting two or more ranges with one. */ + if (lmax == mas->last) + return false; + + /* Overwriting all of offset and a portion of offset + 1. */ + rcu_assign_pointer(wr_mas->slots[offset], wr_mas->entry); + wr_mas->pivots[offset] = mas->last; + goto done; + } + + /* Doesn't end on the next range end. */ + if (lmax != mas->last) + return false; + + /* Overwriting a portion of offset and all of offset + 1 */ + if ((offset + 1 < mt_pivots[wr_mas->type]) && + (wr_mas->entry || wr_mas->pivots[offset + 1])) + wr_mas->pivots[offset + 1] = mas->last; + + rcu_assign_pointer(wr_mas->slots[offset + 1], wr_mas->entry); + wr_mas->pivots[offset] = mas->index - 1; + mas->offset++; /* Keep mas accurate. */ + +done: + trace_ma_write(__func__, mas, 0, wr_mas->entry); + mas_update_gap(mas); + return true; +} + +static inline void mas_wr_end_piv(struct ma_wr_state *wr_mas) +{ + while ((wr_mas->mas->last > wr_mas->end_piv) && + (wr_mas->offset_end < wr_mas->node_end)) + wr_mas->end_piv = wr_mas->pivots[++wr_mas->offset_end]; + + if (wr_mas->mas->last > wr_mas->end_piv) + wr_mas->end_piv = wr_mas->mas->max; +} + +static inline void mas_wr_extend_null(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + if (mas->last < wr_mas->end_piv && !wr_mas->slots[wr_mas->offset_end]) + mas->last = wr_mas->end_piv; + + /* Check next slot(s) if we are overwriting the end */ + if ((mas->last == wr_mas->end_piv) && + (wr_mas->node_end != wr_mas->offset_end) && + !wr_mas->slots[wr_mas->offset_end + 1]) { + wr_mas->offset_end++; + if (wr_mas->offset_end == wr_mas->node_end) + mas->last = mas->max; + else + mas->last = wr_mas->pivots[wr_mas->offset_end]; + wr_mas->end_piv = mas->last; + } + + if (!wr_mas->content) { + /* If this one is null, the next and prev are not */ + mas->index = wr_mas->r_min; + } else { + /* Check prev slot if we are overwriting the start */ + if (mas->index == wr_mas->r_min && mas->offset && + !wr_mas->slots[mas->offset - 1]) { + mas->offset--; + wr_mas->r_min = mas->index = + mas_safe_min(mas, wr_mas->pivots, mas->offset); + wr_mas->r_max = wr_mas->pivots[mas->offset]; + } + } +} + +static inline bool mas_wr_append(struct ma_wr_state *wr_mas) +{ + unsigned char end = wr_mas->node_end; + unsigned char new_end = end + 1; + struct ma_state *mas = wr_mas->mas; + unsigned char node_pivots = mt_pivots[wr_mas->type]; + + if ((mas->index != wr_mas->r_min) && (mas->last == wr_mas->r_max)) { + if (new_end < node_pivots) + wr_mas->pivots[new_end] = wr_mas->pivots[end]; + + if (new_end < node_pivots) + ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); + + rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->entry); + mas->offset = new_end; + wr_mas->pivots[end] = mas->index - 1; + + return true; + } + + if ((mas->index == wr_mas->r_min) && (mas->last < wr_mas->r_max)) { + if (new_end < node_pivots) + wr_mas->pivots[new_end] = wr_mas->pivots[end]; + + rcu_assign_pointer(wr_mas->slots[new_end], wr_mas->content); + if (new_end < node_pivots) + ma_set_meta(wr_mas->node, maple_leaf_64, 0, new_end); + + wr_mas->pivots[end] = mas->last; + rcu_assign_pointer(wr_mas->slots[end], wr_mas->entry); + return true; + } + + return false; +} + +/* + * mas_wr_bnode() - Slow path for a modification. + * @wr_mas: The write maple state + * + * This is where split, rebalance end up. + */ +static void mas_wr_bnode(struct ma_wr_state *wr_mas) +{ + struct maple_big_node b_node; + + trace_ma_write(__func__, wr_mas->mas, 0, wr_mas->entry); + memset(&b_node, 0, sizeof(struct maple_big_node)); + mas_store_b_node(wr_mas, &b_node, wr_mas->offset_end); + mas_commit_b_node(wr_mas, &b_node, wr_mas->node_end); +} + +static inline void mas_wr_modify(struct ma_wr_state *wr_mas) +{ + unsigned char node_slots; + unsigned char node_size; + struct ma_state *mas = wr_mas->mas; + + /* Direct replacement */ + if (wr_mas->r_min == mas->index && wr_mas->r_max == mas->last) { + rcu_assign_pointer(wr_mas->slots[mas->offset], wr_mas->entry); + if (!!wr_mas->entry ^ !!wr_mas->content) + mas_update_gap(mas); + return; + } + + /* Attempt to append */ + node_slots = mt_slots[wr_mas->type]; + node_size = wr_mas->node_end - wr_mas->offset_end + mas->offset + 2; + if (mas->max == ULONG_MAX) + node_size++; + + /* slot and node store will not fit, go to the slow path */ + if (unlikely(node_size >= node_slots)) + goto slow_path; + + if (wr_mas->entry && (wr_mas->node_end < node_slots - 1) && + (mas->offset == wr_mas->node_end) && mas_wr_append(wr_mas)) { + if (!wr_mas->content || !wr_mas->entry) + mas_update_gap(mas); + return; + } + + if ((wr_mas->offset_end - mas->offset <= 1) && mas_wr_slot_store(wr_mas)) + return; + else if (mas_wr_node_store(wr_mas)) + return; + + if (mas_is_err(mas)) + return; + +slow_path: + mas_wr_bnode(wr_mas); +} + +/* + * mas_wr_store_entry() - Internal call to store a value + * @mas: The maple state + * @entry: The entry to store. + * + * Return: The contents that was stored at the index. + */ +static inline void *mas_wr_store_entry(struct ma_wr_state *wr_mas) +{ + struct ma_state *mas = wr_mas->mas; + + wr_mas->content = mas_start(mas); + if (mas_is_none(mas) || mas_is_ptr(mas)) { + mas_store_root(mas, wr_mas->entry); + return wr_mas->content; + } + + if (unlikely(!mas_wr_walk(wr_mas))) { + mas_wr_spanning_store(wr_mas); + return wr_mas->content; + } + + /* At this point, we are at the leaf node that needs to be altered. */ + wr_mas->end_piv = wr_mas->r_max; + mas_wr_end_piv(wr_mas); + + if (!wr_mas->entry) + mas_wr_extend_null(wr_mas); + + /* New root for a single pointer */ + if (unlikely(!mas->index && mas->last == ULONG_MAX)) { + mas_new_root(mas, wr_mas->entry); + return wr_mas->content; + } + + mas_wr_modify(wr_mas); + return wr_mas->content; +} + +/** + * mas_insert() - Internal call to insert a value + * @mas: The maple state + * @entry: The entry to store + * + * Return: %NULL or the contents that already exists at the requested index + * otherwise. The maple state needs to be checked for error conditions. + */ +static inline void *mas_insert(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + /* + * Inserting a new range inserts either 0, 1, or 2 pivots within the + * tree. If the insert fits exactly into an existing gap with a value + * of NULL, then the slot only needs to be written with the new value. + * If the range being inserted is adjacent to another range, then only a + * single pivot needs to be inserted (as well as writing the entry). If + * the new range is within a gap but does not touch any other ranges, + * then two pivots need to be inserted: the start - 1, and the end. As + * usual, the entry must be written. Most operations require a new node + * to be allocated and replace an existing node to ensure RCU safety, + * when in RCU mode. The exception to requiring a newly allocated node + * is when inserting at the end of a node (appending). When done + * carefully, appending can reuse the node in place. + */ + wr_mas.content = mas_start(mas); + if (wr_mas.content) + goto exists; + + if (mas_is_none(mas) || mas_is_ptr(mas)) { + mas_store_root(mas, entry); + return NULL; + } + + /* spanning writes always overwrite something */ + if (!mas_wr_walk(&wr_mas)) + goto exists; + + /* At this point, we are at the leaf node that needs to be altered. */ + wr_mas.offset_end = mas->offset; + wr_mas.end_piv = wr_mas.r_max; + + if (wr_mas.content || (mas->last > wr_mas.r_max)) + goto exists; + + if (!entry) + return NULL; + + mas_wr_modify(&wr_mas); + return wr_mas.content; + +exists: + mas_set_err(mas, -EEXIST); + return wr_mas.content; + +} + +/* + * mas_prev_node() - Find the prev non-null entry at the same level in the + * tree. The prev value will be mas->node[mas->offset] or MAS_NONE. + * @mas: The maple state + * @min: The lower limit to search + * + * The prev node value will be mas->node[mas->offset] or MAS_NONE. + * Return: 1 if the node is dead, 0 otherwise. + */ +static inline int mas_prev_node(struct ma_state *mas, unsigned long min) +{ + enum maple_type mt; + int offset, level; + void __rcu **slots; + struct maple_node *node; + struct maple_enode *enode; + unsigned long *pivots; + + if (mas_is_none(mas)) + return 0; + + level = 0; + do { + node = mas_mn(mas); + if (ma_is_root(node)) + goto no_entry; + + /* Walk up. */ + if (unlikely(mas_ascend(mas))) + return 1; + offset = mas->offset; + level++; + } while (!offset); + + offset--; + mt = mte_node_type(mas->node); + node = mas_mn(mas); + slots = ma_slots(node, mt); + pivots = ma_pivots(node, mt); + mas->max = pivots[offset]; + if (offset) + mas->min = pivots[offset - 1] + 1; + if (unlikely(ma_dead_node(node))) + return 1; + + if (mas->max < min) + goto no_entry_min; + + while (level > 1) { + level--; + enode = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = enode; + mt = mte_node_type(mas->node); + node = mas_mn(mas); + slots = ma_slots(node, mt); + pivots = ma_pivots(node, mt); + offset = ma_data_end(node, mt, pivots, mas->max); + if (offset) + mas->min = pivots[offset - 1] + 1; + + if (offset < mt_pivots[mt]) + mas->max = pivots[offset]; + + if (mas->max < min) + goto no_entry; + } + + mas->node = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(node))) + return 1; + + mas->offset = mas_data_end(mas); + if (unlikely(mte_dead_node(mas->node))) + return 1; + + return 0; + +no_entry_min: + mas->offset = offset; + if (offset) + mas->min = pivots[offset - 1] + 1; +no_entry: + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = MAS_NONE; + return 0; +} + +/* + * mas_next_node() - Get the next node at the same level in the tree. + * @mas: The maple state + * @max: The maximum pivot value to check. + * + * The next value will be mas->node[mas->offset] or MAS_NONE. + * Return: 1 on dead node, 0 otherwise. + */ +static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, + unsigned long max) +{ + unsigned long min, pivot; + unsigned long *pivots; + struct maple_enode *enode; + int level = 0; + unsigned char offset; + enum maple_type mt; + void __rcu **slots; + + if (mas->max >= max) + goto no_entry; + + level = 0; + do { + if (ma_is_root(node)) + goto no_entry; + + min = mas->max + 1; + if (min > max) + goto no_entry; + + if (unlikely(mas_ascend(mas))) + return 1; + + offset = mas->offset; + level++; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + pivots = ma_pivots(node, mt); + } while (unlikely(offset == ma_data_end(node, mt, pivots, mas->max))); + + slots = ma_slots(node, mt); + pivot = mas_safe_pivot(mas, pivots, ++offset, mt); + while (unlikely(level > 1)) { + /* Descend, if necessary */ + enode = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = enode; + level--; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + slots = ma_slots(node, mt); + pivots = ma_pivots(node, mt); + offset = 0; + pivot = pivots[0]; + } + + enode = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = enode; + mas->min = min; + mas->max = pivot; + return 0; + +no_entry: + if (unlikely(ma_dead_node(node))) + return 1; + + mas->node = MAS_NONE; + return 0; +} + +/* + * mas_next_nentry() - Get the next node entry + * @mas: The maple state + * @max: The maximum value to check + * @*range_start: Pointer to store the start of the range. + * + * Sets @mas->offset to the offset of the next node entry, @mas->last to the + * pivot of the entry. + * + * Return: The next entry, %NULL otherwise + */ +static inline void *mas_next_nentry(struct ma_state *mas, + struct maple_node *node, unsigned long max, enum maple_type type) +{ + unsigned char count; + unsigned long pivot; + unsigned long *pivots; + void __rcu **slots; + void *entry; + + if (mas->last == mas->max) { + mas->index = mas->max; + return NULL; + } + + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + mas->index = mas_safe_min(mas, pivots, mas->offset); + if (ma_dead_node(node)) + return NULL; + + if (mas->index > max) + return NULL; + + count = ma_data_end(node, type, pivots, mas->max); + if (mas->offset > count) + return NULL; + + while (mas->offset < count) { + pivot = pivots[mas->offset]; + entry = mas_slot(mas, slots, mas->offset); + if (ma_dead_node(node)) + return NULL; + + if (entry) + goto found; + + if (pivot >= max) + return NULL; + + mas->index = pivot + 1; + mas->offset++; + } + + if (mas->index > mas->max) { + mas->index = mas->last; + return NULL; + } + + pivot = mas_safe_pivot(mas, pivots, mas->offset, type); + entry = mas_slot(mas, slots, mas->offset); + if (ma_dead_node(node)) + return NULL; + + if (!pivot) + return NULL; + + if (!entry) + return NULL; + +found: + mas->last = pivot; + return entry; +} + +static inline void mas_rewalk(struct ma_state *mas, unsigned long index) +{ + +retry: + mas_set(mas, index); + mas_state_walk(mas); + if (mas_is_start(mas)) + goto retry; + + return; + +} + +/* + * mas_next_entry() - Internal function to get the next entry. + * @mas: The maple state + * @limit: The maximum range start. + * + * Set the @mas->node to the next entry and the range_start to + * the beginning value for the entry. Does not check beyond @limit. + * Sets @mas->index and @mas->last to the limit if it is hit. + * Restarts on dead nodes. + * + * Return: the next entry or %NULL. + */ +static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) +{ + void *entry = NULL; + struct maple_enode *prev_node; + struct maple_node *node; + unsigned char offset; + unsigned long last; + enum maple_type mt; + + last = mas->last; +retry: + offset = mas->offset; + prev_node = mas->node; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + mas->offset++; + if (unlikely(mas->offset >= mt_slots[mt])) { + mas->offset = mt_slots[mt] - 1; + goto next_node; + } + + while (!mas_is_none(mas)) { + entry = mas_next_nentry(mas, node, limit, mt); + if (unlikely(ma_dead_node(node))) { + mas_rewalk(mas, last); + goto retry; + } + + if (likely(entry)) + return entry; + + if (unlikely((mas->index > limit))) + break; + +next_node: + prev_node = mas->node; + offset = mas->offset; + if (unlikely(mas_next_node(mas, node, limit))) { + mas_rewalk(mas, last); + goto retry; + } + mas->offset = 0; + node = mas_mn(mas); + mt = mte_node_type(mas->node); + } + + mas->index = mas->last = limit; + mas->offset = offset; + mas->node = prev_node; + return NULL; +} + +/* + * mas_prev_nentry() - Get the previous node entry. + * @mas: The maple state. + * @limit: The lower limit to check for a value. + * + * Return: the entry, %NULL otherwise. + */ +static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit, + unsigned long index) +{ + unsigned long pivot, min; + unsigned char offset; + struct maple_node *mn; + enum maple_type mt; + unsigned long *pivots; + void __rcu **slots; + void *entry; + +retry: + if (!mas->offset) + return NULL; + + mn = mas_mn(mas); + mt = mte_node_type(mas->node); + offset = mas->offset - 1; + if (offset >= mt_slots[mt]) + offset = mt_slots[mt] - 1; + + slots = ma_slots(mn, mt); + pivots = ma_pivots(mn, mt); + if (offset == mt_pivots[mt]) + pivot = mas->max; + else + pivot = pivots[offset]; + + if (unlikely(ma_dead_node(mn))) { + mas_rewalk(mas, index); + goto retry; + } + + while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) || + !pivot)) + pivot = pivots[--offset]; + + min = mas_safe_min(mas, pivots, offset); + entry = mas_slot(mas, slots, offset); + if (unlikely(ma_dead_node(mn))) { + mas_rewalk(mas, index); + goto retry; + } + + if (likely(entry)) { + mas->offset = offset; + mas->last = pivot; + mas->index = min; + } + return entry; +} + +static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min) +{ + void *entry; + +retry: + while (likely(!mas_is_none(mas))) { + entry = mas_prev_nentry(mas, min, mas->index); + if (unlikely(mas->last < min)) + goto not_found; + + if (likely(entry)) + return entry; + + if (unlikely(mas_prev_node(mas, min))) { + mas_rewalk(mas, mas->index); + goto retry; + } + + mas->offset++; + } + + mas->offset--; +not_found: + mas->index = mas->last = min; + return NULL; +} + +/* + * mas_rev_awalk() - Internal function. Reverse allocation walk. Find the + * highest gap address of a given size in a given node and descend. + * @mas: The maple state + * @size: The needed size. + * + * Return: True if found in a leaf, false otherwise. + * + */ +static bool mas_rev_awalk(struct ma_state *mas, unsigned long size) +{ + enum maple_type type = mte_node_type(mas->node); + struct maple_node *node = mas_mn(mas); + unsigned long *pivots, *gaps; + void __rcu **slots; + unsigned long gap = 0; + unsigned long max, min, index; + unsigned char offset; + + if (unlikely(mas_is_err(mas))) + return true; + + if (ma_is_dense(type)) { + /* dense nodes. */ + mas->offset = (unsigned char)(mas->index - mas->min); + return true; + } + + pivots = ma_pivots(node, type); + slots = ma_slots(node, type); + gaps = ma_gaps(node, type); + offset = mas->offset; + min = mas_safe_min(mas, pivots, offset); + /* Skip out of bounds. */ + while (mas->last < min) + min = mas_safe_min(mas, pivots, --offset); + + max = mas_safe_pivot(mas, pivots, offset, type); + index = mas->index; + while (index <= max) { + gap = 0; + if (gaps) + gap = gaps[offset]; + else if (!mas_slot(mas, slots, offset)) + gap = max - min + 1; + + if (gap) { + if ((size <= gap) && (size <= mas->last - min + 1)) + break; + + if (!gaps) { + /* Skip the next slot, it cannot be a gap. */ + if (offset < 2) + goto ascend; + + offset -= 2; + max = pivots[offset]; + min = mas_safe_min(mas, pivots, offset); + continue; + } + } + + if (!offset) + goto ascend; + + offset--; + max = min - 1; + min = mas_safe_min(mas, pivots, offset); + } + + if (unlikely(index > max)) { + mas_set_err(mas, -EBUSY); + return false; + } + + if (unlikely(ma_is_leaf(type))) { + mas->offset = offset; + mas->min = min; + mas->max = min + gap - 1; + return true; + } + + /* descend, only happens under lock. */ + mas->node = mas_slot(mas, slots, offset); + mas->min = min; + mas->max = max; + mas->offset = mas_data_end(mas); + return false; + +ascend: + if (mte_is_root(mas->node)) + mas_set_err(mas, -EBUSY); + + return false; +} + +static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) +{ + enum maple_type type = mte_node_type(mas->node); + unsigned long pivot, min, gap = 0; + unsigned char count, offset; + unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type); + void __rcu **slots = ma_slots(mas_mn(mas), type); + bool found = false; + + if (ma_is_dense(type)) { + mas->offset = (unsigned char)(mas->index - mas->min); + return true; + } + + gaps = ma_gaps(mte_to_node(mas->node), type); + offset = mas->offset; + count = mt_slots[type]; + min = mas_safe_min(mas, pivots, offset); + for (; offset < count; offset++) { + pivot = mas_safe_pivot(mas, pivots, offset, type); + if (offset && !pivot) + break; + + /* Not within lower bounds */ + if (mas->index > pivot) + goto next_slot; + + if (gaps) + gap = gaps[offset]; + else if (!mas_slot(mas, slots, offset)) + gap = min(pivot, mas->last) - max(mas->index, min) + 1; + else + goto next_slot; + + if (gap >= size) { + if (ma_is_leaf(type)) { + found = true; + goto done; + } + if (mas->index <= pivot) { + mas->node = mas_slot(mas, slots, offset); + mas->min = min; + mas->max = pivot; + offset = 0; + type = mte_node_type(mas->node); + count = mt_slots[type]; + break; + } + } +next_slot: + min = pivot + 1; + if (mas->last <= pivot) { + mas_set_err(mas, -EBUSY); + return true; + } + } + + if (mte_is_root(mas->node)) + found = true; +done: + mas->offset = offset; + return found; +} + +/** + * mas_walk() - Search for @mas->index in the tree. + * @mas: The maple state. + * + * mas->index and mas->last will be set to the range if there is a value. If + * mas->node is MAS_NONE, reset to MAS_START. + * + * Return: the entry at the location or %NULL. + */ +void *mas_walk(struct ma_state *mas) +{ + void *entry; + +retry: + entry = mas_state_walk(mas); + if (mas_is_start(mas)) + goto retry; + + if (mas_is_ptr(mas)) { + if (!mas->index) { + mas->last = 0; + } else { + mas->index = 1; + mas->last = ULONG_MAX; + } + return entry; + } + + if (mas_is_none(mas)) { + mas->index = 0; + mas->last = ULONG_MAX; + } + + return entry; +} + +static inline bool mas_rewind_node(struct ma_state *mas) +{ + unsigned char slot; + + do { + if (mte_is_root(mas->node)) { + slot = mas->offset; + if (!slot) + return false; + } else { + mas_ascend(mas); + slot = mas->offset; + } + } while (!slot); + + mas->offset = --slot; + return true; +} + +/* + * mas_skip_node() - Internal function. Skip over a node. + * @mas: The maple state. + * + * Return: true if there is another node, false otherwise. + */ +static inline bool mas_skip_node(struct ma_state *mas) +{ + unsigned char slot, slot_count; + unsigned long *pivots; + enum maple_type mt; + + mt = mte_node_type(mas->node); + slot_count = mt_slots[mt] - 1; + do { + if (mte_is_root(mas->node)) { + slot = mas->offset; + if (slot > slot_count) { + mas_set_err(mas, -EBUSY); + return false; + } + } else { + mas_ascend(mas); + slot = mas->offset; + mt = mte_node_type(mas->node); + slot_count = mt_slots[mt] - 1; + } + } while (slot > slot_count); + + mas->offset = ++slot; + pivots = ma_pivots(mas_mn(mas), mt); + if (slot > 0) + mas->min = pivots[slot - 1] + 1; + + if (slot <= slot_count) + mas->max = pivots[slot]; + + return true; +} + +/* + * mas_awalk() - Allocation walk. Search from low address to high, for a gap of + * @size + * @mas: The maple state + * @size: The size of the gap required + * + * Search between @mas->index and @mas->last for a gap of @size. + */ +static inline void mas_awalk(struct ma_state *mas, unsigned long size) +{ + struct maple_enode *last = NULL; + + /* + * There are 4 options: + * go to child (descend) + * go back to parent (ascend) + * no gap found. (return, slot == MAPLE_NODE_SLOTS) + * found the gap. (return, slot != MAPLE_NODE_SLOTS) + */ + while (!mas_is_err(mas) && !mas_anode_descend(mas, size)) { + if (last == mas->node) + mas_skip_node(mas); + else + last = mas->node; + } +} + +/* + * mas_fill_gap() - Fill a located gap with @entry. + * @mas: The maple state + * @entry: The value to store + * @slot: The offset into the node to store the @entry + * @size: The size of the entry + * @index: The start location + */ +static inline void mas_fill_gap(struct ma_state *mas, void *entry, + unsigned char slot, unsigned long size, unsigned long *index) +{ + MA_WR_STATE(wr_mas, mas, entry); + unsigned char pslot = mte_parent_slot(mas->node); + struct maple_enode *mn = mas->node; + unsigned long *pivots; + enum maple_type ptype; + /* + * mas->index is the start address for the search + * which may no longer be needed. + * mas->last is the end address for the search + */ + + *index = mas->index; + mas->last = mas->index + size - 1; + + /* + * It is possible that using mas->max and mas->min to correctly + * calculate the index and last will cause an issue in the gap + * calculation, so fix the ma_state here + */ + mas_ascend(mas); + ptype = mte_node_type(mas->node); + pivots = ma_pivots(mas_mn(mas), ptype); + mas->max = mas_safe_pivot(mas, pivots, pslot, ptype); + mas->min = mas_safe_min(mas, pivots, pslot); + mas->node = mn; + mas->offset = slot; + mas_wr_store_entry(&wr_mas); +} + +/* + * mas_sparse_area() - Internal function. Return upper or lower limit when + * searching for a gap in an empty tree. + * @mas: The maple state + * @min: the minimum range + * @max: The maximum range + * @size: The size of the gap + * @fwd: Searching forward or back + */ +static inline void mas_sparse_area(struct ma_state *mas, unsigned long min, + unsigned long max, unsigned long size, bool fwd) +{ + unsigned long start = 0; + + if (!unlikely(mas_is_none(mas))) + start++; + /* mas_is_ptr */ + + if (start < min) + start = min; + + if (fwd) { + mas->index = start; + mas->last = start + size - 1; + return; + } + + mas->index = max; +} + +/* + * mas_empty_area() - Get the lowest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ +int mas_empty_area(struct ma_state *mas, unsigned long min, + unsigned long max, unsigned long size) +{ + unsigned char offset; + unsigned long *pivots; + enum maple_type mt; + + if (mas_is_start(mas)) + mas_start(mas); + else if (mas->offset >= 2) + mas->offset -= 2; + else if (!mas_skip_node(mas)) + return -EBUSY; + + /* Empty set */ + if (mas_is_none(mas) || mas_is_ptr(mas)) { + mas_sparse_area(mas, min, max, size, true); + return 0; + } + + /* The start of the window can only be within these values */ + mas->index = min; + mas->last = max; + mas_awalk(mas, size); + + if (unlikely(mas_is_err(mas))) + return xa_err(mas->node); + + offset = mas->offset; + if (unlikely(offset == MAPLE_NODE_SLOTS)) + return -EBUSY; + + mt = mte_node_type(mas->node); + pivots = ma_pivots(mas_mn(mas), mt); + if (offset) + mas->min = pivots[offset - 1] + 1; + + if (offset < mt_pivots[mt]) + mas->max = pivots[offset]; + + if (mas->index < mas->min) + mas->index = mas->min; + + mas->last = mas->index + size - 1; + return 0; +} + +/* + * mas_empty_area_rev() - Get the highest address within the range that is + * sufficient for the size requested. + * @mas: The maple state + * @min: The lowest value of the range + * @max: The highest value of the range + * @size: The size needed + */ +int mas_empty_area_rev(struct ma_state *mas, unsigned long min, + unsigned long max, unsigned long size) +{ + struct maple_enode *last = mas->node; + + if (mas_is_start(mas)) { + mas_start(mas); + mas->offset = mas_data_end(mas); + } else if (mas->offset >= 2) { + mas->offset -= 2; + } else if (!mas_rewind_node(mas)) { + return -EBUSY; + } + + /* Empty set. */ + if (mas_is_none(mas) || mas_is_ptr(mas)) { + mas_sparse_area(mas, min, max, size, false); + return 0; + } + + /* The start of the window can only be within these values. */ + mas->index = min; + mas->last = max; + + while (!mas_rev_awalk(mas, size)) { + if (last == mas->node) { + if (!mas_rewind_node(mas)) + return -EBUSY; + } else { + last = mas->node; + } + } + + if (mas_is_err(mas)) + return xa_err(mas->node); + + if (unlikely(mas->offset == MAPLE_NODE_SLOTS)) + return -EBUSY; + + /* + * mas_rev_awalk() has set mas->min and mas->max to the gap values. If + * the maximum is outside the window we are searching, then use the last + * location in the search. + * mas->max and mas->min is the range of the gap. + * mas->index and mas->last are currently set to the search range. + */ + + /* Trim the upper limit to the max. */ + if (mas->max <= mas->last) + mas->last = mas->max; + + mas->index = mas->last - size + 1; + return 0; +} + +static inline int mas_alloc(struct ma_state *mas, void *entry, + unsigned long size, unsigned long *index) +{ + unsigned long min; + + mas_start(mas); + if (mas_is_none(mas) || mas_is_ptr(mas)) { + mas_root_expand(mas, entry); + if (mas_is_err(mas)) + return xa_err(mas->node); + + if (!mas->index) + return mte_pivot(mas->node, 0); + return mte_pivot(mas->node, 1); + } + + /* Must be walking a tree. */ + mas_awalk(mas, size); + if (mas_is_err(mas)) + return xa_err(mas->node); + + if (mas->offset == MAPLE_NODE_SLOTS) + goto no_gap; + + /* + * At this point, mas->node points to the right node and we have an + * offset that has a sufficient gap. + */ + min = mas->min; + if (mas->offset) + min = mte_pivot(mas->node, mas->offset - 1) + 1; + + if (mas->index < min) + mas->index = min; + + mas_fill_gap(mas, entry, mas->offset, size, index); + return 0; + +no_gap: + return -EBUSY; +} + +static inline int mas_rev_alloc(struct ma_state *mas, unsigned long min, + unsigned long max, void *entry, + unsigned long size, unsigned long *index) +{ + int ret = 0; + + ret = mas_empty_area_rev(mas, min, max, size); + if (ret) + return ret; + + if (mas_is_err(mas)) + return xa_err(mas->node); + + if (mas->offset == MAPLE_NODE_SLOTS) + goto no_gap; + + mas_fill_gap(mas, entry, mas->offset, size, index); + return 0; + +no_gap: + return -EBUSY; +} + +/* + * mas_dead_leaves() - Mark all leaves of a node as dead. + * @mas: The maple state + * @slots: Pointer to the slot array + * + * Must hold the write lock. + * + * Return: The number of leaves marked as dead. + */ +static inline +unsigned char mas_dead_leaves(struct ma_state *mas, void __rcu **slots) +{ + struct maple_node *node; + enum maple_type type; + void *entry; + int offset; + + for (offset = 0; offset < mt_slot_count(mas->node); offset++) { + entry = mas_slot_locked(mas, slots, offset); + type = mte_node_type(entry); + node = mte_to_node(entry); + /* Use both node and type to catch LE & BE metadata */ + if (!node || !type) + break; + + mte_set_node_dead(entry); + smp_wmb(); /* Needed for RCU */ + node->type = type; + rcu_assign_pointer(slots[offset], node); + } + + return offset; +} + +static void __rcu **mas_dead_walk(struct ma_state *mas, unsigned char offset) +{ + struct maple_node *node, *next; + void __rcu **slots = NULL; + + next = mas_mn(mas); + do { + mas->node = ma_enode_ptr(next); + node = mas_mn(mas); + slots = ma_slots(node, node->type); + next = mas_slot_locked(mas, slots, offset); + offset = 0; + } while (!ma_is_leaf(next->type)); + + return slots; +} + +static void mt_free_walk(struct rcu_head *head) +{ + void __rcu **slots; + struct maple_node *node, *start; + struct maple_tree mt; + unsigned char offset; + enum maple_type type; + MA_STATE(mas, &mt, 0, 0); + + node = container_of(head, struct maple_node, rcu); + + if (ma_is_leaf(node->type)) + goto free_leaf; + + mt_init_flags(&mt, node->ma_flags); + mas_lock(&mas); + start = node; + mas.node = mt_mk_node(node, node->type); + slots = mas_dead_walk(&mas, 0); + node = mas_mn(&mas); + do { + mt_free_bulk(node->slot_len, slots); + offset = node->parent_slot + 1; + mas.node = node->piv_parent; + if (mas_mn(&mas) == node) + goto start_slots_free; + + type = mte_node_type(mas.node); + slots = ma_slots(mte_to_node(mas.node), type); + if ((offset < mt_slots[type]) && (slots[offset])) + slots = mas_dead_walk(&mas, offset); + + node = mas_mn(&mas); + } while ((node != start) || (node->slot_len < offset)); + + slots = ma_slots(node, node->type); + mt_free_bulk(node->slot_len, slots); + +start_slots_free: + mas_unlock(&mas); +free_leaf: + mt_free_rcu(&node->rcu); +} + +static inline void __rcu **mas_destroy_descend(struct ma_state *mas, + struct maple_enode *prev, unsigned char offset) +{ + struct maple_node *node; + struct maple_enode *next = mas->node; + void __rcu **slots = NULL; + + do { + mas->node = next; + node = mas_mn(mas); + slots = ma_slots(node, mte_node_type(mas->node)); + next = mas_slot_locked(mas, slots, 0); + if ((mte_dead_node(next))) + next = mas_slot_locked(mas, slots, 1); + + mte_set_node_dead(mas->node); + node->type = mte_node_type(mas->node); + node->piv_parent = prev; + node->parent_slot = offset; + offset = 0; + prev = mas->node; + } while (!mte_is_leaf(next)); + + return slots; +} + +static void mt_destroy_walk(struct maple_enode *enode, unsigned char ma_flags, + bool free) +{ + void __rcu **slots; + struct maple_node *node = mte_to_node(enode); + struct maple_enode *start; + struct maple_tree mt; + + MA_STATE(mas, &mt, 0, 0); + + if (mte_is_leaf(enode)) + goto free_leaf; + + mt_init_flags(&mt, ma_flags); + mas_lock(&mas); + + mas.node = start = enode; + slots = mas_destroy_descend(&mas, start, 0); + node = mas_mn(&mas); + do { + enum maple_type type; + unsigned char offset; + struct maple_enode *parent, *tmp; + + node->slot_len = mas_dead_leaves(&mas, slots); + if (free) + mt_free_bulk(node->slot_len, slots); + offset = node->parent_slot + 1; + mas.node = node->piv_parent; + if (mas_mn(&mas) == node) + goto start_slots_free; + + type = mte_node_type(mas.node); + slots = ma_slots(mte_to_node(mas.node), type); + if (offset >= mt_slots[type]) + goto next; + + tmp = mas_slot_locked(&mas, slots, offset); + if (mte_node_type(tmp) && mte_to_node(tmp)) { + parent = mas.node; + mas.node = tmp; + slots = mas_destroy_descend(&mas, parent, offset); + } +next: + node = mas_mn(&mas); + } while (start != mas.node); + + node = mas_mn(&mas); + node->slot_len = mas_dead_leaves(&mas, slots); + if (free) + mt_free_bulk(node->slot_len, slots); + +start_slots_free: + mas_unlock(&mas); + +free_leaf: + if (free) + mt_free_rcu(&node->rcu); +} + +/* + * mte_destroy_walk() - Free a tree or sub-tree. + * @enode - the encoded maple node (maple_enode) to start + * @mn - the tree to free - needed for node types. + * + * Must hold the write lock. + */ +static inline void mte_destroy_walk(struct maple_enode *enode, + struct maple_tree *mt) +{ + struct maple_node *node = mte_to_node(enode); + + if (mt_in_rcu(mt)) { + mt_destroy_walk(enode, mt->ma_flags, false); + call_rcu(&node->rcu, mt_free_walk); + } else { + mt_destroy_walk(enode, mt->ma_flags, true); + } +} + +static void mas_wr_store_setup(struct ma_wr_state *wr_mas) +{ + if (!mas_is_start(wr_mas->mas)) { + if (mas_is_none(wr_mas->mas)) { + mas_reset(wr_mas->mas); + } else { + wr_mas->r_max = wr_mas->mas->max; + wr_mas->type = mte_node_type(wr_mas->mas->node); + if (mas_is_span_wr(wr_mas)) + mas_reset(wr_mas->mas); + } + } + +} + +/* Interface */ + +/** + * mas_store() - Store an @entry. + * @mas: The maple state. + * @entry: The entry to store. + * + * The @mas->index and @mas->last is used to set the range for the @entry. + * Note: The @mas should have pre-allocated entries to ensure there is memory to + * store the entry. Please see mas_expected_entries()/mas_destroy() for more details. + * + * Return: the first entry between mas->index and mas->last or %NULL. + */ +void *mas_store(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + trace_ma_write(__func__, mas, 0, entry); +#ifdef CONFIG_DEBUG_MAPLE_TREE + if (mas->index > mas->last) + pr_err("Error %lu > %lu %p\n", mas->index, mas->last, entry); + MT_BUG_ON(mas->tree, mas->index > mas->last); + if (mas->index > mas->last) { + mas_set_err(mas, -EINVAL); + return NULL; + } + +#endif + + /* + * Storing is the same operation as insert with the added caveat that it + * can overwrite entries. Although this seems simple enough, one may + * want to examine what happens if a single store operation was to + * overwrite multiple entries within a self-balancing B-Tree. + */ + mas_wr_store_setup(&wr_mas); + mas_wr_store_entry(&wr_mas); + return wr_mas.content; +} + +/** + * mas_store_gfp() - Store a value into the tree. + * @mas: The maple state + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations if necessary. + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) +{ + MA_WR_STATE(wr_mas, mas, entry); + + mas_wr_store_setup(&wr_mas); + trace_ma_write(__func__, mas, 0, entry); +retry: + mas_wr_store_entry(&wr_mas); + if (unlikely(mas_nomem(mas, gfp))) + goto retry; + + if (unlikely(mas_is_err(mas))) + return xa_err(mas->node); + + return 0; +} + +/** + * mas_store_prealloc() - Store a value into the tree using memory + * preallocated in the maple state. + * @mas: The maple state + * @entry: The entry to store. + */ +void mas_store_prealloc(struct ma_state *mas, void *entry) +{ + MA_WR_STATE(wr_mas, mas, entry); + + mas_wr_store_setup(&wr_mas); + trace_ma_write(__func__, mas, 0, entry); + mas_wr_store_entry(&wr_mas); + BUG_ON(mas_is_err(mas)); + mas_destroy(mas); +} + +/** + * mas_preallocate() - Preallocate enough nodes for a store operation + * @mas: The maple state + * @entry: The entry that will be stored + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated. + */ +int mas_preallocate(struct ma_state *mas, void *entry, gfp_t gfp) +{ + int ret; + + mas_node_count_gfp(mas, 1 + mas_mt_height(mas) * 3, gfp); + mas->mas_flags |= MA_STATE_PREALLOC; + if (likely(!mas_is_err(mas))) + return 0; + + mas_set_alloc_req(mas, 0); + ret = xa_err(mas->node); + mas_reset(mas); + mas_destroy(mas); + mas_reset(mas); + return ret; +} + +/* + * mas_destroy() - destroy a maple state. + * @mas: The maple state + * + * Upon completion, check the left-most node and rebalance against the node to + * the right if necessary. Frees any allocated nodes associated with this maple + * state. + */ +void mas_destroy(struct ma_state *mas) +{ + struct maple_alloc *node; + + /* + * When using mas_for_each() to insert an expected number of elements, + * it is possible that the number inserted is less than the expected + * number. To fix an invalid final node, a check is performed here to + * rebalance the previous node with the final node. + */ + if (mas->mas_flags & MA_STATE_REBALANCE) { + unsigned char end; + + if (mas_is_start(mas)) + mas_start(mas); + + mtree_range_walk(mas); + end = mas_data_end(mas) + 1; + if (end < mt_min_slot_count(mas->node) - 1) + mas_destroy_rebalance(mas, end); + + mas->mas_flags &= ~MA_STATE_REBALANCE; + } + mas->mas_flags &= ~(MA_STATE_BULK|MA_STATE_PREALLOC); + + while (mas->alloc && !((unsigned long)mas->alloc & 0x1)) { + node = mas->alloc; + mas->alloc = node->slot[0]; + if (node->node_count > 0) + mt_free_bulk(node->node_count, + (void __rcu **)&node->slot[1]); + kmem_cache_free(maple_node_cache, node); + } + mas->alloc = NULL; +} + +/* + * mas_expected_entries() - Set the expected number of entries that will be inserted. + * @mas: The maple state + * @nr_entries: The number of expected entries. + * + * This will attempt to pre-allocate enough nodes to store the expected number + * of entries. The allocations will occur using the bulk allocator interface + * for speed. Please call mas_destroy() on the @mas after inserting the entries + * to ensure any unused nodes are freed. + * + * Return: 0 on success, -ENOMEM if memory could not be allocated. + */ +int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) +{ + int nonleaf_cap = MAPLE_ARANGE64_SLOTS - 2; + struct maple_enode *enode = mas->node; + int nr_nodes; + int ret; + + /* + * Sometimes it is necessary to duplicate a tree to a new tree, such as + * forking a process and duplicating the VMAs from one tree to a new + * tree. When such a situation arises, it is known that the new tree is + * not going to be used until the entire tree is populated. For + * performance reasons, it is best to use a bulk load with RCU disabled. + * This allows for optimistic splitting that favours the left and reuse + * of nodes during the operation. + */ + + /* Optimize splitting for bulk insert in-order */ + mas->mas_flags |= MA_STATE_BULK; + + /* + * Avoid overflow, assume a gap between each entry and a trailing null. + * If this is wrong, it just means allocation can happen during + * insertion of entries. + */ + nr_nodes = max(nr_entries, nr_entries * 2 + 1); + if (!mt_is_alloc(mas->tree)) + nonleaf_cap = MAPLE_RANGE64_SLOTS - 2; + + /* Leaves; reduce slots to keep space for expansion */ + nr_nodes = DIV_ROUND_UP(nr_nodes, MAPLE_RANGE64_SLOTS - 2); + /* Internal nodes */ + nr_nodes += DIV_ROUND_UP(nr_nodes, nonleaf_cap); + /* Add working room for split (2 nodes) + new parents */ + mas_node_count(mas, nr_nodes + 3); + + /* Detect if allocations run out */ + mas->mas_flags |= MA_STATE_PREALLOC; + + if (!mas_is_err(mas)) + return 0; + + ret = xa_err(mas->node); + mas->node = enode; + mas_destroy(mas); + return ret; + +} + +/** + * mas_next() - Get the next entry. + * @mas: The maple state + * @max: The maximum index to check. + * + * Returns the next entry after @mas->index. + * Must hold rcu_read_lock or the write lock. + * Can return the zero entry. + * + * Return: The next entry or %NULL + */ +void *mas_next(struct ma_state *mas, unsigned long max) +{ + if (mas_is_none(mas) || mas_is_paused(mas)) + mas->node = MAS_START; + + if (mas_is_start(mas)) + mas_walk(mas); /* Retries on dead nodes handled by mas_walk */ + + if (mas_is_ptr(mas)) { + if (!mas->index) { + mas->index = 1; + mas->last = ULONG_MAX; + } + return NULL; + } + + if (mas->last == ULONG_MAX) + return NULL; + + /* Retries on dead nodes handled by mas_next_entry */ + return mas_next_entry(mas, max); +} +EXPORT_SYMBOL_GPL(mas_next); + +/** + * mt_next() - get the next value in the maple tree + * @mt: The maple tree + * @index: The start index + * @max: The maximum index to check + * + * Return: The entry at @index or higher, or %NULL if nothing is found. + */ +void *mt_next(struct maple_tree *mt, unsigned long index, unsigned long max) +{ + void *entry = NULL; + MA_STATE(mas, mt, index, index); + + rcu_read_lock(); + entry = mas_next(&mas, max); + rcu_read_unlock(); + return entry; +} +EXPORT_SYMBOL_GPL(mt_next); + +/** + * mas_prev() - Get the previous entry + * @mas: The maple state + * @min: The minimum value to check. + * + * Must hold rcu_read_lock or the write lock. + * Will reset mas to MAS_START if the node is MAS_NONE. Will stop on not + * searchable nodes. + * + * Return: the previous value or %NULL. + */ +void *mas_prev(struct ma_state *mas, unsigned long min) +{ + if (!mas->index) { + /* Nothing comes before 0 */ + mas->last = 0; + return NULL; + } + + if (unlikely(mas_is_ptr(mas))) + return NULL; + + if (mas_is_none(mas) || mas_is_paused(mas)) + mas->node = MAS_START; + + if (mas_is_start(mas)) { + mas_walk(mas); + if (!mas->index) + return NULL; + } + + if (mas_is_ptr(mas)) { + if (!mas->index) { + mas->last = 0; + return NULL; + } + + mas->index = mas->last = 0; + return mas_root_locked(mas); + } + return mas_prev_entry(mas, min); +} +EXPORT_SYMBOL_GPL(mas_prev); + +/** + * mt_prev() - get the previous value in the maple tree + * @mt: The maple tree + * @index: The start index + * @min: The minimum index to check + * + * Return: The entry at @index or lower, or %NULL if nothing is found. + */ +void *mt_prev(struct maple_tree *mt, unsigned long index, unsigned long min) +{ + void *entry = NULL; + MA_STATE(mas, mt, index, index); + + rcu_read_lock(); + entry = mas_prev(&mas, min); + rcu_read_unlock(); + return entry; +} +EXPORT_SYMBOL_GPL(mt_prev); + +/** + * mas_pause() - Pause a mas_find/mas_for_each to drop the lock. + * @mas: The maple state to pause + * + * Some users need to pause a walk and drop the lock they're holding in + * order to yield to a higher priority thread or carry out an operation + * on an entry. Those users should call this function before they drop + * the lock. It resets the @mas to be suitable for the next iteration + * of the loop after the user has reacquired the lock. If most entries + * found during a walk require you to call mas_pause(), the mt_for_each() + * iterator may be more appropriate. + * + */ +void mas_pause(struct ma_state *mas) +{ + mas->node = MAS_PAUSE; +} +EXPORT_SYMBOL_GPL(mas_pause); + +/** + * mas_find() - On the first call, find the entry at or after mas->index up to + * %max. Otherwise, find the entry after mas->index. + * @mas: The maple state + * @max: The maximum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->node to MAS_NONE. + * + * Return: The entry or %NULL. + */ +void *mas_find(struct ma_state *mas, unsigned long max) +{ + if (unlikely(mas_is_paused(mas))) { + if (unlikely(mas->last == ULONG_MAX)) { + mas->node = MAS_NONE; + return NULL; + } + mas->node = MAS_START; + mas->index = ++mas->last; + } + + if (unlikely(mas_is_start(mas))) { + /* First run or continue */ + void *entry; + + if (mas->index > max) + return NULL; + + entry = mas_walk(mas); + if (entry) + return entry; + } + + if (unlikely(!mas_searchable(mas))) + return NULL; + + /* Retries on dead nodes handled by mas_next_entry */ + return mas_next_entry(mas, max); +} + +/** + * mas_find_rev: On the first call, find the first non-null entry at or below + * mas->index down to %min. Otherwise find the first non-null entry below + * mas->index down to %min. + * @mas: The maple state + * @min: The minimum value to check. + * + * Must hold rcu_read_lock or the write lock. + * If an entry exists, last and index are updated accordingly. + * May set @mas->node to MAS_NONE. + * + * Return: The entry or %NULL. + */ +void *mas_find_rev(struct ma_state *mas, unsigned long min) +{ + if (unlikely(mas_is_paused(mas))) { + if (unlikely(mas->last == ULONG_MAX)) { + mas->node = MAS_NONE; + return NULL; + } + mas->node = MAS_START; + mas->last = --mas->index; + } + + if (unlikely(mas_is_start(mas))) { + /* First run or continue */ + void *entry; + + if (mas->index < min) + return NULL; + + entry = mas_walk(mas); + if (entry) + return entry; + } + + if (unlikely(!mas_searchable(mas))) + return NULL; + + if (mas->index < min) + return NULL; + + /* Retries on dead nodes handled by mas_next_entry */ + return mas_prev_entry(mas, min); +} +EXPORT_SYMBOL_GPL(mas_find); + +/** + * mas_erase() - Find the range in which index resides and erase the entire + * range. + * @mas: The maple state + * + * Must hold the write lock. + * Searches for @mas->index, sets @mas->index and @mas->last to the range and + * erases that range. + * + * Return: the entry that was erased or %NULL, @mas->index and @mas->last are updated. + */ +void *mas_erase(struct ma_state *mas) +{ + void *entry; + MA_WR_STATE(wr_mas, mas, NULL); + + if (mas_is_none(mas) || mas_is_paused(mas)) + mas->node = MAS_START; + + /* Retry unnecessary when holding the write lock. */ + entry = mas_state_walk(mas); + if (!entry) + return NULL; + +write_retry: + /* Must reset to ensure spanning writes of last slot are detected */ + mas_reset(mas); + mas_wr_store_setup(&wr_mas); + mas_wr_store_entry(&wr_mas); + if (mas_nomem(mas, GFP_KERNEL)) + goto write_retry; + + return entry; +} +EXPORT_SYMBOL_GPL(mas_erase); + +/** + * mas_nomem() - Check if there was an error allocating and do the allocation + * if necessary If there are allocations, then free them. + * @mas: The maple state + * @gfp: The GFP_FLAGS to use for allocations + * Return: true on allocation, false otherwise. + */ +bool mas_nomem(struct ma_state *mas, gfp_t gfp) + __must_hold(mas->tree->lock) +{ + if (likely(mas->node != MA_ERROR(-ENOMEM))) { + mas_destroy(mas); + return false; + } + + if (gfpflags_allow_blocking(gfp) && !mt_external_lock(mas->tree)) { + mtree_unlock(mas->tree); + mas_alloc_nodes(mas, gfp); + mtree_lock(mas->tree); + } else { + mas_alloc_nodes(mas, gfp); + } + + if (!mas_allocated(mas)) + return false; + + mas->node = MAS_START; + return true; +} + +void __init maple_tree_init(void) +{ + maple_node_cache = kmem_cache_create("maple_node", + sizeof(struct maple_node), sizeof(struct maple_node), + SLAB_PANIC, NULL); +} + +/** + * mtree_load() - Load a value stored in a maple tree + * @mt: The maple tree + * @index: The index to load + * + * Return: the entry or %NULL + */ +void *mtree_load(struct maple_tree *mt, unsigned long index) +{ + MA_STATE(mas, mt, index, index); + void *entry; + + trace_ma_read(__func__, &mas); + rcu_read_lock(); +retry: + entry = mas_start(&mas); + if (unlikely(mas_is_none(&mas))) + goto unlock; + + if (unlikely(mas_is_ptr(&mas))) { + if (index) + entry = NULL; + + goto unlock; + } + + entry = mtree_lookup_walk(&mas); + if (!entry && unlikely(mas_is_start(&mas))) + goto retry; +unlock: + rcu_read_unlock(); + if (xa_is_zero(entry)) + return NULL; + + return entry; +} +EXPORT_SYMBOL(mtree_load); + +/** + * mtree_store_range() - Store an entry at a given range. + * @mt: The maple tree + * @index: The start of the range + * @last: The end of the range + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mtree_store_range(struct maple_tree *mt, unsigned long index, + unsigned long last, void *entry, gfp_t gfp) +{ + MA_STATE(mas, mt, index, last); + MA_WR_STATE(wr_mas, &mas, entry); + + trace_ma_write(__func__, &mas, 0, entry); + if (WARN_ON_ONCE(xa_is_advanced(entry))) + return -EINVAL; + + if (index > last) + return -EINVAL; + + mtree_lock(mt); +retry: + mas_wr_store_entry(&wr_mas); + if (mas_nomem(&mas, gfp)) + goto retry; + + mtree_unlock(mt); + if (mas_is_err(&mas)) + return xa_err(mas.node); + + return 0; +} +EXPORT_SYMBOL(mtree_store_range); + +/** + * mtree_store() - Store an entry at a given index. + * @mt: The maple tree + * @index: The index to store the value + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations + * + * Return: 0 on success, -EINVAL on invalid request, -ENOMEM if memory could not + * be allocated. + */ +int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, + gfp_t gfp) +{ + return mtree_store_range(mt, index, index, entry, gfp); +} +EXPORT_SYMBOL(mtree_store); + +/** + * mtree_insert_range() - Insert an entry at a give range if there is no value. + * @mt: The maple tree + * @first: The start of the range + * @last: The end of the range + * @entry: The entry to store + * @gfp: The GFP_FLAGS to use for allocations. + * + * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid + * request, -ENOMEM if memory could not be allocated. + */ +int mtree_insert_range(struct maple_tree *mt, unsigned long first, + unsigned long last, void *entry, gfp_t gfp) +{ + MA_STATE(ms, mt, first, last); + + if (WARN_ON_ONCE(xa_is_advanced(entry))) + return -EINVAL; + + if (first > last) + return -EINVAL; + + mtree_lock(mt); +retry: + mas_insert(&ms, entry); + if (mas_nomem(&ms, gfp)) + goto retry; + + mtree_unlock(mt); + if (mas_is_err(&ms)) + return xa_err(ms.node); + + return 0; +} +EXPORT_SYMBOL(mtree_insert_range); + +/** + * mtree_insert() - Insert an entry at a give index if there is no value. + * @mt: The maple tree + * @index : The index to store the value + * @entry: The entry to store + * @gfp: The FGP_FLAGS to use for allocations. + * + * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid + * request, -ENOMEM if memory could not be allocated. + */ +int mtree_insert(struct maple_tree *mt, unsigned long index, void *entry, + gfp_t gfp) +{ + return mtree_insert_range(mt, index, index, entry, gfp); +} +EXPORT_SYMBOL(mtree_insert); + +int mtree_alloc_range(struct maple_tree *mt, unsigned long *startp, + void *entry, unsigned long size, unsigned long min, + unsigned long max, gfp_t gfp) +{ + int ret = 0; + + MA_STATE(mas, mt, min, max - size); + if (!mt_is_alloc(mt)) + return -EINVAL; + + if (WARN_ON_ONCE(mt_is_reserved(entry))) + return -EINVAL; + + if (min > max) + return -EINVAL; + + if (max < size) + return -EINVAL; + + if (!size) + return -EINVAL; + + mtree_lock(mt); +retry: + mas.offset = 0; + mas.index = min; + mas.last = max - size; + ret = mas_alloc(&mas, entry, size, startp); + if (mas_nomem(&mas, gfp)) + goto retry; + + mtree_unlock(mt); + return ret; +} +EXPORT_SYMBOL(mtree_alloc_range); + +int mtree_alloc_rrange(struct maple_tree *mt, unsigned long *startp, + void *entry, unsigned long size, unsigned long min, + unsigned long max, gfp_t gfp) +{ + int ret = 0; + + MA_STATE(mas, mt, min, max - size); + if (!mt_is_alloc(mt)) + return -EINVAL; + + if (WARN_ON_ONCE(mt_is_reserved(entry))) + return -EINVAL; + + if (min >= max) + return -EINVAL; + + if (max < size - 1) + return -EINVAL; + + if (!size) + return -EINVAL; + + mtree_lock(mt); +retry: + ret = mas_rev_alloc(&mas, min, max, entry, size, startp); + if (mas_nomem(&mas, gfp)) + goto retry; + + mtree_unlock(mt); + return ret; +} +EXPORT_SYMBOL(mtree_alloc_rrange); + +/** + * mtree_erase() - Find an index and erase the entire range. + * @mt: The maple tree + * @index: The index to erase + * + * Erasing is the same as a walk to an entry then a store of a NULL to that + * ENTIRE range. In fact, it is implemented as such using the advanced API. + * + * Return: The entry stored at the @index or %NULL + */ +void *mtree_erase(struct maple_tree *mt, unsigned long index) +{ + void *entry = NULL; + + MA_STATE(mas, mt, index, index); + trace_ma_op(__func__, &mas); + + mtree_lock(mt); + entry = mas_erase(&mas); + mtree_unlock(mt); + + return entry; +} +EXPORT_SYMBOL(mtree_erase); + +/** + * __mt_destroy() - Walk and free all nodes of a locked maple tree. + * @mt: The maple tree + * + * Note: Does not handle locking. + */ +void __mt_destroy(struct maple_tree *mt) +{ + void *root = mt_root_locked(mt); + + rcu_assign_pointer(mt->ma_root, NULL); + if (xa_is_node(root)) + mte_destroy_walk(root, mt); + + mt->ma_flags = 0; +} +EXPORT_SYMBOL_GPL(__mt_destroy); + +/** + * mtree_destroy() - Destroy a maple tree + * @mt: The maple tree + * + * Frees all resources used by the tree. Handles locking. + */ +void mtree_destroy(struct maple_tree *mt) +{ + mtree_lock(mt); + __mt_destroy(mt); + mtree_unlock(mt); +} +EXPORT_SYMBOL(mtree_destroy); + +/** + * mt_find() - Search from the start up until an entry is found. + * @mt: The maple tree + * @index: Pointer which contains the start location of the search + * @max: The maximum value to check + * + * Handles locking. @index will be incremented to one beyond the range. + * + * Return: The entry at or after the @index or %NULL + */ +void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max) +{ + MA_STATE(mas, mt, *index, *index); + void *entry; +#ifdef CONFIG_DEBUG_MAPLE_TREE + unsigned long copy = *index; +#endif + + trace_ma_read(__func__, &mas); + + if ((*index) > max) + return NULL; + + rcu_read_lock(); +retry: + entry = mas_state_walk(&mas); + if (mas_is_start(&mas)) + goto retry; + + if (unlikely(xa_is_zero(entry))) + entry = NULL; + + if (entry) + goto unlock; + + while (mas_searchable(&mas) && (mas.index < max)) { + entry = mas_next_entry(&mas, max); + if (likely(entry && !xa_is_zero(entry))) + break; + } + + if (unlikely(xa_is_zero(entry))) + entry = NULL; +unlock: + rcu_read_unlock(); + if (likely(entry)) { + *index = mas.last + 1; +#ifdef CONFIG_DEBUG_MAPLE_TREE + if ((*index) && (*index) <= copy) + pr_err("index not increased! %lx <= %lx\n", + *index, copy); + MT_BUG_ON(mt, (*index) && ((*index) <= copy)); +#endif + } + + return entry; +} +EXPORT_SYMBOL(mt_find); + +/** + * mt_find_after() - Search from the start up until an entry is found. + * @mt: The maple tree + * @index: Pointer which contains the start location of the search + * @max: The maximum value to check + * + * Handles locking, detects wrapping on index == 0 + * + * Return: The entry at or after the @index or %NULL + */ +void *mt_find_after(struct maple_tree *mt, unsigned long *index, + unsigned long max) +{ + if (!(*index)) + return NULL; + + return mt_find(mt, index, max); +} +EXPORT_SYMBOL(mt_find_after); + +#ifdef CONFIG_DEBUG_MAPLE_TREE +atomic_t maple_tree_tests_run; +EXPORT_SYMBOL_GPL(maple_tree_tests_run); +atomic_t maple_tree_tests_passed; +EXPORT_SYMBOL_GPL(maple_tree_tests_passed); + +#ifndef __KERNEL__ +extern void kmem_cache_set_non_kernel(struct kmem_cache *, unsigned int); +void mt_set_non_kernel(unsigned int val) +{ + kmem_cache_set_non_kernel(maple_node_cache, val); +} + +extern unsigned long kmem_cache_get_alloc(struct kmem_cache *); +unsigned long mt_get_alloc_size(void) +{ + return kmem_cache_get_alloc(maple_node_cache); +} + +extern void kmem_cache_zero_nr_tallocated(struct kmem_cache *); +void mt_zero_nr_tallocated(void) +{ + kmem_cache_zero_nr_tallocated(maple_node_cache); +} + +extern unsigned int kmem_cache_nr_tallocated(struct kmem_cache *); +unsigned int mt_nr_tallocated(void) +{ + return kmem_cache_nr_tallocated(maple_node_cache); +} + +extern unsigned int kmem_cache_nr_allocated(struct kmem_cache *); +unsigned int mt_nr_allocated(void) +{ + return kmem_cache_nr_allocated(maple_node_cache); +} + +/* + * mas_dead_node() - Check if the maple state is pointing to a dead node. + * @mas: The maple state + * @index: The index to restore in @mas. + * + * Used in test code. + * Return: 1 if @mas has been reset to MAS_START, 0 otherwise. + */ +static inline int mas_dead_node(struct ma_state *mas, unsigned long index) +{ + if (unlikely(!mas_searchable(mas) || mas_is_start(mas))) + return 0; + + if (likely(!mte_dead_node(mas->node))) + return 0; + + mas_rewalk(mas, index); + return 1; +} +#endif /* not defined __KERNEL__ */ + +/* + * mas_get_slot() - Get the entry in the maple state node stored at @offset. + * @mas: The maple state + * @offset: The offset into the slot array to fetch. + * + * Return: The entry stored at @offset. + */ +static inline struct maple_enode *mas_get_slot(struct ma_state *mas, + unsigned char offset) +{ + return mas_slot(mas, ma_slots(mas_mn(mas), mte_node_type(mas->node)), + offset); +} + + +/* + * mas_first_entry() - Go the first leaf and find the first entry. + * @mas: the maple state. + * @limit: the maximum index to check. + * @*r_start: Pointer to set to the range start. + * + * Sets mas->offset to the offset of the entry, r_start to the range minimum. + * + * Return: The first entry or MAS_NONE. + */ +static inline void *mas_first_entry(struct ma_state *mas, struct maple_node *mn, + unsigned long limit, enum maple_type mt) + +{ + unsigned long max; + unsigned long *pivots; + void __rcu **slots; + void *entry = NULL; + + mas->index = mas->min; + if (mas->index > limit) + goto none; + + max = mas->max; + mas->offset = 0; + while (likely(!ma_is_leaf(mt))) { + MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); + slots = ma_slots(mn, mt); + pivots = ma_pivots(mn, mt); + max = pivots[0]; + entry = mas_slot(mas, slots, 0); + if (unlikely(ma_dead_node(mn))) + return NULL; + mas->node = entry; + mn = mas_mn(mas); + mt = mte_node_type(mas->node); + } + MT_BUG_ON(mas->tree, mte_dead_node(mas->node)); + + mas->max = max; + slots = ma_slots(mn, mt); + entry = mas_slot(mas, slots, 0); + if (unlikely(ma_dead_node(mn))) + return NULL; + + /* Slot 0 or 1 must be set */ + if (mas->index > limit) + goto none; + + if (likely(entry)) + return entry; + + pivots = ma_pivots(mn, mt); + mas->index = pivots[0] + 1; + mas->offset = 1; + entry = mas_slot(mas, slots, 1); + if (unlikely(ma_dead_node(mn))) + return NULL; + + if (mas->index > limit) + goto none; + + if (likely(entry)) + return entry; + +none: + if (likely(!ma_dead_node(mn))) + mas->node = MAS_NONE; + return NULL; +} + +/* Depth first search, post-order */ +static void mas_dfs_postorder(struct ma_state *mas, unsigned long max) +{ + + struct maple_enode *p = MAS_NONE, *mn = mas->node; + unsigned long p_min, p_max; + + mas_next_node(mas, mas_mn(mas), max); + if (!mas_is_none(mas)) + return; + + if (mte_is_root(mn)) + return; + + mas->node = mn; + mas_ascend(mas); + while (mas->node != MAS_NONE) { + p = mas->node; + p_min = mas->min; + p_max = mas->max; + mas_prev_node(mas, 0); + } + + if (p == MAS_NONE) + return; + + mas->node = p; + mas->max = p_max; + mas->min = p_min; +} + +/* Tree validations */ +static void mt_dump_node(const struct maple_tree *mt, void *entry, + unsigned long min, unsigned long max, unsigned int depth); +static void mt_dump_range(unsigned long min, unsigned long max, + unsigned int depth) +{ + static const char spaces[] = " "; + + if (min == max) + pr_info("%.*s%lu: ", depth * 2, spaces, min); + else + pr_info("%.*s%lu-%lu: ", depth * 2, spaces, min, max); +} + +static void mt_dump_entry(void *entry, unsigned long min, unsigned long max, + unsigned int depth) +{ + mt_dump_range(min, max, depth); + + if (xa_is_value(entry)) + pr_cont("value %ld (0x%lx) [%p]\n", xa_to_value(entry), + xa_to_value(entry), entry); + else if (xa_is_zero(entry)) + pr_cont("zero (%ld)\n", xa_to_internal(entry)); + else if (mt_is_reserved(entry)) + pr_cont("UNKNOWN ENTRY (%p)\n", entry); + else + pr_cont("%p\n", entry); +} + +static void mt_dump_range64(const struct maple_tree *mt, void *entry, + unsigned long min, unsigned long max, unsigned int depth) +{ + struct maple_range_64 *node = &mte_to_node(entry)->mr64; + bool leaf = mte_is_leaf(entry); + unsigned long first = min; + int i; + + pr_cont(" contents: "); + for (i = 0; i < MAPLE_RANGE64_SLOTS - 1; i++) + pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + pr_cont("%p\n", node->slot[i]); + for (i = 0; i < MAPLE_RANGE64_SLOTS; i++) { + unsigned long last = max; + + if (i < (MAPLE_RANGE64_SLOTS - 1)) + last = node->pivot[i]; + else if (!node->slot[i] && max != mt_max[mte_node_type(entry)]) + break; + if (last == 0 && i > 0) + break; + if (leaf) + mt_dump_entry(mt_slot(mt, node->slot, i), + first, last, depth + 1); + else if (node->slot[i]) + mt_dump_node(mt, mt_slot(mt, node->slot, i), + first, last, depth + 1); + + if (last == max) + break; + if (last > max) { + pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", + node, last, max, i); + break; + } + first = last + 1; + } +} + +static void mt_dump_arange64(const struct maple_tree *mt, void *entry, + unsigned long min, unsigned long max, unsigned int depth) +{ + struct maple_arange_64 *node = &mte_to_node(entry)->ma64; + bool leaf = mte_is_leaf(entry); + unsigned long first = min; + int i; + + pr_cont(" contents: "); + for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) + pr_cont("%lu ", node->gap[i]); + pr_cont("| %02X %02X| ", node->meta.end, node->meta.gap); + for (i = 0; i < MAPLE_ARANGE64_SLOTS - 1; i++) + pr_cont("%p %lu ", node->slot[i], node->pivot[i]); + pr_cont("%p\n", node->slot[i]); + for (i = 0; i < MAPLE_ARANGE64_SLOTS; i++) { + unsigned long last = max; + + if (i < (MAPLE_ARANGE64_SLOTS - 1)) + last = node->pivot[i]; + else if (!node->slot[i]) + break; + if (last == 0 && i > 0) + break; + if (leaf) + mt_dump_entry(mt_slot(mt, node->slot, i), + first, last, depth + 1); + else if (node->slot[i]) + mt_dump_node(mt, mt_slot(mt, node->slot, i), + first, last, depth + 1); + + if (last == max) + break; + if (last > max) { + pr_err("node %p last (%lu) > max (%lu) at pivot %d!\n", + node, last, max, i); + break; + } + first = last + 1; + } +} + +static void mt_dump_node(const struct maple_tree *mt, void *entry, + unsigned long min, unsigned long max, unsigned int depth) +{ + struct maple_node *node = mte_to_node(entry); + unsigned int type = mte_node_type(entry); + unsigned int i; + + mt_dump_range(min, max, depth); + + pr_cont("node %p depth %d type %d parent %p", node, depth, type, + node ? node->parent : NULL); + switch (type) { + case maple_dense: + pr_cont("\n"); + for (i = 0; i < MAPLE_NODE_SLOTS; i++) { + if (min + i > max) + pr_cont("OUT OF RANGE: "); + mt_dump_entry(mt_slot(mt, node->slot, i), + min + i, min + i, depth); + } + break; + case maple_leaf_64: + case maple_range_64: + mt_dump_range64(mt, entry, min, max, depth); + break; + case maple_arange_64: + mt_dump_arange64(mt, entry, min, max, depth); + break; + + default: + pr_cont(" UNKNOWN TYPE\n"); + } +} + +void mt_dump(const struct maple_tree *mt) +{ + void *entry = rcu_dereference_check(mt->ma_root, mt_locked(mt)); + + pr_info("maple_tree(%p) flags %X, height %u root %p\n", + mt, mt->ma_flags, mt_height(mt), entry); + if (!xa_is_node(entry)) + mt_dump_entry(entry, 0, 0, 0); + else if (entry) + mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0); +} + +/* + * Calculate the maximum gap in a node and check if that's what is reported in + * the parent (unless root). + */ +static void mas_validate_gaps(struct ma_state *mas) +{ + struct maple_enode *mte = mas->node; + struct maple_node *p_mn; + unsigned long gap = 0, max_gap = 0; + unsigned long p_end, p_start = mas->min; + unsigned char p_slot; + unsigned long *gaps = NULL; + unsigned long *pivots = ma_pivots(mte_to_node(mte), mte_node_type(mte)); + int i; + + if (ma_is_dense(mte_node_type(mte))) { + for (i = 0; i < mt_slot_count(mte); i++) { + if (mas_get_slot(mas, i)) { + if (gap > max_gap) + max_gap = gap; + gap = 0; + continue; + } + gap++; + } + goto counted; + } + + gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte)); + for (i = 0; i < mt_slot_count(mte); i++) { + p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte)); + + if (!gaps) { + if (mas_get_slot(mas, i)) { + gap = 0; + goto not_empty; + } + + gap += p_end - p_start + 1; + } else { + void *entry = mas_get_slot(mas, i); + + gap = gaps[i]; + if (!entry) { + if (gap != p_end - p_start + 1) { + pr_err("%p[%u] -> %p %lu != %lu - %lu + 1\n", + mas_mn(mas), i, + mas_get_slot(mas, i), gap, + p_end, p_start); + mt_dump(mas->tree); + + MT_BUG_ON(mas->tree, + gap != p_end - p_start + 1); + } + } else { + if (gap > p_end - p_start + 1) { + pr_err("%p[%u] %lu >= %lu - %lu + 1 (%lu)\n", + mas_mn(mas), i, gap, p_end, p_start, + p_end - p_start + 1); + MT_BUG_ON(mas->tree, + gap > p_end - p_start + 1); + } + } + } + + if (gap > max_gap) + max_gap = gap; +not_empty: + p_start = p_end + 1; + if (p_end >= mas->max) + break; + } + +counted: + if (mte_is_root(mte)) + return; + + p_slot = mte_parent_slot(mas->node); + p_mn = mte_parent(mte); + MT_BUG_ON(mas->tree, max_gap > mas->max); + if (ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap) { + pr_err("gap %p[%u] != %lu\n", p_mn, p_slot, max_gap); + mt_dump(mas->tree); + } + + MT_BUG_ON(mas->tree, + ma_gaps(p_mn, mas_parent_enum(mas, mte))[p_slot] != max_gap); +} + +static void mas_validate_parent_slot(struct ma_state *mas) +{ + struct maple_node *parent; + struct maple_enode *node; + enum maple_type p_type = mas_parent_enum(mas, mas->node); + unsigned char p_slot = mte_parent_slot(mas->node); + void __rcu **slots; + int i; + + if (mte_is_root(mas->node)) + return; + + parent = mte_parent(mas->node); + slots = ma_slots(parent, p_type); + MT_BUG_ON(mas->tree, mas_mn(mas) == parent); + + /* Check prev/next parent slot for duplicate node entry */ + + for (i = 0; i < mt_slots[p_type]; i++) { + node = mas_slot(mas, slots, i); + if (i == p_slot) { + if (node != mas->node) + pr_err("parent %p[%u] does not have %p\n", + parent, i, mas_mn(mas)); + MT_BUG_ON(mas->tree, node != mas->node); + } else if (node == mas->node) { + pr_err("Invalid child %p at parent %p[%u] p_slot %u\n", + mas_mn(mas), parent, i, p_slot); + MT_BUG_ON(mas->tree, node == mas->node); + } + } +} + +static void mas_validate_child_slot(struct ma_state *mas) +{ + enum maple_type type = mte_node_type(mas->node); + void __rcu **slots = ma_slots(mte_to_node(mas->node), type); + unsigned long *pivots = ma_pivots(mte_to_node(mas->node), type); + struct maple_enode *child; + unsigned char i; + + if (mte_is_leaf(mas->node)) + return; + + for (i = 0; i < mt_slots[type]; i++) { + child = mas_slot(mas, slots, i); + if (!pivots[i] || pivots[i] == mas->max) + break; + + if (!child) + break; + + if (mte_parent_slot(child) != i) { + pr_err("Slot error at %p[%u]: child %p has pslot %u\n", + mas_mn(mas), i, mte_to_node(child), + mte_parent_slot(child)); + MT_BUG_ON(mas->tree, 1); + } + + if (mte_parent(child) != mte_to_node(mas->node)) { + pr_err("child %p has parent %p not %p\n", + mte_to_node(child), mte_parent(child), + mte_to_node(mas->node)); + MT_BUG_ON(mas->tree, 1); + } + } +} + +/* + * Validate all pivots are within mas->min and mas->max. + */ +static void mas_validate_limits(struct ma_state *mas) +{ + int i; + unsigned long prev_piv = 0; + enum maple_type type = mte_node_type(mas->node); + void __rcu **slots = ma_slots(mte_to_node(mas->node), type); + unsigned long *pivots = ma_pivots(mas_mn(mas), type); + + /* all limits are fine here. */ + if (mte_is_root(mas->node)) + return; + + for (i = 0; i < mt_slots[type]; i++) { + unsigned long piv; + + piv = mas_safe_pivot(mas, pivots, i, type); + + if (!piv && (i != 0)) + break; + + if (!mte_is_leaf(mas->node)) { + void *entry = mas_slot(mas, slots, i); + + if (!entry) + pr_err("%p[%u] cannot be null\n", + mas_mn(mas), i); + + MT_BUG_ON(mas->tree, !entry); + } + + if (prev_piv > piv) { + pr_err("%p[%u] piv %lu < prev_piv %lu\n", + mas_mn(mas), i, piv, prev_piv); + MT_BUG_ON(mas->tree, piv < prev_piv); + } + + if (piv < mas->min) { + pr_err("%p[%u] %lu < %lu\n", mas_mn(mas), i, + piv, mas->min); + MT_BUG_ON(mas->tree, piv < mas->min); + } + if (piv > mas->max) { + pr_err("%p[%u] %lu > %lu\n", mas_mn(mas), i, + piv, mas->max); + MT_BUG_ON(mas->tree, piv > mas->max); + } + prev_piv = piv; + if (piv == mas->max) + break; + } + for (i += 1; i < mt_slots[type]; i++) { + void *entry = mas_slot(mas, slots, i); + + if (entry && (i != mt_slots[type] - 1)) { + pr_err("%p[%u] should not have entry %p\n", mas_mn(mas), + i, entry); + MT_BUG_ON(mas->tree, entry != NULL); + } + + if (i < mt_pivots[type]) { + unsigned long piv = pivots[i]; + + if (!piv) + continue; + + pr_err("%p[%u] should not have piv %lu\n", + mas_mn(mas), i, piv); + MT_BUG_ON(mas->tree, i < mt_pivots[type] - 1); + } + } +} + +static void mt_validate_nulls(struct maple_tree *mt) +{ + void *entry, *last = (void *)1; + unsigned char offset = 0; + void __rcu **slots; + MA_STATE(mas, mt, 0, 0); + + mas_start(&mas); + if (mas_is_none(&mas) || (mas.node == MAS_ROOT)) + return; + + while (!mte_is_leaf(mas.node)) + mas_descend(&mas); + + slots = ma_slots(mte_to_node(mas.node), mte_node_type(mas.node)); + do { + entry = mas_slot(&mas, slots, offset); + if (!last && !entry) { + pr_err("Sequential nulls end at %p[%u]\n", + mas_mn(&mas), offset); + } + MT_BUG_ON(mt, !last && !entry); + last = entry; + if (offset == mas_data_end(&mas)) { + mas_next_node(&mas, mas_mn(&mas), ULONG_MAX); + if (mas_is_none(&mas)) + return; + offset = 0; + slots = ma_slots(mte_to_node(mas.node), + mte_node_type(mas.node)); + } else { + offset++; + } + + } while (!mas_is_none(&mas)); +} + +/* + * validate a maple tree by checking: + * 1. The limits (pivots are within mas->min to mas->max) + * 2. The gap is correctly set in the parents + */ +void mt_validate(struct maple_tree *mt) +{ + unsigned char end; + + MA_STATE(mas, mt, 0, 0); + rcu_read_lock(); + mas_start(&mas); + if (!mas_searchable(&mas)) + goto done; + + mas_first_entry(&mas, mas_mn(&mas), ULONG_MAX, mte_node_type(mas.node)); + while (!mas_is_none(&mas)) { + MT_BUG_ON(mas.tree, mte_dead_node(mas.node)); + if (!mte_is_root(mas.node)) { + end = mas_data_end(&mas); + if ((end < mt_min_slot_count(mas.node)) && + (mas.max != ULONG_MAX)) { + pr_err("Invalid size %u of %p\n", end, + mas_mn(&mas)); + MT_BUG_ON(mas.tree, 1); + } + + } + mas_validate_parent_slot(&mas); + mas_validate_child_slot(&mas); + mas_validate_limits(&mas); + if (mt_is_alloc(mt)) + mas_validate_gaps(&mas); + mas_dfs_postorder(&mas, ULONG_MAX); + } + mt_validate_nulls(mt); +done: + rcu_read_unlock(); + +} + +#endif /* CONFIG_DEBUG_MAPLE_TREE */ diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c index 62f8ffcbbaa352cd99812f2e38f27b56e6a542f9..2b5cc70ac53fcfb794d37cf69fc5ead928b2873f 100644 --- a/lib/memcpy_kunit.c +++ b/lib/memcpy_kunit.c @@ -29,9 +29,8 @@ struct some_bytes { }; #define check(instance, v) do { \ - int i; \ BUILD_BUG_ON(sizeof(instance.data) != 32); \ - for (i = 0; i < sizeof(instance.data); i++) { \ + for (size_t i = 0; i < sizeof(instance.data); i++) { \ KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \ "line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \ __LINE__, #instance, v, i, instance.data[i]); \ @@ -39,9 +38,8 @@ struct some_bytes { } while (0) #define compare(name, one, two) do { \ - int i; \ BUILD_BUG_ON(sizeof(one) != sizeof(two)); \ - for (i = 0; i < sizeof(one); i++) { \ + for (size_t i = 0; i < sizeof(one); i++) { \ KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \ "line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \ __LINE__, #one, i, one.data[i], #two, i, two.data[i]); \ @@ -272,10 +270,63 @@ static void memset_test(struct kunit *test) #undef TEST_OP } +static void strtomem_test(struct kunit *test) +{ + static const char input[sizeof(unsigned long)] = "hi"; + static const char truncate[] = "this is too long"; + struct { + unsigned long canary1; + unsigned char output[sizeof(unsigned long)] __nonstring; + unsigned long canary2; + } wrap; + + memset(&wrap, 0xFF, sizeof(wrap)); + KUNIT_EXPECT_EQ_MSG(test, wrap.canary1, ULONG_MAX, + "bad initial canary value"); + KUNIT_EXPECT_EQ_MSG(test, wrap.canary2, ULONG_MAX, + "bad initial canary value"); + + /* Check unpadded copy leaves surroundings untouched. */ + strtomem(wrap.output, input); + KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX); + KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]); + KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]); + for (size_t i = 2; i < sizeof(wrap.output); i++) + KUNIT_EXPECT_EQ(test, wrap.output[i], 0xFF); + KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX); + + /* Check truncated copy leaves surroundings untouched. */ + memset(&wrap, 0xFF, sizeof(wrap)); + strtomem(wrap.output, truncate); + KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX); + for (size_t i = 0; i < sizeof(wrap.output); i++) + KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]); + KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX); + + /* Check padded copy leaves only string padded. */ + memset(&wrap, 0xFF, sizeof(wrap)); + strtomem_pad(wrap.output, input, 0xAA); + KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX); + KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]); + KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]); + for (size_t i = 2; i < sizeof(wrap.output); i++) + KUNIT_EXPECT_EQ(test, wrap.output[i], 0xAA); + KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX); + + /* Check truncated padded copy has no padding. */ + memset(&wrap, 0xFF, sizeof(wrap)); + strtomem(wrap.output, truncate); + KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX); + for (size_t i = 0; i < sizeof(wrap.output); i++) + KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]); + KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX); +} + static struct kunit_case memcpy_test_cases[] = { KUNIT_CASE(memset_test), KUNIT_CASE(memcpy_test), KUNIT_CASE(memmove_test), + KUNIT_CASE(strtomem_test), {} }; diff --git a/lib/nlattr.c b/lib/nlattr.c index 86029ad5ead4f23192c423e4e9659befac3c7f01..40f22b177d690c360362530d98f3f4bcaf30bf08 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -159,6 +159,31 @@ void nla_get_range_unsigned(const struct nla_policy *pt, } } +static u64 nla_get_attr_bo(const struct nla_policy *pt, + const struct nlattr *nla) +{ + switch (pt->type) { + case NLA_U16: + if (pt->network_byte_order) + return ntohs(nla_get_be16(nla)); + + return nla_get_u16(nla); + case NLA_U32: + if (pt->network_byte_order) + return ntohl(nla_get_be32(nla)); + + return nla_get_u32(nla); + case NLA_U64: + if (pt->network_byte_order) + return be64_to_cpu(nla_get_be64(nla)); + + return nla_get_u64(nla); + } + + WARN_ON_ONCE(1); + return 0; +} + static int nla_validate_range_unsigned(const struct nla_policy *pt, const struct nlattr *nla, struct netlink_ext_ack *extack, @@ -172,12 +197,10 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt, value = nla_get_u8(nla); break; case NLA_U16: - value = nla_get_u16(nla); - break; case NLA_U32: - value = nla_get_u32(nla); - break; case NLA_U64: + value = nla_get_attr_bo(pt, nla); + break; case NLA_MSECS: value = nla_get_u64(nla); break; diff --git a/lib/once.c b/lib/once.c index 59149bf3bfb4a97e4fa7febee737155d700bae48..2c306f0e891e87ce033fbea2e1d431c3ea6ce4f6 100644 --- a/lib/once.c +++ b/lib/once.c @@ -66,3 +66,33 @@ void __do_once_done(bool *done, struct static_key_true *once_key, once_disable_jump(once_key, mod); } EXPORT_SYMBOL(__do_once_done); + +static DEFINE_MUTEX(once_mutex); + +bool __do_once_sleepable_start(bool *done) + __acquires(once_mutex) +{ + mutex_lock(&once_mutex); + if (*done) { + mutex_unlock(&once_mutex); + /* Keep sparse happy by restoring an even lock count on + * this mutex. In case we return here, we don't call into + * __do_once_done but return early in the DO_ONCE_SLEEPABLE() macro. + */ + __acquire(once_mutex); + return false; + } + + return true; +} +EXPORT_SYMBOL(__do_once_sleepable_start); + +void __do_once_sleepable_done(bool *done, struct static_key_true *once_key, + struct module *mod) + __releases(once_mutex) +{ + *done = true; + mutex_unlock(&once_mutex); + once_disable_jump(once_key, mod); +} +EXPORT_SYMBOL(__do_once_sleepable_done); diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index 7e3e43679b73cf5c46497fee6271d29f8cb80ea5..5369634701fa3442e2f849483c9c0054062f9087 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT /* * Test cases for arithmetic overflow checks. See: - * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests + * "Running tests with kunit_tool" at Documentation/dev-tools/kunit/start.rst * ./tools/testing/kunit/kunit.py run overflow [--raw_output] */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -16,12 +16,15 @@ #include #include -#define DEFINE_TEST_ARRAY(t) \ - static const struct test_ ## t { \ - t a, b; \ - t sum, diff, prod; \ - bool s_of, d_of, p_of; \ - } t ## _tests[] +#define DEFINE_TEST_ARRAY_TYPED(t1, t2, t) \ + static const struct test_ ## t1 ## _ ## t2 ## __ ## t { \ + t1 a; \ + t2 b; \ + t sum, diff, prod; \ + bool s_of, d_of, p_of; \ + } t1 ## _ ## t2 ## __ ## t ## _tests[] + +#define DEFINE_TEST_ARRAY(t) DEFINE_TEST_ARRAY_TYPED(t, t, t) DEFINE_TEST_ARRAY(u8) = { {0, 0, 0, 0, 0, false, false, false}, @@ -222,21 +225,27 @@ DEFINE_TEST_ARRAY(s64) = { }; #endif -#define check_one_op(t, fmt, op, sym, a, b, r, of) do { \ - t _r; \ - bool _of; \ - \ - _of = check_ ## op ## _overflow(a, b, &_r); \ - KUNIT_EXPECT_EQ_MSG(test, _of, of, \ +#define check_one_op(t, fmt, op, sym, a, b, r, of) do { \ + int _a_orig = a, _a_bump = a + 1; \ + int _b_orig = b, _b_bump = b + 1; \ + bool _of; \ + t _r; \ + \ + _of = check_ ## op ## _overflow(a, b, &_r); \ + KUNIT_EXPECT_EQ_MSG(test, _of, of, \ "expected "fmt" "sym" "fmt" to%s overflow (type %s)\n", \ - a, b, of ? "" : " not", #t); \ - KUNIT_EXPECT_EQ_MSG(test, _r, r, \ + a, b, of ? "" : " not", #t); \ + KUNIT_EXPECT_EQ_MSG(test, _r, r, \ "expected "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \ - a, b, r, _r, #t); \ + a, b, r, _r, #t); \ + /* Check for internal macro side-effects. */ \ + _of = check_ ## op ## _overflow(_a_orig++, _b_orig++, &_r); \ + KUNIT_EXPECT_EQ_MSG(test, _a_orig, _a_bump, "Unexpected " #op " macro side-effect!\n"); \ + KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, "Unexpected " #op " macro side-effect!\n"); \ } while (0) -#define DEFINE_TEST_FUNC(t, fmt) \ -static void do_test_ ## t(struct kunit *test, const struct test_ ## t *p) \ +#define DEFINE_TEST_FUNC_TYPED(n, t, fmt) \ +static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ { \ check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of); \ check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of); \ @@ -245,15 +254,18 @@ static void do_test_ ## t(struct kunit *test, const struct test_ ## t *p) \ check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \ } \ \ -static void t ## _overflow_test(struct kunit *test) { \ +static void n ## _overflow_test(struct kunit *test) { \ unsigned i; \ \ - for (i = 0; i < ARRAY_SIZE(t ## _tests); ++i) \ - do_test_ ## t(test, &t ## _tests[i]); \ + for (i = 0; i < ARRAY_SIZE(n ## _tests); ++i) \ + do_test_ ## n(test, &n ## _tests[i]); \ kunit_info(test, "%zu %s arithmetic tests finished\n", \ - ARRAY_SIZE(t ## _tests), #t); \ + ARRAY_SIZE(n ## _tests), #n); \ } +#define DEFINE_TEST_FUNC(t, fmt) \ + DEFINE_TEST_FUNC_TYPED(t ## _ ## t ## __ ## t, t, fmt) + DEFINE_TEST_FUNC(u8, "%d"); DEFINE_TEST_FUNC(s8, "%d"); DEFINE_TEST_FUNC(u16, "%d"); @@ -265,9 +277,32 @@ DEFINE_TEST_FUNC(u64, "%llu"); DEFINE_TEST_FUNC(s64, "%lld"); #endif -static void overflow_shift_test(struct kunit *test) -{ - int count = 0; +DEFINE_TEST_ARRAY_TYPED(u32, u32, u8) = { + {0, 0, 0, 0, 0, false, false, false}, + {U8_MAX, 2, 1, U8_MAX - 2, U8_MAX - 1, true, false, true}, + {U8_MAX + 1, 0, 0, 0, 0, true, true, false}, +}; +DEFINE_TEST_FUNC_TYPED(u32_u32__u8, u8, "%d"); + +DEFINE_TEST_ARRAY_TYPED(u32, u32, int) = { + {0, 0, 0, 0, 0, false, false, false}, + {U32_MAX, 0, -1, -1, 0, true, true, false}, +}; +DEFINE_TEST_FUNC_TYPED(u32_u32__int, int, "%d"); + +DEFINE_TEST_ARRAY_TYPED(u8, u8, int) = { + {0, 0, 0, 0, 0, false, false, false}, + {U8_MAX, U8_MAX, 2 * U8_MAX, 0, U8_MAX * U8_MAX, false, false, false}, + {1, 2, 3, -1, 2, false, false, false}, +}; +DEFINE_TEST_FUNC_TYPED(u8_u8__int, int, "%d"); + +DEFINE_TEST_ARRAY_TYPED(int, int, u8) = { + {0, 0, 0, 0, 0, false, false, false}, + {1, 2, 3, U8_MAX, 2, false, true, false}, + {-1, 0, U8_MAX, U8_MAX, 0, true, true, false}, +}; +DEFINE_TEST_FUNC_TYPED(int_int__u8, u8, "%d"); /* Args are: value, shift, type, expected result, overflow expected */ #define TEST_ONE_SHIFT(a, s, t, expect, of) do { \ @@ -292,6 +327,10 @@ static void overflow_shift_test(struct kunit *test) count++; \ } while (0) +static void shift_sane_test(struct kunit *test) +{ + int count = 0; + /* Sane shifts. */ TEST_ONE_SHIFT(1, 0, u8, 1 << 0, false); TEST_ONE_SHIFT(1, 4, u8, 1 << 4, false); @@ -334,6 +373,13 @@ static void overflow_shift_test(struct kunit *test) TEST_ONE_SHIFT(0, 30, s32, 0, false); TEST_ONE_SHIFT(0, 62, s64, 0, false); + kunit_info(test, "%d sane shift tests finished\n", count); +} + +static void shift_overflow_test(struct kunit *test) +{ + int count = 0; + /* Overflow: shifted the bit off the end. */ TEST_ONE_SHIFT(1, 8, u8, 0, true); TEST_ONE_SHIFT(1, 16, u16, 0, true); @@ -381,6 +427,13 @@ static void overflow_shift_test(struct kunit *test) /* 0100000100001000001000000010000001000010000001000100010001001011 */ TEST_ONE_SHIFT(4686030735197619275LL, 2, s64, 0, true); + kunit_info(test, "%d overflow shift tests finished\n", count); +} + +static void shift_truncate_test(struct kunit *test) +{ + int count = 0; + /* Overflow: values larger than destination type. */ TEST_ONE_SHIFT(0x100, 0, u8, 0, true); TEST_ONE_SHIFT(0xFF, 0, s8, 0, true); @@ -392,6 +445,33 @@ static void overflow_shift_test(struct kunit *test) TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, int, 0, true); TEST_ONE_SHIFT(0xFFFFFFFFFFFFFFFFULL, 0, s64, 0, true); + /* Overflow: shifted at or beyond entire type's bit width. */ + TEST_ONE_SHIFT(0, 8, u8, 0, true); + TEST_ONE_SHIFT(0, 9, u8, 0, true); + TEST_ONE_SHIFT(0, 8, s8, 0, true); + TEST_ONE_SHIFT(0, 9, s8, 0, true); + TEST_ONE_SHIFT(0, 16, u16, 0, true); + TEST_ONE_SHIFT(0, 17, u16, 0, true); + TEST_ONE_SHIFT(0, 16, s16, 0, true); + TEST_ONE_SHIFT(0, 17, s16, 0, true); + TEST_ONE_SHIFT(0, 32, u32, 0, true); + TEST_ONE_SHIFT(0, 33, u32, 0, true); + TEST_ONE_SHIFT(0, 32, int, 0, true); + TEST_ONE_SHIFT(0, 33, int, 0, true); + TEST_ONE_SHIFT(0, 32, s32, 0, true); + TEST_ONE_SHIFT(0, 33, s32, 0, true); + TEST_ONE_SHIFT(0, 64, u64, 0, true); + TEST_ONE_SHIFT(0, 65, u64, 0, true); + TEST_ONE_SHIFT(0, 64, s64, 0, true); + TEST_ONE_SHIFT(0, 65, s64, 0, true); + + kunit_info(test, "%d truncate shift tests finished\n", count); +} + +static void shift_nonsense_test(struct kunit *test) +{ + int count = 0; + /* Nonsense: negative initial value. */ TEST_ONE_SHIFT(-1, 0, s8, 0, true); TEST_ONE_SHIFT(-1, 0, u8, 0, true); @@ -416,26 +496,6 @@ static void overflow_shift_test(struct kunit *test) TEST_ONE_SHIFT(0, -30, s64, 0, true); TEST_ONE_SHIFT(0, -30, u64, 0, true); - /* Overflow: shifted at or beyond entire type's bit width. */ - TEST_ONE_SHIFT(0, 8, u8, 0, true); - TEST_ONE_SHIFT(0, 9, u8, 0, true); - TEST_ONE_SHIFT(0, 8, s8, 0, true); - TEST_ONE_SHIFT(0, 9, s8, 0, true); - TEST_ONE_SHIFT(0, 16, u16, 0, true); - TEST_ONE_SHIFT(0, 17, u16, 0, true); - TEST_ONE_SHIFT(0, 16, s16, 0, true); - TEST_ONE_SHIFT(0, 17, s16, 0, true); - TEST_ONE_SHIFT(0, 32, u32, 0, true); - TEST_ONE_SHIFT(0, 33, u32, 0, true); - TEST_ONE_SHIFT(0, 32, int, 0, true); - TEST_ONE_SHIFT(0, 33, int, 0, true); - TEST_ONE_SHIFT(0, 32, s32, 0, true); - TEST_ONE_SHIFT(0, 33, s32, 0, true); - TEST_ONE_SHIFT(0, 64, u64, 0, true); - TEST_ONE_SHIFT(0, 65, u64, 0, true); - TEST_ONE_SHIFT(0, 64, s64, 0, true); - TEST_ONE_SHIFT(0, 65, s64, 0, true); - /* * Corner case: for unsigned types, we fail when we've shifted * through the entire width of bits. For signed types, we might @@ -451,9 +511,9 @@ static void overflow_shift_test(struct kunit *test) TEST_ONE_SHIFT(0, 31, s32, 0, false); TEST_ONE_SHIFT(0, 63, s64, 0, false); - kunit_info(test, "%d shift tests finished\n", count); -#undef TEST_ONE_SHIFT + kunit_info(test, "%d nonsense shift tests finished\n", count); } +#undef TEST_ONE_SHIFT /* * Deal with the various forms of allocator arguments. See comments above @@ -649,18 +709,25 @@ static void overflow_size_helpers_test(struct kunit *test) } static struct kunit_case overflow_test_cases[] = { - KUNIT_CASE(u8_overflow_test), - KUNIT_CASE(s8_overflow_test), - KUNIT_CASE(u16_overflow_test), - KUNIT_CASE(s16_overflow_test), - KUNIT_CASE(u32_overflow_test), - KUNIT_CASE(s32_overflow_test), + KUNIT_CASE(u8_u8__u8_overflow_test), + KUNIT_CASE(s8_s8__s8_overflow_test), + KUNIT_CASE(u16_u16__u16_overflow_test), + KUNIT_CASE(s16_s16__s16_overflow_test), + KUNIT_CASE(u32_u32__u32_overflow_test), + KUNIT_CASE(s32_s32__s32_overflow_test), /* Clang 13 and earlier generate unwanted libcalls on 32-bit. */ #if BITS_PER_LONG == 64 - KUNIT_CASE(u64_overflow_test), - KUNIT_CASE(s64_overflow_test), + KUNIT_CASE(u64_u64__u64_overflow_test), + KUNIT_CASE(s64_s64__s64_overflow_test), #endif - KUNIT_CASE(overflow_shift_test), + KUNIT_CASE(u32_u32__u8_overflow_test), + KUNIT_CASE(u32_u32__int_overflow_test), + KUNIT_CASE(u8_u8__int_overflow_test), + KUNIT_CASE(int_int__u8_overflow_test), + KUNIT_CASE(shift_sane_test), + KUNIT_CASE(shift_overflow_test), + KUNIT_CASE(shift_truncate_test), + KUNIT_CASE(shift_nonsense_test), KUNIT_CASE(overflow_allocation_test), KUNIT_CASE(overflow_size_helpers_test), {} diff --git a/lib/random32.c b/lib/random32.c index d5d9029362cbb3e6d44e98c41ce16c4f01360429..32060b8526681f069d97b1ffa1d53d3cc1e3e620 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -47,7 +47,7 @@ * @state: pointer to state structure holding seeded state. * * This is used for pseudo-randomness with no outside seeding. - * For more random results, use prandom_u32(). + * For more random results, use get_random_u32(). */ u32 prandom_u32_state(struct rnd_state *state) { @@ -69,7 +69,7 @@ EXPORT_SYMBOL(prandom_u32_state); * @bytes: the requested number of bytes * * This is used for pseudo-randomness with no outside seeding. - * For more random results, use prandom_bytes(). + * For more random results, use get_random_bytes(). */ void prandom_bytes_state(struct rnd_state *state, void *buf, size_t bytes) { diff --git a/lib/ratelimit.c b/lib/ratelimit.c index e01a93f46f833483680b876fe9df7573bf4c2281..ce945c17980b9b8addd4c044e1080f73f21a4f3d 100644 --- a/lib/ratelimit.c +++ b/lib/ratelimit.c @@ -26,10 +26,16 @@ */ int ___ratelimit(struct ratelimit_state *rs, const char *func) { + /* Paired with WRITE_ONCE() in .proc_handler(). + * Changing two values seperately could be inconsistent + * and some message could be lost. (See: net_ratelimit_state). + */ + int interval = READ_ONCE(rs->interval); + int burst = READ_ONCE(rs->burst); unsigned long flags; int ret; - if (!rs->interval) + if (!interval) return 1; /* @@ -44,7 +50,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) if (!rs->begin) rs->begin = jiffies; - if (time_is_before_jiffies(rs->begin + rs->interval)) { + if (time_is_before_jiffies(rs->begin + interval)) { if (rs->missed) { if (!(rs->flags & RATELIMIT_MSG_ON_RELEASE)) { printk_deferred(KERN_WARNING @@ -56,7 +62,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies; rs->printed = 0; } - if (rs->burst && rs->burst > rs->printed) { + if (burst && burst > rs->printed) { rs->printed++; ret = 1; } else { diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c index d9d1c33aebdaee97dae05004bf75f8e95a9edc38..848e7eb5da92170134761624cedd8f837a6bc936 100644 --- a/lib/reed_solomon/test_rslib.c +++ b/lib/reed_solomon/test_rslib.c @@ -164,7 +164,7 @@ static int get_rcw_we(struct rs_control *rs, struct wspace *ws, /* Load c with random data and encode */ for (i = 0; i < dlen; i++) - c[i] = prandom_u32() & nn; + c[i] = get_random_u32() & nn; memset(c + dlen, 0, nroots * sizeof(*c)); encode_rs16(rs, c, dlen, c + dlen, 0); @@ -178,12 +178,12 @@ static int get_rcw_we(struct rs_control *rs, struct wspace *ws, for (i = 0; i < errs; i++) { do { /* Error value must be nonzero */ - errval = prandom_u32() & nn; + errval = get_random_u32() & nn; } while (errval == 0); do { /* Must not choose the same location twice */ - errloc = prandom_u32() % len; + errloc = prandom_u32_max(len); } while (errlocs[errloc] != 0); errlocs[errloc] = 1; @@ -194,19 +194,19 @@ static int get_rcw_we(struct rs_control *rs, struct wspace *ws, for (i = 0; i < eras; i++) { do { /* Must not choose the same location twice */ - errloc = prandom_u32() % len; + errloc = prandom_u32_max(len); } while (errlocs[errloc] != 0); derrlocs[i] = errloc; - if (ewsc && (prandom_u32() & 1)) { + if (ewsc && prandom_u32_max(2)) { /* Erasure with the symbol intact */ errlocs[errloc] = 2; } else { /* Erasure with corrupted symbol */ do { /* Error value must be nonzero */ - errval = prandom_u32() & nn; + errval = get_random_u32() & nn; } while (errval == 0); errlocs[errloc] = 1; diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 29eb0484215afd190d78434850cea9b47b21b848..7280ae8ca88c7c23b272296ed3a4c8953046512e 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -21,7 +21,7 @@ static int init_alloc_hint(struct sbitmap *sb, gfp_t flags) int i; for_each_possible_cpu(i) - *per_cpu_ptr(sb->alloc_hint, i) = prandom_u32() % depth; + *per_cpu_ptr(sb->alloc_hint, i) = prandom_u32_max(depth); } return 0; } @@ -33,7 +33,7 @@ static inline unsigned update_alloc_hint_before_get(struct sbitmap *sb, hint = this_cpu_read(*sb->alloc_hint); if (unlikely(hint >= depth)) { - hint = depth ? prandom_u32() % depth : 0; + hint = depth ? prandom_u32_max(depth) : 0; this_cpu_write(*sb->alloc_hint, hint); } @@ -533,21 +533,20 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, nr = find_first_zero_bit(&map->word, map_depth); if (nr + nr_tags <= map_depth) { atomic_long_t *ptr = (atomic_long_t *) &map->word; - int map_tags = min_t(int, nr_tags, map_depth); - unsigned long val, ret; + unsigned long val; - get_mask = ((1UL << map_tags) - 1) << nr; + get_mask = ((1UL << nr_tags) - 1) << nr; + val = READ_ONCE(map->word); do { - val = READ_ONCE(map->word); if ((val & ~get_mask) != val) goto next; - ret = atomic_long_cmpxchg(ptr, val, get_mask | val); - } while (ret != val); - get_mask = (get_mask & ~ret) >> nr; + } while (!atomic_long_try_cmpxchg(ptr, &val, + get_mask | val)); + get_mask = (get_mask & ~val) >> nr; if (get_mask) { *offset = nr + (index << sb->shift); update_alloc_hint_after_get(sb, depth, hint, - *offset + map_tags - 1); + *offset + nr_tags - 1); return get_mask; } } @@ -588,7 +587,7 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) for (i = 0; i < SBQ_WAIT_QUEUES; i++) { struct sbq_wait_state *ws = &sbq->ws[wake_index]; - if (waitqueue_active(&ws->wait)) { + if (waitqueue_active(&ws->wait) && atomic_read(&ws->wait_cnt)) { if (wake_index != atomic_read(&sbq->wake_index)) atomic_set(&sbq->wake_index, wake_index); return ws; @@ -600,50 +599,82 @@ static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) return NULL; } -static bool __sbq_wake_up(struct sbitmap_queue *sbq) +static bool __sbq_wake_up(struct sbitmap_queue *sbq, int *nr) { struct sbq_wait_state *ws; unsigned int wake_batch; - int wait_cnt; + int wait_cnt, cur, sub; + bool ret; + + if (*nr <= 0) + return false; ws = sbq_wake_ptr(sbq); if (!ws) return false; - wait_cnt = atomic_dec_return(&ws->wait_cnt); - if (wait_cnt <= 0) { - int ret; - - wake_batch = READ_ONCE(sbq->wake_batch); - + cur = atomic_read(&ws->wait_cnt); + do { /* - * Pairs with the memory barrier in sbitmap_queue_resize() to - * ensure that we see the batch size update before the wait - * count is reset. + * For concurrent callers of this, callers should call this + * function again to wakeup a new batch on a different 'ws'. */ - smp_mb__before_atomic(); + if (cur == 0) + return true; + sub = min(*nr, cur); + wait_cnt = cur - sub; + } while (!atomic_try_cmpxchg(&ws->wait_cnt, &cur, wait_cnt)); - /* - * For concurrent callers of this, the one that failed the - * atomic_cmpxhcg() race should call this function again - * to wakeup a new batch on a different 'ws'. - */ - ret = atomic_cmpxchg(&ws->wait_cnt, wait_cnt, wake_batch); - if (ret == wait_cnt) { - sbq_index_atomic_inc(&sbq->wake_index); - wake_up_nr(&ws->wait, wake_batch); - return false; - } + /* + * If we decremented queue without waiters, retry to avoid lost + * wakeups. + */ + if (wait_cnt > 0) + return !waitqueue_active(&ws->wait); - return true; - } + *nr -= sub; - return false; + /* + * When wait_cnt == 0, we have to be particularly careful as we are + * responsible to reset wait_cnt regardless whether we've actually + * woken up anybody. But in case we didn't wakeup anybody, we still + * need to retry. + */ + ret = !waitqueue_active(&ws->wait); + wake_batch = READ_ONCE(sbq->wake_batch); + + /* + * Wake up first in case that concurrent callers decrease wait_cnt + * while waitqueue is empty. + */ + wake_up_nr(&ws->wait, wake_batch); + + /* + * Pairs with the memory barrier in sbitmap_queue_resize() to + * ensure that we see the batch size update before the wait + * count is reset. + * + * Also pairs with the implicit barrier between decrementing wait_cnt + * and checking for waitqueue_active() to make sure waitqueue_active() + * sees result of the wakeup if atomic_dec_return() has seen the result + * of atomic_set(). + */ + smp_mb__before_atomic(); + + /* + * Increase wake_index before updating wait_cnt, otherwise concurrent + * callers can see valid wait_cnt in old waitqueue, which can cause + * invalid wakeup on the old waitqueue. + */ + sbq_index_atomic_inc(&sbq->wake_index); + atomic_set(&ws->wait_cnt, wake_batch); + + return ret || *nr; } -void sbitmap_queue_wake_up(struct sbitmap_queue *sbq) +void sbitmap_queue_wake_up(struct sbitmap_queue *sbq, int nr) { - while (__sbq_wake_up(sbq)) + while (__sbq_wake_up(sbq, &nr)) ; } EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up); @@ -683,7 +714,7 @@ void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset, atomic_long_andnot(mask, (atomic_long_t *) addr); smp_mb__after_atomic(); - sbitmap_queue_wake_up(sbq); + sbitmap_queue_wake_up(sbq, nr_tags); sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(), tags[nr_tags - 1] - offset); } @@ -711,7 +742,7 @@ void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, * waiter. See the comment on waitqueue_active(). */ smp_mb__after_atomic(); - sbitmap_queue_wake_up(sbq); + sbitmap_queue_wake_up(sbq, 1); sbitmap_update_cpu_hint(&sbq->sb, cpu, nr); } EXPORT_SYMBOL_GPL(sbitmap_queue_clear); diff --git a/lib/sg_pool.c b/lib/sg_pool.c index a0b1a52cd6f7510cecdf8800a550441d208879f6..9bfe60ca3f37e143e2d71c0acdf630b1e67dc37e 100644 --- a/lib/sg_pool.c +++ b/lib/sg_pool.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#include +#include #include #include #include @@ -177,16 +177,4 @@ cleanup_sdb: return -ENOMEM; } -static __exit void sg_pool_exit(void) -{ - int i; - - for (i = 0; i < SG_MEMPOOL_NR; i++) { - struct sg_pool *sgp = sg_pools + i; - mempool_destroy(sgp->pool); - kmem_cache_destroy(sgp->slab); - } -} - -module_init(sg_pool_init); -module_exit(sg_pool_exit); +subsys_initcall(sg_pool_init); diff --git a/lib/show_mem.c b/lib/show_mem.c index 1c26c14ffbb9bdfe8d442cb381e7c7d1fd242305..0d7585cde2a69ef1fcdda569b939d6edc602ca99 100644 --- a/lib/show_mem.c +++ b/lib/show_mem.c @@ -8,13 +8,13 @@ #include #include -void show_mem(unsigned int filter, nodemask_t *nodemask) +void __show_mem(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) { pg_data_t *pgdat; unsigned long total = 0, reserved = 0, highmem = 0; printk("Mem-Info:\n"); - show_free_areas(filter, nodemask); + __show_free_areas(filter, nodemask, max_zone_idx); for_each_online_pgdat(pgdat) { int zoneid; diff --git a/lib/stackdepot.c b/lib/stackdepot.c index e73fda23388d8cca22e1f999afa5eb1a6597398f..79e894cf84064ab3b84f8a0a38cc8cb278347e57 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -43,7 +43,8 @@ #define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \ STACK_ALLOC_ALIGN) #define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - \ - STACK_ALLOC_NULL_PROTECTION_BITS - STACK_ALLOC_OFFSET_BITS) + STACK_ALLOC_NULL_PROTECTION_BITS - \ + STACK_ALLOC_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS) #define STACK_ALLOC_SLABS_CAP 8192 #define STACK_ALLOC_MAX_SLABS \ (((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \ @@ -56,6 +57,7 @@ union handle_parts { u32 slabindex : STACK_ALLOC_INDEX_BITS; u32 offset : STACK_ALLOC_OFFSET_BITS; u32 valid : STACK_ALLOC_NULL_PROTECTION_BITS; + u32 extra : STACK_DEPOT_EXTRA_BITS; }; }; @@ -77,6 +79,14 @@ static int next_slab_inited; static size_t depot_offset; static DEFINE_RAW_SPINLOCK(depot_lock); +unsigned int stack_depot_get_extra_bits(depot_stack_handle_t handle) +{ + union handle_parts parts = { .handle = handle }; + + return parts.extra; +} +EXPORT_SYMBOL(stack_depot_get_extra_bits); + static bool init_stack_slab(void **prealloc) { if (!*prealloc) @@ -140,6 +150,7 @@ depot_alloc_stack(unsigned long *entries, int size, u32 hash, void **prealloc) stack->handle.slabindex = depot_index; stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN; stack->handle.valid = 1; + stack->handle.extra = 0; memcpy(stack->entries, entries, flex_array_size(stack, entries, size)); depot_offset += required_size; @@ -382,6 +393,7 @@ EXPORT_SYMBOL_GPL(stack_depot_fetch); * * @entries: Pointer to storage array * @nr_entries: Size of the storage array + * @extra_bits: Flags to store in unused bits of depot_stack_handle_t * @alloc_flags: Allocation gfp flags * @can_alloc: Allocate stack slabs (increased chance of failure if false) * @@ -393,6 +405,10 @@ EXPORT_SYMBOL_GPL(stack_depot_fetch); * If the stack trace in @entries is from an interrupt, only the portion up to * interrupt entry is saved. * + * Additional opaque flags can be passed in @extra_bits, stored in the unused + * bits of the stack handle, and retrieved using stack_depot_get_extra_bits() + * without calling stack_depot_fetch(). + * * Context: Any context, but setting @can_alloc to %false is required if * alloc_pages() cannot be used from the current context. Currently * this is the case from contexts where neither %GFP_ATOMIC nor @@ -402,10 +418,11 @@ EXPORT_SYMBOL_GPL(stack_depot_fetch); */ depot_stack_handle_t __stack_depot_save(unsigned long *entries, unsigned int nr_entries, + unsigned int extra_bits, gfp_t alloc_flags, bool can_alloc) { struct stack_record *found = NULL, **bucket; - depot_stack_handle_t retval = 0; + union handle_parts retval = { .handle = 0 }; struct page *page = NULL; void *prealloc = NULL; unsigned long flags; @@ -489,9 +506,11 @@ exit: free_pages((unsigned long)prealloc, STACK_ALLOC_ORDER); } if (found) - retval = found->handle.handle; + retval.handle = found->handle.handle; fast_exit: - return retval; + retval.extra = extra_bits; + + return retval.handle; } EXPORT_SYMBOL_GPL(__stack_depot_save); @@ -511,6 +530,6 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries, unsigned int nr_entries, gfp_t alloc_flags) { - return __stack_depot_save(entries, nr_entries, alloc_flags, true); + return __stack_depot_save(entries, nr_entries, 0, alloc_flags, true); } EXPORT_SYMBOL_GPL(stack_depot_save); diff --git a/lib/stackinit_kunit.c b/lib/stackinit_kunit.c index 35c69aa425b208ca67a35fa4077af7b896f348ce..4591d6cf5e01800aa14e980c231695b6682f0b91 100644 --- a/lib/stackinit_kunit.c +++ b/lib/stackinit_kunit.c @@ -3,7 +3,7 @@ * Test cases for compiler-based stack variable zeroing via * -ftrivial-auto-var-init={zero,pattern} or CONFIG_GCC_PLUGIN_STRUCTLEAK*. * For example, see: - * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests + * "Running tests with kunit_tool" at Documentation/dev-tools/kunit/start.rst * ./tools/testing/kunit/kunit.py run stackinit [--raw_output] \ * --make_option LLVM=1 \ * --kconfig_add CONFIG_INIT_STACK_ALL_ZERO=y diff --git a/lib/string.c b/lib/string.c index 6f334420f68718f3d25bf470f0feb0842229b38b..3371d26a0e390cd6c3650be0d59b2e78d0b452b3 100644 --- a/lib/string.c +++ b/lib/string.c @@ -197,6 +197,14 @@ ssize_t strscpy(char *dest, const char *src, size_t count) max = 0; #endif + /* + * read_word_at_a_time() below may read uninitialized bytes after the + * trailing zero and use them in comparisons. Disable this optimization + * under KMSAN to prevent false positive reports. + */ + if (IS_ENABLED(CONFIG_KMSAN)) + max = 0; + while (max >= sizeof(unsigned long)) { unsigned long c, data; diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 5ed3beb066e6d09fe8deae3607a8f58416ad13f3..230020a2e076ce9e14e2a2c960218eac5f6e0134 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -131,6 +131,50 @@ void string_get_size(u64 size, u64 blk_size, const enum string_size_units units, } EXPORT_SYMBOL(string_get_size); +/** + * parse_int_array_user - Split string into a sequence of integers + * @from: The user space buffer to read from + * @count: The maximum number of bytes to read + * @array: Returned pointer to sequence of integers + * + * On success @array is allocated and initialized with a sequence of + * integers extracted from the @from plus an additional element that + * begins the sequence and specifies the integers count. + * + * Caller takes responsibility for freeing @array when it is no longer + * needed. + */ +int parse_int_array_user(const char __user *from, size_t count, int **array) +{ + int *ints, nints; + char *buf; + int ret = 0; + + buf = memdup_user_nul(from, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + get_options(buf, 0, &nints); + if (!nints) { + ret = -ENOENT; + goto free_buf; + } + + ints = kcalloc(nints + 1, sizeof(*ints), GFP_KERNEL); + if (!ints) { + ret = -ENOMEM; + goto free_buf; + } + + get_options(buf, nints + 1, ints); + *array = ints; + +free_buf: + kfree(buf); + return ret; +} +EXPORT_SYMBOL(parse_int_array_user); + static bool unescape_space(char **src, char **dst) { char *p = *dst, *q = *src; diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 437d8e6b7cb124798ef0dc3b2ab99e1b0e8bd3fe..86fadd3ba08c58ed23961a2378bb25f82e793d1a 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -587,7 +587,7 @@ static int __init test_string_helpers_init(void) for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++) test_string_unescape("unescape", i, false); test_string_unescape("unescape inplace", - get_random_int() % (UNESCAPE_ANY + 1), true); + prandom_u32_max(UNESCAPE_ANY + 1), true); /* Without dictionary */ for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 98754ff9fe686fab28c6e5c6d2e62cf218b6ea27..a8005ad3bd5893987810dd7fbc44a17c24366479 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -16,6 +16,8 @@ #include "../tools/testing/selftests/kselftest_module.h" +#define EXP1_IN_BITS (sizeof(exp1) * 8) + KSTM_MODULE_GLOBALS(); static char pbl_buffer[PAGE_SIZE] __initdata; @@ -219,6 +221,47 @@ static void __init test_zero_clear(void) expect_eq_pbl("", bmap, 1024); } +static void __init test_find_nth_bit(void) +{ + unsigned long b, bit, cnt = 0; + DECLARE_BITMAP(bmap, 64 * 3); + + bitmap_zero(bmap, 64 * 3); + __set_bit(10, bmap); + __set_bit(20, bmap); + __set_bit(30, bmap); + __set_bit(40, bmap); + __set_bit(50, bmap); + __set_bit(60, bmap); + __set_bit(80, bmap); + __set_bit(123, bmap); + + expect_eq_uint(10, find_nth_bit(bmap, 64 * 3, 0)); + expect_eq_uint(20, find_nth_bit(bmap, 64 * 3, 1)); + expect_eq_uint(30, find_nth_bit(bmap, 64 * 3, 2)); + expect_eq_uint(40, find_nth_bit(bmap, 64 * 3, 3)); + expect_eq_uint(50, find_nth_bit(bmap, 64 * 3, 4)); + expect_eq_uint(60, find_nth_bit(bmap, 64 * 3, 5)); + expect_eq_uint(80, find_nth_bit(bmap, 64 * 3, 6)); + expect_eq_uint(123, find_nth_bit(bmap, 64 * 3, 7)); + expect_eq_uint(64 * 3, find_nth_bit(bmap, 64 * 3, 8)); + + expect_eq_uint(10, find_nth_bit(bmap, 64 * 3 - 1, 0)); + expect_eq_uint(20, find_nth_bit(bmap, 64 * 3 - 1, 1)); + expect_eq_uint(30, find_nth_bit(bmap, 64 * 3 - 1, 2)); + expect_eq_uint(40, find_nth_bit(bmap, 64 * 3 - 1, 3)); + expect_eq_uint(50, find_nth_bit(bmap, 64 * 3 - 1, 4)); + expect_eq_uint(60, find_nth_bit(bmap, 64 * 3 - 1, 5)); + expect_eq_uint(80, find_nth_bit(bmap, 64 * 3 - 1, 6)); + expect_eq_uint(123, find_nth_bit(bmap, 64 * 3 - 1, 7)); + expect_eq_uint(64 * 3 - 1, find_nth_bit(bmap, 64 * 3 - 1, 8)); + + for_each_set_bit(bit, exp1, EXP1_IN_BITS) { + b = find_nth_bit(exp1, EXP1_IN_BITS, cnt++); + expect_eq_uint(b, bit); + } +} + static void __init test_fill_set(void) { DECLARE_BITMAP(bmap, 1024); @@ -557,8 +600,6 @@ static void __init test_bitmap_parse(void) } } -#define EXP1_IN_BITS (sizeof(exp1) * 8) - static void __init test_bitmap_arr32(void) { unsigned int nbits, next_bit; @@ -685,6 +726,239 @@ static void __init test_for_each_set_clump8(void) expect_eq_clump8(start, CLUMP_EXP_NUMBITS, clump_exp, &clump); } +static void __init test_for_each_set_bit_wrap(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int wr, bit; + + bitmap_zero(orig, 500); + + /* Set individual bits */ + for (bit = 0; bit < 500; bit += 10) + bitmap_set(orig, bit, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for (wr = 0; wr < 500; wr++) { + bitmap_zero(copy, 500); + + for_each_set_bit_wrap(bit, orig, 500, wr) + bitmap_set(copy, bit, 1); + + expect_eq_bitmap(orig, copy, 500); + } +} + +static void __init test_for_each_set_bit(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int bit; + + bitmap_zero(orig, 500); + bitmap_zero(copy, 500); + + /* Set individual bits */ + for (bit = 0; bit < 500; bit += 10) + bitmap_set(orig, bit, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for_each_set_bit(bit, orig, 500) + bitmap_set(copy, bit, 1); + + expect_eq_bitmap(orig, copy, 500); +} + +static void __init test_for_each_set_bit_from(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int wr, bit; + + bitmap_zero(orig, 500); + + /* Set individual bits */ + for (bit = 0; bit < 500; bit += 10) + bitmap_set(orig, bit, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for (wr = 0; wr < 500; wr++) { + DECLARE_BITMAP(tmp, 500); + + bitmap_zero(copy, 500); + bit = wr; + + for_each_set_bit_from(bit, orig, 500) + bitmap_set(copy, bit, 1); + + bitmap_copy(tmp, orig, 500); + bitmap_clear(tmp, 0, wr); + expect_eq_bitmap(tmp, copy, 500); + } +} + +static void __init test_for_each_clear_bit(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int bit; + + bitmap_fill(orig, 500); + bitmap_fill(copy, 500); + + /* Set individual bits */ + for (bit = 0; bit < 500; bit += 10) + bitmap_clear(orig, bit, 1); + + /* Set range of bits */ + bitmap_clear(orig, 100, 50); + + for_each_clear_bit(bit, orig, 500) + bitmap_clear(copy, bit, 1); + + expect_eq_bitmap(orig, copy, 500); +} + +static void __init test_for_each_clear_bit_from(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int wr, bit; + + bitmap_fill(orig, 500); + + /* Set individual bits */ + for (bit = 0; bit < 500; bit += 10) + bitmap_clear(orig, bit, 1); + + /* Set range of bits */ + bitmap_clear(orig, 100, 50); + + for (wr = 0; wr < 500; wr++) { + DECLARE_BITMAP(tmp, 500); + + bitmap_fill(copy, 500); + bit = wr; + + for_each_clear_bit_from(bit, orig, 500) + bitmap_clear(copy, bit, 1); + + bitmap_copy(tmp, orig, 500); + bitmap_set(tmp, 0, wr); + expect_eq_bitmap(tmp, copy, 500); + } +} + +static void __init test_for_each_set_bitrange(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int s, e; + + bitmap_zero(orig, 500); + bitmap_zero(copy, 500); + + /* Set individual bits */ + for (s = 0; s < 500; s += 10) + bitmap_set(orig, s, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for_each_set_bitrange(s, e, orig, 500) + bitmap_set(copy, s, e-s); + + expect_eq_bitmap(orig, copy, 500); +} + +static void __init test_for_each_clear_bitrange(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int s, e; + + bitmap_fill(orig, 500); + bitmap_fill(copy, 500); + + /* Set individual bits */ + for (s = 0; s < 500; s += 10) + bitmap_clear(orig, s, 1); + + /* Set range of bits */ + bitmap_clear(orig, 100, 50); + + for_each_clear_bitrange(s, e, orig, 500) + bitmap_clear(copy, s, e-s); + + expect_eq_bitmap(orig, copy, 500); +} + +static void __init test_for_each_set_bitrange_from(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int wr, s, e; + + bitmap_zero(orig, 500); + + /* Set individual bits */ + for (s = 0; s < 500; s += 10) + bitmap_set(orig, s, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for (wr = 0; wr < 500; wr++) { + DECLARE_BITMAP(tmp, 500); + + bitmap_zero(copy, 500); + s = wr; + + for_each_set_bitrange_from(s, e, orig, 500) + bitmap_set(copy, s, e - s); + + bitmap_copy(tmp, orig, 500); + bitmap_clear(tmp, 0, wr); + expect_eq_bitmap(tmp, copy, 500); + } +} + +static void __init test_for_each_clear_bitrange_from(void) +{ + DECLARE_BITMAP(orig, 500); + DECLARE_BITMAP(copy, 500); + unsigned int wr, s, e; + + bitmap_fill(orig, 500); + + /* Set individual bits */ + for (s = 0; s < 500; s += 10) + bitmap_clear(orig, s, 1); + + /* Set range of bits */ + bitmap_set(orig, 100, 50); + + for (wr = 0; wr < 500; wr++) { + DECLARE_BITMAP(tmp, 500); + + bitmap_fill(copy, 500); + s = wr; + + for_each_clear_bitrange_from(s, e, orig, 500) + bitmap_clear(copy, s, e - s); + + bitmap_copy(tmp, orig, 500); + bitmap_set(tmp, 0, wr); + expect_eq_bitmap(tmp, copy, 500); + } +} + struct test_bitmap_cut { unsigned int first; unsigned int cut; @@ -948,10 +1222,21 @@ static void __init selftest(void) test_bitmap_parselist(); test_bitmap_printlist(); test_mem_optimisations(); - test_for_each_set_clump8(); test_bitmap_cut(); test_bitmap_print_buf(); test_bitmap_const_eval(); + + test_find_nth_bit(); + test_for_each_set_bit(); + test_for_each_set_bit_from(); + test_for_each_clear_bit(); + test_for_each_clear_bit_from(); + test_for_each_set_bitrange(); + test_for_each_clear_bitrange(); + test_for_each_set_bitrange_from(); + test_for_each_clear_bitrange_from(); + test_for_each_set_clump8(); + test_for_each_set_bit_wrap(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/test_cpumask.c b/lib/test_cpumask.c deleted file mode 100644 index a31a1622f1f6e8b0fcd901ceb081a1a9445c6022..0000000000000000000000000000000000000000 --- a/lib/test_cpumask.c +++ /dev/null @@ -1,138 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * KUnit tests for cpumask. - * - * Author: Sander Vanheule - */ - -#include -#include -#include - -#define EXPECT_FOR_EACH_CPU_EQ(test, mask) \ - do { \ - const cpumask_t *m = (mask); \ - int mask_weight = cpumask_weight(m); \ - int cpu, iter = 0; \ - for_each_cpu(cpu, m) \ - iter++; \ - KUNIT_EXPECT_EQ((test), mask_weight, iter); \ - } while (0) - -#define EXPECT_FOR_EACH_CPU_NOT_EQ(test, mask) \ - do { \ - const cpumask_t *m = (mask); \ - int mask_weight = cpumask_weight(m); \ - int cpu, iter = 0; \ - for_each_cpu_not(cpu, m) \ - iter++; \ - KUNIT_EXPECT_EQ((test), nr_cpu_ids - mask_weight, iter); \ - } while (0) - -#define EXPECT_FOR_EACH_CPU_WRAP_EQ(test, mask) \ - do { \ - const cpumask_t *m = (mask); \ - int mask_weight = cpumask_weight(m); \ - int cpu, iter = 0; \ - for_each_cpu_wrap(cpu, m, nr_cpu_ids / 2) \ - iter++; \ - KUNIT_EXPECT_EQ((test), mask_weight, iter); \ - } while (0) - -#define EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, name) \ - do { \ - int mask_weight = num_##name##_cpus(); \ - int cpu, iter = 0; \ - for_each_##name##_cpu(cpu) \ - iter++; \ - KUNIT_EXPECT_EQ((test), mask_weight, iter); \ - } while (0) - -static cpumask_t mask_empty; -static cpumask_t mask_all; - -static void test_cpumask_weight(struct kunit *test) -{ - KUNIT_EXPECT_TRUE(test, cpumask_empty(&mask_empty)); - KUNIT_EXPECT_TRUE(test, cpumask_full(cpu_possible_mask)); - KUNIT_EXPECT_TRUE(test, cpumask_full(&mask_all)); - - KUNIT_EXPECT_EQ(test, 0, cpumask_weight(&mask_empty)); - KUNIT_EXPECT_EQ(test, nr_cpu_ids, cpumask_weight(cpu_possible_mask)); - KUNIT_EXPECT_EQ(test, nr_cpumask_bits, cpumask_weight(&mask_all)); -} - -static void test_cpumask_first(struct kunit *test) -{ - KUNIT_EXPECT_LE(test, nr_cpu_ids, cpumask_first(&mask_empty)); - KUNIT_EXPECT_EQ(test, 0, cpumask_first(cpu_possible_mask)); - - KUNIT_EXPECT_EQ(test, 0, cpumask_first_zero(&mask_empty)); - KUNIT_EXPECT_LE(test, nr_cpu_ids, cpumask_first_zero(cpu_possible_mask)); -} - -static void test_cpumask_last(struct kunit *test) -{ - KUNIT_EXPECT_LE(test, nr_cpumask_bits, cpumask_last(&mask_empty)); - KUNIT_EXPECT_EQ(test, nr_cpumask_bits - 1, cpumask_last(cpu_possible_mask)); -} - -static void test_cpumask_next(struct kunit *test) -{ - KUNIT_EXPECT_EQ(test, 0, cpumask_next_zero(-1, &mask_empty)); - KUNIT_EXPECT_LE(test, nr_cpu_ids, cpumask_next_zero(-1, cpu_possible_mask)); - - KUNIT_EXPECT_LE(test, nr_cpu_ids, cpumask_next(-1, &mask_empty)); - KUNIT_EXPECT_EQ(test, 0, cpumask_next(-1, cpu_possible_mask)); -} - -static void test_cpumask_iterators(struct kunit *test) -{ - EXPECT_FOR_EACH_CPU_EQ(test, &mask_empty); - EXPECT_FOR_EACH_CPU_NOT_EQ(test, &mask_empty); - EXPECT_FOR_EACH_CPU_WRAP_EQ(test, &mask_empty); - - EXPECT_FOR_EACH_CPU_EQ(test, cpu_possible_mask); - EXPECT_FOR_EACH_CPU_NOT_EQ(test, cpu_possible_mask); - EXPECT_FOR_EACH_CPU_WRAP_EQ(test, cpu_possible_mask); -} - -static void test_cpumask_iterators_builtin(struct kunit *test) -{ - EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, possible); - - /* Ensure the dynamic masks are stable while running the tests */ - cpu_hotplug_disable(); - - EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, online); - EXPECT_FOR_EACH_CPU_BUILTIN_EQ(test, present); - - cpu_hotplug_enable(); -} - -static int test_cpumask_init(struct kunit *test) -{ - cpumask_clear(&mask_empty); - cpumask_setall(&mask_all); - - return 0; -} - -static struct kunit_case test_cpumask_cases[] = { - KUNIT_CASE(test_cpumask_weight), - KUNIT_CASE(test_cpumask_first), - KUNIT_CASE(test_cpumask_last), - KUNIT_CASE(test_cpumask_next), - KUNIT_CASE(test_cpumask_iterators), - KUNIT_CASE(test_cpumask_iterators_builtin), - {} -}; - -static struct kunit_suite test_cpumask_suite = { - .name = "cpumask", - .init = test_cpumask_init, - .test_cases = test_cpumask_cases, -}; -kunit_test_suite(test_cpumask_suite); - -MODULE_LICENSE("GPL"); diff --git a/lib/test_dynamic_debug.c b/lib/test_dynamic_debug.c new file mode 100644 index 0000000000000000000000000000000000000000..8dd250ad022bb92f8c7cd80813498c5d31ddfbf3 --- /dev/null +++ b/lib/test_dynamic_debug.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel module for testing dynamic_debug + * + * Authors: + * Jim Cromie + */ + +#define pr_fmt(fmt) "test_dd: " fmt + +#include + +/* run tests by reading or writing sysfs node: do_prints */ + +static void do_prints(void); /* device under test */ +static int param_set_do_prints(const char *instr, const struct kernel_param *kp) +{ + do_prints(); + return 0; +} +static int param_get_do_prints(char *buffer, const struct kernel_param *kp) +{ + do_prints(); + return scnprintf(buffer, PAGE_SIZE, "did do_prints\n"); +} +static const struct kernel_param_ops param_ops_do_prints = { + .set = param_set_do_prints, + .get = param_get_do_prints, +}; +module_param_cb(do_prints, ¶m_ops_do_prints, NULL, 0600); + +/* + * Using the CLASSMAP api: + * - classmaps must have corresponding enum + * - enum symbols must match/correlate with class-name strings in the map. + * - base must equal enum's 1st value + * - multiple maps must set their base to share the 0-30 class_id space !! + * (build-bug-on tips welcome) + * Additionally, here: + * - tie together sysname, mapname, bitsname, flagsname + */ +#define DD_SYS_WRAP(_model, _flags) \ + static unsigned long bits_##_model; \ + static struct ddebug_class_param _flags##_model = { \ + .bits = &bits_##_model, \ + .flags = #_flags, \ + .map = &map_##_model, \ + }; \ + module_param_cb(_flags##_##_model, ¶m_ops_dyndbg_classes, &_flags##_model, 0600) + +/* numeric input, independent bits */ +enum cat_disjoint_bits { + D2_CORE = 0, + D2_DRIVER, + D2_KMS, + D2_PRIME, + D2_ATOMIC, + D2_VBL, + D2_STATE, + D2_LEASE, + D2_DP, + D2_DRMRES }; +DECLARE_DYNDBG_CLASSMAP(map_disjoint_bits, DD_CLASS_TYPE_DISJOINT_BITS, 0, + "D2_CORE", + "D2_DRIVER", + "D2_KMS", + "D2_PRIME", + "D2_ATOMIC", + "D2_VBL", + "D2_STATE", + "D2_LEASE", + "D2_DP", + "D2_DRMRES"); +DD_SYS_WRAP(disjoint_bits, p); +DD_SYS_WRAP(disjoint_bits, T); + +/* symbolic input, independent bits */ +enum cat_disjoint_names { LOW = 11, MID, HI }; +DECLARE_DYNDBG_CLASSMAP(map_disjoint_names, DD_CLASS_TYPE_DISJOINT_NAMES, 10, + "LOW", "MID", "HI"); +DD_SYS_WRAP(disjoint_names, p); +DD_SYS_WRAP(disjoint_names, T); + +/* numeric verbosity, V2 > V1 related */ +enum cat_level_num { V0 = 14, V1, V2, V3, V4, V5, V6, V7 }; +DECLARE_DYNDBG_CLASSMAP(map_level_num, DD_CLASS_TYPE_LEVEL_NUM, 14, + "V0", "V1", "V2", "V3", "V4", "V5", "V6", "V7"); +DD_SYS_WRAP(level_num, p); +DD_SYS_WRAP(level_num, T); + +/* symbolic verbosity */ +enum cat_level_names { L0 = 22, L1, L2, L3, L4, L5, L6, L7 }; +DECLARE_DYNDBG_CLASSMAP(map_level_names, DD_CLASS_TYPE_LEVEL_NAMES, 22, + "L0", "L1", "L2", "L3", "L4", "L5", "L6", "L7"); +DD_SYS_WRAP(level_names, p); +DD_SYS_WRAP(level_names, T); + +/* stand-in for all pr_debug etc */ +#define prdbg(SYM) __pr_debug_cls(SYM, #SYM " msg\n") + +static void do_cats(void) +{ + pr_debug("doing categories\n"); + + prdbg(LOW); + prdbg(MID); + prdbg(HI); + + prdbg(D2_CORE); + prdbg(D2_DRIVER); + prdbg(D2_KMS); + prdbg(D2_PRIME); + prdbg(D2_ATOMIC); + prdbg(D2_VBL); + prdbg(D2_STATE); + prdbg(D2_LEASE); + prdbg(D2_DP); + prdbg(D2_DRMRES); +} + +static void do_levels(void) +{ + pr_debug("doing levels\n"); + + prdbg(V1); + prdbg(V2); + prdbg(V3); + prdbg(V4); + prdbg(V5); + prdbg(V6); + prdbg(V7); + + prdbg(L1); + prdbg(L2); + prdbg(L3); + prdbg(L4); + prdbg(L5); + prdbg(L6); + prdbg(L7); +} + +static void do_prints(void) +{ + do_cats(); + do_levels(); +} + +static int __init test_dynamic_debug_init(void) +{ + pr_debug("init start\n"); + do_prints(); + pr_debug("init done\n"); + return 0; +} + +static void __exit test_dynamic_debug_exit(void) +{ + pr_debug("exited\n"); +} + +module_init(test_dynamic_debug_init); +module_exit(test_dynamic_debug_exit); + +MODULE_AUTHOR("Jim Cromie "); +MODULE_LICENSE("GPL"); diff --git a/lib/test_fprobe.c b/lib/test_fprobe.c index ed70637a2ffa4c20a6254fd6448a6fbc698cace3..e0381b3ec410c181fb72fa120f6b107f0250bb50 100644 --- a/lib/test_fprobe.c +++ b/lib/test_fprobe.c @@ -145,7 +145,7 @@ static unsigned long get_ftrace_location(void *func) static int fprobe_test_init(struct kunit *test) { do { - rand1 = prandom_u32(); + rand1 = get_random_u32(); } while (rand1 <= div_factor); target = fprobe_selftest_target; diff --git a/lib/test_hexdump.c b/lib/test_hexdump.c index 5144899d3c6b8b235d33c8a988f3a413fa34aff8..0927f44cd4787f565140c9e75c92c87b0b026d76 100644 --- a/lib/test_hexdump.c +++ b/lib/test_hexdump.c @@ -149,7 +149,7 @@ static void __init test_hexdump(size_t len, int rowsize, int groupsize, static void __init test_hexdump_set(int rowsize, bool ascii) { size_t d = min_t(size_t, sizeof(data_b), rowsize); - size_t len = get_random_int() % d + 1; + size_t len = prandom_u32_max(d) + 1; test_hexdump(len, rowsize, 4, ascii); test_hexdump(len, rowsize, 2, ascii); @@ -208,11 +208,11 @@ static void __init test_hexdump_overflow(size_t buflen, size_t len, static void __init test_hexdump_overflow_set(size_t buflen, bool ascii) { unsigned int i = 0; - int rs = (get_random_int() % 2 + 1) * 16; + int rs = (prandom_u32_max(2) + 1) * 16; do { int gs = 1 << i; - size_t len = get_random_int() % rs + gs; + size_t len = prandom_u32_max(rs) + gs; test_hexdump_overflow(buflen, rounddown(len, gs), rs, gs, ascii); } while (i++ < 3); @@ -223,11 +223,11 @@ static int __init test_hexdump_init(void) unsigned int i; int rowsize; - rowsize = (get_random_int() % 2 + 1) * 16; + rowsize = (prandom_u32_max(2) + 1) * 16; for (i = 0; i < 16; i++) test_hexdump_set(rowsize, false); - rowsize = (get_random_int() % 2 + 1) * 16; + rowsize = (prandom_u32_max(2) + 1) * 16; for (i = 0; i < 16; i++) test_hexdump_set(rowsize, true); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index e3965cafd27cf8ac389187f8b740ae479c7387ac..67e6f83fe0f82ad0f8d73461290b83262493f9dc 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -100,6 +100,7 @@ struct dmirror { struct dmirror_chunk { struct dev_pagemap pagemap; struct dmirror_device *mdevice; + bool remove; }; /* @@ -107,8 +108,8 @@ struct dmirror_chunk { */ struct dmirror_device { struct cdev cdevice; - struct hmm_devmem *devmem; unsigned int zone_device_type; + struct device device; unsigned int devmem_capacity; unsigned int devmem_count; @@ -192,11 +193,15 @@ static int dmirror_fops_release(struct inode *inode, struct file *filp) return 0; } +static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page) +{ + return container_of(page->pgmap, struct dmirror_chunk, pagemap); +} + static struct dmirror_device *dmirror_page_to_device(struct page *page) { - return container_of(page->pgmap, struct dmirror_chunk, - pagemap)->mdevice; + return dmirror_page_to_chunk(page)->mdevice; } static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range) @@ -627,8 +632,8 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) goto error; } + zone_device_page_init(dpage); dpage->zone_device_data = rpage; - lock_page(dpage); return dpage; error: @@ -907,7 +912,7 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, struct vm_area_struct *vma; unsigned long src_pfns[64] = { 0 }; unsigned long dst_pfns[64] = { 0 }; - struct migrate_vma args; + struct migrate_vma args = { 0 }; unsigned long next; int ret; @@ -968,7 +973,7 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror, unsigned long src_pfns[64] = { 0 }; unsigned long dst_pfns[64] = { 0 }; struct dmirror_bounce bounce; - struct migrate_vma args; + struct migrate_vma args = { 0 }; unsigned long next; int ret; @@ -1218,6 +1223,85 @@ static int dmirror_snapshot(struct dmirror *dmirror, return ret; } +static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) +{ + unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; + unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; + unsigned long npages = end_pfn - start_pfn + 1; + unsigned long i; + unsigned long *src_pfns; + unsigned long *dst_pfns; + + src_pfns = kcalloc(npages, sizeof(*src_pfns), GFP_KERNEL); + dst_pfns = kcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL); + + migrate_device_range(src_pfns, start_pfn, npages); + for (i = 0; i < npages; i++) { + struct page *dpage, *spage; + + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + + if (WARN_ON(!is_device_private_page(spage) && + !is_device_coherent_page(spage))) + continue; + spage = BACKING_PAGE(spage); + dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); + lock_page(dpage); + copy_highpage(dpage, spage); + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); + if (src_pfns[i] & MIGRATE_PFN_WRITE) + dst_pfns[i] |= MIGRATE_PFN_WRITE; + } + migrate_device_pages(src_pfns, dst_pfns, npages); + migrate_device_finalize(src_pfns, dst_pfns, npages); + kfree(src_pfns); + kfree(dst_pfns); +} + +/* Removes free pages from the free list so they can't be re-allocated */ +static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) +{ + struct dmirror_device *mdevice = devmem->mdevice; + struct page *page; + + for (page = mdevice->free_pages; page; page = page->zone_device_data) + if (dmirror_page_to_chunk(page) == devmem) + mdevice->free_pages = page->zone_device_data; +} + +static void dmirror_device_remove_chunks(struct dmirror_device *mdevice) +{ + unsigned int i; + + mutex_lock(&mdevice->devmem_lock); + if (mdevice->devmem_chunks) { + for (i = 0; i < mdevice->devmem_count; i++) { + struct dmirror_chunk *devmem = + mdevice->devmem_chunks[i]; + + spin_lock(&mdevice->lock); + devmem->remove = true; + dmirror_remove_free_pages(devmem); + spin_unlock(&mdevice->lock); + + dmirror_device_evict_chunk(devmem); + memunmap_pages(&devmem->pagemap); + if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) + release_mem_region(devmem->pagemap.range.start, + range_len(&devmem->pagemap.range)); + kfree(devmem); + } + mdevice->devmem_count = 0; + mdevice->devmem_capacity = 0; + mdevice->free_pages = NULL; + kfree(mdevice->devmem_chunks); + mdevice->devmem_chunks = NULL; + } + mutex_unlock(&mdevice->devmem_lock); +} + static long dmirror_fops_unlocked_ioctl(struct file *filp, unsigned int command, unsigned long arg) @@ -1272,6 +1356,11 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, ret = dmirror_snapshot(dmirror, &cmd); break; + case HMM_DMIRROR_RELEASE: + dmirror_device_remove_chunks(dmirror->mdevice); + ret = 0; + break; + default: return -EINVAL; } @@ -1326,15 +1415,19 @@ static void dmirror_devmem_free(struct page *page) mdevice = dmirror_page_to_device(page); spin_lock(&mdevice->lock); - mdevice->cfree++; - page->zone_device_data = mdevice->free_pages; - mdevice->free_pages = page; + + /* Return page to our allocator if not freeing the chunk */ + if (!dmirror_page_to_chunk(page)->remove) { + mdevice->cfree++; + page->zone_device_data = mdevice->free_pages; + mdevice->free_pages = page; + } spin_unlock(&mdevice->lock); } static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) { - struct migrate_vma args; + struct migrate_vma args = { 0 }; unsigned long src_pfns = 0; unsigned long dst_pfns = 0; struct page *rpage; @@ -1357,6 +1450,7 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) args.dst = &dst_pfns; args.pgmap_owner = dmirror->mdevice; args.flags = dmirror_select_device(dmirror); + args.fault_page = vmf->page; if (migrate_vma_setup(&args)) return VM_FAULT_SIGBUS; @@ -1390,7 +1484,14 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id) cdev_init(&mdevice->cdevice, &dmirror_fops); mdevice->cdevice.owner = THIS_MODULE; - ret = cdev_add(&mdevice->cdevice, dev, 1); + device_initialize(&mdevice->device); + mdevice->device.devt = dev; + + ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id); + if (ret) + return ret; + + ret = cdev_device_add(&mdevice->cdevice, &mdevice->device); if (ret) return ret; @@ -1400,23 +1501,8 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id) static void dmirror_device_remove(struct dmirror_device *mdevice) { - unsigned int i; - - if (mdevice->devmem_chunks) { - for (i = 0; i < mdevice->devmem_count; i++) { - struct dmirror_chunk *devmem = - mdevice->devmem_chunks[i]; - - memunmap_pages(&devmem->pagemap); - if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE) - release_mem_region(devmem->pagemap.range.start, - range_len(&devmem->pagemap.range)); - kfree(devmem); - } - kfree(mdevice->devmem_chunks); - } - - cdev_del(&mdevice->cdevice); + dmirror_device_remove_chunks(mdevice); + cdev_device_del(&mdevice->cdevice, &mdevice->device); } static int __init hmm_dmirror_init(void) diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index e31d58c9034a764d71a922bdf175c3e03317e45e..8c818a2cf4f69a93317842bfa4aa11bb45626805 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -36,6 +36,7 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x04, struct hmm_dmirror_cmd) #define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd) #define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd) /* * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT. diff --git a/lib/test_kprobes.c b/lib/test_kprobes.c index a5edc2ebc947a6ba770065dc6e8dbe8e2688f7c2..eeb1d728d974679f675100867caff6ef5045186b 100644 --- a/lib/test_kprobes.c +++ b/lib/test_kprobes.c @@ -341,7 +341,7 @@ static int kprobes_test_init(struct kunit *test) stacktrace_driver = kprobe_stacktrace_driver; do { - rand1 = prandom_u32(); + rand1 = get_random_u32(); } while (rand1 <= div_factor); return 0; } diff --git a/lib/test_list_sort.c b/lib/test_list_sort.c index ade7a1ea0c8e25006af3772f91b793602aa4d605..19ff229b9c3a7838a9a216e71bd9fe866c53da2f 100644 --- a/lib/test_list_sort.c +++ b/lib/test_list_sort.c @@ -71,7 +71,7 @@ static void list_sort_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, el); /* force some equivalencies */ - el->value = prandom_u32() % (TEST_LIST_LEN / 3); + el->value = prandom_u32_max(TEST_LIST_LEN / 3); el->serial = i; el->poison1 = TEST_POISON1; el->poison2 = TEST_POISON2; diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c new file mode 100644 index 0000000000000000000000000000000000000000..4f69e009a015d4e8e6e8f6dbe5034d4228fc08be --- /dev/null +++ b/lib/test_maple_tree.c @@ -0,0 +1,38307 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * test_maple_tree.c: Test the maple tree API + * Copyright (c) 2018 Liam R. Howlett + * Author: Liam R. Howlett + */ + +#include +#include +#include +#include + +#define MTREE_ALLOC_MAX 0x2000000000000Ul +#define CONFIG_DEBUG_MAPLE_TREE +#define CONFIG_MAPLE_SEARCH +/* #define BENCH_SLOT_STORE */ +/* #define BENCH_NODE_STORE */ +/* #define BENCH_AWALK */ +/* #define BENCH_WALK */ +/* #define BENCH_MT_FOR_EACH */ +/* #define BENCH_FORK */ +static +int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) +{ + return mtree_insert(mt, index, xa_mk_value(index & LONG_MAX), gfp); +} + +static void mtree_erase_index(struct maple_tree *mt, unsigned long index) +{ + MT_BUG_ON(mt, mtree_erase(mt, index) != xa_mk_value(index & LONG_MAX)); + MT_BUG_ON(mt, mtree_load(mt, index) != NULL); +} + +static int mtree_test_insert(struct maple_tree *mt, unsigned long index, + void *ptr) +{ + return mtree_insert(mt, index, ptr, GFP_KERNEL); +} + +static int mtree_test_store_range(struct maple_tree *mt, unsigned long start, + unsigned long end, void *ptr) +{ + return mtree_store_range(mt, start, end, ptr, GFP_KERNEL); +} + +static int mtree_test_store(struct maple_tree *mt, unsigned long start, + void *ptr) +{ + return mtree_test_store_range(mt, start, start, ptr); +} + +static int mtree_test_insert_range(struct maple_tree *mt, unsigned long start, + unsigned long end, void *ptr) +{ + return mtree_insert_range(mt, start, end, ptr, GFP_KERNEL); +} + +static void *mtree_test_load(struct maple_tree *mt, unsigned long index) +{ + return mtree_load(mt, index); +} + +static void *mtree_test_erase(struct maple_tree *mt, unsigned long index) +{ + return mtree_erase(mt, index); +} + +static noinline void check_mtree_alloc_range(struct maple_tree *mt, + unsigned long start, unsigned long end, unsigned long size, + unsigned long expected, int eret, void *ptr) +{ + + unsigned long result = expected + 1; + int ret; + + ret = mtree_alloc_range(mt, &result, ptr, size, start, end, + GFP_KERNEL); + MT_BUG_ON(mt, ret != eret); + if (ret) + return; + + MT_BUG_ON(mt, result != expected); +} + +static noinline void check_mtree_alloc_rrange(struct maple_tree *mt, + unsigned long start, unsigned long end, unsigned long size, + unsigned long expected, int eret, void *ptr) +{ + + unsigned long result = expected + 1; + int ret; + + ret = mtree_alloc_rrange(mt, &result, ptr, size, start, end - 1, + GFP_KERNEL); + MT_BUG_ON(mt, ret != eret); + if (ret) + return; + + MT_BUG_ON(mt, result != expected); +} + +static noinline void check_load(struct maple_tree *mt, unsigned long index, + void *ptr) +{ + void *ret = mtree_test_load(mt, index); + + if (ret != ptr) + pr_err("Load %lu returned %p expect %p\n", index, ret, ptr); + MT_BUG_ON(mt, ret != ptr); +} + +static noinline void check_store_range(struct maple_tree *mt, + unsigned long start, unsigned long end, void *ptr, int expected) +{ + int ret = -EINVAL; + unsigned long i; + + ret = mtree_test_store_range(mt, start, end, ptr); + MT_BUG_ON(mt, ret != expected); + + if (ret) + return; + + for (i = start; i <= end; i++) + check_load(mt, i, ptr); +} + +static noinline void check_insert_range(struct maple_tree *mt, + unsigned long start, unsigned long end, void *ptr, int expected) +{ + int ret = -EINVAL; + unsigned long i; + + ret = mtree_test_insert_range(mt, start, end, ptr); + MT_BUG_ON(mt, ret != expected); + + if (ret) + return; + + for (i = start; i <= end; i++) + check_load(mt, i, ptr); +} + +static noinline void check_insert(struct maple_tree *mt, unsigned long index, + void *ptr) +{ + int ret = -EINVAL; + + ret = mtree_test_insert(mt, index, ptr); + MT_BUG_ON(mt, ret != 0); +} + +static noinline void check_erase(struct maple_tree *mt, unsigned long index, + void *ptr) +{ + MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); +} + +static noinline void check_dup_insert(struct maple_tree *mt, + unsigned long index, void *ptr) +{ + int ret = -EINVAL; + + ret = mtree_test_insert(mt, index, ptr); + MT_BUG_ON(mt, ret != -EEXIST); +} + + +static noinline +void check_index_load(struct maple_tree *mt, unsigned long index) +{ + return check_load(mt, index, xa_mk_value(index & LONG_MAX)); +} + +static noinline void check_nomem(struct maple_tree *mt) +{ + MA_STATE(ms, mt, 1, 1); + + MT_BUG_ON(mt, !mtree_empty(mt)); + /* Ensure no bypassing of allocation failures */ + mt_set_non_kernel(0); + + /* Storing something at 1 requires memory allocation */ + MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); + /* Storing something at 0 does not */ + MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); + + /* + * Simulate two threads racing; the first one fails to allocate + * memory to insert an entry at 1, then the second one succeeds + * in allocating memory to insert an entry at 2. The first one + * then needs to free the node it allocated. LeakSanitizer will + * notice this, as will the 'nr_allocated' debugging aid in the + * userspace test suite. + */ + mtree_lock(mt); + mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ + MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); + mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ + MT_BUG_ON(mt, ms.node != MAS_START); + mtree_unlock(mt); + MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); + mtree_lock(mt); + mas_store(&ms, &ms); /* insert 1 -> &ms */ + mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ + mtree_unlock(mt); + mtree_destroy(mt); +} + +static inline int not_empty(struct maple_node *node) +{ + int i; + + if (node->parent) + return 1; + + for (i = 0; i < ARRAY_SIZE(node->slot); i++) + if (node->slot[i]) + return 1; + + return 0; +} + +static noinline void check_new_node(struct maple_tree *mt) +{ + + struct maple_node *mn, *mn2, *mn3; + struct maple_alloc *smn; + struct maple_node *nodes[100]; + int i, j, total; + + MA_STATE(mas, mt, 0, 0); + + /* Try allocating 3 nodes */ + mtree_lock(mt); + /* request 3 nodes to be allocated. */ + mas_node_count(&mas, 3); + /* Allocation request of 3. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); + /* Allocate failed. */ + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + mas_push_node(&mas, mn); + mas_nomem(&mas, GFP_KERNEL); /* free */ + mtree_unlock(mt); + + + /* Try allocating 1 node, then 2 more */ + mtree_lock(mt); + /* Set allocation request to 1. */ + mas_set_alloc_req(&mas, 1); + /* Check Allocation request of 1. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + mas_set_err(&mas, -ENOMEM); + /* Validate allocation request. */ + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + /* Eat the requested node. */ + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mn->slot[0] != NULL); + MT_BUG_ON(mt, mn->slot[1] != NULL); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + ma_free_rcu(mn); + mas.node = MAS_START; + mas_nomem(&mas, GFP_KERNEL); + /* Allocate 3 nodes, will fail. */ + mas_node_count(&mas, 3); + /* Drop the lock and allocate 3 nodes. */ + mas_nomem(&mas, GFP_KERNEL); + /* Ensure 3 are allocated. */ + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + /* Allocation request of 0. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); + + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); + /* Ensure we counted 3. */ + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + /* Free. */ + mas_nomem(&mas, GFP_KERNEL); + + /* Set allocation request to 1. */ + mas_set_alloc_req(&mas, 1); + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + mas_set_err(&mas, -ENOMEM); + /* Validate allocation request. */ + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != 1); + /* Check the node is only one node. */ + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mn->slot[0] != NULL); + MT_BUG_ON(mt, mn->slot[1] != NULL); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != 1); + MT_BUG_ON(mt, mas.alloc->node_count); + + mas_set_alloc_req(&mas, 2); /* request 2 more. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); + for (i = 2; i >= 0; i--) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != i); + MT_BUG_ON(mt, !mn); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + + total = 64; + mas_set_alloc_req(&mas, total); /* request 2 more. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != total); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (i = total; i > 0; i--) { + unsigned int e = 0; /* expected node_count */ + + if (i >= 35) + e = i - 35; + else if (i >= 5) + e = i - 5; + else if (i >= 2) + e = i - 2; + MT_BUG_ON(mt, mas.alloc->node_count != e); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); + MT_BUG_ON(mt, !mn); + ma_free_rcu(mn); + } + + total = 100; + for (i = 1; i < total; i++) { + mas_set_alloc_req(&mas, i); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (j = i; j > 0; j--) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); + MT_BUG_ON(mt, !mn); + MT_BUG_ON(mt, not_empty(mn)); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != j); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + mas_set_alloc_req(&mas, i); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (j = 0; j <= i/2; j++) { + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + nodes[j] = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); + } + + while (j) { + j--; + mas_push_node(&mas, nodes[j]); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + } + MT_BUG_ON(mt, mas_allocated(&mas) != i); + for (j = 0; j <= i/2; j++) { + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); + } + MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); + + } + + /* Set allocation request. */ + total = 500; + mas_node_count(&mas, total); + /* Drop the lock and allocate the nodes. */ + mas_nomem(&mas, GFP_KERNEL); + MT_BUG_ON(mt, !mas.alloc); + i = 1; + smn = mas.alloc; + while (i < total) { + for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { + i++; + MT_BUG_ON(mt, !smn->slot[j]); + if (i == total) + break; + } + smn = smn->slot[0]; /* next. */ + } + MT_BUG_ON(mt, mas_allocated(&mas) != total); + mas_nomem(&mas, GFP_KERNEL); /* Free. */ + + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + for (i = 1; i < 128; i++) { + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ + for (j = i; j > 0; j--) { /*Free the requests */ + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + } + + for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { + MA_STATE(mas2, mt, 0, 0); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ + for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + mas_push_node(&mas2, mn); + MT_BUG_ON(mt, mas_allocated(&mas2) != j); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mas_allocated(&mas2) != i); + + for (j = i; j > 0; j--) { /*Free the requests */ + MT_BUG_ON(mt, mas_allocated(&mas2) != j); + mn = mas_pop_node(&mas2); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas2) != 0); + } + + + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); + + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + + /* Check the limit of pop/push/pop */ + mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_alloc_req(&mas)); + MT_BUG_ON(mt, mas.alloc->node_count); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas.alloc->node_count); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + + for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put it back */ + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mn2 = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put them back */ + mas_push_node(&mas, mn2); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mn2 = mas_pop_node(&mas); /* get the next node. */ + mn3 = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put them back */ + mas_push_node(&mas, mn2); + mas_push_node(&mas, mn3); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mas_destroy(&mas); + } + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 5); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != 5); + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 10); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mas.node = MAS_START; + MT_BUG_ON(mt, mas_allocated(&mas) != 10); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mas.node = MAS_START; + MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); + mas_destroy(&mas); + + mtree_unlock(mt); +} + +static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, + bool verbose) +{ + unsigned long i = max, j; + + MT_BUG_ON(mt, !mtree_empty(mt)); + + mt_zero_nr_tallocated(); + while (i) { + MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); + for (j = i; j <= max; j++) + check_index_load(mt, j); + + check_load(mt, i - 1, NULL); + mt_set_in_rcu(mt); + MT_BUG_ON(mt, !mt_height(mt)); + mt_clear_in_rcu(mt); + MT_BUG_ON(mt, !mt_height(mt)); + i--; + } + check_load(mt, max + 1, NULL); + + if (verbose) { + rcu_barrier(); + mt_dump(mt); + pr_info(" %s test of 0-%lu %luK in %d active (%d total)\n", + __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(), + mt_nr_tallocated()); + } +} + +static noinline void check_seq(struct maple_tree *mt, unsigned long max, + bool verbose) +{ + unsigned long i, j; + + MT_BUG_ON(mt, !mtree_empty(mt)); + + mt_zero_nr_tallocated(); + for (i = 0; i <= max; i++) { + MT_BUG_ON(mt, mtree_insert_index(mt, i, GFP_KERNEL)); + for (j = 0; j <= i; j++) + check_index_load(mt, j); + + if (i) + MT_BUG_ON(mt, !mt_height(mt)); + check_load(mt, i + 1, NULL); + } + if (verbose) { + rcu_barrier(); + mt_dump(mt); + pr_info(" seq test of 0-%lu %luK in %d active (%d total)\n", + max, mt_get_alloc_size()/1024, mt_nr_allocated(), + mt_nr_tallocated()); + } +} + +static noinline void check_lb_not_empty(struct maple_tree *mt) +{ + unsigned long i, j; + unsigned long huge = 4000UL * 1000 * 1000; + + + i = huge; + while (i > 4096) { + check_insert(mt, i, (void *) i); + for (j = huge; j >= i; j /= 2) { + check_load(mt, j-1, NULL); + check_load(mt, j, (void *) j); + check_load(mt, j+1, NULL); + } + i /= 2; + } + mtree_destroy(mt); +} + +static noinline void check_lower_bound_split(struct maple_tree *mt) +{ + MT_BUG_ON(mt, !mtree_empty(mt)); + check_lb_not_empty(mt); +} + +static noinline void check_upper_bound_split(struct maple_tree *mt) +{ + unsigned long i, j; + unsigned long huge = 4000UL * 1000 * 1000; + + MT_BUG_ON(mt, !mtree_empty(mt)); + + i = 4096; + while (i < huge) { + check_insert(mt, i, (void *) i); + for (j = i; j >= huge; j *= 2) { + check_load(mt, j-1, NULL); + check_load(mt, j, (void *) j); + check_load(mt, j+1, NULL); + } + i *= 2; + } + mtree_destroy(mt); +} + +static noinline void check_mid_split(struct maple_tree *mt) +{ + unsigned long huge = 8000UL * 1000 * 1000; + + check_insert(mt, huge, (void *) huge); + check_insert(mt, 0, xa_mk_value(0)); + check_lb_not_empty(mt); +} + +static noinline void check_rev_find(struct maple_tree *mt) +{ + int i, nr_entries = 200; + void *val; + MA_STATE(mas, mt, 0, 0); + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + mas_set(&mas, 1000); + val = mas_find_rev(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(100)); + val = mas_find_rev(&mas, 1000); + MT_BUG_ON(mt, val != NULL); + + mas_set(&mas, 999); + val = mas_find_rev(&mas, 997); + MT_BUG_ON(mt, val != NULL); + + mas_set(&mas, 1000); + val = mas_find_rev(&mas, 900); + MT_BUG_ON(mt, val != xa_mk_value(100)); + val = mas_find_rev(&mas, 900); + MT_BUG_ON(mt, val != xa_mk_value(99)); + + mas_set(&mas, 20); + val = mas_find_rev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(2)); + val = mas_find_rev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(1)); + val = mas_find_rev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(0)); + val = mas_find_rev(&mas, 0); + MT_BUG_ON(mt, val != NULL); +} + +static noinline void check_find(struct maple_tree *mt) +{ + unsigned long val = 0; + unsigned long count = 20; + unsigned long max; + unsigned long last = 0, index = 0; + void *entry, *entry2; + + MA_STATE(mas, mt, 0, 0); + + /* Insert 0. */ + MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL)); + + for (int i = 0; i <= count; i++) { + if (val != 64) + MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL)); + else + MT_BUG_ON(mt, mtree_insert(mt, val, + XA_ZERO_ENTRY, GFP_KERNEL)); + + val <<= 2; + } + + val = 0; + mas_set(&mas, val); + mas_lock(&mas); + while ((entry = mas_find(&mas, 268435456)) != NULL) { + if (val != 64) + MT_BUG_ON(mt, xa_mk_value(val) != entry); + else + MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); + + val <<= 2; + /* For zero check. */ + if (!val) + val = 1; + } + mas_unlock(&mas); + + val = 0; + mas_set(&mas, val); + mas_lock(&mas); + mas_for_each(&mas, entry, ULONG_MAX) { + if (val != 64) + MT_BUG_ON(mt, xa_mk_value(val) != entry); + else + MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); + val <<= 2; + /* For zero check. */ + if (!val) + val = 1; + } + mas_unlock(&mas); + + /* Test mas_pause */ + val = 0; + mas_set(&mas, val); + mas_lock(&mas); + mas_for_each(&mas, entry, ULONG_MAX) { + if (val != 64) + MT_BUG_ON(mt, xa_mk_value(val) != entry); + else + MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); + val <<= 2; + /* For zero check. */ + if (!val) + val = 1; + + mas_pause(&mas); + mas_unlock(&mas); + mas_lock(&mas); + } + mas_unlock(&mas); + + val = 0; + max = 300; /* A value big enough to include XA_ZERO_ENTRY at 64. */ + mt_for_each(mt, entry, index, max) { + MT_BUG_ON(mt, xa_mk_value(val) != entry); + val <<= 2; + if (val == 64) /* Skip zero entry. */ + val <<= 2; + /* For zero check. */ + if (!val) + val = 1; + } + + val = 0; + max = 0; + index = 0; + MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); + mt_for_each(mt, entry, index, ULONG_MAX) { + if (val == 4398046511104) + MT_BUG_ON(mt, entry != + xa_mk_value(ULONG_MAX & LONG_MAX)); + else + MT_BUG_ON(mt, xa_mk_value(val) != entry); + val <<= 2; + if (val == 64) /* Skip zero entry. */ + val <<= 2; + /* For zero check. */ + if (!val) + val = 1; + max++; + MT_BUG_ON(mt, max > 25); + } + mtree_erase_index(mt, ULONG_MAX); + + mas_reset(&mas); + index = 17; + entry = mt_find(mt, &index, 512); + MT_BUG_ON(mt, xa_mk_value(256) != entry); + + mas_reset(&mas); + index = 17; + entry = mt_find(mt, &index, 20); + MT_BUG_ON(mt, entry != NULL); + + + /* Range check.. */ + /* Insert ULONG_MAX */ + MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); + + val = 0; + mas_set(&mas, 0); + mas_lock(&mas); + mas_for_each(&mas, entry, ULONG_MAX) { + if (val == 64) + MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); + else if (val == 4398046511104) + MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX)); + else + MT_BUG_ON(mt, xa_mk_value(val) != entry); + val <<= 2; + + /* For zero check. */ + if (!val) + val = 1; + mas_pause(&mas); + mas_unlock(&mas); + mas_lock(&mas); + } + mas_unlock(&mas); + + mas_set(&mas, 1048576); + mas_lock(&mas); + entry = mas_find(&mas, 1048576); + mas_unlock(&mas); + MT_BUG_ON(mas.tree, entry == NULL); + + /* + * Find last value. + * 1. get the expected value, leveraging the existence of an end entry + * 2. delete end entry + * 3. find the last value but searching for ULONG_MAX and then using + * prev + */ + /* First, get the expected result. */ + mas_lock(&mas); + mas_reset(&mas); + mas.index = ULONG_MAX; /* start at max.. */ + entry = mas_find(&mas, ULONG_MAX); + entry = mas_prev(&mas, 0); + index = mas.index; + last = mas.last; + + /* Erase the last entry. */ + mas_reset(&mas); + mas.index = ULONG_MAX; + mas.last = ULONG_MAX; + mas_erase(&mas); + + /* Get the previous value from MAS_START */ + mas_reset(&mas); + entry2 = mas_prev(&mas, 0); + + /* Check results. */ + MT_BUG_ON(mt, entry != entry2); + MT_BUG_ON(mt, index != mas.index); + MT_BUG_ON(mt, last != mas.last); + + + mas.node = MAS_NONE; + mas.index = ULONG_MAX; + mas.last = ULONG_MAX; + entry2 = mas_prev(&mas, 0); + MT_BUG_ON(mt, entry != entry2); + + mas_set(&mas, 0); + MT_BUG_ON(mt, mas_prev(&mas, 0) != NULL); + + mas_unlock(&mas); + mtree_destroy(mt); +} + +static noinline void check_find_2(struct maple_tree *mt) +{ + unsigned long i, j; + void *entry; + + MA_STATE(mas, mt, 0, 0); + rcu_read_lock(); + mas_for_each(&mas, entry, ULONG_MAX) + MT_BUG_ON(mt, true); + rcu_read_unlock(); + + for (i = 0; i < 256; i++) { + mtree_insert_index(mt, i, GFP_KERNEL); + j = 0; + mas_set(&mas, 0); + rcu_read_lock(); + mas_for_each(&mas, entry, ULONG_MAX) { + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j++; + } + rcu_read_unlock(); + MT_BUG_ON(mt, j != i + 1); + } + + for (i = 0; i < 256; i++) { + mtree_erase_index(mt, i); + j = i + 1; + mas_set(&mas, 0); + rcu_read_lock(); + mas_for_each(&mas, entry, ULONG_MAX) { + if (xa_is_zero(entry)) + continue; + + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j++; + } + rcu_read_unlock(); + MT_BUG_ON(mt, j != 256); + } + + /*MT_BUG_ON(mt, !mtree_empty(mt)); */ +} + +#define erase_ptr(i) entry[i%2] +#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) +#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) +#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) + +static noinline void check_erase_testset(struct maple_tree *mt) +{ + unsigned long set[] = { 5015, 5014, 5017, 25, 1000, + 1001, 1002, 1003, 1005, 0, + 6003, 6002, 6008, 6012, 6015, + 7003, 7002, 7008, 7012, 7015, + 8003, 8002, 8008, 8012, 8015, + 9003, 9002, 9008, 9012, 9015, + 10003, 10002, 10008, 10012, 10015, + 11003, 11002, 11008, 11012, 11015, + 12003, 12002, 12008, 12012, 12015, + 13003, 13002, 13008, 13012, 13015, + 14003, 14002, 14008, 14012, 14015, + 15003, 15002, 15008, 15012, 15015, + }; + + + void *ptr = &set; + void *entry[2] = { ptr, mt }; + void *root_node; + + + rcu_register_thread(); + mt_set_in_rcu(mt); + for (int i = 0; i < 4; i++) + erase_check_insert(mt, i); + for (int i = 0; i < 4; i++) + erase_check_load(mt, i); + + mt_set_non_kernel(2); + erase_check_erase(mt, 1); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + for (int i = 2; i < 4; i++) + erase_check_load(mt, i); + + + erase_check_erase(mt, 2); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + check_load(mt, set[2], NULL); + + erase_check_insert(mt, 1); + erase_check_insert(mt, 2); + + for (int i = 0; i < 4; i++) + erase_check_load(mt, i); + + /* Check erase and load without an allocation. */ + erase_check_load(mt, 3); + erase_check_erase(mt, 1); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + for (int i = 2; i < 4; i++) + erase_check_load(mt, i); + + /* + * Set the newly erased node. This will produce a different allocated + * node to avoid busy slots. + */ + root_node = mt->ma_root; + erase_check_insert(mt, 1); + + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + erase_check_load(mt, 2); + check_load(mt, 5018, NULL); + erase_check_load(mt, 3); + + erase_check_erase(mt, 2); /* erase 5017 to check append */ + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + check_load(mt, set[2], NULL); + check_load(mt, 5018, NULL); + + erase_check_load(mt, 3); + + root_node = mt->ma_root; + erase_check_insert(mt, 2); + + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + erase_check_load(mt, 2); + check_load(mt, 5018, NULL); + erase_check_load(mt, 3); + + mt_set_non_kernel(1); + erase_check_erase(mt, 2); /* erase 5017 to check append */ + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + check_load(mt, set[2], NULL); + erase_check_erase(mt, 0); /* erase 5015 to check append */ + check_load(mt, set[0], NULL); + check_load(mt, 5016, NULL); + erase_check_insert(mt, 4); /* 1000 < Should not split. */ + check_load(mt, set[0], NULL); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + check_load(mt, set[2], NULL); + check_load(mt, 5018, NULL); + erase_check_load(mt, 4); + check_load(mt, 999, NULL); + check_load(mt, 1001, NULL); + erase_check_load(mt, 4); + if (mt_in_rcu(mt)) + MT_BUG_ON(mt, root_node == mt->ma_root); + else + MT_BUG_ON(mt, root_node != mt->ma_root); + + /* Should not have split. */ + MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); + + + /* Coalesce testing */ + erase_check_insert(mt, 0); + erase_check_insert(mt, 2); + + for (int i = 5; i < 25; i++) { + erase_check_insert(mt, i); + for (int j = i; j >= 0; j--) + erase_check_load(mt, j); + } + + erase_check_erase(mt, 14); /*6015 */ + for (int i = 0; i < 25; i++) { + if (i == 14) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + erase_check_erase(mt, 16); /*7002 */ + for (int i = 0; i < 25; i++) { + if (i == 16 || i == 14) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + + mt_set_non_kernel(1); + erase_check_erase(mt, 13); /*6012 */ + for (int i = 0; i < 25; i++) { + if (i == 16 || i == 14 || i == 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 15); /*7003 */ + for (int i = 0; i < 25; i++) { + if (i <= 16 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ + for (int i = 0; i < 25; i++) { + if (i <= 17 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 18); /*7012 */ + for (int i = 0; i < 25; i++) { + if (i <= 18 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 19); /*7015 */ + for (int i = 0; i < 25; i++) { + if (i <= 19 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 20); /*8003 */ + for (int i = 0; i < 25; i++) { + if (i <= 20 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 21); /*8002 */ + for (int i = 0; i < 25; i++) { + if (i <= 21 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 22); /*8008 */ + for (int i = 0; i < 25; i++) { + if (i <= 22 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + for (int i = 23; i < 25; i++) + erase_check_erase(mt, i); + + for (int i = 0; i < 25; i++) { + if (i <= 25 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + /* Shrinking tree test. */ + + for (int i = 13; i < ARRAY_SIZE(set); i++) + erase_check_insert(mt, i); + + mt_set_non_kernel(99); + for (int i = 18; i < ARRAY_SIZE(set); i++) { + erase_check_erase(mt, i); + for (int j = 0; j < ARRAY_SIZE(set); j++) { + if (j < 18 || j > i) + erase_check_load(mt, j); + else + check_load(mt, set[j], NULL); + } + } + mt_set_non_kernel(35); + for (int i = 0; i < 18; i++) { + erase_check_erase(mt, i); + for (int j = 0; j < ARRAY_SIZE(set); j++) { + if (j < 18 && j > i) + erase_check_load(mt, j); + else + check_load(mt, set[j], NULL); + } + } + erase_check_insert(mt, 8); + erase_check_insert(mt, 9); + erase_check_erase(mt, 8); + rcu_unregister_thread(); +} + +#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ + a[(i)], a[(i + 1)], ptr) +#define STORE 1 +#define SNULL 2 +#define ERASE 3 +#define ec_type_str(x) \ + (((x) == STORE) ? \ + "STORE" : \ + (((x) == SNULL) ? \ + "SNULL" : "ERASE") \ + ) +#define check_erase2_debug 0 +void *mas_next(struct ma_state *mas, unsigned long max); + +/* Calculate the overwritten entries. */ +int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, + void *s_entry, unsigned long s_min, + void *e_entry, unsigned long e_max, + unsigned long *set, int i, bool null_entry) +{ + int count = 0, span = 0; + unsigned long retry = 0; + void *entry; + struct ma_state tmp; + + + /* count slots */ + memcpy(&tmp, mas_start, sizeof(tmp)); + entry = mas_next(&tmp, mas_end->last); + while (entry) { + BUG_ON(retry > 50); /* stop infinite retry on testing. */ + if (xa_is_zero(s_entry)) { + retry++; + continue; + } + count++; + span++; + entry = mas_next(&tmp, mas_end->last); + } + + if (null_entry) { + /* Check splitting end. */ + if (e_entry && (e_max > mas_end->last)) + count--; + + /* check overwrite of entire start */ + if (s_entry && (s_min == mas_start->index)) + count++; + } else { /* !null_entry (store) */ + bool esplit = e_max > mas_end->last; + bool ssplit = s_min != mas_start->index; + + if (s_entry && e_entry) { + if (esplit && ssplit) + count--; + else if (ssplit) + count--; + else if (esplit) { + if (span) + count--; + } + } else if (s_entry && !e_entry) { + if (ssplit) + count--; + } else if (!s_entry && e_entry) { + if (esplit) + count--; + count--; + } else { + count--; + } + } + return count; +} + +/* + * mas_node_walk() - Walk a maple node to offset of the index. + * @mas: The maple state + * @type: The maple node type + * @*range_min: Pointer to store the minimum range of the offset + * @*range_max: Pointer to store the maximum range of the offset + * + * The offset will be stored in the maple state. + * + */ +static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, + enum maple_type type, unsigned long *range_min, + unsigned long *range_max) + +{ + unsigned long *pivots; + unsigned char count; + unsigned long prev, max; + unsigned char offset; + unsigned long index; + + if (unlikely(ma_is_dense(type))) { + (*range_max) = (*range_min) = mas->index; + if (unlikely(ma_dead_node(node))) + return; + + mas->offset = mas->index = mas->min; + return; + } + + pivots = ma_pivots(node, type); + max = pivots[0]; + if (unlikely(ma_dead_node(node))) + return; + + offset = 0; + prev = mas->min; + index = mas->index; + if (unlikely(index <= max)) + goto offset_zero; + + count = mt_pivots[type]; + while (++offset < count) { + prev = max; + max = pivots[offset]; + if (unlikely(ma_dead_node(node))) + return; + + if (index <= max) + goto offset_found; + else if (unlikely(!max)) + goto mas_max; + } + + prev = max; +mas_max: + max = mas->max; +offset_found: + prev++; +offset_zero: + mas->offset = offset; + if (ma_is_leaf(type)) { + *range_max = max; + *range_min = prev; + } else { + mas->max = max; + mas->min = prev; + } +} + +/* + * mas_descend_walk(): Locates a value and sets the mas->node and slot + * accordingly. range_min and range_max are set to the range which the entry is + * valid. + * @mas: The maple state + * @*range_min: A pointer to store the minimum of the range + * @*range_max: A pointer to store the maximum of the range + * + * Check mas->node is still valid on return of any value. + * + * Return: true if pointing to a valid node and offset. False otherwise. + */ +static inline bool mas_descend_walk(struct ma_state *mas, + unsigned long *range_min, unsigned long *range_max) +{ + struct maple_enode *next; + struct maple_node *node; + enum maple_type type; + + next = mas->node; + while (true) { + node = mte_to_node(next); + type = mte_node_type(next); + mas_node_walk(mas, node, type, range_min, range_max); + next = mas_slot(mas, ma_slots(node, type), mas->offset); + if (unlikely(ma_dead_node(node))) + return false; + + if (unlikely(ma_is_leaf(type))) + return true; + + /* Descend. */ + mas->node = next; + } + return false; +} + +/* + * mas_tree_walk() - Walk to @mas->index and set the range values. + * @mas: The maple state. + * @*range_min: The minimum range to be set. + * @*range_max: The maximum range to be set. + * + * Ranges are only valid if there is a valid entry at @mas->index. + * + * Return: True if a value exists, false otherwise. + */ +static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, + unsigned long *range_max) +{ + bool ret; + +retry: + ret = false; + mas_start(mas); + if (mas_is_none(mas)) + goto not_found; + + if (mas_is_ptr(mas)) { + *range_min = *range_max = 0; + if (!mas->index) + return true; + + goto not_found; + } + + ret = mas_descend_walk(mas, range_min, range_max); + if (unlikely(mte_dead_node(mas->node))) { + mas->node = MAS_START; + goto retry; + } + + return ret; + +not_found: + mas->offset = MAPLE_NODE_SLOTS; + return false; +} + +static inline void *mas_range_load(struct ma_state *mas, + unsigned long *range_min, unsigned long *range_max) + +{ + void *entry = NULL; + unsigned long index = mas->index; + + if (mas_is_none(mas) || mas_is_paused(mas)) + mas->node = MAS_START; +retry: + if (mas_tree_walk(mas, range_min, range_max)) + if (unlikely(mas->node == MAS_ROOT)) + return mas_root(mas); + + if (likely(mas->offset != MAPLE_NODE_SLOTS)) + entry = mas_get_slot(mas, mas->offset); + + if (mas_dead_node(mas, index)) + goto retry; + + return entry; +} +static noinline void check_erase2_testset(struct maple_tree *mt, + unsigned long *set, unsigned long size) +{ + int entry_count = 0; + int check = 0; + void *foo; + unsigned long addr = 0; + void *s_entry = NULL, *e_entry = NULL; + + MA_STATE(mas, mt, 0, 0); + + for (int i = 0; i < size; i += 3) { + unsigned long s_min, s_max; + unsigned long e_min, e_max; + void *value = NULL; + + MA_STATE(mas_start, mt, set[i+1], set[i+1]); + MA_STATE(mas_end, mt, set[i+2], set[i+2]); + mt_set_non_kernel(127); +#if check_erase2_debug + pr_err("%s: %d %s %lu - %lu\n", __func__, i, + ec_type_str(set[i]), + set[i+1], set[i+2]); +#endif + s_entry = mas_range_load(&mas_start, &s_min, &s_max); + e_entry = mas_range_load(&mas_end, &e_min, &e_max); + + switch (set[i]) { + case SNULL: + if ((s_min == set[i+1]) && (s_max == set[i+2])) { + if (s_entry) + entry_count--; + } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { + entry_count++; + } else if ((mas_start.node != mas_end.node) || + (mas_start.offset != mas_end.offset)) { + entry_count -= + mas_ce2_over_count(&mas_start, &mas_end, + s_entry, s_min, + e_entry, e_max, set, i, + true); + } + + + erase_check_store_range(mt, set, i + 1, value); + break; + case STORE: + value = xa_mk_value(set[i + 1]); + if (mas_start.offset > mt_slot_count(mas_start.node)) { + entry_count++; /* appending an entry. */ + } else if ((s_min == e_min) && (s_max == e_max)) { + if (!entry_count) + entry_count++; + + else if (s_entry) { + if (e_max > mas_end.last) + entry_count++; + + if (s_min < mas_start.index) + entry_count++; + + } else { + entry_count++; + } + } else { + entry_count -= + mas_ce2_over_count(&mas_start, &mas_end, + s_entry, s_min, + e_entry, e_max, set, i, + false); + } + + erase_check_store_range(mt, set, i + 1, value); + break; + case ERASE: + if (!s_entry) + break; + check_erase(mt, set[i+1], xa_mk_value(set[i+1])); + entry_count--; + break; + } + mt_validate(mt); + if (entry_count) + MT_BUG_ON(mt, !mt_height(mt)); +#if check_erase2_debug > 1 + mt_dump(mt); +#endif +#if check_erase2_debug + pr_err("Done\n"); +#endif + + check = 0; + addr = 0; + mt_for_each(mt, foo, addr, ULONG_MAX) { + check++; +#if check_erase2_debug > 2 + pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); +#endif + if (check > entry_count) + break; + } + +#if check_erase2_debug > 2 + pr_err("mt_for_each %d and count %d\n", check, entry_count); +#endif + + MT_BUG_ON(mt, check != entry_count); + + check = 0; + addr = 0; + mas_reset(&mas); + mas.index = 0; + rcu_read_lock(); + mas_for_each(&mas, foo, ULONG_MAX) { + if (xa_is_zero(foo)) { + if (addr == mas.index) { + mt_dump(mas.tree); + pr_err("retry failed %lu - %lu\n", + mas.index, mas.last); + MT_BUG_ON(mt, 1); + } + addr = mas.index; + continue; + } +#if check_erase2_debug > 2 + pr_err("mas: %lu -> %p\n", mas.index, foo); +#endif + check++; + if (check > entry_count) + break; + } + rcu_read_unlock(); +#if check_erase2_debug > 2 + pr_err("mas_for_each %d and count %d\n", check, entry_count); + mt_validate(mt); +#endif + + MT_BUG_ON(mt, check != entry_count); + + MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); + } +} + + +/* These tests were pulled from kvm tests. */ +static noinline void check_erase2_sets(struct maple_tree *mt) +{ + void *entry; + unsigned long start = 0; + unsigned long set[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140721266458624, 140737488351231, +ERASE, 140721266458624, 140737488351231, +STORE, 140721266458624, 140721266462719, +STORE, 94735788949504, 94735789121535, +ERASE, 94735788949504, 94735789121535, +STORE, 94735788949504, 94735788965887, +STORE, 94735788965888, 94735789121535, +ERASE, 94735788965888, 94735789121535, +STORE, 94735788965888, 94735789068287, +STORE, 94735789068288, 94735789109247, +STORE, 94735789109248, 94735789121535, +STORE, 140253902692352, 140253902864383, +ERASE, 140253902692352, 140253902864383, +STORE, 140253902692352, 140253902696447, +STORE, 140253902696448, 140253902864383, + }; + unsigned long set2[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735933583360, 140737488351231, +ERASE, 140735933583360, 140737488351231, +STORE, 140735933583360, 140735933587455, +STORE, 94811003260928, 94811003432959, +ERASE, 94811003260928, 94811003432959, +STORE, 94811003260928, 94811003277311, +STORE, 94811003277312, 94811003432959, +ERASE, 94811003277312, 94811003432959, +STORE, 94811003277312, 94811003379711, +STORE, 94811003379712, 94811003420671, +STORE, 94811003420672, 94811003432959, +STORE, 140277094653952, 140277094825983, +ERASE, 140277094653952, 140277094825983, +STORE, 140277094653952, 140277094658047, +STORE, 140277094658048, 140277094825983, +ERASE, 140277094658048, 140277094825983, +STORE, 140277094658048, 140277094780927, +STORE, 140277094780928, 140277094813695, +STORE, 140277094813696, 140277094821887, +STORE, 140277094821888, 140277094825983, +STORE, 140735933906944, 140735933911039, + }; + unsigned long set3[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735790264320, 140737488351231, +ERASE, 140735790264320, 140737488351231, +STORE, 140735790264320, 140735790268415, +STORE, 94016597282816, 94016597454847, +ERASE, 94016597282816, 94016597454847, +STORE, 94016597282816, 94016597299199, +STORE, 94016597299200, 94016597454847, +ERASE, 94016597299200, 94016597454847, +STORE, 94016597299200, 94016597401599, +STORE, 94016597401600, 94016597442559, +STORE, 94016597442560, 94016597454847, +STORE, 140496959283200, 140496959455231, +ERASE, 140496959283200, 140496959455231, +STORE, 140496959283200, 140496959287295, +STORE, 140496959287296, 140496959455231, +ERASE, 140496959287296, 140496959455231, +STORE, 140496959287296, 140496959410175, +STORE, 140496959410176, 140496959442943, +STORE, 140496959442944, 140496959451135, +STORE, 140496959451136, 140496959455231, +STORE, 140735791718400, 140735791722495, +STORE, 140735791706112, 140735791718399, +STORE, 47135835713536, 47135835721727, +STORE, 47135835721728, 47135835729919, +STORE, 47135835729920, 47135835893759, +ERASE, 47135835729920, 47135835893759, +STORE, 47135835729920, 47135835742207, +STORE, 47135835742208, 47135835893759, +STORE, 47135835840512, 47135835893759, +STORE, 47135835742208, 47135835840511, +ERASE, 47135835742208, 47135835840511, +STORE, 47135835742208, 47135835840511, +STORE, 47135835885568, 47135835893759, +STORE, 47135835840512, 47135835885567, +ERASE, 47135835840512, 47135835885567, +STORE, 47135835840512, 47135835893759, +ERASE, 47135835840512, 47135835893759, +STORE, 47135835840512, 47135835885567, +STORE, 47135835885568, 47135835893759, + }; + + unsigned long set4[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140728251703296, 140737488351231, +ERASE, 140728251703296, 140737488351231, +STORE, 140728251703296, 140728251707391, +STORE, 94668429205504, 94668429377535, +ERASE, 94668429205504, 94668429377535, +STORE, 94668429205504, 94668429221887, +STORE, 94668429221888, 94668429377535, +ERASE, 94668429221888, 94668429377535, +STORE, 94668429221888, 94668429324287, +STORE, 94668429324288, 94668429365247, +STORE, 94668429365248, 94668429377535, +STORE, 47646523273216, 47646523445247, +ERASE, 47646523273216, 47646523445247, +STORE, 47646523273216, 47646523277311, +STORE, 47646523277312, 47646523445247, +ERASE, 47646523277312, 47646523445247, +STORE, 47646523277312, 47646523400191, + }; + + unsigned long set5[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140726874062848, 140737488351231, +ERASE, 140726874062848, 140737488351231, +STORE, 140726874062848, 140726874066943, +STORE, 94248892870656, 94248893042687, +ERASE, 94248892870656, 94248893042687, +STORE, 94248892870656, 94248892887039, +STORE, 94248892887040, 94248893042687, +ERASE, 94248892887040, 94248893042687, +STORE, 94248892887040, 94248892989439, +STORE, 94248892989440, 94248893030399, +STORE, 94248893030400, 94248893042687, +STORE, 47884786266112, 47884786438143, +ERASE, 47884786266112, 47884786438143, +STORE, 47884786266112, 47884786270207, +STORE, 47884786270208, 47884786438143, +ERASE, 47884786270208, 47884786438143, +STORE, 47884786270208, 47884786393087, +STORE, 47884786393088, 47884786425855, +STORE, 47884786425856, 47884786434047, +STORE, 47884786434048, 47884786438143, +STORE, 140726874513408, 140726874517503, +STORE, 140726874501120, 140726874513407, +STORE, 47884786438144, 47884786446335, +STORE, 47884786446336, 47884786454527, +STORE, 47884786454528, 47884786618367, +ERASE, 47884786454528, 47884786618367, +STORE, 47884786454528, 47884786466815, +STORE, 47884786466816, 47884786618367, +STORE, 47884786565120, 47884786618367, +STORE, 47884786466816, 47884786565119, +ERASE, 47884786466816, 47884786565119, +STORE, 47884786466816, 47884786565119, +STORE, 47884786610176, 47884786618367, +STORE, 47884786565120, 47884786610175, +ERASE, 47884786565120, 47884786610175, +STORE, 47884786565120, 47884786618367, +ERASE, 47884786565120, 47884786618367, +STORE, 47884786565120, 47884786610175, +STORE, 47884786610176, 47884786618367, +ERASE, 47884786610176, 47884786618367, +STORE, 47884786610176, 47884786618367, +STORE, 47884786618368, 47884789669887, +STORE, 47884787163136, 47884789669887, +STORE, 47884786618368, 47884787163135, +ERASE, 47884787163136, 47884789669887, +STORE, 47884787163136, 47884789448703, +STORE, 47884789448704, 47884789669887, +STORE, 47884788858880, 47884789448703, +STORE, 47884787163136, 47884788858879, +ERASE, 47884787163136, 47884788858879, +STORE, 47884787163136, 47884788858879, +STORE, 47884789444608, 47884789448703, +STORE, 47884788858880, 47884789444607, +ERASE, 47884788858880, 47884789444607, +STORE, 47884788858880, 47884789444607, +STORE, 47884789653504, 47884789669887, +STORE, 47884789448704, 47884789653503, +ERASE, 47884789448704, 47884789653503, +STORE, 47884789448704, 47884789653503, +ERASE, 47884789653504, 47884789669887, +STORE, 47884789653504, 47884789669887, +STORE, 47884789669888, 47884791508991, +STORE, 47884789809152, 47884791508991, +STORE, 47884789669888, 47884789809151, +ERASE, 47884789809152, 47884791508991, +STORE, 47884789809152, 47884791468031, +STORE, 47884791468032, 47884791508991, +STORE, 47884791152640, 47884791468031, +STORE, 47884789809152, 47884791152639, +ERASE, 47884789809152, 47884791152639, +STORE, 47884789809152, 47884791152639, +STORE, 47884791463936, 47884791468031, +STORE, 47884791152640, 47884791463935, +ERASE, 47884791152640, 47884791463935, +STORE, 47884791152640, 47884791463935, +STORE, 47884791492608, 47884791508991, +STORE, 47884791468032, 47884791492607, +ERASE, 47884791468032, 47884791492607, +STORE, 47884791468032, 47884791492607, +ERASE, 47884791492608, 47884791508991, +STORE, 47884791492608, 47884791508991, +STORE, 47884791508992, 47884791644159, +ERASE, 47884791508992, 47884791644159, +STORE, 47884791508992, 47884791533567, +STORE, 47884791533568, 47884791644159, +STORE, 47884791595008, 47884791644159, +STORE, 47884791533568, 47884791595007, +ERASE, 47884791533568, 47884791595007, +STORE, 47884791533568, 47884791595007, +STORE, 47884791619584, 47884791644159, +STORE, 47884791595008, 47884791619583, +ERASE, 47884791595008, 47884791619583, +STORE, 47884791595008, 47884791644159, +ERASE, 47884791595008, 47884791644159, +STORE, 47884791595008, 47884791619583, +STORE, 47884791619584, 47884791644159, +STORE, 47884791627776, 47884791644159, +STORE, 47884791619584, 47884791627775, +ERASE, 47884791619584, 47884791627775, +STORE, 47884791619584, 47884791627775, +ERASE, 47884791627776, 47884791644159, +STORE, 47884791627776, 47884791644159, +STORE, 47884791644160, 47884791664639, +ERASE, 47884791644160, 47884791664639, +STORE, 47884791644160, 47884791648255, +STORE, 47884791648256, 47884791664639, +STORE, 47884791652352, 47884791664639, +STORE, 47884791648256, 47884791652351, +ERASE, 47884791648256, 47884791652351, +STORE, 47884791648256, 47884791652351, +STORE, 47884791656448, 47884791664639, +STORE, 47884791652352, 47884791656447, +ERASE, 47884791652352, 47884791656447, +STORE, 47884791652352, 47884791664639, +ERASE, 47884791652352, 47884791664639, +STORE, 47884791652352, 47884791656447, +STORE, 47884791656448, 47884791664639, +ERASE, 47884791656448, 47884791664639, +STORE, 47884791656448, 47884791664639, +STORE, 47884791664640, 47884791672831, +ERASE, 47884791468032, 47884791492607, +STORE, 47884791468032, 47884791484415, +STORE, 47884791484416, 47884791492607, +ERASE, 47884791656448, 47884791664639, +STORE, 47884791656448, 47884791660543, +STORE, 47884791660544, 47884791664639, +ERASE, 47884791619584, 47884791627775, +STORE, 47884791619584, 47884791623679, +STORE, 47884791623680, 47884791627775, + }; + + unsigned long set6[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722999021568, 140737488351231, +ERASE, 140722999021568, 140737488351231, +STORE, 140722999021568, 140722999025663, +STORE, 94901500268544, 94901500440575, +ERASE, 94901500268544, 94901500440575, +STORE, 94901500268544, 94901500284927, +STORE, 94901500284928, 94901500440575, +ERASE, 94901500284928, 94901500440575, +STORE, 94901500284928, 94901500387327, +STORE, 94901500387328, 94901500428287, +STORE, 94901500428288, 94901500440575, +STORE, 47430426660864, 47430426832895, +ERASE, 47430426660864, 47430426832895, +STORE, 47430426660864, 47430426664959, +STORE, 47430426664960, 47430426832895, +ERASE, 47430426664960, 47430426832895, +STORE, 47430426664960, 47430426787839, +STORE, 47430426787840, 47430426820607, +STORE, 47430426820608, 47430426828799, +STORE, 47430426828800, 47430426832895, +STORE, 140722999115776, 140722999119871, +STORE, 140722999103488, 140722999115775, +STORE, 47430426832896, 47430426841087, +STORE, 47430426841088, 47430426849279, +STORE, 47430426849280, 47430427013119, +ERASE, 47430426849280, 47430427013119, +STORE, 47430426849280, 47430426861567, +STORE, 47430426861568, 47430427013119, +STORE, 47430426959872, 47430427013119, +STORE, 47430426861568, 47430426959871, +ERASE, 47430426861568, 47430426959871, +STORE, 47430426861568, 47430426959871, +STORE, 47430427004928, 47430427013119, +STORE, 47430426959872, 47430427004927, +ERASE, 47430426959872, 47430427004927, +STORE, 47430426959872, 47430427013119, +ERASE, 47430426959872, 47430427013119, +STORE, 47430426959872, 47430427004927, +STORE, 47430427004928, 47430427013119, +ERASE, 47430427004928, 47430427013119, +STORE, 47430427004928, 47430427013119, +STORE, 47430427013120, 47430430064639, +STORE, 47430427557888, 47430430064639, +STORE, 47430427013120, 47430427557887, +ERASE, 47430427557888, 47430430064639, +STORE, 47430427557888, 47430429843455, +STORE, 47430429843456, 47430430064639, +STORE, 47430429253632, 47430429843455, +STORE, 47430427557888, 47430429253631, +ERASE, 47430427557888, 47430429253631, +STORE, 47430427557888, 47430429253631, +STORE, 47430429839360, 47430429843455, +STORE, 47430429253632, 47430429839359, +ERASE, 47430429253632, 47430429839359, +STORE, 47430429253632, 47430429839359, +STORE, 47430430048256, 47430430064639, +STORE, 47430429843456, 47430430048255, +ERASE, 47430429843456, 47430430048255, +STORE, 47430429843456, 47430430048255, +ERASE, 47430430048256, 47430430064639, +STORE, 47430430048256, 47430430064639, +STORE, 47430430064640, 47430431903743, +STORE, 47430430203904, 47430431903743, +STORE, 47430430064640, 47430430203903, +ERASE, 47430430203904, 47430431903743, +STORE, 47430430203904, 47430431862783, +STORE, 47430431862784, 47430431903743, +STORE, 47430431547392, 47430431862783, +STORE, 47430430203904, 47430431547391, +ERASE, 47430430203904, 47430431547391, +STORE, 47430430203904, 47430431547391, +STORE, 47430431858688, 47430431862783, +STORE, 47430431547392, 47430431858687, +ERASE, 47430431547392, 47430431858687, +STORE, 47430431547392, 47430431858687, +STORE, 47430431887360, 47430431903743, +STORE, 47430431862784, 47430431887359, +ERASE, 47430431862784, 47430431887359, +STORE, 47430431862784, 47430431887359, +ERASE, 47430431887360, 47430431903743, +STORE, 47430431887360, 47430431903743, +STORE, 47430431903744, 47430432038911, +ERASE, 47430431903744, 47430432038911, +STORE, 47430431903744, 47430431928319, +STORE, 47430431928320, 47430432038911, +STORE, 47430431989760, 47430432038911, +STORE, 47430431928320, 47430431989759, +ERASE, 47430431928320, 47430431989759, +STORE, 47430431928320, 47430431989759, +STORE, 47430432014336, 47430432038911, +STORE, 47430431989760, 47430432014335, +ERASE, 47430431989760, 47430432014335, +STORE, 47430431989760, 47430432038911, +ERASE, 47430431989760, 47430432038911, +STORE, 47430431989760, 47430432014335, +STORE, 47430432014336, 47430432038911, +STORE, 47430432022528, 47430432038911, +STORE, 47430432014336, 47430432022527, +ERASE, 47430432014336, 47430432022527, +STORE, 47430432014336, 47430432022527, +ERASE, 47430432022528, 47430432038911, +STORE, 47430432022528, 47430432038911, +STORE, 47430432038912, 47430432059391, +ERASE, 47430432038912, 47430432059391, +STORE, 47430432038912, 47430432043007, +STORE, 47430432043008, 47430432059391, +STORE, 47430432047104, 47430432059391, +STORE, 47430432043008, 47430432047103, +ERASE, 47430432043008, 47430432047103, +STORE, 47430432043008, 47430432047103, +STORE, 47430432051200, 47430432059391, +STORE, 47430432047104, 47430432051199, +ERASE, 47430432047104, 47430432051199, +STORE, 47430432047104, 47430432059391, +ERASE, 47430432047104, 47430432059391, +STORE, 47430432047104, 47430432051199, +STORE, 47430432051200, 47430432059391, +ERASE, 47430432051200, 47430432059391, +STORE, 47430432051200, 47430432059391, +STORE, 47430432059392, 47430432067583, +ERASE, 47430431862784, 47430431887359, +STORE, 47430431862784, 47430431879167, +STORE, 47430431879168, 47430431887359, +ERASE, 47430432051200, 47430432059391, +STORE, 47430432051200, 47430432055295, +STORE, 47430432055296, 47430432059391, +ERASE, 47430432014336, 47430432022527, +STORE, 47430432014336, 47430432018431, +STORE, 47430432018432, 47430432022527, + }; + unsigned long set7[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140729808330752, 140737488351231, +ERASE, 140729808330752, 140737488351231, +STORE, 140729808330752, 140729808334847, +STORE, 94629632020480, 94629632192511, +ERASE, 94629632020480, 94629632192511, +STORE, 94629632020480, 94629632036863, +STORE, 94629632036864, 94629632192511, +ERASE, 94629632036864, 94629632192511, +STORE, 94629632036864, 94629632139263, +STORE, 94629632139264, 94629632180223, +STORE, 94629632180224, 94629632192511, +STORE, 47439981776896, 47439981948927, +ERASE, 47439981776896, 47439981948927, +STORE, 47439981776896, 47439981780991, +STORE, 47439981780992, 47439981948927, +ERASE, 47439981780992, 47439981948927, +STORE, 47439981780992, 47439981903871, +STORE, 47439981903872, 47439981936639, +STORE, 47439981936640, 47439981944831, +STORE, 47439981944832, 47439981948927, +STORE, 140729808474112, 140729808478207, +STORE, 140729808461824, 140729808474111, +STORE, 47439981948928, 47439981957119, +STORE, 47439981957120, 47439981965311, +STORE, 47439981965312, 47439982129151, +ERASE, 47439981965312, 47439982129151, +STORE, 47439981965312, 47439981977599, +STORE, 47439981977600, 47439982129151, +STORE, 47439982075904, 47439982129151, +STORE, 47439981977600, 47439982075903, +ERASE, 47439981977600, 47439982075903, +STORE, 47439981977600, 47439982075903, +STORE, 47439982120960, 47439982129151, +STORE, 47439982075904, 47439982120959, +ERASE, 47439982075904, 47439982120959, +STORE, 47439982075904, 47439982129151, +ERASE, 47439982075904, 47439982129151, +STORE, 47439982075904, 47439982120959, +STORE, 47439982120960, 47439982129151, +ERASE, 47439982120960, 47439982129151, +STORE, 47439982120960, 47439982129151, +STORE, 47439982129152, 47439985180671, +STORE, 47439982673920, 47439985180671, +STORE, 47439982129152, 47439982673919, +ERASE, 47439982673920, 47439985180671, +STORE, 47439982673920, 47439984959487, +STORE, 47439984959488, 47439985180671, +STORE, 47439984369664, 47439984959487, +STORE, 47439982673920, 47439984369663, +ERASE, 47439982673920, 47439984369663, +STORE, 47439982673920, 47439984369663, +STORE, 47439984955392, 47439984959487, +STORE, 47439984369664, 47439984955391, +ERASE, 47439984369664, 47439984955391, +STORE, 47439984369664, 47439984955391, +STORE, 47439985164288, 47439985180671, +STORE, 47439984959488, 47439985164287, +ERASE, 47439984959488, 47439985164287, +STORE, 47439984959488, 47439985164287, +ERASE, 47439985164288, 47439985180671, +STORE, 47439985164288, 47439985180671, +STORE, 47439985180672, 47439987019775, +STORE, 47439985319936, 47439987019775, +STORE, 47439985180672, 47439985319935, +ERASE, 47439985319936, 47439987019775, +STORE, 47439985319936, 47439986978815, +STORE, 47439986978816, 47439987019775, +STORE, 47439986663424, 47439986978815, +STORE, 47439985319936, 47439986663423, +ERASE, 47439985319936, 47439986663423, +STORE, 47439985319936, 47439986663423, +STORE, 47439986974720, 47439986978815, +STORE, 47439986663424, 47439986974719, +ERASE, 47439986663424, 47439986974719, +STORE, 47439986663424, 47439986974719, +STORE, 47439987003392, 47439987019775, +STORE, 47439986978816, 47439987003391, +ERASE, 47439986978816, 47439987003391, +STORE, 47439986978816, 47439987003391, +ERASE, 47439987003392, 47439987019775, +STORE, 47439987003392, 47439987019775, +STORE, 47439987019776, 47439987154943, +ERASE, 47439987019776, 47439987154943, +STORE, 47439987019776, 47439987044351, +STORE, 47439987044352, 47439987154943, +STORE, 47439987105792, 47439987154943, +STORE, 47439987044352, 47439987105791, +ERASE, 47439987044352, 47439987105791, +STORE, 47439987044352, 47439987105791, +STORE, 47439987130368, 47439987154943, +STORE, 47439987105792, 47439987130367, +ERASE, 47439987105792, 47439987130367, +STORE, 47439987105792, 47439987154943, +ERASE, 47439987105792, 47439987154943, +STORE, 47439987105792, 47439987130367, +STORE, 47439987130368, 47439987154943, +STORE, 47439987138560, 47439987154943, +STORE, 47439987130368, 47439987138559, +ERASE, 47439987130368, 47439987138559, +STORE, 47439987130368, 47439987138559, +ERASE, 47439987138560, 47439987154943, +STORE, 47439987138560, 47439987154943, +STORE, 47439987154944, 47439987175423, +ERASE, 47439987154944, 47439987175423, +STORE, 47439987154944, 47439987159039, +STORE, 47439987159040, 47439987175423, +STORE, 47439987163136, 47439987175423, +STORE, 47439987159040, 47439987163135, +ERASE, 47439987159040, 47439987163135, +STORE, 47439987159040, 47439987163135, +STORE, 47439987167232, 47439987175423, +STORE, 47439987163136, 47439987167231, +ERASE, 47439987163136, 47439987167231, +STORE, 47439987163136, 47439987175423, +ERASE, 47439987163136, 47439987175423, +STORE, 47439987163136, 47439987167231, +STORE, 47439987167232, 47439987175423, +ERASE, 47439987167232, 47439987175423, +STORE, 47439987167232, 47439987175423, +STORE, 47439987175424, 47439987183615, +ERASE, 47439986978816, 47439987003391, +STORE, 47439986978816, 47439986995199, +STORE, 47439986995200, 47439987003391, +ERASE, 47439987167232, 47439987175423, +STORE, 47439987167232, 47439987171327, +STORE, 47439987171328, 47439987175423, +ERASE, 47439987130368, 47439987138559, +STORE, 47439987130368, 47439987134463, +STORE, 47439987134464, 47439987138559, + }; + unsigned long set8[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722482974720, 140737488351231, +ERASE, 140722482974720, 140737488351231, +STORE, 140722482974720, 140722482978815, +STORE, 94121505034240, 94121505206271, +ERASE, 94121505034240, 94121505206271, +STORE, 94121505034240, 94121505050623, +STORE, 94121505050624, 94121505206271, +ERASE, 94121505050624, 94121505206271, +STORE, 94121505050624, 94121505153023, +STORE, 94121505153024, 94121505193983, +STORE, 94121505193984, 94121505206271, +STORE, 47708483284992, 47708483457023, +ERASE, 47708483284992, 47708483457023, +STORE, 47708483284992, 47708483289087, +STORE, 47708483289088, 47708483457023, +ERASE, 47708483289088, 47708483457023, +STORE, 47708483289088, 47708483411967, +STORE, 47708483411968, 47708483444735, +STORE, 47708483444736, 47708483452927, +STORE, 47708483452928, 47708483457023, +STORE, 140722483142656, 140722483146751, +STORE, 140722483130368, 140722483142655, +STORE, 47708483457024, 47708483465215, +STORE, 47708483465216, 47708483473407, +STORE, 47708483473408, 47708483637247, +ERASE, 47708483473408, 47708483637247, +STORE, 47708483473408, 47708483485695, +STORE, 47708483485696, 47708483637247, +STORE, 47708483584000, 47708483637247, +STORE, 47708483485696, 47708483583999, +ERASE, 47708483485696, 47708483583999, +STORE, 47708483485696, 47708483583999, +STORE, 47708483629056, 47708483637247, +STORE, 47708483584000, 47708483629055, +ERASE, 47708483584000, 47708483629055, +STORE, 47708483584000, 47708483637247, +ERASE, 47708483584000, 47708483637247, +STORE, 47708483584000, 47708483629055, +STORE, 47708483629056, 47708483637247, +ERASE, 47708483629056, 47708483637247, +STORE, 47708483629056, 47708483637247, +STORE, 47708483637248, 47708486688767, +STORE, 47708484182016, 47708486688767, +STORE, 47708483637248, 47708484182015, +ERASE, 47708484182016, 47708486688767, +STORE, 47708484182016, 47708486467583, +STORE, 47708486467584, 47708486688767, +STORE, 47708485877760, 47708486467583, +STORE, 47708484182016, 47708485877759, +ERASE, 47708484182016, 47708485877759, +STORE, 47708484182016, 47708485877759, +STORE, 47708486463488, 47708486467583, +STORE, 47708485877760, 47708486463487, +ERASE, 47708485877760, 47708486463487, +STORE, 47708485877760, 47708486463487, +STORE, 47708486672384, 47708486688767, +STORE, 47708486467584, 47708486672383, +ERASE, 47708486467584, 47708486672383, +STORE, 47708486467584, 47708486672383, +ERASE, 47708486672384, 47708486688767, +STORE, 47708486672384, 47708486688767, +STORE, 47708486688768, 47708488527871, +STORE, 47708486828032, 47708488527871, +STORE, 47708486688768, 47708486828031, +ERASE, 47708486828032, 47708488527871, +STORE, 47708486828032, 47708488486911, +STORE, 47708488486912, 47708488527871, +STORE, 47708488171520, 47708488486911, +STORE, 47708486828032, 47708488171519, +ERASE, 47708486828032, 47708488171519, +STORE, 47708486828032, 47708488171519, +STORE, 47708488482816, 47708488486911, +STORE, 47708488171520, 47708488482815, +ERASE, 47708488171520, 47708488482815, +STORE, 47708488171520, 47708488482815, +STORE, 47708488511488, 47708488527871, +STORE, 47708488486912, 47708488511487, +ERASE, 47708488486912, 47708488511487, +STORE, 47708488486912, 47708488511487, +ERASE, 47708488511488, 47708488527871, +STORE, 47708488511488, 47708488527871, +STORE, 47708488527872, 47708488663039, +ERASE, 47708488527872, 47708488663039, +STORE, 47708488527872, 47708488552447, +STORE, 47708488552448, 47708488663039, +STORE, 47708488613888, 47708488663039, +STORE, 47708488552448, 47708488613887, +ERASE, 47708488552448, 47708488613887, +STORE, 47708488552448, 47708488613887, +STORE, 47708488638464, 47708488663039, +STORE, 47708488613888, 47708488638463, +ERASE, 47708488613888, 47708488638463, +STORE, 47708488613888, 47708488663039, +ERASE, 47708488613888, 47708488663039, +STORE, 47708488613888, 47708488638463, +STORE, 47708488638464, 47708488663039, +STORE, 47708488646656, 47708488663039, +STORE, 47708488638464, 47708488646655, +ERASE, 47708488638464, 47708488646655, +STORE, 47708488638464, 47708488646655, +ERASE, 47708488646656, 47708488663039, +STORE, 47708488646656, 47708488663039, +STORE, 47708488663040, 47708488683519, +ERASE, 47708488663040, 47708488683519, +STORE, 47708488663040, 47708488667135, +STORE, 47708488667136, 47708488683519, +STORE, 47708488671232, 47708488683519, +STORE, 47708488667136, 47708488671231, +ERASE, 47708488667136, 47708488671231, +STORE, 47708488667136, 47708488671231, +STORE, 47708488675328, 47708488683519, +STORE, 47708488671232, 47708488675327, +ERASE, 47708488671232, 47708488675327, +STORE, 47708488671232, 47708488683519, +ERASE, 47708488671232, 47708488683519, +STORE, 47708488671232, 47708488675327, +STORE, 47708488675328, 47708488683519, +ERASE, 47708488675328, 47708488683519, +STORE, 47708488675328, 47708488683519, +STORE, 47708488683520, 47708488691711, +ERASE, 47708488486912, 47708488511487, +STORE, 47708488486912, 47708488503295, +STORE, 47708488503296, 47708488511487, +ERASE, 47708488675328, 47708488683519, +STORE, 47708488675328, 47708488679423, +STORE, 47708488679424, 47708488683519, +ERASE, 47708488638464, 47708488646655, +STORE, 47708488638464, 47708488642559, +STORE, 47708488642560, 47708488646655, + }; + + unsigned long set9[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736427839488, 140737488351231, +ERASE, 140736427839488, 140736427839488, +STORE, 140736427839488, 140736427843583, +STORE, 94071213395968, 94071213567999, +ERASE, 94071213395968, 94071213395968, +STORE, 94071213395968, 94071213412351, +STORE, 94071213412352, 94071213567999, +ERASE, 94071213412352, 94071213412352, +STORE, 94071213412352, 94071213514751, +STORE, 94071213514752, 94071213555711, +STORE, 94071213555712, 94071213567999, +STORE, 139968410644480, 139968410816511, +ERASE, 139968410644480, 139968410644480, +STORE, 139968410644480, 139968410648575, +STORE, 139968410648576, 139968410816511, +ERASE, 139968410648576, 139968410648576, +STORE, 139968410648576, 139968410771455, +STORE, 139968410771456, 139968410804223, +STORE, 139968410804224, 139968410812415, +STORE, 139968410812416, 139968410816511, +STORE, 140736429277184, 140736429281279, +STORE, 140736429264896, 140736429277183, +STORE, 47664384352256, 47664384360447, +STORE, 47664384360448, 47664384368639, +STORE, 47664384368640, 47664384532479, +ERASE, 47664384368640, 47664384368640, +STORE, 47664384368640, 47664384380927, +STORE, 47664384380928, 47664384532479, +STORE, 47664384479232, 47664384532479, +STORE, 47664384380928, 47664384479231, +ERASE, 47664384380928, 47664384380928, +STORE, 47664384380928, 47664384479231, +STORE, 47664384524288, 47664384532479, +STORE, 47664384479232, 47664384524287, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384532479, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384524287, +STORE, 47664384524288, 47664384532479, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384532479, +STORE, 47664384532480, 47664387583999, +STORE, 47664385077248, 47664387583999, +STORE, 47664384532480, 47664385077247, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664387362815, +STORE, 47664387362816, 47664387583999, +STORE, 47664386772992, 47664387362815, +STORE, 47664385077248, 47664386772991, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664386772991, +STORE, 47664387358720, 47664387362815, +STORE, 47664386772992, 47664387358719, +ERASE, 47664386772992, 47664386772992, +STORE, 47664386772992, 47664387358719, +STORE, 47664387567616, 47664387583999, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387567616, 47664387567616, +STORE, 47664387567616, 47664387583999, +STORE, 47664387584000, 47664389423103, +STORE, 47664387723264, 47664389423103, +STORE, 47664387584000, 47664387723263, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389382143, +STORE, 47664389382144, 47664389423103, +STORE, 47664389066752, 47664389382143, +STORE, 47664387723264, 47664389066751, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389066751, +STORE, 47664389378048, 47664389382143, +STORE, 47664389066752, 47664389378047, +ERASE, 47664389066752, 47664389066752, +STORE, 47664389066752, 47664389378047, +STORE, 47664389406720, 47664389423103, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389406720, 47664389406720, +STORE, 47664389406720, 47664389423103, +STORE, 47664389423104, 47664389558271, +ERASE, 47664389423104, 47664389423104, +STORE, 47664389423104, 47664389447679, +STORE, 47664389447680, 47664389558271, +STORE, 47664389509120, 47664389558271, +STORE, 47664389447680, 47664389509119, +ERASE, 47664389447680, 47664389447680, +STORE, 47664389447680, 47664389509119, +STORE, 47664389533696, 47664389558271, +STORE, 47664389509120, 47664389533695, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389558271, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389533695, +STORE, 47664389533696, 47664389558271, +STORE, 47664389541888, 47664389558271, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389541888, 47664389541888, +STORE, 47664389541888, 47664389558271, +STORE, 47664389558272, 47664389578751, +ERASE, 47664389558272, 47664389558272, +STORE, 47664389558272, 47664389562367, +STORE, 47664389562368, 47664389578751, +STORE, 47664389566464, 47664389578751, +STORE, 47664389562368, 47664389566463, +ERASE, 47664389562368, 47664389562368, +STORE, 47664389562368, 47664389566463, +STORE, 47664389570560, 47664389578751, +STORE, 47664389566464, 47664389570559, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389578751, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389570559, +STORE, 47664389570560, 47664389578751, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389578751, +STORE, 47664389578752, 47664389586943, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389398527, +STORE, 47664389398528, 47664389406719, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389574655, +STORE, 47664389574656, 47664389578751, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389537791, +STORE, 47664389537792, 47664389541887, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387559423, +STORE, 47664387559424, 47664387567615, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384528383, +STORE, 47664384528384, 47664384532479, +ERASE, 94071213555712, 94071213555712, +STORE, 94071213555712, 94071213563903, +STORE, 94071213563904, 94071213567999, +ERASE, 139968410804224, 139968410804224, +STORE, 139968410804224, 139968410808319, +STORE, 139968410808320, 139968410812415, +ERASE, 47664384352256, 47664384352256, +STORE, 94071244402688, 94071244537855, +STORE, 140737488347136, 140737488351231, +STORE, 140728271503360, 140737488351231, +ERASE, 140728271503360, 140728271503360, +STORE, 140728271503360, 140728271507455, +STORE, 94410361982976, 94410362155007, +ERASE, 94410361982976, 94410361982976, +STORE, 94410361982976, 94410361999359, +STORE, 94410361999360, 94410362155007, +ERASE, 94410361999360, 94410361999360, +STORE, 94410361999360, 94410362101759, +STORE, 94410362101760, 94410362142719, +STORE, 94410362142720, 94410362155007, +STORE, 140351953997824, 140351954169855, +ERASE, 140351953997824, 140351953997824, +STORE, 140351953997824, 140351954001919, +STORE, 140351954001920, 140351954169855, +ERASE, 140351954001920, 140351954001920, +STORE, 140351954001920, 140351954124799, +STORE, 140351954124800, 140351954157567, +STORE, 140351954157568, 140351954165759, +STORE, 140351954165760, 140351954169855, +STORE, 140728272429056, 140728272433151, +STORE, 140728272416768, 140728272429055, +STORE, 47280840998912, 47280841007103, +STORE, 47280841007104, 47280841015295, +STORE, 47280841015296, 47280841179135, +ERASE, 47280841015296, 47280841015296, +STORE, 47280841015296, 47280841027583, +STORE, 47280841027584, 47280841179135, +STORE, 47280841125888, 47280841179135, +STORE, 47280841027584, 47280841125887, +ERASE, 47280841027584, 47280841027584, +STORE, 47280841027584, 47280841125887, +STORE, 47280841170944, 47280841179135, +STORE, 47280841125888, 47280841170943, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841179135, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841170943, +STORE, 47280841170944, 47280841179135, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841179135, +STORE, 47280841179136, 47280844230655, +STORE, 47280841723904, 47280844230655, +STORE, 47280841179136, 47280841723903, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280844009471, +STORE, 47280844009472, 47280844230655, +STORE, 47280843419648, 47280844009471, +STORE, 47280841723904, 47280843419647, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280843419647, +STORE, 47280844005376, 47280844009471, +STORE, 47280843419648, 47280844005375, +ERASE, 47280843419648, 47280843419648, +STORE, 47280843419648, 47280844005375, +STORE, 47280844214272, 47280844230655, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844214272, 47280844214272, +STORE, 47280844214272, 47280844230655, +STORE, 47280844230656, 47280846069759, +STORE, 47280844369920, 47280846069759, +STORE, 47280844230656, 47280844369919, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280846028799, +STORE, 47280846028800, 47280846069759, +STORE, 47280845713408, 47280846028799, +STORE, 47280844369920, 47280845713407, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280845713407, +STORE, 47280846024704, 47280846028799, +STORE, 47280845713408, 47280846024703, +ERASE, 47280845713408, 47280845713408, +STORE, 47280845713408, 47280846024703, +STORE, 47280846053376, 47280846069759, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846053376, 47280846053376, +STORE, 47280846053376, 47280846069759, +STORE, 47280846069760, 47280846204927, +ERASE, 47280846069760, 47280846069760, +STORE, 47280846069760, 47280846094335, +STORE, 47280846094336, 47280846204927, +STORE, 47280846155776, 47280846204927, +STORE, 47280846094336, 47280846155775, +ERASE, 47280846094336, 47280846094336, +STORE, 47280846094336, 47280846155775, +STORE, 47280846180352, 47280846204927, +STORE, 47280846155776, 47280846180351, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846204927, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846180351, +STORE, 47280846180352, 47280846204927, +STORE, 47280846188544, 47280846204927, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846188544, 47280846188544, +STORE, 47280846188544, 47280846204927, +STORE, 47280846204928, 47280846225407, +ERASE, 47280846204928, 47280846204928, +STORE, 47280846204928, 47280846209023, +STORE, 47280846209024, 47280846225407, +STORE, 47280846213120, 47280846225407, +STORE, 47280846209024, 47280846213119, +ERASE, 47280846209024, 47280846209024, +STORE, 47280846209024, 47280846213119, +STORE, 47280846217216, 47280846225407, +STORE, 47280846213120, 47280846217215, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846225407, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846217215, +STORE, 47280846217216, 47280846225407, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846225407, +STORE, 47280846225408, 47280846233599, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846045183, +STORE, 47280846045184, 47280846053375, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846221311, +STORE, 47280846221312, 47280846225407, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846184447, +STORE, 47280846184448, 47280846188543, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844206079, +STORE, 47280844206080, 47280844214271, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841175039, +STORE, 47280841175040, 47280841179135, +ERASE, 94410362142720, 94410362142720, +STORE, 94410362142720, 94410362150911, +STORE, 94410362150912, 94410362155007, +ERASE, 140351954157568, 140351954157568, +STORE, 140351954157568, 140351954161663, +STORE, 140351954161664, 140351954165759, +ERASE, 47280840998912, 47280840998912, +STORE, 94410379456512, 94410379591679, +STORE, 140737488347136, 140737488351231, +STORE, 140732946362368, 140737488351231, +ERASE, 140732946362368, 140732946362368, +STORE, 140732946362368, 140732946366463, +STORE, 94352937934848, 94352938106879, +ERASE, 94352937934848, 94352937934848, +STORE, 94352937934848, 94352937951231, +STORE, 94352937951232, 94352938106879, +ERASE, 94352937951232, 94352937951232, +STORE, 94352937951232, 94352938053631, +STORE, 94352938053632, 94352938094591, +STORE, 94352938094592, 94352938106879, +STORE, 140595518742528, 140595518914559, +ERASE, 140595518742528, 140595518742528, +STORE, 140595518742528, 140595518746623, +STORE, 140595518746624, 140595518914559, +ERASE, 140595518746624, 140595518746624, +STORE, 140595518746624, 140595518869503, +STORE, 140595518869504, 140595518902271, +STORE, 140595518902272, 140595518910463, +STORE, 140595518910464, 140595518914559, +STORE, 140732947468288, 140732947472383, +STORE, 140732947456000, 140732947468287, +STORE, 47037276254208, 47037276262399, +STORE, 47037276262400, 47037276270591, +STORE, 47037276270592, 47037276434431, +ERASE, 47037276270592, 47037276270592, +STORE, 47037276270592, 47037276282879, +STORE, 47037276282880, 47037276434431, +STORE, 47037276381184, 47037276434431, +STORE, 47037276282880, 47037276381183, +ERASE, 47037276282880, 47037276282880, +STORE, 47037276282880, 47037276381183, +STORE, 47037276426240, 47037276434431, +STORE, 47037276381184, 47037276426239, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276434431, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276426239, +STORE, 47037276426240, 47037276434431, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276434431, +STORE, 47037276434432, 47037279485951, +STORE, 47037276979200, 47037279485951, +STORE, 47037276434432, 47037276979199, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037279264767, +STORE, 47037279264768, 47037279485951, +STORE, 47037278674944, 47037279264767, +STORE, 47037276979200, 47037278674943, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037278674943, +STORE, 47037279260672, 47037279264767, +STORE, 47037278674944, 47037279260671, +ERASE, 47037278674944, 47037278674944, +STORE, 47037278674944, 47037279260671, +STORE, 47037279469568, 47037279485951, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279469568, 47037279469568, +STORE, 47037279469568, 47037279485951, +STORE, 47037279485952, 47037281325055, +STORE, 47037279625216, 47037281325055, +STORE, 47037279485952, 47037279625215, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037281284095, +STORE, 47037281284096, 47037281325055, +STORE, 47037280968704, 47037281284095, +STORE, 47037279625216, 47037280968703, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037280968703, +STORE, 47037281280000, 47037281284095, +STORE, 47037280968704, 47037281279999, +ERASE, 47037280968704, 47037280968704, +STORE, 47037280968704, 47037281279999, +STORE, 47037281308672, 47037281325055, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281308672, 47037281308672, +STORE, 47037281308672, 47037281325055, +STORE, 47037281325056, 47037281460223, +ERASE, 47037281325056, 47037281325056, +STORE, 47037281325056, 47037281349631, +STORE, 47037281349632, 47037281460223, +STORE, 47037281411072, 47037281460223, +STORE, 47037281349632, 47037281411071, +ERASE, 47037281349632, 47037281349632, +STORE, 47037281349632, 47037281411071, +STORE, 47037281435648, 47037281460223, +STORE, 47037281411072, 47037281435647, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281460223, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281435647, +STORE, 47037281435648, 47037281460223, +STORE, 47037281443840, 47037281460223, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281443840, 47037281443840, +STORE, 47037281443840, 47037281460223, +STORE, 47037281460224, 47037281480703, +ERASE, 47037281460224, 47037281460224, +STORE, 47037281460224, 47037281464319, +STORE, 47037281464320, 47037281480703, +STORE, 47037281468416, 47037281480703, +STORE, 47037281464320, 47037281468415, +ERASE, 47037281464320, 47037281464320, +STORE, 47037281464320, 47037281468415, +STORE, 47037281472512, 47037281480703, +STORE, 47037281468416, 47037281472511, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281480703, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281472511, +STORE, 47037281472512, 47037281480703, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281480703, +STORE, 47037281480704, 47037281488895, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281300479, +STORE, 47037281300480, 47037281308671, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281476607, +STORE, 47037281476608, 47037281480703, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281439743, +STORE, 47037281439744, 47037281443839, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279461375, +STORE, 47037279461376, 47037279469567, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276430335, +STORE, 47037276430336, 47037276434431, +ERASE, 94352938094592, 94352938094592, +STORE, 94352938094592, 94352938102783, +STORE, 94352938102784, 94352938106879, +ERASE, 140595518902272, 140595518902272, +STORE, 140595518902272, 140595518906367, +STORE, 140595518906368, 140595518910463, +ERASE, 47037276254208, 47037276254208, +STORE, 94352938438656, 94352938573823, +STORE, 140737488347136, 140737488351231, +STORE, 140733506027520, 140737488351231, +ERASE, 140733506027520, 140733506027520, +STORE, 140733506027520, 140733506031615, +STORE, 94150123073536, 94150123245567, +ERASE, 94150123073536, 94150123073536, +STORE, 94150123073536, 94150123089919, +STORE, 94150123089920, 94150123245567, +ERASE, 94150123089920, 94150123089920, +STORE, 94150123089920, 94150123192319, +STORE, 94150123192320, 94150123233279, +STORE, 94150123233280, 94150123245567, +STORE, 140081290375168, 140081290547199, +ERASE, 140081290375168, 140081290375168, +STORE, 140081290375168, 140081290379263, +STORE, 140081290379264, 140081290547199, +ERASE, 140081290379264, 140081290379264, +STORE, 140081290379264, 140081290502143, +STORE, 140081290502144, 140081290534911, +STORE, 140081290534912, 140081290543103, +STORE, 140081290543104, 140081290547199, +STORE, 140733506707456, 140733506711551, +STORE, 140733506695168, 140733506707455, +STORE, 47551504621568, 47551504629759, +STORE, 47551504629760, 47551504637951, +STORE, 47551504637952, 47551504801791, +ERASE, 47551504637952, 47551504637952, +STORE, 47551504637952, 47551504650239, +STORE, 47551504650240, 47551504801791, +STORE, 47551504748544, 47551504801791, +STORE, 47551504650240, 47551504748543, +ERASE, 47551504650240, 47551504650240, +STORE, 47551504650240, 47551504748543, +STORE, 47551504793600, 47551504801791, +STORE, 47551504748544, 47551504793599, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504801791, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504793599, +STORE, 47551504793600, 47551504801791, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504801791, +STORE, 47551504801792, 47551507853311, +STORE, 47551505346560, 47551507853311, +STORE, 47551504801792, 47551505346559, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507632127, +STORE, 47551507632128, 47551507853311, +STORE, 47551507042304, 47551507632127, +STORE, 47551505346560, 47551507042303, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507042303, +STORE, 47551507628032, 47551507632127, +STORE, 47551507042304, 47551507628031, +ERASE, 47551507042304, 47551507042304, +STORE, 47551507042304, 47551507628031, +STORE, 47551507836928, 47551507853311, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507836928, 47551507836928, +STORE, 47551507836928, 47551507853311, +STORE, 47551507853312, 47551509692415, +STORE, 47551507992576, 47551509692415, +STORE, 47551507853312, 47551507992575, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509651455, +STORE, 47551509651456, 47551509692415, +STORE, 47551509336064, 47551509651455, +STORE, 47551507992576, 47551509336063, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509336063, +STORE, 47551509647360, 47551509651455, +STORE, 47551509336064, 47551509647359, +ERASE, 47551509336064, 47551509336064, +STORE, 47551509336064, 47551509647359, +STORE, 47551509676032, 47551509692415, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509676032, 47551509676032, +STORE, 47551509676032, 47551509692415, +STORE, 47551509692416, 47551509827583, +ERASE, 47551509692416, 47551509692416, +STORE, 47551509692416, 47551509716991, +STORE, 47551509716992, 47551509827583, +STORE, 47551509778432, 47551509827583, +STORE, 47551509716992, 47551509778431, +ERASE, 47551509716992, 47551509716992, +STORE, 47551509716992, 47551509778431, +STORE, 47551509803008, 47551509827583, +STORE, 47551509778432, 47551509803007, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509827583, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509803007, +STORE, 47551509803008, 47551509827583, +STORE, 47551509811200, 47551509827583, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509811200, 47551509811200, +STORE, 47551509811200, 47551509827583, +STORE, 47551509827584, 47551509848063, +ERASE, 47551509827584, 47551509827584, +STORE, 47551509827584, 47551509831679, +STORE, 47551509831680, 47551509848063, +STORE, 47551509835776, 47551509848063, +STORE, 47551509831680, 47551509835775, +ERASE, 47551509831680, 47551509831680, +STORE, 47551509831680, 47551509835775, +STORE, 47551509839872, 47551509848063, +STORE, 47551509835776, 47551509839871, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509848063, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509839871, +STORE, 47551509839872, 47551509848063, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509848063, +STORE, 47551509848064, 47551509856255, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509667839, +STORE, 47551509667840, 47551509676031, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509843967, +STORE, 47551509843968, 47551509848063, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509807103, +STORE, 47551509807104, 47551509811199, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507828735, +STORE, 47551507828736, 47551507836927, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504797695, +STORE, 47551504797696, 47551504801791, +ERASE, 94150123233280, 94150123233280, +STORE, 94150123233280, 94150123241471, +STORE, 94150123241472, 94150123245567, +ERASE, 140081290534912, 140081290534912, +STORE, 140081290534912, 140081290539007, +STORE, 140081290539008, 140081290543103, +ERASE, 47551504621568, 47551504621568, +STORE, 94150148112384, 94150148247551, +STORE, 140737488347136, 140737488351231, +STORE, 140734389334016, 140737488351231, +ERASE, 140734389334016, 140734389334016, +STORE, 140734389334016, 140734389338111, +STORE, 94844636606464, 94844636778495, +ERASE, 94844636606464, 94844636606464, +STORE, 94844636606464, 94844636622847, +STORE, 94844636622848, 94844636778495, +ERASE, 94844636622848, 94844636622848, +STORE, 94844636622848, 94844636725247, +STORE, 94844636725248, 94844636766207, +STORE, 94844636766208, 94844636778495, +STORE, 139922765217792, 139922765389823, +ERASE, 139922765217792, 139922765217792, +STORE, 139922765217792, 139922765221887, +STORE, 139922765221888, 139922765389823, +ERASE, 139922765221888, 139922765221888, +STORE, 139922765221888, 139922765344767, +STORE, 139922765344768, 139922765377535, +STORE, 139922765377536, 139922765385727, +STORE, 139922765385728, 139922765389823, +STORE, 140734389678080, 140734389682175, +STORE, 140734389665792, 140734389678079, +STORE, 47710029778944, 47710029787135, +STORE, 47710029787136, 47710029795327, +STORE, 47710029795328, 47710029959167, +ERASE, 47710029795328, 47710029795328, +STORE, 47710029795328, 47710029807615, +STORE, 47710029807616, 47710029959167, +STORE, 47710029905920, 47710029959167, +STORE, 47710029807616, 47710029905919, +ERASE, 47710029807616, 47710029807616, +STORE, 47710029807616, 47710029905919, +STORE, 47710029950976, 47710029959167, +STORE, 47710029905920, 47710029950975, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029959167, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029950975, +STORE, 47710029950976, 47710029959167, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029959167, +STORE, 47710029959168, 47710033010687, +STORE, 47710030503936, 47710033010687, +STORE, 47710029959168, 47710030503935, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032789503, +STORE, 47710032789504, 47710033010687, +STORE, 47710032199680, 47710032789503, +STORE, 47710030503936, 47710032199679, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032199679, +STORE, 47710032785408, 47710032789503, +STORE, 47710032199680, 47710032785407, +ERASE, 47710032199680, 47710032199680, +STORE, 47710032199680, 47710032785407, +STORE, 47710032994304, 47710033010687, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032994304, 47710032994304, +STORE, 47710032994304, 47710033010687, +STORE, 47710033010688, 47710034849791, +STORE, 47710033149952, 47710034849791, +STORE, 47710033010688, 47710033149951, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034808831, +STORE, 47710034808832, 47710034849791, +STORE, 47710034493440, 47710034808831, +STORE, 47710033149952, 47710034493439, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034493439, +STORE, 47710034804736, 47710034808831, +STORE, 47710034493440, 47710034804735, +ERASE, 47710034493440, 47710034493440, +STORE, 47710034493440, 47710034804735, +STORE, 47710034833408, 47710034849791, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034833408, 47710034833408, +STORE, 47710034833408, 47710034849791, +STORE, 47710034849792, 47710034984959, +ERASE, 47710034849792, 47710034849792, +STORE, 47710034849792, 47710034874367, +STORE, 47710034874368, 47710034984959, +STORE, 47710034935808, 47710034984959, +STORE, 47710034874368, 47710034935807, +ERASE, 47710034874368, 47710034874368, +STORE, 47710034874368, 47710034935807, +STORE, 47710034960384, 47710034984959, +STORE, 47710034935808, 47710034960383, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034984959, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034960383, +STORE, 47710034960384, 47710034984959, +STORE, 47710034968576, 47710034984959, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034968576, 47710034968576, +STORE, 47710034968576, 47710034984959, +STORE, 47710034984960, 47710035005439, +ERASE, 47710034984960, 47710034984960, +STORE, 47710034984960, 47710034989055, +STORE, 47710034989056, 47710035005439, +STORE, 47710034993152, 47710035005439, +STORE, 47710034989056, 47710034993151, +ERASE, 47710034989056, 47710034989056, +STORE, 47710034989056, 47710034993151, +STORE, 47710034997248, 47710035005439, +STORE, 47710034993152, 47710034997247, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710035005439, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710034997247, +STORE, 47710034997248, 47710035005439, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035005439, +STORE, 47710035005440, 47710035013631, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034825215, +STORE, 47710034825216, 47710034833407, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035001343, +STORE, 47710035001344, 47710035005439, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034964479, +STORE, 47710034964480, 47710034968575, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032986111, +STORE, 47710032986112, 47710032994303, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029955071, +STORE, 47710029955072, 47710029959167, +ERASE, 94844636766208, 94844636766208, +STORE, 94844636766208, 94844636774399, +STORE, 94844636774400, 94844636778495, +ERASE, 139922765377536, 139922765377536, +STORE, 139922765377536, 139922765381631, +STORE, 139922765381632, 139922765385727, +ERASE, 47710029778944, 47710029778944, +STORE, 94844641775616, 94844641910783, +STORE, 140737488347136, 140737488351231, +STORE, 140732213886976, 140737488351231, +ERASE, 140732213886976, 140732213886976, +STORE, 140732213886976, 140732213891071, +STORE, 94240508887040, 94240509059071, +ERASE, 94240508887040, 94240508887040, +STORE, 94240508887040, 94240508903423, +STORE, 94240508903424, 94240509059071, +ERASE, 94240508903424, 94240508903424, +STORE, 94240508903424, 94240509005823, +STORE, 94240509005824, 94240509046783, +STORE, 94240509046784, 94240509059071, +STORE, 140275106516992, 140275106689023, +ERASE, 140275106516992, 140275106516992, +STORE, 140275106516992, 140275106521087, +STORE, 140275106521088, 140275106689023, +ERASE, 140275106521088, 140275106521088, +STORE, 140275106521088, 140275106643967, +STORE, 140275106643968, 140275106676735, +STORE, 140275106676736, 140275106684927, +STORE, 140275106684928, 140275106689023, +STORE, 140732213977088, 140732213981183, +STORE, 140732213964800, 140732213977087, +STORE, 47357688479744, 47357688487935, +STORE, 47357688487936, 47357688496127, +STORE, 47357688496128, 47357688659967, +ERASE, 47357688496128, 47357688496128, +STORE, 47357688496128, 47357688508415, +STORE, 47357688508416, 47357688659967, +STORE, 47357688606720, 47357688659967, +STORE, 47357688508416, 47357688606719, +ERASE, 47357688508416, 47357688508416, +STORE, 47357688508416, 47357688606719, +STORE, 47357688651776, 47357688659967, +STORE, 47357688606720, 47357688651775, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688659967, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688651775, +STORE, 47357688651776, 47357688659967, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688659967, +STORE, 47357688659968, 47357691711487, +STORE, 47357689204736, 47357691711487, +STORE, 47357688659968, 47357689204735, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357691490303, +STORE, 47357691490304, 47357691711487, +STORE, 47357690900480, 47357691490303, +STORE, 47357689204736, 47357690900479, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357690900479, +STORE, 47357691486208, 47357691490303, +STORE, 47357690900480, 47357691486207, +ERASE, 47357690900480, 47357690900480, +STORE, 47357690900480, 47357691486207, +STORE, 47357691695104, 47357691711487, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691695104, 47357691695104, +STORE, 47357691695104, 47357691711487, +STORE, 47357691711488, 47357693550591, +STORE, 47357691850752, 47357693550591, +STORE, 47357691711488, 47357691850751, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693509631, +STORE, 47357693509632, 47357693550591, +STORE, 47357693194240, 47357693509631, +STORE, 47357691850752, 47357693194239, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693194239, +STORE, 47357693505536, 47357693509631, +STORE, 47357693194240, 47357693505535, +ERASE, 47357693194240, 47357693194240, +STORE, 47357693194240, 47357693505535, +STORE, 47357693534208, 47357693550591, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693534208, 47357693534208, +STORE, 47357693534208, 47357693550591, +STORE, 47357693550592, 47357693685759, +ERASE, 47357693550592, 47357693550592, +STORE, 47357693550592, 47357693575167, +STORE, 47357693575168, 47357693685759, +STORE, 47357693636608, 47357693685759, +STORE, 47357693575168, 47357693636607, +ERASE, 47357693575168, 47357693575168, +STORE, 47357693575168, 47357693636607, +STORE, 47357693661184, 47357693685759, +STORE, 47357693636608, 47357693661183, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693685759, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693661183, +STORE, 47357693661184, 47357693685759, +STORE, 47357693669376, 47357693685759, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693669376, 47357693669376, +STORE, 47357693669376, 47357693685759, +STORE, 47357693685760, 47357693706239, +ERASE, 47357693685760, 47357693685760, +STORE, 47357693685760, 47357693689855, +STORE, 47357693689856, 47357693706239, +STORE, 47357693693952, 47357693706239, +STORE, 47357693689856, 47357693693951, +ERASE, 47357693689856, 47357693689856, +STORE, 47357693689856, 47357693693951, +STORE, 47357693698048, 47357693706239, +STORE, 47357693693952, 47357693698047, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693706239, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693698047, +STORE, 47357693698048, 47357693706239, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693706239, +STORE, 47357693706240, 47357693714431, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693526015, +STORE, 47357693526016, 47357693534207, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693702143, +STORE, 47357693702144, 47357693706239, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693665279, +STORE, 47357693665280, 47357693669375, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691686911, +STORE, 47357691686912, 47357691695103, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688655871, +STORE, 47357688655872, 47357688659967, +ERASE, 94240509046784, 94240509046784, +STORE, 94240509046784, 94240509054975, +STORE, 94240509054976, 94240509059071, +ERASE, 140275106676736, 140275106676736, +STORE, 140275106676736, 140275106680831, +STORE, 140275106680832, 140275106684927, +ERASE, 47357688479744, 47357688479744, +STORE, 94240518361088, 94240518496255, +STORE, 140737488347136, 140737488351231, +STORE, 140732688277504, 140737488351231, +ERASE, 140732688277504, 140732688277504, +STORE, 140732688277504, 140732688281599, +STORE, 94629171351552, 94629172064255, +ERASE, 94629171351552, 94629171351552, +STORE, 94629171351552, 94629171400703, +STORE, 94629171400704, 94629172064255, +ERASE, 94629171400704, 94629171400704, +STORE, 94629171400704, 94629171945471, +STORE, 94629171945472, 94629172043775, +STORE, 94629172043776, 94629172064255, +STORE, 139770707644416, 139770707816447, +ERASE, 139770707644416, 139770707644416, +STORE, 139770707644416, 139770707648511, +STORE, 139770707648512, 139770707816447, +ERASE, 139770707648512, 139770707648512, +STORE, 139770707648512, 139770707771391, +STORE, 139770707771392, 139770707804159, +STORE, 139770707804160, 139770707812351, +STORE, 139770707812352, 139770707816447, +STORE, 140732689121280, 140732689125375, +STORE, 140732689108992, 140732689121279, +STORE, 47862087352320, 47862087360511, +STORE, 47862087360512, 47862087368703, +STORE, 47862087368704, 47862087475199, +STORE, 47862087385088, 47862087475199, +STORE, 47862087368704, 47862087385087, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087458815, +STORE, 47862087458816, 47862087475199, +STORE, 47862087438336, 47862087458815, +STORE, 47862087385088, 47862087438335, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087438335, +STORE, 47862087454720, 47862087458815, +STORE, 47862087438336, 47862087454719, +ERASE, 47862087438336, 47862087438336, +STORE, 47862087438336, 47862087454719, +STORE, 47862087467008, 47862087475199, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087467008, 47862087467008, +STORE, 47862087467008, 47862087475199, +STORE, 47862087475200, 47862089314303, +STORE, 47862087614464, 47862089314303, +STORE, 47862087475200, 47862087614463, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862089273343, +STORE, 47862089273344, 47862089314303, +STORE, 47862088957952, 47862089273343, +STORE, 47862087614464, 47862088957951, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862088957951, +STORE, 47862089269248, 47862089273343, +STORE, 47862088957952, 47862089269247, +ERASE, 47862088957952, 47862088957952, +STORE, 47862088957952, 47862089269247, +STORE, 47862089297920, 47862089314303, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089297920, 47862089297920, +STORE, 47862089297920, 47862089314303, +STORE, 47862089297920, 47862089326591, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089289727, +STORE, 47862089289728, 47862089297919, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087462911, +STORE, 47862087462912, 47862087467007, +ERASE, 94629172043776, 94629172043776, +STORE, 94629172043776, 94629172060159, +STORE, 94629172060160, 94629172064255, +ERASE, 139770707804160, 139770707804160, +STORE, 139770707804160, 139770707808255, +STORE, 139770707808256, 139770707812351, +ERASE, 47862087352320, 47862087352320, +STORE, 94629197533184, 94629197668351, +STORE, 140737488347136, 140737488351231, +STORE, 140727540711424, 140737488351231, +ERASE, 140727540711424, 140727540711424, +STORE, 140727540711424, 140727540715519, +STORE, 94299865313280, 94299866025983, +ERASE, 94299865313280, 94299865313280, +STORE, 94299865313280, 94299865362431, +STORE, 94299865362432, 94299866025983, +ERASE, 94299865362432, 94299865362432, +STORE, 94299865362432, 94299865907199, +STORE, 94299865907200, 94299866005503, +STORE, 94299866005504, 94299866025983, +STORE, 140680268763136, 140680268935167, +ERASE, 140680268763136, 140680268763136, +STORE, 140680268763136, 140680268767231, +STORE, 140680268767232, 140680268935167, +ERASE, 140680268767232, 140680268767232, +STORE, 140680268767232, 140680268890111, +STORE, 140680268890112, 140680268922879, +STORE, 140680268922880, 140680268931071, +STORE, 140680268931072, 140680268935167, +STORE, 140727541424128, 140727541428223, +STORE, 140727541411840, 140727541424127, +STORE, 46952526233600, 46952526241791, +STORE, 46952526241792, 46952526249983, +STORE, 46952526249984, 46952526356479, +STORE, 46952526266368, 46952526356479, +STORE, 46952526249984, 46952526266367, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526340095, +STORE, 46952526340096, 46952526356479, +STORE, 46952526319616, 46952526340095, +STORE, 46952526266368, 46952526319615, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526319615, +STORE, 46952526336000, 46952526340095, +STORE, 46952526319616, 46952526335999, +ERASE, 46952526319616, 46952526319616, +STORE, 46952526319616, 46952526335999, +STORE, 46952526348288, 46952526356479, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526348288, 46952526348288, +STORE, 46952526348288, 46952526356479, +STORE, 46952526356480, 46952528195583, +STORE, 46952526495744, 46952528195583, +STORE, 46952526356480, 46952526495743, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952528154623, +STORE, 46952528154624, 46952528195583, +STORE, 46952527839232, 46952528154623, +STORE, 46952526495744, 46952527839231, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952527839231, +STORE, 46952528150528, 46952528154623, +STORE, 46952527839232, 46952528150527, +ERASE, 46952527839232, 46952527839232, +STORE, 46952527839232, 46952528150527, +STORE, 46952528179200, 46952528195583, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528179200, 46952528179200, +STORE, 46952528179200, 46952528195583, +STORE, 46952528179200, 46952528207871, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528171007, +STORE, 46952528171008, 46952528179199, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526344191, +STORE, 46952526344192, 46952526348287, +ERASE, 94299866005504, 94299866005504, +STORE, 94299866005504, 94299866021887, +STORE, 94299866021888, 94299866025983, +ERASE, 140680268922880, 140680268922880, +STORE, 140680268922880, 140680268926975, +STORE, 140680268926976, 140680268931071, +ERASE, 46952526233600, 46952526233600, +STORE, 140737488347136, 140737488351231, +STORE, 140722874793984, 140737488351231, +ERASE, 140722874793984, 140722874793984, +STORE, 140722874793984, 140722874798079, +STORE, 94448916213760, 94448916926463, +ERASE, 94448916213760, 94448916213760, +STORE, 94448916213760, 94448916262911, +STORE, 94448916262912, 94448916926463, +ERASE, 94448916262912, 94448916262912, +STORE, 94448916262912, 94448916807679, +STORE, 94448916807680, 94448916905983, +STORE, 94448916905984, 94448916926463, +STORE, 140389117046784, 140389117218815, +ERASE, 140389117046784, 140389117046784, +STORE, 140389117046784, 140389117050879, +STORE, 140389117050880, 140389117218815, +ERASE, 140389117050880, 140389117050880, +STORE, 140389117050880, 140389117173759, +STORE, 140389117173760, 140389117206527, +STORE, 140389117206528, 140389117214719, +STORE, 140389117214720, 140389117218815, +STORE, 140722875297792, 140722875301887, +STORE, 140722875285504, 140722875297791, +STORE, 47243677949952, 47243677958143, +STORE, 47243677958144, 47243677966335, +STORE, 47243677966336, 47243678072831, +STORE, 47243677982720, 47243678072831, +STORE, 47243677966336, 47243677982719, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678056447, +STORE, 47243678056448, 47243678072831, +STORE, 47243678035968, 47243678056447, +STORE, 47243677982720, 47243678035967, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678035967, +STORE, 47243678052352, 47243678056447, +STORE, 47243678035968, 47243678052351, +ERASE, 47243678035968, 47243678035968, +STORE, 47243678035968, 47243678052351, +STORE, 47243678064640, 47243678072831, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678064640, 47243678064640, +STORE, 47243678064640, 47243678072831, +STORE, 47243678072832, 47243679911935, +STORE, 47243678212096, 47243679911935, +STORE, 47243678072832, 47243678212095, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679870975, +STORE, 47243679870976, 47243679911935, +STORE, 47243679555584, 47243679870975, +STORE, 47243678212096, 47243679555583, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679555583, +STORE, 47243679866880, 47243679870975, +STORE, 47243679555584, 47243679866879, +ERASE, 47243679555584, 47243679555584, +STORE, 47243679555584, 47243679866879, +STORE, 47243679895552, 47243679911935, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679895552, 47243679895552, +STORE, 47243679895552, 47243679911935, +STORE, 47243679895552, 47243679924223, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679887359, +STORE, 47243679887360, 47243679895551, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678060543, +STORE, 47243678060544, 47243678064639, +ERASE, 94448916905984, 94448916905984, +STORE, 94448916905984, 94448916922367, +STORE, 94448916922368, 94448916926463, +ERASE, 140389117206528, 140389117206528, +STORE, 140389117206528, 140389117210623, +STORE, 140389117210624, 140389117214719, +ERASE, 47243677949952, 47243677949952, +STORE, 140737488347136, 140737488351231, +STORE, 140733068505088, 140737488351231, +ERASE, 140733068505088, 140733068505088, +STORE, 140733068505088, 140733068509183, +STORE, 94207145750528, 94207146463231, +ERASE, 94207145750528, 94207145750528, +STORE, 94207145750528, 94207145799679, +STORE, 94207145799680, 94207146463231, +ERASE, 94207145799680, 94207145799680, +STORE, 94207145799680, 94207146344447, +STORE, 94207146344448, 94207146442751, +STORE, 94207146442752, 94207146463231, +STORE, 140684504911872, 140684505083903, +ERASE, 140684504911872, 140684504911872, +STORE, 140684504911872, 140684504915967, +STORE, 140684504915968, 140684505083903, +ERASE, 140684504915968, 140684504915968, +STORE, 140684504915968, 140684505038847, +STORE, 140684505038848, 140684505071615, +STORE, 140684505071616, 140684505079807, +STORE, 140684505079808, 140684505083903, +STORE, 140733068607488, 140733068611583, +STORE, 140733068595200, 140733068607487, +STORE, 46948290084864, 46948290093055, +STORE, 46948290093056, 46948290101247, +STORE, 46948290101248, 46948290207743, +STORE, 46948290117632, 46948290207743, +STORE, 46948290101248, 46948290117631, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290191359, +STORE, 46948290191360, 46948290207743, +STORE, 46948290170880, 46948290191359, +STORE, 46948290117632, 46948290170879, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290170879, +STORE, 46948290187264, 46948290191359, +STORE, 46948290170880, 46948290187263, +ERASE, 46948290170880, 46948290170880, +STORE, 46948290170880, 46948290187263, +STORE, 46948290199552, 46948290207743, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290199552, 46948290199552, +STORE, 46948290199552, 46948290207743, +STORE, 46948290207744, 46948292046847, +STORE, 46948290347008, 46948292046847, +STORE, 46948290207744, 46948290347007, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948292005887, +STORE, 46948292005888, 46948292046847, +STORE, 46948291690496, 46948292005887, +STORE, 46948290347008, 46948291690495, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948291690495, +STORE, 46948292001792, 46948292005887, +STORE, 46948291690496, 46948292001791, +ERASE, 46948291690496, 46948291690496, +STORE, 46948291690496, 46948292001791, +STORE, 46948292030464, 46948292046847, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292030464, 46948292030464, +STORE, 46948292030464, 46948292046847, +STORE, 46948292030464, 46948292059135, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292022271, +STORE, 46948292022272, 46948292030463, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290195455, +STORE, 46948290195456, 46948290199551, +ERASE, 94207146442752, 94207146442752, +STORE, 94207146442752, 94207146459135, +STORE, 94207146459136, 94207146463231, +ERASE, 140684505071616, 140684505071616, +STORE, 140684505071616, 140684505075711, +STORE, 140684505075712, 140684505079807, +ERASE, 46948290084864, 46948290084864, +STORE, 140737488347136, 140737488351231, +STORE, 140726367158272, 140737488351231, +ERASE, 140726367158272, 140726367158272, +STORE, 140726367158272, 140726367162367, +STORE, 94436124106752, 94436124819455, +ERASE, 94436124106752, 94436124106752, +STORE, 94436124106752, 94436124155903, +STORE, 94436124155904, 94436124819455, +ERASE, 94436124155904, 94436124155904, +STORE, 94436124155904, 94436124700671, +STORE, 94436124700672, 94436124798975, +STORE, 94436124798976, 94436124819455, +STORE, 140049025044480, 140049025216511, +ERASE, 140049025044480, 140049025044480, +STORE, 140049025044480, 140049025048575, +STORE, 140049025048576, 140049025216511, +ERASE, 140049025048576, 140049025048576, +STORE, 140049025048576, 140049025171455, +STORE, 140049025171456, 140049025204223, +STORE, 140049025204224, 140049025212415, +STORE, 140049025212416, 140049025216511, +STORE, 140726367256576, 140726367260671, +STORE, 140726367244288, 140726367256575, +STORE, 47583769952256, 47583769960447, +STORE, 47583769960448, 47583769968639, +STORE, 47583769968640, 47583770075135, +STORE, 47583769985024, 47583770075135, +STORE, 47583769968640, 47583769985023, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770058751, +STORE, 47583770058752, 47583770075135, +STORE, 47583770038272, 47583770058751, +STORE, 47583769985024, 47583770038271, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770038271, +STORE, 47583770054656, 47583770058751, +STORE, 47583770038272, 47583770054655, +ERASE, 47583770038272, 47583770038272, +STORE, 47583770038272, 47583770054655, +STORE, 47583770066944, 47583770075135, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770066944, 47583770066944, +STORE, 47583770066944, 47583770075135, +STORE, 47583770075136, 47583771914239, +STORE, 47583770214400, 47583771914239, +STORE, 47583770075136, 47583770214399, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771873279, +STORE, 47583771873280, 47583771914239, +STORE, 47583771557888, 47583771873279, +STORE, 47583770214400, 47583771557887, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771557887, +STORE, 47583771869184, 47583771873279, +STORE, 47583771557888, 47583771869183, +ERASE, 47583771557888, 47583771557888, +STORE, 47583771557888, 47583771869183, +STORE, 47583771897856, 47583771914239, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771897856, 47583771897856, +STORE, 47583771897856, 47583771914239, +STORE, 47583771897856, 47583771926527, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771889663, +STORE, 47583771889664, 47583771897855, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770062847, +STORE, 47583770062848, 47583770066943, +ERASE, 94436124798976, 94436124798976, +STORE, 94436124798976, 94436124815359, +STORE, 94436124815360, 94436124819455, +ERASE, 140049025204224, 140049025204224, +STORE, 140049025204224, 140049025208319, +STORE, 140049025208320, 140049025212415, +ERASE, 47583769952256, 47583769952256, +STORE, 140737488347136, 140737488351231, +STORE, 140727116099584, 140737488351231, +ERASE, 140727116099584, 140727116099584, +STORE, 140727116099584, 140727116103679, +STORE, 94166319734784, 94166320447487, +ERASE, 94166319734784, 94166319734784, +STORE, 94166319734784, 94166319783935, +STORE, 94166319783936, 94166320447487, +ERASE, 94166319783936, 94166319783936, +STORE, 94166319783936, 94166320328703, +STORE, 94166320328704, 94166320427007, +STORE, 94166320427008, 94166320447487, +STORE, 139976559542272, 139976559714303, +ERASE, 139976559542272, 139976559542272, +STORE, 139976559542272, 139976559546367, +STORE, 139976559546368, 139976559714303, +ERASE, 139976559546368, 139976559546368, +STORE, 139976559546368, 139976559669247, +STORE, 139976559669248, 139976559702015, +STORE, 139976559702016, 139976559710207, +STORE, 139976559710208, 139976559714303, +STORE, 140727116222464, 140727116226559, +STORE, 140727116210176, 140727116222463, +STORE, 47656235454464, 47656235462655, +STORE, 47656235462656, 47656235470847, +STORE, 47656235470848, 47656235577343, +STORE, 47656235487232, 47656235577343, +STORE, 47656235470848, 47656235487231, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235560959, +STORE, 47656235560960, 47656235577343, +STORE, 47656235540480, 47656235560959, +STORE, 47656235487232, 47656235540479, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235540479, +STORE, 47656235556864, 47656235560959, +STORE, 47656235540480, 47656235556863, +ERASE, 47656235540480, 47656235540480, +STORE, 47656235540480, 47656235556863, +STORE, 47656235569152, 47656235577343, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235569152, 47656235569152, +STORE, 47656235569152, 47656235577343, +STORE, 47656235577344, 47656237416447, +STORE, 47656235716608, 47656237416447, +STORE, 47656235577344, 47656235716607, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237375487, +STORE, 47656237375488, 47656237416447, +STORE, 47656237060096, 47656237375487, +STORE, 47656235716608, 47656237060095, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237060095, +STORE, 47656237371392, 47656237375487, +STORE, 47656237060096, 47656237371391, +ERASE, 47656237060096, 47656237060096, +STORE, 47656237060096, 47656237371391, +STORE, 47656237400064, 47656237416447, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237400064, 47656237400064, +STORE, 47656237400064, 47656237416447, +STORE, 47656237400064, 47656237428735, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237391871, +STORE, 47656237391872, 47656237400063, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235565055, +STORE, 47656235565056, 47656235569151, +ERASE, 94166320427008, 94166320427008, +STORE, 94166320427008, 94166320443391, +STORE, 94166320443392, 94166320447487, +ERASE, 139976559702016, 139976559702016, +STORE, 139976559702016, 139976559706111, +STORE, 139976559706112, 139976559710207, +ERASE, 47656235454464, 47656235454464, +STORE, 94166332153856, 94166332289023, +STORE, 140737488347136, 140737488351231, +STORE, 140726412816384, 140737488351231, +ERASE, 140726412816384, 140726412816384, +STORE, 140726412816384, 140726412820479, +STORE, 94094884507648, 94094885220351, +ERASE, 94094884507648, 94094884507648, +STORE, 94094884507648, 94094884556799, +STORE, 94094884556800, 94094885220351, +ERASE, 94094884556800, 94094884556800, +STORE, 94094884556800, 94094885101567, +STORE, 94094885101568, 94094885199871, +STORE, 94094885199872, 94094885220351, +STORE, 139773773938688, 139773774110719, +ERASE, 139773773938688, 139773773938688, +STORE, 139773773938688, 139773773942783, +STORE, 139773773942784, 139773774110719, +ERASE, 139773773942784, 139773773942784, +STORE, 139773773942784, 139773774065663, +STORE, 139773774065664, 139773774098431, +STORE, 139773774098432, 139773774106623, +STORE, 139773774106624, 139773774110719, +STORE, 140726412963840, 140726412967935, +STORE, 140726412951552, 140726412963839, +STORE, 47859021058048, 47859021066239, +STORE, 47859021066240, 47859021074431, +STORE, 47859021074432, 47859021180927, +STORE, 47859021090816, 47859021180927, +STORE, 47859021074432, 47859021090815, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021164543, +STORE, 47859021164544, 47859021180927, +STORE, 47859021144064, 47859021164543, +STORE, 47859021090816, 47859021144063, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021144063, +STORE, 47859021160448, 47859021164543, +STORE, 47859021144064, 47859021160447, +ERASE, 47859021144064, 47859021144064, +STORE, 47859021144064, 47859021160447, +STORE, 47859021172736, 47859021180927, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021172736, 47859021172736, +STORE, 47859021172736, 47859021180927, +STORE, 47859021180928, 47859023020031, +STORE, 47859021320192, 47859023020031, +STORE, 47859021180928, 47859021320191, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022979071, +STORE, 47859022979072, 47859023020031, +STORE, 47859022663680, 47859022979071, +STORE, 47859021320192, 47859022663679, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022663679, +STORE, 47859022974976, 47859022979071, +STORE, 47859022663680, 47859022974975, +ERASE, 47859022663680, 47859022663680, +STORE, 47859022663680, 47859022974975, +STORE, 47859023003648, 47859023020031, +STORE, 47859022979072, 47859023003647, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859023003647, +ERASE, 47859023003648, 47859023003648, +STORE, 47859023003648, 47859023020031, +STORE, 47859023003648, 47859023032319, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859022995455, +STORE, 47859022995456, 47859023003647, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021168639, +STORE, 47859021168640, 47859021172735, +ERASE, 94094885199872, 94094885199872, +STORE, 94094885199872, 94094885216255, +STORE, 94094885216256, 94094885220351, +ERASE, 139773774098432, 139773774098432, +STORE, 139773774098432, 139773774102527, +STORE, 139773774102528, 139773774106623, +ERASE, 47859021058048, 47859021058048, +STORE, 94094901108736, 94094901243903, +STORE, 140737488347136, 140737488351231, +STORE, 140736567963648, 140737488351231, +ERASE, 140736567963648, 140736567963648, +STORE, 140736567963648, 140736567967743, +STORE, 94924425748480, 94924426461183, +ERASE, 94924425748480, 94924425748480, +STORE, 94924425748480, 94924425797631, +STORE, 94924425797632, 94924426461183, +ERASE, 94924425797632, 94924425797632, +STORE, 94924425797632, 94924426342399, +STORE, 94924426342400, 94924426440703, +STORE, 94924426440704, 94924426461183, +STORE, 140042126319616, 140042126491647, +ERASE, 140042126319616, 140042126319616, +STORE, 140042126319616, 140042126323711, +STORE, 140042126323712, 140042126491647, +ERASE, 140042126323712, 140042126323712, +STORE, 140042126323712, 140042126446591, +STORE, 140042126446592, 140042126479359, +STORE, 140042126479360, 140042126487551, +STORE, 140042126487552, 140042126491647, +STORE, 140736568672256, 140736568676351, +STORE, 140736568659968, 140736568672255, +STORE, 47590668677120, 47590668685311, +STORE, 47590668685312, 47590668693503, +STORE, 47590668693504, 47590668799999, +STORE, 47590668709888, 47590668799999, +STORE, 47590668693504, 47590668709887, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668783615, +STORE, 47590668783616, 47590668799999, +STORE, 47590668763136, 47590668783615, +STORE, 47590668709888, 47590668763135, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668763135, +STORE, 47590668779520, 47590668783615, +STORE, 47590668763136, 47590668779519, +ERASE, 47590668763136, 47590668763136, +STORE, 47590668763136, 47590668779519, +STORE, 47590668791808, 47590668799999, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668791808, 47590668791808, +STORE, 47590668791808, 47590668799999, +STORE, 47590668800000, 47590670639103, +STORE, 47590668939264, 47590670639103, +STORE, 47590668800000, 47590668939263, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670598143, +STORE, 47590670598144, 47590670639103, +STORE, 47590670282752, 47590670598143, +STORE, 47590668939264, 47590670282751, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670282751, +STORE, 47590670594048, 47590670598143, +STORE, 47590670282752, 47590670594047, +ERASE, 47590670282752, 47590670282752, +STORE, 47590670282752, 47590670594047, +STORE, 47590670622720, 47590670639103, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670622720, 47590670622720, +STORE, 47590670622720, 47590670639103, +STORE, 47590670622720, 47590670651391, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670614527, +STORE, 47590670614528, 47590670622719, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668787711, +STORE, 47590668787712, 47590668791807, +ERASE, 94924426440704, 94924426440704, +STORE, 94924426440704, 94924426457087, +STORE, 94924426457088, 94924426461183, +ERASE, 140042126479360, 140042126479360, +STORE, 140042126479360, 140042126483455, +STORE, 140042126483456, 140042126487551, +ERASE, 47590668677120, 47590668677120, +STORE, 140737488347136, 140737488351231, +STORE, 140733281439744, 140737488351231, +ERASE, 140733281439744, 140733281439744, +STORE, 140733281439744, 140733281443839, +STORE, 94490667069440, 94490667782143, +ERASE, 94490667069440, 94490667069440, +STORE, 94490667069440, 94490667118591, +STORE, 94490667118592, 94490667782143, +ERASE, 94490667118592, 94490667118592, +STORE, 94490667118592, 94490667663359, +STORE, 94490667663360, 94490667761663, +STORE, 94490667761664, 94490667782143, +STORE, 139878215118848, 139878215290879, +ERASE, 139878215118848, 139878215118848, +STORE, 139878215118848, 139878215122943, +STORE, 139878215122944, 139878215290879, +ERASE, 139878215122944, 139878215122944, +STORE, 139878215122944, 139878215245823, +STORE, 139878215245824, 139878215278591, +STORE, 139878215278592, 139878215286783, +STORE, 139878215286784, 139878215290879, +STORE, 140733281464320, 140733281468415, +STORE, 140733281452032, 140733281464319, +STORE, 47754579877888, 47754579886079, +STORE, 47754579886080, 47754579894271, +STORE, 47754579894272, 47754580000767, +STORE, 47754579910656, 47754580000767, +STORE, 47754579894272, 47754579910655, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579984383, +STORE, 47754579984384, 47754580000767, +STORE, 47754579963904, 47754579984383, +STORE, 47754579910656, 47754579963903, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579963903, +STORE, 47754579980288, 47754579984383, +STORE, 47754579963904, 47754579980287, +ERASE, 47754579963904, 47754579963904, +STORE, 47754579963904, 47754579980287, +STORE, 47754579992576, 47754580000767, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579992576, 47754579992576, +STORE, 47754579992576, 47754580000767, +STORE, 47754580000768, 47754581839871, +STORE, 47754580140032, 47754581839871, +STORE, 47754580000768, 47754580140031, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581798911, +STORE, 47754581798912, 47754581839871, +STORE, 47754581483520, 47754581798911, +STORE, 47754580140032, 47754581483519, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581483519, +STORE, 47754581794816, 47754581798911, +STORE, 47754581483520, 47754581794815, +ERASE, 47754581483520, 47754581483520, +STORE, 47754581483520, 47754581794815, +STORE, 47754581823488, 47754581839871, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581823488, 47754581823488, +STORE, 47754581823488, 47754581839871, +STORE, 47754581823488, 47754581852159, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581815295, +STORE, 47754581815296, 47754581823487, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579988479, +STORE, 47754579988480, 47754579992575, +ERASE, 94490667761664, 94490667761664, +STORE, 94490667761664, 94490667778047, +STORE, 94490667778048, 94490667782143, +ERASE, 139878215278592, 139878215278592, +STORE, 139878215278592, 139878215282687, +STORE, 139878215282688, 139878215286783, +ERASE, 47754579877888, 47754579877888, +STORE, 94490669649920, 94490669785087, +STORE, 140737488347136, 140737488351231, +STORE, 140735382188032, 140737488351231, +ERASE, 140735382188032, 140735382188032, +STORE, 140735382188032, 140735382192127, +STORE, 94150181302272, 94150182014975, +ERASE, 94150181302272, 94150181302272, +STORE, 94150181302272, 94150181351423, +STORE, 94150181351424, 94150182014975, +ERASE, 94150181351424, 94150181351424, +STORE, 94150181351424, 94150181896191, +STORE, 94150181896192, 94150181994495, +STORE, 94150181994496, 94150182014975, +STORE, 139679752458240, 139679752630271, +ERASE, 139679752458240, 139679752458240, +STORE, 139679752458240, 139679752462335, +STORE, 139679752462336, 139679752630271, +ERASE, 139679752462336, 139679752462336, +STORE, 139679752462336, 139679752585215, +STORE, 139679752585216, 139679752617983, +STORE, 139679752617984, 139679752626175, +STORE, 139679752626176, 139679752630271, +STORE, 140735382536192, 140735382540287, +STORE, 140735382523904, 140735382536191, +STORE, 47953042538496, 47953042546687, +STORE, 47953042546688, 47953042554879, +STORE, 47953042554880, 47953042661375, +STORE, 47953042571264, 47953042661375, +STORE, 47953042554880, 47953042571263, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042644991, +STORE, 47953042644992, 47953042661375, +STORE, 47953042624512, 47953042644991, +STORE, 47953042571264, 47953042624511, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042624511, +STORE, 47953042640896, 47953042644991, +STORE, 47953042624512, 47953042640895, +ERASE, 47953042624512, 47953042624512, +STORE, 47953042624512, 47953042640895, +STORE, 47953042653184, 47953042661375, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042653184, 47953042653184, +STORE, 47953042653184, 47953042661375, +STORE, 47953042661376, 47953044500479, +STORE, 47953042800640, 47953044500479, +STORE, 47953042661376, 47953042800639, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044459519, +STORE, 47953044459520, 47953044500479, +STORE, 47953044144128, 47953044459519, +STORE, 47953042800640, 47953044144127, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044144127, +STORE, 47953044455424, 47953044459519, +STORE, 47953044144128, 47953044455423, +ERASE, 47953044144128, 47953044144128, +STORE, 47953044144128, 47953044455423, +STORE, 47953044484096, 47953044500479, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044484096, 47953044484096, +STORE, 47953044484096, 47953044500479, +STORE, 47953044484096, 47953044512767, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044475903, +STORE, 47953044475904, 47953044484095, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042649087, +STORE, 47953042649088, 47953042653183, +ERASE, 94150181994496, 94150181994496, +STORE, 94150181994496, 94150182010879, +STORE, 94150182010880, 94150182014975, +ERASE, 139679752617984, 139679752617984, +STORE, 139679752617984, 139679752622079, +STORE, 139679752622080, 139679752626175, +ERASE, 47953042538496, 47953042538496, +STORE, 140737488347136, 140737488351231, +STORE, 140737044123648, 140737488351231, +ERASE, 140737044123648, 140737044123648, +STORE, 140737044123648, 140737044127743, +STORE, 94425324294144, 94425325006847, +ERASE, 94425324294144, 94425324294144, +STORE, 94425324294144, 94425324343295, +STORE, 94425324343296, 94425325006847, +ERASE, 94425324343296, 94425324343296, +STORE, 94425324343296, 94425324888063, +STORE, 94425324888064, 94425324986367, +STORE, 94425324986368, 94425325006847, +STORE, 140382015016960, 140382015188991, +ERASE, 140382015016960, 140382015016960, +STORE, 140382015016960, 140382015021055, +STORE, 140382015021056, 140382015188991, +ERASE, 140382015021056, 140382015021056, +STORE, 140382015021056, 140382015143935, +STORE, 140382015143936, 140382015176703, +STORE, 140382015176704, 140382015184895, +STORE, 140382015184896, 140382015188991, +STORE, 140737045585920, 140737045590015, +STORE, 140737045573632, 140737045585919, +STORE, 47250779979776, 47250779987967, +STORE, 47250779987968, 47250779996159, +STORE, 47250779996160, 47250780102655, +STORE, 47250780012544, 47250780102655, +STORE, 47250779996160, 47250780012543, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780086271, +STORE, 47250780086272, 47250780102655, +STORE, 47250780065792, 47250780086271, +STORE, 47250780012544, 47250780065791, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780065791, +STORE, 47250780082176, 47250780086271, +STORE, 47250780065792, 47250780082175, +ERASE, 47250780065792, 47250780065792, +STORE, 47250780065792, 47250780082175, +STORE, 47250780094464, 47250780102655, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780094464, 47250780094464, +STORE, 47250780094464, 47250780102655, +STORE, 47250780102656, 47250781941759, +STORE, 47250780241920, 47250781941759, +STORE, 47250780102656, 47250780241919, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781900799, +STORE, 47250781900800, 47250781941759, +STORE, 47250781585408, 47250781900799, +STORE, 47250780241920, 47250781585407, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781585407, +STORE, 47250781896704, 47250781900799, +STORE, 47250781585408, 47250781896703, +ERASE, 47250781585408, 47250781585408, +STORE, 47250781585408, 47250781896703, +STORE, 47250781925376, 47250781941759, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781925376, 47250781925376, +STORE, 47250781925376, 47250781941759, +STORE, 47250781925376, 47250781954047, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781917183, +STORE, 47250781917184, 47250781925375, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780090367, +STORE, 47250780090368, 47250780094463, +ERASE, 94425324986368, 94425324986368, +STORE, 94425324986368, 94425325002751, +STORE, 94425325002752, 94425325006847, +ERASE, 140382015176704, 140382015176704, +STORE, 140382015176704, 140382015180799, +STORE, 140382015180800, 140382015184895, +ERASE, 47250779979776, 47250779979776, +STORE, 94425351438336, 94425351573503, +STORE, 140737488347136, 140737488351231, +STORE, 140736801144832, 140737488351231, +ERASE, 140736801144832, 140736801144832, +STORE, 140736801144832, 140736801148927, +STORE, 94629429358592, 94629430071295, +ERASE, 94629429358592, 94629429358592, +STORE, 94629429358592, 94629429407743, +STORE, 94629429407744, 94629430071295, +ERASE, 94629429407744, 94629429407744, +STORE, 94629429407744, 94629429952511, +STORE, 94629429952512, 94629430050815, +STORE, 94629430050816, 94629430071295, +STORE, 139801685483520, 139801685655551, +ERASE, 139801685483520, 139801685483520, +STORE, 139801685483520, 139801685487615, +STORE, 139801685487616, 139801685655551, +ERASE, 139801685487616, 139801685487616, +STORE, 139801685487616, 139801685610495, +STORE, 139801685610496, 139801685643263, +STORE, 139801685643264, 139801685651455, +STORE, 139801685651456, 139801685655551, +STORE, 140736801198080, 140736801202175, +STORE, 140736801185792, 140736801198079, +STORE, 47831109513216, 47831109521407, +STORE, 47831109521408, 47831109529599, +STORE, 47831109529600, 47831109636095, +STORE, 47831109545984, 47831109636095, +STORE, 47831109529600, 47831109545983, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109619711, +STORE, 47831109619712, 47831109636095, +STORE, 47831109599232, 47831109619711, +STORE, 47831109545984, 47831109599231, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109599231, +STORE, 47831109615616, 47831109619711, +STORE, 47831109599232, 47831109615615, +ERASE, 47831109599232, 47831109599232, +STORE, 47831109599232, 47831109615615, +STORE, 47831109627904, 47831109636095, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109627904, 47831109627904, +STORE, 47831109627904, 47831109636095, +STORE, 47831109636096, 47831111475199, +STORE, 47831109775360, 47831111475199, +STORE, 47831109636096, 47831109775359, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111434239, +STORE, 47831111434240, 47831111475199, +STORE, 47831111118848, 47831111434239, +STORE, 47831109775360, 47831111118847, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111118847, +STORE, 47831111430144, 47831111434239, +STORE, 47831111118848, 47831111430143, +ERASE, 47831111118848, 47831111118848, +STORE, 47831111118848, 47831111430143, +STORE, 47831111458816, 47831111475199, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111458816, 47831111458816, +STORE, 47831111458816, 47831111475199, +STORE, 47831111458816, 47831111487487, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111450623, +STORE, 47831111450624, 47831111458815, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109623807, +STORE, 47831109623808, 47831109627903, +ERASE, 94629430050816, 94629430050816, +STORE, 94629430050816, 94629430067199, +STORE, 94629430067200, 94629430071295, +ERASE, 139801685643264, 139801685643264, +STORE, 139801685643264, 139801685647359, +STORE, 139801685647360, 139801685651455, +ERASE, 47831109513216, 47831109513216, +STORE, 140737488347136, 140737488351231, +STORE, 140729419612160, 140737488351231, +ERASE, 140729419612160, 140729419612160, +STORE, 140729419612160, 140729419616255, +STORE, 94443354148864, 94443354861567, +ERASE, 94443354148864, 94443354148864, +STORE, 94443354148864, 94443354198015, +STORE, 94443354198016, 94443354861567, +ERASE, 94443354198016, 94443354198016, +STORE, 94443354198016, 94443354742783, +STORE, 94443354742784, 94443354841087, +STORE, 94443354841088, 94443354861567, +STORE, 139741700038656, 139741700210687, +ERASE, 139741700038656, 139741700038656, +STORE, 139741700038656, 139741700042751, +STORE, 139741700042752, 139741700210687, +ERASE, 139741700042752, 139741700042752, +STORE, 139741700042752, 139741700165631, +STORE, 139741700165632, 139741700198399, +STORE, 139741700198400, 139741700206591, +STORE, 139741700206592, 139741700210687, +STORE, 140729420574720, 140729420578815, +STORE, 140729420562432, 140729420574719, +STORE, 47891094958080, 47891094966271, +STORE, 47891094966272, 47891094974463, +STORE, 47891094974464, 47891095080959, +STORE, 47891094990848, 47891095080959, +STORE, 47891094974464, 47891094990847, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095064575, +STORE, 47891095064576, 47891095080959, +STORE, 47891095044096, 47891095064575, +STORE, 47891094990848, 47891095044095, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095044095, +STORE, 47891095060480, 47891095064575, +STORE, 47891095044096, 47891095060479, +ERASE, 47891095044096, 47891095044096, +STORE, 47891095044096, 47891095060479, +STORE, 47891095072768, 47891095080959, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095072768, 47891095072768, +STORE, 47891095072768, 47891095080959, +STORE, 47891095080960, 47891096920063, +STORE, 47891095220224, 47891096920063, +STORE, 47891095080960, 47891095220223, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096879103, +STORE, 47891096879104, 47891096920063, +STORE, 47891096563712, 47891096879103, +STORE, 47891095220224, 47891096563711, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096563711, +STORE, 47891096875008, 47891096879103, +STORE, 47891096563712, 47891096875007, +ERASE, 47891096563712, 47891096563712, +STORE, 47891096563712, 47891096875007, +STORE, 47891096903680, 47891096920063, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096903680, 47891096903680, +STORE, 47891096903680, 47891096920063, +STORE, 47891096903680, 47891096932351, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096895487, +STORE, 47891096895488, 47891096903679, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095068671, +STORE, 47891095068672, 47891095072767, +ERASE, 94443354841088, 94443354841088, +STORE, 94443354841088, 94443354857471, +STORE, 94443354857472, 94443354861567, +ERASE, 139741700198400, 139741700198400, +STORE, 139741700198400, 139741700202495, +STORE, 139741700202496, 139741700206591, +ERASE, 47891094958080, 47891094958080, +STORE, 94443360825344, 94443360960511, +STORE, 140737488347136, 140737488351231, +STORE, 140722961661952, 140737488351231, +ERASE, 140722961661952, 140722961661952, +STORE, 140722961661952, 140722961666047, +STORE, 94878388944896, 94878389657599, +ERASE, 94878388944896, 94878388944896, +STORE, 94878388944896, 94878388994047, +STORE, 94878388994048, 94878389657599, +ERASE, 94878388994048, 94878388994048, +STORE, 94878388994048, 94878389538815, +STORE, 94878389538816, 94878389637119, +STORE, 94878389637120, 94878389657599, +STORE, 140210690056192, 140210690228223, +ERASE, 140210690056192, 140210690056192, +STORE, 140210690056192, 140210690060287, +STORE, 140210690060288, 140210690228223, +ERASE, 140210690060288, 140210690060288, +STORE, 140210690060288, 140210690183167, +STORE, 140210690183168, 140210690215935, +STORE, 140210690215936, 140210690224127, +STORE, 140210690224128, 140210690228223, +STORE, 140722963148800, 140722963152895, +STORE, 140722963136512, 140722963148799, +STORE, 47422104940544, 47422104948735, +STORE, 47422104948736, 47422104956927, +STORE, 47422104956928, 47422105063423, +STORE, 47422104973312, 47422105063423, +STORE, 47422104956928, 47422104973311, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105047039, +STORE, 47422105047040, 47422105063423, +STORE, 47422105026560, 47422105047039, +STORE, 47422104973312, 47422105026559, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105026559, +STORE, 47422105042944, 47422105047039, +STORE, 47422105026560, 47422105042943, +ERASE, 47422105026560, 47422105026560, +STORE, 47422105026560, 47422105042943, +STORE, 47422105055232, 47422105063423, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105055232, 47422105055232, +STORE, 47422105055232, 47422105063423, +STORE, 47422105063424, 47422106902527, +STORE, 47422105202688, 47422106902527, +STORE, 47422105063424, 47422105202687, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106861567, +STORE, 47422106861568, 47422106902527, +STORE, 47422106546176, 47422106861567, +STORE, 47422105202688, 47422106546175, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106546175, +STORE, 47422106857472, 47422106861567, +STORE, 47422106546176, 47422106857471, +ERASE, 47422106546176, 47422106546176, +STORE, 47422106546176, 47422106857471, +STORE, 47422106886144, 47422106902527, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106886144, 47422106886144, +STORE, 47422106886144, 47422106902527, +STORE, 47422106886144, 47422106914815, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106877951, +STORE, 47422106877952, 47422106886143, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105051135, +STORE, 47422105051136, 47422105055231, +ERASE, 94878389637120, 94878389637120, +STORE, 94878389637120, 94878389653503, +STORE, 94878389653504, 94878389657599, +ERASE, 140210690215936, 140210690215936, +STORE, 140210690215936, 140210690220031, +STORE, 140210690220032, 140210690224127, +ERASE, 47422104940544, 47422104940544, +STORE, 140737488347136, 140737488351231, +STORE, 140727690309632, 140737488351231, +ERASE, 140727690309632, 140727690309632, +STORE, 140727690309632, 140727690313727, +STORE, 94121892208640, 94121892921343, +ERASE, 94121892208640, 94121892208640, +STORE, 94121892208640, 94121892257791, +STORE, 94121892257792, 94121892921343, +ERASE, 94121892257792, 94121892257792, +STORE, 94121892257792, 94121892802559, +STORE, 94121892802560, 94121892900863, +STORE, 94121892900864, 94121892921343, +STORE, 140662438326272, 140662438498303, +ERASE, 140662438326272, 140662438326272, +STORE, 140662438326272, 140662438330367, +STORE, 140662438330368, 140662438498303, +ERASE, 140662438330368, 140662438330368, +STORE, 140662438330368, 140662438453247, +STORE, 140662438453248, 140662438486015, +STORE, 140662438486016, 140662438494207, +STORE, 140662438494208, 140662438498303, +STORE, 140727690379264, 140727690383359, +STORE, 140727690366976, 140727690379263, +STORE, 46970356670464, 46970356678655, +STORE, 46970356678656, 46970356686847, +STORE, 46970356686848, 46970356793343, +STORE, 46970356703232, 46970356793343, +STORE, 46970356686848, 46970356703231, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356776959, +STORE, 46970356776960, 46970356793343, +STORE, 46970356756480, 46970356776959, +STORE, 46970356703232, 46970356756479, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356756479, +STORE, 46970356772864, 46970356776959, +STORE, 46970356756480, 46970356772863, +ERASE, 46970356756480, 46970356756480, +STORE, 46970356756480, 46970356772863, +STORE, 46970356785152, 46970356793343, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356785152, 46970356785152, +STORE, 46970356785152, 46970356793343, +STORE, 46970356793344, 46970358632447, +STORE, 46970356932608, 46970358632447, +STORE, 46970356793344, 46970356932607, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358591487, +STORE, 46970358591488, 46970358632447, +STORE, 46970358276096, 46970358591487, +STORE, 46970356932608, 46970358276095, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358276095, +STORE, 46970358587392, 46970358591487, +STORE, 46970358276096, 46970358587391, +ERASE, 46970358276096, 46970358276096, +STORE, 46970358276096, 46970358587391, +STORE, 46970358616064, 46970358632447, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358616064, 46970358616064, +STORE, 46970358616064, 46970358632447, +STORE, 46970358616064, 46970358644735, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358607871, +STORE, 46970358607872, 46970358616063, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356781055, +STORE, 46970356781056, 46970356785151, +ERASE, 94121892900864, 94121892900864, +STORE, 94121892900864, 94121892917247, +STORE, 94121892917248, 94121892921343, +ERASE, 140662438486016, 140662438486016, +STORE, 140662438486016, 140662438490111, +STORE, 140662438490112, 140662438494207, +ERASE, 46970356670464, 46970356670464, +STORE, 94121898610688, 94121898745855, +STORE, 140737488347136, 140737488351231, +STORE, 140737189351424, 140737488351231, +ERASE, 140737189351424, 140737189351424, +STORE, 140737189351424, 140737189355519, +STORE, 93847948832768, 93847949545471, +ERASE, 93847948832768, 93847948832768, +STORE, 93847948832768, 93847948881919, +STORE, 93847948881920, 93847949545471, +ERASE, 93847948881920, 93847948881920, +STORE, 93847948881920, 93847949426687, +STORE, 93847949426688, 93847949524991, +STORE, 93847949524992, 93847949545471, +STORE, 139698989985792, 139698990157823, +ERASE, 139698989985792, 139698989985792, +STORE, 139698989985792, 139698989989887, +STORE, 139698989989888, 139698990157823, +ERASE, 139698989989888, 139698989989888, +STORE, 139698989989888, 139698990112767, +STORE, 139698990112768, 139698990145535, +STORE, 139698990145536, 139698990153727, +STORE, 139698990153728, 139698990157823, +STORE, 140737189744640, 140737189748735, +STORE, 140737189732352, 140737189744639, +STORE, 47933805010944, 47933805019135, +STORE, 47933805019136, 47933805027327, +STORE, 47933805027328, 47933805133823, +STORE, 47933805043712, 47933805133823, +STORE, 47933805027328, 47933805043711, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805117439, +STORE, 47933805117440, 47933805133823, +STORE, 47933805096960, 47933805117439, +STORE, 47933805043712, 47933805096959, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805096959, +STORE, 47933805113344, 47933805117439, +STORE, 47933805096960, 47933805113343, +ERASE, 47933805096960, 47933805096960, +STORE, 47933805096960, 47933805113343, +STORE, 47933805125632, 47933805133823, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805125632, 47933805125632, +STORE, 47933805125632, 47933805133823, +STORE, 47933805133824, 47933806972927, +STORE, 47933805273088, 47933806972927, +STORE, 47933805133824, 47933805273087, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806931967, +STORE, 47933806931968, 47933806972927, +STORE, 47933806616576, 47933806931967, +STORE, 47933805273088, 47933806616575, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806616575, +STORE, 47933806927872, 47933806931967, +STORE, 47933806616576, 47933806927871, +ERASE, 47933806616576, 47933806616576, +STORE, 47933806616576, 47933806927871, +STORE, 47933806956544, 47933806972927, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806956544, 47933806956544, +STORE, 47933806956544, 47933806972927, +STORE, 47933806956544, 47933806985215, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806948351, +STORE, 47933806948352, 47933806956543, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805121535, +STORE, 47933805121536, 47933805125631, +ERASE, 93847949524992, 93847949524992, +STORE, 93847949524992, 93847949541375, +STORE, 93847949541376, 93847949545471, +ERASE, 139698990145536, 139698990145536, +STORE, 139698990145536, 139698990149631, +STORE, 139698990149632, 139698990153727, +ERASE, 47933805010944, 47933805010944, +STORE, 140737488347136, 140737488351231, +STORE, 140725553991680, 140737488351231, +ERASE, 140725553991680, 140725553991680, +STORE, 140725553991680, 140725553995775, +STORE, 93980056248320, 93980056961023, +ERASE, 93980056248320, 93980056248320, +STORE, 93980056248320, 93980056297471, +STORE, 93980056297472, 93980056961023, +ERASE, 93980056297472, 93980056297472, +STORE, 93980056297472, 93980056842239, +STORE, 93980056842240, 93980056940543, +STORE, 93980056940544, 93980056961023, +STORE, 140146588971008, 140146589143039, +ERASE, 140146588971008, 140146588971008, +STORE, 140146588971008, 140146588975103, +STORE, 140146588975104, 140146589143039, +ERASE, 140146588975104, 140146588975104, +STORE, 140146588975104, 140146589097983, +STORE, 140146589097984, 140146589130751, +STORE, 140146589130752, 140146589138943, +STORE, 140146589138944, 140146589143039, +STORE, 140725554860032, 140725554864127, +STORE, 140725554847744, 140725554860031, +STORE, 47486206025728, 47486206033919, +STORE, 47486206033920, 47486206042111, +STORE, 47486206042112, 47486206148607, +STORE, 47486206058496, 47486206148607, +STORE, 47486206042112, 47486206058495, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206132223, +STORE, 47486206132224, 47486206148607, +STORE, 47486206111744, 47486206132223, +STORE, 47486206058496, 47486206111743, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206111743, +STORE, 47486206128128, 47486206132223, +STORE, 47486206111744, 47486206128127, +ERASE, 47486206111744, 47486206111744, +STORE, 47486206111744, 47486206128127, +STORE, 47486206140416, 47486206148607, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206140416, 47486206140416, +STORE, 47486206140416, 47486206148607, +STORE, 47486206148608, 47486207987711, +STORE, 47486206287872, 47486207987711, +STORE, 47486206148608, 47486206287871, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207946751, +STORE, 47486207946752, 47486207987711, +STORE, 47486207631360, 47486207946751, +STORE, 47486206287872, 47486207631359, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207631359, +STORE, 47486207942656, 47486207946751, +STORE, 47486207631360, 47486207942655, +ERASE, 47486207631360, 47486207631360, +STORE, 47486207631360, 47486207942655, +STORE, 47486207971328, 47486207987711, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207971328, 47486207971328, +STORE, 47486207971328, 47486207987711, +STORE, 47486207971328, 47486207999999, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207963135, +STORE, 47486207963136, 47486207971327, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206136319, +STORE, 47486206136320, 47486206140415, +ERASE, 93980056940544, 93980056940544, +STORE, 93980056940544, 93980056956927, +STORE, 93980056956928, 93980056961023, +ERASE, 140146589130752, 140146589130752, +STORE, 140146589130752, 140146589134847, +STORE, 140146589134848, 140146589138943, +ERASE, 47486206025728, 47486206025728, +STORE, 93980070006784, 93980070141951, +STORE, 140737488347136, 140737488351231, +STORE, 140727334776832, 140737488351231, +ERASE, 140727334776832, 140727334776832, +STORE, 140727334776832, 140727334780927, +STORE, 94049747247104, 94049747959807, +ERASE, 94049747247104, 94049747247104, +STORE, 94049747247104, 94049747296255, +STORE, 94049747296256, 94049747959807, +ERASE, 94049747296256, 94049747296256, +STORE, 94049747296256, 94049747841023, +STORE, 94049747841024, 94049747939327, +STORE, 94049747939328, 94049747959807, +STORE, 140227307216896, 140227307388927, +ERASE, 140227307216896, 140227307216896, +STORE, 140227307216896, 140227307220991, +STORE, 140227307220992, 140227307388927, +ERASE, 140227307220992, 140227307220992, +STORE, 140227307220992, 140227307343871, +STORE, 140227307343872, 140227307376639, +STORE, 140227307376640, 140227307384831, +STORE, 140227307384832, 140227307388927, +STORE, 140727335337984, 140727335342079, +STORE, 140727335325696, 140727335337983, +STORE, 47405487779840, 47405487788031, +STORE, 47405487788032, 47405487796223, +STORE, 47405487796224, 47405487902719, +STORE, 47405487812608, 47405487902719, +STORE, 47405487796224, 47405487812607, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487886335, +STORE, 47405487886336, 47405487902719, +STORE, 47405487865856, 47405487886335, +STORE, 47405487812608, 47405487865855, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487865855, +STORE, 47405487882240, 47405487886335, +STORE, 47405487865856, 47405487882239, +ERASE, 47405487865856, 47405487865856, +STORE, 47405487865856, 47405487882239, +STORE, 47405487894528, 47405487902719, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487894528, 47405487894528, +STORE, 47405487894528, 47405487902719, +STORE, 47405487902720, 47405489741823, +STORE, 47405488041984, 47405489741823, +STORE, 47405487902720, 47405488041983, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489700863, +STORE, 47405489700864, 47405489741823, +STORE, 47405489385472, 47405489700863, +STORE, 47405488041984, 47405489385471, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489385471, +STORE, 47405489696768, 47405489700863, +STORE, 47405489385472, 47405489696767, +ERASE, 47405489385472, 47405489385472, +STORE, 47405489385472, 47405489696767, +STORE, 47405489725440, 47405489741823, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489725440, 47405489725440, +STORE, 47405489725440, 47405489741823, +STORE, 47405489725440, 47405489754111, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489717247, +STORE, 47405489717248, 47405489725439, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487890431, +STORE, 47405487890432, 47405487894527, +ERASE, 94049747939328, 94049747939328, +STORE, 94049747939328, 94049747955711, +STORE, 94049747955712, 94049747959807, +ERASE, 140227307376640, 140227307376640, +STORE, 140227307376640, 140227307380735, +STORE, 140227307380736, 140227307384831, +ERASE, 47405487779840, 47405487779840, +STORE, 94049758810112, 94049758945279, +STORE, 140737488347136, 140737488351231, +STORE, 140727079718912, 140737488351231, +ERASE, 140727079718912, 140727079718912, +STORE, 140727079718912, 140727079723007, +STORE, 94250996527104, 94250997239807, +ERASE, 94250996527104, 94250996527104, +STORE, 94250996527104, 94250996576255, +STORE, 94250996576256, 94250997239807, +ERASE, 94250996576256, 94250996576256, +STORE, 94250996576256, 94250997121023, +STORE, 94250997121024, 94250997219327, +STORE, 94250997219328, 94250997239807, +STORE, 140060022587392, 140060022759423, +ERASE, 140060022587392, 140060022587392, +STORE, 140060022587392, 140060022591487, +STORE, 140060022591488, 140060022759423, +ERASE, 140060022591488, 140060022591488, +STORE, 140060022591488, 140060022714367, +STORE, 140060022714368, 140060022747135, +STORE, 140060022747136, 140060022755327, +STORE, 140060022755328, 140060022759423, +STORE, 140727079788544, 140727079792639, +STORE, 140727079776256, 140727079788543, +/* this next one caused issues when lowering the efficiency */ +STORE, 47572772409344, 47572772417535, +STORE, 47572772417536, 47572772425727, +STORE, 47572772425728, 47572772532223, +STORE, 47572772442112, 47572772532223, +STORE, 47572772425728, 47572772442111, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772515839, +STORE, 47572772515840, 47572772532223, +STORE, 47572772495360, 47572772515839, +STORE, 47572772442112, 47572772495359, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772495359, +STORE, 47572772511744, 47572772515839, +STORE, 47572772495360, 47572772511743, +ERASE, 47572772495360, 47572772495360, +STORE, 47572772495360, 47572772511743, +STORE, 47572772524032, 47572772532223, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772524032, 47572772524032, +STORE, 47572772524032, 47572772532223, +STORE, 47572772532224, 47572774371327, +STORE, 47572772671488, 47572774371327, +STORE, 47572772532224, 47572772671487, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774330367, +STORE, 47572774330368, 47572774371327, +STORE, 47572774014976, 47572774330367, +STORE, 47572772671488, 47572774014975, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774014975, +STORE, 47572774326272, 47572774330367, +STORE, 47572774014976, 47572774326271, +ERASE, 47572774014976, 47572774014976, +STORE, 47572774014976, 47572774326271, +STORE, 47572774354944, 47572774371327, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774354944, 47572774354944, +STORE, 47572774354944, 47572774371327, +STORE, 47572774354944, 47572774383615, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774346751, +STORE, 47572774346752, 47572774354943, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772519935, +STORE, 47572772519936, 47572772524031, +ERASE, 94250997219328, 94250997219328, +STORE, 94250997219328, 94250997235711, +STORE, 94250997235712, 94250997239807, +ERASE, 140060022747136, 140060022747136, +STORE, 140060022747136, 140060022751231, +STORE, 140060022751232, 140060022755327, +ERASE, 47572772409344, 47572772409344, +STORE, 94251018305536, 94251018440703, +STORE, 140737488347136, 140737488351231, +STORE, 140730012389376, 140737488351231, +ERASE, 140730012389376, 140730012389376, +STORE, 140730012389376, 140730012393471, +STORE, 94382607675392, 94382607695871, +ERASE, 94382607675392, 94382607675392, +STORE, 94382607675392, 94382607679487, +STORE, 94382607679488, 94382607695871, +ERASE, 94382607679488, 94382607679488, +STORE, 94382607679488, 94382607683583, +STORE, 94382607683584, 94382607687679, +STORE, 94382607687680, 94382607695871, +STORE, 140252451454976, 140252451627007, +ERASE, 140252451454976, 140252451454976, +STORE, 140252451454976, 140252451459071, +STORE, 140252451459072, 140252451627007, +ERASE, 140252451459072, 140252451459072, +STORE, 140252451459072, 140252451581951, +STORE, 140252451581952, 140252451614719, +STORE, 140252451614720, 140252451622911, +STORE, 140252451622912, 140252451627007, +STORE, 140730013548544, 140730013552639, +STORE, 140730013536256, 140730013548543, +STORE, 47380343541760, 47380343549951, +STORE, 47380343549952, 47380343558143, +STORE, 47380343558144, 47380345397247, +STORE, 47380343697408, 47380345397247, +STORE, 47380343558144, 47380343697407, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345356287, +STORE, 47380345356288, 47380345397247, +STORE, 47380345040896, 47380345356287, +STORE, 47380343697408, 47380345040895, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345040895, +STORE, 47380345352192, 47380345356287, +STORE, 47380345040896, 47380345352191, +ERASE, 47380345040896, 47380345040896, +STORE, 47380345040896, 47380345352191, +STORE, 47380345380864, 47380345397247, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345380864, 47380345380864, +STORE, 47380345380864, 47380345397247, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345372671, +STORE, 47380345372672, 47380345380863, +ERASE, 94382607687680, 94382607687680, +STORE, 94382607687680, 94382607691775, +STORE, 94382607691776, 94382607695871, +ERASE, 140252451614720, 140252451614720, +STORE, 140252451614720, 140252451618815, +STORE, 140252451618816, 140252451622911, +ERASE, 47380343541760, 47380343541760, +STORE, 94382626803712, 94382626938879, +STORE, 140737488347136, 140737488351231, +STORE, 140730900271104, 140737488351231, +ERASE, 140730900271104, 140730900271104, +STORE, 140730900271104, 140730900275199, +STORE, 93855478120448, 93855478337535, +ERASE, 93855478120448, 93855478120448, +STORE, 93855478120448, 93855478198271, +STORE, 93855478198272, 93855478337535, +ERASE, 93855478198272, 93855478198272, +STORE, 93855478198272, 93855478243327, +STORE, 93855478243328, 93855478288383, +STORE, 93855478288384, 93855478337535, +STORE, 140092686573568, 140092686745599, +ERASE, 140092686573568, 140092686573568, +STORE, 140092686573568, 140092686577663, +STORE, 140092686577664, 140092686745599, +ERASE, 140092686577664, 140092686577664, +STORE, 140092686577664, 140092686700543, +STORE, 140092686700544, 140092686733311, +STORE, 140092686733312, 140092686741503, +STORE, 140092686741504, 140092686745599, +STORE, 140730900537344, 140730900541439, +STORE, 140730900525056, 140730900537343, +STORE, 47540108423168, 47540108431359, +STORE, 47540108431360, 47540108439551, +STORE, 47540108439552, 47540110278655, +STORE, 47540108578816, 47540110278655, +STORE, 47540108439552, 47540108578815, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540110237695, +STORE, 47540110237696, 47540110278655, +STORE, 47540109922304, 47540110237695, +STORE, 47540108578816, 47540109922303, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540109922303, +STORE, 47540110233600, 47540110237695, +STORE, 47540109922304, 47540110233599, +ERASE, 47540109922304, 47540109922304, +STORE, 47540109922304, 47540110233599, +STORE, 47540110262272, 47540110278655, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110262272, 47540110262272, +STORE, 47540110262272, 47540110278655, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110254079, +STORE, 47540110254080, 47540110262271, +ERASE, 93855478288384, 93855478288384, +STORE, 93855478288384, 93855478333439, +STORE, 93855478333440, 93855478337535, +ERASE, 140092686733312, 140092686733312, +STORE, 140092686733312, 140092686737407, +STORE, 140092686737408, 140092686741503, +ERASE, 47540108423168, 47540108423168, +STORE, 93855492222976, 93855492358143, +STORE, 93855492222976, 93855492493311, +STORE, 140737488347136, 140737488351231, +STORE, 140733498146816, 140737488351231, +ERASE, 140733498146816, 140733498146816, +STORE, 140733498146816, 140733498150911, +STORE, 94170739654656, 94170740367359, +ERASE, 94170739654656, 94170739654656, +STORE, 94170739654656, 94170739703807, +STORE, 94170739703808, 94170740367359, +ERASE, 94170739703808, 94170739703808, +STORE, 94170739703808, 94170740248575, +STORE, 94170740248576, 94170740346879, +STORE, 94170740346880, 94170740367359, +STORE, 140024788877312, 140024789049343, +ERASE, 140024788877312, 140024788877312, +STORE, 140024788877312, 140024788881407, +STORE, 140024788881408, 140024789049343, +ERASE, 140024788881408, 140024788881408, +STORE, 140024788881408, 140024789004287, +STORE, 140024789004288, 140024789037055, +STORE, 140024789037056, 140024789045247, +STORE, 140024789045248, 140024789049343, +STORE, 140733499023360, 140733499027455, +STORE, 140733499011072, 140733499023359, +STORE, 47608006119424, 47608006127615, +STORE, 47608006127616, 47608006135807, +STORE, 47608006135808, 47608006242303, +STORE, 47608006152192, 47608006242303, +STORE, 47608006135808, 47608006152191, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006225919, +STORE, 47608006225920, 47608006242303, +STORE, 47608006205440, 47608006225919, +STORE, 47608006152192, 47608006205439, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006205439, +STORE, 47608006221824, 47608006225919, +STORE, 47608006205440, 47608006221823, +ERASE, 47608006205440, 47608006205440, +STORE, 47608006205440, 47608006221823, +STORE, 47608006234112, 47608006242303, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006234112, 47608006234112, +STORE, 47608006234112, 47608006242303, +STORE, 47608006242304, 47608008081407, +STORE, 47608006381568, 47608008081407, +STORE, 47608006242304, 47608006381567, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608008040447, +STORE, 47608008040448, 47608008081407, +STORE, 47608007725056, 47608008040447, +STORE, 47608006381568, 47608007725055, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608007725055, +STORE, 47608008036352, 47608008040447, +STORE, 47608007725056, 47608008036351, +ERASE, 47608007725056, 47608007725056, +STORE, 47608007725056, 47608008036351, +STORE, 47608008065024, 47608008081407, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008065024, 47608008065024, +STORE, 47608008065024, 47608008081407, +STORE, 47608008065024, 47608008093695, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008056831, +STORE, 47608008056832, 47608008065023, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006230015, +STORE, 47608006230016, 47608006234111, +ERASE, 94170740346880, 94170740346880, +STORE, 94170740346880, 94170740363263, +STORE, 94170740363264, 94170740367359, +ERASE, 140024789037056, 140024789037056, +STORE, 140024789037056, 140024789041151, +STORE, 140024789041152, 140024789045247, +ERASE, 47608006119424, 47608006119424, +STORE, 140737488347136, 140737488351231, +STORE, 140730264326144, 140737488351231, +ERASE, 140730264326144, 140730264326144, +STORE, 140730264326144, 140730264330239, +STORE, 94653216407552, 94653217120255, +ERASE, 94653216407552, 94653216407552, +STORE, 94653216407552, 94653216456703, +STORE, 94653216456704, 94653217120255, +ERASE, 94653216456704, 94653216456704, +STORE, 94653216456704, 94653217001471, +STORE, 94653217001472, 94653217099775, +STORE, 94653217099776, 94653217120255, +STORE, 140103617011712, 140103617183743, +ERASE, 140103617011712, 140103617011712, +STORE, 140103617011712, 140103617015807, +STORE, 140103617015808, 140103617183743, +ERASE, 140103617015808, 140103617015808, +STORE, 140103617015808, 140103617138687, +STORE, 140103617138688, 140103617171455, +STORE, 140103617171456, 140103617179647, +STORE, 140103617179648, 140103617183743, +STORE, 140730265427968, 140730265432063, +STORE, 140730265415680, 140730265427967, +STORE, 47529177985024, 47529177993215, +STORE, 47529177993216, 47529178001407, +STORE, 47529178001408, 47529178107903, +STORE, 47529178017792, 47529178107903, +STORE, 47529178001408, 47529178017791, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178091519, +STORE, 47529178091520, 47529178107903, +STORE, 47529178071040, 47529178091519, +STORE, 47529178017792, 47529178071039, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178071039, +STORE, 47529178087424, 47529178091519, +STORE, 47529178071040, 47529178087423, +ERASE, 47529178071040, 47529178071040, +STORE, 47529178071040, 47529178087423, +STORE, 47529178099712, 47529178107903, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178099712, 47529178099712, +STORE, 47529178099712, 47529178107903, +STORE, 47529178107904, 47529179947007, +STORE, 47529178247168, 47529179947007, +STORE, 47529178107904, 47529178247167, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179906047, +STORE, 47529179906048, 47529179947007, +STORE, 47529179590656, 47529179906047, +STORE, 47529178247168, 47529179590655, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179590655, +STORE, 47529179901952, 47529179906047, +STORE, 47529179590656, 47529179901951, +ERASE, 47529179590656, 47529179590656, +STORE, 47529179590656, 47529179901951, +STORE, 47529179930624, 47529179947007, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179930624, 47529179930624, +STORE, 47529179930624, 47529179947007, +STORE, 47529179930624, 47529179959295, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179922431, +STORE, 47529179922432, 47529179930623, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178095615, +STORE, 47529178095616, 47529178099711, +ERASE, 94653217099776, 94653217099776, +STORE, 94653217099776, 94653217116159, +STORE, 94653217116160, 94653217120255, +ERASE, 140103617171456, 140103617171456, +STORE, 140103617171456, 140103617175551, +STORE, 140103617175552, 140103617179647, +ERASE, 47529177985024, 47529177985024, +STORE, 94653241135104, 94653241270271, +STORE, 140737488347136, 140737488351231, +STORE, 140736284549120, 140737488351231, +ERASE, 140736284549120, 140736284549120, +STORE, 140736284549120, 140736284553215, +STORE, 93963663822848, 93963664506879, +ERASE, 93963663822848, 93963663822848, +STORE, 93963663822848, 93963663884287, +STORE, 93963663884288, 93963664506879, +ERASE, 93963663884288, 93963663884288, +STORE, 93963663884288, 93963664240639, +STORE, 93963664240640, 93963664379903, +STORE, 93963664379904, 93963664506879, +STORE, 140450188439552, 140450188611583, +ERASE, 140450188439552, 140450188439552, +STORE, 140450188439552, 140450188443647, +STORE, 140450188443648, 140450188611583, +ERASE, 140450188443648, 140450188443648, +STORE, 140450188443648, 140450188566527, +STORE, 140450188566528, 140450188599295, +STORE, 140450188599296, 140450188607487, +STORE, 140450188607488, 140450188611583, +STORE, 140736284577792, 140736284581887, +STORE, 140736284565504, 140736284577791, +STORE, 47182606557184, 47182606565375, +STORE, 47182606565376, 47182606573567, +STORE, 47182606573568, 47182608412671, +STORE, 47182606712832, 47182608412671, +STORE, 47182606573568, 47182606712831, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608371711, +STORE, 47182608371712, 47182608412671, +STORE, 47182608056320, 47182608371711, +STORE, 47182606712832, 47182608056319, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608056319, +STORE, 47182608367616, 47182608371711, +STORE, 47182608056320, 47182608367615, +ERASE, 47182608056320, 47182608056320, +STORE, 47182608056320, 47182608367615, +STORE, 47182608396288, 47182608412671, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608396288, 47182608396288, +STORE, 47182608396288, 47182608412671, +STORE, 47182608412672, 47182608523263, +STORE, 47182608429056, 47182608523263, +STORE, 47182608412672, 47182608429055, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608515071, +STORE, 47182608515072, 47182608523263, +STORE, 47182608490496, 47182608515071, +STORE, 47182608429056, 47182608490495, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608490495, +STORE, 47182608510976, 47182608515071, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608490496, 47182608490496, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608523263, +STORE, 47182608523264, 47182608568319, +ERASE, 47182608523264, 47182608523264, +STORE, 47182608523264, 47182608531455, +STORE, 47182608531456, 47182608568319, +STORE, 47182608551936, 47182608568319, +STORE, 47182608531456, 47182608551935, +ERASE, 47182608531456, 47182608531456, +STORE, 47182608531456, 47182608551935, +STORE, 47182608560128, 47182608568319, +STORE, 47182608551936, 47182608560127, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608568319, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608560127, +STORE, 47182608560128, 47182608568319, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608568319, +STORE, 47182608568320, 47182608916479, +STORE, 47182608609280, 47182608916479, +STORE, 47182608568320, 47182608609279, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608891903, +STORE, 47182608891904, 47182608916479, +STORE, 47182608822272, 47182608891903, +STORE, 47182608609280, 47182608822271, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608822271, +STORE, 47182608887808, 47182608891903, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608822272, 47182608822272, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608916479, +STORE, 47182608916480, 47182611177471, +STORE, 47182609068032, 47182611177471, +STORE, 47182608916480, 47182609068031, +ERASE, 47182609068032, 47182609068032, +STORE, 47182609068032, 47182611161087, +STORE, 47182611161088, 47182611177471, +STORE, 47182611169280, 47182611177471, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611169280, 47182611169280, +STORE, 47182611169280, 47182611177471, +STORE, 47182611177472, 47182611312639, +ERASE, 47182611177472, 47182611177472, +STORE, 47182611177472, 47182611202047, +STORE, 47182611202048, 47182611312639, +STORE, 47182611263488, 47182611312639, +STORE, 47182611202048, 47182611263487, +ERASE, 47182611202048, 47182611202048, +STORE, 47182611202048, 47182611263487, +STORE, 47182611288064, 47182611312639, +STORE, 47182611263488, 47182611288063, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611312639, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611288063, +STORE, 47182611288064, 47182611312639, +STORE, 47182611296256, 47182611312639, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611296256, 47182611296256, +STORE, 47182611296256, 47182611312639, +STORE, 47182611296256, 47182611320831, +STORE, 47182611320832, 47182611484671, +ERASE, 47182611320832, 47182611320832, +STORE, 47182611320832, 47182611333119, +STORE, 47182611333120, 47182611484671, +STORE, 47182611431424, 47182611484671, +STORE, 47182611333120, 47182611431423, +ERASE, 47182611333120, 47182611333120, +STORE, 47182611333120, 47182611431423, +STORE, 47182611476480, 47182611484671, +STORE, 47182611431424, 47182611476479, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611484671, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611476479, +STORE, 47182611476480, 47182611484671, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611484671, +STORE, 47182611484672, 47182612082687, +STORE, 47182611603456, 47182612082687, +STORE, 47182611484672, 47182611603455, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182612029439, +STORE, 47182612029440, 47182612082687, +STORE, 47182611918848, 47182612029439, +STORE, 47182611603456, 47182611918847, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182611918847, +STORE, 47182612025344, 47182612029439, +STORE, 47182611918848, 47182612025343, +ERASE, 47182611918848, 47182611918848, +STORE, 47182611918848, 47182612025343, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612082687, +STORE, 47182612082688, 47182615134207, +STORE, 47182612627456, 47182615134207, +STORE, 47182612082688, 47182612627455, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614913023, +STORE, 47182614913024, 47182615134207, +STORE, 47182614323200, 47182614913023, +STORE, 47182612627456, 47182614323199, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614323199, +STORE, 47182614908928, 47182614913023, +STORE, 47182614323200, 47182614908927, +ERASE, 47182614323200, 47182614323200, +STORE, 47182614323200, 47182614908927, +STORE, 47182615117824, 47182615134207, +STORE, 47182614913024, 47182615117823, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615117823, +ERASE, 47182615117824, 47182615117824, +STORE, 47182615117824, 47182615134207, +STORE, 47182615134208, 47182615166975, +ERASE, 47182615134208, 47182615134208, +STORE, 47182615134208, 47182615142399, +STORE, 47182615142400, 47182615166975, +STORE, 47182615154688, 47182615166975, +STORE, 47182615142400, 47182615154687, +ERASE, 47182615142400, 47182615142400, +STORE, 47182615142400, 47182615154687, +STORE, 47182615158784, 47182615166975, +STORE, 47182615154688, 47182615158783, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615166975, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615158783, +STORE, 47182615158784, 47182615166975, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615166975, +STORE, 47182615166976, 47182615203839, +ERASE, 47182615166976, 47182615166976, +STORE, 47182615166976, 47182615175167, +STORE, 47182615175168, 47182615203839, +STORE, 47182615191552, 47182615203839, +STORE, 47182615175168, 47182615191551, +ERASE, 47182615175168, 47182615175168, +STORE, 47182615175168, 47182615191551, +STORE, 47182615195648, 47182615203839, +STORE, 47182615191552, 47182615195647, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615203839, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615195647, +STORE, 47182615195648, 47182615203839, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615203839, +STORE, 47182615203840, 47182615678975, +ERASE, 47182615203840, 47182615203840, +STORE, 47182615203840, 47182615212031, +STORE, 47182615212032, 47182615678975, +STORE, 47182615547904, 47182615678975, +STORE, 47182615212032, 47182615547903, +ERASE, 47182615212032, 47182615212032, +STORE, 47182615212032, 47182615547903, +STORE, 47182615670784, 47182615678975, +STORE, 47182615547904, 47182615670783, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615678975, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615670783, +STORE, 47182615670784, 47182615678975, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615678975, +STORE, 47182615678976, 47182615687167, +STORE, 47182615687168, 47182615707647, +ERASE, 47182615687168, 47182615687168, +STORE, 47182615687168, 47182615691263, +STORE, 47182615691264, 47182615707647, +STORE, 47182615695360, 47182615707647, +STORE, 47182615691264, 47182615695359, +ERASE, 47182615691264, 47182615691264, +STORE, 47182615691264, 47182615695359, +STORE, 47182615699456, 47182615707647, +STORE, 47182615695360, 47182615699455, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615707647, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615699455, +STORE, 47182615699456, 47182615707647, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615707647, +STORE, 47182615707648, 47182615715839, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608388095, +STORE, 47182608388096, 47182608396287, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615703551, +STORE, 47182615703552, 47182615707647, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611292159, +STORE, 47182611292160, 47182611296255, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615674879, +STORE, 47182615674880, 47182615678975, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615199743, +STORE, 47182615199744, 47182615203839, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615162879, +STORE, 47182615162880, 47182615166975, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615109631, +STORE, 47182615109632, 47182615117823, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612066303, +STORE, 47182612066304, 47182612082687, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611480575, +STORE, 47182611480576, 47182611484671, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611165183, +STORE, 47182611165184, 47182611169279, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608912383, +STORE, 47182608912384, 47182608916479, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608564223, +STORE, 47182608564224, 47182608568319, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608519167, +STORE, 47182608519168, 47182608523263, +ERASE, 93963664379904, 93963664379904, +STORE, 93963664379904, 93963664502783, +STORE, 93963664502784, 93963664506879, +ERASE, 140450188599296, 140450188599296, +STORE, 140450188599296, 140450188603391, +STORE, 140450188603392, 140450188607487, +ERASE, 47182606557184, 47182606557184, +STORE, 93963694723072, 93963694858239, +STORE, 140737488347136, 140737488351231, +STORE, 140730313261056, 140737488351231, +ERASE, 140730313261056, 140730313261056, +STORE, 140730313261056, 140730313265151, +STORE, 94386579017728, 94386579697663, +ERASE, 94386579017728, 94386579017728, +STORE, 94386579017728, 94386579083263, +STORE, 94386579083264, 94386579697663, +ERASE, 94386579083264, 94386579083264, +STORE, 94386579083264, 94386579431423, +STORE, 94386579431424, 94386579570687, +STORE, 94386579570688, 94386579697663, +STORE, 140124810838016, 140124811010047, +ERASE, 140124810838016, 140124810838016, +STORE, 140124810838016, 140124810842111, +STORE, 140124810842112, 140124811010047, +ERASE, 140124810842112, 140124810842112, +STORE, 140124810842112, 140124810964991, +STORE, 140124810964992, 140124810997759, +STORE, 140124810997760, 140124811005951, +STORE, 140124811005952, 140124811010047, +STORE, 140730313601024, 140730313605119, +STORE, 140730313588736, 140730313601023, +STORE, 47507984158720, 47507984166911, +STORE, 47507984166912, 47507984175103, +STORE, 47507984175104, 47507986014207, +STORE, 47507984314368, 47507986014207, +STORE, 47507984175104, 47507984314367, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985973247, +STORE, 47507985973248, 47507986014207, +STORE, 47507985657856, 47507985973247, +STORE, 47507984314368, 47507985657855, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985657855, +STORE, 47507985969152, 47507985973247, +STORE, 47507985657856, 47507985969151, +ERASE, 47507985657856, 47507985657856, +STORE, 47507985657856, 47507985969151, +STORE, 47507985997824, 47507986014207, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985997824, 47507985997824, +STORE, 47507985997824, 47507986014207, +STORE, 47507986014208, 47507986124799, +STORE, 47507986030592, 47507986124799, +STORE, 47507986014208, 47507986030591, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986116607, +STORE, 47507986116608, 47507986124799, +STORE, 47507986092032, 47507986116607, +STORE, 47507986030592, 47507986092031, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986092031, +STORE, 47507986112512, 47507986116607, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986092032, 47507986092032, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986124799, +STORE, 47507986124800, 47507986169855, +ERASE, 47507986124800, 47507986124800, +STORE, 47507986124800, 47507986132991, +STORE, 47507986132992, 47507986169855, +STORE, 47507986153472, 47507986169855, +STORE, 47507986132992, 47507986153471, +ERASE, 47507986132992, 47507986132992, +STORE, 47507986132992, 47507986153471, +STORE, 47507986161664, 47507986169855, +STORE, 47507986153472, 47507986161663, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986169855, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986161663, +STORE, 47507986161664, 47507986169855, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986169855, +STORE, 47507986169856, 47507986518015, +STORE, 47507986210816, 47507986518015, +STORE, 47507986169856, 47507986210815, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986493439, +STORE, 47507986493440, 47507986518015, +STORE, 47507986423808, 47507986493439, +STORE, 47507986210816, 47507986423807, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986423807, +STORE, 47507986489344, 47507986493439, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986423808, 47507986423808, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986518015, +STORE, 47507986518016, 47507988779007, +STORE, 47507986669568, 47507988779007, +STORE, 47507986518016, 47507986669567, +ERASE, 47507986669568, 47507986669568, +STORE, 47507986669568, 47507988762623, +STORE, 47507988762624, 47507988779007, +STORE, 47507988770816, 47507988779007, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988770816, 47507988770816, +STORE, 47507988770816, 47507988779007, +STORE, 47507988779008, 47507988914175, +ERASE, 47507988779008, 47507988779008, +STORE, 47507988779008, 47507988803583, +STORE, 47507988803584, 47507988914175, +STORE, 47507988865024, 47507988914175, +STORE, 47507988803584, 47507988865023, +ERASE, 47507988803584, 47507988803584, +STORE, 47507988803584, 47507988865023, +STORE, 47507988889600, 47507988914175, +STORE, 47507988865024, 47507988889599, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988914175, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988889599, +STORE, 47507988889600, 47507988914175, +STORE, 47507988897792, 47507988914175, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988897792, 47507988897792, +STORE, 47507988897792, 47507988914175, +STORE, 47507988897792, 47507988922367, +STORE, 47507988922368, 47507989086207, +ERASE, 47507988922368, 47507988922368, +STORE, 47507988922368, 47507988934655, +STORE, 47507988934656, 47507989086207, +STORE, 47507989032960, 47507989086207, +STORE, 47507988934656, 47507989032959, +ERASE, 47507988934656, 47507988934656, +STORE, 47507988934656, 47507989032959, +STORE, 47507989078016, 47507989086207, +STORE, 47507989032960, 47507989078015, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989086207, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989078015, +STORE, 47507989078016, 47507989086207, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989086207, +STORE, 47507989086208, 47507989684223, +STORE, 47507989204992, 47507989684223, +STORE, 47507989086208, 47507989204991, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989630975, +STORE, 47507989630976, 47507989684223, +STORE, 47507989520384, 47507989630975, +STORE, 47507989204992, 47507989520383, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989520383, +STORE, 47507989626880, 47507989630975, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989520384, 47507989520384, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989684223, +STORE, 47507989684224, 47507992735743, +STORE, 47507990228992, 47507992735743, +STORE, 47507989684224, 47507990228991, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507992514559, +STORE, 47507992514560, 47507992735743, +STORE, 47507991924736, 47507992514559, +STORE, 47507990228992, 47507991924735, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507991924735, +STORE, 47507992510464, 47507992514559, +STORE, 47507991924736, 47507992510463, +ERASE, 47507991924736, 47507991924736, +STORE, 47507991924736, 47507992510463, +STORE, 47507992719360, 47507992735743, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992719360, 47507992719360, +STORE, 47507992719360, 47507992735743, +STORE, 47507992735744, 47507992768511, +ERASE, 47507992735744, 47507992735744, +STORE, 47507992735744, 47507992743935, +STORE, 47507992743936, 47507992768511, +STORE, 47507992756224, 47507992768511, +STORE, 47507992743936, 47507992756223, +ERASE, 47507992743936, 47507992743936, +STORE, 47507992743936, 47507992756223, +STORE, 47507992760320, 47507992768511, +STORE, 47507992756224, 47507992760319, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992768511, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992760319, +STORE, 47507992760320, 47507992768511, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992768511, +STORE, 47507992768512, 47507992805375, +ERASE, 47507992768512, 47507992768512, +STORE, 47507992768512, 47507992776703, +STORE, 47507992776704, 47507992805375, +STORE, 47507992793088, 47507992805375, +STORE, 47507992776704, 47507992793087, +ERASE, 47507992776704, 47507992776704, +STORE, 47507992776704, 47507992793087, +STORE, 47507992797184, 47507992805375, +STORE, 47507992793088, 47507992797183, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992805375, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992797183, +STORE, 47507992797184, 47507992805375, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992805375, +STORE, 47507992805376, 47507993280511, +ERASE, 47507992805376, 47507992805376, +STORE, 47507992805376, 47507992813567, +STORE, 47507992813568, 47507993280511, +STORE, 47507993149440, 47507993280511, +STORE, 47507992813568, 47507993149439, +ERASE, 47507992813568, 47507992813568, +STORE, 47507992813568, 47507993149439, +STORE, 47507993272320, 47507993280511, +STORE, 47507993149440, 47507993272319, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993280511, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993272319, +STORE, 47507993272320, 47507993280511, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993280511, +STORE, 47507993280512, 47507993288703, +STORE, 47507993288704, 47507993309183, +ERASE, 47507993288704, 47507993288704, +STORE, 47507993288704, 47507993292799, +STORE, 47507993292800, 47507993309183, +STORE, 47507993296896, 47507993309183, +STORE, 47507993292800, 47507993296895, +ERASE, 47507993292800, 47507993292800, +STORE, 47507993292800, 47507993296895, +STORE, 47507993300992, 47507993309183, +STORE, 47507993296896, 47507993300991, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993309183, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993300991, +STORE, 47507993300992, 47507993309183, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993309183, +STORE, 47507993309184, 47507993317375, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985989631, +STORE, 47507985989632, 47507985997823, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993305087, +STORE, 47507993305088, 47507993309183, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988893695, +STORE, 47507988893696, 47507988897791, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993276415, +STORE, 47507993276416, 47507993280511, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992801279, +STORE, 47507992801280, 47507992805375, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992764415, +STORE, 47507992764416, 47507992768511, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992711167, +STORE, 47507992711168, 47507992719359, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989667839, +STORE, 47507989667840, 47507989684223, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989082111, +STORE, 47507989082112, 47507989086207, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988766719, +STORE, 47507988766720, 47507988770815, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986513919, +STORE, 47507986513920, 47507986518015, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986165759, +STORE, 47507986165760, 47507986169855, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986120703, +STORE, 47507986120704, 47507986124799, +ERASE, 94386579570688, 94386579570688, +STORE, 94386579570688, 94386579693567, +STORE, 94386579693568, 94386579697663, +ERASE, 140124810997760, 140124810997760, +STORE, 140124810997760, 140124811001855, +STORE, 140124811001856, 140124811005951, +ERASE, 47507984158720, 47507984158720, +STORE, 94386583982080, 94386584117247, +STORE, 94386583982080, 94386584256511, +ERASE, 94386583982080, 94386583982080, +STORE, 94386583982080, 94386584223743, +STORE, 94386584223744, 94386584256511, +ERASE, 94386584223744, 94386584223744, +STORE, 140737488347136, 140737488351231, +STORE, 140733763395584, 140737488351231, +ERASE, 140733763395584, 140733763395584, +STORE, 140733763395584, 140733763399679, +STORE, 94011546472448, 94011547152383, +ERASE, 94011546472448, 94011546472448, +STORE, 94011546472448, 94011546537983, +STORE, 94011546537984, 94011547152383, +ERASE, 94011546537984, 94011546537984, +STORE, 94011546537984, 94011546886143, +STORE, 94011546886144, 94011547025407, +STORE, 94011547025408, 94011547152383, +STORE, 139757597949952, 139757598121983, +ERASE, 139757597949952, 139757597949952, +STORE, 139757597949952, 139757597954047, +STORE, 139757597954048, 139757598121983, +ERASE, 139757597954048, 139757597954048, +STORE, 139757597954048, 139757598076927, +STORE, 139757598076928, 139757598109695, +STORE, 139757598109696, 139757598117887, +STORE, 139757598117888, 139757598121983, +STORE, 140733763596288, 140733763600383, +STORE, 140733763584000, 140733763596287, +STORE, 47875197046784, 47875197054975, +STORE, 47875197054976, 47875197063167, +STORE, 47875197063168, 47875198902271, +STORE, 47875197202432, 47875198902271, +STORE, 47875197063168, 47875197202431, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198861311, +STORE, 47875198861312, 47875198902271, +STORE, 47875198545920, 47875198861311, +STORE, 47875197202432, 47875198545919, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198545919, +STORE, 47875198857216, 47875198861311, +STORE, 47875198545920, 47875198857215, +ERASE, 47875198545920, 47875198545920, +STORE, 47875198545920, 47875198857215, +STORE, 47875198885888, 47875198902271, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198885888, 47875198885888, +STORE, 47875198885888, 47875198902271, +STORE, 47875198902272, 47875199012863, +STORE, 47875198918656, 47875199012863, +STORE, 47875198902272, 47875198918655, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875199004671, +STORE, 47875199004672, 47875199012863, +STORE, 47875198980096, 47875199004671, +STORE, 47875198918656, 47875198980095, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875198980095, +STORE, 47875199000576, 47875199004671, +STORE, 47875198980096, 47875199000575, +ERASE, 47875198980096, 47875198980096, +STORE, 47875198980096, 47875199000575, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199012863, +STORE, 47875199012864, 47875199057919, +ERASE, 47875199012864, 47875199012864, +STORE, 47875199012864, 47875199021055, +STORE, 47875199021056, 47875199057919, +STORE, 47875199041536, 47875199057919, +STORE, 47875199021056, 47875199041535, +ERASE, 47875199021056, 47875199021056, +STORE, 47875199021056, 47875199041535, +STORE, 47875199049728, 47875199057919, +STORE, 47875199041536, 47875199049727, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199057919, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199049727, +STORE, 47875199049728, 47875199057919, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199057919, +STORE, 47875199057920, 47875199406079, +STORE, 47875199098880, 47875199406079, +STORE, 47875199057920, 47875199098879, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199381503, +STORE, 47875199381504, 47875199406079, +STORE, 47875199311872, 47875199381503, +STORE, 47875199098880, 47875199311871, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199311871, +STORE, 47875199377408, 47875199381503, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199311872, 47875199311872, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199406079, +STORE, 47875199406080, 47875201667071, +STORE, 47875199557632, 47875201667071, +STORE, 47875199406080, 47875199557631, +ERASE, 47875199557632, 47875199557632, +STORE, 47875199557632, 47875201650687, +STORE, 47875201650688, 47875201667071, +STORE, 47875201658880, 47875201667071, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201658880, 47875201658880, +STORE, 47875201658880, 47875201667071, +STORE, 47875201667072, 47875201802239, +ERASE, 47875201667072, 47875201667072, +STORE, 47875201667072, 47875201691647, +STORE, 47875201691648, 47875201802239, +STORE, 47875201753088, 47875201802239, +STORE, 47875201691648, 47875201753087, +ERASE, 47875201691648, 47875201691648, +STORE, 47875201691648, 47875201753087, +STORE, 47875201777664, 47875201802239, +STORE, 47875201753088, 47875201777663, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201802239, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201777663, +STORE, 47875201777664, 47875201802239, +STORE, 47875201785856, 47875201802239, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201785856, 47875201785856, +STORE, 47875201785856, 47875201802239, +STORE, 47875201785856, 47875201810431, +STORE, 47875201810432, 47875201974271, +ERASE, 47875201810432, 47875201810432, +STORE, 47875201810432, 47875201822719, +STORE, 47875201822720, 47875201974271, +STORE, 47875201921024, 47875201974271, +STORE, 47875201822720, 47875201921023, +ERASE, 47875201822720, 47875201822720, +STORE, 47875201822720, 47875201921023, +STORE, 47875201966080, 47875201974271, +STORE, 47875201921024, 47875201966079, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201974271, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201966079, +STORE, 47875201966080, 47875201974271, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201974271, +STORE, 47875201974272, 47875202572287, +STORE, 47875202093056, 47875202572287, +STORE, 47875201974272, 47875202093055, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202519039, +STORE, 47875202519040, 47875202572287, +STORE, 47875202408448, 47875202519039, +STORE, 47875202093056, 47875202408447, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202408447, +STORE, 47875202514944, 47875202519039, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202408448, 47875202408448, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202572287, +STORE, 47875202572288, 47875205623807, +STORE, 47875203117056, 47875205623807, +STORE, 47875202572288, 47875203117055, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875205402623, +STORE, 47875205402624, 47875205623807, +STORE, 47875204812800, 47875205402623, +STORE, 47875203117056, 47875204812799, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875204812799, +STORE, 47875205398528, 47875205402623, +STORE, 47875204812800, 47875205398527, +ERASE, 47875204812800, 47875204812800, +STORE, 47875204812800, 47875205398527, +STORE, 47875205607424, 47875205623807, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205607424, 47875205607424, +STORE, 47875205607424, 47875205623807, +STORE, 47875205623808, 47875205656575, +ERASE, 47875205623808, 47875205623808, +STORE, 47875205623808, 47875205631999, +STORE, 47875205632000, 47875205656575, +STORE, 47875205644288, 47875205656575, +STORE, 47875205632000, 47875205644287, +ERASE, 47875205632000, 47875205632000, +STORE, 47875205632000, 47875205644287, +STORE, 47875205648384, 47875205656575, +STORE, 47875205644288, 47875205648383, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205656575, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205648383, +STORE, 47875205648384, 47875205656575, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205656575, +STORE, 47875205656576, 47875205693439, +ERASE, 47875205656576, 47875205656576, +STORE, 47875205656576, 47875205664767, +STORE, 47875205664768, 47875205693439, +STORE, 47875205681152, 47875205693439, +STORE, 47875205664768, 47875205681151, +ERASE, 47875205664768, 47875205664768, +STORE, 47875205664768, 47875205681151, +STORE, 47875205685248, 47875205693439, +STORE, 47875205681152, 47875205685247, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205693439, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205685247, +STORE, 47875205685248, 47875205693439, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205693439, +STORE, 47875205693440, 47875206168575, +ERASE, 47875205693440, 47875205693440, +STORE, 47875205693440, 47875205701631, +STORE, 47875205701632, 47875206168575, +STORE, 47875206037504, 47875206168575, +STORE, 47875205701632, 47875206037503, +ERASE, 47875205701632, 47875205701632, +STORE, 47875205701632, 47875206037503, +STORE, 47875206160384, 47875206168575, +STORE, 47875206037504, 47875206160383, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206168575, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206160383, +STORE, 47875206160384, 47875206168575, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206168575, +STORE, 47875206168576, 47875206176767, +STORE, 47875206176768, 47875206197247, +ERASE, 47875206176768, 47875206176768, +STORE, 47875206176768, 47875206180863, +STORE, 47875206180864, 47875206197247, +STORE, 47875206184960, 47875206197247, +STORE, 47875206180864, 47875206184959, +ERASE, 47875206180864, 47875206180864, +STORE, 47875206180864, 47875206184959, +STORE, 47875206189056, 47875206197247, +STORE, 47875206184960, 47875206189055, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206197247, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206189055, +STORE, 47875206189056, 47875206197247, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206197247, +STORE, 47875206197248, 47875206205439, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198877695, +STORE, 47875198877696, 47875198885887, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206193151, +STORE, 47875206193152, 47875206197247, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201781759, +STORE, 47875201781760, 47875201785855, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206164479, +STORE, 47875206164480, 47875206168575, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205689343, +STORE, 47875205689344, 47875205693439, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205652479, +STORE, 47875205652480, 47875205656575, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205599231, +STORE, 47875205599232, 47875205607423, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202555903, +STORE, 47875202555904, 47875202572287, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201970175, +STORE, 47875201970176, 47875201974271, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201654783, +STORE, 47875201654784, 47875201658879, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199401983, +STORE, 47875199401984, 47875199406079, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199053823, +STORE, 47875199053824, 47875199057919, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199008767, +STORE, 47875199008768, 47875199012863, +ERASE, 94011547025408, 94011547025408, +STORE, 94011547025408, 94011547148287, +STORE, 94011547148288, 94011547152383, +ERASE, 139757598109696, 139757598109696, +STORE, 139757598109696, 139757598113791, +STORE, 139757598113792, 139757598117887, +ERASE, 47875197046784, 47875197046784, +STORE, 94011557584896, 94011557720063, +STORE, 94011557584896, 94011557855231, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557851135, +STORE, 94011557851136, 94011557855231, +ERASE, 94011557851136, 94011557851136, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557847039, +STORE, 94011557847040, 94011557851135, +ERASE, 94011557847040, 94011557847040, +STORE, 94011557584896, 94011557982207, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557978111, +STORE, 94011557978112, 94011557982207, +ERASE, 94011557978112, 94011557978112, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557974015, +STORE, 94011557974016, 94011557978111, +ERASE, 94011557974016, 94011557974016, +STORE, 140737488347136, 140737488351231, +STORE, 140734130360320, 140737488351231, +ERASE, 140734130360320, 140734130360320, +STORE, 140734130360320, 140734130364415, +STORE, 94641232105472, 94641232785407, +ERASE, 94641232105472, 94641232105472, +STORE, 94641232105472, 94641232171007, +STORE, 94641232171008, 94641232785407, +ERASE, 94641232171008, 94641232171008, +STORE, 94641232171008, 94641232519167, +STORE, 94641232519168, 94641232658431, +STORE, 94641232658432, 94641232785407, +STORE, 139726599516160, 139726599688191, +ERASE, 139726599516160, 139726599516160, +STORE, 139726599516160, 139726599520255, +STORE, 139726599520256, 139726599688191, +ERASE, 139726599520256, 139726599520256, +STORE, 139726599520256, 139726599643135, +STORE, 139726599643136, 139726599675903, +STORE, 139726599675904, 139726599684095, +STORE, 139726599684096, 139726599688191, +STORE, 140734130446336, 140734130450431, +STORE, 140734130434048, 140734130446335, +STORE, 47906195480576, 47906195488767, +STORE, 47906195488768, 47906195496959, +STORE, 47906195496960, 47906197336063, +STORE, 47906195636224, 47906197336063, +STORE, 47906195496960, 47906195636223, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906197295103, +STORE, 47906197295104, 47906197336063, +STORE, 47906196979712, 47906197295103, +STORE, 47906195636224, 47906196979711, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906196979711, +STORE, 47906197291008, 47906197295103, +STORE, 47906196979712, 47906197291007, +ERASE, 47906196979712, 47906196979712, +STORE, 47906196979712, 47906197291007, +STORE, 47906197319680, 47906197336063, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197319680, 47906197319680, +STORE, 47906197319680, 47906197336063, +STORE, 47906197336064, 47906197446655, +STORE, 47906197352448, 47906197446655, +STORE, 47906197336064, 47906197352447, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197438463, +STORE, 47906197438464, 47906197446655, +STORE, 47906197413888, 47906197438463, +STORE, 47906197352448, 47906197413887, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197413887, +STORE, 47906197434368, 47906197438463, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197413888, 47906197413888, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197446655, +STORE, 47906197446656, 47906197491711, +ERASE, 47906197446656, 47906197446656, +STORE, 47906197446656, 47906197454847, +STORE, 47906197454848, 47906197491711, +STORE, 47906197475328, 47906197491711, +STORE, 47906197454848, 47906197475327, +ERASE, 47906197454848, 47906197454848, +STORE, 47906197454848, 47906197475327, +STORE, 47906197483520, 47906197491711, +STORE, 47906197475328, 47906197483519, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197491711, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197483519, +STORE, 47906197483520, 47906197491711, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197491711, +STORE, 47906197491712, 47906197839871, +STORE, 47906197532672, 47906197839871, +STORE, 47906197491712, 47906197532671, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197815295, +STORE, 47906197815296, 47906197839871, +STORE, 47906197745664, 47906197815295, +STORE, 47906197532672, 47906197745663, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197745663, +STORE, 47906197811200, 47906197815295, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197745664, 47906197745664, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197839871, +STORE, 47906197839872, 47906200100863, +STORE, 47906197991424, 47906200100863, +STORE, 47906197839872, 47906197991423, +ERASE, 47906197991424, 47906197991424, +STORE, 47906197991424, 47906200084479, +STORE, 47906200084480, 47906200100863, +STORE, 47906200092672, 47906200100863, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200092672, 47906200092672, +STORE, 47906200092672, 47906200100863, +STORE, 47906200100864, 47906200236031, +ERASE, 47906200100864, 47906200100864, +STORE, 47906200100864, 47906200125439, +STORE, 47906200125440, 47906200236031, +STORE, 47906200186880, 47906200236031, +STORE, 47906200125440, 47906200186879, +ERASE, 47906200125440, 47906200125440, +STORE, 47906200125440, 47906200186879, +STORE, 47906200211456, 47906200236031, +STORE, 47906200186880, 47906200211455, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200236031, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200211455, +STORE, 47906200211456, 47906200236031, +STORE, 47906200219648, 47906200236031, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200219648, 47906200219648, +STORE, 47906200219648, 47906200236031, +STORE, 47906200219648, 47906200244223, +STORE, 47906200244224, 47906200408063, +ERASE, 47906200244224, 47906200244224, +STORE, 47906200244224, 47906200256511, +STORE, 47906200256512, 47906200408063, +STORE, 47906200354816, 47906200408063, +STORE, 47906200256512, 47906200354815, +ERASE, 47906200256512, 47906200256512, +STORE, 47906200256512, 47906200354815, +STORE, 47906200399872, 47906200408063, +STORE, 47906200354816, 47906200399871, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200408063, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200399871, +STORE, 47906200399872, 47906200408063, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200408063, +STORE, 47906200408064, 47906201006079, +STORE, 47906200526848, 47906201006079, +STORE, 47906200408064, 47906200526847, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200952831, +STORE, 47906200952832, 47906201006079, +STORE, 47906200842240, 47906200952831, +STORE, 47906200526848, 47906200842239, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200842239, +STORE, 47906200948736, 47906200952831, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200842240, 47906200842240, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906201006079, +STORE, 47906201006080, 47906204057599, +STORE, 47906201550848, 47906204057599, +STORE, 47906201006080, 47906201550847, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203836415, +STORE, 47906203836416, 47906204057599, +STORE, 47906203246592, 47906203836415, +STORE, 47906201550848, 47906203246591, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203246591, +STORE, 47906203832320, 47906203836415, +STORE, 47906203246592, 47906203832319, +ERASE, 47906203246592, 47906203246592, +STORE, 47906203246592, 47906203832319, +STORE, 47906204041216, 47906204057599, +STORE, 47906203836416, 47906204041215, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204041215, +ERASE, 47906204041216, 47906204041216, +STORE, 47906204041216, 47906204057599, +STORE, 47906204057600, 47906204090367, +ERASE, 47906204057600, 47906204057600, +STORE, 47906204057600, 47906204065791, +STORE, 47906204065792, 47906204090367, +STORE, 47906204078080, 47906204090367, +STORE, 47906204065792, 47906204078079, +ERASE, 47906204065792, 47906204065792, +STORE, 47906204065792, 47906204078079, +STORE, 47906204082176, 47906204090367, +STORE, 47906204078080, 47906204082175, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204090367, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204082175, +STORE, 47906204082176, 47906204090367, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204090367, +STORE, 47906204090368, 47906204127231, +ERASE, 47906204090368, 47906204090368, +STORE, 47906204090368, 47906204098559, +STORE, 47906204098560, 47906204127231, +STORE, 47906204114944, 47906204127231, +STORE, 47906204098560, 47906204114943, +ERASE, 47906204098560, 47906204098560, +STORE, 47906204098560, 47906204114943, +STORE, 47906204119040, 47906204127231, +STORE, 47906204114944, 47906204119039, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204127231, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204119039, +STORE, 47906204119040, 47906204127231, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204127231, +STORE, 47906204127232, 47906204602367, +ERASE, 47906204127232, 47906204127232, +STORE, 47906204127232, 47906204135423, +STORE, 47906204135424, 47906204602367, +STORE, 47906204471296, 47906204602367, +STORE, 47906204135424, 47906204471295, +ERASE, 47906204135424, 47906204135424, +STORE, 47906204135424, 47906204471295, +STORE, 47906204594176, 47906204602367, +STORE, 47906204471296, 47906204594175, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204602367, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204594175, +STORE, 47906204594176, 47906204602367, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204602367, +STORE, 47906204602368, 47906204610559, +STORE, 47906204610560, 47906204631039, +ERASE, 47906204610560, 47906204610560, +STORE, 47906204610560, 47906204614655, +STORE, 47906204614656, 47906204631039, +STORE, 47906204618752, 47906204631039, +STORE, 47906204614656, 47906204618751, +ERASE, 47906204614656, 47906204614656, +STORE, 47906204614656, 47906204618751, +STORE, 47906204622848, 47906204631039, +STORE, 47906204618752, 47906204622847, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204631039, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204622847, +STORE, 47906204622848, 47906204631039, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204631039, +STORE, 47906204631040, 47906204639231, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197311487, +STORE, 47906197311488, 47906197319679, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204626943, +STORE, 47906204626944, 47906204631039, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200215551, +STORE, 47906200215552, 47906200219647, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204598271, +STORE, 47906204598272, 47906204602367, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204123135, +STORE, 47906204123136, 47906204127231, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204086271, +STORE, 47906204086272, 47906204090367, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204033023, +STORE, 47906204033024, 47906204041215, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906200989695, +STORE, 47906200989696, 47906201006079, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200403967, +STORE, 47906200403968, 47906200408063, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200088575, +STORE, 47906200088576, 47906200092671, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197835775, +STORE, 47906197835776, 47906197839871, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197487615, +STORE, 47906197487616, 47906197491711, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197442559, +STORE, 47906197442560, 47906197446655, +ERASE, 94641232658432, 94641232658432, +STORE, 94641232658432, 94641232781311, +STORE, 94641232781312, 94641232785407, +ERASE, 139726599675904, 139726599675904, +STORE, 139726599675904, 139726599679999, +STORE, 139726599680000, 139726599684095, +ERASE, 47906195480576, 47906195480576, +STORE, 94641242615808, 94641242750975, + }; + + unsigned long set10[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736427839488, 140737488351231, +ERASE, 140736427839488, 140736427839488, +STORE, 140736427839488, 140736427843583, +STORE, 94071213395968, 94071213567999, +ERASE, 94071213395968, 94071213395968, +STORE, 94071213395968, 94071213412351, +STORE, 94071213412352, 94071213567999, +ERASE, 94071213412352, 94071213412352, +STORE, 94071213412352, 94071213514751, +STORE, 94071213514752, 94071213555711, +STORE, 94071213555712, 94071213567999, +STORE, 139968410644480, 139968410816511, +ERASE, 139968410644480, 139968410644480, +STORE, 139968410644480, 139968410648575, +STORE, 139968410648576, 139968410816511, +ERASE, 139968410648576, 139968410648576, +STORE, 139968410648576, 139968410771455, +STORE, 139968410771456, 139968410804223, +STORE, 139968410804224, 139968410812415, +STORE, 139968410812416, 139968410816511, +STORE, 140736429277184, 140736429281279, +STORE, 140736429264896, 140736429277183, +STORE, 47664384352256, 47664384360447, +STORE, 47664384360448, 47664384368639, +STORE, 47664384368640, 47664384532479, +ERASE, 47664384368640, 47664384368640, +STORE, 47664384368640, 47664384380927, +STORE, 47664384380928, 47664384532479, +STORE, 47664384479232, 47664384532479, +STORE, 47664384380928, 47664384479231, +ERASE, 47664384380928, 47664384380928, +STORE, 47664384380928, 47664384479231, +STORE, 47664384524288, 47664384532479, +STORE, 47664384479232, 47664384524287, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384532479, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384524287, +STORE, 47664384524288, 47664384532479, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384532479, +STORE, 47664384532480, 47664387583999, +STORE, 47664385077248, 47664387583999, +STORE, 47664384532480, 47664385077247, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664387362815, +STORE, 47664387362816, 47664387583999, +STORE, 47664386772992, 47664387362815, +STORE, 47664385077248, 47664386772991, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664386772991, +STORE, 47664387358720, 47664387362815, +STORE, 47664386772992, 47664387358719, +ERASE, 47664386772992, 47664386772992, +STORE, 47664386772992, 47664387358719, +STORE, 47664387567616, 47664387583999, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387567616, 47664387567616, +STORE, 47664387567616, 47664387583999, +STORE, 47664387584000, 47664389423103, +STORE, 47664387723264, 47664389423103, +STORE, 47664387584000, 47664387723263, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389382143, +STORE, 47664389382144, 47664389423103, +STORE, 47664389066752, 47664389382143, +STORE, 47664387723264, 47664389066751, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389066751, +STORE, 47664389378048, 47664389382143, +STORE, 47664389066752, 47664389378047, +ERASE, 47664389066752, 47664389066752, +STORE, 47664389066752, 47664389378047, +STORE, 47664389406720, 47664389423103, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389406720, 47664389406720, +STORE, 47664389406720, 47664389423103, +STORE, 47664389423104, 47664389558271, +ERASE, 47664389423104, 47664389423104, +STORE, 47664389423104, 47664389447679, +STORE, 47664389447680, 47664389558271, +STORE, 47664389509120, 47664389558271, +STORE, 47664389447680, 47664389509119, +ERASE, 47664389447680, 47664389447680, +STORE, 47664389447680, 47664389509119, +STORE, 47664389533696, 47664389558271, +STORE, 47664389509120, 47664389533695, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389558271, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389533695, +STORE, 47664389533696, 47664389558271, +STORE, 47664389541888, 47664389558271, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389541888, 47664389541888, +STORE, 47664389541888, 47664389558271, +STORE, 47664389558272, 47664389578751, +ERASE, 47664389558272, 47664389558272, +STORE, 47664389558272, 47664389562367, +STORE, 47664389562368, 47664389578751, +STORE, 47664389566464, 47664389578751, +STORE, 47664389562368, 47664389566463, +ERASE, 47664389562368, 47664389562368, +STORE, 47664389562368, 47664389566463, +STORE, 47664389570560, 47664389578751, +STORE, 47664389566464, 47664389570559, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389578751, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389570559, +STORE, 47664389570560, 47664389578751, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389578751, +STORE, 47664389578752, 47664389586943, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389398527, +STORE, 47664389398528, 47664389406719, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389574655, +STORE, 47664389574656, 47664389578751, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389537791, +STORE, 47664389537792, 47664389541887, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387559423, +STORE, 47664387559424, 47664387567615, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384528383, +STORE, 47664384528384, 47664384532479, +ERASE, 94071213555712, 94071213555712, +STORE, 94071213555712, 94071213563903, +STORE, 94071213563904, 94071213567999, +ERASE, 139968410804224, 139968410804224, +STORE, 139968410804224, 139968410808319, +STORE, 139968410808320, 139968410812415, +ERASE, 47664384352256, 47664384352256, +STORE, 94071244402688, 94071244537855, +STORE, 140737488347136, 140737488351231, +STORE, 140728271503360, 140737488351231, +ERASE, 140728271503360, 140728271503360, +STORE, 140728271503360, 140728271507455, +STORE, 94410361982976, 94410362155007, +ERASE, 94410361982976, 94410361982976, +STORE, 94410361982976, 94410361999359, +STORE, 94410361999360, 94410362155007, +ERASE, 94410361999360, 94410361999360, +STORE, 94410361999360, 94410362101759, +STORE, 94410362101760, 94410362142719, +STORE, 94410362142720, 94410362155007, +STORE, 140351953997824, 140351954169855, +ERASE, 140351953997824, 140351953997824, +STORE, 140351953997824, 140351954001919, +STORE, 140351954001920, 140351954169855, +ERASE, 140351954001920, 140351954001920, +STORE, 140351954001920, 140351954124799, +STORE, 140351954124800, 140351954157567, +STORE, 140351954157568, 140351954165759, +STORE, 140351954165760, 140351954169855, +STORE, 140728272429056, 140728272433151, +STORE, 140728272416768, 140728272429055, +STORE, 47280840998912, 47280841007103, +STORE, 47280841007104, 47280841015295, +STORE, 47280841015296, 47280841179135, +ERASE, 47280841015296, 47280841015296, +STORE, 47280841015296, 47280841027583, +STORE, 47280841027584, 47280841179135, +STORE, 47280841125888, 47280841179135, +STORE, 47280841027584, 47280841125887, +ERASE, 47280841027584, 47280841027584, +STORE, 47280841027584, 47280841125887, +STORE, 47280841170944, 47280841179135, +STORE, 47280841125888, 47280841170943, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841179135, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841170943, +STORE, 47280841170944, 47280841179135, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841179135, +STORE, 47280841179136, 47280844230655, +STORE, 47280841723904, 47280844230655, +STORE, 47280841179136, 47280841723903, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280844009471, +STORE, 47280844009472, 47280844230655, +STORE, 47280843419648, 47280844009471, +STORE, 47280841723904, 47280843419647, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280843419647, +STORE, 47280844005376, 47280844009471, +STORE, 47280843419648, 47280844005375, +ERASE, 47280843419648, 47280843419648, +STORE, 47280843419648, 47280844005375, +STORE, 47280844214272, 47280844230655, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844214272, 47280844214272, +STORE, 47280844214272, 47280844230655, +STORE, 47280844230656, 47280846069759, +STORE, 47280844369920, 47280846069759, +STORE, 47280844230656, 47280844369919, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280846028799, +STORE, 47280846028800, 47280846069759, +STORE, 47280845713408, 47280846028799, +STORE, 47280844369920, 47280845713407, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280845713407, +STORE, 47280846024704, 47280846028799, +STORE, 47280845713408, 47280846024703, +ERASE, 47280845713408, 47280845713408, +STORE, 47280845713408, 47280846024703, +STORE, 47280846053376, 47280846069759, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846053376, 47280846053376, +STORE, 47280846053376, 47280846069759, +STORE, 47280846069760, 47280846204927, +ERASE, 47280846069760, 47280846069760, +STORE, 47280846069760, 47280846094335, +STORE, 47280846094336, 47280846204927, +STORE, 47280846155776, 47280846204927, +STORE, 47280846094336, 47280846155775, +ERASE, 47280846094336, 47280846094336, +STORE, 47280846094336, 47280846155775, +STORE, 47280846180352, 47280846204927, +STORE, 47280846155776, 47280846180351, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846204927, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846180351, +STORE, 47280846180352, 47280846204927, +STORE, 47280846188544, 47280846204927, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846188544, 47280846188544, +STORE, 47280846188544, 47280846204927, +STORE, 47280846204928, 47280846225407, +ERASE, 47280846204928, 47280846204928, +STORE, 47280846204928, 47280846209023, +STORE, 47280846209024, 47280846225407, +STORE, 47280846213120, 47280846225407, +STORE, 47280846209024, 47280846213119, +ERASE, 47280846209024, 47280846209024, +STORE, 47280846209024, 47280846213119, +STORE, 47280846217216, 47280846225407, +STORE, 47280846213120, 47280846217215, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846225407, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846217215, +STORE, 47280846217216, 47280846225407, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846225407, +STORE, 47280846225408, 47280846233599, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846045183, +STORE, 47280846045184, 47280846053375, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846221311, +STORE, 47280846221312, 47280846225407, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846184447, +STORE, 47280846184448, 47280846188543, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844206079, +STORE, 47280844206080, 47280844214271, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841175039, +STORE, 47280841175040, 47280841179135, +ERASE, 94410362142720, 94410362142720, +STORE, 94410362142720, 94410362150911, +STORE, 94410362150912, 94410362155007, +ERASE, 140351954157568, 140351954157568, +STORE, 140351954157568, 140351954161663, +STORE, 140351954161664, 140351954165759, +ERASE, 47280840998912, 47280840998912, +STORE, 94410379456512, 94410379591679, +STORE, 140737488347136, 140737488351231, +STORE, 140732946362368, 140737488351231, +ERASE, 140732946362368, 140732946362368, +STORE, 140732946362368, 140732946366463, +STORE, 94352937934848, 94352938106879, +ERASE, 94352937934848, 94352937934848, +STORE, 94352937934848, 94352937951231, +STORE, 94352937951232, 94352938106879, +ERASE, 94352937951232, 94352937951232, +STORE, 94352937951232, 94352938053631, +STORE, 94352938053632, 94352938094591, +STORE, 94352938094592, 94352938106879, +STORE, 140595518742528, 140595518914559, +ERASE, 140595518742528, 140595518742528, +STORE, 140595518742528, 140595518746623, +STORE, 140595518746624, 140595518914559, +ERASE, 140595518746624, 140595518746624, +STORE, 140595518746624, 140595518869503, +STORE, 140595518869504, 140595518902271, +STORE, 140595518902272, 140595518910463, +STORE, 140595518910464, 140595518914559, +STORE, 140732947468288, 140732947472383, +STORE, 140732947456000, 140732947468287, +STORE, 47037276254208, 47037276262399, +STORE, 47037276262400, 47037276270591, +STORE, 47037276270592, 47037276434431, +ERASE, 47037276270592, 47037276270592, +STORE, 47037276270592, 47037276282879, +STORE, 47037276282880, 47037276434431, +STORE, 47037276381184, 47037276434431, +STORE, 47037276282880, 47037276381183, +ERASE, 47037276282880, 47037276282880, +STORE, 47037276282880, 47037276381183, +STORE, 47037276426240, 47037276434431, +STORE, 47037276381184, 47037276426239, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276434431, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276426239, +STORE, 47037276426240, 47037276434431, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276434431, +STORE, 47037276434432, 47037279485951, +STORE, 47037276979200, 47037279485951, +STORE, 47037276434432, 47037276979199, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037279264767, +STORE, 47037279264768, 47037279485951, +STORE, 47037278674944, 47037279264767, +STORE, 47037276979200, 47037278674943, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037278674943, +STORE, 47037279260672, 47037279264767, +STORE, 47037278674944, 47037279260671, +ERASE, 47037278674944, 47037278674944, +STORE, 47037278674944, 47037279260671, +STORE, 47037279469568, 47037279485951, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279469568, 47037279469568, +STORE, 47037279469568, 47037279485951, +STORE, 47037279485952, 47037281325055, +STORE, 47037279625216, 47037281325055, +STORE, 47037279485952, 47037279625215, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037281284095, +STORE, 47037281284096, 47037281325055, +STORE, 47037280968704, 47037281284095, +STORE, 47037279625216, 47037280968703, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037280968703, +STORE, 47037281280000, 47037281284095, +STORE, 47037280968704, 47037281279999, +ERASE, 47037280968704, 47037280968704, +STORE, 47037280968704, 47037281279999, +STORE, 47037281308672, 47037281325055, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281308672, 47037281308672, +STORE, 47037281308672, 47037281325055, +STORE, 47037281325056, 47037281460223, +ERASE, 47037281325056, 47037281325056, +STORE, 47037281325056, 47037281349631, +STORE, 47037281349632, 47037281460223, +STORE, 47037281411072, 47037281460223, +STORE, 47037281349632, 47037281411071, +ERASE, 47037281349632, 47037281349632, +STORE, 47037281349632, 47037281411071, +STORE, 47037281435648, 47037281460223, +STORE, 47037281411072, 47037281435647, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281460223, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281435647, +STORE, 47037281435648, 47037281460223, +STORE, 47037281443840, 47037281460223, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281443840, 47037281443840, +STORE, 47037281443840, 47037281460223, +STORE, 47037281460224, 47037281480703, +ERASE, 47037281460224, 47037281460224, +STORE, 47037281460224, 47037281464319, +STORE, 47037281464320, 47037281480703, +STORE, 47037281468416, 47037281480703, +STORE, 47037281464320, 47037281468415, +ERASE, 47037281464320, 47037281464320, +STORE, 47037281464320, 47037281468415, +STORE, 47037281472512, 47037281480703, +STORE, 47037281468416, 47037281472511, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281480703, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281472511, +STORE, 47037281472512, 47037281480703, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281480703, +STORE, 47037281480704, 47037281488895, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281300479, +STORE, 47037281300480, 47037281308671, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281476607, +STORE, 47037281476608, 47037281480703, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281439743, +STORE, 47037281439744, 47037281443839, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279461375, +STORE, 47037279461376, 47037279469567, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276430335, +STORE, 47037276430336, 47037276434431, +ERASE, 94352938094592, 94352938094592, +STORE, 94352938094592, 94352938102783, +STORE, 94352938102784, 94352938106879, +ERASE, 140595518902272, 140595518902272, +STORE, 140595518902272, 140595518906367, +STORE, 140595518906368, 140595518910463, +ERASE, 47037276254208, 47037276254208, +STORE, 94352938438656, 94352938573823, +STORE, 140737488347136, 140737488351231, +STORE, 140733506027520, 140737488351231, +ERASE, 140733506027520, 140733506027520, +STORE, 140733506027520, 140733506031615, +STORE, 94150123073536, 94150123245567, +ERASE, 94150123073536, 94150123073536, +STORE, 94150123073536, 94150123089919, +STORE, 94150123089920, 94150123245567, +ERASE, 94150123089920, 94150123089920, +STORE, 94150123089920, 94150123192319, +STORE, 94150123192320, 94150123233279, +STORE, 94150123233280, 94150123245567, +STORE, 140081290375168, 140081290547199, +ERASE, 140081290375168, 140081290375168, +STORE, 140081290375168, 140081290379263, +STORE, 140081290379264, 140081290547199, +ERASE, 140081290379264, 140081290379264, +STORE, 140081290379264, 140081290502143, +STORE, 140081290502144, 140081290534911, +STORE, 140081290534912, 140081290543103, +STORE, 140081290543104, 140081290547199, +STORE, 140733506707456, 140733506711551, +STORE, 140733506695168, 140733506707455, +STORE, 47551504621568, 47551504629759, +STORE, 47551504629760, 47551504637951, +STORE, 47551504637952, 47551504801791, +ERASE, 47551504637952, 47551504637952, +STORE, 47551504637952, 47551504650239, +STORE, 47551504650240, 47551504801791, +STORE, 47551504748544, 47551504801791, +STORE, 47551504650240, 47551504748543, +ERASE, 47551504650240, 47551504650240, +STORE, 47551504650240, 47551504748543, +STORE, 47551504793600, 47551504801791, +STORE, 47551504748544, 47551504793599, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504801791, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504793599, +STORE, 47551504793600, 47551504801791, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504801791, +STORE, 47551504801792, 47551507853311, +STORE, 47551505346560, 47551507853311, +STORE, 47551504801792, 47551505346559, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507632127, +STORE, 47551507632128, 47551507853311, +STORE, 47551507042304, 47551507632127, +STORE, 47551505346560, 47551507042303, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507042303, +STORE, 47551507628032, 47551507632127, +STORE, 47551507042304, 47551507628031, +ERASE, 47551507042304, 47551507042304, +STORE, 47551507042304, 47551507628031, +STORE, 47551507836928, 47551507853311, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507836928, 47551507836928, +STORE, 47551507836928, 47551507853311, +STORE, 47551507853312, 47551509692415, +STORE, 47551507992576, 47551509692415, +STORE, 47551507853312, 47551507992575, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509651455, +STORE, 47551509651456, 47551509692415, +STORE, 47551509336064, 47551509651455, +STORE, 47551507992576, 47551509336063, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509336063, +STORE, 47551509647360, 47551509651455, +STORE, 47551509336064, 47551509647359, +ERASE, 47551509336064, 47551509336064, +STORE, 47551509336064, 47551509647359, +STORE, 47551509676032, 47551509692415, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509676032, 47551509676032, +STORE, 47551509676032, 47551509692415, +STORE, 47551509692416, 47551509827583, +ERASE, 47551509692416, 47551509692416, +STORE, 47551509692416, 47551509716991, +STORE, 47551509716992, 47551509827583, +STORE, 47551509778432, 47551509827583, +STORE, 47551509716992, 47551509778431, +ERASE, 47551509716992, 47551509716992, +STORE, 47551509716992, 47551509778431, +STORE, 47551509803008, 47551509827583, +STORE, 47551509778432, 47551509803007, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509827583, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509803007, +STORE, 47551509803008, 47551509827583, +STORE, 47551509811200, 47551509827583, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509811200, 47551509811200, +STORE, 47551509811200, 47551509827583, +STORE, 47551509827584, 47551509848063, +ERASE, 47551509827584, 47551509827584, +STORE, 47551509827584, 47551509831679, +STORE, 47551509831680, 47551509848063, +STORE, 47551509835776, 47551509848063, +STORE, 47551509831680, 47551509835775, +ERASE, 47551509831680, 47551509831680, +STORE, 47551509831680, 47551509835775, +STORE, 47551509839872, 47551509848063, +STORE, 47551509835776, 47551509839871, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509848063, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509839871, +STORE, 47551509839872, 47551509848063, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509848063, +STORE, 47551509848064, 47551509856255, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509667839, +STORE, 47551509667840, 47551509676031, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509843967, +STORE, 47551509843968, 47551509848063, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509807103, +STORE, 47551509807104, 47551509811199, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507828735, +STORE, 47551507828736, 47551507836927, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504797695, +STORE, 47551504797696, 47551504801791, +ERASE, 94150123233280, 94150123233280, +STORE, 94150123233280, 94150123241471, +STORE, 94150123241472, 94150123245567, +ERASE, 140081290534912, 140081290534912, +STORE, 140081290534912, 140081290539007, +STORE, 140081290539008, 140081290543103, +ERASE, 47551504621568, 47551504621568, +STORE, 94150148112384, 94150148247551, +STORE, 140737488347136, 140737488351231, +STORE, 140734389334016, 140737488351231, +ERASE, 140734389334016, 140734389334016, +STORE, 140734389334016, 140734389338111, +STORE, 94844636606464, 94844636778495, +ERASE, 94844636606464, 94844636606464, +STORE, 94844636606464, 94844636622847, +STORE, 94844636622848, 94844636778495, +ERASE, 94844636622848, 94844636622848, +STORE, 94844636622848, 94844636725247, +STORE, 94844636725248, 94844636766207, +STORE, 94844636766208, 94844636778495, +STORE, 139922765217792, 139922765389823, +ERASE, 139922765217792, 139922765217792, +STORE, 139922765217792, 139922765221887, +STORE, 139922765221888, 139922765389823, +ERASE, 139922765221888, 139922765221888, +STORE, 139922765221888, 139922765344767, +STORE, 139922765344768, 139922765377535, +STORE, 139922765377536, 139922765385727, +STORE, 139922765385728, 139922765389823, +STORE, 140734389678080, 140734389682175, +STORE, 140734389665792, 140734389678079, +STORE, 47710029778944, 47710029787135, +STORE, 47710029787136, 47710029795327, +STORE, 47710029795328, 47710029959167, +ERASE, 47710029795328, 47710029795328, +STORE, 47710029795328, 47710029807615, +STORE, 47710029807616, 47710029959167, +STORE, 47710029905920, 47710029959167, +STORE, 47710029807616, 47710029905919, +ERASE, 47710029807616, 47710029807616, +STORE, 47710029807616, 47710029905919, +STORE, 47710029950976, 47710029959167, +STORE, 47710029905920, 47710029950975, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029959167, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029950975, +STORE, 47710029950976, 47710029959167, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029959167, +STORE, 47710029959168, 47710033010687, +STORE, 47710030503936, 47710033010687, +STORE, 47710029959168, 47710030503935, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032789503, +STORE, 47710032789504, 47710033010687, +STORE, 47710032199680, 47710032789503, +STORE, 47710030503936, 47710032199679, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032199679, +STORE, 47710032785408, 47710032789503, +STORE, 47710032199680, 47710032785407, +ERASE, 47710032199680, 47710032199680, +STORE, 47710032199680, 47710032785407, +STORE, 47710032994304, 47710033010687, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032994304, 47710032994304, +STORE, 47710032994304, 47710033010687, +STORE, 47710033010688, 47710034849791, +STORE, 47710033149952, 47710034849791, +STORE, 47710033010688, 47710033149951, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034808831, +STORE, 47710034808832, 47710034849791, +STORE, 47710034493440, 47710034808831, +STORE, 47710033149952, 47710034493439, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034493439, +STORE, 47710034804736, 47710034808831, +STORE, 47710034493440, 47710034804735, +ERASE, 47710034493440, 47710034493440, +STORE, 47710034493440, 47710034804735, +STORE, 47710034833408, 47710034849791, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034833408, 47710034833408, +STORE, 47710034833408, 47710034849791, +STORE, 47710034849792, 47710034984959, +ERASE, 47710034849792, 47710034849792, +STORE, 47710034849792, 47710034874367, +STORE, 47710034874368, 47710034984959, +STORE, 47710034935808, 47710034984959, +STORE, 47710034874368, 47710034935807, +ERASE, 47710034874368, 47710034874368, +STORE, 47710034874368, 47710034935807, +STORE, 47710034960384, 47710034984959, +STORE, 47710034935808, 47710034960383, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034984959, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034960383, +STORE, 47710034960384, 47710034984959, +STORE, 47710034968576, 47710034984959, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034968576, 47710034968576, +STORE, 47710034968576, 47710034984959, +STORE, 47710034984960, 47710035005439, +ERASE, 47710034984960, 47710034984960, +STORE, 47710034984960, 47710034989055, +STORE, 47710034989056, 47710035005439, +STORE, 47710034993152, 47710035005439, +STORE, 47710034989056, 47710034993151, +ERASE, 47710034989056, 47710034989056, +STORE, 47710034989056, 47710034993151, +STORE, 47710034997248, 47710035005439, +STORE, 47710034993152, 47710034997247, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710035005439, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710034997247, +STORE, 47710034997248, 47710035005439, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035005439, +STORE, 47710035005440, 47710035013631, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034825215, +STORE, 47710034825216, 47710034833407, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035001343, +STORE, 47710035001344, 47710035005439, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034964479, +STORE, 47710034964480, 47710034968575, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032986111, +STORE, 47710032986112, 47710032994303, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029955071, +STORE, 47710029955072, 47710029959167, +ERASE, 94844636766208, 94844636766208, +STORE, 94844636766208, 94844636774399, +STORE, 94844636774400, 94844636778495, +ERASE, 139922765377536, 139922765377536, +STORE, 139922765377536, 139922765381631, +STORE, 139922765381632, 139922765385727, +ERASE, 47710029778944, 47710029778944, +STORE, 94844641775616, 94844641910783, +STORE, 140737488347136, 140737488351231, +STORE, 140732213886976, 140737488351231, +ERASE, 140732213886976, 140732213886976, +STORE, 140732213886976, 140732213891071, +STORE, 94240508887040, 94240509059071, +ERASE, 94240508887040, 94240508887040, +STORE, 94240508887040, 94240508903423, +STORE, 94240508903424, 94240509059071, +ERASE, 94240508903424, 94240508903424, +STORE, 94240508903424, 94240509005823, +STORE, 94240509005824, 94240509046783, +STORE, 94240509046784, 94240509059071, +STORE, 140275106516992, 140275106689023, +ERASE, 140275106516992, 140275106516992, +STORE, 140275106516992, 140275106521087, +STORE, 140275106521088, 140275106689023, +ERASE, 140275106521088, 140275106521088, +STORE, 140275106521088, 140275106643967, +STORE, 140275106643968, 140275106676735, +STORE, 140275106676736, 140275106684927, +STORE, 140275106684928, 140275106689023, +STORE, 140732213977088, 140732213981183, +STORE, 140732213964800, 140732213977087, +STORE, 47357688479744, 47357688487935, +STORE, 47357688487936, 47357688496127, +STORE, 47357688496128, 47357688659967, +ERASE, 47357688496128, 47357688496128, +STORE, 47357688496128, 47357688508415, +STORE, 47357688508416, 47357688659967, +STORE, 47357688606720, 47357688659967, +STORE, 47357688508416, 47357688606719, +ERASE, 47357688508416, 47357688508416, +STORE, 47357688508416, 47357688606719, +STORE, 47357688651776, 47357688659967, +STORE, 47357688606720, 47357688651775, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688659967, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688651775, +STORE, 47357688651776, 47357688659967, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688659967, +STORE, 47357688659968, 47357691711487, +STORE, 47357689204736, 47357691711487, +STORE, 47357688659968, 47357689204735, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357691490303, +STORE, 47357691490304, 47357691711487, +STORE, 47357690900480, 47357691490303, +STORE, 47357689204736, 47357690900479, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357690900479, +STORE, 47357691486208, 47357691490303, +STORE, 47357690900480, 47357691486207, +ERASE, 47357690900480, 47357690900480, +STORE, 47357690900480, 47357691486207, +STORE, 47357691695104, 47357691711487, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691695104, 47357691695104, +STORE, 47357691695104, 47357691711487, +STORE, 47357691711488, 47357693550591, +STORE, 47357691850752, 47357693550591, +STORE, 47357691711488, 47357691850751, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693509631, +STORE, 47357693509632, 47357693550591, +STORE, 47357693194240, 47357693509631, +STORE, 47357691850752, 47357693194239, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693194239, +STORE, 47357693505536, 47357693509631, +STORE, 47357693194240, 47357693505535, +ERASE, 47357693194240, 47357693194240, +STORE, 47357693194240, 47357693505535, +STORE, 47357693534208, 47357693550591, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693534208, 47357693534208, +STORE, 47357693534208, 47357693550591, +STORE, 47357693550592, 47357693685759, +ERASE, 47357693550592, 47357693550592, +STORE, 47357693550592, 47357693575167, +STORE, 47357693575168, 47357693685759, +STORE, 47357693636608, 47357693685759, +STORE, 47357693575168, 47357693636607, +ERASE, 47357693575168, 47357693575168, +STORE, 47357693575168, 47357693636607, +STORE, 47357693661184, 47357693685759, +STORE, 47357693636608, 47357693661183, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693685759, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693661183, +STORE, 47357693661184, 47357693685759, +STORE, 47357693669376, 47357693685759, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693669376, 47357693669376, +STORE, 47357693669376, 47357693685759, +STORE, 47357693685760, 47357693706239, +ERASE, 47357693685760, 47357693685760, +STORE, 47357693685760, 47357693689855, +STORE, 47357693689856, 47357693706239, +STORE, 47357693693952, 47357693706239, +STORE, 47357693689856, 47357693693951, +ERASE, 47357693689856, 47357693689856, +STORE, 47357693689856, 47357693693951, +STORE, 47357693698048, 47357693706239, +STORE, 47357693693952, 47357693698047, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693706239, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693698047, +STORE, 47357693698048, 47357693706239, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693706239, +STORE, 47357693706240, 47357693714431, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693526015, +STORE, 47357693526016, 47357693534207, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693702143, +STORE, 47357693702144, 47357693706239, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693665279, +STORE, 47357693665280, 47357693669375, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691686911, +STORE, 47357691686912, 47357691695103, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688655871, +STORE, 47357688655872, 47357688659967, +ERASE, 94240509046784, 94240509046784, +STORE, 94240509046784, 94240509054975, +STORE, 94240509054976, 94240509059071, +ERASE, 140275106676736, 140275106676736, +STORE, 140275106676736, 140275106680831, +STORE, 140275106680832, 140275106684927, +ERASE, 47357688479744, 47357688479744, +STORE, 94240518361088, 94240518496255, +STORE, 140737488347136, 140737488351231, +STORE, 140732688277504, 140737488351231, +ERASE, 140732688277504, 140732688277504, +STORE, 140732688277504, 140732688281599, +STORE, 94629171351552, 94629172064255, +ERASE, 94629171351552, 94629171351552, +STORE, 94629171351552, 94629171400703, +STORE, 94629171400704, 94629172064255, +ERASE, 94629171400704, 94629171400704, +STORE, 94629171400704, 94629171945471, +STORE, 94629171945472, 94629172043775, +STORE, 94629172043776, 94629172064255, +STORE, 139770707644416, 139770707816447, +ERASE, 139770707644416, 139770707644416, +STORE, 139770707644416, 139770707648511, +STORE, 139770707648512, 139770707816447, +ERASE, 139770707648512, 139770707648512, +STORE, 139770707648512, 139770707771391, +STORE, 139770707771392, 139770707804159, +STORE, 139770707804160, 139770707812351, +STORE, 139770707812352, 139770707816447, +STORE, 140732689121280, 140732689125375, +STORE, 140732689108992, 140732689121279, +STORE, 47862087352320, 47862087360511, +STORE, 47862087360512, 47862087368703, +STORE, 47862087368704, 47862087475199, +STORE, 47862087385088, 47862087475199, +STORE, 47862087368704, 47862087385087, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087458815, +STORE, 47862087458816, 47862087475199, +STORE, 47862087438336, 47862087458815, +STORE, 47862087385088, 47862087438335, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087438335, +STORE, 47862087454720, 47862087458815, +STORE, 47862087438336, 47862087454719, +ERASE, 47862087438336, 47862087438336, +STORE, 47862087438336, 47862087454719, +STORE, 47862087467008, 47862087475199, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087467008, 47862087467008, +STORE, 47862087467008, 47862087475199, +STORE, 47862087475200, 47862089314303, +STORE, 47862087614464, 47862089314303, +STORE, 47862087475200, 47862087614463, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862089273343, +STORE, 47862089273344, 47862089314303, +STORE, 47862088957952, 47862089273343, +STORE, 47862087614464, 47862088957951, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862088957951, +STORE, 47862089269248, 47862089273343, +STORE, 47862088957952, 47862089269247, +ERASE, 47862088957952, 47862088957952, +STORE, 47862088957952, 47862089269247, +STORE, 47862089297920, 47862089314303, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089297920, 47862089297920, +STORE, 47862089297920, 47862089314303, +STORE, 47862089297920, 47862089326591, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089289727, +STORE, 47862089289728, 47862089297919, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087462911, +STORE, 47862087462912, 47862087467007, +ERASE, 94629172043776, 94629172043776, +STORE, 94629172043776, 94629172060159, +STORE, 94629172060160, 94629172064255, +ERASE, 139770707804160, 139770707804160, +STORE, 139770707804160, 139770707808255, +STORE, 139770707808256, 139770707812351, +ERASE, 47862087352320, 47862087352320, +STORE, 94629197533184, 94629197668351, +STORE, 140737488347136, 140737488351231, +STORE, 140727540711424, 140737488351231, +ERASE, 140727540711424, 140727540711424, +STORE, 140727540711424, 140727540715519, +STORE, 94299865313280, 94299866025983, +ERASE, 94299865313280, 94299865313280, +STORE, 94299865313280, 94299865362431, +STORE, 94299865362432, 94299866025983, +ERASE, 94299865362432, 94299865362432, +STORE, 94299865362432, 94299865907199, +STORE, 94299865907200, 94299866005503, +STORE, 94299866005504, 94299866025983, +STORE, 140680268763136, 140680268935167, +ERASE, 140680268763136, 140680268763136, +STORE, 140680268763136, 140680268767231, +STORE, 140680268767232, 140680268935167, +ERASE, 140680268767232, 140680268767232, +STORE, 140680268767232, 140680268890111, +STORE, 140680268890112, 140680268922879, +STORE, 140680268922880, 140680268931071, +STORE, 140680268931072, 140680268935167, +STORE, 140727541424128, 140727541428223, +STORE, 140727541411840, 140727541424127, +STORE, 46952526233600, 46952526241791, +STORE, 46952526241792, 46952526249983, +STORE, 46952526249984, 46952526356479, +STORE, 46952526266368, 46952526356479, +STORE, 46952526249984, 46952526266367, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526340095, +STORE, 46952526340096, 46952526356479, +STORE, 46952526319616, 46952526340095, +STORE, 46952526266368, 46952526319615, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526319615, +STORE, 46952526336000, 46952526340095, +STORE, 46952526319616, 46952526335999, +ERASE, 46952526319616, 46952526319616, +STORE, 46952526319616, 46952526335999, +STORE, 46952526348288, 46952526356479, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526348288, 46952526348288, +STORE, 46952526348288, 46952526356479, +STORE, 46952526356480, 46952528195583, +STORE, 46952526495744, 46952528195583, +STORE, 46952526356480, 46952526495743, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952528154623, +STORE, 46952528154624, 46952528195583, +STORE, 46952527839232, 46952528154623, +STORE, 46952526495744, 46952527839231, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952527839231, +STORE, 46952528150528, 46952528154623, +STORE, 46952527839232, 46952528150527, +ERASE, 46952527839232, 46952527839232, +STORE, 46952527839232, 46952528150527, +STORE, 46952528179200, 46952528195583, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528179200, 46952528179200, +STORE, 46952528179200, 46952528195583, +STORE, 46952528179200, 46952528207871, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528171007, +STORE, 46952528171008, 46952528179199, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526344191, +STORE, 46952526344192, 46952526348287, +ERASE, 94299866005504, 94299866005504, +STORE, 94299866005504, 94299866021887, +STORE, 94299866021888, 94299866025983, +ERASE, 140680268922880, 140680268922880, +STORE, 140680268922880, 140680268926975, +STORE, 140680268926976, 140680268931071, +ERASE, 46952526233600, 46952526233600, +STORE, 140737488347136, 140737488351231, +STORE, 140722874793984, 140737488351231, +ERASE, 140722874793984, 140722874793984, +STORE, 140722874793984, 140722874798079, +STORE, 94448916213760, 94448916926463, +ERASE, 94448916213760, 94448916213760, +STORE, 94448916213760, 94448916262911, +STORE, 94448916262912, 94448916926463, +ERASE, 94448916262912, 94448916262912, +STORE, 94448916262912, 94448916807679, +STORE, 94448916807680, 94448916905983, +STORE, 94448916905984, 94448916926463, +STORE, 140389117046784, 140389117218815, +ERASE, 140389117046784, 140389117046784, +STORE, 140389117046784, 140389117050879, +STORE, 140389117050880, 140389117218815, +ERASE, 140389117050880, 140389117050880, +STORE, 140389117050880, 140389117173759, +STORE, 140389117173760, 140389117206527, +STORE, 140389117206528, 140389117214719, +STORE, 140389117214720, 140389117218815, +STORE, 140722875297792, 140722875301887, +STORE, 140722875285504, 140722875297791, +STORE, 47243677949952, 47243677958143, +STORE, 47243677958144, 47243677966335, +STORE, 47243677966336, 47243678072831, +STORE, 47243677982720, 47243678072831, +STORE, 47243677966336, 47243677982719, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678056447, +STORE, 47243678056448, 47243678072831, +STORE, 47243678035968, 47243678056447, +STORE, 47243677982720, 47243678035967, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678035967, +STORE, 47243678052352, 47243678056447, +STORE, 47243678035968, 47243678052351, +ERASE, 47243678035968, 47243678035968, +STORE, 47243678035968, 47243678052351, +STORE, 47243678064640, 47243678072831, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678064640, 47243678064640, +STORE, 47243678064640, 47243678072831, +STORE, 47243678072832, 47243679911935, +STORE, 47243678212096, 47243679911935, +STORE, 47243678072832, 47243678212095, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679870975, +STORE, 47243679870976, 47243679911935, +STORE, 47243679555584, 47243679870975, +STORE, 47243678212096, 47243679555583, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679555583, +STORE, 47243679866880, 47243679870975, +STORE, 47243679555584, 47243679866879, +ERASE, 47243679555584, 47243679555584, +STORE, 47243679555584, 47243679866879, +STORE, 47243679895552, 47243679911935, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679895552, 47243679895552, +STORE, 47243679895552, 47243679911935, +STORE, 47243679895552, 47243679924223, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679887359, +STORE, 47243679887360, 47243679895551, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678060543, +STORE, 47243678060544, 47243678064639, +ERASE, 94448916905984, 94448916905984, +STORE, 94448916905984, 94448916922367, +STORE, 94448916922368, 94448916926463, +ERASE, 140389117206528, 140389117206528, +STORE, 140389117206528, 140389117210623, +STORE, 140389117210624, 140389117214719, +ERASE, 47243677949952, 47243677949952, +STORE, 140737488347136, 140737488351231, +STORE, 140733068505088, 140737488351231, +ERASE, 140733068505088, 140733068505088, +STORE, 140733068505088, 140733068509183, +STORE, 94207145750528, 94207146463231, +ERASE, 94207145750528, 94207145750528, +STORE, 94207145750528, 94207145799679, +STORE, 94207145799680, 94207146463231, +ERASE, 94207145799680, 94207145799680, +STORE, 94207145799680, 94207146344447, +STORE, 94207146344448, 94207146442751, +STORE, 94207146442752, 94207146463231, +STORE, 140684504911872, 140684505083903, +ERASE, 140684504911872, 140684504911872, +STORE, 140684504911872, 140684504915967, +STORE, 140684504915968, 140684505083903, +ERASE, 140684504915968, 140684504915968, +STORE, 140684504915968, 140684505038847, +STORE, 140684505038848, 140684505071615, +STORE, 140684505071616, 140684505079807, +STORE, 140684505079808, 140684505083903, +STORE, 140733068607488, 140733068611583, +STORE, 140733068595200, 140733068607487, +STORE, 46948290084864, 46948290093055, +STORE, 46948290093056, 46948290101247, +STORE, 46948290101248, 46948290207743, +STORE, 46948290117632, 46948290207743, +STORE, 46948290101248, 46948290117631, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290191359, +STORE, 46948290191360, 46948290207743, +STORE, 46948290170880, 46948290191359, +STORE, 46948290117632, 46948290170879, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290170879, +STORE, 46948290187264, 46948290191359, +STORE, 46948290170880, 46948290187263, +ERASE, 46948290170880, 46948290170880, +STORE, 46948290170880, 46948290187263, +STORE, 46948290199552, 46948290207743, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290199552, 46948290199552, +STORE, 46948290199552, 46948290207743, +STORE, 46948290207744, 46948292046847, +STORE, 46948290347008, 46948292046847, +STORE, 46948290207744, 46948290347007, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948292005887, +STORE, 46948292005888, 46948292046847, +STORE, 46948291690496, 46948292005887, +STORE, 46948290347008, 46948291690495, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948291690495, +STORE, 46948292001792, 46948292005887, +STORE, 46948291690496, 46948292001791, +ERASE, 46948291690496, 46948291690496, +STORE, 46948291690496, 46948292001791, +STORE, 46948292030464, 46948292046847, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292030464, 46948292030464, +STORE, 46948292030464, 46948292046847, +STORE, 46948292030464, 46948292059135, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292022271, +STORE, 46948292022272, 46948292030463, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290195455, +STORE, 46948290195456, 46948290199551, +ERASE, 94207146442752, 94207146442752, +STORE, 94207146442752, 94207146459135, +STORE, 94207146459136, 94207146463231, +ERASE, 140684505071616, 140684505071616, +STORE, 140684505071616, 140684505075711, +STORE, 140684505075712, 140684505079807, +ERASE, 46948290084864, 46948290084864, +STORE, 140737488347136, 140737488351231, +STORE, 140726367158272, 140737488351231, +ERASE, 140726367158272, 140726367158272, +STORE, 140726367158272, 140726367162367, +STORE, 94436124106752, 94436124819455, +ERASE, 94436124106752, 94436124106752, +STORE, 94436124106752, 94436124155903, +STORE, 94436124155904, 94436124819455, +ERASE, 94436124155904, 94436124155904, +STORE, 94436124155904, 94436124700671, +STORE, 94436124700672, 94436124798975, +STORE, 94436124798976, 94436124819455, +STORE, 140049025044480, 140049025216511, +ERASE, 140049025044480, 140049025044480, +STORE, 140049025044480, 140049025048575, +STORE, 140049025048576, 140049025216511, +ERASE, 140049025048576, 140049025048576, +STORE, 140049025048576, 140049025171455, +STORE, 140049025171456, 140049025204223, +STORE, 140049025204224, 140049025212415, +STORE, 140049025212416, 140049025216511, +STORE, 140726367256576, 140726367260671, +STORE, 140726367244288, 140726367256575, +STORE, 47583769952256, 47583769960447, +STORE, 47583769960448, 47583769968639, +STORE, 47583769968640, 47583770075135, +STORE, 47583769985024, 47583770075135, +STORE, 47583769968640, 47583769985023, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770058751, +STORE, 47583770058752, 47583770075135, +STORE, 47583770038272, 47583770058751, +STORE, 47583769985024, 47583770038271, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770038271, +STORE, 47583770054656, 47583770058751, +STORE, 47583770038272, 47583770054655, +ERASE, 47583770038272, 47583770038272, +STORE, 47583770038272, 47583770054655, +STORE, 47583770066944, 47583770075135, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770066944, 47583770066944, +STORE, 47583770066944, 47583770075135, +STORE, 47583770075136, 47583771914239, +STORE, 47583770214400, 47583771914239, +STORE, 47583770075136, 47583770214399, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771873279, +STORE, 47583771873280, 47583771914239, +STORE, 47583771557888, 47583771873279, +STORE, 47583770214400, 47583771557887, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771557887, +STORE, 47583771869184, 47583771873279, +STORE, 47583771557888, 47583771869183, +ERASE, 47583771557888, 47583771557888, +STORE, 47583771557888, 47583771869183, +STORE, 47583771897856, 47583771914239, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771897856, 47583771897856, +STORE, 47583771897856, 47583771914239, +STORE, 47583771897856, 47583771926527, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771889663, +STORE, 47583771889664, 47583771897855, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770062847, +STORE, 47583770062848, 47583770066943, +ERASE, 94436124798976, 94436124798976, +STORE, 94436124798976, 94436124815359, +STORE, 94436124815360, 94436124819455, +ERASE, 140049025204224, 140049025204224, +STORE, 140049025204224, 140049025208319, +STORE, 140049025208320, 140049025212415, +ERASE, 47583769952256, 47583769952256, +STORE, 140737488347136, 140737488351231, +STORE, 140727116099584, 140737488351231, +ERASE, 140727116099584, 140727116099584, +STORE, 140727116099584, 140727116103679, +STORE, 94166319734784, 94166320447487, +ERASE, 94166319734784, 94166319734784, +STORE, 94166319734784, 94166319783935, +STORE, 94166319783936, 94166320447487, +ERASE, 94166319783936, 94166319783936, +STORE, 94166319783936, 94166320328703, +STORE, 94166320328704, 94166320427007, +STORE, 94166320427008, 94166320447487, +STORE, 139976559542272, 139976559714303, +ERASE, 139976559542272, 139976559542272, +STORE, 139976559542272, 139976559546367, +STORE, 139976559546368, 139976559714303, +ERASE, 139976559546368, 139976559546368, +STORE, 139976559546368, 139976559669247, +STORE, 139976559669248, 139976559702015, +STORE, 139976559702016, 139976559710207, +STORE, 139976559710208, 139976559714303, +STORE, 140727116222464, 140727116226559, +STORE, 140727116210176, 140727116222463, +STORE, 47656235454464, 47656235462655, +STORE, 47656235462656, 47656235470847, +STORE, 47656235470848, 47656235577343, +STORE, 47656235487232, 47656235577343, +STORE, 47656235470848, 47656235487231, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235560959, +STORE, 47656235560960, 47656235577343, +STORE, 47656235540480, 47656235560959, +STORE, 47656235487232, 47656235540479, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235540479, +STORE, 47656235556864, 47656235560959, +STORE, 47656235540480, 47656235556863, +ERASE, 47656235540480, 47656235540480, +STORE, 47656235540480, 47656235556863, +STORE, 47656235569152, 47656235577343, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235569152, 47656235569152, +STORE, 47656235569152, 47656235577343, +STORE, 47656235577344, 47656237416447, +STORE, 47656235716608, 47656237416447, +STORE, 47656235577344, 47656235716607, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237375487, +STORE, 47656237375488, 47656237416447, +STORE, 47656237060096, 47656237375487, +STORE, 47656235716608, 47656237060095, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237060095, +STORE, 47656237371392, 47656237375487, +STORE, 47656237060096, 47656237371391, +ERASE, 47656237060096, 47656237060096, +STORE, 47656237060096, 47656237371391, +STORE, 47656237400064, 47656237416447, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237400064, 47656237400064, +STORE, 47656237400064, 47656237416447, +STORE, 47656237400064, 47656237428735, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237391871, +STORE, 47656237391872, 47656237400063, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235565055, +STORE, 47656235565056, 47656235569151, +ERASE, 94166320427008, 94166320427008, +STORE, 94166320427008, 94166320443391, +STORE, 94166320443392, 94166320447487, +ERASE, 139976559702016, 139976559702016, +STORE, 139976559702016, 139976559706111, +STORE, 139976559706112, 139976559710207, +ERASE, 47656235454464, 47656235454464, +STORE, 94166332153856, 94166332289023, +STORE, 140737488347136, 140737488351231, +STORE, 140726412816384, 140737488351231, +ERASE, 140726412816384, 140726412816384, +STORE, 140726412816384, 140726412820479, +STORE, 94094884507648, 94094885220351, +ERASE, 94094884507648, 94094884507648, +STORE, 94094884507648, 94094884556799, +STORE, 94094884556800, 94094885220351, +ERASE, 94094884556800, 94094884556800, +STORE, 94094884556800, 94094885101567, +STORE, 94094885101568, 94094885199871, +STORE, 94094885199872, 94094885220351, +STORE, 139773773938688, 139773774110719, +ERASE, 139773773938688, 139773773938688, +STORE, 139773773938688, 139773773942783, +STORE, 139773773942784, 139773774110719, +ERASE, 139773773942784, 139773773942784, +STORE, 139773773942784, 139773774065663, +STORE, 139773774065664, 139773774098431, +STORE, 139773774098432, 139773774106623, +STORE, 139773774106624, 139773774110719, +STORE, 140726412963840, 140726412967935, +STORE, 140726412951552, 140726412963839, +STORE, 47859021058048, 47859021066239, +STORE, 47859021066240, 47859021074431, +STORE, 47859021074432, 47859021180927, +STORE, 47859021090816, 47859021180927, +STORE, 47859021074432, 47859021090815, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021164543, +STORE, 47859021164544, 47859021180927, +STORE, 47859021144064, 47859021164543, +STORE, 47859021090816, 47859021144063, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021144063, +STORE, 47859021160448, 47859021164543, +STORE, 47859021144064, 47859021160447, +ERASE, 47859021144064, 47859021144064, +STORE, 47859021144064, 47859021160447, +STORE, 47859021172736, 47859021180927, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021172736, 47859021172736, +STORE, 47859021172736, 47859021180927, +STORE, 47859021180928, 47859023020031, +STORE, 47859021320192, 47859023020031, +STORE, 47859021180928, 47859021320191, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022979071, +STORE, 47859022979072, 47859023020031, +STORE, 47859022663680, 47859022979071, +STORE, 47859021320192, 47859022663679, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022663679, +STORE, 47859022974976, 47859022979071, +STORE, 47859022663680, 47859022974975, +ERASE, 47859022663680, 47859022663680, +STORE, 47859022663680, 47859022974975, +STORE, 47859023003648, 47859023020031, +STORE, 47859022979072, 47859023003647, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859023003647, +ERASE, 47859023003648, 47859023003648, +STORE, 47859023003648, 47859023020031, +STORE, 47859023003648, 47859023032319, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859022995455, +STORE, 47859022995456, 47859023003647, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021168639, +STORE, 47859021168640, 47859021172735, +ERASE, 94094885199872, 94094885199872, +STORE, 94094885199872, 94094885216255, +STORE, 94094885216256, 94094885220351, +ERASE, 139773774098432, 139773774098432, +STORE, 139773774098432, 139773774102527, +STORE, 139773774102528, 139773774106623, +ERASE, 47859021058048, 47859021058048, +STORE, 94094901108736, 94094901243903, +STORE, 140737488347136, 140737488351231, +STORE, 140736567963648, 140737488351231, +ERASE, 140736567963648, 140736567963648, +STORE, 140736567963648, 140736567967743, +STORE, 94924425748480, 94924426461183, +ERASE, 94924425748480, 94924425748480, +STORE, 94924425748480, 94924425797631, +STORE, 94924425797632, 94924426461183, +ERASE, 94924425797632, 94924425797632, +STORE, 94924425797632, 94924426342399, +STORE, 94924426342400, 94924426440703, +STORE, 94924426440704, 94924426461183, +STORE, 140042126319616, 140042126491647, +ERASE, 140042126319616, 140042126319616, +STORE, 140042126319616, 140042126323711, +STORE, 140042126323712, 140042126491647, +ERASE, 140042126323712, 140042126323712, +STORE, 140042126323712, 140042126446591, +STORE, 140042126446592, 140042126479359, +STORE, 140042126479360, 140042126487551, +STORE, 140042126487552, 140042126491647, +STORE, 140736568672256, 140736568676351, +STORE, 140736568659968, 140736568672255, +STORE, 47590668677120, 47590668685311, +STORE, 47590668685312, 47590668693503, +STORE, 47590668693504, 47590668799999, +STORE, 47590668709888, 47590668799999, +STORE, 47590668693504, 47590668709887, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668783615, +STORE, 47590668783616, 47590668799999, +STORE, 47590668763136, 47590668783615, +STORE, 47590668709888, 47590668763135, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668763135, +STORE, 47590668779520, 47590668783615, +STORE, 47590668763136, 47590668779519, +ERASE, 47590668763136, 47590668763136, +STORE, 47590668763136, 47590668779519, +STORE, 47590668791808, 47590668799999, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668791808, 47590668791808, +STORE, 47590668791808, 47590668799999, +STORE, 47590668800000, 47590670639103, +STORE, 47590668939264, 47590670639103, +STORE, 47590668800000, 47590668939263, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670598143, +STORE, 47590670598144, 47590670639103, +STORE, 47590670282752, 47590670598143, +STORE, 47590668939264, 47590670282751, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670282751, +STORE, 47590670594048, 47590670598143, +STORE, 47590670282752, 47590670594047, +ERASE, 47590670282752, 47590670282752, +STORE, 47590670282752, 47590670594047, +STORE, 47590670622720, 47590670639103, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670622720, 47590670622720, +STORE, 47590670622720, 47590670639103, +STORE, 47590670622720, 47590670651391, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670614527, +STORE, 47590670614528, 47590670622719, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668787711, +STORE, 47590668787712, 47590668791807, +ERASE, 94924426440704, 94924426440704, +STORE, 94924426440704, 94924426457087, +STORE, 94924426457088, 94924426461183, +ERASE, 140042126479360, 140042126479360, +STORE, 140042126479360, 140042126483455, +STORE, 140042126483456, 140042126487551, +ERASE, 47590668677120, 47590668677120, +STORE, 140737488347136, 140737488351231, +STORE, 140733281439744, 140737488351231, +ERASE, 140733281439744, 140733281439744, +STORE, 140733281439744, 140733281443839, +STORE, 94490667069440, 94490667782143, +ERASE, 94490667069440, 94490667069440, +STORE, 94490667069440, 94490667118591, +STORE, 94490667118592, 94490667782143, +ERASE, 94490667118592, 94490667118592, +STORE, 94490667118592, 94490667663359, +STORE, 94490667663360, 94490667761663, +STORE, 94490667761664, 94490667782143, +STORE, 139878215118848, 139878215290879, +ERASE, 139878215118848, 139878215118848, +STORE, 139878215118848, 139878215122943, +STORE, 139878215122944, 139878215290879, +ERASE, 139878215122944, 139878215122944, +STORE, 139878215122944, 139878215245823, +STORE, 139878215245824, 139878215278591, +STORE, 139878215278592, 139878215286783, +STORE, 139878215286784, 139878215290879, +STORE, 140733281464320, 140733281468415, +STORE, 140733281452032, 140733281464319, +STORE, 47754579877888, 47754579886079, +STORE, 47754579886080, 47754579894271, +STORE, 47754579894272, 47754580000767, +STORE, 47754579910656, 47754580000767, +STORE, 47754579894272, 47754579910655, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579984383, +STORE, 47754579984384, 47754580000767, +STORE, 47754579963904, 47754579984383, +STORE, 47754579910656, 47754579963903, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579963903, +STORE, 47754579980288, 47754579984383, +STORE, 47754579963904, 47754579980287, +ERASE, 47754579963904, 47754579963904, +STORE, 47754579963904, 47754579980287, +STORE, 47754579992576, 47754580000767, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579992576, 47754579992576, +STORE, 47754579992576, 47754580000767, +STORE, 47754580000768, 47754581839871, +STORE, 47754580140032, 47754581839871, +STORE, 47754580000768, 47754580140031, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581798911, +STORE, 47754581798912, 47754581839871, +STORE, 47754581483520, 47754581798911, +STORE, 47754580140032, 47754581483519, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581483519, +STORE, 47754581794816, 47754581798911, +STORE, 47754581483520, 47754581794815, +ERASE, 47754581483520, 47754581483520, +STORE, 47754581483520, 47754581794815, +STORE, 47754581823488, 47754581839871, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581823488, 47754581823488, +STORE, 47754581823488, 47754581839871, +STORE, 47754581823488, 47754581852159, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581815295, +STORE, 47754581815296, 47754581823487, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579988479, +STORE, 47754579988480, 47754579992575, +ERASE, 94490667761664, 94490667761664, +STORE, 94490667761664, 94490667778047, +STORE, 94490667778048, 94490667782143, +ERASE, 139878215278592, 139878215278592, +STORE, 139878215278592, 139878215282687, +STORE, 139878215282688, 139878215286783, +ERASE, 47754579877888, 47754579877888, +STORE, 94490669649920, 94490669785087, +STORE, 140737488347136, 140737488351231, +STORE, 140735382188032, 140737488351231, +ERASE, 140735382188032, 140735382188032, +STORE, 140735382188032, 140735382192127, +STORE, 94150181302272, 94150182014975, +ERASE, 94150181302272, 94150181302272, +STORE, 94150181302272, 94150181351423, +STORE, 94150181351424, 94150182014975, +ERASE, 94150181351424, 94150181351424, +STORE, 94150181351424, 94150181896191, +STORE, 94150181896192, 94150181994495, +STORE, 94150181994496, 94150182014975, +STORE, 139679752458240, 139679752630271, +ERASE, 139679752458240, 139679752458240, +STORE, 139679752458240, 139679752462335, +STORE, 139679752462336, 139679752630271, +ERASE, 139679752462336, 139679752462336, +STORE, 139679752462336, 139679752585215, +STORE, 139679752585216, 139679752617983, +STORE, 139679752617984, 139679752626175, +STORE, 139679752626176, 139679752630271, +STORE, 140735382536192, 140735382540287, +STORE, 140735382523904, 140735382536191, +STORE, 47953042538496, 47953042546687, +STORE, 47953042546688, 47953042554879, +STORE, 47953042554880, 47953042661375, +STORE, 47953042571264, 47953042661375, +STORE, 47953042554880, 47953042571263, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042644991, +STORE, 47953042644992, 47953042661375, +STORE, 47953042624512, 47953042644991, +STORE, 47953042571264, 47953042624511, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042624511, +STORE, 47953042640896, 47953042644991, +STORE, 47953042624512, 47953042640895, +ERASE, 47953042624512, 47953042624512, +STORE, 47953042624512, 47953042640895, +STORE, 47953042653184, 47953042661375, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042653184, 47953042653184, +STORE, 47953042653184, 47953042661375, +STORE, 47953042661376, 47953044500479, +STORE, 47953042800640, 47953044500479, +STORE, 47953042661376, 47953042800639, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044459519, +STORE, 47953044459520, 47953044500479, +STORE, 47953044144128, 47953044459519, +STORE, 47953042800640, 47953044144127, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044144127, +STORE, 47953044455424, 47953044459519, +STORE, 47953044144128, 47953044455423, +ERASE, 47953044144128, 47953044144128, +STORE, 47953044144128, 47953044455423, +STORE, 47953044484096, 47953044500479, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044484096, 47953044484096, +STORE, 47953044484096, 47953044500479, +STORE, 47953044484096, 47953044512767, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044475903, +STORE, 47953044475904, 47953044484095, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042649087, +STORE, 47953042649088, 47953042653183, +ERASE, 94150181994496, 94150181994496, +STORE, 94150181994496, 94150182010879, +STORE, 94150182010880, 94150182014975, +ERASE, 139679752617984, 139679752617984, +STORE, 139679752617984, 139679752622079, +STORE, 139679752622080, 139679752626175, +ERASE, 47953042538496, 47953042538496, +STORE, 140737488347136, 140737488351231, +STORE, 140737044123648, 140737488351231, +ERASE, 140737044123648, 140737044123648, +STORE, 140737044123648, 140737044127743, +STORE, 94425324294144, 94425325006847, +ERASE, 94425324294144, 94425324294144, +STORE, 94425324294144, 94425324343295, +STORE, 94425324343296, 94425325006847, +ERASE, 94425324343296, 94425324343296, +STORE, 94425324343296, 94425324888063, +STORE, 94425324888064, 94425324986367, +STORE, 94425324986368, 94425325006847, +STORE, 140382015016960, 140382015188991, +ERASE, 140382015016960, 140382015016960, +STORE, 140382015016960, 140382015021055, +STORE, 140382015021056, 140382015188991, +ERASE, 140382015021056, 140382015021056, +STORE, 140382015021056, 140382015143935, +STORE, 140382015143936, 140382015176703, +STORE, 140382015176704, 140382015184895, +STORE, 140382015184896, 140382015188991, +STORE, 140737045585920, 140737045590015, +STORE, 140737045573632, 140737045585919, +STORE, 47250779979776, 47250779987967, +STORE, 47250779987968, 47250779996159, +STORE, 47250779996160, 47250780102655, +STORE, 47250780012544, 47250780102655, +STORE, 47250779996160, 47250780012543, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780086271, +STORE, 47250780086272, 47250780102655, +STORE, 47250780065792, 47250780086271, +STORE, 47250780012544, 47250780065791, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780065791, +STORE, 47250780082176, 47250780086271, +STORE, 47250780065792, 47250780082175, +ERASE, 47250780065792, 47250780065792, +STORE, 47250780065792, 47250780082175, +STORE, 47250780094464, 47250780102655, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780094464, 47250780094464, +STORE, 47250780094464, 47250780102655, +STORE, 47250780102656, 47250781941759, +STORE, 47250780241920, 47250781941759, +STORE, 47250780102656, 47250780241919, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781900799, +STORE, 47250781900800, 47250781941759, +STORE, 47250781585408, 47250781900799, +STORE, 47250780241920, 47250781585407, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781585407, +STORE, 47250781896704, 47250781900799, +STORE, 47250781585408, 47250781896703, +ERASE, 47250781585408, 47250781585408, +STORE, 47250781585408, 47250781896703, +STORE, 47250781925376, 47250781941759, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781925376, 47250781925376, +STORE, 47250781925376, 47250781941759, +STORE, 47250781925376, 47250781954047, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781917183, +STORE, 47250781917184, 47250781925375, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780090367, +STORE, 47250780090368, 47250780094463, +ERASE, 94425324986368, 94425324986368, +STORE, 94425324986368, 94425325002751, +STORE, 94425325002752, 94425325006847, +ERASE, 140382015176704, 140382015176704, +STORE, 140382015176704, 140382015180799, +STORE, 140382015180800, 140382015184895, +ERASE, 47250779979776, 47250779979776, +STORE, 94425351438336, 94425351573503, +STORE, 140737488347136, 140737488351231, +STORE, 140736801144832, 140737488351231, +ERASE, 140736801144832, 140736801144832, +STORE, 140736801144832, 140736801148927, +STORE, 94629429358592, 94629430071295, +ERASE, 94629429358592, 94629429358592, +STORE, 94629429358592, 94629429407743, +STORE, 94629429407744, 94629430071295, +ERASE, 94629429407744, 94629429407744, +STORE, 94629429407744, 94629429952511, +STORE, 94629429952512, 94629430050815, +STORE, 94629430050816, 94629430071295, +STORE, 139801685483520, 139801685655551, +ERASE, 139801685483520, 139801685483520, +STORE, 139801685483520, 139801685487615, +STORE, 139801685487616, 139801685655551, +ERASE, 139801685487616, 139801685487616, +STORE, 139801685487616, 139801685610495, +STORE, 139801685610496, 139801685643263, +STORE, 139801685643264, 139801685651455, +STORE, 139801685651456, 139801685655551, +STORE, 140736801198080, 140736801202175, +STORE, 140736801185792, 140736801198079, +STORE, 47831109513216, 47831109521407, +STORE, 47831109521408, 47831109529599, +STORE, 47831109529600, 47831109636095, +STORE, 47831109545984, 47831109636095, +STORE, 47831109529600, 47831109545983, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109619711, +STORE, 47831109619712, 47831109636095, +STORE, 47831109599232, 47831109619711, +STORE, 47831109545984, 47831109599231, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109599231, +STORE, 47831109615616, 47831109619711, +STORE, 47831109599232, 47831109615615, +ERASE, 47831109599232, 47831109599232, +STORE, 47831109599232, 47831109615615, +STORE, 47831109627904, 47831109636095, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109627904, 47831109627904, +STORE, 47831109627904, 47831109636095, +STORE, 47831109636096, 47831111475199, +STORE, 47831109775360, 47831111475199, +STORE, 47831109636096, 47831109775359, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111434239, +STORE, 47831111434240, 47831111475199, +STORE, 47831111118848, 47831111434239, +STORE, 47831109775360, 47831111118847, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111118847, +STORE, 47831111430144, 47831111434239, +STORE, 47831111118848, 47831111430143, +ERASE, 47831111118848, 47831111118848, +STORE, 47831111118848, 47831111430143, +STORE, 47831111458816, 47831111475199, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111458816, 47831111458816, +STORE, 47831111458816, 47831111475199, +STORE, 47831111458816, 47831111487487, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111450623, +STORE, 47831111450624, 47831111458815, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109623807, +STORE, 47831109623808, 47831109627903, +ERASE, 94629430050816, 94629430050816, +STORE, 94629430050816, 94629430067199, +STORE, 94629430067200, 94629430071295, +ERASE, 139801685643264, 139801685643264, +STORE, 139801685643264, 139801685647359, +STORE, 139801685647360, 139801685651455, +ERASE, 47831109513216, 47831109513216, +STORE, 140737488347136, 140737488351231, +STORE, 140729419612160, 140737488351231, +ERASE, 140729419612160, 140729419612160, +STORE, 140729419612160, 140729419616255, +STORE, 94443354148864, 94443354861567, +ERASE, 94443354148864, 94443354148864, +STORE, 94443354148864, 94443354198015, +STORE, 94443354198016, 94443354861567, +ERASE, 94443354198016, 94443354198016, +STORE, 94443354198016, 94443354742783, +STORE, 94443354742784, 94443354841087, +STORE, 94443354841088, 94443354861567, +STORE, 139741700038656, 139741700210687, +ERASE, 139741700038656, 139741700038656, +STORE, 139741700038656, 139741700042751, +STORE, 139741700042752, 139741700210687, +ERASE, 139741700042752, 139741700042752, +STORE, 139741700042752, 139741700165631, +STORE, 139741700165632, 139741700198399, +STORE, 139741700198400, 139741700206591, +STORE, 139741700206592, 139741700210687, +STORE, 140729420574720, 140729420578815, +STORE, 140729420562432, 140729420574719, +STORE, 47891094958080, 47891094966271, +STORE, 47891094966272, 47891094974463, +STORE, 47891094974464, 47891095080959, +STORE, 47891094990848, 47891095080959, +STORE, 47891094974464, 47891094990847, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095064575, +STORE, 47891095064576, 47891095080959, +STORE, 47891095044096, 47891095064575, +STORE, 47891094990848, 47891095044095, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095044095, +STORE, 47891095060480, 47891095064575, +STORE, 47891095044096, 47891095060479, +ERASE, 47891095044096, 47891095044096, +STORE, 47891095044096, 47891095060479, +STORE, 47891095072768, 47891095080959, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095072768, 47891095072768, +STORE, 47891095072768, 47891095080959, +STORE, 47891095080960, 47891096920063, +STORE, 47891095220224, 47891096920063, +STORE, 47891095080960, 47891095220223, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096879103, +STORE, 47891096879104, 47891096920063, +STORE, 47891096563712, 47891096879103, +STORE, 47891095220224, 47891096563711, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096563711, +STORE, 47891096875008, 47891096879103, +STORE, 47891096563712, 47891096875007, +ERASE, 47891096563712, 47891096563712, +STORE, 47891096563712, 47891096875007, +STORE, 47891096903680, 47891096920063, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096903680, 47891096903680, +STORE, 47891096903680, 47891096920063, +STORE, 47891096903680, 47891096932351, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096895487, +STORE, 47891096895488, 47891096903679, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095068671, +STORE, 47891095068672, 47891095072767, +ERASE, 94443354841088, 94443354841088, +STORE, 94443354841088, 94443354857471, +STORE, 94443354857472, 94443354861567, +ERASE, 139741700198400, 139741700198400, +STORE, 139741700198400, 139741700202495, +STORE, 139741700202496, 139741700206591, +ERASE, 47891094958080, 47891094958080, +STORE, 94443360825344, 94443360960511, +STORE, 140737488347136, 140737488351231, +STORE, 140722961661952, 140737488351231, +ERASE, 140722961661952, 140722961661952, +STORE, 140722961661952, 140722961666047, +STORE, 94878388944896, 94878389657599, +ERASE, 94878388944896, 94878388944896, +STORE, 94878388944896, 94878388994047, +STORE, 94878388994048, 94878389657599, +ERASE, 94878388994048, 94878388994048, +STORE, 94878388994048, 94878389538815, +STORE, 94878389538816, 94878389637119, +STORE, 94878389637120, 94878389657599, +STORE, 140210690056192, 140210690228223, +ERASE, 140210690056192, 140210690056192, +STORE, 140210690056192, 140210690060287, +STORE, 140210690060288, 140210690228223, +ERASE, 140210690060288, 140210690060288, +STORE, 140210690060288, 140210690183167, +STORE, 140210690183168, 140210690215935, +STORE, 140210690215936, 140210690224127, +STORE, 140210690224128, 140210690228223, +STORE, 140722963148800, 140722963152895, +STORE, 140722963136512, 140722963148799, +STORE, 47422104940544, 47422104948735, +STORE, 47422104948736, 47422104956927, +STORE, 47422104956928, 47422105063423, +STORE, 47422104973312, 47422105063423, +STORE, 47422104956928, 47422104973311, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105047039, +STORE, 47422105047040, 47422105063423, +STORE, 47422105026560, 47422105047039, +STORE, 47422104973312, 47422105026559, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105026559, +STORE, 47422105042944, 47422105047039, +STORE, 47422105026560, 47422105042943, +ERASE, 47422105026560, 47422105026560, +STORE, 47422105026560, 47422105042943, +STORE, 47422105055232, 47422105063423, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105055232, 47422105055232, +STORE, 47422105055232, 47422105063423, +STORE, 47422105063424, 47422106902527, +STORE, 47422105202688, 47422106902527, +STORE, 47422105063424, 47422105202687, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106861567, +STORE, 47422106861568, 47422106902527, +STORE, 47422106546176, 47422106861567, +STORE, 47422105202688, 47422106546175, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106546175, +STORE, 47422106857472, 47422106861567, +STORE, 47422106546176, 47422106857471, +ERASE, 47422106546176, 47422106546176, +STORE, 47422106546176, 47422106857471, +STORE, 47422106886144, 47422106902527, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106886144, 47422106886144, +STORE, 47422106886144, 47422106902527, +STORE, 47422106886144, 47422106914815, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106877951, +STORE, 47422106877952, 47422106886143, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105051135, +STORE, 47422105051136, 47422105055231, +ERASE, 94878389637120, 94878389637120, +STORE, 94878389637120, 94878389653503, +STORE, 94878389653504, 94878389657599, +ERASE, 140210690215936, 140210690215936, +STORE, 140210690215936, 140210690220031, +STORE, 140210690220032, 140210690224127, +ERASE, 47422104940544, 47422104940544, +STORE, 140737488347136, 140737488351231, +STORE, 140727690309632, 140737488351231, +ERASE, 140727690309632, 140727690309632, +STORE, 140727690309632, 140727690313727, +STORE, 94121892208640, 94121892921343, +ERASE, 94121892208640, 94121892208640, +STORE, 94121892208640, 94121892257791, +STORE, 94121892257792, 94121892921343, +ERASE, 94121892257792, 94121892257792, +STORE, 94121892257792, 94121892802559, +STORE, 94121892802560, 94121892900863, +STORE, 94121892900864, 94121892921343, +STORE, 140662438326272, 140662438498303, +ERASE, 140662438326272, 140662438326272, +STORE, 140662438326272, 140662438330367, +STORE, 140662438330368, 140662438498303, +ERASE, 140662438330368, 140662438330368, +STORE, 140662438330368, 140662438453247, +STORE, 140662438453248, 140662438486015, +STORE, 140662438486016, 140662438494207, +STORE, 140662438494208, 140662438498303, +STORE, 140727690379264, 140727690383359, +STORE, 140727690366976, 140727690379263, +STORE, 46970356670464, 46970356678655, +STORE, 46970356678656, 46970356686847, +STORE, 46970356686848, 46970356793343, +STORE, 46970356703232, 46970356793343, +STORE, 46970356686848, 46970356703231, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356776959, +STORE, 46970356776960, 46970356793343, +STORE, 46970356756480, 46970356776959, +STORE, 46970356703232, 46970356756479, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356756479, +STORE, 46970356772864, 46970356776959, +STORE, 46970356756480, 46970356772863, +ERASE, 46970356756480, 46970356756480, +STORE, 46970356756480, 46970356772863, +STORE, 46970356785152, 46970356793343, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356785152, 46970356785152, +STORE, 46970356785152, 46970356793343, +STORE, 46970356793344, 46970358632447, +STORE, 46970356932608, 46970358632447, +STORE, 46970356793344, 46970356932607, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358591487, +STORE, 46970358591488, 46970358632447, +STORE, 46970358276096, 46970358591487, +STORE, 46970356932608, 46970358276095, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358276095, +STORE, 46970358587392, 46970358591487, +STORE, 46970358276096, 46970358587391, +ERASE, 46970358276096, 46970358276096, +STORE, 46970358276096, 46970358587391, +STORE, 46970358616064, 46970358632447, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358616064, 46970358616064, +STORE, 46970358616064, 46970358632447, +STORE, 46970358616064, 46970358644735, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358607871, +STORE, 46970358607872, 46970358616063, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356781055, +STORE, 46970356781056, 46970356785151, +ERASE, 94121892900864, 94121892900864, +STORE, 94121892900864, 94121892917247, +STORE, 94121892917248, 94121892921343, +ERASE, 140662438486016, 140662438486016, +STORE, 140662438486016, 140662438490111, +STORE, 140662438490112, 140662438494207, +ERASE, 46970356670464, 46970356670464, +STORE, 94121898610688, 94121898745855, +STORE, 140737488347136, 140737488351231, +STORE, 140737189351424, 140737488351231, +ERASE, 140737189351424, 140737189351424, +STORE, 140737189351424, 140737189355519, +STORE, 93847948832768, 93847949545471, +ERASE, 93847948832768, 93847948832768, +STORE, 93847948832768, 93847948881919, +STORE, 93847948881920, 93847949545471, +ERASE, 93847948881920, 93847948881920, +STORE, 93847948881920, 93847949426687, +STORE, 93847949426688, 93847949524991, +STORE, 93847949524992, 93847949545471, +STORE, 139698989985792, 139698990157823, +ERASE, 139698989985792, 139698989985792, +STORE, 139698989985792, 139698989989887, +STORE, 139698989989888, 139698990157823, +ERASE, 139698989989888, 139698989989888, +STORE, 139698989989888, 139698990112767, +STORE, 139698990112768, 139698990145535, +STORE, 139698990145536, 139698990153727, +STORE, 139698990153728, 139698990157823, +STORE, 140737189744640, 140737189748735, +STORE, 140737189732352, 140737189744639, +STORE, 47933805010944, 47933805019135, +STORE, 47933805019136, 47933805027327, +STORE, 47933805027328, 47933805133823, +STORE, 47933805043712, 47933805133823, +STORE, 47933805027328, 47933805043711, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805117439, +STORE, 47933805117440, 47933805133823, +STORE, 47933805096960, 47933805117439, +STORE, 47933805043712, 47933805096959, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805096959, +STORE, 47933805113344, 47933805117439, +STORE, 47933805096960, 47933805113343, +ERASE, 47933805096960, 47933805096960, +STORE, 47933805096960, 47933805113343, +STORE, 47933805125632, 47933805133823, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805125632, 47933805125632, +STORE, 47933805125632, 47933805133823, +STORE, 47933805133824, 47933806972927, +STORE, 47933805273088, 47933806972927, +STORE, 47933805133824, 47933805273087, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806931967, +STORE, 47933806931968, 47933806972927, +STORE, 47933806616576, 47933806931967, +STORE, 47933805273088, 47933806616575, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806616575, +STORE, 47933806927872, 47933806931967, +STORE, 47933806616576, 47933806927871, +ERASE, 47933806616576, 47933806616576, +STORE, 47933806616576, 47933806927871, +STORE, 47933806956544, 47933806972927, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806956544, 47933806956544, +STORE, 47933806956544, 47933806972927, +STORE, 47933806956544, 47933806985215, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806948351, +STORE, 47933806948352, 47933806956543, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805121535, +STORE, 47933805121536, 47933805125631, +ERASE, 93847949524992, 93847949524992, +STORE, 93847949524992, 93847949541375, +STORE, 93847949541376, 93847949545471, +ERASE, 139698990145536, 139698990145536, +STORE, 139698990145536, 139698990149631, +STORE, 139698990149632, 139698990153727, +ERASE, 47933805010944, 47933805010944, +STORE, 140737488347136, 140737488351231, +STORE, 140725553991680, 140737488351231, +ERASE, 140725553991680, 140725553991680, +STORE, 140725553991680, 140725553995775, +STORE, 93980056248320, 93980056961023, +ERASE, 93980056248320, 93980056248320, +STORE, 93980056248320, 93980056297471, +STORE, 93980056297472, 93980056961023, +ERASE, 93980056297472, 93980056297472, +STORE, 93980056297472, 93980056842239, +STORE, 93980056842240, 93980056940543, +STORE, 93980056940544, 93980056961023, +STORE, 140146588971008, 140146589143039, +ERASE, 140146588971008, 140146588971008, +STORE, 140146588971008, 140146588975103, +STORE, 140146588975104, 140146589143039, +ERASE, 140146588975104, 140146588975104, +STORE, 140146588975104, 140146589097983, +STORE, 140146589097984, 140146589130751, +STORE, 140146589130752, 140146589138943, +STORE, 140146589138944, 140146589143039, +STORE, 140725554860032, 140725554864127, +STORE, 140725554847744, 140725554860031, +STORE, 47486206025728, 47486206033919, +STORE, 47486206033920, 47486206042111, +STORE, 47486206042112, 47486206148607, +STORE, 47486206058496, 47486206148607, +STORE, 47486206042112, 47486206058495, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206132223, +STORE, 47486206132224, 47486206148607, +STORE, 47486206111744, 47486206132223, +STORE, 47486206058496, 47486206111743, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206111743, +STORE, 47486206128128, 47486206132223, +STORE, 47486206111744, 47486206128127, +ERASE, 47486206111744, 47486206111744, +STORE, 47486206111744, 47486206128127, +STORE, 47486206140416, 47486206148607, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206140416, 47486206140416, +STORE, 47486206140416, 47486206148607, +STORE, 47486206148608, 47486207987711, +STORE, 47486206287872, 47486207987711, +STORE, 47486206148608, 47486206287871, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207946751, +STORE, 47486207946752, 47486207987711, +STORE, 47486207631360, 47486207946751, +STORE, 47486206287872, 47486207631359, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207631359, +STORE, 47486207942656, 47486207946751, +STORE, 47486207631360, 47486207942655, +ERASE, 47486207631360, 47486207631360, +STORE, 47486207631360, 47486207942655, +STORE, 47486207971328, 47486207987711, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207971328, 47486207971328, +STORE, 47486207971328, 47486207987711, +STORE, 47486207971328, 47486207999999, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207963135, +STORE, 47486207963136, 47486207971327, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206136319, +STORE, 47486206136320, 47486206140415, +ERASE, 93980056940544, 93980056940544, +STORE, 93980056940544, 93980056956927, +STORE, 93980056956928, 93980056961023, +ERASE, 140146589130752, 140146589130752, +STORE, 140146589130752, 140146589134847, +STORE, 140146589134848, 140146589138943, +ERASE, 47486206025728, 47486206025728, +STORE, 93980070006784, 93980070141951, +STORE, 140737488347136, 140737488351231, +STORE, 140727334776832, 140737488351231, +ERASE, 140727334776832, 140727334776832, +STORE, 140727334776832, 140727334780927, +STORE, 94049747247104, 94049747959807, +ERASE, 94049747247104, 94049747247104, +STORE, 94049747247104, 94049747296255, +STORE, 94049747296256, 94049747959807, +ERASE, 94049747296256, 94049747296256, +STORE, 94049747296256, 94049747841023, +STORE, 94049747841024, 94049747939327, +STORE, 94049747939328, 94049747959807, +STORE, 140227307216896, 140227307388927, +ERASE, 140227307216896, 140227307216896, +STORE, 140227307216896, 140227307220991, +STORE, 140227307220992, 140227307388927, +ERASE, 140227307220992, 140227307220992, +STORE, 140227307220992, 140227307343871, +STORE, 140227307343872, 140227307376639, +STORE, 140227307376640, 140227307384831, +STORE, 140227307384832, 140227307388927, +STORE, 140727335337984, 140727335342079, +STORE, 140727335325696, 140727335337983, +STORE, 47405487779840, 47405487788031, +STORE, 47405487788032, 47405487796223, +STORE, 47405487796224, 47405487902719, +STORE, 47405487812608, 47405487902719, +STORE, 47405487796224, 47405487812607, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487886335, +STORE, 47405487886336, 47405487902719, +STORE, 47405487865856, 47405487886335, +STORE, 47405487812608, 47405487865855, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487865855, +STORE, 47405487882240, 47405487886335, +STORE, 47405487865856, 47405487882239, +ERASE, 47405487865856, 47405487865856, +STORE, 47405487865856, 47405487882239, +STORE, 47405487894528, 47405487902719, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487894528, 47405487894528, +STORE, 47405487894528, 47405487902719, +STORE, 47405487902720, 47405489741823, +STORE, 47405488041984, 47405489741823, +STORE, 47405487902720, 47405488041983, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489700863, +STORE, 47405489700864, 47405489741823, +STORE, 47405489385472, 47405489700863, +STORE, 47405488041984, 47405489385471, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489385471, +STORE, 47405489696768, 47405489700863, +STORE, 47405489385472, 47405489696767, +ERASE, 47405489385472, 47405489385472, +STORE, 47405489385472, 47405489696767, +STORE, 47405489725440, 47405489741823, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489725440, 47405489725440, +STORE, 47405489725440, 47405489741823, +STORE, 47405489725440, 47405489754111, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489717247, +STORE, 47405489717248, 47405489725439, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487890431, +STORE, 47405487890432, 47405487894527, +ERASE, 94049747939328, 94049747939328, +STORE, 94049747939328, 94049747955711, +STORE, 94049747955712, 94049747959807, +ERASE, 140227307376640, 140227307376640, +STORE, 140227307376640, 140227307380735, +STORE, 140227307380736, 140227307384831, +ERASE, 47405487779840, 47405487779840, +STORE, 94049758810112, 94049758945279, +STORE, 140737488347136, 140737488351231, +STORE, 140727079718912, 140737488351231, +ERASE, 140727079718912, 140727079718912, +STORE, 140727079718912, 140727079723007, +STORE, 94250996527104, 94250997239807, +ERASE, 94250996527104, 94250996527104, +STORE, 94250996527104, 94250996576255, +STORE, 94250996576256, 94250997239807, +ERASE, 94250996576256, 94250996576256, +STORE, 94250996576256, 94250997121023, +STORE, 94250997121024, 94250997219327, +STORE, 94250997219328, 94250997239807, +STORE, 140060022587392, 140060022759423, +ERASE, 140060022587392, 140060022587392, +STORE, 140060022587392, 140060022591487, +STORE, 140060022591488, 140060022759423, +ERASE, 140060022591488, 140060022591488, +STORE, 140060022591488, 140060022714367, +STORE, 140060022714368, 140060022747135, +STORE, 140060022747136, 140060022755327, +STORE, 140060022755328, 140060022759423, +STORE, 140727079788544, 140727079792639, +STORE, 140727079776256, 140727079788543, +STORE, 47572772409344, 47572772417535, +STORE, 47572772417536, 47572772425727, +STORE, 47572772425728, 47572772532223, +STORE, 47572772442112, 47572772532223, +STORE, 47572772425728, 47572772442111, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772515839, +STORE, 47572772515840, 47572772532223, +STORE, 47572772495360, 47572772515839, +STORE, 47572772442112, 47572772495359, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772495359, +STORE, 47572772511744, 47572772515839, +STORE, 47572772495360, 47572772511743, +ERASE, 47572772495360, 47572772495360, +STORE, 47572772495360, 47572772511743, +STORE, 47572772524032, 47572772532223, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772524032, 47572772524032, +STORE, 47572772524032, 47572772532223, +STORE, 47572772532224, 47572774371327, +STORE, 47572772671488, 47572774371327, +STORE, 47572772532224, 47572772671487, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774330367, +STORE, 47572774330368, 47572774371327, +STORE, 47572774014976, 47572774330367, +STORE, 47572772671488, 47572774014975, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774014975, +STORE, 47572774326272, 47572774330367, +STORE, 47572774014976, 47572774326271, +ERASE, 47572774014976, 47572774014976, +STORE, 47572774014976, 47572774326271, +STORE, 47572774354944, 47572774371327, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774354944, 47572774354944, +STORE, 47572774354944, 47572774371327, +STORE, 47572774354944, 47572774383615, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774346751, +STORE, 47572774346752, 47572774354943, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772519935, +STORE, 47572772519936, 47572772524031, +ERASE, 94250997219328, 94250997219328, +STORE, 94250997219328, 94250997235711, +STORE, 94250997235712, 94250997239807, +ERASE, 140060022747136, 140060022747136, +STORE, 140060022747136, 140060022751231, +STORE, 140060022751232, 140060022755327, +ERASE, 47572772409344, 47572772409344, +STORE, 94251018305536, 94251018440703, +STORE, 140737488347136, 140737488351231, +STORE, 140730012389376, 140737488351231, +ERASE, 140730012389376, 140730012389376, +STORE, 140730012389376, 140730012393471, +STORE, 94382607675392, 94382607695871, +ERASE, 94382607675392, 94382607675392, +STORE, 94382607675392, 94382607679487, +STORE, 94382607679488, 94382607695871, +ERASE, 94382607679488, 94382607679488, +STORE, 94382607679488, 94382607683583, +STORE, 94382607683584, 94382607687679, +STORE, 94382607687680, 94382607695871, +STORE, 140252451454976, 140252451627007, +ERASE, 140252451454976, 140252451454976, +STORE, 140252451454976, 140252451459071, +STORE, 140252451459072, 140252451627007, +ERASE, 140252451459072, 140252451459072, +STORE, 140252451459072, 140252451581951, +STORE, 140252451581952, 140252451614719, +STORE, 140252451614720, 140252451622911, +STORE, 140252451622912, 140252451627007, +STORE, 140730013548544, 140730013552639, +STORE, 140730013536256, 140730013548543, +STORE, 47380343541760, 47380343549951, +STORE, 47380343549952, 47380343558143, +STORE, 47380343558144, 47380345397247, +STORE, 47380343697408, 47380345397247, +STORE, 47380343558144, 47380343697407, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345356287, +STORE, 47380345356288, 47380345397247, +STORE, 47380345040896, 47380345356287, +STORE, 47380343697408, 47380345040895, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345040895, +STORE, 47380345352192, 47380345356287, +STORE, 47380345040896, 47380345352191, +ERASE, 47380345040896, 47380345040896, +STORE, 47380345040896, 47380345352191, +STORE, 47380345380864, 47380345397247, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345380864, 47380345380864, +STORE, 47380345380864, 47380345397247, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345372671, +STORE, 47380345372672, 47380345380863, +ERASE, 94382607687680, 94382607687680, +STORE, 94382607687680, 94382607691775, +STORE, 94382607691776, 94382607695871, +ERASE, 140252451614720, 140252451614720, +STORE, 140252451614720, 140252451618815, +STORE, 140252451618816, 140252451622911, +ERASE, 47380343541760, 47380343541760, +STORE, 94382626803712, 94382626938879, +STORE, 140737488347136, 140737488351231, +STORE, 140730900271104, 140737488351231, +ERASE, 140730900271104, 140730900271104, +STORE, 140730900271104, 140730900275199, +STORE, 93855478120448, 93855478337535, +ERASE, 93855478120448, 93855478120448, +STORE, 93855478120448, 93855478198271, +STORE, 93855478198272, 93855478337535, +ERASE, 93855478198272, 93855478198272, +STORE, 93855478198272, 93855478243327, +STORE, 93855478243328, 93855478288383, +STORE, 93855478288384, 93855478337535, +STORE, 140092686573568, 140092686745599, +ERASE, 140092686573568, 140092686573568, +STORE, 140092686573568, 140092686577663, +STORE, 140092686577664, 140092686745599, +ERASE, 140092686577664, 140092686577664, +STORE, 140092686577664, 140092686700543, +STORE, 140092686700544, 140092686733311, +STORE, 140092686733312, 140092686741503, +STORE, 140092686741504, 140092686745599, +STORE, 140730900537344, 140730900541439, +STORE, 140730900525056, 140730900537343, +STORE, 47540108423168, 47540108431359, +STORE, 47540108431360, 47540108439551, +STORE, 47540108439552, 47540110278655, +STORE, 47540108578816, 47540110278655, +STORE, 47540108439552, 47540108578815, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540110237695, +STORE, 47540110237696, 47540110278655, +STORE, 47540109922304, 47540110237695, +STORE, 47540108578816, 47540109922303, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540109922303, +STORE, 47540110233600, 47540110237695, +STORE, 47540109922304, 47540110233599, +ERASE, 47540109922304, 47540109922304, +STORE, 47540109922304, 47540110233599, +STORE, 47540110262272, 47540110278655, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110262272, 47540110262272, +STORE, 47540110262272, 47540110278655, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110254079, +STORE, 47540110254080, 47540110262271, +ERASE, 93855478288384, 93855478288384, +STORE, 93855478288384, 93855478333439, +STORE, 93855478333440, 93855478337535, +ERASE, 140092686733312, 140092686733312, +STORE, 140092686733312, 140092686737407, +STORE, 140092686737408, 140092686741503, +ERASE, 47540108423168, 47540108423168, +STORE, 93855492222976, 93855492358143, +STORE, 93855492222976, 93855492493311, +STORE, 140737488347136, 140737488351231, +STORE, 140733498146816, 140737488351231, +ERASE, 140733498146816, 140733498146816, +STORE, 140733498146816, 140733498150911, +STORE, 94170739654656, 94170740367359, +ERASE, 94170739654656, 94170739654656, +STORE, 94170739654656, 94170739703807, +STORE, 94170739703808, 94170740367359, +ERASE, 94170739703808, 94170739703808, +STORE, 94170739703808, 94170740248575, +STORE, 94170740248576, 94170740346879, +STORE, 94170740346880, 94170740367359, +STORE, 140024788877312, 140024789049343, +ERASE, 140024788877312, 140024788877312, +STORE, 140024788877312, 140024788881407, +STORE, 140024788881408, 140024789049343, +ERASE, 140024788881408, 140024788881408, +STORE, 140024788881408, 140024789004287, +STORE, 140024789004288, 140024789037055, +STORE, 140024789037056, 140024789045247, +STORE, 140024789045248, 140024789049343, +STORE, 140733499023360, 140733499027455, +STORE, 140733499011072, 140733499023359, +STORE, 47608006119424, 47608006127615, +STORE, 47608006127616, 47608006135807, +STORE, 47608006135808, 47608006242303, +STORE, 47608006152192, 47608006242303, +STORE, 47608006135808, 47608006152191, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006225919, +STORE, 47608006225920, 47608006242303, +STORE, 47608006205440, 47608006225919, +STORE, 47608006152192, 47608006205439, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006205439, +STORE, 47608006221824, 47608006225919, +STORE, 47608006205440, 47608006221823, +ERASE, 47608006205440, 47608006205440, +STORE, 47608006205440, 47608006221823, +STORE, 47608006234112, 47608006242303, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006234112, 47608006234112, +STORE, 47608006234112, 47608006242303, +STORE, 47608006242304, 47608008081407, +STORE, 47608006381568, 47608008081407, +STORE, 47608006242304, 47608006381567, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608008040447, +STORE, 47608008040448, 47608008081407, +STORE, 47608007725056, 47608008040447, +STORE, 47608006381568, 47608007725055, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608007725055, +STORE, 47608008036352, 47608008040447, +STORE, 47608007725056, 47608008036351, +ERASE, 47608007725056, 47608007725056, +STORE, 47608007725056, 47608008036351, +STORE, 47608008065024, 47608008081407, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008065024, 47608008065024, +STORE, 47608008065024, 47608008081407, +STORE, 47608008065024, 47608008093695, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008056831, +STORE, 47608008056832, 47608008065023, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006230015, +STORE, 47608006230016, 47608006234111, +ERASE, 94170740346880, 94170740346880, +STORE, 94170740346880, 94170740363263, +STORE, 94170740363264, 94170740367359, +ERASE, 140024789037056, 140024789037056, +STORE, 140024789037056, 140024789041151, +STORE, 140024789041152, 140024789045247, +ERASE, 47608006119424, 47608006119424, +STORE, 140737488347136, 140737488351231, +STORE, 140730264326144, 140737488351231, +ERASE, 140730264326144, 140730264326144, +STORE, 140730264326144, 140730264330239, +STORE, 94653216407552, 94653217120255, +ERASE, 94653216407552, 94653216407552, +STORE, 94653216407552, 94653216456703, +STORE, 94653216456704, 94653217120255, +ERASE, 94653216456704, 94653216456704, +STORE, 94653216456704, 94653217001471, +STORE, 94653217001472, 94653217099775, +STORE, 94653217099776, 94653217120255, +STORE, 140103617011712, 140103617183743, +ERASE, 140103617011712, 140103617011712, +STORE, 140103617011712, 140103617015807, +STORE, 140103617015808, 140103617183743, +ERASE, 140103617015808, 140103617015808, +STORE, 140103617015808, 140103617138687, +STORE, 140103617138688, 140103617171455, +STORE, 140103617171456, 140103617179647, +STORE, 140103617179648, 140103617183743, +STORE, 140730265427968, 140730265432063, +STORE, 140730265415680, 140730265427967, +STORE, 47529177985024, 47529177993215, +STORE, 47529177993216, 47529178001407, +STORE, 47529178001408, 47529178107903, +STORE, 47529178017792, 47529178107903, +STORE, 47529178001408, 47529178017791, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178091519, +STORE, 47529178091520, 47529178107903, +STORE, 47529178071040, 47529178091519, +STORE, 47529178017792, 47529178071039, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178071039, +STORE, 47529178087424, 47529178091519, +STORE, 47529178071040, 47529178087423, +ERASE, 47529178071040, 47529178071040, +STORE, 47529178071040, 47529178087423, +STORE, 47529178099712, 47529178107903, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178099712, 47529178099712, +STORE, 47529178099712, 47529178107903, +STORE, 47529178107904, 47529179947007, +STORE, 47529178247168, 47529179947007, +STORE, 47529178107904, 47529178247167, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179906047, +STORE, 47529179906048, 47529179947007, +STORE, 47529179590656, 47529179906047, +STORE, 47529178247168, 47529179590655, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179590655, +STORE, 47529179901952, 47529179906047, +STORE, 47529179590656, 47529179901951, +ERASE, 47529179590656, 47529179590656, +STORE, 47529179590656, 47529179901951, +STORE, 47529179930624, 47529179947007, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179930624, 47529179930624, +STORE, 47529179930624, 47529179947007, +STORE, 47529179930624, 47529179959295, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179922431, +STORE, 47529179922432, 47529179930623, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178095615, +STORE, 47529178095616, 47529178099711, +ERASE, 94653217099776, 94653217099776, +STORE, 94653217099776, 94653217116159, +STORE, 94653217116160, 94653217120255, +ERASE, 140103617171456, 140103617171456, +STORE, 140103617171456, 140103617175551, +STORE, 140103617175552, 140103617179647, +ERASE, 47529177985024, 47529177985024, +STORE, 94653241135104, 94653241270271, +STORE, 140737488347136, 140737488351231, +STORE, 140736284549120, 140737488351231, +ERASE, 140736284549120, 140736284549120, +STORE, 140736284549120, 140736284553215, +STORE, 93963663822848, 93963664506879, +ERASE, 93963663822848, 93963663822848, +STORE, 93963663822848, 93963663884287, +STORE, 93963663884288, 93963664506879, +ERASE, 93963663884288, 93963663884288, +STORE, 93963663884288, 93963664240639, +STORE, 93963664240640, 93963664379903, +STORE, 93963664379904, 93963664506879, +STORE, 140450188439552, 140450188611583, +ERASE, 140450188439552, 140450188439552, +STORE, 140450188439552, 140450188443647, +STORE, 140450188443648, 140450188611583, +ERASE, 140450188443648, 140450188443648, +STORE, 140450188443648, 140450188566527, +STORE, 140450188566528, 140450188599295, +STORE, 140450188599296, 140450188607487, +STORE, 140450188607488, 140450188611583, +STORE, 140736284577792, 140736284581887, +STORE, 140736284565504, 140736284577791, +STORE, 47182606557184, 47182606565375, +STORE, 47182606565376, 47182606573567, +STORE, 47182606573568, 47182608412671, +STORE, 47182606712832, 47182608412671, +STORE, 47182606573568, 47182606712831, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608371711, +STORE, 47182608371712, 47182608412671, +STORE, 47182608056320, 47182608371711, +STORE, 47182606712832, 47182608056319, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608056319, +STORE, 47182608367616, 47182608371711, +STORE, 47182608056320, 47182608367615, +ERASE, 47182608056320, 47182608056320, +STORE, 47182608056320, 47182608367615, +STORE, 47182608396288, 47182608412671, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608396288, 47182608396288, +STORE, 47182608396288, 47182608412671, +STORE, 47182608412672, 47182608523263, +STORE, 47182608429056, 47182608523263, +STORE, 47182608412672, 47182608429055, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608515071, +STORE, 47182608515072, 47182608523263, +STORE, 47182608490496, 47182608515071, +STORE, 47182608429056, 47182608490495, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608490495, +STORE, 47182608510976, 47182608515071, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608490496, 47182608490496, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608523263, +STORE, 47182608523264, 47182608568319, +ERASE, 47182608523264, 47182608523264, +STORE, 47182608523264, 47182608531455, +STORE, 47182608531456, 47182608568319, +STORE, 47182608551936, 47182608568319, +STORE, 47182608531456, 47182608551935, +ERASE, 47182608531456, 47182608531456, +STORE, 47182608531456, 47182608551935, +STORE, 47182608560128, 47182608568319, +STORE, 47182608551936, 47182608560127, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608568319, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608560127, +STORE, 47182608560128, 47182608568319, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608568319, +STORE, 47182608568320, 47182608916479, +STORE, 47182608609280, 47182608916479, +STORE, 47182608568320, 47182608609279, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608891903, +STORE, 47182608891904, 47182608916479, +STORE, 47182608822272, 47182608891903, +STORE, 47182608609280, 47182608822271, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608822271, +STORE, 47182608887808, 47182608891903, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608822272, 47182608822272, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608916479, +STORE, 47182608916480, 47182611177471, +STORE, 47182609068032, 47182611177471, +STORE, 47182608916480, 47182609068031, +ERASE, 47182609068032, 47182609068032, +STORE, 47182609068032, 47182611161087, +STORE, 47182611161088, 47182611177471, +STORE, 47182611169280, 47182611177471, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611169280, 47182611169280, +STORE, 47182611169280, 47182611177471, +STORE, 47182611177472, 47182611312639, +ERASE, 47182611177472, 47182611177472, +STORE, 47182611177472, 47182611202047, +STORE, 47182611202048, 47182611312639, +STORE, 47182611263488, 47182611312639, +STORE, 47182611202048, 47182611263487, +ERASE, 47182611202048, 47182611202048, +STORE, 47182611202048, 47182611263487, +STORE, 47182611288064, 47182611312639, +STORE, 47182611263488, 47182611288063, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611312639, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611288063, +STORE, 47182611288064, 47182611312639, +STORE, 47182611296256, 47182611312639, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611296256, 47182611296256, +STORE, 47182611296256, 47182611312639, +STORE, 47182611296256, 47182611320831, +STORE, 47182611320832, 47182611484671, +ERASE, 47182611320832, 47182611320832, +STORE, 47182611320832, 47182611333119, +STORE, 47182611333120, 47182611484671, +STORE, 47182611431424, 47182611484671, +STORE, 47182611333120, 47182611431423, +ERASE, 47182611333120, 47182611333120, +STORE, 47182611333120, 47182611431423, +STORE, 47182611476480, 47182611484671, +STORE, 47182611431424, 47182611476479, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611484671, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611476479, +STORE, 47182611476480, 47182611484671, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611484671, +STORE, 47182611484672, 47182612082687, +STORE, 47182611603456, 47182612082687, +STORE, 47182611484672, 47182611603455, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182612029439, +STORE, 47182612029440, 47182612082687, +STORE, 47182611918848, 47182612029439, +STORE, 47182611603456, 47182611918847, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182611918847, +STORE, 47182612025344, 47182612029439, +STORE, 47182611918848, 47182612025343, +ERASE, 47182611918848, 47182611918848, +STORE, 47182611918848, 47182612025343, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612082687, +STORE, 47182612082688, 47182615134207, +STORE, 47182612627456, 47182615134207, +STORE, 47182612082688, 47182612627455, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614913023, +STORE, 47182614913024, 47182615134207, +STORE, 47182614323200, 47182614913023, +STORE, 47182612627456, 47182614323199, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614323199, +STORE, 47182614908928, 47182614913023, +STORE, 47182614323200, 47182614908927, +ERASE, 47182614323200, 47182614323200, +STORE, 47182614323200, 47182614908927, +STORE, 47182615117824, 47182615134207, +STORE, 47182614913024, 47182615117823, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615117823, +ERASE, 47182615117824, 47182615117824, +STORE, 47182615117824, 47182615134207, +STORE, 47182615134208, 47182615166975, +ERASE, 47182615134208, 47182615134208, +STORE, 47182615134208, 47182615142399, +STORE, 47182615142400, 47182615166975, +STORE, 47182615154688, 47182615166975, +STORE, 47182615142400, 47182615154687, +ERASE, 47182615142400, 47182615142400, +STORE, 47182615142400, 47182615154687, +STORE, 47182615158784, 47182615166975, +STORE, 47182615154688, 47182615158783, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615166975, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615158783, +STORE, 47182615158784, 47182615166975, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615166975, +STORE, 47182615166976, 47182615203839, +ERASE, 47182615166976, 47182615166976, +STORE, 47182615166976, 47182615175167, +STORE, 47182615175168, 47182615203839, +STORE, 47182615191552, 47182615203839, +STORE, 47182615175168, 47182615191551, +ERASE, 47182615175168, 47182615175168, +STORE, 47182615175168, 47182615191551, +STORE, 47182615195648, 47182615203839, +STORE, 47182615191552, 47182615195647, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615203839, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615195647, +STORE, 47182615195648, 47182615203839, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615203839, +STORE, 47182615203840, 47182615678975, +ERASE, 47182615203840, 47182615203840, +STORE, 47182615203840, 47182615212031, +STORE, 47182615212032, 47182615678975, +STORE, 47182615547904, 47182615678975, +STORE, 47182615212032, 47182615547903, +ERASE, 47182615212032, 47182615212032, +STORE, 47182615212032, 47182615547903, +STORE, 47182615670784, 47182615678975, +STORE, 47182615547904, 47182615670783, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615678975, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615670783, +STORE, 47182615670784, 47182615678975, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615678975, +STORE, 47182615678976, 47182615687167, +STORE, 47182615687168, 47182615707647, +ERASE, 47182615687168, 47182615687168, +STORE, 47182615687168, 47182615691263, +STORE, 47182615691264, 47182615707647, +STORE, 47182615695360, 47182615707647, +STORE, 47182615691264, 47182615695359, +ERASE, 47182615691264, 47182615691264, +STORE, 47182615691264, 47182615695359, +STORE, 47182615699456, 47182615707647, +STORE, 47182615695360, 47182615699455, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615707647, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615699455, +STORE, 47182615699456, 47182615707647, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615707647, +STORE, 47182615707648, 47182615715839, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608388095, +STORE, 47182608388096, 47182608396287, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615703551, +STORE, 47182615703552, 47182615707647, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611292159, +STORE, 47182611292160, 47182611296255, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615674879, +STORE, 47182615674880, 47182615678975, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615199743, +STORE, 47182615199744, 47182615203839, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615162879, +STORE, 47182615162880, 47182615166975, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615109631, +STORE, 47182615109632, 47182615117823, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612066303, +STORE, 47182612066304, 47182612082687, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611480575, +STORE, 47182611480576, 47182611484671, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611165183, +STORE, 47182611165184, 47182611169279, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608912383, +STORE, 47182608912384, 47182608916479, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608564223, +STORE, 47182608564224, 47182608568319, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608519167, +STORE, 47182608519168, 47182608523263, +ERASE, 93963664379904, 93963664379904, +STORE, 93963664379904, 93963664502783, +STORE, 93963664502784, 93963664506879, +ERASE, 140450188599296, 140450188599296, +STORE, 140450188599296, 140450188603391, +STORE, 140450188603392, 140450188607487, +ERASE, 47182606557184, 47182606557184, +STORE, 93963694723072, 93963694858239, +STORE, 140737488347136, 140737488351231, +STORE, 140730313261056, 140737488351231, +ERASE, 140730313261056, 140730313261056, +STORE, 140730313261056, 140730313265151, +STORE, 94386579017728, 94386579697663, +ERASE, 94386579017728, 94386579017728, +STORE, 94386579017728, 94386579083263, +STORE, 94386579083264, 94386579697663, +ERASE, 94386579083264, 94386579083264, +STORE, 94386579083264, 94386579431423, +STORE, 94386579431424, 94386579570687, +STORE, 94386579570688, 94386579697663, +STORE, 140124810838016, 140124811010047, +ERASE, 140124810838016, 140124810838016, +STORE, 140124810838016, 140124810842111, +STORE, 140124810842112, 140124811010047, +ERASE, 140124810842112, 140124810842112, +STORE, 140124810842112, 140124810964991, +STORE, 140124810964992, 140124810997759, +STORE, 140124810997760, 140124811005951, +STORE, 140124811005952, 140124811010047, +STORE, 140730313601024, 140730313605119, +STORE, 140730313588736, 140730313601023, +STORE, 47507984158720, 47507984166911, +STORE, 47507984166912, 47507984175103, +STORE, 47507984175104, 47507986014207, +STORE, 47507984314368, 47507986014207, +STORE, 47507984175104, 47507984314367, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985973247, +STORE, 47507985973248, 47507986014207, +STORE, 47507985657856, 47507985973247, +STORE, 47507984314368, 47507985657855, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985657855, +STORE, 47507985969152, 47507985973247, +STORE, 47507985657856, 47507985969151, +ERASE, 47507985657856, 47507985657856, +STORE, 47507985657856, 47507985969151, +STORE, 47507985997824, 47507986014207, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985997824, 47507985997824, +STORE, 47507985997824, 47507986014207, +STORE, 47507986014208, 47507986124799, +STORE, 47507986030592, 47507986124799, +STORE, 47507986014208, 47507986030591, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986116607, +STORE, 47507986116608, 47507986124799, +STORE, 47507986092032, 47507986116607, +STORE, 47507986030592, 47507986092031, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986092031, +STORE, 47507986112512, 47507986116607, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986092032, 47507986092032, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986124799, +STORE, 47507986124800, 47507986169855, +ERASE, 47507986124800, 47507986124800, +STORE, 47507986124800, 47507986132991, +STORE, 47507986132992, 47507986169855, +STORE, 47507986153472, 47507986169855, +STORE, 47507986132992, 47507986153471, +ERASE, 47507986132992, 47507986132992, +STORE, 47507986132992, 47507986153471, +STORE, 47507986161664, 47507986169855, +STORE, 47507986153472, 47507986161663, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986169855, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986161663, +STORE, 47507986161664, 47507986169855, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986169855, +STORE, 47507986169856, 47507986518015, +STORE, 47507986210816, 47507986518015, +STORE, 47507986169856, 47507986210815, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986493439, +STORE, 47507986493440, 47507986518015, +STORE, 47507986423808, 47507986493439, +STORE, 47507986210816, 47507986423807, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986423807, +STORE, 47507986489344, 47507986493439, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986423808, 47507986423808, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986518015, +STORE, 47507986518016, 47507988779007, +STORE, 47507986669568, 47507988779007, +STORE, 47507986518016, 47507986669567, +ERASE, 47507986669568, 47507986669568, +STORE, 47507986669568, 47507988762623, +STORE, 47507988762624, 47507988779007, +STORE, 47507988770816, 47507988779007, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988770816, 47507988770816, +STORE, 47507988770816, 47507988779007, +STORE, 47507988779008, 47507988914175, +ERASE, 47507988779008, 47507988779008, +STORE, 47507988779008, 47507988803583, +STORE, 47507988803584, 47507988914175, +STORE, 47507988865024, 47507988914175, +STORE, 47507988803584, 47507988865023, +ERASE, 47507988803584, 47507988803584, +STORE, 47507988803584, 47507988865023, +STORE, 47507988889600, 47507988914175, +STORE, 47507988865024, 47507988889599, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988914175, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988889599, +STORE, 47507988889600, 47507988914175, +STORE, 47507988897792, 47507988914175, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988897792, 47507988897792, +STORE, 47507988897792, 47507988914175, +STORE, 47507988897792, 47507988922367, +STORE, 47507988922368, 47507989086207, +ERASE, 47507988922368, 47507988922368, +STORE, 47507988922368, 47507988934655, +STORE, 47507988934656, 47507989086207, +STORE, 47507989032960, 47507989086207, +STORE, 47507988934656, 47507989032959, +ERASE, 47507988934656, 47507988934656, +STORE, 47507988934656, 47507989032959, +STORE, 47507989078016, 47507989086207, +STORE, 47507989032960, 47507989078015, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989086207, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989078015, +STORE, 47507989078016, 47507989086207, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989086207, +STORE, 47507989086208, 47507989684223, +STORE, 47507989204992, 47507989684223, +STORE, 47507989086208, 47507989204991, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989630975, +STORE, 47507989630976, 47507989684223, +STORE, 47507989520384, 47507989630975, +STORE, 47507989204992, 47507989520383, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989520383, +STORE, 47507989626880, 47507989630975, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989520384, 47507989520384, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989684223, +STORE, 47507989684224, 47507992735743, +STORE, 47507990228992, 47507992735743, +STORE, 47507989684224, 47507990228991, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507992514559, +STORE, 47507992514560, 47507992735743, +STORE, 47507991924736, 47507992514559, +STORE, 47507990228992, 47507991924735, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507991924735, +STORE, 47507992510464, 47507992514559, +STORE, 47507991924736, 47507992510463, +ERASE, 47507991924736, 47507991924736, +STORE, 47507991924736, 47507992510463, +STORE, 47507992719360, 47507992735743, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992719360, 47507992719360, +STORE, 47507992719360, 47507992735743, +STORE, 47507992735744, 47507992768511, +ERASE, 47507992735744, 47507992735744, +STORE, 47507992735744, 47507992743935, +STORE, 47507992743936, 47507992768511, +STORE, 47507992756224, 47507992768511, +STORE, 47507992743936, 47507992756223, +ERASE, 47507992743936, 47507992743936, +STORE, 47507992743936, 47507992756223, +STORE, 47507992760320, 47507992768511, +STORE, 47507992756224, 47507992760319, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992768511, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992760319, +STORE, 47507992760320, 47507992768511, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992768511, +STORE, 47507992768512, 47507992805375, +ERASE, 47507992768512, 47507992768512, +STORE, 47507992768512, 47507992776703, +STORE, 47507992776704, 47507992805375, +STORE, 47507992793088, 47507992805375, +STORE, 47507992776704, 47507992793087, +ERASE, 47507992776704, 47507992776704, +STORE, 47507992776704, 47507992793087, +STORE, 47507992797184, 47507992805375, +STORE, 47507992793088, 47507992797183, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992805375, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992797183, +STORE, 47507992797184, 47507992805375, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992805375, +STORE, 47507992805376, 47507993280511, +ERASE, 47507992805376, 47507992805376, +STORE, 47507992805376, 47507992813567, +STORE, 47507992813568, 47507993280511, +STORE, 47507993149440, 47507993280511, +STORE, 47507992813568, 47507993149439, +ERASE, 47507992813568, 47507992813568, +STORE, 47507992813568, 47507993149439, +STORE, 47507993272320, 47507993280511, +STORE, 47507993149440, 47507993272319, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993280511, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993272319, +STORE, 47507993272320, 47507993280511, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993280511, +STORE, 47507993280512, 47507993288703, +STORE, 47507993288704, 47507993309183, +ERASE, 47507993288704, 47507993288704, +STORE, 47507993288704, 47507993292799, +STORE, 47507993292800, 47507993309183, +STORE, 47507993296896, 47507993309183, +STORE, 47507993292800, 47507993296895, +ERASE, 47507993292800, 47507993292800, +STORE, 47507993292800, 47507993296895, +STORE, 47507993300992, 47507993309183, +STORE, 47507993296896, 47507993300991, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993309183, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993300991, +STORE, 47507993300992, 47507993309183, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993309183, +STORE, 47507993309184, 47507993317375, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985989631, +STORE, 47507985989632, 47507985997823, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993305087, +STORE, 47507993305088, 47507993309183, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988893695, +STORE, 47507988893696, 47507988897791, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993276415, +STORE, 47507993276416, 47507993280511, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992801279, +STORE, 47507992801280, 47507992805375, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992764415, +STORE, 47507992764416, 47507992768511, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992711167, +STORE, 47507992711168, 47507992719359, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989667839, +STORE, 47507989667840, 47507989684223, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989082111, +STORE, 47507989082112, 47507989086207, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988766719, +STORE, 47507988766720, 47507988770815, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986513919, +STORE, 47507986513920, 47507986518015, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986165759, +STORE, 47507986165760, 47507986169855, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986120703, +STORE, 47507986120704, 47507986124799, +ERASE, 94386579570688, 94386579570688, +STORE, 94386579570688, 94386579693567, +STORE, 94386579693568, 94386579697663, +ERASE, 140124810997760, 140124810997760, +STORE, 140124810997760, 140124811001855, +STORE, 140124811001856, 140124811005951, +ERASE, 47507984158720, 47507984158720, +STORE, 94386583982080, 94386584117247, +STORE, 94386583982080, 94386584256511, +ERASE, 94386583982080, 94386583982080, +STORE, 94386583982080, 94386584223743, +STORE, 94386584223744, 94386584256511, +ERASE, 94386584223744, 94386584223744, +STORE, 140737488347136, 140737488351231, +STORE, 140733763395584, 140737488351231, +ERASE, 140733763395584, 140733763395584, +STORE, 140733763395584, 140733763399679, +STORE, 94011546472448, 94011547152383, +ERASE, 94011546472448, 94011546472448, +STORE, 94011546472448, 94011546537983, +STORE, 94011546537984, 94011547152383, +ERASE, 94011546537984, 94011546537984, +STORE, 94011546537984, 94011546886143, +STORE, 94011546886144, 94011547025407, +STORE, 94011547025408, 94011547152383, +STORE, 139757597949952, 139757598121983, +ERASE, 139757597949952, 139757597949952, +STORE, 139757597949952, 139757597954047, +STORE, 139757597954048, 139757598121983, +ERASE, 139757597954048, 139757597954048, +STORE, 139757597954048, 139757598076927, +STORE, 139757598076928, 139757598109695, +STORE, 139757598109696, 139757598117887, +STORE, 139757598117888, 139757598121983, +STORE, 140733763596288, 140733763600383, +STORE, 140733763584000, 140733763596287, +STORE, 47875197046784, 47875197054975, +STORE, 47875197054976, 47875197063167, +STORE, 47875197063168, 47875198902271, +STORE, 47875197202432, 47875198902271, +STORE, 47875197063168, 47875197202431, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198861311, +STORE, 47875198861312, 47875198902271, +STORE, 47875198545920, 47875198861311, +STORE, 47875197202432, 47875198545919, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198545919, +STORE, 47875198857216, 47875198861311, +STORE, 47875198545920, 47875198857215, +ERASE, 47875198545920, 47875198545920, +STORE, 47875198545920, 47875198857215, +STORE, 47875198885888, 47875198902271, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198885888, 47875198885888, +STORE, 47875198885888, 47875198902271, +STORE, 47875198902272, 47875199012863, +STORE, 47875198918656, 47875199012863, +STORE, 47875198902272, 47875198918655, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875199004671, +STORE, 47875199004672, 47875199012863, +STORE, 47875198980096, 47875199004671, +STORE, 47875198918656, 47875198980095, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875198980095, +STORE, 47875199000576, 47875199004671, +STORE, 47875198980096, 47875199000575, +ERASE, 47875198980096, 47875198980096, +STORE, 47875198980096, 47875199000575, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199012863, +STORE, 47875199012864, 47875199057919, +ERASE, 47875199012864, 47875199012864, +STORE, 47875199012864, 47875199021055, +STORE, 47875199021056, 47875199057919, +STORE, 47875199041536, 47875199057919, +STORE, 47875199021056, 47875199041535, +ERASE, 47875199021056, 47875199021056, +STORE, 47875199021056, 47875199041535, +STORE, 47875199049728, 47875199057919, +STORE, 47875199041536, 47875199049727, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199057919, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199049727, +STORE, 47875199049728, 47875199057919, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199057919, +STORE, 47875199057920, 47875199406079, +STORE, 47875199098880, 47875199406079, +STORE, 47875199057920, 47875199098879, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199381503, +STORE, 47875199381504, 47875199406079, +STORE, 47875199311872, 47875199381503, +STORE, 47875199098880, 47875199311871, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199311871, +STORE, 47875199377408, 47875199381503, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199311872, 47875199311872, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199406079, +STORE, 47875199406080, 47875201667071, +STORE, 47875199557632, 47875201667071, +STORE, 47875199406080, 47875199557631, +ERASE, 47875199557632, 47875199557632, +STORE, 47875199557632, 47875201650687, +STORE, 47875201650688, 47875201667071, +STORE, 47875201658880, 47875201667071, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201658880, 47875201658880, +STORE, 47875201658880, 47875201667071, +STORE, 47875201667072, 47875201802239, +ERASE, 47875201667072, 47875201667072, +STORE, 47875201667072, 47875201691647, +STORE, 47875201691648, 47875201802239, +STORE, 47875201753088, 47875201802239, +STORE, 47875201691648, 47875201753087, +ERASE, 47875201691648, 47875201691648, +STORE, 47875201691648, 47875201753087, +STORE, 47875201777664, 47875201802239, +STORE, 47875201753088, 47875201777663, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201802239, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201777663, +STORE, 47875201777664, 47875201802239, +STORE, 47875201785856, 47875201802239, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201785856, 47875201785856, +STORE, 47875201785856, 47875201802239, +STORE, 47875201785856, 47875201810431, +STORE, 47875201810432, 47875201974271, +ERASE, 47875201810432, 47875201810432, +STORE, 47875201810432, 47875201822719, +STORE, 47875201822720, 47875201974271, +STORE, 47875201921024, 47875201974271, +STORE, 47875201822720, 47875201921023, +ERASE, 47875201822720, 47875201822720, +STORE, 47875201822720, 47875201921023, +STORE, 47875201966080, 47875201974271, +STORE, 47875201921024, 47875201966079, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201974271, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201966079, +STORE, 47875201966080, 47875201974271, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201974271, +STORE, 47875201974272, 47875202572287, +STORE, 47875202093056, 47875202572287, +STORE, 47875201974272, 47875202093055, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202519039, +STORE, 47875202519040, 47875202572287, +STORE, 47875202408448, 47875202519039, +STORE, 47875202093056, 47875202408447, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202408447, +STORE, 47875202514944, 47875202519039, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202408448, 47875202408448, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202572287, +STORE, 47875202572288, 47875205623807, +STORE, 47875203117056, 47875205623807, +STORE, 47875202572288, 47875203117055, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875205402623, +STORE, 47875205402624, 47875205623807, +STORE, 47875204812800, 47875205402623, +STORE, 47875203117056, 47875204812799, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875204812799, +STORE, 47875205398528, 47875205402623, +STORE, 47875204812800, 47875205398527, +ERASE, 47875204812800, 47875204812800, +STORE, 47875204812800, 47875205398527, +STORE, 47875205607424, 47875205623807, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205607424, 47875205607424, +STORE, 47875205607424, 47875205623807, +STORE, 47875205623808, 47875205656575, +ERASE, 47875205623808, 47875205623808, +STORE, 47875205623808, 47875205631999, +STORE, 47875205632000, 47875205656575, +STORE, 47875205644288, 47875205656575, +STORE, 47875205632000, 47875205644287, +ERASE, 47875205632000, 47875205632000, +STORE, 47875205632000, 47875205644287, +STORE, 47875205648384, 47875205656575, +STORE, 47875205644288, 47875205648383, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205656575, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205648383, +STORE, 47875205648384, 47875205656575, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205656575, +STORE, 47875205656576, 47875205693439, +ERASE, 47875205656576, 47875205656576, +STORE, 47875205656576, 47875205664767, +STORE, 47875205664768, 47875205693439, +STORE, 47875205681152, 47875205693439, +STORE, 47875205664768, 47875205681151, +ERASE, 47875205664768, 47875205664768, +STORE, 47875205664768, 47875205681151, +STORE, 47875205685248, 47875205693439, +STORE, 47875205681152, 47875205685247, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205693439, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205685247, +STORE, 47875205685248, 47875205693439, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205693439, +STORE, 47875205693440, 47875206168575, +ERASE, 47875205693440, 47875205693440, +STORE, 47875205693440, 47875205701631, +STORE, 47875205701632, 47875206168575, +STORE, 47875206037504, 47875206168575, +STORE, 47875205701632, 47875206037503, +ERASE, 47875205701632, 47875205701632, +STORE, 47875205701632, 47875206037503, +STORE, 47875206160384, 47875206168575, +STORE, 47875206037504, 47875206160383, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206168575, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206160383, +STORE, 47875206160384, 47875206168575, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206168575, +STORE, 47875206168576, 47875206176767, +STORE, 47875206176768, 47875206197247, +ERASE, 47875206176768, 47875206176768, +STORE, 47875206176768, 47875206180863, +STORE, 47875206180864, 47875206197247, +STORE, 47875206184960, 47875206197247, +STORE, 47875206180864, 47875206184959, +ERASE, 47875206180864, 47875206180864, +STORE, 47875206180864, 47875206184959, +STORE, 47875206189056, 47875206197247, +STORE, 47875206184960, 47875206189055, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206197247, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206189055, +STORE, 47875206189056, 47875206197247, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206197247, +STORE, 47875206197248, 47875206205439, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198877695, +STORE, 47875198877696, 47875198885887, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206193151, +STORE, 47875206193152, 47875206197247, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201781759, +STORE, 47875201781760, 47875201785855, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206164479, +STORE, 47875206164480, 47875206168575, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205689343, +STORE, 47875205689344, 47875205693439, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205652479, +STORE, 47875205652480, 47875205656575, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205599231, +STORE, 47875205599232, 47875205607423, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202555903, +STORE, 47875202555904, 47875202572287, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201970175, +STORE, 47875201970176, 47875201974271, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201654783, +STORE, 47875201654784, 47875201658879, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199401983, +STORE, 47875199401984, 47875199406079, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199053823, +STORE, 47875199053824, 47875199057919, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199008767, +STORE, 47875199008768, 47875199012863, +ERASE, 94011547025408, 94011547025408, +STORE, 94011547025408, 94011547148287, +STORE, 94011547148288, 94011547152383, +ERASE, 139757598109696, 139757598109696, +STORE, 139757598109696, 139757598113791, +STORE, 139757598113792, 139757598117887, +ERASE, 47875197046784, 47875197046784, +STORE, 94011557584896, 94011557720063, +STORE, 94011557584896, 94011557855231, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557851135, +STORE, 94011557851136, 94011557855231, +ERASE, 94011557851136, 94011557851136, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557847039, +STORE, 94011557847040, 94011557851135, +ERASE, 94011557847040, 94011557847040, +STORE, 94011557584896, 94011557982207, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557978111, +STORE, 94011557978112, 94011557982207, +ERASE, 94011557978112, 94011557978112, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557974015, +STORE, 94011557974016, 94011557978111, +ERASE, 94011557974016, 94011557974016, +STORE, 140737488347136, 140737488351231, +STORE, 140734130360320, 140737488351231, +ERASE, 140734130360320, 140734130360320, +STORE, 140734130360320, 140734130364415, +STORE, 94641232105472, 94641232785407, +ERASE, 94641232105472, 94641232105472, +STORE, 94641232105472, 94641232171007, +STORE, 94641232171008, 94641232785407, +ERASE, 94641232171008, 94641232171008, +STORE, 94641232171008, 94641232519167, +STORE, 94641232519168, 94641232658431, +STORE, 94641232658432, 94641232785407, +STORE, 139726599516160, 139726599688191, +ERASE, 139726599516160, 139726599516160, +STORE, 139726599516160, 139726599520255, +STORE, 139726599520256, 139726599688191, +ERASE, 139726599520256, 139726599520256, +STORE, 139726599520256, 139726599643135, +STORE, 139726599643136, 139726599675903, +STORE, 139726599675904, 139726599684095, +STORE, 139726599684096, 139726599688191, +STORE, 140734130446336, 140734130450431, +STORE, 140734130434048, 140734130446335, +STORE, 47906195480576, 47906195488767, +STORE, 47906195488768, 47906195496959, +STORE, 47906195496960, 47906197336063, +STORE, 47906195636224, 47906197336063, +STORE, 47906195496960, 47906195636223, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906197295103, +STORE, 47906197295104, 47906197336063, +STORE, 47906196979712, 47906197295103, +STORE, 47906195636224, 47906196979711, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906196979711, +STORE, 47906197291008, 47906197295103, +STORE, 47906196979712, 47906197291007, +ERASE, 47906196979712, 47906196979712, +STORE, 47906196979712, 47906197291007, +STORE, 47906197319680, 47906197336063, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197319680, 47906197319680, +STORE, 47906197319680, 47906197336063, +STORE, 47906197336064, 47906197446655, +STORE, 47906197352448, 47906197446655, +STORE, 47906197336064, 47906197352447, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197438463, +STORE, 47906197438464, 47906197446655, +STORE, 47906197413888, 47906197438463, +STORE, 47906197352448, 47906197413887, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197413887, +STORE, 47906197434368, 47906197438463, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197413888, 47906197413888, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197446655, +STORE, 47906197446656, 47906197491711, +ERASE, 47906197446656, 47906197446656, +STORE, 47906197446656, 47906197454847, +STORE, 47906197454848, 47906197491711, +STORE, 47906197475328, 47906197491711, +STORE, 47906197454848, 47906197475327, +ERASE, 47906197454848, 47906197454848, +STORE, 47906197454848, 47906197475327, +STORE, 47906197483520, 47906197491711, +STORE, 47906197475328, 47906197483519, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197491711, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197483519, +STORE, 47906197483520, 47906197491711, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197491711, +STORE, 47906197491712, 47906197839871, +STORE, 47906197532672, 47906197839871, +STORE, 47906197491712, 47906197532671, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197815295, +STORE, 47906197815296, 47906197839871, +STORE, 47906197745664, 47906197815295, +STORE, 47906197532672, 47906197745663, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197745663, +STORE, 47906197811200, 47906197815295, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197745664, 47906197745664, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197839871, +STORE, 47906197839872, 47906200100863, +STORE, 47906197991424, 47906200100863, +STORE, 47906197839872, 47906197991423, +ERASE, 47906197991424, 47906197991424, +STORE, 47906197991424, 47906200084479, +STORE, 47906200084480, 47906200100863, +STORE, 47906200092672, 47906200100863, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200092672, 47906200092672, +STORE, 47906200092672, 47906200100863, +STORE, 47906200100864, 47906200236031, +ERASE, 47906200100864, 47906200100864, +STORE, 47906200100864, 47906200125439, +STORE, 47906200125440, 47906200236031, +STORE, 47906200186880, 47906200236031, +STORE, 47906200125440, 47906200186879, +ERASE, 47906200125440, 47906200125440, +STORE, 47906200125440, 47906200186879, +STORE, 47906200211456, 47906200236031, +STORE, 47906200186880, 47906200211455, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200236031, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200211455, +STORE, 47906200211456, 47906200236031, +STORE, 47906200219648, 47906200236031, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200219648, 47906200219648, +STORE, 47906200219648, 47906200236031, +STORE, 47906200219648, 47906200244223, +STORE, 47906200244224, 47906200408063, +ERASE, 47906200244224, 47906200244224, +STORE, 47906200244224, 47906200256511, +STORE, 47906200256512, 47906200408063, +STORE, 47906200354816, 47906200408063, +STORE, 47906200256512, 47906200354815, +ERASE, 47906200256512, 47906200256512, +STORE, 47906200256512, 47906200354815, +STORE, 47906200399872, 47906200408063, +STORE, 47906200354816, 47906200399871, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200408063, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200399871, +STORE, 47906200399872, 47906200408063, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200408063, +STORE, 47906200408064, 47906201006079, +STORE, 47906200526848, 47906201006079, +STORE, 47906200408064, 47906200526847, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200952831, +STORE, 47906200952832, 47906201006079, +STORE, 47906200842240, 47906200952831, +STORE, 47906200526848, 47906200842239, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200842239, +STORE, 47906200948736, 47906200952831, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200842240, 47906200842240, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906201006079, +STORE, 47906201006080, 47906204057599, +STORE, 47906201550848, 47906204057599, +STORE, 47906201006080, 47906201550847, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203836415, +STORE, 47906203836416, 47906204057599, +STORE, 47906203246592, 47906203836415, +STORE, 47906201550848, 47906203246591, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203246591, +STORE, 47906203832320, 47906203836415, +STORE, 47906203246592, 47906203832319, +ERASE, 47906203246592, 47906203246592, +STORE, 47906203246592, 47906203832319, +STORE, 47906204041216, 47906204057599, +STORE, 47906203836416, 47906204041215, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204041215, +ERASE, 47906204041216, 47906204041216, +STORE, 47906204041216, 47906204057599, +STORE, 47906204057600, 47906204090367, +ERASE, 47906204057600, 47906204057600, +STORE, 47906204057600, 47906204065791, +STORE, 47906204065792, 47906204090367, +STORE, 47906204078080, 47906204090367, +STORE, 47906204065792, 47906204078079, +ERASE, 47906204065792, 47906204065792, +STORE, 47906204065792, 47906204078079, +STORE, 47906204082176, 47906204090367, +STORE, 47906204078080, 47906204082175, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204090367, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204082175, +STORE, 47906204082176, 47906204090367, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204090367, +STORE, 47906204090368, 47906204127231, +ERASE, 47906204090368, 47906204090368, +STORE, 47906204090368, 47906204098559, +STORE, 47906204098560, 47906204127231, +STORE, 47906204114944, 47906204127231, +STORE, 47906204098560, 47906204114943, +ERASE, 47906204098560, 47906204098560, +STORE, 47906204098560, 47906204114943, +STORE, 47906204119040, 47906204127231, +STORE, 47906204114944, 47906204119039, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204127231, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204119039, +STORE, 47906204119040, 47906204127231, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204127231, +STORE, 47906204127232, 47906204602367, +ERASE, 47906204127232, 47906204127232, +STORE, 47906204127232, 47906204135423, +STORE, 47906204135424, 47906204602367, +STORE, 47906204471296, 47906204602367, +STORE, 47906204135424, 47906204471295, +ERASE, 47906204135424, 47906204135424, +STORE, 47906204135424, 47906204471295, +STORE, 47906204594176, 47906204602367, +STORE, 47906204471296, 47906204594175, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204602367, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204594175, +STORE, 47906204594176, 47906204602367, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204602367, +STORE, 47906204602368, 47906204610559, +STORE, 47906204610560, 47906204631039, +ERASE, 47906204610560, 47906204610560, +STORE, 47906204610560, 47906204614655, +STORE, 47906204614656, 47906204631039, +STORE, 47906204618752, 47906204631039, +STORE, 47906204614656, 47906204618751, +ERASE, 47906204614656, 47906204614656, +STORE, 47906204614656, 47906204618751, +STORE, 47906204622848, 47906204631039, +STORE, 47906204618752, 47906204622847, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204631039, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204622847, +STORE, 47906204622848, 47906204631039, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204631039, +STORE, 47906204631040, 47906204639231, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197311487, +STORE, 47906197311488, 47906197319679, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204626943, +STORE, 47906204626944, 47906204631039, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200215551, +STORE, 47906200215552, 47906200219647, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204598271, +STORE, 47906204598272, 47906204602367, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204123135, +STORE, 47906204123136, 47906204127231, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204086271, +STORE, 47906204086272, 47906204090367, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204033023, +STORE, 47906204033024, 47906204041215, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906200989695, +STORE, 47906200989696, 47906201006079, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200403967, +STORE, 47906200403968, 47906200408063, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200088575, +STORE, 47906200088576, 47906200092671, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197835775, +STORE, 47906197835776, 47906197839871, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197487615, +STORE, 47906197487616, 47906197491711, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197442559, +STORE, 47906197442560, 47906197446655, +ERASE, 94641232658432, 94641232658432, +STORE, 94641232658432, 94641232781311, +STORE, 94641232781312, 94641232785407, +ERASE, 139726599675904, 139726599675904, +STORE, 139726599675904, 139726599679999, +STORE, 139726599680000, 139726599684095, +ERASE, 47906195480576, 47906195480576, +STORE, 94641242615808, 94641242750975, + }; + unsigned long set11[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140732658499584, 140737488351231, +ERASE, 140732658499584, 140732658499584, +STORE, 140732658499584, 140732658503679, +STORE, 94029856579584, 94029856751615, +ERASE, 94029856579584, 94029856579584, +STORE, 94029856579584, 94029856595967, +STORE, 94029856595968, 94029856751615, +ERASE, 94029856595968, 94029856595968, +STORE, 94029856595968, 94029856698367, +STORE, 94029856698368, 94029856739327, +STORE, 94029856739328, 94029856751615, +STORE, 140014592573440, 140014592745471, +ERASE, 140014592573440, 140014592573440, +STORE, 140014592573440, 140014592577535, +STORE, 140014592577536, 140014592745471, +ERASE, 140014592577536, 140014592577536, +STORE, 140014592577536, 140014592700415, +STORE, 140014592700416, 140014592733183, +STORE, 140014592733184, 140014592741375, +STORE, 140014592741376, 140014592745471, +STORE, 140732658565120, 140732658569215, +STORE, 140732658552832, 140732658565119, + }; + + unsigned long set12[] = { /* contains 12 values. */ +STORE, 140737488347136, 140737488351231, +STORE, 140732658499584, 140737488351231, +ERASE, 140732658499584, 140732658499584, +STORE, 140732658499584, 140732658503679, +STORE, 94029856579584, 94029856751615, +ERASE, 94029856579584, 94029856579584, +STORE, 94029856579584, 94029856595967, +STORE, 94029856595968, 94029856751615, +ERASE, 94029856595968, 94029856595968, +STORE, 94029856595968, 94029856698367, +STORE, 94029856698368, 94029856739327, +STORE, 94029856739328, 94029856751615, +STORE, 140014592573440, 140014592745471, +ERASE, 140014592573440, 140014592573440, +STORE, 140014592573440, 140014592577535, +STORE, 140014592577536, 140014592745471, +ERASE, 140014592577536, 140014592577536, +STORE, 140014592577536, 140014592700415, +STORE, 140014592700416, 140014592733183, +STORE, 140014592733184, 140014592741375, +STORE, 140014592741376, 140014592745471, +STORE, 140732658565120, 140732658569215, +STORE, 140732658552832, 140732658565119, +STORE, 140014592741375, 140014592741375, /* contrived */ +STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ + }; + unsigned long set13[] = { +STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ +STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ +STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ +STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ +STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ +STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ +STORE, 140373518663680, 140373518684159,/*: deleted (257) */ +STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ +STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ +STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ +STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ + }; + unsigned long set14[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731667996672, 140737488351231, +SNULL, 140731668000767, 140737488351231, +STORE, 140731667996672, 140731668000767, +STORE, 140731667865600, 140731668000767, +STORE, 94077521272832, 94077521313791, +SNULL, 94077521301503, 94077521313791, +STORE, 94077521272832, 94077521301503, +STORE, 94077521301504, 94077521313791, +ERASE, 94077521301504, 94077521313791, +STORE, 94077521305600, 94077521313791, +STORE, 139826134630400, 139826136883199, +SNULL, 139826134773759, 139826136883199, +STORE, 139826134630400, 139826134773759, +STORE, 139826134773760, 139826136883199, +ERASE, 139826134773760, 139826136883199, +STORE, 139826136870912, 139826136879103, +STORE, 139826136879104, 139826136883199, +STORE, 140731668013056, 140731668017151, +STORE, 140731668000768, 140731668013055, +STORE, 139826136862720, 139826136870911, +STORE, 139826132406272, 139826134630399, +SNULL, 139826134056959, 139826134630399, +STORE, 139826132406272, 139826134056959, +STORE, 139826134056960, 139826134630399, +SNULL, 139826134056960, 139826134626303, +STORE, 139826134626304, 139826134630399, +STORE, 139826134056960, 139826134626303, +ERASE, 139826134056960, 139826134626303, +STORE, 139826134056960, 139826134626303, +ERASE, 139826134626304, 139826134630399, +STORE, 139826134626304, 139826134630399, +STORE, 139826136842240, 139826136862719, +STORE, 139826130022400, 139826132406271, +SNULL, 139826130022400, 139826130288639, +STORE, 139826130288640, 139826132406271, +STORE, 139826130022400, 139826130288639, +SNULL, 139826132381695, 139826132406271, +STORE, 139826130288640, 139826132381695, +STORE, 139826132381696, 139826132406271, +SNULL, 139826132381696, 139826132402175, +STORE, 139826132402176, 139826132406271, +STORE, 139826132381696, 139826132402175, +ERASE, 139826132381696, 139826132402175, +STORE, 139826132381696, 139826132402175, +ERASE, 139826132402176, 139826132406271, +STORE, 139826132402176, 139826132406271, +STORE, 139826127806464, 139826130022399, +SNULL, 139826127806464, 139826127904767, +STORE, 139826127904768, 139826130022399, +STORE, 139826127806464, 139826127904767, +SNULL, 139826129997823, 139826130022399, +STORE, 139826127904768, 139826129997823, +STORE, 139826129997824, 139826130022399, +SNULL, 139826129997824, 139826130006015, +STORE, 139826130006016, 139826130022399, +STORE, 139826129997824, 139826130006015, +ERASE, 139826129997824, 139826130006015, +STORE, 139826129997824, 139826130006015, +ERASE, 139826130006016, 139826130022399, +STORE, 139826130006016, 139826130022399, +STORE, 139826124009472, 139826127806463, +SNULL, 139826124009472, 139826125668351, +STORE, 139826125668352, 139826127806463, +STORE, 139826124009472, 139826125668351, +SNULL, 139826127765503, 139826127806463, +STORE, 139826125668352, 139826127765503, +STORE, 139826127765504, 139826127806463, +SNULL, 139826127765504, 139826127790079, +STORE, 139826127790080, 139826127806463, +STORE, 139826127765504, 139826127790079, +ERASE, 139826127765504, 139826127790079, +STORE, 139826127765504, 139826127790079, +ERASE, 139826127790080, 139826127806463, +STORE, 139826127790080, 139826127806463, +STORE, 139826121748480, 139826124009471, +SNULL, 139826121748480, 139826121900031, +STORE, 139826121900032, 139826124009471, +STORE, 139826121748480, 139826121900031, +SNULL, 139826123993087, 139826124009471, +STORE, 139826121900032, 139826123993087, +STORE, 139826123993088, 139826124009471, +SNULL, 139826123993088, 139826124001279, +STORE, 139826124001280, 139826124009471, +STORE, 139826123993088, 139826124001279, +ERASE, 139826123993088, 139826124001279, +STORE, 139826123993088, 139826124001279, +ERASE, 139826124001280, 139826124009471, +STORE, 139826124001280, 139826124009471, +STORE, 139826119626752, 139826121748479, +SNULL, 139826119626752, 139826119643135, +STORE, 139826119643136, 139826121748479, +STORE, 139826119626752, 139826119643135, +SNULL, 139826121740287, 139826121748479, +STORE, 139826119643136, 139826121740287, +STORE, 139826121740288, 139826121748479, +ERASE, 139826121740288, 139826121748479, +STORE, 139826121740288, 139826121748479, +STORE, 139826136834048, 139826136842239, +STORE, 139826117496832, 139826119626751, +SNULL, 139826117496832, 139826117525503, +STORE, 139826117525504, 139826119626751, +STORE, 139826117496832, 139826117525503, +SNULL, 139826119618559, 139826119626751, +STORE, 139826117525504, 139826119618559, +STORE, 139826119618560, 139826119626751, +ERASE, 139826119618560, 139826119626751, +STORE, 139826119618560, 139826119626751, +STORE, 139826115244032, 139826117496831, +SNULL, 139826115244032, 139826115395583, +STORE, 139826115395584, 139826117496831, +STORE, 139826115244032, 139826115395583, +SNULL, 139826117488639, 139826117496831, +STORE, 139826115395584, 139826117488639, +STORE, 139826117488640, 139826117496831, +ERASE, 139826117488640, 139826117496831, +STORE, 139826117488640, 139826117496831, +STORE, 139826113073152, 139826115244031, +SNULL, 139826113073152, 139826113142783, +STORE, 139826113142784, 139826115244031, +STORE, 139826113073152, 139826113142783, +SNULL, 139826115235839, 139826115244031, +STORE, 139826113142784, 139826115235839, +STORE, 139826115235840, 139826115244031, +ERASE, 139826115235840, 139826115244031, +STORE, 139826115235840, 139826115244031, +STORE, 139826109861888, 139826113073151, +SNULL, 139826109861888, 139826110939135, +STORE, 139826110939136, 139826113073151, +STORE, 139826109861888, 139826110939135, +SNULL, 139826113036287, 139826113073151, +STORE, 139826110939136, 139826113036287, +STORE, 139826113036288, 139826113073151, +ERASE, 139826113036288, 139826113073151, +STORE, 139826113036288, 139826113073151, +STORE, 139826107727872, 139826109861887, +SNULL, 139826107727872, 139826107756543, +STORE, 139826107756544, 139826109861887, +STORE, 139826107727872, 139826107756543, +SNULL, 139826109853695, 139826109861887, +STORE, 139826107756544, 139826109853695, +STORE, 139826109853696, 139826109861887, +ERASE, 139826109853696, 139826109861887, +STORE, 139826109853696, 139826109861887, +STORE, 139826105417728, 139826107727871, +SNULL, 139826105417728, 139826105622527, +STORE, 139826105622528, 139826107727871, +STORE, 139826105417728, 139826105622527, +SNULL, 139826107719679, 139826107727871, +STORE, 139826105622528, 139826107719679, +STORE, 139826107719680, 139826107727871, +ERASE, 139826107719680, 139826107727871, +STORE, 139826107719680, 139826107727871, +STORE, 139826136825856, 139826136842239, +STORE, 139826103033856, 139826105417727, +SNULL, 139826103033856, 139826103226367, +STORE, 139826103226368, 139826105417727, +STORE, 139826103033856, 139826103226367, +SNULL, 139826105319423, 139826105417727, +STORE, 139826103226368, 139826105319423, +STORE, 139826105319424, 139826105417727, +ERASE, 139826105319424, 139826105417727, +STORE, 139826105319424, 139826105417727, +STORE, 139826100916224, 139826103033855, +SNULL, 139826100916224, 139826100932607, +STORE, 139826100932608, 139826103033855, +STORE, 139826100916224, 139826100932607, +SNULL, 139826103025663, 139826103033855, +STORE, 139826100932608, 139826103025663, +STORE, 139826103025664, 139826103033855, +ERASE, 139826103025664, 139826103033855, +STORE, 139826103025664, 139826103033855, +STORE, 139826098348032, 139826100916223, +SNULL, 139826098348032, 139826098814975, +STORE, 139826098814976, 139826100916223, +STORE, 139826098348032, 139826098814975, +SNULL, 139826100908031, 139826100916223, +STORE, 139826098814976, 139826100908031, +STORE, 139826100908032, 139826100916223, +ERASE, 139826100908032, 139826100916223, +STORE, 139826100908032, 139826100916223, +STORE, 139826096234496, 139826098348031, +SNULL, 139826096234496, 139826096246783, +STORE, 139826096246784, 139826098348031, +STORE, 139826096234496, 139826096246783, +SNULL, 139826098339839, 139826098348031, +STORE, 139826096246784, 139826098339839, +STORE, 139826098339840, 139826098348031, +ERASE, 139826098339840, 139826098348031, +STORE, 139826098339840, 139826098348031, +STORE, 139826094055424, 139826096234495, +SNULL, 139826094055424, 139826094133247, +STORE, 139826094133248, 139826096234495, +STORE, 139826094055424, 139826094133247, +SNULL, 139826096226303, 139826096234495, +STORE, 139826094133248, 139826096226303, +STORE, 139826096226304, 139826096234495, +ERASE, 139826096226304, 139826096234495, +STORE, 139826096226304, 139826096234495, +STORE, 139826136817664, 139826136842239, +STORE, 139826091937792, 139826094055423, +SNULL, 139826091937792, 139826091954175, +STORE, 139826091954176, 139826094055423, +STORE, 139826091937792, 139826091954175, +SNULL, 139826094047231, 139826094055423, +STORE, 139826091954176, 139826094047231, +STORE, 139826094047232, 139826094055423, +ERASE, 139826094047232, 139826094055423, +STORE, 139826094047232, 139826094055423, +STORE, 139826136809472, 139826136842239, +SNULL, 139826127781887, 139826127790079, +STORE, 139826127765504, 139826127781887, +STORE, 139826127781888, 139826127790079, +SNULL, 139826094051327, 139826094055423, +STORE, 139826094047232, 139826094051327, +STORE, 139826094051328, 139826094055423, +SNULL, 139826096230399, 139826096234495, +STORE, 139826096226304, 139826096230399, +STORE, 139826096230400, 139826096234495, +SNULL, 139826098343935, 139826098348031, +STORE, 139826098339840, 139826098343935, +STORE, 139826098343936, 139826098348031, +SNULL, 139826130001919, 139826130006015, +STORE, 139826129997824, 139826130001919, +STORE, 139826130001920, 139826130006015, +SNULL, 139826100912127, 139826100916223, +STORE, 139826100908032, 139826100912127, +STORE, 139826100912128, 139826100916223, +SNULL, 139826103029759, 139826103033855, +STORE, 139826103025664, 139826103029759, +STORE, 139826103029760, 139826103033855, +SNULL, 139826105413631, 139826105417727, +STORE, 139826105319424, 139826105413631, +STORE, 139826105413632, 139826105417727, +SNULL, 139826107723775, 139826107727871, +STORE, 139826107719680, 139826107723775, +STORE, 139826107723776, 139826107727871, +SNULL, 139826109857791, 139826109861887, +STORE, 139826109853696, 139826109857791, +STORE, 139826109857792, 139826109861887, +SNULL, 139826113044479, 139826113073151, +STORE, 139826113036288, 139826113044479, +STORE, 139826113044480, 139826113073151, +SNULL, 139826115239935, 139826115244031, +STORE, 139826115235840, 139826115239935, +STORE, 139826115239936, 139826115244031, +SNULL, 139826117492735, 139826117496831, +STORE, 139826117488640, 139826117492735, +STORE, 139826117492736, 139826117496831, +SNULL, 139826119622655, 139826119626751, +STORE, 139826119618560, 139826119622655, +STORE, 139826119622656, 139826119626751, +SNULL, 139826121744383, 139826121748479, +STORE, 139826121740288, 139826121744383, +STORE, 139826121744384, 139826121748479, +SNULL, 139826123997183, 139826124001279, +STORE, 139826123993088, 139826123997183, +STORE, 139826123997184, 139826124001279, +SNULL, 139826132398079, 139826132402175, +STORE, 139826132381696, 139826132398079, +STORE, 139826132398080, 139826132402175, +SNULL, 139826134622207, 139826134626303, +STORE, 139826134056960, 139826134622207, +STORE, 139826134622208, 139826134626303, +SNULL, 94077521309695, 94077521313791, +STORE, 94077521305600, 94077521309695, +STORE, 94077521309696, 94077521313791, +SNULL, 139826136875007, 139826136879103, +STORE, 139826136870912, 139826136875007, +STORE, 139826136875008, 139826136879103, +ERASE, 139826136842240, 139826136862719, +STORE, 94077554049024, 94077554184191, +STORE, 139826136543232, 139826136842239, +STORE, 139826136276992, 139826136842239, +STORE, 139826136010752, 139826136842239, +STORE, 139826135744512, 139826136842239, +SNULL, 139826136543231, 139826136842239, +STORE, 139826135744512, 139826136543231, +STORE, 139826136543232, 139826136842239, +SNULL, 139826136543232, 139826136809471, +STORE, 139826136809472, 139826136842239, +STORE, 139826136543232, 139826136809471, + }; + unsigned long set15[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722061451264, 140737488351231, +SNULL, 140722061455359, 140737488351231, +STORE, 140722061451264, 140722061455359, +STORE, 140722061320192, 140722061455359, +STORE, 94728600248320, 94728600289279, +SNULL, 94728600276991, 94728600289279, +STORE, 94728600248320, 94728600276991, +STORE, 94728600276992, 94728600289279, +ERASE, 94728600276992, 94728600289279, +STORE, 94728600281088, 94728600289279, +STORE, 139906806779904, 139906809032703, +SNULL, 139906806923263, 139906809032703, +STORE, 139906806779904, 139906806923263, +STORE, 139906806923264, 139906809032703, +ERASE, 139906806923264, 139906809032703, +STORE, 139906809020416, 139906809028607, +STORE, 139906809028608, 139906809032703, +STORE, 140722061692928, 140722061697023, +STORE, 140722061680640, 140722061692927, +STORE, 139906809012224, 139906809020415, +STORE, 139906804555776, 139906806779903, +SNULL, 139906806206463, 139906806779903, +STORE, 139906804555776, 139906806206463, +STORE, 139906806206464, 139906806779903, +SNULL, 139906806206464, 139906806775807, +STORE, 139906806775808, 139906806779903, +STORE, 139906806206464, 139906806775807, +ERASE, 139906806206464, 139906806775807, +STORE, 139906806206464, 139906806775807, +ERASE, 139906806775808, 139906806779903, +STORE, 139906806775808, 139906806779903, +STORE, 139906808991744, 139906809012223, +STORE, 139906802171904, 139906804555775, +SNULL, 139906802171904, 139906802438143, +STORE, 139906802438144, 139906804555775, +STORE, 139906802171904, 139906802438143, +SNULL, 139906804531199, 139906804555775, +STORE, 139906802438144, 139906804531199, +STORE, 139906804531200, 139906804555775, +SNULL, 139906804531200, 139906804551679, +STORE, 139906804551680, 139906804555775, +STORE, 139906804531200, 139906804551679, +ERASE, 139906804531200, 139906804551679, +STORE, 139906804531200, 139906804551679, +ERASE, 139906804551680, 139906804555775, +STORE, 139906804551680, 139906804555775, +STORE, 139906799955968, 139906802171903, +SNULL, 139906799955968, 139906800054271, +STORE, 139906800054272, 139906802171903, +STORE, 139906799955968, 139906800054271, +SNULL, 139906802147327, 139906802171903, +STORE, 139906800054272, 139906802147327, +STORE, 139906802147328, 139906802171903, +SNULL, 139906802147328, 139906802155519, +STORE, 139906802155520, 139906802171903, +STORE, 139906802147328, 139906802155519, +ERASE, 139906802147328, 139906802155519, +STORE, 139906802147328, 139906802155519, +ERASE, 139906802155520, 139906802171903, +STORE, 139906802155520, 139906802171903, +STORE, 139906796158976, 139906799955967, +SNULL, 139906796158976, 139906797817855, +STORE, 139906797817856, 139906799955967, +STORE, 139906796158976, 139906797817855, +SNULL, 139906799915007, 139906799955967, +STORE, 139906797817856, 139906799915007, +STORE, 139906799915008, 139906799955967, +SNULL, 139906799915008, 139906799939583, +STORE, 139906799939584, 139906799955967, +STORE, 139906799915008, 139906799939583, +ERASE, 139906799915008, 139906799939583, +STORE, 139906799915008, 139906799939583, +ERASE, 139906799939584, 139906799955967, +STORE, 139906799939584, 139906799955967, +STORE, 139906793897984, 139906796158975, +SNULL, 139906793897984, 139906794049535, +STORE, 139906794049536, 139906796158975, +STORE, 139906793897984, 139906794049535, +SNULL, 139906796142591, 139906796158975, +STORE, 139906794049536, 139906796142591, +STORE, 139906796142592, 139906796158975, +SNULL, 139906796142592, 139906796150783, +STORE, 139906796150784, 139906796158975, +STORE, 139906796142592, 139906796150783, +ERASE, 139906796142592, 139906796150783, +STORE, 139906796142592, 139906796150783, +ERASE, 139906796150784, 139906796158975, +STORE, 139906796150784, 139906796158975, +STORE, 139906791776256, 139906793897983, +SNULL, 139906791776256, 139906791792639, +STORE, 139906791792640, 139906793897983, +STORE, 139906791776256, 139906791792639, +SNULL, 139906793889791, 139906793897983, +STORE, 139906791792640, 139906793889791, +STORE, 139906793889792, 139906793897983, +ERASE, 139906793889792, 139906793897983, +STORE, 139906793889792, 139906793897983, +STORE, 139906808983552, 139906808991743, +STORE, 139906789646336, 139906791776255, +SNULL, 139906789646336, 139906789675007, +STORE, 139906789675008, 139906791776255, +STORE, 139906789646336, 139906789675007, +SNULL, 139906791768063, 139906791776255, +STORE, 139906789675008, 139906791768063, +STORE, 139906791768064, 139906791776255, +ERASE, 139906791768064, 139906791776255, +STORE, 139906791768064, 139906791776255, +STORE, 139906787393536, 139906789646335, +SNULL, 139906787393536, 139906787545087, +STORE, 139906787545088, 139906789646335, +STORE, 139906787393536, 139906787545087, +SNULL, 139906789638143, 139906789646335, +STORE, 139906787545088, 139906789638143, +STORE, 139906789638144, 139906789646335, +ERASE, 139906789638144, 139906789646335, +STORE, 139906789638144, 139906789646335, +STORE, 139906785222656, 139906787393535, +SNULL, 139906785222656, 139906785292287, +STORE, 139906785292288, 139906787393535, +STORE, 139906785222656, 139906785292287, +SNULL, 139906787385343, 139906787393535, +STORE, 139906785292288, 139906787385343, +STORE, 139906787385344, 139906787393535, +ERASE, 139906787385344, 139906787393535, +STORE, 139906787385344, 139906787393535, +STORE, 139906782011392, 139906785222655, +SNULL, 139906782011392, 139906783088639, +STORE, 139906783088640, 139906785222655, +STORE, 139906782011392, 139906783088639, +SNULL, 139906785185791, 139906785222655, +STORE, 139906783088640, 139906785185791, +STORE, 139906785185792, 139906785222655, +ERASE, 139906785185792, 139906785222655, +STORE, 139906785185792, 139906785222655, +STORE, 139906779877376, 139906782011391, +SNULL, 139906779877376, 139906779906047, +STORE, 139906779906048, 139906782011391, +STORE, 139906779877376, 139906779906047, +SNULL, 139906782003199, 139906782011391, +STORE, 139906779906048, 139906782003199, +STORE, 139906782003200, 139906782011391, +ERASE, 139906782003200, 139906782011391, +STORE, 139906782003200, 139906782011391, +STORE, 139906777567232, 139906779877375, +SNULL, 139906777567232, 139906777772031, +STORE, 139906777772032, 139906779877375, +STORE, 139906777567232, 139906777772031, +SNULL, 139906779869183, 139906779877375, +STORE, 139906777772032, 139906779869183, +STORE, 139906779869184, 139906779877375, +ERASE, 139906779869184, 139906779877375, +STORE, 139906779869184, 139906779877375, +STORE, 139906808975360, 139906808991743, +STORE, 139906775183360, 139906777567231, +SNULL, 139906775183360, 139906775375871, +STORE, 139906775375872, 139906777567231, +STORE, 139906775183360, 139906775375871, +SNULL, 139906777468927, 139906777567231, +STORE, 139906775375872, 139906777468927, +STORE, 139906777468928, 139906777567231, +ERASE, 139906777468928, 139906777567231, +STORE, 139906777468928, 139906777567231, +STORE, 139906773065728, 139906775183359, +SNULL, 139906773065728, 139906773082111, +STORE, 139906773082112, 139906775183359, +STORE, 139906773065728, 139906773082111, +SNULL, 139906775175167, 139906775183359, +STORE, 139906773082112, 139906775175167, +STORE, 139906775175168, 139906775183359, +ERASE, 139906775175168, 139906775183359, +STORE, 139906775175168, 139906775183359, +STORE, 139906770497536, 139906773065727, +SNULL, 139906770497536, 139906770964479, +STORE, 139906770964480, 139906773065727, +STORE, 139906770497536, 139906770964479, +SNULL, 139906773057535, 139906773065727, +STORE, 139906770964480, 139906773057535, +STORE, 139906773057536, 139906773065727, +ERASE, 139906773057536, 139906773065727, +STORE, 139906773057536, 139906773065727, +STORE, 139906768384000, 139906770497535, +SNULL, 139906768384000, 139906768396287, +STORE, 139906768396288, 139906770497535, +STORE, 139906768384000, 139906768396287, +SNULL, 139906770489343, 139906770497535, +STORE, 139906768396288, 139906770489343, +STORE, 139906770489344, 139906770497535, +ERASE, 139906770489344, 139906770497535, +STORE, 139906770489344, 139906770497535, +STORE, 139906766204928, 139906768383999, +SNULL, 139906766204928, 139906766282751, +STORE, 139906766282752, 139906768383999, +STORE, 139906766204928, 139906766282751, +SNULL, 139906768375807, 139906768383999, +STORE, 139906766282752, 139906768375807, +STORE, 139906768375808, 139906768383999, +ERASE, 139906768375808, 139906768383999, +STORE, 139906768375808, 139906768383999, +STORE, 139906808967168, 139906808991743, +STORE, 139906764087296, 139906766204927, +SNULL, 139906764087296, 139906764103679, +STORE, 139906764103680, 139906766204927, +STORE, 139906764087296, 139906764103679, +SNULL, 139906766196735, 139906766204927, +STORE, 139906764103680, 139906766196735, +STORE, 139906766196736, 139906766204927, +ERASE, 139906766196736, 139906766204927, +STORE, 139906766196736, 139906766204927, +STORE, 139906808958976, 139906808991743, +SNULL, 139906799931391, 139906799939583, +STORE, 139906799915008, 139906799931391, +STORE, 139906799931392, 139906799939583, +SNULL, 139906766200831, 139906766204927, +STORE, 139906766196736, 139906766200831, +STORE, 139906766200832, 139906766204927, +SNULL, 139906768379903, 139906768383999, +STORE, 139906768375808, 139906768379903, +STORE, 139906768379904, 139906768383999, +SNULL, 139906770493439, 139906770497535, +STORE, 139906770489344, 139906770493439, +STORE, 139906770493440, 139906770497535, +SNULL, 139906802151423, 139906802155519, +STORE, 139906802147328, 139906802151423, +STORE, 139906802151424, 139906802155519, +SNULL, 139906773061631, 139906773065727, +STORE, 139906773057536, 139906773061631, +STORE, 139906773061632, 139906773065727, +SNULL, 139906775179263, 139906775183359, +STORE, 139906775175168, 139906775179263, +STORE, 139906775179264, 139906775183359, +SNULL, 139906777563135, 139906777567231, +STORE, 139906777468928, 139906777563135, +STORE, 139906777563136, 139906777567231, +SNULL, 139906779873279, 139906779877375, +STORE, 139906779869184, 139906779873279, +STORE, 139906779873280, 139906779877375, +SNULL, 139906782007295, 139906782011391, +STORE, 139906782003200, 139906782007295, +STORE, 139906782007296, 139906782011391, +SNULL, 139906785193983, 139906785222655, +STORE, 139906785185792, 139906785193983, +STORE, 139906785193984, 139906785222655, +SNULL, 139906787389439, 139906787393535, +STORE, 139906787385344, 139906787389439, +STORE, 139906787389440, 139906787393535, +SNULL, 139906789642239, 139906789646335, +STORE, 139906789638144, 139906789642239, +STORE, 139906789642240, 139906789646335, +SNULL, 139906791772159, 139906791776255, +STORE, 139906791768064, 139906791772159, +STORE, 139906791772160, 139906791776255, +SNULL, 139906793893887, 139906793897983, +STORE, 139906793889792, 139906793893887, +STORE, 139906793893888, 139906793897983, +SNULL, 139906796146687, 139906796150783, +STORE, 139906796142592, 139906796146687, +STORE, 139906796146688, 139906796150783, +SNULL, 139906804547583, 139906804551679, +STORE, 139906804531200, 139906804547583, +STORE, 139906804547584, 139906804551679, +SNULL, 139906806771711, 139906806775807, +STORE, 139906806206464, 139906806771711, +STORE, 139906806771712, 139906806775807, +SNULL, 94728600285183, 94728600289279, +STORE, 94728600281088, 94728600285183, +STORE, 94728600285184, 94728600289279, +SNULL, 139906809024511, 139906809028607, +STORE, 139906809020416, 139906809024511, +STORE, 139906809024512, 139906809028607, +ERASE, 139906808991744, 139906809012223, +STORE, 94728620138496, 94728620273663, +STORE, 139906808692736, 139906808991743, +STORE, 139906808426496, 139906808991743, +STORE, 139906808160256, 139906808991743, +STORE, 139906807894016, 139906808991743, +SNULL, 139906808692735, 139906808991743, +STORE, 139906807894016, 139906808692735, +STORE, 139906808692736, 139906808991743, +SNULL, 139906808692736, 139906808958975, +STORE, 139906808958976, 139906808991743, +STORE, 139906808692736, 139906808958975, + }; + + unsigned long set16[] = { +STORE, 94174808662016, 94174809321471, +STORE, 94174811414528, 94174811426815, +STORE, 94174811426816, 94174811430911, +STORE, 94174811430912, 94174811443199, +STORE, 94174841700352, 94174841835519, +STORE, 140173257838592, 140173259497471, +STORE, 140173259497472, 140173261594623, +STORE, 140173261594624, 140173261611007, +STORE, 140173261611008, 140173261619199, +STORE, 140173261619200, 140173261635583, +STORE, 140173261635584, 140173261778943, +STORE, 140173263863808, 140173263871999, +STORE, 140173263876096, 140173263880191, +STORE, 140173263880192, 140173263884287, +STORE, 140173263884288, 140173263888383, +STORE, 140729801007104, 140729801142271, +STORE, 140729801617408, 140729801629695, +STORE, 140729801629696, 140729801633791, +STORE, 140737488347136, 140737488351231, +STORE, 140728166858752, 140737488351231, +SNULL, 140728166862847, 140737488351231, +STORE, 140728166858752, 140728166862847, +STORE, 140728166727680, 140728166862847, +STORE, 93912949866496, 93912950337535, +SNULL, 93912950288383, 93912950337535, +STORE, 93912949866496, 93912950288383, +STORE, 93912950288384, 93912950337535, +ERASE, 93912950288384, 93912950337535, +STORE, 93912950292480, 93912950337535, +STORE, 139921863385088, 139921865637887, +SNULL, 139921863528447, 139921865637887, +STORE, 139921863385088, 139921863528447, +STORE, 139921863528448, 139921865637887, +ERASE, 139921863528448, 139921865637887, +STORE, 139921865625600, 139921865633791, +STORE, 139921865633792, 139921865637887, +STORE, 140728167899136, 140728167903231, +STORE, 140728167886848, 140728167899135, +STORE, 139921865601024, 139921865625599, +STORE, 139921865592832, 139921865601023, +STORE, 139921861251072, 139921863385087, +SNULL, 139921861251072, 139921861279743, +STORE, 139921861279744, 139921863385087, +STORE, 139921861251072, 139921861279743, +SNULL, 139921863376895, 139921863385087, +STORE, 139921861279744, 139921863376895, +STORE, 139921863376896, 139921863385087, +ERASE, 139921863376896, 139921863385087, +STORE, 139921863376896, 139921863385087, +STORE, 139921858867200, 139921861251071, +SNULL, 139921858867200, 139921859133439, +STORE, 139921859133440, 139921861251071, +STORE, 139921858867200, 139921859133439, +SNULL, 139921861226495, 139921861251071, +STORE, 139921859133440, 139921861226495, +STORE, 139921861226496, 139921861251071, +SNULL, 139921861226496, 139921861246975, +STORE, 139921861246976, 139921861251071, +STORE, 139921861226496, 139921861246975, +ERASE, 139921861226496, 139921861246975, +STORE, 139921861226496, 139921861246975, +ERASE, 139921861246976, 139921861251071, +STORE, 139921861246976, 139921861251071, +STORE, 139921856675840, 139921858867199, +SNULL, 139921856675840, 139921856765951, +STORE, 139921856765952, 139921858867199, +STORE, 139921856675840, 139921856765951, +SNULL, 139921858859007, 139921858867199, +STORE, 139921856765952, 139921858859007, +STORE, 139921858859008, 139921858867199, +ERASE, 139921858859008, 139921858867199, +STORE, 139921858859008, 139921858867199, +STORE, 139921854414848, 139921856675839, +SNULL, 139921854414848, 139921854566399, +STORE, 139921854566400, 139921856675839, +STORE, 139921854414848, 139921854566399, +SNULL, 139921856659455, 139921856675839, +STORE, 139921854566400, 139921856659455, +STORE, 139921856659456, 139921856675839, +SNULL, 139921856659456, 139921856667647, +STORE, 139921856667648, 139921856675839, +STORE, 139921856659456, 139921856667647, +ERASE, 139921856659456, 139921856667647, +STORE, 139921856659456, 139921856667647, +ERASE, 139921856667648, 139921856675839, +STORE, 139921856667648, 139921856675839, +STORE, 139921852284928, 139921854414847, +SNULL, 139921852284928, 139921852313599, +STORE, 139921852313600, 139921854414847, +STORE, 139921852284928, 139921852313599, +SNULL, 139921854406655, 139921854414847, +STORE, 139921852313600, 139921854406655, +STORE, 139921854406656, 139921854414847, +ERASE, 139921854406656, 139921854414847, +STORE, 139921854406656, 139921854414847, +STORE, 139921850068992, 139921852284927, +SNULL, 139921850068992, 139921850167295, +STORE, 139921850167296, 139921852284927, +STORE, 139921850068992, 139921850167295, +SNULL, 139921852260351, 139921852284927, +STORE, 139921850167296, 139921852260351, +STORE, 139921852260352, 139921852284927, +SNULL, 139921852260352, 139921852268543, +STORE, 139921852268544, 139921852284927, +STORE, 139921852260352, 139921852268543, +ERASE, 139921852260352, 139921852268543, +STORE, 139921852260352, 139921852268543, +ERASE, 139921852268544, 139921852284927, +STORE, 139921852268544, 139921852284927, +STORE, 139921865584640, 139921865601023, +STORE, 139921846272000, 139921850068991, +SNULL, 139921846272000, 139921847930879, +STORE, 139921847930880, 139921850068991, +STORE, 139921846272000, 139921847930879, +SNULL, 139921850028031, 139921850068991, +STORE, 139921847930880, 139921850028031, +STORE, 139921850028032, 139921850068991, +SNULL, 139921850028032, 139921850052607, +STORE, 139921850052608, 139921850068991, +STORE, 139921850028032, 139921850052607, +ERASE, 139921850028032, 139921850052607, +STORE, 139921850028032, 139921850052607, +ERASE, 139921850052608, 139921850068991, +STORE, 139921850052608, 139921850068991, +STORE, 139921844154368, 139921846271999, +SNULL, 139921844154368, 139921844170751, +STORE, 139921844170752, 139921846271999, +STORE, 139921844154368, 139921844170751, +SNULL, 139921846263807, 139921846271999, +STORE, 139921844170752, 139921846263807, +STORE, 139921846263808, 139921846271999, +ERASE, 139921846263808, 139921846271999, +STORE, 139921846263808, 139921846271999, +STORE, 139921842036736, 139921844154367, +SNULL, 139921842036736, 139921842053119, +STORE, 139921842053120, 139921844154367, +STORE, 139921842036736, 139921842053119, +SNULL, 139921844146175, 139921844154367, +STORE, 139921842053120, 139921844146175, +STORE, 139921844146176, 139921844154367, +ERASE, 139921844146176, 139921844154367, +STORE, 139921844146176, 139921844154367, +STORE, 139921839468544, 139921842036735, +SNULL, 139921839468544, 139921839935487, +STORE, 139921839935488, 139921842036735, +STORE, 139921839468544, 139921839935487, +SNULL, 139921842028543, 139921842036735, +STORE, 139921839935488, 139921842028543, +STORE, 139921842028544, 139921842036735, +ERASE, 139921842028544, 139921842036735, +STORE, 139921842028544, 139921842036735, +STORE, 139921837355008, 139921839468543, +SNULL, 139921837355008, 139921837367295, +STORE, 139921837367296, 139921839468543, +STORE, 139921837355008, 139921837367295, +SNULL, 139921839460351, 139921839468543, +STORE, 139921837367296, 139921839460351, +STORE, 139921839460352, 139921839468543, +ERASE, 139921839460352, 139921839468543, +STORE, 139921839460352, 139921839468543, +STORE, 139921865576448, 139921865601023, +STORE, 139921865564160, 139921865601023, +SNULL, 139921850044415, 139921850052607, +STORE, 139921850028032, 139921850044415, +STORE, 139921850044416, 139921850052607, +SNULL, 139921839464447, 139921839468543, +STORE, 139921839460352, 139921839464447, +STORE, 139921839464448, 139921839468543, +SNULL, 139921852264447, 139921852268543, +STORE, 139921852260352, 139921852264447, +STORE, 139921852264448, 139921852268543, +SNULL, 139921842032639, 139921842036735, +STORE, 139921842028544, 139921842032639, +STORE, 139921842032640, 139921842036735, +SNULL, 139921844150271, 139921844154367, +STORE, 139921844146176, 139921844150271, +STORE, 139921844150272, 139921844154367, +SNULL, 139921846267903, 139921846271999, +STORE, 139921846263808, 139921846267903, +STORE, 139921846267904, 139921846271999, +SNULL, 139921854410751, 139921854414847, +STORE, 139921854406656, 139921854410751, +STORE, 139921854410752, 139921854414847, +SNULL, 139921856663551, 139921856667647, +STORE, 139921856659456, 139921856663551, +STORE, 139921856663552, 139921856667647, +SNULL, 139921858863103, 139921858867199, +STORE, 139921858859008, 139921858863103, +STORE, 139921858863104, 139921858867199, +SNULL, 139921861242879, 139921861246975, +STORE, 139921861226496, 139921861242879, +STORE, 139921861242880, 139921861246975, +SNULL, 139921863380991, 139921863385087, +STORE, 139921863376896, 139921863380991, +STORE, 139921863380992, 139921863385087, +SNULL, 93912950333439, 93912950337535, +STORE, 93912950292480, 93912950333439, +STORE, 93912950333440, 93912950337535, +SNULL, 139921865629695, 139921865633791, +STORE, 139921865625600, 139921865629695, +STORE, 139921865629696, 139921865633791, +ERASE, 139921865601024, 139921865625599, +STORE, 93912968110080, 93912968245247, +STORE, 139921828913152, 139921837355007, +STORE, 139921865621504, 139921865625599, +STORE, 139921865617408, 139921865621503, +STORE, 139921865613312, 139921865617407, +STORE, 139921865547776, 139921865564159, + }; + + unsigned long set17[] = { +STORE, 94397057224704, 94397057646591, +STORE, 94397057650688, 94397057691647, +STORE, 94397057691648, 94397057695743, +STORE, 94397075271680, 94397075406847, +STORE, 139953169051648, 139953169063935, +STORE, 139953169063936, 139953171156991, +STORE, 139953171156992, 139953171161087, +STORE, 139953171161088, 139953171165183, +STORE, 139953171165184, 139953171632127, +STORE, 139953171632128, 139953173725183, +STORE, 139953173725184, 139953173729279, +STORE, 139953173729280, 139953173733375, +STORE, 139953173733376, 139953173749759, +STORE, 139953173749760, 139953175842815, +STORE, 139953175842816, 139953175846911, +STORE, 139953175846912, 139953175851007, +STORE, 139953175851008, 139953175867391, +STORE, 139953175867392, 139953177960447, +STORE, 139953177960448, 139953177964543, +STORE, 139953177964544, 139953177968639, +STORE, 139953177968640, 139953179627519, +STORE, 139953179627520, 139953181724671, +STORE, 139953181724672, 139953181741055, +STORE, 139953181741056, 139953181749247, +STORE, 139953181749248, 139953181765631, +STORE, 139953181765632, 139953181863935, +STORE, 139953181863936, 139953183956991, +STORE, 139953183956992, 139953183961087, +STORE, 139953183961088, 139953183965183, +STORE, 139953183965184, 139953183981567, +STORE, 139953183981568, 139953184010239, +STORE, 139953184010240, 139953186103295, +STORE, 139953186103296, 139953186107391, +STORE, 139953186107392, 139953186111487, +STORE, 139953186111488, 139953186263039, +STORE, 139953186263040, 139953188356095, +STORE, 139953188356096, 139953188360191, +STORE, 139953188360192, 139953188364287, +STORE, 139953188364288, 139953188372479, +STORE, 139953188372480, 139953188462591, +STORE, 139953188462592, 139953190555647, +STORE, 139953190555648, 139953190559743, +STORE, 139953190559744, 139953190563839, +STORE, 139953190563840, 139953190830079, +STORE, 139953190830080, 139953192923135, +STORE, 139953192923136, 139953192939519, +STORE, 139953192939520, 139953192943615, +STORE, 139953192943616, 139953192947711, +STORE, 139953192947712, 139953192976383, +STORE, 139953192976384, 139953195073535, +STORE, 139953195073536, 139953195077631, +STORE, 139953195077632, 139953195081727, +STORE, 139953195081728, 139953195225087, +STORE, 139953197281280, 139953197318143, +STORE, 139953197322240, 139953197326335, +STORE, 139953197326336, 139953197330431, +STORE, 139953197330432, 139953197334527, +STORE, 140720477511680, 140720477646847, +STORE, 140720478302208, 140720478314495, +STORE, 140720478314496, 140720478318591, + }; + unsigned long set18[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140724953673728, 140737488351231, +SNULL, 140724953677823, 140737488351231, +STORE, 140724953673728, 140724953677823, +STORE, 140724953542656, 140724953677823, +STORE, 94675199266816, 94675199311871, +SNULL, 94675199303679, 94675199311871, +STORE, 94675199266816, 94675199303679, +STORE, 94675199303680, 94675199311871, +ERASE, 94675199303680, 94675199311871, +STORE, 94675199303680, 94675199311871, +STORE, 140222970605568, 140222972858367, +SNULL, 140222970748927, 140222972858367, +STORE, 140222970605568, 140222970748927, +STORE, 140222970748928, 140222972858367, +ERASE, 140222970748928, 140222972858367, +STORE, 140222972846080, 140222972854271, +STORE, 140222972854272, 140222972858367, +STORE, 140724954365952, 140724954370047, +STORE, 140724954353664, 140724954365951, +STORE, 140222972841984, 140222972846079, +STORE, 140222972833792, 140222972841983, +STORE, 140222968475648, 140222970605567, +SNULL, 140222968475648, 140222968504319, +STORE, 140222968504320, 140222970605567, +STORE, 140222968475648, 140222968504319, +SNULL, 140222970597375, 140222970605567, +STORE, 140222968504320, 140222970597375, +STORE, 140222970597376, 140222970605567, +ERASE, 140222970597376, 140222970605567, +STORE, 140222970597376, 140222970605567, + }; + unsigned long set19[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140725182459904, 140737488351231, +SNULL, 140725182463999, 140737488351231, +STORE, 140725182459904, 140725182463999, +STORE, 140725182328832, 140725182463999, +STORE, 94730166636544, 94730166763519, +SNULL, 94730166747135, 94730166763519, +STORE, 94730166636544, 94730166747135, +STORE, 94730166747136, 94730166763519, +ERASE, 94730166747136, 94730166763519, +STORE, 94730166751232, 94730166763519, +STORE, 140656834555904, 140656836808703, +SNULL, 140656834699263, 140656836808703, +STORE, 140656834555904, 140656834699263, +STORE, 140656834699264, 140656836808703, +ERASE, 140656834699264, 140656836808703, +STORE, 140656836796416, 140656836804607, +STORE, 140656836804608, 140656836808703, +STORE, 140725183389696, 140725183393791, +STORE, 140725183377408, 140725183389695, +STORE, 140656836788224, 140656836796415, +STORE, 140656832331776, 140656834555903, +SNULL, 140656833982463, 140656834555903, +STORE, 140656832331776, 140656833982463, +STORE, 140656833982464, 140656834555903, +SNULL, 140656833982464, 140656834551807, +STORE, 140656834551808, 140656834555903, +STORE, 140656833982464, 140656834551807, +ERASE, 140656833982464, 140656834551807, +STORE, 140656833982464, 140656834551807, +ERASE, 140656834551808, 140656834555903, +STORE, 140656834551808, 140656834555903, +STORE, 140656836763648, 140656836788223, +STORE, 140656830070784, 140656832331775, +SNULL, 140656830070784, 140656830222335, +STORE, 140656830222336, 140656832331775, +STORE, 140656830070784, 140656830222335, +SNULL, 140656832315391, 140656832331775, +STORE, 140656830222336, 140656832315391, +STORE, 140656832315392, 140656832331775, +SNULL, 140656832315392, 140656832323583, +STORE, 140656832323584, 140656832331775, +STORE, 140656832315392, 140656832323583, +ERASE, 140656832315392, 140656832323583, +STORE, 140656832315392, 140656832323583, +ERASE, 140656832323584, 140656832331775, +STORE, 140656832323584, 140656832331775, +STORE, 140656827940864, 140656830070783, +SNULL, 140656827940864, 140656827969535, +STORE, 140656827969536, 140656830070783, +STORE, 140656827940864, 140656827969535, +SNULL, 140656830062591, 140656830070783, +STORE, 140656827969536, 140656830062591, +STORE, 140656830062592, 140656830070783, +ERASE, 140656830062592, 140656830070783, +STORE, 140656830062592, 140656830070783, +STORE, 140656825724928, 140656827940863, +SNULL, 140656825724928, 140656825823231, +STORE, 140656825823232, 140656827940863, +STORE, 140656825724928, 140656825823231, +SNULL, 140656827916287, 140656827940863, +STORE, 140656825823232, 140656827916287, +STORE, 140656827916288, 140656827940863, +SNULL, 140656827916288, 140656827924479, +STORE, 140656827924480, 140656827940863, +STORE, 140656827916288, 140656827924479, +ERASE, 140656827916288, 140656827924479, +STORE, 140656827916288, 140656827924479, +ERASE, 140656827924480, 140656827940863, +STORE, 140656827924480, 140656827940863, +STORE, 140656821927936, 140656825724927, +SNULL, 140656821927936, 140656823586815, +STORE, 140656823586816, 140656825724927, +STORE, 140656821927936, 140656823586815, +SNULL, 140656825683967, 140656825724927, +STORE, 140656823586816, 140656825683967, +STORE, 140656825683968, 140656825724927, +SNULL, 140656825683968, 140656825708543, +STORE, 140656825708544, 140656825724927, +STORE, 140656825683968, 140656825708543, +ERASE, 140656825683968, 140656825708543, +STORE, 140656825683968, 140656825708543, +ERASE, 140656825708544, 140656825724927, +STORE, 140656825708544, 140656825724927, +STORE, 140656819806208, 140656821927935, +SNULL, 140656819806208, 140656819822591, +STORE, 140656819822592, 140656821927935, +STORE, 140656819806208, 140656819822591, +SNULL, 140656821919743, 140656821927935, +STORE, 140656819822592, 140656821919743, +STORE, 140656821919744, 140656821927935, +ERASE, 140656821919744, 140656821927935, +STORE, 140656821919744, 140656821927935, +STORE, 140656836755456, 140656836763647, +STORE, 140656817553408, 140656819806207, +SNULL, 140656817553408, 140656817704959, +STORE, 140656817704960, 140656819806207, +STORE, 140656817553408, 140656817704959, +SNULL, 140656819798015, 140656819806207, +STORE, 140656817704960, 140656819798015, +STORE, 140656819798016, 140656819806207, +ERASE, 140656819798016, 140656819806207, +STORE, 140656819798016, 140656819806207, +STORE, 140656815382528, 140656817553407, +SNULL, 140656815382528, 140656815452159, +STORE, 140656815452160, 140656817553407, +STORE, 140656815382528, 140656815452159, +SNULL, 140656817545215, 140656817553407, +STORE, 140656815452160, 140656817545215, +STORE, 140656817545216, 140656817553407, +ERASE, 140656817545216, 140656817553407, +STORE, 140656817545216, 140656817553407, +STORE, 140656812171264, 140656815382527, +SNULL, 140656812171264, 140656813248511, +STORE, 140656813248512, 140656815382527, +STORE, 140656812171264, 140656813248511, +SNULL, 140656815345663, 140656815382527, +STORE, 140656813248512, 140656815345663, +STORE, 140656815345664, 140656815382527, +ERASE, 140656815345664, 140656815382527, +STORE, 140656815345664, 140656815382527, +STORE, 140656810037248, 140656812171263, +SNULL, 140656810037248, 140656810065919, +STORE, 140656810065920, 140656812171263, +STORE, 140656810037248, 140656810065919, +SNULL, 140656812163071, 140656812171263, +STORE, 140656810065920, 140656812163071, +STORE, 140656812163072, 140656812171263, +ERASE, 140656812163072, 140656812171263, +STORE, 140656812163072, 140656812171263, +STORE, 140656807727104, 140656810037247, +SNULL, 140656807727104, 140656807931903, +STORE, 140656807931904, 140656810037247, +STORE, 140656807727104, 140656807931903, +SNULL, 140656810029055, 140656810037247, +STORE, 140656807931904, 140656810029055, +STORE, 140656810029056, 140656810037247, +ERASE, 140656810029056, 140656810037247, +STORE, 140656810029056, 140656810037247, +STORE, 140656805343232, 140656807727103, +SNULL, 140656805343232, 140656805535743, +STORE, 140656805535744, 140656807727103, +STORE, 140656805343232, 140656805535743, +SNULL, 140656807628799, 140656807727103, +STORE, 140656805535744, 140656807628799, +STORE, 140656807628800, 140656807727103, +ERASE, 140656807628800, 140656807727103, +STORE, 140656807628800, 140656807727103, +STORE, 140656836747264, 140656836763647, +STORE, 140656802775040, 140656805343231, +SNULL, 140656802775040, 140656803241983, +STORE, 140656803241984, 140656805343231, +STORE, 140656802775040, 140656803241983, +SNULL, 140656805335039, 140656805343231, +STORE, 140656803241984, 140656805335039, +STORE, 140656805335040, 140656805343231, +ERASE, 140656805335040, 140656805343231, +STORE, 140656805335040, 140656805343231, +STORE, 140656800661504, 140656802775039, +SNULL, 140656800661504, 140656800673791, +STORE, 140656800673792, 140656802775039, +STORE, 140656800661504, 140656800673791, +SNULL, 140656802766847, 140656802775039, +STORE, 140656800673792, 140656802766847, +STORE, 140656802766848, 140656802775039, +ERASE, 140656802766848, 140656802775039, +STORE, 140656802766848, 140656802775039, +STORE, 140656798482432, 140656800661503, +SNULL, 140656798482432, 140656798560255, +STORE, 140656798560256, 140656800661503, +STORE, 140656798482432, 140656798560255, +SNULL, 140656800653311, 140656800661503, +STORE, 140656798560256, 140656800653311, +STORE, 140656800653312, 140656800661503, +ERASE, 140656800653312, 140656800661503, +STORE, 140656800653312, 140656800661503, +STORE, 140656796364800, 140656798482431, +SNULL, 140656796364800, 140656796381183, +STORE, 140656796381184, 140656798482431, +STORE, 140656796364800, 140656796381183, +SNULL, 140656798474239, 140656798482431, +STORE, 140656796381184, 140656798474239, +STORE, 140656798474240, 140656798482431, +ERASE, 140656798474240, 140656798482431, +STORE, 140656798474240, 140656798482431, +STORE, 140656836739072, 140656836763647, +STORE, 140656836726784, 140656836763647, +SNULL, 140656825700351, 140656825708543, +STORE, 140656825683968, 140656825700351, +STORE, 140656825700352, 140656825708543, +SNULL, 140656798478335, 140656798482431, +STORE, 140656798474240, 140656798478335, +STORE, 140656798478336, 140656798482431, +SNULL, 140656800657407, 140656800661503, +STORE, 140656800653312, 140656800657407, +STORE, 140656800657408, 140656800661503, +SNULL, 140656802770943, 140656802775039, +STORE, 140656802766848, 140656802770943, +STORE, 140656802770944, 140656802775039, +SNULL, 140656827920383, 140656827924479, +STORE, 140656827916288, 140656827920383, +STORE, 140656827920384, 140656827924479, +SNULL, 140656805339135, 140656805343231, +STORE, 140656805335040, 140656805339135, +STORE, 140656805339136, 140656805343231, +SNULL, 140656807723007, 140656807727103, +STORE, 140656807628800, 140656807723007, +STORE, 140656807723008, 140656807727103, +SNULL, 140656810033151, 140656810037247, +STORE, 140656810029056, 140656810033151, +STORE, 140656810033152, 140656810037247, +SNULL, 140656812167167, 140656812171263, +STORE, 140656812163072, 140656812167167, +STORE, 140656812167168, 140656812171263, +SNULL, 140656815353855, 140656815382527, +STORE, 140656815345664, 140656815353855, +STORE, 140656815353856, 140656815382527, +SNULL, 140656817549311, 140656817553407, +STORE, 140656817545216, 140656817549311, +STORE, 140656817549312, 140656817553407, +SNULL, 140656819802111, 140656819806207, +STORE, 140656819798016, 140656819802111, +STORE, 140656819802112, 140656819806207, +SNULL, 140656821923839, 140656821927935, +STORE, 140656821919744, 140656821923839, +STORE, 140656821923840, 140656821927935, +SNULL, 140656830066687, 140656830070783, +STORE, 140656830062592, 140656830066687, +STORE, 140656830066688, 140656830070783, +SNULL, 140656832319487, 140656832323583, +STORE, 140656832315392, 140656832319487, +STORE, 140656832319488, 140656832323583, +SNULL, 140656834547711, 140656834551807, +STORE, 140656833982464, 140656834547711, +STORE, 140656834547712, 140656834551807, +SNULL, 94730166759423, 94730166763519, +STORE, 94730166751232, 94730166759423, +STORE, 94730166759424, 94730166763519, +SNULL, 140656836800511, 140656836804607, +STORE, 140656836796416, 140656836800511, +STORE, 140656836800512, 140656836804607, +ERASE, 140656836763648, 140656836788223, +STORE, 94730171318272, 94730171453439, +STORE, 140656836784128, 140656836788223, +STORE, 140656836780032, 140656836784127, +STORE, 140656791920640, 140656796364799, +STORE, 140656836775936, 140656836780031, +STORE, 140656787476480, 140656791920639, +STORE, 140656779083776, 140656787476479, +SNULL, 140656779087871, 140656787476479, +STORE, 140656779083776, 140656779087871, +STORE, 140656779087872, 140656787476479, +STORE, 140656836771840, 140656836775935, +STORE, 140656774639616, 140656779083775, +STORE, 140656766246912, 140656774639615, +SNULL, 140656766251007, 140656774639615, +STORE, 140656766246912, 140656766251007, +STORE, 140656766251008, 140656774639615, +ERASE, 140656791920640, 140656796364799, +ERASE, 140656836780032, 140656836784127, +ERASE, 140656787476480, 140656791920639, +ERASE, 140656836775936, 140656836780031, +STORE, 140656836780032, 140656836784127, +STORE, 140656791920640, 140656796364799, +STORE, 140656836775936, 140656836780031, +STORE, 140656787476480, 140656791920639, +ERASE, 140656774639616, 140656779083775, + }; + unsigned long set20[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735952392192, 140737488351231, +SNULL, 140735952396287, 140737488351231, +STORE, 140735952392192, 140735952396287, +STORE, 140735952261120, 140735952396287, +STORE, 94849008947200, 94849009414143, +SNULL, 94849009364991, 94849009414143, +STORE, 94849008947200, 94849009364991, +STORE, 94849009364992, 94849009414143, +ERASE, 94849009364992, 94849009414143, +STORE, 94849009364992, 94849009414143, +STORE, 140590397943808, 140590400196607, +SNULL, 140590398087167, 140590400196607, +STORE, 140590397943808, 140590398087167, +STORE, 140590398087168, 140590400196607, +ERASE, 140590398087168, 140590400196607, +STORE, 140590400184320, 140590400192511, +STORE, 140590400192512, 140590400196607, +STORE, 140735952850944, 140735952855039, +STORE, 140735952838656, 140735952850943, +STORE, 140590400180224, 140590400184319, +STORE, 140590400172032, 140590400180223, +STORE, 140590395809792, 140590397943807, +SNULL, 140590395809792, 140590395838463, +STORE, 140590395838464, 140590397943807, +STORE, 140590395809792, 140590395838463, +SNULL, 140590397935615, 140590397943807, +STORE, 140590395838464, 140590397935615, +STORE, 140590397935616, 140590397943807, +ERASE, 140590397935616, 140590397943807, +STORE, 140590397935616, 140590397943807, +STORE, 140590393425920, 140590395809791, +SNULL, 140590393425920, 140590393692159, +STORE, 140590393692160, 140590395809791, +STORE, 140590393425920, 140590393692159, +SNULL, 140590395785215, 140590395809791, +STORE, 140590393692160, 140590395785215, +STORE, 140590395785216, 140590395809791, +SNULL, 140590395785216, 140590395805695, +STORE, 140590395805696, 140590395809791, +STORE, 140590395785216, 140590395805695, +ERASE, 140590395785216, 140590395805695, +STORE, 140590395785216, 140590395805695, +ERASE, 140590395805696, 140590395809791, +STORE, 140590395805696, 140590395809791, +STORE, 140590391234560, 140590393425919, +SNULL, 140590391234560, 140590391324671, +STORE, 140590391324672, 140590393425919, +STORE, 140590391234560, 140590391324671, +SNULL, 140590393417727, 140590393425919, +STORE, 140590391324672, 140590393417727, +STORE, 140590393417728, 140590393425919, +ERASE, 140590393417728, 140590393425919, +STORE, 140590393417728, 140590393425919, +STORE, 140590388973568, 140590391234559, +SNULL, 140590388973568, 140590389125119, +STORE, 140590389125120, 140590391234559, +STORE, 140590388973568, 140590389125119, +SNULL, 140590391218175, 140590391234559, +STORE, 140590389125120, 140590391218175, +STORE, 140590391218176, 140590391234559, +SNULL, 140590391218176, 140590391226367, +STORE, 140590391226368, 140590391234559, +STORE, 140590391218176, 140590391226367, +ERASE, 140590391218176, 140590391226367, +STORE, 140590391218176, 140590391226367, +ERASE, 140590391226368, 140590391234559, +STORE, 140590391226368, 140590391234559, +STORE, 140590386843648, 140590388973567, +SNULL, 140590386843648, 140590386872319, +STORE, 140590386872320, 140590388973567, +STORE, 140590386843648, 140590386872319, +SNULL, 140590388965375, 140590388973567, +STORE, 140590386872320, 140590388965375, +STORE, 140590388965376, 140590388973567, +ERASE, 140590388965376, 140590388973567, +STORE, 140590388965376, 140590388973567, +STORE, 140590384627712, 140590386843647, +SNULL, 140590384627712, 140590384726015, +STORE, 140590384726016, 140590386843647, +STORE, 140590384627712, 140590384726015, +SNULL, 140590386819071, 140590386843647, +STORE, 140590384726016, 140590386819071, +STORE, 140590386819072, 140590386843647, +SNULL, 140590386819072, 140590386827263, +STORE, 140590386827264, 140590386843647, +STORE, 140590386819072, 140590386827263, +ERASE, 140590386819072, 140590386827263, +STORE, 140590386819072, 140590386827263, +ERASE, 140590386827264, 140590386843647, +STORE, 140590386827264, 140590386843647, +STORE, 140590400163840, 140590400180223, +STORE, 140590380830720, 140590384627711, +SNULL, 140590380830720, 140590382489599, +STORE, 140590382489600, 140590384627711, +STORE, 140590380830720, 140590382489599, +SNULL, 140590384586751, 140590384627711, +STORE, 140590382489600, 140590384586751, +STORE, 140590384586752, 140590384627711, +SNULL, 140590384586752, 140590384611327, +STORE, 140590384611328, 140590384627711, +STORE, 140590384586752, 140590384611327, +ERASE, 140590384586752, 140590384611327, +STORE, 140590384586752, 140590384611327, +ERASE, 140590384611328, 140590384627711, +STORE, 140590384611328, 140590384627711, +STORE, 140590378713088, 140590380830719, +SNULL, 140590378713088, 140590378729471, +STORE, 140590378729472, 140590380830719, +STORE, 140590378713088, 140590378729471, +SNULL, 140590380822527, 140590380830719, +STORE, 140590378729472, 140590380822527, +STORE, 140590380822528, 140590380830719, +ERASE, 140590380822528, 140590380830719, +STORE, 140590380822528, 140590380830719, +STORE, 140590376595456, 140590378713087, +SNULL, 140590376595456, 140590376611839, +STORE, 140590376611840, 140590378713087, +STORE, 140590376595456, 140590376611839, +SNULL, 140590378704895, 140590378713087, +STORE, 140590376611840, 140590378704895, +STORE, 140590378704896, 140590378713087, +ERASE, 140590378704896, 140590378713087, +STORE, 140590378704896, 140590378713087, +STORE, 140590374027264, 140590376595455, +SNULL, 140590374027264, 140590374494207, +STORE, 140590374494208, 140590376595455, +STORE, 140590374027264, 140590374494207, +SNULL, 140590376587263, 140590376595455, +STORE, 140590374494208, 140590376587263, +STORE, 140590376587264, 140590376595455, +ERASE, 140590376587264, 140590376595455, +STORE, 140590376587264, 140590376595455, +STORE, 140590371913728, 140590374027263, +SNULL, 140590371913728, 140590371926015, +STORE, 140590371926016, 140590374027263, +STORE, 140590371913728, 140590371926015, +SNULL, 140590374019071, 140590374027263, +STORE, 140590371926016, 140590374019071, +STORE, 140590374019072, 140590374027263, +ERASE, 140590374019072, 140590374027263, +STORE, 140590374019072, 140590374027263, +STORE, 140590400155648, 140590400180223, +STORE, 140590400143360, 140590400180223, +SNULL, 140590384603135, 140590384611327, +STORE, 140590384586752, 140590384603135, +STORE, 140590384603136, 140590384611327, +SNULL, 140590374023167, 140590374027263, +STORE, 140590374019072, 140590374023167, +STORE, 140590374023168, 140590374027263, +SNULL, 140590386823167, 140590386827263, +STORE, 140590386819072, 140590386823167, +STORE, 140590386823168, 140590386827263, +SNULL, 140590376591359, 140590376595455, + }; + unsigned long set21[] = { +STORE, 93874710941696, 93874711363583, +STORE, 93874711367680, 93874711408639, +STORE, 93874711408640, 93874711412735, +STORE, 93874720989184, 93874721124351, +STORE, 140708365086720, 140708365099007, +STORE, 140708365099008, 140708367192063, +STORE, 140708367192064, 140708367196159, +STORE, 140708367196160, 140708367200255, +STORE, 140708367200256, 140708367667199, +STORE, 140708367667200, 140708369760255, +STORE, 140708369760256, 140708369764351, +STORE, 140708369764352, 140708369768447, +STORE, 140708369768448, 140708369784831, +STORE, 140708369784832, 140708371877887, +STORE, 140708371877888, 140708371881983, +STORE, 140708371881984, 140708371886079, +STORE, 140708371886080, 140708371902463, +STORE, 140708371902464, 140708373995519, +STORE, 140708373995520, 140708373999615, +STORE, 140708373999616, 140708374003711, +STORE, 140708374003712, 140708375662591, +STORE, 140708375662592, 140708377759743, +STORE, 140708377759744, 140708377776127, +STORE, 140708377776128, 140708377784319, +STORE, 140708377784320, 140708377800703, +STORE, 140708377800704, 140708377899007, +STORE, 140708377899008, 140708379992063, +STORE, 140708379992064, 140708379996159, +STORE, 140708379996160, 140708380000255, +STORE, 140708380000256, 140708380016639, +STORE, 140708380016640, 140708380045311, +STORE, 140708380045312, 140708382138367, +STORE, 140708382138368, 140708382142463, +STORE, 140708382142464, 140708382146559, +STORE, 140708382146560, 140708382298111, +STORE, 140708382298112, 140708384391167, +STORE, 140708384391168, 140708384395263, +STORE, 140708384395264, 140708384399359, +STORE, 140708384399360, 140708384407551, +STORE, 140708384407552, 140708384497663, +STORE, 140708384497664, 140708386590719, +STORE, 140708386590720, 140708386594815, +STORE, 140708386594816, 140708386598911, +STORE, 140708386598912, 140708386865151, +STORE, 140708386865152, 140708388958207, +STORE, 140708388958208, 140708388974591, +STORE, 140708388974592, 140708388978687, +STORE, 140708388978688, 140708388982783, +STORE, 140708388982784, 140708389011455, +STORE, 140708389011456, 140708391108607, +STORE, 140708391108608, 140708391112703, +STORE, 140708391112704, 140708391116799, +STORE, 140708391116800, 140708391260159, +STORE, 140708393291776, 140708393308159, +STORE, 140708393308160, 140708393312255, +STORE, 140708393312256, 140708393316351, +STORE, 140708393316352, 140708393353215, +STORE, 140708393353216, 140708393357311, +STORE, 140708393357312, 140708393361407, +STORE, 140708393361408, 140708393365503, +STORE, 140708393365504, 140708393369599, +STORE, 140730557042688, 140730557177855, +STORE, 140730557235200, 140730557247487, +STORE, 140730557247488, 140730557251583, +ERASE, 140708393353216, 140708393357311, +ERASE, 140708393312256, 140708393316351, +ERASE, 140708393308160, 140708393312255, +ERASE, 140708393291776, 140708393308159, + }; + unsigned long set22[] = { +STORE, 93951397134336, 93951397183487, +STORE, 93951397183488, 93951397728255, +STORE, 93951397728256, 93951397826559, +STORE, 93951397826560, 93951397842943, +STORE, 93951397842944, 93951397847039, +STORE, 93951425974272, 93951426109439, +STORE, 140685152665600, 140685152677887, +STORE, 140685152677888, 140685152829439, +STORE, 140685152829440, 140685154181119, +STORE, 140685154181120, 140685154484223, +STORE, 140685154484224, 140685154496511, +STORE, 140685154496512, 140685154508799, +STORE, 140685154508800, 140685154525183, +STORE, 140685154525184, 140685154541567, +STORE, 140685154541568, 140685154590719, +STORE, 140685154590720, 140685154603007, +STORE, 140685154603008, 140685154607103, +STORE, 140685154607104, 140685154611199, +STORE, 140685154611200, 140685154615295, +STORE, 140685154615296, 140685154631679, +STORE, 140685154639872, 140685154643967, +STORE, 140685154643968, 140685154766847, +STORE, 140685154766848, 140685154799615, +STORE, 140685154803712, 140685154807807, +STORE, 140685154807808, 140685154811903, +STORE, 140685154811904, 140685154815999, +STORE, 140722188902400, 140722189037567, +STORE, 140722189512704, 140722189524991, +STORE, 140722189524992, 140722189529087, +STORE, 140737488347136, 140737488351231, +STORE, 140733429354496, 140737488351231, +SNULL, 140733429358591, 140737488351231, +STORE, 140733429354496, 140733429358591, +STORE, 140733429223424, 140733429358591, +STORE, 94526683537408, 94526683660287, +SNULL, 94526683553791, 94526683660287, +STORE, 94526683537408, 94526683553791, +STORE, 94526683553792, 94526683660287, +ERASE, 94526683553792, 94526683660287, +STORE, 94526683553792, 94526683623423, +STORE, 94526683623424, 94526683647999, +STORE, 94526683652096, 94526683660287, +STORE, 140551363747840, 140551363923967, +SNULL, 140551363751935, 140551363923967, +STORE, 140551363747840, 140551363751935, +STORE, 140551363751936, 140551363923967, +ERASE, 140551363751936, 140551363923967, +STORE, 140551363751936, 140551363874815, +STORE, 140551363874816, 140551363907583, +STORE, 140551363911680, 140551363919871, +STORE, 140551363919872, 140551363923967, +STORE, 140733429690368, 140733429694463, +STORE, 140733429678080, 140733429690367, +STORE, 140551363739648, 140551363747839, +STORE, 140551363731456, 140551363739647, +STORE, 140551363379200, 140551363731455, +SNULL, 140551363379200, 140551363420159, +STORE, 140551363420160, 140551363731455, +STORE, 140551363379200, 140551363420159, +SNULL, 140551363706879, 140551363731455, +STORE, 140551363420160, 140551363706879, +STORE, 140551363706880, 140551363731455, +SNULL, 140551363420160, 140551363637247, +STORE, 140551363637248, 140551363706879, +STORE, 140551363420160, 140551363637247, +ERASE, 140551363420160, 140551363637247, +STORE, 140551363420160, 140551363637247, +SNULL, 140551363637248, 140551363702783, +STORE, 140551363702784, 140551363706879, +STORE, 140551363637248, 140551363702783, +ERASE, 140551363637248, 140551363702783, +STORE, 140551363637248, 140551363702783, +ERASE, 140551363706880, 140551363731455, +STORE, 140551363706880, 140551363731455, +STORE, 140551361531904, 140551363379199, +SNULL, 140551361683455, 140551363379199, +STORE, 140551361531904, 140551361683455, +STORE, 140551361683456, 140551363379199, +SNULL, 140551361683456, 140551363035135, +STORE, 140551363035136, 140551363379199, +STORE, 140551361683456, 140551363035135, +ERASE, 140551361683456, 140551363035135, +STORE, 140551361683456, 140551363035135, +SNULL, 140551363035136, 140551363338239, +STORE, 140551363338240, 140551363379199, +STORE, 140551363035136, 140551363338239, +ERASE, 140551363035136, 140551363338239, +STORE, 140551363035136, 140551363379199, +SNULL, 140551363338239, 140551363379199, +STORE, 140551363035136, 140551363338239, +STORE, 140551363338240, 140551363379199, +SNULL, 140551363338240, 140551363362815, +STORE, 140551363362816, 140551363379199, +STORE, 140551363338240, 140551363362815, +ERASE, 140551363338240, 140551363362815, +STORE, 140551363338240, 140551363362815, +ERASE, 140551363362816, 140551363379199, +STORE, 140551363362816, 140551363379199, +STORE, 140551361519616, 140551361531903, +SNULL, 140551363350527, 140551363362815, +STORE, 140551363338240, 140551363350527, +STORE, 140551363350528, 140551363362815, +SNULL, 140551363727359, 140551363731455, +STORE, 140551363706880, 140551363727359, +STORE, 140551363727360, 140551363731455, +SNULL, 94526683656191, 94526683660287, +STORE, 94526683652096, 94526683656191, +STORE, 94526683656192, 94526683660287, +SNULL, 140551363915775, 140551363919871, +STORE, 140551363911680, 140551363915775, +STORE, 140551363915776, 140551363919871, +ERASE, 140551363739648, 140551363747839, +STORE, 94526715490304, 94526715625471, +STORE, 140551361253376, 140551361531903, +STORE, 140551360987136, 140551361531903, +STORE, 140551360720896, 140551361531903, +STORE, 140551360454656, 140551361531903, +SNULL, 140551361253375, 140551361531903, +STORE, 140551360454656, 140551361253375, +STORE, 140551361253376, 140551361531903, +SNULL, 140551361253376, 140551361519615, +STORE, 140551361519616, 140551361531903, +STORE, 140551361253376, 140551361519615, +ERASE, 140551361253376, 140551361519615, + }; + + unsigned long set23[] = { +STORE, 94014447943680, 94014448156671, +STORE, 94014450253824, 94014450257919, +STORE, 94014450257920, 94014450266111, +STORE, 94014450266112, 94014450278399, +STORE, 94014464225280, 94014464630783, +STORE, 139761764306944, 139761765965823, +STORE, 139761765965824, 139761768062975, +STORE, 139761768062976, 139761768079359, +STORE, 139761768079360, 139761768087551, +STORE, 139761768087552, 139761768103935, +STORE, 139761768103936, 139761768116223, +STORE, 139761768116224, 139761770209279, +STORE, 139761770209280, 139761770213375, +STORE, 139761770213376, 139761770217471, +STORE, 139761770217472, 139761770360831, +STORE, 139761770729472, 139761772412927, +STORE, 139761772412928, 139761772429311, +STORE, 139761772457984, 139761772462079, +STORE, 139761772462080, 139761772466175, +STORE, 139761772466176, 139761772470271, +STORE, 140724336517120, 140724336652287, +STORE, 140724336955392, 140724336967679, +STORE, 140724336967680, 140724336971775, +STORE, 140737488347136, 140737488351231, +STORE, 140721840295936, 140737488351231, +SNULL, 140721840300031, 140737488351231, +STORE, 140721840295936, 140721840300031, +STORE, 140721840164864, 140721840300031, +STORE, 93937913667584, 93937915830271, +SNULL, 93937913729023, 93937915830271, +STORE, 93937913667584, 93937913729023, +STORE, 93937913729024, 93937915830271, +ERASE, 93937913729024, 93937915830271, +STORE, 93937915822080, 93937915830271, +STORE, 140598835335168, 140598837587967, +SNULL, 140598835478527, 140598837587967, +STORE, 140598835335168, 140598835478527, +STORE, 140598835478528, 140598837587967, +ERASE, 140598835478528, 140598837587967, +STORE, 140598837575680, 140598837583871, +STORE, 140598837583872, 140598837587967, +STORE, 140721841086464, 140721841090559, +STORE, 140721841074176, 140721841086463, +STORE, 140598837547008, 140598837575679, +STORE, 140598837538816, 140598837547007, +STORE, 140598831538176, 140598835335167, +SNULL, 140598831538176, 140598833197055, +STORE, 140598833197056, 140598835335167, +STORE, 140598831538176, 140598833197055, +SNULL, 140598835294207, 140598835335167, +STORE, 140598833197056, 140598835294207, +STORE, 140598835294208, 140598835335167, +SNULL, 140598835294208, 140598835318783, +STORE, 140598835318784, 140598835335167, +STORE, 140598835294208, 140598835318783, +ERASE, 140598835294208, 140598835318783, +STORE, 140598835294208, 140598835318783, +ERASE, 140598835318784, 140598835335167, +STORE, 140598835318784, 140598835335167, +SNULL, 140598835310591, 140598835318783, +STORE, 140598835294208, 140598835310591, +STORE, 140598835310592, 140598835318783, +SNULL, 93937915826175, 93937915830271, +STORE, 93937915822080, 93937915826175, +STORE, 93937915826176, 93937915830271, +SNULL, 140598837579775, 140598837583871, +STORE, 140598837575680, 140598837579775, +STORE, 140598837579776, 140598837583871, +ERASE, 140598837547008, 140598837575679, +STORE, 93937929179136, 93937929314303, +STORE, 140598835855360, 140598837538815, +STORE, 140737488347136, 140737488351231, +STORE, 140728187723776, 140737488351231, +SNULL, 140728187727871, 140737488351231, +STORE, 140728187723776, 140728187727871, +STORE, 140728187592704, 140728187727871, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140583951437824, 140583953690623, +SNULL, 140583951581183, 140583953690623, +STORE, 140583951437824, 140583951581183, +STORE, 140583951581184, 140583953690623, +ERASE, 140583951581184, 140583953690623, +STORE, 140583953678336, 140583953686527, +STORE, 140583953686528, 140583953690623, +STORE, 140728189116416, 140728189120511, +STORE, 140728189104128, 140728189116415, +STORE, 140583953649664, 140583953678335, +STORE, 140583953641472, 140583953649663, +STORE, 140583948275712, 140583951437823, +SNULL, 140583948275712, 140583949336575, +STORE, 140583949336576, 140583951437823, +STORE, 140583948275712, 140583949336575, +SNULL, 140583951429631, 140583951437823, +STORE, 140583949336576, 140583951429631, +STORE, 140583951429632, 140583951437823, +ERASE, 140583951429632, 140583951437823, +STORE, 140583951429632, 140583951437823, +STORE, 140583944478720, 140583948275711, +SNULL, 140583944478720, 140583946137599, +STORE, 140583946137600, 140583948275711, +STORE, 140583944478720, 140583946137599, +SNULL, 140583948234751, 140583948275711, +STORE, 140583946137600, 140583948234751, +STORE, 140583948234752, 140583948275711, +SNULL, 140583948234752, 140583948259327, +STORE, 140583948259328, 140583948275711, +STORE, 140583948234752, 140583948259327, +ERASE, 140583948234752, 140583948259327, +STORE, 140583948234752, 140583948259327, +ERASE, 140583948259328, 140583948275711, +STORE, 140583948259328, 140583948275711, +STORE, 140583953629184, 140583953649663, +SNULL, 140583948251135, 140583948259327, +STORE, 140583948234752, 140583948251135, +STORE, 140583948251136, 140583948259327, +SNULL, 140583951433727, 140583951437823, +STORE, 140583951429632, 140583951433727, +STORE, 140583951433728, 140583951437823, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140583953682431, 140583953686527, +STORE, 140583953678336, 140583953682431, +STORE, 140583953682432, 140583953686527, +ERASE, 140583953649664, 140583953678335, +STORE, 17821696, 17956863, +STORE, 17821696, 18104319, +STORE, 140583951945728, 140583953629183, +STORE, 94014447943680, 94014448156671, +STORE, 94014450253824, 94014450257919, +STORE, 94014450257920, 94014450266111, +STORE, 94014450266112, 94014450278399, +STORE, 94014464225280, 94014465196031, +STORE, 139761764306944, 139761765965823, +STORE, 139761765965824, 139761768062975, +STORE, 139761768062976, 139761768079359, +STORE, 139761768079360, 139761768087551, +STORE, 139761768087552, 139761768103935, +STORE, 139761768103936, 139761768116223, +STORE, 139761768116224, 139761770209279, +STORE, 139761770209280, 139761770213375, +STORE, 139761770213376, 139761770217471, +STORE, 139761770217472, 139761770360831, +STORE, 139761770729472, 139761772412927, +STORE, 139761772412928, 139761772429311, +STORE, 139761772457984, 139761772462079, +STORE, 139761772462080, 139761772466175, +STORE, 139761772466176, 139761772470271, +STORE, 140724336517120, 140724336652287, +STORE, 140724336955392, 140724336967679, +STORE, 140724336967680, 140724336971775, +STORE, 140737488347136, 140737488351231, +STORE, 140726063296512, 140737488351231, +SNULL, 140726063300607, 140737488351231, +STORE, 140726063296512, 140726063300607, +STORE, 140726063165440, 140726063300607, +STORE, 94016795934720, 94016798158847, +SNULL, 94016796045311, 94016798158847, +STORE, 94016795934720, 94016796045311, +STORE, 94016796045312, 94016798158847, +ERASE, 94016796045312, 94016798158847, +STORE, 94016798138368, 94016798150655, +STORE, 94016798150656, 94016798158847, +STORE, 139975915966464, 139975918219263, +SNULL, 139975916109823, 139975918219263, +STORE, 139975915966464, 139975916109823, +STORE, 139975916109824, 139975918219263, +ERASE, 139975916109824, 139975918219263, +STORE, 139975918206976, 139975918215167, +STORE, 139975918215168, 139975918219263, +STORE, 140726064541696, 140726064545791, +STORE, 140726064529408, 140726064541695, +STORE, 139975918178304, 139975918206975, +STORE, 139975918170112, 139975918178303, +STORE, 139975912169472, 139975915966463, +SNULL, 139975912169472, 139975913828351, +STORE, 139975913828352, 139975915966463, +STORE, 139975912169472, 139975913828351, +SNULL, 139975915925503, 139975915966463, +STORE, 139975913828352, 139975915925503, +STORE, 139975915925504, 139975915966463, +SNULL, 139975915925504, 139975915950079, +STORE, 139975915950080, 139975915966463, +STORE, 139975915925504, 139975915950079, +ERASE, 139975915925504, 139975915950079, +STORE, 139975915925504, 139975915950079, +ERASE, 139975915950080, 139975915966463, +STORE, 139975915950080, 139975915966463, +SNULL, 139975915941887, 139975915950079, +STORE, 139975915925504, 139975915941887, +STORE, 139975915941888, 139975915950079, +SNULL, 94016798146559, 94016798150655, +STORE, 94016798138368, 94016798146559, +STORE, 94016798146560, 94016798150655, +SNULL, 139975918211071, 139975918215167, +STORE, 139975918206976, 139975918211071, +STORE, 139975918211072, 139975918215167, +ERASE, 139975918178304, 139975918206975, +STORE, 94016804925440, 94016805060607, +STORE, 94596177661952, 94596177772543, +STORE, 94596179865600, 94596179873791, +STORE, 94596179873792, 94596179877887, +STORE, 94596179877888, 94596179886079, +STORE, 94596211597312, 94596211863551, +STORE, 140127351840768, 140127353499647, +STORE, 140127353499648, 140127355596799, +STORE, 140127355596800, 140127355613183, +STORE, 140127355613184, 140127355621375, +STORE, 140127355621376, 140127355637759, +STORE, 140127355637760, 140127355781119, +STORE, 140127357841408, 140127357849599, +STORE, 140127357878272, 140127357882367, +STORE, 140127357882368, 140127357886463, +STORE, 140127357886464, 140127357890559, +STORE, 140726167252992, 140726167392255, +STORE, 140726167838720, 140726167851007, +STORE, 140726167851008, 140726167855103, +STORE, 140737488347136, 140737488351231, +STORE, 140731874017280, 140737488351231, +SNULL, 140731874021375, 140737488351231, +STORE, 140731874017280, 140731874021375, +STORE, 140731873886208, 140731874021375, +STORE, 94178682265600, 94178684489727, +SNULL, 94178682376191, 94178684489727, +STORE, 94178682265600, 94178682376191, +STORE, 94178682376192, 94178684489727, +ERASE, 94178682376192, 94178684489727, +STORE, 94178684469248, 94178684481535, +STORE, 94178684481536, 94178684489727, +STORE, 140460853403648, 140460855656447, +SNULL, 140460853547007, 140460855656447, +STORE, 140460853403648, 140460853547007, +STORE, 140460853547008, 140460855656447, +ERASE, 140460853547008, 140460855656447, +STORE, 140460855644160, 140460855652351, +STORE, 140460855652352, 140460855656447, +STORE, 140731874103296, 140731874107391, +STORE, 140731874091008, 140731874103295, +STORE, 140460855615488, 140460855644159, +STORE, 140460855607296, 140460855615487, +STORE, 140460849606656, 140460853403647, +SNULL, 140460849606656, 140460851265535, +STORE, 140460851265536, 140460853403647, +STORE, 140460849606656, 140460851265535, +SNULL, 140460853362687, 140460853403647, +STORE, 140460851265536, 140460853362687, +STORE, 140460853362688, 140460853403647, +SNULL, 140460853362688, 140460853387263, +STORE, 140460853387264, 140460853403647, +STORE, 140460853362688, 140460853387263, +ERASE, 140460853362688, 140460853387263, +STORE, 140460853362688, 140460853387263, +ERASE, 140460853387264, 140460853403647, +STORE, 140460853387264, 140460853403647, +SNULL, 140460853379071, 140460853387263, +STORE, 140460853362688, 140460853379071, +STORE, 140460853379072, 140460853387263, +SNULL, 94178684477439, 94178684481535, +STORE, 94178684469248, 94178684477439, +STORE, 94178684477440, 94178684481535, +SNULL, 140460855648255, 140460855652351, +STORE, 140460855644160, 140460855648255, +STORE, 140460855648256, 140460855652351, +ERASE, 140460855615488, 140460855644159, +STORE, 94178692063232, 94178692198399, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733096603648, 140737488351231, +SNULL, 140733096611839, 140737488351231, +STORE, 140733096603648, 140733096611839, +STORE, 140733096472576, 140733096611839, +STORE, 94796716122112, 94796718325759, +SNULL, 94796716224511, 94796718325759, +STORE, 94796716122112, 94796716224511, +STORE, 94796716224512, 94796718325759, +ERASE, 94796716224512, 94796718325759, +STORE, 94796718317568, 94796718325759, +STORE, 139667892793344, 139667895046143, +SNULL, 139667892936703, 139667895046143, +STORE, 139667892793344, 139667892936703, +STORE, 139667892936704, 139667895046143, +ERASE, 139667892936704, 139667895046143, +STORE, 139667895033856, 139667895042047, +STORE, 139667895042048, 139667895046143, +STORE, 140733096857600, 140733096861695, +STORE, 140733096845312, 140733096857599, +STORE, 139667895005184, 139667895033855, +STORE, 139667894996992, 139667895005183, +STORE, 139667890532352, 139667892793343, +SNULL, 139667890532352, 139667890683903, +STORE, 139667890683904, 139667892793343, +STORE, 139667890532352, 139667890683903, +SNULL, 139667892776959, 139667892793343, +STORE, 139667890683904, 139667892776959, +STORE, 139667892776960, 139667892793343, +SNULL, 139667892776960, 139667892785151, +STORE, 139667892785152, 139667892793343, +STORE, 139667892776960, 139667892785151, +ERASE, 139667892776960, 139667892785151, +STORE, 139667892776960, 139667892785151, +ERASE, 139667892785152, 139667892793343, +STORE, 139667892785152, 139667892793343, +STORE, 139667886735360, 139667890532351, +SNULL, 139667886735360, 139667888394239, +STORE, 139667888394240, 139667890532351, +STORE, 139667886735360, 139667888394239, +SNULL, 139667890491391, 139667890532351, +STORE, 139667888394240, 139667890491391, +STORE, 139667890491392, 139667890532351, +SNULL, 139667890491392, 139667890515967, +STORE, 139667890515968, 139667890532351, +STORE, 139667890491392, 139667890515967, +ERASE, 139667890491392, 139667890515967, +STORE, 139667890491392, 139667890515967, +ERASE, 139667890515968, 139667890532351, +STORE, 139667890515968, 139667890532351, +STORE, 139667884167168, 139667886735359, +SNULL, 139667884167168, 139667884634111, +STORE, 139667884634112, 139667886735359, +STORE, 139667884167168, 139667884634111, +SNULL, 139667886727167, 139667886735359, +STORE, 139667884634112, 139667886727167, +STORE, 139667886727168, 139667886735359, +ERASE, 139667886727168, 139667886735359, +STORE, 139667886727168, 139667886735359, +STORE, 139667882053632, 139667884167167, +SNULL, 139667882053632, 139667882065919, +STORE, 139667882065920, 139667884167167, +STORE, 139667882053632, 139667882065919, +SNULL, 139667884158975, 139667884167167, +STORE, 139667882065920, 139667884158975, +STORE, 139667884158976, 139667884167167, +ERASE, 139667884158976, 139667884167167, +STORE, 139667884158976, 139667884167167, +STORE, 139667879837696, 139667882053631, +SNULL, 139667879837696, 139667879935999, +STORE, 139667879936000, 139667882053631, +STORE, 139667879837696, 139667879935999, +SNULL, 139667882029055, 139667882053631, +STORE, 139667879936000, 139667882029055, +STORE, 139667882029056, 139667882053631, +SNULL, 139667882029056, 139667882037247, +STORE, 139667882037248, 139667882053631, +STORE, 139667882029056, 139667882037247, +ERASE, 139667882029056, 139667882037247, +STORE, 139667882029056, 139667882037247, +ERASE, 139667882037248, 139667882053631, +STORE, 139667882037248, 139667882053631, +STORE, 139667894988800, 139667895005183, +SNULL, 139667890507775, 139667890515967, +STORE, 139667890491392, 139667890507775, +STORE, 139667890507776, 139667890515967, +SNULL, 139667882033151, 139667882037247, +STORE, 139667882029056, 139667882033151, +STORE, 139667882033152, 139667882037247, +SNULL, 139667884163071, 139667884167167, +STORE, 139667884158976, 139667884163071, +STORE, 139667884163072, 139667884167167, +SNULL, 139667886731263, 139667886735359, +STORE, 139667886727168, 139667886731263, +STORE, 139667886731264, 139667886735359, +SNULL, 139667892781055, 139667892785151, +STORE, 139667892776960, 139667892781055, +STORE, 139667892781056, 139667892785151, +SNULL, 94796718321663, 94796718325759, +STORE, 94796718317568, 94796718321663, +STORE, 94796718321664, 94796718325759, +SNULL, 139667895037951, 139667895042047, +STORE, 139667895033856, 139667895037951, +STORE, 139667895037952, 139667895042047, +ERASE, 139667895005184, 139667895033855, +STORE, 94796726063104, 94796726198271, +STORE, 139667893305344, 139667894988799, +STORE, 139667895005184, 139667895033855, +STORE, 94796726063104, 94796726333439, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722489507840, 140737488351231, +SNULL, 140722489516031, 140737488351231, +STORE, 140722489507840, 140722489516031, +STORE, 140722489376768, 140722489516031, +STORE, 93980993265664, 93980995489791, +SNULL, 93980993376255, 93980995489791, +STORE, 93980993265664, 93980993376255, +STORE, 93980993376256, 93980995489791, +ERASE, 93980993376256, 93980995489791, +STORE, 93980995469312, 93980995481599, +STORE, 93980995481600, 93980995489791, +STORE, 140261313593344, 140261315846143, +SNULL, 140261313736703, 140261315846143, +STORE, 140261313593344, 140261313736703, +STORE, 140261313736704, 140261315846143, +ERASE, 140261313736704, 140261315846143, +STORE, 140261315833856, 140261315842047, +STORE, 140261315842048, 140261315846143, +STORE, 140722489675776, 140722489679871, +STORE, 140722489663488, 140722489675775, +STORE, 140261315805184, 140261315833855, +STORE, 140261315796992, 140261315805183, +STORE, 140261309796352, 140261313593343, +SNULL, 140261309796352, 140261311455231, +STORE, 140261311455232, 140261313593343, +STORE, 140261309796352, 140261311455231, +SNULL, 140261313552383, 140261313593343, +STORE, 140261311455232, 140261313552383, +STORE, 140261313552384, 140261313593343, +SNULL, 140261313552384, 140261313576959, +STORE, 140261313576960, 140261313593343, +STORE, 140261313552384, 140261313576959, +ERASE, 140261313552384, 140261313576959, +STORE, 140261313552384, 140261313576959, +ERASE, 140261313576960, 140261313593343, +STORE, 140261313576960, 140261313593343, +SNULL, 140261313568767, 140261313576959, +STORE, 140261313552384, 140261313568767, +STORE, 140261313568768, 140261313576959, +SNULL, 93980995477503, 93980995481599, +STORE, 93980995469312, 93980995477503, +STORE, 93980995477504, 93980995481599, +SNULL, 140261315837951, 140261315842047, +STORE, 140261315833856, 140261315837951, +STORE, 140261315837952, 140261315842047, +ERASE, 140261315805184, 140261315833855, +STORE, 93980997443584, 93980997578751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140734059450368, 140737488351231, +SNULL, 140734059462655, 140737488351231, +STORE, 140734059450368, 140734059462655, +STORE, 140734059319296, 140734059462655, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140307554983936, 140307557236735, +SNULL, 140307555127295, 140307557236735, +STORE, 140307554983936, 140307555127295, +STORE, 140307555127296, 140307557236735, +ERASE, 140307555127296, 140307557236735, +STORE, 140307557224448, 140307557232639, +STORE, 140307557232640, 140307557236735, +STORE, 140734059483136, 140734059487231, +STORE, 140734059470848, 140734059483135, +STORE, 140307557195776, 140307557224447, +STORE, 140307557187584, 140307557195775, +STORE, 140307551821824, 140307554983935, +SNULL, 140307551821824, 140307552882687, +STORE, 140307552882688, 140307554983935, +STORE, 140307551821824, 140307552882687, +SNULL, 140307554975743, 140307554983935, +STORE, 140307552882688, 140307554975743, +STORE, 140307554975744, 140307554983935, +ERASE, 140307554975744, 140307554983935, +STORE, 140307554975744, 140307554983935, +STORE, 140307548024832, 140307551821823, +SNULL, 140307548024832, 140307549683711, +STORE, 140307549683712, 140307551821823, +STORE, 140307548024832, 140307549683711, +SNULL, 140307551780863, 140307551821823, +STORE, 140307549683712, 140307551780863, +STORE, 140307551780864, 140307551821823, +SNULL, 140307551780864, 140307551805439, +STORE, 140307551805440, 140307551821823, +STORE, 140307551780864, 140307551805439, +ERASE, 140307551780864, 140307551805439, +STORE, 140307551780864, 140307551805439, +ERASE, 140307551805440, 140307551821823, +STORE, 140307551805440, 140307551821823, +STORE, 140307557175296, 140307557195775, +SNULL, 140307551797247, 140307551805439, +STORE, 140307551780864, 140307551797247, +STORE, 140307551797248, 140307551805439, +SNULL, 140307554979839, 140307554983935, +STORE, 140307554975744, 140307554979839, +STORE, 140307554979840, 140307554983935, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140307557228543, 140307557232639, +STORE, 140307557224448, 140307557228543, +STORE, 140307557228544, 140307557232639, +ERASE, 140307557195776, 140307557224447, +STORE, 39698432, 39833599, +STORE, 39698432, 39981055, +STORE, 94306485321728, 94306485432319, +STORE, 94306487525376, 94306487533567, +STORE, 94306487533568, 94306487537663, +STORE, 94306487537664, 94306487545855, +STORE, 94306488868864, 94306489004031, +STORE, 140497673998336, 140497675657215, +STORE, 140497675657216, 140497677754367, +STORE, 140497677754368, 140497677770751, +STORE, 140497677770752, 140497677778943, +STORE, 140497677778944, 140497677795327, +STORE, 140497677795328, 140497677938687, +STORE, 140497679998976, 140497680007167, +STORE, 140497680035840, 140497680039935, +STORE, 140497680039936, 140497680044031, +STORE, 140497680044032, 140497680048127, +STORE, 140732780462080, 140732780601343, +STORE, 140732782239744, 140732782252031, +STORE, 140732782252032, 140732782256127, +STORE, 94236915900416, 94236916011007, +STORE, 94236918104064, 94236918112255, +STORE, 94236918112256, 94236918116351, +STORE, 94236918116352, 94236918124543, +STORE, 94236939489280, 94236939624447, +STORE, 140046091743232, 140046093402111, +STORE, 140046093402112, 140046095499263, +STORE, 140046095499264, 140046095515647, +STORE, 140046095515648, 140046095523839, +STORE, 140046095523840, 140046095540223, +STORE, 140046095540224, 140046095683583, +STORE, 140046097743872, 140046097752063, +STORE, 140046097780736, 140046097784831, +STORE, 140046097784832, 140046097788927, +STORE, 140046097788928, 140046097793023, +STORE, 140726694449152, 140726694588415, +STORE, 140726695313408, 140726695325695, +STORE, 140726695325696, 140726695329791, +STORE, 94894582779904, 94894582992895, +STORE, 94894585090048, 94894585094143, +STORE, 94894585094144, 94894585102335, +STORE, 94894585102336, 94894585114623, +STORE, 94894592868352, 94894594293759, +STORE, 139733563842560, 139733565501439, +STORE, 139733565501440, 139733567598591, +STORE, 139733567598592, 139733567614975, +STORE, 139733567614976, 139733567623167, +STORE, 139733567623168, 139733567639551, +STORE, 139733567639552, 139733567651839, +STORE, 139733567651840, 139733569744895, +STORE, 139733569744896, 139733569748991, +STORE, 139733569748992, 139733569753087, +STORE, 139733569753088, 139733569896447, +STORE, 139733570265088, 139733571948543, +STORE, 139733571948544, 139733571964927, +STORE, 139733571993600, 139733571997695, +STORE, 139733571997696, 139733572001791, +STORE, 139733572001792, 139733572005887, +STORE, 140726369255424, 140726369394687, +STORE, 140726370402304, 140726370414591, +STORE, 140726370414592, 140726370418687, +STORE, 94899236483072, 94899236696063, +STORE, 94899238793216, 94899238797311, +STORE, 94899238797312, 94899238805503, +STORE, 94899238805504, 94899238817791, +STORE, 94899263045632, 94899263979519, +STORE, 140040959893504, 140040961552383, +STORE, 140040961552384, 140040963649535, +STORE, 140040963649536, 140040963665919, +STORE, 140040963665920, 140040963674111, +STORE, 140040963674112, 140040963690495, +STORE, 140040963690496, 140040963702783, +STORE, 140040963702784, 140040965795839, +STORE, 140040965795840, 140040965799935, +STORE, 140040965799936, 140040965804031, +STORE, 140040965804032, 140040965947391, +STORE, 140040966316032, 140040967999487, +STORE, 140040967999488, 140040968015871, +STORE, 140040968044544, 140040968048639, +STORE, 140040968048640, 140040968052735, +STORE, 140040968052736, 140040968056831, +STORE, 140729921359872, 140729921499135, +STORE, 140729921613824, 140729921626111, +STORE, 140729921626112, 140729921630207, +STORE, 94818265190400, 94818265403391, +STORE, 94818267500544, 94818267504639, +STORE, 94818267504640, 94818267512831, +STORE, 94818267512832, 94818267525119, +STORE, 94818283372544, 94818285858815, +STORE, 139818425675776, 139818427334655, +STORE, 139818427334656, 139818429431807, +STORE, 139818429431808, 139818429448191, +STORE, 139818429448192, 139818429456383, +STORE, 139818429456384, 139818429472767, +STORE, 139818429472768, 139818429485055, +STORE, 139818429485056, 139818431578111, +STORE, 139818431578112, 139818431582207, +STORE, 139818431582208, 139818431586303, +STORE, 139818431586304, 139818431729663, +STORE, 139818432098304, 139818433781759, +STORE, 139818433781760, 139818433798143, +STORE, 139818433826816, 139818433830911, +STORE, 139818433830912, 139818433835007, +STORE, 139818433835008, 139818433839103, +STORE, 140726170509312, 140726170648575, +STORE, 140726171824128, 140726171836415, +STORE, 140726171836416, 140726171840511, +STORE, 94611513188352, 94611513401343, +STORE, 94611515498496, 94611515502591, +STORE, 94611515502592, 94611515510783, +STORE, 94611515510784, 94611515523071, +STORE, 94611516502016, 94611516907519, +STORE, 140596246388736, 140596248047615, +STORE, 140596248047616, 140596250144767, +STORE, 140596250144768, 140596250161151, +STORE, 140596250161152, 140596250169343, +STORE, 140596250169344, 140596250185727, +STORE, 140596250185728, 140596250198015, +STORE, 140596250198016, 140596252291071, +STORE, 140596252291072, 140596252295167, +STORE, 140596252295168, 140596252299263, +STORE, 140596252299264, 140596252442623, +STORE, 140596252811264, 140596254494719, +STORE, 140596254494720, 140596254511103, +STORE, 140596254539776, 140596254543871, +STORE, 140596254543872, 140596254547967, +STORE, 140596254547968, 140596254552063, +STORE, 140731551338496, 140731551477759, +STORE, 140731551780864, 140731551793151, +STORE, 140731551793152, 140731551797247, +STORE, 94313835851776, 94313836064767, +STORE, 94313838161920, 94313838166015, +STORE, 94313838166016, 94313838174207, +STORE, 94313838174208, 94313838186495, +STORE, 94313858416640, 94313861906431, +STORE, 140693503918080, 140693505576959, +STORE, 140693505576960, 140693507674111, +STORE, 140693507674112, 140693507690495, +STORE, 140693507690496, 140693507698687, +STORE, 140693507698688, 140693507715071, +STORE, 140693507715072, 140693507727359, +STORE, 140693507727360, 140693509820415, +STORE, 140693509820416, 140693509824511, +STORE, 140693509824512, 140693509828607, +STORE, 140693509828608, 140693509971967, +STORE, 140693510340608, 140693512024063, +STORE, 140693512024064, 140693512040447, +STORE, 140693512069120, 140693512073215, +STORE, 140693512073216, 140693512077311, +STORE, 140693512077312, 140693512081407, +STORE, 140721116065792, 140721116205055, +STORE, 140721117831168, 140721117843455, +STORE, 140721117843456, 140721117847551, +STORE, 94843650150400, 94843650363391, +STORE, 94843652460544, 94843652464639, +STORE, 94843652464640, 94843652472831, +STORE, 94843652472832, 94843652485119, +STORE, 94843685388288, 94843686281215, +STORE, 140484193681408, 140484195340287, +STORE, 140484195340288, 140484197437439, +STORE, 140484197437440, 140484197453823, +STORE, 140484197453824, 140484197462015, +STORE, 140484197462016, 140484197478399, +STORE, 140484197478400, 140484197490687, +STORE, 140484197490688, 140484199583743, +STORE, 140484199583744, 140484199587839, +STORE, 140484199587840, 140484199591935, +STORE, 140484199591936, 140484199735295, +STORE, 140484200103936, 140484201787391, +STORE, 140484201787392, 140484201803775, +STORE, 140484201832448, 140484201836543, +STORE, 140484201836544, 140484201840639, +STORE, 140484201840640, 140484201844735, +STORE, 140726294315008, 140726294454271, +STORE, 140726295646208, 140726295658495, +STORE, 140726295658496, 140726295662591, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720422371328, 140737488351231, +SNULL, 140720422379519, 140737488351231, +STORE, 140720422371328, 140720422379519, +STORE, 140720422240256, 140720422379519, +STORE, 94417967845376, 94417970180095, +SNULL, 94417968058367, 94417970180095, +STORE, 94417967845376, 94417968058367, +STORE, 94417968058368, 94417970180095, +ERASE, 94417968058368, 94417970180095, +STORE, 94417970155520, 94417970167807, +STORE, 94417970167808, 94417970180095, +STORE, 140252450045952, 140252452298751, +SNULL, 140252450189311, 140252452298751, +STORE, 140252450045952, 140252450189311, +STORE, 140252450189312, 140252452298751, +ERASE, 140252450189312, 140252452298751, +STORE, 140252452286464, 140252452294655, +STORE, 140252452294656, 140252452298751, +STORE, 140720422416384, 140720422420479, +STORE, 140720422404096, 140720422416383, +STORE, 140252452257792, 140252452286463, +STORE, 140252452249600, 140252452257791, +STORE, 140252447932416, 140252450045951, +SNULL, 140252447932416, 140252447944703, +STORE, 140252447944704, 140252450045951, +STORE, 140252447932416, 140252447944703, +SNULL, 140252450037759, 140252450045951, +STORE, 140252447944704, 140252450037759, +STORE, 140252450037760, 140252450045951, +ERASE, 140252450037760, 140252450045951, +STORE, 140252450037760, 140252450045951, +STORE, 140252444135424, 140252447932415, +SNULL, 140252444135424, 140252445794303, +STORE, 140252445794304, 140252447932415, +STORE, 140252444135424, 140252445794303, +SNULL, 140252447891455, 140252447932415, +STORE, 140252445794304, 140252447891455, +STORE, 140252447891456, 140252447932415, +SNULL, 140252447891456, 140252447916031, +STORE, 140252447916032, 140252447932415, +STORE, 140252447891456, 140252447916031, +ERASE, 140252447891456, 140252447916031, +STORE, 140252447891456, 140252447916031, +ERASE, 140252447916032, 140252447932415, +STORE, 140252447916032, 140252447932415, +STORE, 140252452241408, 140252452257791, +SNULL, 140252447907839, 140252447916031, +STORE, 140252447891456, 140252447907839, +STORE, 140252447907840, 140252447916031, +SNULL, 140252450041855, 140252450045951, +STORE, 140252450037760, 140252450041855, +STORE, 140252450041856, 140252450045951, +SNULL, 94417970159615, 94417970167807, +STORE, 94417970155520, 94417970159615, +STORE, 94417970159616, 94417970167807, +SNULL, 140252452290559, 140252452294655, +STORE, 140252452286464, 140252452290559, +STORE, 140252452290560, 140252452294655, +ERASE, 140252452257792, 140252452286463, +STORE, 94417996333056, 94417996468223, +STORE, 140252450557952, 140252452241407, +STORE, 94417996333056, 94417996603391, +STORE, 94417996333056, 94417996738559, +STORE, 94417996333056, 94417996910591, +SNULL, 94417996881919, 94417996910591, +STORE, 94417996333056, 94417996881919, +STORE, 94417996881920, 94417996910591, +ERASE, 94417996881920, 94417996910591, +STORE, 94417996333056, 94417997017087, +STORE, 94417996333056, 94417997152255, +SNULL, 94417997135871, 94417997152255, +STORE, 94417996333056, 94417997135871, +STORE, 94417997135872, 94417997152255, +ERASE, 94417997135872, 94417997152255, +STORE, 94417996333056, 94417997291519, +SNULL, 94417997271039, 94417997291519, +STORE, 94417996333056, 94417997271039, +STORE, 94417997271040, 94417997291519, +ERASE, 94417997271040, 94417997291519, +STORE, 94417996333056, 94417997406207, +SNULL, 94417997381631, 94417997406207, +STORE, 94417996333056, 94417997381631, +STORE, 94417997381632, 94417997406207, +ERASE, 94417997381632, 94417997406207, +STORE, 94417996333056, 94417997516799, +SNULL, 94417997488127, 94417997516799, +STORE, 94417996333056, 94417997488127, +STORE, 94417997488128, 94417997516799, +ERASE, 94417997488128, 94417997516799, +STORE, 94417996333056, 94417997643775, +SNULL, 94417997631487, 94417997643775, +STORE, 94417996333056, 94417997631487, +STORE, 94417997631488, 94417997643775, +ERASE, 94417997631488, 94417997643775, +SNULL, 94417997590527, 94417997631487, +STORE, 94417996333056, 94417997590527, +STORE, 94417997590528, 94417997631487, +ERASE, 94417997590528, 94417997631487, +STORE, 94417996333056, 94417997733887, +STORE, 94417996333056, 94417997869055, +STORE, 94417996333056, 94417998004223, +SNULL, 94417998000127, 94417998004223, +STORE, 94417996333056, 94417998000127, +STORE, 94417998000128, 94417998004223, +ERASE, 94417998000128, 94417998004223, +STORE, 94049170993152, 94049171206143, +STORE, 94049173303296, 94049173307391, +STORE, 94049173307392, 94049173315583, +STORE, 94049173315584, 94049173327871, +STORE, 94049176236032, 94049183645695, +STORE, 139807795544064, 139807797202943, +STORE, 139807797202944, 139807799300095, +STORE, 139807799300096, 139807799316479, +STORE, 139807799316480, 139807799324671, +STORE, 139807799324672, 139807799341055, +STORE, 139807799341056, 139807799353343, +STORE, 139807799353344, 139807801446399, +STORE, 139807801446400, 139807801450495, +STORE, 139807801450496, 139807801454591, +STORE, 139807801454592, 139807801597951, +STORE, 139807801966592, 139807803650047, +STORE, 139807803650048, 139807803666431, +STORE, 139807803695104, 139807803699199, +STORE, 139807803699200, 139807803703295, +STORE, 139807803703296, 139807803707391, +STORE, 140727555538944, 140727555678207, +STORE, 140727555940352, 140727555952639, +STORE, 140727555952640, 140727555956735, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722483441664, 140737488351231, +SNULL, 140722483449855, 140737488351231, +STORE, 140722483441664, 140722483449855, +STORE, 140722483310592, 140722483449855, +STORE, 94416704921600, 94416707145727, +SNULL, 94416705032191, 94416707145727, +STORE, 94416704921600, 94416705032191, +STORE, 94416705032192, 94416707145727, +ERASE, 94416705032192, 94416707145727, +STORE, 94416707125248, 94416707137535, +STORE, 94416707137536, 94416707145727, +STORE, 140555439296512, 140555441549311, +SNULL, 140555439439871, 140555441549311, +STORE, 140555439296512, 140555439439871, +STORE, 140555439439872, 140555441549311, +ERASE, 140555439439872, 140555441549311, +STORE, 140555441537024, 140555441545215, +STORE, 140555441545216, 140555441549311, +STORE, 140722484781056, 140722484785151, +STORE, 140722484768768, 140722484781055, +STORE, 140555441508352, 140555441537023, +STORE, 140555441500160, 140555441508351, +STORE, 140555435499520, 140555439296511, +SNULL, 140555435499520, 140555437158399, +STORE, 140555437158400, 140555439296511, +STORE, 140555435499520, 140555437158399, +SNULL, 140555439255551, 140555439296511, +STORE, 140555437158400, 140555439255551, +STORE, 140555439255552, 140555439296511, +SNULL, 140555439255552, 140555439280127, +STORE, 140555439280128, 140555439296511, +STORE, 140555439255552, 140555439280127, +ERASE, 140555439255552, 140555439280127, +STORE, 140555439255552, 140555439280127, +ERASE, 140555439280128, 140555439296511, +STORE, 140555439280128, 140555439296511, +SNULL, 140555439271935, 140555439280127, +STORE, 140555439255552, 140555439271935, +STORE, 140555439271936, 140555439280127, +SNULL, 94416707133439, 94416707137535, +STORE, 94416707125248, 94416707133439, +STORE, 94416707133440, 94416707137535, +SNULL, 140555441541119, 140555441545215, +STORE, 140555441537024, 140555441541119, +STORE, 140555441541120, 140555441545215, +ERASE, 140555441508352, 140555441537023, +STORE, 94416724672512, 94416724807679, +STORE, 94686636953600, 94686637166591, +STORE, 94686639263744, 94686639267839, +STORE, 94686639267840, 94686639276031, +STORE, 94686639276032, 94686639288319, +STORE, 94686662193152, 94686663163903, +STORE, 140312944431104, 140312946089983, +STORE, 140312946089984, 140312948187135, +STORE, 140312948187136, 140312948203519, +STORE, 140312948203520, 140312948211711, +STORE, 140312948211712, 140312948228095, +STORE, 140312948228096, 140312948240383, +STORE, 140312948240384, 140312950333439, +STORE, 140312950333440, 140312950337535, +STORE, 140312950337536, 140312950341631, +STORE, 140312950341632, 140312950484991, +STORE, 140312950853632, 140312952537087, +STORE, 140312952537088, 140312952553471, +STORE, 140312952582144, 140312952586239, +STORE, 140312952586240, 140312952590335, +STORE, 140312952590336, 140312952594431, +STORE, 140730598920192, 140730599059455, +STORE, 140730599108608, 140730599120895, +STORE, 140730599120896, 140730599124991, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726234079232, 140737488351231, +SNULL, 140726234087423, 140737488351231, +STORE, 140726234079232, 140726234087423, +STORE, 140726233948160, 140726234087423, +STORE, 94589467578368, 94589469802495, +SNULL, 94589467688959, 94589469802495, +STORE, 94589467578368, 94589467688959, +STORE, 94589467688960, 94589469802495, +ERASE, 94589467688960, 94589469802495, +STORE, 94589469782016, 94589469794303, +STORE, 94589469794304, 94589469802495, +STORE, 140587082842112, 140587085094911, +SNULL, 140587082985471, 140587085094911, +STORE, 140587082842112, 140587082985471, +STORE, 140587082985472, 140587085094911, +ERASE, 140587082985472, 140587085094911, +STORE, 140587085082624, 140587085090815, +STORE, 140587085090816, 140587085094911, +STORE, 140726234103808, 140726234107903, +STORE, 140726234091520, 140726234103807, +STORE, 140587085053952, 140587085082623, +STORE, 140587085045760, 140587085053951, +STORE, 140587079045120, 140587082842111, +SNULL, 140587079045120, 140587080703999, +STORE, 140587080704000, 140587082842111, +STORE, 140587079045120, 140587080703999, +SNULL, 140587082801151, 140587082842111, +STORE, 140587080704000, 140587082801151, +STORE, 140587082801152, 140587082842111, +SNULL, 140587082801152, 140587082825727, +STORE, 140587082825728, 140587082842111, +STORE, 140587082801152, 140587082825727, +ERASE, 140587082801152, 140587082825727, +STORE, 140587082801152, 140587082825727, +ERASE, 140587082825728, 140587082842111, +STORE, 140587082825728, 140587082842111, +SNULL, 140587082817535, 140587082825727, +STORE, 140587082801152, 140587082817535, +STORE, 140587082817536, 140587082825727, +SNULL, 94589469790207, 94589469794303, +STORE, 94589469782016, 94589469790207, +STORE, 94589469790208, 94589469794303, +SNULL, 140587085086719, 140587085090815, +STORE, 140587085082624, 140587085086719, +STORE, 140587085086720, 140587085090815, +ERASE, 140587085053952, 140587085082623, +STORE, 94589477507072, 94589477642239, +STORE, 94225448325120, 94225448538111, +STORE, 94225450635264, 94225450639359, +STORE, 94225450639360, 94225450647551, +STORE, 94225450647552, 94225450659839, +STORE, 94225470246912, 94225473548287, +STORE, 140199245496320, 140199247155199, +STORE, 140199247155200, 140199249252351, +STORE, 140199249252352, 140199249268735, +STORE, 140199249268736, 140199249276927, +STORE, 140199249276928, 140199249293311, +STORE, 140199249293312, 140199249305599, +STORE, 140199249305600, 140199251398655, +STORE, 140199251398656, 140199251402751, +STORE, 140199251402752, 140199251406847, +STORE, 140199251406848, 140199251550207, +STORE, 140199251918848, 140199253602303, +STORE, 140199253602304, 140199253618687, +STORE, 140199253647360, 140199253651455, +STORE, 140199253651456, 140199253655551, +STORE, 140199253655552, 140199253659647, +STORE, 140726264414208, 140726264553471, +STORE, 140726265843712, 140726265855999, +STORE, 140726265856000, 140726265860095, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733508358144, 140737488351231, +SNULL, 140733508366335, 140737488351231, +STORE, 140733508358144, 140733508366335, +STORE, 140733508227072, 140733508366335, +STORE, 94766263947264, 94766266171391, +SNULL, 94766264057855, 94766266171391, +STORE, 94766263947264, 94766264057855, +STORE, 94766264057856, 94766266171391, +ERASE, 94766264057856, 94766266171391, +STORE, 94766266150912, 94766266163199, +STORE, 94766266163200, 94766266171391, +STORE, 140693985132544, 140693987385343, +SNULL, 140693985275903, 140693987385343, +STORE, 140693985132544, 140693985275903, +STORE, 140693985275904, 140693987385343, +ERASE, 140693985275904, 140693987385343, +STORE, 140693987373056, 140693987381247, +STORE, 140693987381248, 140693987385343, +STORE, 140733509939200, 140733509943295, +STORE, 140733509926912, 140733509939199, +STORE, 140693987344384, 140693987373055, +STORE, 140693987336192, 140693987344383, +STORE, 140693981335552, 140693985132543, +SNULL, 140693981335552, 140693982994431, +STORE, 140693982994432, 140693985132543, +STORE, 140693981335552, 140693982994431, +SNULL, 140693985091583, 140693985132543, +STORE, 140693982994432, 140693985091583, +STORE, 140693985091584, 140693985132543, +SNULL, 140693985091584, 140693985116159, +STORE, 140693985116160, 140693985132543, +STORE, 140693985091584, 140693985116159, +ERASE, 140693985091584, 140693985116159, +STORE, 140693985091584, 140693985116159, +ERASE, 140693985116160, 140693985132543, +STORE, 140693985116160, 140693985132543, +SNULL, 140693985107967, 140693985116159, +STORE, 140693985091584, 140693985107967, +STORE, 140693985107968, 140693985116159, +SNULL, 94766266159103, 94766266163199, +STORE, 94766266150912, 94766266159103, +STORE, 94766266159104, 94766266163199, +SNULL, 140693987377151, 140693987381247, +STORE, 140693987373056, 140693987377151, +STORE, 140693987377152, 140693987381247, +ERASE, 140693987344384, 140693987373055, +STORE, 94766282035200, 94766282170367, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724769353728, 140737488351231, +SNULL, 140724769361919, 140737488351231, +STORE, 140724769353728, 140724769361919, +STORE, 140724769222656, 140724769361919, +STORE, 94710460526592, 94710462750719, +SNULL, 94710460637183, 94710462750719, +STORE, 94710460526592, 94710460637183, +STORE, 94710460637184, 94710462750719, +ERASE, 94710460637184, 94710462750719, +STORE, 94710462730240, 94710462742527, +STORE, 94710462742528, 94710462750719, +STORE, 140469764395008, 140469766647807, +SNULL, 140469764538367, 140469766647807, +STORE, 140469764395008, 140469764538367, +STORE, 140469764538368, 140469766647807, +ERASE, 140469764538368, 140469766647807, +STORE, 140469766635520, 140469766643711, +STORE, 140469766643712, 140469766647807, +STORE, 140724770877440, 140724770881535, +STORE, 140724770865152, 140724770877439, +STORE, 140469766606848, 140469766635519, +STORE, 140469766598656, 140469766606847, +STORE, 140469760598016, 140469764395007, +SNULL, 140469760598016, 140469762256895, +STORE, 140469762256896, 140469764395007, +STORE, 140469760598016, 140469762256895, +SNULL, 140469764354047, 140469764395007, +STORE, 140469762256896, 140469764354047, +STORE, 140469764354048, 140469764395007, +SNULL, 140469764354048, 140469764378623, +STORE, 140469764378624, 140469764395007, +STORE, 140469764354048, 140469764378623, +ERASE, 140469764354048, 140469764378623, +STORE, 140469764354048, 140469764378623, +ERASE, 140469764378624, 140469764395007, +STORE, 140469764378624, 140469764395007, +SNULL, 140469764370431, 140469764378623, +STORE, 140469764354048, 140469764370431, +STORE, 140469764370432, 140469764378623, +SNULL, 94710462738431, 94710462742527, +STORE, 94710462730240, 94710462738431, +STORE, 94710462738432, 94710462742527, +SNULL, 140469766639615, 140469766643711, +STORE, 140469766635520, 140469766639615, +STORE, 140469766639616, 140469766643711, +ERASE, 140469766606848, 140469766635519, +STORE, 94710485581824, 94710485716991, +STORE, 94105755795456, 94105756008447, +STORE, 94105758105600, 94105758109695, +STORE, 94105758109696, 94105758117887, +STORE, 94105758117888, 94105758130175, +STORE, 94105788981248, 94105794871295, +STORE, 140641190031360, 140641191690239, +STORE, 140641191690240, 140641193787391, +STORE, 140641193787392, 140641193803775, +STORE, 140641193803776, 140641193811967, +STORE, 140641193811968, 140641193828351, +STORE, 140641193828352, 140641193840639, +STORE, 140641193840640, 140641195933695, +STORE, 140641195933696, 140641195937791, +STORE, 140641195937792, 140641195941887, +STORE, 140641195941888, 140641196085247, +STORE, 140641196453888, 140641198137343, +STORE, 140641198137344, 140641198153727, +STORE, 140641198182400, 140641198186495, +STORE, 140641198186496, 140641198190591, +STORE, 140641198190592, 140641198194687, +STORE, 140731980034048, 140731980173311, +STORE, 140731981078528, 140731981090815, +STORE, 140731981090816, 140731981094911, +STORE, 93828086431744, 93828086644735, +STORE, 93828088741888, 93828088745983, +STORE, 93828088745984, 93828088754175, +STORE, 93828088754176, 93828088766463, +STORE, 93828094193664, 93828096831487, +STORE, 139844717334528, 139844718993407, +STORE, 139844718993408, 139844721090559, +STORE, 139844721090560, 139844721106943, +STORE, 139844721106944, 139844721115135, +STORE, 139844721115136, 139844721131519, +STORE, 139844721131520, 139844721143807, +STORE, 139844721143808, 139844723236863, +STORE, 139844723236864, 139844723240959, +STORE, 139844723240960, 139844723245055, +STORE, 139844723245056, 139844723388415, +STORE, 139844723757056, 139844725440511, +STORE, 139844725440512, 139844725456895, +STORE, 139844725485568, 139844725489663, +STORE, 139844725489664, 139844725493759, +STORE, 139844725493760, 139844725497855, +STORE, 140729996185600, 140729996324863, +STORE, 140729996828672, 140729996840959, +STORE, 140729996840960, 140729996845055, +STORE, 140737488347136, 140737488351231, +STORE, 140722494771200, 140737488351231, +SNULL, 140722494775295, 140737488351231, +STORE, 140722494771200, 140722494775295, +STORE, 140722494640128, 140722494775295, +STORE, 94324011311104, 94324013535231, +SNULL, 94324011421695, 94324013535231, +STORE, 94324011311104, 94324011421695, +STORE, 94324011421696, 94324013535231, +ERASE, 94324011421696, 94324013535231, +STORE, 94324013514752, 94324013527039, +STORE, 94324013527040, 94324013535231, +STORE, 140151462309888, 140151464562687, +SNULL, 140151462453247, 140151464562687, +STORE, 140151462309888, 140151462453247, +STORE, 140151462453248, 140151464562687, +ERASE, 140151462453248, 140151464562687, +STORE, 140151464550400, 140151464558591, +STORE, 140151464558592, 140151464562687, +STORE, 140722495467520, 140722495471615, +STORE, 140722495455232, 140722495467519, +STORE, 140151464521728, 140151464550399, +STORE, 140151464513536, 140151464521727, +STORE, 140151458512896, 140151462309887, +SNULL, 140151458512896, 140151460171775, +STORE, 140151460171776, 140151462309887, +STORE, 140151458512896, 140151460171775, +SNULL, 140151462268927, 140151462309887, +STORE, 140151460171776, 140151462268927, +STORE, 140151462268928, 140151462309887, +SNULL, 140151462268928, 140151462293503, +STORE, 140151462293504, 140151462309887, +STORE, 140151462268928, 140151462293503, +ERASE, 140151462268928, 140151462293503, +STORE, 140151462268928, 140151462293503, +ERASE, 140151462293504, 140151462309887, +STORE, 140151462293504, 140151462309887, +SNULL, 140151462285311, 140151462293503, +STORE, 140151462268928, 140151462285311, +STORE, 140151462285312, 140151462293503, +SNULL, 94324013522943, 94324013527039, +STORE, 94324013514752, 94324013522943, +STORE, 94324013522944, 94324013527039, +SNULL, 140151464554495, 140151464558591, +STORE, 140151464550400, 140151464554495, +STORE, 140151464554496, 140151464558591, +ERASE, 140151464521728, 140151464550399, +STORE, 94324024778752, 94324024913919, +STORE, 94899262967808, 94899263180799, +STORE, 94899265277952, 94899265282047, +STORE, 94899265282048, 94899265290239, +STORE, 94899265290240, 94899265302527, +STORE, 94899295469568, 94899298689023, +STORE, 140434388418560, 140434390077439, +STORE, 140434390077440, 140434392174591, +STORE, 140434392174592, 140434392190975, +STORE, 140434392190976, 140434392199167, +STORE, 140434392199168, 140434392215551, +STORE, 140434392215552, 140434392227839, +STORE, 140434392227840, 140434394320895, +STORE, 140434394320896, 140434394324991, +STORE, 140434394324992, 140434394329087, +STORE, 140434394329088, 140434394472447, +STORE, 140434394841088, 140434396524543, +STORE, 140434396524544, 140434396540927, +STORE, 140434396569600, 140434396573695, +STORE, 140434396573696, 140434396577791, +STORE, 140434396577792, 140434396581887, +STORE, 140720618135552, 140720618274815, +STORE, 140720618418176, 140720618430463, +STORE, 140720618430464, 140720618434559, +STORE, 94425529798656, 94425530011647, +STORE, 94425532108800, 94425532112895, +STORE, 94425532112896, 94425532121087, +STORE, 94425532121088, 94425532133375, +STORE, 94425557753856, 94425566576639, +STORE, 140600528470016, 140600530128895, +STORE, 140600530128896, 140600532226047, +STORE, 140600532226048, 140600532242431, +STORE, 140600532242432, 140600532250623, +STORE, 140600532250624, 140600532267007, +STORE, 140600532267008, 140600532279295, +STORE, 140600532279296, 140600534372351, +STORE, 140600534372352, 140600534376447, +STORE, 140600534376448, 140600534380543, +STORE, 140600534380544, 140600534523903, +STORE, 140600534892544, 140600536575999, +STORE, 140600536576000, 140600536592383, +STORE, 140600536621056, 140600536625151, +STORE, 140600536625152, 140600536629247, +STORE, 140600536629248, 140600536633343, +STORE, 140721857785856, 140721857925119, +STORE, 140721858068480, 140721858080767, +STORE, 140721858080768, 140721858084863, +STORE, 94425529798656, 94425530011647, +STORE, 94425532108800, 94425532112895, +STORE, 94425532112896, 94425532121087, +STORE, 94425532121088, 94425532133375, +STORE, 94425557753856, 94425568772095, +STORE, 140600528470016, 140600530128895, +STORE, 140600530128896, 140600532226047, +STORE, 140600532226048, 140600532242431, +STORE, 140600532242432, 140600532250623, +STORE, 140600532250624, 140600532267007, +STORE, 140600532267008, 140600532279295, +STORE, 140600532279296, 140600534372351, +STORE, 140600534372352, 140600534376447, +STORE, 140600534376448, 140600534380543, +STORE, 140600534380544, 140600534523903, +STORE, 140600534892544, 140600536575999, +STORE, 140600536576000, 140600536592383, +STORE, 140600536621056, 140600536625151, +STORE, 140600536625152, 140600536629247, +STORE, 140600536629248, 140600536633343, +STORE, 140721857785856, 140721857925119, +STORE, 140721858068480, 140721858080767, +STORE, 140721858080768, 140721858084863, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735611645952, 140737488351231, +SNULL, 140735611654143, 140737488351231, +STORE, 140735611645952, 140735611654143, +STORE, 140735611514880, 140735611654143, +STORE, 94592137641984, 94592139866111, +SNULL, 94592137752575, 94592139866111, +STORE, 94592137641984, 94592137752575, +STORE, 94592137752576, 94592139866111, +ERASE, 94592137752576, 94592139866111, +STORE, 94592139845632, 94592139857919, +STORE, 94592139857920, 94592139866111, +STORE, 140350425030656, 140350427283455, +SNULL, 140350425174015, 140350427283455, +STORE, 140350425030656, 140350425174015, +STORE, 140350425174016, 140350427283455, +ERASE, 140350425174016, 140350427283455, +STORE, 140350427271168, 140350427279359, +STORE, 140350427279360, 140350427283455, +STORE, 140735612043264, 140735612047359, +STORE, 140735612030976, 140735612043263, +STORE, 140350427242496, 140350427271167, +STORE, 140350427234304, 140350427242495, +STORE, 140350421233664, 140350425030655, +SNULL, 140350421233664, 140350422892543, +STORE, 140350422892544, 140350425030655, +STORE, 140350421233664, 140350422892543, +SNULL, 140350424989695, 140350425030655, +STORE, 140350422892544, 140350424989695, +STORE, 140350424989696, 140350425030655, +SNULL, 140350424989696, 140350425014271, +STORE, 140350425014272, 140350425030655, +STORE, 140350424989696, 140350425014271, +ERASE, 140350424989696, 140350425014271, +STORE, 140350424989696, 140350425014271, +ERASE, 140350425014272, 140350425030655, +STORE, 140350425014272, 140350425030655, +SNULL, 140350425006079, 140350425014271, +STORE, 140350424989696, 140350425006079, +STORE, 140350425006080, 140350425014271, +SNULL, 94592139853823, 94592139857919, +STORE, 94592139845632, 94592139853823, +STORE, 94592139853824, 94592139857919, +SNULL, 140350427275263, 140350427279359, +STORE, 140350427271168, 140350427275263, +STORE, 140350427275264, 140350427279359, +ERASE, 140350427242496, 140350427271167, +STORE, 94592164823040, 94592164958207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723500535808, 140737488351231, +SNULL, 140723500543999, 140737488351231, +STORE, 140723500535808, 140723500543999, +STORE, 140723500404736, 140723500543999, +STORE, 94458379010048, 94458381234175, +SNULL, 94458379120639, 94458381234175, +STORE, 94458379010048, 94458379120639, +STORE, 94458379120640, 94458381234175, +ERASE, 94458379120640, 94458381234175, +STORE, 94458381213696, 94458381225983, +STORE, 94458381225984, 94458381234175, +STORE, 139771674230784, 139771676483583, +SNULL, 139771674374143, 139771676483583, +STORE, 139771674230784, 139771674374143, +STORE, 139771674374144, 139771676483583, +ERASE, 139771674374144, 139771676483583, +STORE, 139771676471296, 139771676479487, +STORE, 139771676479488, 139771676483583, +STORE, 140723500769280, 140723500773375, +STORE, 140723500756992, 140723500769279, +STORE, 139771676442624, 139771676471295, +STORE, 139771676434432, 139771676442623, +STORE, 139771670433792, 139771674230783, +SNULL, 139771670433792, 139771672092671, +STORE, 139771672092672, 139771674230783, +STORE, 139771670433792, 139771672092671, +SNULL, 139771674189823, 139771674230783, +STORE, 139771672092672, 139771674189823, +STORE, 139771674189824, 139771674230783, +SNULL, 139771674189824, 139771674214399, +STORE, 139771674214400, 139771674230783, +STORE, 139771674189824, 139771674214399, +ERASE, 139771674189824, 139771674214399, +STORE, 139771674189824, 139771674214399, +ERASE, 139771674214400, 139771674230783, +STORE, 139771674214400, 139771674230783, +SNULL, 139771674206207, 139771674214399, +STORE, 139771674189824, 139771674206207, +STORE, 139771674206208, 139771674214399, +SNULL, 94458381221887, 94458381225983, +STORE, 94458381213696, 94458381221887, +STORE, 94458381221888, 94458381225983, +SNULL, 139771676475391, 139771676479487, +STORE, 139771676471296, 139771676475391, +STORE, 139771676475392, 139771676479487, +ERASE, 139771676442624, 139771676471295, +STORE, 94458401873920, 94458402009087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731316264960, 140737488351231, +SNULL, 140731316273151, 140737488351231, +STORE, 140731316264960, 140731316273151, +STORE, 140731316133888, 140731316273151, +STORE, 94437830881280, 94437833215999, +SNULL, 94437831094271, 94437833215999, +STORE, 94437830881280, 94437831094271, +STORE, 94437831094272, 94437833215999, +ERASE, 94437831094272, 94437833215999, +STORE, 94437833191424, 94437833203711, +STORE, 94437833203712, 94437833215999, +STORE, 140265986031616, 140265988284415, +SNULL, 140265986174975, 140265988284415, +STORE, 140265986031616, 140265986174975, +STORE, 140265986174976, 140265988284415, +ERASE, 140265986174976, 140265988284415, +STORE, 140265988272128, 140265988280319, +STORE, 140265988280320, 140265988284415, +STORE, 140731316318208, 140731316322303, +STORE, 140731316305920, 140731316318207, +STORE, 140265988243456, 140265988272127, +STORE, 140265988235264, 140265988243455, +STORE, 140265983918080, 140265986031615, +SNULL, 140265983918080, 140265983930367, +STORE, 140265983930368, 140265986031615, +STORE, 140265983918080, 140265983930367, +SNULL, 140265986023423, 140265986031615, +STORE, 140265983930368, 140265986023423, +STORE, 140265986023424, 140265986031615, +ERASE, 140265986023424, 140265986031615, +STORE, 140265986023424, 140265986031615, +STORE, 140265980121088, 140265983918079, +SNULL, 140265980121088, 140265981779967, +STORE, 140265981779968, 140265983918079, +STORE, 140265980121088, 140265981779967, +SNULL, 140265983877119, 140265983918079, +STORE, 140265981779968, 140265983877119, +STORE, 140265983877120, 140265983918079, +SNULL, 140265983877120, 140265983901695, +STORE, 140265983901696, 140265983918079, +STORE, 140265983877120, 140265983901695, +ERASE, 140265983877120, 140265983901695, +STORE, 140265983877120, 140265983901695, +ERASE, 140265983901696, 140265983918079, +STORE, 140265983901696, 140265983918079, +STORE, 140265988227072, 140265988243455, +SNULL, 140265983893503, 140265983901695, +STORE, 140265983877120, 140265983893503, +STORE, 140265983893504, 140265983901695, +SNULL, 140265986027519, 140265986031615, +STORE, 140265986023424, 140265986027519, +STORE, 140265986027520, 140265986031615, +SNULL, 94437833195519, 94437833203711, +STORE, 94437833191424, 94437833195519, +STORE, 94437833195520, 94437833203711, +SNULL, 140265988276223, 140265988280319, +STORE, 140265988272128, 140265988276223, +STORE, 140265988276224, 140265988280319, +ERASE, 140265988243456, 140265988272127, +STORE, 94437847638016, 94437847773183, +STORE, 140265986543616, 140265988227071, +STORE, 94437847638016, 94437847908351, +STORE, 94437847638016, 94437848043519, +STORE, 94437847638016, 94437848190975, +SNULL, 94437848178687, 94437848190975, +STORE, 94437847638016, 94437848178687, +STORE, 94437848178688, 94437848190975, +ERASE, 94437848178688, 94437848190975, +STORE, 94437847638016, 94437848330239, +STORE, 94437847638016, 94437848465407, +SNULL, 94437848444927, 94437848465407, +STORE, 94437847638016, 94437848444927, +STORE, 94437848444928, 94437848465407, +ERASE, 94437848444928, 94437848465407, +STORE, 94437847638016, 94437848584191, +STORE, 94437847638016, 94437848719359, +SNULL, 94437848678399, 94437848719359, +STORE, 94437847638016, 94437848678399, +STORE, 94437848678400, 94437848719359, +ERASE, 94437848678400, 94437848719359, +STORE, 94437847638016, 94437848842239, +SNULL, 94437848825855, 94437848842239, +STORE, 94437847638016, 94437848825855, +STORE, 94437848825856, 94437848842239, +ERASE, 94437848825856, 94437848842239, +STORE, 94437847638016, 94437848961023, +STORE, 94437847638016, 94437849096191, +STORE, 94661814710272, 94661814923263, +STORE, 94661817020416, 94661817024511, +STORE, 94661817024512, 94661817032703, +STORE, 94661817032704, 94661817044991, +STORE, 94661840424960, 94661841240063, +STORE, 140582259814400, 140582261473279, +STORE, 140582261473280, 140582263570431, +STORE, 140582263570432, 140582263586815, +STORE, 140582263586816, 140582263595007, +STORE, 140582263595008, 140582263611391, +STORE, 140582263611392, 140582263623679, +STORE, 140582263623680, 140582265716735, +STORE, 140582265716736, 140582265720831, +STORE, 140582265720832, 140582265724927, +STORE, 140582265724928, 140582265868287, +STORE, 140582266236928, 140582267920383, +STORE, 140582267920384, 140582267936767, +STORE, 140582267965440, 140582267969535, +STORE, 140582267969536, 140582267973631, +STORE, 140582267973632, 140582267977727, +STORE, 140735472508928, 140735472648191, +STORE, 140735472672768, 140735472685055, +STORE, 140735472685056, 140735472689151, +STORE, 94440069140480, 94440069353471, +STORE, 94440071450624, 94440071454719, +STORE, 94440071454720, 94440071462911, +STORE, 94440071462912, 94440071475199, +STORE, 94440072122368, 94440079048703, +STORE, 140112218095616, 140112219754495, +STORE, 140112219754496, 140112221851647, +STORE, 140112221851648, 140112221868031, +STORE, 140112221868032, 140112221876223, +STORE, 140112221876224, 140112221892607, +STORE, 140112221892608, 140112221904895, +STORE, 140112221904896, 140112223997951, +STORE, 140112223997952, 140112224002047, +STORE, 140112224002048, 140112224006143, +STORE, 140112224006144, 140112224149503, +STORE, 140112224518144, 140112226201599, +STORE, 140112226201600, 140112226217983, +STORE, 140112226246656, 140112226250751, +STORE, 140112226250752, 140112226254847, +STORE, 140112226254848, 140112226258943, +STORE, 140737460969472, 140737461108735, +STORE, 140737462083584, 140737462095871, +STORE, 140737462095872, 140737462099967, +STORE, 94257654345728, 94257654390783, +STORE, 94257656483840, 94257656487935, +STORE, 94257656487936, 94257656492031, +STORE, 94257656492032, 94257656496127, +STORE, 94257665859584, 94257665994751, +STORE, 140507070345216, 140507070386175, +STORE, 140507070386176, 140507072483327, +STORE, 140507072483328, 140507072487423, +STORE, 140507072487424, 140507072491519, +STORE, 140507072491520, 140507072516095, +STORE, 140507072516096, 140507072561151, +STORE, 140507072561152, 140507074654207, +STORE, 140507074654208, 140507074658303, +STORE, 140507074658304, 140507074662399, +STORE, 140507074662400, 140507074744319, +STORE, 140507074744320, 140507076841471, +STORE, 140507076841472, 140507076845567, +STORE, 140507076845568, 140507076849663, +STORE, 140507076849664, 140507076857855, +STORE, 140507076857856, 140507076886527, +STORE, 140507076886528, 140507078979583, +STORE, 140507078979584, 140507078983679, +STORE, 140507078983680, 140507078987775, +STORE, 140507078987776, 140507079086079, +STORE, 140507079086080, 140507081179135, +STORE, 140507081179136, 140507081183231, +STORE, 140507081183232, 140507081187327, +STORE, 140507081187328, 140507081203711, +STORE, 140507081203712, 140507081220095, +STORE, 140507081220096, 140507083317247, +STORE, 140507083317248, 140507083321343, +STORE, 140507083321344, 140507083325439, +STORE, 140507083325440, 140507083792383, +STORE, 140507083792384, 140507085885439, +STORE, 140507085885440, 140507085889535, +STORE, 140507085889536, 140507085893631, +STORE, 140507085893632, 140507085905919, +STORE, 140507085905920, 140507087998975, +STORE, 140507087998976, 140507088003071, +STORE, 140507088003072, 140507088007167, +STORE, 140507088007168, 140507088125951, +STORE, 140507088125952, 140507090219007, +STORE, 140507090219008, 140507090223103, +STORE, 140507090223104, 140507090227199, +STORE, 140507090227200, 140507090268159, +STORE, 140507090268160, 140507091927039, +STORE, 140507091927040, 140507094024191, +STORE, 140507094024192, 140507094040575, +STORE, 140507094040576, 140507094048767, +STORE, 140507094048768, 140507094065151, +STORE, 140507094065152, 140507094216703, +STORE, 140507094216704, 140507096309759, +STORE, 140507096309760, 140507096313855, +STORE, 140507096313856, 140507096317951, +STORE, 140507096317952, 140507096326143, +STORE, 140507096326144, 140507096379391, +STORE, 140507096379392, 140507098472447, +STORE, 140507098472448, 140507098476543, +STORE, 140507098476544, 140507098480639, +STORE, 140507098480640, 140507098623999, +STORE, 140507098980352, 140507100663807, +STORE, 140507100663808, 140507100692479, +STORE, 140507100721152, 140507100725247, +STORE, 140507100725248, 140507100729343, +STORE, 140507100729344, 140507100733439, +STORE, 140728152780800, 140728152915967, +STORE, 140728153698304, 140728153710591, +STORE, 140728153710592, 140728153714687, +STORE, 140507068137472, 140507070345215, +SNULL, 140507068137472, 140507068190719, +STORE, 140507068190720, 140507070345215, +STORE, 140507068137472, 140507068190719, +SNULL, 140507070287871, 140507070345215, +STORE, 140507068190720, 140507070287871, +STORE, 140507070287872, 140507070345215, +SNULL, 140507070287872, 140507070296063, +STORE, 140507070296064, 140507070345215, +STORE, 140507070287872, 140507070296063, +ERASE, 140507070287872, 140507070296063, +STORE, 140507070287872, 140507070296063, +ERASE, 140507070296064, 140507070345215, +STORE, 140507070296064, 140507070345215, +STORE, 140507100692480, 140507100721151, +STORE, 140507065810944, 140507068137471, +SNULL, 140507065810944, 140507065843711, +STORE, 140507065843712, 140507068137471, +STORE, 140507065810944, 140507065843711, +SNULL, 140507067940863, 140507068137471, +STORE, 140507065843712, 140507067940863, +STORE, 140507067940864, 140507068137471, +SNULL, 140507067940864, 140507067949055, +STORE, 140507067949056, 140507068137471, +STORE, 140507067940864, 140507067949055, +ERASE, 140507067940864, 140507067949055, +STORE, 140507067940864, 140507067949055, +ERASE, 140507067949056, 140507068137471, +STORE, 140507067949056, 140507068137471, +SNULL, 140507067944959, 140507067949055, +STORE, 140507067940864, 140507067944959, +STORE, 140507067944960, 140507067949055, +SNULL, 140507070291967, 140507070296063, +STORE, 140507070287872, 140507070291967, +STORE, 140507070291968, 140507070296063, +ERASE, 140507100692480, 140507100721151, +STORE, 140507063705600, 140507065810943, +SNULL, 140507063705600, 140507063709695, +STORE, 140507063709696, 140507065810943, +STORE, 140507063705600, 140507063709695, +SNULL, 140507065802751, 140507065810943, +STORE, 140507063709696, 140507065802751, +STORE, 140507065802752, 140507065810943, +ERASE, 140507065802752, 140507065810943, +STORE, 140507065802752, 140507065810943, +SNULL, 140507065806847, 140507065810943, +STORE, 140507065802752, 140507065806847, +STORE, 140507065806848, 140507065810943, +STORE, 140507061600256, 140507063705599, +SNULL, 140507061600256, 140507061604351, +STORE, 140507061604352, 140507063705599, +STORE, 140507061600256, 140507061604351, +SNULL, 140507063697407, 140507063705599, +STORE, 140507061604352, 140507063697407, +STORE, 140507063697408, 140507063705599, +ERASE, 140507063697408, 140507063705599, +STORE, 140507063697408, 140507063705599, +SNULL, 140507063701503, 140507063705599, +STORE, 140507063697408, 140507063701503, +STORE, 140507063701504, 140507063705599, +STORE, 140507059490816, 140507061600255, +SNULL, 140507059490816, 140507059499007, +STORE, 140507059499008, 140507061600255, +STORE, 140507059490816, 140507059499007, +SNULL, 140507061592063, 140507061600255, +STORE, 140507059499008, 140507061592063, +STORE, 140507061592064, 140507061600255, +ERASE, 140507061592064, 140507061600255, +STORE, 140507061592064, 140507061600255, +SNULL, 140507061596159, 140507061600255, +STORE, 140507061592064, 140507061596159, +STORE, 140507061596160, 140507061600255, +STORE, 140507057377280, 140507059490815, +SNULL, 140507057377280, 140507057389567, +STORE, 140507057389568, 140507059490815, +STORE, 140507057377280, 140507057389567, +SNULL, 140507059482623, 140507059490815, +STORE, 140507057389568, 140507059482623, +STORE, 140507059482624, 140507059490815, +ERASE, 140507059482624, 140507059490815, +STORE, 140507059482624, 140507059490815, +SNULL, 140507059486719, 140507059490815, +STORE, 140507059482624, 140507059486719, +STORE, 140507059486720, 140507059490815, +STORE, 140507055255552, 140507057377279, +SNULL, 140507055255552, 140507055276031, +STORE, 140507055276032, 140507057377279, +STORE, 140507055255552, 140507055276031, +SNULL, 140507057369087, 140507057377279, +STORE, 140507055276032, 140507057369087, +STORE, 140507057369088, 140507057377279, +ERASE, 140507057369088, 140507057377279, +STORE, 140507057369088, 140507057377279, +SNULL, 140507057373183, 140507057377279, +STORE, 140507057369088, 140507057373183, +STORE, 140507057373184, 140507057377279, +STORE, 140507098693632, 140507098980351, +SNULL, 140507098959871, 140507098980351, +STORE, 140507098693632, 140507098959871, +STORE, 140507098959872, 140507098980351, +SNULL, 140507098959872, 140507098976255, +STORE, 140507098976256, 140507098980351, +STORE, 140507098959872, 140507098976255, +ERASE, 140507098959872, 140507098976255, +STORE, 140507098959872, 140507098976255, +ERASE, 140507098976256, 140507098980351, +STORE, 140507098976256, 140507098980351, +STORE, 140507100692480, 140507100721151, +STORE, 140507053125632, 140507055255551, +SNULL, 140507053125632, 140507053154303, +STORE, 140507053154304, 140507055255551, +STORE, 140507053125632, 140507053154303, +SNULL, 140507055247359, 140507055255551, +STORE, 140507053154304, 140507055247359, +STORE, 140507055247360, 140507055255551, +ERASE, 140507055247360, 140507055255551, +STORE, 140507055247360, 140507055255551, +STORE, 140507051012096, 140507053125631, +SNULL, 140507051012096, 140507051024383, +STORE, 140507051024384, 140507053125631, +STORE, 140507051012096, 140507051024383, +SNULL, 140507053117439, 140507053125631, +STORE, 140507051024384, 140507053117439, +STORE, 140507053117440, 140507053125631, +ERASE, 140507053117440, 140507053125631, +STORE, 140507053117440, 140507053125631, +SNULL, 140507053121535, 140507053125631, +STORE, 140507053117440, 140507053121535, +STORE, 140507053121536, 140507053125631, +SNULL, 140507055251455, 140507055255551, +STORE, 140507055247360, 140507055251455, +STORE, 140507055251456, 140507055255551, +SNULL, 140507098972159, 140507098976255, +STORE, 140507098959872, 140507098972159, +STORE, 140507098972160, 140507098976255, +ERASE, 140507100692480, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100692480, 140507100721151, +ERASE, 140507068137472, 140507068190719, +ERASE, 140507068190720, 140507070287871, +ERASE, 140507070287872, 140507070291967, +ERASE, 140507070291968, 140507070296063, +ERASE, 140507070296064, 140507070345215, +ERASE, 140507065810944, 140507065843711, +ERASE, 140507065843712, 140507067940863, +ERASE, 140507067940864, 140507067944959, +ERASE, 140507067944960, 140507067949055, +ERASE, 140507067949056, 140507068137471, +ERASE, 140507063705600, 140507063709695, +ERASE, 140507063709696, 140507065802751, +ERASE, 140507065802752, 140507065806847, +ERASE, 140507065806848, 140507065810943, +ERASE, 140507061600256, 140507061604351, +ERASE, 140507061604352, 140507063697407, +ERASE, 140507063697408, 140507063701503, +ERASE, 140507063701504, 140507063705599, +ERASE, 140507059490816, 140507059499007, +ERASE, 140507059499008, 140507061592063, +ERASE, 140507061592064, 140507061596159, +ERASE, 140507061596160, 140507061600255, +ERASE, 140507057377280, 140507057389567, +ERASE, 140507057389568, 140507059482623, +ERASE, 140507059482624, 140507059486719, +ERASE, 140507059486720, 140507059490815, +ERASE, 140507055255552, 140507055276031, +ERASE, 140507055276032, 140507057369087, +ERASE, 140507057369088, 140507057373183, +ERASE, 140507057373184, 140507057377279, +ERASE, 140507098693632, 140507098959871, +ERASE, 140507098959872, 140507098972159, +ERASE, 140507098972160, 140507098976255, +ERASE, 140507098976256, 140507098980351, +ERASE, 140507051012096, 140507051024383, +ERASE, 140507051024384, 140507053117439, +ERASE, 140507053117440, 140507053121535, +ERASE, 140507053121536, 140507053125631, +STORE, 94036448296960, 94036448509951, +STORE, 94036450607104, 94036450611199, +STORE, 94036450611200, 94036450619391, +STORE, 94036450619392, 94036450631679, +STORE, 94036482445312, 94036502376447, +STORE, 140469487013888, 140469488672767, +STORE, 140469488672768, 140469490769919, +STORE, 140469490769920, 140469490786303, +STORE, 140469490786304, 140469490794495, +STORE, 140469490794496, 140469490810879, +STORE, 140469490810880, 140469490823167, +STORE, 140469490823168, 140469492916223, +STORE, 140469492916224, 140469492920319, +STORE, 140469492920320, 140469492924415, +STORE, 140469492924416, 140469493067775, +STORE, 140469493436416, 140469495119871, +STORE, 140469495119872, 140469495136255, +STORE, 140469495164928, 140469495169023, +STORE, 140469495169024, 140469495173119, +STORE, 140469495173120, 140469495177215, +STORE, 140732281446400, 140732281585663, +STORE, 140732282736640, 140732282748927, +STORE, 140732282748928, 140732282753023, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723411931136, 140737488351231, +SNULL, 140723411939327, 140737488351231, +STORE, 140723411931136, 140723411939327, +STORE, 140723411800064, 140723411939327, +STORE, 93993768685568, 93993770909695, +SNULL, 93993768796159, 93993770909695, +STORE, 93993768685568, 93993768796159, +STORE, 93993768796160, 93993770909695, +ERASE, 93993768796160, 93993770909695, +STORE, 93993770889216, 93993770901503, +STORE, 93993770901504, 93993770909695, +STORE, 140508681740288, 140508683993087, +SNULL, 140508681883647, 140508683993087, +STORE, 140508681740288, 140508681883647, +STORE, 140508681883648, 140508683993087, +ERASE, 140508681883648, 140508683993087, +STORE, 140508683980800, 140508683988991, +STORE, 140508683988992, 140508683993087, +STORE, 140723412070400, 140723412074495, +STORE, 140723412058112, 140723412070399, +STORE, 140508683952128, 140508683980799, +STORE, 140508683943936, 140508683952127, +STORE, 140508677943296, 140508681740287, +SNULL, 140508677943296, 140508679602175, +STORE, 140508679602176, 140508681740287, +STORE, 140508677943296, 140508679602175, +SNULL, 140508681699327, 140508681740287, +STORE, 140508679602176, 140508681699327, +STORE, 140508681699328, 140508681740287, +SNULL, 140508681699328, 140508681723903, +STORE, 140508681723904, 140508681740287, +STORE, 140508681699328, 140508681723903, +ERASE, 140508681699328, 140508681723903, +STORE, 140508681699328, 140508681723903, +ERASE, 140508681723904, 140508681740287, +STORE, 140508681723904, 140508681740287, +SNULL, 140508681715711, 140508681723903, +STORE, 140508681699328, 140508681715711, +STORE, 140508681715712, 140508681723903, +SNULL, 93993770897407, 93993770901503, +STORE, 93993770889216, 93993770897407, +STORE, 93993770897408, 93993770901503, +SNULL, 140508683984895, 140508683988991, +STORE, 140508683980800, 140508683984895, +STORE, 140508683984896, 140508683988991, +ERASE, 140508683952128, 140508683980799, +STORE, 93993791582208, 93993791717375, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734685458432, 140737488351231, +SNULL, 140734685466623, 140737488351231, +STORE, 140734685458432, 140734685466623, +STORE, 140734685327360, 140734685466623, +STORE, 93832321548288, 93832323772415, +SNULL, 93832321658879, 93832323772415, +STORE, 93832321548288, 93832321658879, +STORE, 93832321658880, 93832323772415, +ERASE, 93832321658880, 93832323772415, +STORE, 93832323751936, 93832323764223, +STORE, 93832323764224, 93832323772415, +STORE, 140650945118208, 140650947371007, +SNULL, 140650945261567, 140650947371007, +STORE, 140650945118208, 140650945261567, +STORE, 140650945261568, 140650947371007, +ERASE, 140650945261568, 140650947371007, +STORE, 140650947358720, 140650947366911, +STORE, 140650947366912, 140650947371007, +STORE, 140734686081024, 140734686085119, +STORE, 140734686068736, 140734686081023, +STORE, 140650947330048, 140650947358719, +STORE, 140650947321856, 140650947330047, +STORE, 140650941321216, 140650945118207, +SNULL, 140650941321216, 140650942980095, +STORE, 140650942980096, 140650945118207, +STORE, 140650941321216, 140650942980095, +SNULL, 140650945077247, 140650945118207, +STORE, 140650942980096, 140650945077247, +STORE, 140650945077248, 140650945118207, +SNULL, 140650945077248, 140650945101823, +STORE, 140650945101824, 140650945118207, +STORE, 140650945077248, 140650945101823, +ERASE, 140650945077248, 140650945101823, +STORE, 140650945077248, 140650945101823, +ERASE, 140650945101824, 140650945118207, +STORE, 140650945101824, 140650945118207, +SNULL, 140650945093631, 140650945101823, +STORE, 140650945077248, 140650945093631, +STORE, 140650945093632, 140650945101823, +SNULL, 93832323760127, 93832323764223, +STORE, 93832323751936, 93832323760127, +STORE, 93832323760128, 93832323764223, +SNULL, 140650947362815, 140650947366911, +STORE, 140650947358720, 140650947362815, +STORE, 140650947362816, 140650947366911, +ERASE, 140650947330048, 140650947358719, +STORE, 93832331890688, 93832332025855, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728333520896, 140737488351231, +SNULL, 140728333529087, 140737488351231, +STORE, 140728333520896, 140728333529087, +STORE, 140728333389824, 140728333529087, +STORE, 94872734732288, 94872736956415, +SNULL, 94872734842879, 94872736956415, +STORE, 94872734732288, 94872734842879, +STORE, 94872734842880, 94872736956415, +ERASE, 94872734842880, 94872736956415, +STORE, 94872736935936, 94872736948223, +STORE, 94872736948224, 94872736956415, +STORE, 139755193257984, 139755195510783, +SNULL, 139755193401343, 139755195510783, +STORE, 139755193257984, 139755193401343, +STORE, 139755193401344, 139755195510783, +ERASE, 139755193401344, 139755195510783, +STORE, 139755195498496, 139755195506687, +STORE, 139755195506688, 139755195510783, +STORE, 140728333926400, 140728333930495, +STORE, 140728333914112, 140728333926399, +STORE, 139755195469824, 139755195498495, +STORE, 139755195461632, 139755195469823, +STORE, 139755189460992, 139755193257983, +SNULL, 139755189460992, 139755191119871, +STORE, 139755191119872, 139755193257983, +STORE, 139755189460992, 139755191119871, +SNULL, 139755193217023, 139755193257983, +STORE, 139755191119872, 139755193217023, +STORE, 139755193217024, 139755193257983, +SNULL, 139755193217024, 139755193241599, +STORE, 139755193241600, 139755193257983, +STORE, 139755193217024, 139755193241599, +ERASE, 139755193217024, 139755193241599, +STORE, 139755193217024, 139755193241599, +ERASE, 139755193241600, 139755193257983, +STORE, 139755193241600, 139755193257983, +SNULL, 139755193233407, 139755193241599, +STORE, 139755193217024, 139755193233407, +STORE, 139755193233408, 139755193241599, +SNULL, 94872736944127, 94872736948223, +STORE, 94872736935936, 94872736944127, +STORE, 94872736944128, 94872736948223, +SNULL, 139755195502591, 139755195506687, +STORE, 139755195498496, 139755195502591, +STORE, 139755195502592, 139755195506687, +ERASE, 139755195469824, 139755195498495, +STORE, 94872749744128, 94872749879295, +STORE, 94720243642368, 94720243855359, +STORE, 94720245952512, 94720245956607, +STORE, 94720245956608, 94720245964799, +STORE, 94720245964800, 94720245977087, +STORE, 94720277745664, 94720278151167, +STORE, 140453174497280, 140453176156159, +STORE, 140453176156160, 140453178253311, +STORE, 140453178253312, 140453178269695, +STORE, 140453178269696, 140453178277887, +STORE, 140453178277888, 140453178294271, +STORE, 140453178294272, 140453178306559, +STORE, 140453178306560, 140453180399615, +STORE, 140453180399616, 140453180403711, +STORE, 140453180403712, 140453180407807, +STORE, 140453180407808, 140453180551167, +STORE, 140453180919808, 140453182603263, +STORE, 140453182603264, 140453182619647, +STORE, 140453182648320, 140453182652415, +STORE, 140453182652416, 140453182656511, +STORE, 140453182656512, 140453182660607, +STORE, 140733223923712, 140733224062975, +STORE, 140733224808448, 140733224820735, +STORE, 140733224820736, 140733224824831, +STORE, 94321091141632, 94321091354623, +STORE, 94321093451776, 94321093455871, +STORE, 94321093455872, 94321093464063, +STORE, 94321093464064, 94321093476351, +STORE, 94321115873280, 94321117229055, +STORE, 139695978840064, 139695980498943, +STORE, 139695980498944, 139695982596095, +STORE, 139695982596096, 139695982612479, +STORE, 139695982612480, 139695982620671, +STORE, 139695982620672, 139695982637055, +STORE, 139695982637056, 139695982649343, +STORE, 139695982649344, 139695984742399, +STORE, 139695984742400, 139695984746495, +STORE, 139695984746496, 139695984750591, +STORE, 139695984750592, 139695984893951, +STORE, 139695985262592, 139695986946047, +STORE, 139695986946048, 139695986962431, +STORE, 139695986991104, 139695986995199, +STORE, 139695986995200, 139695986999295, +STORE, 139695986999296, 139695987003391, +STORE, 140734650564608, 140734650703871, +STORE, 140734650785792, 140734650798079, +STORE, 140734650798080, 140734650802175, +STORE, 94523438456832, 94523438669823, +STORE, 94523440766976, 94523440771071, +STORE, 94523440771072, 94523440779263, +STORE, 94523440779264, 94523440791551, +STORE, 94523464544256, 94523465842687, +STORE, 140453231493120, 140453233151999, +STORE, 140453233152000, 140453235249151, +STORE, 140453235249152, 140453235265535, +STORE, 140453235265536, 140453235273727, +STORE, 140453235273728, 140453235290111, +STORE, 140453235290112, 140453235302399, +STORE, 140453235302400, 140453237395455, +STORE, 140453237395456, 140453237399551, +STORE, 140453237399552, 140453237403647, +STORE, 140453237403648, 140453237547007, +STORE, 140453237915648, 140453239599103, +STORE, 140453239599104, 140453239615487, +STORE, 140453239644160, 140453239648255, +STORE, 140453239648256, 140453239652351, +STORE, 140453239652352, 140453239656447, +STORE, 140734679445504, 140734679584767, +STORE, 140734680018944, 140734680031231, +STORE, 140734680031232, 140734680035327, +STORE, 94614776987648, 94614777200639, +STORE, 94614779297792, 94614779301887, +STORE, 94614779301888, 94614779310079, +STORE, 94614779310080, 94614779322367, +STORE, 94614798467072, 94614800699391, +STORE, 139677037182976, 139677038841855, +STORE, 139677038841856, 139677040939007, +STORE, 139677040939008, 139677040955391, +STORE, 139677040955392, 139677040963583, +STORE, 139677040963584, 139677040979967, +STORE, 139677040979968, 139677040992255, +STORE, 139677040992256, 139677043085311, +STORE, 139677043085312, 139677043089407, +STORE, 139677043089408, 139677043093503, +STORE, 139677043093504, 139677043236863, +STORE, 139677043605504, 139677045288959, +STORE, 139677045288960, 139677045305343, +STORE, 139677045334016, 139677045338111, +STORE, 139677045338112, 139677045342207, +STORE, 139677045342208, 139677045346303, +STORE, 140721604411392, 140721604550655, +STORE, 140721606135808, 140721606148095, +STORE, 140721606148096, 140721606152191, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729280544768, 140737488351231, +SNULL, 140729280552959, 140737488351231, +STORE, 140729280544768, 140729280552959, +STORE, 140729280413696, 140729280552959, +STORE, 94863939334144, 94863941558271, +SNULL, 94863939444735, 94863941558271, +STORE, 94863939334144, 94863939444735, +STORE, 94863939444736, 94863941558271, +ERASE, 94863939444736, 94863941558271, +STORE, 94863941537792, 94863941550079, +STORE, 94863941550080, 94863941558271, +STORE, 139691047276544, 139691049529343, +SNULL, 139691047419903, 139691049529343, +STORE, 139691047276544, 139691047419903, +STORE, 139691047419904, 139691049529343, +ERASE, 139691047419904, 139691049529343, +STORE, 139691049517056, 139691049525247, +STORE, 139691049525248, 139691049529343, +STORE, 140729281679360, 140729281683455, +STORE, 140729281667072, 140729281679359, +STORE, 139691049488384, 139691049517055, +STORE, 139691049480192, 139691049488383, +STORE, 139691043479552, 139691047276543, +SNULL, 139691043479552, 139691045138431, +STORE, 139691045138432, 139691047276543, +STORE, 139691043479552, 139691045138431, +SNULL, 139691047235583, 139691047276543, +STORE, 139691045138432, 139691047235583, +STORE, 139691047235584, 139691047276543, +SNULL, 139691047235584, 139691047260159, +STORE, 139691047260160, 139691047276543, +STORE, 139691047235584, 139691047260159, +ERASE, 139691047235584, 139691047260159, +STORE, 139691047235584, 139691047260159, +ERASE, 139691047260160, 139691047276543, +STORE, 139691047260160, 139691047276543, +SNULL, 139691047251967, 139691047260159, +STORE, 139691047235584, 139691047251967, +STORE, 139691047251968, 139691047260159, +SNULL, 94863941545983, 94863941550079, +STORE, 94863941537792, 94863941545983, +STORE, 94863941545984, 94863941550079, +SNULL, 139691049521151, 139691049525247, +STORE, 139691049517056, 139691049521151, +STORE, 139691049521152, 139691049525247, +ERASE, 139691049488384, 139691049517055, +STORE, 94863951294464, 94863951429631, +STORE, 93998209294336, 93998209507327, +STORE, 93998211604480, 93998211608575, +STORE, 93998211608576, 93998211616767, +STORE, 93998211616768, 93998211629055, +STORE, 93998227210240, 93998227615743, +STORE, 140243029913600, 140243031572479, +STORE, 140243031572480, 140243033669631, +STORE, 140243033669632, 140243033686015, +STORE, 140243033686016, 140243033694207, +STORE, 140243033694208, 140243033710591, +STORE, 140243033710592, 140243033722879, +STORE, 140243033722880, 140243035815935, +STORE, 140243035815936, 140243035820031, +STORE, 140243035820032, 140243035824127, +STORE, 140243035824128, 140243035967487, +STORE, 140243036336128, 140243038019583, +STORE, 140243038019584, 140243038035967, +STORE, 140243038064640, 140243038068735, +STORE, 140243038068736, 140243038072831, +STORE, 140243038072832, 140243038076927, +STORE, 140734976479232, 140734976618495, +STORE, 140734977978368, 140734977990655, +STORE, 140734977990656, 140734977994751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722742775808, 140737488351231, +SNULL, 140722742783999, 140737488351231, +STORE, 140722742775808, 140722742783999, +STORE, 140722742644736, 140722742783999, +STORE, 93857673662464, 93857675997183, +SNULL, 93857673875455, 93857675997183, +STORE, 93857673662464, 93857673875455, +STORE, 93857673875456, 93857675997183, +ERASE, 93857673875456, 93857675997183, +STORE, 93857675972608, 93857675984895, +STORE, 93857675984896, 93857675997183, +STORE, 140629677498368, 140629679751167, +SNULL, 140629677641727, 140629679751167, +STORE, 140629677498368, 140629677641727, +STORE, 140629677641728, 140629679751167, +ERASE, 140629677641728, 140629679751167, +STORE, 140629679738880, 140629679747071, +STORE, 140629679747072, 140629679751167, +STORE, 140722743222272, 140722743226367, +STORE, 140722743209984, 140722743222271, +STORE, 140629679710208, 140629679738879, +STORE, 140629679702016, 140629679710207, +STORE, 140629675384832, 140629677498367, +SNULL, 140629675384832, 140629675397119, +STORE, 140629675397120, 140629677498367, +STORE, 140629675384832, 140629675397119, +SNULL, 140629677490175, 140629677498367, +STORE, 140629675397120, 140629677490175, +STORE, 140629677490176, 140629677498367, +ERASE, 140629677490176, 140629677498367, +STORE, 140629677490176, 140629677498367, +STORE, 140629671587840, 140629675384831, +SNULL, 140629671587840, 140629673246719, +STORE, 140629673246720, 140629675384831, +STORE, 140629671587840, 140629673246719, +SNULL, 140629675343871, 140629675384831, +STORE, 140629673246720, 140629675343871, +STORE, 140629675343872, 140629675384831, +SNULL, 140629675343872, 140629675368447, +STORE, 140629675368448, 140629675384831, +STORE, 140629675343872, 140629675368447, +ERASE, 140629675343872, 140629675368447, +STORE, 140629675343872, 140629675368447, +ERASE, 140629675368448, 140629675384831, +STORE, 140629675368448, 140629675384831, +STORE, 140629679693824, 140629679710207, +SNULL, 140629675360255, 140629675368447, +STORE, 140629675343872, 140629675360255, +STORE, 140629675360256, 140629675368447, +SNULL, 140629677494271, 140629677498367, +STORE, 140629677490176, 140629677494271, +STORE, 140629677494272, 140629677498367, +SNULL, 93857675976703, 93857675984895, +STORE, 93857675972608, 93857675976703, +STORE, 93857675976704, 93857675984895, +SNULL, 140629679742975, 140629679747071, +STORE, 140629679738880, 140629679742975, +STORE, 140629679742976, 140629679747071, +ERASE, 140629679710208, 140629679738879, +STORE, 93857705832448, 93857705967615, +STORE, 140629678010368, 140629679693823, +STORE, 93857705832448, 93857706102783, +STORE, 93857705832448, 93857706237951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735922421760, 140737488351231, +SNULL, 140735922429951, 140737488351231, +STORE, 140735922421760, 140735922429951, +STORE, 140735922290688, 140735922429951, +STORE, 94651136139264, 94651138363391, +SNULL, 94651136249855, 94651138363391, +STORE, 94651136139264, 94651136249855, +STORE, 94651136249856, 94651138363391, +ERASE, 94651136249856, 94651138363391, +STORE, 94651138342912, 94651138355199, +STORE, 94651138355200, 94651138363391, +STORE, 140325788266496, 140325790519295, +SNULL, 140325788409855, 140325790519295, +STORE, 140325788266496, 140325788409855, +STORE, 140325788409856, 140325790519295, +ERASE, 140325788409856, 140325790519295, +STORE, 140325790507008, 140325790515199, +STORE, 140325790515200, 140325790519295, +STORE, 140735923572736, 140735923576831, +STORE, 140735923560448, 140735923572735, +STORE, 140325790478336, 140325790507007, +STORE, 140325790470144, 140325790478335, +STORE, 140325784469504, 140325788266495, +SNULL, 140325784469504, 140325786128383, +STORE, 140325786128384, 140325788266495, +STORE, 140325784469504, 140325786128383, +SNULL, 140325788225535, 140325788266495, +STORE, 140325786128384, 140325788225535, +STORE, 140325788225536, 140325788266495, +SNULL, 140325788225536, 140325788250111, +STORE, 140325788250112, 140325788266495, +STORE, 140325788225536, 140325788250111, +ERASE, 140325788225536, 140325788250111, +STORE, 140325788225536, 140325788250111, +ERASE, 140325788250112, 140325788266495, +STORE, 140325788250112, 140325788266495, +SNULL, 140325788241919, 140325788250111, +STORE, 140325788225536, 140325788241919, +STORE, 140325788241920, 140325788250111, +SNULL, 94651138351103, 94651138355199, +STORE, 94651138342912, 94651138351103, +STORE, 94651138351104, 94651138355199, +SNULL, 140325790511103, 140325790515199, +STORE, 140325790507008, 140325790511103, +STORE, 140325790511104, 140325790515199, +ERASE, 140325790478336, 140325790507007, +STORE, 94651146297344, 94651146432511, +STORE, 94212330168320, 94212330381311, +STORE, 94212332478464, 94212332482559, +STORE, 94212332482560, 94212332490751, +STORE, 94212332490752, 94212332503039, +STORE, 94212348891136, 94212349825023, +STORE, 140611630604288, 140611632263167, +STORE, 140611632263168, 140611634360319, +STORE, 140611634360320, 140611634376703, +STORE, 140611634376704, 140611634384895, +STORE, 140611634384896, 140611634401279, +STORE, 140611634401280, 140611634413567, +STORE, 140611634413568, 140611636506623, +STORE, 140611636506624, 140611636510719, +STORE, 140611636510720, 140611636514815, +STORE, 140611636514816, 140611636658175, +STORE, 140611637026816, 140611638710271, +STORE, 140611638710272, 140611638726655, +STORE, 140611638755328, 140611638759423, +STORE, 140611638759424, 140611638763519, +STORE, 140611638763520, 140611638767615, +STORE, 140726974533632, 140726974672895, +STORE, 140726974943232, 140726974955519, +STORE, 140726974955520, 140726974959615, +STORE, 94572463521792, 94572463734783, +STORE, 94572465831936, 94572465836031, +STORE, 94572465836032, 94572465844223, +STORE, 94572465844224, 94572465856511, +STORE, 94572491534336, 94572492865535, +STORE, 140644351492096, 140644353150975, +STORE, 140644353150976, 140644355248127, +STORE, 140644355248128, 140644355264511, +STORE, 140644355264512, 140644355272703, +STORE, 140644355272704, 140644355289087, +STORE, 140644355289088, 140644355301375, +STORE, 140644355301376, 140644357394431, +STORE, 140644357394432, 140644357398527, +STORE, 140644357398528, 140644357402623, +STORE, 140644357402624, 140644357545983, +STORE, 140644357914624, 140644359598079, +STORE, 140644359598080, 140644359614463, +STORE, 140644359643136, 140644359647231, +STORE, 140644359647232, 140644359651327, +STORE, 140644359651328, 140644359655423, +STORE, 140727841824768, 140727841964031, +STORE, 140727843188736, 140727843201023, +STORE, 140727843201024, 140727843205119, +STORE, 94144315457536, 94144315670527, +STORE, 94144317767680, 94144317771775, +STORE, 94144317771776, 94144317779967, +STORE, 94144317779968, 94144317792255, +STORE, 94144318369792, 94144320815103, +STORE, 140316717645824, 140316719304703, +STORE, 140316719304704, 140316721401855, +STORE, 140316721401856, 140316721418239, +STORE, 140316721418240, 140316721426431, +STORE, 140316721426432, 140316721442815, +STORE, 140316721442816, 140316721455103, +STORE, 140316721455104, 140316723548159, +STORE, 140316723548160, 140316723552255, +STORE, 140316723552256, 140316723556351, +STORE, 140316723556352, 140316723699711, +STORE, 140316724068352, 140316725751807, +STORE, 140316725751808, 140316725768191, +STORE, 140316725796864, 140316725800959, +STORE, 140316725800960, 140316725805055, +STORE, 140316725805056, 140316725809151, +STORE, 140725744283648, 140725744422911, +STORE, 140725745852416, 140725745864703, +STORE, 140725745864704, 140725745868799, +STORE, 94646858846208, 94646859059199, +STORE, 94646861156352, 94646861160447, +STORE, 94646861160448, 94646861168639, +STORE, 94646861168640, 94646861180927, +STORE, 94646879805440, 94646881894399, +STORE, 140435449745408, 140435451404287, +STORE, 140435451404288, 140435453501439, +STORE, 140435453501440, 140435453517823, +STORE, 140435453517824, 140435453526015, +STORE, 140435453526016, 140435453542399, +STORE, 140435453542400, 140435453554687, +STORE, 140435453554688, 140435455647743, +STORE, 140435455647744, 140435455651839, +STORE, 140435455651840, 140435455655935, +STORE, 140435455655936, 140435455799295, +STORE, 140435456167936, 140435457851391, +STORE, 140435457851392, 140435457867775, +STORE, 140435457896448, 140435457900543, +STORE, 140435457900544, 140435457904639, +STORE, 140435457904640, 140435457908735, +STORE, 140721033818112, 140721033957375, +STORE, 140721034018816, 140721034031103, +STORE, 140721034031104, 140721034035199, +STORE, 94872903438336, 94872903651327, +STORE, 94872905748480, 94872905752575, +STORE, 94872905752576, 94872905760767, +STORE, 94872905760768, 94872905773055, +STORE, 94872931246080, 94872931651583, +STORE, 139771607810048, 139771609468927, +STORE, 139771609468928, 139771611566079, +STORE, 139771611566080, 139771611582463, +STORE, 139771611582464, 139771611590655, +STORE, 139771611590656, 139771611607039, +STORE, 139771611607040, 139771611619327, +STORE, 139771611619328, 139771613712383, +STORE, 139771613712384, 139771613716479, +STORE, 139771613716480, 139771613720575, +STORE, 139771613720576, 139771613863935, +STORE, 139771614232576, 139771615916031, +STORE, 139771615916032, 139771615932415, +STORE, 139771615961088, 139771615965183, +STORE, 139771615965184, 139771615969279, +STORE, 139771615969280, 139771615973375, +STORE, 140725402931200, 140725403070463, +STORE, 140725403852800, 140725403865087, +STORE, 140725403865088, 140725403869183, +STORE, 94740737736704, 94740737949695, +STORE, 94740740046848, 94740740050943, +STORE, 94740740050944, 94740740059135, +STORE, 94740740059136, 94740740071423, +STORE, 94740743249920, 94740744724479, +STORE, 140640287010816, 140640288669695, +STORE, 140640288669696, 140640290766847, +STORE, 140640290766848, 140640290783231, +STORE, 140640290783232, 140640290791423, +STORE, 140640290791424, 140640290807807, +STORE, 140640290807808, 140640290820095, +STORE, 140640290820096, 140640292913151, +STORE, 140640292913152, 140640292917247, +STORE, 140640292917248, 140640292921343, +STORE, 140640292921344, 140640293064703, +STORE, 140640293433344, 140640295116799, +STORE, 140640295116800, 140640295133183, +STORE, 140640295161856, 140640295165951, +STORE, 140640295165952, 140640295170047, +STORE, 140640295170048, 140640295174143, +STORE, 140725133303808, 140725133443071, +STORE, 140725133684736, 140725133697023, +STORE, 140725133697024, 140725133701119, +STORE, 140737488347136, 140737488351231, +STORE, 140722826371072, 140737488351231, +SNULL, 140722826375167, 140737488351231, +STORE, 140722826371072, 140722826375167, +STORE, 140722826240000, 140722826375167, +STORE, 94113818611712, 94113820835839, +SNULL, 94113818722303, 94113820835839, +STORE, 94113818611712, 94113818722303, +STORE, 94113818722304, 94113820835839, +ERASE, 94113818722304, 94113820835839, +STORE, 94113820815360, 94113820827647, +STORE, 94113820827648, 94113820835839, +STORE, 139628194508800, 139628196761599, +SNULL, 139628194652159, 139628196761599, +STORE, 139628194508800, 139628194652159, +STORE, 139628194652160, 139628196761599, +ERASE, 139628194652160, 139628196761599, +STORE, 139628196749312, 139628196757503, +STORE, 139628196757504, 139628196761599, +STORE, 140722826727424, 140722826731519, +STORE, 140722826715136, 140722826727423, +STORE, 139628196720640, 139628196749311, +STORE, 139628196712448, 139628196720639, +STORE, 139628190711808, 139628194508799, +SNULL, 139628190711808, 139628192370687, +STORE, 139628192370688, 139628194508799, +STORE, 139628190711808, 139628192370687, +SNULL, 139628194467839, 139628194508799, +STORE, 139628192370688, 139628194467839, +STORE, 139628194467840, 139628194508799, +SNULL, 139628194467840, 139628194492415, +STORE, 139628194492416, 139628194508799, +STORE, 139628194467840, 139628194492415, +ERASE, 139628194467840, 139628194492415, +STORE, 139628194467840, 139628194492415, +ERASE, 139628194492416, 139628194508799, +STORE, 139628194492416, 139628194508799, +SNULL, 139628194484223, 139628194492415, +STORE, 139628194467840, 139628194484223, +STORE, 139628194484224, 139628194492415, +SNULL, 94113820823551, 94113820827647, +STORE, 94113820815360, 94113820823551, +STORE, 94113820823552, 94113820827647, +SNULL, 139628196753407, 139628196757503, +STORE, 139628196749312, 139628196753407, +STORE, 139628196753408, 139628196757503, +ERASE, 139628196720640, 139628196749311, +STORE, 94113830850560, 94113830985727, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731865833472, 140737488351231, +SNULL, 140731865841663, 140737488351231, +STORE, 140731865833472, 140731865841663, +STORE, 140731865702400, 140731865841663, +STORE, 94763339386880, 94763341611007, +SNULL, 94763339497471, 94763341611007, +STORE, 94763339386880, 94763339497471, +STORE, 94763339497472, 94763341611007, +ERASE, 94763339497472, 94763341611007, +STORE, 94763341590528, 94763341602815, +STORE, 94763341602816, 94763341611007, +STORE, 139778398486528, 139778400739327, +SNULL, 139778398629887, 139778400739327, +STORE, 139778398486528, 139778398629887, +STORE, 139778398629888, 139778400739327, +ERASE, 139778398629888, 139778400739327, +STORE, 139778400727040, 139778400735231, +STORE, 139778400735232, 139778400739327, +STORE, 140731865858048, 140731865862143, +STORE, 140731865845760, 140731865858047, +STORE, 139778400698368, 139778400727039, +STORE, 139778400690176, 139778400698367, +STORE, 139778394689536, 139778398486527, +SNULL, 139778394689536, 139778396348415, +STORE, 139778396348416, 139778398486527, +STORE, 139778394689536, 139778396348415, +SNULL, 139778398445567, 139778398486527, +STORE, 139778396348416, 139778398445567, +STORE, 139778398445568, 139778398486527, +SNULL, 139778398445568, 139778398470143, +STORE, 139778398470144, 139778398486527, +STORE, 139778398445568, 139778398470143, +ERASE, 139778398445568, 139778398470143, +STORE, 139778398445568, 139778398470143, +ERASE, 139778398470144, 139778398486527, +STORE, 139778398470144, 139778398486527, +SNULL, 139778398461951, 139778398470143, +STORE, 139778398445568, 139778398461951, +STORE, 139778398461952, 139778398470143, +SNULL, 94763341598719, 94763341602815, +STORE, 94763341590528, 94763341598719, +STORE, 94763341598720, 94763341602815, +SNULL, 139778400731135, 139778400735231, +STORE, 139778400727040, 139778400731135, +STORE, 139778400731136, 139778400735231, +ERASE, 139778400698368, 139778400727039, +STORE, 94763362197504, 94763362332671, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140732053192704, 140737488351231, +SNULL, 140732053204991, 140737488351231, +STORE, 140732053192704, 140732053204991, +STORE, 140732053061632, 140732053204991, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140176018599936, 140176020852735, +SNULL, 140176018743295, 140176020852735, +STORE, 140176018599936, 140176018743295, +STORE, 140176018743296, 140176020852735, +ERASE, 140176018743296, 140176020852735, +STORE, 140176020840448, 140176020848639, +STORE, 140176020848640, 140176020852735, +STORE, 140732053381120, 140732053385215, +STORE, 140732053368832, 140732053381119, +STORE, 140176020811776, 140176020840447, +STORE, 140176020803584, 140176020811775, +STORE, 140176014766080, 140176018599935, +SNULL, 140176014766080, 140176016474111, +STORE, 140176016474112, 140176018599935, +STORE, 140176014766080, 140176016474111, +SNULL, 140176018567167, 140176018599935, +STORE, 140176016474112, 140176018567167, +STORE, 140176018567168, 140176018599935, +ERASE, 140176018567168, 140176018599935, +STORE, 140176018567168, 140176018599935, +STORE, 140176012570624, 140176014766079, +SNULL, 140176012570624, 140176012664831, +STORE, 140176012664832, 140176014766079, +STORE, 140176012570624, 140176012664831, +SNULL, 140176014757887, 140176014766079, +STORE, 140176012664832, 140176014757887, +STORE, 140176014757888, 140176014766079, +ERASE, 140176014757888, 140176014766079, +STORE, 140176014757888, 140176014766079, +STORE, 140176010051584, 140176012570623, +SNULL, 140176010051584, 140176010465279, +STORE, 140176010465280, 140176012570623, +STORE, 140176010051584, 140176010465279, +SNULL, 140176012558335, 140176012570623, +STORE, 140176010465280, 140176012558335, +STORE, 140176012558336, 140176012570623, +ERASE, 140176012558336, 140176012570623, +STORE, 140176012558336, 140176012570623, +STORE, 140176007417856, 140176010051583, +SNULL, 140176007417856, 140176007946239, +STORE, 140176007946240, 140176010051583, +STORE, 140176007417856, 140176007946239, +SNULL, 140176010043391, 140176010051583, +STORE, 140176007946240, 140176010043391, +STORE, 140176010043392, 140176010051583, +ERASE, 140176010043392, 140176010051583, +STORE, 140176010043392, 140176010051583, +STORE, 140176005304320, 140176007417855, +SNULL, 140176005304320, 140176005316607, +STORE, 140176005316608, 140176007417855, +STORE, 140176005304320, 140176005316607, +SNULL, 140176007409663, 140176007417855, +STORE, 140176005316608, 140176007409663, +STORE, 140176007409664, 140176007417855, +ERASE, 140176007409664, 140176007417855, +STORE, 140176007409664, 140176007417855, +STORE, 140176003100672, 140176005304319, +SNULL, 140176003100672, 140176003203071, +STORE, 140176003203072, 140176005304319, +STORE, 140176003100672, 140176003203071, +SNULL, 140176005296127, 140176005304319, +STORE, 140176003203072, 140176005296127, +STORE, 140176005296128, 140176005304319, +ERASE, 140176005296128, 140176005304319, +STORE, 140176005296128, 140176005304319, +STORE, 140176020795392, 140176020811775, +STORE, 140175999938560, 140176003100671, +SNULL, 140175999938560, 140176000999423, +STORE, 140176000999424, 140176003100671, +STORE, 140175999938560, 140176000999423, +SNULL, 140176003092479, 140176003100671, +STORE, 140176000999424, 140176003092479, +STORE, 140176003092480, 140176003100671, +ERASE, 140176003092480, 140176003100671, +STORE, 140176003092480, 140176003100671, +STORE, 140175996141568, 140175999938559, +SNULL, 140175996141568, 140175997800447, +STORE, 140175997800448, 140175999938559, +STORE, 140175996141568, 140175997800447, +SNULL, 140175999897599, 140175999938559, +STORE, 140175997800448, 140175999897599, +STORE, 140175999897600, 140175999938559, +SNULL, 140175999897600, 140175999922175, +STORE, 140175999922176, 140175999938559, +STORE, 140175999897600, 140175999922175, +ERASE, 140175999897600, 140175999922175, +STORE, 140175999897600, 140175999922175, +ERASE, 140175999922176, 140175999938559, +STORE, 140175999922176, 140175999938559, +STORE, 140176020783104, 140176020811775, +SNULL, 140175999913983, 140175999922175, +STORE, 140175999897600, 140175999913983, +STORE, 140175999913984, 140175999922175, +SNULL, 140176003096575, 140176003100671, +STORE, 140176003092480, 140176003096575, +STORE, 140176003096576, 140176003100671, +SNULL, 140176005300223, 140176005304319, +STORE, 140176005296128, 140176005300223, +STORE, 140176005300224, 140176005304319, +SNULL, 140176007413759, 140176007417855, +STORE, 140176007409664, 140176007413759, +STORE, 140176007413760, 140176007417855, +SNULL, 140176010047487, 140176010051583, +STORE, 140176010043392, 140176010047487, +STORE, 140176010047488, 140176010051583, +SNULL, 140176012566527, 140176012570623, +STORE, 140176012558336, 140176012566527, +STORE, 140176012566528, 140176012570623, +SNULL, 140176014761983, 140176014766079, +STORE, 140176014757888, 140176014761983, +STORE, 140176014761984, 140176014766079, +SNULL, 140176018571263, 140176018599935, +STORE, 140176018567168, 140176018571263, +STORE, 140176018571264, 140176018599935, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140176020844543, 140176020848639, +STORE, 140176020840448, 140176020844543, +STORE, 140176020844544, 140176020848639, +ERASE, 140176020811776, 140176020840447, +STORE, 53080064, 53215231, +STORE, 140176019099648, 140176020783103, +STORE, 140176020836352, 140176020840447, +STORE, 140176018964480, 140176019099647, +STORE, 53080064, 53358591, +STORE, 140175994044416, 140175996141567, +STORE, 140176020828160, 140176020840447, +STORE, 140176020819968, 140176020840447, +STORE, 140176020783104, 140176020819967, +STORE, 140176018948096, 140176019099647, +STORE, 53080064, 53493759, +STORE, 53080064, 53649407, +STORE, 140176018939904, 140176019099647, +STORE, 140176018931712, 140176019099647, +STORE, 53080064, 53784575, +STORE, 53080064, 53919743, +STORE, 140176018915328, 140176019099647, +STORE, 140176018907136, 140176019099647, +STORE, 53080064, 54059007, +STORE, 140175993769984, 140175996141567, +STORE, 140176018747392, 140176019099647, +STORE, 53080064, 54198271, +SNULL, 54190079, 54198271, +STORE, 53080064, 54190079, +STORE, 54190080, 54198271, +ERASE, 54190080, 54198271, +SNULL, 54181887, 54190079, +STORE, 53080064, 54181887, +STORE, 54181888, 54190079, +ERASE, 54181888, 54190079, +SNULL, 54173695, 54181887, +STORE, 53080064, 54173695, +STORE, 54173696, 54181887, +ERASE, 54173696, 54181887, +SNULL, 54165503, 54173695, +STORE, 53080064, 54165503, +STORE, 54165504, 54173695, +ERASE, 54165504, 54173695, +STORE, 140175993753600, 140175996141567, +STORE, 140175993688064, 140175996141567, +STORE, 140175993655296, 140175996141567, +STORE, 140175991558144, 140175996141567, +STORE, 140175991492608, 140175996141567, +STORE, 53080064, 54312959, +STORE, 140175991361536, 140175996141567, +STORE, 140175991099392, 140175996141567, +STORE, 140175991091200, 140175996141567, +STORE, 140175991074816, 140175996141567, +STORE, 140175991066624, 140175996141567, +STORE, 140175991058432, 140175996141567, +STORE, 53080064, 54448127, +SNULL, 54439935, 54448127, +STORE, 53080064, 54439935, +STORE, 54439936, 54448127, +ERASE, 54439936, 54448127, +SNULL, 54431743, 54439935, +STORE, 53080064, 54431743, +STORE, 54431744, 54439935, +ERASE, 54431744, 54439935, +SNULL, 54419455, 54431743, +STORE, 53080064, 54419455, +STORE, 54419456, 54431743, +ERASE, 54419456, 54431743, +SNULL, 54403071, 54419455, +STORE, 53080064, 54403071, +STORE, 54403072, 54419455, +ERASE, 54403072, 54419455, +STORE, 140175991042048, 140175996141567, +STORE, 53080064, 54538239, +SNULL, 54534143, 54538239, +STORE, 53080064, 54534143, +STORE, 54534144, 54538239, +ERASE, 54534144, 54538239, +SNULL, 54530047, 54534143, +STORE, 53080064, 54530047, +STORE, 54530048, 54534143, +ERASE, 54530048, 54534143, +SNULL, 54525951, 54530047, +STORE, 53080064, 54525951, +STORE, 54525952, 54530047, +ERASE, 54525952, 54530047, +SNULL, 54521855, 54525951, +STORE, 53080064, 54521855, +STORE, 54521856, 54525951, +ERASE, 54521856, 54525951, +SNULL, 54517759, 54521855, +STORE, 53080064, 54517759, +STORE, 54517760, 54521855, +ERASE, 54517760, 54521855, +SNULL, 54513663, 54517759, +STORE, 53080064, 54513663, +STORE, 54513664, 54517759, +ERASE, 54513664, 54517759, +SNULL, 54509567, 54513663, +STORE, 53080064, 54509567, +STORE, 54509568, 54513663, +ERASE, 54509568, 54513663, +STORE, 140175991025664, 140175996141567, +STORE, 140175990992896, 140175996141567, +STORE, 53080064, 54644735, +SNULL, 54628351, 54644735, +STORE, 53080064, 54628351, +STORE, 54628352, 54644735, +ERASE, 54628352, 54644735, +SNULL, 54616063, 54628351, +STORE, 53080064, 54616063, +STORE, 54616064, 54628351, +ERASE, 54616064, 54628351, +STORE, 140175988895744, 140175996141567, +STORE, 53080064, 54767615, +STORE, 140175988879360, 140175996141567, +STORE, 140175988617216, 140175996141567, +STORE, 140175988609024, 140175996141567, +STORE, 140175988600832, 140175996141567, +STORE, 53080064, 54906879, +SNULL, 54898687, 54906879, +STORE, 53080064, 54898687, +STORE, 54898688, 54906879, +ERASE, 54898688, 54906879, +SNULL, 54853631, 54898687, +STORE, 53080064, 54853631, +STORE, 54853632, 54898687, +ERASE, 54853632, 54898687, +STORE, 140175986503680, 140175996141567, +STORE, 53080064, 54996991, +STORE, 140175986495488, 140175996141567, +STORE, 140175986487296, 140175996141567, +STORE, 140175985438720, 140175996141567, +STORE, 53080064, 55136255, +STORE, 140175985405952, 140175996141567, +STORE, 140175985139712, 140175996141567, +SNULL, 140176018964479, 140176019099647, +STORE, 140176018747392, 140176018964479, +STORE, 140176018964480, 140176019099647, +ERASE, 140176018964480, 140176019099647, +STORE, 140175983042560, 140175996141567, +STORE, 140175982518272, 140175996141567, +STORE, 140175980421120, 140175996141567, +STORE, 53080064, 55287807, +STORE, 53080064, 55427071, +STORE, 140176019091456, 140176019099647, +STORE, 140176019083264, 140176019099647, +STORE, 140176019075072, 140176019099647, +STORE, 140176019066880, 140176019099647, +STORE, 140176019058688, 140176019099647, +STORE, 140175980158976, 140175996141567, +STORE, 140176019050496, 140176019099647, +STORE, 140176019042304, 140176019099647, +STORE, 140176019034112, 140176019099647, +STORE, 140176019025920, 140176019099647, +STORE, 140176019017728, 140176019099647, +STORE, 140176019009536, 140176019099647, +STORE, 140176019001344, 140176019099647, +STORE, 140176018993152, 140176019099647, +STORE, 140176018984960, 140176019099647, +STORE, 140176018976768, 140176019099647, +STORE, 140176018968576, 140176019099647, +STORE, 140175978061824, 140175996141567, +STORE, 53080064, 55603199, +STORE, 140175978029056, 140175996141567, +STORE, 140175977996288, 140175996141567, +STORE, 53080064, 55738367, +STORE, 53080064, 55881727, +STORE, 140175977963520, 140175996141567, +STORE, 140175977930752, 140175996141567, +STORE, 53080064, 56041471, +STORE, 140175977897984, 140175996141567, +STORE, 140175977865216, 140175996141567, +SNULL, 55881727, 56041471, +STORE, 53080064, 55881727, +STORE, 55881728, 56041471, +ERASE, 55881728, 56041471, +SNULL, 55721983, 55881727, +STORE, 53080064, 55721983, +STORE, 55721984, 55881727, +ERASE, 55721984, 55881727, +SNULL, 55570431, 55721983, +STORE, 53080064, 55570431, +STORE, 55570432, 55721983, +ERASE, 55570432, 55721983, +STORE, 140175977857024, 140175996141567, +STORE, 140175975759872, 140175996141567, +STORE, 53080064, 55754751, +STORE, 53080064, 55943167, +STORE, 140175975751680, 140175996141567, +STORE, 140175975743488, 140175996141567, +STORE, 140175975735296, 140175996141567, +STORE, 140175975727104, 140175996141567, +STORE, 140175975718912, 140175996141567, +STORE, 140175975710720, 140175996141567, +STORE, 140175975702528, 140175996141567, +STORE, 140175975694336, 140175996141567, +STORE, 140175975686144, 140175996141567, +STORE, 140175975677952, 140175996141567, +STORE, 140175975669760, 140175996141567, +STORE, 140175974621184, 140175996141567, +STORE, 140175974612992, 140175996141567, +STORE, 53080064, 56139775, +STORE, 140175972515840, 140175996141567, +STORE, 53080064, 56401919, +STORE, 140175970418688, 140175996141567, +STORE, 140175970410496, 140175996141567, +STORE, 140175970402304, 140175996141567, +STORE, 140175970394112, 140175996141567, +STORE, 53080064, 56569855, +STORE, 140175969865728, 140175996141567, +SNULL, 140175985139711, 140175996141567, +STORE, 140175969865728, 140175985139711, +STORE, 140175985139712, 140175996141567, +SNULL, 140175985139712, 140175985405951, +STORE, 140175985405952, 140175996141567, +STORE, 140175985139712, 140175985405951, +ERASE, 140175985139712, 140175985405951, +STORE, 140175965671424, 140175985139711, +STORE, 140175985397760, 140175996141567, +STORE, 140175985389568, 140175996141567, +STORE, 140175985381376, 140175996141567, +STORE, 140175985373184, 140175996141567, +STORE, 140175985364992, 140175996141567, +STORE, 140175985356800, 140175996141567, +STORE, 140175985348608, 140175996141567, +STORE, 140175985340416, 140175996141567, +STORE, 140175985332224, 140175996141567, +STORE, 140175985324032, 140175996141567, +STORE, 140175985315840, 140175996141567, +STORE, 140175985307648, 140175996141567, +STORE, 140175985299456, 140175996141567, +STORE, 140175985291264, 140175996141567, +STORE, 140175985283072, 140175996141567, +STORE, 140175985274880, 140175996141567, +STORE, 140175963574272, 140175985139711, +STORE, 140175985266688, 140175996141567, +STORE, 140175961477120, 140175985139711, +STORE, 53080064, 56831999, +STORE, 140175959379968, 140175985139711, +STORE, 140175985258496, 140175996141567, +STORE, 140175957282816, 140175985139711, +STORE, 140175985250304, 140175996141567, +STORE, 140175985242112, 140175996141567, +STORE, 140175985233920, 140175996141567, +STORE, 140175985225728, 140175996141567, +STORE, 140175985217536, 140175996141567, +STORE, 140175957151744, 140175985139711, +STORE, 140175956627456, 140175985139711, +SNULL, 140175980158975, 140175985139711, +STORE, 140175956627456, 140175980158975, +STORE, 140175980158976, 140175985139711, +SNULL, 140175980158976, 140175980421119, +STORE, 140175980421120, 140175985139711, +STORE, 140175980158976, 140175980421119, +ERASE, 140175980158976, 140175980421119, +STORE, 140175954530304, 140175980158975, +STORE, 140175985209344, 140175996141567, +STORE, 53080064, 57094143, +STORE, 140175952433152, 140175980158975, +STORE, 140175985192960, 140175996141567, +STORE, 140175985184768, 140175996141567, +STORE, 140175985176576, 140175996141567, +STORE, 140175985168384, 140175996141567, +STORE, 140175985160192, 140175996141567, +STORE, 140175985152000, 140175996141567, +STORE, 140175985143808, 140175996141567, +STORE, 140175980412928, 140175985139711, +STORE, 140175980404736, 140175985139711, +STORE, 140175980396544, 140175985139711, +STORE, 140175980388352, 140175985139711, +STORE, 140175980380160, 140175985139711, +STORE, 140175980371968, 140175985139711, +STORE, 140175980363776, 140175985139711, +STORE, 140175980355584, 140175985139711, +STORE, 140175980347392, 140175985139711, +STORE, 140175980339200, 140175985139711, +STORE, 53080064, 57356287, +SNULL, 140176018747392, 140176018907135, +STORE, 140176018907136, 140176018964479, +STORE, 140176018747392, 140176018907135, +ERASE, 140176018747392, 140176018907135, +STORE, 140175952146432, 140175980158975, +STORE, 140175950049280, 140175980158975, +SNULL, 140175952146431, 140175980158975, +STORE, 140175950049280, 140175952146431, +STORE, 140175952146432, 140175980158975, +SNULL, 140175952146432, 140175952433151, +STORE, 140175952433152, 140175980158975, +STORE, 140175952146432, 140175952433151, +ERASE, 140175952146432, 140175952433151, +STORE, 140176018898944, 140176018964479, +STORE, 53080064, 57749503, +STORE, 140175949520896, 140175952146431, +STORE, 140175947423744, 140175952146431, +SNULL, 140175993769983, 140175996141567, +STORE, 140175985143808, 140175993769983, +STORE, 140175993769984, 140175996141567, +SNULL, 140175993769984, 140175994044415, +STORE, 140175994044416, 140175996141567, +STORE, 140175993769984, 140175994044415, +ERASE, 140175993769984, 140175994044415, +STORE, 140176018890752, 140176018964479, +STORE, 140176018882560, 140176018964479, +STORE, 140176018874368, 140176018964479, +STORE, 140176018866176, 140176018964479, +STORE, 140176018849792, 140176018964479, +STORE, 140176018841600, 140176018964479, +STORE, 140176018825216, 140176018964479, +STORE, 140176018817024, 140176018964479, +STORE, 140176018800640, 140176018964479, +STORE, 140176018792448, 140176018964479, +STORE, 140176018759680, 140176018964479, +STORE, 140176018751488, 140176018964479, +STORE, 140175994028032, 140175996141567, +STORE, 140176018743296, 140176018964479, +STORE, 140175994011648, 140175996141567, +STORE, 140175994003456, 140175996141567, +STORE, 140175993987072, 140175996141567, +STORE, 140175993978880, 140175996141567, +STORE, 140175993946112, 140175996141567, +STORE, 140175993937920, 140175996141567, +STORE, 140175993921536, 140175996141567, +STORE, 140175993913344, 140175996141567, +STORE, 140175993896960, 140175996141567, +STORE, 140175993888768, 140175996141567, +STORE, 140175993872384, 140175996141567, +STORE, 140175993864192, 140175996141567, +STORE, 140175993831424, 140175996141567, +STORE, 140175993823232, 140175996141567, +STORE, 140175993806848, 140175996141567, +STORE, 140175993798656, 140175996141567, +STORE, 140175993782272, 140175996141567, +STORE, 140175993774080, 140175996141567, +STORE, 140175980322816, 140175985139711, +STORE, 140175980314624, 140175985139711, +STORE, 140175980281856, 140175985139711, +STORE, 140175980273664, 140175985139711, +STORE, 140175980257280, 140175985139711, +STORE, 140175945326592, 140175952146431, +STORE, 140175980249088, 140175985139711, +STORE, 140175980232704, 140175985139711, +STORE, 140175980224512, 140175985139711, +STORE, 140175980208128, 140175985139711, +STORE, 140175980199936, 140175985139711, +STORE, 140175980167168, 140175985139711, +STORE, 140175952433152, 140175985139711, +STORE, 140175952416768, 140175985139711, +STORE, 140175952408576, 140175985139711, +STORE, 140175952392192, 140175985139711, +STORE, 140175952384000, 140175985139711, +STORE, 140175952367616, 140175985139711, +STORE, 140175943229440, 140175952146431, +STORE, 140175952359424, 140175985139711, +STORE, 140175952326656, 140175985139711, +STORE, 140175952318464, 140175985139711, +STORE, 140175952302080, 140175985139711, +STORE, 140175952293888, 140175985139711, +STORE, 140175952277504, 140175985139711, +STORE, 140175952269312, 140175985139711, +STORE, 140175952252928, 140175985139711, +STORE, 140175952244736, 140175985139711, +STORE, 140175952211968, 140175985139711, +STORE, 140175952203776, 140175985139711, +STORE, 140175952187392, 140175985139711, +STORE, 140175952179200, 140175985139711, +STORE, 140175952162816, 140175985139711, +STORE, 140175952154624, 140175985139711, +STORE, 140175943213056, 140175952146431, +STORE, 140175943213056, 140175985139711, +STORE, 140175943180288, 140175985139711, +STORE, 140175943172096, 140175985139711, +STORE, 140175943155712, 140175985139711, +STORE, 140175943147520, 140175985139711, +STORE, 140175943131136, 140175985139711, +STORE, 140175943122944, 140175985139711, +STORE, 140175943106560, 140175985139711, +STORE, 140175943098368, 140175985139711, +STORE, 140175943065600, 140175985139711, +STORE, 140175943057408, 140175985139711, +STORE, 140175943041024, 140175985139711, +STORE, 140175943032832, 140175985139711, +STORE, 140175943016448, 140175985139711, +STORE, 140175943008256, 140175985139711, +STORE, 140175942991872, 140175985139711, +STORE, 140175942983680, 140175985139711, +STORE, 140175942950912, 140175985139711, +STORE, 140175942942720, 140175985139711, +STORE, 140175942926336, 140175985139711, +STORE, 140175942918144, 140175985139711, +STORE, 140175942901760, 140175985139711, +STORE, 140175942893568, 140175985139711, +STORE, 140175942877184, 140175985139711, +STORE, 140175942868992, 140175985139711, +STORE, 140175942836224, 140175985139711, +STORE, 140175942828032, 140175985139711, +STORE, 140175942811648, 140175985139711, +STORE, 140175942803456, 140175985139711, +STORE, 140175942787072, 140175985139711, +STORE, 140175942778880, 140175985139711, +STORE, 140175942762496, 140175985139711, +STORE, 140175942754304, 140175985139711, +STORE, 140175942721536, 140175985139711, +STORE, 140175942713344, 140175985139711, +STORE, 140175942696960, 140175985139711, +STORE, 140175942688768, 140175985139711, +STORE, 140175942672384, 140175985139711, +STORE, 140175942664192, 140175985139711, +STORE, 140175942647808, 140175985139711, +STORE, 140175942639616, 140175985139711, +STORE, 140175942606848, 140175985139711, +STORE, 140175942598656, 140175985139711, +STORE, 140175942582272, 140175985139711, +STORE, 140175942574080, 140175985139711, +STORE, 140175942557696, 140175985139711, +STORE, 140175942549504, 140175985139711, +STORE, 140175942533120, 140175985139711, +STORE, 140175942524928, 140175985139711, +STORE, 140175942492160, 140175985139711, +STORE, 140175942483968, 140175985139711, +STORE, 140175942467584, 140175985139711, +STORE, 140175942459392, 140175985139711, +STORE, 140175942443008, 140175985139711, +STORE, 140175942434816, 140175985139711, +STORE, 140175942418432, 140175985139711, +STORE, 140175942410240, 140175985139711, +STORE, 140175942377472, 140175985139711, +STORE, 140175942369280, 140175985139711, +STORE, 140175942352896, 140175985139711, +STORE, 140175942344704, 140175985139711, +STORE, 140175942328320, 140175985139711, +STORE, 140175942320128, 140175985139711, +STORE, 140175942303744, 140175985139711, +STORE, 140175942295552, 140175985139711, +STORE, 140175942262784, 140175985139711, +STORE, 140175942254592, 140175985139711, +STORE, 140175942238208, 140175985139711, +STORE, 140175942230016, 140175985139711, +STORE, 140175942213632, 140175985139711, +STORE, 140175942205440, 140175985139711, +STORE, 140175942189056, 140175985139711, +STORE, 140175942180864, 140175985139711, +STORE, 140175942148096, 140175985139711, +STORE, 140175942139904, 140175985139711, +STORE, 140175942123520, 140175985139711, +STORE, 140175942115328, 140175985139711, +STORE, 140175942098944, 140175985139711, +STORE, 140175942090752, 140175985139711, +STORE, 140175942074368, 140175985139711, +STORE, 140175942066176, 140175985139711, +STORE, 140175942033408, 140175985139711, +STORE, 140175942025216, 140175985139711, +STORE, 140175942008832, 140175985139711, +STORE, 140175942000640, 140175985139711, +STORE, 140175941984256, 140175985139711, +STORE, 140175941976064, 140175985139711, +STORE, 140175941959680, 140175985139711, +STORE, 140175939862528, 140175985139711, +STORE, 140175939854336, 140175985139711, +STORE, 140175939821568, 140175985139711, +STORE, 140175939813376, 140175985139711, +STORE, 140175939796992, 140175985139711, +STORE, 140175939788800, 140175985139711, +STORE, 140175939772416, 140175985139711, +STORE, 140175939764224, 140175985139711, +STORE, 140175939747840, 140175985139711, +STORE, 140175939739648, 140175985139711, +STORE, 140175939706880, 140175985139711, +STORE, 140175939698688, 140175985139711, +STORE, 140175939682304, 140175985139711, +STORE, 140175939674112, 140175985139711, +STORE, 140175939657728, 140175985139711, +STORE, 140175939649536, 140175985139711, +STORE, 140175939633152, 140175985139711, +STORE, 140175939624960, 140175985139711, +STORE, 140175939592192, 140175985139711, +STORE, 140175939584000, 140175985139711, +STORE, 140175939567616, 140175985139711, +STORE, 140175939559424, 140175985139711, +STORE, 140175939543040, 140175985139711, +STORE, 140175939534848, 140175985139711, +STORE, 140175939518464, 140175985139711, +STORE, 140175939510272, 140175985139711, +STORE, 140175939477504, 140175985139711, +STORE, 140175939469312, 140175985139711, +STORE, 140175939452928, 140175985139711, +STORE, 140175939444736, 140175985139711, +STORE, 140175939428352, 140175985139711, +STORE, 140175939420160, 140175985139711, +STORE, 140175939403776, 140175985139711, +STORE, 140175939395584, 140175985139711, +STORE, 140175939362816, 140175985139711, +STORE, 140175939354624, 140175985139711, +STORE, 140175939338240, 140175985139711, +STORE, 140175939330048, 140175985139711, +STORE, 140175939313664, 140175985139711, +STORE, 140175939305472, 140175985139711, +STORE, 140175939289088, 140175985139711, +STORE, 140175939280896, 140175985139711, +STORE, 140175939248128, 140175985139711, +STORE, 140175939239936, 140175985139711, +STORE, 140175939223552, 140175985139711, +STORE, 140175939215360, 140175985139711, +STORE, 140175939198976, 140175985139711, +STORE, 140175939190784, 140175985139711, +STORE, 140175939174400, 140175985139711, +STORE, 140175939166208, 140175985139711, +STORE, 140175939133440, 140175985139711, +STORE, 140175939125248, 140175985139711, +STORE, 140175939108864, 140175985139711, +STORE, 140175939100672, 140175985139711, +STORE, 140175939084288, 140175985139711, +STORE, 140175939076096, 140175985139711, +STORE, 140175939059712, 140175985139711, +STORE, 140175939051520, 140175985139711, +STORE, 140175939018752, 140175985139711, +STORE, 140175939010560, 140175985139711, +STORE, 140175938994176, 140175985139711, +STORE, 140175938985984, 140175985139711, +STORE, 140175938969600, 140175985139711, +STORE, 140175938961408, 140175985139711, +STORE, 140175938945024, 140175985139711, +STORE, 140175938936832, 140175985139711, +STORE, 140175938904064, 140175985139711, +STORE, 140175938895872, 140175985139711, +STORE, 140175938879488, 140175985139711, +STORE, 140175938871296, 140175985139711, +STORE, 140175938854912, 140175985139711, +STORE, 140175938846720, 140175985139711, +STORE, 140175938830336, 140175985139711, +STORE, 140175938822144, 140175985139711, +STORE, 140175938789376, 140175985139711, +STORE, 140175938781184, 140175985139711, +STORE, 140175938764800, 140175985139711, +STORE, 140175938756608, 140175985139711, +STORE, 140175938740224, 140175985139711, +STORE, 140175938732032, 140175985139711, +STORE, 140175938715648, 140175985139711, +STORE, 140175938707456, 140175985139711, +STORE, 140175938674688, 140175985139711, +STORE, 140175938666496, 140175985139711, +STORE, 140175938650112, 140175985139711, +STORE, 140175938641920, 140175985139711, +STORE, 140175938625536, 140175985139711, +STORE, 140175938617344, 140175985139711, +STORE, 140175938600960, 140175985139711, +STORE, 140175938592768, 140175985139711, +STORE, 140175938560000, 140175985139711, +STORE, 140175938551808, 140175985139711, +STORE, 140175938535424, 140175985139711, +STORE, 140175938527232, 140175985139711, +STORE, 140175938510848, 140175985139711, +STORE, 140175938502656, 140175985139711, +STORE, 140175938486272, 140175985139711, +STORE, 140175938478080, 140175985139711, +STORE, 140175938445312, 140175985139711, +STORE, 140175938437120, 140175985139711, +STORE, 140175938420736, 140175985139711, +STORE, 140175938412544, 140175985139711, +STORE, 140175938396160, 140175985139711, +STORE, 140175938387968, 140175985139711, +STORE, 140175938371584, 140175985139711, +STORE, 140175938363392, 140175985139711, +STORE, 140175938330624, 140175985139711, +STORE, 140175938322432, 140175985139711, +STORE, 140175938306048, 140175985139711, +STORE, 140175938297856, 140175985139711, +STORE, 140175938281472, 140175985139711, +STORE, 140175938273280, 140175985139711, +STORE, 140175938256896, 140175985139711, +STORE, 140175938248704, 140175985139711, +STORE, 140175938215936, 140175985139711, +STORE, 140175938207744, 140175985139711, +STORE, 140175938191360, 140175985139711, +STORE, 140175938183168, 140175985139711, +STORE, 140175938166784, 140175985139711, +STORE, 140175938158592, 140175985139711, +STORE, 140175938142208, 140175985139711, +STORE, 140175936045056, 140175985139711, +STORE, 140175936036864, 140175985139711, +STORE, 140175936004096, 140175985139711, +STORE, 140175935995904, 140175985139711, +STORE, 140175935979520, 140175985139711, +STORE, 140175935971328, 140175985139711, +STORE, 140175935954944, 140175985139711, +STORE, 140175935946752, 140175985139711, +STORE, 140175935930368, 140175985139711, +STORE, 140175935922176, 140175985139711, +STORE, 140175935889408, 140175985139711, +STORE, 140175935881216, 140175985139711, +STORE, 140175935864832, 140175985139711, +STORE, 140175935856640, 140175985139711, +STORE, 140175935840256, 140175985139711, +STORE, 140175935832064, 140175985139711, +STORE, 140175935815680, 140175985139711, +STORE, 140175935807488, 140175985139711, +STORE, 140175935774720, 140175985139711, +STORE, 140175935766528, 140175985139711, +STORE, 140175935750144, 140175985139711, +STORE, 140175935741952, 140175985139711, +STORE, 140175935725568, 140175985139711, +STORE, 140175935717376, 140175985139711, +STORE, 140175935700992, 140175985139711, +STORE, 140175935692800, 140175985139711, +STORE, 140175935660032, 140175985139711, +STORE, 140175935651840, 140175985139711, +STORE, 140175935635456, 140175985139711, +STORE, 140175935627264, 140175985139711, +STORE, 140175935610880, 140175985139711, +STORE, 140175935602688, 140175985139711, +STORE, 140175935586304, 140175985139711, +STORE, 140175935578112, 140175985139711, +STORE, 140175935545344, 140175985139711, +STORE, 140175935537152, 140175985139711, +STORE, 140175935520768, 140175985139711, +STORE, 140175935512576, 140175985139711, +STORE, 140175935496192, 140175985139711, +STORE, 140175935488000, 140175985139711, +STORE, 140175935471616, 140175985139711, +STORE, 140175935463424, 140175985139711, +STORE, 140175935430656, 140175985139711, +STORE, 140175935422464, 140175985139711, +STORE, 140175935406080, 140175985139711, +STORE, 140175935397888, 140175985139711, +STORE, 140175935381504, 140175985139711, +STORE, 140175935373312, 140175985139711, +STORE, 140175935356928, 140175985139711, +STORE, 140175935348736, 140175985139711, +STORE, 140175935315968, 140175985139711, +STORE, 140175935307776, 140175985139711, +STORE, 140175935291392, 140175985139711, +STORE, 140175935283200, 140175985139711, +STORE, 140175935266816, 140175985139711, +STORE, 140175935258624, 140175985139711, +STORE, 140175935242240, 140175985139711, +STORE, 140175935234048, 140175985139711, +STORE, 140175935201280, 140175985139711, +STORE, 140175935193088, 140175985139711, +STORE, 140175935176704, 140175985139711, +STORE, 140175935168512, 140175985139711, +STORE, 140175935152128, 140175985139711, +STORE, 140175935143936, 140175985139711, +STORE, 140175935127552, 140175985139711, +STORE, 140175935119360, 140175985139711, +STORE, 140175935086592, 140175985139711, +STORE, 140175935078400, 140175985139711, +STORE, 140175935062016, 140175985139711, +STORE, 140175935053824, 140175985139711, +STORE, 140175935037440, 140175985139711, +STORE, 140175935029248, 140175985139711, +STORE, 140175935012864, 140175985139711, +STORE, 140175935004672, 140175985139711, +STORE, 140175934971904, 140175985139711, +STORE, 140175934963712, 140175985139711, +STORE, 140175934947328, 140175985139711, +STORE, 140175934939136, 140175985139711, +STORE, 140175934922752, 140175985139711, +STORE, 140175934914560, 140175985139711, +STORE, 140175934898176, 140175985139711, +STORE, 140175934889984, 140175985139711, +STORE, 140175934857216, 140175985139711, +STORE, 140175934849024, 140175985139711, +STORE, 140175934832640, 140175985139711, +STORE, 140175934824448, 140175985139711, +STORE, 140175934808064, 140175985139711, +STORE, 140175934799872, 140175985139711, +STORE, 140175934783488, 140175985139711, +STORE, 140175934775296, 140175985139711, +STORE, 140175934742528, 140175985139711, +STORE, 140175934734336, 140175985139711, +STORE, 140175934717952, 140175985139711, +STORE, 140175934709760, 140175985139711, +STORE, 140175934693376, 140175985139711, +STORE, 140175934685184, 140175985139711, +STORE, 140175934668800, 140175985139711, +STORE, 140175934660608, 140175985139711, +STORE, 140175934627840, 140175985139711, +STORE, 140175934619648, 140175985139711, +STORE, 140175934603264, 140175985139711, +STORE, 140175934595072, 140175985139711, +STORE, 140175934578688, 140175985139711, +STORE, 140175934570496, 140175985139711, +STORE, 140175934554112, 140175985139711, +STORE, 140175934545920, 140175985139711, +STORE, 140175934513152, 140175985139711, +STORE, 140175934504960, 140175985139711, +STORE, 140175934488576, 140175985139711, +STORE, 140175934480384, 140175985139711, +STORE, 140175934464000, 140175985139711, +STORE, 140175934455808, 140175985139711, +STORE, 140175934439424, 140175985139711, +STORE, 140175934431232, 140175985139711, +STORE, 140175934398464, 140175985139711, +STORE, 140175934390272, 140175985139711, +STORE, 140175934373888, 140175985139711, +STORE, 140175934365696, 140175985139711, +STORE, 140175934349312, 140175985139711, +STORE, 140175934341120, 140175985139711, +STORE, 140175934324736, 140175985139711, +STORE, 140175932227584, 140175985139711, +STORE, 140175932219392, 140175985139711, +STORE, 140175932186624, 140175985139711, +STORE, 140175932178432, 140175985139711, +STORE, 140175932162048, 140175985139711, +STORE, 140175932153856, 140175985139711, +STORE, 140175932137472, 140175985139711, +STORE, 53080064, 57884671, +STORE, 140175932129280, 140175985139711, +STORE, 140175932112896, 140175985139711, +STORE, 140175932104704, 140175985139711, +STORE, 140175932071936, 140175985139711, +STORE, 140175932063744, 140175985139711, +STORE, 140175932047360, 140175985139711, +STORE, 140175932039168, 140175985139711, +STORE, 140175932022784, 140175985139711, +STORE, 140175932014592, 140175985139711, +STORE, 140175931998208, 140175985139711, +STORE, 140175931990016, 140175985139711, +STORE, 140175931957248, 140175985139711, +STORE, 140175931949056, 140175985139711, +STORE, 140175931932672, 140175985139711, +STORE, 140175931924480, 140175985139711, +STORE, 140175931908096, 140175985139711, +STORE, 140175931899904, 140175985139711, +STORE, 140175931883520, 140175985139711, +STORE, 140175931875328, 140175985139711, +STORE, 140175931842560, 140175985139711, +STORE, 140175931834368, 140175985139711, +STORE, 140175931817984, 140175985139711, +STORE, 140175931809792, 140175985139711, +STORE, 140175931793408, 140175985139711, +STORE, 140175931785216, 140175985139711, +STORE, 140175931768832, 140175985139711, +STORE, 140175931760640, 140175985139711, +STORE, 140175931727872, 140175985139711, +STORE, 140175931719680, 140175985139711, +STORE, 140175931703296, 140175985139711, +STORE, 140175931695104, 140175985139711, +STORE, 140175931678720, 140175985139711, +STORE, 140175931670528, 140175985139711, +STORE, 140175931654144, 140175985139711, +STORE, 140175931645952, 140175985139711, +STORE, 140175931613184, 140175985139711, +STORE, 140175931604992, 140175985139711, +STORE, 140175931588608, 140175985139711, +STORE, 140175931580416, 140175985139711, +STORE, 140175931564032, 140175985139711, +STORE, 140175931555840, 140175985139711, +STORE, 140175931539456, 140175985139711, +STORE, 140175931531264, 140175985139711, +STORE, 140175931498496, 140175985139711, +STORE, 140175931490304, 140175985139711, +STORE, 140175931473920, 140175985139711, +STORE, 140175931465728, 140175985139711, +STORE, 140175931449344, 140175985139711, +STORE, 140175931441152, 140175985139711, +STORE, 140175931424768, 140175985139711, +STORE, 140175931416576, 140175985139711, +STORE, 140175931383808, 140175985139711, +STORE, 140175931375616, 140175985139711, +STORE, 140175931359232, 140175985139711, +STORE, 140175931351040, 140175985139711, +STORE, 140175931334656, 140175985139711, +STORE, 140175931326464, 140175985139711, +STORE, 140175931310080, 140175985139711, +STORE, 140175931301888, 140175985139711, +STORE, 140175931269120, 140175985139711, +STORE, 140175931260928, 140175985139711, +STORE, 140175931244544, 140175985139711, +STORE, 140175931236352, 140175985139711, +STORE, 140175931219968, 140175985139711, +STORE, 140175931211776, 140175985139711, +STORE, 140175931195392, 140175985139711, +STORE, 140175931187200, 140175985139711, +STORE, 140175931154432, 140175985139711, +STORE, 140175931146240, 140175985139711, +STORE, 140175931129856, 140175985139711, +STORE, 140175931121664, 140175985139711, +STORE, 140175931105280, 140175985139711, +STORE, 140175931097088, 140175985139711, +STORE, 140175931080704, 140175985139711, +STORE, 140175931072512, 140175985139711, +STORE, 140175931039744, 140175985139711, +STORE, 140175931031552, 140175985139711, +STORE, 140175931015168, 140175985139711, +STORE, 140175931006976, 140175985139711, +STORE, 140175930990592, 140175985139711, +STORE, 140175930982400, 140175985139711, +STORE, 140175930966016, 140175985139711, +STORE, 140175930957824, 140175985139711, +STORE, 140175930925056, 140175985139711, +STORE, 140175930916864, 140175985139711, +STORE, 140175930900480, 140175985139711, +STORE, 140175930892288, 140175985139711, +STORE, 140175930875904, 140175985139711, +STORE, 140175930867712, 140175985139711, +STORE, 140175930851328, 140175985139711, +STORE, 140175930843136, 140175985139711, +STORE, 140175930810368, 140175985139711, +STORE, 140175930802176, 140175985139711, +STORE, 140175930785792, 140175985139711, +STORE, 140175930777600, 140175985139711, +STORE, 140175930761216, 140175985139711, +STORE, 140175930753024, 140175985139711, +STORE, 140175930736640, 140175985139711, +STORE, 140175930728448, 140175985139711, +STORE, 140175930695680, 140175985139711, +STORE, 140175930687488, 140175985139711, +STORE, 140175930671104, 140175985139711, +STORE, 140175930662912, 140175985139711, +STORE, 140175930646528, 140175985139711, +STORE, 140175930638336, 140175985139711, +STORE, 140175930621952, 140175985139711, +STORE, 140175930613760, 140175985139711, +STORE, 140175930580992, 140175985139711, +STORE, 140175930572800, 140175985139711, +STORE, 140175930556416, 140175985139711, +STORE, 140175930548224, 140175985139711, +STORE, 140175930531840, 140175985139711, +STORE, 140175930523648, 140175985139711, +STORE, 140175930507264, 140175985139711, +STORE, 140175928410112, 140175985139711, +STORE, 140175928401920, 140175985139711, +STORE, 140175928369152, 140175985139711, +STORE, 140175928360960, 140175985139711, +STORE, 140175928344576, 140175985139711, +STORE, 140175928336384, 140175985139711, +STORE, 140175928320000, 140175985139711, +STORE, 140175928311808, 140175985139711, +STORE, 140175928295424, 140175985139711, +STORE, 140175927242752, 140175985139711, +SNULL, 140175956627455, 140175985139711, +STORE, 140175927242752, 140175956627455, +STORE, 140175956627456, 140175985139711, + }; + unsigned long set24[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735281639424, 140737488351231, +SNULL, 140735281643519, 140737488351231, +STORE, 140735281639424, 140735281643519, +STORE, 140735281508352, 140735281643519, +STORE, 94717834911744, 94717834928127, +SNULL, 94717834915839, 94717834928127, +STORE, 94717834911744, 94717834915839, +STORE, 94717834915840, 94717834928127, +ERASE, 94717834915840, 94717834928127, +STORE, 94717834919936, 94717834928127, +STORE, 140428246065152, 140428248317951, +SNULL, 140428246208511, 140428248317951, +STORE, 140428246065152, 140428246208511, +STORE, 140428246208512, 140428248317951, +ERASE, 140428246208512, 140428248317951, +STORE, 140428248305664, 140428248313855, +STORE, 140428248313856, 140428248317951, +STORE, 140735281811456, 140735281815551, +STORE, 140735281799168, 140735281811455, +STORE, 140428248297472, 140428248305663, +STORE, 140428243841024, 140428246065151, +SNULL, 140428245491711, 140428246065151, +STORE, 140428243841024, 140428245491711, +STORE, 140428245491712, 140428246065151, +SNULL, 140428245491712, 140428246061055, +STORE, 140428246061056, 140428246065151, +STORE, 140428245491712, 140428246061055, +ERASE, 140428245491712, 140428246061055, +STORE, 140428245491712, 140428246061055, +ERASE, 140428246061056, 140428246065151, +STORE, 140428246061056, 140428246065151, +STORE, 140428248268800, 140428248297471, +STORE, 140428241625088, 140428243841023, +SNULL, 140428241625088, 140428241723391, +STORE, 140428241723392, 140428243841023, +STORE, 140428241625088, 140428241723391, +SNULL, 140428243816447, 140428243841023, +STORE, 140428241723392, 140428243816447, +STORE, 140428243816448, 140428243841023, +SNULL, 140428243816448, 140428243824639, +STORE, 140428243824640, 140428243841023, +STORE, 140428243816448, 140428243824639, +ERASE, 140428243816448, 140428243824639, +STORE, 140428243816448, 140428243824639, +ERASE, 140428243824640, 140428243841023, +STORE, 140428243824640, 140428243841023, +STORE, 140428237828096, 140428241625087, +SNULL, 140428237828096, 140428239486975, +STORE, 140428239486976, 140428241625087, +STORE, 140428237828096, 140428239486975, +SNULL, 140428241584127, 140428241625087, +STORE, 140428239486976, 140428241584127, +STORE, 140428241584128, 140428241625087, +SNULL, 140428241584128, 140428241608703, +STORE, 140428241608704, 140428241625087, +STORE, 140428241584128, 140428241608703, +ERASE, 140428241584128, 140428241608703, +STORE, 140428241584128, 140428241608703, +ERASE, 140428241608704, 140428241625087, +STORE, 140428241608704, 140428241625087, +STORE, 140428235567104, 140428237828095, +SNULL, 140428235567104, 140428235718655, +STORE, 140428235718656, 140428237828095, +STORE, 140428235567104, 140428235718655, +SNULL, 140428237811711, 140428237828095, +STORE, 140428235718656, 140428237811711, +STORE, 140428237811712, 140428237828095, +SNULL, 140428237811712, 140428237819903, +STORE, 140428237819904, 140428237828095, +STORE, 140428237811712, 140428237819903, +ERASE, 140428237811712, 140428237819903, +STORE, 140428237811712, 140428237819903, +ERASE, 140428237819904, 140428237828095, +STORE, 140428237819904, 140428237828095, +STORE, 140428233445376, 140428235567103, +SNULL, 140428233445376, 140428233461759, +STORE, 140428233461760, 140428235567103, +STORE, 140428233445376, 140428233461759, +SNULL, 140428235558911, 140428235567103, +STORE, 140428233461760, 140428235558911, +STORE, 140428235558912, 140428235567103, +ERASE, 140428235558912, 140428235567103, +STORE, 140428235558912, 140428235567103, +STORE, 140428231315456, 140428233445375, +SNULL, 140428231315456, 140428231344127, +STORE, 140428231344128, 140428233445375, +STORE, 140428231315456, 140428231344127, +SNULL, 140428233437183, 140428233445375, +STORE, 140428231344128, 140428233437183, +STORE, 140428233437184, 140428233445375, +ERASE, 140428233437184, 140428233445375, +STORE, 140428233437184, 140428233445375, +STORE, 140428248260608, 140428248268799, +STORE, 140428229062656, 140428231315455, +SNULL, 140428229062656, 140428229214207, +STORE, 140428229214208, 140428231315455, +STORE, 140428229062656, 140428229214207, +SNULL, 140428231307263, 140428231315455, +STORE, 140428229214208, 140428231307263, +STORE, 140428231307264, 140428231315455, +ERASE, 140428231307264, 140428231315455, +STORE, 140428231307264, 140428231315455, +STORE, 140428226891776, 140428229062655, +SNULL, 140428226891776, 140428226961407, +STORE, 140428226961408, 140428229062655, +STORE, 140428226891776, 140428226961407, +SNULL, 140428229054463, 140428229062655, +STORE, 140428226961408, 140428229054463, +STORE, 140428229054464, 140428229062655, +ERASE, 140428229054464, 140428229062655, +STORE, 140428229054464, 140428229062655, +STORE, 140428223680512, 140428226891775, +SNULL, 140428223680512, 140428224757759, +STORE, 140428224757760, 140428226891775, +STORE, 140428223680512, 140428224757759, +SNULL, 140428226854911, 140428226891775, +STORE, 140428224757760, 140428226854911, +STORE, 140428226854912, 140428226891775, +ERASE, 140428226854912, 140428226891775, +STORE, 140428226854912, 140428226891775, +STORE, 140428221546496, 140428223680511, +SNULL, 140428221546496, 140428221575167, +STORE, 140428221575168, 140428223680511, +STORE, 140428221546496, 140428221575167, +SNULL, 140428223672319, 140428223680511, +STORE, 140428221575168, 140428223672319, +STORE, 140428223672320, 140428223680511, +ERASE, 140428223672320, 140428223680511, +STORE, 140428223672320, 140428223680511, +STORE, 140428219236352, 140428221546495, +SNULL, 140428219236352, 140428219441151, +STORE, 140428219441152, 140428221546495, +STORE, 140428219236352, 140428219441151, +SNULL, 140428221538303, 140428221546495, +STORE, 140428219441152, 140428221538303, +STORE, 140428221538304, 140428221546495, +ERASE, 140428221538304, 140428221546495, +STORE, 140428221538304, 140428221546495, +STORE, 140428216852480, 140428219236351, +SNULL, 140428216852480, 140428217044991, +STORE, 140428217044992, 140428219236351, +STORE, 140428216852480, 140428217044991, +SNULL, 140428219138047, 140428219236351, +STORE, 140428217044992, 140428219138047, +STORE, 140428219138048, 140428219236351, +ERASE, 140428219138048, 140428219236351, +STORE, 140428219138048, 140428219236351, +STORE, 140428248252416, 140428248268799, +STORE, 140428214284288, 140428216852479, +SNULL, 140428214284288, 140428214751231, +STORE, 140428214751232, 140428216852479, +STORE, 140428214284288, 140428214751231, +SNULL, 140428216844287, 140428216852479, +STORE, 140428214751232, 140428216844287, +STORE, 140428216844288, 140428216852479, +ERASE, 140428216844288, 140428216852479, +STORE, 140428216844288, 140428216852479, +STORE, 140428212170752, 140428214284287, +SNULL, 140428212170752, 140428212183039, +STORE, 140428212183040, 140428214284287, +STORE, 140428212170752, 140428212183039, +SNULL, 140428214276095, 140428214284287, +STORE, 140428212183040, 140428214276095, +STORE, 140428214276096, 140428214284287, +ERASE, 140428214276096, 140428214284287, +STORE, 140428214276096, 140428214284287, +STORE, 140428209991680, 140428212170751, +SNULL, 140428209991680, 140428210069503, +STORE, 140428210069504, 140428212170751, +STORE, 140428209991680, 140428210069503, +SNULL, 140428212162559, 140428212170751, +STORE, 140428210069504, 140428212162559, +STORE, 140428212162560, 140428212170751, +ERASE, 140428212162560, 140428212170751, +STORE, 140428212162560, 140428212170751, +STORE, 140428207874048, 140428209991679, +SNULL, 140428207874048, 140428207890431, +STORE, 140428207890432, 140428209991679, +STORE, 140428207874048, 140428207890431, +SNULL, 140428209983487, 140428209991679, +STORE, 140428207890432, 140428209983487, +STORE, 140428209983488, 140428209991679, +ERASE, 140428209983488, 140428209991679, +STORE, 140428209983488, 140428209991679, +STORE, 140428248244224, 140428248268799, +STORE, 140428248231936, 140428248268799, +SNULL, 140428241600511, 140428241608703, +STORE, 140428241584128, 140428241600511, +STORE, 140428241600512, 140428241608703, +SNULL, 140428209987583, 140428209991679, +STORE, 140428209983488, 140428209987583, +STORE, 140428209987584, 140428209991679, +SNULL, 140428212166655, 140428212170751, +STORE, 140428212162560, 140428212166655, +STORE, 140428212166656, 140428212170751, +SNULL, 140428214280191, 140428214284287, +STORE, 140428214276096, 140428214280191, +STORE, 140428214280192, 140428214284287, +SNULL, 140428243820543, 140428243824639, +STORE, 140428243816448, 140428243820543, +STORE, 140428243820544, 140428243824639, +SNULL, 140428216848383, 140428216852479, +STORE, 140428216844288, 140428216848383, +STORE, 140428216848384, 140428216852479, +SNULL, 140428219232255, 140428219236351, +STORE, 140428219138048, 140428219232255, +STORE, 140428219232256, 140428219236351, +SNULL, 140428221542399, 140428221546495, +STORE, 140428221538304, 140428221542399, +STORE, 140428221542400, 140428221546495, +SNULL, 140428223676415, 140428223680511, +STORE, 140428223672320, 140428223676415, +STORE, 140428223676416, 140428223680511, +SNULL, 140428226863103, 140428226891775, +STORE, 140428226854912, 140428226863103, +STORE, 140428226863104, 140428226891775, +SNULL, 140428229058559, 140428229062655, +STORE, 140428229054464, 140428229058559, +STORE, 140428229058560, 140428229062655, +SNULL, 140428231311359, 140428231315455, +STORE, 140428231307264, 140428231311359, +STORE, 140428231311360, 140428231315455, +SNULL, 140428233441279, 140428233445375, +STORE, 140428233437184, 140428233441279, +STORE, 140428233441280, 140428233445375, +SNULL, 140428235563007, 140428235567103, +STORE, 140428235558912, 140428235563007, +STORE, 140428235563008, 140428235567103, +SNULL, 140428237815807, 140428237819903, +STORE, 140428237811712, 140428237815807, +STORE, 140428237815808, 140428237819903, +SNULL, 140428246056959, 140428246061055, +STORE, 140428245491712, 140428246056959, +STORE, 140428246056960, 140428246061055, +SNULL, 94717834924031, 94717834928127, +STORE, 94717834919936, 94717834924031, +STORE, 94717834924032, 94717834928127, +SNULL, 140428248309759, 140428248313855, +STORE, 140428248305664, 140428248309759, +STORE, 140428248309760, 140428248313855, +ERASE, 140428248268800, 140428248297471, +STORE, 94717843058688, 94717843193855, +STORE, 94749677137920, 94749677559807, +STORE, 94749677563904, 94749677604863, +STORE, 94749677604864, 94749677608959, +STORE, 94749710970880, 94749711241215, +STORE, 140490884894720, 140490884935679, +STORE, 140490884935680, 140490887032831, +STORE, 140490887032832, 140490887036927, +STORE, 140490887036928, 140490887041023, +STORE, 140490887041024, 140490887065599, +STORE, 140490887065600, 140490887110655, +STORE, 140490887110656, 140490889203711, +STORE, 140490889203712, 140490889207807, +STORE, 140490889207808, 140490889211903, +STORE, 140490889211904, 140490889293823, +STORE, 140490889293824, 140490891390975, +STORE, 140490891390976, 140490891395071, +STORE, 140490891395072, 140490891399167, +STORE, 140490891399168, 140490891407359, +STORE, 140490891407360, 140490891436031, +STORE, 140490891436032, 140490893529087, +STORE, 140490893529088, 140490893533183, +STORE, 140490893533184, 140490893537279, +STORE, 140490893537280, 140490901979135, +STORE, 140490901979136, 140490901991423, +STORE, 140490901991424, 140490904084479, +STORE, 140490904084480, 140490904088575, +STORE, 140490904088576, 140490904092671, +STORE, 140490904092672, 140490904559615, +STORE, 140490904559616, 140490906652671, +STORE, 140490906652672, 140490906656767, +STORE, 140490906656768, 140490906660863, +STORE, 140490906660864, 140490906677247, +STORE, 140490906677248, 140490908770303, +STORE, 140490908770304, 140490908774399, +STORE, 140490908774400, 140490908778495, +STORE, 140490908778496, 140490908794879, +STORE, 140490908794880, 140490910887935, +STORE, 140490910887936, 140490910892031, +STORE, 140490910892032, 140490910896127, +STORE, 140490910896128, 140490912555007, +STORE, 140490912555008, 140490914652159, +STORE, 140490914652160, 140490914668543, +STORE, 140490914668544, 140490914676735, +STORE, 140490914676736, 140490914693119, +STORE, 140490914693120, 140490914791423, +STORE, 140490914791424, 140490916884479, +STORE, 140490916884480, 140490916888575, +STORE, 140490916888576, 140490916892671, +STORE, 140490916892672, 140490916909055, +STORE, 140490916909056, 140490916937727, +STORE, 140490916937728, 140490919030783, +STORE, 140490919030784, 140490919034879, +STORE, 140490919034880, 140490919038975, +STORE, 140490919038976, 140490919190527, +STORE, 140490919190528, 140490921283583, +STORE, 140490921283584, 140490921287679, +STORE, 140490921287680, 140490921291775, +STORE, 140490921291776, 140490921299967, +STORE, 140490921299968, 140490921390079, +STORE, 140490921390080, 140490923483135, +STORE, 140490923483136, 140490923487231, +STORE, 140490923487232, 140490923491327, +STORE, 140490923491328, 140490923757567, +STORE, 140490923757568, 140490925850623, +STORE, 140490925850624, 140490925867007, +STORE, 140490925867008, 140490925871103, +STORE, 140490925871104, 140490925875199, +STORE, 140490925875200, 140490925903871, +STORE, 140490925903872, 140490928001023, +STORE, 140490928001024, 140490928005119, +STORE, 140490928005120, 140490928009215, +STORE, 140490928009216, 140490928152575, +STORE, 140490930184192, 140490930221055, +STORE, 140490930221056, 140490930237439, +STORE, 140490930237440, 140490930241535, +STORE, 140490930241536, 140490930245631, +STORE, 140490930245632, 140490930249727, +STORE, 140490930249728, 140490930253823, +STORE, 140490930253824, 140490930257919, +STORE, 140490930257920, 140490930262015, +STORE, 140724611694592, 140724611829759, +STORE, 140724612427776, 140724612440063, +STORE, 140724612440064, 140724612444159, +STORE, 94103163662336, 94103163772927, +STORE, 94103165865984, 94103165874175, +STORE, 94103165874176, 94103165878271, +STORE, 94103165878272, 94103165886463, +STORE, 94103182548992, 94103182684159, +STORE, 140092694708224, 140092696367103, +STORE, 140092696367104, 140092698464255, +STORE, 140092698464256, 140092698480639, +STORE, 140092698480640, 140092698488831, +STORE, 140092698488832, 140092698505215, +STORE, 140092698505216, 140092698648575, +STORE, 140092700708864, 140092700717055, +STORE, 140092700745728, 140092700749823, +STORE, 140092700749824, 140092700753919, +STORE, 140092700753920, 140092700758015, +STORE, 140736800911360, 140736801046527, +STORE, 140736802308096, 140736802320383, +STORE, 140736802320384, 140736802324479, +STORE, 93948802064384, 93948802174975, +STORE, 93948804268032, 93948804276223, +STORE, 93948804276224, 93948804280319, +STORE, 93948804280320, 93948804288511, +STORE, 93948806266880, 93948806402047, +STORE, 140222999113728, 140223000772607, +STORE, 140223000772608, 140223002869759, +STORE, 140223002869760, 140223002886143, +STORE, 140223002886144, 140223002894335, +STORE, 140223002894336, 140223002910719, +STORE, 140223002910720, 140223003054079, +STORE, 140223005114368, 140223005122559, +STORE, 140223005151232, 140223005155327, +STORE, 140223005155328, 140223005159423, +STORE, 140223005159424, 140223005163519, +STORE, 140720877506560, 140720877641727, +STORE, 140720878231552, 140720878243839, +STORE, 140720878243840, 140720878247935, +STORE, 140737488347136, 140737488351231, +STORE, 140733232087040, 140737488351231, +SNULL, 140733232091135, 140737488351231, +STORE, 140733232087040, 140733232091135, +STORE, 140733231955968, 140733232091135, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140161681321984, 140161683574783, +SNULL, 140161681465343, 140161683574783, +STORE, 140161681321984, 140161681465343, +STORE, 140161681465344, 140161683574783, +ERASE, 140161681465344, 140161683574783, +STORE, 140161683562496, 140161683570687, +STORE, 140161683570688, 140161683574783, +STORE, 140733232214016, 140733232218111, +STORE, 140733232201728, 140733232214015, +STORE, 140161683533824, 140161683562495, +STORE, 140161683525632, 140161683533823, +STORE, 140161678159872, 140161681321983, +SNULL, 140161678159872, 140161679220735, +STORE, 140161679220736, 140161681321983, +STORE, 140161678159872, 140161679220735, +SNULL, 140161681313791, 140161681321983, +STORE, 140161679220736, 140161681313791, +STORE, 140161681313792, 140161681321983, +ERASE, 140161681313792, 140161681321983, +STORE, 140161681313792, 140161681321983, +STORE, 140161674362880, 140161678159871, +SNULL, 140161674362880, 140161676021759, +STORE, 140161676021760, 140161678159871, +STORE, 140161674362880, 140161676021759, +SNULL, 140161678118911, 140161678159871, +STORE, 140161676021760, 140161678118911, +STORE, 140161678118912, 140161678159871, +SNULL, 140161678118912, 140161678143487, +STORE, 140161678143488, 140161678159871, +STORE, 140161678118912, 140161678143487, +ERASE, 140161678118912, 140161678143487, +STORE, 140161678118912, 140161678143487, +ERASE, 140161678143488, 140161678159871, +STORE, 140161678143488, 140161678159871, +STORE, 140161683513344, 140161683533823, +SNULL, 140161678135295, 140161678143487, +STORE, 140161678118912, 140161678135295, +STORE, 140161678135296, 140161678143487, +SNULL, 140161681317887, 140161681321983, +STORE, 140161681313792, 140161681317887, +STORE, 140161681317888, 140161681321983, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140161683566591, 140161683570687, +STORE, 140161683562496, 140161683566591, +STORE, 140161683566592, 140161683570687, +ERASE, 140161683533824, 140161683562495, +STORE, 25477120, 25612287, +STORE, 25477120, 25759743, +STORE, 140161681829888, 140161683513343, +STORE, 25477120, 25915391, +STORE, 25477120, 26054655, +SNULL, 25800703, 26054655, +STORE, 25477120, 25800703, +STORE, 25800704, 26054655, +ERASE, 25800704, 26054655, +STORE, 140737488347136, 140737488351231, +STORE, 140723218452480, 140737488351231, +SNULL, 140723218456575, 140737488351231, +STORE, 140723218452480, 140723218456575, +STORE, 140723218321408, 140723218456575, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140398872264704, 140398874517503, +SNULL, 140398872408063, 140398874517503, +STORE, 140398872264704, 140398872408063, +STORE, 140398872408064, 140398874517503, +ERASE, 140398872408064, 140398874517503, +STORE, 140398874505216, 140398874513407, +STORE, 140398874513408, 140398874517503, +STORE, 140723219247104, 140723219251199, +STORE, 140723219234816, 140723219247103, +STORE, 140398874476544, 140398874505215, +STORE, 140398874468352, 140398874476543, +STORE, 140398868430848, 140398872264703, +SNULL, 140398868430848, 140398870138879, +STORE, 140398870138880, 140398872264703, +STORE, 140398868430848, 140398870138879, +SNULL, 140398872231935, 140398872264703, +STORE, 140398870138880, 140398872231935, +STORE, 140398872231936, 140398872264703, +ERASE, 140398872231936, 140398872264703, +STORE, 140398872231936, 140398872264703, +STORE, 140398866235392, 140398868430847, +SNULL, 140398866235392, 140398866329599, +STORE, 140398866329600, 140398868430847, +STORE, 140398866235392, 140398866329599, +SNULL, 140398868422655, 140398868430847, +STORE, 140398866329600, 140398868422655, +STORE, 140398868422656, 140398868430847, +ERASE, 140398868422656, 140398868430847, +STORE, 140398868422656, 140398868430847, +STORE, 140398863716352, 140398866235391, +SNULL, 140398863716352, 140398864130047, +STORE, 140398864130048, 140398866235391, +STORE, 140398863716352, 140398864130047, +SNULL, 140398866223103, 140398866235391, +STORE, 140398864130048, 140398866223103, +STORE, 140398866223104, 140398866235391, +ERASE, 140398866223104, 140398866235391, +STORE, 140398866223104, 140398866235391, +STORE, 140398861082624, 140398863716351, +SNULL, 140398861082624, 140398861611007, +STORE, 140398861611008, 140398863716351, +STORE, 140398861082624, 140398861611007, +SNULL, 140398863708159, 140398863716351, +STORE, 140398861611008, 140398863708159, +STORE, 140398863708160, 140398863716351, +ERASE, 140398863708160, 140398863716351, +STORE, 140398863708160, 140398863716351, +STORE, 140398858969088, 140398861082623, +SNULL, 140398858969088, 140398858981375, +STORE, 140398858981376, 140398861082623, +STORE, 140398858969088, 140398858981375, +SNULL, 140398861074431, 140398861082623, +STORE, 140398858981376, 140398861074431, +STORE, 140398861074432, 140398861082623, +ERASE, 140398861074432, 140398861082623, +STORE, 140398861074432, 140398861082623, +STORE, 140398856765440, 140398858969087, +SNULL, 140398856765440, 140398856867839, +STORE, 140398856867840, 140398858969087, +STORE, 140398856765440, 140398856867839, +SNULL, 140398858960895, 140398858969087, +STORE, 140398856867840, 140398858960895, +STORE, 140398858960896, 140398858969087, +ERASE, 140398858960896, 140398858969087, +STORE, 140398858960896, 140398858969087, +STORE, 140398874460160, 140398874476543, +STORE, 140398853603328, 140398856765439, +SNULL, 140398853603328, 140398854664191, +STORE, 140398854664192, 140398856765439, +STORE, 140398853603328, 140398854664191, +SNULL, 140398856757247, 140398856765439, +STORE, 140398854664192, 140398856757247, +STORE, 140398856757248, 140398856765439, +ERASE, 140398856757248, 140398856765439, +STORE, 140398856757248, 140398856765439, +STORE, 140398849806336, 140398853603327, +SNULL, 140398849806336, 140398851465215, +STORE, 140398851465216, 140398853603327, +STORE, 140398849806336, 140398851465215, +SNULL, 140398853562367, 140398853603327, +STORE, 140398851465216, 140398853562367, +STORE, 140398853562368, 140398853603327, +SNULL, 140398853562368, 140398853586943, +STORE, 140398853586944, 140398853603327, +STORE, 140398853562368, 140398853586943, +ERASE, 140398853562368, 140398853586943, +STORE, 140398853562368, 140398853586943, +ERASE, 140398853586944, 140398853603327, +STORE, 140398853586944, 140398853603327, +STORE, 140398874447872, 140398874476543, +SNULL, 140398853578751, 140398853586943, +STORE, 140398853562368, 140398853578751, +STORE, 140398853578752, 140398853586943, +SNULL, 140398856761343, 140398856765439, +STORE, 140398856757248, 140398856761343, +STORE, 140398856761344, 140398856765439, +SNULL, 140398858964991, 140398858969087, +STORE, 140398858960896, 140398858964991, +STORE, 140398858964992, 140398858969087, +SNULL, 140398861078527, 140398861082623, +STORE, 140398861074432, 140398861078527, +STORE, 140398861078528, 140398861082623, +SNULL, 140398863712255, 140398863716351, +STORE, 140398863708160, 140398863712255, +STORE, 140398863712256, 140398863716351, +SNULL, 140398866231295, 140398866235391, +STORE, 140398866223104, 140398866231295, +STORE, 140398866231296, 140398866235391, +SNULL, 140398868426751, 140398868430847, +STORE, 140398868422656, 140398868426751, +STORE, 140398868426752, 140398868430847, +SNULL, 140398872236031, 140398872264703, +STORE, 140398872231936, 140398872236031, +STORE, 140398872236032, 140398872264703, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140398874509311, 140398874513407, +STORE, 140398874505216, 140398874509311, +STORE, 140398874509312, 140398874513407, +ERASE, 140398874476544, 140398874505215, +STORE, 43278336, 43413503, +STORE, 140398872764416, 140398874447871, +STORE, 140398874501120, 140398874505215, +STORE, 140398872629248, 140398872764415, +STORE, 43278336, 43556863, +STORE, 140398847709184, 140398849806335, +STORE, 140398874492928, 140398874505215, +STORE, 140398874484736, 140398874505215, +STORE, 140398874447872, 140398874484735, +STORE, 140398872612864, 140398872764415, +STORE, 43278336, 43692031, +STORE, 43278336, 43880447, +STORE, 140398872604672, 140398872764415, +STORE, 140398872596480, 140398872764415, +STORE, 43278336, 44044287, +STORE, 140398872580096, 140398872764415, +STORE, 140737488347136, 140737488351231, +STORE, 140734403092480, 140737488351231, +SNULL, 140734403096575, 140737488351231, +STORE, 140734403092480, 140734403096575, +STORE, 140734402961408, 140734403096575, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140240662380544, 140240664633343, +SNULL, 140240662523903, 140240664633343, +STORE, 140240662380544, 140240662523903, +STORE, 140240662523904, 140240664633343, +ERASE, 140240662523904, 140240664633343, +STORE, 140240664621056, 140240664629247, +STORE, 140240664629248, 140240664633343, +STORE, 140734403145728, 140734403149823, +STORE, 140734403133440, 140734403145727, +STORE, 140240664592384, 140240664621055, +STORE, 140240664584192, 140240664592383, +STORE, 140240659218432, 140240662380543, +SNULL, 140240659218432, 140240660279295, +STORE, 140240660279296, 140240662380543, +STORE, 140240659218432, 140240660279295, +SNULL, 140240662372351, 140240662380543, +STORE, 140240660279296, 140240662372351, +STORE, 140240662372352, 140240662380543, +ERASE, 140240662372352, 140240662380543, +STORE, 140240662372352, 140240662380543, +STORE, 140240655421440, 140240659218431, +SNULL, 140240655421440, 140240657080319, +STORE, 140240657080320, 140240659218431, +STORE, 140240655421440, 140240657080319, +SNULL, 140240659177471, 140240659218431, +STORE, 140240657080320, 140240659177471, +STORE, 140240659177472, 140240659218431, +SNULL, 140240659177472, 140240659202047, +STORE, 140240659202048, 140240659218431, +STORE, 140240659177472, 140240659202047, +ERASE, 140240659177472, 140240659202047, +STORE, 140240659177472, 140240659202047, +ERASE, 140240659202048, 140240659218431, +STORE, 140240659202048, 140240659218431, +STORE, 140240664571904, 140240664592383, +SNULL, 140240659193855, 140240659202047, +STORE, 140240659177472, 140240659193855, +STORE, 140240659193856, 140240659202047, +SNULL, 140240662376447, 140240662380543, +STORE, 140240662372352, 140240662376447, +STORE, 140240662376448, 140240662380543, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140240664625151, 140240664629247, +STORE, 140240664621056, 140240664625151, +STORE, 140240664625152, 140240664629247, +ERASE, 140240664592384, 140240664621055, +STORE, 30646272, 30781439, +STORE, 30646272, 30928895, +STORE, 140240662888448, 140240664571903, +STORE, 94256659468288, 94256659578879, +STORE, 94256661671936, 94256661680127, +STORE, 94256661680128, 94256661684223, +STORE, 94256661684224, 94256661692415, +STORE, 94256687980544, 94256688115711, +STORE, 139801712504832, 139801714163711, +STORE, 139801714163712, 139801716260863, +STORE, 139801716260864, 139801716277247, +STORE, 139801716277248, 139801716285439, +STORE, 139801716285440, 139801716301823, +STORE, 139801716301824, 139801716445183, +STORE, 139801718505472, 139801718513663, +STORE, 139801718542336, 139801718546431, +STORE, 139801718546432, 139801718550527, +STORE, 139801718550528, 139801718554623, +STORE, 140721575538688, 140721575673855, +STORE, 140721577013248, 140721577025535, +STORE, 140721577025536, 140721577029631, +STORE, 140737488347136, 140737488351231, +STORE, 140729259393024, 140737488351231, +SNULL, 140729259397119, 140737488351231, +STORE, 140729259393024, 140729259397119, +STORE, 140729259261952, 140729259397119, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 139682376638464, 139682378891263, +SNULL, 139682376781823, 139682378891263, +STORE, 139682376638464, 139682376781823, +STORE, 139682376781824, 139682378891263, +ERASE, 139682376781824, 139682378891263, +STORE, 139682378878976, 139682378887167, +STORE, 139682378887168, 139682378891263, +STORE, 140729260462080, 140729260466175, +STORE, 140729260449792, 140729260462079, +STORE, 139682378850304, 139682378878975, +STORE, 139682378842112, 139682378850303, +STORE, 139682373476352, 139682376638463, +SNULL, 139682373476352, 139682374537215, +STORE, 139682374537216, 139682376638463, +STORE, 139682373476352, 139682374537215, +SNULL, 139682376630271, 139682376638463, +STORE, 139682374537216, 139682376630271, +STORE, 139682376630272, 139682376638463, +ERASE, 139682376630272, 139682376638463, +STORE, 139682376630272, 139682376638463, +STORE, 139682369679360, 139682373476351, +SNULL, 139682369679360, 139682371338239, +STORE, 139682371338240, 139682373476351, +STORE, 139682369679360, 139682371338239, +SNULL, 139682373435391, 139682373476351, +STORE, 139682371338240, 139682373435391, +STORE, 139682373435392, 139682373476351, +SNULL, 139682373435392, 139682373459967, +STORE, 139682373459968, 139682373476351, +STORE, 139682373435392, 139682373459967, +ERASE, 139682373435392, 139682373459967, +STORE, 139682373435392, 139682373459967, +ERASE, 139682373459968, 139682373476351, +STORE, 139682373459968, 139682373476351, +STORE, 139682378829824, 139682378850303, +SNULL, 139682373451775, 139682373459967, +STORE, 139682373435392, 139682373451775, +STORE, 139682373451776, 139682373459967, +SNULL, 139682376634367, 139682376638463, +STORE, 139682376630272, 139682376634367, +STORE, 139682376634368, 139682376638463, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 139682378883071, 139682378887167, +STORE, 139682378878976, 139682378883071, +STORE, 139682378883072, 139682378887167, +ERASE, 139682378850304, 139682378878975, +STORE, 10022912, 10158079, +STORE, 10022912, 10305535, +STORE, 139682377146368, 139682378829823, +STORE, 140737488347136, 140737488351231, +STORE, 140731831926784, 140737488351231, +SNULL, 140731831930879, 140737488351231, +STORE, 140731831926784, 140731831930879, +STORE, 140731831795712, 140731831930879, +STORE, 94615305261056, 94615307485183, +SNULL, 94615305371647, 94615307485183, +STORE, 94615305261056, 94615305371647, +STORE, 94615305371648, 94615307485183, +ERASE, 94615305371648, 94615307485183, +STORE, 94615307464704, 94615307476991, +STORE, 94615307476992, 94615307485183, +STORE, 140163912994816, 140163915247615, +SNULL, 140163913138175, 140163915247615, +STORE, 140163912994816, 140163913138175, +STORE, 140163913138176, 140163915247615, +ERASE, 140163913138176, 140163915247615, +STORE, 140163915235328, 140163915243519, +STORE, 140163915243520, 140163915247615, +STORE, 140731832217600, 140731832221695, +STORE, 140731832205312, 140731832217599, +STORE, 140163915206656, 140163915235327, +STORE, 140163915198464, 140163915206655, +STORE, 140163909197824, 140163912994815, +SNULL, 140163909197824, 140163910856703, +STORE, 140163910856704, 140163912994815, +STORE, 140163909197824, 140163910856703, +SNULL, 140163912953855, 140163912994815, +STORE, 140163910856704, 140163912953855, +STORE, 140163912953856, 140163912994815, +SNULL, 140163912953856, 140163912978431, +STORE, 140163912978432, 140163912994815, +STORE, 140163912953856, 140163912978431, +ERASE, 140163912953856, 140163912978431, +STORE, 140163912953856, 140163912978431, +ERASE, 140163912978432, 140163912994815, +STORE, 140163912978432, 140163912994815, +SNULL, 140163912970239, 140163912978431, +STORE, 140163912953856, 140163912970239, +STORE, 140163912970240, 140163912978431, +SNULL, 94615307472895, 94615307476991, +STORE, 94615307464704, 94615307472895, +STORE, 94615307472896, 94615307476991, +SNULL, 140163915239423, 140163915243519, +STORE, 140163915235328, 140163915239423, +STORE, 140163915239424, 140163915243519, +ERASE, 140163915206656, 140163915235327, +STORE, 94615330672640, 94615330807807, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725254479872, 140737488351231, +SNULL, 140725254488063, 140737488351231, +STORE, 140725254479872, 140725254488063, +STORE, 140725254348800, 140725254488063, +STORE, 94572781277184, 94572785741823, +SNULL, 94572783312895, 94572785741823, +STORE, 94572781277184, 94572783312895, +STORE, 94572783312896, 94572785741823, +ERASE, 94572783312896, 94572785741823, +STORE, 94572785405952, 94572785455103, +STORE, 94572785455104, 94572785741823, +STORE, 139636001341440, 139636003594239, +SNULL, 139636001484799, 139636003594239, +STORE, 139636001341440, 139636001484799, +STORE, 139636001484800, 139636003594239, +ERASE, 139636001484800, 139636003594239, +STORE, 139636003581952, 139636003590143, +STORE, 139636003590144, 139636003594239, +STORE, 140725255557120, 140725255561215, +STORE, 140725255544832, 140725255557119, +STORE, 139636003553280, 139636003581951, +STORE, 139636003545088, 139636003553279, +STORE, 139635998773248, 139636001341439, +SNULL, 139635998773248, 139635999240191, +STORE, 139635999240192, 139636001341439, +STORE, 139635998773248, 139635999240191, +SNULL, 139636001333247, 139636001341439, +STORE, 139635999240192, 139636001333247, +STORE, 139636001333248, 139636001341439, +ERASE, 139636001333248, 139636001341439, +STORE, 139636001333248, 139636001341439, +STORE, 139635996569600, 139635998773247, +SNULL, 139635996569600, 139635996671999, +STORE, 139635996672000, 139635998773247, +STORE, 139635996569600, 139635996671999, +SNULL, 139635998765055, 139635998773247, +STORE, 139635996672000, 139635998765055, +STORE, 139635998765056, 139635998773247, +ERASE, 139635998765056, 139635998773247, +STORE, 139635998765056, 139635998773247, +STORE, 139635994353664, 139635996569599, +SNULL, 139635994353664, 139635994451967, +STORE, 139635994451968, 139635996569599, +STORE, 139635994353664, 139635994451967, +SNULL, 139635996545023, 139635996569599, +STORE, 139635994451968, 139635996545023, +STORE, 139635996545024, 139635996569599, +SNULL, 139635996545024, 139635996553215, +STORE, 139635996553216, 139635996569599, +STORE, 139635996545024, 139635996553215, +ERASE, 139635996545024, 139635996553215, +STORE, 139635996545024, 139635996553215, +ERASE, 139635996553216, 139635996569599, +STORE, 139635996553216, 139635996569599, +STORE, 139635992223744, 139635994353663, +SNULL, 139635992223744, 139635992252415, +STORE, 139635992252416, 139635994353663, +STORE, 139635992223744, 139635992252415, +SNULL, 139635994345471, 139635994353663, +STORE, 139635992252416, 139635994345471, +STORE, 139635994345472, 139635994353663, +ERASE, 139635994345472, 139635994353663, +STORE, 139635994345472, 139635994353663, +STORE, 139635988426752, 139635992223743, +SNULL, 139635988426752, 139635990085631, +STORE, 139635990085632, 139635992223743, +STORE, 139635988426752, 139635990085631, +SNULL, 139635992182783, 139635992223743, +STORE, 139635990085632, 139635992182783, +STORE, 139635992182784, 139635992223743, +SNULL, 139635992182784, 139635992207359, +STORE, 139635992207360, 139635992223743, +STORE, 139635992182784, 139635992207359, +ERASE, 139635992182784, 139635992207359, +STORE, 139635992182784, 139635992207359, +ERASE, 139635992207360, 139635992223743, +STORE, 139635992207360, 139635992223743, +STORE, 139636003536896, 139636003553279, +SNULL, 139635992199167, 139635992207359, +STORE, 139635992182784, 139635992199167, +STORE, 139635992199168, 139635992207359, +SNULL, 139635996549119, 139635996553215, +STORE, 139635996545024, 139635996549119, +STORE, 139635996549120, 139635996553215, +SNULL, 139635994349567, 139635994353663, +STORE, 139635994345472, 139635994349567, +STORE, 139635994349568, 139635994353663, +SNULL, 139635998769151, 139635998773247, +STORE, 139635998765056, 139635998769151, +STORE, 139635998769152, 139635998773247, +SNULL, 139636001337343, 139636001341439, +STORE, 139636001333248, 139636001337343, +STORE, 139636001337344, 139636001341439, +SNULL, 94572785418239, 94572785455103, +STORE, 94572785405952, 94572785418239, +STORE, 94572785418240, 94572785455103, +SNULL, 139636003586047, 139636003590143, +STORE, 139636003581952, 139636003586047, +STORE, 139636003586048, 139636003590143, +ERASE, 139636003553280, 139636003581951, +STORE, 94572798435328, 94572798570495, +STORE, 139636001853440, 139636003536895, +STORE, 139635981426688, 139635988426751, +STORE, 139635980615680, 139635981426687, +STORE, 94572798435328, 94572798705663, +STORE, 94572798435328, 94572798840831, +STORE, 94572798435328, 94572798975999, +STORE, 94572798435328, 94572799111167, +STORE, 94572798435328, 94572799246335, +STORE, 94572798435328, 94572799381503, +STORE, 94572798435328, 94572799516671, +STORE, 94572798435328, 94572799651839, +STORE, 94572798435328, 94572799787007, +STORE, 94572798435328, 94572799922175, +STORE, 94572798435328, 94572800057343, +STORE, 94572798435328, 94572800192511, +STORE, 94572798435328, 94572800327679, +STORE, 94572798435328, 94572800462847, +STORE, 94572798435328, 94572800598015, +STORE, 94572798435328, 94572800733183, +STORE, 94572798435328, 94572800868351, +STORE, 94572798435328, 94572801003519, +STORE, 94572798435328, 94572801138687, +STORE, 94572798435328, 94572801273855, +STORE, 94572798435328, 94572801409023, +STORE, 94572798435328, 94572801544191, +STORE, 94572798435328, 94572801679359, +STORE, 94572798435328, 94572801814527, +STORE, 94572798435328, 94572801949695, +STORE, 94572798435328, 94572802084863, +STORE, 94572798435328, 94572802220031, +STORE, 94572798435328, 94572802355199, +STORE, 94572798435328, 94572802490367, +STORE, 94572798435328, 94572802625535, +STORE, 94572798435328, 94572802760703, +STORE, 94572798435328, 94572802895871, +STORE, 94572798435328, 94572803031039, +STORE, 94572798435328, 94572803166207, +STORE, 94572798435328, 94572803301375, +STORE, 94572798435328, 94572803436543, +STORE, 94572798435328, 94572803571711, +STORE, 94572798435328, 94572803706879, +STORE, 94572798435328, 94572803842047, +STORE, 94572798435328, 94572803977215, +STORE, 94572798435328, 94572804112383, +STORE, 94572798435328, 94572804247551, +STORE, 94572798435328, 94572804382719, +STORE, 94572798435328, 94572804517887, +STORE, 94572798435328, 94572804653055, +STORE, 94572798435328, 94572804788223, +STORE, 94572798435328, 94572804923391, +STORE, 94572798435328, 94572805058559, +STORE, 94572798435328, 94572805193727, +STORE, 94572798435328, 94572805328895, +STORE, 94572798435328, 94572805464063, +STORE, 94572798435328, 94572805599231, +STORE, 94572798435328, 94572805734399, +STORE, 94572798435328, 94572805869567, +STORE, 94572798435328, 94572806004735, +STORE, 94572798435328, 94572806139903, +STORE, 94572798435328, 94572806275071, +STORE, 94572798435328, 94572806410239, +STORE, 94572798435328, 94572806545407, +STORE, 94572798435328, 94572806680575, +STORE, 94572798435328, 94572806815743, +STORE, 94572798435328, 94572806950911, +STORE, 94572798435328, 94572807086079, +STORE, 94572798435328, 94572807221247, +STORE, 94572798435328, 94572807356415, +STORE, 94572798435328, 94572807491583, +STORE, 94572798435328, 94572807626751, +STORE, 94572798435328, 94572807761919, +STORE, 94572798435328, 94572807897087, +STORE, 94572798435328, 94572808032255, +STORE, 94572798435328, 94572808167423, +STORE, 94572798435328, 94572808302591, +STORE, 94572798435328, 94572808437759, +STORE, 94572798435328, 94572808572927, +ERASE, 139635981426688, 139635988426751, +STORE, 139635985088512, 139635988426751, +STORE, 139635778273280, 139635980615679, +STORE, 139635567632384, 139635778273279, +STORE, 94572798435328, 94572808716287, +STORE, 139635984564224, 139635985088511, +STORE, 139635559239680, 139635567632383, +SNULL, 139635559243775, 139635567632383, +STORE, 139635559239680, 139635559243775, +STORE, 139635559243776, 139635567632383, +STORE, 139635550846976, 139635559239679, +SNULL, 139635550851071, 139635559239679, +STORE, 139635550846976, 139635550851071, +STORE, 139635550851072, 139635559239679, +STORE, 139635542454272, 139635550846975, +STORE, 139635408236544, 139635542454271, +SNULL, 139635408236544, 139635426590719, +STORE, 139635426590720, 139635542454271, +STORE, 139635408236544, 139635426590719, +ERASE, 139635408236544, 139635426590719, +STORE, 139635292372992, 139635542454271, +SNULL, 139635359481855, 139635542454271, +STORE, 139635292372992, 139635359481855, +STORE, 139635359481856, 139635542454271, +SNULL, 139635359481856, 139635426590719, +STORE, 139635426590720, 139635542454271, +STORE, 139635359481856, 139635426590719, +ERASE, 139635359481856, 139635426590719, +SNULL, 139635542458367, 139635550846975, +STORE, 139635542454272, 139635542458367, +STORE, 139635542458368, 139635550846975, +STORE, 139635418198016, 139635426590719, +SNULL, 139635493699583, 139635542454271, +STORE, 139635426590720, 139635493699583, +STORE, 139635493699584, 139635542454271, +ERASE, 139635493699584, 139635542454271, +SNULL, 139635426725887, 139635493699583, +STORE, 139635426590720, 139635426725887, +STORE, 139635426725888, 139635493699583, +SNULL, 139635292508159, 139635359481855, +STORE, 139635292372992, 139635292508159, +STORE, 139635292508160, 139635359481855, +SNULL, 139635418202111, 139635426590719, +STORE, 139635418198016, 139635418202111, +STORE, 139635418202112, 139635426590719, +STORE, 139635225264128, 139635292372991, +STORE, 139635534061568, 139635542454271, +SNULL, 139635534065663, 139635542454271, +STORE, 139635534061568, 139635534065663, +STORE, 139635534065664, 139635542454271, +STORE, 139635525668864, 139635534061567, +SNULL, 139635525672959, 139635534061567, +STORE, 139635525668864, 139635525672959, +STORE, 139635525672960, 139635534061567, +SNULL, 139635225399295, 139635292372991, +STORE, 139635225264128, 139635225399295, +STORE, 139635225399296, 139635292372991, +STORE, 139635091046400, 139635225264127, +SNULL, 139635158155263, 139635225264127, +STORE, 139635091046400, 139635158155263, +STORE, 139635158155264, 139635225264127, +ERASE, 139635158155264, 139635225264127, +STORE, 139634956828672, 139635158155263, +STORE, 139635517276160, 139635525668863, +SNULL, 139635517280255, 139635525668863, +STORE, 139635517276160, 139635517280255, +STORE, 139635517280256, 139635525668863, +SNULL, 139634956828672, 139635091046399, +STORE, 139635091046400, 139635158155263, +STORE, 139634956828672, 139635091046399, +SNULL, 139635091181567, 139635158155263, +STORE, 139635091046400, 139635091181567, +STORE, 139635091181568, 139635158155263, +SNULL, 139635023937535, 139635091046399, +STORE, 139634956828672, 139635023937535, +STORE, 139635023937536, 139635091046399, +ERASE, 139635023937536, 139635091046399, +STORE, 139634956828672, 139635091046399, +SNULL, 139634956828672, 139635023937535, +STORE, 139635023937536, 139635091046399, +STORE, 139634956828672, 139635023937535, +SNULL, 139635024072703, 139635091046399, +STORE, 139635023937536, 139635024072703, +STORE, 139635024072704, 139635091046399, +STORE, 139635508883456, 139635517276159, +SNULL, 139635508887551, 139635517276159, +STORE, 139635508883456, 139635508887551, +STORE, 139635508887552, 139635517276159, +STORE, 139634822610944, 139635023937535, +SNULL, 139634822610944, 139634956828671, +STORE, 139634956828672, 139635023937535, +STORE, 139634822610944, 139634956828671, +SNULL, 139634956963839, 139635023937535, +STORE, 139634956828672, 139634956963839, +STORE, 139634956963840, 139635023937535, +STORE, 139635500490752, 139635508883455, +SNULL, 139634889719807, 139634956828671, +STORE, 139634822610944, 139634889719807, +STORE, 139634889719808, 139634956828671, +ERASE, 139634889719808, 139634956828671, +SNULL, 139635500494847, 139635508883455, +STORE, 139635500490752, 139635500494847, +STORE, 139635500494848, 139635508883455, +SNULL, 139634822746111, 139634889719807, +STORE, 139634822610944, 139634822746111, +STORE, 139634822746112, 139634889719807, +STORE, 139635409805312, 139635418198015, +STORE, 139634822746112, 139634956828671, +SNULL, 139634822746112, 139634889719807, +STORE, 139634889719808, 139634956828671, +STORE, 139634822746112, 139634889719807, +SNULL, 139634889854975, 139634956828671, +STORE, 139634889719808, 139634889854975, +STORE, 139634889854976, 139634956828671, +SNULL, 139635409809407, 139635418198015, +STORE, 139635409805312, 139635409809407, +STORE, 139635409809408, 139635418198015, +STORE, 139635401412608, 139635409805311, +STORE, 139634688393216, 139634822610943, +SNULL, 139634755502079, 139634822610943, +STORE, 139634688393216, 139634755502079, +STORE, 139634755502080, 139634822610943, +ERASE, 139634755502080, 139634822610943, +SNULL, 139635401416703, 139635409805311, +STORE, 139635401412608, 139635401416703, +STORE, 139635401416704, 139635409805311, +STORE, 139634554175488, 139634755502079, +SNULL, 139634554175488, 139634688393215, +STORE, 139634688393216, 139634755502079, +STORE, 139634554175488, 139634688393215, +SNULL, 139634688528383, 139634755502079, +STORE, 139634688393216, 139634688528383, +STORE, 139634688528384, 139634755502079, +STORE, 139635393019904, 139635401412607, +SNULL, 139634621284351, 139634688393215, +STORE, 139634554175488, 139634621284351, +STORE, 139634621284352, 139634688393215, +ERASE, 139634621284352, 139634688393215, +SNULL, 139634554310655, 139634621284351, +STORE, 139634554175488, 139634554310655, +STORE, 139634554310656, 139634621284351, +STORE, 139634554310656, 139634688393215, +SNULL, 139635393023999, 139635401412607, +STORE, 139635393019904, 139635393023999, +STORE, 139635393024000, 139635401412607, +SNULL, 139634554310656, 139634621284351, +STORE, 139634621284352, 139634688393215, +STORE, 139634554310656, 139634621284351, +SNULL, 139634621419519, 139634688393215, +STORE, 139634621284352, 139634621419519, +STORE, 139634621419520, 139634688393215, +STORE, 139635384627200, 139635393019903, +SNULL, 139635384631295, 139635393019903, +STORE, 139635384627200, 139635384631295, +STORE, 139635384631296, 139635393019903, +STORE, 139635376234496, 139635384627199, +SNULL, 139635376238591, 139635384627199, +STORE, 139635376234496, 139635376238591, +STORE, 139635376238592, 139635384627199, +STORE, 139635367841792, 139635376234495, +SNULL, 139635367845887, 139635376234495, +STORE, 139635367841792, 139635367845887, +STORE, 139635367845888, 139635376234495, +STORE, 139634419957760, 139634554175487, +SNULL, 139634487066623, 139634554175487, +STORE, 139634419957760, 139634487066623, +STORE, 139634487066624, 139634554175487, +ERASE, 139634487066624, 139634554175487, +STORE, 139635216871424, 139635225264127, +SNULL, 139635216875519, 139635225264127, +STORE, 139635216871424, 139635216875519, +STORE, 139635216875520, 139635225264127, +SNULL, 139634420092927, 139634487066623, +STORE, 139634419957760, 139634420092927, +STORE, 139634420092928, 139634487066623, +STORE, 139635208478720, 139635216871423, +SNULL, 139635208482815, 139635216871423, +STORE, 139635208478720, 139635208482815, +STORE, 139635208482816, 139635216871423, +STORE, 139635200086016, 139635208478719, +SNULL, 139635200090111, 139635208478719, +STORE, 139635200086016, 139635200090111, +STORE, 139635200090112, 139635208478719, +STORE, 139635191693312, 139635200086015, +SNULL, 139635191697407, 139635200086015, +STORE, 139635191693312, 139635191697407, +STORE, 139635191697408, 139635200086015, +STORE, 139635183300608, 139635191693311, +SNULL, 139635183304703, 139635191693311, +STORE, 139635183300608, 139635183304703, +STORE, 139635183304704, 139635191693311, +STORE, 139634420092928, 139634554175487, +SNULL, 139634420092928, 139634487066623, +STORE, 139634487066624, 139634554175487, +STORE, 139634420092928, 139634487066623, +SNULL, 139634487201791, 139634554175487, +STORE, 139634487066624, 139634487201791, +STORE, 139634487201792, 139634554175487, +ERASE, 139635559239680, 139635559243775, +ERASE, 139635559243776, 139635567632383, +ERASE, 139635550846976, 139635550851071, +ERASE, 139635550851072, 139635559239679, +ERASE, 139635542454272, 139635542458367, +ERASE, 139635542458368, 139635550846975, +ERASE, 139635418198016, 139635418202111, +ERASE, 139635418202112, 139635426590719, +ERASE, 139635534061568, 139635534065663, +ERASE, 139635534065664, 139635542454271, +ERASE, 139635525668864, 139635525672959, +ERASE, 139635525672960, 139635534061567, +ERASE, 139635517276160, 139635517280255, +ERASE, 139635517280256, 139635525668863, +ERASE, 139635508883456, 139635508887551, +ERASE, 139635508887552, 139635517276159, +ERASE, 139635500490752, 139635500494847, +ERASE, 139635500494848, 139635508883455, +ERASE, 139635409805312, 139635409809407, +ERASE, 139635409809408, 139635418198015, +ERASE, 139635401412608, 139635401416703, +ERASE, 139635401416704, 139635409805311, +ERASE, 139635393019904, 139635393023999, +ERASE, 139635393024000, 139635401412607, +ERASE, 139635384627200, 139635384631295, +ERASE, 139635384631296, 139635393019903, + }; + unsigned long set25[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722547441664, 140737488351231, +SNULL, 140722547449855, 140737488351231, +STORE, 140722547441664, 140722547449855, +STORE, 140722547310592, 140722547449855, +STORE, 94827521732608, 94827523956735, +SNULL, 94827521843199, 94827523956735, +STORE, 94827521732608, 94827521843199, +STORE, 94827521843200, 94827523956735, +ERASE, 94827521843200, 94827523956735, +STORE, 94827523936256, 94827523948543, +STORE, 94827523948544, 94827523956735, +STORE, 139816136847360, 139816139100159, +SNULL, 139816136990719, 139816139100159, +STORE, 139816136847360, 139816136990719, +STORE, 139816136990720, 139816139100159, +ERASE, 139816136990720, 139816139100159, +STORE, 139816139087872, 139816139096063, +STORE, 139816139096064, 139816139100159, +STORE, 140722548142080, 140722548146175, +STORE, 140722548129792, 140722548142079, +STORE, 139816139059200, 139816139087871, +STORE, 139816139051008, 139816139059199, +STORE, 139816133050368, 139816136847359, +SNULL, 139816133050368, 139816134709247, +STORE, 139816134709248, 139816136847359, +STORE, 139816133050368, 139816134709247, +SNULL, 139816136806399, 139816136847359, +STORE, 139816134709248, 139816136806399, +STORE, 139816136806400, 139816136847359, +SNULL, 139816136806400, 139816136830975, +STORE, 139816136830976, 139816136847359, +STORE, 139816136806400, 139816136830975, +ERASE, 139816136806400, 139816136830975, +STORE, 139816136806400, 139816136830975, +ERASE, 139816136830976, 139816136847359, +STORE, 139816136830976, 139816136847359, +SNULL, 139816136822783, 139816136830975, +STORE, 139816136806400, 139816136822783, +STORE, 139816136822784, 139816136830975, +SNULL, 94827523944447, 94827523948543, +STORE, 94827523936256, 94827523944447, +STORE, 94827523944448, 94827523948543, +SNULL, 139816139091967, 139816139096063, +STORE, 139816139087872, 139816139091967, +STORE, 139816139091968, 139816139096063, +ERASE, 139816139059200, 139816139087871, +STORE, 94827534970880, 94827535106047, +STORE, 94114394132480, 94114394345471, +STORE, 94114396442624, 94114396446719, +STORE, 94114396446720, 94114396454911, +STORE, 94114396454912, 94114396467199, +STORE, 94114421575680, 94114427715583, +STORE, 139934313955328, 139934315614207, +STORE, 139934315614208, 139934317711359, +STORE, 139934317711360, 139934317727743, +STORE, 139934317727744, 139934317735935, +STORE, 139934317735936, 139934317752319, +STORE, 139934317752320, 139934317764607, +STORE, 139934317764608, 139934319857663, +STORE, 139934319857664, 139934319861759, +STORE, 139934319861760, 139934319865855, +STORE, 139934319865856, 139934320009215, +STORE, 139934320377856, 139934322061311, +STORE, 139934322061312, 139934322077695, +STORE, 139934322106368, 139934322110463, +STORE, 139934322110464, 139934322114559, +STORE, 139934322114560, 139934322118655, +STORE, 140731200376832, 140731200516095, +STORE, 140731200929792, 140731200942079, +STORE, 140731200942080, 140731200946175, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734133174272, 140737488351231, +SNULL, 140734133182463, 140737488351231, +STORE, 140734133174272, 140734133182463, +STORE, 140734133043200, 140734133182463, +STORE, 94412675600384, 94412677824511, +SNULL, 94412675710975, 94412677824511, +STORE, 94412675600384, 94412675710975, +STORE, 94412675710976, 94412677824511, +ERASE, 94412675710976, 94412677824511, +STORE, 94412677804032, 94412677816319, +STORE, 94412677816320, 94412677824511, +STORE, 140320087945216, 140320090198015, +SNULL, 140320088088575, 140320090198015, +STORE, 140320087945216, 140320088088575, +STORE, 140320088088576, 140320090198015, +ERASE, 140320088088576, 140320090198015, +STORE, 140320090185728, 140320090193919, +STORE, 140320090193920, 140320090198015, +STORE, 140734134591488, 140734134595583, +STORE, 140734134579200, 140734134591487, +STORE, 140320090157056, 140320090185727, +STORE, 140320090148864, 140320090157055, +STORE, 140320084148224, 140320087945215, +SNULL, 140320084148224, 140320085807103, +STORE, 140320085807104, 140320087945215, +STORE, 140320084148224, 140320085807103, +SNULL, 140320087904255, 140320087945215, +STORE, 140320085807104, 140320087904255, +STORE, 140320087904256, 140320087945215, +SNULL, 140320087904256, 140320087928831, +STORE, 140320087928832, 140320087945215, +STORE, 140320087904256, 140320087928831, +ERASE, 140320087904256, 140320087928831, +STORE, 140320087904256, 140320087928831, +ERASE, 140320087928832, 140320087945215, +STORE, 140320087928832, 140320087945215, +SNULL, 140320087920639, 140320087928831, +STORE, 140320087904256, 140320087920639, +STORE, 140320087920640, 140320087928831, +SNULL, 94412677812223, 94412677816319, +STORE, 94412677804032, 94412677812223, +STORE, 94412677812224, 94412677816319, +SNULL, 140320090189823, 140320090193919, +STORE, 140320090185728, 140320090189823, +STORE, 140320090189824, 140320090193919, +ERASE, 140320090157056, 140320090185727, +STORE, 94412684546048, 94412684681215, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723005485056, 140737488351231, +SNULL, 140723005493247, 140737488351231, +STORE, 140723005485056, 140723005493247, +STORE, 140723005353984, 140723005493247, +STORE, 94387431936000, 94387434160127, +SNULL, 94387432046591, 94387434160127, +STORE, 94387431936000, 94387432046591, +STORE, 94387432046592, 94387434160127, +ERASE, 94387432046592, 94387434160127, +STORE, 94387434139648, 94387434151935, +STORE, 94387434151936, 94387434160127, +STORE, 140151675392000, 140151677644799, +SNULL, 140151675535359, 140151677644799, +STORE, 140151675392000, 140151675535359, +STORE, 140151675535360, 140151677644799, +ERASE, 140151675535360, 140151677644799, +STORE, 140151677632512, 140151677640703, +STORE, 140151677640704, 140151677644799, +STORE, 140723005784064, 140723005788159, +STORE, 140723005771776, 140723005784063, +STORE, 140151677603840, 140151677632511, +STORE, 140151677595648, 140151677603839, +STORE, 140151671595008, 140151675391999, +SNULL, 140151671595008, 140151673253887, +STORE, 140151673253888, 140151675391999, +STORE, 140151671595008, 140151673253887, +SNULL, 140151675351039, 140151675391999, +STORE, 140151673253888, 140151675351039, +STORE, 140151675351040, 140151675391999, +SNULL, 140151675351040, 140151675375615, +STORE, 140151675375616, 140151675391999, +STORE, 140151675351040, 140151675375615, +ERASE, 140151675351040, 140151675375615, +STORE, 140151675351040, 140151675375615, +ERASE, 140151675375616, 140151675391999, +STORE, 140151675375616, 140151675391999, +SNULL, 140151675367423, 140151675375615, +STORE, 140151675351040, 140151675367423, +STORE, 140151675367424, 140151675375615, +SNULL, 94387434147839, 94387434151935, +STORE, 94387434139648, 94387434147839, +STORE, 94387434147840, 94387434151935, +SNULL, 140151677636607, 140151677640703, +STORE, 140151677632512, 140151677636607, +STORE, 140151677636608, 140151677640703, +ERASE, 140151677603840, 140151677632511, +STORE, 94387458818048, 94387458953215, +STORE, 94909010997248, 94909011210239, +STORE, 94909013307392, 94909013311487, +STORE, 94909013311488, 94909013319679, +STORE, 94909013319680, 94909013331967, +STORE, 94909014827008, 94909023371263, +STORE, 140712411975680, 140712413634559, +STORE, 140712413634560, 140712415731711, +STORE, 140712415731712, 140712415748095, +STORE, 140712415748096, 140712415756287, +STORE, 140712415756288, 140712415772671, +STORE, 140712415772672, 140712415784959, +STORE, 140712415784960, 140712417878015, +STORE, 140712417878016, 140712417882111, +STORE, 140712417882112, 140712417886207, +STORE, 140712417886208, 140712418029567, +STORE, 140712418398208, 140712420081663, +STORE, 140712420081664, 140712420098047, +STORE, 140712420126720, 140712420130815, +STORE, 140712420130816, 140712420134911, +STORE, 140712420134912, 140712420139007, +STORE, 140729293111296, 140729293250559, +STORE, 140729293307904, 140729293320191, +STORE, 140729293320192, 140729293324287, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720541691904, 140737488351231, +SNULL, 140720541700095, 140737488351231, +STORE, 140720541691904, 140720541700095, +STORE, 140720541560832, 140720541700095, +STORE, 94203603419136, 94203605643263, +SNULL, 94203603529727, 94203605643263, +STORE, 94203603419136, 94203603529727, +STORE, 94203603529728, 94203605643263, +ERASE, 94203603529728, 94203605643263, +STORE, 94203605622784, 94203605635071, +STORE, 94203605635072, 94203605643263, +STORE, 139847623081984, 139847625334783, +SNULL, 139847623225343, 139847625334783, +STORE, 139847623081984, 139847623225343, +STORE, 139847623225344, 139847625334783, +ERASE, 139847623225344, 139847625334783, +STORE, 139847625322496, 139847625330687, +STORE, 139847625330688, 139847625334783, +STORE, 140720542547968, 140720542552063, +STORE, 140720542535680, 140720542547967, +STORE, 139847625293824, 139847625322495, +STORE, 139847625285632, 139847625293823, +STORE, 139847619284992, 139847623081983, +SNULL, 139847619284992, 139847620943871, +STORE, 139847620943872, 139847623081983, +STORE, 139847619284992, 139847620943871, +SNULL, 139847623041023, 139847623081983, +STORE, 139847620943872, 139847623041023, +STORE, 139847623041024, 139847623081983, +SNULL, 139847623041024, 139847623065599, +STORE, 139847623065600, 139847623081983, +STORE, 139847623041024, 139847623065599, +ERASE, 139847623041024, 139847623065599, +STORE, 139847623041024, 139847623065599, +ERASE, 139847623065600, 139847623081983, +STORE, 139847623065600, 139847623081983, +SNULL, 139847623057407, 139847623065599, +STORE, 139847623041024, 139847623057407, +STORE, 139847623057408, 139847623065599, +SNULL, 94203605630975, 94203605635071, +STORE, 94203605622784, 94203605630975, +STORE, 94203605630976, 94203605635071, +SNULL, 139847625326591, 139847625330687, +STORE, 139847625322496, 139847625326591, +STORE, 139847625326592, 139847625330687, +ERASE, 139847625293824, 139847625322495, +STORE, 94203634880512, 94203635015679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721428738048, 140737488351231, +SNULL, 140721428746239, 140737488351231, +STORE, 140721428738048, 140721428746239, +STORE, 140721428606976, 140721428746239, +STORE, 93968808378368, 93968810602495, +SNULL, 93968808488959, 93968810602495, +STORE, 93968808378368, 93968808488959, +STORE, 93968808488960, 93968810602495, +ERASE, 93968808488960, 93968810602495, +STORE, 93968810582016, 93968810594303, +STORE, 93968810594304, 93968810602495, +STORE, 140397757026304, 140397759279103, +SNULL, 140397757169663, 140397759279103, +STORE, 140397757026304, 140397757169663, +STORE, 140397757169664, 140397759279103, +ERASE, 140397757169664, 140397759279103, +STORE, 140397759266816, 140397759275007, +STORE, 140397759275008, 140397759279103, +STORE, 140721430368256, 140721430372351, +STORE, 140721430355968, 140721430368255, +STORE, 140397759238144, 140397759266815, +STORE, 140397759229952, 140397759238143, +STORE, 140397753229312, 140397757026303, +SNULL, 140397753229312, 140397754888191, +STORE, 140397754888192, 140397757026303, +STORE, 140397753229312, 140397754888191, +SNULL, 140397756985343, 140397757026303, +STORE, 140397754888192, 140397756985343, +STORE, 140397756985344, 140397757026303, +SNULL, 140397756985344, 140397757009919, +STORE, 140397757009920, 140397757026303, +STORE, 140397756985344, 140397757009919, +ERASE, 140397756985344, 140397757009919, +STORE, 140397756985344, 140397757009919, +ERASE, 140397757009920, 140397757026303, +STORE, 140397757009920, 140397757026303, +SNULL, 140397757001727, 140397757009919, +STORE, 140397756985344, 140397757001727, +STORE, 140397757001728, 140397757009919, +SNULL, 93968810590207, 93968810594303, +STORE, 93968810582016, 93968810590207, +STORE, 93968810590208, 93968810594303, +SNULL, 140397759270911, 140397759275007, +STORE, 140397759266816, 140397759270911, +STORE, 140397759270912, 140397759275007, +ERASE, 140397759238144, 140397759266815, +STORE, 93968837025792, 93968837160959, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721751044096, 140737488351231, +SNULL, 140721751052287, 140737488351231, +STORE, 140721751044096, 140721751052287, +STORE, 140721750913024, 140721751052287, +STORE, 94426051657728, 94426053881855, +SNULL, 94426051768319, 94426053881855, +STORE, 94426051657728, 94426051768319, +STORE, 94426051768320, 94426053881855, +ERASE, 94426051768320, 94426053881855, +STORE, 94426053861376, 94426053873663, +STORE, 94426053873664, 94426053881855, +STORE, 140228456181760, 140228458434559, +SNULL, 140228456325119, 140228458434559, +STORE, 140228456181760, 140228456325119, +STORE, 140228456325120, 140228458434559, +ERASE, 140228456325120, 140228458434559, +STORE, 140228458422272, 140228458430463, +STORE, 140228458430464, 140228458434559, +STORE, 140721751117824, 140721751121919, +STORE, 140721751105536, 140721751117823, +STORE, 140228458393600, 140228458422271, +STORE, 140228458385408, 140228458393599, +STORE, 140228452384768, 140228456181759, +SNULL, 140228452384768, 140228454043647, +STORE, 140228454043648, 140228456181759, +STORE, 140228452384768, 140228454043647, +SNULL, 140228456140799, 140228456181759, +STORE, 140228454043648, 140228456140799, +STORE, 140228456140800, 140228456181759, +SNULL, 140228456140800, 140228456165375, +STORE, 140228456165376, 140228456181759, +STORE, 140228456140800, 140228456165375, +ERASE, 140228456140800, 140228456165375, +STORE, 140228456140800, 140228456165375, +ERASE, 140228456165376, 140228456181759, +STORE, 140228456165376, 140228456181759, +SNULL, 140228456157183, 140228456165375, +STORE, 140228456140800, 140228456157183, +STORE, 140228456157184, 140228456165375, +SNULL, 94426053869567, 94426053873663, +STORE, 94426053861376, 94426053869567, +STORE, 94426053869568, 94426053873663, +SNULL, 140228458426367, 140228458430463, +STORE, 140228458422272, 140228458426367, +STORE, 140228458426368, 140228458430463, +ERASE, 140228458393600, 140228458422271, +STORE, 94426073681920, 94426073817087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732727623680, 140737488351231, +SNULL, 140732727631871, 140737488351231, +STORE, 140732727623680, 140732727631871, +STORE, 140732727492608, 140732727631871, +STORE, 94537485996032, 94537488220159, +SNULL, 94537486106623, 94537488220159, +STORE, 94537485996032, 94537486106623, +STORE, 94537486106624, 94537488220159, +ERASE, 94537486106624, 94537488220159, +STORE, 94537488199680, 94537488211967, +STORE, 94537488211968, 94537488220159, +STORE, 140446578036736, 140446580289535, +SNULL, 140446578180095, 140446580289535, +STORE, 140446578036736, 140446578180095, +STORE, 140446578180096, 140446580289535, +ERASE, 140446578180096, 140446580289535, +STORE, 140446580277248, 140446580285439, +STORE, 140446580285440, 140446580289535, +STORE, 140732727758848, 140732727762943, +STORE, 140732727746560, 140732727758847, +STORE, 140446580248576, 140446580277247, +STORE, 140446580240384, 140446580248575, +STORE, 140446574239744, 140446578036735, +SNULL, 140446574239744, 140446575898623, +STORE, 140446575898624, 140446578036735, +STORE, 140446574239744, 140446575898623, +SNULL, 140446577995775, 140446578036735, +STORE, 140446575898624, 140446577995775, +STORE, 140446577995776, 140446578036735, +SNULL, 140446577995776, 140446578020351, +STORE, 140446578020352, 140446578036735, +STORE, 140446577995776, 140446578020351, +ERASE, 140446577995776, 140446578020351, +STORE, 140446577995776, 140446578020351, +ERASE, 140446578020352, 140446578036735, +STORE, 140446578020352, 140446578036735, +SNULL, 140446578012159, 140446578020351, +STORE, 140446577995776, 140446578012159, +STORE, 140446578012160, 140446578020351, +SNULL, 94537488207871, 94537488211967, +STORE, 94537488199680, 94537488207871, +STORE, 94537488207872, 94537488211967, +SNULL, 140446580281343, 140446580285439, +STORE, 140446580277248, 140446580281343, +STORE, 140446580281344, 140446580285439, +ERASE, 140446580248576, 140446580277247, +STORE, 94537489014784, 94537489149951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728766808064, 140737488351231, +SNULL, 140728766816255, 140737488351231, +STORE, 140728766808064, 140728766816255, +STORE, 140728766676992, 140728766816255, +STORE, 94418513866752, 94418516090879, +SNULL, 94418513977343, 94418516090879, +STORE, 94418513866752, 94418513977343, +STORE, 94418513977344, 94418516090879, +ERASE, 94418513977344, 94418516090879, +STORE, 94418516070400, 94418516082687, +STORE, 94418516082688, 94418516090879, +STORE, 140556479520768, 140556481773567, +SNULL, 140556479664127, 140556481773567, +STORE, 140556479520768, 140556479664127, +STORE, 140556479664128, 140556481773567, +ERASE, 140556479664128, 140556481773567, +STORE, 140556481761280, 140556481769471, +STORE, 140556481769472, 140556481773567, +STORE, 140728767148032, 140728767152127, +STORE, 140728767135744, 140728767148031, +STORE, 140556481732608, 140556481761279, +STORE, 140556481724416, 140556481732607, +STORE, 140556475723776, 140556479520767, +SNULL, 140556475723776, 140556477382655, +STORE, 140556477382656, 140556479520767, +STORE, 140556475723776, 140556477382655, +SNULL, 140556479479807, 140556479520767, +STORE, 140556477382656, 140556479479807, +STORE, 140556479479808, 140556479520767, +SNULL, 140556479479808, 140556479504383, +STORE, 140556479504384, 140556479520767, +STORE, 140556479479808, 140556479504383, +ERASE, 140556479479808, 140556479504383, +STORE, 140556479479808, 140556479504383, +ERASE, 140556479504384, 140556479520767, +STORE, 140556479504384, 140556479520767, +SNULL, 140556479496191, 140556479504383, +STORE, 140556479479808, 140556479496191, +STORE, 140556479496192, 140556479504383, +SNULL, 94418516078591, 94418516082687, +STORE, 94418516070400, 94418516078591, +STORE, 94418516078592, 94418516082687, +SNULL, 140556481765375, 140556481769471, +STORE, 140556481761280, 140556481765375, +STORE, 140556481765376, 140556481769471, +ERASE, 140556481732608, 140556481761279, +STORE, 94418541113344, 94418541248511, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723945873408, 140737488351231, +SNULL, 140723945881599, 140737488351231, +STORE, 140723945873408, 140723945881599, +STORE, 140723945742336, 140723945881599, +STORE, 94543169773568, 94543171997695, +SNULL, 94543169884159, 94543171997695, +STORE, 94543169773568, 94543169884159, +STORE, 94543169884160, 94543171997695, +ERASE, 94543169884160, 94543171997695, +STORE, 94543171977216, 94543171989503, +STORE, 94543171989504, 94543171997695, +STORE, 139890420883456, 139890423136255, +SNULL, 139890421026815, 139890423136255, +STORE, 139890420883456, 139890421026815, +STORE, 139890421026816, 139890423136255, +ERASE, 139890421026816, 139890423136255, +STORE, 139890423123968, 139890423132159, +STORE, 139890423132160, 139890423136255, +STORE, 140723946102784, 140723946106879, +STORE, 140723946090496, 140723946102783, +STORE, 139890423095296, 139890423123967, +STORE, 139890423087104, 139890423095295, +STORE, 139890417086464, 139890420883455, +SNULL, 139890417086464, 139890418745343, +STORE, 139890418745344, 139890420883455, +STORE, 139890417086464, 139890418745343, +SNULL, 139890420842495, 139890420883455, +STORE, 139890418745344, 139890420842495, +STORE, 139890420842496, 139890420883455, +SNULL, 139890420842496, 139890420867071, +STORE, 139890420867072, 139890420883455, +STORE, 139890420842496, 139890420867071, +ERASE, 139890420842496, 139890420867071, +STORE, 139890420842496, 139890420867071, +ERASE, 139890420867072, 139890420883455, +STORE, 139890420867072, 139890420883455, +SNULL, 139890420858879, 139890420867071, +STORE, 139890420842496, 139890420858879, +STORE, 139890420858880, 139890420867071, +SNULL, 94543171985407, 94543171989503, +STORE, 94543171977216, 94543171985407, +STORE, 94543171985408, 94543171989503, +SNULL, 139890423128063, 139890423132159, +STORE, 139890423123968, 139890423128063, +STORE, 139890423128064, 139890423132159, +ERASE, 139890423095296, 139890423123967, +STORE, 94543197097984, 94543197233151, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736205979648, 140737488351231, +SNULL, 140736205987839, 140737488351231, +STORE, 140736205979648, 140736205987839, +STORE, 140736205848576, 140736205987839, +STORE, 94913209913344, 94913212137471, +SNULL, 94913210023935, 94913212137471, +STORE, 94913209913344, 94913210023935, +STORE, 94913210023936, 94913212137471, +ERASE, 94913210023936, 94913212137471, +STORE, 94913212116992, 94913212129279, +STORE, 94913212129280, 94913212137471, +STORE, 140006323052544, 140006325305343, +SNULL, 140006323195903, 140006325305343, +STORE, 140006323052544, 140006323195903, +STORE, 140006323195904, 140006325305343, +ERASE, 140006323195904, 140006325305343, +STORE, 140006325293056, 140006325301247, +STORE, 140006325301248, 140006325305343, +STORE, 140736206716928, 140736206721023, +STORE, 140736206704640, 140736206716927, +STORE, 140006325264384, 140006325293055, +STORE, 140006325256192, 140006325264383, +STORE, 140006319255552, 140006323052543, +SNULL, 140006319255552, 140006320914431, +STORE, 140006320914432, 140006323052543, +STORE, 140006319255552, 140006320914431, +SNULL, 140006323011583, 140006323052543, +STORE, 140006320914432, 140006323011583, +STORE, 140006323011584, 140006323052543, +SNULL, 140006323011584, 140006323036159, +STORE, 140006323036160, 140006323052543, +STORE, 140006323011584, 140006323036159, +ERASE, 140006323011584, 140006323036159, +STORE, 140006323011584, 140006323036159, +ERASE, 140006323036160, 140006323052543, +STORE, 140006323036160, 140006323052543, +SNULL, 140006323027967, 140006323036159, +STORE, 140006323011584, 140006323027967, +STORE, 140006323027968, 140006323036159, +SNULL, 94913212125183, 94913212129279, +STORE, 94913212116992, 94913212125183, +STORE, 94913212125184, 94913212129279, +SNULL, 140006325297151, 140006325301247, +STORE, 140006325293056, 140006325297151, +STORE, 140006325297152, 140006325301247, +ERASE, 140006325264384, 140006325293055, +STORE, 94913239932928, 94913240068095, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726926897152, 140737488351231, +SNULL, 140726926905343, 140737488351231, +STORE, 140726926897152, 140726926905343, +STORE, 140726926766080, 140726926905343, +STORE, 94213246820352, 94213249044479, +SNULL, 94213246930943, 94213249044479, +STORE, 94213246820352, 94213246930943, +STORE, 94213246930944, 94213249044479, +ERASE, 94213246930944, 94213249044479, +STORE, 94213249024000, 94213249036287, +STORE, 94213249036288, 94213249044479, +STORE, 140368830242816, 140368832495615, +SNULL, 140368830386175, 140368832495615, +STORE, 140368830242816, 140368830386175, +STORE, 140368830386176, 140368832495615, +ERASE, 140368830386176, 140368832495615, +STORE, 140368832483328, 140368832491519, +STORE, 140368832491520, 140368832495615, +STORE, 140726926999552, 140726927003647, +STORE, 140726926987264, 140726926999551, +STORE, 140368832454656, 140368832483327, +STORE, 140368832446464, 140368832454655, +STORE, 140368826445824, 140368830242815, +SNULL, 140368826445824, 140368828104703, +STORE, 140368828104704, 140368830242815, +STORE, 140368826445824, 140368828104703, +SNULL, 140368830201855, 140368830242815, +STORE, 140368828104704, 140368830201855, +STORE, 140368830201856, 140368830242815, +SNULL, 140368830201856, 140368830226431, +STORE, 140368830226432, 140368830242815, +STORE, 140368830201856, 140368830226431, +ERASE, 140368830201856, 140368830226431, +STORE, 140368830201856, 140368830226431, +ERASE, 140368830226432, 140368830242815, +STORE, 140368830226432, 140368830242815, +SNULL, 140368830218239, 140368830226431, +STORE, 140368830201856, 140368830218239, +STORE, 140368830218240, 140368830226431, +SNULL, 94213249032191, 94213249036287, +STORE, 94213249024000, 94213249032191, +STORE, 94213249032192, 94213249036287, +SNULL, 140368832487423, 140368832491519, +STORE, 140368832483328, 140368832487423, +STORE, 140368832487424, 140368832491519, +ERASE, 140368832454656, 140368832483327, +STORE, 94213267435520, 94213267570687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728954130432, 140737488351231, +SNULL, 140728954138623, 140737488351231, +STORE, 140728954130432, 140728954138623, +STORE, 140728953999360, 140728954138623, +STORE, 94672570966016, 94672573190143, +SNULL, 94672571076607, 94672573190143, +STORE, 94672570966016, 94672571076607, +STORE, 94672571076608, 94672573190143, +ERASE, 94672571076608, 94672573190143, +STORE, 94672573169664, 94672573181951, +STORE, 94672573181952, 94672573190143, +STORE, 140201696735232, 140201698988031, +SNULL, 140201696878591, 140201698988031, +STORE, 140201696735232, 140201696878591, +STORE, 140201696878592, 140201698988031, +ERASE, 140201696878592, 140201698988031, +STORE, 140201698975744, 140201698983935, +STORE, 140201698983936, 140201698988031, +STORE, 140728954163200, 140728954167295, +STORE, 140728954150912, 140728954163199, +STORE, 140201698947072, 140201698975743, +STORE, 140201698938880, 140201698947071, +STORE, 140201692938240, 140201696735231, +SNULL, 140201692938240, 140201694597119, +STORE, 140201694597120, 140201696735231, +STORE, 140201692938240, 140201694597119, +SNULL, 140201696694271, 140201696735231, +STORE, 140201694597120, 140201696694271, +STORE, 140201696694272, 140201696735231, +SNULL, 140201696694272, 140201696718847, +STORE, 140201696718848, 140201696735231, +STORE, 140201696694272, 140201696718847, +ERASE, 140201696694272, 140201696718847, +STORE, 140201696694272, 140201696718847, +ERASE, 140201696718848, 140201696735231, +STORE, 140201696718848, 140201696735231, +SNULL, 140201696710655, 140201696718847, +STORE, 140201696694272, 140201696710655, +STORE, 140201696710656, 140201696718847, +SNULL, 94672573177855, 94672573181951, +STORE, 94672573169664, 94672573177855, +STORE, 94672573177856, 94672573181951, +SNULL, 140201698979839, 140201698983935, +STORE, 140201698975744, 140201698979839, +STORE, 140201698979840, 140201698983935, +ERASE, 140201698947072, 140201698975743, +STORE, 94672595689472, 94672595824639, +STORE, 94114394132480, 94114394345471, +STORE, 94114396442624, 94114396446719, +STORE, 94114396446720, 94114396454911, +STORE, 94114396454912, 94114396467199, +STORE, 94114421575680, 94114428256255, +STORE, 139934313955328, 139934315614207, +STORE, 139934315614208, 139934317711359, +STORE, 139934317711360, 139934317727743, +STORE, 139934317727744, 139934317735935, +STORE, 139934317735936, 139934317752319, +STORE, 139934317752320, 139934317764607, +STORE, 139934317764608, 139934319857663, +STORE, 139934319857664, 139934319861759, +STORE, 139934319861760, 139934319865855, +STORE, 139934319865856, 139934320009215, +STORE, 139934320377856, 139934322061311, +STORE, 139934322061312, 139934322077695, +STORE, 139934322106368, 139934322110463, +STORE, 139934322110464, 139934322114559, +STORE, 139934322114560, 139934322118655, +STORE, 140731200376832, 140731200516095, +STORE, 140731200929792, 140731200942079, +STORE, 140731200942080, 140731200946175, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721532362752, 140737488351231, +SNULL, 140721532370943, 140737488351231, +STORE, 140721532362752, 140721532370943, +STORE, 140721532231680, 140721532370943, +STORE, 94467222597632, 94467224821759, +SNULL, 94467222708223, 94467224821759, +STORE, 94467222597632, 94467222708223, +STORE, 94467222708224, 94467224821759, +ERASE, 94467222708224, 94467224821759, +STORE, 94467224801280, 94467224813567, +STORE, 94467224813568, 94467224821759, +STORE, 140191433543680, 140191435796479, +SNULL, 140191433687039, 140191435796479, +STORE, 140191433543680, 140191433687039, +STORE, 140191433687040, 140191435796479, +ERASE, 140191433687040, 140191435796479, +STORE, 140191435784192, 140191435792383, +STORE, 140191435792384, 140191435796479, +STORE, 140721533034496, 140721533038591, +STORE, 140721533022208, 140721533034495, +STORE, 140191435755520, 140191435784191, +STORE, 140191435747328, 140191435755519, +STORE, 140191429746688, 140191433543679, +SNULL, 140191429746688, 140191431405567, +STORE, 140191431405568, 140191433543679, +STORE, 140191429746688, 140191431405567, +SNULL, 140191433502719, 140191433543679, +STORE, 140191431405568, 140191433502719, +STORE, 140191433502720, 140191433543679, +SNULL, 140191433502720, 140191433527295, +STORE, 140191433527296, 140191433543679, +STORE, 140191433502720, 140191433527295, +ERASE, 140191433502720, 140191433527295, +STORE, 140191433502720, 140191433527295, +ERASE, 140191433527296, 140191433543679, +STORE, 140191433527296, 140191433543679, +SNULL, 140191433519103, 140191433527295, +STORE, 140191433502720, 140191433519103, +STORE, 140191433519104, 140191433527295, +SNULL, 94467224809471, 94467224813567, +STORE, 94467224801280, 94467224809471, +STORE, 94467224809472, 94467224813567, +SNULL, 140191435788287, 140191435792383, +STORE, 140191435784192, 140191435788287, +STORE, 140191435788288, 140191435792383, +ERASE, 140191435755520, 140191435784191, +STORE, 94467251847168, 94467251982335, +STORE, 94367895400448, 94367895613439, +STORE, 94367897710592, 94367897714687, +STORE, 94367897714688, 94367897722879, +STORE, 94367897722880, 94367897735167, +STORE, 94367925264384, 94367926861823, +STORE, 139801317548032, 139801319206911, +STORE, 139801319206912, 139801321304063, +STORE, 139801321304064, 139801321320447, +STORE, 139801321320448, 139801321328639, +STORE, 139801321328640, 139801321345023, +STORE, 139801321345024, 139801321357311, +STORE, 139801321357312, 139801323450367, +STORE, 139801323450368, 139801323454463, +STORE, 139801323454464, 139801323458559, +STORE, 139801323458560, 139801323601919, +STORE, 139801323970560, 139801325654015, +STORE, 139801325654016, 139801325670399, +STORE, 139801325699072, 139801325703167, +STORE, 139801325703168, 139801325707263, +STORE, 139801325707264, 139801325711359, +STORE, 140724442861568, 140724443000831, +STORE, 140724443611136, 140724443623423, +STORE, 140724443623424, 140724443627519, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731353149440, 140737488351231, +SNULL, 140731353157631, 140737488351231, +STORE, 140731353149440, 140731353157631, +STORE, 140731353018368, 140731353157631, +STORE, 94310379503616, 94310381838335, +SNULL, 94310379716607, 94310381838335, +STORE, 94310379503616, 94310379716607, +STORE, 94310379716608, 94310381838335, +ERASE, 94310379716608, 94310381838335, +STORE, 94310381813760, 94310381826047, +STORE, 94310381826048, 94310381838335, +STORE, 140515434659840, 140515436912639, +SNULL, 140515434803199, 140515436912639, +STORE, 140515434659840, 140515434803199, +STORE, 140515434803200, 140515436912639, +ERASE, 140515434803200, 140515436912639, +STORE, 140515436900352, 140515436908543, +STORE, 140515436908544, 140515436912639, +STORE, 140731353886720, 140731353890815, +STORE, 140731353874432, 140731353886719, +STORE, 140515436871680, 140515436900351, +STORE, 140515436863488, 140515436871679, +STORE, 140515432546304, 140515434659839, +SNULL, 140515432546304, 140515432558591, +STORE, 140515432558592, 140515434659839, +STORE, 140515432546304, 140515432558591, +SNULL, 140515434651647, 140515434659839, +STORE, 140515432558592, 140515434651647, +STORE, 140515434651648, 140515434659839, +ERASE, 140515434651648, 140515434659839, +STORE, 140515434651648, 140515434659839, +STORE, 140515428749312, 140515432546303, +SNULL, 140515428749312, 140515430408191, +STORE, 140515430408192, 140515432546303, +STORE, 140515428749312, 140515430408191, +SNULL, 140515432505343, 140515432546303, +STORE, 140515430408192, 140515432505343, +STORE, 140515432505344, 140515432546303, +SNULL, 140515432505344, 140515432529919, +STORE, 140515432529920, 140515432546303, +STORE, 140515432505344, 140515432529919, +ERASE, 140515432505344, 140515432529919, +STORE, 140515432505344, 140515432529919, +ERASE, 140515432529920, 140515432546303, +STORE, 140515432529920, 140515432546303, +STORE, 140515436855296, 140515436871679, +SNULL, 140515432521727, 140515432529919, +STORE, 140515432505344, 140515432521727, +STORE, 140515432521728, 140515432529919, +SNULL, 140515434655743, 140515434659839, +STORE, 140515434651648, 140515434655743, +STORE, 140515434655744, 140515434659839, +SNULL, 94310381817855, 94310381826047, +STORE, 94310381813760, 94310381817855, +STORE, 94310381817856, 94310381826047, +SNULL, 140515436904447, 140515436908543, +STORE, 140515436900352, 140515436904447, +STORE, 140515436904448, 140515436908543, +ERASE, 140515436871680, 140515436900351, +STORE, 94310395457536, 94310395592703, +STORE, 140515435171840, 140515436855295, +STORE, 94310395457536, 94310395727871, +STORE, 94310395457536, 94310395863039, +STORE, 94310395457536, 94310396047359, +SNULL, 94310396022783, 94310396047359, +STORE, 94310395457536, 94310396022783, +STORE, 94310396022784, 94310396047359, +ERASE, 94310396022784, 94310396047359, +STORE, 94310395457536, 94310396157951, +STORE, 94310395457536, 94310396293119, +SNULL, 94310396276735, 94310396293119, +STORE, 94310395457536, 94310396276735, +STORE, 94310396276736, 94310396293119, +ERASE, 94310396276736, 94310396293119, +STORE, 94310395457536, 94310396411903, +SNULL, 94310396383231, 94310396411903, +STORE, 94310395457536, 94310396383231, +STORE, 94310396383232, 94310396411903, +ERASE, 94310396383232, 94310396411903, +STORE, 94310395457536, 94310396522495, +STORE, 94310395457536, 94310396674047, +SNULL, 94310396657663, 94310396674047, +STORE, 94310395457536, 94310396657663, +STORE, 94310396657664, 94310396674047, +ERASE, 94310396657664, 94310396674047, +SNULL, 94310396624895, 94310396657663, +STORE, 94310395457536, 94310396624895, +STORE, 94310396624896, 94310396657663, +ERASE, 94310396624896, 94310396657663, +STORE, 94310395457536, 94310396776447, +SNULL, 94310396764159, 94310396776447, +STORE, 94310395457536, 94310396764159, +STORE, 94310396764160, 94310396776447, +ERASE, 94310396764160, 94310396776447, +SNULL, 94310396739583, 94310396764159, +STORE, 94310395457536, 94310396739583, +STORE, 94310396739584, 94310396764159, +ERASE, 94310396739584, 94310396764159, +STORE, 94310395457536, 94310396882943, +STORE, 94310395457536, 94310397018111, +STORE, 94310395457536, 94310397161471, +STORE, 94310395457536, 94310397300735, +SNULL, 94310397292543, 94310397300735, +STORE, 94310395457536, 94310397292543, +STORE, 94310397292544, 94310397300735, +ERASE, 94310397292544, 94310397300735, +STORE, 94359222210560, 94359222423551, +STORE, 94359224520704, 94359224524799, +STORE, 94359224524800, 94359224532991, +STORE, 94359224532992, 94359224545279, +STORE, 94359238348800, 94359239385087, +STORE, 140675699838976, 140675701497855, +STORE, 140675701497856, 140675703595007, +STORE, 140675703595008, 140675703611391, +STORE, 140675703611392, 140675703619583, +STORE, 140675703619584, 140675703635967, +STORE, 140675703635968, 140675703648255, +STORE, 140675703648256, 140675705741311, +STORE, 140675705741312, 140675705745407, +STORE, 140675705745408, 140675705749503, +STORE, 140675705749504, 140675705892863, +STORE, 140675706261504, 140675707944959, +STORE, 140675707944960, 140675707961343, +STORE, 140675707990016, 140675707994111, +STORE, 140675707994112, 140675707998207, +STORE, 140675707998208, 140675708002303, +STORE, 140721324634112, 140721324773375, +STORE, 140721324810240, 140721324822527, +STORE, 140721324822528, 140721324826623, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724099678208, 140737488351231, +SNULL, 140724099686399, 140737488351231, +STORE, 140724099678208, 140724099686399, +STORE, 140724099547136, 140724099686399, +STORE, 94586638516224, 94586640850943, +SNULL, 94586638729215, 94586640850943, +STORE, 94586638516224, 94586638729215, +STORE, 94586638729216, 94586640850943, +ERASE, 94586638729216, 94586640850943, +STORE, 94586640826368, 94586640838655, +STORE, 94586640838656, 94586640850943, +STORE, 140371033796608, 140371036049407, +SNULL, 140371033939967, 140371036049407, +STORE, 140371033796608, 140371033939967, +STORE, 140371033939968, 140371036049407, +ERASE, 140371033939968, 140371036049407, +STORE, 140371036037120, 140371036045311, +STORE, 140371036045312, 140371036049407, +STORE, 140724100001792, 140724100005887, +STORE, 140724099989504, 140724100001791, +STORE, 140371036008448, 140371036037119, +STORE, 140371036000256, 140371036008447, +STORE, 140371031683072, 140371033796607, +SNULL, 140371031683072, 140371031695359, +STORE, 140371031695360, 140371033796607, +STORE, 140371031683072, 140371031695359, +SNULL, 140371033788415, 140371033796607, +STORE, 140371031695360, 140371033788415, +STORE, 140371033788416, 140371033796607, +ERASE, 140371033788416, 140371033796607, +STORE, 140371033788416, 140371033796607, +STORE, 140371027886080, 140371031683071, +SNULL, 140371027886080, 140371029544959, +STORE, 140371029544960, 140371031683071, +STORE, 140371027886080, 140371029544959, +SNULL, 140371031642111, 140371031683071, +STORE, 140371029544960, 140371031642111, +STORE, 140371031642112, 140371031683071, +SNULL, 140371031642112, 140371031666687, +STORE, 140371031666688, 140371031683071, +STORE, 140371031642112, 140371031666687, +ERASE, 140371031642112, 140371031666687, +STORE, 140371031642112, 140371031666687, +ERASE, 140371031666688, 140371031683071, +STORE, 140371031666688, 140371031683071, +STORE, 140371035992064, 140371036008447, +SNULL, 140371031658495, 140371031666687, +STORE, 140371031642112, 140371031658495, +STORE, 140371031658496, 140371031666687, +SNULL, 140371033792511, 140371033796607, +STORE, 140371033788416, 140371033792511, +STORE, 140371033792512, 140371033796607, +SNULL, 94586640830463, 94586640838655, +STORE, 94586640826368, 94586640830463, +STORE, 94586640830464, 94586640838655, +SNULL, 140371036041215, 140371036045311, +STORE, 140371036037120, 140371036041215, +STORE, 140371036041216, 140371036045311, +ERASE, 140371036008448, 140371036037119, +STORE, 94586663849984, 94586663985151, +STORE, 140371034308608, 140371035992063, +STORE, 94586663849984, 94586664120319, +STORE, 94586663849984, 94586664255487, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727532937216, 140737488351231, +SNULL, 140727532945407, 140737488351231, +STORE, 140727532937216, 140727532945407, +STORE, 140727532806144, 140727532945407, +STORE, 94849780191232, 94849782525951, +SNULL, 94849780404223, 94849782525951, +STORE, 94849780191232, 94849780404223, +STORE, 94849780404224, 94849782525951, +ERASE, 94849780404224, 94849782525951, +STORE, 94849782501376, 94849782513663, +STORE, 94849782513664, 94849782525951, +STORE, 140382070218752, 140382072471551, +SNULL, 140382070362111, 140382072471551, +STORE, 140382070218752, 140382070362111, +STORE, 140382070362112, 140382072471551, +ERASE, 140382070362112, 140382072471551, +STORE, 140382072459264, 140382072467455, +STORE, 140382072467456, 140382072471551, +STORE, 140727533092864, 140727533096959, +STORE, 140727533080576, 140727533092863, +STORE, 140382072430592, 140382072459263, +STORE, 140382072422400, 140382072430591, +STORE, 140382068105216, 140382070218751, +SNULL, 140382068105216, 140382068117503, +STORE, 140382068117504, 140382070218751, +STORE, 140382068105216, 140382068117503, +SNULL, 140382070210559, 140382070218751, +STORE, 140382068117504, 140382070210559, +STORE, 140382070210560, 140382070218751, +ERASE, 140382070210560, 140382070218751, +STORE, 140382070210560, 140382070218751, +STORE, 140382064308224, 140382068105215, +SNULL, 140382064308224, 140382065967103, +STORE, 140382065967104, 140382068105215, +STORE, 140382064308224, 140382065967103, +SNULL, 140382068064255, 140382068105215, +STORE, 140382065967104, 140382068064255, +STORE, 140382068064256, 140382068105215, +SNULL, 140382068064256, 140382068088831, +STORE, 140382068088832, 140382068105215, +STORE, 140382068064256, 140382068088831, +ERASE, 140382068064256, 140382068088831, +STORE, 140382068064256, 140382068088831, +ERASE, 140382068088832, 140382068105215, +STORE, 140382068088832, 140382068105215, +STORE, 140382072414208, 140382072430591, +SNULL, 140382068080639, 140382068088831, +STORE, 140382068064256, 140382068080639, +STORE, 140382068080640, 140382068088831, +SNULL, 140382070214655, 140382070218751, +STORE, 140382070210560, 140382070214655, +STORE, 140382070214656, 140382070218751, +SNULL, 94849782505471, 94849782513663, +STORE, 94849782501376, 94849782505471, +STORE, 94849782505472, 94849782513663, +SNULL, 140382072463359, 140382072467455, +STORE, 140382072459264, 140382072463359, +STORE, 140382072463360, 140382072467455, +ERASE, 140382072430592, 140382072459263, +STORE, 94849782845440, 94849782980607, +STORE, 140382070730752, 140382072414207, +STORE, 94849782845440, 94849783115775, +STORE, 94849782845440, 94849783250943, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722594377728, 140737488351231, +SNULL, 140722594385919, 140737488351231, +STORE, 140722594377728, 140722594385919, +STORE, 140722594246656, 140722594385919, +STORE, 94421466353664, 94421468577791, +SNULL, 94421466464255, 94421468577791, +STORE, 94421466353664, 94421466464255, +STORE, 94421466464256, 94421468577791, +ERASE, 94421466464256, 94421468577791, +STORE, 94421468557312, 94421468569599, +STORE, 94421468569600, 94421468577791, +STORE, 140345458057216, 140345460310015, +SNULL, 140345458200575, 140345460310015, +STORE, 140345458057216, 140345458200575, +STORE, 140345458200576, 140345460310015, +ERASE, 140345458200576, 140345460310015, +STORE, 140345460297728, 140345460305919, +STORE, 140345460305920, 140345460310015, +STORE, 140722595557376, 140722595561471, +STORE, 140722595545088, 140722595557375, +STORE, 140345460269056, 140345460297727, +STORE, 140345460260864, 140345460269055, +STORE, 140345454260224, 140345458057215, +SNULL, 140345454260224, 140345455919103, +STORE, 140345455919104, 140345458057215, +STORE, 140345454260224, 140345455919103, +SNULL, 140345458016255, 140345458057215, +STORE, 140345455919104, 140345458016255, +STORE, 140345458016256, 140345458057215, +SNULL, 140345458016256, 140345458040831, +STORE, 140345458040832, 140345458057215, +STORE, 140345458016256, 140345458040831, +ERASE, 140345458016256, 140345458040831, +STORE, 140345458016256, 140345458040831, +ERASE, 140345458040832, 140345458057215, +STORE, 140345458040832, 140345458057215, +SNULL, 140345458032639, 140345458040831, +STORE, 140345458016256, 140345458032639, +STORE, 140345458032640, 140345458040831, +SNULL, 94421468565503, 94421468569599, +STORE, 94421468557312, 94421468565503, +STORE, 94421468565504, 94421468569599, +SNULL, 140345460301823, 140345460305919, +STORE, 140345460297728, 140345460301823, +STORE, 140345460301824, 140345460305919, +ERASE, 140345460269056, 140345460297727, +STORE, 94421496004608, 94421496139775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726096302080, 140737488351231, +SNULL, 140726096310271, 140737488351231, +STORE, 140726096302080, 140726096310271, +STORE, 140726096171008, 140726096310271, +STORE, 94101992124416, 94101994459135, +SNULL, 94101992337407, 94101994459135, +STORE, 94101992124416, 94101992337407, +STORE, 94101992337408, 94101994459135, +ERASE, 94101992337408, 94101994459135, +STORE, 94101994434560, 94101994446847, +STORE, 94101994446848, 94101994459135, +STORE, 140192085594112, 140192087846911, +SNULL, 140192085737471, 140192087846911, +STORE, 140192085594112, 140192085737471, +STORE, 140192085737472, 140192087846911, +ERASE, 140192085737472, 140192087846911, +STORE, 140192087834624, 140192087842815, +STORE, 140192087842816, 140192087846911, +STORE, 140726096375808, 140726096379903, +STORE, 140726096363520, 140726096375807, +STORE, 140192087805952, 140192087834623, +STORE, 140192087797760, 140192087805951, +STORE, 140192083480576, 140192085594111, +SNULL, 140192083480576, 140192083492863, +STORE, 140192083492864, 140192085594111, +STORE, 140192083480576, 140192083492863, +SNULL, 140192085585919, 140192085594111, +STORE, 140192083492864, 140192085585919, +STORE, 140192085585920, 140192085594111, +ERASE, 140192085585920, 140192085594111, +STORE, 140192085585920, 140192085594111, +STORE, 140192079683584, 140192083480575, +SNULL, 140192079683584, 140192081342463, +STORE, 140192081342464, 140192083480575, +STORE, 140192079683584, 140192081342463, +SNULL, 140192083439615, 140192083480575, +STORE, 140192081342464, 140192083439615, +STORE, 140192083439616, 140192083480575, +SNULL, 140192083439616, 140192083464191, +STORE, 140192083464192, 140192083480575, +STORE, 140192083439616, 140192083464191, +ERASE, 140192083439616, 140192083464191, +STORE, 140192083439616, 140192083464191, +ERASE, 140192083464192, 140192083480575, +STORE, 140192083464192, 140192083480575, +STORE, 140192087789568, 140192087805951, +SNULL, 140192083455999, 140192083464191, +STORE, 140192083439616, 140192083455999, +STORE, 140192083456000, 140192083464191, +SNULL, 140192085590015, 140192085594111, +STORE, 140192085585920, 140192085590015, +STORE, 140192085590016, 140192085594111, +SNULL, 94101994438655, 94101994446847, +STORE, 94101994434560, 94101994438655, +STORE, 94101994438656, 94101994446847, +SNULL, 140192087838719, 140192087842815, +STORE, 140192087834624, 140192087838719, +STORE, 140192087838720, 140192087842815, +ERASE, 140192087805952, 140192087834623, +STORE, 94102011887616, 94102012022783, +STORE, 140192086106112, 140192087789567, +STORE, 94102011887616, 94102012157951, +STORE, 94102011887616, 94102012293119, +STORE, 94102011887616, 94102012440575, +SNULL, 94102012428287, 94102012440575, +STORE, 94102011887616, 94102012428287, +STORE, 94102012428288, 94102012440575, +ERASE, 94102012428288, 94102012440575, +STORE, 94102011887616, 94102012579839, +STORE, 94102011887616, 94102012715007, +SNULL, 94102012694527, 94102012715007, +STORE, 94102011887616, 94102012694527, +STORE, 94102012694528, 94102012715007, +ERASE, 94102012694528, 94102012715007, +STORE, 94102011887616, 94102012833791, +STORE, 94102011887616, 94102012968959, +SNULL, 94102012927999, 94102012968959, +STORE, 94102011887616, 94102012927999, +STORE, 94102012928000, 94102012968959, +ERASE, 94102012928000, 94102012968959, +STORE, 94102011887616, 94102013091839, +SNULL, 94102013075455, 94102013091839, +STORE, 94102011887616, 94102013075455, +STORE, 94102013075456, 94102013091839, +ERASE, 94102013075456, 94102013091839, +STORE, 94102011887616, 94102013210623, +STORE, 94102011887616, 94102013345791, +STORE, 93968727965696, 93968728178687, +STORE, 93968730275840, 93968730279935, +STORE, 93968730279936, 93968730288127, +STORE, 93968730288128, 93968730300415, +STORE, 93968731140096, 93968732704767, +STORE, 140588443168768, 140588444827647, +STORE, 140588444827648, 140588446924799, +STORE, 140588446924800, 140588446941183, +STORE, 140588446941184, 140588446949375, +STORE, 140588446949376, 140588446965759, +STORE, 140588446965760, 140588446978047, +STORE, 140588446978048, 140588449071103, +STORE, 140588449071104, 140588449075199, +STORE, 140588449075200, 140588449079295, +STORE, 140588449079296, 140588449222655, +STORE, 140588449591296, 140588451274751, +STORE, 140588451274752, 140588451291135, +STORE, 140588451319808, 140588451323903, +STORE, 140588451323904, 140588451327999, +STORE, 140588451328000, 140588451332095, +STORE, 140733877239808, 140733877379071, +STORE, 140733878702080, 140733878714367, +STORE, 140733878714368, 140733878718463, +STORE, 93968727965696, 93968728178687, +STORE, 93968730275840, 93968730279935, +STORE, 93968730279936, 93968730288127, +STORE, 93968730288128, 93968730300415, +STORE, 93968731140096, 93968732991487, +STORE, 140588443168768, 140588444827647, +STORE, 140588444827648, 140588446924799, +STORE, 140588446924800, 140588446941183, +STORE, 140588446941184, 140588446949375, +STORE, 140588446949376, 140588446965759, +STORE, 140588446965760, 140588446978047, +STORE, 140588446978048, 140588449071103, +STORE, 140588449071104, 140588449075199, +STORE, 140588449075200, 140588449079295, +STORE, 140588449079296, 140588449222655, +STORE, 140588449591296, 140588451274751, +STORE, 140588451274752, 140588451291135, +STORE, 140588451319808, 140588451323903, +STORE, 140588451323904, 140588451327999, +STORE, 140588451328000, 140588451332095, +STORE, 140733877239808, 140733877379071, +STORE, 140733878702080, 140733878714367, +STORE, 140733878714368, 140733878718463, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733054472192, 140737488351231, +SNULL, 140733054480383, 140737488351231, +STORE, 140733054472192, 140733054480383, +STORE, 140733054341120, 140733054480383, +STORE, 93992873623552, 93992875847679, +SNULL, 93992873734143, 93992875847679, +STORE, 93992873623552, 93992873734143, +STORE, 93992873734144, 93992875847679, +ERASE, 93992873734144, 93992875847679, +STORE, 93992875827200, 93992875839487, +STORE, 93992875839488, 93992875847679, +STORE, 139790881488896, 139790883741695, +SNULL, 139790881632255, 139790883741695, +STORE, 139790881488896, 139790881632255, +STORE, 139790881632256, 139790883741695, +ERASE, 139790881632256, 139790883741695, +STORE, 139790883729408, 139790883737599, +STORE, 139790883737600, 139790883741695, +STORE, 140733054754816, 140733054758911, +STORE, 140733054742528, 140733054754815, +STORE, 139790883700736, 139790883729407, +STORE, 139790883692544, 139790883700735, +STORE, 139790877691904, 139790881488895, +SNULL, 139790877691904, 139790879350783, +STORE, 139790879350784, 139790881488895, +STORE, 139790877691904, 139790879350783, +SNULL, 139790881447935, 139790881488895, +STORE, 139790879350784, 139790881447935, +STORE, 139790881447936, 139790881488895, +SNULL, 139790881447936, 139790881472511, +STORE, 139790881472512, 139790881488895, +STORE, 139790881447936, 139790881472511, +ERASE, 139790881447936, 139790881472511, +STORE, 139790881447936, 139790881472511, +ERASE, 139790881472512, 139790881488895, +STORE, 139790881472512, 139790881488895, +SNULL, 139790881464319, 139790881472511, +STORE, 139790881447936, 139790881464319, +STORE, 139790881464320, 139790881472511, +SNULL, 93992875835391, 93992875839487, +STORE, 93992875827200, 93992875835391, +STORE, 93992875835392, 93992875839487, +SNULL, 139790883733503, 139790883737599, +STORE, 139790883729408, 139790883733503, +STORE, 139790883733504, 139790883737599, +ERASE, 139790883700736, 139790883729407, +STORE, 93992877031424, 93992877166591, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728550887424, 140737488351231, +SNULL, 140728550895615, 140737488351231, +STORE, 140728550887424, 140728550895615, +STORE, 140728550756352, 140728550895615, +STORE, 94707634077696, 94707636301823, +SNULL, 94707634188287, 94707636301823, +STORE, 94707634077696, 94707634188287, +STORE, 94707634188288, 94707636301823, +ERASE, 94707634188288, 94707636301823, +STORE, 94707636281344, 94707636293631, +STORE, 94707636293632, 94707636301823, +STORE, 140553545666560, 140553547919359, +SNULL, 140553545809919, 140553547919359, +STORE, 140553545666560, 140553545809919, +STORE, 140553545809920, 140553547919359, +ERASE, 140553545809920, 140553547919359, +STORE, 140553547907072, 140553547915263, +STORE, 140553547915264, 140553547919359, +STORE, 140728552374272, 140728552378367, +STORE, 140728552361984, 140728552374271, +STORE, 140553547878400, 140553547907071, +STORE, 140553547870208, 140553547878399, +STORE, 140553541869568, 140553545666559, +SNULL, 140553541869568, 140553543528447, +STORE, 140553543528448, 140553545666559, +STORE, 140553541869568, 140553543528447, +SNULL, 140553545625599, 140553545666559, +STORE, 140553543528448, 140553545625599, +STORE, 140553545625600, 140553545666559, +SNULL, 140553545625600, 140553545650175, +STORE, 140553545650176, 140553545666559, +STORE, 140553545625600, 140553545650175, +ERASE, 140553545625600, 140553545650175, +STORE, 140553545625600, 140553545650175, +ERASE, 140553545650176, 140553545666559, +STORE, 140553545650176, 140553545666559, +SNULL, 140553545641983, 140553545650175, +STORE, 140553545625600, 140553545641983, +STORE, 140553545641984, 140553545650175, +SNULL, 94707636289535, 94707636293631, +STORE, 94707636281344, 94707636289535, +STORE, 94707636289536, 94707636293631, +SNULL, 140553547911167, 140553547915263, +STORE, 140553547907072, 140553547911167, +STORE, 140553547911168, 140553547915263, +ERASE, 140553547878400, 140553547907071, +STORE, 94707651411968, 94707651547135, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732168695808, 140737488351231, +SNULL, 140732168703999, 140737488351231, +STORE, 140732168695808, 140732168703999, +STORE, 140732168564736, 140732168703999, +STORE, 94454287859712, 94454290083839, +SNULL, 94454287970303, 94454290083839, +STORE, 94454287859712, 94454287970303, +STORE, 94454287970304, 94454290083839, +ERASE, 94454287970304, 94454290083839, +STORE, 94454290063360, 94454290075647, +STORE, 94454290075648, 94454290083839, +STORE, 140564947107840, 140564949360639, +SNULL, 140564947251199, 140564949360639, +STORE, 140564947107840, 140564947251199, +STORE, 140564947251200, 140564949360639, +ERASE, 140564947251200, 140564949360639, +STORE, 140564949348352, 140564949356543, +STORE, 140564949356544, 140564949360639, +STORE, 140732168843264, 140732168847359, +STORE, 140732168830976, 140732168843263, +STORE, 140564949319680, 140564949348351, +STORE, 140564949311488, 140564949319679, +STORE, 140564943310848, 140564947107839, +SNULL, 140564943310848, 140564944969727, +STORE, 140564944969728, 140564947107839, +STORE, 140564943310848, 140564944969727, +SNULL, 140564947066879, 140564947107839, +STORE, 140564944969728, 140564947066879, +STORE, 140564947066880, 140564947107839, +SNULL, 140564947066880, 140564947091455, +STORE, 140564947091456, 140564947107839, +STORE, 140564947066880, 140564947091455, +ERASE, 140564947066880, 140564947091455, +STORE, 140564947066880, 140564947091455, +ERASE, 140564947091456, 140564947107839, +STORE, 140564947091456, 140564947107839, +SNULL, 140564947083263, 140564947091455, +STORE, 140564947066880, 140564947083263, +STORE, 140564947083264, 140564947091455, +SNULL, 94454290071551, 94454290075647, +STORE, 94454290063360, 94454290071551, +STORE, 94454290071552, 94454290075647, +SNULL, 140564949352447, 140564949356543, +STORE, 140564949348352, 140564949352447, +STORE, 140564949352448, 140564949356543, +ERASE, 140564949319680, 140564949348351, +STORE, 94454316236800, 94454316371967, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735155617792, 140737488351231, +SNULL, 140735155625983, 140737488351231, +STORE, 140735155617792, 140735155625983, +STORE, 140735155486720, 140735155625983, +STORE, 93915969556480, 93915971780607, +SNULL, 93915969667071, 93915971780607, +STORE, 93915969556480, 93915969667071, +STORE, 93915969667072, 93915971780607, +ERASE, 93915969667072, 93915971780607, +STORE, 93915971760128, 93915971772415, +STORE, 93915971772416, 93915971780607, +STORE, 140141164605440, 140141166858239, +SNULL, 140141164748799, 140141166858239, +STORE, 140141164605440, 140141164748799, +STORE, 140141164748800, 140141166858239, +ERASE, 140141164748800, 140141166858239, +STORE, 140141166845952, 140141166854143, +STORE, 140141166854144, 140141166858239, +STORE, 140735155691520, 140735155695615, +STORE, 140735155679232, 140735155691519, +STORE, 140141166817280, 140141166845951, +STORE, 140141166809088, 140141166817279, +STORE, 140141160808448, 140141164605439, +SNULL, 140141160808448, 140141162467327, +STORE, 140141162467328, 140141164605439, +STORE, 140141160808448, 140141162467327, +SNULL, 140141164564479, 140141164605439, +STORE, 140141162467328, 140141164564479, +STORE, 140141164564480, 140141164605439, +SNULL, 140141164564480, 140141164589055, +STORE, 140141164589056, 140141164605439, +STORE, 140141164564480, 140141164589055, +ERASE, 140141164564480, 140141164589055, +STORE, 140141164564480, 140141164589055, +ERASE, 140141164589056, 140141164605439, +STORE, 140141164589056, 140141164605439, +SNULL, 140141164580863, 140141164589055, +STORE, 140141164564480, 140141164580863, +STORE, 140141164580864, 140141164589055, +SNULL, 93915971768319, 93915971772415, +STORE, 93915971760128, 93915971768319, +STORE, 93915971768320, 93915971772415, +SNULL, 140141166850047, 140141166854143, +STORE, 140141166845952, 140141166850047, +STORE, 140141166850048, 140141166854143, +ERASE, 140141166817280, 140141166845951, +STORE, 93916002775040, 93916002910207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728988409856, 140737488351231, +SNULL, 140728988418047, 140737488351231, +STORE, 140728988409856, 140728988418047, +STORE, 140728988278784, 140728988418047, +STORE, 94021634813952, 94021637038079, +SNULL, 94021634924543, 94021637038079, +STORE, 94021634813952, 94021634924543, +STORE, 94021634924544, 94021637038079, +ERASE, 94021634924544, 94021637038079, +STORE, 94021637017600, 94021637029887, +STORE, 94021637029888, 94021637038079, +STORE, 140638014038016, 140638016290815, +SNULL, 140638014181375, 140638016290815, +STORE, 140638014038016, 140638014181375, +STORE, 140638014181376, 140638016290815, +ERASE, 140638014181376, 140638016290815, +STORE, 140638016278528, 140638016286719, +STORE, 140638016286720, 140638016290815, +STORE, 140728988536832, 140728988540927, +STORE, 140728988524544, 140728988536831, +STORE, 140638016249856, 140638016278527, +STORE, 140638016241664, 140638016249855, +STORE, 140638010241024, 140638014038015, +SNULL, 140638010241024, 140638011899903, +STORE, 140638011899904, 140638014038015, +STORE, 140638010241024, 140638011899903, +SNULL, 140638013997055, 140638014038015, +STORE, 140638011899904, 140638013997055, +STORE, 140638013997056, 140638014038015, +SNULL, 140638013997056, 140638014021631, +STORE, 140638014021632, 140638014038015, +STORE, 140638013997056, 140638014021631, +ERASE, 140638013997056, 140638014021631, +STORE, 140638013997056, 140638014021631, +ERASE, 140638014021632, 140638014038015, +STORE, 140638014021632, 140638014038015, +SNULL, 140638014013439, 140638014021631, +STORE, 140638013997056, 140638014013439, +STORE, 140638014013440, 140638014021631, +SNULL, 94021637025791, 94021637029887, +STORE, 94021637017600, 94021637025791, +STORE, 94021637025792, 94021637029887, +SNULL, 140638016282623, 140638016286719, +STORE, 140638016278528, 140638016282623, +STORE, 140638016282624, 140638016286719, +ERASE, 140638016249856, 140638016278527, +STORE, 94021643124736, 94021643259903, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731219275776, 140737488351231, +SNULL, 140731219283967, 140737488351231, +STORE, 140731219275776, 140731219283967, +STORE, 140731219144704, 140731219283967, +STORE, 93888803647488, 93888805871615, +SNULL, 93888803758079, 93888805871615, +STORE, 93888803647488, 93888803758079, +STORE, 93888803758080, 93888805871615, +ERASE, 93888803758080, 93888805871615, +STORE, 93888805851136, 93888805863423, +STORE, 93888805863424, 93888805871615, +STORE, 139630576934912, 139630579187711, +SNULL, 139630577078271, 139630579187711, +STORE, 139630576934912, 139630577078271, +STORE, 139630577078272, 139630579187711, +ERASE, 139630577078272, 139630579187711, +STORE, 139630579175424, 139630579183615, +STORE, 139630579183616, 139630579187711, +STORE, 140731219718144, 140731219722239, +STORE, 140731219705856, 140731219718143, +STORE, 139630579146752, 139630579175423, +STORE, 139630579138560, 139630579146751, +STORE, 139630573137920, 139630576934911, +SNULL, 139630573137920, 139630574796799, +STORE, 139630574796800, 139630576934911, +STORE, 139630573137920, 139630574796799, +SNULL, 139630576893951, 139630576934911, +STORE, 139630574796800, 139630576893951, +STORE, 139630576893952, 139630576934911, +SNULL, 139630576893952, 139630576918527, +STORE, 139630576918528, 139630576934911, +STORE, 139630576893952, 139630576918527, +ERASE, 139630576893952, 139630576918527, +STORE, 139630576893952, 139630576918527, +ERASE, 139630576918528, 139630576934911, +STORE, 139630576918528, 139630576934911, +SNULL, 139630576910335, 139630576918527, +STORE, 139630576893952, 139630576910335, +STORE, 139630576910336, 139630576918527, +SNULL, 93888805859327, 93888805863423, +STORE, 93888805851136, 93888805859327, +STORE, 93888805859328, 93888805863423, +SNULL, 139630579179519, 139630579183615, +STORE, 139630579175424, 139630579179519, +STORE, 139630579179520, 139630579183615, +ERASE, 139630579146752, 139630579175423, +STORE, 93888822235136, 93888822370303, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733391151104, 140737488351231, +SNULL, 140733391159295, 140737488351231, +STORE, 140733391151104, 140733391159295, +STORE, 140733391020032, 140733391159295, +STORE, 94393875324928, 94393877549055, +SNULL, 94393875435519, 94393877549055, +STORE, 94393875324928, 94393875435519, +STORE, 94393875435520, 94393877549055, +ERASE, 94393875435520, 94393877549055, +STORE, 94393877528576, 94393877540863, +STORE, 94393877540864, 94393877549055, +STORE, 140292111740928, 140292113993727, +SNULL, 140292111884287, 140292113993727, +STORE, 140292111740928, 140292111884287, +STORE, 140292111884288, 140292113993727, +ERASE, 140292111884288, 140292113993727, +STORE, 140292113981440, 140292113989631, +STORE, 140292113989632, 140292113993727, +STORE, 140733391532032, 140733391536127, +STORE, 140733391519744, 140733391532031, +STORE, 140292113952768, 140292113981439, +STORE, 140292113944576, 140292113952767, +STORE, 140292107943936, 140292111740927, +SNULL, 140292107943936, 140292109602815, +STORE, 140292109602816, 140292111740927, +STORE, 140292107943936, 140292109602815, +SNULL, 140292111699967, 140292111740927, +STORE, 140292109602816, 140292111699967, +STORE, 140292111699968, 140292111740927, +SNULL, 140292111699968, 140292111724543, +STORE, 140292111724544, 140292111740927, +STORE, 140292111699968, 140292111724543, +ERASE, 140292111699968, 140292111724543, +STORE, 140292111699968, 140292111724543, +ERASE, 140292111724544, 140292111740927, +STORE, 140292111724544, 140292111740927, +SNULL, 140292111716351, 140292111724543, +STORE, 140292111699968, 140292111716351, +STORE, 140292111716352, 140292111724543, +SNULL, 94393877536767, 94393877540863, +STORE, 94393877528576, 94393877536767, +STORE, 94393877536768, 94393877540863, +SNULL, 140292113985535, 140292113989631, +STORE, 140292113981440, 140292113985535, +STORE, 140292113985536, 140292113989631, +ERASE, 140292113952768, 140292113981439, +STORE, 94393909342208, 94393909477375, +STORE, 94458367512576, 94458367725567, +STORE, 94458369822720, 94458369826815, +STORE, 94458369826816, 94458369835007, +STORE, 94458369835008, 94458369847295, +STORE, 94458393292800, 94458399666175, +STORE, 140619773841408, 140619775500287, +STORE, 140619775500288, 140619777597439, +STORE, 140619777597440, 140619777613823, +STORE, 140619777613824, 140619777622015, +STORE, 140619777622016, 140619777638399, +STORE, 140619777638400, 140619777650687, +STORE, 140619777650688, 140619779743743, +STORE, 140619779743744, 140619779747839, +STORE, 140619779747840, 140619779751935, +STORE, 140619779751936, 140619779895295, +STORE, 140619780263936, 140619781947391, +STORE, 140619781947392, 140619781963775, +STORE, 140619781992448, 140619781996543, +STORE, 140619781996544, 140619782000639, +STORE, 140619782000640, 140619782004735, +STORE, 140725811675136, 140725811814399, +STORE, 140725812813824, 140725812826111, +STORE, 140725812826112, 140725812830207, +STORE, 94458367512576, 94458367725567, +STORE, 94458369822720, 94458369826815, +STORE, 94458369826816, 94458369835007, +STORE, 94458369835008, 94458369847295, +STORE, 94458393292800, 94458400366591, +STORE, 140619773841408, 140619775500287, +STORE, 140619775500288, 140619777597439, +STORE, 140619777597440, 140619777613823, +STORE, 140619777613824, 140619777622015, +STORE, 140619777622016, 140619777638399, +STORE, 140619777638400, 140619777650687, +STORE, 140619777650688, 140619779743743, +STORE, 140619779743744, 140619779747839, +STORE, 140619779747840, 140619779751935, +STORE, 140619779751936, 140619779895295, +STORE, 140619780263936, 140619781947391, +STORE, 140619781947392, 140619781963775, +STORE, 140619781992448, 140619781996543, +STORE, 140619781996544, 140619782000639, +STORE, 140619782000640, 140619782004735, +STORE, 140725811675136, 140725811814399, +STORE, 140725812813824, 140725812826111, +STORE, 140725812826112, 140725812830207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728740679680, 140737488351231, +SNULL, 140728740687871, 140737488351231, +STORE, 140728740679680, 140728740687871, +STORE, 140728740548608, 140728740687871, +STORE, 94764075249664, 94764077473791, +SNULL, 94764075360255, 94764077473791, +STORE, 94764075249664, 94764075360255, +STORE, 94764075360256, 94764077473791, +ERASE, 94764075360256, 94764077473791, +STORE, 94764077453312, 94764077465599, +STORE, 94764077465600, 94764077473791, +STORE, 139766406791168, 139766409043967, +SNULL, 139766406934527, 139766409043967, +STORE, 139766406791168, 139766406934527, +STORE, 139766406934528, 139766409043967, +ERASE, 139766406934528, 139766409043967, +STORE, 139766409031680, 139766409039871, +STORE, 139766409039872, 139766409043967, +STORE, 140728740913152, 140728740917247, +STORE, 140728740900864, 140728740913151, +STORE, 139766409003008, 139766409031679, +STORE, 139766408994816, 139766409003007, +STORE, 139766402994176, 139766406791167, +SNULL, 139766402994176, 139766404653055, +STORE, 139766404653056, 139766406791167, +STORE, 139766402994176, 139766404653055, +SNULL, 139766406750207, 139766406791167, +STORE, 139766404653056, 139766406750207, +STORE, 139766406750208, 139766406791167, +SNULL, 139766406750208, 139766406774783, +STORE, 139766406774784, 139766406791167, +STORE, 139766406750208, 139766406774783, +ERASE, 139766406750208, 139766406774783, +STORE, 139766406750208, 139766406774783, +ERASE, 139766406774784, 139766406791167, +STORE, 139766406774784, 139766406791167, +SNULL, 139766406766591, 139766406774783, +STORE, 139766406750208, 139766406766591, +STORE, 139766406766592, 139766406774783, +SNULL, 94764077461503, 94764077465599, +STORE, 94764077453312, 94764077461503, +STORE, 94764077461504, 94764077465599, +SNULL, 139766409035775, 139766409039871, +STORE, 139766409031680, 139766409035775, +STORE, 139766409035776, 139766409039871, +ERASE, 139766409003008, 139766409031679, +STORE, 94764090458112, 94764090593279, +STORE, 94758057480192, 94758057590783, +STORE, 94758059683840, 94758059692031, +STORE, 94758059692032, 94758059696127, +STORE, 94758059696128, 94758059704319, +STORE, 94758083215360, 94758083350527, +STORE, 139951456772096, 139951458430975, +STORE, 139951458430976, 139951460528127, +STORE, 139951460528128, 139951460544511, +STORE, 139951460544512, 139951460552703, +STORE, 139951460552704, 139951460569087, +STORE, 139951460569088, 139951460712447, +STORE, 139951462772736, 139951462780927, +STORE, 139951462809600, 139951462813695, +STORE, 139951462813696, 139951462817791, +STORE, 139951462817792, 139951462821887, +STORE, 140734098313216, 140734098452479, +STORE, 140734098911232, 140734098923519, +STORE, 140734098923520, 140734098927615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724904095744, 140737488351231, +SNULL, 140724904103935, 140737488351231, +STORE, 140724904095744, 140724904103935, +STORE, 140724903964672, 140724904103935, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140408497864704, 140408500117503, +SNULL, 140408498008063, 140408500117503, +STORE, 140408497864704, 140408498008063, +STORE, 140408498008064, 140408500117503, +ERASE, 140408498008064, 140408500117503, +STORE, 140408500105216, 140408500113407, +STORE, 140408500113408, 140408500117503, +STORE, 140724905369600, 140724905373695, +STORE, 140724905357312, 140724905369599, +STORE, 140408500076544, 140408500105215, +STORE, 140408500068352, 140408500076543, +STORE, 140408494702592, 140408497864703, +SNULL, 140408494702592, 140408495763455, +STORE, 140408495763456, 140408497864703, +STORE, 140408494702592, 140408495763455, +SNULL, 140408497856511, 140408497864703, +STORE, 140408495763456, 140408497856511, +STORE, 140408497856512, 140408497864703, +ERASE, 140408497856512, 140408497864703, +STORE, 140408497856512, 140408497864703, +STORE, 140408490905600, 140408494702591, +SNULL, 140408490905600, 140408492564479, +STORE, 140408492564480, 140408494702591, +STORE, 140408490905600, 140408492564479, +SNULL, 140408494661631, 140408494702591, +STORE, 140408492564480, 140408494661631, +STORE, 140408494661632, 140408494702591, +SNULL, 140408494661632, 140408494686207, +STORE, 140408494686208, 140408494702591, +STORE, 140408494661632, 140408494686207, +ERASE, 140408494661632, 140408494686207, +STORE, 140408494661632, 140408494686207, +ERASE, 140408494686208, 140408494702591, +STORE, 140408494686208, 140408494702591, +STORE, 140408500056064, 140408500076543, +SNULL, 140408494678015, 140408494686207, +STORE, 140408494661632, 140408494678015, +STORE, 140408494678016, 140408494686207, +SNULL, 140408497860607, 140408497864703, +STORE, 140408497856512, 140408497860607, +STORE, 140408497860608, 140408497864703, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140408500109311, 140408500113407, +STORE, 140408500105216, 140408500109311, +STORE, 140408500109312, 140408500113407, +ERASE, 140408500076544, 140408500105215, +STORE, 25235456, 25370623, +STORE, 25235456, 25518079, +STORE, 140408498372608, 140408500056063, +STORE, 94543937388544, 94543937499135, +STORE, 94543939592192, 94543939600383, +STORE, 94543939600384, 94543939604479, +STORE, 94543939604480, 94543939612671, +STORE, 94543941447680, 94543941582847, +STORE, 140282621947904, 140282623606783, +STORE, 140282623606784, 140282625703935, +STORE, 140282625703936, 140282625720319, +STORE, 140282625720320, 140282625728511, +STORE, 140282625728512, 140282625744895, +STORE, 140282625744896, 140282625888255, +STORE, 140282627948544, 140282627956735, +STORE, 140282627985408, 140282627989503, +STORE, 140282627989504, 140282627993599, +STORE, 140282627993600, 140282627997695, +STORE, 140728295723008, 140728295862271, +STORE, 140728296476672, 140728296488959, +STORE, 140728296488960, 140728296493055, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431510691839, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431510691839, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140736944451584, 140737488351231, +SNULL, 140736944463871, 140737488351231, +STORE, 140736944451584, 140736944463871, +STORE, 140736944320512, 140736944463871, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 139693609893888, 139693612146687, +SNULL, 139693610037247, 139693612146687, +STORE, 139693609893888, 139693610037247, +STORE, 139693610037248, 139693612146687, +ERASE, 139693610037248, 139693612146687, +STORE, 139693612134400, 139693612142591, +STORE, 139693612142592, 139693612146687, +STORE, 140736945152000, 140736945156095, +STORE, 140736945139712, 140736945151999, +STORE, 139693612105728, 139693612134399, +STORE, 139693612097536, 139693612105727, +STORE, 139693606060032, 139693609893887, +SNULL, 139693606060032, 139693607768063, +STORE, 139693607768064, 139693609893887, +STORE, 139693606060032, 139693607768063, +SNULL, 139693609861119, 139693609893887, +STORE, 139693607768064, 139693609861119, +STORE, 139693609861120, 139693609893887, +ERASE, 139693609861120, 139693609893887, +STORE, 139693609861120, 139693609893887, +STORE, 139693603864576, 139693606060031, +SNULL, 139693603864576, 139693603958783, +STORE, 139693603958784, 139693606060031, +STORE, 139693603864576, 139693603958783, +SNULL, 139693606051839, 139693606060031, +STORE, 139693603958784, 139693606051839, +STORE, 139693606051840, 139693606060031, +ERASE, 139693606051840, 139693606060031, +STORE, 139693606051840, 139693606060031, +STORE, 139693601345536, 139693603864575, +SNULL, 139693601345536, 139693601759231, +STORE, 139693601759232, 139693603864575, +STORE, 139693601345536, 139693601759231, +SNULL, 139693603852287, 139693603864575, +STORE, 139693601759232, 139693603852287, +STORE, 139693603852288, 139693603864575, +ERASE, 139693603852288, 139693603864575, +STORE, 139693603852288, 139693603864575, +STORE, 139693598711808, 139693601345535, +SNULL, 139693598711808, 139693599240191, +STORE, 139693599240192, 139693601345535, +STORE, 139693598711808, 139693599240191, +SNULL, 139693601337343, 139693601345535, +STORE, 139693599240192, 139693601337343, +STORE, 139693601337344, 139693601345535, +ERASE, 139693601337344, 139693601345535, +STORE, 139693601337344, 139693601345535, +STORE, 139693596598272, 139693598711807, +SNULL, 139693596598272, 139693596610559, +STORE, 139693596610560, 139693598711807, +STORE, 139693596598272, 139693596610559, +SNULL, 139693598703615, 139693598711807, +STORE, 139693596610560, 139693598703615, +STORE, 139693598703616, 139693598711807, +ERASE, 139693598703616, 139693598711807, +STORE, 139693598703616, 139693598711807, +STORE, 139693594394624, 139693596598271, +SNULL, 139693594394624, 139693594497023, +STORE, 139693594497024, 139693596598271, +STORE, 139693594394624, 139693594497023, +SNULL, 139693596590079, 139693596598271, +STORE, 139693594497024, 139693596590079, +STORE, 139693596590080, 139693596598271, +ERASE, 139693596590080, 139693596598271, +STORE, 139693596590080, 139693596598271, +STORE, 139693612089344, 139693612105727, +STORE, 139693591232512, 139693594394623, +SNULL, 139693591232512, 139693592293375, +STORE, 139693592293376, 139693594394623, +STORE, 139693591232512, 139693592293375, +SNULL, 139693594386431, 139693594394623, +STORE, 139693592293376, 139693594386431, +STORE, 139693594386432, 139693594394623, +ERASE, 139693594386432, 139693594394623, +STORE, 139693594386432, 139693594394623, +STORE, 139693587435520, 139693591232511, +SNULL, 139693587435520, 139693589094399, +STORE, 139693589094400, 139693591232511, +STORE, 139693587435520, 139693589094399, +SNULL, 139693591191551, 139693591232511, +STORE, 139693589094400, 139693591191551, +STORE, 139693591191552, 139693591232511, +SNULL, 139693591191552, 139693591216127, +STORE, 139693591216128, 139693591232511, +STORE, 139693591191552, 139693591216127, +ERASE, 139693591191552, 139693591216127, +STORE, 139693591191552, 139693591216127, +ERASE, 139693591216128, 139693591232511, +STORE, 139693591216128, 139693591232511, +STORE, 139693612077056, 139693612105727, +SNULL, 139693591207935, 139693591216127, +STORE, 139693591191552, 139693591207935, +STORE, 139693591207936, 139693591216127, +SNULL, 139693594390527, 139693594394623, +STORE, 139693594386432, 139693594390527, +STORE, 139693594390528, 139693594394623, +SNULL, 139693596594175, 139693596598271, +STORE, 139693596590080, 139693596594175, +STORE, 139693596594176, 139693596598271, +SNULL, 139693598707711, 139693598711807, +STORE, 139693598703616, 139693598707711, +STORE, 139693598707712, 139693598711807, +SNULL, 139693601341439, 139693601345535, +STORE, 139693601337344, 139693601341439, +STORE, 139693601341440, 139693601345535, +SNULL, 139693603860479, 139693603864575, +STORE, 139693603852288, 139693603860479, +STORE, 139693603860480, 139693603864575, +SNULL, 139693606055935, 139693606060031, +STORE, 139693606051840, 139693606055935, +STORE, 139693606055936, 139693606060031, +SNULL, 139693609865215, 139693609893887, +STORE, 139693609861120, 139693609865215, +STORE, 139693609865216, 139693609893887, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 139693612138495, 139693612142591, +STORE, 139693612134400, 139693612138495, +STORE, 139693612138496, 139693612142591, +ERASE, 139693612105728, 139693612134399, +STORE, 39976960, 40112127, +STORE, 139693610393600, 139693612077055, +STORE, 139693612130304, 139693612134399, +STORE, 139693610258432, 139693610393599, +STORE, 39976960, 40255487, +STORE, 139693585338368, 139693587435519, +STORE, 139693612122112, 139693612134399, +STORE, 139693612113920, 139693612134399, +STORE, 139693612077056, 139693612113919, +STORE, 139693610242048, 139693610393599, +STORE, 39976960, 40390655, +STORE, 39976960, 40546303, +STORE, 139693610233856, 139693610393599, +STORE, 139693610225664, 139693610393599, +STORE, 39976960, 40714239, +STORE, 139693610209280, 139693610393599, +STORE, 39976960, 40861695, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431528759295, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729993904128, 140737488351231, +SNULL, 140729993912319, 140737488351231, +STORE, 140729993904128, 140729993912319, +STORE, 140729993773056, 140729993912319, +STORE, 93926271991808, 93926274215935, +SNULL, 93926272102399, 93926274215935, +STORE, 93926271991808, 93926272102399, +STORE, 93926272102400, 93926274215935, +ERASE, 93926272102400, 93926274215935, +STORE, 93926274195456, 93926274207743, +STORE, 93926274207744, 93926274215935, +STORE, 139962167296000, 139962169548799, +SNULL, 139962167439359, 139962169548799, +STORE, 139962167296000, 139962167439359, +STORE, 139962167439360, 139962169548799, +ERASE, 139962167439360, 139962169548799, +STORE, 139962169536512, 139962169544703, +STORE, 139962169544704, 139962169548799, +STORE, 140729995096064, 140729995100159, +STORE, 140729995083776, 140729995096063, +STORE, 139962169507840, 139962169536511, +STORE, 139962169499648, 139962169507839, +STORE, 139962163499008, 139962167295999, +SNULL, 139962163499008, 139962165157887, +STORE, 139962165157888, 139962167295999, +STORE, 139962163499008, 139962165157887, +SNULL, 139962167255039, 139962167295999, +STORE, 139962165157888, 139962167255039, +STORE, 139962167255040, 139962167295999, +SNULL, 139962167255040, 139962167279615, +STORE, 139962167279616, 139962167295999, +STORE, 139962167255040, 139962167279615, +ERASE, 139962167255040, 139962167279615, +STORE, 139962167255040, 139962167279615, +ERASE, 139962167279616, 139962167295999, +STORE, 139962167279616, 139962167295999, +SNULL, 139962167271423, 139962167279615, +STORE, 139962167255040, 139962167271423, +STORE, 139962167271424, 139962167279615, +SNULL, 93926274203647, 93926274207743, +STORE, 93926274195456, 93926274203647, +STORE, 93926274203648, 93926274207743, +SNULL, 139962169540607, 139962169544703, +STORE, 139962169536512, 139962169540607, +STORE, 139962169540608, 139962169544703, +ERASE, 139962169507840, 139962169536511, +STORE, 93926291120128, 93926291255295, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724960579584, 140737488351231, +SNULL, 140724960587775, 140737488351231, +STORE, 140724960579584, 140724960587775, +STORE, 140724960448512, 140724960587775, +STORE, 94246489489408, 94246491713535, +SNULL, 94246489599999, 94246491713535, +STORE, 94246489489408, 94246489599999, +STORE, 94246489600000, 94246491713535, +ERASE, 94246489600000, 94246491713535, +STORE, 94246491693056, 94246491705343, +STORE, 94246491705344, 94246491713535, +STORE, 140098174926848, 140098177179647, +SNULL, 140098175070207, 140098177179647, +STORE, 140098174926848, 140098175070207, +STORE, 140098175070208, 140098177179647, +ERASE, 140098175070208, 140098177179647, +STORE, 140098177167360, 140098177175551, +STORE, 140098177175552, 140098177179647, +STORE, 140724961439744, 140724961443839, +STORE, 140724961427456, 140724961439743, +STORE, 140098177138688, 140098177167359, +STORE, 140098177130496, 140098177138687, +STORE, 140098171129856, 140098174926847, +SNULL, 140098171129856, 140098172788735, +STORE, 140098172788736, 140098174926847, +STORE, 140098171129856, 140098172788735, +SNULL, 140098174885887, 140098174926847, +STORE, 140098172788736, 140098174885887, +STORE, 140098174885888, 140098174926847, +SNULL, 140098174885888, 140098174910463, +STORE, 140098174910464, 140098174926847, +STORE, 140098174885888, 140098174910463, +ERASE, 140098174885888, 140098174910463, +STORE, 140098174885888, 140098174910463, +ERASE, 140098174910464, 140098174926847, +STORE, 140098174910464, 140098174926847, +SNULL, 140098174902271, 140098174910463, +STORE, 140098174885888, 140098174902271, +STORE, 140098174902272, 140098174910463, +SNULL, 94246491701247, 94246491705343, +STORE, 94246491693056, 94246491701247, +STORE, 94246491701248, 94246491705343, +SNULL, 140098177171455, 140098177175551, +STORE, 140098177167360, 140098177171455, +STORE, 140098177171456, 140098177175551, +ERASE, 140098177138688, 140098177167359, +STORE, 94246516998144, 94246517133311, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730522918912, 140737488351231, +SNULL, 140730522927103, 140737488351231, +STORE, 140730522918912, 140730522927103, +STORE, 140730522787840, 140730522927103, +STORE, 94196043120640, 94196045344767, +SNULL, 94196043231231, 94196045344767, +STORE, 94196043120640, 94196043231231, +STORE, 94196043231232, 94196045344767, +ERASE, 94196043231232, 94196045344767, +STORE, 94196045324288, 94196045336575, +STORE, 94196045336576, 94196045344767, +STORE, 139815918940160, 139815921192959, +SNULL, 139815919083519, 139815921192959, +STORE, 139815918940160, 139815919083519, +STORE, 139815919083520, 139815921192959, +ERASE, 139815919083520, 139815921192959, +STORE, 139815921180672, 139815921188863, +STORE, 139815921188864, 139815921192959, +STORE, 140730523344896, 140730523348991, +STORE, 140730523332608, 140730523344895, +STORE, 139815921152000, 139815921180671, +STORE, 139815921143808, 139815921151999, +STORE, 139815915143168, 139815918940159, +SNULL, 139815915143168, 139815916802047, +STORE, 139815916802048, 139815918940159, +STORE, 139815915143168, 139815916802047, +SNULL, 139815918899199, 139815918940159, +STORE, 139815916802048, 139815918899199, +STORE, 139815918899200, 139815918940159, +SNULL, 139815918899200, 139815918923775, +STORE, 139815918923776, 139815918940159, +STORE, 139815918899200, 139815918923775, +ERASE, 139815918899200, 139815918923775, +STORE, 139815918899200, 139815918923775, +ERASE, 139815918923776, 139815918940159, +STORE, 139815918923776, 139815918940159, +SNULL, 139815918915583, 139815918923775, +STORE, 139815918899200, 139815918915583, +STORE, 139815918915584, 139815918923775, +SNULL, 94196045332479, 94196045336575, +STORE, 94196045324288, 94196045332479, +STORE, 94196045332480, 94196045336575, +SNULL, 139815921184767, 139815921188863, +STORE, 139815921180672, 139815921184767, +STORE, 139815921184768, 139815921188863, +ERASE, 139815921152000, 139815921180671, +STORE, 94196076183552, 94196076318719, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722460393472, 140737488351231, +SNULL, 140722460401663, 140737488351231, +STORE, 140722460393472, 140722460401663, +STORE, 140722460262400, 140722460401663, +STORE, 94569810399232, 94569812623359, +SNULL, 94569810509823, 94569812623359, +STORE, 94569810399232, 94569810509823, +STORE, 94569810509824, 94569812623359, +ERASE, 94569810509824, 94569812623359, +STORE, 94569812602880, 94569812615167, +STORE, 94569812615168, 94569812623359, +STORE, 139681565450240, 139681567703039, +SNULL, 139681565593599, 139681567703039, +STORE, 139681565450240, 139681565593599, +STORE, 139681565593600, 139681567703039, +ERASE, 139681565593600, 139681567703039, +STORE, 139681567690752, 139681567698943, +STORE, 139681567698944, 139681567703039, +STORE, 140722460569600, 140722460573695, +STORE, 140722460557312, 140722460569599, +STORE, 139681567662080, 139681567690751, +STORE, 139681567653888, 139681567662079, +STORE, 139681561653248, 139681565450239, +SNULL, 139681561653248, 139681563312127, +STORE, 139681563312128, 139681565450239, +STORE, 139681561653248, 139681563312127, +SNULL, 139681565409279, 139681565450239, +STORE, 139681563312128, 139681565409279, +STORE, 139681565409280, 139681565450239, +SNULL, 139681565409280, 139681565433855, +STORE, 139681565433856, 139681565450239, +STORE, 139681565409280, 139681565433855, +ERASE, 139681565409280, 139681565433855, +STORE, 139681565409280, 139681565433855, +ERASE, 139681565433856, 139681565450239, +STORE, 139681565433856, 139681565450239, +SNULL, 139681565425663, 139681565433855, +STORE, 139681565409280, 139681565425663, +STORE, 139681565425664, 139681565433855, +SNULL, 94569812611071, 94569812615167, +STORE, 94569812602880, 94569812611071, +STORE, 94569812611072, 94569812615167, +SNULL, 139681567694847, 139681567698943, +STORE, 139681567690752, 139681567694847, +STORE, 139681567694848, 139681567698943, +ERASE, 139681567662080, 139681567690751, +STORE, 94569818066944, 94569818202111, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431534280703, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725452365824, 140737488351231, +SNULL, 140725452374015, 140737488351231, +STORE, 140725452365824, 140725452374015, +STORE, 140725452234752, 140725452374015, +STORE, 94395067465728, 94395069689855, +SNULL, 94395067576319, 94395069689855, +STORE, 94395067465728, 94395067576319, +STORE, 94395067576320, 94395069689855, +ERASE, 94395067576320, 94395069689855, +STORE, 94395069669376, 94395069681663, +STORE, 94395069681664, 94395069689855, +STORE, 140269941211136, 140269943463935, +SNULL, 140269941354495, 140269943463935, +STORE, 140269941211136, 140269941354495, +STORE, 140269941354496, 140269943463935, +ERASE, 140269941354496, 140269943463935, +STORE, 140269943451648, 140269943459839, +STORE, 140269943459840, 140269943463935, +STORE, 140725452558336, 140725452562431, +STORE, 140725452546048, 140725452558335, +STORE, 140269943422976, 140269943451647, +STORE, 140269943414784, 140269943422975, +STORE, 140269937414144, 140269941211135, +SNULL, 140269937414144, 140269939073023, +STORE, 140269939073024, 140269941211135, +STORE, 140269937414144, 140269939073023, +SNULL, 140269941170175, 140269941211135, +STORE, 140269939073024, 140269941170175, +STORE, 140269941170176, 140269941211135, +SNULL, 140269941170176, 140269941194751, +STORE, 140269941194752, 140269941211135, +STORE, 140269941170176, 140269941194751, +ERASE, 140269941170176, 140269941194751, +STORE, 140269941170176, 140269941194751, +ERASE, 140269941194752, 140269941211135, +STORE, 140269941194752, 140269941211135, +SNULL, 140269941186559, 140269941194751, +STORE, 140269941170176, 140269941186559, +STORE, 140269941186560, 140269941194751, +SNULL, 94395069677567, 94395069681663, +STORE, 94395069669376, 94395069677567, +STORE, 94395069677568, 94395069681663, +SNULL, 140269943455743, 140269943459839, +STORE, 140269943451648, 140269943455743, +STORE, 140269943455744, 140269943459839, +ERASE, 140269943422976, 140269943451647, +STORE, 94395101691904, 94395101827071, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733860118528, 140737488351231, +SNULL, 140733860126719, 140737488351231, +STORE, 140733860118528, 140733860126719, +STORE, 140733859987456, 140733860126719, +STORE, 94484752990208, 94484755214335, +SNULL, 94484753100799, 94484755214335, +STORE, 94484752990208, 94484753100799, +STORE, 94484753100800, 94484755214335, +ERASE, 94484753100800, 94484755214335, +STORE, 94484755193856, 94484755206143, +STORE, 94484755206144, 94484755214335, +STORE, 139958922309632, 139958924562431, +SNULL, 139958922452991, 139958924562431, +STORE, 139958922309632, 139958922452991, +STORE, 139958922452992, 139958924562431, +ERASE, 139958922452992, 139958924562431, +STORE, 139958924550144, 139958924558335, +STORE, 139958924558336, 139958924562431, +STORE, 140733860253696, 140733860257791, +STORE, 140733860241408, 140733860253695, +STORE, 139958924521472, 139958924550143, +STORE, 139958924513280, 139958924521471, +STORE, 139958918512640, 139958922309631, +SNULL, 139958918512640, 139958920171519, +STORE, 139958920171520, 139958922309631, +STORE, 139958918512640, 139958920171519, +SNULL, 139958922268671, 139958922309631, +STORE, 139958920171520, 139958922268671, +STORE, 139958922268672, 139958922309631, +SNULL, 139958922268672, 139958922293247, +STORE, 139958922293248, 139958922309631, +STORE, 139958922268672, 139958922293247, +ERASE, 139958922268672, 139958922293247, +STORE, 139958922268672, 139958922293247, +ERASE, 139958922293248, 139958922309631, +STORE, 139958922293248, 139958922309631, +SNULL, 139958922285055, 139958922293247, +STORE, 139958922268672, 139958922285055, +STORE, 139958922285056, 139958922293247, +SNULL, 94484755202047, 94484755206143, +STORE, 94484755193856, 94484755202047, +STORE, 94484755202048, 94484755206143, +SNULL, 139958924554239, 139958924558335, +STORE, 139958924550144, 139958924554239, +STORE, 139958924554240, 139958924558335, +ERASE, 139958924521472, 139958924550143, +STORE, 94484777615360, 94484777750527, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731051036672, 140737488351231, +SNULL, 140731051044863, 140737488351231, +STORE, 140731051036672, 140731051044863, +STORE, 140731050905600, 140731051044863, +STORE, 93945822998528, 93945825222655, +SNULL, 93945823109119, 93945825222655, +STORE, 93945822998528, 93945823109119, +STORE, 93945823109120, 93945825222655, +ERASE, 93945823109120, 93945825222655, +STORE, 93945825202176, 93945825214463, +STORE, 93945825214464, 93945825222655, +STORE, 140153503997952, 140153506250751, +SNULL, 140153504141311, 140153506250751, +STORE, 140153503997952, 140153504141311, +STORE, 140153504141312, 140153506250751, +ERASE, 140153504141312, 140153506250751, +STORE, 140153506238464, 140153506246655, +STORE, 140153506246656, 140153506250751, +STORE, 140731051331584, 140731051335679, +STORE, 140731051319296, 140731051331583, +STORE, 140153506209792, 140153506238463, +STORE, 140153506201600, 140153506209791, +STORE, 140153500200960, 140153503997951, +SNULL, 140153500200960, 140153501859839, +STORE, 140153501859840, 140153503997951, +STORE, 140153500200960, 140153501859839, +SNULL, 140153503956991, 140153503997951, +STORE, 140153501859840, 140153503956991, +STORE, 140153503956992, 140153503997951, +SNULL, 140153503956992, 140153503981567, +STORE, 140153503981568, 140153503997951, +STORE, 140153503956992, 140153503981567, +ERASE, 140153503956992, 140153503981567, +STORE, 140153503956992, 140153503981567, +ERASE, 140153503981568, 140153503997951, +STORE, 140153503981568, 140153503997951, +SNULL, 140153503973375, 140153503981567, +STORE, 140153503956992, 140153503973375, +STORE, 140153503973376, 140153503981567, +SNULL, 93945825210367, 93945825214463, +STORE, 93945825202176, 93945825210367, +STORE, 93945825210368, 93945825214463, +SNULL, 140153506242559, 140153506246655, +STORE, 140153506238464, 140153506242559, +STORE, 140153506242560, 140153506246655, +ERASE, 140153506209792, 140153506238463, +STORE, 93945854537728, 93945854672895, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431537885183, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736025325568, 140737488351231, +SNULL, 140736025333759, 140737488351231, +STORE, 140736025325568, 140736025333759, +STORE, 140736025194496, 140736025333759, +STORE, 94809095172096, 94809097396223, +SNULL, 94809095282687, 94809097396223, +STORE, 94809095172096, 94809095282687, +STORE, 94809095282688, 94809097396223, +ERASE, 94809095282688, 94809097396223, +STORE, 94809097375744, 94809097388031, +STORE, 94809097388032, 94809097396223, +STORE, 140194992517120, 140194994769919, +SNULL, 140194992660479, 140194994769919, +STORE, 140194992517120, 140194992660479, +STORE, 140194992660480, 140194994769919, +ERASE, 140194992660480, 140194994769919, +STORE, 140194994757632, 140194994765823, +STORE, 140194994765824, 140194994769919, +STORE, 140736026173440, 140736026177535, +STORE, 140736026161152, 140736026173439, +STORE, 140194994728960, 140194994757631, +STORE, 140194994720768, 140194994728959, +STORE, 140194988720128, 140194992517119, +SNULL, 140194988720128, 140194990379007, +STORE, 140194990379008, 140194992517119, +STORE, 140194988720128, 140194990379007, +SNULL, 140194992476159, 140194992517119, +STORE, 140194990379008, 140194992476159, +STORE, 140194992476160, 140194992517119, +SNULL, 140194992476160, 140194992500735, +STORE, 140194992500736, 140194992517119, +STORE, 140194992476160, 140194992500735, +ERASE, 140194992476160, 140194992500735, +STORE, 140194992476160, 140194992500735, +ERASE, 140194992500736, 140194992517119, +STORE, 140194992500736, 140194992517119, +SNULL, 140194992492543, 140194992500735, +STORE, 140194992476160, 140194992492543, +STORE, 140194992492544, 140194992500735, +SNULL, 94809097383935, 94809097388031, +STORE, 94809097375744, 94809097383935, +STORE, 94809097383936, 94809097388031, +SNULL, 140194994761727, 140194994765823, +STORE, 140194994757632, 140194994761727, +STORE, 140194994761728, 140194994765823, +ERASE, 140194994728960, 140194994757631, +STORE, 94809124286464, 94809124421631, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726342660096, 140737488351231, +SNULL, 140726342668287, 140737488351231, +STORE, 140726342660096, 140726342668287, +STORE, 140726342529024, 140726342668287, +STORE, 94140331462656, 94140333686783, +SNULL, 94140331573247, 94140333686783, +STORE, 94140331462656, 94140331573247, +STORE, 94140331573248, 94140333686783, +ERASE, 94140331573248, 94140333686783, +STORE, 94140333666304, 94140333678591, +STORE, 94140333678592, 94140333686783, +STORE, 140714077208576, 140714079461375, +SNULL, 140714077351935, 140714079461375, +STORE, 140714077208576, 140714077351935, +STORE, 140714077351936, 140714079461375, +ERASE, 140714077351936, 140714079461375, +STORE, 140714079449088, 140714079457279, +STORE, 140714079457280, 140714079461375, +STORE, 140726343933952, 140726343938047, +STORE, 140726343921664, 140726343933951, +STORE, 140714079420416, 140714079449087, +STORE, 140714079412224, 140714079420415, +STORE, 140714073411584, 140714077208575, +SNULL, 140714073411584, 140714075070463, +STORE, 140714075070464, 140714077208575, +STORE, 140714073411584, 140714075070463, +SNULL, 140714077167615, 140714077208575, +STORE, 140714075070464, 140714077167615, +STORE, 140714077167616, 140714077208575, +SNULL, 140714077167616, 140714077192191, +STORE, 140714077192192, 140714077208575, +STORE, 140714077167616, 140714077192191, +ERASE, 140714077167616, 140714077192191, +STORE, 140714077167616, 140714077192191, +ERASE, 140714077192192, 140714077208575, +STORE, 140714077192192, 140714077208575, +SNULL, 140714077183999, 140714077192191, +STORE, 140714077167616, 140714077183999, +STORE, 140714077184000, 140714077192191, +SNULL, 94140333674495, 94140333678591, +STORE, 94140333666304, 94140333674495, +STORE, 94140333674496, 94140333678591, +SNULL, 140714079453183, 140714079457279, +STORE, 140714079449088, 140714079453183, +STORE, 140714079453184, 140714079457279, +ERASE, 140714079420416, 140714079449087, +STORE, 94140341432320, 94140341567487, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431539601407, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725843607552, 140737488351231, +SNULL, 140725843615743, 140737488351231, +STORE, 140725843607552, 140725843615743, +STORE, 140725843476480, 140725843615743, +STORE, 94889043505152, 94889045839871, +SNULL, 94889043718143, 94889045839871, +STORE, 94889043505152, 94889043718143, +STORE, 94889043718144, 94889045839871, +ERASE, 94889043718144, 94889045839871, +STORE, 94889045815296, 94889045827583, +STORE, 94889045827584, 94889045839871, +STORE, 140250965946368, 140250968199167, +SNULL, 140250966089727, 140250968199167, +STORE, 140250965946368, 140250966089727, +STORE, 140250966089728, 140250968199167, +ERASE, 140250966089728, 140250968199167, +STORE, 140250968186880, 140250968195071, +STORE, 140250968195072, 140250968199167, +STORE, 140725844500480, 140725844504575, +STORE, 140725844488192, 140725844500479, +STORE, 140250968158208, 140250968186879, +STORE, 140250968150016, 140250968158207, +STORE, 140250963832832, 140250965946367, +SNULL, 140250963832832, 140250963845119, +STORE, 140250963845120, 140250965946367, +STORE, 140250963832832, 140250963845119, +SNULL, 140250965938175, 140250965946367, +STORE, 140250963845120, 140250965938175, +STORE, 140250965938176, 140250965946367, +ERASE, 140250965938176, 140250965946367, +STORE, 140250965938176, 140250965946367, +STORE, 140250960035840, 140250963832831, +SNULL, 140250960035840, 140250961694719, +STORE, 140250961694720, 140250963832831, +STORE, 140250960035840, 140250961694719, +SNULL, 140250963791871, 140250963832831, +STORE, 140250961694720, 140250963791871, +STORE, 140250963791872, 140250963832831, +SNULL, 140250963791872, 140250963816447, +STORE, 140250963816448, 140250963832831, +STORE, 140250963791872, 140250963816447, +ERASE, 140250963791872, 140250963816447, +STORE, 140250963791872, 140250963816447, +ERASE, 140250963816448, 140250963832831, +STORE, 140250963816448, 140250963832831, +STORE, 140250968141824, 140250968158207, +SNULL, 140250963808255, 140250963816447, +STORE, 140250963791872, 140250963808255, +STORE, 140250963808256, 140250963816447, +SNULL, 140250965942271, 140250965946367, +STORE, 140250965938176, 140250965942271, +STORE, 140250965942272, 140250965946367, +SNULL, 94889045819391, 94889045827583, +STORE, 94889045815296, 94889045819391, +STORE, 94889045819392, 94889045827583, +SNULL, 140250968190975, 140250968195071, +STORE, 140250968186880, 140250968190975, +STORE, 140250968190976, 140250968195071, +ERASE, 140250968158208, 140250968186879, +STORE, 94889052213248, 94889052348415, +STORE, 140250966458368, 140250968141823, +STORE, 94889052213248, 94889052483583, +STORE, 94889052213248, 94889052618751, +STORE, 94170851819520, 94170852032511, +STORE, 94170854129664, 94170854133759, +STORE, 94170854133760, 94170854141951, +STORE, 94170854141952, 94170854154239, +STORE, 94170866515968, 94170867740671, +STORE, 140062030422016, 140062032080895, +STORE, 140062032080896, 140062034178047, +STORE, 140062034178048, 140062034194431, +STORE, 140062034194432, 140062034202623, +STORE, 140062034202624, 140062034219007, +STORE, 140062034219008, 140062034231295, +STORE, 140062034231296, 140062036324351, +STORE, 140062036324352, 140062036328447, +STORE, 140062036328448, 140062036332543, +STORE, 140062036332544, 140062036475903, +STORE, 140062036844544, 140062038527999, +STORE, 140062038528000, 140062038544383, +STORE, 140062038573056, 140062038577151, +STORE, 140062038577152, 140062038581247, +STORE, 140062038581248, 140062038585343, +STORE, 140736210550784, 140736210690047, +STORE, 140736210759680, 140736210771967, +STORE, 140736210771968, 140736210776063, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724272365568, 140737488351231, +SNULL, 140724272373759, 140737488351231, +STORE, 140724272365568, 140724272373759, +STORE, 140724272234496, 140724272373759, +STORE, 94607711965184, 94607714189311, +SNULL, 94607712075775, 94607714189311, +STORE, 94607711965184, 94607712075775, +STORE, 94607712075776, 94607714189311, +ERASE, 94607712075776, 94607714189311, +STORE, 94607714168832, 94607714181119, +STORE, 94607714181120, 94607714189311, +STORE, 140054949253120, 140054951505919, +SNULL, 140054949396479, 140054951505919, +STORE, 140054949253120, 140054949396479, +STORE, 140054949396480, 140054951505919, +ERASE, 140054949396480, 140054951505919, +STORE, 140054951493632, 140054951501823, +STORE, 140054951501824, 140054951505919, +STORE, 140724272992256, 140724272996351, +STORE, 140724272979968, 140724272992255, +STORE, 140054951464960, 140054951493631, +STORE, 140054951456768, 140054951464959, +STORE, 140054945456128, 140054949253119, +SNULL, 140054945456128, 140054947115007, +STORE, 140054947115008, 140054949253119, +STORE, 140054945456128, 140054947115007, +SNULL, 140054949212159, 140054949253119, +STORE, 140054947115008, 140054949212159, +STORE, 140054949212160, 140054949253119, +SNULL, 140054949212160, 140054949236735, +STORE, 140054949236736, 140054949253119, +STORE, 140054949212160, 140054949236735, +ERASE, 140054949212160, 140054949236735, +STORE, 140054949212160, 140054949236735, +ERASE, 140054949236736, 140054949253119, +STORE, 140054949236736, 140054949253119, +SNULL, 140054949228543, 140054949236735, +STORE, 140054949212160, 140054949228543, +STORE, 140054949228544, 140054949236735, +SNULL, 94607714177023, 94607714181119, +STORE, 94607714168832, 94607714177023, +STORE, 94607714177024, 94607714181119, +SNULL, 140054951497727, 140054951501823, +STORE, 140054951493632, 140054951497727, +STORE, 140054951497728, 140054951501823, +ERASE, 140054951464960, 140054951493631, +STORE, 94607733374976, 94607733510143, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733586923520, 140737488351231, +SNULL, 140733586931711, 140737488351231, +STORE, 140733586923520, 140733586931711, +STORE, 140733586792448, 140733586931711, +STORE, 93901634904064, 93901637128191, +SNULL, 93901635014655, 93901637128191, +STORE, 93901634904064, 93901635014655, +STORE, 93901635014656, 93901637128191, +ERASE, 93901635014656, 93901637128191, +STORE, 93901637107712, 93901637119999, +STORE, 93901637120000, 93901637128191, +STORE, 140086104784896, 140086107037695, +SNULL, 140086104928255, 140086107037695, +STORE, 140086104784896, 140086104928255, +STORE, 140086104928256, 140086107037695, +ERASE, 140086104928256, 140086107037695, +STORE, 140086107025408, 140086107033599, +STORE, 140086107033600, 140086107037695, +STORE, 140733587263488, 140733587267583, +STORE, 140733587251200, 140733587263487, +STORE, 140086106996736, 140086107025407, +STORE, 140086106988544, 140086106996735, +STORE, 140086100987904, 140086104784895, +SNULL, 140086100987904, 140086102646783, +STORE, 140086102646784, 140086104784895, +STORE, 140086100987904, 140086102646783, +SNULL, 140086104743935, 140086104784895, +STORE, 140086102646784, 140086104743935, +STORE, 140086104743936, 140086104784895, +SNULL, 140086104743936, 140086104768511, +STORE, 140086104768512, 140086104784895, +STORE, 140086104743936, 140086104768511, +ERASE, 140086104743936, 140086104768511, +STORE, 140086104743936, 140086104768511, +ERASE, 140086104768512, 140086104784895, +STORE, 140086104768512, 140086104784895, +SNULL, 140086104760319, 140086104768511, +STORE, 140086104743936, 140086104760319, +STORE, 140086104760320, 140086104768511, +SNULL, 93901637115903, 93901637119999, +STORE, 93901637107712, 93901637115903, +STORE, 93901637115904, 93901637119999, +SNULL, 140086107029503, 140086107033599, +STORE, 140086107025408, 140086107029503, +STORE, 140086107029504, 140086107033599, +ERASE, 140086106996736, 140086107025407, +STORE, 93901662715904, 93901662851071, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723365613568, 140737488351231, +SNULL, 140723365621759, 140737488351231, +STORE, 140723365613568, 140723365621759, +STORE, 140723365482496, 140723365621759, +STORE, 94759193546752, 94759195770879, +SNULL, 94759193657343, 94759195770879, +STORE, 94759193546752, 94759193657343, +STORE, 94759193657344, 94759195770879, +ERASE, 94759193657344, 94759195770879, +STORE, 94759195750400, 94759195762687, +STORE, 94759195762688, 94759195770879, +STORE, 140607636246528, 140607638499327, +SNULL, 140607636389887, 140607638499327, +STORE, 140607636246528, 140607636389887, +STORE, 140607636389888, 140607638499327, +ERASE, 140607636389888, 140607638499327, +STORE, 140607638487040, 140607638495231, +STORE, 140607638495232, 140607638499327, +STORE, 140723365900288, 140723365904383, +STORE, 140723365888000, 140723365900287, +STORE, 140607638458368, 140607638487039, +STORE, 140607638450176, 140607638458367, +STORE, 140607632449536, 140607636246527, +SNULL, 140607632449536, 140607634108415, +STORE, 140607634108416, 140607636246527, +STORE, 140607632449536, 140607634108415, +SNULL, 140607636205567, 140607636246527, +STORE, 140607634108416, 140607636205567, +STORE, 140607636205568, 140607636246527, +SNULL, 140607636205568, 140607636230143, +STORE, 140607636230144, 140607636246527, +STORE, 140607636205568, 140607636230143, +ERASE, 140607636205568, 140607636230143, +STORE, 140607636205568, 140607636230143, +ERASE, 140607636230144, 140607636246527, +STORE, 140607636230144, 140607636246527, +SNULL, 140607636221951, 140607636230143, +STORE, 140607636205568, 140607636221951, +STORE, 140607636221952, 140607636230143, +SNULL, 94759195758591, 94759195762687, +STORE, 94759195750400, 94759195758591, +STORE, 94759195758592, 94759195762687, +SNULL, 140607638491135, 140607638495231, +STORE, 140607638487040, 140607638491135, +STORE, 140607638491136, 140607638495231, +ERASE, 140607638458368, 140607638487039, +STORE, 94759204995072, 94759205130239, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732503789568, 140737488351231, +SNULL, 140732503797759, 140737488351231, +STORE, 140732503789568, 140732503797759, +STORE, 140732503658496, 140732503797759, +STORE, 94077792956416, 94077795180543, +SNULL, 94077793067007, 94077795180543, +STORE, 94077792956416, 94077793067007, +STORE, 94077793067008, 94077795180543, +ERASE, 94077793067008, 94077795180543, +STORE, 94077795160064, 94077795172351, +STORE, 94077795172352, 94077795180543, +STORE, 140359874252800, 140359876505599, +SNULL, 140359874396159, 140359876505599, +STORE, 140359874252800, 140359874396159, +STORE, 140359874396160, 140359876505599, +ERASE, 140359874396160, 140359876505599, +STORE, 140359876493312, 140359876501503, +STORE, 140359876501504, 140359876505599, +STORE, 140732504465408, 140732504469503, +STORE, 140732504453120, 140732504465407, +STORE, 140359876464640, 140359876493311, +STORE, 140359876456448, 140359876464639, +STORE, 140359870455808, 140359874252799, +SNULL, 140359870455808, 140359872114687, +STORE, 140359872114688, 140359874252799, +STORE, 140359870455808, 140359872114687, +SNULL, 140359874211839, 140359874252799, +STORE, 140359872114688, 140359874211839, +STORE, 140359874211840, 140359874252799, +SNULL, 140359874211840, 140359874236415, +STORE, 140359874236416, 140359874252799, +STORE, 140359874211840, 140359874236415, +ERASE, 140359874211840, 140359874236415, +STORE, 140359874211840, 140359874236415, +ERASE, 140359874236416, 140359874252799, +STORE, 140359874236416, 140359874252799, +SNULL, 140359874228223, 140359874236415, +STORE, 140359874211840, 140359874228223, +STORE, 140359874228224, 140359874236415, +SNULL, 94077795168255, 94077795172351, +STORE, 94077795160064, 94077795168255, +STORE, 94077795168256, 94077795172351, +SNULL, 140359876497407, 140359876501503, +STORE, 140359876493312, 140359876497407, +STORE, 140359876497408, 140359876501503, +ERASE, 140359876464640, 140359876493311, +STORE, 94077808717824, 94077808852991, +STORE, 94549486252032, 94549486465023, +STORE, 94549488562176, 94549488566271, +STORE, 94549488566272, 94549488574463, +STORE, 94549488574464, 94549488586751, +STORE, 94549503492096, 94549506121727, +STORE, 140085800894464, 140085802553343, +STORE, 140085802553344, 140085804650495, +STORE, 140085804650496, 140085804666879, +STORE, 140085804666880, 140085804675071, +STORE, 140085804675072, 140085804691455, +STORE, 140085804691456, 140085804703743, +STORE, 140085804703744, 140085806796799, +STORE, 140085806796800, 140085806800895, +STORE, 140085806800896, 140085806804991, +STORE, 140085806804992, 140085806948351, +STORE, 140085807316992, 140085809000447, +STORE, 140085809000448, 140085809016831, +STORE, 140085809045504, 140085809049599, +STORE, 140085809049600, 140085809053695, +STORE, 140085809053696, 140085809057791, +STORE, 140731810545664, 140731810684927, +STORE, 140731810967552, 140731810979839, +STORE, 140731810979840, 140731810983935, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724752330752, 140737488351231, +SNULL, 140724752338943, 140737488351231, +STORE, 140724752330752, 140724752338943, +STORE, 140724752199680, 140724752338943, +STORE, 94656357539840, 94656359874559, +SNULL, 94656357752831, 94656359874559, +STORE, 94656357539840, 94656357752831, +STORE, 94656357752832, 94656359874559, +ERASE, 94656357752832, 94656359874559, +STORE, 94656359849984, 94656359862271, +STORE, 94656359862272, 94656359874559, +STORE, 139632585203712, 139632587456511, +SNULL, 139632585347071, 139632587456511, +STORE, 139632585203712, 139632585347071, +STORE, 139632585347072, 139632587456511, +ERASE, 139632585347072, 139632587456511, +STORE, 139632587444224, 139632587452415, +STORE, 139632587452416, 139632587456511, +STORE, 139632587440128, 139632587444223, +STORE, 139632587427840, 139632587440127, +STORE, 139632587399168, 139632587427839, +STORE, 139632587390976, 139632587399167, +STORE, 139632583090176, 139632585203711, +SNULL, 139632583090176, 139632583102463, +STORE, 139632583102464, 139632585203711, +STORE, 139632583090176, 139632583102463, +SNULL, 139632585195519, 139632585203711, +STORE, 139632583102464, 139632585195519, +STORE, 139632585195520, 139632585203711, +ERASE, 139632585195520, 139632585203711, +STORE, 139632585195520, 139632585203711, +STORE, 139632579293184, 139632583090175, +SNULL, 139632579293184, 139632580952063, +STORE, 139632580952064, 139632583090175, +STORE, 139632579293184, 139632580952063, +SNULL, 139632583049215, 139632583090175, +STORE, 139632580952064, 139632583049215, +STORE, 139632583049216, 139632583090175, +SNULL, 139632583049216, 139632583073791, +STORE, 139632583073792, 139632583090175, +STORE, 139632583049216, 139632583073791, +ERASE, 139632583049216, 139632583073791, +STORE, 139632583049216, 139632583073791, +ERASE, 139632583073792, 139632583090175, +STORE, 139632583073792, 139632583090175, +STORE, 139632587382784, 139632587399167, +SNULL, 139632583065599, 139632583073791, +STORE, 139632583049216, 139632583065599, +STORE, 139632583065600, 139632583073791, +SNULL, 139632585199615, 139632585203711, +STORE, 139632585195520, 139632585199615, +STORE, 139632585199616, 139632585203711, +SNULL, 94656359854079, 94656359862271, +STORE, 94656359849984, 94656359854079, +STORE, 94656359854080, 94656359862271, +SNULL, 139632587448319, 139632587452415, +STORE, 139632587444224, 139632587448319, +STORE, 139632587448320, 139632587452415, +ERASE, 139632587399168, 139632587427839, +STORE, 94656378912768, 94656379047935, +STORE, 139632585699328, 139632587382783, +STORE, 94656378912768, 94656379183103, +STORE, 94656378912768, 94656379318271, +STORE, 94656378912768, 94656379494399, +SNULL, 94656379469823, 94656379494399, +STORE, 94656378912768, 94656379469823, +STORE, 94656379469824, 94656379494399, +ERASE, 94656379469824, 94656379494399, +STORE, 94656378912768, 94656379621375, +STORE, 94656378912768, 94656379756543, +STORE, 94656378912768, 94656379912191, +STORE, 94656378912768, 94656380055551, +STORE, 94656378912768, 94656380190719, +STORE, 94656378912768, 94656380338175, +SNULL, 94656380313599, 94656380338175, +STORE, 94656378912768, 94656380313599, +STORE, 94656380313600, 94656380338175, +ERASE, 94656380313600, 94656380338175, +STORE, 94656378912768, 94656380448767, +SNULL, 94656380432383, 94656380448767, +STORE, 94656378912768, 94656380432383, +STORE, 94656380432384, 94656380448767, +ERASE, 94656380432384, 94656380448767, +STORE, 94656378912768, 94656380567551, +STORE, 94656378912768, 94656380719103, +STORE, 94656378912768, 94656380858367, +STORE, 94656378912768, 94656380997631, +STORE, 94656378912768, 94656381132799, +SNULL, 94656381124607, 94656381132799, +STORE, 94656378912768, 94656381124607, +STORE, 94656381124608, 94656381132799, +ERASE, 94656381124608, 94656381132799, +STORE, 94656378912768, 94656381276159, +STORE, 94656378912768, 94656381427711, +STORE, 94604087611392, 94604087824383, +STORE, 94604089921536, 94604089925631, +STORE, 94604089925632, 94604089933823, +STORE, 94604089933824, 94604089946111, +STORE, 94604105125888, 94604106424319, +STORE, 140454937694208, 140454939353087, +STORE, 140454939353088, 140454941450239, +STORE, 140454941450240, 140454941466623, +STORE, 140454941466624, 140454941474815, +STORE, 140454941474816, 140454941491199, +STORE, 140454941491200, 140454941503487, +STORE, 140454941503488, 140454943596543, +STORE, 140454943596544, 140454943600639, +STORE, 140454943600640, 140454943604735, +STORE, 140454943604736, 140454943748095, +STORE, 140454944116736, 140454945800191, +STORE, 140454945800192, 140454945816575, +STORE, 140454945845248, 140454945849343, +STORE, 140454945849344, 140454945853439, +STORE, 140454945853440, 140454945857535, +STORE, 140728438214656, 140728438353919, +STORE, 140728439095296, 140728439107583, +STORE, 140728439107584, 140728439111679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727821099008, 140737488351231, +SNULL, 140727821107199, 140737488351231, +STORE, 140727821099008, 140727821107199, +STORE, 140727820967936, 140727821107199, +STORE, 94088457240576, 94088459575295, +SNULL, 94088457453567, 94088459575295, +STORE, 94088457240576, 94088457453567, +STORE, 94088457453568, 94088459575295, +ERASE, 94088457453568, 94088459575295, +STORE, 94088459550720, 94088459563007, +STORE, 94088459563008, 94088459575295, +STORE, 140234378989568, 140234381242367, +SNULL, 140234379132927, 140234381242367, +STORE, 140234378989568, 140234379132927, +STORE, 140234379132928, 140234381242367, +ERASE, 140234379132928, 140234381242367, +STORE, 140234381230080, 140234381238271, +STORE, 140234381238272, 140234381242367, +STORE, 140727822077952, 140727822082047, +STORE, 140727822065664, 140727822077951, +STORE, 140234381201408, 140234381230079, +STORE, 140234381193216, 140234381201407, +STORE, 140234376876032, 140234378989567, +SNULL, 140234376876032, 140234376888319, +STORE, 140234376888320, 140234378989567, +STORE, 140234376876032, 140234376888319, +SNULL, 140234378981375, 140234378989567, +STORE, 140234376888320, 140234378981375, +STORE, 140234378981376, 140234378989567, +ERASE, 140234378981376, 140234378989567, +STORE, 140234378981376, 140234378989567, +STORE, 140234373079040, 140234376876031, +SNULL, 140234373079040, 140234374737919, +STORE, 140234374737920, 140234376876031, +STORE, 140234373079040, 140234374737919, +SNULL, 140234376835071, 140234376876031, +STORE, 140234374737920, 140234376835071, +STORE, 140234376835072, 140234376876031, +SNULL, 140234376835072, 140234376859647, +STORE, 140234376859648, 140234376876031, +STORE, 140234376835072, 140234376859647, +ERASE, 140234376835072, 140234376859647, +STORE, 140234376835072, 140234376859647, +ERASE, 140234376859648, 140234376876031, +STORE, 140234376859648, 140234376876031, +STORE, 140234381185024, 140234381201407, +SNULL, 140234376851455, 140234376859647, +STORE, 140234376835072, 140234376851455, +STORE, 140234376851456, 140234376859647, +SNULL, 140234378985471, 140234378989567, +STORE, 140234378981376, 140234378985471, +STORE, 140234378985472, 140234378989567, +SNULL, 94088459554815, 94088459563007, +STORE, 94088459550720, 94088459554815, +STORE, 94088459554816, 94088459563007, +SNULL, 140234381234175, 140234381238271, +STORE, 140234381230080, 140234381234175, +STORE, 140234381234176, 140234381238271, +ERASE, 140234381201408, 140234381230079, +STORE, 94088468852736, 94088468987903, +STORE, 140234379501568, 140234381185023, +STORE, 94088468852736, 94088469123071, +STORE, 94088468852736, 94088469258239, +STORE, 94110050402304, 94110050615295, +STORE, 94110052712448, 94110052716543, +STORE, 94110052716544, 94110052724735, +STORE, 94110052724736, 94110052737023, +STORE, 94110061875200, 94110062415871, +STORE, 140139439357952, 140139441016831, +STORE, 140139441016832, 140139443113983, +STORE, 140139443113984, 140139443130367, +STORE, 140139443130368, 140139443138559, +STORE, 140139443138560, 140139443154943, +STORE, 140139443154944, 140139443167231, +STORE, 140139443167232, 140139445260287, +STORE, 140139445260288, 140139445264383, +STORE, 140139445264384, 140139445268479, +STORE, 140139445268480, 140139445411839, +STORE, 140139445780480, 140139447463935, +STORE, 140139447463936, 140139447480319, +STORE, 140139447508992, 140139447513087, +STORE, 140139447513088, 140139447517183, +STORE, 140139447517184, 140139447521279, +STORE, 140731901427712, 140731901566975, +STORE, 140731902259200, 140731902271487, +STORE, 140731902271488, 140731902275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727282622464, 140737488351231, +SNULL, 140727282630655, 140737488351231, +STORE, 140727282622464, 140727282630655, +STORE, 140727282491392, 140727282630655, +STORE, 94266649866240, 94266652200959, +SNULL, 94266650079231, 94266652200959, +STORE, 94266649866240, 94266650079231, +STORE, 94266650079232, 94266652200959, +ERASE, 94266650079232, 94266652200959, +STORE, 94266652176384, 94266652188671, +STORE, 94266652188672, 94266652200959, +STORE, 139888497991680, 139888500244479, +SNULL, 139888498135039, 139888500244479, +STORE, 139888497991680, 139888498135039, +STORE, 139888498135040, 139888500244479, +ERASE, 139888498135040, 139888500244479, +STORE, 139888500232192, 139888500240383, +STORE, 139888500240384, 139888500244479, +STORE, 140727283113984, 140727283118079, +STORE, 140727283101696, 140727283113983, +STORE, 139888500203520, 139888500232191, +STORE, 139888500195328, 139888500203519, +STORE, 139888495878144, 139888497991679, +SNULL, 139888495878144, 139888495890431, +STORE, 139888495890432, 139888497991679, +STORE, 139888495878144, 139888495890431, +SNULL, 139888497983487, 139888497991679, +STORE, 139888495890432, 139888497983487, +STORE, 139888497983488, 139888497991679, +ERASE, 139888497983488, 139888497991679, +STORE, 139888497983488, 139888497991679, +STORE, 139888492081152, 139888495878143, +SNULL, 139888492081152, 139888493740031, +STORE, 139888493740032, 139888495878143, +STORE, 139888492081152, 139888493740031, +SNULL, 139888495837183, 139888495878143, +STORE, 139888493740032, 139888495837183, +STORE, 139888495837184, 139888495878143, +SNULL, 139888495837184, 139888495861759, +STORE, 139888495861760, 139888495878143, +STORE, 139888495837184, 139888495861759, +ERASE, 139888495837184, 139888495861759, +STORE, 139888495837184, 139888495861759, +ERASE, 139888495861760, 139888495878143, +STORE, 139888495861760, 139888495878143, +STORE, 139888500187136, 139888500203519, +SNULL, 139888495853567, 139888495861759, +STORE, 139888495837184, 139888495853567, +STORE, 139888495853568, 139888495861759, +SNULL, 139888497987583, 139888497991679, +STORE, 139888497983488, 139888497987583, +STORE, 139888497987584, 139888497991679, +SNULL, 94266652180479, 94266652188671, +STORE, 94266652176384, 94266652180479, +STORE, 94266652180480, 94266652188671, +SNULL, 139888500236287, 139888500240383, +STORE, 139888500232192, 139888500236287, +STORE, 139888500236288, 139888500240383, +ERASE, 139888500203520, 139888500232191, +STORE, 94266678542336, 94266678677503, +STORE, 139888498503680, 139888500187135, +STORE, 94266678542336, 94266678812671, +STORE, 94266678542336, 94266678947839, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722507702272, 140737488351231, +SNULL, 140722507710463, 140737488351231, +STORE, 140722507702272, 140722507710463, +STORE, 140722507571200, 140722507710463, +STORE, 94313981394944, 94313983729663, +SNULL, 94313981607935, 94313983729663, +STORE, 94313981394944, 94313981607935, +STORE, 94313981607936, 94313983729663, +ERASE, 94313981607936, 94313983729663, +STORE, 94313983705088, 94313983717375, +STORE, 94313983717376, 94313983729663, +STORE, 140456286076928, 140456288329727, +SNULL, 140456286220287, 140456288329727, +STORE, 140456286076928, 140456286220287, +STORE, 140456286220288, 140456288329727, +ERASE, 140456286220288, 140456288329727, +STORE, 140456288317440, 140456288325631, +STORE, 140456288325632, 140456288329727, +STORE, 140722507997184, 140722508001279, +STORE, 140722507984896, 140722507997183, +STORE, 140456288288768, 140456288317439, +STORE, 140456288280576, 140456288288767, +STORE, 140456283963392, 140456286076927, +SNULL, 140456283963392, 140456283975679, +STORE, 140456283975680, 140456286076927, +STORE, 140456283963392, 140456283975679, +SNULL, 140456286068735, 140456286076927, +STORE, 140456283975680, 140456286068735, +STORE, 140456286068736, 140456286076927, +ERASE, 140456286068736, 140456286076927, +STORE, 140456286068736, 140456286076927, +STORE, 140456280166400, 140456283963391, +SNULL, 140456280166400, 140456281825279, +STORE, 140456281825280, 140456283963391, +STORE, 140456280166400, 140456281825279, +SNULL, 140456283922431, 140456283963391, +STORE, 140456281825280, 140456283922431, +STORE, 140456283922432, 140456283963391, +SNULL, 140456283922432, 140456283947007, +STORE, 140456283947008, 140456283963391, +STORE, 140456283922432, 140456283947007, +ERASE, 140456283922432, 140456283947007, +STORE, 140456283922432, 140456283947007, +ERASE, 140456283947008, 140456283963391, +STORE, 140456283947008, 140456283963391, +STORE, 140456288272384, 140456288288767, +SNULL, 140456283938815, 140456283947007, +STORE, 140456283922432, 140456283938815, +STORE, 140456283938816, 140456283947007, +SNULL, 140456286072831, 140456286076927, +STORE, 140456286068736, 140456286072831, +STORE, 140456286072832, 140456286076927, +SNULL, 94313983709183, 94313983717375, +STORE, 94313983705088, 94313983709183, +STORE, 94313983709184, 94313983717375, +SNULL, 140456288321535, 140456288325631, +STORE, 140456288317440, 140456288321535, +STORE, 140456288321536, 140456288325631, +ERASE, 140456288288768, 140456288317439, +STORE, 94314006716416, 94314006851583, +STORE, 140456286588928, 140456288272383, +STORE, 94314006716416, 94314006986751, +STORE, 94314006716416, 94314007121919, +STORE, 93948644454400, 93948644667391, +STORE, 93948646764544, 93948646768639, +STORE, 93948646768640, 93948646776831, +STORE, 93948646776832, 93948646789119, +STORE, 93948664999936, 93948667142143, +STORE, 140187350659072, 140187352317951, +STORE, 140187352317952, 140187354415103, +STORE, 140187354415104, 140187354431487, +STORE, 140187354431488, 140187354439679, +STORE, 140187354439680, 140187354456063, +STORE, 140187354456064, 140187354468351, +STORE, 140187354468352, 140187356561407, +STORE, 140187356561408, 140187356565503, +STORE, 140187356565504, 140187356569599, +STORE, 140187356569600, 140187356712959, +STORE, 140187357081600, 140187358765055, +STORE, 140187358765056, 140187358781439, +STORE, 140187358810112, 140187358814207, +STORE, 140187358814208, 140187358818303, +STORE, 140187358818304, 140187358822399, +STORE, 140730484518912, 140730484658175, +STORE, 140730485690368, 140730485702655, +STORE, 140730485702656, 140730485706751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721211551744, 140737488351231, +SNULL, 140721211559935, 140737488351231, +STORE, 140721211551744, 140721211559935, +STORE, 140721211420672, 140721211559935, +STORE, 94105221423104, 94105223757823, +SNULL, 94105221636095, 94105223757823, +STORE, 94105221423104, 94105221636095, +STORE, 94105221636096, 94105223757823, +ERASE, 94105221636096, 94105223757823, +STORE, 94105223733248, 94105223745535, +STORE, 94105223745536, 94105223757823, +STORE, 140474453676032, 140474455928831, +SNULL, 140474453819391, 140474455928831, +STORE, 140474453676032, 140474453819391, +STORE, 140474453819392, 140474455928831, +ERASE, 140474453819392, 140474455928831, +STORE, 140474455916544, 140474455924735, +STORE, 140474455924736, 140474455928831, +STORE, 140721211703296, 140721211707391, +STORE, 140721211691008, 140721211703295, +STORE, 140474455887872, 140474455916543, +STORE, 140474455879680, 140474455887871, +STORE, 140474451562496, 140474453676031, +SNULL, 140474451562496, 140474451574783, +STORE, 140474451574784, 140474453676031, +STORE, 140474451562496, 140474451574783, +SNULL, 140474453667839, 140474453676031, +STORE, 140474451574784, 140474453667839, +STORE, 140474453667840, 140474453676031, +ERASE, 140474453667840, 140474453676031, +STORE, 140474453667840, 140474453676031, +STORE, 140474447765504, 140474451562495, +SNULL, 140474447765504, 140474449424383, +STORE, 140474449424384, 140474451562495, +STORE, 140474447765504, 140474449424383, +SNULL, 140474451521535, 140474451562495, +STORE, 140474449424384, 140474451521535, +STORE, 140474451521536, 140474451562495, +SNULL, 140474451521536, 140474451546111, +STORE, 140474451546112, 140474451562495, +STORE, 140474451521536, 140474451546111, +ERASE, 140474451521536, 140474451546111, +STORE, 140474451521536, 140474451546111, +ERASE, 140474451546112, 140474451562495, +STORE, 140474451546112, 140474451562495, +STORE, 140474455871488, 140474455887871, +SNULL, 140474451537919, 140474451546111, +STORE, 140474451521536, 140474451537919, +STORE, 140474451537920, 140474451546111, +SNULL, 140474453671935, 140474453676031, +STORE, 140474453667840, 140474453671935, +STORE, 140474453671936, 140474453676031, +SNULL, 94105223737343, 94105223745535, +STORE, 94105223733248, 94105223737343, +STORE, 94105223737344, 94105223745535, +SNULL, 140474455920639, 140474455924735, +STORE, 140474455916544, 140474455920639, +STORE, 140474455920640, 140474455924735, +ERASE, 140474455887872, 140474455916543, +STORE, 94105238712320, 94105238847487, +STORE, 140474454188032, 140474455871487, +STORE, 94105238712320, 94105238982655, +STORE, 94105238712320, 94105239117823, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732356354048, 140737488351231, +SNULL, 140732356362239, 140737488351231, +STORE, 140732356354048, 140732356362239, +STORE, 140732356222976, 140732356362239, +STORE, 94461165989888, 94461168324607, +SNULL, 94461166202879, 94461168324607, +STORE, 94461165989888, 94461166202879, +STORE, 94461166202880, 94461168324607, +ERASE, 94461166202880, 94461168324607, +STORE, 94461168300032, 94461168312319, +STORE, 94461168312320, 94461168324607, +STORE, 140317255110656, 140317257363455, +SNULL, 140317255254015, 140317257363455, +STORE, 140317255110656, 140317255254015, +STORE, 140317255254016, 140317257363455, +ERASE, 140317255254016, 140317257363455, +STORE, 140317257351168, 140317257359359, +STORE, 140317257359360, 140317257363455, +STORE, 140732356583424, 140732356587519, +STORE, 140732356571136, 140732356583423, +STORE, 140317257322496, 140317257351167, +STORE, 140317257314304, 140317257322495, +STORE, 140317252997120, 140317255110655, +SNULL, 140317252997120, 140317253009407, +STORE, 140317253009408, 140317255110655, +STORE, 140317252997120, 140317253009407, +SNULL, 140317255102463, 140317255110655, +STORE, 140317253009408, 140317255102463, +STORE, 140317255102464, 140317255110655, +ERASE, 140317255102464, 140317255110655, +STORE, 140317255102464, 140317255110655, +STORE, 140317249200128, 140317252997119, +SNULL, 140317249200128, 140317250859007, +STORE, 140317250859008, 140317252997119, +STORE, 140317249200128, 140317250859007, +SNULL, 140317252956159, 140317252997119, +STORE, 140317250859008, 140317252956159, +STORE, 140317252956160, 140317252997119, +SNULL, 140317252956160, 140317252980735, +STORE, 140317252980736, 140317252997119, +STORE, 140317252956160, 140317252980735, +ERASE, 140317252956160, 140317252980735, +STORE, 140317252956160, 140317252980735, +ERASE, 140317252980736, 140317252997119, +STORE, 140317252980736, 140317252997119, +STORE, 140317257306112, 140317257322495, +SNULL, 140317252972543, 140317252980735, +STORE, 140317252956160, 140317252972543, +STORE, 140317252972544, 140317252980735, +SNULL, 140317255106559, 140317255110655, +STORE, 140317255102464, 140317255106559, +STORE, 140317255106560, 140317255110655, +SNULL, 94461168304127, 94461168312319, +STORE, 94461168300032, 94461168304127, +STORE, 94461168304128, 94461168312319, +SNULL, 140317257355263, 140317257359359, +STORE, 140317257351168, 140317257355263, +STORE, 140317257355264, 140317257359359, +ERASE, 140317257322496, 140317257351167, +STORE, 94461195268096, 94461195403263, +STORE, 140317255622656, 140317257306111, +STORE, 94461195268096, 94461195538431, +STORE, 94461195268096, 94461195673599, +STORE, 94110050402304, 94110050615295, +STORE, 94110052712448, 94110052716543, +STORE, 94110052716544, 94110052724735, +STORE, 94110052724736, 94110052737023, +STORE, 94110061875200, 94110062415871, +STORE, 140139439357952, 140139441016831, +STORE, 140139441016832, 140139443113983, +STORE, 140139443113984, 140139443130367, +STORE, 140139443130368, 140139443138559, +STORE, 140139443138560, 140139443154943, +STORE, 140139443154944, 140139443167231, +STORE, 140139443167232, 140139445260287, +STORE, 140139445260288, 140139445264383, +STORE, 140139445264384, 140139445268479, +STORE, 140139445268480, 140139445411839, +STORE, 140139445780480, 140139447463935, +STORE, 140139447463936, 140139447480319, +STORE, 140139447508992, 140139447513087, +STORE, 140139447513088, 140139447517183, +STORE, 140139447517184, 140139447521279, +STORE, 140731901427712, 140731901566975, +STORE, 140731902259200, 140731902271487, +STORE, 140731902271488, 140731902275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720941613056, 140737488351231, +SNULL, 140720941621247, 140737488351231, +STORE, 140720941613056, 140720941621247, +STORE, 140720941481984, 140720941621247, +STORE, 93902377721856, 93902379945983, +SNULL, 93902377832447, 93902379945983, +STORE, 93902377721856, 93902377832447, +STORE, 93902377832448, 93902379945983, +ERASE, 93902377832448, 93902379945983, +STORE, 93902379925504, 93902379937791, +STORE, 93902379937792, 93902379945983, +STORE, 139836543635456, 139836545888255, +SNULL, 139836543778815, 139836545888255, +STORE, 139836543635456, 139836543778815, +STORE, 139836543778816, 139836545888255, +ERASE, 139836543778816, 139836545888255, +STORE, 139836545875968, 139836545884159, +STORE, 139836545884160, 139836545888255, +STORE, 140720941711360, 140720941715455, +STORE, 140720941699072, 140720941711359, +STORE, 139836545847296, 139836545875967, +STORE, 139836545839104, 139836545847295, +STORE, 139836539838464, 139836543635455, +SNULL, 139836539838464, 139836541497343, +STORE, 139836541497344, 139836543635455, +STORE, 139836539838464, 139836541497343, +SNULL, 139836543594495, 139836543635455, +STORE, 139836541497344, 139836543594495, +STORE, 139836543594496, 139836543635455, +SNULL, 139836543594496, 139836543619071, +STORE, 139836543619072, 139836543635455, +STORE, 139836543594496, 139836543619071, +ERASE, 139836543594496, 139836543619071, +STORE, 139836543594496, 139836543619071, +ERASE, 139836543619072, 139836543635455, +STORE, 139836543619072, 139836543635455, +SNULL, 139836543610879, 139836543619071, +STORE, 139836543594496, 139836543610879, +STORE, 139836543610880, 139836543619071, +SNULL, 93902379933695, 93902379937791, +STORE, 93902379925504, 93902379933695, +STORE, 93902379933696, 93902379937791, +SNULL, 139836545880063, 139836545884159, +STORE, 139836545875968, 139836545880063, +STORE, 139836545880064, 139836545884159, +ERASE, 139836545847296, 139836545875967, +STORE, 93902396891136, 93902397026303, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736538206208, 140737488351231, +SNULL, 140736538214399, 140737488351231, +STORE, 140736538206208, 140736538214399, +STORE, 140736538075136, 140736538214399, +STORE, 94173471399936, 94173473734655, +SNULL, 94173471612927, 94173473734655, +STORE, 94173471399936, 94173471612927, +STORE, 94173471612928, 94173473734655, +ERASE, 94173471612928, 94173473734655, +STORE, 94173473710080, 94173473722367, +STORE, 94173473722368, 94173473734655, +STORE, 140035513556992, 140035515809791, +SNULL, 140035513700351, 140035515809791, +STORE, 140035513556992, 140035513700351, +STORE, 140035513700352, 140035515809791, +ERASE, 140035513700352, 140035515809791, +STORE, 140035515797504, 140035515805695, +STORE, 140035515805696, 140035515809791, +STORE, 140736538329088, 140736538333183, +STORE, 140736538316800, 140736538329087, +STORE, 140035515768832, 140035515797503, +STORE, 140035515760640, 140035515768831, +STORE, 140035511443456, 140035513556991, +SNULL, 140035511443456, 140035511455743, +STORE, 140035511455744, 140035513556991, +STORE, 140035511443456, 140035511455743, +SNULL, 140035513548799, 140035513556991, +STORE, 140035511455744, 140035513548799, +STORE, 140035513548800, 140035513556991, +ERASE, 140035513548800, 140035513556991, +STORE, 140035513548800, 140035513556991, +STORE, 140035507646464, 140035511443455, +SNULL, 140035507646464, 140035509305343, +STORE, 140035509305344, 140035511443455, +STORE, 140035507646464, 140035509305343, +SNULL, 140035511402495, 140035511443455, +STORE, 140035509305344, 140035511402495, +STORE, 140035511402496, 140035511443455, +SNULL, 140035511402496, 140035511427071, +STORE, 140035511427072, 140035511443455, +STORE, 140035511402496, 140035511427071, +ERASE, 140035511402496, 140035511427071, +STORE, 140035511402496, 140035511427071, +ERASE, 140035511427072, 140035511443455, +STORE, 140035511427072, 140035511443455, +STORE, 140035515752448, 140035515768831, +SNULL, 140035511418879, 140035511427071, +STORE, 140035511402496, 140035511418879, +STORE, 140035511418880, 140035511427071, +SNULL, 140035513552895, 140035513556991, +STORE, 140035513548800, 140035513552895, +STORE, 140035513552896, 140035513556991, +SNULL, 94173473714175, 94173473722367, +STORE, 94173473710080, 94173473714175, +STORE, 94173473714176, 94173473722367, +SNULL, 140035515801599, 140035515805695, +STORE, 140035515797504, 140035515801599, +STORE, 140035515801600, 140035515805695, +ERASE, 140035515768832, 140035515797503, +STORE, 94173478645760, 94173478780927, +STORE, 140035514068992, 140035515752447, +STORE, 94173478645760, 94173478916095, +STORE, 94173478645760, 94173479051263, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724216176640, 140737488351231, +SNULL, 140724216184831, 140737488351231, +STORE, 140724216176640, 140724216184831, +STORE, 140724216045568, 140724216184831, +STORE, 94870930628608, 94870932963327, +SNULL, 94870930841599, 94870932963327, +STORE, 94870930628608, 94870930841599, +STORE, 94870930841600, 94870932963327, +ERASE, 94870930841600, 94870932963327, +STORE, 94870932938752, 94870932951039, +STORE, 94870932951040, 94870932963327, +STORE, 140453683736576, 140453685989375, +SNULL, 140453683879935, 140453685989375, +STORE, 140453683736576, 140453683879935, +STORE, 140453683879936, 140453685989375, +ERASE, 140453683879936, 140453685989375, +STORE, 140453685977088, 140453685985279, +STORE, 140453685985280, 140453685989375, +STORE, 140724216832000, 140724216836095, +STORE, 140724216819712, 140724216831999, +STORE, 140453685948416, 140453685977087, +STORE, 140453685940224, 140453685948415, +STORE, 140453681623040, 140453683736575, +SNULL, 140453681623040, 140453681635327, +STORE, 140453681635328, 140453683736575, +STORE, 140453681623040, 140453681635327, +SNULL, 140453683728383, 140453683736575, +STORE, 140453681635328, 140453683728383, +STORE, 140453683728384, 140453683736575, +ERASE, 140453683728384, 140453683736575, +STORE, 140453683728384, 140453683736575, +STORE, 140453677826048, 140453681623039, +SNULL, 140453677826048, 140453679484927, +STORE, 140453679484928, 140453681623039, +STORE, 140453677826048, 140453679484927, +SNULL, 140453681582079, 140453681623039, +STORE, 140453679484928, 140453681582079, +STORE, 140453681582080, 140453681623039, +SNULL, 140453681582080, 140453681606655, +STORE, 140453681606656, 140453681623039, +STORE, 140453681582080, 140453681606655, +ERASE, 140453681582080, 140453681606655, +STORE, 140453681582080, 140453681606655, +ERASE, 140453681606656, 140453681623039, +STORE, 140453681606656, 140453681623039, +STORE, 140453685932032, 140453685948415, +SNULL, 140453681598463, 140453681606655, +STORE, 140453681582080, 140453681598463, +STORE, 140453681598464, 140453681606655, +SNULL, 140453683732479, 140453683736575, +STORE, 140453683728384, 140453683732479, +STORE, 140453683732480, 140453683736575, +SNULL, 94870932942847, 94870932951039, +STORE, 94870932938752, 94870932942847, +STORE, 94870932942848, 94870932951039, +SNULL, 140453685981183, 140453685985279, +STORE, 140453685977088, 140453685981183, +STORE, 140453685981184, 140453685985279, +ERASE, 140453685948416, 140453685977087, +STORE, 94870940565504, 94870940700671, +STORE, 140453684248576, 140453685932031, +STORE, 94870940565504, 94870940835839, +STORE, 94870940565504, 94870940971007, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731275661312, 140737488351231, +SNULL, 140731275669503, 140737488351231, +STORE, 140731275661312, 140731275669503, +STORE, 140731275530240, 140731275669503, +STORE, 94642788548608, 94642790883327, +SNULL, 94642788761599, 94642790883327, +STORE, 94642788548608, 94642788761599, +STORE, 94642788761600, 94642790883327, +ERASE, 94642788761600, 94642790883327, +STORE, 94642790858752, 94642790871039, +STORE, 94642790871040, 94642790883327, +STORE, 140228458749952, 140228461002751, +SNULL, 140228458893311, 140228461002751, +STORE, 140228458749952, 140228458893311, +STORE, 140228458893312, 140228461002751, +ERASE, 140228458893312, 140228461002751, +STORE, 140228460990464, 140228460998655, +STORE, 140228460998656, 140228461002751, +STORE, 140731276349440, 140731276353535, +STORE, 140731276337152, 140731276349439, +STORE, 140228460961792, 140228460990463, +STORE, 140228460953600, 140228460961791, +STORE, 140228456636416, 140228458749951, +SNULL, 140228456636416, 140228456648703, +STORE, 140228456648704, 140228458749951, +STORE, 140228456636416, 140228456648703, +SNULL, 140228458741759, 140228458749951, +STORE, 140228456648704, 140228458741759, +STORE, 140228458741760, 140228458749951, +ERASE, 140228458741760, 140228458749951, +STORE, 140228458741760, 140228458749951, +STORE, 140228452839424, 140228456636415, +SNULL, 140228452839424, 140228454498303, +STORE, 140228454498304, 140228456636415, +STORE, 140228452839424, 140228454498303, +SNULL, 140228456595455, 140228456636415, +STORE, 140228454498304, 140228456595455, +STORE, 140228456595456, 140228456636415, +SNULL, 140228456595456, 140228456620031, +STORE, 140228456620032, 140228456636415, +STORE, 140228456595456, 140228456620031, +ERASE, 140228456595456, 140228456620031, +STORE, 140228456595456, 140228456620031, +ERASE, 140228456620032, 140228456636415, +STORE, 140228456620032, 140228456636415, +STORE, 140228460945408, 140228460961791, +SNULL, 140228456611839, 140228456620031, +STORE, 140228456595456, 140228456611839, +STORE, 140228456611840, 140228456620031, +SNULL, 140228458745855, 140228458749951, +STORE, 140228458741760, 140228458745855, +STORE, 140228458745856, 140228458749951, +SNULL, 94642790862847, 94642790871039, +STORE, 94642790858752, 94642790862847, +STORE, 94642790862848, 94642790871039, +SNULL, 140228460994559, 140228460998655, +STORE, 140228460990464, 140228460994559, +STORE, 140228460994560, 140228460998655, +ERASE, 140228460961792, 140228460990463, +STORE, 94642801549312, 94642801684479, +STORE, 140228459261952, 140228460945407, +STORE, 94642801549312, 94642801819647, +STORE, 94642801549312, 94642801954815, +STORE, 94604087611392, 94604087824383, +STORE, 94604089921536, 94604089925631, +STORE, 94604089925632, 94604089933823, +STORE, 94604089933824, 94604089946111, +STORE, 94604105125888, 94604106424319, +STORE, 140454937694208, 140454939353087, +STORE, 140454939353088, 140454941450239, +STORE, 140454941450240, 140454941466623, +STORE, 140454941466624, 140454941474815, +STORE, 140454941474816, 140454941491199, +STORE, 140454941491200, 140454941503487, +STORE, 140454941503488, 140454943596543, +STORE, 140454943596544, 140454943600639, +STORE, 140454943600640, 140454943604735, +STORE, 140454943604736, 140454943748095, +STORE, 140454944116736, 140454945800191, +STORE, 140454945800192, 140454945816575, +STORE, 140454945845248, 140454945849343, +STORE, 140454945849344, 140454945853439, +STORE, 140454945853440, 140454945857535, +STORE, 140728438214656, 140728438353919, +STORE, 140728439095296, 140728439107583, +STORE, 140728439107584, 140728439111679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721843453952, 140737488351231, +SNULL, 140721843462143, 140737488351231, +STORE, 140721843453952, 140721843462143, +STORE, 140721843322880, 140721843462143, +STORE, 94465962455040, 94465964789759, +SNULL, 94465962668031, 94465964789759, +STORE, 94465962455040, 94465962668031, +STORE, 94465962668032, 94465964789759, +ERASE, 94465962668032, 94465964789759, +STORE, 94465964765184, 94465964777471, +STORE, 94465964777472, 94465964789759, +STORE, 139913488314368, 139913490567167, +SNULL, 139913488457727, 139913490567167, +STORE, 139913488314368, 139913488457727, +STORE, 139913488457728, 139913490567167, +ERASE, 139913488457728, 139913490567167, +STORE, 139913490554880, 139913490563071, +STORE, 139913490563072, 139913490567167, +STORE, 140721843503104, 140721843507199, +STORE, 140721843490816, 140721843503103, +STORE, 139913490526208, 139913490554879, +STORE, 139913490518016, 139913490526207, +STORE, 139913486200832, 139913488314367, +SNULL, 139913486200832, 139913486213119, +STORE, 139913486213120, 139913488314367, +STORE, 139913486200832, 139913486213119, +SNULL, 139913488306175, 139913488314367, +STORE, 139913486213120, 139913488306175, +STORE, 139913488306176, 139913488314367, +ERASE, 139913488306176, 139913488314367, +STORE, 139913488306176, 139913488314367, +STORE, 139913482403840, 139913486200831, +SNULL, 139913482403840, 139913484062719, +STORE, 139913484062720, 139913486200831, +STORE, 139913482403840, 139913484062719, +SNULL, 139913486159871, 139913486200831, +STORE, 139913484062720, 139913486159871, +STORE, 139913486159872, 139913486200831, +SNULL, 139913486159872, 139913486184447, +STORE, 139913486184448, 139913486200831, +STORE, 139913486159872, 139913486184447, +ERASE, 139913486159872, 139913486184447, +STORE, 139913486159872, 139913486184447, +ERASE, 139913486184448, 139913486200831, +STORE, 139913486184448, 139913486200831, +STORE, 139913490509824, 139913490526207, +SNULL, 139913486176255, 139913486184447, +STORE, 139913486159872, 139913486176255, +STORE, 139913486176256, 139913486184447, +SNULL, 139913488310271, 139913488314367, +STORE, 139913488306176, 139913488310271, +STORE, 139913488310272, 139913488314367, +SNULL, 94465964769279, 94465964777471, +STORE, 94465964765184, 94465964769279, +STORE, 94465964769280, 94465964777471, +SNULL, 139913490558975, 139913490563071, +STORE, 139913490554880, 139913490558975, +STORE, 139913490558976, 139913490563071, +ERASE, 139913490526208, 139913490554879, +STORE, 94465970024448, 94465970159615, +STORE, 139913488826368, 139913490509823, +STORE, 94465970024448, 94465970294783, +STORE, 94465970024448, 94465970429951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720583307264, 140737488351231, +SNULL, 140720583315455, 140737488351231, +STORE, 140720583307264, 140720583315455, +STORE, 140720583176192, 140720583315455, +STORE, 94212322082816, 94212324417535, +SNULL, 94212322295807, 94212324417535, +STORE, 94212322082816, 94212322295807, +STORE, 94212322295808, 94212324417535, +ERASE, 94212322295808, 94212324417535, +STORE, 94212324392960, 94212324405247, +STORE, 94212324405248, 94212324417535, +STORE, 139659688538112, 139659690790911, +SNULL, 139659688681471, 139659690790911, +STORE, 139659688538112, 139659688681471, +STORE, 139659688681472, 139659690790911, +ERASE, 139659688681472, 139659690790911, +STORE, 139659690778624, 139659690786815, +STORE, 139659690786816, 139659690790911, +STORE, 140720584781824, 140720584785919, +STORE, 140720584769536, 140720584781823, +STORE, 139659690749952, 139659690778623, +STORE, 139659690741760, 139659690749951, +STORE, 139659686424576, 139659688538111, +SNULL, 139659686424576, 139659686436863, +STORE, 139659686436864, 139659688538111, +STORE, 139659686424576, 139659686436863, +SNULL, 139659688529919, 139659688538111, +STORE, 139659686436864, 139659688529919, +STORE, 139659688529920, 139659688538111, +ERASE, 139659688529920, 139659688538111, +STORE, 139659688529920, 139659688538111, +STORE, 139659682627584, 139659686424575, +SNULL, 139659682627584, 139659684286463, +STORE, 139659684286464, 139659686424575, +STORE, 139659682627584, 139659684286463, +SNULL, 139659686383615, 139659686424575, +STORE, 139659684286464, 139659686383615, +STORE, 139659686383616, 139659686424575, +SNULL, 139659686383616, 139659686408191, +STORE, 139659686408192, 139659686424575, +STORE, 139659686383616, 139659686408191, +ERASE, 139659686383616, 139659686408191, +STORE, 139659686383616, 139659686408191, +ERASE, 139659686408192, 139659686424575, +STORE, 139659686408192, 139659686424575, +STORE, 139659690733568, 139659690749951, +SNULL, 139659686399999, 139659686408191, +STORE, 139659686383616, 139659686399999, +STORE, 139659686400000, 139659686408191, +SNULL, 139659688534015, 139659688538111, +STORE, 139659688529920, 139659688534015, +STORE, 139659688534016, 139659688538111, +SNULL, 94212324397055, 94212324405247, +STORE, 94212324392960, 94212324397055, +STORE, 94212324397056, 94212324405247, +SNULL, 139659690782719, 139659690786815, +STORE, 139659690778624, 139659690782719, +STORE, 139659690782720, 139659690786815, +ERASE, 139659690749952, 139659690778623, +STORE, 94212355014656, 94212355149823, +STORE, 139659689050112, 139659690733567, +STORE, 94212355014656, 94212355284991, +STORE, 94212355014656, 94212355420159, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727689830400, 140737488351231, +SNULL, 140727689838591, 140737488351231, +STORE, 140727689830400, 140727689838591, +STORE, 140727689699328, 140727689838591, +STORE, 94572390281216, 94572392615935, +SNULL, 94572390494207, 94572392615935, +STORE, 94572390281216, 94572390494207, +STORE, 94572390494208, 94572392615935, +ERASE, 94572390494208, 94572392615935, +STORE, 94572392591360, 94572392603647, +STORE, 94572392603648, 94572392615935, +STORE, 140575923769344, 140575926022143, +SNULL, 140575923912703, 140575926022143, +STORE, 140575923769344, 140575923912703, +STORE, 140575923912704, 140575926022143, +ERASE, 140575923912704, 140575926022143, +STORE, 140575926009856, 140575926018047, +STORE, 140575926018048, 140575926022143, +STORE, 140727689871360, 140727689875455, +STORE, 140727689859072, 140727689871359, +STORE, 140575925981184, 140575926009855, +STORE, 140575925972992, 140575925981183, +STORE, 140575921655808, 140575923769343, +SNULL, 140575921655808, 140575921668095, +STORE, 140575921668096, 140575923769343, +STORE, 140575921655808, 140575921668095, +SNULL, 140575923761151, 140575923769343, +STORE, 140575921668096, 140575923761151, +STORE, 140575923761152, 140575923769343, +ERASE, 140575923761152, 140575923769343, +STORE, 140575923761152, 140575923769343, +STORE, 140575917858816, 140575921655807, +SNULL, 140575917858816, 140575919517695, +STORE, 140575919517696, 140575921655807, +STORE, 140575917858816, 140575919517695, +SNULL, 140575921614847, 140575921655807, +STORE, 140575919517696, 140575921614847, +STORE, 140575921614848, 140575921655807, +SNULL, 140575921614848, 140575921639423, +STORE, 140575921639424, 140575921655807, +STORE, 140575921614848, 140575921639423, +ERASE, 140575921614848, 140575921639423, +STORE, 140575921614848, 140575921639423, +ERASE, 140575921639424, 140575921655807, +STORE, 140575921639424, 140575921655807, +STORE, 140575925964800, 140575925981183, +SNULL, 140575921631231, 140575921639423, +STORE, 140575921614848, 140575921631231, +STORE, 140575921631232, 140575921639423, +SNULL, 140575923765247, 140575923769343, +STORE, 140575923761152, 140575923765247, +STORE, 140575923765248, 140575923769343, +SNULL, 94572392595455, 94572392603647, +STORE, 94572392591360, 94572392595455, +STORE, 94572392595456, 94572392603647, +SNULL, 140575926013951, 140575926018047, +STORE, 140575926009856, 140575926013951, +STORE, 140575926013952, 140575926018047, +ERASE, 140575925981184, 140575926009855, +STORE, 94572402278400, 94572402413567, +STORE, 140575924281344, 140575925964799, +STORE, 94572402278400, 94572402548735, +STORE, 94572402278400, 94572402683903, +STORE, 94572402278400, 94572402851839, +SNULL, 94572402827263, 94572402851839, +STORE, 94572402278400, 94572402827263, +STORE, 94572402827264, 94572402851839, +ERASE, 94572402827264, 94572402851839, +STORE, 94572402278400, 94572402966527, +STORE, 94572402278400, 94572403109887, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725520506880, 140737488351231, +SNULL, 140725520515071, 140737488351231, +STORE, 140725520506880, 140725520515071, +STORE, 140725520375808, 140725520515071, +STORE, 93829948788736, 93829951012863, +SNULL, 93829948899327, 93829951012863, +STORE, 93829948788736, 93829948899327, +STORE, 93829948899328, 93829951012863, +ERASE, 93829948899328, 93829951012863, +STORE, 93829950992384, 93829951004671, +STORE, 93829951004672, 93829951012863, +STORE, 140133696794624, 140133699047423, +SNULL, 140133696937983, 140133699047423, +STORE, 140133696794624, 140133696937983, +STORE, 140133696937984, 140133699047423, +ERASE, 140133696937984, 140133699047423, +STORE, 140133699035136, 140133699043327, +STORE, 140133699043328, 140133699047423, +STORE, 140725520875520, 140725520879615, +STORE, 140725520863232, 140725520875519, +STORE, 140133699006464, 140133699035135, +STORE, 140133698998272, 140133699006463, +STORE, 140133692997632, 140133696794623, +SNULL, 140133692997632, 140133694656511, +STORE, 140133694656512, 140133696794623, +STORE, 140133692997632, 140133694656511, +SNULL, 140133696753663, 140133696794623, +STORE, 140133694656512, 140133696753663, +STORE, 140133696753664, 140133696794623, +SNULL, 140133696753664, 140133696778239, +STORE, 140133696778240, 140133696794623, +STORE, 140133696753664, 140133696778239, +ERASE, 140133696753664, 140133696778239, +STORE, 140133696753664, 140133696778239, +ERASE, 140133696778240, 140133696794623, +STORE, 140133696778240, 140133696794623, +SNULL, 140133696770047, 140133696778239, +STORE, 140133696753664, 140133696770047, +STORE, 140133696770048, 140133696778239, +SNULL, 93829951000575, 93829951004671, +STORE, 93829950992384, 93829951000575, +STORE, 93829951000576, 93829951004671, +SNULL, 140133699039231, 140133699043327, +STORE, 140133699035136, 140133699039231, +STORE, 140133699039232, 140133699043327, +ERASE, 140133699006464, 140133699035135, +STORE, 93829978693632, 93829978828799, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736118022144, 140737488351231, +SNULL, 140736118030335, 140737488351231, +STORE, 140736118022144, 140736118030335, +STORE, 140736117891072, 140736118030335, +STORE, 94467663982592, 94467666206719, +SNULL, 94467664093183, 94467666206719, +STORE, 94467663982592, 94467664093183, +STORE, 94467664093184, 94467666206719, +ERASE, 94467664093184, 94467666206719, +STORE, 94467666186240, 94467666198527, +STORE, 94467666198528, 94467666206719, +STORE, 140525377327104, 140525379579903, +SNULL, 140525377470463, 140525379579903, +STORE, 140525377327104, 140525377470463, +STORE, 140525377470464, 140525379579903, +ERASE, 140525377470464, 140525379579903, +STORE, 140525379567616, 140525379575807, +STORE, 140525379575808, 140525379579903, +STORE, 140736118771712, 140736118775807, +STORE, 140736118759424, 140736118771711, +STORE, 140525379538944, 140525379567615, +STORE, 140525379530752, 140525379538943, +STORE, 140525373530112, 140525377327103, +SNULL, 140525373530112, 140525375188991, +STORE, 140525375188992, 140525377327103, +STORE, 140525373530112, 140525375188991, +SNULL, 140525377286143, 140525377327103, +STORE, 140525375188992, 140525377286143, +STORE, 140525377286144, 140525377327103, +SNULL, 140525377286144, 140525377310719, +STORE, 140525377310720, 140525377327103, +STORE, 140525377286144, 140525377310719, +ERASE, 140525377286144, 140525377310719, +STORE, 140525377286144, 140525377310719, +ERASE, 140525377310720, 140525377327103, +STORE, 140525377310720, 140525377327103, +SNULL, 140525377302527, 140525377310719, +STORE, 140525377286144, 140525377302527, +STORE, 140525377302528, 140525377310719, +SNULL, 94467666194431, 94467666198527, +STORE, 94467666186240, 94467666194431, +STORE, 94467666194432, 94467666198527, +SNULL, 140525379571711, 140525379575807, +STORE, 140525379567616, 140525379571711, +STORE, 140525379571712, 140525379575807, +ERASE, 140525379538944, 140525379567615, +STORE, 94467693379584, 94467693514751, +STORE, 94200172744704, 94200172957695, +STORE, 94200175054848, 94200175058943, +STORE, 94200175058944, 94200175067135, +STORE, 94200175067136, 94200175079423, +STORE, 94200196673536, 94200198905855, +STORE, 140053867720704, 140053869379583, +STORE, 140053869379584, 140053871476735, +STORE, 140053871476736, 140053871493119, +STORE, 140053871493120, 140053871501311, +STORE, 140053871501312, 140053871517695, +STORE, 140053871517696, 140053871529983, +STORE, 140053871529984, 140053873623039, +STORE, 140053873623040, 140053873627135, +STORE, 140053873627136, 140053873631231, +STORE, 140053873631232, 140053873774591, +STORE, 140053874143232, 140053875826687, +STORE, 140053875826688, 140053875843071, +STORE, 140053875871744, 140053875875839, +STORE, 140053875875840, 140053875879935, +STORE, 140053875879936, 140053875884031, +STORE, 140728538484736, 140728538623999, +STORE, 140728538652672, 140728538664959, +STORE, 140728538664960, 140728538669055, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732307775488, 140737488351231, +SNULL, 140732307783679, 140737488351231, +STORE, 140732307775488, 140732307783679, +STORE, 140732307644416, 140732307783679, +STORE, 93831417630720, 93831419965439, +SNULL, 93831417843711, 93831419965439, +STORE, 93831417630720, 93831417843711, +STORE, 93831417843712, 93831419965439, +ERASE, 93831417843712, 93831419965439, +STORE, 93831419940864, 93831419953151, +STORE, 93831419953152, 93831419965439, +STORE, 140241062088704, 140241064341503, +SNULL, 140241062232063, 140241064341503, +STORE, 140241062088704, 140241062232063, +STORE, 140241062232064, 140241064341503, +ERASE, 140241062232064, 140241064341503, +STORE, 140241064329216, 140241064337407, +STORE, 140241064337408, 140241064341503, +STORE, 140732308140032, 140732308144127, +STORE, 140732308127744, 140732308140031, +STORE, 140241064300544, 140241064329215, +STORE, 140241064292352, 140241064300543, +STORE, 140241059975168, 140241062088703, +SNULL, 140241059975168, 140241059987455, +STORE, 140241059987456, 140241062088703, +STORE, 140241059975168, 140241059987455, +SNULL, 140241062080511, 140241062088703, +STORE, 140241059987456, 140241062080511, +STORE, 140241062080512, 140241062088703, +ERASE, 140241062080512, 140241062088703, +STORE, 140241062080512, 140241062088703, +STORE, 140241056178176, 140241059975167, +SNULL, 140241056178176, 140241057837055, +STORE, 140241057837056, 140241059975167, +STORE, 140241056178176, 140241057837055, +SNULL, 140241059934207, 140241059975167, +STORE, 140241057837056, 140241059934207, +STORE, 140241059934208, 140241059975167, +SNULL, 140241059934208, 140241059958783, +STORE, 140241059958784, 140241059975167, +STORE, 140241059934208, 140241059958783, +ERASE, 140241059934208, 140241059958783, +STORE, 140241059934208, 140241059958783, +ERASE, 140241059958784, 140241059975167, +STORE, 140241059958784, 140241059975167, +STORE, 140241064284160, 140241064300543, +SNULL, 140241059950591, 140241059958783, +STORE, 140241059934208, 140241059950591, +STORE, 140241059950592, 140241059958783, +SNULL, 140241062084607, 140241062088703, +STORE, 140241062080512, 140241062084607, +STORE, 140241062084608, 140241062088703, +SNULL, 93831419944959, 93831419953151, +STORE, 93831419940864, 93831419944959, +STORE, 93831419944960, 93831419953151, +SNULL, 140241064333311, 140241064337407, +STORE, 140241064329216, 140241064333311, +STORE, 140241064333312, 140241064337407, +ERASE, 140241064300544, 140241064329215, +STORE, 93831435284480, 93831435419647, +STORE, 140241062600704, 140241064284159, +STORE, 93831435284480, 93831435554815, +STORE, 93831435284480, 93831435689983, +STORE, 93831435284480, 93831435862015, +SNULL, 93831435837439, 93831435862015, +STORE, 93831435284480, 93831435837439, +STORE, 93831435837440, 93831435862015, +ERASE, 93831435837440, 93831435862015, +STORE, 93831435284480, 93831435972607, +STORE, 93831435284480, 93831436107775, +SNULL, 93831436091391, 93831436107775, +STORE, 93831435284480, 93831436091391, +STORE, 93831436091392, 93831436107775, +ERASE, 93831436091392, 93831436107775, +STORE, 93831435284480, 93831436226559, +STORE, 93831435284480, 93831436361727, +STORE, 93831435284480, 93831436505087, +STORE, 93831435284480, 93831436652543, +STORE, 93831435284480, 93831436787711, +STORE, 93831435284480, 93831436926975, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728546775040, 140737488351231, +SNULL, 140728546783231, 140737488351231, +STORE, 140728546775040, 140728546783231, +STORE, 140728546643968, 140728546783231, +STORE, 94456178786304, 94456181010431, +SNULL, 94456178896895, 94456181010431, +STORE, 94456178786304, 94456178896895, +STORE, 94456178896896, 94456181010431, +ERASE, 94456178896896, 94456181010431, +STORE, 94456180989952, 94456181002239, +STORE, 94456181002240, 94456181010431, +STORE, 140221893091328, 140221895344127, +SNULL, 140221893234687, 140221895344127, +STORE, 140221893091328, 140221893234687, +STORE, 140221893234688, 140221895344127, +ERASE, 140221893234688, 140221895344127, +STORE, 140221895331840, 140221895340031, +STORE, 140221895340032, 140221895344127, +STORE, 140728547803136, 140728547807231, +STORE, 140728547790848, 140728547803135, +STORE, 140221895303168, 140221895331839, +STORE, 140221895294976, 140221895303167, +STORE, 140221889294336, 140221893091327, +SNULL, 140221889294336, 140221890953215, +STORE, 140221890953216, 140221893091327, +STORE, 140221889294336, 140221890953215, +SNULL, 140221893050367, 140221893091327, +STORE, 140221890953216, 140221893050367, +STORE, 140221893050368, 140221893091327, +SNULL, 140221893050368, 140221893074943, +STORE, 140221893074944, 140221893091327, +STORE, 140221893050368, 140221893074943, +ERASE, 140221893050368, 140221893074943, +STORE, 140221893050368, 140221893074943, +ERASE, 140221893074944, 140221893091327, +STORE, 140221893074944, 140221893091327, +SNULL, 140221893066751, 140221893074943, +STORE, 140221893050368, 140221893066751, +STORE, 140221893066752, 140221893074943, +SNULL, 94456180998143, 94456181002239, +STORE, 94456180989952, 94456180998143, +STORE, 94456180998144, 94456181002239, +SNULL, 140221895335935, 140221895340031, +STORE, 140221895331840, 140221895335935, +STORE, 140221895335936, 140221895340031, +ERASE, 140221895303168, 140221895331839, +STORE, 94456203730944, 94456203866111, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734438637568, 140737488351231, +SNULL, 140734438645759, 140737488351231, +STORE, 140734438637568, 140734438645759, +STORE, 140734438506496, 140734438645759, +STORE, 94652233351168, 94652235575295, +SNULL, 94652233461759, 94652235575295, +STORE, 94652233351168, 94652233461759, +STORE, 94652233461760, 94652235575295, +ERASE, 94652233461760, 94652235575295, +STORE, 94652235554816, 94652235567103, +STORE, 94652235567104, 94652235575295, +STORE, 140536493195264, 140536495448063, +SNULL, 140536493338623, 140536495448063, +STORE, 140536493195264, 140536493338623, +STORE, 140536493338624, 140536495448063, +ERASE, 140536493338624, 140536495448063, +STORE, 140536495435776, 140536495443967, +STORE, 140536495443968, 140536495448063, +STORE, 140734439002112, 140734439006207, +STORE, 140734438989824, 140734439002111, +STORE, 140536495407104, 140536495435775, +STORE, 140536495398912, 140536495407103, +STORE, 140536489398272, 140536493195263, +SNULL, 140536489398272, 140536491057151, +STORE, 140536491057152, 140536493195263, +STORE, 140536489398272, 140536491057151, +SNULL, 140536493154303, 140536493195263, +STORE, 140536491057152, 140536493154303, +STORE, 140536493154304, 140536493195263, +SNULL, 140536493154304, 140536493178879, +STORE, 140536493178880, 140536493195263, +STORE, 140536493154304, 140536493178879, +ERASE, 140536493154304, 140536493178879, +STORE, 140536493154304, 140536493178879, +ERASE, 140536493178880, 140536493195263, +STORE, 140536493178880, 140536493195263, +SNULL, 140536493170687, 140536493178879, +STORE, 140536493154304, 140536493170687, +STORE, 140536493170688, 140536493178879, +SNULL, 94652235563007, 94652235567103, +STORE, 94652235554816, 94652235563007, +STORE, 94652235563008, 94652235567103, +SNULL, 140536495439871, 140536495443967, +STORE, 140536495435776, 140536495439871, +STORE, 140536495439872, 140536495443967, +ERASE, 140536495407104, 140536495435775, +STORE, 94652265619456, 94652265754623, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721814200320, 140737488351231, +SNULL, 140721814208511, 140737488351231, +STORE, 140721814200320, 140721814208511, +STORE, 140721814069248, 140721814208511, +STORE, 94062800691200, 94062802915327, +SNULL, 94062800801791, 94062802915327, +STORE, 94062800691200, 94062800801791, +STORE, 94062800801792, 94062802915327, +ERASE, 94062800801792, 94062802915327, +STORE, 94062802894848, 94062802907135, +STORE, 94062802907136, 94062802915327, +STORE, 139717739700224, 139717741953023, +SNULL, 139717739843583, 139717741953023, +STORE, 139717739700224, 139717739843583, +STORE, 139717739843584, 139717741953023, +ERASE, 139717739843584, 139717741953023, +STORE, 139717741940736, 139717741948927, +STORE, 139717741948928, 139717741953023, +STORE, 140721814224896, 140721814228991, +STORE, 140721814212608, 140721814224895, +STORE, 139717741912064, 139717741940735, +STORE, 139717741903872, 139717741912063, +STORE, 139717735903232, 139717739700223, +SNULL, 139717735903232, 139717737562111, +STORE, 139717737562112, 139717739700223, +STORE, 139717735903232, 139717737562111, +SNULL, 139717739659263, 139717739700223, +STORE, 139717737562112, 139717739659263, +STORE, 139717739659264, 139717739700223, +SNULL, 139717739659264, 139717739683839, +STORE, 139717739683840, 139717739700223, +STORE, 139717739659264, 139717739683839, +ERASE, 139717739659264, 139717739683839, +STORE, 139717739659264, 139717739683839, +ERASE, 139717739683840, 139717739700223, +STORE, 139717739683840, 139717739700223, +SNULL, 139717739675647, 139717739683839, +STORE, 139717739659264, 139717739675647, +STORE, 139717739675648, 139717739683839, +SNULL, 94062802903039, 94062802907135, +STORE, 94062802894848, 94062802903039, +STORE, 94062802903040, 94062802907135, +SNULL, 139717741944831, 139717741948927, +STORE, 139717741940736, 139717741944831, +STORE, 139717741944832, 139717741948927, +ERASE, 139717741912064, 139717741940735, +STORE, 94062814060544, 94062814195711, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723945754624, 140737488351231, +SNULL, 140723945762815, 140737488351231, +STORE, 140723945754624, 140723945762815, +STORE, 140723945623552, 140723945762815, +STORE, 94886119305216, 94886121639935, +SNULL, 94886119518207, 94886121639935, +STORE, 94886119305216, 94886119518207, +STORE, 94886119518208, 94886121639935, +ERASE, 94886119518208, 94886121639935, +STORE, 94886121615360, 94886121627647, +STORE, 94886121627648, 94886121639935, +STORE, 140152532131840, 140152534384639, +SNULL, 140152532275199, 140152534384639, +STORE, 140152532131840, 140152532275199, +STORE, 140152532275200, 140152534384639, +ERASE, 140152532275200, 140152534384639, +STORE, 140152534372352, 140152534380543, +STORE, 140152534380544, 140152534384639, +STORE, 140723946213376, 140723946217471, +STORE, 140723946201088, 140723946213375, +STORE, 140152534343680, 140152534372351, +STORE, 140152534335488, 140152534343679, +STORE, 140152530018304, 140152532131839, +SNULL, 140152530018304, 140152530030591, +STORE, 140152530030592, 140152532131839, +STORE, 140152530018304, 140152530030591, +SNULL, 140152532123647, 140152532131839, +STORE, 140152530030592, 140152532123647, +STORE, 140152532123648, 140152532131839, +ERASE, 140152532123648, 140152532131839, +STORE, 140152532123648, 140152532131839, +STORE, 140152526221312, 140152530018303, +SNULL, 140152526221312, 140152527880191, +STORE, 140152527880192, 140152530018303, +STORE, 140152526221312, 140152527880191, +SNULL, 140152529977343, 140152530018303, +STORE, 140152527880192, 140152529977343, +STORE, 140152529977344, 140152530018303, +SNULL, 140152529977344, 140152530001919, +STORE, 140152530001920, 140152530018303, +STORE, 140152529977344, 140152530001919, +ERASE, 140152529977344, 140152530001919, +STORE, 140152529977344, 140152530001919, +ERASE, 140152530001920, 140152530018303, +STORE, 140152530001920, 140152530018303, +STORE, 140152534327296, 140152534343679, +SNULL, 140152529993727, 140152530001919, +STORE, 140152529977344, 140152529993727, +STORE, 140152529993728, 140152530001919, +SNULL, 140152532127743, 140152532131839, +STORE, 140152532123648, 140152532127743, +STORE, 140152532127744, 140152532131839, +SNULL, 94886121619455, 94886121627647, +STORE, 94886121615360, 94886121619455, +STORE, 94886121619456, 94886121627647, +SNULL, 140152534376447, 140152534380543, +STORE, 140152534372352, 140152534376447, +STORE, 140152534376448, 140152534380543, +ERASE, 140152534343680, 140152534372351, +STORE, 94886129770496, 94886129905663, +STORE, 140152532643840, 140152534327295, +STORE, 94886129770496, 94886130040831, +STORE, 94886129770496, 94886130175999, +STORE, 94886129770496, 94886130348031, +SNULL, 94886130323455, 94886130348031, +STORE, 94886129770496, 94886130323455, +STORE, 94886130323456, 94886130348031, +ERASE, 94886130323456, 94886130348031, +STORE, 94886129770496, 94886130458623, +STORE, 94886129770496, 94886130606079, +SNULL, 94886130573311, 94886130606079, +STORE, 94886129770496, 94886130573311, +STORE, 94886130573312, 94886130606079, +ERASE, 94886130573312, 94886130606079, +STORE, 94886129770496, 94886130724863, +STORE, 94886129770496, 94886130876415, +STORE, 94886129770496, 94886131023871, +STORE, 94886129770496, 94886131175423, +STORE, 94886129770496, 94886131318783, +STORE, 94886129770496, 94886131453951, +SNULL, 94886131449855, 94886131453951, +STORE, 94886129770496, 94886131449855, +STORE, 94886131449856, 94886131453951, +ERASE, 94886131449856, 94886131453951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735450779648, 140737488351231, +SNULL, 140735450787839, 140737488351231, +STORE, 140735450779648, 140735450787839, +STORE, 140735450648576, 140735450787839, +STORE, 93947794079744, 93947796414463, +SNULL, 93947794292735, 93947796414463, +STORE, 93947794079744, 93947794292735, +STORE, 93947794292736, 93947796414463, +ERASE, 93947794292736, 93947796414463, +STORE, 93947796389888, 93947796402175, +STORE, 93947796402176, 93947796414463, +STORE, 139841993433088, 139841995685887, +SNULL, 139841993576447, 139841995685887, +STORE, 139841993433088, 139841993576447, +STORE, 139841993576448, 139841995685887, +ERASE, 139841993576448, 139841995685887, +STORE, 139841995673600, 139841995681791, +STORE, 139841995681792, 139841995685887, +STORE, 140735451308032, 140735451312127, +STORE, 140735451295744, 140735451308031, +STORE, 139841995644928, 139841995673599, +STORE, 139841995636736, 139841995644927, +STORE, 139841991319552, 139841993433087, +SNULL, 139841991319552, 139841991331839, +STORE, 139841991331840, 139841993433087, +STORE, 139841991319552, 139841991331839, +SNULL, 139841993424895, 139841993433087, +STORE, 139841991331840, 139841993424895, +STORE, 139841993424896, 139841993433087, +ERASE, 139841993424896, 139841993433087, +STORE, 139841993424896, 139841993433087, +STORE, 139841987522560, 139841991319551, +SNULL, 139841987522560, 139841989181439, +STORE, 139841989181440, 139841991319551, +STORE, 139841987522560, 139841989181439, +SNULL, 139841991278591, 139841991319551, +STORE, 139841989181440, 139841991278591, +STORE, 139841991278592, 139841991319551, +SNULL, 139841991278592, 139841991303167, +STORE, 139841991303168, 139841991319551, +STORE, 139841991278592, 139841991303167, +ERASE, 139841991278592, 139841991303167, +STORE, 139841991278592, 139841991303167, +ERASE, 139841991303168, 139841991319551, +STORE, 139841991303168, 139841991319551, +STORE, 139841995628544, 139841995644927, +SNULL, 139841991294975, 139841991303167, +STORE, 139841991278592, 139841991294975, +STORE, 139841991294976, 139841991303167, +SNULL, 139841993428991, 139841993433087, +STORE, 139841993424896, 139841993428991, +STORE, 139841993428992, 139841993433087, +SNULL, 93947796393983, 93947796402175, +STORE, 93947796389888, 93947796393983, +STORE, 93947796393984, 93947796402175, +SNULL, 139841995677695, 139841995681791, +STORE, 139841995673600, 139841995677695, +STORE, 139841995677696, 139841995681791, +ERASE, 139841995644928, 139841995673599, +STORE, 93947829739520, 93947829874687, +STORE, 139841993945088, 139841995628543, +STORE, 93947829739520, 93947830009855, +STORE, 93947829739520, 93947830145023, +STORE, 94659351814144, 94659352027135, +STORE, 94659354124288, 94659354128383, +STORE, 94659354128384, 94659354136575, +STORE, 94659354136576, 94659354148863, +STORE, 94659383476224, 94659385057279, +STORE, 139959054557184, 139959056216063, +STORE, 139959056216064, 139959058313215, +STORE, 139959058313216, 139959058329599, +STORE, 139959058329600, 139959058337791, +STORE, 139959058337792, 139959058354175, +STORE, 139959058354176, 139959058366463, +STORE, 139959058366464, 139959060459519, +STORE, 139959060459520, 139959060463615, +STORE, 139959060463616, 139959060467711, +STORE, 139959060467712, 139959060611071, +STORE, 139959060979712, 139959062663167, +STORE, 139959062663168, 139959062679551, +STORE, 139959062708224, 139959062712319, +STORE, 139959062712320, 139959062716415, +STORE, 139959062716416, 139959062720511, +STORE, 140735532539904, 140735532679167, +STORE, 140735532830720, 140735532843007, +STORE, 140735532843008, 140735532847103, +STORE, 93894361829376, 93894362042367, +STORE, 93894364139520, 93894364143615, +STORE, 93894364143616, 93894364151807, +STORE, 93894364151808, 93894364164095, +STORE, 93894396944384, 93894397624319, +STORE, 140075612573696, 140075614232575, +STORE, 140075614232576, 140075616329727, +STORE, 140075616329728, 140075616346111, +STORE, 140075616346112, 140075616354303, +STORE, 140075616354304, 140075616370687, +STORE, 140075616370688, 140075616382975, +STORE, 140075616382976, 140075618476031, +STORE, 140075618476032, 140075618480127, +STORE, 140075618480128, 140075618484223, +STORE, 140075618484224, 140075618627583, +STORE, 140075618996224, 140075620679679, +STORE, 140075620679680, 140075620696063, +STORE, 140075620724736, 140075620728831, +STORE, 140075620728832, 140075620732927, +STORE, 140075620732928, 140075620737023, +STORE, 140720830312448, 140720830451711, +STORE, 140720830631936, 140720830644223, +STORE, 140720830644224, 140720830648319, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735116226560, 140737488351231, +SNULL, 140735116234751, 140737488351231, +STORE, 140735116226560, 140735116234751, +STORE, 140735116095488, 140735116234751, +STORE, 94873398054912, 94873400279039, +SNULL, 94873398165503, 94873400279039, +STORE, 94873398054912, 94873398165503, +STORE, 94873398165504, 94873400279039, +ERASE, 94873398165504, 94873400279039, +STORE, 94873400258560, 94873400270847, +STORE, 94873400270848, 94873400279039, +STORE, 140303828606976, 140303830859775, +SNULL, 140303828750335, 140303830859775, +STORE, 140303828606976, 140303828750335, +STORE, 140303828750336, 140303830859775, +ERASE, 140303828750336, 140303830859775, +STORE, 140303830847488, 140303830855679, +STORE, 140303830855680, 140303830859775, +STORE, 140735116251136, 140735116255231, +STORE, 140735116238848, 140735116251135, +STORE, 140303830818816, 140303830847487, +STORE, 140303830810624, 140303830818815, +STORE, 140303824809984, 140303828606975, +SNULL, 140303824809984, 140303826468863, +STORE, 140303826468864, 140303828606975, +STORE, 140303824809984, 140303826468863, +SNULL, 140303828566015, 140303828606975, +STORE, 140303826468864, 140303828566015, +STORE, 140303828566016, 140303828606975, +SNULL, 140303828566016, 140303828590591, +STORE, 140303828590592, 140303828606975, +STORE, 140303828566016, 140303828590591, +ERASE, 140303828566016, 140303828590591, +STORE, 140303828566016, 140303828590591, +ERASE, 140303828590592, 140303828606975, +STORE, 140303828590592, 140303828606975, +SNULL, 140303828582399, 140303828590591, +STORE, 140303828566016, 140303828582399, +STORE, 140303828582400, 140303828590591, +SNULL, 94873400266751, 94873400270847, +STORE, 94873400258560, 94873400266751, +STORE, 94873400266752, 94873400270847, +SNULL, 140303830851583, 140303830855679, +STORE, 140303830847488, 140303830851583, +STORE, 140303830851584, 140303830855679, +ERASE, 140303830818816, 140303830847487, +STORE, 94873413713920, 94873413849087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732349956096, 140737488351231, +SNULL, 140732349964287, 140737488351231, +STORE, 140732349956096, 140732349964287, +STORE, 140732349825024, 140732349964287, +STORE, 94009652736000, 94009655070719, +SNULL, 94009652948991, 94009655070719, +STORE, 94009652736000, 94009652948991, +STORE, 94009652948992, 94009655070719, +ERASE, 94009652948992, 94009655070719, +STORE, 94009655046144, 94009655058431, +STORE, 94009655058432, 94009655070719, +STORE, 140295688531968, 140295690784767, +SNULL, 140295688675327, 140295690784767, +STORE, 140295688531968, 140295688675327, +STORE, 140295688675328, 140295690784767, +ERASE, 140295688675328, 140295690784767, +STORE, 140295690772480, 140295690780671, +STORE, 140295690780672, 140295690784767, +STORE, 140732350005248, 140732350009343, +STORE, 140732349992960, 140732350005247, +STORE, 140295690743808, 140295690772479, +STORE, 140295690735616, 140295690743807, +STORE, 140295686418432, 140295688531967, +SNULL, 140295686418432, 140295686430719, +STORE, 140295686430720, 140295688531967, +STORE, 140295686418432, 140295686430719, +SNULL, 140295688523775, 140295688531967, +STORE, 140295686430720, 140295688523775, +STORE, 140295688523776, 140295688531967, +ERASE, 140295688523776, 140295688531967, +STORE, 140295688523776, 140295688531967, +STORE, 140295682621440, 140295686418431, +SNULL, 140295682621440, 140295684280319, +STORE, 140295684280320, 140295686418431, +STORE, 140295682621440, 140295684280319, +SNULL, 140295686377471, 140295686418431, +STORE, 140295684280320, 140295686377471, +STORE, 140295686377472, 140295686418431, +SNULL, 140295686377472, 140295686402047, +STORE, 140295686402048, 140295686418431, +STORE, 140295686377472, 140295686402047, +ERASE, 140295686377472, 140295686402047, +STORE, 140295686377472, 140295686402047, +ERASE, 140295686402048, 140295686418431, +STORE, 140295686402048, 140295686418431, +STORE, 140295690727424, 140295690743807, +SNULL, 140295686393855, 140295686402047, +STORE, 140295686377472, 140295686393855, +STORE, 140295686393856, 140295686402047, +SNULL, 140295688527871, 140295688531967, +STORE, 140295688523776, 140295688527871, +STORE, 140295688527872, 140295688531967, +SNULL, 94009655050239, 94009655058431, +STORE, 94009655046144, 94009655050239, +STORE, 94009655050240, 94009655058431, +SNULL, 140295690776575, 140295690780671, +STORE, 140295690772480, 140295690776575, +STORE, 140295690776576, 140295690780671, +ERASE, 140295690743808, 140295690772479, +STORE, 94009672114176, 94009672249343, +STORE, 140295689043968, 140295690727423, +STORE, 94009672114176, 94009672384511, +STORE, 94009672114176, 94009672519679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722376515584, 140737488351231, +SNULL, 140722376523775, 140737488351231, +STORE, 140722376515584, 140722376523775, +STORE, 140722376384512, 140722376523775, +STORE, 94089815773184, 94089818107903, +SNULL, 94089815986175, 94089818107903, +STORE, 94089815773184, 94089815986175, +STORE, 94089815986176, 94089818107903, +ERASE, 94089815986176, 94089818107903, +STORE, 94089818083328, 94089818095615, +STORE, 94089818095616, 94089818107903, +STORE, 140265595711488, 140265597964287, +SNULL, 140265595854847, 140265597964287, +STORE, 140265595711488, 140265595854847, +STORE, 140265595854848, 140265597964287, +ERASE, 140265595854848, 140265597964287, +STORE, 140265597952000, 140265597960191, +STORE, 140265597960192, 140265597964287, +STORE, 140722378297344, 140722378301439, +STORE, 140722378285056, 140722378297343, +STORE, 140265597923328, 140265597951999, +STORE, 140265597915136, 140265597923327, +STORE, 140265593597952, 140265595711487, +SNULL, 140265593597952, 140265593610239, +STORE, 140265593610240, 140265595711487, +STORE, 140265593597952, 140265593610239, +SNULL, 140265595703295, 140265595711487, +STORE, 140265593610240, 140265595703295, +STORE, 140265595703296, 140265595711487, +ERASE, 140265595703296, 140265595711487, +STORE, 140265595703296, 140265595711487, +STORE, 140265589800960, 140265593597951, +SNULL, 140265589800960, 140265591459839, +STORE, 140265591459840, 140265593597951, +STORE, 140265589800960, 140265591459839, +SNULL, 140265593556991, 140265593597951, +STORE, 140265591459840, 140265593556991, +STORE, 140265593556992, 140265593597951, +SNULL, 140265593556992, 140265593581567, +STORE, 140265593581568, 140265593597951, +STORE, 140265593556992, 140265593581567, +ERASE, 140265593556992, 140265593581567, +STORE, 140265593556992, 140265593581567, +ERASE, 140265593581568, 140265593597951, +STORE, 140265593581568, 140265593597951, +STORE, 140265597906944, 140265597923327, +SNULL, 140265593573375, 140265593581567, +STORE, 140265593556992, 140265593573375, +STORE, 140265593573376, 140265593581567, +SNULL, 140265595707391, 140265595711487, +STORE, 140265595703296, 140265595707391, +STORE, 140265595707392, 140265595711487, +SNULL, 94089818087423, 94089818095615, +STORE, 94089818083328, 94089818087423, +STORE, 94089818087424, 94089818095615, +SNULL, 140265597956095, 140265597960191, +STORE, 140265597952000, 140265597956095, +STORE, 140265597956096, 140265597960191, +ERASE, 140265597923328, 140265597951999, +STORE, 94089837146112, 94089837281279, +STORE, 140265596223488, 140265597906943, +STORE, 94089837146112, 94089837416447, +STORE, 94089837146112, 94089837551615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735265218560, 140737488351231, +SNULL, 140735265226751, 140737488351231, +STORE, 140735265218560, 140735265226751, +STORE, 140735265087488, 140735265226751, +STORE, 94250422370304, 94250424705023, +SNULL, 94250422583295, 94250424705023, +STORE, 94250422370304, 94250422583295, +STORE, 94250422583296, 94250424705023, +ERASE, 94250422583296, 94250424705023, +STORE, 94250424680448, 94250424692735, +STORE, 94250424692736, 94250424705023, +STORE, 140344442474496, 140344444727295, +SNULL, 140344442617855, 140344444727295, +STORE, 140344442474496, 140344442617855, +STORE, 140344442617856, 140344444727295, +ERASE, 140344442617856, 140344444727295, +STORE, 140344444715008, 140344444723199, +STORE, 140344444723200, 140344444727295, +STORE, 140735265341440, 140735265345535, +STORE, 140735265329152, 140735265341439, +STORE, 140344444686336, 140344444715007, +STORE, 140344444678144, 140344444686335, +STORE, 140344440360960, 140344442474495, +SNULL, 140344440360960, 140344440373247, +STORE, 140344440373248, 140344442474495, +STORE, 140344440360960, 140344440373247, +SNULL, 140344442466303, 140344442474495, +STORE, 140344440373248, 140344442466303, +STORE, 140344442466304, 140344442474495, +ERASE, 140344442466304, 140344442474495, +STORE, 140344442466304, 140344442474495, +STORE, 140344436563968, 140344440360959, +SNULL, 140344436563968, 140344438222847, +STORE, 140344438222848, 140344440360959, +STORE, 140344436563968, 140344438222847, +SNULL, 140344440319999, 140344440360959, +STORE, 140344438222848, 140344440319999, +STORE, 140344440320000, 140344440360959, +SNULL, 140344440320000, 140344440344575, +STORE, 140344440344576, 140344440360959, +STORE, 140344440320000, 140344440344575, +ERASE, 140344440320000, 140344440344575, +STORE, 140344440320000, 140344440344575, +ERASE, 140344440344576, 140344440360959, +STORE, 140344440344576, 140344440360959, +STORE, 140344444669952, 140344444686335, +SNULL, 140344440336383, 140344440344575, +STORE, 140344440320000, 140344440336383, +STORE, 140344440336384, 140344440344575, +SNULL, 140344442470399, 140344442474495, +STORE, 140344442466304, 140344442470399, +STORE, 140344442470400, 140344442474495, +SNULL, 94250424684543, 94250424692735, +STORE, 94250424680448, 94250424684543, +STORE, 94250424684544, 94250424692735, +SNULL, 140344444719103, 140344444723199, +STORE, 140344444715008, 140344444719103, +STORE, 140344444719104, 140344444723199, +ERASE, 140344444686336, 140344444715007, +STORE, 94250445512704, 94250445647871, +STORE, 140344442986496, 140344444669951, +STORE, 94250445512704, 94250445783039, +STORE, 94250445512704, 94250445918207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725762719744, 140737488351231, +SNULL, 140725762727935, 140737488351231, +STORE, 140725762719744, 140725762727935, +STORE, 140725762588672, 140725762727935, +STORE, 94819009097728, 94819011432447, +SNULL, 94819009310719, 94819011432447, +STORE, 94819009097728, 94819009310719, +STORE, 94819009310720, 94819011432447, +ERASE, 94819009310720, 94819011432447, +STORE, 94819011407872, 94819011420159, +STORE, 94819011420160, 94819011432447, +STORE, 139987985596416, 139987987849215, +SNULL, 139987985739775, 139987987849215, +STORE, 139987985596416, 139987985739775, +STORE, 139987985739776, 139987987849215, +ERASE, 139987985739776, 139987987849215, +STORE, 139987987836928, 139987987845119, +STORE, 139987987845120, 139987987849215, +STORE, 140725763072000, 140725763076095, +STORE, 140725763059712, 140725763071999, +STORE, 139987987808256, 139987987836927, +STORE, 139987987800064, 139987987808255, +STORE, 139987983482880, 139987985596415, +SNULL, 139987983482880, 139987983495167, +STORE, 139987983495168, 139987985596415, +STORE, 139987983482880, 139987983495167, +SNULL, 139987985588223, 139987985596415, +STORE, 139987983495168, 139987985588223, +STORE, 139987985588224, 139987985596415, +ERASE, 139987985588224, 139987985596415, +STORE, 139987985588224, 139987985596415, +STORE, 139987979685888, 139987983482879, +SNULL, 139987979685888, 139987981344767, +STORE, 139987981344768, 139987983482879, +STORE, 139987979685888, 139987981344767, +SNULL, 139987983441919, 139987983482879, +STORE, 139987981344768, 139987983441919, +STORE, 139987983441920, 139987983482879, +SNULL, 139987983441920, 139987983466495, +STORE, 139987983466496, 139987983482879, +STORE, 139987983441920, 139987983466495, +ERASE, 139987983441920, 139987983466495, +STORE, 139987983441920, 139987983466495, +ERASE, 139987983466496, 139987983482879, +STORE, 139987983466496, 139987983482879, +STORE, 139987987791872, 139987987808255, +SNULL, 139987983458303, 139987983466495, +STORE, 139987983441920, 139987983458303, +STORE, 139987983458304, 139987983466495, +SNULL, 139987985592319, 139987985596415, +STORE, 139987985588224, 139987985592319, +STORE, 139987985592320, 139987985596415, +SNULL, 94819011411967, 94819011420159, +STORE, 94819011407872, 94819011411967, +STORE, 94819011411968, 94819011420159, +SNULL, 139987987841023, 139987987845119, +STORE, 139987987836928, 139987987841023, +STORE, 139987987841024, 139987987845119, +ERASE, 139987987808256, 139987987836927, +STORE, 94819028176896, 94819028312063, +STORE, 139987986108416, 139987987791871, +STORE, 94819028176896, 94819028447231, +STORE, 94819028176896, 94819028582399, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722475413504, 140737488351231, +SNULL, 140722475421695, 140737488351231, +STORE, 140722475413504, 140722475421695, +STORE, 140722475282432, 140722475421695, +STORE, 94620599119872, 94620601343999, +SNULL, 94620599230463, 94620601343999, +STORE, 94620599119872, 94620599230463, +STORE, 94620599230464, 94620601343999, +ERASE, 94620599230464, 94620601343999, +STORE, 94620601323520, 94620601335807, +STORE, 94620601335808, 94620601343999, +STORE, 139891763060736, 139891765313535, +SNULL, 139891763204095, 139891765313535, +STORE, 139891763060736, 139891763204095, +STORE, 139891763204096, 139891765313535, +ERASE, 139891763204096, 139891765313535, +STORE, 139891765301248, 139891765309439, +STORE, 139891765309440, 139891765313535, +STORE, 140722475700224, 140722475704319, +STORE, 140722475687936, 140722475700223, +STORE, 139891765272576, 139891765301247, +STORE, 139891765264384, 139891765272575, +STORE, 139891759263744, 139891763060735, +SNULL, 139891759263744, 139891760922623, +STORE, 139891760922624, 139891763060735, +STORE, 139891759263744, 139891760922623, +SNULL, 139891763019775, 139891763060735, +STORE, 139891760922624, 139891763019775, +STORE, 139891763019776, 139891763060735, +SNULL, 139891763019776, 139891763044351, +STORE, 139891763044352, 139891763060735, +STORE, 139891763019776, 139891763044351, +ERASE, 139891763019776, 139891763044351, +STORE, 139891763019776, 139891763044351, +ERASE, 139891763044352, 139891763060735, +STORE, 139891763044352, 139891763060735, +SNULL, 139891763036159, 139891763044351, +STORE, 139891763019776, 139891763036159, +STORE, 139891763036160, 139891763044351, +SNULL, 94620601331711, 94620601335807, +STORE, 94620601323520, 94620601331711, +STORE, 94620601331712, 94620601335807, +SNULL, 139891765305343, 139891765309439, +STORE, 139891765301248, 139891765305343, +STORE, 139891765305344, 139891765309439, +ERASE, 139891765272576, 139891765301247, +STORE, 94620610027520, 94620610162687, +STORE, 94031976210432, 94031976423423, +STORE, 94031978520576, 94031978524671, +STORE, 94031978524672, 94031978532863, +STORE, 94031978532864, 94031978545151, +STORE, 94031990398976, 94031992565759, +STORE, 140336240640000, 140336242298879, +STORE, 140336242298880, 140336244396031, +STORE, 140336244396032, 140336244412415, +STORE, 140336244412416, 140336244420607, +STORE, 140336244420608, 140336244436991, +STORE, 140336244436992, 140336244449279, +STORE, 140336244449280, 140336246542335, +STORE, 140336246542336, 140336246546431, +STORE, 140336246546432, 140336246550527, +STORE, 140336246550528, 140336246693887, +STORE, 140336247062528, 140336248745983, +STORE, 140336248745984, 140336248762367, +STORE, 140336248791040, 140336248795135, +STORE, 140336248795136, 140336248799231, +STORE, 140336248799232, 140336248803327, +STORE, 140728500064256, 140728500203519, +STORE, 140728501501952, 140728501514239, +STORE, 140728501514240, 140728501518335, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730503987200, 140737488351231, +SNULL, 140730503995391, 140737488351231, +STORE, 140730503987200, 140730503995391, +STORE, 140730503856128, 140730503995391, +STORE, 93866544205824, 93866546429951, +SNULL, 93866544316415, 93866546429951, +STORE, 93866544205824, 93866544316415, +STORE, 93866544316416, 93866546429951, +ERASE, 93866544316416, 93866546429951, +STORE, 93866546409472, 93866546421759, +STORE, 93866546421760, 93866546429951, +STORE, 140216311959552, 140216314212351, +SNULL, 140216312102911, 140216314212351, +STORE, 140216311959552, 140216312102911, +STORE, 140216312102912, 140216314212351, +ERASE, 140216312102912, 140216314212351, +STORE, 140216314200064, 140216314208255, +STORE, 140216314208256, 140216314212351, +STORE, 140730504626176, 140730504630271, +STORE, 140730504613888, 140730504626175, +STORE, 140216314171392, 140216314200063, +STORE, 140216314163200, 140216314171391, +STORE, 140216308162560, 140216311959551, +SNULL, 140216308162560, 140216309821439, +STORE, 140216309821440, 140216311959551, +STORE, 140216308162560, 140216309821439, +SNULL, 140216311918591, 140216311959551, +STORE, 140216309821440, 140216311918591, +STORE, 140216311918592, 140216311959551, +SNULL, 140216311918592, 140216311943167, +STORE, 140216311943168, 140216311959551, +STORE, 140216311918592, 140216311943167, +ERASE, 140216311918592, 140216311943167, +STORE, 140216311918592, 140216311943167, +ERASE, 140216311943168, 140216311959551, +STORE, 140216311943168, 140216311959551, +SNULL, 140216311934975, 140216311943167, +STORE, 140216311918592, 140216311934975, +STORE, 140216311934976, 140216311943167, +SNULL, 93866546417663, 93866546421759, +STORE, 93866546409472, 93866546417663, +STORE, 93866546417664, 93866546421759, +SNULL, 140216314204159, 140216314208255, +STORE, 140216314200064, 140216314204159, +STORE, 140216314204160, 140216314208255, +ERASE, 140216314171392, 140216314200063, +STORE, 93866550386688, 93866550521855, +STORE, 94074292674560, 94074292887551, +STORE, 94074294984704, 94074294988799, +STORE, 94074294988800, 94074294996991, +STORE, 94074294996992, 94074295009279, +STORE, 94074300219392, 94074301378559, +STORE, 139781563256832, 139781564915711, +STORE, 139781564915712, 139781567012863, +STORE, 139781567012864, 139781567029247, +STORE, 139781567029248, 139781567037439, +STORE, 139781567037440, 139781567053823, +STORE, 139781567053824, 139781567066111, +STORE, 139781567066112, 139781569159167, +STORE, 139781569159168, 139781569163263, +STORE, 139781569163264, 139781569167359, +STORE, 139781569167360, 139781569310719, +STORE, 139781569679360, 139781571362815, +STORE, 139781571362816, 139781571379199, +STORE, 139781571407872, 139781571411967, +STORE, 139781571411968, 139781571416063, +STORE, 139781571416064, 139781571420159, +STORE, 140723688488960, 140723688628223, +STORE, 140723689005056, 140723689017343, +STORE, 140723689017344, 140723689021439, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735189745664, 140737488351231, +SNULL, 140735189753855, 140737488351231, +STORE, 140735189745664, 140735189753855, +STORE, 140735189614592, 140735189753855, +STORE, 94172072177664, 94172074512383, +SNULL, 94172072390655, 94172074512383, +STORE, 94172072177664, 94172072390655, +STORE, 94172072390656, 94172074512383, +ERASE, 94172072390656, 94172074512383, +STORE, 94172074487808, 94172074500095, +STORE, 94172074500096, 94172074512383, +STORE, 140687827263488, 140687829516287, +SNULL, 140687827406847, 140687829516287, +STORE, 140687827263488, 140687827406847, +STORE, 140687827406848, 140687829516287, +ERASE, 140687827406848, 140687829516287, +STORE, 140687829504000, 140687829512191, +STORE, 140687829512192, 140687829516287, +STORE, 140735189766144, 140735189770239, +STORE, 140735189753856, 140735189766143, +STORE, 140687829475328, 140687829503999, +STORE, 140687829467136, 140687829475327, +STORE, 140687825149952, 140687827263487, +SNULL, 140687825149952, 140687825162239, +STORE, 140687825162240, 140687827263487, +STORE, 140687825149952, 140687825162239, +SNULL, 140687827255295, 140687827263487, +STORE, 140687825162240, 140687827255295, +STORE, 140687827255296, 140687827263487, +ERASE, 140687827255296, 140687827263487, +STORE, 140687827255296, 140687827263487, +STORE, 140687821352960, 140687825149951, +SNULL, 140687821352960, 140687823011839, +STORE, 140687823011840, 140687825149951, +STORE, 140687821352960, 140687823011839, +SNULL, 140687825108991, 140687825149951, +STORE, 140687823011840, 140687825108991, +STORE, 140687825108992, 140687825149951, +SNULL, 140687825108992, 140687825133567, +STORE, 140687825133568, 140687825149951, +STORE, 140687825108992, 140687825133567, +ERASE, 140687825108992, 140687825133567, +STORE, 140687825108992, 140687825133567, +ERASE, 140687825133568, 140687825149951, +STORE, 140687825133568, 140687825149951, +STORE, 140687829458944, 140687829475327, +SNULL, 140687825125375, 140687825133567, +STORE, 140687825108992, 140687825125375, +STORE, 140687825125376, 140687825133567, +SNULL, 140687827259391, 140687827263487, +STORE, 140687827255296, 140687827259391, +STORE, 140687827259392, 140687827263487, +SNULL, 94172074491903, 94172074500095, +STORE, 94172074487808, 94172074491903, +STORE, 94172074491904, 94172074500095, +SNULL, 140687829508095, 140687829512191, +STORE, 140687829504000, 140687829508095, +STORE, 140687829508096, 140687829512191, +ERASE, 140687829475328, 140687829503999, +STORE, 94172092432384, 94172092567551, +STORE, 140687827775488, 140687829458943, +STORE, 94172092432384, 94172092702719, +STORE, 94172092432384, 94172092837887, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737229504512, 140737488351231, +SNULL, 140737229512703, 140737488351231, +STORE, 140737229504512, 140737229512703, +STORE, 140737229373440, 140737229512703, +STORE, 94155246866432, 94155249090559, +SNULL, 94155246977023, 94155249090559, +STORE, 94155246866432, 94155246977023, +STORE, 94155246977024, 94155249090559, +ERASE, 94155246977024, 94155249090559, +STORE, 94155249070080, 94155249082367, +STORE, 94155249082368, 94155249090559, +STORE, 140640993693696, 140640995946495, +SNULL, 140640993837055, 140640995946495, +STORE, 140640993693696, 140640993837055, +STORE, 140640993837056, 140640995946495, +ERASE, 140640993837056, 140640995946495, +STORE, 140640995934208, 140640995942399, +STORE, 140640995942400, 140640995946495, +STORE, 140737230004224, 140737230008319, +STORE, 140737229991936, 140737230004223, +STORE, 140640995905536, 140640995934207, +STORE, 140640995897344, 140640995905535, +STORE, 140640989896704, 140640993693695, +SNULL, 140640989896704, 140640991555583, +STORE, 140640991555584, 140640993693695, +STORE, 140640989896704, 140640991555583, +SNULL, 140640993652735, 140640993693695, +STORE, 140640991555584, 140640993652735, +STORE, 140640993652736, 140640993693695, +SNULL, 140640993652736, 140640993677311, +STORE, 140640993677312, 140640993693695, +STORE, 140640993652736, 140640993677311, +ERASE, 140640993652736, 140640993677311, +STORE, 140640993652736, 140640993677311, +ERASE, 140640993677312, 140640993693695, +STORE, 140640993677312, 140640993693695, +SNULL, 140640993669119, 140640993677311, +STORE, 140640993652736, 140640993669119, +STORE, 140640993669120, 140640993677311, +SNULL, 94155249078271, 94155249082367, +STORE, 94155249070080, 94155249078271, +STORE, 94155249078272, 94155249082367, +SNULL, 140640995938303, 140640995942399, +STORE, 140640995934208, 140640995938303, +STORE, 140640995938304, 140640995942399, +ERASE, 140640995905536, 140640995934207, +STORE, 94155281035264, 94155281170431, +STORE, 94088066453504, 94088066564095, +STORE, 94088068657152, 94088068665343, +STORE, 94088068665344, 94088068669439, +STORE, 94088068669440, 94088068677631, +STORE, 94088090214400, 94088090349567, +STORE, 140503024627712, 140503026286591, +STORE, 140503026286592, 140503028383743, +STORE, 140503028383744, 140503028400127, +STORE, 140503028400128, 140503028408319, +STORE, 140503028408320, 140503028424703, +STORE, 140503028424704, 140503028568063, +STORE, 140503030628352, 140503030636543, +STORE, 140503030665216, 140503030669311, +STORE, 140503030669312, 140503030673407, +STORE, 140503030673408, 140503030677503, +STORE, 140730894725120, 140730894864383, +STORE, 140730894880768, 140730894893055, +STORE, 140730894893056, 140730894897151, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730434342912, 140737488351231, +SNULL, 140730434351103, 140737488351231, +STORE, 140730434342912, 140730434351103, +STORE, 140730434211840, 140730434351103, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140109041938432, 140109044191231, +SNULL, 140109042081791, 140109044191231, +STORE, 140109041938432, 140109042081791, +STORE, 140109042081792, 140109044191231, +ERASE, 140109042081792, 140109044191231, +STORE, 140109044178944, 140109044187135, +STORE, 140109044187136, 140109044191231, +STORE, 140730434850816, 140730434854911, +STORE, 140730434838528, 140730434850815, +STORE, 140109044150272, 140109044178943, +STORE, 140109044142080, 140109044150271, +STORE, 140109038776320, 140109041938431, +SNULL, 140109038776320, 140109039837183, +STORE, 140109039837184, 140109041938431, +STORE, 140109038776320, 140109039837183, +SNULL, 140109041930239, 140109041938431, +STORE, 140109039837184, 140109041930239, +STORE, 140109041930240, 140109041938431, +ERASE, 140109041930240, 140109041938431, +STORE, 140109041930240, 140109041938431, +STORE, 140109034979328, 140109038776319, +SNULL, 140109034979328, 140109036638207, +STORE, 140109036638208, 140109038776319, +STORE, 140109034979328, 140109036638207, +SNULL, 140109038735359, 140109038776319, +STORE, 140109036638208, 140109038735359, +STORE, 140109038735360, 140109038776319, +SNULL, 140109038735360, 140109038759935, +STORE, 140109038759936, 140109038776319, +STORE, 140109038735360, 140109038759935, +ERASE, 140109038735360, 140109038759935, +STORE, 140109038735360, 140109038759935, +ERASE, 140109038759936, 140109038776319, +STORE, 140109038759936, 140109038776319, +STORE, 140109044129792, 140109044150271, +SNULL, 140109038751743, 140109038759935, +STORE, 140109038735360, 140109038751743, +STORE, 140109038751744, 140109038759935, +SNULL, 140109041934335, 140109041938431, +STORE, 140109041930240, 140109041934335, +STORE, 140109041934336, 140109041938431, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140109044183039, 140109044187135, +STORE, 140109044178944, 140109044183039, +STORE, 140109044183040, 140109044187135, +ERASE, 140109044150272, 140109044178943, +STORE, 20000768, 20135935, +STORE, 20000768, 20283391, +STORE, 140109042446336, 140109044129791, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730853408768, 140737488351231, +SNULL, 140730853416959, 140737488351231, +STORE, 140730853408768, 140730853416959, +STORE, 140730853277696, 140730853416959, +STORE, 94865902977024, 94865905311743, +SNULL, 94865903190015, 94865905311743, +STORE, 94865902977024, 94865903190015, +STORE, 94865903190016, 94865905311743, +ERASE, 94865903190016, 94865905311743, +STORE, 94865905287168, 94865905299455, +STORE, 94865905299456, 94865905311743, +STORE, 139768865738752, 139768867991551, +SNULL, 139768865882111, 139768867991551, +STORE, 139768865738752, 139768865882111, +STORE, 139768865882112, 139768867991551, +ERASE, 139768865882112, 139768867991551, +STORE, 139768867979264, 139768867987455, +STORE, 139768867987456, 139768867991551, +STORE, 140730853957632, 140730853961727, +STORE, 140730853945344, 140730853957631, +STORE, 139768867950592, 139768867979263, +STORE, 139768867942400, 139768867950591, +STORE, 139768863625216, 139768865738751, +SNULL, 139768863625216, 139768863637503, +STORE, 139768863637504, 139768865738751, +STORE, 139768863625216, 139768863637503, +SNULL, 139768865730559, 139768865738751, +STORE, 139768863637504, 139768865730559, +STORE, 139768865730560, 139768865738751, +ERASE, 139768865730560, 139768865738751, +STORE, 139768865730560, 139768865738751, +STORE, 139768859828224, 139768863625215, +SNULL, 139768859828224, 139768861487103, +STORE, 139768861487104, 139768863625215, +STORE, 139768859828224, 139768861487103, +SNULL, 139768863584255, 139768863625215, +STORE, 139768861487104, 139768863584255, +STORE, 139768863584256, 139768863625215, +SNULL, 139768863584256, 139768863608831, +STORE, 139768863608832, 139768863625215, +STORE, 139768863584256, 139768863608831, +ERASE, 139768863584256, 139768863608831, +STORE, 139768863584256, 139768863608831, +ERASE, 139768863608832, 139768863625215, +STORE, 139768863608832, 139768863625215, +STORE, 139768867934208, 139768867950591, +SNULL, 139768863600639, 139768863608831, +STORE, 139768863584256, 139768863600639, +STORE, 139768863600640, 139768863608831, +SNULL, 139768865734655, 139768865738751, +STORE, 139768865730560, 139768865734655, +STORE, 139768865734656, 139768865738751, +SNULL, 94865905291263, 94865905299455, +STORE, 94865905287168, 94865905291263, +STORE, 94865905291264, 94865905299455, +SNULL, 139768867983359, 139768867987455, +STORE, 139768867979264, 139768867983359, +STORE, 139768867983360, 139768867987455, +ERASE, 139768867950592, 139768867979263, +STORE, 94865923670016, 94865923805183, +STORE, 139768866250752, 139768867934207, +STORE, 94865923670016, 94865923940351, +STORE, 94865923670016, 94865924075519, +STORE, 94865923670016, 94865924222975, +SNULL, 94865924210687, 94865924222975, +STORE, 94865923670016, 94865924210687, +STORE, 94865924210688, 94865924222975, +ERASE, 94865924210688, 94865924222975, +STORE, 94865923670016, 94865924349951, +STORE, 94865923670016, 94865924493311, +STORE, 94865923670016, 94865924640767, +SNULL, 94865924603903, 94865924640767, +STORE, 94865923670016, 94865924603903, +STORE, 94865924603904, 94865924640767, +ERASE, 94865924603904, 94865924640767, +STORE, 94865923670016, 94865924747263, +STORE, 94865923670016, 94865924898815, +SNULL, 94865924874239, 94865924898815, +STORE, 94865923670016, 94865924874239, +STORE, 94865924874240, 94865924898815, +ERASE, 94865924874240, 94865924898815, +STORE, 94865923670016, 94865925025791, +SNULL, 94865925013503, 94865925025791, +STORE, 94865923670016, 94865925013503, +STORE, 94865925013504, 94865925025791, +ERASE, 94865925013504, 94865925025791, +SNULL, 94865924988927, 94865925013503, +STORE, 94865923670016, 94865924988927, +STORE, 94865924988928, 94865925013503, +ERASE, 94865924988928, 94865925013503, +STORE, 94865923670016, 94865925152767, +SNULL, 94865925136383, 94865925152767, +STORE, 94865923670016, 94865925136383, +STORE, 94865925136384, 94865925152767, +ERASE, 94865925136384, 94865925152767, +STORE, 94865923670016, 94865925292031, +SNULL, 94865925279743, 94865925292031, +STORE, 94865923670016, 94865925279743, +STORE, 94865925279744, 94865925292031, +ERASE, 94865925279744, 94865925292031, +SNULL, 94865925255167, 94865925279743, +STORE, 94865923670016, 94865925255167, +STORE, 94865925255168, 94865925279743, +ERASE, 94865925255168, 94865925279743, +STORE, 94865923670016, 94865925406719, +SNULL, 94865925394431, 94865925406719, +STORE, 94865923670016, 94865925394431, +STORE, 94865925394432, 94865925406719, +ERASE, 94865925394432, 94865925406719, +STORE, 94865923670016, 94865925545983, +SNULL, 94865925533695, 94865925545983, +STORE, 94865923670016, 94865925533695, +STORE, 94865925533696, 94865925545983, +ERASE, 94865925533696, 94865925545983, +SNULL, 94865925492735, 94865925533695, +STORE, 94865923670016, 94865925492735, +STORE, 94865925492736, 94865925533695, +ERASE, 94865925492736, 94865925533695, +STORE, 94865923670016, 94865925627903, +SNULL, 94865925599231, 94865925627903, +STORE, 94865923670016, 94865925599231, +STORE, 94865925599232, 94865925627903, +ERASE, 94865925599232, 94865925627903, +STORE, 94865923670016, 94865925738495, +SNULL, 94865925726207, 94865925738495, +STORE, 94865923670016, 94865925726207, +STORE, 94865925726208, 94865925738495, +ERASE, 94865925726208, 94865925738495, +STORE, 94865923670016, 94865925877759, +SNULL, 94865925865471, 94865925877759, +STORE, 94865923670016, 94865925865471, +STORE, 94865925865472, 94865925877759, +ERASE, 94865925865472, 94865925877759, +STORE, 94865923670016, 94865926021119, +SNULL, 94865926008831, 94865926021119, +STORE, 94865923670016, 94865926008831, +STORE, 94865926008832, 94865926021119, +ERASE, 94865926008832, 94865926021119, +SNULL, 94865925971967, 94865926008831, +STORE, 94865923670016, 94865925971967, +STORE, 94865925971968, 94865926008831, +ERASE, 94865925971968, 94865926008831, +STORE, 94865923670016, 94865926115327, +STORE, 94865923670016, 94865926254591, +SNULL, 94865926246399, 94865926254591, +STORE, 94865923670016, 94865926246399, +STORE, 94865926246400, 94865926254591, +ERASE, 94865926246400, 94865926254591, +STORE, 94865923670016, 94865926385663, +STORE, 94865923670016, 94865926537215, +STORE, 94865923670016, 94865926672383, +STORE, 94865923670016, 94865926815743, +STORE, 94865923670016, 94865926955007, +STORE, 94865923670016, 94865927094271, +STORE, 94865923670016, 94865927233535, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731148435456, 140737488351231, +SNULL, 140731148443647, 140737488351231, +STORE, 140731148435456, 140731148443647, +STORE, 140731148304384, 140731148443647, +STORE, 94090775400448, 94090777735167, +SNULL, 94090775613439, 94090777735167, +STORE, 94090775400448, 94090775613439, +STORE, 94090775613440, 94090777735167, +ERASE, 94090775613440, 94090777735167, +STORE, 94090777710592, 94090777722879, +STORE, 94090777722880, 94090777735167, +STORE, 140301090283520, 140301092536319, +SNULL, 140301090426879, 140301092536319, +STORE, 140301090283520, 140301090426879, +STORE, 140301090426880, 140301092536319, +ERASE, 140301090426880, 140301092536319, +STORE, 140301092524032, 140301092532223, +STORE, 140301092532224, 140301092536319, +STORE, 140731148570624, 140731148574719, +STORE, 140731148558336, 140731148570623, +STORE, 140301092495360, 140301092524031, +STORE, 140301092487168, 140301092495359, +STORE, 140301088169984, 140301090283519, +SNULL, 140301088169984, 140301088182271, +STORE, 140301088182272, 140301090283519, +STORE, 140301088169984, 140301088182271, +SNULL, 140301090275327, 140301090283519, +STORE, 140301088182272, 140301090275327, +STORE, 140301090275328, 140301090283519, +ERASE, 140301090275328, 140301090283519, +STORE, 140301090275328, 140301090283519, +STORE, 140301084372992, 140301088169983, +SNULL, 140301084372992, 140301086031871, +STORE, 140301086031872, 140301088169983, +STORE, 140301084372992, 140301086031871, +SNULL, 140301088129023, 140301088169983, +STORE, 140301086031872, 140301088129023, +STORE, 140301088129024, 140301088169983, +SNULL, 140301088129024, 140301088153599, +STORE, 140301088153600, 140301088169983, +STORE, 140301088129024, 140301088153599, +ERASE, 140301088129024, 140301088153599, +STORE, 140301088129024, 140301088153599, +ERASE, 140301088153600, 140301088169983, +STORE, 140301088153600, 140301088169983, +STORE, 140301092478976, 140301092495359, +SNULL, 140301088145407, 140301088153599, +STORE, 140301088129024, 140301088145407, +STORE, 140301088145408, 140301088153599, +SNULL, 140301090279423, 140301090283519, +STORE, 140301090275328, 140301090279423, +STORE, 140301090279424, 140301090283519, +SNULL, 94090777714687, 94090777722879, +STORE, 94090777710592, 94090777714687, +STORE, 94090777714688, 94090777722879, +SNULL, 140301092528127, 140301092532223, +STORE, 140301092524032, 140301092528127, +STORE, 140301092528128, 140301092532223, +ERASE, 140301092495360, 140301092524031, +STORE, 94090794590208, 94090794725375, +STORE, 140301090795520, 140301092478975, +STORE, 94090794590208, 94090794860543, +STORE, 94090794590208, 94090794995711, +STORE, 94090794590208, 94090795163647, +SNULL, 94090795139071, 94090795163647, +STORE, 94090794590208, 94090795139071, +STORE, 94090795139072, 94090795163647, +ERASE, 94090795139072, 94090795163647, +STORE, 94090794590208, 94090795278335, +STORE, 94090794590208, 94090795425791, +SNULL, 94090795388927, 94090795425791, +STORE, 94090794590208, 94090795388927, +STORE, 94090795388928, 94090795425791, +ERASE, 94090795388928, 94090795425791, +STORE, 94090794590208, 94090795528191, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733084430336, 140737488351231, +SNULL, 140733084438527, 140737488351231, +STORE, 140733084430336, 140733084438527, +STORE, 140733084299264, 140733084438527, +STORE, 94116169183232, 94116171517951, +SNULL, 94116169396223, 94116171517951, +STORE, 94116169183232, 94116169396223, +STORE, 94116169396224, 94116171517951, +ERASE, 94116169396224, 94116171517951, +STORE, 94116171493376, 94116171505663, +STORE, 94116171505664, 94116171517951, +STORE, 139772214128640, 139772216381439, +SNULL, 139772214271999, 139772216381439, +STORE, 139772214128640, 139772214271999, +STORE, 139772214272000, 139772216381439, +ERASE, 139772214272000, 139772216381439, +STORE, 139772216369152, 139772216377343, +STORE, 139772216377344, 139772216381439, +STORE, 140733085270016, 140733085274111, +STORE, 140733085257728, 140733085270015, +STORE, 139772216340480, 139772216369151, +STORE, 139772216332288, 139772216340479, +STORE, 139772212015104, 139772214128639, +SNULL, 139772212015104, 139772212027391, +STORE, 139772212027392, 139772214128639, +STORE, 139772212015104, 139772212027391, +SNULL, 139772214120447, 139772214128639, +STORE, 139772212027392, 139772214120447, +STORE, 139772214120448, 139772214128639, +ERASE, 139772214120448, 139772214128639, +STORE, 139772214120448, 139772214128639, +STORE, 139772208218112, 139772212015103, +SNULL, 139772208218112, 139772209876991, +STORE, 139772209876992, 139772212015103, +STORE, 139772208218112, 139772209876991, +SNULL, 139772211974143, 139772212015103, +STORE, 139772209876992, 139772211974143, +STORE, 139772211974144, 139772212015103, +SNULL, 139772211974144, 139772211998719, +STORE, 139772211998720, 139772212015103, +STORE, 139772211974144, 139772211998719, +ERASE, 139772211974144, 139772211998719, +STORE, 139772211974144, 139772211998719, +ERASE, 139772211998720, 139772212015103, +STORE, 139772211998720, 139772212015103, +STORE, 139772216324096, 139772216340479, +SNULL, 139772211990527, 139772211998719, +STORE, 139772211974144, 139772211990527, +STORE, 139772211990528, 139772211998719, +SNULL, 139772214124543, 139772214128639, +STORE, 139772214120448, 139772214124543, +STORE, 139772214124544, 139772214128639, +SNULL, 94116171497471, 94116171505663, +STORE, 94116171493376, 94116171497471, +STORE, 94116171497472, 94116171505663, +SNULL, 139772216373247, 139772216377343, +STORE, 139772216369152, 139772216373247, +STORE, 139772216373248, 139772216377343, +ERASE, 139772216340480, 139772216369151, +STORE, 94116199383040, 94116199518207, +STORE, 139772214640640, 139772216324095, +STORE, 94116199383040, 94116199653375, +STORE, 94116199383040, 94116199788543, +STORE, 140737488347136, 140737488351231, +STORE, 140726067826688, 140737488351231, +SNULL, 140726067830783, 140737488351231, +STORE, 140726067826688, 140726067830783, +STORE, 140726067695616, 140726067830783, +STORE, 94535150673920, 94535152898047, +SNULL, 94535150784511, 94535152898047, +STORE, 94535150673920, 94535150784511, +STORE, 94535150784512, 94535152898047, +ERASE, 94535150784512, 94535152898047, +STORE, 94535152877568, 94535152889855, +STORE, 94535152889856, 94535152898047, +STORE, 140381257314304, 140381259567103, +SNULL, 140381257457663, 140381259567103, +STORE, 140381257314304, 140381257457663, +STORE, 140381257457664, 140381259567103, +ERASE, 140381257457664, 140381259567103, +STORE, 140381259554816, 140381259563007, +STORE, 140381259563008, 140381259567103, +STORE, 140726068060160, 140726068064255, +STORE, 140726068047872, 140726068060159, +STORE, 140381259526144, 140381259554815, +STORE, 140381259517952, 140381259526143, +STORE, 140381253517312, 140381257314303, +SNULL, 140381253517312, 140381255176191, +STORE, 140381255176192, 140381257314303, +STORE, 140381253517312, 140381255176191, +SNULL, 140381257273343, 140381257314303, +STORE, 140381255176192, 140381257273343, +STORE, 140381257273344, 140381257314303, +SNULL, 140381257273344, 140381257297919, +STORE, 140381257297920, 140381257314303, +STORE, 140381257273344, 140381257297919, +ERASE, 140381257273344, 140381257297919, +STORE, 140381257273344, 140381257297919, +ERASE, 140381257297920, 140381257314303, +STORE, 140381257297920, 140381257314303, +SNULL, 140381257289727, 140381257297919, +STORE, 140381257273344, 140381257289727, +STORE, 140381257289728, 140381257297919, +SNULL, 94535152885759, 94535152889855, +STORE, 94535152877568, 94535152885759, +STORE, 94535152885760, 94535152889855, +SNULL, 140381259558911, 140381259563007, +STORE, 140381259554816, 140381259558911, +STORE, 140381259558912, 140381259563007, +ERASE, 140381259526144, 140381259554815, +STORE, 94535186296832, 94535186431999, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729189425152, 140737488351231, +SNULL, 140729189433343, 140737488351231, +STORE, 140729189425152, 140729189433343, +STORE, 140729189294080, 140729189433343, +STORE, 94428200128512, 94428202352639, +SNULL, 94428200239103, 94428202352639, +STORE, 94428200128512, 94428200239103, +STORE, 94428200239104, 94428202352639, +ERASE, 94428200239104, 94428202352639, +STORE, 94428202332160, 94428202344447, +STORE, 94428202344448, 94428202352639, +STORE, 139707216986112, 139707219238911, +SNULL, 139707217129471, 139707219238911, +STORE, 139707216986112, 139707217129471, +STORE, 139707217129472, 139707219238911, +ERASE, 139707217129472, 139707219238911, +STORE, 139707219226624, 139707219234815, +STORE, 139707219234816, 139707219238911, +STORE, 140729189785600, 140729189789695, +STORE, 140729189773312, 140729189785599, +STORE, 139707219197952, 139707219226623, +STORE, 139707219189760, 139707219197951, +STORE, 139707213189120, 139707216986111, +SNULL, 139707213189120, 139707214847999, +STORE, 139707214848000, 139707216986111, +STORE, 139707213189120, 139707214847999, +SNULL, 139707216945151, 139707216986111, +STORE, 139707214848000, 139707216945151, +STORE, 139707216945152, 139707216986111, +SNULL, 139707216945152, 139707216969727, +STORE, 139707216969728, 139707216986111, +STORE, 139707216945152, 139707216969727, +ERASE, 139707216945152, 139707216969727, +STORE, 139707216945152, 139707216969727, +ERASE, 139707216969728, 139707216986111, +STORE, 139707216969728, 139707216986111, +SNULL, 139707216961535, 139707216969727, +STORE, 139707216945152, 139707216961535, +STORE, 139707216961536, 139707216969727, +SNULL, 94428202340351, 94428202344447, +STORE, 94428202332160, 94428202340351, +STORE, 94428202340352, 94428202344447, +SNULL, 139707219230719, 139707219234815, +STORE, 139707219226624, 139707219230719, +STORE, 139707219230720, 139707219234815, +ERASE, 139707219197952, 139707219226623, +STORE, 94428208599040, 94428208734207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722000953344, 140737488351231, +SNULL, 140722000961535, 140737488351231, +STORE, 140722000953344, 140722000961535, +STORE, 140722000822272, 140722000961535, +STORE, 94636494757888, 94636496982015, +SNULL, 94636494868479, 94636496982015, +STORE, 94636494757888, 94636494868479, +STORE, 94636494868480, 94636496982015, +ERASE, 94636494868480, 94636496982015, +STORE, 94636496961536, 94636496973823, +STORE, 94636496973824, 94636496982015, +STORE, 140142275100672, 140142277353471, +SNULL, 140142275244031, 140142277353471, +STORE, 140142275100672, 140142275244031, +STORE, 140142275244032, 140142277353471, +ERASE, 140142275244032, 140142277353471, +STORE, 140142277341184, 140142277349375, +STORE, 140142277349376, 140142277353471, +STORE, 140722002747392, 140722002751487, +STORE, 140722002735104, 140722002747391, +STORE, 140142277312512, 140142277341183, +STORE, 140142277304320, 140142277312511, +STORE, 140142271303680, 140142275100671, +SNULL, 140142271303680, 140142272962559, +STORE, 140142272962560, 140142275100671, +STORE, 140142271303680, 140142272962559, +SNULL, 140142275059711, 140142275100671, +STORE, 140142272962560, 140142275059711, +STORE, 140142275059712, 140142275100671, +SNULL, 140142275059712, 140142275084287, +STORE, 140142275084288, 140142275100671, +STORE, 140142275059712, 140142275084287, +ERASE, 140142275059712, 140142275084287, +STORE, 140142275059712, 140142275084287, +ERASE, 140142275084288, 140142275100671, +STORE, 140142275084288, 140142275100671, +SNULL, 140142275076095, 140142275084287, +STORE, 140142275059712, 140142275076095, +STORE, 140142275076096, 140142275084287, +SNULL, 94636496969727, 94636496973823, +STORE, 94636496961536, 94636496969727, +STORE, 94636496969728, 94636496973823, +SNULL, 140142277345279, 140142277349375, +STORE, 140142277341184, 140142277345279, +STORE, 140142277345280, 140142277349375, +ERASE, 140142277312512, 140142277341183, +STORE, 94636516286464, 94636516421631, +STORE, 94071103692800, 94071103905791, +STORE, 94071106002944, 94071106007039, +STORE, 94071106007040, 94071106015231, +STORE, 94071106015232, 94071106027519, +STORE, 94071138521088, 94071140368383, +STORE, 140145668190208, 140145669849087, +STORE, 140145669849088, 140145671946239, +STORE, 140145671946240, 140145671962623, +STORE, 140145671962624, 140145671970815, +STORE, 140145671970816, 140145671987199, +STORE, 140145671987200, 140145671999487, +STORE, 140145671999488, 140145674092543, +STORE, 140145674092544, 140145674096639, +STORE, 140145674096640, 140145674100735, +STORE, 140145674100736, 140145674244095, +STORE, 140145674612736, 140145676296191, +STORE, 140145676296192, 140145676312575, +STORE, 140145676341248, 140145676345343, +STORE, 140145676345344, 140145676349439, +STORE, 140145676349440, 140145676353535, +STORE, 140734927740928, 140734927880191, +STORE, 140734928842752, 140734928855039, +STORE, 140734928855040, 140734928859135, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722342535168, 140737488351231, +SNULL, 140722342543359, 140737488351231, +STORE, 140722342535168, 140722342543359, +STORE, 140722342404096, 140722342543359, +STORE, 94399699714048, 94399702048767, +SNULL, 94399699927039, 94399702048767, +STORE, 94399699714048, 94399699927039, +STORE, 94399699927040, 94399702048767, +ERASE, 94399699927040, 94399702048767, +STORE, 94399702024192, 94399702036479, +STORE, 94399702036480, 94399702048767, +STORE, 139811024748544, 139811027001343, +SNULL, 139811024891903, 139811027001343, +STORE, 139811024748544, 139811024891903, +STORE, 139811024891904, 139811027001343, +ERASE, 139811024891904, 139811027001343, +STORE, 139811026989056, 139811026997247, +STORE, 139811026997248, 139811027001343, +STORE, 140722342707200, 140722342711295, +STORE, 140722342694912, 140722342707199, +STORE, 139811026960384, 139811026989055, +STORE, 139811026952192, 139811026960383, +STORE, 139811022635008, 139811024748543, +SNULL, 139811022635008, 139811022647295, +STORE, 139811022647296, 139811024748543, +STORE, 139811022635008, 139811022647295, +SNULL, 139811024740351, 139811024748543, +STORE, 139811022647296, 139811024740351, +STORE, 139811024740352, 139811024748543, +ERASE, 139811024740352, 139811024748543, +STORE, 139811024740352, 139811024748543, +STORE, 139811018838016, 139811022635007, +SNULL, 139811018838016, 139811020496895, +STORE, 139811020496896, 139811022635007, +STORE, 139811018838016, 139811020496895, +SNULL, 139811022594047, 139811022635007, +STORE, 139811020496896, 139811022594047, +STORE, 139811022594048, 139811022635007, +SNULL, 139811022594048, 139811022618623, +STORE, 139811022618624, 139811022635007, +STORE, 139811022594048, 139811022618623, +ERASE, 139811022594048, 139811022618623, +STORE, 139811022594048, 139811022618623, +ERASE, 139811022618624, 139811022635007, +STORE, 139811022618624, 139811022635007, +STORE, 139811026944000, 139811026960383, +SNULL, 139811022610431, 139811022618623, +STORE, 139811022594048, 139811022610431, +STORE, 139811022610432, 139811022618623, +SNULL, 139811024744447, 139811024748543, +STORE, 139811024740352, 139811024744447, +STORE, 139811024744448, 139811024748543, +SNULL, 94399702028287, 94399702036479, +STORE, 94399702024192, 94399702028287, +STORE, 94399702028288, 94399702036479, +SNULL, 139811026993151, 139811026997247, +STORE, 139811026989056, 139811026993151, +STORE, 139811026993152, 139811026997247, +ERASE, 139811026960384, 139811026989055, +STORE, 94399723880448, 94399724015615, +STORE, 139811025260544, 139811026943999, +STORE, 94399723880448, 94399724150783, +STORE, 94399723880448, 94399724285951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735364939776, 140737488351231, +SNULL, 140735364947967, 140737488351231, +STORE, 140735364939776, 140735364947967, +STORE, 140735364808704, 140735364947967, +STORE, 94421528674304, 94421531009023, +SNULL, 94421528887295, 94421531009023, +STORE, 94421528674304, 94421528887295, +STORE, 94421528887296, 94421531009023, +ERASE, 94421528887296, 94421531009023, +STORE, 94421530984448, 94421530996735, +STORE, 94421530996736, 94421531009023, +STORE, 140162004742144, 140162006994943, +SNULL, 140162004885503, 140162006994943, +STORE, 140162004742144, 140162004885503, +STORE, 140162004885504, 140162006994943, +ERASE, 140162004885504, 140162006994943, +STORE, 140162006982656, 140162006990847, +STORE, 140162006990848, 140162006994943, +STORE, 140735365402624, 140735365406719, +STORE, 140735365390336, 140735365402623, +STORE, 140162006953984, 140162006982655, +STORE, 140162006945792, 140162006953983, +STORE, 140162002628608, 140162004742143, +SNULL, 140162002628608, 140162002640895, +STORE, 140162002640896, 140162004742143, +STORE, 140162002628608, 140162002640895, +SNULL, 140162004733951, 140162004742143, +STORE, 140162002640896, 140162004733951, +STORE, 140162004733952, 140162004742143, +ERASE, 140162004733952, 140162004742143, +STORE, 140162004733952, 140162004742143, +STORE, 140161998831616, 140162002628607, +SNULL, 140161998831616, 140162000490495, +STORE, 140162000490496, 140162002628607, +STORE, 140161998831616, 140162000490495, +SNULL, 140162002587647, 140162002628607, +STORE, 140162000490496, 140162002587647, +STORE, 140162002587648, 140162002628607, +SNULL, 140162002587648, 140162002612223, +STORE, 140162002612224, 140162002628607, +STORE, 140162002587648, 140162002612223, +ERASE, 140162002587648, 140162002612223, +STORE, 140162002587648, 140162002612223, +ERASE, 140162002612224, 140162002628607, +STORE, 140162002612224, 140162002628607, +STORE, 140162006937600, 140162006953983, +SNULL, 140162002604031, 140162002612223, +STORE, 140162002587648, 140162002604031, +STORE, 140162002604032, 140162002612223, +SNULL, 140162004738047, 140162004742143, +STORE, 140162004733952, 140162004738047, +STORE, 140162004738048, 140162004742143, +SNULL, 94421530988543, 94421530996735, +STORE, 94421530984448, 94421530988543, +STORE, 94421530988544, 94421530996735, +SNULL, 140162006986751, 140162006990847, +STORE, 140162006982656, 140162006986751, +STORE, 140162006986752, 140162006990847, +ERASE, 140162006953984, 140162006982655, +STORE, 94421551697920, 94421551833087, +STORE, 140162005254144, 140162006937599, +STORE, 94421551697920, 94421551968255, +STORE, 94421551697920, 94421552103423, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733498486784, 140737488351231, +SNULL, 140733498494975, 140737488351231, +STORE, 140733498486784, 140733498494975, +STORE, 140733498355712, 140733498494975, +STORE, 94567985836032, 94567988170751, +SNULL, 94567986049023, 94567988170751, +STORE, 94567985836032, 94567986049023, +STORE, 94567986049024, 94567988170751, +ERASE, 94567986049024, 94567988170751, +STORE, 94567988146176, 94567988158463, +STORE, 94567988158464, 94567988170751, +STORE, 139634278572032, 139634280824831, +SNULL, 139634278715391, 139634280824831, +STORE, 139634278572032, 139634278715391, +STORE, 139634278715392, 139634280824831, +ERASE, 139634278715392, 139634280824831, +STORE, 139634280812544, 139634280820735, +STORE, 139634280820736, 139634280824831, +STORE, 140733498544128, 140733498548223, +STORE, 140733498531840, 140733498544127, +STORE, 139634280783872, 139634280812543, +STORE, 139634280775680, 139634280783871, +STORE, 139634276458496, 139634278572031, +SNULL, 139634276458496, 139634276470783, +STORE, 139634276470784, 139634278572031, +STORE, 139634276458496, 139634276470783, +SNULL, 139634278563839, 139634278572031, +STORE, 139634276470784, 139634278563839, +STORE, 139634278563840, 139634278572031, +ERASE, 139634278563840, 139634278572031, +STORE, 139634278563840, 139634278572031, +STORE, 139634272661504, 139634276458495, +SNULL, 139634272661504, 139634274320383, +STORE, 139634274320384, 139634276458495, +STORE, 139634272661504, 139634274320383, +SNULL, 139634276417535, 139634276458495, +STORE, 139634274320384, 139634276417535, +STORE, 139634276417536, 139634276458495, +SNULL, 139634276417536, 139634276442111, +STORE, 139634276442112, 139634276458495, +STORE, 139634276417536, 139634276442111, +ERASE, 139634276417536, 139634276442111, +STORE, 139634276417536, 139634276442111, +ERASE, 139634276442112, 139634276458495, +STORE, 139634276442112, 139634276458495, +STORE, 139634280767488, 139634280783871, +SNULL, 139634276433919, 139634276442111, +STORE, 139634276417536, 139634276433919, +STORE, 139634276433920, 139634276442111, +SNULL, 139634278567935, 139634278572031, +STORE, 139634278563840, 139634278567935, +STORE, 139634278567936, 139634278572031, +SNULL, 94567988150271, 94567988158463, +STORE, 94567988146176, 94567988150271, +STORE, 94567988150272, 94567988158463, +SNULL, 139634280816639, 139634280820735, +STORE, 139634280812544, 139634280816639, +STORE, 139634280816640, 139634280820735, +ERASE, 139634280783872, 139634280812543, +STORE, 94567996379136, 94567996514303, +STORE, 139634279084032, 139634280767487, +STORE, 94567996379136, 94567996649471, +STORE, 94567996379136, 94567996784639, +STORE, 94567996379136, 94567996960767, +SNULL, 94567996932095, 94567996960767, +STORE, 94567996379136, 94567996932095, +STORE, 94567996932096, 94567996960767, +ERASE, 94567996932096, 94567996960767, +STORE, 94567996379136, 94567997071359, +STORE, 94567996379136, 94567997206527, +SNULL, 94567997186047, 94567997206527, +STORE, 94567996379136, 94567997186047, +STORE, 94567997186048, 94567997206527, +ERASE, 94567997186048, 94567997206527, +STORE, 94567996379136, 94567997358079, +STORE, 94567996379136, 94567997493247, +SNULL, 94567997476863, 94567997493247, +STORE, 94567996379136, 94567997476863, +STORE, 94567997476864, 94567997493247, +ERASE, 94567997476864, 94567997493247, +STORE, 94567996379136, 94567997612031, +STORE, 94567996379136, 94567997767679, +SNULL, 94567997739007, 94567997767679, +STORE, 94567996379136, 94567997739007, +STORE, 94567997739008, 94567997767679, +ERASE, 94567997739008, 94567997767679, +SNULL, 94567997698047, 94567997739007, +STORE, 94567996379136, 94567997698047, +STORE, 94567997698048, 94567997739007, +ERASE, 94567997698048, 94567997739007, +STORE, 94567996379136, 94567997853695, +STORE, 94567996379136, 94567997988863, +STORE, 94567996379136, 94567998132223, +STORE, 94567996379136, 94567998275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723667759104, 140737488351231, +SNULL, 140723667767295, 140737488351231, +STORE, 140723667759104, 140723667767295, +STORE, 140723667628032, 140723667767295, +STORE, 94231598800896, 94231601135615, +SNULL, 94231599013887, 94231601135615, +STORE, 94231598800896, 94231599013887, +STORE, 94231599013888, 94231601135615, +ERASE, 94231599013888, 94231601135615, +STORE, 94231601111040, 94231601123327, +STORE, 94231601123328, 94231601135615, +STORE, 140269472649216, 140269474902015, +SNULL, 140269472792575, 140269474902015, +STORE, 140269472649216, 140269472792575, +STORE, 140269472792576, 140269474902015, +ERASE, 140269472792576, 140269474902015, +STORE, 140269474889728, 140269474897919, +STORE, 140269474897920, 140269474902015, +STORE, 140723667836928, 140723667841023, +STORE, 140723667824640, 140723667836927, +STORE, 140269474861056, 140269474889727, +STORE, 140269474852864, 140269474861055, +STORE, 140269470535680, 140269472649215, +SNULL, 140269470535680, 140269470547967, +STORE, 140269470547968, 140269472649215, +STORE, 140269470535680, 140269470547967, +SNULL, 140269472641023, 140269472649215, +STORE, 140269470547968, 140269472641023, +STORE, 140269472641024, 140269472649215, +ERASE, 140269472641024, 140269472649215, +STORE, 140269472641024, 140269472649215, +STORE, 140269466738688, 140269470535679, +SNULL, 140269466738688, 140269468397567, +STORE, 140269468397568, 140269470535679, +STORE, 140269466738688, 140269468397567, +SNULL, 140269470494719, 140269470535679, +STORE, 140269468397568, 140269470494719, +STORE, 140269470494720, 140269470535679, +SNULL, 140269470494720, 140269470519295, +STORE, 140269470519296, 140269470535679, +STORE, 140269470494720, 140269470519295, +ERASE, 140269470494720, 140269470519295, +STORE, 140269470494720, 140269470519295, +ERASE, 140269470519296, 140269470535679, +STORE, 140269470519296, 140269470535679, +STORE, 140269474844672, 140269474861055, +SNULL, 140269470511103, 140269470519295, +STORE, 140269470494720, 140269470511103, +STORE, 140269470511104, 140269470519295, +SNULL, 140269472645119, 140269472649215, +STORE, 140269472641024, 140269472645119, +STORE, 140269472645120, 140269472649215, +SNULL, 94231601115135, 94231601123327, +STORE, 94231601111040, 94231601115135, +STORE, 94231601115136, 94231601123327, +SNULL, 140269474893823, 140269474897919, +STORE, 140269474889728, 140269474893823, +STORE, 140269474893824, 140269474897919, +ERASE, 140269474861056, 140269474889727, +STORE, 94231626592256, 94231626727423, +STORE, 140269473161216, 140269474844671, +STORE, 94231626592256, 94231626862591, +STORE, 94231626592256, 94231626997759, +STORE, 94327178862592, 94327179075583, +STORE, 94327181172736, 94327181176831, +STORE, 94327181176832, 94327181185023, +STORE, 94327181185024, 94327181197311, +STORE, 94327185715200, 94327186685951, +STORE, 140172071755776, 140172073414655, +STORE, 140172073414656, 140172075511807, +STORE, 140172075511808, 140172075528191, +STORE, 140172075528192, 140172075536383, +STORE, 140172075536384, 140172075552767, +STORE, 140172075552768, 140172075565055, +STORE, 140172075565056, 140172077658111, +STORE, 140172077658112, 140172077662207, +STORE, 140172077662208, 140172077666303, +STORE, 140172077666304, 140172077809663, +STORE, 140172078178304, 140172079861759, +STORE, 140172079861760, 140172079878143, +STORE, 140172079878144, 140172079906815, +STORE, 140172079906816, 140172079910911, +STORE, 140172079910912, 140172079915007, +STORE, 140172079915008, 140172079919103, +STORE, 140720358359040, 140720358494207, +STORE, 140720358498304, 140720358510591, +STORE, 140720358510592, 140720358514687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722548621312, 140737488351231, +SNULL, 140722548629503, 140737488351231, +STORE, 140722548621312, 140722548629503, +STORE, 140722548490240, 140722548629503, +STORE, 93949289504768, 93949291728895, +SNULL, 93949289615359, 93949291728895, +STORE, 93949289504768, 93949289615359, +STORE, 93949289615360, 93949291728895, +ERASE, 93949289615360, 93949291728895, +STORE, 93949291708416, 93949291720703, +STORE, 93949291720704, 93949291728895, +STORE, 140305861902336, 140305864155135, +SNULL, 140305862045695, 140305864155135, +STORE, 140305861902336, 140305862045695, +STORE, 140305862045696, 140305864155135, +ERASE, 140305862045696, 140305864155135, +STORE, 140305864142848, 140305864151039, +STORE, 140305864151040, 140305864155135, +STORE, 140722549821440, 140722549825535, +STORE, 140722549809152, 140722549821439, +STORE, 140305864114176, 140305864142847, +STORE, 140305864105984, 140305864114175, +STORE, 140305858105344, 140305861902335, +SNULL, 140305858105344, 140305859764223, +STORE, 140305859764224, 140305861902335, +STORE, 140305858105344, 140305859764223, +SNULL, 140305861861375, 140305861902335, +STORE, 140305859764224, 140305861861375, +STORE, 140305861861376, 140305861902335, +SNULL, 140305861861376, 140305861885951, +STORE, 140305861885952, 140305861902335, +STORE, 140305861861376, 140305861885951, +ERASE, 140305861861376, 140305861885951, +STORE, 140305861861376, 140305861885951, +ERASE, 140305861885952, 140305861902335, +STORE, 140305861885952, 140305861902335, +SNULL, 140305861877759, 140305861885951, +STORE, 140305861861376, 140305861877759, +STORE, 140305861877760, 140305861885951, +SNULL, 93949291716607, 93949291720703, +STORE, 93949291708416, 93949291716607, +STORE, 93949291716608, 93949291720703, +SNULL, 140305864146943, 140305864151039, +STORE, 140305864142848, 140305864146943, +STORE, 140305864146944, 140305864151039, +ERASE, 140305864114176, 140305864142847, +STORE, 93949324136448, 93949324271615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725754908672, 140737488351231, +SNULL, 140725754916863, 140737488351231, +STORE, 140725754908672, 140725754916863, +STORE, 140725754777600, 140725754916863, +STORE, 94831184375808, 94831186599935, +SNULL, 94831184486399, 94831186599935, +STORE, 94831184375808, 94831184486399, +STORE, 94831184486400, 94831186599935, +ERASE, 94831184486400, 94831186599935, +STORE, 94831186579456, 94831186591743, +STORE, 94831186591744, 94831186599935, +STORE, 140605482479616, 140605484732415, +SNULL, 140605482622975, 140605484732415, +STORE, 140605482479616, 140605482622975, +STORE, 140605482622976, 140605484732415, +ERASE, 140605482622976, 140605484732415, +STORE, 140605484720128, 140605484728319, +STORE, 140605484728320, 140605484732415, +STORE, 140725755670528, 140725755674623, +STORE, 140725755658240, 140725755670527, +STORE, 140605484691456, 140605484720127, +STORE, 140605484683264, 140605484691455, +STORE, 140605478682624, 140605482479615, +SNULL, 140605478682624, 140605480341503, +STORE, 140605480341504, 140605482479615, +STORE, 140605478682624, 140605480341503, +SNULL, 140605482438655, 140605482479615, +STORE, 140605480341504, 140605482438655, +STORE, 140605482438656, 140605482479615, +SNULL, 140605482438656, 140605482463231, +STORE, 140605482463232, 140605482479615, +STORE, 140605482438656, 140605482463231, +ERASE, 140605482438656, 140605482463231, +STORE, 140605482438656, 140605482463231, +ERASE, 140605482463232, 140605482479615, +STORE, 140605482463232, 140605482479615, +SNULL, 140605482455039, 140605482463231, +STORE, 140605482438656, 140605482455039, +STORE, 140605482455040, 140605482463231, +SNULL, 94831186587647, 94831186591743, +STORE, 94831186579456, 94831186587647, +STORE, 94831186587648, 94831186591743, +SNULL, 140605484724223, 140605484728319, +STORE, 140605484720128, 140605484724223, +STORE, 140605484724224, 140605484728319, +ERASE, 140605484691456, 140605484720127, +STORE, 94831217156096, 94831217291263, +STORE, 94327178862592, 94327179075583, +STORE, 94327181172736, 94327181176831, +STORE, 94327181176832, 94327181185023, +STORE, 94327181185024, 94327181197311, +STORE, 94327185715200, 94327186685951, +STORE, 140172071755776, 140172073414655, +STORE, 140172073414656, 140172075511807, +STORE, 140172075511808, 140172075528191, +STORE, 140172075528192, 140172075536383, +STORE, 140172075536384, 140172075552767, +STORE, 140172075552768, 140172075565055, +STORE, 140172075565056, 140172077658111, +STORE, 140172077658112, 140172077662207, +STORE, 140172077662208, 140172077666303, +STORE, 140172077666304, 140172077809663, +STORE, 140172078178304, 140172079861759, +STORE, 140172079861760, 140172079878143, +STORE, 140172079878144, 140172079906815, +STORE, 140172079906816, 140172079910911, +STORE, 140172079910912, 140172079915007, +STORE, 140172079915008, 140172079919103, +STORE, 140720358359040, 140720358494207, +STORE, 140720358498304, 140720358510591, +STORE, 140720358510592, 140720358514687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140734529933312, 140737488351231, +SNULL, 140734529945599, 140737488351231, +STORE, 140734529933312, 140734529945599, +STORE, 140734529802240, 140734529945599, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140249744060416, 140249746313215, +SNULL, 140249744203775, 140249746313215, +STORE, 140249744060416, 140249744203775, +STORE, 140249744203776, 140249746313215, +ERASE, 140249744203776, 140249746313215, +STORE, 140249746300928, 140249746309119, +STORE, 140249746309120, 140249746313215, +STORE, 140734530174976, 140734530179071, +STORE, 140734530162688, 140734530174975, +STORE, 140249746272256, 140249746300927, +STORE, 140249746264064, 140249746272255, +STORE, 140249740226560, 140249744060415, +SNULL, 140249740226560, 140249741934591, +STORE, 140249741934592, 140249744060415, +STORE, 140249740226560, 140249741934591, +SNULL, 140249744027647, 140249744060415, +STORE, 140249741934592, 140249744027647, +STORE, 140249744027648, 140249744060415, +ERASE, 140249744027648, 140249744060415, +STORE, 140249744027648, 140249744060415, +STORE, 140249738031104, 140249740226559, +SNULL, 140249738031104, 140249738125311, +STORE, 140249738125312, 140249740226559, +STORE, 140249738031104, 140249738125311, +SNULL, 140249740218367, 140249740226559, +STORE, 140249738125312, 140249740218367, +STORE, 140249740218368, 140249740226559, +ERASE, 140249740218368, 140249740226559, +STORE, 140249740218368, 140249740226559, +STORE, 140249735512064, 140249738031103, +SNULL, 140249735512064, 140249735925759, +STORE, 140249735925760, 140249738031103, +STORE, 140249735512064, 140249735925759, +SNULL, 140249738018815, 140249738031103, +STORE, 140249735925760, 140249738018815, +STORE, 140249738018816, 140249738031103, +ERASE, 140249738018816, 140249738031103, +STORE, 140249738018816, 140249738031103, +STORE, 140249732878336, 140249735512063, +SNULL, 140249732878336, 140249733406719, +STORE, 140249733406720, 140249735512063, +STORE, 140249732878336, 140249733406719, +SNULL, 140249735503871, 140249735512063, +STORE, 140249733406720, 140249735503871, +STORE, 140249735503872, 140249735512063, +ERASE, 140249735503872, 140249735512063, +STORE, 140249735503872, 140249735512063, +STORE, 140249730764800, 140249732878335, +SNULL, 140249730764800, 140249730777087, +STORE, 140249730777088, 140249732878335, +STORE, 140249730764800, 140249730777087, +SNULL, 140249732870143, 140249732878335, +STORE, 140249730777088, 140249732870143, +STORE, 140249732870144, 140249732878335, +ERASE, 140249732870144, 140249732878335, +STORE, 140249732870144, 140249732878335, +STORE, 140249728561152, 140249730764799, +SNULL, 140249728561152, 140249728663551, +STORE, 140249728663552, 140249730764799, +STORE, 140249728561152, 140249728663551, +SNULL, 140249730756607, 140249730764799, +STORE, 140249728663552, 140249730756607, +STORE, 140249730756608, 140249730764799, +ERASE, 140249730756608, 140249730764799, +STORE, 140249730756608, 140249730764799, +STORE, 140249746255872, 140249746272255, +STORE, 140249725399040, 140249728561151, +SNULL, 140249725399040, 140249726459903, +STORE, 140249726459904, 140249728561151, +STORE, 140249725399040, 140249726459903, +SNULL, 140249728552959, 140249728561151, +STORE, 140249726459904, 140249728552959, +STORE, 140249728552960, 140249728561151, +ERASE, 140249728552960, 140249728561151, +STORE, 140249728552960, 140249728561151, +STORE, 140249721602048, 140249725399039, +SNULL, 140249721602048, 140249723260927, +STORE, 140249723260928, 140249725399039, +STORE, 140249721602048, 140249723260927, +SNULL, 140249725358079, 140249725399039, +STORE, 140249723260928, 140249725358079, +STORE, 140249725358080, 140249725399039, +SNULL, 140249725358080, 140249725382655, +STORE, 140249725382656, 140249725399039, +STORE, 140249725358080, 140249725382655, +ERASE, 140249725358080, 140249725382655, +STORE, 140249725358080, 140249725382655, +ERASE, 140249725382656, 140249725399039, +STORE, 140249725382656, 140249725399039, +STORE, 140249746243584, 140249746272255, +SNULL, 140249725374463, 140249725382655, +STORE, 140249725358080, 140249725374463, +STORE, 140249725374464, 140249725382655, +SNULL, 140249728557055, 140249728561151, +STORE, 140249728552960, 140249728557055, +STORE, 140249728557056, 140249728561151, +SNULL, 140249730760703, 140249730764799, +STORE, 140249730756608, 140249730760703, +STORE, 140249730760704, 140249730764799, +SNULL, 140249732874239, 140249732878335, +STORE, 140249732870144, 140249732874239, +STORE, 140249732874240, 140249732878335, +SNULL, 140249735507967, 140249735512063, +STORE, 140249735503872, 140249735507967, +STORE, 140249735507968, 140249735512063, +SNULL, 140249738027007, 140249738031103, +STORE, 140249738018816, 140249738027007, +STORE, 140249738027008, 140249738031103, +SNULL, 140249740222463, 140249740226559, +STORE, 140249740218368, 140249740222463, +STORE, 140249740222464, 140249740226559, +SNULL, 140249744031743, 140249744060415, +STORE, 140249744027648, 140249744031743, +STORE, 140249744031744, 140249744060415, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140249746305023, 140249746309119, +STORE, 140249746300928, 140249746305023, +STORE, 140249746305024, 140249746309119, +ERASE, 140249746272256, 140249746300927, +STORE, 33853440, 33988607, +STORE, 140249744560128, 140249746243583, +STORE, 140249746296832, 140249746300927, +STORE, 140249744424960, 140249744560127, +STORE, 33853440, 34131967, +STORE, 140249719504896, 140249721602047, +STORE, 140249746288640, 140249746300927, +STORE, 140249746280448, 140249746300927, +STORE, 140249746243584, 140249746280447, +STORE, 140249744408576, 140249744560127, +STORE, 33853440, 34267135, +STORE, 33853440, 34422783, +STORE, 140249744400384, 140249744560127, +STORE, 140249744392192, 140249744560127, +STORE, 33853440, 34557951, +STORE, 33853440, 34693119, +STORE, 140249744375808, 140249744560127, +STORE, 140249744367616, 140249744560127, +STORE, 33853440, 34832383, +STORE, 140249719230464, 140249721602047, +STORE, 140249744207872, 140249744560127, +STORE, 33853440, 34971647, +SNULL, 34963455, 34971647, +STORE, 33853440, 34963455, +STORE, 34963456, 34971647, +ERASE, 34963456, 34971647, +SNULL, 34955263, 34963455, +STORE, 33853440, 34955263, +STORE, 34955264, 34963455, +ERASE, 34955264, 34963455, +SNULL, 34947071, 34955263, +STORE, 33853440, 34947071, +STORE, 34947072, 34955263, +ERASE, 34947072, 34955263, +SNULL, 34938879, 34947071, +STORE, 33853440, 34938879, +STORE, 34938880, 34947071, +ERASE, 34938880, 34947071, +STORE, 140249719214080, 140249721602047, +STORE, 140249719148544, 140249721602047, +STORE, 140249719115776, 140249721602047, +STORE, 140249717018624, 140249721602047, +STORE, 140249716953088, 140249721602047, +STORE, 33853440, 35086335, +STORE, 140249716822016, 140249721602047, +STORE, 140249716559872, 140249721602047, +STORE, 140249716551680, 140249721602047, +STORE, 140249716535296, 140249721602047, +STORE, 140249716527104, 140249721602047, +STORE, 140249716518912, 140249721602047, +STORE, 33853440, 35221503, +SNULL, 35213311, 35221503, +STORE, 33853440, 35213311, +STORE, 35213312, 35221503, +ERASE, 35213312, 35221503, +SNULL, 35205119, 35213311, +STORE, 33853440, 35205119, +STORE, 35205120, 35213311, +ERASE, 35205120, 35213311, +SNULL, 35192831, 35205119, +STORE, 33853440, 35192831, +STORE, 35192832, 35205119, +ERASE, 35192832, 35205119, +SNULL, 35176447, 35192831, +STORE, 33853440, 35176447, +STORE, 35176448, 35192831, +ERASE, 35176448, 35192831, +STORE, 140249716502528, 140249721602047, +STORE, 33853440, 35311615, +SNULL, 35307519, 35311615, +STORE, 33853440, 35307519, +STORE, 35307520, 35311615, +ERASE, 35307520, 35311615, +SNULL, 35303423, 35307519, +STORE, 33853440, 35303423, +STORE, 35303424, 35307519, +ERASE, 35303424, 35307519, +SNULL, 35299327, 35303423, +STORE, 33853440, 35299327, +STORE, 35299328, 35303423, +ERASE, 35299328, 35303423, +SNULL, 35295231, 35299327, +STORE, 33853440, 35295231, +STORE, 35295232, 35299327, +ERASE, 35295232, 35299327, +SNULL, 35291135, 35295231, +STORE, 33853440, 35291135, +STORE, 35291136, 35295231, +ERASE, 35291136, 35295231, +SNULL, 35287039, 35291135, +STORE, 33853440, 35287039, +STORE, 35287040, 35291135, +ERASE, 35287040, 35291135, +SNULL, 35282943, 35287039, +STORE, 33853440, 35282943, +STORE, 35282944, 35287039, +ERASE, 35282944, 35287039, +STORE, 140249716486144, 140249721602047, +STORE, 140249716453376, 140249721602047, +STORE, 33853440, 35418111, +SNULL, 35401727, 35418111, +STORE, 33853440, 35401727, +STORE, 35401728, 35418111, +ERASE, 35401728, 35418111, +SNULL, 35389439, 35401727, +STORE, 33853440, 35389439, +STORE, 35389440, 35401727, +ERASE, 35389440, 35401727, +STORE, 140249714356224, 140249721602047, +STORE, 33853440, 35540991, +STORE, 140249714339840, 140249721602047, +STORE, 140249714077696, 140249721602047, +STORE, 140249714069504, 140249721602047, +STORE, 140249714061312, 140249721602047, +STORE, 33853440, 35680255, +SNULL, 35672063, 35680255, +STORE, 33853440, 35672063, +STORE, 35672064, 35680255, +ERASE, 35672064, 35680255, +SNULL, 35627007, 35672063, +STORE, 33853440, 35627007, +STORE, 35627008, 35672063, +ERASE, 35627008, 35672063, +STORE, 140249711964160, 140249721602047, +STORE, 33853440, 35762175, +SNULL, 35753983, 35762175, +STORE, 33853440, 35753983, +STORE, 35753984, 35762175, +ERASE, 35753984, 35762175, +SNULL, 35745791, 35753983, +STORE, 33853440, 35745791, +STORE, 35745792, 35753983, +ERASE, 35745792, 35753983, +STORE, 140249711955968, 140249721602047, +STORE, 140249711947776, 140249721602047, +STORE, 140249710899200, 140249721602047, +STORE, 140249710866432, 140249721602047, +STORE, 140249710600192, 140249721602047, +SNULL, 140249744424959, 140249744560127, +STORE, 140249744207872, 140249744424959, +STORE, 140249744424960, 140249744560127, +ERASE, 140249744424960, 140249744560127, +STORE, 140249708503040, 140249721602047, +STORE, 33853440, 35885055, +STORE, 140249707978752, 140249721602047, +STORE, 140249705881600, 140249721602047, +STORE, 33853440, 36036607, +STORE, 33853440, 36175871, +STORE, 140249744551936, 140249744560127, +STORE, 140249744543744, 140249744560127, +STORE, 140249744535552, 140249744560127, +STORE, 140249744527360, 140249744560127, +STORE, 140249744519168, 140249744560127, +STORE, 140249705619456, 140249721602047, +STORE, 140249744510976, 140249744560127, +STORE, 140249744502784, 140249744560127, +STORE, 140249744494592, 140249744560127, +STORE, 140249744486400, 140249744560127, +STORE, 140249744478208, 140249744560127, +STORE, 140249744470016, 140249744560127, +STORE, 140249744461824, 140249744560127, +STORE, 140249744453632, 140249744560127, +STORE, 140249744445440, 140249744560127, +STORE, 140249744437248, 140249744560127, +STORE, 140249744429056, 140249744560127, +STORE, 140249703522304, 140249721602047, +STORE, 33853440, 36311039, +STORE, 140249703489536, 140249721602047, +STORE, 33853440, 36474879, +STORE, 140249703456768, 140249721602047, +STORE, 33853440, 36622335, +STORE, 140249703424000, 140249721602047, +STORE, 140249703391232, 140249721602047, +STORE, 33853440, 36810751, +STORE, 140249703358464, 140249721602047, +STORE, 140249703325696, 140249721602047, +SNULL, 36655103, 36810751, +STORE, 33853440, 36655103, +STORE, 36655104, 36810751, +ERASE, 36655104, 36810751, +SNULL, 36438015, 36655103, +STORE, 33853440, 36438015, +STORE, 36438016, 36655103, +ERASE, 36438016, 36655103, +STORE, 140249703317504, 140249721602047, +STORE, 140249701220352, 140249721602047, +STORE, 33853440, 36585471, +STORE, 33853440, 36782079, +STORE, 140249701212160, 140249721602047, +STORE, 140249701203968, 140249721602047, +STORE, 140249701195776, 140249721602047, +STORE, 140249701187584, 140249721602047, +STORE, 140249701179392, 140249721602047, +STORE, 140249701171200, 140249721602047, +STORE, 140249701163008, 140249721602047, +STORE, 140249701154816, 140249721602047, +STORE, 140249701146624, 140249721602047, +STORE, 140249701138432, 140249721602047, +STORE, 140249701130240, 140249721602047, +STORE, 140249700081664, 140249721602047, +STORE, 140249700073472, 140249721602047, +STORE, 33853440, 36978687, +STORE, 140249697976320, 140249721602047, +STORE, 33853440, 37240831, +STORE, 140249695879168, 140249721602047, +STORE, 140249695870976, 140249721602047, +STORE, 140249695862784, 140249721602047, +STORE, 140249695854592, 140249721602047, +STORE, 140249695326208, 140249721602047, +SNULL, 140249710600191, 140249721602047, +STORE, 140249695326208, 140249710600191, +STORE, 140249710600192, 140249721602047, +SNULL, 140249710600192, 140249710866431, +STORE, 140249710866432, 140249721602047, +STORE, 140249710600192, 140249710866431, +ERASE, 140249710600192, 140249710866431, +STORE, 140249691131904, 140249710600191, +STORE, 33853440, 37474303, +STORE, 140249710858240, 140249721602047, +STORE, 140249710850048, 140249721602047, +STORE, 140249710841856, 140249721602047, +STORE, 140249710833664, 140249721602047, +STORE, 140249710825472, 140249721602047, +STORE, 140249710817280, 140249721602047, +STORE, 140249710809088, 140249721602047, +STORE, 140249710800896, 140249721602047, +STORE, 140249710792704, 140249721602047, +STORE, 140249710784512, 140249721602047, +STORE, 140249710776320, 140249721602047, +STORE, 140249710768128, 140249721602047, +STORE, 140249710759936, 140249721602047, +STORE, 140249710751744, 140249721602047, +STORE, 140249710743552, 140249721602047, +STORE, 140249710735360, 140249721602047, +STORE, 140249689034752, 140249710600191, +STORE, 140249710727168, 140249721602047, +STORE, 140249686937600, 140249710600191, +STORE, 33853440, 37867519, +STORE, 140249684840448, 140249710600191, +STORE, 140249710718976, 140249721602047, +STORE, 140249682743296, 140249710600191, +STORE, 140249710710784, 140249721602047, +STORE, 140249710702592, 140249721602047, +STORE, 140249710694400, 140249721602047, +STORE, 140249710686208, 140249721602047, +STORE, 140249710678016, 140249721602047, +STORE, 140249682612224, 140249710600191, +STORE, 140249682087936, 140249710600191, +SNULL, 140249705619455, 140249710600191, +STORE, 140249682087936, 140249705619455, +STORE, 140249705619456, 140249710600191, +SNULL, 140249705619456, 140249705881599, +STORE, 140249705881600, 140249710600191, +STORE, 140249705619456, 140249705881599, +ERASE, 140249705619456, 140249705881599, +STORE, 140249679990784, 140249705619455, +STORE, 140249710669824, 140249721602047, +STORE, 140249677893632, 140249705619455, +STORE, 140249710653440, 140249721602047, +STORE, 140249710645248, 140249721602047, +STORE, 140249710637056, 140249721602047, +STORE, 140249710628864, 140249721602047, +STORE, 140249710620672, 140249721602047, +STORE, 140249710612480, 140249721602047, +STORE, 140249710604288, 140249721602047, +STORE, 140249705873408, 140249710600191, +STORE, 140249705865216, 140249710600191, +STORE, 140249705857024, 140249710600191, +STORE, 140249705848832, 140249710600191, +STORE, 140249705840640, 140249710600191, +STORE, 140249705832448, 140249710600191, +STORE, 140249705824256, 140249710600191, +STORE, 140249705816064, 140249710600191, +STORE, 140249705807872, 140249710600191, +STORE, 140249705799680, 140249710600191, +STORE, 33853440, 38129663, +SNULL, 140249744207872, 140249744367615, +STORE, 140249744367616, 140249744424959, +STORE, 140249744207872, 140249744367615, +ERASE, 140249744207872, 140249744367615, +STORE, 140249677606912, 140249705619455, +STORE, 140249675509760, 140249705619455, +SNULL, 140249677606911, 140249705619455, +STORE, 140249675509760, 140249677606911, +STORE, 140249677606912, 140249705619455, +SNULL, 140249677606912, 140249677893631, +STORE, 140249677893632, 140249705619455, +STORE, 140249677606912, 140249677893631, +ERASE, 140249677606912, 140249677893631, +STORE, 140249744359424, 140249744424959, +STORE, 33853440, 38391807, +STORE, 140249674981376, 140249677606911, +STORE, 140249672884224, 140249677606911, +SNULL, 140249719230463, 140249721602047, +STORE, 140249710604288, 140249719230463, +STORE, 140249719230464, 140249721602047, +SNULL, 140249719230464, 140249719504895, +STORE, 140249719504896, 140249721602047, +STORE, 140249719230464, 140249719504895, +ERASE, 140249719230464, 140249719504895, +STORE, 140249744351232, 140249744424959, +STORE, 140249744343040, 140249744424959, +STORE, 140249744334848, 140249744424959, +STORE, 140249744326656, 140249744424959, +STORE, 140249744310272, 140249744424959, +STORE, 140249744302080, 140249744424959, +STORE, 140249744285696, 140249744424959, +STORE, 140249744277504, 140249744424959, +STORE, 140249744261120, 140249744424959, +STORE, 140249744252928, 140249744424959, +STORE, 140249744220160, 140249744424959, +STORE, 140249744211968, 140249744424959, +STORE, 140249719488512, 140249721602047, +STORE, 140249744203776, 140249744424959, +STORE, 140249719472128, 140249721602047, +STORE, 140249719463936, 140249721602047, +STORE, 140249719447552, 140249721602047, +STORE, 140249719439360, 140249721602047, +STORE, 140249719406592, 140249721602047, +STORE, 140249719398400, 140249721602047, +STORE, 140249719382016, 140249721602047, +STORE, 140249719373824, 140249721602047, +STORE, 140249719357440, 140249721602047, +STORE, 140249719349248, 140249721602047, +STORE, 140249719332864, 140249721602047, +STORE, 140249719324672, 140249721602047, +STORE, 140249719291904, 140249721602047, +STORE, 140249719283712, 140249721602047, +STORE, 140249719267328, 140249721602047, +STORE, 140249719259136, 140249721602047, +STORE, 140249719242752, 140249721602047, +STORE, 140249719234560, 140249721602047, +STORE, 140249705783296, 140249710600191, +STORE, 140249705775104, 140249710600191, +STORE, 140249705742336, 140249710600191, +STORE, 140249705734144, 140249710600191, +STORE, 140249705717760, 140249710600191, +STORE, 140249670787072, 140249677606911, +STORE, 140249705709568, 140249710600191, +STORE, 140249705693184, 140249710600191, +STORE, 140249705684992, 140249710600191, +STORE, 140249705668608, 140249710600191, +STORE, 140249705660416, 140249710600191, +STORE, 140249705627648, 140249710600191, +STORE, 140249677893632, 140249710600191, +STORE, 140249677877248, 140249710600191, +STORE, 140249677869056, 140249710600191, +STORE, 140249677852672, 140249710600191, +STORE, 140249677844480, 140249710600191, +STORE, 140249677828096, 140249710600191, +STORE, 140249668689920, 140249677606911, +STORE, 140249677819904, 140249710600191, +STORE, 140249677787136, 140249710600191, +STORE, 140249677778944, 140249710600191, +STORE, 140249677762560, 140249710600191, +STORE, 140249677754368, 140249710600191, +STORE, 140249677737984, 140249710600191, +STORE, 140249677729792, 140249710600191, +STORE, 140249677713408, 140249710600191, +STORE, 140249677705216, 140249710600191, +STORE, 140249677672448, 140249710600191, +STORE, 140249677664256, 140249710600191, +STORE, 140249677647872, 140249710600191, +STORE, 140249677639680, 140249710600191, +STORE, 140249677623296, 140249710600191, +STORE, 140249677615104, 140249710600191, +STORE, 140249668673536, 140249677606911, +STORE, 140249668673536, 140249710600191, +STORE, 140249668640768, 140249710600191, +STORE, 140249668632576, 140249710600191, +STORE, 140249668616192, 140249710600191, +STORE, 140249668608000, 140249710600191, +STORE, 140249668591616, 140249710600191, +STORE, 140249668583424, 140249710600191, +STORE, 140249668567040, 140249710600191, +STORE, 140249668558848, 140249710600191, +STORE, 140249668526080, 140249710600191, +STORE, 140249668517888, 140249710600191, +STORE, 140249668501504, 140249710600191, +STORE, 140249668493312, 140249710600191, +STORE, 140249668476928, 140249710600191, +STORE, 140249668468736, 140249710600191, +STORE, 140249668452352, 140249710600191, +STORE, 140249668444160, 140249710600191, +STORE, 140249668411392, 140249710600191, +STORE, 140249668403200, 140249710600191, +STORE, 140249668386816, 140249710600191, +STORE, 140249668378624, 140249710600191, +STORE, 140249668362240, 140249710600191, +STORE, 140249668354048, 140249710600191, +STORE, 140249668337664, 140249710600191, +STORE, 140249668329472, 140249710600191, +STORE, 140249668296704, 140249710600191, +STORE, 140249668288512, 140249710600191, +STORE, 140249668272128, 140249710600191, +STORE, 140249668263936, 140249710600191, +STORE, 140249668247552, 140249710600191, +STORE, 140249668239360, 140249710600191, +STORE, 140249668222976, 140249710600191, +STORE, 140249668214784, 140249710600191, +STORE, 140249668182016, 140249710600191, +STORE, 140249668173824, 140249710600191, +STORE, 140249668157440, 140249710600191, +STORE, 140249668149248, 140249710600191, +STORE, 140249668132864, 140249710600191, +STORE, 140249668124672, 140249710600191, +STORE, 140249668108288, 140249710600191, +STORE, 140249668100096, 140249710600191, +STORE, 140249668067328, 140249710600191, +STORE, 140249668059136, 140249710600191, +STORE, 140249668042752, 140249710600191, +STORE, 140249668034560, 140249710600191, +STORE, 140249668018176, 140249710600191, +STORE, 140249668009984, 140249710600191, +STORE, 140249667993600, 140249710600191, +STORE, 140249667985408, 140249710600191, +STORE, 140249667952640, 140249710600191, +STORE, 140249667944448, 140249710600191, +STORE, 140249667928064, 140249710600191, +STORE, 140249667919872, 140249710600191, +STORE, 140249667903488, 140249710600191, +STORE, 140249667895296, 140249710600191, +STORE, 140249667878912, 140249710600191, +STORE, 140249667870720, 140249710600191, +STORE, 140249667837952, 140249710600191, +STORE, 140249667829760, 140249710600191, +STORE, 140249667813376, 140249710600191, +STORE, 140249667805184, 140249710600191, +STORE, 140249667788800, 140249710600191, +STORE, 140249667780608, 140249710600191, +STORE, 140249667764224, 140249710600191, +STORE, 140249667756032, 140249710600191, +STORE, 140249667723264, 140249710600191, +STORE, 140249667715072, 140249710600191, +STORE, 140249667698688, 140249710600191, +STORE, 140249667690496, 140249710600191, +STORE, 140249667674112, 140249710600191, +STORE, 140249667665920, 140249710600191, +STORE, 140249667649536, 140249710600191, +STORE, 140249667641344, 140249710600191, +STORE, 140249667608576, 140249710600191, +STORE, 140249667600384, 140249710600191, +STORE, 140249667584000, 140249710600191, +STORE, 140249667575808, 140249710600191, +STORE, 140249667559424, 140249710600191, +STORE, 140249667551232, 140249710600191, +STORE, 140249667534848, 140249710600191, +STORE, 140249667526656, 140249710600191, +STORE, 140249667493888, 140249710600191, +STORE, 140249667485696, 140249710600191, +STORE, 140249667469312, 140249710600191, +STORE, 140249667461120, 140249710600191, +STORE, 140249667444736, 140249710600191, +STORE, 140249667436544, 140249710600191, +STORE, 140249667420160, 140249710600191, +STORE, 140249665323008, 140249710600191, +STORE, 140249665314816, 140249710600191, +STORE, 140249665282048, 140249710600191, +STORE, 140249665273856, 140249710600191, +STORE, 140249665257472, 140249710600191, +STORE, 140249665249280, 140249710600191, +STORE, 140249665232896, 140249710600191, +STORE, 140249665224704, 140249710600191, +STORE, 140249665208320, 140249710600191, +STORE, 140249665200128, 140249710600191, +STORE, 140249665167360, 140249710600191, +STORE, 140249665159168, 140249710600191, +STORE, 140249665142784, 140249710600191, +STORE, 140249665134592, 140249710600191, +STORE, 140249665118208, 140249710600191, +STORE, 140249665110016, 140249710600191, +STORE, 140249665093632, 140249710600191, +STORE, 140249665085440, 140249710600191, +STORE, 140249665052672, 140249710600191, +STORE, 140249665044480, 140249710600191, +STORE, 140249665028096, 140249710600191, +STORE, 140249665019904, 140249710600191, +STORE, 140249665003520, 140249710600191, +STORE, 140249664995328, 140249710600191, +STORE, 140249664978944, 140249710600191, +STORE, 140249664970752, 140249710600191, +STORE, 140249664937984, 140249710600191, +STORE, 140249664929792, 140249710600191, +STORE, 140249664913408, 140249710600191, +STORE, 140249664905216, 140249710600191, +STORE, 140249664888832, 140249710600191, +STORE, 140249664880640, 140249710600191, +STORE, 140249664864256, 140249710600191, +STORE, 140249664856064, 140249710600191, +STORE, 140249664823296, 140249710600191, +STORE, 140249664815104, 140249710600191, +STORE, 140249664798720, 140249710600191, +STORE, 140249664790528, 140249710600191, +STORE, 140249664774144, 140249710600191, +STORE, 140249664765952, 140249710600191, +STORE, 140249664749568, 140249710600191, +STORE, 140249664741376, 140249710600191, +STORE, 140249664708608, 140249710600191, +STORE, 140249664700416, 140249710600191, +STORE, 140249664684032, 140249710600191, +STORE, 140249664675840, 140249710600191, +STORE, 140249664659456, 140249710600191, +STORE, 140249664651264, 140249710600191, +STORE, 140249664634880, 140249710600191, +STORE, 140249664626688, 140249710600191, +STORE, 140249664593920, 140249710600191, +STORE, 140249664585728, 140249710600191, +STORE, 140249664569344, 140249710600191, +STORE, 140249664561152, 140249710600191, +STORE, 140249664544768, 140249710600191, +STORE, 140249664536576, 140249710600191, +STORE, 140249664520192, 140249710600191, +STORE, 140249664512000, 140249710600191, +STORE, 140249664479232, 140249710600191, +STORE, 140249664471040, 140249710600191, +STORE, 140249664454656, 140249710600191, +STORE, 140249664446464, 140249710600191, +STORE, 140249664430080, 140249710600191, +STORE, 140249664421888, 140249710600191, +STORE, 140249664405504, 140249710600191, +STORE, 140249664397312, 140249710600191, +STORE, 140249664364544, 140249710600191, +STORE, 140249664356352, 140249710600191, +STORE, 140249664339968, 140249710600191, +STORE, 140249664331776, 140249710600191, +STORE, 140249664315392, 140249710600191, +STORE, 140249664307200, 140249710600191, +STORE, 140249664290816, 140249710600191, +STORE, 140249664282624, 140249710600191, +STORE, 140249664249856, 140249710600191, +STORE, 140249664241664, 140249710600191, +STORE, 140249664225280, 140249710600191, +STORE, 140249664217088, 140249710600191, +STORE, 140249664200704, 140249710600191, +STORE, 140249664192512, 140249710600191, +STORE, 140249664176128, 140249710600191, +STORE, 140249664167936, 140249710600191, +STORE, 140249664135168, 140249710600191, +STORE, 140249664126976, 140249710600191, +STORE, 140249664110592, 140249710600191, +STORE, 140249664102400, 140249710600191, +STORE, 140249664086016, 140249710600191, +STORE, 140249664077824, 140249710600191, +STORE, 140249664061440, 140249710600191, +STORE, 140249664053248, 140249710600191, +STORE, 140249664020480, 140249710600191, +STORE, 140249664012288, 140249710600191, +STORE, 140249663995904, 140249710600191, +STORE, 140249663987712, 140249710600191, +STORE, 140249663971328, 140249710600191, +STORE, 140249663963136, 140249710600191, +STORE, 140249663946752, 140249710600191, +STORE, 140249663938560, 140249710600191, +STORE, 140249663905792, 140249710600191, +STORE, 140249663897600, 140249710600191, +STORE, 140249663881216, 140249710600191, +STORE, 140249663873024, 140249710600191, +STORE, 140249663856640, 140249710600191, +STORE, 140249663848448, 140249710600191, +STORE, 140249663832064, 140249710600191, +STORE, 140249663823872, 140249710600191, +STORE, 140249663791104, 140249710600191, +STORE, 140249663782912, 140249710600191, +STORE, 140249663766528, 140249710600191, +STORE, 140249663758336, 140249710600191, +STORE, 140249663741952, 140249710600191, +STORE, 140249663733760, 140249710600191, +STORE, 140249663717376, 140249710600191, +STORE, 140249663709184, 140249710600191, +STORE, 140249663676416, 140249710600191, +STORE, 140249663668224, 140249710600191, +STORE, 140249663651840, 140249710600191, +STORE, 140249663643648, 140249710600191, +STORE, 140249663627264, 140249710600191, +STORE, 33853440, 38526975, +STORE, 140249663619072, 140249710600191, +STORE, 140249663602688, 140249710600191, +STORE, 140249661505536, 140249710600191, +STORE, 140249661497344, 140249710600191, +STORE, 140249661464576, 140249710600191, +STORE, 140249661456384, 140249710600191, +STORE, 140249661440000, 140249710600191, +STORE, 140249661431808, 140249710600191, +STORE, 140249661415424, 140249710600191, +STORE, 140249661407232, 140249710600191, +STORE, 140249661390848, 140249710600191, +STORE, 140249661382656, 140249710600191, +STORE, 140249661349888, 140249710600191, +STORE, 140249661341696, 140249710600191, +STORE, 140249661325312, 140249710600191, +STORE, 140249661317120, 140249710600191, +STORE, 140249661300736, 140249710600191, +STORE, 140249661292544, 140249710600191, +STORE, 140249661276160, 140249710600191, +STORE, 140249661267968, 140249710600191, +STORE, 140249661235200, 140249710600191, +STORE, 140249661227008, 140249710600191, +STORE, 140249661210624, 140249710600191, +STORE, 140249661202432, 140249710600191, +STORE, 140249661186048, 140249710600191, +STORE, 140249661177856, 140249710600191, +STORE, 140249661161472, 140249710600191, +STORE, 140249661153280, 140249710600191, +STORE, 140249661120512, 140249710600191, +STORE, 140249661112320, 140249710600191, +STORE, 140249661095936, 140249710600191, +STORE, 140249661087744, 140249710600191, +STORE, 140249661071360, 140249710600191, +STORE, 140249661063168, 140249710600191, +STORE, 140249661046784, 140249710600191, +STORE, 140249661038592, 140249710600191, +STORE, 140249661005824, 140249710600191, +STORE, 140249660997632, 140249710600191, +STORE, 140249660981248, 140249710600191, +STORE, 140249660973056, 140249710600191, +STORE, 140249660956672, 140249710600191, +STORE, 140249660948480, 140249710600191, +STORE, 140249660932096, 140249710600191, +STORE, 140249660923904, 140249710600191, +STORE, 140249660891136, 140249710600191, +STORE, 140249660882944, 140249710600191, +STORE, 140249660866560, 140249710600191, +STORE, 140249660858368, 140249710600191, +STORE, 140249660841984, 140249710600191, +STORE, 140249660833792, 140249710600191, +STORE, 140249660817408, 140249710600191, +STORE, 140249660809216, 140249710600191, +STORE, 140249660776448, 140249710600191, +STORE, 140249660768256, 140249710600191, +STORE, 140249660751872, 140249710600191, +STORE, 140249660743680, 140249710600191, +STORE, 140249660727296, 140249710600191, +STORE, 140249660719104, 140249710600191, +STORE, 140249660702720, 140249710600191, +STORE, 140249660694528, 140249710600191, +STORE, 140249660661760, 140249710600191, +STORE, 140249660653568, 140249710600191, +STORE, 140249660637184, 140249710600191, +STORE, 140249660628992, 140249710600191, +STORE, 140249660612608, 140249710600191, +STORE, 140249660604416, 140249710600191, +STORE, 140249660588032, 140249710600191, +STORE, 140249660579840, 140249710600191, +STORE, 140249660547072, 140249710600191, +STORE, 140249660538880, 140249710600191, +STORE, 140249660522496, 140249710600191, +STORE, 140249660514304, 140249710600191, +STORE, 140249660497920, 140249710600191, +STORE, 140249660489728, 140249710600191, +STORE, 140249660473344, 140249710600191, +STORE, 140249660465152, 140249710600191, +STORE, 140249660432384, 140249710600191, +STORE, 140249660424192, 140249710600191, +STORE, 140249660407808, 140249710600191, +STORE, 140249660399616, 140249710600191, +STORE, 140249660383232, 140249710600191, +STORE, 140249660375040, 140249710600191, +STORE, 140249660358656, 140249710600191, +STORE, 140249660350464, 140249710600191, +STORE, 140249660317696, 140249710600191, +STORE, 140249660309504, 140249710600191, +STORE, 140249660293120, 140249710600191, +STORE, 140249660284928, 140249710600191, +STORE, 140249660268544, 140249710600191, +STORE, 140249660260352, 140249710600191, +STORE, 140249660243968, 140249710600191, +STORE, 140249660235776, 140249710600191, +STORE, 140249660203008, 140249710600191, +STORE, 140249660194816, 140249710600191, +STORE, 140249660178432, 140249710600191, +STORE, 140249660170240, 140249710600191, +STORE, 140249660153856, 140249710600191, +STORE, 140249660145664, 140249710600191, +STORE, 140249660129280, 140249710600191, +STORE, 140249660121088, 140249710600191, +STORE, 140249660088320, 140249710600191, +STORE, 140249660080128, 140249710600191, +STORE, 140249660063744, 140249710600191, +STORE, 140249660055552, 140249710600191, +STORE, 140249660039168, 140249710600191, +STORE, 140249660030976, 140249710600191, +STORE, 140249660014592, 140249710600191, +STORE, 140249660006400, 140249710600191, +STORE, 140249659973632, 140249710600191, +STORE, 140249659965440, 140249710600191, +STORE, 140249659949056, 140249710600191, +STORE, 140249659940864, 140249710600191, +STORE, 140249659924480, 140249710600191, +STORE, 140249659916288, 140249710600191, +STORE, 140249659899904, 140249710600191, +STORE, 140249659891712, 140249710600191, +STORE, 140249659858944, 140249710600191, +STORE, 140249659850752, 140249710600191, +STORE, 140249659834368, 140249710600191, +STORE, 140249659826176, 140249710600191, +STORE, 140249659809792, 140249710600191, +STORE, 140249659801600, 140249710600191, +STORE, 140249659785216, 140249710600191, +STORE, 140249657688064, 140249710600191, +STORE, 140249657679872, 140249710600191, +STORE, 140249657647104, 140249710600191, +STORE, 140249657638912, 140249710600191, +STORE, 140249657622528, 140249710600191, +STORE, 140249657614336, 140249710600191, +STORE, 140249657597952, 140249710600191, +STORE, 140249657589760, 140249710600191, +STORE, 140249657573376, 140249710600191, +STORE, 140249657565184, 140249710600191, +STORE, 140249657532416, 140249710600191, +STORE, 140249657524224, 140249710600191, +STORE, 140249657507840, 140249710600191, +STORE, 140249657499648, 140249710600191, +STORE, 140249657483264, 140249710600191, +STORE, 140249657475072, 140249710600191, +STORE, 140249657458688, 140249710600191, +STORE, 140249657450496, 140249710600191, +STORE, 140249657417728, 140249710600191, +STORE, 140249657409536, 140249710600191, +STORE, 140249657393152, 140249710600191, +STORE, 140249657384960, 140249710600191, +STORE, 140249657368576, 140249710600191, +STORE, 140249657360384, 140249710600191, +STORE, 140249657344000, 140249710600191, +STORE, 140249657335808, 140249710600191, +STORE, 140249657303040, 140249710600191, +STORE, 140249657294848, 140249710600191, +STORE, 140249657278464, 140249710600191, +STORE, 140249657270272, 140249710600191, +STORE, 140249657253888, 140249710600191, +STORE, 140249657245696, 140249710600191, +STORE, 140249657229312, 140249710600191, +STORE, 140249657221120, 140249710600191, +STORE, 140249657188352, 140249710600191, +STORE, 140249657180160, 140249710600191, +STORE, 140249657163776, 140249710600191, +STORE, 140249657155584, 140249710600191, +STORE, 140249657139200, 140249710600191, +STORE, 140249657131008, 140249710600191, +STORE, 140249657114624, 140249710600191, +STORE, 140249657106432, 140249710600191, +STORE, 140249657073664, 140249710600191, +STORE, 140249657065472, 140249710600191, +STORE, 140249657049088, 140249710600191, +STORE, 140249657040896, 140249710600191, +STORE, 140249657024512, 140249710600191, +STORE, 140249657016320, 140249710600191, +STORE, 140249656999936, 140249710600191, +STORE, 140249656991744, 140249710600191, +STORE, 140249656958976, 140249710600191, +STORE, 140249656950784, 140249710600191, +STORE, 140249656934400, 140249710600191, +STORE, 140249656926208, 140249710600191, +STORE, 140249656909824, 140249710600191, +STORE, 140249656901632, 140249710600191, +STORE, 140249656885248, 140249710600191, +STORE, 140249656877056, 140249710600191, +STORE, 140249656844288, 140249710600191, +STORE, 140249656836096, 140249710600191, +STORE, 140249656819712, 140249710600191, +STORE, 140249656811520, 140249710600191, +STORE, 140249656795136, 140249710600191, +STORE, 33853440, 38662143, +STORE, 140249656786944, 140249710600191, +STORE, 140249656770560, 140249710600191, +STORE, 140249656762368, 140249710600191, +STORE, 140249656729600, 140249710600191, +STORE, 140249656721408, 140249710600191, +STORE, 140249656705024, 140249710600191, +STORE, 140249656696832, 140249710600191, +STORE, 140249656680448, 140249710600191, +STORE, 140249656672256, 140249710600191, +STORE, 140249656655872, 140249710600191, +STORE, 140249656647680, 140249710600191, +STORE, 140249656614912, 140249710600191, +STORE, 140249656606720, 140249710600191, +STORE, 140249656590336, 140249710600191, +STORE, 140249656582144, 140249710600191, +STORE, 140249656565760, 140249710600191, +STORE, 140249656557568, 140249710600191, +STORE, 140249656541184, 140249710600191, +STORE, 140249656532992, 140249710600191, +STORE, 140249656500224, 140249710600191, +STORE, 140249656492032, 140249710600191, +STORE, 140249656475648, 140249710600191, +STORE, 140249656467456, 140249710600191, +STORE, 140249656451072, 140249710600191, +STORE, 140249656442880, 140249710600191, +STORE, 140249656426496, 140249710600191, +STORE, 140249656418304, 140249710600191, +STORE, 140249656385536, 140249710600191, +STORE, 140249656377344, 140249710600191, +STORE, 140249656360960, 140249710600191, +STORE, 140249656352768, 140249710600191, +STORE, 140249656336384, 140249710600191, +STORE, 140249656328192, 140249710600191, +STORE, 140249656311808, 140249710600191, +STORE, 140249656303616, 140249710600191, +STORE, 140249656270848, 140249710600191, +STORE, 140249656262656, 140249710600191, +STORE, 140249656246272, 140249710600191, +STORE, 140249656238080, 140249710600191, +STORE, 140249656221696, 140249710600191, +STORE, 140249656213504, 140249710600191, +STORE, 140249656197120, 140249710600191, +STORE, 140249656188928, 140249710600191, +STORE, 140249656156160, 140249710600191, +STORE, 140249656147968, 140249710600191, +STORE, 140249656131584, 140249710600191, +STORE, 140249656123392, 140249710600191, +STORE, 140249656107008, 140249710600191, +STORE, 140249656098816, 140249710600191, +STORE, 140249656082432, 140249710600191, +STORE, 140249656074240, 140249710600191, +STORE, 140249656041472, 140249710600191, +STORE, 140249656033280, 140249710600191, +STORE, 140249656016896, 140249710600191, +STORE, 140249656008704, 140249710600191, +STORE, 140249655992320, 140249710600191, +STORE, 140249655984128, 140249710600191, +STORE, 140249655967744, 140249710600191, +STORE, 140249653870592, 140249710600191, +STORE, 140249653862400, 140249710600191, +STORE, 140249653829632, 140249710600191, +STORE, 140249653821440, 140249710600191, +STORE, 140249653805056, 140249710600191, +STORE, 140249653796864, 140249710600191, +STORE, 140249653780480, 140249710600191, +STORE, 140249653772288, 140249710600191, +STORE, 140249653755904, 140249710600191, +STORE, 140249652703232, 140249710600191, +SNULL, 140249682087935, 140249710600191, +STORE, 140249652703232, 140249682087935, +STORE, 140249682087936, 140249710600191, + }; + + unsigned long set26[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140729464770560, 140737488351231, +SNULL, 140729464774655, 140737488351231, +STORE, 140729464770560, 140729464774655, +STORE, 140729464639488, 140729464774655, +STORE, 4194304, 5066751, +STORE, 7159808, 7172095, +STORE, 7172096, 7180287, +STORE, 140729465114624, 140729465118719, +STORE, 140729465102336, 140729465114623, +STORE, 30867456, 30875647, +STORE, 30867456, 31010815, +STORE, 140109040988160, 140109042671615, +STORE, 140109040959488, 140109040988159, +STORE, 140109040943104, 140109040959487, +ERASE, 140109040943104, 140109040959487, +STORE, 140109040840704, 140109040959487, +ERASE, 140109040840704, 140109040959487, +STORE, 140109040951296, 140109040959487, +ERASE, 140109040951296, 140109040959487, +STORE, 140109040955392, 140109040959487, +ERASE, 140109040955392, 140109040959487, + }; + unsigned long set27[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140726128070656, 140737488351231, +SNULL, 140726128074751, 140737488351231, +STORE, 140726128070656, 140726128074751, +STORE, 140726127939584, 140726128074751, +STORE, 94478497189888, 94478499303423, +SNULL, 94478497202175, 94478499303423, +STORE, 94478497189888, 94478497202175, +STORE, 94478497202176, 94478499303423, +ERASE, 94478497202176, 94478499303423, +STORE, 94478499295232, 94478499303423, +STORE, 140415605723136, 140415607975935, +SNULL, 140415605866495, 140415607975935, +STORE, 140415605723136, 140415605866495, +STORE, 140415605866496, 140415607975935, +ERASE, 140415605866496, 140415607975935, +STORE, 140415607963648, 140415607971839, +STORE, 140415607971840, 140415607975935, +STORE, 140726130024448, 140726130028543, +STORE, 140726130012160, 140726130024447, +STORE, 140415607934976, 140415607963647, +STORE, 140415607926784, 140415607934975, +STORE, 140415603245056, 140415605723135, +SNULL, 140415603245056, 140415603613695, +STORE, 140415603613696, 140415605723135, +STORE, 140415603245056, 140415603613695, +SNULL, 140415605710847, 140415605723135, +STORE, 140415603613696, 140415605710847, +STORE, 140415605710848, 140415605723135, +ERASE, 140415605710848, 140415605723135, +STORE, 140415605710848, 140415605723135, +STORE, 140415599370240, 140415603245055, +SNULL, 140415599370240, 140415601111039, +STORE, 140415601111040, 140415603245055, +STORE, 140415599370240, 140415601111039, +SNULL, 140415603208191, 140415603245055, +STORE, 140415601111040, 140415603208191, +STORE, 140415603208192, 140415603245055, +ERASE, 140415603208192, 140415603245055, +STORE, 140415603208192, 140415603245055, +STORE, 140415595692032, 140415599370239, +SNULL, 140415595692032, 140415597207551, +STORE, 140415597207552, 140415599370239, +STORE, 140415595692032, 140415597207551, +SNULL, 140415599304703, 140415599370239, +STORE, 140415597207552, 140415599304703, +STORE, 140415599304704, 140415599370239, +SNULL, 140415599304704, 140415599353855, +STORE, 140415599353856, 140415599370239, +STORE, 140415599304704, 140415599353855, +ERASE, 140415599304704, 140415599353855, +STORE, 140415599304704, 140415599353855, +ERASE, 140415599353856, 140415599370239, +STORE, 140415599353856, 140415599370239, +STORE, 140415593500672, 140415595692031, +SNULL, 140415593500672, 140415593590783, +STORE, 140415593590784, 140415595692031, +STORE, 140415593500672, 140415593590783, +SNULL, 140415595683839, 140415595692031, +STORE, 140415593590784, 140415595683839, +STORE, 140415595683840, 140415595692031, +ERASE, 140415595683840, 140415595692031, +STORE, 140415595683840, 140415595692031, +STORE, 140415589703680, 140415593500671, +SNULL, 140415589703680, 140415591362559, +STORE, 140415591362560, 140415593500671, +STORE, 140415589703680, 140415591362559, +SNULL, 140415593459711, 140415593500671, +STORE, 140415591362560, 140415593459711, +STORE, 140415593459712, 140415593500671, +SNULL, 140415593459712, 140415593484287, +STORE, 140415593484288, 140415593500671, +STORE, 140415593459712, 140415593484287, +ERASE, 140415593459712, 140415593484287, +STORE, 140415593459712, 140415593484287, +ERASE, 140415593484288, 140415593500671, +STORE, 140415593484288, 140415593500671, +STORE, 140415587590144, 140415589703679, +SNULL, 140415587590144, 140415587602431, +STORE, 140415587602432, 140415589703679, +STORE, 140415587590144, 140415587602431, +SNULL, 140415589695487, 140415589703679, +STORE, 140415587602432, 140415589695487, +STORE, 140415589695488, 140415589703679, +ERASE, 140415589695488, 140415589703679, +STORE, 140415589695488, 140415589703679, +STORE, 140415607918592, 140415607934975, +STORE, 140415585398784, 140415587590143, +SNULL, 140415585398784, 140415585480703, +STORE, 140415585480704, 140415587590143, +STORE, 140415585398784, 140415585480703, +SNULL, 140415587573759, 140415587590143, +STORE, 140415585480704, 140415587573759, +STORE, 140415587573760, 140415587590143, +SNULL, 140415587573760, 140415587581951, +STORE, 140415587581952, 140415587590143, +STORE, 140415587573760, 140415587581951, +ERASE, 140415587573760, 140415587581951, +STORE, 140415587573760, 140415587581951, +ERASE, 140415587581952, 140415587590143, +STORE, 140415587581952, 140415587590143, +STORE, 140415583182848, 140415585398783, +SNULL, 140415583182848, 140415583281151, +STORE, 140415583281152, 140415585398783, +STORE, 140415583182848, 140415583281151, +SNULL, 140415585374207, 140415585398783, +STORE, 140415583281152, 140415585374207, +STORE, 140415585374208, 140415585398783, +SNULL, 140415585374208, 140415585382399, +STORE, 140415585382400, 140415585398783, +STORE, 140415585374208, 140415585382399, +ERASE, 140415585374208, 140415585382399, +STORE, 140415585374208, 140415585382399, +ERASE, 140415585382400, 140415585398783, +STORE, 140415585382400, 140415585398783, +STORE, 140415580979200, 140415583182847, +SNULL, 140415580979200, 140415581081599, +STORE, 140415581081600, 140415583182847, +STORE, 140415580979200, 140415581081599, +SNULL, 140415583174655, 140415583182847, +STORE, 140415581081600, 140415583174655, +STORE, 140415583174656, 140415583182847, +ERASE, 140415583174656, 140415583182847, +STORE, 140415583174656, 140415583182847, +STORE, 140415578816512, 140415580979199, +SNULL, 140415578816512, 140415578877951, +STORE, 140415578877952, 140415580979199, +STORE, 140415578816512, 140415578877951, +SNULL, 140415580971007, 140415580979199, +STORE, 140415578877952, 140415580971007, +STORE, 140415580971008, 140415580979199, +ERASE, 140415580971008, 140415580979199, +STORE, 140415580971008, 140415580979199, +STORE, 140415576563712, 140415578816511, +SNULL, 140415576563712, 140415576715263, +STORE, 140415576715264, 140415578816511, +STORE, 140415576563712, 140415576715263, +SNULL, 140415578808319, 140415578816511, +STORE, 140415576715264, 140415578808319, +STORE, 140415578808320, 140415578816511, +ERASE, 140415578808320, 140415578816511, +STORE, 140415578808320, 140415578816511, +STORE, 140415574392832, 140415576563711, +SNULL, 140415574392832, 140415574462463, +STORE, 140415574462464, 140415576563711, +STORE, 140415574392832, 140415574462463, +SNULL, 140415576555519, 140415576563711, +STORE, 140415574462464, 140415576555519, +STORE, 140415576555520, 140415576563711, +ERASE, 140415576555520, 140415576563711, +STORE, 140415576555520, 140415576563711, +STORE, 140415607910400, 140415607934975, +STORE, 140415571230720, 140415574392831, +SNULL, 140415571230720, 140415572291583, +STORE, 140415572291584, 140415574392831, +STORE, 140415571230720, 140415572291583, +SNULL, 140415574384639, 140415574392831, +STORE, 140415572291584, 140415574384639, +STORE, 140415574384640, 140415574392831, +ERASE, 140415574384640, 140415574392831, +STORE, 140415574384640, 140415574392831, +STORE, 140415607902208, 140415607934975, +SNULL, 140415593476095, 140415593484287, +STORE, 140415593459712, 140415593476095, +STORE, 140415593476096, 140415593484287, +SNULL, 140415574388735, 140415574392831, +STORE, 140415574384640, 140415574388735, +STORE, 140415574388736, 140415574392831, +SNULL, 140415576559615, 140415576563711, +STORE, 140415576555520, 140415576559615, +STORE, 140415576559616, 140415576563711, +SNULL, 140415589699583, 140415589703679, +STORE, 140415589695488, 140415589699583, +STORE, 140415589699584, 140415589703679, +SNULL, 140415585378303, 140415585382399, +STORE, 140415585374208, 140415585378303, +STORE, 140415585378304, 140415585382399, +SNULL, 140415578812415, 140415578816511, +STORE, 140415578808320, 140415578812415, +STORE, 140415578812416, 140415578816511, +SNULL, 140415580975103, 140415580979199, +STORE, 140415580971008, 140415580975103, +STORE, 140415580975104, 140415580979199, +SNULL, 140415583178751, 140415583182847, +STORE, 140415583174656, 140415583178751, +STORE, 140415583178752, 140415583182847, +SNULL, 140415587577855, 140415587581951, +STORE, 140415587573760, 140415587577855, +STORE, 140415587577856, 140415587581951, +SNULL, 140415595687935, 140415595692031, +STORE, 140415595683840, 140415595687935, +STORE, 140415595687936, 140415595692031, +STORE, 140415607894016, 140415607934975, +SNULL, 140415599345663, 140415599353855, +STORE, 140415599304704, 140415599345663, +STORE, 140415599345664, 140415599353855, +SNULL, 140415603240959, 140415603245055, +STORE, 140415603208192, 140415603240959, +STORE, 140415603240960, 140415603245055, +SNULL, 140415605719039, 140415605723135, +STORE, 140415605710848, 140415605719039, +STORE, 140415605719040, 140415605723135, +SNULL, 94478499299327, 94478499303423, +STORE, 94478499295232, 94478499299327, +STORE, 94478499299328, 94478499303423, +SNULL, 140415607967743, 140415607971839, +STORE, 140415607963648, 140415607967743, +STORE, 140415607967744, 140415607971839, +ERASE, 140415607934976, 140415607963647, +STORE, 94478511173632, 94478511378431, +STORE, 140415606210560, 140415607894015, +STORE, 140415607934976, 140415607963647, +STORE, 94478511173632, 94478511513599, +STORE, 94478511173632, 94478511648767, +SNULL, 94478511615999, 94478511648767, +STORE, 94478511173632, 94478511615999, +STORE, 94478511616000, 94478511648767, +ERASE, 94478511616000, 94478511648767, +STORE, 94478511173632, 94478511751167, +SNULL, 94478511747071, 94478511751167, +STORE, 94478511173632, 94478511747071, +STORE, 94478511747072, 94478511751167, +ERASE, 94478511747072, 94478511751167, +STORE, 94478511173632, 94478511882239, +SNULL, 94478511878143, 94478511882239, +STORE, 94478511173632, 94478511878143, +STORE, 94478511878144, 94478511882239, +ERASE, 94478511878144, 94478511882239, +STORE, 94478511173632, 94478512013311, +SNULL, 94478512009215, 94478512013311, +STORE, 94478511173632, 94478512009215, +STORE, 94478512009216, 94478512013311, +ERASE, 94478512009216, 94478512013311, +STORE, 94478511173632, 94478512144383, +STORE, 94478511173632, 94478512279551, +STORE, 140415606181888, 140415606210559, +STORE, 140415569100800, 140415571230719, +SNULL, 140415569100800, 140415569129471, +STORE, 140415569129472, 140415571230719, +STORE, 140415569100800, 140415569129471, +SNULL, 140415571222527, 140415571230719, +STORE, 140415569129472, 140415571222527, +STORE, 140415571222528, 140415571230719, +ERASE, 140415571222528, 140415571230719, +STORE, 140415571222528, 140415571230719, +STORE, 140415566905344, 140415569100799, +SNULL, 140415566905344, 140415566987263, +STORE, 140415566987264, 140415569100799, +STORE, 140415566905344, 140415566987263, +SNULL, 140415569084415, 140415569100799, +STORE, 140415566987264, 140415569084415, +STORE, 140415569084416, 140415569100799, +SNULL, 140415569084416, 140415569092607, +STORE, 140415569092608, 140415569100799, +STORE, 140415569084416, 140415569092607, +ERASE, 140415569084416, 140415569092607, +STORE, 140415569084416, 140415569092607, +ERASE, 140415569092608, 140415569100799, +STORE, 140415569092608, 140415569100799, +SNULL, 140415569088511, 140415569092607, +STORE, 140415569084416, 140415569088511, +STORE, 140415569088512, 140415569092607, +SNULL, 140415571226623, 140415571230719, +STORE, 140415571222528, 140415571226623, +STORE, 140415571226624, 140415571230719, +ERASE, 140415606181888, 140415606210559, +STORE, 140415606181888, 140415606210559, +STORE, 140415564759040, 140415566905343, +SNULL, 140415564759040, 140415564804095, +STORE, 140415564804096, 140415566905343, +STORE, 140415564759040, 140415564804095, +SNULL, 140415566897151, 140415566905343, +STORE, 140415564804096, 140415566897151, +STORE, 140415566897152, 140415566905343, +ERASE, 140415566897152, 140415566905343, +STORE, 140415566897152, 140415566905343, +STORE, 140415562588160, 140415564759039, +SNULL, 140415562588160, 140415562629119, +STORE, 140415562629120, 140415564759039, +STORE, 140415562588160, 140415562629119, +SNULL, 140415564726271, 140415564759039, +STORE, 140415562629120, 140415564726271, +STORE, 140415564726272, 140415564759039, +SNULL, 140415564726272, 140415564734463, +STORE, 140415564734464, 140415564759039, +STORE, 140415564726272, 140415564734463, +ERASE, 140415564726272, 140415564734463, +STORE, 140415564726272, 140415564734463, +ERASE, 140415564734464, 140415564759039, +STORE, 140415564734464, 140415564759039, +SNULL, 140415564730367, 140415564734463, +STORE, 140415564726272, 140415564730367, +STORE, 140415564730368, 140415564734463, +SNULL, 140415566901247, 140415566905343, +STORE, 140415566897152, 140415566901247, +STORE, 140415566901248, 140415566905343, +ERASE, 140415606181888, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415605944320, 140415606210559, +ERASE, 140415605944320, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 94478511173632, 94478512414719, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 94478511173632, 94478512652287, +STORE, 94478511173632, 94478512787455, +STORE, 94478511173632, 94478512922623, +STORE, 94478511173632, 94478513057791, +STORE, 140415537422336, 140415562588159, +STORE, 94478511173632, 94478513192959, +STORE, 94478511173632, 94478513356799, +STORE, 94478511173632, 94478513491967, +STORE, 94478511173632, 94478513627135, +STORE, 94478511173632, 94478513790975, +STORE, 94478511173632, 94478513926143, +STORE, 94478511173632, 94478514061311, +STORE, 94478511173632, 94478514196479, +STORE, 94478511173632, 94478514331647, +STORE, 94478511173632, 94478514606079, +STORE, 94478511173632, 94478514741247, +STORE, 94478511173632, 94478514876415, +STORE, 94478511173632, 94478515011583, +STORE, 94478511173632, 94478515146751, +STORE, 94478511173632, 94478515281919, +STORE, 94478511173632, 94478515474431, +STORE, 94478511173632, 94478515609599, +STORE, 94478511173632, 94478515744767, +STORE, 140415536922624, 140415562588159, +STORE, 94478511173632, 94478515879935, +STORE, 94478511173632, 94478516015103, +STORE, 94478511173632, 94478516150271, +STORE, 94478511173632, 94478516285439, +STORE, 94478511173632, 94478516420607, +STORE, 94478511173632, 94478516555775, +STORE, 94478511173632, 94478516690943, +STORE, 94478511173632, 94478516826111, +STORE, 94478511173632, 94478516961279, +STORE, 94478511173632, 94478517231615, +STORE, 94478511173632, 94478517366783, +STORE, 94478511173632, 94478517501951, +STORE, 94478511173632, 94478517637119, +STORE, 94478511173632, 94478517772287, +STORE, 94478511173632, 94478517907455, +STORE, 94478511173632, 94478518042623, +STORE, 94478511173632, 94478518177791, +STORE, 94478511173632, 94478518312959, +STORE, 94478511173632, 94478518448127, +STORE, 140415535910912, 140415562588159, +SNULL, 140415536922623, 140415562588159, +STORE, 140415535910912, 140415536922623, +STORE, 140415536922624, 140415562588159, +SNULL, 140415536922624, 140415537422335, +STORE, 140415537422336, 140415562588159, +STORE, 140415536922624, 140415537422335, +ERASE, 140415536922624, 140415537422335, +STORE, 94478511173632, 94478518583295, +STORE, 94478511173632, 94478518718463, +STORE, 94478511173632, 94478518853631, +STORE, 94478511173632, 94478518988799, +STORE, 94478511173632, 94478519123967, +STORE, 94478511173632, 94478519259135, +STORE, 140415509696512, 140415535910911, +ERASE, 140415537422336, 140415562588159, +STORE, 140415482433536, 140415509696511, + }; + unsigned long set28[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722475622400, 140737488351231, +SNULL, 140722475626495, 140737488351231, +STORE, 140722475622400, 140722475626495, +STORE, 140722475491328, 140722475626495, +STORE, 93865834291200, 93865836548095, +SNULL, 93865834422271, 93865836548095, +STORE, 93865834291200, 93865834422271, +STORE, 93865834422272, 93865836548095, +ERASE, 93865834422272, 93865836548095, +STORE, 93865836519424, 93865836527615, +STORE, 93865836527616, 93865836548095, +STORE, 139918411104256, 139918413357055, +SNULL, 139918411247615, 139918413357055, +STORE, 139918411104256, 139918411247615, +STORE, 139918411247616, 139918413357055, +ERASE, 139918411247616, 139918413357055, +STORE, 139918413344768, 139918413352959, +STORE, 139918413352960, 139918413357055, +STORE, 140722476642304, 140722476646399, +STORE, 140722476630016, 140722476642303, +STORE, 139918413316096, 139918413344767, +STORE, 139918413307904, 139918413316095, +STORE, 139918408888320, 139918411104255, +SNULL, 139918408888320, 139918408986623, +STORE, 139918408986624, 139918411104255, +STORE, 139918408888320, 139918408986623, +SNULL, 139918411079679, 139918411104255, +STORE, 139918408986624, 139918411079679, +STORE, 139918411079680, 139918411104255, +SNULL, 139918411079680, 139918411087871, +STORE, 139918411087872, 139918411104255, +STORE, 139918411079680, 139918411087871, +ERASE, 139918411079680, 139918411087871, +STORE, 139918411079680, 139918411087871, +ERASE, 139918411087872, 139918411104255, +STORE, 139918411087872, 139918411104255, +STORE, 139918405091328, 139918408888319, +SNULL, 139918405091328, 139918406750207, +STORE, 139918406750208, 139918408888319, +STORE, 139918405091328, 139918406750207, +SNULL, 139918408847359, 139918408888319, +STORE, 139918406750208, 139918408847359, +STORE, 139918408847360, 139918408888319, +SNULL, 139918408847360, 139918408871935, +STORE, 139918408871936, 139918408888319, +STORE, 139918408847360, 139918408871935, +ERASE, 139918408847360, 139918408871935, +STORE, 139918408847360, 139918408871935, +ERASE, 139918408871936, 139918408888319, +STORE, 139918408871936, 139918408888319, +STORE, 139918413299712, 139918413316095, +SNULL, 139918408863743, 139918408871935, +STORE, 139918408847360, 139918408863743, +STORE, 139918408863744, 139918408871935, +SNULL, 139918411083775, 139918411087871, +STORE, 139918411079680, 139918411083775, +STORE, 139918411083776, 139918411087871, +SNULL, 93865836523519, 93865836527615, +STORE, 93865836519424, 93865836523519, +STORE, 93865836523520, 93865836527615, +SNULL, 139918413348863, 139918413352959, +STORE, 139918413344768, 139918413348863, +STORE, 139918413348864, 139918413352959, +ERASE, 139918413316096, 139918413344767, +STORE, 93865848528896, 93865848664063, + }; + unsigned long set29[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734467944448, 140737488351231, +SNULL, 140734467948543, 140737488351231, +STORE, 140734467944448, 140734467948543, +STORE, 140734467813376, 140734467948543, +STORE, 94880407924736, 94880410177535, +SNULL, 94880408055807, 94880410177535, +STORE, 94880407924736, 94880408055807, +STORE, 94880408055808, 94880410177535, +ERASE, 94880408055808, 94880410177535, +STORE, 94880410148864, 94880410157055, +STORE, 94880410157056, 94880410177535, +STORE, 140143367815168, 140143370067967, +SNULL, 140143367958527, 140143370067967, +STORE, 140143367815168, 140143367958527, +STORE, 140143367958528, 140143370067967, +ERASE, 140143367958528, 140143370067967, +STORE, 140143370055680, 140143370063871, +STORE, 140143370063872, 140143370067967, +STORE, 140734468329472, 140734468333567, +STORE, 140734468317184, 140734468329471, +STORE, 140143370027008, 140143370055679, +STORE, 140143370018816, 140143370027007, +STORE, 140143365599232, 140143367815167, +SNULL, 140143365599232, 140143365697535, +STORE, 140143365697536, 140143367815167, +STORE, 140143365599232, 140143365697535, +SNULL, 140143367790591, 140143367815167, +STORE, 140143365697536, 140143367790591, +STORE, 140143367790592, 140143367815167, +SNULL, 140143367790592, 140143367798783, +STORE, 140143367798784, 140143367815167, +STORE, 140143367790592, 140143367798783, +ERASE, 140143367790592, 140143367798783, +STORE, 140143367790592, 140143367798783, +ERASE, 140143367798784, 140143367815167, +STORE, 140143367798784, 140143367815167, +STORE, 140143361802240, 140143365599231, +SNULL, 140143361802240, 140143363461119, +STORE, 140143363461120, 140143365599231, +STORE, 140143361802240, 140143363461119, +SNULL, 140143365558271, 140143365599231, +STORE, 140143363461120, 140143365558271, +STORE, 140143365558272, 140143365599231, +SNULL, 140143365558272, 140143365582847, +STORE, 140143365582848, 140143365599231, +STORE, 140143365558272, 140143365582847, +ERASE, 140143365558272, 140143365582847, +STORE, 140143365558272, 140143365582847, +ERASE, 140143365582848, 140143365599231, +STORE, 140143365582848, 140143365599231, +STORE, 140143370010624, 140143370027007, +SNULL, 140143365574655, 140143365582847, +STORE, 140143365558272, 140143365574655, +STORE, 140143365574656, 140143365582847, +SNULL, 140143367794687, 140143367798783, +STORE, 140143367790592, 140143367794687, +STORE, 140143367794688, 140143367798783, +SNULL, 94880410152959, 94880410157055, +STORE, 94880410148864, 94880410152959, +STORE, 94880410152960, 94880410157055, +SNULL, 140143370059775, 140143370063871, +STORE, 140143370055680, 140143370059775, +STORE, 140143370059776, 140143370063871, +ERASE, 140143370027008, 140143370055679, +STORE, 94880442400768, 94880442535935, +STORE, 140143353409536, 140143361802239, +SNULL, 140143353413631, 140143361802239, +STORE, 140143353409536, 140143353413631, +STORE, 140143353413632, 140143361802239, +STORE, 140143345016832, 140143353409535, +STORE, 140143210799104, 140143345016831, +SNULL, 140143210799104, 140143239364607, +STORE, 140143239364608, 140143345016831, +STORE, 140143210799104, 140143239364607, +ERASE, 140143210799104, 140143239364607, +SNULL, 140143306473471, 140143345016831, +STORE, 140143239364608, 140143306473471, +STORE, 140143306473472, 140143345016831, +ERASE, 140143306473472, 140143345016831, +SNULL, 140143239499775, 140143306473471, +STORE, 140143239364608, 140143239499775, +STORE, 140143239499776, 140143306473471, +SNULL, 140143345020927, 140143353409535, +STORE, 140143345016832, 140143345020927, +STORE, 140143345020928, 140143353409535, +STORE, 140143336624128, 140143345016831, +SNULL, 140143336628223, 140143345016831, +STORE, 140143336624128, 140143336628223, +STORE, 140143336628224, 140143345016831, +STORE, 140143328231424, 140143336624127, +SNULL, 140143328235519, 140143336624127, +STORE, 140143328231424, 140143328235519, +STORE, 140143328235520, 140143336624127, +STORE, 140143319838720, 140143328231423, +SNULL, 140143319842815, 140143328231423, +STORE, 140143319838720, 140143319842815, +STORE, 140143319842816, 140143328231423, +STORE, 140143311446016, 140143319838719, +STORE, 140143105146880, 140143239364607, +STORE, 140143096754176, 140143105146879, +STORE, 140143029645312, 140143096754175, +ERASE, 140143029645312, 140143096754175, +STORE, 140142962536448, 140143096754175, +SNULL, 140142962536448, 140142970929151, +STORE, 140142970929152, 140143096754175, +STORE, 140142962536448, 140142970929151, +ERASE, 140142962536448, 140142970929151, +STORE, 140142962536448, 140142970929151, +STORE, 140142828318720, 140142962536447, +STORE, 140142819926016, 140142828318719, +SNULL, 140142828318720, 140142836711423, +STORE, 140142836711424, 140142962536447, +STORE, 140142828318720, 140142836711423, +ERASE, 140142828318720, 140142836711423, +SNULL, 140143172255743, 140143239364607, +STORE, 140143105146880, 140143172255743, +STORE, 140143172255744, 140143239364607, +ERASE, 140143172255744, 140143239364607, +SNULL, 140143105282047, 140143172255743, +STORE, 140143105146880, 140143105282047, +STORE, 140143105282048, 140143172255743, +SNULL, 140143038038015, 140143096754175, +STORE, 140142970929152, 140143038038015, +STORE, 140143038038016, 140143096754175, +ERASE, 140143038038016, 140143096754175, +SNULL, 140142971064319, 140143038038015, +STORE, 140142970929152, 140142971064319, +STORE, 140142971064320, 140143038038015, +SNULL, 140142903820287, 140142962536447, +STORE, 140142836711424, 140142903820287, +STORE, 140142903820288, 140142962536447, +ERASE, 140142903820288, 140142962536447, +SNULL, 140142836846591, 140142903820287, +STORE, 140142836711424, 140142836846591, +STORE, 140142836846592, 140142903820287, +STORE, 140142685708288, 140142819926015, +SNULL, 140143311450111, 140143319838719, +STORE, 140143311446016, 140143311450111, +STORE, 140143311450112, 140143319838719, +SNULL, 140142962540543, 140142970929151, +STORE, 140142962536448, 140142962540543, +STORE, 140142962540544, 140142970929151, +SNULL, 140142685708288, 140142702493695, +STORE, 140142702493696, 140142819926015, +STORE, 140142685708288, 140142702493695, +ERASE, 140142685708288, 140142702493695, +SNULL, 140142769602559, 140142819926015, +STORE, 140142702493696, 140142769602559, +STORE, 140142769602560, 140142819926015, +ERASE, 140142769602560, 140142819926015, +SNULL, 140142702628863, 140142769602559, +STORE, 140142702493696, 140142702628863, +STORE, 140142702628864, 140142769602559, +STORE, 140143230971904, 140143239364607, +SNULL, 140143230975999, 140143239364607, +STORE, 140143230971904, 140143230975999, +STORE, 140143230976000, 140143239364607, +SNULL, 140143096758271, 140143105146879, +STORE, 140143096754176, 140143096758271, +STORE, 140143096758272, 140143105146879, +STORE, 140143222579200, 140143230971903, +SNULL, 140143222583295, 140143230971903, +STORE, 140143222579200, 140143222583295, +STORE, 140143222583296, 140143230971903, +STORE, 140143214186496, 140143222579199, +SNULL, 140142819930111, 140142828318719, +STORE, 140142819926016, 140142819930111, +STORE, 140142819930112, 140142828318719, +STORE, 140143205793792, 140143222579199, +SNULL, 140143205793792, 140143214186495, +STORE, 140143214186496, 140143222579199, +STORE, 140143205793792, 140143214186495, +SNULL, 140143214190591, 140143222579199, +STORE, 140143214186496, 140143214190591, +STORE, 140143214190592, 140143222579199, +SNULL, 140143205797887, 140143214186495, +STORE, 140143205793792, 140143205797887, +STORE, 140143205797888, 140143214186495, +STORE, 140143197401088, 140143205793791, +SNULL, 140143197405183, 140143205793791, +STORE, 140143197401088, 140143197405183, +STORE, 140143197405184, 140143205793791, +STORE, 140143189008384, 140143197401087, +STORE, 140143180615680, 140143197401087, +STORE, 140143088361472, 140143096754175, +SNULL, 140143180619775, 140143197401087, +STORE, 140143180615680, 140143180619775, +STORE, 140143180619776, 140143197401087, +SNULL, 140143180619776, 140143189008383, +STORE, 140143189008384, 140143197401087, +STORE, 140143180619776, 140143189008383, +SNULL, 140143189012479, 140143197401087, +STORE, 140143189008384, 140143189012479, +STORE, 140143189012480, 140143197401087, +SNULL, 140143088365567, 140143096754175, +STORE, 140143088361472, 140143088365567, +STORE, 140143088365568, 140143096754175, +STORE, 140143079968768, 140143088361471, +SNULL, 140143079972863, 140143088361471, +STORE, 140143079968768, 140143079972863, +STORE, 140143079972864, 140143088361471, +STORE, 140143071576064, 140143079968767, +SNULL, 140143071580159, 140143079968767, +STORE, 140143071576064, 140143071580159, +STORE, 140143071580160, 140143079968767, +STORE, 140143063183360, 140143071576063, +STORE, 140143054790656, 140143071576063, +SNULL, 140143054794751, 140143071576063, +STORE, 140143054790656, 140143054794751, +STORE, 140143054794752, 140143071576063, +SNULL, 140143054794752, 140143063183359, +STORE, 140143063183360, 140143071576063, +STORE, 140143054794752, 140143063183359, +SNULL, 140143063187455, 140143071576063, +STORE, 140143063183360, 140143063187455, +STORE, 140143063187456, 140143071576063, +STORE, 140143046397952, 140143054790655, +STORE, 140142954143744, 140142962536447, +STORE, 140142945751040, 140142962536447, +STORE, 140142937358336, 140142962536447, +STORE, 140142928965632, 140142962536447, +STORE, 140142568275968, 140142702493695, +SNULL, 140142635384831, 140142702493695, +STORE, 140142568275968, 140142635384831, +STORE, 140142635384832, 140142702493695, +ERASE, 140142635384832, 140142702493695, +STORE, 140142920572928, 140142962536447, +STORE, 140142912180224, 140142962536447, +STORE, 140142568275968, 140142702493695, +SNULL, 140142568275968, 140142635384831, +STORE, 140142635384832, 140142702493695, +STORE, 140142568275968, 140142635384831, +SNULL, 140142635519999, 140142702493695, +STORE, 140142635384832, 140142635519999, +STORE, 140142635520000, 140142702493695, +STORE, 140142819930112, 140142836711423, +STORE, 140142811533312, 140142819926015, +STORE, 140142434058240, 140142635384831, +SNULL, 140142501167103, 140142635384831, +STORE, 140142434058240, 140142501167103, +STORE, 140142501167104, 140142635384831, +SNULL, 140142501167104, 140142568275967, +STORE, 140142568275968, 140142635384831, +STORE, 140142501167104, 140142568275967, +ERASE, 140142501167104, 140142568275967, +STORE, 140142299840512, 140142501167103, +STORE, 140142803140608, 140142819926015, +SNULL, 140142366949375, 140142501167103, +STORE, 140142299840512, 140142366949375, +STORE, 140142366949376, 140142501167103, +SNULL, 140142366949376, 140142434058239, +STORE, 140142434058240, 140142501167103, +STORE, 140142366949376, 140142434058239, +ERASE, 140142366949376, 140142434058239, +STORE, 140142794747904, 140142819926015, +STORE, 140142786355200, 140142819926015, +STORE, 140142299840512, 140142501167103, +STORE, 140142777962496, 140142819926015, +STORE, 140142559883264, 140142568275967, +STORE, 140142232731648, 140142501167103, +STORE, 140142551490560, 140142568275967, +SNULL, 140142777962496, 140142803140607, +STORE, 140142803140608, 140142819926015, +STORE, 140142777962496, 140142803140607, +SNULL, 140142803144703, 140142819926015, +STORE, 140142803140608, 140142803144703, +STORE, 140142803144704, 140142819926015, +STORE, 140142543097856, 140142568275967, +STORE, 140142098513920, 140142501167103, +SNULL, 140142165622783, 140142501167103, +STORE, 140142098513920, 140142165622783, +STORE, 140142165622784, 140142501167103, +SNULL, 140142165622784, 140142232731647, +STORE, 140142232731648, 140142501167103, +STORE, 140142165622784, 140142232731647, +ERASE, 140142165622784, 140142232731647, +SNULL, 140142568411135, 140142635384831, +STORE, 140142568275968, 140142568411135, +STORE, 140142568411136, 140142635384831, +STORE, 140141964296192, 140142165622783, +SNULL, 140142912180224, 140142928965631, +STORE, 140142928965632, 140142962536447, +STORE, 140142912180224, 140142928965631, +SNULL, 140142928969727, 140142962536447, +STORE, 140142928965632, 140142928969727, +STORE, 140142928969728, 140142962536447, +STORE, 140141830078464, 140142165622783, +SNULL, 140142912184319, 140142928965631, +STORE, 140142912180224, 140142912184319, +STORE, 140142912184320, 140142928965631, +SNULL, 140142232731648, 140142434058239, +STORE, 140142434058240, 140142501167103, +STORE, 140142232731648, 140142434058239, +SNULL, 140142434193407, 140142501167103, +STORE, 140142434058240, 140142434193407, +STORE, 140142434193408, 140142501167103, +SNULL, 140142232731648, 140142299840511, +STORE, 140142299840512, 140142434058239, +STORE, 140142232731648, 140142299840511, +SNULL, 140142299975679, 140142434058239, +STORE, 140142299840512, 140142299975679, +STORE, 140142299975680, 140142434058239, +SNULL, 140142928969728, 140142954143743, +STORE, 140142954143744, 140142962536447, +STORE, 140142928969728, 140142954143743, +SNULL, 140142954147839, 140142962536447, +STORE, 140142954143744, 140142954147839, +STORE, 140142954147840, 140142962536447, +STORE, 140141830078464, 140142299840511, +SNULL, 140142543097856, 140142559883263, +STORE, 140142559883264, 140142568275967, +STORE, 140142543097856, 140142559883263, +SNULL, 140142559887359, 140142568275967, +STORE, 140142559883264, 140142559887359, +STORE, 140142559887360, 140142568275967, +STORE, 140142534705152, 140142559883263, +SNULL, 140142928969728, 140142945751039, +STORE, 140142945751040, 140142954143743, +STORE, 140142928969728, 140142945751039, +SNULL, 140142945755135, 140142954143743, +STORE, 140142945751040, 140142945755135, +STORE, 140142945755136, 140142954143743, +SNULL, 140142299975680, 140142366949375, +STORE, 140142366949376, 140142434058239, +STORE, 140142299975680, 140142366949375, +SNULL, 140142367084543, 140142434058239, +STORE, 140142366949376, 140142367084543, +STORE, 140142367084544, 140142434058239, +SNULL, 140142928969728, 140142937358335, +STORE, 140142937358336, 140142945751039, +STORE, 140142928969728, 140142937358335, +SNULL, 140142937362431, 140142945751039, +STORE, 140142937358336, 140142937362431, +STORE, 140142937362432, 140142945751039, +SNULL, 140141830078464, 140142232731647, +STORE, 140142232731648, 140142299840511, +STORE, 140141830078464, 140142232731647, +SNULL, 140142232866815, 140142299840511, +STORE, 140142232731648, 140142232866815, +STORE, 140142232866816, 140142299840511, +SNULL, 140142534705152, 140142543097855, +STORE, 140142543097856, 140142559883263, +STORE, 140142534705152, 140142543097855, +SNULL, 140142543101951, 140142559883263, +STORE, 140142543097856, 140142543101951, +STORE, 140142543101952, 140142559883263, +STORE, 140142526312448, 140142543097855, +STORE, 140142517919744, 140142543097855, +SNULL, 140141830078464, 140142098513919, +STORE, 140142098513920, 140142232731647, +STORE, 140141830078464, 140142098513919, +SNULL, 140142098649087, 140142232731647, +STORE, 140142098513920, 140142098649087, +STORE, 140142098649088, 140142232731647, +SNULL, 140142031405055, 140142098513919, +STORE, 140141830078464, 140142031405055, +STORE, 140142031405056, 140142098513919, +ERASE, 140142031405056, 140142098513919, +SNULL, 140141830078464, 140141964296191, +STORE, 140141964296192, 140142031405055, +STORE, 140141830078464, 140141964296191, +SNULL, 140141964431359, 140142031405055, +STORE, 140141964296192, 140141964431359, +STORE, 140141964431360, 140142031405055, +STORE, 140142509527040, 140142543097855, +SNULL, 140141897187327, 140141964296191, +STORE, 140141830078464, 140141897187327, +STORE, 140141897187328, 140141964296191, +ERASE, 140141897187328, 140141964296191, +SNULL, 140141830213631, 140141897187327, +STORE, 140141830078464, 140141830213631, +STORE, 140141830213632, 140141897187327, +SNULL, 140142803144704, 140142811533311, +STORE, 140142811533312, 140142819926015, +STORE, 140142803144704, 140142811533311, +SNULL, 140142811537407, 140142819926015, +STORE, 140142811533312, 140142811537407, +STORE, 140142811537408, 140142819926015, +SNULL, 140142098649088, 140142165622783, +STORE, 140142165622784, 140142232731647, +STORE, 140142098649088, 140142165622783, +SNULL, 140142165757951, 140142232731647, +STORE, 140142165622784, 140142165757951, +STORE, 140142165757952, 140142232731647, +STORE, 140142090121216, 140142098513919, +SNULL, 140142777962496, 140142786355199, +STORE, 140142786355200, 140142803140607, +STORE, 140142777962496, 140142786355199, +SNULL, 140142786359295, 140142803140607, +STORE, 140142786355200, 140142786359295, +STORE, 140142786359296, 140142803140607, +SNULL, 140142509527040, 140142534705151, +STORE, 140142534705152, 140142543097855, +STORE, 140142509527040, 140142534705151, +SNULL, 140142534709247, 140142543097855, +STORE, 140142534705152, 140142534709247, +STORE, 140142534709248, 140142543097855, +STORE, 140142081728512, 140142098513919, +SNULL, 140142786359296, 140142794747903, +STORE, 140142794747904, 140142803140607, +STORE, 140142786359296, 140142794747903, +SNULL, 140142794751999, 140142803140607, +STORE, 140142794747904, 140142794751999, +STORE, 140142794752000, 140142803140607, +STORE, 140142073335808, 140142098513919, +SNULL, 140142073339903, 140142098513919, +STORE, 140142073335808, 140142073339903, +STORE, 140142073339904, 140142098513919, +SNULL, 140142543101952, 140142551490559, +STORE, 140142551490560, 140142559883263, +STORE, 140142543101952, 140142551490559, +SNULL, 140142551494655, 140142559883263, +STORE, 140142551490560, 140142551494655, +STORE, 140142551494656, 140142559883263, +SNULL, 140142509527040, 140142517919743, +STORE, 140142517919744, 140142534705151, +STORE, 140142509527040, 140142517919743, +SNULL, 140142517923839, 140142534705151, +STORE, 140142517919744, 140142517923839, +STORE, 140142517923840, 140142534705151, +STORE, 140142064943104, 140142073335807, +SNULL, 140142073339904, 140142090121215, +STORE, 140142090121216, 140142098513919, +STORE, 140142073339904, 140142090121215, +SNULL, 140142090125311, 140142098513919, +STORE, 140142090121216, 140142090125311, +STORE, 140142090125312, 140142098513919, +STORE, 140142056550400, 140142073335807, +SNULL, 140142056554495, 140142073335807, +STORE, 140142056550400, 140142056554495, +STORE, 140142056554496, 140142073335807, +STORE, 140142048157696, 140142056550399, +SNULL, 140142509531135, 140142517919743, +STORE, 140142509527040, 140142509531135, +STORE, 140142509531136, 140142517919743, +SNULL, 140142777966591, 140142786355199, +STORE, 140142777962496, 140142777966591, +STORE, 140142777966592, 140142786355199, +SNULL, 140143046402047, 140143054790655, +STORE, 140143046397952, 140143046402047, +STORE, 140143046402048, 140143054790655, +SNULL, 140142912184320, 140142920572927, +STORE, 140142920572928, 140142928965631, +STORE, 140142912184320, 140142920572927, +SNULL, 140142920577023, 140142928965631, +STORE, 140142920572928, 140142920577023, +STORE, 140142920577024, 140142928965631, +STORE, 140142039764992, 140142056550399, +STORE, 140141955903488, 140141964296191, +SNULL, 140142819930112, 140142828318719, +STORE, 140142828318720, 140142836711423, +STORE, 140142819930112, 140142828318719, +SNULL, 140142828322815, 140142836711423, +STORE, 140142828318720, 140142828322815, +STORE, 140142828322816, 140142836711423, +SNULL, 140142517923840, 140142526312447, +STORE, 140142526312448, 140142534705151, +STORE, 140142517923840, 140142526312447, +SNULL, 140142526316543, 140142534705151, +STORE, 140142526312448, 140142526316543, +STORE, 140142526316544, 140142534705151, +STORE, 140141947510784, 140141964296191, +SNULL, 140142056554496, 140142064943103, +STORE, 140142064943104, 140142073335807, +STORE, 140142056554496, 140142064943103, +SNULL, 140142064947199, 140142073335807, +STORE, 140142064943104, 140142064947199, +STORE, 140142064947200, 140142073335807, +SNULL, 140142073339904, 140142081728511, +STORE, 140142081728512, 140142090121215, +STORE, 140142073339904, 140142081728511, +SNULL, 140142081732607, 140142090121215, +STORE, 140142081728512, 140142081732607, +STORE, 140142081732608, 140142090121215, +STORE, 140141939118080, 140141964296191, +STORE, 140141930725376, 140141964296191, +STORE, 140141922332672, 140141964296191, +STORE, 140141913939968, 140141964296191, +SNULL, 140141913939968, 140141922332671, +STORE, 140141922332672, 140141964296191, +STORE, 140141913939968, 140141922332671, +SNULL, 140141922336767, 140141964296191, +STORE, 140141922332672, 140141922336767, +STORE, 140141922336768, 140141964296191, +STORE, 140141905547264, 140141922332671, +SNULL, 140141905551359, 140141922332671, +STORE, 140141905547264, 140141905551359, +STORE, 140141905551360, 140141922332671, +STORE, 140141821685760, 140141830078463, +STORE, 140141813293056, 140141830078463, +STORE, 140141804900352, 140141830078463, +STORE, 140141796507648, 140141830078463, +SNULL, 140141796511743, 140141830078463, +STORE, 140141796507648, 140141796511743, +STORE, 140141796511744, 140141830078463, +SNULL, 140141922336768, 140141955903487, +STORE, 140141955903488, 140141964296191, +STORE, 140141922336768, 140141955903487, +SNULL, 140141955907583, 140141964296191, +STORE, 140141955903488, 140141955907583, +STORE, 140141955907584, 140141964296191, +STORE, 140141788114944, 140141796507647, +STORE, 140141779722240, 140141796507647, +SNULL, 140141779722240, 140141788114943, +STORE, 140141788114944, 140141796507647, +STORE, 140141779722240, 140141788114943, +SNULL, 140141788119039, 140141796507647, +STORE, 140141788114944, 140141788119039, +STORE, 140141788119040, 140141796507647, +SNULL, 140141922336768, 140141947510783, +STORE, 140141947510784, 140141955903487, +STORE, 140141922336768, 140141947510783, +SNULL, 140141947514879, 140141955903487, +STORE, 140141947510784, 140141947514879, +STORE, 140141947514880, 140141955903487, +SNULL, 140142039764992, 140142048157695, +STORE, 140142048157696, 140142056550399, +STORE, 140142039764992, 140142048157695, +SNULL, 140142048161791, 140142056550399, +STORE, 140142048157696, 140142048161791, +STORE, 140142048161792, 140142056550399, +SNULL, 140142039769087, 140142048157695, +STORE, 140142039764992, 140142039769087, +STORE, 140142039769088, 140142048157695, +SNULL, 140141796511744, 140141804900351, +STORE, 140141804900352, 140141830078463, +STORE, 140141796511744, 140141804900351, +SNULL, 140141804904447, 140141830078463, +STORE, 140141804900352, 140141804904447, +STORE, 140141804904448, 140141830078463, +STORE, 140141771329536, 140141788114943, +STORE, 140141762936832, 140141788114943, +STORE, 140141754544128, 140141788114943, +SNULL, 140141804904448, 140141821685759, +STORE, 140141821685760, 140141830078463, +STORE, 140141804904448, 140141821685759, +SNULL, 140141821689855, 140141830078463, +STORE, 140141821685760, 140141821689855, +STORE, 140141821689856, 140141830078463, +SNULL, 140141922336768, 140141939118079, +STORE, 140141939118080, 140141947510783, +STORE, 140141922336768, 140141939118079, +SNULL, 140141939122175, 140141947510783, +STORE, 140141939118080, 140141939122175, +STORE, 140141939122176, 140141947510783, +SNULL, 140141905551360, 140141913939967, +STORE, 140141913939968, 140141922332671, +STORE, 140141905551360, 140141913939967, +SNULL, 140141913944063, 140141922332671, +STORE, 140141913939968, 140141913944063, +STORE, 140141913944064, 140141922332671, +STORE, 140141746151424, 140141788114943, +STORE, 140141737758720, 140141788114943, +SNULL, 140141804904448, 140141813293055, +STORE, 140141813293056, 140141821685759, +STORE, 140141804904448, 140141813293055, +SNULL, 140141813297151, 140141821685759, +STORE, 140141813293056, 140141813297151, +STORE, 140141813297152, 140141821685759, +STORE, 140141729366016, 140141788114943, +STORE, 140141720973312, 140141788114943, +STORE, 140141712580608, 140141788114943, +SNULL, 140141712584703, 140141788114943, +STORE, 140141712580608, 140141712584703, +STORE, 140141712584704, 140141788114943, +SNULL, 140141922336768, 140141930725375, +STORE, 140141930725376, 140141939118079, +STORE, 140141922336768, 140141930725375, +SNULL, 140141930729471, 140141939118079, +STORE, 140141930725376, 140141930729471, +STORE, 140141930729472, 140141939118079, +STORE, 140141704187904, 140141712580607, +SNULL, 140141704191999, 140141712580607, +STORE, 140141704187904, 140141704191999, +STORE, 140141704192000, 140141712580607, +STORE, 140141695795200, 140141704187903, +STORE, 140141687402496, 140141704187903, +SNULL, 140141712584704, 140141771329535, +STORE, 140141771329536, 140141788114943, +STORE, 140141712584704, 140141771329535, +SNULL, 140141771333631, 140141788114943, +STORE, 140141771329536, 140141771333631, +STORE, 140141771333632, 140141788114943, +SNULL, 140141771333632, 140141779722239, +STORE, 140141779722240, 140141788114943, +STORE, 140141771333632, 140141779722239, +SNULL, 140141779726335, 140141788114943, +STORE, 140141779722240, 140141779726335, +STORE, 140141779726336, 140141788114943, +STORE, 140141679009792, 140141704187903, +SNULL, 140141679013887, 140141704187903, +STORE, 140141679009792, 140141679013887, +STORE, 140141679013888, 140141704187903, +STORE, 140141670617088, 140141679009791, +SNULL, 140141670621183, 140141679009791, +STORE, 140141670617088, 140141670621183, +STORE, 140141670621184, 140141679009791, +STORE, 140141662224384, 140141670617087, +SNULL, 140141712584704, 140141737758719, +STORE, 140141737758720, 140141771329535, +STORE, 140141712584704, 140141737758719, +SNULL, 140141737762815, 140141771329535, +STORE, 140141737758720, 140141737762815, +STORE, 140141737762816, 140141771329535, +SNULL, 140141712584704, 140141729366015, +STORE, 140141729366016, 140141737758719, +STORE, 140141712584704, 140141729366015, +SNULL, 140141729370111, 140141737758719, +STORE, 140141729366016, 140141729370111, +STORE, 140141729370112, 140141737758719, +SNULL, 140141737762816, 140141746151423, +STORE, 140141746151424, 140141771329535, +STORE, 140141737762816, 140141746151423, +SNULL, 140141746155519, 140141771329535, +STORE, 140141746151424, 140141746155519, +STORE, 140141746155520, 140141771329535, +STORE, 140141653831680, 140141670617087, +SNULL, 140141746155520, 140141762936831, +STORE, 140141762936832, 140141771329535, +STORE, 140141746155520, 140141762936831, +SNULL, 140141762940927, 140141771329535, +STORE, 140141762936832, 140141762940927, +STORE, 140141762940928, 140141771329535, +STORE, 140141645438976, 140141670617087, +SNULL, 140141645443071, 140141670617087, +STORE, 140141645438976, 140141645443071, +STORE, 140141645443072, 140141670617087, +SNULL, 140141712584704, 140141720973311, +STORE, 140141720973312, 140141729366015, +STORE, 140141712584704, 140141720973311, +SNULL, 140141720977407, 140141729366015, +STORE, 140141720973312, 140141720977407, +STORE, 140141720977408, 140141729366015, +STORE, 140141637046272, 140141645438975, +SNULL, 140141637050367, 140141645438975, +STORE, 140141637046272, 140141637050367, +STORE, 140141637050368, 140141645438975, +STORE, 140141628653568, 140141637046271, +SNULL, 140141628657663, 140141637046271, +STORE, 140141628653568, 140141628657663, +STORE, 140141628657664, 140141637046271, +STORE, 140141620260864, 140141628653567, +SNULL, 140141679013888, 140141687402495, +STORE, 140141687402496, 140141704187903, +STORE, 140141679013888, 140141687402495, +SNULL, 140141687406591, 140141704187903, +STORE, 140141687402496, 140141687406591, +STORE, 140141687406592, 140141704187903, +SNULL, 140141746155520, 140141754544127, +STORE, 140141754544128, 140141762936831, +STORE, 140141746155520, 140141754544127, +SNULL, 140141754548223, 140141762936831, +STORE, 140141754544128, 140141754548223, +STORE, 140141754548224, 140141762936831, +SNULL, 140141687406592, 140141695795199, +STORE, 140141695795200, 140141704187903, +STORE, 140141687406592, 140141695795199, +SNULL, 140141695799295, 140141704187903, +STORE, 140141695795200, 140141695799295, +STORE, 140141695799296, 140141704187903, +STORE, 140141611868160, 140141628653567, +SNULL, 140141611872255, 140141628653567, +STORE, 140141611868160, 140141611872255, +STORE, 140141611872256, 140141628653567, +SNULL, 140141645443072, 140141662224383, +STORE, 140141662224384, 140141670617087, +STORE, 140141645443072, 140141662224383, +SNULL, 140141662228479, 140141670617087, +STORE, 140141662224384, 140141662228479, +STORE, 140141662228480, 140141670617087, +STORE, 140141603475456, 140141611868159, +SNULL, 140141603479551, 140141611868159, +STORE, 140141603475456, 140141603479551, +STORE, 140141603479552, 140141611868159, +STORE, 140141595082752, 140141603475455, +SNULL, 140141645443072, 140141653831679, +STORE, 140141653831680, 140141662224383, +STORE, 140141645443072, 140141653831679, +SNULL, 140141653835775, 140141662224383, +STORE, 140141653831680, 140141653835775, +STORE, 140141653835776, 140141662224383, +STORE, 140141586690048, 140141603475455, +SNULL, 140141611872256, 140141620260863, +STORE, 140141620260864, 140141628653567, +STORE, 140141611872256, 140141620260863, +SNULL, 140141620264959, 140141628653567, +STORE, 140141620260864, 140141620264959, +STORE, 140141620264960, 140141628653567, +SNULL, 140141586690048, 140141595082751, +STORE, 140141595082752, 140141603475455, +STORE, 140141586690048, 140141595082751, +SNULL, 140141595086847, 140141603475455, +STORE, 140141595082752, 140141595086847, +STORE, 140141595086848, 140141603475455, +STORE, 140141578297344, 140141595082751, +SNULL, 140141578301439, 140141595082751, +STORE, 140141578297344, 140141578301439, +STORE, 140141578301440, 140141595082751, +SNULL, 140141578301440, 140141586690047, +STORE, 140141586690048, 140141595082751, +STORE, 140141578301440, 140141586690047, +SNULL, 140141586694143, 140141595082751, +STORE, 140141586690048, 140141586694143, +STORE, 140141586694144, 140141595082751, +STORE, 140143370027008, 140143370055679, +STORE, 140143309254656, 140143311446015, +SNULL, 140143309254656, 140143309344767, +STORE, 140143309344768, 140143311446015, +STORE, 140143309254656, 140143309344767, +SNULL, 140143311437823, 140143311446015, +STORE, 140143309344768, 140143311437823, +STORE, 140143311437824, 140143311446015, +ERASE, 140143311437824, 140143311446015, +STORE, 140143311437824, 140143311446015, +SNULL, 140143311441919, 140143311446015, +STORE, 140143311437824, 140143311441919, +STORE, 140143311441920, 140143311446015, +ERASE, 140143370027008, 140143370055679, +ERASE, 140142912180224, 140142912184319, +ERASE, 140142912184320, 140142920572927, +ERASE, 140142945751040, 140142945755135, +ERASE, 140142945755136, 140142954143743, +ERASE, 140142090121216, 140142090125311, +ERASE, 140142090125312, 140142098513919, +ERASE, 140142794747904, 140142794751999, +ERASE, 140142794752000, 140142803140607, +ERASE, 140141913939968, 140141913944063, +ERASE, 140141913944064, 140141922332671, +ERASE, 140141746151424, 140141746155519, +ERASE, 140141746155520, 140141754544127, +ERASE, 140142954143744, 140142954147839, +ERASE, 140142954147840, 140142962536447, +ERASE, 140142081728512, 140142081732607, +ERASE, 140142081732608, 140142090121215, +ERASE, 140141905547264, 140141905551359, +ERASE, 140141905551360, 140141913939967, +ERASE, 140141729366016, 140141729370111, +ERASE, 140141729370112, 140141737758719, +ERASE, 140142920572928, 140142920577023, +ERASE, 140142920577024, 140142928965631, +ERASE, 140142039764992, 140142039769087, +ERASE, 140142039769088, 140142048157695, +ERASE, 140141679009792, 140141679013887, +ERASE, 140141679013888, 140141687402495, +ERASE, 140142551490560, 140142551494655, +ERASE, 140142551494656, 140142559883263, +ERASE, 140141947510784, 140141947514879, +ERASE, 140141947514880, 140141955903487, +ERASE, 140141771329536, 140141771333631, +ERASE, 140141771333632, 140141779722239, +ERASE, 140142928965632, 140142928969727, +ERASE, 140142928969728, 140142937358335, +ERASE, 140142073335808, 140142073339903, +ERASE, 140142073339904, 140142081728511, +ERASE, 140142543097856, 140142543101951, +ERASE, 140142543101952, 140142551490559, +ERASE, 140141955903488, 140141955907583, +ERASE, 140141955907584, 140141964296191, +ERASE, 140141704187904, 140141704191999, +ERASE, 140141704192000, 140141712580607, +ERASE, 140142786355200, 140142786359295, +ERASE, 140142786359296, 140142794747903, +ERASE, 140142056550400, 140142056554495, +ERASE, 140142056554496, 140142064943103, +ERASE, 140142828318720, 140142828322815, +ERASE, 140142828322816, 140142836711423, +ERASE, 140141788114944, 140141788119039, +ERASE, 140141788119040, 140141796507647, +ERASE, 140141695795200, 140141695799295, +ERASE, 140141695799296, 140141704187903, +ERASE, 140141578297344, 140141578301439, +ERASE, 140141578301440, 140141586690047, +ERASE, 140141611868160, 140141611872255, +ERASE, 140141611872256, 140141620260863, +ERASE, 140142811533312, 140142811537407, +ERASE, 140142811537408, 140142819926015, +ERASE, 140142064943104, 140142064947199, +ERASE, 140142064947200, 140142073335807, +ERASE, 140141628653568, 140141628657663, +ERASE, 140141628657664, 140141637046271, +ERASE, 140143046397952, 140143046402047, +ERASE, 140143046402048, 140143054790655, +ERASE, 140141796507648, 140141796511743, +ERASE, 140141796511744, 140141804900351, +ERASE, 140142803140608, 140142803144703, +ERASE, 140142803144704, 140142811533311, +ERASE, 140142509527040, 140142509531135, +ERASE, 140142509531136, 140142517919743, +ERASE, 140141821685760, 140141821689855, +ERASE, 140141821689856, 140141830078463, +ERASE, 140142777962496, 140142777966591, +ERASE, 140142777966592, 140142786355199, +ERASE, 140141804900352, 140141804904447, +ERASE, 140141804904448, 140141813293055, +ERASE, 140141930725376, 140141930729471, +ERASE, 140141930729472, 140141939118079, +ERASE, 140142937358336, 140142937362431, +ERASE, 140142937362432, 140142945751039, +ERASE, 140142559883264, 140142559887359, +ERASE, 140142559887360, 140142568275967, +ERASE, 140142534705152, 140142534709247, +ERASE, 140142534709248, 140142543097855, +ERASE, 140142048157696, 140142048161791, +ERASE, 140142048161792, 140142056550399, +ERASE, 140141754544128, 140141754548223, +ERASE, 140141754548224, 140141762936831, +ERASE, 140141939118080, 140141939122175, +ERASE, 140141939122176, 140141947510783, +ERASE, 140141653831680, 140141653835775, +ERASE, 140141653835776, 140141662224383, +ERASE, 140141712580608, 140141712584703, +ERASE, 140141712584704, 140141720973311, +ERASE, 140141645438976, 140141645443071, +ERASE, 140141645443072, 140141653831679, +ERASE, 140141687402496, 140141687406591, +ERASE, 140141687406592, 140141695795199, +ERASE, 140141662224384, 140141662228479, +ERASE, 140141662228480, 140141670617087, +ERASE, 140141922332672, 140141922336767, +ERASE, 140141922336768, 140141930725375, +ERASE, 140141737758720, 140141737762815, +ERASE, 140141737762816, 140141746151423, +ERASE, 140141637046272, 140141637050367, +ERASE, 140141637050368, 140141645438975, +ERASE, 140142517919744, 140142517923839, +ERASE, 140142517923840, 140142526312447, +ERASE, 140143096754176, 140143096758271, +ERASE, 140143096758272, 140143105146879, +ERASE, 140141595082752, 140141595086847, +ERASE, 140141595086848, 140141603475455, +ERASE, 140141762936832, 140141762940927, +ERASE, 140141762940928, 140141771329535, +ERASE, 140143311446016, 140143311450111, +ERASE, 140143311450112, 140143319838719, +ERASE, 140142526312448, 140142526316543, +ERASE, 140142526316544, 140142534705151, +ERASE, 140142819926016, 140142819930111, +ERASE, 140142819930112, 140142828318719, +ERASE, 140143180615680, 140143180619775, +ERASE, 140143180619776, 140143189008383, +ERASE, 140142962536448, 140142962540543, +ERASE, 140142962540544, 140142970929151, +ERASE, 140143214186496, 140143214190591, +ERASE, 140143214190592, 140143222579199, +ERASE, 140143088361472, 140143088365567, +ERASE, 140143088365568, 140143096754175, +ERASE, 140141586690048, 140141586694143, +ERASE, 140141586694144, 140141595082751, +ERASE, 140143230971904, 140143230975999, +ERASE, 140143230976000, 140143239364607, +ERASE, 140141779722240, 140141779726335, +ERASE, 140141779726336, 140141788114943, +ERASE, 140141670617088, 140141670621183, +ERASE, 140141670621184, 140141679009791, +ERASE, 140141813293056, 140141813297151, +ERASE, 140141813297152, 140141821685759, +ERASE, 140143222579200, 140143222583295, +ERASE, 140143222583296, 140143230971903, +ERASE, 140143189008384, 140143189012479, +ERASE, 140143189012480, 140143197401087, +ERASE, 140143071576064, 140143071580159, +ERASE, 140143071580160, 140143079968767, +ERASE, 140141620260864, 140141620264959, +ERASE, 140141620264960, 140141628653567, +ERASE, 140141603475456, 140141603479551, +ERASE, 140141603479552, 140141611868159, +ERASE, 140141720973312, 140141720977407, +ERASE, 140141720977408, 140141729366015, +ERASE, 140143079968768, 140143079972863, +ERASE, 140143079972864, 140143088361471, +ERASE, 140143205793792, 140143205797887, +ERASE, 140143205797888, 140143214186495, + }; + unsigned long set30[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140733436743680, 140737488351231, +SNULL, 140733436747775, 140737488351231, +STORE, 140733436743680, 140733436747775, +STORE, 140733436612608, 140733436747775, +STORE, 94630728904704, 94630731157503, +SNULL, 94630729035775, 94630731157503, +STORE, 94630728904704, 94630729035775, +STORE, 94630729035776, 94630731157503, +ERASE, 94630729035776, 94630731157503, +STORE, 94630731128832, 94630731137023, +STORE, 94630731137024, 94630731157503, +STORE, 140165750841344, 140165753094143, +SNULL, 140165750984703, 140165753094143, +STORE, 140165750841344, 140165750984703, +STORE, 140165750984704, 140165753094143, +ERASE, 140165750984704, 140165753094143, +STORE, 140165753081856, 140165753090047, +STORE, 140165753090048, 140165753094143, +STORE, 140733436887040, 140733436891135, +STORE, 140733436874752, 140733436887039, +STORE, 140165753053184, 140165753081855, +STORE, 140165753044992, 140165753053183, +STORE, 140165748625408, 140165750841343, +SNULL, 140165748625408, 140165748723711, +STORE, 140165748723712, 140165750841343, +STORE, 140165748625408, 140165748723711, +SNULL, 140165750816767, 140165750841343, +STORE, 140165748723712, 140165750816767, +STORE, 140165750816768, 140165750841343, +SNULL, 140165750816768, 140165750824959, +STORE, 140165750824960, 140165750841343, +STORE, 140165750816768, 140165750824959, +ERASE, 140165750816768, 140165750824959, +STORE, 140165750816768, 140165750824959, +ERASE, 140165750824960, 140165750841343, +STORE, 140165750824960, 140165750841343, +STORE, 140165744828416, 140165748625407, +SNULL, 140165744828416, 140165746487295, +STORE, 140165746487296, 140165748625407, +STORE, 140165744828416, 140165746487295, +SNULL, 140165748584447, 140165748625407, +STORE, 140165746487296, 140165748584447, +STORE, 140165748584448, 140165748625407, +SNULL, 140165748584448, 140165748609023, +STORE, 140165748609024, 140165748625407, +STORE, 140165748584448, 140165748609023, +ERASE, 140165748584448, 140165748609023, +STORE, 140165748584448, 140165748609023, +ERASE, 140165748609024, 140165748625407, +STORE, 140165748609024, 140165748625407, +STORE, 140165753036800, 140165753053183, +SNULL, 140165748600831, 140165748609023, +STORE, 140165748584448, 140165748600831, +STORE, 140165748600832, 140165748609023, +SNULL, 140165750820863, 140165750824959, +STORE, 140165750816768, 140165750820863, +STORE, 140165750820864, 140165750824959, +SNULL, 94630731132927, 94630731137023, +STORE, 94630731128832, 94630731132927, +STORE, 94630731132928, 94630731137023, +SNULL, 140165753085951, 140165753090047, +STORE, 140165753081856, 140165753085951, +STORE, 140165753085952, 140165753090047, +ERASE, 140165753053184, 140165753081855, +STORE, 94630743547904, 94630743683071, +STORE, 140165736435712, 140165744828415, +SNULL, 140165736439807, 140165744828415, +STORE, 140165736435712, 140165736439807, +STORE, 140165736439808, 140165744828415, +STORE, 140165728043008, 140165736435711, +STORE, 140165593825280, 140165728043007, +SNULL, 140165593825280, 140165653725183, +STORE, 140165653725184, 140165728043007, +STORE, 140165593825280, 140165653725183, +ERASE, 140165593825280, 140165653725183, +SNULL, 140165720834047, 140165728043007, +STORE, 140165653725184, 140165720834047, +STORE, 140165720834048, 140165728043007, +ERASE, 140165720834048, 140165728043007, +SNULL, 140165653860351, 140165720834047, +STORE, 140165653725184, 140165653860351, +STORE, 140165653860352, 140165720834047, +SNULL, 140165728047103, 140165736435711, +STORE, 140165728043008, 140165728047103, +STORE, 140165728047104, 140165736435711, +STORE, 140165645332480, 140165653725183, +SNULL, 140165645336575, 140165653725183, +STORE, 140165645332480, 140165645336575, +STORE, 140165645336576, 140165653725183, +STORE, 140165636939776, 140165645332479, +SNULL, 140165636943871, 140165645332479, +STORE, 140165636939776, 140165636943871, +STORE, 140165636943872, 140165645332479, +STORE, 140165628547072, 140165636939775, +SNULL, 140165628551167, 140165636939775, +STORE, 140165628547072, 140165628551167, +STORE, 140165628551168, 140165636939775, +STORE, 140165620154368, 140165628547071, +STORE, 140165611761664, 140165628547071, +STORE, 140165603368960, 140165628547071, +STORE, 140165469151232, 140165603368959, +SNULL, 140165469151232, 140165519507455, +STORE, 140165519507456, 140165603368959, +STORE, 140165469151232, 140165519507455, +ERASE, 140165469151232, 140165519507455, +SNULL, 140165586616319, 140165603368959, +STORE, 140165519507456, 140165586616319, +STORE, 140165586616320, 140165603368959, +ERASE, 140165586616320, 140165603368959, +STORE, 140165594976256, 140165628547071, +STORE, 140165385289728, 140165586616319, +SNULL, 140165452398591, 140165586616319, +STORE, 140165385289728, 140165452398591, +STORE, 140165452398592, 140165586616319, +SNULL, 140165452398592, 140165519507455, +STORE, 140165519507456, 140165586616319, +STORE, 140165452398592, 140165519507455, +ERASE, 140165452398592, 140165519507455, +STORE, 140165251072000, 140165452398591, +SNULL, 140165318180863, 140165452398591, +STORE, 140165251072000, 140165318180863, +STORE, 140165318180864, 140165452398591, +SNULL, 140165318180864, 140165385289727, +STORE, 140165385289728, 140165452398591, +STORE, 140165318180864, 140165385289727, +ERASE, 140165318180864, 140165385289727, +SNULL, 140165519642623, 140165586616319, +STORE, 140165519507456, 140165519642623, +STORE, 140165519642624, 140165586616319, +SNULL, 140165594976256, 140165611761663, +STORE, 140165611761664, 140165628547071, +STORE, 140165594976256, 140165611761663, +SNULL, 140165611765759, 140165628547071, +STORE, 140165611761664, 140165611765759, +STORE, 140165611765760, 140165628547071, +STORE, 140165385289728, 140165519507455, +SNULL, 140165385424895, 140165519507455, +STORE, 140165385289728, 140165385424895, +STORE, 140165385424896, 140165519507455, +SNULL, 140165594976256, 140165603368959, +STORE, 140165603368960, 140165611761663, +STORE, 140165594976256, 140165603368959, +SNULL, 140165603373055, 140165611761663, +STORE, 140165603368960, 140165603373055, +STORE, 140165603373056, 140165611761663, +SNULL, 140165251207167, 140165318180863, +STORE, 140165251072000, 140165251207167, +STORE, 140165251207168, 140165318180863, +STORE, 140165376897024, 140165385289727, +SNULL, 140165376901119, 140165385289727, +STORE, 140165376897024, 140165376901119, +STORE, 140165376901120, 140165385289727, +SNULL, 140165385424896, 140165452398591, +STORE, 140165452398592, 140165519507455, +STORE, 140165385424896, 140165452398591, +SNULL, 140165452533759, 140165519507455, +STORE, 140165452398592, 140165452533759, +STORE, 140165452533760, 140165519507455, +STORE, 140165368504320, 140165376897023, +SNULL, 140165594980351, 140165603368959, +STORE, 140165594976256, 140165594980351, +STORE, 140165594980352, 140165603368959, +SNULL, 140165368508415, 140165376897023, +STORE, 140165368504320, 140165368508415, +STORE, 140165368508416, 140165376897023, +SNULL, 140165611765760, 140165620154367, +STORE, 140165620154368, 140165628547071, +STORE, 140165611765760, 140165620154367, +SNULL, 140165620158463, 140165628547071, +STORE, 140165620154368, 140165620158463, +STORE, 140165620158464, 140165628547071, +STORE, 140165360111616, 140165368504319, +STORE, 140165351718912, 140165368504319, +STORE, 140165343326208, 140165368504319, +SNULL, 140165343326208, 140165351718911, +STORE, 140165351718912, 140165368504319, +STORE, 140165343326208, 140165351718911, +SNULL, 140165351723007, 140165368504319, +STORE, 140165351718912, 140165351723007, +STORE, 140165351723008, 140165368504319, +SNULL, 140165343330303, 140165351718911, +STORE, 140165343326208, 140165343330303, +STORE, 140165343330304, 140165351718911, +SNULL, 140165351723008, 140165360111615, +STORE, 140165360111616, 140165368504319, +STORE, 140165351723008, 140165360111615, +SNULL, 140165360115711, 140165368504319, +STORE, 140165360111616, 140165360115711, +STORE, 140165360115712, 140165368504319, +STORE, 140165334933504, 140165343326207, +SNULL, 140165334937599, 140165343326207, +STORE, 140165334933504, 140165334937599, +STORE, 140165334937600, 140165343326207, +STORE, 140165326540800, 140165334933503, +STORE, 140165242679296, 140165251071999, +SNULL, 140165242683391, 140165251071999, +STORE, 140165242679296, 140165242683391, +STORE, 140165242683392, 140165251071999, +STORE, 140165234286592, 140165242679295, +STORE, 140165225893888, 140165242679295, +SNULL, 140165225897983, 140165242679295, +STORE, 140165225893888, 140165225897983, +STORE, 140165225897984, 140165242679295, +SNULL, 140165225897984, 140165234286591, +STORE, 140165234286592, 140165242679295, +STORE, 140165225897984, 140165234286591, +SNULL, 140165234290687, 140165242679295, +STORE, 140165234286592, 140165234290687, +STORE, 140165234290688, 140165242679295, +SNULL, 140165326544895, 140165334933503, +STORE, 140165326540800, 140165326544895, +STORE, 140165326544896, 140165334933503, +STORE, 140165217501184, 140165225893887, +STORE, 140165209108480, 140165225893887, +SNULL, 140165209108480, 140165217501183, +STORE, 140165217501184, 140165225893887, +STORE, 140165209108480, 140165217501183, +SNULL, 140165217505279, 140165225893887, +STORE, 140165217501184, 140165217505279, +STORE, 140165217505280, 140165225893887, +SNULL, 140165209112575, 140165217501183, +STORE, 140165209108480, 140165209112575, +STORE, 140165209112576, 140165217501183, +STORE, 140165200715776, 140165209108479, +STORE, 140165066498048, 140165200715775, +SNULL, 140165066498048, 140165116854271, +STORE, 140165116854272, 140165200715775, +STORE, 140165066498048, 140165116854271, +ERASE, 140165066498048, 140165116854271, +SNULL, 140165183963135, 140165200715775, +STORE, 140165116854272, 140165183963135, +STORE, 140165183963136, 140165200715775, +ERASE, 140165183963136, 140165200715775, +SNULL, 140165116989439, 140165183963135, +STORE, 140165116854272, 140165116989439, +STORE, 140165116989440, 140165183963135, +STORE, 140165192323072, 140165209108479, +STORE, 140165108461568, 140165116854271, +STORE, 140164974243840, 140165108461567, +STORE, 140164965851136, 140164974243839, +SNULL, 140164974243840, 140164982636543, +STORE, 140164982636544, 140165108461567, +STORE, 140164974243840, 140164982636543, +ERASE, 140164974243840, 140164982636543, +STORE, 140164965851136, 140164982636543, +STORE, 140164957458432, 140164982636543, +STORE, 140164949065728, 140164982636543, +STORE, 140164940673024, 140164982636543, +STORE, 140164806455296, 140164940673023, +STORE, 140164798062592, 140164806455295, +STORE, 140164789669888, 140164806455295, +STORE, 140164655452160, 140164789669887, +STORE, 140164647059456, 140164655452159, +STORE, 140164638666752, 140164655452159, +SNULL, 140164655452160, 140164714201087, +STORE, 140164714201088, 140164789669887, +STORE, 140164655452160, 140164714201087, +ERASE, 140164655452160, 140164714201087, +STORE, 140164705808384, 140164714201087, +STORE, 140164697415680, 140164714201087, +STORE, 140164504449024, 140164638666751, +SNULL, 140164504449024, 140164512874495, +STORE, 140164512874496, 140164638666751, +STORE, 140164504449024, 140164512874495, +ERASE, 140164504449024, 140164512874495, +STORE, 140164689022976, 140164714201087, +STORE, 140164680630272, 140164714201087, +SNULL, 140164680634367, 140164714201087, +STORE, 140164680630272, 140164680634367, +STORE, 140164680634368, 140164714201087, +STORE, 140164378656768, 140164638666751, +SNULL, 140165192323072, 140165200715775, +STORE, 140165200715776, 140165209108479, +STORE, 140165192323072, 140165200715775, +SNULL, 140165200719871, 140165209108479, +STORE, 140165200715776, 140165200719871, +STORE, 140165200719872, 140165209108479, +SNULL, 140165049745407, 140165108461567, +STORE, 140164982636544, 140165049745407, +STORE, 140165049745408, 140165108461567, +ERASE, 140165049745408, 140165108461567, +SNULL, 140164982771711, 140165049745407, +STORE, 140164982636544, 140164982771711, +STORE, 140164982771712, 140165049745407, +STORE, 140164244439040, 140164638666751, +SNULL, 140164311547903, 140164638666751, +STORE, 140164244439040, 140164311547903, +STORE, 140164311547904, 140164638666751, +SNULL, 140164311547904, 140164378656767, +STORE, 140164378656768, 140164638666751, +STORE, 140164311547904, 140164378656767, +ERASE, 140164311547904, 140164378656767, +SNULL, 140164806455296, 140164848418815, +STORE, 140164848418816, 140164940673023, +STORE, 140164806455296, 140164848418815, +ERASE, 140164806455296, 140164848418815, +SNULL, 140164915527679, 140164940673023, +STORE, 140164848418816, 140164915527679, +STORE, 140164915527680, 140164940673023, +ERASE, 140164915527680, 140164940673023, +STORE, 140164110221312, 140164311547903, +SNULL, 140164177330175, 140164311547903, +STORE, 140164110221312, 140164177330175, +STORE, 140164177330176, 140164311547903, +SNULL, 140164177330176, 140164244439039, +STORE, 140164244439040, 140164311547903, +STORE, 140164177330176, 140164244439039, +ERASE, 140164177330176, 140164244439039, +SNULL, 140164781309951, 140164789669887, +STORE, 140164714201088, 140164781309951, +STORE, 140164781309952, 140164789669887, +ERASE, 140164781309952, 140164789669887, +STORE, 140163976003584, 140164177330175, +SNULL, 140164043112447, 140164177330175, +STORE, 140163976003584, 140164043112447, +STORE, 140164043112448, 140164177330175, +SNULL, 140164043112448, 140164110221311, +STORE, 140164110221312, 140164177330175, +STORE, 140164043112448, 140164110221311, +ERASE, 140164043112448, 140164110221311, +SNULL, 140164579983359, 140164638666751, +STORE, 140164378656768, 140164579983359, +STORE, 140164579983360, 140164638666751, +ERASE, 140164579983360, 140164638666751, +STORE, 140163841785856, 140164043112447, +SNULL, 140163908894719, 140164043112447, +STORE, 140163841785856, 140163908894719, +STORE, 140163908894720, 140164043112447, +SNULL, 140163908894720, 140163976003583, +STORE, 140163976003584, 140164043112447, +STORE, 140163908894720, 140163976003583, +ERASE, 140163908894720, 140163976003583, +SNULL, 140164940673024, 140164965851135, +STORE, 140164965851136, 140164982636543, +STORE, 140164940673024, 140164965851135, +SNULL, 140164965855231, 140164982636543, +STORE, 140164965851136, 140164965855231, +STORE, 140164965855232, 140164982636543, +SNULL, 140164965855232, 140164974243839, +STORE, 140164974243840, 140164982636543, +STORE, 140164965855232, 140164974243839, +SNULL, 140164974247935, 140164982636543, +STORE, 140164974243840, 140164974247935, +STORE, 140164974247936, 140164982636543, +SNULL, 140164445765631, 140164579983359, +STORE, 140164378656768, 140164445765631, +STORE, 140164445765632, 140164579983359, +SNULL, 140164445765632, 140164512874495, +STORE, 140164512874496, 140164579983359, +STORE, 140164445765632, 140164512874495, +ERASE, 140164445765632, 140164512874495, +SNULL, 140164378791935, 140164445765631, +STORE, 140164378656768, 140164378791935, +STORE, 140164378791936, 140164445765631, +SNULL, 140164789673983, 140164806455295, +STORE, 140164789669888, 140164789673983, +STORE, 140164789673984, 140164806455295, +SNULL, 140164789673984, 140164798062591, +STORE, 140164798062592, 140164806455295, +STORE, 140164789673984, 140164798062591, +SNULL, 140164798066687, 140164806455295, +STORE, 140164798062592, 140164798066687, +STORE, 140164798066688, 140164806455295, +SNULL, 140164638670847, 140164655452159, +STORE, 140164638666752, 140164638670847, +STORE, 140164638670848, 140164655452159, +STORE, 140165100068864, 140165116854271, +STORE, 140165091676160, 140165116854271, +STORE, 140165083283456, 140165116854271, +SNULL, 140164244574207, 140164311547903, +STORE, 140164244439040, 140164244574207, +STORE, 140164244574208, 140164311547903, +SNULL, 140164848553983, 140164915527679, +STORE, 140164848418816, 140164848553983, +STORE, 140164848553984, 140164915527679, +SNULL, 140164110356479, 140164177330175, +STORE, 140164110221312, 140164110356479, +STORE, 140164110356480, 140164177330175, +SNULL, 140164714336255, 140164781309951, +STORE, 140164714201088, 140164714336255, +STORE, 140164714336256, 140164781309951, +SNULL, 140163976138751, 140164043112447, +STORE, 140163976003584, 140163976138751, +STORE, 140163976138752, 140164043112447, +SNULL, 140164513009663, 140164579983359, +STORE, 140164512874496, 140164513009663, +STORE, 140164513009664, 140164579983359, +SNULL, 140163841921023, 140163908894719, +STORE, 140163841785856, 140163841921023, +STORE, 140163841921024, 140163908894719, +SNULL, 140165083283456, 140165100068863, +STORE, 140165100068864, 140165116854271, +STORE, 140165083283456, 140165100068863, +SNULL, 140165100072959, 140165116854271, +STORE, 140165100068864, 140165100072959, +STORE, 140165100072960, 140165116854271, +SNULL, 140165100072960, 140165108461567, +STORE, 140165108461568, 140165116854271, +STORE, 140165100072960, 140165108461567, +SNULL, 140165108465663, 140165116854271, +STORE, 140165108461568, 140165108465663, +STORE, 140165108465664, 140165116854271, +STORE, 140165074890752, 140165100068863, +SNULL, 140165074894847, 140165100068863, +STORE, 140165074890752, 140165074894847, +STORE, 140165074894848, 140165100068863, +STORE, 140165066498048, 140165074890751, +STORE, 140165058105344, 140165074890751, +STORE, 140164932280320, 140164965851135, +SNULL, 140165192327167, 140165200715775, +STORE, 140165192323072, 140165192327167, +STORE, 140165192327168, 140165200715775, +STORE, 140164923887616, 140164965851135, +SNULL, 140164923891711, 140164965851135, +STORE, 140164923887616, 140164923891711, +STORE, 140164923891712, 140164965851135, +SNULL, 140164680634368, 140164705808383, +STORE, 140164705808384, 140164714201087, +STORE, 140164680634368, 140164705808383, +SNULL, 140164705812479, 140164714201087, +STORE, 140164705808384, 140164705812479, +STORE, 140164705812480, 140164714201087, +SNULL, 140164680634368, 140164697415679, +STORE, 140164697415680, 140164705808383, +STORE, 140164680634368, 140164697415679, +SNULL, 140164697419775, 140164705808383, +STORE, 140164697415680, 140164697419775, +STORE, 140164697419776, 140164705808383, +STORE, 140164840026112, 140164848418815, +STORE, 140164831633408, 140164848418815, +STORE, 140164823240704, 140164848418815, +SNULL, 140165074894848, 140165083283455, +STORE, 140165083283456, 140165100068863, +STORE, 140165074894848, 140165083283455, +SNULL, 140165083287551, 140165100068863, +STORE, 140165083283456, 140165083287551, +STORE, 140165083287552, 140165100068863, +SNULL, 140165083287552, 140165091676159, +STORE, 140165091676160, 140165100068863, +STORE, 140165083287552, 140165091676159, +SNULL, 140165091680255, 140165100068863, +STORE, 140165091676160, 140165091680255, +STORE, 140165091680256, 140165100068863, +SNULL, 140164638670848, 140164647059455, +STORE, 140164647059456, 140164655452159, +STORE, 140164638670848, 140164647059455, +SNULL, 140164647063551, 140164655452159, +STORE, 140164647059456, 140164647063551, +STORE, 140164647063552, 140164655452159, +SNULL, 140164923891712, 140164940673023, +STORE, 140164940673024, 140164965851135, +STORE, 140164923891712, 140164940673023, +SNULL, 140164940677119, 140164965851135, +STORE, 140164940673024, 140164940677119, +STORE, 140164940677120, 140164965851135, +SNULL, 140164940677120, 140164949065727, +STORE, 140164949065728, 140164965851135, +STORE, 140164940677120, 140164949065727, +SNULL, 140164949069823, 140164965851135, +STORE, 140164949065728, 140164949069823, +STORE, 140164949069824, 140164965851135, +SNULL, 140164949069824, 140164957458431, +STORE, 140164957458432, 140164965851135, +STORE, 140164949069824, 140164957458431, +SNULL, 140164957462527, 140164965851135, +STORE, 140164957458432, 140164957462527, +STORE, 140164957462528, 140164965851135, +SNULL, 140164680634368, 140164689022975, +STORE, 140164689022976, 140164697415679, +STORE, 140164680634368, 140164689022975, +SNULL, 140164689027071, 140164697415679, +STORE, 140164689022976, 140164689027071, +STORE, 140164689027072, 140164697415679, +STORE, 140164814848000, 140164848418815, +SNULL, 140165058105344, 140165066498047, +STORE, 140165066498048, 140165074890751, +STORE, 140165058105344, 140165066498047, +SNULL, 140165066502143, 140165074890751, +STORE, 140165066498048, 140165066502143, +STORE, 140165066502144, 140165074890751, +SNULL, 140165058109439, 140165066498047, +STORE, 140165058105344, 140165058109439, +STORE, 140165058109440, 140165066498047, +STORE, 140164798066688, 140164814847999, +SNULL, 140164798066688, 140164806455295, +STORE, 140164806455296, 140164814847999, +STORE, 140164798066688, 140164806455295, +SNULL, 140164806459391, 140164814847999, +STORE, 140164806455296, 140164806459391, +STORE, 140164806459392, 140164814847999, +SNULL, 140164923891712, 140164932280319, +STORE, 140164932280320, 140164940673023, +STORE, 140164923891712, 140164932280319, +SNULL, 140164932284415, 140164940673023, +STORE, 140164932280320, 140164932284415, +STORE, 140164932284416, 140164940673023, +STORE, 140164672237568, 140164680630271, +STORE, 140164663844864, 140164680630271, +STORE, 140164647063552, 140164680630271, +SNULL, 140164647063552, 140164655452159, +STORE, 140164655452160, 140164680630271, +STORE, 140164647063552, 140164655452159, +SNULL, 140164655456255, 140164680630271, +STORE, 140164655452160, 140164655456255, +STORE, 140164655456256, 140164680630271, +STORE, 140164630274048, 140164638666751, +SNULL, 140164814852095, 140164848418815, +STORE, 140164814848000, 140164814852095, +STORE, 140164814852096, 140164848418815, +SNULL, 140164814852096, 140164831633407, +STORE, 140164831633408, 140164848418815, +STORE, 140164814852096, 140164831633407, +SNULL, 140164831637503, 140164848418815, +STORE, 140164831633408, 140164831637503, +STORE, 140164831637504, 140164848418815, +STORE, 140164621881344, 140164638666751, +SNULL, 140164831637504, 140164840026111, +STORE, 140164840026112, 140164848418815, +STORE, 140164831637504, 140164840026111, +SNULL, 140164840030207, 140164848418815, +STORE, 140164840026112, 140164840030207, +STORE, 140164840030208, 140164848418815, +STORE, 140164613488640, 140164638666751, +SNULL, 140164613492735, 140164638666751, +STORE, 140164613488640, 140164613492735, +STORE, 140164613492736, 140164638666751, +STORE, 140164605095936, 140164613488639, +SNULL, 140164605100031, 140164613488639, +STORE, 140164605095936, 140164605100031, +STORE, 140164605100032, 140164613488639, +STORE, 140164596703232, 140164605095935, +STORE, 140164588310528, 140164605095935, +SNULL, 140164588314623, 140164605095935, +STORE, 140164588310528, 140164588314623, +STORE, 140164588314624, 140164605095935, +STORE, 140164504481792, 140164512874495, +STORE, 140164496089088, 140164512874495, +SNULL, 140164496089088, 140164504481791, +STORE, 140164504481792, 140164512874495, +STORE, 140164496089088, 140164504481791, +SNULL, 140164504485887, 140164512874495, +STORE, 140164504481792, 140164504485887, +STORE, 140164504485888, 140164512874495, +SNULL, 140164613492736, 140164630274047, +STORE, 140164630274048, 140164638666751, +STORE, 140164613492736, 140164630274047, +SNULL, 140164630278143, 140164638666751, +STORE, 140164630274048, 140164630278143, +STORE, 140164630278144, 140164638666751, +STORE, 140164487696384, 140164504481791, +STORE, 140164479303680, 140164504481791, +SNULL, 140164814852096, 140164823240703, +STORE, 140164823240704, 140164831633407, +STORE, 140164814852096, 140164823240703, +SNULL, 140164823244799, 140164831633407, +STORE, 140164823240704, 140164823244799, +STORE, 140164823244800, 140164831633407, +STORE, 140164470910976, 140164504481791, +SNULL, 140164470910976, 140164496089087, +STORE, 140164496089088, 140164504481791, +STORE, 140164470910976, 140164496089087, +SNULL, 140164496093183, 140164504481791, +STORE, 140164496089088, 140164496093183, +STORE, 140164496093184, 140164504481791, +SNULL, 140164655456256, 140164672237567, +STORE, 140164672237568, 140164680630271, +STORE, 140164655456256, 140164672237567, +SNULL, 140164672241663, 140164680630271, +STORE, 140164672237568, 140164672241663, +STORE, 140164672241664, 140164680630271, +STORE, 140164462518272, 140164496089087, +STORE, 140164454125568, 140164496089087, +SNULL, 140164655456256, 140164663844863, +STORE, 140164663844864, 140164672237567, +STORE, 140164655456256, 140164663844863, +SNULL, 140164663848959, 140164672237567, +STORE, 140164663844864, 140164663848959, +STORE, 140164663848960, 140164672237567, +STORE, 140164370264064, 140164378656767, +STORE, 140164361871360, 140164378656767, +STORE, 140164353478656, 140164378656767, +STORE, 140164345085952, 140164378656767, +SNULL, 140164345085952, 140164353478655, +STORE, 140164353478656, 140164378656767, +STORE, 140164345085952, 140164353478655, +SNULL, 140164353482751, 140164378656767, +STORE, 140164353478656, 140164353482751, +STORE, 140164353482752, 140164378656767, +SNULL, 140164454125568, 140164487696383, +STORE, 140164487696384, 140164496089087, +STORE, 140164454125568, 140164487696383, +SNULL, 140164487700479, 140164496089087, +STORE, 140164487696384, 140164487700479, +STORE, 140164487700480, 140164496089087, +STORE, 140164336693248, 140164353478655, +SNULL, 140164336697343, 140164353478655, +STORE, 140164336693248, 140164336697343, +STORE, 140164336697344, 140164353478655, +STORE, 140164328300544, 140164336693247, +SNULL, 140164454125568, 140164479303679, +STORE, 140164479303680, 140164487696383, +STORE, 140164454125568, 140164479303679, +SNULL, 140164479307775, 140164487696383, +STORE, 140164479303680, 140164479307775, +STORE, 140164479307776, 140164487696383, +STORE, 140164319907840, 140164336693247, +STORE, 140164236046336, 140164244439039, +SNULL, 140164588314624, 140164596703231, +STORE, 140164596703232, 140164605095935, +STORE, 140164588314624, 140164596703231, +SNULL, 140164596707327, 140164605095935, +STORE, 140164596703232, 140164596707327, +STORE, 140164596707328, 140164605095935, +SNULL, 140164454125568, 140164462518271, +STORE, 140164462518272, 140164479303679, +STORE, 140164454125568, 140164462518271, +SNULL, 140164462522367, 140164479303679, +STORE, 140164462518272, 140164462522367, +STORE, 140164462522368, 140164479303679, +STORE, 140164227653632, 140164244439039, +SNULL, 140164227657727, 140164244439039, +STORE, 140164227653632, 140164227657727, +STORE, 140164227657728, 140164244439039, +SNULL, 140164462522368, 140164470910975, +STORE, 140164470910976, 140164479303679, +STORE, 140164462522368, 140164470910975, +SNULL, 140164470915071, 140164479303679, +STORE, 140164470910976, 140164470915071, +STORE, 140164470915072, 140164479303679, +SNULL, 140164613492736, 140164621881343, +STORE, 140164621881344, 140164630274047, +STORE, 140164613492736, 140164621881343, +SNULL, 140164621885439, 140164630274047, +STORE, 140164621881344, 140164621885439, +STORE, 140164621885440, 140164630274047, +SNULL, 140164353482752, 140164370264063, +STORE, 140164370264064, 140164378656767, +STORE, 140164353482752, 140164370264063, +SNULL, 140164370268159, 140164378656767, +STORE, 140164370264064, 140164370268159, +STORE, 140164370268160, 140164378656767, +STORE, 140164219260928, 140164227653631, +SNULL, 140164319911935, 140164336693247, +STORE, 140164319907840, 140164319911935, +STORE, 140164319911936, 140164336693247, +SNULL, 140164336697344, 140164345085951, +STORE, 140164345085952, 140164353478655, +STORE, 140164336697344, 140164345085951, +SNULL, 140164345090047, 140164353478655, +STORE, 140164345085952, 140164345090047, +STORE, 140164345090048, 140164353478655, +SNULL, 140164319911936, 140164328300543, +STORE, 140164328300544, 140164336693247, +STORE, 140164319911936, 140164328300543, +SNULL, 140164328304639, 140164336693247, +STORE, 140164328300544, 140164328304639, +STORE, 140164328304640, 140164336693247, +SNULL, 140164454129663, 140164462518271, +STORE, 140164454125568, 140164454129663, +STORE, 140164454129664, 140164462518271, +STORE, 140164210868224, 140164227653631, +STORE, 140164202475520, 140164227653631, +STORE, 140164194082816, 140164227653631, +SNULL, 140164194086911, 140164227653631, +STORE, 140164194082816, 140164194086911, +STORE, 140164194086912, 140164227653631, +SNULL, 140164353482752, 140164361871359, +STORE, 140164361871360, 140164370264063, +STORE, 140164353482752, 140164361871359, +SNULL, 140164361875455, 140164370264063, +STORE, 140164361871360, 140164361875455, +STORE, 140164361875456, 140164370264063, +SNULL, 140164227657728, 140164236046335, +STORE, 140164236046336, 140164244439039, +STORE, 140164227657728, 140164236046335, +SNULL, 140164236050431, 140164244439039, +STORE, 140164236046336, 140164236050431, +STORE, 140164236050432, 140164244439039, +STORE, 140164185690112, 140164194082815, +SNULL, 140164194086912, 140164219260927, +STORE, 140164219260928, 140164227653631, +STORE, 140164194086912, 140164219260927, +SNULL, 140164219265023, 140164227653631, +STORE, 140164219260928, 140164219265023, +STORE, 140164219265024, 140164227653631, +STORE, 140164101828608, 140164110221311, +STORE, 140164093435904, 140164110221311, +STORE, 140164085043200, 140164110221311, +SNULL, 140164085047295, 140164110221311, +STORE, 140164085043200, 140164085047295, +STORE, 140164085047296, 140164110221311, +STORE, 140164076650496, 140164085043199, +SNULL, 140164185694207, 140164194082815, +STORE, 140164185690112, 140164185694207, +STORE, 140164185694208, 140164194082815, +SNULL, 140164085047296, 140164101828607, +STORE, 140164101828608, 140164110221311, +STORE, 140164085047296, 140164101828607, +SNULL, 140164101832703, 140164110221311, +STORE, 140164101828608, 140164101832703, +STORE, 140164101832704, 140164110221311, +SNULL, 140164085047296, 140164093435903, +STORE, 140164093435904, 140164101828607, +STORE, 140164085047296, 140164093435903, +SNULL, 140164093439999, 140164101828607, +STORE, 140164093435904, 140164093439999, +STORE, 140164093440000, 140164101828607, +SNULL, 140164194086912, 140164202475519, +STORE, 140164202475520, 140164219260927, +STORE, 140164194086912, 140164202475519, +SNULL, 140164202479615, 140164219260927, +STORE, 140164202475520, 140164202479615, +STORE, 140164202479616, 140164219260927, +SNULL, 140164202479616, 140164210868223, +STORE, 140164210868224, 140164219260927, +STORE, 140164202479616, 140164210868223, +SNULL, 140164210872319, 140164219260927, +STORE, 140164210868224, 140164210872319, +STORE, 140164210872320, 140164219260927, +SNULL, 140164076654591, 140164085043199, +STORE, 140164076650496, 140164076654591, +STORE, 140164076654592, 140164085043199, +STORE, 140164068257792, 140164076650495, +SNULL, 140164068261887, 140164076650495, +STORE, 140164068257792, 140164068261887, +STORE, 140164068261888, 140164076650495, +STORE, 140165753053184, 140165753081855, +STORE, 140165725851648, 140165728043007, +SNULL, 140165725851648, 140165725941759, +STORE, 140165725941760, 140165728043007, +STORE, 140165725851648, 140165725941759, +SNULL, 140165728034815, 140165728043007, +STORE, 140165725941760, 140165728034815, +STORE, 140165728034816, 140165728043007, +ERASE, 140165728034816, 140165728043007, +STORE, 140165728034816, 140165728043007, +SNULL, 140165728038911, 140165728043007, +STORE, 140165728034816, 140165728038911, +STORE, 140165728038912, 140165728043007, +ERASE, 140165753053184, 140165753081855, +ERASE, 140164638666752, 140164638670847, +ERASE, 140164638670848, 140164647059455, +ERASE, 140165091676160, 140165091680255, +ERASE, 140165091680256, 140165100068863, +ERASE, 140164613488640, 140164613492735, +ERASE, 140164613492736, 140164621881343, +ERASE, 140164319907840, 140164319911935, +ERASE, 140164319911936, 140164328300543, +ERASE, 140165620154368, 140165620158463, +ERASE, 140165620158464, 140165628547071, +ERASE, 140164798062592, 140164798066687, +ERASE, 140164798066688, 140164806455295, +ERASE, 140164789669888, 140164789673983, +ERASE, 140164789673984, 140164798062591, +ERASE, 140164965851136, 140164965855231, +ERASE, 140164965855232, 140164974243839, +ERASE, 140165074890752, 140165074894847, +ERASE, 140165074894848, 140165083283455, +ERASE, 140164672237568, 140164672241663, +ERASE, 140164672241664, 140164680630271, +ERASE, 140164454125568, 140164454129663, +ERASE, 140164454129664, 140164462518271, +ERASE, 140165200715776, 140165200719871, +ERASE, 140165200719872, 140165209108479, +ERASE, 140164932280320, 140164932284415, +ERASE, 140164932284416, 140164940673023, +ERASE, 140164663844864, 140164663848959, +ERASE, 140164663848960, 140164672237567, +ERASE, 140164697415680, 140164697419775, +ERASE, 140164697419776, 140164705808383, +ERASE, 140164831633408, 140164831637503, +ERASE, 140164831637504, 140164840026111, +ERASE, 140165192323072, 140165192327167, +ERASE, 140165192327168, 140165200715775, +ERASE, 140165108461568, 140165108465663, +ERASE, 140165108465664, 140165116854271, +ERASE, 140164840026112, 140164840030207, +ERASE, 140164840030208, 140164848418815, +ERASE, 140164647059456, 140164647063551, +ERASE, 140164647063552, 140164655452159, +ERASE, 140165083283456, 140165083287551, +ERASE, 140165083287552, 140165091676159, +ERASE, 140164923887616, 140164923891711, +ERASE, 140164923891712, 140164932280319, +ERASE, 140164823240704, 140164823244799, +ERASE, 140164823244800, 140164831633407, +ERASE, 140164227653632, 140164227657727, +ERASE, 140164227657728, 140164236046335, +ERASE, 140164957458432, 140164957462527, +ERASE, 140164957462528, 140164965851135, +ERASE, 140164680630272, 140164680634367, +ERASE, 140164680634368, 140164689022975, +ERASE, 140164974243840, 140164974247935, +ERASE, 140164974247936, 140164982636543, +ERASE, 140165066498048, 140165066502143, +ERASE, 140165066502144, 140165074890751, +ERASE, 140164621881344, 140164621885439, +ERASE, 140164621885440, 140164630274047, +ERASE, 140164949065728, 140164949069823, +ERASE, 140164949069824, 140164957458431, +ERASE, 140164588310528, 140164588314623, +ERASE, 140164588314624, 140164596703231, +ERASE, 140164806455296, 140164806459391, +ERASE, 140164806459392, 140164814847999, +ERASE, 140164940673024, 140164940677119, +ERASE, 140164940677120, 140164949065727, +ERASE, 140164596703232, 140164596707327, +ERASE, 140164596707328, 140164605095935, +ERASE, 140164605095936, 140164605100031, +ERASE, 140164605100032, 140164613488639, +ERASE, 140164655452160, 140164655456255, +ERASE, 140164655456256, 140164663844863, +ERASE, 140164705808384, 140164705812479, +ERASE, 140164705812480, 140164714201087, +ERASE, 140164689022976, 140164689027071, +ERASE, 140164689027072, 140164697415679, +ERASE, 140164630274048, 140164630278143, +ERASE, 140164630278144, 140164638666751, +ERASE, 140164479303680, 140164479307775, +ERASE, 140164479307776, 140164487696383, +ERASE, 140164236046336, 140164236050431, +ERASE, 140164236050432, 140164244439039, +ERASE, 140164085043200, 140164085047295, +ERASE, 140164085047296, 140164093435903, +ERASE, 140164345085952, 140164345090047, +ERASE, 140164345090048, 140164353478655, +ERASE, 140164101828608, 140164101832703, +ERASE, 140164101832704, 140164110221311, +ERASE, 140164370264064, 140164370268159, +ERASE, 140164370268160, 140164378656767, +ERASE, 140164336693248, 140164336697343, +ERASE, 140164336697344, 140164345085951, +ERASE, 140164194082816, 140164194086911, +ERASE, 140164194086912, 140164202475519, +ERASE, 140164353478656, 140164353482751, +ERASE, 140164353482752, 140164361871359, +ERASE, 140164210868224, 140164210872319, +ERASE, 140164210872320, 140164219260927, +ERASE, 140164814848000, 140164814852095, +ERASE, 140164814852096, 140164823240703, +ERASE, 140164504481792, 140164504485887, +ERASE, 140164504485888, 140164512874495, +ERASE, 140165100068864, 140165100072959, +ERASE, 140165100072960, 140165108461567, +ERASE, 140164361871360, 140164361875455, +ERASE, 140164361875456, 140164370264063, +ERASE, 140164470910976, 140164470915071, +ERASE, 140164470915072, 140164479303679, +ERASE, 140164076650496, 140164076654591, +ERASE, 140164076654592, 140164085043199, +ERASE, 140164202475520, 140164202479615, +ERASE, 140164202479616, 140164210868223, +ERASE, 140164462518272, 140164462522367, +ERASE, 140164462522368, 140164470910975, +ERASE, 140165351718912, 140165351723007, +ERASE, 140165351723008, 140165360111615, +ERASE, 140164328300544, 140164328304639, +ERASE, 140164328304640, 140164336693247, +ERASE, 140164093435904, 140164093439999, +ERASE, 140164093440000, 140164101828607, +ERASE, 140165603368960, 140165603373055, +ERASE, 140165603373056, 140165611761663, +ERASE, 140165368504320, 140165368508415, +ERASE, 140165368508416, 140165376897023, +ERASE, 140165334933504, 140165334937599, +ERASE, 140165334937600, 140165343326207, +ERASE, 140165594976256, 140165594980351, +ERASE, 140165594980352, 140165603368959, +ERASE, 140164487696384, 140164487700479, +ERASE, 140164487700480, 140164496089087, +ERASE, 140164219260928, 140164219265023, +ERASE, 140164219265024, 140164227653631, +ERASE, 140164185690112, 140164185694207, +ERASE, 140164185694208, 140164194082815, +ERASE, 140164068257792, 140164068261887, +ERASE, 140164068261888, 140164076650495, +ERASE, 140165225893888, 140165225897983, +ERASE, 140165225897984, 140165234286591, +ERASE, 140165058105344, 140165058109439, + }; + unsigned long set31[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730890784768, 140737488351231, +SNULL, 140730890788863, 140737488351231, +STORE, 140730890784768, 140730890788863, +STORE, 140730890653696, 140730890788863, +STORE, 94577123659776, 94577125912575, +SNULL, 94577123790847, 94577125912575, +STORE, 94577123659776, 94577123790847, +STORE, 94577123790848, 94577125912575, +ERASE, 94577123790848, 94577125912575, +STORE, 94577125883904, 94577125892095, +STORE, 94577125892096, 94577125912575, +STORE, 140624060407808, 140624062660607, +SNULL, 140624060551167, 140624062660607, +STORE, 140624060407808, 140624060551167, +STORE, 140624060551168, 140624062660607, +ERASE, 140624060551168, 140624062660607, +STORE, 140624062648320, 140624062656511, +STORE, 140624062656512, 140624062660607, +STORE, 140730892140544, 140730892144639, +STORE, 140730892128256, 140730892140543, +STORE, 140624062619648, 140624062648319, +STORE, 140624062611456, 140624062619647, +STORE, 140624058191872, 140624060407807, +SNULL, 140624058191872, 140624058290175, +STORE, 140624058290176, 140624060407807, +STORE, 140624058191872, 140624058290175, +SNULL, 140624060383231, 140624060407807, +STORE, 140624058290176, 140624060383231, +STORE, 140624060383232, 140624060407807, +SNULL, 140624060383232, 140624060391423, +STORE, 140624060391424, 140624060407807, +STORE, 140624060383232, 140624060391423, +ERASE, 140624060383232, 140624060391423, +STORE, 140624060383232, 140624060391423, +ERASE, 140624060391424, 140624060407807, +STORE, 140624060391424, 140624060407807, +STORE, 140624054394880, 140624058191871, +SNULL, 140624054394880, 140624056053759, +STORE, 140624056053760, 140624058191871, +STORE, 140624054394880, 140624056053759, +SNULL, 140624058150911, 140624058191871, +STORE, 140624056053760, 140624058150911, +STORE, 140624058150912, 140624058191871, +SNULL, 140624058150912, 140624058175487, +STORE, 140624058175488, 140624058191871, +STORE, 140624058150912, 140624058175487, +ERASE, 140624058150912, 140624058175487, +STORE, 140624058150912, 140624058175487, +ERASE, 140624058175488, 140624058191871, +STORE, 140624058175488, 140624058191871, +STORE, 140624062603264, 140624062619647, +SNULL, 140624058167295, 140624058175487, +STORE, 140624058150912, 140624058167295, +STORE, 140624058167296, 140624058175487, +SNULL, 140624060387327, 140624060391423, +STORE, 140624060383232, 140624060387327, +STORE, 140624060387328, 140624060391423, +SNULL, 94577125887999, 94577125892095, +STORE, 94577125883904, 94577125887999, +STORE, 94577125888000, 94577125892095, +SNULL, 140624062652415, 140624062656511, +STORE, 140624062648320, 140624062652415, +STORE, 140624062652416, 140624062656511, +ERASE, 140624062619648, 140624062648319, +STORE, 94577157709824, 94577157844991, +STORE, 140624046002176, 140624054394879, +SNULL, 140624046006271, 140624054394879, +STORE, 140624046002176, 140624046006271, +STORE, 140624046006272, 140624054394879, +STORE, 140624037609472, 140624046002175, +STORE, 140623903391744, 140624037609471, +SNULL, 140623903391744, 140623940157439, +STORE, 140623940157440, 140624037609471, +STORE, 140623903391744, 140623940157439, +ERASE, 140623903391744, 140623940157439, +SNULL, 140624007266303, 140624037609471, +STORE, 140623940157440, 140624007266303, +STORE, 140624007266304, 140624037609471, +ERASE, 140624007266304, 140624037609471, +SNULL, 140623940292607, 140624007266303, +STORE, 140623940157440, 140623940292607, +STORE, 140623940292608, 140624007266303, +SNULL, 140624037613567, 140624046002175, +STORE, 140624037609472, 140624037613567, +STORE, 140624037613568, 140624046002175, +STORE, 140624029216768, 140624037609471, +SNULL, 140624029220863, 140624037609471, +STORE, 140624029216768, 140624029220863, +STORE, 140624029220864, 140624037609471, +STORE, 140624020824064, 140624029216767, +SNULL, 140624020828159, 140624029216767, +STORE, 140624020824064, 140624020828159, +STORE, 140624020828160, 140624029216767, +STORE, 140624012431360, 140624020824063, +SNULL, 140624012435455, 140624020824063, +STORE, 140624012431360, 140624012435455, +STORE, 140624012435456, 140624020824063, +STORE, 140623931764736, 140623940157439, +STORE, 140623797547008, 140623931764735, +SNULL, 140623797547008, 140623805939711, +STORE, 140623805939712, 140623931764735, +STORE, 140623797547008, 140623805939711, +ERASE, 140623797547008, 140623805939711, +SNULL, 140623873048575, 140623931764735, +STORE, 140623805939712, 140623873048575, +STORE, 140623873048576, 140623931764735, +ERASE, 140623873048576, 140623931764735, +STORE, 140623923372032, 140623940157439, +STORE, 140623914979328, 140623940157439, +STORE, 140623906586624, 140623940157439, +STORE, 140623671721984, 140623873048575, +SNULL, 140623738830847, 140623873048575, +STORE, 140623671721984, 140623738830847, +STORE, 140623738830848, 140623873048575, +SNULL, 140623738830848, 140623805939711, +STORE, 140623805939712, 140623873048575, +STORE, 140623738830848, 140623805939711, +ERASE, 140623738830848, 140623805939711, +SNULL, 140623806074879, 140623873048575, +STORE, 140623805939712, 140623806074879, +STORE, 140623806074880, 140623873048575, +SNULL, 140623906586624, 140623931764735, +STORE, 140623931764736, 140623940157439, +STORE, 140623906586624, 140623931764735, +SNULL, 140623931768831, 140623940157439, +STORE, 140623931764736, 140623931768831, +STORE, 140623931768832, 140623940157439, +STORE, 140623537504256, 140623738830847, +SNULL, 140623537504256, 140623671721983, +STORE, 140623671721984, 140623738830847, +STORE, 140623537504256, 140623671721983, +SNULL, 140623671857151, 140623738830847, +STORE, 140623671721984, 140623671857151, +STORE, 140623671857152, 140623738830847, +SNULL, 140623604613119, 140623671721983, +STORE, 140623537504256, 140623604613119, +STORE, 140623604613120, 140623671721983, +ERASE, 140623604613120, 140623671721983, +SNULL, 140623537639423, 140623604613119, +STORE, 140623537504256, 140623537639423, +STORE, 140623537639424, 140623604613119, +STORE, 140623537639424, 140623671721983, +SNULL, 140623537639424, 140623604613119, +STORE, 140623604613120, 140623671721983, +STORE, 140623537639424, 140623604613119, +SNULL, 140623604748287, 140623671721983, +STORE, 140623604613120, 140623604748287, +STORE, 140623604748288, 140623671721983, +STORE, 140623898193920, 140623931764735, +SNULL, 140623898193920, 140623923372031, +STORE, 140623923372032, 140623931764735, +STORE, 140623898193920, 140623923372031, +SNULL, 140623923376127, 140623931764735, +STORE, 140623923372032, 140623923376127, +STORE, 140623923376128, 140623931764735, +STORE, 140623889801216, 140623923372031, +SNULL, 140623889801216, 140623898193919, +STORE, 140623898193920, 140623923372031, +STORE, 140623889801216, 140623898193919, +SNULL, 140623898198015, 140623923372031, +STORE, 140623898193920, 140623898198015, +STORE, 140623898198016, 140623923372031, +SNULL, 140623889805311, 140623898193919, +STORE, 140623889801216, 140623889805311, +STORE, 140623889805312, 140623898193919, +SNULL, 140623898198016, 140623906586623, +STORE, 140623906586624, 140623923372031, +STORE, 140623898198016, 140623906586623, +SNULL, 140623906590719, 140623923372031, +STORE, 140623906586624, 140623906590719, +STORE, 140623906590720, 140623923372031, +STORE, 140623881408512, 140623889801215, +SNULL, 140623906590720, 140623914979327, +STORE, 140623914979328, 140623923372031, +STORE, 140623906590720, 140623914979327, +SNULL, 140623914983423, 140623923372031, +STORE, 140623914979328, 140623914983423, +STORE, 140623914983424, 140623923372031, +SNULL, 140623881412607, 140623889801215, +STORE, 140623881408512, 140623881412607, +STORE, 140623881412608, 140623889801215, +STORE, 140623797547008, 140623805939711, +STORE, 140623789154304, 140623805939711, +STORE, 140623780761600, 140623805939711, +SNULL, 140623780761600, 140623789154303, +STORE, 140623789154304, 140623805939711, +STORE, 140623780761600, 140623789154303, +SNULL, 140623789158399, 140623805939711, +STORE, 140623789154304, 140623789158399, +STORE, 140623789158400, 140623805939711, +STORE, 140623772368896, 140623789154303, +STORE, 140623763976192, 140623789154303, +SNULL, 140623763976192, 140623780761599, +STORE, 140623780761600, 140623789154303, +STORE, 140623763976192, 140623780761599, +SNULL, 140623780765695, 140623789154303, +STORE, 140623780761600, 140623780765695, +STORE, 140623780765696, 140623789154303, +SNULL, 140623789158400, 140623797547007, +STORE, 140623797547008, 140623805939711, +STORE, 140623789158400, 140623797547007, +SNULL, 140623797551103, 140623805939711, +STORE, 140623797547008, 140623797551103, +STORE, 140623797551104, 140623805939711, +SNULL, 140623763976192, 140623772368895, +STORE, 140623772368896, 140623780761599, +STORE, 140623763976192, 140623772368895, +SNULL, 140623772372991, 140623780761599, +STORE, 140623772368896, 140623772372991, +STORE, 140623772372992, 140623780761599, +SNULL, 140623763980287, 140623772368895, +STORE, 140623763976192, 140623763980287, +STORE, 140623763980288, 140623772368895, +STORE, 140623755583488, 140623763976191, +STORE, 140623747190784, 140623763976191, +SNULL, 140623747190784, 140623755583487, +STORE, 140623755583488, 140623763976191, +STORE, 140623747190784, 140623755583487, +SNULL, 140623755587583, 140623763976191, +STORE, 140623755583488, 140623755587583, +STORE, 140623755587584, 140623763976191, +STORE, 140623529111552, 140623537504255, +SNULL, 140623747194879, 140623755583487, +STORE, 140623747190784, 140623747194879, +STORE, 140623747194880, 140623755583487, +SNULL, 140623529115647, 140623537504255, +STORE, 140623529111552, 140623529115647, +STORE, 140623529115648, 140623537504255, +STORE, 140623520718848, 140623529111551, +SNULL, 140623520722943, 140623529111551, +STORE, 140623520718848, 140623520722943, +STORE, 140623520722944, 140623529111551, +STORE, 140623512326144, 140623520718847, +STORE, 140623503933440, 140623520718847, +STORE, 140623495540736, 140623520718847, +STORE, 140623361323008, 140623495540735, +STORE, 140623227105280, 140623495540735, +STORE, 140623218712576, 140623227105279, +STORE, 140623084494848, 140623218712575, +STORE, 140623076102144, 140623084494847, +STORE, 140622941884416, 140623076102143, +SNULL, 140622941884416, 140623000633343, +STORE, 140623000633344, 140623076102143, +STORE, 140622941884416, 140623000633343, +ERASE, 140622941884416, 140623000633343, +STORE, 140622992240640, 140623000633343, +STORE, 140622983847936, 140623000633343, +STORE, 140622849630208, 140622983847935, +STORE, 140622841237504, 140622849630207, +SNULL, 140622849630208, 140622866415615, +STORE, 140622866415616, 140622983847935, +STORE, 140622849630208, 140622866415615, +ERASE, 140622849630208, 140622866415615, +STORE, 140622858022912, 140622866415615, +SNULL, 140622933524479, 140622983847935, +STORE, 140622866415616, 140622933524479, +STORE, 140622933524480, 140622983847935, +ERASE, 140622933524480, 140622983847935, +STORE, 140622975455232, 140623000633343, +STORE, 140622707019776, 140622841237503, +STORE, 140622967062528, 140623000633343, +STORE, 140622572802048, 140622841237503, +STORE, 140622958669824, 140623000633343, +STORE, 140622438584320, 140622841237503, +STORE, 140622950277120, 140623000633343, +SNULL, 140622858027007, 140622866415615, +STORE, 140622858022912, 140622858027007, +STORE, 140622858027008, 140622866415615, +STORE, 140622941884416, 140623000633343, +STORE, 140622841237504, 140622858022911, +SNULL, 140622841237504, 140622849630207, +STORE, 140622849630208, 140622858022911, +STORE, 140622841237504, 140622849630207, +SNULL, 140622849634303, 140622858022911, +STORE, 140622849630208, 140622849634303, +STORE, 140622849634304, 140622858022911, +STORE, 140622430191616, 140622438584319, +SNULL, 140622430195711, 140622438584319, +STORE, 140622430191616, 140622430195711, +STORE, 140622430195712, 140622438584319, +SNULL, 140623361323007, 140623495540735, +STORE, 140623227105280, 140623361323007, +STORE, 140623361323008, 140623495540735, +SNULL, 140623361323008, 140623403286527, +STORE, 140623403286528, 140623495540735, +STORE, 140623361323008, 140623403286527, +ERASE, 140623361323008, 140623403286527, +SNULL, 140623470395391, 140623495540735, +STORE, 140623403286528, 140623470395391, +STORE, 140623470395392, 140623495540735, +ERASE, 140623470395392, 140623495540735, +SNULL, 140623227105280, 140623269068799, +STORE, 140623269068800, 140623361323007, +STORE, 140623227105280, 140623269068799, +ERASE, 140623227105280, 140623269068799, +SNULL, 140623084494848, 140623134851071, +STORE, 140623134851072, 140623218712575, +STORE, 140623084494848, 140623134851071, +ERASE, 140623084494848, 140623134851071, +SNULL, 140623201959935, 140623218712575, +STORE, 140623134851072, 140623201959935, +STORE, 140623201959936, 140623218712575, +ERASE, 140623201959936, 140623218712575, +SNULL, 140623067742207, 140623076102143, +STORE, 140623000633344, 140623067742207, +STORE, 140623067742208, 140623076102143, +ERASE, 140623067742208, 140623076102143, +STORE, 140622295973888, 140622430191615, +SNULL, 140622295973888, 140622329544703, +STORE, 140622329544704, 140622430191615, +STORE, 140622295973888, 140622329544703, +ERASE, 140622295973888, 140622329544703, +SNULL, 140622866550783, 140622933524479, +STORE, 140622866415616, 140622866550783, +STORE, 140622866550784, 140622933524479, +SNULL, 140622707019775, 140622841237503, +STORE, 140622438584320, 140622707019775, +STORE, 140622707019776, 140622841237503, +SNULL, 140622707019776, 140622732197887, +STORE, 140622732197888, 140622841237503, +STORE, 140622707019776, 140622732197887, +ERASE, 140622707019776, 140622732197887, +SNULL, 140622799306751, 140622841237503, +STORE, 140622732197888, 140622799306751, +STORE, 140622799306752, 140622841237503, +ERASE, 140622799306752, 140622841237503, +SNULL, 140622572802047, 140622707019775, +STORE, 140622438584320, 140622572802047, +STORE, 140622572802048, 140622707019775, +SNULL, 140622572802048, 140622597980159, +STORE, 140622597980160, 140622707019775, +STORE, 140622572802048, 140622597980159, +ERASE, 140622572802048, 140622597980159, +SNULL, 140622438584320, 140622463762431, +STORE, 140622463762432, 140622572802047, +STORE, 140622438584320, 140622463762431, +ERASE, 140622438584320, 140622463762431, +SNULL, 140622530871295, 140622572802047, +STORE, 140622463762432, 140622530871295, +STORE, 140622530871296, 140622572802047, +ERASE, 140622530871296, 140622572802047, +STORE, 140622195326976, 140622430191615, +SNULL, 140622262435839, 140622430191615, +STORE, 140622195326976, 140622262435839, +STORE, 140622262435840, 140622430191615, +SNULL, 140622262435840, 140622329544703, +STORE, 140622329544704, 140622430191615, +STORE, 140622262435840, 140622329544703, +ERASE, 140622262435840, 140622329544703, +SNULL, 140622841241599, 140622849630207, +STORE, 140622841237504, 140622841241599, +STORE, 140622841241600, 140622849630207, +STORE, 140623487148032, 140623520718847, +STORE, 140623478755328, 140623520718847, +SNULL, 140622941884416, 140622983847935, +STORE, 140622983847936, 140623000633343, +STORE, 140622941884416, 140622983847935, +SNULL, 140622983852031, 140623000633343, +STORE, 140622983847936, 140622983852031, +STORE, 140622983852032, 140623000633343, +STORE, 140623394893824, 140623403286527, +SNULL, 140623394897919, 140623403286527, +STORE, 140623394893824, 140623394897919, +STORE, 140623394897920, 140623403286527, +SNULL, 140623403421695, 140623470395391, +STORE, 140623403286528, 140623403421695, +STORE, 140623403421696, 140623470395391, +SNULL, 140623478755328, 140623503933439, +STORE, 140623503933440, 140623520718847, +STORE, 140623478755328, 140623503933439, +SNULL, 140623503937535, 140623520718847, +STORE, 140623503933440, 140623503937535, +STORE, 140623503937536, 140623520718847, +SNULL, 140623336177663, 140623361323007, +STORE, 140623269068800, 140623336177663, +STORE, 140623336177664, 140623361323007, +ERASE, 140623336177664, 140623361323007, +SNULL, 140623269203967, 140623336177663, +STORE, 140623269068800, 140623269203967, +STORE, 140623269203968, 140623336177663, +SNULL, 140623134986239, 140623201959935, +STORE, 140623134851072, 140623134986239, +STORE, 140623134986240, 140623201959935, +SNULL, 140623000768511, 140623067742207, +STORE, 140623000633344, 140623000768511, +STORE, 140623000768512, 140623067742207, +SNULL, 140622396653567, 140622430191615, +STORE, 140622329544704, 140622396653567, +STORE, 140622396653568, 140622430191615, +ERASE, 140622396653568, 140622430191615, +SNULL, 140622732333055, 140622799306751, +STORE, 140622732197888, 140622732333055, +STORE, 140622732333056, 140622799306751, +SNULL, 140622941884416, 140622975455231, +STORE, 140622975455232, 140622983847935, +STORE, 140622941884416, 140622975455231, +SNULL, 140622975459327, 140622983847935, +STORE, 140622975455232, 140622975459327, +STORE, 140622975459328, 140622983847935, +SNULL, 140622665089023, 140622707019775, +STORE, 140622597980160, 140622665089023, +STORE, 140622665089024, 140622707019775, +ERASE, 140622665089024, 140622707019775, +SNULL, 140622598115327, 140622665089023, +STORE, 140622597980160, 140622598115327, +STORE, 140622598115328, 140622665089023, +SNULL, 140622463897599, 140622530871295, +STORE, 140622463762432, 140622463897599, +STORE, 140622463897600, 140622530871295, +SNULL, 140622195462143, 140622262435839, +STORE, 140622195326976, 140622195462143, +STORE, 140622195462144, 140622262435839, +STORE, 140623386501120, 140623394893823, +SNULL, 140622941884416, 140622950277119, +STORE, 140622950277120, 140622975455231, +STORE, 140622941884416, 140622950277119, +SNULL, 140622950281215, 140622975455231, +STORE, 140622950277120, 140622950281215, +STORE, 140622950281216, 140622975455231, +SNULL, 140622941888511, 140622950277119, +STORE, 140622941884416, 140622941888511, +STORE, 140622941888512, 140622950277119, +STORE, 140623378108416, 140623394893823, +SNULL, 140623478755328, 140623495540735, +STORE, 140623495540736, 140623503933439, +STORE, 140623478755328, 140623495540735, +SNULL, 140623495544831, 140623503933439, +STORE, 140623495540736, 140623495544831, +STORE, 140623495544832, 140623503933439, +SNULL, 140623478755328, 140623487148031, +STORE, 140623487148032, 140623495540735, +STORE, 140623478755328, 140623487148031, +SNULL, 140623487152127, 140623495540735, +STORE, 140623487148032, 140623487152127, +STORE, 140623487152128, 140623495540735, +SNULL, 140623218716671, 140623227105279, +STORE, 140623218712576, 140623218716671, +STORE, 140623218716672, 140623227105279, +SNULL, 140623076106239, 140623084494847, +STORE, 140623076102144, 140623076106239, +STORE, 140623076106240, 140623084494847, +SNULL, 140622329679871, 140622396653567, +STORE, 140622329544704, 140622329679871, +STORE, 140622329679872, 140622396653567, +SNULL, 140622950281216, 140622958669823, +STORE, 140622958669824, 140622975455231, +STORE, 140622950281216, 140622958669823, +SNULL, 140622958673919, 140622975455231, +STORE, 140622958669824, 140622958673919, +STORE, 140622958673920, 140622975455231, +SNULL, 140623503937536, 140623512326143, +STORE, 140623512326144, 140623520718847, +STORE, 140623503937536, 140623512326143, +SNULL, 140623512330239, 140623520718847, +STORE, 140623512326144, 140623512330239, +STORE, 140623512330240, 140623520718847, +SNULL, 140623378108416, 140623386501119, +STORE, 140623386501120, 140623394893823, +STORE, 140623378108416, 140623386501119, +SNULL, 140623386505215, 140623394893823, +STORE, 140623386501120, 140623386505215, +STORE, 140623386505216, 140623394893823, +STORE, 140623369715712, 140623386501119, +STORE, 140623361323008, 140623386501119, +STORE, 140623352930304, 140623386501119, +SNULL, 140623352930304, 140623361323007, +STORE, 140623361323008, 140623386501119, +STORE, 140623352930304, 140623361323007, +SNULL, 140623361327103, 140623386501119, +STORE, 140623361323008, 140623361327103, +STORE, 140623361327104, 140623386501119, +SNULL, 140623478759423, 140623487148031, +STORE, 140623478755328, 140623478759423, +STORE, 140623478759424, 140623487148031, +STORE, 140623344537600, 140623361323007, +STORE, 140623260676096, 140623269068799, +SNULL, 140622958673920, 140622967062527, +STORE, 140622967062528, 140622975455231, +STORE, 140622958673920, 140622967062527, +SNULL, 140622967066623, 140622975455231, +STORE, 140622967062528, 140622967066623, +STORE, 140622967066624, 140622975455231, +STORE, 140623252283392, 140623269068799, +STORE, 140623243890688, 140623269068799, +SNULL, 140622983852032, 140622992240639, +STORE, 140622992240640, 140623000633343, +STORE, 140622983852032, 140622992240639, +SNULL, 140622992244735, 140623000633343, +STORE, 140622992240640, 140622992244735, +STORE, 140622992244736, 140623000633343, +STORE, 140623235497984, 140623269068799, +STORE, 140623218716672, 140623235497983, +STORE, 140623210319872, 140623218712575, +STORE, 140623126458368, 140623134851071, +SNULL, 140623210323967, 140623218712575, +STORE, 140623210319872, 140623210323967, +STORE, 140623210323968, 140623218712575, +SNULL, 140623218716672, 140623227105279, +STORE, 140623227105280, 140623235497983, +STORE, 140623218716672, 140623227105279, +SNULL, 140623227109375, 140623235497983, +STORE, 140623227105280, 140623227109375, +STORE, 140623227109376, 140623235497983, +STORE, 140623118065664, 140623134851071, +STORE, 140623109672960, 140623134851071, +SNULL, 140623109677055, 140623134851071, +STORE, 140623109672960, 140623109677055, +STORE, 140623109677056, 140623134851071, +STORE, 140623101280256, 140623109672959, +STORE, 140623092887552, 140623109672959, +SNULL, 140623092887552, 140623101280255, +STORE, 140623101280256, 140623109672959, +STORE, 140623092887552, 140623101280255, +SNULL, 140623101284351, 140623109672959, +STORE, 140623101280256, 140623101284351, +STORE, 140623101284352, 140623109672959, +SNULL, 140623361327104, 140623378108415, +STORE, 140623378108416, 140623386501119, +STORE, 140623361327104, 140623378108415, +SNULL, 140623378112511, 140623386501119, +STORE, 140623378108416, 140623378112511, +STORE, 140623378112512, 140623386501119, +SNULL, 140623235497984, 140623243890687, +STORE, 140623243890688, 140623269068799, +STORE, 140623235497984, 140623243890687, +SNULL, 140623243894783, 140623269068799, +STORE, 140623243890688, 140623243894783, +STORE, 140623243894784, 140623269068799, +SNULL, 140623361327104, 140623369715711, +STORE, 140623369715712, 140623378108415, +STORE, 140623361327104, 140623369715711, +SNULL, 140623369719807, 140623378108415, +STORE, 140623369715712, 140623369719807, +STORE, 140623369719808, 140623378108415, +SNULL, 140623243894784, 140623252283391, +STORE, 140623252283392, 140623269068799, +STORE, 140623243894784, 140623252283391, +SNULL, 140623252287487, 140623269068799, +STORE, 140623252283392, 140623252287487, +STORE, 140623252287488, 140623269068799, +SNULL, 140623235502079, 140623243890687, +STORE, 140623235497984, 140623235502079, +STORE, 140623235502080, 140623243890687, +SNULL, 140623344541695, 140623361323007, +STORE, 140623344537600, 140623344541695, +STORE, 140623344541696, 140623361323007, +STORE, 140623076106240, 140623092887551, +SNULL, 140623076106240, 140623084494847, +STORE, 140623084494848, 140623092887551, +STORE, 140623076106240, 140623084494847, +SNULL, 140623084498943, 140623092887551, +STORE, 140623084494848, 140623084498943, +STORE, 140623084498944, 140623092887551, +SNULL, 140623344541696, 140623352930303, +STORE, 140623352930304, 140623361323007, +STORE, 140623344541696, 140623352930303, +SNULL, 140623352934399, 140623361323007, +STORE, 140623352930304, 140623352934399, +STORE, 140623352934400, 140623361323007, +SNULL, 140623109677056, 140623118065663, +STORE, 140623118065664, 140623134851071, +STORE, 140623109677056, 140623118065663, +SNULL, 140623118069759, 140623134851071, +STORE, 140623118065664, 140623118069759, +STORE, 140623118069760, 140623134851071, +STORE, 140622832844800, 140622841237503, +STORE, 140622824452096, 140622841237503, +SNULL, 140622824452096, 140622832844799, +STORE, 140622832844800, 140622841237503, +STORE, 140622824452096, 140622832844799, +SNULL, 140622832848895, 140622841237503, +STORE, 140622832844800, 140622832848895, +STORE, 140622832848896, 140622841237503, +STORE, 140622816059392, 140622832844799, +SNULL, 140623092891647, 140623101280255, +STORE, 140623092887552, 140623092891647, +STORE, 140623092891648, 140623101280255, +SNULL, 140623118069760, 140623126458367, +STORE, 140623126458368, 140623134851071, +STORE, 140623118069760, 140623126458367, +SNULL, 140623126462463, 140623134851071, +STORE, 140623126458368, 140623126462463, +STORE, 140623126462464, 140623134851071, +SNULL, 140623252287488, 140623260676095, +STORE, 140623260676096, 140623269068799, +STORE, 140623252287488, 140623260676095, +SNULL, 140623260680191, 140623269068799, +STORE, 140623260676096, 140623260680191, +STORE, 140623260680192, 140623269068799, +STORE, 140622807666688, 140622832844799, +STORE, 140622723805184, 140622732197887, +STORE, 140622715412480, 140622732197887, +STORE, 140622707019776, 140622732197887, +SNULL, 140622707023871, 140622732197887, +STORE, 140622707019776, 140622707023871, +STORE, 140622707023872, 140622732197887, +STORE, 140622698627072, 140622707019775, +STORE, 140622690234368, 140622707019775, +SNULL, 140622690238463, 140622707019775, +STORE, 140622690234368, 140622690238463, +STORE, 140622690238464, 140622707019775, +SNULL, 140622807666688, 140622816059391, +STORE, 140622816059392, 140622832844799, +STORE, 140622807666688, 140622816059391, +SNULL, 140622816063487, 140622832844799, +STORE, 140622816059392, 140622816063487, +STORE, 140622816063488, 140622832844799, +STORE, 140622681841664, 140622690234367, +STORE, 140622673448960, 140622690234367, +SNULL, 140622673453055, 140622690234367, +STORE, 140622673448960, 140622673453055, +STORE, 140622673453056, 140622690234367, +STORE, 140622589587456, 140622597980159, +SNULL, 140622807670783, 140622816059391, +STORE, 140622807666688, 140622807670783, +STORE, 140622807670784, 140622816059391, +STORE, 140622581194752, 140622597980159, +SNULL, 140622581198847, 140622597980159, +STORE, 140622581194752, 140622581198847, +STORE, 140622581198848, 140622597980159, +SNULL, 140622816063488, 140622824452095, +STORE, 140622824452096, 140622832844799, +STORE, 140622816063488, 140622824452095, +SNULL, 140622824456191, 140622832844799, +STORE, 140622824452096, 140622824456191, +STORE, 140622824456192, 140622832844799, +STORE, 140622572802048, 140622581194751, +SNULL, 140622572806143, 140622581194751, +STORE, 140622572802048, 140622572806143, +STORE, 140622572806144, 140622581194751, +STORE, 140622564409344, 140622572802047, +STORE, 140622556016640, 140622572802047, +SNULL, 140622556016640, 140622564409343, +STORE, 140622564409344, 140622572802047, +STORE, 140622556016640, 140622564409343, +SNULL, 140622564413439, 140622572802047, +STORE, 140622564409344, 140622564413439, +STORE, 140622564413440, 140622572802047, +SNULL, 140622690238464, 140622698627071, +STORE, 140622698627072, 140622707019775, +STORE, 140622690238464, 140622698627071, +SNULL, 140622698631167, 140622707019775, +STORE, 140622698627072, 140622698631167, +STORE, 140622698631168, 140622707019775, +SNULL, 140622707023872, 140622723805183, +STORE, 140622723805184, 140622732197887, +STORE, 140622707023872, 140622723805183, +SNULL, 140622723809279, 140622732197887, +STORE, 140622723805184, 140622723809279, +STORE, 140622723809280, 140622732197887, +SNULL, 140622707023872, 140622715412479, +STORE, 140622715412480, 140622723805183, +STORE, 140622707023872, 140622715412479, +SNULL, 140622715416575, 140622723805183, +STORE, 140622715412480, 140622715416575, +STORE, 140622715416576, 140622723805183, +STORE, 140622547623936, 140622564409343, +SNULL, 140622547628031, 140622564409343, +STORE, 140622547623936, 140622547628031, +STORE, 140622547628032, 140622564409343, +STORE, 140622539231232, 140622547623935, +SNULL, 140622539235327, 140622547623935, +STORE, 140622539231232, 140622539235327, +STORE, 140622539235328, 140622547623935, +SNULL, 140622581198848, 140622589587455, +STORE, 140622589587456, 140622597980159, +STORE, 140622581198848, 140622589587455, +SNULL, 140622589591551, 140622597980159, +STORE, 140622589587456, 140622589591551, +STORE, 140622589591552, 140622597980159, +STORE, 140622455369728, 140622463762431, +SNULL, 140622455373823, 140622463762431, +STORE, 140622455369728, 140622455373823, +STORE, 140622455373824, 140622463762431, +STORE, 140622446977024, 140622455369727, +SNULL, 140622446981119, 140622455369727, +STORE, 140622446977024, 140622446981119, +STORE, 140622446981120, 140622455369727, +SNULL, 140622547628032, 140622556016639, +STORE, 140622556016640, 140622564409343, +STORE, 140622547628032, 140622556016639, +SNULL, 140622556020735, 140622564409343, +STORE, 140622556016640, 140622556020735, +STORE, 140622556020736, 140622564409343, +STORE, 140622430195712, 140622446977023, +STORE, 140622421798912, 140622430191615, +SNULL, 140622430195712, 140622438584319, +STORE, 140622438584320, 140622446977023, +STORE, 140622430195712, 140622438584319, +SNULL, 140622438588415, 140622446977023, +STORE, 140622438584320, 140622438588415, +STORE, 140622438588416, 140622446977023, +STORE, 140622413406208, 140622430191615, +STORE, 140622405013504, 140622430191615, +SNULL, 140622405013504, 140622413406207, +STORE, 140622413406208, 140622430191615, +STORE, 140622405013504, 140622413406207, +SNULL, 140622413410303, 140622430191615, +STORE, 140622413406208, 140622413410303, +STORE, 140622413410304, 140622430191615, +SNULL, 140622673453056, 140622681841663, +STORE, 140622681841664, 140622690234367, +STORE, 140622673453056, 140622681841663, +SNULL, 140622681845759, 140622690234367, +STORE, 140622681841664, 140622681845759, +STORE, 140622681845760, 140622690234367, +STORE, 140622321152000, 140622329544703, +SNULL, 140622413410304, 140622421798911, +STORE, 140622421798912, 140622430191615, +STORE, 140622413410304, 140622421798911, +SNULL, 140622421803007, 140622430191615, +STORE, 140622421798912, 140622421803007, +STORE, 140622421803008, 140622430191615, +STORE, 140622312759296, 140622329544703, +SNULL, 140622312763391, 140622329544703, +STORE, 140622312759296, 140622312763391, +STORE, 140622312763392, 140622329544703, +SNULL, 140622405017599, 140622413406207, +STORE, 140622405013504, 140622405017599, +STORE, 140622405017600, 140622413406207, +STORE, 140622304366592, 140622312759295, +SNULL, 140622304370687, 140622312759295, +STORE, 140622304366592, 140622304370687, +STORE, 140622304370688, 140622312759295, +SNULL, 140622312763392, 140622321151999, +STORE, 140622321152000, 140622329544703, +STORE, 140622312763392, 140622321151999, +SNULL, 140622321156095, 140622329544703, +STORE, 140622321152000, 140622321156095, +STORE, 140622321156096, 140622329544703, +STORE, 140624062619648, 140624062648319, +STORE, 140624010240000, 140624012431359, +SNULL, 140624010240000, 140624010330111, +STORE, 140624010330112, 140624012431359, +STORE, 140624010240000, 140624010330111, +SNULL, 140624012423167, 140624012431359, +STORE, 140624010330112, 140624012423167, +STORE, 140624012423168, 140624012431359, +ERASE, 140624012423168, 140624012431359, +STORE, 140624012423168, 140624012431359, +SNULL, 140624012427263, 140624012431359, +STORE, 140624012423168, 140624012427263, +STORE, 140624012427264, 140624012431359, +ERASE, 140624062619648, 140624062648319, +ERASE, 140622849630208, 140622849634303, +ERASE, 140622849634304, 140622858022911, +ERASE, 140623394893824, 140623394897919, +ERASE, 140623394897920, 140623403286527, +ERASE, 140623361323008, 140623361327103, +ERASE, 140623361327104, 140623369715711, +ERASE, 140623084494848, 140623084498943, +ERASE, 140623084498944, 140623092887551, +ERASE, 140623931764736, 140623931768831, +ERASE, 140623931768832, 140623940157439, +ERASE, 140622841237504, 140622841241599, +ERASE, 140622841241600, 140622849630207, +ERASE, 140623487148032, 140623487152127, +ERASE, 140623487152128, 140623495540735, +ERASE, 140623109672960, 140623109677055, +ERASE, 140623109677056, 140623118065663, +ERASE, 140622983847936, 140622983852031, +ERASE, 140622983852032, 140622992240639, +ERASE, 140623352930304, 140623352934399, +ERASE, 140623352934400, 140623361323007, +ERASE, 140622564409344, 140622564413439, +ERASE, 140622564413440, 140622572802047, +ERASE, 140622430191616, 140622430195711, +ERASE, 140622430195712, 140622438584319, +ERASE, 140622958669824, 140622958673919, +ERASE, 140622958673920, 140622967062527, +ERASE, 140622992240640, 140622992244735, +ERASE, 140622992244736, 140623000633343, +ERASE, 140623227105280, 140623227109375, +ERASE, 140623227109376, 140623235497983, +ERASE, 140622321152000, 140622321156095, +ERASE, 140622321156096, 140622329544703, +ERASE, 140622858022912, 140622858027007, +ERASE, 140622858027008, 140622866415615, +ERASE, 140622975455232, 140622975459327, +ERASE, 140622975459328, 140622983847935, +ERASE, 140623378108416, 140623378112511, +ERASE, 140623378112512, 140623386501119, +ERASE, 140623495540736, 140623495544831, +ERASE, 140623495544832, 140623503933439, +ERASE, 140623118065664, 140623118069759, +ERASE, 140623118069760, 140623126458367, +ERASE, 140622572802048, 140622572806143, +ERASE, 140622572806144, 140622581194751, +ERASE, 140622421798912, 140622421803007, +ERASE, 140622421803008, 140622430191615, +ERASE, 140622967062528, 140622967066623, +ERASE, 140622967066624, 140622975455231, +ERASE, 140623252283392, 140623252287487, +ERASE, 140623252287488, 140623260676095, +ERASE, 140622673448960, 140622673453055, +ERASE, 140622673453056, 140622681841663, +ERASE, 140623076102144, 140623076106239, +ERASE, 140623076106240, 140623084494847, +ERASE, 140623101280256, 140623101284351, +ERASE, 140623101284352, 140623109672959, +ERASE, 140622715412480, 140622715416575, +ERASE, 140622715416576, 140622723805183, +ERASE, 140622405013504, 140622405017599, +ERASE, 140622405017600, 140622413406207, +ERASE, 140623478755328, 140623478759423, +ERASE, 140623478759424, 140623487148031, +ERASE, 140623906586624, 140623906590719, +ERASE, 140623906590720, 140623914979327, +ERASE, 140622950277120, 140622950281215, +ERASE, 140622950281216, 140622958669823, + }; + unsigned long set32[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731244212224, 140737488351231, +SNULL, 140731244216319, 140737488351231, +STORE, 140731244212224, 140731244216319, +STORE, 140731244081152, 140731244216319, +STORE, 94427773984768, 94427776237567, +SNULL, 94427774115839, 94427776237567, +STORE, 94427773984768, 94427774115839, +STORE, 94427774115840, 94427776237567, +ERASE, 94427774115840, 94427776237567, +STORE, 94427776208896, 94427776217087, +STORE, 94427776217088, 94427776237567, +STORE, 140401464893440, 140401467146239, +SNULL, 140401465036799, 140401467146239, +STORE, 140401464893440, 140401465036799, +STORE, 140401465036800, 140401467146239, +ERASE, 140401465036800, 140401467146239, +STORE, 140401467133952, 140401467142143, +STORE, 140401467142144, 140401467146239, +STORE, 140731244507136, 140731244511231, +STORE, 140731244494848, 140731244507135, +STORE, 140401467105280, 140401467133951, +STORE, 140401467097088, 140401467105279, +STORE, 140401462677504, 140401464893439, +SNULL, 140401462677504, 140401462775807, +STORE, 140401462775808, 140401464893439, +STORE, 140401462677504, 140401462775807, +SNULL, 140401464868863, 140401464893439, +STORE, 140401462775808, 140401464868863, +STORE, 140401464868864, 140401464893439, +SNULL, 140401464868864, 140401464877055, +STORE, 140401464877056, 140401464893439, +STORE, 140401464868864, 140401464877055, +ERASE, 140401464868864, 140401464877055, +STORE, 140401464868864, 140401464877055, +ERASE, 140401464877056, 140401464893439, +STORE, 140401464877056, 140401464893439, +STORE, 140401458880512, 140401462677503, +SNULL, 140401458880512, 140401460539391, +STORE, 140401460539392, 140401462677503, +STORE, 140401458880512, 140401460539391, +SNULL, 140401462636543, 140401462677503, +STORE, 140401460539392, 140401462636543, +STORE, 140401462636544, 140401462677503, +SNULL, 140401462636544, 140401462661119, +STORE, 140401462661120, 140401462677503, +STORE, 140401462636544, 140401462661119, +ERASE, 140401462636544, 140401462661119, +STORE, 140401462636544, 140401462661119, +ERASE, 140401462661120, 140401462677503, +STORE, 140401462661120, 140401462677503, +STORE, 140401467088896, 140401467105279, +SNULL, 140401462652927, 140401462661119, +STORE, 140401462636544, 140401462652927, +STORE, 140401462652928, 140401462661119, +SNULL, 140401464872959, 140401464877055, +STORE, 140401464868864, 140401464872959, +STORE, 140401464872960, 140401464877055, +SNULL, 94427776212991, 94427776217087, +STORE, 94427776208896, 94427776212991, +STORE, 94427776212992, 94427776217087, +SNULL, 140401467138047, 140401467142143, +STORE, 140401467133952, 140401467138047, +STORE, 140401467138048, 140401467142143, +ERASE, 140401467105280, 140401467133951, +STORE, 94427784683520, 94427784818687, +STORE, 140401450487808, 140401458880511, +SNULL, 140401450491903, 140401458880511, +STORE, 140401450487808, 140401450491903, +STORE, 140401450491904, 140401458880511, +STORE, 140401442095104, 140401450487807, +STORE, 140401307877376, 140401442095103, +SNULL, 140401307877376, 140401340055551, +STORE, 140401340055552, 140401442095103, +STORE, 140401307877376, 140401340055551, +ERASE, 140401307877376, 140401340055551, +SNULL, 140401407164415, 140401442095103, +STORE, 140401340055552, 140401407164415, +STORE, 140401407164416, 140401442095103, +ERASE, 140401407164416, 140401442095103, +SNULL, 140401340190719, 140401407164415, +STORE, 140401340055552, 140401340190719, +STORE, 140401340190720, 140401407164415, +SNULL, 140401442099199, 140401450487807, +STORE, 140401442095104, 140401442099199, +STORE, 140401442099200, 140401450487807, +STORE, 140401433702400, 140401442095103, +SNULL, 140401433706495, 140401442095103, +STORE, 140401433702400, 140401433706495, +STORE, 140401433706496, 140401442095103, +STORE, 140401425309696, 140401433702399, +SNULL, 140401425313791, 140401433702399, +STORE, 140401425309696, 140401425313791, +STORE, 140401425313792, 140401433702399, +STORE, 140401416916992, 140401425309695, +SNULL, 140401416921087, 140401425309695, +STORE, 140401416916992, 140401416921087, +STORE, 140401416921088, 140401425309695, +STORE, 140401408524288, 140401416916991, +STORE, 140401205837824, 140401340055551, +SNULL, 140401272946687, 140401340055551, +STORE, 140401205837824, 140401272946687, +STORE, 140401272946688, 140401340055551, +ERASE, 140401272946688, 140401340055551, +SNULL, 140401205972991, 140401272946687, +STORE, 140401205837824, 140401205972991, +STORE, 140401205972992, 140401272946687, +STORE, 140401331662848, 140401340055551, +STORE, 140401323270144, 140401340055551, +STORE, 140401138728960, 140401205837823, +STORE, 140401314877440, 140401340055551, +SNULL, 140401408528383, 140401416916991, +STORE, 140401408524288, 140401408528383, +STORE, 140401408528384, 140401416916991, +SNULL, 140401138864127, 140401205837823, +STORE, 140401138728960, 140401138864127, +STORE, 140401138864128, 140401205837823, +STORE, 140401004511232, 140401138728959, +SNULL, 140401071620095, 140401138728959, +STORE, 140401004511232, 140401071620095, +STORE, 140401071620096, 140401138728959, +ERASE, 140401071620096, 140401138728959, +STORE, 140400870293504, 140401071620095, +SNULL, 140400937402367, 140401071620095, +STORE, 140400870293504, 140400937402367, +STORE, 140400937402368, 140401071620095, +SNULL, 140400937402368, 140401004511231, +STORE, 140401004511232, 140401071620095, +STORE, 140400937402368, 140401004511231, +ERASE, 140400937402368, 140401004511231, +STORE, 140401306484736, 140401340055551, +SNULL, 140401306484736, 140401323270143, +STORE, 140401323270144, 140401340055551, +STORE, 140401306484736, 140401323270143, +SNULL, 140401323274239, 140401340055551, +STORE, 140401323270144, 140401323274239, +STORE, 140401323274240, 140401340055551, +SNULL, 140401004646399, 140401071620095, +STORE, 140401004511232, 140401004646399, +STORE, 140401004646400, 140401071620095, +SNULL, 140400870428671, 140400937402367, +STORE, 140400870293504, 140400870428671, +STORE, 140400870428672, 140400937402367, +SNULL, 140401306488831, 140401323270143, +STORE, 140401306484736, 140401306488831, +STORE, 140401306488832, 140401323270143, +STORE, 140401298092032, 140401306484735, +SNULL, 140401306488832, 140401314877439, +STORE, 140401314877440, 140401323270143, +STORE, 140401306488832, 140401314877439, +SNULL, 140401314881535, 140401323270143, +STORE, 140401314877440, 140401314881535, +STORE, 140401314881536, 140401323270143, +SNULL, 140401323274240, 140401331662847, +STORE, 140401331662848, 140401340055551, +STORE, 140401323274240, 140401331662847, +SNULL, 140401331666943, 140401340055551, +STORE, 140401331662848, 140401331666943, +STORE, 140401331666944, 140401340055551, +SNULL, 140401298096127, 140401306484735, +STORE, 140401298092032, 140401298096127, +STORE, 140401298096128, 140401306484735, +STORE, 140401289699328, 140401298092031, +STORE, 140401281306624, 140401298092031, +STORE, 140401130336256, 140401138728959, +SNULL, 140401281306624, 140401289699327, +STORE, 140401289699328, 140401298092031, +STORE, 140401281306624, 140401289699327, +SNULL, 140401289703423, 140401298092031, +STORE, 140401289699328, 140401289703423, +STORE, 140401289703424, 140401298092031, +STORE, 140401121943552, 140401138728959, +STORE, 140401113550848, 140401138728959, +SNULL, 140401281310719, 140401289699327, +STORE, 140401281306624, 140401281310719, +STORE, 140401281310720, 140401289699327, +SNULL, 140401113550848, 140401121943551, +STORE, 140401121943552, 140401138728959, +STORE, 140401113550848, 140401121943551, +SNULL, 140401121947647, 140401138728959, +STORE, 140401121943552, 140401121947647, +STORE, 140401121947648, 140401138728959, +STORE, 140401105158144, 140401121943551, +SNULL, 140401121947648, 140401130336255, +STORE, 140401130336256, 140401138728959, +STORE, 140401121947648, 140401130336255, +SNULL, 140401130340351, 140401138728959, +STORE, 140401130336256, 140401130340351, +STORE, 140401130340352, 140401138728959, +STORE, 140401096765440, 140401121943551, +SNULL, 140401096765440, 140401113550847, +STORE, 140401113550848, 140401121943551, +STORE, 140401096765440, 140401113550847, +SNULL, 140401113554943, 140401121943551, +STORE, 140401113550848, 140401113554943, +STORE, 140401113554944, 140401121943551, +STORE, 140401088372736, 140401113550847, +SNULL, 140401088372736, 140401096765439, +STORE, 140401096765440, 140401113550847, +STORE, 140401088372736, 140401096765439, +SNULL, 140401096769535, 140401113550847, +STORE, 140401096765440, 140401096769535, +STORE, 140401096769536, 140401113550847, +SNULL, 140401096769536, 140401105158143, +STORE, 140401105158144, 140401113550847, +STORE, 140401096769536, 140401105158143, +SNULL, 140401105162239, 140401113550847, +STORE, 140401105158144, 140401105162239, +STORE, 140401105162240, 140401113550847, +SNULL, 140401088376831, 140401096765439, +STORE, 140401088372736, 140401088376831, +STORE, 140401088376832, 140401096765439, +STORE, 140401079980032, 140401088372735, +STORE, 140400996118528, 140401004511231, +SNULL, 140401079984127, 140401088372735, +STORE, 140401079980032, 140401079984127, +STORE, 140401079984128, 140401088372735, +SNULL, 140400996122623, 140401004511231, +STORE, 140400996118528, 140400996122623, +STORE, 140400996122624, 140401004511231, +STORE, 140400987725824, 140400996118527, +STORE, 140400979333120, 140400996118527, +STORE, 140400803184640, 140400870293503, +SNULL, 140400803319807, 140400870293503, +STORE, 140400803184640, 140400803319807, +STORE, 140400803319808, 140400870293503, +SNULL, 140400979333120, 140400987725823, +STORE, 140400987725824, 140400996118527, +STORE, 140400979333120, 140400987725823, +SNULL, 140400987729919, 140400996118527, +STORE, 140400987725824, 140400987729919, +STORE, 140400987729920, 140400996118527, +STORE, 140400970940416, 140400987725823, +STORE, 140400962547712, 140400987725823, +STORE, 140400668966912, 140400803184639, +STORE, 140400954155008, 140400987725823, +STORE, 140400945762304, 140400987725823, +STORE, 140400660574208, 140400668966911, +STORE, 140400593465344, 140400660574207, +STORE, 140400585072640, 140400593465343, +STORE, 140400450854912, 140400585072639, +STORE, 140400442462208, 140400450854911, +STORE, 140400434069504, 140400450854911, +STORE, 140400299851776, 140400434069503, +STORE, 140400291459072, 140400299851775, +SNULL, 140400299851776, 140400333422591, +STORE, 140400333422592, 140400434069503, +STORE, 140400299851776, 140400333422591, +ERASE, 140400299851776, 140400333422591, +STORE, 140400325029888, 140400333422591, +STORE, 140400157241344, 140400291459071, +STORE, 140400316637184, 140400333422591, +STORE, 140400308244480, 140400333422591, +STORE, 140400023023616, 140400291459071, +STORE, 140400291459072, 140400333422591, +SNULL, 140400023023616, 140400064987135, +STORE, 140400064987136, 140400291459071, +STORE, 140400023023616, 140400064987135, +ERASE, 140400023023616, 140400064987135, +STORE, 140400056594432, 140400064987135, +SNULL, 140400056598527, 140400064987135, +STORE, 140400056594432, 140400056598527, +STORE, 140400056598528, 140400064987135, +STORE, 140399989485568, 140400056594431, +SNULL, 140400291459072, 140400316637183, +STORE, 140400316637184, 140400333422591, +STORE, 140400291459072, 140400316637183, +SNULL, 140400316641279, 140400333422591, +STORE, 140400316637184, 140400316641279, +STORE, 140400316641280, 140400333422591, +STORE, 140399855267840, 140400056594431, +SNULL, 140399855267840, 140399863660543, +STORE, 140399863660544, 140400056594431, +STORE, 140399855267840, 140399863660543, +ERASE, 140399855267840, 140399863660543, +SNULL, 140400736075775, 140400803184639, +STORE, 140400668966912, 140400736075775, +STORE, 140400736075776, 140400803184639, +ERASE, 140400736075776, 140400803184639, +SNULL, 140400669102079, 140400736075775, +STORE, 140400668966912, 140400669102079, +STORE, 140400669102080, 140400736075775, +STORE, 140400669102080, 140400803184639, +SNULL, 140400669102080, 140400736075775, +STORE, 140400736075776, 140400803184639, +STORE, 140400669102080, 140400736075775, +SNULL, 140400736210943, 140400803184639, +STORE, 140400736075776, 140400736210943, +STORE, 140400736210944, 140400803184639, +ERASE, 140400593465344, 140400660574207, +SNULL, 140400450854912, 140400467640319, +STORE, 140400467640320, 140400585072639, +STORE, 140400450854912, 140400467640319, +ERASE, 140400450854912, 140400467640319, +STORE, 140399729442816, 140400056594431, +SNULL, 140400400531455, 140400434069503, +STORE, 140400333422592, 140400400531455, +STORE, 140400400531456, 140400434069503, +ERASE, 140400400531456, 140400434069503, +SNULL, 140400333557759, 140400400531455, +STORE, 140400333422592, 140400333557759, +STORE, 140400333557760, 140400400531455, +SNULL, 140400157241343, 140400291459071, +STORE, 140400064987136, 140400157241343, +STORE, 140400157241344, 140400291459071, +SNULL, 140400157241344, 140400199204863, +STORE, 140400199204864, 140400291459071, +STORE, 140400157241344, 140400199204863, +ERASE, 140400157241344, 140400199204863, +SNULL, 140400266313727, 140400291459071, +STORE, 140400199204864, 140400266313727, +STORE, 140400266313728, 140400291459071, +ERASE, 140400266313728, 140400291459071, +SNULL, 140400132095999, 140400157241343, +STORE, 140400064987136, 140400132095999, +STORE, 140400132096000, 140400157241343, +ERASE, 140400132096000, 140400157241343, +SNULL, 140400065122303, 140400132095999, +STORE, 140400064987136, 140400065122303, +STORE, 140400065122304, 140400132095999, +SNULL, 140400945762304, 140400954155007, +STORE, 140400954155008, 140400987725823, +STORE, 140400945762304, 140400954155007, +SNULL, 140400954159103, 140400987725823, +STORE, 140400954155008, 140400954159103, +STORE, 140400954159104, 140400987725823, +SNULL, 140400434069504, 140400442462207, +STORE, 140400442462208, 140400450854911, +STORE, 140400434069504, 140400442462207, +SNULL, 140400442466303, 140400450854911, +STORE, 140400442462208, 140400442466303, +STORE, 140400442466304, 140400450854911, +SNULL, 140400291463167, 140400316637183, +STORE, 140400291459072, 140400291463167, +STORE, 140400291463168, 140400316637183, +STORE, 140400652181504, 140400668966911, +STORE, 140400643788800, 140400668966911, +SNULL, 140400291463168, 140400299851775, +STORE, 140400299851776, 140400316637183, +STORE, 140400291463168, 140400299851775, +SNULL, 140400299855871, 140400316637183, +STORE, 140400299851776, 140400299855871, +STORE, 140400299855872, 140400316637183, +STORE, 140400635396096, 140400668966911, +SNULL, 140400635396096, 140400643788799, +STORE, 140400643788800, 140400668966911, +STORE, 140400635396096, 140400643788799, +SNULL, 140400643792895, 140400668966911, +STORE, 140400643788800, 140400643792895, +STORE, 140400643792896, 140400668966911, +SNULL, 140399989485567, 140400056594431, +STORE, 140399729442816, 140399989485567, +STORE, 140399989485568, 140400056594431, +ERASE, 140399989485568, 140400056594431, +SNULL, 140399930769407, 140399989485567, +STORE, 140399729442816, 140399930769407, +STORE, 140399930769408, 140399989485567, +ERASE, 140399930769408, 140399989485567, +SNULL, 140400945766399, 140400954155007, +STORE, 140400945762304, 140400945766399, +STORE, 140400945766400, 140400954155007, +SNULL, 140400534749183, 140400585072639, +STORE, 140400467640320, 140400534749183, +STORE, 140400534749184, 140400585072639, +ERASE, 140400534749184, 140400585072639, +SNULL, 140399796551679, 140399930769407, +STORE, 140399729442816, 140399796551679, +STORE, 140399796551680, 140399930769407, +SNULL, 140399796551680, 140399863660543, +STORE, 140399863660544, 140399930769407, +STORE, 140399796551680, 140399863660543, +ERASE, 140399796551680, 140399863660543, +SNULL, 140400199340031, 140400266313727, +STORE, 140400199204864, 140400199340031, +STORE, 140400199340032, 140400266313727, +STORE, 140400627003392, 140400643788799, +SNULL, 140400316641280, 140400325029887, +STORE, 140400325029888, 140400333422591, +STORE, 140400316641280, 140400325029887, +SNULL, 140400325033983, 140400333422591, +STORE, 140400325029888, 140400325033983, +STORE, 140400325033984, 140400333422591, +SNULL, 140400627003392, 140400635396095, +STORE, 140400635396096, 140400643788799, +STORE, 140400627003392, 140400635396095, +SNULL, 140400635400191, 140400643788799, +STORE, 140400635396096, 140400635400191, +STORE, 140400635400192, 140400643788799, +SNULL, 140400434073599, 140400442462207, +STORE, 140400434069504, 140400434073599, +STORE, 140400434073600, 140400442462207, +STORE, 140400618610688, 140400635396095, +STORE, 140400610217984, 140400635396095, +SNULL, 140400954159104, 140400962547711, +STORE, 140400962547712, 140400987725823, +STORE, 140400954159104, 140400962547711, +SNULL, 140400962551807, 140400987725823, +STORE, 140400962547712, 140400962551807, +STORE, 140400962551808, 140400987725823, +SNULL, 140400299855872, 140400308244479, +STORE, 140400308244480, 140400316637183, +STORE, 140400299855872, 140400308244479, +SNULL, 140400308248575, 140400316637183, +STORE, 140400308244480, 140400308248575, +STORE, 140400308248576, 140400316637183, +STORE, 140400601825280, 140400635396095, +SNULL, 140400601829375, 140400635396095, +STORE, 140400601825280, 140400601829375, +STORE, 140400601829376, 140400635396095, +STORE, 140400576679936, 140400593465343, +SNULL, 140400576684031, 140400593465343, +STORE, 140400576679936, 140400576684031, +STORE, 140400576684032, 140400593465343, +SNULL, 140400643792896, 140400652181503, +STORE, 140400652181504, 140400668966911, +STORE, 140400643792896, 140400652181503, +SNULL, 140400652185599, 140400668966911, +STORE, 140400652181504, 140400652185599, +STORE, 140400652185600, 140400668966911, +STORE, 140399595225088, 140399796551679, +SNULL, 140399662333951, 140399796551679, +STORE, 140399595225088, 140399662333951, +STORE, 140399662333952, 140399796551679, +SNULL, 140399662333952, 140399729442815, +STORE, 140399729442816, 140399796551679, +STORE, 140399662333952, 140399729442815, +ERASE, 140399662333952, 140399729442815, +SNULL, 140399863795711, 140399930769407, +STORE, 140399863660544, 140399863795711, +STORE, 140399863795712, 140399930769407, +STORE, 140400568287232, 140400576679935, +SNULL, 140400568291327, 140400576679935, +STORE, 140400568287232, 140400568291327, +STORE, 140400568291328, 140400576679935, +SNULL, 140400467775487, 140400534749183, +STORE, 140400467640320, 140400467775487, +STORE, 140400467775488, 140400534749183, +SNULL, 140399729577983, 140399796551679, +STORE, 140399729442816, 140399729577983, +STORE, 140399729577984, 140399796551679, +SNULL, 140400601829376, 140400627003391, +STORE, 140400627003392, 140400635396095, +STORE, 140400601829376, 140400627003391, +SNULL, 140400627007487, 140400635396095, +STORE, 140400627003392, 140400627007487, +STORE, 140400627007488, 140400635396095, +STORE, 140400559894528, 140400568287231, +STORE, 140400551501824, 140400568287231, +STORE, 140400543109120, 140400568287231, +STORE, 140400459247616, 140400467640319, +STORE, 140400442466304, 140400467640319, +SNULL, 140399595360255, 140399662333951, +STORE, 140399595225088, 140399595360255, +STORE, 140399595360256, 140399662333951, +SNULL, 140400962551808, 140400970940415, +STORE, 140400970940416, 140400987725823, +STORE, 140400962551808, 140400970940415, +SNULL, 140400970944511, 140400987725823, +STORE, 140400970940416, 140400970944511, +STORE, 140400970944512, 140400987725823, +SNULL, 140400652185600, 140400660574207, +STORE, 140400660574208, 140400668966911, +STORE, 140400652185600, 140400660574207, +SNULL, 140400660578303, 140400668966911, +STORE, 140400660574208, 140400660578303, +STORE, 140400660578304, 140400668966911, +SNULL, 140400576684032, 140400585072639, +STORE, 140400585072640, 140400593465343, +STORE, 140400576684032, 140400585072639, +SNULL, 140400585076735, 140400593465343, +STORE, 140400585072640, 140400585076735, +STORE, 140400585076736, 140400593465343, +STORE, 140400425676800, 140400434069503, +STORE, 140400417284096, 140400434069503, +STORE, 140400408891392, 140400434069503, +SNULL, 140400408891392, 140400417284095, +STORE, 140400417284096, 140400434069503, +STORE, 140400408891392, 140400417284095, +SNULL, 140400417288191, 140400434069503, +STORE, 140400417284096, 140400417288191, +STORE, 140400417288192, 140400434069503, +STORE, 140400283066368, 140400291459071, +SNULL, 140400601829376, 140400618610687, +STORE, 140400618610688, 140400627003391, +STORE, 140400601829376, 140400618610687, +SNULL, 140400618614783, 140400627003391, +STORE, 140400618610688, 140400618614783, +STORE, 140400618614784, 140400627003391, +SNULL, 140400601829376, 140400610217983, +STORE, 140400610217984, 140400618610687, +STORE, 140400601829376, 140400610217983, +SNULL, 140400610222079, 140400618610687, +STORE, 140400610217984, 140400610222079, +STORE, 140400610222080, 140400618610687, +STORE, 140400274673664, 140400291459071, +STORE, 140400190812160, 140400199204863, +STORE, 140400182419456, 140400199204863, +SNULL, 140400442466304, 140400450854911, +STORE, 140400450854912, 140400467640319, +STORE, 140400442466304, 140400450854911, +SNULL, 140400450859007, 140400467640319, +STORE, 140400450854912, 140400450859007, +STORE, 140400450859008, 140400467640319, +SNULL, 140400543109120, 140400559894527, +STORE, 140400559894528, 140400568287231, +STORE, 140400543109120, 140400559894527, +SNULL, 140400559898623, 140400568287231, +STORE, 140400559894528, 140400559898623, +STORE, 140400559898624, 140400568287231, +SNULL, 140400450859008, 140400459247615, +STORE, 140400459247616, 140400467640319, +STORE, 140400450859008, 140400459247615, +SNULL, 140400459251711, 140400467640319, +STORE, 140400459247616, 140400459251711, +STORE, 140400459251712, 140400467640319, +SNULL, 140400543113215, 140400559894527, +STORE, 140400543109120, 140400543113215, +STORE, 140400543113216, 140400559894527, +SNULL, 140400970944512, 140400979333119, +STORE, 140400979333120, 140400987725823, +STORE, 140400970944512, 140400979333119, +SNULL, 140400979337215, 140400987725823, +STORE, 140400979333120, 140400979337215, +STORE, 140400979337216, 140400987725823, +STORE, 140400174026752, 140400199204863, +SNULL, 140400174030847, 140400199204863, +STORE, 140400174026752, 140400174030847, +STORE, 140400174030848, 140400199204863, +SNULL, 140400274673664, 140400283066367, +STORE, 140400283066368, 140400291459071, +STORE, 140400274673664, 140400283066367, +SNULL, 140400283070463, 140400291459071, +STORE, 140400283066368, 140400283070463, +STORE, 140400283070464, 140400291459071, +STORE, 140400165634048, 140400174026751, +SNULL, 140400165638143, 140400174026751, +STORE, 140400165634048, 140400165638143, +STORE, 140400165638144, 140400174026751, +SNULL, 140400174030848, 140400182419455, +STORE, 140400182419456, 140400199204863, +STORE, 140400174030848, 140400182419455, +SNULL, 140400182423551, 140400199204863, +STORE, 140400182419456, 140400182423551, +STORE, 140400182423552, 140400199204863, +SNULL, 140400182423552, 140400190812159, +STORE, 140400190812160, 140400199204863, +STORE, 140400182423552, 140400190812159, +SNULL, 140400190816255, 140400199204863, +STORE, 140400190812160, 140400190816255, +STORE, 140400190816256, 140400199204863, +STORE, 140400157241344, 140400165634047, +SNULL, 140400157245439, 140400165634047, +STORE, 140400157241344, 140400157245439, +STORE, 140400157245440, 140400165634047, +SNULL, 140400408895487, 140400417284095, +STORE, 140400408891392, 140400408895487, +STORE, 140400408895488, 140400417284095, +SNULL, 140400417288192, 140400425676799, +STORE, 140400425676800, 140400434069503, +STORE, 140400417288192, 140400425676799, +SNULL, 140400425680895, 140400434069503, +STORE, 140400425676800, 140400425680895, +STORE, 140400425680896, 140400434069503, +STORE, 140400148848640, 140400157241343, +SNULL, 140400148852735, 140400157241343, +STORE, 140400148848640, 140400148852735, +STORE, 140400148852736, 140400157241343, +SNULL, 140400543113216, 140400551501823, +STORE, 140400551501824, 140400559894527, +STORE, 140400543113216, 140400551501823, +SNULL, 140400551505919, 140400559894527, +STORE, 140400551501824, 140400551505919, +STORE, 140400551505920, 140400559894527, +STORE, 140400140455936, 140400148848639, +STORE, 140400048201728, 140400056594431, +SNULL, 140400140460031, 140400148848639, +STORE, 140400140455936, 140400140460031, +STORE, 140400140460032, 140400148848639, +STORE, 140400039809024, 140400056594431, +SNULL, 140400039813119, 140400056594431, +STORE, 140400039809024, 140400039813119, +STORE, 140400039813120, 140400056594431, +STORE, 140400031416320, 140400039809023, +STORE, 140400023023616, 140400039809023, +SNULL, 140400274677759, 140400283066367, +STORE, 140400274673664, 140400274677759, +STORE, 140400274677760, 140400283066367, +STORE, 140400014630912, 140400039809023, +STORE, 140400006238208, 140400039809023, +STORE, 140399997845504, 140400039809023, +SNULL, 140399997849599, 140400039809023, +STORE, 140399997845504, 140399997849599, +STORE, 140399997849600, 140400039809023, +STORE, 140399989452800, 140399997845503, +SNULL, 140399989456895, 140399997845503, +STORE, 140399989452800, 140399989456895, +STORE, 140399989456896, 140399997845503, +STORE, 140399981060096, 140399989452799, +SNULL, 140399981064191, 140399989452799, +STORE, 140399981060096, 140399981064191, +STORE, 140399981064192, 140399989452799, +STORE, 140399972667392, 140399981060095, +STORE, 140399964274688, 140399981060095, +SNULL, 140399964278783, 140399981060095, +STORE, 140399964274688, 140399964278783, +STORE, 140399964278784, 140399981060095, +SNULL, 140400039813120, 140400048201727, +STORE, 140400048201728, 140400056594431, +STORE, 140400039813120, 140400048201727, +SNULL, 140400048205823, 140400056594431, +STORE, 140400048201728, 140400048205823, +STORE, 140400048205824, 140400056594431, +SNULL, 140399997849600, 140400031416319, +STORE, 140400031416320, 140400039809023, +STORE, 140399997849600, 140400031416319, +SNULL, 140400031420415, 140400039809023, +STORE, 140400031416320, 140400031420415, +STORE, 140400031420416, 140400039809023, +STORE, 140399955881984, 140399964274687, +SNULL, 140399955886079, 140399964274687, +STORE, 140399955881984, 140399955886079, +STORE, 140399955886080, 140399964274687, +STORE, 140399947489280, 140399955881983, +STORE, 140399939096576, 140399955881983, +STORE, 140399855267840, 140399863660543, +SNULL, 140399939100671, 140399955881983, +STORE, 140399939096576, 140399939100671, +STORE, 140399939100672, 140399955881983, +SNULL, 140399997849600, 140400014630911, +STORE, 140400014630912, 140400031416319, +STORE, 140399997849600, 140400014630911, +SNULL, 140400014635007, 140400031416319, +STORE, 140400014630912, 140400014635007, +STORE, 140400014635008, 140400031416319, +SNULL, 140400014635008, 140400023023615, +STORE, 140400023023616, 140400031416319, +STORE, 140400014635008, 140400023023615, +SNULL, 140400023027711, 140400031416319, +STORE, 140400023023616, 140400023027711, +STORE, 140400023027712, 140400031416319, +SNULL, 140399997849600, 140400006238207, +STORE, 140400006238208, 140400014630911, +STORE, 140399997849600, 140400006238207, +SNULL, 140400006242303, 140400014630911, +STORE, 140400006238208, 140400006242303, +STORE, 140400006242304, 140400014630911, +STORE, 140399846875136, 140399863660543, +STORE, 140399838482432, 140399863660543, +SNULL, 140399838486527, 140399863660543, +STORE, 140399838482432, 140399838486527, +STORE, 140399838486528, 140399863660543, +SNULL, 140399939100672, 140399947489279, +STORE, 140399947489280, 140399955881983, +STORE, 140399939100672, 140399947489279, +SNULL, 140399947493375, 140399955881983, +STORE, 140399947489280, 140399947493375, +STORE, 140399947493376, 140399955881983, +SNULL, 140399964278784, 140399972667391, +STORE, 140399972667392, 140399981060095, +STORE, 140399964278784, 140399972667391, +SNULL, 140399972671487, 140399981060095, +STORE, 140399972667392, 140399972671487, +STORE, 140399972671488, 140399981060095, +SNULL, 140399838486528, 140399855267839, +STORE, 140399855267840, 140399863660543, +STORE, 140399838486528, 140399855267839, +SNULL, 140399855271935, 140399863660543, +STORE, 140399855267840, 140399855271935, +STORE, 140399855271936, 140399863660543, +STORE, 140399830089728, 140399838482431, +SNULL, 140399830093823, 140399838482431, +STORE, 140399830089728, 140399830093823, +STORE, 140399830093824, 140399838482431, +STORE, 140399821697024, 140399830089727, +SNULL, 140399821701119, 140399830089727, +STORE, 140399821697024, 140399821701119, +STORE, 140399821701120, 140399830089727, +SNULL, 140399838486528, 140399846875135, +STORE, 140399846875136, 140399855267839, +STORE, 140399838486528, 140399846875135, +SNULL, 140399846879231, 140399855267839, +STORE, 140399846875136, 140399846879231, +STORE, 140399846879232, 140399855267839, +STORE, 140399813304320, 140399821697023, +STORE, 140399804911616, 140399821697023, +SNULL, 140399804915711, 140399821697023, +STORE, 140399804911616, 140399804915711, +STORE, 140399804915712, 140399821697023, +STORE, 140399721050112, 140399729442815, +SNULL, 140399804915712, 140399813304319, +STORE, 140399813304320, 140399821697023, +STORE, 140399804915712, 140399813304319, +SNULL, 140399813308415, 140399821697023, +STORE, 140399813304320, 140399813308415, +STORE, 140399813308416, 140399821697023, +SNULL, 140399721054207, 140399729442815, +STORE, 140399721050112, 140399721054207, +STORE, 140399721054208, 140399729442815, +STORE, 140401467105280, 140401467133951, +STORE, 140401279115264, 140401281306623, +SNULL, 140401279115264, 140401279205375, +STORE, 140401279205376, 140401281306623, +STORE, 140401279115264, 140401279205375, +SNULL, 140401281298431, 140401281306623, +STORE, 140401279205376, 140401281298431, +STORE, 140401281298432, 140401281306623, +ERASE, 140401281298432, 140401281306623, +STORE, 140401281298432, 140401281306623, +SNULL, 140401281302527, 140401281306623, +STORE, 140401281298432, 140401281302527, +STORE, 140401281302528, 140401281306623, +ERASE, 140401467105280, 140401467133951, +ERASE, 140400056594432, 140400056598527, +ERASE, 140400056598528, 140400064987135, +ERASE, 140400635396096, 140400635400191, +ERASE, 140400635400192, 140400643788799, +ERASE, 140400408891392, 140400408895487, +ERASE, 140400408895488, 140400417284095, +ERASE, 140400299851776, 140400299855871, +ERASE, 140400299855872, 140400308244479, +ERASE, 140400627003392, 140400627007487, +ERASE, 140400627007488, 140400635396095, +ERASE, 140400954155008, 140400954159103, +ERASE, 140400954159104, 140400962547711, +ERASE, 140400291459072, 140400291463167, +ERASE, 140400291463168, 140400299851775, +ERASE, 140400643788800, 140400643792895, +ERASE, 140400643792896, 140400652181503, +ERASE, 140400325029888, 140400325033983, +ERASE, 140400325033984, 140400333422591, +ERASE, 140400610217984, 140400610222079, +ERASE, 140400610222080, 140400618610687, +ERASE, 140400190812160, 140400190816255, +ERASE, 140400190816256, 140400199204863, +ERASE, 140399964274688, 140399964278783, +ERASE, 140399964278784, 140399972667391, +ERASE, 140400945762304, 140400945766399, +ERASE, 140400945766400, 140400954155007, +ERASE, 140400568287232, 140400568291327, +ERASE, 140400568291328, 140400576679935, +ERASE, 140399972667392, 140399972671487, +ERASE, 140399972671488, 140399981060095, +ERASE, 140400962547712, 140400962551807, +ERASE, 140400962551808, 140400970940415, +ERASE, 140400987725824, 140400987729919, +ERASE, 140400987729920, 140400996118527, +ERASE, 140400652181504, 140400652185599, +ERASE, 140400652185600, 140400660574207, +ERASE, 140400450854912, 140400450859007, +ERASE, 140400450859008, 140400459247615, +ERASE, 140400031416320, 140400031420415, +ERASE, 140400031420416, 140400039809023, +ERASE, 140400308244480, 140400308248575, +ERASE, 140400308248576, 140400316637183, +ERASE, 140400434069504, 140400434073599, +ERASE, 140400434073600, 140400442462207, +ERASE, 140400543109120, 140400543113215, +ERASE, 140400543113216, 140400551501823, +ERASE, 140400023023616, 140400023027711, +ERASE, 140400023027712, 140400031416319, +ERASE, 140399813304320, 140399813308415, +ERASE, 140399813308416, 140399821697023, +ERASE, 140400316637184, 140400316641279, +ERASE, 140400316641280, 140400325029887, +ERASE, 140400585072640, 140400585076735, +ERASE, 140400585076736, 140400593465343, +ERASE, 140400148848640, 140400148852735, +ERASE, 140400148852736, 140400157241343, +ERASE, 140399955881984, 140399955886079, +ERASE, 140399955886080, 140399964274687, +ERASE, 140399821697024, 140399821701119, +ERASE, 140399821701120, 140399830089727, +ERASE, 140400601825280, 140400601829375, +ERASE, 140400601829376, 140400610217983, +ERASE, 140400979333120, 140400979337215, +ERASE, 140400979337216, 140400987725823, +ERASE, 140399997845504, 140399997849599, +ERASE, 140399997849600, 140400006238207, +ERASE, 140400459247616, 140400459251711, +ERASE, 140400459251712, 140400467640319, +ERASE, 140400551501824, 140400551505919, +ERASE, 140400551505920, 140400559894527, +ERASE, 140399939096576, 140399939100671, +ERASE, 140399939100672, 140399947489279, +ERASE, 140400442462208, 140400442466303, +ERASE, 140400442466304, 140400450854911, +ERASE, 140400576679936, 140400576684031, +ERASE, 140400576684032, 140400585072639, +ERASE, 140400559894528, 140400559898623, +ERASE, 140400559898624, 140400568287231, +ERASE, 140400417284096, 140400417288191, +ERASE, 140400417288192, 140400425676799, +ERASE, 140400283066368, 140400283070463, +ERASE, 140400283070464, 140400291459071, + }; + unsigned long set33[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734562918400, 140737488351231, +SNULL, 140734562922495, 140737488351231, +STORE, 140734562918400, 140734562922495, +STORE, 140734562787328, 140734562922495, +STORE, 94133878984704, 94133881237503, +SNULL, 94133879115775, 94133881237503, +STORE, 94133878984704, 94133879115775, +STORE, 94133879115776, 94133881237503, +ERASE, 94133879115776, 94133881237503, +STORE, 94133881208832, 94133881217023, +STORE, 94133881217024, 94133881237503, +STORE, 140583654043648, 140583656296447, +SNULL, 140583654187007, 140583656296447, +STORE, 140583654043648, 140583654187007, +STORE, 140583654187008, 140583656296447, +ERASE, 140583654187008, 140583656296447, +STORE, 140583656284160, 140583656292351, +STORE, 140583656292352, 140583656296447, +STORE, 140734564319232, 140734564323327, +STORE, 140734564306944, 140734564319231, +STORE, 140583656255488, 140583656284159, +STORE, 140583656247296, 140583656255487, +STORE, 140583651827712, 140583654043647, +SNULL, 140583651827712, 140583651926015, +STORE, 140583651926016, 140583654043647, +STORE, 140583651827712, 140583651926015, +SNULL, 140583654019071, 140583654043647, +STORE, 140583651926016, 140583654019071, +STORE, 140583654019072, 140583654043647, +SNULL, 140583654019072, 140583654027263, +STORE, 140583654027264, 140583654043647, +STORE, 140583654019072, 140583654027263, +ERASE, 140583654019072, 140583654027263, +STORE, 140583654019072, 140583654027263, +ERASE, 140583654027264, 140583654043647, +STORE, 140583654027264, 140583654043647, +STORE, 140583648030720, 140583651827711, +SNULL, 140583648030720, 140583649689599, +STORE, 140583649689600, 140583651827711, +STORE, 140583648030720, 140583649689599, +SNULL, 140583651786751, 140583651827711, +STORE, 140583649689600, 140583651786751, +STORE, 140583651786752, 140583651827711, +SNULL, 140583651786752, 140583651811327, +STORE, 140583651811328, 140583651827711, +STORE, 140583651786752, 140583651811327, +ERASE, 140583651786752, 140583651811327, +STORE, 140583651786752, 140583651811327, +ERASE, 140583651811328, 140583651827711, +STORE, 140583651811328, 140583651827711, +STORE, 140583656239104, 140583656255487, +SNULL, 140583651803135, 140583651811327, +STORE, 140583651786752, 140583651803135, +STORE, 140583651803136, 140583651811327, +SNULL, 140583654023167, 140583654027263, +STORE, 140583654019072, 140583654023167, +STORE, 140583654023168, 140583654027263, +SNULL, 94133881212927, 94133881217023, +STORE, 94133881208832, 94133881212927, +STORE, 94133881212928, 94133881217023, +SNULL, 140583656288255, 140583656292351, +STORE, 140583656284160, 140583656288255, +STORE, 140583656288256, 140583656292351, +ERASE, 140583656255488, 140583656284159, +STORE, 94133881733120, 94133881868287, +STORE, 140583639638016, 140583648030719, +SNULL, 140583639642111, 140583648030719, +STORE, 140583639638016, 140583639642111, +STORE, 140583639642112, 140583648030719, +STORE, 140583631245312, 140583639638015, +STORE, 140583497027584, 140583631245311, +SNULL, 140583497027584, 140583540621311, +STORE, 140583540621312, 140583631245311, +STORE, 140583497027584, 140583540621311, +ERASE, 140583497027584, 140583540621311, +SNULL, 140583607730175, 140583631245311, +STORE, 140583540621312, 140583607730175, +STORE, 140583607730176, 140583631245311, +ERASE, 140583607730176, 140583631245311, +SNULL, 140583540756479, 140583607730175, +STORE, 140583540621312, 140583540756479, +STORE, 140583540756480, 140583607730175, +SNULL, 140583631249407, 140583639638015, +STORE, 140583631245312, 140583631249407, +STORE, 140583631249408, 140583639638015, +STORE, 140583622852608, 140583631245311, +SNULL, 140583622856703, 140583631245311, +STORE, 140583622852608, 140583622856703, +STORE, 140583622856704, 140583631245311, +STORE, 140583614459904, 140583622852607, +SNULL, 140583614463999, 140583622852607, +STORE, 140583614459904, 140583614463999, +STORE, 140583614464000, 140583622852607, +STORE, 140583532228608, 140583540621311, +SNULL, 140583532232703, 140583540621311, +STORE, 140583532228608, 140583532232703, +STORE, 140583532232704, 140583540621311, +STORE, 140583523835904, 140583532228607, +STORE, 140583515443200, 140583532228607, +STORE, 140583507050496, 140583532228607, +STORE, 140583372832768, 140583507050495, +STORE, 140583364440064, 140583372832767, +STORE, 140583230222336, 140583364440063, +STORE, 140583096004608, 140583364440063, +SNULL, 140583230222335, 140583364440063, +STORE, 140583096004608, 140583230222335, +STORE, 140583230222336, 140583364440063, +SNULL, 140583230222336, 140583272185855, +STORE, 140583272185856, 140583364440063, +STORE, 140583230222336, 140583272185855, +ERASE, 140583230222336, 140583272185855, +STORE, 140582961786880, 140583230222335, +SNULL, 140583372832768, 140583406403583, +STORE, 140583406403584, 140583507050495, +STORE, 140583372832768, 140583406403583, +ERASE, 140583372832768, 140583406403583, +SNULL, 140583473512447, 140583507050495, +STORE, 140583406403584, 140583473512447, +STORE, 140583473512448, 140583507050495, +ERASE, 140583473512448, 140583507050495, +SNULL, 140583096004607, 140583230222335, +STORE, 140582961786880, 140583096004607, +STORE, 140583096004608, 140583230222335, +SNULL, 140583096004608, 140583137968127, +STORE, 140583137968128, 140583230222335, +STORE, 140583096004608, 140583137968127, +ERASE, 140583096004608, 140583137968127, +SNULL, 140583339294719, 140583364440063, +STORE, 140583272185856, 140583339294719, +STORE, 140583339294720, 140583364440063, +ERASE, 140583339294720, 140583364440063, +SNULL, 140583272321023, 140583339294719, +STORE, 140583272185856, 140583272321023, +STORE, 140583272321024, 140583339294719, +SNULL, 140582961786880, 140583003750399, +STORE, 140583003750400, 140583096004607, +STORE, 140582961786880, 140583003750399, +ERASE, 140582961786880, 140583003750399, + }; + + unsigned long set34[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731327180800, 140737488351231, +SNULL, 140731327184895, 140737488351231, +STORE, 140731327180800, 140731327184895, +STORE, 140731327049728, 140731327184895, +STORE, 94632924487680, 94632926740479, +SNULL, 94632924618751, 94632926740479, +STORE, 94632924487680, 94632924618751, +STORE, 94632924618752, 94632926740479, +ERASE, 94632924618752, 94632926740479, +STORE, 94632926711808, 94632926719999, +STORE, 94632926720000, 94632926740479, +STORE, 140012544888832, 140012547141631, +SNULL, 140012545032191, 140012547141631, +STORE, 140012544888832, 140012545032191, +STORE, 140012545032192, 140012547141631, +ERASE, 140012545032192, 140012547141631, +STORE, 140012547129344, 140012547137535, +STORE, 140012547137536, 140012547141631, +STORE, 140731327725568, 140731327729663, +STORE, 140731327713280, 140731327725567, +STORE, 140012547100672, 140012547129343, +STORE, 140012547092480, 140012547100671, +STORE, 140012542672896, 140012544888831, +SNULL, 140012542672896, 140012542771199, +STORE, 140012542771200, 140012544888831, +STORE, 140012542672896, 140012542771199, +SNULL, 140012544864255, 140012544888831, +STORE, 140012542771200, 140012544864255, +STORE, 140012544864256, 140012544888831, +SNULL, 140012544864256, 140012544872447, +STORE, 140012544872448, 140012544888831, +STORE, 140012544864256, 140012544872447, +ERASE, 140012544864256, 140012544872447, +STORE, 140012544864256, 140012544872447, +ERASE, 140012544872448, 140012544888831, +STORE, 140012544872448, 140012544888831, +STORE, 140012538875904, 140012542672895, +SNULL, 140012538875904, 140012540534783, +STORE, 140012540534784, 140012542672895, +STORE, 140012538875904, 140012540534783, +SNULL, 140012542631935, 140012542672895, +STORE, 140012540534784, 140012542631935, +STORE, 140012542631936, 140012542672895, +SNULL, 140012542631936, 140012542656511, +STORE, 140012542656512, 140012542672895, +STORE, 140012542631936, 140012542656511, +ERASE, 140012542631936, 140012542656511, +STORE, 140012542631936, 140012542656511, +ERASE, 140012542656512, 140012542672895, +STORE, 140012542656512, 140012542672895, +STORE, 140012547084288, 140012547100671, +SNULL, 140012542648319, 140012542656511, +STORE, 140012542631936, 140012542648319, +STORE, 140012542648320, 140012542656511, +SNULL, 140012544868351, 140012544872447, +STORE, 140012544864256, 140012544868351, +STORE, 140012544868352, 140012544872447, +SNULL, 94632926715903, 94632926719999, +STORE, 94632926711808, 94632926715903, +STORE, 94632926715904, 94632926719999, +SNULL, 140012547133439, 140012547137535, +STORE, 140012547129344, 140012547133439, +STORE, 140012547133440, 140012547137535, +ERASE, 140012547100672, 140012547129343, +STORE, 94632939606016, 94632939741183, +STORE, 140012530483200, 140012538875903, +SNULL, 140012530487295, 140012538875903, +STORE, 140012530483200, 140012530487295, +STORE, 140012530487296, 140012538875903, +STORE, 140012522090496, 140012530483199, +STORE, 140012387872768, 140012522090495, +SNULL, 140012387872768, 140012444188671, +STORE, 140012444188672, 140012522090495, +STORE, 140012387872768, 140012444188671, +ERASE, 140012387872768, 140012444188671, +SNULL, 140012511297535, 140012522090495, +STORE, 140012444188672, 140012511297535, +STORE, 140012511297536, 140012522090495, +ERASE, 140012511297536, 140012522090495, +SNULL, 140012444323839, 140012511297535, +STORE, 140012444188672, 140012444323839, +STORE, 140012444323840, 140012511297535, +SNULL, 140012522094591, 140012530483199, +STORE, 140012522090496, 140012522094591, +STORE, 140012522094592, 140012530483199, +STORE, 140012513697792, 140012522090495, +SNULL, 140012513701887, 140012522090495, +STORE, 140012513697792, 140012513701887, +STORE, 140012513701888, 140012522090495, +STORE, 140012435795968, 140012444188671, +SNULL, 140012435800063, 140012444188671, +STORE, 140012435795968, 140012435800063, +STORE, 140012435800064, 140012444188671, +STORE, 140012427403264, 140012435795967, +SNULL, 140012427407359, 140012435795967, +STORE, 140012427403264, 140012427407359, +STORE, 140012427407360, 140012435795967, +STORE, 140012419010560, 140012427403263, +STORE, 140012410617856, 140012427403263, +STORE, 140012276400128, 140012410617855, +STORE, 140012268007424, 140012276400127, +STORE, 140012133789696, 140012268007423, +SNULL, 140012133789696, 140012175753215, +STORE, 140012175753216, 140012268007423, +STORE, 140012133789696, 140012175753215, +ERASE, 140012133789696, 140012175753215, +STORE, 140012041535488, 140012268007423, +SNULL, 140012108644351, 140012268007423, +STORE, 140012041535488, 140012108644351, +STORE, 140012108644352, 140012268007423, +SNULL, 140012108644352, 140012175753215, +STORE, 140012175753216, 140012268007423, +STORE, 140012108644352, 140012175753215, +ERASE, 140012108644352, 140012175753215, +SNULL, 140012276400128, 140012309970943, +STORE, 140012309970944, 140012410617855, +STORE, 140012276400128, 140012309970943, +ERASE, 140012276400128, 140012309970943, +STORE, 140012301578240, 140012309970943, +STORE, 140012041535488, 140012268007423, +SNULL, 140012242862079, 140012268007423, +STORE, 140012041535488, 140012242862079, +STORE, 140012242862080, 140012268007423, +ERASE, 140012242862080, 140012268007423, +SNULL, 140012041670655, 140012242862079, +STORE, 140012041535488, 140012041670655, +STORE, 140012041670656, 140012242862079, +SNULL, 140012041670656, 140012108644351, +STORE, 140012108644352, 140012242862079, +STORE, 140012041670656, 140012108644351, +SNULL, 140012108779519, 140012242862079, +STORE, 140012108644352, 140012108779519, +STORE, 140012108779520, 140012242862079, +SNULL, 140012377079807, 140012410617855, +STORE, 140012309970944, 140012377079807, +STORE, 140012377079808, 140012410617855, +ERASE, 140012377079808, 140012410617855, +SNULL, 140012310106111, 140012377079807, +STORE, 140012309970944, 140012310106111, +STORE, 140012310106112, 140012377079807, +SNULL, 140012410621951, 140012427403263, +STORE, 140012410617856, 140012410621951, +STORE, 140012410621952, 140012427403263, +SNULL, 140012108779520, 140012175753215, +STORE, 140012175753216, 140012242862079, +STORE, 140012108779520, 140012175753215, +SNULL, 140012175888383, 140012242862079, +STORE, 140012175753216, 140012175888383, +STORE, 140012175888384, 140012242862079, +SNULL, 140012301582335, 140012309970943, +STORE, 140012301578240, 140012301582335, +STORE, 140012301582336, 140012309970943, +SNULL, 140012410621952, 140012419010559, +STORE, 140012419010560, 140012427403263, +STORE, 140012410621952, 140012419010559, +SNULL, 140012419014655, 140012427403263, +STORE, 140012419010560, 140012419014655, +STORE, 140012419014656, 140012427403263, +SNULL, 140012268011519, 140012276400127, +STORE, 140012268007424, 140012268011519, +STORE, 140012268011520, 140012276400127, +STORE, 140012402225152, 140012410617855, +STORE, 140012393832448, 140012410617855, +SNULL, 140012393832448, 140012402225151, +STORE, 140012402225152, 140012410617855, +STORE, 140012393832448, 140012402225151, +SNULL, 140012402229247, 140012410617855, +STORE, 140012402225152, 140012402229247, +STORE, 140012402229248, 140012410617855, +STORE, 140012385439744, 140012402225151, +SNULL, 140012385439744, 140012393832447, +STORE, 140012393832448, 140012402225151, +STORE, 140012385439744, 140012393832447, +SNULL, 140012393836543, 140012402225151, +STORE, 140012393832448, 140012393836543, +STORE, 140012393836544, 140012402225151, +STORE, 140012293185536, 140012301578239, +STORE, 140012284792832, 140012301578239, +SNULL, 140012284792832, 140012293185535, +STORE, 140012293185536, 140012301578239, +STORE, 140012284792832, 140012293185535, +SNULL, 140012293189631, 140012301578239, +STORE, 140012293185536, 140012293189631, +STORE, 140012293189632, 140012301578239, +STORE, 140012268011520, 140012284792831, +SNULL, 140012385443839, 140012393832447, +STORE, 140012385439744, 140012385443839, +STORE, 140012385443840, 140012393832447, +STORE, 140012259614720, 140012268007423, +SNULL, 140012259618815, 140012268007423, +STORE, 140012259614720, 140012259618815, +STORE, 140012259618816, 140012268007423, +STORE, 140012251222016, 140012259614719, +SNULL, 140012251226111, 140012259614719, +STORE, 140012251222016, 140012251226111, +STORE, 140012251226112, 140012259614719, +SNULL, 140012284796927, 140012293185535, +STORE, 140012284792832, 140012284796927, +STORE, 140012284796928, 140012293185535, +SNULL, 140012268011520, 140012276400127, +STORE, 140012276400128, 140012284792831, +STORE, 140012268011520, 140012276400127, +SNULL, 140012276404223, 140012284792831, +STORE, 140012276400128, 140012276404223, +STORE, 140012276404224, 140012284792831, +STORE, 140012033142784, 140012041535487, +SNULL, 140012033146879, 140012041535487, +STORE, 140012033142784, 140012033146879, +STORE, 140012033146880, 140012041535487, +STORE, 140012024750080, 140012033142783, +STORE, 140012016357376, 140012033142783, +SNULL, 140012016357376, 140012024750079, +STORE, 140012024750080, 140012033142783, +STORE, 140012016357376, 140012024750079, +SNULL, 140012024754175, 140012033142783, +STORE, 140012024750080, 140012024754175, +STORE, 140012024754176, 140012033142783, +SNULL, 140012016361471, 140012024750079, +STORE, 140012016357376, 140012016361471, +STORE, 140012016361472, 140012024750079, +STORE, 140012007964672, 140012016357375, +SNULL, 140012007968767, 140012016357375, +STORE, 140012007964672, 140012007968767, +STORE, 140012007968768, 140012016357375, +STORE, 140011999571968, 140012007964671, +STORE, 140011991179264, 140012007964671, +STORE, 140011856961536, 140011991179263, +STORE, 140011848568832, 140011856961535, +STORE, 140011714351104, 140011848568831, +SNULL, 140011714351104, 140011773100031, +STORE, 140011773100032, 140011848568831, +STORE, 140011714351104, 140011773100031, +ERASE, 140011714351104, 140011773100031, +STORE, 140011764707328, 140011773100031, +STORE, 140011756314624, 140011773100031, +STORE, 140011622096896, 140011756314623, +STORE, 140011613704192, 140011622096895, +STORE, 140011479486464, 140011613704191, +STORE, 140011471093760, 140011479486463, +SNULL, 140011479486464, 140011504664575, +STORE, 140011504664576, 140011613704191, +STORE, 140011479486464, 140011504664575, +ERASE, 140011479486464, 140011504664575, +STORE, 140011496271872, 140011504664575, +STORE, 140011487879168, 140011504664575, +STORE, 140011336876032, 140011471093759, +SNULL, 140011336876032, 140011370446847, +STORE, 140011370446848, 140011471093759, +STORE, 140011336876032, 140011370446847, +ERASE, 140011336876032, 140011370446847, +STORE, 140011471093760, 140011487879167, +STORE, 140011362054144, 140011370446847, +SNULL, 140011362058239, 140011370446847, +STORE, 140011362054144, 140011362058239, +STORE, 140011362058240, 140011370446847, +STORE, 140011353661440, 140011362054143, +STORE, 140011345268736, 140011362054143, +SNULL, 140011345272831, 140011362054143, +STORE, 140011345268736, 140011345272831, +STORE, 140011345272832, 140011362054143, +STORE, 140011336876032, 140011345268735, +STORE, 140011328483328, 140011345268735, +SNULL, 140011328487423, 140011345268735, +STORE, 140011328483328, 140011328487423, +STORE, 140011328487424, 140011345268735, +STORE, 140011320090624, 140011328483327, +STORE, 140011185872896, 140011320090623, +SNULL, 140011185872896, 140011236229119, +STORE, 140011236229120, 140011320090623, +STORE, 140011185872896, 140011236229119, +ERASE, 140011185872896, 140011236229119, +SNULL, 140011856961536, 140011907317759, +STORE, 140011907317760, 140011991179263, +STORE, 140011856961536, 140011907317759, +ERASE, 140011856961536, 140011907317759, +SNULL, 140011974426623, 140011991179263, +STORE, 140011907317760, 140011974426623, +STORE, 140011974426624, 140011991179263, +ERASE, 140011974426624, 140011991179263, +SNULL, 140011840208895, 140011848568831, +STORE, 140011773100032, 140011840208895, +STORE, 140011840208896, 140011848568831, +ERASE, 140011840208896, 140011848568831, +SNULL, 140011773235199, 140011840208895, +STORE, 140011773100032, 140011773235199, +STORE, 140011773235200, 140011840208895, +STORE, 140011102011392, 140011320090623, +SNULL, 140011169120255, 140011320090623, +STORE, 140011102011392, 140011169120255, +STORE, 140011169120256, 140011320090623, +SNULL, 140011169120256, 140011236229119, +STORE, 140011236229120, 140011320090623, +STORE, 140011169120256, 140011236229119, +ERASE, 140011169120256, 140011236229119, +SNULL, 140011622096896, 140011638882303, +STORE, 140011638882304, 140011756314623, +STORE, 140011622096896, 140011638882303, +ERASE, 140011622096896, 140011638882303, +SNULL, 140011705991167, 140011756314623, +STORE, 140011638882304, 140011705991167, +STORE, 140011705991168, 140011756314623, +ERASE, 140011705991168, 140011756314623, +SNULL, 140011571773439, 140011613704191, +STORE, 140011504664576, 140011571773439, +STORE, 140011571773440, 140011613704191, +ERASE, 140011571773440, 140011613704191, +STORE, 140010967793664, 140011169120255, +SNULL, 140011034902527, 140011169120255, +STORE, 140010967793664, 140011034902527, +STORE, 140011034902528, 140011169120255, +SNULL, 140011034902528, 140011102011391, +STORE, 140011102011392, 140011169120255, +STORE, 140011034902528, 140011102011391, +ERASE, 140011034902528, 140011102011391, +STORE, 140010833575936, 140011034902527, +SNULL, 140011437555711, 140011471093759, +STORE, 140011370446848, 140011437555711, +STORE, 140011437555712, 140011471093759, +ERASE, 140011437555712, 140011471093759, +SNULL, 140011370582015, 140011437555711, +STORE, 140011370446848, 140011370582015, +STORE, 140011370582016, 140011437555711, +STORE, 140010699358208, 140011034902527, +SNULL, 140011487883263, 140011504664575, +STORE, 140011487879168, 140011487883263, +STORE, 140011487883264, 140011504664575, +SNULL, 140011345272832, 140011353661439, +STORE, 140011353661440, 140011362054143, +STORE, 140011345272832, 140011353661439, +SNULL, 140011353665535, 140011362054143, +STORE, 140011353661440, 140011353665535, +STORE, 140011353665536, 140011362054143, +SNULL, 140011328487424, 140011336876031, +STORE, 140011336876032, 140011345268735, +STORE, 140011328487424, 140011336876031, +SNULL, 140011336880127, 140011345268735, +STORE, 140011336876032, 140011336880127, +STORE, 140011336880128, 140011345268735, +SNULL, 140011303337983, 140011320090623, +STORE, 140011236229120, 140011303337983, +STORE, 140011303337984, 140011320090623, +ERASE, 140011303337984, 140011320090623, +SNULL, 140011907452927, 140011974426623, +STORE, 140011907317760, 140011907452927, +STORE, 140011907452928, 140011974426623, +SNULL, 140011102146559, 140011169120255, +STORE, 140011102011392, 140011102146559, +STORE, 140011102146560, 140011169120255, +SNULL, 140011639017471, 140011705991167, +STORE, 140011638882304, 140011639017471, +STORE, 140011639017472, 140011705991167, +SNULL, 140011504799743, 140011571773439, +STORE, 140011504664576, 140011504799743, +STORE, 140011504799744, 140011571773439, +SNULL, 140011613708287, 140011622096895, +STORE, 140011613704192, 140011613708287, +STORE, 140011613708288, 140011622096895, +SNULL, 140010699358208, 140010967793663, +STORE, 140010967793664, 140011034902527, +STORE, 140010699358208, 140010967793663, +SNULL, 140010967928831, 140011034902527, +STORE, 140010967793664, 140010967928831, +STORE, 140010967928832, 140011034902527, +SNULL, 140010900684799, 140010967793663, +STORE, 140010699358208, 140010900684799, +STORE, 140010900684800, 140010967793663, +ERASE, 140010900684800, 140010967793663, +SNULL, 140010766467071, 140010900684799, +STORE, 140010699358208, 140010766467071, +STORE, 140010766467072, 140010900684799, +SNULL, 140010766467072, 140010833575935, +STORE, 140010833575936, 140010900684799, +STORE, 140010766467072, 140010833575935, +ERASE, 140010766467072, 140010833575935, +SNULL, 140010699493375, 140010766467071, +STORE, 140010699358208, 140010699493375, +STORE, 140010699493376, 140010766467071, +SNULL, 140011848572927, 140011856961535, +STORE, 140011848568832, 140011848572927, +STORE, 140011848572928, 140011856961535, +STORE, 140011982786560, 140012007964671, +STORE, 140011898925056, 140011907317759, +SNULL, 140011898929151, 140011907317759, +STORE, 140011898925056, 140011898929151, +STORE, 140011898929152, 140011907317759, +SNULL, 140011320094719, 140011328483327, +STORE, 140011320090624, 140011320094719, +STORE, 140011320094720, 140011328483327, +STORE, 140011890532352, 140011898925055, +STORE, 140011882139648, 140011898925055, +SNULL, 140011882143743, 140011898925055, +STORE, 140011882139648, 140011882143743, +STORE, 140011882143744, 140011898925055, +STORE, 140011873746944, 140011882139647, +SNULL, 140011873751039, 140011882139647, +STORE, 140011873746944, 140011873751039, +STORE, 140011873751040, 140011882139647, +SNULL, 140011236364287, 140011303337983, +STORE, 140011236229120, 140011236364287, +STORE, 140011236364288, 140011303337983, +SNULL, 140011756318719, 140011773100031, +STORE, 140011756314624, 140011756318719, +STORE, 140011756318720, 140011773100031, +SNULL, 140011756318720, 140011764707327, +STORE, 140011764707328, 140011773100031, +STORE, 140011756318720, 140011764707327, +SNULL, 140011764711423, 140011773100031, +STORE, 140011764707328, 140011764711423, +STORE, 140011764711424, 140011773100031, +SNULL, 140011471097855, 140011487879167, +STORE, 140011471093760, 140011471097855, +STORE, 140011471097856, 140011487879167, +SNULL, 140010833711103, 140010900684799, +STORE, 140010833575936, 140010833711103, +STORE, 140010833711104, 140010900684799, +SNULL, 140011982790655, 140012007964671, +STORE, 140011982786560, 140011982790655, +STORE, 140011982790656, 140012007964671, +STORE, 140011865354240, 140011873746943, +STORE, 140011848572928, 140011865354239, +SNULL, 140011848572928, 140011856961535, +STORE, 140011856961536, 140011865354239, +STORE, 140011848572928, 140011856961535, +SNULL, 140011856965631, 140011865354239, +STORE, 140011856961536, 140011856965631, +STORE, 140011856965632, 140011865354239, +STORE, 140011747921920, 140011756314623, +STORE, 140011739529216, 140011756314623, +SNULL, 140011471097856, 140011479486463, +STORE, 140011479486464, 140011487879167, +STORE, 140011471097856, 140011479486463, +SNULL, 140011479490559, 140011487879167, +STORE, 140011479486464, 140011479490559, +STORE, 140011479490560, 140011487879167, +STORE, 140011731136512, 140011756314623, +STORE, 140011722743808, 140011756314623, +SNULL, 140011982790656, 140011999571967, +STORE, 140011999571968, 140012007964671, +STORE, 140011982790656, 140011999571967, +SNULL, 140011999576063, 140012007964671, +STORE, 140011999571968, 140011999576063, +STORE, 140011999576064, 140012007964671, +STORE, 140011714351104, 140011756314623, +SNULL, 140011882143744, 140011890532351, +STORE, 140011890532352, 140011898925055, +STORE, 140011882143744, 140011890532351, +SNULL, 140011890536447, 140011898925055, +STORE, 140011890532352, 140011890536447, +STORE, 140011890536448, 140011898925055, +STORE, 140011630489600, 140011638882303, +STORE, 140011613708288, 140011638882303, +STORE, 140011605311488, 140011613704191, +STORE, 140011596918784, 140011613704191, +STORE, 140011588526080, 140011613704191, +SNULL, 140011487883264, 140011496271871, +STORE, 140011496271872, 140011504664575, +STORE, 140011487883264, 140011496271871, +SNULL, 140011496275967, 140011504664575, +STORE, 140011496271872, 140011496275967, +STORE, 140011496275968, 140011504664575, +STORE, 140011580133376, 140011613704191, +SNULL, 140011580137471, 140011613704191, +STORE, 140011580133376, 140011580137471, +STORE, 140011580137472, 140011613704191, +SNULL, 140011982790656, 140011991179263, +STORE, 140011991179264, 140011999571967, +STORE, 140011982790656, 140011991179263, +SNULL, 140011991183359, 140011999571967, +STORE, 140011991179264, 140011991183359, +STORE, 140011991183360, 140011999571967, +SNULL, 140011865358335, 140011873746943, +STORE, 140011865354240, 140011865358335, +STORE, 140011865358336, 140011873746943, +STORE, 140011462701056, 140011471093759, +SNULL, 140011714351104, 140011739529215, +STORE, 140011739529216, 140011756314623, +STORE, 140011714351104, 140011739529215, +SNULL, 140011739533311, 140011756314623, +STORE, 140011739529216, 140011739533311, +STORE, 140011739533312, 140011756314623, +SNULL, 140011739533312, 140011747921919, +STORE, 140011747921920, 140011756314623, +STORE, 140011739533312, 140011747921919, +SNULL, 140011747926015, 140011756314623, +STORE, 140011747921920, 140011747926015, +STORE, 140011747926016, 140011756314623, +SNULL, 140011613708288, 140011630489599, +STORE, 140011630489600, 140011638882303, +STORE, 140011613708288, 140011630489599, +SNULL, 140011630493695, 140011638882303, +STORE, 140011630489600, 140011630493695, +STORE, 140011630493696, 140011638882303, +SNULL, 140011714351104, 140011722743807, +STORE, 140011722743808, 140011739529215, +STORE, 140011714351104, 140011722743807, +SNULL, 140011722747903, 140011739529215, +STORE, 140011722743808, 140011722747903, +STORE, 140011722747904, 140011739529215, +SNULL, 140011714355199, 140011722743807, +STORE, 140011714351104, 140011714355199, +STORE, 140011714355200, 140011722743807, +SNULL, 140011722747904, 140011731136511, +STORE, 140011731136512, 140011739529215, +STORE, 140011722747904, 140011731136511, +SNULL, 140011731140607, 140011739529215, +STORE, 140011731136512, 140011731140607, +STORE, 140011731140608, 140011739529215, +STORE, 140011454308352, 140011471093759, +STORE, 140011445915648, 140011471093759, +SNULL, 140011580137472, 140011588526079, +STORE, 140011588526080, 140011613704191, +STORE, 140011580137472, 140011588526079, +SNULL, 140011588530175, 140011613704191, +STORE, 140011588526080, 140011588530175, +STORE, 140011588530176, 140011613704191, +SNULL, 140011445915648, 140011462701055, +STORE, 140011462701056, 140011471093759, +STORE, 140011445915648, 140011462701055, +SNULL, 140011462705151, 140011471093759, +STORE, 140011462701056, 140011462705151, +STORE, 140011462705152, 140011471093759, +SNULL, 140011588530176, 140011596918783, +STORE, 140011596918784, 140011613704191, +STORE, 140011588530176, 140011596918783, +SNULL, 140011596922879, 140011613704191, +STORE, 140011596918784, 140011596922879, +STORE, 140011596922880, 140011613704191, +SNULL, 140011596922880, 140011605311487, +STORE, 140011605311488, 140011613704191, +STORE, 140011596922880, 140011605311487, +SNULL, 140011605315583, 140011613704191, +STORE, 140011605311488, 140011605315583, +STORE, 140011605315584, 140011613704191, +SNULL, 140011613708288, 140011622096895, +STORE, 140011622096896, 140011630489599, +STORE, 140011613708288, 140011622096895, +SNULL, 140011622100991, 140011630489599, +STORE, 140011622096896, 140011622100991, +STORE, 140011622100992, 140011630489599, +STORE, 140011311697920, 140011320090623, +STORE, 140011227836416, 140011236229119, +STORE, 140011219443712, 140011236229119, +SNULL, 140011219447807, 140011236229119, +STORE, 140011219443712, 140011219447807, +STORE, 140011219447808, 140011236229119, +STORE, 140011211051008, 140011219443711, +STORE, 140011202658304, 140011219443711, +SNULL, 140011202662399, 140011219443711, +STORE, 140011202658304, 140011202662399, +STORE, 140011202662400, 140011219443711, +STORE, 140011194265600, 140011202658303, +STORE, 140011185872896, 140011202658303, +STORE, 140011177480192, 140011202658303, +STORE, 140011093618688, 140011102011391, +SNULL, 140011445915648, 140011454308351, +STORE, 140011454308352, 140011462701055, +STORE, 140011445915648, 140011454308351, +SNULL, 140011454312447, 140011462701055, +STORE, 140011454308352, 140011454312447, +STORE, 140011454312448, 140011462701055, +STORE, 140011085225984, 140011102011391, +SNULL, 140011085230079, 140011102011391, +STORE, 140011085225984, 140011085230079, +STORE, 140011085230080, 140011102011391, +SNULL, 140011177484287, 140011202658303, +STORE, 140011177480192, 140011177484287, +STORE, 140011177484288, 140011202658303, +SNULL, 140011445919743, 140011454308351, +STORE, 140011445915648, 140011445919743, +STORE, 140011445919744, 140011454308351, +SNULL, 140011177484288, 140011185872895, +STORE, 140011185872896, 140011202658303, +STORE, 140011177484288, 140011185872895, +SNULL, 140011185876991, 140011202658303, +STORE, 140011185872896, 140011185876991, +STORE, 140011185876992, 140011202658303, +STORE, 140011076833280, 140011085225983, +SNULL, 140011202662400, 140011211051007, +STORE, 140011211051008, 140011219443711, +STORE, 140011202662400, 140011211051007, +SNULL, 140011211055103, 140011219443711, +STORE, 140011211051008, 140011211055103, +STORE, 140011211055104, 140011219443711, +SNULL, 140011185876992, 140011194265599, +STORE, 140011194265600, 140011202658303, +STORE, 140011185876992, 140011194265599, +SNULL, 140011194269695, 140011202658303, +STORE, 140011194265600, 140011194269695, +STORE, 140011194269696, 140011202658303, +STORE, 140011068440576, 140011085225983, +SNULL, 140011311702015, 140011320090623, +STORE, 140011311697920, 140011311702015, +STORE, 140011311702016, 140011320090623, +STORE, 140011060047872, 140011085225983, +SNULL, 140011060051967, 140011085225983, +STORE, 140011060047872, 140011060051967, +STORE, 140011060051968, 140011085225983, +STORE, 140011051655168, 140011060047871, +STORE, 140011043262464, 140011060047871, +SNULL, 140011043266559, 140011060047871, +STORE, 140011043262464, 140011043266559, +STORE, 140011043266560, 140011060047871, +SNULL, 140011219447808, 140011227836415, +STORE, 140011227836416, 140011236229119, +STORE, 140011219447808, 140011227836415, +SNULL, 140011227840511, 140011236229119, +STORE, 140011227836416, 140011227840511, +STORE, 140011227840512, 140011236229119, +SNULL, 140011085230080, 140011093618687, +STORE, 140011093618688, 140011102011391, +STORE, 140011085230080, 140011093618687, +SNULL, 140011093622783, 140011102011391, +STORE, 140011093618688, 140011093622783, +STORE, 140011093622784, 140011102011391, +STORE, 140010959400960, 140010967793663, +STORE, 140010951008256, 140010967793663, +SNULL, 140010951008256, 140010959400959, +STORE, 140010959400960, 140010967793663, +STORE, 140010951008256, 140010959400959, +SNULL, 140010959405055, 140010967793663, +STORE, 140010959400960, 140010959405055, +STORE, 140010959405056, 140010967793663, +STORE, 140010942615552, 140010959400959, +STORE, 140010934222848, 140010959400959, +SNULL, 140011060051968, 140011076833279, +STORE, 140011076833280, 140011085225983, +STORE, 140011060051968, 140011076833279, +SNULL, 140011076837375, 140011085225983, +STORE, 140011076833280, 140011076837375, +STORE, 140011076837376, 140011085225983, +SNULL, 140011043266560, 140011051655167, +STORE, 140011051655168, 140011060047871, +STORE, 140011043266560, 140011051655167, +SNULL, 140011051659263, 140011060047871, +STORE, 140011051655168, 140011051659263, +STORE, 140011051659264, 140011060047871, +STORE, 140010925830144, 140010959400959, +SNULL, 140011060051968, 140011068440575, +STORE, 140011068440576, 140011076833279, +STORE, 140011060051968, 140011068440575, +SNULL, 140011068444671, 140011076833279, +STORE, 140011068440576, 140011068444671, +STORE, 140011068444672, 140011076833279, +STORE, 140010917437440, 140010959400959, +STORE, 140010909044736, 140010959400959, +STORE, 140010825183232, 140010833575935, +SNULL, 140010909044736, 140010942615551, +STORE, 140010942615552, 140010959400959, +STORE, 140010909044736, 140010942615551, +SNULL, 140010942619647, 140010959400959, +STORE, 140010942615552, 140010942619647, +STORE, 140010942619648, 140010959400959, +SNULL, 140010909044736, 140010934222847, +STORE, 140010934222848, 140010942615551, +STORE, 140010909044736, 140010934222847, +SNULL, 140010934226943, 140010942615551, +STORE, 140010934222848, 140010934226943, +STORE, 140010934226944, 140010942615551, +SNULL, 140010909048831, 140010934222847, +STORE, 140010909044736, 140010909048831, +STORE, 140010909048832, 140010934222847, +STORE, 140010816790528, 140010833575935, +SNULL, 140010816794623, 140010833575935, +STORE, 140010816790528, 140010816794623, +STORE, 140010816794624, 140010833575935, +STORE, 140010808397824, 140010816790527, +SNULL, 140010942619648, 140010951008255, +STORE, 140010951008256, 140010959400959, +STORE, 140010942619648, 140010951008255, +SNULL, 140010951012351, 140010959400959, +STORE, 140010951008256, 140010951012351, +STORE, 140010951012352, 140010959400959, +STORE, 140010800005120, 140010816790527, +SNULL, 140010800009215, 140010816790527, +STORE, 140010800005120, 140010800009215, +STORE, 140010800009216, 140010816790527, +SNULL, 140010909048832, 140010925830143, +STORE, 140010925830144, 140010934222847, +STORE, 140010909048832, 140010925830143, +SNULL, 140010925834239, 140010934222847, +STORE, 140010925830144, 140010925834239, +STORE, 140010925834240, 140010934222847, +SNULL, 140010816794624, 140010825183231, +STORE, 140010825183232, 140010833575935, +STORE, 140010816794624, 140010825183231, +SNULL, 140010825187327, 140010833575935, +STORE, 140010825183232, 140010825187327, +STORE, 140010825187328, 140010833575935, +SNULL, 140010909048832, 140010917437439, +STORE, 140010917437440, 140010925830143, +STORE, 140010909048832, 140010917437439, +SNULL, 140010917441535, 140010925830143, +STORE, 140010917437440, 140010917441535, +STORE, 140010917441536, 140010925830143, +SNULL, 140010800009216, 140010808397823, +STORE, 140010808397824, 140010816790527, +STORE, 140010800009216, 140010808397823, +SNULL, 140010808401919, 140010816790527, +STORE, 140010808397824, 140010808401919, +STORE, 140010808401920, 140010816790527, +STORE, 140010791612416, 140010800005119, +SNULL, 140010791616511, 140010800005119, +STORE, 140010791612416, 140010791616511, +STORE, 140010791616512, 140010800005119, +STORE, 140012547100672, 140012547129343, +STORE, 140012511506432, 140012513697791, +SNULL, 140012511506432, 140012511596543, +STORE, 140012511596544, 140012513697791, +STORE, 140012511506432, 140012511596543, +SNULL, 140012513689599, 140012513697791, +STORE, 140012511596544, 140012513689599, +STORE, 140012513689600, 140012513697791, +ERASE, 140012513689600, 140012513697791, +STORE, 140012513689600, 140012513697791, +SNULL, 140012513693695, 140012513697791, +STORE, 140012513689600, 140012513693695, +STORE, 140012513693696, 140012513697791, +ERASE, 140012547100672, 140012547129343, +ERASE, 140011362054144, 140011362058239, +ERASE, 140011362058240, 140011370446847, +ERASE, 140011882139648, 140011882143743, +ERASE, 140011882143744, 140011890532351, +ERASE, 140011873746944, 140011873751039, +ERASE, 140011873751040, 140011882139647, +ERASE, 140011588526080, 140011588530175, +ERASE, 140011588530176, 140011596918783, +ERASE, 140011328483328, 140011328487423, +ERASE, 140011328487424, 140011336876031, +ERASE, 140011898925056, 140011898929151, +ERASE, 140011898929152, 140011907317759, +ERASE, 140011353661440, 140011353665535, +ERASE, 140011353665536, 140011362054143, +ERASE, 140011336876032, 140011336880127, +ERASE, 140011336880128, 140011345268735, +ERASE, 140011731136512, 140011731140607, +ERASE, 140011731140608, 140011739529215, +ERASE, 140011479486464, 140011479490559, +ERASE, 140011479490560, 140011487879167, +ERASE, 140011756314624, 140011756318719, +ERASE, 140011756318720, 140011764707327, +ERASE, 140011580133376, 140011580137471, +ERASE, 140011580137472, 140011588526079, +ERASE, 140011219443712, 140011219447807, +ERASE, 140011219447808, 140011227836415, +ERASE, 140011051655168, 140011051659263, +ERASE, 140011051659264, 140011060047871, +ERASE, 140011999571968, 140011999576063, +ERASE, 140011999576064, 140012007964671, +ERASE, 140011714351104, 140011714355199, +ERASE, 140011714355200, 140011722743807, +ERASE, 140011739529216, 140011739533311, +ERASE, 140011739533312, 140011747921919, +ERASE, 140011320090624, 140011320094719, +ERASE, 140011320094720, 140011328483327, +ERASE, 140011630489600, 140011630493695, +ERASE, 140011630493696, 140011638882303, +ERASE, 140011345268736, 140011345272831, +ERASE, 140011345272832, 140011353661439, +ERASE, 140011496271872, 140011496275967, +ERASE, 140011496275968, 140011504664575, +ERASE, 140011194265600, 140011194269695, +ERASE, 140011194269696, 140011202658303, +ERASE, 140011068440576, 140011068444671, +ERASE, 140011068444672, 140011076833279, +ERASE, 140010909044736, 140010909048831, +ERASE, 140010909048832, 140010917437439, +ERASE, 140011764707328, 140011764711423, +ERASE, 140011764711424, 140011773100031, +ERASE, 140011462701056, 140011462705151, +ERASE, 140011462705152, 140011471093759, +ERASE, 140011076833280, 140011076837375, +ERASE, 140011076837376, 140011085225983, +ERASE, 140011991179264, 140011991183359, +ERASE, 140011991183360, 140011999571967, +ERASE, 140011211051008, 140011211055103, +ERASE, 140011211055104, 140011219443711, +ERASE, 140010917437440, 140010917441535, +ERASE, 140010917441536, 140010925830143, +ERASE, 140011085225984, 140011085230079, +ERASE, 140011085230080, 140011093618687, +ERASE, 140011487879168, 140011487883263, +ERASE, 140011487883264, 140011496271871, +ERASE, 140011856961536, 140011856965631, +ERASE, 140011856965632, 140011865354239, +ERASE, 140011982786560, 140011982790655, +ERASE, 140011982790656, 140011991179263, +ERASE, 140011722743808, 140011722747903, +ERASE, 140011722747904, 140011731136511, +ERASE, 140011177480192, 140011177484287, +ERASE, 140011177484288, 140011185872895, +ERASE, 140011848568832, 140011848572927, +ERASE, 140011848572928, 140011856961535, +ERASE, 140011890532352, 140011890536447, +ERASE, 140011890536448, 140011898925055, +ERASE, 140011622096896, 140011622100991, +ERASE, 140011622100992, 140011630489599, +ERASE, 140011311697920, 140011311702015, +ERASE, 140011311702016, 140011320090623, +ERASE, 140011471093760, 140011471097855, +ERASE, 140011471097856, 140011479486463, +ERASE, 140011605311488, 140011605315583, +ERASE, 140011605315584, 140011613704191, +ERASE, 140010791612416, 140010791616511, +ERASE, 140010791616512, 140010800005119, +ERASE, 140010959400960, 140010959405055, +ERASE, 140010959405056, 140010967793663, +ERASE, 140011185872896, 140011185876991, +ERASE, 140011185876992, 140011194265599, +ERASE, 140011454308352, 140011454312447, +ERASE, 140011454312448, 140011462701055, +ERASE, 140011596918784, 140011596922879, +ERASE, 140011596922880, 140011605311487, +ERASE, 140011060047872, 140011060051967, +ERASE, 140011060051968, 140011068440575, +ERASE, 140010925830144, 140010925834239, +ERASE, 140010925834240, 140010934222847, +ERASE, 140011747921920, 140011747926015, +ERASE, 140011747926016, 140011756314623, +ERASE, 140011202658304, 140011202662399, +ERASE, 140011202662400, 140011211051007, +ERASE, 140010800005120, 140010800009215, +ERASE, 140010800009216, 140010808397823, +ERASE, 140011093618688, 140011093622783, +ERASE, 140011093622784, 140011102011391, +ERASE, 140010808397824, 140010808401919, +ERASE, 140010808401920, 140010816790527, +ERASE, 140012419010560, 140012419014655, +ERASE, 140012419014656, 140012427403263, +ERASE, 140010934222848, 140010934226943, +ERASE, 140010934226944, 140010942615551, +ERASE, 140010942615552, 140010942619647, +ERASE, 140010942619648, 140010951008255, +ERASE, 140011613704192, 140011613708287, +ERASE, 140011613708288, 140011622096895, +ERASE, 140011865354240, 140011865358335, +ERASE, 140011865358336, 140011873746943, +ERASE, 140012301578240, 140012301582335, +ERASE, 140012301582336, 140012309970943, +ERASE, 140012393832448, 140012393836543, +ERASE, 140012393836544, 140012402225151, +ERASE, 140012410617856, 140012410621951, +ERASE, 140012410621952, 140012419010559, +ERASE, 140012402225152, 140012402229247, +ERASE, 140012402229248, 140012410617855, +ERASE, 140012259614720, 140012259618815, +ERASE, 140012259618816, 140012268007423, +ERASE, 140012251222016, 140012251226111, +ERASE, 140012251226112, 140012259614719, +ERASE, 140012284792832, 140012284796927, +ERASE, 140012284796928, 140012293185535, +ERASE, 140011445915648, 140011445919743, +ERASE, 140011445919744, 140011454308351, +ERASE, 140010951008256, 140010951012351, +ERASE, 140010951012352, 140010959400959, +ERASE, 140011043262464, 140011043266559, +ERASE, 140011043266560, 140011051655167, +ERASE, 140010825183232, 140010825187327, +ERASE, 140010825187328, 140010833575935, +ERASE, 140012293185536, 140012293189631, +ERASE, 140012293189632, 140012301578239, +ERASE, 140012276400128, 140012276404223, +ERASE, 140012276404224, 140012284792831, +ERASE, 140012016357376, 140012016361471, +ERASE, 140012016361472, 140012024750079, +ERASE, 140012024750080, 140012024754175, +ERASE, 140012024754176, 140012033142783, +ERASE, 140011227836416, 140011227840511, +ERASE, 140011227840512, 140011236229119, +ERASE, 140010816790528, 140010816794623, +ERASE, 140010816794624, 140010825183231, +ERASE, 140012268007424, 140012268011519, +ERASE, 140012268011520, 140012276400127, +ERASE, 140012385439744, 140012385443839, +ERASE, 140012385443840, 140012393832447, +ERASE, 140012522090496, 140012522094591, +ERASE, 140012522094592, 140012530483199, +ERASE, 140012033142784, 140012033146879, +ERASE, 140012033146880, 140012041535487, + }; + unsigned long set35[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730536939520, 140737488351231, +SNULL, 140730536943615, 140737488351231, +STORE, 140730536939520, 140730536943615, +STORE, 140730536808448, 140730536943615, +STORE, 94245239877632, 94245242130431, +SNULL, 94245240008703, 94245242130431, +STORE, 94245239877632, 94245240008703, +STORE, 94245240008704, 94245242130431, +ERASE, 94245240008704, 94245242130431, +STORE, 94245242101760, 94245242109951, +STORE, 94245242109952, 94245242130431, +STORE, 140475575263232, 140475577516031, +SNULL, 140475575406591, 140475577516031, +STORE, 140475575263232, 140475575406591, +STORE, 140475575406592, 140475577516031, +ERASE, 140475575406592, 140475577516031, +STORE, 140475577503744, 140475577511935, +STORE, 140475577511936, 140475577516031, +STORE, 140730538164224, 140730538168319, +STORE, 140730538151936, 140730538164223, +STORE, 140475577475072, 140475577503743, +STORE, 140475577466880, 140475577475071, +STORE, 140475573047296, 140475575263231, +SNULL, 140475573047296, 140475573145599, +STORE, 140475573145600, 140475575263231, +STORE, 140475573047296, 140475573145599, +SNULL, 140475575238655, 140475575263231, +STORE, 140475573145600, 140475575238655, +STORE, 140475575238656, 140475575263231, +SNULL, 140475575238656, 140475575246847, +STORE, 140475575246848, 140475575263231, +STORE, 140475575238656, 140475575246847, +ERASE, 140475575238656, 140475575246847, +STORE, 140475575238656, 140475575246847, +ERASE, 140475575246848, 140475575263231, +STORE, 140475575246848, 140475575263231, +STORE, 140475569250304, 140475573047295, +SNULL, 140475569250304, 140475570909183, +STORE, 140475570909184, 140475573047295, +STORE, 140475569250304, 140475570909183, +SNULL, 140475573006335, 140475573047295, +STORE, 140475570909184, 140475573006335, +STORE, 140475573006336, 140475573047295, +SNULL, 140475573006336, 140475573030911, +STORE, 140475573030912, 140475573047295, +STORE, 140475573006336, 140475573030911, +ERASE, 140475573006336, 140475573030911, +STORE, 140475573006336, 140475573030911, +ERASE, 140475573030912, 140475573047295, +STORE, 140475573030912, 140475573047295, +STORE, 140475577458688, 140475577475071, +SNULL, 140475573022719, 140475573030911, +STORE, 140475573006336, 140475573022719, +STORE, 140475573022720, 140475573030911, +SNULL, 140475575242751, 140475575246847, +STORE, 140475575238656, 140475575242751, +STORE, 140475575242752, 140475575246847, +SNULL, 94245242105855, 94245242109951, +STORE, 94245242101760, 94245242105855, +STORE, 94245242105856, 94245242109951, +SNULL, 140475577507839, 140475577511935, +STORE, 140475577503744, 140475577507839, +STORE, 140475577507840, 140475577511935, +ERASE, 140475577475072, 140475577503743, +STORE, 94245271216128, 94245271351295, +STORE, 140475560857600, 140475569250303, +SNULL, 140475560861695, 140475569250303, +STORE, 140475560857600, 140475560861695, +STORE, 140475560861696, 140475569250303, +STORE, 140475552464896, 140475560857599, +STORE, 140475418247168, 140475552464895, +SNULL, 140475418247168, 140475428241407, +STORE, 140475428241408, 140475552464895, +STORE, 140475418247168, 140475428241407, +ERASE, 140475418247168, 140475428241407, +SNULL, 140475495350271, 140475552464895, +STORE, 140475428241408, 140475495350271, +STORE, 140475495350272, 140475552464895, +ERASE, 140475495350272, 140475552464895, +SNULL, 140475428376575, 140475495350271, +STORE, 140475428241408, 140475428376575, +STORE, 140475428376576, 140475495350271, +SNULL, 140475552468991, 140475560857599, +STORE, 140475552464896, 140475552468991, +STORE, 140475552468992, 140475560857599, +STORE, 140475544072192, 140475552464895, +SNULL, 140475544076287, 140475552464895, +STORE, 140475544072192, 140475544076287, +STORE, 140475544076288, 140475552464895, +STORE, 140475535679488, 140475544072191, +SNULL, 140475535683583, 140475544072191, +STORE, 140475535679488, 140475535683583, +STORE, 140475535683584, 140475544072191, +STORE, 140475527286784, 140475535679487, +SNULL, 140475527290879, 140475535679487, +STORE, 140475527286784, 140475527290879, +STORE, 140475527290880, 140475535679487, +STORE, 140475518894080, 140475527286783, +STORE, 140475510501376, 140475527286783, +STORE, 140475502108672, 140475527286783, +STORE, 140475419848704, 140475428241407, +STORE, 140475285630976, 140475419848703, +SNULL, 140475285630976, 140475294023679, +STORE, 140475294023680, 140475419848703, +STORE, 140475285630976, 140475294023679, +ERASE, 140475285630976, 140475294023679, +STORE, 140475159805952, 140475419848703, +STORE, 140475025588224, 140475419848703, +SNULL, 140475092697087, 140475419848703, +STORE, 140475025588224, 140475092697087, +STORE, 140475092697088, 140475419848703, +SNULL, 140475092697088, 140475159805951, +STORE, 140475159805952, 140475419848703, +STORE, 140475092697088, 140475159805951, +ERASE, 140475092697088, 140475159805951, +STORE, 140474891370496, 140475092697087, +SNULL, 140474958479359, 140475092697087, +STORE, 140474891370496, 140474958479359, +STORE, 140474958479360, 140475092697087, +SNULL, 140474958479360, 140475025588223, +STORE, 140475025588224, 140475092697087, +STORE, 140474958479360, 140475025588223, +ERASE, 140474958479360, 140475025588223, +SNULL, 140475361132543, 140475419848703, +STORE, 140475159805952, 140475361132543, +STORE, 140475361132544, 140475419848703, +ERASE, 140475361132544, 140475419848703, +SNULL, 140475159805952, 140475294023679, +STORE, 140475294023680, 140475361132543, +STORE, 140475159805952, 140475294023679, +SNULL, 140475294158847, 140475361132543, +STORE, 140475294023680, 140475294158847, +STORE, 140475294158848, 140475361132543, +SNULL, 140475226914815, 140475294023679, +STORE, 140475159805952, 140475226914815, +STORE, 140475226914816, 140475294023679, +ERASE, 140475226914816, 140475294023679, +SNULL, 140475025723391, 140475092697087, +STORE, 140475025588224, 140475025723391, +STORE, 140475025723392, 140475092697087, +SNULL, 140475159941119, 140475226914815, +STORE, 140475159805952, 140475159941119, +STORE, 140475159941120, 140475226914815, +SNULL, 140474891505663, 140474958479359, +STORE, 140474891370496, 140474891505663, +STORE, 140474891505664, 140474958479359, +SNULL, 140475502108672, 140475518894079, +STORE, 140475518894080, 140475527286783, +STORE, 140475502108672, 140475518894079, +SNULL, 140475518898175, 140475527286783, +STORE, 140475518894080, 140475518898175, +STORE, 140475518898176, 140475527286783, +STORE, 140475411456000, 140475428241407, +SNULL, 140475502112767, 140475518894079, +STORE, 140475502108672, 140475502112767, +STORE, 140475502112768, 140475518894079, +SNULL, 140475411460095, 140475428241407, +STORE, 140475411456000, 140475411460095, +STORE, 140475411460096, 140475428241407, +SNULL, 140475411460096, 140475419848703, +STORE, 140475419848704, 140475428241407, +STORE, 140475411460096, 140475419848703, +SNULL, 140475419852799, 140475428241407, +STORE, 140475419848704, 140475419852799, +STORE, 140475419852800, 140475428241407, +STORE, 140475403063296, 140475411455999, +SNULL, 140475502112768, 140475510501375, +STORE, 140475510501376, 140475518894079, +STORE, 140475502112768, 140475510501375, +SNULL, 140475510505471, 140475518894079, +STORE, 140475510501376, 140475510505471, +STORE, 140475510505472, 140475518894079, +SNULL, 140475403067391, 140475411455999, +STORE, 140475403063296, 140475403067391, +STORE, 140475403067392, 140475411455999, +STORE, 140475394670592, 140475403063295, +SNULL, 140475394674687, 140475403063295, +STORE, 140475394670592, 140475394674687, +STORE, 140475394674688, 140475403063295, +STORE, 140475386277888, 140475394670591, +STORE, 140475377885184, 140475394670591, +STORE, 140475369492480, 140475394670591, +SNULL, 140475369496575, 140475394670591, +STORE, 140475369492480, 140475369496575, +STORE, 140475369496576, 140475394670591, +SNULL, 140475369496576, 140475377885183, +STORE, 140475377885184, 140475394670591, +STORE, 140475369496576, 140475377885183, +SNULL, 140475377889279, 140475394670591, +STORE, 140475377885184, 140475377889279, +STORE, 140475377889280, 140475394670591, +STORE, 140475285630976, 140475294023679, +SNULL, 140475377889280, 140475386277887, +STORE, 140475386277888, 140475394670591, +STORE, 140475377889280, 140475386277887, +SNULL, 140475386281983, 140475394670591, +STORE, 140475386277888, 140475386281983, +STORE, 140475386281984, 140475394670591, +SNULL, 140475285635071, 140475294023679, +STORE, 140475285630976, 140475285635071, +STORE, 140475285635072, 140475294023679, +STORE, 140475277238272, 140475285630975, +STORE, 140475268845568, 140475285630975, +SNULL, 140475268845568, 140475277238271, +STORE, 140475277238272, 140475285630975, +STORE, 140475268845568, 140475277238271, +SNULL, 140475277242367, 140475285630975, +STORE, 140475277238272, 140475277242367, +STORE, 140475277242368, 140475285630975, +STORE, 140475260452864, 140475277238271, +SNULL, 140475260452864, 140475268845567, +STORE, 140475268845568, 140475277238271, +STORE, 140475260452864, 140475268845567, +SNULL, 140475268849663, 140475277238271, +STORE, 140475268845568, 140475268849663, +STORE, 140475268849664, 140475277238271, +SNULL, 140475260456959, 140475268845567, +STORE, 140475260452864, 140475260456959, +STORE, 140475260456960, 140475268845567, +STORE, 140475252060160, 140475260452863, +SNULL, 140475252064255, 140475260452863, +STORE, 140475252060160, 140475252064255, +STORE, 140475252064256, 140475260452863, +STORE, 140475243667456, 140475252060159, +SNULL, 140475243671551, 140475252060159, +STORE, 140475243667456, 140475243671551, +STORE, 140475243671552, 140475252060159, +STORE, 140475235274752, 140475243667455, +STORE, 140475151413248, 140475159805951, +STORE, 140474891505664, 140475025588223, +STORE, 140475143020544, 140475159805951, +SNULL, 140474891505664, 140474958479359, +STORE, 140474958479360, 140475025588223, +STORE, 140474891505664, 140474958479359, +SNULL, 140474958614527, 140475025588223, +STORE, 140474958479360, 140474958614527, +STORE, 140474958614528, 140475025588223, +STORE, 140474824261632, 140474891370495, +SNULL, 140474824396799, 140474891370495, +STORE, 140474824261632, 140474824396799, +STORE, 140474824396800, 140474891370495, +STORE, 140475134627840, 140475159805951, +STORE, 140474690043904, 140474824261631, +STORE, 140475126235136, 140475159805951, +STORE, 140475117842432, 140475159805951, +STORE, 140474622935040, 140474824261631, +STORE, 140475109449728, 140475159805951, +STORE, 140474488717312, 140474824261631, +STORE, 140475101057024, 140475159805951, +STORE, 140474480324608, 140474488717311, +STORE, 140474413215744, 140474480324607, +STORE, 140474404823040, 140474413215743, +ERASE, 140474413215744, 140474480324607, +STORE, 140474471931904, 140474488717311, +STORE, 140474270605312, 140474404823039, +SNULL, 140475101057024, 140475126235135, +STORE, 140475126235136, 140475159805951, +STORE, 140475101057024, 140475126235135, +SNULL, 140475126239231, 140475159805951, +STORE, 140475126235136, 140475126239231, +STORE, 140475126239232, 140475159805951, +STORE, 140474463539200, 140474488717311, +STORE, 140474455146496, 140474488717311, +SNULL, 140474455150591, 140474488717311, +STORE, 140474455146496, 140474455150591, +STORE, 140474455150592, 140474488717311, +STORE, 140474446753792, 140474455146495, +SNULL, 140474446757887, 140474455146495, +STORE, 140474446753792, 140474446757887, +STORE, 140474446757888, 140474455146495, +STORE, 140474438361088, 140474446753791, +STORE, 140474429968384, 140474446753791, +SNULL, 140474429972479, 140474446753791, +STORE, 140474429968384, 140474429972479, +STORE, 140474429972480, 140474446753791, +SNULL, 140475235278847, 140475243667455, +STORE, 140475235274752, 140475235278847, +STORE, 140475235278848, 140475243667455, +SNULL, 140474757152767, 140474824261631, +STORE, 140474488717312, 140474757152767, +STORE, 140474757152768, 140474824261631, +ERASE, 140474757152768, 140474824261631, +SNULL, 140474488717312, 140474690043903, +STORE, 140474690043904, 140474757152767, +STORE, 140474488717312, 140474690043903, +SNULL, 140474690179071, 140474757152767, +STORE, 140474690043904, 140474690179071, +STORE, 140474690179072, 140474757152767, +SNULL, 140474488717312, 140474622935039, +STORE, 140474622935040, 140474690043903, +STORE, 140474488717312, 140474622935039, +SNULL, 140474623070207, 140474690043903, +STORE, 140474622935040, 140474623070207, +STORE, 140474623070208, 140474690043903, +SNULL, 140475101057024, 140475117842431, +STORE, 140475117842432, 140475126235135, +STORE, 140475101057024, 140475117842431, +SNULL, 140475117846527, 140475126235135, +STORE, 140475117842432, 140475117846527, +STORE, 140475117846528, 140475126235135, +SNULL, 140474555826175, 140474622935039, +STORE, 140474488717312, 140474555826175, +STORE, 140474555826176, 140474622935039, +ERASE, 140474555826176, 140474622935039, +STORE, 140474136387584, 140474404823039, +SNULL, 140474136387584, 140474153172991, +STORE, 140474153172992, 140474404823039, +STORE, 140474136387584, 140474153172991, +ERASE, 140474136387584, 140474153172991, +STORE, 140474018955264, 140474404823039, +STORE, 140473884737536, 140474404823039, +SNULL, 140474086064127, 140474404823039, +STORE, 140473884737536, 140474086064127, +STORE, 140474086064128, 140474404823039, +SNULL, 140474086064128, 140474153172991, +STORE, 140474153172992, 140474404823039, +STORE, 140474086064128, 140474153172991, +ERASE, 140474086064128, 140474153172991, +STORE, 140473750519808, 140474086064127, +SNULL, 140473817628671, 140474086064127, +STORE, 140473750519808, 140473817628671, +STORE, 140473817628672, 140474086064127, +SNULL, 140473817628672, 140473884737535, +STORE, 140473884737536, 140474086064127, +STORE, 140473817628672, 140473884737535, +ERASE, 140473817628672, 140473884737535, +SNULL, 140475126239232, 140475151413247, +STORE, 140475151413248, 140475159805951, +STORE, 140475126239232, 140475151413247, +SNULL, 140475151417343, 140475159805951, +STORE, 140475151413248, 140475151417343, +STORE, 140475151417344, 140475159805951, +SNULL, 140474270605311, 140474404823039, +STORE, 140474153172992, 140474270605311, +STORE, 140474270605312, 140474404823039, +SNULL, 140474270605312, 140474287390719, +STORE, 140474287390720, 140474404823039, +STORE, 140474270605312, 140474287390719, +ERASE, 140474270605312, 140474287390719, +SNULL, 140474429972480, 140474438361087, +STORE, 140474438361088, 140474446753791, +STORE, 140474429972480, 140474438361087, +SNULL, 140474438365183, 140474446753791, +STORE, 140474438361088, 140474438365183, +STORE, 140474438365184, 140474446753791, +STORE, 140474815868928, 140474824261631, +SNULL, 140474815873023, 140474824261631, +STORE, 140474815868928, 140474815873023, +STORE, 140474815873024, 140474824261631, +SNULL, 140474220281855, 140474270605311, +STORE, 140474153172992, 140474220281855, +STORE, 140474220281856, 140474270605311, +ERASE, 140474220281856, 140474270605311, +SNULL, 140474488852479, 140474555826175, +STORE, 140474488717312, 140474488852479, +STORE, 140474488852480, 140474555826175, +SNULL, 140475101057024, 140475109449727, +STORE, 140475109449728, 140475117842431, +STORE, 140475101057024, 140475109449727, +SNULL, 140475109453823, 140475117842431, +STORE, 140475109449728, 140475109453823, +STORE, 140475109453824, 140475117842431, +SNULL, 140473951846399, 140474086064127, +STORE, 140473884737536, 140473951846399, +STORE, 140473951846400, 140474086064127, +SNULL, 140473951846400, 140474018955263, +STORE, 140474018955264, 140474086064127, +STORE, 140473951846400, 140474018955263, +ERASE, 140473951846400, 140474018955263, +SNULL, 140473884872703, 140473951846399, +STORE, 140473884737536, 140473884872703, +STORE, 140473884872704, 140473951846399, +SNULL, 140474019090431, 140474086064127, +STORE, 140474018955264, 140474019090431, +STORE, 140474019090432, 140474086064127, +SNULL, 140473750654975, 140473817628671, +STORE, 140473750519808, 140473750654975, +STORE, 140473750654976, 140473817628671, +SNULL, 140474455150592, 140474463539199, +STORE, 140474463539200, 140474488717311, +STORE, 140474455150592, 140474463539199, +SNULL, 140474463543295, 140474488717311, +STORE, 140474463539200, 140474463543295, +STORE, 140474463543296, 140474488717311, +STORE, 140474807476224, 140474815868927, +SNULL, 140474463543296, 140474471931903, +STORE, 140474471931904, 140474488717311, +STORE, 140474463543296, 140474471931903, +SNULL, 140474471935999, 140474488717311, +STORE, 140474471931904, 140474471935999, +STORE, 140474471936000, 140474488717311, +STORE, 140474799083520, 140474815868927, +STORE, 140474790690816, 140474815868927, +SNULL, 140474790690816, 140474799083519, +STORE, 140474799083520, 140474815868927, +STORE, 140474790690816, 140474799083519, +SNULL, 140474799087615, 140474815868927, +STORE, 140474799083520, 140474799087615, +STORE, 140474799087616, 140474815868927, +SNULL, 140474354499583, 140474404823039, +STORE, 140474287390720, 140474354499583, +STORE, 140474354499584, 140474404823039, +ERASE, 140474354499584, 140474404823039, +SNULL, 140474287525887, 140474354499583, +STORE, 140474287390720, 140474287525887, +STORE, 140474287525888, 140474354499583, +STORE, 140474782298112, 140474799083519, +STORE, 140474773905408, 140474799083519, +SNULL, 140474773909503, 140474799083519, +STORE, 140474773905408, 140474773909503, +STORE, 140474773909504, 140474799083519, +SNULL, 140475126239232, 140475134627839, +STORE, 140475134627840, 140475151413247, +STORE, 140475126239232, 140475134627839, +SNULL, 140475134631935, 140475151413247, +STORE, 140475134627840, 140475134631935, +STORE, 140475134631936, 140475151413247, +STORE, 140474765512704, 140474773905407, +STORE, 140474614542336, 140474622935039, +SNULL, 140474153308159, 140474220281855, +STORE, 140474153172992, 140474153308159, +STORE, 140474153308160, 140474220281855, +SNULL, 140474404827135, 140474413215743, +STORE, 140474404823040, 140474404827135, +STORE, 140474404827136, 140474413215743, +STORE, 140474606149632, 140474622935039, +SNULL, 140474606153727, 140474622935039, +STORE, 140474606149632, 140474606153727, +STORE, 140474606153728, 140474622935039, +STORE, 140474597756928, 140474606149631, +SNULL, 140474597761023, 140474606149631, +STORE, 140474597756928, 140474597761023, +STORE, 140474597761024, 140474606149631, +SNULL, 140475134631936, 140475143020543, +STORE, 140475143020544, 140475151413247, +STORE, 140475134631936, 140475143020543, +SNULL, 140475143024639, 140475151413247, +STORE, 140475143020544, 140475143024639, +STORE, 140475143024640, 140475151413247, +STORE, 140474589364224, 140474597756927, +SNULL, 140474606153728, 140474614542335, +STORE, 140474614542336, 140474622935039, +STORE, 140474606153728, 140474614542335, +SNULL, 140474614546431, 140474622935039, +STORE, 140474614542336, 140474614546431, +STORE, 140474614546432, 140474622935039, +SNULL, 140474765516799, 140474773905407, +STORE, 140474765512704, 140474765516799, +STORE, 140474765516800, 140474773905407, +STORE, 140474580971520, 140474597756927, +SNULL, 140474773909504, 140474782298111, +STORE, 140474782298112, 140474799083519, +STORE, 140474773909504, 140474782298111, +SNULL, 140474782302207, 140474799083519, +STORE, 140474782298112, 140474782302207, +STORE, 140474782302208, 140474799083519, +SNULL, 140474471936000, 140474480324607, +STORE, 140474480324608, 140474488717311, +STORE, 140474471936000, 140474480324607, +SNULL, 140474480328703, 140474488717311, +STORE, 140474480324608, 140474480328703, +STORE, 140474480328704, 140474488717311, +STORE, 140474572578816, 140474597756927, +SNULL, 140474572582911, 140474597756927, +STORE, 140474572578816, 140474572582911, +STORE, 140474572582912, 140474597756927, +SNULL, 140474782302208, 140474790690815, +STORE, 140474790690816, 140474799083519, +STORE, 140474782302208, 140474790690815, +SNULL, 140474790694911, 140474799083519, +STORE, 140474790690816, 140474790694911, +STORE, 140474790694912, 140474799083519, +STORE, 140474564186112, 140474572578815, +STORE, 140474421575680, 140474429968383, +STORE, 140474396430336, 140474404823039, +SNULL, 140474396434431, 140474404823039, +STORE, 140474396430336, 140474396434431, +STORE, 140474396434432, 140474404823039, +STORE, 140474388037632, 140474396430335, +SNULL, 140474799087616, 140474807476223, +STORE, 140474807476224, 140474815868927, +STORE, 140474799087616, 140474807476223, +SNULL, 140474807480319, 140474815868927, +STORE, 140474807476224, 140474807480319, +STORE, 140474807480320, 140474815868927, +SNULL, 140475101061119, 140475109449727, +STORE, 140475101057024, 140475101061119, +STORE, 140475101061120, 140475109449727, +STORE, 140474379644928, 140474396430335, +SNULL, 140474572582912, 140474589364223, +STORE, 140474589364224, 140474597756927, +STORE, 140474572582912, 140474589364223, +SNULL, 140474589368319, 140474597756927, +STORE, 140474589364224, 140474589368319, +STORE, 140474589368320, 140474597756927, +STORE, 140474371252224, 140474396430335, +STORE, 140474362859520, 140474396430335, +STORE, 140474278998016, 140474287390719, +STORE, 140474270605312, 140474287390719, +STORE, 140474262212608, 140474287390719, +SNULL, 140474262216703, 140474287390719, +STORE, 140474262212608, 140474262216703, +STORE, 140474262216704, 140474287390719, +STORE, 140474253819904, 140474262212607, +SNULL, 140474253823999, 140474262212607, +STORE, 140474253819904, 140474253823999, +STORE, 140474253824000, 140474262212607, +SNULL, 140474362859520, 140474388037631, +STORE, 140474388037632, 140474396430335, +STORE, 140474362859520, 140474388037631, +SNULL, 140474388041727, 140474396430335, +STORE, 140474388037632, 140474388041727, +STORE, 140474388041728, 140474396430335, +SNULL, 140474362859520, 140474379644927, +STORE, 140474379644928, 140474388037631, +STORE, 140474362859520, 140474379644927, +SNULL, 140474379649023, 140474388037631, +STORE, 140474379644928, 140474379649023, +STORE, 140474379649024, 140474388037631, +STORE, 140474245427200, 140474253819903, +STORE, 140474237034496, 140474253819903, +STORE, 140474228641792, 140474253819903, +STORE, 140474144780288, 140474153172991, +SNULL, 140474228645887, 140474253819903, +STORE, 140474228641792, 140474228645887, +STORE, 140474228645888, 140474253819903, +SNULL, 140474564190207, 140474572578815, +STORE, 140474564186112, 140474564190207, +STORE, 140474564190208, 140474572578815, +STORE, 140474136387584, 140474153172991, +SNULL, 140474362859520, 140474371252223, +STORE, 140474371252224, 140474379644927, +STORE, 140474362859520, 140474371252223, +SNULL, 140474371256319, 140474379644927, +STORE, 140474371252224, 140474371256319, +STORE, 140474371256320, 140474379644927, +STORE, 140474127994880, 140474153172991, +STORE, 140474119602176, 140474153172991, +SNULL, 140474421579775, 140474429968383, +STORE, 140474421575680, 140474421579775, +STORE, 140474421579776, 140474429968383, +STORE, 140474111209472, 140474153172991, +SNULL, 140474111213567, 140474153172991, +STORE, 140474111209472, 140474111213567, +STORE, 140474111213568, 140474153172991, +SNULL, 140474262216704, 140474270605311, +STORE, 140474270605312, 140474287390719, +STORE, 140474262216704, 140474270605311, +SNULL, 140474270609407, 140474287390719, +STORE, 140474270605312, 140474270609407, +STORE, 140474270609408, 140474287390719, +STORE, 140474102816768, 140474111209471, +SNULL, 140474102820863, 140474111209471, +STORE, 140474102816768, 140474102820863, +STORE, 140474102820864, 140474111209471, +SNULL, 140474270609408, 140474278998015, +STORE, 140474278998016, 140474287390719, +STORE, 140474270609408, 140474278998015, +SNULL, 140474279002111, 140474287390719, +STORE, 140474278998016, 140474279002111, +STORE, 140474279002112, 140474287390719, +STORE, 140474094424064, 140474102816767, +SNULL, 140474572582912, 140474580971519, +STORE, 140474580971520, 140474589364223, +STORE, 140474572582912, 140474580971519, +SNULL, 140474580975615, 140474589364223, +STORE, 140474580971520, 140474580975615, +STORE, 140474580975616, 140474589364223, +SNULL, 140474362863615, 140474371252223, +STORE, 140474362859520, 140474362863615, +STORE, 140474362863616, 140474371252223, +STORE, 140474010562560, 140474018955263, +SNULL, 140474228645888, 140474245427199, +STORE, 140474245427200, 140474253819903, +STORE, 140474228645888, 140474245427199, +SNULL, 140474245431295, 140474253819903, +STORE, 140474245427200, 140474245431295, +STORE, 140474245431296, 140474253819903, +SNULL, 140474111213568, 140474136387583, +STORE, 140474136387584, 140474153172991, +STORE, 140474111213568, 140474136387583, +SNULL, 140474136391679, 140474153172991, +STORE, 140474136387584, 140474136391679, +STORE, 140474136391680, 140474153172991, +STORE, 140474002169856, 140474018955263, +STORE, 140473993777152, 140474018955263, +SNULL, 140474111213568, 140474127994879, +STORE, 140474127994880, 140474136387583, +STORE, 140474111213568, 140474127994879, +SNULL, 140474127998975, 140474136387583, +STORE, 140474127994880, 140474127998975, +STORE, 140474127998976, 140474136387583, +SNULL, 140474228645888, 140474237034495, +STORE, 140474237034496, 140474245427199, +STORE, 140474228645888, 140474237034495, +SNULL, 140474237038591, 140474245427199, +STORE, 140474237034496, 140474237038591, +STORE, 140474237038592, 140474245427199, +SNULL, 140474136391680, 140474144780287, +STORE, 140474144780288, 140474153172991, +STORE, 140474136391680, 140474144780287, +SNULL, 140474144784383, 140474153172991, +STORE, 140474144780288, 140474144784383, +STORE, 140474144784384, 140474153172991, +STORE, 140473985384448, 140474018955263, +STORE, 140473976991744, 140474018955263, +STORE, 140473968599040, 140474018955263, +SNULL, 140473968603135, 140474018955263, +STORE, 140473968599040, 140473968603135, +STORE, 140473968603136, 140474018955263, +SNULL, 140474111213568, 140474119602175, +STORE, 140474119602176, 140474127994879, +STORE, 140474111213568, 140474119602175, +SNULL, 140474119606271, 140474127994879, +STORE, 140474119602176, 140474119606271, +STORE, 140474119606272, 140474127994879, +STORE, 140473960206336, 140473968599039, +SNULL, 140474094428159, 140474102816767, +STORE, 140474094424064, 140474094428159, +STORE, 140474094428160, 140474102816767, +STORE, 140473876344832, 140473884737535, +STORE, 140473867952128, 140473884737535, +STORE, 140473859559424, 140473884737535, +SNULL, 140473859563519, 140473884737535, +STORE, 140473859559424, 140473859563519, +STORE, 140473859563520, 140473884737535, +SNULL, 140473968603136, 140473993777151, +STORE, 140473993777152, 140474018955263, +STORE, 140473968603136, 140473993777151, +SNULL, 140473993781247, 140474018955263, +STORE, 140473993777152, 140473993781247, +STORE, 140473993781248, 140474018955263, +SNULL, 140473960210431, 140473968599039, +STORE, 140473960206336, 140473960210431, +STORE, 140473960210432, 140473968599039, +SNULL, 140473993781248, 140474010562559, +STORE, 140474010562560, 140474018955263, +STORE, 140473993781248, 140474010562559, +SNULL, 140474010566655, 140474018955263, +STORE, 140474010562560, 140474010566655, +STORE, 140474010566656, 140474018955263, +SNULL, 140473968603136, 140473985384447, +STORE, 140473985384448, 140473993777151, +STORE, 140473968603136, 140473985384447, +SNULL, 140473985388543, 140473993777151, +STORE, 140473985384448, 140473985388543, +STORE, 140473985388544, 140473993777151, +SNULL, 140473993781248, 140474002169855, +STORE, 140474002169856, 140474010562559, +STORE, 140473993781248, 140474002169855, +SNULL, 140474002173951, 140474010562559, +STORE, 140474002169856, 140474002173951, +STORE, 140474002173952, 140474010562559, +STORE, 140473851166720, 140473859559423, +SNULL, 140473851170815, 140473859559423, +STORE, 140473851166720, 140473851170815, +STORE, 140473851170816, 140473859559423, +SNULL, 140473968603136, 140473976991743, +STORE, 140473976991744, 140473985384447, +STORE, 140473968603136, 140473976991743, +SNULL, 140473976995839, 140473985384447, +STORE, 140473976991744, 140473976995839, +STORE, 140473976995840, 140473985384447, +STORE, 140473842774016, 140473851166719, +SNULL, 140473859563520, 140473867952127, +STORE, 140473867952128, 140473884737535, +STORE, 140473859563520, 140473867952127, +SNULL, 140473867956223, 140473884737535, +STORE, 140473867952128, 140473867956223, +STORE, 140473867956224, 140473884737535, +SNULL, 140473867956224, 140473876344831, +STORE, 140473876344832, 140473884737535, +STORE, 140473867956224, 140473876344831, +SNULL, 140473876348927, 140473884737535, +STORE, 140473876344832, 140473876348927, +STORE, 140473876348928, 140473884737535, +STORE, 140473834381312, 140473851166719, +SNULL, 140473834385407, 140473851166719, +STORE, 140473834381312, 140473834385407, +STORE, 140473834385408, 140473851166719, +SNULL, 140473834385408, 140473842774015, +STORE, 140473842774016, 140473851166719, +STORE, 140473834385408, 140473842774015, +SNULL, 140473842778111, 140473851166719, +STORE, 140473842774016, 140473842778111, +STORE, 140473842778112, 140473851166719, +STORE, 140473825988608, 140473834381311, +SNULL, 140473825992703, 140473834381311, +STORE, 140473825988608, 140473825992703, +STORE, 140473825992704, 140473834381311, +STORE, 140475577475072, 140475577503743, +STORE, 140475499917312, 140475502108671, +SNULL, 140475499917312, 140475500007423, +STORE, 140475500007424, 140475502108671, +STORE, 140475499917312, 140475500007423, +SNULL, 140475502100479, 140475502108671, +STORE, 140475500007424, 140475502100479, +STORE, 140475502100480, 140475502108671, +ERASE, 140475502100480, 140475502108671, +STORE, 140475502100480, 140475502108671, +SNULL, 140475502104575, 140475502108671, +STORE, 140475502100480, 140475502104575, +STORE, 140475502104576, 140475502108671, +ERASE, 140475577475072, 140475577503743, +ERASE, 140475235274752, 140475235278847, +ERASE, 140475235278848, 140475243667455, +ERASE, 140474815868928, 140474815873023, +ERASE, 140474815873024, 140474824261631, +ERASE, 140474606149632, 140474606153727, +ERASE, 140474606153728, 140474614542335, +ERASE, 140474270605312, 140474270609407, +ERASE, 140474270609408, 140474278998015, +ERASE, 140474438361088, 140474438365183, +ERASE, 140474438365184, 140474446753791, +ERASE, 140474597756928, 140474597761023, +ERASE, 140474597761024, 140474606149631, +ERASE, 140475126235136, 140475126239231, +ERASE, 140475126239232, 140475134627839, +ERASE, 140474463539200, 140474463543295, +ERASE, 140474463543296, 140474471931903, +ERASE, 140474388037632, 140474388041727, +ERASE, 140474388041728, 140474396430335, +ERASE, 140474404823040, 140474404827135, +ERASE, 140474404827136, 140474413215743, +ERASE, 140474278998016, 140474279002111, +ERASE, 140474279002112, 140474287390719, +ERASE, 140474094424064, 140474094428159, +ERASE, 140474094428160, 140474102816767, +ERASE, 140473867952128, 140473867956223, +ERASE, 140473867956224, 140473876344831, +ERASE, 140475151413248, 140475151417343, +ERASE, 140475151417344, 140475159805951, +ERASE, 140474455146496, 140474455150591, +ERASE, 140474455150592, 140474463539199, +ERASE, 140474807476224, 140474807480319, +ERASE, 140474807480320, 140474815868927, +ERASE, 140475117842432, 140475117846527, +ERASE, 140475117846528, 140475126235135, +ERASE, 140474446753792, 140474446757887, +ERASE, 140474446757888, 140474455146495, +ERASE, 140474429968384, 140474429972479, +ERASE, 140474429972480, 140474438361087, +ERASE, 140474782298112, 140474782302207, +ERASE, 140474782302208, 140474790690815, +ERASE, 140474136387584, 140474136391679, +ERASE, 140474136391680, 140474144780287, +ERASE, 140474002169856, 140474002173951, +ERASE, 140474002173952, 140474010562559, +ERASE, 140475134627840, 140475134631935, +ERASE, 140475134631936, 140475143020543, +ERASE, 140474471931904, 140474471935999, +ERASE, 140474471936000, 140474480324607, +ERASE, 140474396430336, 140474396434431, +ERASE, 140474396434432, 140474404823039, + }; + unsigned long set36[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140723893125120, 140737488351231, +SNULL, 140723893129215, 140737488351231, +STORE, 140723893125120, 140723893129215, +STORE, 140723892994048, 140723893129215, +STORE, 94076829786112, 94076832038911, +SNULL, 94076829917183, 94076832038911, +STORE, 94076829786112, 94076829917183, +STORE, 94076829917184, 94076832038911, +ERASE, 94076829917184, 94076832038911, +STORE, 94076832010240, 94076832018431, +STORE, 94076832018432, 94076832038911, +STORE, 140122444345344, 140122446598143, +SNULL, 140122444488703, 140122446598143, +STORE, 140122444345344, 140122444488703, +STORE, 140122444488704, 140122446598143, +ERASE, 140122444488704, 140122446598143, +STORE, 140122446585856, 140122446594047, +STORE, 140122446594048, 140122446598143, +STORE, 140723893538816, 140723893542911, +STORE, 140723893526528, 140723893538815, +STORE, 140122446557184, 140122446585855, +STORE, 140122446548992, 140122446557183, +STORE, 140122442129408, 140122444345343, +SNULL, 140122442129408, 140122442227711, +STORE, 140122442227712, 140122444345343, +STORE, 140122442129408, 140122442227711, +SNULL, 140122444320767, 140122444345343, +STORE, 140122442227712, 140122444320767, +STORE, 140122444320768, 140122444345343, +SNULL, 140122444320768, 140122444328959, +STORE, 140122444328960, 140122444345343, +STORE, 140122444320768, 140122444328959, +ERASE, 140122444320768, 140122444328959, +STORE, 140122444320768, 140122444328959, +ERASE, 140122444328960, 140122444345343, +STORE, 140122444328960, 140122444345343, +STORE, 140122438332416, 140122442129407, +SNULL, 140122438332416, 140122439991295, +STORE, 140122439991296, 140122442129407, +STORE, 140122438332416, 140122439991295, +SNULL, 140122442088447, 140122442129407, +STORE, 140122439991296, 140122442088447, +STORE, 140122442088448, 140122442129407, +SNULL, 140122442088448, 140122442113023, +STORE, 140122442113024, 140122442129407, +STORE, 140122442088448, 140122442113023, +ERASE, 140122442088448, 140122442113023, +STORE, 140122442088448, 140122442113023, +ERASE, 140122442113024, 140122442129407, +STORE, 140122442113024, 140122442129407, +STORE, 140122446540800, 140122446557183, +SNULL, 140122442104831, 140122442113023, +STORE, 140122442088448, 140122442104831, +STORE, 140122442104832, 140122442113023, +SNULL, 140122444324863, 140122444328959, +STORE, 140122444320768, 140122444324863, +STORE, 140122444324864, 140122444328959, +SNULL, 94076832014335, 94076832018431, +STORE, 94076832010240, 94076832014335, +STORE, 94076832014336, 94076832018431, +SNULL, 140122446589951, 140122446594047, +STORE, 140122446585856, 140122446589951, +STORE, 140122446589952, 140122446594047, +ERASE, 140122446557184, 140122446585855, +STORE, 94076845723648, 94076845858815, +STORE, 140122429939712, 140122438332415, +SNULL, 140122429943807, 140122438332415, +STORE, 140122429939712, 140122429943807, +STORE, 140122429943808, 140122438332415, +STORE, 140122421547008, 140122429939711, +STORE, 140122287329280, 140122421547007, +SNULL, 140122287329280, 140122301399039, +STORE, 140122301399040, 140122421547007, +STORE, 140122287329280, 140122301399039, +ERASE, 140122287329280, 140122301399039, +SNULL, 140122368507903, 140122421547007, +STORE, 140122301399040, 140122368507903, +STORE, 140122368507904, 140122421547007, +ERASE, 140122368507904, 140122421547007, +SNULL, 140122301534207, 140122368507903, +STORE, 140122301399040, 140122301534207, +STORE, 140122301534208, 140122368507903, +SNULL, 140122421551103, 140122429939711, +STORE, 140122421547008, 140122421551103, +STORE, 140122421551104, 140122429939711, +STORE, 140122413154304, 140122421547007, +SNULL, 140122413158399, 140122421547007, +STORE, 140122413154304, 140122413158399, +STORE, 140122413158400, 140122421547007, +STORE, 140122404761600, 140122413154303, +SNULL, 140122404765695, 140122413154303, +STORE, 140122404761600, 140122404765695, +STORE, 140122404765696, 140122413154303, +STORE, 140122396368896, 140122404761599, +SNULL, 140122396372991, 140122404761599, +STORE, 140122396368896, 140122396372991, +STORE, 140122396372992, 140122404761599, +STORE, 140122387976192, 140122396368895, +STORE, 140122167181312, 140122301399039, +SNULL, 140122234290175, 140122301399039, +STORE, 140122167181312, 140122234290175, +STORE, 140122234290176, 140122301399039, +ERASE, 140122234290176, 140122301399039, +SNULL, 140122167316479, 140122234290175, +STORE, 140122167181312, 140122167316479, +STORE, 140122167316480, 140122234290175, +STORE, 140122379583488, 140122396368895, +STORE, 140122371190784, 140122396368895, +STORE, 140122167316480, 140122301399039, +STORE, 140122158788608, 140122167181311, +SNULL, 140122371190784, 140122387976191, +STORE, 140122387976192, 140122396368895, +STORE, 140122371190784, 140122387976191, +SNULL, 140122387980287, 140122396368895, +STORE, 140122387976192, 140122387980287, +STORE, 140122387980288, 140122396368895, +SNULL, 140122167316480, 140122234290175, +STORE, 140122234290176, 140122301399039, +STORE, 140122167316480, 140122234290175, +SNULL, 140122234425343, 140122301399039, +STORE, 140122234290176, 140122234425343, +STORE, 140122234425344, 140122301399039, +STORE, 140122024570880, 140122158788607, +SNULL, 140122024570880, 140122032963583, +STORE, 140122032963584, 140122158788607, +STORE, 140122024570880, 140122032963583, +ERASE, 140122024570880, 140122032963583, +STORE, 140121898745856, 140122158788607, +STORE, 140121890353152, 140121898745855, +SNULL, 140122100072447, 140122158788607, +STORE, 140121898745856, 140122100072447, +STORE, 140122100072448, 140122158788607, +ERASE, 140122100072448, 140122158788607, +SNULL, 140121965854719, 140122100072447, +STORE, 140121898745856, 140121965854719, +STORE, 140121965854720, 140122100072447, +SNULL, 140121965854720, 140122032963583, +STORE, 140122032963584, 140122100072447, +STORE, 140121965854720, 140122032963583, +ERASE, 140121965854720, 140122032963583, +SNULL, 140121898881023, 140121965854719, +STORE, 140121898745856, 140121898881023, +STORE, 140121898881024, 140121965854719, +SNULL, 140121890357247, 140121898745855, +STORE, 140121890353152, 140121890357247, +STORE, 140121890357248, 140121898745855, +SNULL, 140122371190784, 140122379583487, +STORE, 140122379583488, 140122387976191, +STORE, 140122371190784, 140122379583487, +SNULL, 140122379587583, 140122387976191, +STORE, 140122379583488, 140122379587583, +STORE, 140122379587584, 140122387976191, +SNULL, 140122033098751, 140122100072447, +STORE, 140122032963584, 140122033098751, +STORE, 140122033098752, 140122100072447, +SNULL, 140122158792703, 140122167181311, +STORE, 140122158788608, 140122158792703, +STORE, 140122158792704, 140122167181311, +STORE, 140122150395904, 140122158788607, +STORE, 140122142003200, 140122158788607, +SNULL, 140122142007295, 140122158788607, +STORE, 140122142003200, 140122142007295, +STORE, 140122142007296, 140122158788607, +SNULL, 140122371194879, 140122379583487, +STORE, 140122371190784, 140122371194879, +STORE, 140122371194880, 140122379583487, +SNULL, 140122142007296, 140122150395903, +STORE, 140122150395904, 140122158788607, +STORE, 140122142007296, 140122150395903, +SNULL, 140122150399999, 140122158788607, +STORE, 140122150395904, 140122150399999, +STORE, 140122150400000, 140122158788607, +STORE, 140122133610496, 140122142003199, +STORE, 140122125217792, 140122142003199, +STORE, 140122116825088, 140122142003199, +SNULL, 140122116829183, 140122142003199, +STORE, 140122116825088, 140122116829183, +STORE, 140122116829184, 140122142003199, +SNULL, 140122116829184, 140122133610495, +STORE, 140122133610496, 140122142003199, +STORE, 140122116829184, 140122133610495, +SNULL, 140122133614591, 140122142003199, +STORE, 140122133610496, 140122133614591, +STORE, 140122133614592, 140122142003199, +SNULL, 140122116829184, 140122125217791, +STORE, 140122125217792, 140122133610495, +STORE, 140122116829184, 140122125217791, +SNULL, 140122125221887, 140122133610495, +STORE, 140122125217792, 140122125221887, +STORE, 140122125221888, 140122133610495, +STORE, 140122108432384, 140122116825087, +SNULL, 140122108436479, 140122116825087, +STORE, 140122108432384, 140122108436479, +STORE, 140122108436480, 140122116825087, +STORE, 140122024570880, 140122032963583, +STORE, 140122016178176, 140122032963583, +SNULL, 140122016182271, 140122032963583, +STORE, 140122016178176, 140122016182271, +STORE, 140122016182272, 140122032963583, +SNULL, 140122016182272, 140122024570879, +STORE, 140122024570880, 140122032963583, +STORE, 140122016182272, 140122024570879, +SNULL, 140122024574975, 140122032963583, +STORE, 140122024570880, 140122024574975, +STORE, 140122024574976, 140122032963583, +STORE, 140122007785472, 140122016178175, +SNULL, 140122007789567, 140122016178175, +STORE, 140122007785472, 140122007789567, +STORE, 140122007789568, 140122016178175, +STORE, 140121999392768, 140122007785471, +STORE, 140121991000064, 140122007785471, +SNULL, 140121991004159, 140122007785471, +STORE, 140121991000064, 140121991004159, +STORE, 140121991004160, 140122007785471, +SNULL, 140121991004160, 140121999392767, +STORE, 140121999392768, 140122007785471, +STORE, 140121991004160, 140121999392767, +SNULL, 140121999396863, 140122007785471, +STORE, 140121999392768, 140121999396863, +STORE, 140121999396864, 140122007785471, +STORE, 140121982607360, 140121991000063, +STORE, 140121823244288, 140121890353151, +ERASE, 140121823244288, 140121890353151, +STORE, 140121756135424, 140121890353151, +SNULL, 140121756135424, 140121764528127, +STORE, 140121764528128, 140121890353151, +STORE, 140121756135424, 140121764528127, +ERASE, 140121756135424, 140121764528127, +SNULL, 140121831636991, 140121890353151, +STORE, 140121764528128, 140121831636991, +STORE, 140121831636992, 140121890353151, +ERASE, 140121831636992, 140121890353151, +STORE, 140121974214656, 140121991000063, +STORE, 140121630310400, 140121831636991, +SNULL, 140121697419263, 140121831636991, +STORE, 140121630310400, 140121697419263, +STORE, 140121697419264, 140121831636991, +SNULL, 140121697419264, 140121764528127, +STORE, 140121764528128, 140121831636991, +STORE, 140121697419264, 140121764528127, +ERASE, 140121697419264, 140121764528127, +STORE, 140121881960448, 140121890353151, +STORE, 140121630310400, 140121831636991, +STORE, 140121873567744, 140121890353151, +SNULL, 140121630310400, 140121697419263, +STORE, 140121697419264, 140121831636991, +STORE, 140121630310400, 140121697419263, +SNULL, 140121697554431, 140121831636991, +STORE, 140121697419264, 140121697554431, +STORE, 140121697554432, 140121831636991, +STORE, 140121865175040, 140121890353151, +STORE, 140121856782336, 140121890353151, +STORE, 140121848389632, 140121890353151, +STORE, 140121839996928, 140121890353151, +STORE, 140121496092672, 140121697419263, +STORE, 140121487699968, 140121496092671, +STORE, 140121420591104, 140121487699967, +STORE, 140121412198400, 140121420591103, +ERASE, 140121420591104, 140121487699967, +STORE, 140121479307264, 140121496092671, +STORE, 140121277980672, 140121412198399, +SNULL, 140121277980672, 140121294766079, +STORE, 140121294766080, 140121412198399, +STORE, 140121277980672, 140121294766079, +ERASE, 140121277980672, 140121294766079, +STORE, 140121470914560, 140121496092671, +STORE, 140121462521856, 140121496092671, +STORE, 140121160548352, 140121412198399, +STORE, 140121454129152, 140121496092671, +SNULL, 140121227657215, 140121412198399, +STORE, 140121160548352, 140121227657215, +STORE, 140121227657216, 140121412198399, +SNULL, 140121227657216, 140121294766079, +STORE, 140121294766080, 140121412198399, +STORE, 140121227657216, 140121294766079, +ERASE, 140121227657216, 140121294766079, +STORE, 140121445736448, 140121496092671, +STORE, 140121437343744, 140121496092671, +SNULL, 140121437343744, 140121445736447, +STORE, 140121445736448, 140121496092671, +STORE, 140121437343744, 140121445736447, +SNULL, 140121445740543, 140121496092671, +STORE, 140121445736448, 140121445740543, +STORE, 140121445740544, 140121496092671, +SNULL, 140121697554432, 140121764528127, +STORE, 140121764528128, 140121831636991, +STORE, 140121697554432, 140121764528127, +SNULL, 140121764663295, 140121831636991, +STORE, 140121764528128, 140121764663295, +STORE, 140121764663296, 140121831636991, +SNULL, 140121496092672, 140121630310399, +STORE, 140121630310400, 140121697419263, +STORE, 140121496092672, 140121630310399, +SNULL, 140121630445567, 140121697419263, +STORE, 140121630310400, 140121630445567, +STORE, 140121630445568, 140121697419263, +SNULL, 140121445740544, 140121454129151, +STORE, 140121454129152, 140121496092671, +STORE, 140121445740544, 140121454129151, +SNULL, 140121454133247, 140121496092671, +STORE, 140121454129152, 140121454133247, +STORE, 140121454133248, 140121496092671, +STORE, 140121026330624, 140121227657215, +SNULL, 140121093439487, 140121227657215, +STORE, 140121026330624, 140121093439487, +STORE, 140121093439488, 140121227657215, +SNULL, 140121093439488, 140121160548351, +STORE, 140121160548352, 140121227657215, +STORE, 140121093439488, 140121160548351, +ERASE, 140121093439488, 140121160548351, +SNULL, 140121563201535, 140121630310399, +STORE, 140121496092672, 140121563201535, +STORE, 140121563201536, 140121630310399, +ERASE, 140121563201536, 140121630310399, +STORE, 140120892112896, 140121093439487, +SNULL, 140120959221759, 140121093439487, +STORE, 140120892112896, 140120959221759, +STORE, 140120959221760, 140121093439487, +SNULL, 140120959221760, 140121026330623, +STORE, 140121026330624, 140121093439487, +STORE, 140120959221760, 140121026330623, +ERASE, 140120959221760, 140121026330623, +STORE, 140120757895168, 140120959221759, +SNULL, 140121361874943, 140121412198399, +STORE, 140121294766080, 140121361874943, +STORE, 140121361874944, 140121412198399, +ERASE, 140121361874944, 140121412198399, +SNULL, 140121294901247, 140121361874943, +STORE, 140121294766080, 140121294901247, +STORE, 140121294901248, 140121361874943, +STORE, 140120623677440, 140120959221759, +SNULL, 140120690786303, 140120959221759, +STORE, 140120623677440, 140120690786303, +STORE, 140120690786304, 140120959221759, +SNULL, 140120690786304, 140120757895167, +STORE, 140120757895168, 140120959221759, +STORE, 140120690786304, 140120757895167, +ERASE, 140120690786304, 140120757895167, +SNULL, 140121160683519, 140121227657215, +STORE, 140121160548352, 140121160683519, +STORE, 140121160683520, 140121227657215, +SNULL, 140121974214656, 140121982607359, +STORE, 140121982607360, 140121991000063, +STORE, 140121974214656, 140121982607359, +SNULL, 140121982611455, 140121991000063, +STORE, 140121982607360, 140121982611455, +STORE, 140121982611456, 140121991000063, +SNULL, 140121839996928, 140121873567743, +STORE, 140121873567744, 140121890353151, +STORE, 140121839996928, 140121873567743, +SNULL, 140121873571839, 140121890353151, +STORE, 140121873567744, 140121873571839, +STORE, 140121873571840, 140121890353151, +SNULL, 140121873571840, 140121881960447, +STORE, 140121881960448, 140121890353151, +STORE, 140121873571840, 140121881960447, +SNULL, 140121881964543, 140121890353151, +STORE, 140121881960448, 140121881964543, +STORE, 140121881964544, 140121890353151, +SNULL, 140121840001023, 140121873567743, +STORE, 140121839996928, 140121840001023, +STORE, 140121840001024, 140121873567743, +SNULL, 140121840001024, 140121865175039, +STORE, 140121865175040, 140121873567743, +STORE, 140121840001024, 140121865175039, +SNULL, 140121865179135, 140121873567743, +STORE, 140121865175040, 140121865179135, +STORE, 140121865179136, 140121873567743, +SNULL, 140121437347839, 140121445736447, +STORE, 140121437343744, 140121437347839, +STORE, 140121437347840, 140121445736447, +STORE, 140121621917696, 140121630310399, +STORE, 140121613524992, 140121630310399, +SNULL, 140121026465791, 140121093439487, +STORE, 140121026330624, 140121026465791, +STORE, 140121026465792, 140121093439487, +SNULL, 140121496227839, 140121563201535, +STORE, 140121496092672, 140121496227839, +STORE, 140121496227840, 140121563201535, +SNULL, 140120757895168, 140120892112895, +STORE, 140120892112896, 140120959221759, +STORE, 140120757895168, 140120892112895, +SNULL, 140120892248063, 140120959221759, +STORE, 140120892112896, 140120892248063, +STORE, 140120892248064, 140120959221759, +SNULL, 140120825004031, 140120892112895, +STORE, 140120757895168, 140120825004031, +STORE, 140120825004032, 140120892112895, +ERASE, 140120825004032, 140120892112895, +SNULL, 140120623812607, 140120690786303, +STORE, 140120623677440, 140120623812607, +STORE, 140120623812608, 140120690786303, +SNULL, 140120758030335, 140120825004031, +STORE, 140120757895168, 140120758030335, +STORE, 140120758030336, 140120825004031, +SNULL, 140121454133248, 140121462521855, +STORE, 140121462521856, 140121496092671, +STORE, 140121454133248, 140121462521855, +SNULL, 140121462525951, 140121496092671, +STORE, 140121462521856, 140121462525951, +STORE, 140121462525952, 140121496092671, +STORE, 140121605132288, 140121630310399, +SNULL, 140121605136383, 140121630310399, +STORE, 140121605132288, 140121605136383, +STORE, 140121605136384, 140121630310399, +STORE, 140121596739584, 140121605132287, +SNULL, 140121605136384, 140121621917695, +STORE, 140121621917696, 140121630310399, +STORE, 140121605136384, 140121621917695, +SNULL, 140121621921791, 140121630310399, +STORE, 140121621917696, 140121621921791, +STORE, 140121621921792, 140121630310399, +STORE, 140121588346880, 140121605132287, +STORE, 140121579954176, 140121605132287, +SNULL, 140121412202495, 140121420591103, +STORE, 140121412198400, 140121412202495, +STORE, 140121412202496, 140121420591103, +SNULL, 140121974218751, 140121982607359, +STORE, 140121974214656, 140121974218751, +STORE, 140121974218752, 140121982607359, +SNULL, 140121462525952, 140121479307263, +STORE, 140121479307264, 140121496092671, +STORE, 140121462525952, 140121479307263, +SNULL, 140121479311359, 140121496092671, +STORE, 140121479307264, 140121479311359, +STORE, 140121479311360, 140121496092671, +STORE, 140121571561472, 140121605132287, +SNULL, 140121571565567, 140121605132287, +STORE, 140121571561472, 140121571565567, +STORE, 140121571565568, 140121605132287, +STORE, 140121428951040, 140121437343743, +SNULL, 140121428955135, 140121437343743, +STORE, 140121428951040, 140121428955135, +STORE, 140121428955136, 140121437343743, +SNULL, 140121840001024, 140121856782335, +STORE, 140121856782336, 140121865175039, +STORE, 140121840001024, 140121856782335, +SNULL, 140121856786431, 140121865175039, +STORE, 140121856782336, 140121856786431, +STORE, 140121856786432, 140121865175039, +STORE, 140121403805696, 140121412198399, +SNULL, 140121840001024, 140121848389631, +STORE, 140121848389632, 140121856782335, +STORE, 140121840001024, 140121848389631, +SNULL, 140121848393727, 140121856782335, +STORE, 140121848389632, 140121848393727, +STORE, 140121848393728, 140121856782335, +SNULL, 140121479311360, 140121487699967, +STORE, 140121487699968, 140121496092671, +STORE, 140121479311360, 140121487699967, +SNULL, 140121487704063, 140121496092671, +STORE, 140121487699968, 140121487704063, +STORE, 140121487704064, 140121496092671, +STORE, 140121395412992, 140121412198399, +STORE, 140121387020288, 140121412198399, +SNULL, 140121387024383, 140121412198399, +STORE, 140121387020288, 140121387024383, +STORE, 140121387024384, 140121412198399, +SNULL, 140121605136384, 140121613524991, +STORE, 140121613524992, 140121621917695, +STORE, 140121605136384, 140121613524991, +SNULL, 140121613529087, 140121621917695, +STORE, 140121613524992, 140121613529087, +STORE, 140121613529088, 140121621917695, +SNULL, 140121462525952, 140121470914559, +STORE, 140121470914560, 140121479307263, +STORE, 140121462525952, 140121470914559, +SNULL, 140121470918655, 140121479307263, +STORE, 140121470914560, 140121470918655, +STORE, 140121470918656, 140121479307263, +STORE, 140121378627584, 140121387020287, +SNULL, 140121378631679, 140121387020287, +STORE, 140121378627584, 140121378631679, +STORE, 140121378631680, 140121387020287, +SNULL, 140121571565568, 140121596739583, +STORE, 140121596739584, 140121605132287, +STORE, 140121571565568, 140121596739583, +SNULL, 140121596743679, 140121605132287, +STORE, 140121596739584, 140121596743679, +STORE, 140121596743680, 140121605132287, +SNULL, 140121387024384, 140121403805695, +STORE, 140121403805696, 140121412198399, +STORE, 140121387024384, 140121403805695, +SNULL, 140121403809791, 140121412198399, +STORE, 140121403805696, 140121403809791, +STORE, 140121403809792, 140121412198399, +STORE, 140121370234880, 140121378627583, +SNULL, 140121387024384, 140121395412991, +STORE, 140121395412992, 140121403805695, +STORE, 140121387024384, 140121395412991, +SNULL, 140121395417087, 140121403805695, +STORE, 140121395412992, 140121395417087, +STORE, 140121395417088, 140121403805695, +SNULL, 140121571565568, 140121588346879, +STORE, 140121588346880, 140121596739583, +STORE, 140121571565568, 140121588346879, +SNULL, 140121588350975, 140121596739583, +STORE, 140121588346880, 140121588350975, +STORE, 140121588350976, 140121596739583, +SNULL, 140121571565568, 140121579954175, +STORE, 140121579954176, 140121588346879, +STORE, 140121571565568, 140121579954175, +SNULL, 140121579958271, 140121588346879, +STORE, 140121579954176, 140121579958271, +STORE, 140121579958272, 140121588346879, +STORE, 140121286373376, 140121294766079, +STORE, 140121277980672, 140121294766079, +SNULL, 140121277980672, 140121286373375, +STORE, 140121286373376, 140121294766079, +STORE, 140121277980672, 140121286373375, +SNULL, 140121286377471, 140121294766079, +STORE, 140121286373376, 140121286377471, +STORE, 140121286377472, 140121294766079, +STORE, 140121269587968, 140121286373375, +STORE, 140121261195264, 140121286373375, +SNULL, 140121261195264, 140121269587967, +STORE, 140121269587968, 140121286373375, +STORE, 140121261195264, 140121269587967, +SNULL, 140121269592063, 140121286373375, +STORE, 140121269587968, 140121269592063, +STORE, 140121269592064, 140121286373375, +STORE, 140121252802560, 140121269587967, +SNULL, 140121252806655, 140121269587967, +STORE, 140121252802560, 140121252806655, +STORE, 140121252806656, 140121269587967, +STORE, 140121244409856, 140121252802559, +STORE, 140121236017152, 140121252802559, +SNULL, 140121236017152, 140121244409855, +STORE, 140121244409856, 140121252802559, +STORE, 140121236017152, 140121244409855, +SNULL, 140121244413951, 140121252802559, +STORE, 140121244409856, 140121244413951, +STORE, 140121244413952, 140121252802559, +SNULL, 140121370238975, 140121378627583, +STORE, 140121370234880, 140121370238975, +STORE, 140121370238976, 140121378627583, +STORE, 140121152155648, 140121160548351, +STORE, 140121143762944, 140121160548351, +STORE, 140121135370240, 140121160548351, +SNULL, 140121135374335, 140121160548351, +STORE, 140121135370240, 140121135374335, +STORE, 140121135374336, 140121160548351, +STORE, 140121126977536, 140121135370239, +STORE, 140121118584832, 140121135370239, +STORE, 140121110192128, 140121135370239, +SNULL, 140121110192128, 140121118584831, +STORE, 140121118584832, 140121135370239, +STORE, 140121110192128, 140121118584831, +SNULL, 140121118588927, 140121135370239, +STORE, 140121118584832, 140121118588927, +STORE, 140121118588928, 140121135370239, +STORE, 140121101799424, 140121118584831, +STORE, 140121017937920, 140121026330623, +STORE, 140121009545216, 140121026330623, +SNULL, 140121009545216, 140121017937919, +STORE, 140121017937920, 140121026330623, +STORE, 140121009545216, 140121017937919, +SNULL, 140121017942015, 140121026330623, +STORE, 140121017937920, 140121017942015, +STORE, 140121017942016, 140121026330623, +SNULL, 140121269592064, 140121277980671, +STORE, 140121277980672, 140121286373375, +STORE, 140121269592064, 140121277980671, +SNULL, 140121277984767, 140121286373375, +STORE, 140121277980672, 140121277984767, +STORE, 140121277984768, 140121286373375, +STORE, 140121001152512, 140121017937919, +SNULL, 140121252806656, 140121261195263, +STORE, 140121261195264, 140121269587967, +STORE, 140121252806656, 140121261195263, +SNULL, 140121261199359, 140121269587967, +STORE, 140121261195264, 140121261199359, +STORE, 140121261199360, 140121269587967, +SNULL, 140121135374336, 140121152155647, +STORE, 140121152155648, 140121160548351, +STORE, 140121135374336, 140121152155647, +SNULL, 140121152159743, 140121160548351, +STORE, 140121152155648, 140121152159743, +STORE, 140121152159744, 140121160548351, +STORE, 140120992759808, 140121017937919, +STORE, 140120984367104, 140121017937919, +STORE, 140120975974400, 140121017937919, +SNULL, 140121101799424, 140121110192127, +STORE, 140121110192128, 140121118584831, +STORE, 140121101799424, 140121110192127, +SNULL, 140121110196223, 140121118584831, +STORE, 140121110192128, 140121110196223, +STORE, 140121110196224, 140121118584831, +SNULL, 140121118588928, 140121126977535, +STORE, 140121126977536, 140121135370239, +STORE, 140121118588928, 140121126977535, +SNULL, 140121126981631, 140121135370239, +STORE, 140121126977536, 140121126981631, +STORE, 140121126981632, 140121135370239, +STORE, 140120967581696, 140121017937919, +STORE, 140120883720192, 140120892112895, +SNULL, 140120883724287, 140120892112895, +STORE, 140120883720192, 140120883724287, +STORE, 140120883724288, 140120892112895, +STORE, 140120875327488, 140120883720191, +SNULL, 140121101803519, 140121110192127, +STORE, 140121101799424, 140121101803519, +STORE, 140121101803520, 140121110192127, +SNULL, 140121135374336, 140121143762943, +STORE, 140121143762944, 140121152155647, +STORE, 140121135374336, 140121143762943, +SNULL, 140121143767039, 140121152155647, +STORE, 140121143762944, 140121143767039, +STORE, 140121143767040, 140121152155647, +STORE, 140120866934784, 140120883720191, +SNULL, 140120967581696, 140120984367103, +STORE, 140120984367104, 140121017937919, +STORE, 140120967581696, 140120984367103, +SNULL, 140120984371199, 140121017937919, +STORE, 140120984367104, 140120984371199, +STORE, 140120984371200, 140121017937919, +STORE, 140120858542080, 140120883720191, +SNULL, 140121236021247, 140121244409855, +STORE, 140121236017152, 140121236021247, +STORE, 140121236021248, 140121244409855, +SNULL, 140120984371200, 140121009545215, +STORE, 140121009545216, 140121017937919, +STORE, 140120984371200, 140121009545215, +SNULL, 140121009549311, 140121017937919, +STORE, 140121009545216, 140121009549311, +STORE, 140121009549312, 140121017937919, +SNULL, 140120984371200, 140120992759807, +STORE, 140120992759808, 140121009545215, +STORE, 140120984371200, 140120992759807, +SNULL, 140120992763903, 140121009545215, +STORE, 140120992759808, 140120992763903, +STORE, 140120992763904, 140121009545215, +SNULL, 140120992763904, 140121001152511, +STORE, 140121001152512, 140121009545215, +STORE, 140120992763904, 140121001152511, +SNULL, 140121001156607, 140121009545215, +STORE, 140121001152512, 140121001156607, +STORE, 140121001156608, 140121009545215, +STORE, 140120850149376, 140120883720191, +SNULL, 140120850153471, 140120883720191, +STORE, 140120850149376, 140120850153471, +STORE, 140120850153472, 140120883720191, +SNULL, 140120967585791, 140120984367103, +STORE, 140120967581696, 140120967585791, +STORE, 140120967585792, 140120984367103, +SNULL, 140120850153472, 140120866934783, +STORE, 140120866934784, 140120883720191, +STORE, 140120850153472, 140120866934783, +SNULL, 140120866938879, 140120883720191, +STORE, 140120866934784, 140120866938879, +STORE, 140120866938880, 140120883720191, +STORE, 140120841756672, 140120850149375, +SNULL, 140120967585792, 140120975974399, +STORE, 140120975974400, 140120984367103, +STORE, 140120967585792, 140120975974399, +SNULL, 140120975978495, 140120984367103, +STORE, 140120975974400, 140120975978495, +STORE, 140120975978496, 140120984367103, +SNULL, 140120866938880, 140120875327487, +STORE, 140120875327488, 140120883720191, +STORE, 140120866938880, 140120875327487, +SNULL, 140120875331583, 140120883720191, +STORE, 140120875327488, 140120875331583, +STORE, 140120875331584, 140120883720191, +STORE, 140120833363968, 140120850149375, +STORE, 140120749502464, 140120757895167, +STORE, 140120741109760, 140120757895167, +STORE, 140120732717056, 140120757895167, +STORE, 140120724324352, 140120757895167, +SNULL, 140120724324352, 140120732717055, +STORE, 140120732717056, 140120757895167, +STORE, 140120724324352, 140120732717055, +SNULL, 140120732721151, 140120757895167, +STORE, 140120732717056, 140120732721151, +STORE, 140120732721152, 140120757895167, +STORE, 140120715931648, 140120732717055, +SNULL, 140120715935743, 140120732717055, +STORE, 140120715931648, 140120715935743, +STORE, 140120715935744, 140120732717055, +SNULL, 140120850153472, 140120858542079, +STORE, 140120858542080, 140120866934783, +STORE, 140120850153472, 140120858542079, +SNULL, 140120858546175, 140120866934783, +STORE, 140120858542080, 140120858546175, +STORE, 140120858546176, 140120866934783, +STORE, 140120707538944, 140120715931647, +SNULL, 140120707543039, 140120715931647, +STORE, 140120707538944, 140120707543039, +STORE, 140120707543040, 140120715931647, +SNULL, 140120833368063, 140120850149375, +STORE, 140120833363968, 140120833368063, +STORE, 140120833368064, 140120850149375, +SNULL, 140120833368064, 140120841756671, +STORE, 140120841756672, 140120850149375, +STORE, 140120833368064, 140120841756671, +SNULL, 140120841760767, 140120850149375, +STORE, 140120841756672, 140120841760767, +STORE, 140120841760768, 140120850149375, +STORE, 140120699146240, 140120707538943, +SNULL, 140120715935744, 140120724324351, +STORE, 140120724324352, 140120732717055, +STORE, 140120715935744, 140120724324351, +SNULL, 140120724328447, 140120732717055, +STORE, 140120724324352, 140120724328447, +STORE, 140120724328448, 140120732717055, +SNULL, 140120732721152, 140120741109759, +STORE, 140120741109760, 140120757895167, +STORE, 140120732721152, 140120741109759, +SNULL, 140120741113855, 140120757895167, +STORE, 140120741109760, 140120741113855, +STORE, 140120741113856, 140120757895167, +SNULL, 140120741113856, 140120749502463, +STORE, 140120749502464, 140120757895167, +STORE, 140120741113856, 140120749502463, +SNULL, 140120749506559, 140120757895167, +STORE, 140120749502464, 140120749506559, +STORE, 140120749506560, 140120757895167, +SNULL, 140120699150335, 140120707538943, +STORE, 140120699146240, 140120699150335, +STORE, 140120699150336, 140120707538943, +STORE, 140122446557184, 140122446585855, +STORE, 140122368999424, 140122371190783, +SNULL, 140122368999424, 140122369089535, +STORE, 140122369089536, 140122371190783, +STORE, 140122368999424, 140122369089535, +SNULL, 140122371182591, 140122371190783, +STORE, 140122369089536, 140122371182591, +STORE, 140122371182592, 140122371190783, +ERASE, 140122371182592, 140122371190783, +STORE, 140122371182592, 140122371190783, +SNULL, 140122371186687, 140122371190783, +STORE, 140122371182592, 140122371186687, +STORE, 140122371186688, 140122371190783, +ERASE, 140122446557184, 140122446585855, +ERASE, 140121445736448, 140121445740543, +ERASE, 140121445740544, 140121454129151, +ERASE, 140121621917696, 140121621921791, +ERASE, 140121621921792, 140121630310399, +ERASE, 140121579954176, 140121579958271, +ERASE, 140121579958272, 140121588346879, +ERASE, 140121261195264, 140121261199359, +ERASE, 140121261199360, 140121269587967, +ERASE, 140121454129152, 140121454133247, +ERASE, 140121454133248, 140121462521855, +ERASE, 140121588346880, 140121588350975, +ERASE, 140121588350976, 140121596739583, +ERASE, 140121135370240, 140121135374335, +ERASE, 140121135374336, 140121143762943, +ERASE, 140121881960448, 140121881964543, +ERASE, 140121881964544, 140121890353151, +ERASE, 140121428951040, 140121428955135, +ERASE, 140121428955136, 140121437343743, +ERASE, 140121387020288, 140121387024383, +ERASE, 140121387024384, 140121395412991, +ERASE, 140121487699968, 140121487704063, +ERASE, 140121487704064, 140121496092671, +ERASE, 140121437343744, 140121437347839, +ERASE, 140121437347840, 140121445736447, +ERASE, 140121613524992, 140121613529087, +ERASE, 140121613529088, 140121621917695, +ERASE, 140121856782336, 140121856786431, +ERASE, 140121856786432, 140121865175039, +ERASE, 140121252802560, 140121252806655, +ERASE, 140121252806656, 140121261195263, +ERASE, 140121839996928, 140121840001023, +ERASE, 140121840001024, 140121848389631, +ERASE, 140121596739584, 140121596743679, +ERASE, 140121596743680, 140121605132287, +ERASE, 140121009545216, 140121009549311, +ERASE, 140121009549312, 140121017937919, +ERASE, 140120724324352, 140120724328447, +ERASE, 140120724328448, 140120732717055, +ERASE, 140120883720192, 140120883724287, +ERASE, 140120883724288, 140120892112895, +ERASE, 140121982607360, 140121982611455, +ERASE, 140121982611456, 140121991000063, +ERASE, 140121571561472, 140121571565567, +ERASE, 140121571565568, 140121579954175, +ERASE, 140121286373376, 140121286377471, +ERASE, 140121286377472, 140121294766079, +ERASE, 140120875327488, 140120875331583, +ERASE, 140120875331584, 140120883720191, +ERASE, 140121848389632, 140121848393727, +ERASE, 140121848393728, 140121856782335, +ERASE, 140121370234880, 140121370238975, +ERASE, 140121370238976, 140121378627583, +ERASE, 140121143762944, 140121143767039, +ERASE, 140121143767040, 140121152155647, +ERASE, 140121118584832, 140121118588927, +ERASE, 140121118588928, 140121126977535, +ERASE, 140120866934784, 140120866938879, +ERASE, 140120866938880, 140120875327487, +ERASE, 140120741109760, 140120741113855, +ERASE, 140120741113856, 140120749502463, +ERASE, 140121865175040, 140121865179135, +ERASE, 140121865179136, 140121873567743, +ERASE, 140121403805696, 140121403809791, +ERASE, 140121403809792, 140121412198399, +ERASE, 140121236017152, 140121236021247, +ERASE, 140121236021248, 140121244409855, +ERASE, 140120732717056, 140120732721151, +ERASE, 140120732721152, 140120741109759, +ERASE, 140121017937920, 140121017942015, +ERASE, 140121017942016, 140121026330623, +ERASE, 140121873567744, 140121873571839, +ERASE, 140121873571840, 140121881960447, +ERASE, 140121470914560, 140121470918655, +ERASE, 140121470918656, 140121479307263, +ERASE, 140121126977536, 140121126981631, +ERASE, 140121126981632, 140121135370239, +ERASE, 140120850149376, 140120850153471, +ERASE, 140120850153472, 140120858542079, +ERASE, 140120707538944, 140120707543039, +ERASE, 140120707543040, 140120715931647, +ERASE, 140121479307264, 140121479311359, +ERASE, 140121479311360, 140121487699967, +ERASE, 140120967581696, 140120967585791, +ERASE, 140120967585792, 140120975974399, +ERASE, 140120841756672, 140120841760767, +ERASE, 140120841760768, 140120850149375, +ERASE, 140121412198400, 140121412202495, +ERASE, 140121412202496, 140121420591103, +ERASE, 140122158788608, 140122158792703, +ERASE, 140122158792704, 140122167181311, +ERASE, 140122142003200, 140122142007295, +ERASE, 140122142007296, 140122150395903, +ERASE, 140121101799424, 140121101803519, +ERASE, 140121101803520, 140121110192127, +ERASE, 140120858542080, 140120858546175, +ERASE, 140120858546176, 140120866934783, +ERASE, 140120833363968, 140120833368063, +ERASE, 140120833368064, 140120841756671, +ERASE, 140121277980672, 140121277984767, +ERASE, 140121277984768, 140121286373375, +ERASE, 140121001152512, 140121001156607, +ERASE, 140121001156608, 140121009545215, +ERASE, 140120749502464, 140120749506559, +ERASE, 140120749506560, 140120757895167, +ERASE, 140121605132288, 140121605136383, +ERASE, 140121605136384, 140121613524991, +ERASE, 140121378627584, 140121378631679, +ERASE, 140121378631680, 140121387020287, +ERASE, 140121110192128, 140121110196223, +ERASE, 140121110196224, 140121118584831, +ERASE, 140121462521856, 140121462525951, +ERASE, 140121462525952, 140121470914559, +ERASE, 140121395412992, 140121395417087, +ERASE, 140121395417088, 140121403805695, +ERASE, 140121152155648, 140121152159743, +ERASE, 140121152159744, 140121160548351, +ERASE, 140120992759808, 140120992763903, +ERASE, 140120992763904, 140121001152511, +ERASE, 140122387976192, 140122387980287, +ERASE, 140122387980288, 140122396368895, +ERASE, 140121890353152, 140121890357247, +ERASE, 140121890357248, 140121898745855, +ERASE, 140121269587968, 140121269592063, +ERASE, 140121269592064, 140121277980671, + }; + unsigned long set37[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722404016128, 140737488351231, +SNULL, 140722404020223, 140737488351231, +STORE, 140722404016128, 140722404020223, +STORE, 140722403885056, 140722404020223, +STORE, 94637010001920, 94637012254719, +SNULL, 94637010132991, 94637012254719, +STORE, 94637010001920, 94637010132991, +STORE, 94637010132992, 94637012254719, +ERASE, 94637010132992, 94637012254719, +STORE, 94637012226048, 94637012234239, +STORE, 94637012234240, 94637012254719, +STORE, 139760240594944, 139760242847743, +SNULL, 139760240738303, 139760242847743, +STORE, 139760240594944, 139760240738303, +STORE, 139760240738304, 139760242847743, +ERASE, 139760240738304, 139760242847743, +STORE, 139760242835456, 139760242843647, +STORE, 139760242843648, 139760242847743, +STORE, 140722405232640, 140722405236735, +STORE, 140722405220352, 140722405232639, +STORE, 139760242806784, 139760242835455, +STORE, 139760242798592, 139760242806783, +STORE, 139760238379008, 139760240594943, +SNULL, 139760238379008, 139760238477311, +STORE, 139760238477312, 139760240594943, +STORE, 139760238379008, 139760238477311, +SNULL, 139760240570367, 139760240594943, +STORE, 139760238477312, 139760240570367, +STORE, 139760240570368, 139760240594943, +SNULL, 139760240570368, 139760240578559, +STORE, 139760240578560, 139760240594943, +STORE, 139760240570368, 139760240578559, +ERASE, 139760240570368, 139760240578559, +STORE, 139760240570368, 139760240578559, +ERASE, 139760240578560, 139760240594943, +STORE, 139760240578560, 139760240594943, +STORE, 139760234582016, 139760238379007, +SNULL, 139760234582016, 139760236240895, +STORE, 139760236240896, 139760238379007, +STORE, 139760234582016, 139760236240895, +SNULL, 139760238338047, 139760238379007, +STORE, 139760236240896, 139760238338047, +STORE, 139760238338048, 139760238379007, +SNULL, 139760238338048, 139760238362623, +STORE, 139760238362624, 139760238379007, +STORE, 139760238338048, 139760238362623, +ERASE, 139760238338048, 139760238362623, +STORE, 139760238338048, 139760238362623, +ERASE, 139760238362624, 139760238379007, +STORE, 139760238362624, 139760238379007, +STORE, 139760242790400, 139760242806783, +SNULL, 139760238354431, 139760238362623, +STORE, 139760238338048, 139760238354431, +STORE, 139760238354432, 139760238362623, +SNULL, 139760240574463, 139760240578559, +STORE, 139760240570368, 139760240574463, +STORE, 139760240574464, 139760240578559, +SNULL, 94637012230143, 94637012234239, +STORE, 94637012226048, 94637012230143, +STORE, 94637012230144, 94637012234239, +SNULL, 139760242839551, 139760242843647, +STORE, 139760242835456, 139760242839551, +STORE, 139760242839552, 139760242843647, +ERASE, 139760242806784, 139760242835455, +STORE, 94637033324544, 94637033459711, +STORE, 139760226189312, 139760234582015, +SNULL, 139760226193407, 139760234582015, +STORE, 139760226189312, 139760226193407, +STORE, 139760226193408, 139760234582015, +STORE, 139760217796608, 139760226189311, +STORE, 139760083578880, 139760217796607, +SNULL, 139760083578880, 139760114860031, +STORE, 139760114860032, 139760217796607, +STORE, 139760083578880, 139760114860031, +ERASE, 139760083578880, 139760114860031, +SNULL, 139760181968895, 139760217796607, +STORE, 139760114860032, 139760181968895, +STORE, 139760181968896, 139760217796607, +ERASE, 139760181968896, 139760217796607, +SNULL, 139760114995199, 139760181968895, +STORE, 139760114860032, 139760114995199, +STORE, 139760114995200, 139760181968895, +SNULL, 139760217800703, 139760226189311, +STORE, 139760217796608, 139760217800703, +STORE, 139760217800704, 139760226189311, +STORE, 139760209403904, 139760217796607, +SNULL, 139760209407999, 139760217796607, +STORE, 139760209403904, 139760209407999, +STORE, 139760209408000, 139760217796607, +STORE, 139760201011200, 139760209403903, +SNULL, 139760201015295, 139760209403903, +STORE, 139760201011200, 139760201015295, +STORE, 139760201015296, 139760209403903, +STORE, 139760192618496, 139760201011199, +SNULL, 139760192622591, 139760201011199, +STORE, 139760192618496, 139760192622591, +STORE, 139760192622592, 139760201011199, +STORE, 139760184225792, 139760192618495, +STORE, 139759980642304, 139760114860031, +STORE, 139759972249600, 139759980642303, +STORE, 139759963856896, 139759980642303, +STORE, 139759955464192, 139759980642303, +STORE, 139759888355328, 139759955464191, +SNULL, 139760047751167, 139760114860031, +STORE, 139759980642304, 139760047751167, +STORE, 139760047751168, 139760114860031, +ERASE, 139760047751168, 139760114860031, +SNULL, 139759980777471, 139760047751167, +STORE, 139759980642304, 139759980777471, +STORE, 139759980777472, 139760047751167, +STORE, 139759980777472, 139760114860031, +SNULL, 139759980777472, 139760047751167, +STORE, 139760047751168, 139760114860031, +STORE, 139759980777472, 139760047751167, +SNULL, 139760047886335, 139760114860031, +STORE, 139760047751168, 139760047886335, +STORE, 139760047886336, 139760114860031, +STORE, 139759821246464, 139759955464191, +SNULL, 139759821246464, 139759888355327, +STORE, 139759888355328, 139759955464191, +STORE, 139759821246464, 139759888355327, +ERASE, 139759821246464, 139759888355327, +ERASE, 139759888355328, 139759955464191, + }; + unsigned long set38[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730666221568, 140737488351231, +SNULL, 140730666225663, 140737488351231, +STORE, 140730666221568, 140730666225663, +STORE, 140730666090496, 140730666225663, +STORE, 94177584803840, 94177587056639, +SNULL, 94177584934911, 94177587056639, +STORE, 94177584803840, 94177584934911, +STORE, 94177584934912, 94177587056639, +ERASE, 94177584934912, 94177587056639, +STORE, 94177587027968, 94177587036159, +STORE, 94177587036160, 94177587056639, +STORE, 140614382714880, 140614384967679, +SNULL, 140614382858239, 140614384967679, +STORE, 140614382714880, 140614382858239, +STORE, 140614382858240, 140614384967679, +ERASE, 140614382858240, 140614384967679, +STORE, 140614384955392, 140614384963583, +STORE, 140614384963584, 140614384967679, +STORE, 140730666315776, 140730666319871, +STORE, 140730666303488, 140730666315775, +STORE, 140614384926720, 140614384955391, +STORE, 140614384918528, 140614384926719, +STORE, 140614380498944, 140614382714879, +SNULL, 140614380498944, 140614380597247, +STORE, 140614380597248, 140614382714879, +STORE, 140614380498944, 140614380597247, +SNULL, 140614382690303, 140614382714879, +STORE, 140614380597248, 140614382690303, +STORE, 140614382690304, 140614382714879, +SNULL, 140614382690304, 140614382698495, +STORE, 140614382698496, 140614382714879, +STORE, 140614382690304, 140614382698495, +ERASE, 140614382690304, 140614382698495, +STORE, 140614382690304, 140614382698495, +ERASE, 140614382698496, 140614382714879, +STORE, 140614382698496, 140614382714879, +STORE, 140614376701952, 140614380498943, +SNULL, 140614376701952, 140614378360831, +STORE, 140614378360832, 140614380498943, +STORE, 140614376701952, 140614378360831, +SNULL, 140614380457983, 140614380498943, +STORE, 140614378360832, 140614380457983, +STORE, 140614380457984, 140614380498943, +SNULL, 140614380457984, 140614380482559, +STORE, 140614380482560, 140614380498943, +STORE, 140614380457984, 140614380482559, +ERASE, 140614380457984, 140614380482559, +STORE, 140614380457984, 140614380482559, +ERASE, 140614380482560, 140614380498943, +STORE, 140614380482560, 140614380498943, +STORE, 140614384910336, 140614384926719, +SNULL, 140614380474367, 140614380482559, +STORE, 140614380457984, 140614380474367, +STORE, 140614380474368, 140614380482559, +SNULL, 140614382694399, 140614382698495, +STORE, 140614382690304, 140614382694399, +STORE, 140614382694400, 140614382698495, +SNULL, 94177587032063, 94177587036159, +STORE, 94177587027968, 94177587032063, +STORE, 94177587032064, 94177587036159, +SNULL, 140614384959487, 140614384963583, +STORE, 140614384955392, 140614384959487, +STORE, 140614384959488, 140614384963583, +ERASE, 140614384926720, 140614384955391, +STORE, 94177619791872, 94177619927039, +STORE, 140614368309248, 140614376701951, +SNULL, 140614368313343, 140614376701951, +STORE, 140614368309248, 140614368313343, +STORE, 140614368313344, 140614376701951, +STORE, 140614359916544, 140614368309247, +STORE, 140614225698816, 140614359916543, +SNULL, 140614225698816, 140614276481023, +STORE, 140614276481024, 140614359916543, +STORE, 140614225698816, 140614276481023, +ERASE, 140614225698816, 140614276481023, +SNULL, 140614343589887, 140614359916543, +STORE, 140614276481024, 140614343589887, +STORE, 140614343589888, 140614359916543, +ERASE, 140614343589888, 140614359916543, +SNULL, 140614276616191, 140614343589887, +STORE, 140614276481024, 140614276616191, +STORE, 140614276616192, 140614343589887, +SNULL, 140614359920639, 140614368309247, +STORE, 140614359916544, 140614359920639, +STORE, 140614359920640, 140614368309247, +STORE, 140614351523840, 140614359916543, +SNULL, 140614351527935, 140614359916543, +STORE, 140614351523840, 140614351527935, +STORE, 140614351527936, 140614359916543, +STORE, 140614268088320, 140614276481023, +SNULL, 140614268092415, 140614276481023, +STORE, 140614268088320, 140614268092415, +STORE, 140614268092416, 140614276481023, +STORE, 140614259695616, 140614268088319, +SNULL, 140614259699711, 140614268088319, +STORE, 140614259695616, 140614259699711, +STORE, 140614259699712, 140614268088319, +STORE, 140614251302912, 140614259695615, +STORE, 140614242910208, 140614259695615, +STORE, 140614108692480, 140614242910207, +SNULL, 140614108692480, 140614142263295, +STORE, 140614142263296, 140614242910207, +STORE, 140614108692480, 140614142263295, +ERASE, 140614108692480, 140614142263295, +STORE, 140614133870592, 140614142263295, +STORE, 140613999652864, 140614133870591, +SNULL, 140613999652864, 140614008045567, +STORE, 140614008045568, 140614133870591, +STORE, 140613999652864, 140614008045567, +ERASE, 140613999652864, 140614008045567, +STORE, 140613999652864, 140614008045567, +STORE, 140613865435136, 140613999652863, +SNULL, 140613865435136, 140613873827839, +STORE, 140613873827840, 140613999652863, +STORE, 140613865435136, 140613873827839, +ERASE, 140613865435136, 140613873827839, +SNULL, 140614209372159, 140614242910207, +STORE, 140614142263296, 140614209372159, +STORE, 140614209372160, 140614242910207, +ERASE, 140614209372160, 140614242910207, +SNULL, 140614142398463, 140614209372159, +STORE, 140614142263296, 140614142398463, +STORE, 140614142398464, 140614209372159, +SNULL, 140614075154431, 140614133870591, +STORE, 140614008045568, 140614075154431, +STORE, 140614075154432, 140614133870591, +ERASE, 140614075154432, 140614133870591, +SNULL, 140614008180735, 140614075154431, +STORE, 140614008045568, 140614008180735, +STORE, 140614008180736, 140614075154431, +SNULL, 140613940936703, 140613999652863, +STORE, 140613873827840, 140613940936703, +STORE, 140613940936704, 140613999652863, +ERASE, 140613940936704, 140613999652863, +SNULL, 140614242914303, 140614259695615, +STORE, 140614242910208, 140614242914303, +STORE, 140614242914304, 140614259695615, +STORE, 140613739610112, 140613940936703, +STORE, 140614234517504, 140614242910207, +SNULL, 140614242914304, 140614251302911, +STORE, 140614251302912, 140614259695615, +STORE, 140614242914304, 140614251302911, +SNULL, 140614251307007, 140614259695615, +STORE, 140614251302912, 140614251307007, +STORE, 140614251307008, 140614259695615, +SNULL, 140613739610112, 140613873827839, +STORE, 140613873827840, 140613940936703, +STORE, 140613739610112, 140613873827839, +SNULL, 140613873963007, 140613940936703, +STORE, 140613873827840, 140613873963007, +STORE, 140613873963008, 140613940936703, +SNULL, 140614133874687, 140614142263295, +STORE, 140614133870592, 140614133874687, +STORE, 140614133874688, 140614142263295, +SNULL, 140613806718975, 140613873827839, +STORE, 140613739610112, 140613806718975, +STORE, 140613806718976, 140613873827839, +ERASE, 140613806718976, 140613873827839, +STORE, 140614226124800, 140614242910207, +SNULL, 140613739745279, 140613806718975, +STORE, 140613739610112, 140613739745279, +STORE, 140613739745280, 140613806718975, +SNULL, 140613999656959, 140614008045567, +STORE, 140613999652864, 140613999656959, +STORE, 140613999656960, 140614008045567, +SNULL, 140614226124800, 140614234517503, +STORE, 140614234517504, 140614242910207, +STORE, 140614226124800, 140614234517503, +SNULL, 140614234521599, 140614242910207, +STORE, 140614234517504, 140614234521599, +STORE, 140614234521600, 140614242910207, +STORE, 140614217732096, 140614234517503, +STORE, 140614125477888, 140614133870591, +SNULL, 140614125481983, 140614133870591, +STORE, 140614125477888, 140614125481983, +STORE, 140614125481984, 140614133870591, +STORE, 140614117085184, 140614125477887, +SNULL, 140614217736191, 140614234517503, +STORE, 140614217732096, 140614217736191, +STORE, 140614217736192, 140614234517503, +SNULL, 140614117089279, 140614125477887, +STORE, 140614117085184, 140614117089279, +STORE, 140614117089280, 140614125477887, +SNULL, 140614217736192, 140614226124799, +STORE, 140614226124800, 140614234517503, +STORE, 140614217736192, 140614226124799, +SNULL, 140614226128895, 140614234517503, +STORE, 140614226124800, 140614226128895, +STORE, 140614226128896, 140614234517503, +STORE, 140614108692480, 140614117085183, +STORE, 140614100299776, 140614117085183, +STORE, 140614091907072, 140614117085183, +SNULL, 140614091907072, 140614108692479, +STORE, 140614108692480, 140614117085183, +STORE, 140614091907072, 140614108692479, +SNULL, 140614108696575, 140614117085183, +STORE, 140614108692480, 140614108696575, +STORE, 140614108696576, 140614117085183, +SNULL, 140614091907072, 140614100299775, +STORE, 140614100299776, 140614108692479, +STORE, 140614091907072, 140614100299775, +SNULL, 140614100303871, 140614108692479, +STORE, 140614100299776, 140614100303871, +STORE, 140614100303872, 140614108692479, +STORE, 140614083514368, 140614100299775, +SNULL, 140614083518463, 140614100299775, +STORE, 140614083514368, 140614083518463, +STORE, 140614083518464, 140614100299775, +STORE, 140613991260160, 140613999652863, +SNULL, 140614083518464, 140614091907071, +STORE, 140614091907072, 140614100299775, +STORE, 140614083518464, 140614091907071, +SNULL, 140614091911167, 140614100299775, +STORE, 140614091907072, 140614091911167, +STORE, 140614091911168, 140614100299775, +SNULL, 140613991264255, 140613999652863, +STORE, 140613991260160, 140613991264255, +STORE, 140613991264256, 140613999652863, +STORE, 140613982867456, 140613991260159, +SNULL, 140613982871551, 140613991260159, +STORE, 140613982867456, 140613982871551, +STORE, 140613982871552, 140613991260159, +STORE, 140613974474752, 140613982867455, +SNULL, 140613974478847, 140613982867455, +STORE, 140613974474752, 140613974478847, +STORE, 140613974478848, 140613982867455, +STORE, 140613966082048, 140613974474751, +STORE, 140613739745280, 140613873827839, +SNULL, 140613739745280, 140613806718975, +STORE, 140613806718976, 140613873827839, +STORE, 140613739745280, 140613806718975, +SNULL, 140613806854143, 140613873827839, +STORE, 140613806718976, 140613806854143, +STORE, 140613806854144, 140613873827839, +SNULL, 140613966086143, 140613974474751, +STORE, 140613966082048, 140613966086143, +STORE, 140613966086144, 140613974474751, +STORE, 140613957689344, 140613966082047, +STORE, 140613605392384, 140613739610111, +STORE, 140613949296640, 140613966082047, +STORE, 140613596999680, 140613605392383, +STORE, 140613529890816, 140613596999679, +STORE, 140613521498112, 140613529890815, +STORE, 140613513105408, 140613529890815, +STORE, 140613378887680, 140613513105407, +SNULL, 140613378887680, 140613404065791, +STORE, 140613404065792, 140613513105407, +STORE, 140613378887680, 140613404065791, +ERASE, 140613378887680, 140613404065791, +STORE, 140613395673088, 140613404065791, +STORE, 140613261455360, 140613395673087, +SNULL, 140613261455360, 140613269848063, +STORE, 140613269848064, 140613395673087, +STORE, 140613261455360, 140613269848063, +ERASE, 140613261455360, 140613269848063, +STORE, 140613261455360, 140613269848063, +STORE, 140613253062656, 140613269848063, +STORE, 140613118844928, 140613253062655, +STORE, 140613110452224, 140613118844927, +SNULL, 140613118844928, 140613135630335, +STORE, 140613135630336, 140613253062655, +STORE, 140613118844928, 140613135630335, +ERASE, 140613118844928, 140613135630335, +STORE, 140613127237632, 140613135630335, +STORE, 140613110452224, 140613135630335, +STORE, 140612976234496, 140613110452223, +STORE, 140612967841792, 140612976234495, +STORE, 140612833624064, 140612967841791, +STORE, 140612825231360, 140612833624063, +STORE, 140612816838656, 140612833624063, +STORE, 140612682620928, 140612816838655, +STORE, 140612674228224, 140612682620927, +SNULL, 140612682620928, 140612732977151, +STORE, 140612732977152, 140612816838655, +STORE, 140612682620928, 140612732977151, +ERASE, 140612682620928, 140612732977151, +SNULL, 140613672501247, 140613739610111, +STORE, 140613605392384, 140613672501247, +STORE, 140613672501248, 140613739610111, +ERASE, 140613672501248, 140613739610111, +SNULL, 140613605527551, 140613672501247, +STORE, 140613605392384, 140613605527551, +STORE, 140613605527552, 140613672501247, +ERASE, 140613529890816, 140613596999679, +STORE, 140612540010496, 140612674228223, +SNULL, 140612540010496, 140612598759423, +STORE, 140612598759424, 140612674228223, +STORE, 140612540010496, 140612598759423, +ERASE, 140612540010496, 140612598759423, +SNULL, 140613471174655, 140613513105407, +STORE, 140613404065792, 140613471174655, +STORE, 140613471174656, 140613513105407, +ERASE, 140613471174656, 140613513105407, +SNULL, 140613404200959, 140613471174655, +STORE, 140613404065792, 140613404200959, +STORE, 140613404200960, 140613471174655, +SNULL, 140613336956927, 140613395673087, +STORE, 140613269848064, 140613336956927, +STORE, 140613336956928, 140613395673087, +ERASE, 140613336956928, 140613395673087, +SNULL, 140612833624064, 140612867194879, +STORE, 140612867194880, 140612967841791, +STORE, 140612833624064, 140612867194879, +ERASE, 140612833624064, 140612867194879, +SNULL, 140612976234496, 140613001412607, +STORE, 140613001412608, 140613110452223, +STORE, 140612976234496, 140613001412607, +ERASE, 140612976234496, 140613001412607, +SNULL, 140613202739199, 140613253062655, +STORE, 140613135630336, 140613202739199, +STORE, 140613202739200, 140613253062655, +ERASE, 140613202739200, 140613253062655, +SNULL, 140613135765503, 140613202739199, +STORE, 140613135630336, 140613135765503, +STORE, 140613135765504, 140613202739199, +SNULL, 140612816842751, 140612833624063, +STORE, 140612816838656, 140612816842751, +STORE, 140612816842752, 140612833624063, +SNULL, 140613110456319, 140613135630335, +STORE, 140613110452224, 140613110456319, +STORE, 140613110456320, 140613135630335, +SNULL, 140613949300735, 140613966082047, +STORE, 140613949296640, 140613949300735, +STORE, 140613949300736, 140613966082047, +SNULL, 140613110456320, 140613118844927, +STORE, 140613118844928, 140613135630335, +STORE, 140613110456320, 140613118844927, +SNULL, 140613118849023, 140613135630335, +STORE, 140613118844928, 140613118849023, +STORE, 140613118849024, 140613135630335, +SNULL, 140612800086015, 140612816838655, +STORE, 140612732977152, 140612800086015, +STORE, 140612800086016, 140612816838655, +ERASE, 140612800086016, 140612816838655, +SNULL, 140613253062656, 140613261455359, +STORE, 140613261455360, 140613269848063, +STORE, 140613253062656, 140613261455359, +SNULL, 140613261459455, 140613269848063, +STORE, 140613261455360, 140613261459455, +STORE, 140613261459456, 140613269848063, +SNULL, 140612674232319, 140612682620927, +STORE, 140612674228224, 140612674232319, +STORE, 140612674232320, 140612682620927, +STORE, 140613731217408, 140613739610111, +STORE, 140613722824704, 140613739610111, +SNULL, 140613949300736, 140613957689343, +STORE, 140613957689344, 140613966082047, +STORE, 140613949300736, 140613957689343, +SNULL, 140613957693439, 140613966082047, +STORE, 140613957689344, 140613957693439, +STORE, 140613957693440, 140613966082047, +STORE, 140612464541696, 140612674228223, +SNULL, 140612531650559, 140612674228223, +STORE, 140612464541696, 140612531650559, +STORE, 140612531650560, 140612674228223, +SNULL, 140612531650560, 140612598759423, +STORE, 140612598759424, 140612674228223, +STORE, 140612531650560, 140612598759423, +ERASE, 140612531650560, 140612598759423, +SNULL, 140612665868287, 140612674228223, +STORE, 140612598759424, 140612665868287, +STORE, 140612665868288, 140612674228223, +ERASE, 140612665868288, 140612674228223, +SNULL, 140613269983231, 140613336956927, +STORE, 140613269848064, 140613269983231, +STORE, 140613269983232, 140613336956927, +SNULL, 140612934303743, 140612967841791, +STORE, 140612867194880, 140612934303743, +STORE, 140612934303744, 140612967841791, +ERASE, 140612934303744, 140612967841791, +SNULL, 140613068521471, 140613110452223, +STORE, 140613001412608, 140613068521471, +STORE, 140613068521472, 140613110452223, +ERASE, 140613068521472, 140613110452223, +STORE, 140613714432000, 140613739610111, +SNULL, 140613001547775, 140613068521471, +STORE, 140613001412608, 140613001547775, +STORE, 140613001547776, 140613068521471, +SNULL, 140612733112319, 140612800086015, +STORE, 140612732977152, 140612733112319, +STORE, 140612733112320, 140612800086015, +SNULL, 140613513109503, 140613529890815, +STORE, 140613513105408, 140613513109503, +STORE, 140613513109504, 140613529890815, +STORE, 140613706039296, 140613739610111, +STORE, 140613697646592, 140613739610111, +STORE, 140613689253888, 140613739610111, +SNULL, 140613689257983, 140613739610111, +STORE, 140613689253888, 140613689257983, +STORE, 140613689257984, 140613739610111, +SNULL, 140613253066751, 140613261455359, +STORE, 140613253062656, 140613253066751, +STORE, 140613253066752, 140613261455359, +STORE, 140613680861184, 140613689253887, +STORE, 140613588606976, 140613605392383, +SNULL, 140613689257984, 140613731217407, +STORE, 140613731217408, 140613739610111, +STORE, 140613689257984, 140613731217407, +SNULL, 140613731221503, 140613739610111, +STORE, 140613731217408, 140613731221503, +STORE, 140613731221504, 140613739610111, +STORE, 140613580214272, 140613605392383, +SNULL, 140612464676863, 140612531650559, +STORE, 140612464541696, 140612464676863, +STORE, 140612464676864, 140612531650559, +SNULL, 140612598894591, 140612665868287, +STORE, 140612598759424, 140612598894591, +STORE, 140612598894592, 140612665868287, +SNULL, 140612867330047, 140612934303743, +STORE, 140612867194880, 140612867330047, +STORE, 140612867330048, 140612934303743, +STORE, 140613571821568, 140613605392383, +SNULL, 140613571825663, 140613605392383, +STORE, 140613571821568, 140613571825663, +STORE, 140613571825664, 140613605392383, +SNULL, 140613689257984, 140613722824703, +STORE, 140613722824704, 140613731217407, +STORE, 140613689257984, 140613722824703, +SNULL, 140613722828799, 140613731217407, +STORE, 140613722824704, 140613722828799, +STORE, 140613722828800, 140613731217407, +SNULL, 140613689257984, 140613714431999, +STORE, 140613714432000, 140613722824703, +STORE, 140613689257984, 140613714431999, +SNULL, 140613714436095, 140613722824703, +STORE, 140613714432000, 140613714436095, +STORE, 140613714436096, 140613722824703, +SNULL, 140612816842752, 140612825231359, +STORE, 140612825231360, 140612833624063, +STORE, 140612816842752, 140612825231359, +SNULL, 140612825235455, 140612833624063, +STORE, 140612825231360, 140612825235455, +STORE, 140612825235456, 140612833624063, +SNULL, 140613395677183, 140613404065791, +STORE, 140613395673088, 140613395677183, +STORE, 140613395677184, 140613404065791, +SNULL, 140613689257984, 140613706039295, +STORE, 140613706039296, 140613714431999, +STORE, 140613689257984, 140613706039295, +SNULL, 140613706043391, 140613714431999, +STORE, 140613706039296, 140613706043391, +STORE, 140613706043392, 140613714431999, +SNULL, 140613118849024, 140613127237631, +STORE, 140613127237632, 140613135630335, +STORE, 140613118849024, 140613127237631, +SNULL, 140613127241727, 140613135630335, +STORE, 140613127237632, 140613127241727, +STORE, 140613127241728, 140613135630335, +SNULL, 140613571825664, 140613580214271, +STORE, 140613580214272, 140613605392383, +STORE, 140613571825664, 140613580214271, +SNULL, 140613580218367, 140613605392383, +STORE, 140613580214272, 140613580218367, +STORE, 140613580218368, 140613605392383, +SNULL, 140613689257984, 140613697646591, +STORE, 140613697646592, 140613706039295, +STORE, 140613689257984, 140613697646591, +SNULL, 140613697650687, 140613706039295, +STORE, 140613697646592, 140613697650687, +STORE, 140613697650688, 140613706039295, +SNULL, 140613680865279, 140613689253887, +STORE, 140613680861184, 140613680865279, +STORE, 140613680865280, 140613689253887, +STORE, 140613563428864, 140613571821567, +SNULL, 140613563432959, 140613571821567, +STORE, 140613563428864, 140613563432959, +STORE, 140613563432960, 140613571821567, +SNULL, 140613580218368, 140613588606975, +STORE, 140613588606976, 140613605392383, +STORE, 140613580218368, 140613588606975, +SNULL, 140613588611071, 140613605392383, +STORE, 140613588606976, 140613588611071, +STORE, 140613588611072, 140613605392383, +SNULL, 140613513109504, 140613521498111, +STORE, 140613521498112, 140613529890815, +STORE, 140613513109504, 140613521498111, +SNULL, 140613521502207, 140613529890815, +STORE, 140613521498112, 140613521502207, +STORE, 140613521502208, 140613529890815, +SNULL, 140613588611072, 140613596999679, +STORE, 140613596999680, 140613605392383, +STORE, 140613588611072, 140613596999679, +SNULL, 140613597003775, 140613605392383, +STORE, 140613596999680, 140613597003775, +STORE, 140613597003776, 140613605392383, +STORE, 140613555036160, 140613563428863, +SNULL, 140613555040255, 140613563428863, +STORE, 140613555036160, 140613555040255, +STORE, 140613555040256, 140613563428863, +STORE, 140613546643456, 140613555036159, +STORE, 140613538250752, 140613555036159, +SNULL, 140613538250752, 140613546643455, +STORE, 140613546643456, 140613555036159, +STORE, 140613538250752, 140613546643455, +SNULL, 140613546647551, 140613555036159, +STORE, 140613546643456, 140613546647551, +STORE, 140613546647552, 140613555036159, +STORE, 140613504712704, 140613513105407, +STORE, 140613496320000, 140613513105407, +SNULL, 140613496324095, 140613513105407, +STORE, 140613496320000, 140613496324095, +STORE, 140613496324096, 140613513105407, +STORE, 140613487927296, 140613496319999, +SNULL, 140613487931391, 140613496319999, +STORE, 140613487927296, 140613487931391, +STORE, 140613487931392, 140613496319999, +STORE, 140613479534592, 140613487927295, +SNULL, 140612967845887, 140612976234495, +STORE, 140612967841792, 140612967845887, +STORE, 140612967845888, 140612976234495, +STORE, 140613387280384, 140613395673087, +STORE, 140613378887680, 140613395673087, +SNULL, 140613378887680, 140613387280383, +STORE, 140613387280384, 140613395673087, +STORE, 140613378887680, 140613387280383, +SNULL, 140613387284479, 140613395673087, +STORE, 140613387280384, 140613387284479, +STORE, 140613387284480, 140613395673087, +STORE, 140613370494976, 140613387280383, +STORE, 140613362102272, 140613387280383, +SNULL, 140613479538687, 140613487927295, +STORE, 140613479534592, 140613479538687, +STORE, 140613479538688, 140613487927295, +STORE, 140613353709568, 140613387280383, +STORE, 140613345316864, 140613387280383, +STORE, 140613244669952, 140613253062655, +SNULL, 140613345320959, 140613387280383, +STORE, 140613345316864, 140613345320959, +STORE, 140613345320960, 140613387280383, +SNULL, 140613538254847, 140613546643455, +STORE, 140613538250752, 140613538254847, +STORE, 140613538254848, 140613546643455, +STORE, 140613236277248, 140613253062655, +STORE, 140613227884544, 140613253062655, +STORE, 140613219491840, 140613253062655, +STORE, 140613211099136, 140613253062655, +SNULL, 140613211103231, 140613253062655, +STORE, 140613211099136, 140613211103231, +STORE, 140613211103232, 140613253062655, +STORE, 140613102059520, 140613110452223, +STORE, 140613093666816, 140613110452223, +SNULL, 140613093670911, 140613110452223, +STORE, 140613093666816, 140613093670911, +STORE, 140613093670912, 140613110452223, +STORE, 140613085274112, 140613093666815, +SNULL, 140613496324096, 140613504712703, +STORE, 140613504712704, 140613513105407, +STORE, 140613496324096, 140613504712703, +SNULL, 140613504716799, 140613513105407, +STORE, 140613504712704, 140613504716799, +STORE, 140613504716800, 140613513105407, +SNULL, 140613345320960, 140613378887679, +STORE, 140613378887680, 140613387280383, +STORE, 140613345320960, 140613378887679, +SNULL, 140613378891775, 140613387280383, +STORE, 140613378887680, 140613378891775, +STORE, 140613378891776, 140613387280383, +SNULL, 140613345320960, 140613362102271, +STORE, 140613362102272, 140613378887679, +STORE, 140613345320960, 140613362102271, +SNULL, 140613362106367, 140613378887679, +STORE, 140613362102272, 140613362106367, +STORE, 140613362106368, 140613378887679, +SNULL, 140613362106368, 140613370494975, +STORE, 140613370494976, 140613378887679, +STORE, 140613362106368, 140613370494975, +SNULL, 140613370499071, 140613378887679, +STORE, 140613370494976, 140613370499071, +STORE, 140613370499072, 140613378887679, +STORE, 140613076881408, 140613093666815, +STORE, 140612993019904, 140613001412607, +SNULL, 140613076885503, 140613093666815, +STORE, 140613076881408, 140613076885503, +STORE, 140613076885504, 140613093666815, +SNULL, 140613093670912, 140613102059519, +STORE, 140613102059520, 140613110452223, +STORE, 140613093670912, 140613102059519, +SNULL, 140613102063615, 140613110452223, +STORE, 140613102059520, 140613102063615, +STORE, 140613102063616, 140613110452223, +SNULL, 140613076885504, 140613085274111, +STORE, 140613085274112, 140613093666815, +STORE, 140613076885504, 140613085274111, +SNULL, 140613085278207, 140613093666815, +STORE, 140613085274112, 140613085278207, +STORE, 140613085278208, 140613093666815, +STORE, 140612984627200, 140613001412607, +STORE, 140612967845888, 140612984627199, +SNULL, 140613211103232, 140613219491839, +STORE, 140613219491840, 140613253062655, +STORE, 140613211103232, 140613219491839, +SNULL, 140613219495935, 140613253062655, +STORE, 140613219491840, 140613219495935, +STORE, 140613219495936, 140613253062655, +STORE, 140612959449088, 140612967841791, +STORE, 140612951056384, 140612967841791, +SNULL, 140612951060479, 140612967841791, +STORE, 140612951056384, 140612951060479, +STORE, 140612951060480, 140612967841791, +SNULL, 140613345320960, 140613353709567, +STORE, 140613353709568, 140613362102271, +STORE, 140613345320960, 140613353709567, +SNULL, 140613353713663, 140613362102271, +STORE, 140613353709568, 140613353713663, +STORE, 140613353713664, 140613362102271, +SNULL, 140613219495936, 140613244669951, +STORE, 140613244669952, 140613253062655, +STORE, 140613219495936, 140613244669951, +SNULL, 140613244674047, 140613253062655, +STORE, 140613244669952, 140613244674047, +STORE, 140613244674048, 140613253062655, +STORE, 140612942663680, 140612951056383, +SNULL, 140613219495936, 140613236277247, +STORE, 140613236277248, 140613244669951, +STORE, 140613219495936, 140613236277247, +SNULL, 140613236281343, 140613244669951, +STORE, 140613236277248, 140613236281343, +STORE, 140613236281344, 140613244669951, +SNULL, 140613219495936, 140613227884543, +STORE, 140613227884544, 140613236277247, +STORE, 140613219495936, 140613227884543, +SNULL, 140613227888639, 140613236277247, +STORE, 140613227884544, 140613227888639, +STORE, 140613227888640, 140613236277247, +SNULL, 140612984627200, 140612993019903, +STORE, 140612993019904, 140613001412607, +STORE, 140612984627200, 140612993019903, +SNULL, 140612993023999, 140613001412607, +STORE, 140612993019904, 140612993023999, +STORE, 140612993024000, 140613001412607, +STORE, 140612858802176, 140612867194879, +STORE, 140612850409472, 140612867194879, +SNULL, 140612951060480, 140612959449087, +STORE, 140612959449088, 140612967841791, +STORE, 140612951060480, 140612959449087, +SNULL, 140612959453183, 140612967841791, +STORE, 140612959449088, 140612959453183, +STORE, 140612959453184, 140612967841791, +SNULL, 140612967845888, 140612976234495, +STORE, 140612976234496, 140612984627199, +STORE, 140612967845888, 140612976234495, +SNULL, 140612976238591, 140612984627199, +STORE, 140612976234496, 140612976238591, +STORE, 140612976238592, 140612984627199, +STORE, 140612842016768, 140612867194879, +SNULL, 140612842020863, 140612867194879, +STORE, 140612842016768, 140612842020863, +STORE, 140612842020864, 140612867194879, +SNULL, 140612984631295, 140612993019903, +STORE, 140612984627200, 140612984631295, +STORE, 140612984631296, 140612993019903, +STORE, 140612825235456, 140612842016767, +STORE, 140612808445952, 140612816838655, +SNULL, 140612942667775, 140612951056383, +STORE, 140612942663680, 140612942667775, +STORE, 140612942667776, 140612951056383, +STORE, 140612724584448, 140612732977151, +SNULL, 140612724588543, 140612732977151, +STORE, 140612724584448, 140612724588543, +STORE, 140612724588544, 140612732977151, +STORE, 140612716191744, 140612724584447, +SNULL, 140612842020864, 140612850409471, +STORE, 140612850409472, 140612867194879, +STORE, 140612842020864, 140612850409471, +SNULL, 140612850413567, 140612867194879, +STORE, 140612850409472, 140612850413567, +STORE, 140612850413568, 140612867194879, +SNULL, 140612850413568, 140612858802175, +STORE, 140612858802176, 140612867194879, +STORE, 140612850413568, 140612858802175, +SNULL, 140612858806271, 140612867194879, +STORE, 140612858802176, 140612858806271, +STORE, 140612858806272, 140612867194879, +STORE, 140612707799040, 140612724584447, +SNULL, 140612707803135, 140612724584447, +STORE, 140612707799040, 140612707803135, +STORE, 140612707803136, 140612724584447, +SNULL, 140612707803136, 140612716191743, +STORE, 140612716191744, 140612724584447, +STORE, 140612707803136, 140612716191743, +SNULL, 140612716195839, 140612724584447, +STORE, 140612716191744, 140612716195839, +STORE, 140612716195840, 140612724584447, +SNULL, 140612808450047, 140612816838655, +STORE, 140612808445952, 140612808450047, +STORE, 140612808450048, 140612816838655, +SNULL, 140612825235456, 140612833624063, +STORE, 140612833624064, 140612842016767, +STORE, 140612825235456, 140612833624063, +SNULL, 140612833628159, 140612842016767, +STORE, 140612833624064, 140612833628159, +STORE, 140612833628160, 140612842016767, +STORE, 140612699406336, 140612707799039, +SNULL, 140612699410431, 140612707799039, +STORE, 140612699406336, 140612699410431, +STORE, 140612699410432, 140612707799039, +STORE, 140614384926720, 140614384955391, +STORE, 140614349332480, 140614351523839, +SNULL, 140614349332480, 140614349422591, +STORE, 140614349422592, 140614351523839, +STORE, 140614349332480, 140614349422591, +SNULL, 140614351515647, 140614351523839, +STORE, 140614349422592, 140614351515647, +STORE, 140614351515648, 140614351523839, +ERASE, 140614351515648, 140614351523839, +STORE, 140614351515648, 140614351523839, +SNULL, 140614351519743, 140614351523839, +STORE, 140614351515648, 140614351519743, +STORE, 140614351519744, 140614351523839, +ERASE, 140614384926720, 140614384955391, +ERASE, 140613949296640, 140613949300735, +ERASE, 140613949300736, 140613957689343, +ERASE, 140613689253888, 140613689257983, +ERASE, 140613689257984, 140613697646591, +ERASE, 140613563428864, 140613563432959, +ERASE, 140613563432960, 140613571821567, +ERASE, 140613211099136, 140613211103231, +ERASE, 140613211103232, 140613219491839, +ERASE, 140614133870592, 140614133874687, +ERASE, 140614133874688, 140614142263295, +ERASE, 140612967841792, 140612967845887, +ERASE, 140612967845888, 140612976234495, +ERASE, 140613076881408, 140613076885503, +ERASE, 140613076885504, 140613085274111, +ERASE, 140612850409472, 140612850413567, +ERASE, 140612850413568, 140612858802175, +ERASE, 140613110452224, 140613110456319, +ERASE, 140613110456320, 140613118844927, +ERASE, 140613706039296, 140613706043391, +ERASE, 140613706043392, 140613714431999, +ERASE, 140613521498112, 140613521502207, +ERASE, 140613521502208, 140613529890815, +ERASE, 140613362102272, 140613362106367, +ERASE, 140613362106368, 140613370494975, +ERASE, 140613253062656, 140613253066751, +ERASE, 140613253066752, 140613261455359, +ERASE, 140612816838656, 140612816842751, +ERASE, 140612816842752, 140612825231359, +ERASE, 140613261455360, 140613261459455, +ERASE, 140613261459456, 140613269848063, +ERASE, 140613118844928, 140613118849023, +ERASE, 140613118849024, 140613127237631, +ERASE, 140613714432000, 140613714436095, +ERASE, 140613714436096, 140613722824703, +ERASE, 140613496320000, 140613496324095, +ERASE, 140613496324096, 140613504712703, +ERASE, 140613513105408, 140613513109503, +ERASE, 140613513109504, 140613521498111, +ERASE, 140613697646592, 140613697650687, +ERASE, 140613697650688, 140613706039295, +ERASE, 140613093666816, 140613093670911, +ERASE, 140613093670912, 140613102059519, +ERASE, 140612993019904, 140612993023999, +ERASE, 140612993024000, 140613001412607, +ERASE, 140613127237632, 140613127241727, +ERASE, 140613127241728, 140613135630335, +ERASE, 140613957689344, 140613957693439, +ERASE, 140613957693440, 140613966082047, +ERASE, 140613571821568, 140613571825663, +ERASE, 140613571825664, 140613580214271, +ERASE, 140613479534592, 140613479538687, +ERASE, 140613479538688, 140613487927295, +ERASE, 140612984627200, 140612984631295, +ERASE, 140612984631296, 140612993019903, +ERASE, 140613588606976, 140613588611071, +ERASE, 140613588611072, 140613596999679, +ERASE, 140613680861184, 140613680865279, +ERASE, 140613680865280, 140613689253887, +ERASE, 140613345316864, 140613345320959, +ERASE, 140613345320960, 140613353709567, +ERASE, 140613596999680, 140613597003775, +ERASE, 140613597003776, 140613605392383, +ERASE, 140613966082048, 140613966086143, +ERASE, 140613966086144, 140613974474751, +ERASE, 140613731217408, 140613731221503, +ERASE, 140613731221504, 140613739610111, +ERASE, 140613395673088, 140613395677183, +ERASE, 140613395677184, 140613404065791, +ERASE, 140612825231360, 140612825235455, +ERASE, 140612825235456, 140612833624063, +ERASE, 140612674228224, 140612674232319, +ERASE, 140612674232320, 140612682620927, +ERASE, 140613722824704, 140613722828799, +ERASE, 140613722828800, 140613731217407, +ERASE, 140613487927296, 140613487931391, +ERASE, 140613487931392, 140613496319999, +ERASE, 140613102059520, 140613102063615, +ERASE, 140613102063616, 140613110452223, +ERASE, 140614242910208, 140614242914303, +ERASE, 140614242914304, 140614251302911, +ERASE, 140612808445952, 140612808450047, +ERASE, 140612808450048, 140612816838655, +ERASE, 140613236277248, 140613236281343, +ERASE, 140613236281344, 140613244669951, +ERASE, 140613580214272, 140613580218367, +ERASE, 140613580218368, 140613588606975, +ERASE, 140613370494976, 140613370499071, +ERASE, 140613370499072, 140613378887679, +ERASE, 140613244669952, 140613244674047, +ERASE, 140613244674048, 140613253062655, +ERASE, 140612724584448, 140612724588543, +ERASE, 140612724588544, 140612732977151, +ERASE, 140612707799040, 140612707803135, +ERASE, 140612707803136, 140612716191743, +ERASE, 140613504712704, 140613504716799, +ERASE, 140613504716800, 140613513105407, + }; + + unsigned long set39[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736271417344, 140737488351231, +SNULL, 140736271421439, 140737488351231, +STORE, 140736271417344, 140736271421439, +STORE, 140736271286272, 140736271421439, +STORE, 94412930822144, 94412933074943, +SNULL, 94412930953215, 94412933074943, +STORE, 94412930822144, 94412930953215, +STORE, 94412930953216, 94412933074943, +ERASE, 94412930953216, 94412933074943, +STORE, 94412933046272, 94412933054463, +STORE, 94412933054464, 94412933074943, +STORE, 140326136901632, 140326139154431, +SNULL, 140326137044991, 140326139154431, +STORE, 140326136901632, 140326137044991, +STORE, 140326137044992, 140326139154431, +ERASE, 140326137044992, 140326139154431, +STORE, 140326139142144, 140326139150335, +STORE, 140326139150336, 140326139154431, +STORE, 140736271585280, 140736271589375, +STORE, 140736271572992, 140736271585279, +STORE, 140326139113472, 140326139142143, +STORE, 140326139105280, 140326139113471, +STORE, 140326134685696, 140326136901631, +SNULL, 140326134685696, 140326134783999, +STORE, 140326134784000, 140326136901631, +STORE, 140326134685696, 140326134783999, +SNULL, 140326136877055, 140326136901631, +STORE, 140326134784000, 140326136877055, +STORE, 140326136877056, 140326136901631, +SNULL, 140326136877056, 140326136885247, +STORE, 140326136885248, 140326136901631, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136877056, 140326136885247, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136885248, 140326136901631, +STORE, 140326136885248, 140326136901631, +STORE, 140326130888704, 140326134685695, +SNULL, 140326130888704, 140326132547583, +STORE, 140326132547584, 140326134685695, +STORE, 140326130888704, 140326132547583, +SNULL, 140326134644735, 140326134685695, +STORE, 140326132547584, 140326134644735, +STORE, 140326134644736, 140326134685695, +SNULL, 140326134644736, 140326134669311, +STORE, 140326134669312, 140326134685695, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134644736, 140326134669311, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134669312, 140326134685695, +STORE, 140326134669312, 140326134685695, +STORE, 140326139097088, 140326139113471, +SNULL, 140326134661119, 140326134669311, +STORE, 140326134644736, 140326134661119, +STORE, 140326134661120, 140326134669311, +SNULL, 140326136881151, 140326136885247, +STORE, 140326136877056, 140326136881151, +STORE, 140326136881152, 140326136885247, +SNULL, 94412933050367, 94412933054463, +STORE, 94412933046272, 94412933050367, +STORE, 94412933050368, 94412933054463, +SNULL, 140326139146239, 140326139150335, +STORE, 140326139142144, 140326139146239, +STORE, 140326139146240, 140326139150335, +ERASE, 140326139113472, 140326139142143, +STORE, 94412939493376, 94412939628543, +STORE, 140326122496000, 140326130888703, +SNULL, 140326122500095, 140326130888703, +STORE, 140326122496000, 140326122500095, +STORE, 140326122500096, 140326130888703, +STORE, 140326114103296, 140326122495999, +STORE, 140325979885568, 140326114103295, +SNULL, 140325979885568, 140326043910143, +STORE, 140326043910144, 140326114103295, +STORE, 140325979885568, 140326043910143, +ERASE, 140325979885568, 140326043910143, +SNULL, 140326111019007, 140326114103295, +STORE, 140326043910144, 140326111019007, +STORE, 140326111019008, 140326114103295, +ERASE, 140326111019008, 140326114103295, +SNULL, 140326044045311, 140326111019007, +STORE, 140326043910144, 140326044045311, +STORE, 140326044045312, 140326111019007, +SNULL, 140326114107391, 140326122495999, +STORE, 140326114103296, 140326114107391, +STORE, 140326114107392, 140326122495999, +STORE, 140326035517440, 140326043910143, +SNULL, 140326035521535, 140326043910143, +STORE, 140326035517440, 140326035521535, +STORE, 140326035521536, 140326043910143, +STORE, 140326027124736, 140326035517439, +SNULL, 140326027128831, 140326035517439, +STORE, 140326027124736, 140326027128831, +STORE, 140326027128832, 140326035517439, +STORE, 140326018732032, 140326027124735, +SNULL, 140326018736127, 140326027124735, +STORE, 140326018732032, 140326018736127, +STORE, 140326018736128, 140326027124735, +STORE, 140326010339328, 140326018732031, +STORE, 140326001946624, 140326018732031, +STORE, 140325993553920, 140326018732031, +STORE, 140325859336192, 140325993553919, +SNULL, 140325859336192, 140325909692415, +STORE, 140325909692416, 140325993553919, +STORE, 140325859336192, 140325909692415, +ERASE, 140325859336192, 140325909692415, +SNULL, 140325976801279, 140325993553919, +STORE, 140325909692416, 140325976801279, +STORE, 140325976801280, 140325993553919, +ERASE, 140325976801280, 140325993553919, +STORE, 140325985161216, 140326018732031, +STORE, 140325775474688, 140325976801279, +STORE, 140325708365824, 140325976801279, +SNULL, 140325708500991, 140325976801279, +STORE, 140325708365824, 140325708500991, +STORE, 140325708500992, 140325976801279, +SNULL, 140325708500992, 140325909692415, +STORE, 140325909692416, 140325976801279, +STORE, 140325708500992, 140325909692415, +SNULL, 140325909827583, 140325976801279, +STORE, 140325909692416, 140325909827583, +STORE, 140325909827584, 140325976801279, +SNULL, 140325842583551, 140325909692415, +STORE, 140325708500992, 140325842583551, +STORE, 140325842583552, 140325909692415, +ERASE, 140325842583552, 140325909692415, +SNULL, 140325708500992, 140325775474687, +STORE, 140325775474688, 140325842583551, +STORE, 140325708500992, 140325775474687, +SNULL, 140325775609855, 140325842583551, +STORE, 140325775474688, 140325775609855, +STORE, 140325775609856, 140325842583551, +STORE, 140325775609856, 140325909692415, +SNULL, 140325775609856, 140325842583551, +STORE, 140325842583552, 140325909692415, +STORE, 140325775609856, 140325842583551, +SNULL, 140325842718719, 140325909692415, +STORE, 140325842583552, 140325842718719, +STORE, 140325842718720, 140325909692415, +SNULL, 140325985161216, 140325993553919, +STORE, 140325993553920, 140326018732031, +STORE, 140325985161216, 140325993553919, +SNULL, 140325993558015, 140326018732031, +STORE, 140325993553920, 140325993558015, +STORE, 140325993558016, 140326018732031, +SNULL, 140325985165311, 140325993553919, +STORE, 140325985161216, 140325985165311, +STORE, 140325985165312, 140325993553919, +SNULL, 140325993558016, 140326001946623, +STORE, 140326001946624, 140326018732031, +STORE, 140325993558016, 140326001946623, +SNULL, 140326001950719, 140326018732031, +STORE, 140326001946624, 140326001950719, +STORE, 140326001950720, 140326018732031, +SNULL, 140326001950720, 140326010339327, +STORE, 140326010339328, 140326018732031, +STORE, 140326001950720, 140326010339327, +SNULL, 140326010343423, 140326018732031, +STORE, 140326010339328, 140326010343423, +STORE, 140326010343424, 140326018732031, +STORE, 140325699973120, 140325708365823, +STORE, 140325691580416, 140325708365823, +STORE, 140325683187712, 140325708365823, +SNULL, 140325683191807, 140325708365823, +STORE, 140325683187712, 140325683191807, +STORE, 140325683191808, 140325708365823, +SNULL, 140325683191808, 140325699973119, +STORE, 140325699973120, 140325708365823, +STORE, 140325683191808, 140325699973119, +SNULL, 140325699977215, 140325708365823, +STORE, 140325699973120, 140325699977215, +STORE, 140325699977216, 140325708365823, +STORE, 140325674795008, 140325683187711, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325683187711, +SNULL, 140325658009600, 140325666402303, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325666402303, +SNULL, 140325666406399, 140325683187711, +STORE, 140325666402304, 140325666406399, +STORE, 140325666406400, 140325683187711, +SNULL, 140325683191808, 140325691580415, +STORE, 140325691580416, 140325699973119, +STORE, 140325683191808, 140325691580415, +SNULL, 140325691584511, 140325699973119, +STORE, 140325691580416, 140325691584511, +STORE, 140325691584512, 140325699973119, +SNULL, 140325666406400, 140325674795007, +STORE, 140325674795008, 140325683187711, +STORE, 140325666406400, 140325674795007, +SNULL, 140325674799103, 140325683187711, +STORE, 140325674795008, 140325674799103, +STORE, 140325674799104, 140325683187711, +STORE, 140325649616896, 140325666402303, +SNULL, 140325649616896, 140325658009599, +STORE, 140325658009600, 140325666402303, +STORE, 140325649616896, 140325658009599, +SNULL, 140325658013695, 140325666402303, +STORE, 140325658009600, 140325658013695, +STORE, 140325658013696, 140325666402303, +SNULL, 140325649620991, 140325658009599, +STORE, 140325649616896, 140325649620991, +STORE, 140325649620992, 140325658009599, +STORE, 140325641224192, 140325649616895, +STORE, 140325632831488, 140325649616895, +SNULL, 140325632835583, 140325649616895, +STORE, 140325632831488, 140325632835583, +STORE, 140325632835584, 140325649616895, +STORE, 140325624438784, 140325632831487, +SNULL, 140325624442879, 140325632831487, +STORE, 140325624438784, 140325624442879, +STORE, 140325624442880, 140325632831487, +SNULL, 140325632835584, 140325641224191, +STORE, 140325641224192, 140325649616895, +STORE, 140325632835584, 140325641224191, +SNULL, 140325641228287, 140325649616895, +STORE, 140325641224192, 140325641228287, +STORE, 140325641228288, 140325649616895, +STORE, 140325616046080, 140325624438783, +SNULL, 140325616050175, 140325624438783, +STORE, 140325616046080, 140325616050175, +STORE, 140325616050176, 140325624438783, +STORE, 140325607653376, 140325616046079, +SNULL, 140325607657471, 140325616046079, +STORE, 140325607653376, 140325607657471, +STORE, 140325607657472, 140325616046079, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325607653375, +STORE, 140325456650240, 140325590867967, +SNULL, 140325456650240, 140325507039231, +STORE, 140325507039232, 140325590867967, +STORE, 140325456650240, 140325507039231, +ERASE, 140325456650240, 140325507039231, +STORE, 140325498646528, 140325507039231, +STORE, 140325364428800, 140325498646527, +SNULL, 140325364428800, 140325372821503, +STORE, 140325372821504, 140325498646527, +STORE, 140325364428800, 140325372821503, +ERASE, 140325364428800, 140325372821503, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325372821503, +STORE, 140325221818368, 140325356036095, +SNULL, 140325221818368, 140325238603775, +STORE, 140325238603776, 140325356036095, +STORE, 140325221818368, 140325238603775, +ERASE, 140325221818368, 140325238603775, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325238603775, +STORE, 140325087600640, 140325221818367, +STORE, 140325079207936, 140325087600639, +SNULL, 140325087600640, 140325104386047, +STORE, 140325104386048, 140325221818367, +STORE, 140325087600640, 140325104386047, +ERASE, 140325087600640, 140325104386047, +STORE, 140325095993344, 140325104386047, +STORE, 140325079207936, 140325104386047, +STORE, 140324944990208, 140325079207935, +SNULL, 140324944990208, 140324970168319, +STORE, 140324970168320, 140325079207935, +STORE, 140324944990208, 140324970168319, +ERASE, 140324944990208, 140324970168319, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324970168319, +STORE, 140324819165184, 140324953382911, +STORE, 140324684947456, 140324953382911, +STORE, 140324676554752, 140324684947455, +STORE, 140324668162048, 140324684947455, +STORE, 140324533944320, 140324668162047, +STORE, 140324525551616, 140324533944319, +SNULL, 140324533944320, 140324567515135, +STORE, 140324567515136, 140324668162047, +STORE, 140324533944320, 140324567515135, +ERASE, 140324533944320, 140324567515135, +STORE, 140324559122432, 140324567515135, +STORE, 140324391333888, 140324525551615, +SNULL, 140325574148095, 140325590867967, +STORE, 140325507039232, 140325574148095, +STORE, 140325574148096, 140325590867967, +ERASE, 140325574148096, 140325590867967, +SNULL, 140325439930367, 140325498646527, +STORE, 140325372821504, 140325439930367, +STORE, 140325439930368, 140325498646527, +ERASE, 140325439930368, 140325498646527, +SNULL, 140325305712639, 140325356036095, +STORE, 140325238603776, 140325305712639, +STORE, 140325305712640, 140325356036095, +ERASE, 140325305712640, 140325356036095, +SNULL, 140325171494911, 140325221818367, +STORE, 140325104386048, 140325171494911, +STORE, 140325171494912, 140325221818367, +ERASE, 140325171494912, 140325221818367, +SNULL, 140325104521215, 140325171494911, +STORE, 140325104386048, 140325104521215, +STORE, 140325104521216, 140325171494911, +STORE, 140324257116160, 140324525551615, +SNULL, 140324257116160, 140324299079679, +STORE, 140324299079680, 140324525551615, +STORE, 140324257116160, 140324299079679, +ERASE, 140324257116160, 140324299079679, +SNULL, 140325037277183, 140325079207935, +STORE, 140324970168320, 140325037277183, +STORE, 140325037277184, 140325079207935, +ERASE, 140325037277184, 140325079207935, +SNULL, 140324819165183, 140324953382911, +STORE, 140324684947456, 140324819165183, +STORE, 140324819165184, 140324953382911, +SNULL, 140324819165184, 140324835950591, +STORE, 140324835950592, 140324953382911, +STORE, 140324819165184, 140324835950591, +ERASE, 140324819165184, 140324835950591, +SNULL, 140324903059455, 140324953382911, +STORE, 140324835950592, 140324903059455, +STORE, 140324903059456, 140324953382911, +ERASE, 140324903059456, 140324953382911, +SNULL, 140324684947456, 140324701732863, +STORE, 140324701732864, 140324819165183, +STORE, 140324684947456, 140324701732863, +ERASE, 140324684947456, 140324701732863, +SNULL, 140324768841727, 140324819165183, +STORE, 140324701732864, 140324768841727, +STORE, 140324768841728, 140324819165183, +ERASE, 140324768841728, 140324819165183, +SNULL, 140324634623999, 140324668162047, +STORE, 140324567515136, 140324634623999, +STORE, 140324634624000, 140324668162047, +ERASE, 140324634624000, 140324668162047, +SNULL, 140324391333887, 140324525551615, +STORE, 140324299079680, 140324391333887, +STORE, 140324391333888, 140324525551615, +SNULL, 140324391333888, 140324433297407, +STORE, 140324433297408, 140324525551615, +STORE, 140324391333888, 140324433297407, +ERASE, 140324391333888, 140324433297407, +SNULL, 140325507174399, 140325574148095, +STORE, 140325507039232, 140325507174399, +STORE, 140325507174400, 140325574148095, +SNULL, 140325590867968, 140325599260671, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325599260671, +SNULL, 140325599264767, 140325607653375, +STORE, 140325599260672, 140325599264767, +STORE, 140325599264768, 140325607653375, +SNULL, 140325372956671, 140325439930367, +STORE, 140325372821504, 140325372956671, +STORE, 140325372956672, 140325439930367, +SNULL, 140324668166143, 140324684947455, +STORE, 140324668162048, 140324668166143, +STORE, 140324668166144, 140324684947455, +SNULL, 140324525555711, 140324533944319, +STORE, 140324525551616, 140324525555711, +STORE, 140324525555712, 140324533944319, +SNULL, 140324953382912, 140324961775615, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324961775615, +SNULL, 140324961779711, 140324970168319, +STORE, 140324961775616, 140324961779711, +STORE, 140324961779712, 140324970168319, +SNULL, 140325079212031, 140325104386047, +STORE, 140325079207936, 140325079212031, +STORE, 140325079212032, 140325104386047, +SNULL, 140325221818368, 140325230211071, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325230211071, +SNULL, 140325230215167, 140325238603775, +STORE, 140325230211072, 140325230215167, +STORE, 140325230215168, 140325238603775, +SNULL, 140325356036096, 140325364428799, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325364428799, +SNULL, 140325364432895, 140325372821503, + }; + unsigned long set40[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734309167104, 140737488351231, +SNULL, 140734309171199, 140737488351231, +STORE, 140734309167104, 140734309171199, +STORE, 140734309036032, 140734309171199, +STORE, 94270500081664, 94270502334463, +SNULL, 94270500212735, 94270502334463, +STORE, 94270500081664, 94270500212735, +STORE, 94270500212736, 94270502334463, +ERASE, 94270500212736, 94270502334463, +STORE, 94270502305792, 94270502313983, +STORE, 94270502313984, 94270502334463, +STORE, 140321935110144, 140321937362943, +SNULL, 140321935253503, 140321937362943, +STORE, 140321935110144, 140321935253503, +STORE, 140321935253504, 140321937362943, +ERASE, 140321935253504, 140321937362943, +STORE, 140321937350656, 140321937358847, +STORE, 140321937358848, 140321937362943, +STORE, 140734309625856, 140734309629951, +STORE, 140734309613568, 140734309625855, +STORE, 140321937321984, 140321937350655, +STORE, 140321937313792, 140321937321983, +STORE, 140321932894208, 140321935110143, +SNULL, 140321932894208, 140321932992511, +STORE, 140321932992512, 140321935110143, +STORE, 140321932894208, 140321932992511, +SNULL, 140321935085567, 140321935110143, +STORE, 140321932992512, 140321935085567, +STORE, 140321935085568, 140321935110143, +SNULL, 140321935085568, 140321935093759, +STORE, 140321935093760, 140321935110143, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935085568, 140321935093759, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935093760, 140321935110143, +STORE, 140321935093760, 140321935110143, +STORE, 140321929097216, 140321932894207, +SNULL, 140321929097216, 140321930756095, +STORE, 140321930756096, 140321932894207, +STORE, 140321929097216, 140321930756095, +SNULL, 140321932853247, 140321932894207, +STORE, 140321930756096, 140321932853247, +STORE, 140321932853248, 140321932894207, +SNULL, 140321932853248, 140321932877823, +STORE, 140321932877824, 140321932894207, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932853248, 140321932877823, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932877824, 140321932894207, +STORE, 140321932877824, 140321932894207, +STORE, 140321937305600, 140321937321983, +SNULL, 140321932869631, 140321932877823, +STORE, 140321932853248, 140321932869631, +STORE, 140321932869632, 140321932877823, +SNULL, 140321935089663, 140321935093759, +STORE, 140321935085568, 140321935089663, +STORE, 140321935089664, 140321935093759, +SNULL, 94270502309887, 94270502313983, +STORE, 94270502305792, 94270502309887, +STORE, 94270502309888, 94270502313983, +SNULL, 140321937354751, 140321937358847, +STORE, 140321937350656, 140321937354751, +STORE, 140321937354752, 140321937358847, +ERASE, 140321937321984, 140321937350655, +STORE, 94270507364352, 94270507499519, +STORE, 140321920704512, 140321929097215, +SNULL, 140321920708607, 140321929097215, +STORE, 140321920704512, 140321920708607, +STORE, 140321920708608, 140321929097215, +STORE, 140321912311808, 140321920704511, +STORE, 140321778094080, 140321912311807, +SNULL, 140321778094080, 140321816051711, +STORE, 140321816051712, 140321912311807, +STORE, 140321778094080, 140321816051711, +ERASE, 140321778094080, 140321816051711, +SNULL, 140321883160575, 140321912311807, +STORE, 140321816051712, 140321883160575, +STORE, 140321883160576, 140321912311807, +ERASE, 140321883160576, 140321912311807, +SNULL, 140321816186879, 140321883160575, +STORE, 140321816051712, 140321816186879, +STORE, 140321816186880, 140321883160575, +SNULL, 140321912315903, 140321920704511, +STORE, 140321912311808, 140321912315903, +STORE, 140321912315904, 140321920704511, +STORE, 140321903919104, 140321912311807, +SNULL, 140321903923199, 140321912311807, +STORE, 140321903919104, 140321903923199, +STORE, 140321903923200, 140321912311807, +STORE, 140321895526400, 140321903919103, +SNULL, 140321895530495, 140321903919103, +STORE, 140321895526400, 140321895530495, +STORE, 140321895530496, 140321903919103, +STORE, 140321887133696, 140321895526399, +SNULL, 140321887137791, 140321895526399, +STORE, 140321887133696, 140321887137791, +STORE, 140321887137792, 140321895526399, +STORE, 140321807659008, 140321816051711, +STORE, 140321673441280, 140321807659007, +SNULL, 140321673441280, 140321681833983, +STORE, 140321681833984, 140321807659007, +STORE, 140321673441280, 140321681833983, +ERASE, 140321673441280, 140321681833983, +SNULL, 140321748942847, 140321807659007, +STORE, 140321681833984, 140321748942847, +STORE, 140321748942848, 140321807659007, +ERASE, 140321748942848, 140321807659007, +STORE, 140321799266304, 140321816051711, +STORE, 140321790873600, 140321816051711, +STORE, 140321782480896, 140321816051711, +STORE, 140321547616256, 140321748942847, +SNULL, 140321614725119, 140321748942847, +STORE, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321748942847, +SNULL, 140321614725120, 140321681833983, +STORE, 140321681833984, 140321748942847, +STORE, 140321614725120, 140321681833983, +ERASE, 140321614725120, 140321681833983, +SNULL, 140321681969151, 140321748942847, +STORE, 140321681833984, 140321681969151, +STORE, 140321681969152, 140321748942847, +STORE, 140321547616256, 140321681833983, +SNULL, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321681833983, +STORE, 140321547616256, 140321614725119, +SNULL, 140321614860287, 140321681833983, +STORE, 140321614725120, 140321614860287, +STORE, 140321614860288, 140321681833983, +SNULL, 140321547751423, 140321614725119, +STORE, 140321547616256, 140321547751423, +STORE, 140321547751424, 140321614725119, +STORE, 140321480507392, 140321547616255, +SNULL, 140321782480896, 140321799266303, +STORE, 140321799266304, 140321816051711, +STORE, 140321782480896, 140321799266303, +SNULL, 140321799270399, 140321816051711, +STORE, 140321799266304, 140321799270399, +STORE, 140321799270400, 140321816051711, +STORE, 140321774088192, 140321799266303, +SNULL, 140321774088192, 140321790873599, +STORE, 140321790873600, 140321799266303, +STORE, 140321774088192, 140321790873599, +SNULL, 140321790877695, 140321799266303, +STORE, 140321790873600, 140321790877695, +STORE, 140321790877696, 140321799266303, +SNULL, 140321480642559, 140321547616255, +STORE, 140321480507392, 140321480642559, +STORE, 140321480642560, 140321547616255, +SNULL, 140321774088192, 140321782480895, +STORE, 140321782480896, 140321790873599, +STORE, 140321774088192, 140321782480895, +SNULL, 140321782484991, 140321790873599, +STORE, 140321782480896, 140321782484991, +STORE, 140321782484992, 140321790873599, +SNULL, 140321799270400, 140321807659007, +STORE, 140321807659008, 140321816051711, +STORE, 140321799270400, 140321807659007, +SNULL, 140321807663103, 140321816051711, +STORE, 140321807659008, 140321807663103, +STORE, 140321807663104, 140321816051711, +STORE, 140321765695488, 140321782480895, +STORE, 140321757302784, 140321782480895, +SNULL, 140321757306879, 140321782480895, +STORE, 140321757302784, 140321757306879, +STORE, 140321757306880, 140321782480895, +STORE, 140321472114688, 140321480507391, +STORE, 140321463721984, 140321480507391, +SNULL, 140321463726079, 140321480507391, +STORE, 140321463721984, 140321463726079, +STORE, 140321463726080, 140321480507391, +SNULL, 140321757306880, 140321774088191, +STORE, 140321774088192, 140321782480895, +STORE, 140321757306880, 140321774088191, +SNULL, 140321774092287, 140321782480895, +STORE, 140321774088192, 140321774092287, +STORE, 140321774092288, 140321782480895, +SNULL, 140321463726080, 140321472114687, +STORE, 140321472114688, 140321480507391, +STORE, 140321463726080, 140321472114687, +SNULL, 140321472118783, 140321480507391, +STORE, 140321472114688, 140321472118783, +STORE, 140321472118784, 140321480507391, +SNULL, 140321757306880, 140321765695487, +STORE, 140321765695488, 140321774088191, +STORE, 140321757306880, 140321765695487, +SNULL, 140321765699583, 140321774088191, +STORE, 140321765695488, 140321765699583, +STORE, 140321765699584, 140321774088191, +STORE, 140321455329280, 140321463721983, +SNULL, 140321455333375, 140321463721983, +STORE, 140321455329280, 140321455333375, +STORE, 140321455333376, 140321463721983, +STORE, 140321446936576, 140321455329279, +STORE, 140321438543872, 140321455329279, +STORE, 140321430151168, 140321455329279, +SNULL, 140321430155263, 140321455329279, +STORE, 140321430151168, 140321430155263, +STORE, 140321430155264, 140321455329279, +SNULL, 140321430155264, 140321446936575, +STORE, 140321446936576, 140321455329279, +STORE, 140321430155264, 140321446936575, +SNULL, 140321446940671, 140321455329279, +STORE, 140321446936576, 140321446940671, +STORE, 140321446940672, 140321455329279, +SNULL, 140321430155264, 140321438543871, +STORE, 140321438543872, 140321446936575, +STORE, 140321430155264, 140321438543871, +SNULL, 140321438547967, 140321446936575, +STORE, 140321438543872, 140321438547967, +STORE, 140321438547968, 140321446936575, +STORE, 140321421758464, 140321430151167, +SNULL, 140321421762559, 140321430151167, +STORE, 140321421758464, 140321421762559, +STORE, 140321421762560, 140321430151167, +STORE, 140321413365760, 140321421758463, +SNULL, 140321413369855, 140321421758463, +STORE, 140321413365760, 140321413369855, +STORE, 140321413369856, 140321421758463, +STORE, 140321404973056, 140321413365759, +SNULL, 140321404977151, 140321413365759, +STORE, 140321404973056, 140321404977151, +STORE, 140321404977152, 140321413365759, +STORE, 140321396580352, 140321404973055, +STORE, 140321388187648, 140321404973055, +STORE, 140321253969920, 140321388187647, +SNULL, 140321253969920, 140321279180799, +STORE, 140321279180800, 140321388187647, +STORE, 140321253969920, 140321279180799, +ERASE, 140321253969920, 140321279180799, +SNULL, 140321346289663, 140321388187647, +STORE, 140321279180800, 140321346289663, +STORE, 140321346289664, 140321388187647, +ERASE, 140321346289664, 140321388187647, +STORE, 140321144963072, 140321346289663, +STORE, 140321379794944, 140321404973055, +STORE, 140321371402240, 140321404973055, +STORE, 140321010745344, 140321346289663, +STORE, 140321363009536, 140321404973055, +SNULL, 140321077854207, 140321346289663, +STORE, 140321010745344, 140321077854207, +STORE, 140321077854208, 140321346289663, +SNULL, 140321077854208, 140321144963071, +STORE, 140321144963072, 140321346289663, +STORE, 140321077854208, 140321144963071, +ERASE, 140321077854208, 140321144963071, +STORE, 140321354616832, 140321404973055, +STORE, 140321136570368, 140321144963071, +STORE, 140320943636480, 140321077854207, +STORE, 140320876527616, 140321077854207, +STORE, 140321128177664, 140321144963071, +SNULL, 140320876662783, 140321077854207, +STORE, 140320876527616, 140320876662783, +STORE, 140320876662784, 140321077854207, +STORE, 140321119784960, 140321144963071, +STORE, 140321111392256, 140321144963071, +STORE, 140320742309888, 140320876527615, +STORE, 140321102999552, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320675201023, 140320876527615, +STORE, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +SNULL, 140320675201024, 140320742309887, +STORE, 140320742309888, 140320876527615, +STORE, 140320675201024, 140320742309887, +ERASE, 140320675201024, 140320742309887, +STORE, 140321094606848, 140321144963071, +STORE, 140321086214144, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +STORE, 140320608092160, 140320675201023, +SNULL, 140320675336191, 140320876527615, +STORE, 140320675201024, 140320675336191, +STORE, 140320675336192, 140320876527615, +STORE, 140320599699456, 140320608092159, +STORE, 140320591306752, 140320608092159, +STORE, 140320457089024, 140320591306751, +STORE, 140320448696320, 140320457089023, +STORE, 140320314478592, 140320448696319, +SNULL, 140321144963072, 140321279180799, +STORE, 140321279180800, 140321346289663, +STORE, 140321144963072, 140321279180799, +SNULL, 140321279315967, 140321346289663, +STORE, 140321279180800, 140321279315967, +STORE, 140321279315968, 140321346289663, +SNULL, 140321086214144, 140321136570367, +STORE, 140321136570368, 140321144963071, +STORE, 140321086214144, 140321136570367, +SNULL, 140321136574463, 140321144963071, +STORE, 140321136570368, 140321136574463, +STORE, 140321136574464, 140321144963071, +SNULL, 140321212071935, 140321279180799, +STORE, 140321144963072, 140321212071935, +STORE, 140321212071936, 140321279180799, +ERASE, 140321212071936, 140321279180799, +SNULL, 140321145098239, 140321212071935, +STORE, 140321144963072, 140321145098239, +STORE, 140321145098240, 140321212071935, +SNULL, 140320876662784, 140321010745343, +STORE, 140321010745344, 140321077854207, +STORE, 140320876662784, 140321010745343, +SNULL, 140321010880511, 140321077854207, +STORE, 140321010745344, 140321010880511, +STORE, 140321010880512, 140321077854207, +SNULL, 140321354616832, 140321379794943, +STORE, 140321379794944, 140321404973055, +STORE, 140321354616832, 140321379794943, +SNULL, 140321379799039, 140321404973055, +STORE, 140321379794944, 140321379799039, +STORE, 140321379799040, 140321404973055, +SNULL, 140320876662784, 140320943636479, +STORE, 140320943636480, 140321010745343, +STORE, 140320876662784, 140320943636479, +SNULL, 140320943771647, 140321010745343, +STORE, 140320943636480, 140320943771647, +STORE, 140320943771648, 140321010745343, +SNULL, 140320809418751, 140320876527615, +STORE, 140320675336192, 140320809418751, +STORE, 140320809418752, 140320876527615, +ERASE, 140320809418752, 140320876527615, +SNULL, 140320675336192, 140320742309887, +STORE, 140320742309888, 140320809418751, +STORE, 140320675336192, 140320742309887, +SNULL, 140320742445055, 140320809418751, +STORE, 140320742309888, 140320742445055, +STORE, 140320742445056, 140320809418751, +SNULL, 140320608227327, 140320675201023, +STORE, 140320608092160, 140320608227327, +STORE, 140320608227328, 140320675201023, +SNULL, 140320457089024, 140320473874431, +STORE, 140320473874432, 140320591306751, +STORE, 140320457089024, 140320473874431, +ERASE, 140320457089024, 140320473874431, +SNULL, 140320540983295, 140320591306751, +STORE, 140320473874432, 140320540983295, +STORE, 140320540983296, 140320591306751, +ERASE, 140320540983296, 140320591306751, +SNULL, 140320314478592, 140320339656703, +STORE, 140320339656704, 140320448696319, +STORE, 140320314478592, 140320339656703, +ERASE, 140320314478592, 140320339656703, +SNULL, 140321086214144, 140321128177663, +STORE, 140321128177664, 140321136570367, +STORE, 140321086214144, 140321128177663, +SNULL, 140321128181759, 140321136570367, +STORE, 140321128177664, 140321128181759, +STORE, 140321128181760, 140321136570367, +SNULL, 140321354616832, 140321371402239, +STORE, 140321371402240, 140321379794943, +STORE, 140321354616832, 140321371402239, +SNULL, 140321371406335, 140321379794943, +STORE, 140321371402240, 140321371406335, +STORE, 140321371406336, 140321379794943, +SNULL, 140320591310847, 140320608092159, +STORE, 140320591306752, 140320591310847, +STORE, 140320591310848, 140320608092159, +SNULL, 140321354616832, 140321363009535, +STORE, 140321363009536, 140321371402239, +STORE, 140321354616832, 140321363009535, +SNULL, 140321363013631, 140321371402239, +STORE, 140321363009536, 140321363013631, +STORE, 140321363013632, 140321371402239, +SNULL, 140321086214144, 140321119784959, +STORE, 140321119784960, 140321128177663, +STORE, 140321086214144, 140321119784959, +SNULL, 140321119789055, 140321128177663, +STORE, 140321119784960, 140321119789055, +STORE, 140321119789056, 140321128177663, +SNULL, 140321086218239, 140321119784959, +STORE, 140321086214144, 140321086218239, +STORE, 140321086218240, 140321119784959, +SNULL, 140321086218240, 140321094606847, +STORE, 140321094606848, 140321119784959, +STORE, 140321086218240, 140321094606847, +SNULL, 140321094610943, 140321119784959, +STORE, 140321094606848, 140321094610943, +STORE, 140321094610944, 140321119784959, +SNULL, 140320474009599, 140320540983295, +STORE, 140320473874432, 140320474009599, +STORE, 140320474009600, 140320540983295, +SNULL, 140320406765567, 140320448696319, +STORE, 140320339656704, 140320406765567, +STORE, 140320406765568, 140320448696319, +ERASE, 140320406765568, 140320448696319, +SNULL, 140320339791871, 140320406765567, +STORE, 140320339656704, 140320339791871, +STORE, 140320339791872, 140320406765567, +STORE, 140321270788096, 140321279180799, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321279180799, +SNULL, 140321254002688, 140321262395391, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321262395391, +SNULL, 140321262399487, 140321279180799, +STORE, 140321262395392, 140321262399487, +STORE, 140321262399488, 140321279180799, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321262395391, +SNULL, 140321237217280, 140321245609983, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321245609983, +SNULL, 140321245614079, 140321262395391, +STORE, 140321245609984, 140321245614079, +STORE, 140321245614080, 140321262395391, +SNULL, 140321379799040, 140321388187647, +STORE, 140321388187648, 140321404973055, +STORE, 140321379799040, 140321388187647, +SNULL, 140321388191743, 140321404973055, +STORE, 140321388187648, 140321388191743, +STORE, 140321388191744, 140321404973055, +SNULL, 140321354620927, 140321363009535, +STORE, 140321354616832, 140321354620927, +STORE, 140321354620928, 140321363009535, +SNULL, 140321388191744, 140321396580351, +STORE, 140321396580352, 140321404973055, +STORE, 140321388191744, 140321396580351, +SNULL, 140321396584447, 140321404973055, +STORE, 140321396580352, 140321396584447, +STORE, 140321396584448, 140321404973055, +SNULL, 140321094610944, 140321111392255, +STORE, 140321111392256, 140321119784959, +STORE, 140321094610944, 140321111392255, +SNULL, 140321111396351, 140321119784959, +STORE, 140321111392256, 140321111396351, +STORE, 140321111396352, 140321119784959, +STORE, 140321228824576, 140321245609983, +SNULL, 140321094610944, 140321102999551, +STORE, 140321102999552, 140321111392255, +STORE, 140321094610944, 140321102999551, +SNULL, 140321103003647, 140321111392255, +STORE, 140321102999552, 140321103003647, +STORE, 140321103003648, 140321111392255, +STORE, 140321220431872, 140321245609983, +SNULL, 140321220435967, 140321245609983, +STORE, 140321220431872, 140321220435967, +STORE, 140321220435968, 140321245609983, +STORE, 140320868134912, 140320876527615, +SNULL, 140320868139007, 140320876527615, +STORE, 140320868134912, 140320868139007, +STORE, 140320868139008, 140320876527615, +SNULL, 140320591310848, 140320599699455, +STORE, 140320599699456, 140320608092159, +STORE, 140320591310848, 140320599699455, +SNULL, 140320599703551, 140320608092159, +STORE, 140320599699456, 140320599703551, +STORE, 140320599703552, 140320608092159, +STORE, 140320859742208, 140320868134911, +SNULL, 140321262399488, 140321270788095, +STORE, 140321270788096, 140321279180799, +STORE, 140321262399488, 140321270788095, +SNULL, 140321270792191, 140321279180799, +STORE, 140321270788096, 140321270792191, +STORE, 140321270792192, 140321279180799, +STORE, 140320851349504, 140320868134911, +STORE, 140320842956800, 140320868134911, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320868134911, +SNULL, 140320826171392, 140320834564095, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320834564095, +SNULL, 140320834568191, 140320868134911, +STORE, 140320834564096, 140320834568191, +STORE, 140320834568192, 140320868134911, +SNULL, 140321220435968, 140321228824575, +STORE, 140321228824576, 140321245609983, +STORE, 140321220435968, 140321228824575, +SNULL, 140321228828671, 140321245609983, +STORE, 140321228824576, 140321228828671, +STORE, 140321228828672, 140321245609983, +STORE, 140320817778688, 140320834564095, +SNULL, 140320817782783, 140320834564095, +STORE, 140320817778688, 140320817782783, +STORE, 140320817782784, 140320834564095, +STORE, 140320582914048, 140320591306751, +SNULL, 140321228828672, 140321237217279, +STORE, 140321237217280, 140321245609983, +STORE, 140321228828672, 140321237217279, +SNULL, 140321237221375, 140321245609983, +STORE, 140321237217280, 140321237221375, +STORE, 140321237221376, 140321245609983, +SNULL, 140320448700415, 140320457089023, +STORE, 140320448696320, 140320448700415, +STORE, 140320448700416, 140320457089023, +SNULL, 140321245614080, 140321254002687, +STORE, 140321254002688, 140321262395391, +STORE, 140321245614080, 140321254002687, +SNULL, 140321254006783, 140321262395391, +STORE, 140321254002688, 140321254006783, +STORE, 140321254006784, 140321262395391, +STORE, 140320574521344, 140320591306751, +SNULL, 140320574525439, 140320591306751, +STORE, 140320574521344, 140320574525439, +STORE, 140320574525440, 140320591306751, +STORE, 140320566128640, 140320574521343, +SNULL, 140320566132735, 140320574521343, +STORE, 140320566128640, 140320566132735, +STORE, 140320566132736, 140320574521343, +SNULL, 140320574525440, 140320582914047, +STORE, 140320582914048, 140320591306751, +STORE, 140320574525440, 140320582914047, +SNULL, 140320582918143, 140320591306751, +STORE, 140320582914048, 140320582918143, +STORE, 140320582918144, 140320591306751, +STORE, 140320557735936, 140320566128639, +SNULL, 140320557740031, 140320566128639, +STORE, 140320557735936, 140320557740031, +STORE, 140320557740032, 140320566128639, +STORE, 140320549343232, 140320557735935, +STORE, 140320465481728, 140320473874431, +STORE, 140320448700416, 140320473874431, +SNULL, 140320834568192, 140320859742207, +STORE, 140320859742208, 140320868134911, +STORE, 140320834568192, 140320859742207, +SNULL, 140320859746303, 140320868134911, +STORE, 140320859742208, 140320859746303, +STORE, 140320859746304, 140320868134911, +STORE, 140320440303616, 140320448696319, +STORE, 140320431910912, 140320448696319, +SNULL, 140320834568192, 140320851349503, +STORE, 140320851349504, 140320859742207, +STORE, 140320834568192, 140320851349503, +SNULL, 140320851353599, 140320859742207, +STORE, 140320851349504, 140320851353599, +STORE, 140320851353600, 140320859742207, +SNULL, 140320817782784, 140320826171391, +STORE, 140320826171392, 140320834564095, +STORE, 140320817782784, 140320826171391, +SNULL, 140320826175487, 140320834564095, +STORE, 140320826171392, 140320826175487, +STORE, 140320826175488, 140320834564095, +SNULL, 140320834568192, 140320842956799, +STORE, 140320842956800, 140320851349503, +STORE, 140320834568192, 140320842956799, +SNULL, 140320842960895, 140320851349503, +STORE, 140320842956800, 140320842960895, +STORE, 140320842960896, 140320851349503, +STORE, 140320423518208, 140320448696319, +SNULL, 140320423522303, 140320448696319, +STORE, 140320423518208, 140320423522303, +STORE, 140320423522304, 140320448696319, +STORE, 140320415125504, 140320423518207, +STORE, 140320331264000, 140320339656703, +STORE, 140320322871296, 140320339656703, +STORE, 140320314478592, 140320339656703, +SNULL, 140320314482687, 140320339656703, +STORE, 140320314478592, 140320314482687, +STORE, 140320314482688, 140320339656703, +STORE, 140320306085888, 140320314478591, +SNULL, 140320306089983, 140320314478591, +STORE, 140320306085888, 140320306089983, +STORE, 140320306089984, 140320314478591, +STORE, 140320297693184, 140320306085887, +SNULL, 140320297697279, 140320306085887, +STORE, 140320297693184, 140320297697279, +STORE, 140320297697280, 140320306085887, +STORE, 140320289300480, 140320297693183, +STORE, 140320280907776, 140320297693183, +SNULL, 140320280911871, 140320297693183, +STORE, 140320280907776, 140320280911871, +STORE, 140320280911872, 140320297693183, +SNULL, 140320423522304, 140320431910911, +STORE, 140320431910912, 140320448696319, +STORE, 140320423522304, 140320431910911, +SNULL, 140320431915007, 140320448696319, +STORE, 140320431910912, 140320431915007, +STORE, 140320431915008, 140320448696319, +SNULL, 140320549347327, 140320557735935, +STORE, 140320549343232, 140320549347327, +STORE, 140320549347328, 140320557735935, +STORE, 140320272515072, 140320280907775, +SNULL, 140320448700416, 140320457089023, +STORE, 140320457089024, 140320473874431, +STORE, 140320448700416, 140320457089023, +SNULL, 140320457093119, 140320473874431, +STORE, 140320457089024, 140320457093119, +STORE, 140320457093120, 140320473874431, +STORE, 140320264122368, 140320280907775, +SNULL, 140320457093120, 140320465481727, +STORE, 140320465481728, 140320473874431, +STORE, 140320457093120, 140320465481727, +SNULL, 140320465485823, 140320473874431, +STORE, 140320465481728, 140320465485823, +STORE, 140320465485824, 140320473874431, +SNULL, 140320431915008, 140320440303615, +STORE, 140320440303616, 140320448696319, +STORE, 140320431915008, 140320440303615, +SNULL, 140320440307711, 140320448696319, +STORE, 140320440303616, 140320440307711, +STORE, 140320440307712, 140320448696319, +STORE, 140320255729664, 140320280907775, +STORE, 140320247336960, 140320280907775, +SNULL, 140320247341055, 140320280907775, +STORE, 140320247336960, 140320247341055, +STORE, 140320247341056, 140320280907775, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320247336959, +SNULL, 140320230551552, 140320238944255, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320238944255, +SNULL, 140320238948351, 140320247336959, +STORE, 140320238944256, 140320238948351, +STORE, 140320238948352, 140320247336959, +SNULL, 140320314482688, 140320331263999, +STORE, 140320331264000, 140320339656703, +STORE, 140320314482688, 140320331263999, +SNULL, 140320331268095, 140320339656703, +STORE, 140320331264000, 140320331268095, +STORE, 140320331268096, 140320339656703, +SNULL, 140320280911872, 140320289300479, +STORE, 140320289300480, 140320297693183, +STORE, 140320280911872, 140320289300479, +SNULL, 140320289304575, 140320297693183, +STORE, 140320289300480, 140320289304575, +STORE, 140320289304576, 140320297693183, +SNULL, 140320415129599, 140320423518207, +STORE, 140320415125504, 140320415129599, +STORE, 140320415129600, 140320423518207, +STORE, 140320222158848, 140320238944255, +STORE, 140320213766144, 140320238944255, +STORE, 140320205373440, 140320238944255, +SNULL, 140320205377535, 140320238944255, +STORE, 140320205373440, 140320205377535, +STORE, 140320205377536, 140320238944255, +SNULL, 140320314482688, 140320322871295, +STORE, 140320322871296, 140320331263999, +STORE, 140320314482688, 140320322871295, +SNULL, 140320322875391, 140320331263999, +STORE, 140320322871296, 140320322875391, +STORE, 140320322875392, 140320331263999, +SNULL, 140320247341056, 140320272515071, +STORE, 140320272515072, 140320280907775, +STORE, 140320247341056, 140320272515071, +SNULL, 140320272519167, 140320280907775, +STORE, 140320272515072, 140320272519167, +STORE, 140320272519168, 140320280907775, +SNULL, 140320247341056, 140320264122367, +STORE, 140320264122368, 140320272515071, +STORE, 140320247341056, 140320264122367, +SNULL, 140320264126463, 140320272515071, +STORE, 140320264122368, 140320264126463, +STORE, 140320264126464, 140320272515071, +SNULL, 140320205377536, 140320230551551, +STORE, 140320230551552, 140320238944255, +STORE, 140320205377536, 140320230551551, +SNULL, 140320230555647, 140320238944255, +STORE, 140320230551552, 140320230555647, +STORE, 140320230555648, 140320238944255, +STORE, 140320196980736, 140320205373439, +SNULL, 140320196984831, 140320205373439, +STORE, 140320196980736, 140320196984831, +STORE, 140320196984832, 140320205373439, +STORE, 140320188588032, 140320196980735, +SNULL, 140320247341056, 140320255729663, +STORE, 140320255729664, 140320264122367, +STORE, 140320247341056, 140320255729663, +SNULL, 140320255733759, 140320264122367, +STORE, 140320255729664, 140320255733759, +STORE, 140320255733760, 140320264122367, +STORE, 140320180195328, 140320196980735, +SNULL, 140320180199423, 140320196980735, +STORE, 140320180195328, 140320180199423, +STORE, 140320180199424, 140320196980735, +STORE, 140320171802624, 140320180195327, +STORE, 140320163409920, 140320180195327, +SNULL, 140320163414015, 140320180195327, +STORE, 140320163409920, 140320163414015, +STORE, 140320163414016, 140320180195327, +SNULL, 140320205377536, 140320222158847, +STORE, 140320222158848, 140320230551551, +STORE, 140320205377536, 140320222158847, +SNULL, 140320222162943, 140320230551551, +STORE, 140320222158848, 140320222162943, +STORE, 140320222162944, 140320230551551, +SNULL, 140320205377536, 140320213766143, +STORE, 140320213766144, 140320222158847, +STORE, 140320205377536, 140320213766143, +SNULL, 140320213770239, 140320222158847, +STORE, 140320213766144, 140320213770239, +STORE, 140320213770240, 140320222158847, +STORE, 140320155017216, 140320163409919, +SNULL, 140320180199424, 140320188588031, +STORE, 140320188588032, 140320196980735, +STORE, 140320180199424, 140320188588031, +SNULL, 140320188592127, 140320196980735, +STORE, 140320188588032, 140320188592127, +STORE, 140320188592128, 140320196980735, +SNULL, 140320155021311, 140320163409919, +STORE, 140320155017216, 140320155021311, +STORE, 140320155021312, 140320163409919, +SNULL, 140320163414016, 140320171802623, +STORE, 140320171802624, 140320180195327, +STORE, 140320163414016, 140320171802623, +SNULL, 140320171806719, 140320180195327, +STORE, 140320171802624, 140320171806719, +STORE, 140320171806720, 140320180195327, +STORE, 140320146624512, 140320155017215, +SNULL, 140320146628607, 140320155017215, +STORE, 140320146624512, 140320146628607, +STORE, 140320146628608, 140320155017215, +STORE, 140321937321984, 140321937350655, +STORE, 140321884942336, 140321887133695, +SNULL, 140321884942336, 140321885032447, +STORE, 140321885032448, 140321887133695, +STORE, 140321884942336, 140321885032447, +SNULL, 140321887125503, 140321887133695, +STORE, 140321885032448, 140321887125503, +STORE, 140321887125504, 140321887133695, +ERASE, 140321887125504, 140321887133695, +STORE, 140321887125504, 140321887133695, +SNULL, 140321887129599, 140321887133695, +STORE, 140321887125504, 140321887129599, +STORE, 140321887129600, 140321887133695, +ERASE, 140321937321984, 140321937350655, +ERASE, 140321086214144, 140321086218239, +ERASE, 140321086218240, 140321094606847, +ERASE, 140321119784960, 140321119789055, +ERASE, 140321119789056, 140321128177663, +ERASE, 140321245609984, 140321245614079, +ERASE, 140321245614080, 140321254002687, +ERASE, 140320574521344, 140320574525439, +ERASE, 140320574525440, 140320582914047, +ERASE, 140320297693184, 140320297697279, +ERASE, 140320297697280, 140320306085887, +ERASE, 140321354616832, 140321354620927, +ERASE, 140321354620928, 140321363009535, +ERASE, 140320834564096, 140320834568191, +ERASE, 140320834568192, 140320842956799, +ERASE, 140320591306752, 140320591310847, +ERASE, 140320591310848, 140320599699455, +ERASE, 140321136570368, 140321136574463, +ERASE, 140321136574464, 140321144963071, +ERASE, 140321237217280, 140321237221375, +ERASE, 140321237221376, 140321245609983, +ERASE, 140321363009536, 140321363013631, +ERASE, 140321363013632, 140321371402239, +ERASE, 140320599699456, 140320599703551, +ERASE, 140320599703552, 140320608092159, +ERASE, 140321396580352, 140321396584447, +ERASE, 140321396584448, 140321404973055, +ERASE, 140320566128640, 140320566132735, +ERASE, 140320566132736, 140320574521343, +ERASE, 140321094606848, 140321094610943, +ERASE, 140321094610944, 140321102999551, +ERASE, 140320582914048, 140320582918143, +ERASE, 140320582918144, 140320591306751, +ERASE, 140320289300480, 140320289304575, +ERASE, 140320289304576, 140320297693183, +ERASE, 140320163409920, 140320163414015, + }; + unsigned long set41[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140728157171712, 140737488351231, +SNULL, 140728157175807, 140737488351231, +STORE, 140728157171712, 140728157175807, +STORE, 140728157040640, 140728157175807, +STORE, 94376106364928, 94376108613631, +SNULL, 94376106487807, 94376108613631, +STORE, 94376106364928, 94376106487807, +STORE, 94376106487808, 94376108613631, +SNULL, 94376106487808, 94376108613631, +STORE, 94376108584960, 94376108593151, +STORE, 94376108593152, 94376108613631, +STORE, 140113496432640, 140113498685439, +SNULL, 140113496575999, 140113498685439, +STORE, 140113496432640, 140113496575999, +STORE, 140113496576000, 140113498685439, +SNULL, 140113496576000, 140113498685439, +STORE, 140113498673152, 140113498681343, +STORE, 140113498681344, 140113498685439, +STORE, 140728157609984, 140728157618175, +STORE, 140728157593600, 140728157609983, +STORE, 140113498636288, 140113498673151, +STORE, 140113498628096, 140113498636287, +STORE, 140113492635648, 140113496432639, +SNULL, 140113492635648, 140113494294527, +STORE, 140113494294528, 140113496432639, +STORE, 140113492635648, 140113494294527, +SNULL, 140113496391679, 140113496432639, +STORE, 140113494294528, 140113496391679, +STORE, 140113496391680, 140113496432639, +SNULL, 140113496391680, 140113496416255, +STORE, 140113496416256, 140113496432639, +STORE, 140113496391680, 140113496416255, +SNULL, 140113496391680, 140113496416255, +STORE, 140113496391680, 140113496416255, +SNULL, 140113496416256, 140113496432639, +STORE, 140113496416256, 140113496432639, +SNULL, 140113496408063, 140113496416255, +STORE, 140113496391680, 140113496408063, +STORE, 140113496408064, 140113496416255, +SNULL, 94376108589055, 94376108593151, +STORE, 94376108584960, 94376108589055, +STORE, 94376108589056, 94376108593151, +SNULL, 140113498677247, 140113498681343, +STORE, 140113498673152, 140113498677247, +STORE, 140113498677248, 140113498681343, +SNULL, 140113498636288, 140113498673151, +STORE, 94376135090176, 94376135094271, +STORE, 94376135090176, 94376135098367, +STORE, 94376139288576, 94376139292671, +STORE, 94376143482880, 94376143486975, +STORE, 94376147677184, 94376147681279, +STORE, 94376151871488, 94376151875583, +STORE, 94376156065792, 94376156069887, +STORE, 94376160260096, 94376160264191, +STORE, 94376164454400, 94376164458495, +STORE, 94376168648704, 94376168652799, +STORE, 94376172843008, 94376172847103, +STORE, 94376177037312, 94376177041407, +STORE, 94376181231616, 94376181235711, +STORE, 94376185425920, 94376185430015, +STORE, 94376189620224, 94376189624319, +STORE, 94376193814528, 94376193818623, +STORE, 94376198008832, 94376198012927, +STORE, 94376202203136, 94376202207231, +STORE, 94376206397440, 94376206401535, +STORE, 94376210591744, 94376210595839, +STORE, 94376214786048, 94376214790143, +STORE, 94376218980352, 94376218984447, +STORE, 94376223174656, 94376223178751, +STORE, 94376227368960, 94376227373055, +STORE, 94376231563264, 94376231567359, +STORE, 94376235757568, 94376235761663, +STORE, 94376239951872, 94376239955967, +STORE, 94376244146176, 94376244150271, +STORE, 94376248340480, 94376248344575, +STORE, 94376252534784, 94376252538879, +STORE, 94376256729088, 94376256733183, +STORE, 94376260923392, 94376260927487, +STORE, 94376265117696, 94376265121791, +STORE, 94376269312000, 94376269316095, +STORE, 94376273506304, 94376273510399, +STORE, 94376277700608, 94376277704703, +STORE, 94376281894912, 94376281899007, +STORE, 94376286089216, 94376286093311, +STORE, 94376290283520, 94376290287615, +STORE, 94376294477824, 94376294481919, +STORE, 94376298672128, 94376298676223, +STORE, 94376302866432, 94376302870527, +STORE, 94376307060736, 94376307064831, +STORE, 94376311255040, 94376311259135, +STORE, 94376315449344, 94376315453439, +STORE, 94376319643648, 94376319647743, +STORE, 94376323837952, 94376323842047, +STORE, 94376328032256, 94376328036351, +STORE, 94376332226560, 94376332230655, +STORE, 94376336420864, 94376336424959, +STORE, 94376340615168, 94376340619263, +STORE, 94376344809472, 94376344813567, +STORE, 94376349003776, 94376349007871, +STORE, 94376353198080, 94376353202175, +STORE, 94376357392384, 94376357396479, +STORE, 94376361586688, 94376361590783, +STORE, 94376365780992, 94376365785087, +STORE, 94376369975296, 94376369979391, +STORE, 94376374169600, 94376374173695, +STORE, 94376378363904, 94376378367999, +STORE, 94376382558208, 94376382562303, +STORE, 94376386752512, 94376386756607, +STORE, 94376390946816, 94376390950911, +STORE, 94376395141120, 94376395145215, +STORE, 94376399335424, 94376399339519, +STORE, 94376403529728, 94376403533823, +STORE, 94376407724032, 94376407728127, +STORE, 94376411918336, 94376411922431, +STORE, 94376416112640, 94376416116735, +STORE, 94376420306944, 94376420311039, +STORE, 94376424501248, 94376424505343, +STORE, 94376428695552, 94376428699647, +STORE, 94376432889856, 94376432893951, +STORE, 94376437084160, 94376437088255, +STORE, 94376441278464, 94376441282559, +STORE, 94376445472768, 94376445476863, +STORE, 94376449667072, 94376449671167, +STORE, 94376453861376, 94376453865471, +STORE, 94376458055680, 94376458059775, +STORE, 94376462249984, 94376462254079, +STORE, 94376466444288, 94376466448383, +STORE, 94376470638592, 94376470642687, +STORE, 94376474832896, 94376474836991, +STORE, 94376479027200, 94376479031295, +STORE, 94376483221504, 94376483225599, +STORE, 94376487415808, 94376487419903, +STORE, 94376491610112, 94376491614207, +STORE, 94376495804416, 94376495808511, +STORE, 94376499998720, 94376500002815, +STORE, 94376504193024, 94376504197119, +STORE, 94376508387328, 94376508391423, +STORE, 94376512581632, 94376512585727, +STORE, 94376516775936, 94376516780031, +STORE, 94376520970240, 94376520974335, +STORE, 94376525164544, 94376525168639, +STORE, 94376529358848, 94376529362943, +STORE, 94376533553152, 94376533557247, +STORE, 94376537747456, 94376537751551, +STORE, 94376541941760, 94376541945855, +STORE, 94376546136064, 94376546140159, +STORE, 94376550330368, 94376550334463, +STORE, 94376554524672, 94376554528767, +STORE, 94376558718976, 94376558723071, +STORE, 94376562913280, 94376562917375, +STORE, 94376567107584, 94376567111679, +STORE, 94376571301888, 94376571305983, +STORE, 94376575496192, 94376575500287, +STORE, 94376579690496, 94376579694591, +STORE, 94376583884800, 94376583888895, +STORE, 94376588079104, 94376588083199, +STORE, 94376592273408, 94376592277503, +STORE, 94376596467712, 94376596471807, +STORE, 94376600662016, 94376600666111, +STORE, 94376604856320, 94376604860415, +STORE, 94376609050624, 94376609054719, +STORE, 94376613244928, 94376613249023, +STORE, 94376617439232, 94376617443327, +STORE, 94376621633536, 94376621637631, +STORE, 94376625827840, 94376625831935, +STORE, 94376630022144, 94376630026239, +STORE, 94376634216448, 94376634220543, +STORE, 94376638410752, 94376638414847, +STORE, 94376642605056, 94376642609151, +STORE, 94376646799360, 94376646803455, +STORE, 94376650993664, 94376650997759, +STORE, 94376655187968, 94376655192063, +STORE, 94376659382272, 94376659386367, +STORE, 94376663576576, 94376663580671, +STORE, 94376667770880, 94376667774975, +STORE, 94376671965184, 94376671969279, +STORE, 94376676159488, 94376676163583, +STORE, 94376680353792, 94376680357887, +STORE, 94376684548096, 94376684552191, +STORE, 94376688742400, 94376688746495, +STORE, 94376692936704, 94376692940799, +STORE, 94376697131008, 94376697135103, +STORE, 94376701325312, 94376701329407, +STORE, 94376705519616, 94376705523711, +STORE, 94376709713920, 94376709718015, +STORE, 94376713908224, 94376713912319, +STORE, 94376718102528, 94376718106623, +STORE, 94376722296832, 94376722300927, +STORE, 94376726491136, 94376726495231, +STORE, 94376730685440, 94376730689535, +STORE, 94376734879744, 94376734883839, +STORE, 94376739074048, 94376739078143, +STORE, 94376743268352, 94376743272447, +STORE, 94376747462656, 94376747466751, +STORE, 94376751656960, 94376751661055, +STORE, 94376755851264, 94376755855359, +STORE, 94376760045568, 94376760049663, +STORE, 94376764239872, 94376764243967, +STORE, 94376768434176, 94376768438271, +STORE, 94376772628480, 94376772632575, +STORE, 94376776822784, 94376776826879, +STORE, 94376781017088, 94376781021183, +STORE, 94376785211392, 94376785215487, +STORE, 94376789405696, 94376789409791, +STORE, 94376793600000, 94376793604095, +STORE, 94376797794304, 94376797798399, +STORE, 94376801988608, 94376801992703, +STORE, 94376806182912, 94376806187007, +STORE, 94376810377216, 94376810381311, +STORE, 94376814571520, 94376814575615, +STORE, 94376818765824, 94376818769919, +STORE, 94376822960128, 94376822964223, +STORE, 94376827154432, 94376827158527, +STORE, 94376831348736, 94376831352831, +STORE, 94376835543040, 94376835547135, +STORE, 94376839737344, 94376839741439, +STORE, 94376843931648, 94376843935743, +STORE, 94376848125952, 94376848130047, +STORE, 94376852320256, 94376852324351, +STORE, 94376856514560, 94376856518655, +STORE, 94376860708864, 94376860712959, +STORE, 94376864903168, 94376864907263, +STORE, 94376869097472, 94376869101567, +STORE, 94376873291776, 94376873295871, +STORE, 94376877486080, 94376877490175, +STORE, 94376881680384, 94376881684479, +STORE, 94376885874688, 94376885878783, +STORE, 94376890068992, 94376890073087, +STORE, 94376894263296, 94376894267391, +STORE, 94376898457600, 94376898461695, +STORE, 94376902651904, 94376902655999, +STORE, 94376906846208, 94376906850303, +STORE, 94376911040512, 94376911044607, +STORE, 94376915234816, 94376915238911, +STORE, 94376919429120, 94376919433215, +STORE, 94376923623424, 94376923627519, +STORE, 94376927817728, 94376927821823, +STORE, 94376932012032, 94376932016127, +STORE, 94376936206336, 94376936210431, +STORE, 94376940400640, 94376940404735, +STORE, 94376944594944, 94376944599039, +STORE, 94376948789248, 94376948793343, +STORE, 94376952983552, 94376952987647, +STORE, 94376957177856, 94376957181951, +STORE, 94376961372160, 94376961376255, +STORE, 94376965566464, 94376965570559, +STORE, 94376969760768, 94376969764863, +STORE, 94376973955072, 94376973959167, +STORE, 94376978149376, 94376978153471, +STORE, 94376982343680, 94376982347775, +STORE, 94376986537984, 94376986542079, +STORE, 94376990732288, 94376990736383, +STORE, 94376994926592, 94376994930687, +STORE, 94376999120896, 94376999124991, +STORE, 94377003315200, 94377003319295, +STORE, 94377007509504, 94377007513599, +STORE, 94377011703808, 94377011707903, +STORE, 94377015898112, 94377015902207, +STORE, 94377020092416, 94377020096511, +STORE, 94377024286720, 94377024290815, +STORE, 94377028481024, 94377028485119, +STORE, 94377032675328, 94377032679423, +STORE, 94377036869632, 94377036873727, +STORE, 94377041063936, 94377041068031, +STORE, 94377045258240, 94377045262335, +STORE, 94377049452544, 94377049456639, +STORE, 94377053646848, 94377053650943, +STORE, 94377057841152, 94377057845247, +STORE, 94377062035456, 94377062039551, +STORE, 94377066229760, 94377066233855, +STORE, 94377070424064, 94377070428159, +STORE, 94377074618368, 94377074622463, +STORE, 94377078812672, 94377078816767, +STORE, 94377083006976, 94377083011071, +STORE, 94377087201280, 94377087205375, +STORE, 94377091395584, 94377091399679, +STORE, 94377095589888, 94377095593983, +STORE, 94377099784192, 94377099788287, +STORE, 94377103978496, 94377103982591, +STORE, 94377108172800, 94377108176895, +STORE, 94377112367104, 94377112371199, +STORE, 94377116561408, 94377116565503, +STORE, 94377120755712, 94377120759807, +STORE, 94377124950016, 94377124954111, +STORE, 94377129144320, 94377129148415, +STORE, 94377133338624, 94377133342719, +STORE, 94377137532928, 94377137537023, +STORE, 94377141727232, 94377141731327, +STORE, 94377145921536, 94377145925631, +STORE, 94377150115840, 94377150119935, +STORE, 94377154310144, 94377154314239, +STORE, 94377158504448, 94377158508543, +STORE, 94377162698752, 94377162702847, +STORE, 94377166893056, 94377166897151, +STORE, 94377171087360, 94377171091455, +STORE, 94377175281664, 94377175285759, +STORE, 94377179475968, 94377179480063, +STORE, 94377183670272, 94377183674367, +STORE, 94377187864576, 94377187868671, +STORE, 94377192058880, 94377192062975, +STORE, 94377196253184, 94377196257279, +STORE, 94377200447488, 94377200451583, +STORE, 94377204641792, 94377204645887, +SNULL, 94376135094271, 94376135098367, +STORE, 94376135090176, 94376135094271, +STORE, 94376135094272, 94376135098367, +SNULL, 94376135094272, 94377208836095, + }; + unsigned long set42[] = { +STORE, 314572800, 1388314623, +STORE, 1462157312, 1462169599, +STORE, 1462169600, 1462185983, +STORE, 1462185984, 1462190079, +STORE, 1462190080, 1462194175, +STORE, 1462194176, 1462198271, +STORE, 1879986176, 1881800703, +STORE, 1881800704, 1882034175, +STORE, 1882034176, 1882193919, +STORE, 1882193920, 1882406911, +STORE, 1882406912, 1882451967, +STORE, 1882451968, 1882996735, +STORE, 1882996736, 1885892607, +STORE, 1885892608, 1885896703, +STORE, 1885896704, 1885904895, +STORE, 1885904896, 1885908991, +STORE, 1885908992, 1885913087, +STORE, 1885913088, 1885966335, +STORE, 1885966336, 1886232575, +STORE, 1886232576, 1886236671, +STORE, 1886236672, 1886240767, +STORE, 1886240768, 1886244863, +STORE, 1886244864, 1886248959, +STORE, 1886248960, 1886294015, +STORE, 1886294016, 1886494719, +STORE, 1886494720, 1886498815, +STORE, 1886498816, 1886502911, +STORE, 1886502912, 1886507007, +STORE, 1886507008, 1886511103, +STORE, 1886511104, 1886556159, +STORE, 1886556160, 1886629887, +STORE, 1886629888, 1886633983, +STORE, 1886633984, 1886638079, +STORE, 1886638080, 1886642175, +STORE, 1886642176, 1886646271, +STORE, 1886646272, 1886666751, +STORE, 1886666752, 1886670847, +STORE, 1886670848, 1886674943, +STORE, 1886674944, 1886679039, +STORE, 1886679040, 1895419903, +STORE, 1895419904, 1895550975, +STORE, 1895550976, 1896148991, +STORE, 1896148992, 1897189375, +STORE, 1897189376, 1897701375, +STORE, 1897701376, 1897803775, +STORE, 1897803776, 1897816063, +STORE, 1897816064, 1899913215, +STORE, 1899913216, 1909379071, +STORE, 1909379072, 1909387263, +STORE, 1909387264, 1909391359, +STORE, 1909391360, 1909432319, +STORE, 1909432320, 1909436415, +STORE, 1909436416, 1909440511, +STORE, 1909440512, 1909460991, +STORE, 1909460992, 1909547007, +STORE, 1909547008, 1909551103, +STORE, 1909551104, 1909555199, +STORE, 1909555200, 1909559295, +STORE, 1909559296, 1909563391, +STORE, 1909563392, 1909739519, +STORE, 1909739520, 1910566911, +STORE, 1910566912, 1910571007, +STORE, 1910571008, 1910575103, +STORE, 1910575104, 1910579199, +STORE, 1910579200, 1910583295, +STORE, 1910583296, 1910587391, +STORE, 1910587392, 1910620159, +STORE, 1910620160, 1910624255, +STORE, 1910624256, 1910628351, +STORE, 1910628352, 1910632447, +STORE, 1910632448, 1910652927, +STORE, 1910652928, 1910657023, +STORE, 1910657024, 1910661119, +STORE, 1910661120, 1910665215, +STORE, 1910665216, 1910669311, +STORE, 1910669312, 1910677503, +STORE, 1910677504, 1910681599, +STORE, 1910681600, 1910685695, +STORE, 1910685696, 1910689791, +STORE, 1910689792, 1910697983, +STORE, 1910697984, 1910702079, +STORE, 1910702080, 1910706175, +STORE, 1910706176, 1910710271, +STORE, 1910710272, 1914093567, +STORE, 1914093568, 1914097663, +STORE, 1914097664, 1969434623, +STORE, 1969434624, 1977819135, +STORE, 3290435584, 3426750463, +STORE, 3426750464, 3426754559, +STORE, 3426754560, 3426762751, +STORE, 3426762752, 3426766847, +STORE, 3426766848, 3426770943, +STORE, 3427037184, 3427061759, +STORE, 3427061760, 3427135487, +STORE, 3427135488, 3427143679, +STORE, 3427143680, 3427147775, +STORE, 3427147776, 3427209215, +STORE, 3427319808, 3432116223, +STORE, 3432116224, 3450130431, +STORE, 3450130432, 3451027455, +STORE, 3451027456, 3451031551, +STORE, 3451031552, 3451461631, +STORE, 3451736064, 3456688127, +STORE, 3456688128, 3475222527, +STORE, 3475222528, 3476119551, +STORE, 3476119552, 3476127743, +STORE, 3476127744, 3476553727, +STORE, 3476631552, 3477315583, +STORE, 3477315584, 3479949311, +STORE, 3479949312, 3480002559, +STORE, 3480002560, 3480006655, +STORE, 3480006656, 3480432639, +STORE, 3480539136, 3480543231, +STORE, 3480543232, 3480547327, +STORE, 3480547328, 3480555519, +STORE, 3480854528, 3480903679, +STORE, 3480903680, 3480969215, +STORE, 3480969216, 3480977407, +STORE, 3480977408, 3480981503, +STORE, 3481030656, 3481092095, +STORE, 3481092096, 3481235455, +STORE, 3481235456, 3481243647, +STORE, 3481243648, 3481247743, +STORE, 3481436160, 3481444351, +STORE, 3481444352, 3481456639, +STORE, 3481456640, 3481460735, +STORE, 3481460736, 3481464831, +STORE, 3481587712, 3481645055, +STORE, 3481645056, 3481772031, +STORE, 3481772032, 3481776127, +STORE, 3481776128, 3481780223, +STORE, 3481874432, 3481935871, +STORE, 3481935872, 3482030079, +STORE, 3482030080, 3482038271, +STORE, 3482038272, 3482042367, +STORE, 3482198016, 3482230783, +STORE, 3482230784, 3482271743, +STORE, 3482271744, 3482279935, +STORE, 3482279936, 3482284031, +STORE, 3482562560, 3482566655, +STORE, 3482566656, 3482570751, +STORE, 3482570752, 3482574847, +STORE, 3482636288, 3482689535, +STORE, 3482689536, 3482746879, +STORE, 3482746880, 3482755071, +STORE, 3482755072, 3482759167, +STORE, 3482972160, 3483062271, +STORE, 3483062272, 3483242495, +STORE, 3483242496, 3483246591, +STORE, 3483246592, 3483250687, +STORE, 3483398144, 3483688959, +STORE, 3483688960, 3484114943, +STORE, 3484114944, 3484131327, +STORE, 3484131328, 3484135423, +STORE, 3484135424, 3484143615, +STORE, 3484184576, 3484475391, +STORE, 3484475392, 3485028351, +STORE, 3485028352, 3485057023, +STORE, 3485057024, 3485061119, +STORE, 3485360128, 3485364223, +STORE, 3485364224, 3485368319, +STORE, 3485368320, 3485372415, +STORE, 3485589504, 3485593599, +STORE, 3485593600, 3485597695, +STORE, 3485597696, 3485601791, +STORE, 3485913088, 3485937663, +STORE, 3485937664, 3485974527, +STORE, 3485974528, 3485982719, +STORE, 3485982720, 3485986815, +STORE, 3486052352, 3486056447, +STORE, 3486056448, 3486064639, +STORE, 3486064640, 3486068735, +STORE, 3486068736, 3486072831, +STORE, 3486294016, 3486302207, +STORE, 3486302208, 3486306303, +STORE, 3486306304, 3486310399, +STORE, 3486310400, 3486314495, +STORE, 3486670848, 3486679039, +STORE, 3486679040, 3486683135, +STORE, 3486683136, 3486687231, +STORE, 3486687232, 3486691327, +STORE, 3486863360, 3486871551, +STORE, 3486871552, 3486875647, +STORE, 3486875648, 3486879743, +STORE, 3486879744, 3486883839, +STORE, 3487584256, 3522543615, +STORE, 3522543616, 3523321855, +STORE, 3523321856, 3523342335, +STORE, 3523342336, 3523387391, +STORE, 3523387392, 3523391487, +STORE, 3523391488, 3523395583, +STORE, 3523477504, 3523686399, +STORE, 3523686400, 3523981311, +STORE, 3523981312, 3523997695, +STORE, 3523997696, 3524001791, +STORE, 3524177920, 3525013503, +STORE, 3525013504, 3526582271, +STORE, 3526582272, 3526606847, +STORE, 3526606848, 3526610943, +STORE, 3526610944, 3526615039, +STORE, 3526672384, 3526746111, +STORE, 3526746112, 3526860799, +STORE, 3526860800, 3526868991, +STORE, 3526868992, 3526873087, +STORE, 3527000064, 3527475199, +STORE, 3527475200, 3527479295, +STORE, 3527479296, 3527573503, +STORE, 3527573504, 3527581695, +STORE, 3527581696, 3527585791, +STORE, 3527585792, 3527606271, +STORE, 3527909376, 3527913471, +STORE, 3527913472, 3527917567, +STORE, 3527917568, 3527921663, +STORE, 3527950336, 3528011775, +STORE, 3528011776, 3528093695, +STORE, 3528093696, 3528101887, +STORE, 3528101888, 3528105983, +STORE, 3528228864, 3528241151, +STORE, 3528241152, 3528261631, +STORE, 3528261632, 3528265727, +STORE, 3528273920, 3528593407, +STORE, 3528593408, 3528609791, +STORE, 3528609792, 3528638463, +STORE, 3528638464, 3528642559, +STORE, 3528642560, 3528646655, +STORE, 3528880128, 3528912895, +STORE, 3528912896, 3528962047, +STORE, 3528962048, 3528966143, +STORE, 3528966144, 3528970239, +STORE, 3528982528, 3530293247, +STORE, 3530366976, 3530825727, +STORE, 3530825728, 3531317247, +STORE, 3531317248, 3541041151, +STORE, 3541041152, 3541303295, +STORE, 3541430272, 3566206975, +STORE, 3566206976, 3566993407, +STORE, 3567239168, 3587571711, +STORE, 3587571712, 3588284415, +STORE, 3588284416, 3588661247, +STORE, 3588661248, 3589066751, +STORE, 3589066752, 3589574655, +STORE, 3589574656, 3590078463, +STORE, 3590078464, 3590373375, +STORE, 3590373376, 3590668287, +STORE, 3590668288, 3590963199, +STORE, 3590963200, 3591294975, +STORE, 3591294976, 3591602175, +STORE, 3591602176, 3591933951, +STORE, 3591933952, 3592241151, +STORE, 3592241152, 3592572927, +STORE, 3592572928, 3592876031, +STORE, 3592876032, 3593211903, +STORE, 3593211904, 3593547775, +STORE, 3593547776, 3593650175, +STORE, 3593650176, 3593928703, +STORE, 3593928704, 3593936895, +STORE, 3593936896, 3593940991, +STORE, 3594006528, 3594301439, +STORE, 3594301440, 3594739711, +STORE, 3594739712, 3594756095, +STORE, 3594756096, 3594760191, +STORE, 3594760192, 3594768383, +STORE, 3594952704, 3595051007, +STORE, 3595051008, 3595223039, +STORE, 3595223040, 3595227135, +STORE, 3595227136, 3595235327, +STORE, 3595431936, 3595775999, +STORE, 3595776000, 3596701695, +STORE, 3596701696, 3596742655, +STORE, 3596742656, 3596746751, +STORE, 3596746752, 3596750847, +STORE, 3596767232, 3597070335, +STORE, 3597070336, 3597402111, +STORE, 3597402112, 3598188543, +STORE, 3598262272, 3623428095, +STORE, 3623428096, 3623432191, +STORE, 3623432192, 3623436287, +STORE, 3623436288, 3623440383, +STORE, 3623616512, 3623878655, +STORE, 3624169472, 3624300543, +STORE, 3627524096, 3628523519, +STORE, 3628523520, 3629522943, +STORE, 3696631808, 3730186239, +STORE, 3730186240, 3763740671, +STORE, 3763740672, 3764027391, +STORE, 3764027392, 3765133311, +STORE, 3765133312, 3765145599, +STORE, 3765145600, 3765149695, +STORE, 3765178368, 3766022143, +STORE, 3766022144, 3768791039, +STORE, 3768791040, 3768840191, +STORE, 3768840192, 3768844287, +STORE, 3768897536, 3768913919, +STORE, 3768913920, 3768934399, +STORE, 3768934400, 3768938495, +STORE, 3769016320, 3769147391, +STORE, 3769147392, 3769233407, +STORE, 3769233408, 3769356287, +STORE, 3769356288, 3769360383, +STORE, 3769360384, 3769368575, +STORE, 3769376768, 3794542591, +STORE, 3794542592, 3794599935, +STORE, 3794599936, 3794731007, +STORE, 3794731008, 3794735103, +STORE, 3794735104, 3794743295, +STORE, 3794849792, 3794980863, +STORE, 3794980864, 3794984959, +STORE, 3794984960, 3794989055, +STORE, 3794989056, 3794993151, +STORE, 3794993152, 3794997247, +STORE, 3795103744, 3795128319, +STORE, 3795128320, 3795165183, +STORE, 3795165184, 3795169279, +STORE, 3795169280, 3795173375, +STORE, 3795210240, 3795357695, +STORE, 3795357696, 3795365887, +STORE, 3795365888, 3795374079, +STORE, 3795374080, 3795378175, +STORE, 3795378176, 3795382271, +STORE, 3795406848, 3795738623, +STORE, 3795738624, 3795742719, +STORE, 3795742720, 3795755007, +STORE, 3795755008, 3795759103, +STORE, 3795763200, 3795894271, +STORE, 3795894272, 3796041727, +STORE, 3796041728, 3796054015, +STORE, 3796054016, 3796066303, +STORE, 3796066304, 3796070399, +STORE, 3796176896, 3796205567, +STORE, 3796205568, 3796250623, +STORE, 3796250624, 3796254719, +STORE, 3796254720, 3796258815, +STORE, 3796262912, 3796393983, +STORE, 3796393984, 3796516863, +STORE, 3796516864, 3796873215, +STORE, 3796873216, 3796885503, +STORE, 3796885504, 3796889599, +STORE, 3796963328, 3796967423, +STORE, 3796967424, 3796975615, +STORE, 3796975616, 3796979711, +STORE, 3797000192, 3797307391, +STORE, 3797307392, 3797311487, +STORE, 3797311488, 3797315583, +STORE, 3797315584, 3797323775, +STORE, 3797327872, 3797450751, +STORE, 3797450752, 3797458943, +STORE, 3797458944, 3797471231, +STORE, 3797471232, 3797475327, +STORE, 3797577728, 3797700607, +STORE, 3797700608, 3797721087, +STORE, 3797721088, 3797733375, +STORE, 3797733376, 3797741567, +STORE, 3797741568, 3797864447, +STORE, 3797864448, 3797995519, +STORE, 3797995520, 3798048767, +STORE, 3798048768, 3798179839, +STORE, 3798179840, 3798188031, +STORE, 3798188032, 3798192127, +STORE, 3798290432, 3798302719, +STORE, 3798302720, 3798323199, +STORE, 3798323200, 3798327295, +STORE, 3798327296, 3798331391, +STORE, 3798429696, 3798433791, +STORE, 3798433792, 3798552575, +STORE, 3798552576, 3798556671, +STORE, 3798556672, 3798568959, +STORE, 3798568960, 3798573055, +STORE, 3798573056, 3798581247, +STORE, 3798618112, 3798749183, +STORE, 3798749184, 3798855679, +STORE, 3798855680, 3798966271, +STORE, 3798966272, 3798982655, +STORE, 3798982656, 3798986751, +STORE, 3799101440, 3799171071, +STORE, 3799171072, 3799240703, +STORE, 3799240704, 3799248895, +STORE, 3799248896, 3799252991, +STORE, 3799326720, 3799650303, +STORE, 3799650304, 3800629247, +STORE, 3800629248, 3800641535, +STORE, 3800641536, 3800645631, +STORE, 3800645632, 3800649727, +STORE, 3800649728, 3800903679, +STORE, 3800903680, 3800936447, +STORE, 3800936448, 3800969215, +STORE, 3800969216, 3800981503, +STORE, 3800981504, 3800985599, +STORE, 3801001984, 3801133055, +STORE, 3801133056, 3801202687, +STORE, 3801202688, 3801591807, +STORE, 3801591808, 3801599999, +STORE, 3801600000, 3801604095, +STORE, 3801604096, 3801608191, +STORE, 3801608192, 3801739263, +STORE, 3801739264, 3801755647, +STORE, 3801755648, 3801796607, +STORE, 3801796608, 3801804799, +STORE, 3801804800, 3801808895, +STORE, 3801878528, 3801944063, +STORE, 3801944064, 3802116095, +STORE, 3802116096, 3802124287, +STORE, 3802124288, 3802128383, +STORE, 3802136576, 3803447295, +STORE, 3803492352, 3803553791, +STORE, 3803553792, 3804233727, +STORE, 3804233728, 3806068735, +STORE, 3806121984, 3806253055, +STORE, 3806253056, 3806674943, +STORE, 3806674944, 3807117311, +STORE, 3807117312, 3807379455, +STORE, 3807379456, 3807432703, +STORE, 3807432704, 3807563775, +STORE, 3807563776, 3809202175, +STORE, 3809202176, 3810250751, +STORE, 3810250752, 3827027967, +STORE, 3827027968, 3829125119, +STORE, 3829125120, 3837513727, +STORE, 3837513728, 3839610879, +STORE, 3839610880, 3847999487, +STORE, 3847999488, 3856392191, +STORE, 3856392192, 3864784895, +STORE, 3864784896, 3868983295, +STORE, 3868983296, 3885760511, +STORE, 3885760512, 3886809087, +STORE, 3886809088, 3887857663, +STORE, 3887857664, 3888119807, +STORE, 3888144384, 3888148479, +STORE, 3888148480, 3888218111, +STORE, 3888218112, 3888222207, +STORE, 3888222208, 3888353279, +STORE, 3888353280, 3889172479, +STORE, 3889172480, 3892314111, +STORE, 3892314112, 3892576255, +STORE, 3892588544, 3892637695, +STORE, 3892637696, 3892686847, +STORE, 3892686848, 3892744191, +STORE, 3892748288, 3892785151, +STORE, 3892785152, 3895459839, +STORE, 3895459840, 3895721983, +STORE, 3895738368, 3895885823, +STORE, 3895885824, 3897081855, +STORE, 3897081856, 3906482175, +STORE, 3906482176, 3916144639, +STORE, 3916144640, 3925766143, +STORE, 3925766144, 3926974463, +STORE, 3926974464, 3928367103, +STORE, 3928367104, 3928911871, +STORE, 3928911872, 3933995007, +STORE, 3933995008, 3935830015, +STORE, 3935830016, 3935846399, +STORE, 3935879168, 3936010239, +STORE, 3936010240, 3936026623, +STORE, 3936026624, 3936034815, +STORE, 3936034816, 3936051199, +STORE, 3936051200, 3936055295, +STORE, 3936071680, 3936137215, +STORE, 3936137216, 3936202751, +STORE, 3936202752, 3936219135, +STORE, 3936235520, 3936251903, +STORE, 3936268288, 3936276479, +STORE, 3936276480, 3936284671, +STORE, 3936284672, 3936288767, +STORE, 3936288768, 3936292863, +STORE, 3936296960, 3936354303, +STORE, 3936354304, 3936616447, +STORE, 3936628736, 3936669695, +STORE, 3936669696, 3936747519, +STORE, 3936747520, 3936870399, +STORE, 3936870400, 3936874495, +STORE, 3936874496, 3936878591, +STORE, 3936882688, 3936903167, +STORE, 3936911360, 3936948223, +STORE, 3936948224, 3936964607, +STORE, 3936964608, 3937103871, +STORE, 3937103872, 3937107967, +STORE, 3937132544, 3937161215, +STORE, 3937189888, 3937255423, +STORE, 3937255424, 3938512895, +STORE, 3938512896, 3945435135, +STORE, 3945435136, 3945476095, +STORE, 3945476096, 3945484287, +STORE, 3945484288, 3945496575, +STORE, 3945500672, 3945541631, +STORE, 3945558016, 3945566207, +STORE, 3945566208, 3945594879, +STORE, 3945594880, 3945598975, +STORE, 3945598976, 3945603071, +STORE, 3945611264, 3945742335, +STORE, 3945742336, 3945844735, +STORE, 3945844736, 3945848831, +STORE, 3945848832, 3945861119, +STORE, 3945861120, 3945865215, +STORE, 3945869312, 3945897983, +STORE, 3945897984, 3946303487, +STORE, 3946303488, 3946397695, +STORE, 3946397696, 3946569727, +STORE, 3946569728, 3946573823, +STORE, 3946573824, 3946594303, +STORE, 3946594304, 3946663935, +STORE, 3946663936, 3946708991, +STORE, 3946708992, 3946823679, +STORE, 3946823680, 3946827775, +STORE, 3946827776, 3946831871, +STORE, 3946831872, 3946860543, +STORE, 3946893312, 3946897407, +STORE, 3946897408, 3946905599, +STORE, 3946905600, 3946909695, +STORE, 3946909696, 3946913791, +STORE, 3946913792, 3946930175, +STORE, 3946930176, 3946967039, +STORE, 3946967040, 3947102207, +STORE, 3947102208, 3948412927, +STORE, 3948441600, 3948556287, +STORE, 3948556288, 3948576767, +STORE, 3948576768, 3948597247, +STORE, 3948597248, 3948605439, +STORE, 3948605440, 3948609535, +STORE, 3948609536, 3948654591, +STORE, 3948654592, 3948781567, +STORE, 3948781568, 3948822527, +STORE, 3948822528, 3948904447, +STORE, 3948904448, 3948908543, +STORE, 3948908544, 3948912639, +STORE, 3948945408, 3949043711, +STORE, 3949043712, 3949174783, +STORE, 3949174784, 3949191167, +STORE, 3949191168, 3949195263, +STORE, 3949207552, 3949252607, +STORE, 3949252608, 3949256703, +STORE, 3949256704, 3949363199, +STORE, 3949363200, 3949367295, +STORE, 3949367296, 3949379583, +STORE, 3949379584, 3949383679, +STORE, 3949383680, 3949400063, +STORE, 3949400064, 3949404159, +STORE, 3949416448, 3949481983, +STORE, 3949481984, 3949486079, +STORE, 3949486080, 3949592575, +STORE, 3949592576, 3949596671, +STORE, 3949596672, 3949621247, +STORE, 3949621248, 3949662207, +STORE, 3949662208, 3949666303, +STORE, 3949694976, 3949727743, +STORE, 3949727744, 3949731839, +STORE, 3949731840, 3949838335, +STORE, 3949838336, 3949842431, +STORE, 3949842432, 3949846527, +STORE, 3949846528, 3949854719, +STORE, 3949854720, 3949858815, +STORE, 3949858816, 3949862911, +STORE, 3949867008, 3949891583, +STORE, 3949891584, 3949928447, +STORE, 3949928448, 3949993983, +STORE, 3949993984, 3950043135, +STORE, 3950043136, 3950059519, +STORE, 3950059520, 3950096383, +STORE, 3950096384, 3950100479, +STORE, 3950100480, 3950104575, +STORE, 3950104576, 3950157823, +STORE, 3950157824, 3950292991, +STORE, 3950292992, 3950346239, +STORE, 3950346240, 3950477311, +STORE, 3950477312, 3950485503, +STORE, 3950485504, 3950489599, +STORE, 3950493696, 3950510079, +STORE, 3950510080, 3950661631, +STORE, 3950661632, 3951005695, +STORE, 3951005696, 3951026175, +STORE, 3951026176, 3951030271, +STORE, 3951030272, 3951054847, +STORE, 3951054848, 3951116287, +STORE, 3951116288, 3951144959, +STORE, 3951144960, 3951149055, +STORE, 3951149056, 3951194111, +STORE, 3951194112, 3951202303, +STORE, 3951202304, 3951206399, +STORE, 3951210496, 3951226879, +STORE, 3951226880, 3951329279, +STORE, 3951329280, 3951366143, +STORE, 3951366144, 3951411199, +STORE, 3951411200, 3951415295, +STORE, 3951415296, 3951419391, +STORE, 3951419392, 3951452159, +STORE, 3951452160, 3951566847, +STORE, 3951566848, 3951812607, +STORE, 3951812608, 3952173055, +STORE, 3952173056, 3952214015, +STORE, 3952214016, 3952218111, +STORE, 3952222208, 3952250879, +STORE, 3952250880, 3952369663, +STORE, 3952369664, 3952488447, +STORE, 3952488448, 3952627711, +STORE, 3952627712, 3952635903, +STORE, 3952635904, 3952639999, +STORE, 3952652288, 3952668671, +STORE, 3952668672, 3953000447, +STORE, 3953000448, 3953004543, +STORE, 3953004544, 3953008639, +STORE, 3953008640, 3953012735, +STORE, 3953012736, 3953037311, +STORE, 3953037312, 3953151999, +STORE, 3953152000, 3953291263, +STORE, 3953291264, 3953324031, +STORE, 3953324032, 3953364991, +STORE, 3953364992, 3953373183, +STORE, 3953373184, 3953377279, +STORE, 3953381376, 3953410047, +STORE, 3953410048, 3953491967, +STORE, 3953491968, 3953643519, +STORE, 3953643520, 3953651711, +STORE, 3953651712, 3953655807, +STORE, 3953659904, 3953766399, +STORE, 3953766400, 3953774591, +STORE, 3953774592, 3953786879, +STORE, 3953786880, 3953790975, +STORE, 3953790976, 3953823743, +STORE, 3953823744, 3953963007, +STORE, 3953963008, 3954024447, +STORE, 3954024448, 3954118655, +STORE, 3954118656, 3954122751, +STORE, 3954122752, 3954126847, +STORE, 3954130944, 3954184191, +STORE, 3954184192, 3954294783, +STORE, 3954294784, 3954323455, +STORE, 3954323456, 3954393087, +STORE, 3954393088, 3954397183, +STORE, 3954397184, 3954401279, +STORE, 3954401280, 3954405375, +STORE, 3954409472, 3954528255, +STORE, 3954528256, 3954737151, +STORE, 3954737152, 3955052543, +STORE, 3955052544, 3955060735, +STORE, 3955060736, 3955064831, +STORE, 3955068928, 3955105791, +STORE, 3955105792, 3955167231, +STORE, 3955167232, 3955277823, +STORE, 3955277824, 3955310591, +STORE, 3955310592, 3955351551, +STORE, 3955351552, 3955359743, +STORE, 3955359744, 3955363839, +STORE, 3955363840, 3955392511, +STORE, 3955392512, 3955453951, +STORE, 3955453952, 3955601407, +STORE, 3955601408, 3955777535, +STORE, 3955777536, 3955982335, +STORE, 3955982336, 3956011007, +STORE, 3956011008, 3956015103, +STORE, 3956023296, 3956039679, +STORE, 3956039680, 3956125695, +STORE, 3956125696, 3956129791, +STORE, 3956129792, 3956133887, +STORE, 3956133888, 3956137983, +STORE, 3956142080, 3956449279, +STORE, 3956449280, 3956543487, +STORE, 3956543488, 3956719615, +STORE, 3956719616, 3956731903, +STORE, 3956731904, 3956735999, +STORE, 3956744192, 3956793343, +STORE, 3956793344, 3956887551, +STORE, 3956887552, 3956953087, +STORE, 3956953088, 3957035007, +STORE, 3957035008, 3957039103, +STORE, 3957039104, 3957047295, +STORE, 3957047296, 3957071871, +STORE, 3957071872, 3957231615, +STORE, 3957231616, 3957563391, +STORE, 3957563392, 3957579775, +STORE, 3957579776, 3957583871, +STORE, 3957592064, 3957608447, +STORE, 3957608448, 3957878783, +STORE, 3957878784, 3958591487, +STORE, 3958591488, 3958599679, +STORE, 3958599680, 3958607871, +STORE, 3958607872, 3958620159, +STORE, 3958620160, 3958624255, +STORE, 3958624256, 3963199487, +STORE, 3963199488, 3963285503, +STORE, 3963285504, 3963371519, +STORE, 3963371520, 3963428863, +STORE, 3963428864, 3963555839, +STORE, 3963555840, 3963559935, +STORE, 3963559936, 3963564031, +STORE, 3963568128, 3963596799, +STORE, 3963596800, 3963682815, +STORE, 3963682816, 3963695103, +STORE, 3963695104, 3963711487, +STORE, 3963711488, 3963715583, +STORE, 3963719680, 3963752447, +STORE, 3963752448, 3963846655, +STORE, 3963846656, 3963932671, +STORE, 3963932672, 3964444671, +STORE, 3964444672, 3964448767, +STORE, 3964448768, 3965808639, +STORE, 3965808640, 3965845503, +STORE, 3965845504, 3965849599, +STORE, 3965853696, 3965935615, +STORE, 3965935616, 3966017535, +STORE, 3966017536, 3966103551, +STORE, 3966103552, 3966685183, +STORE, 3966685184, 3967705087, +STORE, 3967705088, 3967758335, +STORE, 3967758336, 3967762431, +STORE, 3967762432, 3967770623, +STORE, 3967770624, 3967799295, +STORE, 3967799296, 3967848447, +STORE, 3967848448, 3967868927, +STORE, 3967868928, 3967901695, +STORE, 3967901696, 3967905791, +STORE, 3967905792, 3967909887, +STORE, 3967909888, 3967995903, +STORE, 3967995904, 3968077823, +STORE, 3968077824, 3968159743, +STORE, 3968159744, 3968167935, +STORE, 3968167936, 3968172031, +STORE, 3968172032, 3968192511, +STORE, 3968192512, 3968196607, +STORE, 3968196608, 3968200703, +STORE, 3968208896, 3968516095, +STORE, 3968516096, 3968528383, +STORE, 3968528384, 3968552959, +STORE, 3968552960, 3968557055, +STORE, 3968561152, 3968593919, +STORE, 3968593920, 3968626687, +STORE, 3968626688, 3971153919, +STORE, 3971153920, 3973754879, +STORE, 3973754880, 3973804031, +STORE, 3973804032, 3973820415, +STORE, 3973820416, 3973832703, +STORE, 3973840896, 3973873663, +STORE, 3973873664, 3973967871, +STORE, 3973967872, 3973976063, +STORE, 3973976064, 3973984255, +STORE, 3973984256, 3973988351, +STORE, 3973988352, 3973992447, +STORE, 3973996544, 3974008831, +STORE, 3974008832, 3974045695, +STORE, 3974045696, 3974139903, +STORE, 3974139904, 3974254591, +STORE, 3974254592, 3974275071, +STORE, 3974275072, 3974291455, +STORE, 3974291456, 3974295551, +STORE, 3974295552, 3974373375, +STORE, 3974373376, 3974524927, +STORE, 3974524928, 3974529023, +STORE, 3974529024, 3974537215, +STORE, 3974537216, 3974541311, +STORE, 3974541312, 3974545407, +STORE, 3974545408, 3974627327, +STORE, 3974627328, 3974680575, +STORE, 3974680576, 3974811647, +STORE, 3974811648, 3974819839, +STORE, 3974819840, 3974823935, +STORE, 3974832128, 3974918143, +STORE, 3974918144, 3974963199, +STORE, 3974963200, 3975077887, +STORE, 3975077888, 3975090175, +STORE, 3975090176, 3975094271, +STORE, 3975094272, 3975102463, +STORE, 3975102464, 3975114751, +STORE, 3975114752, 3975266303, +STORE, 3975266304, 3975274495, +STORE, 3975274496, 3975286783, +STORE, 3975286784, 3975290879, +STORE, 3975290880, 3975299071, +STORE, 3975299072, 3975315455, +STORE, 3975315456, 3975430143, +STORE, 3975430144, 3975536639, +STORE, 3975536640, 3975651327, +STORE, 3975651328, 3975655423, +STORE, 3975655424, 3975659519, +STORE, 3975659520, 3975770111, +STORE, 3975770112, 3975778303, +STORE, 3975778304, 3975790591, +STORE, 3975790592, 3975794687, +STORE, 3975794688, 3975798783, +STORE, 3975798784, 3975831551, +STORE, 3975831552, 3975872511, +STORE, 3975872512, 3975987199, +STORE, 3975987200, 3976134655, +STORE, 3976134656, 3977175039, +STORE, 3977175040, 3977183231, +STORE, 3977183232, 3977191423, +STORE, 3977191424, 3977195519, +STORE, 3977199616, 3977248767, +STORE, 3977248768, 3977539583, +STORE, 3977539584, 3977965567, +STORE, 3977965568, 3977981951, +STORE, 3977981952, 3977986047, +STORE, 3977986048, 3977994239, +STORE, 3977994240, 3978002431, +STORE, 3978002432, 3978084351, +STORE, 3978084352, 3978125311, +STORE, 3978125312, 3978174463, +STORE, 3978174464, 3978178559, +STORE, 3978178560, 3978182655, +STORE, 3978182656, 3978207231, +STORE, 3978207232, 3978297343, +STORE, 3978297344, 3978301439, +STORE, 3978301440, 3978305535, +STORE, 3978305536, 3978309631, +STORE, 3978309632, 3978317823, +STORE, 3978317824, 3978625023, +STORE, 3978625024, 3978657791, +STORE, 3978657792, 3978727423, +STORE, 3978727424, 3978735615, +STORE, 3978735616, 3978739711, +STORE, 3978739712, 3978760191, +STORE, 3978760192, 3978842111, +STORE, 3978842112, 3978850303, +STORE, 3978850304, 3978858495, +STORE, 3978858496, 3978862591, +STORE, 3978862592, 3978895359, +STORE, 3978895360, 3979014143, +STORE, 3979014144, 3979132927, +STORE, 3979132928, 3979288575, +STORE, 3979288576, 3979481087, +STORE, 3979481088, 3979489279, +STORE, 3979489280, 3979493375, +STORE, 3979497472, 3979583487, +STORE, 3979583488, 3979673599, +STORE, 3979673600, 3979718655, +STORE, 3979718656, 3979829247, +STORE, 3979829248, 3979841535, +STORE, 3979841536, 3979882495, +STORE, 3979882496, 3979964415, +STORE, 3979964416, 3980013567, +STORE, 3980013568, 3980148735, +STORE, 3980148736, 3980152831, +STORE, 3980152832, 3980320767, +STORE, 3980320768, 3980337151, +STORE, 3980337152, 3980341247, +STORE, 3980345344, 3980365823, +STORE, 3980365824, 3980423167, +STORE, 3980423168, 3980460031, +STORE, 3980460032, 3980500991, +STORE, 3980500992, 3980509183, +STORE, 3980509184, 3980513279, +STORE, 3980513280, 3980546047, +STORE, 3980546048, 3980660735, +STORE, 3980660736, 3980951551, +STORE, 3980951552, 3981500415, +STORE, 3981500416, 3981529087, +STORE, 3981529088, 3981533183, +STORE, 3981537280, 3981549567, +STORE, 3981549568, 3981598719, +STORE, 3981598720, 3981717503, +STORE, 3981717504, 3982127103, +STORE, 3982127104, 3982675967, +STORE, 3982675968, 3982733311, +STORE, 3982733312, 3982737407, +STORE, 3982741504, 3982860287, +STORE, 3982860288, 3982905343, +STORE, 3982905344, 3982966783, +STORE, 3982966784, 3982974975, +STORE, 3982974976, 3982979071, +STORE, 3982979072, 3983032319, +STORE, 3983032320, 3983085567, +STORE, 3983085568, 3983208447, +STORE, 3983208448, 3983212543, +STORE, 3983212544, 3983220735, +STORE, 3983220736, 3983224831, +STORE, 3983224832, 3983237119, +STORE, 3983237120, 3983351807, +STORE, 3983351808, 3983376383, +STORE, 3983376384, 3983392767, +STORE, 3983392768, 3983396863, +STORE, 3983396864, 3983400959, +STORE, 3983400960, 3983417343, +STORE, 3983417344, 3983753215, +STORE, 3983753216, 3983757311, +STORE, 3983757312, 3983761407, +STORE, 3983761408, 3983765503, +STORE, 3983765504, 3983769599, +STORE, 3983769600, 3983880191, +STORE, 3983880192, 3983892479, +STORE, 3983892480, 3983900671, +STORE, 3983900672, 3983904767, +STORE, 3983904768, 3983908863, +STORE, 3983908864, 3983941631, +STORE, 3983941632, 3983990783, +STORE, 3983990784, 3984097279, +STORE, 3984097280, 3984105471, +STORE, 3984105472, 3984117759, +STORE, 3984117760, 3984121855, +STORE, 3984121856, 3984125951, +STORE, 3984125952, 3984134143, +STORE, 3984134144, 3984150527, +STORE, 3984150528, 3984416767, +STORE, 3984416768, 3984470015, +STORE, 3984470016, 3984564223, +STORE, 3984564224, 3984568319, +STORE, 3984572416, 3984629759, +STORE, 3984629760, 3984805887, +STORE, 3984805888, 3985096703, +STORE, 3985096704, 3985104895, +STORE, 3985104896, 3985108991, +STORE, 3985113088, 3986862079, +STORE, 3986862080, 3993640959, +STORE, 3993640960, 3993739263, +STORE, 3993739264, 3993743359, +STORE, 3993743360, 3993759743, +STORE, 3993759744, 3993780223, +STORE, 3993780224, 3993784319, +STORE, 3993784320, 3993792511, +STORE, 3993792512, 3993796607, +STORE, 3993796608, 3993800703, +STORE, 3993804800, 3994214399, +STORE, 3994214400, 3994218495, +STORE, 3994218496, 3994222591, +STORE, 3994222592, 3994226687, +STORE, 3994230784, 3994243071, +STORE, 3994243072, 3994255359, +STORE, 3994255360, 3994304511, +STORE, 3994304512, 3994386431, +STORE, 3994386432, 3994509311, +STORE, 3994509312, 3994521599, +STORE, 3994521600, 3994525695, +STORE, 3994529792, 3994542079, +STORE, 3994542080, 3994660863, +STORE, 3994660864, 3994705919, +STORE, 3994705920, 3994796031, +STORE, 3994796032, 3994800127, +STORE, 3994800128, 3994804223, +STORE, 3994804224, 3994812415, +STORE, 3994812416, 3994845183, +STORE, 3994845184, 3994898431, +STORE, 3994898432, 3994902527, +STORE, 3994902528, 3994906623, +STORE, 3994910720, 3994931199, +STORE, 3994931200, 3995181055, +STORE, 3995181056, 3995222015, +STORE, 3995222016, 3995275263, +STORE, 3995275264, 3995279359, +STORE, 3995279360, 3995283455, +STORE, 3995283456, 3995291647, +STORE, 3995291648, 3995324415, +STORE, 3995324416, 3995451391, +STORE, 3995451392, 3995697151, +STORE, 3995697152, 3996078079, +STORE, 3996078080, 3996086271, +STORE, 3996086272, 3996090367, +STORE, 3996094464, 3996119039, +STORE, 3996119040, 3996200959, +STORE, 3996200960, 3996229631, +STORE, 3996229632, 3996233727, +STORE, 3996233728, 3996282879, +STORE, 3996282880, 3996291071, +STORE, 3996291072, 3996295167, +STORE, 3996299264, 3996311551, +STORE, 3996311552, 3996430335, +STORE, 3996430336, 3996467199, +STORE, 3996467200, 3996504063, +STORE, 3996504064, 3996512255, +STORE, 3996512256, 3996516351, +STORE, 3996516352, 3996540927, +STORE, 3996540928, 3996671999, +STORE, 3996672000, 3996676095, +STORE, 3996676096, 3996684287, +STORE, 3996684288, 3996688383, +STORE, 3996688384, 3996692479, +STORE, 3996692480, 3996717055, +STORE, 3996717056, 3997048831, +STORE, 3997048832, 3997057023, +STORE, 3997057024, 3997073407, +STORE, 3997073408, 3997077503, +STORE, 3997077504, 3997081599, +STORE, 3997081600, 3997097983, +STORE, 3997097984, 3997179903, +STORE, 3997179904, 3997356031, +STORE, 3997356032, 3997650943, +STORE, 3997650944, 3997675519, +STORE, 3997675520, 3997679615, +STORE, 3997683712, 3997700095, +STORE, 3997700096, 3997745151, +STORE, 3997745152, 3997802495, +STORE, 3997802496, 3997810687, +STORE, 3997810688, 3997814783, +STORE, 3997814784, 3998064639, +STORE, 3998064640, 3998081023, +STORE, 3998081024, 3998085119, +STORE, 3998085120, 3998130175, +STORE, 3998130176, 3998134271, +STORE, 3998134272, 3998142463, +STORE, 3998142464, 3998179327, +STORE, 3998179328, 3998212095, +STORE, 3998212096, 3998326783, +STORE, 3998326784, 3998351359, +STORE, 3998351360, 3998392319, +STORE, 3998392320, 3998396415, +STORE, 3998396416, 3998400511, +STORE, 3998400512, 3998433279, +STORE, 3998433280, 3998466047, +STORE, 3998466048, 3998613503, +STORE, 3998613504, 3998666751, +STORE, 3998666752, 3998724095, +STORE, 3998724096, 3998732287, +STORE, 3998732288, 3998736383, +STORE, 3998736384, 3998760959, +STORE, 3998760960, 3998777343, +STORE, 3998777344, 3998822399, +STORE, 3998822400, 3998826495, +STORE, 3998826496, 3998830591, +STORE, 3998830592, 3998863359, +STORE, 3998863360, 3998900223, +STORE, 3998900224, 3999043583, +STORE, 3999043584, 3999121407, +STORE, 3999121408, 3999215615, +STORE, 3999215616, 3999223807, +STORE, 3999223808, 3999227903, +STORE, 3999227904, 3999236095, +STORE, 3999236096, 3999268863, +STORE, 3999268864, 3999301631, +STORE, 3999301632, 3999354879, +STORE, 3999354880, 3999428607, +STORE, 3999428608, 3999436799, +STORE, 3999436800, 3999440895, +STORE, 3999444992, 3999461375, +STORE, 3999461376, 3999584255, +STORE, 3999584256, 3999760383, +STORE, 3999760384, 4000219135, +STORE, 4000219136, 4000235519, +STORE, 4000235520, 4000251903, +STORE, 4000251904, 4000501759, +STORE, 4000501760, 4000505855, +STORE, 4000505856, 4000509951, +STORE, 4000509952, 4000518143, +STORE, 4000518144, 4000522239, +STORE, 4000522240, 4000587775, +STORE, 4000587776, 4000645119, +STORE, 4000645120, 4000813055, +STORE, 4000813056, 4000817151, +STORE, 4000821248, 4000837631, +STORE, 4000837632, 4000870399, +STORE, 4000870400, 4000874495, +STORE, 4000874496, 4000878591, +STORE, 4000878592, 4000882687, +STORE, 4000882688, 4000886783, +STORE, 4000886784, 4000890879, +STORE, 4000890880, 4000907263, +STORE, 4000907264, 4001214463, +STORE, 4001214464, 4001558527, +STORE, 4001558528, 4002484223, +STORE, 4002484224, 4002525183, +STORE, 4002525184, 4002529279, +STORE, 4002529280, 4002533375, +STORE, 4002533376, 4002537471, +STORE, 4002537472, 4002660351, +STORE, 4002660352, 4002779135, +STORE, 4002779136, 4002791423, +STORE, 4002791424, 4002799615, +STORE, 4002799616, 4002807807, +STORE, 4002807808, 4002811903, +STORE, 4002811904, 4002828287, +STORE, 4002828288, 4002910207, +STORE, 4002910208, 4003028991, +STORE, 4003028992, 4003037183, +STORE, 4003037184, 4003045375, +STORE, 4003045376, 4003049471, +STORE, 4003049472, 4003053567, +STORE, 4003053568, 4003057663, +STORE, 4003057664, 4003065855, +STORE, 4003065856, 4003135487, +STORE, 4003135488, 4003446783, +STORE, 4003446784, 4003450879, +STORE, 4003450880, 4003454975, +STORE, 4003454976, 4003459071, +STORE, 4003459072, 4003463167, +STORE, 4003463168, 4003495935, +STORE, 4003495936, 4003569663, +STORE, 4003569664, 4003573759, +STORE, 4003573760, 4003704831, +STORE, 4003704832, 4003708927, +STORE, 4003708928, 4003713023, +STORE, 4003713024, 4003737599, +STORE, 4003737600, 4003770367, +STORE, 4003770368, 4003876863, +STORE, 4003876864, 4003880959, +STORE, 4003880960, 4003885055, +STORE, 4003885056, 4003889151, +STORE, 4003889152, 4003893247, +STORE, 4003893248, 4003897343, +STORE, 4003897344, 4003962879, +STORE, 4003962880, 4004069375, +STORE, 4004069376, 4004093951, +STORE, 4004093952, 4004118527, +STORE, 4004118528, 4004122623, +STORE, 4004122624, 4004126719, +STORE, 4004126720, 4004155391, +STORE, 4004155392, 4004286463, +STORE, 4004286464, 4004384767, +STORE, 4004384768, 4004388863, +STORE, 4004388864, 4004646911, +STORE, 4004646912, 4004655103, +STORE, 4004655104, 4004659199, +STORE, 4004659200, 4004667391, +STORE, 4004667392, 4004683775, +STORE, 4004683776, 4004814847, +STORE, 4004814848, 4004818943, +STORE, 4004818944, 4004823039, +STORE, 4004823040, 4004827135, +STORE, 4004827136, 4004835327, +STORE, 4004835328, 4004954111, +STORE, 4004954112, 4005085183, +STORE, 4005085184, 4005306367, +STORE, 4005306368, 4005765119, +STORE, 4005765120, 4005789695, +STORE, 4005789696, 4005793791, +STORE, 4005793792, 4005801983, +STORE, 4005801984, 4005920767, +STORE, 4005920768, 4005945343, +STORE, 4005945344, 4005949439, +STORE, 4005949440, 4005986303, +STORE, 4005986304, 4005990399, +STORE, 4005990400, 4005994495, +STORE, 4005994496, 4006002687, +STORE, 4006002688, 4006109183, +STORE, 4006109184, 4006117375, +STORE, 4006117376, 4006121471, +STORE, 4006121472, 4006133759, +STORE, 4006133760, 4006137855, +STORE, 4006137856, 4006141951, +STORE, 4006141952, 4006150143, +STORE, 4006150144, 4006391807, +STORE, 4006391808, 4006445055, +STORE, 4006445056, 4006563839, +STORE, 4006563840, 4006572031, +STORE, 4006572032, 4006576127, +STORE, 4006576128, 4006584319, +STORE, 4006584320, 4006694911, +STORE, 4006694912, 4006739967, +STORE, 4006739968, 4006776831, +STORE, 4006776832, 4006785023, +STORE, 4006785024, 4006789119, +STORE, 4006789120, 4006797311, +STORE, 4006797312, 4006813695, +STORE, 4006813696, 4006846463, +STORE, 4006846464, 4006977535, +STORE, 4006977536, 4007006207, +STORE, 4007006208, 4007010303, +STORE, 4007010304, 4007067647, +STORE, 4007067648, 4007075839, +STORE, 4007075840, 4007084031, +STORE, 4007084032, 4007100415, +STORE, 4007100416, 4007116799, +STORE, 4007116800, 4007133183, +STORE, 4007133184, 4007153663, +STORE, 4007153664, 4007178239, +STORE, 4007178240, 4007202815, +STORE, 4007202816, 4007206911, +STORE, 4007206912, 4007272447, +STORE, 4007272448, 4007276543, +STORE, 4007276544, 4007280639, +STORE, 4007280640, 4007284735, +STORE, 4007284736, 4007292927, +STORE, 4007292928, 4007423999, +STORE, 4007424000, 4007448575, +STORE, 4007448576, 4007452671, +STORE, 4007452672, 4007505919, +STORE, 4007505920, 4007510015, +STORE, 4007510016, 4007514111, +STORE, 4007514112, 4007645183, +STORE, 4007645184, 4007776255, +STORE, 4007776256, 4007780351, +STORE, 4007780352, 4007784447, +STORE, 4007784448, 4007788543, +STORE, 4007788544, 4007809023, +STORE, 4007809024, 4007829503, +STORE, 4007829504, 4007960575, +STORE, 4007960576, 4008091647, +STORE, 4008091648, 4008296447, +STORE, 4008296448, 4008890367, +STORE, 4008890368, 4008898559, +STORE, 4008898560, 4008902655, +STORE, 4008902656, 4008996863, +STORE, 4008996864, 4009041919, +STORE, 4009041920, 4009082879, +STORE, 4009082880, 4009091071, +STORE, 4009091072, 4009107455, +STORE, 4009107456, 4009349119, +STORE, 4009349120, 4009373695, +STORE, 4009373696, 4009414655, +STORE, 4009414656, 4009422847, +STORE, 4009422848, 4009426943, +STORE, 4009426944, 4009447423, +STORE, 4009447424, 4009471999, +STORE, 4009472000, 4009512959, +STORE, 4009512960, 4009594879, +STORE, 4009594880, 4009598975, +STORE, 4009598976, 4009697279, +STORE, 4009697280, 4009713663, +STORE, 4009713664, 4009717759, +STORE, 4009717760, 4009721855, +STORE, 4009721856, 4009730047, +STORE, 4009730048, 4009861119, +STORE, 4009861120, 4009951231, +STORE, 4009951232, 4010131455, +STORE, 4010131456, 4010135551, +STORE, 4010135552, 4010139647, +STORE, 4010139648, 4010143743, +STORE, 4010143744, 4010164223, +STORE, 4010164224, 4010295295, +STORE, 4010295296, 4010299391, +STORE, 4010299392, 4010491903, +STORE, 4010491904, 4010495999, +STORE, 4010496000, 4010668031, +STORE, 4010668032, 4011028479, +STORE, 4011028480, 4011053055, +STORE, 4011053056, 4011057151, +STORE, 4011057152, 4011118591, +STORE, 4011118592, 4011126783, +STORE, 4011126784, 4011130879, +STORE, 4011130880, 4011143167, +STORE, 4011143168, 4011147263, +STORE, 4011147264, 4011167743, +STORE, 4011167744, 4011171839, +STORE, 4011171840, 4011360255, +STORE, 4011360256, 4011364351, +STORE, 4011364352, 4011626495, +STORE, 4011626496, 4012216319, +STORE, 4012216320, 4012228607, +STORE, 4012228608, 4012232703, +STORE, 4012232704, 4012236799, +STORE, 4012236800, 4012240895, +STORE, 4012240896, 4012261375, +STORE, 4012261376, 4012392447, +STORE, 4012392448, 4012466175, +STORE, 4012466176, 4012597247, +STORE, 4012597248, 4012601343, +STORE, 4012601344, 4012605439, +STORE, 4012605440, 4012609535, +STORE, 4012609536, 4012679167, +STORE, 4012679168, 4013563903, +STORE, 4013563904, 4015366143, +STORE, 4015366144, 4015411199, +STORE, 4015411200, 4015415295, +STORE, 4015415296, 4015419391, +STORE, 4015419392, 4015542271, +STORE, 4015542272, 4015550463, +STORE, 4015550464, 4015558655, +STORE, 4015558656, 4015562751, +STORE, 4015562752, 4015583231, +STORE, 4015583232, 4015587327, +STORE, 4015587328, 4015603711, +STORE, 4015665152, 4015669247, +STORE, 4015669248, 4015812607, +STORE, 4015812608, 4015816703, +STORE, 4015816704, 4016111615, +STORE, 4016111616, 4016467967, +STORE, 4016467968, 4016508927, +STORE, 4016508928, 4016517119, +STORE, 4016517120, 4016525311, +STORE, 4016525312, 4016586751, +STORE, 4016586752, 4016664575, +STORE, 4016664576, 4016697343, +STORE, 4016697344, 4016742399, +STORE, 4016742400, 4016746495, +STORE, 4016746496, 4016750591, +STORE, 4016750592, 4016758783, +STORE, 4016799744, 4016844799, +STORE, 4016844800, 4016902143, +STORE, 4016902144, 4016992255, +STORE, 4016992256, 4017000447, +STORE, 4017000448, 4017004543, +STORE, 4017004544, 4017008639, +STORE, 4017008640, 4017016831, +STORE, 4017016832, 4017020927, +STORE, 4017020928, 4017127423, +STORE, 4017127424, 4017131519, +STORE, 4017131520, 4017229823, +STORE, 4017229824, 4017422335, +STORE, 4017422336, 4017438719, +STORE, 4017438720, 4017442815, +STORE, 4017442816, 4017446911, +STORE, 4017446912, 4017455103, +STORE, 4017455104, 4017766399, +STORE, 4017766400, 4017909759, +STORE, 4017909760, 4018081791, +STORE, 4018081792, 4018089983, +STORE, 4018089984, 4018094079, +STORE, 4018094080, 4018098175, +STORE, 4018098176, 4018327551, +STORE, 4018327552, 4018331647, +STORE, 4018331648, 4018339839, +STORE, 4018339840, 4018348031, +STORE, 4018348032, 4018610175, +STORE, 4018610176, 4018626559, +STORE, 4018626560, 4018647039, +STORE, 4018647040, 4018651135, +STORE, 4018651136, 4018749439, +STORE, 4018749440, 4018761727, +STORE, 4018761728, 4018802687, +STORE, 4018802688, 4018806783, +STORE, 4018806784, 4018810879, +STORE, 4018810880, 4018814975, +STORE, 4018814976, 4018823167, +STORE, 4018823168, 4018954239, +STORE, 4018954240, 4019007487, +STORE, 4019007488, 4019068927, +STORE, 4019068928, 4019077119, +STORE, 4019077120, 4019081215, +STORE, 4019081216, 4019093503, +STORE, 4019093504, 4019208191, +STORE, 4019208192, 4019232767, +STORE, 4019232768, 4019265535, +STORE, 4019265536, 4019269631, +STORE, 4019269632, 4019277823, +STORE, 4019277824, 4019458047, +STORE, 4019458048, 4019519487, +STORE, 4019519488, 4019613695, +STORE, 4019613696, 4019621887, +STORE, 4019621888, 4019625983, +STORE, 4019625984, 4019630079, +STORE, 4019630080, 4019744767, +STORE, 4019744768, 4019822591, +STORE, 4019822592, 4019929087, +STORE, 4019929088, 4019941375, +STORE, 4019941376, 4019945471, +STORE, 4019945472, 4019961855, +STORE, 4019961856, 4019994623, +STORE, 4019994624, 4019998719, +STORE, 4019998720, 4020002815, +STORE, 4020002816, 4020006911, +STORE, 4020006912, 4020011007, +STORE, 4020011008, 4020256767, +STORE, 4020256768, 4020326399, +STORE, 4020326400, 4020457471, +STORE, 4020457472, 4020469759, +STORE, 4020469760, 4020473855, +STORE, 4020473856, 4020482047, +STORE, 4020482048, 4020711423, +STORE, 4020711424, 4020715519, +STORE, 4020715520, 4020719615, +STORE, 4020719616, 4020723711, +STORE, 4020723712, 4020805631, +STORE, 4020805632, 4021051391, +STORE, 4021051392, 4021460991, +STORE, 4021460992, 4021469183, +STORE, 4021469184, 4021473279, +STORE, 4021473280, 4021571583, +STORE, 4021571584, 4021633023, +STORE, 4021633024, 4021727231, +STORE, 4021727232, 4021735423, +STORE, 4021735424, 4021739519, +STORE, 4021739520, 4021747711, +STORE, 4021747712, 4021829631, +STORE, 4021829632, 4021866495, +STORE, 4021866496, 4021919743, +STORE, 4021919744, 4021927935, +STORE, 4021927936, 4021932031, +STORE, 4021932032, 4021944319, +STORE, 4021944320, 4022157311, +STORE, 4022157312, 4022161407, +STORE, 4022161408, 4022173695, +STORE, 4022173696, 4022177791, +STORE, 4022177792, 4022472703, +STORE, 4022472704, 4022509567, +STORE, 4022509568, 4022583295, +STORE, 4022583296, 4022587391, +STORE, 4022587392, 4022591487, +STORE, 4022591488, 4022607871, +STORE, 4022607872, 4022657023, +STORE, 4022657024, 4022722559, +STORE, 4022722560, 4022730751, +STORE, 4022730752, 4022734847, +STORE, 4022734848, 4022865919, +STORE, 4022865920, 4022943743, +STORE, 4022943744, 4023062527, +STORE, 4023062528, 4023074815, +STORE, 4023074816, 4023078911, +STORE, 4023078912, 4023128063, +STORE, 4023128064, 4023218175, +STORE, 4023218176, 4023361535, +STORE, 4023361536, 4023373823, +STORE, 4023373824, 4023377919, +STORE, 4023377920, 4023558143, +STORE, 4023558144, 4023631871, +STORE, 4023631872, 4023816191, +STORE, 4023816192, 4023820287, +STORE, 4023820288, 4023824383, +STORE, 4023824384, 4023832575, +STORE, 4023832576, 4024078335, +STORE, 4024078336, 4024197119, +STORE, 4024197120, 4024389631, +STORE, 4024389632, 4024406015, +STORE, 4024406016, 4024410111, +STORE, 4024410112, 4024422399, +STORE, 4024422400, 4024619007, +STORE, 4024619008, 4024639487, +STORE, 4024639488, 4024655871, +STORE, 4024655872, 4024664063, +STORE, 4024664064, 4024668159, +STORE, 4024668160, 4024676351, +STORE, 4024676352, 4024905727, +STORE, 4024905728, 4024909823, +STORE, 4024909824, 4024918015, +STORE, 4024918016, 4024922111, +STORE, 4024922112, 4024930303, +STORE, 4024930304, 4025110527, +STORE, 4025110528, 4025176063, +STORE, 4025176064, 4025208831, +STORE, 4025208832, 4025212927, +STORE, 4025212928, 4025217023, +STORE, 4025217024, 4025348095, +STORE, 4025348096, 4025372671, +STORE, 4025372672, 4025458687, +STORE, 4025458688, 4025466879, +STORE, 4025466880, 4025565183, +STORE, 4025565184, 4025757695, +STORE, 4025757696, 4026249215, +STORE, 4026249216, 4026261503, +STORE, 4026261504, 4026265599, +STORE, 4026265600, 4026269695, +STORE, 4026269696, 4026302463, +STORE, 4026302464, 4026306559, +STORE, 4026306560, 4026314751, +STORE, 4026314752, 4026318847, +STORE, 4026318848, 4026322943, +STORE, 4026322944, 4026327039, +STORE, 4026327040, 4026654719, +STORE, 4026654720, 4026671103, +STORE, 4026671104, 4026720255, +STORE, 4026720256, 4026724351, +STORE, 4026724352, 4026728447, +STORE, 4026728448, 4026732543, +STORE, 4026732544, 4026863615, +STORE, 4026863616, 4027027455, +STORE, 4027027456, 4027031551, +STORE, 4027031552, 4027514879, +STORE, 4027514880, 4027531263, +STORE, 4027531264, 4027535359, +STORE, 4027535360, 4027539455, +STORE, 4027539456, 4027785215, +STORE, 4027785216, 4027789311, +STORE, 4027789312, 4027793407, +STORE, 4027793408, 4027797503, +STORE, 4027797504, 4027863039, +STORE, 4027863040, 4027899903, +STORE, 4027899904, 4027949055, +STORE, 4027949056, 4027957247, +STORE, 4027957248, 4027961343, +STORE, 4027961344, 4027965439, +STORE, 4027965440, 4028194815, +STORE, 4028194816, 4028252159, +STORE, 4028252160, 4028338175, +STORE, 4028338176, 4028350463, +STORE, 4028350464, 4028354559, +STORE, 4028354560, 4028452863, +STORE, 4028452864, 4028489727, +STORE, 4028489728, 4028530687, +STORE, 4028530688, 4028538879, +STORE, 4028538880, 4028542975, +STORE, 4028542976, 4028551167, +STORE, 4028551168, 4028665855, +STORE, 4028665856, 4029349887, +STORE, 4029349888, 4030468095, +STORE, 4030468096, 4030513151, +STORE, 4030513152, 4030517247, +STORE, 4030517248, 4030525439, +STORE, 4030525440, 4030529535, +STORE, 4030529536, 4030758911, +STORE, 4030758912, 4030828543, +STORE, 4030828544, 4030943231, +STORE, 4030943232, 4030951423, +STORE, 4030951424, 4030955519, +STORE, 4030955520, 4030967807, +STORE, 4030967808, 4031131647, +STORE, 4031131648, 4031135743, +STORE, 4031135744, 4031139839, +STORE, 4031139840, 4031148031, +STORE, 4031148032, 4031152127, +STORE, 4031152128, 4031160319, +STORE, 4031160320, 4031504383, +STORE, 4031504384, 4031598591, +STORE, 4031598592, 4031754239, +STORE, 4031754240, 4031766527, +STORE, 4031766528, 4031770623, +STORE, 4031770624, 4031774719, +STORE, 4031774720, 4031782911, +STORE, 4031782912, 4031799295, +STORE, 4031799296, 4031856639, +STORE, 4031856640, 4031983615, +STORE, 4031983616, 4031987711, +STORE, 4031987712, 4031991807, +STORE, 4031991808, 4032270335, +STORE, 4032270336, 4032274431, +STORE, 4032274432, 4032282623, +STORE, 4032282624, 4032286719, +STORE, 4032286720, 4032290815, +STORE, 4032290816, 4032389119, +STORE, 4032389120, 4032397311, +STORE, 4032397312, 4032405503, +STORE, 4032405504, 4032413695, +STORE, 4032413696, 4032417791, +STORE, 4032417792, 4032565247, +STORE, 4032565248, 4032593919, +STORE, 4032593920, 4032737279, +STORE, 4032737280, 4032741375, +STORE, 4032741376, 4032745471, +STORE, 4032745472, 4032770047, +STORE, 4032770048, 4032933887, +STORE, 4032933888, 4032999423, +STORE, 4032999424, 4033032191, +STORE, 4033032192, 4033036287, +STORE, 4033036288, 4033040383, +STORE, 4033040384, 4033105919, +STORE, 4033105920, 4033396735, +STORE, 4033396736, 4033822719, +STORE, 4033822720, 4033839103, +STORE, 4033839104, 4033843199, +STORE, 4033843200, 4033851391, +STORE, 4033851392, 4033863679, +STORE, 4033863680, 4033880063, +STORE, 4033880064, 4033933311, +STORE, 4033933312, 4034023423, +STORE, 4034023424, 4034031615, +STORE, 4034031616, 4034035711, +STORE, 4034035712, 4034043903, +STORE, 4034043904, 4034142207, +STORE, 4034142208, 4034191359, +STORE, 4034191360, 4034260991, +STORE, 4034260992, 4034269183, +STORE, 4034269184, 4034273279, +STORE, 4034273280, 4034281471, +STORE, 4034281472, 4034412543, +STORE, 4034412544, 4034445311, +STORE, 4034445312, 4034490367, +STORE, 4034490368, 4034494463, +STORE, 4034494464, 4034498559, +STORE, 4034498560, 4034662399, +STORE, 4034662400, 4034666495, +STORE, 4034666496, 4034670591, +STORE, 4034670592, 4034674687, +STORE, 4034674688, 4034678783, +STORE, 4034678784, 4034682879, +STORE, 4034682880, 4034781183, +STORE, 4034781184, 4035043327, +STORE, 4035043328, 4035047423, +STORE, 4035047424, 4035055615, +STORE, 4035055616, 4035059711, +STORE, 4035059712, 4035063807, +STORE, 4035063808, 4035067903, +STORE, 4035067904, 4035100671, +STORE, 4035100672, 4035375103, +STORE, 4035375104, 4035383295, +STORE, 4035383296, 4035395583, +STORE, 4035395584, 4035399679, +STORE, 4035399680, 4035403775, +STORE, 4035403776, 4035407871, +STORE, 4035407872, 4035411967, +STORE, 4035411968, 4035477503, +STORE, 4035477504, 4035608575, +STORE, 4035608576, 4035641343, +STORE, 4035641344, 4035682303, +STORE, 4035682304, 4035686399, +STORE, 4035686400, 4035690495, +STORE, 4035690496, 4035694591, +STORE, 4035694592, 4035743743, +STORE, 4035743744, 4035784703, +STORE, 4035784704, 4035829759, +STORE, 4035829760, 4035837951, +STORE, 4035837952, 4035842047, +STORE, 4035842048, 4035846143, +STORE, 4035846144, 4035850239, +STORE, 4035850240, 4036001791, +STORE, 4036001792, 4036005887, +STORE, 4036005888, 4036214783, +STORE, 4036214784, 4036218879, +STORE, 4036218880, 4036603903, +STORE, 4036603904, 4036648959, +STORE, 4036648960, 4036653055, +STORE, 4036653056, 4036657151, +STORE, 4036657152, 4036665343, +STORE, 4036665344, 4036780031, +STORE, 4036780032, 4036829183, +STORE, 4036829184, 4036984831, +STORE, 4036984832, 4036993023, +STORE, 4036993024, 4036997119, +STORE, 4036997120, 4037001215, +STORE, 4037001216, 4037009407, +STORE, 4037009408, 4037025791, +STORE, 4037025792, 4037095423, +STORE, 4037095424, 4037181439, +STORE, 4037181440, 4037193727, +STORE, 4037193728, 4037197823, +STORE, 4037197824, 4037206015, +STORE, 4037206016, 4037320703, +STORE, 4037320704, 4037337087, +STORE, 4037337088, 4037349375, +STORE, 4037349376, 4037357567, +STORE, 4037357568, 4037361663, +STORE, 4037369856, 4037386239, +STORE, 4037386240, 4037672959, +STORE, 4037672960, 4037689343, +STORE, 4037689344, 4037730303, +STORE, 4037730304, 4037734399, +STORE, 4037734400, 4037738495, +STORE, 4037738496, 4037742591, +STORE, 4037742592, 4037758975, +STORE, 4037758976, 4037890047, +STORE, 4037890048, 4037931007, +STORE, 4037931008, 4037976063, +STORE, 4037976064, 4037984255, +STORE, 4037984256, 4037988351, +STORE, 4037988352, 4038053887, +STORE, 4038053888, 4038184959, +STORE, 4038184960, 4038189055, +STORE, 4038189056, 4038197247, +STORE, 4038197248, 4038201343, +STORE, 4038201344, 4038205439, +STORE, 4038205440, 4038209535, +STORE, 4038217728, 4038250495, +STORE, 4038250496, 4038512639, +STORE, 4038512640, 4038516735, +STORE, 4038516736, 4038520831, +STORE, 4038520832, 4038524927, +STORE, 4038524928, 4038529023, +STORE, 4038529024, 4038533119, +STORE, 4038541312, 4038623231, +STORE, 4038623232, 4038754303, +STORE, 4038754304, 4038885375, +STORE, 4038885376, 4038889471, +STORE, 4038897664, 4038963199, +STORE, 4038963200, 4038967295, +STORE, 4038967296, 4038983679, +STORE, 4038983680, 4039114751, +STORE, 4039114752, 4039245823, +STORE, 4039245824, 4039376895, +STORE, 4039376896, 4040687615, +STORE, 4040687616, 4040691711, +STORE, 4040691712, 4040806399, +STORE, 4040806400, 4040937471, +STORE, 4040937472, 4040941567, +STORE, 4040945664, 4040949759, +STORE, 4040949760, 4041080831, +STORE, 4041080832, 4041211903, +STORE, 4041211904, 4043046911, +STORE, 4043046912, 4043051007, +STORE, 4043051008, 4043055103, +STORE, 4043055104, 4043137023, +STORE, 4043137024, 4043141119, +STORE, 4043141120, 4043145215, +STORE, 4043145216, 4043153407, +STORE, 4043153408, 4043186175, +STORE, 4043186176, 4043317247, +STORE, 4043317248, 4043448319, +STORE, 4043448320, 4043579391, +STORE, 4043579392, 4043583487, +STORE, 4043583488, 4043599871, +STORE, 4043599872, 4043661311, +STORE, 4043661312, 4043792383, +STORE, 4043792384, 4043796479, +STORE, 4043796480, 4043800575, +STORE, 4043800576, 4043816959, +STORE, 4043816960, 4043821055, +STORE, 4043821056, 4043825151, +STORE, 4043825152, 4043829247, +STORE, 4043829248, 4043833343, +STORE, 4043833344, 4047241215, +STORE, 4047241216, 4047249407, +STORE, 4047249408, 4047253503, +STORE, 4047253504, 4047323135, +STORE, 4047323136, 4047327231, +STORE, 4047327232, 4047458303, +STORE, 4047458304, 4047589375, +STORE, 4047589376, 4047720447, +STORE, 4047720448, 4047773695, +STORE, 4047773696, 4047790079, +STORE, 4047790080, 4047921151, +STORE, 4047921152, 4048052223, +STORE, 4048052224, 4048183295, +STORE, 4048183296, 4049002495, +STORE, 4049002496, 4049133567, +STORE, 4049133568, 4049154047, +STORE, 4049154048, 4049158143, +STORE, 4049158144, 4049162239, +STORE, 4049162240, 4049166335, +STORE, 4049166336, 4049174527, +STORE, 4049174528, 4049182719, +STORE, 4049182720, 4049186815, +STORE, 4049186816, 4049190911, +STORE, 4049190912, 4049195007, +STORE, 4049195008, 4049203199, +STORE, 4049203200, 4049207295, +STORE, 4049207296, 4049211391, +STORE, 4049211392, 4049215487, +STORE, 4049215488, 4049219583, +STORE, 4049219584, 4049227775, +STORE, 4049227776, 4049231871, +STORE, 4049231872, 4049235967, +STORE, 4049235968, 4049244159, +STORE, 4049244160, 4049248255, +STORE, 4049248256, 4049252351, +STORE, 4049252352, 4049256447, +STORE, 4049256448, 4049268735, +STORE, 4049268736, 4049272831, +STORE, 4049272832, 4049313791, +STORE, 4049313792, 4049723391, +STORE, 4049723392, 4049727487, +STORE, 4049727488, 4049858559, +STORE, 4049858560, 4049989631, +STORE, 4049989632, 4049993727, +STORE, 4049993728, 4050026495, +STORE, 4050026496, 4050030591, +STORE, 4050030592, 4050161663, +STORE, 4050161664, 4050169855, +STORE, 4050169856, 4050223103, +STORE, 4050223104, 4050632703, +STORE, 4050632704, 4050636799, +STORE, 4050636800, 4050640895, +STORE, 4050640896, 4050644991, +STORE, 4050644992, 4050661375, +STORE, 4050661376, 4050665471, +STORE, 4050665472, 4050673663, +STORE, 4050673664, 4050677759, +STORE, 4050677760, 4050694143, +STORE, 4050694144, 4050702335, +STORE, 4050702336, 4050956287, +STORE, 4050956288, 4051963903, +STORE, 4051963904, 4051980287, +STORE, 4051980288, 4051988479, +STORE, 4051988480, 4052000767, +STORE, 4052000768, 4052004863, +STORE, 4052004864, 4052029439, +STORE, 4284014592, 4284018687, +STORE, 4284018688, 4292403199, +SNULL, 4041080832, 4041211903, +SNULL, 3795763200, 3795894271, +STORE, 3629522944, 3696631807, +SNULL, 3663077375, 3696631807, +STORE, 3629522944, 3663077375, +STORE, 3663077376, 3696631807, +SNULL, 3663077376, 3696631807, +STORE, 3663077376, 3696631807, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3626471424, 3627524095, +SNULL, 3626471424, 3626475519, +STORE, 3626475520, 3627524095, +STORE, 3626471424, 3626475519, +SNULL, 3627519999, 3627524095, +STORE, 3626475520, 3627519999, +STORE, 3627520000, 3627524095, +STORE, 3625418752, 3626475519, +SNULL, 3625418752, 3625422847, +STORE, 3625422848, 3626475519, +STORE, 3625418752, 3625422847, +SNULL, 3626467327, 3626475519, +STORE, 3625422848, 3626467327, +STORE, 3626467328, 3626475519, +STORE, 3624366080, 3625422847, +SNULL, 3624366080, 3624370175, +STORE, 3624370176, 3625422847, +STORE, 3624366080, 3624370175, +SNULL, 3625414655, 3625422847, +STORE, 3624370176, 3625414655, +STORE, 3625414656, 3625422847, +STORE, 4041191424, 4041211903, +SNULL, 4041195519, 4041211903, +STORE, 4041191424, 4041195519, +STORE, 4041195520, 4041211903, +STORE, 4041170944, 4041191423, +SNULL, 4041175039, 4041191423, +STORE, 4041170944, 4041175039, +STORE, 4041175040, 4041191423, +SNULL, 3625426943, 3626467327, +STORE, 3625422848, 3625426943, +STORE, 3625426944, 3626467327, +STORE, 4041162752, 4041170943, +SNULL, 3626479615, 3627519999, +STORE, 3626475520, 3626479615, +STORE, 3626479616, 3627519999, +STORE, 4041154560, 4041162751, +STORE, 4041154560, 4041170943, +STORE, 4041134080, 4041154559, +SNULL, 4041138175, 4041154559, +STORE, 4041134080, 4041138175, +STORE, 4041138176, 4041154559, +SNULL, 3624374271, 3625414655, +STORE, 3624370176, 3624374271, +STORE, 3624374272, 3625414655, +STORE, 4041125888, 4041134079, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +STORE, 4041121792, 4041125887, +SNULL, 4041121792, 4041125887, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +STORE, 3222274048, 3223326719, +SNULL, 3222274048, 3222278143, +STORE, 3222278144, 3223326719, +STORE, 3222274048, 3222278143, +SNULL, 3223322623, 3223326719, +STORE, 3222278144, 3223322623, +STORE, 3223322624, 3223326719, +STORE, 3221221376, 3222278143, +SNULL, 3221221376, 3221225471, +STORE, 3221225472, 3222278143, +STORE, 3221221376, 3221225471, +SNULL, 3222269951, 3222278143, +STORE, 3221225472, 3222269951, +STORE, 3222269952, 3222278143, +STORE, 3220168704, 3221225471, +SNULL, 3220168704, 3220172799, +STORE, 3220172800, 3221225471, +STORE, 3220168704, 3220172799, +SNULL, 3221217279, 3221225471, +STORE, 3220172800, 3221217279, +STORE, 3221217280, 3221225471, +STORE, 4041117696, 4041125887, +STORE, 4041117696, 4041134079, +STORE, 3219083264, 3220172799, +SNULL, 3219083264, 3219087359, +STORE, 3219087360, 3220172799, +STORE, 3219083264, 3219087359, +SNULL, 3220164607, 3220172799, +STORE, 3219087360, 3220164607, +STORE, 3220164608, 3220172799, +STORE, 4041109504, 4041117695, +STORE, 4041109504, 4041134079, +STORE, 3217997824, 3219087359, +SNULL, 3217997824, 3218001919, +STORE, 3218001920, 3219087359, +STORE, 3217997824, 3218001919, +SNULL, 3219079167, 3219087359, +STORE, 3218001920, 3219079167, +STORE, 3219079168, 3219087359, +STORE, 4041101312, 4041109503, +STORE, 4041101312, 4041134079, +STORE, 3216912384, 3218001919, +SNULL, 3216912384, 3216916479, +STORE, 3216916480, 3218001919, +STORE, 3216912384, 3216916479, +SNULL, 3217993727, 3218001919, +STORE, 3216916480, 3217993727, +STORE, 3217993728, 3218001919, +STORE, 4041093120, 4041101311, +STORE, 4041093120, 4041134079, +STORE, 3215826944, 3216916479, +SNULL, 3215826944, 3215831039, +STORE, 3215831040, 3216916479, +STORE, 3215826944, 3215831039, +SNULL, 3216908287, 3216916479, +STORE, 3215831040, 3216908287, +STORE, 3216908288, 3216916479, +STORE, 4016779264, 4016799743, +SNULL, 4016783359, 4016799743, +STORE, 4016779264, 4016783359, +STORE, 4016783360, 4016799743, +STORE, 4016758784, 4016779263, +SNULL, 4016762879, 4016779263, +STORE, 4016758784, 4016762879, +STORE, 4016762880, 4016779263, +SNULL, 3222282239, 3223322623, +STORE, 3222278144, 3222282239, +STORE, 3222282240, 3223322623, +STORE, 4041084928, 4041093119, +STORE, 4041084928, 4041134079, +SNULL, 3221229567, 3222269951, +STORE, 3221225472, 3221229567, +STORE, 3221229568, 3222269951, +STORE, 4015644672, 4015665151, +STORE, 4038889472, 4038897663, +SNULL, 4015648767, 4015665151, +STORE, 4015644672, 4015648767, +STORE, 4015648768, 4015665151, +STORE, 4015624192, 4015644671, +SNULL, 4015628287, 4015644671, +STORE, 4015624192, 4015628287, +STORE, 4015628288, 4015644671, +SNULL, 3219091455, 3220164607, +STORE, 3219087360, 3219091455, +STORE, 3219091456, 3220164607, +STORE, 4015603712, 4015624191, +SNULL, 4015607807, 4015624191, +STORE, 4015603712, 4015607807, +STORE, 4015607808, 4015624191, +SNULL, 3218006015, 3219079167, +STORE, 3218001920, 3218006015, +STORE, 3218006016, 3219079167, +STORE, 3949674496, 3949694975, +SNULL, 3949678591, 3949694975, +STORE, 3949674496, 3949678591, +STORE, 3949678592, 3949694975, +SNULL, 3216920575, 3217993727, +STORE, 3216916480, 3216920575, +STORE, 3216920576, 3217993727, +STORE, 3948924928, 3948945407, +SNULL, 3948929023, 3948945407, +STORE, 3948924928, 3948929023, +STORE, 3948929024, 3948945407, +SNULL, 3215835135, 3216908287, +STORE, 3215831040, 3215835135, +STORE, 3215835136, 3216908287, +SNULL, 3220176895, 3221217279, +STORE, 3220172800, 3220176895, +STORE, 3220176896, 3221217279, +STORE, 3214786560, 3215826943, +STORE, 3213733888, 3214786559, +SNULL, 3213733888, 3213737983, +STORE, 3213737984, 3214786559, +STORE, 3213733888, 3213737983, +SNULL, 3214782463, 3214786559, +STORE, 3213737984, 3214782463, +STORE, 3214782464, 3214786559, +STORE, 4038533120, 4038541311, +STORE, 3948421120, 3948441599, +SNULL, 3948425215, 3948441599, +STORE, 3948421120, 3948425215, +STORE, 3948425216, 3948441599, +SNULL, 3213742079, 3214782463, +STORE, 3213737984, 3213742079, +STORE, 3213742080, 3214782463, +STORE, 4038209536, 4038217727, +STORE, 3212681216, 3213737983, +SNULL, 3212681216, 3212685311, +STORE, 3212685312, 3213737983, +STORE, 3212681216, 3212685311, +SNULL, 3213729791, 3213737983, +STORE, 3212685312, 3213729791, +STORE, 3213729792, 3213737983, +STORE, 3795763200, 3795894271, +STORE, 3946872832, 3946893311, +SNULL, 3946876927, 3946893311, +STORE, 3946872832, 3946876927, +STORE, 3946876928, 3946893311, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +SNULL, 3212689407, 3213729791, +STORE, 3212685312, 3212689407, +STORE, 3212689408, 3213729791, +STORE, 4041080832, 4041084927, +STORE, 4040941568, 4040945663, +STORE, 4037361664, 4037369855, +STORE, 4000817152, 4000821247, +STORE, 3999440896, 3999444991, +STORE, 3212161024, 3212681215, +SNULL, 3212161024, 3212439551, +STORE, 3212439552, 3212681215, +STORE, 3212161024, 3212439551, +SNULL, 3212161024, 3212439551, +SNULL, 3212464127, 3212681215, +STORE, 3212439552, 3212464127, +STORE, 3212464128, 3212681215, +SNULL, 3212464128, 3212681215, +SNULL, 3212439552, 3212451839, +STORE, 3212451840, 3212464127, +STORE, 3212439552, 3212451839, +SNULL, 3212439552, 3212451839, +STORE, 3212439552, 3212451839, +SNULL, 3212451840, 3212455935, +STORE, 3212455936, 3212464127, +STORE, 3212451840, 3212455935, +SNULL, 3212451840, 3212455935, +STORE, 3212451840, 3212455935, +SNULL, 3212455936, 3212460031, +STORE, 3212460032, 3212464127, +STORE, 3212455936, 3212460031, +SNULL, 3212455936, 3212460031, +STORE, 3212455936, 3212460031, +SNULL, 3212460032, 3212464127, +STORE, 3212460032, 3212464127, +STORE, 3997679616, 3997683711, +SNULL, 4049235968, 4049240063, +STORE, 4049240064, 4049244159, +STORE, 4049235968, 4049240063, +SNULL, 4049240064, 4049244159, +STORE, 4049240064, 4049244159, +SNULL, 3997679616, 3997683711, +SNULL, 3999440896, 3999444991, +SNULL, 4000817152, 4000821247, +SNULL, 4040941568, 4040945663, +SNULL, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +SNULL, 3212451840, 3212455935, +STORE, 3212451840, 3212455935, +STORE, 4041080832, 4041084927, +STORE, 3623890944, 3624169471, +SNULL, 4041080832, 4041084927, +STORE, 4041080832, 4041084927, +SNULL, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3211386880, 3212439551, +SNULL, 3211386880, 3211390975, +STORE, 3211390976, 3212439551, +STORE, 3211386880, 3211390975, +SNULL, 3212435455, 3212439551, +STORE, 3211390976, 3212435455, +STORE, 3212435456, 3212439551, +STORE, 4040941568, 4040945663, +STORE, 3937169408, 3937189887, +STORE, 3623485440, 3623616511, +SNULL, 717225983, 1388314623, +STORE, 314572800, 717225983, +STORE, 717225984, 1388314623, +SNULL, 717225984, 1388314623, +STORE, 3937112064, 3937132543, +SNULL, 3937116159, 3937132543, +STORE, 3937112064, 3937116159, +STORE, 3937116160, 3937132543, +SNULL, 3211395071, 3212435455, +STORE, 3211390976, 3211395071, +STORE, 3211395072, 3212435455, +STORE, 4000817152, 4000821247, +STORE, 3974823936, 3974832127, +STORE, 3595284480, 3595431935, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +STORE, 3999440896, 3999444991, +STORE, 3997679616, 3997683711, +STORE, 3996295168, 3996299263, +STORE, 3996090368, 3996094463, +STORE, 3210866688, 3211386879, +SNULL, 3210866688, 3211001855, +STORE, 3211001856, 3211386879, +STORE, 3210866688, 3211001855, +SNULL, 3210866688, 3211001855, +SNULL, 3211038719, 3211386879, +STORE, 3211001856, 3211038719, +STORE, 3211038720, 3211386879, +SNULL, 3211038720, 3211386879, +SNULL, 3211001856, 3211022335, +STORE, 3211022336, 3211038719, +STORE, 3211001856, 3211022335, +SNULL, 3211001856, 3211022335, +STORE, 3211001856, 3211022335, +SNULL, 3211022336, 3211030527, +STORE, 3211030528, 3211038719, +STORE, 3211022336, 3211030527, +SNULL, 3211022336, 3211030527, +STORE, 3211022336, 3211030527, +SNULL, 3211030528, 3211034623, +STORE, 3211034624, 3211038719, +STORE, 3211030528, 3211034623, +SNULL, 3211030528, 3211034623, +STORE, 3211030528, 3211034623, +SNULL, 3211034624, 3211038719, +STORE, 3211034624, 3211038719, +STORE, 3994906624, 3994910719, +SNULL, 4049240064, 4049244159, +STORE, 4049240064, 4049244159, +SNULL, 3994906624, 3994910719, +SNULL, 3996090368, 3996094463, +SNULL, 3996295168, 3996299263, +SNULL, 3997679616, 3997683711, +SNULL, 3999440896, 3999444991, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +SNULL, 3211022336, 3211030527, +STORE, 3211022336, 3211030527, +STORE, 3999440896, 3999444991, +STORE, 3210199040, 3211001855, +SNULL, 3999440896, 3999444991, +STORE, 3999440896, 3999444991, +SNULL, 3999440896, 3999444991, +STORE, 3594821632, 3594952703, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 1914101759, 1969434623, +STORE, 1914097664, 1914101759, +STORE, 1914101760, 1969434623, +STORE, 3567108096, 3567239167, +STORE, 3973832704, 3973840895, +STORE, 3209113600, 3210199039, +SNULL, 3209113600, 3209117695, +STORE, 3209117696, 3210199039, +STORE, 3209113600, 3209117695, +SNULL, 3210194943, 3210199039, +STORE, 3209117696, 3210194943, +STORE, 3210194944, 3210199039, +STORE, 3935858688, 3935879167, +SNULL, 3935862783, 3935879167, +STORE, 3935858688, 3935862783, +STORE, 3935862784, 3935879167, +SNULL, 3209121791, 3210194943, +STORE, 3209117696, 3209121791, +STORE, 3209121792, 3210194943, +STORE, 3528749056, 3528880127, +STORE, 3968200704, 3968208895, +STORE, 3208028160, 3209117695, +SNULL, 3208028160, 3208032255, +STORE, 3208032256, 3209117695, +STORE, 3208028160, 3208032255, +SNULL, 3209109503, 3209117695, +STORE, 3208032256, 3209109503, +STORE, 3209109504, 3209117695, +STORE, 3888123904, 3888144383, +SNULL, 3888127999, 3888144383, +STORE, 3888123904, 3888127999, +STORE, 3888128000, 3888144383, +SNULL, 3208036351, 3209109503, +STORE, 3208032256, 3208036351, +STORE, 3208036352, 3209109503, +SNULL, 3968200704, 3968208895, +SNULL, 3888123904, 3888144383, +SNULL, 3209109504, 3209113599, +STORE, 3209113600, 3209117695, +STORE, 3209109504, 3209113599, +SNULL, 3208028160, 3209113599, +STORE, 3208060928, 3209117695, +SNULL, 3208060928, 3208065023, +STORE, 3208065024, 3209117695, +STORE, 3208060928, 3208065023, +SNULL, 3209109503, 3209117695, +STORE, 3208065024, 3209109503, +STORE, 3209109504, 3209117695, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3888123904, 3888144383, +SNULL, 3888127999, 3888144383, +STORE, 3888123904, 3888127999, +STORE, 3888128000, 3888144383, +SNULL, 3208069119, 3209109503, +STORE, 3208065024, 3208069119, +STORE, 3208069120, 3209109503, +STORE, 3968200704, 3968208895, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3527778304, 3527909375, +STORE, 3999440896, 3999444991, +STORE, 3997679616, 3997683711, +STORE, 1914097664, 1914105855, +STORE, 1914105856, 1969434623, +STORE, 3957583872, 3957592063, +STORE, 3206975488, 3208065023, +SNULL, 3206975488, 3206979583, +STORE, 3206979584, 3208065023, +STORE, 3206975488, 3206979583, +SNULL, 3208056831, 3208065023, +STORE, 3206979584, 3208056831, +STORE, 3208056832, 3208065023, +STORE, 3956736000, 3956744191, +STORE, 3205890048, 3206979583, +SNULL, 3205890048, 3205894143, +STORE, 3205894144, 3206979583, +STORE, 3205890048, 3205894143, +SNULL, 3206971391, 3206979583, +STORE, 3205894144, 3206971391, +STORE, 3206971392, 3206979583, +STORE, 3806101504, 3806121983, +SNULL, 3806105599, 3806121983, +STORE, 3806101504, 3806105599, +STORE, 3806105600, 3806121983, +SNULL, 3206983679, 3208056831, +STORE, 3206979584, 3206983679, +STORE, 3206983680, 3208056831, +STORE, 3806081024, 3806101503, +SNULL, 3806085119, 3806101503, +STORE, 3806081024, 3806085119, +STORE, 3806085120, 3806101503, +SNULL, 3205898239, 3206971391, +STORE, 3205894144, 3205898239, +STORE, 3205898240, 3206971391, +STORE, 3956015104, 3956023295, +STORE, 3204804608, 3205894143, +SNULL, 3204804608, 3204808703, +STORE, 3204808704, 3205894143, +STORE, 3204804608, 3204808703, +SNULL, 3205885951, 3205894143, +STORE, 3204808704, 3205885951, +STORE, 3205885952, 3205894143, +STORE, 3803471872, 3803492351, +STORE, 3803451392, 3803471871, +STORE, 3803451392, 3803492351, +SNULL, 3957583872, 3957592063, +SNULL, 3806101504, 3806121983, +SNULL, 3206975487, 3206979583, +STORE, 3206971392, 3206975487, +STORE, 3206975488, 3206979583, +SNULL, 3208056832, 3208060927, +STORE, 3208060928, 3208065023, +STORE, 3208056832, 3208060927, +SNULL, 3206975488, 3208060927, +STORE, 3801845760, 3801878527, +STORE, 3806101504, 3806121983, +SNULL, 3806105599, 3806121983, +STORE, 3806101504, 3806105599, +STORE, 3806105600, 3806121983, +SNULL, 3204812799, 3205885951, +STORE, 3204808704, 3204812799, +STORE, 3204812800, 3205885951, +STORE, 1914097664, 1914109951, +STORE, 1914109952, 1969434623, +STORE, 3957583872, 3957592063, +STORE, 3206971392, 3208065023, +SNULL, 3206971392, 3206979583, +STORE, 3206979584, 3208065023, +STORE, 3206971392, 3206979583, +SNULL, 3208056831, 3208065023, +STORE, 3206979584, 3208056831, +STORE, 3208056832, 3208065023, +STORE, 3801825280, 3801845759, +SNULL, 3801829375, 3801845759, +STORE, 3801825280, 3801829375, +STORE, 3801829376, 3801845759, +SNULL, 3206983679, 3208056831, +STORE, 3206979584, 3206983679, +STORE, 3206983680, 3208056831, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3202707456, 3204804607, +STORE, 3200610304, 3202707455, +SNULL, 3202707456, 3204804607, +SNULL, 3200610304, 3202707455, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3527647232, 3527778303, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +STORE, 3487059968, 3487584255, +SNULL, 3487059968, 3487301631, +STORE, 3487301632, 3487584255, +STORE, 3487059968, 3487301631, +SNULL, 3487059968, 3487301631, +SNULL, 3487563775, 3487584255, +STORE, 3487301632, 3487563775, +STORE, 3487563776, 3487584255, +SNULL, 3487563776, 3487584255, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3524046848, 3524177919, +STORE, 3487170560, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3487039488, 3487170559, +STORE, 3487039488, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3204280320, 3204804607, +SNULL, 3204280320, 3204448255, +STORE, 3204448256, 3204804607, +STORE, 3204280320, 3204448255, +SNULL, 3204280320, 3204448255, +SNULL, 3204710399, 3204804607, +STORE, 3204448256, 3204710399, +STORE, 3204710400, 3204804607, +SNULL, 3204710400, 3204804607, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3996295168, 3996299263, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +SNULL, 3996295168, 3996299263, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3486908416, 3487039487, +STORE, 3486908416, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +STORE, 3201826816, 3202351103, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +SNULL, 3803471871, 3803492351, +STORE, 3803451392, 3803471871, +STORE, 3803471872, 3803492351, +SNULL, 3803471872, 3803492351, +SNULL, 3803451392, 3803471871, +STORE, 3798999040, 3799101439, +SNULL, 3798999040, 3799101439, +STORE, 3952644096, 3952652287, +STORE, 3203362816, 3204448255, +SNULL, 3203362816, 3203366911, +STORE, 3203366912, 3204448255, +STORE, 3203362816, 3203366911, +SNULL, 3204444159, 3204448255, +STORE, 3203366912, 3204444159, +STORE, 3204444160, 3204448255, +STORE, 3803471872, 3803492351, +SNULL, 3803475967, 3803492351, +STORE, 3803471872, 3803475967, +STORE, 3803475968, 3803492351, +SNULL, 3203371007, 3204444159, +STORE, 3203366912, 3203371007, +STORE, 3203371008, 3204444159, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3200774144, 3201826815, +SNULL, 3200774144, 3200778239, +STORE, 3200778240, 3201826815, +STORE, 3200774144, 3200778239, +SNULL, 3201822719, 3201826815, +STORE, 3200778240, 3201822719, +STORE, 3201822720, 3201826815, +STORE, 3803451392, 3803471871, +SNULL, 3803455487, 3803471871, +STORE, 3803451392, 3803455487, +STORE, 3803455488, 3803471871, +SNULL, 3200782335, 3201822719, +STORE, 3200778240, 3200782335, +STORE, 3200782336, 3201822719, +STORE, 3949666304, 3949674495, +STORE, 3949408256, 3949416447, +STORE, 3199688704, 3200778239, +SNULL, 3199688704, 3199692799, +STORE, 3199692800, 3200778239, +STORE, 3199688704, 3199692799, +SNULL, 3200770047, 3200778239, +STORE, 3199692800, 3200770047, +STORE, 3200770048, 3200778239, +STORE, 3799306240, 3799326719, +SNULL, 3799310335, 3799326719, +STORE, 3799306240, 3799310335, +STORE, 3799310336, 3799326719, +SNULL, 3199696895, 3200770047, +STORE, 3199692800, 3199696895, +STORE, 3199696896, 3200770047, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3799277568, 3799306239, +SNULL, 3799277568, 3799306239, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +SNULL, 4041162751, 4041170943, +STORE, 4041154560, 4041162751, +STORE, 4041162752, 4041170943, +SNULL, 4041162752, 4041170943, +SNULL, 4041154560, 4041162751, +SNULL, 4041191424, 4041211903, +SNULL, 4041170944, 4041191423, +SNULL, 3626471423, 3626475519, +STORE, 3626467328, 3626471423, +STORE, 3626471424, 3626475519, +SNULL, 3626471424, 3627524095, +SNULL, 3625418751, 3625422847, +STORE, 3625414656, 3625418751, +STORE, 3625418752, 3625422847, +SNULL, 3625418752, 3626471423, +STORE, 3627393024, 3627524095, +STORE, 3627261952, 3627393023, +STORE, 3627261952, 3627524095, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3195494400, 3197591551, +SNULL, 3197591552, 3199688703, +SNULL, 3195494400, 3197591551, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3195494400, 3197591551, +SNULL, 3197591552, 3199688703, +SNULL, 3195494400, 3197591551, +STORE, 3798999040, 3799101439, +SNULL, 3798999040, 3799101439, +/* + * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 + * Gap was found: mt 4041162752 gap_end 4041183232 + * mmap: window was 4052029440 - 4096 size 28672 + * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 + * mmap: mas.index 4041162752 align mask 0 offset 0 + * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) + */ + }; + + unsigned long set43[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734187720704, 140737488351231, +SNULL, 140734187724800, 140737488351231, +STORE, 140734187589632, 140734187724799, +STORE, 4194304, 6443007, +STORE, 4337664, 6443007, +STORE, 4194304, 4337663, +SNULL, 4337664, 6443007, +STORE, 6430720, 6443007, +STORE, 206158430208, 206160674815, +STORE, 206158569472, 206160674815, +STORE, 206158430208, 206158569471, +SNULL, 206158569472, 206160674815, +STORE, 206160662528, 206160670719, +STORE, 206160670720, 206160674815, +STORE, 140734188756992, 140734188765183, +STORE, 140734188740608, 140734188756991, +STORE, 140501948112896, 140501948116991, + }; + + int count = 0; + void *ptr = NULL; + + MA_STATE(mas, mt, 0, 0); + + mt_set_non_kernel(3); + check_erase2_testset(mt, set, ARRAY_SIZE(set)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); + start = 140735933894656; + MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); + mtree_destroy(mt); + + mt_set_non_kernel(2); + mt_init_flags(mt, 0); + check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, 0); + check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); + rcu_read_lock(); + mas_for_each(&mas, entry, ULONG_MAX) { + if (xa_is_zero(entry)) + continue; + } + rcu_read_unlock(); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mt_set_non_kernel(100); + check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); + rcu_barrier(); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); + rcu_barrier(); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); + rcu_barrier(); + mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); + MT_BUG_ON(mt, mas.last != 140014592573439); + mtree_destroy(mt); + + mas_reset(&mas); + mas.tree = mt; + count = 0; + mas.index = 0; + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); + rcu_barrier(); + mas_for_each(&mas, entry, ULONG_MAX) { + if (xa_is_zero(entry)) + continue; + BUG_ON(count > 12); + count++; + } + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); + mtree_erase(mt, 140373516443648); + rcu_read_lock(); + mas_empty_area_rev(&mas, 0, 140373518663680, 4096); + rcu_read_unlock(); + mtree_destroy(mt); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); + rcu_barrier(); + mtree_destroy(mt); + + /* set16 was to find a bug on limit updating at slot 0. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); + MT_BUG_ON(mt, mas.last != 139921865547775); + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set17 found a bug in walking backwards and not counting nulls at + * the end. This could cause a gap to be missed if the null had any + * size. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); + MT_BUG_ON(mt, mas.last != 139953197322239); +/* MT_BUG_ON(mt, mas.index != 139953197318144); */ + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set18 found a bug in walking backwards and not setting the max from + * the node, but using the parent node. This was only an issue if the + * next slot in the parent had what we needed. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); + MT_BUG_ON(mt, mas.last != 140222968475647); + /*MT_BUG_ON(mt, mas.index != 140222966259712); */ + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set19 found 2 bugs in prev. + * 1. If we hit root without finding anything, then there was an + * infinite loop. + * 2. The first ascending wasn't using the correct slot which may have + * caused missed entries. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); + rcu_barrier(); + mas.index = 140656779083776; + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); + entry = mas_prev(&mas, 0); + MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set20 found a bug in mas_may_move_gap due to the slot being + * overwritten during the __mas_add operation and setting it to zero. + */ + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); + rcu_barrier(); + check_load(mt, 94849009414144, NULL); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); + rcu_barrier(); + mt_validate(mt); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(999); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); + rcu_barrier(); + mt_validate(mt); + ptr = mtree_load(mt, 140551363362816); + MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Split on NULL followed by delete - causes gap issues. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); + MT_BUG_ON(mt, mas.last != 140109040959487); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Split on NULL followed by delete - causes gap issues. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); + /* Search for the size of gap then align it (offset 0) */ + mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); + MT_BUG_ON(mt, mas.index != 139918401601536); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found issues with retry moving rebalanced nodes so the + * incorrect parent pivot was updated. + */ + mt_set_non_kernel(999); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found issues with deleting all entries in a node when + * surrounded by entries in the next nodes, then deleting the entries + * surrounding the node filled with deleted entries. + */ + mt_set_non_kernel(999); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found an issue with deleting all entries in a node that was + * the end node and mas_gap incorrectly set next = curr, and curr = prev + * then moved next to the left, losing data. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + +/* + * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: + * mt 140582827569152 gap_end 140582869532672 + * mmap: window was 140583656296448 - 4096 size 134217728 + * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 + * mmap: mas.index 140582827569152 align mask 0 offset 0 + * mmap: rb_find_vma find on + * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) + */ + + /* move gap failed due to an entirely empty node */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); + MT_BUG_ON(mt, mas.last != 140583003750399); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* + * Incorrect gap in tree caused by mas_prev not setting the limits + * correctly while walking down. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Empty leaf at the end of a parent caused incorrect gap. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + /* Empty leaf at the end of a parent caused incorrect gap. */ + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + /* move gap failed due to an entirely empty node. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 4052029440, 28672); + MT_BUG_ON(mt, mas.last != 4041211903); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* gap calc off by one */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); +} + +static noinline void check_alloc_rev_range(struct maple_tree *mt) +{ + /* + * Generated by: + * cat /proc/self/maps | awk '{print $1}'| + * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' + */ + + unsigned long range[] = { + /* Inclusive , Exclusive. */ + 0x565234af2000, 0x565234af4000, + 0x565234af4000, 0x565234af9000, + 0x565234af9000, 0x565234afb000, + 0x565234afc000, 0x565234afd000, + 0x565234afd000, 0x565234afe000, + 0x565235def000, 0x565235e10000, + 0x7f36d4bfd000, 0x7f36d4ee2000, + 0x7f36d4ee2000, 0x7f36d4f04000, + 0x7f36d4f04000, 0x7f36d504c000, + 0x7f36d504c000, 0x7f36d5098000, + 0x7f36d5098000, 0x7f36d5099000, + 0x7f36d5099000, 0x7f36d509d000, + 0x7f36d509d000, 0x7f36d509f000, + 0x7f36d509f000, 0x7f36d50a5000, + 0x7f36d50b9000, 0x7f36d50db000, + 0x7f36d50db000, 0x7f36d50dc000, + 0x7f36d50dc000, 0x7f36d50fa000, + 0x7f36d50fa000, 0x7f36d5102000, + 0x7f36d5102000, 0x7f36d5103000, + 0x7f36d5103000, 0x7f36d5104000, + 0x7f36d5104000, 0x7f36d5105000, + 0x7fff5876b000, 0x7fff5878d000, + 0x7fff5878e000, 0x7fff58791000, + 0x7fff58791000, 0x7fff58793000, + }; + + unsigned long holes[] = { + /* + * Note: start of hole is INCLUSIVE + * end of hole is EXCLUSIVE + * (opposite of the above table.) + * Start of hole, end of hole, size of hole (+1) + */ + 0x565234afb000, 0x565234afc000, 0x1000, + 0x565234afe000, 0x565235def000, 0x12F1000, + 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, + }; + + /* + * req_range consists of 4 values. + * 1. min index + * 2. max index + * 3. size + * 4. number that should be returned. + * 5. return value + */ + unsigned long req_range[] = { + 0x565234af9000, /* Min */ + 0x7fff58791000, /* Max */ + 0x1000, /* Size */ + 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ + 0, /* Return value success. */ + + 0x0, /* Min */ + 0x565234AF1 << 12, /* Max */ + 0x3000, /* Size */ + 0x565234AEE << 12, /* max - 3. */ + 0, /* Return value success. */ + + 0x0, /* Min */ + -1, /* Max */ + 0x1000, /* Size */ + 562949953421311 << 12,/* First rev hole of size 0x1000 */ + 0, /* Return value success. */ + + 0x0, /* Min */ + 0x7F36D510A << 12, /* Max */ + 0x4000, /* Size */ + 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ + 0, /* Return value success. */ + + /* Ascend test. */ + 0x0, + 34148798629 << 12, + 19 << 12, + 34148797418 << 12, + 0x0, + + /* Too big test. */ + 0x0, + 18446744073709551615UL, + 562915594369134UL << 12, + 0x0, + -EBUSY, + + }; + + int i, range_count = ARRAY_SIZE(range); + int req_range_count = ARRAY_SIZE(req_range); + unsigned long min = 0; + + MA_STATE(mas, mt, 0, 0); + + mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, + GFP_KERNEL); +#define DEBUG_REV_RANGE 0 + for (i = 0; i < range_count; i += 2) { + /* Inclusive, Inclusive (with the -1) */ + +#if DEBUG_REV_RANGE + pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, + (range[i + 1] >> 12) - 1); +#endif + check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, + xa_mk_value(range[i] >> 12), 0); + mt_validate(mt); + } + + + for (i = 0; i < ARRAY_SIZE(holes); i += 3) { +#if DEBUG_REV_RANGE + pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", + min, holes[i+1]>>12, holes[i+2]>>12, + holes[i] >> 12); +#endif + MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, + holes[i+1] >> 12, + holes[i+2] >> 12)); +#if DEBUG_REV_RANGE + pr_debug("Found %lu %lu\n", mas.index, mas.last); + pr_debug("gap %lu %lu\n", (holes[i] >> 12), + (holes[i+1] >> 12)); +#endif + MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); + MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); + min = holes[i+1] >> 12; + mas_reset(&mas); + } + + for (i = 0; i < req_range_count; i += 5) { +#if DEBUG_REV_RANGE + pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", + req_range[i] >> 12, + (req_range[i + 1] >> 12) - 1, + req_range[i+2] >> 12, + req_range[i+3] >> 12); +#endif + check_mtree_alloc_rrange(mt, + req_range[i] >> 12, /* start */ + req_range[i+1] >> 12, /* end */ + req_range[i+2] >> 12, /* size */ + req_range[i+3] >> 12, /* expected address */ + req_range[i+4], /* expected return */ + xa_mk_value(req_range[i] >> 12)); /* pointer */ + mt_validate(mt); + } + + mt_set_non_kernel(1); + mtree_erase(mt, 34148798727); /* create a deleted range. */ + check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, + 34148798725, 0, mt); + + mtree_destroy(mt); +} + +static noinline void check_alloc_range(struct maple_tree *mt) +{ + /* + * Generated by: + * cat /proc/self/maps|awk '{print $1}'| + * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' + */ + + unsigned long range[] = { + /* Inclusive , Exclusive. */ + 0x565234af2000, 0x565234af4000, + 0x565234af4000, 0x565234af9000, + 0x565234af9000, 0x565234afb000, + 0x565234afc000, 0x565234afd000, + 0x565234afd000, 0x565234afe000, + 0x565235def000, 0x565235e10000, + 0x7f36d4bfd000, 0x7f36d4ee2000, + 0x7f36d4ee2000, 0x7f36d4f04000, + 0x7f36d4f04000, 0x7f36d504c000, + 0x7f36d504c000, 0x7f36d5098000, + 0x7f36d5098000, 0x7f36d5099000, + 0x7f36d5099000, 0x7f36d509d000, + 0x7f36d509d000, 0x7f36d509f000, + 0x7f36d509f000, 0x7f36d50a5000, + 0x7f36d50b9000, 0x7f36d50db000, + 0x7f36d50db000, 0x7f36d50dc000, + 0x7f36d50dc000, 0x7f36d50fa000, + 0x7f36d50fa000, 0x7f36d5102000, + 0x7f36d5102000, 0x7f36d5103000, + 0x7f36d5103000, 0x7f36d5104000, + 0x7f36d5104000, 0x7f36d5105000, + 0x7fff5876b000, 0x7fff5878d000, + 0x7fff5878e000, 0x7fff58791000, + 0x7fff58791000, 0x7fff58793000, + }; + unsigned long holes[] = { + /* Start of hole, end of hole, size of hole (+1) */ + 0x565234afb000, 0x565234afc000, 0x1000, + 0x565234afe000, 0x565235def000, 0x12F1000, + 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, + }; + + /* + * req_range consists of 4 values. + * 1. min index + * 2. max index + * 3. size + * 4. number that should be returned. + * 5. return value + */ + unsigned long req_range[] = { + 0x565234af9000, /* Min */ + 0x7fff58791000, /* Max */ + 0x1000, /* Size */ + 0x565234afb000, /* First hole in our data of size 1000. */ + 0, /* Return value success. */ + + 0x0, /* Min */ + 0x7fff58791000, /* Max */ + 0x1F00, /* Size */ + 0x0, /* First hole in our data of size 2000. */ + 0, /* Return value success. */ + + /* Test ascend. */ + 34148797436 << 12, /* Min */ + 0x7fff587AF000, /* Max */ + 0x3000, /* Size */ + 34148798629 << 12, /* Expected location */ + 0, /* Return value success. */ + + /* Test failing. */ + 34148798623 << 12, /* Min */ + 34148798683 << 12, /* Max */ + 0x15000, /* Size */ + 0, /* Expected location */ + -EBUSY, /* Return value failed. */ + + /* Test filling entire gap. */ + 34148798623 << 12, /* Min */ + 0x7fff587AF000, /* Max */ + 0x10000, /* Size */ + 34148798632 << 12, /* Expected location */ + 0, /* Return value success. */ + + /* Test walking off the end of root. */ + 0, /* Min */ + -1, /* Max */ + -1, /* Size */ + 0, /* Expected location */ + -EBUSY, /* Return value failure. */ + + /* Test looking for too large a hole across entire range. */ + 0, /* Min */ + -1, /* Max */ + 4503599618982063UL << 12, /* Size */ + 34359052178 << 12, /* Expected location */ + -EBUSY, /* Return failure. */ + }; + int i, range_count = ARRAY_SIZE(range); + int req_range_count = ARRAY_SIZE(req_range); + unsigned long min = 0x565234af2000; + + mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, + GFP_KERNEL); + for (i = 0; i < range_count; i += 2) { +#define DEBUG_ALLOC_RANGE 0 +#if DEBUG_ALLOC_RANGE + pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, + (range[i + 1] >> 12) - 1); + mt_dump(mt); +#endif + check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, + xa_mk_value(range[i] >> 12), 0); + mt_validate(mt); + } + + + MA_STATE(mas, mt, 0, 0); + + for (i = 0; i < ARRAY_SIZE(holes); i += 3) { + +#if DEBUG_ALLOC_RANGE + pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, + holes[i+1] >> 12, holes[i+2] >> 12, + min, holes[i+1]); +#endif + MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, + holes[i+1] >> 12, + holes[i+2] >> 12)); + MT_BUG_ON(mt, mas.index != holes[i] >> 12); + min = holes[i+1]; + mas_reset(&mas); + } + for (i = 0; i < req_range_count; i += 5) { +#if DEBUG_ALLOC_RANGE + pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", + i/5, req_range[i] >> 12, req_range[i + 1] >> 12, + req_range[i + 2] >> 12, req_range[i + 3] >> 12, + req_range[i], req_range[i+1]); +#endif + check_mtree_alloc_range(mt, + req_range[i] >> 12, /* start */ + req_range[i+1] >> 12, /* end */ + req_range[i+2] >> 12, /* size */ + req_range[i+3] >> 12, /* expected address */ + req_range[i+4], /* expected return */ + xa_mk_value(req_range[i] >> 12)); /* pointer */ + mt_validate(mt); +#if DEBUG_ALLOC_RANGE + mt_dump(mt); +#endif + } + + mtree_destroy(mt); +} + +static noinline void check_ranges(struct maple_tree *mt) +{ + int i, val, val2; + unsigned long r[] = { + 10, 15, + 20, 25, + 17, 22, /* Overlaps previous range. */ + 9, 1000, /* Huge. */ + 100, 200, + 45, 168, + 118, 128, + }; + + MT_BUG_ON(mt, !mtree_empty(mt)); + check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); + check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); + check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); + MT_BUG_ON(mt, !mt_height(mt)); + /* Store */ + check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); + check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); + check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + MT_BUG_ON(mt, mt_height(mt)); + + check_seq(mt, 50, false); + mt_set_non_kernel(4); + check_store_range(mt, 5, 47, xa_mk_value(47), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + /* Create tree of 1-100 */ + check_seq(mt, 100, false); + /* Store 45-168 */ + mt_set_non_kernel(10); + check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + /* Create tree of 1-200 */ + check_seq(mt, 200, false); + /* Store 45-168 */ + check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + check_seq(mt, 30, false); + check_store_range(mt, 6, 18, xa_mk_value(6), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + /* Overwrite across multiple levels. */ + /* Create tree of 1-400 */ + check_seq(mt, 400, false); + mt_set_non_kernel(50); + /* Store 118-128 */ + check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); + mt_set_non_kernel(50); + mtree_test_erase(mt, 140); + mtree_test_erase(mt, 141); + mtree_test_erase(mt, 142); + mtree_test_erase(mt, 143); + mtree_test_erase(mt, 130); + mtree_test_erase(mt, 131); + mtree_test_erase(mt, 132); + mtree_test_erase(mt, 133); + mtree_test_erase(mt, 134); + mtree_test_erase(mt, 135); + check_load(mt, r[12], xa_mk_value(r[12])); + check_load(mt, r[13], xa_mk_value(r[12])); + check_load(mt, r[13] - 1, xa_mk_value(r[12])); + check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); + check_load(mt, 135, NULL); + check_load(mt, 140, NULL); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + + + /* Overwrite multiple levels at the end of the tree (slot 7) */ + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 353, 361, xa_mk_value(353), 0); + check_store_range(mt, 347, 352, xa_mk_value(347), 0); + + check_load(mt, 346, xa_mk_value(346)); + for (i = 347; i <= 352; i++) + check_load(mt, i, xa_mk_value(347)); + for (i = 353; i <= 361; i++) + check_load(mt, i, xa_mk_value(353)); + check_load(mt, 362, xa_mk_value(362)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 352, 364, NULL, 0); + check_store_range(mt, 351, 363, xa_mk_value(352), 0); + check_load(mt, 350, xa_mk_value(350)); + check_load(mt, 351, xa_mk_value(352)); + for (i = 352; i <= 363; i++) + check_load(mt, i, xa_mk_value(352)); + check_load(mt, 364, NULL); + check_load(mt, 365, xa_mk_value(365)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + mt_set_non_kernel(5); + check_seq(mt, 400, false); + check_store_range(mt, 352, 364, NULL, 0); + check_store_range(mt, 351, 364, xa_mk_value(352), 0); + check_load(mt, 350, xa_mk_value(350)); + check_load(mt, 351, xa_mk_value(352)); + for (i = 352; i <= 364; i++) + check_load(mt, i, xa_mk_value(352)); + check_load(mt, 365, xa_mk_value(365)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 362, 367, xa_mk_value(362), 0); + check_store_range(mt, 353, 361, xa_mk_value(353), 0); + mt_set_non_kernel(0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + /* + * Interesting cases: + * 1. Overwrite the end of a node and end in the first entry of the next + * node. + * 2. Split a single range + * 3. Overwrite the start of a range + * 4. Overwrite the end of a range + * 5. Overwrite the entire range + * 6. Overwrite a range that causes multiple parent nodes to be + * combined + * 7. Overwrite a range that causes multiple parent nodes and part of + * root to be combined + * 8. Overwrite the whole tree + * 9. Try to overwrite the zero entry of an alloc tree. + * 10. Write a range larger than a nodes current pivot + */ + + mt_set_non_kernel(50); + for (i = 0; i <= 500; i++) { + val = i*5; + val2 = (i+1)*5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); + check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); + check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); + check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); + check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); + mtree_destroy(mt); + mt_set_non_kernel(0); + + mt_set_non_kernel(50); + for (i = 0; i <= 500; i++) { + val = i*5; + val2 = (i+1)*5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); + check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); + check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); + check_store_range(mt, 2460, 2470, NULL, 0); + check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); + check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + /* Test rebalance gaps */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mt_set_non_kernel(50); + for (i = 0; i <= 50; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 161, 161, xa_mk_value(161), 0); + check_store_range(mt, 162, 162, xa_mk_value(162), 0); + check_store_range(mt, 163, 163, xa_mk_value(163), 0); + check_store_range(mt, 240, 249, NULL, 0); + mtree_erase(mt, 200); + mtree_erase(mt, 210); + mtree_erase(mt, 220); + mtree_erase(mt, 230); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 500; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 500; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); + check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); + check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); + check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); + check_store_range(mt, 4842, 4849, NULL, 0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 1300; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); + } + /* Cause a 3 child split all the way up the tree. */ + for (i = 5; i < 215; i += 10) + check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); + for (i = 5; i < 65; i += 10) + check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); + + MT_BUG_ON(mt, mt_height(mt) >= 4); + for (i = 5; i < 45; i += 10) + check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); + MT_BUG_ON(mt, mt_height(mt) < 4); + mtree_destroy(mt); + + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 1200; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); + } + /* Fill parents and leaves before split. */ + for (i = 5; i < 455; i += 10) + check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); + + for (i = 1; i < 16; i++) + check_store_range(mt, 8185 + i, 8185 + i + 1, + xa_mk_value(8185+i), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); + /* triple split across multiple levels. */ + check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); + MT_BUG_ON(mt, mt_height(mt) != 4); +} + +static noinline void check_next_entry(struct maple_tree *mt) +{ + void *entry = NULL; + unsigned long limit = 30, i = 0; + + MT_BUG_ON(mt, !mtree_empty(mt)); + MA_STATE(mas, mt, i, i); + + check_seq(mt, limit, false); + rcu_read_lock(); + + /* Check the first one and get ma_state in the correct state. */ + MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); + for ( ; i <= limit + 1; i++) { + entry = mas_next(&mas, limit); + if (i > limit) + MT_BUG_ON(mt, entry != NULL); + else + MT_BUG_ON(mt, xa_mk_value(i) != entry); + } + rcu_read_unlock(); + mtree_destroy(mt); +} + +static noinline void check_prev_entry(struct maple_tree *mt) +{ + unsigned long index = 16; + void *value; + int i; + + MA_STATE(mas, mt, index, index); + + MT_BUG_ON(mt, !mtree_empty(mt)); + check_seq(mt, 30, false); + + rcu_read_lock(); + value = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, value != xa_mk_value(index)); + value = mas_prev(&mas, 0); + MT_BUG_ON(mt, value != xa_mk_value(index - 1)); + rcu_read_unlock(); + mtree_destroy(mt); + + /* Check limits on prev */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_lock(&mas); + for (i = 0; i <= index; i++) { + mas_set_range(&mas, i*10, i*10+5); + mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); + } + + mas_set(&mas, 20); + value = mas_walk(&mas); + MT_BUG_ON(mt, value != xa_mk_value(2)); + + value = mas_prev(&mas, 19); + MT_BUG_ON(mt, value != NULL); + + mas_set(&mas, 80); + value = mas_walk(&mas); + MT_BUG_ON(mt, value != xa_mk_value(8)); + + value = mas_prev(&mas, 76); + MT_BUG_ON(mt, value != NULL); + + mas_unlock(&mas); +} + +static noinline void check_root_expand(struct maple_tree *mt) +{ + MA_STATE(mas, mt, 0, 0); + void *ptr; + + + mas_lock(&mas); + mas_set(&mas, 3); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != ULONG_MAX); + + ptr = &check_prev_entry; + mas_set(&mas, 1); + mas_store_gfp(&mas, ptr, GFP_KERNEL); + + mas_set(&mas, 0); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + + mas_set(&mas, 1); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != &check_prev_entry); + + mas_set(&mas, 2); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + mas_unlock(&mas); + mtree_destroy(mt); + + + mt_init_flags(mt, 0); + mas_lock(&mas); + + mas_set(&mas, 0); + ptr = &check_prev_entry; + mas_store_gfp(&mas, ptr, GFP_KERNEL); + + mas_set(&mas, 5); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, mas.index != 1); + MT_BUG_ON(mt, mas.last != ULONG_MAX); + + mas_set_range(&mas, 0, 100); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != &check_prev_entry); + MT_BUG_ON(mt, mas.last != 0); + mas_unlock(&mas); + mtree_destroy(mt); + + mt_init_flags(mt, 0); + mas_lock(&mas); + + mas_set(&mas, 0); + ptr = (void *)((unsigned long) check_prev_entry | 1UL); + mas_store_gfp(&mas, ptr, GFP_KERNEL); + ptr = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); + + mas_set(&mas, 1); + ptr = mas_prev(&mas, 0); + MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); + MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); + + mas_unlock(&mas); + + mtree_destroy(mt); + + mt_init_flags(mt, 0); + mas_lock(&mas); + mas_set(&mas, 0); + ptr = (void *)((unsigned long) check_prev_entry | 2UL); + mas_store_gfp(&mas, ptr, GFP_KERNEL); + ptr = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); + + mas_set(&mas, 1); + ptr = mas_prev(&mas, 0); + MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); + MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); + + + mas_unlock(&mas); +} + +static noinline void check_prealloc(struct maple_tree *mt) +{ + unsigned long i, max = 100; + unsigned long allocated; + unsigned char height; + struct maple_node *mn; + void *ptr = check_prealloc; + MA_STATE(mas, mt, 10, 20); + + mt_set_non_kernel(1000); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + ma_free_rcu(mn); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + ma_free_rcu(mn); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mt_set_non_kernel(1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated != 0); + mas_destroy(&mas); + + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mt_set_non_kernel(1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated != 0); +} + +static noinline void check_spanning_write(struct maple_tree *mt) +{ + unsigned long i, max = 5000; + MA_STATE(mas, mt, 1200, 2380); + + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + for (i = 1; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mt_validate(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a right cousin rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test non-alloc tree spanning store */ + mt_init_flags(mt, 0); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 300); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 15); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a right sibling rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 12865); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 15); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a left sibling rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 90, 13665); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 95); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a left cousin rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 46805, 49995); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 46815); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* + * Test spanning store that requires a left cousin rebalance all the way + * to root + */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 32395, 49995); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 46815); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* + * Test spanning store that requires a right cousin rebalance all the + * way to root + */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mas_set_range(&mas, 38875, 43190); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 38900); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store ending at full node (depth 2)*/ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mtree_lock(mt); + mas_set(&mas, 47606); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47607); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47608); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47609); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + /* Ensure the parent node is full */ + mas_ascend(&mas); + MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); + mas_set_range(&mas, 11516, 48940); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning write with many levels of no siblings */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mas_set_range(&mas, 43200, 49999); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 43200); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 100; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_set_range(&mas, 76, 875); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mtree_unlock(mt); +} + +static noinline void check_null_expand(struct maple_tree *mt) +{ + unsigned long i, max = 100; + unsigned char data_end; + MA_STATE(mas, mt, 959, 959); + + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + /* Test expanding null at start. */ + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 959, 963); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); + MT_BUG_ON(mt, data_end != mas_data_end(&mas)); + + /* Test expanding null at end. */ + mas_set(&mas, 880); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 884, 887); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); + MT_BUG_ON(mt, data_end != mas_data_end(&mas)); + + /* Test expanding null at start and end. */ + mas_set(&mas, 890); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 900, 905); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); + MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); + + /* Test expanding null across multiple slots. */ + mas_set(&mas, 800); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 810, 825); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); + MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); +} + +static noinline void check_gap_combining(struct maple_tree *mt) +{ + struct maple_enode *mn1, *mn2; + void *entry; + + unsigned long seq100[] = { + /* 0-5 */ + 74, 75, 76, + 50, 100, 2, + + /* 6-12 */ + 44, 45, 46, 43, + 20, 50, 3, + + /* 13-20*/ + 80, 81, 82, + 76, 2, 79, 85, 4, + }; + unsigned long seq2000[] = { + 1152, 1151, + 1100, 1200, 2, + }; + unsigned long seq400[] = { + 286, 318, + 256, 260, 266, 270, 275, 280, 290, 398, + 286, 310, + }; + + unsigned long index = seq100[0]; + + MA_STATE(mas, mt, index, index); + + MT_BUG_ON(mt, !mtree_empty(mt)); + check_seq(mt, 100, false); /* create 100 singletons. */ + + mt_set_non_kernel(1); + mtree_test_erase(mt, seq100[2]); + check_load(mt, seq100[2], NULL); + mtree_test_erase(mt, seq100[1]); + check_load(mt, seq100[1], NULL); + + rcu_read_lock(); + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index)); + mn1 = mas.node; + mas_next(&mas, ULONG_MAX); + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); + mn2 = mas.node; + MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ + + /* + * At this point, there is a gap of 2 at index + 1 between seq100[3] and + * seq100[4]. Search for the gap. + */ + mt_set_non_kernel(1); + mas_reset(&mas); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], + seq100[5])); + MT_BUG_ON(mt, mas.index != index + 1); + rcu_read_unlock(); + + mtree_test_erase(mt, seq100[6]); + check_load(mt, seq100[6], NULL); + mtree_test_erase(mt, seq100[7]); + check_load(mt, seq100[7], NULL); + mtree_test_erase(mt, seq100[8]); + index = seq100[9]; + + rcu_read_lock(); + mas.index = index; + mas.last = index; + mas_reset(&mas); + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index)); + mn1 = mas.node; + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); + mas_next(&mas, ULONG_MAX); /* go to the next entry. */ + mn2 = mas.node; + MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ + + /* + * At this point, there is a gap of 3 at seq100[6]. Find it by + * searching 20 - 50 for size 3. + */ + mas_reset(&mas); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], + seq100[12])); + MT_BUG_ON(mt, mas.index != seq100[6]); + rcu_read_unlock(); + + mt_set_non_kernel(1); + mtree_store(mt, seq100[13], NULL, GFP_KERNEL); + check_load(mt, seq100[13], NULL); + check_load(mt, seq100[14], xa_mk_value(seq100[14])); + mtree_store(mt, seq100[14], NULL, GFP_KERNEL); + check_load(mt, seq100[13], NULL); + check_load(mt, seq100[14], NULL); + + mas_reset(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], + seq100[17])); + MT_BUG_ON(mt, mas.index != seq100[13]); + mt_validate(mt); + rcu_read_unlock(); + + /* + * *DEPRECATED: no retries anymore* Test retry entry in the start of a + * gap. + */ + mt_set_non_kernel(2); + mtree_test_store_range(mt, seq100[18], seq100[14], NULL); + mtree_test_erase(mt, seq100[15]); + mas_reset(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], + seq100[20])); + rcu_read_unlock(); + MT_BUG_ON(mt, mas.index != seq100[18]); + mt_validate(mt); + mtree_destroy(mt); + + /* seq 2000 tests are for multi-level tree gaps */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 2000, false); + mt_set_non_kernel(1); + mtree_test_erase(mt, seq2000[0]); + mtree_test_erase(mt, seq2000[1]); + + mt_set_non_kernel(2); + mas_reset(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], + seq2000[4])); + MT_BUG_ON(mt, mas.index != seq2000[1]); + rcu_read_unlock(); + mt_validate(mt); + mtree_destroy(mt); + + /* seq 400 tests rebalancing over two levels. */ + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 400, false); + mtree_test_store_range(mt, seq400[0], seq400[1], NULL); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 400, false); + mt_set_non_kernel(50); + mtree_test_store_range(mt, seq400[2], seq400[9], + xa_mk_value(seq400[2])); + mtree_test_store_range(mt, seq400[3], seq400[9], + xa_mk_value(seq400[3])); + mtree_test_store_range(mt, seq400[4], seq400[9], + xa_mk_value(seq400[4])); + mtree_test_store_range(mt, seq400[5], seq400[9], + xa_mk_value(seq400[5])); + mtree_test_store_range(mt, seq400[0], seq400[9], + xa_mk_value(seq400[0])); + mtree_test_store_range(mt, seq400[6], seq400[9], + xa_mk_value(seq400[6])); + mtree_test_store_range(mt, seq400[7], seq400[9], + xa_mk_value(seq400[7])); + mtree_test_store_range(mt, seq400[8], seq400[9], + xa_mk_value(seq400[8])); + mtree_test_store_range(mt, seq400[10], seq400[11], + xa_mk_value(seq400[10])); + mt_validate(mt); + mt_set_non_kernel(0); + mtree_destroy(mt); +} +static noinline void check_node_overwrite(struct maple_tree *mt) +{ + int i, max = 4000; + + for (i = 0; i < max; i++) + mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); + + mtree_test_store_range(mt, 319951, 367950, NULL); + /*mt_dump(mt); */ + mt_validate(mt); +} + +static void mas_dfs_preorder(struct ma_state *mas) +{ + + struct maple_enode *prev; + unsigned char end, slot = 0; + + if (mas_is_start(mas)) { + mas_start(mas); + return; + } + + if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) + goto done; + +walk_up: + end = mas_data_end(mas); + if (mte_is_leaf(mas->node) || + (slot > end)) { + if (mte_is_root(mas->node)) + goto done; + + slot = mte_parent_slot(mas->node) + 1; + mas_ascend(mas); + goto walk_up; + } + + prev = mas->node; + mas->node = mas_get_slot(mas, slot); + if (!mas->node || slot > end) { + if (mte_is_root(prev)) + goto done; + + mas->node = prev; + slot = mte_parent_slot(mas->node) + 1; + mas_ascend(mas); + goto walk_up; + } + + return; +done: + mas->node = MAS_NONE; +} + + +static void check_dfs_preorder(struct maple_tree *mt) +{ + unsigned long count = 0, max = 1000; + + MA_STATE(mas, mt, 0, 0); + + check_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + MT_BUG_ON(mt, count != 74); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + count = 0; + check_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + /*printk("count %lu\n", count); */ + MT_BUG_ON(mt, count != 77); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + count = 0; + check_rev_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + /*printk("count %lu\n", count); */ + MT_BUG_ON(mt, count != 77); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + mt_zero_nr_tallocated(); + mt_set_non_kernel(200); + mas_expected_entries(&mas, max); + for (count = 0; count <= max; count++) { + mas.index = mas.last = count; + mas_store(&mas, xa_mk_value(count)); + MT_BUG_ON(mt, mas_is_err(&mas)); + } + mas_destroy(&mas); + rcu_barrier(); + /* + * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", + * max, mt_get_alloc_size()/1024, mt_nr_allocated(), + * mt_nr_tallocated()); + */ + +} + +#if defined(BENCH_SLOT_STORE) +static noinline void bench_slot_store(struct maple_tree *mt) +{ + int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; + + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < count; i++) { + mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); + mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), + GFP_KERNEL); + } +} +#endif + +#if defined(BENCH_NODE_STORE) +static noinline void bench_node_store(struct maple_tree *mt) +{ + int i, overwrite = 76, max = 240, count = 20000000; + + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < count; i++) { + mtree_store_range(mt, overwrite, overwrite + 15, + xa_mk_value(overwrite), GFP_KERNEL); + + overwrite += 5; + if (overwrite >= 135) + overwrite = 76; + } +} +#endif + +#if defined(BENCH_AWALK) +static noinline void bench_awalk(struct maple_tree *mt) +{ + int i, max = 2500, count = 50000000; + MA_STATE(mas, mt, 1470, 1470); + + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + + mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); + + for (i = 0; i < count; i++) { + mas_empty_area_rev(&mas, 0, 2000, 10); + mas_reset(&mas); + } +} +#endif +#if defined(BENCH_WALK) +static noinline void bench_walk(struct maple_tree *mt) +{ + int i, max = 2500, count = 550000000; + MA_STATE(mas, mt, 1470, 1470); + + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < count; i++) { + mas_walk(&mas); + mas_reset(&mas); + } + +} +#endif + +#if defined(BENCH_MT_FOR_EACH) +static noinline void bench_mt_for_each(struct maple_tree *mt) +{ + int i, count = 1000000; + unsigned long max = 2500, index = 0; + void *entry; + + for (i = 0; i < max; i += 5) + mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < count; i++) { + unsigned long j = 0; + + mt_for_each(mt, entry, index, max) { + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j += 5; + } + + index = 0; + } + +} +#endif + +static noinline void check_forking(struct maple_tree *mt) +{ + + struct maple_tree newmt; + int i, nr_entries = 134; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + mas_reset(&newmas); + mas_reset(&mas); + mas.index = 0; + mas.last = 0; + if (mas_expected_entries(&newmas, nr_entries)) { + pr_err("OOM!"); + BUG_ON(1); + } + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store(&newmas, val); + } + mas_destroy(&newmas); + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); +} + +static noinline void check_mas_store_gfp(struct maple_tree *mt) +{ + + struct maple_tree newmt; + int i, nr_entries = 135; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + mas_reset(&newmas); + mas_set(&mas, 0); + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store_gfp(&newmas, val, GFP_KERNEL); + } + + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); +} + +#if defined(BENCH_FORK) +static noinline void bench_forking(struct maple_tree *mt) +{ + + struct maple_tree newmt; + int i, nr_entries = 134, nr_fork = 80000; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < nr_fork; i++) { + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + mas_reset(&newmas); + mas_reset(&mas); + mas.index = 0; + mas.last = 0; + if (mas_expected_entries(&newmas, nr_entries)) { + printk("OOM!"); + BUG_ON(1); + } + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store(&newmas, val); + } + mas_destroy(&newmas); + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); + } +} +#endif + +static noinline void next_prev_test(struct maple_tree *mt) +{ + int i, nr_entries = 200; + void *val; + MA_STATE(mas, mt, 0, 0); + struct maple_enode *mn; + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i <= nr_entries / 2; i++) { + mas_next(&mas, 1000); + if (mas_is_none(&mas)) + break; + + } + mas_reset(&mas); + mas_set(&mas, 0); + i = 0; + mas_for_each(&mas, val, 1000) { + i++; + } + + mas_reset(&mas); + mas_set(&mas, 0); + i = 0; + mas_for_each(&mas, val, 1000) { + mas_pause(&mas); + i++; + } + + /* + * 680 - 685 = 0x61a00001930c + * 686 - 689 = NULL; + * 690 - 695 = 0x61a00001930c + * Check simple next/prev + */ + mas_set(&mas, 686); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != NULL); + + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); + MT_BUG_ON(mt, mas.index != 690); + MT_BUG_ON(mt, mas.last != 695); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); + MT_BUG_ON(mt, mas.index != 680); + MT_BUG_ON(mt, mas.last != 685); + + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); + MT_BUG_ON(mt, mas.index != 690); + MT_BUG_ON(mt, mas.last != 695); + + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); + MT_BUG_ON(mt, mas.index != 700); + MT_BUG_ON(mt, mas.last != 705); + + /* Check across node boundaries of the tree */ + mas_set(&mas, 70); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); + MT_BUG_ON(mt, mas.index != 70); + MT_BUG_ON(mt, mas.last != 75); + + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); + MT_BUG_ON(mt, mas.index != 80); + MT_BUG_ON(mt, mas.last != 85); + + val = mas_prev(&mas, 70); + MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); + MT_BUG_ON(mt, mas.index != 70); + MT_BUG_ON(mt, mas.last != 75); + + /* Check across two levels of the tree */ + mas_reset(&mas); + mas_set(&mas, 707); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != NULL); + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); + MT_BUG_ON(mt, mas.index != 710); + MT_BUG_ON(mt, mas.last != 715); + mn = mas.node; + + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(720 / 10)); + MT_BUG_ON(mt, mas.index != 720); + MT_BUG_ON(mt, mas.last != 725); + MT_BUG_ON(mt, mn == mas.node); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); + MT_BUG_ON(mt, mas.index != 710); + MT_BUG_ON(mt, mas.last != 715); + + /* Check running off the end and back on */ + mas_reset(&mas); + mas_set(&mas, 2000); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); + MT_BUG_ON(mt, mas.index != 2000); + MT_BUG_ON(mt, mas.last != 2005); + + val = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != ULONG_MAX); + MT_BUG_ON(mt, mas.last != ULONG_MAX); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); + MT_BUG_ON(mt, mas.index != 2000); + MT_BUG_ON(mt, mas.last != 2005); + + /* Check running off the start and back on */ + mas_reset(&mas); + mas_set(&mas, 10); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(1)); + MT_BUG_ON(mt, mas.index != 10); + MT_BUG_ON(mt, mas.last != 15); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(0)); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 5); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0); + + mas.index = 0; + mas.last = 5; + mas_store(&mas, NULL); + mas_reset(&mas); + mas_set(&mas, 10); + mas_walk(&mas); + + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0); + + mtree_destroy(mt); + + mt_init(mt); + mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); + mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); + mas_set(&mas, 5); + val = mas_prev(&mas, 4); + MT_BUG_ON(mt, val != NULL); +} + +#define RCU_RANGE_COUNT 1000 +#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);} +struct rcu_test_struct2 { + struct maple_tree *mt; + + bool start; + bool stop; + unsigned int thread_count; + + unsigned int seen_toggle; + unsigned int seen_added; + unsigned int seen_modified; + unsigned int seen_deleted; + int pause; + + unsigned long index[RCU_RANGE_COUNT]; + unsigned long last[RCU_RANGE_COUNT]; +}; + +struct rcu_reader_struct { + unsigned int id; + int mod; + int del; + int flip; + int add; + int next; + struct rcu_test_struct2 *test; +}; + +/* RCU reader helper function */ +static void rcu_reader_register(struct rcu_test_struct2 *test) +{ + rcu_register_thread(); + uatomic_inc(&test->thread_count); + + while (!test->start) + usleep(test->pause * 100); +} + +static void rcu_reader_setup(struct rcu_reader_struct *reader, + unsigned int id, struct rcu_test_struct2 *test) +{ + reader->id = id; + reader->test = test; + reader->mod = reader->id % 10; + reader->del = (reader->mod + 1) % 10; + reader->flip = (reader->mod + 2) % 10; + reader->add = (reader->mod + 3) % 10; + reader->next = (reader->mod + 4) % 10; +} + +/* RCU reader in increasing index */ +static void *rcu_reader_fwd(void *ptr) +{ + struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; + struct rcu_test_struct2 *test = reader->test; + unsigned long index = reader->id; + bool toggled, modified, deleted, added; + int i; + void *entry, *prev = NULL; + MA_STATE(mas, test->mt, 0, 0); + + rcu_reader_register(test); + toggled = modified = deleted = added = false; + + while (!test->stop) { + i = 0; + /* mas_for_each ?*/ + rcu_read_lock(); + mas_set(&mas, test->index[index]); + mas_for_each(&mas, entry, test->last[index + 9]) { + unsigned long r_start, r_end, alt_start; + void *expected, *alt; + + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + + if (i == reader->del) { + if (!deleted) { + alt_start = test->index[index + reader->flip]; + /* delete occurred. */ + if (mas.index == alt_start) { + uatomic_inc(&test->seen_deleted); + deleted = true; + } + } + if (deleted) { + i = reader->flip; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + if (!added && (i == reader->add)) { + alt_start = test->index[index + reader->next]; + if (mas.index == r_start) { + uatomic_inc(&test->seen_added); + added = true; + } else if (mas.index == alt_start) { + i = reader->next; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + RCU_MT_BUG_ON(test, mas.index != r_start); + RCU_MT_BUG_ON(test, mas.last != r_end); + + if (i == reader->flip) { + alt = xa_mk_value(index + i + RCU_RANGE_COUNT); + if (prev) { + if (toggled && entry == expected) + uatomic_inc(&test->seen_toggle); + else if (!toggled && entry == alt) + uatomic_inc(&test->seen_toggle); + } + + if (entry == expected) + toggled = false; + else if (entry == alt) + toggled = true; + else { + printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); + RCU_MT_BUG_ON(test, 1); + } + + prev = entry; + } else if (i == reader->mod) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + if (entry != expected) { + if (!modified) + uatomic_inc(&test->seen_modified); + modified = true; + } else { + if (modified) + uatomic_inc(&test->seen_modified); + modified = false; + } + + if (modified) + RCU_MT_BUG_ON(test, entry != alt); + + } else { + if (entry != expected) + printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); + RCU_MT_BUG_ON(test, entry != expected); + } + + i++; + } + rcu_read_unlock(); + usleep(test->pause); + } + + rcu_unregister_thread(); + return NULL; +} + +/* RCU reader in decreasing index */ +static void *rcu_reader_rev(void *ptr) +{ + struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; + struct rcu_test_struct2 *test = reader->test; + unsigned long index = reader->id; + bool toggled, modified, deleted, added; + int i; + void *prev = NULL; + MA_STATE(mas, test->mt, 0, 0); + + rcu_reader_register(test); + toggled = modified = deleted = added = false; + + + while (!test->stop) { + void *entry; + + i = 9; + mas_set(&mas, test->index[index + i]); + + rcu_read_lock(); + while (i--) { + unsigned long r_start, r_end, alt_start; + void *expected, *alt; + int line = __LINE__; + + entry = mas_prev(&mas, test->index[index]); + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + + if (i == reader->del) { + alt_start = test->index[index + reader->mod]; + if (mas.index == alt_start) { + line = __LINE__; + if (!deleted) + uatomic_inc(&test->seen_deleted); + deleted = true; + } + if (deleted) { + line = __LINE__; + i = reader->mod; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + if (!added && (i == reader->add)) { + alt_start = test->index[index + reader->flip]; + if (mas.index == r_start) { + line = __LINE__; + uatomic_inc(&test->seen_added); + added = true; + } else if (mas.index == alt_start) { + line = __LINE__; + i = reader->flip; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + if (i == reader->mod) + line = __LINE__; + else if (i == reader->flip) + line = __LINE__; + + if (mas.index != r_start) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + mt_dump(test->mt); + printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", + mas.index, mas.last, entry, + r_start, r_end, expected, alt, + line, i); + } + RCU_MT_BUG_ON(test, mas.index != r_start); + RCU_MT_BUG_ON(test, mas.last != r_end); + + if (i == reader->mod) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + + if (entry != expected) { + if (!modified) + uatomic_inc(&test->seen_modified); + modified = true; + } else { + if (modified) + uatomic_inc(&test->seen_modified); + modified = false; + } + if (modified) + RCU_MT_BUG_ON(test, entry != alt); + + + } else if (i == reader->flip) { + alt = xa_mk_value(index + i + + RCU_RANGE_COUNT); + if (prev) { + if (toggled && entry == expected) + uatomic_inc(&test->seen_toggle); + else if (!toggled && entry == alt) + uatomic_inc(&test->seen_toggle); + } + + if (entry == expected) + toggled = false; + else if (entry == alt) + toggled = true; + else { + printk("%lu-%lu %p != %p or %p\n", + mas.index, mas.last, entry, + expected, alt); + RCU_MT_BUG_ON(test, 1); + } + + prev = entry; + } else { + if (entry != expected) + printk("%lu-%lu %p != %p\n", mas.index, + mas.last, entry, expected); + RCU_MT_BUG_ON(test, entry != expected); + } + } + rcu_read_unlock(); + usleep(test->pause); + } + + rcu_unregister_thread(); + return NULL; +} + +static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, + int count, struct rcu_reader_struct *test_reader) +{ + int i, j = 10000; + bool toggle = true; + + test->start = true; /* Release the hounds! */ + usleep(5); + + while (j--) { + toggle = !toggle; + i = count; + while (i--) { + unsigned long start, end; + struct rcu_reader_struct *this = &test_reader[i]; + + /* Mod offset */ + if (j == 600) { + start = test->index[this->id + this->mod]; + end = test->last[this->id + this->mod]; + mtree_store_range(mt, start, end, + xa_mk_value(this->id + this->mod * 2 + + 1 + RCU_RANGE_COUNT), + GFP_KERNEL); + } + + /* Toggle */ + if (!(j % 5)) { + start = test->index[this->id + this->flip]; + end = test->last[this->id + this->flip]; + mtree_store_range(mt, start, end, + xa_mk_value((toggle ? start : + this->id + this->flip + + RCU_RANGE_COUNT)), + GFP_KERNEL); + } + + /* delete */ + if (j == 400) { + start = test->index[this->id + this->del]; + end = test->last[this->id + this->del]; + mtree_store_range(mt, start, end, NULL, GFP_KERNEL); + } + + /* add */ + if (j == 500) { + start = test->index[this->id + this->add]; + end = test->last[this->id + this->add]; + mtree_store_range(mt, start, end, + xa_mk_value(start), GFP_KERNEL); + } + } + usleep(test->pause); + /* If a test fails, don't flood the console */ + if (test->stop) + break; + } +} + +static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, + int count, struct rcu_reader_struct *test_reader) +{ + int j, i; + bool toggle = true; + + test->start = true; /* Release the hounds! */ + usleep(5); + for (j = 0; j < 10000; j++) { + toggle = !toggle; + for (i = 0; i < count; i++) { + unsigned long start, end; + struct rcu_reader_struct *this = &test_reader[i]; + + /* Mod offset */ + if (j == 600) { + start = test->index[this->id + this->mod]; + end = test->last[this->id + this->mod]; + mtree_store_range(mt, start, end, + xa_mk_value(this->id + this->mod * 2 + + 1 + RCU_RANGE_COUNT), + GFP_KERNEL); + } + + /* Toggle */ + if (!(j % 5)) { + start = test->index[this->id + this->flip]; + end = test->last[this->id + this->flip]; + mtree_store_range(mt, start, end, + xa_mk_value((toggle ? start : + this->id + this->flip + + RCU_RANGE_COUNT)), + GFP_KERNEL); + } + + /* delete */ + if (j == 400) { + start = test->index[this->id + this->del]; + end = test->last[this->id + this->del]; + mtree_store_range(mt, start, end, NULL, GFP_KERNEL); + } + + /* add */ + if (j == 500) { + start = test->index[this->id + this->add]; + end = test->last[this->id + this->add]; + mtree_store_range(mt, start, end, + xa_mk_value(start), GFP_KERNEL); + } + } + usleep(test->pause); + /* If a test fails, don't flood the console */ + if (test->stop) + break; + } +} + +/* + * This is to check: + * 1. Range that is not ever present + * 2. Range that is always present + * 3. Things being added but not removed. + * 4. Things being removed but not added. + * 5. Things are being added and removed, searches my succeed or fail + * + * This sets up two readers for every 10 entries; one forward and one reverse + * reading. + */ +static void rcu_stress(struct maple_tree *mt, bool forward) +{ + unsigned int count, i; + unsigned long r, seed; + pthread_t readers[RCU_RANGE_COUNT / 5]; + struct rcu_test_struct2 test; + struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; + void *(*function)(void *); + + /* Test setup */ + test.mt = mt; + test.pause = 5; + test.seen_toggle = 0; + test.seen_deleted = 0; + test.seen_added = 0; + test.seen_modified = 0; + test.thread_count = 0; + test.start = test.stop = false; + seed = time(NULL); + srand(seed); + for (i = 0; i < RCU_RANGE_COUNT; i++) { + r = seed + rand(); + mtree_store_range(mt, seed, r, + xa_mk_value(seed), GFP_KERNEL); + + /* Record start and end of entry */ + test.index[i] = seed; + test.last[i] = r; + seed = 1 + r + rand() % 10; + } + + i = count = ARRAY_SIZE(readers); + while (i--) { + unsigned long id; + + id = i / 2 * 10; + if (i % 2) + function = rcu_reader_fwd; + else + function = rcu_reader_rev; + + rcu_reader_setup(&test_reader[i], id, &test); + if (pthread_create(&readers[i], NULL, *function, + &test_reader[i])) { + perror("creating reader thread"); + exit(1); + } + } + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + struct rcu_reader_struct *this = &test_reader[i]; + int add = this->id + this->add; + + /* Remove add entries from the tree for later addition */ + mtree_store_range(mt, test.index[add], test.last[add], + NULL, GFP_KERNEL); + } + + mt_set_in_rcu(mt); + do { + usleep(5); + } while (test.thread_count > ARRAY_SIZE(readers)); + + if (forward) + rcu_stress_fwd(mt, &test, count, test_reader); + else + rcu_stress_rev(mt, &test, count, test_reader); + + test.stop = true; + while (count--) + pthread_join(readers[count], NULL); + + mt_validate(mt); +} + + +struct rcu_test_struct { + struct maple_tree *mt; /* the maple tree */ + int count; /* Number of times to check value(s) */ + unsigned long index; /* The first index to check */ + void *entry1; /* The first entry value */ + void *entry2; /* The second entry value */ + void *entry3; /* The third entry value */ + + bool update_2; + bool update_3; + unsigned long range_start; + unsigned long range_end; + unsigned int loop_sleep; + unsigned int val_sleep; + + unsigned int failed; /* failed detection for other threads */ + unsigned int seen_entry2; /* Number of threads that have seen the new value */ + unsigned int seen_entry3; /* Number of threads that have seen the new value */ + unsigned int seen_both; /* Number of threads that have seen both new values */ + unsigned int seen_toggle; + unsigned int seen_added; + unsigned int seen_removed; + unsigned long last; /* The end of the range to write. */ + + unsigned long removed; /* The index of the removed entry */ + unsigned long added; /* The index of the removed entry */ + unsigned long toggle; /* The index of the removed entry */ +}; + +static inline +int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, + bool *update_3) +{ + if (entry == test->entry1) + return 0; + + if (entry == test->entry2) { + if (!(*update_2)) { + uatomic_inc(&test->seen_entry2); + *update_2 = true; + if (update_3) + uatomic_inc(&test->seen_both); + } + return 0; + } + + if (entry == test->entry3) { + if (!(*update_3)) { + uatomic_inc(&test->seen_entry3); + *update_3 = true; + if (update_2) + uatomic_inc(&test->seen_both); + } + return 0; + } + + return 1; +} + +/* + * rcu_val() - Read a given value in the tree test->count times using the + * regular API + * + * @ptr: The pointer to the rcu_test_struct + */ +static void *rcu_val(void *ptr) +{ + struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; + unsigned long count = test->count; + bool update_2 = false; + bool update_3 = false; + void *entry; + + rcu_register_thread(); + while (count--) { + usleep(test->val_sleep); + /* + * No locking required, regular API locking is handled in the + * maple tree code + */ + entry = mtree_load(test->mt, test->index); + MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, + &update_3)); + } + rcu_unregister_thread(); + return NULL; +} + +/* + * rcu_loop() - Loop over a section of the maple tree, checking for an expected + * value using the advanced API + * + * @ptr - The pointer to the rcu_test_struct + */ +static void *rcu_loop(void *ptr) +{ + struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; + unsigned long count = test->count; + void *entry, *expected; + bool update_2 = false; + bool update_3 = false; + MA_STATE(mas, test->mt, test->range_start, test->range_start); + + rcu_register_thread(); + + /* + * Loop through the test->range_start - test->range_end test->count + * times + */ + while (count--) { + usleep(test->loop_sleep); + rcu_read_lock(); + mas_for_each(&mas, entry, test->range_end) { + /* The expected value is based on the start range. */ + expected = xa_mk_value(mas.index ? mas.index / 10 : 0); + + /* Out of the interesting range */ + if (mas.index < test->index || mas.index > test->last) { + if (entry != expected) { + printk("%lx - %lx = %p not %p\n", + mas.index, mas.last, entry, expected); + } + MT_BUG_ON(test->mt, entry != expected); + continue; + } + + if (entry == expected) + continue; /* Not seen. */ + + /* In the interesting range */ + MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, + &update_2, + &update_3)); + } + rcu_read_unlock(); + mas_set(&mas, test->range_start); + } + + rcu_unregister_thread(); + return NULL; +} + +static noinline +void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) +{ + + int i; + void *(*function)(void *); + pthread_t readers[20]; + + mt_set_in_rcu(mt); + MT_BUG_ON(mt, !mt_in_rcu(mt)); + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + if (i % 2) + function = rcu_loop; + else + function = rcu_val; + + if (pthread_create(&readers[i], NULL, *function, vals)) { + perror("creating reader thread"); + exit(1); + } + } + + usleep(5); /* small yield to ensure all threads are at least started. */ + mtree_store_range(mt, vals->index, vals->last, vals->entry2, + GFP_KERNEL); + while (i--) + pthread_join(readers[i], NULL); + + /* Make sure the test caught at least one update. */ + MT_BUG_ON(mt, !vals->seen_entry2); +} + +static noinline +void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) +{ + + int i; + void *(*function)(void *); + pthread_t readers[20]; + unsigned int index = vals->index; + + mt_set_in_rcu(mt); + MT_BUG_ON(mt, !mt_in_rcu(mt)); + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + if (i % 2) + function = rcu_loop; + else + function = rcu_val; + + if (pthread_create(&readers[i], NULL, *function, vals)) { + perror("creating reader thread"); + exit(1); + } + } + + usleep(5); /* small yield to ensure all threads are at least started. */ + + while (index <= vals->last) { + mtree_store(mt, index, + (index % 2 ? vals->entry2 : vals->entry3), + GFP_KERNEL); + index++; + usleep(5); + } + + while (i--) + pthread_join(readers[i], NULL); + + /* Make sure the test caught at least one update. */ + MT_BUG_ON(mt, !vals->seen_entry2); + MT_BUG_ON(mt, !vals->seen_entry3); + MT_BUG_ON(mt, !vals->seen_both); +} +static noinline void check_rcu_simulated(struct maple_tree *mt) +{ + unsigned long i, nr_entries = 1000; + unsigned long target = 4320; + unsigned long val = 0xDEAD; + + MA_STATE(mas_writer, mt, 0, 0); + MA_STATE(mas_reader, mt, target, target); + + rcu_register_thread(); + + mt_set_in_rcu(mt); + mas_lock(&mas_writer); + for (i = 0; i <= nr_entries; i++) { + mas_writer.index = i * 10; + mas_writer.last = i * 10 + 5; + mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); + } + mas_unlock(&mas_writer); + + /* Overwrite one entry with a new value. */ + mas_set_range(&mas_writer, target, target + 5); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + + /* Overwrite 1/2 the entry */ + mas_set_range(&mas_writer, target, target + 2); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite last 1/2 the entry */ + mas_set_range(&mas_writer, target + 2, target + 5); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + rcu_read_unlock(); + + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite more than the entry */ + mas_set_range(&mas_writer, target - 5, target + 15); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite more than the node. */ + mas_set_range(&mas_writer, target - 400, target + 400); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite the tree */ + mas_set_range(&mas_writer, 0, ULONG_MAX); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Clear out tree & recreate it */ + mas_lock(&mas_writer); + mas_set_range(&mas_writer, 0, ULONG_MAX); + mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); + mas_set_range(&mas_writer, 0, 0); + for (i = 0; i <= nr_entries; i++) { + mas_writer.index = i * 10; + mas_writer.last = i * 10 + 5; + mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); + } + mas_unlock(&mas_writer); + + /* next check */ + /* Overwrite one entry with a new value. */ + mas_reset(&mas_reader); + mas_set_range(&mas_writer, target, target + 5); + mas_set_range(&mas_reader, target, target); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_prev(&mas_reader, 0); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + + /* prev check */ + /* Overwrite one entry with a new value. */ + mas_reset(&mas_reader); + mas_set_range(&mas_writer, target, target + 5); + mas_set_range(&mas_reader, target, target); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_next(&mas_reader, ULONG_MAX); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); + rcu_read_unlock(); + + rcu_unregister_thread(); +} + +static noinline void check_rcu_threaded(struct maple_tree *mt) +{ + unsigned long i, nr_entries = 1000; + struct rcu_test_struct vals; + + vals.val_sleep = 200; + vals.loop_sleep = 110; + + rcu_register_thread(); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + /* Store across several slots. */ + vals.count = 1000; + vals.mt = mt; + vals.index = 8650; + vals.last = 8666; + vals.entry1 = xa_mk_value(865); + vals.entry2 = xa_mk_value(8650); + vals.entry3 = xa_mk_value(8650); + vals.range_start = 0; + vals.range_end = ULONG_MAX; + vals.seen_entry2 = 0; + vals.seen_entry3 = 0; + + run_check_rcu(mt, &vals); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + /* 4390-4395: value 439 (0x1b7) [0x36f] */ + /* Store across several slots. */ + /* Spanning store. */ + vals.count = 10000; + vals.mt = mt; + vals.index = 4390; + vals.last = 4398; + vals.entry1 = xa_mk_value(4390); + vals.entry2 = xa_mk_value(439); + vals.entry3 = xa_mk_value(439); + vals.seen_entry2 = 0; + vals.range_start = 4316; + vals.range_end = 5035; + run_check_rcu(mt, &vals); + mtree_destroy(mt); + + + /* Forward writer for rcu stress */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + rcu_stress(mt, true); + mtree_destroy(mt); + + /* Reverse writer for rcu stress */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + rcu_stress(mt, false); + mtree_destroy(mt); + + /* Slow reader test with spanning store. */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + /* 4390-4395: value 439 (0x1b7) [0x36f] */ + /* Store across several slots. */ + /* Spanning store. */ + vals.count = 15000; + vals.mt = mt; + vals.index = 4390; + vals.last = 4398; + vals.entry1 = xa_mk_value(4390); + vals.entry2 = xa_mk_value(439); + vals.entry3 = xa_mk_value(4391); + vals.seen_toggle = 0; + vals.seen_added = 0; + vals.seen_removed = 0; + vals.range_start = 4316; + vals.range_end = 5035; + vals.removed = 4360; + vals.added = 4396; + vals.toggle = 4347; + vals.val_sleep = 400; + vals.loop_sleep = 200; + vals.seen_entry2 = 0; + vals.seen_entry3 = 0; + vals.seen_both = 0; + vals.entry3 = xa_mk_value(438); + + run_check_rcu_slowread(mt, &vals); + rcu_unregister_thread(); +} + +extern void test_kmem_cache_bulk(void); + +/* Test spanning writes that require balancing right sibling or right cousin */ +static noinline void check_spanning_relatives(struct maple_tree *mt) +{ + + unsigned long i, nr_entries = 1000; + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + + mtree_store_range(mt, 9365, 9955, NULL, GFP_KERNEL); +} + +static noinline void check_fuzzer(struct maple_tree *mt) +{ + /* + * 1. Causes a spanning rebalance of a single root node. + * Fixed by setting the correct limit in mast_cp_to_nodes() when the + * entire right side is consumed. + */ + mtree_test_insert(mt, 88, (void *)0xb1); + mtree_test_insert(mt, 84, (void *)0xa9); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_insert(mt, 14, (void *)0x1d); + mtree_test_insert(mt, 7, (void *)0xf); + mtree_test_insert(mt, 12, (void *)0x19); + mtree_test_insert(mt, 18, (void *)0x25); + mtree_test_store_range(mt, 8, 18, (void *)0x11); + mtree_destroy(mt); + + + /* + * 2. Cause a spanning rebalance of two nodes in root. + * Fixed by setting mast->r->max correctly. + */ + mt_init_flags(mt, 0); + mtree_test_store(mt, 87, (void *)0xaf); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_load(mt, 4); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_store(mt, 8, (void *)0x11); + mtree_test_store(mt, 44, (void *)0x59); + mtree_test_store(mt, 68, (void *)0x89); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_insert(mt, 43, (void *)0x57); + mtree_test_insert(mt, 24, (void *)0x31); + mtree_test_insert(mt, 844, (void *)0x699); + mtree_test_store(mt, 84, (void *)0xa9); + mtree_test_store(mt, 4, (void *)0x9); + mtree_test_erase(mt, 4); + mtree_test_load(mt, 5); + mtree_test_erase(mt, 0); + mtree_destroy(mt); + + /* + * 3. Cause a node overflow on copy + * Fixed by using the correct check for node size in mas_wr_modify() + * Also discovered issue with metadata setting. + */ + mt_init_flags(mt, 0); + mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1); + mtree_test_store(mt, 4, (void *)0x9); + mtree_test_erase(mt, 5); + mtree_test_erase(mt, 0); + mtree_test_erase(mt, 4); + mtree_test_store(mt, 5, (void *)0xb); + mtree_test_erase(mt, 5); + mtree_test_store(mt, 5, (void *)0xb); + mtree_test_erase(mt, 5); + mtree_test_erase(mt, 4); + mtree_test_store(mt, 4, (void *)0x9); + mtree_test_store(mt, 444, (void *)0x379); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_load(mt, 0); + mtree_test_store(mt, 5, (void *)0xb); + mtree_test_erase(mt, 0); + mtree_destroy(mt); + + /* + * 4. spanning store failure due to writing incorrect pivot value at + * last slot. + * Fixed by setting mast->r->max correctly in mast_cp_to_nodes() + * + */ + mt_init_flags(mt, 0); + mtree_test_insert(mt, 261, (void *)0x20b); + mtree_test_store(mt, 516, (void *)0x409); + mtree_test_store(mt, 6, (void *)0xd); + mtree_test_insert(mt, 5, (void *)0xb); + mtree_test_insert(mt, 1256, (void *)0x9d1); + mtree_test_store(mt, 4, (void *)0x9); + mtree_test_erase(mt, 1); + mtree_test_store(mt, 56, (void *)0x71); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_store(mt, 24, (void *)0x31); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 2263, (void *)0x11af); + mtree_test_insert(mt, 446, (void *)0x37d); + mtree_test_store_range(mt, 6, 45, (void *)0xd); + mtree_test_store_range(mt, 3, 446, (void *)0x7); + mtree_destroy(mt); + + /* + * 5. mas_wr_extend_null() may overflow slots. + * Fix by checking against wr_mas->node_end. + */ + mt_init_flags(mt, 0); + mtree_test_store(mt, 48, (void *)0x61); + mtree_test_store(mt, 3, (void *)0x7); + mtree_test_load(mt, 0); + mtree_test_store(mt, 88, (void *)0xb1); + mtree_test_store(mt, 81, (void *)0xa3); + mtree_test_insert(mt, 0, (void *)0x1); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_insert(mt, 2480, (void *)0x1361); + mtree_test_insert(mt, 18446744073709551615UL, + (void *)0xffffffffffffffff); + mtree_test_erase(mt, 18446744073709551615UL); + mtree_destroy(mt); + + /* + * 6. When reusing a node with an implied pivot and the node is + * shrinking, old data would be left in the implied slot + * Fixed by checking the last pivot for the mas->max and clear + * accordingly. This only affected the left-most node as that node is + * the only one allowed to end in NULL. + */ + mt_init_flags(mt, 0); + mtree_test_erase(mt, 3); + mtree_test_insert(mt, 22, (void *)0x2d); + mtree_test_insert(mt, 15, (void *)0x1f); + mtree_test_load(mt, 2); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 5, (void *)0xb); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 3); + mtree_test_insert(mt, 22, (void *)0x2d); + mtree_test_insert(mt, 15, (void *)0x1f); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_load(mt, 2); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_insert(mt, 5, (void *)0xb); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 3); + mtree_test_insert(mt, 22, (void *)0x2d); + mtree_test_insert(mt, 15, (void *)0x1f); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_insert(mt, 12, (void *)0x19); + mtree_test_erase(mt, 1); + mtree_test_store_range(mt, 4, 62, (void *)0x9); + mtree_test_erase(mt, 62); + mtree_test_store_range(mt, 1, 0, (void *)0x3); + mtree_test_insert(mt, 11, (void *)0x17); + mtree_test_insert(mt, 3, (void *)0x7); + mtree_test_insert(mt, 3, (void *)0x7); + mtree_test_store(mt, 62, (void *)0x7d); + mtree_test_erase(mt, 62); + mtree_test_store_range(mt, 1, 15, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 22, (void *)0x2d); + mtree_test_insert(mt, 12, (void *)0x19); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 3, (void *)0x7); + mtree_test_store(mt, 62, (void *)0x7d); + mtree_test_erase(mt, 62); + mtree_test_insert(mt, 122, (void *)0xf5); + mtree_test_store(mt, 3, (void *)0x7); + mtree_test_insert(mt, 0, (void *)0x1); + mtree_test_store_range(mt, 0, 1, (void *)0x1); + mtree_test_insert(mt, 85, (void *)0xab); + mtree_test_insert(mt, 72, (void *)0x91); + mtree_test_insert(mt, 81, (void *)0xa3); + mtree_test_insert(mt, 726, (void *)0x5ad); + mtree_test_insert(mt, 0, (void *)0x1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_store(mt, 51, (void *)0x67); + mtree_test_insert(mt, 611, (void *)0x4c7); + mtree_test_insert(mt, 485, (void *)0x3cb); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 0, (void *)0x1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert_range(mt, 26, 1, (void *)0x35); + mtree_test_load(mt, 1); + mtree_test_store_range(mt, 1, 22, (void *)0x3); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_load(mt, 53); + mtree_test_load(mt, 1); + mtree_test_store_range(mt, 1, 1, (void *)0x3); + mtree_test_insert(mt, 222, (void *)0x1bd); + mtree_test_insert(mt, 485, (void *)0x3cb); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_load(mt, 0); + mtree_test_insert(mt, 21, (void *)0x2b); + mtree_test_insert(mt, 3, (void *)0x7); + mtree_test_store(mt, 621, (void *)0x4db); + mtree_test_insert(mt, 0, (void *)0x1); + mtree_test_erase(mt, 5); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_store(mt, 62, (void *)0x7d); + mtree_test_erase(mt, 62); + mtree_test_store_range(mt, 1, 0, (void *)0x3); + mtree_test_insert(mt, 22, (void *)0x2d); + mtree_test_insert(mt, 12, (void *)0x19); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_store_range(mt, 4, 62, (void *)0x9); + mtree_test_erase(mt, 62); + mtree_test_erase(mt, 1); + mtree_test_load(mt, 1); + mtree_test_store_range(mt, 1, 22, (void *)0x3); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_load(mt, 53); + mtree_test_load(mt, 1); + mtree_test_store_range(mt, 1, 1, (void *)0x3); + mtree_test_insert(mt, 222, (void *)0x1bd); + mtree_test_insert(mt, 485, (void *)0x3cb); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_load(mt, 0); + mtree_test_load(mt, 0); + mtree_destroy(mt); + + /* + * 7. Previous fix was incomplete, fix mas_resuse_node() clearing of old + * data by overwriting it first - that way metadata is of no concern. + */ + mt_init_flags(mt, 0); + mtree_test_load(mt, 1); + mtree_test_insert(mt, 102, (void *)0xcd); + mtree_test_erase(mt, 2); + mtree_test_erase(mt, 0); + mtree_test_load(mt, 0); + mtree_test_insert(mt, 4, (void *)0x9); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 110, (void *)0xdd); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_insert_range(mt, 5, 0, (void *)0xb); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_store(mt, 112, (void *)0xe1); + mtree_test_insert(mt, 21, (void *)0x2b); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_insert_range(mt, 110, 2, (void *)0xdd); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_load(mt, 22); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 210, (void *)0x1a5); + mtree_test_store_range(mt, 0, 2, (void *)0x1); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_erase(mt, 2); + mtree_test_erase(mt, 22); + mtree_test_erase(mt, 1); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_load(mt, 112); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_insert_range(mt, 1, 2, (void *)0x3); + mtree_test_erase(mt, 0); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_erase(mt, 0); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_erase(mt, 2); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert_range(mt, 1, 2, (void *)0x3); + mtree_test_erase(mt, 0); + mtree_test_erase(mt, 2); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_load(mt, 112); + mtree_test_store_range(mt, 110, 12, (void *)0xdd); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_load(mt, 110); + mtree_test_insert_range(mt, 4, 71, (void *)0x9); + mtree_test_load(mt, 2); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_insert_range(mt, 11, 22, (void *)0x17); + mtree_test_erase(mt, 12); + mtree_test_store(mt, 2, (void *)0x5); + mtree_test_load(mt, 22); + mtree_destroy(mt); + + + /* + * 8. When rebalancing or spanning_rebalance(), the max of the new node + * may be set incorrectly to the final pivot and not the right max. + * Fix by setting the left max to orig right max if the entire node is + * consumed. + */ + mt_init_flags(mt, 0); + mtree_test_store(mt, 6, (void *)0xd); + mtree_test_store(mt, 67, (void *)0x87); + mtree_test_insert(mt, 15, (void *)0x1f); + mtree_test_insert(mt, 6716, (void *)0x3479); + mtree_test_store(mt, 61, (void *)0x7b); + mtree_test_insert(mt, 13, (void *)0x1b); + mtree_test_store(mt, 8, (void *)0x11); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_load(mt, 0); + mtree_test_erase(mt, 67167); + mtree_test_insert_range(mt, 6, 7167, (void *)0xd); + mtree_test_insert(mt, 6, (void *)0xd); + mtree_test_erase(mt, 67); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 667167); + mtree_test_insert(mt, 6, (void *)0xd); + mtree_test_store(mt, 67, (void *)0x87); + mtree_test_insert(mt, 5, (void *)0xb); + mtree_test_erase(mt, 1); + mtree_test_insert(mt, 6, (void *)0xd); + mtree_test_erase(mt, 67); + mtree_test_insert(mt, 15, (void *)0x1f); + mtree_test_insert(mt, 67167, (void *)0x20cbf); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_load(mt, 7); + mtree_test_insert(mt, 16, (void *)0x21); + mtree_test_insert(mt, 36, (void *)0x49); + mtree_test_store(mt, 67, (void *)0x87); + mtree_test_store(mt, 6, (void *)0xd); + mtree_test_insert(mt, 367, (void *)0x2df); + mtree_test_insert(mt, 115, (void *)0xe7); + mtree_test_store(mt, 0, (void *)0x1); + mtree_test_store_range(mt, 1, 3, (void *)0x3); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_erase(mt, 67167); + mtree_test_insert_range(mt, 6, 47, (void *)0xd); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_insert_range(mt, 1, 67, (void *)0x3); + mtree_test_load(mt, 67); + mtree_test_insert(mt, 1, (void *)0x3); + mtree_test_erase(mt, 67167); + mtree_destroy(mt); + + /* + * 9. spanning store to the end of data caused an invalid metadata + * length which resulted in a crash eventually. + * Fix by checking if there is a value in pivot before incrementing the + * metadata end in mab_mas_cp(). To ensure this doesn't happen again, + * abstract the two locations this happens into a function called + * mas_leaf_set_meta(). + */ + mt_init_flags(mt, 0); + mtree_test_insert(mt, 21, (void *)0x2b); + mtree_test_insert(mt, 12, (void *)0x19); + mtree_test_insert(mt, 6, (void *)0xd); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 91, (void *)0xb7); + mtree_test_insert(mt, 18, (void *)0x25); + mtree_test_insert(mt, 81, (void *)0xa3); + mtree_test_store_range(mt, 0, 128, (void *)0x1); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_erase(mt, 8); + mtree_test_insert(mt, 11, (void *)0x17); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_insert(mt, 21, (void *)0x2b); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); + mtree_test_erase(mt, 18446744073709551605UL); + mtree_test_store_range(mt, 0, 281, (void *)0x1); + mtree_test_erase(mt, 2); + mtree_test_insert(mt, 1211, (void *)0x977); + mtree_test_insert(mt, 111, (void *)0xdf); + mtree_test_insert(mt, 13, (void *)0x1b); + mtree_test_insert(mt, 211, (void *)0x1a7); + mtree_test_insert(mt, 11, (void *)0x17); + mtree_test_insert(mt, 5, (void *)0xb); + mtree_test_insert(mt, 1218, (void *)0x985); + mtree_test_insert(mt, 61, (void *)0x7b); + mtree_test_store(mt, 1, (void *)0x3); + mtree_test_insert(mt, 121, (void *)0xf3); + mtree_test_insert(mt, 8, (void *)0x11); + mtree_test_insert(mt, 21, (void *)0x2b); + mtree_test_insert(mt, 2, (void *)0x5); + mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); + mtree_test_erase(mt, 18446744073709551605UL); +} +static noinline void check_dup_gaps(struct maple_tree *mt, + unsigned long nr_entries, bool zero_start, + unsigned long gap) +{ + unsigned long i = 0; + struct maple_tree newmt; + int ret; + void *tmp; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, &newmt, 0, 0); + + + if (!zero_start) + i = 1; + + mt_zero_nr_tallocated(); + for (; i <= nr_entries; i++) + mtree_store_range(mt, i*10, (i+1)*10 - gap, + xa_mk_value(i), GFP_KERNEL); + + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + mt_set_non_kernel(99999); + ret = mas_expected_entries(&newmas, nr_entries); + mt_set_non_kernel(0); + MT_BUG_ON(mt, ret != 0); + + mas_for_each(&mas, tmp, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store(&newmas, tmp); + } + + mas_destroy(&mas); + mas_destroy(&newmas); + mtree_destroy(&newmt); +} + +static noinline void check_dup(struct maple_tree *mt) +{ + int i; + + /* Check with a value at zero */ + for (i = 10; i < 1000; i++) { + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_dup_gaps(mt, i, true, 5); + mtree_destroy(mt); + } + + /* Check with a value at zero, no gap */ + for (i = 1000; i < 2000; i++) { + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_dup_gaps(mt, i, true, 0); + mtree_destroy(mt); + } + + /* Check with a value at zero and unreasonably large */ + for (i = 100010; i < 100020; i++) { + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_dup_gaps(mt, i, true, 5); + mtree_destroy(mt); + } + + /* Small to medium size not starting at zero*/ + for (i = 200; i < 1000; i++) { + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_dup_gaps(mt, i, false, 5); + mtree_destroy(mt); + } + + /* Unreasonably large not starting at zero*/ + for (i = 100010; i < 100020; i++) { + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_dup_gaps(mt, i, false, 5); + mtree_destroy(mt); + } + + /* Check non-allocation tree not starting at zero */ + for (i = 1500; i < 3000; i++) { + mt_init_flags(mt, 0); + check_dup_gaps(mt, i, false, 5); + mtree_destroy(mt); + } + + /* Check non-allocation tree starting at zero */ + for (i = 200; i < 1000; i++) { + mt_init_flags(mt, 0); + check_dup_gaps(mt, i, true, 5); + mtree_destroy(mt); + } + + /* Unreasonably large */ + for (i = 100015; i < 100020; i++) { + mt_init_flags(mt, 0); + check_dup_gaps(mt, i, true, 5); + mtree_destroy(mt); + } + +} + +static DEFINE_MTREE(tree); +static int maple_tree_seed(void) +{ + unsigned long set[] = {5015, 5014, 5017, 25, 1000, + 1001, 1002, 1003, 1005, 0, + 5003, 5002}; + void *ptr = &set; + + pr_info("\nTEST STARTING\n\n"); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_root_expand(&tree); + mtree_destroy(&tree); + +#if defined(BENCH_SLOT_STORE) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_slot_store(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_NODE_STORE) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_node_store(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_AWALK) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_awalk(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_WALK) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_walk(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_FORK) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_forking(&tree); + mtree_destroy(&tree); + goto skip; +#endif +#if defined(BENCH_MT_FOR_EACH) +#define BENCH + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + bench_mt_for_each(&tree); + mtree_destroy(&tree); + goto skip; +#endif + + test_kmem_cache_bulk(); + + mt_init_flags(&tree, 0); + check_new_node(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_prealloc(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_spanning_write(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_null_expand(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_dfs_preorder(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_forking(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_mas_store_gfp(&tree); + mtree_destroy(&tree); + + /* Test ranges (store and insert) */ + mt_init_flags(&tree, 0); + check_ranges(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_alloc_range(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_alloc_rev_range(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + + check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ + + check_insert(&tree, set[9], &tree); /* Insert 0 */ + check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ + check_load(&tree, set[0], NULL); /* See if 5015 -> NULL */ + + check_insert(&tree, set[10], ptr); /* Insert 5003 */ + check_load(&tree, set[9], &tree); /* See if 0 -> &tree */ + check_load(&tree, set[11], NULL); /* See if 5002 -> NULL */ + check_load(&tree, set[10], ptr); /* See if 5003 -> ptr */ + + /* Clear out the tree */ + mtree_destroy(&tree); + + /* Try to insert, insert a dup, and load back what was inserted. */ + mt_init_flags(&tree, 0); + check_insert(&tree, set[0], &tree); /* Insert 5015 */ + check_dup_insert(&tree, set[0], &tree); /* Insert 5015 again */ + check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ + + /* + * Second set of tests try to load a value that doesn't exist, inserts + * a second value, then loads the value again + */ + check_load(&tree, set[1], NULL); /* See if 5014 -> NULL */ + check_insert(&tree, set[1], ptr); /* insert 5014 -> ptr */ + check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ + check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ + /* + * Tree currently contains: + * p[0]: 14 -> (nil) p[1]: 15 -> ptr p[2]: 16 -> &tree p[3]: 0 -> (nil) + */ + check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ + check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ + + check_load(&tree, set[0], &tree); /* See if 5015 -> &tree */ + check_load(&tree, set[1], ptr); /* See if 5014 -> ptr */ + check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ + check_load(&tree, set[7], &tree); /* 1003 = &tree ? */ + + /* Clear out tree */ + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + /* Test inserting into a NULL hole. */ + check_insert(&tree, set[5], ptr); /* insert 1001 -> ptr */ + check_insert(&tree, set[7], &tree); /* insert 1003 -> &tree */ + check_insert(&tree, set[6], ptr); /* insert 1002 -> ptr */ + check_load(&tree, set[5], ptr); /* See if 1001 -> ptr */ + check_load(&tree, set[6], ptr); /* See if 1002 -> ptr */ + check_load(&tree, set[7], &tree); /* See if 1003 -> &tree */ + + /* Clear out the tree */ + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_erase_testset(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + /* + * set[] = {5015, 5014, 5017, 25, 1000, + * 1001, 1002, 1003, 1005, 0, + * 5003, 5002}; + */ + + check_insert(&tree, set[0], ptr); /* 5015 */ + check_insert(&tree, set[1], &tree); /* 5014 */ + check_insert(&tree, set[2], ptr); /* 5017 */ + check_insert(&tree, set[3], &tree); /* 25 */ + check_load(&tree, set[0], ptr); + check_load(&tree, set[1], &tree); + check_load(&tree, set[2], ptr); + check_load(&tree, set[3], &tree); + check_insert(&tree, set[4], ptr); /* 1000 < Should split. */ + check_load(&tree, set[0], ptr); + check_load(&tree, set[1], &tree); + check_load(&tree, set[2], ptr); + check_load(&tree, set[3], &tree); /*25 */ + check_load(&tree, set[4], ptr); + check_insert(&tree, set[5], &tree); /* 1001 */ + check_load(&tree, set[0], ptr); + check_load(&tree, set[1], &tree); + check_load(&tree, set[2], ptr); + check_load(&tree, set[3], &tree); + check_load(&tree, set[4], ptr); + check_load(&tree, set[5], &tree); + check_insert(&tree, set[6], ptr); + check_load(&tree, set[0], ptr); + check_load(&tree, set[1], &tree); + check_load(&tree, set[2], ptr); + check_load(&tree, set[3], &tree); + check_load(&tree, set[4], ptr); + check_load(&tree, set[5], &tree); + check_load(&tree, set[6], ptr); + check_insert(&tree, set[7], &tree); + check_load(&tree, set[0], ptr); + check_insert(&tree, set[8], ptr); + + check_insert(&tree, set[9], &tree); + + check_load(&tree, set[0], ptr); + check_load(&tree, set[1], &tree); + check_load(&tree, set[2], ptr); + check_load(&tree, set[3], &tree); + check_load(&tree, set[4], ptr); + check_load(&tree, set[5], &tree); + check_load(&tree, set[6], ptr); + check_load(&tree, set[9], &tree); + mtree_destroy(&tree); + + check_nomem(&tree); + mt_init_flags(&tree, 0); + check_seq(&tree, 16, false); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_seq(&tree, 1000, true); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rev_seq(&tree, 1000, true); + mtree_destroy(&tree); + + check_lower_bound_split(&tree); + check_upper_bound_split(&tree); + check_mid_split(&tree); + + mt_init_flags(&tree, 0); + check_next_entry(&tree); + check_find(&tree); + check_find_2(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_prev_entry(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_erase2_sets(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_gap_combining(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_node_overwrite(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + next_prev_test(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rcu_simulated(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rcu_threaded(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_spanning_relatives(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rev_find(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_fuzzer(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_dup(&tree); + mtree_destroy(&tree); + +#if defined(BENCH) +skip: +#endif + rcu_barrier(); + pr_info("maple_tree: %u of %u tests passed\n", + atomic_read(&maple_tree_tests_passed), + atomic_read(&maple_tree_tests_run)); + if (atomic_read(&maple_tree_tests_run) == + atomic_read(&maple_tree_tests_passed)) + return 0; + + return -EINVAL; +} + +static void maple_tree_harvest(void) +{ + +} + +module_init(maple_tree_seed); +module_exit(maple_tree_harvest); +MODULE_AUTHOR("Liam R. Howlett "); +MODULE_LICENSE("GPL"); diff --git a/lib/test_meminit.c b/lib/test_meminit.c index c95db11a690648a7cd424a83a6840647455ed04e..60e1984c060facca2db2e28ade9c3dddbf1627a6 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -67,17 +67,24 @@ static int __init do_alloc_pages_order(int order, int *total_failures) size_t size = PAGE_SIZE << order; page = alloc_pages(GFP_KERNEL, order); + if (!page) + goto err; buf = page_address(page); fill_with_garbage(buf, size); __free_pages(page, order); page = alloc_pages(GFP_KERNEL, order); + if (!page) + goto err; buf = page_address(page); if (count_nonzero_bytes(buf, size)) (*total_failures)++; fill_with_garbage(buf, size); __free_pages(page, order); return 1; +err: + (*total_failures)++; + return 1; } /* Test the page allocator by calling alloc_pages with different orders. */ @@ -100,15 +107,22 @@ static int __init do_kmalloc_size(size_t size, int *total_failures) void *buf; buf = kmalloc(size, GFP_KERNEL); + if (!buf) + goto err; fill_with_garbage(buf, size); kfree(buf); buf = kmalloc(size, GFP_KERNEL); + if (!buf) + goto err; if (count_nonzero_bytes(buf, size)) (*total_failures)++; fill_with_garbage(buf, size); kfree(buf); return 1; +err: + (*total_failures)++; + return 1; } /* Test vmalloc() with given parameters. */ @@ -117,15 +131,22 @@ static int __init do_vmalloc_size(size_t size, int *total_failures) void *buf; buf = vmalloc(size); + if (!buf) + goto err; fill_with_garbage(buf, size); vfree(buf); buf = vmalloc(size); + if (!buf) + goto err; if (count_nonzero_bytes(buf, size)) (*total_failures)++; fill_with_garbage(buf, size); vfree(buf); return 1; +err: + (*total_failures)++; + return 1; } /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ diff --git a/lib/test_min_heap.c b/lib/test_min_heap.c index d19c8080fd4d146e237a72754fdde0f907a03885..7b01b4387cfbcc3f55c44ed84648011553c40d81 100644 --- a/lib/test_min_heap.c +++ b/lib/test_min_heap.c @@ -83,7 +83,7 @@ static __init int test_heapify_all(bool min_heap) /* Test with randomly generated values. */ heap.nr = ARRAY_SIZE(values); for (i = 0; i < heap.nr; i++) - values[i] = get_random_int(); + values[i] = get_random_u32(); min_heapify_all(&heap, &funcs); err += pop_verify_heap(min_heap, &heap, &funcs); @@ -116,7 +116,7 @@ static __init int test_heap_push(bool min_heap) /* Test with randomly generated values. */ while (heap.nr < heap.size) { - temp = get_random_int(); + temp = get_random_u32(); min_heap_push(&heap, &temp, &funcs); } err += pop_verify_heap(min_heap, &heap, &funcs); @@ -158,7 +158,7 @@ static __init int test_heap_pop_push(bool min_heap) /* Test with randomly generated values. */ for (i = 0; i < ARRAY_SIZE(data); i++) { - temp = get_random_int(); + temp = get_random_u32(); min_heap_pop_push(&heap, &temp, &funcs); } err += pop_verify_heap(min_heap, &heap, &funcs); diff --git a/lib/test_objagg.c b/lib/test_objagg.c index da137939a410072751320cd0a0c5109f656742db..c0c957c50635415b00b5ce526c2b25efc4ac370e 100644 --- a/lib/test_objagg.c +++ b/lib/test_objagg.c @@ -157,7 +157,7 @@ static int test_nodelta_obj_get(struct world *world, struct objagg *objagg, int err; if (should_create_root) - prandom_bytes(world->next_root_buf, + get_random_bytes(world->next_root_buf, sizeof(world->next_root_buf)); objagg_obj = world_obj_get(world, objagg, key_id); diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index 5a1dd4736b56f510a3ae449bccf6b486065c6374..b358a74ed7ed8a27345182cd909755d2e10730a7 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -291,7 +291,7 @@ static int __init test_rhltable(unsigned int entries) if (WARN_ON(err)) goto out_free; - k = prandom_u32(); + k = get_random_u32(); ret = 0; for (i = 0; i < entries; i++) { rhl_test_objects[i].value.id = k; @@ -369,12 +369,12 @@ static int __init test_rhltable(unsigned int entries) pr_info("test %d random rhlist add/delete operations\n", entries); for (j = 0; j < entries; j++) { u32 i = prandom_u32_max(entries); - u32 prand = prandom_u32(); + u32 prand = get_random_u32(); cond_resched(); if (prand == 0) - prand = prandom_u32(); + prand = get_random_u32(); if (prand & 1) { prand >>= 1; diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c index 4f2f2d1bac562dee432d9ec23b8e303d4fbb96c8..cf7780572f5b4a4bfd3b2deea5c44807e930d9ea 100644 --- a/lib/test_vmalloc.c +++ b/lib/test_vmalloc.c @@ -80,7 +80,7 @@ static int random_size_align_alloc_test(void) int i; for (i = 0; i < test_loop_count; i++) { - rnd = prandom_u32(); + rnd = get_random_u8(); /* * Maximum 1024 pages, if PAGE_SIZE is 4096. @@ -151,9 +151,7 @@ static int random_size_alloc_test(void) int i; for (i = 0; i < test_loop_count; i++) { - n = prandom_u32(); - n = (n % 100) + 1; - + n = prandom_u32_max(100) + 1; p = vmalloc(n * PAGE_SIZE); if (!p) @@ -293,16 +291,12 @@ pcpu_alloc_test(void) return -1; for (i = 0; i < 35000; i++) { - unsigned int r; - - r = prandom_u32(); - size = (r % (PAGE_SIZE / 4)) + 1; + size = prandom_u32_max(PAGE_SIZE / 4) + 1; /* * Maximum PAGE_SIZE */ - r = prandom_u32(); - align = 1 << ((r % 11) + 1); + align = 1 << (prandom_u32_max(11) + 1); pcpu[i] = __alloc_percpu(size, align); if (!pcpu[i]) @@ -393,14 +387,11 @@ static struct test_driver { static void shuffle_array(int *arr, int n) { - unsigned int rnd; int i, j; for (i = n - 1; i > 0; i--) { - rnd = prandom_u32(); - /* Cut the range. */ - j = rnd % i; + j = prandom_u32_max(i); /* Swap indexes. */ swap(arr[i], arr[j]); diff --git a/lib/usercopy.c b/lib/usercopy.c index 7413dd300516e5405d92858a033b36c0eeb0399c..1505a52f23a01945ef30bb675d647a524e01680d 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,8 +12,9 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n unsigned long res = n; might_fault(); if (!should_fail_usercopy() && likely(access_ok(from, n))) { - instrument_copy_from_user(to, from, n); + instrument_copy_from_user_before(to, from, n); res = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, res); } if (unlikely(res)) memset(to + (n - res), 0, res); diff --git a/lib/uuid.c b/lib/uuid.c index 562d53977cabb99b84ad9dfb5a52440db260ed09..e309b4c5be3df0eb74861506d8cbbc3dd206c39b 100644 --- a/lib/uuid.c +++ b/lib/uuid.c @@ -52,7 +52,7 @@ EXPORT_SYMBOL(generate_random_guid); static void __uuid_gen_common(__u8 b[16]) { - prandom_bytes(b, 16); + get_random_bytes(b, 16); /* reversion 0b10 */ b[8] = (b[8] & 0x3F) | 0x80; } diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 3c1853a9d1c0908f51713773917b2baaf66a11e2..24f37bab8bc1f7a9ae5c533e6f155fa2aa30e83b 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -750,37 +750,42 @@ static int __init debug_boot_weak_hash_enable(char *str) } early_param("debug_boot_weak_hash", debug_boot_weak_hash_enable); -static DEFINE_STATIC_KEY_FALSE(filled_random_ptr_key); +static bool filled_random_ptr_key __read_mostly; +static siphash_key_t ptr_key __read_mostly; +static void fill_ptr_key_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(fill_ptr_key_work, fill_ptr_key_workfn); -static void enable_ptr_key_workfn(struct work_struct *work) +static void fill_ptr_key_workfn(struct work_struct *work) { - static_branch_enable(&filled_random_ptr_key); + if (!rng_is_initialized()) { + queue_delayed_work(system_unbound_wq, &fill_ptr_key_work, HZ * 2); + return; + } + + get_random_bytes(&ptr_key, sizeof(ptr_key)); + + /* Pairs with smp_rmb() before reading ptr_key. */ + smp_wmb(); + WRITE_ONCE(filled_random_ptr_key, true); +} + +static int __init vsprintf_init_hashval(void) +{ + fill_ptr_key_workfn(NULL); + return 0; } +subsys_initcall(vsprintf_init_hashval) /* Maps a pointer to a 32 bit unique identifier. */ static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out) { - static siphash_key_t ptr_key __read_mostly; unsigned long hashval; - if (!static_branch_likely(&filled_random_ptr_key)) { - static bool filled = false; - static DEFINE_SPINLOCK(filling); - static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn); - unsigned long flags; - - if (!system_unbound_wq || !rng_is_initialized() || - !spin_trylock_irqsave(&filling, flags)) - return -EAGAIN; - - if (!filled) { - get_random_bytes(&ptr_key, sizeof(ptr_key)); - queue_work(system_unbound_wq, &enable_ptr_key_work); - filled = true; - } - spin_unlock_irqrestore(&filling, flags); - } + if (!READ_ONCE(filled_random_ptr_key)) + return -EBUSY; + /* Pairs with smp_wmb() after writing ptr_key. */ + smp_rmb(); #ifdef CONFIG_64BIT hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key); @@ -1189,7 +1194,7 @@ char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec, } static noinline_for_stack -char *bitmap_string(char *buf, char *end, unsigned long *bitmap, +char *bitmap_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { const int CHUNKSZ = 32; @@ -1233,7 +1238,7 @@ char *bitmap_string(char *buf, char *end, unsigned long *bitmap, } static noinline_for_stack -char *bitmap_list_string(char *buf, char *end, unsigned long *bitmap, +char *bitmap_list_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { int nr_bits = max_t(int, spec.field_width, 0); @@ -2246,6 +2251,9 @@ int __init no_hash_pointers_enable(char *str) } early_param("no_hash_pointers", no_hash_pointers_enable); +/* Used for Rust formatting ('%pA'). */ +char *rust_fmt_argument(char *buf, char *end, void *ptr); + /* * Show a '%p' thing. A kernel extension is that the '%p' is followed * by an extra set of alphanumeric characters that are extended format @@ -2372,6 +2380,10 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * * Note: The default behaviour (unadorned %p) is to hash the address, * rendering it useful as a unique identifier. + * + * There is also a '%pA' format specifier, but it is only intended to be used + * from Rust code to format core::fmt::Arguments. Do *not* use it from C. + * See rust/kernel/print.rs for details. */ static noinline_for_stack char *pointer(const char *fmt, char *buf, char *end, void *ptr, @@ -2444,6 +2456,12 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, return device_node_string(buf, end, ptr, spec, fmt + 1); case 'f': return fwnode_string(buf, end, ptr, spec, fmt + 1); + case 'A': + if (!IS_ENABLED(CONFIG_RUST)) { + WARN_ONCE(1, "Please remove %%pA from non-Rust code\n"); + return error_string(buf, end, "(%pA?)", spec); + } + return rust_fmt_argument(buf, end, ptr); case 'x': return pointer_string(buf, end, ptr, spec); case 'e': diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile index fc45339fc3a3659e56d112979891a619f8b862cb..440bd0007ae207dcf17a211aeac2f87f60204825 100644 --- a/lib/zstd/Makefile +++ b/lib/zstd/Makefile @@ -10,14 +10,10 @@ # ################################################################ obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o +obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o zstd_compress-y := \ zstd_compress_module.o \ - common/debug.o \ - common/entropy_common.o \ - common/error_private.o \ - common/fse_decompress.o \ - common/zstd_common.o \ compress/fse_compress.o \ compress/hist.o \ compress/huf_compress.o \ @@ -33,12 +29,14 @@ zstd_compress-y := \ zstd_decompress-y := \ zstd_decompress_module.o \ + decompress/huf_decompress.o \ + decompress/zstd_ddict.o \ + decompress/zstd_decompress.o \ + decompress/zstd_decompress_block.o \ + +zstd_common-y := \ common/debug.o \ common/entropy_common.o \ common/error_private.o \ common/fse_decompress.o \ common/zstd_common.o \ - decompress/huf_decompress.o \ - decompress/zstd_ddict.o \ - decompress/zstd_decompress.o \ - decompress/zstd_decompress_block.o \ diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c index 53b47a2b52ff21b14d221510edea599ae2cd0285..a311808c0d5662155f156d06d771b169018c69f9 100644 --- a/lib/zstd/common/entropy_common.c +++ b/lib/zstd/common/entropy_common.c @@ -15,6 +15,7 @@ /* ************************************* * Dependencies ***************************************/ +#include #include "mem.h" #include "error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ @@ -239,7 +240,7 @@ size_t FSE_readNCount( { return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); } - +EXPORT_SYMBOL_GPL(FSE_readNCount); /*! HUF_readStats() : Read compact Huffman tree, saved by HUF_writeCTable(). @@ -255,6 +256,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); } +EXPORT_SYMBOL_GPL(HUF_readStats); FORCE_INLINE_TEMPLATE size_t HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, @@ -355,3 +357,4 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, (void)bmi2; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } +EXPORT_SYMBOL_GPL(HUF_readStats_wksp); diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c index 3d7e35b309b5d10e8a7d8f408f722b24006bb6ba..0f1f63be25d9fd5b038cce0e424ae9b44614bb11 100644 --- a/lib/zstd/common/zstd_common.c +++ b/lib/zstd/common/zstd_common.c @@ -13,6 +13,7 @@ /*-************************************* * Dependencies ***************************************/ +#include #define ZSTD_DEPS_NEED_MALLOC #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ #include "error_private.h" @@ -35,14 +36,17 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } * tells if a return value is an error code * symbol is required for external callers */ unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } +EXPORT_SYMBOL_GPL(ZSTD_isError); /*! ZSTD_getErrorName() : * provides error code string from function result (useful for debugging) */ const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } +EXPORT_SYMBOL_GPL(ZSTD_getErrorName); /*! ZSTD_getError() : * convert a `size_t` function result into a proper ZSTD_errorCode enum */ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } +EXPORT_SYMBOL_GPL(ZSTD_getErrorCode); /*! ZSTD_getErrorString() : * provides error code string from enum */ @@ -59,6 +63,7 @@ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) return customMem.customAlloc(customMem.opaque, size); return ZSTD_malloc(size); } +EXPORT_SYMBOL_GPL(ZSTD_customMalloc); void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) { @@ -71,6 +76,7 @@ void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) } return ZSTD_calloc(1, size); } +EXPORT_SYMBOL_GPL(ZSTD_customCalloc); void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) { @@ -81,3 +87,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) ZSTD_free(ptr); } } +EXPORT_SYMBOL_GPL(ZSTD_customFree); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Zstd Common"); diff --git a/mm/Kconfig b/mm/Kconfig index 0331f1461f81cd5fa9a08264faa3a348fa7af7a3..57e1d8c5b505287c3c0c81474d0daaa8f333e3d1 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -23,7 +23,7 @@ menuconfig SWAP in your computer. If unsure say Y. config ZSWAP - bool "Compressed cache for swap pages (EXPERIMENTAL)" + bool "Compressed cache for swap pages" depends on SWAP select FRONTSWAP select CRYPTO @@ -36,12 +36,6 @@ config ZSWAP in the case where decompressing from RAM is faster than swap device reads, can also improve workload performance. - This is marked experimental because it is a new feature (as of - v3.11) that interacts heavily with memory reclaim. While these - interactions don't cause any known issues on simple memory setups, - they have not be fully explored on the large set of potential - configurations and workloads that exist. - config ZSWAP_DEFAULT_ON bool "Enable the compressed cache for swap pages by default" depends on ZSWAP @@ -579,6 +573,12 @@ config COMPACTION it and then we would be really interested to hear about that at linux-mm@kvack.org. +config COMPACT_UNEVICTABLE_DEFAULT + int + depends on COMPACTION + default 0 if PREEMPT_RT + default 1 + # # support for free page reporting config PAGE_REPORTING @@ -1124,6 +1124,32 @@ config PTE_MARKER_UFFD_WP purposes. It is required to enable userfaultfd write protection on file-backed memory types like shmem and hugetlbfs. +# multi-gen LRU { +config LRU_GEN + bool "Multi-Gen LRU" + depends on MMU + # make sure folio->flags has enough spare bits + depends on 64BIT || !SPARSEMEM || SPARSEMEM_VMEMMAP + help + A high performance LRU implementation to overcommit memory. See + Documentation/admin-guide/mm/multigen_lru.rst for details. + +config LRU_GEN_ENABLED + bool "Enable by default" + depends on LRU_GEN + help + This option enables the multi-gen LRU by default. + +config LRU_GEN_STATS + bool "Full stats for debugging" + depends on LRU_GEN + help + Do not enable this option unless you plan to look at historical stats + from evicted generations for debugging purpose. + + This option has a per-memcg and per-node memory overhead. +# } + source "mm/damon/Kconfig" endmenu diff --git a/mm/Makefile b/mm/Makefile index 9a564f836403597c16fe1bddafc5aa91d7e25d99..8e105e5b3e293843176d01e08b518b25fff59c9f 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -52,7 +52,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ util.o mmzone.o vmstat.o backing-dev.o \ mm_init.o percpu.o slab_common.o \ - compaction.o vmacache.o \ + compaction.o \ interval_tree.o list_lru.o workingset.o \ debug.o gup.o mmap_lock.o $(mmu-y) @@ -89,14 +89,18 @@ obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_SLUB) += slub.o obj-$(CONFIG_KASAN) += kasan/ obj-$(CONFIG_KFENCE) += kfence/ +obj-$(CONFIG_KMSAN) += kmsan/ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_MEMTEST) += memtest.o obj-$(CONFIG_MIGRATION) += migrate.o +obj-$(CONFIG_NUMA) += memory-tiers.o obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o obj-$(CONFIG_PAGE_COUNTER) += page_counter.o obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o -obj-$(CONFIG_MEMCG_SWAP) += swap_cgroup.o +ifdef CONFIG_SWAP +obj-$(CONFIG_MEMCG) += swap_cgroup.o +endif obj-$(CONFIG_CGROUP_HUGETLB) += hugetlb_cgroup.o obj-$(CONFIG_GUP_TEST) += gup_test.o obj-$(CONFIG_MEMORY_FAILURE) += memory-failure.o diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 95550b8fa7fe2e359b55eb3d4cb50001296e9f20..c30419a5e1197dbc9700cc8d22d5a07ed185a3cb 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -260,10 +260,10 @@ void wb_wakeup_delayed(struct bdi_writeback *wb) unsigned long timeout; timeout = msecs_to_jiffies(dirty_writeback_interval * 10); - spin_lock_bh(&wb->work_lock); + spin_lock_irq(&wb->work_lock); if (test_bit(WB_registered, &wb->state)) queue_delayed_work(bdi_wq, &wb->dwork, timeout); - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); } static void wb_update_bandwidth_workfn(struct work_struct *work) @@ -334,12 +334,12 @@ static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb); static void wb_shutdown(struct bdi_writeback *wb) { /* Make sure nobody queues further work */ - spin_lock_bh(&wb->work_lock); + spin_lock_irq(&wb->work_lock); if (!test_and_clear_bit(WB_registered, &wb->state)) { - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); return; } - spin_unlock_bh(&wb->work_lock); + spin_unlock_irq(&wb->work_lock); cgwb_remove_from_bdi_list(wb); /* @@ -776,8 +776,6 @@ static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb) int bdi_init(struct backing_dev_info *bdi) { - int ret; - bdi->dev = NULL; kref_init(&bdi->refcnt); @@ -788,9 +786,7 @@ int bdi_init(struct backing_dev_info *bdi) INIT_LIST_HEAD(&bdi->wb_list); init_waitqueue_head(&bdi->wb_waitq); - ret = cgwb_bdi_init(bdi); - - return ret; + return cgwb_bdi_init(bdi); } struct backing_dev_info *bdi_alloc(int node_id) diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c index f18a631e74797b1f7c5902cca59a8e49a5f603d1..b1efebfcf94bb9a5282ca69826dd31063d9058d1 100644 --- a/mm/bootmem_info.c +++ b/mm/bootmem_info.c @@ -12,6 +12,7 @@ #include #include #include +#include void get_page_bootmem(unsigned long info, struct page *page, unsigned long type) { @@ -33,6 +34,7 @@ void put_page_bootmem(struct page *page) ClearPagePrivate(page); set_page_private(page, 0); INIT_LIST_HEAD(&page->lru); + kmemleak_free_part(page_to_virt(page), PAGE_SIZE); free_reserved_page(page); } } diff --git a/mm/cma_debug.c b/mm/cma_debug.c index c3ffe253e055271010a07d525cc65857a19cf5a8..602fff89b15fa309d1f03d6c43c2ead8f007c362 100644 --- a/mm/cma_debug.c +++ b/mm/cma_debug.c @@ -163,11 +163,8 @@ DEFINE_DEBUGFS_ATTRIBUTE(cma_alloc_fops, NULL, cma_alloc_write, "%llu\n"); static void cma_debugfs_add_one(struct cma *cma, struct dentry *root_dentry) { struct dentry *tmp; - char name[CMA_MAX_NAME]; - scnprintf(name, sizeof(name), "cma-%s", cma->name); - - tmp = debugfs_create_dir(name, root_dentry); + tmp = debugfs_create_dir(cma->name, root_dentry); debugfs_create_file("alloc", 0200, tmp, cma, &cma_alloc_fops); debugfs_create_file("free", 0200, tmp, cma, &cma_free_fops); diff --git a/mm/compaction.c b/mm/compaction.c index 640fa76228dd927b0004ad8f9e47d0d49e12346f..c51f7f545afe98644da0f1485f43c35448fd7406 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -52,8 +52,6 @@ static inline void count_compact_events(enum vm_event_item item, long delta) #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) -#define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order) -#define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order) /* * Page order with-respect-to which proactive compaction @@ -404,7 +402,7 @@ static bool test_and_set_skip(struct compact_control *cc, struct page *page, if (cc->ignore_skip_hint) return false; - if (!IS_ALIGNED(pfn, pageblock_nr_pages)) + if (!pageblock_aligned(pfn)) return false; skip = get_pageblock_skip(page); @@ -886,7 +884,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, * COMPACT_CLUSTER_MAX at a time so the second call must * not falsely conclude that the block should be skipped. */ - if (!valid_page && IS_ALIGNED(low_pfn, pageblock_nr_pages)) { + if (!valid_page && pageblock_aligned(low_pfn)) { if (!isolation_suitable(cc, page)) { low_pfn = end_pfn; page = NULL; @@ -1727,11 +1725,7 @@ typedef enum { * Allow userspace to control policy on scanning the unevictable LRU for * compactable pages. */ -#ifdef CONFIG_PREEMPT_RT -int sysctl_compact_unevictable_allowed __read_mostly = 0; -#else -int sysctl_compact_unevictable_allowed __read_mostly = 1; -#endif +int sysctl_compact_unevictable_allowed __read_mostly = CONFIG_COMPACT_UNEVICTABLE_DEFAULT; static inline void update_fast_start_pfn(struct compact_control *cc, unsigned long pfn) @@ -1853,7 +1847,6 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) pfn = cc->zone->zone_start_pfn; cc->fast_search_fail = 0; found_block = true; - set_pageblock_skip(freepage); break; } } @@ -1939,7 +1932,7 @@ static isolate_migrate_t isolate_migratepages(struct compact_control *cc) * before making it "skip" so other compaction instances do * not scan the same block. */ - if (IS_ALIGNED(low_pfn, pageblock_nr_pages) && + if (pageblock_aligned(low_pfn) && !fast_find_block && !isolation_suitable(cc, page)) continue; @@ -1981,9 +1974,21 @@ static inline bool is_via_compact_memory(int order) return order == -1; } +/* + * Determine whether kswapd is (or recently was!) running on this node. + * + * pgdat_kswapd_lock() pins pgdat->kswapd, so a concurrent kswapd_stop() can't + * zero it. + */ static bool kswapd_is_running(pg_data_t *pgdat) { - return pgdat->kswapd && task_is_running(pgdat->kswapd); + bool running; + + pgdat_kswapd_lock(pgdat); + running = pgdat->kswapd && task_is_running(pgdat->kswapd); + pgdat_kswapd_unlock(pgdat); + + return running; } /* @@ -2113,7 +2118,7 @@ static enum compact_result __compact_finished(struct compact_control *cc) * migration source is unmovable/reclaimable but it's not worth * special casing. */ - if (!IS_ALIGNED(cc->migrate_pfn, pageblock_nr_pages)) + if (!pageblock_aligned(cc->migrate_pfn)) return COMPACT_CONTINUE; /* Direct compactor: Is a suitable page free? */ diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 66265e3a9c659a95906455492ecb0e21ac6130fd..7821fcb3f25869d72673f2246d045db60ea46f72 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -68,6 +68,9 @@ config DAMON_DBGFS If unsure, say N. + This will be removed after >5.15.y LTS kernel is released, so users + should move to the sysfs interface (DAMON_SYSFS). + config DAMON_DBGFS_KUNIT_TEST bool "Test for damon debugfs interface" if !KUNIT_ALL_TESTS depends on DAMON_DBGFS && KUNIT=y diff --git a/mm/damon/core-test.h b/mm/damon/core-test.h index 573669566f8467772f211b9f0ead619cd679d4eb..3db9b73687562aebfb6d27e62bd716bc7e4b94c9 100644 --- a/mm/damon/core-test.h +++ b/mm/damon/core-test.h @@ -126,7 +126,7 @@ static void damon_test_split_at(struct kunit *test) t = damon_new_target(); r = damon_new_region(0, 100); damon_add_region(r, t); - damon_split_region_at(c, t, r, 25); + damon_split_region_at(t, r, 25); KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -219,14 +219,14 @@ static void damon_test_split_regions_of(struct kunit *test) t = damon_new_target(); r = damon_new_region(0, 22); damon_add_region(r, t); - damon_split_regions_of(c, t, 2); + damon_split_regions_of(t, 2); KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u); damon_free_target(t); t = damon_new_target(); r = damon_new_region(0, 220); damon_add_region(r, t); - damon_split_regions_of(c, t, 4); + damon_split_regions_of(t, 4); KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u); damon_free_target(t); damon_destroy_ctx(c); @@ -267,6 +267,28 @@ static void damon_test_ops_registration(struct kunit *test) KUNIT_EXPECT_EQ(test, damon_register_ops(&ops), -EINVAL); } +static void damon_test_set_regions(struct kunit *test) +{ + struct damon_target *t = damon_new_target(); + struct damon_region *r1 = damon_new_region(4, 16); + struct damon_region *r2 = damon_new_region(24, 32); + struct damon_addr_range range = {.start = 8, .end = 28}; + unsigned long expects[] = {8, 16, 16, 24, 24, 28}; + int expect_idx = 0; + struct damon_region *r; + + damon_add_region(r1, t); + damon_add_region(r2, t); + damon_set_regions(t, &range, 1); + + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 3); + damon_for_each_region(r, t) { + KUNIT_EXPECT_EQ(test, r->ar.start, expects[expect_idx++]); + KUNIT_EXPECT_EQ(test, r->ar.end, expects[expect_idx++]); + } + damon_destroy_target(t); +} + static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_target), KUNIT_CASE(damon_test_regions), @@ -276,6 +298,7 @@ static struct kunit_case damon_test_cases[] = { KUNIT_CASE(damon_test_merge_regions_of), KUNIT_CASE(damon_test_split_regions_of), KUNIT_CASE(damon_test_ops_registration), + KUNIT_CASE(damon_test_set_regions), {}, }; diff --git a/mm/damon/core.c b/mm/damon/core.c index 7d25dc582fe34427d2063793f5667e9fe3a0e423..36d098d06c5583db2d555e40b7075ea1760b1468 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -29,6 +29,8 @@ static bool running_exclusive_ctxs; static DEFINE_MUTEX(damon_ops_lock); static struct damon_operations damon_registered_ops[NR_DAMON_OPS]; +static struct kmem_cache *damon_region_cache __ro_after_init; + /* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */ static bool __damon_is_registered_ops(enum damon_ops_id id) { @@ -119,7 +121,7 @@ struct damon_region *damon_new_region(unsigned long start, unsigned long end) { struct damon_region *region; - region = kmalloc(sizeof(*region), GFP_KERNEL); + region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL); if (!region) return NULL; @@ -148,7 +150,7 @@ static void damon_del_region(struct damon_region *r, struct damon_target *t) static void damon_free_region(struct damon_region *r) { - kfree(r); + kmem_cache_free(damon_region_cache, r); } void damon_destroy_region(struct damon_region *r, struct damon_target *t) @@ -168,6 +170,30 @@ static bool damon_intersect(struct damon_region *r, return !(r->ar.end <= re->start || re->end <= r->ar.start); } +/* + * Fill holes in regions with new regions. + */ +static int damon_fill_regions_holes(struct damon_region *first, + struct damon_region *last, struct damon_target *t) +{ + struct damon_region *r = first; + + damon_for_each_region_from(r, t) { + struct damon_region *next, *newr; + + if (r == last) + break; + next = damon_next_region(r); + if (r->ar.end != next->ar.start) { + newr = damon_new_region(r->ar.end, next->ar.start); + if (!newr) + return -ENOMEM; + damon_insert_region(newr, r, next, t); + } + } + return 0; +} + /* * damon_set_regions() - Set regions of a target for given address ranges. * @t: the given target. @@ -184,6 +210,7 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, { struct damon_region *r, *next; unsigned int i; + int err; /* Remove regions which are not in the new ranges */ damon_for_each_region_safe(r, next, t) { @@ -195,6 +222,7 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, damon_destroy_region(r, t); } + r = damon_first_region(t); /* Add new regions or resize existing regions to fit in the ranges */ for (i = 0; i < nr_ranges; i++) { struct damon_region *first = NULL, *last, *newr; @@ -202,7 +230,7 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, range = &ranges[i]; /* Get the first/last regions intersecting with the range */ - damon_for_each_region(r, t) { + damon_for_each_region_from(r, t) { if (damon_intersect(r, range)) { if (!first) first = r; @@ -225,52 +253,46 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, first->ar.start = ALIGN_DOWN(range->start, DAMON_MIN_REGION); last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); + + /* fill possible holes in the range */ + err = damon_fill_regions_holes(first, last, t); + if (err) + return err; } } return 0; } -struct damos *damon_new_scheme( - unsigned long min_sz_region, unsigned long max_sz_region, - unsigned int min_nr_accesses, unsigned int max_nr_accesses, - unsigned int min_age_region, unsigned int max_age_region, - enum damos_action action, struct damos_quota *quota, - struct damos_watermarks *wmarks) +/* initialize private fields of damos_quota and return the pointer */ +static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota) +{ + quota->total_charged_sz = 0; + quota->total_charged_ns = 0; + quota->esz = 0; + quota->charged_sz = 0; + quota->charged_from = 0; + quota->charge_target_from = NULL; + quota->charge_addr_from = 0; + return quota; +} + +struct damos *damon_new_scheme(struct damos_access_pattern *pattern, + enum damos_action action, struct damos_quota *quota, + struct damos_watermarks *wmarks) { struct damos *scheme; scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); if (!scheme) return NULL; - scheme->min_sz_region = min_sz_region; - scheme->max_sz_region = max_sz_region; - scheme->min_nr_accesses = min_nr_accesses; - scheme->max_nr_accesses = max_nr_accesses; - scheme->min_age_region = min_age_region; - scheme->max_age_region = max_age_region; + scheme->pattern = *pattern; scheme->action = action; scheme->stat = (struct damos_stat){}; INIT_LIST_HEAD(&scheme->list); - scheme->quota.ms = quota->ms; - scheme->quota.sz = quota->sz; - scheme->quota.reset_interval = quota->reset_interval; - scheme->quota.weight_sz = quota->weight_sz; - scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; - scheme->quota.weight_age = quota->weight_age; - scheme->quota.total_charged_sz = 0; - scheme->quota.total_charged_ns = 0; - scheme->quota.esz = 0; - scheme->quota.charged_sz = 0; - scheme->quota.charged_from = 0; - scheme->quota.charge_target_from = NULL; - scheme->quota.charge_addr_from = 0; - - scheme->wmarks.metric = wmarks->metric; - scheme->wmarks.interval = wmarks->interval; - scheme->wmarks.high = wmarks->high; - scheme->wmarks.mid = wmarks->mid; - scheme->wmarks.low = wmarks->low; + scheme->quota = *(damos_quota_init_priv(quota)); + + scheme->wmarks = *wmarks; scheme->wmarks.activated = true; return scheme; @@ -313,6 +335,7 @@ struct damon_target *damon_new_target(void) t->pid = NULL; t->nr_regions = 0; INIT_LIST_HEAD(&t->regions_list); + INIT_LIST_HEAD(&t->list); return t; } @@ -360,17 +383,17 @@ struct damon_ctx *damon_new_ctx(void) if (!ctx) return NULL; - ctx->sample_interval = 5 * 1000; - ctx->aggr_interval = 100 * 1000; - ctx->ops_update_interval = 60 * 1000 * 1000; + ctx->attrs.sample_interval = 5 * 1000; + ctx->attrs.aggr_interval = 100 * 1000; + ctx->attrs.ops_update_interval = 60 * 1000 * 1000; ktime_get_coarse_ts64(&ctx->last_aggregation); ctx->last_ops_update = ctx->last_aggregation; mutex_init(&ctx->kdamond_lock); - ctx->min_nr_regions = 10; - ctx->max_nr_regions = 1000; + ctx->attrs.min_nr_regions = 10; + ctx->attrs.max_nr_regions = 1000; INIT_LIST_HEAD(&ctx->adaptive_targets); INIT_LIST_HEAD(&ctx->schemes); @@ -406,32 +429,21 @@ void damon_destroy_ctx(struct damon_ctx *ctx) /** * damon_set_attrs() - Set attributes for the monitoring. * @ctx: monitoring context - * @sample_int: time interval between samplings - * @aggr_int: time interval between aggregations - * @ops_upd_int: time interval between monitoring operations updates - * @min_nr_reg: minimal number of regions - * @max_nr_reg: maximum number of regions + * @attrs: monitoring attributes * * This function should not be called while the kdamond is running. * Every time interval is in micro-seconds. * * Return: 0 on success, negative error code otherwise. */ -int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, - unsigned long aggr_int, unsigned long ops_upd_int, - unsigned long min_nr_reg, unsigned long max_nr_reg) +int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs) { - if (min_nr_reg < 3) + if (attrs->min_nr_regions < 3) return -EINVAL; - if (min_nr_reg > max_nr_reg) + if (attrs->min_nr_regions > attrs->max_nr_regions) return -EINVAL; - ctx->sample_interval = sample_int; - ctx->aggr_interval = aggr_int; - ctx->ops_update_interval = ops_upd_int; - ctx->min_nr_regions = min_nr_reg; - ctx->max_nr_regions = max_nr_reg; - + ctx->attrs = *attrs; return 0; } @@ -443,10 +455,8 @@ int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, * * This function should not be called while the kdamond of the context is * running. - * - * Return: 0 if success, or negative error code otherwise. */ -int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, +void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, ssize_t nr_schemes) { struct damos *s, *next; @@ -456,7 +466,6 @@ int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, damon_destroy_scheme(s); for (i = 0; i < nr_schemes; i++) damon_add_scheme(ctx, schemes[i]); - return 0; } /** @@ -482,11 +491,11 @@ static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) - sz += r->ar.end - r->ar.start; + sz += damon_sz_region(r); } - if (ctx->min_nr_regions) - sz /= ctx->min_nr_regions; + if (ctx->attrs.min_nr_regions) + sz /= ctx->attrs.min_nr_regions; if (sz < DAMON_MIN_REGION) sz = DAMON_MIN_REGION; @@ -635,7 +644,7 @@ static bool damon_check_reset_time_interval(struct timespec64 *baseline, static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) { return damon_check_reset_time_interval(&ctx->last_aggregation, - ctx->aggr_interval); + ctx->attrs.aggr_interval); } /* @@ -658,19 +667,20 @@ static void kdamond_reset_aggregated(struct damon_ctx *c) } } -static void damon_split_region_at(struct damon_ctx *ctx, - struct damon_target *t, struct damon_region *r, - unsigned long sz_r); +static void damon_split_region_at(struct damon_target *t, + struct damon_region *r, unsigned long sz_r); static bool __damos_valid_target(struct damon_region *r, struct damos *s) { unsigned long sz; - sz = r->ar.end - r->ar.start; - return s->min_sz_region <= sz && sz <= s->max_sz_region && - s->min_nr_accesses <= r->nr_accesses && - r->nr_accesses <= s->max_nr_accesses && - s->min_age_region <= r->age && r->age <= s->max_age_region; + sz = damon_sz_region(r); + return s->pattern.min_sz_region <= sz && + sz <= s->pattern.max_sz_region && + s->pattern.min_nr_accesses <= r->nr_accesses && + r->nr_accesses <= s->pattern.max_nr_accesses && + s->pattern.min_age_region <= r->age && + r->age <= s->pattern.max_age_region; } static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, @@ -692,7 +702,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, damon_for_each_scheme(s, c) { struct damos_quota *quota = &s->quota; - unsigned long sz = r->ar.end - r->ar.start; + unsigned long sz = damon_sz_region(r); struct timespec64 begin, end; unsigned long sz_applied = 0; @@ -721,14 +731,14 @@ static void damon_do_apply_schemes(struct damon_ctx *c, sz = ALIGN_DOWN(quota->charge_addr_from - r->ar.start, DAMON_MIN_REGION); if (!sz) { - if (r->ar.end - r->ar.start <= - DAMON_MIN_REGION) + if (damon_sz_region(r) <= + DAMON_MIN_REGION) continue; sz = DAMON_MIN_REGION; } - damon_split_region_at(c, t, r, sz); + damon_split_region_at(t, r, sz); r = damon_next_region(r); - sz = r->ar.end - r->ar.start; + sz = damon_sz_region(r); } quota->charge_target_from = NULL; quota->charge_addr_from = 0; @@ -745,7 +755,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c, DAMON_MIN_REGION); if (!sz) goto update_stat; - damon_split_region_at(c, t, r, sz); + damon_split_region_at(t, r, sz); } ktime_get_coarse_ts64(&begin); sz_applied = c->ops.apply_scheme(c, t, r, s); @@ -833,8 +843,7 @@ static void kdamond_apply_schemes(struct damon_ctx *c) continue; score = c->ops.get_scheme_score( c, t, r, s); - quota->histogram[score] += - r->ar.end - r->ar.start; + quota->histogram[score] += damon_sz_region(r); if (score > max_score) max_score = score; } @@ -855,18 +864,13 @@ static void kdamond_apply_schemes(struct damon_ctx *c) } } -static inline unsigned long sz_damon_region(struct damon_region *r) -{ - return r->ar.end - r->ar.start; -} - /* * Merge two adjacent regions into one region */ static void damon_merge_two_regions(struct damon_target *t, struct damon_region *l, struct damon_region *r) { - unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); + unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r); l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / (sz_l + sz_r); @@ -895,7 +899,7 @@ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, if (prev && prev->ar.end == r->ar.start && abs(prev->nr_accesses - r->nr_accesses) <= thres && - sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) + damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) damon_merge_two_regions(t, prev, r); else prev = r; @@ -928,9 +932,8 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, * r the region to be split * sz_r size of the first sub-region that will be made */ -static void damon_split_region_at(struct damon_ctx *ctx, - struct damon_target *t, struct damon_region *r, - unsigned long sz_r) +static void damon_split_region_at(struct damon_target *t, + struct damon_region *r, unsigned long sz_r) { struct damon_region *new; @@ -947,15 +950,14 @@ static void damon_split_region_at(struct damon_ctx *ctx, } /* Split every region in the given target into 'nr_subs' regions */ -static void damon_split_regions_of(struct damon_ctx *ctx, - struct damon_target *t, int nr_subs) +static void damon_split_regions_of(struct damon_target *t, int nr_subs) { struct damon_region *r, *next; unsigned long sz_region, sz_sub = 0; int i; damon_for_each_region_safe(r, next, t) { - sz_region = r->ar.end - r->ar.start; + sz_region = damon_sz_region(r); for (i = 0; i < nr_subs - 1 && sz_region > 2 * DAMON_MIN_REGION; i++) { @@ -969,7 +971,7 @@ static void damon_split_regions_of(struct damon_ctx *ctx, if (sz_sub == 0 || sz_sub >= sz_region) continue; - damon_split_region_at(ctx, t, r, sz_sub); + damon_split_region_at(t, r, sz_sub); sz_region = sz_sub; } } @@ -995,16 +997,16 @@ static void kdamond_split_regions(struct damon_ctx *ctx) damon_for_each_target(t, ctx) nr_regions += damon_nr_regions(t); - if (nr_regions > ctx->max_nr_regions / 2) + if (nr_regions > ctx->attrs.max_nr_regions / 2) return; /* Maybe the middle of the region has different access frequency */ if (last_nr_regions == nr_regions && - nr_regions < ctx->max_nr_regions / 3) + nr_regions < ctx->attrs.max_nr_regions / 3) nr_subregions = 3; damon_for_each_target(t, ctx) - damon_split_regions_of(ctx, t, nr_subregions); + damon_split_regions_of(t, nr_subregions); last_nr_regions = nr_regions; } @@ -1018,7 +1020,7 @@ static void kdamond_split_regions(struct damon_ctx *ctx) static bool kdamond_need_update_operations(struct damon_ctx *ctx) { return damon_check_reset_time_interval(&ctx->last_ops_update, - ctx->ops_update_interval); + ctx->attrs.ops_update_interval); } /* @@ -1142,32 +1144,27 @@ static int kdamond_fn(void *data) struct damon_region *r, *next; unsigned int max_nr_accesses = 0; unsigned long sz_limit = 0; - bool done = false; pr_debug("kdamond (%d) starts\n", current->pid); if (ctx->ops.init) ctx->ops.init(ctx); if (ctx->callback.before_start && ctx->callback.before_start(ctx)) - done = true; + goto done; sz_limit = damon_region_sz_limit(ctx); - while (!kdamond_need_stop(ctx) && !done) { - if (kdamond_wait_activation(ctx)) { - done = true; - continue; - } + while (!kdamond_need_stop(ctx)) { + if (kdamond_wait_activation(ctx)) + break; if (ctx->ops.prepare_access_checks) ctx->ops.prepare_access_checks(ctx); if (ctx->callback.after_sampling && - ctx->callback.after_sampling(ctx)) { - done = true; - continue; - } + ctx->callback.after_sampling(ctx)) + break; - kdamond_usleep(ctx->sample_interval); + kdamond_usleep(ctx->attrs.sample_interval); if (ctx->ops.check_accesses) max_nr_accesses = ctx->ops.check_accesses(ctx); @@ -1177,10 +1174,8 @@ static int kdamond_fn(void *data) max_nr_accesses / 10, sz_limit); if (ctx->callback.after_aggregation && - ctx->callback.after_aggregation(ctx)) { - done = true; - continue; - } + ctx->callback.after_aggregation(ctx)) + break; kdamond_apply_schemes(ctx); kdamond_reset_aggregated(ctx); kdamond_split_regions(ctx); @@ -1194,6 +1189,7 @@ static int kdamond_fn(void *data) sz_limit = damon_region_sz_limit(ctx); } } +done: damon_for_each_target(t, ctx) { damon_for_each_region_safe(r, next, t) damon_destroy_region(r, t); @@ -1218,4 +1214,90 @@ static int kdamond_fn(void *data) return 0; } +/* + * struct damon_system_ram_region - System RAM resource address region of + * [@start, @end). + * @start: Start address of the region (inclusive). + * @end: End address of the region (exclusive). + */ +struct damon_system_ram_region { + unsigned long start; + unsigned long end; +}; + +static int walk_system_ram(struct resource *res, void *arg) +{ + struct damon_system_ram_region *a = arg; + + if (a->end - a->start < resource_size(res)) { + a->start = res->start; + a->end = res->end; + } + return 0; +} + +/* + * Find biggest 'System RAM' resource and store its start and end address in + * @start and @end, respectively. If no System RAM is found, returns false. + */ +static bool damon_find_biggest_system_ram(unsigned long *start, + unsigned long *end) + +{ + struct damon_system_ram_region arg = {}; + + walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); + if (arg.end <= arg.start) + return false; + + *start = arg.start; + *end = arg.end; + return true; +} + +/** + * damon_set_region_biggest_system_ram_default() - Set the region of the given + * monitoring target as requested, or biggest 'System RAM'. + * @t: The monitoring target to set the region. + * @start: The pointer to the start address of the region. + * @end: The pointer to the end address of the region. + * + * This function sets the region of @t as requested by @start and @end. If the + * values of @start and @end are zero, however, this function finds the biggest + * 'System RAM' resource and sets the region to cover the resource. In the + * latter case, this function saves the start and end addresses of the resource + * in @start and @end, respectively. + * + * Return: 0 on success, negative error code otherwise. + */ +int damon_set_region_biggest_system_ram_default(struct damon_target *t, + unsigned long *start, unsigned long *end) +{ + struct damon_addr_range addr_range; + + if (*start > *end) + return -EINVAL; + + if (!*start && !*end && + !damon_find_biggest_system_ram(start, end)) + return -EINVAL; + + addr_range.start = *start; + addr_range.end = *end; + return damon_set_regions(t, &addr_range, 1); +} + +static int __init damon_init(void) +{ + damon_region_cache = KMEM_CACHE(damon_region, 0); + if (unlikely(!damon_region_cache)) { + pr_err("creating damon_region_cache fails\n"); + return -ENOMEM; + } + + return 0; +} + +subsys_initcall(damon_init); + #include "core-test.h" diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index cb8a7e9926a40fe07b65ec23f61d9058b17d4718..6f0ae7d3ae39bb62125cb654d1083c96aa4dab43 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -55,9 +55,9 @@ static ssize_t dbgfs_attrs_read(struct file *file, mutex_lock(&ctx->kdamond_lock); ret = scnprintf(kbuf, ARRAY_SIZE(kbuf), "%lu %lu %lu %lu %lu\n", - ctx->sample_interval, ctx->aggr_interval, - ctx->ops_update_interval, ctx->min_nr_regions, - ctx->max_nr_regions); + ctx->attrs.sample_interval, ctx->attrs.aggr_interval, + ctx->attrs.ops_update_interval, + ctx->attrs.min_nr_regions, ctx->attrs.max_nr_regions); mutex_unlock(&ctx->kdamond_lock); return simple_read_from_buffer(buf, count, ppos, kbuf, ret); @@ -67,7 +67,7 @@ static ssize_t dbgfs_attrs_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct damon_ctx *ctx = file->private_data; - unsigned long s, a, r, minr, maxr; + struct damon_attrs attrs; char *kbuf; ssize_t ret; @@ -76,7 +76,10 @@ static ssize_t dbgfs_attrs_write(struct file *file, return PTR_ERR(kbuf); if (sscanf(kbuf, "%lu %lu %lu %lu %lu", - &s, &a, &r, &minr, &maxr) != 5) { + &attrs.sample_interval, &attrs.aggr_interval, + &attrs.ops_update_interval, + &attrs.min_nr_regions, + &attrs.max_nr_regions) != 5) { ret = -EINVAL; goto out; } @@ -87,7 +90,7 @@ static ssize_t dbgfs_attrs_write(struct file *file, goto unlock_out; } - ret = damon_set_attrs(ctx, s, a, r, minr, maxr); + ret = damon_set_attrs(ctx, &attrs); if (!ret) ret = count; unlock_out: @@ -131,9 +134,12 @@ static ssize_t sprint_schemes(struct damon_ctx *c, char *buf, ssize_t len) damon_for_each_scheme(s, c) { rc = scnprintf(&buf[written], len - written, "%lu %lu %u %u %u %u %d %lu %lu %lu %u %u %u %d %lu %lu %lu %lu %lu %lu %lu %lu %lu\n", - s->min_sz_region, s->max_sz_region, - s->min_nr_accesses, s->max_nr_accesses, - s->min_age_region, s->max_age_region, + s->pattern.min_sz_region, + s->pattern.max_sz_region, + s->pattern.min_nr_accesses, + s->pattern.max_nr_accesses, + s->pattern.min_age_region, + s->pattern.max_age_region, damos_action_to_dbgfs_scheme_action(s->action), s->quota.ms, s->quota.sz, s->quota.reset_interval, @@ -221,8 +227,6 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, struct damos *scheme, **schemes; const int max_nr_schemes = 256; int pos = 0, parsed, ret; - unsigned long min_sz, max_sz; - unsigned int min_nr_a, max_nr_a, min_age, max_age; unsigned int action_input; enum damos_action action; @@ -233,13 +237,18 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, *nr_schemes = 0; while (pos < len && *nr_schemes < max_nr_schemes) { + struct damos_access_pattern pattern = {}; struct damos_quota quota = {}; struct damos_watermarks wmarks; ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u %lu %lu %lu %u %u %u %u %lu %lu %lu %lu%n", - &min_sz, &max_sz, &min_nr_a, &max_nr_a, - &min_age, &max_age, &action_input, "a.ms, + &pattern.min_sz_region, &pattern.max_sz_region, + &pattern.min_nr_accesses, + &pattern.max_nr_accesses, + &pattern.min_age_region, + &pattern.max_age_region, + &action_input, "a.ms, "a.sz, "a.reset_interval, "a.weight_sz, "a.weight_nr_accesses, "a.weight_age, &wmarks.metric, @@ -251,7 +260,9 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, if ((int)action < 0) goto fail; - if (min_sz > max_sz || min_nr_a > max_nr_a || min_age > max_age) + if (pattern.min_sz_region > pattern.max_sz_region || + pattern.min_nr_accesses > pattern.max_nr_accesses || + pattern.min_age_region > pattern.max_age_region) goto fail; if (wmarks.high < wmarks.mid || wmarks.high < wmarks.low || @@ -259,8 +270,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len, goto fail; pos += parsed; - scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a, - min_age, max_age, action, "a, &wmarks); + scheme = damon_new_scheme(&pattern, action, "a, &wmarks); if (!scheme) goto fail; @@ -297,11 +307,9 @@ static ssize_t dbgfs_schemes_write(struct file *file, const char __user *buf, goto unlock_out; } - ret = damon_set_schemes(ctx, schemes, nr_schemes); - if (!ret) { - ret = count; - nr_schemes = 0; - } + damon_set_schemes(ctx, schemes, nr_schemes); + ret = count; + nr_schemes = 0; unlock_out: mutex_unlock(&ctx->kdamond_lock); @@ -818,6 +826,9 @@ static int dbgfs_mk_context(char *name) return -ENOENT; new_dir = debugfs_create_dir(name, root); + /* Below check is required for a potential duplicated name case */ + if (IS_ERR(new_dir)) + return PTR_ERR(new_dir); dbgfs_dirs[dbgfs_nr_ctxs] = new_dir; new_ctx = dbgfs_new_ctx(); @@ -881,6 +892,7 @@ static int dbgfs_rm_context(char *name) struct dentry *root, *dir, **new_dirs; struct damon_ctx **new_ctxs; int i, j; + int ret = 0; if (damon_nr_running_ctxs()) return -EBUSY; @@ -895,14 +907,16 @@ static int dbgfs_rm_context(char *name) new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs), GFP_KERNEL); - if (!new_dirs) - return -ENOMEM; + if (!new_dirs) { + ret = -ENOMEM; + goto out_dput; + } new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs), GFP_KERNEL); if (!new_ctxs) { - kfree(new_dirs); - return -ENOMEM; + ret = -ENOMEM; + goto out_new_dirs; } for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) { @@ -922,7 +936,13 @@ static int dbgfs_rm_context(char *name) dbgfs_ctxs = new_ctxs; dbgfs_nr_ctxs--; - return 0; + goto out_dput; + +out_new_dirs: + kfree(new_dirs); +out_dput: + dput(dir); + return ret; } static ssize_t dbgfs_rm_context_write(struct file *file, @@ -1041,7 +1061,7 @@ static int __init __damon_dbgfs_init(void) fops[i]); dbgfs_fill_ctx_dir(dbgfs_root, dbgfs_ctxs[0]); - dbgfs_dirs = kmalloc_array(1, sizeof(dbgfs_root), GFP_KERNEL); + dbgfs_dirs = kmalloc(sizeof(dbgfs_root), GFP_KERNEL); if (!dbgfs_dirs) { debugfs_remove(dbgfs_root); return -ENOMEM; diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 9de6f00a71c5dd8165ddf7249867caddaadfd34f..efbc2bda8b9cdaa4448437d350f37a705712d6b5 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -13,6 +13,8 @@ #include #include +#include "modules-common.h" + #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif @@ -63,109 +65,35 @@ module_param(hot_thres_access_freq, ulong, 0600); static unsigned long cold_min_age __read_mostly = 120000000; module_param(cold_min_age, ulong, 0600); -/* - * Limit of time for trying the LRU lists sorting in milliseconds. - * - * DAMON_LRU_SORT tries to use only up to this time within a time window - * (quota_reset_interval_ms) for trying LRU lists sorting. This can be used - * for limiting CPU consumption of DAMON_LRU_SORT. If the value is zero, the - * limit is disabled. - * - * 10 ms by default. - */ -static unsigned long quota_ms __read_mostly = 10; -module_param(quota_ms, ulong, 0600); - -/* - * The time quota charge reset interval in milliseconds. - * - * The charge reset interval for the quota of time (quota_ms). That is, - * DAMON_LRU_SORT does not try LRU-lists sorting for more than quota_ms - * milliseconds or quota_sz bytes within quota_reset_interval_ms milliseconds. - * - * 1 second by default. - */ -static unsigned long quota_reset_interval_ms __read_mostly = 1000; -module_param(quota_reset_interval_ms, ulong, 0600); - -/* - * The watermarks check time interval in microseconds. - * - * Minimal time to wait before checking the watermarks, when DAMON_LRU_SORT is - * enabled but inactive due to its watermarks rule. 5 seconds by default. - */ -static unsigned long wmarks_interval __read_mostly = 5000000; -module_param(wmarks_interval, ulong, 0600); - -/* - * Free memory rate (per thousand) for the high watermark. - * - * If free memory of the system in bytes per thousand bytes is higher than - * this, DAMON_LRU_SORT becomes inactive, so it does nothing but periodically - * checks the watermarks. 200 (20%) by default. - */ -static unsigned long wmarks_high __read_mostly = 200; -module_param(wmarks_high, ulong, 0600); - -/* - * Free memory rate (per thousand) for the middle watermark. - * - * If free memory of the system in bytes per thousand bytes is between this and - * the low watermark, DAMON_LRU_SORT becomes active, so starts the monitoring - * and the LRU-lists sorting. 150 (15%) by default. - */ -static unsigned long wmarks_mid __read_mostly = 150; -module_param(wmarks_mid, ulong, 0600); - -/* - * Free memory rate (per thousand) for the low watermark. - * - * If free memory of the system in bytes per thousand bytes is lower than this, - * DAMON_LRU_SORT becomes inactive, so it does nothing but periodically checks - * the watermarks. 50 (5%) by default. - */ -static unsigned long wmarks_low __read_mostly = 50; -module_param(wmarks_low, ulong, 0600); - -/* - * Sampling interval for the monitoring in microseconds. - * - * The sampling interval of DAMON for the hot/cold memory monitoring. Please - * refer to the DAMON documentation for more detail. 5 ms by default. - */ -static unsigned long sample_interval __read_mostly = 5000; -module_param(sample_interval, ulong, 0600); - -/* - * Aggregation interval for the monitoring in microseconds. - * - * The aggregation interval of DAMON for the hot/cold memory monitoring. - * Please refer to the DAMON documentation for more detail. 100 ms by default. - */ -static unsigned long aggr_interval __read_mostly = 100000; -module_param(aggr_interval, ulong, 0600); - -/* - * Minimum number of monitoring regions. - * - * The minimal number of monitoring regions of DAMON for the hot/cold memory - * monitoring. This can be used to set lower-bound of the monitoring quality. - * But, setting this too high could result in increased monitoring overhead. - * Please refer to the DAMON documentation for more detail. 10 by default. - */ -static unsigned long min_nr_regions __read_mostly = 10; -module_param(min_nr_regions, ulong, 0600); - -/* - * Maximum number of monitoring regions. - * - * The maximum number of monitoring regions of DAMON for the hot/cold memory - * monitoring. This can be used to set upper-bound of the monitoring overhead. - * However, setting this too low could result in bad monitoring quality. - * Please refer to the DAMON documentation for more detail. 1000 by default. - */ -static unsigned long max_nr_regions __read_mostly = 1000; -module_param(max_nr_regions, ulong, 0600); +static struct damos_quota damon_lru_sort_quota = { + /* Use up to 10 ms per 1 sec, by default */ + .ms = 10, + .sz = 0, + .reset_interval = 1000, + /* Within the quota, mark hotter regions accessed first. */ + .weight_sz = 0, + .weight_nr_accesses = 1, + .weight_age = 0, +}; +DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(damon_lru_sort_quota); + +static struct damos_watermarks damon_lru_sort_wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = 5000000, /* 5 seconds */ + .high = 200, /* 20 percent */ + .mid = 150, /* 15 percent */ + .low = 50, /* 5 percent */ +}; +DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_lru_sort_wmarks); + +static struct damon_attrs damon_lru_sort_mon_attrs = { + .sample_interval = 5000, /* 5 ms */ + .aggr_interval = 100000, /* 100 ms */ + .ops_update_interval = 0, + .min_nr_regions = 10, + .max_nr_regions = 1000, +}; +DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs); /* * Start of the target memory region in physical address. @@ -194,222 +122,97 @@ module_param(monitor_region_end, ulong, 0600); static int kdamond_pid __read_mostly = -1; module_param(kdamond_pid, int, 0400); -/* - * Number of hot memory regions that tried to be LRU-sorted. - */ -static unsigned long nr_lru_sort_tried_hot_regions __read_mostly; -module_param(nr_lru_sort_tried_hot_regions, ulong, 0400); - -/* - * Total bytes of hot memory regions that tried to be LRU-sorted. - */ -static unsigned long bytes_lru_sort_tried_hot_regions __read_mostly; -module_param(bytes_lru_sort_tried_hot_regions, ulong, 0400); - -/* - * Number of hot memory regions that successfully be LRU-sorted. - */ -static unsigned long nr_lru_sorted_hot_regions __read_mostly; -module_param(nr_lru_sorted_hot_regions, ulong, 0400); - -/* - * Total bytes of hot memory regions that successfully be LRU-sorted. - */ -static unsigned long bytes_lru_sorted_hot_regions __read_mostly; -module_param(bytes_lru_sorted_hot_regions, ulong, 0400); - -/* - * Number of times that the time quota limit for hot regions have exceeded - */ -static unsigned long nr_hot_quota_exceeds __read_mostly; -module_param(nr_hot_quota_exceeds, ulong, 0400); - -/* - * Number of cold memory regions that tried to be LRU-sorted. - */ -static unsigned long nr_lru_sort_tried_cold_regions __read_mostly; -module_param(nr_lru_sort_tried_cold_regions, ulong, 0400); - -/* - * Total bytes of cold memory regions that tried to be LRU-sorted. - */ -static unsigned long bytes_lru_sort_tried_cold_regions __read_mostly; -module_param(bytes_lru_sort_tried_cold_regions, ulong, 0400); - -/* - * Number of cold memory regions that successfully be LRU-sorted. - */ -static unsigned long nr_lru_sorted_cold_regions __read_mostly; -module_param(nr_lru_sorted_cold_regions, ulong, 0400); - -/* - * Total bytes of cold memory regions that successfully be LRU-sorted. - */ -static unsigned long bytes_lru_sorted_cold_regions __read_mostly; -module_param(bytes_lru_sorted_cold_regions, ulong, 0400); - -/* - * Number of times that the time quota limit for cold regions have exceeded - */ -static unsigned long nr_cold_quota_exceeds __read_mostly; -module_param(nr_cold_quota_exceeds, ulong, 0400); +static struct damos_stat damon_lru_sort_hot_stat; +DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, + lru_sort_tried_hot_regions, lru_sorted_hot_regions, + hot_quota_exceeds); + +static struct damos_stat damon_lru_sort_cold_stat; +DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_cold_stat, + lru_sort_tried_cold_regions, lru_sorted_cold_regions, + cold_quota_exceeds); + +static struct damos_access_pattern damon_lru_sort_stub_pattern = { + /* Find regions having PAGE_SIZE or larger size */ + .min_sz_region = PAGE_SIZE, + .max_sz_region = ULONG_MAX, + /* no matter its access frequency */ + .min_nr_accesses = 0, + .max_nr_accesses = UINT_MAX, + /* no matter its age */ + .min_age_region = 0, + .max_age_region = UINT_MAX, +}; static struct damon_ctx *ctx; static struct damon_target *target; -struct damon_lru_sort_ram_walk_arg { - unsigned long start; - unsigned long end; -}; - -static int walk_system_ram(struct resource *res, void *arg) -{ - struct damon_lru_sort_ram_walk_arg *a = arg; - - if (a->end - a->start < resource_size(res)) { - a->start = res->start; - a->end = res->end; - } - return 0; -} - -/* - * Find biggest 'System RAM' resource and store its start and end address in - * @start and @end, respectively. If no System RAM is found, returns false. - */ -static bool get_monitoring_region(unsigned long *start, unsigned long *end) +static struct damos *damon_lru_sort_new_scheme( + struct damos_access_pattern *pattern, enum damos_action action) { - struct damon_lru_sort_ram_walk_arg arg = {}; + struct damos_quota quota = damon_lru_sort_quota; - walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); - if (arg.end <= arg.start) - return false; + /* Use half of total quota for hot/cold pages sorting */ + quota.ms = quota.ms / 2; - *start = arg.start; - *end = arg.end; - return true; + return damon_new_scheme( + /* find the pattern, and */ + pattern, + /* (de)prioritize on LRU-lists */ + action, + /* under the quota. */ + "a, + /* (De)activate this according to the watermarks. */ + &damon_lru_sort_wmarks); } /* Create a DAMON-based operation scheme for hot memory regions */ static struct damos *damon_lru_sort_new_hot_scheme(unsigned int hot_thres) { - struct damos_watermarks wmarks = { - .metric = DAMOS_WMARK_FREE_MEM_RATE, - .interval = wmarks_interval, - .high = wmarks_high, - .mid = wmarks_mid, - .low = wmarks_low, - }; - struct damos_quota quota = { - /* - * Do not try LRU-lists sorting of hot pages for more than half - * of quota_ms milliseconds within quota_reset_interval_ms. - */ - .ms = quota_ms / 2, - .sz = 0, - .reset_interval = quota_reset_interval_ms, - /* Within the quota, mark hotter regions accessed first. */ - .weight_sz = 0, - .weight_nr_accesses = 1, - .weight_age = 0, - }; - struct damos *scheme = damon_new_scheme( - /* Find regions having PAGE_SIZE or larger size */ - PAGE_SIZE, ULONG_MAX, - /* and accessed for more than the threshold */ - hot_thres, UINT_MAX, - /* no matter its age */ - 0, UINT_MAX, - /* prioritize those on LRU lists, as soon as found */ - DAMOS_LRU_PRIO, - /* under the quota. */ - "a, - /* (De)activate this according to the watermarks. */ - &wmarks); + struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; - return scheme; + pattern.min_nr_accesses = hot_thres; + return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_PRIO); } /* Create a DAMON-based operation scheme for cold memory regions */ static struct damos *damon_lru_sort_new_cold_scheme(unsigned int cold_thres) { - struct damos_watermarks wmarks = { - .metric = DAMOS_WMARK_FREE_MEM_RATE, - .interval = wmarks_interval, - .high = wmarks_high, - .mid = wmarks_mid, - .low = wmarks_low, - }; - struct damos_quota quota = { - /* - * Do not try LRU-lists sorting of cold pages for more than - * half of quota_ms milliseconds within - * quota_reset_interval_ms. - */ - .ms = quota_ms / 2, - .sz = 0, - .reset_interval = quota_reset_interval_ms, - /* Within the quota, mark colder regions not accessed first. */ - .weight_sz = 0, - .weight_nr_accesses = 0, - .weight_age = 1, - }; - struct damos *scheme = damon_new_scheme( - /* Find regions having PAGE_SIZE or larger size */ - PAGE_SIZE, ULONG_MAX, - /* and not accessed at all */ - 0, 0, - /* for cold_thres or more micro-seconds, and */ - cold_thres, UINT_MAX, - /* mark those as not accessed, as soon as found */ - DAMOS_LRU_DEPRIO, - /* under the quota. */ - "a, - /* (De)activate this according to the watermarks. */ - &wmarks); + struct damos_access_pattern pattern = damon_lru_sort_stub_pattern; - return scheme; + pattern.max_nr_accesses = 0; + pattern.min_age_region = cold_thres; + return damon_lru_sort_new_scheme(&pattern, DAMOS_LRU_DEPRIO); } static int damon_lru_sort_apply_parameters(void) { - struct damos *scheme, *next_scheme; - struct damon_addr_range addr_range; + struct damos *scheme; unsigned int hot_thres, cold_thres; int err = 0; - err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, - min_nr_regions, max_nr_regions); + err = damon_set_attrs(ctx, &damon_lru_sort_mon_attrs); if (err) return err; - /* free previously set schemes */ - damon_for_each_scheme_safe(scheme, next_scheme, ctx) - damon_destroy_scheme(scheme); - /* aggr_interval / sample_interval is the maximum nr_accesses */ - hot_thres = aggr_interval / sample_interval * hot_thres_access_freq / - 1000; + hot_thres = damon_lru_sort_mon_attrs.aggr_interval / + damon_lru_sort_mon_attrs.sample_interval * + hot_thres_access_freq / 1000; scheme = damon_lru_sort_new_hot_scheme(hot_thres); if (!scheme) return -ENOMEM; - damon_add_scheme(ctx, scheme); + damon_set_schemes(ctx, &scheme, 1); - cold_thres = cold_min_age / aggr_interval; + cold_thres = cold_min_age / damon_lru_sort_mon_attrs.aggr_interval; scheme = damon_lru_sort_new_cold_scheme(cold_thres); if (!scheme) return -ENOMEM; damon_add_scheme(ctx, scheme); - if (monitor_region_start > monitor_region_end) - return -EINVAL; - if (!monitor_region_start && !monitor_region_end && - !get_monitoring_region(&monitor_region_start, - &monitor_region_end)) - return -EINVAL; - addr_range.start = monitor_region_start; - addr_range.end = monitor_region_end; - return damon_set_regions(target, &addr_range, 1); + return damon_set_region_biggest_system_ram_default(target, + &monitor_region_start, + &monitor_region_end); } static int damon_lru_sort_turn(bool on) @@ -495,19 +298,10 @@ static int damon_lru_sort_after_aggregation(struct damon_ctx *c) /* update the stats parameter */ damon_for_each_scheme(s, c) { - if (s->action == DAMOS_LRU_PRIO) { - nr_lru_sort_tried_hot_regions = s->stat.nr_tried; - bytes_lru_sort_tried_hot_regions = s->stat.sz_tried; - nr_lru_sorted_hot_regions = s->stat.nr_applied; - bytes_lru_sorted_hot_regions = s->stat.sz_applied; - nr_hot_quota_exceeds = s->stat.qt_exceeds; - } else if (s->action == DAMOS_LRU_DEPRIO) { - nr_lru_sort_tried_cold_regions = s->stat.nr_tried; - bytes_lru_sort_tried_cold_regions = s->stat.sz_tried; - nr_lru_sorted_cold_regions = s->stat.nr_applied; - bytes_lru_sorted_cold_regions = s->stat.sz_applied; - nr_cold_quota_exceeds = s->stat.qt_exceeds; - } + if (s->action == DAMOS_LRU_PRIO) + damon_lru_sort_hot_stat = s->stat; + else if (s->action == DAMOS_LRU_DEPRIO) + damon_lru_sort_cold_stat = s->stat; } return damon_lru_sort_handle_commit_inputs(); diff --git a/mm/damon/modules-common.h b/mm/damon/modules-common.h new file mode 100644 index 0000000000000000000000000000000000000000..5a4921851d326380aaa32a524831a4047c4f6e6c --- /dev/null +++ b/mm/damon/modules-common.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common Primitives for DAMON Modules + * + * Author: SeongJae Park + */ + +#include + +#define DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(attrs) \ + module_param_named(sample_interval, attrs.sample_interval, \ + ulong, 0600); \ + module_param_named(aggr_interval, attrs.aggr_interval, ulong, \ + 0600); \ + module_param_named(min_nr_regions, attrs.min_nr_regions, ulong, \ + 0600); \ + module_param_named(max_nr_regions, attrs.max_nr_regions, ulong, \ + 0600); + +#define DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(quota) \ + module_param_named(quota_ms, quota.ms, ulong, 0600); \ + module_param_named(quota_reset_interval_ms, \ + quota.reset_interval, ulong, 0600); + +#define DEFINE_DAMON_MODULES_DAMOS_QUOTAS(quota) \ + DEFINE_DAMON_MODULES_DAMOS_TIME_QUOTA(quota) \ + module_param_named(quota_sz, quota.sz, ulong, 0600); + +#define DEFINE_DAMON_MODULES_WMARKS_PARAMS(wmarks) \ + module_param_named(wmarks_interval, wmarks.interval, ulong, \ + 0600); \ + module_param_named(wmarks_high, wmarks.high, ulong, 0600); \ + module_param_named(wmarks_mid, wmarks.mid, ulong, 0600); \ + module_param_named(wmarks_low, wmarks.low, ulong, 0600); + +#define DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(stat, try_name, \ + succ_name, qt_exceed_name) \ + module_param_named(nr_##try_name, stat.nr_tried, ulong, 0400); \ + module_param_named(bytes_##try_name, stat.sz_tried, ulong, \ + 0400); \ + module_param_named(nr_##succ_name, stat.nr_applied, ulong, \ + 0400); \ + module_param_named(bytes_##succ_name, stat.sz_applied, ulong, \ + 0400); \ + module_param_named(nr_##qt_exceed_name, stat.qt_exceeds, ulong, \ + 0400); diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index b1335de200e77c6d6e34b91227f8c30945187373..75409601f9349f65241a6eb9b6558bdf574e1f9b 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -88,7 +88,7 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr) #define DAMON_MAX_SUBSCORE (100) #define DAMON_MAX_AGE_IN_LOG (32) -int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, +int damon_hot_score(struct damon_ctx *c, struct damon_region *r, struct damos *s) { unsigned int max_nr_accesses; @@ -99,10 +99,10 @@ int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - max_nr_accesses = c->aggr_interval / c->sample_interval; + max_nr_accesses = c->attrs.aggr_interval / c->attrs.sample_interval; freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; - age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; + age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; age_in_log++, age_in_sec >>= 1) ; @@ -127,48 +127,14 @@ int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, */ hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; - /* Return coldness of the region */ - return DAMOS_MAX_SCORE - hotness; + return hotness; } -int damon_hot_score(struct damon_ctx *c, struct damon_region *r, +int damon_cold_score(struct damon_ctx *c, struct damon_region *r, struct damos *s) { - unsigned int max_nr_accesses; - int freq_subscore; - unsigned int age_in_sec; - int age_in_log, age_subscore; - unsigned int freq_weight = s->quota.weight_nr_accesses; - unsigned int age_weight = s->quota.weight_age; - int hotness; - - max_nr_accesses = c->aggr_interval / c->sample_interval; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses; - - age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000; - for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; - age_in_log++, age_in_sec >>= 1) - ; + int hotness = damon_hot_score(c, r, s); - /* If frequency is 0, higher age means it's colder */ - if (freq_subscore == 0) - age_in_log *= -1; - - /* - * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG]. - * Scale it to be in [0, 100] and set it as age subscore. - */ - age_in_log += DAMON_MAX_AGE_IN_LOG; - age_subscore = age_in_log * DAMON_MAX_SUBSCORE / - DAMON_MAX_AGE_IN_LOG / 2; - - hotness = (freq_weight * freq_subscore + age_weight * age_subscore); - if (freq_weight + age_weight) - hotness /= freq_weight + age_weight; - /* - * Transform it to fit in [0, DAMOS_MAX_SCORE] - */ - hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE; - - return hotness; + /* Return coldness of the region */ + return DAMOS_MAX_SCORE - hotness; } diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h index 52329ff361cd050850f000d513e8ea16276aa1d8..8d82d37222042ffd1f52e6d73e0d46a0a8e74f34 100644 --- a/mm/damon/ops-common.h +++ b/mm/damon/ops-common.h @@ -12,7 +12,7 @@ struct page *damon_get_page(unsigned long pfn); void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr); void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr); -int damon_pageout_score(struct damon_ctx *c, struct damon_region *r, +int damon_cold_score(struct damon_ctx *c, struct damon_region *r, struct damos *s); int damon_hot_score(struct damon_ctx *c, struct damon_region *r, struct damos *s); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index dc131c6a5403875484baa059eef779254bd0166d..e1a4315c4be6aa21149b7571ee31329621a2d0c0 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -63,8 +63,7 @@ out: folio_put(folio); } -static void __damon_pa_prepare_access_check(struct damon_ctx *ctx, - struct damon_region *r) +static void __damon_pa_prepare_access_check(struct damon_region *r) { r->sampling_addr = damon_rand(r->ar.start, r->ar.end); @@ -78,7 +77,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) - __damon_pa_prepare_access_check(ctx, r); + __damon_pa_prepare_access_check(r); } } @@ -166,8 +165,7 @@ out: return result.accessed; } -static void __damon_pa_check_access(struct damon_ctx *ctx, - struct damon_region *r) +static void __damon_pa_check_access(struct damon_region *r) { static unsigned long last_addr; static unsigned long last_page_sz = PAGE_SIZE; @@ -196,7 +194,7 @@ static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) damon_for_each_target(t, ctx) { damon_for_each_region(r, t) { - __damon_pa_check_access(ctx, r); + __damon_pa_check_access(r); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); } } @@ -233,7 +231,8 @@ static unsigned long damon_pa_pageout(struct damon_region *r) return applied * PAGE_SIZE; } -static unsigned long damon_pa_mark_accessed(struct damon_region *r) +static inline unsigned long damon_pa_mark_accessed_or_deactivate( + struct damon_region *r, bool mark_accessed) { unsigned long addr, applied = 0; @@ -242,27 +241,24 @@ static unsigned long damon_pa_mark_accessed(struct damon_region *r) if (!page) continue; - mark_page_accessed(page); + if (mark_accessed) + mark_page_accessed(page); + else + deactivate_page(page); put_page(page); applied++; } return applied * PAGE_SIZE; } -static unsigned long damon_pa_deactivate_pages(struct damon_region *r) +static unsigned long damon_pa_mark_accessed(struct damon_region *r) { - unsigned long addr, applied = 0; - - for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) { - struct page *page = damon_get_page(PHYS_PFN(addr)); + return damon_pa_mark_accessed_or_deactivate(r, true); +} - if (!page) - continue; - deactivate_page(page); - put_page(page); - applied++; - } - return applied * PAGE_SIZE; +static unsigned long damon_pa_deactivate_pages(struct damon_region *r) +{ + return damon_pa_mark_accessed_or_deactivate(r, false); } static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, @@ -276,7 +272,10 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx, return damon_pa_mark_accessed(r); case DAMOS_LRU_DEPRIO: return damon_pa_deactivate_pages(r); + case DAMOS_STAT: + break; default: + /* DAMOS actions that not yet supported by 'paddr'. */ break; } return 0; @@ -288,11 +287,11 @@ static int damon_pa_scheme_score(struct damon_ctx *context, { switch (scheme->action) { case DAMOS_PAGEOUT: - return damon_pageout_score(context, r, scheme); + return damon_cold_score(context, r, scheme); case DAMOS_LRU_PRIO: return damon_hot_score(context, r, scheme); case DAMOS_LRU_DEPRIO: - return damon_pageout_score(context, r, scheme); + return damon_cold_score(context, r, scheme); default: break; } diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index a7faf51b4bd4adaec23221468340cd9a6e83f166..162c9b1ca00fd0de35264bc9f215a797a981a44a 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -13,6 +13,8 @@ #include #include +#include "modules-common.h" + #ifdef MODULE_PARAM_PREFIX #undef MODULE_PARAM_PREFIX #endif @@ -50,124 +52,35 @@ module_param(commit_inputs, bool, 0600); static unsigned long min_age __read_mostly = 120000000; module_param(min_age, ulong, 0600); -/* - * Limit of time for trying the reclamation in milliseconds. - * - * DAMON_RECLAIM tries to use only up to this time within a time window - * (quota_reset_interval_ms) for trying reclamation of cold pages. This can be - * used for limiting CPU consumption of DAMON_RECLAIM. If the value is zero, - * the limit is disabled. - * - * 10 ms by default. - */ -static unsigned long quota_ms __read_mostly = 10; -module_param(quota_ms, ulong, 0600); - -/* - * Limit of size of memory for the reclamation in bytes. - * - * DAMON_RECLAIM charges amount of memory which it tried to reclaim within a - * time window (quota_reset_interval_ms) and makes no more than this limit is - * tried. This can be used for limiting consumption of CPU and IO. If this - * value is zero, the limit is disabled. - * - * 128 MiB by default. - */ -static unsigned long quota_sz __read_mostly = 128 * 1024 * 1024; -module_param(quota_sz, ulong, 0600); - -/* - * The time/size quota charge reset interval in milliseconds. - * - * The charge reset interval for the quota of time (quota_ms) and size - * (quota_sz). That is, DAMON_RECLAIM does not try reclamation for more than - * quota_ms milliseconds or quota_sz bytes within quota_reset_interval_ms - * milliseconds. - * - * 1 second by default. - */ -static unsigned long quota_reset_interval_ms __read_mostly = 1000; -module_param(quota_reset_interval_ms, ulong, 0600); - -/* - * The watermarks check time interval in microseconds. - * - * Minimal time to wait before checking the watermarks, when DAMON_RECLAIM is - * enabled but inactive due to its watermarks rule. 5 seconds by default. - */ -static unsigned long wmarks_interval __read_mostly = 5000000; -module_param(wmarks_interval, ulong, 0600); - -/* - * Free memory rate (per thousand) for the high watermark. - * - * If free memory of the system in bytes per thousand bytes is higher than - * this, DAMON_RECLAIM becomes inactive, so it does nothing but periodically - * checks the watermarks. 500 (50%) by default. - */ -static unsigned long wmarks_high __read_mostly = 500; -module_param(wmarks_high, ulong, 0600); - -/* - * Free memory rate (per thousand) for the middle watermark. - * - * If free memory of the system in bytes per thousand bytes is between this and - * the low watermark, DAMON_RECLAIM becomes active, so starts the monitoring - * and the reclaiming. 400 (40%) by default. - */ -static unsigned long wmarks_mid __read_mostly = 400; -module_param(wmarks_mid, ulong, 0600); - -/* - * Free memory rate (per thousand) for the low watermark. - * - * If free memory of the system in bytes per thousand bytes is lower than this, - * DAMON_RECLAIM becomes inactive, so it does nothing but periodically checks - * the watermarks. In the case, the system falls back to the LRU-based page - * granularity reclamation logic. 200 (20%) by default. - */ -static unsigned long wmarks_low __read_mostly = 200; -module_param(wmarks_low, ulong, 0600); - -/* - * Sampling interval for the monitoring in microseconds. - * - * The sampling interval of DAMON for the cold memory monitoring. Please refer - * to the DAMON documentation for more detail. 5 ms by default. - */ -static unsigned long sample_interval __read_mostly = 5000; -module_param(sample_interval, ulong, 0600); - -/* - * Aggregation interval for the monitoring in microseconds. - * - * The aggregation interval of DAMON for the cold memory monitoring. Please - * refer to the DAMON documentation for more detail. 100 ms by default. - */ -static unsigned long aggr_interval __read_mostly = 100000; -module_param(aggr_interval, ulong, 0600); - -/* - * Minimum number of monitoring regions. - * - * The minimal number of monitoring regions of DAMON for the cold memory - * monitoring. This can be used to set lower-bound of the monitoring quality. - * But, setting this too high could result in increased monitoring overhead. - * Please refer to the DAMON documentation for more detail. 10 by default. - */ -static unsigned long min_nr_regions __read_mostly = 10; -module_param(min_nr_regions, ulong, 0600); - -/* - * Maximum number of monitoring regions. - * - * The maximum number of monitoring regions of DAMON for the cold memory - * monitoring. This can be used to set upper-bound of the monitoring overhead. - * However, setting this too low could result in bad monitoring quality. - * Please refer to the DAMON documentation for more detail. 1000 by default. - */ -static unsigned long max_nr_regions __read_mostly = 1000; -module_param(max_nr_regions, ulong, 0600); +static struct damos_quota damon_reclaim_quota = { + /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ + .ms = 10, + .sz = 128 * 1024 * 1024, + .reset_interval = 1000, + /* Within the quota, page out older regions first. */ + .weight_sz = 0, + .weight_nr_accesses = 0, + .weight_age = 1 +}; +DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); + +static struct damos_watermarks damon_reclaim_wmarks = { + .metric = DAMOS_WMARK_FREE_MEM_RATE, + .interval = 5000000, /* 5 seconds */ + .high = 500, /* 50 percent */ + .mid = 400, /* 40 percent */ + .low = 200, /* 20 percent */ +}; +DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); + +static struct damon_attrs damon_reclaim_mon_attrs = { + .sample_interval = 5000, /* 5 ms */ + .aggr_interval = 100000, /* 100 ms */ + .ops_update_interval = 0, + .min_nr_regions = 10, + .max_nr_regions = 1000, +}; +DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); /* * Start of the target memory region in physical address. @@ -196,119 +109,44 @@ module_param(monitor_region_end, ulong, 0600); static int kdamond_pid __read_mostly = -1; module_param(kdamond_pid, int, 0400); -/* - * Number of memory regions that tried to be reclaimed. - */ -static unsigned long nr_reclaim_tried_regions __read_mostly; -module_param(nr_reclaim_tried_regions, ulong, 0400); - -/* - * Total bytes of memory regions that tried to be reclaimed. - */ -static unsigned long bytes_reclaim_tried_regions __read_mostly; -module_param(bytes_reclaim_tried_regions, ulong, 0400); - -/* - * Number of memory regions that successfully be reclaimed. - */ -static unsigned long nr_reclaimed_regions __read_mostly; -module_param(nr_reclaimed_regions, ulong, 0400); - -/* - * Total bytes of memory regions that successfully be reclaimed. - */ -static unsigned long bytes_reclaimed_regions __read_mostly; -module_param(bytes_reclaimed_regions, ulong, 0400); - -/* - * Number of times that the time/space quota limits have exceeded - */ -static unsigned long nr_quota_exceeds __read_mostly; -module_param(nr_quota_exceeds, ulong, 0400); +static struct damos_stat damon_reclaim_stat; +DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, + reclaim_tried_regions, reclaimed_regions, quota_exceeds); static struct damon_ctx *ctx; static struct damon_target *target; -struct damon_reclaim_ram_walk_arg { - unsigned long start; - unsigned long end; -}; - -static int walk_system_ram(struct resource *res, void *arg) -{ - struct damon_reclaim_ram_walk_arg *a = arg; - - if (a->end - a->start < resource_size(res)) { - a->start = res->start; - a->end = res->end; - } - return 0; -} - -/* - * Find biggest 'System RAM' resource and store its start and end address in - * @start and @end, respectively. If no System RAM is found, returns false. - */ -static bool get_monitoring_region(unsigned long *start, unsigned long *end) -{ - struct damon_reclaim_ram_walk_arg arg = {}; - - walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); - if (arg.end <= arg.start) - return false; - - *start = arg.start; - *end = arg.end; - return true; -} - static struct damos *damon_reclaim_new_scheme(void) { - struct damos_watermarks wmarks = { - .metric = DAMOS_WMARK_FREE_MEM_RATE, - .interval = wmarks_interval, - .high = wmarks_high, - .mid = wmarks_mid, - .low = wmarks_low, - }; - struct damos_quota quota = { - /* - * Do not try reclamation for more than quota_ms milliseconds - * or quota_sz bytes within quota_reset_interval_ms. - */ - .ms = quota_ms, - .sz = quota_sz, - .reset_interval = quota_reset_interval_ms, - /* Within the quota, page out older regions first. */ - .weight_sz = 0, - .weight_nr_accesses = 0, - .weight_age = 1 + struct damos_access_pattern pattern = { + /* Find regions having PAGE_SIZE or larger size */ + .min_sz_region = PAGE_SIZE, + .max_sz_region = ULONG_MAX, + /* and not accessed at all */ + .min_nr_accesses = 0, + .max_nr_accesses = 0, + /* for min_age or more micro-seconds */ + .min_age_region = min_age / + damon_reclaim_mon_attrs.aggr_interval, + .max_age_region = UINT_MAX, }; - struct damos *scheme = damon_new_scheme( - /* Find regions having PAGE_SIZE or larger size */ - PAGE_SIZE, ULONG_MAX, - /* and not accessed at all */ - 0, 0, - /* for min_age or more micro-seconds, and */ - min_age / aggr_interval, UINT_MAX, + + return damon_new_scheme( + &pattern, /* page out those, as soon as found */ DAMOS_PAGEOUT, /* under the quota. */ - "a, + &damon_reclaim_quota, /* (De)activate this according to the watermarks. */ - &wmarks); - - return scheme; + &damon_reclaim_wmarks); } static int damon_reclaim_apply_parameters(void) { struct damos *scheme; - struct damon_addr_range addr_range; int err = 0; - err = damon_set_attrs(ctx, sample_interval, aggr_interval, 0, - min_nr_regions, max_nr_regions); + err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); if (err) return err; @@ -316,19 +154,11 @@ static int damon_reclaim_apply_parameters(void) scheme = damon_reclaim_new_scheme(); if (!scheme) return -ENOMEM; - err = damon_set_schemes(ctx, &scheme, 1); - if (err) - return err; + damon_set_schemes(ctx, &scheme, 1); - if (monitor_region_start > monitor_region_end) - return -EINVAL; - if (!monitor_region_start && !monitor_region_end && - !get_monitoring_region(&monitor_region_start, - &monitor_region_end)) - return -EINVAL; - addr_range.start = monitor_region_start; - addr_range.end = monitor_region_end; - return damon_set_regions(target, &addr_range, 1); + return damon_set_region_biggest_system_ram_default(target, + &monitor_region_start, + &monitor_region_end); } static int damon_reclaim_turn(bool on) @@ -413,13 +243,8 @@ static int damon_reclaim_after_aggregation(struct damon_ctx *c) struct damos *s; /* update the stats parameter */ - damon_for_each_scheme(s, c) { - nr_reclaim_tried_regions = s->stat.nr_tried; - bytes_reclaim_tried_regions = s->stat.sz_tried; - nr_reclaimed_regions = s->stat.nr_applied; - bytes_reclaimed_regions = s->stat.sz_applied; - nr_quota_exceeds = s->stat.qt_exceeds; - } + damon_for_each_scheme(s, c) + damon_reclaim_stat = s->stat; return damon_reclaim_handle_commit_inputs(); } diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index 7488e27c87c37748179134f0a5cecc5f7bb6d79f..9f1219a67e3f17bf49794581fa266f6b45782d59 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -58,7 +58,7 @@ static ssize_t min_store(struct kobject *kobj, struct kobj_attribute *attr, err = kstrtoul(buf, 0, &min); if (err) - return -EINVAL; + return err; range->min = min; return count; @@ -83,7 +83,7 @@ static ssize_t max_store(struct kobject *kobj, struct kobj_attribute *attr, err = kstrtoul(buf, 0, &max); if (err) - return -EINVAL; + return err; range->max = max; return count; @@ -291,9 +291,7 @@ static ssize_t interval_us_store(struct kobject *kobj, struct damon_sysfs_watermarks, kobj); int err = kstrtoul(buf, 0, &watermarks->interval_us); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t high_show(struct kobject *kobj, @@ -312,9 +310,7 @@ static ssize_t high_store(struct kobject *kobj, struct damon_sysfs_watermarks, kobj); int err = kstrtoul(buf, 0, &watermarks->high); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t mid_show(struct kobject *kobj, @@ -333,9 +329,7 @@ static ssize_t mid_store(struct kobject *kobj, struct damon_sysfs_watermarks, kobj); int err = kstrtoul(buf, 0, &watermarks->mid); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t low_show(struct kobject *kobj, @@ -354,9 +348,7 @@ static ssize_t low_store(struct kobject *kobj, struct damon_sysfs_watermarks, kobj); int err = kstrtoul(buf, 0, &watermarks->low); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static void damon_sysfs_watermarks_release(struct kobject *kobj) @@ -437,9 +429,7 @@ static ssize_t sz_permil_store(struct kobject *kobj, struct damon_sysfs_weights, kobj); int err = kstrtouint(buf, 0, &weights->sz); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t nr_accesses_permil_show(struct kobject *kobj, @@ -458,9 +448,7 @@ static ssize_t nr_accesses_permil_store(struct kobject *kobj, struct damon_sysfs_weights, kobj); int err = kstrtouint(buf, 0, &weights->nr_accesses); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t age_permil_show(struct kobject *kobj, @@ -479,9 +467,7 @@ static ssize_t age_permil_store(struct kobject *kobj, struct damon_sysfs_weights, kobj); int err = kstrtouint(buf, 0, &weights->age); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static void damon_sysfs_weights_release(struct kobject *kobj) @@ -1031,8 +1017,7 @@ static ssize_t nr_schemes_show(struct kobject *kobj, static ssize_t nr_schemes_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct damon_sysfs_schemes *schemes = container_of(kobj, - struct damon_sysfs_schemes, kobj); + struct damon_sysfs_schemes *schemes; int nr, err = kstrtoint(buf, 0, &nr); if (err) @@ -1040,6 +1025,8 @@ static ssize_t nr_schemes_store(struct kobject *kobj, if (nr < 0) return -EINVAL; + schemes = container_of(kobj, struct damon_sysfs_schemes, kobj); + if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; err = damon_sysfs_schemes_add_dirs(schemes, nr); @@ -1110,9 +1097,7 @@ static ssize_t start_store(struct kobject *kobj, struct kobj_attribute *attr, struct damon_sysfs_region, kobj); int err = kstrtoul(buf, 0, ®ion->start); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static ssize_t end_show(struct kobject *kobj, struct kobj_attribute *attr, @@ -1131,9 +1116,7 @@ static ssize_t end_store(struct kobject *kobj, struct kobj_attribute *attr, struct damon_sysfs_region, kobj); int err = kstrtoul(buf, 0, ®ion->end); - if (err) - return -EINVAL; - return count; + return err ? err : count; } static void damon_sysfs_region_release(struct kobject *kobj) @@ -1237,8 +1220,7 @@ static ssize_t nr_regions_show(struct kobject *kobj, static ssize_t nr_regions_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct damon_sysfs_regions *regions = container_of(kobj, - struct damon_sysfs_regions, kobj); + struct damon_sysfs_regions *regions; int nr, err = kstrtoint(buf, 0, &nr); if (err) @@ -1246,6 +1228,8 @@ static ssize_t nr_regions_store(struct kobject *kobj, if (nr < 0) return -EINVAL; + regions = container_of(kobj, struct damon_sysfs_regions, kobj); + if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; err = damon_sysfs_regions_add_dirs(regions, nr); @@ -1440,8 +1424,7 @@ static ssize_t nr_targets_show(struct kobject *kobj, static ssize_t nr_targets_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct damon_sysfs_targets *targets = container_of(kobj, - struct damon_sysfs_targets, kobj); + struct damon_sysfs_targets *targets; int nr, err = kstrtoint(buf, 0, &nr); if (err) @@ -1449,6 +1432,8 @@ static ssize_t nr_targets_store(struct kobject *kobj, if (nr < 0) return -EINVAL; + targets = container_of(kobj, struct damon_sysfs_targets, kobj); + if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; err = damon_sysfs_targets_add_dirs(targets, nr); @@ -1525,7 +1510,7 @@ static ssize_t sample_us_store(struct kobject *kobj, int err = kstrtoul(buf, 0, &us); if (err) - return -EINVAL; + return err; intervals->sample_us = us; return count; @@ -1549,7 +1534,7 @@ static ssize_t aggr_us_store(struct kobject *kobj, struct kobj_attribute *attr, int err = kstrtoul(buf, 0, &us); if (err) - return -EINVAL; + return err; intervals->aggr_us = us; return count; @@ -1573,7 +1558,7 @@ static ssize_t update_us_store(struct kobject *kobj, int err = kstrtoul(buf, 0, &us); if (err) - return -EINVAL; + return err; intervals->update_us = us; return count; @@ -1962,8 +1947,7 @@ static ssize_t nr_contexts_show(struct kobject *kobj, static ssize_t nr_contexts_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct damon_sysfs_contexts *contexts = container_of(kobj, - struct damon_sysfs_contexts, kobj); + struct damon_sysfs_contexts *contexts; int nr, err; err = kstrtoint(buf, 0, &nr); @@ -1973,6 +1957,7 @@ static ssize_t nr_contexts_store(struct kobject *kobj, if (nr < 0 || 1 < nr) return -EINVAL; + contexts = container_of(kobj, struct damon_sysfs_contexts, kobj); if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; err = damon_sysfs_contexts_add_dirs(contexts, nr); @@ -2127,18 +2112,23 @@ static int damon_sysfs_set_attrs(struct damon_ctx *ctx, struct damon_sysfs_intervals *sys_intervals = sys_attrs->intervals; struct damon_sysfs_ul_range *sys_nr_regions = sys_attrs->nr_regions_range; - - return damon_set_attrs(ctx, sys_intervals->sample_us, - sys_intervals->aggr_us, sys_intervals->update_us, - sys_nr_regions->min, sys_nr_regions->max); + struct damon_attrs attrs = { + .sample_interval = sys_intervals->sample_us, + .aggr_interval = sys_intervals->aggr_us, + .ops_update_interval = sys_intervals->update_us, + .min_nr_regions = sys_nr_regions->min, + .max_nr_regions = sys_nr_regions->max, + }; + return damon_set_attrs(ctx, &attrs); } static void damon_sysfs_destroy_targets(struct damon_ctx *ctx) { struct damon_target *t, *next; + bool has_pid = damon_target_has_pid(ctx); damon_for_each_target_safe(t, next, ctx) { - if (damon_target_has_pid(ctx)) + if (has_pid) put_pid(t->pid); damon_destroy_target(t); } @@ -2182,12 +2172,12 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target, if (!t) return -ENOMEM; + damon_add_target(ctx, t); if (damon_target_has_pid(ctx)) { t->pid = find_get_pid(sys_target->pid); if (!t->pid) goto destroy_targets_out; } - damon_add_target(ctx, t); err = damon_sysfs_set_regions(t, sys_target->regions); if (err) goto destroy_targets_out; @@ -2259,11 +2249,20 @@ static int damon_sysfs_set_targets(struct damon_ctx *ctx, static struct damos *damon_sysfs_mk_scheme( struct damon_sysfs_scheme *sysfs_scheme) { - struct damon_sysfs_access_pattern *pattern = + struct damon_sysfs_access_pattern *access_pattern = sysfs_scheme->access_pattern; struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas; struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights; struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks; + + struct damos_access_pattern pattern = { + .min_sz_region = access_pattern->sz->min, + .max_sz_region = access_pattern->sz->max, + .min_nr_accesses = access_pattern->nr_accesses->min, + .max_nr_accesses = access_pattern->nr_accesses->max, + .min_age_region = access_pattern->age->min, + .max_age_region = access_pattern->age->max, + }; struct damos_quota quota = { .ms = sysfs_quotas->ms, .sz = sysfs_quotas->sz, @@ -2280,10 +2279,8 @@ static struct damos *damon_sysfs_mk_scheme( .low = sysfs_wmarks->low, }; - return damon_new_scheme(pattern->sz->min, pattern->sz->max, - pattern->nr_accesses->min, pattern->nr_accesses->max, - pattern->age->min, pattern->age->max, - sysfs_scheme->action, "a, &wmarks); + return damon_new_scheme(&pattern, sysfs_scheme->action, "a, + &wmarks); } static int damon_sysfs_set_schemes(struct damon_ctx *ctx, @@ -2309,7 +2306,7 @@ static void damon_sysfs_before_terminate(struct damon_ctx *ctx) { struct damon_target *t, *next; - if (ctx->ops.id != DAMON_OPS_VADDR && ctx->ops.id != DAMON_OPS_FVADDR) + if (!damon_target_has_pid(ctx)) return; mutex_lock(&ctx->kdamond_lock); @@ -2455,8 +2452,7 @@ static int damon_sysfs_turn_damon_on(struct damon_sysfs_kdamond *kdamond) struct damon_ctx *ctx; int err; - if (kdamond->damon_ctx && - damon_sysfs_ctx_running(kdamond->damon_ctx)) + if (damon_sysfs_kdamond_running(kdamond)) return -EBUSY; if (damon_sysfs_cmd_request.kdamond == kdamond) return -EBUSY; @@ -2579,19 +2575,16 @@ static ssize_t pid_show(struct kobject *kobj, struct damon_sysfs_kdamond *kdamond = container_of(kobj, struct damon_sysfs_kdamond, kobj); struct damon_ctx *ctx; - int pid; + int pid = -1; if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; ctx = kdamond->damon_ctx; - if (!ctx) { - pid = -1; + if (!ctx) goto out; - } + mutex_lock(&ctx->kdamond_lock); - if (!ctx->kdamond) - pid = -1; - else + if (ctx->kdamond) pid = ctx->kdamond->pid; mutex_unlock(&ctx->kdamond_lock); out: @@ -2657,23 +2650,18 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) kdamonds->kdamonds_arr = NULL; } -static int damon_sysfs_nr_running_ctxs(struct damon_sysfs_kdamond **kdamonds, +static bool damon_sysfs_kdamonds_busy(struct damon_sysfs_kdamond **kdamonds, int nr_kdamonds) { - int nr_running_ctxs = 0; int i; for (i = 0; i < nr_kdamonds; i++) { - struct damon_ctx *ctx = kdamonds[i]->damon_ctx; - - if (!ctx) - continue; - mutex_lock(&ctx->kdamond_lock); - if (ctx->kdamond) - nr_running_ctxs++; - mutex_unlock(&ctx->kdamond_lock); + if (damon_sysfs_kdamond_running(kdamonds[i]) || + damon_sysfs_cmd_request.kdamond == kdamonds[i]) + return true; } - return nr_running_ctxs; + + return false; } static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, @@ -2682,15 +2670,9 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, struct damon_sysfs_kdamond **kdamonds_arr, *kdamond; int err, i; - if (damon_sysfs_nr_running_ctxs(kdamonds->kdamonds_arr, kdamonds->nr)) + if (damon_sysfs_kdamonds_busy(kdamonds->kdamonds_arr, kdamonds->nr)) return -EBUSY; - for (i = 0; i < kdamonds->nr; i++) { - if (damon_sysfs_cmd_request.kdamond == - kdamonds->kdamonds_arr[i]) - return -EBUSY; - } - damon_sysfs_kdamonds_rm_dirs(kdamonds); if (!nr_kdamonds) return 0; @@ -2741,8 +2723,7 @@ static ssize_t nr_kdamonds_show(struct kobject *kobj, static ssize_t nr_kdamonds_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - struct damon_sysfs_kdamonds *kdamonds = container_of(kobj, - struct damon_sysfs_kdamonds, kobj); + struct damon_sysfs_kdamonds *kdamonds; int nr, err; err = kstrtoint(buf, 0, &nr); @@ -2751,6 +2732,8 @@ static ssize_t nr_kdamonds_store(struct kobject *kobj, if (nr < 0) return -EINVAL; + kdamonds = container_of(kobj, struct damon_sysfs_kdamonds, kobj); + if (!mutex_trylock(&damon_sysfs_lock)) return -EBUSY; err = damon_sysfs_kdamonds_add_dirs(kdamonds, nr); diff --git a/mm/damon/vaddr-test.h b/mm/damon/vaddr-test.h index d4f55f3491007b4966a1ba4c4019b5aaa8ee0488..bce37c4875402d439895430cc4b35b1b75778f89 100644 --- a/mm/damon/vaddr-test.h +++ b/mm/damon/vaddr-test.h @@ -14,33 +14,19 @@ #include -static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) +static void __link_vmas(struct maple_tree *mt, struct vm_area_struct *vmas, + ssize_t nr_vmas) { - int i, j; - unsigned long largest_gap, gap; + int i; + MA_STATE(mas, mt, 0, 0); if (!nr_vmas) return; - for (i = 0; i < nr_vmas - 1; i++) { - vmas[i].vm_next = &vmas[i + 1]; - - vmas[i].vm_rb.rb_left = NULL; - vmas[i].vm_rb.rb_right = &vmas[i + 1].vm_rb; - - largest_gap = 0; - for (j = i; j < nr_vmas; j++) { - if (j == 0) - continue; - gap = vmas[j].vm_start - vmas[j - 1].vm_end; - if (gap > largest_gap) - largest_gap = gap; - } - vmas[i].rb_subtree_gap = largest_gap; - } - vmas[i].vm_next = NULL; - vmas[i].vm_rb.rb_right = NULL; - vmas[i].rb_subtree_gap = 0; + mas_lock(&mas); + for (i = 0; i < nr_vmas; i++) + vma_mas_store(&vmas[i], &mas); + mas_unlock(&mas); } /* @@ -72,6 +58,7 @@ static void __link_vmas(struct vm_area_struct *vmas, ssize_t nr_vmas) */ static void damon_test_three_regions_in_vmas(struct kunit *test) { + static struct mm_struct mm; struct damon_addr_range regions[3] = {0,}; /* 10-20-25, 200-210-220, 300-305, 307-330 */ struct vm_area_struct vmas[] = { @@ -83,9 +70,10 @@ static void damon_test_three_regions_in_vmas(struct kunit *test) (struct vm_area_struct) {.vm_start = 307, .vm_end = 330}, }; - __link_vmas(vmas, 6); + mt_init_flags(&mm.mm_mt, MM_MT_FLAGS); + __link_vmas(&mm.mm_mt, vmas, ARRAY_SIZE(vmas)); - __damon_va_three_regions(&vmas[0], regions); + __damon_va_three_regions(&mm, regions); KUNIT_EXPECT_EQ(test, 10ul, regions[0].start); KUNIT_EXPECT_EQ(test, 25ul, regions[0].end); diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 3c7b9d6dca95d30dfce3e0a41fc63febeb9e0dce..15f03df66db60e5db517bc73413a72e834f53563 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -72,7 +72,7 @@ static int damon_va_evenly_split_region(struct damon_target *t, return -EINVAL; orig_end = r->ar.end; - sz_orig = r->ar.end - r->ar.start; + sz_orig = damon_sz_region(r); sz_piece = ALIGN_DOWN(sz_orig / nr_pieces, DAMON_MIN_REGION); if (!sz_piece) @@ -113,37 +113,38 @@ static unsigned long sz_range(struct damon_addr_range *r) * * Returns 0 if success, or negative error code otherwise. */ -static int __damon_va_three_regions(struct vm_area_struct *vma, +static int __damon_va_three_regions(struct mm_struct *mm, struct damon_addr_range regions[3]) { - struct damon_addr_range gap = {0}, first_gap = {0}, second_gap = {0}; - struct vm_area_struct *last_vma = NULL; - unsigned long start = 0; - struct rb_root rbroot; - - /* Find two biggest gaps so that first_gap > second_gap > others */ - for (; vma; vma = vma->vm_next) { - if (!last_vma) { - start = vma->vm_start; - goto next; - } + struct damon_addr_range first_gap = {0}, second_gap = {0}; + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma, *prev = NULL; + unsigned long start; - if (vma->rb_subtree_gap <= sz_range(&second_gap)) { - rbroot.rb_node = &vma->vm_rb; - vma = rb_entry(rb_last(&rbroot), - struct vm_area_struct, vm_rb); + /* + * Find the two biggest gaps so that first_gap > second_gap > others. + * If this is too slow, it can be optimised to examine the maple + * tree gaps. + */ + for_each_vma(vmi, vma) { + unsigned long gap; + + if (!prev) { + start = vma->vm_start; goto next; } - - gap.start = last_vma->vm_end; - gap.end = vma->vm_start; - if (sz_range(&gap) > sz_range(&second_gap)) { - swap(gap, second_gap); - if (sz_range(&second_gap) > sz_range(&first_gap)) - swap(second_gap, first_gap); + gap = vma->vm_start - prev->vm_end; + + if (gap > sz_range(&first_gap)) { + second_gap = first_gap; + first_gap.start = prev->vm_end; + first_gap.end = vma->vm_start; + } else if (gap > sz_range(&second_gap)) { + second_gap.start = prev->vm_end; + second_gap.end = vma->vm_start; } next: - last_vma = vma; + prev = vma; } if (!sz_range(&second_gap) || !sz_range(&first_gap)) @@ -159,7 +160,7 @@ next: regions[1].start = ALIGN(first_gap.end, DAMON_MIN_REGION); regions[1].end = ALIGN(second_gap.start, DAMON_MIN_REGION); regions[2].start = ALIGN(second_gap.end, DAMON_MIN_REGION); - regions[2].end = ALIGN(last_vma->vm_end, DAMON_MIN_REGION); + regions[2].end = ALIGN(prev->vm_end, DAMON_MIN_REGION); return 0; } @@ -180,7 +181,7 @@ static int damon_va_three_regions(struct damon_target *t, return -EINVAL; mmap_read_lock(mm); - rc = __damon_va_three_regions(mm->mmap, regions); + rc = __damon_va_three_regions(mm, regions); mmap_read_unlock(mm); mmput(mm); @@ -250,8 +251,8 @@ static void __damon_va_init_regions(struct damon_ctx *ctx, for (i = 0; i < 3; i++) sz += regions[i].end - regions[i].start; - if (ctx->min_nr_regions) - sz /= ctx->min_nr_regions; + if (ctx->attrs.min_nr_regions) + sz /= ctx->attrs.min_nr_regions; if (sz < DAMON_MIN_REGION) sz = DAMON_MIN_REGION; @@ -302,9 +303,14 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr, pte_t *pte; spinlock_t *ptl; - if (pmd_huge(*pmd)) { + if (pmd_trans_huge(*pmd)) { ptl = pmd_lock(walk->mm, pmd); - if (pmd_huge(*pmd)) { + if (!pmd_present(*pmd)) { + spin_unlock(ptl); + return 0; + } + + if (pmd_trans_huge(*pmd)) { damon_pmdp_mkold(pmd, walk->mm, addr); spin_unlock(ptl); return 0; @@ -391,8 +397,8 @@ static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) * Functions for the access checking of the regions */ -static void __damon_va_prepare_access_check(struct damon_ctx *ctx, - struct mm_struct *mm, struct damon_region *r) +static void __damon_va_prepare_access_check(struct mm_struct *mm, + struct damon_region *r) { r->sampling_addr = damon_rand(r->ar.start, r->ar.end); @@ -410,7 +416,7 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) if (!mm) continue; damon_for_each_region(r, t) - __damon_va_prepare_access_check(ctx, mm, r); + __damon_va_prepare_access_check(mm, r); mmput(mm); } } @@ -429,9 +435,14 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, struct damon_young_walk_private *priv = walk->private; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (pmd_huge(*pmd)) { + if (pmd_trans_huge(*pmd)) { ptl = pmd_lock(walk->mm, pmd); - if (!pmd_huge(*pmd)) { + if (!pmd_present(*pmd)) { + spin_unlock(ptl); + return 0; + } + + if (!pmd_trans_huge(*pmd)) { spin_unlock(ptl); goto regular_page; } @@ -532,16 +543,15 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * mm 'mm_struct' for the given virtual address space * r the region to be checked */ -static void __damon_va_check_access(struct damon_ctx *ctx, - struct mm_struct *mm, struct damon_region *r) +static void __damon_va_check_access(struct mm_struct *mm, + struct damon_region *r, bool same_target) { - static struct mm_struct *last_mm; static unsigned long last_addr; static unsigned long last_page_sz = PAGE_SIZE; static bool last_accessed; /* If the region is in the last checked page, reuse the result */ - if (mm == last_mm && (ALIGN_DOWN(last_addr, last_page_sz) == + if (same_target && (ALIGN_DOWN(last_addr, last_page_sz) == ALIGN_DOWN(r->sampling_addr, last_page_sz))) { if (last_accessed) r->nr_accesses++; @@ -552,7 +562,6 @@ static void __damon_va_check_access(struct damon_ctx *ctx, if (last_accessed) r->nr_accesses++; - last_mm = mm; last_addr = r->sampling_addr; } @@ -562,14 +571,17 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; + bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); if (!mm) continue; + same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(ctx, mm, r); + __damon_va_check_access(mm, r, same_target); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); + same_target = true; } mmput(mm); } @@ -581,9 +593,8 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) * Functions for the target validity check and cleanup */ -static bool damon_va_target_valid(void *target) +static bool damon_va_target_valid(struct damon_target *t) { - struct damon_target *t = target; struct task_struct *task; task = damon_get_task_struct(t); @@ -607,7 +618,7 @@ static unsigned long damos_madvise(struct damon_target *target, { struct mm_struct *mm; unsigned long start = PAGE_ALIGN(r->ar.start); - unsigned long len = PAGE_ALIGN(r->ar.end - r->ar.start); + unsigned long len = PAGE_ALIGN(damon_sz_region(r)); unsigned long applied; mm = damon_get_mm(target); @@ -646,6 +657,9 @@ static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx, case DAMOS_STAT: return 0; default: + /* + * DAMOS actions that are not yet supported by 'vaddr'. + */ return 0; } @@ -659,7 +673,7 @@ static int damon_va_scheme_score(struct damon_ctx *context, switch (scheme->action) { case DAMOS_PAGEOUT: - return damon_pageout_score(context, r, scheme); + return damon_cold_score(context, r, scheme); default: break; } diff --git a/mm/debug.c b/mm/debug.c index bef329bf28f01aef04256db49f0790fcb65ea8d1..0fd15ba70d16317d76163ba7340e2d54cfd064d9 100644 --- a/mm/debug.c +++ b/mm/debug.c @@ -139,13 +139,11 @@ EXPORT_SYMBOL(dump_page); void dump_vma(const struct vm_area_struct *vma) { - pr_emerg("vma %px start %px end %px\n" - "next %px prev %px mm %px\n" + pr_emerg("vma %px start %px end %px mm %px\n" "prot %lx anon_vma %px vm_ops %px\n" "pgoff %lx file %px private_data %px\n" "flags: %#lx(%pGv)\n", - vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_next, - vma->vm_prev, vma->vm_mm, + vma, (void *)vma->vm_start, (void *)vma->vm_end, vma->vm_mm, (unsigned long)pgprot_val(vma->vm_page_prot), vma->anon_vma, vma->vm_ops, vma->vm_pgoff, vma->vm_file, vma->vm_private_data, @@ -155,11 +153,11 @@ EXPORT_SYMBOL(dump_vma); void dump_mm(const struct mm_struct *mm) { - pr_emerg("mm %px mmap %px seqnum %llu task_size %lu\n" + pr_emerg("mm %px task_size %lu\n" #ifdef CONFIG_MMU "get_unmapped_area %px\n" #endif - "mmap_base %lu mmap_legacy_base %lu highest_vm_end %lu\n" + "mmap_base %lu mmap_legacy_base %lu\n" "pgd %px mm_users %d mm_count %d pgtables_bytes %lu map_count %d\n" "hiwater_rss %lx hiwater_vm %lx total_vm %lx locked_vm %lx\n" "pinned_vm %llx data_vm %lx exec_vm %lx stack_vm %lx\n" @@ -183,11 +181,11 @@ void dump_mm(const struct mm_struct *mm) "tlb_flush_pending %d\n" "def_flags: %#lx(%pGv)\n", - mm, mm->mmap, (long long) mm->vmacache_seqnum, mm->task_size, + mm, mm->task_size, #ifdef CONFIG_MMU mm->get_unmapped_area, #endif - mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end, + mm->mmap_base, mm->mmap_legacy_base, mm->pgd, atomic_read(&mm->mm_users), atomic_read(&mm->mm_count), mm_pgtables_bytes(mm), diff --git a/mm/filemap.c b/mm/filemap.c index 15800334147b3cdf6f098b84306412f0e1130fab..08341616ae7a1edf70b55c8bbc35a53a35ac57ab 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -632,22 +632,23 @@ bool filemap_range_has_writeback(struct address_space *mapping, { XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT); pgoff_t max = end_byte >> PAGE_SHIFT; - struct page *page; + struct folio *folio; if (end_byte < start_byte) return false; rcu_read_lock(); - xas_for_each(&xas, page, max) { - if (xas_retry(&xas, page)) + xas_for_each(&xas, folio, max) { + if (xas_retry(&xas, folio)) continue; - if (xa_is_value(page)) + if (xa_is_value(folio)) continue; - if (PageDirty(page) || PageLocked(page) || PageWriteback(page)) + if (folio_test_dirty(folio) || folio_test_locked(folio) || + folio_test_writeback(folio)) break; } rcu_read_unlock(); - return page != NULL; + return folio != NULL; } EXPORT_SYMBOL_GPL(filemap_range_has_writeback); @@ -1221,15 +1222,12 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, struct wait_page_queue wait_page; wait_queue_entry_t *wait = &wait_page.wait; bool thrashing = false; - bool delayacct = false; unsigned long pflags; + bool in_thrashing; if (bit_nr == PG_locked && !folio_test_uptodate(folio) && folio_test_workingset(folio)) { - if (!folio_test_swapbacked(folio)) { - delayacct_thrashing_start(); - delayacct = true; - } + delayacct_thrashing_start(&in_thrashing); psi_memstall_enter(&pflags); thrashing = true; } @@ -1329,8 +1327,7 @@ repeat: finish_wait(q, wait); if (thrashing) { - if (delayacct) - delayacct_thrashing_end(); + delayacct_thrashing_end(&in_thrashing); psi_memstall_leave(&pflags); } @@ -1378,17 +1375,14 @@ void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep, struct wait_page_queue wait_page; wait_queue_entry_t *wait = &wait_page.wait; bool thrashing = false; - bool delayacct = false; unsigned long pflags; + bool in_thrashing; wait_queue_head_t *q; struct folio *folio = page_folio(pfn_swap_entry_to_page(entry)); q = folio_waitqueue(folio); if (!folio_test_uptodate(folio) && folio_test_workingset(folio)) { - if (!folio_test_swapbacked(folio)) { - delayacct_thrashing_start(); - delayacct = true; - } + delayacct_thrashing_start(&in_thrashing); psi_memstall_enter(&pflags); thrashing = true; } @@ -1435,8 +1429,7 @@ void migration_entry_wait_on_locked(swp_entry_t entry, pte_t *ptep, finish_wait(q, wait); if (thrashing) { - if (delayacct) - delayacct_thrashing_end(); + delayacct_thrashing_end(&in_thrashing); psi_memstall_leave(&pflags); } } @@ -1467,7 +1460,7 @@ EXPORT_SYMBOL(folio_wait_bit_killable); * * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal. */ -int folio_put_wait_locked(struct folio *folio, int state) +static int folio_put_wait_locked(struct folio *folio, int state) { return folio_wait_bit_common(folio, PG_locked, state, DROP); } @@ -1633,24 +1626,26 @@ EXPORT_SYMBOL(folio_end_writeback); */ void page_endio(struct page *page, bool is_write, int err) { + struct folio *folio = page_folio(page); + if (!is_write) { if (!err) { - SetPageUptodate(page); + folio_mark_uptodate(folio); } else { - ClearPageUptodate(page); - SetPageError(page); + folio_clear_uptodate(folio); + folio_set_error(folio); } - unlock_page(page); + folio_unlock(folio); } else { if (err) { struct address_space *mapping; - SetPageError(page); - mapping = page_mapping(page); + folio_set_error(folio); + mapping = folio_mapping(folio); if (mapping) mapping_set_error(mapping, err); } - end_page_writeback(page); + folio_end_writeback(folio); } } EXPORT_SYMBOL_GPL(page_endio); @@ -2195,30 +2190,31 @@ bool folio_more_pages(struct folio *folio, pgoff_t index, pgoff_t max) } /** - * find_get_pages_contig - gang contiguous pagecache lookup + * filemap_get_folios_contig - Get a batch of contiguous folios * @mapping: The address_space to search - * @index: The starting page index - * @nr_pages: The maximum number of pages - * @pages: Where the resulting pages are placed + * @start: The starting page index + * @end: The final page index (inclusive) + * @fbatch: The batch to fill * - * find_get_pages_contig() works exactly like find_get_pages_range(), - * except that the returned number of pages are guaranteed to be - * contiguous. + * filemap_get_folios_contig() works exactly like filemap_get_folios(), + * except the returned folios are guaranteed to be contiguous. This may + * not return all contiguous folios if the batch gets filled up. * - * Return: the number of pages which were found. + * Return: The number of folios found. + * Also update @start to be positioned for traversal of the next folio. */ -unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, - unsigned int nr_pages, struct page **pages) + +unsigned filemap_get_folios_contig(struct address_space *mapping, + pgoff_t *start, pgoff_t end, struct folio_batch *fbatch) { - XA_STATE(xas, &mapping->i_pages, index); + XA_STATE(xas, &mapping->i_pages, *start); + unsigned long nr; struct folio *folio; - unsigned int ret = 0; - - if (unlikely(!nr_pages)) - return 0; rcu_read_lock(); - for (folio = xas_load(&xas); folio; folio = xas_next(&xas)) { + + for (folio = xas_load(&xas); folio && xas.xa_index <= end; + folio = xas_next(&xas)) { if (xas_retry(&xas, folio)) continue; /* @@ -2226,33 +2222,45 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, * No current caller is looking for DAX entries. */ if (xa_is_value(folio)) - break; + goto update_start; if (!folio_try_get_rcu(folio)) goto retry; if (unlikely(folio != xas_reload(&xas))) - goto put_page; + goto put_folio; -again: - pages[ret] = folio_file_page(folio, xas.xa_index); - if (++ret == nr_pages) - break; - if (folio_more_pages(folio, xas.xa_index, ULONG_MAX)) { - xas.xa_index++; - folio_ref_inc(folio); - goto again; + if (!folio_batch_add(fbatch, folio)) { + nr = folio_nr_pages(folio); + + if (folio_test_hugetlb(folio)) + nr = 1; + *start = folio->index + nr; + goto out; } continue; -put_page: +put_folio: folio_put(folio); + retry: xas_reset(&xas); } + +update_start: + nr = folio_batch_count(fbatch); + + if (nr) { + folio = fbatch->folios[nr - 1]; + if (folio_test_hugetlb(folio)) + *start = folio->index + 1; + else + *start = folio->index + folio_nr_pages(folio); + } +out: rcu_read_unlock(); - return ret; + return folio_batch_count(fbatch); } -EXPORT_SYMBOL(find_get_pages_contig); +EXPORT_SYMBOL(filemap_get_folios_contig); /** * find_get_pages_range_tag - Find and return head pages matching @tag. @@ -2382,6 +2390,8 @@ retry: static int filemap_read_folio(struct file *file, filler_t filler, struct folio *folio) { + bool workingset = folio_test_workingset(folio); + unsigned long pflags; int error; /* @@ -2390,8 +2400,13 @@ static int filemap_read_folio(struct file *file, filler_t filler, * fails. */ folio_clear_error(folio); + /* Start the actual read. The read will unlock the page. */ + if (unlikely(workingset)) + psi_memstall_enter(&pflags); error = filler(file, folio); + if (unlikely(workingset)) + psi_memstall_leave(&pflags); if (error) return error; @@ -3712,7 +3727,7 @@ ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) unsigned long offset; /* Offset into pagecache page */ unsigned long bytes; /* Bytes to write to page */ size_t copied; /* Bytes copied from user */ - void *fsdata; + void *fsdata = NULL; offset = (pos & (PAGE_SIZE - 1)); bytes = min_t(unsigned long, PAGE_SIZE - offset, diff --git a/mm/folio-compat.c b/mm/folio-compat.c index 458618c7302c39c7fe2afaad9f1cac56d4cb7d62..e1e23b4947d73bd769bd319c821956911ff41fa6 100644 --- a/mm/folio-compat.c +++ b/mm/folio-compat.c @@ -88,6 +88,12 @@ void lru_cache_add(struct page *page) } EXPORT_SYMBOL(lru_cache_add); +void lru_cache_add_inactive_or_unevictable(struct page *page, + struct vm_area_struct *vma) +{ + folio_add_lru_vma(page_folio(page), vma); +} + int add_to_page_cache_lru(struct page *page, struct address_space *mapping, pgoff_t index, gfp_t gfp) { diff --git a/mm/frontswap.c b/mm/frontswap.c index 1a97610308cbc28a8b16b6eeb16bd978a71d1be7..279e55b4ed8739f51e49959854555fedb536ddf2 100644 --- a/mm/frontswap.c +++ b/mm/frontswap.c @@ -125,6 +125,9 @@ void frontswap_init(unsigned type, unsigned long *map) * p->frontswap set to something valid to work properly. */ frontswap_map_set(sis, map); + + if (!frontswap_enabled()) + return; frontswap_ops->init(type); } diff --git a/mm/gup.c b/mm/gup.c index 7328251574307b17423d410c554e2879ffff85f2..fe195d47de74a74259cb10729b184ed4ee887ad8 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -158,6 +158,13 @@ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags) else folio_ref_add(folio, refs * (GUP_PIN_COUNTING_BIAS - 1)); + /* + * Adjust the pincount before re-checking the PTE for changes. + * This is essentially a smp_mb() and is paired with a memory + * barrier in page_try_share_anon_rmap(). + */ + smp_mb__after_atomic(); + node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs); return folio; @@ -478,14 +485,42 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address, return -EEXIST; } -/* - * FOLL_FORCE can write to even unwritable pte's, but only - * after we've gone through a COW cycle and they are dirty. - */ -static inline bool can_follow_write_pte(pte_t pte, unsigned int flags) +/* FOLL_FORCE can write to even unwritable PTEs in COW mappings. */ +static inline bool can_follow_write_pte(pte_t pte, struct page *page, + struct vm_area_struct *vma, + unsigned int flags) { - return pte_write(pte) || - ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte)); + /* If the pte is writable, we can write to the page. */ + if (pte_write(pte)) + return true; + + /* Maybe FOLL_FORCE is set to override it? */ + if (!(flags & FOLL_FORCE)) + return false; + + /* But FOLL_FORCE has no effect on shared mappings */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return false; + + /* ... or read-only private ones */ + if (!(vma->vm_flags & VM_MAYWRITE)) + return false; + + /* ... or already writable ones that just need to take a write fault */ + if (vma->vm_flags & VM_WRITE) + return false; + + /* + * See can_change_pte_writable(): we broke COW and could map the page + * writable if we have an exclusive anonymous page ... + */ + if (!page || !PageAnon(page) || !PageAnonExclusive(page)) + return false; + + /* ... and a write-fault isn't required for other reasons. */ + if (vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte)) + return false; + return !userfaultfd_pte_wp(vma, pte); } static struct page *follow_page_pte(struct vm_area_struct *vma, @@ -502,6 +537,18 @@ static struct page *follow_page_pte(struct vm_area_struct *vma, if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == (FOLL_PIN | FOLL_GET))) return ERR_PTR(-EINVAL); + + /* + * Considering PTE level hugetlb, like continuous-PTE hugetlb on + * ARM64 architecture. + */ + if (is_vm_hugetlb_page(vma)) { + page = follow_huge_pmd_pte(vma, address, flags); + if (page) + return page; + return no_page_table(vma, flags); + } + retry: if (unlikely(pmd_bad(*pmd))) return no_page_table(vma, flags); @@ -526,14 +573,21 @@ retry: migration_entry_wait(mm, pmd, address); goto retry; } - if ((flags & FOLL_NUMA) && pte_protnone(pte)) + if (pte_protnone(pte) && !gup_can_follow_protnone(flags)) goto no_page; - if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) { - pte_unmap_unlock(ptep, ptl); - return NULL; - } page = vm_normal_page(vma, address, pte); + + /* + * We only care about anon pages in can_follow_write_pte() and don't + * have to worry about pte_devmap() because they are never anon. + */ + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) { + page = NULL; + goto out; + } + if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) { /* * Only return device mapping pages in the FOLL_GET or FOLL_PIN @@ -627,7 +681,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, if (pmd_none(pmdval)) return no_page_table(vma, flags); if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) { - page = follow_huge_pmd(mm, address, pmd, flags); + page = follow_huge_pmd_pte(vma, address, flags); if (page) return page; return no_page_table(vma, flags); @@ -672,7 +726,7 @@ retry: if (likely(!pmd_trans_huge(pmdval))) return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap); - if ((flags & FOLL_NUMA) && pmd_protnone(pmdval)) + if (pmd_protnone(pmdval) && !gup_can_follow_protnone(flags)) return no_page_table(vma, flags); retry_locked: @@ -986,17 +1040,6 @@ static int faultin_page(struct vm_area_struct *vma, return -EBUSY; } - /* - * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when - * necessary, even if maybe_mkwrite decided not to set pte_write. We - * can thus safely do subsequent page lookups as if they were reads. - * But only do so when looping for pte_write is futile: in some cases - * userspace may also be wanting to write to the gotten user page, - * which a read fault here might prevent (a readonly page might get - * reCOWed by userspace write). - */ - if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE)) - *flags |= FOLL_COW; return 0; } @@ -1129,14 +1172,6 @@ static long __get_user_pages(struct mm_struct *mm, VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); - /* - * If FOLL_FORCE is set then do not force a full fault as the hinting - * fault information is unrelated to the reference behaviour of a task - * using the address space - */ - if (!(gup_flags & FOLL_FORCE)) - gup_flags |= FOLL_NUMA; - do { struct page *page; unsigned int foll_flags = gup_flags; @@ -1643,10 +1678,11 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) if (!locked) { locked = 1; mmap_read_lock(mm); - vma = find_vma(mm, nstart); + vma = find_vma_intersection(mm, nstart, end); } else if (nstart >= vma->vm_end) - vma = vma->vm_next; - if (!vma || vma->vm_start >= end) + vma = find_vma_intersection(mm, vma->vm_end, end); + + if (!vma) break; /* * Set [nstart; nend) to intersection of desired address @@ -1903,20 +1939,16 @@ struct page *get_dump_page(unsigned long addr) #ifdef CONFIG_MIGRATION /* - * Check whether all pages are pinnable, if so return number of pages. If some - * pages are not pinnable, migrate them, and unpin all pages. Return zero if - * pages were migrated, or if some pages were not successfully isolated. - * Return negative error if migration fails. + * Returns the number of collected pages. Return value is always >= 0. */ -static long check_and_migrate_movable_pages(unsigned long nr_pages, - struct page **pages, - unsigned int gup_flags) +static unsigned long collect_longterm_unpinnable_pages( + struct list_head *movable_page_list, + unsigned long nr_pages, + struct page **pages) { - unsigned long isolation_error_count = 0, i; + unsigned long i, collected = 0; struct folio *prev_folio = NULL; - LIST_HEAD(movable_page_list); - bool drain_allow = true, coherent_pages = false; - int ret = 0; + bool drain_allow = true; for (i = 0; i < nr_pages; i++) { struct folio *folio = page_folio(pages[i]); @@ -1925,45 +1957,16 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, continue; prev_folio = folio; - /* - * Device coherent pages are managed by a driver and should not - * be pinned indefinitely as it prevents the driver moving the - * page. So when trying to pin with FOLL_LONGTERM instead try - * to migrate the page out of device memory. - */ - if (folio_is_device_coherent(folio)) { - /* - * We always want a new GUP lookup with device coherent - * pages. - */ - pages[i] = 0; - coherent_pages = true; - - /* - * Migration will fail if the page is pinned, so convert - * the pin on the source page to a normal reference. - */ - if (gup_flags & FOLL_PIN) { - get_page(&folio->page); - unpin_user_page(&folio->page); - } + if (folio_is_longterm_pinnable(folio)) + continue; - ret = migrate_device_coherent_page(&folio->page); - if (ret) - goto unpin_pages; + collected++; + if (folio_is_device_coherent(folio)) continue; - } - if (folio_is_longterm_pinnable(folio)) - continue; - /* - * Try to move out any movable page before pinning the range. - */ if (folio_test_hugetlb(folio)) { - if (isolate_hugetlb(&folio->page, - &movable_page_list)) - isolation_error_count++; + isolate_hugetlb(&folio->page, movable_page_list); continue; } @@ -1972,63 +1975,124 @@ static long check_and_migrate_movable_pages(unsigned long nr_pages, drain_allow = false; } - if (folio_isolate_lru(folio)) { - isolation_error_count++; + if (!folio_isolate_lru(folio)) continue; - } - list_add_tail(&folio->lru, &movable_page_list); + + list_add_tail(&folio->lru, movable_page_list); node_stat_mod_folio(folio, NR_ISOLATED_ANON + folio_is_file_lru(folio), folio_nr_pages(folio)); } - if (!list_empty(&movable_page_list) || isolation_error_count || - coherent_pages) - goto unpin_pages; + return collected; +} - /* - * If list is empty, and no isolation errors, means that all pages are - * in the correct zone. - */ - return nr_pages; +/* + * Unpins all pages and migrates device coherent pages and movable_page_list. + * Returns -EAGAIN if all pages were successfully migrated or -errno for failure + * (or partial success). + */ +static int migrate_longterm_unpinnable_pages( + struct list_head *movable_page_list, + unsigned long nr_pages, + struct page **pages) +{ + int ret; + unsigned long i; -unpin_pages: - /* - * pages[i] might be NULL if any device coherent pages were found. - */ for (i = 0; i < nr_pages; i++) { - if (!pages[i]) + struct folio *folio = page_folio(pages[i]); + + if (folio_is_device_coherent(folio)) { + /* + * Migration will fail if the page is pinned, so convert + * the pin on the source page to a normal reference. + */ + pages[i] = NULL; + folio_get(folio); + gup_put_folio(folio, 1, FOLL_PIN); + + if (migrate_device_coherent_page(&folio->page)) { + ret = -EBUSY; + goto err; + } + continue; + } - if (gup_flags & FOLL_PIN) - unpin_user_page(pages[i]); - else - put_page(pages[i]); + /* + * We can't migrate pages with unexpected references, so drop + * the reference obtained by __get_user_pages_locked(). + * Migrating pages have been added to movable_page_list after + * calling folio_isolate_lru() which takes a reference so the + * page won't be freed if it's migrating. + */ + unpin_user_page(pages[i]); + pages[i] = NULL; } - if (!list_empty(&movable_page_list)) { + if (!list_empty(movable_page_list)) { struct migration_target_control mtc = { .nid = NUMA_NO_NODE, .gfp_mask = GFP_USER | __GFP_NOWARN, }; - ret = migrate_pages(&movable_page_list, alloc_migration_target, - NULL, (unsigned long)&mtc, MIGRATE_SYNC, - MR_LONGTERM_PIN, NULL); - if (ret > 0) /* number of pages not migrated */ + if (migrate_pages(movable_page_list, alloc_migration_target, + NULL, (unsigned long)&mtc, MIGRATE_SYNC, + MR_LONGTERM_PIN, NULL)) { ret = -ENOMEM; + goto err; + } } - if (ret && !list_empty(&movable_page_list)) - putback_movable_pages(&movable_page_list); + putback_movable_pages(movable_page_list); + + return -EAGAIN; + +err: + for (i = 0; i < nr_pages; i++) + if (pages[i]) + unpin_user_page(pages[i]); + putback_movable_pages(movable_page_list); + return ret; } + +/* + * Check whether all pages are *allowed* to be pinned. Rather confusingly, all + * pages in the range are required to be pinned via FOLL_PIN, before calling + * this routine. + * + * If any pages in the range are not allowed to be pinned, then this routine + * will migrate those pages away, unpin all the pages in the range and return + * -EAGAIN. The caller should re-pin the entire range with FOLL_PIN and then + * call this routine again. + * + * If an error other than -EAGAIN occurs, this indicates a migration failure. + * The caller should give up, and propagate the error back up the call stack. + * + * If everything is OK and all pages in the range are allowed to be pinned, then + * this routine leaves all pages pinned and returns zero for success. + */ +static long check_and_migrate_movable_pages(unsigned long nr_pages, + struct page **pages) +{ + unsigned long collected; + LIST_HEAD(movable_page_list); + + collected = collect_longterm_unpinnable_pages(&movable_page_list, + nr_pages, pages); + if (!collected) + return 0; + + return migrate_longterm_unpinnable_pages(&movable_page_list, nr_pages, + pages); +} #else static long check_and_migrate_movable_pages(unsigned long nr_pages, - struct page **pages, - unsigned int gup_flags) + struct page **pages) { - return nr_pages; + return 0; } #endif /* CONFIG_MIGRATION */ @@ -2044,22 +2108,36 @@ static long __gup_longterm_locked(struct mm_struct *mm, unsigned int gup_flags) { unsigned int flags; - long rc; + long rc, nr_pinned_pages; if (!(gup_flags & FOLL_LONGTERM)) return __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL, gup_flags); + + /* + * If we get to this point then FOLL_LONGTERM is set, and FOLL_LONGTERM + * implies FOLL_PIN (although the reverse is not true). Therefore it is + * correct to unconditionally call check_and_migrate_movable_pages() + * which assumes pages have been pinned via FOLL_PIN. + * + * Enforce the above reasoning by asserting that FOLL_PIN is set. + */ + if (WARN_ON(!(gup_flags & FOLL_PIN))) + return -EINVAL; flags = memalloc_pin_save(); do { - rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, - NULL, gup_flags); - if (rc <= 0) + nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages, + pages, vmas, NULL, + gup_flags); + if (nr_pinned_pages <= 0) { + rc = nr_pinned_pages; break; - rc = check_and_migrate_movable_pages(rc, pages, gup_flags); - } while (!rc); + } + rc = check_and_migrate_movable_pages(nr_pinned_pages, pages); + } while (rc == -EAGAIN); memalloc_pin_restore(flags); - return rc; + return rc ? rc : nr_pinned_pages; } static bool is_valid_gup_flags(unsigned int gup_flags) @@ -2321,8 +2399,28 @@ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start, } #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL -static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +/* + * Fast-gup relies on pte change detection to avoid concurrent pgtable + * operations. + * + * To pin the page, fast-gup needs to do below in order: + * (1) pin the page (by prefetching pte), then (2) check pte not changed. + * + * For the rest of pgtable operations where pgtable updates can be racy + * with fast-gup, we need to do (1) clear pte, then (2) check whether page + * is pinned. + * + * Above will work for all pte-level operations, including THP split. + * + * For THP collapse, it's a bit more complicated because fast-gup may be + * walking a pgtable page that is being freed (pte is still valid but pmd + * can be cleared already). To avoid race in such condition, we need to + * also check pmd here to make sure pmd doesn't change (corresponds to + * pmdp_collapse_flush() in the THP collapse code path). + */ +static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { struct dev_pagemap *pgmap = NULL; int nr_start = *nr, ret = 0; @@ -2334,11 +2432,7 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, struct page *page; struct folio *folio; - /* - * Similar to the PMD case below, NUMA hinting must take slow - * path using the pte_protnone check. - */ - if (pte_protnone(pte)) + if (pte_protnone(pte) && !gup_can_follow_protnone(flags)) goto pte_unmap; if (!pte_access_permitted(pte, flags & FOLL_WRITE)) @@ -2368,7 +2462,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, goto pte_unmap; } - if (unlikely(pte_val(pte) != pte_val(*ptep))) { + if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) || + unlikely(pte_val(pte) != pte_val(*ptep))) { gup_put_folio(folio, 1, flags); goto pte_unmap; } @@ -2415,8 +2510,9 @@ pte_unmap: * get_user_pages_fast_only implementation that can pin pages. Thus it's still * useful to have gup_huge_pmd even if we can't operate on ptes. */ -static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, - unsigned int flags, struct page **pages, int *nr) +static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, + unsigned long end, unsigned int flags, + struct page **pages, int *nr) { return 0; } @@ -2720,12 +2816,8 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) || pmd_devmap(pmd))) { - /* - * NUMA hinting faults need to be handled in the GUP - * slowpath for accounting purposes and so that they - * can be serialised against THP migration. - */ - if (pmd_protnone(pmd)) + if (pmd_protnone(pmd) && + !gup_can_follow_protnone(flags)) return 0; if (!gup_huge_pmd(pmd, pmdp, addr, next, flags, @@ -2740,7 +2832,7 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr, PMD_SHIFT, next, flags, pages, nr)) return 0; - } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr)) + } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr)) return 0; } while (pmdp++, addr = next, addr != end); diff --git a/mm/highmem.c b/mm/highmem.c index c707d7202d5f756b7fad7cfbba07a03849e13bde..db251e77f98f87253a377571334b06aee8caceeb 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -30,6 +30,17 @@ #include #include +#ifdef CONFIG_KMAP_LOCAL +static inline int kmap_local_calc_idx(int idx) +{ + return idx + KM_MAX_IDX * smp_processor_id(); +} + +#ifndef arch_kmap_local_map_idx +#define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx) +#endif +#endif /* CONFIG_KMAP_LOCAL */ + /* * Virtual_count is not a pure "count". * 0 means that it is not mapped, and has not been mapped @@ -142,12 +153,29 @@ pte_t *pkmap_page_table; struct page *__kmap_to_page(void *vaddr) { + unsigned long base = (unsigned long) vaddr & PAGE_MASK; + struct kmap_ctrl *kctrl = ¤t->kmap_ctrl; unsigned long addr = (unsigned long)vaddr; + int i; + + /* kmap() mappings */ + if (WARN_ON_ONCE(addr >= PKMAP_ADDR(0) && + addr < PKMAP_ADDR(LAST_PKMAP))) + return pte_page(pkmap_page_table[PKMAP_NR(addr)]); - if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) { - int i = PKMAP_NR(addr); + /* kmap_local_page() mappings */ + if (WARN_ON_ONCE(base >= __fix_to_virt(FIX_KMAP_END) && + base < __fix_to_virt(FIX_KMAP_BEGIN))) { + for (i = 0; i < kctrl->idx; i++) { + unsigned long base_addr; + int idx; - return pte_page(pkmap_page_table[i]); + idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); + base_addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); + + if (base_addr == base) + return pte_page(kctrl->pteval[i]); + } } return virt_to_page(vaddr); @@ -462,10 +490,6 @@ static inline void kmap_local_idx_pop(void) # define arch_kmap_local_post_unmap(vaddr) do { } while (0) #endif -#ifndef arch_kmap_local_map_idx -#define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx) -#endif - #ifndef arch_kmap_local_unmap_idx #define arch_kmap_local_unmap_idx(idx, vaddr) kmap_local_calc_idx(idx) #endif @@ -494,11 +518,6 @@ static inline bool kmap_high_unmap_local(unsigned long vaddr) return false; } -static inline int kmap_local_calc_idx(int idx) -{ - return idx + KM_MAX_IDX * smp_processor_id(); -} - static pte_t *__kmap_pte; static pte_t *kmap_get_pte(unsigned long vaddr, int idx) diff --git a/mm/hmm.c b/mm/hmm.c index f2aa63b94d9bd8903157a9d25e48df96f762b3a7..3850fb625dda18e118d25d94af68b66b451d30ee 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -253,7 +253,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, cpu_flags = HMM_PFN_VALID; if (is_writable_device_private_entry(entry)) cpu_flags |= HMM_PFN_WRITE; - *hmm_pfn = swp_offset(entry) | cpu_flags; + *hmm_pfn = swp_offset_pfn(entry) | cpu_flags; return 0; } diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8a7c1b344abefb4b1b3903ecc61d143d98d6a9bc..1cc4a5f4791e92d459612bbda25c726b2b4720f9 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -70,9 +71,8 @@ static atomic_t huge_zero_refcount; struct page *huge_zero_page __read_mostly; unsigned long huge_zero_pfn __read_mostly = ~0UL; -bool hugepage_vma_check(struct vm_area_struct *vma, - unsigned long vm_flags, - bool smaps, bool in_pf) +bool hugepage_vma_check(struct vm_area_struct *vma, unsigned long vm_flags, + bool smaps, bool in_pf, bool enforce_sysfs) { if (!vma->vm_mm) /* vdso */ return false; @@ -119,13 +119,12 @@ bool hugepage_vma_check(struct vm_area_struct *vma, * own flags. */ if (!in_pf && shmem_file(vma->vm_file)) - return shmem_huge_enabled(vma); + return shmem_huge_enabled(vma, !enforce_sysfs); - if (!hugepage_flags_enabled()) - return false; - - /* THP settings require madvise. */ - if (!(vm_flags & VM_HUGEPAGE) && !hugepage_flags_always()) + /* Enforce sysfs THP requirements as necessary */ + if (enforce_sysfs && + (!hugepage_flags_enabled() || (!(vm_flags & VM_HUGEPAGE) && + !hugepage_flags_always()))) return false; /* Only regular file is valid */ @@ -164,7 +163,6 @@ retry: count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); return false; } - count_vm_event(THP_ZERO_PAGE_ALLOC); preempt_disable(); if (cmpxchg(&huge_zero_page, NULL, zero_page)) { preempt_enable(); @@ -176,6 +174,7 @@ retry: /* We take additional reference here. It will be put back by shrinker */ atomic_set(&huge_zero_refcount, 2); preempt_enable(); + count_vm_event(THP_ZERO_PAGE_ALLOC); return true; } @@ -772,8 +771,7 @@ static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, return; entry = mk_pmd(zero_page, vma->vm_page_prot); entry = pmd_mkhuge(entry); - if (pgtable) - pgtable_trans_huge_deposit(mm, pmd, pgtable); + pgtable_trans_huge_deposit(mm, pmd, pgtable); set_pmd_at(mm, haddr, pmd, entry); mm_inc_nr_ptes(mm); } @@ -1040,12 +1038,6 @@ struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr, assert_spin_locked(pmd_lockptr(mm, pmd)); - /* - * When we COW a devmap PMD entry, we split it into PTEs, so we should - * not be in this function with `flags & FOLL_COW` set. - */ - WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set"); - /* FOLL_GET and FOLL_PIN are mutually exclusive. */ if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) == (FOLL_PIN | FOLL_GET))) @@ -1313,6 +1305,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) { const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; + struct folio *folio; struct page *page; unsigned long haddr = vmf->address & HPAGE_PMD_MASK; pmd_t orig_pmd = vmf->orig_pmd; @@ -1334,46 +1327,48 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) } page = pmd_page(orig_pmd); + folio = page_folio(page); VM_BUG_ON_PAGE(!PageHead(page), page); /* Early check when only holding the PT lock. */ if (PageAnonExclusive(page)) goto reuse; - if (!trylock_page(page)) { - get_page(page); + if (!folio_trylock(folio)) { + folio_get(folio); spin_unlock(vmf->ptl); - lock_page(page); + folio_lock(folio); spin_lock(vmf->ptl); if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { spin_unlock(vmf->ptl); - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); return 0; } - put_page(page); + folio_put(folio); } /* Recheck after temporarily dropping the PT lock. */ if (PageAnonExclusive(page)) { - unlock_page(page); + folio_unlock(folio); goto reuse; } /* - * See do_wp_page(): we can only reuse the page exclusively if there are - * no additional references. Note that we always drain the LRU - * pagevecs immediately after adding a THP. + * See do_wp_page(): we can only reuse the folio exclusively if + * there are no additional references. Note that we always drain + * the LRU pagevecs immediately after adding a THP. */ - if (page_count(page) > 1 + PageSwapCache(page) * thp_nr_pages(page)) + if (folio_ref_count(folio) > + 1 + folio_test_swapcache(folio) * folio_nr_pages(folio)) goto unlock_fallback; - if (PageSwapCache(page)) - try_to_free_swap(page); - if (page_count(page) == 1) { + if (folio_test_swapcache(folio)) + folio_free_swap(folio); + if (folio_ref_count(folio) == 1) { pmd_t entry; page_move_anon_rmap(page, vma); - unlock_page(page); + folio_unlock(folio); reuse: if (unlikely(unshare)) { spin_unlock(vmf->ptl); @@ -1388,21 +1383,49 @@ reuse: } unlock_fallback: - unlock_page(page); + folio_unlock(folio); spin_unlock(vmf->ptl); fallback: __split_huge_pmd(vma, vmf->pmd, vmf->address, false, NULL); return VM_FAULT_FALLBACK; } -/* - * FOLL_FORCE can write to even unwritable pmd's, but only - * after we've gone through a COW cycle and they are dirty. - */ -static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags) +/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */ +static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page, + struct vm_area_struct *vma, + unsigned int flags) { - return pmd_write(pmd) || - ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd)); + /* If the pmd is writable, we can write to the page. */ + if (pmd_write(pmd)) + return true; + + /* Maybe FOLL_FORCE is set to override it? */ + if (!(flags & FOLL_FORCE)) + return false; + + /* But FOLL_FORCE has no effect on shared mappings */ + if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED)) + return false; + + /* ... or read-only private ones */ + if (!(vma->vm_flags & VM_MAYWRITE)) + return false; + + /* ... or already writable ones that just need to take a write fault */ + if (vma->vm_flags & VM_WRITE) + return false; + + /* + * See can_change_pte_writable(): we broke COW and could map the page + * writable if we have an exclusive anonymous page ... + */ + if (!page || !PageAnon(page) || !PageAnonExclusive(page)) + return false; + + /* ... and a write-fault isn't required for other reasons. */ + if (vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd)) + return false; + return !userfaultfd_huge_pmd_wp(vma, pmd); } struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, @@ -1411,23 +1434,24 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, unsigned int flags) { struct mm_struct *mm = vma->vm_mm; - struct page *page = NULL; + struct page *page; assert_spin_locked(pmd_lockptr(mm, pmd)); - if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags)) - goto out; + page = pmd_page(*pmd); + VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); + + if ((flags & FOLL_WRITE) && + !can_follow_write_pmd(*pmd, page, vma, flags)) + return NULL; /* Avoid dumping huge zero page */ if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd)) return ERR_PTR(-EFAULT); /* Full NUMA hinting faults to serialise migration in fault paths */ - if ((flags & FOLL_NUMA) && pmd_protnone(*pmd)) - goto out; - - page = pmd_page(*pmd); - VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page); + if (pmd_protnone(*pmd) && !gup_can_follow_protnone(flags)) + return NULL; if (!pmd_write(*pmd) && gup_must_unshare(flags, page)) return ERR_PTR(-EMLINK); @@ -1444,7 +1468,6 @@ struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT; VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page); -out: return page; } @@ -1457,7 +1480,7 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) struct page *page; unsigned long haddr = vmf->address & HPAGE_PMD_MASK; int page_nid = NUMA_NO_NODE; - int target_nid, last_cpupid = -1; + int target_nid, last_cpupid = (-1 & LAST_CPUPID_MASK); bool migrated = false; bool was_writable = pmd_savedwrite(oldpmd); int flags = 0; @@ -1478,7 +1501,12 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) flags |= TNF_NO_GROUP; page_nid = page_to_nid(page); - last_cpupid = page_cpupid_last(page); + /* + * For memory tiering mode, cpupid of slow memory page is used + * to record page access time. So use default value. + */ + if (node_is_toptier(page_nid)) + last_cpupid = page_cpupid_last(page); target_nid = numa_migrate_prep(page, vma, haddr, page_nid, &flags); @@ -1802,6 +1830,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (prot_numa) { struct page *page; + bool toptier; /* * Avoid trapping faults against the zero page. The read-only * data is likely to be read-cached on the local CPU and @@ -1814,13 +1843,18 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; page = pmd_page(*pmd); + toptier = node_is_toptier(page_to_nid(page)); /* * Skip scanning top tier node if normal numa * balancing is disabled */ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && - node_is_toptier(page_to_nid(page))) + toptier) goto unlock; + + if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && + !toptier) + xchg_page_access_time(page, jiffies_to_msecs(jiffies)); } /* * In case prot_numa, we are under mmap_read_lock(mm). It's critical @@ -2007,7 +2041,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, pgtable_t pgtable; pmd_t old_pmd, _pmd; bool young, write, soft_dirty, pmd_migration = false, uffd_wp = false; - bool anon_exclusive = false; + bool anon_exclusive = false, dirty = false; unsigned long addr; int i; @@ -2091,13 +2125,16 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = is_writable_migration_entry(entry); if (PageAnon(page)) anon_exclusive = is_readable_exclusive_migration_entry(entry); - young = false; + young = is_migration_entry_young(entry); + dirty = is_migration_entry_dirty(entry); soft_dirty = pmd_swp_soft_dirty(old_pmd); uffd_wp = pmd_swp_uffd_wp(old_pmd); } else { page = pmd_page(old_pmd); - if (pmd_dirty(old_pmd)) + if (pmd_dirty(old_pmd)) { + dirty = true; SetPageDirty(page); + } write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); @@ -2118,6 +2155,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, * * In case we cannot clear PageAnonExclusive(), split the PMD * only and let try_to_migrate_one() fail later. + * + * See page_try_share_anon_rmap(): invalidate PMD first. */ anon_exclusive = PageAnon(page) && PageAnonExclusive(page); if (freeze && anon_exclusive && page_try_share_anon_rmap(page)) @@ -2149,6 +2188,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, else swp_entry = make_readable_migration_entry( page_to_pfn(page + i)); + if (young) + swp_entry = make_migration_entry_young(swp_entry); + if (dirty) + swp_entry = make_migration_entry_dirty(swp_entry); entry = swp_entry_to_pte(swp_entry); if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); @@ -2163,6 +2206,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry = pte_wrprotect(entry); if (!young) entry = pte_mkold(entry); + /* NOTE: this may set soft-dirty too on some archs */ + if (dirty) + entry = pte_mkdirty(entry); if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) @@ -2266,25 +2312,11 @@ out: void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, bool freeze, struct folio *folio) { - pgd_t *pgd; - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; + pmd_t *pmd = mm_find_pmd(vma->vm_mm, address); - pgd = pgd_offset(vma->vm_mm, address); - if (!pgd_present(*pgd)) + if (!pmd) return; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) - return; - - pud = pud_offset(p4d, address); - if (!pud_present(*pud)) - return; - - pmd = pmd_offset(pud, address); - __split_huge_pmd(vma, pmd, address, freeze, folio); } @@ -2312,24 +2344,23 @@ void vma_adjust_trans_huge(struct vm_area_struct *vma, split_huge_pmd_if_needed(vma, end); /* - * If we're also updating the vma->vm_next->vm_start, + * If we're also updating the next vma vm_start, * check if we need to split it. */ if (adjust_next > 0) { - struct vm_area_struct *next = vma->vm_next; + struct vm_area_struct *next = find_vma(vma->vm_mm, vma->vm_end); unsigned long nstart = next->vm_start; nstart += adjust_next; split_huge_pmd_if_needed(next, nstart); } } -static void unmap_page(struct page *page) +static void unmap_folio(struct folio *folio) { - struct folio *folio = page_folio(page); enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD | TTU_SYNC; - VM_BUG_ON_PAGE(!PageHead(page), page); + VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); /* * Anon pages need migration entries to preserve them, but file @@ -2346,7 +2377,7 @@ static void remap_page(struct folio *folio, unsigned long nr) { int i = 0; - /* If unmap_page() uses try_to_migrate() on file, remove this check */ + /* If unmap_folio() uses try_to_migrate() on file, remove this check */ if (!folio_test_anon(folio)) return; for (;;) { @@ -2396,7 +2427,7 @@ static void __split_huge_page_tail(struct page *head, int tail, * for example lock_page() which set PG_waiters. * * Note that for mapped sub-pages of an anonymous THP, - * PG_anon_exclusive has been cleared in unmap_page() and is stored in + * PG_anon_exclusive has been cleared in unmap_folio() and is stored in * the migration entry instead from where remap_page() will restore it. * We can still have PG_anon_exclusive set on effectively unmapped and * unreferenced sub-pages of an anonymous THP: we can simply drop @@ -2416,7 +2447,8 @@ static void __split_huge_page_tail(struct page *head, int tail, #ifdef CONFIG_64BIT (1L << PG_arch_2) | #endif - (1L << PG_dirty))); + (1L << PG_dirty) | + LRU_GEN_MASK | LRU_REFS_MASK)); /* ->mapping in first tail page is compound_mapcount */ VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, @@ -2589,27 +2621,26 @@ bool can_split_folio(struct folio *folio, int *pextra_pins) int split_huge_page_to_list(struct page *page, struct list_head *list) { struct folio *folio = page_folio(page); - struct page *head = &folio->page; - struct deferred_split *ds_queue = get_deferred_split_queue(head); - XA_STATE(xas, &head->mapping->i_pages, head->index); + struct deferred_split *ds_queue = get_deferred_split_queue(&folio->page); + XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct anon_vma *anon_vma = NULL; struct address_space *mapping = NULL; int extra_pins, ret; pgoff_t end; bool is_hzp; - VM_BUG_ON_PAGE(!PageLocked(head), head); - VM_BUG_ON_PAGE(!PageCompound(head), head); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_test_large(folio), folio); - is_hzp = is_huge_zero_page(head); - VM_WARN_ON_ONCE_PAGE(is_hzp, head); + is_hzp = is_huge_zero_page(&folio->page); + VM_WARN_ON_ONCE_FOLIO(is_hzp, folio); if (is_hzp) return -EBUSY; - if (PageWriteback(head)) + if (folio_test_writeback(folio)) return -EBUSY; - if (PageAnon(head)) { + if (folio_test_anon(folio)) { /* * The caller does not necessarily hold an mmap_lock that would * prevent the anon_vma disappearing so we first we take a @@ -2618,7 +2649,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) * is taken to serialise against parallel split or collapse * operations. */ - anon_vma = page_get_anon_vma(head); + anon_vma = folio_get_anon_vma(folio); if (!anon_vma) { ret = -EBUSY; goto out; @@ -2627,7 +2658,9 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) mapping = NULL; anon_vma_lock_write(anon_vma); } else { - mapping = head->mapping; + gfp_t gfp; + + mapping = folio->mapping; /* Truncated ? */ if (!mapping) { @@ -2635,8 +2668,16 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) goto out; } - xas_split_alloc(&xas, head, compound_order(head), - mapping_gfp_mask(mapping) & GFP_RECLAIM_MASK); + gfp = current_gfp_context(mapping_gfp_mask(mapping) & + GFP_RECLAIM_MASK); + + if (folio_test_private(folio) && + !filemap_release_folio(folio, gfp)) { + ret = -EBUSY; + goto out; + } + + xas_split_alloc(&xas, folio, folio_order(folio), gfp); if (xas_error(&xas)) { ret = xas_error(&xas); goto out; @@ -2650,7 +2691,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) * but on 32-bit, i_size_read() takes an irq-unsafe seqlock, * which cannot be nested inside the page tree lock. So note * end now: i_size itself may be changed at any moment, but - * head page lock is good enough to serialize the trimming. + * folio lock is good enough to serialize the trimming. */ end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE); if (shmem_mapping(mapping)) @@ -2658,7 +2699,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) } /* - * Racy check if we can split the page, before unmap_page() will + * Racy check if we can split the page, before unmap_folio() will * split PMDs */ if (!can_split_folio(folio, &extra_pins)) { @@ -2666,38 +2707,38 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) goto out_unlock; } - unmap_page(head); + unmap_folio(folio); /* block interrupt reentry in xa_lock and spinlock */ local_irq_disable(); if (mapping) { /* - * Check if the head page is present in page cache. - * We assume all tail are present too, if head is there. + * Check if the folio is present in page cache. + * We assume all tail are present too, if folio is there. */ xas_lock(&xas); xas_reset(&xas); - if (xas_load(&xas) != head) + if (xas_load(&xas) != folio) goto fail; } /* Prevent deferred_split_scan() touching ->_refcount */ spin_lock(&ds_queue->split_queue_lock); - if (page_ref_freeze(head, 1 + extra_pins)) { - if (!list_empty(page_deferred_list(head))) { + if (folio_ref_freeze(folio, 1 + extra_pins)) { + if (!list_empty(page_deferred_list(&folio->page))) { ds_queue->split_queue_len--; - list_del(page_deferred_list(head)); + list_del(page_deferred_list(&folio->page)); } spin_unlock(&ds_queue->split_queue_lock); if (mapping) { - int nr = thp_nr_pages(head); + int nr = folio_nr_pages(folio); - xas_split(&xas, head, thp_order(head)); - if (PageSwapBacked(head)) { - __mod_lruvec_page_state(head, NR_SHMEM_THPS, + xas_split(&xas, folio, folio_order(folio)); + if (folio_test_swapbacked(folio)) { + __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr); } else { - __mod_lruvec_page_state(head, NR_FILE_THPS, + __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); } @@ -2872,11 +2913,9 @@ static void split_huge_pages_all(void) max_zone_pfn = zone_end_pfn(zone); for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) { int nr_pages; - if (!pfn_valid(pfn)) - continue; - page = pfn_to_page(pfn); - if (!get_page_unless_zero(page)) + page = pfn_to_online_page(pfn); + if (!page || !get_page_unless_zero(page)) continue; if (zone != page_zone(page)) @@ -2963,7 +3002,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, /* FOLL_DUMP to ignore special (like zero) pages */ page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP); - if (IS_ERR_OR_NULL(page) || is_zone_device_page(page)) + if (IS_ERR_OR_NULL(page)) continue; if (!is_transparent_hugepage(page)) @@ -3155,6 +3194,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); pmdval = pmdp_invalidate(vma, address, pvmw->pmd); + /* See page_try_share_anon_rmap(): invalidate PMD first. */ anon_exclusive = PageAnon(page) && PageAnonExclusive(page); if (anon_exclusive && page_try_share_anon_rmap(page)) { set_pmd_at(mm, address, pvmw->pmd, pmdval); @@ -3169,6 +3209,10 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); else entry = make_readable_migration_entry(page_to_pfn(page)); + if (pmd_young(pmdval)) + entry = make_migration_entry_young(entry); + if (pmd_dirty(pmdval)) + entry = make_migration_entry_dirty(entry); pmdswp = swp_entry_to_pmd(entry); if (pmd_soft_dirty(pmdval)) pmdswp = pmd_swp_mksoft_dirty(pmdswp); @@ -3194,13 +3238,18 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) entry = pmd_to_swp_entry(*pvmw->pmd); get_page(new); - pmde = pmd_mkold(mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot))); + pmde = mk_huge_pmd(new, READ_ONCE(vma->vm_page_prot)); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_mksoft_dirty(pmde); if (is_writable_migration_entry(entry)) pmde = maybe_pmd_mkwrite(pmde, vma); if (pmd_swp_uffd_wp(*pvmw->pmd)) pmde = pmd_wrprotect(pmd_mkuffd_wp(pmde)); + if (!is_migration_entry_young(entry)) + pmde = pmd_mkold(pmde); + /* NOTE: this may contain setting soft-dirty on some archs */ + if (PageDirty(new) && is_migration_entry_dirty(entry)) + pmde = pmd_mkdirty(pmde); if (PageAnon(new)) { rmap_t rmap_flags = RMAP_COMPOUND; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 0aee2f3ae15c8251338746659e1b9ea73b1f1807..b586cdd75930b93db9f293c5e190906cd30ffd3a 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -90,6 +91,9 @@ struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp; /* Forward declaration */ static int hugetlb_acct_memory(struct hstate *h, long delta); +static void hugetlb_vma_lock_free(struct vm_area_struct *vma); +static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); +static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); static inline bool subpool_is_free(struct hugepage_subpool *spool) { @@ -257,7 +261,7 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) static struct file_region * get_file_region_entry_from_cache(struct resv_map *resv, long from, long to) { - struct file_region *nrg = NULL; + struct file_region *nrg; VM_BUG_ON(resv->region_cache_count <= 0); @@ -339,7 +343,7 @@ static bool has_same_uncharge_info(struct file_region *rg, static void coalesce_file_region(struct resv_map *resv, struct file_region *rg) { - struct file_region *nrg = NULL, *prg = NULL; + struct file_region *nrg, *prg; prg = list_prev_entry(rg, link); if (&prg->link != &resv->regions && prg->to == rg->from && @@ -456,14 +460,12 @@ static int allocate_file_region_entries(struct resv_map *resv, int regions_needed) __must_hold(&resv->lock) { - struct list_head allocated_regions; + LIST_HEAD(allocated_regions); int to_allocate = 0, i = 0; struct file_region *trg = NULL, *rg = NULL; VM_BUG_ON(regions_needed < 0); - INIT_LIST_HEAD(&allocated_regions); - /* * Check for sufficient descriptors in the cache to accommodate * the number of in progress add operations plus regions_needed. @@ -860,7 +862,7 @@ __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) * faults in a MAP_PRIVATE mapping. Only the process that called mmap() * is guaranteed to have their future faults succeed. * - * With the exception of reset_vma_resv_huge_pages() which is called at fork(), + * With the exception of hugetlb_dup_vma_private() which is called at fork(), * the reserve counters are updated with the hugetlb_lock held. It is safe * to reset the VMA at fork() time as it is not in use yet and there is no * chance of the global counters getting corrupted as a result of the values. @@ -1007,12 +1009,20 @@ static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag) return (get_vma_private_data(vma) & flag) != 0; } -/* Reset counters to 0 and clear all HPAGE_RESV_* flags */ -void reset_vma_resv_huge_pages(struct vm_area_struct *vma) +void hugetlb_dup_vma_private(struct vm_area_struct *vma) { VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma); + /* + * Clear vm_private_data + * - For MAP_PRIVATE mappings, this is the reserve map which does + * not apply to children. Faults generated by the children are + * not guaranteed to succeed, even if read-only. + * - For shared mappings this is a per-vma semaphore that may be + * allocated in a subsequent call to hugetlb_vm_op_open. + */ + vma->vm_private_data = (void *)0; if (!(vma->vm_flags & VM_MAYSHARE)) - vma->vm_private_data = (void *)0; + return; } /* @@ -1043,7 +1053,7 @@ void clear_vma_resv_huge_pages(struct vm_area_struct *vma) kref_put(&reservations->refs, resv_map_release); } - reset_vma_resv_huge_pages(vma); + hugetlb_dup_vma_private(vma); } /* Returns true if the VMA has associated reserve pages */ @@ -1182,6 +1192,11 @@ retry_cpuset: return NULL; } +static unsigned long available_huge_pages(struct hstate *h) +{ + return h->free_huge_pages - h->resv_huge_pages; +} + static struct page *dequeue_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, unsigned long address, int avoid_reserve, @@ -1198,12 +1213,11 @@ static struct page *dequeue_huge_page_vma(struct hstate *h, * have no page reserves. This check ensures that reservations are * not "stolen". The child may still get SIGKILLed */ - if (!vma_has_reserves(vma, chg) && - h->free_huge_pages - h->resv_huge_pages == 0) + if (!vma_has_reserves(vma, chg) && !available_huge_pages(h)) goto err; /* If reserves cannot be used, ensure enough pages are in the pool */ - if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) + if (avoid_reserve && !available_huge_pages(h)) goto err; gfp_mask = htlb_alloc_mask(h); @@ -1308,12 +1322,13 @@ static void __destroy_compound_gigantic_page(struct page *page, { int i; int nr_pages = 1 << order; - struct page *p = page + 1; + struct page *p; atomic_set(compound_mapcount_ptr(page), 0); atomic_set(compound_pincount_ptr(page), 0); - for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { + for (i = 1; i < nr_pages; i++) { + p = nth_page(page, i); p->mapping = NULL; clear_compound_head(p); if (!demote) @@ -1506,6 +1521,10 @@ static void add_hugetlb_page(struct hstate *h, struct page *page, set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); set_page_private(page, 0); + /* + * We have to set HPageVmemmapOptimized again as above + * set_page_private(page, 0) cleared it. + */ SetHPageVmemmapOptimized(page); /* @@ -1530,7 +1549,7 @@ static void add_hugetlb_page(struct hstate *h, struct page *page, static void __update_and_free_page(struct hstate *h, struct page *page) { int i; - struct page *subpage = page; + struct page *subpage; if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported()) return; @@ -1561,8 +1580,8 @@ static void __update_and_free_page(struct hstate *h, struct page *page) if (unlikely(PageHWPoison(page))) hugetlb_clear_page_hwpoison(page); - for (i = 0; i < pages_per_huge_page(h); - i++, subpage = mem_map_next(subpage, page, i)) { + for (i = 0; i < pages_per_huge_page(h); i++) { + subpage = nth_page(page, i); subpage->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | 1 << PG_dirty | 1 << PG_active | 1 << PG_private | @@ -1769,13 +1788,14 @@ static bool __prep_compound_gigantic_page(struct page *page, unsigned int order, { int i, j; int nr_pages = 1 << order; - struct page *p = page + 1; + struct page *p; /* we rely on prep_new_huge_page to set the destructor */ set_compound_order(page, order); - __ClearPageReserved(page); __SetPageHead(page); - for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) { + for (i = 0; i < nr_pages; i++) { + p = nth_page(page, i); + /* * For gigantic hugepages allocated through bootmem at * boot, it's safer to be consistent with the not-gigantic @@ -1814,22 +1834,26 @@ static bool __prep_compound_gigantic_page(struct page *page, unsigned int order, } else { VM_BUG_ON_PAGE(page_count(p), p); } - set_compound_head(p, page); + if (i != 0) + set_compound_head(p, page); } atomic_set(compound_mapcount_ptr(page), -1); atomic_set(compound_pincount_ptr(page), 0); return true; out_error: - /* undo tail page modifications made above */ - p = page + 1; - for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) { - clear_compound_head(p); + /* undo page modifications made above */ + for (j = 0; j < i; j++) { + p = nth_page(page, j); + if (j != 0) + clear_compound_head(p); set_page_refcounted(p); } /* need to clear PG_reserved on remaining tail pages */ - for (; j < nr_pages; j++, p = mem_map_next(p, page, j)) + for (; j < nr_pages; j++) { + p = nth_page(page, j); __ClearPageReserved(p); + } set_compound_order(page, 0); #ifdef CONFIG_64BIT page[1].compound_nr = 0; @@ -1918,6 +1942,7 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, int order = huge_page_order(h); struct page *page; bool alloc_try_hard = true; + bool retry = true; /* * By default we always try hard to allocate the page with @@ -1933,7 +1958,21 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, gfp_mask |= __GFP_RETRY_MAYFAIL; if (nid == NUMA_NO_NODE) nid = numa_mem_id(); +retry: page = __alloc_pages(gfp_mask, order, nid, nmask); + + /* Freeze head page */ + if (page && !page_ref_freeze(page, 1)) { + __free_pages(page, order); + if (retry) { /* retry once */ + retry = false; + goto retry; + } + /* WOW! twice in a row. */ + pr_warn("HugeTLB head page unexpected inflated ref count\n"); + page = NULL; + } + if (page) __count_vm_event(HTLB_BUDDY_PGALLOC); else @@ -1961,6 +2000,9 @@ static struct page *alloc_buddy_huge_page(struct hstate *h, /* * Common helper to allocate a fresh hugetlb page. All specific allocators * should use this function to get new hugetlb pages + * + * Note that returned page is 'frozen': ref count of head page and all tail + * pages is zero. */ static struct page *alloc_fresh_huge_page(struct hstate *h, gfp_t gfp_mask, int nid, nodemask_t *nmask, @@ -2018,7 +2060,7 @@ static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed, if (!page) return 0; - put_page(page); /* free it into the hugepage allocator */ + free_huge_page(page); /* free it into the hugepage allocator */ return 1; } @@ -2087,7 +2129,7 @@ retry: if (!page_count(page)) { struct page *head = compound_head(page); struct hstate *h = page_hstate(head); - if (h->free_huge_pages - h->resv_huge_pages == 0) + if (!available_huge_pages(h)) goto out; /* @@ -2175,10 +2217,9 @@ int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) * Allocates a fresh surplus page from the page allocator. */ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, - int nid, nodemask_t *nmask, bool zero_ref) + int nid, nodemask_t *nmask) { struct page *page = NULL; - bool retry = false; if (hstate_is_gigantic(h)) return NULL; @@ -2188,7 +2229,6 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, goto out_unlock; spin_unlock_irq(&hugetlb_lock); -retry: page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); if (!page) return NULL; @@ -2204,34 +2244,10 @@ retry: if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { SetHPageTemporary(page); spin_unlock_irq(&hugetlb_lock); - put_page(page); + free_huge_page(page); return NULL; } - if (zero_ref) { - /* - * Caller requires a page with zero ref count. - * We will drop ref count here. If someone else is holding - * a ref, the page will be freed when they drop it. Abuse - * temporary page flag to accomplish this. - */ - SetHPageTemporary(page); - if (!put_page_testzero(page)) { - /* - * Unexpected inflated ref count on freshly allocated - * huge. Retry once. - */ - pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n"); - spin_unlock_irq(&hugetlb_lock); - if (retry) - return NULL; - - retry = true; - goto retry; - } - ClearHPageTemporary(page); - } - h->surplus_huge_pages++; h->surplus_huge_pages_node[page_to_nid(page)]++; @@ -2253,6 +2269,9 @@ static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask, if (!page) return NULL; + /* fresh huge pages are frozen */ + set_page_refcounted(page); + /* * We do not account these pages as surplus because they are only * temporary and will be released properly on the last reference @@ -2280,14 +2299,14 @@ struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h, gfp_t gfp = gfp_mask | __GFP_NOWARN; gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - page = alloc_surplus_huge_page(h, gfp, nid, nodemask, false); + page = alloc_surplus_huge_page(h, gfp, nid, nodemask); /* Fallback to all nodes if page==NULL */ nodemask = NULL; } if (!page) - page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask, false); + page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask); mpol_cond_put(mpol); return page; } @@ -2297,7 +2316,7 @@ struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, nodemask_t *nmask, gfp_t gfp_mask) { spin_lock_irq(&hugetlb_lock); - if (h->free_huge_pages - h->resv_huge_pages > 0) { + if (available_huge_pages(h)) { struct page *page; page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask); @@ -2336,7 +2355,7 @@ struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, static int gather_surplus_pages(struct hstate *h, long delta) __must_hold(&hugetlb_lock) { - struct list_head surplus_list; + LIST_HEAD(surplus_list); struct page *page, *tmp; int ret; long i; @@ -2351,14 +2370,13 @@ static int gather_surplus_pages(struct hstate *h, long delta) } allocated = 0; - INIT_LIST_HEAD(&surplus_list); ret = -ENOMEM; retry: spin_unlock_irq(&hugetlb_lock); for (i = 0; i < needed; i++) { page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), - NUMA_NO_NODE, NULL, true); + NUMA_NO_NODE, NULL); if (!page) { alloc_ok = false; break; @@ -2720,7 +2738,6 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page, { gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; int nid = page_to_nid(old_page); - bool alloc_retry = false; struct page *new_page; int ret = 0; @@ -2731,30 +2748,9 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page, * the pool. This simplifies and let us do most of the processing * under the lock. */ -alloc_retry: new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL); if (!new_page) return -ENOMEM; - /* - * If all goes well, this page will be directly added to the free - * list in the pool. For this the ref count needs to be zero. - * Attempt to drop now, and retry once if needed. It is VERY - * unlikely there is another ref on the page. - * - * If someone else has a reference to the page, it will be freed - * when they drop their ref. Abuse temporary page flag to accomplish - * this. Retry once if there is an inflated ref count. - */ - SetHPageTemporary(new_page); - if (!put_page_testzero(new_page)) { - if (alloc_retry) - return -EBUSY; - - alloc_retry = true; - goto alloc_retry; - } - ClearHPageTemporary(new_page); - __prep_new_huge_page(h, new_page); retry: @@ -2934,6 +2930,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, } spin_lock_irq(&hugetlb_lock); list_add(&page->lru, &h->hugepage_activelist); + set_page_refcounted(page); /* Fall through */ } hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); @@ -3038,7 +3035,7 @@ static void __init gather_bootmem_prealloc(void) if (prep_compound_gigantic_page(page, huge_page_order(h))) { WARN_ON(PageReserved(page)); prep_new_huge_page(h, page, page_to_nid(page)); - put_page(page); /* add to the hugepage allocator */ + free_huge_page(page); /* add to the hugepage allocator */ } else { /* VERY unlikely inflated ref count on a tail page */ free_gigantic_page(page, huge_page_order(h)); @@ -3070,7 +3067,7 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) &node_states[N_MEMORY], NULL); if (!page) break; - put_page(page); /* free it into the hugepage allocator */ + free_huge_page(page); /* free it into the hugepage allocator */ } cond_resched(); } @@ -3420,6 +3417,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) { int i, nid = page_to_nid(page); struct hstate *target_hstate; + struct page *subpage; int rc = 0; target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order); @@ -3453,15 +3451,15 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) mutex_lock(&target_hstate->resize_lock); for (i = 0; i < pages_per_huge_page(h); i += pages_per_huge_page(target_hstate)) { + subpage = nth_page(page, i); if (hstate_is_gigantic(target_hstate)) - prep_compound_gigantic_page_for_demote(page + i, + prep_compound_gigantic_page_for_demote(subpage, target_hstate->order); else - prep_compound_page(page + i, target_hstate->order); - set_page_private(page + i, 0); - set_page_refcounted(page + i); - prep_new_huge_page(target_hstate, page + i, nid); - put_page(page + i); + prep_compound_page(subpage, target_hstate->order); + set_page_private(subpage, 0); + prep_new_huge_page(target_hstate, subpage, nid); + free_huge_page(subpage); } mutex_unlock(&target_hstate->resize_lock); @@ -3472,7 +3470,8 @@ static int demote_free_huge_page(struct hstate *h, struct page *page) * based on pool changes for the demoted page. */ h->max_huge_pages--; - target_hstate->max_huge_pages += pages_per_huge_page(h); + target_hstate->max_huge_pages += + pages_per_huge_page(h) / pages_per_huge_page(target_hstate); return rc; } @@ -3714,7 +3713,7 @@ static ssize_t demote_store(struct kobject *kobj, unsigned long nr_available; nodemask_t nodes_allowed, *n_mask; struct hstate *h; - int err = 0; + int err; int nid; err = kstrtoul(buf, 10, &nr_demote); @@ -3765,8 +3764,7 @@ HSTATE_ATTR_WO(demote); static ssize_t demote_size_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - int nid; - struct hstate *h = kobj_to_hstate(kobj, &nid); + struct hstate *h = kobj_to_hstate(kobj, NULL); unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K; return sysfs_emit(buf, "%lukB\n", demote_size); @@ -3779,7 +3777,6 @@ static ssize_t demote_size_store(struct kobject *kobj, struct hstate *h, *demote_hstate; unsigned long demote_size; unsigned int demote_order; - int nid; demote_size = (unsigned long)memparse(buf, NULL); @@ -3791,7 +3788,7 @@ static ssize_t demote_size_store(struct kobject *kobj, return -EINVAL; /* demote order must be smaller than hstate order */ - h = kobj_to_hstate(kobj, &nid); + h = kobj_to_hstate(kobj, NULL); if (demote_order >= h->order) return -EINVAL; @@ -3845,35 +3842,26 @@ static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent, if (retval) { kobject_put(hstate_kobjs[hi]); hstate_kobjs[hi] = NULL; + return retval; } if (h->demote_order) { - if (sysfs_create_group(hstate_kobjs[hi], - &hstate_demote_attr_group)) + retval = sysfs_create_group(hstate_kobjs[hi], + &hstate_demote_attr_group); + if (retval) { pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name); + sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group); + kobject_put(hstate_kobjs[hi]); + hstate_kobjs[hi] = NULL; + return retval; + } } - return retval; -} - -static void __init hugetlb_sysfs_init(void) -{ - struct hstate *h; - int err; - - hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); - if (!hugepages_kobj) - return; - - for_each_hstate(h) { - err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, - hstate_kobjs, &hstate_attr_group); - if (err) - pr_err("HugeTLB: Unable to add hstate %s", h->name); - } + return 0; } #ifdef CONFIG_NUMA +static bool hugetlb_sysfs_initialized __ro_after_init; /* * node_hstate/s - associate per node hstate attributes, via their kobjects, @@ -3929,7 +3917,7 @@ static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp) * Unregister hstate attributes from a single node device. * No-op if no hstate attributes attached. */ -static void hugetlb_unregister_node(struct node *node) +void hugetlb_unregister_node(struct node *node) { struct hstate *h; struct node_hstate *nhs = &node_hstates[node->dev.id]; @@ -3939,10 +3927,15 @@ static void hugetlb_unregister_node(struct node *node) for_each_hstate(h) { int idx = hstate_index(h); - if (nhs->hstate_kobjs[idx]) { - kobject_put(nhs->hstate_kobjs[idx]); - nhs->hstate_kobjs[idx] = NULL; - } + struct kobject *hstate_kobj = nhs->hstate_kobjs[idx]; + + if (!hstate_kobj) + continue; + if (h->demote_order) + sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group); + sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group); + kobject_put(hstate_kobj); + nhs->hstate_kobjs[idx] = NULL; } kobject_put(nhs->hugepages_kobj); @@ -3954,12 +3947,15 @@ static void hugetlb_unregister_node(struct node *node) * Register hstate attributes for a single node device. * No-op if attributes already registered. */ -static void hugetlb_register_node(struct node *node) +void hugetlb_register_node(struct node *node) { struct hstate *h; struct node_hstate *nhs = &node_hstates[node->dev.id]; int err; + if (!hugetlb_sysfs_initialized) + return; + if (nhs->hugepages_kobj) return; /* already allocated */ @@ -3990,18 +3986,8 @@ static void __init hugetlb_register_all_nodes(void) { int nid; - for_each_node_state(nid, N_MEMORY) { - struct node *node = node_devices[nid]; - if (node->dev.id == nid) - hugetlb_register_node(node); - } - - /* - * Let the node device driver know we're here so it can - * [un]register hstate attributes on node hotplug. - */ - register_hugetlbfs_with_node(hugetlb_register_node, - hugetlb_unregister_node); + for_each_online_node(nid) + hugetlb_register_node(node_devices[nid]); } #else /* !CONFIG_NUMA */ @@ -4017,6 +4003,36 @@ static void hugetlb_register_all_nodes(void) { } #endif +#ifdef CONFIG_CMA +static void __init hugetlb_cma_check(void); +#else +static inline __init void hugetlb_cma_check(void) +{ +} +#endif + +static void __init hugetlb_sysfs_init(void) +{ + struct hstate *h; + int err; + + hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj); + if (!hugepages_kobj) + return; + + for_each_hstate(h) { + err = hugetlb_sysfs_add_hstate(h, hugepages_kobj, + hstate_kobjs, &hstate_attr_group); + if (err) + pr_err("HugeTLB: Unable to add hstate %s", h->name); + } + +#ifdef CONFIG_NUMA + hugetlb_sysfs_initialized = true; +#endif + hugetlb_register_all_nodes(); +} + static int __init hugetlb_init(void) { int i; @@ -4071,7 +4087,6 @@ static int __init hugetlb_init(void) report_hugepages(); hugetlb_sysfs_init(); - hugetlb_register_all_nodes(); hugetlb_cgroup_file_init(); #ifdef CONFIG_SMP @@ -4116,7 +4131,7 @@ void __init hugetlb_add_hstate(unsigned int order) h->next_nid_to_alloc = first_memory_node; h->next_nid_to_free = first_memory_node; snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB", - huge_page_size(h)/1024); + huge_page_size(h)/SZ_1K); parsed_hstate = h; } @@ -4131,11 +4146,11 @@ static void __init hugepages_clear_pages_in_node(void) if (!hugetlb_max_hstate) { default_hstate_max_huge_pages = 0; memset(default_hugepages_in_node, 0, - MAX_NUMNODES * sizeof(unsigned int)); + sizeof(default_hugepages_in_node)); } else { parsed_hstate->max_huge_pages = 0; memset(parsed_hstate->max_huge_pages_node, 0, - MAX_NUMNODES * sizeof(unsigned int)); + sizeof(parsed_hstate->max_huge_pages_node)); } } @@ -4330,18 +4345,34 @@ static int __init default_hugepagesz_setup(char *s) } __setup("default_hugepagesz=", default_hugepagesz_setup); +static nodemask_t *policy_mbind_nodemask(gfp_t gfp) +{ +#ifdef CONFIG_NUMA + struct mempolicy *mpol = get_task_policy(current); + + /* + * Only enforce MPOL_BIND policy which overlaps with cpuset policy + * (from policy_nodemask) specifically for hugetlb case + */ + if (mpol->mode == MPOL_BIND && + (apply_policy_zone(mpol, gfp_zone(gfp)) && + cpuset_nodemask_valid_mems_allowed(&mpol->nodes))) + return &mpol->nodes; +#endif + return NULL; +} + static unsigned int allowed_mems_nr(struct hstate *h) { int node; unsigned int nr = 0; - nodemask_t *mpol_allowed; + nodemask_t *mbind_nodemask; unsigned int *array = h->free_huge_pages_node; gfp_t gfp_mask = htlb_alloc_mask(h); - mpol_allowed = policy_nodemask_current(gfp_mask); - + mbind_nodemask = policy_mbind_nodemask(gfp_mask); for_each_node_mask(node, cpuset_current_mems_allowed) { - if (!mpol_allowed || node_isset(node, *mpol_allowed)) + if (!mbind_nodemask || node_isset(node, *mbind_nodemask)) nr += array[node]; } @@ -4581,16 +4612,28 @@ static void hugetlb_vm_op_open(struct vm_area_struct *vma) resv_map_dup_hugetlb_cgroup_uncharge_info(resv); kref_get(&resv->refs); } + + /* + * vma_lock structure for sharable mappings is vma specific. + * Clear old pointer (if copied via vm_area_dup) and create new. + */ + if (vma->vm_flags & VM_MAYSHARE) { + vma->vm_private_data = NULL; + hugetlb_vma_lock_alloc(vma); + } } static void hugetlb_vm_op_close(struct vm_area_struct *vma) { struct hstate *h = hstate_vma(vma); - struct resv_map *resv = vma_resv_map(vma); + struct resv_map *resv; struct hugepage_subpool *spool = subpool_vma(vma); unsigned long reserve, start, end; long gbl_reserve; + hugetlb_vma_lock_free(vma); + + resv = vma_resv_map(vma); if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER)) return; @@ -4721,14 +4764,13 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - pte_t *src_pte, *dst_pte, entry, dst_entry; + pte_t *src_pte, *dst_pte, entry; struct page *ptepage; unsigned long addr; bool cow = is_cow_mapping(src_vma->vm_flags); struct hstate *h = hstate_vma(src_vma); unsigned long sz = huge_page_size(h); unsigned long npages = pages_per_huge_page(h); - struct address_space *mapping = src_vma->vm_file->f_mapping; struct mmu_notifier_range range; unsigned long last_addr_mask; int ret = 0; @@ -4742,12 +4784,12 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, raw_write_seqcount_begin(&src->write_protect_seq); } else { /* - * For shared mappings i_mmap_rwsem must be held to call - * huge_pte_alloc, otherwise the returned ptep could go - * away if part of a shared pmd and another thread calls - * huge_pmd_unshare. + * For shared mappings the vma lock must be held before + * calling huge_pte_offset in the src vma. Otherwise, the + * returned ptep could go away if part of a shared pmd and + * another thread calls huge_pmd_unshare. */ - i_mmap_lock_read(mapping); + hugetlb_vma_lock_read(src_vma); } last_addr_mask = hugetlb_mask_last_page(h); @@ -4766,15 +4808,13 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, /* * If the pagetables are shared don't copy or take references. - * dst_pte == src_pte is the common case of src/dest sharing. * + * dst_pte == src_pte is the common case of src/dest sharing. * However, src could have 'unshared' and dst shares with - * another vma. If dst_pte !none, this implies sharing. - * Check here before taking page table lock, and once again - * after taking the lock below. + * another vma. So page_count of ptep page is checked instead + * to reliably determine whether pte is shared. */ - dst_entry = huge_ptep_get(dst_pte); - if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) { + if (page_count(virt_to_page(dst_pte)) > 1) { addr |= last_addr_mask; continue; } @@ -4783,13 +4823,10 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, src_ptl = huge_pte_lockptr(h, src, src_pte); spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); entry = huge_ptep_get(src_pte); - dst_entry = huge_ptep_get(dst_pte); again: - if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) { + if (huge_pte_none(entry)) { /* - * Skip if src entry none. Also, skip in the - * unlikely case dst entry !none as this implies - * sharing with another vma. + * Skip if src entry none. */ ; } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) { @@ -4868,7 +4905,7 @@ again: restore_reserve_on_error(h, dst_vma, addr, new); put_page(new); - /* dst_entry won't change as in child */ + /* huge_ptep of dst_pte won't change as in child */ goto again; } hugetlb_install_page(dst_vma, dst_pte, addr, new); @@ -4900,7 +4937,7 @@ again: raw_write_seqcount_end(&src->write_protect_seq); mmu_notifier_invalidate_range_end(&range); } else { - i_mmap_unlock_read(mapping); + hugetlb_vma_unlock_read(src_vma); } return ret; @@ -4959,6 +4996,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, mmu_notifier_invalidate_range_start(&range); last_addr_mask = hugetlb_mask_last_page(h); /* Prevent race with file truncation */ + hugetlb_vma_lock_write(vma); i_mmap_lock_write(mapping); for (; old_addr < old_end; old_addr += sz, new_addr += sz) { src_pte = huge_pte_offset(mm, old_addr, sz); @@ -4990,6 +5028,7 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma, flush_tlb_range(vma, old_end - len, old_end); mmu_notifier_invalidate_range_end(&range); i_mmap_unlock_write(mapping); + hugetlb_vma_unlock_write(vma); return len + old_addr - old_end; } @@ -5057,6 +5096,7 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct * unmapped and its refcount is dropped, so just clear pte here. */ if (unlikely(!pte_present(pte))) { +#ifdef CONFIG_PTE_MARKER_UFFD_WP /* * If the pte was wr-protected by uffd-wp in any of the * swap forms, meanwhile the caller does not want to @@ -5068,6 +5108,7 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP)); else +#endif huge_pte_clear(mm, address, ptep, sz); spin_unlock(ptl); continue; @@ -5096,11 +5137,13 @@ static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct tlb_remove_huge_tlb_entry(h, tlb, ptep, address); if (huge_pte_dirty(pte)) set_page_dirty(page); +#ifdef CONFIG_PTE_MARKER_UFFD_WP /* Leave a uffd-wp pte marker if needed */ if (huge_pte_uffd_wp(pte) && !(zap_flags & ZAP_FLAG_DROP_MARKER)) set_huge_pte_at(mm, address, ptep, make_pte_marker(PTE_MARKER_UFFD_WP)); +#endif hugetlb_count_sub(pages_per_huge_page(h), mm); page_remove_rmap(page, vma, true); @@ -5137,19 +5180,22 @@ void __unmap_hugepage_range_final(struct mmu_gather *tlb, unsigned long end, struct page *ref_page, zap_flags_t zap_flags) { + hugetlb_vma_lock_write(vma); + i_mmap_lock_write(vma->vm_file->f_mapping); + __unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags); /* - * Clear this flag so that x86's huge_pmd_share page_table_shareable - * test will fail on a vma being torn down, and not grab a page table - * on its way out. We're lucky that the flag has such an appropriate - * name, and can in fact be safely cleared here. We could clear it - * before the __unmap_hugepage_range above, but all that's necessary - * is to clear it before releasing the i_mmap_rwsem. This works - * because in the context this is called, the VMA is about to be - * destroyed and the i_mmap_rwsem is held. + * Unlock and free the vma lock before releasing i_mmap_rwsem. When + * the vma_lock is freed, this makes the vma ineligible for pmd + * sharing. And, i_mmap_rwsem is required to set up pmd sharing. + * This is important as page tables for this unmapped range will + * be asynchrously deleted. If the page tables are shared, there + * will be issues when accessed by someone else. */ - vma->vm_flags &= ~VM_MAYSHARE; + __hugetlb_vma_unlock_write_free(vma); + + i_mmap_unlock_write(vma->vm_file->f_mapping); } void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, @@ -5241,6 +5287,21 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma, VM_BUG_ON(unshare && (flags & FOLL_WRITE)); VM_BUG_ON(!unshare && !(flags & FOLL_WRITE)); + /* + * hugetlb does not support FOLL_FORCE-style write faults that keep the + * PTE mapped R/O such as maybe_mkwrite() would do. + */ + if (WARN_ON_ONCE(!unshare && !(vma->vm_flags & VM_WRITE))) + return VM_FAULT_SIGSEGV; + + /* Let's take out MAP_SHARED mappings first. */ + if (vma->vm_flags & VM_MAYSHARE) { + if (unlikely(unshare)) + return 0; + set_huge_ptep_writable(vma, haddr, ptep); + return 0; + } + pte = huge_ptep_get(ptep); old_page = pte_page(pte); @@ -5299,11 +5360,10 @@ retry_avoidcopy: u32 hash; put_page(old_page); - BUG_ON(huge_pte_none(pte)); /* - * Drop hugetlb_fault_mutex and i_mmap_rwsem before - * unmapping. unmapping needs to hold i_mmap_rwsem - * in write mode. Dropping i_mmap_rwsem in read mode + * Drop hugetlb_fault_mutex and vma_lock before + * unmapping. unmapping needs to hold vma_lock + * in write mode. Dropping vma_lock in read mode * here is OK as COW mappings do not interact with * PMD sharing. * @@ -5311,13 +5371,13 @@ retry_avoidcopy: */ idx = vma_hugecache_offset(h, vma, haddr); hash = hugetlb_fault_mutex_hash(mapping, idx); + hugetlb_vma_unlock_read(vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); unmap_ref_private(mm, vma, old_page, haddr); - i_mmap_lock_read(mapping); mutex_lock(&hugetlb_fault_mutex_table[hash]); + hugetlb_vma_lock_read(vma); spin_lock(ptl); ptep = huge_pte_offset(mm, haddr, huge_page_size(h)); if (likely(ptep && @@ -5391,19 +5451,6 @@ out_release_old: return ret; } -/* Return the pagecache page at a given address within a VMA */ -static struct page *hugetlbfs_pagecache_page(struct hstate *h, - struct vm_area_struct *vma, unsigned long address) -{ - struct address_space *mapping; - pgoff_t idx; - - mapping = vma->vm_file->f_mapping; - idx = vma_hugecache_offset(h, vma, address); - - return find_lock_page(mapping, idx); -} - /* * Return whether there is a pagecache page to back given address within VMA. * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page. @@ -5424,7 +5471,7 @@ static bool hugetlbfs_pagecache_present(struct hstate *h, return page != NULL; } -int huge_add_to_page_cache(struct page *page, struct address_space *mapping, +int hugetlb_add_to_page_cache(struct page *page, struct address_space *mapping, pgoff_t idx) { struct folio *folio = page_folio(page); @@ -5461,7 +5508,6 @@ static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma, unsigned long addr, unsigned long reason) { - vm_fault_t ret; u32 hash; struct vm_fault vmf = { .vma = vma, @@ -5479,18 +5525,31 @@ static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma, }; /* - * hugetlb_fault_mutex and i_mmap_rwsem must be - * dropped before handling userfault. Reacquire - * after handling fault to make calling code simpler. + * vma_lock and hugetlb_fault_mutex must be dropped before handling + * userfault. Also mmap_lock could be dropped due to handling + * userfault, any vma operation should be careful from here. */ + hugetlb_vma_unlock_read(vma); hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); - ret = handle_userfault(&vmf, reason); - i_mmap_lock_read(mapping); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + return handle_userfault(&vmf, reason); +} - return ret; +/* + * Recheck pte with pgtable lock. Returns true if pte didn't change, or + * false if pte changed or is changing. + */ +static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm, + pte_t *ptep, pte_t old_pte) +{ + spinlock_t *ptl; + bool same; + + ptl = huge_pte_lock(h, mm, ptep); + same = pte_same(huge_ptep_get(ptep), old_pte); + spin_unlock(ptl); + + return same; } static vm_fault_t hugetlb_no_page(struct mm_struct *mm, @@ -5508,6 +5567,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, spinlock_t *ptl; unsigned long haddr = address & huge_page_mask(h); bool new_page, new_pagecache_page = false; + u32 hash = hugetlb_fault_mutex_hash(mapping, idx); /* * Currently, we are forced to kill the process in the event the @@ -5518,28 +5578,46 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm, if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) { pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n", current->pid); - return ret; + goto out; } /* - * We can not race with truncation due to holding i_mmap_rwsem. - * i_size is modified when holding i_mmap_rwsem, so check here - * once for faults beyond end of file. + * Use page lock to guard against racing truncation + * before we get page_table_lock. */ - size = i_size_read(mapping->host) >> huge_page_shift(h); - if (idx >= size) - goto out; - -retry: new_page = false; page = find_lock_page(mapping, idx); if (!page) { + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto out; /* Check for page in userfault range */ if (userfaultfd_missing(vma)) { - ret = hugetlb_handle_userfault(vma, mapping, idx, - flags, haddr, address, - VM_UFFD_MISSING); - goto out; + /* + * Since hugetlb_no_page() was examining pte + * without pgtable lock, we need to re-test under + * lock because the pte may not be stable and could + * have changed from under us. Try to detect + * either changed or during-changing ptes and retry + * properly when needed. + * + * Note that userfaultfd is actually fine with + * false positives (e.g. caused by pte changed), + * but not wrong logical events (e.g. caused by + * reading a pte during changing). The latter can + * confuse the userspace, so the strictness is very + * much preferred. E.g., MISSING event should + * never happen on the page after UFFDIO_COPY has + * correctly installed the page and returned. + */ + if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { + ret = 0; + goto out; + } + + return hugetlb_handle_userfault(vma, mapping, idx, flags, + haddr, address, + VM_UFFD_MISSING); } page = alloc_huge_page(vma, haddr, 0); @@ -5556,11 +5634,10 @@ retry: * here. Before returning error, get ptl and make * sure there really is no pte entry. */ - ptl = huge_pte_lock(h, mm, ptep); - ret = 0; - if (huge_pte_none(huge_ptep_get(ptep))) + if (hugetlb_pte_stable(h, mm, ptep, old_pte)) ret = vmf_error(PTR_ERR(page)); - spin_unlock(ptl); + else + ret = 0; goto out; } clear_huge_page(page, address, pages_per_huge_page(h)); @@ -5568,11 +5645,17 @@ retry: new_page = true; if (vma->vm_flags & VM_MAYSHARE) { - int err = huge_add_to_page_cache(page, mapping, idx); + int err = hugetlb_add_to_page_cache(page, mapping, idx); if (err) { + /* + * err can't be -EEXIST which implies someone + * else consumed the reservation since hugetlb + * fault mutex is held when add a hugetlb page + * to the page cache. So it's safe to call + * restore_reserve_on_error() here. + */ + restore_reserve_on_error(h, vma, haddr, page); put_page(page); - if (err == -EEXIST) - goto retry; goto out; } new_pagecache_page = true; @@ -5600,10 +5683,14 @@ retry: if (userfaultfd_minor(vma)) { unlock_page(page); put_page(page); - ret = hugetlb_handle_userfault(vma, mapping, idx, - flags, haddr, address, - VM_UFFD_MINOR); - goto out; + /* See comment in userfaultfd_missing() block above */ + if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) { + ret = 0; + goto out; + } + return hugetlb_handle_userfault(vma, mapping, idx, flags, + haddr, address, + VM_UFFD_MINOR); } } @@ -5661,15 +5748,17 @@ retry: unlock_page(page); out: + hugetlb_vma_unlock_read(vma); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); return ret; backout: spin_unlock(ptl); backout_unlocked: - unlock_page(page); - /* restore reserve for newly allocated pages not in page cache */ if (new_page && !new_pagecache_page) restore_reserve_on_error(h, vma, haddr, page); + + unlock_page(page); put_page(page); goto out; } @@ -5730,40 +5819,41 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, } /* - * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold - * until finished with ptep. This serves two purposes: - * 1) It prevents huge_pmd_unshare from being called elsewhere - * and making the ptep no longer valid. - * 2) It synchronizes us with i_size modifications during truncation. + * Serialize hugepage allocation and instantiation, so that we don't + * get spurious allocation failures if two CPUs race to instantiate + * the same page in the page cache. + */ + mapping = vma->vm_file->f_mapping; + idx = vma_hugecache_offset(h, vma, haddr); + hash = hugetlb_fault_mutex_hash(mapping, idx); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * Acquire vma lock before calling huge_pte_alloc and hold + * until finished with ptep. This prevents huge_pmd_unshare from + * being called elsewhere and making the ptep no longer valid. * * ptep could have already be assigned via huge_pte_offset. That * is OK, as huge_pte_alloc will return the same value unless * something has changed. */ - mapping = vma->vm_file->f_mapping; - i_mmap_lock_read(mapping); + hugetlb_vma_lock_read(vma); ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h)); if (!ptep) { - i_mmap_unlock_read(mapping); + hugetlb_vma_unlock_read(vma); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); return VM_FAULT_OOM; } - /* - * Serialize hugepage allocation and instantiation, so that we don't - * get spurious allocation failures if two CPUs race to instantiate - * the same page in the page cache. - */ - idx = vma_hugecache_offset(h, vma, haddr); - hash = hugetlb_fault_mutex_hash(mapping, idx); - mutex_lock(&hugetlb_fault_mutex_table[hash]); - entry = huge_ptep_get(ptep); /* PTE markers should be handled the same way as none pte */ - if (huge_pte_none_mostly(entry)) { - ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, + if (huge_pte_none_mostly(entry)) + /* + * hugetlb_no_page will drop vma lock and hugetlb fault + * mutex internally, which make us return immediately. + */ + return hugetlb_no_page(mm, vma, mapping, idx, address, ptep, entry, flags); - goto out_mutex; - } ret = 0; @@ -5781,12 +5871,11 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, * If we are going to COW/unshare the mapping later, we examine the * pending reservations for this page now. This will ensure that any * allocations necessary to record that reservation occur outside the - * spinlock. For private mappings, we also lookup the pagecache - * page now as it is used to determine if a reservation has been - * consumed. + * spinlock. Also lookup the pagecache page now as it is used to + * determine if a reservation has been consumed. */ if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && - !huge_pte_write(entry)) { + !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) { if (vma_needs_reservation(h, vma, haddr) < 0) { ret = VM_FAULT_OOM; goto out_mutex; @@ -5794,9 +5883,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, /* Just decrements count, does not deallocate */ vma_end_reservation(h, vma, haddr); - if (!(vma->vm_flags & VM_MAYSHARE)) - pagecache_page = hugetlbfs_pagecache_page(h, - vma, haddr); + pagecache_page = find_lock_page(mapping, idx); } ptl = huge_pte_lock(h, mm, ptep); @@ -5820,8 +5907,8 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, unlock_page(pagecache_page); put_page(pagecache_page); } + hugetlb_vma_unlock_read(vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); return handle_userfault(&vmf, VM_UFFD_WP); } @@ -5864,8 +5951,8 @@ out_ptl: put_page(pagecache_page); } out_mutex: + hugetlb_vma_unlock_read(vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); /* * Generally it's safe to hold refcount during waiting page lock. But * here we just wait to defer the next page fault to avoid busy loop and @@ -5993,43 +6080,28 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, /* * Serialization between remove_inode_hugepages() and - * huge_add_to_page_cache() below happens through the + * hugetlb_add_to_page_cache() below happens through the * hugetlb_fault_mutex_table that here must be hold by * the caller. */ - ret = huge_add_to_page_cache(page, mapping, idx); + ret = hugetlb_add_to_page_cache(page, mapping, idx); if (ret) goto out_release_nounlock; page_in_pagecache = true; } - ptl = huge_pte_lockptr(h, dst_mm, dst_pte); - spin_lock(ptl); + ptl = huge_pte_lock(h, dst_mm, dst_pte); - /* - * Recheck the i_size after holding PT lock to make sure not - * to leave any page mapped (as page_mapped()) beyond the end - * of the i_size (remove_inode_hugepages() is strict about - * enforcing that). If we bail out here, we'll also leave a - * page in the radix tree in the vm_shared case beyond the end - * of the i_size, but remove_inode_hugepages() will take care - * of it as soon as we drop the hugetlb_fault_mutex_table. - */ - size = i_size_read(mapping->host) >> huge_page_shift(h); - ret = -EFAULT; - if (idx >= size) - goto out_release_unlock; - - ret = -EEXIST; /* * We allow to overwrite a pte marker: consider when both MISSING|WP * registered, we firstly wr-protect a none pte which has no page cache * page backing it, then access the page. */ + ret = -EEXIST; if (!huge_pte_none_mostly(huge_ptep_get(dst_pte))) goto out_release_unlock; - if (vm_shared) { + if (page_in_pagecache) { page_dup_file_rmap(page, true); } else { ClearHPageRestoreReserve(page); @@ -6093,7 +6165,7 @@ static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma, for (nr = 0; nr < refs; nr++) { if (likely(pages)) - pages[nr] = mem_map_offset(page, nr); + pages[nr] = nth_page(page, nr); if (vmas) vmas[nr] = vma; } @@ -6257,7 +6329,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, (vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT); if (pages || vmas) - record_subpages_vmas(mem_map_offset(page, pfn_offset), + record_subpages_vmas(nth_page(page, pfn_offset), vma, refs, likely(pages) ? pages + i : NULL, vmas ? vmas + i : NULL); @@ -6328,8 +6400,9 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, flush_cache_range(vma, range.start, range.end); mmu_notifier_invalidate_range_start(&range); - last_addr_mask = hugetlb_mask_last_page(h); + hugetlb_vma_lock_write(vma); i_mmap_lock_write(vma->vm_file->f_mapping); + last_addr_mask = hugetlb_mask_last_page(h); for (; address < end; address += psize) { spinlock_t *ptl; ptep = huge_pte_offset(mm, address, psize); @@ -6428,6 +6501,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma, * See Documentation/mm/mmu_notifier.rst */ i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); mmu_notifier_invalidate_range_end(&range); return pages << h->order; @@ -6452,6 +6526,11 @@ bool hugetlb_reserve_pages(struct inode *inode, return false; } + /* + * vma specific semaphore used for pmd sharing synchronization + */ + hugetlb_vma_lock_alloc(vma); + /* * Only apply hugepage reservation if asked. At fault time, an * attempt will be made for VM_NORESERVE to allocate a page @@ -6475,12 +6554,11 @@ bool hugetlb_reserve_pages(struct inode *inode, resv_map = inode_resv_map(inode); chg = region_chg(resv_map, from, to, ®ions_needed); - } else { /* Private mapping. */ resv_map = resv_map_alloc(); if (!resv_map) - return false; + goto out_err; chg = to - from; @@ -6575,6 +6653,7 @@ out_uncharge_cgroup: hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h), chg * pages_per_huge_page(h), h_cg); out_err: + hugetlb_vma_lock_free(vma); if (!vma || vma->vm_flags & VM_MAYSHARE) /* Only call region_abort if the region_chg succeeded but the * region_add failed or didn't run. @@ -6644,35 +6723,37 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma, /* * match the virtual addresses, permission and the alignment of the * page table page. + * + * Also, vma_lock (vm_private_data) is required for sharing. */ if (pmd_index(addr) != pmd_index(saddr) || vm_flags != svm_flags || - !range_in_vma(svma, sbase, s_end)) + !range_in_vma(svma, sbase, s_end) || + !svma->vm_private_data) return 0; return saddr; } -static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr) -{ - unsigned long base = addr & PUD_MASK; - unsigned long end = base + PUD_SIZE; - - /* - * check on proper vm_flags and page table alignment - */ - if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end)) - return true; - return false; -} - bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr) { + unsigned long start = addr & PUD_MASK; + unsigned long end = start + PUD_SIZE; + #ifdef CONFIG_USERFAULTFD if (uffd_disable_huge_pmd_share(vma)) return false; #endif - return vma_shareable(vma, addr); + /* + * check on proper vm_flags and page table alignment + */ + if (!(vma->vm_flags & VM_MAYSHARE)) + return false; + if (!vma->vm_private_data) /* vma lock required for sharing */ + return false; + if (!range_in_vma(vma, start, end)) + return false; + return true; } /* @@ -6702,16 +6783,157 @@ void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma, *end = ALIGN(*end, PUD_SIZE); } +static bool __vma_shareable_flags_pmd(struct vm_area_struct *vma) +{ + return vma->vm_flags & (VM_MAYSHARE | VM_SHARED) && + vma->vm_private_data; +} + +void hugetlb_vma_lock_read(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + down_read(&vma_lock->rw_sema); + } +} + +void hugetlb_vma_unlock_read(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + up_read(&vma_lock->rw_sema); + } +} + +void hugetlb_vma_lock_write(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + down_write(&vma_lock->rw_sema); + } +} + +void hugetlb_vma_unlock_write(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + up_write(&vma_lock->rw_sema); + } +} + +int hugetlb_vma_trylock_write(struct vm_area_struct *vma) +{ + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + if (!__vma_shareable_flags_pmd(vma)) + return 1; + + return down_write_trylock(&vma_lock->rw_sema); +} + +void hugetlb_vma_assert_locked(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + lockdep_assert_held(&vma_lock->rw_sema); + } +} + +void hugetlb_vma_lock_release(struct kref *kref) +{ + struct hugetlb_vma_lock *vma_lock = container_of(kref, + struct hugetlb_vma_lock, refs); + + kfree(vma_lock); +} + +static void __hugetlb_vma_unlock_write_put(struct hugetlb_vma_lock *vma_lock) +{ + struct vm_area_struct *vma = vma_lock->vma; + + /* + * vma_lock structure may or not be released as a result of put, + * it certainly will no longer be attached to vma so clear pointer. + * Semaphore synchronizes access to vma_lock->vma field. + */ + vma_lock->vma = NULL; + vma->vm_private_data = NULL; + up_write(&vma_lock->rw_sema); + kref_put(&vma_lock->refs, hugetlb_vma_lock_release); +} + +static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma) +{ + if (__vma_shareable_flags_pmd(vma)) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + __hugetlb_vma_unlock_write_put(vma_lock); + } +} + +static void hugetlb_vma_lock_free(struct vm_area_struct *vma) +{ + /* + * Only present in sharable vmas. + */ + if (!vma || !__vma_shareable_flags_pmd(vma)) + return; + + if (vma->vm_private_data) { + struct hugetlb_vma_lock *vma_lock = vma->vm_private_data; + + down_write(&vma_lock->rw_sema); + __hugetlb_vma_unlock_write_put(vma_lock); + } +} + +static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma) +{ + struct hugetlb_vma_lock *vma_lock; + + /* Only establish in (flags) sharable vmas */ + if (!vma || !(vma->vm_flags & VM_MAYSHARE)) + return; + + /* Should never get here with non-NULL vm_private_data */ + if (vma->vm_private_data) + return; + + vma_lock = kmalloc(sizeof(*vma_lock), GFP_KERNEL); + if (!vma_lock) { + /* + * If we can not allocate structure, then vma can not + * participate in pmd sharing. This is only a possible + * performance enhancement and memory saving issue. + * However, the lock is also used to synchronize page + * faults with truncation. If the lock is not present, + * unlikely races could leave pages in a file past i_size + * until the file is removed. Warn in the unlikely case of + * allocation failure. + */ + pr_warn_once("HugeTLB: unable to allocate vma specific lock\n"); + return; + } + + kref_init(&vma_lock->refs); + init_rwsem(&vma_lock->rw_sema); + vma_lock->vma = vma; + vma->vm_private_data = vma_lock; +} + /* * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc() * and returns the corresponding pte. While this is not necessary for the * !shared pmd case because we can allocate the pmd later as well, it makes the - * code much cleaner. - * - * This routine must be called with i_mmap_rwsem held in at least read mode if - * sharing is possible. For hugetlbfs, this prevents removal of any page - * table entries associated with the address space. This is important as we - * are setting up sharing based on existing page table entries (mappings). + * code much cleaner. pmd allocation is essential for the shared case because + * pud has to be populated inside the same i_mmap_rwsem section - otherwise + * racing tasks could either miss the sharing (see huge_pte_offset) or select a + * bad pmd for sharing. */ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud) @@ -6725,7 +6947,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *pte; spinlock_t *ptl; - i_mmap_assert_locked(mapping); + i_mmap_lock_read(mapping); vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) { if (svma == vma) continue; @@ -6755,6 +6977,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, spin_unlock(ptl); out: pte = (pte_t *)pmd_alloc(mm, pud, addr); + i_mmap_unlock_read(mapping); return pte; } @@ -6765,7 +6988,7 @@ out: * indicated by page_count > 1, unmap is achieved by clearing pud and * decrementing the ref count. If count == 1, the pte page is not shared. * - * Called with page table lock held and i_mmap_rwsem held in write mode. + * Called with page table lock held. * * returns: 1 successfully unmapped a shared pte page * 0 the underlying pte page is not shared, or it is the last user @@ -6778,6 +7001,7 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, pud_t *pud = pud_offset(p4d, addr); i_mmap_assert_write_locked(vma->vm_file->f_mapping); + hugetlb_vma_assert_locked(vma); BUG_ON(page_count(virt_to_page(ptep)) == 0); if (page_count(virt_to_page(ptep)) == 1) return 0; @@ -6789,6 +7013,48 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma, } #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */ + +void hugetlb_vma_lock_read(struct vm_area_struct *vma) +{ +} + +void hugetlb_vma_unlock_read(struct vm_area_struct *vma) +{ +} + +void hugetlb_vma_lock_write(struct vm_area_struct *vma) +{ +} + +void hugetlb_vma_unlock_write(struct vm_area_struct *vma) +{ +} + +int hugetlb_vma_trylock_write(struct vm_area_struct *vma) +{ + return 1; +} + +void hugetlb_vma_assert_locked(struct vm_area_struct *vma) +{ +} + +void hugetlb_vma_lock_release(struct kref *kref) +{ +} + +static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma) +{ +} + +static void hugetlb_vma_lock_free(struct vm_area_struct *vma) +{ +} + +static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma) +{ +} + pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pud_t *pud) { @@ -6932,12 +7198,13 @@ follow_huge_pd(struct vm_area_struct *vma, } struct page * __weak -follow_huge_pmd(struct mm_struct *mm, unsigned long address, - pmd_t *pmd, int flags) +follow_huge_pmd_pte(struct vm_area_struct *vma, unsigned long address, int flags) { + struct hstate *h = hstate_vma(vma); + struct mm_struct *mm = vma->vm_mm; struct page *page = NULL; spinlock_t *ptl; - pte_t pte; + pte_t *ptep, pte; /* * FOLL_PIN is not supported for follow_page(). Ordinary GUP goes via @@ -6947,17 +7214,15 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, return NULL; retry: - ptl = pmd_lockptr(mm, pmd); - spin_lock(ptl); - /* - * make sure that the address range covered by this pmd is not - * unmapped from other threads. - */ - if (!pmd_huge(*pmd)) - goto out; - pte = huge_ptep_get((pte_t *)pmd); + ptep = huge_pte_offset(mm, address, huge_page_size(h)); + if (!ptep) + return NULL; + + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(ptep); if (pte_present(pte)) { - page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT); + page = pte_page(pte) + + ((address & ~huge_page_mask(h)) >> PAGE_SHIFT); /* * try_grab_page() should always succeed here, because: a) we * hold the pmd (ptl) lock, and b) we've just checked that the @@ -6973,7 +7238,7 @@ retry: } else { if (is_hugetlb_entry_migration(pte)) { spin_unlock(ptl); - __migration_entry_wait_huge((pte_t *)pmd, ptl); + __migration_entry_wait_huge(ptep, ptl); goto retry; } /* @@ -7159,6 +7424,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, start, end); mmu_notifier_invalidate_range_start(&range); + hugetlb_vma_lock_write(vma); i_mmap_lock_write(vma->vm_file->f_mapping); for (address = start; address < end; address += PUD_SIZE) { ptep = huge_pte_offset(mm, address, sz); @@ -7170,6 +7436,7 @@ void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) } flush_hugetlb_tlb_range(vma, start, end); i_mmap_unlock_write(vma->vm_file->f_mapping); + hugetlb_vma_unlock_write(vma); /* * No need to call mmu_notifier_invalidate_range(), see * Documentation/mm/mmu_notifier.rst. @@ -7320,7 +7587,7 @@ void __init hugetlb_cma_reserve(int order) hugetlb_cma_size = 0; } -void __init hugetlb_cma_check(void) +static void __init hugetlb_cma_check(void) { if (!hugetlb_cma_size || cma_reserve_called) return; diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index c86691c431fd7bf891bde3e89dfee8cae31d7120..f61d132df52b34e304553ee899052c07a3929ac3 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -75,11 +75,11 @@ parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg) static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) { - int idx; + struct hstate *h; - for (idx = 0; idx < hugetlb_max_hstate; idx++) { + for_each_hstate(h) { if (page_counter_read( - hugetlb_cgroup_counter_from_cgroup(h_cg, idx))) + hugetlb_cgroup_counter_from_cgroup(h_cg, hstate_index(h)))) return true; } return false; @@ -154,9 +154,9 @@ hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) * function. */ for_each_node(node) { - /* Set node_to_alloc to -1 for offline nodes. */ + /* Set node_to_alloc to NUMA_NO_NODE for offline nodes. */ int node_to_alloc = - node_state(node, N_NORMAL_MEMORY) ? node : -1; + node_state(node, N_NORMAL_MEMORY) ? node : NUMA_NO_NODE; h_cgroup->nodeinfo[node] = kzalloc_node(sizeof(struct hugetlb_cgroup_per_node), GFP_KERNEL, node_to_alloc); @@ -225,17 +225,14 @@ static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); struct hstate *h; struct page *page; - int idx; do { - idx = 0; for_each_hstate(h) { spin_lock_irq(&hugetlb_lock); list_for_each_entry(page, &h->hugepage_activelist, lru) - hugetlb_cgroup_move_parent(idx, h_cg, page); + hugetlb_cgroup_move_parent(hstate_index(h), h_cg, page); spin_unlock_irq(&hugetlb_lock); - idx++; } cond_resched(); } while (hugetlb_cgroup_have_usage(h_cg)); @@ -442,7 +439,7 @@ void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv, if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages) return; - if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 && + if (rg->reservation_counter && resv->pages_per_hpage && !resv->reservation_counter) { page_counter_uncharge(rg->reservation_counter, nr_pages * resv->pages_per_hpage); @@ -675,12 +672,12 @@ static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, static char *mem_fmt(char *buf, int size, unsigned long hsize) { - if (hsize >= (1UL << 30)) - snprintf(buf, size, "%luGB", hsize >> 30); - else if (hsize >= (1UL << 20)) - snprintf(buf, size, "%luMB", hsize >> 20); + if (hsize >= SZ_1G) + snprintf(buf, size, "%luGB", hsize / SZ_1G); + else if (hsize >= SZ_1M) + snprintf(buf, size, "%luMB", hsize / SZ_1M); else - snprintf(buf, size, "%luKB", hsize >> 10); + snprintf(buf, size, "%luKB", hsize / SZ_1K); return buf; } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 20f414c0379f9ef16e9d44d7afda718e575ce59e..ba2a2596fb4e853dfdfa86476eb938ffe1430e26 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -265,11 +265,10 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr, static inline void reset_struct_pages(struct page *start) { - int i; struct page *from = start + NR_RESET_STRUCT_PAGE; - for (i = 0; i < NR_RESET_STRUCT_PAGE; i++) - memcpy(start + i, from, sizeof(*from)); + BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page)); + memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE); } static void vmemmap_restore_pte(pte_t *pte, unsigned long addr, @@ -287,6 +286,11 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr, copy_page(to, (void *)walk->reuse_addr); reset_struct_pages(to); + /* + * Makes sure that preceding stores to the page contents become visible + * before the set_pte_at() write. + */ + smp_wmb(); set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot)); } diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c index 65e242b5a432716e2218786a6b787009dec056ff..d0548e382b6ba2ba6104b7cc513e9a07815263e4 100644 --- a/mm/hwpoison-inject.c +++ b/mm/hwpoison-inject.c @@ -63,13 +63,13 @@ static int hwpoison_unpoison(void *data, u64 val) DEFINE_DEBUGFS_ATTRIBUTE(hwpoison_fops, NULL, hwpoison_inject, "%lli\n"); DEFINE_DEBUGFS_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n"); -static void pfn_inject_exit(void) +static void __exit pfn_inject_exit(void) { hwpoison_filter_enable = 0; debugfs_remove_recursive(hwpoison_dir); } -static int pfn_inject_init(void) +static int __init pfn_inject_init(void) { hwpoison_dir = debugfs_create_dir("hwpoison", NULL); diff --git a/mm/init-mm.c b/mm/init-mm.c index fbe7844d0912f54a79114db2ec7c205fa1cdcb5b..c9327abb771c54be8ca69273d81cef0b5755e8c7 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include -#include +#include #include #include #include @@ -28,7 +28,7 @@ * and size this cpu_bitmask to NR_CPUS. */ struct mm_struct init_mm = { - .mm_rb = RB_ROOT, + .mm_mt = MTREE_INIT_EXT(mm_mt, MM_MT_FLAGS, init_mm.mmap_lock), .pgd = swapper_pg_dir, .mm_users = ATOMIC_INIT(2), .mm_count = ATOMIC_INIT(1), diff --git a/mm/internal.h b/mm/internal.h index 785409805ed797be3792ea99b360d2b23f7832a4..6b7ef495b56d3d9ef383f5b63db2c1a28ed44c20 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -83,9 +83,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf); void folio_rotate_reclaimable(struct folio *folio); bool __folio_end_writeback(struct folio *folio); void deactivate_file_folio(struct folio *folio); +void folio_activate(struct folio *folio); -void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma, - unsigned long floor, unsigned long ceiling); +void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, + struct vm_area_struct *start_vma, unsigned long floor, + unsigned long ceiling); void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte); struct zap_details; @@ -187,7 +189,7 @@ extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason /* * in mm/rmap.c: */ -extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); +pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address); /* * in mm/page_alloc.c @@ -365,7 +367,6 @@ extern int user_min_free_kbytes; extern void free_unref_page(struct page *page, unsigned int order); extern void free_unref_page_list(struct list_head *list); -extern void zone_pcp_update(struct zone *zone, int cpu_online); extern void zone_pcp_reset(struct zone *zone); extern void zone_pcp_disable(struct zone *zone); extern void zone_pcp_enable(struct zone *zone); @@ -479,9 +480,6 @@ static inline bool is_data_mapping(vm_flags_t flags) } /* mm/util.c */ -void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *prev); -void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma); struct anon_vma *folio_anon_vma(struct folio *folio); #ifdef CONFIG_MMU @@ -639,34 +637,6 @@ static inline void vunmap_range_noflush(unsigned long start, unsigned long end) } #endif /* !CONFIG_MMU */ -/* - * Return the mem_map entry representing the 'offset' subpage within - * the maximally aligned gigantic page 'base'. Handle any discontiguity - * in the mem_map at MAX_ORDER_NR_PAGES boundaries. - */ -static inline struct page *mem_map_offset(struct page *base, int offset) -{ - if (unlikely(offset >= MAX_ORDER_NR_PAGES)) - return nth_page(base, offset); - return base + offset; -} - -/* - * Iterator over all subpages within the maximally aligned gigantic - * page 'base'. Handle any discontiguity in the mem_map. - */ -static inline struct page *mem_map_next(struct page *iter, - struct page *base, int offset) -{ - if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) { - unsigned long pfn = page_to_pfn(base) + offset; - if (!pfn_valid(pfn)) - return NULL; - return pfn_to_page(pfn); - } - return iter + 1; -} - /* Memory initialisation debug and verification */ enum mminit_level { MMINIT_WARNING, @@ -847,8 +817,14 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end, } #endif +int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift); + void vunmap_range_noflush(unsigned long start, unsigned long end); +void __vunmap_range_noflush(unsigned long start, unsigned long end); + int numa_migrate_prep(struct page *page, struct vm_area_struct *vma, unsigned long addr, int page_nid, int *flags); @@ -860,8 +836,6 @@ int migrate_device_coherent_page(struct page *page); */ struct folio *try_grab_folio(struct page *page, int refs, unsigned int flags); -DECLARE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); - extern bool mirrored_kernelcore; static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma) diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile index 1f84df9c302e73500a1256548a61f28296aa7fbd..d4837bff3b60f2b254a0a2b67e904628df1524ba 100644 --- a/mm/kasan/Makefile +++ b/mm/kasan/Makefile @@ -35,7 +35,15 @@ CFLAGS_shadow.o := $(CC_FLAGS_KASAN_RUNTIME) CFLAGS_hw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) CFLAGS_sw_tags.o := $(CC_FLAGS_KASAN_RUNTIME) +CFLAGS_KASAN_TEST := $(CFLAGS_KASAN) -fno-builtin $(call cc-disable-warning, vla) + +CFLAGS_kasan_test.o := $(CFLAGS_KASAN_TEST) +CFLAGS_kasan_test_module.o := $(CFLAGS_KASAN_TEST) + obj-y := common.o report.o obj-$(CONFIG_KASAN_GENERIC) += init.o generic.o report_generic.o shadow.o quarantine.o obj-$(CONFIG_KASAN_HW_TAGS) += hw_tags.o report_hw_tags.o tags.o report_tags.o obj-$(CONFIG_KASAN_SW_TAGS) += init.o report_sw_tags.o shadow.o sw_tags.o tags.o report_tags.o + +obj-$(CONFIG_KASAN_KUNIT_TEST) += kasan_test.o +obj-$(CONFIG_KASAN_MODULE_TEST) += kasan_test_module.o diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 69f583855c8be04e80e60d36ad2d427e96a8a9f1..833bf2cfd2a3985bcaeff91db0fc76ff7f93a212 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -30,13 +30,20 @@ #include "kasan.h" #include "../slab.h" +struct slab *kasan_addr_to_slab(const void *addr) +{ + if (virt_addr_valid(addr)) + return virt_to_slab(addr); + return NULL; +} + depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc) { unsigned long entries[KASAN_STACK_DEPTH]; unsigned int nr_entries; nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); - return __stack_depot_save(entries, nr_entries, flags, can_alloc); + return __stack_depot_save(entries, nr_entries, 0, flags, can_alloc); } void kasan_set_track(struct kasan_track *track, gfp_t flags) @@ -88,17 +95,6 @@ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) } #endif /* CONFIG_KASAN_STACK */ -/* - * Only allow cache merging when stack collection is disabled and no metadata - * is present. - */ -slab_flags_t __kasan_never_merge(void) -{ - if (kasan_stack_collection_enabled()) - return SLAB_KASAN; - return 0; -} - void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init) { u8 tag; @@ -121,132 +117,11 @@ void __kasan_poison_pages(struct page *page, unsigned int order, bool init) KASAN_PAGE_FREE, init); } -/* - * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. - * For larger allocations larger redzones are used. - */ -static inline unsigned int optimal_redzone(unsigned int object_size) -{ - return - object_size <= 64 - 16 ? 16 : - object_size <= 128 - 32 ? 32 : - object_size <= 512 - 64 ? 64 : - object_size <= 4096 - 128 ? 128 : - object_size <= (1 << 14) - 256 ? 256 : - object_size <= (1 << 15) - 512 ? 512 : - object_size <= (1 << 16) - 1024 ? 1024 : 2048; -} - -void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size, - slab_flags_t *flags) -{ - unsigned int ok_size; - unsigned int optimal_size; - - /* - * SLAB_KASAN is used to mark caches as ones that are sanitized by - * KASAN. Currently this flag is used in two places: - * 1. In slab_ksize() when calculating the size of the accessible - * memory within the object. - * 2. In slab_common.c to prevent merging of sanitized caches. - */ - *flags |= SLAB_KASAN; - - if (!kasan_stack_collection_enabled()) - return; - - ok_size = *size; - - /* Add alloc meta into redzone. */ - cache->kasan_info.alloc_meta_offset = *size; - *size += sizeof(struct kasan_alloc_meta); - - /* - * If alloc meta doesn't fit, don't add it. - * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal - * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for - * larger sizes. - */ - if (*size > KMALLOC_MAX_SIZE) { - cache->kasan_info.alloc_meta_offset = 0; - *size = ok_size; - /* Continue, since free meta might still fit. */ - } - - /* Only the generic mode uses free meta or flexible redzones. */ - if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { - cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; - return; - } - - /* - * Add free meta into redzone when it's not possible to store - * it in the object. This is the case when: - * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can - * be touched after it was freed, or - * 2. Object has a constructor, which means it's expected to - * retain its content until the next allocation, or - * 3. Object is too small. - * Otherwise cache->kasan_info.free_meta_offset = 0 is implied. - */ - if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor || - cache->object_size < sizeof(struct kasan_free_meta)) { - ok_size = *size; - - cache->kasan_info.free_meta_offset = *size; - *size += sizeof(struct kasan_free_meta); - - /* If free meta doesn't fit, don't add it. */ - if (*size > KMALLOC_MAX_SIZE) { - cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; - *size = ok_size; - } - } - - /* Calculate size with optimal redzone. */ - optimal_size = cache->object_size + optimal_redzone(cache->object_size); - /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */ - if (optimal_size > KMALLOC_MAX_SIZE) - optimal_size = KMALLOC_MAX_SIZE; - /* Use optimal size if the size with added metas is not large enough. */ - if (*size < optimal_size) - *size = optimal_size; -} - void __kasan_cache_create_kmalloc(struct kmem_cache *cache) { cache->kasan_info.is_kmalloc = true; } -size_t __kasan_metadata_size(struct kmem_cache *cache) -{ - if (!kasan_stack_collection_enabled()) - return 0; - return (cache->kasan_info.alloc_meta_offset ? - sizeof(struct kasan_alloc_meta) : 0) + - (cache->kasan_info.free_meta_offset ? - sizeof(struct kasan_free_meta) : 0); -} - -struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, - const void *object) -{ - if (!cache->kasan_info.alloc_meta_offset) - return NULL; - return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset; -} - -#ifdef CONFIG_KASAN_GENERIC -struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, - const void *object) -{ - BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); - if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META) - return NULL; - return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset; -} -#endif - void __kasan_poison_slab(struct slab *slab) { struct page *page = slab_page(slab); @@ -312,13 +187,9 @@ static inline u8 assign_tag(struct kmem_cache *cache, void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache, const void *object) { - struct kasan_alloc_meta *alloc_meta; - - if (kasan_stack_collection_enabled()) { - alloc_meta = kasan_get_alloc_meta(cache, object); - if (alloc_meta) - __memset(alloc_meta, 0, sizeof(*alloc_meta)); - } + /* Initialize per-object metadata if it is present. */ + if (kasan_requires_meta()) + kasan_init_object_meta(cache, object); /* Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS */ object = set_tag(object, assign_tag(cache, object, true)); @@ -329,13 +200,11 @@ void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache, static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip, bool quarantine, bool init) { - u8 tag; void *tagged_object; if (!kasan_arch_is_ready()) return false; - tag = get_tag(object); tagged_object = object; object = kasan_reset_tag(object); @@ -364,7 +233,7 @@ static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object, return false; if (kasan_stack_collection_enabled()) - kasan_set_free_info(cache, object, tag); + kasan_save_free_info(cache, tagged_object); return kasan_quarantine_put(cache, object); } @@ -423,20 +292,6 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip) } } -static void set_alloc_info(struct kmem_cache *cache, void *object, - gfp_t flags, bool is_kmalloc) -{ - struct kasan_alloc_meta *alloc_meta; - - /* Don't save alloc info for kmalloc caches in kasan_slab_alloc(). */ - if (cache->kasan_info.is_kmalloc && !is_kmalloc) - return; - - alloc_meta = kasan_get_alloc_meta(cache, object); - if (alloc_meta) - kasan_set_track(&alloc_meta->alloc_track, flags); -} - void * __must_check __kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags, bool init) { @@ -466,8 +321,8 @@ void * __must_check __kasan_slab_alloc(struct kmem_cache *cache, kasan_unpoison(tagged_object, cache->object_size, init); /* Save alloc info (if possible) for non-kmalloc() allocations. */ - if (kasan_stack_collection_enabled()) - set_alloc_info(cache, (void *)object, flags, false); + if (kasan_stack_collection_enabled() && !cache->kasan_info.is_kmalloc) + kasan_save_alloc_info(cache, tagged_object, flags); return tagged_object; } @@ -512,8 +367,8 @@ static inline void *____kasan_kmalloc(struct kmem_cache *cache, * Save alloc info (if possible) for kmalloc() allocations. * This also rewrites the alloc info when called from kasan_krealloc(). */ - if (kasan_stack_collection_enabled()) - set_alloc_info(cache, (void *)object, flags, true); + if (kasan_stack_collection_enabled() && cache->kasan_info.is_kmalloc) + kasan_save_alloc_info(cache, (void *)object, flags); /* Keep the tag that was set by kasan_slab_alloc(). */ return (void *)object; diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 437fcc7e77cf277484d0b280b2f765468e3fcafc..d8b5590f9484bb6ce8ae5bee152f361bc41c4de8 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -328,6 +328,139 @@ DEFINE_ASAN_SET_SHADOW(f3); DEFINE_ASAN_SET_SHADOW(f5); DEFINE_ASAN_SET_SHADOW(f8); +/* Only allow cache merging when no per-object metadata is present. */ +slab_flags_t kasan_never_merge(void) +{ + if (!kasan_requires_meta()) + return 0; + return SLAB_KASAN; +} + +/* + * Adaptive redzone policy taken from the userspace AddressSanitizer runtime. + * For larger allocations larger redzones are used. + */ +static inline unsigned int optimal_redzone(unsigned int object_size) +{ + return + object_size <= 64 - 16 ? 16 : + object_size <= 128 - 32 ? 32 : + object_size <= 512 - 64 ? 64 : + object_size <= 4096 - 128 ? 128 : + object_size <= (1 << 14) - 256 ? 256 : + object_size <= (1 << 15) - 512 ? 512 : + object_size <= (1 << 16) - 1024 ? 1024 : 2048; +} + +void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, + slab_flags_t *flags) +{ + unsigned int ok_size; + unsigned int optimal_size; + + if (!kasan_requires_meta()) + return; + + /* + * SLAB_KASAN is used to mark caches that are sanitized by KASAN + * and that thus have per-object metadata. + * Currently this flag is used in two places: + * 1. In slab_ksize() to account for per-object metadata when + * calculating the size of the accessible memory within the object. + * 2. In slab_common.c via kasan_never_merge() to prevent merging of + * caches with per-object metadata. + */ + *flags |= SLAB_KASAN; + + ok_size = *size; + + /* Add alloc meta into redzone. */ + cache->kasan_info.alloc_meta_offset = *size; + *size += sizeof(struct kasan_alloc_meta); + + /* + * If alloc meta doesn't fit, don't add it. + * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal + * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for + * larger sizes. + */ + if (*size > KMALLOC_MAX_SIZE) { + cache->kasan_info.alloc_meta_offset = 0; + *size = ok_size; + /* Continue, since free meta might still fit. */ + } + + /* + * Add free meta into redzone when it's not possible to store + * it in the object. This is the case when: + * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can + * be touched after it was freed, or + * 2. Object has a constructor, which means it's expected to + * retain its content until the next allocation, or + * 3. Object is too small. + * Otherwise cache->kasan_info.free_meta_offset = 0 is implied. + */ + if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor || + cache->object_size < sizeof(struct kasan_free_meta)) { + ok_size = *size; + + cache->kasan_info.free_meta_offset = *size; + *size += sizeof(struct kasan_free_meta); + + /* If free meta doesn't fit, don't add it. */ + if (*size > KMALLOC_MAX_SIZE) { + cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; + *size = ok_size; + } + } + + /* Calculate size with optimal redzone. */ + optimal_size = cache->object_size + optimal_redzone(cache->object_size); + /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */ + if (optimal_size > KMALLOC_MAX_SIZE) + optimal_size = KMALLOC_MAX_SIZE; + /* Use optimal size if the size with added metas is not large enough. */ + if (*size < optimal_size) + *size = optimal_size; +} + +struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, + const void *object) +{ + if (!cache->kasan_info.alloc_meta_offset) + return NULL; + return (void *)object + cache->kasan_info.alloc_meta_offset; +} + +struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, + const void *object) +{ + BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); + if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META) + return NULL; + return (void *)object + cache->kasan_info.free_meta_offset; +} + +void kasan_init_object_meta(struct kmem_cache *cache, const void *object) +{ + struct kasan_alloc_meta *alloc_meta; + + alloc_meta = kasan_get_alloc_meta(cache, object); + if (alloc_meta) + __memset(alloc_meta, 0, sizeof(*alloc_meta)); +} + +size_t kasan_metadata_size(struct kmem_cache *cache) +{ + if (!kasan_requires_meta()) + return 0; + return (cache->kasan_info.alloc_meta_offset ? + sizeof(struct kasan_alloc_meta) : 0) + + ((cache->kasan_info.free_meta_offset && + cache->kasan_info.free_meta_offset != KASAN_NO_FREE_META) ? + sizeof(struct kasan_free_meta) : 0); +} + static void __kasan_record_aux_stack(void *addr, bool can_alloc) { struct slab *slab = kasan_addr_to_slab(addr); @@ -358,8 +491,16 @@ void kasan_record_aux_stack_noalloc(void *addr) return __kasan_record_aux_stack(addr, false); } -void kasan_set_free_info(struct kmem_cache *cache, - void *object, u8 tag) +void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) +{ + struct kasan_alloc_meta *alloc_meta; + + alloc_meta = kasan_get_alloc_meta(cache, object); + if (alloc_meta) + kasan_set_track(&alloc_meta->alloc_track, flags); +} + +void kasan_save_free_info(struct kmem_cache *cache, void *object) { struct kasan_free_meta *free_meta; @@ -371,12 +512,3 @@ void kasan_set_free_info(struct kmem_cache *cache, /* The object was freed and has free track set. */ *(u8 *)kasan_mem_to_shadow(object) = KASAN_SLAB_FREETRACK; } - -struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, - void *object, u8 tag) -{ - if (*(u8 *)kasan_mem_to_shadow(object) != KASAN_SLAB_FREETRACK) - return NULL; - /* Free meta must be present with KASAN_SLAB_FREETRACK. */ - return &kasan_get_free_meta(cache, object)->free_track; -} diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 9ad8eff71b28ddcc607441ad940ec0cb37889df8..b22c4f461cb0b991666bfb9beb1e228d82609922 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -38,16 +38,9 @@ enum kasan_arg_vmalloc { KASAN_ARG_VMALLOC_ON, }; -enum kasan_arg_stacktrace { - KASAN_ARG_STACKTRACE_DEFAULT, - KASAN_ARG_STACKTRACE_OFF, - KASAN_ARG_STACKTRACE_ON, -}; - static enum kasan_arg kasan_arg __ro_after_init; static enum kasan_arg_mode kasan_arg_mode __ro_after_init; static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; -static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; /* * Whether KASAN is enabled at all. @@ -66,9 +59,6 @@ EXPORT_SYMBOL_GPL(kasan_mode); /* Whether to enable vmalloc tagging. */ DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); -/* Whether to collect alloc/free stack traces. */ -DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); - /* kasan=off/on */ static int __init early_kasan_flag(char *arg) { @@ -122,23 +112,6 @@ static int __init early_kasan_flag_vmalloc(char *arg) } early_param("kasan.vmalloc", early_kasan_flag_vmalloc); -/* kasan.stacktrace=off/on */ -static int __init early_kasan_flag_stacktrace(char *arg) -{ - if (!arg) - return -EINVAL; - - if (!strcmp(arg, "off")) - kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; - else if (!strcmp(arg, "on")) - kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; - else - return -EINVAL; - - return 0; -} -early_param("kasan.stacktrace", early_kasan_flag_stacktrace); - static inline const char *kasan_mode_info(void) { if (kasan_mode == KASAN_MODE_ASYNC) @@ -213,17 +186,7 @@ void __init kasan_init_hw_tags(void) break; } - switch (kasan_arg_stacktrace) { - case KASAN_ARG_STACKTRACE_DEFAULT: - /* Default is specified by kasan_flag_stacktrace definition. */ - break; - case KASAN_ARG_STACKTRACE_OFF: - static_branch_disable(&kasan_flag_stacktrace); - break; - case KASAN_ARG_STACKTRACE_ON: - static_branch_enable(&kasan_flag_stacktrace); - break; - } + kasan_init_tags(); /* KASAN is now initialized, enable it. */ static_branch_enable(&kasan_flag_enabled); diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 01c03e45acd42afef5469ba3cf2e6d46a1233e06..abbcc1b0eec50d29c9d0cec3675e036c5e2d920d 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -2,18 +2,37 @@ #ifndef __MM_KASAN_KASAN_H #define __MM_KASAN_KASAN_H +#include #include #include #include #include -#ifdef CONFIG_KASAN_HW_TAGS +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) #include + +DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace); + +static inline bool kasan_stack_collection_enabled(void) +{ + return static_branch_unlikely(&kasan_flag_stacktrace); +} + +#else /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ + +static inline bool kasan_stack_collection_enabled(void) +{ + return true; +} + +#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ + +#ifdef CONFIG_KASAN_HW_TAGS + #include "../slab.h" DECLARE_STATIC_KEY_TRUE(kasan_flag_vmalloc); -DECLARE_STATIC_KEY_TRUE(kasan_flag_stacktrace); enum kasan_mode { KASAN_MODE_SYNC, @@ -28,11 +47,6 @@ static inline bool kasan_vmalloc_enabled(void) return static_branch_likely(&kasan_flag_vmalloc); } -static inline bool kasan_stack_collection_enabled(void) -{ - return static_branch_unlikely(&kasan_flag_stacktrace); -} - static inline bool kasan_async_fault_possible(void) { return kasan_mode == KASAN_MODE_ASYNC || kasan_mode == KASAN_MODE_ASYMM; @@ -43,12 +57,7 @@ static inline bool kasan_sync_fault_possible(void) return kasan_mode == KASAN_MODE_SYNC || kasan_mode == KASAN_MODE_ASYMM; } -#else - -static inline bool kasan_stack_collection_enabled(void) -{ - return true; -} +#else /* CONFIG_KASAN_HW_TAGS */ static inline bool kasan_async_fault_possible(void) { @@ -60,7 +69,31 @@ static inline bool kasan_sync_fault_possible(void) return true; } -#endif +#endif /* CONFIG_KASAN_HW_TAGS */ + +#ifdef CONFIG_KASAN_GENERIC + +/* Generic KASAN uses per-object metadata to store stack traces. */ +static inline bool kasan_requires_meta(void) +{ + /* + * Technically, Generic KASAN always collects stack traces right now. + * However, let's use kasan_stack_collection_enabled() in case the + * kasan.stacktrace command-line argument is changed to affect + * Generic KASAN. + */ + return kasan_stack_collection_enabled(); +} + +#else /* CONFIG_KASAN_GENERIC */ + +/* Tag-based KASAN modes do not use per-object metadata. */ +static inline bool kasan_requires_meta(void) +{ + return false; +} + +#endif /* CONFIG_KASAN_GENERIC */ #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) #define KASAN_GRANULE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) @@ -122,6 +155,13 @@ static inline bool kasan_sync_fault_possible(void) #define META_MEM_BYTES_PER_ROW (META_BYTES_PER_ROW * KASAN_GRANULE_SIZE) #define META_ROWS_AROUND_ADDR 2 +#define KASAN_STACK_DEPTH 64 + +struct kasan_track { + u32 pid; + depot_stack_handle_t stack; +}; + enum kasan_report_type { KASAN_REPORT_ACCESS, KASAN_REPORT_INVALID_FREE, @@ -129,12 +169,22 @@ enum kasan_report_type { }; struct kasan_report_info { + /* Filled in by kasan_report_*(). */ enum kasan_report_type type; void *access_addr; - void *first_bad_addr; size_t access_size; bool is_write; unsigned long ip; + + /* Filled in by the common reporting code. */ + void *first_bad_addr; + struct kmem_cache *cache; + void *object; + + /* Filled in by the mode-specific reporting code. */ + const char *bug_type; + struct kasan_track alloc_track; + struct kasan_track free_track; }; /* Do not change the struct layout: compiler ABI. */ @@ -160,33 +210,14 @@ struct kasan_global { #endif }; -/* Structures for keeping alloc and free tracks. */ +/* Structures for keeping alloc and free meta. */ -#define KASAN_STACK_DEPTH 64 - -struct kasan_track { - u32 pid; - depot_stack_handle_t stack; -}; - -#if defined(CONFIG_KASAN_TAGS_IDENTIFY) && defined(CONFIG_KASAN_SW_TAGS) -#define KASAN_NR_FREE_STACKS 5 -#else -#define KASAN_NR_FREE_STACKS 1 -#endif +#ifdef CONFIG_KASAN_GENERIC struct kasan_alloc_meta { struct kasan_track alloc_track; - /* Generic mode stores free track in kasan_free_meta. */ -#ifdef CONFIG_KASAN_GENERIC + /* Free track is stored in kasan_free_meta. */ depot_stack_handle_t aux_stack[2]; -#else - struct kasan_track free_track[KASAN_NR_FREE_STACKS]; -#endif -#ifdef CONFIG_KASAN_TAGS_IDENTIFY - u8 free_pointer_tag[KASAN_NR_FREE_STACKS]; - u8 free_track_idx; -#endif }; struct qlist_node { @@ -205,12 +236,31 @@ struct qlist_node { * After that, slab allocator stores the freelist pointer in the object. */ struct kasan_free_meta { -#ifdef CONFIG_KASAN_GENERIC struct qlist_node quarantine_link; struct kasan_track free_track; -#endif }; +#endif /* CONFIG_KASAN_GENERIC */ + +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) + +struct kasan_stack_ring_entry { + void *ptr; + size_t size; + u32 pid; + depot_stack_handle_t stack; + bool is_free; +}; + +struct kasan_stack_ring { + rwlock_t lock; + size_t size; + atomic64_t pos; + struct kasan_stack_ring_entry *entries; +}; + +#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ + #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) /* Used in KUnit-compatible KASAN tests. */ struct kunit_kasan_status { @@ -219,13 +269,6 @@ struct kunit_kasan_status { }; #endif -struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, - const void *object); -#ifdef CONFIG_KASAN_GENERIC -struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, - const void *object); -#endif - #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) static inline const void *kasan_shadow_to_mem(const void *shadow_addr) @@ -260,34 +303,50 @@ static inline bool addr_has_metadata(const void *addr) #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ +void *kasan_find_first_bad_addr(void *addr, size_t size); +void kasan_complete_mode_report_info(struct kasan_report_info *info); +void kasan_metadata_fetch_row(char *buffer, void *row); + #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) void kasan_print_tags(u8 addr_tag, const void *addr); #else static inline void kasan_print_tags(u8 addr_tag, const void *addr) { } #endif -void *kasan_find_first_bad_addr(void *addr, size_t size); -const char *kasan_get_bug_type(struct kasan_report_info *info); -void kasan_metadata_fetch_row(char *buffer, void *row); - #if defined(CONFIG_KASAN_STACK) void kasan_print_address_stack_frame(const void *addr); #else static inline void kasan_print_address_stack_frame(const void *addr) { } #endif +#ifdef CONFIG_KASAN_GENERIC +void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object); +#else +static inline void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object) { } +#endif + bool kasan_report(unsigned long addr, size_t size, bool is_write, unsigned long ip); void kasan_report_invalid_free(void *object, unsigned long ip, enum kasan_report_type type); -struct page *kasan_addr_to_page(const void *addr); struct slab *kasan_addr_to_slab(const void *addr); +#ifdef CONFIG_KASAN_GENERIC +void kasan_init_cache_meta(struct kmem_cache *cache, unsigned int *size); +void kasan_init_object_meta(struct kmem_cache *cache, const void *object); +struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, + const void *object); +struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, + const void *object); +#else +static inline void kasan_init_cache_meta(struct kmem_cache *cache, unsigned int *size) { } +static inline void kasan_init_object_meta(struct kmem_cache *cache, const void *object) { } +#endif + depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc); void kasan_set_track(struct kasan_track *track, gfp_t flags); -void kasan_set_free_info(struct kmem_cache *cache, void *object, u8 tag); -struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, - void *object, u8 tag); +void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags); +void kasan_save_free_info(struct kmem_cache *cache, void *object); #if defined(CONFIG_KASAN_GENERIC) && \ (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) @@ -358,6 +417,10 @@ static inline void kasan_enable_tagging(void) { } #endif /* CONFIG_KASAN_HW_TAGS */ +#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS) +void __init kasan_init_tags(void); +#endif /* CONFIG_KASAN_SW_TAGS || CONFIG_KASAN_HW_TAGS */ + #if defined(CONFIG_KASAN_HW_TAGS) && IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) void kasan_force_async_fault(void); diff --git a/lib/test_kasan.c b/mm/kasan/kasan_test.c similarity index 97% rename from lib/test_kasan.c rename to mm/kasan/kasan_test.c index 58c1b01ccfe2029d1d5aaf61c99eb2b77c933fff..0d59098f087613d11ef37cdd02c50894eb67e9e1 100644 --- a/lib/test_kasan.c +++ b/mm/kasan/kasan_test.c @@ -25,7 +25,7 @@ #include -#include "../mm/kasan/kasan.h" +#include "kasan.h" #define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE) @@ -295,6 +295,9 @@ static void krealloc_more_oob_helper(struct kunit *test, ptr2 = krealloc(ptr1, size2, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + /* Suppress -Warray-bounds warnings. */ + OPTIMIZER_HIDE_VAR(ptr2); + /* All offsets up to size2 must be accessible. */ ptr2[size1 - 1] = 'x'; ptr2[size1] = 'x'; @@ -327,6 +330,9 @@ static void krealloc_less_oob_helper(struct kunit *test, ptr2 = krealloc(ptr1, size2, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + /* Suppress -Warray-bounds warnings. */ + OPTIMIZER_HIDE_VAR(ptr2); + /* Must be accessible for all modes. */ ptr2[size2 - 1] = 'x'; @@ -540,13 +546,14 @@ static void kmalloc_memmove_invalid_size(struct kunit *test) { char *ptr; size_t size = 64; - volatile size_t invalid_size = size; + size_t invalid_size = size; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); memset((char *)ptr, 0, 64); OPTIMIZER_HIDE_VAR(ptr); + OPTIMIZER_HIDE_VAR(invalid_size); KUNIT_EXPECT_KASAN_FAIL(test, memmove((char *)ptr, (char *)ptr + 4, invalid_size)); kfree(ptr); @@ -612,6 +619,29 @@ again: kfree(ptr2); } +/* + * Check that KASAN detects use-after-free when another object was allocated in + * the same slot. Relevant for the tag-based modes, which do not use quarantine. + */ +static void kmalloc_uaf3(struct kunit *test) +{ + char *ptr1, *ptr2; + size_t size = 100; + + /* This test is specifically crafted for tag-based modes. */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + ptr1 = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr1); + kfree(ptr1); + + ptr2 = kmalloc(size, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr2); + kfree(ptr2); + + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr1)[8]); +} + static void kfree_via_page(struct kunit *test) { char *ptr; @@ -1269,7 +1299,7 @@ static void match_all_not_assigned(struct kunit *test) KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); for (i = 0; i < 256; i++) { - size = (get_random_int() % 1024) + 1; + size = prandom_u32_max(1024) + 1; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); @@ -1278,7 +1308,7 @@ static void match_all_not_assigned(struct kunit *test) } for (i = 0; i < 256; i++) { - order = (get_random_int() % 4) + 1; + order = prandom_u32_max(4) + 1; pages = alloc_pages(GFP_KERNEL, order); ptr = page_address(pages); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); @@ -1291,7 +1321,7 @@ static void match_all_not_assigned(struct kunit *test) return; for (i = 0; i < 256; i++) { - size = (get_random_int() % 1024) + 1; + size = prandom_u32_max(1024) + 1; ptr = vmalloc(size); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); @@ -1382,6 +1412,7 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kmalloc_uaf), KUNIT_CASE(kmalloc_uaf_memset), KUNIT_CASE(kmalloc_uaf2), + KUNIT_CASE(kmalloc_uaf3), KUNIT_CASE(kfree_via_page), KUNIT_CASE(kfree_via_phys), KUNIT_CASE(kmem_cache_oob), diff --git a/lib/test_kasan_module.c b/mm/kasan/kasan_test_module.c similarity index 99% rename from lib/test_kasan_module.c rename to mm/kasan/kasan_test_module.c index b112cbc835e902357707c906f5095835d7ca6066..e4ca82dc2c16da7929e6476004cd764c045291a2 100644 --- a/lib/test_kasan_module.c +++ b/mm/kasan/kasan_test_module.c @@ -13,7 +13,7 @@ #include #include -#include "../mm/kasan/kasan.h" +#include "kasan.h" static noinline void __init copy_user_test(void) { diff --git a/mm/kasan/report.c b/mm/kasan/report.c index fe3f606b3a9867af9fc8db923fc638f923a0f4d3..df3602062bfd61040d5def7d6dcc345ac3396ce5 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -175,18 +175,14 @@ static void end_report(unsigned long *flags, void *addr) static void print_error_description(struct kasan_report_info *info) { - if (info->type == KASAN_REPORT_INVALID_FREE) { - pr_err("BUG: KASAN: invalid-free in %pS\n", (void *)info->ip); - return; - } + pr_err("BUG: KASAN: %s in %pS\n", info->bug_type, (void *)info->ip); - if (info->type == KASAN_REPORT_DOUBLE_FREE) { - pr_err("BUG: KASAN: double-free in %pS\n", (void *)info->ip); + if (info->type != KASAN_REPORT_ACCESS) { + pr_err("Free of addr %px by task %s/%d\n", + info->access_addr, current->comm, task_pid_nr(current)); return; } - pr_err("BUG: KASAN: %s in %pS\n", - kasan_get_bug_type(info), (void *)info->ip); if (info->access_size) pr_err("%s of size %zu at addr %px by task %s/%d\n", info->is_write ? "Write" : "Read", info->access_size, @@ -200,31 +196,21 @@ static void print_error_description(struct kasan_report_info *info) static void print_track(struct kasan_track *track, const char *prefix) { pr_err("%s by task %u:\n", prefix, track->pid); - if (track->stack) { + if (track->stack) stack_depot_print(track->stack); - } else { + else pr_err("(stack is not available)\n"); - } } -struct page *kasan_addr_to_page(const void *addr) +static inline struct page *addr_to_page(const void *addr) { - if ((addr >= (void *)PAGE_OFFSET) && - (addr < high_memory)) + if (virt_addr_valid(addr)) return virt_to_head_page(addr); return NULL; } -struct slab *kasan_addr_to_slab(const void *addr) -{ - if ((addr >= (void *)PAGE_OFFSET) && - (addr < high_memory)) - return virt_to_slab(addr); - return NULL; -} - -static void describe_object_addr(struct kmem_cache *cache, void *object, - const void *addr) +static void describe_object_addr(const void *addr, struct kmem_cache *cache, + void *object) { unsigned long access_addr = (unsigned long)addr; unsigned long object_addr = (unsigned long)object; @@ -252,46 +238,26 @@ static void describe_object_addr(struct kmem_cache *cache, void *object, (void *)(object_addr + cache->object_size)); } -static void describe_object_stacks(struct kmem_cache *cache, void *object, - const void *addr, u8 tag) +static void describe_object_stacks(struct kasan_report_info *info) { - struct kasan_alloc_meta *alloc_meta; - struct kasan_track *free_track; - - alloc_meta = kasan_get_alloc_meta(cache, object); - if (alloc_meta) { - print_track(&alloc_meta->alloc_track, "Allocated"); + if (info->alloc_track.stack) { + print_track(&info->alloc_track, "Allocated"); pr_err("\n"); } - free_track = kasan_get_free_track(cache, object, tag); - if (free_track) { - print_track(free_track, "Freed"); + if (info->free_track.stack) { + print_track(&info->free_track, "Freed"); pr_err("\n"); } -#ifdef CONFIG_KASAN_GENERIC - if (!alloc_meta) - return; - if (alloc_meta->aux_stack[0]) { - pr_err("Last potentially related work creation:\n"); - stack_depot_print(alloc_meta->aux_stack[0]); - pr_err("\n"); - } - if (alloc_meta->aux_stack[1]) { - pr_err("Second to last potentially related work creation:\n"); - stack_depot_print(alloc_meta->aux_stack[1]); - pr_err("\n"); - } -#endif + kasan_print_aux_stacks(info->cache, info->object); } -static void describe_object(struct kmem_cache *cache, void *object, - const void *addr, u8 tag) +static void describe_object(const void *addr, struct kasan_report_info *info) { if (kasan_stack_collection_enabled()) - describe_object_stacks(cache, object, addr, tag); - describe_object_addr(cache, object, addr); + describe_object_stacks(info); + describe_object_addr(addr, info->cache, info->object); } static inline bool kernel_or_module_addr(const void *addr) @@ -310,19 +276,16 @@ static inline bool init_task_stack_addr(const void *addr) sizeof(init_thread_union.stack)); } -static void print_address_description(void *addr, u8 tag) +static void print_address_description(void *addr, u8 tag, + struct kasan_report_info *info) { - struct page *page = kasan_addr_to_page(addr); + struct page *page = addr_to_page(addr); dump_stack_lvl(KERN_ERR); pr_err("\n"); - if (page && PageSlab(page)) { - struct slab *slab = page_slab(page); - struct kmem_cache *cache = slab->slab_cache; - void *object = nearest_obj(cache, slab, addr); - - describe_object(cache, object, addr, tag); + if (info->cache && info->object) { + describe_object(addr, info); pr_err("\n"); } @@ -420,23 +383,56 @@ static void print_memory_metadata(const void *addr) static void print_report(struct kasan_report_info *info) { - void *tagged_addr = info->access_addr; - void *untagged_addr = kasan_reset_tag(tagged_addr); - u8 tag = get_tag(tagged_addr); + void *addr = kasan_reset_tag(info->access_addr); + u8 tag = get_tag(info->access_addr); print_error_description(info); - if (addr_has_metadata(untagged_addr)) + if (addr_has_metadata(addr)) kasan_print_tags(tag, info->first_bad_addr); pr_err("\n"); - if (addr_has_metadata(untagged_addr)) { - print_address_description(untagged_addr, tag); + if (addr_has_metadata(addr)) { + print_address_description(addr, tag, info); print_memory_metadata(info->first_bad_addr); } else { dump_stack_lvl(KERN_ERR); } } +static void complete_report_info(struct kasan_report_info *info) +{ + void *addr = kasan_reset_tag(info->access_addr); + struct slab *slab; + + if (info->type == KASAN_REPORT_ACCESS) + info->first_bad_addr = kasan_find_first_bad_addr( + info->access_addr, info->access_size); + else + info->first_bad_addr = addr; + + slab = kasan_addr_to_slab(addr); + if (slab) { + info->cache = slab->slab_cache; + info->object = nearest_obj(info->cache, slab, addr); + } else + info->cache = info->object = NULL; + + switch (info->type) { + case KASAN_REPORT_INVALID_FREE: + info->bug_type = "invalid-free"; + break; + case KASAN_REPORT_DOUBLE_FREE: + info->bug_type = "double-free"; + break; + default: + /* bug_type filled in by kasan_complete_mode_report_info. */ + break; + } + + /* Fill in mode-specific report info fields. */ + kasan_complete_mode_report_info(info); +} + void kasan_report_invalid_free(void *ptr, unsigned long ip, enum kasan_report_type type) { unsigned long flags; @@ -452,13 +448,15 @@ void kasan_report_invalid_free(void *ptr, unsigned long ip, enum kasan_report_ty start_report(&flags, true); + memset(&info, 0, sizeof(info)); info.type = type; info.access_addr = ptr; - info.first_bad_addr = kasan_reset_tag(ptr); info.access_size = 0; info.is_write = false; info.ip = ip; + complete_report_info(&info); + print_report(&info); end_report(&flags, ptr); @@ -485,13 +483,15 @@ bool kasan_report(unsigned long addr, size_t size, bool is_write, start_report(&irq_flags, true); + memset(&info, 0, sizeof(info)); info.type = KASAN_REPORT_ACCESS; info.access_addr = ptr; - info.first_bad_addr = kasan_find_first_bad_addr(ptr, size); info.access_size = size; info.is_write = is_write; info.ip = ip; + complete_report_info(&info); + print_report(&info); end_report(&irq_flags, ptr); diff --git a/mm/kasan/report_generic.c b/mm/kasan/report_generic.c index 6689fb9a919b1d72a5afc56cf05f1f46b6244b30..043c94b04605402860a0553ef63b4bcd495f9917 100644 --- a/mm/kasan/report_generic.c +++ b/mm/kasan/report_generic.c @@ -109,7 +109,7 @@ static const char *get_wild_bug_type(struct kasan_report_info *info) return bug_type; } -const char *kasan_get_bug_type(struct kasan_report_info *info) +static const char *get_bug_type(struct kasan_report_info *info) { /* * If access_size is a negative number, then it has reason to be @@ -127,11 +127,55 @@ const char *kasan_get_bug_type(struct kasan_report_info *info) return get_wild_bug_type(info); } +void kasan_complete_mode_report_info(struct kasan_report_info *info) +{ + struct kasan_alloc_meta *alloc_meta; + struct kasan_free_meta *free_meta; + + if (!info->bug_type) + info->bug_type = get_bug_type(info); + + if (!info->cache || !info->object) + return; + + alloc_meta = kasan_get_alloc_meta(info->cache, info->object); + if (alloc_meta) + memcpy(&info->alloc_track, &alloc_meta->alloc_track, + sizeof(info->alloc_track)); + + if (*(u8 *)kasan_mem_to_shadow(info->object) == KASAN_SLAB_FREETRACK) { + /* Free meta must be present with KASAN_SLAB_FREETRACK. */ + free_meta = kasan_get_free_meta(info->cache, info->object); + memcpy(&info->free_track, &free_meta->free_track, + sizeof(info->free_track)); + } +} + void kasan_metadata_fetch_row(char *buffer, void *row) { memcpy(buffer, kasan_mem_to_shadow(row), META_BYTES_PER_ROW); } +void kasan_print_aux_stacks(struct kmem_cache *cache, const void *object) +{ + struct kasan_alloc_meta *alloc_meta; + + alloc_meta = kasan_get_alloc_meta(cache, object); + if (!alloc_meta) + return; + + if (alloc_meta->aux_stack[0]) { + pr_err("Last potentially related work creation:\n"); + stack_depot_print(alloc_meta->aux_stack[0]); + pr_err("\n"); + } + if (alloc_meta->aux_stack[1]) { + pr_err("Second to last potentially related work creation:\n"); + stack_depot_print(alloc_meta->aux_stack[1]); + pr_err("\n"); + } +} + #ifdef CONFIG_KASAN_STACK static bool __must_check tokenize_frame_descr(const char **frame_descr, char *token, size_t max_tok_len, diff --git a/mm/kasan/report_tags.c b/mm/kasan/report_tags.c index e25d2166e813d6ce421f9a6e2ddfecb05745e305..ecede06ef374aafc7cf0d5f275667ee0499a956a 100644 --- a/mm/kasan/report_tags.c +++ b/mm/kasan/report_tags.c @@ -4,38 +4,14 @@ * Copyright (c) 2020 Google, Inc. */ +#include + #include "kasan.h" -#include "../slab.h" -const char *kasan_get_bug_type(struct kasan_report_info *info) -{ -#ifdef CONFIG_KASAN_TAGS_IDENTIFY - struct kasan_alloc_meta *alloc_meta; - struct kmem_cache *cache; - struct slab *slab; - const void *addr; - void *object; - u8 tag; - int i; - - tag = get_tag(info->access_addr); - addr = kasan_reset_tag(info->access_addr); - slab = kasan_addr_to_slab(addr); - if (slab) { - cache = slab->slab_cache; - object = nearest_obj(cache, slab, (void *)addr); - alloc_meta = kasan_get_alloc_meta(cache, object); - - if (alloc_meta) { - for (i = 0; i < KASAN_NR_FREE_STACKS; i++) { - if (alloc_meta->free_pointer_tag[i] == tag) - return "use-after-free"; - } - } - return "out-of-bounds"; - } -#endif +extern struct kasan_stack_ring stack_ring; +static const char *get_common_bug_type(struct kasan_report_info *info) +{ /* * If access_size is a negative number, then it has reason to be * defined as out-of-bounds bug type. @@ -49,3 +25,92 @@ const char *kasan_get_bug_type(struct kasan_report_info *info) return "invalid-access"; } + +void kasan_complete_mode_report_info(struct kasan_report_info *info) +{ + unsigned long flags; + u64 pos; + struct kasan_stack_ring_entry *entry; + void *ptr; + u32 pid; + depot_stack_handle_t stack; + bool is_free; + bool alloc_found = false, free_found = false; + + if ((!info->cache || !info->object) && !info->bug_type) { + info->bug_type = get_common_bug_type(info); + return; + } + + write_lock_irqsave(&stack_ring.lock, flags); + + pos = atomic64_read(&stack_ring.pos); + + /* + * The loop below tries to find stack ring entries relevant to the + * buggy object. This is a best-effort process. + * + * First, another object with the same tag can be allocated in place of + * the buggy object. Also, since the number of entries is limited, the + * entries relevant to the buggy object can be overwritten. + */ + + for (u64 i = pos - 1; i != pos - 1 - stack_ring.size; i--) { + if (alloc_found && free_found) + break; + + entry = &stack_ring.entries[i % stack_ring.size]; + + /* Paired with smp_store_release() in save_stack_info(). */ + ptr = (void *)smp_load_acquire(&entry->ptr); + + if (kasan_reset_tag(ptr) != info->object || + get_tag(ptr) != get_tag(info->access_addr)) + continue; + + pid = READ_ONCE(entry->pid); + stack = READ_ONCE(entry->stack); + is_free = READ_ONCE(entry->is_free); + + if (is_free) { + /* + * Second free of the same object. + * Give up on trying to find the alloc entry. + */ + if (free_found) + break; + + info->free_track.pid = pid; + info->free_track.stack = stack; + free_found = true; + + /* + * If a free entry is found first, the bug is likely + * a use-after-free. + */ + if (!info->bug_type) + info->bug_type = "use-after-free"; + } else { + /* Second alloc of the same object. Give up. */ + if (alloc_found) + break; + + info->alloc_track.pid = pid; + info->alloc_track.stack = stack; + alloc_found = true; + + /* + * If an alloc entry is found first, the bug is likely + * an out-of-bounds. + */ + if (!info->bug_type) + info->bug_type = "slab-out-of-bounds"; + } + } + + write_unlock_irqrestore(&stack_ring.lock, flags); + + /* Assign the common bug type if no entries were found. */ + if (!info->bug_type) + info->bug_type = get_common_bug_type(info); +} diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index 77f13f391b577a539de520f1ef2d6e801c750ab3..a3afaf2ad1b110c3fb7756cd2e44c4e82669f428 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -42,7 +42,10 @@ void __init kasan_init_sw_tags(void) for_each_possible_cpu(cpu) per_cpu(prng_state, cpu) = (u32)get_cycles(); - pr_info("KernelAddressSanitizer initialized (sw-tags)\n"); + kasan_init_tags(); + + pr_info("KernelAddressSanitizer initialized (sw-tags, stacktrace=%s)\n", + kasan_stack_collection_enabled() ? "on" : "off"); } /* diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c index 8f48b9502a177dab50a66b47cff527bd8fb8ec83..67a222586846e8d86feeb4971dfc1067477791c5 100644 --- a/mm/kasan/tags.c +++ b/mm/kasan/tags.c @@ -6,9 +6,11 @@ * Copyright (c) 2020 Google, Inc. */ +#include #include #include #include +#include #include #include #include @@ -16,44 +18,127 @@ #include #include "kasan.h" +#include "../slab.h" -void kasan_set_free_info(struct kmem_cache *cache, - void *object, u8 tag) -{ - struct kasan_alloc_meta *alloc_meta; - u8 idx = 0; +#define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10) + +enum kasan_arg_stacktrace { + KASAN_ARG_STACKTRACE_DEFAULT, + KASAN_ARG_STACKTRACE_OFF, + KASAN_ARG_STACKTRACE_ON, +}; + +static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; + +/* Whether to collect alloc/free stack traces. */ +DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); - alloc_meta = kasan_get_alloc_meta(cache, object); - if (!alloc_meta) - return; +/* Non-zero, as initial pointer values are 0. */ +#define STACK_RING_BUSY_PTR ((void *)1) + +struct kasan_stack_ring stack_ring = { + .lock = __RW_LOCK_UNLOCKED(stack_ring.lock) +}; + +/* kasan.stacktrace=off/on */ +static int __init early_kasan_flag_stacktrace(char *arg) +{ + if (!arg) + return -EINVAL; -#ifdef CONFIG_KASAN_TAGS_IDENTIFY - idx = alloc_meta->free_track_idx; - alloc_meta->free_pointer_tag[idx] = tag; - alloc_meta->free_track_idx = (idx + 1) % KASAN_NR_FREE_STACKS; -#endif + if (!strcmp(arg, "off")) + kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; + else if (!strcmp(arg, "on")) + kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; + else + return -EINVAL; - kasan_set_track(&alloc_meta->free_track[idx], GFP_NOWAIT); + return 0; } +early_param("kasan.stacktrace", early_kasan_flag_stacktrace); -struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, - void *object, u8 tag) +/* kasan.stack_ring_size= */ +static int __init early_kasan_flag_stack_ring_size(char *arg) { - struct kasan_alloc_meta *alloc_meta; - int i = 0; + if (!arg) + return -EINVAL; - alloc_meta = kasan_get_alloc_meta(cache, object); - if (!alloc_meta) - return NULL; + return kstrtoul(arg, 0, &stack_ring.size); +} +early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size); + +void __init kasan_init_tags(void) +{ + switch (kasan_arg_stacktrace) { + case KASAN_ARG_STACKTRACE_DEFAULT: + /* Default is specified by kasan_flag_stacktrace definition. */ + break; + case KASAN_ARG_STACKTRACE_OFF: + static_branch_disable(&kasan_flag_stacktrace); + break; + case KASAN_ARG_STACKTRACE_ON: + static_branch_enable(&kasan_flag_stacktrace); + break; + } -#ifdef CONFIG_KASAN_TAGS_IDENTIFY - for (i = 0; i < KASAN_NR_FREE_STACKS; i++) { - if (alloc_meta->free_pointer_tag[i] == tag) - break; + if (kasan_stack_collection_enabled()) { + if (!stack_ring.size) + stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT; + stack_ring.entries = memblock_alloc( + sizeof(stack_ring.entries[0]) * stack_ring.size, + SMP_CACHE_BYTES); + if (WARN_ON(!stack_ring.entries)) + static_branch_disable(&kasan_flag_stacktrace); } - if (i == KASAN_NR_FREE_STACKS) - i = alloc_meta->free_track_idx; -#endif +} + +static void save_stack_info(struct kmem_cache *cache, void *object, + gfp_t gfp_flags, bool is_free) +{ + unsigned long flags; + depot_stack_handle_t stack; + u64 pos; + struct kasan_stack_ring_entry *entry; + void *old_ptr; + + stack = kasan_save_stack(gfp_flags, true); + + /* + * Prevent save_stack_info() from modifying stack ring + * when kasan_complete_mode_report_info() is walking it. + */ + read_lock_irqsave(&stack_ring.lock, flags); + +next: + pos = atomic64_fetch_add(1, &stack_ring.pos); + entry = &stack_ring.entries[pos % stack_ring.size]; + + /* Detect stack ring entry slots that are being written to. */ + old_ptr = READ_ONCE(entry->ptr); + if (old_ptr == STACK_RING_BUSY_PTR) + goto next; /* Busy slot. */ + if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR)) + goto next; /* Busy slot. */ + + WRITE_ONCE(entry->size, cache->object_size); + WRITE_ONCE(entry->pid, current->pid); + WRITE_ONCE(entry->stack, stack); + WRITE_ONCE(entry->is_free, is_free); + + /* + * Paired with smp_load_acquire() in kasan_complete_mode_report_info(). + */ + smp_store_release(&entry->ptr, (s64)object); - return &alloc_meta->free_track[i]; + read_unlock_irqrestore(&stack_ring.lock, flags); +} + +void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) +{ + save_stack_info(cache, object, flags, false); +} + +void kasan_save_free_info(struct kmem_cache *cache, void *object) +{ + save_stack_info(cache, object, GFP_NOWAIT, true); } diff --git a/mm/kfence/core.c b/mm/kfence/core.c index c252081b11dfe57f5d608f385785787c06b72347..141788858b7082e43c1c639059b038a20ff6ea39 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -719,24 +719,13 @@ static int show_object(struct seq_file *seq, void *v) return 0; } -static const struct seq_operations object_seqops = { +static const struct seq_operations objects_sops = { .start = start_object, .next = next_object, .stop = stop_object, .show = show_object, }; - -static int open_objects(struct inode *inode, struct file *file) -{ - return seq_open(file, &object_seqops); -} - -static const struct file_operations objects_fops = { - .open = open_objects, - .read = seq_read, - .llseek = seq_lseek, - .release = seq_release, -}; +DEFINE_SEQ_ATTRIBUTE(objects); static int __init kfence_debugfs_init(void) { @@ -864,7 +853,7 @@ static void kfence_init_enable(void) void __init kfence_init(void) { - stack_hash_seed = (u32)random_get_entropy(); + stack_hash_seed = get_random_u32(); /* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ if (!kfence_sample_interval) @@ -1003,6 +992,13 @@ void *__kfence_alloc(struct kmem_cache *s, size_t size, gfp_t flags) return NULL; } + /* + * Skip allocations for this slab, if KFENCE has been disabled for + * this slab. + */ + if (s->flags & SLAB_SKIP_KFENCE) + return NULL; + if (atomic_inc_return(&kfence_allocation_gate) > 1) return NULL; #ifdef CONFIG_KFENCE_STATIC_KEYS diff --git a/mm/kfence/report.c b/mm/kfence/report.c index f5a6d8ba3e21fef25cee0ec6b449d9ddd2de646d..7e496856c2ebeb6b5f0b7dc88aa8c6904e8d5db3 100644 --- a/mm/kfence/report.c +++ b/mm/kfence/report.c @@ -86,6 +86,7 @@ static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries /* Also the *_bulk() variants by only checking prefixes. */ if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || + str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") || str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) goto found; diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 01f71786d530353c2c9a14adaf068c13cb204f83..4734315f79407bc75db0f995445621f41b1d2f17 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -23,16 +23,20 @@ #include #include #include "internal.h" +#include "mm_slot.h" enum scan_result { SCAN_FAIL, SCAN_SUCCEED, SCAN_PMD_NULL, + SCAN_PMD_NONE, + SCAN_PMD_MAPPED, SCAN_EXCEED_NONE_PTE, SCAN_EXCEED_SWAP_PTE, SCAN_EXCEED_SHARED_PTE, SCAN_PTE_NON_PRESENT, SCAN_PTE_UFFD_WP, + SCAN_PTE_MAPPED_HUGEPAGE, SCAN_PAGE_RO, SCAN_LACK_REFERENCED_PAGE, SCAN_PAGE_NULL, @@ -73,6 +77,8 @@ static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait); * default collapse hugepages if there is at least one pte mapped like * it would have happened if the vma was large enough during page * fault. + * + * Note that these are only respected if collapse was initiated by khugepaged. */ static unsigned int khugepaged_max_ptes_none __read_mostly; static unsigned int khugepaged_max_ptes_swap __read_mostly; @@ -85,18 +91,24 @@ static struct kmem_cache *mm_slot_cache __read_mostly; #define MAX_PTE_MAPPED_THP 8 +struct collapse_control { + bool is_khugepaged; + + /* Num pages scanned per node */ + u32 node_load[MAX_NUMNODES]; + + /* Last target selected in hpage_collapse_find_target_node() */ + int last_target_node; +}; + /** - * struct mm_slot - hash lookup from mm to mm_slot - * @hash: hash collision list - * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head - * @mm: the mm that this information is valid for + * struct khugepaged_mm_slot - khugepaged information per mm that is being scanned + * @slot: hash lookup from mm to mm_slot * @nr_pte_mapped_thp: number of pte mapped THP * @pte_mapped_thp: address array corresponding pte mapped THP */ -struct mm_slot { - struct hlist_node hash; - struct list_head mm_node; - struct mm_struct *mm; +struct khugepaged_mm_slot { + struct mm_slot slot; /* pte-mapped THP in this mm */ int nr_pte_mapped_thp; @@ -113,7 +125,7 @@ struct mm_slot { */ struct khugepaged_scan { struct list_head mm_head; - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; unsigned long address; }; @@ -377,8 +389,9 @@ int hugepage_madvise(struct vm_area_struct *vma, int __init khugepaged_init(void) { mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", - sizeof(struct mm_slot), - __alignof__(struct mm_slot), 0, NULL); + sizeof(struct khugepaged_mm_slot), + __alignof__(struct khugepaged_mm_slot), + 0, NULL); if (!mm_slot_cache) return -ENOMEM; @@ -395,65 +408,38 @@ void __init khugepaged_destroy(void) kmem_cache_destroy(mm_slot_cache); } -static inline struct mm_slot *alloc_mm_slot(void) -{ - if (!mm_slot_cache) /* initialization failed */ - return NULL; - return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); -} - -static inline void free_mm_slot(struct mm_slot *mm_slot) -{ - kmem_cache_free(mm_slot_cache, mm_slot); -} - -static struct mm_slot *get_mm_slot(struct mm_struct *mm) -{ - struct mm_slot *mm_slot; - - hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) - if (mm == mm_slot->mm) - return mm_slot; - - return NULL; -} - -static void insert_to_mm_slots_hash(struct mm_struct *mm, - struct mm_slot *mm_slot) -{ - mm_slot->mm = mm; - hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); -} - -static inline int khugepaged_test_exit(struct mm_struct *mm) +static inline int hpage_collapse_test_exit(struct mm_struct *mm) { return atomic_read(&mm->mm_users) == 0; } void __khugepaged_enter(struct mm_struct *mm) { - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; + struct mm_slot *slot; int wakeup; - mm_slot = alloc_mm_slot(); + mm_slot = mm_slot_alloc(mm_slot_cache); if (!mm_slot) return; + slot = &mm_slot->slot; + /* __khugepaged_exit() must not run from under us */ - VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); + VM_BUG_ON_MM(hpage_collapse_test_exit(mm), mm); if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { - free_mm_slot(mm_slot); + mm_slot_free(mm_slot_cache, mm_slot); return; } spin_lock(&khugepaged_mm_lock); - insert_to_mm_slots_hash(mm, mm_slot); + mm_slot_insert(mm_slots_hash, mm, slot); /* * Insert just behind the scanning cursor, to let the area settle * down a little. */ wakeup = list_empty(&khugepaged_scan.mm_head); - list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); + list_add_tail(&slot->mm_node, &khugepaged_scan.mm_head); spin_unlock(&khugepaged_mm_lock); mmgrab(mm); @@ -466,37 +452,38 @@ void khugepaged_enter_vma(struct vm_area_struct *vma, { if (!test_bit(MMF_VM_HUGEPAGE, &vma->vm_mm->flags) && hugepage_flags_enabled()) { - if (hugepage_vma_check(vma, vm_flags, false, false)) + if (hugepage_vma_check(vma, vm_flags, false, false, true)) __khugepaged_enter(vma->vm_mm); } } void __khugepaged_exit(struct mm_struct *mm) { - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; + struct mm_slot *slot; int free = 0; spin_lock(&khugepaged_mm_lock); - mm_slot = get_mm_slot(mm); + slot = mm_slot_lookup(mm_slots_hash, mm); + mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { - hash_del(&mm_slot->hash); - list_del(&mm_slot->mm_node); + hash_del(&slot->hash); + list_del(&slot->mm_node); free = 1; } spin_unlock(&khugepaged_mm_lock); if (free) { clear_bit(MMF_VM_HUGEPAGE, &mm->flags); - free_mm_slot(mm_slot); + mm_slot_free(mm_slot_cache, mm_slot); mmdrop(mm); } else if (mm_slot) { /* * This is required to serialize against - * khugepaged_test_exit() (which is guaranteed to run - * under mmap sem read mode). Stop here (after we - * return all pagetables will be destroyed) until - * khugepaged has finished working on the pagetables - * under the mmap_lock. + * hpage_collapse_test_exit() (which is guaranteed to run + * under mmap sem read mode). Stop here (after we return all + * pagetables will be destroyed) until khugepaged has finished + * working on the pagetables under the mmap_lock. */ mmap_write_lock(mm); mmap_write_unlock(mm); @@ -546,11 +533,12 @@ static bool is_refcount_suitable(struct page *page) static int __collapse_huge_page_isolate(struct vm_area_struct *vma, unsigned long address, pte_t *pte, + struct collapse_control *cc, struct list_head *compound_pagelist) { struct page *page = NULL; pte_t *_pte; - int none_or_zero = 0, shared = 0, result = 0, referenced = 0; + int none_or_zero = 0, shared = 0, result = SCAN_FAIL, referenced = 0; bool writable = false; for (_pte = pte; _pte < pte + HPAGE_PMD_NR; @@ -558,8 +546,10 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, pte_t pteval = *_pte; if (pte_none(pteval) || (pte_present(pteval) && is_zero_pfn(pte_pfn(pteval)))) { + ++none_or_zero; if (!userfaultfd_armed(vma) && - ++none_or_zero <= khugepaged_max_ptes_none) { + (!cc->is_khugepaged || + none_or_zero <= khugepaged_max_ptes_none)) { continue; } else { result = SCAN_EXCEED_NONE_PTE; @@ -579,11 +569,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, VM_BUG_ON_PAGE(!PageAnon(page), page); - if (page_mapcount(page) > 1 && - ++shared > khugepaged_max_ptes_shared) { - result = SCAN_EXCEED_SHARED_PTE; - count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); - goto out; + if (page_mapcount(page) > 1) { + ++shared; + if (cc->is_khugepaged && + shared > khugepaged_max_ptes_shared) { + result = SCAN_EXCEED_SHARED_PTE; + count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); + goto out; + } } if (PageCompound(page)) { @@ -646,10 +639,14 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma, if (PageCompound(page)) list_add_tail(&page->lru, compound_pagelist); next: - /* There should be enough young pte to collapse the page */ - if (pte_young(pteval) || - page_is_young(page) || PageReferenced(page) || - mmu_notifier_test_young(vma->vm_mm, address)) + /* + * If collapse was initiated by khugepaged, check that there is + * enough young pte to justify collapsing the page + */ + if (cc->is_khugepaged && + (pte_young(pteval) || page_is_young(page) || + PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, + address))) referenced++; if (pte_write(pteval)) @@ -658,19 +655,19 @@ next: if (unlikely(!writable)) { result = SCAN_PAGE_RO; - } else if (unlikely(!referenced)) { + } else if (unlikely(cc->is_khugepaged && !referenced)) { result = SCAN_LACK_REFERENCED_PAGE; } else { result = SCAN_SUCCEED; trace_mm_collapse_huge_page_isolate(page, none_or_zero, referenced, writable, result); - return 1; + return result; } out: release_pte_pages(pte, _pte, compound_pagelist); trace_mm_collapse_huge_page_isolate(page, none_or_zero, referenced, writable, result); - return 0; + return result; } static void __collapse_huge_page_copy(pte_t *pte, struct page *page, @@ -730,14 +727,17 @@ static void khugepaged_alloc_sleep(void) DEFINE_WAIT(wait); add_wait_queue(&khugepaged_wait, &wait); - freezable_schedule_timeout_interruptible( - msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); + schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); remove_wait_queue(&khugepaged_wait, &wait); } -static int khugepaged_node_load[MAX_NUMNODES]; +struct collapse_control khugepaged_collapse_control = { + .is_khugepaged = true, + .last_target_node = NUMA_NO_NODE, +}; -static bool khugepaged_scan_abort(int nid) +static bool hpage_collapse_scan_abort(int nid, struct collapse_control *cc) { int i; @@ -749,11 +749,11 @@ static bool khugepaged_scan_abort(int nid) return false; /* If there is a count for this node already, it must be acceptable */ - if (khugepaged_node_load[nid]) + if (cc->node_load[nid]) return false; for (i = 0; i < MAX_NUMNODES; i++) { - if (!khugepaged_node_load[i]) + if (!cc->node_load[i]) continue; if (node_distance(nid, i) > node_reclaim_distance) return true; @@ -772,146 +772,63 @@ static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) } #ifdef CONFIG_NUMA -static int khugepaged_find_target_node(void) +static int hpage_collapse_find_target_node(struct collapse_control *cc) { - static int last_khugepaged_target_node = NUMA_NO_NODE; int nid, target_node = 0, max_value = 0; /* find first node with max normal pages hit */ for (nid = 0; nid < MAX_NUMNODES; nid++) - if (khugepaged_node_load[nid] > max_value) { - max_value = khugepaged_node_load[nid]; + if (cc->node_load[nid] > max_value) { + max_value = cc->node_load[nid]; target_node = nid; } /* do some balance if several nodes have the same hit record */ - if (target_node <= last_khugepaged_target_node) - for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; - nid++) - if (max_value == khugepaged_node_load[nid]) { + if (target_node <= cc->last_target_node) + for (nid = cc->last_target_node + 1; nid < MAX_NUMNODES; + nid++) + if (max_value == cc->node_load[nid]) { target_node = nid; break; } - last_khugepaged_target_node = target_node; + cc->last_target_node = target_node; return target_node; } - -static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) +#else +static int hpage_collapse_find_target_node(struct collapse_control *cc) { - if (IS_ERR(*hpage)) { - if (!*wait) - return false; - - *wait = false; - *hpage = NULL; - khugepaged_alloc_sleep(); - } else if (*hpage) { - put_page(*hpage); - *hpage = NULL; - } - - return true; + return 0; } +#endif -static struct page * -khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) +static bool hpage_collapse_alloc_page(struct page **hpage, gfp_t gfp, int node) { - VM_BUG_ON_PAGE(*hpage, *hpage); - *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); if (unlikely(!*hpage)) { count_vm_event(THP_COLLAPSE_ALLOC_FAILED); - *hpage = ERR_PTR(-ENOMEM); - return NULL; + return false; } prep_transhuge_page(*hpage); count_vm_event(THP_COLLAPSE_ALLOC); - return *hpage; -} -#else -static int khugepaged_find_target_node(void) -{ - return 0; -} - -static inline struct page *alloc_khugepaged_hugepage(void) -{ - struct page *page; - - page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(), - HPAGE_PMD_ORDER); - if (page) - prep_transhuge_page(page); - return page; -} - -static struct page *khugepaged_alloc_hugepage(bool *wait) -{ - struct page *hpage; - - do { - hpage = alloc_khugepaged_hugepage(); - if (!hpage) { - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); - if (!*wait) - return NULL; - - *wait = false; - khugepaged_alloc_sleep(); - } else - count_vm_event(THP_COLLAPSE_ALLOC); - } while (unlikely(!hpage) && likely(hugepage_flags_enabled())); - - return hpage; -} - -static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) -{ - /* - * If the hpage allocated earlier was briefly exposed in page cache - * before collapse_file() failed, it is possible that racing lookups - * have not yet completed, and would then be unpleasantly surprised by - * finding the hpage reused for the same mapping at a different offset. - * Just release the previous allocation if there is any danger of that. - */ - if (*hpage && page_count(*hpage) > 1) { - put_page(*hpage); - *hpage = NULL; - } - - if (!*hpage) - *hpage = khugepaged_alloc_hugepage(wait); - - if (unlikely(!*hpage)) - return false; - return true; } -static struct page * -khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) -{ - VM_BUG_ON(!*hpage); - - return *hpage; -} -#endif - /* * If mmap_lock temporarily dropped, revalidate vma * before taking mmap_lock. - * Return 0 if succeeds, otherwise return none-zero - * value (scan code). + * Returns enum scan_result value. */ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, - struct vm_area_struct **vmap) + bool expect_anon, + struct vm_area_struct **vmap, + struct collapse_control *cc) { struct vm_area_struct *vma; - if (unlikely(khugepaged_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit(mm))) return SCAN_ANY_PROCESS; *vmap = vma = find_vma(mm, address); @@ -920,7 +837,8 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, if (!transhuge_vma_suitable(vma, address)) return SCAN_ADDRESS_RANGE; - if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) + if (!hugepage_vma_check(vma, vma->vm_flags, false, false, + cc->is_khugepaged)) return SCAN_VMA_CHECK; /* * Anon VMA expected, the address may be unmapped then @@ -929,23 +847,62 @@ static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address, * hugepage_vma_check may return true for qualified file * vmas. */ - if (!vma->anon_vma || !vma_is_anonymous(vma)) - return SCAN_VMA_CHECK; - return 0; + if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap))) + return SCAN_PAGE_ANON; + return SCAN_SUCCEED; +} + +static int find_pmd_or_thp_or_none(struct mm_struct *mm, + unsigned long address, + pmd_t **pmd) +{ + pmd_t pmde; + + *pmd = mm_find_pmd(mm, address); + if (!*pmd) + return SCAN_PMD_NULL; + + pmde = pmd_read_atomic(*pmd); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + /* See comments in pmd_none_or_trans_huge_or_clear_bad() */ + barrier(); +#endif + if (pmd_none(pmde)) + return SCAN_PMD_NONE; + if (pmd_trans_huge(pmde)) + return SCAN_PMD_MAPPED; + if (pmd_bad(pmde)) + return SCAN_PMD_NULL; + return SCAN_SUCCEED; +} + +static int check_pmd_still_valid(struct mm_struct *mm, + unsigned long address, + pmd_t *pmd) +{ + pmd_t *new_pmd; + int result = find_pmd_or_thp_or_none(mm, address, &new_pmd); + + if (result != SCAN_SUCCEED) + return result; + if (new_pmd != pmd) + return SCAN_FAIL; + return SCAN_SUCCEED; } /* * Bring missing pages in from swap, to complete THP collapse. - * Only done if khugepaged_scan_pmd believes it is worthwhile. + * Only done if hpage_collapse_scan_pmd believes it is worthwhile. * * Called and returns without pte mapped or spinlocks held. * Note that if false is returned, mmap_lock will be released. */ -static bool __collapse_huge_page_swapin(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long haddr, pmd_t *pmd, - int referenced) +static int __collapse_huge_page_swapin(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long haddr, pmd_t *pmd, + int referenced) { int swapped_in = 0; vm_fault_t ret = 0; @@ -976,12 +933,13 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, */ if (ret & VM_FAULT_RETRY) { trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); - return false; + /* Likely, but not guaranteed, that page lock failed */ + return SCAN_PAGE_LOCK; } if (ret & VM_FAULT_ERROR) { mmap_read_unlock(mm); trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 0); - return false; + return SCAN_FAIL; } swapped_in++; } @@ -991,30 +949,41 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm, lru_add_drain(); trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, 1); - return true; + return SCAN_SUCCEED; } -static void collapse_huge_page(struct mm_struct *mm, - unsigned long address, - struct page **hpage, - int node, int referenced, int unmapped) +static int alloc_charge_hpage(struct page **hpage, struct mm_struct *mm, + struct collapse_control *cc) +{ + /* Only allocate from the target node */ + gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : + GFP_TRANSHUGE) | __GFP_THISNODE; + int node = hpage_collapse_find_target_node(cc); + + if (!hpage_collapse_alloc_page(hpage, gfp, node)) + return SCAN_ALLOC_HUGE_PAGE_FAIL; + if (unlikely(mem_cgroup_charge(page_folio(*hpage), mm, gfp))) + return SCAN_CGROUP_CHARGE_FAIL; + count_memcg_page_event(*hpage, THP_COLLAPSE_ALLOC); + return SCAN_SUCCEED; +} + +static int collapse_huge_page(struct mm_struct *mm, unsigned long address, + int referenced, int unmapped, + struct collapse_control *cc) { LIST_HEAD(compound_pagelist); pmd_t *pmd, _pmd; pte_t *pte; pgtable_t pgtable; - struct page *new_page; + struct page *hpage; spinlock_t *pmd_ptl, *pte_ptl; - int isolated = 0, result = 0; + int result = SCAN_FAIL; struct vm_area_struct *vma; struct mmu_notifier_range range; - gfp_t gfp; VM_BUG_ON(address & ~HPAGE_PMD_MASK); - /* Only allocate from the target node */ - gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; - /* * Before allocating the hugepage, release the mmap_lock read lock. * The allocation can take potentially a long time if it involves @@ -1022,40 +991,34 @@ static void collapse_huge_page(struct mm_struct *mm, * that. We will recheck the vma after taking it again in write mode. */ mmap_read_unlock(mm); - new_page = khugepaged_alloc_page(hpage, gfp, node); - if (!new_page) { - result = SCAN_ALLOC_HUGE_PAGE_FAIL; - goto out_nolock; - } - if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { - result = SCAN_CGROUP_CHARGE_FAIL; + result = alloc_charge_hpage(&hpage, mm, cc); + if (result != SCAN_SUCCEED) goto out_nolock; - } - count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); mmap_read_lock(mm); - result = hugepage_vma_revalidate(mm, address, &vma); - if (result) { + result = hugepage_vma_revalidate(mm, address, true, &vma, cc); + if (result != SCAN_SUCCEED) { mmap_read_unlock(mm); goto out_nolock; } - pmd = mm_find_pmd(mm, address); - if (!pmd) { - result = SCAN_PMD_NULL; + result = find_pmd_or_thp_or_none(mm, address, &pmd); + if (result != SCAN_SUCCEED) { mmap_read_unlock(mm); goto out_nolock; } - /* - * __collapse_huge_page_swapin will return with mmap_lock released - * when it fails. So we jump out_nolock directly in that case. - * Continuing to collapse causes inconsistency. - */ - if (unmapped && !__collapse_huge_page_swapin(mm, vma, address, - pmd, referenced)) { - goto out_nolock; + if (unmapped) { + /* + * __collapse_huge_page_swapin will return with mmap_lock + * released when it fails. So we jump out_nolock directly in + * that case. Continuing to collapse causes inconsistency. + */ + result = __collapse_huge_page_swapin(mm, vma, address, pmd, + referenced); + if (result != SCAN_SUCCEED) + goto out_nolock; } mmap_read_unlock(mm); @@ -1065,11 +1028,12 @@ static void collapse_huge_page(struct mm_struct *mm, * handled by the anon_vma lock + PG_lock. */ mmap_write_lock(mm); - result = hugepage_vma_revalidate(mm, address, &vma); - if (result) + result = hugepage_vma_revalidate(mm, address, true, &vma, cc); + if (result != SCAN_SUCCEED) goto out_up_write; /* check if the pmd is still valid */ - if (mm_find_pmd(mm, address) != pmd) + result = check_pmd_still_valid(mm, address, pmd); + if (result != SCAN_SUCCEED) goto out_up_write; anon_vma_lock_write(vma->anon_vma); @@ -1083,21 +1047,23 @@ static void collapse_huge_page(struct mm_struct *mm, pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */ /* - * After this gup_fast can't run anymore. This also removes - * any huge TLB entry from the CPU so we won't allow - * huge and small TLB entries for the same virtual address - * to avoid the risk of CPU bugs in that area. + * This removes any huge TLB entry from the CPU so we won't allow + * huge and small TLB entries for the same virtual address to + * avoid the risk of CPU bugs in that area. + * + * Parallel fast GUP is fine since fast GUP will back off when + * it detects PMD is changed. */ _pmd = pmdp_collapse_flush(vma, address, pmd); spin_unlock(pmd_ptl); mmu_notifier_invalidate_range_end(&range); spin_lock(pte_ptl); - isolated = __collapse_huge_page_isolate(vma, address, pte, - &compound_pagelist); + result = __collapse_huge_page_isolate(vma, address, pte, cc, + &compound_pagelist); spin_unlock(pte_ptl); - if (unlikely(!isolated)) { + if (unlikely(result != SCAN_SUCCEED)) { pte_unmap(pte); spin_lock(pmd_ptl); BUG_ON(!pmd_none(*pmd)); @@ -1109,7 +1075,6 @@ static void collapse_huge_page(struct mm_struct *mm, pmd_populate(mm, pmd, pmd_pgtable(_pmd)); spin_unlock(pmd_ptl); anon_vma_unlock_write(vma->anon_vma); - result = SCAN_FAIL; goto out_up_write; } @@ -1119,8 +1084,8 @@ static void collapse_huge_page(struct mm_struct *mm, */ anon_vma_unlock_write(vma->anon_vma); - __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl, - &compound_pagelist); + __collapse_huge_page_copy(pte, hpage, vma, address, pte_ptl, + &compound_pagelist); pte_unmap(pte); /* * spin_lock() below is not the equivalent of smp_wmb(), but @@ -1128,42 +1093,43 @@ static void collapse_huge_page(struct mm_struct *mm, * avoid the copy_huge_page writes to become visible after * the set_pmd_at() write. */ - __SetPageUptodate(new_page); + __SetPageUptodate(hpage); pgtable = pmd_pgtable(_pmd); - _pmd = mk_huge_pmd(new_page, vma->vm_page_prot); + _pmd = mk_huge_pmd(hpage, vma->vm_page_prot); _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma); spin_lock(pmd_ptl); BUG_ON(!pmd_none(*pmd)); - page_add_new_anon_rmap(new_page, vma, address); - lru_cache_add_inactive_or_unevictable(new_page, vma); + page_add_new_anon_rmap(hpage, vma, address); + lru_cache_add_inactive_or_unevictable(hpage, vma); pgtable_trans_huge_deposit(mm, pmd, pgtable); set_pmd_at(mm, address, pmd, _pmd); update_mmu_cache_pmd(vma, address, pmd); spin_unlock(pmd_ptl); - *hpage = NULL; + hpage = NULL; - khugepaged_pages_collapsed++; result = SCAN_SUCCEED; out_up_write: mmap_write_unlock(mm); out_nolock: - if (!IS_ERR_OR_NULL(*hpage)) - mem_cgroup_uncharge(page_folio(*hpage)); - trace_mm_collapse_huge_page(mm, isolated, result); - return; + if (hpage) { + mem_cgroup_uncharge(page_folio(hpage)); + put_page(hpage); + } + trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result); + return result; } -static int khugepaged_scan_pmd(struct mm_struct *mm, - struct vm_area_struct *vma, - unsigned long address, - struct page **hpage) +static int hpage_collapse_scan_pmd(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long address, bool *mmap_locked, + struct collapse_control *cc) { pmd_t *pmd; pte_t *pte, *_pte; - int ret = 0, result = 0, referenced = 0; + int result = SCAN_FAIL, referenced = 0; int none_or_zero = 0, shared = 0; struct page *page = NULL; unsigned long _address; @@ -1173,19 +1139,19 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, VM_BUG_ON(address & ~HPAGE_PMD_MASK); - pmd = mm_find_pmd(mm, address); - if (!pmd) { - result = SCAN_PMD_NULL; + result = find_pmd_or_thp_or_none(mm, address, &pmd); + if (result != SCAN_SUCCEED) goto out; - } - memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); + memset(cc->node_load, 0, sizeof(cc->node_load)); pte = pte_offset_map_lock(mm, pmd, address, &ptl); for (_address = address, _pte = pte; _pte < pte + HPAGE_PMD_NR; _pte++, _address += PAGE_SIZE) { pte_t pteval = *_pte; if (is_swap_pte(pteval)) { - if (++unmapped <= khugepaged_max_ptes_swap) { + ++unmapped; + if (!cc->is_khugepaged || + unmapped <= khugepaged_max_ptes_swap) { /* * Always be strict with uffd-wp * enabled swap entries. Please see @@ -1203,8 +1169,10 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, } } if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { + ++none_or_zero; if (!userfaultfd_armed(vma) && - ++none_or_zero <= khugepaged_max_ptes_none) { + (!cc->is_khugepaged || + none_or_zero <= khugepaged_max_ptes_none)) { continue; } else { result = SCAN_EXCEED_NONE_PTE; @@ -1234,27 +1202,30 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, goto out_unmap; } - if (page_mapcount(page) > 1 && - ++shared > khugepaged_max_ptes_shared) { - result = SCAN_EXCEED_SHARED_PTE; - count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); - goto out_unmap; + if (page_mapcount(page) > 1) { + ++shared; + if (cc->is_khugepaged && + shared > khugepaged_max_ptes_shared) { + result = SCAN_EXCEED_SHARED_PTE; + count_vm_event(THP_SCAN_EXCEED_SHARED_PTE); + goto out_unmap; + } } page = compound_head(page); /* * Record which node the original page is from and save this - * information to khugepaged_node_load[]. + * information to cc->node_load[]. * Khugepaged will allocate hugepage from the node has the max * hit record. */ node = page_to_nid(page); - if (khugepaged_scan_abort(node)) { + if (hpage_collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; goto out_unmap; } - khugepaged_node_load[node]++; + cc->node_load[node]++; if (!PageLRU(page)) { result = SCAN_PAGE_LRU; goto out_unmap; @@ -1289,43 +1260,51 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, result = SCAN_PAGE_COUNT; goto out_unmap; } - if (pte_young(pteval) || - page_is_young(page) || PageReferenced(page) || - mmu_notifier_test_young(vma->vm_mm, address)) + + /* + * If collapse was initiated by khugepaged, check that there is + * enough young pte to justify collapsing the page + */ + if (cc->is_khugepaged && + (pte_young(pteval) || page_is_young(page) || + PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, + address))) referenced++; } if (!writable) { result = SCAN_PAGE_RO; - } else if (!referenced || (unmapped && referenced < HPAGE_PMD_NR/2)) { + } else if (cc->is_khugepaged && + (!referenced || + (unmapped && referenced < HPAGE_PMD_NR / 2))) { result = SCAN_LACK_REFERENCED_PAGE; } else { result = SCAN_SUCCEED; - ret = 1; } out_unmap: pte_unmap_unlock(pte, ptl); - if (ret) { - node = khugepaged_find_target_node(); + if (result == SCAN_SUCCEED) { + result = collapse_huge_page(mm, address, referenced, + unmapped, cc); /* collapse_huge_page will return with the mmap_lock released */ - collapse_huge_page(mm, address, hpage, node, - referenced, unmapped); + *mmap_locked = false; } out: trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced, none_or_zero, result, unmapped); - return ret; + return result; } -static void collect_mm_slot(struct mm_slot *mm_slot) +static void collect_mm_slot(struct khugepaged_mm_slot *mm_slot) { - struct mm_struct *mm = mm_slot->mm; + struct mm_slot *slot = &mm_slot->slot; + struct mm_struct *mm = slot->mm; lockdep_assert_held(&khugepaged_mm_lock); - if (khugepaged_test_exit(mm)) { + if (hpage_collapse_test_exit(mm)) { /* free mm_slot */ - hash_del(&mm_slot->hash); - list_del(&mm_slot->mm_node); + hash_del(&slot->hash); + list_del(&slot->mm_node); /* * Not strictly needed because the mm exited already. @@ -1334,7 +1313,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) */ /* khugepaged_mm_lock actually not necessary for the below */ - free_mm_slot(mm_slot); + mm_slot_free(mm_slot_cache, mm_slot); mmdrop(mm); } } @@ -1343,19 +1322,66 @@ static void collect_mm_slot(struct mm_slot *mm_slot) /* * Notify khugepaged that given addr of the mm is pte-mapped THP. Then * khugepaged should try to collapse the page table. + * + * Note that following race exists: + * (1) khugepaged calls khugepaged_collapse_pte_mapped_thps() for mm_struct A, + * emptying the A's ->pte_mapped_thp[] array. + * (2) MADV_COLLAPSE collapses some file extent with target mm_struct B, and + * retract_page_tables() finds a VMA in mm_struct A mapping the same extent + * (at virtual address X) and adds an entry (for X) into mm_struct A's + * ->pte-mapped_thp[] array. + * (3) khugepaged calls khugepaged_collapse_scan_file() for mm_struct A at X, + * sees a pte-mapped THP (SCAN_PTE_MAPPED_HUGEPAGE) and adds an entry + * (for X) into mm_struct A's ->pte-mapped_thp[] array. + * Thus, it's possible the same address is added multiple times for the same + * mm_struct. Should this happen, we'll simply attempt + * collapse_pte_mapped_thp() multiple times for the same address, under the same + * exclusive mmap_lock, and assuming the first call is successful, subsequent + * attempts will return quickly (without grabbing any additional locks) when + * a huge pmd is found in find_pmd_or_thp_or_none(). Since this is a cheap + * check, and since this is a rare occurrence, the cost of preventing this + * "multiple-add" is thought to be more expensive than just handling it, should + * it occur. */ -static void khugepaged_add_pte_mapped_thp(struct mm_struct *mm, +static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) { - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; + struct mm_slot *slot; + bool ret = false; VM_BUG_ON(addr & ~HPAGE_PMD_MASK); spin_lock(&khugepaged_mm_lock); - mm_slot = get_mm_slot(mm); - if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) + slot = mm_slot_lookup(mm_slots_hash, mm); + mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); + if (likely(mm_slot && mm_slot->nr_pte_mapped_thp < MAX_PTE_MAPPED_THP)) { mm_slot->pte_mapped_thp[mm_slot->nr_pte_mapped_thp++] = addr; + ret = true; + } spin_unlock(&khugepaged_mm_lock); + return ret; +} + +/* hpage must be locked, and mmap_lock must be held in write */ +static int set_huge_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmdp, struct page *hpage) +{ + struct vm_fault vmf = { + .vma = vma, + .address = addr, + .flags = 0, + .pmd = pmdp, + }; + + VM_BUG_ON(!PageTransHuge(hpage)); + mmap_assert_write_locked(vma->vm_mm); + + if (do_set_pmd(&vmf, hpage)) + return SCAN_FAIL; + + get_page(hpage); + return SCAN_SUCCEED; } static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *vma, @@ -1379,52 +1405,80 @@ static void collapse_and_free_pmd(struct mm_struct *mm, struct vm_area_struct *v * * @mm: process address space where collapse happens * @addr: THP collapse address + * @install_pmd: If a huge PMD should be installed * * This function checks whether all the PTEs in the PMD are pointing to the * right THP. If so, retract the page table so the THP can refault in with - * as pmd-mapped. + * as pmd-mapped. Possibly install a huge PMD mapping the THP. */ -void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) +int collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr, + bool install_pmd) { unsigned long haddr = addr & HPAGE_PMD_MASK; - struct vm_area_struct *vma = find_vma(mm, haddr); + struct vm_area_struct *vma = vma_lookup(mm, haddr); struct page *hpage; pte_t *start_pte, *pte; pmd_t *pmd; spinlock_t *ptl; - int count = 0; + int count = 0, result = SCAN_FAIL; int i; + mmap_assert_write_locked(mm); + + /* Fast check before locking page if already PMD-mapped */ + result = find_pmd_or_thp_or_none(mm, haddr, &pmd); + if (result == SCAN_PMD_MAPPED) + return result; + if (!vma || !vma->vm_file || !range_in_vma(vma, haddr, haddr + HPAGE_PMD_SIZE)) - return; + return SCAN_VMA_CHECK; /* - * This vm_flags may not have VM_HUGEPAGE if the page was not - * collapsed by this mm. But we can still collapse if the page is - * the valid THP. Add extra VM_HUGEPAGE so hugepage_vma_check() - * will not fail the vma for missing VM_HUGEPAGE + * If we are here, we've succeeded in replacing all the native pages + * in the page cache with a single hugepage. If a mm were to fault-in + * this memory (mapped by a suitably aligned VMA), we'd get the hugepage + * and map it by a PMD, regardless of sysfs THP settings. As such, let's + * analogously elide sysfs THP settings here. */ - if (!hugepage_vma_check(vma, vma->vm_flags | VM_HUGEPAGE, false, false)) - return; + if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) + return SCAN_VMA_CHECK; /* Keep pmd pgtable for uffd-wp; see comment in retract_page_tables() */ if (userfaultfd_wp(vma)) - return; + return SCAN_PTE_UFFD_WP; hpage = find_lock_page(vma->vm_file->f_mapping, linear_page_index(vma, haddr)); if (!hpage) - return; + return SCAN_PAGE_NULL; + + if (!PageHead(hpage)) { + result = SCAN_FAIL; + goto drop_hpage; + } - if (!PageHead(hpage)) + if (compound_order(hpage) != HPAGE_PMD_ORDER) { + result = SCAN_PAGE_COMPOUND; goto drop_hpage; + } - pmd = mm_find_pmd(mm, haddr); - if (!pmd) + switch (result) { + case SCAN_SUCCEED: + break; + case SCAN_PMD_NONE: + /* + * In MADV_COLLAPSE path, possible race with khugepaged where + * all pte entries have been removed and pmd cleared. If so, + * skip all the pte checks and just update the pmd mapping. + */ + goto maybe_install_pmd; + default: goto drop_hpage; + } start_pte = pte_offset_map_lock(mm, pmd, haddr, &ptl); + result = SCAN_FAIL; /* step 1: check all mapped PTEs are to the right huge page */ for (i = 0, addr = haddr, pte = start_pte; @@ -1436,8 +1490,10 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) continue; /* page swapped out, abort */ - if (!pte_present(*pte)) + if (!pte_present(*pte)) { + result = SCAN_PTE_NON_PRESENT; goto abort; + } page = vm_normal_page(vma, addr, *pte); if (WARN_ON_ONCE(page && is_zone_device_page(page))) @@ -1472,21 +1528,29 @@ void collapse_pte_mapped_thp(struct mm_struct *mm, unsigned long addr) add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); } - /* step 4: collapse pmd */ + /* step 4: remove pte entries */ collapse_and_free_pmd(mm, vma, haddr, pmd); + +maybe_install_pmd: + /* step 5: install pmd entry */ + result = install_pmd + ? set_huge_pmd(vma, haddr, pmd, hpage) + : SCAN_SUCCEED; + drop_hpage: unlock_page(hpage); put_page(hpage); - return; + return result; abort: pte_unmap_unlock(start_pte, ptl); goto drop_hpage; } -static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) +static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) { - struct mm_struct *mm = mm_slot->mm; + struct mm_slot *slot = &mm_slot->slot; + struct mm_struct *mm = slot->mm; int i; if (likely(mm_slot->nr_pte_mapped_thp == 0)) @@ -1495,26 +1559,33 @@ static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) if (!mmap_write_trylock(mm)) return; - if (unlikely(khugepaged_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit(mm))) goto out; for (i = 0; i < mm_slot->nr_pte_mapped_thp; i++) - collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i]); + collapse_pte_mapped_thp(mm, mm_slot->pte_mapped_thp[i], false); out: mm_slot->nr_pte_mapped_thp = 0; mmap_write_unlock(mm); } -static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) +static int retract_page_tables(struct address_space *mapping, pgoff_t pgoff, + struct mm_struct *target_mm, + unsigned long target_addr, struct page *hpage, + struct collapse_control *cc) { struct vm_area_struct *vma; - struct mm_struct *mm; - unsigned long addr; - pmd_t *pmd; + int target_result = SCAN_FAIL; i_mmap_lock_write(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { + int result = SCAN_FAIL; + struct mm_struct *mm = NULL; + unsigned long addr = 0; + pmd_t *pmd; + bool is_target = false; + /* * Check vma->anon_vma to exclude MAP_PRIVATE mappings that * got written to. These VMAs are likely not worth investing @@ -1531,25 +1602,34 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * ptl. It has higher chance to recover THP for the VMA, but * has higher cost too. */ - if (vma->anon_vma) - continue; + if (vma->anon_vma) { + result = SCAN_PAGE_ANON; + goto next; + } addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); - if (addr & ~HPAGE_PMD_MASK) - continue; - if (vma->vm_end < addr + HPAGE_PMD_SIZE) - continue; + if (addr & ~HPAGE_PMD_MASK || + vma->vm_end < addr + HPAGE_PMD_SIZE) { + result = SCAN_VMA_CHECK; + goto next; + } mm = vma->vm_mm; - pmd = mm_find_pmd(mm, addr); - if (!pmd) - continue; + is_target = mm == target_mm && addr == target_addr; + result = find_pmd_or_thp_or_none(mm, addr, &pmd); + if (result != SCAN_SUCCEED) + goto next; /* * We need exclusive mmap_lock to retract page table. * * We use trylock due to lock inversion: we need to acquire * mmap_lock while holding page lock. Fault path does it in * reverse order. Trylock is a way to avoid deadlock. + * + * Also, it's not MADV_COLLAPSE's job to collapse other + * mappings - let khugepaged take care of them later. */ - if (mmap_write_trylock(mm)) { + result = SCAN_PTE_MAPPED_HUGEPAGE; + if ((cc->is_khugepaged || is_target) && + mmap_write_trylock(mm)) { /* * When a vma is registered with uffd-wp, we can't * recycle the pmd pgtable because there can be pte @@ -1558,25 +1638,48 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * it'll always mapped in small page size for uffd-wp * registered ranges. */ - if (!khugepaged_test_exit(mm) && !userfaultfd_wp(vma)) - collapse_and_free_pmd(mm, vma, addr, pmd); + if (hpage_collapse_test_exit(mm)) { + result = SCAN_ANY_PROCESS; + goto unlock_next; + } + if (userfaultfd_wp(vma)) { + result = SCAN_PTE_UFFD_WP; + goto unlock_next; + } + collapse_and_free_pmd(mm, vma, addr, pmd); + if (!cc->is_khugepaged && is_target) + result = set_huge_pmd(vma, addr, pmd, hpage); + else + result = SCAN_SUCCEED; + +unlock_next: mmap_write_unlock(mm); - } else { - /* Try again later */ + goto next; + } + /* + * Calling context will handle target mm/addr. Otherwise, let + * khugepaged try again later. + */ + if (!is_target) { khugepaged_add_pte_mapped_thp(mm, addr); + continue; } +next: + if (is_target) + target_result = result; } i_mmap_unlock_write(mapping); + return target_result; } /** * collapse_file - collapse filemap/tmpfs/shmem pages into huge one. * * @mm: process address space where collapse happens + * @addr: virtual collapse start address * @file: file that collapse on * @start: collapse start address - * @hpage: new allocated huge page for collapse - * @node: appointed node the new huge page allocate from + * @cc: collapse context and scratchpad * * Basic scheme is simple, details are more complex: * - allocate and lock a new huge page; @@ -1593,13 +1696,12 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + restore gaps in the page cache; * + unlock and free huge page; */ -static void collapse_file(struct mm_struct *mm, - struct file *file, pgoff_t start, - struct page **hpage, int node) +static int collapse_file(struct mm_struct *mm, unsigned long addr, + struct file *file, pgoff_t start, + struct collapse_control *cc) { struct address_space *mapping = file->f_mapping; - gfp_t gfp; - struct page *new_page; + struct page *hpage; pgoff_t index, end = start + HPAGE_PMD_NR; LIST_HEAD(pagelist); XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); @@ -1610,20 +1712,9 @@ static void collapse_file(struct mm_struct *mm, VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); - /* Only allocate from the target node */ - gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_THISNODE; - - new_page = khugepaged_alloc_page(hpage, gfp, node); - if (!new_page) { - result = SCAN_ALLOC_HUGE_PAGE_FAIL; - goto out; - } - - if (unlikely(mem_cgroup_charge(page_folio(new_page), mm, gfp))) { - result = SCAN_CGROUP_CHARGE_FAIL; + result = alloc_charge_hpage(&hpage, mm, cc); + if (result != SCAN_SUCCEED) goto out; - } - count_memcg_page_event(new_page, THP_COLLAPSE_ALLOC); /* * Ensure we have slots for all the pages in the range. This is @@ -1641,14 +1732,14 @@ static void collapse_file(struct mm_struct *mm, } } while (1); - __SetPageLocked(new_page); + __SetPageLocked(hpage); if (is_shmem) - __SetPageSwapBacked(new_page); - new_page->index = start; - new_page->mapping = mapping; + __SetPageSwapBacked(hpage); + hpage->index = start; + hpage->mapping = mapping; /* - * At this point the new_page is locked and not up-to-date. + * At this point the hpage is locked and not up-to-date. * It's safe to insert it into the page cache, because nobody would * be able to map it or use it in another way until we unlock it. */ @@ -1676,19 +1767,22 @@ static void collapse_file(struct mm_struct *mm, result = SCAN_FAIL; goto xa_locked; } - xas_store(&xas, new_page); + xas_store(&xas, hpage); nr_none++; continue; } if (xa_is_value(page) || !PageUptodate(page)) { + struct folio *folio; + xas_unlock_irq(&xas); /* swap in or instantiate fallocated page */ - if (shmem_getpage(mapping->host, index, &page, - SGP_NOALLOC)) { + if (shmem_get_folio(mapping->host, index, + &folio, SGP_NOALLOC)) { result = SCAN_FAIL; goto xa_unlocked; } + page = folio_file_page(folio, index); } else if (trylock_page(page)) { get_page(page); xas_unlock_irq(&xas); @@ -1755,9 +1849,16 @@ static void collapse_file(struct mm_struct *mm, /* * If file was truncated then extended, or hole-punched, before * we locked the first page, then a THP might be there already. + * This will be discovered on the first iteration. */ if (PageTransCompound(page)) { - result = SCAN_PAGE_COMPOUND; + struct page *head = compound_head(page); + + result = compound_order(head) == HPAGE_PMD_ORDER && + head->index == start + /* Maybe PMD-mapped */ + ? SCAN_PTE_MAPPED_HUGEPAGE + : SCAN_PAGE_COMPOUND; goto out_unlock; } @@ -1818,19 +1919,19 @@ static void collapse_file(struct mm_struct *mm, list_add_tail(&page->lru, &pagelist); /* Finally, replace with the new page. */ - xas_store(&xas, new_page); + xas_store(&xas, hpage); continue; out_unlock: unlock_page(page); put_page(page); goto xa_unlocked; } - nr = thp_nr_pages(new_page); + nr = thp_nr_pages(hpage); if (is_shmem) - __mod_lruvec_page_state(new_page, NR_SHMEM_THPS, nr); + __mod_lruvec_page_state(hpage, NR_SHMEM_THPS, nr); else { - __mod_lruvec_page_state(new_page, NR_FILE_THPS, nr); + __mod_lruvec_page_state(hpage, NR_FILE_THPS, nr); filemap_nr_thps_inc(mapping); /* * Paired with smp_mb() in do_dentry_open() to ensure @@ -1841,21 +1942,21 @@ out_unlock: smp_mb(); if (inode_is_open_for_write(mapping->host)) { result = SCAN_FAIL; - __mod_lruvec_page_state(new_page, NR_FILE_THPS, -nr); + __mod_lruvec_page_state(hpage, NR_FILE_THPS, -nr); filemap_nr_thps_dec(mapping); goto xa_locked; } } if (nr_none) { - __mod_lruvec_page_state(new_page, NR_FILE_PAGES, nr_none); + __mod_lruvec_page_state(hpage, NR_FILE_PAGES, nr_none); /* nr_none is always 0 for non-shmem. */ - __mod_lruvec_page_state(new_page, NR_SHMEM, nr_none); + __mod_lruvec_page_state(hpage, NR_SHMEM, nr_none); } /* Join all the small entries into a single multi-index entry */ xas_set_order(&xas, start, HPAGE_PMD_ORDER); - xas_store(&xas, new_page); + xas_store(&xas, hpage); xa_locked: xas_unlock_irq(&xas); xa_unlocked: @@ -1877,11 +1978,11 @@ xa_unlocked: index = start; list_for_each_entry_safe(page, tmp, &pagelist, lru) { while (index < page->index) { - clear_highpage(new_page + (index % HPAGE_PMD_NR)); + clear_highpage(hpage + (index % HPAGE_PMD_NR)); index++; } - copy_highpage(new_page + (page->index % HPAGE_PMD_NR), - page); + copy_highpage(hpage + (page->index % HPAGE_PMD_NR), + page); list_del(&page->lru); page->mapping = NULL; page_ref_unfreeze(page, 1); @@ -1892,23 +1993,23 @@ xa_unlocked: index++; } while (index < end) { - clear_highpage(new_page + (index % HPAGE_PMD_NR)); + clear_highpage(hpage + (index % HPAGE_PMD_NR)); index++; } - SetPageUptodate(new_page); - page_ref_add(new_page, HPAGE_PMD_NR - 1); + SetPageUptodate(hpage); + page_ref_add(hpage, HPAGE_PMD_NR - 1); if (is_shmem) - set_page_dirty(new_page); - lru_cache_add(new_page); + set_page_dirty(hpage); + lru_cache_add(hpage); /* * Remove pte page tables, so we can re-fault the page as huge. */ - retract_page_tables(mapping, start); - *hpage = NULL; - - khugepaged_pages_collapsed++; + result = retract_page_tables(mapping, start, mm, addr, hpage, + cc); + unlock_page(hpage); + hpage = NULL; } else { struct page *page; @@ -1947,19 +2048,24 @@ xa_unlocked: VM_BUG_ON(nr_none); xas_unlock_irq(&xas); - new_page->mapping = NULL; + hpage->mapping = NULL; } - unlock_page(new_page); + if (hpage) + unlock_page(hpage); out: VM_BUG_ON(!list_empty(&pagelist)); - if (!IS_ERR_OR_NULL(*hpage)) - mem_cgroup_uncharge(page_folio(*hpage)); + if (hpage) { + mem_cgroup_uncharge(page_folio(hpage)); + put_page(hpage); + } /* TODO: tracepoints */ + return result; } -static void khugepaged_scan_file(struct mm_struct *mm, - struct file *file, pgoff_t start, struct page **hpage) +static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, + struct file *file, pgoff_t start, + struct collapse_control *cc) { struct page *page = NULL; struct address_space *mapping = file->f_mapping; @@ -1970,14 +2076,16 @@ static void khugepaged_scan_file(struct mm_struct *mm, present = 0; swap = 0; - memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); + memset(cc->node_load, 0, sizeof(cc->node_load)); rcu_read_lock(); xas_for_each(&xas, page, start + HPAGE_PMD_NR - 1) { if (xas_retry(&xas, page)) continue; if (xa_is_value(page)) { - if (++swap > khugepaged_max_ptes_swap) { + ++swap; + if (cc->is_khugepaged && + swap > khugepaged_max_ptes_swap) { result = SCAN_EXCEED_SWAP_PTE; count_vm_event(THP_SCAN_EXCEED_SWAP_PTE); break; @@ -1986,20 +2094,32 @@ static void khugepaged_scan_file(struct mm_struct *mm, } /* - * XXX: khugepaged should compact smaller compound pages + * TODO: khugepaged should compact smaller compound pages * into a PMD sized page */ if (PageTransCompound(page)) { - result = SCAN_PAGE_COMPOUND; + struct page *head = compound_head(page); + + result = compound_order(head) == HPAGE_PMD_ORDER && + head->index == start + /* Maybe PMD-mapped */ + ? SCAN_PTE_MAPPED_HUGEPAGE + : SCAN_PAGE_COMPOUND; + /* + * For SCAN_PTE_MAPPED_HUGEPAGE, further processing + * by the caller won't touch the page cache, and so + * it's safe to skip LRU and refcount checks before + * returning. + */ break; } node = page_to_nid(page); - if (khugepaged_scan_abort(node)) { + if (hpage_collapse_scan_abort(node, cc)) { result = SCAN_SCAN_ABORT; break; } - khugepaged_node_load[node]++; + cc->node_load[node]++; if (!PageLRU(page)) { result = SCAN_PAGE_LRU; @@ -2028,54 +2148,68 @@ static void khugepaged_scan_file(struct mm_struct *mm, rcu_read_unlock(); if (result == SCAN_SUCCEED) { - if (present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { + if (cc->is_khugepaged && + present < HPAGE_PMD_NR - khugepaged_max_ptes_none) { result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - node = khugepaged_find_target_node(); - collapse_file(mm, file, start, hpage, node); + result = collapse_file(mm, addr, file, start, cc); } } - /* TODO: tracepoints */ + trace_mm_khugepaged_scan_file(mm, page, file->f_path.dentry->d_iname, + present, swap, result); + return result; } #else -static void khugepaged_scan_file(struct mm_struct *mm, - struct file *file, pgoff_t start, struct page **hpage) +static int hpage_collapse_scan_file(struct mm_struct *mm, unsigned long addr, + struct file *file, pgoff_t start, + struct collapse_control *cc) { BUILD_BUG(); } -static void khugepaged_collapse_pte_mapped_thps(struct mm_slot *mm_slot) +static void khugepaged_collapse_pte_mapped_thps(struct khugepaged_mm_slot *mm_slot) { } + +static bool khugepaged_add_pte_mapped_thp(struct mm_struct *mm, + unsigned long addr) +{ + return false; +} #endif -static unsigned int khugepaged_scan_mm_slot(unsigned int pages, - struct page **hpage) +static unsigned int khugepaged_scan_mm_slot(unsigned int pages, int *result, + struct collapse_control *cc) __releases(&khugepaged_mm_lock) __acquires(&khugepaged_mm_lock) { - struct mm_slot *mm_slot; + struct vma_iterator vmi; + struct khugepaged_mm_slot *mm_slot; + struct mm_slot *slot; struct mm_struct *mm; struct vm_area_struct *vma; int progress = 0; VM_BUG_ON(!pages); lockdep_assert_held(&khugepaged_mm_lock); + *result = SCAN_FAIL; - if (khugepaged_scan.mm_slot) + if (khugepaged_scan.mm_slot) { mm_slot = khugepaged_scan.mm_slot; - else { - mm_slot = list_entry(khugepaged_scan.mm_head.next, + slot = &mm_slot->slot; + } else { + slot = list_entry(khugepaged_scan.mm_head.next, struct mm_slot, mm_node); + mm_slot = mm_slot_entry(slot, struct khugepaged_mm_slot, slot); khugepaged_scan.address = 0; khugepaged_scan.mm_slot = mm_slot; } spin_unlock(&khugepaged_mm_lock); khugepaged_collapse_pte_mapped_thps(mm_slot); - mm = mm_slot->mm; + mm = slot->mm; /* * Don't wait for semaphore (to avoid long wait times). Just move to * the next mm on the list. @@ -2083,19 +2217,21 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, vma = NULL; if (unlikely(!mmap_read_trylock(mm))) goto breakouterloop_mmap_lock; - if (likely(!khugepaged_test_exit(mm))) - vma = find_vma(mm, khugepaged_scan.address); progress++; - for (; vma; vma = vma->vm_next) { + if (unlikely(hpage_collapse_test_exit(mm))) + goto breakouterloop; + + vma_iter_init(&vmi, mm, khugepaged_scan.address); + for_each_vma(vmi, vma) { unsigned long hstart, hend; cond_resched(); - if (unlikely(khugepaged_test_exit(mm))) { + if (unlikely(hpage_collapse_test_exit(mm))) { progress++; break; } - if (!hugepage_vma_check(vma, vma->vm_flags, false, false)) { + if (!hugepage_vma_check(vma, vma->vm_flags, false, false, true)) { skip: progress++; continue; @@ -2109,9 +2245,10 @@ skip: VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); while (khugepaged_scan.address < hend) { - int ret; + bool mmap_locked = true; + cond_resched(); - if (unlikely(khugepaged_test_exit(mm))) + if (unlikely(hpage_collapse_test_exit(mm))) goto breakouterloop; VM_BUG_ON(khugepaged_scan.address < hstart || @@ -2123,19 +2260,48 @@ skip: khugepaged_scan.address); mmap_read_unlock(mm); - ret = 1; - khugepaged_scan_file(mm, file, pgoff, hpage); + *result = hpage_collapse_scan_file(mm, + khugepaged_scan.address, + file, pgoff, cc); + mmap_locked = false; fput(file); } else { - ret = khugepaged_scan_pmd(mm, vma, - khugepaged_scan.address, - hpage); + *result = hpage_collapse_scan_pmd(mm, vma, + khugepaged_scan.address, + &mmap_locked, + cc); + } + switch (*result) { + case SCAN_PTE_MAPPED_HUGEPAGE: { + pmd_t *pmd; + + *result = find_pmd_or_thp_or_none(mm, + khugepaged_scan.address, + &pmd); + if (*result != SCAN_SUCCEED) + break; + if (!khugepaged_add_pte_mapped_thp(mm, + khugepaged_scan.address)) + break; + } fallthrough; + case SCAN_SUCCEED: + ++khugepaged_pages_collapsed; + break; + default: + break; } + /* move to next address */ khugepaged_scan.address += HPAGE_PMD_SIZE; progress += HPAGE_PMD_NR; - if (ret) - /* we released mmap_lock so break loop */ + if (!mmap_locked) + /* + * We released mmap_lock so break loop. Note + * that we drop mmap_lock before all hugepage + * allocations, so if allocation fails, we are + * guaranteed to break here and report the + * correct result back to caller. + */ goto breakouterloop_mmap_lock; if (progress >= pages) goto breakouterloop; @@ -2151,16 +2317,17 @@ breakouterloop_mmap_lock: * Release the current mm_slot if this mm is about to die, or * if we scanned all vmas of this mm. */ - if (khugepaged_test_exit(mm) || !vma) { + if (hpage_collapse_test_exit(mm) || !vma) { /* * Make sure that if mm_users is reaching zero while * khugepaged runs here, khugepaged_exit will find * mm_slot not pointing to the exiting mm. */ - if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) { - khugepaged_scan.mm_slot = list_entry( - mm_slot->mm_node.next, - struct mm_slot, mm_node); + if (slot->mm_node.next != &khugepaged_scan.mm_head) { + slot = list_entry(slot->mm_node.next, + struct mm_slot, mm_node); + khugepaged_scan.mm_slot = + mm_slot_entry(slot, struct khugepaged_mm_slot, slot); khugepaged_scan.address = 0; } else { khugepaged_scan.mm_slot = NULL; @@ -2185,19 +2352,16 @@ static int khugepaged_wait_event(void) kthread_should_stop(); } -static void khugepaged_do_scan(void) +static void khugepaged_do_scan(struct collapse_control *cc) { - struct page *hpage = NULL; unsigned int progress = 0, pass_through_head = 0; unsigned int pages = READ_ONCE(khugepaged_pages_to_scan); bool wait = true; + int result = SCAN_SUCCEED; lru_add_drain_all(); - while (progress < pages) { - if (!khugepaged_prealloc_page(&hpage, &wait)) - break; - + while (true) { cond_resched(); if (unlikely(kthread_should_stop() || try_to_freeze())) @@ -2209,14 +2373,25 @@ static void khugepaged_do_scan(void) if (khugepaged_has_work() && pass_through_head < 2) progress += khugepaged_scan_mm_slot(pages - progress, - &hpage); + &result, cc); else progress = pages; spin_unlock(&khugepaged_mm_lock); - } - if (!IS_ERR_OR_NULL(hpage)) - put_page(hpage); + if (progress >= pages) + break; + + if (result == SCAN_ALLOC_HUGE_PAGE_FAIL) { + /* + * If fail to allocate the first time, try to sleep for + * a while. When hit again, cancel the scan. + */ + if (!wait) + break; + wait = false; + khugepaged_alloc_sleep(); + } + } } static bool khugepaged_should_wakeup(void) @@ -2247,13 +2422,13 @@ static void khugepaged_wait_work(void) static int khugepaged(void *none) { - struct mm_slot *mm_slot; + struct khugepaged_mm_slot *mm_slot; set_freezable(); set_user_nice(current, MAX_NICE); while (!kthread_should_stop()) { - khugepaged_do_scan(); + khugepaged_do_scan(&khugepaged_collapse_control); khugepaged_wait_work(); } @@ -2352,3 +2527,140 @@ void khugepaged_min_free_kbytes_update(void) set_recommended_min_free_kbytes(); mutex_unlock(&khugepaged_mutex); } + +static int madvise_collapse_errno(enum scan_result r) +{ + /* + * MADV_COLLAPSE breaks from existing madvise(2) conventions to provide + * actionable feedback to caller, so they may take an appropriate + * fallback measure depending on the nature of the failure. + */ + switch (r) { + case SCAN_ALLOC_HUGE_PAGE_FAIL: + return -ENOMEM; + case SCAN_CGROUP_CHARGE_FAIL: + return -EBUSY; + /* Resource temporary unavailable - trying again might succeed */ + case SCAN_PAGE_LOCK: + case SCAN_PAGE_LRU: + case SCAN_DEL_PAGE_LRU: + return -EAGAIN; + /* + * Other: Trying again likely not to succeed / error intrinsic to + * specified memory range. khugepaged likely won't be able to collapse + * either. + */ + default: + return -EINVAL; + } +} + +int madvise_collapse(struct vm_area_struct *vma, struct vm_area_struct **prev, + unsigned long start, unsigned long end) +{ + struct collapse_control *cc; + struct mm_struct *mm = vma->vm_mm; + unsigned long hstart, hend, addr; + int thps = 0, last_fail = SCAN_FAIL; + bool mmap_locked = true; + + BUG_ON(vma->vm_start > start); + BUG_ON(vma->vm_end < end); + + *prev = vma; + + if (!hugepage_vma_check(vma, vma->vm_flags, false, false, false)) + return -EINVAL; + + cc = kmalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + cc->is_khugepaged = false; + cc->last_target_node = NUMA_NO_NODE; + + mmgrab(mm); + lru_add_drain_all(); + + hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; + hend = end & HPAGE_PMD_MASK; + + for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) { + int result = SCAN_FAIL; + + if (!mmap_locked) { + cond_resched(); + mmap_read_lock(mm); + mmap_locked = true; + result = hugepage_vma_revalidate(mm, addr, false, &vma, + cc); + if (result != SCAN_SUCCEED) { + last_fail = result; + goto out_nolock; + } + + hend = vma->vm_end & HPAGE_PMD_MASK; + } + mmap_assert_locked(mm); + memset(cc->node_load, 0, sizeof(cc->node_load)); + if (IS_ENABLED(CONFIG_SHMEM) && vma->vm_file) { + struct file *file = get_file(vma->vm_file); + pgoff_t pgoff = linear_page_index(vma, addr); + + mmap_read_unlock(mm); + mmap_locked = false; + result = hpage_collapse_scan_file(mm, addr, file, pgoff, + cc); + fput(file); + } else { + result = hpage_collapse_scan_pmd(mm, vma, addr, + &mmap_locked, cc); + } + if (!mmap_locked) + *prev = NULL; /* Tell caller we dropped mmap_lock */ + +handle_result: + switch (result) { + case SCAN_SUCCEED: + case SCAN_PMD_MAPPED: + ++thps; + break; + case SCAN_PTE_MAPPED_HUGEPAGE: + BUG_ON(mmap_locked); + BUG_ON(*prev); + mmap_write_lock(mm); + result = collapse_pte_mapped_thp(mm, addr, true); + mmap_write_unlock(mm); + goto handle_result; + /* Whitelisted set of results where continuing OK */ + case SCAN_PMD_NULL: + case SCAN_PTE_NON_PRESENT: + case SCAN_PTE_UFFD_WP: + case SCAN_PAGE_RO: + case SCAN_LACK_REFERENCED_PAGE: + case SCAN_PAGE_NULL: + case SCAN_PAGE_COUNT: + case SCAN_PAGE_LOCK: + case SCAN_PAGE_COMPOUND: + case SCAN_PAGE_LRU: + case SCAN_DEL_PAGE_LRU: + last_fail = result; + break; + default: + last_fail = result; + /* Other error, exit */ + goto out_maybelock; + } + } + +out_maybelock: + /* Caller expects us to hold mmap_lock on return */ + if (!mmap_locked) + mmap_read_lock(mm); +out_nolock: + mmap_assert_locked(mm); + mmdrop(mm); + kfree(cc); + + return thps == ((hend - hstart) >> HPAGE_PMD_SHIFT) ? 0 + : madvise_collapse_errno(last_fail); +} diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1eddc0132f7f56563dc5130eb1e4c2247d019a84..37af2dc8dac933510ba8e94f90a48dcc124711a7 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -604,9 +604,8 @@ static int __save_stack_trace(unsigned long *trace) * memory block and add it to the object_list and object_tree_root (or * object_phys_tree_root). */ -static struct kmemleak_object *__create_object(unsigned long ptr, size_t size, - int min_count, gfp_t gfp, - bool is_phys) +static void __create_object(unsigned long ptr, size_t size, + int min_count, gfp_t gfp, bool is_phys) { unsigned long flags; struct kmemleak_object *object, *parent; @@ -618,7 +617,7 @@ static struct kmemleak_object *__create_object(unsigned long ptr, size_t size, if (!object) { pr_warn("Cannot allocate a kmemleak_object structure\n"); kmemleak_disable(); - return NULL; + return; } INIT_LIST_HEAD(&object->object_list); @@ -687,7 +686,6 @@ static struct kmemleak_object *__create_object(unsigned long ptr, size_t size, */ dump_object_info(parent); kmem_cache_free(object_cache, object); - object = NULL; goto out; } } @@ -698,21 +696,20 @@ static struct kmemleak_object *__create_object(unsigned long ptr, size_t size, list_add_tail_rcu(&object->object_list, &object_list); out: raw_spin_unlock_irqrestore(&kmemleak_lock, flags); - return object; } /* Create kmemleak object which allocated with virtual address. */ -static struct kmemleak_object *create_object(unsigned long ptr, size_t size, - int min_count, gfp_t gfp) +static void create_object(unsigned long ptr, size_t size, + int min_count, gfp_t gfp) { - return __create_object(ptr, size, min_count, gfp, false); + __create_object(ptr, size, min_count, gfp, false); } /* Create kmemleak object which allocated with physical address. */ -static struct kmemleak_object *create_object_phys(unsigned long ptr, size_t size, - int min_count, gfp_t gfp) +static void create_object_phys(unsigned long ptr, size_t size, + int min_count, gfp_t gfp) { - return __create_object(ptr, size, min_count, gfp, true); + __create_object(ptr, size, min_count, gfp, true); } /* diff --git a/mm/kmsan/Makefile b/mm/kmsan/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..98eab2856626fa1b8f802dab3c1e620007478fe6 --- /dev/null +++ b/mm/kmsan/Makefile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for KernelMemorySanitizer (KMSAN). +# +# +obj-y := core.o instrumentation.o init.o hooks.o report.o shadow.o + +KMSAN_SANITIZE := n +KCOV_INSTRUMENT := n +UBSAN_SANITIZE := n + +# Disable instrumentation of KMSAN runtime with other tools. +CC_FLAGS_KMSAN_RUNTIME := -fno-stack-protector +CC_FLAGS_KMSAN_RUNTIME += $(call cc-option,-fno-conserve-stack) +CC_FLAGS_KMSAN_RUNTIME += -DDISABLE_BRANCH_PROFILING + +CFLAGS_REMOVE.o = $(CC_FLAGS_FTRACE) + +CFLAGS_core.o := $(CC_FLAGS_KMSAN_RUNTIME) +CFLAGS_hooks.o := $(CC_FLAGS_KMSAN_RUNTIME) +CFLAGS_init.o := $(CC_FLAGS_KMSAN_RUNTIME) +CFLAGS_instrumentation.o := $(CC_FLAGS_KMSAN_RUNTIME) +CFLAGS_report.o := $(CC_FLAGS_KMSAN_RUNTIME) +CFLAGS_shadow.o := $(CC_FLAGS_KMSAN_RUNTIME) + +obj-$(CONFIG_KMSAN_KUNIT_TEST) += kmsan_test.o +KMSAN_SANITIZE_kmsan_test.o := y +CFLAGS_kmsan_test.o += $(call cc-disable-warning, uninitialized) diff --git a/mm/kmsan/core.c b/mm/kmsan/core.c new file mode 100644 index 0000000000000000000000000000000000000000..112dce135c7f6d857b7773d9adbe9d3c403ee3b5 --- /dev/null +++ b/mm/kmsan/core.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN runtime library. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../slab.h" +#include "kmsan.h" + +bool kmsan_enabled __read_mostly; + +/* + * Per-CPU KMSAN context to be used in interrupts, where current->kmsan is + * unavaliable. + */ +DEFINE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); + +void kmsan_internal_task_create(struct task_struct *task) +{ + struct kmsan_ctx *ctx = &task->kmsan_ctx; + struct thread_info *info = current_thread_info(); + + __memset(ctx, 0, sizeof(*ctx)); + ctx->allow_reporting = true; + kmsan_internal_unpoison_memory(info, sizeof(*info), false); +} + +void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags, + unsigned int poison_flags) +{ + u32 extra_bits = + kmsan_extra_bits(/*depth*/ 0, poison_flags & KMSAN_POISON_FREE); + bool checked = poison_flags & KMSAN_POISON_CHECK; + depot_stack_handle_t handle; + + handle = kmsan_save_stack_with_flags(flags, extra_bits); + kmsan_internal_set_shadow_origin(address, size, -1, handle, checked); +} + +void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked) +{ + kmsan_internal_set_shadow_origin(address, size, 0, 0, checked); +} + +depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags, + unsigned int extra) +{ + unsigned long entries[KMSAN_STACK_DEPTH]; + unsigned int nr_entries; + + nr_entries = stack_trace_save(entries, KMSAN_STACK_DEPTH, 0); + + /* Don't sleep (see might_sleep_if() in __alloc_pages_nodemask()). */ + flags &= ~__GFP_DIRECT_RECLAIM; + + return __stack_depot_save(entries, nr_entries, extra, flags, true); +} + +/* Copy the metadata following the memmove() behavior. */ +void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n) +{ + depot_stack_handle_t old_origin = 0, new_origin = 0; + int src_slots, dst_slots, i, iter, step, skip_bits; + depot_stack_handle_t *origin_src, *origin_dst; + void *shadow_src, *shadow_dst; + u32 *align_shadow_src, shadow; + bool backwards; + + shadow_dst = kmsan_get_metadata(dst, KMSAN_META_SHADOW); + if (!shadow_dst) + return; + KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(dst, n)); + + shadow_src = kmsan_get_metadata(src, KMSAN_META_SHADOW); + if (!shadow_src) { + /* + * @src is untracked: zero out destination shadow, ignore the + * origins, we're done. + */ + __memset(shadow_dst, 0, n); + return; + } + KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(src, n)); + + __memmove(shadow_dst, shadow_src, n); + + origin_dst = kmsan_get_metadata(dst, KMSAN_META_ORIGIN); + origin_src = kmsan_get_metadata(src, KMSAN_META_ORIGIN); + KMSAN_WARN_ON(!origin_dst || !origin_src); + src_slots = (ALIGN((u64)src + n, KMSAN_ORIGIN_SIZE) - + ALIGN_DOWN((u64)src, KMSAN_ORIGIN_SIZE)) / + KMSAN_ORIGIN_SIZE; + dst_slots = (ALIGN((u64)dst + n, KMSAN_ORIGIN_SIZE) - + ALIGN_DOWN((u64)dst, KMSAN_ORIGIN_SIZE)) / + KMSAN_ORIGIN_SIZE; + KMSAN_WARN_ON((src_slots < 1) || (dst_slots < 1)); + KMSAN_WARN_ON((src_slots - dst_slots > 1) || + (dst_slots - src_slots < -1)); + + backwards = dst > src; + i = backwards ? min(src_slots, dst_slots) - 1 : 0; + iter = backwards ? -1 : 1; + + align_shadow_src = + (u32 *)ALIGN_DOWN((u64)shadow_src, KMSAN_ORIGIN_SIZE); + for (step = 0; step < min(src_slots, dst_slots); step++, i += iter) { + KMSAN_WARN_ON(i < 0); + shadow = align_shadow_src[i]; + if (i == 0) { + /* + * If @src isn't aligned on KMSAN_ORIGIN_SIZE, don't + * look at the first @src % KMSAN_ORIGIN_SIZE bytes + * of the first shadow slot. + */ + skip_bits = ((u64)src % KMSAN_ORIGIN_SIZE) * 8; + shadow = (shadow >> skip_bits) << skip_bits; + } + if (i == src_slots - 1) { + /* + * If @src + n isn't aligned on + * KMSAN_ORIGIN_SIZE, don't look at the last + * (@src + n) % KMSAN_ORIGIN_SIZE bytes of the + * last shadow slot. + */ + skip_bits = (((u64)src + n) % KMSAN_ORIGIN_SIZE) * 8; + shadow = (shadow << skip_bits) >> skip_bits; + } + /* + * Overwrite the origin only if the corresponding + * shadow is nonempty. + */ + if (origin_src[i] && (origin_src[i] != old_origin) && shadow) { + old_origin = origin_src[i]; + new_origin = kmsan_internal_chain_origin(old_origin); + /* + * kmsan_internal_chain_origin() may return + * NULL, but we don't want to lose the previous + * origin value. + */ + if (!new_origin) + new_origin = old_origin; + } + if (shadow) + origin_dst[i] = new_origin; + else + origin_dst[i] = 0; + } + /* + * If dst_slots is greater than src_slots (i.e. + * dst_slots == src_slots + 1), there is an extra origin slot at the + * beginning or end of the destination buffer, for which we take the + * origin from the previous slot. + * This is only done if the part of the source shadow corresponding to + * slot is non-zero. + * + * E.g. if we copy 8 aligned bytes that are marked as uninitialized + * and have origins o111 and o222, to an unaligned buffer with offset 1, + * these two origins are copied to three origin slots, so one of then + * needs to be duplicated, depending on the copy direction (@backwards) + * + * src shadow: |uuuu|uuuu|....| + * src origin: |o111|o222|....| + * + * backwards = 0: + * dst shadow: |.uuu|uuuu|u...| + * dst origin: |....|o111|o222| - fill the empty slot with o111 + * backwards = 1: + * dst shadow: |.uuu|uuuu|u...| + * dst origin: |o111|o222|....| - fill the empty slot with o222 + */ + if (src_slots < dst_slots) { + if (backwards) { + shadow = align_shadow_src[src_slots - 1]; + skip_bits = (((u64)dst + n) % KMSAN_ORIGIN_SIZE) * 8; + shadow = (shadow << skip_bits) >> skip_bits; + if (shadow) + /* src_slots > 0, therefore dst_slots is at least 2 */ + origin_dst[dst_slots - 1] = + origin_dst[dst_slots - 2]; + } else { + shadow = align_shadow_src[0]; + skip_bits = ((u64)dst % KMSAN_ORIGIN_SIZE) * 8; + shadow = (shadow >> skip_bits) << skip_bits; + if (shadow) + origin_dst[0] = origin_dst[1]; + } + } +} + +depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id) +{ + unsigned long entries[3]; + u32 extra_bits; + int depth; + bool uaf; + + if (!id) + return id; + /* + * Make sure we have enough spare bits in @id to hold the UAF bit and + * the chain depth. + */ + BUILD_BUG_ON( + (1 << STACK_DEPOT_EXTRA_BITS) <= (KMSAN_MAX_ORIGIN_DEPTH << 1)); + + extra_bits = stack_depot_get_extra_bits(id); + depth = kmsan_depth_from_eb(extra_bits); + uaf = kmsan_uaf_from_eb(extra_bits); + + /* + * Stop chaining origins once the depth reached KMSAN_MAX_ORIGIN_DEPTH. + * This mostly happens in the case structures with uninitialized padding + * are copied around many times. Origin chains for such structures are + * usually periodic, and it does not make sense to fully store them. + */ + if (depth == KMSAN_MAX_ORIGIN_DEPTH) + return id; + + depth++; + extra_bits = kmsan_extra_bits(depth, uaf); + + entries[0] = KMSAN_CHAIN_MAGIC_ORIGIN; + entries[1] = kmsan_save_stack_with_flags(GFP_ATOMIC, 0); + entries[2] = id; + /* + * @entries is a local var in non-instrumented code, so KMSAN does not + * know it is initialized. Explicitly unpoison it to avoid false + * positives when __stack_depot_save() passes it to instrumented code. + */ + kmsan_internal_unpoison_memory(entries, sizeof(entries), false); + return __stack_depot_save(entries, ARRAY_SIZE(entries), extra_bits, + GFP_ATOMIC, true); +} + +void kmsan_internal_set_shadow_origin(void *addr, size_t size, int b, + u32 origin, bool checked) +{ + u64 address = (u64)addr; + void *shadow_start; + u32 *origin_start; + size_t pad = 0; + + KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size)); + shadow_start = kmsan_get_metadata(addr, KMSAN_META_SHADOW); + if (!shadow_start) { + /* + * kmsan_metadata_is_contiguous() is true, so either all shadow + * and origin pages are NULL, or all are non-NULL. + */ + if (checked) { + pr_err("%s: not memsetting %ld bytes starting at %px, because the shadow is NULL\n", + __func__, size, addr); + KMSAN_WARN_ON(true); + } + return; + } + __memset(shadow_start, b, size); + + if (!IS_ALIGNED(address, KMSAN_ORIGIN_SIZE)) { + pad = address % KMSAN_ORIGIN_SIZE; + address -= pad; + size += pad; + } + size = ALIGN(size, KMSAN_ORIGIN_SIZE); + origin_start = + (u32 *)kmsan_get_metadata((void *)address, KMSAN_META_ORIGIN); + + for (int i = 0; i < size / KMSAN_ORIGIN_SIZE; i++) + origin_start[i] = origin; +} + +struct page *kmsan_vmalloc_to_page_or_null(void *vaddr) +{ + struct page *page; + + if (!kmsan_internal_is_vmalloc_addr(vaddr) && + !kmsan_internal_is_module_addr(vaddr)) + return NULL; + page = vmalloc_to_page(vaddr); + if (pfn_valid(page_to_pfn(page))) + return page; + else + return NULL; +} + +void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr, + int reason) +{ + depot_stack_handle_t cur_origin = 0, new_origin = 0; + unsigned long addr64 = (unsigned long)addr; + depot_stack_handle_t *origin = NULL; + unsigned char *shadow = NULL; + int cur_off_start = -1; + int chunk_size; + size_t pos = 0; + + if (!size) + return; + KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(addr, size)); + while (pos < size) { + chunk_size = min(size - pos, + PAGE_SIZE - ((addr64 + pos) % PAGE_SIZE)); + shadow = kmsan_get_metadata((void *)(addr64 + pos), + KMSAN_META_SHADOW); + if (!shadow) { + /* + * This page is untracked. If there were uninitialized + * bytes before, report them. + */ + if (cur_origin) { + kmsan_enter_runtime(); + kmsan_report(cur_origin, addr, size, + cur_off_start, pos - 1, user_addr, + reason); + kmsan_leave_runtime(); + } + cur_origin = 0; + cur_off_start = -1; + pos += chunk_size; + continue; + } + for (int i = 0; i < chunk_size; i++) { + if (!shadow[i]) { + /* + * This byte is unpoisoned. If there were + * poisoned bytes before, report them. + */ + if (cur_origin) { + kmsan_enter_runtime(); + kmsan_report(cur_origin, addr, size, + cur_off_start, pos + i - 1, + user_addr, reason); + kmsan_leave_runtime(); + } + cur_origin = 0; + cur_off_start = -1; + continue; + } + origin = kmsan_get_metadata((void *)(addr64 + pos + i), + KMSAN_META_ORIGIN); + KMSAN_WARN_ON(!origin); + new_origin = *origin; + /* + * Encountered new origin - report the previous + * uninitialized range. + */ + if (cur_origin != new_origin) { + if (cur_origin) { + kmsan_enter_runtime(); + kmsan_report(cur_origin, addr, size, + cur_off_start, pos + i - 1, + user_addr, reason); + kmsan_leave_runtime(); + } + cur_origin = new_origin; + cur_off_start = pos + i; + } + } + pos += chunk_size; + } + KMSAN_WARN_ON(pos != size); + if (cur_origin) { + kmsan_enter_runtime(); + kmsan_report(cur_origin, addr, size, cur_off_start, pos - 1, + user_addr, reason); + kmsan_leave_runtime(); + } +} + +bool kmsan_metadata_is_contiguous(void *addr, size_t size) +{ + char *cur_shadow = NULL, *next_shadow = NULL, *cur_origin = NULL, + *next_origin = NULL; + u64 cur_addr = (u64)addr, next_addr = cur_addr + PAGE_SIZE; + depot_stack_handle_t *origin_p; + bool all_untracked = false; + + if (!size) + return true; + + /* The whole range belongs to the same page. */ + if (ALIGN_DOWN(cur_addr + size - 1, PAGE_SIZE) == + ALIGN_DOWN(cur_addr, PAGE_SIZE)) + return true; + + cur_shadow = kmsan_get_metadata((void *)cur_addr, /*is_origin*/ false); + if (!cur_shadow) + all_untracked = true; + cur_origin = kmsan_get_metadata((void *)cur_addr, /*is_origin*/ true); + if (all_untracked && cur_origin) + goto report; + + for (; next_addr < (u64)addr + size; + cur_addr = next_addr, cur_shadow = next_shadow, + cur_origin = next_origin, next_addr += PAGE_SIZE) { + next_shadow = kmsan_get_metadata((void *)next_addr, false); + next_origin = kmsan_get_metadata((void *)next_addr, true); + if (all_untracked) { + if (next_shadow || next_origin) + goto report; + if (!next_shadow && !next_origin) + continue; + } + if (((u64)cur_shadow == ((u64)next_shadow - PAGE_SIZE)) && + ((u64)cur_origin == ((u64)next_origin - PAGE_SIZE))) + continue; + goto report; + } + return true; + +report: + pr_err("%s: attempting to access two shadow page ranges.\n", __func__); + pr_err("Access of size %ld at %px.\n", size, addr); + pr_err("Addresses belonging to different ranges: %px and %px\n", + (void *)cur_addr, (void *)next_addr); + pr_err("page[0].shadow: %px, page[1].shadow: %px\n", cur_shadow, + next_shadow); + pr_err("page[0].origin: %px, page[1].origin: %px\n", cur_origin, + next_origin); + origin_p = kmsan_get_metadata(addr, KMSAN_META_ORIGIN); + if (origin_p) { + pr_err("Origin: %08x\n", *origin_p); + kmsan_print_origin(*origin_p); + } else { + pr_err("Origin: unavailable\n"); + } + return false; +} diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c new file mode 100644 index 0000000000000000000000000000000000000000..35f6b6e6a908c2844ec27cd283ec88896803159d --- /dev/null +++ b/mm/kmsan/hooks.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN hooks for kernel subsystems. + * + * These functions handle creation of KMSAN metadata for memory allocations. + * + * Copyright (C) 2018-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../internal.h" +#include "../slab.h" +#include "kmsan.h" + +/* + * Instrumented functions shouldn't be called under + * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to + * skipping effects of functions like memset() inside instrumented code. + */ + +void kmsan_task_create(struct task_struct *task) +{ + kmsan_enter_runtime(); + kmsan_internal_task_create(task); + kmsan_leave_runtime(); +} + +void kmsan_task_exit(struct task_struct *task) +{ + struct kmsan_ctx *ctx = &task->kmsan_ctx; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + ctx->allow_reporting = false; +} + +void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags) +{ + if (unlikely(object == NULL)) + return; + if (!kmsan_enabled || kmsan_in_runtime()) + return; + /* + * There's a ctor or this is an RCU cache - do nothing. The memory + * status hasn't changed since last use. + */ + if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU)) + return; + + kmsan_enter_runtime(); + if (flags & __GFP_ZERO) + kmsan_internal_unpoison_memory(object, s->object_size, + KMSAN_POISON_CHECK); + else + kmsan_internal_poison_memory(object, s->object_size, flags, + KMSAN_POISON_CHECK); + kmsan_leave_runtime(); +} + +void kmsan_slab_free(struct kmem_cache *s, void *object) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + /* RCU slabs could be legally used after free within the RCU period */ + if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))) + return; + /* + * If there's a constructor, freed memory must remain in the same state + * until the next allocation. We cannot save its state to detect + * use-after-free bugs, instead we just keep it unpoisoned. + */ + if (s->ctor) + return; + kmsan_enter_runtime(); + kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL, + KMSAN_POISON_CHECK | KMSAN_POISON_FREE); + kmsan_leave_runtime(); +} + +void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags) +{ + if (unlikely(ptr == NULL)) + return; + if (!kmsan_enabled || kmsan_in_runtime()) + return; + kmsan_enter_runtime(); + if (flags & __GFP_ZERO) + kmsan_internal_unpoison_memory((void *)ptr, size, + /*checked*/ true); + else + kmsan_internal_poison_memory((void *)ptr, size, flags, + KMSAN_POISON_CHECK); + kmsan_leave_runtime(); +} + +void kmsan_kfree_large(const void *ptr) +{ + struct page *page; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + kmsan_enter_runtime(); + page = virt_to_head_page((void *)ptr); + KMSAN_WARN_ON(ptr != page_address(page)); + kmsan_internal_poison_memory((void *)ptr, + PAGE_SIZE << compound_order(page), + GFP_KERNEL, + KMSAN_POISON_CHECK | KMSAN_POISON_FREE); + kmsan_leave_runtime(); +} + +static unsigned long vmalloc_shadow(unsigned long addr) +{ + return (unsigned long)kmsan_get_metadata((void *)addr, + KMSAN_META_SHADOW); +} + +static unsigned long vmalloc_origin(unsigned long addr) +{ + return (unsigned long)kmsan_get_metadata((void *)addr, + KMSAN_META_ORIGIN); +} + +void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end) +{ + __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end)); + __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end)); + flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); + flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); +} + +/* + * This function creates new shadow/origin pages for the physical pages mapped + * into the virtual memory. If those physical pages already had shadow/origin, + * those are ignored. + */ +void kmsan_ioremap_page_range(unsigned long start, unsigned long end, + phys_addr_t phys_addr, pgprot_t prot, + unsigned int page_shift) +{ + gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO; + struct page *shadow, *origin; + unsigned long off = 0; + int nr; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + nr = (end - start) / PAGE_SIZE; + kmsan_enter_runtime(); + for (int i = 0; i < nr; i++, off += PAGE_SIZE) { + shadow = alloc_pages(gfp_mask, 1); + origin = alloc_pages(gfp_mask, 1); + __vmap_pages_range_noflush( + vmalloc_shadow(start + off), + vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow, + PAGE_SHIFT); + __vmap_pages_range_noflush( + vmalloc_origin(start + off), + vmalloc_origin(start + off + PAGE_SIZE), prot, &origin, + PAGE_SHIFT); + } + flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); + flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); + kmsan_leave_runtime(); +} + +void kmsan_iounmap_page_range(unsigned long start, unsigned long end) +{ + unsigned long v_shadow, v_origin; + struct page *shadow, *origin; + int nr; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + nr = (end - start) / PAGE_SIZE; + kmsan_enter_runtime(); + v_shadow = (unsigned long)vmalloc_shadow(start); + v_origin = (unsigned long)vmalloc_origin(start); + for (int i = 0; i < nr; + i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) { + shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow); + origin = kmsan_vmalloc_to_page_or_null((void *)v_origin); + __vunmap_range_noflush(v_shadow, vmalloc_shadow(end)); + __vunmap_range_noflush(v_origin, vmalloc_origin(end)); + if (shadow) + __free_pages(shadow, 1); + if (origin) + __free_pages(origin, 1); + } + flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); + flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); + kmsan_leave_runtime(); +} + +void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy, + size_t left) +{ + unsigned long ua_flags; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + /* + * At this point we've copied the memory already. It's hard to check it + * before copying, as the size of actually copied buffer is unknown. + */ + + /* copy_to_user() may copy zero bytes. No need to check. */ + if (!to_copy) + return; + /* Or maybe copy_to_user() failed to copy anything. */ + if (to_copy <= left) + return; + + ua_flags = user_access_save(); + if ((u64)to < TASK_SIZE) { + /* This is a user memory access, check it. */ + kmsan_internal_check_memory((void *)from, to_copy - left, to, + REASON_COPY_TO_USER); + } else { + /* Otherwise this is a kernel memory access. This happens when a + * compat syscall passes an argument allocated on the kernel + * stack to a real syscall. + * Don't check anything, just copy the shadow of the copied + * bytes. + */ + kmsan_internal_memmove_metadata((void *)to, (void *)from, + to_copy - left); + } + user_access_restore(ua_flags); +} +EXPORT_SYMBOL(kmsan_copy_to_user); + +/* Helper function to check an URB. */ +void kmsan_handle_urb(const struct urb *urb, bool is_out) +{ + if (!urb) + return; + if (is_out) + kmsan_internal_check_memory(urb->transfer_buffer, + urb->transfer_buffer_length, + /*user_addr*/ 0, REASON_SUBMIT_URB); + else + kmsan_internal_unpoison_memory(urb->transfer_buffer, + urb->transfer_buffer_length, + /*checked*/ false); +} + +static void kmsan_handle_dma_page(const void *addr, size_t size, + enum dma_data_direction dir) +{ + switch (dir) { + case DMA_BIDIRECTIONAL: + kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0, + REASON_ANY); + kmsan_internal_unpoison_memory((void *)addr, size, + /*checked*/ false); + break; + case DMA_TO_DEVICE: + kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0, + REASON_ANY); + break; + case DMA_FROM_DEVICE: + kmsan_internal_unpoison_memory((void *)addr, size, + /*checked*/ false); + break; + case DMA_NONE: + break; + } +} + +/* Helper function to handle DMA data transfers. */ +void kmsan_handle_dma(struct page *page, size_t offset, size_t size, + enum dma_data_direction dir) +{ + u64 page_offset, to_go, addr; + + if (PageHighMem(page)) + return; + addr = (u64)page_address(page) + offset; + /* + * The kernel may occasionally give us adjacent DMA pages not belonging + * to the same allocation. Process them separately to avoid triggering + * internal KMSAN checks. + */ + while (size > 0) { + page_offset = addr % PAGE_SIZE; + to_go = min(PAGE_SIZE - page_offset, (u64)size); + kmsan_handle_dma_page((void *)addr, to_go, dir); + addr += to_go; + size -= to_go; + } +} + +void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, + enum dma_data_direction dir) +{ + struct scatterlist *item; + int i; + + for_each_sg(sg, item, nents, i) + kmsan_handle_dma(sg_page(item), item->offset, item->length, + dir); +} + +/* Functions from kmsan-checks.h follow. */ +void kmsan_poison_memory(const void *address, size_t size, gfp_t flags) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + kmsan_enter_runtime(); + /* The users may want to poison/unpoison random memory. */ + kmsan_internal_poison_memory((void *)address, size, flags, + KMSAN_POISON_NOCHECK); + kmsan_leave_runtime(); +} +EXPORT_SYMBOL(kmsan_poison_memory); + +void kmsan_unpoison_memory(const void *address, size_t size) +{ + unsigned long ua_flags; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + ua_flags = user_access_save(); + kmsan_enter_runtime(); + /* The users may want to poison/unpoison random memory. */ + kmsan_internal_unpoison_memory((void *)address, size, + KMSAN_POISON_NOCHECK); + kmsan_leave_runtime(); + user_access_restore(ua_flags); +} +EXPORT_SYMBOL(kmsan_unpoison_memory); + +/* + * Version of kmsan_unpoison_memory() that can be called from within the KMSAN + * runtime. + * + * Non-instrumented IRQ entry functions receive struct pt_regs from assembly + * code. Those regs need to be unpoisoned, otherwise using them will result in + * false positives. + * Using kmsan_unpoison_memory() is not an option in entry code, because the + * return value of in_task() is inconsistent - as a result, certain calls to + * kmsan_unpoison_memory() are ignored. kmsan_unpoison_entry_regs() ensures that + * the registers are unpoisoned even if kmsan_in_runtime() is true in the early + * entry code. + */ +void kmsan_unpoison_entry_regs(const struct pt_regs *regs) +{ + unsigned long ua_flags; + + if (!kmsan_enabled) + return; + + ua_flags = user_access_save(); + kmsan_internal_unpoison_memory((void *)regs, sizeof(*regs), + KMSAN_POISON_NOCHECK); + user_access_restore(ua_flags); +} + +void kmsan_check_memory(const void *addr, size_t size) +{ + if (!kmsan_enabled) + return; + return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0, + REASON_ANY); +} +EXPORT_SYMBOL(kmsan_check_memory); diff --git a/mm/kmsan/init.c b/mm/kmsan/init.c new file mode 100644 index 0000000000000000000000000000000000000000..7fb794242fad01cab93a654003af0b9a99381359 --- /dev/null +++ b/mm/kmsan/init.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN initialization routines. + * + * Copyright (C) 2017-2021 Google LLC + * Author: Alexander Potapenko + * + */ + +#include "kmsan.h" + +#include +#include +#include + +#include "../internal.h" + +#define NUM_FUTURE_RANGES 128 +struct start_end_pair { + u64 start, end; +}; + +static struct start_end_pair start_end_pairs[NUM_FUTURE_RANGES] __initdata; +static int future_index __initdata; + +/* + * Record a range of memory for which the metadata pages will be created once + * the page allocator becomes available. + */ +static void __init kmsan_record_future_shadow_range(void *start, void *end) +{ + u64 nstart = (u64)start, nend = (u64)end, cstart, cend; + bool merged = false; + + KMSAN_WARN_ON(future_index == NUM_FUTURE_RANGES); + KMSAN_WARN_ON((nstart >= nend) || !nstart || !nend); + nstart = ALIGN_DOWN(nstart, PAGE_SIZE); + nend = ALIGN(nend, PAGE_SIZE); + + /* + * Scan the existing ranges to see if any of them overlaps with + * [start, end). In that case, merge the two ranges instead of + * creating a new one. + * The number of ranges is less than 20, so there is no need to organize + * them into a more intelligent data structure. + */ + for (int i = 0; i < future_index; i++) { + cstart = start_end_pairs[i].start; + cend = start_end_pairs[i].end; + if ((cstart < nstart && cend < nstart) || + (cstart > nend && cend > nend)) + /* ranges are disjoint - do not merge */ + continue; + start_end_pairs[i].start = min(nstart, cstart); + start_end_pairs[i].end = max(nend, cend); + merged = true; + break; + } + if (merged) + return; + start_end_pairs[future_index].start = nstart; + start_end_pairs[future_index].end = nend; + future_index++; +} + +/* + * Initialize the shadow for existing mappings during kernel initialization. + * These include kernel text/data sections, NODE_DATA and future ranges + * registered while creating other data (e.g. percpu). + * + * Allocations via memblock can be only done before slab is initialized. + */ +void __init kmsan_init_shadow(void) +{ + const size_t nd_size = roundup(sizeof(pg_data_t), PAGE_SIZE); + phys_addr_t p_start, p_end; + u64 loop; + int nid; + + for_each_reserved_mem_range(loop, &p_start, &p_end) + kmsan_record_future_shadow_range(phys_to_virt(p_start), + phys_to_virt(p_end)); + /* Allocate shadow for .data */ + kmsan_record_future_shadow_range(_sdata, _edata); + + for_each_online_node(nid) + kmsan_record_future_shadow_range( + NODE_DATA(nid), (char *)NODE_DATA(nid) + nd_size); + + for (int i = 0; i < future_index; i++) + kmsan_init_alloc_meta_for_range( + (void *)start_end_pairs[i].start, + (void *)start_end_pairs[i].end); +} + +struct metadata_page_pair { + struct page *shadow, *origin; +}; +static struct metadata_page_pair held_back[MAX_ORDER] __initdata; + +/* + * Eager metadata allocation. When the memblock allocator is freeing pages to + * pagealloc, we use 2/3 of them as metadata for the remaining 1/3. + * We store the pointers to the returned blocks of pages in held_back[] grouped + * by their order: when kmsan_memblock_free_pages() is called for the first + * time with a certain order, it is reserved as a shadow block, for the second + * time - as an origin block. On the third time the incoming block receives its + * shadow and origin ranges from the previously saved shadow and origin blocks, + * after which held_back[order] can be used again. + * + * At the very end there may be leftover blocks in held_back[]. They are + * collected later by kmsan_memblock_discard(). + */ +bool kmsan_memblock_free_pages(struct page *page, unsigned int order) +{ + struct page *shadow, *origin; + + if (!held_back[order].shadow) { + held_back[order].shadow = page; + return false; + } + if (!held_back[order].origin) { + held_back[order].origin = page; + return false; + } + shadow = held_back[order].shadow; + origin = held_back[order].origin; + kmsan_setup_meta(page, shadow, origin, order); + + held_back[order].shadow = NULL; + held_back[order].origin = NULL; + return true; +} + +#define MAX_BLOCKS 8 +struct smallstack { + struct page *items[MAX_BLOCKS]; + int index; + int order; +}; + +static struct smallstack collect = { + .index = 0, + .order = MAX_ORDER, +}; + +static void smallstack_push(struct smallstack *stack, struct page *pages) +{ + KMSAN_WARN_ON(stack->index == MAX_BLOCKS); + stack->items[stack->index] = pages; + stack->index++; +} +#undef MAX_BLOCKS + +static struct page *smallstack_pop(struct smallstack *stack) +{ + struct page *ret; + + KMSAN_WARN_ON(stack->index == 0); + stack->index--; + ret = stack->items[stack->index]; + stack->items[stack->index] = NULL; + return ret; +} + +static void do_collection(void) +{ + struct page *page, *shadow, *origin; + + while (collect.index >= 3) { + page = smallstack_pop(&collect); + shadow = smallstack_pop(&collect); + origin = smallstack_pop(&collect); + kmsan_setup_meta(page, shadow, origin, collect.order); + __free_pages_core(page, collect.order); + } +} + +static void collect_split(void) +{ + struct smallstack tmp = { + .order = collect.order - 1, + .index = 0, + }; + struct page *page; + + if (!collect.order) + return; + while (collect.index) { + page = smallstack_pop(&collect); + smallstack_push(&tmp, &page[0]); + smallstack_push(&tmp, &page[1 << tmp.order]); + } + __memcpy(&collect, &tmp, sizeof(tmp)); +} + +/* + * Memblock is about to go away. Split the page blocks left over in held_back[] + * and return 1/3 of that memory to the system. + */ +static void kmsan_memblock_discard(void) +{ + /* + * For each order=N: + * - push held_back[N].shadow and .origin to @collect; + * - while there are >= 3 elements in @collect, do garbage collection: + * - pop 3 ranges from @collect; + * - use two of them as shadow and origin for the third one; + * - repeat; + * - split each remaining element from @collect into 2 ranges of + * order=N-1, + * - repeat. + */ + collect.order = MAX_ORDER - 1; + for (int i = MAX_ORDER - 1; i >= 0; i--) { + if (held_back[i].shadow) + smallstack_push(&collect, held_back[i].shadow); + if (held_back[i].origin) + smallstack_push(&collect, held_back[i].origin); + held_back[i].shadow = NULL; + held_back[i].origin = NULL; + do_collection(); + collect_split(); + } +} + +void __init kmsan_init_runtime(void) +{ + /* Assuming current is init_task */ + kmsan_internal_task_create(current); + kmsan_memblock_discard(); + pr_info("Starting KernelMemorySanitizer\n"); + pr_info("ATTENTION: KMSAN is a debugging tool! Do not use it on production machines!\n"); + kmsan_enabled = true; +} diff --git a/mm/kmsan/instrumentation.c b/mm/kmsan/instrumentation.c new file mode 100644 index 0000000000000000000000000000000000000000..280d15413268494c969342e61e522f47e1aa0c69 --- /dev/null +++ b/mm/kmsan/instrumentation.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN compiler API. + * + * This file implements __msan_XXX hooks that Clang inserts into the code + * compiled with -fsanitize=kernel-memory. + * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN + * instrumentation works. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#include "kmsan.h" +#include +#include +#include + +static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store) +{ + if ((u64)addr < TASK_SIZE) + return true; + if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW)) + return true; + return false; +} + +static inline struct shadow_origin_ptr +get_shadow_origin_ptr(void *addr, u64 size, bool store) +{ + unsigned long ua_flags = user_access_save(); + struct shadow_origin_ptr ret; + + ret = kmsan_get_shadow_origin_ptr(addr, size, store); + user_access_restore(ua_flags); + return ret; +} + +/* Get shadow and origin pointers for a memory load with non-standard size. */ +struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr, + uintptr_t size) +{ + return get_shadow_origin_ptr(addr, size, /*store*/ false); +} +EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n); + +/* Get shadow and origin pointers for a memory store with non-standard size. */ +struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr, + uintptr_t size) +{ + return get_shadow_origin_ptr(addr, size, /*store*/ true); +} +EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n); + +/* + * Declare functions that obtain shadow/origin pointers for loads and stores + * with fixed size. + */ +#define DECLARE_METADATA_PTR_GETTER(size) \ + struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size( \ + void *addr) \ + { \ + return get_shadow_origin_ptr(addr, size, /*store*/ false); \ + } \ + EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size); \ + struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size( \ + void *addr) \ + { \ + return get_shadow_origin_ptr(addr, size, /*store*/ true); \ + } \ + EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size) + +DECLARE_METADATA_PTR_GETTER(1); +DECLARE_METADATA_PTR_GETTER(2); +DECLARE_METADATA_PTR_GETTER(4); +DECLARE_METADATA_PTR_GETTER(8); + +/* + * Handle a memory store performed by inline assembly. KMSAN conservatively + * attempts to unpoison the outputs of asm() directives to prevent false + * positives caused by missed stores. + */ +void __msan_instrument_asm_store(void *addr, uintptr_t size) +{ + unsigned long ua_flags; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + ua_flags = user_access_save(); + /* + * Most of the accesses are below 32 bytes. The two exceptions so far + * are clwb() (64 bytes) and FPU state (512 bytes). + * It's unlikely that the assembly will touch more than 512 bytes. + */ + if (size > 512) { + WARN_ONCE(1, "assembly store size too big: %ld\n", size); + size = 8; + } + if (is_bad_asm_addr(addr, size, /*is_store*/ true)) { + user_access_restore(ua_flags); + return; + } + kmsan_enter_runtime(); + /* Unpoisoning the memory on best effort. */ + kmsan_internal_unpoison_memory(addr, size, /*checked*/ false); + kmsan_leave_runtime(); + user_access_restore(ua_flags); +} +EXPORT_SYMBOL(__msan_instrument_asm_store); + +/* + * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset + * intrinsics with calls to respective __msan_ functions. We use + * get_param0_metadata() and set_retval_metadata() to store the shadow/origin + * values for the destination argument of these functions and use them for the + * functions' return values. + */ +static inline void get_param0_metadata(u64 *shadow, + depot_stack_handle_t *origin) +{ + struct kmsan_ctx *ctx = kmsan_get_context(); + + *shadow = *(u64 *)(ctx->cstate.param_tls); + *origin = ctx->cstate.param_origin_tls[0]; +} + +static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin) +{ + struct kmsan_ctx *ctx = kmsan_get_context(); + + *(u64 *)(ctx->cstate.retval_tls) = shadow; + ctx->cstate.retval_origin_tls = origin; +} + +/* Handle llvm.memmove intrinsic. */ +void *__msan_memmove(void *dst, const void *src, uintptr_t n) +{ + depot_stack_handle_t origin; + void *result; + u64 shadow; + + get_param0_metadata(&shadow, &origin); + result = __memmove(dst, src, n); + if (!n) + /* Some people call memmove() with zero length. */ + return result; + if (!kmsan_enabled || kmsan_in_runtime()) + return result; + + kmsan_enter_runtime(); + kmsan_internal_memmove_metadata(dst, (void *)src, n); + kmsan_leave_runtime(); + + set_retval_metadata(shadow, origin); + return result; +} +EXPORT_SYMBOL(__msan_memmove); + +/* Handle llvm.memcpy intrinsic. */ +void *__msan_memcpy(void *dst, const void *src, uintptr_t n) +{ + depot_stack_handle_t origin; + void *result; + u64 shadow; + + get_param0_metadata(&shadow, &origin); + result = __memcpy(dst, src, n); + if (!n) + /* Some people call memcpy() with zero length. */ + return result; + + if (!kmsan_enabled || kmsan_in_runtime()) + return result; + + kmsan_enter_runtime(); + /* Using memmove instead of memcpy doesn't affect correctness. */ + kmsan_internal_memmove_metadata(dst, (void *)src, n); + kmsan_leave_runtime(); + + set_retval_metadata(shadow, origin); + return result; +} +EXPORT_SYMBOL(__msan_memcpy); + +/* Handle llvm.memset intrinsic. */ +void *__msan_memset(void *dst, int c, uintptr_t n) +{ + depot_stack_handle_t origin; + void *result; + u64 shadow; + + get_param0_metadata(&shadow, &origin); + result = __memset(dst, c, n); + if (!kmsan_enabled || kmsan_in_runtime()) + return result; + + kmsan_enter_runtime(); + /* + * Clang doesn't pass parameter metadata here, so it is impossible to + * use shadow of @c to set up the shadow for @dst. + */ + kmsan_internal_unpoison_memory(dst, n, /*checked*/ false); + kmsan_leave_runtime(); + + set_retval_metadata(shadow, origin); + return result; +} +EXPORT_SYMBOL(__msan_memset); + +/* + * Create a new origin from an old one. This is done when storing an + * uninitialized value to memory. When reporting an error, KMSAN unrolls and + * prints the whole chain of stores that preceded the use of this value. + */ +depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin) +{ + depot_stack_handle_t ret = 0; + unsigned long ua_flags; + + if (!kmsan_enabled || kmsan_in_runtime()) + return ret; + + ua_flags = user_access_save(); + + /* Creating new origins may allocate memory. */ + kmsan_enter_runtime(); + ret = kmsan_internal_chain_origin(origin); + kmsan_leave_runtime(); + user_access_restore(ua_flags); + return ret; +} +EXPORT_SYMBOL(__msan_chain_origin); + +/* Poison a local variable when entering a function. */ +void __msan_poison_alloca(void *address, uintptr_t size, char *descr) +{ + depot_stack_handle_t handle; + unsigned long entries[4]; + unsigned long ua_flags; + + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + ua_flags = user_access_save(); + entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN; + entries[1] = (u64)descr; + entries[2] = (u64)__builtin_return_address(0); + /* + * With frame pointers enabled, it is possible to quickly fetch the + * second frame of the caller stack without calling the unwinder. + * Without them, simply do not bother. + */ + if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) + entries[3] = (u64)__builtin_return_address(1); + else + entries[3] = 0; + + /* stack_depot_save() may allocate memory. */ + kmsan_enter_runtime(); + handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC); + kmsan_leave_runtime(); + + kmsan_internal_set_shadow_origin(address, size, -1, handle, + /*checked*/ true); + user_access_restore(ua_flags); +} +EXPORT_SYMBOL(__msan_poison_alloca); + +/* Unpoison a local variable. */ +void __msan_unpoison_alloca(void *address, uintptr_t size) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + + kmsan_enter_runtime(); + kmsan_internal_unpoison_memory(address, size, /*checked*/ true); + kmsan_leave_runtime(); +} +EXPORT_SYMBOL(__msan_unpoison_alloca); + +/* + * Report that an uninitialized value with the given origin was used in a way + * that constituted undefined behavior. + */ +void __msan_warning(u32 origin) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + kmsan_enter_runtime(); + kmsan_report(origin, /*address*/ 0, /*size*/ 0, + /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ 0, + REASON_ANY); + kmsan_leave_runtime(); +} +EXPORT_SYMBOL(__msan_warning); + +/* + * At the beginning of an instrumented function, obtain the pointer to + * `struct kmsan_context_state` holding the metadata for function parameters. + */ +struct kmsan_context_state *__msan_get_context_state(void) +{ + return &kmsan_get_context()->cstate; +} +EXPORT_SYMBOL(__msan_get_context_state); diff --git a/mm/kmsan/kmsan.h b/mm/kmsan/kmsan.h new file mode 100644 index 0000000000000000000000000000000000000000..7019c46d33a745be541f78d7e270cc769677eafe --- /dev/null +++ b/mm/kmsan/kmsan.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Functions used by the KMSAN runtime. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#ifndef __MM_KMSAN_KMSAN_H +#define __MM_KMSAN_KMSAN_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#define KMSAN_ALLOCA_MAGIC_ORIGIN 0xabcd0100 +#define KMSAN_CHAIN_MAGIC_ORIGIN 0xabcd0200 + +#define KMSAN_POISON_NOCHECK 0x0 +#define KMSAN_POISON_CHECK 0x1 +#define KMSAN_POISON_FREE 0x2 + +#define KMSAN_ORIGIN_SIZE 4 +#define KMSAN_MAX_ORIGIN_DEPTH 7 + +#define KMSAN_STACK_DEPTH 64 + +#define KMSAN_META_SHADOW (false) +#define KMSAN_META_ORIGIN (true) + +extern bool kmsan_enabled; +extern int panic_on_kmsan; + +/* + * KMSAN performs a lot of consistency checks that are currently enabled by + * default. BUG_ON is normally discouraged in the kernel, unless used for + * debugging, but KMSAN itself is a debugging tool, so it makes little sense to + * recover if something goes wrong. + */ +#define KMSAN_WARN_ON(cond) \ + ({ \ + const bool __cond = WARN_ON(cond); \ + if (unlikely(__cond)) { \ + WRITE_ONCE(kmsan_enabled, false); \ + if (panic_on_kmsan) { \ + /* Can't call panic() here because */ \ + /* of uaccess checks. */ \ + BUG(); \ + } \ + } \ + __cond; \ + }) + +/* + * A pair of metadata pointers to be returned by the instrumentation functions. + */ +struct shadow_origin_ptr { + void *shadow, *origin; +}; + +struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size, + bool store); +void *kmsan_get_metadata(void *addr, bool is_origin); +void __init kmsan_init_alloc_meta_for_range(void *start, void *end); + +enum kmsan_bug_reason { + REASON_ANY, + REASON_COPY_TO_USER, + REASON_SUBMIT_URB, +}; + +void kmsan_print_origin(depot_stack_handle_t origin); + +/** + * kmsan_report() - Report a use of uninitialized value. + * @origin: Stack ID of the uninitialized value. + * @address: Address at which the memory access happens. + * @size: Memory access size. + * @off_first: Offset (from @address) of the first byte to be reported. + * @off_last: Offset (from @address) of the last byte to be reported. + * @user_addr: When non-NULL, denotes the userspace address to which the kernel + * is leaking data. + * @reason: Error type from enum kmsan_bug_reason. + * + * kmsan_report() prints an error message for a consequent group of bytes + * sharing the same origin. If an uninitialized value is used in a comparison, + * this function is called once without specifying the addresses. When checking + * a memory range, KMSAN may call kmsan_report() multiple times with the same + * @address, @size, @user_addr and @reason, but different @off_first and + * @off_last corresponding to different @origin values. + */ +void kmsan_report(depot_stack_handle_t origin, void *address, int size, + int off_first, int off_last, const void *user_addr, + enum kmsan_bug_reason reason); + +DECLARE_PER_CPU(struct kmsan_ctx, kmsan_percpu_ctx); + +static __always_inline struct kmsan_ctx *kmsan_get_context(void) +{ + return in_task() ? ¤t->kmsan_ctx : raw_cpu_ptr(&kmsan_percpu_ctx); +} + +/* + * When a compiler hook or KMSAN runtime function is invoked, it may make a + * call to instrumented code and eventually call itself recursively. To avoid + * that, we guard the runtime entry regions with + * kmsan_enter_runtime()/kmsan_leave_runtime() and exit the hook if + * kmsan_in_runtime() is true. + * + * Non-runtime code may occasionally get executed in nested IRQs from the + * runtime code (e.g. when called via smp_call_function_single()). Because some + * KMSAN routines may take locks (e.g. for memory allocation), we conservatively + * bail out instead of calling them. To minimize the effect of this (potentially + * missing initialization events) kmsan_in_runtime() is not checked in + * non-blocking runtime functions. + */ +static __always_inline bool kmsan_in_runtime(void) +{ + if ((hardirq_count() >> HARDIRQ_SHIFT) > 1) + return true; + return kmsan_get_context()->kmsan_in_runtime; +} + +static __always_inline void kmsan_enter_runtime(void) +{ + struct kmsan_ctx *ctx; + + ctx = kmsan_get_context(); + KMSAN_WARN_ON(ctx->kmsan_in_runtime++); +} + +static __always_inline void kmsan_leave_runtime(void) +{ + struct kmsan_ctx *ctx = kmsan_get_context(); + + KMSAN_WARN_ON(--ctx->kmsan_in_runtime); +} + +depot_stack_handle_t kmsan_save_stack(void); +depot_stack_handle_t kmsan_save_stack_with_flags(gfp_t flags, + unsigned int extra_bits); + +/* + * Pack and unpack the origin chain depth and UAF flag to/from the extra bits + * provided by the stack depot. + * The UAF flag is stored in the lowest bit, followed by the depth in the upper + * bits. + * set_dsh_extra_bits() is responsible for clamping the value. + */ +static __always_inline unsigned int kmsan_extra_bits(unsigned int depth, + bool uaf) +{ + return (depth << 1) | uaf; +} + +static __always_inline bool kmsan_uaf_from_eb(unsigned int extra_bits) +{ + return extra_bits & 1; +} + +static __always_inline unsigned int kmsan_depth_from_eb(unsigned int extra_bits) +{ + return extra_bits >> 1; +} + +/* + * kmsan_internal_ functions are supposed to be very simple and not require the + * kmsan_in_runtime() checks. + */ +void kmsan_internal_memmove_metadata(void *dst, void *src, size_t n); +void kmsan_internal_poison_memory(void *address, size_t size, gfp_t flags, + unsigned int poison_flags); +void kmsan_internal_unpoison_memory(void *address, size_t size, bool checked); +void kmsan_internal_set_shadow_origin(void *address, size_t size, int b, + u32 origin, bool checked); +depot_stack_handle_t kmsan_internal_chain_origin(depot_stack_handle_t id); + +void kmsan_internal_task_create(struct task_struct *task); + +bool kmsan_metadata_is_contiguous(void *addr, size_t size); +void kmsan_internal_check_memory(void *addr, size_t size, const void *user_addr, + int reason); + +struct page *kmsan_vmalloc_to_page_or_null(void *vaddr); +void kmsan_setup_meta(struct page *page, struct page *shadow, + struct page *origin, int order); + +/* + * kmsan_internal_is_module_addr() and kmsan_internal_is_vmalloc_addr() are + * non-instrumented versions of is_module_address() and is_vmalloc_addr() that + * are safe to call from KMSAN runtime without recursion. + */ +static inline bool kmsan_internal_is_module_addr(void *vaddr) +{ + return ((u64)vaddr >= MODULES_VADDR) && ((u64)vaddr < MODULES_END); +} + +static inline bool kmsan_internal_is_vmalloc_addr(void *addr) +{ + return ((u64)addr >= VMALLOC_START) && ((u64)addr < VMALLOC_END); +} + +#endif /* __MM_KMSAN_KMSAN_H */ diff --git a/mm/kmsan/kmsan_test.c b/mm/kmsan/kmsan_test.c new file mode 100644 index 0000000000000000000000000000000000000000..9a29ea2dbfb9bb9f825bca33c8d879a264107d98 --- /dev/null +++ b/mm/kmsan/kmsan_test.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for KMSAN. + * For each test case checks the presence (or absence) of generated reports. + * Relies on 'console' tracepoint to capture reports as they appear in the + * kernel log. + * + * Copyright (C) 2021-2022, Google LLC. + * Author: Alexander Potapenko + * + */ + +#include +#include "kmsan.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_PER_CPU(int, per_cpu_var); + +/* Report as observed from console. */ +static struct { + spinlock_t lock; + bool available; + bool ignore; /* Stop console output collection. */ + char header[256]; +} observed = { + .lock = __SPIN_LOCK_UNLOCKED(observed.lock), +}; + +/* Probe for console output: obtains observed lines of interest. */ +static void probe_console(void *ignore, const char *buf, size_t len) +{ + unsigned long flags; + + if (observed.ignore) + return; + spin_lock_irqsave(&observed.lock, flags); + + if (strnstr(buf, "BUG: KMSAN: ", len)) { + /* + * KMSAN report and related to the test. + * + * The provided @buf is not NUL-terminated; copy no more than + * @len bytes and let strscpy() add the missing NUL-terminator. + */ + strscpy(observed.header, buf, + min(len + 1, sizeof(observed.header))); + WRITE_ONCE(observed.available, true); + observed.ignore = true; + } + spin_unlock_irqrestore(&observed.lock, flags); +} + +/* Check if a report related to the test exists. */ +static bool report_available(void) +{ + return READ_ONCE(observed.available); +} + +/* Information we expect in a report. */ +struct expect_report { + const char *error_type; /* Error type. */ + /* + * Kernel symbol from the error header, or NULL if no report is + * expected. + */ + const char *symbol; +}; + +/* Check observed report matches information in @r. */ +static bool report_matches(const struct expect_report *r) +{ + typeof(observed.header) expected_header; + unsigned long flags; + bool ret = false; + const char *end; + char *cur; + + /* Doubled-checked locking. */ + if (!report_available() || !r->symbol) + return (!report_available() && !r->symbol); + + /* Generate expected report contents. */ + + /* Title */ + cur = expected_header; + end = &expected_header[sizeof(expected_header) - 1]; + + cur += scnprintf(cur, end - cur, "BUG: KMSAN: %s", r->error_type); + + scnprintf(cur, end - cur, " in %s", r->symbol); + /* The exact offset won't match, remove it; also strip module name. */ + cur = strchr(expected_header, '+'); + if (cur) + *cur = '\0'; + + spin_lock_irqsave(&observed.lock, flags); + if (!report_available()) + goto out; /* A new report is being captured. */ + + /* Finally match expected output to what we actually observed. */ + ret = strstr(observed.header, expected_header); +out: + spin_unlock_irqrestore(&observed.lock, flags); + + return ret; +} + +/* ===== Test cases ===== */ + +/* Prevent replacing branch with select in LLVM. */ +static noinline void check_true(char *arg) +{ + pr_info("%s is true\n", arg); +} + +static noinline void check_false(char *arg) +{ + pr_info("%s is false\n", arg); +} + +#define USE(x) \ + do { \ + if (x) \ + check_true(#x); \ + else \ + check_false(#x); \ + } while (0) + +#define EXPECTATION_ETYPE_FN(e, reason, fn) \ + struct expect_report e = { \ + .error_type = reason, \ + .symbol = fn, \ + } + +#define EXPECTATION_NO_REPORT(e) EXPECTATION_ETYPE_FN(e, NULL, NULL) +#define EXPECTATION_UNINIT_VALUE_FN(e, fn) \ + EXPECTATION_ETYPE_FN(e, "uninit-value", fn) +#define EXPECTATION_UNINIT_VALUE(e) EXPECTATION_UNINIT_VALUE_FN(e, __func__) +#define EXPECTATION_USE_AFTER_FREE(e) \ + EXPECTATION_ETYPE_FN(e, "use-after-free", __func__) + +/* Test case: ensure that kmalloc() returns uninitialized memory. */ +static void test_uninit_kmalloc(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE(expect); + int *ptr; + + kunit_info(test, "uninitialized kmalloc test (UMR report)\n"); + ptr = kmalloc(sizeof(*ptr), GFP_KERNEL); + USE(*ptr); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that kmalloc'ed memory becomes initialized after memset(). + */ +static void test_init_kmalloc(struct kunit *test) +{ + EXPECTATION_NO_REPORT(expect); + int *ptr; + + kunit_info(test, "initialized kmalloc test (no reports)\n"); + ptr = kmalloc(sizeof(*ptr), GFP_KERNEL); + memset(ptr, 0, sizeof(*ptr)); + USE(*ptr); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* Test case: ensure that kzalloc() returns initialized memory. */ +static void test_init_kzalloc(struct kunit *test) +{ + EXPECTATION_NO_REPORT(expect); + int *ptr; + + kunit_info(test, "initialized kzalloc test (no reports)\n"); + ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); + USE(*ptr); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* Test case: ensure that local variables are uninitialized by default. */ +static void test_uninit_stack_var(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE(expect); + volatile int cond; + + kunit_info(test, "uninitialized stack variable (UMR report)\n"); + USE(cond); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* Test case: ensure that local variables with initializers are initialized. */ +static void test_init_stack_var(struct kunit *test) +{ + EXPECTATION_NO_REPORT(expect); + volatile int cond = 1; + + kunit_info(test, "initialized stack variable (no reports)\n"); + USE(cond); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +static noinline void two_param_fn_2(int arg1, int arg2) +{ + USE(arg1); + USE(arg2); +} + +static noinline void one_param_fn(int arg) +{ + two_param_fn_2(arg, arg); + USE(arg); +} + +static noinline void two_param_fn(int arg1, int arg2) +{ + int init = 0; + + one_param_fn(init); + USE(arg1); + USE(arg2); +} + +static void test_params(struct kunit *test) +{ +#ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL + /* + * With eager param/retval checking enabled, KMSAN will report an error + * before the call to two_param_fn(). + */ + EXPECTATION_UNINIT_VALUE_FN(expect, "test_params"); +#else + EXPECTATION_UNINIT_VALUE_FN(expect, "two_param_fn"); +#endif + volatile int uninit, init = 1; + + kunit_info(test, + "uninit passed through a function parameter (UMR report)\n"); + two_param_fn(uninit, init); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +static int signed_sum3(int a, int b, int c) +{ + return a + b + c; +} + +/* + * Test case: ensure that uninitialized values are tracked through function + * arguments. + */ +static void test_uninit_multiple_params(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE(expect); + volatile char b = 3, c; + volatile int a; + + kunit_info(test, "uninitialized local passed to fn (UMR report)\n"); + USE(signed_sum3(a, b, c)); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* Helper function to make an array uninitialized. */ +static noinline void do_uninit_local_array(char *array, int start, int stop) +{ + volatile char uninit; + + for (int i = start; i < stop; i++) + array[i] = uninit; +} + +/* + * Test case: ensure kmsan_check_memory() reports an error when checking + * uninitialized memory. + */ +static void test_uninit_kmsan_check_memory(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE_FN(expect, "test_uninit_kmsan_check_memory"); + volatile char local_array[8]; + + kunit_info( + test, + "kmsan_check_memory() called on uninit local (UMR report)\n"); + do_uninit_local_array((char *)local_array, 5, 7); + + kmsan_check_memory((char *)local_array, 8); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: check that a virtual memory range created with vmap() from + * initialized pages is still considered as initialized. + */ +static void test_init_kmsan_vmap_vunmap(struct kunit *test) +{ + EXPECTATION_NO_REPORT(expect); + const int npages = 2; + struct page **pages; + void *vbuf; + + kunit_info(test, "pages initialized via vmap (no reports)\n"); + + pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL); + for (int i = 0; i < npages; i++) + pages[i] = alloc_page(GFP_KERNEL); + vbuf = vmap(pages, npages, VM_MAP, PAGE_KERNEL); + memset(vbuf, 0xfe, npages * PAGE_SIZE); + for (int i = 0; i < npages; i++) + kmsan_check_memory(page_address(pages[i]), PAGE_SIZE); + + if (vbuf) + vunmap(vbuf); + for (int i = 0; i < npages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that memset() can initialize a buffer allocated via + * vmalloc(). + */ +static void test_init_vmalloc(struct kunit *test) +{ + EXPECTATION_NO_REPORT(expect); + int npages = 8; + char *buf; + + kunit_info(test, "vmalloc buffer can be initialized (no reports)\n"); + buf = vmalloc(PAGE_SIZE * npages); + buf[0] = 1; + memset(buf, 0xfe, PAGE_SIZE * npages); + USE(buf[0]); + for (int i = 0; i < npages; i++) + kmsan_check_memory(&buf[PAGE_SIZE * i], PAGE_SIZE); + vfree(buf); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* Test case: ensure that use-after-free reporting works. */ +static void test_uaf(struct kunit *test) +{ + EXPECTATION_USE_AFTER_FREE(expect); + volatile int value; + volatile int *var; + + kunit_info(test, "use-after-free in kmalloc-ed buffer (UMR report)\n"); + var = kmalloc(80, GFP_KERNEL); + var[3] = 0xfeedface; + kfree((int *)var); + /* Copy the invalid value before checking it. */ + value = var[3]; + USE(value); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that uninitialized values are propagated through per-CPU + * memory. + */ +static void test_percpu_propagate(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE(expect); + volatile int uninit, check; + + kunit_info(test, + "uninit local stored to per_cpu memory (UMR report)\n"); + + this_cpu_write(per_cpu_var, uninit); + check = this_cpu_read(per_cpu_var); + USE(check); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that passing uninitialized values to printk() leads to an + * error report. + */ +static void test_printk(struct kunit *test) +{ +#ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL + /* + * With eager param/retval checking enabled, KMSAN will report an error + * before the call to pr_info(). + */ + EXPECTATION_UNINIT_VALUE_FN(expect, "test_printk"); +#else + EXPECTATION_UNINIT_VALUE_FN(expect, "number"); +#endif + volatile int uninit; + + kunit_info(test, "uninit local passed to pr_info() (UMR report)\n"); + pr_info("%px contains %d\n", &uninit, uninit); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that memcpy() correctly copies uninitialized values between + * aligned `src` and `dst`. + */ +static void test_memcpy_aligned_to_aligned(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE_FN(expect, "test_memcpy_aligned_to_aligned"); + volatile int uninit_src; + volatile int dst = 0; + + kunit_info( + test, + "memcpy()ing aligned uninit src to aligned dst (UMR report)\n"); + memcpy((void *)&dst, (void *)&uninit_src, sizeof(uninit_src)); + kmsan_check_memory((void *)&dst, sizeof(dst)); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that memcpy() correctly copies uninitialized values between + * aligned `src` and unaligned `dst`. + * + * Copying aligned 4-byte value to an unaligned one leads to touching two + * aligned 4-byte values. This test case checks that KMSAN correctly reports an + * error on the first of the two values. + */ +static void test_memcpy_aligned_to_unaligned(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE_FN(expect, "test_memcpy_aligned_to_unaligned"); + volatile int uninit_src; + volatile char dst[8] = { 0 }; + + kunit_info( + test, + "memcpy()ing aligned uninit src to unaligned dst (UMR report)\n"); + memcpy((void *)&dst[1], (void *)&uninit_src, sizeof(uninit_src)); + kmsan_check_memory((void *)dst, 4); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +/* + * Test case: ensure that memcpy() correctly copies uninitialized values between + * aligned `src` and unaligned `dst`. + * + * Copying aligned 4-byte value to an unaligned one leads to touching two + * aligned 4-byte values. This test case checks that KMSAN correctly reports an + * error on the second of the two values. + */ +static void test_memcpy_aligned_to_unaligned2(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE_FN(expect, + "test_memcpy_aligned_to_unaligned2"); + volatile int uninit_src; + volatile char dst[8] = { 0 }; + + kunit_info( + test, + "memcpy()ing aligned uninit src to unaligned dst - part 2 (UMR report)\n"); + memcpy((void *)&dst[1], (void *)&uninit_src, sizeof(uninit_src)); + kmsan_check_memory((void *)&dst[4], sizeof(uninit_src)); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +static noinline void fibonacci(int *array, int size, int start) { + if (start < 2 || (start == size)) + return; + array[start] = array[start - 1] + array[start - 2]; + fibonacci(array, size, start + 1); +} + +static void test_long_origin_chain(struct kunit *test) +{ + EXPECTATION_UNINIT_VALUE_FN(expect, + "test_long_origin_chain"); + /* (KMSAN_MAX_ORIGIN_DEPTH * 2) recursive calls to fibonacci(). */ + volatile int accum[KMSAN_MAX_ORIGIN_DEPTH * 2 + 2]; + int last = ARRAY_SIZE(accum) - 1; + + kunit_info( + test, + "origin chain exceeding KMSAN_MAX_ORIGIN_DEPTH (UMR report)\n"); + /* + * We do not set accum[1] to 0, so the uninitializedness will be carried + * over to accum[2..last]. + */ + accum[0] = 1; + fibonacci((int *)accum, ARRAY_SIZE(accum), 2); + kmsan_check_memory((void *)&accum[last], sizeof(int)); + KUNIT_EXPECT_TRUE(test, report_matches(&expect)); +} + +static struct kunit_case kmsan_test_cases[] = { + KUNIT_CASE(test_uninit_kmalloc), + KUNIT_CASE(test_init_kmalloc), + KUNIT_CASE(test_init_kzalloc), + KUNIT_CASE(test_uninit_stack_var), + KUNIT_CASE(test_init_stack_var), + KUNIT_CASE(test_params), + KUNIT_CASE(test_uninit_multiple_params), + KUNIT_CASE(test_uninit_kmsan_check_memory), + KUNIT_CASE(test_init_kmsan_vmap_vunmap), + KUNIT_CASE(test_init_vmalloc), + KUNIT_CASE(test_uaf), + KUNIT_CASE(test_percpu_propagate), + KUNIT_CASE(test_printk), + KUNIT_CASE(test_memcpy_aligned_to_aligned), + KUNIT_CASE(test_memcpy_aligned_to_unaligned), + KUNIT_CASE(test_memcpy_aligned_to_unaligned2), + KUNIT_CASE(test_long_origin_chain), + {}, +}; + +/* ===== End test cases ===== */ + +static int test_init(struct kunit *test) +{ + unsigned long flags; + + spin_lock_irqsave(&observed.lock, flags); + observed.header[0] = '\0'; + observed.ignore = false; + observed.available = false; + spin_unlock_irqrestore(&observed.lock, flags); + + return 0; +} + +static void test_exit(struct kunit *test) +{ +} + +static void register_tracepoints(struct tracepoint *tp, void *ignore) +{ + check_trace_callback_type_console(probe_console); + if (!strcmp(tp->name, "console")) + WARN_ON(tracepoint_probe_register(tp, probe_console, NULL)); +} + +static void unregister_tracepoints(struct tracepoint *tp, void *ignore) +{ + if (!strcmp(tp->name, "console")) + tracepoint_probe_unregister(tp, probe_console, NULL); +} + +static int kmsan_suite_init(struct kunit_suite *suite) +{ + /* + * Because we want to be able to build the test as a module, we need to + * iterate through all known tracepoints, since the static registration + * won't work here. + */ + for_each_kernel_tracepoint(register_tracepoints, NULL); + return 0; +} + +static void kmsan_suite_exit(struct kunit_suite *suite) +{ + for_each_kernel_tracepoint(unregister_tracepoints, NULL); + tracepoint_synchronize_unregister(); +} + +static struct kunit_suite kmsan_test_suite = { + .name = "kmsan", + .test_cases = kmsan_test_cases, + .init = test_init, + .exit = test_exit, + .suite_init = kmsan_suite_init, + .suite_exit = kmsan_suite_exit, +}; +kunit_test_suites(&kmsan_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Alexander Potapenko "); diff --git a/mm/kmsan/report.c b/mm/kmsan/report.c new file mode 100644 index 0000000000000000000000000000000000000000..02736ec757f2c4c269be61350e7bd6bbb3ede377 --- /dev/null +++ b/mm/kmsan/report.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN error reporting routines. + * + * Copyright (C) 2019-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#include +#include +#include +#include +#include + +#include "kmsan.h" + +static DEFINE_RAW_SPINLOCK(kmsan_report_lock); +#define DESCR_SIZE 128 +/* Protected by kmsan_report_lock */ +static char report_local_descr[DESCR_SIZE]; +int panic_on_kmsan __read_mostly; + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "kmsan." +module_param_named(panic, panic_on_kmsan, int, 0); + +/* + * Skip internal KMSAN frames. + */ +static int get_stack_skipnr(const unsigned long stack_entries[], + int num_entries) +{ + int len, skip; + char buf[64]; + + for (skip = 0; skip < num_entries; ++skip) { + len = scnprintf(buf, sizeof(buf), "%ps", + (void *)stack_entries[skip]); + + /* Never show __msan_* or kmsan_* functions. */ + if ((strnstr(buf, "__msan_", len) == buf) || + (strnstr(buf, "kmsan_", len) == buf)) + continue; + + /* + * No match for runtime functions -- @skip entries to skip to + * get to first frame of interest. + */ + break; + } + + return skip; +} + +/* + * Currently the descriptions of locals generated by Clang look as follows: + * ----local_name@function_name + * We want to print only the name of the local, as other information in that + * description can be confusing. + * The meaningful part of the description is copied to a global buffer to avoid + * allocating memory. + */ +static char *pretty_descr(char *descr) +{ + int pos = 0, len = strlen(descr); + + for (int i = 0; i < len; i++) { + if (descr[i] == '@') + break; + if (descr[i] == '-') + continue; + report_local_descr[pos] = descr[i]; + if (pos + 1 == DESCR_SIZE) + break; + pos++; + } + report_local_descr[pos] = 0; + return report_local_descr; +} + +void kmsan_print_origin(depot_stack_handle_t origin) +{ + unsigned long *entries = NULL, *chained_entries = NULL; + unsigned int nr_entries, chained_nr_entries, skipnr; + void *pc1 = NULL, *pc2 = NULL; + depot_stack_handle_t head; + unsigned long magic; + char *descr = NULL; + unsigned int depth; + + if (!origin) + return; + + while (true) { + nr_entries = stack_depot_fetch(origin, &entries); + depth = kmsan_depth_from_eb(stack_depot_get_extra_bits(origin)); + magic = nr_entries ? entries[0] : 0; + if ((nr_entries == 4) && (magic == KMSAN_ALLOCA_MAGIC_ORIGIN)) { + descr = (char *)entries[1]; + pc1 = (void *)entries[2]; + pc2 = (void *)entries[3]; + pr_err("Local variable %s created at:\n", + pretty_descr(descr)); + if (pc1) + pr_err(" %pSb\n", pc1); + if (pc2) + pr_err(" %pSb\n", pc2); + break; + } + if ((nr_entries == 3) && (magic == KMSAN_CHAIN_MAGIC_ORIGIN)) { + /* + * Origin chains deeper than KMSAN_MAX_ORIGIN_DEPTH are + * not stored, so the output may be incomplete. + */ + if (depth == KMSAN_MAX_ORIGIN_DEPTH) + pr_err("\n\n"); + head = entries[1]; + origin = entries[2]; + pr_err("Uninit was stored to memory at:\n"); + chained_nr_entries = + stack_depot_fetch(head, &chained_entries); + kmsan_internal_unpoison_memory( + chained_entries, + chained_nr_entries * sizeof(*chained_entries), + /*checked*/ false); + skipnr = get_stack_skipnr(chained_entries, + chained_nr_entries); + stack_trace_print(chained_entries + skipnr, + chained_nr_entries - skipnr, 0); + pr_err("\n"); + continue; + } + pr_err("Uninit was created at:\n"); + if (nr_entries) { + skipnr = get_stack_skipnr(entries, nr_entries); + stack_trace_print(entries + skipnr, nr_entries - skipnr, + 0); + } else { + pr_err("(stack is not available)\n"); + } + break; + } +} + +void kmsan_report(depot_stack_handle_t origin, void *address, int size, + int off_first, int off_last, const void *user_addr, + enum kmsan_bug_reason reason) +{ + unsigned long stack_entries[KMSAN_STACK_DEPTH]; + int num_stack_entries, skipnr; + char *bug_type = NULL; + unsigned long ua_flags; + bool is_uaf; + + if (!kmsan_enabled) + return; + if (!current->kmsan_ctx.allow_reporting) + return; + if (!origin) + return; + + current->kmsan_ctx.allow_reporting = false; + ua_flags = user_access_save(); + raw_spin_lock(&kmsan_report_lock); + pr_err("=====================================================\n"); + is_uaf = kmsan_uaf_from_eb(stack_depot_get_extra_bits(origin)); + switch (reason) { + case REASON_ANY: + bug_type = is_uaf ? "use-after-free" : "uninit-value"; + break; + case REASON_COPY_TO_USER: + bug_type = is_uaf ? "kernel-infoleak-after-free" : + "kernel-infoleak"; + break; + case REASON_SUBMIT_URB: + bug_type = is_uaf ? "kernel-usb-infoleak-after-free" : + "kernel-usb-infoleak"; + break; + } + + num_stack_entries = + stack_trace_save(stack_entries, KMSAN_STACK_DEPTH, 1); + skipnr = get_stack_skipnr(stack_entries, num_stack_entries); + + pr_err("BUG: KMSAN: %s in %pSb\n", bug_type, + (void *)stack_entries[skipnr]); + stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, + 0); + pr_err("\n"); + + kmsan_print_origin(origin); + + if (size) { + pr_err("\n"); + if (off_first == off_last) + pr_err("Byte %d of %d is uninitialized\n", off_first, + size); + else + pr_err("Bytes %d-%d of %d are uninitialized\n", + off_first, off_last, size); + } + if (address) + pr_err("Memory access of size %d starts at %px\n", size, + address); + if (user_addr && reason == REASON_COPY_TO_USER) + pr_err("Data copied to user address %px\n", user_addr); + pr_err("\n"); + dump_stack_print_info(KERN_ERR); + pr_err("=====================================================\n"); + add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); + raw_spin_unlock(&kmsan_report_lock); + if (panic_on_kmsan) + panic("kmsan.panic set ...\n"); + user_access_restore(ua_flags); + current->kmsan_ctx.allow_reporting = true; +} diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c new file mode 100644 index 0000000000000000000000000000000000000000..21e3e196ec3cfc47c180412365e0034ac4ded66d --- /dev/null +++ b/mm/kmsan/shadow.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KMSAN shadow implementation. + * + * Copyright (C) 2017-2022 Google LLC + * Author: Alexander Potapenko + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../internal.h" +#include "kmsan.h" + +#define shadow_page_for(page) ((page)->kmsan_shadow) + +#define origin_page_for(page) ((page)->kmsan_origin) + +static void *shadow_ptr_for(struct page *page) +{ + return page_address(shadow_page_for(page)); +} + +static void *origin_ptr_for(struct page *page) +{ + return page_address(origin_page_for(page)); +} + +static bool page_has_metadata(struct page *page) +{ + return shadow_page_for(page) && origin_page_for(page); +} + +static void set_no_shadow_origin_page(struct page *page) +{ + shadow_page_for(page) = NULL; + origin_page_for(page) = NULL; +} + +/* + * Dummy load and store pages to be used when the real metadata is unavailable. + * There are separate pages for loads and stores, so that every load returns a + * zero, and every store doesn't affect other loads. + */ +static char dummy_load_page[PAGE_SIZE] __aligned(PAGE_SIZE); +static char dummy_store_page[PAGE_SIZE] __aligned(PAGE_SIZE); + +static unsigned long vmalloc_meta(void *addr, bool is_origin) +{ + unsigned long addr64 = (unsigned long)addr, off; + + KMSAN_WARN_ON(is_origin && !IS_ALIGNED(addr64, KMSAN_ORIGIN_SIZE)); + if (kmsan_internal_is_vmalloc_addr(addr)) { + off = addr64 - VMALLOC_START; + return off + (is_origin ? KMSAN_VMALLOC_ORIGIN_START : + KMSAN_VMALLOC_SHADOW_START); + } + if (kmsan_internal_is_module_addr(addr)) { + off = addr64 - MODULES_VADDR; + return off + (is_origin ? KMSAN_MODULES_ORIGIN_START : + KMSAN_MODULES_SHADOW_START); + } + return 0; +} + +static struct page *virt_to_page_or_null(void *vaddr) +{ + if (kmsan_virt_addr_valid(vaddr)) + return virt_to_page(vaddr); + else + return NULL; +} + +struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *address, u64 size, + bool store) +{ + struct shadow_origin_ptr ret; + void *shadow; + + /* + * Even if we redirect this memory access to the dummy page, it will + * go out of bounds. + */ + KMSAN_WARN_ON(size > PAGE_SIZE); + + if (!kmsan_enabled) + goto return_dummy; + + KMSAN_WARN_ON(!kmsan_metadata_is_contiguous(address, size)); + shadow = kmsan_get_metadata(address, KMSAN_META_SHADOW); + if (!shadow) + goto return_dummy; + + ret.shadow = shadow; + ret.origin = kmsan_get_metadata(address, KMSAN_META_ORIGIN); + return ret; + +return_dummy: + if (store) { + /* Ignore this store. */ + ret.shadow = dummy_store_page; + ret.origin = dummy_store_page; + } else { + /* This load will return zero. */ + ret.shadow = dummy_load_page; + ret.origin = dummy_load_page; + } + return ret; +} + +/* + * Obtain the shadow or origin pointer for the given address, or NULL if there's + * none. The caller must check the return value for being non-NULL if needed. + * The return value of this function should not depend on whether we're in the + * runtime or not. + */ +void *kmsan_get_metadata(void *address, bool is_origin) +{ + u64 addr = (u64)address, pad, off; + struct page *page; + void *ret; + + if (is_origin && !IS_ALIGNED(addr, KMSAN_ORIGIN_SIZE)) { + pad = addr % KMSAN_ORIGIN_SIZE; + addr -= pad; + } + address = (void *)addr; + if (kmsan_internal_is_vmalloc_addr(address) || + kmsan_internal_is_module_addr(address)) + return (void *)vmalloc_meta(address, is_origin); + + ret = arch_kmsan_get_meta_or_null(address, is_origin); + if (ret) + return ret; + + page = virt_to_page_or_null(address); + if (!page) + return NULL; + if (!page_has_metadata(page)) + return NULL; + off = addr % PAGE_SIZE; + + return (is_origin ? origin_ptr_for(page) : shadow_ptr_for(page)) + off; +} + +void kmsan_copy_page_meta(struct page *dst, struct page *src) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + if (!dst || !page_has_metadata(dst)) + return; + if (!src || !page_has_metadata(src)) { + kmsan_internal_unpoison_memory(page_address(dst), PAGE_SIZE, + /*checked*/ false); + return; + } + + kmsan_enter_runtime(); + __memcpy(shadow_ptr_for(dst), shadow_ptr_for(src), PAGE_SIZE); + __memcpy(origin_ptr_for(dst), origin_ptr_for(src), PAGE_SIZE); + kmsan_leave_runtime(); +} + +void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags) +{ + bool initialized = (flags & __GFP_ZERO) || !kmsan_enabled; + struct page *shadow, *origin; + depot_stack_handle_t handle; + int pages = 1 << order; + + if (!page) + return; + + shadow = shadow_page_for(page); + origin = origin_page_for(page); + + if (initialized) { + __memset(page_address(shadow), 0, PAGE_SIZE * pages); + __memset(page_address(origin), 0, PAGE_SIZE * pages); + return; + } + + /* Zero pages allocated by the runtime should also be initialized. */ + if (kmsan_in_runtime()) + return; + + __memset(page_address(shadow), -1, PAGE_SIZE * pages); + kmsan_enter_runtime(); + handle = kmsan_save_stack_with_flags(flags, /*extra_bits*/ 0); + kmsan_leave_runtime(); + /* + * Addresses are page-aligned, pages are contiguous, so it's ok + * to just fill the origin pages with @handle. + */ + for (int i = 0; i < PAGE_SIZE * pages / sizeof(handle); i++) + ((depot_stack_handle_t *)page_address(origin))[i] = handle; +} + +void kmsan_free_page(struct page *page, unsigned int order) +{ + if (!kmsan_enabled || kmsan_in_runtime()) + return; + kmsan_enter_runtime(); + kmsan_internal_poison_memory(page_address(page), + PAGE_SIZE << compound_order(page), + GFP_KERNEL, + KMSAN_POISON_CHECK | KMSAN_POISON_FREE); + kmsan_leave_runtime(); +} + +void kmsan_vmap_pages_range_noflush(unsigned long start, unsigned long end, + pgprot_t prot, struct page **pages, + unsigned int page_shift) +{ + unsigned long shadow_start, origin_start, shadow_end, origin_end; + struct page **s_pages, **o_pages; + int nr, mapped; + + if (!kmsan_enabled) + return; + + shadow_start = vmalloc_meta((void *)start, KMSAN_META_SHADOW); + shadow_end = vmalloc_meta((void *)end, KMSAN_META_SHADOW); + if (!shadow_start) + return; + + nr = (end - start) / PAGE_SIZE; + s_pages = kcalloc(nr, sizeof(*s_pages), GFP_KERNEL); + o_pages = kcalloc(nr, sizeof(*o_pages), GFP_KERNEL); + if (!s_pages || !o_pages) + goto ret; + for (int i = 0; i < nr; i++) { + s_pages[i] = shadow_page_for(pages[i]); + o_pages[i] = origin_page_for(pages[i]); + } + prot = __pgprot(pgprot_val(prot) | _PAGE_NX); + prot = PAGE_KERNEL; + + origin_start = vmalloc_meta((void *)start, KMSAN_META_ORIGIN); + origin_end = vmalloc_meta((void *)end, KMSAN_META_ORIGIN); + kmsan_enter_runtime(); + mapped = __vmap_pages_range_noflush(shadow_start, shadow_end, prot, + s_pages, page_shift); + KMSAN_WARN_ON(mapped); + mapped = __vmap_pages_range_noflush(origin_start, origin_end, prot, + o_pages, page_shift); + KMSAN_WARN_ON(mapped); + kmsan_leave_runtime(); + flush_tlb_kernel_range(shadow_start, shadow_end); + flush_tlb_kernel_range(origin_start, origin_end); + flush_cache_vmap(shadow_start, shadow_end); + flush_cache_vmap(origin_start, origin_end); + +ret: + kfree(s_pages); + kfree(o_pages); +} + +/* Allocate metadata for pages allocated at boot time. */ +void __init kmsan_init_alloc_meta_for_range(void *start, void *end) +{ + struct page *shadow_p, *origin_p; + void *shadow, *origin; + struct page *page; + u64 size; + + start = (void *)ALIGN_DOWN((u64)start, PAGE_SIZE); + size = ALIGN((u64)end - (u64)start, PAGE_SIZE); + shadow = memblock_alloc(size, PAGE_SIZE); + origin = memblock_alloc(size, PAGE_SIZE); + for (u64 addr = 0; addr < size; addr += PAGE_SIZE) { + page = virt_to_page_or_null((char *)start + addr); + shadow_p = virt_to_page_or_null((char *)shadow + addr); + set_no_shadow_origin_page(shadow_p); + shadow_page_for(page) = shadow_p; + origin_p = virt_to_page_or_null((char *)origin + addr); + set_no_shadow_origin_page(origin_p); + origin_page_for(page) = origin_p; + } +} + +void kmsan_setup_meta(struct page *page, struct page *shadow, + struct page *origin, int order) +{ + for (int i = 0; i < (1 << order); i++) { + set_no_shadow_origin_page(&shadow[i]); + set_no_shadow_origin_page(&origin[i]); + shadow_page_for(&page[i]) = &shadow[i]; + origin_page_for(&page[i]) = &origin[i]; + } +} diff --git a/mm/ksm.c b/mm/ksm.c index 42ab153335a2d788edc87ca4fb73873e4babe846..c19fcca9bc03dcd9511486000c6cadb11d724c29 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -42,6 +42,7 @@ #include #include "internal.h" +#include "mm_slot.h" #ifdef CONFIG_NUMA #define NUMA(x) (x) @@ -82,7 +83,7 @@ * different KSM page copy of that content * * Internally, the regular nodes, "dups" and "chains" are represented - * using the same struct stable_node structure. + * using the same struct ksm_stable_node structure. * * In addition to the stable tree, KSM uses a second data structure called the * unstable tree: this tree holds pointers to pages which have been found to @@ -112,17 +113,13 @@ */ /** - * struct mm_slot - ksm information per mm that is being scanned - * @link: link to the mm_slots hash list - * @mm_list: link into the mm_slots list, rooted in ksm_mm_head + * struct ksm_mm_slot - ksm information per mm that is being scanned + * @slot: hash lookup from mm to mm_slot * @rmap_list: head for this mm_slot's singly-linked list of rmap_items - * @mm: the mm that this information is valid for */ -struct mm_slot { - struct hlist_node link; - struct list_head mm_list; - struct rmap_item *rmap_list; - struct mm_struct *mm; +struct ksm_mm_slot { + struct mm_slot slot; + struct ksm_rmap_item *rmap_list; }; /** @@ -135,14 +132,14 @@ struct mm_slot { * There is only the one ksm_scan instance of this cursor structure. */ struct ksm_scan { - struct mm_slot *mm_slot; + struct ksm_mm_slot *mm_slot; unsigned long address; - struct rmap_item **rmap_list; + struct ksm_rmap_item **rmap_list; unsigned long seqnr; }; /** - * struct stable_node - node of the stable rbtree + * struct ksm_stable_node - node of the stable rbtree * @node: rb node of this ksm page in the stable tree * @head: (overlaying parent) &migrate_nodes indicates temporarily on that list * @hlist_dup: linked into the stable_node->hlist with a stable_node chain @@ -153,7 +150,7 @@ struct ksm_scan { * @rmap_hlist_len: number of rmap_item entries in hlist or STABLE_NODE_CHAIN * @nid: NUMA node id of stable tree in which linked (may not match kpfn) */ -struct stable_node { +struct ksm_stable_node { union { struct rb_node node; /* when node of stable tree */ struct { /* when listed for migration */ @@ -182,7 +179,7 @@ struct stable_node { }; /** - * struct rmap_item - reverse mapping item for virtual addresses + * struct ksm_rmap_item - reverse mapping item for virtual addresses * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree * @nid: NUMA node id of unstable tree in which linked (may not match page) @@ -193,8 +190,8 @@ struct stable_node { * @head: pointer to stable_node heading this list in the stable tree * @hlist: link into hlist of rmap_items hanging off that stable_node */ -struct rmap_item { - struct rmap_item *rmap_list; +struct ksm_rmap_item { + struct ksm_rmap_item *rmap_list; union { struct anon_vma *anon_vma; /* when stable */ #ifdef CONFIG_NUMA @@ -207,7 +204,7 @@ struct rmap_item { union { struct rb_node node; /* when node of unstable tree */ struct { /* when listed from stable tree */ - struct stable_node *head; + struct ksm_stable_node *head; struct hlist_node hlist; }; }; @@ -230,8 +227,8 @@ static LIST_HEAD(migrate_nodes); #define MM_SLOTS_HASH_BITS 10 static DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS); -static struct mm_slot ksm_mm_head = { - .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list), +static struct ksm_mm_slot ksm_mm_head = { + .slot.mm_node = LIST_HEAD_INIT(ksm_mm_head.slot.mm_node), }; static struct ksm_scan ksm_scan = { .mm_slot = &ksm_mm_head, @@ -298,21 +295,21 @@ static DECLARE_WAIT_QUEUE_HEAD(ksm_iter_wait); static DEFINE_MUTEX(ksm_thread_mutex); static DEFINE_SPINLOCK(ksm_mmlist_lock); -#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\ +#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\ sizeof(struct __struct), __alignof__(struct __struct),\ (__flags), NULL) static int __init ksm_slab_init(void) { - rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0); + rmap_item_cache = KSM_KMEM_CACHE(ksm_rmap_item, 0); if (!rmap_item_cache) goto out; - stable_node_cache = KSM_KMEM_CACHE(stable_node, 0); + stable_node_cache = KSM_KMEM_CACHE(ksm_stable_node, 0); if (!stable_node_cache) goto out_free1; - mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0); + mm_slot_cache = KSM_KMEM_CACHE(ksm_mm_slot, 0); if (!mm_slot_cache) goto out_free2; @@ -334,18 +331,18 @@ static void __init ksm_slab_free(void) mm_slot_cache = NULL; } -static __always_inline bool is_stable_node_chain(struct stable_node *chain) +static __always_inline bool is_stable_node_chain(struct ksm_stable_node *chain) { return chain->rmap_hlist_len == STABLE_NODE_CHAIN; } -static __always_inline bool is_stable_node_dup(struct stable_node *dup) +static __always_inline bool is_stable_node_dup(struct ksm_stable_node *dup) { return dup->head == STABLE_NODE_DUP_HEAD; } -static inline void stable_node_chain_add_dup(struct stable_node *dup, - struct stable_node *chain) +static inline void stable_node_chain_add_dup(struct ksm_stable_node *dup, + struct ksm_stable_node *chain) { VM_BUG_ON(is_stable_node_dup(dup)); dup->head = STABLE_NODE_DUP_HEAD; @@ -354,14 +351,14 @@ static inline void stable_node_chain_add_dup(struct stable_node *dup, ksm_stable_node_dups++; } -static inline void __stable_node_dup_del(struct stable_node *dup) +static inline void __stable_node_dup_del(struct ksm_stable_node *dup) { VM_BUG_ON(!is_stable_node_dup(dup)); hlist_del(&dup->hlist_dup); ksm_stable_node_dups--; } -static inline void stable_node_dup_del(struct stable_node *dup) +static inline void stable_node_dup_del(struct ksm_stable_node *dup) { VM_BUG_ON(is_stable_node_chain(dup)); if (is_stable_node_dup(dup)) @@ -373,9 +370,9 @@ static inline void stable_node_dup_del(struct stable_node *dup) #endif } -static inline struct rmap_item *alloc_rmap_item(void) +static inline struct ksm_rmap_item *alloc_rmap_item(void) { - struct rmap_item *rmap_item; + struct ksm_rmap_item *rmap_item; rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); @@ -384,14 +381,15 @@ static inline struct rmap_item *alloc_rmap_item(void) return rmap_item; } -static inline void free_rmap_item(struct rmap_item *rmap_item) +static inline void free_rmap_item(struct ksm_rmap_item *rmap_item) { ksm_rmap_items--; + rmap_item->mm->ksm_rmap_items--; rmap_item->mm = NULL; /* debug safety */ kmem_cache_free(rmap_item_cache, rmap_item); } -static inline struct stable_node *alloc_stable_node(void) +static inline struct ksm_stable_node *alloc_stable_node(void) { /* * The allocation can take too long with GFP_KERNEL when memory is under @@ -401,43 +399,13 @@ static inline struct stable_node *alloc_stable_node(void) return kmem_cache_alloc(stable_node_cache, GFP_KERNEL | __GFP_HIGH); } -static inline void free_stable_node(struct stable_node *stable_node) +static inline void free_stable_node(struct ksm_stable_node *stable_node) { VM_BUG_ON(stable_node->rmap_hlist_len && !is_stable_node_chain(stable_node)); kmem_cache_free(stable_node_cache, stable_node); } -static inline struct mm_slot *alloc_mm_slot(void) -{ - if (!mm_slot_cache) /* initialization failed */ - return NULL; - return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); -} - -static inline void free_mm_slot(struct mm_slot *mm_slot) -{ - kmem_cache_free(mm_slot_cache, mm_slot); -} - -static struct mm_slot *get_mm_slot(struct mm_struct *mm) -{ - struct mm_slot *slot; - - hash_for_each_possible(mm_slots_hash, slot, link, (unsigned long)mm) - if (slot->mm == mm) - return slot; - - return NULL; -} - -static void insert_to_mm_slots_hash(struct mm_struct *mm, - struct mm_slot *mm_slot) -{ - mm_slot->mm = mm; - hash_add(mm_slots_hash, &mm_slot->link, (unsigned long)mm); -} - /* * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's * page tables after it has passed through ksm_exit() - which, if necessary, @@ -475,7 +443,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) cond_resched(); page = follow_page(vma, addr, FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE); - if (IS_ERR_OR_NULL(page) || is_zone_device_page(page)) + if (IS_ERR_OR_NULL(page)) break; if (PageKsm(page)) ret = handle_mm_fault(vma, addr, @@ -528,7 +496,7 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, return vma; } -static void break_cow(struct rmap_item *rmap_item) +static void break_cow(struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; unsigned long addr = rmap_item->address; @@ -547,7 +515,7 @@ static void break_cow(struct rmap_item *rmap_item) mmap_read_unlock(mm); } -static struct page *get_mergeable_page(struct rmap_item *rmap_item) +static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; unsigned long addr = rmap_item->address; @@ -560,12 +528,15 @@ static struct page *get_mergeable_page(struct rmap_item *rmap_item) goto out; page = follow_page(vma, addr, FOLL_GET); - if (IS_ERR_OR_NULL(page) || is_zone_device_page(page)) + if (IS_ERR_OR_NULL(page)) goto out; + if (is_zone_device_page(page)) + goto out_putpage; if (PageAnon(page)) { flush_anon_page(vma, page, addr); flush_dcache_page(page); } else { +out_putpage: put_page(page); out: page = NULL; @@ -585,10 +556,10 @@ static inline int get_kpfn_nid(unsigned long kpfn) return ksm_merge_across_nodes ? 0 : NUMA(pfn_to_nid(kpfn)); } -static struct stable_node *alloc_stable_node_chain(struct stable_node *dup, +static struct ksm_stable_node *alloc_stable_node_chain(struct ksm_stable_node *dup, struct rb_root *root) { - struct stable_node *chain = alloc_stable_node(); + struct ksm_stable_node *chain = alloc_stable_node(); VM_BUG_ON(is_stable_node_chain(dup)); if (likely(chain)) { INIT_HLIST_HEAD(&chain->hlist); @@ -618,7 +589,7 @@ static struct stable_node *alloc_stable_node_chain(struct stable_node *dup, return chain; } -static inline void free_stable_node_chain(struct stable_node *chain, +static inline void free_stable_node_chain(struct ksm_stable_node *chain, struct rb_root *root) { rb_erase(&chain->node, root); @@ -626,9 +597,9 @@ static inline void free_stable_node_chain(struct stable_node *chain, ksm_stable_node_chains--; } -static void remove_node_from_stable_tree(struct stable_node *stable_node) +static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node) { - struct rmap_item *rmap_item; + struct ksm_rmap_item *rmap_item; /* check it's not STABLE_NODE_CHAIN or negative */ BUG_ON(stable_node->rmap_hlist_len < 0); @@ -690,7 +661,7 @@ enum get_ksm_page_flags { * a page to put something that might look like our key in page->mapping. * is on its way to being freed; but it is an anomaly to bear in mind. */ -static struct page *get_ksm_page(struct stable_node *stable_node, +static struct page *get_ksm_page(struct ksm_stable_node *stable_node, enum get_ksm_page_flags flags) { struct page *page; @@ -769,10 +740,10 @@ stale: * Removing rmap_item from stable or unstable tree. * This function will clean the information from the stable/unstable tree. */ -static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) +static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item) { if (rmap_item->address & STABLE_FLAG) { - struct stable_node *stable_node; + struct ksm_stable_node *stable_node; struct page *page; stable_node = rmap_item->head; @@ -819,10 +790,10 @@ out: cond_resched(); /* we're called from many long loops */ } -static void remove_trailing_rmap_items(struct rmap_item **rmap_list) +static void remove_trailing_rmap_items(struct ksm_rmap_item **rmap_list) { while (*rmap_list) { - struct rmap_item *rmap_item = *rmap_list; + struct ksm_rmap_item *rmap_item = *rmap_list; *rmap_list = rmap_item->rmap_list; remove_rmap_item_from_tree(rmap_item); free_rmap_item(rmap_item); @@ -859,18 +830,18 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma, return err; } -static inline struct stable_node *folio_stable_node(struct folio *folio) +static inline struct ksm_stable_node *folio_stable_node(struct folio *folio) { return folio_test_ksm(folio) ? folio_raw_mapping(folio) : NULL; } -static inline struct stable_node *page_stable_node(struct page *page) +static inline struct ksm_stable_node *page_stable_node(struct page *page) { return folio_stable_node(page_folio(page)); } static inline void set_page_stable_node(struct page *page, - struct stable_node *stable_node) + struct ksm_stable_node *stable_node) { VM_BUG_ON_PAGE(PageAnon(page) && PageAnonExclusive(page), page); page->mapping = (void *)((unsigned long)stable_node | PAGE_MAPPING_KSM); @@ -880,7 +851,7 @@ static inline void set_page_stable_node(struct page *page, /* * Only called through the sysfs control interface: */ -static int remove_stable_node(struct stable_node *stable_node) +static int remove_stable_node(struct ksm_stable_node *stable_node) { struct page *page; int err; @@ -918,10 +889,10 @@ static int remove_stable_node(struct stable_node *stable_node) return err; } -static int remove_stable_node_chain(struct stable_node *stable_node, +static int remove_stable_node_chain(struct ksm_stable_node *stable_node, struct rb_root *root) { - struct stable_node *dup; + struct ksm_stable_node *dup; struct hlist_node *hlist_safe; if (!is_stable_node_chain(stable_node)) { @@ -945,14 +916,14 @@ static int remove_stable_node_chain(struct stable_node *stable_node, static int remove_all_stable_nodes(void) { - struct stable_node *stable_node, *next; + struct ksm_stable_node *stable_node, *next; int nid; int err = 0; for (nid = 0; nid < ksm_nr_node_ids; nid++) { while (root_stable_tree[nid].rb_node) { stable_node = rb_entry(root_stable_tree[nid].rb_node, - struct stable_node, node); + struct ksm_stable_node, node); if (remove_stable_node_chain(stable_node, root_stable_tree + nid)) { err = -EBUSY; @@ -971,21 +942,25 @@ static int remove_all_stable_nodes(void) static int unmerge_and_remove_all_rmap_items(void) { - struct mm_slot *mm_slot; + struct ksm_mm_slot *mm_slot; + struct mm_slot *slot; struct mm_struct *mm; struct vm_area_struct *vma; int err = 0; spin_lock(&ksm_mmlist_lock); - ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next, - struct mm_slot, mm_list); + slot = list_entry(ksm_mm_head.slot.mm_node.next, + struct mm_slot, mm_node); + ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); spin_unlock(&ksm_mmlist_lock); - for (mm_slot = ksm_scan.mm_slot; - mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) { - mm = mm_slot->mm; + for (mm_slot = ksm_scan.mm_slot; mm_slot != &ksm_mm_head; + mm_slot = ksm_scan.mm_slot) { + VMA_ITERATOR(vmi, mm_slot->slot.mm, 0); + + mm = mm_slot->slot.mm; mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (ksm_test_exit(mm)) break; if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma) @@ -1000,14 +975,15 @@ static int unmerge_and_remove_all_rmap_items(void) mmap_read_unlock(mm); spin_lock(&ksm_mmlist_lock); - ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next, - struct mm_slot, mm_list); + slot = list_entry(mm_slot->slot.mm_node.next, + struct mm_slot, mm_node); + ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (ksm_test_exit(mm)) { - hash_del(&mm_slot->link); - list_del(&mm_slot->mm_list); + hash_del(&mm_slot->slot.hash); + list_del(&mm_slot->slot.mm_node); spin_unlock(&ksm_mmlist_lock); - free_mm_slot(mm_slot); + mm_slot_free(mm_slot_cache, mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmdrop(mm); } else @@ -1095,6 +1071,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct page *page, goto out_unlock; } + /* See page_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && page_try_share_anon_rmap(page)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; @@ -1133,7 +1110,9 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, struct page *kpage, pte_t orig_pte) { struct mm_struct *mm = vma->vm_mm; + struct folio *folio; pmd_t *pmd; + pmd_t pmde; pte_t *ptep; pte_t newpte; spinlock_t *ptl; @@ -1148,6 +1127,15 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, pmd = mm_find_pmd(mm, addr); if (!pmd) goto out; + /* + * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() + * without holding anon_vma lock for write. So when looking for a + * genuine pmde (in which to find pte), test present and !THP together. + */ + pmde = *pmd; + barrier(); + if (!pmd_present(pmde) || pmd_trans_huge(pmde)) + goto out; mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, addr, addr + PAGE_SIZE); @@ -1191,10 +1179,11 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, ptep_clear_flush(vma, addr, ptep); set_pte_at_notify(mm, addr, ptep, newpte); + folio = page_folio(page); page_remove_rmap(page, vma, false); - if (!page_mapped(page)) - try_to_free_swap(page); - put_page(page); + if (!folio_mapped(folio)) + folio_free_swap(folio); + folio_put(folio); pte_unmap_unlock(ptep, ptl); err = 0; @@ -1278,7 +1267,7 @@ out: * * This function returns 0 if the pages were merged, -EFAULT otherwise. */ -static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item, +static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item, struct page *page, struct page *kpage) { struct mm_struct *mm = rmap_item->mm; @@ -1315,9 +1304,9 @@ out: * Note that this function upgrades page to ksm page: if one of the pages * is already a ksm page, try_to_merge_with_ksm_page should be used. */ -static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, +static struct page *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item, struct page *page, - struct rmap_item *tree_rmap_item, + struct ksm_rmap_item *tree_rmap_item, struct page *tree_page) { int err; @@ -1337,7 +1326,7 @@ static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item, } static __always_inline -bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset) +bool __is_page_sharing_candidate(struct ksm_stable_node *stable_node, int offset) { VM_BUG_ON(stable_node->rmap_hlist_len < 0); /* @@ -1351,17 +1340,17 @@ bool __is_page_sharing_candidate(struct stable_node *stable_node, int offset) } static __always_inline -bool is_page_sharing_candidate(struct stable_node *stable_node) +bool is_page_sharing_candidate(struct ksm_stable_node *stable_node) { return __is_page_sharing_candidate(stable_node, 0); } -static struct page *stable_node_dup(struct stable_node **_stable_node_dup, - struct stable_node **_stable_node, +static struct page *stable_node_dup(struct ksm_stable_node **_stable_node_dup, + struct ksm_stable_node **_stable_node, struct rb_root *root, bool prune_stale_stable_nodes) { - struct stable_node *dup, *found = NULL, *stable_node = *_stable_node; + struct ksm_stable_node *dup, *found = NULL, *stable_node = *_stable_node; struct hlist_node *hlist_safe; struct page *_tree_page, *tree_page = NULL; int nr = 0; @@ -1475,7 +1464,7 @@ static struct page *stable_node_dup(struct stable_node **_stable_node_dup, return tree_page; } -static struct stable_node *stable_node_dup_any(struct stable_node *stable_node, +static struct ksm_stable_node *stable_node_dup_any(struct ksm_stable_node *stable_node, struct rb_root *root) { if (!is_stable_node_chain(stable_node)) @@ -1502,12 +1491,12 @@ static struct stable_node *stable_node_dup_any(struct stable_node *stable_node, * function and will be overwritten in all cases, the caller doesn't * need to initialize it. */ -static struct page *__stable_node_chain(struct stable_node **_stable_node_dup, - struct stable_node **_stable_node, +static struct page *__stable_node_chain(struct ksm_stable_node **_stable_node_dup, + struct ksm_stable_node **_stable_node, struct rb_root *root, bool prune_stale_stable_nodes) { - struct stable_node *stable_node = *_stable_node; + struct ksm_stable_node *stable_node = *_stable_node; if (!is_stable_node_chain(stable_node)) { if (is_page_sharing_candidate(stable_node)) { *_stable_node_dup = stable_node; @@ -1524,18 +1513,18 @@ static struct page *__stable_node_chain(struct stable_node **_stable_node_dup, prune_stale_stable_nodes); } -static __always_inline struct page *chain_prune(struct stable_node **s_n_d, - struct stable_node **s_n, +static __always_inline struct page *chain_prune(struct ksm_stable_node **s_n_d, + struct ksm_stable_node **s_n, struct rb_root *root) { return __stable_node_chain(s_n_d, s_n, root, true); } -static __always_inline struct page *chain(struct stable_node **s_n_d, - struct stable_node *s_n, +static __always_inline struct page *chain(struct ksm_stable_node **s_n_d, + struct ksm_stable_node *s_n, struct rb_root *root) { - struct stable_node *old_stable_node = s_n; + struct ksm_stable_node *old_stable_node = s_n; struct page *tree_page; tree_page = __stable_node_chain(s_n_d, &s_n, root, false); @@ -1559,8 +1548,8 @@ static struct page *stable_tree_search(struct page *page) struct rb_root *root; struct rb_node **new; struct rb_node *parent; - struct stable_node *stable_node, *stable_node_dup, *stable_node_any; - struct stable_node *page_node; + struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any; + struct ksm_stable_node *page_node; page_node = page_stable_node(page); if (page_node && page_node->head != &migrate_nodes) { @@ -1580,7 +1569,7 @@ again: int ret; cond_resched(); - stable_node = rb_entry(*new, struct stable_node, node); + stable_node = rb_entry(*new, struct ksm_stable_node, node); stable_node_any = NULL; tree_page = chain_prune(&stable_node_dup, &stable_node, root); /* @@ -1803,14 +1792,14 @@ chain_append: * This function returns the stable tree node just allocated on success, * NULL otherwise. */ -static struct stable_node *stable_tree_insert(struct page *kpage) +static struct ksm_stable_node *stable_tree_insert(struct page *kpage) { int nid; unsigned long kpfn; struct rb_root *root; struct rb_node **new; struct rb_node *parent; - struct stable_node *stable_node, *stable_node_dup, *stable_node_any; + struct ksm_stable_node *stable_node, *stable_node_dup, *stable_node_any; bool need_chain = false; kpfn = page_to_pfn(kpage); @@ -1825,7 +1814,7 @@ again: int ret; cond_resched(); - stable_node = rb_entry(*new, struct stable_node, node); + stable_node = rb_entry(*new, struct ksm_stable_node, node); stable_node_any = NULL; tree_page = chain(&stable_node_dup, stable_node, root); if (!stable_node_dup) { @@ -1894,7 +1883,7 @@ again: rb_insert_color(&stable_node_dup->node, root); } else { if (!is_stable_node_chain(stable_node)) { - struct stable_node *orig = stable_node; + struct ksm_stable_node *orig = stable_node; /* chain is missing so create it */ stable_node = alloc_stable_node_chain(orig, root); if (!stable_node) { @@ -1923,7 +1912,7 @@ again: * the same walking algorithm in an rbtree. */ static -struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, +struct ksm_rmap_item *unstable_tree_search_insert(struct ksm_rmap_item *rmap_item, struct page *page, struct page **tree_pagep) { @@ -1937,12 +1926,12 @@ struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, new = &root->rb_node; while (*new) { - struct rmap_item *tree_rmap_item; + struct ksm_rmap_item *tree_rmap_item; struct page *tree_page; int ret; cond_resched(); - tree_rmap_item = rb_entry(*new, struct rmap_item, node); + tree_rmap_item = rb_entry(*new, struct ksm_rmap_item, node); tree_page = get_mergeable_page(tree_rmap_item); if (!tree_page) return NULL; @@ -1994,8 +1983,8 @@ struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item, * rmap_items hanging off a given node of the stable tree, all sharing * the same ksm page. */ -static void stable_tree_append(struct rmap_item *rmap_item, - struct stable_node *stable_node, +static void stable_tree_append(struct ksm_rmap_item *rmap_item, + struct ksm_stable_node *stable_node, bool max_page_sharing_bypass) { /* @@ -2037,12 +2026,12 @@ static void stable_tree_append(struct rmap_item *rmap_item, * @page: the page that we are searching identical page to. * @rmap_item: the reverse mapping into the virtual address of this page */ -static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) +static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_item) { struct mm_struct *mm = rmap_item->mm; - struct rmap_item *tree_rmap_item; + struct ksm_rmap_item *tree_rmap_item; struct page *tree_page = NULL; - struct stable_node *stable_node; + struct ksm_stable_node *stable_node; struct page *kpage; unsigned int checksum; int err; @@ -2198,11 +2187,11 @@ static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item) } } -static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, - struct rmap_item **rmap_list, +static struct ksm_rmap_item *get_next_rmap_item(struct ksm_mm_slot *mm_slot, + struct ksm_rmap_item **rmap_list, unsigned long addr) { - struct rmap_item *rmap_item; + struct ksm_rmap_item *rmap_item; while (*rmap_list) { rmap_item = *rmap_list; @@ -2218,7 +2207,8 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, rmap_item = alloc_rmap_item(); if (rmap_item) { /* It has already been zeroed */ - rmap_item->mm = mm_slot->mm; + rmap_item->mm = mm_slot->slot.mm; + rmap_item->mm->ksm_rmap_items++; rmap_item->address = addr; rmap_item->rmap_list = *rmap_list; *rmap_list = rmap_item; @@ -2226,19 +2216,21 @@ static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot, return rmap_item; } -static struct rmap_item *scan_get_next_rmap_item(struct page **page) +static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) { struct mm_struct *mm; + struct ksm_mm_slot *mm_slot; struct mm_slot *slot; struct vm_area_struct *vma; - struct rmap_item *rmap_item; + struct ksm_rmap_item *rmap_item; + struct vma_iterator vmi; int nid; - if (list_empty(&ksm_mm_head.mm_list)) + if (list_empty(&ksm_mm_head.slot.mm_node)) return NULL; - slot = ksm_scan.mm_slot; - if (slot == &ksm_mm_head) { + mm_slot = ksm_scan.mm_slot; + if (mm_slot == &ksm_mm_head) { /* * A number of pages can hang around indefinitely on per-cpu * pagevecs, raised page count preventing write_protect_page @@ -2258,7 +2250,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) * so prune them once before each full scan. */ if (!ksm_merge_across_nodes) { - struct stable_node *stable_node, *next; + struct ksm_stable_node *stable_node, *next; struct page *page; list_for_each_entry_safe(stable_node, next, @@ -2275,28 +2267,31 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) root_unstable_tree[nid] = RB_ROOT; spin_lock(&ksm_mmlist_lock); - slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list); - ksm_scan.mm_slot = slot; + slot = list_entry(mm_slot->slot.mm_node.next, + struct mm_slot, mm_node); + mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); + ksm_scan.mm_slot = mm_slot; spin_unlock(&ksm_mmlist_lock); /* * Although we tested list_empty() above, a racing __ksm_exit * of the last mm on the list may have removed it since then. */ - if (slot == &ksm_mm_head) + if (mm_slot == &ksm_mm_head) return NULL; next_mm: ksm_scan.address = 0; - ksm_scan.rmap_list = &slot->rmap_list; + ksm_scan.rmap_list = &mm_slot->rmap_list; } + slot = &mm_slot->slot; mm = slot->mm; + vma_iter_init(&vmi, mm, ksm_scan.address); + mmap_read_lock(mm); if (ksm_test_exit(mm)) - vma = NULL; - else - vma = find_vma(mm, ksm_scan.address); + goto no_vmas; - for (; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (!(vma->vm_flags & VM_MERGEABLE)) continue; if (ksm_scan.address < vma->vm_start) @@ -2308,15 +2303,17 @@ next_mm: if (ksm_test_exit(mm)) break; *page = follow_page(vma, ksm_scan.address, FOLL_GET); - if (IS_ERR_OR_NULL(*page) || is_zone_device_page(*page)) { + if (IS_ERR_OR_NULL(*page)) { ksm_scan.address += PAGE_SIZE; cond_resched(); continue; } + if (is_zone_device_page(*page)) + goto next_page; if (PageAnon(*page)) { flush_anon_page(vma, *page, ksm_scan.address); flush_dcache_page(*page); - rmap_item = get_next_rmap_item(slot, + rmap_item = get_next_rmap_item(mm_slot, ksm_scan.rmap_list, ksm_scan.address); if (rmap_item) { ksm_scan.rmap_list = @@ -2327,6 +2324,7 @@ next_mm: mmap_read_unlock(mm); return rmap_item; } +next_page: put_page(*page); ksm_scan.address += PAGE_SIZE; cond_resched(); @@ -2334,8 +2332,9 @@ next_mm: } if (ksm_test_exit(mm)) { +no_vmas: ksm_scan.address = 0; - ksm_scan.rmap_list = &slot->rmap_list; + ksm_scan.rmap_list = &mm_slot->rmap_list; } /* * Nuke all the rmap_items that are above this current rmap: @@ -2344,8 +2343,9 @@ next_mm: remove_trailing_rmap_items(ksm_scan.rmap_list); spin_lock(&ksm_mmlist_lock); - ksm_scan.mm_slot = list_entry(slot->mm_list.next, - struct mm_slot, mm_list); + slot = list_entry(mm_slot->slot.mm_node.next, + struct mm_slot, mm_node); + ksm_scan.mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (ksm_scan.address == 0) { /* * We've completed a full scan of all vmas, holding mmap_lock @@ -2356,11 +2356,11 @@ next_mm: * or when all VM_MERGEABLE areas have been unmapped (and * mmap_lock then protects against race with MADV_MERGEABLE). */ - hash_del(&slot->link); - list_del(&slot->mm_list); + hash_del(&mm_slot->slot.hash); + list_del(&mm_slot->slot.mm_node); spin_unlock(&ksm_mmlist_lock); - free_mm_slot(slot); + mm_slot_free(mm_slot_cache, mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmap_read_unlock(mm); mmdrop(mm); @@ -2377,8 +2377,8 @@ next_mm: } /* Repeat until we've completed scanning the whole list */ - slot = ksm_scan.mm_slot; - if (slot != &ksm_mm_head) + mm_slot = ksm_scan.mm_slot; + if (mm_slot != &ksm_mm_head) goto next_mm; ksm_scan.seqnr++; @@ -2391,7 +2391,7 @@ next_mm: */ static void ksm_do_scan(unsigned int scan_npages) { - struct rmap_item *rmap_item; + struct ksm_rmap_item *rmap_item; struct page *page; while (scan_npages-- && likely(!freezing(current))) { @@ -2406,7 +2406,7 @@ static void ksm_do_scan(unsigned int scan_npages) static int ksmd_should_run(void) { - return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list); + return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.slot.mm_node); } static int ksm_scan_thread(void *nothing) @@ -2495,18 +2495,21 @@ EXPORT_SYMBOL_GPL(ksm_madvise); int __ksm_enter(struct mm_struct *mm) { - struct mm_slot *mm_slot; + struct ksm_mm_slot *mm_slot; + struct mm_slot *slot; int needs_wakeup; - mm_slot = alloc_mm_slot(); + mm_slot = mm_slot_alloc(mm_slot_cache); if (!mm_slot) return -ENOMEM; + slot = &mm_slot->slot; + /* Check ksm_run too? Would need tighter locking */ - needs_wakeup = list_empty(&ksm_mm_head.mm_list); + needs_wakeup = list_empty(&ksm_mm_head.slot.mm_node); spin_lock(&ksm_mmlist_lock); - insert_to_mm_slots_hash(mm, mm_slot); + mm_slot_insert(mm_slots_hash, mm, slot); /* * When KSM_RUN_MERGE (or KSM_RUN_STOP), * insert just behind the scanning cursor, to let the area settle @@ -2518,9 +2521,9 @@ int __ksm_enter(struct mm_struct *mm) * missed: then we might as well insert at the end of the list. */ if (ksm_run & KSM_RUN_UNMERGE) - list_add_tail(&mm_slot->mm_list, &ksm_mm_head.mm_list); + list_add_tail(&slot->mm_node, &ksm_mm_head.slot.mm_node); else - list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list); + list_add_tail(&slot->mm_node, &ksm_scan.mm_slot->slot.mm_node); spin_unlock(&ksm_mmlist_lock); set_bit(MMF_VM_MERGEABLE, &mm->flags); @@ -2534,7 +2537,8 @@ int __ksm_enter(struct mm_struct *mm) void __ksm_exit(struct mm_struct *mm) { - struct mm_slot *mm_slot; + struct ksm_mm_slot *mm_slot; + struct mm_slot *slot; int easy_to_free = 0; /* @@ -2547,21 +2551,22 @@ void __ksm_exit(struct mm_struct *mm) */ spin_lock(&ksm_mmlist_lock); - mm_slot = get_mm_slot(mm); + slot = mm_slot_lookup(mm_slots_hash, mm); + mm_slot = mm_slot_entry(slot, struct ksm_mm_slot, slot); if (mm_slot && ksm_scan.mm_slot != mm_slot) { if (!mm_slot->rmap_list) { - hash_del(&mm_slot->link); - list_del(&mm_slot->mm_list); + hash_del(&slot->hash); + list_del(&slot->mm_node); easy_to_free = 1; } else { - list_move(&mm_slot->mm_list, - &ksm_scan.mm_slot->mm_list); + list_move(&slot->mm_node, + &ksm_scan.mm_slot->slot.mm_node); } } spin_unlock(&ksm_mmlist_lock); if (easy_to_free) { - free_mm_slot(mm_slot); + mm_slot_free(mm_slot_cache, mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmdrop(mm); } else if (mm_slot) { @@ -2612,8 +2617,8 @@ struct page *ksm_might_need_to_copy(struct page *page, void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) { - struct stable_node *stable_node; - struct rmap_item *rmap_item; + struct ksm_stable_node *stable_node; + struct ksm_rmap_item *rmap_item; int search_new_forks = 0; VM_BUG_ON_FOLIO(!folio_test_ksm(folio), folio); @@ -2683,7 +2688,7 @@ again: #ifdef CONFIG_MIGRATION void folio_migrate_ksm(struct folio *newfolio, struct folio *folio) { - struct stable_node *stable_node; + struct ksm_stable_node *stable_node; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_locked(newfolio), newfolio); @@ -2716,7 +2721,7 @@ static void wait_while_offlining(void) } } -static bool stable_node_dup_remove_range(struct stable_node *stable_node, +static bool stable_node_dup_remove_range(struct ksm_stable_node *stable_node, unsigned long start_pfn, unsigned long end_pfn) { @@ -2732,12 +2737,12 @@ static bool stable_node_dup_remove_range(struct stable_node *stable_node, return false; } -static bool stable_node_chain_remove_range(struct stable_node *stable_node, +static bool stable_node_chain_remove_range(struct ksm_stable_node *stable_node, unsigned long start_pfn, unsigned long end_pfn, struct rb_root *root) { - struct stable_node *dup; + struct ksm_stable_node *dup; struct hlist_node *hlist_safe; if (!is_stable_node_chain(stable_node)) { @@ -2761,14 +2766,14 @@ static bool stable_node_chain_remove_range(struct stable_node *stable_node, static void ksm_check_stable_tree(unsigned long start_pfn, unsigned long end_pfn) { - struct stable_node *stable_node, *next; + struct ksm_stable_node *stable_node, *next; struct rb_node *node; int nid; for (nid = 0; nid < ksm_nr_node_ids; nid++) { node = rb_first(root_stable_tree + nid); while (node) { - stable_node = rb_entry(node, struct stable_node, node); + stable_node = rb_entry(node, struct ksm_stable_node, node); if (stable_node_chain_remove_range(stable_node, start_pfn, end_pfn, root_stable_tree + diff --git a/mm/madvise.c b/mm/madvise.c index 5f0f0948a50e4399a3115f6cd12ae8deae25f1bd..2baa93ca2310967ba897146480e4a7d6b86f4203 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -59,6 +59,7 @@ static int madvise_need_mmap_write(int behavior) case MADV_FREE: case MADV_POPULATE_READ: case MADV_POPULATE_WRITE: + case MADV_COLLAPSE: return 0; default: /* be safe, default to 1. list exceptions explicitly */ @@ -451,8 +452,11 @@ regular_page: continue; } - /* Do not interfere with other mappings of this page */ - if (page_mapcount(page) != 1) + /* + * Do not interfere with other mappings of this page and + * non-LRU page. + */ + if (!PageLRU(page) || page_mapcount(page) != 1) continue; VM_BUG_ON_PAGE(PageTransCompound(page), page); @@ -597,6 +601,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, struct vm_area_struct *vma = walk->vma; spinlock_t *ptl; pte_t *orig_pte, *pte, ptent; + struct folio *folio; struct page *page; int nr_swap = 0; unsigned long next; @@ -641,56 +646,56 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, page = vm_normal_page(vma, addr, ptent); if (!page || is_zone_device_page(page)) continue; + folio = page_folio(page); /* - * If pmd isn't transhuge but the page is THP and + * If pmd isn't transhuge but the folio is large and * is owned by only this process, split it and * deactivate all pages. */ - if (PageTransCompound(page)) { - if (page_mapcount(page) != 1) + if (folio_test_large(folio)) { + if (folio_mapcount(folio) != 1) goto out; - get_page(page); - if (!trylock_page(page)) { - put_page(page); + folio_get(folio); + if (!folio_trylock(folio)) { + folio_put(folio); goto out; } pte_unmap_unlock(orig_pte, ptl); - if (split_huge_page(page)) { - unlock_page(page); - put_page(page); + if (split_folio(folio)) { + folio_unlock(folio); + folio_put(folio); orig_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); goto out; } - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); orig_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); pte--; addr -= PAGE_SIZE; continue; } - VM_BUG_ON_PAGE(PageTransCompound(page), page); - - if (PageSwapCache(page) || PageDirty(page)) { - if (!trylock_page(page)) + if (folio_test_swapcache(folio) || folio_test_dirty(folio)) { + if (!folio_trylock(folio)) continue; /* - * If page is shared with others, we couldn't clear - * PG_dirty of the page. + * If folio is shared with others, we mustn't clear + * the folio's dirty flag. */ - if (page_mapcount(page) != 1) { - unlock_page(page); + if (folio_mapcount(folio) != 1) { + folio_unlock(folio); continue; } - if (PageSwapCache(page) && !try_to_free_swap(page)) { - unlock_page(page); + if (folio_test_swapcache(folio) && + !folio_free_swap(folio)) { + folio_unlock(folio); continue; } - ClearPageDirty(page); - unlock_page(page); + folio_clear_dirty(folio); + folio_unlock(folio); } if (pte_young(ptent) || pte_dirty(ptent)) { @@ -708,7 +713,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, set_pte_at(mm, addr, pte, ptent); tlb_remove_tlb_entry(tlb, pte, addr); } - mark_page_lazyfree(page); + mark_page_lazyfree(&folio->page); } out: if (nr_swap) { @@ -1057,6 +1062,8 @@ static int madvise_vma_behavior(struct vm_area_struct *vma, if (error) goto out; break; + case MADV_COLLAPSE: + return madvise_collapse(vma, prev, start, end); } anon_name = anon_vma_name(vma); @@ -1150,6 +1157,7 @@ madvise_behavior_valid(int behavior) #ifdef CONFIG_TRANSPARENT_HUGEPAGE case MADV_HUGEPAGE: case MADV_NOHUGEPAGE: + case MADV_COLLAPSE: #endif case MADV_DONTDUMP: case MADV_DODUMP: @@ -1166,13 +1174,13 @@ madvise_behavior_valid(int behavior) } } -static bool -process_madvise_behavior_valid(int behavior) +static bool process_madvise_behavior_valid(int behavior) { switch (behavior) { case MADV_COLD: case MADV_PAGEOUT: case MADV_WILLNEED: + case MADV_COLLAPSE: return true; default: return false; @@ -1238,7 +1246,7 @@ int madvise_walk_vmas(struct mm_struct *mm, unsigned long start, if (start >= end) break; if (prev) - vma = prev->vm_next; + vma = find_vma(mm, prev->vm_end); else /* madvise_remove dropped mmap_lock */ vma = find_vma(mm, start); } @@ -1339,6 +1347,7 @@ int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, * MADV_NOHUGEPAGE - mark the given range as not worth being backed by * transparent huge pages so the existing pages will not be * coalesced into THP and new pages will not be allocated as THP. + * MADV_COLLAPSE - synchronously coalesce pages into new THP. * MADV_DONTDUMP - the application wants to prevent pages in the given range * from being included in its core dump. * MADV_DODUMP - cancel MADV_DONTDUMP: no longer exclude from core dump. diff --git a/mm/memblock.c b/mm/memblock.c index b5d3026979fccbb767458c2844ba1043ca47dd06..511d4783dcf1d86374c14d528cfec69c51247749 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2000,7 +2000,7 @@ static void __init free_unused_memmap(void) * presume that there are no holes in the memory map inside * a pageblock */ - start = round_down(start, pageblock_nr_pages); + start = pageblock_start_pfn(start); /* * If we had a previous bank, and there is a space @@ -2014,12 +2014,12 @@ static void __init free_unused_memmap(void) * presume that there are no holes in the memory map inside * a pageblock */ - prev_end = ALIGN(end, pageblock_nr_pages); + prev_end = pageblock_align(end); } #ifdef CONFIG_SPARSEMEM if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) { - prev_end = ALIGN(end, pageblock_nr_pages); + prev_end = pageblock_align(end); free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION)); } #endif diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b69979c9ced5c26d0bd238e0e090ce3851daa321..2d8549ae1b3004f78c244e578d3bf0ce36b0b844 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -88,13 +88,6 @@ static bool cgroup_memory_nosocket __ro_after_init; /* Kernel memory accounting disabled? */ static bool cgroup_memory_nokmem __ro_after_init; -/* Whether the swap controller is active */ -#ifdef CONFIG_MEMCG_SWAP -static bool cgroup_memory_noswap __ro_after_init; -#else -#define cgroup_memory_noswap 1 -#endif - #ifdef CONFIG_CGROUP_WRITEBACK static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); #endif @@ -102,7 +95,7 @@ static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq); /* Whether legacy memory+swap accounting is active */ static bool do_memsw_account(void) { - return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap; + return !cgroup_subsys_on_dfl(memory_cgrp_subsys); } #define THRESHOLDS_EVENTS_TARGET 128 @@ -597,25 +590,18 @@ static u64 flush_next_time; */ static void memcg_stats_lock(void) { -#ifdef CONFIG_PREEMPT_RT - preempt_disable(); -#else - VM_BUG_ON(!irqs_disabled()); -#endif + preempt_disable_nested(); + VM_WARN_ON_IRQS_ENABLED(); } static void __memcg_stats_lock(void) { -#ifdef CONFIG_PREEMPT_RT - preempt_disable(); -#endif + preempt_disable_nested(); } static void memcg_stats_unlock(void) { -#ifdef CONFIG_PREEMPT_RT - preempt_enable(); -#endif + preempt_enable_nested(); } static inline void memcg_rstat_updated(struct mem_cgroup *memcg, int val) @@ -669,6 +655,81 @@ static void flush_memcg_stats_dwork(struct work_struct *w) queue_delayed_work(system_unbound_wq, &stats_flush_dwork, FLUSH_TIME); } +/* Subset of vm_event_item to report for memcg event stats */ +static const unsigned int memcg_vm_event_stat[] = { + PGPGIN, + PGPGOUT, + PGSCAN_KSWAPD, + PGSCAN_DIRECT, + PGSTEAL_KSWAPD, + PGSTEAL_DIRECT, + PGFAULT, + PGMAJFAULT, + PGREFILL, + PGACTIVATE, + PGDEACTIVATE, + PGLAZYFREE, + PGLAZYFREED, +#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) + ZSWPIN, + ZSWPOUT, +#endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + THP_FAULT_ALLOC, + THP_COLLAPSE_ALLOC, +#endif +}; + +#define NR_MEMCG_EVENTS ARRAY_SIZE(memcg_vm_event_stat) +static int mem_cgroup_events_index[NR_VM_EVENT_ITEMS] __read_mostly; + +static void init_memcg_events(void) +{ + int i; + + for (i = 0; i < NR_MEMCG_EVENTS; ++i) + mem_cgroup_events_index[memcg_vm_event_stat[i]] = i + 1; +} + +static inline int memcg_events_index(enum vm_event_item idx) +{ + return mem_cgroup_events_index[idx] - 1; +} + +struct memcg_vmstats_percpu { + /* Local (CPU and cgroup) page state & events */ + long state[MEMCG_NR_STAT]; + unsigned long events[NR_MEMCG_EVENTS]; + + /* Delta calculation for lockless upward propagation */ + long state_prev[MEMCG_NR_STAT]; + unsigned long events_prev[NR_MEMCG_EVENTS]; + + /* Cgroup1: threshold notifications & softlimit tree updates */ + unsigned long nr_page_events; + unsigned long targets[MEM_CGROUP_NTARGETS]; +}; + +struct memcg_vmstats { + /* Aggregated (CPU and subtree) page state & events */ + long state[MEMCG_NR_STAT]; + unsigned long events[NR_MEMCG_EVENTS]; + + /* Pending child counts during tree propagation */ + long state_pending[MEMCG_NR_STAT]; + unsigned long events_pending[NR_MEMCG_EVENTS]; +}; + +unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +{ + long x = READ_ONCE(memcg->vmstats->state[idx]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + /** * __mod_memcg_state - update cgroup memory statistics * @memcg: the memory cgroup @@ -715,7 +776,7 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, * interrupt context while other caller need to have disabled interrupt. */ __memcg_stats_lock(); - if (IS_ENABLED(CONFIG_DEBUG_VM) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { + if (IS_ENABLED(CONFIG_DEBUG_VM)) { switch (idx) { case NR_ANON_MAPPED: case NR_FILE_MAPPED: @@ -725,7 +786,7 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx, WARN_ON_ONCE(!in_task()); break; default: - WARN_ON_ONCE(!irqs_disabled()); + VM_WARN_ON_IRQS_ENABLED(); } } @@ -816,27 +877,37 @@ void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val) void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count) { - if (mem_cgroup_disabled()) + int index = memcg_events_index(idx); + + if (mem_cgroup_disabled() || index < 0) return; memcg_stats_lock(); - __this_cpu_add(memcg->vmstats_percpu->events[idx], count); + __this_cpu_add(memcg->vmstats_percpu->events[index], count); memcg_rstat_updated(memcg, count); memcg_stats_unlock(); } static unsigned long memcg_events(struct mem_cgroup *memcg, int event) { - return READ_ONCE(memcg->vmstats.events[event]); + int index = memcg_events_index(event); + + if (index < 0) + return 0; + return READ_ONCE(memcg->vmstats->events[index]); } static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event) { long x = 0; int cpu; + int index = memcg_events_index(event); + + if (index < 0) + return 0; for_each_possible_cpu(cpu) - x += per_cpu(memcg->vmstats_percpu->events[event], cpu); + x += per_cpu(memcg->vmstats_percpu->events[index], cpu); return x; } @@ -1143,7 +1214,7 @@ static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg) } while ((memcg = parent_mem_cgroup(memcg))); /* - * When cgruop1 non-hierarchy mode is used, + * When cgroup1 non-hierarchy mode is used, * parent_mem_cgroup() does not walk all the way up to the * cgroup root (root_mem_cgroup). So we have to handle * dead_memcg from cgroup root separately. @@ -1401,6 +1472,7 @@ static const struct memory_stat memory_stats[] = { { "kernel", MEMCG_KMEM }, { "kernel_stack", NR_KERNEL_STACK_KB }, { "pagetables", NR_PAGETABLE }, + { "sec_pagetables", NR_SECONDARY_PAGETABLE }, { "percpu", MEMCG_PERCPU_B }, { "sock", MEMCG_SOCK }, { "vmalloc", MEMCG_VMALLOC }, @@ -1467,29 +1539,6 @@ static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, return memcg_page_state(memcg, item) * memcg_page_state_unit(item); } -/* Subset of vm_event_item to report for memcg event stats */ -static const unsigned int memcg_vm_event_stat[] = { - PGSCAN_KSWAPD, - PGSCAN_DIRECT, - PGSTEAL_KSWAPD, - PGSTEAL_DIRECT, - PGFAULT, - PGMAJFAULT, - PGREFILL, - PGACTIVATE, - PGDEACTIVATE, - PGLAZYFREE, - PGLAZYFREED, -#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP) - ZSWPIN, - ZSWPOUT, -#endif -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - THP_FAULT_ALLOC, - THP_COLLAPSE_ALLOC, -#endif -}; - static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize) { struct seq_buf s; @@ -1530,10 +1579,15 @@ static void memory_stat_format(struct mem_cgroup *memcg, char *buf, int bufsize) memcg_events(memcg, PGSTEAL_KSWAPD) + memcg_events(memcg, PGSTEAL_DIRECT)); - for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) + for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) { + if (memcg_vm_event_stat[i] == PGPGIN || + memcg_vm_event_stat[i] == PGPGOUT) + continue; + seq_buf_printf(&s, "%s %lu\n", vm_event_name(memcg_vm_event_stat[i]), memcg_events(memcg, memcg_vm_event_stat[i])); + } /* The above should easily fit into one page */ WARN_ON_ONCE(seq_buf_has_overflowed(&s)); @@ -1607,17 +1661,17 @@ unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) { unsigned long max = READ_ONCE(memcg->memory.max); - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) { - if (mem_cgroup_swappiness(memcg)) - max += min(READ_ONCE(memcg->swap.max), - (unsigned long)total_swap_pages); - } else { /* v1 */ + if (do_memsw_account()) { if (mem_cgroup_swappiness(memcg)) { /* Calculate swap excess capacity from memsw limit */ unsigned long swap = READ_ONCE(memcg->memsw.max) - max; max += min(swap, (unsigned long)total_swap_pages); } + } else { + if (mem_cgroup_swappiness(memcg)) + max += min(READ_ONCE(memcg->swap.max), + (unsigned long)total_swap_pages); } return max; } @@ -2789,6 +2843,7 @@ static void commit_charge(struct folio *folio, struct mem_cgroup *memcg) * - LRU isolation * - lock_page_memcg() * - exclusive reference + * - mem_cgroup_trylock_pages() */ folio->memcg_data = (unsigned long)memcg; } @@ -3362,7 +3417,7 @@ void split_page_memcg(struct page *head, unsigned int nr) css_get_many(&memcg->css, nr - 1); } -#ifdef CONFIG_MEMCG_SWAP +#ifdef CONFIG_SWAP /** * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record. * @entry: swap entry to be moved @@ -3975,6 +4030,8 @@ static const unsigned int memcg1_stats[] = { NR_FILE_MAPPED, NR_FILE_DIRTY, NR_WRITEBACK, + WORKINGSET_REFAULT_ANON, + WORKINGSET_REFAULT_FILE, MEMCG_SWAP, }; @@ -3988,6 +4045,8 @@ static const char *const memcg1_stat_names[] = { "mapped_file", "dirty", "writeback", + "workingset_refault_anon", + "workingset_refault_file", "swap", }; @@ -4016,7 +4075,8 @@ static int memcg_stat_show(struct seq_file *m, void *v) if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account()) continue; nr = memcg_page_state_local(memcg, memcg1_stats[i]); - seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE); + seq_printf(m, "%s %lu\n", memcg1_stat_names[i], + nr * memcg_page_state_unit(memcg1_stats[i])); } for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) @@ -4047,7 +4107,7 @@ static int memcg_stat_show(struct seq_file *m, void *v) continue; nr = memcg_page_state(memcg, memcg1_stats[i]); seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i], - (u64)nr * PAGE_SIZE); + (u64)nr * memcg_page_state_unit(memcg1_stats[i])); } for (i = 0; i < ARRAY_SIZE(memcg1_events); i++) @@ -5110,8 +5170,8 @@ struct mem_cgroup *mem_cgroup_get_from_ino(unsigned long ino) struct mem_cgroup *memcg; cgrp = cgroup_get_from_id(ino); - if (!cgrp) - return ERR_PTR(-ENOENT); + if (IS_ERR(cgrp)) + return ERR_CAST(cgrp); css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); if (css) @@ -5164,12 +5224,14 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) for_each_node(node) free_mem_cgroup_per_node_info(memcg, node); + kfree(memcg->vmstats); free_percpu(memcg->vmstats_percpu); kfree(memcg); } static void mem_cgroup_free(struct mem_cgroup *memcg) { + lru_gen_exit_memcg(memcg); memcg_wb_domain_exit(memcg); __mem_cgroup_free(memcg); } @@ -5192,6 +5254,10 @@ static struct mem_cgroup *mem_cgroup_alloc(void) goto fail; } + memcg->vmstats = kzalloc(sizeof(struct memcg_vmstats), GFP_KERNEL); + if (!memcg->vmstats) + goto fail; + memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu, GFP_KERNEL_ACCOUNT); if (!memcg->vmstats_percpu) @@ -5228,6 +5294,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) memcg->deferred_split_queue.split_queue_len = 0; #endif idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); + lru_gen_init_memcg(memcg); return memcg; fail: mem_cgroup_id_remove(memcg); @@ -5262,6 +5329,7 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_init(&memcg->kmem, &parent->kmem); page_counter_init(&memcg->tcpmem, &parent->tcpmem); } else { + init_memcg_events(); page_counter_init(&memcg->memory, NULL); page_counter_init(&memcg->swap, NULL); page_counter_init(&memcg->kmem, NULL); @@ -5410,9 +5478,9 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) * below us. We're in a per-cpu loop here and this is * a global counter, so the first cycle will get them. */ - delta = memcg->vmstats.state_pending[i]; + delta = memcg->vmstats->state_pending[i]; if (delta) - memcg->vmstats.state_pending[i] = 0; + memcg->vmstats->state_pending[i] = 0; /* Add CPU changes on this level since the last flush */ v = READ_ONCE(statc->state[i]); @@ -5425,15 +5493,15 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) continue; /* Aggregate counts on this level and propagate upwards */ - memcg->vmstats.state[i] += delta; + memcg->vmstats->state[i] += delta; if (parent) - parent->vmstats.state_pending[i] += delta; + parent->vmstats->state_pending[i] += delta; } - for (i = 0; i < NR_VM_EVENT_ITEMS; i++) { - delta = memcg->vmstats.events_pending[i]; + for (i = 0; i < NR_MEMCG_EVENTS; i++) { + delta = memcg->vmstats->events_pending[i]; if (delta) - memcg->vmstats.events_pending[i] = 0; + memcg->vmstats->events_pending[i] = 0; v = READ_ONCE(statc->events[i]); if (v != statc->events_prev[i]) { @@ -5444,9 +5512,9 @@ static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu) if (!delta) continue; - memcg->vmstats.events[i] += delta; + memcg->vmstats->events[i] += delta; if (parent) - parent->vmstats.events_pending[i] += delta; + parent->vmstats->events_pending[i] += delta; } for_each_node_state(nid, N_MEMORY) { @@ -5561,7 +5629,7 @@ static struct page *mc_handle_swap_pte(struct vm_area_struct *vma, return NULL; /* - * Because lookup_swap_cache() updates some statistics counter, + * Because swap_cache_get_folio() updates some statistics counter, * we call find_get_page() with swapper_space directly. */ page = find_get_page(swap_address_space(ent), swp_offset(ent)); @@ -5871,7 +5939,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm) unsigned long precharge; mmap_read_lock(mm); - walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL); + walk_page_range(mm, 0, ULONG_MAX, &precharge_walk_ops, NULL); mmap_read_unlock(mm); precharge = mc.precharge; @@ -6169,9 +6237,7 @@ retry: * When we have consumed all precharges and failed in doing * additional charge, the page walk just aborts. */ - walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops, - NULL); - + walk_page_range(mc.mm, 0, ULONG_MAX, &charge_walk_ops, NULL); mmap_read_unlock(mc.mm); atomic_dec(&mc.from->moving_account); } @@ -6196,6 +6262,30 @@ static void mem_cgroup_move_task(void) } #endif +#ifdef CONFIG_LRU_GEN +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *css; + + /* find the first leader if there is any */ + cgroup_taskset_for_each_leader(task, css, tset) + break; + + if (!task) + return; + + task_lock(task); + if (task->mm && READ_ONCE(task->mm->owner) == task) + lru_gen_migrate_mm(task->mm); + task_unlock(task); +} +#else +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ +} +#endif /* CONFIG_LRU_GEN */ + static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) { if (value == PAGE_COUNTER_MAX) @@ -6601,6 +6691,7 @@ struct cgroup_subsys memory_cgrp_subsys = { .css_reset = mem_cgroup_css_reset, .css_rstat_flush = mem_cgroup_css_rstat_flush, .can_attach = mem_cgroup_can_attach, + .attach = mem_cgroup_attach, .cancel_attach = mem_cgroup_cancel_attach, .post_attach = mem_cgroup_move_task, .dfl_cftypes = memory_files, @@ -6813,21 +6904,20 @@ int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp) } /** - * mem_cgroup_swapin_charge_page - charge a newly allocated page for swapin - * @page: page to charge + * mem_cgroup_swapin_charge_folio - Charge a newly allocated folio for swapin. + * @folio: folio to charge. * @mm: mm context of the victim * @gfp: reclaim mode - * @entry: swap entry for which the page is allocated + * @entry: swap entry for which the folio is allocated * - * This function charges a page allocated for swapin. Please call this before - * adding the page to the swapcache. + * This function charges a folio allocated for swapin. Please call this before + * adding the folio to the swapcache. * * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, +int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, gfp_t gfp, swp_entry_t entry) { - struct folio *folio = page_folio(page); struct mem_cgroup *memcg; unsigned short id; int ret; @@ -7200,7 +7290,7 @@ static int __init mem_cgroup_init(void) } subsys_initcall(mem_cgroup_init); -#ifdef CONFIG_MEMCG_SWAP +#ifdef CONFIG_SWAP static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg) { while (!refcount_inc_not_zero(&memcg->id.ref)) { @@ -7238,7 +7328,7 @@ void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) if (mem_cgroup_disabled()) return; - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) + if (!do_memsw_account()) return; memcg = folio_memcg(folio); @@ -7267,7 +7357,7 @@ void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry) if (!mem_cgroup_is_root(memcg)) page_counter_uncharge(&memcg->memory, nr_entries); - if (!cgroup_memory_noswap && memcg != swap_memcg) { + if (memcg != swap_memcg) { if (!mem_cgroup_is_root(swap_memcg)) page_counter_charge(&swap_memcg->memsw, nr_entries); page_counter_uncharge(&memcg->memsw, nr_entries); @@ -7303,7 +7393,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry) struct mem_cgroup *memcg; unsigned short oldid; - if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) + if (do_memsw_account()) return 0; memcg = folio_memcg(folio); @@ -7319,7 +7409,7 @@ int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry) memcg = mem_cgroup_id_get_online(memcg); - if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) && + if (!mem_cgroup_is_root(memcg) && !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) { memcg_memory_event(memcg, MEMCG_SWAP_MAX); memcg_memory_event(memcg, MEMCG_SWAP_FAIL); @@ -7347,15 +7437,18 @@ void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) struct mem_cgroup *memcg; unsigned short id; + if (mem_cgroup_disabled()) + return; + id = swap_cgroup_record(entry, 0, nr_pages); rcu_read_lock(); memcg = mem_cgroup_from_id(id); if (memcg) { - if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) { - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - page_counter_uncharge(&memcg->swap, nr_pages); - else + if (!mem_cgroup_is_root(memcg)) { + if (do_memsw_account()) page_counter_uncharge(&memcg->memsw, nr_pages); + else + page_counter_uncharge(&memcg->swap, nr_pages); } mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages); mem_cgroup_id_put_many(memcg, nr_pages); @@ -7367,7 +7460,7 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) { long nr_swap_pages = get_nr_swap_pages(); - if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) + if (mem_cgroup_disabled() || do_memsw_account()) return nr_swap_pages; for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) nr_swap_pages = min_t(long, nr_swap_pages, @@ -7376,18 +7469,18 @@ long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) return nr_swap_pages; } -bool mem_cgroup_swap_full(struct page *page) +bool mem_cgroup_swap_full(struct folio *folio) { struct mem_cgroup *memcg; - VM_BUG_ON_PAGE(!PageLocked(page), page); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); if (vm_swap_full()) return true; - if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys)) + if (do_memsw_account()) return false; - memcg = page_memcg(page); + memcg = folio_memcg(folio); if (!memcg) return false; @@ -7404,10 +7497,9 @@ bool mem_cgroup_swap_full(struct page *page) static int __init setup_swap_account(char *s) { - if (!strcmp(s, "1")) - cgroup_memory_noswap = false; - else if (!strcmp(s, "0")) - cgroup_memory_noswap = true; + pr_warn_once("The swapaccount= commandline option is deprecated. " + "Please report your usecase to linux-mm@kvack.org if you " + "depend on this functionality.\n"); return 1; } __setup("swapaccount=", setup_swap_account); @@ -7676,20 +7768,9 @@ static struct cftype zswap_files[] = { }; #endif /* CONFIG_MEMCG_KMEM && CONFIG_ZSWAP */ -/* - * If mem_cgroup_swap_init() is implemented as a subsys_initcall() - * instead of a core_initcall(), this could mean cgroup_memory_noswap still - * remains set to false even when memcg is disabled via "cgroup_disable=memory" - * boot parameter. This may result in premature OOPS inside - * mem_cgroup_get_nr_swap_pages() function in corner cases. - */ static int __init mem_cgroup_swap_init(void) { - /* No memory control -> no swap control */ if (mem_cgroup_disabled()) - cgroup_memory_noswap = true; - - if (cgroup_memory_noswap) return 0; WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files)); @@ -7699,6 +7780,6 @@ static int __init mem_cgroup_swap_init(void) #endif return 0; } -core_initcall(mem_cgroup_swap_init); +subsys_initcall(mem_cgroup_swap_init); -#endif /* CONFIG_MEMCG_SWAP */ +#endif /* CONFIG_SWAP */ diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 14439806b5efc40327aefa05777422bcfca50611..145bb561ddb3ad2b8a0b723a4fec4fc2ef487254 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -277,7 +277,7 @@ static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) * to SIG_IGN, but hopefully no one will do that? */ ret = send_sig_mceerr(BUS_MCEERR_AO, (void __user *)tk->addr, - addr_lsb, t); /* synchronous? */ + addr_lsb, t); if (ret < 0) pr_info("Error sending signal to %s:%d: %d\n", t->comm, t->pid, ret); @@ -345,13 +345,17 @@ static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, * not much we can do. We just print a message and ignore otherwise. */ +#define FSDAX_INVALID_PGOFF ULONG_MAX + /* * Schedule a process for later kill. * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM. * - * Notice: @fsdax_pgoff is used only when @p is a fsdax page. - * In other cases, such as anonymous and file-backend page, the address to be - * killed can be caculated by @p itself. + * Note: @fsdax_pgoff is used only when @p is a fsdax page and a + * filesystem with a memory failure handler has claimed the + * memory_failure event. In all other cases, page->index and + * page->mapping are sufficient for mapping the page back to its + * corresponding user virtual address. */ static void add_to_kill(struct task_struct *tsk, struct page *p, pgoff_t fsdax_pgoff, struct vm_area_struct *vma, @@ -367,11 +371,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p, tk->addr = page_address_in_vma(p, vma); if (is_zone_device_page(p)) { - /* - * Since page->mapping is not used for fsdax, we need - * calculate the address based on the vma. - */ - if (p->pgmap->type == MEMORY_DEVICE_FS_DAX) + if (fsdax_pgoff != FSDAX_INVALID_PGOFF) tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma); tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr); } else @@ -413,7 +413,7 @@ static void kill_procs(struct list_head *to_kill, int forcekill, bool fail, { struct to_kill *tk, *next; - list_for_each_entry_safe (tk, next, to_kill, nd) { + list_for_each_entry_safe(tk, next, to_kill, nd) { if (forcekill) { /* * In case something went wrong with munmapping @@ -437,6 +437,7 @@ static void kill_procs(struct list_head *to_kill, int forcekill, bool fail, pr_err("%#lx: Cannot send advisory machine check signal to %s:%d\n", pfn, tk->tsk->comm, tk->tsk->pid); } + list_del(&tk->nd); put_task_struct(tk->tsk); kfree(tk); } @@ -520,14 +521,15 @@ static void collect_procs_anon(struct page *page, struct list_head *to_kill, anon_vma_interval_tree_foreach(vmac, &av->rb_root, pgoff, pgoff) { vma = vmac->vma; + if (vma->vm_mm != t->mm) + continue; if (!page_mapped_in_vma(page, vma)) continue; - if (vma->vm_mm == t->mm) - add_to_kill(t, page, 0, vma, to_kill); + add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, to_kill); } } read_unlock(&tasklist_lock); - page_unlock_anon_vma_read(av); + anon_vma_unlock_read(av); } /* @@ -559,7 +561,8 @@ static void collect_procs_file(struct page *page, struct list_head *to_kill, * to be informed of all such data corruptions. */ if (vma->vm_mm == t->mm) - add_to_kill(t, page, 0, vma, to_kill); + add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma, + to_kill); } } read_unlock(&tasklist_lock); @@ -632,7 +635,7 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, swp_entry_t swp = pte_to_swp_entry(pte); if (is_hwpoison_entry(swp)) - pfn = hwpoison_entry_to_pfn(swp); + pfn = swp_offset_pfn(swp); } if (!pfn || pfn != poisoned_pfn) @@ -743,6 +746,9 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn, }; priv.tk.tsk = p; + if (!p->mm) + return -EFAULT; + mmap_read_lock(p->mm); ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwp_walk_ops, (void *)&priv); @@ -1243,9 +1249,9 @@ static int __get_hwpoison_page(struct page *page, unsigned long flags) return ret; /* - * This check prevents from calling get_hwpoison_unless_zero() - * for any unsupported type of page in order to reduce the risk of - * unexpected races caused by taking a page refcount. + * This check prevents from calling get_page_unless_zero() for any + * unsupported type of page in order to reduce the risk of unexpected + * races caused by taking a page refcount. */ if (!HWPoisonHandlable(head, flags)) return -EBUSY; @@ -1396,14 +1402,14 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, struct address_space *mapping; LIST_HEAD(tokill); bool unmap_success; - int kill = 1, forcekill; + int forcekill; bool mlocked = PageMlocked(hpage); /* * Here we are interested only in user-mapped pages, so skip any * other types of pages. */ - if (PageReserved(p) || PageSlab(p)) + if (PageReserved(p) || PageSlab(p) || PageTable(p)) return true; if (!(PageLRU(hpage) || PageHuge(p))) return true; @@ -1437,7 +1443,6 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, if (page_mkclean(hpage)) { SetPageDirty(hpage); } else { - kill = 0; ttu |= TTU_IGNORE_HWPOISON; pr_info("%#lx: corrupted page was clean: dropped without side effects\n", pfn); @@ -1448,12 +1453,8 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * First collect all the processes that have the page * mapped in dirty form. This has to be done before try_to_unmap, * because ttu takes the rmap data structures down. - * - * Error handling: We ignore errors here because - * there's nothing that can be done. */ - if (kill) - collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); + collect_procs(hpage, &tokill, flags & MF_ACTION_REQUIRED); if (PageHuge(hpage) && !PageAnon(hpage)) { /* @@ -1495,7 +1496,8 @@ static bool hwpoison_user_mappings(struct page *p, unsigned long pfn, * use a more force-full uncatchable kill to prevent * any accesses to the poisoned memory. */ - forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL); + forcekill = PageDirty(hpage) || (flags & MF_MUST_KILL) || + !unmap_success; kill_procs(&tokill, forcekill, !unmap_success, pfn, flags); return unmap_success; @@ -1524,20 +1526,18 @@ static int identify_page_state(unsigned long pfn, struct page *p, return page_action(ps, p, pfn); } -static int try_to_split_thp_page(struct page *page, const char *msg) +static int try_to_split_thp_page(struct page *page) { + int ret; + lock_page(page); - if (unlikely(split_huge_page(page))) { - unsigned long pfn = page_to_pfn(page); + ret = split_huge_page(page); + unlock_page(page); - unlock_page(page); - pr_info("%s: %#lx: thp split failed\n", msg, pfn); + if (unlikely(ret)) put_page(page); - return -EBUSY; - } - unlock_page(page); - return 0; + return ret; } static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn, @@ -1862,8 +1862,10 @@ retry: if (hwpoison_filter(p)) { hugetlb_clear_page_hwpoison(head); - res = -EOPNOTSUPP; - goto out; + unlock_page(head); + if (res == 1) + put_page(head); + return -EOPNOTSUPP; } /* @@ -1928,7 +1930,7 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags, * Call driver's implementation to handle the memory failure, otherwise * fall back to generic handler. */ - if (pgmap->ops->memory_failure) { + if (pgmap_has_memory_failure(pgmap)) { rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); /* * Fall back to generic handler too if operation is not @@ -2026,7 +2028,7 @@ try_again: /* * We need/can do nothing about count=0 pages. * 1) it's a free page, and therefore in safe hand: - * prep_new_page() will be the gate keeper. + * check_new_page() will be the gate keeper. * 2) it's part of a non-compound high order page. * Implies some kernel user: cannot stop them from * R/W the page; let's pray that the page has been @@ -2079,7 +2081,7 @@ try_again: * page is a valid handlable page. */ SetPageHasHWPoisoned(hpage); - if (try_to_split_thp_page(p, "Memory Failure") < 0) { + if (try_to_split_thp_page(p) < 0) { action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED); res = -EBUSY; goto unlock_mutex; @@ -2129,7 +2131,7 @@ try_again: page_flags = p->flags; if (hwpoison_filter(p)) { - TestClearPageHWPoison(p); + ClearPageHWPoison(p); unlock_page(p); put_page(p); res = -EOPNOTSUPP; @@ -2354,7 +2356,7 @@ int unpoison_memory(unsigned long pfn) goto unlock_mutex; } - if (PageSlab(page) || PageTable(page)) + if (PageSlab(page) || PageTable(page) || PageReserved(page)) goto unlock_mutex; ret = get_hwpoison_page(p, MF_UNPOISON); @@ -2378,13 +2380,14 @@ int unpoison_memory(unsigned long pfn) count = free_raw_hwp_pages(page, false); if (count == 0) { ret = -EBUSY; + put_page(page); goto unlock_mutex; } } freeit = !!TestClearPageHWPoison(p); put_page(page); - if (freeit && !(pfn == my_zero_pfn(0) && page_count(p) == 1)) { + if (freeit) { put_page(page); ret = 0; } @@ -2404,24 +2407,26 @@ EXPORT_SYMBOL(unpoison_memory); static bool isolate_page(struct page *page, struct list_head *pagelist) { bool isolated = false; - bool lru = PageLRU(page); if (PageHuge(page)) { isolated = !isolate_hugetlb(page, pagelist); } else { + bool lru = !__PageMovable(page); + if (lru) isolated = !isolate_lru_page(page); else - isolated = !isolate_movable_page(page, ISOLATE_UNEVICTABLE); + isolated = !isolate_movable_page(page, + ISOLATE_UNEVICTABLE); - if (isolated) + if (isolated) { list_add(&page->lru, pagelist); + if (lru) + inc_node_page_state(page, NR_ISOLATED_ANON + + page_is_file_lru(page)); + } } - if (isolated && lru) - inc_node_page_state(page, NR_ISOLATED_ANON + - page_is_file_lru(page)); - /* * If we succeed to isolate the page, we grabbed another refcount on * the page, so we can safely drop the one we got from get_any_pages(). @@ -2434,11 +2439,11 @@ static bool isolate_page(struct page *page, struct list_head *pagelist) } /* - * __soft_offline_page handles hugetlb-pages and non-hugetlb pages. + * soft_offline_in_use_page handles hugetlb-pages and non-hugetlb pages. * If the page is a non-dirty unmapped page-cache page, it simply invalidates. * If the page is mapped, it migrates the contents over. */ -static int __soft_offline_page(struct page *page) +static int soft_offline_in_use_page(struct page *page) { long ret = 0; unsigned long pfn = page_to_pfn(page); @@ -2451,6 +2456,14 @@ static int __soft_offline_page(struct page *page) .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL, }; + if (!huge && PageTransHuge(hpage)) { + if (try_to_split_thp_page(page)) { + pr_info("soft offline: %#lx: thp split failed\n", pfn); + return -EBUSY; + } + hpage = page; + } + lock_page(page); if (!PageHuge(page)) wait_on_page_writeback(page); @@ -2500,26 +2513,6 @@ static int __soft_offline_page(struct page *page) return ret; } -static int soft_offline_in_use_page(struct page *page) -{ - struct page *hpage = compound_head(page); - - if (!PageHuge(page) && PageTransHuge(hpage)) - if (try_to_split_thp_page(page, "soft offline") < 0) - return -EBUSY; - return __soft_offline_page(page); -} - -static int soft_offline_free_page(struct page *page) -{ - int rc = 0; - - if (!page_handle_poison(page, true, false)) - rc = -EBUSY; - - return rc; -} - static void put_ref_page(struct page *page) { if (page) @@ -2587,8 +2580,6 @@ retry: if (hwpoison_filter(page)) { if (ret > 0) put_page(page); - else - put_ref_page(ref_page); mutex_unlock(&mf_mutex); return -EOPNOTSUPP; @@ -2597,7 +2588,7 @@ retry: if (ret > 0) { ret = soft_offline_in_use_page(page); } else if (ret == 0) { - if (soft_offline_free_page(page) && try_again) { + if (!page_handle_poison(page, true, false) && try_again) { try_again = false; flags &= ~MF_COUNT_INCREASED; goto retry; @@ -2611,7 +2602,7 @@ retry: void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) { - int i; + int i, total = 0; /* * A further optimization is to have per section refcounted @@ -2624,8 +2615,10 @@ void clear_hwpoisoned_pages(struct page *memmap, int nr_pages) for (i = 0; i < nr_pages; i++) { if (PageHWPoison(&memmap[i])) { - num_poisoned_pages_dec(); + total++; ClearPageHWPoison(&memmap[i]); } } + if (total) + num_poisoned_pages_sub(total); } diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c new file mode 100644 index 0000000000000000000000000000000000000000..f116b7b6333e50f1c3f6c2a7717afe30aa2b1629 --- /dev/null +++ b/mm/memory-tiers.c @@ -0,0 +1,732 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct memory_tier { + /* hierarchy of memory tiers */ + struct list_head list; + /* list of all memory types part of this tier */ + struct list_head memory_types; + /* + * start value of abstract distance. memory tier maps + * an abstract distance range, + * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE + */ + int adistance_start; + struct device dev; + /* All the nodes that are part of all the lower memory tiers. */ + nodemask_t lower_tier_mask; +}; + +struct demotion_nodes { + nodemask_t preferred; +}; + +struct node_memory_type_map { + struct memory_dev_type *memtype; + int map_count; +}; + +static DEFINE_MUTEX(memory_tier_lock); +static LIST_HEAD(memory_tiers); +static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; +static struct memory_dev_type *default_dram_type; + +static struct bus_type memory_tier_subsys = { + .name = "memory_tiering", + .dev_name = "memory_tier", +}; + +#ifdef CONFIG_MIGRATION +static int top_tier_adistance; +/* + * node_demotion[] examples: + * + * Example 1: + * + * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes. + * + * node distances: + * node 0 1 2 3 + * 0 10 20 30 40 + * 1 20 10 40 30 + * 2 30 40 10 40 + * 3 40 30 40 10 + * + * memory_tiers0 = 0-1 + * memory_tiers1 = 2-3 + * + * node_demotion[0].preferred = 2 + * node_demotion[1].preferred = 3 + * node_demotion[2].preferred = + * node_demotion[3].preferred = + * + * Example 2: + * + * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node. + * + * node distances: + * node 0 1 2 + * 0 10 20 30 + * 1 20 10 30 + * 2 30 30 10 + * + * memory_tiers0 = 0-2 + * + * node_demotion[0].preferred = + * node_demotion[1].preferred = + * node_demotion[2].preferred = + * + * Example 3: + * + * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node. + * + * node distances: + * node 0 1 2 + * 0 10 20 30 + * 1 20 10 40 + * 2 30 40 10 + * + * memory_tiers0 = 1 + * memory_tiers1 = 0 + * memory_tiers2 = 2 + * + * node_demotion[0].preferred = 2 + * node_demotion[1].preferred = 0 + * node_demotion[2].preferred = + * + */ +static struct demotion_nodes *node_demotion __read_mostly; +#endif /* CONFIG_MIGRATION */ + +static inline struct memory_tier *to_memory_tier(struct device *device) +{ + return container_of(device, struct memory_tier, dev); +} + +static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier) +{ + nodemask_t nodes = NODE_MASK_NONE; + struct memory_dev_type *memtype; + + list_for_each_entry(memtype, &memtier->memory_types, tier_sibiling) + nodes_or(nodes, nodes, memtype->nodes); + + return nodes; +} + +static void memory_tier_device_release(struct device *dev) +{ + struct memory_tier *tier = to_memory_tier(dev); + /* + * synchronize_rcu in clear_node_memory_tier makes sure + * we don't have rcu access to this memory tier. + */ + kfree(tier); +} + +static ssize_t nodes_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int ret; + nodemask_t nmask; + + mutex_lock(&memory_tier_lock); + nmask = get_memtier_nodemask(to_memory_tier(dev)); + ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask)); + mutex_unlock(&memory_tier_lock); + return ret; +} +static DEVICE_ATTR_RO(nodes); + +static struct attribute *memtier_dev_attrs[] = { + &dev_attr_nodes.attr, + NULL +}; + +static const struct attribute_group memtier_dev_group = { + .attrs = memtier_dev_attrs, +}; + +static const struct attribute_group *memtier_dev_groups[] = { + &memtier_dev_group, + NULL +}; + +static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype) +{ + int ret; + bool found_slot = false; + struct memory_tier *memtier, *new_memtier; + int adistance = memtype->adistance; + unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE; + + lockdep_assert_held_once(&memory_tier_lock); + + adistance = round_down(adistance, memtier_adistance_chunk_size); + /* + * If the memtype is already part of a memory tier, + * just return that. + */ + if (!list_empty(&memtype->tier_sibiling)) { + list_for_each_entry(memtier, &memory_tiers, list) { + if (adistance == memtier->adistance_start) + return memtier; + } + WARN_ON(1); + return ERR_PTR(-EINVAL); + } + + list_for_each_entry(memtier, &memory_tiers, list) { + if (adistance == memtier->adistance_start) { + goto link_memtype; + } else if (adistance < memtier->adistance_start) { + found_slot = true; + break; + } + } + + new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL); + if (!new_memtier) + return ERR_PTR(-ENOMEM); + + new_memtier->adistance_start = adistance; + INIT_LIST_HEAD(&new_memtier->list); + INIT_LIST_HEAD(&new_memtier->memory_types); + if (found_slot) + list_add_tail(&new_memtier->list, &memtier->list); + else + list_add_tail(&new_memtier->list, &memory_tiers); + + new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; + new_memtier->dev.bus = &memory_tier_subsys; + new_memtier->dev.release = memory_tier_device_release; + new_memtier->dev.groups = memtier_dev_groups; + + ret = device_register(&new_memtier->dev); + if (ret) { + list_del(&memtier->list); + put_device(&memtier->dev); + return ERR_PTR(ret); + } + memtier = new_memtier; + +link_memtype: + list_add(&memtype->tier_sibiling, &memtier->memory_types); + return memtier; +} + +static struct memory_tier *__node_get_memory_tier(int node) +{ + pg_data_t *pgdat; + + pgdat = NODE_DATA(node); + if (!pgdat) + return NULL; + /* + * Since we hold memory_tier_lock, we can avoid + * RCU read locks when accessing the details. No + * parallel updates are possible here. + */ + return rcu_dereference_check(pgdat->memtier, + lockdep_is_held(&memory_tier_lock)); +} + +#ifdef CONFIG_MIGRATION +bool node_is_toptier(int node) +{ + bool toptier; + pg_data_t *pgdat; + struct memory_tier *memtier; + + pgdat = NODE_DATA(node); + if (!pgdat) + return false; + + rcu_read_lock(); + memtier = rcu_dereference(pgdat->memtier); + if (!memtier) { + toptier = true; + goto out; + } + if (memtier->adistance_start <= top_tier_adistance) + toptier = true; + else + toptier = false; +out: + rcu_read_unlock(); + return toptier; +} + +void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) +{ + struct memory_tier *memtier; + + /* + * pg_data_t.memtier updates includes a synchronize_rcu() + * which ensures that we either find NULL or a valid memtier + * in NODE_DATA. protect the access via rcu_read_lock(); + */ + rcu_read_lock(); + memtier = rcu_dereference(pgdat->memtier); + if (memtier) + *targets = memtier->lower_tier_mask; + else + *targets = NODE_MASK_NONE; + rcu_read_unlock(); +} + +/** + * next_demotion_node() - Get the next node in the demotion path + * @node: The starting node to lookup the next node + * + * Return: node id for next memory node in the demotion path hierarchy + * from @node; NUMA_NO_NODE if @node is terminal. This does not keep + * @node online or guarantee that it *continues* to be the next demotion + * target. + */ +int next_demotion_node(int node) +{ + struct demotion_nodes *nd; + int target; + + if (!node_demotion) + return NUMA_NO_NODE; + + nd = &node_demotion[node]; + + /* + * node_demotion[] is updated without excluding this + * function from running. + * + * Make sure to use RCU over entire code blocks if + * node_demotion[] reads need to be consistent. + */ + rcu_read_lock(); + /* + * If there are multiple target nodes, just select one + * target node randomly. + * + * In addition, we can also use round-robin to select + * target node, but we should introduce another variable + * for node_demotion[] to record last selected target node, + * that may cause cache ping-pong due to the changing of + * last target node. Or introducing per-cpu data to avoid + * caching issue, which seems more complicated. So selecting + * target node randomly seems better until now. + */ + target = node_random(&nd->preferred); + rcu_read_unlock(); + + return target; +} + +static void disable_all_demotion_targets(void) +{ + struct memory_tier *memtier; + int node; + + for_each_node_state(node, N_MEMORY) { + node_demotion[node].preferred = NODE_MASK_NONE; + /* + * We are holding memory_tier_lock, it is safe + * to access pgda->memtier. + */ + memtier = __node_get_memory_tier(node); + if (memtier) + memtier->lower_tier_mask = NODE_MASK_NONE; + } + /* + * Ensure that the "disable" is visible across the system. + * Readers will see either a combination of before+disable + * state or disable+after. They will never see before and + * after state together. + */ + synchronize_rcu(); +} + +/* + * Find an automatic demotion target for all memory + * nodes. Failing here is OK. It might just indicate + * being at the end of a chain. + */ +static void establish_demotion_targets(void) +{ + struct memory_tier *memtier; + struct demotion_nodes *nd; + int target = NUMA_NO_NODE, node; + int distance, best_distance; + nodemask_t tier_nodes, lower_tier; + + lockdep_assert_held_once(&memory_tier_lock); + + if (!node_demotion || !IS_ENABLED(CONFIG_MIGRATION)) + return; + + disable_all_demotion_targets(); + + for_each_node_state(node, N_MEMORY) { + best_distance = -1; + nd = &node_demotion[node]; + + memtier = __node_get_memory_tier(node); + if (!memtier || list_is_last(&memtier->list, &memory_tiers)) + continue; + /* + * Get the lower memtier to find the demotion node list. + */ + memtier = list_next_entry(memtier, list); + tier_nodes = get_memtier_nodemask(memtier); + /* + * find_next_best_node, use 'used' nodemask as a skip list. + * Add all memory nodes except the selected memory tier + * nodelist to skip list so that we find the best node from the + * memtier nodelist. + */ + nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes); + + /* + * Find all the nodes in the memory tier node list of same best distance. + * add them to the preferred mask. We randomly select between nodes + * in the preferred mask when allocating pages during demotion. + */ + do { + target = find_next_best_node(node, &tier_nodes); + if (target == NUMA_NO_NODE) + break; + + distance = node_distance(node, target); + if (distance == best_distance || best_distance == -1) { + best_distance = distance; + node_set(target, nd->preferred); + } else { + break; + } + } while (1); + } + /* + * Promotion is allowed from a memory tier to higher + * memory tier only if the memory tier doesn't include + * compute. We want to skip promotion from a memory tier, + * if any node that is part of the memory tier have CPUs. + * Once we detect such a memory tier, we consider that tier + * as top tiper from which promotion is not allowed. + */ + list_for_each_entry_reverse(memtier, &memory_tiers, list) { + tier_nodes = get_memtier_nodemask(memtier); + nodes_and(tier_nodes, node_states[N_CPU], tier_nodes); + if (!nodes_empty(tier_nodes)) { + /* + * abstract distance below the max value of this memtier + * is considered toptier. + */ + top_tier_adistance = memtier->adistance_start + + MEMTIER_CHUNK_SIZE - 1; + break; + } + } + /* + * Now build the lower_tier mask for each node collecting node mask from + * all memory tier below it. This allows us to fallback demotion page + * allocation to a set of nodes that is closer the above selected + * perferred node. + */ + lower_tier = node_states[N_MEMORY]; + list_for_each_entry(memtier, &memory_tiers, list) { + /* + * Keep removing current tier from lower_tier nodes, + * This will remove all nodes in current and above + * memory tier from the lower_tier mask. + */ + tier_nodes = get_memtier_nodemask(memtier); + nodes_andnot(lower_tier, lower_tier, tier_nodes); + memtier->lower_tier_mask = lower_tier; + } +} + +#else +static inline void disable_all_demotion_targets(void) {} +static inline void establish_demotion_targets(void) {} +#endif /* CONFIG_MIGRATION */ + +static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype) +{ + if (!node_memory_types[node].memtype) + node_memory_types[node].memtype = memtype; + /* + * for each device getting added in the same NUMA node + * with this specific memtype, bump the map count. We + * Only take memtype device reference once, so that + * changing a node memtype can be done by droping the + * only reference count taken here. + */ + + if (node_memory_types[node].memtype == memtype) { + if (!node_memory_types[node].map_count++) + kref_get(&memtype->kref); + } +} + +static struct memory_tier *set_node_memory_tier(int node) +{ + struct memory_tier *memtier; + struct memory_dev_type *memtype; + pg_data_t *pgdat = NODE_DATA(node); + + + lockdep_assert_held_once(&memory_tier_lock); + + if (!node_state(node, N_MEMORY)) + return ERR_PTR(-EINVAL); + + __init_node_memory_type(node, default_dram_type); + + memtype = node_memory_types[node].memtype; + node_set(node, memtype->nodes); + memtier = find_create_memory_tier(memtype); + if (!IS_ERR(memtier)) + rcu_assign_pointer(pgdat->memtier, memtier); + return memtier; +} + +static void destroy_memory_tier(struct memory_tier *memtier) +{ + list_del(&memtier->list); + device_unregister(&memtier->dev); +} + +static bool clear_node_memory_tier(int node) +{ + bool cleared = false; + pg_data_t *pgdat; + struct memory_tier *memtier; + + pgdat = NODE_DATA(node); + if (!pgdat) + return false; + + /* + * Make sure that anybody looking at NODE_DATA who finds + * a valid memtier finds memory_dev_types with nodes still + * linked to the memtier. We achieve this by waiting for + * rcu read section to finish using synchronize_rcu. + * This also enables us to free the destroyed memory tier + * with kfree instead of kfree_rcu + */ + memtier = __node_get_memory_tier(node); + if (memtier) { + struct memory_dev_type *memtype; + + rcu_assign_pointer(pgdat->memtier, NULL); + synchronize_rcu(); + memtype = node_memory_types[node].memtype; + node_clear(node, memtype->nodes); + if (nodes_empty(memtype->nodes)) { + list_del_init(&memtype->tier_sibiling); + if (list_empty(&memtier->memory_types)) + destroy_memory_tier(memtier); + } + cleared = true; + } + return cleared; +} + +static void release_memtype(struct kref *kref) +{ + struct memory_dev_type *memtype; + + memtype = container_of(kref, struct memory_dev_type, kref); + kfree(memtype); +} + +struct memory_dev_type *alloc_memory_type(int adistance) +{ + struct memory_dev_type *memtype; + + memtype = kmalloc(sizeof(*memtype), GFP_KERNEL); + if (!memtype) + return ERR_PTR(-ENOMEM); + + memtype->adistance = adistance; + INIT_LIST_HEAD(&memtype->tier_sibiling); + memtype->nodes = NODE_MASK_NONE; + kref_init(&memtype->kref); + return memtype; +} +EXPORT_SYMBOL_GPL(alloc_memory_type); + +void destroy_memory_type(struct memory_dev_type *memtype) +{ + kref_put(&memtype->kref, release_memtype); +} +EXPORT_SYMBOL_GPL(destroy_memory_type); + +void init_node_memory_type(int node, struct memory_dev_type *memtype) +{ + + mutex_lock(&memory_tier_lock); + __init_node_memory_type(node, memtype); + mutex_unlock(&memory_tier_lock); +} +EXPORT_SYMBOL_GPL(init_node_memory_type); + +void clear_node_memory_type(int node, struct memory_dev_type *memtype) +{ + mutex_lock(&memory_tier_lock); + if (node_memory_types[node].memtype == memtype) + node_memory_types[node].map_count--; + /* + * If we umapped all the attached devices to this node, + * clear the node memory type. + */ + if (!node_memory_types[node].map_count) { + node_memory_types[node].memtype = NULL; + kref_put(&memtype->kref, release_memtype); + } + mutex_unlock(&memory_tier_lock); +} +EXPORT_SYMBOL_GPL(clear_node_memory_type); + +static int __meminit memtier_hotplug_callback(struct notifier_block *self, + unsigned long action, void *_arg) +{ + struct memory_tier *memtier; + struct memory_notify *arg = _arg; + + /* + * Only update the node migration order when a node is + * changing status, like online->offline. + */ + if (arg->status_change_nid < 0) + return notifier_from_errno(0); + + switch (action) { + case MEM_OFFLINE: + mutex_lock(&memory_tier_lock); + if (clear_node_memory_tier(arg->status_change_nid)) + establish_demotion_targets(); + mutex_unlock(&memory_tier_lock); + break; + case MEM_ONLINE: + mutex_lock(&memory_tier_lock); + memtier = set_node_memory_tier(arg->status_change_nid); + if (!IS_ERR(memtier)) + establish_demotion_targets(); + mutex_unlock(&memory_tier_lock); + break; + } + + return notifier_from_errno(0); +} + +static int __init memory_tier_init(void) +{ + int ret, node; + struct memory_tier *memtier; + + ret = subsys_virtual_register(&memory_tier_subsys, NULL); + if (ret) + panic("%s() failed to register memory tier subsystem\n", __func__); + +#ifdef CONFIG_MIGRATION + node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes), + GFP_KERNEL); + WARN_ON(!node_demotion); +#endif + mutex_lock(&memory_tier_lock); + /* + * For now we can have 4 faster memory tiers with smaller adistance + * than default DRAM tier. + */ + default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); + if (!default_dram_type) + panic("%s() failed to allocate default DRAM tier\n", __func__); + + /* + * Look at all the existing N_MEMORY nodes and add them to + * default memory tier or to a tier if we already have memory + * types assigned. + */ + for_each_node_state(node, N_MEMORY) { + memtier = set_node_memory_tier(node); + if (IS_ERR(memtier)) + /* + * Continue with memtiers we are able to setup + */ + break; + } + establish_demotion_targets(); + mutex_unlock(&memory_tier_lock); + + hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRIO); + return 0; +} +subsys_initcall(memory_tier_init); + +bool numa_demotion_enabled = false; + +#ifdef CONFIG_MIGRATION +#ifdef CONFIG_SYSFS +static ssize_t numa_demotion_enabled_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", + numa_demotion_enabled ? "true" : "false"); +} + +static ssize_t numa_demotion_enabled_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret; + + ret = kstrtobool(buf, &numa_demotion_enabled); + if (ret) + return ret; + + return count; +} + +static struct kobj_attribute numa_demotion_enabled_attr = + __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, + numa_demotion_enabled_store); + +static struct attribute *numa_attrs[] = { + &numa_demotion_enabled_attr.attr, + NULL, +}; + +static const struct attribute_group numa_attr_group = { + .attrs = numa_attrs, +}; + +static int __init numa_init_sysfs(void) +{ + int err; + struct kobject *numa_kobj; + + numa_kobj = kobject_create_and_add("numa", mm_kobj); + if (!numa_kobj) { + pr_err("failed to create numa kobject\n"); + return -ENOMEM; + } + err = sysfs_create_group(numa_kobj, &numa_attr_group); + if (err) { + pr_err("failed to register numa group\n"); + goto delete_obj; + } + return 0; + +delete_obj: + kobject_put(numa_kobj); + return err; +} +subsys_initcall(numa_init_sysfs); +#endif /* CONFIG_SYSFS */ +#endif diff --git a/mm/memory.c b/mm/memory.c index 4ba73f5aa8bb7771c584ac7097c66fba8222ddaa..f88c351aecd4177b84b4bc34d5c2ed438d76a6b0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +76,7 @@ #include #include #include +#include #include @@ -125,18 +128,6 @@ int randomize_va_space __read_mostly = 2; #endif -#ifndef arch_faults_on_old_pte -static inline bool arch_faults_on_old_pte(void) -{ - /* - * Those arches which don't have hw access flag feature need to - * implement their own helper. By default, "true" means pagefault - * will be hit on old pte. - */ - return true; -} -#endif - #ifndef arch_wants_old_prefaulted_pte static inline bool arch_wants_old_prefaulted_pte(void) { @@ -402,12 +393,21 @@ void free_pgd_range(struct mmu_gather *tlb, } while (pgd++, addr = next, addr != end); } -void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, - unsigned long floor, unsigned long ceiling) +void free_pgtables(struct mmu_gather *tlb, struct maple_tree *mt, + struct vm_area_struct *vma, unsigned long floor, + unsigned long ceiling) { - while (vma) { - struct vm_area_struct *next = vma->vm_next; + MA_STATE(mas, mt, vma->vm_end, vma->vm_end); + + do { unsigned long addr = vma->vm_start; + struct vm_area_struct *next; + + /* + * Note: USER_PGTABLES_CEILING may be passed as ceiling and may + * be 0. This will underflow and is okay. + */ + next = mas_find(&mas, ceiling - 1); /* * Hide vma from rmap and truncate_pagecache before freeing @@ -426,7 +426,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, while (next && next->vm_start <= vma->vm_end + PMD_SIZE && !is_vm_hugetlb_page(next)) { vma = next; - next = vma->vm_next; + next = mas_find(&mas, ceiling - 1); unlink_anon_vmas(vma); unlink_file_vma(vma); } @@ -434,7 +434,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma, floor, next ? next->vm_start : ceiling); } vma = next; - } + } while (vma); } void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) @@ -1393,10 +1393,12 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, pte_t *pte, struct zap_details *details, pte_t pteval) { +#ifdef CONFIG_PTE_MARKER_UFFD_WP if (zap_drop_file_uffd_wp(details)) return; pte_install_uffd_wp_if_needed(vma, addr, pte, pteval); +#endif } static unsigned long zap_pte_range(struct mmu_gather *tlb, @@ -1685,10 +1687,8 @@ static void unmap_single_vma(struct mmu_gather *tlb, if (vma->vm_file) { zap_flags_t zap_flags = details ? details->zap_flags : 0; - i_mmap_lock_write(vma->vm_file->f_mapping); __unmap_hugepage_range_final(tlb, vma, start, end, NULL, zap_flags); - i_mmap_unlock_write(vma->vm_file->f_mapping); } } else unmap_page_range(tlb, vma, start, end, details); @@ -1698,6 +1698,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, /** * unmap_vmas - unmap a range of memory covered by a list of vma's * @tlb: address of the caller's struct mmu_gather + * @mt: the maple tree * @vma: the starting vma * @start_addr: virtual address at which to start unmapping * @end_addr: virtual address at which to end unmapping @@ -1713,7 +1714,7 @@ static void unmap_single_vma(struct mmu_gather *tlb, * ensure that any thus-far unmapped pages are flushed before unmap_vmas() * drops the lock and schedules. */ -void unmap_vmas(struct mmu_gather *tlb, +void unmap_vmas(struct mmu_gather *tlb, struct maple_tree *mt, struct vm_area_struct *vma, unsigned long start_addr, unsigned long end_addr) { @@ -1723,12 +1724,14 @@ void unmap_vmas(struct mmu_gather *tlb, /* Careful - we need to zap private pages too! */ .even_cows = true, }; + MA_STATE(mas, mt, vma->vm_end, vma->vm_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, start_addr, end_addr); mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) + do { unmap_single_vma(tlb, vma, start_addr, end_addr, &details); + } while ((vma = mas_find(&mas, end_addr - 1)) != NULL); mmu_notifier_invalidate_range_end(&range); } @@ -1743,8 +1746,11 @@ void unmap_vmas(struct mmu_gather *tlb, void zap_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long size) { + struct maple_tree *mt = &vma->vm_mm->mm_mt; + unsigned long end = start + size; struct mmu_notifier_range range; struct mmu_gather tlb; + MA_STATE(mas, mt, vma->vm_end, vma->vm_end); lru_add_drain(); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm, @@ -1752,8 +1758,9 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start, tlb_gather_mmu(&tlb, vma->vm_mm); update_hiwater_rss(vma->vm_mm); mmu_notifier_invalidate_range_start(&range); - for ( ; vma && vma->vm_start < range.end; vma = vma->vm_next) + do { unmap_single_vma(&tlb, vma, start, range.end, NULL); + } while ((vma = mas_find(&mas, end - 1)) != NULL); mmu_notifier_invalidate_range_end(&range); tlb_finish_mmu(&tlb); } @@ -2870,7 +2877,7 @@ static inline bool __wp_page_copy_user(struct page *dst, struct page *src, * On architectures with software "accessed" bits, we would * take a double page fault, so mark it accessed here. */ - if (arch_faults_on_old_pte() && !pte_young(vmf->orig_pte)) { + if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) { pte_t entry; vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); @@ -3128,6 +3135,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return 0; } + kmsan_copy_page_meta(new_page, old_page); } if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL)) @@ -3362,6 +3370,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) { const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; + struct folio *folio; VM_BUG_ON(unshare && (vmf->flags & FAULT_FLAG_WRITE)); VM_BUG_ON(!unshare && !(vmf->flags & FAULT_FLAG_WRITE)); @@ -3408,48 +3417,47 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) * Take out anonymous pages first, anonymous shared vmas are * not dirty accountable. */ - if (PageAnon(vmf->page)) { - struct page *page = vmf->page; - + folio = page_folio(vmf->page); + if (folio_test_anon(folio)) { /* * If the page is exclusive to this process we must reuse the * page without further checks. */ - if (PageAnonExclusive(page)) + if (PageAnonExclusive(vmf->page)) goto reuse; /* - * We have to verify under page lock: these early checks are - * just an optimization to avoid locking the page and freeing + * We have to verify under folio lock: these early checks are + * just an optimization to avoid locking the folio and freeing * the swapcache if there is little hope that we can reuse. * - * PageKsm() doesn't necessarily raise the page refcount. + * KSM doesn't necessarily raise the folio refcount. */ - if (PageKsm(page) || page_count(page) > 3) + if (folio_test_ksm(folio) || folio_ref_count(folio) > 3) goto copy; - if (!PageLRU(page)) + if (!folio_test_lru(folio)) /* * Note: We cannot easily detect+handle references from - * remote LRU pagevecs or references to PageLRU() pages. + * remote LRU pagevecs or references to LRU folios. */ lru_add_drain(); - if (page_count(page) > 1 + PageSwapCache(page)) + if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio)) goto copy; - if (!trylock_page(page)) + if (!folio_trylock(folio)) goto copy; - if (PageSwapCache(page)) - try_to_free_swap(page); - if (PageKsm(page) || page_count(page) != 1) { - unlock_page(page); + if (folio_test_swapcache(folio)) + folio_free_swap(folio); + if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) { + folio_unlock(folio); goto copy; } /* - * Ok, we've got the only page reference from our mapping - * and the page is locked, it's dark out, and we're wearing + * Ok, we've got the only folio reference from our mapping + * and the folio is locked, it's dark out, and we're wearing * sunglasses. Hit it. */ - page_move_anon_rmap(page, vma); - unlock_page(page); + page_move_anon_rmap(vmf->page, vma); + folio_unlock(folio); reuse: if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); @@ -3612,11 +3620,11 @@ EXPORT_SYMBOL(unmap_mapping_range); */ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) { - struct page *page = vmf->page; + struct folio *folio = page_folio(vmf->page); struct vm_area_struct *vma = vmf->vma; struct mmu_notifier_range range; - if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) + if (!folio_lock_or_retry(folio, vma->vm_mm, vmf->flags)) return VM_FAULT_RETRY; mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma, vma->vm_mm, vmf->address & PAGE_MASK, @@ -3626,23 +3634,23 @@ static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (likely(pte_same(*vmf->pte, vmf->orig_pte))) - restore_exclusive_pte(vma, page, vmf->address, vmf->pte); + restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); - unlock_page(page); + folio_unlock(folio); mmu_notifier_invalidate_range_end(&range); return 0; } -static inline bool should_try_to_free_swap(struct page *page, +static inline bool should_try_to_free_swap(struct folio *folio, struct vm_area_struct *vma, unsigned int fault_flags) { - if (!PageSwapCache(page)) + if (!folio_test_swapcache(folio)) return false; - if (mem_cgroup_swap_full(page) || (vma->vm_flags & VM_LOCKED) || - PageMlocked(page)) + if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) || + folio_test_mlocked(folio)) return true; /* * If we want to map a page that's in the swapcache writable, we @@ -3650,8 +3658,8 @@ static inline bool should_try_to_free_swap(struct page *page, * user. Try freeing the swapcache to get rid of the swapcache * reference only in case it's likely that we'll be the exlusive user. */ - return (fault_flags & FAULT_FLAG_WRITE) && !PageKsm(page) && - page_count(page) == 2; + return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) && + folio_ref_count(folio) == 2; } static vm_fault_t pte_marker_clear(struct vm_fault *vmf) @@ -3718,7 +3726,8 @@ static vm_fault_t handle_pte_marker(struct vm_fault *vmf) vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; - struct page *page = NULL, *swapcache; + struct folio *swapcache, *folio = NULL; + struct page *page; struct swap_info_struct *si = NULL; rmap_t rmap_flags = RMAP_NONE; bool exclusive = false; @@ -3741,7 +3750,21 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) ret = remove_device_exclusive_entry(vmf); } else if (is_device_private_entry(entry)) { vmf->page = pfn_swap_entry_to_page(entry); - ret = vmf->page->pgmap->ops->migrate_to_ram(vmf); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) { + spin_unlock(vmf->ptl); + goto out; + } + + /* + * Get a page reference while we know the page can't be + * freed. + */ + get_page(vmf->page); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->page->pgmap->ops->migrate_to_ram(vmf); + put_page(vmf->page); } else if (is_hwpoison_entry(entry)) { ret = VM_FAULT_HWPOISON; } else if (is_swapin_error_entry(entry)) { @@ -3760,21 +3783,25 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (unlikely(!si)) goto out; - page = lookup_swap_cache(entry, vma, vmf->address); - swapcache = page; + folio = swap_cache_get_folio(entry, vma, vmf->address); + if (folio) + page = folio_file_page(folio, swp_offset(entry)); + swapcache = folio; - if (!page) { + if (!folio) { if (data_race(si->flags & SWP_SYNCHRONOUS_IO) && __swap_count(entry) == 1) { /* skip swapcache */ - page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, - vmf->address); - if (page) { - __SetPageLocked(page); - __SetPageSwapBacked(page); - - if (mem_cgroup_swapin_charge_page(page, - vma->vm_mm, GFP_KERNEL, entry)) { + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, + vma, vmf->address, false); + page = &folio->page; + if (folio) { + __folio_set_locked(folio); + __folio_set_swapbacked(folio); + + if (mem_cgroup_swapin_charge_folio(folio, + vma->vm_mm, GFP_KERNEL, + entry)) { ret = VM_FAULT_OOM; goto out_page; } @@ -3782,23 +3809,24 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) shadow = get_shadow_from_swap_cache(entry); if (shadow) - workingset_refault(page_folio(page), - shadow); + workingset_refault(folio, shadow); - lru_cache_add(page); + folio_add_lru(folio); /* To provide entry to swap_readpage() */ - set_page_private(page, entry.val); + folio_set_swap_entry(folio, entry); swap_readpage(page, true, NULL); - set_page_private(page, 0); + folio->private = NULL; } } else { page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); - swapcache = page; + if (page) + folio = page_folio(page); + swapcache = folio; } - if (!page) { + if (!folio) { /* * Back out if somebody else faulted in this pte * while we released the pte lock. @@ -3823,7 +3851,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out_release; } - locked = lock_page_or_retry(page, vma->vm_mm, vmf->flags); + locked = folio_lock_or_retry(folio, vma->vm_mm, vmf->flags); if (!locked) { ret |= VM_FAULT_RETRY; @@ -3832,13 +3860,13 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (swapcache) { /* - * Make sure try_to_free_swap or swapoff did not release the + * Make sure folio_free_swap() or swapoff did not release the * swapcache from under us. The page pin, and pte_same test * below, are not enough to exclude that. Even if it is still * swapcache, we need to check that the page's swap has not * changed. */ - if (unlikely(!PageSwapCache(page) || + if (unlikely(!folio_test_swapcache(folio) || page_private(page) != entry.val)) goto out_page; @@ -3850,9 +3878,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) page = ksm_might_need_to_copy(page, vma, vmf->address); if (unlikely(!page)) { ret = VM_FAULT_OOM; - page = swapcache; goto out_page; } + folio = page_folio(page); /* * If we want to map a page that's in the swapcache writable, we @@ -3860,8 +3888,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * owner. Try removing the extra reference from the local LRU * pagevecs if required. */ - if ((vmf->flags & FAULT_FLAG_WRITE) && page == swapcache && - !PageKsm(page) && !PageLRU(page)) + if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache && + !folio_test_ksm(folio) && !folio_test_lru(folio)) lru_add_drain(); } @@ -3875,7 +3903,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) goto out_nomap; - if (unlikely(!PageUptodate(page))) { + if (unlikely(!folio_test_uptodate(folio))) { ret = VM_FAULT_SIGBUS; goto out_nomap; } @@ -3888,26 +3916,26 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * check after taking the PT lock and making sure that nobody * concurrently faulted in this page and set PG_anon_exclusive. */ - BUG_ON(!PageAnon(page) && PageMappedToDisk(page)); - BUG_ON(PageAnon(page) && PageAnonExclusive(page)); + BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio)); + BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page)); /* * Check under PT lock (to protect against concurrent fork() sharing * the swap entry concurrently) for certainly exclusive pages. */ - if (!PageKsm(page)) { + if (!folio_test_ksm(folio)) { /* * Note that pte_swp_exclusive() == false for architectures * without __HAVE_ARCH_PTE_SWP_EXCLUSIVE. */ exclusive = pte_swp_exclusive(vmf->orig_pte); - if (page != swapcache) { + if (folio != swapcache) { /* * We have a fresh page that is not exposed to the * swapcache -> certainly exclusive. */ exclusive = true; - } else if (exclusive && PageWriteback(page) && + } else if (exclusive && folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES)) { /* * This is tricky: not all swap backends support @@ -3937,8 +3965,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * yet. */ swap_free(entry); - if (should_try_to_free_swap(page, vma, vmf->flags)) - try_to_free_swap(page); + if (should_try_to_free_swap(folio, vma, vmf->flags)) + folio_free_swap(folio); inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS); @@ -3950,7 +3978,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * exposing them to the swapcache or because the swap entry indicates * exclusivity. */ - if (!PageKsm(page) && (exclusive || page_count(page) == 1)) { + if (!folio_test_ksm(folio) && + (exclusive || folio_ref_count(folio) == 1)) { if (vmf->flags & FAULT_FLAG_WRITE) { pte = maybe_mkwrite(pte_mkdirty(pte), vma); vmf->flags &= ~FAULT_FLAG_WRITE; @@ -3968,19 +3997,20 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) vmf->orig_pte = pte; /* ksm created a completely new copy */ - if (unlikely(page != swapcache && swapcache)) { + if (unlikely(folio != swapcache && swapcache)) { page_add_new_anon_rmap(page, vma, vmf->address); - lru_cache_add_inactive_or_unevictable(page, vma); + folio_add_lru_vma(folio, vma); } else { page_add_anon_rmap(page, vma, vmf->address, rmap_flags); } - VM_BUG_ON(!PageAnon(page) || (pte_write(pte) && !PageAnonExclusive(page))); + VM_BUG_ON(!folio_test_anon(folio) || + (pte_write(pte) && !PageAnonExclusive(page))); set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); arch_do_swap_page(vma->vm_mm, vma, vmf->address, pte, vmf->orig_pte); - unlock_page(page); - if (page != swapcache && swapcache) { + folio_unlock(folio); + if (folio != swapcache && swapcache) { /* * Hold the lock to avoid the swap entry to be reused * until we take the PT lock for the pte_same() check @@ -3989,8 +4019,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) * so that the swap count won't change under a * parallel locked swapcache. */ - unlock_page(swapcache); - put_page(swapcache); + folio_unlock(swapcache); + folio_put(swapcache); } if (vmf->flags & FAULT_FLAG_WRITE) { @@ -4011,12 +4041,12 @@ out: out_nomap: pte_unmap_unlock(vmf->pte, vmf->ptl); out_page: - unlock_page(page); + folio_unlock(folio); out_release: - put_page(page); - if (page != swapcache && swapcache) { - unlock_page(swapcache); - put_page(swapcache); + folio_put(folio); + if (folio != swapcache && swapcache) { + folio_unlock(swapcache); + folio_put(swapcache); } if (si) put_swap_device(si); @@ -4104,7 +4134,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (!pte_none(*vmf->pte)) { - update_mmu_cache(vma, vmf->address, vmf->pte); + update_mmu_tlb(vma, vmf->address, vmf->pte); goto release; } @@ -4386,14 +4416,20 @@ vm_fault_t finish_fault(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); - ret = 0; + /* Re-check under ptl */ - if (likely(!vmf_pte_changed(vmf))) + if (likely(!vmf_pte_changed(vmf))) { do_set_pte(vmf, page, vmf->address); - else + + /* no need to invalidate: a not-present page won't be cached */ + update_mmu_cache(vma, vmf->address, vmf->pte); + + ret = 0; + } else { + update_mmu_tlb(vma, vmf->address, vmf->pte); ret = VM_FAULT_NOPAGE; + } - update_mmu_tlb(vma, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); return ret; } @@ -4725,8 +4761,16 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED)) flags |= TNF_SHARED; - last_cpupid = page_cpupid_last(page); page_nid = page_to_nid(page); + /* + * For memory tiering mode, cpupid of slow memory page is used + * to record page access time. So use default value. + */ + if ((sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && + !node_is_toptier(page_nid)) + last_cpupid = (-1 & LAST_CPUPID_MASK); + else + last_cpupid = page_cpupid_last(page); target_nid = numa_migrate_prep(page, vma, vmf->address, page_nid, &flags); if (target_nid == NUMA_NO_NODE) { @@ -4985,7 +5029,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, return VM_FAULT_OOM; retry_pud: if (pud_none(*vmf.pud) && - hugepage_vma_check(vma, vm_flags, false, true)) { + hugepage_vma_check(vma, vm_flags, false, true, true)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5019,7 +5063,7 @@ retry_pud: goto retry_pud; if (pmd_none(*vmf.pmd) && - hugepage_vma_check(vma, vm_flags, false, true)) { + hugepage_vma_check(vma, vm_flags, false, true, true)) { ret = create_huge_pmd(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; @@ -5114,6 +5158,27 @@ static inline void mm_account_fault(struct pt_regs *regs, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address); } +#ifdef CONFIG_LRU_GEN +static void lru_gen_enter_fault(struct vm_area_struct *vma) +{ + /* the LRU algorithm doesn't apply to sequential or random reads */ + current->in_lru_fault = !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)); +} + +static void lru_gen_exit_fault(void) +{ + current->in_lru_fault = false; +} +#else +static void lru_gen_enter_fault(struct vm_area_struct *vma) +{ +} + +static void lru_gen_exit_fault(void) +{ +} +#endif /* CONFIG_LRU_GEN */ + /* * By the time we get here, we already hold the mm semaphore * @@ -5145,11 +5210,15 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address, if (flags & FAULT_FLAG_USER) mem_cgroup_enter_user_fault(); + lru_gen_enter_fault(vma); + if (unlikely(is_vm_hugetlb_page(vma))) ret = hugetlb_fault(vma->vm_mm, vma, address, flags); else ret = __handle_mm_fault(vma, address, flags); + lru_gen_exit_fault(); + if (flags & FAULT_FLAG_USER) { mem_cgroup_exit_user_fault(); /* @@ -5637,11 +5706,11 @@ static void clear_gigantic_page(struct page *page, unsigned int pages_per_huge_page) { int i; - struct page *p = page; + struct page *p; might_sleep(); - for (i = 0; i < pages_per_huge_page; - i++, p = mem_map_next(p, page, i)) { + for (i = 0; i < pages_per_huge_page; i++) { + p = nth_page(page, i); cond_resched(); clear_user_highpage(p, addr + i * PAGE_SIZE); } @@ -5677,13 +5746,12 @@ static void copy_user_gigantic_page(struct page *dst, struct page *src, struct page *dst_base = dst; struct page *src_base = src; - for (i = 0; i < pages_per_huge_page; ) { + for (i = 0; i < pages_per_huge_page; i++) { + dst = nth_page(dst_base, i); + src = nth_page(src_base, i); + cond_resched(); copy_user_highpage(dst, src, addr + i*PAGE_SIZE, vma); - - i++; - dst = mem_map_next(dst, dst_base, i); - src = mem_map_next(src, src_base, i); } } @@ -5730,10 +5798,10 @@ long copy_huge_page_from_user(struct page *dst_page, void *page_kaddr; unsigned long i, rc = 0; unsigned long ret_val = pages_per_huge_page * PAGE_SIZE; - struct page *subpage = dst_page; + struct page *subpage; - for (i = 0; i < pages_per_huge_page; - i++, subpage = mem_map_next(subpage, dst_page, i)) { + for (i = 0; i < pages_per_huge_page; i++) { + subpage = nth_page(dst_page, i); if (allow_pagefault) page_kaddr = kmap(subpage); else diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index fad6d1f2262af652d30e40e344cb225ebf991d19..fd40f7e9f17635135b0c6bc8046931772f888ec0 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -1085,8 +1085,7 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, * of the physical memory space for vmemmaps. That space is pageblock * aligned. */ - if (WARN_ON_ONCE(!nr_pages || - !IS_ALIGNED(pfn, pageblock_nr_pages) || + if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(pfn) || !IS_ALIGNED(pfn + nr_pages, PAGES_PER_SECTION))) return -EINVAL; @@ -1806,8 +1805,7 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages, * of the physical memory space for vmemmaps. That space is pageblock * aligned. */ - if (WARN_ON_ONCE(!nr_pages || - !IS_ALIGNED(start_pfn, pageblock_nr_pages) || + if (WARN_ON_ONCE(!nr_pages || !pageblock_aligned(start_pfn) || !IS_ALIGNED(start_pfn + nr_pages, PAGES_PER_SECTION))) return -EINVAL; @@ -1940,8 +1938,8 @@ int __ref offline_pages(unsigned long start_pfn, unsigned long nr_pages, node_states_clear_node(node, &arg); if (arg.status_change_nid >= 0) { - kswapd_stop(node); kcompactd_stop(node); + kswapd_stop(node); } writeback_set_ratelimit(); @@ -1969,11 +1967,10 @@ failed_removal: static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) { - int ret = !is_memblock_offlined(mem); int *nid = arg; *nid = mem->nid; - if (unlikely(ret)) { + if (unlikely(mem->state != MEM_OFFLINE)) { phys_addr_t beginpa, endpa; beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b73d3248d976a075d2919cfc0fef6ec1043e313a..a937eaec5b68d82ff830b11faf46be17e2211fcf 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -381,9 +381,10 @@ void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new) { struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); mmap_write_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) + for_each_vma(vmi, vma) mpol_rebind_policy(vma->vm_policy, new); mmap_write_unlock(mm); } @@ -654,7 +655,7 @@ static unsigned long change_prot_numa(struct vm_area_struct *vma, static int queue_pages_test_walk(unsigned long start, unsigned long end, struct mm_walk *walk) { - struct vm_area_struct *vma = walk->vma; + struct vm_area_struct *next, *vma = walk->vma; struct queue_pages *qp = walk->private; unsigned long endvma = vma->vm_end; unsigned long flags = qp->flags; @@ -669,9 +670,10 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end, /* hole at head side of range */ return -EFAULT; } + next = find_vma(vma->vm_mm, vma->vm_end); if (!(flags & MPOL_MF_DISCONTIG_OK) && ((vma->vm_end < qp->end) && - (!vma->vm_next || vma->vm_end < vma->vm_next->vm_start))) + (!next || vma->vm_end < next->vm_start))) /* hole at middle or tail of range */ return -EFAULT; @@ -785,26 +787,24 @@ static int vma_replace_policy(struct vm_area_struct *vma, static int mbind_range(struct mm_struct *mm, unsigned long start, unsigned long end, struct mempolicy *new_pol) { + MA_STATE(mas, &mm->mm_mt, start - 1, start - 1); struct vm_area_struct *prev; struct vm_area_struct *vma; int err = 0; pgoff_t pgoff; - unsigned long vmstart; - unsigned long vmend; - vma = find_vma(mm, start); - VM_BUG_ON(!vma); - - prev = vma->vm_prev; - if (start > vma->vm_start) - prev = vma; + prev = mas_find_rev(&mas, 0); + if (prev && (start < prev->vm_end)) + vma = prev; + else + vma = mas_next(&mas, end - 1); - for (; vma && vma->vm_start < end; prev = vma, vma = vma->vm_next) { - vmstart = max(start, vma->vm_start); - vmend = min(end, vma->vm_end); + for (; vma; vma = mas_next(&mas, end - 1)) { + unsigned long vmstart = max(start, vma->vm_start); + unsigned long vmend = min(end, vma->vm_end); if (mpol_equal(vma_policy(vma), new_pol)) - continue; + goto next; pgoff = vma->vm_pgoff + ((vmstart - vma->vm_start) >> PAGE_SHIFT); @@ -813,6 +813,8 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, new_pol, vma->vm_userfaultfd_ctx, anon_vma_name(vma)); if (prev) { + /* vma_merge() invalidated the mas */ + mas_pause(&mas); vma = prev; goto replace; } @@ -820,19 +822,25 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, err = split_vma(vma->vm_mm, vma, vmstart, 1); if (err) goto out; + /* split_vma() invalidated the mas */ + mas_pause(&mas); } if (vma->vm_end != vmend) { err = split_vma(vma->vm_mm, vma, vmend, 0); if (err) goto out; + /* split_vma() invalidated the mas */ + mas_pause(&mas); } - replace: +replace: err = vma_replace_policy(vma, new_pol); if (err) goto out; +next: + prev = vma; } - out: +out: return err; } @@ -853,12 +861,14 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, goto out; } + task_lock(current); ret = mpol_set_nodemask(new, nodes, scratch); if (ret) { + task_unlock(current); mpol_put(new); goto out; } - task_lock(current); + old = current->mempolicy; current->mempolicy = new; if (new && new->mode == MPOL_INTERLEAVE) @@ -1047,6 +1057,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, int flags) { nodemask_t nmask; + struct vm_area_struct *vma; LIST_HEAD(pagelist); int err = 0; struct migration_target_control mtc = { @@ -1062,8 +1073,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, * need migration. Between passing in the full user address * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. */ + vma = find_vma(mm, 0); VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); - queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, + queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask, flags | MPOL_MF_DISCONTIG_OK, &pagelist); if (!list_empty(&pagelist)) { @@ -1193,14 +1205,13 @@ static struct page *new_page(struct page *page, unsigned long start) struct folio *dst, *src = page_folio(page); struct vm_area_struct *vma; unsigned long address; + VMA_ITERATOR(vmi, current->mm, start); gfp_t gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL; - vma = find_vma(current->mm, start); - while (vma) { + for_each_vma(vmi, vma) { address = page_address_in_vma(page, vma); if (address != -EFAULT) break; - vma = vma->vm_next; } if (folio_test_hugetlb(src)) @@ -1259,7 +1270,7 @@ static long do_mbind(unsigned long start, unsigned long len, if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = (len + PAGE_SIZE - 1) & PAGE_MASK; + len = PAGE_ALIGN(len); end = start + len; if (end < start) @@ -1478,6 +1489,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le unsigned long vmend; unsigned long end; int err = -ENOENT; + VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); if (start & ~PAGE_MASK) @@ -1495,7 +1507,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = (len + PAGE_SIZE - 1) & PAGE_MASK; + len = PAGE_ALIGN(len); end = start + len; if (end < start) @@ -1503,9 +1515,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (end == start) return 0; mmap_write_lock(mm); - vma = find_vma(mm, start); - for (; vma && vma->vm_start < end; vma = vma->vm_next) { - + for_each_vma_range(vmi, vma, end) { vmstart = max(start, vma->vm_start); vmend = min(end, vma->vm_end); new = mpol_dup(vma_policy(vma)); @@ -1803,7 +1813,7 @@ bool vma_policy_mof(struct vm_area_struct *vma) return pol->flags & MPOL_F_MOF; } -static int apply_policy_zone(struct mempolicy *policy, enum zone_type zone) +bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) { enum zone_type dynamic_policy_zone = policy_zone; diff --git a/mm/memremap.c b/mm/memremap.c index 58b20c3c300b871a5aa6028e2a24dd87d79ed277..421bec3a29ee730c5f6fd6532f37abc28962b339 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -138,8 +138,11 @@ void memunmap_pages(struct dev_pagemap *pgmap) int i; percpu_ref_kill(&pgmap->ref); - for (i = 0; i < pgmap->nr_range; i++) - percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i)); + if (pgmap->type != MEMORY_DEVICE_PRIVATE && + pgmap->type != MEMORY_DEVICE_COHERENT) + for (i = 0; i < pgmap->nr_range; i++) + percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i)); + wait_for_completion(&pgmap->done); for (i = 0; i < pgmap->nr_range; i++) @@ -264,7 +267,9 @@ static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params, memmap_init_zone_device(&NODE_DATA(nid)->node_zones[ZONE_DEVICE], PHYS_PFN(range->start), PHYS_PFN(range_len(range)), pgmap); - percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id)); + if (pgmap->type != MEMORY_DEVICE_PRIVATE && + pgmap->type != MEMORY_DEVICE_COHERENT) + percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id)); return 0; err_add_memory: @@ -454,7 +459,7 @@ struct dev_pagemap *get_dev_pagemap(unsigned long pfn, /* fall back to slow path lookup */ rcu_read_lock(); pgmap = xa_load(&pgmap_array, PHYS_PFN(phys)); - if (pgmap && !percpu_ref_tryget_live(&pgmap->ref)) + if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref)) pgmap = NULL; rcu_read_unlock(); @@ -502,11 +507,28 @@ void free_zone_device_page(struct page *page) page->mapping = NULL; page->pgmap->ops->page_free(page); + if (page->pgmap->type != MEMORY_DEVICE_PRIVATE && + page->pgmap->type != MEMORY_DEVICE_COHERENT) + /* + * Reset the page count to 1 to prepare for handing out the page + * again. + */ + set_page_count(page, 1); + else + put_dev_pagemap(page->pgmap); +} + +void zone_device_page_init(struct page *page) +{ /* - * Reset the page count to 1 to prepare for handing out the page again. + * Drivers shouldn't be allocating pages after calling + * memunmap_pages(). */ + WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref)); set_page_count(page, 1); + lock_page(page); } +EXPORT_SYMBOL_GPL(zone_device_page_init); #ifdef CONFIG_FS_DAX bool __put_devmap_managed_page_refs(struct page *page, int refs) diff --git a/mm/migrate.c b/mm/migrate.c index 6a1597c92261d345c5ac95a28625346b04598b8c..1379e1912772e0409b78e776b2777a0c3cf0a33b 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -50,6 +50,7 @@ #include #include #include +#include #include @@ -198,7 +199,7 @@ static bool remove_migration_pte(struct folio *folio, #endif folio_get(folio); - pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot))); + pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); if (pte_swp_soft_dirty(*pvmw.pte)) pte = pte_mksoft_dirty(pte); @@ -206,6 +207,10 @@ static bool remove_migration_pte(struct folio *folio, * Recheck VMA as permissions can change since migration started */ entry = pte_to_swp_entry(*pvmw.pte); + if (!is_migration_entry_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && is_migration_entry_dirty(entry)) + pte = pte_mkdirty(pte); if (is_writable_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); else if (pte_swp_uffd_wp(*pvmw.pte)) @@ -560,6 +565,18 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio) * future migrations of this same page. */ cpupid = page_cpupid_xchg_last(&folio->page, -1); + /* + * For memory tiering mode, when migrate between slow and fast + * memory node, reset cpupid, because that is used to record + * page access time in slow memory node. + */ + if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) { + bool f_toptier = node_is_toptier(page_to_nid(&folio->page)); + bool t_toptier = node_is_toptier(page_to_nid(&newfolio->page)); + + if (f_toptier != t_toptier) + cpupid = -1; + } page_cpupid_xchg_last(&newfolio->page, cpupid); folio_migrate_ksm(newfolio, folio); @@ -608,6 +625,25 @@ EXPORT_SYMBOL(folio_migrate_copy); * Migration functions ***********************************************************/ +int migrate_folio_extra(struct address_space *mapping, struct folio *dst, + struct folio *src, enum migrate_mode mode, int extra_count) +{ + int rc; + + BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */ + + rc = folio_migrate_mapping(mapping, dst, src, extra_count); + + if (rc != MIGRATEPAGE_SUCCESS) + return rc; + + if (mode != MIGRATE_SYNC_NO_COPY) + folio_migrate_copy(dst, src); + else + folio_migrate_flags(dst, src); + return MIGRATEPAGE_SUCCESS; +} + /** * migrate_folio() - Simple folio migration. * @mapping: The address_space containing the folio. @@ -623,20 +659,7 @@ EXPORT_SYMBOL(folio_migrate_copy); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode) { - int rc; - - BUG_ON(folio_test_writeback(src)); /* Writeback must be complete */ - - rc = folio_migrate_mapping(mapping, dst, src, 0); - - if (rc != MIGRATEPAGE_SUCCESS) - return rc; - - if (mode != MIGRATE_SYNC_NO_COPY) - folio_migrate_copy(dst, src); - else - folio_migrate_flags(dst, src); - return MIGRATEPAGE_SUCCESS; + return migrate_folio_extra(mapping, dst, src, mode, 0); } EXPORT_SYMBOL(migrate_folio); @@ -976,17 +999,15 @@ out: return rc; } -static int __unmap_and_move(struct page *page, struct page *newpage, +static int __unmap_and_move(struct folio *src, struct folio *dst, int force, enum migrate_mode mode) { - struct folio *folio = page_folio(page); - struct folio *dst = page_folio(newpage); int rc = -EAGAIN; bool page_was_mapped = false; struct anon_vma *anon_vma = NULL; - bool is_lru = !__PageMovable(page); + bool is_lru = !__PageMovable(&src->page); - if (!trylock_page(page)) { + if (!folio_trylock(src)) { if (!force || mode == MIGRATE_ASYNC) goto out; @@ -1006,10 +1027,10 @@ static int __unmap_and_move(struct page *page, struct page *newpage, if (current->flags & PF_MEMALLOC) goto out; - lock_page(page); + folio_lock(src); } - if (PageWriteback(page)) { + if (folio_test_writeback(src)) { /* * Only in the case of a full synchronous migration is it * necessary to wait for PageWriteback. In the async case, @@ -1026,39 +1047,39 @@ static int __unmap_and_move(struct page *page, struct page *newpage, } if (!force) goto out_unlock; - wait_on_page_writeback(page); + folio_wait_writeback(src); } /* - * By try_to_migrate(), page->mapcount goes down to 0 here. In this case, - * we cannot notice that anon_vma is freed while we migrates a page. + * By try_to_migrate(), src->mapcount goes down to 0 here. In this case, + * we cannot notice that anon_vma is freed while we migrate a page. * This get_anon_vma() delays freeing anon_vma pointer until the end * of migration. File cache pages are no problem because of page_lock() * File Caches may use write_page() or lock_page() in migration, then, * just care Anon page here. * - * Only page_get_anon_vma() understands the subtleties of + * Only folio_get_anon_vma() understands the subtleties of * getting a hold on an anon_vma from outside one of its mms. * But if we cannot get anon_vma, then we won't need it anyway, * because that implies that the anon page is no longer mapped * (and cannot be remapped so long as we hold the page lock). */ - if (PageAnon(page) && !PageKsm(page)) - anon_vma = page_get_anon_vma(page); + if (folio_test_anon(src) && !folio_test_ksm(src)) + anon_vma = folio_get_anon_vma(src); /* * Block others from accessing the new page when we get around to * establishing additional references. We are usually the only one - * holding a reference to newpage at this point. We used to have a BUG - * here if trylock_page(newpage) fails, but would like to allow for - * cases where there might be a race with the previous use of newpage. + * holding a reference to dst at this point. We used to have a BUG + * here if folio_trylock(dst) fails, but would like to allow for + * cases where there might be a race with the previous use of dst. * This is much like races on refcount of oldpage: just don't BUG(). */ - if (unlikely(!trylock_page(newpage))) + if (unlikely(!folio_trylock(dst))) goto out_unlock; if (unlikely(!is_lru)) { - rc = move_to_new_folio(dst, folio, mode); + rc = move_to_new_folio(dst, src, mode); goto out_unlock_both; } @@ -1066,7 +1087,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage, * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU * and treated as swapcache but it has no rmap yet. - * Calling try_to_unmap() against a page->mapping==NULL page will + * Calling try_to_unmap() against a src->mapping==NULL page will * trigger a BUG. So handle it here. * 2. An orphaned page (see truncate_cleanup_page) might have * fs-private metadata. The page can be picked up due to memory @@ -1074,57 +1095,56 @@ static int __unmap_and_move(struct page *page, struct page *newpage, * invisible to the vm, so the page can not be migrated. So try to * free the metadata, so the page can be freed. */ - if (!page->mapping) { - VM_BUG_ON_PAGE(PageAnon(page), page); - if (page_has_private(page)) { - try_to_free_buffers(folio); + if (!src->mapping) { + if (folio_test_private(src)) { + try_to_free_buffers(src); goto out_unlock_both; } - } else if (page_mapped(page)) { + } else if (folio_mapped(src)) { /* Establish migration ptes */ - VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, - page); - try_to_migrate(folio, 0); + VM_BUG_ON_FOLIO(folio_test_anon(src) && + !folio_test_ksm(src) && !anon_vma, src); + try_to_migrate(src, 0); page_was_mapped = true; } - if (!page_mapped(page)) - rc = move_to_new_folio(dst, folio, mode); + if (!folio_mapped(src)) + rc = move_to_new_folio(dst, src, mode); /* - * When successful, push newpage to LRU immediately: so that if it + * When successful, push dst to LRU immediately: so that if it * turns out to be an mlocked page, remove_migration_ptes() will - * automatically build up the correct newpage->mlock_count for it. + * automatically build up the correct dst->mlock_count for it. * * We would like to do something similar for the old page, when * unsuccessful, and other cases when a page has been temporarily * isolated from the unevictable LRU: but this case is the easiest. */ if (rc == MIGRATEPAGE_SUCCESS) { - lru_cache_add(newpage); + folio_add_lru(dst); if (page_was_mapped) lru_add_drain(); } if (page_was_mapped) - remove_migration_ptes(folio, - rc == MIGRATEPAGE_SUCCESS ? dst : folio, false); + remove_migration_ptes(src, + rc == MIGRATEPAGE_SUCCESS ? dst : src, false); out_unlock_both: - unlock_page(newpage); + folio_unlock(dst); out_unlock: /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - unlock_page(page); + folio_unlock(src); out: /* - * If migration is successful, decrease refcount of the newpage, + * If migration is successful, decrease refcount of dst, * which will not free the page because new page owner increased * refcounter. */ if (rc == MIGRATEPAGE_SUCCESS) - put_page(newpage); + folio_put(dst); return rc; } @@ -1140,6 +1160,7 @@ static int unmap_and_move(new_page_t get_new_page, enum migrate_reason reason, struct list_head *ret) { + struct folio *dst, *src = page_folio(page); int rc = MIGRATEPAGE_SUCCESS; struct page *newpage = NULL; @@ -1157,9 +1178,10 @@ static int unmap_and_move(new_page_t get_new_page, newpage = get_new_page(page, private); if (!newpage) return -ENOMEM; + dst = page_folio(newpage); newpage->private = 0; - rc = __unmap_and_move(page, newpage, force, mode); + rc = __unmap_and_move(src, dst, force, mode); if (rc == MIGRATEPAGE_SUCCESS) set_page_owner_migrate_reason(newpage, reason); @@ -1244,12 +1266,10 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, * tables or check whether the hugepage is pmd-based or not before * kicking migration. */ - if (!hugepage_migration_supported(page_hstate(hpage))) { - list_move_tail(&hpage->lru, ret); + if (!hugepage_migration_supported(page_hstate(hpage))) return -ENOSYS; - } - if (page_count(hpage) == 1) { + if (folio_ref_count(src) == 1) { /* page was freed from under us. So we are done. */ putback_active_hugepage(hpage); return MIGRATEPAGE_SUCCESS; @@ -1260,7 +1280,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, return -ENOMEM; dst = page_folio(new_hpage); - if (!trylock_page(hpage)) { + if (!folio_trylock(src)) { if (!force) goto out; switch (mode) { @@ -1270,29 +1290,29 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, default: goto out; } - lock_page(hpage); + folio_lock(src); } /* * Check for pages which are in the process of being freed. Without - * page_mapping() set, hugetlbfs specific move page routine will not + * folio_mapping() set, hugetlbfs specific move page routine will not * be called and we could leak usage counts for subpools. */ - if (hugetlb_page_subpool(hpage) && !page_mapping(hpage)) { + if (hugetlb_page_subpool(hpage) && !folio_mapping(src)) { rc = -EBUSY; goto out_unlock; } - if (PageAnon(hpage)) - anon_vma = page_get_anon_vma(hpage); + if (folio_test_anon(src)) + anon_vma = folio_get_anon_vma(src); - if (unlikely(!trylock_page(new_hpage))) + if (unlikely(!folio_trylock(dst))) goto put_anon; - if (page_mapped(hpage)) { + if (folio_mapped(src)) { enum ttu_flags ttu = 0; - if (!PageAnon(hpage)) { + if (!folio_test_anon(src)) { /* * In shared mappings, try_to_unmap could potentially * call huge_pmd_unshare. Because of this, take @@ -1313,7 +1333,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, i_mmap_unlock_write(mapping); } - if (!page_mapped(hpage)) + if (!folio_mapped(src)) rc = move_to_new_folio(dst, src, mode); if (page_was_mapped) @@ -1321,7 +1341,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, rc == MIGRATEPAGE_SUCCESS ? dst : src, false); unlock_put_anon: - unlock_page(new_hpage); + folio_unlock(dst); put_anon: if (anon_vma) @@ -1333,12 +1353,12 @@ put_anon: } out_unlock: - unlock_page(hpage); + folio_unlock(src); out: if (rc == MIGRATEPAGE_SUCCESS) putback_active_hugepage(hpage); else if (rc != -EAGAIN) - list_move_tail(&hpage->lru, ret); + list_move_tail(&src->lru, ret); /* * If migration was not successful and there's a freeing callback, use @@ -1353,16 +1373,15 @@ out: return rc; } -static inline int try_split_thp(struct page *page, struct page **page2, - struct list_head *from) +static inline int try_split_thp(struct page *page, struct list_head *split_pages) { - int rc = 0; + int rc; lock_page(page); - rc = split_huge_page_to_list(page, from); + rc = split_huge_page_to_list(page, split_pages); unlock_page(page); if (!rc) - list_safe_reset_next(page, *page2, lru); + list_move_tail(&page->lru, split_pages); return rc; } @@ -1400,6 +1419,7 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, int thp_retry = 1; int nr_failed = 0; int nr_failed_pages = 0; + int nr_retry_pages = 0; int nr_succeeded = 0; int nr_thp_succeeded = 0; int nr_thp_failed = 0; @@ -1420,9 +1440,9 @@ thp_subpage_migration: for (pass = 0; pass < 10 && (retry || thp_retry); pass++) { retry = 0; thp_retry = 0; + nr_retry_pages = 0; list_for_each_entry_safe(page, page2, from, lru) { -retry: /* * THP statistics is based on the source huge page. * Capture required information that might get lost @@ -1447,6 +1467,7 @@ retry: * page will be put back * -EAGAIN: stay on the from list * -ENOMEM: stay on the from list + * -ENOSYS: stay on the from list * Other errno: put on ret_pages list then splice to * from list */ @@ -1457,18 +1478,17 @@ retry: * retry on the same page with the THP split * to base pages. * - * Head page is retried immediately and tail - * pages are added to the tail of the list so - * we encounter them after the rest of the list - * is processed. + * Sub-pages are put in thp_split_pages, and + * we will migrate them after the rest of the + * list is processed. */ case -ENOSYS: /* THP migration is unsupported */ if (is_thp) { nr_thp_failed++; - if (!try_split_thp(page, &page2, &thp_split_pages)) { + if (!try_split_thp(page, &thp_split_pages)) { nr_thp_split++; - goto retry; + break; } /* Hugetlb migration is unsupported */ } else if (!no_subpage_counting) { @@ -1476,24 +1496,25 @@ retry: } nr_failed_pages += nr_subpages; + list_move_tail(&page->lru, &ret_pages); break; case -ENOMEM: /* * When memory is low, don't bother to try to migrate * other pages, just exit. - * THP NUMA faulting doesn't split THP to retry. */ - if (is_thp && !nosplit) { + if (is_thp) { nr_thp_failed++; - if (!try_split_thp(page, &page2, &thp_split_pages)) { + /* THP NUMA faulting doesn't split THP to retry. */ + if (!nosplit && !try_split_thp(page, &thp_split_pages)) { nr_thp_split++; - goto retry; + break; } } else if (!no_subpage_counting) { nr_failed++; } - nr_failed_pages += nr_subpages; + nr_failed_pages += nr_subpages + nr_retry_pages; /* * There might be some subpages of fail-to-migrate THPs * left in thp_split_pages list. Move them back to migration @@ -1501,13 +1522,15 @@ retry: * the caller otherwise the page refcnt will be leaked. */ list_splice_init(&thp_split_pages, from); + /* nr_failed isn't updated for not used */ nr_thp_failed += thp_retry; goto out; case -EAGAIN: if (is_thp) thp_retry++; - else + else if (!no_subpage_counting) retry++; + nr_retry_pages += nr_subpages; break; case MIGRATEPAGE_SUCCESS: nr_succeeded += nr_subpages; @@ -1533,6 +1556,7 @@ retry: } nr_failed += retry; nr_thp_failed += thp_retry; + nr_failed_pages += nr_retry_pages; /* * Try to migrate subpages of fail-to-migrate THPs, no nr_failed * counting in this round, since all subpages of a THP is counted @@ -1672,9 +1696,12 @@ static int add_page_for_migration(struct mm_struct *mm, unsigned long addr, goto out; err = -ENOENT; - if (!page || is_zone_device_page(page)) + if (!page) goto out; + if (is_zone_device_page(page)) + goto out_putpage; + err = 0; if (page_to_nid(page) == node) goto out_putpage; @@ -1735,7 +1762,7 @@ static int move_pages_and_store_status(struct mm_struct *mm, int node, * well. */ if (err > 0) - err += nr_pages - i - 1; + err += nr_pages - i; return err; } return store_status(status, start, node, i - start); @@ -1821,8 +1848,12 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes, err = move_pages_and_store_status(mm, current_node, &pagelist, status, start, i, nr_pages); - if (err) + if (err) { + /* We have accounted for page i */ + if (err > 0) + err--; goto out; + } current_node = NUMA_NO_NODE; } out_flush: @@ -1848,6 +1879,7 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, for (i = 0; i < nr_pages; i++) { unsigned long addr = (unsigned long)(*pages); + unsigned int foll_flags = FOLL_DUMP; struct vm_area_struct *vma; struct page *page; int err = -EFAULT; @@ -1856,19 +1888,26 @@ static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, if (!vma) goto set_status; + /* Not all huge page follow APIs support 'FOLL_GET' */ + if (!is_vm_hugetlb_page(vma)) + foll_flags |= FOLL_GET; + /* FOLL_DUMP to ignore special (like zero) pages */ - page = follow_page(vma, addr, FOLL_GET | FOLL_DUMP); + page = follow_page(vma, addr, foll_flags); err = PTR_ERR(page); if (IS_ERR(page)) goto set_status; - if (page && !is_zone_device_page(page)) { + err = -ENOENT; + if (!page) + goto set_status; + + if (!is_zone_device_page(page)) err = page_to_nid(page); + + if (foll_flags & FOLL_GET) put_page(page); - } else { - err = -ENOENT; - } set_status: *status = err; @@ -2170,456 +2209,4 @@ out: return 0; } #endif /* CONFIG_NUMA_BALANCING */ - -/* - * node_demotion[] example: - * - * Consider a system with two sockets. Each socket has - * three classes of memory attached: fast, medium and slow. - * Each memory class is placed in its own NUMA node. The - * CPUs are placed in the node with the "fast" memory. The - * 6 NUMA nodes (0-5) might be split among the sockets like - * this: - * - * Socket A: 0, 1, 2 - * Socket B: 3, 4, 5 - * - * When Node 0 fills up, its memory should be migrated to - * Node 1. When Node 1 fills up, it should be migrated to - * Node 2. The migration path start on the nodes with the - * processors (since allocations default to this node) and - * fast memory, progress through medium and end with the - * slow memory: - * - * 0 -> 1 -> 2 -> stop - * 3 -> 4 -> 5 -> stop - * - * This is represented in the node_demotion[] like this: - * - * { nr=1, nodes[0]=1 }, // Node 0 migrates to 1 - * { nr=1, nodes[0]=2 }, // Node 1 migrates to 2 - * { nr=0, nodes[0]=-1 }, // Node 2 does not migrate - * { nr=1, nodes[0]=4 }, // Node 3 migrates to 4 - * { nr=1, nodes[0]=5 }, // Node 4 migrates to 5 - * { nr=0, nodes[0]=-1 }, // Node 5 does not migrate - * - * Moreover some systems may have multiple slow memory nodes. - * Suppose a system has one socket with 3 memory nodes, node 0 - * is fast memory type, and node 1/2 both are slow memory - * type, and the distance between fast memory node and slow - * memory node is same. So the migration path should be: - * - * 0 -> 1/2 -> stop - * - * This is represented in the node_demotion[] like this: - * { nr=2, {nodes[0]=1, nodes[1]=2} }, // Node 0 migrates to node 1 and node 2 - * { nr=0, nodes[0]=-1, }, // Node 1 dose not migrate - * { nr=0, nodes[0]=-1, }, // Node 2 does not migrate - */ - -/* - * Writes to this array occur without locking. Cycles are - * not allowed: Node X demotes to Y which demotes to X... - * - * If multiple reads are performed, a single rcu_read_lock() - * must be held over all reads to ensure that no cycles are - * observed. - */ -#define DEFAULT_DEMOTION_TARGET_NODES 15 - -#if MAX_NUMNODES < DEFAULT_DEMOTION_TARGET_NODES -#define DEMOTION_TARGET_NODES (MAX_NUMNODES - 1) -#else -#define DEMOTION_TARGET_NODES DEFAULT_DEMOTION_TARGET_NODES -#endif - -struct demotion_nodes { - unsigned short nr; - short nodes[DEMOTION_TARGET_NODES]; -}; - -static struct demotion_nodes *node_demotion __read_mostly; - -/** - * next_demotion_node() - Get the next node in the demotion path - * @node: The starting node to lookup the next node - * - * Return: node id for next memory node in the demotion path hierarchy - * from @node; NUMA_NO_NODE if @node is terminal. This does not keep - * @node online or guarantee that it *continues* to be the next demotion - * target. - */ -int next_demotion_node(int node) -{ - struct demotion_nodes *nd; - unsigned short target_nr, index; - int target; - - if (!node_demotion) - return NUMA_NO_NODE; - - nd = &node_demotion[node]; - - /* - * node_demotion[] is updated without excluding this - * function from running. RCU doesn't provide any - * compiler barriers, so the READ_ONCE() is required - * to avoid compiler reordering or read merging. - * - * Make sure to use RCU over entire code blocks if - * node_demotion[] reads need to be consistent. - */ - rcu_read_lock(); - target_nr = READ_ONCE(nd->nr); - - switch (target_nr) { - case 0: - target = NUMA_NO_NODE; - goto out; - case 1: - index = 0; - break; - default: - /* - * If there are multiple target nodes, just select one - * target node randomly. - * - * In addition, we can also use round-robin to select - * target node, but we should introduce another variable - * for node_demotion[] to record last selected target node, - * that may cause cache ping-pong due to the changing of - * last target node. Or introducing per-cpu data to avoid - * caching issue, which seems more complicated. So selecting - * target node randomly seems better until now. - */ - index = get_random_int() % target_nr; - break; - } - - target = READ_ONCE(nd->nodes[index]); - -out: - rcu_read_unlock(); - return target; -} - -/* Disable reclaim-based migration. */ -static void __disable_all_migrate_targets(void) -{ - int node, i; - - if (!node_demotion) - return; - - for_each_online_node(node) { - node_demotion[node].nr = 0; - for (i = 0; i < DEMOTION_TARGET_NODES; i++) - node_demotion[node].nodes[i] = NUMA_NO_NODE; - } -} - -static void disable_all_migrate_targets(void) -{ - __disable_all_migrate_targets(); - - /* - * Ensure that the "disable" is visible across the system. - * Readers will see either a combination of before+disable - * state or disable+after. They will never see before and - * after state together. - * - * The before+after state together might have cycles and - * could cause readers to do things like loop until this - * function finishes. This ensures they can only see a - * single "bad" read and would, for instance, only loop - * once. - */ - synchronize_rcu(); -} - -/* - * Find an automatic demotion target for 'node'. - * Failing here is OK. It might just indicate - * being at the end of a chain. - */ -static int establish_migrate_target(int node, nodemask_t *used, - int best_distance) -{ - int migration_target, index, val; - struct demotion_nodes *nd; - - if (!node_demotion) - return NUMA_NO_NODE; - - nd = &node_demotion[node]; - - migration_target = find_next_best_node(node, used); - if (migration_target == NUMA_NO_NODE) - return NUMA_NO_NODE; - - /* - * If the node has been set a migration target node before, - * which means it's the best distance between them. Still - * check if this node can be demoted to other target nodes - * if they have a same best distance. - */ - if (best_distance != -1) { - val = node_distance(node, migration_target); - if (val > best_distance) - goto out_clear; - } - - index = nd->nr; - if (WARN_ONCE(index >= DEMOTION_TARGET_NODES, - "Exceeds maximum demotion target nodes\n")) - goto out_clear; - - nd->nodes[index] = migration_target; - nd->nr++; - - return migration_target; -out_clear: - node_clear(migration_target, *used); - return NUMA_NO_NODE; -} - -/* - * When memory fills up on a node, memory contents can be - * automatically migrated to another node instead of - * discarded at reclaim. - * - * Establish a "migration path" which will start at nodes - * with CPUs and will follow the priorities used to build the - * page allocator zonelists. - * - * The difference here is that cycles must be avoided. If - * node0 migrates to node1, then neither node1, nor anything - * node1 migrates to can migrate to node0. Also one node can - * be migrated to multiple nodes if the target nodes all have - * a same best-distance against the source node. - * - * This function can run simultaneously with readers of - * node_demotion[]. However, it can not run simultaneously - * with itself. Exclusion is provided by memory hotplug events - * being single-threaded. - */ -static void __set_migration_target_nodes(void) -{ - nodemask_t next_pass; - nodemask_t this_pass; - nodemask_t used_targets = NODE_MASK_NONE; - int node, best_distance; - - /* - * Avoid any oddities like cycles that could occur - * from changes in the topology. This will leave - * a momentary gap when migration is disabled. - */ - disable_all_migrate_targets(); - - /* - * Allocations go close to CPUs, first. Assume that - * the migration path starts at the nodes with CPUs. - */ - next_pass = node_states[N_CPU]; -again: - this_pass = next_pass; - next_pass = NODE_MASK_NONE; - /* - * To avoid cycles in the migration "graph", ensure - * that migration sources are not future targets by - * setting them in 'used_targets'. Do this only - * once per pass so that multiple source nodes can - * share a target node. - * - * 'used_targets' will become unavailable in future - * passes. This limits some opportunities for - * multiple source nodes to share a destination. - */ - nodes_or(used_targets, used_targets, this_pass); - - for_each_node_mask(node, this_pass) { - best_distance = -1; - - /* - * Try to set up the migration path for the node, and the target - * migration nodes can be multiple, so doing a loop to find all - * the target nodes if they all have a best node distance. - */ - do { - int target_node = - establish_migrate_target(node, &used_targets, - best_distance); - - if (target_node == NUMA_NO_NODE) - break; - - if (best_distance == -1) - best_distance = node_distance(node, target_node); - - /* - * Visit targets from this pass in the next pass. - * Eventually, every node will have been part of - * a pass, and will become set in 'used_targets'. - */ - node_set(target_node, next_pass); - } while (1); - } - /* - * 'next_pass' contains nodes which became migration - * targets in this pass. Make additional passes until - * no more migrations targets are available. - */ - if (!nodes_empty(next_pass)) - goto again; -} - -/* - * For callers that do not hold get_online_mems() already. - */ -void set_migration_target_nodes(void) -{ - get_online_mems(); - __set_migration_target_nodes(); - put_online_mems(); -} - -/* - * This leaves migrate-on-reclaim transiently disabled between - * the MEM_GOING_OFFLINE and MEM_OFFLINE events. This runs - * whether reclaim-based migration is enabled or not, which - * ensures that the user can turn reclaim-based migration at - * any time without needing to recalculate migration targets. - * - * These callbacks already hold get_online_mems(). That is why - * __set_migration_target_nodes() can be used as opposed to - * set_migration_target_nodes(). - */ -#ifdef CONFIG_MEMORY_HOTPLUG -static int __meminit migrate_on_reclaim_callback(struct notifier_block *self, - unsigned long action, void *_arg) -{ - struct memory_notify *arg = _arg; - - /* - * Only update the node migration order when a node is - * changing status, like online->offline. This avoids - * the overhead of synchronize_rcu() in most cases. - */ - if (arg->status_change_nid < 0) - return notifier_from_errno(0); - - switch (action) { - case MEM_GOING_OFFLINE: - /* - * Make sure there are not transient states where - * an offline node is a migration target. This - * will leave migration disabled until the offline - * completes and the MEM_OFFLINE case below runs. - */ - disable_all_migrate_targets(); - break; - case MEM_OFFLINE: - case MEM_ONLINE: - /* - * Recalculate the target nodes once the node - * reaches its final state (online or offline). - */ - __set_migration_target_nodes(); - break; - case MEM_CANCEL_OFFLINE: - /* - * MEM_GOING_OFFLINE disabled all the migration - * targets. Reenable them. - */ - __set_migration_target_nodes(); - break; - case MEM_GOING_ONLINE: - case MEM_CANCEL_ONLINE: - break; - } - - return notifier_from_errno(0); -} -#endif - -void __init migrate_on_reclaim_init(void) -{ - node_demotion = kcalloc(nr_node_ids, - sizeof(struct demotion_nodes), - GFP_KERNEL); - WARN_ON(!node_demotion); -#ifdef CONFIG_MEMORY_HOTPLUG - hotplug_memory_notifier(migrate_on_reclaim_callback, 100); -#endif - /* - * At this point, all numa nodes with memory/CPus have their state - * properly set, so we can build the demotion order now. - * Let us hold the cpu_hotplug lock just, as we could possibily have - * CPU hotplug events during boot. - */ - cpus_read_lock(); - set_migration_target_nodes(); - cpus_read_unlock(); -} - -bool numa_demotion_enabled = false; - -#ifdef CONFIG_SYSFS -static ssize_t numa_demotion_enabled_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%s\n", - numa_demotion_enabled ? "true" : "false"); -} - -static ssize_t numa_demotion_enabled_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buf, size_t count) -{ - ssize_t ret; - - ret = kstrtobool(buf, &numa_demotion_enabled); - if (ret) - return ret; - - return count; -} - -static struct kobj_attribute numa_demotion_enabled_attr = - __ATTR(demotion_enabled, 0644, numa_demotion_enabled_show, - numa_demotion_enabled_store); - -static struct attribute *numa_attrs[] = { - &numa_demotion_enabled_attr.attr, - NULL, -}; - -static const struct attribute_group numa_attr_group = { - .attrs = numa_attrs, -}; - -static int __init numa_init_sysfs(void) -{ - int err; - struct kobject *numa_kobj; - - numa_kobj = kobject_create_and_add("numa", mm_kobj); - if (!numa_kobj) { - pr_err("failed to create numa kobject\n"); - return -ENOMEM; - } - err = sysfs_create_group(numa_kobj, &numa_attr_group); - if (err) { - pr_err("failed to register numa group\n"); - goto delete_obj; - } - return 0; - -delete_obj: - kobject_put(numa_kobj); - return err; -} -subsys_initcall(numa_init_sysfs); -#endif /* CONFIG_SYSFS */ #endif /* CONFIG_NUMA */ diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 27fb37d65476c24a3103a6c3cf830f4982255c7d..6fa682eef7a002737e35fd32a96e2afadc184ebd 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -185,18 +186,25 @@ again: get_page(page); /* - * Optimize for the common case where page is only mapped once - * in one process. If we can lock the page, then we can safely - * set up a special migration page table entry now. + * We rely on trylock_page() to avoid deadlock between + * concurrent migrations where each is waiting on the others + * page lock. If we can't immediately lock the page we fail this + * migration as it is only best effort anyway. + * + * If we can lock the page it's safe to set up a migration entry + * now. In the common case where the page is mapped once in a + * single process setting up the migration entry now is an + * optimisation to avoid walking the rmap later with + * try_to_migrate(). */ if (trylock_page(page)) { bool anon_exclusive; pte_t swp_pte; + flush_cache_page(vma, addr, pte_pfn(*ptep)); anon_exclusive = PageAnon(page) && PageAnonExclusive(page); if (anon_exclusive) { - flush_cache_page(vma, addr, pte_pfn(*ptep)); - ptep_clear_flush(vma, addr, ptep); + pte = ptep_clear_flush(vma, addr, ptep); if (page_try_share_anon_rmap(page)) { set_pte_at(mm, addr, ptep, pte); @@ -206,11 +214,15 @@ again: goto next; } } else { - ptep_get_and_clear(mm, addr, ptep); + pte = ptep_get_and_clear(mm, addr, ptep); } migrate->cpages++; + /* Set the dirty flag on the folio now the pte is gone. */ + if (pte_dirty(pte)) + folio_mark_dirty(page_folio(page)); + /* Setup special migration page table entry */ if (mpfn & MIGRATE_PFN_WRITE) entry = make_writable_migration_entry( @@ -221,6 +233,12 @@ again: else entry = make_readable_migration_entry( page_to_pfn(page)); + if (pte_present(pte)) { + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + } swp_pte = swp_entry_to_pte(entry); if (pte_present(pte)) { if (pte_soft_dirty(pte)) @@ -254,13 +272,14 @@ next: migrate->dst[migrate->npages] = 0; migrate->src[migrate->npages++] = mpfn; } - arch_leave_lazy_mmu_mode(); - pte_unmap_unlock(ptep - 1, ptl); /* Only flush the TLB if we actually modified any entries */ if (unmapped) flush_tlb_range(walk->vma, start, end); + arch_leave_lazy_mmu_mode(); + pte_unmap_unlock(ptep - 1, ptl); + return 0; } @@ -306,14 +325,14 @@ static void migrate_vma_collect(struct migrate_vma *migrate) * folio_migrate_mapping(), except that here we allow migration of a * ZONE_DEVICE page. */ -static bool migrate_vma_check_page(struct page *page) +static bool migrate_vma_check_page(struct page *page, struct page *fault_page) { /* * One extra ref because caller holds an extra reference, either from * isolate_lru_page() for a regular page, or migrate_vma_collect() for * a device page. */ - int extra = 1; + int extra = 1 + (page == fault_page); /* * FIXME support THP (transparent huge page), it is bit more complex to @@ -338,26 +357,20 @@ static bool migrate_vma_check_page(struct page *page) } /* - * migrate_vma_unmap() - replace page mapping with special migration pte entry - * @migrate: migrate struct containing all migration information - * - * Isolate pages from the LRU and replace mappings (CPU page table pte) with a - * special migration pte entry and check if it has been pinned. Pinned pages are - * restored because we cannot migrate them. - * - * This is the last step before we call the device driver callback to allocate - * destination memory and copy contents of original page over to new page. + * Unmaps pages for migration. Returns number of unmapped pages. */ -static void migrate_vma_unmap(struct migrate_vma *migrate) +static unsigned long migrate_device_unmap(unsigned long *src_pfns, + unsigned long npages, + struct page *fault_page) { - const unsigned long npages = migrate->npages; unsigned long i, restore = 0; bool allow_drain = true; + unsigned long unmapped = 0; lru_add_drain(); for (i = 0; i < npages; i++) { - struct page *page = migrate_pfn_to_page(migrate->src[i]); + struct page *page = migrate_pfn_to_page(src_pfns[i]); struct folio *folio; if (!page) @@ -372,8 +385,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) } if (isolate_lru_page(page)) { - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; - migrate->cpages--; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; restore++; continue; } @@ -386,34 +398,55 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) if (folio_mapped(folio)) try_to_migrate(folio, 0); - if (page_mapped(page) || !migrate_vma_check_page(page)) { + if (page_mapped(page) || + !migrate_vma_check_page(page, fault_page)) { if (!is_zone_device_page(page)) { get_page(page); putback_lru_page(page); } - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; - migrate->cpages--; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; restore++; continue; } + + unmapped++; } for (i = 0; i < npages && restore; i++) { - struct page *page = migrate_pfn_to_page(migrate->src[i]); + struct page *page = migrate_pfn_to_page(src_pfns[i]); struct folio *folio; - if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE)) + if (!page || (src_pfns[i] & MIGRATE_PFN_MIGRATE)) continue; folio = page_folio(page); remove_migration_ptes(folio, folio, false); - migrate->src[i] = 0; + src_pfns[i] = 0; folio_unlock(folio); folio_put(folio); restore--; } + + return unmapped; +} + +/* + * migrate_vma_unmap() - replace page mapping with special migration pte entry + * @migrate: migrate struct containing all migration information + * + * Isolate pages from the LRU and replace mappings (CPU page table pte) with a + * special migration pte entry and check if it has been pinned. Pinned pages are + * restored because we cannot migrate them. + * + * This is the last step before we call the device driver callback to allocate + * destination memory and copy contents of original page over to new page. + */ +static void migrate_vma_unmap(struct migrate_vma *migrate) +{ + migrate->cpages = migrate_device_unmap(migrate->src, migrate->npages, + migrate->fault_page); } /** @@ -498,6 +531,8 @@ int migrate_vma_setup(struct migrate_vma *args) return -EINVAL; if (!args->src || !args->dst) return -EINVAL; + if (args->fault_page && !is_device_private_page(args->fault_page)) + return -EINVAL; memset(args->src, 0, sizeof(*args->src) * nr_pages); args->cpages = 0; @@ -658,42 +693,38 @@ abort: *src &= ~MIGRATE_PFN_MIGRATE; } -/** - * migrate_vma_pages() - migrate meta-data from src page to dst page - * @migrate: migrate struct containing all migration information - * - * This migrates struct page meta-data from source struct page to destination - * struct page. This effectively finishes the migration from source page to the - * destination page. - */ -void migrate_vma_pages(struct migrate_vma *migrate) +static void __migrate_device_pages(unsigned long *src_pfns, + unsigned long *dst_pfns, unsigned long npages, + struct migrate_vma *migrate) { - const unsigned long npages = migrate->npages; - const unsigned long start = migrate->start; struct mmu_notifier_range range; - unsigned long addr, i; + unsigned long i; bool notified = false; - for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) { - struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); - struct page *page = migrate_pfn_to_page(migrate->src[i]); + for (i = 0; i < npages; i++) { + struct page *newpage = migrate_pfn_to_page(dst_pfns[i]); + struct page *page = migrate_pfn_to_page(src_pfns[i]); struct address_space *mapping; int r; if (!newpage) { - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; continue; } if (!page) { + unsigned long addr; + + if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + /* * The only time there is no vma is when called from * migrate_device_coherent_page(). However this isn't * called if the page could not be unmapped. */ - VM_BUG_ON(!migrate->vma); - if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) - continue; + VM_BUG_ON(!migrate); + addr = migrate->start + i*PAGE_SIZE; if (!notified) { notified = true; @@ -704,7 +735,7 @@ void migrate_vma_pages(struct migrate_vma *migrate) mmu_notifier_invalidate_range_start(&range); } migrate_vma_insert_page(migrate, addr, newpage, - &migrate->src[i]); + &src_pfns[i]); continue; } @@ -717,21 +748,26 @@ void migrate_vma_pages(struct migrate_vma *migrate) * device private or coherent memory. */ if (mapping) { - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; continue; } } else if (is_zone_device_page(newpage)) { /* * Other types of ZONE_DEVICE page are not supported. */ - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; continue; } - r = migrate_folio(mapping, page_folio(newpage), - page_folio(page), MIGRATE_SYNC_NO_COPY); + if (migrate && migrate->fault_page == page) + r = migrate_folio_extra(mapping, page_folio(newpage), + page_folio(page), + MIGRATE_SYNC_NO_COPY, 1); + else + r = migrate_folio(mapping, page_folio(newpage), + page_folio(page), MIGRATE_SYNC_NO_COPY); if (r != MIGRATEPAGE_SUCCESS) - migrate->src[i] &= ~MIGRATE_PFN_MIGRATE; + src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; } /* @@ -742,28 +778,56 @@ void migrate_vma_pages(struct migrate_vma *migrate) if (notified) mmu_notifier_invalidate_range_only_end(&range); } -EXPORT_SYMBOL(migrate_vma_pages); /** - * migrate_vma_finalize() - restore CPU page table entry + * migrate_device_pages() - migrate meta-data from src page to dst page + * @src_pfns: src_pfns returned from migrate_device_range() + * @dst_pfns: array of pfns allocated by the driver to migrate memory to + * @npages: number of pages in the range + * + * Equivalent to migrate_vma_pages(). This is called to migrate struct page + * meta-data from source struct page to destination. + */ +void migrate_device_pages(unsigned long *src_pfns, unsigned long *dst_pfns, + unsigned long npages) +{ + __migrate_device_pages(src_pfns, dst_pfns, npages, NULL); +} +EXPORT_SYMBOL(migrate_device_pages); + +/** + * migrate_vma_pages() - migrate meta-data from src page to dst page * @migrate: migrate struct containing all migration information * - * This replaces the special migration pte entry with either a mapping to the - * new page if migration was successful for that page, or to the original page - * otherwise. + * This migrates struct page meta-data from source struct page to destination + * struct page. This effectively finishes the migration from source page to the + * destination page. + */ +void migrate_vma_pages(struct migrate_vma *migrate) +{ + __migrate_device_pages(migrate->src, migrate->dst, migrate->npages, migrate); +} +EXPORT_SYMBOL(migrate_vma_pages); + +/* + * migrate_device_finalize() - complete page migration + * @src_pfns: src_pfns returned from migrate_device_range() + * @dst_pfns: array of pfns allocated by the driver to migrate memory to + * @npages: number of pages in the range * - * This also unlocks the pages and puts them back on the lru, or drops the extra - * refcount, for device pages. + * Completes migration of the page by removing special migration entries. + * Drivers must ensure copying of page data is complete and visible to the CPU + * before calling this. */ -void migrate_vma_finalize(struct migrate_vma *migrate) +void migrate_device_finalize(unsigned long *src_pfns, + unsigned long *dst_pfns, unsigned long npages) { - const unsigned long npages = migrate->npages; unsigned long i; for (i = 0; i < npages; i++) { struct folio *dst, *src; - struct page *newpage = migrate_pfn_to_page(migrate->dst[i]); - struct page *page = migrate_pfn_to_page(migrate->src[i]); + struct page *newpage = migrate_pfn_to_page(dst_pfns[i]); + struct page *page = migrate_pfn_to_page(src_pfns[i]); if (!page) { if (newpage) { @@ -773,7 +837,7 @@ void migrate_vma_finalize(struct migrate_vma *migrate) continue; } - if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) { + if (!(src_pfns[i] & MIGRATE_PFN_MIGRATE) || !newpage) { if (newpage) { unlock_page(newpage); put_page(newpage); @@ -800,8 +864,72 @@ void migrate_vma_finalize(struct migrate_vma *migrate) } } } +EXPORT_SYMBOL(migrate_device_finalize); + +/** + * migrate_vma_finalize() - restore CPU page table entry + * @migrate: migrate struct containing all migration information + * + * This replaces the special migration pte entry with either a mapping to the + * new page if migration was successful for that page, or to the original page + * otherwise. + * + * This also unlocks the pages and puts them back on the lru, or drops the extra + * refcount, for device pages. + */ +void migrate_vma_finalize(struct migrate_vma *migrate) +{ + migrate_device_finalize(migrate->src, migrate->dst, migrate->npages); +} EXPORT_SYMBOL(migrate_vma_finalize); +/** + * migrate_device_range() - migrate device private pfns to normal memory. + * @src_pfns: array large enough to hold migrating source device private pfns. + * @start: starting pfn in the range to migrate. + * @npages: number of pages to migrate. + * + * migrate_vma_setup() is similar in concept to migrate_vma_setup() except that + * instead of looking up pages based on virtual address mappings a range of + * device pfns that should be migrated to system memory is used instead. + * + * This is useful when a driver needs to free device memory but doesn't know the + * virtual mappings of every page that may be in device memory. For example this + * is often the case when a driver is being unloaded or unbound from a device. + * + * Like migrate_vma_setup() this function will take a reference and lock any + * migrating pages that aren't free before unmapping them. Drivers may then + * allocate destination pages and start copying data from the device to CPU + * memory before calling migrate_device_pages(). + */ +int migrate_device_range(unsigned long *src_pfns, unsigned long start, + unsigned long npages) +{ + unsigned long i, pfn; + + for (pfn = start, i = 0; i < npages; pfn++, i++) { + struct page *page = pfn_to_page(pfn); + + if (!get_page_unless_zero(page)) { + src_pfns[i] = 0; + continue; + } + + if (!trylock_page(page)) { + src_pfns[i] = 0; + put_page(page); + continue; + } + + src_pfns[i] = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE; + } + + migrate_device_unmap(src_pfns, npages, NULL); + + return 0; +} +EXPORT_SYMBOL(migrate_device_range); + /* * Migrate a device coherent page back to normal memory. The caller should have * a reference on page which will be copied to the new page if migration is @@ -810,25 +938,19 @@ EXPORT_SYMBOL(migrate_vma_finalize); int migrate_device_coherent_page(struct page *page) { unsigned long src_pfn, dst_pfn = 0; - struct migrate_vma args; struct page *dpage; WARN_ON_ONCE(PageCompound(page)); lock_page(page); src_pfn = migrate_pfn(page_to_pfn(page)) | MIGRATE_PFN_MIGRATE; - args.src = &src_pfn; - args.dst = &dst_pfn; - args.cpages = 1; - args.npages = 1; - args.vma = NULL; /* * We don't have a VMA and don't need to walk the page tables to find * the source page. So call migrate_vma_unmap() directly to unmap the * page as migrate_vma_setup() will fail if args.vma == NULL. */ - migrate_vma_unmap(&args); + migrate_device_unmap(&src_pfn, 1, NULL); if (!(src_pfn & MIGRATE_PFN_MIGRATE)) return -EBUSY; @@ -838,10 +960,10 @@ int migrate_device_coherent_page(struct page *page) dst_pfn = migrate_pfn(page_to_pfn(dpage)); } - migrate_vma_pages(&args); + migrate_device_pages(&src_pfn, &dst_pfn, 1); if (src_pfn & MIGRATE_PFN_MIGRATE) copy_highpage(dpage, page); - migrate_vma_finalize(&args); + migrate_device_finalize(&src_pfn, &dst_pfn, 1); if (src_pfn & MIGRATE_PFN_MIGRATE) return 0; diff --git a/mm/mlock.c b/mm/mlock.c index b14e929084ccaa5b86d5c9c199d7054b10e0ccab..7032f6dd0ce198301598ef4811864f83bb5c2c47 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -471,6 +471,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, unsigned long nstart, end, tmp; struct vm_area_struct *vma, *prev; int error; + MA_STATE(mas, ¤t->mm->mm_mt, start, start); VM_BUG_ON(offset_in_page(start)); VM_BUG_ON(len != PAGE_ALIGN(len)); @@ -479,13 +480,14 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, return -EINVAL; if (end == start) return 0; - vma = find_vma(current->mm, start); - if (!vma || vma->vm_start > start) + vma = mas_walk(&mas); + if (!vma) return -ENOMEM; - prev = vma->vm_prev; if (start > vma->vm_start) prev = vma; + else + prev = mas_prev(&mas, 0); for (nstart = start ; ; ) { vm_flags_t newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; @@ -505,7 +507,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, if (nstart >= end) break; - vma = prev->vm_next; + vma = find_vma(prev->vm_mm, prev->vm_end); if (!vma || vma->vm_start != nstart) { error = -ENOMEM; break; @@ -526,24 +528,21 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, { struct vm_area_struct *vma; unsigned long count = 0; + unsigned long end; + VMA_ITERATOR(vmi, mm, start); - if (mm == NULL) - mm = current->mm; + /* Don't overflow past ULONG_MAX */ + if (unlikely(ULONG_MAX - len < start)) + end = ULONG_MAX; + else + end = start + len; - vma = find_vma(mm, start); - if (vma == NULL) - return 0; - - for (; vma ; vma = vma->vm_next) { - if (start >= vma->vm_end) - continue; - if (start + len <= vma->vm_start) - break; + for_each_vma_range(vmi, vma, end) { if (vma->vm_flags & VM_LOCKED) { if (start > vma->vm_start) count -= (start - vma->vm_start); - if (start + len < vma->vm_end) { - count += start + len - vma->vm_start; + if (end < vma->vm_end) { + count += end - vma->vm_start; break; } count += vma->vm_end - vma->vm_start; @@ -659,6 +658,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) */ static int apply_mlockall_flags(int flags) { + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); struct vm_area_struct *vma, *prev = NULL; vm_flags_t to_add = 0; @@ -679,7 +679,7 @@ static int apply_mlockall_flags(int flags) to_add |= VM_LOCKONFAULT; } - for (vma = current->mm->mmap; vma ; vma = prev->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { vm_flags_t newflags; newflags = vma->vm_flags & VM_LOCKED_CLEAR_MASK; @@ -687,6 +687,7 @@ static int apply_mlockall_flags(int flags) /* Ignore errors */ mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags); + mas_pause(&mas); cond_resched(); } out: diff --git a/mm/mm_init.c b/mm/mm_init.c index 9ddaf0e1b0ab95fba2a5d7eee30899ccbd7de701..0d7b2bd2454a1f36b7d82976fda0ae10f4ab5dff 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -65,14 +65,16 @@ void __init mminit_verify_pageflags_layout(void) shift = 8 * sizeof(unsigned long); width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH - - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH; + - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", - "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Flags %d\n", + "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", SECTIONS_WIDTH, NODES_WIDTH, ZONES_WIDTH, LAST_CPUPID_WIDTH, KASAN_TAG_WIDTH, + LRU_GEN_WIDTH, + LRU_REFS_WIDTH, NR_PAGEFLAGS); mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", diff --git a/mm/mm_slot.h b/mm/mm_slot.h new file mode 100644 index 0000000000000000000000000000000000000000..83f18ed1c4bde71eaab92a67ae64db3e466d65f3 --- /dev/null +++ b/mm/mm_slot.h @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef _LINUX_MM_SLOT_H +#define _LINUX_MM_SLOT_H + +#include +#include + +/* + * struct mm_slot - hash lookup from mm to mm_slot + * @hash: link to the mm_slots hash list + * @mm_node: link into the mm_slots list + * @mm: the mm that this information is valid for + */ +struct mm_slot { + struct hlist_node hash; + struct list_head mm_node; + struct mm_struct *mm; +}; + +#define mm_slot_entry(ptr, type, member) \ + container_of(ptr, type, member) + +static inline void *mm_slot_alloc(struct kmem_cache *cache) +{ + if (!cache) /* initialization failed */ + return NULL; + return kmem_cache_zalloc(cache, GFP_KERNEL); +} + +static inline void mm_slot_free(struct kmem_cache *cache, void *objp) +{ + kmem_cache_free(cache, objp); +} + +#define mm_slot_lookup(_hashtable, _mm) \ +({ \ + struct mm_slot *tmp_slot, *mm_slot = NULL; \ + \ + hash_for_each_possible(_hashtable, tmp_slot, hash, (unsigned long)_mm) \ + if (_mm == tmp_slot->mm) { \ + mm_slot = tmp_slot; \ + break; \ + } \ + \ + mm_slot; \ +}) + +#define mm_slot_insert(_hashtable, _mm, _mm_slot) \ +({ \ + _mm_slot->mm = _mm; \ + hash_add(_hashtable, &_mm_slot->hash, (unsigned long)_mm); \ +}) + +#endif /* _LINUX_MM_SLOT_H */ diff --git a/mm/mmap.c b/mm/mmap.c index c035020d0c896971488ead38bacbfb68743ceaf6..bf2122af94e7a3ffda516efb9c5270537326120d 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +38,6 @@ #include #include #include -#include #include #include #include @@ -77,9 +75,10 @@ int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS; static bool ignore_rlimit_data; core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644); -static void unmap_region(struct mm_struct *mm, +static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, struct vm_area_struct *vma, struct vm_area_struct *prev, - unsigned long start, unsigned long end); + struct vm_area_struct *next, unsigned long start, + unsigned long end); static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags) { @@ -132,12 +131,10 @@ void unlink_file_vma(struct vm_area_struct *vma) } /* - * Close a vm structure and free it, returning the next. + * Close a vm structure and free it. */ -static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) +static void remove_vma(struct vm_area_struct *vma) { - struct vm_area_struct *next = vma->vm_next; - might_sleep(); if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); @@ -145,20 +142,41 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) fput(vma->vm_file); mpol_put(vma_policy(vma)); vm_area_free(vma); - return next; } -static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, - struct list_head *uf); +/* + * check_brk_limits() - Use platform specific check of range & verify mlock + * limits. + * @addr: The address to check + * @len: The size of increase. + * + * Return: 0 on success. + */ +static int check_brk_limits(unsigned long addr, unsigned long len) +{ + unsigned long mapped_addr; + + mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); + if (IS_ERR_VALUE(mapped_addr)) + return mapped_addr; + + return mlock_future_check(current->mm, current->mm->def_flags, len); +} +static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, + unsigned long newbrk, unsigned long oldbrk, + struct list_head *uf); +static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *brkvma, + unsigned long addr, unsigned long request, unsigned long flags); SYSCALL_DEFINE1(brk, unsigned long, brk) { unsigned long newbrk, oldbrk, origbrk; struct mm_struct *mm = current->mm; - struct vm_area_struct *next; + struct vm_area_struct *brkvma, *next = NULL; unsigned long min_brk; bool populate; bool downgraded = false; LIST_HEAD(uf); + MA_STATE(mas, &mm->mm_mt, 0, 0); if (mmap_write_lock_killable(mm)) return -EINTR; @@ -200,35 +218,51 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) /* * Always allow shrinking brk. - * __do_munmap() may downgrade mmap_lock to read. + * do_brk_munmap() may downgrade mmap_lock to read. */ if (brk <= mm->brk) { int ret; + /* Search one past newbrk */ + mas_set(&mas, newbrk); + brkvma = mas_find(&mas, oldbrk); + BUG_ON(brkvma == NULL); + if (brkvma->vm_start >= oldbrk) + goto out; /* mapping intersects with an existing non-brk vma. */ /* - * mm->brk must to be protected by write mmap_lock so update it - * before downgrading mmap_lock. When __do_munmap() fails, - * mm->brk will be restored from origbrk. + * mm->brk must be protected by write mmap_lock. + * do_brk_munmap() may downgrade the lock, so update it + * before calling do_brk_munmap(). */ mm->brk = brk; - ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true); - if (ret < 0) { - mm->brk = origbrk; - goto out; - } else if (ret == 1) { + ret = do_brk_munmap(&mas, brkvma, newbrk, oldbrk, &uf); + if (ret == 1) { downgraded = true; - } - goto success; + goto success; + } else if (!ret) + goto success; + + mm->brk = origbrk; + goto out; } - /* Check against existing mmap mappings. */ - next = find_vma(mm, oldbrk); + if (check_brk_limits(oldbrk, newbrk - oldbrk)) + goto out; + + /* + * Only check if the next VMA is within the stack_guard_gap of the + * expansion area + */ + mas_set(&mas, oldbrk); + next = mas_find(&mas, newbrk - 1 + PAGE_SIZE + stack_guard_gap); if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) goto out; + brkvma = mas_prev(&mas, mm->start_brk); /* Ok, looks good - let it rip. */ - if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) + if (do_brk_flags(&mas, brkvma, oldbrk, newbrk - oldbrk, 0) < 0) goto out; + mm->brk = brk; success: @@ -247,104 +281,45 @@ out: return origbrk; } -static inline unsigned long vma_compute_gap(struct vm_area_struct *vma) -{ - unsigned long gap, prev_end; - - /* - * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we - * allow two stack_guard_gaps between them here, and when choosing - * an unmapped area; whereas when expanding we only require one. - * That's a little inconsistent, but keeps the code here simpler. - */ - gap = vm_start_gap(vma); - if (vma->vm_prev) { - prev_end = vm_end_gap(vma->vm_prev); - if (gap > prev_end) - gap -= prev_end; - else - gap = 0; - } - return gap; -} - -#ifdef CONFIG_DEBUG_VM_RB -static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma) -{ - unsigned long max = vma_compute_gap(vma), subtree_gap; - if (vma->vm_rb.rb_left) { - subtree_gap = rb_entry(vma->vm_rb.rb_left, - struct vm_area_struct, vm_rb)->rb_subtree_gap; - if (subtree_gap > max) - max = subtree_gap; - } - if (vma->vm_rb.rb_right) { - subtree_gap = rb_entry(vma->vm_rb.rb_right, - struct vm_area_struct, vm_rb)->rb_subtree_gap; - if (subtree_gap > max) - max = subtree_gap; - } - return max; -} - -static int browse_rb(struct mm_struct *mm) -{ - struct rb_root *root = &mm->mm_rb; - int i = 0, j, bug = 0; - struct rb_node *nd, *pn = NULL; - unsigned long prev = 0, pend = 0; - - for (nd = rb_first(root); nd; nd = rb_next(nd)) { - struct vm_area_struct *vma; - vma = rb_entry(nd, struct vm_area_struct, vm_rb); - if (vma->vm_start < prev) { - pr_emerg("vm_start %lx < prev %lx\n", - vma->vm_start, prev); - bug = 1; - } - if (vma->vm_start < pend) { - pr_emerg("vm_start %lx < pend %lx\n", - vma->vm_start, pend); - bug = 1; - } - if (vma->vm_start > vma->vm_end) { - pr_emerg("vm_start %lx > vm_end %lx\n", - vma->vm_start, vma->vm_end); - bug = 1; - } - spin_lock(&mm->page_table_lock); - if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) { - pr_emerg("free gap %lx, correct %lx\n", - vma->rb_subtree_gap, - vma_compute_subtree_gap(vma)); - bug = 1; +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) +extern void mt_validate(struct maple_tree *mt); +extern void mt_dump(const struct maple_tree *mt); + +/* Validate the maple tree */ +static void validate_mm_mt(struct mm_struct *mm) +{ + struct maple_tree *mt = &mm->mm_mt; + struct vm_area_struct *vma_mt; + + MA_STATE(mas, mt, 0, 0); + + mt_validate(&mm->mm_mt); + mas_for_each(&mas, vma_mt, ULONG_MAX) { + if ((vma_mt->vm_start != mas.index) || + (vma_mt->vm_end - 1 != mas.last)) { + pr_emerg("issue in %s\n", current->comm); + dump_stack(); + dump_vma(vma_mt); + pr_emerg("mt piv: %p %lu - %lu\n", vma_mt, + mas.index, mas.last); + pr_emerg("mt vma: %p %lu - %lu\n", vma_mt, + vma_mt->vm_start, vma_mt->vm_end); + + mt_dump(mas.tree); + if (vma_mt->vm_end != mas.last + 1) { + pr_err("vma: %p vma_mt %lu-%lu\tmt %lu-%lu\n", + mm, vma_mt->vm_start, vma_mt->vm_end, + mas.index, mas.last); + mt_dump(mas.tree); + } + VM_BUG_ON_MM(vma_mt->vm_end != mas.last + 1, mm); + if (vma_mt->vm_start != mas.index) { + pr_err("vma: %p vma_mt %p %lu - %lu doesn't match\n", + mm, vma_mt, vma_mt->vm_start, vma_mt->vm_end); + mt_dump(mas.tree); + } + VM_BUG_ON_MM(vma_mt->vm_start != mas.index, mm); } - spin_unlock(&mm->page_table_lock); - i++; - pn = nd; - prev = vma->vm_start; - pend = vma->vm_end; - } - j = 0; - for (nd = pn; nd; nd = rb_prev(nd)) - j++; - if (i != j) { - pr_emerg("backwards %d, forwards %d\n", j, i); - bug = 1; - } - return bug ? -1 : i; -} - -static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore) -{ - struct rb_node *nd; - - for (nd = rb_first(root); nd; nd = rb_next(nd)) { - struct vm_area_struct *vma; - vma = rb_entry(nd, struct vm_area_struct, vm_rb); - VM_BUG_ON_VMA(vma != ignore && - vma->rb_subtree_gap != vma_compute_subtree_gap(vma), - vma); } } @@ -352,10 +327,13 @@ static void validate_mm(struct mm_struct *mm) { int bug = 0; int i = 0; - unsigned long highest_address = 0; - struct vm_area_struct *vma = mm->mmap; + struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, 0, 0); - while (vma) { + validate_mm_mt(mm); + + mas_for_each(&mas, vma, ULONG_MAX) { +#ifdef CONFIG_DEBUG_VM_RB struct anon_vma *anon_vma = vma->anon_vma; struct anon_vma_chain *avc; @@ -365,93 +343,20 @@ static void validate_mm(struct mm_struct *mm) anon_vma_interval_tree_verify(avc); anon_vma_unlock_read(anon_vma); } - - highest_address = vm_end_gap(vma); - vma = vma->vm_next; +#endif i++; } if (i != mm->map_count) { - pr_emerg("map_count %d vm_next %d\n", mm->map_count, i); - bug = 1; - } - if (highest_address != mm->highest_vm_end) { - pr_emerg("mm->highest_vm_end %lx, found %lx\n", - mm->highest_vm_end, highest_address); - bug = 1; - } - i = browse_rb(mm); - if (i != mm->map_count) { - if (i != -1) - pr_emerg("map_count %d rb %d\n", mm->map_count, i); + pr_emerg("map_count %d mas_for_each %d\n", mm->map_count, i); bug = 1; } VM_BUG_ON_MM(bug, mm); } -#else -#define validate_mm_rb(root, ignore) do { } while (0) -#define validate_mm(mm) do { } while (0) -#endif - -RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks, - struct vm_area_struct, vm_rb, - unsigned long, rb_subtree_gap, vma_compute_gap) - -/* - * Update augmented rbtree rb_subtree_gap values after vma->vm_start or - * vma->vm_prev->vm_end values changed, without modifying the vma's position - * in the rbtree. - */ -static void vma_gap_update(struct vm_area_struct *vma) -{ - /* - * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created - * a callback function that does exactly what we want. - */ - vma_gap_callbacks_propagate(&vma->vm_rb, NULL); -} - -static inline void vma_rb_insert(struct vm_area_struct *vma, - struct rb_root *root) -{ - /* All rb_subtree_gap values must be consistent prior to insertion */ - validate_mm_rb(root, NULL); - - rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks); -} - -static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root) -{ - /* - * Note rb_erase_augmented is a fairly large inline function, - * so make sure we instantiate it only once with our desired - * augmented rbtree callbacks. - */ - rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks); -} - -static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma, - struct rb_root *root, - struct vm_area_struct *ignore) -{ - /* - * All rb_subtree_gap values must be consistent prior to erase, - * with the possible exception of - * - * a. the "next" vma being erased if next->vm_start was reduced in - * __vma_adjust() -> __vma_unlink() - * b. the vma being erased in detach_vmas_to_be_unmapped() -> - * vma_rb_erase() - */ - validate_mm_rb(root, ignore); - - __vma_rb_erase(vma, root); -} -static __always_inline void vma_rb_erase(struct vm_area_struct *vma, - struct rb_root *root) -{ - vma_rb_erase_ignore(vma, root, vma); -} +#else /* !CONFIG_DEBUG_VM_MAPLE_TREE */ +#define validate_mm_mt(root) do { } while (0) +#define validate_mm(mm) do { } while (0) +#endif /* CONFIG_DEBUG_VM_MAPLE_TREE */ /* * vma has some anon_vma assigned, and is already inserted on that @@ -485,208 +390,220 @@ anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma) anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); } -static int find_vma_links(struct mm_struct *mm, unsigned long addr, - unsigned long end, struct vm_area_struct **pprev, - struct rb_node ***rb_link, struct rb_node **rb_parent) +static unsigned long count_vma_pages_range(struct mm_struct *mm, + unsigned long addr, unsigned long end) { - struct rb_node **__rb_link, *__rb_parent, *rb_prev; + VMA_ITERATOR(vmi, mm, addr); + struct vm_area_struct *vma; + unsigned long nr_pages = 0; - mmap_assert_locked(mm); - __rb_link = &mm->mm_rb.rb_node; - rb_prev = __rb_parent = NULL; + for_each_vma_range(vmi, vma, end) { + unsigned long vm_start = max(addr, vma->vm_start); + unsigned long vm_end = min(end, vma->vm_end); - while (*__rb_link) { - struct vm_area_struct *vma_tmp; + nr_pages += PHYS_PFN(vm_end - vm_start); + } - __rb_parent = *__rb_link; - vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb); + return nr_pages; +} - if (vma_tmp->vm_end > addr) { - /* Fail if an existing vma overlaps the area */ - if (vma_tmp->vm_start < end) - return -ENOMEM; - __rb_link = &__rb_parent->rb_left; - } else { - rb_prev = __rb_parent; - __rb_link = &__rb_parent->rb_right; - } - } +static void __vma_link_file(struct vm_area_struct *vma, + struct address_space *mapping) +{ + if (vma->vm_flags & VM_SHARED) + mapping_allow_writable(mapping); - *pprev = NULL; - if (rb_prev) - *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); - *rb_link = __rb_link; - *rb_parent = __rb_parent; - return 0; + flush_dcache_mmap_lock(mapping); + vma_interval_tree_insert(vma, &mapping->i_mmap); + flush_dcache_mmap_unlock(mapping); } /* - * vma_next() - Get the next VMA. - * @mm: The mm_struct. - * @vma: The current vma. + * vma_mas_store() - Store a VMA in the maple tree. + * @vma: The vm_area_struct + * @mas: The maple state * - * If @vma is NULL, return the first vma in the mm. + * Efficient way to store a VMA in the maple tree when the @mas has already + * walked to the correct location. * - * Returns: The next VMA after @vma. + * Note: the end address is inclusive in the maple tree. */ -static inline struct vm_area_struct *vma_next(struct mm_struct *mm, - struct vm_area_struct *vma) +void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) { - if (!vma) - return mm->mmap; - - return vma->vm_next; + trace_vma_store(mas->tree, vma); + mas_set_range(mas, vma->vm_start, vma->vm_end - 1); + mas_store_prealloc(mas, vma); } /* - * munmap_vma_range() - munmap VMAs that overlap a range. - * @mm: The mm struct - * @start: The start of the range. - * @len: The length of the range. - * @pprev: pointer to the pointer that will be set to previous vm_area_struct - * @rb_link: the rb_node - * @rb_parent: the parent rb_node - * - * Find all the vm_area_struct that overlap from @start to - * @end and munmap them. Set @pprev to the previous vm_area_struct. + * vma_mas_remove() - Remove a VMA from the maple tree. + * @vma: The vm_area_struct + * @mas: The maple state * - * Returns: -ENOMEM on munmap failure or 0 on success. + * Efficient way to remove a VMA from the maple tree when the @mas has already + * been established and points to the correct location. + * Note: the end address is inclusive in the maple tree. */ -static inline int -munmap_vma_range(struct mm_struct *mm, unsigned long start, unsigned long len, - struct vm_area_struct **pprev, struct rb_node ***link, - struct rb_node **parent, struct list_head *uf) +void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) { - - while (find_vma_links(mm, start, start + len, pprev, link, parent)) - if (do_munmap(mm, start, len, uf)) - return -ENOMEM; - - return 0; + trace_vma_mas_szero(mas->tree, vma->vm_start, vma->vm_end - 1); + mas->index = vma->vm_start; + mas->last = vma->vm_end - 1; + mas_store_prealloc(mas, NULL); } -static unsigned long count_vma_pages_range(struct mm_struct *mm, - unsigned long addr, unsigned long end) + +/* + * vma_mas_szero() - Set a given range to zero. Used when modifying a + * vm_area_struct start or end. + * + * @mm: The struct_mm + * @start: The start address to zero + * @end: The end address to zero. + */ +static inline void vma_mas_szero(struct ma_state *mas, unsigned long start, + unsigned long end) { - unsigned long nr_pages = 0; - struct vm_area_struct *vma; + trace_vma_mas_szero(mas->tree, start, end - 1); + mas_set_range(mas, start, end - 1); + mas_store_prealloc(mas, NULL); +} - /* Find first overlapping mapping */ - vma = find_vma_intersection(mm, addr, end); - if (!vma) - return 0; +static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) +{ + MA_STATE(mas, &mm->mm_mt, 0, 0); + struct address_space *mapping = NULL; - nr_pages = (min(end, vma->vm_end) - - max(addr, vma->vm_start)) >> PAGE_SHIFT; + if (mas_preallocate(&mas, vma, GFP_KERNEL)) + return -ENOMEM; - /* Iterate over the rest of the overlaps */ - for (vma = vma->vm_next; vma; vma = vma->vm_next) { - unsigned long overlap_len; + if (vma->vm_file) { + mapping = vma->vm_file->f_mapping; + i_mmap_lock_write(mapping); + } - if (vma->vm_start > end) - break; + vma_mas_store(vma, &mas); - overlap_len = min(end, vma->vm_end) - vma->vm_start; - nr_pages += overlap_len >> PAGE_SHIFT; + if (mapping) { + __vma_link_file(vma, mapping); + i_mmap_unlock_write(mapping); } - return nr_pages; + mm->map_count++; + validate_mm(mm); + return 0; } -void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma, - struct rb_node **rb_link, struct rb_node *rb_parent) +/* + * vma_expand - Expand an existing VMA + * + * @mas: The maple state + * @vma: The vma to expand + * @start: The start of the vma + * @end: The exclusive end of the vma + * @pgoff: The page offset of vma + * @next: The current of next vma. + * + * Expand @vma to @start and @end. Can expand off the start and end. Will + * expand over @next if it's different from @vma and @end == @next->vm_end. + * Checking if the @vma can expand and merge with @next needs to be handled by + * the caller. + * + * Returns: 0 on success + */ +inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, + unsigned long start, unsigned long end, pgoff_t pgoff, + struct vm_area_struct *next) { - /* Update tracking information for the gap following the new vma. */ - if (vma->vm_next) - vma_gap_update(vma->vm_next); - else - mm->highest_vm_end = vm_end_gap(vma); + struct mm_struct *mm = vma->vm_mm; + struct address_space *mapping = NULL; + struct rb_root_cached *root = NULL; + struct anon_vma *anon_vma = vma->anon_vma; + struct file *file = vma->vm_file; + bool remove_next = false; - /* - * vma->vm_prev wasn't known when we followed the rbtree to find the - * correct insertion point for that vma. As a result, we could not - * update the vma vm_rb parents rb_subtree_gap values on the way down. - * So, we first insert the vma with a zero rb_subtree_gap value - * (to be consistent with what we did on the way down), and then - * immediately update the gap to the correct value. Finally we - * rebalance the rbtree after all augmented values have been set. - */ - rb_link_node(&vma->vm_rb, rb_parent, rb_link); - vma->rb_subtree_gap = 0; - vma_gap_update(vma); - vma_rb_insert(vma, &mm->mm_rb); -} + if (next && (vma != next) && (end == next->vm_end)) { + remove_next = true; + if (next->anon_vma && !vma->anon_vma) { + int error; -static void __vma_link_file(struct vm_area_struct *vma) -{ - struct file *file; + anon_vma = next->anon_vma; + vma->anon_vma = anon_vma; + error = anon_vma_clone(vma, next); + if (error) + return error; + } + } + + /* Not merging but overwriting any part of next is not handled. */ + VM_BUG_ON(next && !remove_next && next != vma && end > next->vm_start); + /* Only handles expanding */ + VM_BUG_ON(vma->vm_start < start || vma->vm_end > end); + + if (mas_preallocate(mas, vma, GFP_KERNEL)) + goto nomem; + + vma_adjust_trans_huge(vma, start, end, 0); - file = vma->vm_file; if (file) { - struct address_space *mapping = file->f_mapping; + mapping = file->f_mapping; + root = &mapping->i_mmap; + uprobe_munmap(vma, vma->vm_start, vma->vm_end); + i_mmap_lock_write(mapping); + } - if (vma->vm_flags & VM_SHARED) - mapping_allow_writable(mapping); + if (anon_vma) { + anon_vma_lock_write(anon_vma); + anon_vma_interval_tree_pre_update_vma(vma); + } + if (file) { flush_dcache_mmap_lock(mapping); - vma_interval_tree_insert(vma, &mapping->i_mmap); - flush_dcache_mmap_unlock(mapping); + vma_interval_tree_remove(vma, root); } -} -static void -__vma_link(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *prev, struct rb_node **rb_link, - struct rb_node *rb_parent) -{ - __vma_link_list(mm, vma, prev); - __vma_link_rb(mm, vma, rb_link, rb_parent); -} + vma->vm_start = start; + vma->vm_end = end; + vma->vm_pgoff = pgoff; + /* Note: mas must be pointing to the expanding VMA */ + vma_mas_store(vma, mas); -static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *prev, struct rb_node **rb_link, - struct rb_node *rb_parent) -{ - struct address_space *mapping = NULL; + if (file) { + vma_interval_tree_insert(vma, root); + flush_dcache_mmap_unlock(mapping); + } - if (vma->vm_file) { - mapping = vma->vm_file->f_mapping; - i_mmap_lock_write(mapping); + /* Expanding over the next vma */ + if (remove_next && file) { + __remove_shared_vm_struct(next, file, mapping); } - __vma_link(mm, vma, prev, rb_link, rb_parent); - __vma_link_file(vma); + if (anon_vma) { + anon_vma_interval_tree_post_update_vma(vma); + anon_vma_unlock_write(anon_vma); + } - if (mapping) + if (file) { i_mmap_unlock_write(mapping); + uprobe_mmap(vma); + } - mm->map_count++; - validate_mm(mm); -} - -/* - * Helper for vma_adjust() in the split_vma insert case: insert a vma into the - * mm's list and rbtree. It has already been inserted into the interval tree. - */ -static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) -{ - struct vm_area_struct *prev; - struct rb_node **rb_link, *rb_parent; + if (remove_next) { + if (file) { + uprobe_munmap(next, next->vm_start, next->vm_end); + fput(file); + } + if (next->anon_vma) + anon_vma_merge(vma, next); + mm->map_count--; + mpol_put(vma_policy(next)); + vm_area_free(next); + } - if (find_vma_links(mm, vma->vm_start, vma->vm_end, - &prev, &rb_link, &rb_parent)) - BUG(); - __vma_link(mm, vma, prev, rb_link, rb_parent); - mm->map_count++; -} + validate_mm(mm); + return 0; -static __always_inline void __vma_unlink(struct mm_struct *mm, - struct vm_area_struct *vma, - struct vm_area_struct *ignore) -{ - vma_rb_erase_ignore(vma, &mm->mm_rb, ignore); - __vma_unlink_list(mm, vma); - /* Kill the cache */ - vmacache_invalidate(mm); +nomem: + return -ENOMEM; } /* @@ -701,18 +618,19 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, struct vm_area_struct *expand) { struct mm_struct *mm = vma->vm_mm; - struct vm_area_struct *next = vma->vm_next, *orig_vma = vma; + struct vm_area_struct *next_next, *next = find_vma(mm, vma->vm_end); + struct vm_area_struct *orig_vma = vma; struct address_space *mapping = NULL; struct rb_root_cached *root = NULL; struct anon_vma *anon_vma = NULL; struct file *file = vma->vm_file; - bool start_changed = false, end_changed = false; + bool vma_changed = false; long adjust_next = 0; int remove_next = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); + struct vm_area_struct *exporter = NULL, *importer = NULL; if (next && !insert) { - struct vm_area_struct *exporter = NULL, *importer = NULL; - if (end >= next->vm_end) { /* * vma expands, overlapping all the next, and @@ -741,10 +659,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, * remove_next == 1 is case 1 or 7. */ remove_next = 1 + (end > next->vm_end); + if (remove_next == 2) + next_next = find_vma(mm, next->vm_end); + VM_WARN_ON(remove_next == 2 && - end != next->vm_next->vm_end); - /* trim end to next, for case 6 first pass */ - end = next->vm_end; + end != next_next->vm_end); } exporter = next; @@ -755,7 +674,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, * next, if the vma overlaps with it. */ if (remove_next == 2 && !next->anon_vma) - exporter = next->vm_next; + exporter = next_next; } else if (end > next->vm_start) { /* @@ -792,9 +711,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, return error; } } -again: - vma_adjust_trans_huge(orig_vma, start, end, adjust_next); + if (mas_preallocate(&mas, vma, GFP_KERNEL)) + return -ENOMEM; + + vma_adjust_trans_huge(orig_vma, start, end, adjust_next); if (file) { mapping = file->f_mapping; root = &mapping->i_mmap; @@ -804,14 +725,14 @@ again: uprobe_munmap(next, next->vm_start, next->vm_end); i_mmap_lock_write(mapping); - if (insert) { + if (insert && insert->vm_file) { /* * Put into interval tree now, so instantiated pages * are visible to arm/parisc __flush_dcache_page * throughout; but we cannot insert into address * space until vma start or end is updated. */ - __vma_link_file(insert); + __vma_link_file(insert, insert->vm_file->f_mapping); } } @@ -835,17 +756,37 @@ again: } if (start != vma->vm_start) { + if ((vma->vm_start < start) && + (!insert || (insert->vm_end != start))) { + vma_mas_szero(&mas, vma->vm_start, start); + VM_WARN_ON(insert && insert->vm_start > vma->vm_start); + } else { + vma_changed = true; + } vma->vm_start = start; - start_changed = true; } if (end != vma->vm_end) { + if (vma->vm_end > end) { + if (!insert || (insert->vm_start != end)) { + vma_mas_szero(&mas, end, vma->vm_end); + mas_reset(&mas); + VM_WARN_ON(insert && + insert->vm_end < vma->vm_end); + } + } else { + vma_changed = true; + } vma->vm_end = end; - end_changed = true; } + + if (vma_changed) + vma_mas_store(vma, &mas); + vma->vm_pgoff = pgoff; if (adjust_next) { next->vm_start += adjust_next; next->vm_pgoff += adjust_next >> PAGE_SHIFT; + vma_mas_store(next, &mas); } if (file) { @@ -855,42 +796,19 @@ again: flush_dcache_mmap_unlock(mapping); } - if (remove_next) { - /* - * vma_merge has merged next into vma, and needs - * us to remove next before dropping the locks. - */ - if (remove_next != 3) - __vma_unlink(mm, next, next); - else - /* - * vma is not before next if they've been - * swapped. - * - * pre-swap() next->vm_start was reduced so - * tell validate_mm_rb to ignore pre-swap() - * "next" (which is stored in post-swap() - * "vma"). - */ - __vma_unlink(mm, next, vma); - if (file) - __remove_shared_vm_struct(next, file, mapping); + if (remove_next && file) { + __remove_shared_vm_struct(next, file, mapping); + if (remove_next == 2) + __remove_shared_vm_struct(next_next, file, mapping); } else if (insert) { /* * split_vma has split insert from vma, and needs * us to insert it before dropping the locks * (it may either follow vma or precede it). */ - __insert_vm_struct(mm, insert); - } else { - if (start_changed) - vma_gap_update(vma); - if (end_changed) { - if (!next) - mm->highest_vm_end = vm_end_gap(vma); - else if (!adjust_next) - vma_gap_update(next); - } + mas_reset(&mas); + vma_mas_store(insert, &mas); + mm->map_count++; } if (anon_vma) { @@ -909,6 +827,7 @@ again: } if (remove_next) { +again: if (file) { uprobe_munmap(next, next->vm_start, next->vm_end); fput(file); @@ -917,66 +836,24 @@ again: anon_vma_merge(vma, next); mm->map_count--; mpol_put(vma_policy(next)); + if (remove_next != 2) + BUG_ON(vma->vm_end < next->vm_end); vm_area_free(next); + /* * In mprotect's case 6 (see comments on vma_merge), - * we must remove another next too. It would clutter - * up the code too much to do both in one go. + * we must remove next_next too. */ - if (remove_next != 3) { - /* - * If "next" was removed and vma->vm_end was - * expanded (up) over it, in turn - * "next->vm_prev->vm_end" changed and the - * "vma->vm_next" gap must be updated. - */ - next = vma->vm_next; - } else { - /* - * For the scope of the comment "next" and - * "vma" considered pre-swap(): if "vma" was - * removed, next->vm_start was expanded (down) - * over it and the "next" gap must be updated. - * Because of the swap() the post-swap() "vma" - * actually points to pre-swap() "next" - * (post-swap() "next" as opposed is now a - * dangling pointer). - */ - next = vma; - } if (remove_next == 2) { remove_next = 1; - end = next->vm_end; + next = next_next; goto again; } - else if (next) - vma_gap_update(next); - else { - /* - * If remove_next == 2 we obviously can't - * reach this path. - * - * If remove_next == 3 we can't reach this - * path because pre-swap() next is always not - * NULL. pre-swap() "next" is not being - * removed and its next->vm_end is not altered - * (and furthermore "end" already matches - * next->vm_end in remove_next == 3). - * - * We reach this only in the remove_next == 1 - * case if the "next" vma that was removed was - * the highest vma of the mm. However in such - * case next->vm_end == "end" and the extended - * "vma" has vma->vm_end == next->vm_end so - * mm->highest_vm_end doesn't need any update - * in remove_next == 1 case. - */ - VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma)); - } } if (insert && file) uprobe_mmap(insert); + mas_destroy(&mas); validate_mm(mm); return 0; @@ -1128,8 +1005,10 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, struct anon_vma_name *anon_name) { pgoff_t pglen = (end - addr) >> PAGE_SHIFT; - struct vm_area_struct *area, *next; - int err; + struct vm_area_struct *mid, *next, *res; + int err = -1; + bool merge_prev = false; + bool merge_next = false; /* * We later require that vma->vm_flags == vm_flags, @@ -1138,76 +1017,61 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, if (vm_flags & VM_SPECIAL) return NULL; - next = vma_next(mm, prev); - area = next; - if (area && area->vm_end == end) /* cases 6, 7, 8 */ - next = next->vm_next; + next = find_vma(mm, prev ? prev->vm_end : 0); + mid = next; + if (next && next->vm_end == end) /* cases 6, 7, 8 */ + next = find_vma(mm, next->vm_end); /* verify some invariant that must be enforced by the caller */ VM_WARN_ON(prev && addr <= prev->vm_start); - VM_WARN_ON(area && end > area->vm_end); + VM_WARN_ON(mid && end > mid->vm_end); VM_WARN_ON(addr >= end); - /* - * Can it merge with the predecessor? - */ + /* Can we merge the predecessor? */ if (prev && prev->vm_end == addr && mpol_equal(vma_policy(prev), policy) && can_vma_merge_after(prev, vm_flags, anon_vma, file, pgoff, vm_userfaultfd_ctx, anon_name)) { - /* - * OK, it can. Can we now merge in the successor as well? - */ - if (next && end == next->vm_start && - mpol_equal(policy, vma_policy(next)) && - can_vma_merge_before(next, vm_flags, - anon_vma, file, - pgoff+pglen, - vm_userfaultfd_ctx, anon_name) && - is_mergeable_anon_vma(prev->anon_vma, - next->anon_vma, NULL)) { - /* cases 1, 6 */ - err = __vma_adjust(prev, prev->vm_start, - next->vm_end, prev->vm_pgoff, NULL, - prev); - } else /* cases 2, 5, 7 */ - err = __vma_adjust(prev, prev->vm_start, - end, prev->vm_pgoff, NULL, prev); - if (err) - return NULL; - khugepaged_enter_vma(prev, vm_flags); - return prev; + merge_prev = true; } - - /* - * Can this new request be merged in front of next? - */ + /* Can we merge the successor? */ if (next && end == next->vm_start && mpol_equal(policy, vma_policy(next)) && can_vma_merge_before(next, vm_flags, anon_vma, file, pgoff+pglen, vm_userfaultfd_ctx, anon_name)) { + merge_next = true; + } + /* Can we merge both the predecessor and the successor? */ + if (merge_prev && merge_next && + is_mergeable_anon_vma(prev->anon_vma, + next->anon_vma, NULL)) { /* cases 1, 6 */ + err = __vma_adjust(prev, prev->vm_start, + next->vm_end, prev->vm_pgoff, NULL, + prev); + res = prev; + } else if (merge_prev) { /* cases 2, 5, 7 */ + err = __vma_adjust(prev, prev->vm_start, + end, prev->vm_pgoff, NULL, prev); + res = prev; + } else if (merge_next) { if (prev && addr < prev->vm_end) /* case 4 */ err = __vma_adjust(prev, prev->vm_start, - addr, prev->vm_pgoff, NULL, next); - else { /* cases 3, 8 */ - err = __vma_adjust(area, addr, next->vm_end, - next->vm_pgoff - pglen, NULL, next); - /* - * In case 3 area is already equal to next and - * this is a noop, but in case 8 "area" has - * been removed and next was expanded over it. - */ - area = next; - } - if (err) - return NULL; - khugepaged_enter_vma(area, vm_flags); - return area; + addr, prev->vm_pgoff, NULL, next); + else /* cases 3, 8 */ + err = __vma_adjust(mid, addr, next->vm_end, + next->vm_pgoff - pglen, NULL, next); + res = next; } - return NULL; + /* + * Cannot merge with predecessor or successor or error in __vma_adjust? + */ + if (err) + return NULL; + khugepaged_enter_vma(res, vm_flags); + return res; } /* @@ -1275,18 +1139,24 @@ static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_ */ struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma) { + MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_end, vma->vm_end); struct anon_vma *anon_vma = NULL; + struct vm_area_struct *prev, *next; /* Try next first. */ - if (vma->vm_next) { - anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next); + next = mas_walk(&mas); + if (next) { + anon_vma = reusable_anon_vma(next, vma, next); if (anon_vma) return anon_vma; } + prev = mas_prev(&mas, 0); + VM_BUG_ON_VMA(prev != vma, vma); + prev = mas_prev(&mas, 0); /* Try prev next. */ - if (vma->vm_prev) - anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma); + if (prev) + anon_vma = reusable_anon_vma(prev, prev, vma); /* * We might reach here with anon_vma == NULL if we can't find @@ -1375,6 +1245,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags_t vm_flags; int pkey = 0; + validate_mm(mm); *populate = 0; if (!len) @@ -1646,8 +1517,11 @@ int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot) pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags))) return 0; - /* Do we need to track softdirty? */ - if (vma_soft_dirty_enabled(vma)) + /* + * Do we need to track softdirty? hugetlb does not support softdirty + * tracking yet. + */ + if (vma_soft_dirty_enabled(vma) && !is_vm_hugetlb_page(vma)) return 1; /* Specialty mapping? */ @@ -1675,388 +1549,63 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; } -unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf) -{ - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma, *prev, *merge; - int error; - struct rb_node **rb_link, *rb_parent; - unsigned long charged = 0; - - /* Check against address space limit. */ - if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { - unsigned long nr_pages; - - /* - * MAP_FIXED may remove pages of mappings that intersects with - * requested mapping. Account for the pages it would unmap. - */ - nr_pages = count_vma_pages_range(mm, addr, addr + len); - - if (!may_expand_vm(mm, vm_flags, - (len >> PAGE_SHIFT) - nr_pages)) - return -ENOMEM; - } - - /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ - if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) - return -ENOMEM; - /* - * Private writable mapping: check memory availability - */ - if (accountable_mapping(file, vm_flags)) { - charged = len >> PAGE_SHIFT; - if (security_vm_enough_memory_mm(mm, charged)) - return -ENOMEM; - vm_flags |= VM_ACCOUNT; - } - - /* - * Can we just expand an old mapping? - */ - vma = vma_merge(mm, prev, addr, addr + len, vm_flags, - NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); - if (vma) - goto out; - - /* - * Determine the object being mapped and call the appropriate - * specific mapper. the address has already been validated, but - * not unmapped, but the maps are removed from the list. - */ - vma = vm_area_alloc(mm); - if (!vma) { - error = -ENOMEM; - goto unacct_error; - } - - vma->vm_start = addr; - vma->vm_end = addr + len; - vma->vm_flags = vm_flags; - vma->vm_page_prot = vm_get_page_prot(vm_flags); - vma->vm_pgoff = pgoff; - - if (file) { - if (vm_flags & VM_SHARED) { - error = mapping_map_writable(file->f_mapping); - if (error) - goto free_vma; - } - - vma->vm_file = get_file(file); - error = call_mmap(file, vma); - if (error) - goto unmap_and_free_vma; - - /* Can addr have changed?? - * - * Answer: Yes, several device drivers can do it in their - * f_op->mmap method. -DaveM - * Bug: If addr is changed, prev, rb_link, rb_parent should - * be updated for vma_link() - */ - WARN_ON_ONCE(addr != vma->vm_start); - - addr = vma->vm_start; - - /* If vm_flags changed after call_mmap(), we should try merge vma again - * as we may succeed this time. - */ - if (unlikely(vm_flags != vma->vm_flags && prev)) { - merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, - NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); - if (merge) { - /* ->mmap() can change vma->vm_file and fput the original file. So - * fput the vma->vm_file here or we would add an extra fput for file - * and cause general protection fault ultimately. - */ - fput(vma->vm_file); - vm_area_free(vma); - vma = merge; - /* Update vm_flags to pick up the change. */ - vm_flags = vma->vm_flags; - goto unmap_writable; - } - } - - vm_flags = vma->vm_flags; - } else if (vm_flags & VM_SHARED) { - error = shmem_zero_setup(vma); - if (error) - goto free_vma; - } else { - vma_set_anonymous(vma); - } - - /* Allow architectures to sanity-check the vm_flags */ - if (!arch_validate_flags(vma->vm_flags)) { - error = -EINVAL; - if (file) - goto unmap_and_free_vma; - else - goto free_vma; - } - - vma_link(mm, vma, prev, rb_link, rb_parent); - - /* - * vma_merge() calls khugepaged_enter_vma() either, the below - * call covers the non-merge case. - */ - khugepaged_enter_vma(vma, vma->vm_flags); - - /* Once vma denies write, undo our temporary denial count */ -unmap_writable: - if (file && vm_flags & VM_SHARED) - mapping_unmap_writable(file->f_mapping); - file = vma->vm_file; -out: - perf_event_mmap(vma); - - vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); - if (vm_flags & VM_LOCKED) { - if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || - is_vm_hugetlb_page(vma) || - vma == get_gate_vma(current->mm)) - vma->vm_flags &= VM_LOCKED_CLEAR_MASK; - else - mm->locked_vm += (len >> PAGE_SHIFT); - } - - if (file) - uprobe_mmap(vma); - - /* - * New (or expanded) vma always get soft dirty status. - * Otherwise user-space soft-dirty page tracker won't - * be able to distinguish situation when vma area unmapped, - * then new mapped in-place (which must be aimed as - * a completely new data area). - */ - vma->vm_flags |= VM_SOFTDIRTY; - - vma_set_page_prot(vma); - - return addr; - -unmap_and_free_vma: - fput(vma->vm_file); - vma->vm_file = NULL; - - /* Undo any partial mapping done by a device driver. */ - unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); - if (vm_flags & VM_SHARED) - mapping_unmap_writable(file->f_mapping); -free_vma: - vm_area_free(vma); -unacct_error: - if (charged) - vm_unacct_memory(charged); - return error; -} - +/** + * unmapped_area() - Find an area between the low_limit and the high_limit with + * the correct alignment and offset, all from @info. Note: current->mm is used + * for the search. + * + * @info: The unmapped area information including the range (low_limit - + * hight_limit), the alignment offset and mask. + * + * Return: A memory address or -ENOMEM. + */ static unsigned long unmapped_area(struct vm_unmapped_area_info *info) { - /* - * We implement the search by looking for an rbtree node that - * immediately follows a suitable gap. That is, - * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length; - * - gap_end = vma->vm_start >= info->low_limit + length; - * - gap_end - gap_start >= length - */ + unsigned long length, gap; - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - unsigned long length, low_limit, high_limit, gap_start, gap_end; + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); /* Adjust search length to account for worst case alignment overhead */ length = info->length + info->align_mask; if (length < info->length) return -ENOMEM; - /* Adjust search limits by the desired length */ - if (info->high_limit < length) - return -ENOMEM; - high_limit = info->high_limit - length; - - if (info->low_limit > high_limit) - return -ENOMEM; - low_limit = info->low_limit + length; - - /* Check if rbtree root looks promising */ - if (RB_EMPTY_ROOT(&mm->mm_rb)) - goto check_highest; - vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); - if (vma->rb_subtree_gap < length) - goto check_highest; - - while (true) { - /* Visit left subtree if it looks promising */ - gap_end = vm_start_gap(vma); - if (gap_end >= low_limit && vma->vm_rb.rb_left) { - struct vm_area_struct *left = - rb_entry(vma->vm_rb.rb_left, - struct vm_area_struct, vm_rb); - if (left->rb_subtree_gap >= length) { - vma = left; - continue; - } - } - - gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; -check_current: - /* Check if current node has a suitable gap */ - if (gap_start > high_limit) - return -ENOMEM; - if (gap_end >= low_limit && - gap_end > gap_start && gap_end - gap_start >= length) - goto found; - - /* Visit right subtree if it looks promising */ - if (vma->vm_rb.rb_right) { - struct vm_area_struct *right = - rb_entry(vma->vm_rb.rb_right, - struct vm_area_struct, vm_rb); - if (right->rb_subtree_gap >= length) { - vma = right; - continue; - } - } - - /* Go back up the rbtree to find next candidate node */ - while (true) { - struct rb_node *prev = &vma->vm_rb; - if (!rb_parent(prev)) - goto check_highest; - vma = rb_entry(rb_parent(prev), - struct vm_area_struct, vm_rb); - if (prev == vma->vm_rb.rb_left) { - gap_start = vm_end_gap(vma->vm_prev); - gap_end = vm_start_gap(vma); - goto check_current; - } - } - } - -check_highest: - /* Check highest gap, which does not precede any rbtree node */ - gap_start = mm->highest_vm_end; - gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */ - if (gap_start > high_limit) + if (mas_empty_area(&mas, info->low_limit, info->high_limit - 1, + length)) return -ENOMEM; -found: - /* We found a suitable gap. Clip it with the original low_limit. */ - if (gap_start < info->low_limit) - gap_start = info->low_limit; - - /* Adjust gap address to the desired alignment */ - gap_start += (info->align_offset - gap_start) & info->align_mask; - - VM_BUG_ON(gap_start + info->length > info->high_limit); - VM_BUG_ON(gap_start + info->length > gap_end); - return gap_start; + gap = mas.index; + gap += (info->align_offset - gap) & info->align_mask; + return gap; } +/** + * unmapped_area_topdown() - Find an area between the low_limit and the + * high_limit with * the correct alignment and offset at the highest available + * address, all from @info. Note: current->mm is used for the search. + * + * @info: The unmapped area information including the range (low_limit - + * hight_limit), the alignment offset and mask. + * + * Return: A memory address or -ENOMEM. + */ static unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info) { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma; - unsigned long length, low_limit, high_limit, gap_start, gap_end; + unsigned long length, gap; + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); /* Adjust search length to account for worst case alignment overhead */ length = info->length + info->align_mask; if (length < info->length) return -ENOMEM; - /* - * Adjust search limits by the desired length. - * See implementation comment at top of unmapped_area(). - */ - gap_end = info->high_limit; - if (gap_end < length) - return -ENOMEM; - high_limit = gap_end - length; - - if (info->low_limit > high_limit) - return -ENOMEM; - low_limit = info->low_limit + length; - - /* Check highest gap, which does not precede any rbtree node */ - gap_start = mm->highest_vm_end; - if (gap_start <= high_limit) - goto found_highest; - - /* Check if rbtree root looks promising */ - if (RB_EMPTY_ROOT(&mm->mm_rb)) - return -ENOMEM; - vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb); - if (vma->rb_subtree_gap < length) + if (mas_empty_area_rev(&mas, info->low_limit, info->high_limit - 1, + length)) return -ENOMEM; - while (true) { - /* Visit right subtree if it looks promising */ - gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0; - if (gap_start <= high_limit && vma->vm_rb.rb_right) { - struct vm_area_struct *right = - rb_entry(vma->vm_rb.rb_right, - struct vm_area_struct, vm_rb); - if (right->rb_subtree_gap >= length) { - vma = right; - continue; - } - } - -check_current: - /* Check if current node has a suitable gap */ - gap_end = vm_start_gap(vma); - if (gap_end < low_limit) - return -ENOMEM; - if (gap_start <= high_limit && - gap_end > gap_start && gap_end - gap_start >= length) - goto found; - - /* Visit left subtree if it looks promising */ - if (vma->vm_rb.rb_left) { - struct vm_area_struct *left = - rb_entry(vma->vm_rb.rb_left, - struct vm_area_struct, vm_rb); - if (left->rb_subtree_gap >= length) { - vma = left; - continue; - } - } - - /* Go back up the rbtree to find next candidate node */ - while (true) { - struct rb_node *prev = &vma->vm_rb; - if (!rb_parent(prev)) - return -ENOMEM; - vma = rb_entry(rb_parent(prev), - struct vm_area_struct, vm_rb); - if (prev == vma->vm_rb.rb_right) { - gap_start = vma->vm_prev ? - vm_end_gap(vma->vm_prev) : 0; - goto check_current; - } - } - } - -found: - /* We found a suitable gap. Clip it with the original high_limit. */ - if (gap_end > info->high_limit) - gap_end = info->high_limit; - -found_highest: - /* Compute highest gap address at the desired alignment */ - gap_end -= info->length; - gap_end -= (gap_end - info->align_offset) & info->align_mask; - - VM_BUG_ON(gap_end < info->low_limit); - VM_BUG_ON(gap_end < gap_start); - return gap_end; + gap = mas.last + 1 - info->length; + gap -= (gap - info->align_offset) & info->align_mask; + return gap; } /* @@ -2229,6 +1778,9 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, */ pgoff = 0; get_area = shmem_get_unmapped_area; + } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + /* Ensures that larger anonymous mappings are THP aligned. */ + get_area = thp_get_unmapped_area; } addr = get_area(file, addr, len, pgoff, flags); @@ -2246,58 +1798,67 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, EXPORT_SYMBOL(get_unmapped_area); -/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */ -struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +/** + * find_vma_intersection() - Look up the first VMA which intersects the interval + * @mm: The process address space. + * @start_addr: The inclusive start user address. + * @end_addr: The exclusive end user address. + * + * Returns: The first VMA within the provided range, %NULL otherwise. Assumes + * start_addr < end_addr. + */ +struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, + unsigned long start_addr, + unsigned long end_addr) { - struct rb_node *rb_node; - struct vm_area_struct *vma; + unsigned long index = start_addr; mmap_assert_locked(mm); - /* Check the cache first. */ - vma = vmacache_find(mm, addr); - if (likely(vma)) - return vma; - - rb_node = mm->mm_rb.rb_node; - - while (rb_node) { - struct vm_area_struct *tmp; - - tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb); + return mt_find(&mm->mm_mt, &index, end_addr - 1); +} +EXPORT_SYMBOL(find_vma_intersection); - if (tmp->vm_end > addr) { - vma = tmp; - if (tmp->vm_start <= addr) - break; - rb_node = rb_node->rb_left; - } else - rb_node = rb_node->rb_right; - } +/** + * find_vma() - Find the VMA for a given address, or the next VMA. + * @mm: The mm_struct to check + * @addr: The address + * + * Returns: The VMA associated with addr, or the next VMA. + * May return %NULL in the case of no VMA at addr or above. + */ +struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) +{ + unsigned long index = addr; - if (vma) - vmacache_update(addr, vma); - return vma; + mmap_assert_locked(mm); + return mt_find(&mm->mm_mt, &index, ULONG_MAX); } - EXPORT_SYMBOL(find_vma); -/* - * Same as find_vma, but also return a pointer to the previous VMA in *pprev. +/** + * find_vma_prev() - Find the VMA for a given address, or the next vma and + * set %pprev to the previous VMA, if any. + * @mm: The mm_struct to check + * @addr: The address + * @pprev: The pointer to set to the previous VMA + * + * Note that RCU lock is missing here since the external mmap_lock() is used + * instead. + * + * Returns: The VMA associated with @addr, or the next vma. + * May return %NULL in the case of no vma at addr or above. */ struct vm_area_struct * find_vma_prev(struct mm_struct *mm, unsigned long addr, struct vm_area_struct **pprev) { struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, addr, addr); - vma = find_vma(mm, addr); - if (vma) { - *pprev = vma->vm_prev; - } else { - struct rb_node *rb_node = rb_last(&mm->mm_rb); - - *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL; - } + vma = mas_walk(&mas); + *pprev = mas_prev(&mas, 0); + if (!vma) + vma = mas_next(&mas, ULONG_MAX); return vma; } @@ -2351,6 +1912,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) struct vm_area_struct *next; unsigned long gap_addr; int error = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); if (!(vma->vm_flags & VM_GROWSUP)) return -EFAULT; @@ -2368,17 +1930,22 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) if (gap_addr < address || gap_addr > TASK_SIZE) gap_addr = TASK_SIZE; - next = vma->vm_next; - if (next && next->vm_start < gap_addr && vma_is_accessible(next)) { + next = find_vma_intersection(mm, vma->vm_end, gap_addr); + if (next && vma_is_accessible(next)) { if (!(next->vm_flags & VM_GROWSUP)) return -ENOMEM; /* Check that both stack segments have the same anon_vma? */ } - /* We must make sure the anon_vma is allocated. */ - if (unlikely(anon_vma_prepare(vma))) + if (mas_preallocate(&mas, vma, GFP_KERNEL)) return -ENOMEM; + /* We must make sure the anon_vma is allocated. */ + if (unlikely(anon_vma_prepare(vma))) { + mas_destroy(&mas); + return -ENOMEM; + } + /* * vma->vm_start/vm_end cannot change under us because the caller * is required to hold the mmap_lock in read mode. We need the @@ -2398,15 +1965,13 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) error = acct_stack_growth(vma, size, grow); if (!error) { /* - * vma_gap_update() doesn't support concurrent - * updates, but we only hold a shared mmap_lock - * lock here, so we need to protect against - * concurrent vma expansions. - * anon_vma_lock_write() doesn't help here, as - * we don't guarantee that all growable vmas - * in a mm share the same root anon vma. - * So, we reuse mm->page_table_lock to guard - * against concurrent vma expansions. + * We only hold a shared mmap_lock lock here, so + * we need to protect against concurrent vma + * expansions. anon_vma_lock_write() doesn't + * help here, as we don't guarantee that all + * growable vmas in a mm share the same root + * anon vma. So, we reuse mm->page_table_lock + * to guard against concurrent vma expansions. */ spin_lock(&mm->page_table_lock); if (vma->vm_flags & VM_LOCKED) @@ -2414,11 +1979,9 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; + /* Overwrite old entry in mtree. */ + vma_mas_store(vma, &mas); anon_vma_interval_tree_post_update_vma(vma); - if (vma->vm_next) - vma_gap_update(vma->vm_next); - else - mm->highest_vm_end = vm_end_gap(vma); spin_unlock(&mm->page_table_lock); perf_event_mmap(vma); @@ -2427,7 +1990,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) } anon_vma_unlock_write(vma->anon_vma); khugepaged_enter_vma(vma, vma->vm_flags); - validate_mm(mm); + mas_destroy(&mas); return error; } #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ @@ -2435,10 +1998,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) /* * vma is the first one with address < vma->vm_start. Have to extend vma. */ -int expand_downwards(struct vm_area_struct *vma, - unsigned long address) +int expand_downwards(struct vm_area_struct *vma, unsigned long address) { struct mm_struct *mm = vma->vm_mm; + MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start); struct vm_area_struct *prev; int error = 0; @@ -2447,7 +2010,7 @@ int expand_downwards(struct vm_area_struct *vma, return -EPERM; /* Enforce stack_guard_gap */ - prev = vma->vm_prev; + prev = mas_prev(&mas, 0); /* Check that both stack segments have the same anon_vma? */ if (prev && !(prev->vm_flags & VM_GROWSDOWN) && vma_is_accessible(prev)) { @@ -2455,9 +2018,14 @@ int expand_downwards(struct vm_area_struct *vma, return -ENOMEM; } + if (mas_preallocate(&mas, vma, GFP_KERNEL)) + return -ENOMEM; + /* We must make sure the anon_vma is allocated. */ - if (unlikely(anon_vma_prepare(vma))) + if (unlikely(anon_vma_prepare(vma))) { + mas_destroy(&mas); return -ENOMEM; + } /* * vma->vm_start/vm_end cannot change under us because the caller @@ -2478,15 +2046,13 @@ int expand_downwards(struct vm_area_struct *vma, error = acct_stack_growth(vma, size, grow); if (!error) { /* - * vma_gap_update() doesn't support concurrent - * updates, but we only hold a shared mmap_lock - * lock here, so we need to protect against - * concurrent vma expansions. - * anon_vma_lock_write() doesn't help here, as - * we don't guarantee that all growable vmas - * in a mm share the same root anon vma. - * So, we reuse mm->page_table_lock to guard - * against concurrent vma expansions. + * We only hold a shared mmap_lock lock here, so + * we need to protect against concurrent vma + * expansions. anon_vma_lock_write() doesn't + * help here, as we don't guarantee that all + * growable vmas in a mm share the same root + * anon vma. So, we reuse mm->page_table_lock + * to guard against concurrent vma expansions. */ spin_lock(&mm->page_table_lock); if (vma->vm_flags & VM_LOCKED) @@ -2495,8 +2061,9 @@ int expand_downwards(struct vm_area_struct *vma, anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; vma->vm_pgoff -= grow; + /* Overwrite old entry in mtree. */ + vma_mas_store(vma, &mas); anon_vma_interval_tree_post_update_vma(vma); - vma_gap_update(vma); spin_unlock(&mm->page_table_lock); perf_event_mmap(vma); @@ -2505,7 +2072,7 @@ int expand_downwards(struct vm_area_struct *vma, } anon_vma_unlock_write(vma->anon_vma); khugepaged_enter_vma(vma, vma->vm_flags); - validate_mm(mm); + mas_destroy(&mas); return error; } @@ -2578,25 +2145,26 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr) EXPORT_SYMBOL_GPL(find_extend_vma); /* - * Ok - we have the memory areas we should free on the vma list, - * so release them, and do the vma updates. + * Ok - we have the memory areas we should free on a maple tree so release them, + * and do the vma updates. * * Called with the mm semaphore held. */ -static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) +static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) { unsigned long nr_accounted = 0; + struct vm_area_struct *vma; /* Update high watermark before we lower total_vm */ update_hiwater_vm(mm); - do { + mas_for_each(mas, vma, ULONG_MAX) { long nrpages = vma_pages(vma); if (vma->vm_flags & VM_ACCOUNT) nr_accounted += nrpages; vm_stat_account(mm, vma->vm_flags, -nrpages); - vma = remove_vma(vma); - } while (vma); + remove_vma(vma); + } vm_unacct_memory(nr_accounted); validate_mm(mm); } @@ -2606,66 +2174,22 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma) * * Called with the mm semaphore held. */ -static void unmap_region(struct mm_struct *mm, +static void unmap_region(struct mm_struct *mm, struct maple_tree *mt, struct vm_area_struct *vma, struct vm_area_struct *prev, + struct vm_area_struct *next, unsigned long start, unsigned long end) { - struct vm_area_struct *next = vma_next(mm, prev); struct mmu_gather tlb; lru_add_drain(); tlb_gather_mmu(&tlb, mm); update_hiwater_rss(mm); - unmap_vmas(&tlb, vma, start, end); - free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, + unmap_vmas(&tlb, mt, vma, start, end); + free_pgtables(&tlb, mt, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS, next ? next->vm_start : USER_PGTABLES_CEILING); tlb_finish_mmu(&tlb); } -/* - * Create a list of vma's touched by the unmap, removing them from the mm's - * vma list as we go.. - */ -static bool -detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *prev, unsigned long end) -{ - struct vm_area_struct **insertion_point; - struct vm_area_struct *tail_vma = NULL; - - insertion_point = (prev ? &prev->vm_next : &mm->mmap); - vma->vm_prev = NULL; - do { - vma_rb_erase(vma, &mm->mm_rb); - if (vma->vm_flags & VM_LOCKED) - mm->locked_vm -= vma_pages(vma); - mm->map_count--; - tail_vma = vma; - vma = vma->vm_next; - } while (vma && vma->vm_start < end); - *insertion_point = vma; - if (vma) { - vma->vm_prev = prev; - vma_gap_update(vma); - } else - mm->highest_vm_end = prev ? vm_end_gap(prev) : 0; - tail_vma->vm_next = NULL; - - /* Kill the cache */ - vmacache_invalidate(mm); - - /* - * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or - * VM_GROWSUP VMA. Such VMAs can change their size under - * down_read(mmap_lock) and collide with the VMA we are about to unmap. - */ - if (vma && (vma->vm_flags & VM_GROWSDOWN)) - return false; - if (prev && (prev->vm_flags & VM_GROWSUP)) - return false; - return true; -} - /* * __split_vma() bypasses sysctl_max_map_count checking. We use this where it * has already been checked or doesn't make sense to fail. @@ -2675,6 +2199,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, { struct vm_area_struct *new; int err; + validate_mm_mt(mm); if (vma->vm_ops && vma->vm_ops->may_split) { err = vma->vm_ops->may_split(vma, addr); @@ -2717,6 +2242,9 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (!err) return 0; + /* Avoid vm accounting in close() operation */ + new->vm_start = new->vm_end; + new->vm_pgoff = 0; /* Clean everything up if vma_adjust failed. */ if (new->vm_ops && new->vm_ops->close) new->vm_ops->close(new); @@ -2727,6 +2255,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, mpol_put(vma_policy(new)); out_free_vma: vm_area_free(new); + validate_mm_mt(mm); return err; } @@ -2743,38 +2272,48 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, return __split_vma(mm, vma, addr, new_below); } -/* Munmap is split into 2 main parts -- this part which finds - * what needs doing, and the areas themselves, which do the - * work. This now handles partial unmappings. - * Jeremy Fitzhardinge - */ -int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, - struct list_head *uf, bool downgrade) +static inline int munmap_sidetree(struct vm_area_struct *vma, + struct ma_state *mas_detach) { - unsigned long end; - struct vm_area_struct *vma, *prev, *last; - - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) - return -EINVAL; + mas_set_range(mas_detach, vma->vm_start, vma->vm_end - 1); + if (mas_store_gfp(mas_detach, vma, GFP_KERNEL)) + return -ENOMEM; - len = PAGE_ALIGN(len); - end = start + len; - if (len == 0) - return -EINVAL; + if (vma->vm_flags & VM_LOCKED) + vma->vm_mm->locked_vm -= vma_pages(vma); - /* - * arch_unmap() might do unmaps itself. It must be called - * and finish any rbtree manipulation before this code - * runs and also starts to manipulate the rbtree. - */ - arch_unmap(mm, start, end); + return 0; +} - /* Find the first overlapping VMA where start < vma->vm_end */ - vma = find_vma_intersection(mm, start, end); - if (!vma) - return 0; - prev = vma->vm_prev; +/* + * do_mas_align_munmap() - munmap the aligned region from @start to @end. + * @mas: The maple_state, ideally set up to alter the correct tree location. + * @vma: The starting vm_area_struct + * @mm: The mm_struct + * @start: The aligned start address to munmap. + * @end: The aligned end address to munmap. + * @uf: The userfaultfd list_head + * @downgrade: Set to true to attempt a write downgrade of the mmap_sem + * + * If @downgrade is true, check return code for potential release of the lock. + */ +static int +do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, + struct mm_struct *mm, unsigned long start, + unsigned long end, struct list_head *uf, bool downgrade) +{ + struct vm_area_struct *prev, *next = NULL; + struct maple_tree mt_detach; + int count = 0; + int error = -ENOMEM; + MA_STATE(mas_detach, &mt_detach, 0, 0); + mt_init_flags(&mt_detach, MT_FLAGS_LOCK_EXTERN); + mt_set_external_lock(&mt_detach, &mm->mmap_lock); + + if (mas_preallocate(mas, vma, GFP_KERNEL)) + return -ENOMEM; + mas->last = end - 1; /* * If we need to split any vma, do it now to save pain later. * @@ -2782,8 +2321,9 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, * unmapped vm_area_struct will remain in use: so lower split_vma * places tmp vma above, and higher split_vma places tmp vma below. */ + + /* Does it split the first one? */ if (start > vma->vm_start) { - int error; /* * Make sure that map_count on return from munmap() will @@ -2791,22 +2331,61 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, * its limit temporarily, to help free resources as expected. */ if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count) - return -ENOMEM; + goto map_count_exceeded; + /* + * mas_pause() is not needed since mas->index needs to be set + * differently than vma->vm_end anyways. + */ error = __split_vma(mm, vma, start, 0); if (error) - return error; - prev = vma; + goto start_split_failed; + + mas_set(mas, start); + vma = mas_walk(mas); } - /* Does it split the last one? */ - last = find_vma(mm, end); - if (last && end > last->vm_start) { - int error = __split_vma(mm, last, end, 1); + prev = mas_prev(mas, 0); + if (unlikely((!prev))) + mas_set(mas, start); + + /* + * Detach a range of VMAs from the mm. Using next as a temp variable as + * it is always overwritten. + */ + mas_for_each(mas, next, end - 1) { + /* Does it split the end? */ + if (next->vm_end > end) { + struct vm_area_struct *split; + + error = __split_vma(mm, next, end, 1); + if (error) + goto end_split_failed; + + mas_set(mas, end); + split = mas_prev(mas, 0); + error = munmap_sidetree(split, &mas_detach); + if (error) + goto munmap_sidetree_failed; + + count++; + if (vma == next) + vma = split; + break; + } + error = munmap_sidetree(next, &mas_detach); if (error) - return error; + goto munmap_sidetree_failed; + + count++; +#ifdef CONFIG_DEBUG_VM_MAPLE_TREE + BUG_ON(next->vm_start < start); + BUG_ON(next->vm_start > end); +#endif } - vma = vma_next(mm, prev); + + if (!next) + next = mas_next(mas, ULONG_MAX); if (unlikely(uf)) { /* @@ -2818,30 +2397,369 @@ int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len, * split, despite we could. This is unlikely enough * failure that it's not worth optimizing it for. */ - int error = userfaultfd_unmap_prep(vma, start, end, uf); + error = userfaultfd_unmap_prep(mm, start, end, uf); + if (error) - return error; + goto userfaultfd_error; + } + + /* Point of no return */ + mas_set_range(mas, start, end - 1); +#if defined(CONFIG_DEBUG_VM_MAPLE_TREE) + /* Make sure no VMAs are about to be lost. */ + { + MA_STATE(test, &mt_detach, start, end - 1); + struct vm_area_struct *vma_mas, *vma_test; + int test_count = 0; + + rcu_read_lock(); + vma_test = mas_find(&test, end - 1); + mas_for_each(mas, vma_mas, end - 1) { + BUG_ON(vma_mas != vma_test); + test_count++; + vma_test = mas_next(&test, end - 1); + } + rcu_read_unlock(); + BUG_ON(count != test_count); + mas_set_range(mas, start, end - 1); } +#endif + mas_store_prealloc(mas, NULL); + mm->map_count -= count; + /* + * Do not downgrade mmap_lock if we are next to VM_GROWSDOWN or + * VM_GROWSUP VMA. Such VMAs can change their size under + * down_read(mmap_lock) and collide with the VMA we are about to unmap. + */ + if (downgrade) { + if (next && (next->vm_flags & VM_GROWSDOWN)) + downgrade = false; + else if (prev && (prev->vm_flags & VM_GROWSUP)) + downgrade = false; + else + mmap_write_downgrade(mm); + } + + unmap_region(mm, &mt_detach, vma, prev, next, start, end); + /* Statistics and freeing VMAs */ + mas_set(&mas_detach, start); + remove_mt(mm, &mas_detach); + __mt_destroy(&mt_detach); - /* Detach vmas from rbtree */ - if (!detach_vmas_to_be_unmapped(mm, vma, prev, end)) - downgrade = false; - if (downgrade) - mmap_write_downgrade(mm); + validate_mm(mm); + return downgrade ? 1 : 0; + +userfaultfd_error: +munmap_sidetree_failed: +end_split_failed: + __mt_destroy(&mt_detach); +start_split_failed: +map_count_exceeded: + mas_destroy(mas); + return error; +} - unmap_region(mm, vma, prev, start, end); +/* + * do_mas_munmap() - munmap a given range. + * @mas: The maple state + * @mm: The mm_struct + * @start: The start address to munmap + * @len: The length of the range to munmap + * @uf: The userfaultfd list_head + * @downgrade: set to true if the user wants to attempt to write_downgrade the + * mmap_sem + * + * This function takes a @mas that is either pointing to the previous VMA or set + * to MA_START and sets it up to remove the mapping(s). The @len will be + * aligned and any arch_unmap work will be preformed. + * + * Returns: -EINVAL on failure, 1 on success and unlock, 0 otherwise. + */ +int do_mas_munmap(struct ma_state *mas, struct mm_struct *mm, + unsigned long start, size_t len, struct list_head *uf, + bool downgrade) +{ + unsigned long end; + struct vm_area_struct *vma; - /* Fix up all other VM information */ - remove_vma_list(mm, vma); + if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + return -EINVAL; - return downgrade ? 1 : 0; + end = start + PAGE_ALIGN(len); + if (end == start) + return -EINVAL; + + /* arch_unmap() might do unmaps itself. */ + arch_unmap(mm, start, end); + + /* Find the first overlapping VMA */ + vma = mas_find(mas, end - 1); + if (!vma) + return 0; + + return do_mas_align_munmap(mas, vma, mm, start, end, uf, downgrade); } +/* do_munmap() - Wrapper function for non-maple tree aware do_munmap() calls. + * @mm: The mm_struct + * @start: The start address to munmap + * @len: The length to be munmapped. + * @uf: The userfaultfd list_head + */ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) { - return __do_munmap(mm, start, len, uf, false); + MA_STATE(mas, &mm->mm_mt, start, start); + + return do_mas_munmap(&mas, mm, start, len, uf, false); +} + +unsigned long mmap_region(struct file *file, unsigned long addr, + unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + struct list_head *uf) +{ + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = NULL; + struct vm_area_struct *next, *prev, *merge; + pgoff_t pglen = len >> PAGE_SHIFT; + unsigned long charged = 0; + unsigned long end = addr + len; + unsigned long merge_start = addr, merge_end = end; + pgoff_t vm_pgoff; + int error; + MA_STATE(mas, &mm->mm_mt, addr, end - 1); + + /* Check against address space limit. */ + if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) { + unsigned long nr_pages; + + /* + * MAP_FIXED may remove pages of mappings that intersects with + * requested mapping. Account for the pages it would unmap. + */ + nr_pages = count_vma_pages_range(mm, addr, end); + + if (!may_expand_vm(mm, vm_flags, + (len >> PAGE_SHIFT) - nr_pages)) + return -ENOMEM; + } + + /* Unmap any existing mapping in the area */ + if (do_mas_munmap(&mas, mm, addr, len, uf, false)) + return -ENOMEM; + + /* + * Private writable mapping: check memory availability + */ + if (accountable_mapping(file, vm_flags)) { + charged = len >> PAGE_SHIFT; + if (security_vm_enough_memory_mm(mm, charged)) + return -ENOMEM; + vm_flags |= VM_ACCOUNT; + } + + next = mas_next(&mas, ULONG_MAX); + prev = mas_prev(&mas, 0); + if (vm_flags & VM_SPECIAL) + goto cannot_expand; + + /* Attempt to expand an old mapping */ + /* Check next */ + if (next && next->vm_start == end && !vma_policy(next) && + can_vma_merge_before(next, vm_flags, NULL, file, pgoff+pglen, + NULL_VM_UFFD_CTX, NULL)) { + merge_end = next->vm_end; + vma = next; + vm_pgoff = next->vm_pgoff - pglen; + } + + /* Check prev */ + if (prev && prev->vm_end == addr && !vma_policy(prev) && + (vma ? can_vma_merge_after(prev, vm_flags, vma->anon_vma, file, + pgoff, vma->vm_userfaultfd_ctx, NULL) : + can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, + NULL_VM_UFFD_CTX, NULL))) { + merge_start = prev->vm_start; + vma = prev; + vm_pgoff = prev->vm_pgoff; + } + + + /* Actually expand, if possible */ + if (vma && + !vma_expand(&mas, vma, merge_start, merge_end, vm_pgoff, next)) { + khugepaged_enter_vma(vma, vm_flags); + goto expanded; + } + + mas.index = addr; + mas.last = end - 1; +cannot_expand: + /* + * Determine the object being mapped and call the appropriate + * specific mapper. the address has already been validated, but + * not unmapped, but the maps are removed from the list. + */ + vma = vm_area_alloc(mm); + if (!vma) { + error = -ENOMEM; + goto unacct_error; + } + + vma->vm_start = addr; + vma->vm_end = end; + vma->vm_flags = vm_flags; + vma->vm_page_prot = vm_get_page_prot(vm_flags); + vma->vm_pgoff = pgoff; + + if (file) { + if (vm_flags & VM_SHARED) { + error = mapping_map_writable(file->f_mapping); + if (error) + goto free_vma; + } + + vma->vm_file = get_file(file); + error = call_mmap(file, vma); + if (error) + goto unmap_and_free_vma; + + /* Can addr have changed?? + * + * Answer: Yes, several device drivers can do it in their + * f_op->mmap method. -DaveM + */ + WARN_ON_ONCE(addr != vma->vm_start); + + addr = vma->vm_start; + mas_reset(&mas); + + /* + * If vm_flags changed after call_mmap(), we should try merge + * vma again as we may succeed this time. + */ + if (unlikely(vm_flags != vma->vm_flags && prev)) { + merge = vma_merge(mm, prev, vma->vm_start, vma->vm_end, vma->vm_flags, + NULL, vma->vm_file, vma->vm_pgoff, NULL, NULL_VM_UFFD_CTX, NULL); + if (merge) { + /* + * ->mmap() can change vma->vm_file and fput + * the original file. So fput the vma->vm_file + * here or we would add an extra fput for file + * and cause general protection fault + * ultimately. + */ + fput(vma->vm_file); + vm_area_free(vma); + vma = merge; + /* Update vm_flags to pick up the change. */ + addr = vma->vm_start; + vm_flags = vma->vm_flags; + goto unmap_writable; + } + } + + vm_flags = vma->vm_flags; + } else if (vm_flags & VM_SHARED) { + error = shmem_zero_setup(vma); + if (error) + goto free_vma; + } else { + vma_set_anonymous(vma); + } + + /* Allow architectures to sanity-check the vm_flags */ + if (!arch_validate_flags(vma->vm_flags)) { + error = -EINVAL; + if (file) + goto close_and_free_vma; + else + goto free_vma; + } + + if (mas_preallocate(&mas, vma, GFP_KERNEL)) { + error = -ENOMEM; + if (file) + goto unmap_and_free_vma; + else + goto free_vma; + } + + if (vma->vm_file) + i_mmap_lock_write(vma->vm_file->f_mapping); + + vma_mas_store(vma, &mas); + mm->map_count++; + if (vma->vm_file) { + if (vma->vm_flags & VM_SHARED) + mapping_allow_writable(vma->vm_file->f_mapping); + + flush_dcache_mmap_lock(vma->vm_file->f_mapping); + vma_interval_tree_insert(vma, &vma->vm_file->f_mapping->i_mmap); + flush_dcache_mmap_unlock(vma->vm_file->f_mapping); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + /* + * vma_merge() calls khugepaged_enter_vma() either, the below + * call covers the non-merge case. + */ + khugepaged_enter_vma(vma, vma->vm_flags); + + /* Once vma denies write, undo our temporary denial count */ +unmap_writable: + if (file && vm_flags & VM_SHARED) + mapping_unmap_writable(file->f_mapping); + file = vma->vm_file; +expanded: + perf_event_mmap(vma); + + vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT); + if (vm_flags & VM_LOCKED) { + if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || + is_vm_hugetlb_page(vma) || + vma == get_gate_vma(current->mm)) + vma->vm_flags &= VM_LOCKED_CLEAR_MASK; + else + mm->locked_vm += (len >> PAGE_SHIFT); + } + + if (file) + uprobe_mmap(vma); + + /* + * New (or expanded) vma always get soft dirty status. + * Otherwise user-space soft-dirty page tracker won't + * be able to distinguish situation when vma area unmapped, + * then new mapped in-place (which must be aimed as + * a completely new data area). + */ + vma->vm_flags |= VM_SOFTDIRTY; + + vma_set_page_prot(vma); + + validate_mm(mm); + return addr; + +close_and_free_vma: + if (vma->vm_ops && vma->vm_ops->close) + vma->vm_ops->close(vma); +unmap_and_free_vma: + fput(vma->vm_file); + vma->vm_file = NULL; + + /* Undo any partial mapping done by a device driver. */ + unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); + if (vm_flags & VM_SHARED) + mapping_unmap_writable(file->f_mapping); +free_vma: + vm_area_free(vma); +unacct_error: + if (charged) + vm_unacct_memory(charged); + validate_mm(mm); + return error; } static int __vm_munmap(unsigned long start, size_t len, bool downgrade) @@ -2849,11 +2767,12 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) int ret; struct mm_struct *mm = current->mm; LIST_HEAD(uf); + MA_STATE(mas, &mm->mm_mt, start, start); if (mmap_write_lock_killable(mm)) return -EINTR; - ret = __do_munmap(mm, start, len, &uf, downgrade); + ret = do_mas_munmap(&mas, mm, start, len, &uf, downgrade); /* * Returning 1 indicates mmap_lock is downgraded. * But 1 is not legal return value of vm_munmap() and munmap(), reset @@ -2919,11 +2838,12 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, goto out; if (start + size > vma->vm_end) { - struct vm_area_struct *next; + VMA_ITERATOR(vmi, mm, vma->vm_end); + struct vm_area_struct *next, *prev = vma; - for (next = vma->vm_next; next; next = next->vm_next) { + for_each_vma_range(vmi, next, start + size) { /* hole between vmas ? */ - if (next->vm_start != next->vm_prev->vm_end) + if (next->vm_start != prev->vm_end) goto out; if (next->vm_file != vma->vm_file) @@ -2932,8 +2852,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, if (next->vm_flags != vma->vm_flags) goto out; - if (start + size <= next->vm_end) - break; + prev = next; } if (!next) @@ -2963,37 +2882,53 @@ out: } /* - * this is really a simplified "do_mmap". it only handles - * anonymous maps. eventually we may be able to do some - * brk-specific accounting here. + * brk_munmap() - Unmap a parital vma. + * @mas: The maple tree state. + * @vma: The vma to be modified + * @newbrk: the start of the address to unmap + * @oldbrk: The end of the address to unmap + * @uf: The userfaultfd list_head + * + * Returns: 1 on success. + * unmaps a partial VMA mapping. Does not handle alignment, downgrades lock if + * possible. */ -static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf) +static int do_brk_munmap(struct ma_state *mas, struct vm_area_struct *vma, + unsigned long newbrk, unsigned long oldbrk, + struct list_head *uf) { - struct mm_struct *mm = current->mm; - struct vm_area_struct *vma, *prev; - struct rb_node **rb_link, *rb_parent; - pgoff_t pgoff = addr >> PAGE_SHIFT; - int error; - unsigned long mapped_addr; - - /* Until we need other flags, refuse anything except VM_EXEC. */ - if ((flags & (~VM_EXEC)) != 0) - return -EINVAL; - flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; - - mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); - if (IS_ERR_VALUE(mapped_addr)) - return mapped_addr; + struct mm_struct *mm = vma->vm_mm; + int ret; - error = mlock_future_check(mm, mm->def_flags, len); - if (error) - return error; + arch_unmap(mm, newbrk, oldbrk); + ret = do_mas_align_munmap(mas, vma, mm, newbrk, oldbrk, uf, true); + validate_mm_mt(mm); + return ret; +} - /* Clear old maps, set up prev, rb_link, rb_parent, and uf */ - if (munmap_vma_range(mm, addr, len, &prev, &rb_link, &rb_parent, uf)) - return -ENOMEM; +/* + * do_brk_flags() - Increase the brk vma if the flags match. + * @mas: The maple tree state. + * @addr: The start address + * @len: The length of the increase + * @vma: The vma, + * @flags: The VMA Flags + * + * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags + * do not match then create a new anonymous VMA. Eventually we may be able to + * do some brk-specific accounting here. + */ +static int do_brk_flags(struct ma_state *mas, struct vm_area_struct *vma, + unsigned long addr, unsigned long len, unsigned long flags) +{ + struct mm_struct *mm = current->mm; - /* Check against address space limits *after* clearing old maps... */ + validate_mm_mt(mm); + /* + * Check against address space limits by the changed size + * Note: This happens *after* clearing old mappings in some code paths. + */ + flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) return -ENOMEM; @@ -3003,28 +2938,50 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) return -ENOMEM; - /* Can we just expand an old private anonymous mapping? */ - vma = vma_merge(mm, prev, addr, addr + len, flags, - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, NULL); - if (vma) - goto out; - /* - * create a vma struct for an anonymous mapping + * Expand the existing vma if possible; Note that singular lists do not + * occur after forking, so the expand will only happen on new VMAs. */ - vma = vm_area_alloc(mm); - if (!vma) { - vm_unacct_memory(len >> PAGE_SHIFT); - return -ENOMEM; + if (vma && + (!vma->anon_vma || list_is_singular(&vma->anon_vma_chain)) && + ((vma->vm_flags & ~VM_SOFTDIRTY) == flags)) { + mas_set_range(mas, vma->vm_start, addr + len - 1); + if (mas_preallocate(mas, vma, GFP_KERNEL)) + return -ENOMEM; + + vma_adjust_trans_huge(vma, vma->vm_start, addr + len, 0); + if (vma->anon_vma) { + anon_vma_lock_write(vma->anon_vma); + anon_vma_interval_tree_pre_update_vma(vma); + } + vma->vm_end = addr + len; + vma->vm_flags |= VM_SOFTDIRTY; + mas_store_prealloc(mas, vma); + + if (vma->anon_vma) { + anon_vma_interval_tree_post_update_vma(vma); + anon_vma_unlock_write(vma->anon_vma); + } + khugepaged_enter_vma(vma, flags); + goto out; } + /* create a vma struct for an anonymous mapping */ + vma = vm_area_alloc(mm); + if (!vma) + goto vma_alloc_fail; + vma_set_anonymous(vma); vma->vm_start = addr; vma->vm_end = addr + len; - vma->vm_pgoff = pgoff; + vma->vm_pgoff = addr >> PAGE_SHIFT; vma->vm_flags = flags; vma->vm_page_prot = vm_get_page_prot(flags); - vma_link(mm, vma, prev, rb_link, rb_parent); + mas_set_range(mas, vma->vm_start, addr + len - 1); + if (mas_store_gfp(mas, vma, GFP_KERNEL)) + goto mas_store_fail; + + mm->map_count++; out: perf_event_mmap(vma); mm->total_vm += len >> PAGE_SHIFT; @@ -3032,16 +2989,25 @@ out: if (flags & VM_LOCKED) mm->locked_vm += (len >> PAGE_SHIFT); vma->vm_flags |= VM_SOFTDIRTY; + validate_mm(mm); return 0; + +mas_store_fail: + vm_area_free(vma); +vma_alloc_fail: + vm_unacct_memory(len >> PAGE_SHIFT); + return -ENOMEM; } int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) { struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = NULL; unsigned long len; int ret; bool populate; LIST_HEAD(uf); + MA_STATE(mas, &mm->mm_mt, addr, addr); len = PAGE_ALIGN(request); if (len < request) @@ -3052,13 +3018,36 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_brk_flags(addr, len, flags, &uf); + /* Until we need other flags, refuse anything except VM_EXEC. */ + if ((flags & (~VM_EXEC)) != 0) + return -EINVAL; + + ret = check_brk_limits(addr, len); + if (ret) + goto limits_failed; + + ret = do_mas_munmap(&mas, mm, addr, len, &uf, 0); + if (ret) + goto munmap_failed; + + vma = mas_prev(&mas, 0); + if (!vma || vma->vm_end != addr || vma_policy(vma) || + !can_vma_merge_after(vma, flags, NULL, NULL, + addr >> PAGE_SHIFT, NULL_VM_UFFD_CTX, NULL)) + vma = NULL; + + ret = do_brk_flags(&mas, vma, addr, len, flags); populate = ((mm->def_flags & VM_LOCKED) != 0); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate && !ret) mm_populate(addr, len); return ret; + +munmap_failed: +limits_failed: + mmap_write_unlock(mm); + return ret; } EXPORT_SYMBOL(vm_brk_flags); @@ -3074,34 +3063,19 @@ void exit_mmap(struct mm_struct *mm) struct mmu_gather tlb; struct vm_area_struct *vma; unsigned long nr_accounted = 0; + MA_STATE(mas, &mm->mm_mt, 0, 0); + int count = 0; /* mm's last user has gone, and its about to be pulled down */ mmu_notifier_release(mm); - if (unlikely(mm_is_oom_victim(mm))) { - /* - * Manually reap the mm to free as much memory as possible. - * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard - * this mm from further consideration. Taking mm->mmap_lock for - * write after setting MMF_OOM_SKIP will guarantee that the oom - * reaper will not run on this mm again after mmap_lock is - * dropped. - * - * Nothing can be holding mm->mmap_lock here and the above call - * to mmu_notifier_release(mm) ensures mmu notifier callbacks in - * __oom_reap_task_mm() will not block. - */ - (void)__oom_reap_task_mm(mm); - set_bit(MMF_OOM_SKIP, &mm->flags); - } - - mmap_write_lock(mm); + mmap_read_lock(mm); arch_exit_mmap(mm); - vma = mm->mmap; + vma = mas_find(&mas, ULONG_MAX); if (!vma) { /* Can happen if dup_mmap() received an OOM */ - mmap_write_unlock(mm); + mmap_read_unlock(mm); return; } @@ -3109,19 +3083,37 @@ void exit_mmap(struct mm_struct *mm) flush_cache_mm(mm); tlb_gather_mmu_fullmm(&tlb, mm); /* update_hiwater_rss(mm) here? but nobody should be looking */ - /* Use -1 here to ensure all VMAs in the mm are unmapped */ - unmap_vmas(&tlb, vma, 0, -1); - free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING); + /* Use ULONG_MAX here to ensure all VMAs in the mm are unmapped */ + unmap_vmas(&tlb, &mm->mm_mt, vma, 0, ULONG_MAX); + mmap_read_unlock(mm); + + /* + * Set MMF_OOM_SKIP to hide this task from the oom killer/reaper + * because the memory has been already freed. + */ + set_bit(MMF_OOM_SKIP, &mm->flags); + mmap_write_lock(mm); + free_pgtables(&tlb, &mm->mm_mt, vma, FIRST_USER_ADDRESS, + USER_PGTABLES_CEILING); tlb_finish_mmu(&tlb); - /* Walk the list again, actually closing and freeing it. */ - while (vma) { + /* + * Walk the list again, actually closing and freeing it, with preemption + * enabled, without holding any MM locks besides the unreachable + * mmap_write_lock. + */ + do { if (vma->vm_flags & VM_ACCOUNT) nr_accounted += vma_pages(vma); - vma = remove_vma(vma); + remove_vma(vma); + count++; cond_resched(); - } - mm->mmap = NULL; + } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL); + + BUG_ON(count != mm->map_count); + + trace_exit_mmap(mm); + __mt_destroy(&mm->mm_mt); mmap_write_unlock(mm); vm_unacct_memory(nr_accounted); } @@ -3132,14 +3124,14 @@ void exit_mmap(struct mm_struct *mm) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - struct vm_area_struct *prev; - struct rb_node **rb_link, *rb_parent; + unsigned long charged = vma_pages(vma); + - if (find_vma_links(mm, vma->vm_start, vma->vm_end, - &prev, &rb_link, &rb_parent)) + if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; + if ((vma->vm_flags & VM_ACCOUNT) && - security_vm_enough_memory_mm(mm, vma_pages(vma))) + security_vm_enough_memory_mm(mm, charged)) return -ENOMEM; /* @@ -3159,7 +3151,11 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; } - vma_link(mm, vma, prev, rb_link, rb_parent); + if (vma_link(mm, vma)) { + vm_unacct_memory(charged); + return -ENOMEM; + } + return 0; } @@ -3175,9 +3171,9 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long vma_start = vma->vm_start; struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *new_vma, *prev; - struct rb_node **rb_link, *rb_parent; bool faulted_in_anon_vma = true; + validate_mm_mt(mm); /* * If anonymous vma has not yet been faulted, update new pgoff * to match new location, to increase its chance of merging. @@ -3187,8 +3183,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, faulted_in_anon_vma = false; } - if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent)) + new_vma = find_vma_prev(mm, addr, &prev); + if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ + new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), vma->vm_userfaultfd_ctx, anon_vma_name(vma)); @@ -3229,16 +3227,27 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, get_file(new_vma->vm_file); if (new_vma->vm_ops && new_vma->vm_ops->open) new_vma->vm_ops->open(new_vma); - vma_link(mm, new_vma, prev, rb_link, rb_parent); + if (vma_link(mm, new_vma)) + goto out_vma_link; *need_rmap_locks = false; } + validate_mm_mt(mm); return new_vma; +out_vma_link: + if (new_vma->vm_ops && new_vma->vm_ops->close) + new_vma->vm_ops->close(new_vma); + + if (new_vma->vm_file) + fput(new_vma->vm_file); + + unlink_anon_vmas(new_vma); out_free_mempol: mpol_put(vma_policy(new_vma)); out_free_vma: vm_area_free(new_vma); out: + validate_mm_mt(mm); return NULL; } @@ -3375,6 +3384,7 @@ static struct vm_area_struct *__install_special_mapping( int ret; struct vm_area_struct *vma; + validate_mm_mt(mm); vma = vm_area_alloc(mm); if (unlikely(vma == NULL)) return ERR_PTR(-ENOMEM); @@ -3397,10 +3407,12 @@ static struct vm_area_struct *__install_special_mapping( perf_event_mmap(vma); + validate_mm_mt(mm); return vma; out: vm_area_free(vma); + validate_mm_mt(mm); return ERR_PTR(ret); } @@ -3525,12 +3537,13 @@ int mm_take_all_locks(struct mm_struct *mm) { struct vm_area_struct *vma; struct anon_vma_chain *avc; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_assert_write_locked(mm); mutex_lock(&mm_all_locks_mutex); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (signal_pending(current)) goto out_unlock; if (vma->vm_file && vma->vm_file->f_mapping && @@ -3538,7 +3551,8 @@ int mm_take_all_locks(struct mm_struct *mm) vm_lock_mapping(mm, vma->vm_file->f_mapping); } - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_set(&mas, 0); + mas_for_each(&mas, vma, ULONG_MAX) { if (signal_pending(current)) goto out_unlock; if (vma->vm_file && vma->vm_file->f_mapping && @@ -3546,7 +3560,8 @@ int mm_take_all_locks(struct mm_struct *mm) vm_lock_mapping(mm, vma->vm_file->f_mapping); } - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_set(&mas, 0); + mas_for_each(&mas, vma, ULONG_MAX) { if (signal_pending(current)) goto out_unlock; if (vma->anon_vma) @@ -3605,11 +3620,12 @@ void mm_drop_all_locks(struct mm_struct *mm) { struct vm_area_struct *vma; struct anon_vma_chain *avc; + MA_STATE(mas, &mm->mm_mt, 0, 0); mmap_assert_write_locked(mm); BUG_ON(!mutex_is_locked(&mm_all_locks_mutex)); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + mas_for_each(&mas, vma, ULONG_MAX) { if (vma->anon_vma) list_for_each_entry(avc, &vma->anon_vma_chain, same_vma) vm_unlock_anon_vma(avc->anon_vma); diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index a71924bd38c0de1b734016300a96da047d126a51..add4244e5790df28140b8ea7e41125be37b59fbb 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -265,6 +266,15 @@ void tlb_flush_mmu(struct mmu_gather *tlb) static void __tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm) { + /* + * struct mmu_gather contains 7 1-bit fields packed into a 32-bit + * unsigned int value. The remaining 25 bits remain uninitialized + * and are never used, but KMSAN updates the origin for them in + * zap_pXX_range() in mm/memory.c, thus creating very long origin + * chains. This is technically correct, but consumes too much memory. + * Unpoisoning the whole structure will prevent creating such chains. + */ + kmsan_unpoison_memory(tlb, sizeof(*tlb)); tlb->mm = mm; tlb->fullmm = fullmm; diff --git a/mm/mmzone.c b/mm/mmzone.c index 0ae7571e35abb07795037c510ec1c161ce1e8b82..68e1511be12de6052b91eed6c86287042dd69d73 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -88,6 +88,8 @@ void lruvec_init(struct lruvec *lruvec) * Poison its list head, so that any operations on it would crash. */ list_del(&lruvec->lists[LRU_UNEVICTABLE]); + + lru_gen_init_lruvec(lruvec); } #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) diff --git a/mm/mprotect.c b/mm/mprotect.c index 3a23dde73723bc8adb1847c403b99a3d66970b88..668bfaa6ed2aebdbc64182e7d2c8a007a2d4bae7 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,7 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, if (prot_numa) { struct page *page; int nid; + bool toptier; /* Avoid TLB flush if possible */ if (pte_protnone(oldpte)) @@ -150,14 +152,19 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, nid = page_to_nid(page); if (target_node == nid) continue; + toptier = node_is_toptier(nid); /* * Skip scanning top tier node if normal numa * balancing is disabled */ if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && - node_is_toptier(nid)) + toptier) continue; + if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING && + !toptier) + xchg_page_access_time(page, + jiffies_to_msecs(jiffies)); } oldpte = ptep_modify_prot_start(vma, addr, pte); @@ -196,10 +203,11 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, pages++; } else if (is_swap_pte(oldpte)) { swp_entry_t entry = pte_to_swp_entry(oldpte); - struct page *page = pfn_swap_entry_to_page(entry); pte_t newpte; if (is_writable_migration_entry(entry)) { + struct page *page = pfn_swap_entry_to_page(entry); + /* * A protection check is difficult so * just be safe and disable write @@ -259,6 +267,7 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, } else { /* It must be an none page, or what else?.. */ WARN_ON_ONCE(!pte_none(oldpte)); +#ifdef CONFIG_PTE_MARKER_UFFD_WP if (unlikely(uffd_wp && !vma_is_anonymous(vma))) { /* * For file-backed mem, we need to be able to @@ -270,6 +279,7 @@ static unsigned long change_pte_range(struct mmu_gather *tlb, make_pte_marker(PTE_MARKER_UFFD_WP)); pages++; } +#endif } } while (pte++, addr += PAGE_SIZE, addr != end); arch_leave_lazy_mmu_mode(); @@ -668,6 +678,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); struct mmu_gather tlb; + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); start = untagged_addr(start); @@ -699,7 +710,8 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; - vma = find_vma(current->mm, start); + mas_set(&mas, start); + vma = mas_find(&mas, ULONG_MAX); error = -ENOMEM; if (!vma) goto out; @@ -725,7 +737,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (start > vma->vm_start) prev = vma; else - prev = vma->vm_prev; + prev = mas_prev(&mas, 0); tlb_gather_mmu(&tlb, current->mm); for (nstart = start ; ; ) { @@ -788,7 +800,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (nstart >= end) break; - vma = prev->vm_next; + vma = find_vma(current->mm, prev->vm_end); if (!vma || vma->vm_start != nstart) { error = -ENOMEM; break; diff --git a/mm/mremap.c b/mm/mremap.c index b522cd0259a0f1553645d22eb682a489ea1c5345..e465ffe279bb03ac38d20bf5b83ae1d4eb700c3c 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -716,7 +718,7 @@ static unsigned long move_vma(struct vm_area_struct *vma, if (excess) { vma->vm_flags |= VM_ACCOUNT; if (split) - vma->vm_next->vm_flags |= VM_ACCOUNT; + find_vma(mm, vma->vm_end)->vm_flags |= VM_ACCOUNT; } return new_addr; @@ -866,9 +868,10 @@ out: static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) { unsigned long end = vma->vm_end + delta; + if (end < vma->vm_end) /* overflow */ return 0; - if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ + if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, 0, MAP_FIXED) & ~PAGE_MASK) @@ -975,20 +978,23 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, /* * Always allow a shrinking remap: that just unmaps * the unnecessary pages.. - * __do_munmap does all the needed commit accounting, and + * do_mas_munmap does all the needed commit accounting, and * downgrades mmap_lock to read if so directed. */ if (old_len >= new_len) { int retval; + MA_STATE(mas, &mm->mm_mt, addr + new_len, addr + new_len); - retval = __do_munmap(mm, addr+new_len, old_len - new_len, - &uf_unmap, true); - if (retval < 0 && old_len != new_len) { - ret = retval; - goto out; + retval = do_mas_munmap(&mas, mm, addr + new_len, + old_len - new_len, &uf_unmap, true); /* Returning 1 indicates mmap_lock is downgraded to read. */ - } else if (retval == 1) + if (retval == 1) { downgraded = true; + } else if (retval < 0 && old_len != new_len) { + ret = retval; + goto out; + } + ret = addr; goto out; } @@ -1008,6 +1014,9 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, /* can we just expand the current mapping? */ if (vma_expandable(vma, new_len - old_len)) { long pages = (new_len - old_len) >> PAGE_SHIFT; + unsigned long extension_start = addr + old_len; + unsigned long extension_end = addr + new_len; + pgoff_t extension_pgoff = vma->vm_pgoff + (old_len >> PAGE_SHIFT); if (vma->vm_flags & VM_ACCOUNT) { if (security_vm_enough_memory_mm(mm, pages)) { @@ -1016,8 +1025,18 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, } } - if (vma_adjust(vma, vma->vm_start, addr + new_len, - vma->vm_pgoff, NULL)) { + /* + * Function vma_merge() is called on the extension we are adding to + * the already existing vma, vma_merge() will merge this extension with + * the already existing vma (expand operation itself) and possibly also + * with the next vma if it becomes adjacent to the expanded vma and + * otherwise compatible. + */ + vma = vma_merge(mm, vma, extension_start, extension_end, + vma->vm_flags, vma->anon_vma, vma->vm_file, + extension_pgoff, vma_policy(vma), + vma->vm_userfaultfd_ctx, anon_vma_name(vma)); + if (!vma) { vm_unacct_memory(pages); ret = -ENOMEM; goto out; diff --git a/mm/msync.c b/mm/msync.c index 137d1c104f3e94399d6706c5e342af0048615a4d..ac4c9bfea2e7faff18c3cf3bd8b74766cdbee166 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -104,7 +104,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) error = 0; goto out_unlock; } - vma = vma->vm_next; + vma = find_vma(mm, vma->vm_end); } } out_unlock: diff --git a/mm/nommu.c b/mm/nommu.c index e819cbc21b396b6ac00f6485c5fb7bc2c8b87649..214c70e1d05942e8a0a9a2e4ece7a6f5086f9c62 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -545,26 +544,27 @@ static void put_nommu_region(struct vm_region *region) __put_nommu_region(region); } -/* - * add a VMA into a process's mm_struct in the appropriate place in the list - * and tree and add to the address space's page tree also if not an anonymous - * page - * - should be called with mm->mmap_lock held writelocked - */ -static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) +void vma_mas_store(struct vm_area_struct *vma, struct ma_state *mas) { - struct vm_area_struct *pvma, *prev; - struct address_space *mapping; - struct rb_node **p, *parent, *rb_prev; + mas_set_range(mas, vma->vm_start, vma->vm_end - 1); + mas_store_prealloc(mas, vma); +} - BUG_ON(!vma->vm_region); +void vma_mas_remove(struct vm_area_struct *vma, struct ma_state *mas) +{ + mas->index = vma->vm_start; + mas->last = vma->vm_end - 1; + mas_store_prealloc(mas, NULL); +} +static void setup_vma_to_mm(struct vm_area_struct *vma, struct mm_struct *mm) +{ mm->map_count++; vma->vm_mm = mm; /* add the VMA to the mapping */ if (vma->vm_file) { - mapping = vma->vm_file->f_mapping; + struct address_space *mapping = vma->vm_file->f_mapping; i_mmap_lock_write(mapping); flush_dcache_mmap_lock(mapping); @@ -572,67 +572,51 @@ static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } +} - /* add the VMA to the tree */ - parent = rb_prev = NULL; - p = &mm->mm_rb.rb_node; - while (*p) { - parent = *p; - pvma = rb_entry(parent, struct vm_area_struct, vm_rb); - - /* sort by: start addr, end addr, VMA struct addr in that order - * (the latter is necessary as we may get identical VMAs) */ - if (vma->vm_start < pvma->vm_start) - p = &(*p)->rb_left; - else if (vma->vm_start > pvma->vm_start) { - rb_prev = parent; - p = &(*p)->rb_right; - } else if (vma->vm_end < pvma->vm_end) - p = &(*p)->rb_left; - else if (vma->vm_end > pvma->vm_end) { - rb_prev = parent; - p = &(*p)->rb_right; - } else if (vma < pvma) - p = &(*p)->rb_left; - else if (vma > pvma) { - rb_prev = parent; - p = &(*p)->rb_right; - } else - BUG(); - } - - rb_link_node(&vma->vm_rb, parent, p); - rb_insert_color(&vma->vm_rb, &mm->mm_rb); +/* + * mas_add_vma_to_mm() - Maple state variant of add_mas_to_mm(). + * @mas: The maple state with preallocations. + * @mm: The mm_struct + * @vma: The vma to add + * + */ +static void mas_add_vma_to_mm(struct ma_state *mas, struct mm_struct *mm, + struct vm_area_struct *vma) +{ + BUG_ON(!vma->vm_region); - /* add VMA to the VMA list also */ - prev = NULL; - if (rb_prev) - prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); + setup_vma_to_mm(vma, mm); - __vma_link_list(mm, vma, prev); + /* add the VMA to the tree */ + vma_mas_store(vma, mas); } /* - * delete a VMA from its owning mm_struct and address space + * add a VMA into a process's mm_struct in the appropriate place in the list + * and tree and add to the address space's page tree also if not an anonymous + * page + * - should be called with mm->mmap_lock held writelocked */ -static void delete_vma_from_mm(struct vm_area_struct *vma) -{ - int i; - struct address_space *mapping; - struct mm_struct *mm = vma->vm_mm; - struct task_struct *curr = current; - - mm->map_count--; - for (i = 0; i < VMACACHE_SIZE; i++) { - /* if the vma is cached, invalidate the entire cache */ - if (curr->vmacache.vmas[i] == vma) { - vmacache_invalidate(mm); - break; - } +static int add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) +{ + MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); + + if (mas_preallocate(&mas, vma, GFP_KERNEL)) { + pr_warn("Allocation of vma tree for process %d failed\n", + current->pid); + return -ENOMEM; } + mas_add_vma_to_mm(&mas, mm, vma); + return 0; +} +static void cleanup_vma_from_mm(struct vm_area_struct *vma) +{ + vma->vm_mm->map_count--; /* remove the VMA from the mapping */ if (vma->vm_file) { + struct address_space *mapping; mapping = vma->vm_file->f_mapping; i_mmap_lock_write(mapping); @@ -641,11 +625,24 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) flush_dcache_mmap_unlock(mapping); i_mmap_unlock_write(mapping); } +} +/* + * delete a VMA from its owning mm_struct and address space + */ +static int delete_vma_from_mm(struct vm_area_struct *vma) +{ + MA_STATE(mas, &vma->vm_mm->mm_mt, 0, 0); - /* remove from the MM's tree and list */ - rb_erase(&vma->vm_rb, &mm->mm_rb); + if (mas_preallocate(&mas, vma, GFP_KERNEL)) { + pr_warn("Allocation of vma tree for process %d failed\n", + current->pid); + return -ENOMEM; + } + cleanup_vma_from_mm(vma); - __vma_unlink_list(mm, vma); + /* remove from the MM's tree and list */ + vma_mas_remove(vma, &mas); + return 0; } /* @@ -661,31 +658,26 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) vm_area_free(vma); } +struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, + unsigned long start_addr, + unsigned long end_addr) +{ + unsigned long index = start_addr; + + mmap_assert_locked(mm); + return mt_find(&mm->mm_mt, &index, end_addr - 1); +} +EXPORT_SYMBOL(find_vma_intersection); + /* * look up the first VMA in which addr resides, NULL if none * - should be called with mm->mmap_lock at least held readlocked */ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) { - struct vm_area_struct *vma; + MA_STATE(mas, &mm->mm_mt, addr, addr); - /* check the cache first */ - vma = vmacache_find(mm, addr); - if (likely(vma)) - return vma; - - /* trawl the list (there may be multiple mappings in which addr - * resides) */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->vm_start > addr) - return NULL; - if (vma->vm_end > addr) { - vmacache_update(addr, vma); - return vma; - } - } - - return NULL; + return mas_walk(&mas); } EXPORT_SYMBOL(find_vma); @@ -717,26 +709,17 @@ static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, { struct vm_area_struct *vma; unsigned long end = addr + len; + MA_STATE(mas, &mm->mm_mt, addr, addr); - /* check the cache first */ - vma = vmacache_find_exact(mm, addr, end); - if (vma) - return vma; - - /* trawl the list (there may be multiple mappings in which addr - * resides) */ - for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->vm_start < addr) - continue; - if (vma->vm_start > addr) - return NULL; - if (vma->vm_end == end) { - vmacache_update(addr, vma); - return vma; - } - } + vma = mas_walk(&mas); + if (!vma) + return NULL; + if (vma->vm_start != addr) + return NULL; + if (vma->vm_end != end) + return NULL; - return NULL; + return vma; } /* @@ -1069,6 +1052,7 @@ unsigned long do_mmap(struct file *file, vm_flags_t vm_flags; unsigned long capabilities, result; int ret; + MA_STATE(mas, ¤t->mm->mm_mt, 0, 0); *populate = 0; @@ -1087,6 +1071,7 @@ unsigned long do_mmap(struct file *file, * now know into VMA flags */ vm_flags = determine_vm_flags(file, prot, flags, capabilities); + /* we're going to need to record the mapping */ region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); if (!region) @@ -1096,6 +1081,9 @@ unsigned long do_mmap(struct file *file, if (!vma) goto error_getting_vma; + if (mas_preallocate(&mas, vma, GFP_KERNEL)) + goto error_maple_preallocate; + region->vm_usage = 1; region->vm_flags = vm_flags; region->vm_pgoff = pgoff; @@ -1236,7 +1224,7 @@ unsigned long do_mmap(struct file *file, current->mm->total_vm += len >> PAGE_SHIFT; share: - add_vma_to_mm(current->mm, vma); + mas_add_vma_to_mm(&mas, current->mm, vma); /* we flush the region from the icache only when the first executable * mapping of it is made */ @@ -1262,6 +1250,7 @@ error: sharing_violation: up_write(&nommu_region_sem); + mas_destroy(&mas); pr_warn("Attempt to share mismatched mappings\n"); ret = -EINVAL; goto error; @@ -1278,6 +1267,14 @@ error_getting_region: len, current->pid); show_free_areas(0, NULL); return -ENOMEM; + +error_maple_preallocate: + kmem_cache_free(vm_region_jar, region); + vm_area_free(vma); + pr_warn("Allocation of vma tree for process %d failed\n", current->pid); + show_free_areas(0, NULL); + return -ENOMEM; + } unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, @@ -1343,6 +1340,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *new; struct vm_region *region; unsigned long npages; + MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_end); /* we're only permitted to split anonymous regions (these should have * only a single usage on the region) */ @@ -1357,9 +1355,13 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, return -ENOMEM; new = vm_area_dup(vma); - if (!new) { - kmem_cache_free(vm_region_jar, region); - return -ENOMEM; + if (!new) + goto err_vma_dup; + + if (mas_preallocate(&mas, vma, GFP_KERNEL)) { + pr_warn("Allocation of vma tree for process %d failed\n", + current->pid); + goto err_mas_preallocate; } /* most fields are the same, copy all, and then fixup */ @@ -1378,7 +1380,6 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); - delete_vma_from_mm(vma); down_write(&nommu_region_sem); delete_nommu_region(vma->vm_region); if (new_below) { @@ -1391,9 +1392,19 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, add_nommu_region(vma->vm_region); add_nommu_region(new->vm_region); up_write(&nommu_region_sem); - add_vma_to_mm(mm, vma); - add_vma_to_mm(mm, new); + + setup_vma_to_mm(vma, mm); + setup_vma_to_mm(new, mm); + mas_set_range(&mas, vma->vm_start, vma->vm_end - 1); + mas_store(&mas, vma); + vma_mas_store(new, &mas); return 0; + +err_mas_preallocate: + vm_area_free(new); +err_vma_dup: + kmem_cache_free(vm_region_jar, region); + return -ENOMEM; } /* @@ -1408,12 +1419,14 @@ static int shrink_vma(struct mm_struct *mm, /* adjust the VMA's pointers, which may reposition it in the MM's tree * and list */ - delete_vma_from_mm(vma); + if (delete_vma_from_mm(vma)) + return -ENOMEM; if (from > vma->vm_start) vma->vm_end = from; else vma->vm_start = to; - add_vma_to_mm(mm, vma); + if (add_vma_to_mm(mm, vma)) + return -ENOMEM; /* cut the backing region down to size */ region = vma->vm_region; @@ -1441,9 +1454,10 @@ static int shrink_vma(struct mm_struct *mm, */ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) { + MA_STATE(mas, &mm->mm_mt, start, start); struct vm_area_struct *vma; unsigned long end; - int ret; + int ret = 0; len = PAGE_ALIGN(len); if (len == 0) @@ -1452,7 +1466,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list end = start + len; /* find the first potentially overlapping VMA */ - vma = find_vma(mm, start); + vma = mas_find(&mas, end - 1); if (!vma) { static int limit; if (limit < 5) { @@ -1471,7 +1485,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list return -EINVAL; if (end == vma->vm_end) goto erase_whole_vma; - vma = vma->vm_next; + vma = mas_next(&mas, end - 1); } while (vma); return -EINVAL; } else { @@ -1493,9 +1507,10 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list } erase_whole_vma: - delete_vma_from_mm(vma); + if (delete_vma_from_mm(vma)) + ret = -ENOMEM; delete_vma(mm, vma); - return 0; + return ret; } int vm_munmap(unsigned long addr, size_t len) @@ -1520,6 +1535,7 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) */ void exit_mmap(struct mm_struct *mm) { + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; if (!mm) @@ -1527,12 +1543,18 @@ void exit_mmap(struct mm_struct *mm) mm->total_vm = 0; - while ((vma = mm->mmap)) { - mm->mmap = vma->vm_next; - delete_vma_from_mm(vma); + /* + * Lock the mm to avoid assert complaining even though this is the only + * user of the mm + */ + mmap_write_lock(mm); + for_each_vma(vmi, vma) { + cleanup_vma_from_mm(vma); delete_vma(mm, vma); cond_resched(); } + __mt_destroy(&mm->mm_mt); + mmap_write_unlock(mm); } int vm_brk(unsigned long addr, unsigned long len) diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 3c6cf9e3cd66ea67cfe317dbe0f91c54d2e4faa8..1276e49b31b0a0dfd5ca625486ee50f4fc6131a1 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -461,7 +461,7 @@ static void dump_header(struct oom_control *oc, struct task_struct *p) if (is_memcg_oom(oc)) mem_cgroup_print_oom_meminfo(oc->memcg); else { - show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask); + __show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask, gfp_zone(oc->gfp_mask)); if (should_dump_unreclaim_slab()) dump_unreclaimable_slab(); } @@ -509,10 +509,11 @@ static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait); static struct task_struct *oom_reaper_list; static DEFINE_SPINLOCK(oom_reaper_lock); -bool __oom_reap_task_mm(struct mm_struct *mm) +static bool __oom_reap_task_mm(struct mm_struct *mm) { struct vm_area_struct *vma; bool ret = true; + VMA_ITERATOR(vmi, mm, 0); /* * Tell all users of get_user/copy_from_user etc... that the content @@ -522,7 +523,7 @@ bool __oom_reap_task_mm(struct mm_struct *mm) */ set_bit(MMF_UNSTABLE, &mm->flags); - for (vma = mm->mmap ; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (vma->vm_flags & (VM_HUGETLB|VM_PFNMAP)) continue; @@ -764,10 +765,8 @@ static void mark_oom_victim(struct task_struct *tsk) return; /* oom_mm is bound to the signal struct life time. */ - if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) { + if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) mmgrab(tsk->signal->oom_mm); - set_bit(MMF_OOM_VICTIM, &mm->flags); - } /* * Make sure that the task is woken up from uninterruptible sleep diff --git a/mm/page-writeback.c b/mm/page-writeback.c index d0d466a5c804ca0fa8709dd991dcd62f738cd8a3..7e9d8d857eccae53047699fe39549877cbec7b0e 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -1933,6 +1933,7 @@ int balance_dirty_pages_ratelimited_flags(struct address_space *mapping, wb_put(wb); return ret; } +EXPORT_SYMBOL_GPL(balance_dirty_pages_ratelimited_flags); /** * balance_dirty_pages_ratelimited - balance dirty memory state. @@ -2892,6 +2893,7 @@ static void wb_inode_writeback_start(struct bdi_writeback *wb) static void wb_inode_writeback_end(struct bdi_writeback *wb) { + unsigned long flags; atomic_dec(&wb->writeback_inodes); /* * Make sure estimate of writeback throughput gets updated after @@ -2900,7 +2902,10 @@ static void wb_inode_writeback_end(struct bdi_writeback *wb) * that if multiple inodes end writeback at a similar time, they get * batched into one bandwidth update. */ - queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL); + spin_lock_irqsave(&wb->work_lock, flags); + if (test_bit(WB_registered, &wb->state)) + queue_delayed_work(bdi_wq, &wb->bw_dwork, BANDWIDTH_INTERVAL); + spin_unlock_irqrestore(&wb->work_lock, flags); } bool __folio_end_writeback(struct folio *folio) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index e5486d47406e81c3c1f96551c1eded4551e1a38b..e20ade858e71c80f857f8044477cd4918c336258 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -482,6 +483,8 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) { static unsigned long prev_end_pfn, nr_initialised; + if (early_page_ext_enabled()) + return false; /* * prev_end_pfn static that contains the end of previous zone * No need to protect because called very early in boot before smp_init. @@ -542,7 +545,7 @@ static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn) #ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1); #else - pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); + pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn); #endif /* CONFIG_SPARSEMEM */ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; } @@ -870,7 +873,8 @@ static inline bool set_page_guard(struct zone *zone, struct page *page, INIT_LIST_HEAD(&page->buddy_list); set_page_private(page, order); /* Guard pages are not available for any usage */ - __mod_zone_freepage_state(zone, -(1 << order), migratetype); + if (!is_migrate_isolate(migratetype)) + __mod_zone_freepage_state(zone, -(1 << order), migratetype); return true; } @@ -900,7 +904,7 @@ static inline void clear_page_guard(struct zone *zone, struct page *page, * order of appearance. So we need to first gather the full picture of what was * enabled, and then make decisions. */ -void init_mem_debugging_and_hardening(void) +void __init init_mem_debugging_and_hardening(void) { bool page_poisoning_requested = false; @@ -935,6 +939,10 @@ void init_mem_debugging_and_hardening(void) else static_branch_disable(&init_on_free); + if (IS_ENABLED(CONFIG_KMSAN) && + (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) + pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); + #ifdef CONFIG_DEBUG_PAGEALLOC if (!debug_pagealloc_enabled()) return; @@ -1105,7 +1113,7 @@ static inline void __free_one_page(struct page *page, int migratetype, fpi_t fpi_flags) { struct capture_control *capc = task_capc(zone); - unsigned long buddy_pfn; + unsigned long buddy_pfn = 0; unsigned long combined_pfn; struct page *buddy; bool to_tail; @@ -1283,20 +1291,20 @@ static const char *page_bad_reason(struct page *page, unsigned long flags) return bad_reason; } -static void check_free_page_bad(struct page *page) +static void free_page_is_bad_report(struct page *page) { bad_page(page, page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE)); } -static inline int check_free_page(struct page *page) +static inline bool free_page_is_bad(struct page *page) { if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE))) - return 0; + return false; /* Something has gone sideways, find it */ - check_free_page_bad(page); - return 1; + free_page_is_bad_report(page); + return true; } static int free_tail_pages_check(struct page *head_page, struct page *page) @@ -1398,6 +1406,7 @@ static __always_inline bool free_pages_prepare(struct page *page, VM_BUG_ON_PAGE(PageTail(page), page); trace_mm_page_free(page, order); + kmsan_free_page(page, order); if (unlikely(PageHWPoison(page)) && !order) { /* @@ -1428,7 +1437,7 @@ static __always_inline bool free_pages_prepare(struct page *page, for (i = 1; i < (1 << order); i++) { if (compound) bad += free_tail_pages_check(page, page + i); - if (unlikely(check_free_page(page + i))) { + if (unlikely(free_page_is_bad(page + i))) { bad++; continue; } @@ -1439,8 +1448,8 @@ static __always_inline bool free_pages_prepare(struct page *page, page->mapping = NULL; if (memcg_kmem_enabled() && PageMemcgKmem(page)) __memcg_kmem_uncharge_page(page, order); - if (check_free) - bad += check_free_page(page); + if (check_free && free_page_is_bad(page)) + bad++; if (bad) return false; @@ -1499,10 +1508,11 @@ static bool free_pcp_prepare(struct page *page, unsigned int order) return free_pages_prepare(page, order, true, FPI_NONE); } +/* return true if this page has an inappropriate state */ static bool bulkfree_pcp_prepare(struct page *page) { if (debug_pagealloc_enabled_static()) - return check_free_page(page); + return free_page_is_bad(page); else return false; } @@ -1523,7 +1533,7 @@ static bool free_pcp_prepare(struct page *page, unsigned int order) static bool bulkfree_pcp_prepare(struct page *page) { - return check_free_page(page); + return free_page_is_bad(page); } #endif /* CONFIG_DEBUG_VM */ @@ -1575,7 +1585,6 @@ static void free_pcppages_bulk(struct zone *zone, int count, order = pindex_to_order(pindex); nr_pages = 1 << order; - BUILD_BUG_ON(MAX_ORDER >= (1<zone_start_pfn; unsigned long block_end_pfn; - block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages); + block_end_pfn = pageblock_end_pfn(block_start_pfn); for (; block_start_pfn < zone_end_pfn(zone); block_start_pfn = block_end_pfn, block_end_pfn += pageblock_nr_pages) { @@ -1890,15 +1903,14 @@ static void __init deferred_free_range(unsigned long pfn, page = pfn_to_page(pfn); /* Free a large naturally-aligned chunk if possible */ - if (nr_pages == pageblock_nr_pages && - (pfn & (pageblock_nr_pages - 1)) == 0) { + if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) { set_pageblock_migratetype(page, MIGRATE_MOVABLE); __free_pages_core(page, pageblock_order); return; } for (i = 0; i < nr_pages; i++, page++, pfn++) { - if ((pfn & (pageblock_nr_pages - 1)) == 0) + if (pageblock_aligned(pfn)) set_pageblock_migratetype(page, MIGRATE_MOVABLE); __free_pages_core(page, 0); } @@ -1917,16 +1929,12 @@ static inline void __init pgdat_init_report_one_done(void) /* * Returns true if page needs to be initialized or freed to buddy allocator. * - * First we check if pfn is valid on architectures where it is possible to have - * holes within pageblock_nr_pages. On systems where it is not possible, this - * function is optimized out. - * - * Then, we check if a current large page is valid by only checking the validity + * We check if a current large page is valid by only checking the validity * of the head pfn. */ static inline bool __init deferred_pfn_valid(unsigned long pfn) { - if (!(pfn & (pageblock_nr_pages - 1)) && !pfn_valid(pfn)) + if (pageblock_aligned(pfn) && !pfn_valid(pfn)) return false; return true; } @@ -1938,14 +1946,13 @@ static inline bool __init deferred_pfn_valid(unsigned long pfn) static void __init deferred_free_pages(unsigned long pfn, unsigned long end_pfn) { - unsigned long nr_pgmask = pageblock_nr_pages - 1; unsigned long nr_free = 0; for (; pfn < end_pfn; pfn++) { if (!deferred_pfn_valid(pfn)) { deferred_free_range(pfn - nr_free, nr_free); nr_free = 0; - } else if (!(pfn & nr_pgmask)) { + } else if (pageblock_aligned(pfn)) { deferred_free_range(pfn - nr_free, nr_free); nr_free = 1; } else { @@ -1965,7 +1972,6 @@ static unsigned long __init deferred_init_pages(struct zone *zone, unsigned long pfn, unsigned long end_pfn) { - unsigned long nr_pgmask = pageblock_nr_pages - 1; int nid = zone_to_nid(zone); unsigned long nr_pages = 0; int zid = zone_idx(zone); @@ -1975,7 +1981,7 @@ static unsigned long __init deferred_init_pages(struct zone *zone, if (!deferred_pfn_valid(pfn)) { page = NULL; continue; - } else if (!page || !(pfn & nr_pgmask)) { + } else if (!page || pageblock_aligned(pfn)) { page = pfn_to_page(pfn); } else { page++; @@ -2651,8 +2657,8 @@ int move_freepages_block(struct zone *zone, struct page *page, *num_movable = 0; pfn = page_to_pfn(page); - start_pfn = pfn & ~(pageblock_nr_pages - 1); - end_pfn = start_pfn + pageblock_nr_pages - 1; + start_pfn = pageblock_start_pfn(pfn); + end_pfn = pageblock_end_pfn(pfn) - 1; /* Do not cross zone boundaries */ if (!zone_spans_pfn(zone, start_pfn)) @@ -3010,7 +3016,7 @@ __rmqueue_fallback(struct zone *zone, int order, int start_migratetype, * i.e. orders < pageblock_order. If there are no local zones free, * the zonelists will be reiterated without ALLOC_NOFRAGMENT. */ - if (alloc_flags & ALLOC_NOFRAGMENT) + if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT) min_order = pageblock_order; /* @@ -3440,7 +3446,7 @@ static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp, int pindex; bool free_high; - __count_vm_event(PGFREE); + __count_vm_events(PGFREE, 1 << order); pindex = order_to_pindex(migratetype, order); list_add(&page->pcp_list, &pcp->lists[pindex]); pcp->count += 1 << order; @@ -3598,16 +3604,11 @@ EXPORT_SYMBOL_GPL(split_page); int __isolate_free_page(struct page *page, unsigned int order) { - unsigned long watermark; - struct zone *zone; - int mt; - - BUG_ON(!PageBuddy(page)); - - zone = page_zone(page); - mt = get_pageblock_migratetype(page); + struct zone *zone = page_zone(page); + int mt = get_pageblock_migratetype(page); if (!is_migrate_isolate(mt)) { + unsigned long watermark; /* * Obey watermarks as if the page was being allocated. We can * emulate a high-order watermark check with a raised order-0 @@ -3621,8 +3622,6 @@ int __isolate_free_page(struct page *page, unsigned int order) __mod_zone_freepage_state(zone, -(1UL << order), mt); } - /* Remove page from free list */ - del_page_from_free_list(page, zone, order); /* @@ -3643,7 +3642,6 @@ int __isolate_free_page(struct page *page, unsigned int order) } } - return 1UL << order; } @@ -3670,8 +3668,6 @@ void __putback_isolated_page(struct page *page, unsigned int order, int mt) /* * Update NUMA hit/miss statistics - * - * Must be called with interrupts disabled. */ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z, long nr_account) @@ -3777,8 +3773,7 @@ struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order, /* Lock and remove page from the per-cpu list */ static struct page *rmqueue_pcplist(struct zone *preferred_zone, struct zone *zone, unsigned int order, - gfp_t gfp_flags, int migratetype, - unsigned int alloc_flags) + int migratetype, unsigned int alloc_flags) { struct per_cpu_pages *pcp; struct list_head *list; @@ -3808,15 +3803,24 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, pcp_spin_unlock_irqrestore(pcp, flags); pcp_trylock_finish(UP_flags); if (page) { - __count_zid_vm_events(PGALLOC, page_zonenum(page), 1); + __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); zone_statistics(preferred_zone, zone, 1); } return page; } /* - * Allocate a page from the given zone. Use pcplists for order-0 allocations. + * Allocate a page from the given zone. + * Use pcplists for THP or "cheap" high-order allocations. + */ + +/* + * Do not instrument rmqueue() with KMSAN. This function may call + * __msan_poison_alloca() through a call to set_pfnblock_flags_mask(). + * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it + * may call rmqueue() again, which will result in a deadlock. */ +__no_sanitize_memory static inline struct page *rmqueue(struct zone *preferred_zone, struct zone *zone, unsigned int order, @@ -3839,7 +3843,7 @@ struct page *rmqueue(struct zone *preferred_zone, if (!IS_ENABLED(CONFIG_CMA) || alloc_flags & ALLOC_CMA || migratetype != MIGRATE_MOVABLE) { page = rmqueue_pcplist(preferred_zone, zone, order, - gfp_flags, migratetype, alloc_flags); + migratetype, alloc_flags); if (likely(page)) goto out; } @@ -4329,7 +4333,7 @@ static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask) if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM)) filter &= ~SHOW_MEM_FILTER_NODES; - show_mem(filter, nodemask); + __show_mem(filter, nodemask, gfp_zone(gfp_mask)); } void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) @@ -4708,6 +4712,30 @@ void fs_reclaim_release(gfp_t gfp_mask) EXPORT_SYMBOL_GPL(fs_reclaim_release); #endif +/* + * Zonelists may change due to hotplug during allocation. Detect when zonelists + * have been rebuilt so allocation retries. Reader side does not lock and + * retries the allocation if zonelist changes. Writer side is protected by the + * embedded spin_lock. + */ +static DEFINE_SEQLOCK(zonelist_update_seq); + +static unsigned int zonelist_iter_begin(void) +{ + if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) + return read_seqbegin(&zonelist_update_seq); + + return 0; +} + +static unsigned int check_retry_zonelist(unsigned int seq) +{ + if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE)) + return read_seqretry(&zonelist_update_seq, seq); + + return seq; +} + /* Perform direct synchronous page reclaim */ static unsigned long __perform_reclaim(gfp_t gfp_mask, unsigned int order, @@ -5001,6 +5029,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, int compaction_retries; int no_progress_loops; unsigned int cpuset_mems_cookie; + unsigned int zonelist_iter_cookie; int reserve_flags; /* @@ -5011,11 +5040,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM))) gfp_mask &= ~__GFP_ATOMIC; -retry_cpuset: +restart: compaction_retries = 0; no_progress_loops = 0; compact_priority = DEF_COMPACT_PRIORITY; cpuset_mems_cookie = read_mems_allowed_begin(); + zonelist_iter_cookie = zonelist_iter_begin(); /* * The fast path uses conservative alloc_flags to succeed only until @@ -5121,7 +5151,8 @@ retry: reserve_flags = __gfp_pfmemalloc_flags(gfp_mask); if (reserve_flags) - alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags); + alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) | + (alloc_flags & ALLOC_KSWAPD); /* * Reset the nodemask and zonelist iterators if memory policies can be @@ -5187,9 +5218,13 @@ retry: goto retry; - /* Deal with possible cpuset update races before we start OOM killing */ - if (check_retry_cpuset(cpuset_mems_cookie, ac)) - goto retry_cpuset; + /* + * Deal with possible cpuset update races or zonelist updates to avoid + * a unnecessary OOM kill. + */ + if (check_retry_cpuset(cpuset_mems_cookie, ac) || + check_retry_zonelist(zonelist_iter_cookie)) + goto restart; /* Reclaim has failed us, start killing things */ page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress); @@ -5209,9 +5244,13 @@ retry: } nopage: - /* Deal with possible cpuset update races before we fail */ - if (check_retry_cpuset(cpuset_mems_cookie, ac)) - goto retry_cpuset; + /* + * Deal with possible cpuset update races or zonelist updates to avoid + * a unnecessary OOM kill. + */ + if (check_retry_cpuset(cpuset_mems_cookie, ac) || + check_retry_zonelist(zonelist_iter_cookie)) + goto restart; /* * Make sure that __GFP_NOFAIL request doesn't leak out and make sure @@ -5238,7 +5277,7 @@ nopage: * so that we can identify them and convert them to something * else. */ - WARN_ON_ONCE_GFP(order > PAGE_ALLOC_COSTLY_ORDER, gfp_mask); + WARN_ON_ONCE_GFP(costly_order, gfp_mask); /* * Help non-failing allocations by giving them access to memory @@ -5535,6 +5574,7 @@ out: } trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype); + kmsan_alloc_page(page, order, alloc_gfp); return page; } @@ -5706,6 +5746,18 @@ refill: /* reset page count bias and offset to start of new frag */ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; offset = size - fragsz; + if (unlikely(offset < 0)) { + /* + * The caller is trying to allocate a fragment + * with fragsz > PAGE_SIZE but the cache isn't big + * enough to satisfy the request, this may + * happen in low memory conditions. + * We don't release the cache page because + * it could make memory pressure worse + * so we simply return NULL here. + */ + return NULL; + } } nc->pagecnt_bias--; @@ -6011,6 +6063,15 @@ static void show_migration_types(unsigned char type) printk(KERN_CONT "(%s) ", tmp); } +static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx) +{ + int zone_idx; + for (zone_idx = 0; zone_idx <= max_zone_idx; zone_idx++) + if (zone_managed_pages(pgdat->node_zones + zone_idx)) + return true; + return false; +} + /* * Show free area list (used inside shift_scroll-lock stuff) * We also calculate the percentage fragmentation. We do this by counting the @@ -6020,7 +6081,7 @@ static void show_migration_types(unsigned char type) * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's * cpuset. */ -void show_free_areas(unsigned int filter, nodemask_t *nodemask) +void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx) { unsigned long free_pcp = 0; int cpu, nid; @@ -6028,6 +6089,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) pg_data_t *pgdat; for_each_populated_zone(zone) { + if (zone_idx(zone) > max_zone_idx) + continue; if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) continue; @@ -6039,7 +6102,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " active_file:%lu inactive_file:%lu isolated_file:%lu\n" " unevictable:%lu dirty:%lu writeback:%lu\n" " slab_reclaimable:%lu slab_unreclaimable:%lu\n" - " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" + " mapped:%lu shmem:%lu pagetables:%lu\n" + " sec_pagetables:%lu bounce:%lu\n" " kernel_misc_reclaimable:%lu\n" " free:%lu free_pcp:%lu free_cma:%lu\n", global_node_page_state(NR_ACTIVE_ANON), @@ -6056,6 +6120,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) global_node_page_state(NR_FILE_MAPPED), global_node_page_state(NR_SHMEM), global_node_page_state(NR_PAGETABLE), + global_node_page_state(NR_SECONDARY_PAGETABLE), global_zone_page_state(NR_BOUNCE), global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE), global_zone_page_state(NR_FREE_PAGES), @@ -6065,6 +6130,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) for_each_online_pgdat(pgdat) { if (show_mem_node_skip(filter, pgdat->node_id, nodemask)) continue; + if (!node_has_managed_zones(pgdat, max_zone_idx)) + continue; printk("Node %d" " active_anon:%lukB" @@ -6089,6 +6156,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) " shadow_call_stack:%lukB" #endif " pagetables:%lukB" + " sec_pagetables:%lukB" " all_unreclaimable? %s" "\n", pgdat->node_id, @@ -6114,6 +6182,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) node_page_state(pgdat, NR_KERNEL_SCS_KB), #endif K(node_page_state(pgdat, NR_PAGETABLE)), + K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)), pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ? "yes" : "no"); } @@ -6121,6 +6190,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) for_each_populated_zone(zone) { int i; + if (zone_idx(zone) > max_zone_idx) + continue; if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) continue; @@ -6182,6 +6253,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) unsigned long nr[MAX_ORDER], flags, total = 0; unsigned char types[MAX_ORDER]; + if (zone_idx(zone) > max_zone_idx) + continue; if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask)) continue; show_node(zone); @@ -6507,16 +6580,15 @@ static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonesta #define BOOT_PAGESET_BATCH 1 static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset); static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats); -DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); +static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); static void __build_all_zonelists(void *data) { int nid; int __maybe_unused cpu; pg_data_t *self = data; - static DEFINE_SPINLOCK(lock); - spin_lock(&lock); + write_seqlock(&zonelist_update_seq); #ifdef CONFIG_NUMA memset(node_load, 0, sizeof(node_load)); @@ -6553,7 +6625,7 @@ static void __build_all_zonelists(void *data) #endif } - spin_unlock(&lock); + write_sequnlock(&zonelist_update_seq); } static noinline void __init @@ -6704,7 +6776,7 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone * such that unmovable allocations won't be scattered all * over the place during system boot. */ - if (IS_ALIGNED(pfn, pageblock_nr_pages)) { + if (pageblock_aligned(pfn)) { set_pageblock_migratetype(page, migratetype); cond_resched(); } @@ -6747,10 +6819,18 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, * Please note that MEMINIT_HOTPLUG path doesn't clear memmap * because this is done early in section_activate() */ - if (IS_ALIGNED(pfn, pageblock_nr_pages)) { + if (pageblock_aligned(pfn)) { set_pageblock_migratetype(page, MIGRATE_MOVABLE); cond_resched(); } + + /* + * ZONE_DEVICE pages are released directly to the driver page allocator + * which will set the page count to 1 when allocating the page. + */ + if (pgmap->type == MEMORY_DEVICE_PRIVATE || + pgmap->type == MEMORY_DEVICE_COHERENT) + set_page_count(page, 0); } /* @@ -6810,7 +6890,7 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long start = jiffies; int nid = pgdat->node_id; - if (WARN_ON_ONCE(!pgmap || zone_idx(zone) != ZONE_DEVICE)) + if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) return; /* @@ -6879,9 +6959,8 @@ static void __init init_unavailable_range(unsigned long spfn, u64 pgcnt = 0; for (pfn = spfn; pfn < epfn; pfn++) { - if (!pfn_valid(ALIGN_DOWN(pfn, pageblock_nr_pages))) { - pfn = ALIGN_DOWN(pfn, pageblock_nr_pages) - + pageblock_nr_pages - 1; + if (!pfn_valid(pageblock_start_pfn(pfn))) { + pfn = pageblock_end_pfn(pfn) - 1; continue; } __init_single_page(pfn_to_page(pfn), pfn, zone, node); @@ -6986,7 +7065,7 @@ static int zone_batchsize(struct zone *zone) * size is striking a balance between allocation latency * and zone lock contention. */ - batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE); + batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE); batch /= 4; /* We effectively *= 4 below */ if (batch < 1) batch = 1; @@ -7170,6 +7249,17 @@ void __meminit setup_zone_pageset(struct zone *zone) zone_set_pageset_high_and_batch(zone, 0); } +/* + * The zone indicated has a new number of managed_pages; batch sizes and percpu + * page high values need to be recalculated. + */ +static void zone_pcp_update(struct zone *zone, int cpu_online) +{ + mutex_lock(&pcp_batch_high_lock); + zone_set_pageset_high_and_batch(zone, cpu_online); + mutex_unlock(&pcp_batch_high_lock); +} + /* * Allocate per cpu pagesets and initialize them. * Before this call only boot pagesets were available. @@ -7614,6 +7704,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat) int i; pgdat_resize_init(pgdat); + pgdat_kswapd_lock_init(pgdat); pgdat_init_split_queue(pgdat); pgdat_init_kcompactd(pgdat); @@ -7908,17 +7999,6 @@ unsigned long __init node_map_pfn_alignment(void) return ~accl_mask + 1; } -/** - * find_min_pfn_with_active_regions - Find the minimum PFN registered - * - * Return: the minimum PFN based on information provided via - * memblock_set_node(). - */ -unsigned long __init find_min_pfn_with_active_regions(void) -{ - return PHYS_PFN(memblock_start_of_DRAM()); -} - /* * early_calculate_totalpages() * Sum pages in active regions for movable zone. @@ -8211,7 +8291,7 @@ void __init free_area_init(unsigned long *max_zone_pfn) memset(arch_zone_highest_possible_pfn, 0, sizeof(arch_zone_highest_possible_pfn)); - start_pfn = find_min_pfn_with_active_regions(); + start_pfn = PHYS_PFN(memblock_start_of_DRAM()); descending = arch_has_descending_max_zone_pfns(); for (i = 0; i < MAX_NR_ZONES; i++) { @@ -8460,8 +8540,8 @@ void __init mem_init_print_info(void) #endif ")\n", K(nr_free_pages()), K(physpages), - codesize >> 10, datasize >> 10, rosize >> 10, - (init_data_size + init_code_size) >> 10, bss_size >> 10, + codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, + (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, K(physpages - totalram_pages() - totalcma_pages), K(totalcma_pages) #ifdef CONFIG_HIGHMEM @@ -8974,7 +9054,7 @@ void *__init alloc_large_system_hash(const char *tablename, { unsigned long long max = high_limit; unsigned long log2qty, size; - void *table = NULL; + void *table; gfp_t gfp_flags; bool virt; bool huge; @@ -8986,8 +9066,8 @@ void *__init alloc_large_system_hash(const char *tablename, numentries -= arch_reserved_kernel_pages(); /* It isn't necessary when PAGE_SIZE >= 1MB */ - if (PAGE_SHIFT < 20) - numentries = round_up(numentries, (1<<20)/PAGE_SIZE); + if (PAGE_SIZE < SZ_1M) + numentries = round_up(numentries, SZ_1M / PAGE_SIZE); #if __BITS_PER_LONG > 32 if (!high_limit) { @@ -9411,17 +9491,6 @@ void free_contig_range(unsigned long pfn, unsigned long nr_pages) } EXPORT_SYMBOL(free_contig_range); -/* - * The zone indicated has a new number of managed_pages; batch sizes and percpu - * page high values need to be recalculated. - */ -void zone_pcp_update(struct zone *zone, int cpu_online) -{ - mutex_lock(&pcp_batch_high_lock); - zone_set_pageset_high_and_batch(zone, cpu_online); - mutex_unlock(&pcp_batch_high_lock); -} - /* * Effectively disable pcplists for the zone by setting the high limit to 0 * and draining all cpus. A concurrent page freeing on another CPU that's about @@ -9454,9 +9523,11 @@ void zone_pcp_reset(struct zone *zone) drain_zonestat(zone, pzstats); } free_percpu(zone->per_cpu_pageset); - free_percpu(zone->per_cpu_zonestats); zone->per_cpu_pageset = &boot_pageset; - zone->per_cpu_zonestats = &boot_zonestats; + if (zone->per_cpu_zonestats != &boot_zonestats) { + free_percpu(zone->per_cpu_zonestats); + zone->per_cpu_zonestats = &boot_zonestats; + } } } diff --git a/mm/page_counter.c b/mm/page_counter.c index eb156ff5d60303c9ce69977ff2bc7c3204fa1331..db20d6452b715cdfd143f8e1bdf085a406a8f094 100644 --- a/mm/page_counter.c +++ b/mm/page_counter.c @@ -17,24 +17,23 @@ static void propagate_protected_usage(struct page_counter *c, unsigned long usage) { unsigned long protected, old_protected; - unsigned long low, min; long delta; if (!c->parent) return; - min = READ_ONCE(c->min); - if (min || atomic_long_read(&c->min_usage)) { - protected = min(usage, min); + protected = min(usage, READ_ONCE(c->min)); + old_protected = atomic_long_read(&c->min_usage); + if (protected != old_protected) { old_protected = atomic_long_xchg(&c->min_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_min_usage); } - low = READ_ONCE(c->low); - if (low || atomic_long_read(&c->low_usage)) { - protected = min(usage, low); + protected = min(usage, READ_ONCE(c->low)); + old_protected = atomic_long_read(&c->low_usage); + if (protected != old_protected) { old_protected = atomic_long_xchg(&c->low_usage, protected); delta = protected - old_protected; if (delta) @@ -193,7 +192,7 @@ int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages) old = xchg(&counter->max, nr_pages); - if (page_counter_read(counter) <= usage) + if (page_counter_read(counter) <= usage || nr_pages >= old) return 0; counter->max = old; diff --git a/mm/page_ext.c b/mm/page_ext.c index 3dc715d7ac29ed5703150ea3813ef9b4bf750d5b..affe80243b6d66bdc4a7fb75f575177c7e839c36 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -9,6 +9,7 @@ #include #include #include +#include /* * struct page extension @@ -59,6 +60,10 @@ * can utilize this callback to initialize the state of it correctly. */ +#ifdef CONFIG_SPARSEMEM +#define PAGE_EXT_INVALID (0x1) +#endif + #if defined(CONFIG_PAGE_IDLE_FLAG) && !defined(CONFIG_64BIT) static bool need_page_idle(void) { @@ -84,6 +89,15 @@ static struct page_ext_operations *page_ext_ops[] __initdata = { unsigned long page_ext_size = sizeof(struct page_ext); static unsigned long total_usage; +static struct page_ext *lookup_page_ext(const struct page *page); + +bool early_page_ext; +static int __init setup_early_page_ext(char *str) +{ + early_page_ext = true; + return 0; +} +early_param("early_page_ext", setup_early_page_ext); static bool __init invoke_need_callbacks(void) { @@ -125,6 +139,48 @@ static inline struct page_ext *get_entry(void *base, unsigned long index) return base + page_ext_size * index; } +/** + * page_ext_get() - Get the extended information for a page. + * @page: The page we're interested in. + * + * Ensures that the page_ext will remain valid until page_ext_put() + * is called. + * + * Return: NULL if no page_ext exists for this page. + * Context: Any context. Caller may not sleep until they have called + * page_ext_put(). + */ +struct page_ext *page_ext_get(struct page *page) +{ + struct page_ext *page_ext; + + rcu_read_lock(); + page_ext = lookup_page_ext(page); + if (!page_ext) { + rcu_read_unlock(); + return NULL; + } + + return page_ext; +} + +/** + * page_ext_put() - Working with page extended information is done. + * @page_ext - Page extended information received from page_ext_get(). + * + * The page extended information of the page may not be valid after this + * function is called. + * + * Return: None. + * Context: Any context with corresponding page_ext_get() is called. + */ +void page_ext_put(struct page_ext *page_ext) +{ + if (unlikely(!page_ext)) + return; + + rcu_read_unlock(); +} #ifndef CONFIG_SPARSEMEM @@ -133,12 +189,13 @@ void __meminit pgdat_page_ext_init(struct pglist_data *pgdat) pgdat->node_page_ext = NULL; } -struct page_ext *lookup_page_ext(const struct page *page) +static struct page_ext *lookup_page_ext(const struct page *page) { unsigned long pfn = page_to_pfn(page); unsigned long index; struct page_ext *base; + WARN_ON_ONCE(!rcu_read_lock_held()); base = NODE_DATA(page_to_nid(page))->node_page_ext; /* * The sanity checks the page allocator does upon freeing a @@ -206,20 +263,27 @@ fail: } #else /* CONFIG_SPARSEMEM */ +static bool page_ext_invalid(struct page_ext *page_ext) +{ + return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID); +} -struct page_ext *lookup_page_ext(const struct page *page) +static struct page_ext *lookup_page_ext(const struct page *page) { unsigned long pfn = page_to_pfn(page); struct mem_section *section = __pfn_to_section(pfn); + struct page_ext *page_ext = READ_ONCE(section->page_ext); + + WARN_ON_ONCE(!rcu_read_lock_held()); /* * The sanity checks the page allocator does upon freeing a * page can reach here before the page_ext arrays are * allocated when feeding a range of pages to the allocator * for the first time during bootup or memory hotplug. */ - if (!section->page_ext) + if (page_ext_invalid(page_ext)) return NULL; - return get_entry(section->page_ext, pfn); + return get_entry(page_ext, pfn); } static void *__meminit alloc_page_ext(size_t size, int nid) @@ -298,9 +362,30 @@ static void __free_page_ext(unsigned long pfn) ms = __pfn_to_section(pfn); if (!ms || !ms->page_ext) return; - base = get_entry(ms->page_ext, pfn); + + base = READ_ONCE(ms->page_ext); + /* + * page_ext here can be valid while doing the roll back + * operation in online_page_ext(). + */ + if (page_ext_invalid(base)) + base = (void *)base - PAGE_EXT_INVALID; + WRITE_ONCE(ms->page_ext, NULL); + + base = get_entry(base, pfn); free_page_ext(base); - ms->page_ext = NULL; +} + +static void __invalidate_page_ext(unsigned long pfn) +{ + struct mem_section *ms; + void *val; + + ms = __pfn_to_section(pfn); + if (!ms || !ms->page_ext) + return; + val = (void *)ms->page_ext + PAGE_EXT_INVALID; + WRITE_ONCE(ms->page_ext, val); } static int __meminit online_page_ext(unsigned long start_pfn, @@ -336,13 +421,27 @@ static int __meminit online_page_ext(unsigned long start_pfn, } static int __meminit offline_page_ext(unsigned long start_pfn, - unsigned long nr_pages, int nid) + unsigned long nr_pages) { unsigned long start, end, pfn; start = SECTION_ALIGN_DOWN(start_pfn); end = SECTION_ALIGN_UP(start_pfn + nr_pages); + /* + * Freeing of page_ext is done in 3 steps to avoid + * use-after-free of it: + * 1) Traverse all the sections and mark their page_ext + * as invalid. + * 2) Wait for all the existing users of page_ext who + * started before invalidation to finish. + * 3) Free the page_ext. + */ + for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) + __invalidate_page_ext(pfn); + + synchronize_rcu(); + for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) __free_page_ext(pfn); return 0; @@ -362,11 +461,11 @@ static int __meminit page_ext_callback(struct notifier_block *self, break; case MEM_OFFLINE: offline_page_ext(mn->start_pfn, - mn->nr_pages, mn->status_change_nid); + mn->nr_pages); break; case MEM_CANCEL_ONLINE: offline_page_ext(mn->start_pfn, - mn->nr_pages, mn->status_change_nid); + mn->nr_pages); break; case MEM_GOING_OFFLINE: break; diff --git a/mm/page_io.c b/mm/page_io.c index 68318134dc923411ca530b868a33cf1a5e3c52e6..2af34dd8fa4db25fb60ed3c6fcce395eeb407f85 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -28,7 +28,7 @@ #include #include "swap.h" -void end_swap_bio_write(struct bio *bio) +static void end_swap_bio_write(struct bio *bio) { struct page *page = bio_first_page_all(bio); @@ -180,29 +180,30 @@ bad_bmap: */ int swap_writepage(struct page *page, struct writeback_control *wbc) { + struct folio *folio = page_folio(page); int ret = 0; - if (try_to_free_swap(page)) { - unlock_page(page); + if (folio_free_swap(folio)) { + folio_unlock(folio); goto out; } /* * Arch code may have to preserve more data than just the page * contents, e.g. memory tags. */ - ret = arch_prepare_to_swap(page); + ret = arch_prepare_to_swap(&folio->page); if (ret) { - set_page_dirty(page); - unlock_page(page); + folio_mark_dirty(folio); + folio_unlock(folio); goto out; } - if (frontswap_store(page) == 0) { - set_page_writeback(page); - unlock_page(page); - end_page_writeback(page); + if (frontswap_store(&folio->page) == 0) { + folio_start_writeback(folio); + folio_unlock(folio); + folio_end_writeback(folio); goto out; } - ret = __swap_writepage(page, wbc, end_swap_bio_write); + ret = __swap_writepage(&folio->page, wbc); out: return ret; } @@ -332,8 +333,7 @@ static int swap_writepage_fs(struct page *page, struct writeback_control *wbc) return 0; } -int __swap_writepage(struct page *page, struct writeback_control *wbc, - bio_end_io_t end_write_func) +int __swap_writepage(struct page *page, struct writeback_control *wbc) { struct bio *bio; int ret; @@ -358,7 +358,7 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc, REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc), GFP_NOIO); bio->bi_iter.bi_sector = swap_page_sector(page); - bio->bi_end_io = end_write_func; + bio->bi_end_io = end_swap_bio_write; bio_add_page(bio, page, thp_size(page), 0); bio_associate_blkg_from_page(bio, page); @@ -453,18 +453,21 @@ int swap_readpage(struct page *page, bool synchronous, struct swap_info_struct *sis = page_swap_info(page); bool workingset = PageWorkingset(page); unsigned long pflags; + bool in_thrashing; VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageUptodate(page), page); /* - * Count submission time as memory stall. When the device is congested, - * or the submitting cgroup IO-throttled, submission can be a - * significant part of overall IO time. + * Count submission time as memory stall and delay. When the device + * is congested, or the submitting cgroup IO-throttled, submission + * can be a significant part of overall IO time. */ - if (workingset) + if (workingset) { + delayacct_thrashing_start(&in_thrashing); psi_memstall_enter(&pflags); + } delayacct_swapin_start(); if (frontswap_load(page) == 0) { @@ -513,8 +516,10 @@ int swap_readpage(struct page *page, bool synchronous, bio_put(bio); out: - if (workingset) + if (workingset) { + delayacct_thrashing_end(&in_thrashing); psi_memstall_leave(&pflags); + } delayacct_swapin_end(); return ret; } diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 9d73dc38e3d75cdc6ec5a1efc3cbc178dbc89464..04141a9bea7049513286047fe312161fe522dbc9 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -37,8 +37,8 @@ static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long e struct zone *zone = page_zone(page); unsigned long pfn; - VM_BUG_ON(ALIGN_DOWN(start_pfn, pageblock_nr_pages) != - ALIGN_DOWN(end_pfn - 1, pageblock_nr_pages)); + VM_BUG_ON(pageblock_start_pfn(start_pfn) != + pageblock_start_pfn(end_pfn - 1)); if (is_migrate_cma_page(page)) { /* @@ -172,7 +172,7 @@ static int set_migratetype_isolate(struct page *page, int migratetype, int isol_ * to avoid redundant checks. */ check_unmovable_start = max(page_to_pfn(page), start_pfn); - check_unmovable_end = min(ALIGN(page_to_pfn(page) + 1, pageblock_nr_pages), + check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)), end_pfn); unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end, @@ -288,6 +288,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * @isolate_before: isolate the pageblock before the boundary_pfn * @skip_isolation: the flag to skip the pageblock isolation in second * isolate_single_pageblock() + * @migratetype: migrate type to set in error recovery. * * Free and in-use pages can be as big as MAX_ORDER-1 and contain more than one * pageblock. When not all pageblocks within a page are isolated at the same @@ -302,16 +303,16 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * the in-use page then splitting the free page. */ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, - gfp_t gfp_flags, bool isolate_before, bool skip_isolation) + gfp_t gfp_flags, bool isolate_before, bool skip_isolation, + int migratetype) { - unsigned char saved_mt; unsigned long start_pfn; unsigned long isolate_pageblock; unsigned long pfn; struct zone *zone; int ret; - VM_BUG_ON(!IS_ALIGNED(boundary_pfn, pageblock_nr_pages)); + VM_BUG_ON(!pageblock_aligned(boundary_pfn)); if (isolate_before) isolate_pageblock = boundary_pfn - pageblock_nr_pages; @@ -328,13 +329,13 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, start_pfn = max(ALIGN_DOWN(isolate_pageblock, MAX_ORDER_NR_PAGES), zone->zone_start_pfn); - saved_mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock)); + if (skip_isolation) { + int mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock)); - if (skip_isolation) - VM_BUG_ON(!is_migrate_isolate(saved_mt)); - else { - ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt, flags, - isolate_pageblock, isolate_pageblock + pageblock_nr_pages); + VM_BUG_ON(!is_migrate_isolate(mt)); + } else { + ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype, + flags, isolate_pageblock, isolate_pageblock + pageblock_nr_pages); if (ret) return ret; @@ -475,7 +476,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, failed: /* restore the original migratetype */ if (!skip_isolation) - unset_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt); + unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype); return -EBUSY; } @@ -531,13 +532,14 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, unsigned long pfn; struct page *page; /* isolation is done at page block granularity */ - unsigned long isolate_start = ALIGN_DOWN(start_pfn, pageblock_nr_pages); - unsigned long isolate_end = ALIGN(end_pfn, pageblock_nr_pages); + unsigned long isolate_start = pageblock_start_pfn(start_pfn); + unsigned long isolate_end = pageblock_align(end_pfn); int ret; bool skip_isolation = false; /* isolate [isolate_start, isolate_start + pageblock_nr_pages) pageblock */ - ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false, skip_isolation); + ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false, + skip_isolation, migratetype); if (ret) return ret; @@ -545,7 +547,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, skip_isolation = true; /* isolate [isolate_end - pageblock_nr_pages, isolate_end) pageblock */ - ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true, skip_isolation); + ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true, + skip_isolation, migratetype); if (ret) { unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype); return ret; @@ -576,9 +579,8 @@ void undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, { unsigned long pfn; struct page *page; - unsigned long isolate_start = ALIGN_DOWN(start_pfn, pageblock_nr_pages); - unsigned long isolate_end = ALIGN(end_pfn, pageblock_nr_pages); - + unsigned long isolate_start = pageblock_start_pfn(start_pfn); + unsigned long isolate_end = pageblock_align(end_pfn); for (pfn = isolate_start; pfn < isolate_end; diff --git a/mm/page_owner.c b/mm/page_owner.c index e4c6f3f1695bf2229cd732d46c5faa9a3b485070..2d27f532df4c1064530b8178a03830998288bb45 100644 --- a/mm/page_owner.c +++ b/mm/page_owner.c @@ -141,7 +141,7 @@ void __reset_page_owner(struct page *page, unsigned short order) struct page_owner *page_owner; u64 free_ts_nsec = local_clock(); - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); if (unlikely(!page_ext)) return; @@ -153,6 +153,7 @@ void __reset_page_owner(struct page *page, unsigned short order) page_owner->free_ts_nsec = free_ts_nsec; page_ext = page_ext_next(page_ext); } + page_ext_put(page_ext); } static inline void __set_page_owner_handle(struct page_ext *page_ext, @@ -183,19 +184,21 @@ static inline void __set_page_owner_handle(struct page_ext *page_ext, noinline void __set_page_owner(struct page *page, unsigned short order, gfp_t gfp_mask) { - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext; depot_stack_handle_t handle; + handle = save_stack(gfp_mask); + + page_ext = page_ext_get(page); if (unlikely(!page_ext)) return; - - handle = save_stack(gfp_mask); __set_page_owner_handle(page_ext, handle, order, gfp_mask); + page_ext_put(page_ext); } void __set_page_owner_migrate_reason(struct page *page, int reason) { - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext = page_ext_get(page); struct page_owner *page_owner; if (unlikely(!page_ext)) @@ -203,12 +206,13 @@ void __set_page_owner_migrate_reason(struct page *page, int reason) page_owner = get_page_owner(page_ext); page_owner->last_migrate_reason = reason; + page_ext_put(page_ext); } void __split_page_owner(struct page *page, unsigned int nr) { int i; - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext = page_ext_get(page); struct page_owner *page_owner; if (unlikely(!page_ext)) @@ -219,17 +223,25 @@ void __split_page_owner(struct page *page, unsigned int nr) page_owner->order = 0; page_ext = page_ext_next(page_ext); } + page_ext_put(page_ext); } void __folio_copy_owner(struct folio *newfolio, struct folio *old) { - struct page_ext *old_ext = lookup_page_ext(&old->page); - struct page_ext *new_ext = lookup_page_ext(&newfolio->page); + struct page_ext *old_ext; + struct page_ext *new_ext; struct page_owner *old_page_owner, *new_page_owner; - if (unlikely(!old_ext || !new_ext)) + old_ext = page_ext_get(&old->page); + if (unlikely(!old_ext)) return; + new_ext = page_ext_get(&newfolio->page); + if (unlikely(!new_ext)) { + page_ext_put(old_ext); + return; + } + old_page_owner = get_page_owner(old_ext); new_page_owner = get_page_owner(new_ext); new_page_owner->order = old_page_owner->order; @@ -254,6 +266,8 @@ void __folio_copy_owner(struct folio *newfolio, struct folio *old) */ __set_bit(PAGE_EXT_OWNER, &new_ext->flags); __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); + page_ext_put(new_ext); + page_ext_put(old_ext); } void pagetypeinfo_showmixedcount_print(struct seq_file *m, @@ -283,7 +297,7 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, continue; } - block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); + block_end_pfn = pageblock_end_pfn(pfn); block_end_pfn = min(block_end_pfn, end_pfn); pageblock_mt = get_pageblock_migratetype(page); @@ -307,12 +321,12 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, if (PageReserved(page)) continue; - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); if (unlikely(!page_ext)) continue; if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) - continue; + goto ext_put_continue; page_owner = get_page_owner(page_ext); page_mt = gfp_migratetype(page_owner->gfp_mask); @@ -323,9 +337,12 @@ void pagetypeinfo_showmixedcount_print(struct seq_file *m, count[pageblock_mt]++; pfn = block_end_pfn; + page_ext_put(page_ext); break; } pfn += (1UL << page_owner->order) - 1; +ext_put_continue: + page_ext_put(page_ext); } } @@ -435,7 +452,7 @@ err: void __dump_page_owner(const struct page *page) { - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext = page_ext_get((void *)page); struct page_owner *page_owner; depot_stack_handle_t handle; gfp_t gfp_mask; @@ -452,6 +469,7 @@ void __dump_page_owner(const struct page *page) if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { pr_alert("page_owner info is not present (never set?)\n"); + page_ext_put(page_ext); return; } @@ -482,6 +500,7 @@ void __dump_page_owner(const struct page *page) if (page_owner->last_migrate_reason != -1) pr_alert("page has been migrated, last migrate reason: %s\n", migrate_reason_names[page_owner->last_migrate_reason]); + page_ext_put(page_ext); } static ssize_t @@ -497,16 +516,24 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) return -EINVAL; page = NULL; - pfn = min_low_pfn + *ppos; - + if (*ppos == 0) + pfn = min_low_pfn; + else + pfn = *ppos; /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) pfn++; - drain_all_pages(NULL); - /* Find an allocated page */ for (; pfn < max_pfn; pfn++) { + /* + * This temporary page_owner is required so + * that we can avoid the context switches while holding + * the rcu lock and copying the page owner information to + * user through copy_to_user() or GFP_KERNEL allocations. + */ + struct page_owner page_owner_tmp; + /* * If the new page is in a new MAX_ORDER_NR_PAGES area, * validate the area as existing, skip it if not @@ -525,7 +552,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) continue; } - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); if (unlikely(!page_ext)) continue; @@ -534,14 +561,14 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) * because we don't hold the zone lock. */ if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) - continue; + goto ext_put_continue; /* * Although we do have the info about past allocation of free * pages, it's not relevant for current memory usage. */ if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) - continue; + goto ext_put_continue; page_owner = get_page_owner(page_ext); @@ -550,7 +577,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) * would inflate the stats. */ if (!IS_ALIGNED(pfn, 1 << page_owner->order)) - continue; + goto ext_put_continue; /* * Access to page_ext->handle isn't synchronous so we should @@ -558,18 +585,37 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) */ handle = READ_ONCE(page_owner->handle); if (!handle) - continue; + goto ext_put_continue; /* Record the next PFN to read in the file offset */ - *ppos = (pfn - min_low_pfn) + 1; + *ppos = pfn + 1; + page_owner_tmp = *page_owner; + page_ext_put(page_ext); return print_page_owner(buf, count, pfn, page, - page_owner, handle); + &page_owner_tmp, handle); +ext_put_continue: + page_ext_put(page_ext); } return 0; } +static loff_t lseek_page_owner(struct file *file, loff_t offset, int orig) +{ + switch (orig) { + case SEEK_SET: + file->f_pos = offset; + break; + case SEEK_CUR: + file->f_pos += offset; + break; + default: + return -EINVAL; + } + return file->f_pos; +} + static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) { unsigned long pfn = zone->zone_start_pfn; @@ -589,7 +635,7 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) continue; } - block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); + block_end_pfn = pageblock_end_pfn(pfn); block_end_pfn = min(block_end_pfn, end_pfn); for (; pfn < block_end_pfn; pfn++) { @@ -617,18 +663,20 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) if (PageReserved(page)) continue; - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); if (unlikely(!page_ext)) continue; /* Maybe overlapping zone */ if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) - continue; + goto ext_put_continue; /* Found early allocated page */ __set_page_owner_handle(page_ext, early_handle, 0, 0); count++; +ext_put_continue: + page_ext_put(page_ext); } cond_resched(); } @@ -660,6 +708,7 @@ static void init_early_allocated_pages(void) static const struct file_operations proc_page_owner_operations = { .read = read_page_owner, + .llseek = lseek_page_owner, }; static int __init pageowner_init(void) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index e2062748791affbd805baa1b635e42a8d17468aa..433dbce13fe1d27aecf914cb319246e4c19b831c 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -53,7 +53,7 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) } /* - * An enty is removed from the page table, decrement the counters for that page + * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. */ static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, @@ -68,7 +68,7 @@ static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, return; page = pfn_to_page(pfn); - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); anon = PageAnon(page); for (i = 0; i < pgcnt; i++) { @@ -83,10 +83,11 @@ static void page_table_check_clear(struct mm_struct *mm, unsigned long addr, } page_ext = page_ext_next(page_ext); } + page_ext_put(page_ext); } /* - * A new enty is added to the page table, increment the counters for that page + * A new entry is added to the page table, increment the counters for that page * verify that it is of correct type and is not being mapped with a different * type to a different process. */ @@ -103,7 +104,7 @@ static void page_table_check_set(struct mm_struct *mm, unsigned long addr, return; page = pfn_to_page(pfn); - page_ext = lookup_page_ext(page); + page_ext = page_ext_get(page); anon = PageAnon(page); for (i = 0; i < pgcnt; i++) { @@ -118,6 +119,7 @@ static void page_table_check_set(struct mm_struct *mm, unsigned long addr, } page_ext = page_ext_next(page_ext); } + page_ext_put(page_ext); } /* @@ -126,9 +128,10 @@ static void page_table_check_set(struct mm_struct *mm, unsigned long addr, */ void __page_table_check_zero(struct page *page, unsigned int order) { - struct page_ext *page_ext = lookup_page_ext(page); + struct page_ext *page_ext; unsigned long i; + page_ext = page_ext_get(page); BUG_ON(!page_ext); for (i = 0; i < (1ul << order); i++) { struct page_table_check *ptc = get_page_table_check(page_ext); @@ -137,6 +140,7 @@ void __page_table_check_zero(struct page *page, unsigned int order) BUG_ON(atomic_read(&ptc->file_map_count)); page_ext = page_ext_next(page_ext); } + page_ext_put(page_ext); } void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 8e9e574d535aaf02b7cdbac42cdf3ad25447a45b..93e13fc17d3cbf402906497debc04041cec4d2d6 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -86,7 +86,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw) !is_device_exclusive_entry(entry)) return false; - pfn = swp_offset(entry); + pfn = swp_offset_pfn(entry); } else if (is_swap_pte(*pvmw->pte)) { swp_entry_t entry; @@ -96,7 +96,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw) !is_device_exclusive_entry(entry)) return false; - pfn = swp_offset(entry); + pfn = swp_offset_pfn(entry); } else { if (!pte_present(*pvmw->pte)) return false; @@ -221,7 +221,7 @@ restart: return not_found(pvmw); entry = pmd_to_swp_entry(pmde); if (!is_migration_entry(entry) || - !check_pmd(swp_offset(entry), pvmw)) + !check_pmd(swp_offset_pfn(entry), pvmw)) return not_found(pvmw); return true; } diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 9b3db11a4d1db85e4b9f2c771c21aa84584729c8..2ff3a5bebcebb88778259a8814bb6d17cc83c878 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -110,7 +110,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, do { again: next = pmd_addr_end(addr, end); - if (pmd_none(*pmd) || (!walk->vma && !walk->no_vma)) { + if (pmd_none(*pmd)) { if (ops->pte_hole) err = ops->pte_hole(addr, next, depth, walk); if (err) @@ -171,7 +171,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, do { again: next = pud_addr_end(addr, end); - if (pud_none(*pud) || (!walk->vma && !walk->no_vma)) { + if (pud_none(*pud)) { if (ops->pte_hole) err = ops->pte_hole(addr, next, depth, walk); if (err) @@ -366,19 +366,19 @@ static int __walk_page_range(unsigned long start, unsigned long end, struct vm_area_struct *vma = walk->vma; const struct mm_walk_ops *ops = walk->ops; - if (vma && ops->pre_vma) { + if (ops->pre_vma) { err = ops->pre_vma(start, end, walk); if (err) return err; } - if (vma && is_vm_hugetlb_page(vma)) { + if (is_vm_hugetlb_page(vma)) { if (ops->hugetlb_entry) err = walk_hugetlb_range(start, end, walk); } else err = walk_pgd_range(start, end, walk); - if (vma && ops->post_vma) + if (ops->post_vma) ops->post_vma(walk); return err; @@ -450,13 +450,17 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, if (!vma) { /* after the last vma */ walk.vma = NULL; next = end; + if (ops->pte_hole) + err = ops->pte_hole(start, next, -1, &walk); } else if (start < vma->vm_start) { /* outside vma */ walk.vma = NULL; next = min(end, vma->vm_start); + if (ops->pte_hole) + err = ops->pte_hole(start, next, -1, &walk); } else { /* inside vma */ walk.vma = vma; next = min(end, vma->vm_end); - vma = vma->vm_next; + vma = find_vma(mm, vma->vm_end); err = walk_page_test(start, next, &walk); if (err > 0) { @@ -470,16 +474,23 @@ int walk_page_range(struct mm_struct *mm, unsigned long start, } if (err < 0) break; - } - if (walk.vma || walk.ops->pte_hole) err = __walk_page_range(start, next, &walk); + } if (err) break; } while (start = next, start < end); return err; } -/* +/** + * walk_page_range_novma - walk a range of pagetables not backed by a vma + * @mm: mm_struct representing the target process of page table walk + * @start: start address of the virtual address range + * @end: end address of the virtual address range + * @ops: operation to call during the walk + * @pgd: pgd to walk if different from mm->pgd + * @private: private data for callbacks' usage + * * Similar to walk_page_range() but can walk any page tables even if they are * not backed by VMAs. Because 'unusual' entries may be walked this function * will also not lock the PTEs for the pte_entry() callback. This is useful for @@ -501,9 +512,9 @@ int walk_page_range_novma(struct mm_struct *mm, unsigned long start, if (start >= end || !walk.mm) return -EINVAL; - mmap_assert_locked(walk.mm); + mmap_assert_write_locked(walk.mm); - return __walk_page_range(start, end, &walk); + return walk_pgd_range(start, end, &walk); } int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops, diff --git a/mm/ptdump.c b/mm/ptdump.c index eea3d28d173c26b5b63f8e1340f8c0df2e417ce2..8adab455a68b3c1adab99d480c7430fcf9011b30 100644 --- a/mm/ptdump.c +++ b/mm/ptdump.c @@ -152,13 +152,13 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) { const struct ptdump_range *range = st->range; - mmap_read_lock(mm); + mmap_write_lock(mm); while (range->start != range->end) { walk_page_range_novma(mm, range->start, range->end, &ptdump_ops, pgd, st); range++; } - mmap_read_unlock(mm); + mmap_write_unlock(mm); /* Flush out the last page */ st->note_page(st, 0, -1, 0); diff --git a/mm/readahead.c b/mm/readahead.c index fdcd28cbd92ded7a19e45d2737bcf96649a2f91b..b10f0cf81d804ea62aba61ce46b1b1ff60d95a9c 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -122,6 +122,7 @@ #include #include #include +#include #include #include #include @@ -152,6 +153,8 @@ static void read_pages(struct readahead_control *rac) if (!readahead_count(rac)) return; + if (unlikely(rac->_workingset)) + psi_memstall_enter(&rac->_pflags); blk_start_plug(&plug); if (aops->readahead) { @@ -179,6 +182,9 @@ static void read_pages(struct readahead_control *rac) } blk_finish_plug(&plug); + if (unlikely(rac->_workingset)) + psi_memstall_leave(&rac->_pflags); + rac->_workingset = false; BUG_ON(readahead_count(rac)); } @@ -252,6 +258,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, } if (i == nr_to_read - lookahead_size) folio_set_readahead(folio); + ractl->_workingset |= folio_test_workingset(folio); ractl->_nr_pages++; } @@ -480,11 +487,14 @@ static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index, if (index == mark) folio_set_readahead(folio); err = filemap_add_folio(ractl->mapping, folio, index, gfp); - if (err) + if (err) { folio_put(folio); - else - ractl->_nr_pages += 1UL << order; - return err; + return err; + } + + ractl->_nr_pages += 1UL << order; + ractl->_workingset |= folio_test_workingset(folio); + return 0; } void page_cache_ra_order(struct readahead_control *ractl, @@ -826,6 +836,10 @@ void readahead_expand(struct readahead_control *ractl, put_page(page); return; } + if (unlikely(PageWorkingset(page)) && !ractl->_workingset) { + ractl->_workingset = true; + psi_memstall_enter(&ractl->_pflags); + } ractl->_nr_pages++; if (ra) { ra->size++; diff --git a/mm/rmap.c b/mm/rmap.c index edc06c52bc82e7fa831678322e70a7701d7e75de..2ec925e5fa6a9cdb0e954ac21362a632fa452062 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -23,10 +23,9 @@ * inode->i_rwsem (while writing or truncating, not reading or faulting) * mm->mmap_lock * mapping->invalidate_lock (in filemap_fault) - * page->flags PG_locked (lock_page) * (see hugetlbfs below) - * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) + * page->flags PG_locked (lock_page) + * hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share, see hugetlbfs below) * mapping->i_mmap_rwsem - * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) * anon_vma->rwsem * mm->page_table_lock or pte_lock * swap_lock (in swap_duplicate, swap_info_get) @@ -46,10 +45,11 @@ * ->tasklist_lock * pte map lock * - * * hugetlbfs PageHuge() pages take locks in this order: - * mapping->i_mmap_rwsem - * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) - * page->flags PG_locked (lock_page) + * hugetlbfs PageHuge() take locks in this order: + * hugetlb_fault_mutex (hugetlbfs specific page fault mutex) + * vma_lock (hugetlb specific lock for pmd_sharing) + * mapping->i_mmap_rwsem (also used for hugetlb pmd sharing) + * page->flags PG_locked (lock_page) */ #include @@ -93,7 +93,8 @@ static inline struct anon_vma *anon_vma_alloc(void) anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); - anon_vma->degree = 1; /* Reference for first vma */ + anon_vma->num_children = 0; + anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; /* * Initialise the anon_vma root to point to itself. If called @@ -201,6 +202,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = anon_vma_alloc(); if (unlikely(!anon_vma)) goto out_enomem_free_avc; + anon_vma->num_children++; /* self-parent link for new root */ allocated = anon_vma; } @@ -210,8 +212,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; anon_vma_chain_link(vma, avc, anon_vma); - /* vma reference or self-parent link for new root */ - anon_vma->degree++; + anon_vma->num_active_vmas++; allocated = NULL; avc = NULL; } @@ -296,19 +297,19 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) anon_vma_chain_link(dst, avc, anon_vma); /* - * Reuse existing anon_vma if its degree lower than two, - * that means it has no vma and only one anon_vma child. + * Reuse existing anon_vma if it has no vma and only one + * anon_vma child. * - * Do not choose parent anon_vma, otherwise first child - * will always reuse it. Root anon_vma is never reused: + * Root anon_vma is never reused: * it has self-parent reference and at least one child. */ if (!dst->anon_vma && src->anon_vma && - anon_vma != src->anon_vma && anon_vma->degree < 2) + anon_vma->num_children < 2 && + anon_vma->num_active_vmas == 0) dst->anon_vma = anon_vma; } if (dst->anon_vma) - dst->anon_vma->degree++; + dst->anon_vma->num_active_vmas++; unlock_anon_vma_root(root); return 0; @@ -358,6 +359,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) anon_vma = anon_vma_alloc(); if (!anon_vma) goto out_error; + anon_vma->num_active_vmas++; avc = anon_vma_chain_alloc(GFP_KERNEL); if (!avc) goto out_error_free_anon_vma; @@ -378,7 +380,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) vma->anon_vma = anon_vma; anon_vma_lock_write(anon_vma); anon_vma_chain_link(vma, avc, anon_vma); - anon_vma->parent->degree++; + anon_vma->parent->num_children++; anon_vma_unlock_write(anon_vma); return 0; @@ -410,7 +412,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) * to free them outside the lock. */ if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) { - anon_vma->parent->degree--; + anon_vma->parent->num_children--; continue; } @@ -418,7 +420,7 @@ void unlink_anon_vmas(struct vm_area_struct *vma) anon_vma_chain_free(avc); } if (vma->anon_vma) { - vma->anon_vma->degree--; + vma->anon_vma->num_active_vmas--; /* * vma would still be needed after unlink, and anon_vma will be prepared @@ -436,7 +438,8 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - VM_WARN_ON(anon_vma->degree); + VM_WARN_ON(anon_vma->num_children); + VM_WARN_ON(anon_vma->num_active_vmas); put_anon_vma(anon_vma); list_del(&avc->same_vma); @@ -486,16 +489,16 @@ void __init anon_vma_init(void) * if there is a mapcount, we can dereference the anon_vma after observing * those. */ -struct anon_vma *page_get_anon_vma(struct page *page) +struct anon_vma *folio_get_anon_vma(struct folio *folio) { struct anon_vma *anon_vma = NULL; unsigned long anon_mapping; rcu_read_lock(); - anon_mapping = (unsigned long)READ_ONCE(page->mapping); + anon_mapping = (unsigned long)READ_ONCE(folio->mapping); if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) goto out; - if (!page_mapped(page)) + if (!folio_mapped(folio)) goto out; anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); @@ -505,13 +508,13 @@ struct anon_vma *page_get_anon_vma(struct page *page) } /* - * If this page is still mapped, then its anon_vma cannot have been + * If this folio is still mapped, then its anon_vma cannot have been * freed. But if it has been unmapped, we have no security against the * anon_vma structure being freed and reused (for another anon_vma: * SLAB_TYPESAFE_BY_RCU guarantees that - so the atomic_inc_not_zero() * above cannot corrupt). */ - if (!page_mapped(page)) { + if (!folio_mapped(folio)) { rcu_read_unlock(); put_anon_vma(anon_vma); return NULL; @@ -523,11 +526,11 @@ out: } /* - * Similar to page_get_anon_vma() except it locks the anon_vma. + * Similar to folio_get_anon_vma() except it locks the anon_vma. * * Its a little more complex as it tries to keep the fast path to a single * atomic op -- the trylock. If we fail the trylock, we fall back to getting a - * reference like with page_get_anon_vma() and then block on the mutex + * reference like with folio_get_anon_vma() and then block on the mutex * on !rwc->try_lock case. */ struct anon_vma *folio_lock_anon_vma_read(struct folio *folio, @@ -599,11 +602,6 @@ out: return anon_vma; } -void page_unlock_anon_vma_read(struct anon_vma *anon_vma) -{ - anon_vma_unlock_read(anon_vma); -} - #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH /* * Flush TLB entries for recently unmapped pages from remote CPUs. It is @@ -767,13 +765,17 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) return vma_address(page, vma); } +/* + * Returns the actual pmd_t* where we expect 'address' to be mapped from, or + * NULL if it doesn't exist. No guarantees / checks on what the pmd_t* + * represents. + */ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd = NULL; - pmd_t pmde; pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) @@ -788,15 +790,6 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) goto out; pmd = pmd_offset(pud, address); - /* - * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() - * without holding anon_vma lock for write. So when looking for a - * genuine pmde (in which to find pte), test present and !THP together. - */ - pmde = *pmd; - barrier(); - if (!pmd_present(pmde) || pmd_trans_huge(pmde)) - pmd = NULL; out: return pmd; } @@ -830,6 +823,12 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte) { + if (lru_gen_enabled() && pte_young(*pvmw.pte) && + !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) { + lru_gen_look_around(&pvmw); + referenced++; + } + if (ptep_clear_flush_young_notify(vma, address, pvmw.pte)) { /* @@ -1098,22 +1097,20 @@ int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, */ void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) { - struct anon_vma *anon_vma = vma->anon_vma; - struct page *subpage = page; - - page = compound_head(page); + void *anon_vma = vma->anon_vma; + struct folio *folio = page_folio(page); - VM_BUG_ON_PAGE(!PageLocked(page), page); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_VMA(!anon_vma, vma); - anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; + anon_vma += PAGE_MAPPING_ANON; /* * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written * simultaneously, so a concurrent reader (eg folio_referenced()'s * folio_test_anon()) will not see one without the other. */ - WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); - SetPageAnonExclusive(subpage); + WRITE_ONCE(folio->mapping, anon_vma); + SetPageAnonExclusive(page); } /** @@ -1557,33 +1554,45 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * To call huge_pmd_unshare, i_mmap_rwsem must be * held in write mode. Caller needs to explicitly * do this outside rmap routines. + * + * We also must hold hugetlb vma_lock in write mode. + * Lock order dictates acquiring vma_lock BEFORE + * i_mmap_rwsem. We can only try lock here and fail + * if unsuccessful. */ - VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED)); - if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) { - flush_tlb_range(vma, range.start, range.end); - mmu_notifier_invalidate_range(mm, range.start, - range.end); - - /* - * The ref count of the PMD page was dropped - * which is part of the way map counting - * is done for shared PMDs. Return 'true' - * here. When there is no other sharing, - * huge_pmd_unshare returns false and we will - * unmap the actual page and drop map count - * to zero. - */ - page_vma_mapped_walk_done(&pvmw); - break; + if (!anon) { + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + page_vma_mapped_walk_done(&pvmw); + ret = false; + break; + } + if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + flush_tlb_range(vma, + range.start, range.end); + mmu_notifier_invalidate_range(mm, + range.start, range.end); + /* + * The ref count of the PMD page was + * dropped which is part of the way map + * counting is done for shared PMDs. + * Return 'true' here. When there is + * no other sharing, huge_pmd_unshare + * returns false and we will unmap the + * actual page and drop map count + * to zero. + */ + page_vma_mapped_walk_done(&pvmw); + break; + } + hugetlb_vma_unlock_write(vma); } pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); } else { flush_cache_page(vma, address, pte_pfn(*pvmw.pte)); - /* - * Nuke the page table entry. When having to clear - * PageAnonExclusive(), we always have to flush. - */ - if (should_defer_flush(mm, flags) && !anon_exclusive) { + /* Nuke the page table entry. */ + if (should_defer_flush(mm, flags)) { /* * We clear the PTE but do not flush so potentially * a remote CPU could still be writing to the folio. @@ -1714,6 +1723,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, page_vma_mapped_walk_done(&pvmw); break; } + + /* See page_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && page_try_share_anon_rmap(subpage)) { swap_free(entry); @@ -1933,26 +1944,41 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * To call huge_pmd_unshare, i_mmap_rwsem must be * held in write mode. Caller needs to explicitly * do this outside rmap routines. + * + * We also must hold hugetlb vma_lock in write mode. + * Lock order dictates acquiring vma_lock BEFORE + * i_mmap_rwsem. We can only try lock here and + * fail if unsuccessful. */ - VM_BUG_ON(!anon && !(flags & TTU_RMAP_LOCKED)); - if (!anon && huge_pmd_unshare(mm, vma, address, pvmw.pte)) { - flush_tlb_range(vma, range.start, range.end); - mmu_notifier_invalidate_range(mm, range.start, - range.end); - - /* - * The ref count of the PMD page was dropped - * which is part of the way map counting - * is done for shared PMDs. Return 'true' - * here. When there is no other sharing, - * huge_pmd_unshare returns false and we will - * unmap the actual page and drop map count - * to zero. - */ - page_vma_mapped_walk_done(&pvmw); - break; + if (!anon) { + VM_BUG_ON(!(flags & TTU_RMAP_LOCKED)); + if (!hugetlb_vma_trylock_write(vma)) { + page_vma_mapped_walk_done(&pvmw); + ret = false; + break; + } + if (huge_pmd_unshare(mm, vma, address, pvmw.pte)) { + hugetlb_vma_unlock_write(vma); + flush_tlb_range(vma, + range.start, range.end); + mmu_notifier_invalidate_range(mm, + range.start, range.end); + + /* + * The ref count of the PMD page was + * dropped which is part of the way map + * counting is done for shared PMDs. + * Return 'true' here. When there is + * no other sharing, huge_pmd_unshare + * returns false and we will unmap the + * actual page and drop map count + * to zero. + */ + page_vma_mapped_walk_done(&pvmw); + break; + } + hugetlb_vma_unlock_write(vma); } - /* Nuke the hugetlb page table entry */ pteval = huge_ptep_clear_flush(vma, address, pvmw.pte); } else { @@ -2045,6 +2071,8 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, } VM_BUG_ON_PAGE(pte_write(pteval) && folio_test_anon(folio) && !anon_exclusive, subpage); + + /* See page_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && page_try_share_anon_rmap(subpage)) { if (folio_test_hugetlb(folio)) @@ -2070,7 +2098,10 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, else entry = make_readable_migration_entry( page_to_pfn(subpage)); - + if (pte_young(pteval)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pteval)) + entry = make_migration_entry_dirty(entry); swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); diff --git a/mm/rodata_test.c b/mm/rodata_test.c index 2613371945b7ec6f8f1ce24ad378ba539f4d9bc0..6d783436951f324f769c28ad6091c9a62bf85690 100644 --- a/mm/rodata_test.c +++ b/mm/rodata_test.c @@ -9,13 +9,13 @@ #include #include +#include #include static const int rodata_test_data = 0xC3; void rodata_test(void) { - unsigned long start, end; int zero = 0; /* test 1: read the value */ @@ -39,13 +39,11 @@ void rodata_test(void) } /* test 4: check if the rodata section is PAGE_SIZE aligned */ - start = (unsigned long)__start_rodata; - end = (unsigned long)__end_rodata; - if (start & (PAGE_SIZE - 1)) { + if (!PAGE_ALIGNED(__start_rodata)) { pr_err("start of .rodata is not page size aligned\n"); return; } - if (end & (PAGE_SIZE - 1)) { + if (!PAGE_ALIGNED(__end_rodata)) { pr_err("end of .rodata is not page size aligned\n"); return; } diff --git a/mm/secretmem.c b/mm/secretmem.c index e3e9590c6fb30e41e5c54a31d896e92a05077a9a..04c3ac9448a1883d490729a60f493c9b2caa2ac5 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -276,20 +276,18 @@ static struct file_system_type secretmem_fs = { .kill_sb = kill_anon_super, }; -static int secretmem_init(void) +static int __init secretmem_init(void) { - int ret = 0; - if (!secretmem_enable) - return ret; + return 0; secretmem_mnt = kern_mount(&secretmem_fs); if (IS_ERR(secretmem_mnt)) - ret = PTR_ERR(secretmem_mnt); + return PTR_ERR(secretmem_mnt); /* prevent secretmem mappings from ever getting PROT_EXEC */ secretmem_mnt->mnt_flags |= MNT_NOEXEC; - return ret; + return 0; } fs_initcall(secretmem_init); diff --git a/mm/shmem.c b/mm/shmem.c index 5783f11351bb0806c3828a4ada7690b8d5343552..8280a5cb48dfc813e9a84df70fddb3bbcadabc28 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "swap.h" static struct vfsmount *shm_mnt; @@ -139,17 +140,6 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, struct folio **foliop, enum sgp_type sgp, gfp_t gfp, struct vm_area_struct *vma, vm_fault_t *fault_type); -static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp, - gfp_t gfp, struct vm_area_struct *vma, - struct vm_fault *vmf, vm_fault_t *fault_type); - -int shmem_getpage(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp) -{ - return shmem_getpage_gfp(inode, index, pagep, sgp, - mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); -} static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) { @@ -190,7 +180,7 @@ static inline int shmem_reacct_size(unsigned long flags, /* * ... whereas tmpfs objects are accounted incrementally as * pages are allocated, in order to allow large sparse files. - * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, + * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM, * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. */ static inline int shmem_acct_block(unsigned long flags, long pages) @@ -472,20 +462,22 @@ static bool shmem_confirm_swap(struct address_space *mapping, static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; -bool shmem_is_huge(struct vm_area_struct *vma, - struct inode *inode, pgoff_t index) +bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, + pgoff_t index, bool shmem_huge_force) { loff_t i_size; if (!S_ISREG(inode->i_mode)) return false; - if (shmem_huge == SHMEM_HUGE_DENY) - return false; if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))) return false; + if (shmem_huge_force) + return true; if (shmem_huge == SHMEM_HUGE_FORCE) return true; + if (shmem_huge == SHMEM_HUGE_DENY) + return false; switch (SHMEM_SB(inode->i_sb)->huge) { case SHMEM_HUGE_ALWAYS: @@ -629,7 +621,7 @@ next: goto move_back; } - ret = split_huge_page(&folio->page); + ret = split_folio(folio); folio_unlock(folio); folio_put(folio); @@ -680,8 +672,8 @@ static long shmem_unused_huge_count(struct super_block *sb, #define shmem_huge SHMEM_HUGE_DENY -bool shmem_is_huge(struct vm_area_struct *vma, - struct inode *inode, pgoff_t index) +bool shmem_is_huge(struct vm_area_struct *vma, struct inode *inode, + pgoff_t index, bool shmem_huge_force) { return false; } @@ -763,23 +755,22 @@ error: } /* - * Like delete_from_page_cache, but substitutes swap for page. + * Like delete_from_page_cache, but substitutes swap for @folio. */ -static void shmem_delete_from_page_cache(struct page *page, void *radswap) +static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) { - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; + long nr = folio_nr_pages(folio); int error; - VM_BUG_ON_PAGE(PageCompound(page), page); - xa_lock_irq(&mapping->i_pages); - error = shmem_replace_entry(mapping, page->index, page, radswap); - page->mapping = NULL; - mapping->nrpages--; - __dec_lruvec_page_state(page, NR_FILE_PAGES); - __dec_lruvec_page_state(page, NR_SHMEM); + error = shmem_replace_entry(mapping, folio->index, folio, radswap); + folio->mapping = NULL; + mapping->nrpages -= nr; + __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); + __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); xa_unlock_irq(&mapping->i_pages); - put_page(page); + folio_put(folio); BUG_ON(error); } @@ -886,10 +877,9 @@ void shmem_unlock_mapping(struct address_space *mapping) static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) { struct folio *folio; - struct page *page; /* - * At first avoid shmem_getpage(,,,SGP_READ): that fails + * At first avoid shmem_get_folio(,,,SGP_READ): that fails * beyond i_size, and reports fallocated pages as holes. */ folio = __filemap_get_folio(inode->i_mapping, index, @@ -900,9 +890,9 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) * But read a page back from swap if any of it is within i_size * (although in some cases this is just a waste of time). */ - page = NULL; - shmem_getpage(inode, index, &page, SGP_READ); - return page ? page_folio(page) : NULL; + folio = NULL; + shmem_get_folio(inode, index, &folio, SGP_READ); + return folio; } /* @@ -1043,6 +1033,7 @@ void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) { shmem_undo_range(inode, lstart, lend, false); inode->i_ctime = inode->i_mtime = current_time(inode); + inode_inc_iversion(inode); } EXPORT_SYMBOL_GPL(shmem_truncate_range); @@ -1069,7 +1060,7 @@ static int shmem_getattr(struct user_namespace *mnt_userns, STATX_ATTR_NODUMP); generic_fillattr(&init_user_ns, inode, stat); - if (shmem_is_huge(NULL, inode, 0)) + if (shmem_is_huge(NULL, inode, 0, false)) stat->blksize = HPAGE_PMD_SIZE; if (request_mask & STATX_BTIME) { @@ -1087,6 +1078,8 @@ static int shmem_setattr(struct user_namespace *mnt_userns, struct inode *inode = d_inode(dentry); struct shmem_inode_info *info = SHMEM_I(inode); int error; + bool update_mtime = false; + bool update_ctime = true; error = setattr_prepare(&init_user_ns, dentry, attr); if (error) @@ -1107,7 +1100,9 @@ static int shmem_setattr(struct user_namespace *mnt_userns, if (error) return error; i_size_write(inode, newsize); - inode->i_ctime = inode->i_mtime = current_time(inode); + update_mtime = true; + } else { + update_ctime = false; } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); @@ -1127,6 +1122,12 @@ static int shmem_setattr(struct user_namespace *mnt_userns, setattr_copy(&init_user_ns, inode, attr); if (attr->ia_valid & ATTR_MODE) error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); + if (!error && update_ctime) { + inode->i_ctime = current_time(inode); + if (update_mtime) + inode->i_mtime = inode->i_ctime; + inode_inc_iversion(inode); + } return error; } @@ -1328,17 +1329,18 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, * and its shmem_writeback() needs them to be split when swapping. */ - if (PageTransCompound(page)) { + if (folio_test_large(folio)) { /* Ensure the subpages are still dirty */ - SetPageDirty(page); + folio_test_set_dirty(folio); if (split_huge_page(page) < 0) goto redirty; - ClearPageDirty(page); + folio = page_folio(page); + folio_clear_dirty(folio); } - BUG_ON(!PageLocked(page)); - mapping = page->mapping; - index = page->index; + BUG_ON(!folio_test_locked(folio)); + mapping = folio->mapping; + index = folio->index; inode = mapping->host; info = SHMEM_I(inode); if (info->flags & VM_LOCKED) @@ -1361,15 +1363,15 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) /* * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC * value into swapfile.c, the only way we can correctly account for a - * fallocated page arriving here is now to initialize it and write it. + * fallocated folio arriving here is now to initialize it and write it. * - * That's okay for a page already fallocated earlier, but if we have + * That's okay for a folio already fallocated earlier, but if we have * not yet completed the fallocation, then (a) we want to keep track - * of this page in case we have to undo it, and (b) it may not be a + * of this folio in case we have to undo it, and (b) it may not be a * good idea to continue anyway, once we're pushing into swap. So - * reactivate the page, and let shmem_fallocate() quit when too many. + * reactivate the folio, and let shmem_fallocate() quit when too many. */ - if (!PageUptodate(page)) { + if (!folio_test_uptodate(folio)) { if (inode->i_private) { struct shmem_falloc *shmem_falloc; spin_lock(&inode->i_lock); @@ -1385,9 +1387,9 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) if (shmem_falloc) goto redirty; } - clear_highpage(page); - flush_dcache_page(page); - SetPageUptodate(page); + folio_zero_range(folio, 0, folio_size(folio)); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); } swap = folio_alloc_swap(folio); @@ -1396,7 +1398,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) /* * Add inode to shmem_unuse()'s list of swapped-out inodes, - * if it's not already there. Do it now before the page is + * if it's not already there. Do it now before the folio is * moved to swap cache, when its pagelock no longer protects * the inode from eviction. But don't unlock the mutex until * we've incremented swapped, because shmem_unuse_inode() will @@ -1406,7 +1408,7 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) if (list_empty(&info->swaplist)) list_add(&info->swaplist, &shmem_swaplist); - if (add_to_swap_cache(page, swap, + if (add_to_swap_cache(folio, swap, __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, NULL) == 0) { spin_lock_irq(&info->lock); @@ -1415,21 +1417,21 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) spin_unlock_irq(&info->lock); swap_shmem_alloc(swap); - shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); + shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap)); mutex_unlock(&shmem_swaplist_mutex); - BUG_ON(page_mapped(page)); - swap_writepage(page, wbc); + BUG_ON(folio_mapped(folio)); + swap_writepage(&folio->page, wbc); return 0; } mutex_unlock(&shmem_swaplist_mutex); - put_swap_page(page, swap); + put_swap_folio(folio, swap); redirty: - set_page_dirty(page); + folio_mark_dirty(folio); if (wbc->for_reclaim) - return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ - unlock_page(page); + return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ + folio_unlock(folio); return 0; } @@ -1486,7 +1488,7 @@ static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) mpol_cond_put(vma->vm_policy); } -static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, +static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp, struct shmem_inode_info *info, pgoff_t index) { struct vm_area_struct pvma; @@ -1499,7 +1501,9 @@ static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, page = swap_cluster_readahead(swap, gfp, &vmf); shmem_pseudo_vma_destroy(&pvma); - return page; + if (!page) + return NULL; + return page_folio(page); } /* @@ -1560,12 +1564,6 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, return folio; } -static struct page *shmem_alloc_page(gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) -{ - return &shmem_alloc_folio(gfp, info, index)->page; -} - static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode, pgoff_t index, bool huge) { @@ -1599,7 +1597,7 @@ failed: /* * When a page is moved from swapcache to shmem filecache (either by the - * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of + * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of * shmem_unuse_inode()), it may have been read in earlier from swap, in * ignorance of the mapping it belongs to. If that mapping has special * constraints (like the gma500 GEM driver, which requires RAM below 4GB), @@ -1614,52 +1612,52 @@ static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp) return folio_zonenum(folio) > gfp_zone(gfp); } -static int shmem_replace_page(struct page **pagep, gfp_t gfp, +static int shmem_replace_folio(struct folio **foliop, gfp_t gfp, struct shmem_inode_info *info, pgoff_t index) { - struct page *oldpage, *newpage; struct folio *old, *new; struct address_space *swap_mapping; swp_entry_t entry; pgoff_t swap_index; int error; - oldpage = *pagep; - entry.val = page_private(oldpage); + old = *foliop; + entry = folio_swap_entry(old); swap_index = swp_offset(entry); - swap_mapping = page_mapping(oldpage); + swap_mapping = swap_address_space(entry); /* * We have arrived here because our zones are constrained, so don't * limit chance of success by further cpuset and node constraints. */ gfp &= ~GFP_CONSTRAINT_MASK; - newpage = shmem_alloc_page(gfp, info, index); - if (!newpage) + VM_BUG_ON_FOLIO(folio_test_large(old), old); + new = shmem_alloc_folio(gfp, info, index); + if (!new) return -ENOMEM; - get_page(newpage); - copy_highpage(newpage, oldpage); - flush_dcache_page(newpage); + folio_get(new); + folio_copy(new, old); + flush_dcache_folio(new); - __SetPageLocked(newpage); - __SetPageSwapBacked(newpage); - SetPageUptodate(newpage); - set_page_private(newpage, entry.val); - SetPageSwapCache(newpage); + __folio_set_locked(new); + __folio_set_swapbacked(new); + folio_mark_uptodate(new); + folio_set_swap_entry(new, entry); + folio_set_swapcache(new); /* * Our caller will very soon move newpage out of swapcache, but it's * a nice clean interface for us to replace oldpage by newpage there. */ xa_lock_irq(&swap_mapping->i_pages); - error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage); + error = shmem_replace_entry(swap_mapping, swap_index, old, new); if (!error) { - old = page_folio(oldpage); - new = page_folio(newpage); mem_cgroup_migrate(old, new); - __inc_lruvec_page_state(newpage, NR_FILE_PAGES); - __dec_lruvec_page_state(oldpage, NR_FILE_PAGES); + __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1); + __lruvec_stat_mod_folio(new, NR_SHMEM, 1); + __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1); + __lruvec_stat_mod_folio(old, NR_SHMEM, -1); } xa_unlock_irq(&swap_mapping->i_pages); @@ -1669,18 +1667,17 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, * both PageSwapCache and page_private after getting page lock; * but be defensive. Reverse old to newpage for clear and free. */ - oldpage = newpage; + old = new; } else { - lru_cache_add(newpage); - *pagep = newpage; + folio_add_lru(new); + *foliop = new; } - ClearPageSwapCache(oldpage); - set_page_private(oldpage, 0); + folio_clear_swapcache(old); + old->private = NULL; - unlock_page(oldpage); - put_page(oldpage); - put_page(oldpage); + folio_unlock(old); + folio_put_refs(old, 2); return error; } @@ -1728,7 +1725,6 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, struct address_space *mapping = inode->i_mapping; struct shmem_inode_info *info = SHMEM_I(inode); struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL; - struct page *page; struct folio *folio = NULL; swp_entry_t swap; int error; @@ -1741,8 +1737,8 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, return -EIO; /* Look it up and read it in.. */ - page = lookup_swap_cache(swap, NULL, 0); - if (!page) { + folio = swap_cache_get_folio(swap, NULL, 0); + if (!folio) { /* Or update major stats only when swapin succeeds?? */ if (fault_type) { *fault_type |= VM_FAULT_MAJOR; @@ -1750,13 +1746,12 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, count_memcg_event_mm(charge_mm, PGMAJFAULT); } /* Here we actually start the io */ - page = shmem_swapin(swap, gfp, info, index); - if (!page) { + folio = shmem_swapin(swap, gfp, info, index); + if (!folio) { error = -ENOMEM; goto failed; } } - folio = page_folio(page); /* We have to do this with folio locked to prevent races */ folio_lock(folio); @@ -1779,7 +1774,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index, arch_swap_restore(swap, folio); if (shmem_should_replace_folio(folio, gfp)) { - error = shmem_replace_page(&page, gfp, info, index); + error = shmem_replace_folio(&folio, gfp, info, index); if (error) goto failed; } @@ -1819,7 +1814,7 @@ unlock: } /* - * shmem_getpage_gfp - find page in cache, or get from swap, or allocate + * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate * * If we allocate a new one we do not mark it dirty. That's up to the * vm. If we swap it in we mark it dirty since we also free the swap @@ -1828,10 +1823,10 @@ unlock: * vma, vmf, and fault_type are only supplied by shmem_fault: * otherwise they are NULL. */ -static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp, gfp_t gfp, - struct vm_area_struct *vma, struct vm_fault *vmf, - vm_fault_t *fault_type) +static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, + struct folio **foliop, enum sgp_type sgp, gfp_t gfp, + struct vm_area_struct *vma, struct vm_fault *vmf, + vm_fault_t *fault_type) { struct address_space *mapping = inode->i_mapping; struct shmem_inode_info *info = SHMEM_I(inode); @@ -1871,7 +1866,7 @@ repeat: if (error == -EEXIST) goto repeat; - *pagep = &folio->page; + *foliop = folio; return error; } @@ -1881,7 +1876,7 @@ repeat: folio_mark_accessed(folio); if (folio_test_uptodate(folio)) goto out; - /* fallocated page */ + /* fallocated folio */ if (sgp != SGP_READ) goto clear; folio_unlock(folio); @@ -1889,10 +1884,10 @@ repeat: } /* - * SGP_READ: succeed on hole, with NULL page, letting caller zero. - * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail. + * SGP_READ: succeed on hole, with NULL folio, letting caller zero. + * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail. */ - *pagep = NULL; + *foliop = NULL; if (sgp == SGP_READ) return 0; if (sgp == SGP_NOALLOC) @@ -1907,7 +1902,7 @@ repeat: return 0; } - if (!shmem_is_huge(vma, inode, index)) + if (!shmem_is_huge(vma, inode, index, false)) goto alloc_nohuge; huge_gfp = vma_thp_gfp_mask(vma); @@ -1925,7 +1920,7 @@ alloc_nohuge: if (error != -ENOSPC) goto unlock; /* - * Try to reclaim some space by splitting a huge page + * Try to reclaim some space by splitting a large folio * beyond i_size on the filesystem. */ while (retry--) { @@ -1961,9 +1956,9 @@ alloc_nohuge: if (folio_test_pmd_mappable(folio) && DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < - hindex + HPAGE_PMD_NR - 1) { + folio_next_index(folio) - 1) { /* - * Part of the huge page is beyond i_size: subject + * Part of the large folio is beyond i_size: subject * to shrink under memory pressure. */ spin_lock(&sbinfo->shrinklist_lock); @@ -1980,14 +1975,14 @@ alloc_nohuge: } /* - * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. + * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio. */ if (sgp == SGP_FALLOC) sgp = SGP_WRITE; clear: /* - * Let SGP_WRITE caller clear ends if write does not fill page; - * but SGP_FALLOC on a page fallocated earlier must initialize + * Let SGP_WRITE caller clear ends if write does not fill folio; + * but SGP_FALLOC on a folio fallocated earlier must initialize * it now, lest undo on failure cancel our earlier guarantee. */ if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) { @@ -2013,7 +2008,7 @@ clear: goto unlock; } out: - *pagep = folio_page(folio, index - hindex); + *foliop = folio; return 0; /* @@ -2043,6 +2038,13 @@ unlock: return error; } +int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop, + enum sgp_type sgp) +{ + return shmem_get_folio_gfp(inode, index, foliop, sgp, + mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); +} + /* * This is like autoremove_wake_function, but it removes the wait queue * entry unconditionally - even if something else had already woken the @@ -2060,6 +2062,7 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct inode *inode = file_inode(vma->vm_file); gfp_t gfp = mapping_gfp_mask(inode->i_mapping); + struct folio *folio = NULL; int err; vm_fault_t ret = VM_FAULT_LOCKED; @@ -2122,10 +2125,12 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) spin_unlock(&inode->i_lock); } - err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE, + err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE, gfp, vma, vmf, &ret); if (err) return vmf_error(err); + if (folio) + vmf->page = folio_file_page(folio, vmf->pgoff); return ret; } @@ -2281,16 +2286,34 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } -/* Mask out flags that are inappropriate for the given type of inode. */ -static unsigned shmem_mask_flags(umode_t mode, __u32 flags) +#ifdef CONFIG_TMPFS_XATTR +static int shmem_initxattrs(struct inode *, const struct xattr *, void *); + +/* + * chattr's fsflags are unrelated to extended attributes, + * but tmpfs has chosen to enable them under the same config option. + */ +static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) +{ + unsigned int i_flags = 0; + + if (fsflags & FS_NOATIME_FL) + i_flags |= S_NOATIME; + if (fsflags & FS_APPEND_FL) + i_flags |= S_APPEND; + if (fsflags & FS_IMMUTABLE_FL) + i_flags |= S_IMMUTABLE; + /* + * But FS_NODUMP_FL does not require any action in i_flags. + */ + inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE); +} +#else +static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) { - if (S_ISDIR(mode)) - return flags; - else if (S_ISREG(mode)) - return flags & SHMEM_REG_FLMASK; - else - return flags & SHMEM_OTHER_FLMASK; } +#define shmem_initxattrs NULL +#endif static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir, umode_t mode, dev_t dev, unsigned long flags) @@ -2309,7 +2332,7 @@ static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir, inode_init_owner(&init_user_ns, inode, dir, mode); inode->i_blocks = 0; inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); - inode->i_generation = prandom_u32(); + inode->i_generation = get_random_u32(); info = SHMEM_I(inode); memset(info, 0, (char *)inode - (char *)info); spin_lock_init(&info->lock); @@ -2319,7 +2342,8 @@ static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir, info->i_crtime = inode->i_mtime; info->fsflags = (dir == NULL) ? 0 : SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; - info->fsflags = shmem_mask_flags(mode, info->fsflags); + if (info->fsflags) + shmem_set_inode_flags(inode, info->fsflags); INIT_LIST_HEAD(&info->shrinklist); INIT_LIST_HEAD(&info->swaplist); simple_xattrs_init(&info->xattrs); @@ -2376,7 +2400,6 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); void *page_kaddr; struct folio *folio; - struct page *page; int ret; pgoff_t max_off; @@ -2395,53 +2418,53 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, if (!*pagep) { ret = -ENOMEM; - page = shmem_alloc_page(gfp, info, pgoff); - if (!page) + folio = shmem_alloc_folio(gfp, info, pgoff); + if (!folio) goto out_unacct_blocks; if (!zeropage) { /* COPY */ - page_kaddr = kmap_atomic(page); + page_kaddr = kmap_local_folio(folio, 0); ret = copy_from_user(page_kaddr, (const void __user *)src_addr, PAGE_SIZE); - kunmap_atomic(page_kaddr); + kunmap_local(page_kaddr); /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { - *pagep = page; + *pagep = &folio->page; ret = -ENOENT; /* don't free the page */ goto out_unacct_blocks; } - flush_dcache_page(page); + flush_dcache_folio(folio); } else { /* ZEROPAGE */ - clear_user_highpage(page, dst_addr); + clear_user_highpage(&folio->page, dst_addr); } } else { - page = *pagep; + folio = page_folio(*pagep); + VM_BUG_ON_FOLIO(folio_test_large(folio), folio); *pagep = NULL; } - VM_BUG_ON(PageLocked(page)); - VM_BUG_ON(PageSwapBacked(page)); - __SetPageLocked(page); - __SetPageSwapBacked(page); - __SetPageUptodate(page); + VM_BUG_ON(folio_test_locked(folio)); + VM_BUG_ON(folio_test_swapbacked(folio)); + __folio_set_locked(folio); + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); ret = -EFAULT; max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); if (unlikely(pgoff >= max_off)) goto out_release; - folio = page_folio(page); ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp & GFP_RECLAIM_MASK, dst_mm); if (ret) goto out_release; ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, - page, true, wp_copy); + &folio->page, true, wp_copy); if (ret) goto out_delete_from_cache; @@ -2451,13 +2474,13 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, shmem_recalc_inode(inode); spin_unlock_irq(&info->lock); - unlock_page(page); + folio_unlock(folio); return 0; out_delete_from_cache: - delete_from_page_cache(page); + filemap_remove_folio(folio); out_release: - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); out_unacct_blocks: shmem_inode_unacct_blocks(inode, 1); return ret; @@ -2468,12 +2491,6 @@ out_unacct_blocks: static const struct inode_operations shmem_symlink_inode_operations; static const struct inode_operations shmem_short_symlink_operations; -#ifdef CONFIG_TMPFS_XATTR -static int shmem_initxattrs(struct inode *, const struct xattr *, void *); -#else -#define shmem_initxattrs NULL -#endif - static int shmem_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, @@ -2482,6 +2499,7 @@ shmem_write_begin(struct file *file, struct address_space *mapping, struct inode *inode = mapping->host; struct shmem_inode_info *info = SHMEM_I(inode); pgoff_t index = pos >> PAGE_SHIFT; + struct folio *folio; int ret = 0; /* i_rwsem is held by caller */ @@ -2493,14 +2511,15 @@ shmem_write_begin(struct file *file, struct address_space *mapping, return -EPERM; } - ret = shmem_getpage(inode, index, pagep, SGP_WRITE); + ret = shmem_get_folio(inode, index, &folio, SGP_WRITE); if (ret) return ret; + *pagep = folio_file_page(folio, index); if (PageHWPoison(*pagep)) { - unlock_page(*pagep); - put_page(*pagep); + folio_unlock(folio); + folio_put(folio); *pagep = NULL; return -EIO; } @@ -2559,6 +2578,7 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) offset = *ppos & ~PAGE_MASK; for (;;) { + struct folio *folio = NULL; struct page *page = NULL; pgoff_t end_index; unsigned long nr, ret; @@ -2573,17 +2593,18 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) break; } - error = shmem_getpage(inode, index, &page, SGP_READ); + error = shmem_get_folio(inode, index, &folio, SGP_READ); if (error) { if (error == -EINVAL) error = 0; break; } - if (page) { - unlock_page(page); + if (folio) { + folio_unlock(folio); + page = folio_file_page(folio, index); if (PageHWPoison(page)) { - put_page(page); + folio_put(folio); error = -EIO; break; } @@ -2599,14 +2620,14 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) if (index == end_index) { nr = i_size & ~PAGE_MASK; if (nr <= offset) { - if (page) - put_page(page); + if (folio) + folio_put(folio); break; } } nr -= offset; - if (page) { + if (folio) { /* * If users can be writing to this page using arbitrary * virtual addresses, take care about potential aliasing @@ -2618,13 +2639,13 @@ static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) * Mark the page accessed if we read the beginning. */ if (!offset) - mark_page_accessed(page); + folio_mark_accessed(folio); /* * Ok, we have the page, and it's up-to-date, so * now we can copy it to user space... */ ret = copy_page_to_iter(page, offset, nr, to); - put_page(page); + folio_put(folio); } else if (user_backed_iter(to)) { /* @@ -2767,7 +2788,7 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, info->fallocend = end; for (index = start; index < end; ) { - struct page *page; + struct folio *folio; /* * Good, the fallocate(2) manpage permits EINTR: we may have @@ -2778,10 +2799,11 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) error = -ENOMEM; else - error = shmem_getpage(inode, index, &page, SGP_FALLOC); + error = shmem_get_folio(inode, index, &folio, + SGP_FALLOC); if (error) { info->fallocend = undo_fallocend; - /* Remove the !PageUptodate pages we added */ + /* Remove the !uptodate folios we added */ if (index > start) { shmem_undo_range(inode, (loff_t)start << PAGE_SHIFT, @@ -2790,48 +2812,46 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, goto undone; } - index++; /* * Here is a more important optimization than it appears: - * a second SGP_FALLOC on the same huge page will clear it, - * making it PageUptodate and un-undoable if we fail later. + * a second SGP_FALLOC on the same large folio will clear it, + * making it uptodate and un-undoable if we fail later. */ - if (PageTransCompound(page)) { - index = round_up(index, HPAGE_PMD_NR); - /* Beware 32-bit wraparound */ - if (!index) - index--; - } + index = folio_next_index(folio); + /* Beware 32-bit wraparound */ + if (!index) + index--; /* * Inform shmem_writepage() how far we have reached. * No need for lock or barrier: we have the page lock. */ - if (!PageUptodate(page)) + if (!folio_test_uptodate(folio)) shmem_falloc.nr_falloced += index - shmem_falloc.next; shmem_falloc.next = index; /* - * If !PageUptodate, leave it that way so that freeable pages + * If !uptodate, leave it that way so that freeable folios * can be recognized if we need to rollback on error later. - * But set_page_dirty so that memory pressure will swap rather - * than free the pages we are allocating (and SGP_CACHE pages + * But mark it dirty so that memory pressure will swap rather + * than free the folios we are allocating (and SGP_CACHE folios * might still be clean: we now need to mark those dirty too). */ - set_page_dirty(page); - unlock_page(page); - put_page(page); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); cond_resched(); } if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) i_size_write(inode, offset + len); - inode->i_ctime = current_time(inode); undone: spin_lock(&inode->i_lock); inode->i_private = NULL; spin_unlock(&inode->i_lock); out: + if (!error) + file_modified(file); inode_unlock(inode); return error; } @@ -2884,6 +2904,7 @@ shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, error = 0; dir->i_size += BOGO_DIRENT_SIZE; dir->i_ctime = dir->i_mtime = current_time(dir); + inode_inc_iversion(dir); d_instantiate(dentry, inode); dget(dentry); /* Extra count - pin the dentry in core */ } @@ -2895,7 +2916,7 @@ out_iput: static int shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, - struct dentry *dentry, umode_t mode) + struct file *file, umode_t mode) { struct inode *inode; int error = -ENOSPC; @@ -2910,9 +2931,9 @@ shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, error = simple_acl_create(dir, inode); if (error) goto out_iput; - d_tmpfile(dentry, inode); + d_tmpfile(file, inode); } - return error; + return finish_open_simple(file, error); out_iput: iput(inode); return error; @@ -2959,6 +2980,7 @@ static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentr dir->i_size += BOGO_DIRENT_SIZE; inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); + inode_inc_iversion(dir); inc_nlink(inode); ihold(inode); /* New dentry reference */ dget(dentry); /* Extra pinning count for the created dentry */ @@ -2976,6 +2998,7 @@ static int shmem_unlink(struct inode *dir, struct dentry *dentry) dir->i_size -= BOGO_DIRENT_SIZE; inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); + inode_inc_iversion(dir); drop_nlink(inode); dput(dentry); /* Undo the count from "create" - this does all the work */ return 0; @@ -3065,6 +3088,8 @@ static int shmem_rename2(struct user_namespace *mnt_userns, old_dir->i_ctime = old_dir->i_mtime = new_dir->i_ctime = new_dir->i_mtime = inode->i_ctime = current_time(old_dir); + inode_inc_iversion(old_dir); + inode_inc_iversion(new_dir); return 0; } @@ -3074,7 +3099,7 @@ static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, int error; int len; struct inode *inode; - struct page *page; + struct folio *folio; len = strlen(symname) + 1; if (len > PAGE_SIZE) @@ -3102,21 +3127,22 @@ static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, inode->i_op = &shmem_short_symlink_operations; } else { inode_nohighmem(inode); - error = shmem_getpage(inode, 0, &page, SGP_WRITE); + error = shmem_get_folio(inode, 0, &folio, SGP_WRITE); if (error) { iput(inode); return error; } inode->i_mapping->a_ops = &shmem_aops; inode->i_op = &shmem_symlink_inode_operations; - memcpy(page_address(page), symname, len); - SetPageUptodate(page); - set_page_dirty(page); - unlock_page(page); - put_page(page); + memcpy(folio_address(folio), symname, len); + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); } dir->i_size += BOGO_DIRENT_SIZE; dir->i_ctime = dir->i_mtime = current_time(dir); + inode_inc_iversion(dir); d_instantiate(dentry, inode); dget(dentry); return 0; @@ -3124,40 +3150,41 @@ static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, static void shmem_put_link(void *arg) { - mark_page_accessed(arg); - put_page(arg); + folio_mark_accessed(arg); + folio_put(arg); } static const char *shmem_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { - struct page *page = NULL; + struct folio *folio = NULL; int error; + if (!dentry) { - page = find_get_page(inode->i_mapping, 0); - if (!page) + folio = filemap_get_folio(inode->i_mapping, 0); + if (!folio) return ERR_PTR(-ECHILD); - if (PageHWPoison(page) || - !PageUptodate(page)) { - put_page(page); + if (PageHWPoison(folio_page(folio, 0)) || + !folio_test_uptodate(folio)) { + folio_put(folio); return ERR_PTR(-ECHILD); } } else { - error = shmem_getpage(inode, 0, &page, SGP_READ); + error = shmem_get_folio(inode, 0, &folio, SGP_READ); if (error) return ERR_PTR(error); - if (!page) + if (!folio) return ERR_PTR(-ECHILD); - if (PageHWPoison(page)) { - unlock_page(page); - put_page(page); + if (PageHWPoison(folio_page(folio, 0))) { + folio_unlock(folio); + folio_put(folio); return ERR_PTR(-ECHILD); } - unlock_page(page); + folio_unlock(folio); } - set_delayed_call(done, shmem_put_link, page); - return page_address(page); + set_delayed_call(done, shmem_put_link, folio); + return folio_address(folio); } #ifdef CONFIG_TMPFS_XATTR @@ -3179,19 +3206,15 @@ static int shmem_fileattr_set(struct user_namespace *mnt_userns, if (fileattr_has_fsx(fa)) return -EOPNOTSUPP; + if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE) + return -EOPNOTSUPP; info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) | (fa->flags & SHMEM_FL_USER_MODIFIABLE); - inode->i_flags &= ~(S_APPEND | S_IMMUTABLE | S_NOATIME); - if (info->fsflags & FS_APPEND_FL) - inode->i_flags |= S_APPEND; - if (info->fsflags & FS_IMMUTABLE_FL) - inode->i_flags |= S_IMMUTABLE; - if (info->fsflags & FS_NOATIME_FL) - inode->i_flags |= S_NOATIME; - + shmem_set_inode_flags(inode, info->fsflags); inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); return 0; } @@ -3255,9 +3278,15 @@ static int shmem_xattr_handler_set(const struct xattr_handler *handler, size_t size, int flags) { struct shmem_inode_info *info = SHMEM_I(inode); + int err; name = xattr_full_name(handler, name); - return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); + err = simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); + if (!err) { + inode->i_ctime = current_time(inode); + inode_inc_iversion(inode); + } + return err; } static const struct xattr_handler shmem_security_xattr_handler = { @@ -3720,7 +3749,7 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_flags |= SB_NOUSER; } sb->s_export_op = &shmem_export_ops; - sb->s_flags |= SB_NOSEC; + sb->s_flags |= SB_NOSEC | SB_I_VERSION; #else sb->s_flags |= SB_NOUSER; #endif @@ -4254,18 +4283,20 @@ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, { #ifdef CONFIG_SHMEM struct inode *inode = mapping->host; + struct folio *folio; struct page *page; int error; BUG_ON(!shmem_mapping(mapping)); - error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, + error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE, gfp, NULL, NULL, NULL); if (error) return ERR_PTR(error); - unlock_page(page); + folio_unlock(folio); + page = folio_file_page(folio, index); if (PageHWPoison(page)) { - put_page(page); + folio_put(folio); return ERR_PTR(-EIO); } diff --git a/mm/shuffle.c b/mm/shuffle.c index c13c33b247e8747c4bed412edc8776ef53e8973c..fb1393b8b3a9d61e7bee560bd434e7de3b46b635 100644 --- a/mm/shuffle.c +++ b/mm/shuffle.c @@ -12,23 +12,22 @@ DEFINE_STATIC_KEY_FALSE(page_alloc_shuffle_key); static bool shuffle_param; -static int shuffle_show(char *buffer, const struct kernel_param *kp) -{ - return sprintf(buffer, "%c\n", shuffle_param ? 'Y' : 'N'); -} -static __meminit int shuffle_store(const char *val, +static __meminit int shuffle_param_set(const char *val, const struct kernel_param *kp) { - int rc = param_set_bool(val, kp); - - if (rc < 0) - return rc; - if (shuffle_param) + if (param_set_bool(val, kp)) + return -EINVAL; + if (*(bool *)kp->arg) static_branch_enable(&page_alloc_shuffle_key); return 0; } -module_param_call(shuffle, shuffle_store, shuffle_show, &shuffle_param, 0400); + +static const struct kernel_param_ops shuffle_param_ops = { + .set = shuffle_param_set, + .get = param_get_bool, +}; +module_param_cb(shuffle, &shuffle_param_ops, &shuffle_param, 0400); /* * For two pages to be swapped in the shuffle, they must be free (on a diff --git a/mm/slab.c b/mm/slab.c index 10e96137b44f5ab275a1e01ed2092f7151ba5a83..59c8e28f7b6ab7498ba13a244168f3e021f21155 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1619,7 +1619,7 @@ static void slab_destroy(struct kmem_cache *cachep, struct slab *slab) * although actual page can be freed in rcu context */ if (OFF_SLAB(cachep)) - kmem_cache_free(cachep->freelist_cache, freelist); + kfree(freelist); } /* @@ -1671,21 +1671,27 @@ static size_t calculate_slab_order(struct kmem_cache *cachep, if (flags & CFLGS_OFF_SLAB) { struct kmem_cache *freelist_cache; size_t freelist_size; + size_t freelist_cache_size; freelist_size = num * sizeof(freelist_idx_t); - freelist_cache = kmalloc_slab(freelist_size, 0u); - if (!freelist_cache) - continue; - - /* - * Needed to avoid possible looping condition - * in cache_grow_begin() - */ - if (OFF_SLAB(freelist_cache)) - continue; + if (freelist_size > KMALLOC_MAX_CACHE_SIZE) { + freelist_cache_size = PAGE_SIZE << get_order(freelist_size); + } else { + freelist_cache = kmalloc_slab(freelist_size, 0u); + if (!freelist_cache) + continue; + freelist_cache_size = freelist_cache->size; + + /* + * Needed to avoid possible looping condition + * in cache_grow_begin() + */ + if (OFF_SLAB(freelist_cache)) + continue; + } /* check if off slab has enough benefit */ - if (freelist_cache->size > cachep->size / 2) + if (freelist_cache_size > cachep->size / 2) continue; } @@ -2061,11 +2067,6 @@ done: cachep->flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); #endif - if (OFF_SLAB(cachep)) { - cachep->freelist_cache = - kmalloc_slab(cachep->freelist_size, 0u); - } - err = setup_cpu_cache(cachep, gfp); if (err) { __kmem_cache_release(cachep); @@ -2292,7 +2293,7 @@ static void *alloc_slabmgmt(struct kmem_cache *cachep, freelist = NULL; else if (OFF_SLAB(cachep)) { /* Slab management obj is off-slab. */ - freelist = kmem_cache_alloc_node(cachep->freelist_cache, + freelist = kmalloc_node(cachep->freelist_size, local_flags, nodeid); } else { /* We will use last bytes at the slab for freelist */ @@ -2380,7 +2381,7 @@ static bool freelist_state_initialize(union freelist_init_state *state, unsigned int rand; /* Use best entropy available to define a random shift */ - rand = get_random_int(); + rand = get_random_u32(); /* Use a random state if the pre-computed list is not available */ if (!cachep->random_seq) { @@ -3181,84 +3182,46 @@ must_grow: } static __always_inline void * -slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size, - unsigned long caller) +__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid) { - unsigned long save_flags; - void *ptr; + void *objp = NULL; int slab_node = numa_mem_id(); - struct obj_cgroup *objcg = NULL; - bool init = false; - - flags &= gfp_allowed_mask; - cachep = slab_pre_alloc_hook(cachep, NULL, &objcg, 1, flags); - if (unlikely(!cachep)) - return NULL; - - ptr = kfence_alloc(cachep, orig_size, flags); - if (unlikely(ptr)) - goto out_hooks; - - local_irq_save(save_flags); - - if (nodeid == NUMA_NO_NODE) - nodeid = slab_node; - if (unlikely(!get_node(cachep, nodeid))) { - /* Node not bootstrapped yet */ - ptr = fallback_alloc(cachep, flags); - goto out; - } - - if (nodeid == slab_node) { + if (nodeid == NUMA_NO_NODE) { + if (current->mempolicy || cpuset_do_slab_mem_spread()) { + objp = alternate_node_alloc(cachep, flags); + if (objp) + goto out; + } /* * Use the locally cached objects if possible. * However ____cache_alloc does not allow fallback * to other nodes. It may fail while we still have * objects on other nodes available. */ - ptr = ____cache_alloc(cachep, flags); - if (ptr) - goto out; - } - /* ___cache_alloc_node can fall back to other nodes */ - ptr = ____cache_alloc_node(cachep, flags, nodeid); -out: - local_irq_restore(save_flags); - ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); - init = slab_want_init_on_alloc(flags, cachep); - -out_hooks: - slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init); - return ptr; -} - -static __always_inline void * -__do_cache_alloc(struct kmem_cache *cache, gfp_t flags) -{ - void *objp; - - if (current->mempolicy || cpuset_do_slab_mem_spread()) { - objp = alternate_node_alloc(cache, flags); - if (objp) - goto out; + objp = ____cache_alloc(cachep, flags); + nodeid = slab_node; + } else if (nodeid == slab_node) { + objp = ____cache_alloc(cachep, flags); + } else if (!get_node(cachep, nodeid)) { + /* Node not bootstrapped yet */ + objp = fallback_alloc(cachep, flags); + goto out; } - objp = ____cache_alloc(cache, flags); /* * We may just have run out of memory on the local node. * ____cache_alloc_node() knows how to locate memory on other nodes */ if (!objp) - objp = ____cache_alloc_node(cache, flags, numa_mem_id()); - + objp = ____cache_alloc_node(cachep, flags, nodeid); out: return objp; } #else static __always_inline void * -__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) +__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid __maybe_unused) { return ____cache_alloc(cachep, flags); } @@ -3266,8 +3229,8 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) #endif /* CONFIG_NUMA */ static __always_inline void * -slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, - size_t orig_size, unsigned long caller) +slab_alloc_node(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, + int nodeid, size_t orig_size, unsigned long caller) { unsigned long save_flags; void *objp; @@ -3284,7 +3247,7 @@ slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, goto out; local_irq_save(save_flags); - objp = __do_cache_alloc(cachep, flags); + objp = __do_cache_alloc(cachep, flags, nodeid); local_irq_restore(save_flags); objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); prefetchw(objp); @@ -3295,6 +3258,14 @@ out: return objp; } +static __always_inline void * +slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags, + size_t orig_size, unsigned long caller) +{ + return slab_alloc_node(cachep, lru, flags, NUMA_NO_NODE, orig_size, + caller); +} + /* * Caller needs to acquire correct kmem_cache_node's list_lock * @list: List of detached free slabs should be freed by caller @@ -3470,8 +3441,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, { void *ret = slab_alloc(cachep, lru, flags, cachep->object_size, _RET_IP_); - trace_kmem_cache_alloc(_RET_IP_, ret, cachep, - cachep->object_size, cachep->size, flags); + trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, NUMA_NO_NODE); return ret; } @@ -3521,7 +3491,8 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, local_irq_disable(); for (i = 0; i < size; i++) { - void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags); + void *objp = kfence_alloc(s, s->object_size, flags) ?: + __do_cache_alloc(s, flags, NUMA_NO_NODE); if (unlikely(!objp)) goto error; @@ -3548,23 +3519,6 @@ error: } EXPORT_SYMBOL(kmem_cache_alloc_bulk); -#ifdef CONFIG_TRACING -void * -kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) -{ - void *ret; - - ret = slab_alloc(cachep, NULL, flags, size, _RET_IP_); - - ret = kasan_kmalloc(cachep, ret, size, flags); - trace_kmalloc(_RET_IP_, ret, cachep, - size, cachep->size, flags); - return ret; -} -EXPORT_SYMBOL(kmem_cache_alloc_trace); -#endif - -#ifdef CONFIG_NUMA /** * kmem_cache_alloc_node - Allocate an object on the specified node * @cachep: The cache to allocate from. @@ -3580,65 +3534,21 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace); */ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) { - void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_); + void *ret = slab_alloc_node(cachep, NULL, flags, nodeid, cachep->object_size, _RET_IP_); - trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep, - cachep->object_size, cachep->size, - flags, nodeid); + trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, nodeid); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_node); -#ifdef CONFIG_TRACING -void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, - gfp_t flags, - int nodeid, - size_t size) -{ - void *ret; - - ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_); - - ret = kasan_kmalloc(cachep, ret, size, flags); - trace_kmalloc_node(_RET_IP_, ret, cachep, - size, cachep->size, - flags, nodeid); - return ret; -} -EXPORT_SYMBOL(kmem_cache_alloc_node_trace); -#endif - -static __always_inline void * -__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) -{ - struct kmem_cache *cachep; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) - return NULL; - cachep = kmalloc_slab(size, flags); - if (unlikely(ZERO_OR_NULL_PTR(cachep))) - return cachep; - ret = kmem_cache_alloc_node_trace(cachep, flags, node, size); - ret = kasan_kmalloc(cachep, ret, size, flags); - - return ret; -} - -void *__kmalloc_node(size_t size, gfp_t flags, int node) +void *__kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, + int nodeid, size_t orig_size, + unsigned long caller) { - return __do_kmalloc_node(size, flags, node, _RET_IP_); + return slab_alloc_node(cachep, NULL, flags, nodeid, + orig_size, caller); } -EXPORT_SYMBOL(__kmalloc_node); - -void *__kmalloc_node_track_caller(size_t size, gfp_t flags, - int node, unsigned long caller) -{ - return __do_kmalloc_node(size, flags, node, caller); -} -EXPORT_SYMBOL(__kmalloc_node_track_caller); -#endif /* CONFIG_NUMA */ #ifdef CONFIG_PRINTK void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) @@ -3662,45 +3572,25 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) } #endif -/** - * __do_kmalloc - allocate memory - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate (see kmalloc). - * @caller: function caller for debug tracking of the caller - * - * Return: pointer to the allocated memory or %NULL in case of error - */ -static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, - unsigned long caller) +static __always_inline +void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp, + unsigned long caller) { - struct kmem_cache *cachep; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) - return NULL; - cachep = kmalloc_slab(size, flags); - if (unlikely(ZERO_OR_NULL_PTR(cachep))) - return cachep; - ret = slab_alloc(cachep, NULL, flags, size, caller); - - ret = kasan_kmalloc(cachep, ret, size, flags); - trace_kmalloc(caller, ret, cachep, - size, cachep->size, flags); - - return ret; -} + unsigned long flags; -void *__kmalloc(size_t size, gfp_t flags) -{ - return __do_kmalloc(size, flags, _RET_IP_); + local_irq_save(flags); + debug_check_no_locks_freed(objp, cachep->object_size); + if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) + debug_check_no_obj_freed(objp, cachep->object_size); + __cache_free(cachep, objp, caller); + local_irq_restore(flags); } -EXPORT_SYMBOL(__kmalloc); -void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller) +void __kmem_cache_free(struct kmem_cache *cachep, void *objp, + unsigned long caller) { - return __do_kmalloc(size, flags, caller); + __do_kmem_cache_free(cachep, objp, caller); } -EXPORT_SYMBOL(__kmalloc_track_caller); /** * kmem_cache_free - Deallocate an object @@ -3712,34 +3602,38 @@ EXPORT_SYMBOL(__kmalloc_track_caller); */ void kmem_cache_free(struct kmem_cache *cachep, void *objp) { - unsigned long flags; cachep = cache_from_obj(cachep, objp); if (!cachep) return; - trace_kmem_cache_free(_RET_IP_, objp, cachep->name); - local_irq_save(flags); - debug_check_no_locks_freed(objp, cachep->object_size); - if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) - debug_check_no_obj_freed(objp, cachep->object_size); - __cache_free(cachep, objp, _RET_IP_); - local_irq_restore(flags); + trace_kmem_cache_free(_RET_IP_, objp, cachep); + __do_kmem_cache_free(cachep, objp, _RET_IP_); } EXPORT_SYMBOL(kmem_cache_free); void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) { - struct kmem_cache *s; - size_t i; local_irq_disable(); - for (i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { void *objp = p[i]; + struct kmem_cache *s; - if (!orig_s) /* called via kfree_bulk */ - s = virt_to_cache(objp); - else + if (!orig_s) { + struct folio *folio = virt_to_folio(objp); + + /* called via kfree_bulk */ + if (!folio_test_slab(folio)) { + local_irq_enable(); + free_large_kmalloc(folio, objp); + local_irq_disable(); + continue; + } + s = folio_slab(folio)->slab_cache; + } else { s = cache_from_obj(orig_s, objp); + } + if (!s) continue; @@ -3755,39 +3649,6 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p) } EXPORT_SYMBOL(kmem_cache_free_bulk); -/** - * kfree - free previously allocated memory - * @objp: pointer returned by kmalloc. - * - * If @objp is NULL, no operation is performed. - * - * Don't free memory not originally allocated by kmalloc() - * or you will run into trouble. - */ -void kfree(const void *objp) -{ - struct kmem_cache *c; - unsigned long flags; - - trace_kfree(_RET_IP_, objp); - - if (unlikely(ZERO_OR_NULL_PTR(objp))) - return; - local_irq_save(flags); - kfree_debugcheck(objp); - c = virt_to_cache(objp); - if (!c) { - local_irq_restore(flags); - return; - } - debug_check_no_locks_freed(objp, c->object_size); - - debug_check_no_obj_freed(objp, c->object_size); - __cache_free(c, (void *)objp, _RET_IP_); - local_irq_restore(flags); -} -EXPORT_SYMBOL(kfree); - /* * This initializes kmem_cache_node or resizes various caches for all nodes. */ @@ -4190,28 +4051,3 @@ void __check_heap_object(const void *ptr, unsigned long n, usercopy_abort("SLAB object", cachep->name, to_user, offset, n); } #endif /* CONFIG_HARDENED_USERCOPY */ - -/** - * __ksize -- Uninstrumented ksize. - * @objp: pointer to the object - * - * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same - * safety checks as ksize() with KASAN instrumentation enabled. - * - * Return: size of the actual memory used by @objp in bytes - */ -size_t __ksize(const void *objp) -{ - struct kmem_cache *c; - size_t size; - - BUG_ON(!objp); - if (unlikely(objp == ZERO_SIZE_PTR)) - return 0; - - c = virt_to_cache(objp); - size = c ? c->object_size : 0; - - return size; -} -EXPORT_SYMBOL(__ksize); diff --git a/mm/slab.h b/mm/slab.h index 4ec82bec15ecd3b4f50e67bdde6a954bc50781df..0202a8c2f0d25dc4d00ae339e0d1ab06d6f4518b 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -273,6 +273,11 @@ void create_kmalloc_caches(slab_flags_t); /* Find the kmalloc slab corresponding for a certain size */ struct kmem_cache *kmalloc_slab(size_t, gfp_t); + +void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t orig_size, + unsigned long caller); +void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller); #endif gfp_t kmalloc_fix_flags(gfp_t flags); @@ -658,8 +663,13 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) print_tracking(cachep, x); return cachep; } + +void free_large_kmalloc(struct folio *folio, void *object); + #endif /* CONFIG_SLOB */ +size_t __ksize(const void *objp); + static inline size_t slab_ksize(const struct kmem_cache *s) { #ifndef CONFIG_SLUB @@ -729,6 +739,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s, memset(p[i], 0, s->object_size); kmemleak_alloc_recursive(p[i], s->object_size, 1, s->flags, flags); + kmsan_slab_alloc(s, p[i], flags); } memcg_slab_post_alloc_hook(s, objcg, flags, size, p); diff --git a/mm/slab_common.c b/mm/slab_common.c index 17996649cfe3e9efb174a16c00b5c8c22176067b..33b1886b06ebfd4a137c91ae7d4ad22d1e4265f7 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -392,6 +392,28 @@ kmem_cache_create(const char *name, unsigned int size, unsigned int align, } EXPORT_SYMBOL(kmem_cache_create); +#ifdef SLAB_SUPPORTS_SYSFS +/* + * For a given kmem_cache, kmem_cache_destroy() should only be called + * once or there will be a use-after-free problem. The actual deletion + * and release of the kobject does not need slab_mutex or cpu_hotplug_lock + * protection. So they are now done without holding those locks. + * + * Note that there will be a slight delay in the deletion of sysfs files + * if kmem_cache_release() is called indrectly from a work function. + */ +static void kmem_cache_release(struct kmem_cache *s) +{ + sysfs_slab_unlink(s); + sysfs_slab_release(s); +} +#else +static void kmem_cache_release(struct kmem_cache *s) +{ + slab_kmem_cache_release(s); +} +#endif + static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) { LIST_HEAD(to_destroy); @@ -418,11 +440,7 @@ static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) list_for_each_entry_safe(s, s2, &to_destroy, list) { debugfs_slab_release(s); kfence_shutdown_cache(s); -#ifdef SLAB_SUPPORTS_SYSFS - sysfs_slab_release(s); -#else - slab_kmem_cache_release(s); -#endif + kmem_cache_release(s); } } @@ -437,20 +455,11 @@ static int shutdown_cache(struct kmem_cache *s) list_del(&s->list); if (s->flags & SLAB_TYPESAFE_BY_RCU) { -#ifdef SLAB_SUPPORTS_SYSFS - sysfs_slab_unlink(s); -#endif list_add_tail(&s->list, &slab_caches_to_rcu_destroy); schedule_work(&slab_caches_to_rcu_destroy_work); } else { kfence_shutdown_cache(s); debugfs_slab_release(s); -#ifdef SLAB_SUPPORTS_SYSFS - sysfs_slab_unlink(s); - sysfs_slab_release(s); -#else - slab_kmem_cache_release(s); -#endif } return 0; @@ -465,14 +474,19 @@ void slab_kmem_cache_release(struct kmem_cache *s) void kmem_cache_destroy(struct kmem_cache *s) { + int refcnt; + bool rcu_set; + if (unlikely(!s) || !kasan_check_byte(s)) return; cpus_read_lock(); mutex_lock(&slab_mutex); - s->refcount--; - if (s->refcount) + rcu_set = s->flags & SLAB_TYPESAFE_BY_RCU; + + refcnt = --s->refcount; + if (refcnt) goto out_unlock; WARN(shutdown_cache(s), @@ -481,6 +495,8 @@ void kmem_cache_destroy(struct kmem_cache *s) out_unlock: mutex_unlock(&slab_mutex); cpus_read_unlock(); + if (!refcnt && !rcu_set) + kmem_cache_release(s); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -495,13 +511,9 @@ EXPORT_SYMBOL(kmem_cache_destroy); */ int kmem_cache_shrink(struct kmem_cache *cachep) { - int ret; - - kasan_cache_shrink(cachep); - ret = __kmem_cache_shrink(cachep); - return ret; + return __kmem_cache_shrink(cachep); } EXPORT_SYMBOL(kmem_cache_shrink); @@ -649,7 +661,8 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name, if (!s) panic("Out of memory when creating slab %s\n", name); - create_boot_cache(s, name, size, flags, useroffset, usersize); + create_boot_cache(s, name, size, flags | SLAB_KMALLOC, useroffset, + usersize); kasan_cache_create_kmalloc(s); list_add(&s->list, &slab_caches); s->refcount = 1; @@ -721,6 +734,26 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) return kmalloc_caches[kmalloc_type(flags)][index]; } +size_t kmalloc_size_roundup(size_t size) +{ + struct kmem_cache *c; + + /* Short-circuit the 0 size case. */ + if (unlikely(size == 0)) + return 0; + /* Short-circuit saturated "too-large" case. */ + if (unlikely(size == SIZE_MAX)) + return SIZE_MAX; + /* Above the smaller buckets, size is a multiple of page size. */ + if (size > KMALLOC_MAX_CACHE_SIZE) + return PAGE_SIZE << get_order(size); + + /* The flags don't matter since size_index is common to all. */ + c = kmalloc_slab(size, GFP_KERNEL); + return c ? c->object_size : 0; +} +EXPORT_SYMBOL(kmalloc_size_roundup); + #ifdef CONFIG_ZONE_DMA #define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz, #else @@ -744,8 +777,8 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) /* * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. - * kmalloc_index() supports up to 2^25=32MB, so the final entry of the table is - * kmalloc-32M. + * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is + * kmalloc-2M. */ const struct kmalloc_info_struct kmalloc_info[] __initconst = { INIT_KMALLOC_INFO(0, 0), @@ -769,11 +802,7 @@ const struct kmalloc_info_struct kmalloc_info[] __initconst = { INIT_KMALLOC_INFO(262144, 256k), INIT_KMALLOC_INFO(524288, 512k), INIT_KMALLOC_INFO(1048576, 1M), - INIT_KMALLOC_INFO(2097152, 2M), - INIT_KMALLOC_INFO(4194304, 4M), - INIT_KMALLOC_INFO(8388608, 8M), - INIT_KMALLOC_INFO(16777216, 16M), - INIT_KMALLOC_INFO(33554432, 32M) + INIT_KMALLOC_INFO(2097152, 2M) }; /* @@ -886,6 +915,156 @@ void __init create_kmalloc_caches(slab_flags_t flags) /* Kmalloc array is now usable */ slab_state = UP; } + +void free_large_kmalloc(struct folio *folio, void *object) +{ + unsigned int order = folio_order(folio); + + if (WARN_ON_ONCE(order == 0)) + pr_warn_once("object pointer: 0x%p\n", object); + + kmemleak_free(object); + kasan_kfree_large(object); + kmsan_kfree_large(object); + + mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B, + -(PAGE_SIZE << order)); + __free_pages(folio_page(folio, 0), order); +} + +static void *__kmalloc_large_node(size_t size, gfp_t flags, int node); +static __always_inline +void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) +{ + struct kmem_cache *s; + void *ret; + + if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { + ret = __kmalloc_large_node(size, flags, node); + trace_kmalloc(_RET_IP_, ret, size, + PAGE_SIZE << get_order(size), flags, node); + return ret; + } + + s = kmalloc_slab(size, flags); + + if (unlikely(ZERO_OR_NULL_PTR(s))) + return s; + + ret = __kmem_cache_alloc_node(s, flags, node, size, caller); + ret = kasan_kmalloc(s, ret, size, flags); + trace_kmalloc(_RET_IP_, ret, size, s->size, flags, node); + return ret; +} + +void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ + return __do_kmalloc_node(size, flags, node, _RET_IP_); +} +EXPORT_SYMBOL(__kmalloc_node); + +void *__kmalloc(size_t size, gfp_t flags) +{ + return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_); +} +EXPORT_SYMBOL(__kmalloc); + +void *__kmalloc_node_track_caller(size_t size, gfp_t flags, + int node, unsigned long caller) +{ + return __do_kmalloc_node(size, flags, node, caller); +} +EXPORT_SYMBOL(__kmalloc_node_track_caller); + +/** + * kfree - free previously allocated memory + * @object: pointer returned by kmalloc. + * + * If @object is NULL, no operation is performed. + * + * Don't free memory not originally allocated by kmalloc() + * or you will run into trouble. + */ +void kfree(const void *object) +{ + struct folio *folio; + struct slab *slab; + struct kmem_cache *s; + + trace_kfree(_RET_IP_, object); + + if (unlikely(ZERO_OR_NULL_PTR(object))) + return; + + folio = virt_to_folio(object); + if (unlikely(!folio_test_slab(folio))) { + free_large_kmalloc(folio, (void *)object); + return; + } + + slab = folio_slab(folio); + s = slab->slab_cache; + __kmem_cache_free(s, (void *)object, _RET_IP_); +} +EXPORT_SYMBOL(kfree); + +/** + * __ksize -- Report full size of underlying allocation + * @objp: pointer to the object + * + * This should only be used internally to query the true size of allocations. + * It is not meant to be a way to discover the usable size of an allocation + * after the fact. Instead, use kmalloc_size_roundup(). Using memory beyond + * the originally requested allocation size may trigger KASAN, UBSAN_BOUNDS, + * and/or FORTIFY_SOURCE. + * + * Return: size of the actual memory used by @objp in bytes + */ +size_t __ksize(const void *object) +{ + struct folio *folio; + + if (unlikely(object == ZERO_SIZE_PTR)) + return 0; + + folio = virt_to_folio(object); + + if (unlikely(!folio_test_slab(folio))) { + if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE)) + return 0; + if (WARN_ON(object != folio_address(folio))) + return 0; + return folio_size(folio); + } + + return slab_ksize(folio_slab(folio)->slab_cache); +} + +#ifdef CONFIG_TRACING +void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) +{ + void *ret = __kmem_cache_alloc_node(s, gfpflags, NUMA_NO_NODE, + size, _RET_IP_); + + trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); + + ret = kasan_kmalloc(s, ret, size, gfpflags); + return ret; +} +EXPORT_SYMBOL(kmalloc_trace); + +void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t size) +{ + void *ret = __kmem_cache_alloc_node(s, gfpflags, node, size, _RET_IP_); + + trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); + + ret = kasan_kmalloc(s, ret, size, gfpflags); + return ret; +} +EXPORT_SYMBOL(kmalloc_node_trace); +#endif /* !CONFIG_TRACING */ #endif /* !CONFIG_SLOB */ gfp_t kmalloc_fix_flags(gfp_t flags) @@ -905,37 +1084,51 @@ gfp_t kmalloc_fix_flags(gfp_t flags) * directly to the page allocator. We use __GFP_COMP, because we will need to * know the allocation order to free the pages properly in kfree. */ -void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) + +static void *__kmalloc_large_node(size_t size, gfp_t flags, int node) { - void *ret = NULL; struct page *page; + void *ptr = NULL; + unsigned int order = get_order(size); if (unlikely(flags & GFP_SLAB_BUG_MASK)) flags = kmalloc_fix_flags(flags); flags |= __GFP_COMP; - page = alloc_pages(flags, order); - if (likely(page)) { - ret = page_address(page); + page = alloc_pages_node(node, flags, order); + if (page) { + ptr = page_address(page); mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, PAGE_SIZE << order); } - ret = kasan_kmalloc_large(ret, size, flags); - /* As ret might get tagged, call kmemleak hook after KASAN. */ - kmemleak_alloc(ret, size, 1, flags); + + ptr = kasan_kmalloc_large(ptr, size, flags); + /* As ptr might get tagged, call kmemleak hook after KASAN. */ + kmemleak_alloc(ptr, size, 1, flags); + kmsan_kmalloc_large(ptr, size, flags); + + return ptr; +} + +void *kmalloc_large(size_t size, gfp_t flags) +{ + void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE); + + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), + flags, NUMA_NO_NODE); return ret; } -EXPORT_SYMBOL(kmalloc_order); +EXPORT_SYMBOL(kmalloc_large); -#ifdef CONFIG_TRACING -void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) +void *kmalloc_large_node(size_t size, gfp_t flags, int node) { - void *ret = kmalloc_order(size, flags, order); - trace_kmalloc(_RET_IP_, ret, NULL, size, PAGE_SIZE << order, flags); + void *ret = __kmalloc_large_node(size, flags, node); + + trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size), + flags, node); return ret; } -EXPORT_SYMBOL(kmalloc_order_trace); -#endif +EXPORT_SYMBOL(kmalloc_large_node); #ifdef CONFIG_SLAB_FREELIST_RANDOM /* Randomize a generic freelist */ @@ -1134,8 +1327,8 @@ module_init(slab_proc_init); #endif /* CONFIG_SLAB || CONFIG_SLUB_DEBUG */ -static __always_inline void *__do_krealloc(const void *p, size_t new_size, - gfp_t flags) +static __always_inline __realloc_size(2) void * +__do_krealloc(const void *p, size_t new_size, gfp_t flags) { void *ret; size_t ks; @@ -1267,8 +1460,6 @@ EXPORT_SYMBOL(ksize); /* Tracepoints definitions. */ EXPORT_TRACEPOINT_SYMBOL(kmalloc); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); -EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); -EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); EXPORT_TRACEPOINT_SYMBOL(kfree); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); diff --git a/mm/slob.c b/mm/slob.c index 2bd4f476c34085ce7fafc57b4c22331fd7c4dc61..fe567fcfa3a39ef41f939f734c2eab41e86de3c2 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -507,8 +507,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) *m = size; ret = (void *)m + minalign; - trace_kmalloc_node(caller, ret, NULL, - size, size + minalign, gfp, node); + trace_kmalloc(caller, ret, size, size + minalign, gfp, node); } else { unsigned int order = get_order(size); @@ -516,8 +515,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller) gfp |= __GFP_COMP; ret = slob_new_pages(gfp, order, node); - trace_kmalloc_node(caller, ret, NULL, - size, PAGE_SIZE << order, gfp, node); + trace_kmalloc(caller, ret, size, PAGE_SIZE << order, gfp, node); } kmemleak_alloc(ret, size, 1, gfp); @@ -530,20 +528,12 @@ void *__kmalloc(size_t size, gfp_t gfp) } EXPORT_SYMBOL(__kmalloc); -void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller) -{ - return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, caller); -} -EXPORT_SYMBOL(__kmalloc_track_caller); - -#ifdef CONFIG_NUMA void *__kmalloc_node_track_caller(size_t size, gfp_t gfp, int node, unsigned long caller) { return __do_kmalloc_node(size, gfp, node, caller); } EXPORT_SYMBOL(__kmalloc_node_track_caller); -#endif void kfree(const void *block) { @@ -574,6 +564,20 @@ void kfree(const void *block) } EXPORT_SYMBOL(kfree); +size_t kmalloc_size_roundup(size_t size) +{ + /* Short-circuit the 0 size case. */ + if (unlikely(size == 0)) + return 0; + /* Short-circuit saturated "too-large" case. */ + if (unlikely(size == SIZE_MAX)) + return SIZE_MAX; + + return ALIGN(size, ARCH_KMALLOC_MINALIGN); +} + +EXPORT_SYMBOL(kmalloc_size_roundup); + /* can't use ksize for kmem_cache_alloc memory, only kmalloc */ size_t __ksize(const void *block) { @@ -594,7 +598,6 @@ size_t __ksize(const void *block) m = (unsigned int *)(block - align); return SLOB_UNITS(*m) * SLOB_UNIT; } -EXPORT_SYMBOL(__ksize); int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags) { @@ -602,6 +605,9 @@ int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags) /* leave room for rcu footer at the end of object */ c->size += sizeof(struct slob_rcu); } + + /* Actual size allocated */ + c->size = SLOB_UNITS(c->size) * SLOB_UNIT; c->flags = flags; return 0; } @@ -616,14 +622,10 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node) if (c->size < PAGE_SIZE) { b = slob_alloc(c->size, flags, c->align, node, 0); - trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size, - SLOB_UNITS(c->size) * SLOB_UNIT, - flags, node); + trace_kmem_cache_alloc(_RET_IP_, b, c, flags, node); } else { b = slob_new_pages(flags, get_order(c->size), node); - trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size, - PAGE_SIZE << get_order(c->size), - flags, node); + trace_kmem_cache_alloc(_RET_IP_, b, c, flags, node); } if (b && c->ctor) { @@ -647,7 +649,7 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, gfp_ return slob_alloc_node(cachep, flags, NUMA_NO_NODE); } EXPORT_SYMBOL(kmem_cache_alloc_lru); -#ifdef CONFIG_NUMA + void *__kmalloc_node(size_t size, gfp_t gfp, int node) { return __do_kmalloc_node(size, gfp, node, _RET_IP_); @@ -659,7 +661,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node) return slob_alloc_node(cachep, gfp, node); } EXPORT_SYMBOL(kmem_cache_alloc_node); -#endif static void __kmem_cache_free(void *b, int size) { @@ -680,7 +681,7 @@ static void kmem_rcu_free(struct rcu_head *head) void kmem_cache_free(struct kmem_cache *c, void *b) { kmemleak_free_recursive(b, c->flags); - trace_kmem_cache_free(_RET_IP_, b, c->name); + trace_kmem_cache_free(_RET_IP_, b, c); if (unlikely(c->flags & SLAB_TYPESAFE_BY_RCU)) { struct slob_rcu *slob_rcu; slob_rcu = b + (c->size - sizeof(struct slob_rcu)); diff --git a/mm/slub.c b/mm/slub.c index 862dbd9af4f521fe820a9b5bdb69e7a726ba3d5d..157527d7101be0da54026376ff4f4095deb9ec2d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,7 @@ * 1. slab_mutex (Global Mutex) * 2. node->list_lock (Spinlock) * 3. kmem_cache->cpu_slab->lock (Local lock) - * 4. slab_lock(slab) (Only on some arches or for debugging) + * 4. slab_lock(slab) (Only on some arches) * 5. object_map_lock (Only for debugging) * * slab_mutex @@ -64,8 +65,9 @@ * The slab_lock is a wrapper around the page lock, thus it is a bit * spinlock. * - * The slab_lock is only used for debugging and on arches that do not - * have the ability to do a cmpxchg_double. It only protects: + * The slab_lock is only used on arches that do not have the ability + * to do a cmpxchg_double. It only protects: + * * A. slab->freelist -> List of free objects in a slab * B. slab->inuse -> Number of objects in use * C. slab->objects -> Number of objects in slab @@ -94,15 +96,20 @@ * allocating a long series of objects that fill up slabs does not require * the list lock. * + * For debug caches, all allocations are forced to go through a list_lock + * protected region to serialize against concurrent validation. + * * cpu_slab->lock local lock * * This locks protect slowpath manipulation of all kmem_cache_cpu fields * except the stat counters. This is a percpu structure manipulated only by * the local cpu, so the lock protects against being preempted or interrupted * by an irq. Fast path operations rely on lockless operations instead. - * On PREEMPT_RT, the local lock does not actually disable irqs (and thus - * prevent the lockless operations), so fastpath operations also need to take - * the lock and are no longer lockless. + * + * On PREEMPT_RT, the local lock neither disables interrupts nor preemption + * which means the lockless fastpath cannot be used as it might interfere with + * an in-progress slow path operations. In this case the local lock is always + * taken but it still utilizes the freelist for the common operations. * * lockless fastpaths * @@ -163,8 +170,9 @@ * function call even on !PREEMPT_RT, use inline preempt_disable() there. */ #ifndef CONFIG_PREEMPT_RT -#define slub_get_cpu_ptr(var) get_cpu_ptr(var) -#define slub_put_cpu_ptr(var) put_cpu_ptr(var) +#define slub_get_cpu_ptr(var) get_cpu_ptr(var) +#define slub_put_cpu_ptr(var) put_cpu_ptr(var) +#define USE_LOCKLESS_FAST_PATH() (true) #else #define slub_get_cpu_ptr(var) \ ({ \ @@ -176,6 +184,7 @@ do { \ (void)(var); \ migrate_enable(); \ } while (0) +#define USE_LOCKLESS_FAST_PATH() (false) #endif #ifdef CONFIG_SLUB_DEBUG @@ -186,11 +195,24 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); #endif #endif /* CONFIG_SLUB_DEBUG */ +/* Structure holding parameters for get_partial() call chain */ +struct partial_context { + struct slab **slab; + gfp_t flags; + unsigned int orig_size; +}; + static inline bool kmem_cache_debug(struct kmem_cache *s) { return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS); } +static inline bool slub_debug_orig_size(struct kmem_cache *s) +{ + return (kmem_cache_debug_flags(s, SLAB_STORE_USER) && + (s->flags & SLAB_KMALLOC)); +} + void *fixup_red_left(struct kmem_cache *s, void *p) { if (kmem_cache_debug_flags(s, SLAB_RED_ZONE)) @@ -310,6 +332,11 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si) */ static nodemask_t slab_nodes; +/* + * Workqueue used for flush_cpu_slab(). + */ +static struct workqueue_struct *flushwq; + /******************************************************************** * Core slab cache functions *******************************************************************/ @@ -359,6 +386,17 @@ static void prefetch_freepointer(const struct kmem_cache *s, void *object) prefetchw(object + s->offset); } +/* + * When running under KMSAN, get_freepointer_safe() may return an uninitialized + * pointer value in the case the current thread loses the race for the next + * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in + * slab_alloc_node() will fail, so the uninitialized value won't be used, but + * KMSAN will still check all arguments of cmpxchg because of imperfect + * handling of inline assembly. + * To work around this problem, we apply __no_kmsan_checks to ensure that + * get_freepointer_safe() returns initialized memory. + */ +__no_kmsan_checks static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) { unsigned long freepointer_addr; @@ -442,7 +480,7 @@ slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects) /* * Per slab locking using the pagelock */ -static __always_inline void __slab_lock(struct slab *slab) +static __always_inline void slab_lock(struct slab *slab) { struct page *page = slab_page(slab); @@ -450,7 +488,7 @@ static __always_inline void __slab_lock(struct slab *slab) bit_spin_lock(PG_locked, &page->flags); } -static __always_inline void __slab_unlock(struct slab *slab) +static __always_inline void slab_unlock(struct slab *slab) { struct page *page = slab_page(slab); @@ -458,31 +496,19 @@ static __always_inline void __slab_unlock(struct slab *slab) __bit_spin_unlock(PG_locked, &page->flags); } -static __always_inline void slab_lock(struct slab *slab, unsigned long *flags) -{ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - local_irq_save(*flags); - __slab_lock(slab); -} - -static __always_inline void slab_unlock(struct slab *slab, unsigned long *flags) -{ - __slab_unlock(slab); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - local_irq_restore(*flags); -} - /* * Interrupts must be disabled (for the fallback code to work right), typically - * by an _irqsave() lock variant. Except on PREEMPT_RT where locks are different - * so we disable interrupts as part of slab_[un]lock(). + * by an _irqsave() lock variant. On PREEMPT_RT the preempt_disable(), which is + * part of bit_spin_lock(), is sufficient because the policy is not to allow any + * allocation/ free operation in hardirq context. Therefore nothing can + * interrupt the operation. */ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab, void *freelist_old, unsigned long counters_old, void *freelist_new, unsigned long counters_new, const char *n) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + if (USE_LOCKLESS_FAST_PATH()) lockdep_assert_irqs_disabled(); #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) @@ -494,18 +520,15 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab } else #endif { - /* init to 0 to prevent spurious warnings */ - unsigned long flags = 0; - - slab_lock(slab, &flags); + slab_lock(slab); if (slab->freelist == freelist_old && slab->counters == counters_old) { slab->freelist = freelist_new; slab->counters = counters_new; - slab_unlock(slab, &flags); + slab_unlock(slab); return true; } - slab_unlock(slab, &flags); + slab_unlock(slab); } cpu_relax(); @@ -536,16 +559,16 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab, unsigned long flags; local_irq_save(flags); - __slab_lock(slab); + slab_lock(slab); if (slab->freelist == freelist_old && slab->counters == counters_old) { slab->freelist = freelist_new; slab->counters = counters_new; - __slab_unlock(slab); + slab_unlock(slab); local_irq_restore(flags); return true; } - __slab_unlock(slab); + slab_unlock(slab); local_irq_restore(flags); } @@ -561,7 +584,7 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct slab *slab, #ifdef CONFIG_SLUB_DEBUG static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)]; -static DEFINE_RAW_SPINLOCK(object_map_lock); +static DEFINE_SPINLOCK(object_map_lock); static void __fill_map(unsigned long *obj_map, struct kmem_cache *s, struct slab *slab) @@ -595,30 +618,6 @@ static bool slab_add_kunit_errors(void) static inline bool slab_add_kunit_errors(void) { return false; } #endif -/* - * Determine a map of objects in use in a slab. - * - * Node listlock must be held to guarantee that the slab does - * not vanish from under us. - */ -static unsigned long *get_map(struct kmem_cache *s, struct slab *slab) - __acquires(&object_map_lock) -{ - VM_BUG_ON(!irqs_disabled()); - - raw_spin_lock(&object_map_lock); - - __fill_map(object_map, s, slab); - - return object_map; -} - -static void put_map(unsigned long *map) __releases(&object_map_lock) -{ - VM_BUG_ON(map != object_map); - raw_spin_unlock(&object_map_lock); -} - static inline unsigned int size_from_object(struct kmem_cache *s) { if (s->flags & SLAB_RED_ZONE) @@ -816,6 +815,39 @@ static void print_slab_info(const struct slab *slab) folio_flags(folio, 0)); } +/* + * kmalloc caches has fixed sizes (mostly power of 2), and kmalloc() API + * family will round up the real request size to these fixed ones, so + * there could be an extra area than what is requested. Save the original + * request size in the meta data area, for better debug and sanity check. + */ +static inline void set_orig_size(struct kmem_cache *s, + void *object, unsigned int orig_size) +{ + void *p = kasan_reset_tag(object); + + if (!slub_debug_orig_size(s)) + return; + + p += get_info_end(s); + p += sizeof(struct track) * 2; + + *(unsigned int *)p = orig_size; +} + +static inline unsigned int get_orig_size(struct kmem_cache *s, void *object) +{ + void *p = kasan_reset_tag(object); + + if (!slub_debug_orig_size(s)) + return s->object_size; + + p += get_info_end(s); + p += sizeof(struct track) * 2; + + return *(unsigned int *)p; +} + static void slab_bug(struct kmem_cache *s, char *fmt, ...) { struct va_format vaf; @@ -875,6 +907,9 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p) if (s->flags & SLAB_STORE_USER) off += 2 * sizeof(struct track); + if (slub_debug_orig_size(s)) + off += sizeof(unsigned int); + off += kasan_metadata_size(s); if (off != size_from_object(s)) @@ -1008,7 +1043,8 @@ skip_bug_print: * * A. Free pointer (if we cannot overwrite object on free) * B. Tracking data for SLAB_STORE_USER - * C. Padding to reach required alignment boundary or at minimum + * C. Original request size for kmalloc object (SLAB_STORE_USER enabled) + * D. Padding to reach required alignment boundary or at minimum * one word if debugging is on to be able to detect writes * before the word boundary. * @@ -1026,10 +1062,14 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p) { unsigned long off = get_info_end(s); /* The end of info */ - if (s->flags & SLAB_STORE_USER) + if (s->flags & SLAB_STORE_USER) { /* We also have user information there */ off += 2 * sizeof(struct track); + if (s->flags & SLAB_KMALLOC) + off += sizeof(unsigned int); + } + off += kasan_metadata_size(s); if (size_from_object(s) == off) @@ -1324,18 +1364,16 @@ static inline int alloc_consistency_checks(struct kmem_cache *s, } static noinline int alloc_debug_processing(struct kmem_cache *s, - struct slab *slab, - void *object, unsigned long addr) + struct slab *slab, void *object, int orig_size) { if (s->flags & SLAB_CONSISTENCY_CHECKS) { if (!alloc_consistency_checks(s, slab, object)) goto bad; } - /* Success perform special debug activities for allocs */ - if (s->flags & SLAB_STORE_USER) - set_track(s, object, TRACK_ALLOC, addr); + /* Success. Perform special debug activities for allocs */ trace(s, slab, object, 1); + set_orig_size(s, object, orig_size); init_object(s, object, SLUB_RED_ACTIVE); return 1; @@ -1385,63 +1423,6 @@ static inline int free_consistency_checks(struct kmem_cache *s, return 1; } -/* Supports checking bulk free of a constructed freelist */ -static noinline int free_debug_processing( - struct kmem_cache *s, struct slab *slab, - void *head, void *tail, int bulk_cnt, - unsigned long addr) -{ - struct kmem_cache_node *n = get_node(s, slab_nid(slab)); - void *object = head; - int cnt = 0; - unsigned long flags, flags2; - int ret = 0; - depot_stack_handle_t handle = 0; - - if (s->flags & SLAB_STORE_USER) - handle = set_track_prepare(); - - spin_lock_irqsave(&n->list_lock, flags); - slab_lock(slab, &flags2); - - if (s->flags & SLAB_CONSISTENCY_CHECKS) { - if (!check_slab(s, slab)) - goto out; - } - -next_object: - cnt++; - - if (s->flags & SLAB_CONSISTENCY_CHECKS) { - if (!free_consistency_checks(s, slab, object, addr)) - goto out; - } - - if (s->flags & SLAB_STORE_USER) - set_track_update(s, object, TRACK_FREE, addr, handle); - trace(s, slab, object, 0); - /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ - init_object(s, object, SLUB_RED_INACTIVE); - - /* Reached end of constructed freelist yet? */ - if (object != tail) { - object = get_freepointer(s, object); - goto next_object; - } - ret = 1; - -out: - if (cnt != bulk_cnt) - slab_err(s, slab, "Bulk freelist count(%d) invalid(%d)\n", - bulk_cnt, cnt); - - slab_unlock(slab, &flags2); - spin_unlock_irqrestore(&n->list_lock, flags); - if (!ret) - slab_fix(s, "Object at 0x%p not freed", object); - return ret; -} - /* * Parse a block of slub_debug options. Blocks are delimited by ';' * @@ -1661,16 +1642,18 @@ static inline void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {} static inline int alloc_debug_processing(struct kmem_cache *s, - struct slab *slab, void *object, unsigned long addr) { return 0; } + struct slab *slab, void *object, int orig_size) { return 0; } -static inline int free_debug_processing( +static inline void free_debug_processing( struct kmem_cache *s, struct slab *slab, void *head, void *tail, int bulk_cnt, - unsigned long addr) { return 0; } + unsigned long addr) {} static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {} static inline int check_object(struct kmem_cache *s, struct slab *slab, void *object, u8 val) { return 1; } +static inline void set_track(struct kmem_cache *s, void *object, + enum track_item alloc, unsigned long addr) {} static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab) {} static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, @@ -1704,24 +1687,11 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab, * Hooks for other subsystems that check memory allocations. In a typical * production configuration these hooks all should produce no code at all. */ -static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) -{ - ptr = kasan_kmalloc_large(ptr, size, flags); - /* As ptr might get tagged, call kmemleak hook after KASAN. */ - kmemleak_alloc(ptr, size, 1, flags); - return ptr; -} - -static __always_inline void kfree_hook(void *x) -{ - kmemleak_free(x); - kasan_kfree_large(x); -} - static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x, bool init) { kmemleak_free_recursive(x, s->flags); + kmsan_slab_free(s, x); debug_check_no_locks_freed(x, s->object_size); @@ -1911,7 +1881,7 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab) return false; freelist_count = oo_objects(s->oo); - pos = get_random_int() % freelist_count; + pos = prandom_u32_max(freelist_count); page_limit = slab->objects * s->size; start = fixup_red_left(s, slab_address(slab)); @@ -1976,11 +1946,13 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) */ slab = alloc_slab_page(alloc_gfp, node, oo); if (unlikely(!slab)) - goto out; + return NULL; stat(s, ORDER_FALLBACK); } slab->objects = oo_objects(oo); + slab->inuse = 0; + slab->frozen = 0; account_slab(slab, oo_order(oo), s, flags); @@ -2007,15 +1979,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) set_freepointer(s, p, NULL); } - slab->inuse = slab->objects; - slab->frozen = 1; - -out: - if (!slab) - return NULL; - - inc_slabs_node(s, slab_nid(slab), slab->objects); - return slab; } @@ -2102,6 +2065,75 @@ static inline void remove_partial(struct kmem_cache_node *n, n->nr_partial--; } +/* + * Called only for kmem_cache_debug() caches instead of acquire_slab(), with a + * slab from the n->partial list. Remove only a single object from the slab, do + * the alloc_debug_processing() checks and leave the slab on the list, or move + * it to full list if it was the last free object. + */ +static void *alloc_single_from_partial(struct kmem_cache *s, + struct kmem_cache_node *n, struct slab *slab, int orig_size) +{ + void *object; + + lockdep_assert_held(&n->list_lock); + + object = slab->freelist; + slab->freelist = get_freepointer(s, object); + slab->inuse++; + + if (!alloc_debug_processing(s, slab, object, orig_size)) { + remove_partial(n, slab); + return NULL; + } + + if (slab->inuse == slab->objects) { + remove_partial(n, slab); + add_full(s, n, slab); + } + + return object; +} + +/* + * Called only for kmem_cache_debug() caches to allocate from a freshly + * allocated slab. Allocate a single object instead of whole freelist + * and put the slab to the partial (or full) list. + */ +static void *alloc_single_from_new_slab(struct kmem_cache *s, + struct slab *slab, int orig_size) +{ + int nid = slab_nid(slab); + struct kmem_cache_node *n = get_node(s, nid); + unsigned long flags; + void *object; + + + object = slab->freelist; + slab->freelist = get_freepointer(s, object); + slab->inuse = 1; + + if (!alloc_debug_processing(s, slab, object, orig_size)) + /* + * It's not really expected that this would fail on a + * freshly allocated slab, but a concurrent memory + * corruption in theory could cause that. + */ + return NULL; + + spin_lock_irqsave(&n->list_lock, flags); + + if (slab->inuse == slab->objects) + add_full(s, n, slab); + else + add_partial(n, slab, DEACTIVATE_TO_HEAD); + + inc_slabs_node(s, nid, slab->objects); + spin_unlock_irqrestore(&n->list_lock, flags); + + return object; +} + /* * Remove slab from the partial list, freeze it and * return the pointer to the freelist. @@ -2159,7 +2191,7 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags); * Try to allocate a partial slab from a specific node. */ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, - struct slab **ret_slab, gfp_t gfpflags) + struct partial_context *pc) { struct slab *slab, *slab2; void *object = NULL; @@ -2179,15 +2211,23 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { void *t; - if (!pfmemalloc_match(slab, gfpflags)) + if (!pfmemalloc_match(slab, pc->flags)) continue; + if (kmem_cache_debug(s)) { + object = alloc_single_from_partial(s, n, slab, + pc->orig_size); + if (object) + break; + continue; + } + t = acquire_slab(s, n, slab, object == NULL); if (!t) break; if (!object) { - *ret_slab = slab; + *pc->slab = slab; stat(s, ALLOC_FROM_PARTIAL); object = t; } else { @@ -2211,14 +2251,13 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, /* * Get a slab from somewhere. Search in increasing NUMA distances. */ -static void *get_any_partial(struct kmem_cache *s, gfp_t flags, - struct slab **ret_slab) +static void *get_any_partial(struct kmem_cache *s, struct partial_context *pc) { #ifdef CONFIG_NUMA struct zonelist *zonelist; struct zoneref *z; struct zone *zone; - enum zone_type highest_zoneidx = gfp_zone(flags); + enum zone_type highest_zoneidx = gfp_zone(pc->flags); void *object; unsigned int cpuset_mems_cookie; @@ -2246,15 +2285,15 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = read_mems_allowed_begin(); - zonelist = node_zonelist(mempolicy_slab_node(), flags); + zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) { struct kmem_cache_node *n; n = get_node(s, zone_to_nid(zone)); - if (n && cpuset_zone_allowed(zone, flags) && + if (n && cpuset_zone_allowed(zone, pc->flags) && n->nr_partial > s->min_partial) { - object = get_partial_node(s, n, ret_slab, flags); + object = get_partial_node(s, n, pc); if (object) { /* * Don't check read_mems_allowed_retry() @@ -2275,8 +2314,7 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags, /* * Get a partial slab, lock it and return it. */ -static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, - struct slab **ret_slab) +static void *get_partial(struct kmem_cache *s, int node, struct partial_context *pc) { void *object; int searchnode = node; @@ -2284,11 +2322,11 @@ static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, if (node == NUMA_NO_NODE) searchnode = numa_mem_id(); - object = get_partial_node(s, get_node(s, searchnode), ret_slab, flags); + object = get_partial_node(s, get_node(s, searchnode), pc); if (object || node != NUMA_NO_NODE) return object; - return get_any_partial(s, flags, ret_slab); + return get_any_partial(s, pc); } #ifdef CONFIG_PREEMPTION @@ -2730,7 +2768,7 @@ static void flush_all_cpus_locked(struct kmem_cache *s) INIT_WORK(&sfw->work, flush_cpu_slab); sfw->skip = false; sfw->s = s; - schedule_work_on(cpu, &sfw->work); + queue_work_on(cpu, flushwq, &sfw->work); } for_each_online_cpu(cpu) { @@ -2788,6 +2826,113 @@ static inline unsigned long node_nr_objs(struct kmem_cache_node *n) { return atomic_long_read(&n->total_objects); } + +/* Supports checking bulk free of a constructed freelist */ +static noinline void free_debug_processing( + struct kmem_cache *s, struct slab *slab, + void *head, void *tail, int bulk_cnt, + unsigned long addr) +{ + struct kmem_cache_node *n = get_node(s, slab_nid(slab)); + struct slab *slab_free = NULL; + void *object = head; + int cnt = 0; + unsigned long flags; + bool checks_ok = false; + depot_stack_handle_t handle = 0; + + if (s->flags & SLAB_STORE_USER) + handle = set_track_prepare(); + + spin_lock_irqsave(&n->list_lock, flags); + + if (s->flags & SLAB_CONSISTENCY_CHECKS) { + if (!check_slab(s, slab)) + goto out; + } + + if (slab->inuse < bulk_cnt) { + slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n", + slab->inuse, bulk_cnt); + goto out; + } + +next_object: + + if (++cnt > bulk_cnt) + goto out_cnt; + + if (s->flags & SLAB_CONSISTENCY_CHECKS) { + if (!free_consistency_checks(s, slab, object, addr)) + goto out; + } + + if (s->flags & SLAB_STORE_USER) + set_track_update(s, object, TRACK_FREE, addr, handle); + trace(s, slab, object, 0); + /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */ + init_object(s, object, SLUB_RED_INACTIVE); + + /* Reached end of constructed freelist yet? */ + if (object != tail) { + object = get_freepointer(s, object); + goto next_object; + } + checks_ok = true; + +out_cnt: + if (cnt != bulk_cnt) + slab_err(s, slab, "Bulk free expected %d objects but found %d\n", + bulk_cnt, cnt); + +out: + if (checks_ok) { + void *prior = slab->freelist; + + /* Perform the actual freeing while we still hold the locks */ + slab->inuse -= cnt; + set_freepointer(s, tail, prior); + slab->freelist = head; + + /* + * If the slab is empty, and node's partial list is full, + * it should be discarded anyway no matter it's on full or + * partial list. + */ + if (slab->inuse == 0 && n->nr_partial >= s->min_partial) + slab_free = slab; + + if (!prior) { + /* was on full list */ + remove_full(s, n, slab); + if (!slab_free) { + add_partial(n, slab, DEACTIVATE_TO_TAIL); + stat(s, FREE_ADD_PARTIAL); + } + } else if (slab_free) { + remove_partial(n, slab); + stat(s, FREE_REMOVE_PARTIAL); + } + } + + if (slab_free) { + /* + * Update the counters while still holding n->list_lock to + * prevent spurious validation warnings + */ + dec_slabs_node(s, slab_nid(slab_free), slab_free->objects); + } + + spin_unlock_irqrestore(&n->list_lock, flags); + + if (!checks_ok) + slab_fix(s, "Object at 0x%p not freed", object); + + if (slab_free) { + stat(s, FREE_SLAB); + free_slab(s, slab_free); + } +} #endif /* CONFIG_SLUB_DEBUG */ #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) @@ -2905,11 +3050,12 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab) * already disabled (which is the case for bulk allocation). */ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) + unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) { void *freelist; struct slab *slab; unsigned long flags; + struct partial_context pc; stat(s, ALLOC_SLOWPATH); @@ -3023,7 +3169,10 @@ new_slab: new_objects: - freelist = get_partial(s, gfpflags, node, &slab); + pc.flags = gfpflags; + pc.slab = &slab; + pc.orig_size = orig_size; + freelist = get_partial(s, node, &pc); if (freelist) goto check_new_slab; @@ -3036,36 +3185,53 @@ new_objects: return NULL; } + stat(s, ALLOC_SLAB); + + if (kmem_cache_debug(s)) { + freelist = alloc_single_from_new_slab(s, slab, orig_size); + + if (unlikely(!freelist)) + goto new_objects; + + if (s->flags & SLAB_STORE_USER) + set_track(s, freelist, TRACK_ALLOC, addr); + + return freelist; + } + /* * No other reference to the slab yet so we can * muck around with it freely without cmpxchg */ freelist = slab->freelist; slab->freelist = NULL; + slab->inuse = slab->objects; + slab->frozen = 1; - stat(s, ALLOC_SLAB); + inc_slabs_node(s, slab_nid(slab), slab->objects); check_new_slab: if (kmem_cache_debug(s)) { - if (!alloc_debug_processing(s, slab, freelist, addr)) { - /* Slab failed checks. Next slab needed */ - goto new_slab; - } else { - /* - * For debug case, we don't load freelist so that all - * allocations go through alloc_debug_processing() - */ - goto return_single; - } + /* + * For debug caches here we had to go through + * alloc_single_from_partial() so just store the tracking info + * and return the object + */ + if (s->flags & SLAB_STORE_USER) + set_track(s, freelist, TRACK_ALLOC, addr); + + return freelist; } - if (unlikely(!pfmemalloc_match(slab, gfpflags))) + if (unlikely(!pfmemalloc_match(slab, gfpflags))) { /* * For !pfmemalloc_match() case we don't load freelist so that * we don't make further mismatched allocations easier. */ - goto return_single; + deactivate_slab(s, slab, get_freepointer(s, freelist)); + return freelist; + } retry_load_slab: @@ -3089,11 +3255,6 @@ retry_load_slab: c->slab = slab; goto load_freelist; - -return_single: - - deactivate_slab(s, slab, get_freepointer(s, freelist)); - return freelist; } /* @@ -3102,7 +3263,7 @@ return_single: * pointer. */ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) + unsigned long addr, struct kmem_cache_cpu *c, unsigned int orig_size) { void *p; @@ -3115,7 +3276,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, c = slub_get_cpu_ptr(s->cpu_slab); #endif - p = ___slab_alloc(s, gfpflags, node, addr, c); + p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size); #ifdef CONFIG_PREEMPT_COUNT slub_put_cpu_ptr(s->cpu_slab); #endif @@ -3197,16 +3358,10 @@ redo: object = c->freelist; slab = c->slab; - /* - * We cannot use the lockless fastpath on PREEMPT_RT because if a - * slowpath has taken the local_lock_irqsave(), it is not protected - * against a fast path operation in an irq handler. So we need to take - * the slow path which uses local_lock. It is still relatively fast if - * there is a suitable cpu freelist. - */ - if (IS_ENABLED(CONFIG_PREEMPT_RT) || + + if (!USE_LOCKLESS_FAST_PATH() || unlikely(!object || !slab || !node_match(slab, node))) { - object = __slab_alloc(s, gfpflags, node, addr, c); + object = __slab_alloc(s, gfpflags, node, addr, c, orig_size); } else { void *next_object = get_freepointer_safe(s, object); @@ -3257,8 +3412,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, { void *ret = slab_alloc(s, lru, gfpflags, _RET_IP_, s->object_size); - trace_kmem_cache_alloc(_RET_IP_, ret, s, s->object_size, - s->size, gfpflags); + trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); return ret; } @@ -3276,46 +3430,24 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, } EXPORT_SYMBOL(kmem_cache_alloc_lru); -#ifdef CONFIG_TRACING -void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) +void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, + int node, size_t orig_size, + unsigned long caller) { - void *ret = slab_alloc(s, NULL, gfpflags, _RET_IP_, size); - trace_kmalloc(_RET_IP_, ret, s, size, s->size, gfpflags); - ret = kasan_kmalloc(s, ret, size, gfpflags); - return ret; + return slab_alloc_node(s, NULL, gfpflags, node, + caller, orig_size); } -EXPORT_SYMBOL(kmem_cache_alloc_trace); -#endif -#ifdef CONFIG_NUMA void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) { void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); - trace_kmem_cache_alloc_node(_RET_IP_, ret, s, - s->object_size, s->size, gfpflags, node); + trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); return ret; } EXPORT_SYMBOL(kmem_cache_alloc_node); -#ifdef CONFIG_TRACING -void *kmem_cache_alloc_node_trace(struct kmem_cache *s, - gfp_t gfpflags, - int node, size_t size) -{ - void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); - - trace_kmalloc_node(_RET_IP_, ret, s, - size, s->size, gfpflags, node); - - ret = kasan_kmalloc(s, ret, size, gfpflags); - return ret; -} -EXPORT_SYMBOL(kmem_cache_alloc_node_trace); -#endif -#endif /* CONFIG_NUMA */ - /* * Slow path handling. This may still be called frequently since objects * have a longer lifetime than the cpu slabs in most processing loads. @@ -3341,9 +3473,10 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, if (kfence_free(head)) return; - if (kmem_cache_debug(s) && - !free_debug_processing(s, slab, head, tail, cnt, addr)) + if (kmem_cache_debug(s)) { + free_debug_processing(s, slab, head, tail, cnt, addr); return; + } do { if (unlikely(n)) { @@ -3463,6 +3596,7 @@ static __always_inline void do_slab_free(struct kmem_cache *s, void *tail_obj = tail ? : head; struct kmem_cache_cpu *c; unsigned long tid; + void **freelist; redo: /* @@ -3477,9 +3611,13 @@ redo: /* Same with comment on barrier() in slab_alloc_node() */ barrier(); - if (likely(slab == c->slab)) { -#ifndef CONFIG_PREEMPT_RT - void **freelist = READ_ONCE(c->freelist); + if (unlikely(slab != c->slab)) { + __slab_free(s, slab, head, tail_obj, cnt, addr); + return; + } + + if (USE_LOCKLESS_FAST_PATH()) { + freelist = READ_ONCE(c->freelist); set_freepointer(s, tail_obj, freelist); @@ -3491,16 +3629,8 @@ redo: note_cmpxchg_failure("slab_free", s, tid); goto redo; } -#else /* CONFIG_PREEMPT_RT */ - /* - * We cannot use the lockless fastpath on PREEMPT_RT because if - * a slowpath has taken the local_lock_irqsave(), it is not - * protected against a fast path operation in an irq handler. So - * we need to take the local_lock. We shouldn't simply defer to - * __slab_free() as that wouldn't use the cpu freelist at all. - */ - void **freelist; - + } else { + /* Update the free list under the local lock */ local_lock(&s->cpu_slab->lock); c = this_cpu_ptr(s->cpu_slab); if (unlikely(slab != c->slab)) { @@ -3515,11 +3645,8 @@ redo: c->tid = next_tid(tid); local_unlock(&s->cpu_slab->lock); -#endif - stat(s, FREE_FASTPATH); - } else - __slab_free(s, slab, head, tail_obj, cnt, addr); - + } + stat(s, FREE_FASTPATH); } static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab, @@ -3542,12 +3669,17 @@ void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr) } #endif +void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller) +{ + slab_free(s, virt_to_slab(x), x, NULL, &x, 1, caller); +} + void kmem_cache_free(struct kmem_cache *s, void *x) { s = cache_from_obj(s, x); if (!s) return; - trace_kmem_cache_free(_RET_IP_, x, s->name); + trace_kmem_cache_free(_RET_IP_, x, s); slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_); } EXPORT_SYMBOL(kmem_cache_free); @@ -3560,19 +3692,6 @@ struct detached_freelist { struct kmem_cache *s; }; -static inline void free_large_kmalloc(struct folio *folio, void *object) -{ - unsigned int order = folio_order(folio); - - if (WARN_ON_ONCE(order == 0)) - pr_warn_once("object pointer: 0x%p\n", object); - - kfree_hook(object); - mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B, - -(PAGE_SIZE << order)); - __free_pages(folio_page(folio, 0), order); -} - /* * This function progressively scans the array with free objects (with * a limited look ahead) and extract objects belonging to the same @@ -3709,7 +3828,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, * of re-populating per CPU c->freelist */ p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, - _RET_IP_, c); + _RET_IP_, c, s->object_size); if (unlikely(!p[i])) goto error; @@ -3936,6 +4055,7 @@ static void early_kmem_cache_node_alloc(int node) slab = new_slab(kmem_cache_node, GFP_NOWAIT, node); BUG_ON(!slab); + inc_slabs_node(kmem_cache_node, slab_nid(slab), slab->objects); if (slab_nid(slab) != node) { pr_err("SLUB: Unable to allocate memory from node %d\n", node); pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); @@ -3950,7 +4070,6 @@ static void early_kmem_cache_node_alloc(int node) n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false); slab->freelist = get_freepointer(kmem_cache_node, n); slab->inuse = 1; - slab->frozen = 0; kmem_cache_node->node[node] = n; init_kmem_cache_node(n); inc_slabs_node(kmem_cache_node, node, slab->objects); @@ -4112,12 +4231,17 @@ static int calculate_sizes(struct kmem_cache *s) } #ifdef CONFIG_SLUB_DEBUG - if (flags & SLAB_STORE_USER) + if (flags & SLAB_STORE_USER) { /* * Need to store information about allocs and frees after * the object. */ size += 2 * sizeof(struct track); + + /* Save the original kmalloc request size */ + if (flags & SLAB_KMALLOC) + size += sizeof(unsigned int); + } #endif kasan_cache_create(s, &size, &s->flags); @@ -4237,23 +4361,21 @@ static void list_slab_objects(struct kmem_cache *s, struct slab *slab, { #ifdef CONFIG_SLUB_DEBUG void *addr = slab_address(slab); - unsigned long flags; - unsigned long *map; void *p; slab_err(s, slab, text, s->name); - slab_lock(slab, &flags); - map = get_map(s, slab); + spin_lock(&object_map_lock); + __fill_map(object_map, s, slab); + for_each_object(p, s, addr, slab->objects) { - if (!test_bit(__obj_to_index(s, addr, p), map)) { + if (!test_bit(__obj_to_index(s, addr, p), object_map)) { pr_err("Object 0x%p @offset=%tu\n", p, p - addr); print_tracking(s, p); } } - put_map(map); - slab_unlock(slab, &flags); + spin_unlock(&object_map_lock); #endif } @@ -4404,78 +4526,6 @@ static int __init setup_slub_min_objects(char *str) __setup("slub_min_objects=", setup_slub_min_objects); -void *__kmalloc(size_t size, gfp_t flags) -{ - struct kmem_cache *s; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) - return kmalloc_large(size, flags); - - s = kmalloc_slab(size, flags); - - if (unlikely(ZERO_OR_NULL_PTR(s))) - return s; - - ret = slab_alloc(s, NULL, flags, _RET_IP_, size); - - trace_kmalloc(_RET_IP_, ret, s, size, s->size, flags); - - ret = kasan_kmalloc(s, ret, size, flags); - - return ret; -} -EXPORT_SYMBOL(__kmalloc); - -#ifdef CONFIG_NUMA -static void *kmalloc_large_node(size_t size, gfp_t flags, int node) -{ - struct page *page; - void *ptr = NULL; - unsigned int order = get_order(size); - - flags |= __GFP_COMP; - page = alloc_pages_node(node, flags, order); - if (page) { - ptr = page_address(page); - mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, - PAGE_SIZE << order); - } - - return kmalloc_large_node_hook(ptr, size, flags); -} - -void *__kmalloc_node(size_t size, gfp_t flags, int node) -{ - struct kmem_cache *s; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { - ret = kmalloc_large_node(size, flags, node); - - trace_kmalloc_node(_RET_IP_, ret, NULL, - size, PAGE_SIZE << get_order(size), - flags, node); - - return ret; - } - - s = kmalloc_slab(size, flags); - - if (unlikely(ZERO_OR_NULL_PTR(s))) - return s; - - ret = slab_alloc_node(s, NULL, flags, node, _RET_IP_, size); - - trace_kmalloc_node(_RET_IP_, ret, s, size, s->size, flags, node); - - ret = kasan_kmalloc(s, ret, size, flags); - - return ret; -} -EXPORT_SYMBOL(__kmalloc_node); -#endif /* CONFIG_NUMA */ - #ifdef CONFIG_HARDENED_USERCOPY /* * Rejects incorrectly sized objects and objects that are to be copied @@ -4526,43 +4576,6 @@ void __check_heap_object(const void *ptr, unsigned long n, } #endif /* CONFIG_HARDENED_USERCOPY */ -size_t __ksize(const void *object) -{ - struct folio *folio; - - if (unlikely(object == ZERO_SIZE_PTR)) - return 0; - - folio = virt_to_folio(object); - - if (unlikely(!folio_test_slab(folio))) - return folio_size(folio); - - return slab_ksize(folio_slab(folio)->slab_cache); -} -EXPORT_SYMBOL(__ksize); - -void kfree(const void *x) -{ - struct folio *folio; - struct slab *slab; - void *object = (void *)x; - - trace_kfree(_RET_IP_, x); - - if (unlikely(ZERO_OR_NULL_PTR(x))) - return; - - folio = virt_to_folio(x); - if (unlikely(!folio_test_slab(folio))) { - free_large_kmalloc(folio, object); - return; - } - slab = folio_slab(folio); - slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_); -} -EXPORT_SYMBOL(kfree); - #define SHRINK_PROMOTE_MAX 32 /* @@ -4611,6 +4624,7 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) if (free == slab->objects) { list_move(&slab->slab_list, &discard); n->nr_partial--; + dec_slabs_node(s, node, slab->objects); } else if (free <= SHRINK_PROMOTE_MAX) list_move(&slab->slab_list, promote + free - 1); } @@ -4626,7 +4640,7 @@ static int __kmem_cache_do_shrink(struct kmem_cache *s) /* Release empty slabs */ list_for_each_entry_safe(slab, t, &discard, slab_list) - discard_slab(s, slab); + free_slab(s, slab); if (slabs_node(s, node)) ret = 1; @@ -4858,6 +4872,8 @@ void __init kmem_cache_init(void) void __init kmem_cache_init_late(void) { + flushwq = alloc_workqueue("slub_flushwq", WQ_MEM_RECLAIM, 0); + WARN_ON(!flushwq); } struct kmem_cache * @@ -4908,60 +4924,6 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags) return 0; } -void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) -{ - struct kmem_cache *s; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) - return kmalloc_large(size, gfpflags); - - s = kmalloc_slab(size, gfpflags); - - if (unlikely(ZERO_OR_NULL_PTR(s))) - return s; - - ret = slab_alloc(s, NULL, gfpflags, caller, size); - - /* Honor the call site pointer we received. */ - trace_kmalloc(caller, ret, s, size, s->size, gfpflags); - - return ret; -} -EXPORT_SYMBOL(__kmalloc_track_caller); - -#ifdef CONFIG_NUMA -void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, - int node, unsigned long caller) -{ - struct kmem_cache *s; - void *ret; - - if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { - ret = kmalloc_large_node(size, gfpflags, node); - - trace_kmalloc_node(caller, ret, NULL, - size, PAGE_SIZE << get_order(size), - gfpflags, node); - - return ret; - } - - s = kmalloc_slab(size, gfpflags); - - if (unlikely(ZERO_OR_NULL_PTR(s))) - return s; - - ret = slab_alloc_node(s, NULL, gfpflags, node, caller, size); - - /* Honor the call site pointer we received. */ - trace_kmalloc_node(caller, ret, s, size, s->size, gfpflags, node); - - return ret; -} -EXPORT_SYMBOL(__kmalloc_node_track_caller); -#endif - #ifdef CONFIG_SYSFS static int count_inuse(struct slab *slab) { @@ -4980,12 +4942,9 @@ static void validate_slab(struct kmem_cache *s, struct slab *slab, { void *p; void *addr = slab_address(slab); - unsigned long flags; - - slab_lock(slab, &flags); if (!check_slab(s, slab) || !on_freelist(s, slab, NULL)) - goto unlock; + return; /* Now we know that a valid freelist exists */ __fill_map(obj_map, s, slab); @@ -4996,8 +4955,6 @@ static void validate_slab(struct kmem_cache *s, struct slab *slab, if (!check_object(s, slab, p, val)) break; } -unlock: - slab_unlock(slab, &flags); } static int validate_slab_node(struct kmem_cache *s, @@ -5068,6 +5025,7 @@ struct location { depot_stack_handle_t handle; unsigned long count; unsigned long addr; + unsigned long waste; long long sum_time; long min_time; long max_time; @@ -5114,13 +5072,15 @@ static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) } static int add_location(struct loc_track *t, struct kmem_cache *s, - const struct track *track) + const struct track *track, + unsigned int orig_size) { long start, end, pos; struct location *l; - unsigned long caddr, chandle; + unsigned long caddr, chandle, cwaste; unsigned long age = jiffies - track->when; depot_stack_handle_t handle = 0; + unsigned int waste = s->object_size - orig_size; #ifdef CONFIG_STACKDEPOT handle = READ_ONCE(track->handle); @@ -5138,11 +5098,13 @@ static int add_location(struct loc_track *t, struct kmem_cache *s, if (pos == end) break; - caddr = t->loc[pos].addr; - chandle = t->loc[pos].handle; - if ((track->addr == caddr) && (handle == chandle)) { + l = &t->loc[pos]; + caddr = l->addr; + chandle = l->handle; + cwaste = l->waste; + if ((track->addr == caddr) && (handle == chandle) && + (waste == cwaste)) { - l = &t->loc[pos]; l->count++; if (track->when) { l->sum_time += age; @@ -5167,6 +5129,9 @@ static int add_location(struct loc_track *t, struct kmem_cache *s, end = pos; else if (track->addr == caddr && handle < chandle) end = pos; + else if (track->addr == caddr && handle == chandle && + waste < cwaste) + end = pos; else start = pos; } @@ -5190,6 +5155,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s, l->min_pid = track->pid; l->max_pid = track->pid; l->handle = handle; + l->waste = waste; cpumask_clear(to_cpumask(l->cpus)); cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); nodes_clear(l->nodes); @@ -5202,13 +5168,16 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s, unsigned long *obj_map) { void *addr = slab_address(slab); + bool is_alloc = (alloc == TRACK_ALLOC); void *p; __fill_map(obj_map, s, slab); for_each_object(p, s, addr, slab->objects) if (!test_bit(__obj_to_index(s, addr, p), obj_map)) - add_location(t, s, get_track(s, p, alloc)); + add_location(t, s, get_track(s, p, alloc), + is_alloc ? get_orig_size(s, p) : + s->object_size); } #endif /* CONFIG_DEBUG_FS */ #endif /* CONFIG_SLUB_DEBUG */ @@ -5601,7 +5570,7 @@ static ssize_t validate_store(struct kmem_cache *s, { int ret = -EINVAL; - if (buf[0] == '1') { + if (buf[0] == '1' && kmem_cache_debug(s)) { ret = validate_slab_cache(s); if (ret >= 0) ret = length; @@ -5745,6 +5714,29 @@ STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); #endif /* CONFIG_SLUB_STATS */ +#ifdef CONFIG_KFENCE +static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf) +{ + return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE)); +} + +static ssize_t skip_kfence_store(struct kmem_cache *s, + const char *buf, size_t length) +{ + int ret = length; + + if (buf[0] == '0') + s->flags &= ~SLAB_SKIP_KFENCE; + else if (buf[0] == '1') + s->flags |= SLAB_SKIP_KFENCE; + else + ret = -EINVAL; + + return ret; +} +SLAB_ATTR(skip_kfence); +#endif + static struct attribute *slab_attrs[] = { &slab_size_attr.attr, &object_size_attr.attr, @@ -5812,6 +5804,9 @@ static struct attribute *slab_attrs[] = { &failslab_attr.attr, #endif &usersize_attr.attr, +#ifdef CONFIG_KFENCE + &skip_kfence_attr.attr, +#endif NULL }; @@ -5826,7 +5821,6 @@ static ssize_t slab_attr_show(struct kobject *kobj, { struct slab_attribute *attribute; struct kmem_cache *s; - int err; attribute = to_slab_attr(attr); s = to_slab(kobj); @@ -5834,9 +5828,7 @@ static ssize_t slab_attr_show(struct kobject *kobj, if (!attribute->show) return -EIO; - err = attribute->show(s, buf); - - return err; + return attribute->show(s, buf); } static ssize_t slab_attr_store(struct kobject *kobj, @@ -5845,7 +5837,6 @@ static ssize_t slab_attr_store(struct kobject *kobj, { struct slab_attribute *attribute; struct kmem_cache *s; - int err; attribute = to_slab_attr(attr); s = to_slab(kobj); @@ -5853,8 +5844,7 @@ static ssize_t slab_attr_store(struct kobject *kobj, if (!attribute->store) return -EIO; - err = attribute->store(s, buf, len); - return err; + return attribute->store(s, buf, len); } static void kmem_cache_release(struct kobject *k) @@ -5879,7 +5869,7 @@ static inline struct kset *cache_kset(struct kmem_cache *s) return slab_kset; } -#define ID_STR_LENGTH 64 +#define ID_STR_LENGTH 32 /* Create a unique string id for a slab cache: * @@ -5890,7 +5880,8 @@ static char *create_unique_id(struct kmem_cache *s) char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); char *p = name; - BUG_ON(!name); + if (!name) + return ERR_PTR(-ENOMEM); *p++ = ':'; /* @@ -5912,9 +5903,13 @@ static char *create_unique_id(struct kmem_cache *s) *p++ = 'A'; if (p != name + 1) *p++ = '-'; - p += sprintf(p, "%07u", s->size); + p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size); - BUG_ON(p > name + ID_STR_LENGTH - 1); + if (WARN_ON(p > name + ID_STR_LENGTH - 1)) { + kfree(name); + return ERR_PTR(-EINVAL); + } + kmsan_unpoison_memory(name, p - name); return name; } @@ -5948,6 +5943,8 @@ static int sysfs_slab_add(struct kmem_cache *s) * for the symlinks. */ name = create_unique_id(s); + if (IS_ERR(name)) + return PTR_ERR(name); } s->kobj.kset = kset; @@ -6016,6 +6013,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name) al->name = name; al->next = alias_list; alias_list = al; + kmsan_unpoison_memory(al, sizeof(*al)); return 0; } @@ -6078,6 +6076,10 @@ static int slab_debugfs_show(struct seq_file *seq, void *v) else seq_puts(seq, ""); + if (l->waste) + seq_printf(seq, " waste=%lu/%lu", + l->count * l->waste, l->waste); + if (l->sum_time != l->min_time) { seq_printf(seq, " age=%ld/%llu/%ld", l->min_time, div_u64(l->sum_time, l->count), diff --git a/mm/swap.c b/mm/swap.c index 9cee7f6a380942e9bb60e783b4abdea07e9aaa69..955930f41d20c6d491bb3cf0347086443f16b4c6 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -366,7 +366,7 @@ static void folio_activate_drain(int cpu) folio_batch_move_lru(fbatch, folio_activate_fn); } -static void folio_activate(struct folio *folio) +void folio_activate(struct folio *folio) { if (folio_test_lru(folio) && !folio_test_active(folio) && !folio_test_unevictable(folio)) { @@ -385,7 +385,7 @@ static inline void folio_activate_drain(int cpu) { } -static void folio_activate(struct folio *folio) +void folio_activate(struct folio *folio) { struct lruvec *lruvec; @@ -428,6 +428,40 @@ static void __lru_cache_activate_folio(struct folio *folio) local_unlock(&cpu_fbatches.lock); } +#ifdef CONFIG_LRU_GEN +static void folio_inc_refs(struct folio *folio) +{ + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + if (folio_test_unevictable(folio)) + return; + + if (!folio_test_referenced(folio)) { + folio_set_referenced(folio); + return; + } + + if (!folio_test_workingset(folio)) { + folio_set_workingset(folio); + return; + } + + /* see the comment on MAX_NR_TIERS */ + do { + new_flags = old_flags & LRU_REFS_MASK; + if (new_flags == LRU_REFS_MASK) + break; + + new_flags += BIT(LRU_REFS_PGOFF); + new_flags |= old_flags & ~LRU_REFS_MASK; + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); +} +#else +static void folio_inc_refs(struct folio *folio) +{ +} +#endif /* CONFIG_LRU_GEN */ + /* * Mark a page as having seen activity. * @@ -440,6 +474,11 @@ static void __lru_cache_activate_folio(struct folio *folio) */ void folio_mark_accessed(struct folio *folio) { + if (lru_gen_enabled()) { + folio_inc_refs(folio); + return; + } + if (!folio_test_referenced(folio)) { folio_set_referenced(folio); } else if (folio_test_unevictable(folio)) { @@ -484,6 +523,11 @@ void folio_add_lru(struct folio *folio) folio_test_unevictable(folio), folio); VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + /* see the comment in lru_gen_add_folio() */ + if (lru_gen_enabled() && !folio_test_unevictable(folio) && + lru_gen_in_fault() && !(current->flags & PF_MEMALLOC)) + folio_set_active(folio); + folio_get(folio); local_lock(&cpu_fbatches.lock); fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); @@ -493,22 +537,21 @@ void folio_add_lru(struct folio *folio) EXPORT_SYMBOL(folio_add_lru); /** - * lru_cache_add_inactive_or_unevictable - * @page: the page to be added to LRU - * @vma: vma in which page is mapped for determining reclaimability + * folio_add_lru_vma() - Add a folio to the appropate LRU list for this VMA. + * @folio: The folio to be added to the LRU. + * @vma: VMA in which the folio is mapped. * - * Place @page on the inactive or unevictable LRU list, depending on its - * evictability. + * If the VMA is mlocked, @folio is added to the unevictable list. + * Otherwise, it is treated the same way as folio_add_lru(). */ -void lru_cache_add_inactive_or_unevictable(struct page *page, - struct vm_area_struct *vma) +void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma) { - VM_BUG_ON_PAGE(PageLRU(page), page); + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); if (unlikely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED)) - mlock_new_page(page); + mlock_new_page(&folio->page); else - lru_cache_add(page); + folio_add_lru(folio); } /* @@ -575,7 +618,7 @@ static void lru_deactivate_file_fn(struct lruvec *lruvec, struct folio *folio) static void lru_deactivate_fn(struct lruvec *lruvec, struct folio *folio) { - if (folio_test_active(folio) && !folio_test_unevictable(folio)) { + if (!folio_test_unevictable(folio) && (folio_test_active(folio) || lru_gen_enabled())) { long nr_pages = folio_nr_pages(folio); lruvec_del_folio(lruvec, folio); @@ -688,8 +731,8 @@ void deactivate_page(struct page *page) { struct folio *folio = page_folio(page); - if (folio_test_lru(folio) && folio_test_active(folio) && - !folio_test_unevictable(folio)) { + if (folio_test_lru(folio) && !folio_test_unevictable(folio) && + (folio_test_active(folio) || lru_gen_enabled())) { struct folio_batch *fbatch; folio_get(folio); diff --git a/mm/swap.h b/mm/swap.h index 17936e068c1c8fab4dfc652705a813ab5a4d1af5..cc08c459c6190cd33a9e634b270873d4afa210a6 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -18,9 +18,7 @@ static inline void swap_read_unplug(struct swap_iocb *plug) } void swap_write_unplug(struct swap_iocb *sio); int swap_writepage(struct page *page, struct writeback_control *wbc); -void end_swap_bio_write(struct bio *bio); -int __swap_writepage(struct page *page, struct writeback_control *wbc, - bio_end_io_t end_write_func); +int __swap_writepage(struct page *page, struct writeback_control *wbc); /* linux/mm/swap_state.c */ /* One swap address space for each 64M swap space */ @@ -34,16 +32,15 @@ extern struct address_space *swapper_spaces[]; void show_swap_cache_info(void); bool add_to_swap(struct folio *folio); void *get_shadow_from_swap_cache(swp_entry_t entry); -int add_to_swap_cache(struct page *page, swp_entry_t entry, +int add_to_swap_cache(struct folio *folio, swp_entry_t entry, gfp_t gfp, void **shadowp); void __delete_from_swap_cache(struct folio *folio, swp_entry_t entry, void *shadow); void delete_from_swap_cache(struct folio *folio); void clear_shadow_from_swap_cache(int type, unsigned long begin, unsigned long end); -struct page *lookup_swap_cache(swp_entry_t entry, - struct vm_area_struct *vma, - unsigned long addr); +struct folio *swap_cache_get_folio(swp_entry_t entry, + struct vm_area_struct *vma, unsigned long addr); struct page *find_get_incore_page(struct address_space *mapping, pgoff_t index); struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, @@ -101,9 +98,8 @@ static inline int swap_writepage(struct page *p, struct writeback_control *wbc) return 0; } -static inline struct page *lookup_swap_cache(swp_entry_t swp, - struct vm_area_struct *vma, - unsigned long addr) +static inline struct folio *swap_cache_get_folio(swp_entry_t entry, + struct vm_area_struct *vma, unsigned long addr) { return NULL; } @@ -124,7 +120,7 @@ static inline void *get_shadow_from_swap_cache(swp_entry_t entry) return NULL; } -static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, +static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry, gfp_t gfp_mask, void **shadowp) { return -1; diff --git a/mm/swap_cgroup.c b/mm/swap_cgroup.c index 5a9442979a1859227e18914678f7d0e4162c503a..db6c4a26cf5931c314ae5e7de4dd6430c07fab32 100644 --- a/mm/swap_cgroup.c +++ b/mm/swap_cgroup.c @@ -170,6 +170,9 @@ int swap_cgroup_swapon(int type, unsigned long max_pages) unsigned long length; struct swap_cgroup_ctrl *ctrl; + if (mem_cgroup_disabled()) + return 0; + length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); array = vcalloc(length, sizeof(void *)); @@ -204,6 +207,9 @@ void swap_cgroup_swapoff(int type) unsigned long i, length; struct swap_cgroup_ctrl *ctrl; + if (mem_cgroup_disabled()) + return; + mutex_lock(&swap_cgroup_mutex); ctrl = &swap_cgroup_ctrl[type]; map = ctrl->map; diff --git a/mm/swap_slots.c b/mm/swap_slots.c index 10b94d64cc257aadcc826ed4102a94def8fa1c6e..0bec1f705f8e09313e1fcdcf87568cd5bf68da38 100644 --- a/mm/swap_slots.c +++ b/mm/swap_slots.c @@ -343,7 +343,7 @@ repeat: get_swap_pages(1, &entry, 1); out: if (mem_cgroup_try_charge_swap(folio, entry)) { - put_swap_page(&folio->page, entry); + put_swap_folio(folio, entry); entry.val = 0; } return entry; diff --git a/mm/swap_state.c b/mm/swap_state.c index e166051566f42425c3eacccb1451bc47c88bd37b..438d0676c5be2d3463f08890531a7038b0c89c17 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -85,21 +85,21 @@ void *get_shadow_from_swap_cache(swp_entry_t entry) * add_to_swap_cache resembles filemap_add_folio on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */ -int add_to_swap_cache(struct page *page, swp_entry_t entry, +int add_to_swap_cache(struct folio *folio, swp_entry_t entry, gfp_t gfp, void **shadowp) { struct address_space *address_space = swap_address_space(entry); pgoff_t idx = swp_offset(entry); - XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); - unsigned long i, nr = thp_nr_pages(page); + XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio)); + unsigned long i, nr = folio_nr_pages(folio); void *old; - VM_BUG_ON_PAGE(!PageLocked(page), page); - VM_BUG_ON_PAGE(PageSwapCache(page), page); - VM_BUG_ON_PAGE(!PageSwapBacked(page), page); + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio); - page_ref_add(page, nr); - SetPageSwapCache(page); + folio_ref_add(folio, nr); + folio_set_swapcache(folio); do { xas_lock_irq(&xas); @@ -107,19 +107,19 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, if (xas_error(&xas)) goto unlock; for (i = 0; i < nr; i++) { - VM_BUG_ON_PAGE(xas.xa_index != idx + i, page); + VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio); old = xas_load(&xas); if (xa_is_value(old)) { if (shadowp) *shadowp = old; } - set_page_private(page + i, entry.val + i); - xas_store(&xas, page); + set_page_private(folio_page(folio, i), entry.val + i); + xas_store(&xas, folio); xas_next(&xas); } address_space->nrpages += nr; - __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); - __mod_lruvec_page_state(page, NR_SWAPCACHE, nr); + __node_stat_mod_folio(folio, NR_FILE_PAGES, nr); + __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr); unlock: xas_unlock_irq(&xas); } while (xas_nomem(&xas, gfp)); @@ -127,8 +127,8 @@ unlock: if (!xas_error(&xas)) return 0; - ClearPageSwapCache(page); - page_ref_sub(page, nr); + folio_clear_swapcache(folio); + folio_ref_sub(folio, nr); return xas_error(&xas); } @@ -151,7 +151,7 @@ void __delete_from_swap_cache(struct folio *folio, for (i = 0; i < nr; i++) { void *entry = xas_store(&xas, shadow); - VM_BUG_ON_FOLIO(entry != folio, folio); + VM_BUG_ON_PAGE(entry != folio, entry); set_page_private(folio_page(folio, i), 0); xas_next(&xas); } @@ -194,7 +194,7 @@ bool add_to_swap(struct folio *folio) /* * Add it to the swap cache. */ - err = add_to_swap_cache(&folio->page, entry, + err = add_to_swap_cache(folio, entry, __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL); if (err) /* @@ -218,7 +218,7 @@ bool add_to_swap(struct folio *folio) return true; fail: - put_swap_page(&folio->page, entry); + put_swap_folio(folio, entry); return false; } @@ -237,7 +237,7 @@ void delete_from_swap_cache(struct folio *folio) __delete_from_swap_cache(folio, entry, NULL); xa_unlock_irq(&address_space->i_pages); - put_swap_page(&folio->page, entry); + put_swap_folio(folio, entry); folio_ref_sub(folio, folio_nr_pages(folio)); } @@ -272,16 +272,19 @@ void clear_shadow_from_swap_cache(int type, unsigned long begin, /* * If we are the only user, then try to free up the swap cache. * - * Its ok to check for PageSwapCache without the page lock + * Its ok to check the swapcache flag without the folio lock * here because we are going to recheck again inside - * try_to_free_swap() _with_ the lock. + * folio_free_swap() _with_ the lock. * - Marcelo */ void free_swap_cache(struct page *page) { - if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) { - try_to_free_swap(page); - unlock_page(page); + struct folio *folio = page_folio(page); + + if (folio_test_swapcache(folio) && !folio_mapped(folio) && + folio_trylock(folio)) { + folio_free_swap(folio); + folio_unlock(folio); } } @@ -317,24 +320,24 @@ static inline bool swap_use_vma_readahead(void) } /* - * Lookup a swap entry in the swap cache. A found page will be returned + * Lookup a swap entry in the swap cache. A found folio will be returned * unlocked and with its refcount incremented - we rely on the kernel - * lock getting page table operations atomic even if we drop the page + * lock getting page table operations atomic even if we drop the folio * lock before returning. */ -struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, - unsigned long addr) +struct folio *swap_cache_get_folio(swp_entry_t entry, + struct vm_area_struct *vma, unsigned long addr) { - struct page *page; + struct folio *folio; struct swap_info_struct *si; si = get_swap_device(entry); if (!si) return NULL; - page = find_get_page(swap_address_space(entry), swp_offset(entry)); + folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry)); put_swap_device(si); - if (page) { + if (folio) { bool vma_ra = swap_use_vma_readahead(); bool readahead; @@ -342,10 +345,10 @@ struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, * At the moment, we don't support PG_readahead for anon THP * so let's bail out rather than confusing the readahead stat. */ - if (unlikely(PageTransCompound(page))) - return page; + if (unlikely(folio_test_large(folio))) + return folio; - readahead = TestClearPageReadahead(page); + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; int win, hits; @@ -366,7 +369,7 @@ struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, } } - return page; + return folio; } /** @@ -411,7 +414,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, bool *new_page_allocated) { struct swap_info_struct *si; - struct page *page; + struct folio *folio; void *shadow = NULL; *new_page_allocated = false; @@ -420,17 +423,17 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, int err; /* * First check the swap cache. Since this is normally - * called after lookup_swap_cache() failed, re-calling + * called after swap_cache_get_folio() failed, re-calling * that would confuse statistics. */ si = get_swap_device(entry); if (!si) return NULL; - page = find_get_page(swap_address_space(entry), - swp_offset(entry)); + folio = filemap_get_folio(swap_address_space(entry), + swp_offset(entry)); put_swap_device(si); - if (page) - return page; + if (folio) + return folio_file_page(folio, swp_offset(entry)); /* * Just skip read ahead for unused swap slot. @@ -448,8 +451,8 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will * cause any racers to loop around until we add it to cache. */ - page = alloc_page_vma(gfp_mask, vma, addr); - if (!page) + folio = vma_alloc_folio(gfp_mask, 0, vma, addr, false); + if (!folio) return NULL; /* @@ -459,7 +462,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, if (!err) break; - put_page(page); + folio_put(folio); if (err != -EEXIST) return NULL; @@ -477,30 +480,30 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, * The swap entry is ours to swap in. Prepare the new page. */ - __SetPageLocked(page); - __SetPageSwapBacked(page); + __folio_set_locked(folio); + __folio_set_swapbacked(folio); - if (mem_cgroup_swapin_charge_page(page, NULL, gfp_mask, entry)) + if (mem_cgroup_swapin_charge_folio(folio, NULL, gfp_mask, entry)) goto fail_unlock; /* May fail (-ENOMEM) if XArray node allocation failed. */ - if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) + if (add_to_swap_cache(folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) goto fail_unlock; mem_cgroup_swapin_uncharge_swap(entry); if (shadow) - workingset_refault(page_folio(page), shadow); + workingset_refault(folio, shadow); - /* Caller will initiate read into locked page */ - lru_cache_add(page); + /* Caller will initiate read into locked folio */ + folio_add_lru(folio); *new_page_allocated = true; - return page; + return &folio->page; fail_unlock: - put_swap_page(page, entry); - unlock_page(page); - put_page(page); + put_swap_folio(folio, entry); + folio_unlock(folio); + folio_put(folio); return NULL; } diff --git a/mm/swapfile.c b/mm/swapfile.c index 1fdccd2f1422eba16a3770ed36657f22e8e2fdab..5fc1237a9f214d45a30984ea45a842f8e62d522b 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -63,6 +63,10 @@ EXPORT_SYMBOL_GPL(nr_swap_pages); /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */ long total_swap_pages; static int least_priority = -1; +unsigned long swapfile_maximum_size; +#ifdef CONFIG_MIGRATION +bool swap_migration_ad_supported; +#endif /* CONFIG_MIGRATION */ static const char Bad_file[] = "Bad swap file entry "; static const char Unused_file[] = "Unused swap file entry "; @@ -128,27 +132,27 @@ static int __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset, unsigned long flags) { swp_entry_t entry = swp_entry(si->type, offset); - struct page *page; + struct folio *folio; int ret = 0; - page = find_get_page(swap_address_space(entry), offset); - if (!page) + folio = filemap_get_folio(swap_address_space(entry), offset); + if (!folio) return 0; /* * When this function is called from scan_swap_map_slots() and it's - * called by vmscan.c at reclaiming pages. So, we hold a lock on a page, + * called by vmscan.c at reclaiming folios. So we hold a folio lock * here. We have to use trylock for avoiding deadlock. This is a special - * case and you should use try_to_free_swap() with explicit lock_page() + * case and you should use folio_free_swap() with explicit folio_lock() * in usual operations. */ - if (trylock_page(page)) { + if (folio_trylock(folio)) { if ((flags & TTRS_ANYWAY) || - ((flags & TTRS_UNMAPPED) && !page_mapped(page)) || - ((flags & TTRS_FULL) && mem_cgroup_swap_full(page))) - ret = try_to_free_swap(page); - unlock_page(page); + ((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) || + ((flags & TTRS_FULL) && mem_cgroup_swap_full(folio))) + ret = folio_free_swap(folio); + folio_unlock(folio); } - put_page(page); + folio_put(folio); return ret; } @@ -1328,7 +1332,7 @@ void swap_free(swp_entry_t entry) /* * Called after dropping swapcache to decrease refcnt to swap entries. */ -void put_swap_page(struct page *page, swp_entry_t entry) +void put_swap_folio(struct folio *folio, swp_entry_t entry) { unsigned long offset = swp_offset(entry); unsigned long idx = offset / SWAPFILE_CLUSTER; @@ -1337,7 +1341,7 @@ void put_swap_page(struct page *page, swp_entry_t entry) unsigned char *map; unsigned int i, free_entries = 0; unsigned char val; - int size = swap_entry_size(thp_nr_pages(page)); + int size = swap_entry_size(folio_nr_pages(folio)); si = _swap_info_get(entry); if (!si) @@ -1427,30 +1431,6 @@ void swapcache_free_entries(swp_entry_t *entries, int n) spin_unlock(&p->lock); } -/* - * How many references to page are currently swapped out? - * This does not give an exact answer when swap count is continued, - * but does include the high COUNT_CONTINUED flag to allow for that. - */ -static int page_swapcount(struct page *page) -{ - int count = 0; - struct swap_info_struct *p; - struct swap_cluster_info *ci; - swp_entry_t entry; - unsigned long offset; - - entry.val = page_private(page); - p = _swap_info_get(entry); - if (p) { - offset = swp_offset(entry); - ci = lock_cluster_or_swap_info(p, offset); - count = swap_count(p->swap_map[offset]); - unlock_cluster_or_swap_info(p, ci); - } - return count; -} - int __swap_count(swp_entry_t entry) { struct swap_info_struct *si; @@ -1465,11 +1445,16 @@ int __swap_count(swp_entry_t entry) return count; } +/* + * How many references to @entry are currently swapped out? + * This does not give an exact answer when swap count is continued, + * but does include the high COUNT_CONTINUED flag to allow for that. + */ static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) { - int count = 0; pgoff_t offset = swp_offset(entry); struct swap_cluster_info *ci; + int count; ci = lock_cluster_or_swap_info(si, offset); count = swap_count(si->swap_map[offset]); @@ -1570,56 +1555,59 @@ unlock_out: static bool folio_swapped(struct folio *folio) { - swp_entry_t entry; - struct swap_info_struct *si; + swp_entry_t entry = folio_swap_entry(folio); + struct swap_info_struct *si = _swap_info_get(entry); + + if (!si) + return false; if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio))) - return page_swapcount(&folio->page) != 0; + return swap_swapcount(si, entry) != 0; - entry = folio_swap_entry(folio); - si = _swap_info_get(entry); - if (si) - return swap_page_trans_huge_swapped(si, entry); - return false; + return swap_page_trans_huge_swapped(si, entry); } -/* - * If swap is getting full, or if there are no more mappings of this page, - * then try_to_free_swap is called to free its swap space. +/** + * folio_free_swap() - Free the swap space used for this folio. + * @folio: The folio to remove. + * + * If swap is getting full, or if there are no more mappings of this folio, + * then call folio_free_swap to free its swap space. + * + * Return: true if we were able to release the swap space. */ -int try_to_free_swap(struct page *page) +bool folio_free_swap(struct folio *folio) { - struct folio *folio = page_folio(page); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); if (!folio_test_swapcache(folio)) - return 0; + return false; if (folio_test_writeback(folio)) - return 0; + return false; if (folio_swapped(folio)) - return 0; + return false; /* * Once hibernation has begun to create its image of memory, - * there's a danger that one of the calls to try_to_free_swap() + * there's a danger that one of the calls to folio_free_swap() * - most probably a call from __try_to_reclaim_swap() while * hibernation is allocating its own swap pages for the image, * but conceivably even a call from memory reclaim - will free - * the swap from a page which has already been recorded in the - * image as a clean swapcache page, and then reuse its swap for + * the swap from a folio which has already been recorded in the + * image as a clean swapcache folio, and then reuse its swap for * another page of the image. On waking from hibernation, the - * original page might be freed under memory pressure, then + * original folio might be freed under memory pressure, then * later read back in from swap, now with the wrong data. * * Hibernation suspends storage while it is writing the image * to disk so check that here. */ if (pm_suspended_storage()) - return 0; + return false; delete_from_swap_cache(folio); folio_set_dirty(folio); - return 1; + return true; } /* @@ -1770,8 +1758,9 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) * force COW, vm_page_prot omits write permission from any private vma. */ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct page *page) + unsigned long addr, swp_entry_t entry, struct folio *folio) { + struct page *page = folio_file_page(folio, swp_offset(entry)); struct page *swapcache; spinlock_t *ptl; pte_t *pte, new_pte; @@ -1843,17 +1832,18 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, unsigned int type) { - struct page *page; swp_entry_t entry; pte_t *pte; struct swap_info_struct *si; - unsigned long offset; int ret = 0; volatile unsigned char *swap_map; si = swap_info[type]; pte = pte_offset_map(pmd, addr); do { + struct folio *folio; + unsigned long offset; + if (!is_swap_pte(*pte)) continue; @@ -1864,8 +1854,9 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, offset = swp_offset(entry); pte_unmap(pte); swap_map = &si->swap_map[offset]; - page = lookup_swap_cache(entry, vma, addr); - if (!page) { + folio = swap_cache_get_folio(entry, vma, addr); + if (!folio) { + struct page *page; struct vm_fault vmf = { .vma = vma, .address = addr, @@ -1875,25 +1866,27 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, &vmf); + if (page) + folio = page_folio(page); } - if (!page) { + if (!folio) { if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD) goto try_next; return -ENOMEM; } - lock_page(page); - wait_on_page_writeback(page); - ret = unuse_pte(vma, pmd, addr, entry, page); + folio_lock(folio); + folio_wait_writeback(folio); + ret = unuse_pte(vma, pmd, addr, entry, folio); if (ret < 0) { - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); goto out; } - try_to_free_swap(page); - unlock_page(page); - put_page(page); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); try_next: pte = pte_offset_map(pmd, addr); } while (pte++, addr += PAGE_SIZE, addr != end); @@ -1990,14 +1983,16 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type) { struct vm_area_struct *vma; int ret = 0; + VMA_ITERATOR(vmi, mm, 0); mmap_read_lock(mm); - for (vma = mm->mmap; vma; vma = vma->vm_next) { + for_each_vma(vmi, vma) { if (vma->anon_vma) { ret = unuse_vma(vma, type); if (ret) break; } + cond_resched(); } mmap_read_unlock(mm); @@ -2042,7 +2037,7 @@ static int try_to_unuse(unsigned int type) struct list_head *p; int retval = 0; struct swap_info_struct *si = swap_info[type]; - struct page *page; + struct folio *folio; swp_entry_t entry; unsigned int i; @@ -2092,21 +2087,21 @@ retry: (i = find_next_to_unuse(si, i)) != 0) { entry = swp_entry(type, i); - page = find_get_page(swap_address_space(entry), i); - if (!page) + folio = filemap_get_folio(swap_address_space(entry), i); + if (!folio) continue; /* - * It is conceivable that a racing task removed this page from - * swap cache just before we acquired the page lock. The page + * It is conceivable that a racing task removed this folio from + * swap cache just before we acquired the page lock. The folio * might even be back in swap cache on another swap area. But - * that is okay, try_to_free_swap() only removes stale pages. + * that is okay, folio_free_swap() only removes stale folios. */ - lock_page(page); - wait_on_page_writeback(page); - try_to_free_swap(page); - unlock_page(page); - put_page(page); + folio_lock(folio); + folio_wait_writeback(folio); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); } /* @@ -2816,7 +2811,7 @@ unsigned long generic_max_swapfile_size(void) } /* Can be overridden by an architecture for additional checks. */ -__weak unsigned long max_swapfile_size(void) +__weak unsigned long arch_max_swapfile_size(void) { return generic_max_swapfile_size(); } @@ -2856,7 +2851,7 @@ static unsigned long read_swap_header(struct swap_info_struct *p, p->cluster_next = 1; p->cluster_nr = 0; - maxpages = max_swapfile_size(); + maxpages = swapfile_maximum_size; last_page = swap_header->info.last_page; if (!last_page) { pr_warn("Empty swap-file\n"); @@ -3655,7 +3650,7 @@ void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], avail_lists[nid]) { if (si->bdev) { - blkcg_schedule_throttle(bdev_get_queue(si->bdev), true); + blkcg_schedule_throttle(si->bdev->bd_disk, true); break; } } @@ -3677,6 +3672,13 @@ static int __init swapfile_init(void) for_each_node(nid) plist_head_init(&swap_avail_heads[nid]); + swapfile_maximum_size = arch_max_swapfile_size(); + +#ifdef CONFIG_MIGRATION + if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS)) + swap_migration_ad_supported = true; +#endif /* CONFIG_MIGRATION */ + return 0; } subsys_initcall(swapfile_init); diff --git a/mm/truncate.c b/mm/truncate.c index 0b0708bf935f3e9b80a8042157c38b9219745401..c0be77e5c0083c6b993dd39247dd203e0ff627da 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -240,7 +240,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end) folio_invalidate(folio, offset, length); if (!folio_test_large(folio)) return true; - if (split_huge_page(&folio->page) == 0) + if (split_folio(folio) == 0) return true; if (folio_test_dirty(folio)) return false; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 07d3befc80e4134dd6b54be127197ffaad5c10e2..e24e8a47ce8a2805b8e5513845cd9912c370d02f 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -243,20 +243,22 @@ static int mcontinue_atomic_pte(struct mm_struct *dst_mm, { struct inode *inode = file_inode(dst_vma->vm_file); pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); + struct folio *folio; struct page *page; int ret; - ret = shmem_getpage(inode, pgoff, &page, SGP_NOALLOC); - /* Our caller expects us to return -EFAULT if we failed to find page. */ + ret = shmem_get_folio(inode, pgoff, &folio, SGP_NOALLOC); + /* Our caller expects us to return -EFAULT if we failed to find folio */ if (ret == -ENOENT) ret = -EFAULT; if (ret) goto out; - if (!page) { + if (!folio) { ret = -EFAULT; goto out; } + page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; goto out_release; @@ -267,13 +269,13 @@ static int mcontinue_atomic_pte(struct mm_struct *dst_mm, if (ret) goto out_release; - unlock_page(page); + folio_unlock(folio); ret = 0; out: return ret; out_release: - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); goto out; } @@ -377,30 +379,30 @@ retry: BUG_ON(dst_addr >= dst_start + len); /* - * Serialize via i_mmap_rwsem and hugetlb_fault_mutex. - * i_mmap_rwsem ensures the dst_pte remains valid even + * Serialize via vma_lock and hugetlb_fault_mutex. + * vma_lock ensures the dst_pte remains valid even * in the case of shared pmds. fault mutex prevents * races with other faulting threads. */ - mapping = dst_vma->vm_file->f_mapping; - i_mmap_lock_read(mapping); idx = linear_page_index(dst_vma, dst_addr); + mapping = dst_vma->vm_file->f_mapping; hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); + hugetlb_vma_lock_read(dst_vma); err = -ENOMEM; dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); if (!dst_pte) { + hugetlb_vma_unlock_read(dst_vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); goto out_unlock; } if (mode != MCOPY_ATOMIC_CONTINUE && !huge_pte_none_mostly(huge_ptep_get(dst_pte))) { err = -EEXIST; + hugetlb_vma_unlock_read(dst_vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); goto out_unlock; } @@ -408,8 +410,8 @@ retry: dst_addr, src_addr, mode, &page, wp_copy); + hugetlb_vma_unlock_read(dst_vma); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - i_mmap_unlock_read(mapping); cond_resched(); @@ -703,14 +705,29 @@ ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start, mmap_changing, 0); } +void uffd_wp_range(struct mm_struct *dst_mm, struct vm_area_struct *dst_vma, + unsigned long start, unsigned long len, bool enable_wp) +{ + struct mmu_gather tlb; + pgprot_t newprot; + + if (enable_wp) + newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE)); + else + newprot = vm_get_page_prot(dst_vma->vm_flags); + + tlb_gather_mmu(&tlb, dst_mm); + change_protection(&tlb, dst_vma, start, start + len, newprot, + enable_wp ? MM_CP_UFFD_WP : MM_CP_UFFD_WP_RESOLVE); + tlb_finish_mmu(&tlb); +} + int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, unsigned long len, bool enable_wp, atomic_t *mmap_changing) { struct vm_area_struct *dst_vma; unsigned long page_mask; - struct mmu_gather tlb; - pgprot_t newprot; int err; /* @@ -750,15 +767,7 @@ int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start, goto out_unlock; } - if (enable_wp) - newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE)); - else - newprot = vm_get_page_prot(dst_vma->vm_flags); - - tlb_gather_mmu(&tlb, dst_mm); - change_protection(&tlb, dst_vma, start, start + len, newprot, - enable_wp ? MM_CP_UFFD_WP : MM_CP_UFFD_WP_RESOLVE); - tlb_finish_mmu(&tlb); + uffd_wp_range(dst_mm, dst_vma, start, len, enable_wp); err = 0; out_unlock: diff --git a/mm/util.c b/mm/util.c index c9439c66d8cf4d496b9cdaa224501bd8c803059c..12984e76767ebd2b8e488a00d485fb4b82c20e41 100644 --- a/mm/util.c +++ b/mm/util.c @@ -272,38 +272,6 @@ void *memdup_user_nul(const void __user *src, size_t len) } EXPORT_SYMBOL(memdup_user_nul); -void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma, - struct vm_area_struct *prev) -{ - struct vm_area_struct *next; - - vma->vm_prev = prev; - if (prev) { - next = prev->vm_next; - prev->vm_next = vma; - } else { - next = mm->mmap; - mm->mmap = vma; - } - vma->vm_next = next; - if (next) - next->vm_prev = vma; -} - -void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma) -{ - struct vm_area_struct *prev, *next; - - next = vma->vm_next; - prev = vma->vm_prev; - if (prev) - prev->vm_next = next; - else - mm->mmap = next; - if (next) - next->vm_prev = prev; -} - /* Check if the vma is being used as a stack by this task */ int vma_is_stack_for_current(struct vm_area_struct *vma) { @@ -619,6 +587,10 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node) if (ret || size <= PAGE_SIZE) return ret; + /* non-sleeping allocations are not supported by vmalloc */ + if (!gfpflags_allow_blocking(flags)) + return NULL; + /* Don't even allow crazy sizes */ if (unlikely(size > INT_MAX)) { WARN_ON_ONCE(!(flags & __GFP_NOWARN)); @@ -850,10 +822,10 @@ int folio_mapcount(struct folio *folio) return atomic_read(&folio->_mapcount) + 1; compound = folio_entire_mapcount(folio); - nr = folio_nr_pages(folio); if (folio_test_hugetlb(folio)) return compound; ret = compound; + nr = folio_nr_pages(folio); for (i = 0; i < nr; i++) ret += atomic_read(&folio_page(folio, i)->_mapcount) + 1; /* File pages has compound_mapcount included in _mapcount */ @@ -1052,6 +1024,8 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) if (percpu_counter_read_positive(&vm_committed_as) < allowed) return 0; error: + pr_warn_ratelimited("%s: pid: %d, comm: %s, no enough memory for the allocation\n", + __func__, current->pid, current->comm); vm_unacct_memory(pages); return -ENOMEM; diff --git a/mm/vmacache.c b/mm/vmacache.c deleted file mode 100644 index 01a6e6688ec1fbb8b30e0025e4fab1374664ed9d..0000000000000000000000000000000000000000 --- a/mm/vmacache.c +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2014 Davidlohr Bueso. - */ -#include -#include -#include -#include - -/* - * Hash based on the pmd of addr if configured with MMU, which provides a good - * hit rate for workloads with spatial locality. Otherwise, use pages. - */ -#ifdef CONFIG_MMU -#define VMACACHE_SHIFT PMD_SHIFT -#else -#define VMACACHE_SHIFT PAGE_SHIFT -#endif -#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) - -/* - * This task may be accessing a foreign mm via (for example) - * get_user_pages()->find_vma(). The vmacache is task-local and this - * task's vmacache pertains to a different mm (ie, its own). There is - * nothing we can do here. - * - * Also handle the case where a kernel thread has adopted this mm via - * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. - */ -static inline bool vmacache_valid_mm(struct mm_struct *mm) -{ - return current->mm == mm && !(current->flags & PF_KTHREAD); -} - -void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) -{ - if (vmacache_valid_mm(newvma->vm_mm)) - current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; -} - -static bool vmacache_valid(struct mm_struct *mm) -{ - struct task_struct *curr; - - if (!vmacache_valid_mm(mm)) - return false; - - curr = current; - if (mm->vmacache_seqnum != curr->vmacache.seqnum) { - /* - * First attempt will always be invalid, initialize - * the new cache for this task here. - */ - curr->vmacache.seqnum = mm->vmacache_seqnum; - vmacache_flush(curr); - return false; - } - return true; -} - -struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) -{ - int idx = VMACACHE_HASH(addr); - int i; - - count_vm_vmacache_event(VMACACHE_FIND_CALLS); - - if (!vmacache_valid(mm)) - return NULL; - - for (i = 0; i < VMACACHE_SIZE; i++) { - struct vm_area_struct *vma = current->vmacache.vmas[idx]; - - if (vma) { -#ifdef CONFIG_DEBUG_VM_VMACACHE - if (WARN_ON_ONCE(vma->vm_mm != mm)) - break; -#endif - if (vma->vm_start <= addr && vma->vm_end > addr) { - count_vm_vmacache_event(VMACACHE_FIND_HITS); - return vma; - } - } - if (++idx == VMACACHE_SIZE) - idx = 0; - } - - return NULL; -} - -#ifndef CONFIG_MMU -struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, - unsigned long start, - unsigned long end) -{ - int idx = VMACACHE_HASH(start); - int i; - - count_vm_vmacache_event(VMACACHE_FIND_CALLS); - - if (!vmacache_valid(mm)) - return NULL; - - for (i = 0; i < VMACACHE_SIZE; i++) { - struct vm_area_struct *vma = current->vmacache.vmas[idx]; - - if (vma && vma->vm_start == start && vma->vm_end == end) { - count_vm_vmacache_event(VMACACHE_FIND_HITS); - return vma; - } - if (++idx == VMACACHE_SIZE) - idx = 0; - } - - return NULL; -} -#endif diff --git a/mm/vmalloc.c b/mm/vmalloc.c index dd6cdb2011953d6eac7455055bd589280d442ed7..ccaa461998f3c37981df44b89a4ecdcd0c2f4f76 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -320,6 +320,9 @@ int ioremap_page_range(unsigned long addr, unsigned long end, err = vmap_range_noflush(addr, end, phys_addr, pgprot_nx(prot), ioremap_max_page_shift); flush_cache_vmap(addr, end); + if (!err) + kmsan_ioremap_page_range(addr, end, phys_addr, prot, + ioremap_max_page_shift); return err; } @@ -416,7 +419,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, * * This is an internal function only. Do not use outside mm/. */ -void vunmap_range_noflush(unsigned long start, unsigned long end) +void __vunmap_range_noflush(unsigned long start, unsigned long end) { unsigned long next; pgd_t *pgd; @@ -438,6 +441,12 @@ void vunmap_range_noflush(unsigned long start, unsigned long end) arch_sync_kernel_mappings(start, end); } +void vunmap_range_noflush(unsigned long start, unsigned long end) +{ + kmsan_vunmap_range_noflush(start, end); + __vunmap_range_noflush(start, end); +} + /** * vunmap_range - unmap kernel virtual addresses * @addr: start of the VM area to unmap @@ -575,7 +584,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, * * This is an internal function only. Do not use outside mm/. */ -int vmap_pages_range_noflush(unsigned long addr, unsigned long end, +int __vmap_pages_range_noflush(unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, unsigned int page_shift) { unsigned int i, nr = (end - addr) >> PAGE_SHIFT; @@ -590,7 +599,7 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end, int err; err = vmap_range_noflush(addr, addr + (1UL << page_shift), - __pa(page_address(pages[i])), prot, + page_to_phys(pages[i]), prot, page_shift); if (err) return err; @@ -601,6 +610,13 @@ int vmap_pages_range_noflush(unsigned long addr, unsigned long end, return 0; } +int vmap_pages_range_noflush(unsigned long addr, unsigned long end, + pgprot_t prot, struct page **pages, unsigned int page_shift) +{ + kmsan_vmap_pages_range_noflush(addr, end, prot, pages, page_shift); + return __vmap_pages_range_noflush(addr, end, prot, pages, page_shift); +} + /** * vmap_pages_range - map pages to a kernel virtual address * @addr: start of the VM area to map @@ -1300,12 +1316,12 @@ find_vmap_lowest_match(struct rb_root *root, unsigned long size, #include static struct vmap_area * -find_vmap_lowest_linear_match(unsigned long size, +find_vmap_lowest_linear_match(struct list_head *head, unsigned long size, unsigned long align, unsigned long vstart) { struct vmap_area *va; - list_for_each_entry(va, &free_vmap_area_list, list) { + list_for_each_entry(va, head, list) { if (!is_within_this_va(va, size, align, vstart)) continue; @@ -1316,7 +1332,8 @@ find_vmap_lowest_linear_match(unsigned long size, } static void -find_vmap_lowest_match_check(unsigned long size, unsigned long align) +find_vmap_lowest_match_check(struct rb_root *root, struct list_head *head, + unsigned long size, unsigned long align) { struct vmap_area *va_1, *va_2; unsigned long vstart; @@ -1325,8 +1342,8 @@ find_vmap_lowest_match_check(unsigned long size, unsigned long align) get_random_bytes(&rnd, sizeof(rnd)); vstart = VMALLOC_START + rnd; - va_1 = find_vmap_lowest_match(size, align, vstart, false); - va_2 = find_vmap_lowest_linear_match(size, align, vstart); + va_1 = find_vmap_lowest_match(root, size, align, vstart, false); + va_2 = find_vmap_lowest_linear_match(head, size, align, vstart); if (va_1 != va_2) pr_emerg("not lowest: t: 0x%p, l: 0x%p, v: 0x%lx\n", @@ -1513,7 +1530,7 @@ __alloc_vmap_area(struct rb_root *root, struct list_head *head, return vend; #if DEBUG_AUGMENT_LOWEST_MATCH_CHECK - find_vmap_lowest_match_check(size, align); + find_vmap_lowest_match_check(root, head, size, align); #endif return nva_start_addr; diff --git a/mm/vmscan.c b/mm/vmscan.c index b2b1431352dcd8ea791cef0a5fec25602a3b94a3..04d8b88e521648752342e3319d3136a02a838a5a 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -43,12 +43,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include @@ -85,7 +90,7 @@ struct scan_control { unsigned long anon_cost; unsigned long file_cost; - /* Can active pages be deactivated as part of reclaim? */ + /* Can active folios be deactivated as part of reclaim? */ #define DEACTIVATE_ANON 1 #define DEACTIVATE_FILE 2 unsigned int may_deactivate:2; @@ -95,10 +100,10 @@ struct scan_control { /* Writepage batching in laptop mode; RECLAIM_WRITE */ unsigned int may_writepage:1; - /* Can mapped pages be reclaimed? */ + /* Can mapped folios be reclaimed? */ unsigned int may_unmap:1; - /* Can pages be swapped as part of reclaim? */ + /* Can folios be swapped as part of reclaim? */ unsigned int may_swap:1; /* Proactive reclaim invoked by userspace through memory.reclaim */ @@ -123,19 +128,25 @@ struct scan_control { /* There is easily reclaimable cold cache in the current node */ unsigned int cache_trim_mode:1; - /* The file pages on the current node are dangerously low */ + /* The file folios on the current node are dangerously low */ unsigned int file_is_tiny:1; /* Always discard instead of demoting to lower tier memory */ unsigned int no_demotion:1; +#ifdef CONFIG_LRU_GEN + /* help kswapd make better choices among multiple memcgs */ + unsigned int memcgs_need_aging:1; + unsigned long last_reclaimed; +#endif + /* Allocation order */ s8 order; /* Scan (total_size >> priority) pages at once */ s8 priority; - /* The highest zone to isolate pages for reclaim from */ + /* The highest zone to isolate folios for reclaim from */ s8 reclaim_idx; /* This context's GFP mask */ @@ -443,7 +454,7 @@ static bool cgroup_reclaim(struct scan_control *sc) * * The normal page dirty throttling mechanism in balance_dirty_pages() is * completely broken with the legacy memcg and direct stalling in - * shrink_page_list() is used for throttling instead, which lacks all the + * shrink_folio_list() is used for throttling instead, which lacks all the * niceties such as fairness, adaptive pausing, bandwidth proportional * allocation and configurability. * @@ -564,9 +575,9 @@ static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, } /* - * This misses isolated pages which are not accounted for to save counters. + * This misses isolated folios which are not accounted for to save counters. * As the data only determines if reclaim or compaction continues, it is - * not expected that isolated pages will be a dominating factor. + * not expected that isolated folios will be a dominating factor. */ unsigned long zone_reclaimable_pages(struct zone *zone) { @@ -1039,9 +1050,9 @@ void drop_slab(void) static inline int is_page_cache_freeable(struct folio *folio) { /* - * A freeable page cache page is referenced only by the caller - * that isolated the page, the page cache and optional buffer - * heads at page->private. + * A freeable page cache folio is referenced only by the caller + * that isolated the folio, the page cache and optional filesystem + * private data at folio->private. */ return folio_ref_count(folio) - folio_test_private(folio) == 1 + folio_nr_pages(folio); @@ -1081,8 +1092,8 @@ static bool skip_throttle_noprogress(pg_data_t *pgdat) return true; /* - * If there are a lot of dirty/writeback pages then do not - * throttle as throttling will occur when the pages cycle + * If there are a lot of dirty/writeback folios then do not + * throttle as throttling will occur when the folios cycle * towards the end of the LRU if still under writeback. */ for (i = 0; i < MAX_NR_ZONES; i++) { @@ -1125,7 +1136,7 @@ void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) * short. Failing to make progress or waiting on writeback are * potentially long-lived events so use a longer timeout. This is shaky * logic as a failure to make progress could be due to anything from - * writeback to a slow device to excessive references pages at the tail + * writeback to a slow device to excessive referenced folios at the tail * of the inactive LRU. */ switch(reason) { @@ -1171,8 +1182,8 @@ void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) } /* - * Account for pages written if tasks are throttled waiting on dirty - * pages to clean. If enough pages have been cleaned since throttling + * Account for folios written if tasks are throttled waiting on dirty + * folios to clean. If enough folios have been cleaned since throttling * started then wakeup the throttled tasks. */ void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, @@ -1198,18 +1209,18 @@ void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, /* possible outcome of pageout() */ typedef enum { - /* failed to write page out, page is locked */ + /* failed to write folio out, folio is locked */ PAGE_KEEP, - /* move page to the active list, page is locked */ + /* move folio to the active list, folio is locked */ PAGE_ACTIVATE, - /* page has been sent to the disk successfully, page is unlocked */ + /* folio has been sent to the disk successfully, folio is unlocked */ PAGE_SUCCESS, - /* page is clean and locked */ + /* folio is clean and locked */ PAGE_CLEAN, } pageout_t; /* - * pageout is called by shrink_page_list() for each dirty page. + * pageout is called by shrink_folio_list() for each dirty folio. * Calls ->writepage(). */ static pageout_t pageout(struct folio *folio, struct address_space *mapping, @@ -1283,7 +1294,7 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping, } /* - * Same as remove_mapping, but if the page is removed from the mapping, it + * Same as remove_mapping, but if the folio is removed from the mapping, it * gets returned with a refcount of 0. */ static int __remove_mapping(struct address_space *mapping, struct folio *folio, @@ -1299,34 +1310,34 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); /* - * The non racy check for a busy page. + * The non racy check for a busy folio. * * Must be careful with the order of the tests. When someone has - * a ref to the page, it may be possible that they dirty it then - * drop the reference. So if PageDirty is tested before page_count - * here, then the following race may occur: + * a ref to the folio, it may be possible that they dirty it then + * drop the reference. So if the dirty flag is tested before the + * refcount here, then the following race may occur: * * get_user_pages(&page); * [user mapping goes away] * write_to(page); - * !PageDirty(page) [good] - * SetPageDirty(page); - * put_page(page); - * !page_count(page) [good, discard it] + * !folio_test_dirty(folio) [good] + * folio_set_dirty(folio); + * folio_put(folio); + * !refcount(folio) [good, discard it] * * [oops, our write_to data is lost] * * Reversing the order of the tests ensures such a situation cannot - * escape unnoticed. The smp_rmb is needed to ensure the page->flags - * load is not satisfied before that of page->_refcount. + * escape unnoticed. The smp_rmb is needed to ensure the folio->flags + * load is not satisfied before that of folio->_refcount. * - * Note that if SetPageDirty is always performed via set_page_dirty, + * Note that if the dirty flag is always set via folio_mark_dirty, * and thus under the i_pages lock, then this ordering is not required. */ refcount = 1 + folio_nr_pages(folio); if (!folio_ref_freeze(folio, refcount)) goto cannot_free; - /* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */ + /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ if (unlikely(folio_test_dirty(folio))) { folio_ref_unfreeze(folio, refcount); goto cannot_free; @@ -1334,12 +1345,14 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, if (folio_test_swapcache(folio)) { swp_entry_t swap = folio_swap_entry(folio); - mem_cgroup_swapout(folio, swap); + + /* get a shadow entry before mem_cgroup_swapout() clears folio_memcg() */ if (reclaimed && !mapping_exiting(mapping)) shadow = workingset_eviction(folio, target_memcg); + mem_cgroup_swapout(folio, swap); __delete_from_swap_cache(folio, swap, shadow); xa_unlock_irq(&mapping->i_pages); - put_swap_page(&folio->page, swap); + put_swap_folio(folio, swap); } else { void (*free_folio)(struct folio *); @@ -1355,7 +1368,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, * back. * * We also don't store shadows for DAX mappings because the - * only page cache pages found in these are zero pages + * only page cache folios found in these are zero pages * covering holes, and because we don't want to mix DAX * exceptional entries and shadow exceptional entries in the * same address_space. @@ -1423,14 +1436,14 @@ void folio_putback_lru(struct folio *folio) folio_put(folio); /* drop ref from isolate */ } -enum page_references { - PAGEREF_RECLAIM, - PAGEREF_RECLAIM_CLEAN, - PAGEREF_KEEP, - PAGEREF_ACTIVATE, +enum folio_references { + FOLIOREF_RECLAIM, + FOLIOREF_RECLAIM_CLEAN, + FOLIOREF_KEEP, + FOLIOREF_ACTIVATE, }; -static enum page_references folio_check_references(struct folio *folio, +static enum folio_references folio_check_references(struct folio *folio, struct scan_control *sc) { int referenced_ptes, referenced_folio; @@ -1445,11 +1458,11 @@ static enum page_references folio_check_references(struct folio *folio, * Let the folio, now marked Mlocked, be moved to the unevictable list. */ if (vm_flags & VM_LOCKED) - return PAGEREF_ACTIVATE; + return FOLIOREF_ACTIVATE; /* rmap lock contention: rotate */ if (referenced_ptes == -1) - return PAGEREF_KEEP; + return FOLIOREF_KEEP; if (referenced_ptes) { /* @@ -1469,34 +1482,34 @@ static enum page_references folio_check_references(struct folio *folio, folio_set_referenced(folio); if (referenced_folio || referenced_ptes > 1) - return PAGEREF_ACTIVATE; + return FOLIOREF_ACTIVATE; /* * Activate file-backed executable folios after first usage. */ if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) - return PAGEREF_ACTIVATE; + return FOLIOREF_ACTIVATE; - return PAGEREF_KEEP; + return FOLIOREF_KEEP; } /* Reclaim if clean, defer dirty folios to writeback */ if (referenced_folio && folio_is_file_lru(folio)) - return PAGEREF_RECLAIM_CLEAN; + return FOLIOREF_RECLAIM_CLEAN; - return PAGEREF_RECLAIM; + return FOLIOREF_RECLAIM; } -/* Check if a page is dirty or under writeback */ +/* Check if a folio is dirty or under writeback */ static void folio_check_dirty_writeback(struct folio *folio, bool *dirty, bool *writeback) { struct address_space *mapping; /* - * Anonymous pages are not handled by flushers and must be written + * Anonymous folios are not handled by flushers and must be written * from reclaim context. Do not stall reclaim based on them. - * MADV_FREE anonymous pages are put into inactive file list too. + * MADV_FREE anonymous folios are put into inactive file list too. * They could be mistakenly treated as file lru. So further anon * test is needed. */ @@ -1520,44 +1533,71 @@ static void folio_check_dirty_writeback(struct folio *folio, mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); } -static struct page *alloc_demote_page(struct page *page, unsigned long node) +static struct page *alloc_demote_page(struct page *page, unsigned long private) { - struct migration_target_control mtc = { - /* - * Allocate from 'node', or fail quickly and quietly. - * When this happens, 'page' will likely just be discarded - * instead of migrated. - */ - .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | - __GFP_THISNODE | __GFP_NOWARN | - __GFP_NOMEMALLOC | GFP_NOWAIT, - .nid = node - }; + struct page *target_page; + nodemask_t *allowed_mask; + struct migration_target_control *mtc; + + mtc = (struct migration_target_control *)private; + + allowed_mask = mtc->nmask; + /* + * make sure we allocate from the target node first also trying to + * demote or reclaim pages from the target node via kswapd if we are + * low on free memory on target node. If we don't do this and if + * we have free memory on the slower(lower) memtier, we would start + * allocating pages from slower(lower) memory tiers without even forcing + * a demotion of cold pages from the target memtier. This can result + * in the kernel placing hot pages in slower(lower) memory tiers. + */ + mtc->nmask = NULL; + mtc->gfp_mask |= __GFP_THISNODE; + target_page = alloc_migration_target(page, (unsigned long)mtc); + if (target_page) + return target_page; - return alloc_migration_target(page, (unsigned long)&mtc); + mtc->gfp_mask &= ~__GFP_THISNODE; + mtc->nmask = allowed_mask; + + return alloc_migration_target(page, (unsigned long)mtc); } /* - * Take pages on @demote_list and attempt to demote them to - * another node. Pages which are not demoted are left on - * @demote_pages. + * Take folios on @demote_folios and attempt to demote them to another node. + * Folios which are not demoted are left on @demote_folios. */ -static unsigned int demote_page_list(struct list_head *demote_pages, +static unsigned int demote_folio_list(struct list_head *demote_folios, struct pglist_data *pgdat) { int target_nid = next_demotion_node(pgdat->node_id); unsigned int nr_succeeded; + nodemask_t allowed_mask; + + struct migration_target_control mtc = { + /* + * Allocate from 'node', or fail quickly and quietly. + * When this happens, 'page' will likely just be discarded + * instead of migrated. + */ + .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | + __GFP_NOMEMALLOC | GFP_NOWAIT, + .nid = target_nid, + .nmask = &allowed_mask + }; - if (list_empty(demote_pages)) + if (list_empty(demote_folios)) return 0; if (target_nid == NUMA_NO_NODE) return 0; + node_get_allowed_targets(pgdat, &allowed_mask); + /* Demotion ignores all cpuset and mempolicy settings */ - migrate_pages(demote_pages, alloc_demote_page, NULL, - target_nid, MIGRATE_ASYNC, MR_DEMOTION, - &nr_succeeded); + migrate_pages(demote_folios, alloc_demote_page, NULL, + (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, + &nr_succeeded); if (current_is_kswapd()) __count_vm_events(PGDEMOTE_KSWAPD, nr_succeeded); @@ -1584,17 +1624,15 @@ static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) } /* - * shrink_page_list() returns the number of reclaimed pages + * shrink_folio_list() returns the number of reclaimed pages */ -static unsigned int shrink_page_list(struct list_head *page_list, - struct pglist_data *pgdat, - struct scan_control *sc, - struct reclaim_stat *stat, - bool ignore_references) -{ - LIST_HEAD(ret_pages); - LIST_HEAD(free_pages); - LIST_HEAD(demote_pages); +static unsigned int shrink_folio_list(struct list_head *folio_list, + struct pglist_data *pgdat, struct scan_control *sc, + struct reclaim_stat *stat, bool ignore_references) +{ + LIST_HEAD(ret_folios); + LIST_HEAD(free_folios); + LIST_HEAD(demote_folios); unsigned int nr_reclaimed = 0; unsigned int pgactivate = 0; bool do_demote_pass; @@ -1605,16 +1643,16 @@ static unsigned int shrink_page_list(struct list_head *page_list, do_demote_pass = can_demote(pgdat->node_id, sc); retry: - while (!list_empty(page_list)) { + while (!list_empty(folio_list)) { struct address_space *mapping; struct folio *folio; - enum page_references references = PAGEREF_RECLAIM; + enum folio_references references = FOLIOREF_RECLAIM; bool dirty, writeback; unsigned int nr_pages; cond_resched(); - folio = lru_to_folio(page_list); + folio = lru_to_folio(folio_list); list_del(&folio->lru); if (!folio_trylock(folio)) @@ -1633,6 +1671,11 @@ retry: if (!sc->may_unmap && folio_mapped(folio)) goto keep_locked; + /* folio_update_gen() tried to promote this page? */ + if (lru_gen_enabled() && !ignore_references && + folio_mapped(folio) && folio_test_referenced(folio)) + goto keep_locked; + /* * The number of dirty pages determines if a node is marked * reclaim_congested. kswapd will stall and start writing @@ -1733,7 +1776,7 @@ retry: folio_unlock(folio); folio_wait_writeback(folio); /* then go back and try same folio again */ - list_add_tail(&folio->lru, page_list); + list_add_tail(&folio->lru, folio_list); continue; } } @@ -1742,13 +1785,13 @@ retry: references = folio_check_references(folio, sc); switch (references) { - case PAGEREF_ACTIVATE: + case FOLIOREF_ACTIVATE: goto activate_locked; - case PAGEREF_KEEP: + case FOLIOREF_KEEP: stat->nr_ref_keep += nr_pages; goto keep_locked; - case PAGEREF_RECLAIM: - case PAGEREF_RECLAIM_CLEAN: + case FOLIOREF_RECLAIM: + case FOLIOREF_RECLAIM_CLEAN: ; /* try to reclaim the folio below */ } @@ -1758,7 +1801,7 @@ retry: */ if (do_demote_pass && (thp_migration_supported() || !folio_test_large(folio))) { - list_add(&folio->lru, &demote_pages); + list_add(&folio->lru, &demote_folios); folio_unlock(folio); continue; } @@ -1785,7 +1828,7 @@ retry: */ if (!folio_entire_mapcount(folio) && split_folio_to_list(folio, - page_list)) + folio_list)) goto activate_locked; } if (!add_to_swap(folio)) { @@ -1793,7 +1836,7 @@ retry: goto activate_locked_split; /* Fallback to swap normal pages */ if (split_folio_to_list(folio, - page_list)) + folio_list)) goto activate_locked; #ifdef CONFIG_TRANSPARENT_HUGEPAGE count_vm_event(THP_SWPOUT_FALLBACK); @@ -1805,7 +1848,7 @@ retry: } else if (folio_test_swapbacked(folio) && folio_test_large(folio)) { /* Split shmem folio */ - if (split_folio_to_list(folio, page_list)) + if (split_folio_to_list(folio, folio_list)) goto keep_locked; } @@ -1870,7 +1913,7 @@ retry: goto activate_locked; } - if (references == PAGEREF_RECLAIM_CLEAN) + if (references == FOLIOREF_RECLAIM_CLEAN) goto keep_locked; if (!may_enter_fs(folio, sc->gfp_mask)) goto keep_locked; @@ -1983,13 +2026,13 @@ free_it: nr_reclaimed += nr_pages; /* - * Is there need to periodically free_page_list? It would + * Is there need to periodically free_folio_list? It would * appear not as the counts should be low */ if (unlikely(folio_test_large(folio))) destroy_large_folio(folio); else - list_add(&folio->lru, &free_pages); + list_add(&folio->lru, &free_folios); continue; activate_locked_split: @@ -2004,9 +2047,8 @@ activate_locked_split: activate_locked: /* Not a candidate for swapping, so reclaim swap space. */ if (folio_test_swapcache(folio) && - (mem_cgroup_swap_full(&folio->page) || - folio_test_mlocked(folio))) - try_to_free_swap(&folio->page); + (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) + folio_free_swap(folio); VM_BUG_ON_FOLIO(folio_test_active(folio), folio); if (!folio_test_mlocked(folio)) { int type = folio_is_file_lru(folio); @@ -2017,29 +2059,29 @@ activate_locked: keep_locked: folio_unlock(folio); keep: - list_add(&folio->lru, &ret_pages); + list_add(&folio->lru, &ret_folios); VM_BUG_ON_FOLIO(folio_test_lru(folio) || folio_test_unevictable(folio), folio); } - /* 'page_list' is always empty here */ + /* 'folio_list' is always empty here */ /* Migrate folios selected for demotion */ - nr_reclaimed += demote_page_list(&demote_pages, pgdat); - /* Folios that could not be demoted are still in @demote_pages */ - if (!list_empty(&demote_pages)) { - /* Folios which weren't demoted go back on @page_list for retry: */ - list_splice_init(&demote_pages, page_list); + nr_reclaimed += demote_folio_list(&demote_folios, pgdat); + /* Folios that could not be demoted are still in @demote_folios */ + if (!list_empty(&demote_folios)) { + /* Folios which weren't demoted go back on @folio_list for retry: */ + list_splice_init(&demote_folios, folio_list); do_demote_pass = false; goto retry; } pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; - mem_cgroup_uncharge_list(&free_pages); + mem_cgroup_uncharge_list(&free_folios); try_to_unmap_flush(); - free_unref_page_list(&free_pages); + free_unref_page_list(&free_folios); - list_splice(&ret_pages, page_list); + list_splice(&ret_folios, folio_list); count_vm_events(PGACTIVATE, pgactivate); if (plug) @@ -2048,7 +2090,7 @@ keep: } unsigned int reclaim_clean_pages_from_list(struct zone *zone, - struct list_head *folio_list) + struct list_head *folio_list) { struct scan_control sc = { .gfp_mask = GFP_KERNEL, @@ -2076,7 +2118,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone, * change in the future. */ noreclaim_flag = memalloc_noreclaim_save(); - nr_reclaimed = shrink_page_list(&clean_folios, zone->zone_pgdat, &sc, + nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, &stat, true); memalloc_noreclaim_restore(noreclaim_flag); @@ -2135,7 +2177,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec, * * returns how many pages were moved onto *@dst. */ -static unsigned long isolate_lru_pages(unsigned long nr_to_scan, +static unsigned long isolate_lru_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct list_head *dst, unsigned long *nr_scanned, struct scan_control *sc, enum lru_list lru) @@ -2242,8 +2284,8 @@ move: * * Context: * - * (1) Must be called with an elevated refcount on the page. This is a - * fundamental difference from isolate_lru_pages() (which is called + * (1) Must be called with an elevated refcount on the folio. This is a + * fundamental difference from isolate_lru_folios() (which is called * without a stable reference). * (2) The lru_lock must not be held. * (3) Interrupts must be enabled. @@ -2315,13 +2357,13 @@ static int too_many_isolated(struct pglist_data *pgdat, int file, } /* - * move_pages_to_lru() moves folios from private @list to appropriate LRU list. + * move_folios_to_lru() moves folios from private @list to appropriate LRU list. * On return, @list is reused as a list of folios to be freed by the caller. * * Returns the number of pages moved to the given lruvec. */ -static unsigned int move_pages_to_lru(struct lruvec *lruvec, - struct list_head *list) +static unsigned int move_folios_to_lru(struct lruvec *lruvec, + struct list_head *list) { int nr_pages, nr_moved = 0; LIST_HEAD(folios_to_free); @@ -2341,7 +2383,7 @@ static unsigned int move_pages_to_lru(struct lruvec *lruvec, /* * The folio_set_lru needs to be kept here for list integrity. * Otherwise: - * #0 move_pages_to_lru #1 release_pages + * #0 move_folios_to_lru #1 release_pages * if (!folio_put_testzero()) * if (folio_put_testzero()) * !lru //skip lru_lock @@ -2398,11 +2440,11 @@ static int current_may_throttle(void) * shrink_inactive_list() is a helper for shrink_node(). It returns the number * of reclaimed pages */ -static unsigned long -shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, - struct scan_control *sc, enum lru_list lru) +static unsigned long shrink_inactive_list(unsigned long nr_to_scan, + struct lruvec *lruvec, struct scan_control *sc, + enum lru_list lru) { - LIST_HEAD(page_list); + LIST_HEAD(folio_list); unsigned long nr_scanned; unsigned int nr_reclaimed = 0; unsigned long nr_taken; @@ -2429,7 +2471,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, spin_lock_irq(&lruvec->lru_lock); - nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list, + nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, &nr_scanned, sc, lru); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); @@ -2444,10 +2486,10 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, if (nr_taken == 0) return 0; - nr_reclaimed = shrink_page_list(&page_list, pgdat, sc, &stat, false); + nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false); spin_lock_irq(&lruvec->lru_lock); - move_pages_to_lru(lruvec, &page_list); + move_folios_to_lru(lruvec, &folio_list); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; @@ -2458,16 +2500,16 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, spin_unlock_irq(&lruvec->lru_lock); lru_note_cost(lruvec, file, stat.nr_pageout); - mem_cgroup_uncharge_list(&page_list); - free_unref_page_list(&page_list); + mem_cgroup_uncharge_list(&folio_list); + free_unref_page_list(&folio_list); /* - * If dirty pages are scanned that are not queued for IO, it + * If dirty folios are scanned that are not queued for IO, it * implies that flushers are not doing their job. This can - * happen when memory pressure pushes dirty pages to the end of + * happen when memory pressure pushes dirty folios to the end of * the LRU before the dirty limits are breached and the dirty * data has expired. It can also happen when the proportion of - * dirty pages grows not through writes but through memory + * dirty folios grows not through writes but through memory * pressure reclaiming all the clean cache. And in some cases, * the flushers simply cannot keep up with the allocation * rate. Nudge the flusher threads in case they are asleep. @@ -2526,7 +2568,7 @@ static void shrink_active_list(unsigned long nr_to_scan, spin_lock_irq(&lruvec->lru_lock); - nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, + nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, &nr_scanned, sc, lru); __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); @@ -2550,8 +2592,8 @@ static void shrink_active_list(unsigned long nr_to_scan, } if (unlikely(buffer_heads_over_limit)) { - if (folio_get_private(folio) && folio_trylock(folio)) { - if (folio_get_private(folio)) + if (folio_test_private(folio) && folio_trylock(folio)) { + if (folio_test_private(folio)) filemap_release_folio(folio, 0); folio_unlock(folio); } @@ -2586,8 +2628,8 @@ static void shrink_active_list(unsigned long nr_to_scan, */ spin_lock_irq(&lruvec->lru_lock); - nr_activate = move_pages_to_lru(lruvec, &l_active); - nr_deactivate = move_pages_to_lru(lruvec, &l_inactive); + nr_activate = move_folios_to_lru(lruvec, &l_active); + nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); /* Keep all free folios in l_active list */ list_splice(&l_inactive, &l_active); @@ -2603,7 +2645,7 @@ static void shrink_active_list(unsigned long nr_to_scan, nr_deactivate, nr_rotated, sc->priority, file); } -static unsigned int reclaim_page_list(struct list_head *page_list, +static unsigned int reclaim_folio_list(struct list_head *folio_list, struct pglist_data *pgdat) { struct reclaim_stat dummy_stat; @@ -2617,9 +2659,9 @@ static unsigned int reclaim_page_list(struct list_head *page_list, .no_demotion = 1, }; - nr_reclaimed = shrink_page_list(page_list, pgdat, &sc, &dummy_stat, false); - while (!list_empty(page_list)) { - folio = lru_to_folio(page_list); + nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, false); + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); list_del(&folio->lru); folio_putback_lru(folio); } @@ -2649,11 +2691,11 @@ unsigned long reclaim_pages(struct list_head *folio_list) continue; } - nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); nid = folio_nid(lru_to_folio(folio_list)); } while (!list_empty(folio_list)); - nr_reclaimed += reclaim_page_list(&node_folio_list, NODE_DATA(nid)); + nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); memalloc_noreclaim_restore(noreclaim_flag); @@ -2683,13 +2725,13 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, * but large enough to avoid thrashing the aggregate readahead window. * * Both inactive lists should also be large enough that each inactive - * page has a chance to be referenced again before it is reclaimed. + * folio has a chance to be referenced again before it is reclaimed. * * If that fails and refaulting is observed, the inactive list grows. * - * The inactive_ratio is the target ratio of ACTIVE to INACTIVE pages + * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios * on this LRU, maintained by the pageout code. An inactive_ratio - * of 3 means 3:1 or 25% of the pages are kept on the inactive list. + * of 3 means 3:1 or 25% of the folios are kept on the inactive list. * * total target max * memory ratio inactive @@ -2728,12 +2770,118 @@ enum scan_balance { SCAN_FILE, }; +static void prepare_scan_count(pg_data_t *pgdat, struct scan_control *sc) +{ + unsigned long file; + struct lruvec *target_lruvec; + + if (lru_gen_enabled()) + return; + + target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); + + /* + * Flush the memory cgroup stats, so that we read accurate per-memcg + * lruvec stats for heuristics. + */ + mem_cgroup_flush_stats(); + + /* + * Determine the scan balance between anon and file LRUs. + */ + spin_lock_irq(&target_lruvec->lru_lock); + sc->anon_cost = target_lruvec->anon_cost; + sc->file_cost = target_lruvec->file_cost; + spin_unlock_irq(&target_lruvec->lru_lock); + + /* + * Target desirable inactive:active list ratios for the anon + * and file LRU lists. + */ + if (!sc->force_deactivate) { + unsigned long refaults; + + /* + * When refaults are being observed, it means a new + * workingset is being established. Deactivate to get + * rid of any stale active pages quickly. + */ + refaults = lruvec_page_state(target_lruvec, + WORKINGSET_ACTIVATE_ANON); + if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || + inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) + sc->may_deactivate |= DEACTIVATE_ANON; + else + sc->may_deactivate &= ~DEACTIVATE_ANON; + + refaults = lruvec_page_state(target_lruvec, + WORKINGSET_ACTIVATE_FILE); + if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || + inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) + sc->may_deactivate |= DEACTIVATE_FILE; + else + sc->may_deactivate &= ~DEACTIVATE_FILE; + } else + sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; + + /* + * If we have plenty of inactive file pages that aren't + * thrashing, try to reclaim those first before touching + * anonymous pages. + */ + file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); + if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) + sc->cache_trim_mode = 1; + else + sc->cache_trim_mode = 0; + + /* + * Prevent the reclaimer from falling into the cache trap: as + * cache pages start out inactive, every cache fault will tip + * the scan balance towards the file LRU. And as the file LRU + * shrinks, so does the window for rotation from references. + * This means we have a runaway feedback loop where a tiny + * thrashing file LRU becomes infinitely more attractive than + * anon pages. Try to detect this based on file LRU size. + */ + if (!cgroup_reclaim(sc)) { + unsigned long total_high_wmark = 0; + unsigned long free, anon; + int z; + + free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); + file = node_page_state(pgdat, NR_ACTIVE_FILE) + + node_page_state(pgdat, NR_INACTIVE_FILE); + + for (z = 0; z < MAX_NR_ZONES; z++) { + struct zone *zone = &pgdat->node_zones[z]; + + if (!managed_zone(zone)) + continue; + + total_high_wmark += high_wmark_pages(zone); + } + + /* + * Consider anon: if that's low too, this isn't a + * runaway file reclaim problem, but rather just + * extreme pressure. Reclaim as per usual then. + */ + anon = node_page_state(pgdat, NR_INACTIVE_ANON); + + sc->file_is_tiny = + file + free <= total_high_wmark && + !(sc->may_deactivate & DEACTIVATE_ANON) && + anon >> sc->priority; + } +} + /* * Determine how aggressively the anon and file LRU lists should be * scanned. * - * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan - * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan + * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan + * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan */ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, unsigned long *nr) @@ -2748,7 +2896,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, unsigned long ap, fp; enum lru_list lru; - /* If we have no swap space, do not bother scanning anon pages. */ + /* If we have no swap space, do not bother scanning anon folios. */ if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { scan_balance = SCAN_FILE; goto out; @@ -2947,136 +3095,2882 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, return can_demote(pgdat->node_id, sc); } -static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) -{ - unsigned long nr[NR_LRU_LISTS]; - unsigned long targets[NR_LRU_LISTS]; - unsigned long nr_to_scan; - enum lru_list lru; - unsigned long nr_reclaimed = 0; - unsigned long nr_to_reclaim = sc->nr_to_reclaim; - struct blk_plug plug; - bool scan_adjusted; - - get_scan_count(lruvec, sc, nr); - - /* Record the original scan target for proportional adjustments later */ - memcpy(targets, nr, sizeof(nr)); +#ifdef CONFIG_LRU_GEN - /* - * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal - * event that can occur when there is little memory pressure e.g. - * multiple streaming readers/writers. Hence, we do not abort scanning - * when the requested number of pages are reclaimed when scanning at - * DEF_PRIORITY on the assumption that the fact we are direct - * reclaiming implies that kswapd is not keeping up and it is best to - * do a batch of work at once. For memcg reclaim one check is made to - * abort proportional reclaim if either the file or anon lru has already - * dropped to zero at the first pass. - */ - scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && - sc->priority == DEF_PRIORITY); +#ifdef CONFIG_LRU_GEN_ENABLED +DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) +#else +DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); +#define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) +#endif - blk_start_plug(&plug); - while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || - nr[LRU_INACTIVE_FILE]) { - unsigned long nr_anon, nr_file, percentage; - unsigned long nr_scanned; +/****************************************************************************** + * shorthand helpers + ******************************************************************************/ - for_each_evictable_lru(lru) { - if (nr[lru]) { - nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); - nr[lru] -= nr_to_scan; +#define LRU_REFS_FLAGS (BIT(PG_referenced) | BIT(PG_workingset)) - nr_reclaimed += shrink_list(lru, nr_to_scan, - lruvec, sc); - } - } +#define DEFINE_MAX_SEQ(lruvec) \ + unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) - cond_resched(); +#define DEFINE_MIN_SEQ(lruvec) \ + unsigned long min_seq[ANON_AND_FILE] = { \ + READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ + READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ + } - if (nr_reclaimed < nr_to_reclaim || scan_adjusted) - continue; +#define for_each_gen_type_zone(gen, type, zone) \ + for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ + for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ + for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) - /* - * For kswapd and memcg, reclaim at least the number of pages - * requested. Ensure that the anon and file LRUs are scanned - * proportionally what was requested by get_scan_count(). We - * stop reclaiming one LRU and reduce the amount scanning - * proportional to the original scan target. - */ - nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; - nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; +static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) +{ + struct pglist_data *pgdat = NODE_DATA(nid); - /* - * It's just vindictive to attack the larger once the smaller - * has gone to zero. And given the way we stop scanning the - * smaller below, this makes sure that we only make one nudge - * towards proportionality once we've got nr_to_reclaim. - */ - if (!nr_file || !nr_anon) - break; +#ifdef CONFIG_MEMCG + if (memcg) { + struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; - if (nr_file > nr_anon) { - unsigned long scan_target = targets[LRU_INACTIVE_ANON] + - targets[LRU_ACTIVE_ANON] + 1; - lru = LRU_BASE; - percentage = nr_anon * 100 / scan_target; - } else { - unsigned long scan_target = targets[LRU_INACTIVE_FILE] + - targets[LRU_ACTIVE_FILE] + 1; - lru = LRU_FILE; - percentage = nr_file * 100 / scan_target; - } + /* for hotadd_new_pgdat() */ + if (!lruvec->pgdat) + lruvec->pgdat = pgdat; - /* Stop scanning the smaller of the LRU */ - nr[lru] = 0; - nr[lru + LRU_ACTIVE] = 0; + return lruvec; + } +#endif + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); - /* - * Recalculate the other LRU scan count based on its original - * scan target and the percentage scanning already complete - */ - lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; - nr_scanned = targets[lru] - nr[lru]; - nr[lru] = targets[lru] * (100 - percentage) / 100; - nr[lru] -= min(nr[lru], nr_scanned); + return pgdat ? &pgdat->__lruvec : NULL; +} - lru += LRU_ACTIVE; - nr_scanned = targets[lru] - nr[lru]; - nr[lru] = targets[lru] * (100 - percentage) / 100; - nr[lru] -= min(nr[lru], nr_scanned); +static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) +{ + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct pglist_data *pgdat = lruvec_pgdat(lruvec); - scan_adjusted = true; - } - blk_finish_plug(&plug); - sc->nr_reclaimed += nr_reclaimed; + if (!can_demote(pgdat->node_id, sc) && + mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) + return 0; - /* - * Even if we did not try to evict anon pages at all, we want to - * rebalance the anon lru active/inactive ratio. - */ - if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && - inactive_is_low(lruvec, LRU_INACTIVE_ANON)) - shrink_active_list(SWAP_CLUSTER_MAX, lruvec, - sc, LRU_ACTIVE_ANON); + return mem_cgroup_swappiness(memcg); } -/* Use reclaim/compaction for costly allocs or under memory pressure */ -static bool in_reclaim_compaction(struct scan_control *sc) +static int get_nr_gens(struct lruvec *lruvec, int type) { - if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && - (sc->order > PAGE_ALLOC_COSTLY_ORDER || - sc->priority < DEF_PRIORITY - 2)) - return true; + return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; +} - return false; +static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) +{ + /* see the comment on lru_gen_struct */ + return get_nr_gens(lruvec, LRU_GEN_FILE) >= MIN_NR_GENS && + get_nr_gens(lruvec, LRU_GEN_FILE) <= get_nr_gens(lruvec, LRU_GEN_ANON) && + get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; } -/* - * Reclaim/compaction is used for high-order allocation requests. It reclaims - * order-0 pages before compacting the zone. should_continue_reclaim() returns - * true if more pages should be reclaimed such that when the page allocator +/****************************************************************************** + * mm_struct list + ******************************************************************************/ + +static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) +{ + static struct lru_gen_mm_list mm_list = { + .fifo = LIST_HEAD_INIT(mm_list.fifo), + .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), + }; + +#ifdef CONFIG_MEMCG + if (memcg) + return &memcg->mm_list; +#endif + VM_WARN_ON_ONCE(!mem_cgroup_disabled()); + + return &mm_list; +} + +void lru_gen_add_mm(struct mm_struct *mm) +{ + int nid; + struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + + VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); +#ifdef CONFIG_MEMCG + VM_WARN_ON_ONCE(mm->lru_gen.memcg); + mm->lru_gen.memcg = memcg; +#endif + spin_lock(&mm_list->lock); + + for_each_node_state(nid, N_MEMORY) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + /* the first addition since the last iteration */ + if (lruvec->mm_state.tail == &mm_list->fifo) + lruvec->mm_state.tail = &mm->lru_gen.list; + } + + list_add_tail(&mm->lru_gen.list, &mm_list->fifo); + + spin_unlock(&mm_list->lock); +} + +void lru_gen_del_mm(struct mm_struct *mm) +{ + int nid; + struct lru_gen_mm_list *mm_list; + struct mem_cgroup *memcg = NULL; + + if (list_empty(&mm->lru_gen.list)) + return; + +#ifdef CONFIG_MEMCG + memcg = mm->lru_gen.memcg; +#endif + mm_list = get_mm_list(memcg); + + spin_lock(&mm_list->lock); + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + /* where the last iteration ended (exclusive) */ + if (lruvec->mm_state.tail == &mm->lru_gen.list) + lruvec->mm_state.tail = lruvec->mm_state.tail->next; + + /* where the current iteration continues (inclusive) */ + if (lruvec->mm_state.head != &mm->lru_gen.list) + continue; + + lruvec->mm_state.head = lruvec->mm_state.head->next; + /* the deletion ends the current iteration */ + if (lruvec->mm_state.head == &mm_list->fifo) + WRITE_ONCE(lruvec->mm_state.seq, lruvec->mm_state.seq + 1); + } + + list_del_init(&mm->lru_gen.list); + + spin_unlock(&mm_list->lock); + +#ifdef CONFIG_MEMCG + mem_cgroup_put(mm->lru_gen.memcg); + mm->lru_gen.memcg = NULL; +#endif +} + +#ifdef CONFIG_MEMCG +void lru_gen_migrate_mm(struct mm_struct *mm) +{ + struct mem_cgroup *memcg; + struct task_struct *task = rcu_dereference_protected(mm->owner, true); + + VM_WARN_ON_ONCE(task->mm != mm); + lockdep_assert_held(&task->alloc_lock); + + /* for mm_update_next_owner() */ + if (mem_cgroup_disabled()) + return; + + rcu_read_lock(); + memcg = mem_cgroup_from_task(task); + rcu_read_unlock(); + if (memcg == mm->lru_gen.memcg) + return; + + VM_WARN_ON_ONCE(!mm->lru_gen.memcg); + VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); + + lru_gen_del_mm(mm); + lru_gen_add_mm(mm); +} +#endif + +/* + * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when + * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of + * bits in a bitmap, k is the number of hash functions and n is the number of + * inserted items. + * + * Page table walkers use one of the two filters to reduce their search space. + * To get rid of non-leaf entries that no longer have enough leaf entries, the + * aging uses the double-buffering technique to flip to the other filter each + * time it produces a new generation. For non-leaf entries that have enough + * leaf entries, the aging carries them over to the next generation in + * walk_pmd_range(); the eviction also report them when walking the rmap + * in lru_gen_look_around(). + * + * For future optimizations: + * 1. It's not necessary to keep both filters all the time. The spare one can be + * freed after the RCU grace period and reallocated if needed again. + * 2. And when reallocating, it's worth scaling its size according to the number + * of inserted entries in the other filter, to reduce the memory overhead on + * small systems and false positives on large systems. + * 3. Jenkins' hash function is an alternative to Knuth's. + */ +#define BLOOM_FILTER_SHIFT 15 + +static inline int filter_gen_from_seq(unsigned long seq) +{ + return seq % NR_BLOOM_FILTERS; +} + +static void get_item_key(void *item, int *key) +{ + u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); + + BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); + + key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); + key[1] = hash >> BLOOM_FILTER_SHIFT; +} + +static void reset_bloom_filter(struct lruvec *lruvec, unsigned long seq) +{ + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = lruvec->mm_state.filters[gen]; + if (filter) { + bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); + return; + } + + filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), + __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + WRITE_ONCE(lruvec->mm_state.filters[gen], filter); +} + +static void update_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ + int key[2]; + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = READ_ONCE(lruvec->mm_state.filters[gen]); + if (!filter) + return; + + get_item_key(item, key); + + if (!test_bit(key[0], filter)) + set_bit(key[0], filter); + if (!test_bit(key[1], filter)) + set_bit(key[1], filter); +} + +static bool test_bloom_filter(struct lruvec *lruvec, unsigned long seq, void *item) +{ + int key[2]; + unsigned long *filter; + int gen = filter_gen_from_seq(seq); + + filter = READ_ONCE(lruvec->mm_state.filters[gen]); + if (!filter) + return true; + + get_item_key(item, key); + + return test_bit(key[0], filter) && test_bit(key[1], filter); +} + +static void reset_mm_stats(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, bool last) +{ + int i; + int hist; + + lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); + + if (walk) { + hist = lru_hist_from_seq(walk->max_seq); + + for (i = 0; i < NR_MM_STATS; i++) { + WRITE_ONCE(lruvec->mm_state.stats[hist][i], + lruvec->mm_state.stats[hist][i] + walk->mm_stats[i]); + walk->mm_stats[i] = 0; + } + } + + if (NR_HIST_GENS > 1 && last) { + hist = lru_hist_from_seq(lruvec->mm_state.seq + 1); + + for (i = 0; i < NR_MM_STATS; i++) + WRITE_ONCE(lruvec->mm_state.stats[hist][i], 0); + } +} + +static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) +{ + int type; + unsigned long size = 0; + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); + + if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) + return true; + + clear_bit(key, &mm->lru_gen.bitmap); + + for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { + size += type ? get_mm_counter(mm, MM_FILEPAGES) : + get_mm_counter(mm, MM_ANONPAGES) + + get_mm_counter(mm, MM_SHMEMPAGES); + } + + if (size < MIN_LRU_BATCH) + return true; + + return !mmget_not_zero(mm); +} + +static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, + struct mm_struct **iter) +{ + bool first = false; + bool last = true; + struct mm_struct *mm = NULL; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + + /* + * There are four interesting cases for this page table walker: + * 1. It tries to start a new iteration of mm_list with a stale max_seq; + * there is nothing left to do. + * 2. It's the first of the current generation, and it needs to reset + * the Bloom filter for the next generation. + * 3. It reaches the end of mm_list, and it needs to increment + * mm_state->seq; the iteration is done. + * 4. It's the last of the current generation, and it needs to reset the + * mm stats counters for the next generation. + */ + spin_lock(&mm_list->lock); + + VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq); + VM_WARN_ON_ONCE(*iter && mm_state->seq > walk->max_seq); + VM_WARN_ON_ONCE(*iter && !mm_state->nr_walkers); + + if (walk->max_seq <= mm_state->seq) { + if (!*iter) + last = false; + goto done; + } + + if (!mm_state->nr_walkers) { + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + + mm_state->head = mm_list->fifo.next; + first = true; + } + + while (!mm && mm_state->head != &mm_list->fifo) { + mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); + + mm_state->head = mm_state->head->next; + + /* force scan for those added after the last iteration */ + if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { + mm_state->tail = mm_state->head; + walk->force_scan = true; + } + + if (should_skip_mm(mm, walk)) + mm = NULL; + } + + if (mm_state->head == &mm_list->fifo) + WRITE_ONCE(mm_state->seq, mm_state->seq + 1); +done: + if (*iter && !mm) + mm_state->nr_walkers--; + if (!*iter && mm) + mm_state->nr_walkers++; + + if (mm_state->nr_walkers) + last = false; + + if (*iter || last) + reset_mm_stats(lruvec, walk, last); + + spin_unlock(&mm_list->lock); + + if (mm && first) + reset_bloom_filter(lruvec, walk->max_seq + 1); + + if (*iter) + mmput_async(*iter); + + *iter = mm; + + return last; +} + +static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq) +{ + bool success = false; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct lru_gen_mm_list *mm_list = get_mm_list(memcg); + struct lru_gen_mm_state *mm_state = &lruvec->mm_state; + + spin_lock(&mm_list->lock); + + VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq); + + if (max_seq > mm_state->seq && !mm_state->nr_walkers) { + VM_WARN_ON_ONCE(mm_state->head && mm_state->head != &mm_list->fifo); + + WRITE_ONCE(mm_state->seq, mm_state->seq + 1); + reset_mm_stats(lruvec, NULL, true); + success = true; + } + + spin_unlock(&mm_list->lock); + + return success; +} + +/****************************************************************************** + * refault feedback loop + ******************************************************************************/ + +/* + * A feedback loop based on Proportional-Integral-Derivative (PID) controller. + * + * The P term is refaulted/(evicted+protected) from a tier in the generation + * currently being evicted; the I term is the exponential moving average of the + * P term over the generations previously evicted, using the smoothing factor + * 1/2; the D term isn't supported. + * + * The setpoint (SP) is always the first tier of one type; the process variable + * (PV) is either any tier of the other type or any other tier of the same + * type. + * + * The error is the difference between the SP and the PV; the correction is to + * turn off protection when SP>PV or turn on protection when SPlrugen; + int hist = lru_hist_from_seq(lrugen->min_seq[type]); + + pos->refaulted = lrugen->avg_refaulted[type][tier] + + atomic_long_read(&lrugen->refaulted[hist][type][tier]); + pos->total = lrugen->avg_total[type][tier] + + atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + pos->total += lrugen->protected[hist][type][tier - 1]; + pos->gain = gain; +} + +static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) +{ + int hist, tier; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; + unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; + + lockdep_assert_held(&lruvec->lru_lock); + + if (!carryover && !clear) + return; + + hist = lru_hist_from_seq(seq); + + for (tier = 0; tier < MAX_NR_TIERS; tier++) { + if (carryover) { + unsigned long sum; + + sum = lrugen->avg_refaulted[type][tier] + + atomic_long_read(&lrugen->refaulted[hist][type][tier]); + WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); + + sum = lrugen->avg_total[type][tier] + + atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + sum += lrugen->protected[hist][type][tier - 1]; + WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); + } + + if (clear) { + atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); + atomic_long_set(&lrugen->evicted[hist][type][tier], 0); + if (tier) + WRITE_ONCE(lrugen->protected[hist][type][tier - 1], 0); + } + } +} + +static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) +{ + /* + * Return true if the PV has a limited number of refaults or a lower + * refaulted/total than the SP. + */ + return pv->refaulted < MIN_LRU_BATCH || + pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= + (sp->refaulted + 1) * pv->total * pv->gain; +} + +/****************************************************************************** + * the aging + ******************************************************************************/ + +/* promote pages accessed through page tables */ +static int folio_update_gen(struct folio *folio, int gen) +{ + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(!rcu_read_lock_held()); + + do { + /* lru_gen_del_folio() has isolated this page? */ + if (!(old_flags & LRU_GEN_MASK)) { + /* for shrink_folio_list() */ + new_flags = old_flags | BIT(PG_referenced); + continue; + } + + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); + new_flags |= (gen + 1UL) << LRU_GEN_PGOFF; + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + + return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +} + +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) +{ + int type = folio_is_file_lru(folio); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + unsigned long new_flags, old_flags = READ_ONCE(folio->flags); + + VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); + + do { + new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + /* folio_update_gen() has promoted this page? */ + if (new_gen >= 0 && new_gen != old_gen) + return new_gen; + + new_gen = (old_gen + 1) % MAX_NR_GENS; + + new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_MASK | LRU_REFS_FLAGS); + new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; + /* for folio_end_writeback() */ + if (reclaiming) + new_flags |= BIT(PG_reclaim); + } while (!try_cmpxchg(&folio->flags, &old_flags, new_flags)); + + lru_gen_update_size(lruvec, folio, old_gen, new_gen); + + return new_gen; +} + +static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, + int old_gen, int new_gen) +{ + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + + VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); + VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); + + walk->batched++; + + walk->nr_pages[old_gen][type][zone] -= delta; + walk->nr_pages[new_gen][type][zone] += delta; +} + +static void reset_batch_size(struct lruvec *lruvec, struct lru_gen_mm_walk *walk) +{ + int gen, type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + walk->batched = 0; + + for_each_gen_type_zone(gen, type, zone) { + enum lru_list lru = type * LRU_INACTIVE_FILE; + int delta = walk->nr_pages[gen][type][zone]; + + if (!delta) + continue; + + walk->nr_pages[gen][type][zone] = 0; + WRITE_ONCE(lrugen->nr_pages[gen][type][zone], + lrugen->nr_pages[gen][type][zone] + delta); + + if (lru_gen_is_active(lruvec, gen)) + lru += LRU_ACTIVE; + __update_lru_size(lruvec, lru, zone, delta); + } +} + +static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) +{ + struct address_space *mapping; + struct vm_area_struct *vma = args->vma; + struct lru_gen_mm_walk *walk = args->private; + + if (!vma_is_accessible(vma)) + return true; + + if (is_vm_hugetlb_page(vma)) + return true; + + if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL | VM_SEQ_READ | VM_RAND_READ)) + return true; + + if (vma == get_gate_vma(vma->vm_mm)) + return true; + + if (vma_is_anonymous(vma)) + return !walk->can_swap; + + if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) + return true; + + mapping = vma->vm_file->f_mapping; + if (mapping_unevictable(mapping)) + return true; + + if (shmem_mapping(mapping)) + return !walk->can_swap; + + /* to exclude special mappings like dax, etc. */ + return !mapping->a_ops->read_folio; +} + +/* + * Some userspace memory allocators map many single-page VMAs. Instead of + * returning back to the PGD table for each of such VMAs, finish an entire PMD + * table to reduce zigzags and improve cache performance. + */ +static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, + unsigned long *vm_start, unsigned long *vm_end) +{ + unsigned long start = round_up(*vm_end, size); + unsigned long end = (start | ~mask) + 1; + VMA_ITERATOR(vmi, args->mm, start); + + VM_WARN_ON_ONCE(mask & size); + VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); + + for_each_vma(vmi, args->vma) { + if (end && end <= args->vma->vm_start) + return false; + + if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) + continue; + + *vm_start = max(start, args->vma->vm_start); + *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; + + return true; + } + + return false; +} + +static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pte_pfn(pte); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pte_present(pte) || is_zero_pfn(pfn)) + return -1; + + if (WARN_ON_ONCE(pte_devmap(pte) || pte_special(pte))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long pfn = pmd_pfn(pmd); + + VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); + + if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) + return -1; + + if (WARN_ON_ONCE(pmd_devmap(pmd))) + return -1; + + if (WARN_ON_ONCE(!pfn_valid(pfn))) + return -1; + + return pfn; +} +#endif + +static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, + struct pglist_data *pgdat, bool can_swap) +{ + struct folio *folio; + + /* try to avoid unnecessary memory loads */ + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + return NULL; + + folio = pfn_folio(pfn); + if (folio_nid(folio) != pgdat->node_id) + return NULL; + + if (folio_memcg_rcu(folio) != memcg) + return NULL; + + /* file VMAs can contain anon pages from COW */ + if (!folio_is_file_lru(folio) && !can_swap) + return NULL; + + return folio; +} + +static bool suitable_to_scan(int total, int young) +{ + int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); + + /* suitable if the average number of young PTEs per cacheline is >=1 */ + return young * n >= total; +} + +static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pte_t *pte; + spinlock_t *ptl; + unsigned long addr; + int total = 0; + int young = 0; + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + + VM_WARN_ON_ONCE(pmd_leaf(*pmd)); + + ptl = pte_lockptr(args->mm, pmd); + if (!spin_trylock(ptl)) + return false; + + arch_enter_lazy_mmu_mode(); + + pte = pte_offset_map(pmd, start & PMD_MASK); +restart: + for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { + unsigned long pfn; + struct folio *folio; + + total++; + walk->mm_stats[MM_LEAF_TOTAL]++; + + pfn = get_pte_pfn(pte[i], args->vma, addr); + if (pfn == -1) + continue; + + if (!pte_young(pte[i])) { + walk->mm_stats[MM_LEAF_OLD]++; + continue; + } + + folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); + if (!folio) + continue; + + if (!ptep_test_and_clear_young(args->vma, addr, pte + i)) + VM_WARN_ON_ONCE(true); + + young++; + walk->mm_stats[MM_LEAF_YOUNG]++; + + if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen >= 0 && old_gen != new_gen) + update_batch_size(walk, folio, old_gen, new_gen); + } + + if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + goto restart; + + pte_unmap(pte); + + arch_leave_lazy_mmu_mode(); + spin_unlock(ptl); + + return suitable_to_scan(total, young); +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) +static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) +{ + int i; + pmd_t *pmd; + spinlock_t *ptl; + struct lru_gen_mm_walk *walk = args->private; + struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + int old_gen, new_gen = lru_gen_from_seq(walk->max_seq); + + VM_WARN_ON_ONCE(pud_leaf(*pud)); + + /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ + if (*start == -1) { + *start = next; + return; + } + + i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start); + if (i && i <= MIN_LRU_BATCH) { + __set_bit(i - 1, bitmap); + return; + } + + pmd = pmd_offset(pud, *start); + + ptl = pmd_lockptr(args->mm, pmd); + if (!spin_trylock(ptl)) + goto done; + + arch_enter_lazy_mmu_mode(); + + do { + unsigned long pfn; + struct folio *folio; + unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start; + + pfn = get_pmd_pfn(pmd[i], vma, addr); + if (pfn == -1) + goto next; + + if (!pmd_trans_huge(pmd[i])) { + if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && + get_cap(LRU_GEN_NONLEAF_YOUNG)) + pmdp_test_and_clear_young(vma, addr, pmd + i); + goto next; + } + + folio = get_pfn_folio(pfn, memcg, pgdat, walk->can_swap); + if (!folio) + goto next; + + if (!pmdp_test_and_clear_young(vma, addr, pmd + i)) + goto next; + + walk->mm_stats[MM_LEAF_YOUNG]++; + + if (pmd_dirty(pmd[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen >= 0 && old_gen != new_gen) + update_batch_size(walk, folio, old_gen, new_gen); +next: + i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; + } while (i <= MIN_LRU_BATCH); + + arch_leave_lazy_mmu_mode(); + spin_unlock(ptl); +done: + *start = -1; + bitmap_zero(bitmap, MIN_LRU_BATCH); +} +#else +static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma, + struct mm_walk *args, unsigned long *bitmap, unsigned long *start) +{ +} +#endif + +static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pmd_t *pmd; + unsigned long next; + unsigned long addr; + struct vm_area_struct *vma; + unsigned long pos = -1; + struct lru_gen_mm_walk *walk = args->private; + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; + + VM_WARN_ON_ONCE(pud_leaf(*pud)); + + /* + * Finish an entire PMD in two passes: the first only reaches to PTE + * tables to avoid taking the PMD lock; the second, if necessary, takes + * the PMD lock to clear the accessed bit in PMD entries. + */ + pmd = pmd_offset(pud, start & PUD_MASK); +restart: + /* walk_pte_range() may call get_next_vma() */ + vma = args->vma; + for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + pmd_t val = pmd_read_atomic(pmd + i); + + /* for pmd_read_atomic() */ + barrier(); + + next = pmd_addr_end(addr, end); + + if (!pmd_present(val) || is_huge_zero_pmd(val)) { + walk->mm_stats[MM_LEAF_TOTAL]++; + continue; + } + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_trans_huge(val)) { + unsigned long pfn = pmd_pfn(val); + struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); + + walk->mm_stats[MM_LEAF_TOTAL]++; + + if (!pmd_young(val)) { + walk->mm_stats[MM_LEAF_OLD]++; + continue; + } + + /* try to avoid unnecessary memory loads */ + if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) + continue; + + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); + continue; + } +#endif + walk->mm_stats[MM_NONLEAF_TOTAL]++; + +#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG + if (get_cap(LRU_GEN_NONLEAF_YOUNG)) { + if (!pmd_young(val)) + continue; + + walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); + } +#endif + if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) + continue; + + walk->mm_stats[MM_NONLEAF_FOUND]++; + + if (!walk_pte_range(&val, addr, next, args)) + continue; + + walk->mm_stats[MM_NONLEAF_ADDED]++; + + /* carry over to the next generation */ + update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i); + } + + walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos); + + if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + goto restart; +} + +static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, + struct mm_walk *args) +{ + int i; + pud_t *pud; + unsigned long addr; + unsigned long next; + struct lru_gen_mm_walk *walk = args->private; + + VM_WARN_ON_ONCE(p4d_leaf(*p4d)); + + pud = pud_offset(p4d, start & P4D_MASK); +restart: + for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + pud_t val = READ_ONCE(pud[i]); + + next = pud_addr_end(addr, end); + + if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) + continue; + + walk_pmd_range(&val, addr, next, args); + + /* a racy check to curtail the waiting time */ + if (wq_has_sleeper(&walk->lruvec->mm_state.wait)) + return 1; + + if (need_resched() || walk->batched >= MAX_LRU_BATCH) { + end = (addr | ~PUD_MASK) + 1; + goto done; + } + } + + if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + goto restart; + + end = round_up(end, P4D_SIZE); +done: + if (!end || !args->vma) + return 1; + + walk->next_addr = max(end, args->vma->vm_start); + + return -EAGAIN; +} + +static void walk_mm(struct lruvec *lruvec, struct mm_struct *mm, struct lru_gen_mm_walk *walk) +{ + static const struct mm_walk_ops mm_walk_ops = { + .test_walk = should_skip_vma, + .p4d_entry = walk_pud_range, + }; + + int err; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + walk->next_addr = FIRST_USER_ADDRESS; + + do { + err = -EBUSY; + + /* folio_update_gen() requires stable folio_memcg() */ + if (!mem_cgroup_trylock_pages(memcg)) + break; + + /* the caller might be holding the lock for write */ + if (mmap_read_trylock(mm)) { + err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); + + mmap_read_unlock(mm); + } + + mem_cgroup_unlock_pages(); + + if (walk->batched) { + spin_lock_irq(&lruvec->lru_lock); + reset_batch_size(lruvec, walk); + spin_unlock_irq(&lruvec->lru_lock); + } + + cond_resched(); + } while (err == -EAGAIN); +} + +static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat) +{ + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + + if (pgdat && current_is_kswapd()) { + VM_WARN_ON_ONCE(walk); + + walk = &pgdat->mm_walk; + } else if (!pgdat && !walk) { + VM_WARN_ON_ONCE(current_is_kswapd()); + + walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); + } + + current->reclaim_state->mm_walk = walk; + + return walk; +} + +static void clear_mm_walk(void) +{ + struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; + + VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); + VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); + + current->reclaim_state->mm_walk = NULL; + + if (!current_is_kswapd()) + kfree(walk); +} + +static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) +{ + int zone; + int remaining = MAX_LRU_BATCH; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); + + if (type == LRU_GEN_ANON && !can_swap) + goto done; + + /* prevent cold/hot inversion if force_scan is true */ + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *head = &lrugen->lists[old_gen][type][zone]; + + while (!list_empty(head)) { + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + new_gen = folio_inc_gen(lruvec, folio, false); + list_move_tail(&folio->lru, &lrugen->lists[new_gen][type][zone]); + + if (!--remaining) + return false; + } + } +done: + reset_ctrl_pos(lruvec, type, true); + WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); + + return true; +} + +static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) +{ + int gen, type, zone; + bool success = false; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + DEFINE_MIN_SEQ(lruvec); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + + /* find the oldest populated generation */ + for (type = !can_swap; type < ANON_AND_FILE; type++) { + while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { + gen = lru_gen_from_seq(min_seq[type]); + + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + if (!list_empty(&lrugen->lists[gen][type][zone])) + goto next; + } + + min_seq[type]++; + } +next: + ; + } + + /* see the comment on lru_gen_struct */ + if (can_swap) { + min_seq[LRU_GEN_ANON] = min(min_seq[LRU_GEN_ANON], min_seq[LRU_GEN_FILE]); + min_seq[LRU_GEN_FILE] = max(min_seq[LRU_GEN_ANON], lrugen->min_seq[LRU_GEN_FILE]); + } + + for (type = !can_swap; type < ANON_AND_FILE; type++) { + if (min_seq[type] == lrugen->min_seq[type]) + continue; + + reset_ctrl_pos(lruvec, type, true); + WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); + success = true; + } + + return success; +} + +static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) +{ + int prev, next; + int type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + spin_lock_irq(&lruvec->lru_lock); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + + for (type = ANON_AND_FILE - 1; type >= 0; type--) { + if (get_nr_gens(lruvec, type) != MAX_NR_GENS) + continue; + + VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); + + while (!inc_min_seq(lruvec, type, can_swap)) { + spin_unlock_irq(&lruvec->lru_lock); + cond_resched(); + spin_lock_irq(&lruvec->lru_lock); + } + } + + /* + * Update the active/inactive LRU sizes for compatibility. Both sides of + * the current max_seq need to be covered, since max_seq+1 can overlap + * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do + * overlap, cold/hot inversion happens. + */ + prev = lru_gen_from_seq(lrugen->max_seq - 1); + next = lru_gen_from_seq(lrugen->max_seq + 1); + + for (type = 0; type < ANON_AND_FILE; type++) { + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + enum lru_list lru = type * LRU_INACTIVE_FILE; + long delta = lrugen->nr_pages[prev][type][zone] - + lrugen->nr_pages[next][type][zone]; + + if (!delta) + continue; + + __update_lru_size(lruvec, lru, zone, delta); + __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); + } + } + + for (type = 0; type < ANON_AND_FILE; type++) + reset_ctrl_pos(lruvec, type, false); + + WRITE_ONCE(lrugen->timestamps[next], jiffies); + /* make sure preceding modifications appear */ + smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); + + spin_unlock_irq(&lruvec->lru_lock); +} + +static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, + struct scan_control *sc, bool can_swap, bool force_scan) +{ + bool success; + struct lru_gen_mm_walk *walk; + struct mm_struct *mm = NULL; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq)); + + /* see the comment in iterate_mm_list() */ + if (max_seq <= READ_ONCE(lruvec->mm_state.seq)) { + success = false; + goto done; + } + + /* + * If the hardware doesn't automatically set the accessed bit, fallback + * to lru_gen_look_around(), which only clears the accessed bit in a + * handful of PTEs. Spreading the work out over a period of time usually + * is less efficient, but it avoids bursty page faults. + */ + if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { + success = iterate_mm_list_nowalk(lruvec, max_seq); + goto done; + } + + walk = set_mm_walk(NULL); + if (!walk) { + success = iterate_mm_list_nowalk(lruvec, max_seq); + goto done; + } + + walk->lruvec = lruvec; + walk->max_seq = max_seq; + walk->can_swap = can_swap; + walk->force_scan = force_scan; + + do { + success = iterate_mm_list(lruvec, walk, &mm); + if (mm) + walk_mm(lruvec, mm, walk); + + cond_resched(); + } while (mm); +done: + if (!success) { + if (sc->priority <= DEF_PRIORITY - 2) + wait_event_killable(lruvec->mm_state.wait, + max_seq < READ_ONCE(lrugen->max_seq)); + + return max_seq < READ_ONCE(lrugen->max_seq); + } + + VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); + + inc_max_seq(lruvec, can_swap, force_scan); + /* either this sees any waiters or they will see updated max_seq */ + if (wq_has_sleeper(&lruvec->mm_state.wait)) + wake_up_all(&lruvec->mm_state.wait); + + return true; +} + +static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, + struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) +{ + int gen, type, zone; + unsigned long old = 0; + unsigned long young = 0; + unsigned long total = 0; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + for (type = !can_swap; type < ANON_AND_FILE; type++) { + unsigned long seq; + + for (seq = min_seq[type]; seq <= max_seq; seq++) { + unsigned long size = 0; + + gen = lru_gen_from_seq(seq); + + for (zone = 0; zone < MAX_NR_ZONES; zone++) + size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + + total += size; + if (seq == max_seq) + young += size; + else if (seq + MIN_NR_GENS == max_seq) + old += size; + } + } + + /* try to scrape all its memory if this memcg was deleted */ + *nr_to_scan = mem_cgroup_online(memcg) ? (total >> sc->priority) : total; + + /* + * The aging tries to be lazy to reduce the overhead, while the eviction + * stalls when the number of generations reaches MIN_NR_GENS. Hence, the + * ideal number of generations is MIN_NR_GENS+1. + */ + if (min_seq[!can_swap] + MIN_NR_GENS > max_seq) + return true; + if (min_seq[!can_swap] + MIN_NR_GENS < max_seq) + return false; + + /* + * It's also ideal to spread pages out evenly, i.e., 1/(MIN_NR_GENS+1) + * of the total number of pages for each generation. A reasonable range + * for this average portion is [1/MIN_NR_GENS, 1/(MIN_NR_GENS+2)]. The + * aging cares about the upper bound of hot pages, while the eviction + * cares about the lower bound of cold pages. + */ + if (young * MIN_NR_GENS > total) + return true; + if (old * (MIN_NR_GENS + 2) < total) + return true; + + return false; +} + +static bool age_lruvec(struct lruvec *lruvec, struct scan_control *sc, unsigned long min_ttl) +{ + bool need_aging; + unsigned long nr_to_scan; + int swappiness = get_swappiness(lruvec, sc); + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + VM_WARN_ON_ONCE(sc->memcg_low_reclaim); + + mem_cgroup_calculate_protection(NULL, memcg); + + if (mem_cgroup_below_min(memcg)) + return false; + + need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); + + if (min_ttl) { + int gen = lru_gen_from_seq(min_seq[LRU_GEN_FILE]); + unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + + if (time_is_after_jiffies(birth + min_ttl)) + return false; + + /* the size is likely too small to be helpful */ + if (!nr_to_scan && sc->priority != DEF_PRIORITY) + return false; + } + + if (need_aging) + try_to_inc_max_seq(lruvec, max_seq, sc, swappiness, false); + + return true; +} + +/* to protect the working set of the last N jiffies */ +static unsigned long lru_gen_min_ttl __read_mostly; + +static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ + struct mem_cgroup *memcg; + bool success = false; + unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); + + VM_WARN_ON_ONCE(!current_is_kswapd()); + + sc->last_reclaimed = sc->nr_reclaimed; + + /* + * To reduce the chance of going into the aging path, which can be + * costly, optimistically skip it if the flag below was cleared in the + * eviction path. This improves the overall performance when multiple + * memcgs are available. + */ + if (!sc->memcgs_need_aging) { + sc->memcgs_need_aging = true; + return; + } + + set_mm_walk(pgdat); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + + if (age_lruvec(lruvec, sc, min_ttl)) + success = true; + + cond_resched(); + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + + clear_mm_walk(); + + /* check the order to exclude compaction-induced reclaim */ + if (success || !min_ttl || sc->order) + return; + + /* + * The main goal is to OOM kill if every generation from all memcgs is + * younger than min_ttl. However, another possibility is all memcgs are + * either below min or empty. + */ + if (mutex_trylock(&oom_lock)) { + struct oom_control oc = { + .gfp_mask = sc->gfp_mask, + }; + + out_of_memory(&oc); + + mutex_unlock(&oom_lock); + } +} + +/* + * This function exploits spatial locality when shrink_folio_list() walks the + * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If + * the scan was done cacheline efficiently, it adds the PMD entry pointing to + * the PTE table to the Bloom filter. This forms a feedback loop between the + * eviction and the aging. + */ +void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +{ + int i; + pte_t *pte; + unsigned long start; + unsigned long end; + unsigned long addr; + struct lru_gen_mm_walk *walk; + int young = 0; + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; + struct folio *folio = pfn_folio(pvmw->pfn); + struct mem_cgroup *memcg = folio_memcg(folio); + struct pglist_data *pgdat = folio_pgdat(folio); + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); + DEFINE_MAX_SEQ(lruvec); + int old_gen, new_gen = lru_gen_from_seq(max_seq); + + lockdep_assert_held(pvmw->ptl); + VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); + + if (spin_is_contended(pvmw->ptl)) + return; + + /* avoid taking the LRU lock under the PTL when possible */ + walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; + + start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); + end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; + + if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { + if (pvmw->address - start < MIN_LRU_BATCH * PAGE_SIZE / 2) + end = start + MIN_LRU_BATCH * PAGE_SIZE; + else if (end - pvmw->address < MIN_LRU_BATCH * PAGE_SIZE / 2) + start = end - MIN_LRU_BATCH * PAGE_SIZE; + else { + start = pvmw->address - MIN_LRU_BATCH * PAGE_SIZE / 2; + end = pvmw->address + MIN_LRU_BATCH * PAGE_SIZE / 2; + } + } + + pte = pvmw->pte - (pvmw->address - start) / PAGE_SIZE; + + rcu_read_lock(); + arch_enter_lazy_mmu_mode(); + + for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { + unsigned long pfn; + + pfn = get_pte_pfn(pte[i], pvmw->vma, addr); + if (pfn == -1) + continue; + + if (!pte_young(pte[i])) + continue; + + folio = get_pfn_folio(pfn, memcg, pgdat, !walk || walk->can_swap); + if (!folio) + continue; + + if (!ptep_test_and_clear_young(pvmw->vma, addr, pte + i)) + VM_WARN_ON_ONCE(true); + + young++; + + if (pte_dirty(pte[i]) && !folio_test_dirty(folio) && + !(folio_test_anon(folio) && folio_test_swapbacked(folio) && + !folio_test_swapcache(folio))) + folio_mark_dirty(folio); + + old_gen = folio_lru_gen(folio); + if (old_gen < 0) + folio_set_referenced(folio); + else if (old_gen != new_gen) + __set_bit(i, bitmap); + } + + arch_leave_lazy_mmu_mode(); + rcu_read_unlock(); + + /* feedback from rmap walkers to page table walkers */ + if (suitable_to_scan(i, young)) + update_bloom_filter(lruvec, max_seq, pvmw->pmd); + + if (!walk && bitmap_weight(bitmap, MIN_LRU_BATCH) < PAGEVEC_SIZE) { + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { + folio = pfn_folio(pte_pfn(pte[i])); + folio_activate(folio); + } + return; + } + + /* folio_update_gen() requires stable folio_memcg() */ + if (!mem_cgroup_trylock_pages(memcg)) + return; + + if (!walk) { + spin_lock_irq(&lruvec->lru_lock); + new_gen = lru_gen_from_seq(lruvec->lrugen.max_seq); + } + + for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { + folio = pfn_folio(pte_pfn(pte[i])); + if (folio_memcg_rcu(folio) != memcg) + continue; + + old_gen = folio_update_gen(folio, new_gen); + if (old_gen < 0 || old_gen == new_gen) + continue; + + if (walk) + update_batch_size(walk, folio, old_gen, new_gen); + else + lru_gen_update_size(lruvec, folio, old_gen, new_gen); + } + + if (!walk) + spin_unlock_irq(&lruvec->lru_lock); + + mem_cgroup_unlock_pages(); +} + +/****************************************************************************** + * the eviction + ******************************************************************************/ + +static bool sort_folio(struct lruvec *lruvec, struct folio *folio, int tier_idx) +{ + bool success; + int gen = folio_lru_gen(folio); + int type = folio_is_file_lru(folio); + int zone = folio_zonenum(folio); + int delta = folio_nr_pages(folio); + int refs = folio_lru_refs(folio); + int tier = lru_tier_from_refs(refs); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); + + /* unevictable */ + if (!folio_evictable(folio)) { + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + folio_set_unevictable(folio); + lruvec_add_folio(lruvec, folio); + __count_vm_events(UNEVICTABLE_PGCULLED, delta); + return true; + } + + /* dirty lazyfree */ + if (type == LRU_GEN_FILE && folio_test_anon(folio) && folio_test_dirty(folio)) { + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + folio_set_swapbacked(folio); + lruvec_add_folio_tail(lruvec, folio); + return true; + } + + /* promoted */ + if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { + list_move(&folio->lru, &lrugen->lists[gen][type][zone]); + return true; + } + + /* protected */ + if (tier > tier_idx) { + int hist = lru_hist_from_seq(lrugen->min_seq[type]); + + gen = folio_inc_gen(lruvec, folio, false); + list_move_tail(&folio->lru, &lrugen->lists[gen][type][zone]); + + WRITE_ONCE(lrugen->protected[hist][type][tier - 1], + lrugen->protected[hist][type][tier - 1] + delta); + __mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta); + return true; + } + + /* waiting for writeback */ + if (folio_test_locked(folio) || folio_test_writeback(folio) || + (type == LRU_GEN_FILE && folio_test_dirty(folio))) { + gen = folio_inc_gen(lruvec, folio, true); + list_move(&folio->lru, &lrugen->lists[gen][type][zone]); + return true; + } + + return false; +} + +static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) +{ + bool success; + + /* unmapping inhibited */ + if (!sc->may_unmap && folio_mapped(folio)) + return false; + + /* swapping inhibited */ + if (!(sc->may_writepage && (sc->gfp_mask & __GFP_IO)) && + (folio_test_dirty(folio) || + (folio_test_anon(folio) && !folio_test_swapcache(folio)))) + return false; + + /* raced with release_pages() */ + if (!folio_try_get(folio)) + return false; + + /* raced with another isolation */ + if (!folio_test_clear_lru(folio)) { + folio_put(folio); + return false; + } + + /* see the comment on MAX_NR_TIERS */ + if (!folio_test_referenced(folio)) + set_mask_bits(&folio->flags, LRU_REFS_MASK | LRU_REFS_FLAGS, 0); + + /* for shrink_folio_list() */ + folio_clear_reclaim(folio); + folio_clear_referenced(folio); + + success = lru_gen_del_folio(lruvec, folio, true); + VM_WARN_ON_ONCE_FOLIO(!success, folio); + + return true; +} + +static int scan_folios(struct lruvec *lruvec, struct scan_control *sc, + int type, int tier, struct list_head *list) +{ + int gen, zone; + enum vm_event_item item; + int sorted = 0; + int scanned = 0; + int isolated = 0; + int remaining = MAX_LRU_BATCH; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + VM_WARN_ON_ONCE(!list_empty(list)); + + if (get_nr_gens(lruvec, type) == MIN_NR_GENS) + return 0; + + gen = lru_gen_from_seq(lrugen->min_seq[type]); + + for (zone = sc->reclaim_idx; zone >= 0; zone--) { + LIST_HEAD(moved); + int skipped = 0; + struct list_head *head = &lrugen->lists[gen][type][zone]; + + while (!list_empty(head)) { + struct folio *folio = lru_to_folio(head); + int delta = folio_nr_pages(folio); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + scanned += delta; + + if (sort_folio(lruvec, folio, tier)) + sorted += delta; + else if (isolate_folio(lruvec, folio, sc)) { + list_add(&folio->lru, list); + isolated += delta; + } else { + list_move(&folio->lru, &moved); + skipped += delta; + } + + if (!--remaining || max(isolated, skipped) >= MIN_LRU_BATCH) + break; + } + + if (skipped) { + list_splice(&moved, head); + __count_zid_vm_events(PGSCAN_SKIP, zone, skipped); + } + + if (!remaining || isolated >= MIN_LRU_BATCH) + break; + } + + item = current_is_kswapd() ? PGSCAN_KSWAPD : PGSCAN_DIRECT; + if (!cgroup_reclaim(sc)) { + __count_vm_events(item, isolated); + __count_vm_events(PGREFILL, sorted); + } + __count_memcg_events(memcg, item, isolated); + __count_memcg_events(memcg, PGREFILL, sorted); + __count_vm_events(PGSCAN_ANON + type, isolated); + + /* + * There might not be eligible pages due to reclaim_idx, may_unmap and + * may_writepage. Check the remaining to prevent livelock if it's not + * making progress. + */ + return isolated || !remaining ? scanned : 0; +} + +static int get_tier_idx(struct lruvec *lruvec, int type) +{ + int tier; + struct ctrl_pos sp, pv; + + /* + * To leave a margin for fluctuations, use a larger gain factor (1:2). + * This value is chosen because any other tier would have at least twice + * as many refaults as the first tier. + */ + read_ctrl_pos(lruvec, type, 0, 1, &sp); + for (tier = 1; tier < MAX_NR_TIERS; tier++) { + read_ctrl_pos(lruvec, type, tier, 2, &pv); + if (!positive_ctrl_err(&sp, &pv)) + break; + } + + return tier - 1; +} + +static int get_type_to_scan(struct lruvec *lruvec, int swappiness, int *tier_idx) +{ + int type, tier; + struct ctrl_pos sp, pv; + int gain[ANON_AND_FILE] = { swappiness, 200 - swappiness }; + + /* + * Compare the first tier of anon with that of file to determine which + * type to scan. Also need to compare other tiers of the selected type + * with the first tier of the other type to determine the last tier (of + * the selected type) to evict. + */ + read_ctrl_pos(lruvec, LRU_GEN_ANON, 0, gain[LRU_GEN_ANON], &sp); + read_ctrl_pos(lruvec, LRU_GEN_FILE, 0, gain[LRU_GEN_FILE], &pv); + type = positive_ctrl_err(&sp, &pv); + + read_ctrl_pos(lruvec, !type, 0, gain[!type], &sp); + for (tier = 1; tier < MAX_NR_TIERS; tier++) { + read_ctrl_pos(lruvec, type, tier, gain[type], &pv); + if (!positive_ctrl_err(&sp, &pv)) + break; + } + + *tier_idx = tier - 1; + + return type; +} + +static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, + int *type_scanned, struct list_head *list) +{ + int i; + int type; + int scanned; + int tier = -1; + DEFINE_MIN_SEQ(lruvec); + + /* + * Try to make the obvious choice first. When anon and file are both + * available from the same generation, interpret swappiness 1 as file + * first and 200 as anon first. + */ + if (!swappiness) + type = LRU_GEN_FILE; + else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) + type = LRU_GEN_ANON; + else if (swappiness == 1) + type = LRU_GEN_FILE; + else if (swappiness == 200) + type = LRU_GEN_ANON; + else + type = get_type_to_scan(lruvec, swappiness, &tier); + + for (i = !swappiness; i < ANON_AND_FILE; i++) { + if (tier < 0) + tier = get_tier_idx(lruvec, type); + + scanned = scan_folios(lruvec, sc, type, tier, list); + if (scanned) + break; + + type = !type; + tier = -1; + } + + *type_scanned = type; + + return scanned; +} + +static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness, + bool *need_swapping) +{ + int type; + int scanned; + int reclaimed; + LIST_HEAD(list); + struct folio *folio; + enum vm_event_item item; + struct reclaim_stat stat; + struct lru_gen_mm_walk *walk; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + + spin_lock_irq(&lruvec->lru_lock); + + scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); + + scanned += try_to_inc_min_seq(lruvec, swappiness); + + if (get_nr_gens(lruvec, !swappiness) == MIN_NR_GENS) + scanned = 0; + + spin_unlock_irq(&lruvec->lru_lock); + + if (list_empty(&list)) + return scanned; + + reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false); + + list_for_each_entry(folio, &list, lru) { + /* restore LRU_REFS_FLAGS cleared by isolate_folio() */ + if (folio_test_workingset(folio)) + folio_set_referenced(folio); + + /* don't add rejected pages to the oldest generation */ + if (folio_test_reclaim(folio) && + (folio_test_dirty(folio) || folio_test_writeback(folio))) + folio_clear_active(folio); + else + folio_set_active(folio); + } + + spin_lock_irq(&lruvec->lru_lock); + + move_folios_to_lru(lruvec, &list); + + walk = current->reclaim_state->mm_walk; + if (walk && walk->batched) + reset_batch_size(lruvec, walk); + + item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; + if (!cgroup_reclaim(sc)) + __count_vm_events(item, reclaimed); + __count_memcg_events(memcg, item, reclaimed); + __count_vm_events(PGSTEAL_ANON + type, reclaimed); + + spin_unlock_irq(&lruvec->lru_lock); + + mem_cgroup_uncharge_list(&list); + free_unref_page_list(&list); + + sc->nr_reclaimed += reclaimed; + + if (need_swapping && type == LRU_GEN_ANON) + *need_swapping = true; + + return scanned; +} + +/* + * For future optimizations: + * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg + * reclaim. + */ +static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, + bool can_swap, bool *need_aging) +{ + unsigned long nr_to_scan; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (mem_cgroup_below_min(memcg) || + (mem_cgroup_below_low(memcg) && !sc->memcg_low_reclaim)) + return 0; + + *need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, can_swap, &nr_to_scan); + if (!*need_aging) + return nr_to_scan; + + /* skip the aging path at the default priority */ + if (sc->priority == DEF_PRIORITY) + goto done; + + /* leave the work to lru_gen_age_node() */ + if (current_is_kswapd()) + return 0; + + if (try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, false)) + return nr_to_scan; +done: + return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; +} + +static bool should_abort_scan(struct lruvec *lruvec, unsigned long seq, + struct scan_control *sc, bool need_swapping) +{ + int i; + DEFINE_MAX_SEQ(lruvec); + + if (!current_is_kswapd()) { + /* age each memcg at most once to ensure fairness */ + if (max_seq - seq > 1) + return true; + + /* over-swapping can increase allocation latency */ + if (sc->nr_reclaimed >= sc->nr_to_reclaim && need_swapping) + return true; + + /* give this thread a chance to exit and free its memory */ + if (fatal_signal_pending(current)) { + sc->nr_reclaimed += MIN_LRU_BATCH; + return true; + } + + if (cgroup_reclaim(sc)) + return false; + } else if (sc->nr_reclaimed - sc->last_reclaimed < sc->nr_to_reclaim) + return false; + + /* keep scanning at low priorities to ensure fairness */ + if (sc->priority > DEF_PRIORITY - 2) + return false; + + /* + * A minimum amount of work was done under global memory pressure. For + * kswapd, it may be overshooting. For direct reclaim, the allocation + * may succeed if all suitable zones are somewhat safe. In either case, + * it's better to stop now, and restart later if necessary. + */ + for (i = 0; i <= sc->reclaim_idx; i++) { + unsigned long wmark; + struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; + + if (!managed_zone(zone)) + continue; + + wmark = current_is_kswapd() ? high_wmark_pages(zone) : low_wmark_pages(zone); + if (wmark > zone_page_state(zone, NR_FREE_PAGES)) + return false; + } + + sc->nr_reclaimed += MIN_LRU_BATCH; + + return true; +} + +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ + struct blk_plug plug; + bool need_aging = false; + bool need_swapping = false; + unsigned long scanned = 0; + unsigned long reclaimed = sc->nr_reclaimed; + DEFINE_MAX_SEQ(lruvec); + + lru_add_drain(); + + blk_start_plug(&plug); + + set_mm_walk(lruvec_pgdat(lruvec)); + + while (true) { + int delta; + int swappiness; + unsigned long nr_to_scan; + + if (sc->may_swap) + swappiness = get_swappiness(lruvec, sc); + else if (!cgroup_reclaim(sc) && get_swappiness(lruvec, sc)) + swappiness = 1; + else + swappiness = 0; + + nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness, &need_aging); + if (!nr_to_scan) + goto done; + + delta = evict_folios(lruvec, sc, swappiness, &need_swapping); + if (!delta) + goto done; + + scanned += delta; + if (scanned >= nr_to_scan) + break; + + if (should_abort_scan(lruvec, max_seq, sc, need_swapping)) + break; + + cond_resched(); + } + + /* see the comment in lru_gen_age_node() */ + if (sc->nr_reclaimed - reclaimed >= MIN_LRU_BATCH && !need_aging) + sc->memcgs_need_aging = false; +done: + clear_mm_walk(); + + blk_finish_plug(&plug); +} + +/****************************************************************************** + * state change + ******************************************************************************/ + +static bool __maybe_unused state_is_valid(struct lruvec *lruvec) +{ + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + if (lrugen->enabled) { + enum lru_list lru; + + for_each_evictable_lru(lru) { + if (!list_empty(&lruvec->lists[lru])) + return false; + } + } else { + int gen, type, zone; + + for_each_gen_type_zone(gen, type, zone) { + if (!list_empty(&lrugen->lists[gen][type][zone])) + return false; + } + } + + return true; +} + +static bool fill_evictable(struct lruvec *lruvec) +{ + enum lru_list lru; + int remaining = MAX_LRU_BATCH; + + for_each_evictable_lru(lru) { + int type = is_file_lru(lru); + bool active = is_active_lru(lru); + struct list_head *head = &lruvec->lists[lru]; + + while (!list_empty(head)) { + bool success; + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); + + lruvec_del_folio(lruvec, folio); + success = lru_gen_add_folio(lruvec, folio, false); + VM_WARN_ON_ONCE(!success); + + if (!--remaining) + return false; + } + } + + return true; +} + +static bool drain_evictable(struct lruvec *lruvec) +{ + int gen, type, zone; + int remaining = MAX_LRU_BATCH; + + for_each_gen_type_zone(gen, type, zone) { + struct list_head *head = &lruvec->lrugen.lists[gen][type][zone]; + + while (!list_empty(head)) { + bool success; + struct folio *folio = lru_to_folio(head); + + VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); + VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); + VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); + + success = lru_gen_del_folio(lruvec, folio, false); + VM_WARN_ON_ONCE(!success); + lruvec_add_folio(lruvec, folio); + + if (!--remaining) + return false; + } + } + + return true; +} + +static void lru_gen_change_state(bool enabled) +{ + static DEFINE_MUTEX(state_mutex); + + struct mem_cgroup *memcg; + + cgroup_lock(); + cpus_read_lock(); + get_online_mems(); + mutex_lock(&state_mutex); + + if (enabled == lru_gen_enabled()) + goto unlock; + + if (enabled) + static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); + else + static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + int nid; + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + if (!lruvec) + continue; + + spin_lock_irq(&lruvec->lru_lock); + + VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); + VM_WARN_ON_ONCE(!state_is_valid(lruvec)); + + lruvec->lrugen.enabled = enabled; + + while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { + spin_unlock_irq(&lruvec->lru_lock); + cond_resched(); + spin_lock_irq(&lruvec->lru_lock); + } + + spin_unlock_irq(&lruvec->lru_lock); + } + + cond_resched(); + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); +unlock: + mutex_unlock(&state_mutex); + put_online_mems(); + cpus_read_unlock(); + cgroup_unlock(); +} + +/****************************************************************************** + * sysfs interface + ******************************************************************************/ + +static ssize_t show_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t store_min_ttl(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) +{ + unsigned int msecs; + + if (kstrtouint(buf, 0, &msecs)) + return -EINVAL; + + WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); + + return len; +} + +static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR( + min_ttl_ms, 0644, show_min_ttl, store_min_ttl +); + +static ssize_t show_enabled(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + unsigned int caps = 0; + + if (get_cap(LRU_GEN_CORE)) + caps |= BIT(LRU_GEN_CORE); + + if (arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK)) + caps |= BIT(LRU_GEN_MM_WALK); + + if (IS_ENABLED(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) && get_cap(LRU_GEN_NONLEAF_YOUNG)) + caps |= BIT(LRU_GEN_NONLEAF_YOUNG); + + return snprintf(buf, PAGE_SIZE, "0x%04x\n", caps); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t store_enabled(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t len) +{ + int i; + unsigned int caps; + + if (tolower(*buf) == 'n') + caps = 0; + else if (tolower(*buf) == 'y') + caps = -1; + else if (kstrtouint(buf, 0, &caps)) + return -EINVAL; + + for (i = 0; i < NR_LRU_GEN_CAPS; i++) { + bool enabled = caps & BIT(i); + + if (i == LRU_GEN_CORE) + lru_gen_change_state(enabled); + else if (enabled) + static_branch_enable(&lru_gen_caps[i]); + else + static_branch_disable(&lru_gen_caps[i]); + } + + return len; +} + +static struct kobj_attribute lru_gen_enabled_attr = __ATTR( + enabled, 0644, show_enabled, store_enabled +); + +static struct attribute *lru_gen_attrs[] = { + &lru_gen_min_ttl_attr.attr, + &lru_gen_enabled_attr.attr, + NULL +}; + +static struct attribute_group lru_gen_attr_group = { + .name = "lru_gen", + .attrs = lru_gen_attrs, +}; + +/****************************************************************************** + * debugfs interface + ******************************************************************************/ + +static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) +{ + struct mem_cgroup *memcg; + loff_t nr_to_skip = *pos; + + m->private = kvmalloc(PATH_MAX, GFP_KERNEL); + if (!m->private) + return ERR_PTR(-ENOMEM); + + memcg = mem_cgroup_iter(NULL, NULL, NULL); + do { + int nid; + + for_each_node_state(nid, N_MEMORY) { + if (!nr_to_skip--) + return get_lruvec(memcg, nid); + } + } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); + + return NULL; +} + +static void lru_gen_seq_stop(struct seq_file *m, void *v) +{ + if (!IS_ERR_OR_NULL(v)) + mem_cgroup_iter_break(NULL, lruvec_memcg(v)); + + kvfree(m->private); + m->private = NULL; +} + +static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + int nid = lruvec_pgdat(v)->node_id; + struct mem_cgroup *memcg = lruvec_memcg(v); + + ++*pos; + + nid = next_memory_node(nid); + if (nid == MAX_NUMNODES) { + memcg = mem_cgroup_iter(NULL, memcg, NULL); + if (!memcg) + return NULL; + + nid = first_memory_node; + } + + return get_lruvec(memcg, nid); +} + +static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, + unsigned long max_seq, unsigned long *min_seq, + unsigned long seq) +{ + int i; + int type, tier; + int hist = lru_hist_from_seq(seq); + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + for (tier = 0; tier < MAX_NR_TIERS; tier++) { + seq_printf(m, " %10d", tier); + for (type = 0; type < ANON_AND_FILE; type++) { + const char *s = " "; + unsigned long n[3] = {}; + + if (seq == max_seq) { + s = "RT "; + n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); + n[1] = READ_ONCE(lrugen->avg_total[type][tier]); + } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { + s = "rep"; + n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); + n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); + if (tier) + n[2] = READ_ONCE(lrugen->protected[hist][type][tier - 1]); + } + + for (i = 0; i < 3; i++) + seq_printf(m, " %10lu%c", n[i], s[i]); + } + seq_putc(m, '\n'); + } + + seq_puts(m, " "); + for (i = 0; i < NR_MM_STATS; i++) { + const char *s = " "; + unsigned long n = 0; + + if (seq == max_seq && NR_HIST_GENS == 1) { + s = "LOYNFA"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } else if (seq != max_seq && NR_HIST_GENS > 1) { + s = "loynfa"; + n = READ_ONCE(lruvec->mm_state.stats[hist][i]); + } + + seq_printf(m, " %10lu%c", n, s[i]); + } + seq_putc(m, '\n'); +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static int lru_gen_seq_show(struct seq_file *m, void *v) +{ + unsigned long seq; + bool full = !debugfs_real_fops(m->file)->write; + struct lruvec *lruvec = v; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + int nid = lruvec_pgdat(lruvec)->node_id; + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (nid == first_memory_node) { + const char *path = memcg ? m->private : ""; + +#ifdef CONFIG_MEMCG + if (memcg) + cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); +#endif + seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); + } + + seq_printf(m, " node %5d\n", nid); + + if (!full) + seq = min_seq[LRU_GEN_ANON]; + else if (max_seq >= MAX_NR_GENS) + seq = max_seq - MAX_NR_GENS + 1; + else + seq = 0; + + for (; seq <= max_seq; seq++) { + int type, zone; + int gen = lru_gen_from_seq(seq); + unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); + + seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); + + for (type = 0; type < ANON_AND_FILE; type++) { + unsigned long size = 0; + char mark = full && seq < min_seq[type] ? 'x' : ' '; + + for (zone = 0; zone < MAX_NR_ZONES; zone++) + size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + + seq_printf(m, " %10lu%c", size, mark); + } + + seq_putc(m, '\n'); + + if (full) + lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); + } + + return 0; +} + +static const struct seq_operations lru_gen_seq_ops = { + .start = lru_gen_seq_start, + .stop = lru_gen_seq_stop, + .next = lru_gen_seq_next, + .show = lru_gen_seq_show, +}; + +static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, + bool can_swap, bool force_scan) +{ + DEFINE_MAX_SEQ(lruvec); + DEFINE_MIN_SEQ(lruvec); + + if (seq < max_seq) + return 0; + + if (seq > max_seq) + return -EINVAL; + + if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) + return -ERANGE; + + try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); + + return 0; +} + +static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, + int swappiness, unsigned long nr_to_reclaim) +{ + DEFINE_MAX_SEQ(lruvec); + + if (seq + MIN_NR_GENS > max_seq) + return -EINVAL; + + sc->nr_reclaimed = 0; + + while (!signal_pending(current)) { + DEFINE_MIN_SEQ(lruvec); + + if (seq < min_seq[!swappiness]) + return 0; + + if (sc->nr_reclaimed >= nr_to_reclaim) + return 0; + + if (!evict_folios(lruvec, sc, swappiness, NULL)) + return 0; + + cond_resched(); + } + + return -EINTR; +} + +static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, + struct scan_control *sc, int swappiness, unsigned long opt) +{ + struct lruvec *lruvec; + int err = -EINVAL; + struct mem_cgroup *memcg = NULL; + + if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) + return -EINVAL; + + if (!mem_cgroup_disabled()) { + rcu_read_lock(); + memcg = mem_cgroup_from_id(memcg_id); +#ifdef CONFIG_MEMCG + if (memcg && !css_tryget(&memcg->css)) + memcg = NULL; +#endif + rcu_read_unlock(); + + if (!memcg) + return -EINVAL; + } + + if (memcg_id != mem_cgroup_id(memcg)) + goto done; + + lruvec = get_lruvec(memcg, nid); + + if (swappiness < 0) + swappiness = get_swappiness(lruvec, sc); + else if (swappiness > 200) + goto done; + + switch (cmd) { + case '+': + err = run_aging(lruvec, seq, sc, swappiness, opt); + break; + case '-': + err = run_eviction(lruvec, seq, sc, swappiness, opt); + break; + } +done: + mem_cgroup_put(memcg); + + return err; +} + +/* see Documentation/admin-guide/mm/multigen_lru.rst for details */ +static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, + size_t len, loff_t *pos) +{ + void *buf; + char *cur, *next; + unsigned int flags; + struct blk_plug plug; + int err = -EINVAL; + struct scan_control sc = { + .may_writepage = true, + .may_unmap = true, + .may_swap = true, + .reclaim_idx = MAX_NR_ZONES - 1, + .gfp_mask = GFP_KERNEL, + }; + + buf = kvmalloc(len + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, src, len)) { + kvfree(buf); + return -EFAULT; + } + + set_task_reclaim_state(current, &sc.reclaim_state); + flags = memalloc_noreclaim_save(); + blk_start_plug(&plug); + if (!set_mm_walk(NULL)) { + err = -ENOMEM; + goto done; + } + + next = buf; + next[len] = '\0'; + + while ((cur = strsep(&next, ",;\n"))) { + int n; + int end; + char cmd; + unsigned int memcg_id; + unsigned int nid; + unsigned long seq; + unsigned int swappiness = -1; + unsigned long opt = -1; + + cur = skip_spaces(cur); + if (!*cur) + continue; + + n = sscanf(cur, "%c %u %u %lu %n %u %n %lu %n", &cmd, &memcg_id, &nid, + &seq, &end, &swappiness, &end, &opt, &end); + if (n < 4 || cur[end]) { + err = -EINVAL; + break; + } + + err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); + if (err) + break; + } +done: + clear_mm_walk(); + blk_finish_plug(&plug); + memalloc_noreclaim_restore(flags); + set_task_reclaim_state(current, NULL); + + kvfree(buf); + + return err ? : len; +} + +static int lru_gen_seq_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &lru_gen_seq_ops); +} + +static const struct file_operations lru_gen_rw_fops = { + .open = lru_gen_seq_open, + .read = seq_read, + .write = lru_gen_seq_write, + .llseek = seq_lseek, + .release = seq_release, +}; + +static const struct file_operations lru_gen_ro_fops = { + .open = lru_gen_seq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +/****************************************************************************** + * initialization + ******************************************************************************/ + +void lru_gen_init_lruvec(struct lruvec *lruvec) +{ + int i; + int gen, type, zone; + struct lru_gen_struct *lrugen = &lruvec->lrugen; + + lrugen->max_seq = MIN_NR_GENS + 1; + lrugen->enabled = lru_gen_enabled(); + + for (i = 0; i <= MIN_NR_GENS + 1; i++) + lrugen->timestamps[i] = jiffies; + + for_each_gen_type_zone(gen, type, zone) + INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); + + lruvec->mm_state.seq = MIN_NR_GENS; + init_waitqueue_head(&lruvec->mm_state.wait); +} + +#ifdef CONFIG_MEMCG +void lru_gen_init_memcg(struct mem_cgroup *memcg) +{ + INIT_LIST_HEAD(&memcg->mm_list.fifo); + spin_lock_init(&memcg->mm_list.lock); +} + +void lru_gen_exit_memcg(struct mem_cgroup *memcg) +{ + int i; + int nid; + + for_each_node(nid) { + struct lruvec *lruvec = get_lruvec(memcg, nid); + + VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, + sizeof(lruvec->lrugen.nr_pages))); + + for (i = 0; i < NR_BLOOM_FILTERS; i++) { + bitmap_free(lruvec->mm_state.filters[i]); + lruvec->mm_state.filters[i] = NULL; + } + } +} +#endif + +static int __init init_lru_gen(void) +{ + BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); + BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); + + if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) + pr_err("lru_gen: failed to create sysfs group\n"); + + debugfs_create_file("lru_gen", 0644, NULL, NULL, &lru_gen_rw_fops); + debugfs_create_file("lru_gen_full", 0444, NULL, NULL, &lru_gen_ro_fops); + + return 0; +}; +late_initcall(init_lru_gen); + +#else /* !CONFIG_LRU_GEN */ + +static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +{ +} + +static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ +} + +#endif /* CONFIG_LRU_GEN */ + +static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) +{ + unsigned long nr[NR_LRU_LISTS]; + unsigned long targets[NR_LRU_LISTS]; + unsigned long nr_to_scan; + enum lru_list lru; + unsigned long nr_reclaimed = 0; + unsigned long nr_to_reclaim = sc->nr_to_reclaim; + struct blk_plug plug; + bool scan_adjusted; + + if (lru_gen_enabled()) { + lru_gen_shrink_lruvec(lruvec, sc); + return; + } + + get_scan_count(lruvec, sc, nr); + + /* Record the original scan target for proportional adjustments later */ + memcpy(targets, nr, sizeof(nr)); + + /* + * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal + * event that can occur when there is little memory pressure e.g. + * multiple streaming readers/writers. Hence, we do not abort scanning + * when the requested number of pages are reclaimed when scanning at + * DEF_PRIORITY on the assumption that the fact we are direct + * reclaiming implies that kswapd is not keeping up and it is best to + * do a batch of work at once. For memcg reclaim one check is made to + * abort proportional reclaim if either the file or anon lru has already + * dropped to zero at the first pass. + */ + scan_adjusted = (!cgroup_reclaim(sc) && !current_is_kswapd() && + sc->priority == DEF_PRIORITY); + + blk_start_plug(&plug); + while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || + nr[LRU_INACTIVE_FILE]) { + unsigned long nr_anon, nr_file, percentage; + unsigned long nr_scanned; + + for_each_evictable_lru(lru) { + if (nr[lru]) { + nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); + nr[lru] -= nr_to_scan; + + nr_reclaimed += shrink_list(lru, nr_to_scan, + lruvec, sc); + } + } + + cond_resched(); + + if (nr_reclaimed < nr_to_reclaim || scan_adjusted) + continue; + + /* + * For kswapd and memcg, reclaim at least the number of pages + * requested. Ensure that the anon and file LRUs are scanned + * proportionally what was requested by get_scan_count(). We + * stop reclaiming one LRU and reduce the amount scanning + * proportional to the original scan target. + */ + nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; + nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; + + /* + * It's just vindictive to attack the larger once the smaller + * has gone to zero. And given the way we stop scanning the + * smaller below, this makes sure that we only make one nudge + * towards proportionality once we've got nr_to_reclaim. + */ + if (!nr_file || !nr_anon) + break; + + if (nr_file > nr_anon) { + unsigned long scan_target = targets[LRU_INACTIVE_ANON] + + targets[LRU_ACTIVE_ANON] + 1; + lru = LRU_BASE; + percentage = nr_anon * 100 / scan_target; + } else { + unsigned long scan_target = targets[LRU_INACTIVE_FILE] + + targets[LRU_ACTIVE_FILE] + 1; + lru = LRU_FILE; + percentage = nr_file * 100 / scan_target; + } + + /* Stop scanning the smaller of the LRU */ + nr[lru] = 0; + nr[lru + LRU_ACTIVE] = 0; + + /* + * Recalculate the other LRU scan count based on its original + * scan target and the percentage scanning already complete + */ + lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; + nr_scanned = targets[lru] - nr[lru]; + nr[lru] = targets[lru] * (100 - percentage) / 100; + nr[lru] -= min(nr[lru], nr_scanned); + + lru += LRU_ACTIVE; + nr_scanned = targets[lru] - nr[lru]; + nr[lru] = targets[lru] * (100 - percentage) / 100; + nr[lru] -= min(nr[lru], nr_scanned); + + scan_adjusted = true; + } + blk_finish_plug(&plug); + sc->nr_reclaimed += nr_reclaimed; + + /* + * Even if we did not try to evict anon pages at all, we want to + * rebalance the anon lru active/inactive ratio. + */ + if (can_age_anon_pages(lruvec_pgdat(lruvec), sc) && + inactive_is_low(lruvec, LRU_INACTIVE_ANON)) + shrink_active_list(SWAP_CLUSTER_MAX, lruvec, + sc, LRU_ACTIVE_ANON); +} + +/* Use reclaim/compaction for costly allocs or under memory pressure */ +static bool in_reclaim_compaction(struct scan_control *sc) +{ + if (IS_ENABLED(CONFIG_COMPACTION) && sc->order && + (sc->order > PAGE_ALLOC_COSTLY_ORDER || + sc->priority < DEF_PRIORITY - 2)) + return true; + + return false; +} + +/* + * Reclaim/compaction is used for high-order allocation requests. It reclaims + * order-0 pages before compacting the zone. should_continue_reclaim() returns + * true if more pages should be reclaimed such that when the page allocator * calls try_to_compact_pages() that it will have enough free pages to succeed. * It will give up earlier than that if there is difficulty reclaiming pages. */ @@ -3197,109 +6091,16 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) unsigned long nr_reclaimed, nr_scanned; struct lruvec *target_lruvec; bool reclaimable = false; - unsigned long file; target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); again: - /* - * Flush the memory cgroup stats, so that we read accurate per-memcg - * lruvec stats for heuristics. - */ - mem_cgroup_flush_stats(); - memset(&sc->nr, 0, sizeof(sc->nr)); nr_reclaimed = sc->nr_reclaimed; nr_scanned = sc->nr_scanned; - /* - * Determine the scan balance between anon and file LRUs. - */ - spin_lock_irq(&target_lruvec->lru_lock); - sc->anon_cost = target_lruvec->anon_cost; - sc->file_cost = target_lruvec->file_cost; - spin_unlock_irq(&target_lruvec->lru_lock); - - /* - * Target desirable inactive:active list ratios for the anon - * and file LRU lists. - */ - if (!sc->force_deactivate) { - unsigned long refaults; - - refaults = lruvec_page_state(target_lruvec, - WORKINGSET_ACTIVATE_ANON); - if (refaults != target_lruvec->refaults[0] || - inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) - sc->may_deactivate |= DEACTIVATE_ANON; - else - sc->may_deactivate &= ~DEACTIVATE_ANON; - - /* - * When refaults are being observed, it means a new - * workingset is being established. Deactivate to get - * rid of any stale active pages quickly. - */ - refaults = lruvec_page_state(target_lruvec, - WORKINGSET_ACTIVATE_FILE); - if (refaults != target_lruvec->refaults[1] || - inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) - sc->may_deactivate |= DEACTIVATE_FILE; - else - sc->may_deactivate &= ~DEACTIVATE_FILE; - } else - sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; - - /* - * If we have plenty of inactive file pages that aren't - * thrashing, try to reclaim those first before touching - * anonymous pages. - */ - file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); - if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE)) - sc->cache_trim_mode = 1; - else - sc->cache_trim_mode = 0; - - /* - * Prevent the reclaimer from falling into the cache trap: as - * cache pages start out inactive, every cache fault will tip - * the scan balance towards the file LRU. And as the file LRU - * shrinks, so does the window for rotation from references. - * This means we have a runaway feedback loop where a tiny - * thrashing file LRU becomes infinitely more attractive than - * anon pages. Try to detect this based on file LRU size. - */ - if (!cgroup_reclaim(sc)) { - unsigned long total_high_wmark = 0; - unsigned long free, anon; - int z; - - free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); - file = node_page_state(pgdat, NR_ACTIVE_FILE) + - node_page_state(pgdat, NR_INACTIVE_FILE); - - for (z = 0; z < MAX_NR_ZONES; z++) { - struct zone *zone = &pgdat->node_zones[z]; - if (!managed_zone(zone)) - continue; - - total_high_wmark += high_wmark_pages(zone); - } - - /* - * Consider anon: if that's low too, this isn't a - * runaway file reclaim problem, but rather just - * extreme pressure. Reclaim as per usual then. - */ - anon = node_page_state(pgdat, NR_INACTIVE_ANON); - - sc->file_is_tiny = - file + free <= total_high_wmark && - !(sc->may_deactivate & DEACTIVATE_ANON) && - anon >> sc->priority; - } + prepare_scan_count(pgdat, sc); shrink_node_memcgs(pgdat, sc); @@ -3557,11 +6358,14 @@ static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) struct lruvec *target_lruvec; unsigned long refaults; + if (lru_gen_enabled()) + return; + target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); - target_lruvec->refaults[0] = refaults; + target_lruvec->refaults[WORKINGSET_ANON] = refaults; refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); - target_lruvec->refaults[1] = refaults; + target_lruvec->refaults[WORKINGSET_FILE] = refaults; } /* @@ -3923,12 +6727,16 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, } #endif -static void age_active_anon(struct pglist_data *pgdat, - struct scan_control *sc) +static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) { struct mem_cgroup *memcg; struct lruvec *lruvec; + if (lru_gen_enabled()) { + lru_gen_age_node(pgdat, sc); + return; + } + if (!can_age_anon_pages(pgdat, sc)) return; @@ -4248,12 +7056,11 @@ restart: sc.may_swap = !nr_boost_reclaim; /* - * Do some background aging of the anon list, to give - * pages a chance to be referenced before reclaiming. All - * pages are rotated regardless of classzone as this is - * about consistent aging. + * Do some background aging, to give pages a chance to be + * referenced before reclaiming. All pages are rotated + * regardless of classzone as this is about consistent aging. */ - age_active_anon(pgdat, &sc); + kswapd_age_node(pgdat, &sc); /* * If we're getting trouble reclaiming, start doing writepage @@ -4643,16 +7450,17 @@ void kswapd_run(int nid) { pg_data_t *pgdat = NODE_DATA(nid); - if (pgdat->kswapd) - return; - - pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); - if (IS_ERR(pgdat->kswapd)) { - /* failure at boot is fatal */ - BUG_ON(system_state < SYSTEM_RUNNING); - pr_err("Failed to start kswapd on node %d\n", nid); - pgdat->kswapd = NULL; + pgdat_kswapd_lock(pgdat); + if (!pgdat->kswapd) { + pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid); + if (IS_ERR(pgdat->kswapd)) { + /* failure at boot is fatal */ + BUG_ON(system_state < SYSTEM_RUNNING); + pr_err("Failed to start kswapd on node %d\n", nid); + pgdat->kswapd = NULL; + } } + pgdat_kswapd_unlock(pgdat); } /* @@ -4661,12 +7469,16 @@ void kswapd_run(int nid) */ void kswapd_stop(int nid) { - struct task_struct *kswapd = NODE_DATA(nid)->kswapd; + pg_data_t *pgdat = NODE_DATA(nid); + struct task_struct *kswapd; + pgdat_kswapd_lock(pgdat); + kswapd = pgdat->kswapd; if (kswapd) { kthread_stop(kswapd); - NODE_DATA(nid)->kswapd = NULL; + pgdat->kswapd = NULL; } + pgdat_kswapd_unlock(pgdat); } static int __init kswapd_init(void) diff --git a/mm/vmstat.c b/mm/vmstat.c index 373d2730fcf2157562f343ec4b3160e8f4f49eac..b2371d745e007f0ac5812c9727ffe636c66cb8c2 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -28,7 +28,6 @@ #include #include #include -#include #include "internal.h" @@ -355,8 +354,7 @@ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, * CPU migrations and preemption potentially corrupts a counter so * disable preemption. */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); x = delta + __this_cpu_read(*p); @@ -368,8 +366,7 @@ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, } __this_cpu_write(*p, x); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } EXPORT_SYMBOL(__mod_zone_page_state); @@ -393,8 +390,7 @@ void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, } /* See __mod_node_page_state */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); x = delta + __this_cpu_read(*p); @@ -406,8 +402,7 @@ void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item, } __this_cpu_write(*p, x); - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } EXPORT_SYMBOL(__mod_node_page_state); @@ -441,8 +436,7 @@ void __inc_zone_state(struct zone *zone, enum zone_stat_item item) s8 v, t; /* See __mod_node_page_state */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); v = __this_cpu_inc_return(*p); t = __this_cpu_read(pcp->stat_threshold); @@ -453,8 +447,7 @@ void __inc_zone_state(struct zone *zone, enum zone_stat_item item) __this_cpu_write(*p, -overstep); } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) @@ -466,8 +459,7 @@ void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); /* See __mod_node_page_state */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); v = __this_cpu_inc_return(*p); t = __this_cpu_read(pcp->stat_threshold); @@ -478,8 +470,7 @@ void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item) __this_cpu_write(*p, -overstep); } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } void __inc_zone_page_state(struct page *page, enum zone_stat_item item) @@ -501,8 +492,7 @@ void __dec_zone_state(struct zone *zone, enum zone_stat_item item) s8 v, t; /* See __mod_node_page_state */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); v = __this_cpu_dec_return(*p); t = __this_cpu_read(pcp->stat_threshold); @@ -513,8 +503,7 @@ void __dec_zone_state(struct zone *zone, enum zone_stat_item item) __this_cpu_write(*p, overstep); } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) @@ -526,8 +515,7 @@ void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); /* See __mod_node_page_state */ - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_disable(); + preempt_disable_nested(); v = __this_cpu_dec_return(*p); t = __this_cpu_read(pcp->stat_threshold); @@ -538,8 +526,7 @@ void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item) __this_cpu_write(*p, overstep); } - if (IS_ENABLED(CONFIG_PREEMPT_RT)) - preempt_enable(); + preempt_enable_nested(); } void __dec_zone_page_state(struct page *page, enum zone_stat_item item) @@ -1168,8 +1155,15 @@ int fragmentation_index(struct zone *zone, unsigned int order) #define TEXT_FOR_HIGHMEM(xx) #endif +#ifdef CONFIG_ZONE_DEVICE +#define TEXT_FOR_DEVICE(xx) xx "_device", +#else +#define TEXT_FOR_DEVICE(xx) +#endif + #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \ - TEXT_FOR_HIGHMEM(xx) xx "_movable", + TEXT_FOR_HIGHMEM(xx) xx "_movable", \ + TEXT_FOR_DEVICE(xx) const char * const vmstat_text[] = { /* enum zone_stat_item counters */ @@ -1240,11 +1234,13 @@ const char * const vmstat_text[] = { "nr_shadow_call_stack", #endif "nr_page_table_pages", + "nr_sec_page_table_pages", #ifdef CONFIG_SWAP "nr_swapcached", #endif #ifdef CONFIG_NUMA_BALANCING "pgpromote_success", + "pgpromote_candidate", #endif /* enum writeback_stat_item counters */ @@ -1382,10 +1378,6 @@ const char * const vmstat_text[] = { "nr_tlb_local_flush_one", #endif /* CONFIG_DEBUG_TLBFLUSH */ -#ifdef CONFIG_DEBUG_VM_VMACACHE - "vmacache_find_calls", - "vmacache_find_hits", -#endif #ifdef CONFIG_SWAP "swap_ra", "swap_ra_hit", @@ -2060,7 +2052,6 @@ static int vmstat_cpu_online(unsigned int cpu) if (!node_state(cpu_to_node(cpu), N_CPU)) { node_set_state(cpu_to_node(cpu), N_CPU); - set_migration_target_nodes(); } return 0; @@ -2085,7 +2076,6 @@ static int vmstat_cpu_dead(unsigned int cpu) return 0; node_clear_state(node, N_CPU); - set_migration_target_nodes(); return 0; } @@ -2118,7 +2108,6 @@ void __init init_mm_internals(void) start_shepherd_timer(); #endif - migrate_on_reclaim_init(); #ifdef CONFIG_PROC_FS proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op); proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op); diff --git a/mm/workingset.c b/mm/workingset.c index a5e84862fc86881e09ad611ebbca6e3c156ea234..ae7e984b23c6b08e99c25adaf1b8533e623e4c49 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -187,7 +187,6 @@ static unsigned int bucket_order __read_mostly; static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, bool workingset) { - eviction >>= bucket_order; eviction &= EVICTION_MASK; eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; eviction = (eviction << NODES_SHIFT) | pgdat->node_id; @@ -212,10 +211,107 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, *memcgidp = memcgid; *pgdat = NODE_DATA(nid); - *evictionp = entry << bucket_order; + *evictionp = entry; *workingsetp = workingset; } +#ifdef CONFIG_LRU_GEN + +static void *lru_gen_eviction(struct folio *folio) +{ + int hist; + unsigned long token; + unsigned long min_seq; + struct lruvec *lruvec; + struct lru_gen_struct *lrugen; + int type = folio_is_file_lru(folio); + int delta = folio_nr_pages(folio); + int refs = folio_lru_refs(folio); + int tier = lru_tier_from_refs(refs); + struct mem_cgroup *memcg = folio_memcg(folio); + struct pglist_data *pgdat = folio_pgdat(folio); + + BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH > BITS_PER_LONG - EVICTION_SHIFT); + + lruvec = mem_cgroup_lruvec(memcg, pgdat); + lrugen = &lruvec->lrugen; + min_seq = READ_ONCE(lrugen->min_seq[type]); + token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0); + + hist = lru_hist_from_seq(min_seq); + atomic_long_add(delta, &lrugen->evicted[hist][type][tier]); + + return pack_shadow(mem_cgroup_id(memcg), pgdat, token, refs); +} + +static void lru_gen_refault(struct folio *folio, void *shadow) +{ + int hist, tier, refs; + int memcg_id; + bool workingset; + unsigned long token; + unsigned long min_seq; + struct lruvec *lruvec; + struct lru_gen_struct *lrugen; + struct mem_cgroup *memcg; + struct pglist_data *pgdat; + int type = folio_is_file_lru(folio); + int delta = folio_nr_pages(folio); + + unpack_shadow(shadow, &memcg_id, &pgdat, &token, &workingset); + + if (pgdat != folio_pgdat(folio)) + return; + + rcu_read_lock(); + + memcg = folio_memcg_rcu(folio); + if (memcg_id != mem_cgroup_id(memcg)) + goto unlock; + + lruvec = mem_cgroup_lruvec(memcg, pgdat); + lrugen = &lruvec->lrugen; + + min_seq = READ_ONCE(lrugen->min_seq[type]); + if ((token >> LRU_REFS_WIDTH) != (min_seq & (EVICTION_MASK >> LRU_REFS_WIDTH))) + goto unlock; + + hist = lru_hist_from_seq(min_seq); + /* see the comment in folio_lru_refs() */ + refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + workingset; + tier = lru_tier_from_refs(refs); + + atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]); + mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); + + /* + * Count the following two cases as stalls: + * 1. For pages accessed through page tables, hotter pages pushed out + * hot pages which refaulted immediately. + * 2. For pages accessed multiple times through file descriptors, + * numbers of accesses might have been out of the range. + */ + if (lru_gen_in_fault() || refs == BIT(LRU_REFS_WIDTH)) { + folio_set_workingset(folio); + mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta); + } +unlock: + rcu_read_unlock(); +} + +#else /* !CONFIG_LRU_GEN */ + +static void *lru_gen_eviction(struct folio *folio) +{ + return NULL; +} + +static void lru_gen_refault(struct folio *folio, void *shadow) +{ +} + +#endif /* CONFIG_LRU_GEN */ + /** * workingset_age_nonresident - age non-resident entries as LRU ages * @lruvec: the lruvec that was aged @@ -264,10 +360,14 @@ void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg) VM_BUG_ON_FOLIO(folio_ref_count(folio), folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (lru_gen_enabled()) + return lru_gen_eviction(folio); + lruvec = mem_cgroup_lruvec(target_memcg, pgdat); /* XXX: target_memcg can be NULL, go through lruvec */ memcgid = mem_cgroup_id(lruvec_memcg(lruvec)); eviction = atomic_long_read(&lruvec->nonresident_age); + eviction >>= bucket_order; workingset_age_nonresident(lruvec, folio_nr_pages(folio)); return pack_shadow(memcgid, pgdat, eviction, folio_test_workingset(folio)); @@ -298,7 +398,13 @@ void workingset_refault(struct folio *folio, void *shadow) int memcgid; long nr; + if (lru_gen_enabled()) { + lru_gen_refault(folio, shadow); + return; + } + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset); + eviction <<= bucket_order; rcu_read_lock(); /* diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 34f784a1604b7a60153567767f715fa3371eb675..525758713a553d152084a0e3442e0f40fe132cf3 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -472,12 +472,12 @@ static inline struct page *get_first_page(struct zspage *zspage) return first_page; } -static inline int get_first_obj_offset(struct page *page) +static inline unsigned int get_first_obj_offset(struct page *page) { return page->page_type; } -static inline void set_first_obj_offset(struct page *page, int offset) +static inline void set_first_obj_offset(struct page *page, unsigned int offset) { page->page_type = offset; } @@ -1487,7 +1487,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle) struct size_class *class; enum fullness_group fullness; - if (unlikely(!handle)) + if (IS_ERR_OR_NULL((void *)handle)) return; /* @@ -1555,6 +1555,13 @@ static void zs_object_copy(struct size_class *class, unsigned long dst, d_off += size; d_size -= size; + /* + * Calling kunmap_atomic(d_addr) is necessary. kunmap_atomic() + * calls must occurs in reverse order of calls to kmap_atomic(). + * So, to call kunmap_atomic(s_addr) we should first call + * kunmap_atomic(d_addr). For more details see + * Documentation/mm/highmem.rst. + */ if (s_off >= PAGE_SIZE) { kunmap_atomic(d_addr); kunmap_atomic(s_addr); @@ -1585,7 +1592,7 @@ static void zs_object_copy(struct size_class *class, unsigned long dst, static unsigned long find_alloced_obj(struct size_class *class, struct page *page, int *obj_idx) { - int offset = 0; + unsigned int offset; int index = *obj_idx; unsigned long handle = 0; void *addr = kmap_atomic(page); @@ -1839,7 +1846,7 @@ static int zs_page_migrate(struct page *newpage, struct page *page, struct zspage *zspage; struct page *dummy; void *s_addr, *d_addr, *addr; - int offset; + unsigned int offset; unsigned long handle; unsigned long old_obj, new_obj; unsigned int obj_idx; @@ -2103,8 +2110,6 @@ unsigned long zs_compact(struct zs_pool *pool) for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { class = pool->size_class[i]; - if (!class) - continue; if (class->index != i) continue; pages_freed += __zs_compact(pool, class); @@ -2149,8 +2154,6 @@ static unsigned long zs_shrinker_count(struct shrinker *shrinker, for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) { class = pool->size_class[i]; - if (!class) - continue; if (class->index != i) continue; @@ -2308,9 +2311,6 @@ void zs_destroy_pool(struct zs_pool *pool) int fg; struct size_class *class = pool->size_class[i]; - if (!class) - continue; - if (class->index != i) continue; diff --git a/mm/zswap.c b/mm/zswap.c index 104835b379ec9376ca19a90c447cf4dad8ddb6c1..2d48fd59cc7ab67e0d22781012fa06bb37c54a93 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1026,7 +1026,7 @@ static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) SetPageReclaim(page); /* start writeback */ - __swap_writepage(page, &wbc, end_swap_bio_write); + __swap_writepage(page, &wbc); put_page(page); zswap_written_back_pages++; diff --git a/net/802/garp.c b/net/802/garp.c index f6012f8e59f00521e36a7e251ce3b452818a010e..fc9eb02a912f819459d6ba2922619bd85d8f6ba2 100644 --- a/net/802/garp.c +++ b/net/802/garp.c @@ -407,7 +407,7 @@ static void garp_join_timer_arm(struct garp_applicant *app) { unsigned long delay; - delay = (u64)msecs_to_jiffies(garp_join_time) * prandom_u32() >> 32; + delay = prandom_u32_max(msecs_to_jiffies(garp_join_time)); mod_timer(&app->join_timer, jiffies + delay); } diff --git a/net/802/mrp.c b/net/802/mrp.c index 35e04cc5390c49e54b3bef5e7157c9e3db2514b3..155f74d8b14f4b66fad5451a88a10be01604b57d 100644 --- a/net/802/mrp.c +++ b/net/802/mrp.c @@ -592,7 +592,7 @@ static void mrp_join_timer_arm(struct mrp_applicant *app) { unsigned long delay; - delay = (u64)msecs_to_jiffies(mrp_join_time) * prandom_u32() >> 32; + delay = prandom_u32_max(msecs_to_jiffies(mrp_join_time)); mod_timer(&app->join_timer, jiffies + delay); } diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index 5aa8144101dc6c82b6251ac9e2b25cc6eeda8fb4..0beb44f2fe1f0df45b8125cf4b7b9610fa7503e9 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c @@ -467,12 +467,9 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head, off_vlan = skb_gro_offset(skb); hlen = off_vlan + sizeof(*vhdr); - vhdr = skb_gro_header_fast(skb, off_vlan); - if (skb_gro_header_hard(skb, hlen)) { - vhdr = skb_gro_header_slow(skb, hlen, off_vlan); - if (unlikely(!vhdr)) - goto out; - } + vhdr = skb_gro_header(skb, hlen, off_vlan); + if (unlikely(!vhdr)) + goto out; type = vhdr->h_vlan_encapsulated_proto; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 035812b0461cc4b403f1a80bfdb9cfd9f44e4b45..e1bb41a443c433bb14a0731fa875e8a931abcdde 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -674,9 +674,9 @@ static int vlan_ethtool_get_link_ksettings(struct net_device *dev, static void vlan_ethtool_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, vlan_fullname, sizeof(info->driver)); - strlcpy(info->version, vlan_version, sizeof(info->version)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->driver, vlan_fullname, sizeof(info->driver)); + strscpy(info->version, vlan_version, sizeof(info->version)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); } static int vlan_ethtool_get_ts_info(struct net_device *dev, diff --git a/net/9p/client.c b/net/9p/client.c index 0a6110e15d0f85c6278eff862a0850351f7ac12a..aaa37b07e30a5c61e6191905dc466a7c0cc0c9df 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -255,24 +255,42 @@ static struct kmem_cache *p9_req_cache; * p9_tag_alloc - Allocate a new request. * @c: Client session. * @type: Transaction type. - * @max_size: Maximum packet size for this request. + * @t_size: Buffer size for holding this request + * (automatic calculation by format template if 0). + * @r_size: Buffer size for holding server's reply on this request + * (automatic calculation by format template if 0). + * @fmt: Format template for assembling 9p request message + * (see p9pdu_vwritef). + * @ap: Variable arguments to be fed to passed format template + * (see p9pdu_vwritef). * * Context: Process context. * Return: Pointer to new request. */ static struct p9_req_t * -p9_tag_alloc(struct p9_client *c, int8_t type, unsigned int max_size) +p9_tag_alloc(struct p9_client *c, int8_t type, uint t_size, uint r_size, + const char *fmt, va_list ap) { struct p9_req_t *req = kmem_cache_alloc(p9_req_cache, GFP_NOFS); - int alloc_msize = min(c->msize, max_size); + int alloc_tsize; + int alloc_rsize; int tag; + va_list apc; + + va_copy(apc, ap); + alloc_tsize = min_t(size_t, c->msize, + t_size ?: p9_msg_buf_size(c, type, fmt, apc)); + va_end(apc); + + alloc_rsize = min_t(size_t, c->msize, + r_size ?: p9_msg_buf_size(c, type + 1, fmt, ap)); if (!req) return ERR_PTR(-ENOMEM); - if (p9_fcall_init(c, &req->tc, alloc_msize)) + if (p9_fcall_init(c, &req->tc, alloc_tsize)) goto free_req; - if (p9_fcall_init(c, &req->rc, alloc_msize)) + if (p9_fcall_init(c, &req->rc, alloc_rsize)) goto free; p9pdu_reset(&req->tc); @@ -592,11 +610,12 @@ static int p9_client_flush(struct p9_client *c, struct p9_req_t *oldreq) } static struct p9_req_t *p9_client_prepare_req(struct p9_client *c, - int8_t type, int req_size, + int8_t type, uint t_size, uint r_size, const char *fmt, va_list ap) { int err; struct p9_req_t *req; + va_list apc; p9_debug(P9_DEBUG_MUX, "client %p op %d\n", c, type); @@ -608,7 +627,9 @@ static struct p9_req_t *p9_client_prepare_req(struct p9_client *c, if (c->status == BeginDisconnect && type != P9_TCLUNK) return ERR_PTR(-EIO); - req = p9_tag_alloc(c, type, req_size); + va_copy(apc, ap); + req = p9_tag_alloc(c, type, t_size, r_size, fmt, apc); + va_end(apc); if (IS_ERR(req)) return req; @@ -643,9 +664,18 @@ p9_client_rpc(struct p9_client *c, int8_t type, const char *fmt, ...) int sigpending, err; unsigned long flags; struct p9_req_t *req; + /* Passing zero for tsize/rsize to p9_client_prepare_req() tells it to + * auto determine an appropriate (small) request/response size + * according to actual message data being sent. Currently RDMA + * transport is excluded from this response message size optimization, + * as it would not cope with it, due to its pooled response buffers + * (using an optimized request size for RDMA as well though). + */ + const uint tsize = 0; + const uint rsize = c->trans_mod->pooled_rbuffers ? c->msize : 0; va_start(ap, fmt); - req = p9_client_prepare_req(c, type, c->msize, fmt, ap); + req = p9_client_prepare_req(c, type, tsize, rsize, fmt, ap); va_end(ap); if (IS_ERR(req)) return req; @@ -743,7 +773,7 @@ static struct p9_req_t *p9_client_zc_rpc(struct p9_client *c, int8_t type, /* We allocate a inline protocol data of only 4k bytes. * The actual content is passed in zero-copy fashion. */ - req = p9_client_prepare_req(c, type, P9_ZC_HDR_SZ, fmt, ap); + req = p9_client_prepare_req(c, type, P9_ZC_HDR_SZ, P9_ZC_HDR_SZ, fmt, ap); va_end(ap); if (IS_ERR(req)) return req; diff --git a/net/9p/protocol.c b/net/9p/protocol.c index 83694c6319890bb26893afc094b9645acf55ce66..4e3a2a1ffcb3faa34cad770ec76a7f15e37c5cc9 100644 --- a/net/9p/protocol.c +++ b/net/9p/protocol.c @@ -23,6 +23,173 @@ #include +/* len[2] text[len] */ +#define P9_STRLEN(s) \ + (2 + min_t(size_t, s ? strlen(s) : 0, USHRT_MAX)) + +/** + * p9_msg_buf_size - Returns a buffer size sufficiently large to hold the + * intended 9p message. + * @c: client + * @type: message type + * @fmt: format template for assembling request message + * (see p9pdu_vwritef) + * @ap: variable arguments to be fed to passed format template + * (see p9pdu_vwritef) + * + * Note: Even for response types (P9_R*) the format template and variable + * arguments must always be for the originating request type (P9_T*). + */ +size_t p9_msg_buf_size(struct p9_client *c, enum p9_msg_t type, + const char *fmt, va_list ap) +{ + /* size[4] type[1] tag[2] */ + const int hdr = 4 + 1 + 2; + /* ename[s] errno[4] */ + const int rerror_size = hdr + P9_ERRMAX + 4; + /* ecode[4] */ + const int rlerror_size = hdr + 4; + const int err_size = + c->proto_version == p9_proto_2000L ? rlerror_size : rerror_size; + + static_assert(NAME_MAX <= 4*1024, "p9_msg_buf_size() currently assumes " + "a max. allowed directory entry name length of 4k"); + + switch (type) { + + /* message types not used at all */ + case P9_TERROR: + case P9_TLERROR: + case P9_TAUTH: + case P9_RAUTH: + BUG(); + + /* variable length & potentially large message types */ + case P9_TATTACH: + BUG_ON(strcmp("ddss?u", fmt)); + va_arg(ap, int32_t); + va_arg(ap, int32_t); + { + const char *uname = va_arg(ap, const char *); + const char *aname = va_arg(ap, const char *); + /* fid[4] afid[4] uname[s] aname[s] n_uname[4] */ + return hdr + 4 + 4 + P9_STRLEN(uname) + P9_STRLEN(aname) + 4; + } + case P9_TWALK: + BUG_ON(strcmp("ddT", fmt)); + va_arg(ap, int32_t); + va_arg(ap, int32_t); + { + uint i, nwname = va_arg(ap, int); + size_t wname_all; + const char **wnames = va_arg(ap, const char **); + for (i = 0, wname_all = 0; i < nwname; ++i) { + wname_all += P9_STRLEN(wnames[i]); + } + /* fid[4] newfid[4] nwname[2] nwname*(wname[s]) */ + return hdr + 4 + 4 + 2 + wname_all; + } + case P9_RWALK: + BUG_ON(strcmp("ddT", fmt)); + va_arg(ap, int32_t); + va_arg(ap, int32_t); + { + uint nwname = va_arg(ap, int); + /* nwqid[2] nwqid*(wqid[13]) */ + return max_t(size_t, hdr + 2 + nwname * 13, err_size); + } + case P9_TCREATE: + BUG_ON(strcmp("dsdb?s", fmt)); + va_arg(ap, int32_t); + { + const char *name = va_arg(ap, const char *); + if (c->proto_version == p9_proto_legacy) { + /* fid[4] name[s] perm[4] mode[1] */ + return hdr + 4 + P9_STRLEN(name) + 4 + 1; + } else { + va_arg(ap, int32_t); + va_arg(ap, int); + { + const char *ext = va_arg(ap, const char *); + /* fid[4] name[s] perm[4] mode[1] extension[s] */ + return hdr + 4 + P9_STRLEN(name) + 4 + 1 + P9_STRLEN(ext); + } + } + } + case P9_TLCREATE: + BUG_ON(strcmp("dsddg", fmt)); + va_arg(ap, int32_t); + { + const char *name = va_arg(ap, const char *); + /* fid[4] name[s] flags[4] mode[4] gid[4] */ + return hdr + 4 + P9_STRLEN(name) + 4 + 4 + 4; + } + case P9_RREAD: + case P9_RREADDIR: + BUG_ON(strcmp("dqd", fmt)); + va_arg(ap, int32_t); + va_arg(ap, int64_t); + { + const int32_t count = va_arg(ap, int32_t); + /* count[4] data[count] */ + return max_t(size_t, hdr + 4 + count, err_size); + } + case P9_TWRITE: + BUG_ON(strcmp("dqV", fmt)); + va_arg(ap, int32_t); + va_arg(ap, int64_t); + { + const int32_t count = va_arg(ap, int32_t); + /* fid[4] offset[8] count[4] data[count] */ + return hdr + 4 + 8 + 4 + count; + } + case P9_TRENAMEAT: + BUG_ON(strcmp("dsds", fmt)); + va_arg(ap, int32_t); + { + const char *oldname, *newname; + oldname = va_arg(ap, const char *); + va_arg(ap, int32_t); + newname = va_arg(ap, const char *); + /* olddirfid[4] oldname[s] newdirfid[4] newname[s] */ + return hdr + 4 + P9_STRLEN(oldname) + 4 + P9_STRLEN(newname); + } + case P9_TSYMLINK: + BUG_ON(strcmp("dssg", fmt)); + va_arg(ap, int32_t); + { + const char *name = va_arg(ap, const char *); + const char *symtgt = va_arg(ap, const char *); + /* fid[4] name[s] symtgt[s] gid[4] */ + return hdr + 4 + P9_STRLEN(name) + P9_STRLEN(symtgt) + 4; + } + + case P9_RERROR: + return rerror_size; + case P9_RLERROR: + return rlerror_size; + + /* small message types */ + case P9_TWSTAT: + case P9_RSTAT: + case P9_RREADLINK: + case P9_TXATTRWALK: + case P9_TXATTRCREATE: + case P9_TLINK: + case P9_TMKDIR: + case P9_TMKNOD: + case P9_TRENAME: + case P9_TUNLINKAT: + case P9_TLOCK: + return 8 * 1024; + + /* tiny message types */ + default: + return 4 * 1024; + + } +} + static int p9pdu_writef(struct p9_fcall *pdu, int proto_version, const char *fmt, ...); diff --git a/net/9p/protocol.h b/net/9p/protocol.h index 6d719c30331ac9133bd4df7af4c1f00d2012e369..ad2283d1f96be85f92e519dd7d584fb777371d4a 100644 --- a/net/9p/protocol.h +++ b/net/9p/protocol.h @@ -8,6 +8,8 @@ * Copyright (C) 2008 by IBM, Corp. */ +size_t p9_msg_buf_size(struct p9_client *c, enum p9_msg_t type, + const char *fmt, va_list ap); int p9pdu_vwritef(struct p9_fcall *pdu, int proto_version, const char *fmt, va_list ap); int p9pdu_readf(struct p9_fcall *pdu, int proto_version, const char *fmt, ...); diff --git a/net/9p/trans_fd.c b/net/9p/trans_fd.c index e758978b44bee9a10963eb60cff042d7b129636d..56a1867687501a21d85dfd84105ef8f76b9c4d1d 100644 --- a/net/9p/trans_fd.c +++ b/net/9p/trans_fd.c @@ -91,6 +91,7 @@ struct p9_poll_wait { * @mux_list: list link for mux to manage multiple connections (?) * @client: reference to client instance for this connection * @err: error state + * @req_lock: lock protecting req_list and requests statuses * @req_list: accounting for requests which have been sent * @unsent_req_list: accounting for requests that haven't been sent * @rreq: read request @@ -114,6 +115,7 @@ struct p9_conn { struct list_head mux_list; struct p9_client *client; int err; + spinlock_t req_lock; struct list_head req_list; struct list_head unsent_req_list; struct p9_req_t *rreq; @@ -189,10 +191,10 @@ static void p9_conn_cancel(struct p9_conn *m, int err) p9_debug(P9_DEBUG_ERROR, "mux %p err %d\n", m, err); - spin_lock(&m->client->lock); + spin_lock(&m->req_lock); if (m->err) { - spin_unlock(&m->client->lock); + spin_unlock(&m->req_lock); return; } @@ -205,6 +207,8 @@ static void p9_conn_cancel(struct p9_conn *m, int err) list_move(&req->req_list, &cancel_list); } + spin_unlock(&m->req_lock); + list_for_each_entry_safe(req, rtmp, &cancel_list, req_list) { p9_debug(P9_DEBUG_ERROR, "call back req %p\n", req); list_del(&req->req_list); @@ -212,7 +216,6 @@ static void p9_conn_cancel(struct p9_conn *m, int err) req->t_err = err; p9_client_cb(m->client, req, REQ_STATUS_ERROR); } - spin_unlock(&m->client->lock); } static __poll_t @@ -359,7 +362,7 @@ static void p9_read_work(struct work_struct *work) if ((m->rreq) && (m->rc.offset == m->rc.capacity)) { p9_debug(P9_DEBUG_TRANS, "got new packet\n"); m->rreq->rc.size = m->rc.offset; - spin_lock(&m->client->lock); + spin_lock(&m->req_lock); if (m->rreq->status == REQ_STATUS_SENT) { list_del(&m->rreq->req_list); p9_client_cb(m->client, m->rreq, REQ_STATUS_RCVD); @@ -368,14 +371,14 @@ static void p9_read_work(struct work_struct *work) p9_debug(P9_DEBUG_TRANS, "Ignore replies associated with a cancelled request\n"); } else { - spin_unlock(&m->client->lock); + spin_unlock(&m->req_lock); p9_debug(P9_DEBUG_ERROR, "Request tag %d errored out while we were reading the reply\n", m->rc.tag); err = -EIO; goto error; } - spin_unlock(&m->client->lock); + spin_unlock(&m->req_lock); m->rc.sdata = NULL; m->rc.offset = 0; m->rc.capacity = 0; @@ -453,10 +456,10 @@ static void p9_write_work(struct work_struct *work) } if (!m->wsize) { - spin_lock(&m->client->lock); + spin_lock(&m->req_lock); if (list_empty(&m->unsent_req_list)) { clear_bit(Wworksched, &m->wsched); - spin_unlock(&m->client->lock); + spin_unlock(&m->req_lock); return; } @@ -471,7 +474,7 @@ static void p9_write_work(struct work_struct *work) m->wpos = 0; p9_req_get(req); m->wreq = req; - spin_unlock(&m->client->lock); + spin_unlock(&m->req_lock); } p9_debug(P9_DEBUG_TRANS, "mux %p pos %d size %d\n", @@ -588,6 +591,7 @@ static void p9_conn_create(struct p9_client *client) INIT_LIST_HEAD(&m->mux_list); m->client = client; + spin_lock_init(&m->req_lock); INIT_LIST_HEAD(&m->req_list); INIT_LIST_HEAD(&m->unsent_req_list); INIT_WORK(&m->rq, p9_read_work); @@ -669,10 +673,10 @@ static int p9_fd_request(struct p9_client *client, struct p9_req_t *req) if (m->err < 0) return m->err; - spin_lock(&client->lock); + spin_lock(&m->req_lock); req->status = REQ_STATUS_UNSENT; list_add_tail(&req->req_list, &m->unsent_req_list); - spin_unlock(&client->lock); + spin_unlock(&m->req_lock); if (test_and_clear_bit(Wpending, &m->wsched)) n = EPOLLOUT; @@ -687,11 +691,13 @@ static int p9_fd_request(struct p9_client *client, struct p9_req_t *req) static int p9_fd_cancel(struct p9_client *client, struct p9_req_t *req) { + struct p9_trans_fd *ts = client->trans; + struct p9_conn *m = &ts->conn; int ret = 1; p9_debug(P9_DEBUG_TRANS, "client %p req %p\n", client, req); - spin_lock(&client->lock); + spin_lock(&m->req_lock); if (req->status == REQ_STATUS_UNSENT) { list_del(&req->req_list); @@ -699,21 +705,24 @@ static int p9_fd_cancel(struct p9_client *client, struct p9_req_t *req) p9_req_put(client, req); ret = 0; } - spin_unlock(&client->lock); + spin_unlock(&m->req_lock); return ret; } static int p9_fd_cancelled(struct p9_client *client, struct p9_req_t *req) { + struct p9_trans_fd *ts = client->trans; + struct p9_conn *m = &ts->conn; + p9_debug(P9_DEBUG_TRANS, "client %p req %p\n", client, req); - spin_lock(&client->lock); + spin_lock(&m->req_lock); /* Ignore cancelled request if message has been received * before lock. */ if (req->status == REQ_STATUS_RCVD) { - spin_unlock(&client->lock); + spin_unlock(&m->req_lock); return 0; } @@ -722,7 +731,8 @@ static int p9_fd_cancelled(struct p9_client *client, struct p9_req_t *req) */ list_del(&req->req_list); req->status = REQ_STATUS_FLSHD; - spin_unlock(&client->lock); + spin_unlock(&m->req_lock); + p9_req_put(client, req); return 0; @@ -821,11 +831,14 @@ static int p9_fd_open(struct p9_client *client, int rfd, int wfd) goto out_free_ts; if (!(ts->rd->f_mode & FMODE_READ)) goto out_put_rd; + /* prevent workers from hanging on IO when fd is a pipe */ + ts->rd->f_flags |= O_NONBLOCK; ts->wr = fget(wfd); if (!ts->wr) goto out_put_rd; if (!(ts->wr->f_mode & FMODE_WRITE)) goto out_put_wr; + ts->wr->f_flags |= O_NONBLOCK; client->trans = ts; client->status = Connected; @@ -1061,7 +1074,9 @@ p9_fd_create(struct p9_client *client, const char *addr, char *args) int err; struct p9_fd_opts opts; - parse_opts(args, &opts); + err = parse_opts(args, &opts); + if (err < 0) + return err; client->trans_opts.fd.rfd = opts.rfd; client->trans_opts.fd.wfd = opts.wfd; @@ -1082,6 +1097,7 @@ p9_fd_create(struct p9_client *client, const char *addr, char *args) static struct p9_trans_module p9_tcp_trans = { .name = "tcp", .maxsize = MAX_SOCK_BUF, + .pooled_rbuffers = false, .def = 0, .create = p9_fd_create_tcp, .close = p9_fd_close, diff --git a/net/9p/trans_rdma.c b/net/9p/trans_rdma.c index d817d3745238b4c9a22f60125799253015b42dd6..6ff706760676e02995514199f5a575840c85831a 100644 --- a/net/9p/trans_rdma.c +++ b/net/9p/trans_rdma.c @@ -739,6 +739,7 @@ error: static struct p9_trans_module p9_rdma_trans = { .name = "rdma", .maxsize = P9_RDMA_MAXSIZE, + .pooled_rbuffers = true, .def = 0, .owner = THIS_MODULE, .create = rdma_create_trans, diff --git a/net/9p/trans_virtio.c b/net/9p/trans_virtio.c index b84d35cf68994631483f2be687ddfbe88739bd0d..e757f0601304361d31d9e0b00166278fb2c0e3ae 100644 --- a/net/9p/trans_virtio.c +++ b/net/9p/trans_virtio.c @@ -802,6 +802,7 @@ static struct p9_trans_module p9_virtio_trans = { * page in zero copy. */ .maxsize = PAGE_SIZE * (VIRTQUEUE_NUM - 3), + .pooled_rbuffers = false, .def = 1, .owner = THIS_MODULE, }; diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index 227f89cc7237ca9105d3fa4842aebf756ab9dacf..b15c64128c3e58e7c93bcfe94a3ffa28ffad2611 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -246,6 +246,7 @@ static irqreturn_t xen_9pfs_front_event_handler(int irq, void *r) static struct p9_trans_module p9_xen_trans = { .name = "xen", .maxsize = 1 << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT - 2), + .pooled_rbuffers = false, .def = 1, .create = p9_xen_create, .close = p9_xen_close, @@ -510,7 +511,7 @@ static struct xenbus_driver xen_9pfs_front_driver = { .otherend_changed = xen_9pfs_front_changed, }; -static int p9_trans_xen_init(void) +static int __init p9_trans_xen_init(void) { int rc; @@ -529,7 +530,7 @@ static int p9_trans_xen_init(void) module_init(p9_trans_xen_init); MODULE_ALIAS_9P("xen"); -static void p9_trans_xen_exit(void) +static void __exit p9_trans_xen_exit(void) { v9fs_unregister_trans(&p9_xen_trans); return xenbus_unregister_driver(&xen_9pfs_front_driver); diff --git a/net/Kconfig b/net/Kconfig index 6b78f695caa6f57ea28f90143d342d4eb9503b06..48c33c2221999e575c83a409ab773b9cc3656eab 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -204,7 +204,6 @@ config BRIDGE_NETFILTER source "net/netfilter/Kconfig" source "net/ipv4/netfilter/Kconfig" source "net/ipv6/netfilter/Kconfig" -source "net/decnet/netfilter/Kconfig" source "net/bridge/netfilter/Kconfig" endif @@ -221,7 +220,6 @@ source "net/802/Kconfig" source "net/bridge/Kconfig" source "net/dsa/Kconfig" source "net/8021q/Kconfig" -source "net/decnet/Kconfig" source "net/llc/Kconfig" source "drivers/net/appletalk/Kconfig" source "net/x25/Kconfig" diff --git a/net/Kconfig.debug b/net/Kconfig.debug index e6ae11cc2fb7cb90e6a0c063d2135ee1839c3ead..5e3fffe707ddcef45acb9355ee12cd9d58b385bf 100644 --- a/net/Kconfig.debug +++ b/net/Kconfig.debug @@ -2,7 +2,7 @@ config NET_DEV_REFCNT_TRACKER bool "Enable net device refcount tracking" - depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT && NET select REF_TRACKER default n help @@ -11,7 +11,7 @@ config NET_DEV_REFCNT_TRACKER config NET_NS_REFCNT_TRACKER bool "Enable networking namespace refcount tracking" - depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT && NET select REF_TRACKER default n help diff --git a/net/Makefile b/net/Makefile index fbfeb8a0bb379594cfd9d4e4a3bfd314c997c39d..6a62e5b273781b1d5f02bbcf67b1d0ee911220c3 100644 --- a/net/Makefile +++ b/net/Makefile @@ -38,7 +38,6 @@ obj-$(CONFIG_AF_KCM) += kcm/ obj-$(CONFIG_STREAM_PARSER) += strparser/ obj-$(CONFIG_ATM) += atm/ obj-$(CONFIG_L2TP) += l2tp/ -obj-$(CONFIG_DECNET) += decnet/ obj-$(CONFIG_PHONET) += phonet/ ifneq ($(CONFIG_VLAN_8021Q),) obj-y += 8021q/ diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c index d82a51e69386b207130d9adfa2c0c8e56fd7e878..6b4c25a9237746265158900ab92d7f411b77ab79 100644 --- a/net/ax25/af_ax25.c +++ b/net/ax25/af_ax25.c @@ -778,7 +778,7 @@ static int ax25_getsockopt(struct socket *sock, int level, int optname, ax25_dev = ax25->ax25_dev; if (ax25_dev != NULL && ax25_dev->dev != NULL) { - strlcpy(devname, ax25_dev->dev->name, sizeof(devname)); + strscpy(devname, ax25_dev->dev->name, sizeof(devname)); length = strlen(devname) + 1; } else { *devname = '\0'; diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c index b6db999abf75e3930102a00c627a2ea8588bf891..f1741fbfb6178d573f2acc1885928df2851ccf0e 100644 --- a/net/batman-adv/bat_v_elp.c +++ b/net/batman-adv/bat_v_elp.c @@ -125,7 +125,6 @@ static u32 batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh) /* if not a wifi interface, check if this device provides data via * ethtool (e.g. an Ethernet adapter) */ - memset(&link_settings, 0, sizeof(link_settings)); rtnl_lock(); ret = __ethtool_get_link_ksettings(hard_iface->net_dev, &link_settings); rtnl_unlock(); diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index b8f8da7ee3dea4ba09f958e476c6b6f77b05e23a..41c1ad33d009fc7de81cd9f330cbc7d6c7f7f05a 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -700,6 +701,9 @@ int batadv_hardif_enable_interface(struct batadv_hard_iface *hard_iface, int max_header_len = batadv_max_header_len(); int ret; + if (hard_iface->net_dev->mtu < ETH_MIN_MTU + max_header_len) + return -EINVAL; + if (hard_iface->if_status != BATADV_IF_NOT_IN_USE) goto out; diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h index 23f3d53f4b51b17ecdebc403d1b913c0f394ac04..c48803b32bb018b9471f65c97a2cca657e2ce611 100644 --- a/net/batman-adv/main.h +++ b/net/batman-adv/main.h @@ -13,7 +13,7 @@ #define BATADV_DRIVER_DEVICE "batman-adv" #ifndef BATADV_SOURCE_VERSION -#define BATADV_SOURCE_VERSION "2022.2" +#define BATADV_SOURCE_VERSION "2022.3" #endif /* B.A.T.M.A.N. parameters */ diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c index 00875e1d8c44cde121c936a16965fb171ec6f3f1..a5e4a4e976cf3c8dc042ede58e812bbae5d5401f 100644 --- a/net/batman-adv/netlink.c +++ b/net/batman-adv/netlink.c @@ -1493,6 +1493,7 @@ struct genl_family batadv_netlink_family __ro_after_init = { .module = THIS_MODULE, .small_ops = batadv_netlink_ops, .n_small_ops = ARRAY_SIZE(batadv_netlink_ops), + .resv_start_op = BATADV_CMD_SET_VLAN + 1, .mcgrps = batadv_netlink_mcgrps, .n_mcgrps = ARRAY_SIZE(batadv_netlink_mcgrps), }; diff --git a/net/batman-adv/trace.h b/net/batman-adv/trace.h index 31c8f922651d587538508375fa413c9e6b416e56..5dd52bc5cabb92b46a50f2bc5283d787cc62db2c 100644 --- a/net/batman-adv/trace.h +++ b/net/batman-adv/trace.h @@ -9,8 +9,6 @@ #include "main.h" -#include -#include #include #include #include diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index 2be5d4a712c5c2ddc4a04d7296a6d4e4b57a7478..758cd797a063b565ef1d2512793020db90374624 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -1740,45 +1740,6 @@ struct batadv_priv { #endif }; -/** - * struct batadv_socket_client - layer2 icmp socket client data - */ -struct batadv_socket_client { - /** - * @queue_list: packet queue for packets destined for this socket client - */ - struct list_head queue_list; - - /** @queue_len: number of packets in the packet queue (queue_list) */ - unsigned int queue_len; - - /** @index: socket client's index in the batadv_socket_client_hash */ - unsigned char index; - - /** @lock: lock protecting queue_list, queue_len & index */ - spinlock_t lock; - - /** @queue_wait: socket client's wait queue */ - wait_queue_head_t queue_wait; - - /** @bat_priv: pointer to soft_iface this client belongs to */ - struct batadv_priv *bat_priv; -}; - -/** - * struct batadv_socket_packet - layer2 icmp packet for socket client - */ -struct batadv_socket_packet { - /** @list: list node for &batadv_socket_client.queue_list */ - struct list_head list; - - /** @icmp_len: size of the layer2 icmp packet */ - size_t icmp_len; - - /** @icmp_packet: layer2 icmp packet */ - u8 icmp_packet[BATADV_ICMP_MAX_PACKET_SIZE]; -}; - #ifdef CONFIG_BATMAN_ADV_BLA /** diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 9777e7b109eee47dfeab79b3dbfa0b91703e8d77..7a59c44870503ee12d37f64b8428b5c5cc7c4a69 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -44,6 +44,11 @@ struct sco_param { u8 retrans_effort; }; +struct conn_handle_t { + struct hci_conn *conn; + __u16 handle; +}; + static const struct sco_param esco_param_cvsd[] = { { EDR_ESCO_MASK & ~ESCO_2EV3, 0x000a, 0x01 }, /* S3 */ { EDR_ESCO_MASK & ~ESCO_2EV3, 0x0007, 0x01 }, /* S2 */ @@ -316,17 +321,60 @@ static bool find_next_esco_param(struct hci_conn *conn, return conn->attempt <= size; } -static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle) +static int configure_datapath_sync(struct hci_dev *hdev, struct bt_codec *codec) { - struct hci_dev *hdev = conn->hdev; + int err; + __u8 vnd_len, *vnd_data = NULL; + struct hci_op_configure_data_path *cmd = NULL; + + err = hdev->get_codec_config_data(hdev, ESCO_LINK, codec, &vnd_len, + &vnd_data); + if (err < 0) + goto error; + + cmd = kzalloc(sizeof(*cmd) + vnd_len, GFP_KERNEL); + if (!cmd) { + err = -ENOMEM; + goto error; + } + + err = hdev->get_data_path_id(hdev, &cmd->data_path_id); + if (err < 0) + goto error; + + cmd->vnd_len = vnd_len; + memcpy(cmd->vnd_data, vnd_data, vnd_len); + + cmd->direction = 0x00; + __hci_cmd_sync_status(hdev, HCI_CONFIGURE_DATA_PATH, + sizeof(*cmd) + vnd_len, cmd, HCI_CMD_TIMEOUT); + + cmd->direction = 0x01; + err = __hci_cmd_sync_status(hdev, HCI_CONFIGURE_DATA_PATH, + sizeof(*cmd) + vnd_len, cmd, + HCI_CMD_TIMEOUT); +error: + + kfree(cmd); + kfree(vnd_data); + return err; +} + +static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) +{ + struct conn_handle_t *conn_handle = data; + struct hci_conn *conn = conn_handle->conn; + __u16 handle = conn_handle->handle; struct hci_cp_enhanced_setup_sync_conn cp; const struct sco_param *param; + kfree(conn_handle); + bt_dev_dbg(hdev, "hcon %p", conn); /* for offload use case, codec needs to configured before opening SCO */ if (conn->codec.data_path) - hci_req_configure_datapath(hdev, &conn->codec); + configure_datapath_sync(hdev, &conn->codec); conn->state = BT_CONNECT; conn->out = true; @@ -344,7 +392,7 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle) case BT_CODEC_MSBC: if (!find_next_esco_param(conn, esco_param_msbc, ARRAY_SIZE(esco_param_msbc))) - return false; + return -EINVAL; param = &esco_param_msbc[conn->attempt - 1]; cp.tx_coding_format.id = 0x05; @@ -396,11 +444,11 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle) if (lmp_esco_capable(conn->link)) { if (!find_next_esco_param(conn, esco_param_cvsd, ARRAY_SIZE(esco_param_cvsd))) - return false; + return -EINVAL; param = &esco_param_cvsd[conn->attempt - 1]; } else { if (conn->attempt > ARRAY_SIZE(sco_param_cvsd)) - return false; + return -EINVAL; param = &sco_param_cvsd[conn->attempt - 1]; } cp.tx_coding_format.id = 2; @@ -423,7 +471,7 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle) cp.out_transport_unit_size = 16; break; default: - return false; + return -EINVAL; } cp.retrans_effort = param->retrans_effort; @@ -431,9 +479,9 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle) cp.max_latency = __cpu_to_le16(param->max_latency); if (hci_send_cmd(hdev, HCI_OP_ENHANCED_SETUP_SYNC_CONN, sizeof(cp), &cp) < 0) - return false; + return -EIO; - return true; + return 0; } static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) @@ -490,8 +538,24 @@ static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) bool hci_setup_sync(struct hci_conn *conn, __u16 handle) { - if (enhanced_sync_conn_capable(conn->hdev)) - return hci_enhanced_setup_sync_conn(conn, handle); + int result; + struct conn_handle_t *conn_handle; + + if (enhanced_sync_conn_capable(conn->hdev)) { + conn_handle = kzalloc(sizeof(*conn_handle), GFP_KERNEL); + + if (!conn_handle) + return false; + + conn_handle->conn = conn; + conn_handle->handle = handle; + result = hci_cmd_sync_queue(conn->hdev, hci_enhanced_setup_sync, + conn_handle, NULL); + if (result < 0) + kfree(conn_handle); + + return result == 0; + } return hci_setup_sync_conn(conn, handle); } @@ -2696,3 +2760,79 @@ u32 hci_conn_get_phy(struct hci_conn *conn) return phys; } + +int hci_abort_conn(struct hci_conn *conn, u8 reason) +{ + int r = 0; + + switch (conn->state) { + case BT_CONNECTED: + case BT_CONFIG: + if (conn->type == AMP_LINK) { + struct hci_cp_disconn_phy_link cp; + + cp.phy_handle = HCI_PHY_HANDLE(conn->handle); + cp.reason = reason; + r = hci_send_cmd(conn->hdev, HCI_OP_DISCONN_PHY_LINK, + sizeof(cp), &cp); + } else { + struct hci_cp_disconnect dc; + + dc.handle = cpu_to_le16(conn->handle); + dc.reason = reason; + r = hci_send_cmd(conn->hdev, HCI_OP_DISCONNECT, + sizeof(dc), &dc); + } + + conn->state = BT_DISCONN; + + break; + case BT_CONNECT: + if (conn->type == LE_LINK) { + if (test_bit(HCI_CONN_SCANNING, &conn->flags)) + break; + r = hci_send_cmd(conn->hdev, + HCI_OP_LE_CREATE_CONN_CANCEL, 0, NULL); + } else if (conn->type == ACL_LINK) { + if (conn->hdev->hci_ver < BLUETOOTH_VER_1_2) + break; + r = hci_send_cmd(conn->hdev, + HCI_OP_CREATE_CONN_CANCEL, + 6, &conn->dst); + } + break; + case BT_CONNECT2: + if (conn->type == ACL_LINK) { + struct hci_cp_reject_conn_req rej; + + bacpy(&rej.bdaddr, &conn->dst); + rej.reason = reason; + + r = hci_send_cmd(conn->hdev, + HCI_OP_REJECT_CONN_REQ, + sizeof(rej), &rej); + } else if (conn->type == SCO_LINK || conn->type == ESCO_LINK) { + struct hci_cp_reject_sync_conn_req rej; + + bacpy(&rej.bdaddr, &conn->dst); + + /* SCO rejection has its own limited set of + * allowed error values (0x0D-0x0F) which isn't + * compatible with most values passed to this + * function. To be safe hard-code one of the + * values that's suitable for SCO. + */ + rej.reason = HCI_ERROR_REJ_LIMITED_RESOURCES; + + r = hci_send_cmd(conn->hdev, + HCI_OP_REJECT_SYNC_CONN_REQ, + sizeof(rej), &rej); + } + break; + default: + conn->state = BT_CLOSED; + break; + } + + return r; +} diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index b3a5a3cc937202bf94d4d4c1619d7b07e4437d45..0540555b370481f6399227161897c02bc0af650e 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -597,6 +597,15 @@ static int hci_dev_do_reset(struct hci_dev *hdev) /* Cancel these to avoid queueing non-chained pending work */ hci_dev_set_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE); + /* Wait for + * + * if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) + * queue_delayed_work(&hdev->{cmd,ncmd}_timer) + * + * inside RCU section to see the flag or complete scheduling. + */ + synchronize_rcu(); + /* Explicitly cancel works in case scheduled after setting the flag. */ cancel_delayed_work(&hdev->cmd_timer); cancel_delayed_work(&hdev->ncmd_timer); @@ -714,7 +723,7 @@ static void hci_update_passive_scan_state(struct hci_dev *hdev, u8 scan) hci_dev_set_flag(hdev, HCI_BREDR_ENABLED); if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) - hci_req_update_adv_data(hdev, hdev->cur_adv_instance); + hci_update_adv_data(hdev, hdev->cur_adv_instance); mgmt_new_settings(hdev); } @@ -1706,7 +1715,8 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, u32 flags, u16 adv_data_len, u8 *adv_data, u16 scan_rsp_len, u8 *scan_rsp_data, u16 timeout, u16 duration, s8 tx_power, - u32 min_interval, u32 max_interval) + u32 min_interval, u32 max_interval, + u8 mesh_handle) { struct adv_info *adv; @@ -1717,7 +1727,7 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, memset(adv->per_adv_data, 0, sizeof(adv->per_adv_data)); } else { if (hdev->adv_instance_cnt >= hdev->le_num_of_adv_sets || - instance < 1 || instance > hdev->le_num_of_adv_sets) + instance < 1 || instance > hdev->le_num_of_adv_sets + 1) return ERR_PTR(-EOVERFLOW); adv = kzalloc(sizeof(*adv), GFP_KERNEL); @@ -1734,6 +1744,11 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance, adv->min_interval = min_interval; adv->max_interval = max_interval; adv->tx_power = tx_power; + /* Defining a mesh_handle changes the timing units to ms, + * rather than seconds, and ties the instance to the requested + * mesh_tx queue. + */ + adv->mesh = mesh_handle; hci_set_adv_instance_data(hdev, instance, adv_data_len, adv_data, scan_rsp_len, scan_rsp_data); @@ -1762,7 +1777,7 @@ struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance, adv = hci_add_adv_instance(hdev, instance, flags, 0, NULL, 0, NULL, 0, 0, HCI_ADV_TX_POWER_NO_PREFERENCE, - min_interval, max_interval); + min_interval, max_interval, 0); if (IS_ERR(adv)) return adv; @@ -2391,6 +2406,10 @@ static int hci_suspend_notifier(struct notifier_block *nb, unsigned long action, container_of(nb, struct hci_dev, suspend_notifier); int ret = 0; + /* Userspace has full control of this device. Do nothing. */ + if (hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) + return NOTIFY_DONE; + if (action == PM_SUSPEND_PREPARE) ret = hci_suspend_dev(hdev); else if (action == PM_POST_SUSPEND) @@ -2486,6 +2505,7 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv) mutex_init(&hdev->lock); mutex_init(&hdev->req_lock); + INIT_LIST_HEAD(&hdev->mesh_pending); INIT_LIST_HEAD(&hdev->mgmt_pending); INIT_LIST_HEAD(&hdev->reject_list); INIT_LIST_HEAD(&hdev->accept_list); @@ -3469,15 +3489,27 @@ static inline int __get_blocks(struct hci_dev *hdev, struct sk_buff *skb) return DIV_ROUND_UP(skb->len - HCI_ACL_HDR_SIZE, hdev->block_len); } -static void __check_timeout(struct hci_dev *hdev, unsigned int cnt) +static void __check_timeout(struct hci_dev *hdev, unsigned int cnt, u8 type) { - if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) { - /* ACL tx timeout must be longer than maximum - * link supervision timeout (40.9 seconds) */ - if (!cnt && time_after(jiffies, hdev->acl_last_tx + - HCI_ACL_TX_TIMEOUT)) - hci_link_tx_to(hdev, ACL_LINK); + unsigned long last_tx; + + if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) + return; + + switch (type) { + case LE_LINK: + last_tx = hdev->le_last_tx; + break; + default: + last_tx = hdev->acl_last_tx; + break; } + + /* tx timeout must be longer than maximum link supervision timeout + * (40.9 seconds) + */ + if (!cnt && time_after(jiffies, last_tx + HCI_ACL_TX_TIMEOUT)) + hci_link_tx_to(hdev, type); } /* Schedule SCO */ @@ -3535,7 +3567,7 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev) struct sk_buff *skb; int quote; - __check_timeout(hdev, cnt); + __check_timeout(hdev, cnt, ACL_LINK); while (hdev->acl_cnt && (chan = hci_chan_sent(hdev, ACL_LINK, "e))) { @@ -3578,8 +3610,6 @@ static void hci_sched_acl_blk(struct hci_dev *hdev) int quote; u8 type; - __check_timeout(hdev, cnt); - BT_DBG("%s", hdev->name); if (hdev->dev_type == HCI_AMP) @@ -3587,6 +3617,8 @@ static void hci_sched_acl_blk(struct hci_dev *hdev) else type = ACL_LINK; + __check_timeout(hdev, cnt, type); + while (hdev->block_cnt > 0 && (chan = hci_chan_sent(hdev, type, "e))) { u32 priority = (skb_peek(&chan->data_q))->priority; @@ -3660,7 +3692,7 @@ static void hci_sched_le(struct hci_dev *hdev) cnt = hdev->le_pkts ? hdev->le_cnt : hdev->acl_cnt; - __check_timeout(hdev, cnt); + __check_timeout(hdev, cnt, LE_LINK); tmp = cnt; while (cnt && (chan = hci_chan_sent(hdev, LE_LINK, "e))) { @@ -4056,12 +4088,14 @@ static void hci_cmd_work(struct work_struct *work) if (res < 0) __hci_cmd_sync_cancel(hdev, -res); + rcu_read_lock(); if (test_bit(HCI_RESET, &hdev->flags) || hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) cancel_delayed_work(&hdev->cmd_timer); else - schedule_delayed_work(&hdev->cmd_timer, - HCI_CMD_TIMEOUT); + queue_delayed_work(hdev->workqueue, &hdev->cmd_timer, + HCI_CMD_TIMEOUT); + rcu_read_unlock(); } else { skb_queue_head(&hdev->cmd_q, skb); queue_work(hdev->workqueue, &hdev->cmd_work); diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c index 902b40a90b912fe2522ed180943f78e2a7cfc05c..3f401ec5bb0c66e4660eb2163b783bcec69ba9d9 100644 --- a/net/bluetooth/hci_debugfs.c +++ b/net/bluetooth/hci_debugfs.c @@ -1245,7 +1245,7 @@ void hci_debugfs_create_conn(struct hci_conn *conn) struct hci_dev *hdev = conn->hdev; char name[6]; - if (IS_ERR_OR_NULL(hdev->debugfs)) + if (IS_ERR_OR_NULL(hdev->debugfs) || conn->debugfs) return; snprintf(name, sizeof(name), "%u", conn->handle); diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 485c814cf44aa04123d57a428a6e1a4d6de946a4..faca701bce2a38ae8ee223600746003adbd921dd 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -712,6 +712,47 @@ static u8 hci_cc_read_local_version(struct hci_dev *hdev, void *data, return rp->status; } +static u8 hci_cc_read_enc_key_size(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_rp_read_enc_key_size *rp = data; + struct hci_conn *conn; + u16 handle; + u8 status = rp->status; + + bt_dev_dbg(hdev, "status 0x%2.2x", status); + + handle = le16_to_cpu(rp->handle); + + hci_dev_lock(hdev); + + conn = hci_conn_hash_lookup_handle(hdev, handle); + if (!conn) { + status = 0xFF; + goto done; + } + + /* While unexpected, the read_enc_key_size command may fail. The most + * secure approach is to then assume the key size is 0 to force a + * disconnection. + */ + if (status) { + bt_dev_err(hdev, "failed to read key size for handle %u", + handle); + conn->enc_key_size = 0; + } else { + conn->enc_key_size = rp->key_size; + status = 0; + } + + hci_encrypt_cfm(conn, 0); + +done: + hci_dev_unlock(hdev); + + return status; +} + static u8 hci_cc_read_local_commands(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -1715,6 +1756,8 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable) hci_dev_set_flag(hdev, HCI_LE_SCAN); if (hdev->le_scan_type == LE_SCAN_ACTIVE) clear_pending_adv_report(hdev); + if (hci_dev_test_flag(hdev, HCI_MESH)) + hci_discovery_set_state(hdev, DISCOVERY_FINDING); break; case LE_SCAN_DISABLE: @@ -1729,7 +1772,7 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable) d->last_adv_addr_type, NULL, d->last_adv_rssi, d->last_adv_flags, d->last_adv_data, - d->last_adv_data_len, NULL, 0); + d->last_adv_data_len, NULL, 0, 0); } /* Cancel this timer so that we don't try to disable scanning @@ -1745,6 +1788,9 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable) */ if (hci_dev_test_and_clear_flag(hdev, HCI_LE_SCAN_INTERRUPTED)) hci_discovery_set_state(hdev, DISCOVERY_STOPPED); + else if (!hci_dev_test_flag(hdev, HCI_LE_ADV) && + hdev->discovery.state == DISCOVERY_FINDING) + queue_work(hdev->workqueue, &hdev->reenable_adv_work); break; @@ -2152,7 +2198,7 @@ static u8 hci_cc_set_ext_adv_param(struct hci_dev *hdev, void *data, adv_instance->tx_power = rp->tx_power; } /* Update adv data as tx power is known now */ - hci_req_update_adv_data(hdev, cp->handle); + hci_update_adv_data(hdev, cp->handle); hci_dev_unlock(hdev); @@ -3071,7 +3117,7 @@ static void hci_inquiry_result_evt(struct hci_dev *hdev, void *edata, mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00, info->dev_class, HCI_RSSI_INVALID, - flags, NULL, 0, NULL, 0); + flags, NULL, 0, NULL, 0, 0); } hci_dev_unlock(hdev); @@ -3534,47 +3580,6 @@ unlock: hci_dev_unlock(hdev); } -static void read_enc_key_size_complete(struct hci_dev *hdev, u8 status, - u16 opcode, struct sk_buff *skb) -{ - const struct hci_rp_read_enc_key_size *rp; - struct hci_conn *conn; - u16 handle; - - BT_DBG("%s status 0x%02x", hdev->name, status); - - if (!skb || skb->len < sizeof(*rp)) { - bt_dev_err(hdev, "invalid read key size response"); - return; - } - - rp = (void *)skb->data; - handle = le16_to_cpu(rp->handle); - - hci_dev_lock(hdev); - - conn = hci_conn_hash_lookup_handle(hdev, handle); - if (!conn) - goto unlock; - - /* While unexpected, the read_enc_key_size command may fail. The most - * secure approach is to then assume the key size is 0 to force a - * disconnection. - */ - if (rp->status) { - bt_dev_err(hdev, "failed to read key size for handle %u", - handle); - conn->enc_key_size = 0; - } else { - conn->enc_key_size = rp->key_size; - } - - hci_encrypt_cfm(conn, 0); - -unlock: - hci_dev_unlock(hdev); -} - static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -3639,7 +3644,6 @@ static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data, /* Try reading the encryption key size for encrypted ACL links */ if (!ev->status && ev->encrypt && conn->type == ACL_LINK) { struct hci_cp_read_enc_key_size cp; - struct hci_request req; /* Only send HCI_Read_Encryption_Key_Size if the * controller really supports it. If it doesn't, assume @@ -3650,12 +3654,9 @@ static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data, goto notify; } - hci_req_init(&req, hdev); - cp.handle = cpu_to_le16(conn->handle); - hci_req_add(&req, HCI_OP_READ_ENC_KEY_SIZE, sizeof(cp), &cp); - - if (hci_req_run_skb(&req, read_enc_key_size_complete)) { + if (hci_send_cmd(hdev, HCI_OP_READ_ENC_KEY_SIZE, + sizeof(cp), &cp)) { bt_dev_err(hdev, "sending read key size failed"); conn->enc_key_size = HCI_LINK_KEY_SIZE; goto notify; @@ -3766,16 +3767,18 @@ static inline void handle_cmd_cnt_and_timer(struct hci_dev *hdev, u8 ncmd) { cancel_delayed_work(&hdev->cmd_timer); + rcu_read_lock(); if (!test_bit(HCI_RESET, &hdev->flags)) { if (ncmd) { cancel_delayed_work(&hdev->ncmd_timer); atomic_set(&hdev->cmd_cnt, 1); } else { if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE)) - schedule_delayed_work(&hdev->ncmd_timer, - HCI_NCMD_TIMEOUT); + queue_delayed_work(hdev->workqueue, &hdev->ncmd_timer, + HCI_NCMD_TIMEOUT); } } + rcu_read_unlock(); } static u8 hci_cc_le_read_buffer_size_v2(struct hci_dev *hdev, void *data, @@ -4037,6 +4040,8 @@ static const struct hci_cc { sizeof(struct hci_rp_read_local_amp_info)), HCI_CC(HCI_OP_READ_CLOCK, hci_cc_read_clock, sizeof(struct hci_rp_read_clock)), + HCI_CC(HCI_OP_READ_ENC_KEY_SIZE, hci_cc_read_enc_key_size, + sizeof(struct hci_rp_read_enc_key_size)), HCI_CC(HCI_OP_READ_INQ_RSP_TX_POWER, hci_cc_read_inq_rsp_tx_power, sizeof(struct hci_rp_read_inq_rsp_tx_power)), HCI_CC(HCI_OP_READ_DEF_ERR_DATA_REPORTING, @@ -4179,6 +4184,17 @@ static void hci_cmd_complete_evt(struct hci_dev *hdev, void *data, } } + if (i == ARRAY_SIZE(hci_cc_table)) { + /* Unknown opcode, assume byte 0 contains the status, so + * that e.g. __hci_cmd_sync() properly returns errors + * for vendor specific commands send by HCI drivers. + * If a vendor doesn't actually follow this convention we may + * need to introduce a vendor CC table in order to properly set + * the status. + */ + *status = skb->data[0]; + } + handle_cmd_cnt_and_timer(hdev, ev->ncmd); hci_req_cmd_complete(hdev, *opcode, *status, req_complete, @@ -4818,7 +4834,7 @@ static void hci_inquiry_result_with_rssi_evt(struct hci_dev *hdev, void *edata, mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00, info->dev_class, info->rssi, - flags, NULL, 0, NULL, 0); + flags, NULL, 0, NULL, 0, 0); } } else if (skb->len == array_size(ev->num, sizeof(struct inquiry_info_rssi))) { @@ -4849,7 +4865,7 @@ static void hci_inquiry_result_with_rssi_evt(struct hci_dev *hdev, void *edata, mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00, info->dev_class, info->rssi, - flags, NULL, 0, NULL, 0); + flags, NULL, 0, NULL, 0, 0); } } else { bt_dev_err(hdev, "Malformed HCI Event: 0x%2.2x", @@ -5105,7 +5121,7 @@ static void hci_extended_inquiry_result_evt(struct hci_dev *hdev, void *edata, mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00, info->dev_class, info->rssi, - flags, info->data, eir_len, NULL, 0); + flags, info->data, eir_len, NULL, 0, 0); } hci_dev_unlock(hdev); @@ -5790,7 +5806,7 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, */ hci_dev_clear_flag(hdev, HCI_LE_ADV); - conn = hci_lookup_le_connect(hdev); + conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, bdaddr); if (!conn) { /* In case of error status and there is no connection pending * just unlock as there is nothing to cleanup. @@ -6161,7 +6177,7 @@ static struct hci_conn *check_pending_le_conn(struct hci_dev *hdev, static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, u8 bdaddr_type, bdaddr_t *direct_addr, u8 direct_addr_type, s8 rssi, u8 *data, u8 len, - bool ext_adv) + bool ext_adv, bool ctl_time, u64 instant) { struct discovery_state *d = &hdev->discovery; struct smp_irk *irk; @@ -6209,7 +6225,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, * important to see if the address is matching the local * controller address. */ - if (direct_addr) { + if (!hci_dev_test_flag(hdev, HCI_MESH) && direct_addr) { direct_addr_type = ev_bdaddr_type(hdev, direct_addr_type, &bdaddr_resolved); @@ -6257,6 +6273,18 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, conn->le_adv_data_len = len; } + if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND) + flags = MGMT_DEV_FOUND_NOT_CONNECTABLE; + else + flags = 0; + + /* All scan results should be sent up for Mesh systems */ + if (hci_dev_test_flag(hdev, HCI_MESH)) { + mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL, + rssi, flags, data, len, NULL, 0, instant); + return; + } + /* Passive scanning shouldn't trigger any device found events, * except for devices marked as CONN_REPORT for which we do send * device found events, or advertisement monitoring requested. @@ -6270,12 +6298,8 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, idr_is_empty(&hdev->adv_monitors_idr)) return; - if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND) - flags = MGMT_DEV_FOUND_NOT_CONNECTABLE; - else - flags = 0; mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL, - rssi, flags, data, len, NULL, 0); + rssi, flags, data, len, NULL, 0, 0); return; } @@ -6294,11 +6318,8 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, * and just sends a scan response event, then it is marked as * not connectable as well. */ - if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND || - type == LE_ADV_SCAN_RSP) + if (type == LE_ADV_SCAN_RSP) flags = MGMT_DEV_FOUND_NOT_CONNECTABLE; - else - flags = 0; /* If there's nothing pending either store the data from this * event or send an immediate device found event if the data @@ -6315,7 +6336,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, } mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL, - rssi, flags, data, len, NULL, 0); + rssi, flags, data, len, NULL, 0, 0); return; } @@ -6334,7 +6355,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, d->last_adv_addr_type, NULL, d->last_adv_rssi, d->last_adv_flags, d->last_adv_data, - d->last_adv_data_len, NULL, 0); + d->last_adv_data_len, NULL, 0, 0); /* If the new report will trigger a SCAN_REQ store it for * later merging. @@ -6351,7 +6372,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, */ clear_pending_adv_report(hdev); mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL, - rssi, flags, data, len, NULL, 0); + rssi, flags, data, len, NULL, 0, 0); return; } @@ -6361,7 +6382,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr, */ mgmt_device_found(hdev, &d->last_adv_addr, LE_LINK, d->last_adv_addr_type, NULL, rssi, d->last_adv_flags, - d->last_adv_data, d->last_adv_data_len, data, len); + d->last_adv_data, d->last_adv_data_len, data, len, 0); clear_pending_adv_report(hdev); } @@ -6369,6 +6390,7 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { struct hci_ev_le_advertising_report *ev = data; + u64 instant = jiffies; if (!ev->num) return; @@ -6393,7 +6415,8 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data, rssi = info->data[info->length]; process_adv_report(hdev, info->type, &info->bdaddr, info->bdaddr_type, NULL, 0, rssi, - info->data, info->length, false); + info->data, info->length, false, + false, instant); } else { bt_dev_err(hdev, "Dropping invalid advertising data"); } @@ -6450,6 +6473,7 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { struct hci_ev_le_ext_adv_report *ev = data; + u64 instant = jiffies; if (!ev->num) return; @@ -6476,7 +6500,8 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data, process_adv_report(hdev, legacy_evt_type, &info->bdaddr, info->bdaddr_type, NULL, 0, info->rssi, info->data, info->length, - !(evt_type & LE_EXT_ADV_LEGACY_PDU)); + !(evt_type & LE_EXT_ADV_LEGACY_PDU), + false, instant); } } @@ -6699,6 +6724,7 @@ static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { struct hci_ev_le_direct_adv_report *ev = data; + u64 instant = jiffies; int i; if (!hci_le_ev_skb_pull(hdev, skb, HCI_EV_LE_DIRECT_ADV_REPORT, @@ -6716,7 +6742,7 @@ static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data, process_adv_report(hdev, info->type, &info->bdaddr, info->bdaddr_type, &info->direct_addr, info->direct_addr_type, info->rssi, NULL, 0, - false); + false, false, instant); } hci_dev_unlock(hdev); @@ -6765,6 +6791,13 @@ static void hci_le_cis_estabilished_evt(struct hci_dev *hdev, void *data, goto unlock; } + if (conn->type != ISO_LINK) { + bt_dev_err(hdev, + "Invalid connection link type handle 0x%4.4x", + handle); + goto unlock; + } + if (conn->role == HCI_ROLE_SLAVE) { __le32 interval; @@ -6885,6 +6918,13 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data, if (!conn) goto unlock; + if (conn->type != ISO_LINK) { + bt_dev_err(hdev, + "Invalid connection link type handle 0x%2.2x", + ev->handle); + goto unlock; + } + if (ev->num_bis) conn->handle = __le16_to_cpu(ev->bis_handle[0]); diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index e64d558e5d69e74765b1cab97f67367390083abf..5a0296a4352eff79e49c86cd5bdf7ddafb16dfab 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -269,42 +269,10 @@ void hci_req_add_ev(struct hci_request *req, u16 opcode, u32 plen, void hci_req_add(struct hci_request *req, u16 opcode, u32 plen, const void *param) { + bt_dev_err(req->hdev, "HCI_REQ-0x%4.4x", opcode); hci_req_add_ev(req, opcode, plen, param, 0); } -void __hci_req_write_fast_connectable(struct hci_request *req, bool enable) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_page_scan_activity acp; - u8 type; - - if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) - return; - - if (hdev->hci_ver < BLUETOOTH_VER_1_2) - return; - - if (enable) { - type = PAGE_SCAN_TYPE_INTERLACED; - - /* 160 msec page scan interval */ - acp.interval = cpu_to_le16(0x0100); - } else { - type = hdev->def_page_scan_type; - acp.interval = cpu_to_le16(hdev->def_page_scan_int); - } - - acp.window = cpu_to_le16(hdev->def_page_scan_window); - - if (__cpu_to_le16(hdev->page_scan_interval) != acp.interval || - __cpu_to_le16(hdev->page_scan_window) != acp.window) - hci_req_add(req, HCI_OP_WRITE_PAGE_SCAN_ACTIVITY, - sizeof(acp), &acp); - - if (hdev->page_scan_type != type) - hci_req_add(req, HCI_OP_WRITE_PAGE_SCAN_TYPE, 1, &type); -} - static void start_interleave_scan(struct hci_dev *hdev) { hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER; @@ -357,45 +325,6 @@ static bool __hci_update_interleaved_scan(struct hci_dev *hdev) return false; } -void __hci_req_update_name(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_local_name cp; - - memcpy(cp.name, hdev->dev_name, sizeof(cp.name)); - - hci_req_add(req, HCI_OP_WRITE_LOCAL_NAME, sizeof(cp), &cp); -} - -void __hci_req_update_eir(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_eir cp; - - if (!hdev_is_powered(hdev)) - return; - - if (!lmp_ext_inq_capable(hdev)) - return; - - if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) - return; - - if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) - return; - - memset(&cp, 0, sizeof(cp)); - - eir_create(hdev, cp.data); - - if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) - return; - - memcpy(hdev->eir, cp.data, sizeof(cp.data)); - - hci_req_add(req, HCI_OP_WRITE_EIR, sizeof(cp), &cp); -} - void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn) { struct hci_dev *hdev = req->hdev; @@ -721,6 +650,96 @@ static inline bool hci_is_le_conn_scanning(struct hci_dev *hdev) return false; } +static void set_random_addr(struct hci_request *req, bdaddr_t *rpa); +static int hci_update_random_address(struct hci_request *req, + bool require_privacy, bool use_rpa, + u8 *own_addr_type) +{ + struct hci_dev *hdev = req->hdev; + int err; + + /* If privacy is enabled use a resolvable private address. If + * current RPA has expired or there is something else than + * the current RPA in use, then generate a new one. + */ + if (use_rpa) { + /* If Controller supports LL Privacy use own address type is + * 0x03 + */ + if (use_ll_privacy(hdev)) + *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; + else + *own_addr_type = ADDR_LE_DEV_RANDOM; + + if (rpa_valid(hdev)) + return 0; + + err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa); + if (err < 0) { + bt_dev_err(hdev, "failed to generate new RPA"); + return err; + } + + set_random_addr(req, &hdev->rpa); + + return 0; + } + + /* In case of required privacy without resolvable private address, + * use an non-resolvable private address. This is useful for active + * scanning and non-connectable advertising. + */ + if (require_privacy) { + bdaddr_t nrpa; + + while (true) { + /* The non-resolvable private address is generated + * from random six bytes with the two most significant + * bits cleared. + */ + get_random_bytes(&nrpa, 6); + nrpa.b[5] &= 0x3f; + + /* The non-resolvable private address shall not be + * equal to the public address. + */ + if (bacmp(&hdev->bdaddr, &nrpa)) + break; + } + + *own_addr_type = ADDR_LE_DEV_RANDOM; + set_random_addr(req, &nrpa); + return 0; + } + + /* If forcing static address is in use or there is no public + * address use the static address as random address (but skip + * the HCI command if the current random address is already the + * static one. + * + * In case BR/EDR has been disabled on a dual-mode controller + * and a static address has been configured, then use that + * address instead of the public BR/EDR address. + */ + if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) || + !bacmp(&hdev->bdaddr, BDADDR_ANY) || + (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) && + bacmp(&hdev->static_addr, BDADDR_ANY))) { + *own_addr_type = ADDR_LE_DEV_RANDOM; + if (bacmp(&hdev->static_addr, &hdev->random_addr)) + hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, + &hdev->static_addr); + return 0; + } + + /* Neither privacy nor static address is being used so use a + * public address. + */ + *own_addr_type = ADDR_LE_DEV_PUBLIC; + + return 0; +} + /* Ensure to call hci_req_add_le_scan_disable() first to disable the * controller based address resolution to be able to reconfigure * resolving list. @@ -810,1465 +829,94 @@ void hci_req_add_le_passive_scan(struct hci_request *req) addr_resolv); } -static void cancel_adv_timeout(struct hci_dev *hdev) +static int hci_req_add_le_interleaved_scan(struct hci_request *req, + unsigned long opt) { - if (hdev->adv_instance_timeout) { - hdev->adv_instance_timeout = 0; - cancel_delayed_work(&hdev->adv_instance_expire); - } -} + struct hci_dev *hdev = req->hdev; + int ret = 0; -static bool adv_cur_instance_is_scannable(struct hci_dev *hdev) -{ - return hci_adv_instance_is_scannable(hdev, hdev->cur_adv_instance); -} + hci_dev_lock(hdev); -void __hci_req_disable_advertising(struct hci_request *req) -{ - if (ext_adv_capable(req->hdev)) { - __hci_req_disable_ext_adv_instance(req, 0x00); - } else { - u8 enable = 0x00; + if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) + hci_req_add_le_scan_disable(req, false); + hci_req_add_le_passive_scan(req); - hci_req_add(req, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable); + switch (hdev->interleave_scan_state) { + case INTERLEAVE_SCAN_ALLOWLIST: + bt_dev_dbg(hdev, "next state: allowlist"); + hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER; + break; + case INTERLEAVE_SCAN_NO_FILTER: + bt_dev_dbg(hdev, "next state: no filter"); + hdev->interleave_scan_state = INTERLEAVE_SCAN_ALLOWLIST; + break; + case INTERLEAVE_SCAN_NONE: + BT_ERR("unexpected error"); + ret = -1; } -} -static bool adv_use_rpa(struct hci_dev *hdev, uint32_t flags) -{ - /* If privacy is not enabled don't use RPA */ - if (!hci_dev_test_flag(hdev, HCI_PRIVACY)) - return false; - - /* If basic privacy mode is enabled use RPA */ - if (!hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) - return true; - - /* If limited privacy mode is enabled don't use RPA if we're - * both discoverable and bondable. - */ - if ((flags & MGMT_ADV_FLAG_DISCOV) && - hci_dev_test_flag(hdev, HCI_BONDABLE)) - return false; + hci_dev_unlock(hdev); - /* We're neither bondable nor discoverable in the limited - * privacy mode, therefore use RPA. - */ - return true; + return ret; } -static bool is_advertising_allowed(struct hci_dev *hdev, bool connectable) +static void interleave_scan_work(struct work_struct *work) { - /* If there is no connection we are OK to advertise. */ - if (hci_conn_num(hdev, LE_LINK) == 0) - return true; - - /* Check le_states if there is any connection in peripheral role. */ - if (hdev->conn_hash.le_num_peripheral > 0) { - /* Peripheral connection state and non connectable mode bit 20. - */ - if (!connectable && !(hdev->le_states[2] & 0x10)) - return false; + struct hci_dev *hdev = container_of(work, struct hci_dev, + interleave_scan.work); + u8 status; + unsigned long timeout; - /* Peripheral connection state and connectable mode bit 38 - * and scannable bit 21. - */ - if (connectable && (!(hdev->le_states[4] & 0x40) || - !(hdev->le_states[2] & 0x20))) - return false; + if (hdev->interleave_scan_state == INTERLEAVE_SCAN_ALLOWLIST) { + timeout = msecs_to_jiffies(hdev->advmon_allowlist_duration); + } else if (hdev->interleave_scan_state == INTERLEAVE_SCAN_NO_FILTER) { + timeout = msecs_to_jiffies(hdev->advmon_no_filter_duration); + } else { + bt_dev_err(hdev, "unexpected error"); + return; } - /* Check le_states if there is any connection in central role. */ - if (hci_conn_num(hdev, LE_LINK) != hdev->conn_hash.le_num_peripheral) { - /* Central connection state and non connectable mode bit 18. */ - if (!connectable && !(hdev->le_states[2] & 0x02)) - return false; - - /* Central connection state and connectable mode bit 35 and - * scannable 19. - */ - if (connectable && (!(hdev->le_states[4] & 0x08) || - !(hdev->le_states[2] & 0x08))) - return false; - } + hci_req_sync(hdev, hci_req_add_le_interleaved_scan, 0, + HCI_CMD_TIMEOUT, &status); - return true; + /* Don't continue interleaving if it was canceled */ + if (is_interleave_scanning(hdev)) + queue_delayed_work(hdev->req_workqueue, + &hdev->interleave_scan, timeout); } -void __hci_req_enable_advertising(struct hci_request *req) +static void set_random_addr(struct hci_request *req, bdaddr_t *rpa) { struct hci_dev *hdev = req->hdev; - struct adv_info *adv; - struct hci_cp_le_set_adv_param cp; - u8 own_addr_type, enable = 0x01; - bool connectable; - u16 adv_min_interval, adv_max_interval; - u32 flags; - - flags = hci_adv_instance_flags(hdev, hdev->cur_adv_instance); - adv = hci_find_adv_instance(hdev, hdev->cur_adv_instance); - - /* If the "connectable" instance flag was not set, then choose between - * ADV_IND and ADV_NONCONN_IND based on the global connectable setting. - */ - connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || - mgmt_get_connectable(hdev); - - if (!is_advertising_allowed(hdev, connectable)) - return; - - if (hci_dev_test_flag(hdev, HCI_LE_ADV)) - __hci_req_disable_advertising(req); - - /* Clear the HCI_LE_ADV bit temporarily so that the - * hci_update_random_address knows that it's safe to go ahead - * and write a new random address. The flag will be set back on - * as soon as the SET_ADV_ENABLE HCI command completes. - */ - hci_dev_clear_flag(hdev, HCI_LE_ADV); - /* Set require_privacy to true only when non-connectable - * advertising is used. In that case it is fine to use a - * non-resolvable private address. + /* If we're advertising or initiating an LE connection we can't + * go ahead and change the random address at this time. This is + * because the eventual initiator address used for the + * subsequently created connection will be undefined (some + * controllers use the new address and others the one we had + * when the operation started). + * + * In this kind of scenario skip the update and let the random + * address be updated at the next cycle. */ - if (hci_update_random_address(req, !connectable, - adv_use_rpa(hdev, flags), - &own_addr_type) < 0) + if (hci_dev_test_flag(hdev, HCI_LE_ADV) || + hci_lookup_le_connect(hdev)) { + bt_dev_dbg(hdev, "Deferring random address update"); + hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); return; - - memset(&cp, 0, sizeof(cp)); - - if (adv) { - adv_min_interval = adv->min_interval; - adv_max_interval = adv->max_interval; - } else { - adv_min_interval = hdev->le_adv_min_interval; - adv_max_interval = hdev->le_adv_max_interval; } - if (connectable) { - cp.type = LE_ADV_IND; - } else { - if (adv_cur_instance_is_scannable(hdev)) - cp.type = LE_ADV_SCAN_IND; - else - cp.type = LE_ADV_NONCONN_IND; - - if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE) || - hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) { - adv_min_interval = DISCOV_LE_FAST_ADV_INT_MIN; - adv_max_interval = DISCOV_LE_FAST_ADV_INT_MAX; - } - } - - cp.min_interval = cpu_to_le16(adv_min_interval); - cp.max_interval = cpu_to_le16(adv_max_interval); - cp.own_address_type = own_addr_type; - cp.channel_map = hdev->le_adv_channel_map; - - hci_req_add(req, HCI_OP_LE_SET_ADV_PARAM, sizeof(cp), &cp); - - hci_req_add(req, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable); + hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa); } -void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance) +void hci_request_setup(struct hci_dev *hdev) { - struct hci_dev *hdev = req->hdev; - u8 len; - - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) - return; - - if (ext_adv_capable(hdev)) { - struct { - struct hci_cp_le_set_ext_scan_rsp_data cp; - u8 data[HCI_MAX_EXT_AD_LENGTH]; - } pdu; - - memset(&pdu, 0, sizeof(pdu)); - - len = eir_create_scan_rsp(hdev, instance, pdu.data); - - if (hdev->scan_rsp_data_len == len && - !memcmp(pdu.data, hdev->scan_rsp_data, len)) - return; - - memcpy(hdev->scan_rsp_data, pdu.data, len); - hdev->scan_rsp_data_len = len; - - pdu.cp.handle = instance; - pdu.cp.length = len; - pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; - pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; - - hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, - sizeof(pdu.cp) + len, &pdu.cp); - } else { - struct hci_cp_le_set_scan_rsp_data cp; - - memset(&cp, 0, sizeof(cp)); - - len = eir_create_scan_rsp(hdev, instance, cp.data); - - if (hdev->scan_rsp_data_len == len && - !memcmp(cp.data, hdev->scan_rsp_data, len)) - return; - - memcpy(hdev->scan_rsp_data, cp.data, sizeof(cp.data)); - hdev->scan_rsp_data_len = len; - - cp.length = len; - - hci_req_add(req, HCI_OP_LE_SET_SCAN_RSP_DATA, sizeof(cp), &cp); - } + INIT_DELAYED_WORK(&hdev->interleave_scan, interleave_scan_work); } -void __hci_req_update_adv_data(struct hci_request *req, u8 instance) -{ - struct hci_dev *hdev = req->hdev; - u8 len; - - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) - return; - - if (ext_adv_capable(hdev)) { - struct { - struct hci_cp_le_set_ext_adv_data cp; - u8 data[HCI_MAX_EXT_AD_LENGTH]; - } pdu; - - memset(&pdu, 0, sizeof(pdu)); - - len = eir_create_adv_data(hdev, instance, pdu.data); - - /* There's nothing to do if the data hasn't changed */ - if (hdev->adv_data_len == len && - memcmp(pdu.data, hdev->adv_data, len) == 0) - return; - - memcpy(hdev->adv_data, pdu.data, len); - hdev->adv_data_len = len; - - pdu.cp.length = len; - pdu.cp.handle = instance; - pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; - pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; - - hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_DATA, - sizeof(pdu.cp) + len, &pdu.cp); - } else { - struct hci_cp_le_set_adv_data cp; - - memset(&cp, 0, sizeof(cp)); - - len = eir_create_adv_data(hdev, instance, cp.data); - - /* There's nothing to do if the data hasn't changed */ - if (hdev->adv_data_len == len && - memcmp(cp.data, hdev->adv_data, len) == 0) - return; - - memcpy(hdev->adv_data, cp.data, sizeof(cp.data)); - hdev->adv_data_len = len; - - cp.length = len; - - hci_req_add(req, HCI_OP_LE_SET_ADV_DATA, sizeof(cp), &cp); - } -} - -int hci_req_update_adv_data(struct hci_dev *hdev, u8 instance) -{ - struct hci_request req; - - hci_req_init(&req, hdev); - __hci_req_update_adv_data(&req, instance); - - return hci_req_run(&req, NULL); -} - -static void enable_addr_resolution_complete(struct hci_dev *hdev, u8 status, - u16 opcode) -{ - BT_DBG("%s status %u", hdev->name, status); -} - -void hci_req_disable_address_resolution(struct hci_dev *hdev) -{ - struct hci_request req; - __u8 enable = 0x00; - - if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) - return; - - hci_req_init(&req, hdev); - - hci_req_add(&req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable); - - hci_req_run(&req, enable_addr_resolution_complete); -} - -static void adv_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode) -{ - bt_dev_dbg(hdev, "status %u", status); -} - -void hci_req_reenable_advertising(struct hci_dev *hdev) -{ - struct hci_request req; - - if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) && - list_empty(&hdev->adv_instances)) - return; - - hci_req_init(&req, hdev); - - if (hdev->cur_adv_instance) { - __hci_req_schedule_adv_instance(&req, hdev->cur_adv_instance, - true); - } else { - if (ext_adv_capable(hdev)) { - __hci_req_start_ext_adv(&req, 0x00); - } else { - __hci_req_update_adv_data(&req, 0x00); - __hci_req_update_scan_rsp_data(&req, 0x00); - __hci_req_enable_advertising(&req); - } - } - - hci_req_run(&req, adv_enable_complete); -} - -static void adv_timeout_expire(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - adv_instance_expire.work); - - struct hci_request req; - u8 instance; - - bt_dev_dbg(hdev, ""); - - hci_dev_lock(hdev); - - hdev->adv_instance_timeout = 0; - - instance = hdev->cur_adv_instance; - if (instance == 0x00) - goto unlock; - - hci_req_init(&req, hdev); - - hci_req_clear_adv_instance(hdev, NULL, &req, instance, false); - - if (list_empty(&hdev->adv_instances)) - __hci_req_disable_advertising(&req); - - hci_req_run(&req, NULL); - -unlock: - hci_dev_unlock(hdev); -} - -static int hci_req_add_le_interleaved_scan(struct hci_request *req, - unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - int ret = 0; - - hci_dev_lock(hdev); - - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) - hci_req_add_le_scan_disable(req, false); - hci_req_add_le_passive_scan(req); - - switch (hdev->interleave_scan_state) { - case INTERLEAVE_SCAN_ALLOWLIST: - bt_dev_dbg(hdev, "next state: allowlist"); - hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER; - break; - case INTERLEAVE_SCAN_NO_FILTER: - bt_dev_dbg(hdev, "next state: no filter"); - hdev->interleave_scan_state = INTERLEAVE_SCAN_ALLOWLIST; - break; - case INTERLEAVE_SCAN_NONE: - BT_ERR("unexpected error"); - ret = -1; - } - - hci_dev_unlock(hdev); - - return ret; -} - -static void interleave_scan_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - interleave_scan.work); - u8 status; - unsigned long timeout; - - if (hdev->interleave_scan_state == INTERLEAVE_SCAN_ALLOWLIST) { - timeout = msecs_to_jiffies(hdev->advmon_allowlist_duration); - } else if (hdev->interleave_scan_state == INTERLEAVE_SCAN_NO_FILTER) { - timeout = msecs_to_jiffies(hdev->advmon_no_filter_duration); - } else { - bt_dev_err(hdev, "unexpected error"); - return; - } - - hci_req_sync(hdev, hci_req_add_le_interleaved_scan, 0, - HCI_CMD_TIMEOUT, &status); - - /* Don't continue interleaving if it was canceled */ - if (is_interleave_scanning(hdev)) - queue_delayed_work(hdev->req_workqueue, - &hdev->interleave_scan, timeout); -} - -int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, - bool use_rpa, struct adv_info *adv_instance, - u8 *own_addr_type, bdaddr_t *rand_addr) -{ - int err; - - bacpy(rand_addr, BDADDR_ANY); - - /* If privacy is enabled use a resolvable private address. If - * current RPA has expired then generate a new one. - */ - if (use_rpa) { - /* If Controller supports LL Privacy use own address type is - * 0x03 - */ - if (use_ll_privacy(hdev)) - *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; - else - *own_addr_type = ADDR_LE_DEV_RANDOM; - - if (adv_instance) { - if (adv_rpa_valid(adv_instance)) - return 0; - } else { - if (rpa_valid(hdev)) - return 0; - } - - err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa); - if (err < 0) { - bt_dev_err(hdev, "failed to generate new RPA"); - return err; - } - - bacpy(rand_addr, &hdev->rpa); - - return 0; - } - - /* In case of required privacy without resolvable private address, - * use an non-resolvable private address. This is useful for - * non-connectable advertising. - */ - if (require_privacy) { - bdaddr_t nrpa; - - while (true) { - /* The non-resolvable private address is generated - * from random six bytes with the two most significant - * bits cleared. - */ - get_random_bytes(&nrpa, 6); - nrpa.b[5] &= 0x3f; - - /* The non-resolvable private address shall not be - * equal to the public address. - */ - if (bacmp(&hdev->bdaddr, &nrpa)) - break; - } - - *own_addr_type = ADDR_LE_DEV_RANDOM; - bacpy(rand_addr, &nrpa); - - return 0; - } - - /* No privacy so use a public address. */ - *own_addr_type = ADDR_LE_DEV_PUBLIC; - - return 0; -} - -void __hci_req_clear_ext_adv_sets(struct hci_request *req) -{ - hci_req_add(req, HCI_OP_LE_CLEAR_ADV_SETS, 0, NULL); -} - -static void set_random_addr(struct hci_request *req, bdaddr_t *rpa) -{ - struct hci_dev *hdev = req->hdev; - - /* If we're advertising or initiating an LE connection we can't - * go ahead and change the random address at this time. This is - * because the eventual initiator address used for the - * subsequently created connection will be undefined (some - * controllers use the new address and others the one we had - * when the operation started). - * - * In this kind of scenario skip the update and let the random - * address be updated at the next cycle. - */ - if (hci_dev_test_flag(hdev, HCI_LE_ADV) || - hci_lookup_le_connect(hdev)) { - bt_dev_dbg(hdev, "Deferring random address update"); - hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); - return; - } - - hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa); -} - -int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) -{ - struct hci_cp_le_set_ext_adv_params cp; - struct hci_dev *hdev = req->hdev; - bool connectable; - u32 flags; - bdaddr_t random_addr; - u8 own_addr_type; - int err; - struct adv_info *adv; - bool secondary_adv, require_privacy; - - if (instance > 0) { - adv = hci_find_adv_instance(hdev, instance); - if (!adv) - return -EINVAL; - } else { - adv = NULL; - } - - flags = hci_adv_instance_flags(hdev, instance); - - /* If the "connectable" instance flag was not set, then choose between - * ADV_IND and ADV_NONCONN_IND based on the global connectable setting. - */ - connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || - mgmt_get_connectable(hdev); - - if (!is_advertising_allowed(hdev, connectable)) - return -EPERM; - - /* Set require_privacy to true only when non-connectable - * advertising is used. In that case it is fine to use a - * non-resolvable private address. - */ - require_privacy = !connectable; - - /* Don't require privacy for periodic adv? */ - if (adv && adv->periodic) - require_privacy = false; - - err = hci_get_random_address(hdev, require_privacy, - adv_use_rpa(hdev, flags), adv, - &own_addr_type, &random_addr); - if (err < 0) - return err; - - memset(&cp, 0, sizeof(cp)); - - if (adv) { - hci_cpu_to_le24(adv->min_interval, cp.min_interval); - hci_cpu_to_le24(adv->max_interval, cp.max_interval); - cp.tx_power = adv->tx_power; - } else { - hci_cpu_to_le24(hdev->le_adv_min_interval, cp.min_interval); - hci_cpu_to_le24(hdev->le_adv_max_interval, cp.max_interval); - cp.tx_power = HCI_ADV_TX_POWER_NO_PREFERENCE; - } - - secondary_adv = (flags & MGMT_ADV_FLAG_SEC_MASK); - - if (connectable) { - if (secondary_adv) - cp.evt_properties = cpu_to_le16(LE_EXT_ADV_CONN_IND); - else - cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_IND); - } else if (hci_adv_instance_is_scannable(hdev, instance) || - (flags & MGMT_ADV_PARAM_SCAN_RSP)) { - if (secondary_adv) - cp.evt_properties = cpu_to_le16(LE_EXT_ADV_SCAN_IND); - else - cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_SCAN_IND); - } else { - /* Secondary and periodic cannot use legacy PDUs */ - if (secondary_adv || (adv && adv->periodic)) - cp.evt_properties = cpu_to_le16(LE_EXT_ADV_NON_CONN_IND); - else - cp.evt_properties = cpu_to_le16(LE_LEGACY_NONCONN_IND); - } - - cp.own_addr_type = own_addr_type; - cp.channel_map = hdev->le_adv_channel_map; - cp.handle = instance; - - if (flags & MGMT_ADV_FLAG_SEC_2M) { - cp.primary_phy = HCI_ADV_PHY_1M; - cp.secondary_phy = HCI_ADV_PHY_2M; - } else if (flags & MGMT_ADV_FLAG_SEC_CODED) { - cp.primary_phy = HCI_ADV_PHY_CODED; - cp.secondary_phy = HCI_ADV_PHY_CODED; - } else { - /* In all other cases use 1M */ - cp.primary_phy = HCI_ADV_PHY_1M; - cp.secondary_phy = HCI_ADV_PHY_1M; - } - - hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(cp), &cp); - - if ((own_addr_type == ADDR_LE_DEV_RANDOM || - own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) && - bacmp(&random_addr, BDADDR_ANY)) { - struct hci_cp_le_set_adv_set_rand_addr cp; - - /* Check if random address need to be updated */ - if (adv) { - if (!bacmp(&random_addr, &adv->random_addr)) - return 0; - } else { - if (!bacmp(&random_addr, &hdev->random_addr)) - return 0; - /* Instance 0x00 doesn't have an adv_info, instead it - * uses hdev->random_addr to track its address so - * whenever it needs to be updated this also set the - * random address since hdev->random_addr is shared with - * scan state machine. - */ - set_random_addr(req, &random_addr); - } - - memset(&cp, 0, sizeof(cp)); - - cp.handle = instance; - bacpy(&cp.bdaddr, &random_addr); - - hci_req_add(req, - HCI_OP_LE_SET_ADV_SET_RAND_ADDR, - sizeof(cp), &cp); - } - - return 0; -} - -int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_le_set_ext_adv_enable *cp; - struct hci_cp_ext_adv_set *adv_set; - u8 data[sizeof(*cp) + sizeof(*adv_set) * 1]; - struct adv_info *adv_instance; - - if (instance > 0) { - adv_instance = hci_find_adv_instance(hdev, instance); - if (!adv_instance) - return -EINVAL; - } else { - adv_instance = NULL; - } - - cp = (void *) data; - adv_set = (void *) cp->data; - - memset(cp, 0, sizeof(*cp)); - - cp->enable = 0x01; - cp->num_of_sets = 0x01; - - memset(adv_set, 0, sizeof(*adv_set)); - - adv_set->handle = instance; - - /* Set duration per instance since controller is responsible for - * scheduling it. - */ - if (adv_instance && adv_instance->duration) { - u16 duration = adv_instance->timeout * MSEC_PER_SEC; - - /* Time = N * 10 ms */ - adv_set->duration = cpu_to_le16(duration / 10); - } - - hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_ENABLE, - sizeof(*cp) + sizeof(*adv_set) * cp->num_of_sets, - data); - - return 0; -} - -int __hci_req_disable_ext_adv_instance(struct hci_request *req, u8 instance) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_le_set_ext_adv_enable *cp; - struct hci_cp_ext_adv_set *adv_set; - u8 data[sizeof(*cp) + sizeof(*adv_set) * 1]; - u8 req_size; - - /* If request specifies an instance that doesn't exist, fail */ - if (instance > 0 && !hci_find_adv_instance(hdev, instance)) - return -EINVAL; - - memset(data, 0, sizeof(data)); - - cp = (void *)data; - adv_set = (void *)cp->data; - - /* Instance 0x00 indicates all advertising instances will be disabled */ - cp->num_of_sets = !!instance; - cp->enable = 0x00; - - adv_set->handle = instance; - - req_size = sizeof(*cp) + sizeof(*adv_set) * cp->num_of_sets; - hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_ENABLE, req_size, data); - - return 0; -} - -int __hci_req_remove_ext_adv_instance(struct hci_request *req, u8 instance) -{ - struct hci_dev *hdev = req->hdev; - - /* If request specifies an instance that doesn't exist, fail */ - if (instance > 0 && !hci_find_adv_instance(hdev, instance)) - return -EINVAL; - - hci_req_add(req, HCI_OP_LE_REMOVE_ADV_SET, sizeof(instance), &instance); - - return 0; -} - -int __hci_req_start_ext_adv(struct hci_request *req, u8 instance) -{ - struct hci_dev *hdev = req->hdev; - struct adv_info *adv_instance = hci_find_adv_instance(hdev, instance); - int err; - - /* If instance isn't pending, the chip knows about it, and it's safe to - * disable - */ - if (adv_instance && !adv_instance->pending) - __hci_req_disable_ext_adv_instance(req, instance); - - err = __hci_req_setup_ext_adv_instance(req, instance); - if (err < 0) - return err; - - __hci_req_update_scan_rsp_data(req, instance); - __hci_req_enable_ext_advertising(req, instance); - - return 0; -} - -int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance, - bool force) -{ - struct hci_dev *hdev = req->hdev; - struct adv_info *adv_instance = NULL; - u16 timeout; - - if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || - list_empty(&hdev->adv_instances)) - return -EPERM; - - if (hdev->adv_instance_timeout) - return -EBUSY; - - adv_instance = hci_find_adv_instance(hdev, instance); - if (!adv_instance) - return -ENOENT; - - /* A zero timeout means unlimited advertising. As long as there is - * only one instance, duration should be ignored. We still set a timeout - * in case further instances are being added later on. - * - * If the remaining lifetime of the instance is more than the duration - * then the timeout corresponds to the duration, otherwise it will be - * reduced to the remaining instance lifetime. - */ - if (adv_instance->timeout == 0 || - adv_instance->duration <= adv_instance->remaining_time) - timeout = adv_instance->duration; - else - timeout = adv_instance->remaining_time; - - /* The remaining time is being reduced unless the instance is being - * advertised without time limit. - */ - if (adv_instance->timeout) - adv_instance->remaining_time = - adv_instance->remaining_time - timeout; - - /* Only use work for scheduling instances with legacy advertising */ - if (!ext_adv_capable(hdev)) { - hdev->adv_instance_timeout = timeout; - queue_delayed_work(hdev->req_workqueue, - &hdev->adv_instance_expire, - msecs_to_jiffies(timeout * 1000)); - } - - /* If we're just re-scheduling the same instance again then do not - * execute any HCI commands. This happens when a single instance is - * being advertised. - */ - if (!force && hdev->cur_adv_instance == instance && - hci_dev_test_flag(hdev, HCI_LE_ADV)) - return 0; - - hdev->cur_adv_instance = instance; - if (ext_adv_capable(hdev)) { - __hci_req_start_ext_adv(req, instance); - } else { - __hci_req_update_adv_data(req, instance); - __hci_req_update_scan_rsp_data(req, instance); - __hci_req_enable_advertising(req); - } - - return 0; -} - -/* For a single instance: - * - force == true: The instance will be removed even when its remaining - * lifetime is not zero. - * - force == false: the instance will be deactivated but kept stored unless - * the remaining lifetime is zero. - * - * For instance == 0x00: - * - force == true: All instances will be removed regardless of their timeout - * setting. - * - force == false: Only instances that have a timeout will be removed. - */ -void hci_req_clear_adv_instance(struct hci_dev *hdev, struct sock *sk, - struct hci_request *req, u8 instance, - bool force) -{ - struct adv_info *adv_instance, *n, *next_instance = NULL; - int err; - u8 rem_inst; - - /* Cancel any timeout concerning the removed instance(s). */ - if (!instance || hdev->cur_adv_instance == instance) - cancel_adv_timeout(hdev); - - /* Get the next instance to advertise BEFORE we remove - * the current one. This can be the same instance again - * if there is only one instance. - */ - if (instance && hdev->cur_adv_instance == instance) - next_instance = hci_get_next_instance(hdev, instance); - - if (instance == 0x00) { - list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances, - list) { - if (!(force || adv_instance->timeout)) - continue; - - rem_inst = adv_instance->instance; - err = hci_remove_adv_instance(hdev, rem_inst); - if (!err) - mgmt_advertising_removed(sk, hdev, rem_inst); - } - } else { - adv_instance = hci_find_adv_instance(hdev, instance); - - if (force || (adv_instance && adv_instance->timeout && - !adv_instance->remaining_time)) { - /* Don't advertise a removed instance. */ - if (next_instance && - next_instance->instance == instance) - next_instance = NULL; - - err = hci_remove_adv_instance(hdev, instance); - if (!err) - mgmt_advertising_removed(sk, hdev, instance); - } - } - - if (!req || !hdev_is_powered(hdev) || - hci_dev_test_flag(hdev, HCI_ADVERTISING)) - return; - - if (next_instance && !ext_adv_capable(hdev)) - __hci_req_schedule_adv_instance(req, next_instance->instance, - false); -} - -int hci_update_random_address(struct hci_request *req, bool require_privacy, - bool use_rpa, u8 *own_addr_type) -{ - struct hci_dev *hdev = req->hdev; - int err; - - /* If privacy is enabled use a resolvable private address. If - * current RPA has expired or there is something else than - * the current RPA in use, then generate a new one. - */ - if (use_rpa) { - /* If Controller supports LL Privacy use own address type is - * 0x03 - */ - if (use_ll_privacy(hdev)) - *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; - else - *own_addr_type = ADDR_LE_DEV_RANDOM; - - if (rpa_valid(hdev)) - return 0; - - err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa); - if (err < 0) { - bt_dev_err(hdev, "failed to generate new RPA"); - return err; - } - - set_random_addr(req, &hdev->rpa); - - return 0; - } - - /* In case of required privacy without resolvable private address, - * use an non-resolvable private address. This is useful for active - * scanning and non-connectable advertising. - */ - if (require_privacy) { - bdaddr_t nrpa; - - while (true) { - /* The non-resolvable private address is generated - * from random six bytes with the two most significant - * bits cleared. - */ - get_random_bytes(&nrpa, 6); - nrpa.b[5] &= 0x3f; - - /* The non-resolvable private address shall not be - * equal to the public address. - */ - if (bacmp(&hdev->bdaddr, &nrpa)) - break; - } - - *own_addr_type = ADDR_LE_DEV_RANDOM; - set_random_addr(req, &nrpa); - return 0; - } - - /* If forcing static address is in use or there is no public - * address use the static address as random address (but skip - * the HCI command if the current random address is already the - * static one. - * - * In case BR/EDR has been disabled on a dual-mode controller - * and a static address has been configured, then use that - * address instead of the public BR/EDR address. - */ - if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) || - !bacmp(&hdev->bdaddr, BDADDR_ANY) || - (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) && - bacmp(&hdev->static_addr, BDADDR_ANY))) { - *own_addr_type = ADDR_LE_DEV_RANDOM; - if (bacmp(&hdev->static_addr, &hdev->random_addr)) - hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, - &hdev->static_addr); - return 0; - } - - /* Neither privacy nor static address is being used so use a - * public address. - */ - *own_addr_type = ADDR_LE_DEV_PUBLIC; - - return 0; -} - -static bool disconnected_accept_list_entries(struct hci_dev *hdev) -{ - struct bdaddr_list *b; - - list_for_each_entry(b, &hdev->accept_list, list) { - struct hci_conn *conn; - - conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &b->bdaddr); - if (!conn) - return true; - - if (conn->state != BT_CONNECTED && conn->state != BT_CONFIG) - return true; - } - - return false; -} - -void __hci_req_update_scan(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - u8 scan; - - if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) - return; - - if (!hdev_is_powered(hdev)) - return; - - if (mgmt_powering_down(hdev)) - return; - - if (hdev->scanning_paused) - return; - - if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) || - disconnected_accept_list_entries(hdev)) - scan = SCAN_PAGE; - else - scan = SCAN_DISABLED; - - if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) - scan |= SCAN_INQUIRY; - - if (test_bit(HCI_PSCAN, &hdev->flags) == !!(scan & SCAN_PAGE) && - test_bit(HCI_ISCAN, &hdev->flags) == !!(scan & SCAN_INQUIRY)) - return; - - hci_req_add(req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan); -} - -static u8 get_service_classes(struct hci_dev *hdev) -{ - struct bt_uuid *uuid; - u8 val = 0; - - list_for_each_entry(uuid, &hdev->uuids, list) - val |= uuid->svc_hint; - - return val; -} - -void __hci_req_update_class(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - u8 cod[3]; - - bt_dev_dbg(hdev, ""); - - if (!hdev_is_powered(hdev)) - return; - - if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) - return; - - if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) - return; - - cod[0] = hdev->minor_class; - cod[1] = hdev->major_class; - cod[2] = get_service_classes(hdev); - - if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) - cod[1] |= 0x20; - - if (memcmp(cod, hdev->dev_class, 3) == 0) - return; - - hci_req_add(req, HCI_OP_WRITE_CLASS_OF_DEV, sizeof(cod), cod); -} - -void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn, - u8 reason) -{ - switch (conn->state) { - case BT_CONNECTED: - case BT_CONFIG: - if (conn->type == AMP_LINK) { - struct hci_cp_disconn_phy_link cp; - - cp.phy_handle = HCI_PHY_HANDLE(conn->handle); - cp.reason = reason; - hci_req_add(req, HCI_OP_DISCONN_PHY_LINK, sizeof(cp), - &cp); - } else { - struct hci_cp_disconnect dc; - - dc.handle = cpu_to_le16(conn->handle); - dc.reason = reason; - hci_req_add(req, HCI_OP_DISCONNECT, sizeof(dc), &dc); - } - - conn->state = BT_DISCONN; - - break; - case BT_CONNECT: - if (conn->type == LE_LINK) { - if (test_bit(HCI_CONN_SCANNING, &conn->flags)) - break; - hci_req_add(req, HCI_OP_LE_CREATE_CONN_CANCEL, - 0, NULL); - } else if (conn->type == ACL_LINK) { - if (req->hdev->hci_ver < BLUETOOTH_VER_1_2) - break; - hci_req_add(req, HCI_OP_CREATE_CONN_CANCEL, - 6, &conn->dst); - } - break; - case BT_CONNECT2: - if (conn->type == ACL_LINK) { - struct hci_cp_reject_conn_req rej; - - bacpy(&rej.bdaddr, &conn->dst); - rej.reason = reason; - - hci_req_add(req, HCI_OP_REJECT_CONN_REQ, - sizeof(rej), &rej); - } else if (conn->type == SCO_LINK || conn->type == ESCO_LINK) { - struct hci_cp_reject_sync_conn_req rej; - - bacpy(&rej.bdaddr, &conn->dst); - - /* SCO rejection has its own limited set of - * allowed error values (0x0D-0x0F) which isn't - * compatible with most values passed to this - * function. To be safe hard-code one of the - * values that's suitable for SCO. - */ - rej.reason = HCI_ERROR_REJ_LIMITED_RESOURCES; - - hci_req_add(req, HCI_OP_REJECT_SYNC_CONN_REQ, - sizeof(rej), &rej); - } - break; - default: - conn->state = BT_CLOSED; - break; - } -} - -static void abort_conn_complete(struct hci_dev *hdev, u8 status, u16 opcode) -{ - if (status) - bt_dev_dbg(hdev, "Failed to abort connection: status 0x%2.2x", status); -} - -int hci_abort_conn(struct hci_conn *conn, u8 reason) -{ - struct hci_request req; - int err; - - hci_req_init(&req, conn->hdev); - - __hci_abort_conn(&req, conn, reason); - - err = hci_req_run(&req, abort_conn_complete); - if (err && err != -ENODATA) { - bt_dev_err(conn->hdev, "failed to run HCI request: err %d", err); - return err; - } - - return 0; -} - -static int le_scan_disable(struct hci_request *req, unsigned long opt) -{ - hci_req_add_le_scan_disable(req, false); - return 0; -} - -static int bredr_inquiry(struct hci_request *req, unsigned long opt) -{ - u8 length = opt; - const u8 giac[3] = { 0x33, 0x8b, 0x9e }; - const u8 liac[3] = { 0x00, 0x8b, 0x9e }; - struct hci_cp_inquiry cp; - - if (test_bit(HCI_INQUIRY, &req->hdev->flags)) - return 0; - - bt_dev_dbg(req->hdev, ""); - - hci_dev_lock(req->hdev); - hci_inquiry_cache_flush(req->hdev); - hci_dev_unlock(req->hdev); - - memset(&cp, 0, sizeof(cp)); - - if (req->hdev->discovery.limited) - memcpy(&cp.lap, liac, sizeof(cp.lap)); - else - memcpy(&cp.lap, giac, sizeof(cp.lap)); - - cp.length = length; - - hci_req_add(req, HCI_OP_INQUIRY, sizeof(cp), &cp); - - return 0; -} - -static void le_scan_disable_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - le_scan_disable.work); - u8 status; - - bt_dev_dbg(hdev, ""); - - if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) - return; - - cancel_delayed_work(&hdev->le_scan_restart); - - hci_req_sync(hdev, le_scan_disable, 0, HCI_CMD_TIMEOUT, &status); - if (status) { - bt_dev_err(hdev, "failed to disable LE scan: status 0x%02x", - status); - return; - } - - hdev->discovery.scan_start = 0; - - /* If we were running LE only scan, change discovery state. If - * we were running both LE and BR/EDR inquiry simultaneously, - * and BR/EDR inquiry is already finished, stop discovery, - * otherwise BR/EDR inquiry will stop discovery when finished. - * If we will resolve remote device name, do not change - * discovery state. - */ - - if (hdev->discovery.type == DISCOV_TYPE_LE) - goto discov_stopped; - - if (hdev->discovery.type != DISCOV_TYPE_INTERLEAVED) - return; - - if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks)) { - if (!test_bit(HCI_INQUIRY, &hdev->flags) && - hdev->discovery.state != DISCOVERY_RESOLVING) - goto discov_stopped; - - return; - } - - hci_req_sync(hdev, bredr_inquiry, DISCOV_INTERLEAVED_INQUIRY_LEN, - HCI_CMD_TIMEOUT, &status); - if (status) { - bt_dev_err(hdev, "inquiry failed: status 0x%02x", status); - goto discov_stopped; - } - - return; - -discov_stopped: - hci_dev_lock(hdev); - hci_discovery_set_state(hdev, DISCOVERY_STOPPED); - hci_dev_unlock(hdev); -} - -static int le_scan_restart(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - /* If controller is not scanning we are done. */ - if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) - return 0; - - if (hdev->scanning_paused) { - bt_dev_dbg(hdev, "Scanning is paused for suspend"); - return 0; - } - - hci_req_add_le_scan_disable(req, false); - - if (use_ext_scan(hdev)) { - struct hci_cp_le_set_ext_scan_enable ext_enable_cp; - - memset(&ext_enable_cp, 0, sizeof(ext_enable_cp)); - ext_enable_cp.enable = LE_SCAN_ENABLE; - ext_enable_cp.filter_dup = LE_SCAN_FILTER_DUP_ENABLE; - - hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_ENABLE, - sizeof(ext_enable_cp), &ext_enable_cp); - } else { - struct hci_cp_le_set_scan_enable cp; - - memset(&cp, 0, sizeof(cp)); - cp.enable = LE_SCAN_ENABLE; - cp.filter_dup = LE_SCAN_FILTER_DUP_ENABLE; - hci_req_add(req, HCI_OP_LE_SET_SCAN_ENABLE, sizeof(cp), &cp); - } - - return 0; -} - -static void le_scan_restart_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - le_scan_restart.work); - unsigned long timeout, duration, scan_start, now; - u8 status; - - bt_dev_dbg(hdev, ""); - - hci_req_sync(hdev, le_scan_restart, 0, HCI_CMD_TIMEOUT, &status); - if (status) { - bt_dev_err(hdev, "failed to restart LE scan: status %d", - status); - return; - } - - hci_dev_lock(hdev); - - if (!test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) || - !hdev->discovery.scan_start) - goto unlock; - - /* When the scan was started, hdev->le_scan_disable has been queued - * after duration from scan_start. During scan restart this job - * has been canceled, and we need to queue it again after proper - * timeout, to make sure that scan does not run indefinitely. - */ - duration = hdev->discovery.scan_duration; - scan_start = hdev->discovery.scan_start; - now = jiffies; - if (now - scan_start <= duration) { - int elapsed; - - if (now >= scan_start) - elapsed = now - scan_start; - else - elapsed = ULONG_MAX - scan_start + now; - - timeout = duration - elapsed; - } else { - timeout = 0; - } - - queue_delayed_work(hdev->req_workqueue, - &hdev->le_scan_disable, timeout); - -unlock: - hci_dev_unlock(hdev); -} - -bool hci_req_stop_discovery(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct discovery_state *d = &hdev->discovery; - struct hci_cp_remote_name_req_cancel cp; - struct inquiry_entry *e; - bool ret = false; - - bt_dev_dbg(hdev, "state %u", hdev->discovery.state); - - if (d->state == DISCOVERY_FINDING || d->state == DISCOVERY_STOPPING) { - if (test_bit(HCI_INQUIRY, &hdev->flags)) - hci_req_add(req, HCI_OP_INQUIRY_CANCEL, 0, NULL); - - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - cancel_delayed_work(&hdev->le_scan_disable); - cancel_delayed_work(&hdev->le_scan_restart); - hci_req_add_le_scan_disable(req, false); - } - - ret = true; - } else { - /* Passive scanning */ - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - hci_req_add_le_scan_disable(req, false); - ret = true; - } - } - - /* No further actions needed for LE-only discovery */ - if (d->type == DISCOV_TYPE_LE) - return ret; - - if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) { - e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY, - NAME_PENDING); - if (!e) - return ret; - - bacpy(&cp.bdaddr, &e->data.bdaddr); - hci_req_add(req, HCI_OP_REMOTE_NAME_REQ_CANCEL, sizeof(cp), - &cp); - ret = true; - } - - return ret; -} - -static void config_data_path_complete(struct hci_dev *hdev, u8 status, - u16 opcode) -{ - bt_dev_dbg(hdev, "status %u", status); -} - -int hci_req_configure_datapath(struct hci_dev *hdev, struct bt_codec *codec) -{ - struct hci_request req; - int err; - __u8 vnd_len, *vnd_data = NULL; - struct hci_op_configure_data_path *cmd = NULL; - - hci_req_init(&req, hdev); - - err = hdev->get_codec_config_data(hdev, ESCO_LINK, codec, &vnd_len, - &vnd_data); - if (err < 0) - goto error; - - cmd = kzalloc(sizeof(*cmd) + vnd_len, GFP_KERNEL); - if (!cmd) { - err = -ENOMEM; - goto error; - } - - err = hdev->get_data_path_id(hdev, &cmd->data_path_id); - if (err < 0) - goto error; - - cmd->vnd_len = vnd_len; - memcpy(cmd->vnd_data, vnd_data, vnd_len); - - cmd->direction = 0x00; - hci_req_add(&req, HCI_CONFIGURE_DATA_PATH, sizeof(*cmd) + vnd_len, cmd); - - cmd->direction = 0x01; - hci_req_add(&req, HCI_CONFIGURE_DATA_PATH, sizeof(*cmd) + vnd_len, cmd); - - err = hci_req_run(&req, config_data_path_complete); -error: - - kfree(cmd); - kfree(vnd_data); - return err; -} - -void hci_request_setup(struct hci_dev *hdev) -{ - INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable_work); - INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart_work); - INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire); - INIT_DELAYED_WORK(&hdev->interleave_scan, interleave_scan_work); -} - -void hci_request_cancel_all(struct hci_dev *hdev) +void hci_request_cancel_all(struct hci_dev *hdev) { __hci_cmd_sync_cancel(hdev, ENODEV); - cancel_delayed_work_sync(&hdev->le_scan_disable); - cancel_delayed_work_sync(&hdev->le_scan_restart); - - if (hdev->adv_instance_timeout) { - cancel_delayed_work_sync(&hdev->adv_instance_expire); - hdev->adv_instance_timeout = 0; - } - cancel_interleave_scan(hdev); } diff --git a/net/bluetooth/hci_request.h b/net/bluetooth/hci_request.h index 39d001fa3acf70b236e4c6afee2108eed64e27c0..b9c5a98238374cccb4a59fbc3d41c7c465db1afd 100644 --- a/net/bluetooth/hci_request.h +++ b/net/bluetooth/hci_request.h @@ -68,63 +68,10 @@ int __hci_req_sync(struct hci_dev *hdev, int (*func)(struct hci_request *req, struct sk_buff *hci_prepare_cmd(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param); -void __hci_req_write_fast_connectable(struct hci_request *req, bool enable); -void __hci_req_update_name(struct hci_request *req); -void __hci_req_update_eir(struct hci_request *req); - void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn); void hci_req_add_le_passive_scan(struct hci_request *req); void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next); -void hci_req_disable_address_resolution(struct hci_dev *hdev); -void hci_req_reenable_advertising(struct hci_dev *hdev); -void __hci_req_enable_advertising(struct hci_request *req); -void __hci_req_disable_advertising(struct hci_request *req); -void __hci_req_update_adv_data(struct hci_request *req, u8 instance); -int hci_req_update_adv_data(struct hci_dev *hdev, u8 instance); -int hci_req_start_per_adv(struct hci_dev *hdev, u8 instance, u32 flags, - u16 min_interval, u16 max_interval, - u16 sync_interval); -void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance); - -int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance, - bool force); -void hci_req_clear_adv_instance(struct hci_dev *hdev, struct sock *sk, - struct hci_request *req, u8 instance, - bool force); - -int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance); -int __hci_req_setup_per_adv_instance(struct hci_request *req, u8 instance, - u16 min_interval, u16 max_interval); -int __hci_req_start_ext_adv(struct hci_request *req, u8 instance); -int __hci_req_start_per_adv(struct hci_request *req, u8 instance, u32 flags, - u16 min_interval, u16 max_interval, - u16 sync_interval); -int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance); -int __hci_req_enable_per_advertising(struct hci_request *req, u8 instance); -int __hci_req_disable_ext_adv_instance(struct hci_request *req, u8 instance); -int __hci_req_remove_ext_adv_instance(struct hci_request *req, u8 instance); -void __hci_req_clear_ext_adv_sets(struct hci_request *req); -int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, - bool use_rpa, struct adv_info *adv_instance, - u8 *own_addr_type, bdaddr_t *rand_addr); - -void __hci_req_update_class(struct hci_request *req); - -/* Returns true if HCI commands were queued */ -bool hci_req_stop_discovery(struct hci_request *req); - -int hci_req_configure_datapath(struct hci_dev *hdev, struct bt_codec *codec); - -void __hci_req_update_scan(struct hci_request *req); - -int hci_update_random_address(struct hci_request *req, bool require_privacy, - bool use_rpa, u8 *own_addr_type); - -int hci_abort_conn(struct hci_conn *conn, u8 reason); -void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn, - u8 reason); - void hci_request_setup(struct hci_dev *hdev); void hci_request_cancel_all(struct hci_dev *hdev); diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 0d015d4a8e41464a45fbc78b793ba4ec614626bd..06581223238c54b05e90c0a19818754f061a325f 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -887,7 +887,6 @@ static int hci_sock_release(struct socket *sock) */ hci_dev_do_close(hdev); hci_dev_clear_flag(hdev, HCI_USER_CHANNEL); - hci_register_suspend_notifier(hdev); mgmt_index_added(hdev); } @@ -1216,7 +1215,6 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr, } mgmt_index_removed(hdev); - hci_unregister_suspend_notifier(hdev); err = hci_dev_open(hdev->id); if (err) { @@ -1231,7 +1229,6 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr, err = 0; } else { hci_dev_clear_flag(hdev, HCI_USER_CHANNEL); - hci_register_suspend_notifier(hdev); mgmt_index_added(hdev); hci_dev_put(hdev); goto done; @@ -2065,6 +2062,7 @@ static int hci_sock_getsockopt(struct socket *sock, int level, int optname, static void hci_sock_destruct(struct sock *sk) { + mgmt_cleanup(sk); skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); } diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index e6d804b82b674a7ae8d6e8dbd67626d14d6b6fef..76c3107c9f91f4172612fe9bb5145ee458f6c576 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -246,7 +246,7 @@ int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen, skb = __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout, sk); if (IS_ERR(skb)) { bt_dev_err(hdev, "Opcode 0x%4x failed: %ld", opcode, - PTR_ERR(skb)); + PTR_ERR(skb)); return PTR_ERR(skb); } @@ -321,6 +321,307 @@ static void hci_cmd_sync_cancel_work(struct work_struct *work) wake_up_interruptible(&hdev->req_wait_q); } +static int hci_scan_disable_sync(struct hci_dev *hdev); +static int scan_disable_sync(struct hci_dev *hdev, void *data) +{ + return hci_scan_disable_sync(hdev); +} + +static int hci_inquiry_sync(struct hci_dev *hdev, u8 length); +static int interleaved_inquiry_sync(struct hci_dev *hdev, void *data) +{ + return hci_inquiry_sync(hdev, DISCOV_INTERLEAVED_INQUIRY_LEN); +} + +static void le_scan_disable(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, + le_scan_disable.work); + int status; + + bt_dev_dbg(hdev, ""); + hci_dev_lock(hdev); + + if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) + goto _return; + + cancel_delayed_work(&hdev->le_scan_restart); + + status = hci_cmd_sync_queue(hdev, scan_disable_sync, NULL, NULL); + if (status) { + bt_dev_err(hdev, "failed to disable LE scan: %d", status); + goto _return; + } + + hdev->discovery.scan_start = 0; + + /* If we were running LE only scan, change discovery state. If + * we were running both LE and BR/EDR inquiry simultaneously, + * and BR/EDR inquiry is already finished, stop discovery, + * otherwise BR/EDR inquiry will stop discovery when finished. + * If we will resolve remote device name, do not change + * discovery state. + */ + + if (hdev->discovery.type == DISCOV_TYPE_LE) + goto discov_stopped; + + if (hdev->discovery.type != DISCOV_TYPE_INTERLEAVED) + goto _return; + + if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks)) { + if (!test_bit(HCI_INQUIRY, &hdev->flags) && + hdev->discovery.state != DISCOVERY_RESOLVING) + goto discov_stopped; + + goto _return; + } + + status = hci_cmd_sync_queue(hdev, interleaved_inquiry_sync, NULL, NULL); + if (status) { + bt_dev_err(hdev, "inquiry failed: status %d", status); + goto discov_stopped; + } + + goto _return; + +discov_stopped: + hci_discovery_set_state(hdev, DISCOVERY_STOPPED); + +_return: + hci_dev_unlock(hdev); +} + +static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, + u8 filter_dup); +static int hci_le_scan_restart_sync(struct hci_dev *hdev) +{ + /* If controller is not scanning we are done. */ + if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) + return 0; + + if (hdev->scanning_paused) { + bt_dev_dbg(hdev, "Scanning is paused for suspend"); + return 0; + } + + hci_le_set_scan_enable_sync(hdev, LE_SCAN_DISABLE, 0x00); + return hci_le_set_scan_enable_sync(hdev, LE_SCAN_ENABLE, + LE_SCAN_FILTER_DUP_ENABLE); +} + +static int le_scan_restart_sync(struct hci_dev *hdev, void *data) +{ + return hci_le_scan_restart_sync(hdev); +} + +static void le_scan_restart(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, + le_scan_restart.work); + unsigned long timeout, duration, scan_start, now; + int status; + + bt_dev_dbg(hdev, ""); + + hci_dev_lock(hdev); + + status = hci_cmd_sync_queue(hdev, le_scan_restart_sync, NULL, NULL); + if (status) { + bt_dev_err(hdev, "failed to restart LE scan: status %d", + status); + goto unlock; + } + + if (!test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) || + !hdev->discovery.scan_start) + goto unlock; + + /* When the scan was started, hdev->le_scan_disable has been queued + * after duration from scan_start. During scan restart this job + * has been canceled, and we need to queue it again after proper + * timeout, to make sure that scan does not run indefinitely. + */ + duration = hdev->discovery.scan_duration; + scan_start = hdev->discovery.scan_start; + now = jiffies; + if (now - scan_start <= duration) { + int elapsed; + + if (now >= scan_start) + elapsed = now - scan_start; + else + elapsed = ULONG_MAX - scan_start + now; + + timeout = duration - elapsed; + } else { + timeout = 0; + } + + queue_delayed_work(hdev->req_workqueue, + &hdev->le_scan_disable, timeout); + +unlock: + hci_dev_unlock(hdev); +} + +static int reenable_adv_sync(struct hci_dev *hdev, void *data) +{ + bt_dev_dbg(hdev, ""); + + if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) && + list_empty(&hdev->adv_instances)) + return 0; + + if (hdev->cur_adv_instance) { + return hci_schedule_adv_instance_sync(hdev, + hdev->cur_adv_instance, + true); + } else { + if (ext_adv_capable(hdev)) { + hci_start_ext_adv_sync(hdev, 0x00); + } else { + hci_update_adv_data_sync(hdev, 0x00); + hci_update_scan_rsp_data_sync(hdev, 0x00); + hci_enable_advertising_sync(hdev); + } + } + + return 0; +} + +static void reenable_adv(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, + reenable_adv_work); + int status; + + bt_dev_dbg(hdev, ""); + + hci_dev_lock(hdev); + + status = hci_cmd_sync_queue(hdev, reenable_adv_sync, NULL, NULL); + if (status) + bt_dev_err(hdev, "failed to reenable ADV: %d", status); + + hci_dev_unlock(hdev); +} + +static void cancel_adv_timeout(struct hci_dev *hdev) +{ + if (hdev->adv_instance_timeout) { + hdev->adv_instance_timeout = 0; + cancel_delayed_work(&hdev->adv_instance_expire); + } +} + +/* For a single instance: + * - force == true: The instance will be removed even when its remaining + * lifetime is not zero. + * - force == false: the instance will be deactivated but kept stored unless + * the remaining lifetime is zero. + * + * For instance == 0x00: + * - force == true: All instances will be removed regardless of their timeout + * setting. + * - force == false: Only instances that have a timeout will be removed. + */ +int hci_clear_adv_instance_sync(struct hci_dev *hdev, struct sock *sk, + u8 instance, bool force) +{ + struct adv_info *adv_instance, *n, *next_instance = NULL; + int err; + u8 rem_inst; + + /* Cancel any timeout concerning the removed instance(s). */ + if (!instance || hdev->cur_adv_instance == instance) + cancel_adv_timeout(hdev); + + /* Get the next instance to advertise BEFORE we remove + * the current one. This can be the same instance again + * if there is only one instance. + */ + if (instance && hdev->cur_adv_instance == instance) + next_instance = hci_get_next_instance(hdev, instance); + + if (instance == 0x00) { + list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances, + list) { + if (!(force || adv_instance->timeout)) + continue; + + rem_inst = adv_instance->instance; + err = hci_remove_adv_instance(hdev, rem_inst); + if (!err) + mgmt_advertising_removed(sk, hdev, rem_inst); + } + } else { + adv_instance = hci_find_adv_instance(hdev, instance); + + if (force || (adv_instance && adv_instance->timeout && + !adv_instance->remaining_time)) { + /* Don't advertise a removed instance. */ + if (next_instance && + next_instance->instance == instance) + next_instance = NULL; + + err = hci_remove_adv_instance(hdev, instance); + if (!err) + mgmt_advertising_removed(sk, hdev, instance); + } + } + + if (!hdev_is_powered(hdev) || hci_dev_test_flag(hdev, HCI_ADVERTISING)) + return 0; + + if (next_instance && !ext_adv_capable(hdev)) + return hci_schedule_adv_instance_sync(hdev, + next_instance->instance, + false); + + return 0; +} + +static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) +{ + u8 instance = *(u8 *)data; + + kfree(data); + + hci_clear_adv_instance_sync(hdev, NULL, instance, false); + + if (list_empty(&hdev->adv_instances)) + return hci_disable_advertising_sync(hdev); + + return 0; +} + +static void adv_timeout_expire(struct work_struct *work) +{ + u8 *inst_ptr; + struct hci_dev *hdev = container_of(work, struct hci_dev, + adv_instance_expire.work); + + bt_dev_dbg(hdev, ""); + + hci_dev_lock(hdev); + + hdev->adv_instance_timeout = 0; + + if (hdev->cur_adv_instance == 0x00) + goto unlock; + + inst_ptr = kmalloc(1, GFP_KERNEL); + if (!inst_ptr) + goto unlock; + + *inst_ptr = hdev->cur_adv_instance; + hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL); + +unlock: + hci_dev_unlock(hdev); +} + void hci_cmd_sync_init(struct hci_dev *hdev) { INIT_WORK(&hdev->cmd_sync_work, hci_cmd_sync_work); @@ -328,6 +629,10 @@ void hci_cmd_sync_init(struct hci_dev *hdev) mutex_init(&hdev->cmd_sync_work_lock); INIT_WORK(&hdev->cmd_sync_cancel_work, hci_cmd_sync_cancel_work); + INIT_WORK(&hdev->reenable_adv_work, reenable_adv); + INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable); + INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart); + INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire); } void hci_cmd_sync_clear(struct hci_dev *hdev) @@ -335,6 +640,7 @@ void hci_cmd_sync_clear(struct hci_dev *hdev) struct hci_cmd_sync_work_entry *entry, *tmp; cancel_work_sync(&hdev->cmd_sync_work); + cancel_work_sync(&hdev->reenable_adv_work); list_for_each_entry_safe(entry, tmp, &hdev->cmd_sync_work_list, list) { if (entry->destroy) @@ -1333,14 +1639,6 @@ int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason) sizeof(cp), &cp, HCI_CMD_TIMEOUT); } -static void cancel_adv_timeout(struct hci_dev *hdev) -{ - if (hdev->adv_instance_timeout) { - hdev->adv_instance_timeout = 0; - cancel_delayed_work(&hdev->adv_instance_expire); - } -} - static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) { struct { @@ -1492,10 +1790,13 @@ static int hci_clear_adv_sets_sync(struct hci_dev *hdev, struct sock *sk) static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force) { struct adv_info *adv, *n; + int err = 0; if (ext_adv_capable(hdev)) /* Remove all existing sets */ - return hci_clear_adv_sets_sync(hdev, sk); + err = hci_clear_adv_sets_sync(hdev, sk); + if (ext_adv_capable(hdev)) + return err; /* This is safe as long as there is no command send while the lock is * held. @@ -1523,11 +1824,13 @@ static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force) static int hci_remove_adv_sync(struct hci_dev *hdev, u8 instance, struct sock *sk) { - int err; + int err = 0; /* If we use extended advertising, instance has to be removed first. */ if (ext_adv_capable(hdev)) - return hci_remove_ext_adv_instance_sync(hdev, instance, sk); + err = hci_remove_ext_adv_instance_sync(hdev, instance, sk); + if (ext_adv_capable(hdev)) + return err; /* This is safe as long as there is no command send while the lock is * held. @@ -1626,13 +1929,16 @@ int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type) int hci_disable_advertising_sync(struct hci_dev *hdev) { u8 enable = 0x00; + int err = 0; /* If controller is not advertising we are done. */ if (!hci_dev_test_flag(hdev, HCI_LE_ADV)) return 0; if (ext_adv_capable(hdev)) - return hci_disable_ext_adv_instance_sync(hdev, 0x00); + err = hci_disable_ext_adv_instance_sync(hdev, 0x00); + if (ext_adv_capable(hdev)) + return err; return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable, HCI_CMD_TIMEOUT); @@ -1645,7 +1951,11 @@ static int hci_le_set_ext_scan_enable_sync(struct hci_dev *hdev, u8 val, memset(&cp, 0, sizeof(cp)); cp.enable = val; - cp.filter_dup = filter_dup; + + if (hci_dev_test_flag(hdev, HCI_MESH)) + cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE; + else + cp.filter_dup = filter_dup; return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_ENABLE, sizeof(cp), &cp, HCI_CMD_TIMEOUT); @@ -1661,7 +1971,11 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, memset(&cp, 0, sizeof(cp)); cp.enable = val; - cp.filter_dup = filter_dup; + + if (val && hci_dev_test_flag(hdev, HCI_MESH)) + cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE; + else + cp.filter_dup = filter_dup; return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_ENABLE, sizeof(cp), &cp, HCI_CMD_TIMEOUT); @@ -2300,6 +2614,7 @@ static int hci_passive_scan_sync(struct hci_dev *hdev) u8 own_addr_type; u8 filter_policy; u16 window, interval; + u8 filter_dups = LE_SCAN_FILTER_DUP_ENABLE; int err; if (hdev->scanning_paused) { @@ -2362,11 +2677,16 @@ static int hci_passive_scan_sync(struct hci_dev *hdev) interval = hdev->le_scan_interval; } + /* Disable all filtering for Mesh */ + if (hci_dev_test_flag(hdev, HCI_MESH)) { + filter_policy = 0; + filter_dups = LE_SCAN_FILTER_DUP_DISABLE; + } + bt_dev_dbg(hdev, "LE passive scan with acceptlist = %d", filter_policy); return hci_start_scan_sync(hdev, LE_SCAN_PASSIVE, interval, window, - own_addr_type, filter_policy, - LE_SCAN_FILTER_DUP_ENABLE); + own_addr_type, filter_policy, filter_dups); } /* This function controls the passive scanning based on hdev->pend_le_conns @@ -2416,7 +2736,8 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev) bt_dev_dbg(hdev, "ADV monitoring is %s", hci_is_adv_monitoring(hdev) ? "on" : "off"); - if (list_empty(&hdev->pend_le_conns) && + if (!hci_dev_test_flag(hdev, HCI_MESH) && + list_empty(&hdev->pend_le_conns) && list_empty(&hdev->pend_le_reports) && !hci_is_adv_monitoring(hdev) && !hci_dev_test_flag(hdev, HCI_PA_SYNC)) { @@ -3018,12 +3339,6 @@ static const struct hci_init_stage amp_init2[] = { /* Read Buffer Size (ACL mtu, max pkt, etc.) */ static int hci_read_buffer_size_sync(struct hci_dev *hdev) { - /* Use Read LE Buffer Size V2 if supported */ - if (hdev->commands[41] & 0x20) - return __hci_cmd_sync_status(hdev, - HCI_OP_LE_READ_BUFFER_SIZE_V2, - 0, NULL, HCI_CMD_TIMEOUT); - return __hci_cmd_sync_status(hdev, HCI_OP_READ_BUFFER_SIZE, 0, NULL, HCI_CMD_TIMEOUT); } @@ -3237,6 +3552,12 @@ static const struct hci_init_stage hci_init2[] = { /* Read LE Buffer Size */ static int hci_le_read_buffer_size_sync(struct hci_dev *hdev) { + /* Use Read LE Buffer Size V2 if supported */ + if (hdev->commands[41] & 0x20) + return __hci_cmd_sync_status(hdev, + HCI_OP_LE_READ_BUFFER_SIZE_V2, + 0, NULL, HCI_CMD_TIMEOUT); + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_BUFFER_SIZE, 0, NULL, HCI_CMD_TIMEOUT); } @@ -4355,6 +4676,7 @@ int hci_dev_open_sync(struct hci_dev *hdev) hci_dev_test_flag(hdev, HCI_MGMT) && hdev->dev_type == HCI_PRIMARY) { ret = hci_powered_update_sync(hdev); + mgmt_power_on(hdev, ret); } } else { /* Init failed, cleanup */ @@ -4406,6 +4728,31 @@ static void hci_pend_le_actions_clear(struct hci_dev *hdev) BT_DBG("All LE pending actions cleared"); } +static int hci_dev_shutdown(struct hci_dev *hdev) +{ + int err = 0; + /* Similar to how we first do setup and then set the exclusive access + * bit for userspace, we must first unset userchannel and then clean up. + * Otherwise, the kernel can't properly use the hci channel to clean up + * the controller (some shutdown routines require sending additional + * commands to the controller for example). + */ + bool was_userchannel = + hci_dev_test_and_clear_flag(hdev, HCI_USER_CHANNEL); + + if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) && + test_bit(HCI_UP, &hdev->flags)) { + /* Execute vendor specific shutdown routine */ + if (hdev->shutdown) + err = hdev->shutdown(hdev); + } + + if (was_userchannel) + hci_dev_set_flag(hdev, HCI_USER_CHANNEL); + + return err; +} + int hci_dev_close_sync(struct hci_dev *hdev) { bool auto_off; @@ -4415,17 +4762,18 @@ int hci_dev_close_sync(struct hci_dev *hdev) cancel_delayed_work(&hdev->power_off); cancel_delayed_work(&hdev->ncmd_timer); + cancel_delayed_work(&hdev->le_scan_disable); + cancel_delayed_work(&hdev->le_scan_restart); hci_request_cancel_all(hdev); - if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - test_bit(HCI_UP, &hdev->flags)) { - /* Execute vendor specific shutdown routine */ - if (hdev->shutdown) - err = hdev->shutdown(hdev); + if (hdev->adv_instance_timeout) { + cancel_delayed_work_sync(&hdev->adv_instance_expire); + hdev->adv_instance_timeout = 0; } + err = hci_dev_shutdown(hdev); + if (!test_and_clear_bit(HCI_UP, &hdev->flags)) { cancel_delayed_work_sync(&hdev->cmd_timer); return err; @@ -4773,9 +5121,11 @@ int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, u8 reason) /* Cleanup hci_conn object if it cannot be cancelled as it * likelly means the controller and host stack are out of sync. */ - if (err) + if (err) { + hci_dev_lock(hdev); hci_conn_failed(conn, err); - + hci_dev_unlock(hdev); + } return err; case BT_CONNECT2: return hci_reject_conn_sync(hdev, conn, reason); @@ -5021,7 +5371,7 @@ static int hci_active_scan_sync(struct hci_dev *hdev, uint16_t interval) /* Pause advertising since active scanning disables address resolution * which advertising depend on in order to generate its RPAs. */ - if (use_ll_privacy(hdev)) { + if (use_ll_privacy(hdev) && hci_dev_test_flag(hdev, HCI_PRIVACY)) { err = hci_pause_advertising_sync(hdev); if (err) { bt_dev_err(hdev, "pause advertising failed: %d", err); @@ -5288,17 +5638,21 @@ int hci_suspend_sync(struct hci_dev *hdev) /* Prevent disconnects from causing scanning to be re-enabled */ hci_pause_scan_sync(hdev); - /* Soft disconnect everything (power off) */ - err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF); - if (err) { - /* Set state to BT_RUNNING so resume doesn't notify */ - hdev->suspend_state = BT_RUNNING; - hci_resume_sync(hdev); - return err; - } + if (hci_conn_count(hdev)) { + /* Soft disconnect everything (power off) */ + err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF); + if (err) { + /* Set state to BT_RUNNING so resume doesn't notify */ + hdev->suspend_state = BT_RUNNING; + hci_resume_sync(hdev); + return err; + } - /* Update event mask so only the allowed event can wakeup the host */ - hci_set_event_mask_sync(hdev); + /* Update event mask so only the allowed event can wakeup the + * host. + */ + hci_set_event_mask_sync(hdev); + } /* Only configure accept list if disconnect succeeded and wake * isn't being prevented. @@ -5731,3 +6085,96 @@ int hci_le_pa_terminate_sync(struct hci_dev *hdev, u16 handle) return __hci_cmd_sync_status(hdev, HCI_OP_LE_PA_TERM_SYNC, sizeof(cp), &cp, HCI_CMD_TIMEOUT); } + +int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, + bool use_rpa, struct adv_info *adv_instance, + u8 *own_addr_type, bdaddr_t *rand_addr) +{ + int err; + + bacpy(rand_addr, BDADDR_ANY); + + /* If privacy is enabled use a resolvable private address. If + * current RPA has expired then generate a new one. + */ + if (use_rpa) { + /* If Controller supports LL Privacy use own address type is + * 0x03 + */ + if (use_ll_privacy(hdev)) + *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; + else + *own_addr_type = ADDR_LE_DEV_RANDOM; + + if (adv_instance) { + if (adv_rpa_valid(adv_instance)) + return 0; + } else { + if (rpa_valid(hdev)) + return 0; + } + + err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa); + if (err < 0) { + bt_dev_err(hdev, "failed to generate new RPA"); + return err; + } + + bacpy(rand_addr, &hdev->rpa); + + return 0; + } + + /* In case of required privacy without resolvable private address, + * use an non-resolvable private address. This is useful for + * non-connectable advertising. + */ + if (require_privacy) { + bdaddr_t nrpa; + + while (true) { + /* The non-resolvable private address is generated + * from random six bytes with the two most significant + * bits cleared. + */ + get_random_bytes(&nrpa, 6); + nrpa.b[5] &= 0x3f; + + /* The non-resolvable private address shall not be + * equal to the public address. + */ + if (bacmp(&hdev->bdaddr, &nrpa)) + break; + } + + *own_addr_type = ADDR_LE_DEV_RANDOM; + bacpy(rand_addr, &nrpa); + + return 0; + } + + /* No privacy so use a public address. */ + *own_addr_type = ADDR_LE_DEV_PUBLIC; + + return 0; +} + +static int _update_adv_data_sync(struct hci_dev *hdev, void *data) +{ + u8 instance = *(u8 *)data; + + kfree(data); + + return hci_update_adv_data_sync(hdev, instance); +} + +int hci_update_adv_data(struct hci_dev *hdev, u8 instance) +{ + u8 *inst_ptr = kmalloc(1, GFP_KERNEL); + + if (!inst_ptr) + return -ENOMEM; + + *inst_ptr = instance; + return hci_cmd_sync_queue(hdev, _update_adv_data_sync, inst_ptr, NULL); +} diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c index 4e3e0451b08c12925dadba7412c1cc3dcf9e81d1..08542dfc2dc53b21082a3a9fe34e4c4fded87c57 100644 --- a/net/bluetooth/hci_sysfs.c +++ b/net/bluetooth/hci_sysfs.c @@ -48,6 +48,9 @@ void hci_conn_add_sysfs(struct hci_conn *conn) BT_DBG("conn %p", conn); + if (device_is_registered(&conn->dev)) + return; + dev_set_name(&conn->dev, "%s:%d", hdev->name, conn->handle); if (device_add(&conn->dev) < 0) { diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 5940744a8cd8b682ed6d4e013778dbba1d17ec65..cc20e706c63914f1879e8eb6fb221f711e8c34b1 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -83,14 +83,14 @@ static void hidp_copy_session(struct hidp_session *session, struct hidp_conninfo ci->product = session->input->id.product; ci->version = session->input->id.version; if (session->input->name) - strlcpy(ci->name, session->input->name, 128); + strscpy(ci->name, session->input->name, 128); else - strlcpy(ci->name, "HID Boot Device", 128); + strscpy(ci->name, "HID Boot Device", 128); } else if (session->hid) { ci->vendor = session->hid->vendor; ci->product = session->hid->product; ci->version = session->hid->version; - strlcpy(ci->name, session->hid->name, 128); + strscpy(ci->name, session->hid->name, 128); } } diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index ced8ad4fed4fe04ed433068c175eeec3d49b8492..613039ba5dbf54b05f2cee49e1588c1816fddd29 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -1309,7 +1309,7 @@ static int iso_sock_shutdown(struct socket *sock, int how) struct sock *sk = sock->sk; int err = 0; - BT_DBG("sock %p, sk %p", sock, sk); + BT_DBG("sock %p, sk %p, how %d", sock, sk, how); if (!sk) return 0; @@ -1317,17 +1317,32 @@ static int iso_sock_shutdown(struct socket *sock, int how) sock_hold(sk); lock_sock(sk); - if (!sk->sk_shutdown) { - sk->sk_shutdown = SHUTDOWN_MASK; - iso_sock_clear_timer(sk); - __iso_sock_close(sk); - - if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && - !(current->flags & PF_EXITING)) - err = bt_sock_wait_state(sk, BT_CLOSED, - sk->sk_lingertime); + switch (how) { + case SHUT_RD: + if (sk->sk_shutdown & RCV_SHUTDOWN) + goto unlock; + sk->sk_shutdown |= RCV_SHUTDOWN; + break; + case SHUT_WR: + if (sk->sk_shutdown & SEND_SHUTDOWN) + goto unlock; + sk->sk_shutdown |= SEND_SHUTDOWN; + break; + case SHUT_RDWR: + if (sk->sk_shutdown & SHUTDOWN_MASK) + goto unlock; + sk->sk_shutdown |= SHUTDOWN_MASK; + break; } + iso_sock_clear_timer(sk); + __iso_sock_close(sk); + + if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && + !(current->flags & PF_EXITING)) + err = bt_sock_wait_state(sk, BT_CLOSED, sk->sk_lingertime); + +unlock: release_sock(sk); sock_put(sk); diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index cbe0cae73434f7a1631940a58d1b1a5036c6f830..1f34b82ca0ec93669c9b4e2478756e0634aaecf5 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -61,6 +61,9 @@ static void l2cap_send_disconn_req(struct l2cap_chan *chan, int err); static void l2cap_tx(struct l2cap_chan *chan, struct l2cap_ctrl *control, struct sk_buff_head *skbs, u8 event); +static void l2cap_retrans_timeout(struct work_struct *work); +static void l2cap_monitor_timeout(struct work_struct *work); +static void l2cap_ack_timeout(struct work_struct *work); static inline u8 bdaddr_type(u8 link_type, u8 bdaddr_type) { @@ -476,6 +479,9 @@ struct l2cap_chan *l2cap_chan_create(void) write_unlock(&chan_list_lock); INIT_DELAYED_WORK(&chan->chan_timer, l2cap_chan_timeout); + INIT_DELAYED_WORK(&chan->retrans_timer, l2cap_retrans_timeout); + INIT_DELAYED_WORK(&chan->monitor_timer, l2cap_monitor_timeout); + INIT_DELAYED_WORK(&chan->ack_timer, l2cap_ack_timeout); chan->state = BT_OPEN; @@ -1992,11 +1998,11 @@ static struct l2cap_chan *l2cap_global_chan_by_psm(int state, __le16 psm, src_match = !bacmp(&c->src, src); dst_match = !bacmp(&c->dst, dst); if (src_match && dst_match) { - c = l2cap_chan_hold_unless_zero(c); - if (c) { - read_unlock(&chan_list_lock); - return c; - } + if (!l2cap_chan_hold_unless_zero(c)) + continue; + + read_unlock(&chan_list_lock); + return c; } /* Closest match */ @@ -3320,10 +3326,6 @@ int l2cap_ertm_init(struct l2cap_chan *chan) chan->rx_state = L2CAP_RX_STATE_RECV; chan->tx_state = L2CAP_TX_STATE_XMIT; - INIT_DELAYED_WORK(&chan->retrans_timer, l2cap_retrans_timeout); - INIT_DELAYED_WORK(&chan->monitor_timer, l2cap_monitor_timeout); - INIT_DELAYED_WORK(&chan->ack_timer, l2cap_ack_timeout); - skb_queue_head_init(&chan->srej_q); err = l2cap_seq_list_init(&chan->srej_list, chan->tx_win); @@ -4307,6 +4309,12 @@ static int l2cap_connect_create_rsp(struct l2cap_conn *conn, } } + chan = l2cap_chan_hold_unless_zero(chan); + if (!chan) { + err = -EBADSLT; + goto unlock; + } + err = 0; l2cap_chan_lock(chan); @@ -4336,6 +4344,7 @@ static int l2cap_connect_create_rsp(struct l2cap_conn *conn, } l2cap_chan_unlock(chan); + l2cap_chan_put(chan); unlock: mutex_unlock(&conn->chan_lock); diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 6e31023b84f5fc7d5fd6ae5587672940691303cc..a92e7e485febab36fb6de3525710bdea79025011 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -129,6 +129,10 @@ static const u16 mgmt_commands[] = { MGMT_OP_ADD_EXT_ADV_PARAMS, MGMT_OP_ADD_EXT_ADV_DATA, MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI, + MGMT_OP_SET_MESH_RECEIVER, + MGMT_OP_MESH_READ_FEATURES, + MGMT_OP_MESH_SEND, + MGMT_OP_MESH_SEND_CANCEL, }; static const u16 mgmt_events[] = { @@ -1048,9 +1052,66 @@ static void discov_off(struct work_struct *work) hci_dev_unlock(hdev); } +static int send_settings_rsp(struct sock *sk, u16 opcode, struct hci_dev *hdev); + +static void mesh_send_complete(struct hci_dev *hdev, + struct mgmt_mesh_tx *mesh_tx, bool silent) +{ + u8 handle = mesh_tx->handle; + + if (!silent) + mgmt_event(MGMT_EV_MESH_PACKET_CMPLT, hdev, &handle, + sizeof(handle), NULL); + + mgmt_mesh_remove(mesh_tx); +} + +static int mesh_send_done_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_mesh_tx *mesh_tx; + + hci_dev_clear_flag(hdev, HCI_MESH_SENDING); + hci_disable_advertising_sync(hdev); + mesh_tx = mgmt_mesh_next(hdev, NULL); + + if (mesh_tx) + mesh_send_complete(hdev, mesh_tx, false); + + return 0; +} + +static int mesh_send_sync(struct hci_dev *hdev, void *data); +static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err); +static void mesh_next(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_mesh_tx *mesh_tx = mgmt_mesh_next(hdev, NULL); + + if (!mesh_tx) + return; + + err = hci_cmd_sync_queue(hdev, mesh_send_sync, mesh_tx, + mesh_send_start_complete); + + if (err < 0) + mesh_send_complete(hdev, mesh_tx, false); + else + hci_dev_set_flag(hdev, HCI_MESH_SENDING); +} + +static void mesh_send_done(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, + mesh_send_done.work); + + if (!hci_dev_test_flag(hdev, HCI_MESH_SENDING)) + return; + + hci_cmd_sync_queue(hdev, mesh_send_done_sync, NULL, mesh_next); +} + static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev) { - if (hci_dev_test_and_set_flag(hdev, HCI_MGMT)) + if (hci_dev_test_flag(hdev, HCI_MGMT)) return; BT_INFO("MGMT ver %d.%d", MGMT_VERSION, MGMT_REVISION); @@ -1058,6 +1119,7 @@ static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev) INIT_DELAYED_WORK(&hdev->discov_off, discov_off); INIT_DELAYED_WORK(&hdev->service_cache, service_cache_off); INIT_DELAYED_WORK(&hdev->rpa_expired, rpa_expired); + INIT_DELAYED_WORK(&hdev->mesh_send_done, mesh_send_done); /* Non-mgmt controlled devices get this bit set * implicitly so that pairing works for them, however @@ -1065,6 +1127,8 @@ static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev) * it */ hci_dev_clear_flag(hdev, HCI_BONDABLE); + + hci_dev_set_flag(hdev, HCI_MGMT); } static int read_controller_info(struct sock *sk, struct hci_dev *hdev, @@ -2058,6 +2122,8 @@ static int set_le_sync(struct hci_dev *hdev, void *data) int err; if (!val) { + hci_clear_adv_instance_sync(hdev, NULL, 0x00, true); + if (hci_dev_test_flag(hdev, HCI_LE_ADV)) hci_disable_advertising_sync(hdev); @@ -2092,6 +2158,317 @@ static int set_le_sync(struct hci_dev *hdev, void *data) return err; } +static void set_mesh_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + u8 status = mgmt_status(err); + struct sock *sk = cmd->sk; + + if (status) { + mgmt_pending_foreach(MGMT_OP_SET_MESH_RECEIVER, hdev, + cmd_status_rsp, &status); + return; + } + + mgmt_pending_remove(cmd); + mgmt_cmd_complete(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, 0, NULL, 0); +} + +static int set_mesh_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_set_mesh *cp = cmd->param; + size_t len = cmd->param_len; + + memset(hdev->mesh_ad_types, 0, sizeof(hdev->mesh_ad_types)); + + if (cp->enable) + hci_dev_set_flag(hdev, HCI_MESH); + else + hci_dev_clear_flag(hdev, HCI_MESH); + + len -= sizeof(*cp); + + /* If filters don't fit, forward all adv pkts */ + if (len <= sizeof(hdev->mesh_ad_types)) + memcpy(hdev->mesh_ad_types, cp->ad_types, len); + + hci_update_passive_scan_sync(hdev); + return 0; +} + +static int set_mesh(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) +{ + struct mgmt_cp_set_mesh *cp = data; + struct mgmt_pending_cmd *cmd; + int err = 0; + + bt_dev_dbg(hdev, "sock %p", sk); + + if (!lmp_le_capable(hdev) || + !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_NOT_SUPPORTED); + + if (cp->enable != 0x00 && cp->enable != 0x01) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_INVALID_PARAMS); + + hci_dev_lock(hdev); + + cmd = mgmt_pending_add(sk, MGMT_OP_SET_MESH_RECEIVER, hdev, data, len); + if (!cmd) + err = -ENOMEM; + else + err = hci_cmd_sync_queue(hdev, set_mesh_sync, cmd, + set_mesh_complete); + + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, + MGMT_STATUS_FAILED); + + if (cmd) + mgmt_pending_remove(cmd); + } + + hci_dev_unlock(hdev); + return err; +} + +static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_mesh_tx *mesh_tx = data; + struct mgmt_cp_mesh_send *send = (void *)mesh_tx->param; + unsigned long mesh_send_interval; + u8 mgmt_err = mgmt_status(err); + + /* Report any errors here, but don't report completion */ + + if (mgmt_err) { + hci_dev_clear_flag(hdev, HCI_MESH_SENDING); + /* Send Complete Error Code for handle */ + mesh_send_complete(hdev, mesh_tx, false); + return; + } + + mesh_send_interval = msecs_to_jiffies((send->cnt) * 25); + queue_delayed_work(hdev->req_workqueue, &hdev->mesh_send_done, + mesh_send_interval); +} + +static int mesh_send_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_mesh_tx *mesh_tx = data; + struct mgmt_cp_mesh_send *send = (void *)mesh_tx->param; + struct adv_info *adv, *next_instance; + u8 instance = hdev->le_num_of_adv_sets + 1; + u16 timeout, duration; + int err = 0; + + if (hdev->le_num_of_adv_sets <= hdev->adv_instance_cnt) + return MGMT_STATUS_BUSY; + + timeout = 1000; + duration = send->cnt * INTERVAL_TO_MS(hdev->le_adv_max_interval); + adv = hci_add_adv_instance(hdev, instance, 0, + send->adv_data_len, send->adv_data, + 0, NULL, + timeout, duration, + HCI_ADV_TX_POWER_NO_PREFERENCE, + hdev->le_adv_min_interval, + hdev->le_adv_max_interval, + mesh_tx->handle); + + if (!IS_ERR(adv)) + mesh_tx->instance = instance; + else + err = PTR_ERR(adv); + + if (hdev->cur_adv_instance == instance) { + /* If the currently advertised instance is being changed then + * cancel the current advertising and schedule the next + * instance. If there is only one instance then the overridden + * advertising data will be visible right away. + */ + cancel_adv_timeout(hdev); + + next_instance = hci_get_next_instance(hdev, instance); + if (next_instance) + instance = next_instance->instance; + else + instance = 0; + } else if (hdev->adv_instance_timeout) { + /* Immediately advertise the new instance if no other, or + * let it go naturally from queue if ADV is already happening + */ + instance = 0; + } + + if (instance) + return hci_schedule_adv_instance_sync(hdev, instance, true); + + return err; +} + +static void send_count(struct mgmt_mesh_tx *mesh_tx, void *data) +{ + struct mgmt_rp_mesh_read_features *rp = data; + + if (rp->used_handles >= rp->max_handles) + return; + + rp->handles[rp->used_handles++] = mesh_tx->handle; +} + +static int mesh_features(struct sock *sk, struct hci_dev *hdev, + void *data, u16 len) +{ + struct mgmt_rp_mesh_read_features rp; + + if (!lmp_le_capable(hdev) || + !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_READ_FEATURES, + MGMT_STATUS_NOT_SUPPORTED); + + memset(&rp, 0, sizeof(rp)); + rp.index = cpu_to_le16(hdev->id); + if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + rp.max_handles = MESH_HANDLES_MAX; + + hci_dev_lock(hdev); + + if (rp.max_handles) + mgmt_mesh_foreach(hdev, send_count, &rp, sk); + + mgmt_cmd_complete(sk, hdev->id, MGMT_OP_MESH_READ_FEATURES, 0, &rp, + rp.used_handles + sizeof(rp) - MESH_HANDLES_MAX); + + hci_dev_unlock(hdev); + return 0; +} + +static int send_cancel(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_mesh_send_cancel *cancel = (void *)cmd->param; + struct mgmt_mesh_tx *mesh_tx; + + if (!cancel->handle) { + do { + mesh_tx = mgmt_mesh_next(hdev, cmd->sk); + + if (mesh_tx) + mesh_send_complete(hdev, mesh_tx, false); + } while (mesh_tx); + } else { + mesh_tx = mgmt_mesh_find(hdev, cancel->handle); + + if (mesh_tx && mesh_tx->sk == cmd->sk) + mesh_send_complete(hdev, mesh_tx, false); + } + + mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, + 0, NULL, 0); + mgmt_pending_free(cmd); + + return 0; +} + +static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, + void *data, u16 len) +{ + struct mgmt_pending_cmd *cmd; + int err; + + if (!lmp_le_capable(hdev) || + !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, + MGMT_STATUS_NOT_SUPPORTED); + + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, + MGMT_STATUS_REJECTED); + + hci_dev_lock(hdev); + cmd = mgmt_pending_new(sk, MGMT_OP_MESH_SEND_CANCEL, hdev, data, len); + if (!cmd) + err = -ENOMEM; + else + err = hci_cmd_sync_queue(hdev, send_cancel, cmd, NULL); + + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, + MGMT_STATUS_FAILED); + + if (cmd) + mgmt_pending_free(cmd); + } + + hci_dev_unlock(hdev); + return err; +} + +static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) +{ + struct mgmt_mesh_tx *mesh_tx; + struct mgmt_cp_mesh_send *send = data; + struct mgmt_rp_mesh_read_features rp; + bool sending; + int err = 0; + + if (!lmp_le_capable(hdev) || + !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_NOT_SUPPORTED); + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) || + len <= MGMT_MESH_SEND_SIZE || + len > (MGMT_MESH_SEND_SIZE + 31)) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_REJECTED); + + hci_dev_lock(hdev); + + memset(&rp, 0, sizeof(rp)); + rp.max_handles = MESH_HANDLES_MAX; + + mgmt_mesh_foreach(hdev, send_count, &rp, sk); + + if (rp.max_handles <= rp.used_handles) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_BUSY); + goto done; + } + + sending = hci_dev_test_flag(hdev, HCI_MESH_SENDING); + mesh_tx = mgmt_mesh_add(sk, hdev, send, len); + + if (!mesh_tx) + err = -ENOMEM; + else if (!sending) + err = hci_cmd_sync_queue(hdev, mesh_send_sync, mesh_tx, + mesh_send_start_complete); + + if (err < 0) { + bt_dev_err(hdev, "Send Mesh Failed %d", err); + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND, + MGMT_STATUS_FAILED); + + if (mesh_tx) { + if (sending) + mgmt_mesh_remove(mesh_tx); + } + } else { + hci_dev_set_flag(hdev, HCI_MESH_SENDING); + + mgmt_cmd_complete(sk, hdev->id, MGMT_OP_MESH_SEND, 0, + &mesh_tx->handle, 1); + } + +done: + hci_dev_unlock(hdev); + return err; +} + static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_mode *cp = data; @@ -2131,9 +2508,6 @@ static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) val = !!cp->val; enabled = lmp_host_le_capable(hdev); - if (!val) - hci_req_clear_adv_instance(hdev, NULL, NULL, 0x00, true); - if (!hdev_is_powered(hdev) || val == enabled) { bool changed = false; @@ -3186,6 +3560,18 @@ unlock: return err; } +static int abort_conn_sync(struct hci_dev *hdev, void *data) +{ + struct hci_conn *conn; + u16 handle = PTR_ERR(data); + + conn = hci_conn_hash_lookup_handle(hdev, handle); + if (!conn) + return 0; + + return hci_abort_conn_sync(hdev, conn, HCI_ERROR_REMOTE_USER_TERM); +} + static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -3236,7 +3622,8 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, le_addr_type(addr->type)); if (conn->conn_reason == CONN_REASON_PAIR_DEVICE) - hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_cmd_sync_queue(hdev, abort_conn_sync, ERR_PTR(conn->handle), + NULL); unlock: hci_dev_unlock(hdev); @@ -3991,17 +4378,28 @@ static const u8 iso_socket_uuid[16] = { 0x6a, 0x49, 0xe0, 0x05, 0x88, 0xf1, 0xba, 0x6f, }; +/* 2ce463d7-7a03-4d8d-bf05-5f24e8f36e76 */ +static const u8 mgmt_mesh_uuid[16] = { + 0x76, 0x6e, 0xf3, 0xe8, 0x24, 0x5f, 0x05, 0xbf, + 0x8d, 0x4d, 0x03, 0x7a, 0xd7, 0x63, 0xe4, 0x2c, +}; + static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { - char buf[122]; /* Enough space for 6 features: 2 + 20 * 6 */ - struct mgmt_rp_read_exp_features_info *rp = (void *)buf; + struct mgmt_rp_read_exp_features_info *rp; + size_t len; u16 idx = 0; u32 flags; + int status; bt_dev_dbg(hdev, "sock %p", sk); - memset(&buf, 0, sizeof(buf)); + /* Enough space for 7 features */ + len = sizeof(*rp) + (sizeof(rp->features[0]) * 7); + rp = kzalloc(len, GFP_KERNEL); + if (!rp) + return -ENOMEM; #ifdef CONFIG_BT_FEATURE_DEBUG if (!hdev) { @@ -4065,6 +4463,17 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, idx++; } + if (hdev && lmp_le_capable(hdev)) { + if (hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL)) + flags = BIT(0); + else + flags = 0; + + memcpy(rp->features[idx].uuid, mgmt_mesh_uuid, 16); + rp->features[idx].flags = cpu_to_le32(flags); + idx++; + } + rp->feature_count = cpu_to_le16(idx); /* After reading the experimental features information, enable @@ -4072,9 +4481,12 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, */ hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS); - return mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE, - MGMT_OP_READ_EXP_FEATURES_INFO, - 0, rp, sizeof(*rp) + (20 * idx)); + status = mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE, + MGMT_OP_READ_EXP_FEATURES_INFO, + 0, rp, sizeof(*rp) + (20 * idx)); + + kfree(rp); + return status; } static int exp_ll_privacy_feature_changed(bool enabled, struct hci_dev *hdev, @@ -4202,6 +4614,63 @@ static int set_debug_func(struct sock *sk, struct hci_dev *hdev, } #endif +static int set_mgmt_mesh_func(struct sock *sk, struct hci_dev *hdev, + struct mgmt_cp_set_exp_feature *cp, u16 data_len) +{ + struct mgmt_rp_set_exp_feature rp; + bool val, changed; + int err; + + /* Command requires to use the controller index */ + if (!hdev) + return mgmt_cmd_status(sk, MGMT_INDEX_NONE, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_INDEX); + + /* Changes can only be made when controller is powered down */ + if (hdev_is_powered(hdev)) + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_REJECTED); + + /* Parameters are limited to a single octet */ + if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1) + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_PARAMS); + + /* Only boolean on/off is supported */ + if (cp->param[0] != 0x00 && cp->param[0] != 0x01) + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_EXP_FEATURE, + MGMT_STATUS_INVALID_PARAMS); + + val = !!cp->param[0]; + + if (val) { + changed = !hci_dev_test_and_set_flag(hdev, + HCI_MESH_EXPERIMENTAL); + } else { + hci_dev_clear_flag(hdev, HCI_MESH); + changed = hci_dev_test_and_clear_flag(hdev, + HCI_MESH_EXPERIMENTAL); + } + + memcpy(rp.uuid, mgmt_mesh_uuid, 16); + rp.flags = cpu_to_le32(val ? BIT(0) : 0); + + hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS); + + err = mgmt_cmd_complete(sk, hdev->id, + MGMT_OP_SET_EXP_FEATURE, 0, + &rp, sizeof(rp)); + + if (changed) + exp_feature_changed(hdev, mgmt_mesh_uuid, val, sk); + + return err; +} + static int set_rpa_resolution_func(struct sock *sk, struct hci_dev *hdev, struct mgmt_cp_set_exp_feature *cp, u16 data_len) @@ -4517,6 +4986,7 @@ static const struct mgmt_exp_feature { #ifdef CONFIG_BT_FEATURE_DEBUG EXP_FEAT(debug_uuid, set_debug_func), #endif + EXP_FEAT(mgmt_mesh_uuid, set_mgmt_mesh_func), EXP_FEAT(rpa_resolution_uuid, set_rpa_resolution_func), EXP_FEAT(quality_report_uuid, set_quality_report_func), EXP_FEAT(offload_codecs_uuid, set_offload_codec_func), @@ -4547,6 +5017,22 @@ static int set_exp_feature(struct sock *sk, struct hci_dev *hdev, MGMT_STATUS_NOT_SUPPORTED); } +static u32 get_params_flags(struct hci_dev *hdev, + struct hci_conn_params *params) +{ + u32 flags = hdev->conn_flags; + + /* Devices using RPAs can only be programmed in the acceptlist if + * LL Privacy has been enable otherwise they cannot mark + * HCI_CONN_FLAG_REMOTE_WAKEUP. + */ + if ((flags & HCI_CONN_FLAG_REMOTE_WAKEUP) && !use_ll_privacy(hdev) && + hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type)) + flags &= ~HCI_CONN_FLAG_REMOTE_WAKEUP; + + return flags; +} + static int get_device_flags(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { @@ -4578,10 +5064,10 @@ static int get_device_flags(struct sock *sk, struct hci_dev *hdev, void *data, } else { params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); - if (!params) goto done; + supported_flags = get_params_flags(hdev, params); current_flags = params->flags; } @@ -4649,38 +5135,35 @@ static int set_device_flags(struct sock *sk, struct hci_dev *hdev, void *data, bt_dev_warn(hdev, "No such BR/EDR device %pMR (0x%x)", &cp->addr.bdaddr, cp->addr.type); } - } else { - params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, - le_addr_type(cp->addr.type)); - if (params) { - /* Devices using RPAs can only be programmed in the - * acceptlist LL Privacy has been enable otherwise they - * cannot mark HCI_CONN_FLAG_REMOTE_WAKEUP. - */ - if ((current_flags & HCI_CONN_FLAG_REMOTE_WAKEUP) && - !use_ll_privacy(hdev) && - hci_find_irk_by_addr(hdev, ¶ms->addr, - params->addr_type)) { - bt_dev_warn(hdev, - "Cannot set wakeable for RPA"); - goto unlock; - } - params->flags = current_flags; - status = MGMT_STATUS_SUCCESS; + goto unlock; + } - /* Update passive scan if HCI_CONN_FLAG_DEVICE_PRIVACY - * has been set. - */ - if (params->flags & HCI_CONN_FLAG_DEVICE_PRIVACY) - hci_update_passive_scan(hdev); - } else { - bt_dev_warn(hdev, "No such LE device %pMR (0x%x)", - &cp->addr.bdaddr, - le_addr_type(cp->addr.type)); - } + params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, + le_addr_type(cp->addr.type)); + if (!params) { + bt_dev_warn(hdev, "No such LE device %pMR (0x%x)", + &cp->addr.bdaddr, le_addr_type(cp->addr.type)); + goto unlock; + } + + supported_flags = get_params_flags(hdev, params); + + if ((supported_flags | current_flags) != supported_flags) { + bt_dev_warn(hdev, "Bad flag given (0x%x) vs supported (0x%0x)", + current_flags, supported_flags); + goto unlock; } + params->flags = current_flags; + status = MGMT_STATUS_SUCCESS; + + /* Update passive scan if HCI_CONN_FLAG_DEVICE_PRIVACY + * has been set. + */ + if (params->flags & HCI_CONN_FLAG_DEVICE_PRIVACY) + hci_update_passive_scan(hdev); + unlock: hci_dev_unlock(hdev); @@ -5054,7 +5537,6 @@ static int remove_adv_monitor(struct sock *sk, struct hci_dev *hdev, else status = MGMT_STATUS_FAILED; - mgmt_pending_remove(cmd); goto unlock; } @@ -5969,6 +6451,7 @@ static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data, if (!hdev_is_powered(hdev) || (val == hci_dev_test_flag(hdev, HCI_ADVERTISING) && (cp->val == 0x02) == hci_dev_test_flag(hdev, HCI_ADVERTISING_CONNECTABLE)) || + hci_dev_test_flag(hdev, HCI_MESH) || hci_conn_num(hdev, LE_LINK) > 0 || (hci_dev_test_flag(hdev, HCI_LE_SCAN) && hdev->le_scan_type == LE_SCAN_ACTIVE)) { @@ -7897,8 +8380,7 @@ static u32 get_supported_adv_flags(struct hci_dev *hdev) /* In extended adv TX_POWER returned from Set Adv Param * will be always valid. */ - if ((hdev->adv_tx_power != HCI_TX_POWER_INVALID) || - ext_adv_capable(hdev)) + if (hdev->adv_tx_power != HCI_TX_POWER_INVALID || ext_adv_capable(hdev)) flags |= MGMT_ADV_FLAG_TX_POWER; if (ext_adv_capable(hdev)) { @@ -7951,8 +8433,14 @@ static int read_adv_features(struct sock *sk, struct hci_dev *hdev, instance = rp->instance; list_for_each_entry(adv_instance, &hdev->adv_instances, list) { - *instance = adv_instance->instance; - instance++; + /* Only instances 1-le_num_of_adv_sets are externally visible */ + if (adv_instance->instance <= hdev->adv_instance_cnt) { + *instance = adv_instance->instance; + instance++; + } else { + rp->num_instances--; + rp_len--; + } } hci_dev_unlock(hdev); @@ -8214,7 +8702,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, timeout, duration, HCI_ADV_TX_POWER_NO_PREFERENCE, hdev->le_adv_min_interval, - hdev->le_adv_max_interval); + hdev->le_adv_max_interval, 0); if (IS_ERR(adv)) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, MGMT_STATUS_FAILED); @@ -8418,7 +8906,7 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, /* Create advertising instance with no advertising or response data */ adv = hci_add_adv_instance(hdev, cp->instance, flags, 0, NULL, 0, NULL, timeout, duration, tx_power, min_interval, - max_interval); + max_interval, 0); if (IS_ERR(adv)) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_PARAMS, @@ -8864,8 +9352,13 @@ static const struct hci_mgmt_handler mgmt_handlers[] = { { add_ext_adv_data, MGMT_ADD_EXT_ADV_DATA_SIZE, HCI_MGMT_VAR_LEN }, { add_adv_patterns_monitor_rssi, - MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE, + MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE }, + { set_mesh, MGMT_SET_MESH_RECEIVER_SIZE, + HCI_MGMT_VAR_LEN }, + { mesh_features, MGMT_MESH_READ_FEATURES_SIZE }, + { mesh_send, MGMT_MESH_SEND_SIZE, HCI_MGMT_VAR_LEN }, + { mesh_send_cancel, MGMT_MESH_SEND_CANCEL_SIZE }, }; void mgmt_index_added(struct hci_dev *hdev) @@ -9805,14 +10298,86 @@ static void mgmt_adv_monitor_device_found(struct hci_dev *hdev, kfree_skb(skb); } +static void mesh_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, + u8 addr_type, s8 rssi, u32 flags, u8 *eir, + u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len, + u64 instant) +{ + struct sk_buff *skb; + struct mgmt_ev_mesh_device_found *ev; + int i, j; + + if (!hdev->mesh_ad_types[0]) + goto accepted; + + /* Scan for requested AD types */ + if (eir_len > 0) { + for (i = 0; i + 1 < eir_len; i += eir[i] + 1) { + for (j = 0; j < sizeof(hdev->mesh_ad_types); j++) { + if (!hdev->mesh_ad_types[j]) + break; + + if (hdev->mesh_ad_types[j] == eir[i + 1]) + goto accepted; + } + } + } + + if (scan_rsp_len > 0) { + for (i = 0; i + 1 < scan_rsp_len; i += scan_rsp[i] + 1) { + for (j = 0; j < sizeof(hdev->mesh_ad_types); j++) { + if (!hdev->mesh_ad_types[j]) + break; + + if (hdev->mesh_ad_types[j] == scan_rsp[i + 1]) + goto accepted; + } + } + } + + return; + +accepted: + skb = mgmt_alloc_skb(hdev, MGMT_EV_MESH_DEVICE_FOUND, + sizeof(*ev) + eir_len + scan_rsp_len); + if (!skb) + return; + + ev = skb_put(skb, sizeof(*ev)); + + bacpy(&ev->addr.bdaddr, bdaddr); + ev->addr.type = link_to_bdaddr(LE_LINK, addr_type); + ev->rssi = rssi; + ev->flags = cpu_to_le32(flags); + ev->instant = cpu_to_le64(instant); + + if (eir_len > 0) + /* Copy EIR or advertising data into event */ + skb_put_data(skb, eir, eir_len); + + if (scan_rsp_len > 0) + /* Append scan response data to event */ + skb_put_data(skb, scan_rsp, scan_rsp_len); + + ev->eir_len = cpu_to_le16(eir_len + scan_rsp_len); + + mgmt_event_skb(skb, NULL); +} + void mgmt_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, u8 *dev_class, s8 rssi, u32 flags, - u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len) + u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len, + u64 instant) { struct sk_buff *skb; struct mgmt_ev_device_found *ev; bool report_device = hci_discovery_active(hdev); + if (hci_dev_test_flag(hdev, HCI_MESH) && link_type == LE_LINK) + mesh_device_found(hdev, bdaddr, addr_type, rssi, flags, + eir, eir_len, scan_rsp, scan_rsp_len, + instant); + /* Don't send events for a non-kernel initiated discovery. With * LE one exception is if we have pend_le_reports > 0 in which * case we're doing passive scanning and want these events. @@ -9971,3 +10536,22 @@ void mgmt_exit(void) { hci_mgmt_chan_unregister(&chan); } + +void mgmt_cleanup(struct sock *sk) +{ + struct mgmt_mesh_tx *mesh_tx; + struct hci_dev *hdev; + + read_lock(&hci_dev_list_lock); + + list_for_each_entry(hdev, &hci_dev_list, list) { + do { + mesh_tx = mgmt_mesh_next(hdev, sk); + + if (mesh_tx) + mesh_send_complete(hdev, mesh_tx, true); + } while (mesh_tx); + } + + read_unlock(&hci_dev_list_lock); +} diff --git a/net/bluetooth/mgmt_util.c b/net/bluetooth/mgmt_util.c index b69cfed62088ded11384ede77b4a6d258e2f70b4..0115f783bde8055f9f0b8b46a9a84321e75abe39 100644 --- a/net/bluetooth/mgmt_util.c +++ b/net/bluetooth/mgmt_util.c @@ -314,3 +314,77 @@ void mgmt_pending_remove(struct mgmt_pending_cmd *cmd) list_del(&cmd->list); mgmt_pending_free(cmd); } + +void mgmt_mesh_foreach(struct hci_dev *hdev, + void (*cb)(struct mgmt_mesh_tx *mesh_tx, void *data), + void *data, struct sock *sk) +{ + struct mgmt_mesh_tx *mesh_tx, *tmp; + + list_for_each_entry_safe(mesh_tx, tmp, &hdev->mgmt_pending, list) { + if (!sk || mesh_tx->sk == sk) + cb(mesh_tx, data); + } +} + +struct mgmt_mesh_tx *mgmt_mesh_next(struct hci_dev *hdev, struct sock *sk) +{ + struct mgmt_mesh_tx *mesh_tx; + + if (list_empty(&hdev->mesh_pending)) + return NULL; + + list_for_each_entry(mesh_tx, &hdev->mesh_pending, list) { + if (!sk || mesh_tx->sk == sk) + return mesh_tx; + } + + return NULL; +} + +struct mgmt_mesh_tx *mgmt_mesh_find(struct hci_dev *hdev, u8 handle) +{ + struct mgmt_mesh_tx *mesh_tx; + + if (list_empty(&hdev->mesh_pending)) + return NULL; + + list_for_each_entry(mesh_tx, &hdev->mesh_pending, list) { + if (mesh_tx->handle == handle) + return mesh_tx; + } + + return NULL; +} + +struct mgmt_mesh_tx *mgmt_mesh_add(struct sock *sk, struct hci_dev *hdev, + void *data, u16 len) +{ + struct mgmt_mesh_tx *mesh_tx; + + mesh_tx = kzalloc(sizeof(*mesh_tx), GFP_KERNEL); + if (!mesh_tx) + return NULL; + + hdev->mesh_send_ref++; + if (!hdev->mesh_send_ref) + hdev->mesh_send_ref++; + + mesh_tx->handle = hdev->mesh_send_ref; + mesh_tx->index = hdev->id; + memcpy(mesh_tx->param, data, len); + mesh_tx->param_len = len; + mesh_tx->sk = sk; + sock_hold(sk); + + list_add_tail(&mesh_tx->list, &hdev->mesh_pending); + + return mesh_tx; +} + +void mgmt_mesh_remove(struct mgmt_mesh_tx *mesh_tx) +{ + list_del(&mesh_tx->list); + sock_put(mesh_tx->sk); + kfree(mesh_tx); +} diff --git a/net/bluetooth/mgmt_util.h b/net/bluetooth/mgmt_util.h index 98e40395a3834a7564a2c4cc7b057864a4bfd12e..6a8b7e84293dfd436af59a433f4d82e92497867e 100644 --- a/net/bluetooth/mgmt_util.h +++ b/net/bluetooth/mgmt_util.h @@ -20,6 +20,16 @@ SOFTWARE IS DISCLAIMED. */ +struct mgmt_mesh_tx { + struct list_head list; + int index; + size_t param_len; + struct sock *sk; + u8 handle; + u8 instance; + u8 param[sizeof(struct mgmt_cp_mesh_send) + 29]; +}; + struct mgmt_pending_cmd { struct list_head list; u16 opcode; @@ -59,3 +69,11 @@ struct mgmt_pending_cmd *mgmt_pending_new(struct sock *sk, u16 opcode, void *data, u16 len); void mgmt_pending_free(struct mgmt_pending_cmd *cmd); void mgmt_pending_remove(struct mgmt_pending_cmd *cmd); +void mgmt_mesh_foreach(struct hci_dev *hdev, + void (*cb)(struct mgmt_mesh_tx *mesh_tx, void *data), + void *data, struct sock *sk); +struct mgmt_mesh_tx *mgmt_mesh_find(struct hci_dev *hdev, u8 handle); +struct mgmt_mesh_tx *mgmt_mesh_next(struct hci_dev *hdev, struct sock *sk); +struct mgmt_mesh_tx *mgmt_mesh_add(struct sock *sk, struct hci_dev *hdev, + void *data, u16 len); +void mgmt_mesh_remove(struct mgmt_mesh_tx *mesh_tx); diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index 4bf4ea6cbb5eeed122cb2241ceb69701cd463c09..21e24da4847f05a96145815e76e19c3e543a46f0 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -902,7 +902,10 @@ static int rfcomm_sock_shutdown(struct socket *sock, int how) lock_sock(sk); if (!sk->sk_shutdown) { sk->sk_shutdown = SHUTDOWN_MASK; + + release_sock(sk); __rfcomm_sock_close(sk); + lock_sock(sk); if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime && !(current->flags & PF_EXITING)) diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index ebd78fdbd6e8bce1d09ca1c677f3539c77f0e8ae..8009e0e932162d77d58343332ba0ac671411df15 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -35,7 +35,6 @@ #include #include -#define RFCOMM_TTY_MAGIC 0x6d02 /* magic number for rfcomm struct */ #define RFCOMM_TTY_PORTS RFCOMM_MAX_DEV /* whole lotta rfcomm devices */ #define RFCOMM_TTY_MAJOR 216 /* device node major id of the usb/bluetooth.c driver */ #define RFCOMM_TTY_MINOR 0 @@ -855,7 +854,8 @@ static int rfcomm_tty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned l return -ENOIOCTLCMD; } -static void rfcomm_tty_set_termios(struct tty_struct *tty, struct ktermios *old) +static void rfcomm_tty_set_termios(struct tty_struct *tty, + const struct ktermios *old) { struct ktermios *new = &tty->termios; int old_baud_rate = tty_termios_baud_rate(old); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index d11209367dd0066c7b6bd3977f6b8f79bdfd9a05..13d578ce2a096a85c62f22d6bc728552d4746e33 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -606,6 +606,38 @@ noinline void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p) WARN_ON_ONCE(1); } +static int *__bpf_kfunc_call_test_get_mem(struct prog_test_ref_kfunc *p, const int size) +{ + if (size > 2 * sizeof(int)) + return NULL; + + return (int *)p; +} + +noinline int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) +{ + return __bpf_kfunc_call_test_get_mem(p, rdwr_buf_size); +} + +noinline int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) +{ + return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size); +} + +/* the next 2 ones can't be really used for testing expect to ensure + * that the verifier rejects the call. + * Acquire functions must return struct pointers, so these ones are + * failing. + */ +noinline int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) +{ + return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size); +} + +noinline void bpf_kfunc_call_int_mem_release(int *p) +{ +} + noinline struct prog_test_ref_kfunc * bpf_kfunc_call_test_kptr_get(struct prog_test_ref_kfunc **pp, int a, int b) { @@ -695,6 +727,10 @@ noinline void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p) { } +noinline void bpf_kfunc_call_test_destructive(void) +{ +} + __diag_pop(); ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO); @@ -708,6 +744,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdwr_mem, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdonly_mem, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_acq_rdonly_mem, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_kfunc_call_test_kptr_get, KF_ACQUIRE | KF_RET_NULL | KF_KPTR_GET) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1) @@ -719,6 +759,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1) BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE) BTF_SET8_END(test_sk_check_kfunc_ids) static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, @@ -1629,6 +1670,7 @@ static int __init bpf_prog_test_run_init(void) ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set); return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc, ARRAY_SIZE(bpf_prog_test_dtor_kfunc), THIS_MODULE); diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index 58a4f70e01e360f44f5599e637ac388c60dd01fc..b82906fc999a32e3809612e48bdc1d88c6709b65 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -251,10 +251,10 @@ static int br_set_mac_address(struct net_device *dev, void *p) static void br_getinfo(struct net_device *dev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "bridge", sizeof(info->driver)); - strlcpy(info->version, BR_VERSION, sizeof(info->version)); - strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); - strlcpy(info->bus_info, "N/A", sizeof(info->bus_info)); + strscpy(info->driver, "bridge", sizeof(info->driver)); + strscpy(info->version, BR_VERSION, sizeof(info->version)); + strscpy(info->fw_version, "N/A", sizeof(info->fw_version)); + strscpy(info->bus_info, "N/A", sizeof(info->bus_info)); } static int br_get_link_ksettings(struct net_device *dev, diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index a84a7cfb9d6dc856070dc2f24fa9ab0444dad6c7..228fd5b20f109e4190f6e0e169c77683c8d2ec61 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -40,12 +40,21 @@ static int port_cost(struct net_device *dev) switch (ecmd.base.speed) { case SPEED_10000: return 2; - case SPEED_1000: + case SPEED_5000: + return 3; + case SPEED_2500: return 4; + case SPEED_1000: + return 5; case SPEED_100: return 19; case SPEED_10: return 100; + case SPEED_UNKNOWN: + return 100; + default: + if (ecmd.base.speed > SPEED_10000) + return 1; } } @@ -568,26 +577,6 @@ int br_add_if(struct net_bridge *br, struct net_device *dev, !is_valid_ether_addr(dev->dev_addr)) return -EINVAL; - /* Also don't allow bridging of net devices that are DSA masters, since - * the bridge layer rx_handler prevents the DSA fake ethertype handler - * to be invoked, so we don't get the chance to strip off and parse the - * DSA switch tag protocol header (the bridge layer just returns - * RX_HANDLER_CONSUMED, stopping RX processing for these frames). - * The only case where that would not be an issue is when bridging can - * already be offloaded, such as when the DSA master is itself a DSA - * or plain switchdev port, and is bridged only with other ports from - * the same hardware device. - */ - if (netdev_uses_dsa(dev)) { - list_for_each_entry(p, &br->port_list, list) { - if (!netdev_port_same_parent_id(dev, p->dev)) { - NL_SET_ERR_MSG(extack, - "Cannot do software bridging with a DSA master"); - return -EINVAL; - } - } - } - /* No bridging of bridges */ if (dev->netdev_ops->ndo_start_xmit == br_dev_xmit) { NL_SET_ERR_MSG(extack, diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c index ff47790366497f6dad7a7be272330fec3b1a34a2..f20f4373ff408ffaca8e9337d4bab3a31711b5f0 100644 --- a/net/bridge/br_netfilter_hooks.c +++ b/net/bridge/br_netfilter_hooks.c @@ -384,6 +384,7 @@ static int br_nf_pre_routing_finish(struct net *net, struct sock *sk, struct sk_ /* - Bridged-and-DNAT'ed traffic doesn't * require ip_forwarding. */ if (rt->dst.dev == dev) { + skb_dst_drop(skb); skb_dst_set(skb, &rt->dst); goto bridged_dnat; } @@ -413,6 +414,7 @@ bridged_dnat: kfree_skb(skb); return 0; } + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); } diff --git a/net/bridge/br_netfilter_ipv6.c b/net/bridge/br_netfilter_ipv6.c index e4e0c836c3f51d9edfa351b67fed34549ff9e01d..6b07f30675bb0c3114a111411455e3f3770232a3 100644 --- a/net/bridge/br_netfilter_ipv6.c +++ b/net/bridge/br_netfilter_ipv6.c @@ -197,6 +197,7 @@ static int br_nf_pre_routing_finish_ipv6(struct net *net, struct sock *sk, struc kfree_skb(skb); return 0; } + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); } diff --git a/net/bridge/br_sysfs_if.c b/net/bridge/br_sysfs_if.c index 07fa760805125a73ca45b0a96f1fc38266b70660..74fdd8105dca9ec24fbc9379c1353a25dad2ceed 100644 --- a/net/bridge/br_sysfs_if.c +++ b/net/bridge/br_sysfs_if.c @@ -384,7 +384,7 @@ int br_sysfs_addif(struct net_bridge_port *p) return err; } - strlcpy(p->sysfs_name, p->dev->name, IFNAMSIZ); + strscpy(p->sysfs_name, p->dev->name, IFNAMSIZ); return sysfs_create_link(br->ifobj, &p->kobj, p->sysfs_name); } @@ -406,7 +406,7 @@ int br_sysfs_renameif(struct net_bridge_port *p) netdev_notice(br->dev, "unable to rename link %s to %s", p->sysfs_name, p->dev->name); else - strlcpy(p->sysfs_name, p->dev->name, IFNAMSIZ); + strscpy(p->sysfs_name, p->dev->name, IFNAMSIZ); return err; } diff --git a/net/bridge/netfilter/ebtable_broute.c b/net/bridge/netfilter/ebtable_broute.c index 1a11064f9990719588c44d80a93c3269f4582c00..8f19253024b0aa4624bb7c8dac836d5c2fa3a01e 100644 --- a/net/bridge/netfilter/ebtable_broute.c +++ b/net/bridge/netfilter/ebtable_broute.c @@ -36,18 +36,10 @@ static struct ebt_replace_kernel initial_table = { .entries = (char *)&initial_chain, }; -static int check(const struct ebt_table_info *info, unsigned int valid_hooks) -{ - if (valid_hooks & ~(1 << NF_BR_BROUTING)) - return -EINVAL; - return 0; -} - static const struct ebt_table broute_table = { .name = "broute", .table = &initial_table, .valid_hooks = 1 << NF_BR_BROUTING, - .check = check, .me = THIS_MODULE, }; diff --git a/net/bridge/netfilter/ebtable_filter.c b/net/bridge/netfilter/ebtable_filter.c index cb949436bc0e34c2a721d5ca423c8db07e4bac2b..278f324e67524a8933345f48feeb267d0a9e2dfa 100644 --- a/net/bridge/netfilter/ebtable_filter.c +++ b/net/bridge/netfilter/ebtable_filter.c @@ -43,18 +43,10 @@ static struct ebt_replace_kernel initial_table = { .entries = (char *)initial_chains, }; -static int check(const struct ebt_table_info *info, unsigned int valid_hooks) -{ - if (valid_hooks & ~FILTER_VALID_HOOKS) - return -EINVAL; - return 0; -} - static const struct ebt_table frame_filter = { .name = "filter", .table = &initial_table, .valid_hooks = FILTER_VALID_HOOKS, - .check = check, .me = THIS_MODULE, }; diff --git a/net/bridge/netfilter/ebtable_nat.c b/net/bridge/netfilter/ebtable_nat.c index 5ee0531ae50610e456b07f051cf7769bb5cb004e..9066f7f376d57ea509f4af6cfc7d94cd157aae9b 100644 --- a/net/bridge/netfilter/ebtable_nat.c +++ b/net/bridge/netfilter/ebtable_nat.c @@ -43,18 +43,10 @@ static struct ebt_replace_kernel initial_table = { .entries = (char *)initial_chains, }; -static int check(const struct ebt_table_info *info, unsigned int valid_hooks) -{ - if (valid_hooks & ~NAT_VALID_HOOKS) - return -EINVAL; - return 0; -} - static const struct ebt_table frame_nat = { .name = "nat", .table = &initial_table, .valid_hooks = NAT_VALID_HOOKS, - .check = check, .me = THIS_MODULE, }; diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index f2dbefb61ce8368103ee4aa8b20f6b18edbd16ca..ce5dfa3babd26bcc5581ac01c963a7f9cc17961a 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -1040,9 +1040,10 @@ static int do_replace_finish(struct net *net, struct ebt_replace *repl, goto free_iterate; } - /* the table doesn't like it */ - if (t->check && (ret = t->check(newinfo, repl->valid_hooks))) + if (repl->valid_hooks != t->valid_hooks) { + ret = -EINVAL; goto free_unlock; + } if (repl->num_counters && repl->num_counters != t->private->nentries) { ret = -EINVAL; @@ -1231,11 +1232,6 @@ int ebt_register_table(struct net *net, const struct ebt_table *input_table, if (ret != 0) goto free_chainstack; - if (table->check && table->check(newinfo, table->valid_hooks)) { - ret = -EINVAL; - goto free_chainstack; - } - table->private = newinfo; rwlock_init(&table->lock); mutex_lock(&ebt_mutex); @@ -1446,7 +1442,7 @@ static inline int ebt_obj_to_user(char __user *um, const char *_name, /* ebtables expects 31 bytes long names but xt_match names are 29 bytes * long. Copy 29 bytes and fill remaining bytes with zeroes. */ - strlcpy(name, _name, sizeof(name)); + strscpy(name, _name, sizeof(name)); if (copy_to_user(um, name, EBT_EXTENSION_MAXNAMELEN) || put_user(revision, (u8 __user *)(um + EBT_EXTENSION_MAXNAMELEN)) || put_user(datasize, (int __user *)(um + EBT_EXTENSION_MAXNAMELEN + 1)) || diff --git a/net/caif/caif_dev.c b/net/caif/caif_dev.c index 52dd0b6835bc81d3b895f2932c7a94898f16b755..6a0cba4fc29fdf7bad134d786413a5bce66622a9 100644 --- a/net/caif/caif_dev.c +++ b/net/caif/caif_dev.c @@ -342,7 +342,7 @@ int caif_enroll_dev(struct net_device *dev, struct caif_dev_common *caifdev, mutex_lock(&caifdevs->lock); list_add_rcu(&caifd->list, &caifdevs->list); - strlcpy(caifd->layer.name, dev->name, + strscpy(caifd->layer.name, dev->name, sizeof(caifd->layer.name)); caifd->layer.transmit = transmit; res = cfcnfg_add_phy_layer(cfg, diff --git a/net/caif/caif_usb.c b/net/caif/caif_usb.c index 4be6b04879a16042207cc52baf7886fb16b6e88f..ebc202ffdd8d8d8fbb14e1009a86cbcad48094b4 100644 --- a/net/caif/caif_usb.c +++ b/net/caif/caif_usb.c @@ -184,7 +184,7 @@ static int cfusbl_device_notify(struct notifier_block *me, unsigned long what, dev_add_pack(&caif_usb_type); pack_added = true; - strlcpy(layer->name, dev->name, sizeof(layer->name)); + strscpy(layer->name, dev->name, sizeof(layer->name)); return 0; err: diff --git a/net/caif/cfcnfg.c b/net/caif/cfcnfg.c index 23267c8db7c410396aa25ee30ba87a0a487e3d7c..52509e1859601f158f1da3d951fc70038fbd1b54 100644 --- a/net/caif/cfcnfg.c +++ b/net/caif/cfcnfg.c @@ -268,14 +268,14 @@ static int caif_connect_req_to_link_param(struct cfcnfg *cnfg, case CAIFPROTO_RFM: l->linktype = CFCTRL_SRV_RFM; l->u.datagram.connid = s->sockaddr.u.rfm.connection_id; - strlcpy(l->u.rfm.volume, s->sockaddr.u.rfm.volume, + strscpy(l->u.rfm.volume, s->sockaddr.u.rfm.volume, sizeof(l->u.rfm.volume)); break; case CAIFPROTO_UTIL: l->linktype = CFCTRL_SRV_UTIL; l->endpoint = 0x00; l->chtype = 0x00; - strlcpy(l->u.utility.name, s->sockaddr.u.util.service, + strscpy(l->u.utility.name, s->sockaddr.u.util.service, sizeof(l->u.utility.name)); caif_assert(sizeof(l->u.utility.name) > 10); l->u.utility.paramlen = s->param.size; diff --git a/net/caif/cfctrl.c b/net/caif/cfctrl.c index 2809cbd6b7f7478bad724214246d8141515ec488..cc405d8c7c303f301303a6c643cb22b384713461 100644 --- a/net/caif/cfctrl.c +++ b/net/caif/cfctrl.c @@ -258,7 +258,7 @@ int cfctrl_linkup_request(struct cflayer *layer, tmp16 = cpu_to_le16(param->u.utility.fifosize_bufs); cfpkt_add_body(pkt, &tmp16, 2); memset(utility_name, 0, sizeof(utility_name)); - strlcpy(utility_name, param->u.utility.name, + strscpy(utility_name, param->u.utility.name, UTILITY_NAME_LENGTH); cfpkt_add_body(pkt, utility_name, UTILITY_NAME_LENGTH); tmp8 = param->u.utility.paramlen; diff --git a/net/can/af_can.c b/net/can/af_can.c index 1fb49d51b25d6d345e2865171e14de12d775312c..9503ab10f9b8f7a9735b9646d1e82d6c7295c981 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -199,27 +199,26 @@ static int can_create(struct net *net, struct socket *sock, int protocol, int can_send(struct sk_buff *skb, int loop) { struct sk_buff *newskb = NULL; - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; struct can_pkg_stats *pkg_stats = dev_net(skb->dev)->can.pkg_stats; int err = -EINVAL; - if (skb->len == CAN_MTU) { + if (can_is_canxl_skb(skb)) { + skb->protocol = htons(ETH_P_CANXL); + } else if (can_is_can_skb(skb)) { skb->protocol = htons(ETH_P_CAN); - if (unlikely(cfd->len > CAN_MAX_DLEN)) - goto inval_skb; - } else if (skb->len == CANFD_MTU) { + } else if (can_is_canfd_skb(skb)) { + struct canfd_frame *cfd = (struct canfd_frame *)skb->data; + skb->protocol = htons(ETH_P_CANFD); - if (unlikely(cfd->len > CANFD_MAX_DLEN)) - goto inval_skb; + + /* set CAN FD flag for CAN FD frames by default */ + cfd->flags |= CANFD_FDF; } else { goto inval_skb; } - /* Make sure the CAN frame can pass the selected CAN netdevice. - * As structs can_frame and canfd_frame are similar, we can provide - * CAN FD frames to legacy CAN drivers as long as the length is <= 8 - */ - if (unlikely(skb->len > skb->dev->mtu && cfd->len > CAN_MAX_DLEN)) { + /* Make sure the CAN frame can pass the selected CAN netdevice. */ + if (unlikely(skb->len > skb->dev->mtu)) { err = -EMSGSIZE; goto inval_skb; } @@ -678,53 +677,46 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev) static int can_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - - if (unlikely(dev->type != ARPHRD_CAN || skb->len != CAN_MTU)) { + if (unlikely(dev->type != ARPHRD_CAN || (!can_is_can_skb(skb)))) { pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d\n", dev->type, skb->len); - goto free_skb; - } - /* This check is made separately since cfd->len would be uninitialized if skb->len = 0. */ - if (unlikely(cfd->len > CAN_MAX_DLEN)) { - pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d, datalen %d\n", - dev->type, skb->len, cfd->len); - goto free_skb; + kfree_skb(skb); + return NET_RX_DROP; } can_receive(skb, dev); return NET_RX_SUCCESS; - -free_skb: - kfree_skb(skb); - return NET_RX_DROP; } static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { - struct canfd_frame *cfd = (struct canfd_frame *)skb->data; - - if (unlikely(dev->type != ARPHRD_CAN || skb->len != CANFD_MTU)) { + if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canfd_skb(skb)))) { pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d\n", dev->type, skb->len); - goto free_skb; - } - /* This check is made separately since cfd->len would be uninitialized if skb->len = 0. */ - if (unlikely(cfd->len > CANFD_MAX_DLEN)) { - pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d, datalen %d\n", - dev->type, skb->len, cfd->len); - goto free_skb; + kfree_skb(skb); + return NET_RX_DROP; } can_receive(skb, dev); return NET_RX_SUCCESS; +} -free_skb: - kfree_skb(skb); - return NET_RX_DROP; +static int canxl_rcv(struct sk_buff *skb, struct net_device *dev, + struct packet_type *pt, struct net_device *orig_dev) +{ + if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canxl_skb(skb)))) { + pr_warn_once("PF_CAN: dropped non conform CAN XL skbuff: dev type %d, len %d\n", + dev->type, skb->len); + + kfree_skb(skb); + return NET_RX_DROP; + } + + can_receive(skb, dev); + return NET_RX_SUCCESS; } /* af_can protocol functions */ @@ -851,6 +843,11 @@ static struct packet_type canfd_packet __read_mostly = { .func = canfd_rcv, }; +static struct packet_type canxl_packet __read_mostly = { + .type = cpu_to_be16(ETH_P_CANXL), + .func = canxl_rcv, +}; + static const struct net_proto_family can_family_ops = { .family = PF_CAN, .create = can_create, @@ -890,6 +887,7 @@ static __init int can_init(void) dev_add_pack(&can_packet); dev_add_pack(&canfd_packet); + dev_add_pack(&canxl_packet); return 0; diff --git a/net/can/bcm.c b/net/can/bcm.c index e60161bec850a6561f3358a89d7a6ef254bc5a07..27706f6ace34a0296392ae66ca5b0ab1fad26e04 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -274,6 +274,7 @@ static void bcm_can_tx(struct bcm_op *op) struct sk_buff *skb; struct net_device *dev; struct canfd_frame *cf = op->frames + op->cfsiz * op->currframe; + int err; /* no target device? => exit */ if (!op->ifindex) @@ -298,11 +299,11 @@ static void bcm_can_tx(struct bcm_op *op) /* send with loopback */ skb->dev = dev; can_skb_set_owner(skb, op->sk); - can_send(skb, 1); + err = can_send(skb, 1); + if (!err) + op->frames_abs++; - /* update statistics */ op->currframe++; - op->frames_abs++; /* reached last frame? */ if (op->currframe >= op->nframes) @@ -648,8 +649,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) return; /* make sure to handle the correct frame type (CAN / CAN FD) */ - if (skb->len != op->cfsiz) - return; + if (op->flags & CAN_FD_FRAME) { + if (!can_is_canfd_skb(skb)) + return; + } else { + if (!can_is_can_skb(skb)) + return; + } /* disable timeout */ hrtimer_cancel(&op->timer); @@ -1744,15 +1750,27 @@ static int __init bcm_module_init(void) pr_info("can: broadcast manager protocol\n"); + err = register_pernet_subsys(&canbcm_pernet_ops); + if (err) + return err; + + err = register_netdevice_notifier(&canbcm_notifier); + if (err) + goto register_notifier_failed; + err = can_proto_register(&bcm_can_proto); if (err < 0) { printk(KERN_ERR "can: registration of bcm protocol failed\n"); - return err; + goto register_proto_failed; } - register_pernet_subsys(&canbcm_pernet_ops); - register_netdevice_notifier(&canbcm_notifier); return 0; + +register_proto_failed: + unregister_netdevice_notifier(&canbcm_notifier); +register_notifier_failed: + unregister_pernet_subsys(&canbcm_pernet_ops); + return err; } static void __exit bcm_module_exit(void) diff --git a/net/can/gw.c b/net/can/gw.c index 1ea4cc527db3af4cc7b987dea58eea182c5bfc23..23a3d89cad81d82a3590f6b397b113736081cc17 100644 --- a/net/can/gw.c +++ b/net/can/gw.c @@ -463,10 +463,10 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) /* process strictly Classic CAN or CAN FD frames */ if (gwj->flags & CGW_FLAGS_CAN_FD) { - if (skb->len != CANFD_MTU) + if (!can_is_canfd_skb(skb)) return; } else { - if (skb->len != CAN_MTU) + if (!can_is_can_skb(skb)) return; } diff --git a/net/can/isotp.c b/net/can/isotp.c index 43a27d19cdacfddeace6d7a740b0531ab4c31b96..a9d1357f8489f4803128a94896811c814c960628 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -669,7 +669,7 @@ static void isotp_rcv(struct sk_buff *skb, void *data) if (cf->len <= CAN_MAX_DLEN) { isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl); } else { - if (skb->len == CANFD_MTU) { + if (can_is_canfd_skb(skb)) { /* We have a CAN FD frame and CAN_DL is greater than 8: * Only frames with the SF_DL == 0 ESC value are valid. * diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c index 8452b0fbb78c9a4b327fbb0f998d2df0eab12495..144c86b0e3ffe3b99937d0b08dd9170ba5e479a3 100644 --- a/net/can/j1939/main.c +++ b/net/can/j1939/main.c @@ -42,6 +42,10 @@ static void j1939_can_recv(struct sk_buff *iskb, void *data) struct j1939_sk_buff_cb *skcb, *iskcb; struct can_frame *cf; + /* make sure we only get Classical CAN frames */ + if (!can_is_can_skb(iskb)) + return; + /* create a copy of the skb * j1939 only delivers the real data bytes, * the header goes into sockaddr. diff --git a/net/can/raw.c b/net/can/raw.c index d1bd9cc51ebeaf286db88f9dac8faf06cfeaeada..3eb7d3e2b541f2ba40deaecde15bb58bdf6fbfc1 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -50,6 +50,7 @@ #include #include #include +#include /* for can_is_canxl_dev_mtu() */ #include #include #include @@ -87,6 +88,7 @@ struct raw_sock { int loopback; int recv_own_msgs; int fd_frames; + int xl_frames; int join_filters; int count; /* number of active filters */ struct can_filter dfilter; /* default/single filter */ @@ -129,21 +131,21 @@ static void raw_rcv(struct sk_buff *oskb, void *data) if (!ro->recv_own_msgs && oskb->sk == sk) return; - /* do not pass non-CAN2.0 frames to a legacy socket */ - if (!ro->fd_frames && oskb->len != CAN_MTU) + /* make sure to not pass oversized frames to the socket */ + if ((can_is_canfd_skb(oskb) && !ro->fd_frames && !ro->xl_frames) || + (can_is_canxl_skb(oskb) && !ro->xl_frames)) return; /* eliminate multiple filter matches for the same skb */ if (this_cpu_ptr(ro->uniq)->skb == oskb && this_cpu_ptr(ro->uniq)->skbcnt == can_skb_prv(oskb)->skbcnt) { - if (ro->join_filters) { - this_cpu_inc(ro->uniq->join_rx_count); - /* drop frame until all enabled filters matched */ - if (this_cpu_ptr(ro->uniq)->join_rx_count < ro->count) - return; - } else { + if (!ro->join_filters) + return; + + this_cpu_inc(ro->uniq->join_rx_count); + /* drop frame until all enabled filters matched */ + if (this_cpu_ptr(ro->uniq)->join_rx_count < ro->count) return; - } } else { this_cpu_ptr(ro->uniq)->skb = oskb; this_cpu_ptr(ro->uniq)->skbcnt = can_skb_prv(oskb)->skbcnt; @@ -346,6 +348,7 @@ static int raw_init(struct sock *sk) ro->loopback = 1; ro->recv_own_msgs = 0; ro->fd_frames = 0; + ro->xl_frames = 0; ro->join_filters = 0; /* alloc_percpu provides zero'ed memory */ @@ -669,6 +672,15 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, break; + case CAN_RAW_XL_FRAMES: + if (optlen != sizeof(ro->xl_frames)) + return -EINVAL; + + if (copy_from_sockptr(&ro->xl_frames, optval, optlen)) + return -EFAULT; + + break; + case CAN_RAW_JOIN_FILTERS: if (optlen != sizeof(ro->join_filters)) return -EINVAL; @@ -751,6 +763,12 @@ static int raw_getsockopt(struct socket *sock, int level, int optname, val = &ro->fd_frames; break; + case CAN_RAW_XL_FRAMES: + if (len > sizeof(int)) + len = sizeof(int); + val = &ro->xl_frames; + break; + case CAN_RAW_JOIN_FILTERS: if (len > sizeof(int)) len = sizeof(int); @@ -776,7 +794,11 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct sk_buff *skb; struct net_device *dev; int ifindex; - int err; + int err = -EINVAL; + + /* check for valid CAN frame sizes */ + if (size < CANXL_HDR_SIZE + CANXL_MIN_DLEN || size > CANXL_MTU) + return -EINVAL; if (msg->msg_name) { DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name); @@ -796,15 +818,6 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) if (!dev) return -ENXIO; - err = -EINVAL; - if (ro->fd_frames && dev->mtu == CANFD_MTU) { - if (unlikely(size != CANFD_MTU && size != CAN_MTU)) - goto put_dev; - } else { - if (unlikely(size != CAN_MTU)) - goto put_dev; - } - skb = sock_alloc_send_skb(sk, size + sizeof(struct can_skb_priv), msg->msg_flags & MSG_DONTWAIT, &err); if (!skb) @@ -814,10 +827,27 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) can_skb_prv(skb)->ifindex = dev->ifindex; can_skb_prv(skb)->skbcnt = 0; + /* fill the skb before testing for valid CAN frames */ err = memcpy_from_msg(skb_put(skb, size), msg, size); if (err < 0) goto free_skb; + err = -EINVAL; + if (ro->xl_frames && can_is_canxl_dev_mtu(dev->mtu)) { + /* CAN XL, CAN FD and Classical CAN */ + if (!can_is_canxl_skb(skb) && !can_is_canfd_skb(skb) && + !can_is_can_skb(skb)) + goto free_skb; + } else if (ro->fd_frames && dev->mtu == CANFD_MTU) { + /* CAN FD and Classical CAN */ + if (!can_is_canfd_skb(skb) && !can_is_can_skb(skb)) + goto free_skb; + } else { + /* Classical CAN */ + if (!can_is_can_skb(skb)) + goto free_skb; + } + sockcm_init(&sockc, sk); if (msg->msg_controllen) { err = sock_cmsg_send(sk, msg, &sockc); @@ -942,12 +972,20 @@ static __init int raw_module_init(void) pr_info("can: raw protocol\n"); + err = register_netdevice_notifier(&canraw_notifier); + if (err) + return err; + err = can_proto_register(&raw_can_proto); - if (err < 0) + if (err < 0) { pr_err("can: registration of raw protocol failed\n"); - else - register_netdevice_notifier(&canraw_notifier); + goto register_proto_failed; + } + + return 0; +register_proto_failed: + unregister_netdevice_notifier(&canraw_notifier); return err; } diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index d3bb656308b432e0abc3ce2a3c221f40470bd5ef..dfa237fbd5a325ab96bce3bfe05ceb78b11eceac 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -728,7 +728,6 @@ static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor, it->iter.bi_size = cursor->resid; BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter)); - cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter); } static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor, @@ -754,10 +753,8 @@ static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor, cursor->resid -= bytes; bio_advance_iter(it->bio, &it->iter, bytes); - if (!cursor->resid) { - BUG_ON(!cursor->last_piece); + if (!cursor->resid) return false; /* no more data */ - } if (!bytes || (it->iter.bi_size && it->iter.bi_bvec_done && page == bio_iter_page(it->bio, it->iter))) @@ -770,9 +767,7 @@ static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor, it->iter.bi_size = cursor->resid; } - BUG_ON(cursor->last_piece); BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter)); - cursor->last_piece = cursor->resid == bio_iter_len(it->bio, it->iter); return true; } #endif /* CONFIG_BLOCK */ @@ -788,8 +783,6 @@ static void ceph_msg_data_bvecs_cursor_init(struct ceph_msg_data_cursor *cursor, cursor->bvec_iter.bi_size = cursor->resid; BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter)); - cursor->last_piece = - cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter); } static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor, @@ -815,19 +808,14 @@ static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor, cursor->resid -= bytes; bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes); - if (!cursor->resid) { - BUG_ON(!cursor->last_piece); + if (!cursor->resid) return false; /* no more data */ - } if (!bytes || (cursor->bvec_iter.bi_bvec_done && page == bvec_iter_page(bvecs, cursor->bvec_iter))) return false; /* more bytes to process in this segment */ - BUG_ON(cursor->last_piece); BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter)); - cursor->last_piece = - cursor->resid == bvec_iter_len(bvecs, cursor->bvec_iter); return true; } @@ -853,7 +841,6 @@ static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor, BUG_ON(page_count > (int)USHRT_MAX); cursor->page_count = (unsigned short)page_count; BUG_ON(length > SIZE_MAX - cursor->page_offset); - cursor->last_piece = cursor->page_offset + cursor->resid <= PAGE_SIZE; } static struct page * @@ -868,11 +855,7 @@ ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor, BUG_ON(cursor->page_offset >= PAGE_SIZE); *page_offset = cursor->page_offset; - if (cursor->last_piece) - *length = cursor->resid; - else - *length = PAGE_SIZE - *page_offset; - + *length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset); return data->pages[cursor->page_index]; } @@ -897,8 +880,6 @@ static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor, BUG_ON(cursor->page_index >= cursor->page_count); cursor->page_index++; - cursor->last_piece = cursor->resid <= PAGE_SIZE; - return true; } @@ -928,7 +909,6 @@ ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor, cursor->resid = min(length, pagelist->length); cursor->page = page; cursor->offset = 0; - cursor->last_piece = cursor->resid <= PAGE_SIZE; } static struct page * @@ -948,11 +928,7 @@ ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor, /* offset of first page in pagelist is always 0 */ *page_offset = cursor->offset & ~PAGE_MASK; - if (cursor->last_piece) - *length = cursor->resid; - else - *length = PAGE_SIZE - *page_offset; - + *length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset); return cursor->page; } @@ -985,8 +961,6 @@ static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor, BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head)); cursor->page = list_next_entry(cursor->page, lru); - cursor->last_piece = cursor->resid <= PAGE_SIZE; - return true; } @@ -1044,8 +1018,7 @@ void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor, * Indicate whether this is the last piece in this data item. */ struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor, - size_t *page_offset, size_t *length, - bool *last_piece) + size_t *page_offset, size_t *length) { struct page *page; @@ -1074,8 +1047,6 @@ struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor, BUG_ON(*page_offset + *length > PAGE_SIZE); BUG_ON(!*length); BUG_ON(*length > cursor->resid); - if (last_piece) - *last_piece = cursor->last_piece; return page; } @@ -1112,7 +1083,6 @@ void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes) cursor->total_resid -= bytes; if (!cursor->resid && cursor->total_resid) { - WARN_ON(!cursor->last_piece); cursor->data++; __ceph_msg_data_cursor_init(cursor); new_piece = true; diff --git a/net/ceph/messenger_v1.c b/net/ceph/messenger_v1.c index 6b014eca3a1305e48aeb62d533951ad98c68ad78..3ddbde87e4d6e393777becc8ad3fff6c6fbf416a 100644 --- a/net/ceph/messenger_v1.c +++ b/net/ceph/messenger_v1.c @@ -495,7 +495,7 @@ static int write_partial_message_data(struct ceph_connection *con) continue; } - page = ceph_msg_data_next(cursor, &page_offset, &length, NULL); + page = ceph_msg_data_next(cursor, &page_offset, &length); if (length == cursor->total_resid) more = MSG_MORE; ret = ceph_tcp_sendpage(con->sock, page, page_offset, length, @@ -1008,7 +1008,7 @@ static int read_partial_msg_data(struct ceph_connection *con) continue; } - page = ceph_msg_data_next(cursor, &page_offset, &length, NULL); + page = ceph_msg_data_next(cursor, &page_offset, &length); ret = ceph_tcp_recvpage(con->sock, page, page_offset, length); if (ret <= 0) { if (do_datacrc) @@ -1050,7 +1050,7 @@ static int read_partial_msg_data_bounce(struct ceph_connection *con) continue; } - page = ceph_msg_data_next(cursor, &off, &len, NULL); + page = ceph_msg_data_next(cursor, &off, &len); ret = ceph_tcp_recvpage(con->sock, con->bounce_page, 0, len); if (ret <= 0) { con->in_data_crc = crc; diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index c6e5bfc717d5460fb8b4238da7484aadeee2a6d8..cc8ff81a50b7f1abe3ff4d58d924aa277ace4269 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -862,7 +862,7 @@ static void get_bvec_at(struct ceph_msg_data_cursor *cursor, ceph_msg_data_advance(cursor, 0); /* get a piece of data, cursor isn't advanced */ - page = ceph_msg_data_next(cursor, &off, &len, NULL); + page = ceph_msg_data_next(cursor, &off, &len); bv->bv_page = page; bv->bv_offset = off; diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index 6a6898ee40495d9de053396ae29cf482c0a62c66..db60217f911b31ca04a768b3dee562d1d165ca02 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -222,7 +222,7 @@ static void pick_new_mon(struct ceph_mon_client *monc) max--; } - n = prandom_u32() % max; + n = prandom_u32_max(max); if (o >= 0 && n >= o) n++; diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 87b883c7bfd6492ac95c550af64db09ea9acc612..4e4f1e4bc265a9294103131b565ac955a8f25c3f 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -1479,7 +1479,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc, static int pick_random_replica(const struct ceph_osds *acting) { - int i = prandom_u32() % acting->size; + int i = prandom_u32_max(acting->size); dout("%s picked osd%d, primary osd%d\n", __func__, acting->osds[i], acting->primary); diff --git a/net/compat.c b/net/compat.c index fe9be3c56ef71431599cfd4e22d1d85c55a3949b..385f04a6be2f7a9ef1258619c06c5fdf37b093c5 100644 --- a/net/compat.c +++ b/net/compat.c @@ -52,6 +52,7 @@ int __get_compat_msghdr(struct msghdr *kmsg, kmsg->msg_namelen = sizeof(struct sockaddr_storage); kmsg->msg_control_is_user = true; + kmsg->msg_get_inq = 0; kmsg->msg_control_user = compat_ptr(msg->msg_control); kmsg->msg_controllen = msg->msg_controllen; diff --git a/net/core/.gitignore b/net/core/.gitignore deleted file mode 100644 index df1e74372cce788f13b6e2bef19964b8cf235065..0000000000000000000000000000000000000000 --- a/net/core/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dropreason_str.c diff --git a/net/core/Makefile b/net/core/Makefile index e8ce3bd283a6363e7d9db3f2a14f8d1408a6e130..5857cec87b8394604c707eb52e9fc61fe6b81917 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -5,7 +5,7 @@ obj-y := sock.o request_sock.o skbuff.o datagram.o stream.o scm.o \ gen_stats.o gen_estimator.o net_namespace.o secure_seq.o \ - flow_dissector.o dropreason_str.o + flow_dissector.o obj-$(CONFIG_SYSCTL) += sysctl_net_core.o @@ -40,23 +40,3 @@ obj-$(CONFIG_NET_SOCK_MSG) += skmsg.o obj-$(CONFIG_BPF_SYSCALL) += sock_map.o obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o obj-$(CONFIG_OF) += of_net.o - -clean-files := dropreason_str.c - -quiet_cmd_dropreason_str = GEN $@ -cmd_dropreason_str = awk -F ',' 'BEGIN{ print "\#include \n"; \ - print "const char * const drop_reasons[] = {" }\ - /^enum skb_drop/ { dr=1; }\ - /^\};/ { dr=0; }\ - /^\tSKB_DROP_REASON_/ {\ - if (dr) {\ - sub(/\tSKB_DROP_REASON_/, "", $$1);\ - printf "\t[SKB_DROP_REASON_%s] = \"%s\",\n", $$1, $$1;\ - }\ - }\ - END{ print "};" }' $< > $@ - -$(obj)/dropreason_str.c: $(srctree)/include/net/dropreason.h - $(call cmd,dropreason_str) - -$(obj)/dropreason_str.o: $(obj)/dropreason_str.c diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 1b7f385643b4c90c02f0659e64d2f5d371f839dc..94374d529ea4211f516d2a0d10a3345fa3f3c924 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -310,11 +310,12 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk) static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { + int optmem_max = READ_ONCE(sysctl_optmem_max); struct sock *sk = (struct sock *)owner; /* same check as in sock_kmalloc() */ - if (size <= sysctl_optmem_max && - atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { + if (size <= optmem_max && + atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { atomic_add(size, &sk->sk_omem_alloc); return 0; } diff --git a/net/core/datagram.c b/net/core/datagram.c index 7255531f63ae279204bd6dd1a592ded789d4ac0e..e4ff2db40c9810a4cbbf6540ab5365d79bcdbbe0 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -677,7 +677,7 @@ int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk, page_ref_sub(last_head, refs); refs = 0; } - skb_fill_page_desc(skb, frag++, head, start, size); + skb_fill_page_desc_noacc(skb, frag++, head, start, size); } if (refs) page_ref_sub(last_head, refs); diff --git a/net/core/dev.c b/net/core/dev.c index 716df64fcfa570c29101da230aeb4ec6e644d803..fa53830d0683944f6558deefb9ae1d6d988cb73b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -1100,7 +1100,7 @@ static int dev_alloc_name_ns(struct net *net, BUG_ON(!net); ret = __dev_alloc_name(net, name, buf); if (ret >= 0) - strlcpy(dev->name, buf, IFNAMSIZ); + strscpy(dev->name, buf, IFNAMSIZ); return ret; } @@ -1137,7 +1137,7 @@ static int dev_get_valid_name(struct net *net, struct net_device *dev, else if (netdev_name_in_use(net, name)) return -EEXIST; else if (dev->name != name) - strlcpy(dev->name, name, IFNAMSIZ); + strscpy(dev->name, name, IFNAMSIZ); return 0; } @@ -4624,7 +4624,7 @@ static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) struct softnet_data *sd; unsigned int old_flow, new_flow; - if (qlen < (netdev_max_backlog >> 1)) + if (qlen < (READ_ONCE(netdev_max_backlog) >> 1)) return false; sd = this_cpu_ptr(&softnet_data); @@ -4672,7 +4672,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, if (!netif_running(skb->dev)) goto drop; qlen = skb_queue_len(&sd->input_pkt_queue); - if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { + if (qlen <= READ_ONCE(netdev_max_backlog) && !skb_flow_limit(skb, qlen)) { if (qlen) { enqueue: __skb_queue_tail(&sd->input_pkt_queue, skb); @@ -4928,7 +4928,7 @@ static int netif_rx_internal(struct sk_buff *skb) { int ret; - net_timestamp_check(netdev_tstamp_prequeue, skb); + net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb); trace_netif_rx(skb); @@ -5281,7 +5281,7 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, int ret = NET_RX_DROP; __be16 type; - net_timestamp_check(!netdev_tstamp_prequeue, skb); + net_timestamp_check(!READ_ONCE(netdev_tstamp_prequeue), skb); trace_netif_receive_skb(skb); @@ -5664,7 +5664,7 @@ static int netif_receive_skb_internal(struct sk_buff *skb) { int ret; - net_timestamp_check(netdev_tstamp_prequeue, skb); + net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb); if (skb_defer_rx_timestamp(skb)) return NET_RX_SUCCESS; @@ -5694,7 +5694,7 @@ void netif_receive_skb_list_internal(struct list_head *head) INIT_LIST_HEAD(&sublist); list_for_each_entry_safe(skb, next, head, list) { - net_timestamp_check(netdev_tstamp_prequeue, skb); + net_timestamp_check(READ_ONCE(netdev_tstamp_prequeue), skb); skb_list_del_init(skb); if (!skb_defer_rx_timestamp(skb)) list_add_tail(&skb->list, &sublist); @@ -5918,7 +5918,7 @@ static int process_backlog(struct napi_struct *napi, int quota) net_rps_action_and_irq_enable(sd); } - napi->weight = dev_rx_weight; + napi->weight = READ_ONCE(dev_rx_weight); while (again) { struct sk_buff *skb; @@ -6358,23 +6358,6 @@ int dev_set_threaded(struct net_device *dev, bool threaded) } EXPORT_SYMBOL(dev_set_threaded); -/* Double check that napi_get_frags() allocates skbs with - * skb->head being backed by slab, not a page fragment. - * This is to make sure bug fixed in 3226b158e67c - * ("net: avoid 32 x truesize under-estimation for tiny skbs") - * does not accidentally come back. - */ -static void napi_get_frags_check(struct napi_struct *napi) -{ - struct sk_buff *skb; - - local_bh_disable(); - skb = napi_get_frags(napi); - WARN_ON_ONCE(skb && skb->head_frag); - napi_free_frags(napi); - local_bh_enable(); -} - void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi, int (*poll)(struct napi_struct *, int), int weight) { @@ -6665,8 +6648,8 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) { struct softnet_data *sd = this_cpu_ptr(&softnet_data); unsigned long time_limit = jiffies + - usecs_to_jiffies(netdev_budget_usecs); - int budget = netdev_budget; + usecs_to_jiffies(READ_ONCE(netdev_budget_usecs)); + int budget = READ_ONCE(netdev_budget); LIST_HEAD(list); LIST_HEAD(repoll); @@ -10284,7 +10267,7 @@ static struct net_device *netdev_wait_allrefs_any(struct list_head *list) return dev; if (time_after(jiffies, warning_time + - netdev_unregister_timeout_secs * HZ)) { + READ_ONCE(netdev_unregister_timeout_secs) * HZ)) { list_for_each_entry(dev, list, todo_list) { pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", dev->name, netdev_refcnt_read(dev)); @@ -10370,9 +10353,7 @@ void netdev_run_todo(void) BUG_ON(!list_empty(&dev->ptype_specific)); WARN_ON(rcu_access_pointer(dev->ip_ptr)); WARN_ON(rcu_access_pointer(dev->ip6_ptr)); -#if IS_ENABLED(CONFIG_DECNET) - WARN_ON(dev->dn_ptr); -#endif + if (dev->priv_destructor) dev->priv_destructor(dev); if (dev->needs_free_netdev) diff --git a/net/core/devlink.c b/net/core/devlink.c index b50bcc18b8d9e034f8706bdf1afe6b7aa2c2395d..89baa7c0938b9580b322070da5069086cabd3bc7 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -371,6 +371,13 @@ static struct devlink *devlink_get_from_attrs(struct net *net, return ERR_PTR(-ENODEV); } +#define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port) \ + WARN_ON_ONCE(!(devlink_port)->registered) +#define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port) \ + WARN_ON_ONCE((devlink_port)->registered) +#define ASSERT_DEVLINK_PORT_INITIALIZED(devlink_port) \ + WARN_ON_ONCE(!(devlink_port)->initialized) + static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink, unsigned int port_index) { @@ -1710,7 +1717,7 @@ static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb, struct devlink *devlink = info->user_ptr[0]; u32 count; - if (!info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT)) return -EINVAL; if (!devlink->ops->port_split) return -EOPNOTSUPP; @@ -1838,7 +1845,7 @@ static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb, if (!devlink->ops->port_del) return -EOPNOTSUPP; - if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) { + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_INDEX)) { NL_SET_ERR_MSG_MOD(extack, "Port index is not specified"); return -EINVAL; } @@ -2690,7 +2697,7 @@ static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb, if (err) return err; - if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_POOL_SIZE)) return -EINVAL; size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]); @@ -2900,7 +2907,7 @@ static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb, if (err) return err; - if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD)) return -EINVAL; threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]); @@ -3156,7 +3163,7 @@ static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb, if (err) return err; - if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD)) return -EINVAL; threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]); @@ -3845,7 +3852,7 @@ static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb, struct devlink_dpipe_table *table; const char *table_name; - if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME)) return -EINVAL; table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]); @@ -4029,8 +4036,9 @@ static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb, const char *table_name; bool counters_enable; - if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] || - !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME) || + GENL_REQ_ATTR_CHECK(info, + DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED)) return -EINVAL; table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]); @@ -4119,8 +4127,8 @@ static int devlink_nl_cmd_resource_set(struct sk_buff *skb, u64 size; int err; - if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] || - !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_ID) || + GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_SIZE)) return -EINVAL; resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]); @@ -4742,10 +4750,76 @@ void devlink_flash_update_timeout_notify(struct devlink *devlink, } EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify); +struct devlink_info_req { + struct sk_buff *msg; + void (*version_cb)(const char *version_name, + enum devlink_info_version_type version_type, + void *version_cb_priv); + void *version_cb_priv; +}; + +struct devlink_flash_component_lookup_ctx { + const char *lookup_name; + bool lookup_name_found; +}; + +static void +devlink_flash_component_lookup_cb(const char *version_name, + enum devlink_info_version_type version_type, + void *version_cb_priv) +{ + struct devlink_flash_component_lookup_ctx *lookup_ctx = version_cb_priv; + + if (version_type != DEVLINK_INFO_VERSION_TYPE_COMPONENT || + lookup_ctx->lookup_name_found) + return; + + lookup_ctx->lookup_name_found = + !strcmp(lookup_ctx->lookup_name, version_name); +} + +static int devlink_flash_component_get(struct devlink *devlink, + struct nlattr *nla_component, + const char **p_component, + struct netlink_ext_ack *extack) +{ + struct devlink_flash_component_lookup_ctx lookup_ctx = {}; + struct devlink_info_req req = {}; + const char *component; + int ret; + + if (!nla_component) + return 0; + + component = nla_data(nla_component); + + if (!devlink->ops->info_get) { + NL_SET_ERR_MSG_ATTR(extack, nla_component, + "component update is not supported by this device"); + return -EOPNOTSUPP; + } + + lookup_ctx.lookup_name = component; + req.version_cb = devlink_flash_component_lookup_cb; + req.version_cb_priv = &lookup_ctx; + + ret = devlink->ops->info_get(devlink, &req, NULL); + if (ret) + return ret; + + if (!lookup_ctx.lookup_name_found) { + NL_SET_ERR_MSG_ATTR(extack, nla_component, + "selected component is not supported by this device"); + return -EINVAL; + } + *p_component = component; + return 0; +} + static int devlink_nl_cmd_flash_update(struct sk_buff *skb, struct genl_info *info) { - struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name; + struct nlattr *nla_overwrite_mask, *nla_file_name; struct devlink_flash_update_params params = {}; struct devlink *devlink = info->user_ptr[0]; const char *file_name; @@ -4755,20 +4829,16 @@ static int devlink_nl_cmd_flash_update(struct sk_buff *skb, if (!devlink->ops->flash_update) return -EOPNOTSUPP; - if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME)) return -EINVAL; - supported_params = devlink->ops->supported_flash_update_params; + ret = devlink_flash_component_get(devlink, + info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT], + ¶ms.component, info->extack); + if (ret) + return ret; - nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT]; - if (nla_component) { - if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) { - NL_SET_ERR_MSG_ATTR(info->extack, nla_component, - "component update is not supported by this device"); - return -EOPNOTSUPP; - } - params.component = nla_data(nla_component); - } + supported_params = devlink->ops->supported_flash_update_params; nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK]; if (nla_overwrite_mask) { @@ -4936,10 +5006,8 @@ static int devlink_nl_cmd_selftests_run(struct sk_buff *skb, if (!devlink->ops->selftest_run || !devlink->ops->selftest_check) return -EOPNOTSUPP; - if (!info->attrs[DEVLINK_ATTR_SELFTESTS]) { - NL_SET_ERR_MSG_MOD(info->extack, "selftest required"); + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SELFTESTS)) return -EINVAL; - } attrs = info->attrs[DEVLINK_ATTR_SELFTESTS]; @@ -5393,7 +5461,7 @@ static int devlink_param_type_get_from_info(struct genl_info *info, enum devlink_param_type *param_type) { - if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_TYPE)) return -EINVAL; switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) { @@ -5470,7 +5538,7 @@ devlink_param_get_from_info(struct list_head *param_list, { char *param_name; - if (!info->attrs[DEVLINK_ATTR_PARAM_NAME]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_NAME)) return NULL; param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]); @@ -5536,7 +5604,7 @@ static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink, return err; } - if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_VALUE_CMODE)) return -EINVAL; cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]); if (!devlink_param_cmode_is_supported(param, cmode)) @@ -5574,89 +5642,22 @@ static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb, static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) { - struct devlink_param_item *param_item; - struct devlink_port *devlink_port; - struct devlink *devlink; - int start = cb->args[0]; - unsigned long index; - int idx = 0; - int err = 0; - - devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) { - devl_lock(devlink); - list_for_each_entry(devlink_port, &devlink->port_list, list) { - list_for_each_entry(param_item, - &devlink_port->param_list, list) { - if (idx < start) { - idx++; - continue; - } - err = devlink_nl_param_fill(msg, - devlink_port->devlink, - devlink_port->index, param_item, - DEVLINK_CMD_PORT_PARAM_GET, - NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, - NLM_F_MULTI); - if (err == -EOPNOTSUPP) { - err = 0; - } else if (err) { - devl_unlock(devlink); - devlink_put(devlink); - goto out; - } - idx++; - } - } - devl_unlock(devlink); - devlink_put(devlink); - } -out: - if (err != -EMSGSIZE) - return err; - - cb->args[0] = idx; + NL_SET_ERR_MSG_MOD(cb->extack, "Port params are not supported"); return msg->len; } static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb, struct genl_info *info) { - struct devlink_port *devlink_port = info->user_ptr[1]; - struct devlink_param_item *param_item; - struct sk_buff *msg; - int err; - - param_item = devlink_param_get_from_info(&devlink_port->param_list, - info); - if (!param_item) - return -EINVAL; - - msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (!msg) - return -ENOMEM; - - err = devlink_nl_param_fill(msg, devlink_port->devlink, - devlink_port->index, param_item, - DEVLINK_CMD_PORT_PARAM_GET, - info->snd_portid, info->snd_seq, 0); - if (err) { - nlmsg_free(msg); - return err; - } - - return genlmsg_reply(msg, info); + NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported"); + return -EINVAL; } static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb, struct genl_info *info) { - struct devlink_port *devlink_port = info->user_ptr[1]; - - return __devlink_nl_cmd_param_set_doit(devlink_port->devlink, - devlink_port->index, - &devlink_port->param_list, info, - DEVLINK_CMD_PORT_PARAM_NEW); + NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported"); + return -EINVAL; } static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg, @@ -6056,7 +6057,7 @@ static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb, unsigned int index; int err; - if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME)) return -EINVAL; if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) { @@ -6189,8 +6190,8 @@ static int devlink_nl_cmd_region_del(struct sk_buff *skb, unsigned int index; u32 snapshot_id; - if (!info->attrs[DEVLINK_ATTR_REGION_NAME] || - !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID]) + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME) || + GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_SNAPSHOT_ID)) return -EINVAL; region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]); @@ -6238,7 +6239,7 @@ devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info) u8 *data; int err; - if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) { + if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME)) { NL_SET_ERR_MSG_MOD(info->extack, "No region name provided"); return -EINVAL; } @@ -6553,18 +6554,18 @@ out_unlock: return err; } -struct devlink_info_req { - struct sk_buff *msg; -}; - int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name) { + if (!req->msg) + return 0; return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name); } EXPORT_SYMBOL_GPL(devlink_info_driver_name_put); int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn) { + if (!req->msg) + return 0; return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn); } EXPORT_SYMBOL_GPL(devlink_info_serial_number_put); @@ -6572,6 +6573,8 @@ EXPORT_SYMBOL_GPL(devlink_info_serial_number_put); int devlink_info_board_serial_number_put(struct devlink_info_req *req, const char *bsn) { + if (!req->msg) + return 0; return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER, bsn); } @@ -6579,11 +6582,19 @@ EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put); static int devlink_info_version_put(struct devlink_info_req *req, int attr, const char *version_name, - const char *version_value) + const char *version_value, + enum devlink_info_version_type version_type) { struct nlattr *nest; int err; + if (req->version_cb) + req->version_cb(version_name, version_type, + req->version_cb_priv); + + if (!req->msg) + return 0; + nest = nla_nest_start_noflag(req->msg, attr); if (!nest) return -EMSGSIZE; @@ -6612,7 +6623,8 @@ int devlink_info_version_fixed_put(struct devlink_info_req *req, const char *version_value) { return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED, - version_name, version_value); + version_name, version_value, + DEVLINK_INFO_VERSION_TYPE_NONE); } EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put); @@ -6621,25 +6633,49 @@ int devlink_info_version_stored_put(struct devlink_info_req *req, const char *version_value) { return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED, - version_name, version_value); + version_name, version_value, + DEVLINK_INFO_VERSION_TYPE_NONE); } EXPORT_SYMBOL_GPL(devlink_info_version_stored_put); +int devlink_info_version_stored_put_ext(struct devlink_info_req *req, + const char *version_name, + const char *version_value, + enum devlink_info_version_type version_type) +{ + return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED, + version_name, version_value, + version_type); +} +EXPORT_SYMBOL_GPL(devlink_info_version_stored_put_ext); + int devlink_info_version_running_put(struct devlink_info_req *req, const char *version_name, const char *version_value) { return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING, - version_name, version_value); + version_name, version_value, + DEVLINK_INFO_VERSION_TYPE_NONE); } EXPORT_SYMBOL_GPL(devlink_info_version_running_put); +int devlink_info_version_running_put_ext(struct devlink_info_req *req, + const char *version_name, + const char *version_value, + enum devlink_info_version_type version_type) +{ + return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING, + version_name, version_value, + version_type); +} +EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext); + static int devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink, enum devlink_command cmd, u32 portid, u32 seq, int flags, struct netlink_ext_ack *extack) { - struct devlink_info_req req; + struct devlink_info_req req = {}; void *hdr; int err; @@ -9513,6 +9549,7 @@ static struct genl_family devlink_nl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = devlink_nl_ops, .n_small_ops = ARRAY_SIZE(devlink_nl_ops), + .resv_start_op = DEVLINK_CMD_SELFTESTS_RUN + 1, .mcgrps = devlink_nl_mcgrps, .n_mcgrps = ARRAY_SIZE(devlink_nl_mcgrps), }; @@ -9817,6 +9854,44 @@ static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port) cancel_delayed_work_sync(&devlink_port->type_warn_dw); } +/** + * devlink_port_init() - Init devlink port + * + * @devlink: devlink + * @devlink_port: devlink port + * + * Initialize essencial stuff that is needed for functions + * that may be called before devlink port registration. + * Call to this function is optional and not needed + * in case the driver does not use such functions. + */ +void devlink_port_init(struct devlink *devlink, + struct devlink_port *devlink_port) +{ + if (devlink_port->initialized) + return; + devlink_port->devlink = devlink; + INIT_LIST_HEAD(&devlink_port->region_list); + devlink_port->initialized = true; +} +EXPORT_SYMBOL_GPL(devlink_port_init); + +/** + * devlink_port_fini() - Deinitialize devlink port + * + * @devlink_port: devlink port + * + * Deinitialize essencial stuff that is in use for functions + * that may be called after devlink port unregistration. + * Call to this function is optional and not needed + * in case the driver does not use such functions. + */ +void devlink_port_fini(struct devlink_port *devlink_port) +{ + WARN_ON(!list_empty(&devlink_port->region_list)); +} +EXPORT_SYMBOL_GPL(devlink_port_fini); + /** * devl_port_register() - Register devlink port * @@ -9839,15 +9914,15 @@ int devl_port_register(struct devlink *devlink, if (devlink_port_index_exists(devlink, port_index)) return -EEXIST; - WARN_ON(devlink_port->devlink); - devlink_port->devlink = devlink; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + + devlink_port_init(devlink, devlink_port); + devlink_port->registered = true; devlink_port->index = port_index; spin_lock_init(&devlink_port->type_lock); INIT_LIST_HEAD(&devlink_port->reporter_list); mutex_init(&devlink_port->reporters_lock); list_add_tail(&devlink_port->list, &devlink->port_list); - INIT_LIST_HEAD(&devlink_port->param_list); - INIT_LIST_HEAD(&devlink_port->region_list); INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn); devlink_port_type_warn_schedule(devlink_port); @@ -9897,8 +9972,8 @@ void devl_port_unregister(struct devlink_port *devlink_port) devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL); list_del(&devlink_port->list); WARN_ON(!list_empty(&devlink_port->reporter_list)); - WARN_ON(!list_empty(&devlink_port->region_list)); mutex_destroy(&devlink_port->reporters_lock); + devlink_port->registered = false; } EXPORT_SYMBOL_GPL(devl_port_unregister); @@ -9923,8 +9998,8 @@ static void __devlink_port_type_set(struct devlink_port *devlink_port, enum devlink_port_type type, void *type_dev) { - if (WARN_ON(!devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_REGISTERED(devlink_port); + devlink_port_type_warn_cancel(devlink_port); spin_lock_bh(&devlink_port->type_lock); devlink_port->type = type; @@ -10043,8 +10118,8 @@ void devlink_port_attrs_set(struct devlink_port *devlink_port, { int ret; - if (WARN_ON(devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + devlink_port->attrs = *attrs; ret = __devlink_port_attrs_set(devlink_port, attrs->flavour); if (ret) @@ -10067,8 +10142,8 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro struct devlink_port_attrs *attrs = &devlink_port->attrs; int ret; - if (WARN_ON(devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + ret = __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_PF); if (ret) @@ -10094,8 +10169,8 @@ void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 contro struct devlink_port_attrs *attrs = &devlink_port->attrs; int ret; - if (WARN_ON(devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + ret = __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_VF); if (ret) @@ -10122,8 +10197,8 @@ void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 contro struct devlink_port_attrs *attrs = &devlink_port->attrs; int ret; - if (WARN_ON(devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + ret = __devlink_port_attrs_set(devlink_port, DEVLINK_PORT_FLAVOUR_PCI_SF); if (ret) @@ -10238,8 +10313,8 @@ EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy); void devlink_port_linecard_set(struct devlink_port *devlink_port, struct devlink_linecard *linecard) { - if (WARN_ON(devlink_port->devlink)) - return; + ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port); + devlink_port->linecard = linecard; } EXPORT_SYMBOL_GPL(devlink_port_linecard_set); @@ -11310,6 +11385,8 @@ devlink_port_region_create(struct devlink_port *port, struct devlink_region *region; int err = 0; + ASSERT_DEVLINK_PORT_INITIALIZED(port); + if (WARN_ON(!ops) || WARN_ON(!ops->destructor)) return ERR_PTR(-EINVAL); @@ -12306,8 +12383,8 @@ EXPORT_SYMBOL_GPL(devl_trap_policers_unregister); static void __devlink_compat_running_version(struct devlink *devlink, char *buf, size_t len) { + struct devlink_info_req req = {}; const struct nlattr *nlattr; - struct devlink_info_req req; struct sk_buff *msg; int rem, err; diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 75501e1bdd25b3d2c04a512351c817e6a921f718..f084a4a6b7ab2c7e5b822be89a34d3c4dd2cc0f6 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -464,7 +464,7 @@ net_dm_hw_trap_summary_probe(void *ignore, const struct devlink *devlink, goto out; hw_entry = &hw_entries->entries[hw_entries->num_entries]; - strlcpy(hw_entry->trap_name, metadata->trap_name, + strscpy(hw_entry->trap_name, metadata->trap_name, NET_DM_MAX_HW_TRAP_NAME_LEN - 1); hw_entry->count = 1; hw_entries->num_entries++; @@ -1645,6 +1645,7 @@ static struct genl_family net_drop_monitor_family __ro_after_init = { .module = THIS_MODULE, .small_ops = dropmon_ops, .n_small_ops = ARRAY_SIZE(dropmon_ops), + .resv_start_op = NET_DM_CMD_STATS_GET + 1, .mcgrps = dropmon_mcgrps, .n_mcgrps = ARRAY_SIZE(dropmon_mcgrps), }; diff --git a/net/core/filter.c b/net/core/filter.c index e8508aaafd27d75c6cbf872eeeb60e608ecdcd6f..bb0136e7a8e422b010477333c811db79aaf56e3f 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -1214,10 +1215,11 @@ void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp) static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp) { u32 filter_size = bpf_prog_size(fp->prog->len); + int optmem_max = READ_ONCE(sysctl_optmem_max); /* same check as in sock_kmalloc() */ - if (filter_size <= sysctl_optmem_max && - atomic_read(&sk->sk_omem_alloc) + filter_size < sysctl_optmem_max) { + if (filter_size <= optmem_max && + atomic_read(&sk->sk_omem_alloc) + filter_size < optmem_max) { atomic_add(filter_size, &sk->sk_omem_alloc); return true; } @@ -1548,7 +1550,7 @@ int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk) if (IS_ERR(prog)) return PTR_ERR(prog); - if (bpf_prog_size(prog->len) > sysctl_optmem_max) + if (bpf_prog_size(prog->len) > READ_ONCE(sysctl_optmem_max)) err = -ENOMEM; else err = reuseport_attach_prog(sk, prog); @@ -1615,7 +1617,7 @@ int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk) } } else { /* BPF_PROG_TYPE_SOCKET_FILTER */ - if (bpf_prog_size(prog->len) > sysctl_optmem_max) { + if (bpf_prog_size(prog->len) > READ_ONCE(sysctl_optmem_max)) { err = -ENOMEM; goto err_prog_put; } @@ -3009,7 +3011,7 @@ BPF_CALL_0(bpf_get_cgroup_classid_curr) return __task_get_classid(current); } -static const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = { +const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = { .func = bpf_get_cgroup_classid_curr, .gpl_only = false, .ret_type = RET_INTEGER, @@ -4488,7 +4490,8 @@ BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key void *to_orig = to; int err; - if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6)))) { + if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6 | + BPF_F_TUNINFO_FLAGS)))) { err = -EINVAL; goto err_clear; } @@ -4520,7 +4523,10 @@ set_compat: to->tunnel_id = be64_to_cpu(info->key.tun_id); to->tunnel_tos = info->key.tos; to->tunnel_ttl = info->key.ttl; - to->tunnel_ext = 0; + if (flags & BPF_F_TUNINFO_FLAGS) + to->tunnel_flags = info->key.tun_flags; + else + to->tunnel_ext = 0; if (flags & BPF_F_TUNINFO_IPV6) { memcpy(to->remote_ipv6, &info->key.u.ipv6.src, @@ -5013,359 +5019,303 @@ static const struct bpf_func_proto bpf_get_socket_uid_proto = { .arg1_type = ARG_PTR_TO_CTX, }; -static int __bpf_setsockopt(struct sock *sk, int level, int optname, - char *optval, int optlen) +static int sol_socket_sockopt(struct sock *sk, int optname, + char *optval, int *optlen, + bool getopt) +{ + switch (optname) { + case SO_REUSEADDR: + case SO_SNDBUF: + case SO_RCVBUF: + case SO_KEEPALIVE: + case SO_PRIORITY: + case SO_REUSEPORT: + case SO_RCVLOWAT: + case SO_MARK: + case SO_MAX_PACING_RATE: + case SO_BINDTOIFINDEX: + case SO_TXREHASH: + if (*optlen != sizeof(int)) + return -EINVAL; + break; + case SO_BINDTODEVICE: + break; + default: + return -EINVAL; + } + + if (getopt) { + if (optname == SO_BINDTODEVICE) + return -EINVAL; + return sk_getsockopt(sk, SOL_SOCKET, optname, + KERNEL_SOCKPTR(optval), + KERNEL_SOCKPTR(optlen)); + } + + return sk_setsockopt(sk, SOL_SOCKET, optname, + KERNEL_SOCKPTR(optval), *optlen); +} + +static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname, + char *optval, int optlen) { - char devname[IFNAMSIZ]; - int val, valbool; - struct net *net; - int ifindex; - int ret = 0; + struct tcp_sock *tp = tcp_sk(sk); + unsigned long timeout; + int val; - if (!sk_fullsock(sk)) + if (optlen != sizeof(int)) return -EINVAL; - if (level == SOL_SOCKET) { - if (optlen != sizeof(int) && optname != SO_BINDTODEVICE) + val = *(int *)optval; + + /* Only some options are supported */ + switch (optname) { + case TCP_BPF_IW: + if (val <= 0 || tp->data_segs_out > tp->syn_data) return -EINVAL; - val = *((int *)optval); - valbool = val ? 1 : 0; - - /* Only some socketops are supported */ - switch (optname) { - case SO_RCVBUF: - val = min_t(u32, val, sysctl_rmem_max); - val = min_t(int, val, INT_MAX / 2); - sk->sk_userlocks |= SOCK_RCVBUF_LOCK; - WRITE_ONCE(sk->sk_rcvbuf, - max_t(int, val * 2, SOCK_MIN_RCVBUF)); - break; - case SO_SNDBUF: - val = min_t(u32, val, sysctl_wmem_max); - val = min_t(int, val, INT_MAX / 2); - sk->sk_userlocks |= SOCK_SNDBUF_LOCK; - WRITE_ONCE(sk->sk_sndbuf, - max_t(int, val * 2, SOCK_MIN_SNDBUF)); - break; - case SO_MAX_PACING_RATE: /* 32bit version */ - if (val != ~0U) - cmpxchg(&sk->sk_pacing_status, - SK_PACING_NONE, - SK_PACING_NEEDED); - sk->sk_max_pacing_rate = (val == ~0U) ? - ~0UL : (unsigned int)val; - sk->sk_pacing_rate = min(sk->sk_pacing_rate, - sk->sk_max_pacing_rate); - break; - case SO_PRIORITY: - sk->sk_priority = val; - break; - case SO_RCVLOWAT: - if (val < 0) - val = INT_MAX; - if (sk->sk_socket && sk->sk_socket->ops->set_rcvlowat) - ret = sk->sk_socket->ops->set_rcvlowat(sk, val); - else - WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); - break; - case SO_MARK: - if (sk->sk_mark != val) { - sk->sk_mark = val; - sk_dst_reset(sk); - } - break; - case SO_BINDTODEVICE: - optlen = min_t(long, optlen, IFNAMSIZ - 1); - strncpy(devname, optval, optlen); - devname[optlen] = 0; + tcp_snd_cwnd_set(tp, val); + break; + case TCP_BPF_SNDCWND_CLAMP: + if (val <= 0) + return -EINVAL; + tp->snd_cwnd_clamp = val; + tp->snd_ssthresh = val; + break; + case TCP_BPF_DELACK_MAX: + timeout = usecs_to_jiffies(val); + if (timeout > TCP_DELACK_MAX || + timeout < TCP_TIMEOUT_MIN) + return -EINVAL; + inet_csk(sk)->icsk_delack_max = timeout; + break; + case TCP_BPF_RTO_MIN: + timeout = usecs_to_jiffies(val); + if (timeout > TCP_RTO_MIN || + timeout < TCP_TIMEOUT_MIN) + return -EINVAL; + inet_csk(sk)->icsk_rto_min = timeout; + break; + default: + return -EINVAL; + } - ifindex = 0; - if (devname[0] != '\0') { - struct net_device *dev; + return 0; +} - ret = -ENODEV; +static int sol_tcp_sockopt_congestion(struct sock *sk, char *optval, + int *optlen, bool getopt) +{ + struct tcp_sock *tp; + int ret; - net = sock_net(sk); - dev = dev_get_by_name(net, devname); - if (!dev) - break; - ifindex = dev->ifindex; - dev_put(dev); - } - fallthrough; - case SO_BINDTOIFINDEX: - if (optname == SO_BINDTOIFINDEX) - ifindex = val; - ret = sock_bindtoindex(sk, ifindex, false); - break; - case SO_KEEPALIVE: - if (sk->sk_prot->keepalive) - sk->sk_prot->keepalive(sk, valbool); - sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool); - break; - case SO_REUSEPORT: - sk->sk_reuseport = valbool; - break; - case SO_TXREHASH: - if (val < -1 || val > 1) { - ret = -EINVAL; - break; - } - sk->sk_txrehash = (u8)val; - break; - default: - ret = -EINVAL; - } -#ifdef CONFIG_INET - } else if (level == SOL_IP) { - if (optlen != sizeof(int) || sk->sk_family != AF_INET) + if (*optlen < 2) + return -EINVAL; + + if (getopt) { + if (!inet_csk(sk)->icsk_ca_ops) return -EINVAL; + /* BPF expects NULL-terminated tcp-cc string */ + optval[--(*optlen)] = '\0'; + return do_tcp_getsockopt(sk, SOL_TCP, TCP_CONGESTION, + KERNEL_SOCKPTR(optval), + KERNEL_SOCKPTR(optlen)); + } - val = *((int *)optval); - /* Only some options are supported */ - switch (optname) { - case IP_TOS: - if (val < -1 || val > 0xff) { - ret = -EINVAL; - } else { - struct inet_sock *inet = inet_sk(sk); + /* "cdg" is the only cc that alloc a ptr + * in inet_csk_ca area. The bpf-tcp-cc may + * overwrite this ptr after switching to cdg. + */ + if (*optlen >= sizeof("cdg") - 1 && !strncmp("cdg", optval, *optlen)) + return -ENOTSUPP; - if (val == -1) - val = 0; - inet->tos = val; - } - break; - default: - ret = -EINVAL; - } -#if IS_ENABLED(CONFIG_IPV6) - } else if (level == SOL_IPV6) { - if (optlen != sizeof(int) || sk->sk_family != AF_INET6) - return -EINVAL; + /* It stops this looping + * + * .init => bpf_setsockopt(tcp_cc) => .init => + * bpf_setsockopt(tcp_cc)" => .init => .... + * + * The second bpf_setsockopt(tcp_cc) is not allowed + * in order to break the loop when both .init + * are the same bpf prog. + * + * This applies even the second bpf_setsockopt(tcp_cc) + * does not cause a loop. This limits only the first + * '.init' can call bpf_setsockopt(TCP_CONGESTION) to + * pick a fallback cc (eg. peer does not support ECN) + * and the second '.init' cannot fallback to + * another. + */ + tp = tcp_sk(sk); + if (tp->bpf_chg_cc_inprogress) + return -EBUSY; + + tp->bpf_chg_cc_inprogress = 1; + ret = do_tcp_setsockopt(sk, SOL_TCP, TCP_CONGESTION, + KERNEL_SOCKPTR(optval), *optlen); + tp->bpf_chg_cc_inprogress = 0; + return ret; +} - val = *((int *)optval); - /* Only some options are supported */ - switch (optname) { - case IPV6_TCLASS: - if (val < -1 || val > 0xff) { - ret = -EINVAL; - } else { - struct ipv6_pinfo *np = inet6_sk(sk); +static int sol_tcp_sockopt(struct sock *sk, int optname, + char *optval, int *optlen, + bool getopt) +{ + if (sk->sk_prot->setsockopt != tcp_setsockopt) + return -EINVAL; - if (val == -1) - val = 0; - np->tclass = val; - } - break; - default: - ret = -EINVAL; - } -#endif - } else if (level == SOL_TCP && - sk->sk_prot->setsockopt == tcp_setsockopt) { - if (optname == TCP_CONGESTION) { - char name[TCP_CA_NAME_MAX]; - - strncpy(name, optval, min_t(long, optlen, - TCP_CA_NAME_MAX-1)); - name[TCP_CA_NAME_MAX-1] = 0; - ret = tcp_set_congestion_control(sk, name, false, true); - } else { - struct inet_connection_sock *icsk = inet_csk(sk); + switch (optname) { + case TCP_NODELAY: + case TCP_MAXSEG: + case TCP_KEEPIDLE: + case TCP_KEEPINTVL: + case TCP_KEEPCNT: + case TCP_SYNCNT: + case TCP_WINDOW_CLAMP: + case TCP_THIN_LINEAR_TIMEOUTS: + case TCP_USER_TIMEOUT: + case TCP_NOTSENT_LOWAT: + case TCP_SAVE_SYN: + if (*optlen != sizeof(int)) + return -EINVAL; + break; + case TCP_CONGESTION: + return sol_tcp_sockopt_congestion(sk, optval, optlen, getopt); + case TCP_SAVED_SYN: + if (*optlen < 1) + return -EINVAL; + break; + default: + if (getopt) + return -EINVAL; + return bpf_sol_tcp_setsockopt(sk, optname, optval, *optlen); + } + + if (getopt) { + if (optname == TCP_SAVED_SYN) { struct tcp_sock *tp = tcp_sk(sk); - unsigned long timeout; - if (optlen != sizeof(int)) + if (!tp->saved_syn || + *optlen > tcp_saved_syn_len(tp->saved_syn)) return -EINVAL; - - val = *((int *)optval); - /* Only some options are supported */ - switch (optname) { - case TCP_BPF_IW: - if (val <= 0 || tp->data_segs_out > tp->syn_data) - ret = -EINVAL; - else - tcp_snd_cwnd_set(tp, val); - break; - case TCP_BPF_SNDCWND_CLAMP: - if (val <= 0) { - ret = -EINVAL; - } else { - tp->snd_cwnd_clamp = val; - tp->snd_ssthresh = val; - } - break; - case TCP_BPF_DELACK_MAX: - timeout = usecs_to_jiffies(val); - if (timeout > TCP_DELACK_MAX || - timeout < TCP_TIMEOUT_MIN) - return -EINVAL; - inet_csk(sk)->icsk_delack_max = timeout; - break; - case TCP_BPF_RTO_MIN: - timeout = usecs_to_jiffies(val); - if (timeout > TCP_RTO_MIN || - timeout < TCP_TIMEOUT_MIN) - return -EINVAL; - inet_csk(sk)->icsk_rto_min = timeout; - break; - case TCP_SAVE_SYN: - if (val < 0 || val > 1) - ret = -EINVAL; - else - tp->save_syn = val; - break; - case TCP_KEEPIDLE: - ret = tcp_sock_set_keepidle_locked(sk, val); - break; - case TCP_KEEPINTVL: - if (val < 1 || val > MAX_TCP_KEEPINTVL) - ret = -EINVAL; - else - tp->keepalive_intvl = val * HZ; - break; - case TCP_KEEPCNT: - if (val < 1 || val > MAX_TCP_KEEPCNT) - ret = -EINVAL; - else - tp->keepalive_probes = val; - break; - case TCP_SYNCNT: - if (val < 1 || val > MAX_TCP_SYNCNT) - ret = -EINVAL; - else - icsk->icsk_syn_retries = val; - break; - case TCP_USER_TIMEOUT: - if (val < 0) - ret = -EINVAL; - else - icsk->icsk_user_timeout = val; - break; - case TCP_NOTSENT_LOWAT: - tp->notsent_lowat = val; - sk->sk_write_space(sk); - break; - case TCP_WINDOW_CLAMP: - ret = tcp_set_window_clamp(sk, val); - break; - default: - ret = -EINVAL; - } + memcpy(optval, tp->saved_syn->data, *optlen); + /* It cannot free tp->saved_syn here because it + * does not know if the user space still needs it. + */ + return 0; } -#endif - } else { - ret = -EINVAL; + + return do_tcp_getsockopt(sk, SOL_TCP, optname, + KERNEL_SOCKPTR(optval), + KERNEL_SOCKPTR(optlen)); } - return ret; + + return do_tcp_setsockopt(sk, SOL_TCP, optname, + KERNEL_SOCKPTR(optval), *optlen); } -static int _bpf_setsockopt(struct sock *sk, int level, int optname, - char *optval, int optlen) +static int sol_ip_sockopt(struct sock *sk, int optname, + char *optval, int *optlen, + bool getopt) { - if (sk_fullsock(sk)) - sock_owned_by_me(sk); - return __bpf_setsockopt(sk, level, optname, optval, optlen); + if (sk->sk_family != AF_INET) + return -EINVAL; + + switch (optname) { + case IP_TOS: + if (*optlen != sizeof(int)) + return -EINVAL; + break; + default: + return -EINVAL; + } + + if (getopt) + return do_ip_getsockopt(sk, SOL_IP, optname, + KERNEL_SOCKPTR(optval), + KERNEL_SOCKPTR(optlen)); + + return do_ip_setsockopt(sk, SOL_IP, optname, + KERNEL_SOCKPTR(optval), *optlen); } -static int __bpf_getsockopt(struct sock *sk, int level, int optname, - char *optval, int optlen) +static int sol_ipv6_sockopt(struct sock *sk, int optname, + char *optval, int *optlen, + bool getopt) { - if (!sk_fullsock(sk)) - goto err_clear; + if (sk->sk_family != AF_INET6) + return -EINVAL; - if (level == SOL_SOCKET) { - if (optlen != sizeof(int)) - goto err_clear; + switch (optname) { + case IPV6_TCLASS: + case IPV6_AUTOFLOWLABEL: + if (*optlen != sizeof(int)) + return -EINVAL; + break; + default: + return -EINVAL; + } - switch (optname) { - case SO_RCVBUF: - *((int *)optval) = sk->sk_rcvbuf; - break; - case SO_SNDBUF: - *((int *)optval) = sk->sk_sndbuf; - break; - case SO_MARK: - *((int *)optval) = sk->sk_mark; - break; - case SO_PRIORITY: - *((int *)optval) = sk->sk_priority; - break; - case SO_BINDTOIFINDEX: - *((int *)optval) = sk->sk_bound_dev_if; - break; - case SO_REUSEPORT: - *((int *)optval) = sk->sk_reuseport; - break; - case SO_TXREHASH: - *((int *)optval) = sk->sk_txrehash; - break; - default: - goto err_clear; - } -#ifdef CONFIG_INET - } else if (level == SOL_TCP && sk->sk_prot->getsockopt == tcp_getsockopt) { - struct inet_connection_sock *icsk; - struct tcp_sock *tp; + if (getopt) + return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname, + KERNEL_SOCKPTR(optval), + KERNEL_SOCKPTR(optlen)); - switch (optname) { - case TCP_CONGESTION: - icsk = inet_csk(sk); + return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname, + KERNEL_SOCKPTR(optval), *optlen); +} - if (!icsk->icsk_ca_ops || optlen <= 1) - goto err_clear; - strncpy(optval, icsk->icsk_ca_ops->name, optlen); - optval[optlen - 1] = 0; - break; - case TCP_SAVED_SYN: - tp = tcp_sk(sk); +static int __bpf_setsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) +{ + if (!sk_fullsock(sk)) + return -EINVAL; - if (optlen <= 0 || !tp->saved_syn || - optlen > tcp_saved_syn_len(tp->saved_syn)) - goto err_clear; - memcpy(optval, tp->saved_syn->data, optlen); - break; - default: - goto err_clear; - } - } else if (level == SOL_IP) { - struct inet_sock *inet = inet_sk(sk); + if (level == SOL_SOCKET) + return sol_socket_sockopt(sk, optname, optval, &optlen, false); + else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP) + return sol_ip_sockopt(sk, optname, optval, &optlen, false); + else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6) + return sol_ipv6_sockopt(sk, optname, optval, &optlen, false); + else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP) + return sol_tcp_sockopt(sk, optname, optval, &optlen, false); - if (optlen != sizeof(int) || sk->sk_family != AF_INET) - goto err_clear; + return -EINVAL; +} - /* Only some options are supported */ - switch (optname) { - case IP_TOS: - *((int *)optval) = (int)inet->tos; - break; - default: - goto err_clear; - } -#if IS_ENABLED(CONFIG_IPV6) - } else if (level == SOL_IPV6) { - struct ipv6_pinfo *np = inet6_sk(sk); +static int _bpf_setsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) +{ + if (sk_fullsock(sk)) + sock_owned_by_me(sk); + return __bpf_setsockopt(sk, level, optname, optval, optlen); +} - if (optlen != sizeof(int) || sk->sk_family != AF_INET6) - goto err_clear; +static int __bpf_getsockopt(struct sock *sk, int level, int optname, + char *optval, int optlen) +{ + int err, saved_optlen = optlen; - /* Only some options are supported */ - switch (optname) { - case IPV6_TCLASS: - *((int *)optval) = (int)np->tclass; - break; - default: - goto err_clear; - } -#endif -#endif - } else { - goto err_clear; + if (!sk_fullsock(sk)) { + err = -EINVAL; + goto done; } - return 0; -err_clear: - memset(optval, 0, optlen); - return -EINVAL; + + if (level == SOL_SOCKET) + err = sol_socket_sockopt(sk, optname, optval, &optlen, true); + else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP) + err = sol_tcp_sockopt(sk, optname, optval, &optlen, true); + else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP) + err = sol_ip_sockopt(sk, optname, optval, &optlen, true); + else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6) + err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true); + else + err = -EINVAL; + +done: + if (err) + optlen = 0; + if (optlen < saved_optlen) + memset(optval + optlen, 0, saved_optlen - optlen); + return err; } static int _bpf_getsockopt(struct sock *sk, int level, int optname, @@ -5379,12 +5329,6 @@ static int _bpf_getsockopt(struct sock *sk, int level, int optname, BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level, int, optname, char *, optval, int, optlen) { - if (level == SOL_TCP && optname == TCP_CONGESTION) { - if (optlen >= sizeof("cdg") - 1 && - !strncmp("cdg", optval, optlen)) - return -ENOTSUPP; - } - return _bpf_setsockopt(sk, level, optname, optval, optlen); } @@ -6468,6 +6412,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = { static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple, int dif, int sdif, u8 family, u8 proto) { + struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo; bool refcounted = false; struct sock *sk = NULL; @@ -6476,7 +6421,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple, __be32 dst4 = tuple->ipv4.daddr; if (proto == IPPROTO_TCP) - sk = __inet_lookup(net, &tcp_hashinfo, NULL, 0, + sk = __inet_lookup(net, hinfo, NULL, 0, src4, tuple->ipv4.sport, dst4, tuple->ipv4.dport, dif, sdif, &refcounted); @@ -6490,7 +6435,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple, struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr; if (proto == IPPROTO_TCP) - sk = __inet6_lookup(net, &tcp_hashinfo, NULL, 0, + sk = __inet6_lookup(net, hinfo, NULL, 0, src6, tuple->ipv6.sport, dst6, ntohs(tuple->ipv6.dport), dif, sdif, &refcounted); @@ -7666,34 +7611,23 @@ const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak; static const struct bpf_func_proto * sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + + func_proto = cgroup_current_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { - /* inet and inet6 sockets are created in a process - * context so there is always a valid uid/gid - */ - case BPF_FUNC_get_current_uid_gid: - return &bpf_get_current_uid_gid_proto; - case BPF_FUNC_get_local_storage: - return &bpf_get_local_storage_proto; case BPF_FUNC_get_socket_cookie: return &bpf_get_socket_cookie_sock_proto; case BPF_FUNC_get_netns_cookie: return &bpf_get_netns_cookie_sock_proto; case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; - case BPF_FUNC_get_current_pid_tgid: - return &bpf_get_current_pid_tgid_proto; - case BPF_FUNC_get_current_comm: - return &bpf_get_current_comm_proto; -#ifdef CONFIG_CGROUPS - case BPF_FUNC_get_current_cgroup_id: - return &bpf_get_current_cgroup_id_proto; - case BPF_FUNC_get_current_ancestor_cgroup_id: - return &bpf_get_current_ancestor_cgroup_id_proto; -#endif -#ifdef CONFIG_CGROUP_NET_CLASSID - case BPF_FUNC_get_cgroup_classid: - return &bpf_get_cgroup_classid_curr_proto; -#endif case BPF_FUNC_sk_storage_get: return &bpf_sk_storage_get_cg_sock_proto; case BPF_FUNC_ktime_get_coarse_ns: @@ -7706,12 +7640,17 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) static const struct bpf_func_proto * sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + + func_proto = cgroup_current_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { - /* inet and inet6 sockets are created in a process - * context so there is always a valid uid/gid - */ - case BPF_FUNC_get_current_uid_gid: - return &bpf_get_current_uid_gid_proto; case BPF_FUNC_bind: switch (prog->expected_attach_type) { case BPF_CGROUP_INET4_CONNECT: @@ -7724,24 +7663,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_socket_cookie_sock_addr_proto; case BPF_FUNC_get_netns_cookie: return &bpf_get_netns_cookie_sock_addr_proto; - case BPF_FUNC_get_local_storage: - return &bpf_get_local_storage_proto; case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; - case BPF_FUNC_get_current_pid_tgid: - return &bpf_get_current_pid_tgid_proto; - case BPF_FUNC_get_current_comm: - return &bpf_get_current_comm_proto; -#ifdef CONFIG_CGROUPS - case BPF_FUNC_get_current_cgroup_id: - return &bpf_get_current_cgroup_id_proto; - case BPF_FUNC_get_current_ancestor_cgroup_id: - return &bpf_get_current_ancestor_cgroup_id_proto; -#endif -#ifdef CONFIG_CGROUP_NET_CLASSID - case BPF_FUNC_get_cgroup_classid: - return &bpf_get_cgroup_classid_curr_proto; -#endif #ifdef CONFIG_INET case BPF_FUNC_sk_lookup_tcp: return &bpf_sock_addr_sk_lookup_tcp_proto; @@ -7822,9 +7745,13 @@ const struct bpf_func_proto bpf_sk_storage_delete_proto __weak; static const struct bpf_func_proto * cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { - case BPF_FUNC_get_local_storage: - return &bpf_get_local_storage_proto; case BPF_FUNC_sk_fullsock: return &bpf_sk_fullsock_proto; case BPF_FUNC_sk_storage_get: @@ -8064,6 +7991,12 @@ const struct bpf_func_proto bpf_sock_hash_update_proto __weak; static const struct bpf_func_proto * sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { + const struct bpf_func_proto *func_proto; + + func_proto = cgroup_common_func_proto(func_id, prog); + if (func_proto) + return func_proto; + switch (func_id) { case BPF_FUNC_setsockopt: return &bpf_sock_ops_setsockopt_proto; @@ -8077,8 +8010,6 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_sock_hash_update_proto; case BPF_FUNC_get_socket_cookie: return &bpf_get_socket_cookie_sock_ops_proto; - case BPF_FUNC_get_local_storage: - return &bpf_get_local_storage_proto; case BPF_FUNC_perf_event_output: return &bpf_event_output_data_proto; case BPF_FUNC_sk_storage_get: @@ -8713,6 +8644,36 @@ static bool tc_cls_act_is_valid_access(int off, int size, return bpf_skb_is_valid_access(off, size, type, prog, info); } +DEFINE_MUTEX(nf_conn_btf_access_lock); +EXPORT_SYMBOL_GPL(nf_conn_btf_access_lock); + +int (*nfct_btf_struct_access)(struct bpf_verifier_log *log, const struct btf *btf, + const struct btf_type *t, int off, int size, + enum bpf_access_type atype, u32 *next_btf_id, + enum bpf_type_flag *flag); +EXPORT_SYMBOL_GPL(nfct_btf_struct_access); + +static int tc_cls_act_btf_struct_access(struct bpf_verifier_log *log, + const struct btf *btf, + const struct btf_type *t, int off, + int size, enum bpf_access_type atype, + u32 *next_btf_id, + enum bpf_type_flag *flag) +{ + int ret = -EACCES; + + if (atype == BPF_READ) + return btf_struct_access(log, btf, t, off, size, atype, next_btf_id, + flag); + + mutex_lock(&nf_conn_btf_access_lock); + if (nfct_btf_struct_access) + ret = nfct_btf_struct_access(log, btf, t, off, size, atype, next_btf_id, flag); + mutex_unlock(&nf_conn_btf_access_lock); + + return ret; +} + static bool __is_valid_xdp_access(int off, int size) { if (off < 0 || off >= sizeof(struct xdp_md)) @@ -8772,6 +8733,27 @@ void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog, } EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action); +static int xdp_btf_struct_access(struct bpf_verifier_log *log, + const struct btf *btf, + const struct btf_type *t, int off, + int size, enum bpf_access_type atype, + u32 *next_btf_id, + enum bpf_type_flag *flag) +{ + int ret = -EACCES; + + if (atype == BPF_READ) + return btf_struct_access(log, btf, t, off, size, atype, next_btf_id, + flag); + + mutex_lock(&nf_conn_btf_access_lock); + if (nfct_btf_struct_access) + ret = nfct_btf_struct_access(log, btf, t, off, size, atype, next_btf_id, flag); + mutex_unlock(&nf_conn_btf_access_lock); + + return ret; +} + static bool sock_addr_is_valid_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, @@ -10666,6 +10648,7 @@ const struct bpf_verifier_ops tc_cls_act_verifier_ops = { .convert_ctx_access = tc_cls_act_convert_ctx_access, .gen_prologue = tc_cls_act_prologue, .gen_ld_abs = bpf_gen_ld_abs, + .btf_struct_access = tc_cls_act_btf_struct_access, }; const struct bpf_prog_ops tc_cls_act_prog_ops = { @@ -10677,6 +10660,7 @@ const struct bpf_verifier_ops xdp_verifier_ops = { .is_valid_access = xdp_is_valid_access, .convert_ctx_access = xdp_convert_ctx_access, .gen_prologue = bpf_noop_prologue, + .btf_struct_access = xdp_btf_struct_access, }; const struct bpf_prog_ops xdp_prog_ops = { @@ -10811,14 +10795,13 @@ int sk_detach_filter(struct sock *sk) } EXPORT_SYMBOL_GPL(sk_detach_filter); -int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, - unsigned int len) +int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len) { struct sock_fprog_kern *fprog; struct sk_filter *filter; int ret = 0; - lock_sock(sk); + sockopt_lock_sock(sk); filter = rcu_dereference_protected(sk->sk_filter, lockdep_sock_is_held(sk)); if (!filter) @@ -10843,7 +10826,7 @@ int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, goto out; ret = -EFAULT; - if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog))) + if (copy_to_sockptr(optval, fprog->filter, bpf_classic_proglen(fprog))) goto out; /* Instead of bytes, the API requests to return the number @@ -10851,7 +10834,7 @@ int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf, */ ret = fprog->len; out: - release_sock(sk); + sockopt_release_sock(sk); return ret; } diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 764c4cb3fe8f41d3ff9c4f9a981563d7296b2c72..25cd35f5922e6a3197ad7e5dcf3c05fefc0bdd1d 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -204,6 +204,30 @@ static void __skb_flow_dissect_icmp(const struct sk_buff *skb, skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen); } +static void __skb_flow_dissect_l2tpv3(const struct sk_buff *skb, + struct flow_dissector *flow_dissector, + void *target_container, const void *data, + int nhoff, int hlen) +{ + struct flow_dissector_key_l2tpv3 *key_l2tpv3; + struct { + __be32 session_id; + } *hdr, _hdr; + + if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_L2TPV3)) + return; + + hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); + if (!hdr) + return; + + key_l2tpv3 = skb_flow_dissector_target(flow_dissector, + FLOW_DISSECTOR_KEY_L2TPV3, + target_container); + + key_l2tpv3->session_id = hdr->session_id; +} + void skb_flow_dissect_meta(const struct sk_buff *skb, struct flow_dissector *flow_dissector, void *target_container) @@ -866,8 +890,8 @@ static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys, } } -bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, - __be16 proto, int nhoff, int hlen, unsigned int flags) +u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, + __be16 proto, int nhoff, int hlen, unsigned int flags) { struct bpf_flow_keys *flow_keys = ctx->flow_keys; u32 result; @@ -892,7 +916,7 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, flow_keys->thoff = clamp_t(u16, flow_keys->thoff, flow_keys->nhoff, hlen); - return result == BPF_OK; + return result; } static bool is_pppoe_ses_hdr_valid(const struct pppoe_hdr *hdr) @@ -1008,6 +1032,7 @@ bool __skb_flow_dissect(const struct net *net, }; __be16 n_proto = proto; struct bpf_prog *prog; + u32 result; if (skb) { ctx.skb = skb; @@ -1019,13 +1044,16 @@ bool __skb_flow_dissect(const struct net *net, } prog = READ_ONCE(run_array->items[0].prog); - ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff, - hlen, flags); + result = bpf_flow_dissect(prog, &ctx, n_proto, nhoff, + hlen, flags); + if (result == BPF_FLOW_DISSECTOR_CONTINUE) + goto dissect_continue; __skb_flow_bpf_to_target(&flow_keys, flow_dissector, target_container); rcu_read_unlock(); - return ret; + return result == BPF_OK; } +dissect_continue: rcu_read_unlock(); } @@ -1173,8 +1201,8 @@ proto_again: nhoff += sizeof(*vlan); } - if (dissector_uses_key(flow_dissector, - FLOW_DISSECTOR_KEY_NUM_OF_VLANS)) { + if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_NUM_OF_VLANS) && + !(key_control->flags & FLOW_DIS_ENCAPSULATION)) { struct flow_dissector_key_num_of_vlans *key_nvs; key_nvs = skb_flow_dissector_target(flow_dissector, @@ -1497,6 +1525,10 @@ ip_proto_again: __skb_flow_dissect_icmp(skb, flow_dissector, target_container, data, nhoff, hlen); break; + case IPPROTO_L2TP: + __skb_flow_dissect_l2tpv3(skb, flow_dissector, target_container, + data, nhoff, hlen); + break; default: break; @@ -1611,9 +1643,8 @@ static inline void __flow_hash_consistentify(struct flow_keys *keys) switch (keys->control.addr_type) { case FLOW_DISSECTOR_KEY_IPV4_ADDRS: - addr_diff = (__force u32)keys->addrs.v4addrs.dst - - (__force u32)keys->addrs.v4addrs.src; - if (addr_diff < 0) + if ((__force u32)keys->addrs.v4addrs.dst < + (__force u32)keys->addrs.v4addrs.src) swap(keys->addrs.v4addrs.src, keys->addrs.v4addrs.dst); if ((__force u16)keys->ports.dst < diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c index 8cfb63528d183a877e85885fcf29e80bd6525b6f..abe423fd57369037d632e5b2123b424b53f1d69d 100644 --- a/net/core/flow_offload.c +++ b/net/core/flow_offload.c @@ -237,6 +237,13 @@ void flow_rule_match_pppoe(const struct flow_rule *rule, } EXPORT_SYMBOL(flow_rule_match_pppoe); +void flow_rule_match_l2tpv3(const struct flow_rule *rule, + struct flow_match_l2tpv3 *out) +{ + FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_L2TPV3, out); +} +EXPORT_SYMBOL(flow_rule_match_l2tpv3); + struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, void *cb_ident, void *cb_priv, void (*release)(void *cb_priv)) diff --git a/net/core/gro.c b/net/core/gro.c index b4190eb084672fb4f2be8b437eccb4e8507ff63f..bc9451743307bc380cca96ae6995aa0a3b83d185 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -160,6 +160,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) unsigned int gro_max_size; unsigned int new_truesize; struct sk_buff *lp; + int segs; /* pairs with WRITE_ONCE() in netif_set_gro_max_size() */ gro_max_size = READ_ONCE(p->dev->gro_max_size); @@ -175,6 +176,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) return -E2BIG; } + segs = NAPI_GRO_CB(skb)->count; lp = NAPI_GRO_CB(p)->last; pinfo = skb_shinfo(lp); @@ -265,7 +267,7 @@ merge: lp = p; done: - NAPI_GRO_CB(p)->count++; + NAPI_GRO_CB(p)->count += segs; p->data_len += len; p->truesize += delta_truesize; p->len += len; @@ -496,8 +498,15 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed), sizeof(u32))); /* Avoid slow unaligned acc */ *(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0; - NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); + NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb); NAPI_GRO_CB(skb)->is_atomic = 1; + NAPI_GRO_CB(skb)->count = 1; + if (unlikely(skb_is_gso(skb))) { + NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs; + /* Only support TCP at the moment. */ + if (!skb_is_gso_tcp(skb)) + NAPI_GRO_CB(skb)->flush = 1; + } /* Setup for GRO checksum validation */ switch (skb->ip_summed) { @@ -545,10 +554,10 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff else gro_list->count++; - NAPI_GRO_CB(skb)->count = 1; NAPI_GRO_CB(skb)->age = jiffies; NAPI_GRO_CB(skb)->last = skb; - skb_shinfo(skb)->gso_size = skb_gro_len(skb); + if (!skb_is_gso(skb)) + skb_shinfo(skb)->gso_size = skb_gro_len(skb); list_add(&skb->list, &gro_list->list); ret = GRO_HELD; @@ -660,6 +669,7 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) skb->encapsulation = 0; skb_shinfo(skb)->gso_type = 0; + skb_shinfo(skb)->gso_size = 0; if (unlikely(skb->slow_gro)) { skb_orphan(skb); skb_ext_reset(skb); diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c index 541c7a72a28a4b00e7e196eca01df42842ea103f..ed5ec5de47f670753924bd0c72db1e3ceb9b9e7a 100644 --- a/net/core/gro_cells.c +++ b/net/core/gro_cells.c @@ -26,7 +26,7 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) cell = this_cpu_ptr(gcells->cells); - if (skb_queue_len(&cell->napi_skbs) > netdev_max_backlog) { + if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(netdev_max_backlog)) { drop: dev_core_stats_rx_dropped_inc(dev); kfree_skb(skb); @@ -81,8 +81,7 @@ int gro_cells_init(struct gro_cells *gcells, struct net_device *dev) set_bit(NAPI_STATE_NO_BUSY_POLL, &cell->napi.state); - netif_napi_add(dev, &cell->napi, gro_cell_poll, - NAPI_POLL_WEIGHT); + netif_napi_add(dev, &cell->napi, gro_cell_poll); napi_enable(&cell->napi); } return 0; diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c index 9ccd64e8a666ad0c82d64b79cdd8da8308e80148..6fac2f0ef0742d46db21d955b284db451f1e109a 100644 --- a/net/core/lwtunnel.c +++ b/net/core/lwtunnel.c @@ -50,6 +50,7 @@ static const char *lwtunnel_encap_str(enum lwtunnel_encap_types encap_type) return "IOAM6"; case LWTUNNEL_ENCAP_IP6: case LWTUNNEL_ENCAP_IP: + case LWTUNNEL_ENCAP_XFRM: case LWTUNNEL_ENCAP_NONE: case __LWTUNNEL_ENCAP_MAX: /* should not have got here */ diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 5b669eb802708e6a4c735d57370fa65b77d16a7a..3c4786b9990703814995832027af16d8c8e06c72 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -111,7 +111,7 @@ static void neigh_cleanup_and_release(struct neighbour *neigh) unsigned long neigh_rand_reach_time(unsigned long base) { - return base ? (prandom_u32() % base) + (base >> 1) : 0; + return base ? prandom_u32_max(base) + (base >> 1) : 0; } EXPORT_SYMBOL(neigh_rand_reach_time); @@ -309,14 +309,17 @@ static int neigh_del_timer(struct neighbour *n) static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net) { + struct sk_buff_head tmp; unsigned long flags; struct sk_buff *skb; + skb_queue_head_init(&tmp); spin_lock_irqsave(&list->lock, flags); skb = skb_peek(list); while (skb != NULL) { struct sk_buff *skb_next = skb_peek_next(skb, list); struct net_device *dev = skb->dev; + if (net == NULL || net_eq(dev_net(dev), net)) { struct in_device *in_dev; @@ -326,13 +329,16 @@ static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net) in_dev->arp_parms->qlen--; rcu_read_unlock(); __skb_unlink(skb, list); - - dev_put(dev); - kfree_skb(skb); + __skb_queue_tail(&tmp, skb); } skb = skb_next; } spin_unlock_irqrestore(&list->lock, flags); + + while ((skb = __skb_dequeue(&tmp))) { + dev_put(skb->dev); + kfree_skb(skb); + } } static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev, @@ -1847,9 +1853,6 @@ static struct neigh_table *neigh_find_table(int family) case AF_INET6: tbl = neigh_tables[NEIGH_ND_TABLE]; break; - case AF_DECnet: - tbl = neigh_tables[NEIGH_DN_TABLE]; - break; } return tbl; diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index d61afd21aab5c7ba33b631e29f99bcd50bb6510c..8409d41405dfe4a4448f3b7f070bdf9f856d74b8 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -59,7 +59,7 @@ static ssize_t netdev_show(const struct device *dev, #define NETDEVICE_SHOW(field, format_string) \ static ssize_t format_##field(const struct net_device *dev, char *buf) \ { \ - return sprintf(buf, format_string, dev->field); \ + return sysfs_emit(buf, format_string, dev->field); \ } \ static ssize_t field##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ @@ -118,13 +118,13 @@ static ssize_t iflink_show(struct device *dev, struct device_attribute *attr, { struct net_device *ndev = to_net_dev(dev); - return sprintf(buf, fmt_dec, dev_get_iflink(ndev)); + return sysfs_emit(buf, fmt_dec, dev_get_iflink(ndev)); } static DEVICE_ATTR_RO(iflink); static ssize_t format_name_assign_type(const struct net_device *dev, char *buf) { - return sprintf(buf, fmt_dec, dev->name_assign_type); + return sysfs_emit(buf, fmt_dec, dev->name_assign_type); } static ssize_t name_assign_type_show(struct device *dev, @@ -194,7 +194,7 @@ static ssize_t carrier_show(struct device *dev, struct net_device *netdev = to_net_dev(dev); if (netif_running(netdev)) - return sprintf(buf, fmt_dec, !!netif_carrier_ok(netdev)); + return sysfs_emit(buf, fmt_dec, !!netif_carrier_ok(netdev)); return -EINVAL; } @@ -219,7 +219,7 @@ static ssize_t speed_show(struct device *dev, struct ethtool_link_ksettings cmd; if (!__ethtool_get_link_ksettings(netdev, &cmd)) - ret = sprintf(buf, fmt_dec, cmd.base.speed); + ret = sysfs_emit(buf, fmt_dec, cmd.base.speed); } rtnl_unlock(); return ret; @@ -258,7 +258,7 @@ static ssize_t duplex_show(struct device *dev, duplex = "unknown"; break; } - ret = sprintf(buf, "%s\n", duplex); + ret = sysfs_emit(buf, "%s\n", duplex); } } rtnl_unlock(); @@ -272,7 +272,7 @@ static ssize_t testing_show(struct device *dev, struct net_device *netdev = to_net_dev(dev); if (netif_running(netdev)) - return sprintf(buf, fmt_dec, !!netif_testing(netdev)); + return sysfs_emit(buf, fmt_dec, !!netif_testing(netdev)); return -EINVAL; } @@ -284,7 +284,7 @@ static ssize_t dormant_show(struct device *dev, struct net_device *netdev = to_net_dev(dev); if (netif_running(netdev)) - return sprintf(buf, fmt_dec, !!netif_dormant(netdev)); + return sysfs_emit(buf, fmt_dec, !!netif_dormant(netdev)); return -EINVAL; } @@ -315,7 +315,7 @@ static ssize_t operstate_show(struct device *dev, if (operstate >= ARRAY_SIZE(operstates)) return -EINVAL; /* should not happen */ - return sprintf(buf, "%s\n", operstates[operstate]); + return sysfs_emit(buf, "%s\n", operstates[operstate]); } static DEVICE_ATTR_RO(operstate); @@ -325,9 +325,9 @@ static ssize_t carrier_changes_show(struct device *dev, { struct net_device *netdev = to_net_dev(dev); - return sprintf(buf, fmt_dec, - atomic_read(&netdev->carrier_up_count) + - atomic_read(&netdev->carrier_down_count)); + return sysfs_emit(buf, fmt_dec, + atomic_read(&netdev->carrier_up_count) + + atomic_read(&netdev->carrier_down_count)); } static DEVICE_ATTR_RO(carrier_changes); @@ -337,7 +337,7 @@ static ssize_t carrier_up_count_show(struct device *dev, { struct net_device *netdev = to_net_dev(dev); - return sprintf(buf, fmt_dec, atomic_read(&netdev->carrier_up_count)); + return sysfs_emit(buf, fmt_dec, atomic_read(&netdev->carrier_up_count)); } static DEVICE_ATTR_RO(carrier_up_count); @@ -347,7 +347,7 @@ static ssize_t carrier_down_count_show(struct device *dev, { struct net_device *netdev = to_net_dev(dev); - return sprintf(buf, fmt_dec, atomic_read(&netdev->carrier_down_count)); + return sysfs_emit(buf, fmt_dec, atomic_read(&netdev->carrier_down_count)); } static DEVICE_ATTR_RO(carrier_down_count); @@ -462,7 +462,7 @@ static ssize_t ifalias_show(struct device *dev, ret = dev_get_alias(netdev, tmp, sizeof(tmp)); if (ret > 0) - ret = sprintf(buf, "%s\n", tmp); + ret = sysfs_emit(buf, "%s\n", tmp); return ret; } static DEVICE_ATTR_RW(ifalias); @@ -514,7 +514,7 @@ static ssize_t phys_port_id_show(struct device *dev, ret = dev_get_phys_port_id(netdev, &ppid); if (!ret) - ret = sprintf(buf, "%*phN\n", ppid.id_len, ppid.id); + ret = sysfs_emit(buf, "%*phN\n", ppid.id_len, ppid.id); } rtnl_unlock(); @@ -543,7 +543,7 @@ static ssize_t phys_port_name_show(struct device *dev, ret = dev_get_phys_port_name(netdev, name, sizeof(name)); if (!ret) - ret = sprintf(buf, "%s\n", name); + ret = sysfs_emit(buf, "%s\n", name); } rtnl_unlock(); @@ -573,7 +573,7 @@ static ssize_t phys_switch_id_show(struct device *dev, ret = dev_get_port_parent_id(netdev, &ppid, false); if (!ret) - ret = sprintf(buf, "%*phN\n", ppid.id_len, ppid.id); + ret = sysfs_emit(buf, "%*phN\n", ppid.id_len, ppid.id); } rtnl_unlock(); @@ -591,7 +591,7 @@ static ssize_t threaded_show(struct device *dev, return restart_syscall(); if (dev_isalive(netdev)) - ret = sprintf(buf, fmt_dec, netdev->threaded); + ret = sysfs_emit(buf, fmt_dec, netdev->threaded); rtnl_unlock(); return ret; @@ -673,7 +673,7 @@ static ssize_t netstat_show(const struct device *d, struct rtnl_link_stats64 temp; const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp); - ret = sprintf(buf, fmt_u64, *(u64 *)(((u8 *)stats) + offset)); + ret = sysfs_emit(buf, fmt_u64, *(u64 *)(((u8 *)stats) + offset)); } read_unlock(&dev_base_lock); return ret; @@ -824,7 +824,7 @@ static ssize_t show_rps_map(struct netdev_rx_queue *queue, char *buf) for (i = 0; i < map->len; i++) cpumask_set_cpu(map->cpus[i], mask); - len = snprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask)); + len = sysfs_emit(buf, "%*pb\n", cpumask_pr_args(mask)); rcu_read_unlock(); free_cpumask_var(mask); @@ -910,7 +910,7 @@ static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue, val = (unsigned long)flow_table->mask + 1; rcu_read_unlock(); - return sprintf(buf, "%lu\n", val); + return sysfs_emit(buf, "%lu\n", val); } static void rps_dev_flow_table_release(struct rcu_head *rcu) @@ -1208,7 +1208,7 @@ static ssize_t tx_timeout_show(struct netdev_queue *queue, char *buf) { unsigned long trans_timeout = atomic_long_read(&queue->trans_timeout); - return sprintf(buf, fmt_ulong, trans_timeout); + return sysfs_emit(buf, fmt_ulong, trans_timeout); } static unsigned int get_netdev_queue_index(struct netdev_queue *queue) @@ -1255,15 +1255,15 @@ static ssize_t traffic_class_show(struct netdev_queue *queue, * belongs to the root device it will be reported with just the * traffic class, so just "0" for TC 0 for example. */ - return num_tc < 0 ? sprintf(buf, "%d%d\n", tc, num_tc) : - sprintf(buf, "%d\n", tc); + return num_tc < 0 ? sysfs_emit(buf, "%d%d\n", tc, num_tc) : + sysfs_emit(buf, "%d\n", tc); } #ifdef CONFIG_XPS static ssize_t tx_maxrate_show(struct netdev_queue *queue, char *buf) { - return sprintf(buf, "%lu\n", queue->tx_maxrate); + return sysfs_emit(buf, "%lu\n", queue->tx_maxrate); } static ssize_t tx_maxrate_store(struct netdev_queue *queue, @@ -1317,7 +1317,7 @@ static struct netdev_queue_attribute queue_traffic_class __ro_after_init */ static ssize_t bql_show(char *buf, unsigned int value) { - return sprintf(buf, "%u\n", value); + return sysfs_emit(buf, "%u\n", value); } static ssize_t bql_set(const char *buf, const size_t count, @@ -1346,7 +1346,7 @@ static ssize_t bql_show_hold_time(struct netdev_queue *queue, { struct dql *dql = &queue->dql; - return sprintf(buf, "%u\n", jiffies_to_msecs(dql->slack_hold_time)); + return sysfs_emit(buf, "%u\n", jiffies_to_msecs(dql->slack_hold_time)); } static ssize_t bql_set_hold_time(struct netdev_queue *queue, @@ -1374,7 +1374,7 @@ static ssize_t bql_show_inflight(struct netdev_queue *queue, { struct dql *dql = &queue->dql; - return sprintf(buf, "%u\n", dql->num_queued - dql->num_completed); + return sysfs_emit(buf, "%u\n", dql->num_queued - dql->num_completed); } static struct netdev_queue_attribute bql_inflight_attribute __ro_after_init = diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 6b9f19122ec1aef67e4a91d3e19ea8b28ad33b6d..0ec2f5906a27c7f930e832835682d69a32e3c8e1 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include @@ -1144,13 +1143,7 @@ static int __register_pernet_operations(struct list_head *list, * setup_net() and cleanup_net() are not possible. */ for_each_net(net) { - struct mem_cgroup *old, *memcg; - - memcg = mem_cgroup_or_root(get_mem_cgroup_from_obj(net)); - old = set_active_memcg(memcg); error = ops_init(ops, net); - set_active_memcg(old); - mem_cgroup_put(memcg); if (error) goto out_undo; list_add_tail(&net->exit_list, &net_exit_list); diff --git a/net/core/netclassid_cgroup.c b/net/core/netclassid_cgroup.c index 1a6a86693b745984dbc3fe0c3b39d30042a47186..d6a70aeaa50372b4992fafd888ddde4f0fc43336 100644 --- a/net/core/netclassid_cgroup.c +++ b/net/core/netclassid_cgroup.c @@ -66,7 +66,7 @@ struct update_classid_context { #define UPDATE_CLASSID_BATCH 1000 -static int update_classid_sock(const void *v, struct file *file, unsigned n) +static int update_classid_sock(const void *v, struct file *file, unsigned int n) { struct update_classid_context *ctx = (void *)v; struct socket *sock = sock_from_file(file); diff --git a/net/core/netpoll.c b/net/core/netpoll.c index 5d27067b72d565ed7977984ace9ccd9188416c62..9be762e1d042897f09b682bf28d2473133d9fefa 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -556,7 +556,7 @@ int netpoll_parse_options(struct netpoll *np, char *opt) if ((delim = strchr(cur, ',')) == NULL) goto parse_failed; *delim = 0; - strlcpy(np->dev_name, cur, sizeof(np->dev_name)); + strscpy(np->dev_name, cur, sizeof(np->dev_name)); cur = delim; } cur++; @@ -610,7 +610,7 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev) int err; np->dev = ndev; - strlcpy(np->dev_name, ndev->name, IFNAMSIZ); + strscpy(np->dev_name, ndev->name, IFNAMSIZ); if (ndev->priv_flags & IFF_DISABLE_NETPOLL) { np_err(np, "%s doesn't support polling, aborting\n", diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 88906ba6d9a7851dfbb4762d3504cc20423e1053..c3763056c554ad9c5f7849040903811a7542f992 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2324,7 +2324,7 @@ static inline int f_pick(struct pktgen_dev *pkt_dev) pkt_dev->curfl = 0; /*reset */ } } else { - flow = prandom_u32() % pkt_dev->cflows; + flow = prandom_u32_max(pkt_dev->cflows); pkt_dev->curfl = flow; if (pkt_dev->flows[flow].count > pkt_dev->lflow) { @@ -2380,10 +2380,9 @@ static void set_cur_queue_map(struct pktgen_dev *pkt_dev) else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) { __u16 t; if (pkt_dev->flags & F_QUEUE_MAP_RND) { - t = prandom_u32() % - (pkt_dev->queue_map_max - - pkt_dev->queue_map_min + 1) - + pkt_dev->queue_map_min; + t = prandom_u32_max(pkt_dev->queue_map_max - + pkt_dev->queue_map_min + 1) + + pkt_dev->queue_map_min; } else { t = pkt_dev->cur_queue_map + 1; if (t > pkt_dev->queue_map_max) @@ -2412,7 +2411,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) __u32 tmp; if (pkt_dev->flags & F_MACSRC_RND) - mc = prandom_u32() % pkt_dev->src_mac_count; + mc = prandom_u32_max(pkt_dev->src_mac_count); else { mc = pkt_dev->cur_src_mac_offset++; if (pkt_dev->cur_src_mac_offset >= @@ -2438,7 +2437,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) __u32 tmp; if (pkt_dev->flags & F_MACDST_RND) - mc = prandom_u32() % pkt_dev->dst_mac_count; + mc = prandom_u32_max(pkt_dev->dst_mac_count); else { mc = pkt_dev->cur_dst_mac_offset++; @@ -2465,23 +2464,23 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) for (i = 0; i < pkt_dev->nr_labels; i++) if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM) pkt_dev->labels[i] = MPLS_STACK_BOTTOM | - ((__force __be32)prandom_u32() & + ((__force __be32)get_random_u32() & htonl(0x000fffff)); } if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) { - pkt_dev->vlan_id = prandom_u32() & (4096 - 1); + pkt_dev->vlan_id = prandom_u32_max(4096); } if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) { - pkt_dev->svlan_id = prandom_u32() & (4096 - 1); + pkt_dev->svlan_id = prandom_u32_max(4096); } if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { if (pkt_dev->flags & F_UDPSRC_RND) - pkt_dev->cur_udp_src = prandom_u32() % - (pkt_dev->udp_src_max - pkt_dev->udp_src_min) - + pkt_dev->udp_src_min; + pkt_dev->cur_udp_src = prandom_u32_max( + pkt_dev->udp_src_max - pkt_dev->udp_src_min) + + pkt_dev->udp_src_min; else { pkt_dev->cur_udp_src++; @@ -2492,9 +2491,9 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { if (pkt_dev->flags & F_UDPDST_RND) { - pkt_dev->cur_udp_dst = prandom_u32() % - (pkt_dev->udp_dst_max - pkt_dev->udp_dst_min) - + pkt_dev->udp_dst_min; + pkt_dev->cur_udp_dst = prandom_u32_max( + pkt_dev->udp_dst_max - pkt_dev->udp_dst_min) + + pkt_dev->udp_dst_min; } else { pkt_dev->cur_udp_dst++; if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) @@ -2509,7 +2508,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) if (imn < imx) { __u32 t; if (pkt_dev->flags & F_IPSRC_RND) - t = prandom_u32() % (imx - imn) + imn; + t = prandom_u32_max(imx - imn) + imn; else { t = ntohl(pkt_dev->cur_saddr); t++; @@ -2531,8 +2530,8 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) if (pkt_dev->flags & F_IPDST_RND) { do { - t = prandom_u32() % - (imx - imn) + imn; + t = prandom_u32_max(imx - imn) + + imn; s = htonl(t); } while (ipv4_is_loopback(s) || ipv4_is_multicast(s) || @@ -2569,7 +2568,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) for (i = 0; i < 4; i++) { pkt_dev->cur_in6_daddr.s6_addr32[i] = - (((__force __be32)prandom_u32() | + (((__force __be32)get_random_u32() | pkt_dev->min_in6_daddr.s6_addr32[i]) & pkt_dev->max_in6_daddr.s6_addr32[i]); } @@ -2579,9 +2578,9 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { __u32 t; if (pkt_dev->flags & F_TXSIZE_RND) { - t = prandom_u32() % - (pkt_dev->max_pkt_size - pkt_dev->min_pkt_size) - + pkt_dev->min_pkt_size; + t = prandom_u32_max(pkt_dev->max_pkt_size - + pkt_dev->min_pkt_size) + + pkt_dev->min_pkt_size; } else { t = pkt_dev->cur_pkt_size + 1; if (t > pkt_dev->max_pkt_size) @@ -2590,7 +2589,7 @@ static void mod_cur_headers(struct pktgen_dev *pkt_dev) pkt_dev->cur_pkt_size = t; } else if (pkt_dev->n_imix_entries > 0) { struct imix_pkt *entry; - __u32 t = prandom_u32() % IMIX_PRECISION; + __u32 t = prandom_u32_max(IMIX_PRECISION); __u8 entry_index = pkt_dev->imix_distribution[t]; entry = &pkt_dev->imix_entries[entry_index]; diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 4b5b15c684ed63522325740dea0678a71cd07206..74864dc46a7effc32dfcb75ffe9629af45add077 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -866,14 +866,12 @@ static void set_operstate(struct net_device *dev, unsigned char transition) break; case IF_OPER_TESTING: - if (operstate == IF_OPER_UP || - operstate == IF_OPER_UNKNOWN) + if (netif_oper_up(dev)) operstate = IF_OPER_TESTING; break; case IF_OPER_DORMANT: - if (operstate == IF_OPER_UP || - operstate == IF_OPER_UNKNOWN) + if (netif_oper_up(dev)) operstate = IF_OPER_DORMANT; break; } @@ -1059,6 +1057,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + nla_total_size(4) /* IFLA_MASTER */ + nla_total_size(1) /* IFLA_CARRIER */ + nla_total_size(4) /* IFLA_PROMISCUITY */ + + nla_total_size(4) /* IFLA_ALLMULTI */ + nla_total_size(4) /* IFLA_NUM_TX_QUEUES */ + nla_total_size(4) /* IFLA_NUM_RX_QUEUES */ + nla_total_size(4) /* IFLA_GSO_MAX_SEGS */ @@ -1767,6 +1766,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, nla_put_u32(skb, IFLA_MAX_MTU, dev->max_mtu) || nla_put_u32(skb, IFLA_GROUP, dev->group) || nla_put_u32(skb, IFLA_PROMISCUITY, dev->promiscuity) || + nla_put_u32(skb, IFLA_ALLMULTI, dev->allmulti) || nla_put_u32(skb, IFLA_NUM_TX_QUEUES, dev->num_tx_queues) || nla_put_u32(skb, IFLA_GSO_MAX_SEGS, dev->gso_max_segs) || nla_put_u32(skb, IFLA_GSO_MAX_SIZE, dev->gso_max_size) || @@ -1928,6 +1928,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_GRO_MAX_SIZE] = { .type = NLA_U32 }, [IFLA_TSO_MAX_SIZE] = { .type = NLA_REJECT }, [IFLA_TSO_MAX_SEGS] = { .type = NLA_REJECT }, + [IFLA_ALLMULTI] = { .type = NLA_REJECT }, }; static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = { @@ -2776,13 +2777,6 @@ static int do_setlink(const struct sk_buff *skb, call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); } - if (ifm->ifi_flags || ifm->ifi_change) { - err = dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm), - extack); - if (err < 0) - goto errout; - } - if (tb[IFLA_MASTER]) { err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack); if (err) @@ -2790,6 +2784,13 @@ static int do_setlink(const struct sk_buff *skb, status |= DO_SETLINK_MODIFIED; } + if (ifm->ifi_flags || ifm->ifi_change) { + err = dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm), + extack); + if (err < 0) + goto errout; + } + if (tb[IFLA_CARRIER]) { err = dev_change_carrier(dev, nla_get_u8(tb[IFLA_CARRIER])); if (err) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 974bbbbe7138a447e04d3a773d8c875574413516..1d9719e72f9d9ea6ca40979ff3ba95afec4f5b37 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -91,7 +91,11 @@ static struct kmem_cache *skbuff_ext_cache __ro_after_init; int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; EXPORT_SYMBOL(sysctl_max_skb_frags); -/* The array 'drop_reasons' is auto-generated in dropreason_str.c */ +#undef FN +#define FN(reason) [SKB_DROP_REASON_##reason] = #reason, +const char * const drop_reasons[] = { + DEFINE_DROP_REASON(FN, FN) +}; EXPORT_SYMBOL(drop_reasons); /** @@ -130,8 +134,66 @@ static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) #define NAPI_SKB_CACHE_BULK 16 #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) +#if PAGE_SIZE == SZ_4K + +#define NAPI_HAS_SMALL_PAGE_FRAG 1 +#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) ((nc).pfmemalloc) + +/* specialized page frag allocator using a single order 0 page + * and slicing it into 1K sized fragment. Constrained to systems + * with a very limited amount of 1K fragments fitting a single + * page - to avoid excessive truesize underestimation + */ + +struct page_frag_1k { + void *va; + u16 offset; + bool pfmemalloc; +}; + +static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp) +{ + struct page *page; + int offset; + + offset = nc->offset - SZ_1K; + if (likely(offset >= 0)) + goto use_frag; + + page = alloc_pages_node(NUMA_NO_NODE, gfp, 0); + if (!page) + return NULL; + + nc->va = page_address(page); + nc->pfmemalloc = page_is_pfmemalloc(page); + offset = PAGE_SIZE - SZ_1K; + page_ref_add(page, offset / SZ_1K); + +use_frag: + nc->offset = offset; + return nc->va + offset; +} +#else + +/* the small page is actually unused in this build; add dummy helpers + * to please the compiler and avoid later preprocessor's conditionals + */ +#define NAPI_HAS_SMALL_PAGE_FRAG 0 +#define NAPI_SMALL_PAGE_PFMEMALLOC(nc) false + +struct page_frag_1k { +}; + +static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask) +{ + return NULL; +} + +#endif + struct napi_alloc_cache { struct page_frag_cache page; + struct page_frag_1k page_small; unsigned int skb_count; void *skb_cache[NAPI_SKB_CACHE_SIZE]; }; @@ -139,6 +201,23 @@ struct napi_alloc_cache { static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); +/* Double check that napi_get_frags() allocates skbs with + * skb->head being backed by slab, not a page fragment. + * This is to make sure bug fixed in 3226b158e67c + * ("net: avoid 32 x truesize under-estimation for tiny skbs") + * does not accidentally come back. + */ +void napi_get_frags_check(struct napi_struct *napi) +{ + struct sk_buff *skb; + + local_bh_disable(); + skb = napi_get_frags(napi); + WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag); + napi_free_frags(napi); + local_bh_enable(); +} + void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) { struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); @@ -557,6 +636,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, { struct napi_alloc_cache *nc; struct sk_buff *skb; + bool pfmemalloc; void *data; DEBUG_NET_WARN_ON_ONCE(!in_softirq()); @@ -564,8 +644,10 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, /* If requested length is either too small or too big, * we use kmalloc() for skb->head allocation. + * When the small frag allocator is available, prefer it over kmalloc + * for small fragments */ - if (len <= SKB_WITH_OVERHEAD(1024) || + if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) || len > SKB_WITH_OVERHEAD(PAGE_SIZE) || (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, @@ -576,13 +658,33 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, } nc = this_cpu_ptr(&napi_alloc_cache); - len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - len = SKB_DATA_ALIGN(len); if (sk_memalloc_socks()) gfp_mask |= __GFP_MEMALLOC; - data = page_frag_alloc(&nc->page, len, gfp_mask); + if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) { + /* we are artificially inflating the allocation size, but + * that is not as bad as it may look like, as: + * - 'len' less than GRO_MAX_HEAD makes little sense + * - On most systems, larger 'len' values lead to fragment + * size above 512 bytes + * - kmalloc would use the kmalloc-1k slab for such values + * - Builds with smaller GRO_MAX_HEAD will very likely do + * little networking, as that implies no WiFi and no + * tunnels support, and 32 bits arches. + */ + len = SZ_1K; + + data = page_frag_alloc_1k(&nc->page_small, gfp_mask); + pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small); + } else { + len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + len = SKB_DATA_ALIGN(len); + + data = page_frag_alloc(&nc->page, len, gfp_mask); + pfmemalloc = nc->page.pfmemalloc; + } + if (unlikely(!data)) return NULL; @@ -592,7 +694,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, return NULL; } - if (nc->page.pfmemalloc) + if (pfmemalloc) skb->pfmemalloc = 1; skb->head_frag = 1; @@ -777,9 +879,10 @@ EXPORT_SYMBOL(__kfree_skb); * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' * tracepoint. */ -void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) +void __fix_address +kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) { - if (!skb_unref(skb)) + if (unlikely(!skb_unref(skb))) return; DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX); @@ -1183,7 +1286,7 @@ EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) { - struct ubuf_info *uarg; + struct ubuf_info_msgzc *uarg; struct sk_buff *skb; WARN_ON_ONCE(!in_task()); @@ -1201,19 +1304,19 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) return NULL; } - uarg->callback = msg_zerocopy_callback; + uarg->ubuf.callback = msg_zerocopy_callback; uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; uarg->len = 1; uarg->bytelen = size; uarg->zerocopy = 1; - uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; - refcount_set(&uarg->refcnt, 1); + uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&uarg->ubuf.refcnt, 1); sock_hold(sk); - return uarg; + return &uarg->ubuf; } -static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) +static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg) { return container_of((void *)uarg, struct sk_buff, cb); } @@ -1222,6 +1325,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, struct ubuf_info *uarg) { if (uarg) { + struct ubuf_info_msgzc *uarg_zc; const u32 byte_limit = 1 << 19; /* limit to a few TSO */ u32 bytelen, next; @@ -1237,8 +1341,9 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, return NULL; } - bytelen = uarg->bytelen + size; - if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { + uarg_zc = uarg_to_msgzc(uarg); + bytelen = uarg_zc->bytelen + size; + if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) { /* TCP can create new skb to attach new uarg */ if (sk->sk_type == SOCK_STREAM) goto new_alloc; @@ -1246,11 +1351,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, } next = (u32)atomic_read(&sk->sk_zckey); - if ((u32)(uarg->id + uarg->len) == next) { - if (mm_account_pinned_pages(&uarg->mmp, size)) + if ((u32)(uarg_zc->id + uarg_zc->len) == next) { + if (mm_account_pinned_pages(&uarg_zc->mmp, size)) return NULL; - uarg->len++; - uarg->bytelen = bytelen; + uarg_zc->len++; + uarg_zc->bytelen = bytelen; atomic_set(&sk->sk_zckey, ++next); /* no extra ref when appending to datagram (MSG_MORE) */ @@ -1286,7 +1391,7 @@ static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) return true; } -static void __msg_zerocopy_callback(struct ubuf_info *uarg) +static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg) { struct sk_buff *tail, *skb = skb_from_uarg(uarg); struct sock_exterr_skb *serr; @@ -1339,19 +1444,21 @@ release: void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, bool success) { - uarg->zerocopy = uarg->zerocopy & success; + struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg); + + uarg_zc->zerocopy = uarg_zc->zerocopy & success; if (refcount_dec_and_test(&uarg->refcnt)) - __msg_zerocopy_callback(uarg); + __msg_zerocopy_callback(uarg_zc); } EXPORT_SYMBOL_GPL(msg_zerocopy_callback); void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) { - struct sock *sk = skb_from_uarg(uarg)->sk; + struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk; atomic_dec(&sk->sk_zckey); - uarg->len--; + uarg_to_msgzc(uarg)->len--; if (have_uref) msg_zerocopy_callback(NULL, uarg, true); @@ -4205,9 +4312,8 @@ normal: SKB_GSO_CB(nskb)->csum_start = skb_headroom(nskb) + doffset; } else { - skb_copy_bits(head_skb, offset, - skb_put(nskb, len), - len); + if (skb_copy_bits(head_skb, offset, skb_put(nskb, len), len)) + goto err; } continue; } @@ -4798,7 +4904,7 @@ static bool skb_may_tx_timestamp(struct sock *sk, bool tsonly) { bool ret; - if (likely(sysctl_tstamp_allow_data || tsonly)) + if (likely(READ_ONCE(sysctl_tstamp_allow_data) || tsonly)) return true; read_lock_bh(&sk->sk_callback_lock); diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 59e75ffcc1f4085710c56c9461c4a82dd1b3882c..ca70525621c7162da52b9ae446337cbaac378f78 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -434,8 +434,10 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, if (copied + copy > len) copy = len - copied; copy = copy_page_to_iter(page, sge->offset, copy, iter); - if (!copy) - return copied ? copied : -EFAULT; + if (!copy) { + copied = copied ? copied : -EFAULT; + goto out; + } copied += copy; if (likely(!peek)) { @@ -455,13 +457,13 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, * didn't copy the entire length lets just break. */ if (copy != sge->length) - return copied; + goto out; sk_msg_iter_var_next(i); } if (copied == len) break; - } while (!sg_is_last(sge)); + } while ((i != msg_rx->sg.end) && !sg_is_last(sge)); if (unlikely(peek)) { msg_rx = sk_psock_next_msg(psock, msg_rx); @@ -471,13 +473,15 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg, } msg_rx->sg.start = i; - if (!sge->length && sg_is_last(sge)) { + if (!sge->length && (i == msg_rx->sg.end || sg_is_last(sge))) { msg_rx = sk_psock_dequeue_msg(psock); kfree_sk_msg(msg_rx); } msg_rx = sk_psock_peek_msg(psock); } - +out: + if (psock->work_state.skb && copied > 0) + schedule_work(&psock->work); return copied; } EXPORT_SYMBOL_GPL(sk_msg_recvmsg); diff --git a/net/core/sock.c b/net/core/sock.c index 4cb957d934a252f00e4aabf8c5fa4631fd0b529a..a3ba0358c77c0e44db1cfbaeb420f8b80ad7cf98 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -703,15 +703,17 @@ static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen) goto out; } - return sock_bindtoindex(sk, index, true); + sockopt_lock_sock(sk); + ret = sock_bindtoindex_locked(sk, index); + sockopt_release_sock(sk); out: #endif return ret; } -static int sock_getbindtodevice(struct sock *sk, char __user *optval, - int __user *optlen, int len) +static int sock_getbindtodevice(struct sock *sk, sockptr_t optval, + sockptr_t optlen, int len) { int ret = -ENOPROTOOPT; #ifdef CONFIG_NETDEVICES @@ -735,12 +737,12 @@ static int sock_getbindtodevice(struct sock *sk, char __user *optval, len = strlen(devname) + 1; ret = -EFAULT; - if (copy_to_user(optval, devname, len)) + if (copy_to_sockptr(optval, devname, len)) goto out; zero: ret = -EFAULT; - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) goto out; ret = 0; @@ -1036,17 +1038,51 @@ static int sock_reserve_memory(struct sock *sk, int bytes) return 0; } +void sockopt_lock_sock(struct sock *sk) +{ + /* When current->bpf_ctx is set, the setsockopt is called from + * a bpf prog. bpf has ensured the sk lock has been + * acquired before calling setsockopt(). + */ + if (has_current_bpf_ctx()) + return; + + lock_sock(sk); +} +EXPORT_SYMBOL(sockopt_lock_sock); + +void sockopt_release_sock(struct sock *sk) +{ + if (has_current_bpf_ctx()) + return; + + release_sock(sk); +} +EXPORT_SYMBOL(sockopt_release_sock); + +bool sockopt_ns_capable(struct user_namespace *ns, int cap) +{ + return has_current_bpf_ctx() || ns_capable(ns, cap); +} +EXPORT_SYMBOL(sockopt_ns_capable); + +bool sockopt_capable(int cap) +{ + return has_current_bpf_ctx() || capable(cap); +} +EXPORT_SYMBOL(sockopt_capable); + /* * This is meant for all protocols to use and covers goings on * at the socket level. Everything here is generic. */ -int sock_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen) +int sk_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen) { struct so_timestamping timestamping; + struct socket *sock = sk->sk_socket; struct sock_txtime sk_txtime; - struct sock *sk = sock->sk; int val; int valbool; struct linger ling; @@ -1067,11 +1103,11 @@ int sock_setsockopt(struct socket *sock, int level, int optname, valbool = val ? 1 : 0; - lock_sock(sk); + sockopt_lock_sock(sk); switch (optname) { case SO_DEBUG: - if (val && !capable(CAP_NET_ADMIN)) + if (val && !sockopt_capable(CAP_NET_ADMIN)) ret = -EACCES; else sock_valbool_flag(sk, SOCK_DBG, valbool); @@ -1101,7 +1137,7 @@ int sock_setsockopt(struct socket *sock, int level, int optname, * play 'guess the biggest size' games. RCVBUF/SNDBUF * are treated in BSD as hints */ - val = min_t(u32, val, sysctl_wmem_max); + val = min_t(u32, val, READ_ONCE(sysctl_wmem_max)); set_sndbuf: /* Ensure val * 2 fits into an int, to prevent max_t() * from treating it as a negative value. @@ -1115,7 +1151,7 @@ set_sndbuf: break; case SO_SNDBUFFORCE: - if (!capable(CAP_NET_ADMIN)) { + if (!sockopt_capable(CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1133,11 +1169,11 @@ set_sndbuf: * play 'guess the biggest size' games. RCVBUF/SNDBUF * are treated in BSD as hints */ - __sock_set_rcvbuf(sk, min_t(u32, val, sysctl_rmem_max)); + __sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max))); break; case SO_RCVBUFFORCE: - if (!capable(CAP_NET_ADMIN)) { + if (!sockopt_capable(CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1164,8 +1200,8 @@ set_sndbuf: case SO_PRIORITY: if ((val >= 0 && val <= 6) || - ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || - ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) + sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || + sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) sk->sk_priority = val; else ret = -EPERM; @@ -1228,7 +1264,7 @@ set_sndbuf: case SO_RCVLOWAT: if (val < 0) val = INT_MAX; - if (sock->ops->set_rcvlowat) + if (sock && sock->ops->set_rcvlowat) ret = sock->ops->set_rcvlowat(sk, val); else WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); @@ -1310,8 +1346,8 @@ set_sndbuf: clear_bit(SOCK_PASSSEC, &sock->flags); break; case SO_MARK: - if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && - !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && + !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1319,8 +1355,8 @@ set_sndbuf: __sock_set_mark(sk, val); break; case SO_RCVMARK: - if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && - !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && + !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1354,7 +1390,7 @@ set_sndbuf: #ifdef CONFIG_NET_RX_BUSY_POLL case SO_BUSY_POLL: /* allow unprivileged users to decrease the value */ - if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN)) + if ((val > sk->sk_ll_usec) && !sockopt_capable(CAP_NET_ADMIN)) ret = -EPERM; else { if (val < 0) @@ -1364,13 +1400,13 @@ set_sndbuf: } break; case SO_PREFER_BUSY_POLL: - if (valbool && !capable(CAP_NET_ADMIN)) + if (valbool && !sockopt_capable(CAP_NET_ADMIN)) ret = -EPERM; else WRITE_ONCE(sk->sk_prefer_busy_poll, valbool); break; case SO_BUSY_POLL_BUDGET: - if (val > READ_ONCE(sk->sk_busy_poll_budget) && !capable(CAP_NET_ADMIN)) { + if (val > READ_ONCE(sk->sk_busy_poll_budget) && !sockopt_capable(CAP_NET_ADMIN)) { ret = -EPERM; } else { if (val < 0 || val > U16_MAX) @@ -1441,7 +1477,7 @@ set_sndbuf: * scheduler has enough safe guards. */ if (sk_txtime.clockid != CLOCK_MONOTONIC && - !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1496,9 +1532,16 @@ set_sndbuf: ret = -ENOPROTOOPT; break; } - release_sock(sk); + sockopt_release_sock(sk); return ret; } + +int sock_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + return sk_setsockopt(sock->sk, level, optname, + optval, optlen); +} EXPORT_SYMBOL(sock_setsockopt); static const struct cred *sk_get_peer_cred(struct sock *sk) @@ -1525,22 +1568,25 @@ static void cred_to_ucred(struct pid *pid, const struct cred *cred, } } -static int groups_to_user(gid_t __user *dst, const struct group_info *src) +static int groups_to_user(sockptr_t dst, const struct group_info *src) { struct user_namespace *user_ns = current_user_ns(); int i; - for (i = 0; i < src->ngroups; i++) - if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i)) + for (i = 0; i < src->ngroups; i++) { + gid_t gid = from_kgid_munged(user_ns, src->gid[i]); + + if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid))) return -EFAULT; + } return 0; } -int sock_getsockopt(struct socket *sock, int level, int optname, - char __user *optval, int __user *optlen) +int sk_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen) { - struct sock *sk = sock->sk; + struct socket *sock = sk->sk_socket; union { int val; @@ -1557,7 +1603,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, int lv = sizeof(int); int len; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len < 0) return -EINVAL; @@ -1692,7 +1738,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred); spin_unlock(&sk->sk_peer_lock); - if (copy_to_user(optval, &peercred, len)) + if (copy_to_sockptr(optval, &peercred, len)) return -EFAULT; goto lenout; } @@ -1710,11 +1756,11 @@ int sock_getsockopt(struct socket *sock, int level, int optname, if (len < n * sizeof(gid_t)) { len = n * sizeof(gid_t); put_cred(cred); - return put_user(len, optlen) ? -EFAULT : -ERANGE; + return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE; } len = n * sizeof(gid_t); - ret = groups_to_user((gid_t __user *)optval, cred->group_info); + ret = groups_to_user(optval, cred->group_info); put_cred(cred); if (ret) return ret; @@ -1730,7 +1776,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, return -ENOTCONN; if (lv < len) return -EINVAL; - if (copy_to_user(optval, address, len)) + if (copy_to_sockptr(optval, address, len)) return -EFAULT; goto lenout; } @@ -1747,7 +1793,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, break; case SO_PEERSEC: - return security_socket_getpeersec_stream(sock, optval, optlen, len); + return security_socket_getpeersec_stream(sock, optval.user, optlen.user, len); case SO_MARK: v.val = sk->sk_mark; @@ -1779,7 +1825,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, return sock_getbindtodevice(sk, optval, optlen, len); case SO_GET_FILTER: - len = sk_get_filter(sk, (struct sock_filter __user *)optval, len); + len = sk_get_filter(sk, optval, len); if (len < 0) return len; @@ -1827,7 +1873,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname, sk_get_meminfo(sk, meminfo); len = min_t(unsigned int, len, sizeof(meminfo)); - if (copy_to_user(optval, &meminfo, len)) + if (copy_to_sockptr(optval, &meminfo, len)) return -EFAULT; goto lenout; @@ -1896,14 +1942,22 @@ int sock_getsockopt(struct socket *sock, int level, int optname, if (len > lv) len = lv; - if (copy_to_user(optval, &v, len)) + if (copy_to_sockptr(optval, &v, len)) return -EFAULT; lenout: - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; return 0; } +int sock_getsockopt(struct socket *sock, int level, int optname, + char __user *optval, int __user *optlen) +{ + return sk_getsockopt(sock->sk, level, optname, + USER_SOCKPTR(optval), + USER_SOCKPTR(optlen)); +} + /* * Initialize an sk_lock. * @@ -2536,7 +2590,7 @@ struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */ if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) > - sysctl_optmem_max) + READ_ONCE(sysctl_optmem_max)) return NULL; skb = alloc_skb(size, priority); @@ -2554,8 +2608,10 @@ struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, */ void *sock_kmalloc(struct sock *sk, int size, gfp_t priority) { - if ((unsigned int)size <= sysctl_optmem_max && - atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) { + int optmem_max = READ_ONCE(sysctl_optmem_max); + + if ((unsigned int)size <= optmem_max && + atomic_read(&sk->sk_omem_alloc) + size < optmem_max) { void *mem; /* First do the add, to avoid the race if kmalloc * might sleep. @@ -3309,8 +3365,8 @@ void sock_init_data(struct socket *sock, struct sock *sk) timer_setup(&sk->sk_timer, NULL, 0); sk->sk_allocation = GFP_KERNEL; - sk->sk_rcvbuf = sysctl_rmem_default; - sk->sk_sndbuf = sysctl_wmem_default; + sk->sk_rcvbuf = READ_ONCE(sysctl_rmem_default); + sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); sk->sk_state = TCP_CLOSE; sk_set_socket(sk, sock); @@ -3365,7 +3421,7 @@ void sock_init_data(struct socket *sock, struct sock *sk) #ifdef CONFIG_NET_RX_BUSY_POLL sk->sk_napi_id = 0; - sk->sk_ll_usec = sysctl_net_busy_read; + sk->sk_ll_usec = READ_ONCE(sysctl_net_busy_read); #endif sk->sk_max_pacing_rate = ~0UL; @@ -3554,7 +3610,8 @@ int sock_common_getsockopt(struct socket *sock, int level, int optname, { struct sock *sk = sock->sk; - return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen); + /* IPV6_ADDRFORM can change sk->sk_prot under us. */ + return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen); } EXPORT_SYMBOL(sock_common_getsockopt); @@ -3580,7 +3637,8 @@ int sock_common_setsockopt(struct socket *sock, int level, int optname, { struct sock *sk = sock->sk; - return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen); + /* IPV6_ADDRFORM can change sk->sk_prot under us. */ + return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen); } EXPORT_SYMBOL(sock_common_setsockopt); diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 9a9fb9487d636ac223b91d438c4add1ece7732c4..a660baedd9e799e0e90bdc005144a96d84560a47 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -41,7 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr) attr->map_flags & ~SOCK_CREATE_FLAG_MASK) return ERR_PTR(-EINVAL); - stab = kzalloc(sizeof(*stab), GFP_USER | __GFP_ACCOUNT); + stab = bpf_map_area_alloc(sizeof(*stab), NUMA_NO_NODE); if (!stab) return ERR_PTR(-ENOMEM); @@ -52,7 +52,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr) sizeof(struct sock *), stab->map.numa_node); if (!stab->sks) { - kfree(stab); + bpf_map_area_free(stab); return ERR_PTR(-ENOMEM); } @@ -361,7 +361,7 @@ static void sock_map_free(struct bpf_map *map) synchronize_rcu(); bpf_map_area_free(stab->sks); - kfree(stab); + bpf_map_area_free(stab); } static void sock_map_release_progs(struct bpf_map *map) @@ -1085,7 +1085,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr) if (attr->key_size > MAX_BPF_STACK) return ERR_PTR(-E2BIG); - htab = kzalloc(sizeof(*htab), GFP_USER | __GFP_ACCOUNT); + htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE); if (!htab) return ERR_PTR(-ENOMEM); @@ -1115,7 +1115,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr) return &htab->map; free_htab: - kfree(htab); + bpf_map_area_free(htab); return ERR_PTR(err); } @@ -1168,7 +1168,7 @@ static void sock_hash_free(struct bpf_map *map) synchronize_rcu(); bpf_map_area_free(htab->buckets); - kfree(htab); + bpf_map_area_free(htab); } static void *sock_hash_lookup_sys(struct bpf_map *map, void *key) diff --git a/net/core/stream.c b/net/core/stream.c index ccc083cdef23266dc100368a7183bdeb0e99cbb7..75fded8495f5b70700637b125a4993021deac407 100644 --- a/net/core/stream.c +++ b/net/core/stream.c @@ -123,7 +123,7 @@ int sk_stream_wait_memory(struct sock *sk, long *timeo_p) DEFINE_WAIT_FUNC(wait, woken_wake_function); if (sk_stream_memory_free(sk)) - current_timeo = vm_wait = (prandom_u32() % (HZ / 5)) + 2; + current_timeo = vm_wait = prandom_u32_max(HZ / 5) + 2; add_wait_queue(sk_sleep(sk), &wait); @@ -159,7 +159,8 @@ int sk_stream_wait_memory(struct sock *sk, long *timeo_p) *timeo_p = current_timeo; } out: - remove_wait_queue(sk_sleep(sk), &wait); + if (!sock_flag(sk, SOCK_DEAD)) + remove_wait_queue(sk_sleep(sk), &wait); return err; do_error: diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index 71a13596ea2bffb64b48399d9aeb2dc04c3446c6..5b1ce656baa1d4701a1b62724edc437d847d8f5f 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -29,7 +29,6 @@ static int int_3600 = 3600; static int min_sndbuf = SOCK_MIN_SNDBUF; static int min_rcvbuf = SOCK_MIN_RCVBUF; static int max_skb_frags = MAX_SKB_FRAGS; -static long long_max __maybe_unused = LONG_MAX; static int net_msg_warn; /* Unused, but still a sysctl */ @@ -234,14 +233,17 @@ static int set_default_qdisc(struct ctl_table *table, int write, static int proc_do_dev_weight(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { - int ret; + static DEFINE_MUTEX(dev_weight_mutex); + int ret, weight; + mutex_lock(&dev_weight_mutex); ret = proc_dointvec(table, write, buffer, lenp, ppos); - if (ret != 0) - return ret; - - dev_rx_weight = weight_p * dev_weight_rx_bias; - dev_tx_weight = weight_p * dev_weight_tx_bias; + if (!ret && write) { + weight = READ_ONCE(weight_p); + WRITE_ONCE(dev_rx_weight, weight * dev_weight_rx_bias); + WRITE_ONCE(dev_tx_weight, weight * dev_weight_tx_bias); + } + mutex_unlock(&dev_weight_mutex); return ret; } diff --git a/net/core/xdp.c b/net/core/xdp.c index 24420209bf0e45e55d122bb8c2bc4923a4c8b90f..844c9d99dc0ecf882f2730069a048256895b62f1 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -375,19 +375,17 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model); void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct, struct xdp_buff *xdp) { - struct xdp_mem_allocator *xa; struct page *page; switch (mem->type) { case MEM_TYPE_PAGE_POOL: - rcu_read_lock(); - /* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */ - xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params); page = virt_to_head_page(data); if (napi_direct && xdp_return_frame_no_direct()) napi_direct = false; - page_pool_put_full_page(xa->page_pool, page, napi_direct); - rcu_read_unlock(); + /* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE) + * as mem->type knows this a page_pool page + */ + page_pool_put_full_page(page->pp, page, napi_direct); break; case MEM_TYPE_PAGE_SHARED: page_frag_free(data); diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c index da6e3b20cd758211f07b8d4058a700715a6afeac..713b7b8dad7e58844e67da727b831d0f9f28dc38 100644 --- a/net/dccp/ipv4.c +++ b/net/dccp/ipv4.c @@ -45,10 +45,11 @@ static unsigned int dccp_v4_pernet_id __read_mostly; int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { const struct sockaddr_in *usin = (struct sockaddr_in *)uaddr; + struct inet_bind_hashbucket *prev_addr_hashbucket = NULL; + __be32 daddr, nexthop, prev_sk_rcv_saddr; struct inet_sock *inet = inet_sk(sk); struct dccp_sock *dp = dccp_sk(sk); __be16 orig_sport, orig_dport; - __be32 daddr, nexthop; struct flowi4 *fl4; struct rtable *rt; int err; @@ -89,9 +90,29 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) if (inet_opt == NULL || !inet_opt->opt.srr) daddr = fl4->daddr; - if (inet->inet_saddr == 0) + if (inet->inet_saddr == 0) { + if (inet_csk(sk)->icsk_bind2_hash) { + prev_addr_hashbucket = + inet_bhashfn_portaddr(&dccp_hashinfo, sk, + sock_net(sk), + inet->inet_num); + prev_sk_rcv_saddr = sk->sk_rcv_saddr; + } inet->inet_saddr = fl4->saddr; + } + sk_rcv_saddr_set(sk, inet->inet_saddr); + + if (prev_addr_hashbucket) { + err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk); + if (err) { + inet->inet_saddr = 0; + sk_rcv_saddr_set(sk, prev_sk_rcv_saddr); + ip_rt_put(rt); + return err; + } + } + inet->inet_dport = usin->sin_port; sk_daddr_set(sk, daddr); @@ -123,7 +144,7 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) inet->inet_daddr, inet->inet_sport, inet->inet_dport); - inet->inet_id = prandom_u32(); + inet->inet_id = get_random_u16(); err = dccp_connect(sk); rt = NULL; @@ -422,7 +443,7 @@ struct sock *dccp_v4_request_recv_sock(const struct sock *sk, RCU_INIT_POINTER(newinet->inet_opt, rcu_dereference(ireq->ireq_opt)); newinet->mc_index = inet_iif(skb); newinet->mc_ttl = ip_hdr(skb)->ttl; - newinet->inet_id = prandom_u32(); + newinet->inet_id = get_random_u16(); if (dst == NULL && (dst = inet_csk_route_child_sock(sk, newsk, req)) == NULL) goto put_and_exit; diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c index fd44638ec16b846777655d4a4a13cdde12c1fe4d..e57b43006074627776c8119a9b349d99ea27b1d4 100644 --- a/net/dccp/ipv6.c +++ b/net/dccp/ipv6.c @@ -934,8 +934,26 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr, } if (saddr == NULL) { + struct inet_bind_hashbucket *prev_addr_hashbucket = NULL; + struct in6_addr prev_v6_rcv_saddr; + + if (icsk->icsk_bind2_hash) { + prev_addr_hashbucket = inet_bhashfn_portaddr(&dccp_hashinfo, + sk, sock_net(sk), + inet->inet_num); + prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr; + } + saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + + if (prev_addr_hashbucket) { + err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk); + if (err) { + sk->sk_v6_rcv_saddr = prev_v6_rcv_saddr; + goto failure; + } + } } /* set the source address */ diff --git a/net/dccp/proto.c b/net/dccp/proto.c index e13641c65f88e0e0c1bf5f748898bd99b73a9722..c548ca3e9b0e81a27380d8e026f5a5a420f96737 100644 --- a/net/dccp/proto.c +++ b/net/dccp/proto.c @@ -1120,6 +1120,12 @@ static int __init dccp_init(void) SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); if (!dccp_hashinfo.bind_bucket_cachep) goto out_free_hashinfo2; + dccp_hashinfo.bind2_bucket_cachep = + kmem_cache_create("dccp_bind2_bucket", + sizeof(struct inet_bind2_bucket), 0, + SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL); + if (!dccp_hashinfo.bind2_bucket_cachep) + goto out_free_bind_bucket_cachep; /* * Size and allocate the main established and bind bucket @@ -1150,7 +1156,7 @@ static int __init dccp_init(void) if (!dccp_hashinfo.ehash) { DCCP_CRIT("Failed to allocate DCCP established hash table"); - goto out_free_bind_bucket_cachep; + goto out_free_bind2_bucket_cachep; } for (i = 0; i <= dccp_hashinfo.ehash_mask; i++) @@ -1176,14 +1182,26 @@ static int __init dccp_init(void) goto out_free_dccp_locks; } + dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *) + __get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order); + + if (!dccp_hashinfo.bhash2) { + DCCP_CRIT("Failed to allocate DCCP bind2 hash table"); + goto out_free_dccp_bhash; + } + for (i = 0; i < dccp_hashinfo.bhash_size; i++) { spin_lock_init(&dccp_hashinfo.bhash[i].lock); INIT_HLIST_HEAD(&dccp_hashinfo.bhash[i].chain); + spin_lock_init(&dccp_hashinfo.bhash2[i].lock); + INIT_HLIST_HEAD(&dccp_hashinfo.bhash2[i].chain); } + dccp_hashinfo.pernet = false; + rc = dccp_mib_init(); if (rc) - goto out_free_dccp_bhash; + goto out_free_dccp_bhash2; rc = dccp_ackvec_init(); if (rc) @@ -1207,30 +1225,38 @@ out_ackvec_exit: dccp_ackvec_exit(); out_free_dccp_mib: dccp_mib_exit(); +out_free_dccp_bhash2: + free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order); out_free_dccp_bhash: free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order); out_free_dccp_locks: inet_ehash_locks_free(&dccp_hashinfo); out_free_dccp_ehash: free_pages((unsigned long)dccp_hashinfo.ehash, ehash_order); +out_free_bind2_bucket_cachep: + kmem_cache_destroy(dccp_hashinfo.bind2_bucket_cachep); out_free_bind_bucket_cachep: kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep); out_free_hashinfo2: inet_hashinfo2_free_mod(&dccp_hashinfo); out_fail: dccp_hashinfo.bhash = NULL; + dccp_hashinfo.bhash2 = NULL; dccp_hashinfo.ehash = NULL; dccp_hashinfo.bind_bucket_cachep = NULL; + dccp_hashinfo.bind2_bucket_cachep = NULL; return rc; } static void __exit dccp_fini(void) { + int bhash_order = get_order(dccp_hashinfo.bhash_size * + sizeof(struct inet_bind_hashbucket)); + ccid_cleanup_builtins(); dccp_mib_exit(); - free_pages((unsigned long)dccp_hashinfo.bhash, - get_order(dccp_hashinfo.bhash_size * - sizeof(struct inet_bind_hashbucket))); + free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order); + free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order); free_pages((unsigned long)dccp_hashinfo.ehash, get_order((dccp_hashinfo.ehash_mask + 1) * sizeof(struct inet_ehash_bucket))); diff --git a/net/decnet/Kconfig b/net/decnet/Kconfig deleted file mode 100644 index 24336bdb1054662b22d25c94b5cc1ccf5c071200..0000000000000000000000000000000000000000 --- a/net/decnet/Kconfig +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# DECnet configuration -# -config DECNET - tristate "DECnet Support" - help - The DECnet networking protocol was used in many products made by - Digital (now Compaq). It provides reliable stream and sequenced - packet communications over which run a variety of services similar - to those which run over TCP/IP. - - To find some tools to use with the kernel layer support, please - look at Patrick Caulfield's web site: - . - - More detailed documentation is available in - . - - Be sure to say Y to "/proc file system support" and "Sysctl support" - below when using DECnet, since you will need sysctl support to aid - in configuration at run time. - - The DECnet code is also available as a module ( = code which can be - inserted in and removed from the running kernel whenever you want). - The module is called decnet. - -config DECNET_ROUTER - bool "DECnet: router support" - depends on DECNET - select FIB_RULES - help - Add support for turning your DECnet Endnode into a level 1 or 2 - router. This is an experimental, but functional option. If you - do say Y here, then make sure that you also say Y to "Kernel/User - network link driver", "Routing messages" and "Network packet - filtering". The first two are required to allow configuration via - rtnetlink (you will need Alexey Kuznetsov's iproute2 package - from ). The "Network packet - filtering" option will be required for the forthcoming routing daemon - to work. - - See for more information. diff --git a/net/decnet/Makefile b/net/decnet/Makefile deleted file mode 100644 index 07b38e441b2d0f6a6fb54ad35b838e0311cce22f..0000000000000000000000000000000000000000 --- a/net/decnet/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_DECNET) += decnet.o - -decnet-y := af_decnet.o dn_nsp_in.o dn_nsp_out.o \ - dn_route.o dn_dev.o dn_neigh.o dn_timer.o -decnet-$(CONFIG_DECNET_ROUTER) += dn_fib.o dn_rules.o dn_table.o -decnet-y += sysctl_net_decnet.o - -obj-$(CONFIG_NETFILTER) += netfilter/ diff --git a/net/decnet/README b/net/decnet/README deleted file mode 100644 index 60e7ec88c81fd14a7d1f14ce51b34681fe279f50..0000000000000000000000000000000000000000 --- a/net/decnet/README +++ /dev/null @@ -1,8 +0,0 @@ - Linux DECnet Project - ====================== - -The documentation for this kernel subsystem is available in the -Documentation/networking subdirectory of this distribution and also -on line at http://www.chygwyn.com/DECnet/ - -Steve Whitehouse diff --git a/net/decnet/af_decnet.c b/net/decnet/af_decnet.c deleted file mode 100644 index 6582dfdfb93270881d1c65b4d095614300bdde5d..0000000000000000000000000000000000000000 --- a/net/decnet/af_decnet.c +++ /dev/null @@ -1,2404 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later - -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Socket Layer Interface - * - * Authors: Eduardo Marcelo Serrat - * Patrick Caulfield - * - * Changes: - * Steve Whitehouse: Copied from Eduardo Serrat and Patrick Caulfield's - * version of the code. Original copyright preserved - * below. - * Steve Whitehouse: Some bug fixes, cleaning up some code to make it - * compatible with my routing layer. - * Steve Whitehouse: Merging changes from Eduardo Serrat and Patrick - * Caulfield. - * Steve Whitehouse: Further bug fixes, checking module code still works - * with new routing layer. - * Steve Whitehouse: Additional set/get_sockopt() calls. - * Steve Whitehouse: Fixed TIOCINQ ioctl to be same as Eduardo's new - * code. - * Steve Whitehouse: recvmsg() changed to try and behave in a POSIX like - * way. Didn't manage it entirely, but its better. - * Steve Whitehouse: ditto for sendmsg(). - * Steve Whitehouse: A selection of bug fixes to various things. - * Steve Whitehouse: Added TIOCOUTQ ioctl. - * Steve Whitehouse: Fixes to username2sockaddr & sockaddr2username. - * Steve Whitehouse: Fixes to connect() error returns. - * Patrick Caulfield: Fixes to delayed acceptance logic. - * David S. Miller: New socket locking - * Steve Whitehouse: Socket list hashing/locking - * Arnaldo C. Melo: use capable, not suser - * Steve Whitehouse: Removed unused code. Fix to use sk->allocation - * when required. - * Patrick Caulfield: /proc/net/decnet now has object name/number - * Steve Whitehouse: Fixed local port allocation, hashed sk list - * Matthew Wilcox: Fixes for dn_ioctl() - * Steve Whitehouse: New connect/accept logic to allow timeouts and - * prepare for sendpage etc. - */ - - -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - - -HISTORY: - -Version Kernel Date Author/Comments -------- ------ ---- --------------- -Version 0.0.1 2.0.30 01-dic-97 Eduardo Marcelo Serrat - (emserrat@geocities.com) - - First Development of DECnet Socket La- - yer for Linux. Only supports outgoing - connections. - -Version 0.0.2 2.1.105 20-jun-98 Patrick J. Caulfield - (patrick@pandh.demon.co.uk) - - Port to new kernel development version. - -Version 0.0.3 2.1.106 25-jun-98 Eduardo Marcelo Serrat - (emserrat@geocities.com) - _ - Added support for incoming connections - so we can start developing server apps - on Linux. - - - Module Support -Version 0.0.4 2.1.109 21-jul-98 Eduardo Marcelo Serrat - (emserrat@geocities.com) - _ - Added support for X11R6.4. Now we can - use DECnet transport for X on Linux!!! - - -Version 0.0.5 2.1.110 01-aug-98 Eduardo Marcelo Serrat - (emserrat@geocities.com) - Removed bugs on flow control - Removed bugs on incoming accessdata - order - - -Version 0.0.6 2.1.110 07-aug-98 Eduardo Marcelo Serrat - dn_recvmsg fixes - - Patrick J. Caulfield - dn_bind fixes -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct dn_sock { - struct sock sk; - struct dn_scp scp; -}; - -static void dn_keepalive(struct sock *sk); - -#define DN_SK_HASH_SHIFT 8 -#define DN_SK_HASH_SIZE (1 << DN_SK_HASH_SHIFT) -#define DN_SK_HASH_MASK (DN_SK_HASH_SIZE - 1) - - -static const struct proto_ops dn_proto_ops; -static DEFINE_RWLOCK(dn_hash_lock); -static struct hlist_head dn_sk_hash[DN_SK_HASH_SIZE]; -static struct hlist_head dn_wild_sk; -static atomic_long_t decnet_memory_allocated; -static DEFINE_PER_CPU(int, decnet_memory_per_cpu_fw_alloc); - -static int __dn_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen, int flags); -static int __dn_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen, int flags); - -static struct hlist_head *dn_find_list(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - if (scp->addr.sdn_flags & SDF_WILD) - return hlist_empty(&dn_wild_sk) ? &dn_wild_sk : NULL; - - return &dn_sk_hash[le16_to_cpu(scp->addrloc) & DN_SK_HASH_MASK]; -} - -/* - * Valid ports are those greater than zero and not already in use. - */ -static int check_port(__le16 port) -{ - struct sock *sk; - - if (port == 0) - return -1; - - sk_for_each(sk, &dn_sk_hash[le16_to_cpu(port) & DN_SK_HASH_MASK]) { - struct dn_scp *scp = DN_SK(sk); - if (scp->addrloc == port) - return -1; - } - return 0; -} - -static unsigned short port_alloc(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - static unsigned short port = 0x2000; - unsigned short i_port = port; - - while(check_port(cpu_to_le16(++port)) != 0) { - if (port == i_port) - return 0; - } - - scp->addrloc = cpu_to_le16(port); - - return 1; -} - -/* - * Since this is only ever called from user - * level, we don't need a write_lock() version - * of this. - */ -static int dn_hash_sock(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - struct hlist_head *list; - int rv = -EUSERS; - - BUG_ON(sk_hashed(sk)); - - write_lock_bh(&dn_hash_lock); - - if (!scp->addrloc && !port_alloc(sk)) - goto out; - - rv = -EADDRINUSE; - if ((list = dn_find_list(sk)) == NULL) - goto out; - - sk_add_node(sk, list); - rv = 0; -out: - write_unlock_bh(&dn_hash_lock); - return rv; -} - -static void dn_unhash_sock(struct sock *sk) -{ - write_lock(&dn_hash_lock); - sk_del_node_init(sk); - write_unlock(&dn_hash_lock); -} - -static void dn_unhash_sock_bh(struct sock *sk) -{ - write_lock_bh(&dn_hash_lock); - sk_del_node_init(sk); - write_unlock_bh(&dn_hash_lock); -} - -static struct hlist_head *listen_hash(struct sockaddr_dn *addr) -{ - int i; - unsigned int hash = addr->sdn_objnum; - - if (hash == 0) { - hash = addr->sdn_objnamel; - for(i = 0; i < le16_to_cpu(addr->sdn_objnamel); i++) { - hash ^= addr->sdn_objname[i]; - hash ^= (hash << 3); - } - } - - return &dn_sk_hash[hash & DN_SK_HASH_MASK]; -} - -/* - * Called to transform a socket from bound (i.e. with a local address) - * into a listening socket (doesn't need a local port number) and rehashes - * based upon the object name/number. - */ -static void dn_rehash_sock(struct sock *sk) -{ - struct hlist_head *list; - struct dn_scp *scp = DN_SK(sk); - - if (scp->addr.sdn_flags & SDF_WILD) - return; - - write_lock_bh(&dn_hash_lock); - sk_del_node_init(sk); - DN_SK(sk)->addrloc = 0; - list = listen_hash(&DN_SK(sk)->addr); - sk_add_node(sk, list); - write_unlock_bh(&dn_hash_lock); -} - -int dn_sockaddr2username(struct sockaddr_dn *sdn, unsigned char *buf, unsigned char type) -{ - int len = 2; - - *buf++ = type; - - switch (type) { - case 0: - *buf++ = sdn->sdn_objnum; - break; - case 1: - *buf++ = 0; - *buf++ = le16_to_cpu(sdn->sdn_objnamel); - memcpy(buf, sdn->sdn_objname, le16_to_cpu(sdn->sdn_objnamel)); - len = 3 + le16_to_cpu(sdn->sdn_objnamel); - break; - case 2: - memset(buf, 0, 5); - buf += 5; - *buf++ = le16_to_cpu(sdn->sdn_objnamel); - memcpy(buf, sdn->sdn_objname, le16_to_cpu(sdn->sdn_objnamel)); - len = 7 + le16_to_cpu(sdn->sdn_objnamel); - break; - } - - return len; -} - -/* - * On reception of usernames, we handle types 1 and 0 for destination - * addresses only. Types 2 and 4 are used for source addresses, but the - * UIC, GIC are ignored and they are both treated the same way. Type 3 - * is never used as I've no idea what its purpose might be or what its - * format is. - */ -int dn_username2sockaddr(unsigned char *data, int len, struct sockaddr_dn *sdn, unsigned char *fmt) -{ - unsigned char type; - int size = len; - int namel = 12; - - sdn->sdn_objnum = 0; - sdn->sdn_objnamel = cpu_to_le16(0); - memset(sdn->sdn_objname, 0, DN_MAXOBJL); - - if (len < 2) - return -1; - - len -= 2; - *fmt = *data++; - type = *data++; - - switch (*fmt) { - case 0: - sdn->sdn_objnum = type; - return 2; - case 1: - namel = 16; - break; - case 2: - len -= 4; - data += 4; - break; - case 4: - len -= 8; - data += 8; - break; - default: - return -1; - } - - len -= 1; - - if (len < 0) - return -1; - - sdn->sdn_objnamel = cpu_to_le16(*data++); - len -= le16_to_cpu(sdn->sdn_objnamel); - - if ((len < 0) || (le16_to_cpu(sdn->sdn_objnamel) > namel)) - return -1; - - memcpy(sdn->sdn_objname, data, le16_to_cpu(sdn->sdn_objnamel)); - - return size - len; -} - -struct sock *dn_sklist_find_listener(struct sockaddr_dn *addr) -{ - struct hlist_head *list = listen_hash(addr); - struct sock *sk; - - read_lock(&dn_hash_lock); - sk_for_each(sk, list) { - struct dn_scp *scp = DN_SK(sk); - if (sk->sk_state != TCP_LISTEN) - continue; - if (scp->addr.sdn_objnum) { - if (scp->addr.sdn_objnum != addr->sdn_objnum) - continue; - } else { - if (addr->sdn_objnum) - continue; - if (scp->addr.sdn_objnamel != addr->sdn_objnamel) - continue; - if (memcmp(scp->addr.sdn_objname, addr->sdn_objname, le16_to_cpu(addr->sdn_objnamel)) != 0) - continue; - } - sock_hold(sk); - read_unlock(&dn_hash_lock); - return sk; - } - - sk = sk_head(&dn_wild_sk); - if (sk) { - if (sk->sk_state == TCP_LISTEN) - sock_hold(sk); - else - sk = NULL; - } - - read_unlock(&dn_hash_lock); - return sk; -} - -struct sock *dn_find_by_skb(struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct sock *sk; - struct dn_scp *scp; - - read_lock(&dn_hash_lock); - sk_for_each(sk, &dn_sk_hash[le16_to_cpu(cb->dst_port) & DN_SK_HASH_MASK]) { - scp = DN_SK(sk); - if (cb->src != dn_saddr2dn(&scp->peer)) - continue; - if (cb->dst_port != scp->addrloc) - continue; - if (scp->addrrem && (cb->src_port != scp->addrrem)) - continue; - sock_hold(sk); - goto found; - } - sk = NULL; -found: - read_unlock(&dn_hash_lock); - return sk; -} - - - -static void dn_destruct(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - skb_queue_purge(&scp->data_xmit_queue); - skb_queue_purge(&scp->other_xmit_queue); - skb_queue_purge(&scp->other_receive_queue); - - dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1)); -} - -static unsigned long dn_memory_pressure; - -static void dn_enter_memory_pressure(struct sock *sk) -{ - if (!dn_memory_pressure) { - dn_memory_pressure = 1; - } -} - -static struct proto dn_proto = { - .name = "NSP", - .owner = THIS_MODULE, - .enter_memory_pressure = dn_enter_memory_pressure, - .memory_pressure = &dn_memory_pressure, - - .memory_allocated = &decnet_memory_allocated, - .per_cpu_fw_alloc = &decnet_memory_per_cpu_fw_alloc, - - .sysctl_mem = sysctl_decnet_mem, - .sysctl_wmem = sysctl_decnet_wmem, - .sysctl_rmem = sysctl_decnet_rmem, - .max_header = DN_MAX_NSP_DATA_HEADER + 64, - .obj_size = sizeof(struct dn_sock), -}; - -static struct sock *dn_alloc_sock(struct net *net, struct socket *sock, gfp_t gfp, int kern) -{ - struct dn_scp *scp; - struct sock *sk = sk_alloc(net, PF_DECnet, gfp, &dn_proto, kern); - - if (!sk) - goto out; - - if (sock) - sock->ops = &dn_proto_ops; - sock_init_data(sock, sk); - - sk->sk_backlog_rcv = dn_nsp_backlog_rcv; - sk->sk_destruct = dn_destruct; - sk->sk_no_check_tx = 1; - sk->sk_family = PF_DECnet; - sk->sk_protocol = 0; - sk->sk_allocation = gfp; - sk->sk_sndbuf = READ_ONCE(sysctl_decnet_wmem[1]); - sk->sk_rcvbuf = READ_ONCE(sysctl_decnet_rmem[1]); - - /* Initialization of DECnet Session Control Port */ - scp = DN_SK(sk); - scp->state = DN_O; /* Open */ - scp->numdat = 1; /* Next data seg to tx */ - scp->numoth = 1; /* Next oth data to tx */ - scp->ackxmt_dat = 0; /* Last data seg ack'ed */ - scp->ackxmt_oth = 0; /* Last oth data ack'ed */ - scp->ackrcv_dat = 0; /* Highest data ack recv*/ - scp->ackrcv_oth = 0; /* Last oth data ack rec*/ - scp->flowrem_sw = DN_SEND; - scp->flowloc_sw = DN_SEND; - scp->flowrem_dat = 0; - scp->flowrem_oth = 1; - scp->flowloc_dat = 0; - scp->flowloc_oth = 1; - scp->services_rem = 0; - scp->services_loc = 1 | NSP_FC_NONE; - scp->info_rem = 0; - scp->info_loc = 0x03; /* NSP version 4.1 */ - scp->segsize_rem = 230 - DN_MAX_NSP_DATA_HEADER; /* Default: Updated by remote segsize */ - scp->nonagle = 0; - scp->multi_ireq = 1; - scp->accept_mode = ACC_IMMED; - scp->addr.sdn_family = AF_DECnet; - scp->peer.sdn_family = AF_DECnet; - scp->accessdata.acc_accl = 5; - memcpy(scp->accessdata.acc_acc, "LINUX", 5); - - scp->max_window = NSP_MAX_WINDOW; - scp->snd_window = NSP_MIN_WINDOW; - scp->nsp_srtt = NSP_INITIAL_SRTT; - scp->nsp_rttvar = NSP_INITIAL_RTTVAR; - scp->nsp_rxtshift = 0; - - skb_queue_head_init(&scp->data_xmit_queue); - skb_queue_head_init(&scp->other_xmit_queue); - skb_queue_head_init(&scp->other_receive_queue); - - scp->persist = 0; - scp->persist_fxn = NULL; - scp->keepalive = 10 * HZ; - scp->keepalive_fxn = dn_keepalive; - - dn_start_slow_timer(sk); -out: - return sk; -} - -/* - * Keepalive timer. - * FIXME: Should respond to SO_KEEPALIVE etc. - */ -static void dn_keepalive(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - /* - * By checking the other_data transmit queue is empty - * we are double checking that we are not sending too - * many of these keepalive frames. - */ - if (skb_queue_empty(&scp->other_xmit_queue)) - dn_nsp_send_link(sk, DN_NOCHANGE, 0); -} - - -/* - * Timer for shutdown/destroyed sockets. - * When socket is dead & no packets have been sent for a - * certain amount of time, they are removed by this - * routine. Also takes care of sending out DI & DC - * frames at correct times. - */ -int dn_destroy_timer(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - scp->persist = dn_nsp_persist(sk); - - switch (scp->state) { - case DN_DI: - dn_nsp_send_disc(sk, NSP_DISCINIT, 0, GFP_ATOMIC); - if (scp->nsp_rxtshift >= decnet_di_count) - scp->state = DN_CN; - return 0; - - case DN_DR: - dn_nsp_send_disc(sk, NSP_DISCINIT, 0, GFP_ATOMIC); - if (scp->nsp_rxtshift >= decnet_dr_count) - scp->state = DN_DRC; - return 0; - - case DN_DN: - if (scp->nsp_rxtshift < decnet_dn_count) { - /* printk(KERN_DEBUG "dn_destroy_timer: DN\n"); */ - dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC, - GFP_ATOMIC); - return 0; - } - } - - scp->persist = (HZ * decnet_time_wait); - - if (sk->sk_socket) - return 0; - - if (time_after_eq(jiffies, scp->stamp + HZ * decnet_time_wait)) { - dn_unhash_sock(sk); - sock_put(sk); - return 1; - } - - return 0; -} - -static void dn_destroy_sock(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - scp->nsp_rxtshift = 0; /* reset back off */ - - if (sk->sk_socket) { - if (sk->sk_socket->state != SS_UNCONNECTED) - sk->sk_socket->state = SS_DISCONNECTING; - } - - sk->sk_state = TCP_CLOSE; - - switch (scp->state) { - case DN_DN: - dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC, - sk->sk_allocation); - scp->persist_fxn = dn_destroy_timer; - scp->persist = dn_nsp_persist(sk); - break; - case DN_CR: - scp->state = DN_DR; - goto disc_reject; - case DN_RUN: - scp->state = DN_DI; - fallthrough; - case DN_DI: - case DN_DR: -disc_reject: - dn_nsp_send_disc(sk, NSP_DISCINIT, 0, sk->sk_allocation); - fallthrough; - case DN_NC: - case DN_NR: - case DN_RJ: - case DN_DIC: - case DN_CN: - case DN_DRC: - case DN_CI: - case DN_CD: - scp->persist_fxn = dn_destroy_timer; - scp->persist = dn_nsp_persist(sk); - break; - default: - printk(KERN_DEBUG "DECnet: dn_destroy_sock passed socket in invalid state\n"); - fallthrough; - case DN_O: - dn_stop_slow_timer(sk); - - dn_unhash_sock_bh(sk); - sock_put(sk); - - break; - } -} - -char *dn_addr2asc(__u16 addr, char *buf) -{ - unsigned short node, area; - - node = addr & 0x03ff; - area = addr >> 10; - sprintf(buf, "%hd.%hd", area, node); - - return buf; -} - - - -static int dn_create(struct net *net, struct socket *sock, int protocol, - int kern) -{ - struct sock *sk; - - if (protocol < 0 || protocol > U8_MAX) - return -EINVAL; - - if (!net_eq(net, &init_net)) - return -EAFNOSUPPORT; - - switch (sock->type) { - case SOCK_SEQPACKET: - if (protocol != DNPROTO_NSP) - return -EPROTONOSUPPORT; - break; - case SOCK_STREAM: - break; - default: - return -ESOCKTNOSUPPORT; - } - - - if ((sk = dn_alloc_sock(net, sock, GFP_KERNEL, kern)) == NULL) - return -ENOBUFS; - - sk->sk_protocol = protocol; - - return 0; -} - - -static int -dn_release(struct socket *sock) -{ - struct sock *sk = sock->sk; - - if (sk) { - sock_orphan(sk); - sock_hold(sk); - lock_sock(sk); - dn_destroy_sock(sk); - release_sock(sk); - sock_put(sk); - } - - return 0; -} - -static int dn_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - struct sockaddr_dn *saddr = (struct sockaddr_dn *)uaddr; - struct net_device *dev, *ldev; - int rv; - - if (addr_len != sizeof(struct sockaddr_dn)) - return -EINVAL; - - if (saddr->sdn_family != AF_DECnet) - return -EINVAL; - - if (le16_to_cpu(saddr->sdn_nodeaddrl) && (le16_to_cpu(saddr->sdn_nodeaddrl) != 2)) - return -EINVAL; - - if (le16_to_cpu(saddr->sdn_objnamel) > DN_MAXOBJL) - return -EINVAL; - - if (saddr->sdn_flags & ~SDF_WILD) - return -EINVAL; - - if (!capable(CAP_NET_BIND_SERVICE) && (saddr->sdn_objnum || - (saddr->sdn_flags & SDF_WILD))) - return -EACCES; - - if (!(saddr->sdn_flags & SDF_WILD)) { - if (le16_to_cpu(saddr->sdn_nodeaddrl)) { - rcu_read_lock(); - ldev = NULL; - for_each_netdev_rcu(&init_net, dev) { - if (!dev->dn_ptr) - continue; - if (dn_dev_islocal(dev, dn_saddr2dn(saddr))) { - ldev = dev; - break; - } - } - rcu_read_unlock(); - if (ldev == NULL) - return -EADDRNOTAVAIL; - } - } - - rv = -EINVAL; - lock_sock(sk); - if (sock_flag(sk, SOCK_ZAPPED)) { - memcpy(&scp->addr, saddr, addr_len); - sock_reset_flag(sk, SOCK_ZAPPED); - - rv = dn_hash_sock(sk); - if (rv) - sock_set_flag(sk, SOCK_ZAPPED); - } - release_sock(sk); - - return rv; -} - - -static int dn_auto_bind(struct socket *sock) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - int rv; - - sock_reset_flag(sk, SOCK_ZAPPED); - - scp->addr.sdn_flags = 0; - scp->addr.sdn_objnum = 0; - - /* - * This stuff is to keep compatibility with Eduardo's - * patch. I hope I can dispense with it shortly... - */ - if ((scp->accessdata.acc_accl != 0) && - (scp->accessdata.acc_accl <= 12)) { - - scp->addr.sdn_objnamel = cpu_to_le16(scp->accessdata.acc_accl); - memcpy(scp->addr.sdn_objname, scp->accessdata.acc_acc, le16_to_cpu(scp->addr.sdn_objnamel)); - - scp->accessdata.acc_accl = 0; - memset(scp->accessdata.acc_acc, 0, 40); - } - /* End of compatibility stuff */ - - scp->addr.sdn_add.a_len = cpu_to_le16(2); - rv = dn_dev_bind_default((__le16 *)scp->addr.sdn_add.a_addr); - if (rv == 0) { - rv = dn_hash_sock(sk); - if (rv) - sock_set_flag(sk, SOCK_ZAPPED); - } - - return rv; -} - -static int dn_confirm_accept(struct sock *sk, long *timeo, gfp_t allocation) -{ - struct dn_scp *scp = DN_SK(sk); - DEFINE_WAIT_FUNC(wait, woken_wake_function); - int err; - - if (scp->state != DN_CR) - return -EINVAL; - - scp->state = DN_CC; - scp->segsize_loc = dst_metric_advmss(__sk_dst_get(sk)); - dn_send_conn_conf(sk, allocation); - - add_wait_queue(sk_sleep(sk), &wait); - for(;;) { - release_sock(sk); - if (scp->state == DN_CC) - *timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo); - lock_sock(sk); - err = 0; - if (scp->state == DN_RUN) - break; - err = sock_error(sk); - if (err) - break; - err = sock_intr_errno(*timeo); - if (signal_pending(current)) - break; - err = -EAGAIN; - if (!*timeo) - break; - } - remove_wait_queue(sk_sleep(sk), &wait); - if (err == 0) { - sk->sk_socket->state = SS_CONNECTED; - } else if (scp->state != DN_CC) { - sk->sk_socket->state = SS_UNCONNECTED; - } - return err; -} - -static int dn_wait_run(struct sock *sk, long *timeo) -{ - struct dn_scp *scp = DN_SK(sk); - DEFINE_WAIT_FUNC(wait, woken_wake_function); - int err = 0; - - if (scp->state == DN_RUN) - goto out; - - if (!*timeo) - return -EALREADY; - - add_wait_queue(sk_sleep(sk), &wait); - for(;;) { - release_sock(sk); - if (scp->state == DN_CI || scp->state == DN_CC) - *timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo); - lock_sock(sk); - err = 0; - if (scp->state == DN_RUN) - break; - err = sock_error(sk); - if (err) - break; - err = sock_intr_errno(*timeo); - if (signal_pending(current)) - break; - err = -ETIMEDOUT; - if (!*timeo) - break; - } - remove_wait_queue(sk_sleep(sk), &wait); -out: - if (err == 0) { - sk->sk_socket->state = SS_CONNECTED; - } else if (scp->state != DN_CI && scp->state != DN_CC) { - sk->sk_socket->state = SS_UNCONNECTED; - } - return err; -} - -static int __dn_connect(struct sock *sk, struct sockaddr_dn *addr, int addrlen, long *timeo, int flags) -{ - struct socket *sock = sk->sk_socket; - struct dn_scp *scp = DN_SK(sk); - int err = -EISCONN; - struct flowidn fld; - struct dst_entry *dst; - - if (sock->state == SS_CONNECTED) - goto out; - - if (sock->state == SS_CONNECTING) { - err = 0; - if (scp->state == DN_RUN) { - sock->state = SS_CONNECTED; - goto out; - } - err = -ECONNREFUSED; - if (scp->state != DN_CI && scp->state != DN_CC) { - sock->state = SS_UNCONNECTED; - goto out; - } - return dn_wait_run(sk, timeo); - } - - err = -EINVAL; - if (scp->state != DN_O) - goto out; - - if (addr == NULL || addrlen != sizeof(struct sockaddr_dn)) - goto out; - if (addr->sdn_family != AF_DECnet) - goto out; - if (addr->sdn_flags & SDF_WILD) - goto out; - - if (sock_flag(sk, SOCK_ZAPPED)) { - err = dn_auto_bind(sk->sk_socket); - if (err) - goto out; - } - - memcpy(&scp->peer, addr, sizeof(struct sockaddr_dn)); - - err = -EHOSTUNREACH; - memset(&fld, 0, sizeof(fld)); - fld.flowidn_oif = sk->sk_bound_dev_if; - fld.daddr = dn_saddr2dn(&scp->peer); - fld.saddr = dn_saddr2dn(&scp->addr); - dn_sk_ports_copy(&fld, scp); - fld.flowidn_proto = DNPROTO_NSP; - if (dn_route_output_sock(&sk->sk_dst_cache, &fld, sk, flags) < 0) - goto out; - dst = __sk_dst_get(sk); - sk->sk_route_caps = dst->dev->features; - sock->state = SS_CONNECTING; - scp->state = DN_CI; - scp->segsize_loc = dst_metric_advmss(dst); - - dn_nsp_send_conninit(sk, NSP_CI); - err = -EINPROGRESS; - if (*timeo) { - err = dn_wait_run(sk, timeo); - } -out: - return err; -} - -static int dn_connect(struct socket *sock, struct sockaddr *uaddr, int addrlen, int flags) -{ - struct sockaddr_dn *addr = (struct sockaddr_dn *)uaddr; - struct sock *sk = sock->sk; - int err; - long timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); - - lock_sock(sk); - err = __dn_connect(sk, addr, addrlen, &timeo, 0); - release_sock(sk); - - return err; -} - -static inline int dn_check_state(struct sock *sk, struct sockaddr_dn *addr, int addrlen, long *timeo, int flags) -{ - struct dn_scp *scp = DN_SK(sk); - - switch (scp->state) { - case DN_RUN: - return 0; - case DN_CR: - return dn_confirm_accept(sk, timeo, sk->sk_allocation); - case DN_CI: - case DN_CC: - return dn_wait_run(sk, timeo); - case DN_O: - return __dn_connect(sk, addr, addrlen, timeo, flags); - } - - return -EINVAL; -} - - -static void dn_access_copy(struct sk_buff *skb, struct accessdata_dn *acc) -{ - unsigned char *ptr = skb->data; - - acc->acc_userl = *ptr++; - memcpy(&acc->acc_user, ptr, acc->acc_userl); - ptr += acc->acc_userl; - - acc->acc_passl = *ptr++; - memcpy(&acc->acc_pass, ptr, acc->acc_passl); - ptr += acc->acc_passl; - - acc->acc_accl = *ptr++; - memcpy(&acc->acc_acc, ptr, acc->acc_accl); - - skb_pull(skb, acc->acc_accl + acc->acc_passl + acc->acc_userl + 3); - -} - -static void dn_user_copy(struct sk_buff *skb, struct optdata_dn *opt) -{ - unsigned char *ptr = skb->data; - u16 len = *ptr++; /* yes, it's 8bit on the wire */ - - BUG_ON(len > 16); /* we've checked the contents earlier */ - opt->opt_optl = cpu_to_le16(len); - opt->opt_status = 0; - memcpy(opt->opt_data, ptr, len); - skb_pull(skb, len + 1); -} - -static struct sk_buff *dn_wait_for_connect(struct sock *sk, long *timeo) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - struct sk_buff *skb = NULL; - int err = 0; - - add_wait_queue(sk_sleep(sk), &wait); - for(;;) { - release_sock(sk); - skb = skb_dequeue(&sk->sk_receive_queue); - if (skb == NULL) { - *timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo); - skb = skb_dequeue(&sk->sk_receive_queue); - } - lock_sock(sk); - if (skb != NULL) - break; - err = -EINVAL; - if (sk->sk_state != TCP_LISTEN) - break; - err = sock_intr_errno(*timeo); - if (signal_pending(current)) - break; - err = -EAGAIN; - if (!*timeo) - break; - } - remove_wait_queue(sk_sleep(sk), &wait); - - return skb == NULL ? ERR_PTR(err) : skb; -} - -static int dn_accept(struct socket *sock, struct socket *newsock, int flags, - bool kern) -{ - struct sock *sk = sock->sk, *newsk; - struct sk_buff *skb = NULL; - struct dn_skb_cb *cb; - unsigned char menuver; - int err = 0; - unsigned char type; - long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); - struct dst_entry *dst; - - lock_sock(sk); - - if (sk->sk_state != TCP_LISTEN || DN_SK(sk)->state != DN_O) { - release_sock(sk); - return -EINVAL; - } - - skb = skb_dequeue(&sk->sk_receive_queue); - if (skb == NULL) { - skb = dn_wait_for_connect(sk, &timeo); - if (IS_ERR(skb)) { - release_sock(sk); - return PTR_ERR(skb); - } - } - - cb = DN_SKB_CB(skb); - sk_acceptq_removed(sk); - newsk = dn_alloc_sock(sock_net(sk), newsock, sk->sk_allocation, kern); - if (newsk == NULL) { - release_sock(sk); - kfree_skb(skb); - return -ENOBUFS; - } - release_sock(sk); - - dst = skb_dst(skb); - sk_dst_set(newsk, dst); - skb_dst_set(skb, NULL); - - DN_SK(newsk)->state = DN_CR; - DN_SK(newsk)->addrrem = cb->src_port; - DN_SK(newsk)->services_rem = cb->services; - DN_SK(newsk)->info_rem = cb->info; - DN_SK(newsk)->segsize_rem = cb->segsize; - DN_SK(newsk)->accept_mode = DN_SK(sk)->accept_mode; - - if (DN_SK(newsk)->segsize_rem < 230) - DN_SK(newsk)->segsize_rem = 230; - - if ((DN_SK(newsk)->services_rem & NSP_FC_MASK) == NSP_FC_NONE) - DN_SK(newsk)->max_window = decnet_no_fc_max_cwnd; - - newsk->sk_state = TCP_LISTEN; - memcpy(&(DN_SK(newsk)->addr), &(DN_SK(sk)->addr), sizeof(struct sockaddr_dn)); - - /* - * If we are listening on a wild socket, we don't want - * the newly created socket on the wrong hash queue. - */ - DN_SK(newsk)->addr.sdn_flags &= ~SDF_WILD; - - skb_pull(skb, dn_username2sockaddr(skb->data, skb->len, &(DN_SK(newsk)->addr), &type)); - skb_pull(skb, dn_username2sockaddr(skb->data, skb->len, &(DN_SK(newsk)->peer), &type)); - *(__le16 *)(DN_SK(newsk)->peer.sdn_add.a_addr) = cb->src; - *(__le16 *)(DN_SK(newsk)->addr.sdn_add.a_addr) = cb->dst; - - menuver = *skb->data; - skb_pull(skb, 1); - - if (menuver & DN_MENUVER_ACC) - dn_access_copy(skb, &(DN_SK(newsk)->accessdata)); - - if (menuver & DN_MENUVER_USR) - dn_user_copy(skb, &(DN_SK(newsk)->conndata_in)); - - if (menuver & DN_MENUVER_PRX) - DN_SK(newsk)->peer.sdn_flags |= SDF_PROXY; - - if (menuver & DN_MENUVER_UIC) - DN_SK(newsk)->peer.sdn_flags |= SDF_UICPROXY; - - kfree_skb(skb); - - memcpy(&(DN_SK(newsk)->conndata_out), &(DN_SK(sk)->conndata_out), - sizeof(struct optdata_dn)); - memcpy(&(DN_SK(newsk)->discdata_out), &(DN_SK(sk)->discdata_out), - sizeof(struct optdata_dn)); - - lock_sock(newsk); - err = dn_hash_sock(newsk); - if (err == 0) { - sock_reset_flag(newsk, SOCK_ZAPPED); - dn_send_conn_ack(newsk); - - /* - * Here we use sk->sk_allocation since although the conn conf is - * for the newsk, the context is the old socket. - */ - if (DN_SK(newsk)->accept_mode == ACC_IMMED) - err = dn_confirm_accept(newsk, &timeo, - sk->sk_allocation); - } - release_sock(newsk); - return err; -} - - -static int dn_getname(struct socket *sock, struct sockaddr *uaddr,int peer) -{ - struct sockaddr_dn *sa = (struct sockaddr_dn *)uaddr; - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - - lock_sock(sk); - - if (peer) { - if ((sock->state != SS_CONNECTED && - sock->state != SS_CONNECTING) && - scp->accept_mode == ACC_IMMED) { - release_sock(sk); - return -ENOTCONN; - } - - memcpy(sa, &scp->peer, sizeof(struct sockaddr_dn)); - } else { - memcpy(sa, &scp->addr, sizeof(struct sockaddr_dn)); - } - - release_sock(sk); - - return sizeof(struct sockaddr_dn); -} - - -static __poll_t dn_poll(struct file *file, struct socket *sock, poll_table *wait) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - __poll_t mask = datagram_poll(file, sock, wait); - - if (!skb_queue_empty_lockless(&scp->other_receive_queue)) - mask |= EPOLLRDBAND; - - return mask; -} - -static int dn_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - int err = -EOPNOTSUPP; - long amount = 0; - struct sk_buff *skb; - int val; - - switch(cmd) - { - case SIOCGIFADDR: - case SIOCSIFADDR: - return dn_dev_ioctl(cmd, (void __user *)arg); - - case SIOCATMARK: - lock_sock(sk); - val = !skb_queue_empty(&scp->other_receive_queue); - if (scp->state != DN_RUN) - val = -ENOTCONN; - release_sock(sk); - return val; - - case TIOCOUTQ: - amount = sk->sk_sndbuf - sk_wmem_alloc_get(sk); - if (amount < 0) - amount = 0; - err = put_user(amount, (int __user *)arg); - break; - - case TIOCINQ: - lock_sock(sk); - skb = skb_peek(&scp->other_receive_queue); - if (skb) { - amount = skb->len; - } else { - skb_queue_walk(&sk->sk_receive_queue, skb) - amount += skb->len; - } - release_sock(sk); - err = put_user(amount, (int __user *)arg); - break; - - default: - err = -ENOIOCTLCMD; - break; - } - - return err; -} - -static int dn_listen(struct socket *sock, int backlog) -{ - struct sock *sk = sock->sk; - int err = -EINVAL; - - lock_sock(sk); - - if (sock_flag(sk, SOCK_ZAPPED)) - goto out; - - if ((DN_SK(sk)->state != DN_O) || (sk->sk_state == TCP_LISTEN)) - goto out; - - sk->sk_max_ack_backlog = backlog; - sk->sk_ack_backlog = 0; - sk->sk_state = TCP_LISTEN; - err = 0; - dn_rehash_sock(sk); - -out: - release_sock(sk); - - return err; -} - - -static int dn_shutdown(struct socket *sock, int how) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - int err = -ENOTCONN; - - lock_sock(sk); - - if (sock->state == SS_UNCONNECTED) - goto out; - - err = 0; - if (sock->state == SS_DISCONNECTING) - goto out; - - err = -EINVAL; - if (scp->state == DN_O) - goto out; - - if (how != SHUT_RDWR) - goto out; - - sk->sk_shutdown = SHUTDOWN_MASK; - dn_destroy_sock(sk); - err = 0; - -out: - release_sock(sk); - - return err; -} - -static int dn_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen) -{ - struct sock *sk = sock->sk; - int err; - - lock_sock(sk); - err = __dn_setsockopt(sock, level, optname, optval, optlen, 0); - release_sock(sk); -#ifdef CONFIG_NETFILTER - /* we need to exclude all possible ENOPROTOOPTs except default case */ - if (err == -ENOPROTOOPT && optname != DSO_LINKINFO && - optname != DSO_STREAM && optname != DSO_SEQPACKET) - err = nf_setsockopt(sk, PF_DECnet, optname, optval, optlen); -#endif - - return err; -} - -static int __dn_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen, int flags) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - long timeo; - union { - struct optdata_dn opt; - struct accessdata_dn acc; - int mode; - unsigned long win; - int val; - unsigned char services; - unsigned char info; - } u; - int err; - - if (optlen && sockptr_is_null(optval)) - return -EINVAL; - - if (optlen > sizeof(u)) - return -EINVAL; - - if (copy_from_sockptr(&u, optval, optlen)) - return -EFAULT; - - switch (optname) { - case DSO_CONDATA: - if (sock->state == SS_CONNECTED) - return -EISCONN; - if ((scp->state != DN_O) && (scp->state != DN_CR)) - return -EINVAL; - - if (optlen != sizeof(struct optdata_dn)) - return -EINVAL; - - if (le16_to_cpu(u.opt.opt_optl) > 16) - return -EINVAL; - - memcpy(&scp->conndata_out, &u.opt, optlen); - break; - - case DSO_DISDATA: - if (sock->state != SS_CONNECTED && - scp->accept_mode == ACC_IMMED) - return -ENOTCONN; - - if (optlen != sizeof(struct optdata_dn)) - return -EINVAL; - - if (le16_to_cpu(u.opt.opt_optl) > 16) - return -EINVAL; - - memcpy(&scp->discdata_out, &u.opt, optlen); - break; - - case DSO_CONACCESS: - if (sock->state == SS_CONNECTED) - return -EISCONN; - if (scp->state != DN_O) - return -EINVAL; - - if (optlen != sizeof(struct accessdata_dn)) - return -EINVAL; - - if ((u.acc.acc_accl > DN_MAXACCL) || - (u.acc.acc_passl > DN_MAXACCL) || - (u.acc.acc_userl > DN_MAXACCL)) - return -EINVAL; - - memcpy(&scp->accessdata, &u.acc, optlen); - break; - - case DSO_ACCEPTMODE: - if (sock->state == SS_CONNECTED) - return -EISCONN; - if (scp->state != DN_O) - return -EINVAL; - - if (optlen != sizeof(int)) - return -EINVAL; - - if ((u.mode != ACC_IMMED) && (u.mode != ACC_DEFER)) - return -EINVAL; - - scp->accept_mode = (unsigned char)u.mode; - break; - - case DSO_CONACCEPT: - if (scp->state != DN_CR) - return -EINVAL; - timeo = sock_rcvtimeo(sk, 0); - err = dn_confirm_accept(sk, &timeo, sk->sk_allocation); - return err; - - case DSO_CONREJECT: - if (scp->state != DN_CR) - return -EINVAL; - - scp->state = DN_DR; - sk->sk_shutdown = SHUTDOWN_MASK; - dn_nsp_send_disc(sk, 0x38, 0, sk->sk_allocation); - break; - - case DSO_MAXWINDOW: - if (optlen != sizeof(unsigned long)) - return -EINVAL; - if (u.win > NSP_MAX_WINDOW) - u.win = NSP_MAX_WINDOW; - if (u.win == 0) - return -EINVAL; - scp->max_window = u.win; - if (scp->snd_window > u.win) - scp->snd_window = u.win; - break; - - case DSO_NODELAY: - if (optlen != sizeof(int)) - return -EINVAL; - if (scp->nonagle == TCP_NAGLE_CORK) - return -EINVAL; - scp->nonagle = (u.val == 0) ? 0 : TCP_NAGLE_OFF; - /* if (scp->nonagle == 1) { Push pending frames } */ - break; - - case DSO_CORK: - if (optlen != sizeof(int)) - return -EINVAL; - if (scp->nonagle == TCP_NAGLE_OFF) - return -EINVAL; - scp->nonagle = (u.val == 0) ? 0 : TCP_NAGLE_CORK; - /* if (scp->nonagle == 0) { Push pending frames } */ - break; - - case DSO_SERVICES: - if (optlen != sizeof(unsigned char)) - return -EINVAL; - if ((u.services & ~NSP_FC_MASK) != 0x01) - return -EINVAL; - if ((u.services & NSP_FC_MASK) == NSP_FC_MASK) - return -EINVAL; - scp->services_loc = u.services; - break; - - case DSO_INFO: - if (optlen != sizeof(unsigned char)) - return -EINVAL; - if (u.info & 0xfc) - return -EINVAL; - scp->info_loc = u.info; - break; - - case DSO_LINKINFO: - case DSO_STREAM: - case DSO_SEQPACKET: - default: - return -ENOPROTOOPT; - } - - return 0; -} - -static int dn_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) -{ - struct sock *sk = sock->sk; - int err; - - lock_sock(sk); - err = __dn_getsockopt(sock, level, optname, optval, optlen, 0); - release_sock(sk); -#ifdef CONFIG_NETFILTER - if (err == -ENOPROTOOPT && optname != DSO_STREAM && - optname != DSO_SEQPACKET && optname != DSO_CONACCEPT && - optname != DSO_CONREJECT) { - int len; - - if (get_user(len, optlen)) - return -EFAULT; - - err = nf_getsockopt(sk, PF_DECnet, optname, optval, &len); - if (err >= 0) - err = put_user(len, optlen); - } -#endif - - return err; -} - -static int __dn_getsockopt(struct socket *sock, int level,int optname, char __user *optval,int __user *optlen, int flags) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - struct linkinfo_dn link; - unsigned int r_len; - void *r_data = NULL; - unsigned int val; - - if(get_user(r_len , optlen)) - return -EFAULT; - - switch (optname) { - case DSO_CONDATA: - if (r_len > sizeof(struct optdata_dn)) - r_len = sizeof(struct optdata_dn); - r_data = &scp->conndata_in; - break; - - case DSO_DISDATA: - if (r_len > sizeof(struct optdata_dn)) - r_len = sizeof(struct optdata_dn); - r_data = &scp->discdata_in; - break; - - case DSO_CONACCESS: - if (r_len > sizeof(struct accessdata_dn)) - r_len = sizeof(struct accessdata_dn); - r_data = &scp->accessdata; - break; - - case DSO_ACCEPTMODE: - if (r_len > sizeof(unsigned char)) - r_len = sizeof(unsigned char); - r_data = &scp->accept_mode; - break; - - case DSO_LINKINFO: - if (r_len > sizeof(struct linkinfo_dn)) - r_len = sizeof(struct linkinfo_dn); - - memset(&link, 0, sizeof(link)); - - switch (sock->state) { - case SS_CONNECTING: - link.idn_linkstate = LL_CONNECTING; - break; - case SS_DISCONNECTING: - link.idn_linkstate = LL_DISCONNECTING; - break; - case SS_CONNECTED: - link.idn_linkstate = LL_RUNNING; - break; - default: - link.idn_linkstate = LL_INACTIVE; - } - - link.idn_segsize = scp->segsize_rem; - r_data = &link; - break; - - case DSO_MAXWINDOW: - if (r_len > sizeof(unsigned long)) - r_len = sizeof(unsigned long); - r_data = &scp->max_window; - break; - - case DSO_NODELAY: - if (r_len > sizeof(int)) - r_len = sizeof(int); - val = (scp->nonagle == TCP_NAGLE_OFF); - r_data = &val; - break; - - case DSO_CORK: - if (r_len > sizeof(int)) - r_len = sizeof(int); - val = (scp->nonagle == TCP_NAGLE_CORK); - r_data = &val; - break; - - case DSO_SERVICES: - if (r_len > sizeof(unsigned char)) - r_len = sizeof(unsigned char); - r_data = &scp->services_rem; - break; - - case DSO_INFO: - if (r_len > sizeof(unsigned char)) - r_len = sizeof(unsigned char); - r_data = &scp->info_rem; - break; - - case DSO_STREAM: - case DSO_SEQPACKET: - case DSO_CONACCEPT: - case DSO_CONREJECT: - default: - return -ENOPROTOOPT; - } - - if (r_data) { - if (copy_to_user(optval, r_data, r_len)) - return -EFAULT; - if (put_user(r_len, optlen)) - return -EFAULT; - } - - return 0; -} - - -static int dn_data_ready(struct sock *sk, struct sk_buff_head *q, int flags, int target) -{ - struct sk_buff *skb; - int len = 0; - - if (flags & MSG_OOB) - return !skb_queue_empty(q) ? 1 : 0; - - skb_queue_walk(q, skb) { - struct dn_skb_cb *cb = DN_SKB_CB(skb); - len += skb->len; - - if (cb->nsp_flags & 0x40) { - /* SOCK_SEQPACKET reads to EOM */ - if (sk->sk_type == SOCK_SEQPACKET) - return 1; - /* so does SOCK_STREAM unless WAITALL is specified */ - if (!(flags & MSG_WAITALL)) - return 1; - } - - /* minimum data length for read exceeded */ - if (len >= target) - return 1; - } - - return 0; -} - - -static int dn_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, - int flags) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - struct sk_buff_head *queue = &sk->sk_receive_queue; - size_t target = size > 1 ? 1 : 0; - size_t copied = 0; - int rv = 0; - struct sk_buff *skb, *n; - struct dn_skb_cb *cb = NULL; - unsigned char eor = 0; - long timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); - - lock_sock(sk); - - if (sock_flag(sk, SOCK_ZAPPED)) { - rv = -EADDRNOTAVAIL; - goto out; - } - - if (sk->sk_shutdown & RCV_SHUTDOWN) { - rv = 0; - goto out; - } - - rv = dn_check_state(sk, NULL, 0, &timeo, flags); - if (rv) - goto out; - - if (flags & ~(MSG_CMSG_COMPAT|MSG_PEEK|MSG_OOB|MSG_WAITALL|MSG_DONTWAIT|MSG_NOSIGNAL)) { - rv = -EOPNOTSUPP; - goto out; - } - - if (flags & MSG_OOB) - queue = &scp->other_receive_queue; - - if (flags & MSG_WAITALL) - target = size; - - - /* - * See if there is data ready to read, sleep if there isn't - */ - for(;;) { - DEFINE_WAIT_FUNC(wait, woken_wake_function); - - if (sk->sk_err) - goto out; - - if (!skb_queue_empty(&scp->other_receive_queue)) { - if (!(flags & MSG_OOB)) { - msg->msg_flags |= MSG_OOB; - if (!scp->other_report) { - scp->other_report = 1; - goto out; - } - } - } - - if (scp->state != DN_RUN) - goto out; - - if (signal_pending(current)) { - rv = sock_intr_errno(timeo); - goto out; - } - - if (dn_data_ready(sk, queue, flags, target)) - break; - - if (flags & MSG_DONTWAIT) { - rv = -EWOULDBLOCK; - goto out; - } - - add_wait_queue(sk_sleep(sk), &wait); - sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); - sk_wait_event(sk, &timeo, dn_data_ready(sk, queue, flags, target), &wait); - sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); - remove_wait_queue(sk_sleep(sk), &wait); - } - - skb_queue_walk_safe(queue, skb, n) { - unsigned int chunk = skb->len; - cb = DN_SKB_CB(skb); - - if ((chunk + copied) > size) - chunk = size - copied; - - if (memcpy_to_msg(msg, skb->data, chunk)) { - rv = -EFAULT; - break; - } - copied += chunk; - - if (!(flags & MSG_PEEK)) - skb_pull(skb, chunk); - - eor = cb->nsp_flags & 0x40; - - if (skb->len == 0) { - skb_unlink(skb, queue); - kfree_skb(skb); - /* - * N.B. Don't refer to skb or cb after this point - * in loop. - */ - if ((scp->flowloc_sw == DN_DONTSEND) && !dn_congested(sk)) { - scp->flowloc_sw = DN_SEND; - dn_nsp_send_link(sk, DN_SEND, 0); - } - } - - if (eor) { - if (sk->sk_type == SOCK_SEQPACKET) - break; - if (!(flags & MSG_WAITALL)) - break; - } - - if (flags & MSG_OOB) - break; - - if (copied >= target) - break; - } - - rv = copied; - - - if (eor && (sk->sk_type == SOCK_SEQPACKET)) - msg->msg_flags |= MSG_EOR; - -out: - if (rv == 0) - rv = (flags & MSG_PEEK) ? -sk->sk_err : sock_error(sk); - - if ((rv >= 0) && msg->msg_name) { - __sockaddr_check_size(sizeof(struct sockaddr_dn)); - memcpy(msg->msg_name, &scp->peer, sizeof(struct sockaddr_dn)); - msg->msg_namelen = sizeof(struct sockaddr_dn); - } - - release_sock(sk); - - return rv; -} - - -static inline int dn_queue_too_long(struct dn_scp *scp, struct sk_buff_head *queue, int flags) -{ - unsigned char fctype = scp->services_rem & NSP_FC_MASK; - if (skb_queue_len(queue) >= scp->snd_window) - return 1; - if (fctype != NSP_FC_NONE) { - if (flags & MSG_OOB) { - if (scp->flowrem_oth == 0) - return 1; - } else { - if (scp->flowrem_dat == 0) - return 1; - } - } - return 0; -} - -/* - * The DECnet spec requires that the "routing layer" accepts packets which - * are at least 230 bytes in size. This excludes any headers which the NSP - * layer might add, so we always assume that we'll be using the maximal - * length header on data packets. The variation in length is due to the - * inclusion (or not) of the two 16 bit acknowledgement fields so it doesn't - * make much practical difference. - */ -unsigned int dn_mss_from_pmtu(struct net_device *dev, int mtu) -{ - unsigned int mss = 230 - DN_MAX_NSP_DATA_HEADER; - if (dev) { - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - mtu -= LL_RESERVED_SPACE(dev); - if (dn_db->use_long) - mtu -= 21; - else - mtu -= 6; - mtu -= DN_MAX_NSP_DATA_HEADER; - } else { - /* - * 21 = long header, 16 = guess at MAC header length - */ - mtu -= (21 + DN_MAX_NSP_DATA_HEADER + 16); - } - if (mtu > mss) - mss = mtu; - return mss; -} - -static inline unsigned int dn_current_mss(struct sock *sk, int flags) -{ - struct dst_entry *dst = __sk_dst_get(sk); - struct dn_scp *scp = DN_SK(sk); - int mss_now = min_t(int, scp->segsize_loc, scp->segsize_rem); - - /* Other data messages are limited to 16 bytes per packet */ - if (flags & MSG_OOB) - return 16; - - /* This works out the maximum size of segment we can send out */ - if (dst) { - u32 mtu = dst_mtu(dst); - mss_now = min_t(int, dn_mss_from_pmtu(dst->dev, mtu), mss_now); - } - - return mss_now; -} - -/* - * N.B. We get the timeout wrong here, but then we always did get it - * wrong before and this is another step along the road to correcting - * it. It ought to get updated each time we pass through the routine, - * but in practise it probably doesn't matter too much for now. - */ -static inline struct sk_buff *dn_alloc_send_pskb(struct sock *sk, - unsigned long datalen, int noblock, - int *errcode) -{ - struct sk_buff *skb = sock_alloc_send_skb(sk, datalen, - noblock, errcode); - if (skb) { - skb->protocol = htons(ETH_P_DNA_RT); - skb->pkt_type = PACKET_OUTGOING; - } - return skb; -} - -static int dn_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) -{ - struct sock *sk = sock->sk; - struct dn_scp *scp = DN_SK(sk); - size_t mss; - struct sk_buff_head *queue = &scp->data_xmit_queue; - int flags = msg->msg_flags; - int err = 0; - size_t sent = 0; - int addr_len = msg->msg_namelen; - DECLARE_SOCKADDR(struct sockaddr_dn *, addr, msg->msg_name); - struct sk_buff *skb = NULL; - struct dn_skb_cb *cb; - size_t len; - unsigned char fctype; - long timeo; - - if (flags & ~(MSG_TRYHARD|MSG_OOB|MSG_DONTWAIT|MSG_EOR|MSG_NOSIGNAL|MSG_MORE|MSG_CMSG_COMPAT)) - return -EOPNOTSUPP; - - if (addr_len && (addr_len != sizeof(struct sockaddr_dn))) - return -EINVAL; - - lock_sock(sk); - timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); - /* - * The only difference between stream sockets and sequenced packet - * sockets is that the stream sockets always behave as if MSG_EOR - * has been set. - */ - if (sock->type == SOCK_STREAM) { - if (flags & MSG_EOR) { - err = -EINVAL; - goto out; - } - flags |= MSG_EOR; - } - - - err = dn_check_state(sk, addr, addr_len, &timeo, flags); - if (err) - goto out_err; - - if (sk->sk_shutdown & SEND_SHUTDOWN) { - err = -EPIPE; - if (!(flags & MSG_NOSIGNAL)) - send_sig(SIGPIPE, current, 0); - goto out_err; - } - - if ((flags & MSG_TRYHARD) && sk->sk_dst_cache) - dst_negative_advice(sk); - - mss = scp->segsize_rem; - fctype = scp->services_rem & NSP_FC_MASK; - - mss = dn_current_mss(sk, flags); - - if (flags & MSG_OOB) { - queue = &scp->other_xmit_queue; - if (size > mss) { - err = -EMSGSIZE; - goto out; - } - } - - scp->persist_fxn = dn_nsp_xmit_timeout; - - while(sent < size) { - err = sock_error(sk); - if (err) - goto out; - - if (signal_pending(current)) { - err = sock_intr_errno(timeo); - goto out; - } - - /* - * Calculate size that we wish to send. - */ - len = size - sent; - - if (len > mss) - len = mss; - - /* - * Wait for queue size to go down below the window - * size. - */ - if (dn_queue_too_long(scp, queue, flags)) { - DEFINE_WAIT_FUNC(wait, woken_wake_function); - - if (flags & MSG_DONTWAIT) { - err = -EWOULDBLOCK; - goto out; - } - - add_wait_queue(sk_sleep(sk), &wait); - sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); - sk_wait_event(sk, &timeo, - !dn_queue_too_long(scp, queue, flags), &wait); - sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); - remove_wait_queue(sk_sleep(sk), &wait); - continue; - } - - /* - * Get a suitably sized skb. - * 64 is a bit of a hack really, but its larger than any - * link-layer headers and has served us well as a good - * guess as to their real length. - */ - skb = dn_alloc_send_pskb(sk, len + 64 + DN_MAX_NSP_DATA_HEADER, - flags & MSG_DONTWAIT, &err); - - if (err) - break; - - if (!skb) - continue; - - cb = DN_SKB_CB(skb); - - skb_reserve(skb, 64 + DN_MAX_NSP_DATA_HEADER); - - if (memcpy_from_msg(skb_put(skb, len), msg, len)) { - err = -EFAULT; - goto out; - } - - if (flags & MSG_OOB) { - cb->nsp_flags = 0x30; - if (fctype != NSP_FC_NONE) - scp->flowrem_oth--; - } else { - cb->nsp_flags = 0x00; - if (scp->seg_total == 0) - cb->nsp_flags |= 0x20; - - scp->seg_total += len; - - if (((sent + len) == size) && (flags & MSG_EOR)) { - cb->nsp_flags |= 0x40; - scp->seg_total = 0; - if (fctype == NSP_FC_SCMC) - scp->flowrem_dat--; - } - if (fctype == NSP_FC_SRC) - scp->flowrem_dat--; - } - - sent += len; - dn_nsp_queue_xmit(sk, skb, sk->sk_allocation, flags & MSG_OOB); - skb = NULL; - - scp->persist = dn_nsp_persist(sk); - - } -out: - - kfree_skb(skb); - - release_sock(sk); - - return sent ? sent : err; - -out_err: - err = sk_stream_error(sk, flags, err); - release_sock(sk); - return err; -} - -static int dn_device_event(struct notifier_block *this, unsigned long event, - void *ptr) -{ - struct net_device *dev = netdev_notifier_info_to_dev(ptr); - - if (!net_eq(dev_net(dev), &init_net)) - return NOTIFY_DONE; - - switch (event) { - case NETDEV_UP: - dn_dev_up(dev); - break; - case NETDEV_DOWN: - dn_dev_down(dev); - break; - default: - break; - } - - return NOTIFY_DONE; -} - -static struct notifier_block dn_dev_notifier = { - .notifier_call = dn_device_event, -}; - -static struct packet_type dn_dix_packet_type __read_mostly = { - .type = cpu_to_be16(ETH_P_DNA_RT), - .func = dn_route_rcv, -}; - -#ifdef CONFIG_PROC_FS -struct dn_iter_state { - int bucket; -}; - -static struct sock *dn_socket_get_first(struct seq_file *seq) -{ - struct dn_iter_state *state = seq->private; - struct sock *n = NULL; - - for(state->bucket = 0; - state->bucket < DN_SK_HASH_SIZE; - ++state->bucket) { - n = sk_head(&dn_sk_hash[state->bucket]); - if (n) - break; - } - - return n; -} - -static struct sock *dn_socket_get_next(struct seq_file *seq, - struct sock *n) -{ - struct dn_iter_state *state = seq->private; - - n = sk_next(n); - while (!n) { - if (++state->bucket >= DN_SK_HASH_SIZE) - break; - n = sk_head(&dn_sk_hash[state->bucket]); - } - return n; -} - -static struct sock *socket_get_idx(struct seq_file *seq, loff_t *pos) -{ - struct sock *sk = dn_socket_get_first(seq); - - if (sk) { - while(*pos && (sk = dn_socket_get_next(seq, sk))) - --*pos; - } - return *pos ? NULL : sk; -} - -static void *dn_socket_get_idx(struct seq_file *seq, loff_t pos) -{ - void *rc; - read_lock_bh(&dn_hash_lock); - rc = socket_get_idx(seq, &pos); - if (!rc) { - read_unlock_bh(&dn_hash_lock); - } - return rc; -} - -static void *dn_socket_seq_start(struct seq_file *seq, loff_t *pos) -{ - return *pos ? dn_socket_get_idx(seq, *pos - 1) : SEQ_START_TOKEN; -} - -static void *dn_socket_seq_next(struct seq_file *seq, void *v, loff_t *pos) -{ - void *rc; - - if (v == SEQ_START_TOKEN) { - rc = dn_socket_get_idx(seq, 0); - goto out; - } - - rc = dn_socket_get_next(seq, v); - if (rc) - goto out; - read_unlock_bh(&dn_hash_lock); -out: - ++*pos; - return rc; -} - -static void dn_socket_seq_stop(struct seq_file *seq, void *v) -{ - if (v && v != SEQ_START_TOKEN) - read_unlock_bh(&dn_hash_lock); -} - -#define IS_NOT_PRINTABLE(x) ((x) < 32 || (x) > 126) - -static void dn_printable_object(struct sockaddr_dn *dn, unsigned char *buf) -{ - int i; - - switch (le16_to_cpu(dn->sdn_objnamel)) { - case 0: - sprintf(buf, "%d", dn->sdn_objnum); - break; - default: - for (i = 0; i < le16_to_cpu(dn->sdn_objnamel); i++) { - buf[i] = dn->sdn_objname[i]; - if (IS_NOT_PRINTABLE(buf[i])) - buf[i] = '.'; - } - buf[i] = 0; - } -} - -static char *dn_state2asc(unsigned char state) -{ - switch (state) { - case DN_O: - return "OPEN"; - case DN_CR: - return " CR"; - case DN_DR: - return " DR"; - case DN_DRC: - return " DRC"; - case DN_CC: - return " CC"; - case DN_CI: - return " CI"; - case DN_NR: - return " NR"; - case DN_NC: - return " NC"; - case DN_CD: - return " CD"; - case DN_RJ: - return " RJ"; - case DN_RUN: - return " RUN"; - case DN_DI: - return " DI"; - case DN_DIC: - return " DIC"; - case DN_DN: - return " DN"; - case DN_CL: - return " CL"; - case DN_CN: - return " CN"; - } - - return "????"; -} - -static inline void dn_socket_format_entry(struct seq_file *seq, struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - char buf1[DN_ASCBUF_LEN]; - char buf2[DN_ASCBUF_LEN]; - char local_object[DN_MAXOBJL+3]; - char remote_object[DN_MAXOBJL+3]; - - dn_printable_object(&scp->addr, local_object); - dn_printable_object(&scp->peer, remote_object); - - seq_printf(seq, - "%6s/%04X %04d:%04d %04d:%04d %01d %-16s " - "%6s/%04X %04d:%04d %04d:%04d %01d %-16s %4s %s\n", - dn_addr2asc(le16_to_cpu(dn_saddr2dn(&scp->addr)), buf1), - scp->addrloc, - scp->numdat, - scp->numoth, - scp->ackxmt_dat, - scp->ackxmt_oth, - scp->flowloc_sw, - local_object, - dn_addr2asc(le16_to_cpu(dn_saddr2dn(&scp->peer)), buf2), - scp->addrrem, - scp->numdat_rcv, - scp->numoth_rcv, - scp->ackrcv_dat, - scp->ackrcv_oth, - scp->flowrem_sw, - remote_object, - dn_state2asc(scp->state), - ((scp->accept_mode == ACC_IMMED) ? "IMMED" : "DEFER")); -} - -static int dn_socket_seq_show(struct seq_file *seq, void *v) -{ - if (v == SEQ_START_TOKEN) { - seq_puts(seq, "Local Remote\n"); - } else { - dn_socket_format_entry(seq, v); - } - return 0; -} - -static const struct seq_operations dn_socket_seq_ops = { - .start = dn_socket_seq_start, - .next = dn_socket_seq_next, - .stop = dn_socket_seq_stop, - .show = dn_socket_seq_show, -}; -#endif - -static const struct net_proto_family dn_family_ops = { - .family = AF_DECnet, - .create = dn_create, - .owner = THIS_MODULE, -}; - -static const struct proto_ops dn_proto_ops = { - .family = AF_DECnet, - .owner = THIS_MODULE, - .release = dn_release, - .bind = dn_bind, - .connect = dn_connect, - .socketpair = sock_no_socketpair, - .accept = dn_accept, - .getname = dn_getname, - .poll = dn_poll, - .ioctl = dn_ioctl, - .listen = dn_listen, - .shutdown = dn_shutdown, - .setsockopt = dn_setsockopt, - .getsockopt = dn_getsockopt, - .sendmsg = dn_sendmsg, - .recvmsg = dn_recvmsg, - .mmap = sock_no_mmap, - .sendpage = sock_no_sendpage, -}; - -MODULE_DESCRIPTION("The Linux DECnet Network Protocol"); -MODULE_AUTHOR("Linux DECnet Project Team"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_NETPROTO(PF_DECnet); - -static const char banner[] __initconst = KERN_INFO -"NET4: DECnet for Linux: V.2.5.68s (C) 1995-2003 Linux DECnet Project Team\n"; - -static int __init decnet_init(void) -{ - int rc; - - printk(banner); - - rc = proto_register(&dn_proto, 1); - if (rc != 0) - goto out; - - dn_neigh_init(); - dn_dev_init(); - dn_route_init(); - dn_fib_init(); - - sock_register(&dn_family_ops); - dev_add_pack(&dn_dix_packet_type); - register_netdevice_notifier(&dn_dev_notifier); - - proc_create_seq_private("decnet", 0444, init_net.proc_net, - &dn_socket_seq_ops, sizeof(struct dn_iter_state), - NULL); - dn_register_sysctl(); -out: - return rc; - -} -module_init(decnet_init); - -/* - * Prevent DECnet module unloading until its fixed properly. - * Requires an audit of the code to check for memory leaks and - * initialisation problems etc. - */ -#if 0 -static void __exit decnet_exit(void) -{ - sock_unregister(AF_DECnet); - rtnl_unregister_all(PF_DECnet); - dev_remove_pack(&dn_dix_packet_type); - - dn_unregister_sysctl(); - - unregister_netdevice_notifier(&dn_dev_notifier); - - dn_route_cleanup(); - dn_dev_cleanup(); - dn_neigh_cleanup(); - dn_fib_cleanup(); - - remove_proc_entry("decnet", init_net.proc_net); - - proto_unregister(&dn_proto); - - rcu_barrier(); /* Wait for completion of call_rcu()'s */ -} -module_exit(decnet_exit); -#endif diff --git a/net/decnet/dn_dev.c b/net/decnet/dn_dev.c deleted file mode 100644 index a09ba642b5e76abdbfd0d844d12be9572c001abc..0000000000000000000000000000000000000000 --- a/net/decnet/dn_dev.c +++ /dev/null @@ -1,1433 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Device Layer - * - * Authors: Steve Whitehouse - * Eduardo Marcelo Serrat - * - * Changes: - * Steve Whitehouse : Devices now see incoming frames so they - * can mark on who it came from. - * Steve Whitehouse : Fixed bug in creating neighbours. Each neighbour - * can now have a device specific setup func. - * Steve Whitehouse : Added /proc/sys/net/decnet/conf// - * Steve Whitehouse : Fixed bug which sometimes killed timer - * Steve Whitehouse : Multiple ifaddr support - * Steve Whitehouse : SIOCGIFCONF is now a compile time option - * Steve Whitehouse : /proc/sys/net/decnet/conf//forwarding - * Steve Whitehouse : Removed timer1 - it's a user space issue now - * Patrick Caulfield : Fixed router hello message format - * Steve Whitehouse : Got rid of constant sizes for blksize for - * devices. All mtu based now. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define DN_IFREQ_SIZE (offsetof(struct ifreq, ifr_ifru) + sizeof(struct sockaddr_dn)) - -static char dn_rt_all_end_mcast[ETH_ALEN] = {0xAB,0x00,0x00,0x04,0x00,0x00}; -static char dn_rt_all_rt_mcast[ETH_ALEN] = {0xAB,0x00,0x00,0x03,0x00,0x00}; -static char dn_hiord[ETH_ALEN] = {0xAA,0x00,0x04,0x00,0x00,0x00}; -static unsigned char dn_eco_version[3] = {0x02,0x00,0x00}; - -extern struct neigh_table dn_neigh_table; - -/* - * decnet_address is kept in network order. - */ -__le16 decnet_address = 0; - -static DEFINE_SPINLOCK(dndev_lock); -static struct net_device *decnet_default_device; -static BLOCKING_NOTIFIER_HEAD(dnaddr_chain); - -static struct dn_dev *dn_dev_create(struct net_device *dev, int *err); -static void dn_dev_delete(struct net_device *dev); -static void dn_ifaddr_notify(int event, struct dn_ifaddr *ifa); - -static int dn_eth_up(struct net_device *); -static void dn_eth_down(struct net_device *); -static void dn_send_brd_hello(struct net_device *dev, struct dn_ifaddr *ifa); -static void dn_send_ptp_hello(struct net_device *dev, struct dn_ifaddr *ifa); - -static struct dn_dev_parms dn_dev_list[] = { -{ - .type = ARPHRD_ETHER, /* Ethernet */ - .mode = DN_DEV_BCAST, - .state = DN_DEV_S_RU, - .t2 = 1, - .t3 = 10, - .name = "ethernet", - .up = dn_eth_up, - .down = dn_eth_down, - .timer3 = dn_send_brd_hello, -}, -{ - .type = ARPHRD_IPGRE, /* DECnet tunneled over GRE in IP */ - .mode = DN_DEV_BCAST, - .state = DN_DEV_S_RU, - .t2 = 1, - .t3 = 10, - .name = "ipgre", - .timer3 = dn_send_brd_hello, -}, -#if 0 -{ - .type = ARPHRD_X25, /* Bog standard X.25 */ - .mode = DN_DEV_UCAST, - .state = DN_DEV_S_DS, - .t2 = 1, - .t3 = 120, - .name = "x25", - .timer3 = dn_send_ptp_hello, -}, -#endif -#if 0 -{ - .type = ARPHRD_PPP, /* DECnet over PPP */ - .mode = DN_DEV_BCAST, - .state = DN_DEV_S_RU, - .t2 = 1, - .t3 = 10, - .name = "ppp", - .timer3 = dn_send_brd_hello, -}, -#endif -{ - .type = ARPHRD_DDCMP, /* DECnet over DDCMP */ - .mode = DN_DEV_UCAST, - .state = DN_DEV_S_DS, - .t2 = 1, - .t3 = 120, - .name = "ddcmp", - .timer3 = dn_send_ptp_hello, -}, -{ - .type = ARPHRD_LOOPBACK, /* Loopback interface - always last */ - .mode = DN_DEV_BCAST, - .state = DN_DEV_S_RU, - .t2 = 1, - .t3 = 10, - .name = "loopback", - .timer3 = dn_send_brd_hello, -} -}; - -#define DN_DEV_LIST_SIZE ARRAY_SIZE(dn_dev_list) - -#define DN_DEV_PARMS_OFFSET(x) offsetof(struct dn_dev_parms, x) - -#ifdef CONFIG_SYSCTL - -static int min_t2[] = { 1 }; -static int max_t2[] = { 60 }; /* No max specified, but this seems sensible */ -static int min_t3[] = { 1 }; -static int max_t3[] = { 8191 }; /* Must fit in 16 bits when multiplied by BCT3MULT or T3MULT */ - -static int min_priority[1]; -static int max_priority[] = { 127 }; /* From DECnet spec */ - -static int dn_forwarding_proc(struct ctl_table *, int, void *, size_t *, - loff_t *); -static struct dn_dev_sysctl_table { - struct ctl_table_header *sysctl_header; - struct ctl_table dn_dev_vars[5]; -} dn_dev_sysctl = { - NULL, - { - { - .procname = "forwarding", - .data = (void *)DN_DEV_PARMS_OFFSET(forwarding), - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = dn_forwarding_proc, - }, - { - .procname = "priority", - .data = (void *)DN_DEV_PARMS_OFFSET(priority), - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_priority, - .extra2 = &max_priority - }, - { - .procname = "t2", - .data = (void *)DN_DEV_PARMS_OFFSET(t2), - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_t2, - .extra2 = &max_t2 - }, - { - .procname = "t3", - .data = (void *)DN_DEV_PARMS_OFFSET(t3), - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_t3, - .extra2 = &max_t3 - }, - { } - }, -}; - -static void dn_dev_sysctl_register(struct net_device *dev, struct dn_dev_parms *parms) -{ - struct dn_dev_sysctl_table *t; - int i; - - char path[sizeof("net/decnet/conf/") + IFNAMSIZ]; - - t = kmemdup(&dn_dev_sysctl, sizeof(*t), GFP_KERNEL); - if (t == NULL) - return; - - for(i = 0; i < ARRAY_SIZE(t->dn_dev_vars) - 1; i++) { - long offset = (long)t->dn_dev_vars[i].data; - t->dn_dev_vars[i].data = ((char *)parms) + offset; - } - - snprintf(path, sizeof(path), "net/decnet/conf/%s", - dev? dev->name : parms->name); - - t->dn_dev_vars[0].extra1 = (void *)dev; - - t->sysctl_header = register_net_sysctl(&init_net, path, t->dn_dev_vars); - if (t->sysctl_header == NULL) - kfree(t); - else - parms->sysctl = t; -} - -static void dn_dev_sysctl_unregister(struct dn_dev_parms *parms) -{ - if (parms->sysctl) { - struct dn_dev_sysctl_table *t = parms->sysctl; - parms->sysctl = NULL; - unregister_net_sysctl_table(t->sysctl_header); - kfree(t); - } -} - -static int dn_forwarding_proc(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ -#ifdef CONFIG_DECNET_ROUTER - struct net_device *dev = table->extra1; - struct dn_dev *dn_db; - int err; - int tmp, old; - - if (table->extra1 == NULL) - return -EINVAL; - - dn_db = rcu_dereference_raw(dev->dn_ptr); - old = dn_db->parms.forwarding; - - err = proc_dointvec(table, write, buffer, lenp, ppos); - - if ((err >= 0) && write) { - if (dn_db->parms.forwarding < 0) - dn_db->parms.forwarding = 0; - if (dn_db->parms.forwarding > 2) - dn_db->parms.forwarding = 2; - /* - * What an ugly hack this is... its works, just. It - * would be nice if sysctl/proc were just that little - * bit more flexible so I don't have to write a special - * routine, or suffer hacks like this - SJW - */ - tmp = dn_db->parms.forwarding; - dn_db->parms.forwarding = old; - if (dn_db->parms.down) - dn_db->parms.down(dev); - dn_db->parms.forwarding = tmp; - if (dn_db->parms.up) - dn_db->parms.up(dev); - } - - return err; -#else - return -EINVAL; -#endif -} - -#else /* CONFIG_SYSCTL */ -static void dn_dev_sysctl_unregister(struct dn_dev_parms *parms) -{ -} -static void dn_dev_sysctl_register(struct net_device *dev, struct dn_dev_parms *parms) -{ -} - -#endif /* CONFIG_SYSCTL */ - -static inline __u16 mtu2blksize(struct net_device *dev) -{ - u32 blksize = dev->mtu; - if (blksize > 0xffff) - blksize = 0xffff; - - if (dev->type == ARPHRD_ETHER || - dev->type == ARPHRD_PPP || - dev->type == ARPHRD_IPGRE || - dev->type == ARPHRD_LOOPBACK) - blksize -= 2; - - return (__u16)blksize; -} - -static struct dn_ifaddr *dn_dev_alloc_ifa(void) -{ - struct dn_ifaddr *ifa; - - ifa = kzalloc(sizeof(*ifa), GFP_KERNEL); - - return ifa; -} - -static void dn_dev_free_ifa(struct dn_ifaddr *ifa) -{ - kfree_rcu(ifa, rcu); -} - -static void dn_dev_del_ifa(struct dn_dev *dn_db, struct dn_ifaddr __rcu **ifap, int destroy) -{ - struct dn_ifaddr *ifa1 = rtnl_dereference(*ifap); - unsigned char mac_addr[6]; - struct net_device *dev = dn_db->dev; - - ASSERT_RTNL(); - - *ifap = ifa1->ifa_next; - - if (dn_db->dev->type == ARPHRD_ETHER) { - if (ifa1->ifa_local != dn_eth2dn(dev->dev_addr)) { - dn_dn2eth(mac_addr, ifa1->ifa_local); - dev_mc_del(dev, mac_addr); - } - } - - dn_ifaddr_notify(RTM_DELADDR, ifa1); - blocking_notifier_call_chain(&dnaddr_chain, NETDEV_DOWN, ifa1); - if (destroy) { - dn_dev_free_ifa(ifa1); - - if (dn_db->ifa_list == NULL) - dn_dev_delete(dn_db->dev); - } -} - -static int dn_dev_insert_ifa(struct dn_dev *dn_db, struct dn_ifaddr *ifa) -{ - struct net_device *dev = dn_db->dev; - struct dn_ifaddr *ifa1; - unsigned char mac_addr[6]; - - ASSERT_RTNL(); - - /* Check for duplicates */ - for (ifa1 = rtnl_dereference(dn_db->ifa_list); - ifa1 != NULL; - ifa1 = rtnl_dereference(ifa1->ifa_next)) { - if (ifa1->ifa_local == ifa->ifa_local) - return -EEXIST; - } - - if (dev->type == ARPHRD_ETHER) { - if (ifa->ifa_local != dn_eth2dn(dev->dev_addr)) { - dn_dn2eth(mac_addr, ifa->ifa_local); - dev_mc_add(dev, mac_addr); - } - } - - ifa->ifa_next = dn_db->ifa_list; - rcu_assign_pointer(dn_db->ifa_list, ifa); - - dn_ifaddr_notify(RTM_NEWADDR, ifa); - blocking_notifier_call_chain(&dnaddr_chain, NETDEV_UP, ifa); - - return 0; -} - -static int dn_dev_set_ifa(struct net_device *dev, struct dn_ifaddr *ifa) -{ - struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr); - int rv; - - if (dn_db == NULL) { - int err; - dn_db = dn_dev_create(dev, &err); - if (dn_db == NULL) - return err; - } - - ifa->ifa_dev = dn_db; - - if (dev->flags & IFF_LOOPBACK) - ifa->ifa_scope = RT_SCOPE_HOST; - - rv = dn_dev_insert_ifa(dn_db, ifa); - if (rv) - dn_dev_free_ifa(ifa); - return rv; -} - - -int dn_dev_ioctl(unsigned int cmd, void __user *arg) -{ - char buffer[DN_IFREQ_SIZE]; - struct ifreq *ifr = (struct ifreq *)buffer; - struct sockaddr_dn *sdn = (struct sockaddr_dn *)&ifr->ifr_addr; - struct dn_dev *dn_db; - struct net_device *dev; - struct dn_ifaddr *ifa = NULL; - struct dn_ifaddr __rcu **ifap = NULL; - int ret = 0; - - if (copy_from_user(ifr, arg, DN_IFREQ_SIZE)) - return -EFAULT; - ifr->ifr_name[IFNAMSIZ-1] = 0; - - dev_load(&init_net, ifr->ifr_name); - - switch (cmd) { - case SIOCGIFADDR: - break; - case SIOCSIFADDR: - if (!capable(CAP_NET_ADMIN)) - return -EACCES; - if (sdn->sdn_family != AF_DECnet) - return -EINVAL; - break; - default: - return -EINVAL; - } - - rtnl_lock(); - - if ((dev = __dev_get_by_name(&init_net, ifr->ifr_name)) == NULL) { - ret = -ENODEV; - goto done; - } - - if ((dn_db = rtnl_dereference(dev->dn_ptr)) != NULL) { - for (ifap = &dn_db->ifa_list; - (ifa = rtnl_dereference(*ifap)) != NULL; - ifap = &ifa->ifa_next) - if (strcmp(ifr->ifr_name, ifa->ifa_label) == 0) - break; - } - - if (ifa == NULL && cmd != SIOCSIFADDR) { - ret = -EADDRNOTAVAIL; - goto done; - } - - switch (cmd) { - case SIOCGIFADDR: - *((__le16 *)sdn->sdn_nodeaddr) = ifa->ifa_local; - if (copy_to_user(arg, ifr, DN_IFREQ_SIZE)) - ret = -EFAULT; - break; - - case SIOCSIFADDR: - if (!ifa) { - if ((ifa = dn_dev_alloc_ifa()) == NULL) { - ret = -ENOBUFS; - break; - } - memcpy(ifa->ifa_label, dev->name, IFNAMSIZ); - } else { - if (ifa->ifa_local == dn_saddr2dn(sdn)) - break; - dn_dev_del_ifa(dn_db, ifap, 0); - } - - ifa->ifa_local = ifa->ifa_address = dn_saddr2dn(sdn); - - ret = dn_dev_set_ifa(dev, ifa); - } -done: - rtnl_unlock(); - - return ret; -} - -struct net_device *dn_dev_get_default(void) -{ - struct net_device *dev; - - spin_lock(&dndev_lock); - dev = decnet_default_device; - if (dev) { - if (dev->dn_ptr) - dev_hold(dev); - else - dev = NULL; - } - spin_unlock(&dndev_lock); - - return dev; -} - -int dn_dev_set_default(struct net_device *dev, int force) -{ - struct net_device *old = NULL; - int rv = -EBUSY; - if (!dev->dn_ptr) - return -ENODEV; - - spin_lock(&dndev_lock); - if (force || decnet_default_device == NULL) { - old = decnet_default_device; - decnet_default_device = dev; - rv = 0; - } - spin_unlock(&dndev_lock); - - dev_put(old); - return rv; -} - -static void dn_dev_check_default(struct net_device *dev) -{ - spin_lock(&dndev_lock); - if (dev == decnet_default_device) { - decnet_default_device = NULL; - } else { - dev = NULL; - } - spin_unlock(&dndev_lock); - - dev_put(dev); -} - -/* - * Called with RTNL - */ -static struct dn_dev *dn_dev_by_index(int ifindex) -{ - struct net_device *dev; - struct dn_dev *dn_dev = NULL; - - dev = __dev_get_by_index(&init_net, ifindex); - if (dev) - dn_dev = rtnl_dereference(dev->dn_ptr); - - return dn_dev; -} - -static const struct nla_policy dn_ifa_policy[IFA_MAX+1] = { - [IFA_ADDRESS] = { .type = NLA_U16 }, - [IFA_LOCAL] = { .type = NLA_U16 }, - [IFA_LABEL] = { .type = NLA_STRING, - .len = IFNAMSIZ - 1 }, - [IFA_FLAGS] = { .type = NLA_U32 }, -}; - -static int dn_nl_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(skb->sk); - struct nlattr *tb[IFA_MAX+1]; - struct dn_dev *dn_db; - struct ifaddrmsg *ifm; - struct dn_ifaddr *ifa; - struct dn_ifaddr __rcu **ifap; - int err = -EINVAL; - - if (!netlink_capable(skb, CAP_NET_ADMIN)) - return -EPERM; - - if (!net_eq(net, &init_net)) - goto errout; - - err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX, - dn_ifa_policy, extack); - if (err < 0) - goto errout; - - err = -ENODEV; - ifm = nlmsg_data(nlh); - if ((dn_db = dn_dev_by_index(ifm->ifa_index)) == NULL) - goto errout; - - err = -EADDRNOTAVAIL; - for (ifap = &dn_db->ifa_list; - (ifa = rtnl_dereference(*ifap)) != NULL; - ifap = &ifa->ifa_next) { - if (tb[IFA_LOCAL] && - nla_memcmp(tb[IFA_LOCAL], &ifa->ifa_local, 2)) - continue; - - if (tb[IFA_LABEL] && nla_strcmp(tb[IFA_LABEL], ifa->ifa_label)) - continue; - - dn_dev_del_ifa(dn_db, ifap, 1); - return 0; - } - -errout: - return err; -} - -static int dn_nl_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(skb->sk); - struct nlattr *tb[IFA_MAX+1]; - struct net_device *dev; - struct dn_dev *dn_db; - struct ifaddrmsg *ifm; - struct dn_ifaddr *ifa; - int err; - - if (!netlink_capable(skb, CAP_NET_ADMIN)) - return -EPERM; - - if (!net_eq(net, &init_net)) - return -EINVAL; - - err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX, - dn_ifa_policy, extack); - if (err < 0) - return err; - - if (tb[IFA_LOCAL] == NULL) - return -EINVAL; - - ifm = nlmsg_data(nlh); - if ((dev = __dev_get_by_index(&init_net, ifm->ifa_index)) == NULL) - return -ENODEV; - - if ((dn_db = rtnl_dereference(dev->dn_ptr)) == NULL) { - dn_db = dn_dev_create(dev, &err); - if (!dn_db) - return err; - } - - if ((ifa = dn_dev_alloc_ifa()) == NULL) - return -ENOBUFS; - - if (tb[IFA_ADDRESS] == NULL) - tb[IFA_ADDRESS] = tb[IFA_LOCAL]; - - ifa->ifa_local = nla_get_le16(tb[IFA_LOCAL]); - ifa->ifa_address = nla_get_le16(tb[IFA_ADDRESS]); - ifa->ifa_flags = tb[IFA_FLAGS] ? nla_get_u32(tb[IFA_FLAGS]) : - ifm->ifa_flags; - ifa->ifa_scope = ifm->ifa_scope; - ifa->ifa_dev = dn_db; - - if (tb[IFA_LABEL]) - nla_strscpy(ifa->ifa_label, tb[IFA_LABEL], IFNAMSIZ); - else - memcpy(ifa->ifa_label, dev->name, IFNAMSIZ); - - err = dn_dev_insert_ifa(dn_db, ifa); - if (err) - dn_dev_free_ifa(ifa); - - return err; -} - -static inline size_t dn_ifaddr_nlmsg_size(void) -{ - return NLMSG_ALIGN(sizeof(struct ifaddrmsg)) - + nla_total_size(IFNAMSIZ) /* IFA_LABEL */ - + nla_total_size(2) /* IFA_ADDRESS */ - + nla_total_size(2) /* IFA_LOCAL */ - + nla_total_size(4); /* IFA_FLAGS */ -} - -static int dn_nl_fill_ifaddr(struct sk_buff *skb, struct dn_ifaddr *ifa, - u32 portid, u32 seq, int event, unsigned int flags) -{ - struct ifaddrmsg *ifm; - struct nlmsghdr *nlh; - u32 ifa_flags = ifa->ifa_flags | IFA_F_PERMANENT; - - nlh = nlmsg_put(skb, portid, seq, event, sizeof(*ifm), flags); - if (nlh == NULL) - return -EMSGSIZE; - - ifm = nlmsg_data(nlh); - ifm->ifa_family = AF_DECnet; - ifm->ifa_prefixlen = 16; - ifm->ifa_flags = ifa_flags; - ifm->ifa_scope = ifa->ifa_scope; - ifm->ifa_index = ifa->ifa_dev->dev->ifindex; - - if ((ifa->ifa_address && - nla_put_le16(skb, IFA_ADDRESS, ifa->ifa_address)) || - (ifa->ifa_local && - nla_put_le16(skb, IFA_LOCAL, ifa->ifa_local)) || - (ifa->ifa_label[0] && - nla_put_string(skb, IFA_LABEL, ifa->ifa_label)) || - nla_put_u32(skb, IFA_FLAGS, ifa_flags)) - goto nla_put_failure; - nlmsg_end(skb, nlh); - return 0; - -nla_put_failure: - nlmsg_cancel(skb, nlh); - return -EMSGSIZE; -} - -static void dn_ifaddr_notify(int event, struct dn_ifaddr *ifa) -{ - struct sk_buff *skb; - int err = -ENOBUFS; - - skb = alloc_skb(dn_ifaddr_nlmsg_size(), GFP_KERNEL); - if (skb == NULL) - goto errout; - - err = dn_nl_fill_ifaddr(skb, ifa, 0, 0, event, 0); - if (err < 0) { - /* -EMSGSIZE implies BUG in dn_ifaddr_nlmsg_size() */ - WARN_ON(err == -EMSGSIZE); - kfree_skb(skb); - goto errout; - } - rtnl_notify(skb, &init_net, 0, RTNLGRP_DECnet_IFADDR, NULL, GFP_KERNEL); - return; -errout: - if (err < 0) - rtnl_set_sk_err(&init_net, RTNLGRP_DECnet_IFADDR, err); -} - -static int dn_nl_dump_ifaddr(struct sk_buff *skb, struct netlink_callback *cb) -{ - struct net *net = sock_net(skb->sk); - int idx, dn_idx = 0, skip_ndevs, skip_naddr; - struct net_device *dev; - struct dn_dev *dn_db; - struct dn_ifaddr *ifa; - - if (!net_eq(net, &init_net)) - return 0; - - skip_ndevs = cb->args[0]; - skip_naddr = cb->args[1]; - - idx = 0; - rcu_read_lock(); - for_each_netdev_rcu(&init_net, dev) { - if (idx < skip_ndevs) - goto cont; - else if (idx > skip_ndevs) { - /* Only skip over addresses for first dev dumped - * in this iteration (idx == skip_ndevs) */ - skip_naddr = 0; - } - - if ((dn_db = rcu_dereference(dev->dn_ptr)) == NULL) - goto cont; - - for (ifa = rcu_dereference(dn_db->ifa_list), dn_idx = 0; ifa; - ifa = rcu_dereference(ifa->ifa_next), dn_idx++) { - if (dn_idx < skip_naddr) - continue; - - if (dn_nl_fill_ifaddr(skb, ifa, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, RTM_NEWADDR, - NLM_F_MULTI) < 0) - goto done; - } -cont: - idx++; - } -done: - rcu_read_unlock(); - cb->args[0] = idx; - cb->args[1] = dn_idx; - - return skb->len; -} - -static int dn_dev_get_first(struct net_device *dev, __le16 *addr) -{ - struct dn_dev *dn_db; - struct dn_ifaddr *ifa; - int rv = -ENODEV; - - rcu_read_lock(); - dn_db = rcu_dereference(dev->dn_ptr); - if (dn_db == NULL) - goto out; - - ifa = rcu_dereference(dn_db->ifa_list); - if (ifa != NULL) { - *addr = ifa->ifa_local; - rv = 0; - } -out: - rcu_read_unlock(); - return rv; -} - -/* - * Find a default address to bind to. - * - * This is one of those areas where the initial VMS concepts don't really - * map onto the Linux concepts, and since we introduced multiple addresses - * per interface we have to cope with slightly odd ways of finding out what - * "our address" really is. Mostly it's not a problem; for this we just guess - * a sensible default. Eventually the routing code will take care of all the - * nasties for us I hope. - */ -int dn_dev_bind_default(__le16 *addr) -{ - struct net_device *dev; - int rv; - dev = dn_dev_get_default(); -last_chance: - if (dev) { - rv = dn_dev_get_first(dev, addr); - dev_put(dev); - if (rv == 0 || dev == init_net.loopback_dev) - return rv; - } - dev = init_net.loopback_dev; - dev_hold(dev); - goto last_chance; -} - -static void dn_send_endnode_hello(struct net_device *dev, struct dn_ifaddr *ifa) -{ - struct endnode_hello_message *msg; - struct sk_buff *skb = NULL; - __le16 *pktlen; - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - - if ((skb = dn_alloc_skb(NULL, sizeof(*msg), GFP_ATOMIC)) == NULL) - return; - - skb->dev = dev; - - msg = skb_put(skb, sizeof(*msg)); - - msg->msgflg = 0x0D; - memcpy(msg->tiver, dn_eco_version, 3); - dn_dn2eth(msg->id, ifa->ifa_local); - msg->iinfo = DN_RT_INFO_ENDN; - msg->blksize = cpu_to_le16(mtu2blksize(dev)); - msg->area = 0x00; - memset(msg->seed, 0, 8); - memcpy(msg->neighbor, dn_hiord, ETH_ALEN); - - if (dn_db->router) { - struct dn_neigh *dn = container_of(dn_db->router, struct dn_neigh, n); - dn_dn2eth(msg->neighbor, dn->addr); - } - - msg->timer = cpu_to_le16((unsigned short)dn_db->parms.t3); - msg->mpd = 0x00; - msg->datalen = 0x02; - memset(msg->data, 0xAA, 2); - - pktlen = skb_push(skb, 2); - *pktlen = cpu_to_le16(skb->len - 2); - - skb_reset_network_header(skb); - - dn_rt_finish_output(skb, dn_rt_all_rt_mcast, msg->id); -} - - -#define DRDELAY (5 * HZ) - -static int dn_am_i_a_router(struct dn_neigh *dn, struct dn_dev *dn_db, struct dn_ifaddr *ifa) -{ - /* First check time since device went up */ - if (time_before(jiffies, dn_db->uptime + DRDELAY)) - return 0; - - /* If there is no router, then yes... */ - if (!dn_db->router) - return 1; - - /* otherwise only if we have a higher priority or.. */ - if (dn->priority < dn_db->parms.priority) - return 1; - - /* if we have equal priority and a higher node number */ - if (dn->priority != dn_db->parms.priority) - return 0; - - if (le16_to_cpu(dn->addr) < le16_to_cpu(ifa->ifa_local)) - return 1; - - return 0; -} - -static void dn_send_router_hello(struct net_device *dev, struct dn_ifaddr *ifa) -{ - int n; - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - struct dn_neigh *dn = container_of(dn_db->router, struct dn_neigh, n); - struct sk_buff *skb; - size_t size; - unsigned char *ptr; - unsigned char *i1, *i2; - __le16 *pktlen; - char *src; - - if (mtu2blksize(dev) < (26 + 7)) - return; - - n = mtu2blksize(dev) - 26; - n /= 7; - - if (n > 32) - n = 32; - - size = 2 + 26 + 7 * n; - - if ((skb = dn_alloc_skb(NULL, size, GFP_ATOMIC)) == NULL) - return; - - skb->dev = dev; - ptr = skb_put(skb, size); - - *ptr++ = DN_RT_PKT_CNTL | DN_RT_PKT_ERTH; - *ptr++ = 2; /* ECO */ - *ptr++ = 0; - *ptr++ = 0; - dn_dn2eth(ptr, ifa->ifa_local); - src = ptr; - ptr += ETH_ALEN; - *ptr++ = dn_db->parms.forwarding == 1 ? - DN_RT_INFO_L1RT : DN_RT_INFO_L2RT; - *((__le16 *)ptr) = cpu_to_le16(mtu2blksize(dev)); - ptr += 2; - *ptr++ = dn_db->parms.priority; /* Priority */ - *ptr++ = 0; /* Area: Reserved */ - *((__le16 *)ptr) = cpu_to_le16((unsigned short)dn_db->parms.t3); - ptr += 2; - *ptr++ = 0; /* MPD: Reserved */ - i1 = ptr++; - memset(ptr, 0, 7); /* Name: Reserved */ - ptr += 7; - i2 = ptr++; - - n = dn_neigh_elist(dev, ptr, n); - - *i2 = 7 * n; - *i1 = 8 + *i2; - - skb_trim(skb, (27 + *i2)); - - pktlen = skb_push(skb, 2); - *pktlen = cpu_to_le16(skb->len - 2); - - skb_reset_network_header(skb); - - if (dn_am_i_a_router(dn, dn_db, ifa)) { - struct sk_buff *skb2 = skb_copy(skb, GFP_ATOMIC); - if (skb2) { - dn_rt_finish_output(skb2, dn_rt_all_end_mcast, src); - } - } - - dn_rt_finish_output(skb, dn_rt_all_rt_mcast, src); -} - -static void dn_send_brd_hello(struct net_device *dev, struct dn_ifaddr *ifa) -{ - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - - if (dn_db->parms.forwarding == 0) - dn_send_endnode_hello(dev, ifa); - else - dn_send_router_hello(dev, ifa); -} - -static void dn_send_ptp_hello(struct net_device *dev, struct dn_ifaddr *ifa) -{ - int tdlen = 16; - int size = dev->hard_header_len + 2 + 4 + tdlen; - struct sk_buff *skb = dn_alloc_skb(NULL, size, GFP_ATOMIC); - int i; - unsigned char *ptr; - char src[ETH_ALEN]; - - if (skb == NULL) - return ; - - skb->dev = dev; - skb_push(skb, dev->hard_header_len); - ptr = skb_put(skb, 2 + 4 + tdlen); - - *ptr++ = DN_RT_PKT_HELO; - *((__le16 *)ptr) = ifa->ifa_local; - ptr += 2; - *ptr++ = tdlen; - - for(i = 0; i < tdlen; i++) - *ptr++ = 0252; - - dn_dn2eth(src, ifa->ifa_local); - dn_rt_finish_output(skb, dn_rt_all_rt_mcast, src); -} - -static int dn_eth_up(struct net_device *dev) -{ - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - - if (dn_db->parms.forwarding == 0) - dev_mc_add(dev, dn_rt_all_end_mcast); - else - dev_mc_add(dev, dn_rt_all_rt_mcast); - - dn_db->use_long = 1; - - return 0; -} - -static void dn_eth_down(struct net_device *dev) -{ - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - - if (dn_db->parms.forwarding == 0) - dev_mc_del(dev, dn_rt_all_end_mcast); - else - dev_mc_del(dev, dn_rt_all_rt_mcast); -} - -static void dn_dev_set_timer(struct net_device *dev); - -static void dn_dev_timer_func(struct timer_list *t) -{ - struct dn_dev *dn_db = from_timer(dn_db, t, timer); - struct net_device *dev; - struct dn_ifaddr *ifa; - - rcu_read_lock(); - dev = dn_db->dev; - if (dn_db->t3 <= dn_db->parms.t2) { - if (dn_db->parms.timer3) { - for (ifa = rcu_dereference(dn_db->ifa_list); - ifa; - ifa = rcu_dereference(ifa->ifa_next)) { - if (!(ifa->ifa_flags & IFA_F_SECONDARY)) - dn_db->parms.timer3(dev, ifa); - } - } - dn_db->t3 = dn_db->parms.t3; - } else { - dn_db->t3 -= dn_db->parms.t2; - } - rcu_read_unlock(); - dn_dev_set_timer(dev); -} - -static void dn_dev_set_timer(struct net_device *dev) -{ - struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr); - - if (dn_db->parms.t2 > dn_db->parms.t3) - dn_db->parms.t2 = dn_db->parms.t3; - - dn_db->timer.expires = jiffies + (dn_db->parms.t2 * HZ); - - add_timer(&dn_db->timer); -} - -static struct dn_dev *dn_dev_create(struct net_device *dev, int *err) -{ - int i; - struct dn_dev_parms *p = dn_dev_list; - struct dn_dev *dn_db; - - for(i = 0; i < DN_DEV_LIST_SIZE; i++, p++) { - if (p->type == dev->type) - break; - } - - *err = -ENODEV; - if (i == DN_DEV_LIST_SIZE) - return NULL; - - *err = -ENOBUFS; - if ((dn_db = kzalloc(sizeof(struct dn_dev), GFP_ATOMIC)) == NULL) - return NULL; - - memcpy(&dn_db->parms, p, sizeof(struct dn_dev_parms)); - - rcu_assign_pointer(dev->dn_ptr, dn_db); - dn_db->dev = dev; - timer_setup(&dn_db->timer, dn_dev_timer_func, 0); - - dn_db->uptime = jiffies; - - dn_db->neigh_parms = neigh_parms_alloc(dev, &dn_neigh_table); - if (!dn_db->neigh_parms) { - RCU_INIT_POINTER(dev->dn_ptr, NULL); - kfree(dn_db); - return NULL; - } - - if (dn_db->parms.up) { - if (dn_db->parms.up(dev) < 0) { - neigh_parms_release(&dn_neigh_table, dn_db->neigh_parms); - dev->dn_ptr = NULL; - kfree(dn_db); - return NULL; - } - } - - dn_dev_sysctl_register(dev, &dn_db->parms); - - dn_dev_set_timer(dev); - - *err = 0; - return dn_db; -} - - -/* - * This processes a device up event. We only start up - * the loopback device & ethernet devices with correct - * MAC addresses automatically. Others must be started - * specifically. - * - * FIXME: How should we configure the loopback address ? If we could dispense - * with using decnet_address here and for autobind, it will be one less thing - * for users to worry about setting up. - */ - -void dn_dev_up(struct net_device *dev) -{ - struct dn_ifaddr *ifa; - __le16 addr = decnet_address; - int maybe_default = 0; - struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr); - - if ((dev->type != ARPHRD_ETHER) && (dev->type != ARPHRD_LOOPBACK)) - return; - - /* - * Need to ensure that loopback device has a dn_db attached to it - * to allow creation of neighbours against it, even though it might - * not have a local address of its own. Might as well do the same for - * all autoconfigured interfaces. - */ - if (dn_db == NULL) { - int err; - dn_db = dn_dev_create(dev, &err); - if (dn_db == NULL) - return; - } - - if (dev->type == ARPHRD_ETHER) { - if (memcmp(dev->dev_addr, dn_hiord, 4) != 0) - return; - addr = dn_eth2dn(dev->dev_addr); - maybe_default = 1; - } - - if (addr == 0) - return; - - if ((ifa = dn_dev_alloc_ifa()) == NULL) - return; - - ifa->ifa_local = ifa->ifa_address = addr; - ifa->ifa_flags = 0; - ifa->ifa_scope = RT_SCOPE_UNIVERSE; - strcpy(ifa->ifa_label, dev->name); - - dn_dev_set_ifa(dev, ifa); - - /* - * Automagically set the default device to the first automatically - * configured ethernet card in the system. - */ - if (maybe_default) { - dev_hold(dev); - if (dn_dev_set_default(dev, 0)) - dev_put(dev); - } -} - -static void dn_dev_delete(struct net_device *dev) -{ - struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr); - - if (dn_db == NULL) - return; - - del_timer_sync(&dn_db->timer); - dn_dev_sysctl_unregister(&dn_db->parms); - dn_dev_check_default(dev); - neigh_ifdown(&dn_neigh_table, dev); - - if (dn_db->parms.down) - dn_db->parms.down(dev); - - dev->dn_ptr = NULL; - - neigh_parms_release(&dn_neigh_table, dn_db->neigh_parms); - neigh_ifdown(&dn_neigh_table, dev); - - if (dn_db->router) - neigh_release(dn_db->router); - if (dn_db->peer) - neigh_release(dn_db->peer); - - kfree(dn_db); -} - -void dn_dev_down(struct net_device *dev) -{ - struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr); - struct dn_ifaddr *ifa; - - if (dn_db == NULL) - return; - - while ((ifa = rtnl_dereference(dn_db->ifa_list)) != NULL) { - dn_dev_del_ifa(dn_db, &dn_db->ifa_list, 0); - dn_dev_free_ifa(ifa); - } - - dn_dev_delete(dev); -} - -void dn_dev_init_pkt(struct sk_buff *skb) -{ -} - -void dn_dev_veri_pkt(struct sk_buff *skb) -{ -} - -void dn_dev_hello(struct sk_buff *skb) -{ -} - -void dn_dev_devices_off(void) -{ - struct net_device *dev; - - rtnl_lock(); - for_each_netdev(&init_net, dev) - dn_dev_down(dev); - rtnl_unlock(); - -} - -void dn_dev_devices_on(void) -{ - struct net_device *dev; - - rtnl_lock(); - for_each_netdev(&init_net, dev) { - if (dev->flags & IFF_UP) - dn_dev_up(dev); - } - rtnl_unlock(); -} - -int register_dnaddr_notifier(struct notifier_block *nb) -{ - return blocking_notifier_chain_register(&dnaddr_chain, nb); -} - -int unregister_dnaddr_notifier(struct notifier_block *nb) -{ - return blocking_notifier_chain_unregister(&dnaddr_chain, nb); -} - -#ifdef CONFIG_PROC_FS -static inline int is_dn_dev(struct net_device *dev) -{ - return dev->dn_ptr != NULL; -} - -static void *dn_dev_seq_start(struct seq_file *seq, loff_t *pos) - __acquires(RCU) -{ - int i; - struct net_device *dev; - - rcu_read_lock(); - - if (*pos == 0) - return SEQ_START_TOKEN; - - i = 1; - for_each_netdev_rcu(&init_net, dev) { - if (!is_dn_dev(dev)) - continue; - - if (i++ == *pos) - return dev; - } - - return NULL; -} - -static void *dn_dev_seq_next(struct seq_file *seq, void *v, loff_t *pos) -{ - struct net_device *dev; - - ++*pos; - - dev = v; - if (v == SEQ_START_TOKEN) - dev = net_device_entry(&init_net.dev_base_head); - - for_each_netdev_continue_rcu(&init_net, dev) { - if (!is_dn_dev(dev)) - continue; - - return dev; - } - - return NULL; -} - -static void dn_dev_seq_stop(struct seq_file *seq, void *v) - __releases(RCU) -{ - rcu_read_unlock(); -} - -static char *dn_type2asc(char type) -{ - switch (type) { - case DN_DEV_BCAST: - return "B"; - case DN_DEV_UCAST: - return "U"; - case DN_DEV_MPOINT: - return "M"; - } - - return "?"; -} - -static int dn_dev_seq_show(struct seq_file *seq, void *v) -{ - if (v == SEQ_START_TOKEN) - seq_puts(seq, "Name Flags T1 Timer1 T3 Timer3 BlkSize Pri State DevType Router Peer\n"); - else { - struct net_device *dev = v; - char peer_buf[DN_ASCBUF_LEN]; - char router_buf[DN_ASCBUF_LEN]; - struct dn_dev *dn_db = rcu_dereference(dev->dn_ptr); - - seq_printf(seq, "%-8s %1s %04u %04u %04lu %04lu" - " %04hu %03d %02x %-10s %-7s %-7s\n", - dev->name, - dn_type2asc(dn_db->parms.mode), - 0, 0, - dn_db->t3, dn_db->parms.t3, - mtu2blksize(dev), - dn_db->parms.priority, - dn_db->parms.state, dn_db->parms.name, - dn_db->router ? dn_addr2asc(le16_to_cpu(*(__le16 *)dn_db->router->primary_key), router_buf) : "", - dn_db->peer ? dn_addr2asc(le16_to_cpu(*(__le16 *)dn_db->peer->primary_key), peer_buf) : ""); - } - return 0; -} - -static const struct seq_operations dn_dev_seq_ops = { - .start = dn_dev_seq_start, - .next = dn_dev_seq_next, - .stop = dn_dev_seq_stop, - .show = dn_dev_seq_show, -}; -#endif /* CONFIG_PROC_FS */ - -static int addr[2]; -module_param_array(addr, int, NULL, 0444); -MODULE_PARM_DESC(addr, "The DECnet address of this machine: area,node"); - -void __init dn_dev_init(void) -{ - if (addr[0] > 63 || addr[0] < 0) { - printk(KERN_ERR "DECnet: Area must be between 0 and 63"); - return; - } - - if (addr[1] > 1023 || addr[1] < 0) { - printk(KERN_ERR "DECnet: Node must be between 0 and 1023"); - return; - } - - decnet_address = cpu_to_le16((addr[0] << 10) | addr[1]); - - dn_dev_devices_on(); - - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_NEWADDR, - dn_nl_newaddr, NULL, 0); - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_DELADDR, - dn_nl_deladdr, NULL, 0); - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETADDR, - NULL, dn_nl_dump_ifaddr, 0); - - proc_create_seq("decnet_dev", 0444, init_net.proc_net, &dn_dev_seq_ops); - -#ifdef CONFIG_SYSCTL - { - int i; - for(i = 0; i < DN_DEV_LIST_SIZE; i++) - dn_dev_sysctl_register(NULL, &dn_dev_list[i]); - } -#endif /* CONFIG_SYSCTL */ -} - -void __exit dn_dev_cleanup(void) -{ -#ifdef CONFIG_SYSCTL - { - int i; - for(i = 0; i < DN_DEV_LIST_SIZE; i++) - dn_dev_sysctl_unregister(&dn_dev_list[i]); - } -#endif /* CONFIG_SYSCTL */ - - remove_proc_entry("decnet_dev", init_net.proc_net); - - dn_dev_devices_off(); -} diff --git a/net/decnet/dn_fib.c b/net/decnet/dn_fib.c deleted file mode 100644 index 269c029ad74fce39af444df003b645605048045b..0000000000000000000000000000000000000000 --- a/net/decnet/dn_fib.c +++ /dev/null @@ -1,798 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Routing Forwarding Information Base (Glue/Info List) - * - * Author: Steve Whitehouse - * - * - * Changes: - * Alexey Kuznetsov : SMP locking changes - * Steve Whitehouse : Rewrote it... Well to be more correct, I - * copied most of it from the ipv4 fib code. - * Steve Whitehouse : Updated it in style and fixed a few bugs - * which were fixed in the ipv4 code since - * this code was copied from it. - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define RT_MIN_TABLE 1 - -#define for_fib_info() { struct dn_fib_info *fi;\ - for(fi = dn_fib_info_list; fi; fi = fi->fib_next) -#define endfor_fib_info() } - -#define for_nexthops(fi) { int nhsel; const struct dn_fib_nh *nh;\ - for(nhsel = 0, nh = (fi)->fib_nh; nhsel < (fi)->fib_nhs; nh++, nhsel++) - -#define change_nexthops(fi) { int nhsel; struct dn_fib_nh *nh;\ - for(nhsel = 0, nh = (struct dn_fib_nh *)((fi)->fib_nh); nhsel < (fi)->fib_nhs; nh++, nhsel++) - -#define endfor_nexthops(fi) } - -static DEFINE_SPINLOCK(dn_fib_multipath_lock); -static struct dn_fib_info *dn_fib_info_list; -static DEFINE_SPINLOCK(dn_fib_info_lock); - -static struct -{ - int error; - u8 scope; -} dn_fib_props[RTN_MAX+1] = { - [RTN_UNSPEC] = { .error = 0, .scope = RT_SCOPE_NOWHERE }, - [RTN_UNICAST] = { .error = 0, .scope = RT_SCOPE_UNIVERSE }, - [RTN_LOCAL] = { .error = 0, .scope = RT_SCOPE_HOST }, - [RTN_BROADCAST] = { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE }, - [RTN_ANYCAST] = { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE }, - [RTN_MULTICAST] = { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE }, - [RTN_BLACKHOLE] = { .error = -EINVAL, .scope = RT_SCOPE_UNIVERSE }, - [RTN_UNREACHABLE] = { .error = -EHOSTUNREACH, .scope = RT_SCOPE_UNIVERSE }, - [RTN_PROHIBIT] = { .error = -EACCES, .scope = RT_SCOPE_UNIVERSE }, - [RTN_THROW] = { .error = -EAGAIN, .scope = RT_SCOPE_UNIVERSE }, - [RTN_NAT] = { .error = 0, .scope = RT_SCOPE_NOWHERE }, - [RTN_XRESOLVE] = { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE }, -}; - -static int dn_fib_sync_down(__le16 local, struct net_device *dev, int force); -static int dn_fib_sync_up(struct net_device *dev); - -void dn_fib_free_info(struct dn_fib_info *fi) -{ - if (fi->fib_dead == 0) { - printk(KERN_DEBUG "DECnet: BUG! Attempt to free alive dn_fib_info\n"); - return; - } - - change_nexthops(fi) { - dev_put(nh->nh_dev); - nh->nh_dev = NULL; - } endfor_nexthops(fi); - kfree(fi); -} - -void dn_fib_release_info(struct dn_fib_info *fi) -{ - spin_lock(&dn_fib_info_lock); - if (fi && refcount_dec_and_test(&fi->fib_treeref)) { - if (fi->fib_next) - fi->fib_next->fib_prev = fi->fib_prev; - if (fi->fib_prev) - fi->fib_prev->fib_next = fi->fib_next; - if (fi == dn_fib_info_list) - dn_fib_info_list = fi->fib_next; - fi->fib_dead = 1; - dn_fib_info_put(fi); - } - spin_unlock(&dn_fib_info_lock); -} - -static inline int dn_fib_nh_comp(const struct dn_fib_info *fi, const struct dn_fib_info *ofi) -{ - const struct dn_fib_nh *onh = ofi->fib_nh; - - for_nexthops(fi) { - if (nh->nh_oif != onh->nh_oif || - nh->nh_gw != onh->nh_gw || - nh->nh_scope != onh->nh_scope || - nh->nh_weight != onh->nh_weight || - ((nh->nh_flags^onh->nh_flags)&~RTNH_F_DEAD)) - return -1; - onh++; - } endfor_nexthops(fi); - return 0; -} - -static inline struct dn_fib_info *dn_fib_find_info(const struct dn_fib_info *nfi) -{ - for_fib_info() { - if (fi->fib_nhs != nfi->fib_nhs) - continue; - if (nfi->fib_protocol == fi->fib_protocol && - nfi->fib_prefsrc == fi->fib_prefsrc && - nfi->fib_priority == fi->fib_priority && - memcmp(nfi->fib_metrics, fi->fib_metrics, sizeof(fi->fib_metrics)) == 0 && - ((nfi->fib_flags^fi->fib_flags)&~RTNH_F_DEAD) == 0 && - (nfi->fib_nhs == 0 || dn_fib_nh_comp(fi, nfi) == 0)) - return fi; - } endfor_fib_info(); - return NULL; -} - -static int dn_fib_count_nhs(const struct nlattr *attr) -{ - struct rtnexthop *nhp = nla_data(attr); - int nhs = 0, nhlen = nla_len(attr); - - while (rtnh_ok(nhp, nhlen)) { - nhs++; - nhp = rtnh_next(nhp, &nhlen); - } - - /* leftover implies invalid nexthop configuration, discard it */ - return nhlen > 0 ? 0 : nhs; -} - -static int dn_fib_get_nhs(struct dn_fib_info *fi, const struct nlattr *attr, - const struct rtmsg *r) -{ - struct rtnexthop *nhp = nla_data(attr); - int nhlen = nla_len(attr); - - change_nexthops(fi) { - int attrlen; - - if (!rtnh_ok(nhp, nhlen)) - return -EINVAL; - - nh->nh_flags = (r->rtm_flags&~0xFF) | nhp->rtnh_flags; - nh->nh_oif = nhp->rtnh_ifindex; - nh->nh_weight = nhp->rtnh_hops + 1; - - attrlen = rtnh_attrlen(nhp); - if (attrlen > 0) { - struct nlattr *gw_attr; - - gw_attr = nla_find((struct nlattr *) (nhp + 1), attrlen, RTA_GATEWAY); - nh->nh_gw = gw_attr ? nla_get_le16(gw_attr) : 0; - } - - nhp = rtnh_next(nhp, &nhlen); - } endfor_nexthops(fi); - - return 0; -} - - -static int dn_fib_check_nh(const struct rtmsg *r, struct dn_fib_info *fi, struct dn_fib_nh *nh) -{ - int err; - - if (nh->nh_gw) { - struct flowidn fld; - struct dn_fib_res res; - - if (nh->nh_flags&RTNH_F_ONLINK) { - struct net_device *dev; - - if (r->rtm_scope >= RT_SCOPE_LINK) - return -EINVAL; - if (dnet_addr_type(nh->nh_gw) != RTN_UNICAST) - return -EINVAL; - if ((dev = __dev_get_by_index(&init_net, nh->nh_oif)) == NULL) - return -ENODEV; - if (!(dev->flags&IFF_UP)) - return -ENETDOWN; - nh->nh_dev = dev; - dev_hold(dev); - nh->nh_scope = RT_SCOPE_LINK; - return 0; - } - - memset(&fld, 0, sizeof(fld)); - fld.daddr = nh->nh_gw; - fld.flowidn_oif = nh->nh_oif; - fld.flowidn_scope = r->rtm_scope + 1; - - if (fld.flowidn_scope < RT_SCOPE_LINK) - fld.flowidn_scope = RT_SCOPE_LINK; - - if ((err = dn_fib_lookup(&fld, &res)) != 0) - return err; - - err = -EINVAL; - if (res.type != RTN_UNICAST && res.type != RTN_LOCAL) - goto out; - nh->nh_scope = res.scope; - nh->nh_oif = DN_FIB_RES_OIF(res); - nh->nh_dev = DN_FIB_RES_DEV(res); - if (nh->nh_dev == NULL) - goto out; - dev_hold(nh->nh_dev); - err = -ENETDOWN; - if (!(nh->nh_dev->flags & IFF_UP)) - goto out; - err = 0; -out: - dn_fib_res_put(&res); - return err; - } else { - struct net_device *dev; - - if (nh->nh_flags&(RTNH_F_PERVASIVE|RTNH_F_ONLINK)) - return -EINVAL; - - dev = __dev_get_by_index(&init_net, nh->nh_oif); - if (dev == NULL || dev->dn_ptr == NULL) - return -ENODEV; - if (!(dev->flags&IFF_UP)) - return -ENETDOWN; - nh->nh_dev = dev; - dev_hold(nh->nh_dev); - nh->nh_scope = RT_SCOPE_HOST; - } - - return 0; -} - - -struct dn_fib_info *dn_fib_create_info(const struct rtmsg *r, struct nlattr *attrs[], - const struct nlmsghdr *nlh, int *errp) -{ - int err; - struct dn_fib_info *fi = NULL; - struct dn_fib_info *ofi; - int nhs = 1; - - if (r->rtm_type > RTN_MAX) - goto err_inval; - - if (dn_fib_props[r->rtm_type].scope > r->rtm_scope) - goto err_inval; - - if (attrs[RTA_MULTIPATH] && - (nhs = dn_fib_count_nhs(attrs[RTA_MULTIPATH])) == 0) - goto err_inval; - - fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL); - err = -ENOBUFS; - if (fi == NULL) - goto failure; - - fi->fib_protocol = r->rtm_protocol; - fi->fib_nhs = nhs; - fi->fib_flags = r->rtm_flags; - - if (attrs[RTA_PRIORITY]) - fi->fib_priority = nla_get_u32(attrs[RTA_PRIORITY]); - - if (attrs[RTA_METRICS]) { - struct nlattr *attr; - int rem; - - nla_for_each_nested(attr, attrs[RTA_METRICS], rem) { - int type = nla_type(attr); - - if (type) { - if (type > RTAX_MAX || type == RTAX_CC_ALGO || - nla_len(attr) < 4) - goto err_inval; - - fi->fib_metrics[type-1] = nla_get_u32(attr); - } - } - } - - if (attrs[RTA_PREFSRC]) - fi->fib_prefsrc = nla_get_le16(attrs[RTA_PREFSRC]); - - if (attrs[RTA_MULTIPATH]) { - if ((err = dn_fib_get_nhs(fi, attrs[RTA_MULTIPATH], r)) != 0) - goto failure; - - if (attrs[RTA_OIF] && - fi->fib_nh->nh_oif != nla_get_u32(attrs[RTA_OIF])) - goto err_inval; - - if (attrs[RTA_GATEWAY] && - fi->fib_nh->nh_gw != nla_get_le16(attrs[RTA_GATEWAY])) - goto err_inval; - } else { - struct dn_fib_nh *nh = fi->fib_nh; - - if (attrs[RTA_OIF]) - nh->nh_oif = nla_get_u32(attrs[RTA_OIF]); - - if (attrs[RTA_GATEWAY]) - nh->nh_gw = nla_get_le16(attrs[RTA_GATEWAY]); - - nh->nh_flags = r->rtm_flags; - nh->nh_weight = 1; - } - - if (r->rtm_type == RTN_NAT) { - if (!attrs[RTA_GATEWAY] || nhs != 1 || attrs[RTA_OIF]) - goto err_inval; - - fi->fib_nh->nh_gw = nla_get_le16(attrs[RTA_GATEWAY]); - goto link_it; - } - - if (dn_fib_props[r->rtm_type].error) { - if (attrs[RTA_GATEWAY] || attrs[RTA_OIF] || attrs[RTA_MULTIPATH]) - goto err_inval; - - goto link_it; - } - - if (r->rtm_scope > RT_SCOPE_HOST) - goto err_inval; - - if (r->rtm_scope == RT_SCOPE_HOST) { - struct dn_fib_nh *nh = fi->fib_nh; - - /* Local address is added */ - if (nhs != 1 || nh->nh_gw) - goto err_inval; - nh->nh_scope = RT_SCOPE_NOWHERE; - nh->nh_dev = dev_get_by_index(&init_net, fi->fib_nh->nh_oif); - err = -ENODEV; - if (nh->nh_dev == NULL) - goto failure; - } else { - change_nexthops(fi) { - if ((err = dn_fib_check_nh(r, fi, nh)) != 0) - goto failure; - } endfor_nexthops(fi) - } - - if (fi->fib_prefsrc) { - if (r->rtm_type != RTN_LOCAL || !attrs[RTA_DST] || - fi->fib_prefsrc != nla_get_le16(attrs[RTA_DST])) - if (dnet_addr_type(fi->fib_prefsrc) != RTN_LOCAL) - goto err_inval; - } - -link_it: - if ((ofi = dn_fib_find_info(fi)) != NULL) { - fi->fib_dead = 1; - dn_fib_free_info(fi); - refcount_inc(&ofi->fib_treeref); - return ofi; - } - - refcount_set(&fi->fib_treeref, 1); - refcount_set(&fi->fib_clntref, 1); - spin_lock(&dn_fib_info_lock); - fi->fib_next = dn_fib_info_list; - fi->fib_prev = NULL; - if (dn_fib_info_list) - dn_fib_info_list->fib_prev = fi; - dn_fib_info_list = fi; - spin_unlock(&dn_fib_info_lock); - return fi; - -err_inval: - err = -EINVAL; - -failure: - *errp = err; - if (fi) { - fi->fib_dead = 1; - dn_fib_free_info(fi); - } - - return NULL; -} - -int dn_fib_semantic_match(int type, struct dn_fib_info *fi, const struct flowidn *fld, struct dn_fib_res *res) -{ - int err = dn_fib_props[type].error; - - if (err == 0) { - if (fi->fib_flags & RTNH_F_DEAD) - return 1; - - res->fi = fi; - - switch (type) { - case RTN_NAT: - DN_FIB_RES_RESET(*res); - refcount_inc(&fi->fib_clntref); - return 0; - case RTN_UNICAST: - case RTN_LOCAL: - for_nexthops(fi) { - if (nh->nh_flags & RTNH_F_DEAD) - continue; - if (!fld->flowidn_oif || - fld->flowidn_oif == nh->nh_oif) - break; - } - if (nhsel < fi->fib_nhs) { - res->nh_sel = nhsel; - refcount_inc(&fi->fib_clntref); - return 0; - } - endfor_nexthops(fi); - res->fi = NULL; - return 1; - default: - net_err_ratelimited("DECnet: impossible routing event : dn_fib_semantic_match type=%d\n", - type); - res->fi = NULL; - return -EINVAL; - } - } - return err; -} - -void dn_fib_select_multipath(const struct flowidn *fld, struct dn_fib_res *res) -{ - struct dn_fib_info *fi = res->fi; - int w; - - spin_lock_bh(&dn_fib_multipath_lock); - if (fi->fib_power <= 0) { - int power = 0; - change_nexthops(fi) { - if (!(nh->nh_flags&RTNH_F_DEAD)) { - power += nh->nh_weight; - nh->nh_power = nh->nh_weight; - } - } endfor_nexthops(fi); - fi->fib_power = power; - if (power < 0) { - spin_unlock_bh(&dn_fib_multipath_lock); - res->nh_sel = 0; - return; - } - } - - w = jiffies % fi->fib_power; - - change_nexthops(fi) { - if (!(nh->nh_flags&RTNH_F_DEAD) && nh->nh_power) { - if ((w -= nh->nh_power) <= 0) { - nh->nh_power--; - fi->fib_power--; - res->nh_sel = nhsel; - spin_unlock_bh(&dn_fib_multipath_lock); - return; - } - } - } endfor_nexthops(fi); - res->nh_sel = 0; - spin_unlock_bh(&dn_fib_multipath_lock); -} - -static inline u32 rtm_get_table(struct nlattr *attrs[], u8 table) -{ - if (attrs[RTA_TABLE]) - table = nla_get_u32(attrs[RTA_TABLE]); - - return table; -} - -static int dn_fib_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(skb->sk); - struct dn_fib_table *tb; - struct rtmsg *r = nlmsg_data(nlh); - struct nlattr *attrs[RTA_MAX+1]; - int err; - - if (!netlink_capable(skb, CAP_NET_ADMIN)) - return -EPERM; - - if (!net_eq(net, &init_net)) - return -EINVAL; - - err = nlmsg_parse_deprecated(nlh, sizeof(*r), attrs, RTA_MAX, - rtm_dn_policy, extack); - if (err < 0) - return err; - - tb = dn_fib_get_table(rtm_get_table(attrs, r->rtm_table), 0); - if (!tb) - return -ESRCH; - - return tb->delete(tb, r, attrs, nlh, &NETLINK_CB(skb)); -} - -static int dn_fib_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(skb->sk); - struct dn_fib_table *tb; - struct rtmsg *r = nlmsg_data(nlh); - struct nlattr *attrs[RTA_MAX+1]; - int err; - - if (!netlink_capable(skb, CAP_NET_ADMIN)) - return -EPERM; - - if (!net_eq(net, &init_net)) - return -EINVAL; - - err = nlmsg_parse_deprecated(nlh, sizeof(*r), attrs, RTA_MAX, - rtm_dn_policy, extack); - if (err < 0) - return err; - - tb = dn_fib_get_table(rtm_get_table(attrs, r->rtm_table), 1); - if (!tb) - return -ENOBUFS; - - return tb->insert(tb, r, attrs, nlh, &NETLINK_CB(skb)); -} - -static void fib_magic(int cmd, int type, __le16 dst, int dst_len, struct dn_ifaddr *ifa) -{ - struct dn_fib_table *tb; - struct { - struct nlmsghdr nlh; - struct rtmsg rtm; - } req; - struct { - struct nlattr hdr; - __le16 dst; - } dst_attr = { - .dst = dst, - }; - struct { - struct nlattr hdr; - __le16 prefsrc; - } prefsrc_attr = { - .prefsrc = ifa->ifa_local, - }; - struct { - struct nlattr hdr; - u32 oif; - } oif_attr = { - .oif = ifa->ifa_dev->dev->ifindex, - }; - struct nlattr *attrs[RTA_MAX+1] = { - [RTA_DST] = (struct nlattr *) &dst_attr, - [RTA_PREFSRC] = (struct nlattr * ) &prefsrc_attr, - [RTA_OIF] = (struct nlattr *) &oif_attr, - }; - - memset(&req.rtm, 0, sizeof(req.rtm)); - - if (type == RTN_UNICAST) - tb = dn_fib_get_table(RT_MIN_TABLE, 1); - else - tb = dn_fib_get_table(RT_TABLE_LOCAL, 1); - - if (tb == NULL) - return; - - req.nlh.nlmsg_len = sizeof(req); - req.nlh.nlmsg_type = cmd; - req.nlh.nlmsg_flags = NLM_F_REQUEST|NLM_F_CREATE|NLM_F_APPEND; - req.nlh.nlmsg_pid = 0; - req.nlh.nlmsg_seq = 0; - - req.rtm.rtm_dst_len = dst_len; - req.rtm.rtm_table = tb->n; - req.rtm.rtm_protocol = RTPROT_KERNEL; - req.rtm.rtm_scope = (type != RTN_LOCAL ? RT_SCOPE_LINK : RT_SCOPE_HOST); - req.rtm.rtm_type = type; - - if (cmd == RTM_NEWROUTE) - tb->insert(tb, &req.rtm, attrs, &req.nlh, NULL); - else - tb->delete(tb, &req.rtm, attrs, &req.nlh, NULL); -} - -static void dn_fib_add_ifaddr(struct dn_ifaddr *ifa) -{ - - fib_magic(RTM_NEWROUTE, RTN_LOCAL, ifa->ifa_local, 16, ifa); - -#if 0 - if (!(dev->flags&IFF_UP)) - return; - /* In the future, we will want to add default routes here */ - -#endif -} - -static void dn_fib_del_ifaddr(struct dn_ifaddr *ifa) -{ - int found_it = 0; - struct net_device *dev; - struct dn_dev *dn_db; - struct dn_ifaddr *ifa2; - - ASSERT_RTNL(); - - /* Scan device list */ - rcu_read_lock(); - for_each_netdev_rcu(&init_net, dev) { - dn_db = rcu_dereference(dev->dn_ptr); - if (dn_db == NULL) - continue; - for (ifa2 = rcu_dereference(dn_db->ifa_list); - ifa2 != NULL; - ifa2 = rcu_dereference(ifa2->ifa_next)) { - if (ifa2->ifa_local == ifa->ifa_local) { - found_it = 1; - break; - } - } - } - rcu_read_unlock(); - - if (found_it == 0) { - fib_magic(RTM_DELROUTE, RTN_LOCAL, ifa->ifa_local, 16, ifa); - - if (dnet_addr_type(ifa->ifa_local) != RTN_LOCAL) { - if (dn_fib_sync_down(ifa->ifa_local, NULL, 0)) - dn_fib_flush(); - } - } -} - -static void dn_fib_disable_addr(struct net_device *dev, int force) -{ - if (dn_fib_sync_down(0, dev, force)) - dn_fib_flush(); - dn_rt_cache_flush(0); - neigh_ifdown(&dn_neigh_table, dev); -} - -static int dn_fib_dnaddr_event(struct notifier_block *this, unsigned long event, void *ptr) -{ - struct dn_ifaddr *ifa = (struct dn_ifaddr *)ptr; - - switch (event) { - case NETDEV_UP: - dn_fib_add_ifaddr(ifa); - dn_fib_sync_up(ifa->ifa_dev->dev); - dn_rt_cache_flush(-1); - break; - case NETDEV_DOWN: - dn_fib_del_ifaddr(ifa); - if (ifa->ifa_dev && ifa->ifa_dev->ifa_list == NULL) { - dn_fib_disable_addr(ifa->ifa_dev->dev, 1); - } else { - dn_rt_cache_flush(-1); - } - break; - } - return NOTIFY_DONE; -} - -static int dn_fib_sync_down(__le16 local, struct net_device *dev, int force) -{ - int ret = 0; - int scope = RT_SCOPE_NOWHERE; - - if (force) - scope = -1; - - for_fib_info() { - /* - * This makes no sense for DECnet.... we will almost - * certainly have more than one local address the same - * over all our interfaces. It needs thinking about - * some more. - */ - if (local && fi->fib_prefsrc == local) { - fi->fib_flags |= RTNH_F_DEAD; - ret++; - } else if (dev && fi->fib_nhs) { - int dead = 0; - - change_nexthops(fi) { - if (nh->nh_flags&RTNH_F_DEAD) - dead++; - else if (nh->nh_dev == dev && - nh->nh_scope != scope) { - spin_lock_bh(&dn_fib_multipath_lock); - nh->nh_flags |= RTNH_F_DEAD; - fi->fib_power -= nh->nh_power; - nh->nh_power = 0; - spin_unlock_bh(&dn_fib_multipath_lock); - dead++; - } - } endfor_nexthops(fi) - if (dead == fi->fib_nhs) { - fi->fib_flags |= RTNH_F_DEAD; - ret++; - } - } - } endfor_fib_info(); - return ret; -} - - -static int dn_fib_sync_up(struct net_device *dev) -{ - int ret = 0; - - if (!(dev->flags&IFF_UP)) - return 0; - - for_fib_info() { - int alive = 0; - - change_nexthops(fi) { - if (!(nh->nh_flags&RTNH_F_DEAD)) { - alive++; - continue; - } - if (nh->nh_dev == NULL || !(nh->nh_dev->flags&IFF_UP)) - continue; - if (nh->nh_dev != dev || dev->dn_ptr == NULL) - continue; - alive++; - spin_lock_bh(&dn_fib_multipath_lock); - nh->nh_power = 0; - nh->nh_flags &= ~RTNH_F_DEAD; - spin_unlock_bh(&dn_fib_multipath_lock); - } endfor_nexthops(fi); - - if (alive > 0) { - fi->fib_flags &= ~RTNH_F_DEAD; - ret++; - } - } endfor_fib_info(); - return ret; -} - -static struct notifier_block dn_fib_dnaddr_notifier = { - .notifier_call = dn_fib_dnaddr_event, -}; - -void __exit dn_fib_cleanup(void) -{ - dn_fib_table_cleanup(); - dn_fib_rules_cleanup(); - - unregister_dnaddr_notifier(&dn_fib_dnaddr_notifier); -} - - -void __init dn_fib_init(void) -{ - dn_fib_table_init(); - dn_fib_rules_init(); - - register_dnaddr_notifier(&dn_fib_dnaddr_notifier); - - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_NEWROUTE, - dn_fib_rtm_newroute, NULL, 0); - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_DELROUTE, - dn_fib_rtm_delroute, NULL, 0); -} diff --git a/net/decnet/dn_neigh.c b/net/decnet/dn_neigh.c deleted file mode 100644 index 7c569bcc0aca185266ff63cd8655d1d578808d72..0000000000000000000000000000000000000000 --- a/net/decnet/dn_neigh.c +++ /dev/null @@ -1,607 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Neighbour Functions (Adjacency Database and - * On-Ethernet Cache) - * - * Author: Steve Whitehouse - * - * - * Changes: - * Steve Whitehouse : Fixed router listing routine - * Steve Whitehouse : Added error_report functions - * Steve Whitehouse : Added default router detection - * Steve Whitehouse : Hop counts in outgoing messages - * Steve Whitehouse : Fixed src/dst in outgoing messages so - * forwarding now stands a good chance of - * working. - * Steve Whitehouse : Fixed neighbour states (for now anyway). - * Steve Whitehouse : Made error_report functions dummies. This - * is not the right place to return skbs. - * Steve Whitehouse : Convert to seq_file - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static int dn_neigh_construct(struct neighbour *); -static void dn_neigh_error_report(struct neighbour *, struct sk_buff *); -static int dn_neigh_output(struct neighbour *neigh, struct sk_buff *skb); - -/* - * Operations for adding the link layer header. - */ -static const struct neigh_ops dn_neigh_ops = { - .family = AF_DECnet, - .error_report = dn_neigh_error_report, - .output = dn_neigh_output, - .connected_output = dn_neigh_output, -}; - -static u32 dn_neigh_hash(const void *pkey, - const struct net_device *dev, - __u32 *hash_rnd) -{ - return jhash_2words(*(__u16 *)pkey, 0, hash_rnd[0]); -} - -static bool dn_key_eq(const struct neighbour *neigh, const void *pkey) -{ - return neigh_key_eq16(neigh, pkey); -} - -struct neigh_table dn_neigh_table = { - .family = PF_DECnet, - .entry_size = NEIGH_ENTRY_SIZE(sizeof(struct dn_neigh)), - .key_len = sizeof(__le16), - .protocol = cpu_to_be16(ETH_P_DNA_RT), - .hash = dn_neigh_hash, - .key_eq = dn_key_eq, - .constructor = dn_neigh_construct, - .id = "dn_neigh_cache", - .parms ={ - .tbl = &dn_neigh_table, - .reachable_time = 30 * HZ, - .data = { - [NEIGH_VAR_MCAST_PROBES] = 0, - [NEIGH_VAR_UCAST_PROBES] = 0, - [NEIGH_VAR_APP_PROBES] = 0, - [NEIGH_VAR_RETRANS_TIME] = 1 * HZ, - [NEIGH_VAR_BASE_REACHABLE_TIME] = 30 * HZ, - [NEIGH_VAR_DELAY_PROBE_TIME] = 5 * HZ, - [NEIGH_VAR_INTERVAL_PROBE_TIME_MS] = 5 * HZ, - [NEIGH_VAR_GC_STALETIME] = 60 * HZ, - [NEIGH_VAR_QUEUE_LEN_BYTES] = SK_WMEM_MAX, - [NEIGH_VAR_PROXY_QLEN] = 0, - [NEIGH_VAR_ANYCAST_DELAY] = 0, - [NEIGH_VAR_PROXY_DELAY] = 0, - [NEIGH_VAR_LOCKTIME] = 1 * HZ, - }, - }, - .gc_interval = 30 * HZ, - .gc_thresh1 = 128, - .gc_thresh2 = 512, - .gc_thresh3 = 1024, -}; - -static int dn_neigh_construct(struct neighbour *neigh) -{ - struct net_device *dev = neigh->dev; - struct dn_neigh *dn = container_of(neigh, struct dn_neigh, n); - struct dn_dev *dn_db; - struct neigh_parms *parms; - - rcu_read_lock(); - dn_db = rcu_dereference(dev->dn_ptr); - if (dn_db == NULL) { - rcu_read_unlock(); - return -EINVAL; - } - - parms = dn_db->neigh_parms; - if (!parms) { - rcu_read_unlock(); - return -EINVAL; - } - - __neigh_parms_put(neigh->parms); - neigh->parms = neigh_parms_clone(parms); - rcu_read_unlock(); - - neigh->ops = &dn_neigh_ops; - neigh->nud_state = NUD_NOARP; - neigh->output = neigh->ops->connected_output; - - if ((dev->type == ARPHRD_IPGRE) || (dev->flags & IFF_POINTOPOINT)) - memcpy(neigh->ha, dev->broadcast, dev->addr_len); - else if ((dev->type == ARPHRD_ETHER) || (dev->type == ARPHRD_LOOPBACK)) - dn_dn2eth(neigh->ha, dn->addr); - else { - net_dbg_ratelimited("Trying to create neigh for hw %d\n", - dev->type); - return -EINVAL; - } - - /* - * Make an estimate of the remote block size by assuming that its - * two less then the device mtu, which it true for ethernet (and - * other things which support long format headers) since there is - * an extra length field (of 16 bits) which isn't part of the - * ethernet headers and which the DECnet specs won't admit is part - * of the DECnet routing headers either. - * - * If we over estimate here its no big deal, the NSP negotiations - * will prevent us from sending packets which are too large for the - * remote node to handle. In any case this figure is normally updated - * by a hello message in most cases. - */ - dn->blksize = dev->mtu - 2; - - return 0; -} - -static void dn_neigh_error_report(struct neighbour *neigh, struct sk_buff *skb) -{ - printk(KERN_DEBUG "dn_neigh_error_report: called\n"); - kfree_skb(skb); -} - -static int dn_neigh_output(struct neighbour *neigh, struct sk_buff *skb) -{ - struct dst_entry *dst = skb_dst(skb); - struct dn_route *rt = (struct dn_route *)dst; - struct net_device *dev = neigh->dev; - char mac_addr[ETH_ALEN]; - unsigned int seq; - int err; - - dn_dn2eth(mac_addr, rt->rt_local_src); - do { - seq = read_seqbegin(&neigh->ha_lock); - err = dev_hard_header(skb, dev, ntohs(skb->protocol), - neigh->ha, mac_addr, skb->len); - } while (read_seqretry(&neigh->ha_lock, seq)); - - if (err >= 0) - err = dev_queue_xmit(skb); - else { - kfree_skb(skb); - err = -EINVAL; - } - return err; -} - -static int dn_neigh_output_packet(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct dst_entry *dst = skb_dst(skb); - struct dn_route *rt = (struct dn_route *)dst; - struct neighbour *neigh = rt->n; - - return neigh->output(neigh, skb); -} - -/* - * For talking to broadcast devices: Ethernet & PPP - */ -static int dn_long_output(struct neighbour *neigh, struct sock *sk, - struct sk_buff *skb) -{ - struct net_device *dev = neigh->dev; - int headroom = dev->hard_header_len + sizeof(struct dn_long_packet) + 3; - unsigned char *data; - struct dn_long_packet *lp; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - - if (skb_headroom(skb) < headroom) { - struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom); - if (skb2 == NULL) { - net_crit_ratelimited("dn_long_output: no memory\n"); - kfree_skb(skb); - return -ENOBUFS; - } - consume_skb(skb); - skb = skb2; - net_info_ratelimited("dn_long_output: Increasing headroom\n"); - } - - data = skb_push(skb, sizeof(struct dn_long_packet) + 3); - lp = (struct dn_long_packet *)(data+3); - - *((__le16 *)data) = cpu_to_le16(skb->len - 2); - *(data + 2) = 1 | DN_RT_F_PF; /* Padding */ - - lp->msgflg = DN_RT_PKT_LONG|(cb->rt_flags&(DN_RT_F_IE|DN_RT_F_RQR|DN_RT_F_RTS)); - lp->d_area = lp->d_subarea = 0; - dn_dn2eth(lp->d_id, cb->dst); - lp->s_area = lp->s_subarea = 0; - dn_dn2eth(lp->s_id, cb->src); - lp->nl2 = 0; - lp->visit_ct = cb->hops & 0x3f; - lp->s_class = 0; - lp->pt = 0; - - skb_reset_network_header(skb); - - return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING, - &init_net, sk, skb, NULL, neigh->dev, - dn_neigh_output_packet); -} - -/* - * For talking to pointopoint and multidrop devices: DDCMP and X.25 - */ -static int dn_short_output(struct neighbour *neigh, struct sock *sk, - struct sk_buff *skb) -{ - struct net_device *dev = neigh->dev; - int headroom = dev->hard_header_len + sizeof(struct dn_short_packet) + 2; - struct dn_short_packet *sp; - unsigned char *data; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - - if (skb_headroom(skb) < headroom) { - struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom); - if (skb2 == NULL) { - net_crit_ratelimited("dn_short_output: no memory\n"); - kfree_skb(skb); - return -ENOBUFS; - } - consume_skb(skb); - skb = skb2; - net_info_ratelimited("dn_short_output: Increasing headroom\n"); - } - - data = skb_push(skb, sizeof(struct dn_short_packet) + 2); - *((__le16 *)data) = cpu_to_le16(skb->len - 2); - sp = (struct dn_short_packet *)(data+2); - - sp->msgflg = DN_RT_PKT_SHORT|(cb->rt_flags&(DN_RT_F_RQR|DN_RT_F_RTS)); - sp->dstnode = cb->dst; - sp->srcnode = cb->src; - sp->forward = cb->hops & 0x3f; - - skb_reset_network_header(skb); - - return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING, - &init_net, sk, skb, NULL, neigh->dev, - dn_neigh_output_packet); -} - -/* - * For talking to DECnet phase III nodes - * Phase 3 output is the same as short output, execpt that - * it clears the area bits before transmission. - */ -static int dn_phase3_output(struct neighbour *neigh, struct sock *sk, - struct sk_buff *skb) -{ - struct net_device *dev = neigh->dev; - int headroom = dev->hard_header_len + sizeof(struct dn_short_packet) + 2; - struct dn_short_packet *sp; - unsigned char *data; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - if (skb_headroom(skb) < headroom) { - struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom); - if (skb2 == NULL) { - net_crit_ratelimited("dn_phase3_output: no memory\n"); - kfree_skb(skb); - return -ENOBUFS; - } - consume_skb(skb); - skb = skb2; - net_info_ratelimited("dn_phase3_output: Increasing headroom\n"); - } - - data = skb_push(skb, sizeof(struct dn_short_packet) + 2); - *((__le16 *)data) = cpu_to_le16(skb->len - 2); - sp = (struct dn_short_packet *)(data + 2); - - sp->msgflg = DN_RT_PKT_SHORT|(cb->rt_flags&(DN_RT_F_RQR|DN_RT_F_RTS)); - sp->dstnode = cb->dst & cpu_to_le16(0x03ff); - sp->srcnode = cb->src & cpu_to_le16(0x03ff); - sp->forward = cb->hops & 0x3f; - - skb_reset_network_header(skb); - - return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING, - &init_net, sk, skb, NULL, neigh->dev, - dn_neigh_output_packet); -} - -int dn_to_neigh_output(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct dst_entry *dst = skb_dst(skb); - struct dn_route *rt = (struct dn_route *) dst; - struct neighbour *neigh = rt->n; - struct dn_neigh *dn = container_of(neigh, struct dn_neigh, n); - struct dn_dev *dn_db; - bool use_long; - - rcu_read_lock(); - dn_db = rcu_dereference(neigh->dev->dn_ptr); - if (dn_db == NULL) { - rcu_read_unlock(); - return -EINVAL; - } - use_long = dn_db->use_long; - rcu_read_unlock(); - - if (dn->flags & DN_NDFLAG_P3) - return dn_phase3_output(neigh, sk, skb); - if (use_long) - return dn_long_output(neigh, sk, skb); - else - return dn_short_output(neigh, sk, skb); -} - -/* - * Unfortunately, the neighbour code uses the device in its hash - * function, so we don't get any advantage from it. This function - * basically does a neigh_lookup(), but without comparing the device - * field. This is required for the On-Ethernet cache - */ - -/* - * Pointopoint link receives a hello message - */ -void dn_neigh_pointopoint_hello(struct sk_buff *skb) -{ - kfree_skb(skb); -} - -/* - * Ethernet router hello message received - */ -int dn_neigh_router_hello(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct rtnode_hello_message *msg = (struct rtnode_hello_message *)skb->data; - - struct neighbour *neigh; - struct dn_neigh *dn; - struct dn_dev *dn_db; - __le16 src; - - src = dn_eth2dn(msg->id); - - neigh = __neigh_lookup(&dn_neigh_table, &src, skb->dev, 1); - - dn = container_of(neigh, struct dn_neigh, n); - - if (neigh) { - write_lock(&neigh->lock); - - neigh->used = jiffies; - dn_db = rcu_dereference(neigh->dev->dn_ptr); - - if (!(neigh->nud_state & NUD_PERMANENT)) { - neigh->updated = jiffies; - - if (neigh->dev->type == ARPHRD_ETHER) - memcpy(neigh->ha, ð_hdr(skb)->h_source, ETH_ALEN); - - dn->blksize = le16_to_cpu(msg->blksize); - dn->priority = msg->priority; - - dn->flags &= ~DN_NDFLAG_P3; - - switch (msg->iinfo & DN_RT_INFO_TYPE) { - case DN_RT_INFO_L1RT: - dn->flags &=~DN_NDFLAG_R2; - dn->flags |= DN_NDFLAG_R1; - break; - case DN_RT_INFO_L2RT: - dn->flags |= DN_NDFLAG_R2; - } - } - - /* Only use routers in our area */ - if ((le16_to_cpu(src)>>10) == (le16_to_cpu((decnet_address))>>10)) { - if (!dn_db->router) { - dn_db->router = neigh_clone(neigh); - } else { - if (msg->priority > container_of(dn_db->router, - struct dn_neigh, n)->priority) - neigh_release(xchg(&dn_db->router, neigh_clone(neigh))); - } - } - write_unlock(&neigh->lock); - neigh_release(neigh); - } - - kfree_skb(skb); - return 0; -} - -/* - * Endnode hello message received - */ -int dn_neigh_endnode_hello(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct endnode_hello_message *msg = (struct endnode_hello_message *)skb->data; - struct neighbour *neigh; - struct dn_neigh *dn; - __le16 src; - - src = dn_eth2dn(msg->id); - - neigh = __neigh_lookup(&dn_neigh_table, &src, skb->dev, 1); - - dn = container_of(neigh, struct dn_neigh, n); - - if (neigh) { - write_lock(&neigh->lock); - - neigh->used = jiffies; - - if (!(neigh->nud_state & NUD_PERMANENT)) { - neigh->updated = jiffies; - - if (neigh->dev->type == ARPHRD_ETHER) - memcpy(neigh->ha, ð_hdr(skb)->h_source, ETH_ALEN); - dn->flags &= ~(DN_NDFLAG_R1 | DN_NDFLAG_R2); - dn->blksize = le16_to_cpu(msg->blksize); - dn->priority = 0; - } - - write_unlock(&neigh->lock); - neigh_release(neigh); - } - - kfree_skb(skb); - return 0; -} - -static char *dn_find_slot(char *base, int max, int priority) -{ - int i; - unsigned char *min = NULL; - - base += 6; /* skip first id */ - - for(i = 0; i < max; i++) { - if (!min || (*base < *min)) - min = base; - base += 7; /* find next priority */ - } - - if (!min) - return NULL; - - return (*min < priority) ? (min - 6) : NULL; -} - -struct elist_cb_state { - struct net_device *dev; - unsigned char *ptr; - unsigned char *rs; - int t, n; -}; - -static void neigh_elist_cb(struct neighbour *neigh, void *_info) -{ - struct elist_cb_state *s = _info; - struct dn_neigh *dn; - - if (neigh->dev != s->dev) - return; - - dn = container_of(neigh, struct dn_neigh, n); - if (!(dn->flags & (DN_NDFLAG_R1|DN_NDFLAG_R2))) - return; - - if (s->t == s->n) - s->rs = dn_find_slot(s->ptr, s->n, dn->priority); - else - s->t++; - if (s->rs == NULL) - return; - - dn_dn2eth(s->rs, dn->addr); - s->rs += 6; - *(s->rs) = neigh->nud_state & NUD_CONNECTED ? 0x80 : 0x0; - *(s->rs) |= dn->priority; - s->rs++; -} - -int dn_neigh_elist(struct net_device *dev, unsigned char *ptr, int n) -{ - struct elist_cb_state state; - - state.dev = dev; - state.t = 0; - state.n = n; - state.ptr = ptr; - state.rs = ptr; - - neigh_for_each(&dn_neigh_table, neigh_elist_cb, &state); - - return state.t; -} - - -#ifdef CONFIG_PROC_FS - -static inline void dn_neigh_format_entry(struct seq_file *seq, - struct neighbour *n) -{ - struct dn_neigh *dn = container_of(n, struct dn_neigh, n); - char buf[DN_ASCBUF_LEN]; - - read_lock(&n->lock); - seq_printf(seq, "%-7s %s%s%s %02x %02d %07ld %-8s\n", - dn_addr2asc(le16_to_cpu(dn->addr), buf), - (dn->flags&DN_NDFLAG_R1) ? "1" : "-", - (dn->flags&DN_NDFLAG_R2) ? "2" : "-", - (dn->flags&DN_NDFLAG_P3) ? "3" : "-", - dn->n.nud_state, - refcount_read(&dn->n.refcnt), - dn->blksize, - (dn->n.dev) ? dn->n.dev->name : "?"); - read_unlock(&n->lock); -} - -static int dn_neigh_seq_show(struct seq_file *seq, void *v) -{ - if (v == SEQ_START_TOKEN) { - seq_puts(seq, "Addr Flags State Use Blksize Dev\n"); - } else { - dn_neigh_format_entry(seq, v); - } - - return 0; -} - -static void *dn_neigh_seq_start(struct seq_file *seq, loff_t *pos) -{ - return neigh_seq_start(seq, pos, &dn_neigh_table, - NEIGH_SEQ_NEIGH_ONLY); -} - -static const struct seq_operations dn_neigh_seq_ops = { - .start = dn_neigh_seq_start, - .next = neigh_seq_next, - .stop = neigh_seq_stop, - .show = dn_neigh_seq_show, -}; -#endif - -void __init dn_neigh_init(void) -{ - neigh_table_init(NEIGH_DN_TABLE, &dn_neigh_table); - proc_create_net("decnet_neigh", 0444, init_net.proc_net, - &dn_neigh_seq_ops, sizeof(struct neigh_seq_state)); -} - -void __exit dn_neigh_cleanup(void) -{ - remove_proc_entry("decnet_neigh", init_net.proc_net); - neigh_table_clear(NEIGH_DN_TABLE, &dn_neigh_table); -} diff --git a/net/decnet/dn_nsp_in.c b/net/decnet/dn_nsp_in.c deleted file mode 100644 index c59be5b04479b4dccef992adbbbe71d99c961ffe..0000000000000000000000000000000000000000 --- a/net/decnet/dn_nsp_in.c +++ /dev/null @@ -1,907 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Network Services Protocol (Input) - * - * Author: Eduardo Marcelo Serrat - * - * Changes: - * - * Steve Whitehouse: Split into dn_nsp_in.c and dn_nsp_out.c from - * original dn_nsp.c. - * Steve Whitehouse: Updated to work with my new routing architecture. - * Steve Whitehouse: Add changes from Eduardo Serrat's patches. - * Steve Whitehouse: Put all ack handling code in a common routine. - * Steve Whitehouse: Put other common bits into dn_nsp_rx() - * Steve Whitehouse: More checks on skb->len to catch bogus packets - * Fixed various race conditions and possible nasties. - * Steve Whitehouse: Now handles returned conninit frames. - * David S. Miller: New socket locking - * Steve Whitehouse: Fixed lockup when socket filtering was enabled. - * Paul Koning: Fix to push CC sockets into RUN when acks are - * received. - * Steve Whitehouse: - * Patrick Caulfield: Checking conninits for correctness & sending of error - * responses. - * Steve Whitehouse: Added backlog congestion level return codes. - * Patrick Caulfield: - * Steve Whitehouse: Added flow control support (outbound) - * Steve Whitehouse: Prepare for nonlinear skbs - */ - -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern int decnet_log_martians; - -static void dn_log_martian(struct sk_buff *skb, const char *msg) -{ - if (decnet_log_martians) { - char *devname = skb->dev ? skb->dev->name : "???"; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - net_info_ratelimited("DECnet: Martian packet (%s) dev=%s src=0x%04hx dst=0x%04hx srcport=0x%04hx dstport=0x%04hx\n", - msg, devname, - le16_to_cpu(cb->src), - le16_to_cpu(cb->dst), - le16_to_cpu(cb->src_port), - le16_to_cpu(cb->dst_port)); - } -} - -/* - * For this function we've flipped the cross-subchannel bit - * if the message is an otherdata or linkservice message. Thus - * we can use it to work out what to update. - */ -static void dn_ack(struct sock *sk, struct sk_buff *skb, unsigned short ack) -{ - struct dn_scp *scp = DN_SK(sk); - unsigned short type = ((ack >> 12) & 0x0003); - int wakeup = 0; - - switch (type) { - case 0: /* ACK - Data */ - if (dn_after(ack, scp->ackrcv_dat)) { - scp->ackrcv_dat = ack & 0x0fff; - wakeup |= dn_nsp_check_xmit_queue(sk, skb, - &scp->data_xmit_queue, - ack); - } - break; - case 1: /* NAK - Data */ - break; - case 2: /* ACK - OtherData */ - if (dn_after(ack, scp->ackrcv_oth)) { - scp->ackrcv_oth = ack & 0x0fff; - wakeup |= dn_nsp_check_xmit_queue(sk, skb, - &scp->other_xmit_queue, - ack); - } - break; - case 3: /* NAK - OtherData */ - break; - } - - if (wakeup && !sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); -} - -/* - * This function is a universal ack processor. - */ -static int dn_process_ack(struct sock *sk, struct sk_buff *skb, int oth) -{ - __le16 *ptr = (__le16 *)skb->data; - int len = 0; - unsigned short ack; - - if (skb->len < 2) - return len; - - if ((ack = le16_to_cpu(*ptr)) & 0x8000) { - skb_pull(skb, 2); - ptr++; - len += 2; - if ((ack & 0x4000) == 0) { - if (oth) - ack ^= 0x2000; - dn_ack(sk, skb, ack); - } - } - - if (skb->len < 2) - return len; - - if ((ack = le16_to_cpu(*ptr)) & 0x8000) { - skb_pull(skb, 2); - len += 2; - if ((ack & 0x4000) == 0) { - if (oth) - ack ^= 0x2000; - dn_ack(sk, skb, ack); - } - } - - return len; -} - - -/** - * dn_check_idf - Check an image data field format is correct. - * @pptr: Pointer to pointer to image data - * @len: Pointer to length of image data - * @max: The maximum allowed length of the data in the image data field - * @follow_on: Check that this many bytes exist beyond the end of the image data - * - * Returns: 0 if ok, -1 on error - */ -static inline int dn_check_idf(unsigned char **pptr, int *len, unsigned char max, unsigned char follow_on) -{ - unsigned char *ptr = *pptr; - unsigned char flen = *ptr++; - - (*len)--; - if (flen > max) - return -1; - if ((flen + follow_on) > *len) - return -1; - - *len -= flen; - *pptr = ptr + flen; - return 0; -} - -/* - * Table of reason codes to pass back to node which sent us a badly - * formed message, plus text messages for the log. A zero entry in - * the reason field means "don't reply" otherwise a disc init is sent with - * the specified reason code. - */ -static struct { - unsigned short reason; - const char *text; -} ci_err_table[] = { - { 0, "CI: Truncated message" }, - { NSP_REASON_ID, "CI: Destination username error" }, - { NSP_REASON_ID, "CI: Destination username type" }, - { NSP_REASON_US, "CI: Source username error" }, - { 0, "CI: Truncated at menuver" }, - { 0, "CI: Truncated before access or user data" }, - { NSP_REASON_IO, "CI: Access data format error" }, - { NSP_REASON_IO, "CI: User data format error" } -}; - -/* - * This function uses a slightly different lookup method - * to find its sockets, since it searches on object name/number - * rather than port numbers. Various tests are done to ensure that - * the incoming data is in the correct format before it is queued to - * a socket. - */ -static struct sock *dn_find_listener(struct sk_buff *skb, unsigned short *reason) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct nsp_conn_init_msg *msg = (struct nsp_conn_init_msg *)skb->data; - struct sockaddr_dn dstaddr; - struct sockaddr_dn srcaddr; - unsigned char type = 0; - int dstlen; - int srclen; - unsigned char *ptr; - int len; - int err = 0; - unsigned char menuver; - - memset(&dstaddr, 0, sizeof(struct sockaddr_dn)); - memset(&srcaddr, 0, sizeof(struct sockaddr_dn)); - - /* - * 1. Decode & remove message header - */ - cb->src_port = msg->srcaddr; - cb->dst_port = msg->dstaddr; - cb->services = msg->services; - cb->info = msg->info; - cb->segsize = le16_to_cpu(msg->segsize); - - if (!pskb_may_pull(skb, sizeof(*msg))) - goto err_out; - - skb_pull(skb, sizeof(*msg)); - - len = skb->len; - ptr = skb->data; - - /* - * 2. Check destination end username format - */ - dstlen = dn_username2sockaddr(ptr, len, &dstaddr, &type); - err++; - if (dstlen < 0) - goto err_out; - - err++; - if (type > 1) - goto err_out; - - len -= dstlen; - ptr += dstlen; - - /* - * 3. Check source end username format - */ - srclen = dn_username2sockaddr(ptr, len, &srcaddr, &type); - err++; - if (srclen < 0) - goto err_out; - - len -= srclen; - ptr += srclen; - err++; - if (len < 1) - goto err_out; - - menuver = *ptr; - ptr++; - len--; - - /* - * 4. Check that optional data actually exists if menuver says it does - */ - err++; - if ((menuver & (DN_MENUVER_ACC | DN_MENUVER_USR)) && (len < 1)) - goto err_out; - - /* - * 5. Check optional access data format - */ - err++; - if (menuver & DN_MENUVER_ACC) { - if (dn_check_idf(&ptr, &len, 39, 1)) - goto err_out; - if (dn_check_idf(&ptr, &len, 39, 1)) - goto err_out; - if (dn_check_idf(&ptr, &len, 39, (menuver & DN_MENUVER_USR) ? 1 : 0)) - goto err_out; - } - - /* - * 6. Check optional user data format - */ - err++; - if (menuver & DN_MENUVER_USR) { - if (dn_check_idf(&ptr, &len, 16, 0)) - goto err_out; - } - - /* - * 7. Look up socket based on destination end username - */ - return dn_sklist_find_listener(&dstaddr); -err_out: - dn_log_martian(skb, ci_err_table[err].text); - *reason = ci_err_table[err].reason; - return NULL; -} - - -static void dn_nsp_conn_init(struct sock *sk, struct sk_buff *skb) -{ - if (sk_acceptq_is_full(sk)) { - kfree_skb(skb); - return; - } - - sk_acceptq_added(sk); - skb_queue_tail(&sk->sk_receive_queue, skb); - sk->sk_state_change(sk); -} - -static void dn_nsp_conn_conf(struct sock *sk, struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct dn_scp *scp = DN_SK(sk); - unsigned char *ptr; - - if (skb->len < 4) - goto out; - - ptr = skb->data; - cb->services = *ptr++; - cb->info = *ptr++; - cb->segsize = le16_to_cpu(*(__le16 *)ptr); - - if ((scp->state == DN_CI) || (scp->state == DN_CD)) { - scp->persist = 0; - scp->addrrem = cb->src_port; - sk->sk_state = TCP_ESTABLISHED; - scp->state = DN_RUN; - scp->services_rem = cb->services; - scp->info_rem = cb->info; - scp->segsize_rem = cb->segsize; - - if ((scp->services_rem & NSP_FC_MASK) == NSP_FC_NONE) - scp->max_window = decnet_no_fc_max_cwnd; - - if (skb->len > 0) { - u16 dlen = *skb->data; - if ((dlen <= 16) && (dlen <= skb->len)) { - scp->conndata_in.opt_optl = cpu_to_le16(dlen); - skb_copy_from_linear_data_offset(skb, 1, - scp->conndata_in.opt_data, dlen); - } - } - dn_nsp_send_link(sk, DN_NOCHANGE, 0); - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); - } - -out: - kfree_skb(skb); -} - -static void dn_nsp_conn_ack(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - - if (scp->state == DN_CI) { - scp->state = DN_CD; - scp->persist = 0; - } - - kfree_skb(skb); -} - -static void dn_nsp_disc_init(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - struct dn_skb_cb *cb = DN_SKB_CB(skb); - unsigned short reason; - - if (skb->len < 2) - goto out; - - reason = le16_to_cpu(*(__le16 *)skb->data); - skb_pull(skb, 2); - - scp->discdata_in.opt_status = cpu_to_le16(reason); - scp->discdata_in.opt_optl = 0; - memset(scp->discdata_in.opt_data, 0, 16); - - if (skb->len > 0) { - u16 dlen = *skb->data; - if ((dlen <= 16) && (dlen <= skb->len)) { - scp->discdata_in.opt_optl = cpu_to_le16(dlen); - skb_copy_from_linear_data_offset(skb, 1, scp->discdata_in.opt_data, dlen); - } - } - - scp->addrrem = cb->src_port; - sk->sk_state = TCP_CLOSE; - - switch (scp->state) { - case DN_CI: - case DN_CD: - scp->state = DN_RJ; - sk->sk_err = ECONNREFUSED; - break; - case DN_RUN: - sk->sk_shutdown |= SHUTDOWN_MASK; - scp->state = DN_DN; - break; - case DN_DI: - scp->state = DN_DIC; - break; - } - - if (!sock_flag(sk, SOCK_DEAD)) { - if (sk->sk_socket->state != SS_UNCONNECTED) - sk->sk_socket->state = SS_DISCONNECTING; - sk->sk_state_change(sk); - } - - /* - * It appears that its possible for remote machines to send disc - * init messages with no port identifier if we are in the CI and - * possibly also the CD state. Obviously we shouldn't reply with - * a message if we don't know what the end point is. - */ - if (scp->addrrem) { - dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC, GFP_ATOMIC); - } - scp->persist_fxn = dn_destroy_timer; - scp->persist = dn_nsp_persist(sk); - -out: - kfree_skb(skb); -} - -/* - * disc_conf messages are also called no_resources or no_link - * messages depending upon the "reason" field. - */ -static void dn_nsp_disc_conf(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - unsigned short reason; - - if (skb->len != 2) - goto out; - - reason = le16_to_cpu(*(__le16 *)skb->data); - - sk->sk_state = TCP_CLOSE; - - switch (scp->state) { - case DN_CI: - scp->state = DN_NR; - break; - case DN_DR: - if (reason == NSP_REASON_DC) - scp->state = DN_DRC; - if (reason == NSP_REASON_NL) - scp->state = DN_CN; - break; - case DN_DI: - scp->state = DN_DIC; - break; - case DN_RUN: - sk->sk_shutdown |= SHUTDOWN_MASK; - fallthrough; - case DN_CC: - scp->state = DN_CN; - } - - if (!sock_flag(sk, SOCK_DEAD)) { - if (sk->sk_socket->state != SS_UNCONNECTED) - sk->sk_socket->state = SS_DISCONNECTING; - sk->sk_state_change(sk); - } - - scp->persist_fxn = dn_destroy_timer; - scp->persist = dn_nsp_persist(sk); - -out: - kfree_skb(skb); -} - -static void dn_nsp_linkservice(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - unsigned short segnum; - unsigned char lsflags; - signed char fcval; - int wake_up = 0; - char *ptr = skb->data; - unsigned char fctype = scp->services_rem & NSP_FC_MASK; - - if (skb->len != 4) - goto out; - - segnum = le16_to_cpu(*(__le16 *)ptr); - ptr += 2; - lsflags = *(unsigned char *)ptr++; - fcval = *ptr; - - /* - * Here we ignore erroneous packets which should really - * should cause a connection abort. It is not critical - * for now though. - */ - if (lsflags & 0xf8) - goto out; - - if (seq_next(scp->numoth_rcv, segnum)) { - seq_add(&scp->numoth_rcv, 1); - switch(lsflags & 0x04) { /* FCVAL INT */ - case 0x00: /* Normal Request */ - switch(lsflags & 0x03) { /* FCVAL MOD */ - case 0x00: /* Request count */ - if (fcval < 0) { - unsigned char p_fcval = -fcval; - if ((scp->flowrem_dat > p_fcval) && - (fctype == NSP_FC_SCMC)) { - scp->flowrem_dat -= p_fcval; - } - } else if (fcval > 0) { - scp->flowrem_dat += fcval; - wake_up = 1; - } - break; - case 0x01: /* Stop outgoing data */ - scp->flowrem_sw = DN_DONTSEND; - break; - case 0x02: /* Ok to start again */ - scp->flowrem_sw = DN_SEND; - dn_nsp_output(sk); - wake_up = 1; - } - break; - case 0x04: /* Interrupt Request */ - if (fcval > 0) { - scp->flowrem_oth += fcval; - wake_up = 1; - } - break; - } - if (wake_up && !sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); - } - - dn_nsp_send_oth_ack(sk); - -out: - kfree_skb(skb); -} - -/* - * Copy of sock_queue_rcv_skb (from sock.h) without - * bh_lock_sock() (its already held when this is called) which - * also allows data and other data to be queued to a socket. - */ -static __inline__ int dn_queue_skb(struct sock *sk, struct sk_buff *skb, int sig, struct sk_buff_head *queue) -{ - int err; - - /* Cast skb->rcvbuf to unsigned... It's pointless, but reduces - number of warnings when compiling with -W --ANK - */ - if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= - (unsigned int)sk->sk_rcvbuf) { - err = -ENOMEM; - goto out; - } - - err = sk_filter(sk, skb); - if (err) - goto out; - - skb_set_owner_r(skb, sk); - skb_queue_tail(queue, skb); - - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk); -out: - return err; -} - -static void dn_nsp_otherdata(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - unsigned short segnum; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - int queued = 0; - - if (skb->len < 2) - goto out; - - cb->segnum = segnum = le16_to_cpu(*(__le16 *)skb->data); - skb_pull(skb, 2); - - if (seq_next(scp->numoth_rcv, segnum)) { - - if (dn_queue_skb(sk, skb, SIGURG, &scp->other_receive_queue) == 0) { - seq_add(&scp->numoth_rcv, 1); - scp->other_report = 0; - queued = 1; - } - } - - dn_nsp_send_oth_ack(sk); -out: - if (!queued) - kfree_skb(skb); -} - -static void dn_nsp_data(struct sock *sk, struct sk_buff *skb) -{ - int queued = 0; - unsigned short segnum; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct dn_scp *scp = DN_SK(sk); - - if (skb->len < 2) - goto out; - - cb->segnum = segnum = le16_to_cpu(*(__le16 *)skb->data); - skb_pull(skb, 2); - - if (seq_next(scp->numdat_rcv, segnum)) { - if (dn_queue_skb(sk, skb, SIGIO, &sk->sk_receive_queue) == 0) { - seq_add(&scp->numdat_rcv, 1); - queued = 1; - } - - if ((scp->flowloc_sw == DN_SEND) && dn_congested(sk)) { - scp->flowloc_sw = DN_DONTSEND; - dn_nsp_send_link(sk, DN_DONTSEND, 0); - } - } - - dn_nsp_send_data_ack(sk); -out: - if (!queued) - kfree_skb(skb); -} - -/* - * If one of our conninit messages is returned, this function - * deals with it. It puts the socket into the NO_COMMUNICATION - * state. - */ -static void dn_returned_conn_init(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - - if (scp->state == DN_CI) { - scp->state = DN_NC; - sk->sk_state = TCP_CLOSE; - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); - } - - kfree_skb(skb); -} - -static int dn_nsp_no_socket(struct sk_buff *skb, unsigned short reason) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - int ret = NET_RX_DROP; - - /* Must not reply to returned packets */ - if (cb->rt_flags & DN_RT_F_RTS) - goto out; - - if ((reason != NSP_REASON_OK) && ((cb->nsp_flags & 0x0c) == 0x08)) { - switch (cb->nsp_flags & 0x70) { - case 0x10: - case 0x60: /* (Retransmitted) Connect Init */ - dn_nsp_return_disc(skb, NSP_DISCINIT, reason); - ret = NET_RX_SUCCESS; - break; - case 0x20: /* Connect Confirm */ - dn_nsp_return_disc(skb, NSP_DISCCONF, reason); - ret = NET_RX_SUCCESS; - break; - } - } - -out: - kfree_skb(skb); - return ret; -} - -static int dn_nsp_rx_packet(struct net *net, struct sock *sk2, - struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct sock *sk = NULL; - unsigned char *ptr = (unsigned char *)skb->data; - unsigned short reason = NSP_REASON_NL; - - if (!pskb_may_pull(skb, 2)) - goto free_out; - - skb_reset_transport_header(skb); - cb->nsp_flags = *ptr++; - - if (decnet_debug_level & 2) - printk(KERN_DEBUG "dn_nsp_rx: Message type 0x%02x\n", (int)cb->nsp_flags); - - if (cb->nsp_flags & 0x83) - goto free_out; - - /* - * Filter out conninits and useless packet types - */ - if ((cb->nsp_flags & 0x0c) == 0x08) { - switch (cb->nsp_flags & 0x70) { - case 0x00: /* NOP */ - case 0x70: /* Reserved */ - case 0x50: /* Reserved, Phase II node init */ - goto free_out; - case 0x10: - case 0x60: - if (unlikely(cb->rt_flags & DN_RT_F_RTS)) - goto free_out; - sk = dn_find_listener(skb, &reason); - goto got_it; - } - } - - if (!pskb_may_pull(skb, 3)) - goto free_out; - - /* - * Grab the destination address. - */ - cb->dst_port = *(__le16 *)ptr; - cb->src_port = 0; - ptr += 2; - - /* - * If not a connack, grab the source address too. - */ - if (pskb_may_pull(skb, 5)) { - cb->src_port = *(__le16 *)ptr; - ptr += 2; - skb_pull(skb, 5); - } - - /* - * Returned packets... - * Swap src & dst and look up in the normal way. - */ - if (unlikely(cb->rt_flags & DN_RT_F_RTS)) { - swap(cb->dst_port, cb->src_port); - swap(cb->dst, cb->src); - } - - /* - * Find the socket to which this skb is destined. - */ - sk = dn_find_by_skb(skb); -got_it: - if (sk != NULL) { - struct dn_scp *scp = DN_SK(sk); - - /* Reset backoff */ - scp->nsp_rxtshift = 0; - - /* - * We linearize everything except data segments here. - */ - if (cb->nsp_flags & ~0x60) { - if (unlikely(skb_linearize(skb))) - goto free_out; - } - - return sk_receive_skb(sk, skb, 0); - } - - return dn_nsp_no_socket(skb, reason); - -free_out: - kfree_skb(skb); - return NET_RX_DROP; -} - -int dn_nsp_rx(struct sk_buff *skb) -{ - return NF_HOOK(NFPROTO_DECNET, NF_DN_LOCAL_IN, - &init_net, NULL, skb, skb->dev, NULL, - dn_nsp_rx_packet); -} - -/* - * This is the main receive routine for sockets. It is called - * from the above when the socket is not busy, and also from - * sock_release() when there is a backlog queued up. - */ -int dn_nsp_backlog_rcv(struct sock *sk, struct sk_buff *skb) -{ - struct dn_scp *scp = DN_SK(sk); - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - if (cb->rt_flags & DN_RT_F_RTS) { - if (cb->nsp_flags == 0x18 || cb->nsp_flags == 0x68) - dn_returned_conn_init(sk, skb); - else - kfree_skb(skb); - return NET_RX_SUCCESS; - } - - /* - * Control packet. - */ - if ((cb->nsp_flags & 0x0c) == 0x08) { - switch (cb->nsp_flags & 0x70) { - case 0x10: - case 0x60: - dn_nsp_conn_init(sk, skb); - break; - case 0x20: - dn_nsp_conn_conf(sk, skb); - break; - case 0x30: - dn_nsp_disc_init(sk, skb); - break; - case 0x40: - dn_nsp_disc_conf(sk, skb); - break; - } - - } else if (cb->nsp_flags == 0x24) { - /* - * Special for connacks, 'cos they don't have - * ack data or ack otherdata info. - */ - dn_nsp_conn_ack(sk, skb); - } else { - int other = 1; - - /* both data and ack frames can kick a CC socket into RUN */ - if ((scp->state == DN_CC) && !sock_flag(sk, SOCK_DEAD)) { - scp->state = DN_RUN; - sk->sk_state = TCP_ESTABLISHED; - sk->sk_state_change(sk); - } - - if ((cb->nsp_flags & 0x1c) == 0) - other = 0; - if (cb->nsp_flags == 0x04) - other = 0; - - /* - * Read out ack data here, this applies equally - * to data, other data, link service and both - * ack data and ack otherdata. - */ - dn_process_ack(sk, skb, other); - - /* - * If we've some sort of data here then call a - * suitable routine for dealing with it, otherwise - * the packet is an ack and can be discarded. - */ - if ((cb->nsp_flags & 0x0c) == 0) { - - if (scp->state != DN_RUN) - goto free_out; - - switch (cb->nsp_flags) { - case 0x10: /* LS */ - dn_nsp_linkservice(sk, skb); - break; - case 0x30: /* OD */ - dn_nsp_otherdata(sk, skb); - break; - default: - dn_nsp_data(sk, skb); - } - - } else { /* Ack, chuck it out here */ -free_out: - kfree_skb(skb); - } - } - - return NET_RX_SUCCESS; -} diff --git a/net/decnet/dn_nsp_out.c b/net/decnet/dn_nsp_out.c deleted file mode 100644 index b05639bdfc8f5b73e126a06f693b1c2c58d357a9..0000000000000000000000000000000000000000 --- a/net/decnet/dn_nsp_out.c +++ /dev/null @@ -1,696 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Network Services Protocol (Output) - * - * Author: Eduardo Marcelo Serrat - * - * Changes: - * - * Steve Whitehouse: Split into dn_nsp_in.c and dn_nsp_out.c from - * original dn_nsp.c. - * Steve Whitehouse: Updated to work with my new routing architecture. - * Steve Whitehouse: Added changes from Eduardo Serrat's patches. - * Steve Whitehouse: Now conninits have the "return" bit set. - * Steve Whitehouse: Fixes to check alloc'd skbs are non NULL! - * Moved output state machine into one function - * Steve Whitehouse: New output state machine - * Paul Koning: Connect Confirm message fix. - * Eduardo Serrat: Fix to stop dn_nsp_do_disc() sending malformed packets. - * Steve Whitehouse: dn_nsp_output() and friends needed a spring clean - * Steve Whitehouse: Moved dn_nsp_send() in here from route.h - */ - -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -static int nsp_backoff[NSP_MAXRXTSHIFT + 1] = { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 }; - -static void dn_nsp_send(struct sk_buff *skb) -{ - struct sock *sk = skb->sk; - struct dn_scp *scp = DN_SK(sk); - struct dst_entry *dst; - struct flowidn fld; - - skb_reset_transport_header(skb); - scp->stamp = jiffies; - - dst = sk_dst_check(sk, 0); - if (dst) { -try_again: - skb_dst_set(skb, dst); - dst_output(&init_net, skb->sk, skb); - return; - } - - memset(&fld, 0, sizeof(fld)); - fld.flowidn_oif = sk->sk_bound_dev_if; - fld.saddr = dn_saddr2dn(&scp->addr); - fld.daddr = dn_saddr2dn(&scp->peer); - dn_sk_ports_copy(&fld, scp); - fld.flowidn_proto = DNPROTO_NSP; - if (dn_route_output_sock(&sk->sk_dst_cache, &fld, sk, 0) == 0) { - dst = sk_dst_get(sk); - sk->sk_route_caps = dst->dev->features; - goto try_again; - } - - sk->sk_err = EHOSTUNREACH; - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_state_change(sk); -} - - -/* - * If sk == NULL, then we assume that we are supposed to be making - * a routing layer skb. If sk != NULL, then we are supposed to be - * creating an skb for the NSP layer. - * - * The eventual aim is for each socket to have a cached header size - * for its outgoing packets, and to set hdr from this when sk != NULL. - */ -struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri) -{ - struct sk_buff *skb; - int hdr = 64; - - if ((skb = alloc_skb(size + hdr, pri)) == NULL) - return NULL; - - skb->protocol = htons(ETH_P_DNA_RT); - skb->pkt_type = PACKET_OUTGOING; - - if (sk) - skb_set_owner_w(skb, sk); - - skb_reserve(skb, hdr); - - return skb; -} - -/* - * Calculate persist timer based upon the smoothed round - * trip time and the variance. Backoff according to the - * nsp_backoff[] array. - */ -unsigned long dn_nsp_persist(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - unsigned long t = ((scp->nsp_srtt >> 2) + scp->nsp_rttvar) >> 1; - - t *= nsp_backoff[scp->nsp_rxtshift]; - - if (t < HZ) t = HZ; - if (t > (600*HZ)) t = (600*HZ); - - if (scp->nsp_rxtshift < NSP_MAXRXTSHIFT) - scp->nsp_rxtshift++; - - /* printk(KERN_DEBUG "rxtshift %lu, t=%lu\n", scp->nsp_rxtshift, t); */ - - return t; -} - -/* - * This is called each time we get an estimate for the rtt - * on the link. - */ -static void dn_nsp_rtt(struct sock *sk, long rtt) -{ - struct dn_scp *scp = DN_SK(sk); - long srtt = (long)scp->nsp_srtt; - long rttvar = (long)scp->nsp_rttvar; - long delta; - - /* - * If the jiffies clock flips over in the middle of timestamp - * gathering this value might turn out negative, so we make sure - * that is it always positive here. - */ - if (rtt < 0) - rtt = -rtt; - /* - * Add new rtt to smoothed average - */ - delta = ((rtt << 3) - srtt); - srtt += (delta >> 3); - if (srtt >= 1) - scp->nsp_srtt = (unsigned long)srtt; - else - scp->nsp_srtt = 1; - - /* - * Add new rtt variance to smoothed varience - */ - delta >>= 1; - rttvar += ((((delta>0)?(delta):(-delta)) - rttvar) >> 2); - if (rttvar >= 1) - scp->nsp_rttvar = (unsigned long)rttvar; - else - scp->nsp_rttvar = 1; - - /* printk(KERN_DEBUG "srtt=%lu rttvar=%lu\n", scp->nsp_srtt, scp->nsp_rttvar); */ -} - -/** - * dn_nsp_clone_and_send - Send a data packet by cloning it - * @skb: The packet to clone and transmit - * @gfp: memory allocation flag - * - * Clone a queued data or other data packet and transmit it. - * - * Returns: The number of times the packet has been sent previously - */ -static inline unsigned int dn_nsp_clone_and_send(struct sk_buff *skb, - gfp_t gfp) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct sk_buff *skb2; - int ret = 0; - - if ((skb2 = skb_clone(skb, gfp)) != NULL) { - ret = cb->xmit_count; - cb->xmit_count++; - cb->stamp = jiffies; - skb2->sk = skb->sk; - dn_nsp_send(skb2); - } - - return ret; -} - -/** - * dn_nsp_output - Try and send something from socket queues - * @sk: The socket whose queues are to be investigated - * - * Try and send the packet on the end of the data and other data queues. - * Other data gets priority over data, and if we retransmit a packet we - * reduce the window by dividing it in two. - * - */ -void dn_nsp_output(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - struct sk_buff *skb; - unsigned int reduce_win = 0; - - /* - * First we check for otherdata/linkservice messages - */ - if ((skb = skb_peek(&scp->other_xmit_queue)) != NULL) - reduce_win = dn_nsp_clone_and_send(skb, GFP_ATOMIC); - - /* - * If we may not send any data, we don't. - * If we are still trying to get some other data down the - * channel, we don't try and send any data. - */ - if (reduce_win || (scp->flowrem_sw != DN_SEND)) - goto recalc_window; - - if ((skb = skb_peek(&scp->data_xmit_queue)) != NULL) - reduce_win = dn_nsp_clone_and_send(skb, GFP_ATOMIC); - - /* - * If we've sent any frame more than once, we cut the - * send window size in half. There is always a minimum - * window size of one available. - */ -recalc_window: - if (reduce_win) { - scp->snd_window >>= 1; - if (scp->snd_window < NSP_MIN_WINDOW) - scp->snd_window = NSP_MIN_WINDOW; - } -} - -int dn_nsp_xmit_timeout(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - dn_nsp_output(sk); - - if (!skb_queue_empty(&scp->data_xmit_queue) || - !skb_queue_empty(&scp->other_xmit_queue)) - scp->persist = dn_nsp_persist(sk); - - return 0; -} - -static inline __le16 *dn_mk_common_header(struct dn_scp *scp, struct sk_buff *skb, unsigned char msgflag, int len) -{ - unsigned char *ptr = skb_push(skb, len); - - BUG_ON(len < 5); - - *ptr++ = msgflag; - *((__le16 *)ptr) = scp->addrrem; - ptr += 2; - *((__le16 *)ptr) = scp->addrloc; - ptr += 2; - return (__le16 __force *)ptr; -} - -static __le16 *dn_mk_ack_header(struct sock *sk, struct sk_buff *skb, unsigned char msgflag, int hlen, int other) -{ - struct dn_scp *scp = DN_SK(sk); - unsigned short acknum = scp->numdat_rcv & 0x0FFF; - unsigned short ackcrs = scp->numoth_rcv & 0x0FFF; - __le16 *ptr; - - BUG_ON(hlen < 9); - - scp->ackxmt_dat = acknum; - scp->ackxmt_oth = ackcrs; - acknum |= 0x8000; - ackcrs |= 0x8000; - - /* If this is an "other data/ack" message, swap acknum and ackcrs */ - if (other) - swap(acknum, ackcrs); - - /* Set "cross subchannel" bit in ackcrs */ - ackcrs |= 0x2000; - - ptr = dn_mk_common_header(scp, skb, msgflag, hlen); - - *ptr++ = cpu_to_le16(acknum); - *ptr++ = cpu_to_le16(ackcrs); - - return ptr; -} - -static __le16 *dn_nsp_mk_data_header(struct sock *sk, struct sk_buff *skb, int oth) -{ - struct dn_scp *scp = DN_SK(sk); - struct dn_skb_cb *cb = DN_SKB_CB(skb); - __le16 *ptr = dn_mk_ack_header(sk, skb, cb->nsp_flags, 11, oth); - - if (unlikely(oth)) { - cb->segnum = scp->numoth; - seq_add(&scp->numoth, 1); - } else { - cb->segnum = scp->numdat; - seq_add(&scp->numdat, 1); - } - *(ptr++) = cpu_to_le16(cb->segnum); - - return ptr; -} - -void dn_nsp_queue_xmit(struct sock *sk, struct sk_buff *skb, - gfp_t gfp, int oth) -{ - struct dn_scp *scp = DN_SK(sk); - struct dn_skb_cb *cb = DN_SKB_CB(skb); - unsigned long t = ((scp->nsp_srtt >> 2) + scp->nsp_rttvar) >> 1; - - cb->xmit_count = 0; - dn_nsp_mk_data_header(sk, skb, oth); - - /* - * Slow start: If we have been idle for more than - * one RTT, then reset window to min size. - */ - if (time_is_before_jiffies(scp->stamp + t)) - scp->snd_window = NSP_MIN_WINDOW; - - if (oth) - skb_queue_tail(&scp->other_xmit_queue, skb); - else - skb_queue_tail(&scp->data_xmit_queue, skb); - - if (scp->flowrem_sw != DN_SEND) - return; - - dn_nsp_clone_and_send(skb, gfp); -} - - -int dn_nsp_check_xmit_queue(struct sock *sk, struct sk_buff *skb, struct sk_buff_head *q, unsigned short acknum) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct dn_scp *scp = DN_SK(sk); - struct sk_buff *skb2, *n, *ack = NULL; - int wakeup = 0; - int try_retrans = 0; - unsigned long reftime = cb->stamp; - unsigned long pkttime; - unsigned short xmit_count; - unsigned short segnum; - - skb_queue_walk_safe(q, skb2, n) { - struct dn_skb_cb *cb2 = DN_SKB_CB(skb2); - - if (dn_before_or_equal(cb2->segnum, acknum)) - ack = skb2; - - /* printk(KERN_DEBUG "ack: %s %04x %04x\n", ack ? "ACK" : "SKIP", (int)cb2->segnum, (int)acknum); */ - - if (ack == NULL) - continue; - - /* printk(KERN_DEBUG "check_xmit_queue: %04x, %d\n", acknum, cb2->xmit_count); */ - - /* Does _last_ packet acked have xmit_count > 1 */ - try_retrans = 0; - /* Remember to wake up the sending process */ - wakeup = 1; - /* Keep various statistics */ - pkttime = cb2->stamp; - xmit_count = cb2->xmit_count; - segnum = cb2->segnum; - /* Remove and drop ack'ed packet */ - skb_unlink(ack, q); - kfree_skb(ack); - ack = NULL; - - /* - * We don't expect to see acknowledgements for packets we - * haven't sent yet. - */ - WARN_ON(xmit_count == 0); - - /* - * If the packet has only been sent once, we can use it - * to calculate the RTT and also open the window a little - * further. - */ - if (xmit_count == 1) { - if (dn_equal(segnum, acknum)) - dn_nsp_rtt(sk, (long)(pkttime - reftime)); - - if (scp->snd_window < scp->max_window) - scp->snd_window++; - } - - /* - * Packet has been sent more than once. If this is the last - * packet to be acknowledged then we want to send the next - * packet in the send queue again (assumes the remote host does - * go-back-N error control). - */ - if (xmit_count > 1) - try_retrans = 1; - } - - if (try_retrans) - dn_nsp_output(sk); - - return wakeup; -} - -void dn_nsp_send_data_ack(struct sock *sk) -{ - struct sk_buff *skb = NULL; - - if ((skb = dn_alloc_skb(sk, 9, GFP_ATOMIC)) == NULL) - return; - - skb_reserve(skb, 9); - dn_mk_ack_header(sk, skb, 0x04, 9, 0); - dn_nsp_send(skb); -} - -void dn_nsp_send_oth_ack(struct sock *sk) -{ - struct sk_buff *skb = NULL; - - if ((skb = dn_alloc_skb(sk, 9, GFP_ATOMIC)) == NULL) - return; - - skb_reserve(skb, 9); - dn_mk_ack_header(sk, skb, 0x14, 9, 1); - dn_nsp_send(skb); -} - - -void dn_send_conn_ack (struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - struct sk_buff *skb = NULL; - struct nsp_conn_ack_msg *msg; - - if ((skb = dn_alloc_skb(sk, 3, sk->sk_allocation)) == NULL) - return; - - msg = skb_put(skb, 3); - msg->msgflg = 0x24; - msg->dstaddr = scp->addrrem; - - dn_nsp_send(skb); -} - -static int dn_nsp_retrans_conn_conf(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - if (scp->state == DN_CC) - dn_send_conn_conf(sk, GFP_ATOMIC); - - return 0; -} - -void dn_send_conn_conf(struct sock *sk, gfp_t gfp) -{ - struct dn_scp *scp = DN_SK(sk); - struct sk_buff *skb = NULL; - struct nsp_conn_init_msg *msg; - __u8 len = (__u8)le16_to_cpu(scp->conndata_out.opt_optl); - - if ((skb = dn_alloc_skb(sk, 50 + len, gfp)) == NULL) - return; - - msg = skb_put(skb, sizeof(*msg)); - msg->msgflg = 0x28; - msg->dstaddr = scp->addrrem; - msg->srcaddr = scp->addrloc; - msg->services = scp->services_loc; - msg->info = scp->info_loc; - msg->segsize = cpu_to_le16(scp->segsize_loc); - - skb_put_u8(skb, len); - - if (len > 0) - skb_put_data(skb, scp->conndata_out.opt_data, len); - - - dn_nsp_send(skb); - - scp->persist = dn_nsp_persist(sk); - scp->persist_fxn = dn_nsp_retrans_conn_conf; -} - - -static __inline__ void dn_nsp_do_disc(struct sock *sk, unsigned char msgflg, - unsigned short reason, gfp_t gfp, - struct dst_entry *dst, - int ddl, unsigned char *dd, __le16 rem, __le16 loc) -{ - struct sk_buff *skb = NULL; - int size = 7 + ddl + ((msgflg == NSP_DISCINIT) ? 1 : 0); - unsigned char *msg; - - if ((dst == NULL) || (rem == 0)) { - net_dbg_ratelimited("DECnet: dn_nsp_do_disc: BUG! Please report this to SteveW@ACM.org rem=%u dst=%p\n", - le16_to_cpu(rem), dst); - return; - } - - if ((skb = dn_alloc_skb(sk, size, gfp)) == NULL) - return; - - msg = skb_put(skb, size); - *msg++ = msgflg; - *(__le16 *)msg = rem; - msg += 2; - *(__le16 *)msg = loc; - msg += 2; - *(__le16 *)msg = cpu_to_le16(reason); - msg += 2; - if (msgflg == NSP_DISCINIT) - *msg++ = ddl; - - if (ddl) { - memcpy(msg, dd, ddl); - } - - /* - * This doesn't go via the dn_nsp_send() function since we need - * to be able to send disc packets out which have no socket - * associations. - */ - skb_dst_set(skb, dst_clone(dst)); - dst_output(&init_net, skb->sk, skb); -} - - -void dn_nsp_send_disc(struct sock *sk, unsigned char msgflg, - unsigned short reason, gfp_t gfp) -{ - struct dn_scp *scp = DN_SK(sk); - int ddl = 0; - - if (msgflg == NSP_DISCINIT) - ddl = le16_to_cpu(scp->discdata_out.opt_optl); - - if (reason == 0) - reason = le16_to_cpu(scp->discdata_out.opt_status); - - dn_nsp_do_disc(sk, msgflg, reason, gfp, __sk_dst_get(sk), ddl, - scp->discdata_out.opt_data, scp->addrrem, scp->addrloc); -} - - -void dn_nsp_return_disc(struct sk_buff *skb, unsigned char msgflg, - unsigned short reason) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - int ddl = 0; - gfp_t gfp = GFP_ATOMIC; - - dn_nsp_do_disc(NULL, msgflg, reason, gfp, skb_dst(skb), ddl, - NULL, cb->src_port, cb->dst_port); -} - - -void dn_nsp_send_link(struct sock *sk, unsigned char lsflags, char fcval) -{ - struct dn_scp *scp = DN_SK(sk); - struct sk_buff *skb; - unsigned char *ptr; - gfp_t gfp = GFP_ATOMIC; - - if ((skb = dn_alloc_skb(sk, DN_MAX_NSP_DATA_HEADER + 2, gfp)) == NULL) - return; - - skb_reserve(skb, DN_MAX_NSP_DATA_HEADER); - ptr = skb_put(skb, 2); - DN_SKB_CB(skb)->nsp_flags = 0x10; - *ptr++ = lsflags; - *ptr = fcval; - - dn_nsp_queue_xmit(sk, skb, gfp, 1); - - scp->persist = dn_nsp_persist(sk); - scp->persist_fxn = dn_nsp_xmit_timeout; -} - -static int dn_nsp_retrans_conninit(struct sock *sk) -{ - struct dn_scp *scp = DN_SK(sk); - - if (scp->state == DN_CI) - dn_nsp_send_conninit(sk, NSP_RCI); - - return 0; -} - -void dn_nsp_send_conninit(struct sock *sk, unsigned char msgflg) -{ - struct dn_scp *scp = DN_SK(sk); - struct nsp_conn_init_msg *msg; - unsigned char aux; - unsigned char menuver; - struct dn_skb_cb *cb; - unsigned char type = 1; - gfp_t allocation = (msgflg == NSP_CI) ? sk->sk_allocation : GFP_ATOMIC; - struct sk_buff *skb = dn_alloc_skb(sk, 200, allocation); - - if (!skb) - return; - - cb = DN_SKB_CB(skb); - msg = skb_put(skb, sizeof(*msg)); - - msg->msgflg = msgflg; - msg->dstaddr = 0x0000; /* Remote Node will assign it*/ - - msg->srcaddr = scp->addrloc; - msg->services = scp->services_loc; /* Requested flow control */ - msg->info = scp->info_loc; /* Version Number */ - msg->segsize = cpu_to_le16(scp->segsize_loc); /* Max segment size */ - - if (scp->peer.sdn_objnum) - type = 0; - - skb_put(skb, dn_sockaddr2username(&scp->peer, - skb_tail_pointer(skb), type)); - skb_put(skb, dn_sockaddr2username(&scp->addr, - skb_tail_pointer(skb), 2)); - - menuver = DN_MENUVER_ACC | DN_MENUVER_USR; - if (scp->peer.sdn_flags & SDF_PROXY) - menuver |= DN_MENUVER_PRX; - if (scp->peer.sdn_flags & SDF_UICPROXY) - menuver |= DN_MENUVER_UIC; - - skb_put_u8(skb, menuver); /* Menu Version */ - - aux = scp->accessdata.acc_userl; - skb_put_u8(skb, aux); - if (aux > 0) - skb_put_data(skb, scp->accessdata.acc_user, aux); - - aux = scp->accessdata.acc_passl; - skb_put_u8(skb, aux); - if (aux > 0) - skb_put_data(skb, scp->accessdata.acc_pass, aux); - - aux = scp->accessdata.acc_accl; - skb_put_u8(skb, aux); - if (aux > 0) - skb_put_data(skb, scp->accessdata.acc_acc, aux); - - aux = (__u8)le16_to_cpu(scp->conndata_out.opt_optl); - skb_put_u8(skb, aux); - if (aux > 0) - skb_put_data(skb, scp->conndata_out.opt_data, aux); - - scp->persist = dn_nsp_persist(sk); - scp->persist_fxn = dn_nsp_retrans_conninit; - - cb->rt_flags = DN_RT_F_RQR; - - dn_nsp_send(skb); -} diff --git a/net/decnet/dn_route.c b/net/decnet/dn_route.c deleted file mode 100644 index ac2ee1689111205f088bf89896f52e9f7925cf7e..0000000000000000000000000000000000000000 --- a/net/decnet/dn_route.c +++ /dev/null @@ -1,1922 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Routing Functions (Endnode and Router) - * - * Authors: Steve Whitehouse - * Eduardo Marcelo Serrat - * - * Changes: - * Steve Whitehouse : Fixes to allow "intra-ethernet" and - * "return-to-sender" bits on outgoing - * packets. - * Steve Whitehouse : Timeouts for cached routes. - * Steve Whitehouse : Use dst cache for input routes too. - * Steve Whitehouse : Fixed error values in dn_send_skb. - * Steve Whitehouse : Rework routing functions to better fit - * DECnet routing design - * Alexey Kuznetsov : New SMP locking - * Steve Whitehouse : More SMP locking changes & dn_cache_dump() - * Steve Whitehouse : Prerouting NF hook, now really is prerouting. - * Fixed possible skb leak in rtnetlink funcs. - * Steve Whitehouse : Dave Miller's dynamic hash table sizing and - * Alexey Kuznetsov's finer grained locking - * from ipv4/route.c. - * Steve Whitehouse : Routing is now starting to look like a - * sensible set of code now, mainly due to - * my copying the IPv4 routing code. The - * hooks here are modified and will continue - * to evolve for a while. - * Steve Whitehouse : Real SMP at last :-) Also new netfilter - * stuff. Look out raw sockets your days - * are numbered! - * Steve Whitehouse : Added return-to-sender functions. Added - * backlog congestion level return codes. - * Steve Whitehouse : Fixed bug where routes were set up with - * no ref count on net devices. - * Steve Whitehouse : RCU for the route cache - * Steve Whitehouse : Preparations for the flow cache - * Steve Whitehouse : Prepare for nonlinear skbs - */ - -/****************************************************************************** - (c) 1995-1998 E.M. Serrat emserrat@geocities.com - -*******************************************************************************/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct dn_rt_hash_bucket { - struct dn_route __rcu *chain; - spinlock_t lock; -}; - -extern struct neigh_table dn_neigh_table; - - -static unsigned char dn_hiord_addr[6] = {0xAA, 0x00, 0x04, 0x00, 0x00, 0x00}; - -static const int dn_rt_min_delay = 2 * HZ; -static const int dn_rt_max_delay = 10 * HZ; -static const int dn_rt_mtu_expires = 10 * 60 * HZ; - -static unsigned long dn_rt_deadline; - -static int dn_dst_gc(struct dst_ops *ops); -static struct dst_entry *dn_dst_check(struct dst_entry *, __u32); -static unsigned int dn_dst_default_advmss(const struct dst_entry *dst); -static unsigned int dn_dst_mtu(const struct dst_entry *dst); -static void dn_dst_destroy(struct dst_entry *); -static void dn_dst_ifdown(struct dst_entry *, struct net_device *dev, int how); -static struct dst_entry *dn_dst_negative_advice(struct dst_entry *); -static void dn_dst_link_failure(struct sk_buff *); -static void dn_dst_update_pmtu(struct dst_entry *dst, struct sock *sk, - struct sk_buff *skb , u32 mtu, - bool confirm_neigh); -static void dn_dst_redirect(struct dst_entry *dst, struct sock *sk, - struct sk_buff *skb); -static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst, - struct sk_buff *skb, - const void *daddr); -static int dn_route_input(struct sk_buff *); -static void dn_run_flush(struct timer_list *unused); - -static struct dn_rt_hash_bucket *dn_rt_hash_table; -static unsigned int dn_rt_hash_mask; - -static struct timer_list dn_route_timer; -static DEFINE_TIMER(dn_rt_flush_timer, dn_run_flush); -int decnet_dst_gc_interval = 2; - -static struct dst_ops dn_dst_ops = { - .family = PF_DECnet, - .gc_thresh = 128, - .gc = dn_dst_gc, - .check = dn_dst_check, - .default_advmss = dn_dst_default_advmss, - .mtu = dn_dst_mtu, - .cow_metrics = dst_cow_metrics_generic, - .destroy = dn_dst_destroy, - .ifdown = dn_dst_ifdown, - .negative_advice = dn_dst_negative_advice, - .link_failure = dn_dst_link_failure, - .update_pmtu = dn_dst_update_pmtu, - .redirect = dn_dst_redirect, - .neigh_lookup = dn_dst_neigh_lookup, -}; - -static void dn_dst_destroy(struct dst_entry *dst) -{ - struct dn_route *rt = (struct dn_route *) dst; - - if (rt->n) - neigh_release(rt->n); - dst_destroy_metrics_generic(dst); -} - -static void dn_dst_ifdown(struct dst_entry *dst, struct net_device *dev, int how) -{ - if (how) { - struct dn_route *rt = (struct dn_route *) dst; - struct neighbour *n = rt->n; - - if (n && n->dev == dev) { - n->dev = blackhole_netdev; - dev_hold(n->dev); - dev_put(dev); - } - } -} - -static __inline__ unsigned int dn_hash(__le16 src, __le16 dst) -{ - __u16 tmp = (__u16 __force)(src ^ dst); - tmp ^= (tmp >> 3); - tmp ^= (tmp >> 5); - tmp ^= (tmp >> 10); - return dn_rt_hash_mask & (unsigned int)tmp; -} - -static void dn_dst_check_expire(struct timer_list *unused) -{ - int i; - struct dn_route *rt; - struct dn_route __rcu **rtp; - unsigned long now = jiffies; - unsigned long expire = 120 * HZ; - - for (i = 0; i <= dn_rt_hash_mask; i++) { - rtp = &dn_rt_hash_table[i].chain; - - spin_lock(&dn_rt_hash_table[i].lock); - while ((rt = rcu_dereference_protected(*rtp, - lockdep_is_held(&dn_rt_hash_table[i].lock))) != NULL) { - if (atomic_read(&rt->dst.__refcnt) > 1 || - (now - rt->dst.lastuse) < expire) { - rtp = &rt->dn_next; - continue; - } - *rtp = rt->dn_next; - rt->dn_next = NULL; - dst_dev_put(&rt->dst); - dst_release(&rt->dst); - } - spin_unlock(&dn_rt_hash_table[i].lock); - - if (jiffies != now) - break; - } - - mod_timer(&dn_route_timer, now + decnet_dst_gc_interval * HZ); -} - -static int dn_dst_gc(struct dst_ops *ops) -{ - struct dn_route *rt; - struct dn_route __rcu **rtp; - int i; - unsigned long now = jiffies; - unsigned long expire = 10 * HZ; - - for (i = 0; i <= dn_rt_hash_mask; i++) { - - spin_lock_bh(&dn_rt_hash_table[i].lock); - rtp = &dn_rt_hash_table[i].chain; - - while ((rt = rcu_dereference_protected(*rtp, - lockdep_is_held(&dn_rt_hash_table[i].lock))) != NULL) { - if (atomic_read(&rt->dst.__refcnt) > 1 || - (now - rt->dst.lastuse) < expire) { - rtp = &rt->dn_next; - continue; - } - *rtp = rt->dn_next; - rt->dn_next = NULL; - dst_dev_put(&rt->dst); - dst_release(&rt->dst); - break; - } - spin_unlock_bh(&dn_rt_hash_table[i].lock); - } - - return 0; -} - -/* - * The decnet standards don't impose a particular minimum mtu, what they - * do insist on is that the routing layer accepts a datagram of at least - * 230 bytes long. Here we have to subtract the routing header length from - * 230 to get the minimum acceptable mtu. If there is no neighbour, then we - * assume the worst and use a long header size. - * - * We update both the mtu and the advertised mss (i.e. the segment size we - * advertise to the other end). - */ -static void dn_dst_update_pmtu(struct dst_entry *dst, struct sock *sk, - struct sk_buff *skb, u32 mtu, - bool confirm_neigh) -{ - struct dn_route *rt = (struct dn_route *) dst; - struct neighbour *n = rt->n; - u32 min_mtu = 230; - struct dn_dev *dn; - - dn = n ? rcu_dereference_raw(n->dev->dn_ptr) : NULL; - - if (dn && dn->use_long == 0) - min_mtu -= 6; - else - min_mtu -= 21; - - if (dst_metric(dst, RTAX_MTU) > mtu && mtu >= min_mtu) { - if (!(dst_metric_locked(dst, RTAX_MTU))) { - dst_metric_set(dst, RTAX_MTU, mtu); - dst_set_expires(dst, dn_rt_mtu_expires); - } - if (!(dst_metric_locked(dst, RTAX_ADVMSS))) { - u32 mss = mtu - DN_MAX_NSP_DATA_HEADER; - u32 existing_mss = dst_metric_raw(dst, RTAX_ADVMSS); - if (!existing_mss || existing_mss > mss) - dst_metric_set(dst, RTAX_ADVMSS, mss); - } - } -} - -static void dn_dst_redirect(struct dst_entry *dst, struct sock *sk, - struct sk_buff *skb) -{ -} - -/* - * When a route has been marked obsolete. (e.g. routing cache flush) - */ -static struct dst_entry *dn_dst_check(struct dst_entry *dst, __u32 cookie) -{ - return NULL; -} - -static struct dst_entry *dn_dst_negative_advice(struct dst_entry *dst) -{ - dst_release(dst); - return NULL; -} - -static void dn_dst_link_failure(struct sk_buff *skb) -{ -} - -static inline int compare_keys(struct flowidn *fl1, struct flowidn *fl2) -{ - return ((fl1->daddr ^ fl2->daddr) | - (fl1->saddr ^ fl2->saddr) | - (fl1->flowidn_mark ^ fl2->flowidn_mark) | - (fl1->flowidn_scope ^ fl2->flowidn_scope) | - (fl1->flowidn_oif ^ fl2->flowidn_oif) | - (fl1->flowidn_iif ^ fl2->flowidn_iif)) == 0; -} - -static int dn_insert_route(struct dn_route *rt, unsigned int hash, struct dn_route **rp) -{ - struct dn_route *rth; - struct dn_route __rcu **rthp; - unsigned long now = jiffies; - - rthp = &dn_rt_hash_table[hash].chain; - - spin_lock_bh(&dn_rt_hash_table[hash].lock); - while ((rth = rcu_dereference_protected(*rthp, - lockdep_is_held(&dn_rt_hash_table[hash].lock))) != NULL) { - if (compare_keys(&rth->fld, &rt->fld)) { - /* Put it first */ - *rthp = rth->dn_next; - rcu_assign_pointer(rth->dn_next, - dn_rt_hash_table[hash].chain); - rcu_assign_pointer(dn_rt_hash_table[hash].chain, rth); - - dst_hold_and_use(&rth->dst, now); - spin_unlock_bh(&dn_rt_hash_table[hash].lock); - - dst_release_immediate(&rt->dst); - *rp = rth; - return 0; - } - rthp = &rth->dn_next; - } - - rcu_assign_pointer(rt->dn_next, dn_rt_hash_table[hash].chain); - rcu_assign_pointer(dn_rt_hash_table[hash].chain, rt); - - dst_hold_and_use(&rt->dst, now); - spin_unlock_bh(&dn_rt_hash_table[hash].lock); - *rp = rt; - return 0; -} - -static void dn_run_flush(struct timer_list *unused) -{ - int i; - struct dn_route *rt, *next; - - for (i = 0; i < dn_rt_hash_mask; i++) { - spin_lock_bh(&dn_rt_hash_table[i].lock); - - rt = xchg((struct dn_route **)&dn_rt_hash_table[i].chain, NULL); - if (!rt) - goto nothing_to_declare; - - for (; rt; rt = next) { - next = rcu_dereference_raw(rt->dn_next); - RCU_INIT_POINTER(rt->dn_next, NULL); - dst_dev_put(&rt->dst); - dst_release(&rt->dst); - } - -nothing_to_declare: - spin_unlock_bh(&dn_rt_hash_table[i].lock); - } -} - -static DEFINE_SPINLOCK(dn_rt_flush_lock); - -void dn_rt_cache_flush(int delay) -{ - unsigned long now = jiffies; - int user_mode = !in_interrupt(); - - if (delay < 0) - delay = dn_rt_min_delay; - - spin_lock_bh(&dn_rt_flush_lock); - - if (del_timer(&dn_rt_flush_timer) && delay > 0 && dn_rt_deadline) { - long tmo = (long)(dn_rt_deadline - now); - - if (user_mode && tmo < dn_rt_max_delay - dn_rt_min_delay) - tmo = 0; - - if (delay > tmo) - delay = tmo; - } - - if (delay <= 0) { - spin_unlock_bh(&dn_rt_flush_lock); - dn_run_flush(NULL); - return; - } - - if (dn_rt_deadline == 0) - dn_rt_deadline = now + dn_rt_max_delay; - - dn_rt_flush_timer.expires = now + delay; - add_timer(&dn_rt_flush_timer); - spin_unlock_bh(&dn_rt_flush_lock); -} - -/** - * dn_return_short - Return a short packet to its sender - * @skb: The packet to return - * - */ -static int dn_return_short(struct sk_buff *skb) -{ - struct dn_skb_cb *cb; - unsigned char *ptr; - __le16 *src; - __le16 *dst; - - /* Add back headers */ - skb_push(skb, skb->data - skb_network_header(skb)); - - skb = skb_unshare(skb, GFP_ATOMIC); - if (!skb) - return NET_RX_DROP; - - cb = DN_SKB_CB(skb); - /* Skip packet length and point to flags */ - ptr = skb->data + 2; - *ptr++ = (cb->rt_flags & ~DN_RT_F_RQR) | DN_RT_F_RTS; - - dst = (__le16 *)ptr; - ptr += 2; - src = (__le16 *)ptr; - ptr += 2; - *ptr = 0; /* Zero hop count */ - - swap(*src, *dst); - - skb->pkt_type = PACKET_OUTGOING; - dn_rt_finish_output(skb, NULL, NULL); - return NET_RX_SUCCESS; -} - -/** - * dn_return_long - Return a long packet to its sender - * @skb: The long format packet to return - * - */ -static int dn_return_long(struct sk_buff *skb) -{ - struct dn_skb_cb *cb; - unsigned char *ptr; - unsigned char *src_addr, *dst_addr; - unsigned char tmp[ETH_ALEN]; - - /* Add back all headers */ - skb_push(skb, skb->data - skb_network_header(skb)); - - skb = skb_unshare(skb, GFP_ATOMIC); - if (!skb) - return NET_RX_DROP; - - cb = DN_SKB_CB(skb); - /* Ignore packet length and point to flags */ - ptr = skb->data + 2; - - /* Skip padding */ - if (*ptr & DN_RT_F_PF) { - char padlen = (*ptr & ~DN_RT_F_PF); - ptr += padlen; - } - - *ptr++ = (cb->rt_flags & ~DN_RT_F_RQR) | DN_RT_F_RTS; - ptr += 2; - dst_addr = ptr; - ptr += 8; - src_addr = ptr; - ptr += 6; - *ptr = 0; /* Zero hop count */ - - /* Swap source and destination */ - memcpy(tmp, src_addr, ETH_ALEN); - memcpy(src_addr, dst_addr, ETH_ALEN); - memcpy(dst_addr, tmp, ETH_ALEN); - - skb->pkt_type = PACKET_OUTGOING; - dn_rt_finish_output(skb, dst_addr, src_addr); - return NET_RX_SUCCESS; -} - -/** - * dn_route_rx_packet - Try and find a route for an incoming packet - * @net: The applicable net namespace - * @sk: Socket packet transmitted on - * @skb: The packet to find a route for - * - * Returns: result of input function if route is found, error code otherwise - */ -static int dn_route_rx_packet(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct dn_skb_cb *cb; - int err; - - err = dn_route_input(skb); - if (err == 0) - return dst_input(skb); - - cb = DN_SKB_CB(skb); - if (decnet_debug_level & 4) { - char *devname = skb->dev ? skb->dev->name : "???"; - - printk(KERN_DEBUG - "DECnet: dn_route_rx_packet: rt_flags=0x%02x dev=%s len=%d src=0x%04hx dst=0x%04hx err=%d type=%d\n", - (int)cb->rt_flags, devname, skb->len, - le16_to_cpu(cb->src), le16_to_cpu(cb->dst), - err, skb->pkt_type); - } - - if ((skb->pkt_type == PACKET_HOST) && (cb->rt_flags & DN_RT_F_RQR)) { - switch (cb->rt_flags & DN_RT_PKT_MSK) { - case DN_RT_PKT_SHORT: - return dn_return_short(skb); - case DN_RT_PKT_LONG: - return dn_return_long(skb); - } - } - - kfree_skb(skb); - return NET_RX_DROP; -} - -static int dn_route_rx_long(struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - unsigned char *ptr = skb->data; - - if (!pskb_may_pull(skb, 21)) /* 20 for long header, 1 for shortest nsp */ - goto drop_it; - - skb_pull(skb, 20); - skb_reset_transport_header(skb); - - /* Destination info */ - ptr += 2; - cb->dst = dn_eth2dn(ptr); - if (memcmp(ptr, dn_hiord_addr, 4) != 0) - goto drop_it; - ptr += 6; - - - /* Source info */ - ptr += 2; - cb->src = dn_eth2dn(ptr); - if (memcmp(ptr, dn_hiord_addr, 4) != 0) - goto drop_it; - ptr += 6; - /* Other junk */ - ptr++; - cb->hops = *ptr++; /* Visit Count */ - - return NF_HOOK(NFPROTO_DECNET, NF_DN_PRE_ROUTING, - &init_net, NULL, skb, skb->dev, NULL, - dn_route_rx_packet); - -drop_it: - kfree_skb(skb); - return NET_RX_DROP; -} - - - -static int dn_route_rx_short(struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - unsigned char *ptr = skb->data; - - if (!pskb_may_pull(skb, 6)) /* 5 for short header + 1 for shortest nsp */ - goto drop_it; - - skb_pull(skb, 5); - skb_reset_transport_header(skb); - - cb->dst = *(__le16 *)ptr; - ptr += 2; - cb->src = *(__le16 *)ptr; - ptr += 2; - cb->hops = *ptr & 0x3f; - - return NF_HOOK(NFPROTO_DECNET, NF_DN_PRE_ROUTING, - &init_net, NULL, skb, skb->dev, NULL, - dn_route_rx_packet); - -drop_it: - kfree_skb(skb); - return NET_RX_DROP; -} - -static int dn_route_discard(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - /* - * I know we drop the packet here, but that's considered success in - * this case - */ - kfree_skb(skb); - return NET_RX_SUCCESS; -} - -static int dn_route_ptp_hello(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - dn_dev_hello(skb); - dn_neigh_pointopoint_hello(skb); - return NET_RX_SUCCESS; -} - -int dn_route_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) -{ - struct dn_skb_cb *cb; - unsigned char flags = 0; - __u16 len = le16_to_cpu(*(__le16 *)skb->data); - struct dn_dev *dn = rcu_dereference(dev->dn_ptr); - unsigned char padlen = 0; - - if (!net_eq(dev_net(dev), &init_net)) - goto dump_it; - - if (dn == NULL) - goto dump_it; - - skb = skb_share_check(skb, GFP_ATOMIC); - if (!skb) - goto out; - - if (!pskb_may_pull(skb, 3)) - goto dump_it; - - skb_pull(skb, 2); - - if (len > skb->len) - goto dump_it; - - skb_trim(skb, len); - - flags = *skb->data; - - cb = DN_SKB_CB(skb); - cb->stamp = jiffies; - cb->iif = dev->ifindex; - - /* - * If we have padding, remove it. - */ - if (flags & DN_RT_F_PF) { - padlen = flags & ~DN_RT_F_PF; - if (!pskb_may_pull(skb, padlen + 1)) - goto dump_it; - skb_pull(skb, padlen); - flags = *skb->data; - } - - skb_reset_network_header(skb); - - /* - * Weed out future version DECnet - */ - if (flags & DN_RT_F_VER) - goto dump_it; - - cb->rt_flags = flags; - - if (decnet_debug_level & 1) - printk(KERN_DEBUG - "dn_route_rcv: got 0x%02x from %s [%d %d %d]\n", - (int)flags, dev->name, len, skb->len, - padlen); - - if (flags & DN_RT_PKT_CNTL) { - if (unlikely(skb_linearize(skb))) - goto dump_it; - - switch (flags & DN_RT_CNTL_MSK) { - case DN_RT_PKT_INIT: - dn_dev_init_pkt(skb); - break; - case DN_RT_PKT_VERI: - dn_dev_veri_pkt(skb); - break; - } - - if (dn->parms.state != DN_DEV_S_RU) - goto dump_it; - - switch (flags & DN_RT_CNTL_MSK) { - case DN_RT_PKT_HELO: - return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO, - &init_net, NULL, skb, skb->dev, NULL, - dn_route_ptp_hello); - - case DN_RT_PKT_L1RT: - case DN_RT_PKT_L2RT: - return NF_HOOK(NFPROTO_DECNET, NF_DN_ROUTE, - &init_net, NULL, skb, skb->dev, NULL, - dn_route_discard); - case DN_RT_PKT_ERTH: - return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO, - &init_net, NULL, skb, skb->dev, NULL, - dn_neigh_router_hello); - - case DN_RT_PKT_EEDH: - return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO, - &init_net, NULL, skb, skb->dev, NULL, - dn_neigh_endnode_hello); - } - } else { - if (dn->parms.state != DN_DEV_S_RU) - goto dump_it; - - skb_pull(skb, 1); /* Pull flags */ - - switch (flags & DN_RT_PKT_MSK) { - case DN_RT_PKT_LONG: - return dn_route_rx_long(skb); - case DN_RT_PKT_SHORT: - return dn_route_rx_short(skb); - } - } - -dump_it: - kfree_skb(skb); -out: - return NET_RX_DROP; -} - -static int dn_output(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct dst_entry *dst = skb_dst(skb); - struct dn_route *rt = (struct dn_route *)dst; - struct net_device *dev = dst->dev; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - int err = -EINVAL; - - if (rt->n == NULL) - goto error; - - skb->dev = dev; - - cb->src = rt->rt_saddr; - cb->dst = rt->rt_daddr; - - /* - * Always set the Intra-Ethernet bit on all outgoing packets - * originated on this node. Only valid flag from upper layers - * is return-to-sender-requested. Set hop count to 0 too. - */ - cb->rt_flags &= ~DN_RT_F_RQR; - cb->rt_flags |= DN_RT_F_IE; - cb->hops = 0; - - return NF_HOOK(NFPROTO_DECNET, NF_DN_LOCAL_OUT, - &init_net, sk, skb, NULL, dev, - dn_to_neigh_output); - -error: - net_dbg_ratelimited("dn_output: This should not happen\n"); - - kfree_skb(skb); - - return err; -} - -static int dn_forward(struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct dst_entry *dst = skb_dst(skb); - struct dn_dev *dn_db = rcu_dereference(dst->dev->dn_ptr); - struct dn_route *rt; - int header_len; - struct net_device *dev = skb->dev; - - if (skb->pkt_type != PACKET_HOST) - goto drop; - - /* Ensure that we have enough space for headers */ - rt = (struct dn_route *)skb_dst(skb); - header_len = dn_db->use_long ? 21 : 6; - if (skb_cow(skb, LL_RESERVED_SPACE(rt->dst.dev)+header_len)) - goto drop; - - /* - * Hop count exceeded. - */ - if (++cb->hops > 30) - goto drop; - - skb->dev = rt->dst.dev; - - /* - * If packet goes out same interface it came in on, then set - * the Intra-Ethernet bit. This has no effect for short - * packets, so we don't need to test for them here. - */ - cb->rt_flags &= ~DN_RT_F_IE; - if (rt->rt_flags & RTCF_DOREDIRECT) - cb->rt_flags |= DN_RT_F_IE; - - return NF_HOOK(NFPROTO_DECNET, NF_DN_FORWARD, - &init_net, NULL, skb, dev, skb->dev, - dn_to_neigh_output); - -drop: - kfree_skb(skb); - return NET_RX_DROP; -} - -/* - * Used to catch bugs. This should never normally get - * called. - */ -static int dn_rt_bug_out(struct net *net, struct sock *sk, struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - net_dbg_ratelimited("dn_rt_bug: skb from:%04x to:%04x\n", - le16_to_cpu(cb->src), le16_to_cpu(cb->dst)); - - kfree_skb(skb); - - return NET_RX_DROP; -} - -static int dn_rt_bug(struct sk_buff *skb) -{ - struct dn_skb_cb *cb = DN_SKB_CB(skb); - - net_dbg_ratelimited("dn_rt_bug: skb from:%04x to:%04x\n", - le16_to_cpu(cb->src), le16_to_cpu(cb->dst)); - - kfree_skb(skb); - - return NET_RX_DROP; -} - -static unsigned int dn_dst_default_advmss(const struct dst_entry *dst) -{ - return dn_mss_from_pmtu(dst->dev, dst_mtu(dst)); -} - -static unsigned int dn_dst_mtu(const struct dst_entry *dst) -{ - unsigned int mtu = dst_metric_raw(dst, RTAX_MTU); - - return mtu ? : dst->dev->mtu; -} - -static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst, - struct sk_buff *skb, - const void *daddr) -{ - return __neigh_lookup_errno(&dn_neigh_table, daddr, dst->dev); -} - -static int dn_rt_set_next_hop(struct dn_route *rt, struct dn_fib_res *res) -{ - struct dn_fib_info *fi = res->fi; - struct net_device *dev = rt->dst.dev; - unsigned int mss_metric; - struct neighbour *n; - - if (fi) { - if (DN_FIB_RES_GW(*res) && - DN_FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK) - rt->rt_gateway = DN_FIB_RES_GW(*res); - dst_init_metrics(&rt->dst, fi->fib_metrics, true); - } - rt->rt_type = res->type; - - if (dev != NULL && rt->n == NULL) { - n = __neigh_lookup_errno(&dn_neigh_table, &rt->rt_gateway, dev); - if (IS_ERR(n)) - return PTR_ERR(n); - rt->n = n; - } - - if (dst_metric(&rt->dst, RTAX_MTU) > rt->dst.dev->mtu) - dst_metric_set(&rt->dst, RTAX_MTU, rt->dst.dev->mtu); - mss_metric = dst_metric_raw(&rt->dst, RTAX_ADVMSS); - if (mss_metric) { - unsigned int mss = dn_mss_from_pmtu(dev, dst_mtu(&rt->dst)); - if (mss_metric > mss) - dst_metric_set(&rt->dst, RTAX_ADVMSS, mss); - } - return 0; -} - -static inline int dn_match_addr(__le16 addr1, __le16 addr2) -{ - __u16 tmp = le16_to_cpu(addr1) ^ le16_to_cpu(addr2); - int match = 16; - while (tmp) { - tmp >>= 1; - match--; - } - return match; -} - -static __le16 dnet_select_source(const struct net_device *dev, __le16 daddr, int scope) -{ - __le16 saddr = 0; - struct dn_dev *dn_db; - struct dn_ifaddr *ifa; - int best_match = 0; - int ret; - - rcu_read_lock(); - dn_db = rcu_dereference(dev->dn_ptr); - for (ifa = rcu_dereference(dn_db->ifa_list); - ifa != NULL; - ifa = rcu_dereference(ifa->ifa_next)) { - if (ifa->ifa_scope > scope) - continue; - if (!daddr) { - saddr = ifa->ifa_local; - break; - } - ret = dn_match_addr(daddr, ifa->ifa_local); - if (ret > best_match) - saddr = ifa->ifa_local; - if (best_match == 0) - saddr = ifa->ifa_local; - } - rcu_read_unlock(); - - return saddr; -} - -static inline __le16 __dn_fib_res_prefsrc(struct dn_fib_res *res) -{ - return dnet_select_source(DN_FIB_RES_DEV(*res), DN_FIB_RES_GW(*res), res->scope); -} - -static inline __le16 dn_fib_rules_map_destination(__le16 daddr, struct dn_fib_res *res) -{ - __le16 mask = dnet_make_mask(res->prefixlen); - return (daddr&~mask)|res->fi->fib_nh->nh_gw; -} - -static int dn_route_output_slow(struct dst_entry **pprt, const struct flowidn *oldflp, int try_hard) -{ - struct flowidn fld = { - .daddr = oldflp->daddr, - .saddr = oldflp->saddr, - .flowidn_scope = RT_SCOPE_UNIVERSE, - .flowidn_mark = oldflp->flowidn_mark, - .flowidn_iif = LOOPBACK_IFINDEX, - .flowidn_oif = oldflp->flowidn_oif, - }; - struct dn_route *rt = NULL; - struct net_device *dev_out = NULL, *dev; - struct neighbour *neigh = NULL; - unsigned int hash; - unsigned int flags = 0; - struct dn_fib_res res = { .fi = NULL, .type = RTN_UNICAST }; - int err; - int free_res = 0; - __le16 gateway = 0; - - if (decnet_debug_level & 16) - printk(KERN_DEBUG - "dn_route_output_slow: dst=%04x src=%04x mark=%d" - " iif=%d oif=%d\n", le16_to_cpu(oldflp->daddr), - le16_to_cpu(oldflp->saddr), - oldflp->flowidn_mark, LOOPBACK_IFINDEX, - oldflp->flowidn_oif); - - /* If we have an output interface, verify its a DECnet device */ - if (oldflp->flowidn_oif) { - dev_out = dev_get_by_index(&init_net, oldflp->flowidn_oif); - err = -ENODEV; - if (dev_out && dev_out->dn_ptr == NULL) { - dev_put(dev_out); - dev_out = NULL; - } - if (dev_out == NULL) - goto out; - } - - /* If we have a source address, verify that its a local address */ - if (oldflp->saddr) { - err = -EADDRNOTAVAIL; - - if (dev_out) { - if (dn_dev_islocal(dev_out, oldflp->saddr)) - goto source_ok; - dev_put(dev_out); - goto out; - } - rcu_read_lock(); - for_each_netdev_rcu(&init_net, dev) { - if (!dev->dn_ptr) - continue; - if (!dn_dev_islocal(dev, oldflp->saddr)) - continue; - if ((dev->flags & IFF_LOOPBACK) && - oldflp->daddr && - !dn_dev_islocal(dev, oldflp->daddr)) - continue; - - dev_out = dev; - break; - } - rcu_read_unlock(); - if (dev_out == NULL) - goto out; - dev_hold(dev_out); -source_ok: - ; - } - - /* No destination? Assume its local */ - if (!fld.daddr) { - fld.daddr = fld.saddr; - - dev_put(dev_out); - err = -EINVAL; - dev_out = init_net.loopback_dev; - if (!dev_out->dn_ptr) - goto out; - err = -EADDRNOTAVAIL; - dev_hold(dev_out); - if (!fld.daddr) { - fld.daddr = - fld.saddr = dnet_select_source(dev_out, 0, - RT_SCOPE_HOST); - if (!fld.daddr) - goto done; - } - fld.flowidn_oif = LOOPBACK_IFINDEX; - res.type = RTN_LOCAL; - goto make_route; - } - - if (decnet_debug_level & 16) - printk(KERN_DEBUG - "dn_route_output_slow: initial checks complete." - " dst=%04x src=%04x oif=%d try_hard=%d\n", - le16_to_cpu(fld.daddr), le16_to_cpu(fld.saddr), - fld.flowidn_oif, try_hard); - - /* - * N.B. If the kernel is compiled without router support then - * dn_fib_lookup() will evaluate to non-zero so this if () block - * will always be executed. - */ - err = -ESRCH; - if (try_hard || (err = dn_fib_lookup(&fld, &res)) != 0) { - struct dn_dev *dn_db; - if (err != -ESRCH) - goto out; - /* - * Here the fallback is basically the standard algorithm for - * routing in endnodes which is described in the DECnet routing - * docs - * - * If we are not trying hard, look in neighbour cache. - * The result is tested to ensure that if a specific output - * device/source address was requested, then we honour that - * here - */ - if (!try_hard) { - neigh = neigh_lookup_nodev(&dn_neigh_table, &init_net, &fld.daddr); - if (neigh) { - if ((oldflp->flowidn_oif && - (neigh->dev->ifindex != oldflp->flowidn_oif)) || - (oldflp->saddr && - (!dn_dev_islocal(neigh->dev, - oldflp->saddr)))) { - neigh_release(neigh); - neigh = NULL; - } else { - dev_put(dev_out); - if (dn_dev_islocal(neigh->dev, fld.daddr)) { - dev_out = init_net.loopback_dev; - res.type = RTN_LOCAL; - } else { - dev_out = neigh->dev; - } - dev_hold(dev_out); - goto select_source; - } - } - } - - /* Not there? Perhaps its a local address */ - if (dev_out == NULL) - dev_out = dn_dev_get_default(); - err = -ENODEV; - if (dev_out == NULL) - goto out; - dn_db = rcu_dereference_raw(dev_out->dn_ptr); - if (!dn_db) - goto e_inval; - /* Possible improvement - check all devices for local addr */ - if (dn_dev_islocal(dev_out, fld.daddr)) { - dev_put(dev_out); - dev_out = init_net.loopback_dev; - dev_hold(dev_out); - res.type = RTN_LOCAL; - goto select_source; - } - /* Not local either.... try sending it to the default router */ - neigh = neigh_clone(dn_db->router); - BUG_ON(neigh && neigh->dev != dev_out); - - /* Ok then, we assume its directly connected and move on */ -select_source: - if (neigh) - gateway = container_of(neigh, struct dn_neigh, n)->addr; - if (gateway == 0) - gateway = fld.daddr; - if (fld.saddr == 0) { - fld.saddr = dnet_select_source(dev_out, gateway, - res.type == RTN_LOCAL ? - RT_SCOPE_HOST : - RT_SCOPE_LINK); - if (fld.saddr == 0 && res.type != RTN_LOCAL) - goto e_addr; - } - fld.flowidn_oif = dev_out->ifindex; - goto make_route; - } - free_res = 1; - - if (res.type == RTN_NAT) - goto e_inval; - - if (res.type == RTN_LOCAL) { - if (!fld.saddr) - fld.saddr = fld.daddr; - dev_put(dev_out); - dev_out = init_net.loopback_dev; - dev_hold(dev_out); - if (!dev_out->dn_ptr) - goto e_inval; - fld.flowidn_oif = dev_out->ifindex; - if (res.fi) - dn_fib_info_put(res.fi); - res.fi = NULL; - goto make_route; - } - - if (res.fi->fib_nhs > 1 && fld.flowidn_oif == 0) - dn_fib_select_multipath(&fld, &res); - - /* - * We could add some logic to deal with default routes here and - * get rid of some of the special casing above. - */ - - if (!fld.saddr) - fld.saddr = DN_FIB_RES_PREFSRC(res); - - dev_put(dev_out); - dev_out = DN_FIB_RES_DEV(res); - dev_hold(dev_out); - fld.flowidn_oif = dev_out->ifindex; - gateway = DN_FIB_RES_GW(res); - -make_route: - if (dev_out->flags & IFF_LOOPBACK) - flags |= RTCF_LOCAL; - - rt = dst_alloc(&dn_dst_ops, dev_out, 0, DST_OBSOLETE_NONE, 0); - if (rt == NULL) - goto e_nobufs; - - rt->dn_next = NULL; - memset(&rt->fld, 0, sizeof(rt->fld)); - rt->fld.saddr = oldflp->saddr; - rt->fld.daddr = oldflp->daddr; - rt->fld.flowidn_oif = oldflp->flowidn_oif; - rt->fld.flowidn_iif = 0; - rt->fld.flowidn_mark = oldflp->flowidn_mark; - - rt->rt_saddr = fld.saddr; - rt->rt_daddr = fld.daddr; - rt->rt_gateway = gateway ? gateway : fld.daddr; - rt->rt_local_src = fld.saddr; - - rt->rt_dst_map = fld.daddr; - rt->rt_src_map = fld.saddr; - - rt->n = neigh; - neigh = NULL; - - rt->dst.lastuse = jiffies; - rt->dst.output = dn_output; - rt->dst.input = dn_rt_bug; - rt->rt_flags = flags; - if (flags & RTCF_LOCAL) - rt->dst.input = dn_nsp_rx; - - err = dn_rt_set_next_hop(rt, &res); - if (err) - goto e_neighbour; - - hash = dn_hash(rt->fld.saddr, rt->fld.daddr); - /* dn_insert_route() increments dst->__refcnt */ - dn_insert_route(rt, hash, (struct dn_route **)pprt); - -done: - if (neigh) - neigh_release(neigh); - if (free_res) - dn_fib_res_put(&res); - dev_put(dev_out); -out: - return err; - -e_addr: - err = -EADDRNOTAVAIL; - goto done; -e_inval: - err = -EINVAL; - goto done; -e_nobufs: - err = -ENOBUFS; - goto done; -e_neighbour: - dst_release_immediate(&rt->dst); - goto e_nobufs; -} - - -/* - * N.B. The flags may be moved into the flowi at some future stage. - */ -static int __dn_route_output_key(struct dst_entry **pprt, const struct flowidn *flp, int flags) -{ - unsigned int hash = dn_hash(flp->saddr, flp->daddr); - struct dn_route *rt = NULL; - - if (!(flags & MSG_TRYHARD)) { - rcu_read_lock_bh(); - for (rt = rcu_dereference_bh(dn_rt_hash_table[hash].chain); rt; - rt = rcu_dereference_bh(rt->dn_next)) { - if ((flp->daddr == rt->fld.daddr) && - (flp->saddr == rt->fld.saddr) && - (flp->flowidn_mark == rt->fld.flowidn_mark) && - dn_is_output_route(rt) && - (rt->fld.flowidn_oif == flp->flowidn_oif)) { - dst_hold_and_use(&rt->dst, jiffies); - rcu_read_unlock_bh(); - *pprt = &rt->dst; - return 0; - } - } - rcu_read_unlock_bh(); - } - - return dn_route_output_slow(pprt, flp, flags); -} - -static int dn_route_output_key(struct dst_entry **pprt, struct flowidn *flp, int flags) -{ - int err; - - err = __dn_route_output_key(pprt, flp, flags); - if (err == 0 && flp->flowidn_proto) { - *pprt = xfrm_lookup(&init_net, *pprt, - flowidn_to_flowi(flp), NULL, 0); - if (IS_ERR(*pprt)) { - err = PTR_ERR(*pprt); - *pprt = NULL; - } - } - return err; -} - -int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *fl, struct sock *sk, int flags) -{ - int err; - - err = __dn_route_output_key(pprt, fl, flags & MSG_TRYHARD); - if (err == 0 && fl->flowidn_proto) { - *pprt = xfrm_lookup(&init_net, *pprt, - flowidn_to_flowi(fl), sk, 0); - if (IS_ERR(*pprt)) { - err = PTR_ERR(*pprt); - *pprt = NULL; - } - } - return err; -} - -static int dn_route_input_slow(struct sk_buff *skb) -{ - struct dn_route *rt = NULL; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - struct net_device *in_dev = skb->dev; - struct net_device *out_dev = NULL; - struct dn_dev *dn_db; - struct neighbour *neigh = NULL; - unsigned int hash; - int flags = 0; - __le16 gateway = 0; - __le16 local_src = 0; - struct flowidn fld = { - .daddr = cb->dst, - .saddr = cb->src, - .flowidn_scope = RT_SCOPE_UNIVERSE, - .flowidn_mark = skb->mark, - .flowidn_iif = skb->dev->ifindex, - }; - struct dn_fib_res res = { .fi = NULL, .type = RTN_UNREACHABLE }; - int err = -EINVAL; - int free_res = 0; - - dev_hold(in_dev); - - dn_db = rcu_dereference(in_dev->dn_ptr); - if (!dn_db) - goto out; - - /* Zero source addresses are not allowed */ - if (fld.saddr == 0) - goto out; - - /* - * In this case we've just received a packet from a source - * outside ourselves pretending to come from us. We don't - * allow it any further to prevent routing loops, spoofing and - * other nasties. Loopback packets already have the dst attached - * so this only affects packets which have originated elsewhere. - */ - err = -ENOTUNIQ; - if (dn_dev_islocal(in_dev, cb->src)) - goto out; - - err = dn_fib_lookup(&fld, &res); - if (err) { - if (err != -ESRCH) - goto out; - /* - * Is the destination us ? - */ - if (!dn_dev_islocal(in_dev, cb->dst)) - goto e_inval; - - res.type = RTN_LOCAL; - } else { - __le16 src_map = fld.saddr; - free_res = 1; - - out_dev = DN_FIB_RES_DEV(res); - if (out_dev == NULL) { - net_crit_ratelimited("Bug in dn_route_input_slow() No output device\n"); - goto e_inval; - } - dev_hold(out_dev); - - if (res.r) - src_map = fld.saddr; /* no NAT support for now */ - - gateway = DN_FIB_RES_GW(res); - if (res.type == RTN_NAT) { - fld.daddr = dn_fib_rules_map_destination(fld.daddr, &res); - dn_fib_res_put(&res); - free_res = 0; - if (dn_fib_lookup(&fld, &res)) - goto e_inval; - free_res = 1; - if (res.type != RTN_UNICAST) - goto e_inval; - flags |= RTCF_DNAT; - gateway = fld.daddr; - } - fld.saddr = src_map; - } - - switch (res.type) { - case RTN_UNICAST: - /* - * Forwarding check here, we only check for forwarding - * being turned off, if you want to only forward intra - * area, its up to you to set the routing tables up - * correctly. - */ - if (dn_db->parms.forwarding == 0) - goto e_inval; - - if (res.fi->fib_nhs > 1 && fld.flowidn_oif == 0) - dn_fib_select_multipath(&fld, &res); - - /* - * Check for out_dev == in_dev. We use the RTCF_DOREDIRECT - * flag as a hint to set the intra-ethernet bit when - * forwarding. If we've got NAT in operation, we don't do - * this optimisation. - */ - if (out_dev == in_dev && !(flags & RTCF_NAT)) - flags |= RTCF_DOREDIRECT; - - local_src = DN_FIB_RES_PREFSRC(res); - break; - case RTN_BLACKHOLE: - case RTN_UNREACHABLE: - break; - case RTN_LOCAL: - flags |= RTCF_LOCAL; - fld.saddr = cb->dst; - fld.daddr = cb->src; - - /* Routing tables gave us a gateway */ - if (gateway) - goto make_route; - - /* Packet was intra-ethernet, so we know its on-link */ - if (cb->rt_flags & DN_RT_F_IE) { - gateway = cb->src; - goto make_route; - } - - /* Use the default router if there is one */ - neigh = neigh_clone(dn_db->router); - if (neigh) { - gateway = container_of(neigh, struct dn_neigh, n)->addr; - goto make_route; - } - - /* Close eyes and pray */ - gateway = cb->src; - goto make_route; - default: - goto e_inval; - } - -make_route: - rt = dst_alloc(&dn_dst_ops, out_dev, 1, DST_OBSOLETE_NONE, 0); - if (rt == NULL) - goto e_nobufs; - - rt->dn_next = NULL; - memset(&rt->fld, 0, sizeof(rt->fld)); - rt->rt_saddr = fld.saddr; - rt->rt_daddr = fld.daddr; - rt->rt_gateway = fld.daddr; - if (gateway) - rt->rt_gateway = gateway; - rt->rt_local_src = local_src ? local_src : rt->rt_saddr; - - rt->rt_dst_map = fld.daddr; - rt->rt_src_map = fld.saddr; - - rt->fld.saddr = cb->src; - rt->fld.daddr = cb->dst; - rt->fld.flowidn_oif = 0; - rt->fld.flowidn_iif = in_dev->ifindex; - rt->fld.flowidn_mark = fld.flowidn_mark; - - rt->n = neigh; - rt->dst.lastuse = jiffies; - rt->dst.output = dn_rt_bug_out; - switch (res.type) { - case RTN_UNICAST: - rt->dst.input = dn_forward; - break; - case RTN_LOCAL: - rt->dst.output = dn_output; - rt->dst.input = dn_nsp_rx; - rt->dst.dev = in_dev; - flags |= RTCF_LOCAL; - break; - default: - case RTN_UNREACHABLE: - case RTN_BLACKHOLE: - rt->dst.input = dst_discard; - } - rt->rt_flags = flags; - - err = dn_rt_set_next_hop(rt, &res); - if (err) - goto e_neighbour; - - hash = dn_hash(rt->fld.saddr, rt->fld.daddr); - /* dn_insert_route() increments dst->__refcnt */ - dn_insert_route(rt, hash, &rt); - skb_dst_set(skb, &rt->dst); - -done: - if (neigh) - neigh_release(neigh); - if (free_res) - dn_fib_res_put(&res); - dev_put(in_dev); - dev_put(out_dev); -out: - return err; - -e_inval: - err = -EINVAL; - goto done; - -e_nobufs: - err = -ENOBUFS; - goto done; - -e_neighbour: - dst_release_immediate(&rt->dst); - goto done; -} - -static int dn_route_input(struct sk_buff *skb) -{ - struct dn_route *rt; - struct dn_skb_cb *cb = DN_SKB_CB(skb); - unsigned int hash = dn_hash(cb->src, cb->dst); - - if (skb_dst(skb)) - return 0; - - rcu_read_lock(); - for (rt = rcu_dereference(dn_rt_hash_table[hash].chain); rt != NULL; - rt = rcu_dereference(rt->dn_next)) { - if ((rt->fld.saddr == cb->src) && - (rt->fld.daddr == cb->dst) && - (rt->fld.flowidn_oif == 0) && - (rt->fld.flowidn_mark == skb->mark) && - (rt->fld.flowidn_iif == cb->iif)) { - dst_hold_and_use(&rt->dst, jiffies); - rcu_read_unlock(); - skb_dst_set(skb, (struct dst_entry *)rt); - return 0; - } - } - rcu_read_unlock(); - - return dn_route_input_slow(skb); -} - -static int dn_rt_fill_info(struct sk_buff *skb, u32 portid, u32 seq, - int event, int nowait, unsigned int flags) -{ - struct dn_route *rt = (struct dn_route *)skb_dst(skb); - struct rtmsg *r; - struct nlmsghdr *nlh; - long expires; - - nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags); - if (!nlh) - return -EMSGSIZE; - - r = nlmsg_data(nlh); - r->rtm_family = AF_DECnet; - r->rtm_dst_len = 16; - r->rtm_src_len = 0; - r->rtm_tos = 0; - r->rtm_table = RT_TABLE_MAIN; - r->rtm_type = rt->rt_type; - r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED; - r->rtm_scope = RT_SCOPE_UNIVERSE; - r->rtm_protocol = RTPROT_UNSPEC; - - if (rt->rt_flags & RTCF_NOTIFY) - r->rtm_flags |= RTM_F_NOTIFY; - - if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN) < 0 || - nla_put_le16(skb, RTA_DST, rt->rt_daddr) < 0) - goto errout; - - if (rt->fld.saddr) { - r->rtm_src_len = 16; - if (nla_put_le16(skb, RTA_SRC, rt->fld.saddr) < 0) - goto errout; - } - if (rt->dst.dev && - nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex) < 0) - goto errout; - - /* - * Note to self - change this if input routes reverse direction when - * they deal only with inputs and not with replies like they do - * currently. - */ - if (nla_put_le16(skb, RTA_PREFSRC, rt->rt_local_src) < 0) - goto errout; - - if (rt->rt_daddr != rt->rt_gateway && - nla_put_le16(skb, RTA_GATEWAY, rt->rt_gateway) < 0) - goto errout; - - if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0) - goto errout; - - expires = rt->dst.expires ? rt->dst.expires - jiffies : 0; - if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, - rt->dst.error) < 0) - goto errout; - - if (dn_is_input_route(rt) && - nla_put_u32(skb, RTA_IIF, rt->fld.flowidn_iif) < 0) - goto errout; - - nlmsg_end(skb, nlh); - return 0; - -errout: - nlmsg_cancel(skb, nlh); - return -EMSGSIZE; -} - -const struct nla_policy rtm_dn_policy[RTA_MAX + 1] = { - [RTA_DST] = { .type = NLA_U16 }, - [RTA_SRC] = { .type = NLA_U16 }, - [RTA_IIF] = { .type = NLA_U32 }, - [RTA_OIF] = { .type = NLA_U32 }, - [RTA_GATEWAY] = { .type = NLA_U16 }, - [RTA_PRIORITY] = { .type = NLA_U32 }, - [RTA_PREFSRC] = { .type = NLA_U16 }, - [RTA_METRICS] = { .type = NLA_NESTED }, - [RTA_MULTIPATH] = { .type = NLA_NESTED }, - [RTA_TABLE] = { .type = NLA_U32 }, - [RTA_MARK] = { .type = NLA_U32 }, -}; - -/* - * This is called by both endnodes and routers now. - */ -static int dn_cache_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh, - struct netlink_ext_ack *extack) -{ - struct net *net = sock_net(in_skb->sk); - struct rtmsg *rtm = nlmsg_data(nlh); - struct dn_route *rt = NULL; - struct dn_skb_cb *cb; - int err; - struct sk_buff *skb; - struct flowidn fld; - struct nlattr *tb[RTA_MAX+1]; - - if (!net_eq(net, &init_net)) - return -EINVAL; - - err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX, - rtm_dn_policy, extack); - if (err < 0) - return err; - - memset(&fld, 0, sizeof(fld)); - fld.flowidn_proto = DNPROTO_NSP; - - skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (skb == NULL) - return -ENOBUFS; - skb_reset_mac_header(skb); - cb = DN_SKB_CB(skb); - - if (tb[RTA_SRC]) - fld.saddr = nla_get_le16(tb[RTA_SRC]); - - if (tb[RTA_DST]) - fld.daddr = nla_get_le16(tb[RTA_DST]); - - if (tb[RTA_IIF]) - fld.flowidn_iif = nla_get_u32(tb[RTA_IIF]); - - if (fld.flowidn_iif) { - struct net_device *dev; - dev = __dev_get_by_index(&init_net, fld.flowidn_iif); - if (!dev || !dev->dn_ptr) { - kfree_skb(skb); - return -ENODEV; - } - skb->protocol = htons(ETH_P_DNA_RT); - skb->dev = dev; - cb->src = fld.saddr; - cb->dst = fld.daddr; - local_bh_disable(); - err = dn_route_input(skb); - local_bh_enable(); - memset(cb, 0, sizeof(struct dn_skb_cb)); - rt = (struct dn_route *)skb_dst(skb); - if (!err && -rt->dst.error) - err = rt->dst.error; - } else { - if (tb[RTA_OIF]) - fld.flowidn_oif = nla_get_u32(tb[RTA_OIF]); - - err = dn_route_output_key((struct dst_entry **)&rt, &fld, 0); - } - - skb->dev = NULL; - if (err) - goto out_free; - skb_dst_set(skb, &rt->dst); - if (rtm->rtm_flags & RTM_F_NOTIFY) - rt->rt_flags |= RTCF_NOTIFY; - - err = dn_rt_fill_info(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, RTM_NEWROUTE, 0, 0); - if (err < 0) { - err = -EMSGSIZE; - goto out_free; - } - - return rtnl_unicast(skb, &init_net, NETLINK_CB(in_skb).portid); - -out_free: - kfree_skb(skb); - return err; -} - -/* - * For routers, this is called from dn_fib_dump, but for endnodes its - * called directly from the rtnetlink dispatch table. - */ -int dn_cache_dump(struct sk_buff *skb, struct netlink_callback *cb) -{ - struct net *net = sock_net(skb->sk); - struct dn_route *rt; - int h, s_h; - int idx, s_idx; - struct rtmsg *rtm; - - if (!net_eq(net, &init_net)) - return 0; - - if (nlmsg_len(cb->nlh) < sizeof(struct rtmsg)) - return -EINVAL; - - rtm = nlmsg_data(cb->nlh); - if (!(rtm->rtm_flags & RTM_F_CLONED)) - return 0; - - s_h = cb->args[0]; - s_idx = idx = cb->args[1]; - for (h = 0; h <= dn_rt_hash_mask; h++) { - if (h < s_h) - continue; - if (h > s_h) - s_idx = 0; - rcu_read_lock_bh(); - for (rt = rcu_dereference_bh(dn_rt_hash_table[h].chain), idx = 0; - rt; - rt = rcu_dereference_bh(rt->dn_next), idx++) { - if (idx < s_idx) - continue; - skb_dst_set(skb, dst_clone(&rt->dst)); - if (dn_rt_fill_info(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, RTM_NEWROUTE, - 1, NLM_F_MULTI) < 0) { - skb_dst_drop(skb); - rcu_read_unlock_bh(); - goto done; - } - skb_dst_drop(skb); - } - rcu_read_unlock_bh(); - } - -done: - cb->args[0] = h; - cb->args[1] = idx; - return skb->len; -} - -#ifdef CONFIG_PROC_FS -struct dn_rt_cache_iter_state { - int bucket; -}; - -static struct dn_route *dn_rt_cache_get_first(struct seq_file *seq) -{ - struct dn_route *rt = NULL; - struct dn_rt_cache_iter_state *s = seq->private; - - for (s->bucket = dn_rt_hash_mask; s->bucket >= 0; --s->bucket) { - rcu_read_lock_bh(); - rt = rcu_dereference_bh(dn_rt_hash_table[s->bucket].chain); - if (rt) - break; - rcu_read_unlock_bh(); - } - return rt; -} - -static struct dn_route *dn_rt_cache_get_next(struct seq_file *seq, struct dn_route *rt) -{ - struct dn_rt_cache_iter_state *s = seq->private; - - rt = rcu_dereference_bh(rt->dn_next); - while (!rt) { - rcu_read_unlock_bh(); - if (--s->bucket < 0) - break; - rcu_read_lock_bh(); - rt = rcu_dereference_bh(dn_rt_hash_table[s->bucket].chain); - } - return rt; -} - -static void *dn_rt_cache_seq_start(struct seq_file *seq, loff_t *pos) -{ - struct dn_route *rt = dn_rt_cache_get_first(seq); - - if (rt) { - while (*pos && (rt = dn_rt_cache_get_next(seq, rt))) - --*pos; - } - return *pos ? NULL : rt; -} - -static void *dn_rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos) -{ - struct dn_route *rt = dn_rt_cache_get_next(seq, v); - ++*pos; - return rt; -} - -static void dn_rt_cache_seq_stop(struct seq_file *seq, void *v) -{ - if (v) - rcu_read_unlock_bh(); -} - -static int dn_rt_cache_seq_show(struct seq_file *seq, void *v) -{ - struct dn_route *rt = v; - char buf1[DN_ASCBUF_LEN], buf2[DN_ASCBUF_LEN]; - - seq_printf(seq, "%-8s %-7s %-7s %04d %04d %04d\n", - rt->dst.dev ? rt->dst.dev->name : "*", - dn_addr2asc(le16_to_cpu(rt->rt_daddr), buf1), - dn_addr2asc(le16_to_cpu(rt->rt_saddr), buf2), - atomic_read(&rt->dst.__refcnt), - rt->dst.__use, 0); - return 0; -} - -static const struct seq_operations dn_rt_cache_seq_ops = { - .start = dn_rt_cache_seq_start, - .next = dn_rt_cache_seq_next, - .stop = dn_rt_cache_seq_stop, - .show = dn_rt_cache_seq_show, -}; -#endif /* CONFIG_PROC_FS */ - -void __init dn_route_init(void) -{ - int i, goal, order; - - dn_dst_ops.kmem_cachep = - kmem_cache_create("dn_dst_cache", sizeof(struct dn_route), 0, - SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); - dst_entries_init(&dn_dst_ops); - timer_setup(&dn_route_timer, dn_dst_check_expire, 0); - dn_route_timer.expires = jiffies + decnet_dst_gc_interval * HZ; - add_timer(&dn_route_timer); - - goal = totalram_pages() >> (26 - PAGE_SHIFT); - - for (order = 0; (1UL << order) < goal; order++) - /* NOTHING */; - - /* - * Only want 1024 entries max, since the table is very, very unlikely - * to be larger than that. - */ - while (order && ((((1UL << order) * PAGE_SIZE) / - sizeof(struct dn_rt_hash_bucket)) >= 2048)) - order--; - - do { - dn_rt_hash_mask = (1UL << order) * PAGE_SIZE / - sizeof(struct dn_rt_hash_bucket); - while (dn_rt_hash_mask & (dn_rt_hash_mask - 1)) - dn_rt_hash_mask--; - dn_rt_hash_table = (struct dn_rt_hash_bucket *) - __get_free_pages(GFP_ATOMIC, order); - } while (dn_rt_hash_table == NULL && --order > 0); - - if (!dn_rt_hash_table) - panic("Failed to allocate DECnet route cache hash table\n"); - - printk(KERN_INFO - "DECnet: Routing cache hash table of %u buckets, %ldKbytes\n", - dn_rt_hash_mask, - (long)(dn_rt_hash_mask*sizeof(struct dn_rt_hash_bucket))/1024); - - dn_rt_hash_mask--; - for (i = 0; i <= dn_rt_hash_mask; i++) { - spin_lock_init(&dn_rt_hash_table[i].lock); - dn_rt_hash_table[i].chain = NULL; - } - - dn_dst_ops.gc_thresh = (dn_rt_hash_mask + 1); - - proc_create_seq_private("decnet_cache", 0444, init_net.proc_net, - &dn_rt_cache_seq_ops, - sizeof(struct dn_rt_cache_iter_state), NULL); - -#ifdef CONFIG_DECNET_ROUTER - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETROUTE, - dn_cache_getroute, dn_fib_dump, 0); -#else - rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETROUTE, - dn_cache_getroute, dn_cache_dump, 0); -#endif -} - -void __exit dn_route_cleanup(void) -{ - del_timer(&dn_route_timer); - dn_run_flush(NULL); - - remove_proc_entry("decnet_cache", init_net.proc_net); - dst_entries_destroy(&dn_dst_ops); -} diff --git a/net/decnet/dn_rules.c b/net/decnet/dn_rules.c deleted file mode 100644 index ee73057529cf9bb342529492f07fd0ed341837f2..0000000000000000000000000000000000000000 --- a/net/decnet/dn_rules.c +++ /dev/null @@ -1,253 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Routing Forwarding Information Base (Rules) - * - * Author: Steve Whitehouse - * Mostly copied from Alexey Kuznetsov's ipv4/fib_rules.c - * - * - * Changes: - * Steve Whitehouse - * Updated for Thomas Graf's generic rules - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static struct fib_rules_ops *dn_fib_rules_ops; - -struct dn_fib_rule -{ - struct fib_rule common; - unsigned char dst_len; - unsigned char src_len; - __le16 src; - __le16 srcmask; - __le16 dst; - __le16 dstmask; - __le16 srcmap; - u8 flags; -}; - - -int dn_fib_lookup(struct flowidn *flp, struct dn_fib_res *res) -{ - struct fib_lookup_arg arg = { - .result = res, - }; - int err; - - err = fib_rules_lookup(dn_fib_rules_ops, - flowidn_to_flowi(flp), 0, &arg); - res->r = arg.rule; - - return err; -} - -static int dn_fib_rule_action(struct fib_rule *rule, struct flowi *flp, - int flags, struct fib_lookup_arg *arg) -{ - struct flowidn *fld = &flp->u.dn; - int err = -EAGAIN; - struct dn_fib_table *tbl; - - switch(rule->action) { - case FR_ACT_TO_TBL: - break; - - case FR_ACT_UNREACHABLE: - err = -ENETUNREACH; - goto errout; - - case FR_ACT_PROHIBIT: - err = -EACCES; - goto errout; - - case FR_ACT_BLACKHOLE: - default: - err = -EINVAL; - goto errout; - } - - tbl = dn_fib_get_table(rule->table, 0); - if (tbl == NULL) - goto errout; - - err = tbl->lookup(tbl, fld, (struct dn_fib_res *)arg->result); - if (err > 0) - err = -EAGAIN; -errout: - return err; -} - -static int dn_fib_rule_match(struct fib_rule *rule, struct flowi *fl, int flags) -{ - struct dn_fib_rule *r = (struct dn_fib_rule *)rule; - struct flowidn *fld = &fl->u.dn; - __le16 daddr = fld->daddr; - __le16 saddr = fld->saddr; - - if (((saddr ^ r->src) & r->srcmask) || - ((daddr ^ r->dst) & r->dstmask)) - return 0; - - return 1; -} - -static int dn_fib_rule_configure(struct fib_rule *rule, struct sk_buff *skb, - struct fib_rule_hdr *frh, - struct nlattr **tb, - struct netlink_ext_ack *extack) -{ - int err = -EINVAL; - struct dn_fib_rule *r = (struct dn_fib_rule *)rule; - - if (frh->tos) { - NL_SET_ERR_MSG(extack, "Invalid tos value"); - goto errout; - } - - if (rule->table == RT_TABLE_UNSPEC) { - if (rule->action == FR_ACT_TO_TBL) { - struct dn_fib_table *table; - - table = dn_fib_empty_table(); - if (table == NULL) { - err = -ENOBUFS; - goto errout; - } - - rule->table = table->n; - } - } - - if (frh->src_len) - r->src = nla_get_le16(tb[FRA_SRC]); - - if (frh->dst_len) - r->dst = nla_get_le16(tb[FRA_DST]); - - r->src_len = frh->src_len; - r->srcmask = dnet_make_mask(r->src_len); - r->dst_len = frh->dst_len; - r->dstmask = dnet_make_mask(r->dst_len); - err = 0; -errout: - return err; -} - -static int dn_fib_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh, - struct nlattr **tb) -{ - struct dn_fib_rule *r = (struct dn_fib_rule *)rule; - - if (frh->src_len && (r->src_len != frh->src_len)) - return 0; - - if (frh->dst_len && (r->dst_len != frh->dst_len)) - return 0; - - if (frh->src_len && (r->src != nla_get_le16(tb[FRA_SRC]))) - return 0; - - if (frh->dst_len && (r->dst != nla_get_le16(tb[FRA_DST]))) - return 0; - - return 1; -} - -unsigned int dnet_addr_type(__le16 addr) -{ - struct flowidn fld = { .daddr = addr }; - struct dn_fib_res res; - unsigned int ret = RTN_UNICAST; - struct dn_fib_table *tb = dn_fib_get_table(RT_TABLE_LOCAL, 0); - - res.r = NULL; - - if (tb) { - if (!tb->lookup(tb, &fld, &res)) { - ret = res.type; - dn_fib_res_put(&res); - } - } - return ret; -} - -static int dn_fib_rule_fill(struct fib_rule *rule, struct sk_buff *skb, - struct fib_rule_hdr *frh) -{ - struct dn_fib_rule *r = (struct dn_fib_rule *)rule; - - frh->dst_len = r->dst_len; - frh->src_len = r->src_len; - frh->tos = 0; - - if ((r->dst_len && - nla_put_le16(skb, FRA_DST, r->dst)) || - (r->src_len && - nla_put_le16(skb, FRA_SRC, r->src))) - goto nla_put_failure; - return 0; - -nla_put_failure: - return -ENOBUFS; -} - -static void dn_fib_rule_flush_cache(struct fib_rules_ops *ops) -{ - dn_rt_cache_flush(-1); -} - -static const struct fib_rules_ops __net_initconst dn_fib_rules_ops_template = { - .family = AF_DECnet, - .rule_size = sizeof(struct dn_fib_rule), - .addr_size = sizeof(u16), - .action = dn_fib_rule_action, - .match = dn_fib_rule_match, - .configure = dn_fib_rule_configure, - .compare = dn_fib_rule_compare, - .fill = dn_fib_rule_fill, - .flush_cache = dn_fib_rule_flush_cache, - .nlgroup = RTNLGRP_DECnet_RULE, - .owner = THIS_MODULE, - .fro_net = &init_net, -}; - -void __init dn_fib_rules_init(void) -{ - dn_fib_rules_ops = - fib_rules_register(&dn_fib_rules_ops_template, &init_net); - BUG_ON(IS_ERR(dn_fib_rules_ops)); - BUG_ON(fib_default_rule_add(dn_fib_rules_ops, 0x7fff, - RT_TABLE_MAIN, 0)); -} - -void __exit dn_fib_rules_cleanup(void) -{ - rtnl_lock(); - fib_rules_unregister(dn_fib_rules_ops); - rtnl_unlock(); - rcu_barrier(); -} diff --git a/net/decnet/dn_table.c b/net/decnet/dn_table.c deleted file mode 100644 index 4086f9c746af4eb9ce6c42bcf3be2bdfaa2317ee..0000000000000000000000000000000000000000 --- a/net/decnet/dn_table.c +++ /dev/null @@ -1,929 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Routing Forwarding Information Base (Routing Tables) - * - * Author: Steve Whitehouse - * Mostly copied from the IPv4 routing code - * - * - * Changes: - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* RTF_xxx */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct dn_zone -{ - struct dn_zone *dz_next; - struct dn_fib_node **dz_hash; - int dz_nent; - int dz_divisor; - u32 dz_hashmask; -#define DZ_HASHMASK(dz) ((dz)->dz_hashmask) - int dz_order; - __le16 dz_mask; -#define DZ_MASK(dz) ((dz)->dz_mask) -}; - -struct dn_hash -{ - struct dn_zone *dh_zones[17]; - struct dn_zone *dh_zone_list; -}; - -#define dz_key_0(key) ((key).datum = 0) - -#define for_nexthops(fi) { int nhsel; const struct dn_fib_nh *nh;\ - for(nhsel = 0, nh = (fi)->fib_nh; nhsel < (fi)->fib_nhs; nh++, nhsel++) - -#define endfor_nexthops(fi) } - -#define DN_MAX_DIVISOR 1024 -#define DN_S_ZOMBIE 1 -#define DN_S_ACCESSED 2 - -#define DN_FIB_SCAN(f, fp) \ -for( ; ((f) = *(fp)) != NULL; (fp) = &(f)->fn_next) - -#define DN_FIB_SCAN_KEY(f, fp, key) \ -for( ; ((f) = *(fp)) != NULL && dn_key_eq((f)->fn_key, (key)); (fp) = &(f)->fn_next) - -#define RT_TABLE_MIN 1 -#define DN_FIB_TABLE_HASHSZ 256 -static struct hlist_head dn_fib_table_hash[DN_FIB_TABLE_HASHSZ]; -static DEFINE_RWLOCK(dn_fib_tables_lock); - -static struct kmem_cache *dn_hash_kmem __read_mostly; -static int dn_fib_hash_zombies; - -static inline dn_fib_idx_t dn_hash(dn_fib_key_t key, struct dn_zone *dz) -{ - u16 h = le16_to_cpu(key.datum)>>(16 - dz->dz_order); - h ^= (h >> 10); - h ^= (h >> 6); - h &= DZ_HASHMASK(dz); - return *(dn_fib_idx_t *)&h; -} - -static inline dn_fib_key_t dz_key(__le16 dst, struct dn_zone *dz) -{ - dn_fib_key_t k; - k.datum = dst & DZ_MASK(dz); - return k; -} - -static inline struct dn_fib_node **dn_chain_p(dn_fib_key_t key, struct dn_zone *dz) -{ - return &dz->dz_hash[dn_hash(key, dz).datum]; -} - -static inline struct dn_fib_node *dz_chain(dn_fib_key_t key, struct dn_zone *dz) -{ - return dz->dz_hash[dn_hash(key, dz).datum]; -} - -static inline int dn_key_eq(dn_fib_key_t a, dn_fib_key_t b) -{ - return a.datum == b.datum; -} - -static inline int dn_key_leq(dn_fib_key_t a, dn_fib_key_t b) -{ - return a.datum <= b.datum; -} - -static inline void dn_rebuild_zone(struct dn_zone *dz, - struct dn_fib_node **old_ht, - int old_divisor) -{ - struct dn_fib_node *f, **fp, *next; - int i; - - for(i = 0; i < old_divisor; i++) { - for(f = old_ht[i]; f; f = next) { - next = f->fn_next; - for(fp = dn_chain_p(f->fn_key, dz); - *fp && dn_key_leq((*fp)->fn_key, f->fn_key); - fp = &(*fp)->fn_next) - /* NOTHING */; - f->fn_next = *fp; - *fp = f; - } - } -} - -static void dn_rehash_zone(struct dn_zone *dz) -{ - struct dn_fib_node **ht, **old_ht; - int old_divisor, new_divisor; - u32 new_hashmask; - - old_divisor = dz->dz_divisor; - - switch (old_divisor) { - case 16: - new_divisor = 256; - new_hashmask = 0xFF; - break; - default: - printk(KERN_DEBUG "DECnet: dn_rehash_zone: BUG! %d\n", - old_divisor); - fallthrough; - case 256: - new_divisor = 1024; - new_hashmask = 0x3FF; - break; - } - - ht = kcalloc(new_divisor, sizeof(struct dn_fib_node*), GFP_KERNEL); - if (ht == NULL) - return; - - write_lock_bh(&dn_fib_tables_lock); - old_ht = dz->dz_hash; - dz->dz_hash = ht; - dz->dz_hashmask = new_hashmask; - dz->dz_divisor = new_divisor; - dn_rebuild_zone(dz, old_ht, old_divisor); - write_unlock_bh(&dn_fib_tables_lock); - kfree(old_ht); -} - -static void dn_free_node(struct dn_fib_node *f) -{ - dn_fib_release_info(DN_FIB_INFO(f)); - kmem_cache_free(dn_hash_kmem, f); -} - - -static struct dn_zone *dn_new_zone(struct dn_hash *table, int z) -{ - int i; - struct dn_zone *dz = kzalloc(sizeof(struct dn_zone), GFP_KERNEL); - if (!dz) - return NULL; - - if (z) { - dz->dz_divisor = 16; - dz->dz_hashmask = 0x0F; - } else { - dz->dz_divisor = 1; - dz->dz_hashmask = 0; - } - - dz->dz_hash = kcalloc(dz->dz_divisor, sizeof(struct dn_fib_node *), GFP_KERNEL); - if (!dz->dz_hash) { - kfree(dz); - return NULL; - } - - dz->dz_order = z; - dz->dz_mask = dnet_make_mask(z); - - for(i = z + 1; i <= 16; i++) - if (table->dh_zones[i]) - break; - - write_lock_bh(&dn_fib_tables_lock); - if (i>16) { - dz->dz_next = table->dh_zone_list; - table->dh_zone_list = dz; - } else { - dz->dz_next = table->dh_zones[i]->dz_next; - table->dh_zones[i]->dz_next = dz; - } - table->dh_zones[z] = dz; - write_unlock_bh(&dn_fib_tables_lock); - return dz; -} - - -static int dn_fib_nh_match(struct rtmsg *r, struct nlmsghdr *nlh, struct nlattr *attrs[], struct dn_fib_info *fi) -{ - struct rtnexthop *nhp; - int nhlen; - - if (attrs[RTA_PRIORITY] && - nla_get_u32(attrs[RTA_PRIORITY]) != fi->fib_priority) - return 1; - - if (attrs[RTA_OIF] || attrs[RTA_GATEWAY]) { - if ((!attrs[RTA_OIF] || nla_get_u32(attrs[RTA_OIF]) == fi->fib_nh->nh_oif) && - (!attrs[RTA_GATEWAY] || nla_get_le16(attrs[RTA_GATEWAY]) != fi->fib_nh->nh_gw)) - return 0; - return 1; - } - - if (!attrs[RTA_MULTIPATH]) - return 0; - - nhp = nla_data(attrs[RTA_MULTIPATH]); - nhlen = nla_len(attrs[RTA_MULTIPATH]); - - for_nexthops(fi) { - int attrlen = nhlen - sizeof(struct rtnexthop); - __le16 gw; - - if (attrlen < 0 || (nhlen -= nhp->rtnh_len) < 0) - return -EINVAL; - if (nhp->rtnh_ifindex && nhp->rtnh_ifindex != nh->nh_oif) - return 1; - if (attrlen) { - struct nlattr *gw_attr; - - gw_attr = nla_find((struct nlattr *) (nhp + 1), attrlen, RTA_GATEWAY); - gw = gw_attr ? nla_get_le16(gw_attr) : 0; - - if (gw && gw != nh->nh_gw) - return 1; - } - nhp = RTNH_NEXT(nhp); - } endfor_nexthops(fi); - - return 0; -} - -static inline size_t dn_fib_nlmsg_size(struct dn_fib_info *fi) -{ - size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg)) - + nla_total_size(4) /* RTA_TABLE */ - + nla_total_size(2) /* RTA_DST */ - + nla_total_size(4) /* RTA_PRIORITY */ - + nla_total_size(TCP_CA_NAME_MAX); /* RTAX_CC_ALGO */ - - /* space for nested metrics */ - payload += nla_total_size((RTAX_MAX * nla_total_size(4))); - - if (fi->fib_nhs) { - /* Also handles the special case fib_nhs == 1 */ - - /* each nexthop is packed in an attribute */ - size_t nhsize = nla_total_size(sizeof(struct rtnexthop)); - - /* may contain a gateway attribute */ - nhsize += nla_total_size(4); - - /* all nexthops are packed in a nested attribute */ - payload += nla_total_size(fi->fib_nhs * nhsize); - } - - return payload; -} - -static int dn_fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event, - u32 tb_id, u8 type, u8 scope, void *dst, int dst_len, - struct dn_fib_info *fi, unsigned int flags) -{ - struct rtmsg *rtm; - struct nlmsghdr *nlh; - - nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags); - if (!nlh) - return -EMSGSIZE; - - rtm = nlmsg_data(nlh); - rtm->rtm_family = AF_DECnet; - rtm->rtm_dst_len = dst_len; - rtm->rtm_src_len = 0; - rtm->rtm_tos = 0; - rtm->rtm_table = tb_id; - rtm->rtm_flags = fi->fib_flags; - rtm->rtm_scope = scope; - rtm->rtm_type = type; - rtm->rtm_protocol = fi->fib_protocol; - - if (nla_put_u32(skb, RTA_TABLE, tb_id) < 0) - goto errout; - - if (rtm->rtm_dst_len && - nla_put(skb, RTA_DST, 2, dst) < 0) - goto errout; - - if (fi->fib_priority && - nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority) < 0) - goto errout; - - if (rtnetlink_put_metrics(skb, fi->fib_metrics) < 0) - goto errout; - - if (fi->fib_nhs == 1) { - if (fi->fib_nh->nh_gw && - nla_put_le16(skb, RTA_GATEWAY, fi->fib_nh->nh_gw) < 0) - goto errout; - - if (fi->fib_nh->nh_oif && - nla_put_u32(skb, RTA_OIF, fi->fib_nh->nh_oif) < 0) - goto errout; - } - - if (fi->fib_nhs > 1) { - struct rtnexthop *nhp; - struct nlattr *mp_head; - - mp_head = nla_nest_start_noflag(skb, RTA_MULTIPATH); - if (!mp_head) - goto errout; - - for_nexthops(fi) { - if (!(nhp = nla_reserve_nohdr(skb, sizeof(*nhp)))) - goto errout; - - nhp->rtnh_flags = nh->nh_flags & 0xFF; - nhp->rtnh_hops = nh->nh_weight - 1; - nhp->rtnh_ifindex = nh->nh_oif; - - if (nh->nh_gw && - nla_put_le16(skb, RTA_GATEWAY, nh->nh_gw) < 0) - goto errout; - - nhp->rtnh_len = skb_tail_pointer(skb) - (unsigned char *)nhp; - } endfor_nexthops(fi); - - nla_nest_end(skb, mp_head); - } - - nlmsg_end(skb, nlh); - return 0; - -errout: - nlmsg_cancel(skb, nlh); - return -EMSGSIZE; -} - - -static void dn_rtmsg_fib(int event, struct dn_fib_node *f, int z, u32 tb_id, - struct nlmsghdr *nlh, struct netlink_skb_parms *req) -{ - struct sk_buff *skb; - u32 portid = req ? req->portid : 0; - int err = -ENOBUFS; - - skb = nlmsg_new(dn_fib_nlmsg_size(DN_FIB_INFO(f)), GFP_KERNEL); - if (skb == NULL) - goto errout; - - err = dn_fib_dump_info(skb, portid, nlh->nlmsg_seq, event, tb_id, - f->fn_type, f->fn_scope, &f->fn_key, z, - DN_FIB_INFO(f), 0); - if (err < 0) { - /* -EMSGSIZE implies BUG in dn_fib_nlmsg_size() */ - WARN_ON(err == -EMSGSIZE); - kfree_skb(skb); - goto errout; - } - rtnl_notify(skb, &init_net, portid, RTNLGRP_DECnet_ROUTE, nlh, GFP_KERNEL); - return; -errout: - if (err < 0) - rtnl_set_sk_err(&init_net, RTNLGRP_DECnet_ROUTE, err); -} - -static __inline__ int dn_hash_dump_bucket(struct sk_buff *skb, - struct netlink_callback *cb, - struct dn_fib_table *tb, - struct dn_zone *dz, - struct dn_fib_node *f) -{ - int i, s_i; - - s_i = cb->args[4]; - for(i = 0; f; i++, f = f->fn_next) { - if (i < s_i) - continue; - if (f->fn_state & DN_S_ZOMBIE) - continue; - if (dn_fib_dump_info(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, - RTM_NEWROUTE, - tb->n, - (f->fn_state & DN_S_ZOMBIE) ? 0 : f->fn_type, - f->fn_scope, &f->fn_key, dz->dz_order, - f->fn_info, NLM_F_MULTI) < 0) { - cb->args[4] = i; - return -1; - } - } - cb->args[4] = i; - return skb->len; -} - -static __inline__ int dn_hash_dump_zone(struct sk_buff *skb, - struct netlink_callback *cb, - struct dn_fib_table *tb, - struct dn_zone *dz) -{ - int h, s_h; - - s_h = cb->args[3]; - for(h = 0; h < dz->dz_divisor; h++) { - if (h < s_h) - continue; - if (h > s_h) - memset(&cb->args[4], 0, sizeof(cb->args) - 4*sizeof(cb->args[0])); - if (dz->dz_hash == NULL || dz->dz_hash[h] == NULL) - continue; - if (dn_hash_dump_bucket(skb, cb, tb, dz, dz->dz_hash[h]) < 0) { - cb->args[3] = h; - return -1; - } - } - cb->args[3] = h; - return skb->len; -} - -static int dn_fib_table_dump(struct dn_fib_table *tb, struct sk_buff *skb, - struct netlink_callback *cb) -{ - int m, s_m; - struct dn_zone *dz; - struct dn_hash *table = (struct dn_hash *)tb->data; - - s_m = cb->args[2]; - read_lock(&dn_fib_tables_lock); - for(dz = table->dh_zone_list, m = 0; dz; dz = dz->dz_next, m++) { - if (m < s_m) - continue; - if (m > s_m) - memset(&cb->args[3], 0, sizeof(cb->args) - 3*sizeof(cb->args[0])); - - if (dn_hash_dump_zone(skb, cb, tb, dz) < 0) { - cb->args[2] = m; - read_unlock(&dn_fib_tables_lock); - return -1; - } - } - read_unlock(&dn_fib_tables_lock); - cb->args[2] = m; - - return skb->len; -} - -int dn_fib_dump(struct sk_buff *skb, struct netlink_callback *cb) -{ - struct net *net = sock_net(skb->sk); - unsigned int h, s_h; - unsigned int e = 0, s_e; - struct dn_fib_table *tb; - int dumped = 0; - - if (!net_eq(net, &init_net)) - return 0; - - if (nlmsg_len(cb->nlh) >= sizeof(struct rtmsg) && - ((struct rtmsg *)nlmsg_data(cb->nlh))->rtm_flags&RTM_F_CLONED) - return dn_cache_dump(skb, cb); - - s_h = cb->args[0]; - s_e = cb->args[1]; - - for (h = s_h; h < DN_FIB_TABLE_HASHSZ; h++, s_h = 0) { - e = 0; - hlist_for_each_entry(tb, &dn_fib_table_hash[h], hlist) { - if (e < s_e) - goto next; - if (dumped) - memset(&cb->args[2], 0, sizeof(cb->args) - - 2 * sizeof(cb->args[0])); - if (tb->dump(tb, skb, cb) < 0) - goto out; - dumped = 1; -next: - e++; - } - } -out: - cb->args[1] = e; - cb->args[0] = h; - - return skb->len; -} - -static int dn_fib_table_insert(struct dn_fib_table *tb, struct rtmsg *r, struct nlattr *attrs[], - struct nlmsghdr *n, struct netlink_skb_parms *req) -{ - struct dn_hash *table = (struct dn_hash *)tb->data; - struct dn_fib_node *new_f, *f, **fp, **del_fp; - struct dn_zone *dz; - struct dn_fib_info *fi; - int z = r->rtm_dst_len; - int type = r->rtm_type; - dn_fib_key_t key; - int err; - - if (z > 16) - return -EINVAL; - - dz = table->dh_zones[z]; - if (!dz && !(dz = dn_new_zone(table, z))) - return -ENOBUFS; - - dz_key_0(key); - if (attrs[RTA_DST]) { - __le16 dst = nla_get_le16(attrs[RTA_DST]); - if (dst & ~DZ_MASK(dz)) - return -EINVAL; - key = dz_key(dst, dz); - } - - if ((fi = dn_fib_create_info(r, attrs, n, &err)) == NULL) - return err; - - if (dz->dz_nent > (dz->dz_divisor << 2) && - dz->dz_divisor > DN_MAX_DIVISOR && - (z==16 || (1< dz->dz_divisor)) - dn_rehash_zone(dz); - - fp = dn_chain_p(key, dz); - - DN_FIB_SCAN(f, fp) { - if (dn_key_leq(key, f->fn_key)) - break; - } - - del_fp = NULL; - - if (f && (f->fn_state & DN_S_ZOMBIE) && - dn_key_eq(f->fn_key, key)) { - del_fp = fp; - fp = &f->fn_next; - f = *fp; - goto create; - } - - DN_FIB_SCAN_KEY(f, fp, key) { - if (fi->fib_priority <= DN_FIB_INFO(f)->fib_priority) - break; - } - - if (f && dn_key_eq(f->fn_key, key) && - fi->fib_priority == DN_FIB_INFO(f)->fib_priority) { - struct dn_fib_node **ins_fp; - - err = -EEXIST; - if (n->nlmsg_flags & NLM_F_EXCL) - goto out; - - if (n->nlmsg_flags & NLM_F_REPLACE) { - del_fp = fp; - fp = &f->fn_next; - f = *fp; - goto replace; - } - - ins_fp = fp; - err = -EEXIST; - - DN_FIB_SCAN_KEY(f, fp, key) { - if (fi->fib_priority != DN_FIB_INFO(f)->fib_priority) - break; - if (f->fn_type == type && - f->fn_scope == r->rtm_scope && - DN_FIB_INFO(f) == fi) - goto out; - } - - if (!(n->nlmsg_flags & NLM_F_APPEND)) { - fp = ins_fp; - f = *fp; - } - } - -create: - err = -ENOENT; - if (!(n->nlmsg_flags & NLM_F_CREATE)) - goto out; - -replace: - err = -ENOBUFS; - new_f = kmem_cache_zalloc(dn_hash_kmem, GFP_KERNEL); - if (new_f == NULL) - goto out; - - new_f->fn_key = key; - new_f->fn_type = type; - new_f->fn_scope = r->rtm_scope; - DN_FIB_INFO(new_f) = fi; - - new_f->fn_next = f; - write_lock_bh(&dn_fib_tables_lock); - *fp = new_f; - write_unlock_bh(&dn_fib_tables_lock); - dz->dz_nent++; - - if (del_fp) { - f = *del_fp; - write_lock_bh(&dn_fib_tables_lock); - *del_fp = f->fn_next; - write_unlock_bh(&dn_fib_tables_lock); - - if (!(f->fn_state & DN_S_ZOMBIE)) - dn_rtmsg_fib(RTM_DELROUTE, f, z, tb->n, n, req); - if (f->fn_state & DN_S_ACCESSED) - dn_rt_cache_flush(-1); - dn_free_node(f); - dz->dz_nent--; - } else { - dn_rt_cache_flush(-1); - } - - dn_rtmsg_fib(RTM_NEWROUTE, new_f, z, tb->n, n, req); - - return 0; -out: - dn_fib_release_info(fi); - return err; -} - - -static int dn_fib_table_delete(struct dn_fib_table *tb, struct rtmsg *r, struct nlattr *attrs[], - struct nlmsghdr *n, struct netlink_skb_parms *req) -{ - struct dn_hash *table = (struct dn_hash*)tb->data; - struct dn_fib_node **fp, **del_fp, *f; - int z = r->rtm_dst_len; - struct dn_zone *dz; - dn_fib_key_t key; - int matched; - - - if (z > 16) - return -EINVAL; - - if ((dz = table->dh_zones[z]) == NULL) - return -ESRCH; - - dz_key_0(key); - if (attrs[RTA_DST]) { - __le16 dst = nla_get_le16(attrs[RTA_DST]); - if (dst & ~DZ_MASK(dz)) - return -EINVAL; - key = dz_key(dst, dz); - } - - fp = dn_chain_p(key, dz); - - DN_FIB_SCAN(f, fp) { - if (dn_key_eq(f->fn_key, key)) - break; - if (dn_key_leq(key, f->fn_key)) - return -ESRCH; - } - - matched = 0; - del_fp = NULL; - DN_FIB_SCAN_KEY(f, fp, key) { - struct dn_fib_info *fi = DN_FIB_INFO(f); - - if (f->fn_state & DN_S_ZOMBIE) - return -ESRCH; - - matched++; - - if (del_fp == NULL && - (!r->rtm_type || f->fn_type == r->rtm_type) && - (r->rtm_scope == RT_SCOPE_NOWHERE || f->fn_scope == r->rtm_scope) && - (!r->rtm_protocol || - fi->fib_protocol == r->rtm_protocol) && - dn_fib_nh_match(r, n, attrs, fi) == 0) - del_fp = fp; - } - - if (del_fp) { - f = *del_fp; - dn_rtmsg_fib(RTM_DELROUTE, f, z, tb->n, n, req); - - if (matched != 1) { - write_lock_bh(&dn_fib_tables_lock); - *del_fp = f->fn_next; - write_unlock_bh(&dn_fib_tables_lock); - - if (f->fn_state & DN_S_ACCESSED) - dn_rt_cache_flush(-1); - dn_free_node(f); - dz->dz_nent--; - } else { - f->fn_state |= DN_S_ZOMBIE; - if (f->fn_state & DN_S_ACCESSED) { - f->fn_state &= ~DN_S_ACCESSED; - dn_rt_cache_flush(-1); - } - if (++dn_fib_hash_zombies > 128) - dn_fib_flush(); - } - - return 0; - } - - return -ESRCH; -} - -static inline int dn_flush_list(struct dn_fib_node **fp, int z, struct dn_hash *table) -{ - int found = 0; - struct dn_fib_node *f; - - while((f = *fp) != NULL) { - struct dn_fib_info *fi = DN_FIB_INFO(f); - - if (fi && ((f->fn_state & DN_S_ZOMBIE) || (fi->fib_flags & RTNH_F_DEAD))) { - write_lock_bh(&dn_fib_tables_lock); - *fp = f->fn_next; - write_unlock_bh(&dn_fib_tables_lock); - - dn_free_node(f); - found++; - continue; - } - fp = &f->fn_next; - } - - return found; -} - -static int dn_fib_table_flush(struct dn_fib_table *tb) -{ - struct dn_hash *table = (struct dn_hash *)tb->data; - struct dn_zone *dz; - int found = 0; - - dn_fib_hash_zombies = 0; - for(dz = table->dh_zone_list; dz; dz = dz->dz_next) { - int i; - int tmp = 0; - for(i = dz->dz_divisor-1; i >= 0; i--) - tmp += dn_flush_list(&dz->dz_hash[i], dz->dz_order, table); - dz->dz_nent -= tmp; - found += tmp; - } - - return found; -} - -static int dn_fib_table_lookup(struct dn_fib_table *tb, const struct flowidn *flp, struct dn_fib_res *res) -{ - int err; - struct dn_zone *dz; - struct dn_hash *t = (struct dn_hash *)tb->data; - - read_lock(&dn_fib_tables_lock); - for(dz = t->dh_zone_list; dz; dz = dz->dz_next) { - struct dn_fib_node *f; - dn_fib_key_t k = dz_key(flp->daddr, dz); - - for(f = dz_chain(k, dz); f; f = f->fn_next) { - if (!dn_key_eq(k, f->fn_key)) { - if (dn_key_leq(k, f->fn_key)) - break; - else - continue; - } - - f->fn_state |= DN_S_ACCESSED; - - if (f->fn_state&DN_S_ZOMBIE) - continue; - - if (f->fn_scope < flp->flowidn_scope) - continue; - - err = dn_fib_semantic_match(f->fn_type, DN_FIB_INFO(f), flp, res); - - if (err == 0) { - res->type = f->fn_type; - res->scope = f->fn_scope; - res->prefixlen = dz->dz_order; - goto out; - } - if (err < 0) - goto out; - } - } - err = 1; -out: - read_unlock(&dn_fib_tables_lock); - return err; -} - - -struct dn_fib_table *dn_fib_get_table(u32 n, int create) -{ - struct dn_fib_table *t; - unsigned int h; - - if (n < RT_TABLE_MIN) - return NULL; - - if (n > RT_TABLE_MAX) - return NULL; - - h = n & (DN_FIB_TABLE_HASHSZ - 1); - rcu_read_lock(); - hlist_for_each_entry_rcu(t, &dn_fib_table_hash[h], hlist) { - if (t->n == n) { - rcu_read_unlock(); - return t; - } - } - rcu_read_unlock(); - - if (!create) - return NULL; - - if (in_interrupt()) { - net_dbg_ratelimited("DECnet: BUG! Attempt to create routing table from interrupt\n"); - return NULL; - } - - t = kzalloc(sizeof(struct dn_fib_table) + sizeof(struct dn_hash), - GFP_KERNEL); - if (t == NULL) - return NULL; - - t->n = n; - t->insert = dn_fib_table_insert; - t->delete = dn_fib_table_delete; - t->lookup = dn_fib_table_lookup; - t->flush = dn_fib_table_flush; - t->dump = dn_fib_table_dump; - hlist_add_head_rcu(&t->hlist, &dn_fib_table_hash[h]); - - return t; -} - -struct dn_fib_table *dn_fib_empty_table(void) -{ - u32 id; - - for(id = RT_TABLE_MIN; id <= RT_TABLE_MAX; id++) - if (dn_fib_get_table(id, 0) == NULL) - return dn_fib_get_table(id, 1); - return NULL; -} - -void dn_fib_flush(void) -{ - int flushed = 0; - struct dn_fib_table *tb; - unsigned int h; - - for (h = 0; h < DN_FIB_TABLE_HASHSZ; h++) { - hlist_for_each_entry(tb, &dn_fib_table_hash[h], hlist) - flushed += tb->flush(tb); - } - - if (flushed) - dn_rt_cache_flush(-1); -} - -void __init dn_fib_table_init(void) -{ - dn_hash_kmem = kmem_cache_create("dn_fib_info_cache", - sizeof(struct dn_fib_info), - 0, SLAB_HWCACHE_ALIGN, - NULL); -} - -void __exit dn_fib_table_cleanup(void) -{ - struct dn_fib_table *t; - struct hlist_node *next; - unsigned int h; - - write_lock(&dn_fib_tables_lock); - for (h = 0; h < DN_FIB_TABLE_HASHSZ; h++) { - hlist_for_each_entry_safe(t, next, &dn_fib_table_hash[h], - hlist) { - hlist_del(&t->hlist); - kfree(t); - } - } - write_unlock(&dn_fib_tables_lock); -} diff --git a/net/decnet/dn_timer.c b/net/decnet/dn_timer.c deleted file mode 100644 index aa4155875ca84eabb75ab445c4f1adf2612eeac2..0000000000000000000000000000000000000000 --- a/net/decnet/dn_timer.c +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Socket Timer Functions - * - * Author: Steve Whitehouse - * - * - * Changes: - * Steve Whitehouse : Made keepalive timer part of the same - * timer idea. - * Steve Whitehouse : Added checks for sk->sock_readers - * David S. Miller : New socket locking - * Steve Whitehouse : Timer grabs socket ref. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Slow timer is for everything else (n * 500mS) - */ - -#define SLOW_INTERVAL (HZ/2) - -static void dn_slow_timer(struct timer_list *t); - -void dn_start_slow_timer(struct sock *sk) -{ - timer_setup(&sk->sk_timer, dn_slow_timer, 0); - sk_reset_timer(sk, &sk->sk_timer, jiffies + SLOW_INTERVAL); -} - -void dn_stop_slow_timer(struct sock *sk) -{ - sk_stop_timer(sk, &sk->sk_timer); -} - -static void dn_slow_timer(struct timer_list *t) -{ - struct sock *sk = from_timer(sk, t, sk_timer); - struct dn_scp *scp = DN_SK(sk); - - bh_lock_sock(sk); - - if (sock_owned_by_user(sk)) { - sk_reset_timer(sk, &sk->sk_timer, jiffies + HZ / 10); - goto out; - } - - /* - * The persist timer is the standard slow timer used for retransmits - * in both connection establishment and disconnection as well as - * in the RUN state. The different states are catered for by changing - * the function pointer in the socket. Setting the timer to a value - * of zero turns it off. We allow the persist_fxn to turn the - * timer off in a permant way by returning non-zero, so that - * timer based routines may remove sockets. This is why we have a - * sock_hold()/sock_put() around the timer to prevent the socket - * going away in the middle. - */ - if (scp->persist && scp->persist_fxn) { - if (scp->persist <= SLOW_INTERVAL) { - scp->persist = 0; - - if (scp->persist_fxn(sk)) - goto out; - } else { - scp->persist -= SLOW_INTERVAL; - } - } - - /* - * Check for keepalive timeout. After the other timer 'cos if - * the previous timer caused a retransmit, we don't need to - * do this. scp->stamp is the last time that we sent a packet. - * The keepalive function sends a link service packet to the - * other end. If it remains unacknowledged, the standard - * socket timers will eventually shut the socket down. Each - * time we do this, scp->stamp will be updated, thus - * we won't try and send another until scp->keepalive has passed - * since the last successful transmission. - */ - if (scp->keepalive && scp->keepalive_fxn && (scp->state == DN_RUN)) { - if (time_after_eq(jiffies, scp->stamp + scp->keepalive)) - scp->keepalive_fxn(sk); - } - - sk_reset_timer(sk, &sk->sk_timer, jiffies + SLOW_INTERVAL); -out: - bh_unlock_sock(sk); - sock_put(sk); -} diff --git a/net/decnet/netfilter/Kconfig b/net/decnet/netfilter/Kconfig deleted file mode 100644 index 14ec4ef95fab1789e0690fbe1fa6b1237439c6ac..0000000000000000000000000000000000000000 --- a/net/decnet/netfilter/Kconfig +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# DECnet netfilter configuration -# - -menu "DECnet: Netfilter Configuration" - depends on DECNET && NETFILTER - depends on NETFILTER_ADVANCED - -config DECNET_NF_GRABULATOR - tristate "Routing message grabulator (for userland routing daemon)" - help - Enable this module if you want to use the userland DECnet routing - daemon. You will also need to enable routing support for DECnet - unless you just want to monitor routing messages from other nodes. - -endmenu diff --git a/net/decnet/netfilter/Makefile b/net/decnet/netfilter/Makefile deleted file mode 100644 index 429c84289d0ff693ab16a398c8cbd5e98fe1a7ed..0000000000000000000000000000000000000000 --- a/net/decnet/netfilter/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for DECnet netfilter modules -# - -obj-$(CONFIG_DECNET_NF_GRABULATOR) += dn_rtmsg.o diff --git a/net/decnet/netfilter/dn_rtmsg.c b/net/decnet/netfilter/dn_rtmsg.c deleted file mode 100644 index 26a9193df78319f89bf7e039ce569080215374ca..0000000000000000000000000000000000000000 --- a/net/decnet/netfilter/dn_rtmsg.c +++ /dev/null @@ -1,158 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet Routing Message Grabulator - * - * (C) 2000 ChyGwyn Limited - https://www.chygwyn.com/ - * - * Author: Steven Whitehouse - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -static struct sock *dnrmg = NULL; - - -static struct sk_buff *dnrmg_build_message(struct sk_buff *rt_skb, int *errp) -{ - struct sk_buff *skb = NULL; - size_t size; - sk_buff_data_t old_tail; - struct nlmsghdr *nlh; - unsigned char *ptr; - struct nf_dn_rtmsg *rtm; - - size = NLMSG_ALIGN(rt_skb->len) + - NLMSG_ALIGN(sizeof(struct nf_dn_rtmsg)); - skb = nlmsg_new(size, GFP_ATOMIC); - if (!skb) { - *errp = -ENOMEM; - return NULL; - } - old_tail = skb->tail; - nlh = nlmsg_put(skb, 0, 0, 0, size, 0); - if (!nlh) { - kfree_skb(skb); - *errp = -ENOMEM; - return NULL; - } - rtm = (struct nf_dn_rtmsg *)nlmsg_data(nlh); - rtm->nfdn_ifindex = rt_skb->dev->ifindex; - ptr = NFDN_RTMSG(rtm); - skb_copy_from_linear_data(rt_skb, ptr, rt_skb->len); - nlh->nlmsg_len = skb->tail - old_tail; - return skb; -} - -static void dnrmg_send_peer(struct sk_buff *skb) -{ - struct sk_buff *skb2; - int status = 0; - int group = 0; - unsigned char flags = *skb->data; - - switch (flags & DN_RT_CNTL_MSK) { - case DN_RT_PKT_L1RT: - group = DNRNG_NLGRP_L1; - break; - case DN_RT_PKT_L2RT: - group = DNRNG_NLGRP_L2; - break; - default: - return; - } - - skb2 = dnrmg_build_message(skb, &status); - if (skb2 == NULL) - return; - NETLINK_CB(skb2).dst_group = group; - netlink_broadcast(dnrmg, skb2, 0, group, GFP_ATOMIC); -} - - -static unsigned int dnrmg_hook(void *priv, - struct sk_buff *skb, - const struct nf_hook_state *state) -{ - dnrmg_send_peer(skb); - return NF_ACCEPT; -} - - -#define RCV_SKB_FAIL(err) do { netlink_ack(skb, nlh, (err), NULL); return; } while (0) - -static inline void dnrmg_receive_user_skb(struct sk_buff *skb) -{ - struct nlmsghdr *nlh = nlmsg_hdr(skb); - - if (skb->len < sizeof(*nlh) || - nlh->nlmsg_len < sizeof(*nlh) || - skb->len < nlh->nlmsg_len) - return; - - if (!netlink_capable(skb, CAP_NET_ADMIN)) - RCV_SKB_FAIL(-EPERM); - - /* Eventually we might send routing messages too */ - - RCV_SKB_FAIL(-EINVAL); -} - -static const struct nf_hook_ops dnrmg_ops = { - .hook = dnrmg_hook, - .pf = NFPROTO_DECNET, - .hooknum = NF_DN_ROUTE, - .priority = NF_DN_PRI_DNRTMSG, -}; - -static int __init dn_rtmsg_init(void) -{ - int rv = 0; - struct netlink_kernel_cfg cfg = { - .groups = DNRNG_NLGRP_MAX, - .input = dnrmg_receive_user_skb, - }; - - dnrmg = netlink_kernel_create(&init_net, NETLINK_DNRTMSG, &cfg); - if (dnrmg == NULL) { - printk(KERN_ERR "dn_rtmsg: Cannot create netlink socket"); - return -ENOMEM; - } - - rv = nf_register_net_hook(&init_net, &dnrmg_ops); - if (rv) { - netlink_kernel_release(dnrmg); - } - - return rv; -} - -static void __exit dn_rtmsg_fini(void) -{ - nf_unregister_net_hook(&init_net, &dnrmg_ops); - netlink_kernel_release(dnrmg); -} - - -MODULE_DESCRIPTION("DECnet Routing Message Grabulator"); -MODULE_AUTHOR("Steven Whitehouse "); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_DNRTMSG); - -module_init(dn_rtmsg_init); -module_exit(dn_rtmsg_fini); diff --git a/net/decnet/sysctl_net_decnet.c b/net/decnet/sysctl_net_decnet.c deleted file mode 100644 index 67b5ab2657b7c9c0d20581a499e9f1bbbda798d5..0000000000000000000000000000000000000000 --- a/net/decnet/sysctl_net_decnet.c +++ /dev/null @@ -1,362 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DECnet An implementation of the DECnet protocol suite for the LINUX - * operating system. DECnet is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * DECnet sysctl support functions - * - * Author: Steve Whitehouse - * - * - * Changes: - * Steve Whitehouse - C99 changes and default device handling - * Steve Whitehouse - Memory buffer settings, like the tcp ones - * - */ -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - - -int decnet_debug_level; -int decnet_time_wait = 30; -int decnet_dn_count = 1; -int decnet_di_count = 3; -int decnet_dr_count = 3; -int decnet_log_martians = 1; -int decnet_no_fc_max_cwnd = NSP_MIN_WINDOW; - -/* Reasonable defaults, I hope, based on tcp's defaults */ -long sysctl_decnet_mem[3] = { 768 << 3, 1024 << 3, 1536 << 3 }; -int sysctl_decnet_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 }; -int sysctl_decnet_rmem[3] = { 4 * 1024, 87380, 87380 * 2 }; - -#ifdef CONFIG_SYSCTL -extern int decnet_dst_gc_interval; -static int min_decnet_time_wait[] = { 5 }; -static int max_decnet_time_wait[] = { 600 }; -static int min_state_count[] = { 1 }; -static int max_state_count[] = { NSP_MAXRXTSHIFT }; -static int min_decnet_dst_gc_interval[] = { 1 }; -static int max_decnet_dst_gc_interval[] = { 60 }; -static int min_decnet_no_fc_max_cwnd[] = { NSP_MIN_WINDOW }; -static int max_decnet_no_fc_max_cwnd[] = { NSP_MAX_WINDOW }; -static char node_name[7] = "???"; - -static struct ctl_table_header *dn_table_header = NULL; - -/* - * ctype.h :-) - */ -#define ISNUM(x) (((x) >= '0') && ((x) <= '9')) -#define ISLOWER(x) (((x) >= 'a') && ((x) <= 'z')) -#define ISUPPER(x) (((x) >= 'A') && ((x) <= 'Z')) -#define ISALPHA(x) (ISLOWER(x) || ISUPPER(x)) -#define INVALID_END_CHAR(x) (ISNUM(x) || ISALPHA(x)) - -static void strip_it(char *str) -{ - for(;;) { - switch (*str) { - case ' ': - case '\n': - case '\r': - case ':': - *str = 0; - fallthrough; - case 0: - return; - } - str++; - } -} - -/* - * Simple routine to parse an ascii DECnet address - * into a network order address. - */ -static int parse_addr(__le16 *addr, char *str) -{ - __u16 area, node; - - while(*str && !ISNUM(*str)) str++; - - if (*str == 0) - return -1; - - area = (*str++ - '0'); - if (ISNUM(*str)) { - area *= 10; - area += (*str++ - '0'); - } - - if (*str++ != '.') - return -1; - - if (!ISNUM(*str)) - return -1; - - node = *str++ - '0'; - if (ISNUM(*str)) { - node *= 10; - node += (*str++ - '0'); - } - if (ISNUM(*str)) { - node *= 10; - node += (*str++ - '0'); - } - if (ISNUM(*str)) { - node *= 10; - node += (*str++ - '0'); - } - - if ((node > 1023) || (area > 63)) - return -1; - - if (INVALID_END_CHAR(*str)) - return -1; - - *addr = cpu_to_le16((area << 10) | node); - - return 0; -} - -static int dn_node_address_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - char addr[DN_ASCBUF_LEN]; - size_t len; - __le16 dnaddr; - - if (!*lenp || (*ppos && !write)) { - *lenp = 0; - return 0; - } - - if (write) { - len = (*lenp < DN_ASCBUF_LEN) ? *lenp : (DN_ASCBUF_LEN-1); - memcpy(addr, buffer, len); - addr[len] = 0; - strip_it(addr); - - if (parse_addr(&dnaddr, addr)) - return -EINVAL; - - dn_dev_devices_off(); - - decnet_address = dnaddr; - - dn_dev_devices_on(); - - *ppos += len; - - return 0; - } - - dn_addr2asc(le16_to_cpu(decnet_address), addr); - len = strlen(addr); - addr[len++] = '\n'; - - if (len > *lenp) - len = *lenp; - memcpy(buffer, addr, len); - *lenp = len; - *ppos += len; - - return 0; -} - -static int dn_def_dev_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - size_t len; - struct net_device *dev; - char devname[17]; - - if (!*lenp || (*ppos && !write)) { - *lenp = 0; - return 0; - } - - if (write) { - if (*lenp > 16) - return -E2BIG; - - memcpy(devname, buffer, *lenp); - devname[*lenp] = 0; - strip_it(devname); - - dev = dev_get_by_name(&init_net, devname); - if (dev == NULL) - return -ENODEV; - - if (dev->dn_ptr == NULL) { - dev_put(dev); - return -ENODEV; - } - - if (dn_dev_set_default(dev, 1)) { - dev_put(dev); - return -ENODEV; - } - *ppos += *lenp; - - return 0; - } - - dev = dn_dev_get_default(); - if (dev == NULL) { - *lenp = 0; - return 0; - } - - strcpy(devname, dev->name); - dev_put(dev); - len = strlen(devname); - devname[len++] = '\n'; - - if (len > *lenp) len = *lenp; - - memcpy(buffer, devname, len); - *lenp = len; - *ppos += len; - - return 0; -} - -static struct ctl_table dn_table[] = { - { - .procname = "node_address", - .maxlen = 7, - .mode = 0644, - .proc_handler = dn_node_address_handler, - }, - { - .procname = "node_name", - .data = node_name, - .maxlen = 7, - .mode = 0644, - .proc_handler = proc_dostring, - }, - { - .procname = "default_device", - .maxlen = 16, - .mode = 0644, - .proc_handler = dn_def_dev_handler, - }, - { - .procname = "time_wait", - .data = &decnet_time_wait, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_decnet_time_wait, - .extra2 = &max_decnet_time_wait - }, - { - .procname = "dn_count", - .data = &decnet_dn_count, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_state_count, - .extra2 = &max_state_count - }, - { - .procname = "di_count", - .data = &decnet_di_count, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_state_count, - .extra2 = &max_state_count - }, - { - .procname = "dr_count", - .data = &decnet_dr_count, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_state_count, - .extra2 = &max_state_count - }, - { - .procname = "dst_gc_interval", - .data = &decnet_dst_gc_interval, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_decnet_dst_gc_interval, - .extra2 = &max_decnet_dst_gc_interval - }, - { - .procname = "no_fc_max_cwnd", - .data = &decnet_no_fc_max_cwnd, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &min_decnet_no_fc_max_cwnd, - .extra2 = &max_decnet_no_fc_max_cwnd - }, - { - .procname = "decnet_mem", - .data = &sysctl_decnet_mem, - .maxlen = sizeof(sysctl_decnet_mem), - .mode = 0644, - .proc_handler = proc_doulongvec_minmax - }, - { - .procname = "decnet_rmem", - .data = &sysctl_decnet_rmem, - .maxlen = sizeof(sysctl_decnet_rmem), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "decnet_wmem", - .data = &sysctl_decnet_wmem, - .maxlen = sizeof(sysctl_decnet_wmem), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "debug", - .data = &decnet_debug_level, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { } -}; - -void dn_register_sysctl(void) -{ - dn_table_header = register_net_sysctl(&init_net, "net/decnet", dn_table); -} - -void dn_unregister_sysctl(void) -{ - unregister_net_sysctl_table(dn_table_header); -} - -#else /* CONFIG_SYSCTL */ -void dn_unregister_sysctl(void) -{ -} -void dn_register_sysctl(void) -{ -} - -#endif diff --git a/net/dsa/Makefile b/net/dsa/Makefile index af28c24ead1854c97f7350e5f1e1f3f4d8e5bbbe..bf57ef3bce2a47b3af5e4d938b543a369507b01d 100644 --- a/net/dsa/Makefile +++ b/net/dsa/Makefile @@ -1,7 +1,15 @@ # SPDX-License-Identifier: GPL-2.0 # the core obj-$(CONFIG_NET_DSA) += dsa_core.o -dsa_core-y += dsa.o dsa2.o master.o port.o slave.o switch.o tag_8021q.o +dsa_core-y += \ + dsa.o \ + dsa2.o \ + master.o \ + netlink.o \ + port.o \ + slave.o \ + switch.o \ + tag_8021q.o # tagging formats obj-$(CONFIG_NET_DSA_TAG_AR9331) += tag_ar9331.o diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c index be7b320cda76946231624e3cf7cdbf7617f6ef78..64b14f655b23eaa3012ed6acb84ef69a76b525da 100644 --- a/net/dsa/dsa.c +++ b/net/dsa/dsa.c @@ -536,8 +536,16 @@ static int __init dsa_init_module(void) dsa_tag_driver_register(&DSA_TAG_DRIVER_NAME(none_ops), THIS_MODULE); + rc = rtnl_link_register(&dsa_link_ops); + if (rc) + goto netlink_register_fail; + return 0; +netlink_register_fail: + dsa_tag_driver_unregister(&DSA_TAG_DRIVER_NAME(none_ops)); + dsa_slave_unregister_notifier(); + dev_remove_pack(&dsa_pack_type); register_notifier_fail: destroy_workqueue(dsa_owq); @@ -547,6 +555,7 @@ module_init(dsa_init_module); static void __exit dsa_cleanup_module(void) { + rtnl_link_unregister(&dsa_link_ops); dsa_tag_driver_unregister(&DSA_TAG_DRIVER_NAME(none_ops)); dsa_slave_unregister_notifier(); diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index cac48a741f27c419f68a2258c0ff0a254687db55..af0e2c0394ac3f1589ddd80f503f2d070fb91493 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -387,6 +387,20 @@ static struct dsa_port *dsa_tree_find_first_cpu(struct dsa_switch_tree *dst) return NULL; } +struct net_device *dsa_tree_find_first_master(struct dsa_switch_tree *dst) +{ + struct device_node *ethernet; + struct net_device *master; + struct dsa_port *cpu_dp; + + cpu_dp = dsa_tree_find_first_cpu(dst); + ethernet = of_parse_phandle(cpu_dp->dn, "ethernet", 0); + master = of_find_net_device_by_node(ethernet); + of_node_put(ethernet); + + return master; +} + /* Assign the default CPU port (the first one in the tree) to all ports of the * fabric which don't already have one as part of their own switch. */ @@ -447,6 +461,72 @@ static void dsa_tree_teardown_cpu_ports(struct dsa_switch_tree *dst) dp->cpu_dp = NULL; } +static int dsa_port_devlink_setup(struct dsa_port *dp) +{ + struct devlink_port *dlp = &dp->devlink_port; + struct dsa_switch_tree *dst = dp->ds->dst; + struct devlink_port_attrs attrs = {}; + struct devlink *dl = dp->ds->devlink; + struct dsa_switch *ds = dp->ds; + const unsigned char *id; + unsigned char len; + int err; + + memset(dlp, 0, sizeof(*dlp)); + devlink_port_init(dl, dlp); + + if (ds->ops->port_setup) { + err = ds->ops->port_setup(ds, dp->index); + if (err) + return err; + } + + id = (const unsigned char *)&dst->index; + len = sizeof(dst->index); + + attrs.phys.port_number = dp->index; + memcpy(attrs.switch_id.id, id, len); + attrs.switch_id.id_len = len; + + switch (dp->type) { + case DSA_PORT_TYPE_UNUSED: + attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED; + break; + case DSA_PORT_TYPE_CPU: + attrs.flavour = DEVLINK_PORT_FLAVOUR_CPU; + break; + case DSA_PORT_TYPE_DSA: + attrs.flavour = DEVLINK_PORT_FLAVOUR_DSA; + break; + case DSA_PORT_TYPE_USER: + attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; + break; + } + + devlink_port_attrs_set(dlp, &attrs); + err = devlink_port_register(dl, dlp, dp->index); + if (err) { + if (ds->ops->port_teardown) + ds->ops->port_teardown(ds, dp->index); + return err; + } + + return 0; +} + +static void dsa_port_devlink_teardown(struct dsa_port *dp) +{ + struct devlink_port *dlp = &dp->devlink_port; + struct dsa_switch *ds = dp->ds; + + devlink_port_unregister(dlp); + + if (ds->ops->port_teardown) + ds->ops->port_teardown(ds, dp->index); + + devlink_port_fini(dlp); +} + static int dsa_port_setup(struct dsa_port *dp) { struct devlink_port *dlp = &dp->devlink_port; @@ -458,21 +538,25 @@ static int dsa_port_setup(struct dsa_port *dp) if (dp->setup) return 0; - if (ds->ops->port_setup) { - err = ds->ops->port_setup(ds, dp->index); - if (err) - return err; - } + err = dsa_port_devlink_setup(dp); + if (err) + return err; switch (dp->type) { case DSA_PORT_TYPE_UNUSED: dsa_port_disable(dp); break; case DSA_PORT_TYPE_CPU: - err = dsa_port_link_register_of(dp); - if (err) - break; - dsa_port_link_registered = true; + if (dp->dn) { + err = dsa_shared_port_link_register_of(dp); + if (err) + break; + dsa_port_link_registered = true; + } else { + dev_warn(ds->dev, + "skipping link registration for CPU port %d\n", + dp->index); + } err = dsa_port_enable(dp, NULL); if (err) @@ -481,10 +565,16 @@ static int dsa_port_setup(struct dsa_port *dp) break; case DSA_PORT_TYPE_DSA: - err = dsa_port_link_register_of(dp); - if (err) - break; - dsa_port_link_registered = true; + if (dp->dn) { + err = dsa_shared_port_link_register_of(dp); + if (err) + break; + dsa_port_link_registered = true; + } else { + dev_warn(ds->dev, + "skipping link registration for DSA port %d\n", + dp->index); + } err = dsa_port_enable(dp, NULL); if (err) @@ -505,10 +595,9 @@ static int dsa_port_setup(struct dsa_port *dp) if (err && dsa_port_enabled) dsa_port_disable(dp); if (err && dsa_port_link_registered) - dsa_port_link_unregister_of(dp); + dsa_shared_port_link_unregister_of(dp); if (err) { - if (ds->ops->port_teardown) - ds->ops->port_teardown(ds, dp->index); + dsa_port_devlink_teardown(dp); return err; } @@ -517,59 +606,13 @@ static int dsa_port_setup(struct dsa_port *dp) return 0; } -static int dsa_port_devlink_setup(struct dsa_port *dp) -{ - struct devlink_port *dlp = &dp->devlink_port; - struct dsa_switch_tree *dst = dp->ds->dst; - struct devlink_port_attrs attrs = {}; - struct devlink *dl = dp->ds->devlink; - const unsigned char *id; - unsigned char len; - int err; - - id = (const unsigned char *)&dst->index; - len = sizeof(dst->index); - - attrs.phys.port_number = dp->index; - memcpy(attrs.switch_id.id, id, len); - attrs.switch_id.id_len = len; - memset(dlp, 0, sizeof(*dlp)); - - switch (dp->type) { - case DSA_PORT_TYPE_UNUSED: - attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED; - break; - case DSA_PORT_TYPE_CPU: - attrs.flavour = DEVLINK_PORT_FLAVOUR_CPU; - break; - case DSA_PORT_TYPE_DSA: - attrs.flavour = DEVLINK_PORT_FLAVOUR_DSA; - break; - case DSA_PORT_TYPE_USER: - attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; - break; - } - - devlink_port_attrs_set(dlp, &attrs); - err = devlink_port_register(dl, dlp, dp->index); - - if (!err) - dp->devlink_port_setup = true; - - return err; -} - static void dsa_port_teardown(struct dsa_port *dp) { struct devlink_port *dlp = &dp->devlink_port; - struct dsa_switch *ds = dp->ds; if (!dp->setup) return; - if (ds->ops->port_teardown) - ds->ops->port_teardown(ds, dp->index); - devlink_port_type_clear(dlp); switch (dp->type) { @@ -577,11 +620,13 @@ static void dsa_port_teardown(struct dsa_port *dp) break; case DSA_PORT_TYPE_CPU: dsa_port_disable(dp); - dsa_port_link_unregister_of(dp); + if (dp->dn) + dsa_shared_port_link_unregister_of(dp); break; case DSA_PORT_TYPE_DSA: dsa_port_disable(dp); - dsa_port_link_unregister_of(dp); + if (dp->dn) + dsa_shared_port_link_unregister_of(dp); break; case DSA_PORT_TYPE_USER: if (dp->slave) { @@ -591,46 +636,15 @@ static void dsa_port_teardown(struct dsa_port *dp) break; } - dp->setup = false; -} - -static void dsa_port_devlink_teardown(struct dsa_port *dp) -{ - struct devlink_port *dlp = &dp->devlink_port; + dsa_port_devlink_teardown(dp); - if (dp->devlink_port_setup) - devlink_port_unregister(dlp); - dp->devlink_port_setup = false; + dp->setup = false; } -/* Destroy the current devlink port, and create a new one which has the UNUSED - * flavour. At this point, any call to ds->ops->port_setup has been already - * balanced out by a call to ds->ops->port_teardown, so we know that any - * devlink port regions the driver had are now unregistered. We then call its - * ds->ops->port_setup again, in order for the driver to re-create them on the - * new devlink port. - */ -static int dsa_port_reinit_as_unused(struct dsa_port *dp) +static int dsa_port_setup_as_unused(struct dsa_port *dp) { - struct dsa_switch *ds = dp->ds; - int err; - - dsa_port_devlink_teardown(dp); dp->type = DSA_PORT_TYPE_UNUSED; - err = dsa_port_devlink_setup(dp); - if (err) - return err; - - if (ds->ops->port_setup) { - /* On error, leave the devlink port registered, - * dsa_switch_teardown will clean it up later. - */ - err = ds->ops->port_setup(ds, dp->index); - if (err) - return err; - } - - return 0; + return dsa_port_setup(dp); } static int dsa_devlink_info_get(struct devlink *dl, @@ -854,7 +868,6 @@ static int dsa_switch_setup(struct dsa_switch *ds) { struct dsa_devlink_priv *dl_priv; struct device_node *dn; - struct dsa_port *dp; int err; if (ds->setup) @@ -877,18 +890,9 @@ static int dsa_switch_setup(struct dsa_switch *ds) dl_priv = devlink_priv(ds->devlink); dl_priv->ds = ds; - /* Setup devlink port instances now, so that the switch - * setup() can register regions etc, against the ports - */ - dsa_switch_for_each_port(dp, ds) { - err = dsa_port_devlink_setup(dp); - if (err) - goto unregister_devlink_ports; - } - err = dsa_switch_register_notifier(ds); if (err) - goto unregister_devlink_ports; + goto devlink_free; ds->configure_vlan_while_not_filtering = true; @@ -929,9 +933,7 @@ teardown: ds->ops->teardown(ds); unregister_notifier: dsa_switch_unregister_notifier(ds); -unregister_devlink_ports: - dsa_switch_for_each_port(dp, ds) - dsa_port_devlink_teardown(dp); +devlink_free: devlink_free(ds->devlink); ds->devlink = NULL; return err; @@ -939,8 +941,6 @@ unregister_devlink_ports: static void dsa_switch_teardown(struct dsa_switch *ds) { - struct dsa_port *dp; - if (!ds->setup) return; @@ -959,8 +959,6 @@ static void dsa_switch_teardown(struct dsa_switch *ds) dsa_switch_unregister_notifier(ds); if (ds->devlink) { - dsa_switch_for_each_port(dp, ds) - dsa_port_devlink_teardown(dp); devlink_free(ds->devlink); ds->devlink = NULL; } @@ -1013,7 +1011,7 @@ static int dsa_tree_setup_ports(struct dsa_switch_tree *dst) if (dsa_port_is_user(dp) || dsa_port_is_unused(dp)) { err = dsa_port_setup(dp); if (err) { - err = dsa_port_reinit_as_unused(dp); + err = dsa_port_setup_as_unused(dp); if (err) goto teardown; } @@ -1046,26 +1044,24 @@ static int dsa_tree_setup_switches(struct dsa_switch_tree *dst) static int dsa_tree_setup_master(struct dsa_switch_tree *dst) { - struct dsa_port *dp; + struct dsa_port *cpu_dp; int err = 0; rtnl_lock(); - list_for_each_entry(dp, &dst->ports, list) { - if (dsa_port_is_cpu(dp)) { - struct net_device *master = dp->master; - bool admin_up = (master->flags & IFF_UP) && - !qdisc_tx_is_noop(master); + dsa_tree_for_each_cpu_port(cpu_dp, dst) { + struct net_device *master = cpu_dp->master; + bool admin_up = (master->flags & IFF_UP) && + !qdisc_tx_is_noop(master); - err = dsa_master_setup(master, dp); - if (err) - break; + err = dsa_master_setup(master, cpu_dp); + if (err) + break; - /* Replay master state event */ - dsa_tree_master_admin_state_change(dst, master, admin_up); - dsa_tree_master_oper_state_change(dst, master, - netif_oper_up(master)); - } + /* Replay master state event */ + dsa_tree_master_admin_state_change(dst, master, admin_up); + dsa_tree_master_oper_state_change(dst, master, + netif_oper_up(master)); } rtnl_unlock(); @@ -1075,22 +1071,20 @@ static int dsa_tree_setup_master(struct dsa_switch_tree *dst) static void dsa_tree_teardown_master(struct dsa_switch_tree *dst) { - struct dsa_port *dp; + struct dsa_port *cpu_dp; rtnl_lock(); - list_for_each_entry(dp, &dst->ports, list) { - if (dsa_port_is_cpu(dp)) { - struct net_device *master = dp->master; + dsa_tree_for_each_cpu_port(cpu_dp, dst) { + struct net_device *master = cpu_dp->master; - /* Synthesizing an "admin down" state is sufficient for - * the switches to get a notification if the master is - * currently up and running. - */ - dsa_tree_master_admin_state_change(dst, master, false); + /* Synthesizing an "admin down" state is sufficient for + * the switches to get a notification if the master is + * currently up and running. + */ + dsa_tree_master_admin_state_change(dst, master, false); - dsa_master_teardown(master); - } + dsa_master_teardown(master); } rtnl_unlock(); @@ -1238,7 +1232,6 @@ out_disconnect: * they would have formed disjoint trees (different "dsa,member" values). */ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, - struct net_device *master, const struct dsa_device_ops *tag_ops, const struct dsa_device_ops *old_tag_ops) { @@ -1254,12 +1247,9 @@ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, * attempts to change the tagging protocol. If we ever lift the IFF_UP * restriction, there needs to be another mutex which serializes this. */ - if (master->flags & IFF_UP) - goto out_unlock; - - list_for_each_entry(dp, &dst->ports, list) { - if (!dsa_port_is_user(dp)) - continue; + dsa_tree_for_each_user_port(dp, dst) { + if (dsa_port_to_master(dp)->flags & IFF_UP) + goto out_unlock; if (dp->slave->flags & IFF_UP) goto out_unlock; @@ -1306,6 +1296,12 @@ void dsa_tree_master_admin_state_change(struct dsa_switch_tree *dst, struct dsa_port *cpu_dp = master->dsa_ptr; bool notify = false; + /* Don't keep track of admin state on LAG DSA masters, + * but rather just of physical DSA masters + */ + if (netif_is_lag_master(master)) + return; + if ((dsa_port_master_is_operational(cpu_dp)) != (up && cpu_dp->master_oper_up)) notify = true; @@ -1323,6 +1319,12 @@ void dsa_tree_master_oper_state_change(struct dsa_switch_tree *dst, struct dsa_port *cpu_dp = master->dsa_ptr; bool notify = false; + /* Don't keep track of oper state on LAG DSA masters, + * but rather just of physical DSA masters + */ + if (netif_is_lag_master(master)) + return; + if ((dsa_port_master_is_operational(cpu_dp)) != (cpu_dp->master_admin_up && up)) notify = true; @@ -1791,7 +1793,7 @@ void dsa_switch_shutdown(struct dsa_switch *ds) rtnl_lock(); dsa_switch_for_each_user_port(dp, ds) { - master = dp->cpu_dp->master; + master = dsa_port_to_master(dp); slave_dev = dp->slave; netdev_upper_dev_unlink(master, slave_dev); diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index d9722e49864b06176fa781706e3f76c57f3bff3e..6e65c7ffd6f32f778b490d4fc94a7269104fb390 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -88,6 +88,7 @@ struct dsa_notifier_lag_info { const struct dsa_port *dp; struct dsa_lag lag; struct netdev_lag_upper_info *info; + struct netlink_ext_ack *extack; }; /* DSA_NOTIFIER_VLAN_* */ @@ -184,6 +185,11 @@ static inline int dsa_tag_protocol_overhead(const struct dsa_device_ops *ops) /* master.c */ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp); void dsa_master_teardown(struct net_device *dev); +int dsa_master_lag_setup(struct net_device *lag_dev, struct dsa_port *cpu_dp, + struct netdev_lag_upper_info *uinfo, + struct netlink_ext_ack *extack); +void dsa_master_lag_teardown(struct net_device *lag_dev, + struct dsa_port *cpu_dp); static inline struct net_device *dsa_master_find_slave(struct net_device *dev, int device, int port) @@ -200,6 +206,9 @@ static inline struct net_device *dsa_master_find_slave(struct net_device *dev, return NULL; } +/* netlink.c */ +extern struct rtnl_link_ops dsa_link_ops __read_mostly; + /* port.c */ void dsa_port_set_tag_protocol(struct dsa_port *cpu_dp, const struct dsa_device_ops *tag_ops); @@ -285,13 +294,16 @@ int dsa_port_mrp_add_ring_role(const struct dsa_port *dp, int dsa_port_mrp_del_ring_role(const struct dsa_port *dp, const struct switchdev_obj_ring_role_mrp *mrp); int dsa_port_phylink_create(struct dsa_port *dp); -int dsa_port_link_register_of(struct dsa_port *dp); -void dsa_port_link_unregister_of(struct dsa_port *dp); +void dsa_port_phylink_destroy(struct dsa_port *dp); +int dsa_shared_port_link_register_of(struct dsa_port *dp); +void dsa_shared_port_link_unregister_of(struct dsa_port *dp); int dsa_port_hsr_join(struct dsa_port *dp, struct net_device *hsr); void dsa_port_hsr_leave(struct dsa_port *dp, struct net_device *hsr); int dsa_port_tag_8021q_vlan_add(struct dsa_port *dp, u16 vid, bool broadcast); void dsa_port_tag_8021q_vlan_del(struct dsa_port *dp, u16 vid, bool broadcast); void dsa_port_set_host_flood(struct dsa_port *dp, bool uc, bool mc); +int dsa_port_change_master(struct dsa_port *dp, struct net_device *master, + struct netlink_ext_ack *extack); /* slave.c */ extern const struct dsa_device_ops notag_netdev_ops; @@ -305,8 +317,12 @@ int dsa_slave_suspend(struct net_device *slave_dev); int dsa_slave_resume(struct net_device *slave_dev); int dsa_slave_register_notifier(void); void dsa_slave_unregister_notifier(void); +void dsa_slave_sync_ha(struct net_device *dev); +void dsa_slave_unsync_ha(struct net_device *dev); void dsa_slave_setup_tagger(struct net_device *slave); int dsa_slave_change_mtu(struct net_device *dev, int new_mtu); +int dsa_slave_change_master(struct net_device *dev, struct net_device *master, + struct netlink_ext_ack *extack); int dsa_slave_manage_vlan_filtering(struct net_device *dev, bool vlan_filtering); @@ -322,7 +338,7 @@ dsa_slave_to_master(const struct net_device *dev) { struct dsa_port *dp = dsa_slave_to_port(dev); - return dp->cpu_dp->master; + return dsa_port_to_master(dp); } /* If under a bridge with vlan_filtering=0, make sure to send pvid-tagged @@ -542,10 +558,10 @@ void dsa_lag_map(struct dsa_switch_tree *dst, struct dsa_lag *lag); void dsa_lag_unmap(struct dsa_switch_tree *dst, struct dsa_lag *lag); struct dsa_lag *dsa_tree_lag_find(struct dsa_switch_tree *dst, const struct net_device *lag_dev); +struct net_device *dsa_tree_find_first_master(struct dsa_switch_tree *dst); int dsa_tree_notify(struct dsa_switch_tree *dst, unsigned long e, void *v); int dsa_broadcast(unsigned long e, void *v); int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, - struct net_device *master, const struct dsa_device_ops *tag_ops, const struct dsa_device_ops *old_tag_ops); void dsa_tree_master_admin_state_change(struct dsa_switch_tree *dst, diff --git a/net/dsa/master.c b/net/dsa/master.c index 2851e44c4cf0dc5860f7e60545f5c346c7ecf6bd..40367ab41cf8f7d7cdcda922f0dc0e21d2f532b4 100644 --- a/net/dsa/master.c +++ b/net/dsa/master.c @@ -58,7 +58,7 @@ static void dsa_master_get_regs(struct net_device *dev, } cpu_info = (struct ethtool_drvinfo *)data; - strlcpy(cpu_info->driver, "dsa", sizeof(cpu_info->driver)); + strscpy(cpu_info->driver, "dsa", sizeof(cpu_info->driver)); data += sizeof(*cpu_info); cpu_regs = (struct ethtool_regs *)data; data += sizeof(*cpu_regs); @@ -226,6 +226,9 @@ static int dsa_master_ethtool_setup(struct net_device *dev) struct dsa_switch *ds = cpu_dp->ds; struct ethtool_ops *ops; + if (netif_is_lag_master(dev)) + return 0; + ops = devm_kzalloc(ds->dev, sizeof(*ops), GFP_KERNEL); if (!ops) return -ENOMEM; @@ -250,6 +253,9 @@ static void dsa_master_ethtool_teardown(struct net_device *dev) { struct dsa_port *cpu_dp = dev->dsa_ptr; + if (netif_is_lag_master(dev)) + return; + dev->ethtool_ops = cpu_dp->orig_ethtool_ops; cpu_dp->orig_ethtool_ops = NULL; } @@ -257,6 +263,9 @@ static void dsa_master_ethtool_teardown(struct net_device *dev) static void dsa_netdev_ops_set(struct net_device *dev, const struct dsa_netdevice_ops *ops) { + if (netif_is_lag_master(dev)) + return; + dev->dsa_ptr->netdev_ops = ops; } @@ -307,7 +316,7 @@ static ssize_t tagging_store(struct device *d, struct device_attribute *attr, */ goto out; - err = dsa_tree_change_tag_proto(cpu_dp->ds->dst, dev, new_tag_ops, + err = dsa_tree_change_tag_proto(cpu_dp->ds->dst, new_tag_ops, old_tag_ops); if (err) { /* On failure the old tagger is restored, so we don't need the @@ -355,12 +364,14 @@ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp) mtu = ETH_DATA_LEN + dsa_tag_protocol_overhead(tag_ops); /* The DSA master must use SET_NETDEV_DEV for this to work. */ - consumer_link = device_link_add(ds->dev, dev->dev.parent, - DL_FLAG_AUTOREMOVE_CONSUMER); - if (!consumer_link) - netdev_err(dev, - "Failed to create a device link to DSA switch %s\n", - dev_name(ds->dev)); + if (!netif_is_lag_master(dev)) { + consumer_link = device_link_add(ds->dev, dev->dev.parent, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!consumer_link) + netdev_err(dev, + "Failed to create a device link to DSA switch %s\n", + dev_name(ds->dev)); + } /* The switch driver may not implement ->port_change_mtu(), case in * which dsa_slave_change_mtu() will not update the master MTU either, @@ -417,3 +428,52 @@ void dsa_master_teardown(struct net_device *dev) */ wmb(); } + +int dsa_master_lag_setup(struct net_device *lag_dev, struct dsa_port *cpu_dp, + struct netdev_lag_upper_info *uinfo, + struct netlink_ext_ack *extack) +{ + bool master_setup = false; + int err; + + if (!netdev_uses_dsa(lag_dev)) { + err = dsa_master_setup(lag_dev, cpu_dp); + if (err) + return err; + + master_setup = true; + } + + err = dsa_port_lag_join(cpu_dp, lag_dev, uinfo, extack); + if (err) { + if (extack && !extack->_msg) + NL_SET_ERR_MSG_MOD(extack, + "CPU port failed to join LAG"); + goto out_master_teardown; + } + + return 0; + +out_master_teardown: + if (master_setup) + dsa_master_teardown(lag_dev); + return err; +} + +/* Tear down a master if there isn't any other user port on it, + * optionally also destroying LAG information. + */ +void dsa_master_lag_teardown(struct net_device *lag_dev, + struct dsa_port *cpu_dp) +{ + struct net_device *upper; + struct list_head *iter; + + dsa_port_lag_leave(cpu_dp, lag_dev); + + netdev_for_each_upper_dev_rcu(lag_dev, upper, iter) + if (dsa_slave_dev_check(upper)) + return; + + dsa_master_teardown(lag_dev); +} diff --git a/net/dsa/netlink.c b/net/dsa/netlink.c new file mode 100644 index 0000000000000000000000000000000000000000..ecf9ed1de1857be64d82761e3fe31e0bf4bc8c98 --- /dev/null +++ b/net/dsa/netlink.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2022 NXP + */ +#include +#include + +#include "dsa_priv.h" + +static const struct nla_policy dsa_policy[IFLA_DSA_MAX + 1] = { + [IFLA_DSA_MASTER] = { .type = NLA_U32 }, +}; + +static int dsa_changelink(struct net_device *dev, struct nlattr *tb[], + struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + int err; + + if (!data) + return 0; + + if (data[IFLA_DSA_MASTER]) { + u32 ifindex = nla_get_u32(data[IFLA_DSA_MASTER]); + struct net_device *master; + + master = __dev_get_by_index(dev_net(dev), ifindex); + if (!master) + return -EINVAL; + + err = dsa_slave_change_master(dev, master, extack); + if (err) + return err; + } + + return 0; +} + +static size_t dsa_get_size(const struct net_device *dev) +{ + return nla_total_size(sizeof(u32)) + /* IFLA_DSA_MASTER */ + 0; +} + +static int dsa_fill_info(struct sk_buff *skb, const struct net_device *dev) +{ + struct net_device *master = dsa_slave_to_master(dev); + + if (nla_put_u32(skb, IFLA_DSA_MASTER, master->ifindex)) + return -EMSGSIZE; + + return 0; +} + +struct rtnl_link_ops dsa_link_ops __read_mostly = { + .kind = "dsa", + .priv_size = sizeof(struct dsa_port), + .maxtype = IFLA_DSA_MAX, + .policy = dsa_policy, + .changelink = dsa_changelink, + .get_size = dsa_get_size, + .fill_info = dsa_fill_info, + .netns_refund = true, +}; diff --git a/net/dsa/port.c b/net/dsa/port.c index a8895ee3cd600ca8cfd1cc88a6614a910cc76b7f..208168276995aba29afab8c29312454598dc19b2 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -634,6 +635,7 @@ int dsa_port_lag_join(struct dsa_port *dp, struct net_device *lag_dev, struct dsa_notifier_lag_info info = { .dp = dp, .info = uinfo, + .extack = extack, }; struct net_device *bridge_dev; int err; @@ -1026,7 +1028,7 @@ int dsa_port_standalone_host_fdb_add(struct dsa_port *dp, int dsa_port_bridge_host_fdb_add(struct dsa_port *dp, const unsigned char *addr, u16 vid) { - struct dsa_port *cpu_dp = dp->cpu_dp; + struct net_device *master = dsa_port_to_master(dp); struct dsa_db db = { .type = DSA_DB_BRIDGE, .bridge = *dp->bridge, @@ -1037,8 +1039,8 @@ int dsa_port_bridge_host_fdb_add(struct dsa_port *dp, * requires rtnl_lock(), since we can't guarantee that is held here, * and we can't take it either. */ - if (cpu_dp->master->priv_flags & IFF_UNICAST_FLT) { - err = dev_uc_add(cpu_dp->master, addr); + if (master->priv_flags & IFF_UNICAST_FLT) { + err = dev_uc_add(master, addr); if (err) return err; } @@ -1077,15 +1079,15 @@ int dsa_port_standalone_host_fdb_del(struct dsa_port *dp, int dsa_port_bridge_host_fdb_del(struct dsa_port *dp, const unsigned char *addr, u16 vid) { - struct dsa_port *cpu_dp = dp->cpu_dp; + struct net_device *master = dsa_port_to_master(dp); struct dsa_db db = { .type = DSA_DB_BRIDGE, .bridge = *dp->bridge, }; int err; - if (cpu_dp->master->priv_flags & IFF_UNICAST_FLT) { - err = dev_uc_del(cpu_dp->master, addr); + if (master->priv_flags & IFF_UNICAST_FLT) { + err = dev_uc_del(master, addr); if (err) return err; } @@ -1208,14 +1210,14 @@ int dsa_port_standalone_host_mdb_add(const struct dsa_port *dp, int dsa_port_bridge_host_mdb_add(const struct dsa_port *dp, const struct switchdev_obj_port_mdb *mdb) { - struct dsa_port *cpu_dp = dp->cpu_dp; + struct net_device *master = dsa_port_to_master(dp); struct dsa_db db = { .type = DSA_DB_BRIDGE, .bridge = *dp->bridge, }; int err; - err = dev_mc_add(cpu_dp->master, mdb->addr); + err = dev_mc_add(master, mdb->addr); if (err) return err; @@ -1252,14 +1254,14 @@ int dsa_port_standalone_host_mdb_del(const struct dsa_port *dp, int dsa_port_bridge_host_mdb_del(const struct dsa_port *dp, const struct switchdev_obj_port_mdb *mdb) { - struct dsa_port *cpu_dp = dp->cpu_dp; + struct net_device *master = dsa_port_to_master(dp); struct dsa_db db = { .type = DSA_DB_BRIDGE, .bridge = *dp->bridge, }; int err; - err = dev_mc_del(cpu_dp->master, mdb->addr); + err = dev_mc_del(master, mdb->addr); if (err) return err; @@ -1294,19 +1296,19 @@ int dsa_port_host_vlan_add(struct dsa_port *dp, const struct switchdev_obj_port_vlan *vlan, struct netlink_ext_ack *extack) { + struct net_device *master = dsa_port_to_master(dp); struct dsa_notifier_vlan_info info = { .dp = dp, .vlan = vlan, .extack = extack, }; - struct dsa_port *cpu_dp = dp->cpu_dp; int err; err = dsa_port_notify(dp, DSA_NOTIFIER_HOST_VLAN_ADD, &info); if (err && err != -EOPNOTSUPP) return err; - vlan_vid_add(cpu_dp->master, htons(ETH_P_8021Q), vlan->vid); + vlan_vid_add(master, htons(ETH_P_8021Q), vlan->vid); return err; } @@ -1314,18 +1316,18 @@ int dsa_port_host_vlan_add(struct dsa_port *dp, int dsa_port_host_vlan_del(struct dsa_port *dp, const struct switchdev_obj_port_vlan *vlan) { + struct net_device *master = dsa_port_to_master(dp); struct dsa_notifier_vlan_info info = { .dp = dp, .vlan = vlan, }; - struct dsa_port *cpu_dp = dp->cpu_dp; int err; err = dsa_port_notify(dp, DSA_NOTIFIER_HOST_VLAN_DEL, &info); if (err && err != -EOPNOTSUPP) return err; - vlan_vid_del(cpu_dp->master, htons(ETH_P_8021Q), vlan->vid); + vlan_vid_del(master, htons(ETH_P_8021Q), vlan->vid); return err; } @@ -1374,6 +1376,136 @@ int dsa_port_mrp_del_ring_role(const struct dsa_port *dp, return ds->ops->port_mrp_del_ring_role(ds, dp->index, mrp); } +static int dsa_port_assign_master(struct dsa_port *dp, + struct net_device *master, + struct netlink_ext_ack *extack, + bool fail_on_err) +{ + struct dsa_switch *ds = dp->ds; + int port = dp->index, err; + + err = ds->ops->port_change_master(ds, port, master, extack); + if (err && !fail_on_err) + dev_err(ds->dev, "port %d failed to assign master %s: %pe\n", + port, master->name, ERR_PTR(err)); + + if (err && fail_on_err) + return err; + + dp->cpu_dp = master->dsa_ptr; + dp->cpu_port_in_lag = netif_is_lag_master(master); + + return 0; +} + +/* Change the dp->cpu_dp affinity for a user port. Note that both cross-chip + * notifiers and drivers have implicit assumptions about user-to-CPU-port + * mappings, so we unfortunately cannot delay the deletion of the objects + * (switchdev, standalone addresses, standalone VLANs) on the old CPU port + * until the new CPU port has been set up. So we need to completely tear down + * the old CPU port before changing it, and restore it on errors during the + * bringup of the new one. + */ +int dsa_port_change_master(struct dsa_port *dp, struct net_device *master, + struct netlink_ext_ack *extack) +{ + struct net_device *bridge_dev = dsa_port_bridge_dev_get(dp); + struct net_device *old_master = dsa_port_to_master(dp); + struct net_device *dev = dp->slave; + struct dsa_switch *ds = dp->ds; + bool vlan_filtering; + int err, tmp; + + /* Bridges may hold host FDB, MDB and VLAN objects. These need to be + * migrated, so dynamically unoffload and later reoffload the bridge + * port. + */ + if (bridge_dev) { + dsa_port_pre_bridge_leave(dp, bridge_dev); + dsa_port_bridge_leave(dp, bridge_dev); + } + + /* The port might still be VLAN filtering even if it's no longer + * under a bridge, either due to ds->vlan_filtering_is_global or + * ds->needs_standalone_vlan_filtering. In turn this means VLANs + * on the CPU port. + */ + vlan_filtering = dsa_port_is_vlan_filtering(dp); + if (vlan_filtering) { + err = dsa_slave_manage_vlan_filtering(dev, false); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to remove standalone VLANs"); + goto rewind_old_bridge; + } + } + + /* Standalone addresses, and addresses of upper interfaces like + * VLAN, LAG, HSR need to be migrated. + */ + dsa_slave_unsync_ha(dev); + + err = dsa_port_assign_master(dp, master, extack, true); + if (err) + goto rewind_old_addrs; + + dsa_slave_sync_ha(dev); + + if (vlan_filtering) { + err = dsa_slave_manage_vlan_filtering(dev, true); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to restore standalone VLANs"); + goto rewind_new_addrs; + } + } + + if (bridge_dev) { + err = dsa_port_bridge_join(dp, bridge_dev, extack); + if (err && err == -EOPNOTSUPP) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to reoffload bridge"); + goto rewind_new_vlan; + } + } + + return 0; + +rewind_new_vlan: + if (vlan_filtering) + dsa_slave_manage_vlan_filtering(dev, false); + +rewind_new_addrs: + dsa_slave_unsync_ha(dev); + + dsa_port_assign_master(dp, old_master, NULL, false); + +/* Restore the objects on the old CPU port */ +rewind_old_addrs: + dsa_slave_sync_ha(dev); + + if (vlan_filtering) { + tmp = dsa_slave_manage_vlan_filtering(dev, true); + if (tmp) { + dev_err(ds->dev, + "port %d failed to restore standalone VLANs: %pe\n", + dp->index, ERR_PTR(tmp)); + } + } + +rewind_old_bridge: + if (bridge_dev) { + tmp = dsa_port_bridge_join(dp, bridge_dev, extack); + if (tmp) { + dev_err(ds->dev, + "port %d failed to rejoin bridge %s: %pe\n", + dp->index, bridge_dev->name, ERR_PTR(tmp)); + } + } + + return err; +} + void dsa_port_set_tag_protocol(struct dsa_port *cpu_dp, const struct dsa_device_ops *tag_ops) { @@ -1529,6 +1661,7 @@ int dsa_port_phylink_create(struct dsa_port *dp) { struct dsa_switch *ds = dp->ds; phy_interface_t mode; + struct phylink *pl; int err; err = of_get_phy_mode(dp->dn, &mode); @@ -1545,17 +1678,25 @@ int dsa_port_phylink_create(struct dsa_port *dp) if (ds->ops->phylink_get_caps) ds->ops->phylink_get_caps(ds, dp->index, &dp->pl_config); - dp->pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn), - mode, &dsa_port_phylink_mac_ops); - if (IS_ERR(dp->pl)) { - pr_err("error creating PHYLINK: %ld\n", PTR_ERR(dp->pl)); - return PTR_ERR(dp->pl); + pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn), + mode, &dsa_port_phylink_mac_ops); + if (IS_ERR(pl)) { + pr_err("error creating PHYLINK: %ld\n", PTR_ERR(pl)); + return PTR_ERR(pl); } + dp->pl = pl; + return 0; } -static int dsa_port_setup_phy_of(struct dsa_port *dp, bool enable) +void dsa_port_phylink_destroy(struct dsa_port *dp) +{ + phylink_destroy(dp->pl); + dp->pl = NULL; +} + +static int dsa_shared_port_setup_phy_of(struct dsa_port *dp, bool enable) { struct dsa_switch *ds = dp->ds; struct phy_device *phydev; @@ -1593,7 +1734,7 @@ err_put_dev: return err; } -static int dsa_port_fixed_link_register_of(struct dsa_port *dp) +static int dsa_shared_port_fixed_link_register_of(struct dsa_port *dp) { struct device_node *dn = dp->dn; struct dsa_switch *ds = dp->ds; @@ -1627,7 +1768,7 @@ static int dsa_port_fixed_link_register_of(struct dsa_port *dp) return 0; } -static int dsa_port_phylink_register(struct dsa_port *dp) +static int dsa_shared_port_phylink_register(struct dsa_port *dp) { struct dsa_switch *ds = dp->ds; struct device_node *port_dn = dp->dn; @@ -1649,26 +1790,188 @@ static int dsa_port_phylink_register(struct dsa_port *dp) return 0; err_phy_connect: - phylink_destroy(dp->pl); + dsa_port_phylink_destroy(dp); return err; } -int dsa_port_link_register_of(struct dsa_port *dp) +/* During the initial DSA driver migration to OF, port nodes were sometimes + * added to device trees with no indication of how they should operate from a + * link management perspective (phy-handle, fixed-link, etc). Additionally, the + * phy-mode may be absent. The interpretation of these port OF nodes depends on + * their type. + * + * User ports with no phy-handle or fixed-link are expected to connect to an + * internal PHY located on the ds->slave_mii_bus at an MDIO address equal to + * the port number. This description is still actively supported. + * + * Shared (CPU and DSA) ports with no phy-handle or fixed-link are expected to + * operate at the maximum speed that their phy-mode is capable of. If the + * phy-mode is absent, they are expected to operate using the phy-mode + * supported by the port that gives the highest link speed. It is unspecified + * if the port should use flow control or not, half duplex or full duplex, or + * if the phy-mode is a SERDES link, whether in-band autoneg is expected to be + * enabled or not. + * + * In the latter case of shared ports, omitting the link management description + * from the firmware node is deprecated and strongly discouraged. DSA uses + * phylink, which rejects the firmware nodes of these ports for lacking + * required properties. + * + * For switches in this table, DSA will skip enforcing validation and will + * later omit registering a phylink instance for the shared ports, if they lack + * a fixed-link, a phy-handle, or a managed = "in-band-status" property. + * It becomes the responsibility of the driver to ensure that these ports + * operate at the maximum speed (whatever this means) and will interoperate + * with the DSA master or other cascade port, since phylink methods will not be + * invoked for them. + * + * If you are considering expanding this table for newly introduced switches, + * think again. It is OK to remove switches from this table if there aren't DT + * blobs in circulation which rely on defaulting the shared ports. + */ +static const char * const dsa_switches_apply_workarounds[] = { +#if IS_ENABLED(CONFIG_NET_DSA_XRS700X) + "arrow,xrs7003e", + "arrow,xrs7003f", + "arrow,xrs7004e", + "arrow,xrs7004f", +#endif +#if IS_ENABLED(CONFIG_B53) + "brcm,bcm5325", + "brcm,bcm53115", + "brcm,bcm53125", + "brcm,bcm53128", + "brcm,bcm5365", + "brcm,bcm5389", + "brcm,bcm5395", + "brcm,bcm5397", + "brcm,bcm5398", + "brcm,bcm53010-srab", + "brcm,bcm53011-srab", + "brcm,bcm53012-srab", + "brcm,bcm53018-srab", + "brcm,bcm53019-srab", + "brcm,bcm5301x-srab", + "brcm,bcm11360-srab", + "brcm,bcm58522-srab", + "brcm,bcm58525-srab", + "brcm,bcm58535-srab", + "brcm,bcm58622-srab", + "brcm,bcm58623-srab", + "brcm,bcm58625-srab", + "brcm,bcm88312-srab", + "brcm,cygnus-srab", + "brcm,nsp-srab", + "brcm,omega-srab", + "brcm,bcm3384-switch", + "brcm,bcm6328-switch", + "brcm,bcm6368-switch", + "brcm,bcm63xx-switch", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_BCM_SF2) + "brcm,bcm7445-switch-v4.0", + "brcm,bcm7278-switch-v4.0", + "brcm,bcm7278-switch-v4.8", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_LANTIQ_GSWIP) + "lantiq,xrx200-gswip", + "lantiq,xrx300-gswip", + "lantiq,xrx330-gswip", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_MV88E6060) + "marvell,mv88e6060", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_MV88E6XXX) + "marvell,mv88e6085", + "marvell,mv88e6190", + "marvell,mv88e6250", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON) + "microchip,ksz8765", + "microchip,ksz8794", + "microchip,ksz8795", + "microchip,ksz8863", + "microchip,ksz8873", + "microchip,ksz9477", + "microchip,ksz9897", + "microchip,ksz9893", + "microchip,ksz9563", + "microchip,ksz8563", + "microchip,ksz9567", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_SMSC_LAN9303_MDIO) + "smsc,lan9303-mdio", +#endif +#if IS_ENABLED(CONFIG_NET_DSA_SMSC_LAN9303_I2C) + "smsc,lan9303-i2c", +#endif + NULL, +}; + +static void dsa_shared_port_validate_of(struct dsa_port *dp, + bool *missing_phy_mode, + bool *missing_link_description) { + struct device_node *dn = dp->dn, *phy_np; struct dsa_switch *ds = dp->ds; - struct device_node *phy_np; + phy_interface_t mode; + + *missing_phy_mode = false; + *missing_link_description = false; + + if (of_get_phy_mode(dn, &mode)) { + *missing_phy_mode = true; + dev_err(ds->dev, + "OF node %pOF of %s port %d lacks the required \"phy-mode\" property\n", + dn, dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index); + } + + /* Note: of_phy_is_fixed_link() also returns true for + * managed = "in-band-status" + */ + if (of_phy_is_fixed_link(dn)) + return; + + phy_np = of_parse_phandle(dn, "phy-handle", 0); + if (phy_np) { + of_node_put(phy_np); + return; + } + + *missing_link_description = true; + + dev_err(ds->dev, + "OF node %pOF of %s port %d lacks the required \"phy-handle\", \"fixed-link\" or \"managed\" properties\n", + dn, dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index); +} + +int dsa_shared_port_link_register_of(struct dsa_port *dp) +{ + struct dsa_switch *ds = dp->ds; + bool missing_link_description; + bool missing_phy_mode; int port = dp->index; + dsa_shared_port_validate_of(dp, &missing_phy_mode, + &missing_link_description); + + if ((missing_phy_mode || missing_link_description) && + !of_device_compatible_match(ds->dev->of_node, + dsa_switches_apply_workarounds)) + return -EINVAL; + if (!ds->ops->adjust_link) { - phy_np = of_parse_phandle(dp->dn, "phy-handle", 0); - if (of_phy_is_fixed_link(dp->dn) || phy_np) { + if (missing_link_description) { + dev_warn(ds->dev, + "Skipping phylink registration for %s port %d\n", + dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index); + } else { if (ds->ops->phylink_mac_link_down) ds->ops->phylink_mac_link_down(ds, port, MLO_AN_FIXED, PHY_INTERFACE_MODE_NA); - of_node_put(phy_np); - return dsa_port_phylink_register(dp); + + return dsa_shared_port_phylink_register(dp); } - of_node_put(phy_np); return 0; } @@ -1676,12 +1979,12 @@ int dsa_port_link_register_of(struct dsa_port *dp) "Using legacy PHYLIB callbacks. Please migrate to PHYLINK!\n"); if (of_phy_is_fixed_link(dp->dn)) - return dsa_port_fixed_link_register_of(dp); + return dsa_shared_port_fixed_link_register_of(dp); else - return dsa_port_setup_phy_of(dp, true); + return dsa_shared_port_setup_phy_of(dp, true); } -void dsa_port_link_unregister_of(struct dsa_port *dp) +void dsa_shared_port_link_unregister_of(struct dsa_port *dp) { struct dsa_switch *ds = dp->ds; @@ -1689,15 +1992,14 @@ void dsa_port_link_unregister_of(struct dsa_port *dp) rtnl_lock(); phylink_disconnect_phy(dp->pl); rtnl_unlock(); - phylink_destroy(dp->pl); - dp->pl = NULL; + dsa_port_phylink_destroy(dp); return; } if (of_phy_is_fixed_link(dp->dn)) of_phy_deregister_fixed_link(dp->dn); else - dsa_port_setup_phy_of(dp, false); + dsa_shared_port_setup_phy_of(dp, false); } int dsa_port_hsr_join(struct dsa_port *dp, struct net_device *hsr) diff --git a/net/dsa/slave.c b/net/dsa/slave.c index ad6a6663feeb5ecce4559dcb783608edb82030c2..1a59918d3b3051e5656742b990f3621cd6ce0428 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -164,6 +164,48 @@ static int dsa_slave_unsync_mc(struct net_device *dev, return dsa_slave_schedule_standalone_work(dev, DSA_MC_DEL, addr, 0); } +void dsa_slave_sync_ha(struct net_device *dev) +{ + struct dsa_port *dp = dsa_slave_to_port(dev); + struct dsa_switch *ds = dp->ds; + struct netdev_hw_addr *ha; + + netif_addr_lock_bh(dev); + + netdev_for_each_synced_mc_addr(ha, dev) + dsa_slave_sync_mc(dev, ha->addr); + + netdev_for_each_synced_uc_addr(ha, dev) + dsa_slave_sync_uc(dev, ha->addr); + + netif_addr_unlock_bh(dev); + + if (dsa_switch_supports_uc_filtering(ds) || + dsa_switch_supports_mc_filtering(ds)) + dsa_flush_workqueue(); +} + +void dsa_slave_unsync_ha(struct net_device *dev) +{ + struct dsa_port *dp = dsa_slave_to_port(dev); + struct dsa_switch *ds = dp->ds; + struct netdev_hw_addr *ha; + + netif_addr_lock_bh(dev); + + netdev_for_each_synced_uc_addr(ha, dev) + dsa_slave_unsync_uc(dev, ha->addr); + + netdev_for_each_synced_mc_addr(ha, dev) + dsa_slave_unsync_mc(dev, ha->addr); + + netif_addr_unlock_bh(dev); + + if (dsa_switch_supports_uc_filtering(ds) || + dsa_switch_supports_mc_filtering(ds)) + dsa_flush_workqueue(); +} + /* slave mii_bus handling ***************************************************/ static int dsa_slave_phy_read(struct mii_bus *bus, int addr, int reg) { @@ -826,9 +868,9 @@ static netdev_tx_t dsa_slave_xmit(struct sk_buff *skb, struct net_device *dev) static void dsa_slave_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo) { - strlcpy(drvinfo->driver, "dsa", sizeof(drvinfo->driver)); - strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); - strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); + strscpy(drvinfo->driver, "dsa", sizeof(drvinfo->driver)); + strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version)); + strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info)); } static int dsa_slave_get_regs_len(struct net_device *dev) @@ -1503,8 +1545,7 @@ static int dsa_slave_setup_tc_block(struct net_device *dev, static int dsa_slave_setup_ft_block(struct dsa_switch *ds, int port, void *type_data) { - struct dsa_port *cpu_dp = dsa_to_port(ds, port)->cpu_dp; - struct net_device *master = cpu_dp->master; + struct net_device *master = dsa_port_to_master(dsa_to_port(ds, port)); if (!master->netdev_ops->ndo_setup_tc) return -EOPNOTSUPP; @@ -2147,13 +2188,14 @@ static int dsa_slave_fill_forward_path(struct net_device_path_ctx *ctx, struct net_device_path *path) { struct dsa_port *dp = dsa_slave_to_port(ctx->dev); + struct net_device *master = dsa_port_to_master(dp); struct dsa_port *cpu_dp = dp->cpu_dp; path->dev = ctx->dev; path->type = DEV_PATH_DSA; path->dsa.proto = cpu_dp->tag_ops->proto; path->dsa.port = dp->index; - ctx->dev = cpu_dp->master; + ctx->dev = master; return 0; } @@ -2262,7 +2304,7 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev) if (ret) { netdev_err(slave_dev, "failed to connect to PHY: %pe\n", ERR_PTR(ret)); - phylink_destroy(dp->pl); + dsa_port_phylink_destroy(dp); } return ret; @@ -2271,9 +2313,9 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev) void dsa_slave_setup_tagger(struct net_device *slave) { struct dsa_port *dp = dsa_slave_to_port(slave); + struct net_device *master = dsa_port_to_master(dp); struct dsa_slave_priv *p = netdev_priv(slave); const struct dsa_port *cpu_dp = dp->cpu_dp; - struct net_device *master = cpu_dp->master; const struct dsa_switch *ds = dp->ds; slave->needed_headroom = cpu_dp->tag_ops->needed_headroom; @@ -2330,8 +2372,7 @@ int dsa_slave_resume(struct net_device *slave_dev) int dsa_slave_create(struct dsa_port *port) { - const struct dsa_port *cpu_dp = port->cpu_dp; - struct net_device *master = cpu_dp->master; + struct net_device *master = dsa_port_to_master(port); struct dsa_switch *ds = port->ds; const char *name = port->name; struct net_device *slave_dev; @@ -2347,6 +2388,7 @@ int dsa_slave_create(struct dsa_port *port) if (slave_dev == NULL) return -ENOMEM; + slave_dev->rtnl_link_ops = &dsa_link_ops; slave_dev->ethtool_ops = &dsa_slave_ethtool_ops; #if IS_ENABLED(CONFIG_DCB) slave_dev->dcbnl_ops = &dsa_slave_dcbnl_ops; @@ -2434,7 +2476,7 @@ out_phy: rtnl_lock(); phylink_disconnect_phy(p->dp->pl); rtnl_unlock(); - phylink_destroy(p->dp->pl); + dsa_port_phylink_destroy(p->dp); out_gcells: gro_cells_destroy(&p->gcells); out_free: @@ -2457,12 +2499,89 @@ void dsa_slave_destroy(struct net_device *slave_dev) phylink_disconnect_phy(dp->pl); rtnl_unlock(); - phylink_destroy(dp->pl); + dsa_port_phylink_destroy(dp); gro_cells_destroy(&p->gcells); free_percpu(slave_dev->tstats); free_netdev(slave_dev); } +int dsa_slave_change_master(struct net_device *dev, struct net_device *master, + struct netlink_ext_ack *extack) +{ + struct net_device *old_master = dsa_slave_to_master(dev); + struct dsa_port *dp = dsa_slave_to_port(dev); + struct dsa_switch *ds = dp->ds; + struct net_device *upper; + struct list_head *iter; + int err; + + if (master == old_master) + return 0; + + if (!ds->ops->port_change_master) { + NL_SET_ERR_MSG_MOD(extack, + "Driver does not support changing DSA master"); + return -EOPNOTSUPP; + } + + if (!netdev_uses_dsa(master)) { + NL_SET_ERR_MSG_MOD(extack, + "Interface not eligible as DSA master"); + return -EOPNOTSUPP; + } + + netdev_for_each_upper_dev_rcu(master, upper, iter) { + if (dsa_slave_dev_check(upper)) + continue; + if (netif_is_bridge_master(upper)) + continue; + NL_SET_ERR_MSG_MOD(extack, "Cannot join master with unknown uppers"); + return -EOPNOTSUPP; + } + + /* Since we allow live-changing the DSA master, plus we auto-open the + * DSA master when the user port opens => we need to ensure that the + * new DSA master is open too. + */ + if (dev->flags & IFF_UP) { + err = dev_open(master, extack); + if (err) + return err; + } + + netdev_upper_dev_unlink(old_master, dev); + + err = netdev_upper_dev_link(master, dev, extack); + if (err) + goto out_revert_old_master_unlink; + + err = dsa_port_change_master(dp, master, extack); + if (err) + goto out_revert_master_link; + + /* Update the MTU of the new CPU port through cross-chip notifiers */ + err = dsa_slave_change_mtu(dev, dev->mtu); + if (err && err != -EOPNOTSUPP) { + netdev_warn(dev, + "nonfatal error updating MTU with new master: %pe\n", + ERR_PTR(err)); + } + + /* If the port doesn't have its own MAC address and relies on the DSA + * master's one, inherit it again from the new DSA master. + */ + if (is_zero_ether_addr(dp->mac)) + eth_hw_addr_inherit(dev, master); + + return 0; + +out_revert_master_link: + netdev_upper_dev_unlink(master, dev); +out_revert_old_master_unlink: + netdev_upper_dev_link(old_master, dev, NULL); + return err; +} + bool dsa_slave_dev_check(const struct net_device *dev) { return dev->netdev_ops == &dsa_slave_netdev_ops; @@ -2476,6 +2595,9 @@ static int dsa_slave_changeupper(struct net_device *dev, struct netlink_ext_ack *extack; int err = NOTIFY_DONE; + if (!dsa_slave_dev_check(dev)) + return err; + extack = netdev_notifier_info_to_extack(&info->info); if (netif_is_bridge_master(info->upper_dev)) { @@ -2484,7 +2606,7 @@ static int dsa_slave_changeupper(struct net_device *dev, if (!err) dsa_bridge_mtu_normalization(dp); if (err == -EOPNOTSUPP) { - if (!extack->_msg) + if (extack && !extack->_msg) NL_SET_ERR_MSG_MOD(extack, "Offloading not supported"); err = 0; @@ -2531,6 +2653,9 @@ static int dsa_slave_prechangeupper(struct net_device *dev, { struct dsa_port *dp = dsa_slave_to_port(dev); + if (!dsa_slave_dev_check(dev)) + return NOTIFY_DONE; + if (netif_is_bridge_master(info->upper_dev) && !info->linking) dsa_port_pre_bridge_leave(dp, info->upper_dev); else if (netif_is_lag_master(info->upper_dev) && !info->linking) @@ -2551,6 +2676,9 @@ dsa_slave_lag_changeupper(struct net_device *dev, int err = NOTIFY_DONE; struct dsa_port *dp; + if (!netif_is_lag_master(dev)) + return err; + netdev_for_each_lower_dev(dev, lower, iter) { if (!dsa_slave_dev_check(lower)) continue; @@ -2580,6 +2708,9 @@ dsa_slave_lag_prechangeupper(struct net_device *dev, int err = NOTIFY_DONE; struct dsa_port *dp; + if (!netif_is_lag_master(dev)) + return err; + netdev_for_each_lower_dev(dev, lower, iter) { if (!dsa_slave_dev_check(lower)) continue; @@ -2687,6 +2818,277 @@ dsa_slave_prechangeupper_sanity_check(struct net_device *dev, return NOTIFY_DONE; } +/* To be eligible as a DSA master, a LAG must have all lower interfaces be + * eligible DSA masters. Additionally, all LAG slaves must be DSA masters of + * switches in the same switch tree. + */ +static int dsa_lag_master_validate(struct net_device *lag_dev, + struct netlink_ext_ack *extack) +{ + struct net_device *lower1, *lower2; + struct list_head *iter1, *iter2; + + netdev_for_each_lower_dev(lag_dev, lower1, iter1) { + netdev_for_each_lower_dev(lag_dev, lower2, iter2) { + if (!netdev_uses_dsa(lower1) || + !netdev_uses_dsa(lower2)) { + NL_SET_ERR_MSG_MOD(extack, + "All LAG ports must be eligible as DSA masters"); + return notifier_from_errno(-EINVAL); + } + + if (lower1 == lower2) + continue; + + if (!dsa_port_tree_same(lower1->dsa_ptr, + lower2->dsa_ptr)) { + NL_SET_ERR_MSG_MOD(extack, + "LAG contains DSA masters of disjoint switch trees"); + return notifier_from_errno(-EINVAL); + } + } + } + + return NOTIFY_DONE; +} + +static int +dsa_master_prechangeupper_sanity_check(struct net_device *master, + struct netdev_notifier_changeupper_info *info) +{ + struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(&info->info); + + if (!netdev_uses_dsa(master)) + return NOTIFY_DONE; + + if (!info->linking) + return NOTIFY_DONE; + + /* Allow DSA switch uppers */ + if (dsa_slave_dev_check(info->upper_dev)) + return NOTIFY_DONE; + + /* Allow bridge uppers of DSA masters, subject to further + * restrictions in dsa_bridge_prechangelower_sanity_check() + */ + if (netif_is_bridge_master(info->upper_dev)) + return NOTIFY_DONE; + + /* Allow LAG uppers, subject to further restrictions in + * dsa_lag_master_prechangelower_sanity_check() + */ + if (netif_is_lag_master(info->upper_dev)) + return dsa_lag_master_validate(info->upper_dev, extack); + + NL_SET_ERR_MSG_MOD(extack, + "DSA master cannot join unknown upper interfaces"); + return notifier_from_errno(-EBUSY); +} + +static int +dsa_lag_master_prechangelower_sanity_check(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(&info->info); + struct net_device *lag_dev = info->upper_dev; + struct net_device *lower; + struct list_head *iter; + + if (!netdev_uses_dsa(lag_dev) || !netif_is_lag_master(lag_dev)) + return NOTIFY_DONE; + + if (!info->linking) + return NOTIFY_DONE; + + if (!netdev_uses_dsa(dev)) { + NL_SET_ERR_MSG(extack, + "Only DSA masters can join a LAG DSA master"); + return notifier_from_errno(-EINVAL); + } + + netdev_for_each_lower_dev(lag_dev, lower, iter) { + if (!dsa_port_tree_same(dev->dsa_ptr, lower->dsa_ptr)) { + NL_SET_ERR_MSG(extack, + "Interface is DSA master for a different switch tree than this LAG"); + return notifier_from_errno(-EINVAL); + } + + break; + } + + return NOTIFY_DONE; +} + +/* Don't allow bridging of DSA masters, since the bridge layer rx_handler + * prevents the DSA fake ethertype handler to be invoked, so we don't get the + * chance to strip off and parse the DSA switch tag protocol header (the bridge + * layer just returns RX_HANDLER_CONSUMED, stopping RX processing for these + * frames). + * The only case where that would not be an issue is when bridging can already + * be offloaded, such as when the DSA master is itself a DSA or plain switchdev + * port, and is bridged only with other ports from the same hardware device. + */ +static int +dsa_bridge_prechangelower_sanity_check(struct net_device *new_lower, + struct netdev_notifier_changeupper_info *info) +{ + struct net_device *br = info->upper_dev; + struct netlink_ext_ack *extack; + struct net_device *lower; + struct list_head *iter; + + if (!netif_is_bridge_master(br)) + return NOTIFY_DONE; + + if (!info->linking) + return NOTIFY_DONE; + + extack = netdev_notifier_info_to_extack(&info->info); + + netdev_for_each_lower_dev(br, lower, iter) { + if (!netdev_uses_dsa(new_lower) && !netdev_uses_dsa(lower)) + continue; + + if (!netdev_port_same_parent_id(lower, new_lower)) { + NL_SET_ERR_MSG(extack, + "Cannot do software bridging with a DSA master"); + return notifier_from_errno(-EINVAL); + } + } + + return NOTIFY_DONE; +} + +static void dsa_tree_migrate_ports_from_lag_master(struct dsa_switch_tree *dst, + struct net_device *lag_dev) +{ + struct net_device *new_master = dsa_tree_find_first_master(dst); + struct dsa_port *dp; + int err; + + dsa_tree_for_each_user_port(dp, dst) { + if (dsa_port_to_master(dp) != lag_dev) + continue; + + err = dsa_slave_change_master(dp->slave, new_master, NULL); + if (err) { + netdev_err(dp->slave, + "failed to restore master to %s: %pe\n", + new_master->name, ERR_PTR(err)); + } + } +} + +static int dsa_master_lag_join(struct net_device *master, + struct net_device *lag_dev, + struct netdev_lag_upper_info *uinfo, + struct netlink_ext_ack *extack) +{ + struct dsa_port *cpu_dp = master->dsa_ptr; + struct dsa_switch_tree *dst = cpu_dp->dst; + struct dsa_port *dp; + int err; + + err = dsa_master_lag_setup(lag_dev, cpu_dp, uinfo, extack); + if (err) + return err; + + dsa_tree_for_each_user_port(dp, dst) { + if (dsa_port_to_master(dp) != master) + continue; + + err = dsa_slave_change_master(dp->slave, lag_dev, extack); + if (err) + goto restore; + } + + return 0; + +restore: + dsa_tree_for_each_user_port_continue_reverse(dp, dst) { + if (dsa_port_to_master(dp) != lag_dev) + continue; + + err = dsa_slave_change_master(dp->slave, master, NULL); + if (err) { + netdev_err(dp->slave, + "failed to restore master to %s: %pe\n", + master->name, ERR_PTR(err)); + } + } + + dsa_master_lag_teardown(lag_dev, master->dsa_ptr); + + return err; +} + +static void dsa_master_lag_leave(struct net_device *master, + struct net_device *lag_dev) +{ + struct dsa_port *dp, *cpu_dp = lag_dev->dsa_ptr; + struct dsa_switch_tree *dst = cpu_dp->dst; + struct dsa_port *new_cpu_dp = NULL; + struct net_device *lower; + struct list_head *iter; + + netdev_for_each_lower_dev(lag_dev, lower, iter) { + if (netdev_uses_dsa(lower)) { + new_cpu_dp = lower->dsa_ptr; + break; + } + } + + if (new_cpu_dp) { + /* Update the CPU port of the user ports still under the LAG + * so that dsa_port_to_master() continues to work properly + */ + dsa_tree_for_each_user_port(dp, dst) + if (dsa_port_to_master(dp) == lag_dev) + dp->cpu_dp = new_cpu_dp; + + /* Update the index of the virtual CPU port to match the lowest + * physical CPU port + */ + lag_dev->dsa_ptr = new_cpu_dp; + wmb(); + } else { + /* If the LAG DSA master has no ports left, migrate back all + * user ports to the first physical CPU port + */ + dsa_tree_migrate_ports_from_lag_master(dst, lag_dev); + } + + /* This DSA master has left its LAG in any case, so let + * the CPU port leave the hardware LAG as well + */ + dsa_master_lag_teardown(lag_dev, master->dsa_ptr); +} + +static int dsa_master_changeupper(struct net_device *dev, + struct netdev_notifier_changeupper_info *info) +{ + struct netlink_ext_ack *extack; + int err = NOTIFY_DONE; + + if (!netdev_uses_dsa(dev)) + return err; + + extack = netdev_notifier_info_to_extack(&info->info); + + if (netif_is_lag_master(info->upper_dev)) { + if (info->linking) { + err = dsa_master_lag_join(dev, info->upper_dev, + info->upper_info, extack); + err = notifier_from_errno(err); + } else { + dsa_master_lag_leave(dev, info->upper_dev); + err = NOTIFY_OK; + } + } + + return err; +} + static int dsa_slave_netdevice_event(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -2698,36 +3100,68 @@ static int dsa_slave_netdevice_event(struct notifier_block *nb, int err; err = dsa_slave_prechangeupper_sanity_check(dev, info); - if (err != NOTIFY_DONE) + if (notifier_to_errno(err)) + return err; + + err = dsa_master_prechangeupper_sanity_check(dev, info); + if (notifier_to_errno(err)) + return err; + + err = dsa_lag_master_prechangelower_sanity_check(dev, info); + if (notifier_to_errno(err)) + return err; + + err = dsa_bridge_prechangelower_sanity_check(dev, info); + if (notifier_to_errno(err)) return err; - if (dsa_slave_dev_check(dev)) - return dsa_slave_prechangeupper(dev, ptr); + err = dsa_slave_prechangeupper(dev, ptr); + if (notifier_to_errno(err)) + return err; - if (netif_is_lag_master(dev)) - return dsa_slave_lag_prechangeupper(dev, ptr); + err = dsa_slave_lag_prechangeupper(dev, ptr); + if (notifier_to_errno(err)) + return err; break; } - case NETDEV_CHANGEUPPER: - if (dsa_slave_dev_check(dev)) - return dsa_slave_changeupper(dev, ptr); + case NETDEV_CHANGEUPPER: { + int err; - if (netif_is_lag_master(dev)) - return dsa_slave_lag_changeupper(dev, ptr); + err = dsa_slave_changeupper(dev, ptr); + if (notifier_to_errno(err)) + return err; + + err = dsa_slave_lag_changeupper(dev, ptr); + if (notifier_to_errno(err)) + return err; + + err = dsa_master_changeupper(dev, ptr); + if (notifier_to_errno(err)) + return err; break; + } case NETDEV_CHANGELOWERSTATE: { struct netdev_notifier_changelowerstate_info *info = ptr; struct dsa_port *dp; int err; - if (!dsa_slave_dev_check(dev)) - break; + if (dsa_slave_dev_check(dev)) { + dp = dsa_slave_to_port(dev); - dp = dsa_slave_to_port(dev); + err = dsa_port_lag_change(dp, info->lower_state_info); + } + + /* Mirror LAG port events on DSA masters that are in + * a LAG towards their respective switch CPU ports + */ + if (netdev_uses_dsa(dev)) { + dp = dev->dsa_ptr; + + err = dsa_port_lag_change(dp, info->lower_state_info); + } - err = dsa_port_lag_change(dp, info->lower_state_info); return notifier_from_errno(err); } case NETDEV_CHANGE: @@ -2777,6 +3211,9 @@ static int dsa_slave_netdevice_event(struct notifier_block *nb, if (!dsa_port_is_user(dp)) continue; + if (dp->cpu_dp != cpu_dp) + continue; + list_add(&dp->slave->close_list, &close_list); } diff --git a/net/dsa/switch.c b/net/dsa/switch.c index 4dfd68cf61c5a32184c1408816431a5cf80cafcd..ce56acdba2031e79e4396c8e9306c00c89d895b4 100644 --- a/net/dsa/switch.c +++ b/net/dsa/switch.c @@ -398,8 +398,15 @@ static int dsa_switch_host_fdb_add(struct dsa_switch *ds, dsa_switch_for_each_port(dp, ds) { if (dsa_port_host_address_match(dp, info->dp)) { - err = dsa_port_do_fdb_add(dp, info->addr, info->vid, - info->db); + if (dsa_port_is_cpu(dp) && info->dp->cpu_port_in_lag) { + err = dsa_switch_do_lag_fdb_add(ds, dp->lag, + info->addr, + info->vid, + info->db); + } else { + err = dsa_port_do_fdb_add(dp, info->addr, + info->vid, info->db); + } if (err) break; } @@ -419,8 +426,15 @@ static int dsa_switch_host_fdb_del(struct dsa_switch *ds, dsa_switch_for_each_port(dp, ds) { if (dsa_port_host_address_match(dp, info->dp)) { - err = dsa_port_do_fdb_del(dp, info->addr, info->vid, - info->db); + if (dsa_port_is_cpu(dp) && info->dp->cpu_port_in_lag) { + err = dsa_switch_do_lag_fdb_del(ds, dp->lag, + info->addr, + info->vid, + info->db); + } else { + err = dsa_port_do_fdb_del(dp, info->addr, + info->vid, info->db); + } if (err) break; } @@ -507,12 +521,12 @@ static int dsa_switch_lag_join(struct dsa_switch *ds, { if (info->dp->ds == ds && ds->ops->port_lag_join) return ds->ops->port_lag_join(ds, info->dp->index, info->lag, - info->info); + info->info, info->extack); if (info->dp->ds != ds && ds->ops->crosschip_lag_join) return ds->ops->crosschip_lag_join(ds, info->dp->ds->index, info->dp->index, info->lag, - info->info); + info->info, info->extack); return -EOPNOTSUPP; } diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c index 01a427800797c5cc93c20df18b5892a09e3d2ccc..34e5ec5d3e236ecb8478950019e6f16b48877583 100644 --- a/net/dsa/tag_8021q.c +++ b/net/dsa/tag_8021q.c @@ -2,9 +2,7 @@ /* Copyright (c) 2019, Vladimir Oltean * * This module is not a complete tagger implementation. It only provides - * primitives for taggers that rely on 802.1Q VLAN tags to use. The - * dsa_8021q_netdev_ops is registered for API compliance and not used - * directly by callers. + * primitives for taggers that rely on 802.1Q VLAN tags to use. */ #include #include @@ -332,7 +330,7 @@ static int dsa_tag_8021q_port_setup(struct dsa_switch *ds, int port) if (!dsa_port_is_user(dp)) return 0; - master = dp->cpu_dp->master; + master = dsa_port_to_master(dp); err = dsa_port_tag_8021q_vlan_add(dp, vid, false); if (err) { @@ -361,7 +359,7 @@ static void dsa_tag_8021q_port_teardown(struct dsa_switch *ds, int port) if (!dsa_port_is_user(dp)) return; - master = dp->cpu_dp->master; + master = dsa_port_to_master(dp); dsa_port_tag_8021q_vlan_del(dp, vid, false); diff --git a/net/dsa/tag_hellcreek.c b/net/dsa/tag_hellcreek.c index eb204ad36eeec083f9f4bf8f9ff89baf06eea72f..846588c0070a5e2a43afee4a1bb553203643d51f 100644 --- a/net/dsa/tag_hellcreek.c +++ b/net/dsa/tag_hellcreek.c @@ -45,7 +45,7 @@ static struct sk_buff *hellcreek_rcv(struct sk_buff *skb, skb->dev = dsa_master_find_slave(dev, 0, port); if (!skb->dev) { - netdev_warn(dev, "Failed to get source port: %d\n", port); + netdev_warn_once(dev, "Failed to get source port: %d\n", port); return NULL; } diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index 62b89d6f54fd2c6ed1b75064ba8f1db4c1200a23..e02daa74e8334508d17611b323699d5c9fc2ae59 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -414,12 +414,9 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb) off_eth = skb_gro_offset(skb); hlen = off_eth + sizeof(*eh); - eh = skb_gro_header_fast(skb, off_eth); - if (skb_gro_header_hard(skb, hlen)) { - eh = skb_gro_header_slow(skb, hlen, off_eth); - if (unlikely(!eh)) - goto out; - } + eh = skb_gro_header(skb, hlen, off_eth); + if (unlikely(!eh)) + goto out; flush = 0; diff --git a/net/ethtool/Makefile b/net/ethtool/Makefile index b76432e70e6baa1492ef2e1dceaa4bcb141fc518..72ab0944262af4fe880d180898c511e893827aee 100644 --- a/net/ethtool/Makefile +++ b/net/ethtool/Makefile @@ -7,4 +7,5 @@ obj-$(CONFIG_ETHTOOL_NETLINK) += ethtool_nl.o ethtool_nl-y := netlink.o bitset.o strset.o linkinfo.o linkmodes.o \ linkstate.o debug.o wol.o features.o privflags.o rings.o \ channels.o coalesce.o pause.o eee.o tsinfo.o cabletest.o \ - tunnels.o fec.o eeprom.o stats.o phc_vclocks.o module.o + tunnels.o fec.o eeprom.o stats.o phc_vclocks.o module.o \ + pse-pd.o diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 2dc2b80aea5f55f042a301b938c5c09deb2e8c1f..c1779657e074f25998901deee41a41613172b024 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -46,6 +46,7 @@ int ethtool_get_max_rxfh_channel(struct net_device *dev, u32 *max); int __ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info); extern const struct ethtool_phy_ops *ethtool_phy_ops; +extern const struct ethtool_pse_ops *ethtool_pse_ops; int ethtool_get_module_info_call(struct net_device *dev, struct ethtool_modinfo *modinfo); diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 6a7308de192d90fba124d360818a58d2b310a959..57e7238a4136bbb309d0e3549aa0b49c13ecefd9 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -571,6 +571,7 @@ static int ethtool_get_link_ksettings(struct net_device *dev, = __ETHTOOL_LINK_MODE_MASK_NU32; link_ksettings.base.master_slave_cfg = MASTER_SLAVE_CFG_UNSUPPORTED; link_ksettings.base.master_slave_state = MASTER_SLAVE_STATE_UNSUPPORTED; + link_ksettings.base.rate_matching = RATE_MATCH_NONE; return store_link_ksettings_for_user(useraddr, &link_ksettings); } @@ -714,16 +715,16 @@ ethtool_get_drvinfo(struct net_device *dev, struct ethtool_devlink_compat *rsp) const struct ethtool_ops *ops = dev->ethtool_ops; rsp->info.cmd = ETHTOOL_GDRVINFO; - strlcpy(rsp->info.version, UTS_RELEASE, sizeof(rsp->info.version)); + strscpy(rsp->info.version, UTS_RELEASE, sizeof(rsp->info.version)); if (ops->get_drvinfo) { ops->get_drvinfo(dev, &rsp->info); } else if (dev->dev.parent && dev->dev.parent->driver) { - strlcpy(rsp->info.bus_info, dev_name(dev->dev.parent), + strscpy(rsp->info.bus_info, dev_name(dev->dev.parent), sizeof(rsp->info.bus_info)); - strlcpy(rsp->info.driver, dev->dev.parent->driver->name, + strscpy(rsp->info.driver, dev->dev.parent->driver->name, sizeof(rsp->info.driver)); } else if (dev->rtnl_link_ops) { - strlcpy(rsp->info.driver, dev->rtnl_link_ops->kind, + strscpy(rsp->info.driver, dev->rtnl_link_ops->kind, sizeof(rsp->info.driver)); } else { return -EOPNOTSUPP; diff --git a/net/ethtool/linkmodes.c b/net/ethtool/linkmodes.c index 99b29b4fe9472f5e2de1c10ecd33d8248cd1687c..126e06c713a3ac1cf8d57c179a088995ff2e268a 100644 --- a/net/ethtool/linkmodes.c +++ b/net/ethtool/linkmodes.c @@ -70,6 +70,7 @@ static int linkmodes_reply_size(const struct ethnl_req_info *req_base, + nla_total_size(sizeof(u32)) /* LINKMODES_SPEED */ + nla_total_size(sizeof(u32)) /* LINKMODES_LANES */ + nla_total_size(sizeof(u8)) /* LINKMODES_DUPLEX */ + + nla_total_size(sizeof(u8)) /* LINKMODES_RATE_MATCHING */ + 0; ret = ethnl_bitset_size(ksettings->link_modes.advertising, ksettings->link_modes.supported, @@ -143,6 +144,10 @@ static int linkmodes_fill_reply(struct sk_buff *skb, lsettings->master_slave_state)) return -EMSGSIZE; + if (nla_put_u8(skb, ETHTOOL_A_LINKMODES_RATE_MATCHING, + lsettings->rate_matching)) + return -EMSGSIZE; + return 0; } diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index e26079e11835ceeda18b7356b787d75f547d9946..1a4c11356c96cdadfff40754d9c01ca72774ccde 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -286,6 +286,7 @@ ethnl_default_requests[__ETHTOOL_MSG_USER_CNT] = { [ETHTOOL_MSG_STATS_GET] = ðnl_stats_request_ops, [ETHTOOL_MSG_PHC_VCLOCKS_GET] = ðnl_phc_vclocks_request_ops, [ETHTOOL_MSG_MODULE_GET] = ðnl_module_request_ops, + [ETHTOOL_MSG_PSE_GET] = ðnl_pse_request_ops, }; static struct ethnl_dump_ctx *ethnl_dump_context(struct netlink_callback *cb) @@ -361,6 +362,9 @@ static int ethnl_default_doit(struct sk_buff *skb, struct genl_info *info) ops = ethnl_default_requests[cmd]; if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", cmd)) return -EOPNOTSUPP; + if (GENL_REQ_ATTR_CHECK(info, ops->hdr_attr)) + return -EINVAL; + req_info = kzalloc(ops->req_info_size, GFP_KERNEL); if (!req_info) return -ENOMEM; @@ -1020,6 +1024,22 @@ static const struct genl_ops ethtool_genl_ops[] = { .policy = ethnl_module_set_policy, .maxattr = ARRAY_SIZE(ethnl_module_set_policy) - 1, }, + { + .cmd = ETHTOOL_MSG_PSE_GET, + .doit = ethnl_default_doit, + .start = ethnl_default_start, + .dumpit = ethnl_default_dumpit, + .done = ethnl_default_done, + .policy = ethnl_pse_get_policy, + .maxattr = ARRAY_SIZE(ethnl_pse_get_policy) - 1, + }, + { + .cmd = ETHTOOL_MSG_PSE_SET, + .flags = GENL_UNS_ADMIN_PERM, + .doit = ethnl_set_pse, + .policy = ethnl_pse_set_policy, + .maxattr = ARRAY_SIZE(ethnl_pse_set_policy) - 1, + }, }; static const struct genl_multicast_group ethtool_nl_mcgrps[] = { @@ -1033,6 +1053,7 @@ static struct genl_family ethtool_genl_family __ro_after_init = { .parallel_ops = true, .ops = ethtool_genl_ops, .n_ops = ARRAY_SIZE(ethtool_genl_ops), + .resv_start_op = ETHTOOL_MSG_MODULE_GET + 1, .mcgrps = ethtool_nl_mcgrps, .n_mcgrps = ARRAY_SIZE(ethtool_nl_mcgrps), }; diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index c0d587611854619244329257d22dbcbe4e58e944..1bfd374f97188bfbe58c8cbfb9a26a2cd9b7410f 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -345,6 +345,7 @@ extern const struct ethnl_request_ops ethnl_module_eeprom_request_ops; extern const struct ethnl_request_ops ethnl_stats_request_ops; extern const struct ethnl_request_ops ethnl_phc_vclocks_request_ops; extern const struct ethnl_request_ops ethnl_module_request_ops; +extern const struct ethnl_request_ops ethnl_pse_request_ops; extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_FLAGS + 1]; extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_FLAGS + 1]; @@ -383,6 +384,8 @@ extern const struct nla_policy ethnl_stats_get_policy[ETHTOOL_A_STATS_GROUPS + 1 extern const struct nla_policy ethnl_phc_vclocks_get_policy[ETHTOOL_A_PHC_VCLOCKS_HEADER + 1]; extern const struct nla_policy ethnl_module_get_policy[ETHTOOL_A_MODULE_HEADER + 1]; extern const struct nla_policy ethnl_module_set_policy[ETHTOOL_A_MODULE_POWER_MODE_POLICY + 1]; +extern const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1]; +extern const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1]; int ethnl_set_linkinfo(struct sk_buff *skb, struct genl_info *info); int ethnl_set_linkmodes(struct sk_buff *skb, struct genl_info *info); @@ -402,6 +405,7 @@ int ethnl_tunnel_info_start(struct netlink_callback *cb); int ethnl_tunnel_info_dumpit(struct sk_buff *skb, struct netlink_callback *cb); int ethnl_set_fec(struct sk_buff *skb, struct genl_info *info); int ethnl_set_module(struct sk_buff *skb, struct genl_info *info); +int ethnl_set_pse(struct sk_buff *skb, struct genl_info *info); extern const char stats_std_names[__ETHTOOL_STATS_CNT][ETH_GSTRING_LEN]; extern const char stats_eth_phy_names[__ETHTOOL_A_STATS_ETH_PHY_CNT][ETH_GSTRING_LEN]; diff --git a/net/ethtool/pse-pd.c b/net/ethtool/pse-pd.c new file mode 100644 index 0000000000000000000000000000000000000000..5a471e115b66acf5c0bc04bd4905c3a3b7b83c78 --- /dev/null +++ b/net/ethtool/pse-pd.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// ethtool interface for for Ethernet PSE (Power Sourcing Equipment) +// and PD (Powered Device) +// +// Copyright (c) 2022 Pengutronix, Oleksij Rempel +// + +#include "common.h" +#include "linux/pse-pd/pse.h" +#include "netlink.h" +#include +#include +#include + +struct pse_req_info { + struct ethnl_req_info base; +}; + +struct pse_reply_data { + struct ethnl_reply_data base; + struct pse_control_status status; +}; + +#define PSE_REPDATA(__reply_base) \ + container_of(__reply_base, struct pse_reply_data, base) + +/* PSE_GET */ + +const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1] = { + [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), +}; + +static int pse_get_pse_attributes(struct net_device *dev, + struct netlink_ext_ack *extack, + struct pse_reply_data *data) +{ + struct phy_device *phydev = dev->phydev; + + if (!phydev) { + NL_SET_ERR_MSG(extack, "No PHY is attached"); + return -EOPNOTSUPP; + } + + if (!phydev->psec) { + NL_SET_ERR_MSG(extack, "No PSE is attached"); + return -EOPNOTSUPP; + } + + memset(&data->status, 0, sizeof(data->status)); + + return pse_ethtool_get_status(phydev->psec, extack, &data->status); +} + +static int pse_prepare_data(const struct ethnl_req_info *req_base, + struct ethnl_reply_data *reply_base, + struct genl_info *info) +{ + struct pse_reply_data *data = PSE_REPDATA(reply_base); + struct net_device *dev = reply_base->dev; + int ret; + + ret = ethnl_ops_begin(dev); + if (ret < 0) + return ret; + + ret = pse_get_pse_attributes(dev, info->extack, data); + + ethnl_ops_complete(dev); + + return ret; +} + +static int pse_reply_size(const struct ethnl_req_info *req_base, + const struct ethnl_reply_data *reply_base) +{ + const struct pse_reply_data *data = PSE_REPDATA(reply_base); + const struct pse_control_status *st = &data->status; + int len = 0; + + if (st->podl_admin_state > 0) + len += nla_total_size(sizeof(u32)); /* _PODL_PSE_ADMIN_STATE */ + if (st->podl_pw_status > 0) + len += nla_total_size(sizeof(u32)); /* _PODL_PSE_PW_D_STATUS */ + + return len; +} + +static int pse_fill_reply(struct sk_buff *skb, + const struct ethnl_req_info *req_base, + const struct ethnl_reply_data *reply_base) +{ + const struct pse_reply_data *data = PSE_REPDATA(reply_base); + const struct pse_control_status *st = &data->status; + + if (st->podl_admin_state > 0 && + nla_put_u32(skb, ETHTOOL_A_PODL_PSE_ADMIN_STATE, + st->podl_admin_state)) + return -EMSGSIZE; + + if (st->podl_pw_status > 0 && + nla_put_u32(skb, ETHTOOL_A_PODL_PSE_PW_D_STATUS, + st->podl_pw_status)) + return -EMSGSIZE; + + return 0; +} + +const struct ethnl_request_ops ethnl_pse_request_ops = { + .request_cmd = ETHTOOL_MSG_PSE_GET, + .reply_cmd = ETHTOOL_MSG_PSE_GET_REPLY, + .hdr_attr = ETHTOOL_A_PSE_HEADER, + .req_info_size = sizeof(struct pse_req_info), + .reply_data_size = sizeof(struct pse_reply_data), + + .prepare_data = pse_prepare_data, + .reply_size = pse_reply_size, + .fill_reply = pse_fill_reply, +}; + +/* PSE_SET */ + +const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1] = { + [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), + [ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] = + NLA_POLICY_RANGE(NLA_U32, ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED, + ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED), +}; + +static int pse_set_pse_config(struct net_device *dev, + struct netlink_ext_ack *extack, + struct nlattr **tb) +{ + struct phy_device *phydev = dev->phydev; + struct pse_control_config config = {}; + + /* Optional attribute. Do not return error if not set. */ + if (!tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]) + return 0; + + /* this values are already validated by the ethnl_pse_set_policy */ + config.admin_cotrol = nla_get_u32(tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]); + + if (!phydev) { + NL_SET_ERR_MSG(extack, "No PHY is attached"); + return -EOPNOTSUPP; + } + + if (!phydev->psec) { + NL_SET_ERR_MSG(extack, "No PSE is attached"); + return -EOPNOTSUPP; + } + + return pse_ethtool_set_config(phydev->psec, extack, &config); +} + +int ethnl_set_pse(struct sk_buff *skb, struct genl_info *info) +{ + struct ethnl_req_info req_info = {}; + struct nlattr **tb = info->attrs; + struct net_device *dev; + int ret; + + ret = ethnl_parse_header_dev_get(&req_info, tb[ETHTOOL_A_PSE_HEADER], + genl_info_net(info), info->extack, + true); + if (ret < 0) + return ret; + + dev = req_info.dev; + + rtnl_lock(); + ret = ethnl_ops_begin(dev); + if (ret < 0) + goto out_rtnl; + + ret = pse_set_pse_config(dev, info->extack, tb); + ethnl_ops_complete(dev); +out_rtnl: + rtnl_unlock(); + + ethnl_parse_header_dev_put(&req_info); + + return ret; +} diff --git a/net/ethtool/strset.c b/net/ethtool/strset.c index 2d51b7ab4dc524473c17b44d58460dfc6bfc0fc0..3f7de54d85fb9c20fc344fc840a4ad6de5fe2bff 100644 --- a/net/ethtool/strset.c +++ b/net/ethtool/strset.c @@ -167,7 +167,7 @@ static int strset_get_id(const struct nlattr *nest, u32 *val, get_stringset_policy, extack); if (ret < 0) return ret; - if (!tb[ETHTOOL_A_STRINGSET_ID]) + if (NL_REQ_ATTR_CHECK(extack, nest, tb, ETHTOOL_A_STRINGSET_ID)) return -EINVAL; *val = nla_get_u32(tb[ETHTOOL_A_STRINGSET_ID]); diff --git a/net/ethtool/tunnels.c b/net/ethtool/tunnels.c index efde3353668734d7d0457d721f8ecd7a4e0bc551..67fb414ca859b75b5a0c9f2cd20a1f4a4045f5be 100644 --- a/net/ethtool/tunnels.c +++ b/net/ethtool/tunnels.c @@ -136,6 +136,8 @@ ethnl_tunnel_info_fill_reply(const struct ethnl_req_info *req_base, goto err_cancel_table; entry = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY); + if (!entry) + goto err_cancel_entry; if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT, htons(IANA_VXLAN_UDP_PORT)) || diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c index 1405c037cf7ab8b42986362336160396206744f0..7174a9092900266c43ac00da4e62130da8ee7dc3 100644 --- a/net/hsr/hsr_netlink.c +++ b/net/hsr/hsr_netlink.c @@ -522,6 +522,7 @@ static struct genl_family hsr_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = hsr_ops, .n_small_ops = ARRAY_SIZE(hsr_ops), + .resv_start_op = HSR_C_SET_NODE_LIST + 1, .mcgrps = hsr_mcgrps, .n_mcgrps = ARRAY_SIZE(hsr_mcgrps), }; diff --git a/net/ieee802154/netlink.c b/net/ieee802154/netlink.c index b07abc38b4b3d6a73b7a6968aa2d6fc5f7f1f293..7d2de4ee6992b75a0b86eb5e44e5d40d20a98968 100644 --- a/net/ieee802154/netlink.c +++ b/net/ieee802154/netlink.c @@ -132,6 +132,7 @@ struct genl_family nl802154_family __ro_after_init = { .module = THIS_MODULE, .small_ops = ieee802154_ops, .n_small_ops = ARRAY_SIZE(ieee802154_ops), + .resv_start_op = IEEE802154_LLSEC_DEL_SECLEVEL + 1, .mcgrps = ieee802154_mcgrps, .n_mcgrps = ARRAY_SIZE(ieee802154_mcgrps), }; diff --git a/net/ieee802154/nl802154.c b/net/ieee802154/nl802154.c index e0b072aecf0f30fe5b410ef0418fc01c765474c9..38c4f3cb010e89f8cd31c453e2fdd1f2291886c5 100644 --- a/net/ieee802154/nl802154.c +++ b/net/ieee802154/nl802154.c @@ -2500,6 +2500,7 @@ static struct genl_family nl802154_fam __ro_after_init = { .module = THIS_MODULE, .ops = nl802154_ops, .n_ops = ARRAY_SIZE(nl802154_ops), + .resv_start_op = NL802154_CMD_DEL_SEC_LEVEL + 1, .mcgrps = nl802154_mcgrps, .n_mcgrps = ARRAY_SIZE(nl802154_mcgrps), }; diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index 718fb77bb372c67f3514205d649f287bdbc4a609..6e55fae4c68604ac5d33eb79c8c33b006be89bbe 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -200,8 +200,9 @@ static int raw_bind(struct sock *sk, struct sockaddr *_uaddr, int len) int err = 0; struct net_device *dev = NULL; - if (len < sizeof(*uaddr)) - return -EINVAL; + err = ieee802154_sockaddr_check_size(uaddr, len); + if (err < 0) + return err; uaddr = (struct sockaddr_ieee802154 *)_uaddr; if (uaddr->family != AF_IEEE802154) @@ -271,6 +272,10 @@ static int raw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) err = -EMSGSIZE; goto out_dev; } + if (!size) { + err = 0; + goto out_dev; + } hlen = LL_RESERVED_SPACE(dev); tlen = dev->needed_tailroom; @@ -493,7 +498,8 @@ static int dgram_bind(struct sock *sk, struct sockaddr *uaddr, int len) ro->bound = 0; - if (len < sizeof(*addr)) + err = ieee802154_sockaddr_check_size(addr, len); + if (err < 0) goto out; if (addr->family != AF_IEEE802154) @@ -564,8 +570,9 @@ static int dgram_connect(struct sock *sk, struct sockaddr *uaddr, struct dgram_sock *ro = dgram_sk(sk); int err = 0; - if (len < sizeof(*addr)) - return -EINVAL; + err = ieee802154_sockaddr_check_size(addr, len); + if (err < 0) + return err; if (addr->family != AF_IEEE802154) return -EINVAL; @@ -604,6 +611,7 @@ static int dgram_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) struct ieee802154_mac_cb *cb; struct dgram_sock *ro = dgram_sk(sk); struct ieee802154_addr dst_addr; + DECLARE_SOCKADDR(struct sockaddr_ieee802154*, daddr, msg->msg_name); int hlen, tlen; int err; @@ -612,10 +620,20 @@ static int dgram_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) return -EOPNOTSUPP; } - if (!ro->connected && !msg->msg_name) - return -EDESTADDRREQ; - else if (ro->connected && msg->msg_name) - return -EISCONN; + if (msg->msg_name) { + if (ro->connected) + return -EISCONN; + if (msg->msg_namelen < IEEE802154_MIN_NAMELEN) + return -EINVAL; + err = ieee802154_sockaddr_check_size(daddr, msg->msg_namelen); + if (err < 0) + return err; + ieee802154_addr_from_sa(&dst_addr, &daddr->addr); + } else { + if (!ro->connected) + return -EDESTADDRREQ; + dst_addr = ro->dst_addr; + } if (!ro->bound) dev = dev_getfirstbyhwtype(sock_net(sk), ARPHRD_IEEE802154); @@ -651,16 +669,6 @@ static int dgram_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) cb = mac_cb_init(skb); cb->type = IEEE802154_FC_TYPE_DATA; cb->ackreq = ro->want_ack; - - if (msg->msg_name) { - DECLARE_SOCKADDR(struct sockaddr_ieee802154*, - daddr, msg->msg_name); - - ieee802154_addr_from_sa(&dst_addr, &daddr->addr); - } else { - dst_addr = ro->dst_addr; - } - cb->secen = ro->secen; cb->secen_override = ro->secen_override; cb->seclevel = ro->seclevel; diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 3ca0cc4678862798b243fd21871b8d885e927157..3dd02396517df599cf4ff3b9ab8463ea959770a1 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -558,22 +558,27 @@ int inet_dgram_connect(struct socket *sock, struct sockaddr *uaddr, int addr_len, int flags) { struct sock *sk = sock->sk; + const struct proto *prot; int err; if (addr_len < sizeof(uaddr->sa_family)) return -EINVAL; + + /* IPV6_ADDRFORM can change sk->sk_prot under us. */ + prot = READ_ONCE(sk->sk_prot); + if (uaddr->sa_family == AF_UNSPEC) - return sk->sk_prot->disconnect(sk, flags); + return prot->disconnect(sk, flags); if (BPF_CGROUP_PRE_CONNECT_ENABLED(sk)) { - err = sk->sk_prot->pre_connect(sk, uaddr, addr_len); + err = prot->pre_connect(sk, uaddr, addr_len); if (err) return err; } if (data_race(!inet_sk(sk)->inet_num) && inet_autobind(sk)) return -EAGAIN; - return sk->sk_prot->connect(sk, uaddr, addr_len); + return prot->connect(sk, uaddr, addr_len); } EXPORT_SYMBOL(inet_dgram_connect); @@ -734,10 +739,11 @@ EXPORT_SYMBOL(inet_stream_connect); int inet_accept(struct socket *sock, struct socket *newsock, int flags, bool kern) { - struct sock *sk1 = sock->sk; + struct sock *sk1 = sock->sk, *sk2; int err = -EINVAL; - struct sock *sk2 = sk1->sk_prot->accept(sk1, flags, &err, kern); + /* IPV6_ADDRFORM can change sk->sk_prot under us. */ + sk2 = READ_ONCE(sk1->sk_prot)->accept(sk1, flags, &err, kern); if (!sk2) goto do_err; @@ -825,12 +831,15 @@ ssize_t inet_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags) { struct sock *sk = sock->sk; + const struct proto *prot; if (unlikely(inet_send_prepare(sk))) return -EAGAIN; - if (sk->sk_prot->sendpage) - return sk->sk_prot->sendpage(sk, page, offset, size, flags); + /* IPV6_ADDRFORM can change sk->sk_prot under us. */ + prot = READ_ONCE(sk->sk_prot); + if (prot->sendpage) + return prot->sendpage(sk, page, offset, size, flags); return sock_no_sendpage(sock, page, offset, size, flags); } EXPORT_SYMBOL(inet_sendpage); @@ -1219,6 +1228,7 @@ EXPORT_SYMBOL(inet_unregister_protosw); static int inet_sk_reselect_saddr(struct sock *sk) { + struct inet_bind_hashbucket *prev_addr_hashbucket; struct inet_sock *inet = inet_sk(sk); __be32 old_saddr = inet->inet_saddr; __be32 daddr = inet->inet_daddr; @@ -1226,6 +1236,7 @@ static int inet_sk_reselect_saddr(struct sock *sk) struct rtable *rt; __be32 new_saddr; struct ip_options_rcu *inet_opt; + int err; inet_opt = rcu_dereference_protected(inet->inet_opt, lockdep_sock_is_held(sk)); @@ -1240,20 +1251,34 @@ static int inet_sk_reselect_saddr(struct sock *sk) if (IS_ERR(rt)) return PTR_ERR(rt); - sk_setup_caps(sk, &rt->dst); - new_saddr = fl4->saddr; - if (new_saddr == old_saddr) + if (new_saddr == old_saddr) { + sk_setup_caps(sk, &rt->dst); return 0; + } + + prev_addr_hashbucket = + inet_bhashfn_portaddr(tcp_or_dccp_get_hashinfo(sk), sk, + sock_net(sk), inet->inet_num); + + inet->inet_saddr = inet->inet_rcv_saddr = new_saddr; + + err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk); + if (err) { + inet->inet_saddr = old_saddr; + inet->inet_rcv_saddr = old_saddr; + ip_rt_put(rt); + return err; + } + + sk_setup_caps(sk, &rt->dst); if (READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) > 1) { pr_info("%s(): shifting inet->saddr from %pI4 to %pI4\n", __func__, &old_saddr, &new_saddr); } - inet->inet_saddr = inet->inet_rcv_saddr = new_saddr; - /* * XXX The only one ugly spot where we need to * XXX really change the sockets identity after @@ -1448,12 +1473,9 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) off = skb_gro_offset(skb); hlen = off + sizeof(*iph); - iph = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) { - iph = skb_gro_header_slow(skb, hlen, off); - if (unlikely(!iph)) - goto out; - } + iph = skb_gro_header(skb, hlen, off); + if (unlikely(!iph)) + goto out; proto = iph->protocol; diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c index f8ad04470d3ace7f2e59ee5890467d689be52752..ee4e578c7f20191768c0042df696b2631c080cbb 100644 --- a/net/ipv4/ah4.c +++ b/net/ipv4/ah4.c @@ -471,30 +471,38 @@ static int ah4_err(struct sk_buff *skb, u32 info) return 0; } -static int ah_init_state(struct xfrm_state *x) +static int ah_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { struct ah_data *ahp = NULL; struct xfrm_algo_desc *aalg_desc; struct crypto_ahash *ahash; - if (!x->aalg) + if (!x->aalg) { + NL_SET_ERR_MSG(extack, "AH requires a state with an AUTH algorithm"); goto error; + } - if (x->encap) + if (x->encap) { + NL_SET_ERR_MSG(extack, "AH is not compatible with encapsulation"); goto error; + } ahp = kzalloc(sizeof(*ahp), GFP_KERNEL); if (!ahp) return -ENOMEM; ahash = crypto_alloc_ahash(x->aalg->alg_name, 0, 0); - if (IS_ERR(ahash)) + if (IS_ERR(ahash)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } ahp->ahash = ahash; if (crypto_ahash_setkey(ahash, x->aalg->alg_key, - (x->aalg->alg_key_len + 7) / 8)) + (x->aalg->alg_key_len + 7) / 8)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } /* * Lookup the algorithm description maintained by xfrm_algo, @@ -507,10 +515,7 @@ static int ah_init_state(struct xfrm_state *x) if (aalg_desc->uinfo.auth.icv_fullbits/8 != crypto_ahash_digestsize(ahash)) { - pr_info("%s: %s digestsize %u != %u\n", - __func__, x->aalg->alg_name, - crypto_ahash_digestsize(ahash), - aalg_desc->uinfo.auth.icv_fullbits / 8); + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; } diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index 87c7e3fc5197342ce49f10f15f14bdf0036331ea..4f7237661afb9b2c1a11a1740eae823443ebdea4 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -1129,7 +1129,7 @@ static int arp_req_get(struct arpreq *r, struct net_device *dev) r->arp_flags = arp_state_to_flags(neigh); read_unlock_bh(&neigh->lock); r->arp_ha.sa_family = dev->type; - strlcpy(r->arp_dev, dev->name, sizeof(r->arp_dev)); + strscpy(r->arp_dev, dev->name, sizeof(r->arp_dev)); err = 0; } neigh_release(neigh); diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c index 85a9e500c42d4f8599f6fb9e1abe4987a7d8b770..6da16ae6a96243b363889c690d76d28a3897669b 100644 --- a/net/ipv4/bpf_tcp_ca.c +++ b/net/ipv4/bpf_tcp_ca.c @@ -124,7 +124,7 @@ static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log, return -EACCES; } - return NOT_INIT; + return 0; } BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt) diff --git a/net/ipv4/datagram.c b/net/ipv4/datagram.c index ffd57523331fdae4c157cd75c430522dbc2e1b54..0ee7fd2597300f1e63396ea7025b47f194de2a02 100644 --- a/net/ipv4/datagram.c +++ b/net/ipv4/datagram.c @@ -42,6 +42,8 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len oif = inet->mc_index; if (!saddr) saddr = inet->mc_addr; + } else if (!oif) { + oif = inet->uc_index; } fl4 = &inet->cork.fl.u.ip4; rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr, oif, @@ -71,7 +73,7 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len reuseport_has_conns(sk, true); sk->sk_state = TCP_ESTABLISHED; sk_set_txhash(sk); - inet->inet_id = prandom_u32(); + inet->inet_id = get_random_u16(); sk_dst_set(sk, &rt->dst); err = 0; diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index 92b778e423df824d45ffbfa53a975af14c6c5713..e8b9a9202fecd913137f169f161dfdccc16f7edf 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -2682,23 +2682,27 @@ static __net_init int devinet_init_net(struct net *net) #endif if (!net_eq(net, &init_net)) { - if (IS_ENABLED(CONFIG_SYSCTL) && - sysctl_devconf_inherit_init_net == 3) { + switch (net_inherit_devconf()) { + case 3: /* copy from the current netns */ memcpy(all, current->nsproxy->net_ns->ipv4.devconf_all, sizeof(ipv4_devconf)); memcpy(dflt, current->nsproxy->net_ns->ipv4.devconf_dflt, sizeof(ipv4_devconf_dflt)); - } else if (!IS_ENABLED(CONFIG_SYSCTL) || - sysctl_devconf_inherit_init_net != 2) { - /* inherit == 0 or 1: copy from init_net */ + break; + case 0: + case 1: + /* copy from init_net */ memcpy(all, init_net.ipv4.devconf_all, sizeof(ipv4_devconf)); memcpy(dflt, init_net.ipv4.devconf_dflt, sizeof(ipv4_devconf_dflt)); + break; + case 2: + /* use compiled values */ + break; } - /* else inherit == 2: use compiled values */ } #ifdef CONFIG_SYSCTL diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 5c03eba787e52ffc22438676cd96ba7ff265b29e..52c8047efedbba2c57a43ba4ae7cfef3df8afef3 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -134,6 +134,7 @@ static void esp_free_tcp_sk(struct rcu_head *head) static struct sock *esp_find_tcp_sk(struct xfrm_state *x) { struct xfrm_encap_tmpl *encap = x->encap; + struct net *net = xs_net(x); struct esp_tcp_sk *esk; __be16 sport, dport; struct sock *nsk; @@ -160,7 +161,7 @@ static struct sock *esp_find_tcp_sk(struct xfrm_state *x) } spin_unlock_bh(&x->lock); - sk = inet_lookup_established(xs_net(x), &tcp_hashinfo, x->id.daddr.a4, + sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, x->id.daddr.a4, dport, x->props.saddr.a4, sport, 0); if (!sk) return ERR_PTR(-ENOENT); @@ -1007,16 +1008,17 @@ static void esp_destroy(struct xfrm_state *x) crypto_free_aead(aead); } -static int esp_init_aead(struct xfrm_state *x) +static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack) { char aead_name[CRYPTO_MAX_ALG_NAME]; struct crypto_aead *aead; int err; - err = -ENAMETOOLONG; if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", - x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) - goto error; + x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); + return -ENAMETOOLONG; + } aead = crypto_alloc_aead(aead_name, 0, 0); err = PTR_ERR(aead); @@ -1034,11 +1036,15 @@ static int esp_init_aead(struct xfrm_state *x) if (err) goto error; + return 0; + error: + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); return err; } -static int esp_init_authenc(struct xfrm_state *x) +static int esp_init_authenc(struct xfrm_state *x, + struct netlink_ext_ack *extack) { struct crypto_aead *aead; struct crypto_authenc_key_param *param; @@ -1049,10 +1055,6 @@ static int esp_init_authenc(struct xfrm_state *x) unsigned int keylen; int err; - err = -EINVAL; - if (!x->ealg) - goto error; - err = -ENAMETOOLONG; if ((x->props.flags & XFRM_STATE_ESN)) { @@ -1061,22 +1063,28 @@ static int esp_init_authenc(struct xfrm_state *x) x->geniv ?: "", x->geniv ? "(" : "", x->aalg ? x->aalg->alg_name : "digest_null", x->ealg->alg_name, - x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) + x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); goto error; + } } else { if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, "%s%sauthenc(%s,%s)%s", x->geniv ?: "", x->geniv ? "(" : "", x->aalg ? x->aalg->alg_name : "digest_null", x->ealg->alg_name, - x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) + x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); goto error; + } } aead = crypto_alloc_aead(authenc_name, 0, 0); err = PTR_ERR(aead); - if (IS_ERR(aead)) + if (IS_ERR(aead)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } x->data = aead; @@ -1106,17 +1114,16 @@ static int esp_init_authenc(struct xfrm_state *x) err = -EINVAL; if (aalg_desc->uinfo.auth.icv_fullbits / 8 != crypto_aead_authsize(aead)) { - pr_info("ESP: %s digestsize %u != %u\n", - x->aalg->alg_name, - crypto_aead_authsize(aead), - aalg_desc->uinfo.auth.icv_fullbits / 8); + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto free_key; } err = crypto_aead_setauthsize( aead, x->aalg->alg_trunc_len / 8); - if (err) + if (err) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto free_key; + } } param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8); @@ -1131,7 +1138,7 @@ error: return err; } -static int esp_init_state(struct xfrm_state *x) +static int esp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { struct crypto_aead *aead; u32 align; @@ -1139,10 +1146,14 @@ static int esp_init_state(struct xfrm_state *x) x->data = NULL; - if (x->aead) - err = esp_init_aead(x); - else - err = esp_init_authenc(x); + if (x->aead) { + err = esp_init_aead(x, extack); + } else if (x->ealg) { + err = esp_init_authenc(x, extack); + } else { + NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided"); + err = -EINVAL; + } if (err) goto error; @@ -1160,6 +1171,7 @@ static int esp_init_state(struct xfrm_state *x) switch (encap->encap_type) { default: + NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP"); err = -EINVAL; goto error; case UDP_ENCAP_ESPINUDP: diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 935026f4c807e5f02129dd31190617b3e066ff1d..170152772d3328421bd4bcdb86b9222de38a72e8 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -110,7 +110,10 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - return skb_eth_gso_segment(skb, features, htons(ETH_P_IP)); + __be16 type = x->inner_mode.family == AF_INET6 ? htons(ETH_P_IPV6) + : htons(ETH_P_IP); + + return skb_eth_gso_segment(skb, features, type); } static struct sk_buff *xfrm4_transport_gso_segment(struct xfrm_state *x, diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index f361d3d56be27591b548b088a44d995d9aa38bbf..943edf4ad4db0df876def32d88d5dbdfea93958e 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -389,7 +389,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst, dev_match = dev_match || (res.type == RTN_LOCAL && dev == net->loopback_dev); if (dev_match) { - ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST; + ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_LINK; return ret; } if (no_addr) @@ -401,7 +401,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst, ret = 0; if (fib_lookup(net, &fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE) == 0) { if (res.type == RTN_UNICAST) - ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST; + ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_LINK; } return ret; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 2dc97583d279061461b3613d35808b5b50362ecf..e9a7f70a54df4920f6f41e4ed0967069adb05e98 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -888,13 +888,13 @@ int fib_nh_match(struct net *net, struct fib_config *cfg, struct fib_info *fi, return 1; } + /* cannot match on nexthop object attributes */ + if (fi->nh) + return 1; + if (cfg->fc_oif || cfg->fc_gw_family) { struct fib_nh *nh; - /* cannot match on nexthop object attributes */ - if (fi->nh) - return 1; - nh = fib_info_nh(fi, 0); if (cfg->fc_encap) { if (fib_encap_match(net, cfg->fc_encap_type, diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c index 025a33c1b04d6b761dd7eade5ba9130f3a2c8d5b..0c3c6d0cee290cfef07ec8f3fa7e2089f68db341 100644 --- a/net/ipv4/fou.c +++ b/net/ipv4/fou.c @@ -323,12 +323,9 @@ static struct sk_buff *gue_gro_receive(struct sock *sk, off = skb_gro_offset(skb); len = off + sizeof(*guehdr); - guehdr = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, len)) { - guehdr = skb_gro_header_slow(skb, len, off); - if (unlikely(!guehdr)) - goto out; - } + guehdr = skb_gro_header(skb, len, off); + if (unlikely(!guehdr)) + goto out; switch (guehdr->version) { case 0: @@ -931,6 +928,7 @@ static struct genl_family fou_nl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = fou_nl_ops, .n_small_ops = ARRAY_SIZE(fou_nl_ops), + .resv_start_op = FOU_CMD_GET + 1, }; size_t fou_encap_hlen(struct ip_tunnel_encap *e) diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c index 07073fa35205ef80882c854c6a7b3eebe746f365..2b9cb5398335bc1a0c7b48b638360016141e1ba5 100644 --- a/net/ipv4/gre_offload.c +++ b/net/ipv4/gre_offload.c @@ -137,12 +137,9 @@ static struct sk_buff *gre_gro_receive(struct list_head *head, off = skb_gro_offset(skb); hlen = off + sizeof(*greh); - greh = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) { - greh = skb_gro_header_slow(skb, hlen, off); - if (unlikely(!greh)) - goto out; - } + greh = skb_gro_header(skb, hlen, off); + if (unlikely(!greh)) + goto out; /* Only support version 0 and K (key), C (csum) flags. Note that * although the support for the S (seq#) flag can be added easily diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index e3ab0cb616246b2b120515996e23ce1e748f2fb6..81be3e0f0e70471f40db9d6a0b832cc8d3398a81 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -213,7 +213,7 @@ static void igmp_stop_timer(struct ip_mc_list *im) /* It must be called with locked im->lock */ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) { - int tv = prandom_u32() % max_delay; + int tv = prandom_u32_max(max_delay); im->tm_running = 1; if (!mod_timer(&im->timer, jiffies+tv+2)) @@ -222,7 +222,7 @@ static void igmp_start_timer(struct ip_mc_list *im, int max_delay) static void igmp_gq_start_timer(struct in_device *in_dev) { - int tv = prandom_u32() % in_dev->mr_maxdelay; + int tv = prandom_u32_max(in_dev->mr_maxdelay); unsigned long exp = jiffies + tv + 2; if (in_dev->mr_gq_running && @@ -236,7 +236,7 @@ static void igmp_gq_start_timer(struct in_device *in_dev) static void igmp_ifc_start_timer(struct in_device *in_dev, int delay) { - int tv = prandom_u32() % delay; + int tv = prandom_u32_max(delay); if (!mod_timer(&in_dev->mr_ifc_timer, jiffies+tv+2)) in_dev_hold(in_dev); @@ -2529,11 +2529,10 @@ done: err = ip_mc_leave_group(sk, &imr); return err; } - int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, - struct ip_msfilter __user *optval, int __user *optlen) + sockptr_t optval, sockptr_t optlen) { - int err, len, count, copycount; + int err, len, count, copycount, msf_size; struct ip_mreqn imr; __be32 addr = msf->imsf_multiaddr; struct ip_mc_socklist *pmc; @@ -2575,12 +2574,15 @@ int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, copycount = count < msf->imsf_numsrc ? count : msf->imsf_numsrc; len = flex_array_size(psl, sl_addr, copycount); msf->imsf_numsrc = count; - if (put_user(IP_MSFILTER_SIZE(copycount), optlen) || - copy_to_user(optval, msf, IP_MSFILTER_SIZE(0))) { + msf_size = IP_MSFILTER_SIZE(copycount); + if (copy_to_sockptr(optlen, &msf_size, sizeof(int)) || + copy_to_sockptr(optval, msf, IP_MSFILTER_SIZE(0))) { return -EFAULT; } if (len && - copy_to_user(&optval->imsf_slist_flex[0], psl->sl_addr, len)) + copy_to_sockptr_offset(optval, + offsetof(struct ip_msfilter, imsf_slist_flex), + psl->sl_addr, len)) return -EFAULT; return 0; done: @@ -2588,7 +2590,7 @@ done: } int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf, - struct sockaddr_storage __user *p) + sockptr_t optval, size_t ss_offset) { int i, count, copycount; struct sockaddr_in *psin; @@ -2618,15 +2620,17 @@ int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf, count = psl ? psl->sl_count : 0; copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc; gsf->gf_numsrc = count; - for (i = 0; i < copycount; i++, p++) { + for (i = 0; i < copycount; i++) { struct sockaddr_storage ss; psin = (struct sockaddr_in *)&ss; memset(&ss, 0, sizeof(ss)); psin->sin_family = AF_INET; psin->sin_addr.s_addr = psl->sl_addr[i]; - if (copy_to_user(p, &ss, sizeof(ss))) + if (copy_to_sockptr_offset(optval, ss_offset, + &ss, sizeof(ss))) return -EFAULT; + ss_offset += sizeof(ss); } return 0; } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index eb31c7158b39cbcefbef4d4f3ee9ce0d8d606ca9..4e84ed21d16fedc7515728f94a5b560589c989e1 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -130,14 +130,75 @@ void inet_get_local_port_range(struct net *net, int *low, int *high) } EXPORT_SYMBOL(inet_get_local_port_range); +static bool inet_use_bhash2_on_bind(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) { + int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr); + + return addr_type != IPV6_ADDR_ANY && + addr_type != IPV6_ADDR_MAPPED; + } +#endif + return sk->sk_rcv_saddr != htonl(INADDR_ANY); +} + +static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2, + kuid_t sk_uid, bool relax, + bool reuseport_cb_ok, bool reuseport_ok) +{ + int bound_dev_if2; + + if (sk == sk2) + return false; + + bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if); + + if (!sk->sk_bound_dev_if || !bound_dev_if2 || + sk->sk_bound_dev_if == bound_dev_if2) { + if (sk->sk_reuse && sk2->sk_reuse && + sk2->sk_state != TCP_LISTEN) { + if (!relax || (!reuseport_ok && sk->sk_reuseport && + sk2->sk_reuseport && reuseport_cb_ok && + (sk2->sk_state == TCP_TIME_WAIT || + uid_eq(sk_uid, sock_i_uid(sk2))))) + return true; + } else if (!reuseport_ok || !sk->sk_reuseport || + !sk2->sk_reuseport || !reuseport_cb_ok || + (sk2->sk_state != TCP_TIME_WAIT && + !uid_eq(sk_uid, sock_i_uid(sk2)))) { + return true; + } + } + return false; +} + +static bool inet_bhash2_conflict(const struct sock *sk, + const struct inet_bind2_bucket *tb2, + kuid_t sk_uid, + bool relax, bool reuseport_cb_ok, + bool reuseport_ok) +{ + struct sock *sk2; + + sk_for_each_bound_bhash2(sk2, &tb2->owners) { + if (sk->sk_family == AF_INET && ipv6_only_sock(sk2)) + continue; + + if (inet_bind_conflict(sk, sk2, sk_uid, relax, + reuseport_cb_ok, reuseport_ok)) + return true; + } + return false; +} + +/* This should be called only when the tb and tb2 hashbuckets' locks are held */ static int inet_csk_bind_conflict(const struct sock *sk, const struct inet_bind_bucket *tb, + const struct inet_bind2_bucket *tb2, /* may be null */ bool relax, bool reuseport_ok) { - struct sock *sk2; bool reuseport_cb_ok; - bool reuse = sk->sk_reuse; - bool reuseport = !!sk->sk_reuseport; struct sock_reuseport *reuseport_cb; kuid_t uid = sock_i_uid((struct sock *)sk); @@ -150,58 +211,88 @@ static int inet_csk_bind_conflict(const struct sock *sk, /* * Unlike other sk lookup places we do not check * for sk_net here, since _all_ the socks listed - * in tb->owners list belong to the same net - the - * one this bucket belongs to. + * in tb->owners and tb2->owners list belong + * to the same net - the one this bucket belongs to. */ - sk_for_each_bound(sk2, &tb->owners) { - int bound_dev_if2; + if (!inet_use_bhash2_on_bind(sk)) { + struct sock *sk2; - if (sk == sk2) - continue; - bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if); - if ((!sk->sk_bound_dev_if || - !bound_dev_if2 || - sk->sk_bound_dev_if == bound_dev_if2)) { - if (reuse && sk2->sk_reuse && - sk2->sk_state != TCP_LISTEN) { - if ((!relax || - (!reuseport_ok && - reuseport && sk2->sk_reuseport && - reuseport_cb_ok && - (sk2->sk_state == TCP_TIME_WAIT || - uid_eq(uid, sock_i_uid(sk2))))) && - inet_rcv_saddr_equal(sk, sk2, true)) - break; - } else if (!reuseport_ok || - !reuseport || !sk2->sk_reuseport || - !reuseport_cb_ok || - (sk2->sk_state != TCP_TIME_WAIT && - !uid_eq(uid, sock_i_uid(sk2)))) { - if (inet_rcv_saddr_equal(sk, sk2, true)) - break; - } - } + sk_for_each_bound(sk2, &tb->owners) + if (inet_bind_conflict(sk, sk2, uid, relax, + reuseport_cb_ok, reuseport_ok) && + inet_rcv_saddr_equal(sk, sk2, true)) + return true; + + return false; + } + + /* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if + * ipv4) should have been checked already. We need to do these two + * checks separately because their spinlocks have to be acquired/released + * independently of each other, to prevent possible deadlocks + */ + return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, + reuseport_ok); +} + +/* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or + * INADDR_ANY (if ipv4) socket. + * + * Caller must hold bhash hashbucket lock with local bh disabled, to protect + * against concurrent binds on the port for addr any + */ +static bool inet_bhash2_addr_any_conflict(const struct sock *sk, int port, int l3mdev, + bool relax, bool reuseport_ok) +{ + kuid_t uid = sock_i_uid((struct sock *)sk); + const struct net *net = sock_net(sk); + struct sock_reuseport *reuseport_cb; + struct inet_bind_hashbucket *head2; + struct inet_bind2_bucket *tb2; + bool reuseport_cb_ok; + + rcu_read_lock(); + reuseport_cb = rcu_dereference(sk->sk_reuseport_cb); + /* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */ + reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks); + rcu_read_unlock(); + + head2 = inet_bhash2_addr_any_hashbucket(sk, net, port); + + spin_lock(&head2->lock); + + inet_bind_bucket_for_each(tb2, &head2->chain) + if (inet_bind2_bucket_match_addr_any(tb2, net, port, l3mdev, sk)) + break; + + if (tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok, + reuseport_ok)) { + spin_unlock(&head2->lock); + return true; } - return sk2 != NULL; + + spin_unlock(&head2->lock); + return false; } /* * Find an open port number for the socket. Returns with the - * inet_bind_hashbucket lock held. + * inet_bind_hashbucket locks held if successful. */ static struct inet_bind_hashbucket * -inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret) +inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret, + struct inet_bind2_bucket **tb2_ret, + struct inet_bind_hashbucket **head2_ret, int *port_ret) { - struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; - int port = 0; - struct inet_bind_hashbucket *head; + struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); + int i, low, high, attempt_half, port, l3mdev; + struct inet_bind_hashbucket *head, *head2; struct net *net = sock_net(sk); - bool relax = false; - int i, low, high, attempt_half; + struct inet_bind2_bucket *tb2; struct inet_bind_bucket *tb; u32 remaining, offset; - int l3mdev; + bool relax = false; l3mdev = inet_sk_bound_l3mdev(sk); ports_exhausted: @@ -223,7 +314,7 @@ other_half_scan: if (likely(remaining > 1)) remaining &= ~1U; - offset = prandom_u32() % remaining; + offset = prandom_u32_max(remaining); /* __inet_hash_connect() favors ports having @low parity * We do the opposite to not pollute connect() users. */ @@ -239,11 +330,20 @@ other_parity_scan: head = &hinfo->bhash[inet_bhashfn(net, port, hinfo->bhash_size)]; spin_lock_bh(&head->lock); + if (inet_use_bhash2_on_bind(sk)) { + if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false)) + goto next_port; + } + + head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); + spin_lock(&head2->lock); + tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); inet_bind_bucket_for_each(tb, &head->chain) - if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev && - tb->port == port) { - if (!inet_csk_bind_conflict(sk, tb, relax, false)) + if (inet_bind_bucket_match(tb, net, port, l3mdev)) { + if (!inet_csk_bind_conflict(sk, tb, tb2, + relax, false)) goto success; + spin_unlock(&head2->lock); goto next_port; } tb = NULL; @@ -272,6 +372,8 @@ next_port: success: *port_ret = port; *tb_ret = tb; + *tb2_ret = tb2; + *head2_ret = head2; return head; } @@ -365,56 +467,97 @@ void inet_csk_update_fastreuse(struct inet_bind_bucket *tb, */ int inet_csk_get_port(struct sock *sk, unsigned short snum) { + struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN; - struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo; - int ret = 1, port = snum; - struct inet_bind_hashbucket *head; - struct net *net = sock_net(sk); + bool found_port = false, check_bind_conflict = true; + bool bhash_created = false, bhash2_created = false; + struct inet_bind_hashbucket *head, *head2; + struct inet_bind2_bucket *tb2 = NULL; struct inet_bind_bucket *tb = NULL; - int l3mdev; + bool head2_lock_acquired = false; + int ret = 1, port = snum, l3mdev; + struct net *net = sock_net(sk); l3mdev = inet_sk_bound_l3mdev(sk); if (!port) { - head = inet_csk_find_open_port(sk, &tb, &port); + head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port); if (!head) return ret; + + head2_lock_acquired = true; + + if (tb && tb2) + goto success; + found_port = true; + } else { + head = &hinfo->bhash[inet_bhashfn(net, port, + hinfo->bhash_size)]; + spin_lock_bh(&head->lock); + inet_bind_bucket_for_each(tb, &head->chain) + if (inet_bind_bucket_match(tb, net, port, l3mdev)) + break; + } + + if (!tb) { + tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net, + head, port, l3mdev); if (!tb) - goto tb_not_found; - goto success; + goto fail_unlock; + bhash_created = true; } - head = &hinfo->bhash[inet_bhashfn(net, port, - hinfo->bhash_size)]; - spin_lock_bh(&head->lock); - inet_bind_bucket_for_each(tb, &head->chain) - if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev && - tb->port == port) - goto tb_found; -tb_not_found: - tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, - net, head, port, l3mdev); - if (!tb) - goto fail_unlock; -tb_found: - if (!hlist_empty(&tb->owners)) { - if (sk->sk_reuse == SK_FORCE_REUSE) - goto success; - if ((tb->fastreuse > 0 && reuse) || - sk_reuseport_match(tb, sk)) - goto success; - if (inet_csk_bind_conflict(sk, tb, true, true)) + if (!found_port) { + if (!hlist_empty(&tb->owners)) { + if (sk->sk_reuse == SK_FORCE_REUSE || + (tb->fastreuse > 0 && reuse) || + sk_reuseport_match(tb, sk)) + check_bind_conflict = false; + } + + if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) { + if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true)) + goto fail_unlock; + } + + head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); + spin_lock(&head2->lock); + head2_lock_acquired = true; + tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); + } + + if (!tb2) { + tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, + net, head2, port, l3mdev, sk); + if (!tb2) goto fail_unlock; + bhash2_created = true; } + + if (!found_port && check_bind_conflict) { + if (inet_csk_bind_conflict(sk, tb, tb2, true, true)) + goto fail_unlock; + } + success: inet_csk_update_fastreuse(tb, sk); if (!inet_csk(sk)->icsk_bind_hash) - inet_bind_hash(sk, tb, port); + inet_bind_hash(sk, tb, tb2, port); WARN_ON(inet_csk(sk)->icsk_bind_hash != tb); + WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2); ret = 0; fail_unlock: + if (ret) { + if (bhash_created) + inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); + if (bhash2_created) + inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, + tb2); + } + if (head2_lock_acquired) + spin_unlock(&head2->lock); spin_unlock_bh(&head->lock); return ret; } @@ -763,14 +906,15 @@ static void reqsk_migrate_reset(struct request_sock *req) /* return true if req was found in the ehash table */ static bool reqsk_queue_unlink(struct request_sock *req) { - struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo; + struct sock *sk = req_to_sk(req); bool found = false; - if (sk_hashed(req_to_sk(req))) { + if (sk_hashed(sk)) { + struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash); spin_lock(lock); - found = __sk_nulls_del_node_init_rcu(req_to_sk(req)); + found = __sk_nulls_del_node_init_rcu(sk); spin_unlock(lock); } if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer)) @@ -962,6 +1106,7 @@ struct sock *inet_csk_clone_lock(const struct sock *sk, inet_sk_set_state(newsk, TCP_SYN_RECV); newicsk->icsk_bind_hash = NULL; + newicsk->icsk_bind2_hash = NULL; inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port; inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num; diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index b9d995b5ce24cc259dfc7c8b42b636eca4bfc78e..d3dc281566229479aa9027229a37973f8507d014 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -92,12 +92,79 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket } } +bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, const struct net *net, + unsigned short port, int l3mdev) +{ + return net_eq(ib_net(tb), net) && tb->port == port && + tb->l3mdev == l3mdev; +} + +static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb, + struct net *net, + struct inet_bind_hashbucket *head, + unsigned short port, int l3mdev, + const struct sock *sk) +{ + write_pnet(&tb->ib_net, net); + tb->l3mdev = l3mdev; + tb->port = port; +#if IS_ENABLED(CONFIG_IPV6) + tb->family = sk->sk_family; + if (sk->sk_family == AF_INET6) + tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr; + else +#endif + tb->rcv_saddr = sk->sk_rcv_saddr; + INIT_HLIST_HEAD(&tb->owners); + hlist_add_head(&tb->node, &head->chain); +} + +struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep, + struct net *net, + struct inet_bind_hashbucket *head, + unsigned short port, + int l3mdev, + const struct sock *sk) +{ + struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); + + if (tb) + inet_bind2_bucket_init(tb, net, head, port, l3mdev, sk); + + return tb; +} + +/* Caller must hold hashbucket lock for this tb with local BH disabled */ +void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb) +{ + if (hlist_empty(&tb->owners)) { + __hlist_del(&tb->node); + kmem_cache_free(cachep, tb); + } +} + +static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2, + const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family != tb2->family) + return false; + + if (sk->sk_family == AF_INET6) + return ipv6_addr_equal(&tb2->v6_rcv_saddr, + &sk->sk_v6_rcv_saddr); +#endif + return tb2->rcv_saddr == sk->sk_rcv_saddr; +} + void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, - const unsigned short snum) + struct inet_bind2_bucket *tb2, unsigned short port) { - inet_sk(sk)->inet_num = snum; + inet_sk(sk)->inet_num = port; sk_add_bind_node(sk, &tb->owners); inet_csk(sk)->icsk_bind_hash = tb; + sk_add_bind2_node(sk, &tb2->owners); + inet_csk(sk)->icsk_bind2_hash = tb2; } /* @@ -105,11 +172,15 @@ void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, */ static void __inet_put_port(struct sock *sk) { - struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; - const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num, - hashinfo->bhash_size); - struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; + struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); + struct inet_bind_hashbucket *head, *head2; + struct net *net = sock_net(sk); struct inet_bind_bucket *tb; + int bhash; + + bhash = inet_bhashfn(net, inet_sk(sk)->inet_num, hashinfo->bhash_size); + head = &hashinfo->bhash[bhash]; + head2 = inet_bhashfn_portaddr(hashinfo, sk, net, inet_sk(sk)->inet_num); spin_lock(&head->lock); tb = inet_csk(sk)->icsk_bind_hash; @@ -117,6 +188,17 @@ static void __inet_put_port(struct sock *sk) inet_csk(sk)->icsk_bind_hash = NULL; inet_sk(sk)->inet_num = 0; inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); + + spin_lock(&head2->lock); + if (inet_csk(sk)->icsk_bind2_hash) { + struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash; + + __sk_del_bind2_node(sk); + inet_csk(sk)->icsk_bind2_hash = NULL; + inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2); + } + spin_unlock(&head2->lock); + spin_unlock(&head->lock); } @@ -130,17 +212,26 @@ EXPORT_SYMBOL(inet_put_port); int __inet_inherit_port(const struct sock *sk, struct sock *child) { - struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; + struct inet_hashinfo *table = tcp_or_dccp_get_hashinfo(sk); unsigned short port = inet_sk(child)->inet_num; - const int bhash = inet_bhashfn(sock_net(sk), port, - table->bhash_size); - struct inet_bind_hashbucket *head = &table->bhash[bhash]; + struct inet_bind_hashbucket *head, *head2; + bool created_inet_bind_bucket = false; + struct net *net = sock_net(sk); + bool update_fastreuse = false; + struct inet_bind2_bucket *tb2; struct inet_bind_bucket *tb; - int l3mdev; + int bhash, l3mdev; + + bhash = inet_bhashfn(net, port, table->bhash_size); + head = &table->bhash[bhash]; + head2 = inet_bhashfn_portaddr(table, child, net, port); spin_lock(&head->lock); + spin_lock(&head2->lock); tb = inet_csk(sk)->icsk_bind_hash; - if (unlikely(!tb)) { + tb2 = inet_csk(sk)->icsk_bind2_hash; + if (unlikely(!tb || !tb2)) { + spin_unlock(&head2->lock); spin_unlock(&head->lock); return -ENOENT; } @@ -153,25 +244,49 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child) * as that of the child socket. We have to look up or * create a new bind bucket for the child here. */ inet_bind_bucket_for_each(tb, &head->chain) { - if (net_eq(ib_net(tb), sock_net(sk)) && - tb->l3mdev == l3mdev && tb->port == port) + if (inet_bind_bucket_match(tb, net, port, l3mdev)) break; } if (!tb) { tb = inet_bind_bucket_create(table->bind_bucket_cachep, - sock_net(sk), head, port, - l3mdev); + net, head, port, l3mdev); if (!tb) { + spin_unlock(&head2->lock); spin_unlock(&head->lock); return -ENOMEM; } + created_inet_bind_bucket = true; + } + update_fastreuse = true; + + goto bhash2_find; + } else if (!inet_bind2_bucket_addr_match(tb2, child)) { + l3mdev = inet_sk_bound_l3mdev(sk); + +bhash2_find: + tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, child); + if (!tb2) { + tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep, + net, head2, port, + l3mdev, child); + if (!tb2) + goto error; } - inet_csk_update_fastreuse(tb, child); } - inet_bind_hash(child, tb, port); + if (update_fastreuse) + inet_csk_update_fastreuse(tb, child); + inet_bind_hash(child, tb, tb2, port); + spin_unlock(&head2->lock); spin_unlock(&head->lock); return 0; + +error: + if (created_inet_bind_bucket) + inet_bind_bucket_destroy(table->bind_bucket_cachep, tb); + spin_unlock(&head2->lock); + spin_unlock(&head->lock); + return -ENOMEM; } EXPORT_SYMBOL_GPL(__inet_inherit_port); @@ -275,7 +390,7 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net, struct sock *sk, *reuse_sk; bool no_reuseport; - if (hashinfo != &tcp_hashinfo) + if (hashinfo != net->ipv4.tcp_death_row.hashinfo) return NULL; /* only TCP is supported */ no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport, @@ -518,9 +633,9 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk, */ bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk) { - struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; - struct hlist_nulls_head *list; + struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); struct inet_ehash_bucket *head; + struct hlist_nulls_head *list; spinlock_t *lock; bool ret = true; @@ -590,7 +705,7 @@ static int inet_reuseport_add_sock(struct sock *sk, int __inet_hash(struct sock *sk, struct sock *osk) { - struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; + struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); struct inet_listen_hashbucket *ilb2; int err = 0; @@ -636,7 +751,7 @@ EXPORT_SYMBOL_GPL(inet_hash); void inet_unhash(struct sock *sk) { - struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; + struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk); if (sk_unhashed(sk)) return; @@ -675,6 +790,118 @@ void inet_unhash(struct sock *sk) } EXPORT_SYMBOL_GPL(inet_unhash); +static bool inet_bind2_bucket_match(const struct inet_bind2_bucket *tb, + const struct net *net, unsigned short port, + int l3mdev, const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family != tb->family) + return false; + + if (sk->sk_family == AF_INET6) + return net_eq(ib2_net(tb), net) && tb->port == port && + tb->l3mdev == l3mdev && + ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr); + else +#endif + return net_eq(ib2_net(tb), net) && tb->port == port && + tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr; +} + +bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb, const struct net *net, + unsigned short port, int l3mdev, const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr addr_any = {}; + + if (sk->sk_family != tb->family) + return false; + + if (sk->sk_family == AF_INET6) + return net_eq(ib2_net(tb), net) && tb->port == port && + tb->l3mdev == l3mdev && + ipv6_addr_equal(&tb->v6_rcv_saddr, &addr_any); + else +#endif + return net_eq(ib2_net(tb), net) && tb->port == port && + tb->l3mdev == l3mdev && tb->rcv_saddr == 0; +} + +/* The socket's bhash2 hashbucket spinlock must be held when this is called */ +struct inet_bind2_bucket * +inet_bind2_bucket_find(const struct inet_bind_hashbucket *head, const struct net *net, + unsigned short port, int l3mdev, const struct sock *sk) +{ + struct inet_bind2_bucket *bhash2 = NULL; + + inet_bind_bucket_for_each(bhash2, &head->chain) + if (inet_bind2_bucket_match(bhash2, net, port, l3mdev, sk)) + break; + + return bhash2; +} + +struct inet_bind_hashbucket * +inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port) +{ + struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); + u32 hash; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr addr_any = {}; + + if (sk->sk_family == AF_INET6) + hash = ipv6_portaddr_hash(net, &addr_any, port); + else +#endif + hash = ipv4_portaddr_hash(net, 0, port); + + return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; +} + +int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk) +{ + struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk); + struct inet_bind2_bucket *tb2, *new_tb2; + int l3mdev = inet_sk_bound_l3mdev(sk); + struct inet_bind_hashbucket *head2; + int port = inet_sk(sk)->inet_num; + struct net *net = sock_net(sk); + + /* Allocate a bind2 bucket ahead of time to avoid permanently putting + * the bhash2 table in an inconsistent state if a new tb2 bucket + * allocation fails. + */ + new_tb2 = kmem_cache_alloc(hinfo->bind2_bucket_cachep, GFP_ATOMIC); + if (!new_tb2) + return -ENOMEM; + + head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); + + if (prev_saddr) { + spin_lock_bh(&prev_saddr->lock); + __sk_del_bind2_node(sk); + inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep, + inet_csk(sk)->icsk_bind2_hash); + spin_unlock_bh(&prev_saddr->lock); + } + + spin_lock_bh(&head2->lock); + tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); + if (!tb2) { + tb2 = new_tb2; + inet_bind2_bucket_init(tb2, net, head2, port, l3mdev, sk); + } + sk_add_bind2_node(sk, &tb2->owners); + inet_csk(sk)->icsk_bind2_hash = tb2; + spin_unlock_bh(&head2->lock); + + if (tb2 != new_tb2) + kmem_cache_free(hinfo->bind2_bucket_cachep, new_tb2); + + return 0; +} +EXPORT_SYMBOL_GPL(inet_bhash2_update_saddr); + /* RFC 6056 3.3.4. Algorithm 4: Double-Hash Port Selection Algorithm * Note that we use 32bit integers (vs RFC 'short integers') * because 2^16 is not a multiple of num_ephemeral and this @@ -694,11 +921,13 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *, __u16, struct inet_timewait_sock **)) { struct inet_hashinfo *hinfo = death_row->hashinfo; + struct inet_bind_hashbucket *head, *head2; struct inet_timewait_sock *tw = NULL; - struct inet_bind_hashbucket *head; int port = inet_sk(sk)->inet_num; struct net *net = sock_net(sk); + struct inet_bind2_bucket *tb2; struct inet_bind_bucket *tb; + bool tb_created = false; u32 remaining, offset; int ret, i, low, high; int l3mdev; @@ -729,8 +958,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row, if (likely(remaining > 1)) remaining &= ~1U; - net_get_random_once(table_perturb, - INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb)); + get_random_sleepable_once(table_perturb, + INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb)); index = port_offset & (INET_TABLE_PERTURB_SIZE - 1); offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32); @@ -755,8 +984,7 @@ other_parity_scan: * the established check is already unique enough. */ inet_bind_bucket_for_each(tb, &head->chain) { - if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev && - tb->port == port) { + if (inet_bind_bucket_match(tb, net, port, l3mdev)) { if (tb->fastreuse >= 0 || tb->fastreuseport >= 0) goto next_port; @@ -774,6 +1002,7 @@ other_parity_scan: spin_unlock_bh(&head->lock); return -ENOMEM; } + tb_created = true; tb->fastreuse = -1; tb->fastreuseport = -1; goto ok; @@ -789,16 +1018,33 @@ next_port: return -EADDRNOTAVAIL; ok: + /* Find the corresponding tb2 bucket since we need to + * add the socket to the bhash2 table as well + */ + head2 = inet_bhashfn_portaddr(hinfo, sk, net, port); + spin_lock(&head2->lock); + + tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk); + if (!tb2) { + tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net, + head2, port, l3mdev, sk); + if (!tb2) + goto error; + } + /* Here we want to add a little bit of randomness to the next source * port that will be chosen. We use a max() with a random here so that * on low contention the randomness is maximal and on high contention * it may be inexistent. */ - i = max_t(int, i, (prandom_u32() & 7) * 2); + i = max_t(int, i, prandom_u32_max(8) * 2); WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2); /* Head lock still held and bh's disabled */ - inet_bind_hash(sk, tb, port); + inet_bind_hash(sk, tb, tb2, port); + + spin_unlock(&head2->lock); + if (sk_unhashed(sk)) { inet_sk(sk)->inet_sport = htons(port); inet_ehash_nolisten(sk, (struct sock *)tw, NULL); @@ -810,6 +1056,13 @@ ok: inet_twsk_deschedule_put(tw); local_bh_enable(); return 0; + +error: + spin_unlock(&head2->lock); + if (tb_created) + inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb); + spin_unlock_bh(&head->lock); + return -ENOMEM; } /* @@ -902,3 +1155,50 @@ int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo) return 0; } EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc); + +struct inet_hashinfo *inet_pernet_hashinfo_alloc(struct inet_hashinfo *hashinfo, + unsigned int ehash_entries) +{ + struct inet_hashinfo *new_hashinfo; + int i; + + new_hashinfo = kmemdup(hashinfo, sizeof(*hashinfo), GFP_KERNEL); + if (!new_hashinfo) + goto err; + + new_hashinfo->ehash = vmalloc_huge(ehash_entries * sizeof(struct inet_ehash_bucket), + GFP_KERNEL_ACCOUNT); + if (!new_hashinfo->ehash) + goto free_hashinfo; + + new_hashinfo->ehash_mask = ehash_entries - 1; + + if (inet_ehash_locks_alloc(new_hashinfo)) + goto free_ehash; + + for (i = 0; i < ehash_entries; i++) + INIT_HLIST_NULLS_HEAD(&new_hashinfo->ehash[i].chain, i); + + new_hashinfo->pernet = true; + + return new_hashinfo; + +free_ehash: + vfree(new_hashinfo->ehash); +free_hashinfo: + kfree(new_hashinfo); +err: + return NULL; +} +EXPORT_SYMBOL_GPL(inet_pernet_hashinfo_alloc); + +void inet_pernet_hashinfo_free(struct inet_hashinfo *hashinfo) +{ + if (!hashinfo->pernet) + return; + + inet_ehash_locks_free(hashinfo); + vfree(hashinfo->ehash); + kfree(hashinfo); +} +EXPORT_SYMBOL_GPL(inet_pernet_hashinfo_free); diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c index 47ccc343c9fb0a995b817d938f25a779dd8221c0..66fc940f9521abb5c9f83b70cdb482afa1ea6ede 100644 --- a/net/ipv4/inet_timewait_sock.c +++ b/net/ipv4/inet_timewait_sock.c @@ -59,9 +59,7 @@ static void inet_twsk_kill(struct inet_timewait_sock *tw) inet_twsk_bind_unhash(tw, hashinfo); spin_unlock(&bhead->lock); - if (refcount_dec_and_test(&tw->tw_dr->tw_refcount)) - kfree(tw->tw_dr); - + refcount_dec(&tw->tw_dr->tw_refcount); inet_twsk_put(tw); } @@ -270,8 +268,21 @@ restart_rcu: rcu_read_lock(); restart: sk_nulls_for_each_rcu(sk, node, &head->chain) { - if (sk->sk_state != TCP_TIME_WAIT) + if (sk->sk_state != TCP_TIME_WAIT) { + /* A kernel listener socket might not hold refcnt for net, + * so reqsk_timer_handler() could be fired after net is + * freed. Userspace listener and reqsk never exist here. + */ + if (unlikely(sk->sk_state == TCP_NEW_SYN_RECV && + hashinfo->pernet)) { + struct request_sock *req = inet_reqsk(sk); + + inet_csk_reqsk_queue_drop_and_put(req->rsk_listener, req); + } + continue; + } + tw = inet_twsk(sk); if ((tw->tw_family != family) || refcount_read(&twsk_net(tw)->ns.count)) diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 5c58e21f724e98f4d7a450f9be2e5b25e98ecc8d..f866d6282b2b32031b60d495eaa683f757c44fd4 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -609,7 +609,7 @@ static int gre_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) ip_tunnel_init_flow(&fl4, IPPROTO_GRE, key->u.ipv4.dst, key->u.ipv4.src, tunnel_id_to_key32(key->tun_id), key->tos & ~INET_ECN_MASK, dev_net(dev), 0, - skb->mark, skb_get_hash(skb)); + skb->mark, skb_get_hash(skb), key->flow_flags); rt = ip_route_output_key(dev_net(dev), &fl4); if (IS_ERR(rt)) return PTR_ERR(rt); diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index d7bd1daf022b5a6a2c025e83346d347714f672c8..922c87ef1ab588205edda64fe09c2d7d460e67db 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -172,7 +172,7 @@ int ip_build_and_send_pkt(struct sk_buff *skb, const struct sock *sk, * Avoid using the hashed IP ident generator. */ if (sk->sk_protocol == IPPROTO_TCP) - iph->id = (__force __be16)prandom_u32(); + iph->id = (__force __be16)get_random_u16(); else __ip_select_ident(net, iph, 1); } @@ -1043,7 +1043,7 @@ static int __ip_append_data(struct sock *sk, paged = true; zc = true; } else { - uarg->zerocopy = 0; + uarg_to_msgzc(uarg)->zerocopy = 0; skb_zcopy_set(skb, uarg, &extra_uref); } } @@ -1109,10 +1109,7 @@ alloc_new_skb: (fraglen + alloc_extra < SKB_MAX_ALLOC || !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; - else if (!zc) { - alloclen = min_t(int, fraglen, MAX_HEADER); - pagedlen = fraglen - alloclen; - } else { + else { alloclen = fragheaderlen + transhdrlen; pagedlen = datalen - transhdrlen; } @@ -1730,7 +1727,7 @@ void ip_send_unicast_reply(struct sock *sk, struct sk_buff *skb, sk->sk_protocol = ip_hdr(skb)->protocol; sk->sk_bound_dev_if = arg->bound_dev_if; - sk->sk_sndbuf = sysctl_wmem_default; + sk->sk_sndbuf = READ_ONCE(sysctl_wmem_default); ipc.sockc.mark = fl4.flowi4_mark; err = ip_append_data(sk, &fl4, ip_reply_glue_bits, arg->iov->iov_base, len, 0, &ipc, &rt, MSG_DONTWAIT); diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index a8a323ecbb54b702e0a744e44f34ddcef7e2d383..6e19cad154f5cdf7e4aecb745e0c85002d27369e 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -772,7 +772,7 @@ static int ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, int optlen) if (optlen < GROUP_FILTER_SIZE(0)) return -EINVAL; - if (optlen > sysctl_optmem_max) + if (optlen > READ_ONCE(sysctl_optmem_max)) return -ENOBUFS; gsf = memdup_sockptr(optval, optlen); @@ -808,7 +808,7 @@ static int compat_ip_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < size0) return -EINVAL; - if (optlen > sysctl_optmem_max - 4) + if (optlen > READ_ONCE(sysctl_optmem_max) - 4) return -ENOBUFS; p = kmalloc(optlen + 4, GFP_KERNEL); @@ -888,8 +888,8 @@ static int compat_ip_mcast_join_leave(struct sock *sk, int optname, DEFINE_STATIC_KEY_FALSE(ip4_min_ttl); -static int do_ip_setsockopt(struct sock *sk, int level, int optname, - sockptr_t optval, unsigned int optlen) +int do_ip_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen) { struct inet_sock *inet = inet_sk(sk); struct net *net = sock_net(sk); @@ -944,7 +944,7 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname, err = 0; if (needs_rtnl) rtnl_lock(); - lock_sock(sk); + sockopt_lock_sock(sk); switch (optname) { case IP_OPTIONS: @@ -1233,7 +1233,7 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname, if (optlen < IP_MSFILTER_SIZE(0)) goto e_inval; - if (optlen > sysctl_optmem_max) { + if (optlen > READ_ONCE(sysctl_optmem_max)) { err = -ENOBUFS; break; } @@ -1333,14 +1333,14 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname, case IP_IPSEC_POLICY: case IP_XFRM_POLICY: err = -EPERM; - if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) + if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) break; err = xfrm_user_policy(sk, optname, optval, optlen); break; case IP_TRANSPARENT: - if (!!val && !ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && - !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + if (!!val && !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && + !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { err = -EPERM; break; } @@ -1368,13 +1368,13 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname, err = -ENOPROTOOPT; break; } - release_sock(sk); + sockopt_release_sock(sk); if (needs_rtnl) rtnl_unlock(); return err; e_inval: - release_sock(sk); + sockopt_release_sock(sk); if (needs_rtnl) rtnl_unlock(); return -EINVAL; @@ -1462,37 +1462,37 @@ static bool getsockopt_needs_rtnl(int optname) return false; } -static int ip_get_mcast_msfilter(struct sock *sk, void __user *optval, - int __user *optlen, int len) +static int ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval, + sockptr_t optlen, int len) { const int size0 = offsetof(struct group_filter, gf_slist_flex); - struct group_filter __user *p = optval; struct group_filter gsf; - int num; + int num, gsf_size; int err; if (len < size0) return -EINVAL; - if (copy_from_user(&gsf, p, size0)) + if (copy_from_sockptr(&gsf, optval, size0)) return -EFAULT; num = gsf.gf_numsrc; - err = ip_mc_gsfget(sk, &gsf, p->gf_slist_flex); + err = ip_mc_gsfget(sk, &gsf, optval, + offsetof(struct group_filter, gf_slist_flex)); if (err) return err; if (gsf.gf_numsrc < num) num = gsf.gf_numsrc; - if (put_user(GROUP_FILTER_SIZE(num), optlen) || - copy_to_user(p, &gsf, size0)) + gsf_size = GROUP_FILTER_SIZE(num); + if (copy_to_sockptr(optlen, &gsf_size, sizeof(int)) || + copy_to_sockptr(optval, &gsf, size0)) return -EFAULT; return 0; } -static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval, - int __user *optlen, int len) +static int compat_ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval, + sockptr_t optlen, int len) { const int size0 = offsetof(struct compat_group_filter, gf_slist_flex); - struct compat_group_filter __user *p = optval; struct compat_group_filter gf32; struct group_filter gf; int num; @@ -1500,7 +1500,7 @@ static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval, if (len < size0) return -EINVAL; - if (copy_from_user(&gf32, p, size0)) + if (copy_from_sockptr(&gf32, optval, size0)) return -EFAULT; gf.gf_interface = gf32.gf_interface; @@ -1508,21 +1508,24 @@ static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval, num = gf.gf_numsrc = gf32.gf_numsrc; gf.gf_group = gf32.gf_group; - err = ip_mc_gsfget(sk, &gf, p->gf_slist_flex); + err = ip_mc_gsfget(sk, &gf, optval, + offsetof(struct compat_group_filter, gf_slist_flex)); if (err) return err; if (gf.gf_numsrc < num) num = gf.gf_numsrc; len = GROUP_FILTER_SIZE(num) - (sizeof(gf) - sizeof(gf32)); - if (put_user(len, optlen) || - put_user(gf.gf_fmode, &p->gf_fmode) || - put_user(gf.gf_numsrc, &p->gf_numsrc)) + if (copy_to_sockptr(optlen, &len, sizeof(int)) || + copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode), + &gf.gf_fmode, sizeof(gf.gf_fmode)) || + copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc), + &gf.gf_numsrc, sizeof(gf.gf_numsrc))) return -EFAULT; return 0; } -static int do_ip_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen) +int do_ip_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen) { struct inet_sock *inet = inet_sk(sk); bool needs_rtnl = getsockopt_needs_rtnl(optname); @@ -1535,14 +1538,14 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, if (ip_mroute_opt(optname)) return ip_mroute_getsockopt(sk, optname, optval, optlen); - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len < 0) return -EINVAL; if (needs_rtnl) rtnl_lock(); - lock_sock(sk); + sockopt_lock_sock(sk); switch (optname) { case IP_OPTIONS: @@ -1558,17 +1561,19 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, memcpy(optbuf, &inet_opt->opt, sizeof(struct ip_options) + inet_opt->opt.optlen); - release_sock(sk); + sockopt_release_sock(sk); - if (opt->optlen == 0) - return put_user(0, optlen); + if (opt->optlen == 0) { + len = 0; + return copy_to_sockptr(optlen, &len, sizeof(int)); + } ip_options_undo(opt); len = min_t(unsigned int, len, opt->optlen); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, opt->__data, len)) + if (copy_to_sockptr(optval, opt->__data, len)) return -EFAULT; return 0; } @@ -1632,7 +1637,7 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, dst_release(dst); } if (!val) { - release_sock(sk); + sockopt_release_sock(sk); return -ENOTCONN; } break; @@ -1657,11 +1662,11 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, struct in_addr addr; len = min_t(unsigned int, len, sizeof(struct in_addr)); addr.s_addr = inet->mc_addr; - release_sock(sk); + sockopt_release_sock(sk); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &addr, len)) + if (copy_to_sockptr(optval, &addr, len)) return -EFAULT; return 0; } @@ -1673,12 +1678,11 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, err = -EINVAL; goto out; } - if (copy_from_user(&msf, optval, IP_MSFILTER_SIZE(0))) { + if (copy_from_sockptr(&msf, optval, IP_MSFILTER_SIZE(0))) { err = -EFAULT; goto out; } - err = ip_mc_msfget(sk, &msf, - (struct ip_msfilter __user *)optval, optlen); + err = ip_mc_msfget(sk, &msf, optval, optlen); goto out; } case MCAST_MSFILTER: @@ -1695,13 +1699,18 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, { struct msghdr msg; - release_sock(sk); + sockopt_release_sock(sk); if (sk->sk_type != SOCK_STREAM) return -ENOPROTOOPT; - msg.msg_control_is_user = true; - msg.msg_control_user = optval; + if (optval.is_kernel) { + msg.msg_control_is_user = false; + msg.msg_control = optval.kernel; + } else { + msg.msg_control_is_user = true; + msg.msg_control_user = optval.user; + } msg.msg_controllen = len; msg.msg_flags = in_compat_syscall() ? MSG_CMSG_COMPAT : 0; @@ -1722,7 +1731,7 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, put_cmsg(&msg, SOL_IP, IP_TOS, sizeof(tos), &tos); } len -= msg.msg_controllen; - return put_user(len, optlen); + return copy_to_sockptr(optlen, &len, sizeof(int)); } case IP_FREEBIND: val = inet->freebind; @@ -1734,29 +1743,29 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname, val = inet->min_ttl; break; default: - release_sock(sk); + sockopt_release_sock(sk); return -ENOPROTOOPT; } - release_sock(sk); + sockopt_release_sock(sk); if (len < sizeof(int) && len > 0 && val >= 0 && val <= 255) { unsigned char ucval = (unsigned char)val; len = 1; - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &ucval, 1)) + if (copy_to_sockptr(optval, &ucval, 1)) return -EFAULT; } else { len = min_t(unsigned int, sizeof(int), len); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &val, len)) + if (copy_to_sockptr(optval, &val, len)) return -EFAULT; } return 0; out: - release_sock(sk); + sockopt_release_sock(sk); if (needs_rtnl) rtnl_unlock(); return err; @@ -1767,7 +1776,8 @@ int ip_getsockopt(struct sock *sk, int level, { int err; - err = do_ip_getsockopt(sk, level, optname, optval, optlen); + err = do_ip_getsockopt(sk, level, optname, + USER_SOCKPTR(optval), USER_SOCKPTR(optlen)); #if IS_ENABLED(CONFIG_BPFILTER_UMH) if (optname >= BPFILTER_IPT_SO_GET_INFO && diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index e65e948cab9f2a598cac79406f2b8e0ab9c79aba..019f3b0839c5225c7055f97cf15c96533fe32a2f 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -295,7 +295,7 @@ static int ip_tunnel_bind_dev(struct net_device *dev) ip_tunnel_init_flow(&fl4, iph->protocol, iph->daddr, iph->saddr, tunnel->parms.o_key, RT_TOS(iph->tos), dev_net(dev), - tunnel->parms.link, tunnel->fwmark, 0); + tunnel->parms.link, tunnel->fwmark, 0, 0); rt = ip_route_output_key(tunnel->net, &fl4); if (!IS_ERR(rt)) { @@ -570,7 +570,8 @@ void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, } ip_tunnel_init_flow(&fl4, proto, key->u.ipv4.dst, key->u.ipv4.src, tunnel_id_to_key32(key->tun_id), RT_TOS(tos), - dev_net(dev), 0, skb->mark, skb_get_hash(skb)); + dev_net(dev), 0, skb->mark, skb_get_hash(skb), + key->flow_flags); if (tunnel->encap.type != TUNNEL_ENCAP_NONE) goto tx_error; @@ -729,7 +730,7 @@ void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, ip_tunnel_init_flow(&fl4, protocol, dst, tnl_params->saddr, tunnel->parms.o_key, RT_TOS(tos), dev_net(dev), tunnel->parms.link, - tunnel->fwmark, skb_get_hash(skb)); + tunnel->fwmark, skb_get_hash(skb), 0); if (ip_tunnel_encap(skb, tunnel, &protocol, &fl4) < 0) goto tx_error; diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index cc1caab4a654921a267bac1cca56bb91244b9ea9..92c02c886fe735ba0dd784181fe297acf8a98faa 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -1079,3 +1079,70 @@ EXPORT_SYMBOL(ip_tunnel_parse_protocol); const struct header_ops ip_tunnel_header_ops = { .parse_protocol = ip_tunnel_parse_protocol }; EXPORT_SYMBOL(ip_tunnel_header_ops); + +/* This function returns true when ENCAP attributes are present in the nl msg */ +bool ip_tunnel_netlink_encap_parms(struct nlattr *data[], + struct ip_tunnel_encap *encap) +{ + bool ret = false; + + memset(encap, 0, sizeof(*encap)); + + if (!data) + return ret; + + if (data[IFLA_IPTUN_ENCAP_TYPE]) { + ret = true; + encap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]); + } + + if (data[IFLA_IPTUN_ENCAP_FLAGS]) { + ret = true; + encap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]); + } + + if (data[IFLA_IPTUN_ENCAP_SPORT]) { + ret = true; + encap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]); + } + + if (data[IFLA_IPTUN_ENCAP_DPORT]) { + ret = true; + encap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]); + } + + return ret; +} +EXPORT_SYMBOL_GPL(ip_tunnel_netlink_encap_parms); + +void ip_tunnel_netlink_parms(struct nlattr *data[], + struct ip_tunnel_parm *parms) +{ + if (data[IFLA_IPTUN_LINK]) + parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]); + + if (data[IFLA_IPTUN_LOCAL]) + parms->iph.saddr = nla_get_be32(data[IFLA_IPTUN_LOCAL]); + + if (data[IFLA_IPTUN_REMOTE]) + parms->iph.daddr = nla_get_be32(data[IFLA_IPTUN_REMOTE]); + + if (data[IFLA_IPTUN_TTL]) { + parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]); + if (parms->iph.ttl) + parms->iph.frag_off = htons(IP_DF); + } + + if (data[IFLA_IPTUN_TOS]) + parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]); + + if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC])) + parms->iph.frag_off = htons(IP_DF); + + if (data[IFLA_IPTUN_FLAGS]) + parms->i_flags = nla_get_be16(data[IFLA_IPTUN_FLAGS]); + + if (data[IFLA_IPTUN_PROTO]) + parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]); +} +EXPORT_SYMBOL_GPL(ip_tunnel_netlink_parms); diff --git a/net/ipv4/ipcomp.c b/net/ipv4/ipcomp.c index 366094c1ce6caf4dfe7b09b679c5caa0b4797ab5..5a4fb2539b08b60212a42638a8e6c4e39eaa472c 100644 --- a/net/ipv4/ipcomp.c +++ b/net/ipv4/ipcomp.c @@ -117,7 +117,8 @@ out: return err; } -static int ipcomp4_init_state(struct xfrm_state *x) +static int ipcomp4_init_state(struct xfrm_state *x, + struct netlink_ext_ack *extack) { int err = -EINVAL; @@ -129,17 +130,20 @@ static int ipcomp4_init_state(struct xfrm_state *x) x->props.header_len += sizeof(struct iphdr); break; default: + NL_SET_ERR_MSG(extack, "Unsupported XFRM mode for IPcomp"); goto out; } - err = ipcomp_init_state(x); + err = ipcomp_init_state(x, extack); if (err) goto out; if (x->props.mode == XFRM_MODE_TUNNEL) { err = ipcomp_tunnel_attach(x); - if (err) + if (err) { + NL_SET_ERR_MSG(extack, "Kernel error: failed to initialize the associated state"); goto out; + } } err = 0; diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 123ea63a04cbf04786bd7a170ea942301cc9e0bb..180f9daf5bec577b8f6062185ea13f7af9e5fe33 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -417,29 +417,7 @@ static void ipip_netlink_parms(struct nlattr *data[], if (!data) return; - if (data[IFLA_IPTUN_LINK]) - parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]); - - if (data[IFLA_IPTUN_LOCAL]) - parms->iph.saddr = nla_get_in_addr(data[IFLA_IPTUN_LOCAL]); - - if (data[IFLA_IPTUN_REMOTE]) - parms->iph.daddr = nla_get_in_addr(data[IFLA_IPTUN_REMOTE]); - - if (data[IFLA_IPTUN_TTL]) { - parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]); - if (parms->iph.ttl) - parms->iph.frag_off = htons(IP_DF); - } - - if (data[IFLA_IPTUN_TOS]) - parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]); - - if (data[IFLA_IPTUN_PROTO]) - parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]); - - if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC])) - parms->iph.frag_off = htons(IP_DF); + ip_tunnel_netlink_parms(data, parms); if (data[IFLA_IPTUN_COLLECT_METADATA]) *collect_md = true; @@ -448,40 +426,6 @@ static void ipip_netlink_parms(struct nlattr *data[], *fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]); } -/* This function returns true when ENCAP attributes are present in the nl msg */ -static bool ipip_netlink_encap_parms(struct nlattr *data[], - struct ip_tunnel_encap *ipencap) -{ - bool ret = false; - - memset(ipencap, 0, sizeof(*ipencap)); - - if (!data) - return ret; - - if (data[IFLA_IPTUN_ENCAP_TYPE]) { - ret = true; - ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]); - } - - if (data[IFLA_IPTUN_ENCAP_FLAGS]) { - ret = true; - ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]); - } - - if (data[IFLA_IPTUN_ENCAP_SPORT]) { - ret = true; - ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]); - } - - if (data[IFLA_IPTUN_ENCAP_DPORT]) { - ret = true; - ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]); - } - - return ret; -} - static int ipip_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) @@ -491,7 +435,7 @@ static int ipip_newlink(struct net *src_net, struct net_device *dev, struct ip_tunnel_encap ipencap; __u32 fwmark = 0; - if (ipip_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { int err = ip_tunnel_encap_setup(t, &ipencap); if (err < 0) @@ -512,7 +456,7 @@ static int ipip_changelink(struct net_device *dev, struct nlattr *tb[], bool collect_md; __u32 fwmark = t->fwmark; - if (ipip_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { int err = ip_tunnel_encap_setup(t, &ipencap); if (err < 0) diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 73651d17e51f31c8755da6ac3c1c2763a99b1117..e04544ac4b4545f5beb1fc7c5dc864aefc1b7324 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -1004,7 +1004,9 @@ static void ipmr_cache_resolve(struct net *net, struct mr_table *mrt, rtnl_unicast(skb, net, NETLINK_CB(skb).portid); } else { + rcu_read_lock(); ip_mr_forward(net, mrt, skb->dev, skb, c, 0); + rcu_read_unlock(); } } } @@ -1546,7 +1548,8 @@ out: } /* Getsock opt support for the multicast routing system. */ -int ip_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, int __user *optlen) +int ip_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval, + sockptr_t optlen) { int olr; int val; @@ -1577,14 +1580,14 @@ int ip_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, int return -ENOPROTOOPT; } - if (get_user(olr, optlen)) + if (copy_from_sockptr(&olr, optlen, sizeof(int))) return -EFAULT; olr = min_t(unsigned int, olr, sizeof(int)); if (olr < 0) return -EINVAL; - if (put_user(olr, optlen)) + if (copy_to_sockptr(optlen, &olr, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &val, olr)) + if (copy_to_sockptr(optval, &val, olr)) return -EFAULT; return 0; } diff --git a/net/ipv4/netfilter/ipt_rpfilter.c b/net/ipv4/netfilter/ipt_rpfilter.c index 8cd3224d913e0ca5cc0c227e813b5f279fed5545..ff85db52b2e56ecef625511ad51b6bf9b866c341 100644 --- a/net/ipv4/netfilter/ipt_rpfilter.c +++ b/net/ipv4/netfilter/ipt_rpfilter.c @@ -33,7 +33,6 @@ static bool rpfilter_lookup_reverse(struct net *net, struct flowi4 *fl4, const struct net_device *dev, u8 flags) { struct fib_result res; - int ret __maybe_unused; if (fib_lookup(net, fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE)) return false; @@ -78,7 +77,7 @@ static bool rpfilter_mt(const struct sk_buff *skb, struct xt_action_param *par) flow.flowi4_mark = info->flags & XT_RPFILTER_VALID_MARK ? skb->mark : 0; flow.flowi4_tos = iph->tos & IPTOS_RT_MASK; flow.flowi4_scope = RT_SCOPE_UNIVERSE; - flow.flowi4_oif = l3mdev_master_ifindex_rcu(xt_in(par)); + flow.flowi4_l3mdev = l3mdev_master_ifindex_rcu(xt_in(par)); return rpfilter_lookup_reverse(xt_net(par), &flow, xt_in(par), info->flags) ^ invert; } diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c index a334f0dcc2d0dbd430f836266b822334d9cb8f00..faee20af485613132c874442fa3942018e3f3ecb 100644 --- a/net/ipv4/netfilter/nf_nat_h323.c +++ b/net/ipv4/netfilter/nf_nat_h323.c @@ -291,20 +291,7 @@ static int nat_t120(struct sk_buff *skb, struct nf_conn *ct, exp->expectfn = nf_nat_follow_master; exp->dir = !dir; - /* Try to get same port: if not, try to change it. */ - for (; nated_port != 0; nated_port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(nated_port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - nated_port = 0; - break; - } - } - + nated_port = nf_nat_exp_find_port(exp, nated_port); if (nated_port == 0) { /* No port available */ net_notice_ratelimited("nf_nat_h323: out of TCP ports\n"); return 0; @@ -347,20 +334,7 @@ static int nat_h245(struct sk_buff *skb, struct nf_conn *ct, if (info->sig_port[dir] == port) nated_port = ntohs(info->sig_port[!dir]); - /* Try to get same port: if not, try to change it. */ - for (; nated_port != 0; nated_port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(nated_port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - nated_port = 0; - break; - } - } - + nated_port = nf_nat_exp_find_port(exp, nated_port); if (nated_port == 0) { /* No port available */ net_notice_ratelimited("nf_nat_q931: out of TCP ports\n"); return 0; @@ -439,20 +413,7 @@ static int nat_q931(struct sk_buff *skb, struct nf_conn *ct, if (info->sig_port[dir] == port) nated_port = ntohs(info->sig_port[!dir]); - /* Try to get same port: if not, try to change it. */ - for (; nated_port != 0; nated_port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(nated_port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - nated_port = 0; - break; - } - } - + nated_port = nf_nat_exp_find_port(exp, nated_port); if (nated_port == 0) { /* No port available */ net_notice_ratelimited("nf_nat_ras: out of TCP ports\n"); return 0; @@ -532,20 +493,7 @@ static int nat_callforwarding(struct sk_buff *skb, struct nf_conn *ct, exp->expectfn = ip_nat_callforwarding_expect; exp->dir = !dir; - /* Try to get same port: if not, try to change it. */ - for (nated_port = ntohs(port); nated_port != 0; nated_port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(nated_port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - nated_port = 0; - break; - } - } - + nated_port = nf_nat_exp_find_port(exp, ntohs(port)); if (nated_port == 0) { /* No port available */ net_notice_ratelimited("nf_nat_q931: out of TCP ports\n"); return 0; diff --git a/net/ipv4/netfilter/nf_socket_ipv4.c b/net/ipv4/netfilter/nf_socket_ipv4.c index 2d42e4c35a2069a3f1c04da128476838b6a477ac..a1350fc25838755934f06d24aa08f493c4a19785 100644 --- a/net/ipv4/netfilter/nf_socket_ipv4.c +++ b/net/ipv4/netfilter/nf_socket_ipv4.c @@ -71,8 +71,8 @@ nf_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff, { switch (protocol) { case IPPROTO_TCP: - return inet_lookup(net, &tcp_hashinfo, skb, doff, - saddr, sport, daddr, dport, + return inet_lookup(net, net->ipv4.tcp_death_row.hashinfo, + skb, doff, saddr, sport, daddr, dport, in->ifindex); case IPPROTO_UDP: return udp4_lib_lookup(net, saddr, sport, daddr, dport, diff --git a/net/ipv4/netfilter/nf_tproxy_ipv4.c b/net/ipv4/netfilter/nf_tproxy_ipv4.c index b2bae0b0e42a1e983210b37b32b50186de1b0eff..b22b2c745c76c317dad4f4d65d6c5ac48624d24d 100644 --- a/net/ipv4/netfilter/nf_tproxy_ipv4.c +++ b/net/ipv4/netfilter/nf_tproxy_ipv4.c @@ -79,6 +79,7 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb, const struct net_device *in, const enum nf_tproxy_lookup_t lookup_type) { + struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo; struct sock *sk; switch (protocol) { @@ -92,12 +93,10 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb, switch (lookup_type) { case NF_TPROXY_LOOKUP_LISTENER: - sk = inet_lookup_listener(net, &tcp_hashinfo, skb, - ip_hdrlen(skb) + - __tcp_hdrlen(hp), - saddr, sport, - daddr, dport, - in->ifindex, 0); + sk = inet_lookup_listener(net, hinfo, skb, + ip_hdrlen(skb) + __tcp_hdrlen(hp), + saddr, sport, daddr, dport, + in->ifindex, 0); if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) sk = NULL; @@ -108,9 +107,8 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb, */ break; case NF_TPROXY_LOOKUP_ESTABLISHED: - sk = inet_lookup_established(net, &tcp_hashinfo, - saddr, sport, daddr, dport, - in->ifindex); + sk = inet_lookup_established(net, hinfo, saddr, sport, + daddr, dport, in->ifindex); break; default: BUG(); diff --git a/net/ipv4/netfilter/nft_fib_ipv4.c b/net/ipv4/netfilter/nft_fib_ipv4.c index b75cac69bd7e6b95eafc2b07aab571b93a8321fc..e886147eed11d22992525b0c1a41ae6532daa631 100644 --- a/net/ipv4/netfilter/nft_fib_ipv4.c +++ b/net/ipv4/netfilter/nft_fib_ipv4.c @@ -83,6 +83,9 @@ void nft_fib4_eval(const struct nft_expr *expr, struct nft_regs *regs, else oif = NULL; + if (priv->flags & NFTA_FIB_F_IIF) + fl4.flowi4_l3mdev = l3mdev_master_ifindex_rcu(oif); + if (nft_hook(pkt) == NF_INET_PRE_ROUTING && nft_fib_is_loopback(pkt->skb, nft_in(pkt))) { nft_fib_store_result(dest, priv, nft_in(pkt)); diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index b83c2bd9d722388e948fc37a6388ce04fa8b0dbe..bde333b24837aef2f23f588210de483540e9f252 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -295,6 +296,19 @@ void ping_close(struct sock *sk, long timeout) } EXPORT_SYMBOL_GPL(ping_close); +static int ping_pre_connect(struct sock *sk, struct sockaddr *uaddr, + int addr_len) +{ + /* This check is replicated from __ip4_datagram_connect() and + * intended to prevent BPF program called below from accessing bytes + * that are out of the bound specified by user in addr_len. + */ + if (addr_len < sizeof(struct sockaddr_in)) + return -EINVAL; + + return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr); +} + /* Checks the bind address and possibly modifies sk->sk_bound_dev_if. */ static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, struct sockaddr *uaddr, int addr_len) @@ -603,21 +617,9 @@ int ping_getfrag(void *from, char *to, { struct pingfakehdr *pfh = from; - if (offset == 0) { - fraglen -= sizeof(struct icmphdr); - if (fraglen < 0) - BUG(); - if (!csum_and_copy_from_iter_full(to + sizeof(struct icmphdr), - fraglen, &pfh->wcheck, - &pfh->msg->msg_iter)) - return -EFAULT; - } else if (offset < sizeof(struct icmphdr)) { - BUG(); - } else { - if (!csum_and_copy_from_iter_full(to, fraglen, &pfh->wcheck, - &pfh->msg->msg_iter)) - return -EFAULT; - } + if (!csum_and_copy_from_iter_full(to, fraglen, &pfh->wcheck, + &pfh->msg->msg_iter)) + return -EFAULT; #if IS_ENABLED(CONFIG_IPV6) /* For IPv6, checksum each skb as we go along, as expected by @@ -625,7 +627,7 @@ int ping_getfrag(void *from, char *to, * wcheck, it will be finalized in ping_v4_push_pending_frames. */ if (pfh->family == AF_INET6) { - skb->csum = pfh->wcheck; + skb->csum = csum_block_add(skb->csum, pfh->wcheck, odd); skb->ip_summed = CHECKSUM_NONE; pfh->wcheck = 0; } @@ -828,7 +830,8 @@ back_from_confirm: pfh.family = AF_INET; err = ip_append_data(sk, &fl4, ping_getfrag, &pfh, len, - 0, &ipc, &rt, msg->msg_flags); + sizeof(struct icmphdr), &ipc, &rt, + msg->msg_flags); if (err) ip_flush_pending_frames(sk); else @@ -1009,6 +1012,7 @@ struct proto ping_prot = { .owner = THIS_MODULE, .init = ping_init_sock, .close = ping_close, + .pre_connect = ping_pre_connect, .connect = ip4_datagram_connect, .disconnect = __udp_disconnect, .setsockopt = ip_setsockopt, diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index 0088a4c64d77ed89b86b736d1c40b099a817ea27..5386f460bd208c8f30902ea8b6aa613449d59ee0 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -59,7 +59,7 @@ static int sockstat_seq_show(struct seq_file *seq, void *v) socket_seq_show(seq); seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %ld\n", sock_prot_inuse_get(net, &tcp_prot), orphans, - refcount_read(&net->ipv4.tcp_death_row->tw_refcount) - 1, + refcount_read(&net->ipv4.tcp_death_row.tw_refcount) - 1, sockets, proto_memory_allocated(&tcp_prot)); seq_printf(seq, "UDP: inuse %d mem %ld\n", sock_prot_inuse_get(net, &udp_prot), diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 795cbe1de912403b367c92947516382339437b06..cd1fa9f70f1a17276648f1ec693c3b40cb6e11b4 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -3664,7 +3664,7 @@ static __net_init int rt_genid_init(struct net *net) { atomic_set(&net->ipv4.rt_genid, 0); atomic_set(&net->fnhe_genid, 0); - atomic_set(&net->ipv4.dev_addr_genid, get_random_int()); + atomic_set(&net->ipv4.dev_addr_genid, get_random_u32()); return 0; } @@ -3719,7 +3719,7 @@ int __init ip_rt_init(void) ip_idents = idents_hash; - prandom_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents)); + get_random_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents)); ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents); diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 5490c285668b93b5683328a2c284af66e2f19b0d..9b8a6db7a66b310ba9983afb4f64dfa1e1846e64 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -39,6 +39,7 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ; static int one_day_secs = 24 * 3600; static u32 fib_multipath_hash_fields_all_mask __maybe_unused = FIB_MULTIPATH_HASH_FIELD_ALL_MASK; +static unsigned int tcp_child_ehash_entries_max = 16 * 1024 * 1024; /* obsolete */ static int sysctl_tcp_low_latency __read_mostly; @@ -382,6 +383,29 @@ static int proc_tcp_available_ulp(struct ctl_table *ctl, return ret; } +static int proc_tcp_ehash_entries(struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + struct net *net = container_of(table->data, struct net, + ipv4.sysctl_tcp_child_ehash_entries); + struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo; + int tcp_ehash_entries; + struct ctl_table tbl; + + tcp_ehash_entries = hinfo->ehash_mask + 1; + + /* A negative number indicates that the child netns + * shares the global ehash. + */ + if (!net_eq(net, &init_net) && !hinfo->pernet) + tcp_ehash_entries *= -1; + + tbl.data = &tcp_ehash_entries; + tbl.maxlen = sizeof(int); + + return proc_dointvec(&tbl, write, buffer, lenp, ppos); +} + #ifdef CONFIG_IP_ROUTE_MULTIPATH static int proc_fib_multipath_hash_policy(struct ctl_table *table, int write, void *buffer, size_t *lenp, @@ -530,10 +554,9 @@ static struct ctl_table ipv4_table[] = { }; static struct ctl_table ipv4_net_table[] = { - /* tcp_max_tw_buckets must be first in this table. */ { .procname = "tcp_max_tw_buckets", -/* .data = &init_net.ipv4.tcp_death_row.sysctl_max_tw_buckets, */ + .data = &init_net.ipv4.tcp_death_row.sysctl_max_tw_buckets, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec @@ -1321,6 +1344,21 @@ static struct ctl_table ipv4_net_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "tcp_ehash_entries", + .data = &init_net.ipv4.sysctl_tcp_child_ehash_entries, + .mode = 0444, + .proc_handler = proc_tcp_ehash_entries, + }, + { + .procname = "tcp_child_ehash_entries", + .data = &init_net.ipv4.sysctl_tcp_child_ehash_entries, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &tcp_child_ehash_entries_max, + }, { .procname = "udp_rmem_min", .data = &init_net.ipv4.sysctl_udp_rmem_min, @@ -1361,8 +1399,7 @@ static __net_init int ipv4_sysctl_init_net(struct net *net) if (!table) goto err_alloc; - /* skip first entry (sysctl_max_tw_buckets) */ - for (i = 1; i < ARRAY_SIZE(ipv4_net_table) - 1; i++) { + for (i = 0; i < ARRAY_SIZE(ipv4_net_table) - 1; i++) { if (table[i].data) { /* Update the variables to point into * the current struct net @@ -1377,8 +1414,6 @@ static __net_init int ipv4_sysctl_init_net(struct net *net) } } - table[0].data = &net->ipv4.tcp_death_row->sysctl_max_tw_buckets; - net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", table); if (!net->ipv4.ipv4_hdr) goto err_reg; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index bbe2187536620a9300ef7350e08c861625c042c0..f8232811a5be17ec7652ff47ffde6341b2a76d1e 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1000,7 +1000,7 @@ new_segment: i = skb_shinfo(skb)->nr_frags; can_coalesce = skb_can_coalesce(skb, i, page, offset); - if (!can_coalesce && i >= sysctl_max_skb_frags) { + if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) { tcp_mark_push(tp, skb); goto new_segment; } @@ -1015,7 +1015,7 @@ new_segment: skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); } else { get_page(page); - skb_fill_page_desc(skb, i, page, offset, copy); + skb_fill_page_desc_noacc(skb, i, page, offset, copy); } if (!(flags & MSG_NO_SHARED_FRAGS)) @@ -1162,9 +1162,8 @@ void tcp_free_fastopen_req(struct tcp_sock *tp) } } -static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, - int *copied, size_t size, - struct ubuf_info *uarg) +int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied, + size_t size, struct ubuf_info *uarg) { struct tcp_sock *tp = tcp_sk(sk); struct inet_sock *inet = inet_sk(sk); @@ -1239,7 +1238,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) } zc = sk->sk_route_caps & NETIF_F_SG; if (!zc) - uarg->zerocopy = 0; + uarg_to_msgzc(uarg)->zerocopy = 0; } } @@ -1354,7 +1353,7 @@ new_segment: if (!skb_can_coalesce(skb, i, pfrag->page, pfrag->offset)) { - if (i >= sysctl_max_skb_frags) { + if (i >= READ_ONCE(sysctl_max_skb_frags)) { tcp_mark_push(tp, skb); goto new_segment; } @@ -1761,19 +1760,28 @@ int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) if (sk->sk_state == TCP_LISTEN) return -ENOTCONN; - skb = tcp_recv_skb(sk, seq, &offset); - if (!skb) - return 0; + while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { + u8 tcp_flags; + int used; - __skb_unlink(skb, &sk->sk_receive_queue); - WARN_ON(!skb_set_owner_sk_safe(skb, sk)); - copied = recv_actor(sk, skb); - if (copied >= 0) { - seq += copied; - if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) + __skb_unlink(skb, &sk->sk_receive_queue); + WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk)); + tcp_flags = TCP_SKB_CB(skb)->tcp_flags; + used = recv_actor(sk, skb); + consume_skb(skb); + if (used < 0) { + if (!copied) + copied = used; + break; + } + seq += used; + copied += used; + + if (tcp_flags & TCPHDR_FIN) { ++seq; + break; + } } - consume_skb(skb); WRITE_ONCE(tp->copied_seq, seq); tcp_rcv_space_adjust(sk); @@ -3128,6 +3136,8 @@ int tcp_disconnect(struct sock *sk, int flags) tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; tcp_snd_cwnd_set(tp, TCP_INIT_CWND); tp->snd_cwnd_cnt = 0; + tp->is_cwnd_limited = 0; + tp->max_packets_out = 0; tp->window_clamp = 0; tp->delivered = 0; tp->delivered_ce = 0; @@ -3199,7 +3209,7 @@ EXPORT_SYMBOL(tcp_disconnect); static inline bool tcp_can_repair_sock(const struct sock *sk) { - return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && + return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) && (sk->sk_state != TCP_LISTEN); } @@ -3476,8 +3486,8 @@ int tcp_set_window_clamp(struct sock *sk, int val) /* * Socket option code for TCP. */ -static int do_tcp_setsockopt(struct sock *sk, int level, int optname, - sockptr_t optval, unsigned int optlen) +int do_tcp_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen) { struct tcp_sock *tp = tcp_sk(sk); struct inet_connection_sock *icsk = inet_csk(sk); @@ -3499,11 +3509,11 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname, return -EFAULT; name[val] = 0; - lock_sock(sk); - err = tcp_set_congestion_control(sk, name, true, - ns_capable(sock_net(sk)->user_ns, - CAP_NET_ADMIN)); - release_sock(sk); + sockopt_lock_sock(sk); + err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(), + sockopt_ns_capable(sock_net(sk)->user_ns, + CAP_NET_ADMIN)); + sockopt_release_sock(sk); return err; } case TCP_ULP: { @@ -3519,9 +3529,9 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname, return -EFAULT; name[val] = 0; - lock_sock(sk); + sockopt_lock_sock(sk); err = tcp_set_ulp(sk, name); - release_sock(sk); + sockopt_release_sock(sk); return err; } case TCP_FASTOPEN_KEY: { @@ -3554,7 +3564,7 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname, if (copy_from_sockptr(&val, optval, sizeof(val))) return -EFAULT; - lock_sock(sk); + sockopt_lock_sock(sk); switch (optname) { case TCP_MAXSEG: @@ -3776,7 +3786,7 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname, break; } - release_sock(sk); + sockopt_release_sock(sk); return err; } @@ -3786,8 +3796,9 @@ int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, const struct inet_connection_sock *icsk = inet_csk(sk); if (level != SOL_TCP) - return icsk->icsk_af_ops->setsockopt(sk, level, optname, - optval, optlen); + /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ + return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname, + optval, optlen); return do_tcp_setsockopt(sk, level, optname, optval, optlen); } EXPORT_SYMBOL(tcp_setsockopt); @@ -4040,15 +4051,15 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, return stats; } -static int do_tcp_getsockopt(struct sock *sk, int level, - int optname, char __user *optval, int __user *optlen) +int do_tcp_getsockopt(struct sock *sk, int level, + int optname, sockptr_t optval, sockptr_t optlen) { struct inet_connection_sock *icsk = inet_csk(sk); struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); int val, len; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; len = min_t(unsigned int, len, sizeof(int)); @@ -4098,15 +4109,15 @@ static int do_tcp_getsockopt(struct sock *sk, int level, case TCP_INFO: { struct tcp_info info; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; tcp_get_info(sk, &info); len = min_t(unsigned int, len, sizeof(info)); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &info, len)) + if (copy_to_sockptr(optval, &info, len)) return -EFAULT; return 0; } @@ -4116,7 +4127,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level, size_t sz = 0; int attr; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; ca_ops = icsk->icsk_ca_ops; @@ -4124,9 +4135,9 @@ static int do_tcp_getsockopt(struct sock *sk, int level, sz = ca_ops->get_info(sk, ~0U, &attr, &info); len = min_t(unsigned int, len, sz); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &info, len)) + if (copy_to_sockptr(optval, &info, len)) return -EFAULT; return 0; } @@ -4135,27 +4146,28 @@ static int do_tcp_getsockopt(struct sock *sk, int level, break; case TCP_CONGESTION: - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; len = min_t(unsigned int, len, TCP_CA_NAME_MAX); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, icsk->icsk_ca_ops->name, len)) + if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len)) return -EFAULT; return 0; case TCP_ULP: - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; len = min_t(unsigned int, len, TCP_ULP_NAME_MAX); if (!icsk->icsk_ulp_ops) { - if (put_user(0, optlen)) + len = 0; + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; return 0; } - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len)) + if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len)) return -EFAULT; return 0; @@ -4163,15 +4175,15 @@ static int do_tcp_getsockopt(struct sock *sk, int level, u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)]; unsigned int key_len; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; key_len = tcp_fastopen_get_cipher(net, icsk, key) * TCP_FASTOPEN_KEY_LENGTH; len = min_t(unsigned int, len, key_len); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, key, len)) + if (copy_to_sockptr(optval, key, len)) return -EFAULT; return 0; } @@ -4197,7 +4209,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level, case TCP_REPAIR_WINDOW: { struct tcp_repair_window opt; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len != sizeof(opt)) @@ -4212,7 +4224,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level, opt.rcv_wnd = tp->rcv_wnd; opt.rcv_wup = tp->rcv_wup; - if (copy_to_user(optval, &opt, len)) + if (copy_to_sockptr(optval, &opt, len)) return -EFAULT; return 0; } @@ -4258,35 +4270,35 @@ static int do_tcp_getsockopt(struct sock *sk, int level, val = tp->save_syn; break; case TCP_SAVED_SYN: { - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; - lock_sock(sk); + sockopt_lock_sock(sk); if (tp->saved_syn) { if (len < tcp_saved_syn_len(tp->saved_syn)) { - if (put_user(tcp_saved_syn_len(tp->saved_syn), - optlen)) { - release_sock(sk); + len = tcp_saved_syn_len(tp->saved_syn); + if (copy_to_sockptr(optlen, &len, sizeof(int))) { + sockopt_release_sock(sk); return -EFAULT; } - release_sock(sk); + sockopt_release_sock(sk); return -EINVAL; } len = tcp_saved_syn_len(tp->saved_syn); - if (put_user(len, optlen)) { - release_sock(sk); + if (copy_to_sockptr(optlen, &len, sizeof(int))) { + sockopt_release_sock(sk); return -EFAULT; } - if (copy_to_user(optval, tp->saved_syn->data, len)) { - release_sock(sk); + if (copy_to_sockptr(optval, tp->saved_syn->data, len)) { + sockopt_release_sock(sk); return -EFAULT; } tcp_saved_syn_free(tp); - release_sock(sk); + sockopt_release_sock(sk); } else { - release_sock(sk); + sockopt_release_sock(sk); len = 0; - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; } return 0; @@ -4297,31 +4309,31 @@ static int do_tcp_getsockopt(struct sock *sk, int level, struct tcp_zerocopy_receive zc = {}; int err; - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; if (len < 0 || len < offsetofend(struct tcp_zerocopy_receive, length)) return -EINVAL; if (unlikely(len > sizeof(zc))) { - err = check_zeroed_user(optval + sizeof(zc), - len - sizeof(zc)); + err = check_zeroed_sockptr(optval, sizeof(zc), + len - sizeof(zc)); if (err < 1) return err == 0 ? -EINVAL : err; len = sizeof(zc); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; } - if (copy_from_user(&zc, optval, len)) + if (copy_from_sockptr(&zc, optval, len)) return -EFAULT; if (zc.reserved) return -EINVAL; if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS)) return -EINVAL; - lock_sock(sk); + sockopt_lock_sock(sk); err = tcp_zerocopy_receive(sk, &zc, &tss); err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); - release_sock(sk); + sockopt_release_sock(sk); if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) goto zerocopy_rcv_cmsg; switch (len) { @@ -4351,7 +4363,7 @@ zerocopy_rcv_sk_err: zerocopy_rcv_inq: zc.inq = tcp_inq_hint(sk); zerocopy_rcv_out: - if (!err && copy_to_user(optval, &zc, len)) + if (!err && copy_to_sockptr(optval, &zc, len)) err = -EFAULT; return err; } @@ -4360,9 +4372,9 @@ zerocopy_rcv_out: return -ENOPROTOOPT; } - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &val, len)) + if (copy_to_sockptr(optval, &val, len)) return -EFAULT; return 0; } @@ -4385,9 +4397,11 @@ int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, struct inet_connection_sock *icsk = inet_csk(sk); if (level != SOL_TCP) - return icsk->icsk_af_ops->getsockopt(sk, level, optname, - optval, optlen); - return do_tcp_getsockopt(sk, level, optname, optval, optlen); + /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */ + return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname, + optval, optlen); + return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval), + USER_SOCKPTR(optlen)); } EXPORT_SYMBOL(tcp_getsockopt); @@ -4433,12 +4447,16 @@ static void __tcp_alloc_md5sig_pool(void) * to memory. See smp_rmb() in tcp_get_md5sig_pool() */ smp_wmb(); - tcp_md5sig_pool_populated = true; + /* Paired with READ_ONCE() from tcp_alloc_md5sig_pool() + * and tcp_get_md5sig_pool(). + */ + WRITE_ONCE(tcp_md5sig_pool_populated, true); } bool tcp_alloc_md5sig_pool(void) { - if (unlikely(!tcp_md5sig_pool_populated)) { + /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */ + if (unlikely(!READ_ONCE(tcp_md5sig_pool_populated))) { mutex_lock(&tcp_md5sig_mutex); if (!tcp_md5sig_pool_populated) { @@ -4449,7 +4467,8 @@ bool tcp_alloc_md5sig_pool(void) mutex_unlock(&tcp_md5sig_mutex); } - return tcp_md5sig_pool_populated; + /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */ + return READ_ONCE(tcp_md5sig_pool_populated); } EXPORT_SYMBOL(tcp_alloc_md5sig_pool); @@ -4465,7 +4484,8 @@ struct tcp_md5sig_pool *tcp_get_md5sig_pool(void) { local_bh_disable(); - if (tcp_md5sig_pool_populated) { + /* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */ + if (READ_ONCE(tcp_md5sig_pool_populated)) { /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */ smp_rmb(); return this_cpu_ptr(&tcp_md5sig_pool); @@ -4736,6 +4756,12 @@ void __init tcp_init(void) SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT, NULL); + tcp_hashinfo.bind2_bucket_cachep = + kmem_cache_create("tcp_bind2_bucket", + sizeof(struct inet_bind2_bucket), 0, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT, + NULL); /* Size and allocate the main established and bind bucket * hash tables. @@ -4759,7 +4785,7 @@ void __init tcp_init(void) panic("TCP: failed to alloc ehash_locks"); tcp_hashinfo.bhash = alloc_large_system_hash("TCP bind", - sizeof(struct inet_bind_hashbucket), + 2 * sizeof(struct inet_bind_hashbucket), tcp_hashinfo.ehash_mask + 1, 17, /* one slot per 128 KB of memory */ 0, @@ -4768,11 +4794,15 @@ void __init tcp_init(void) 0, 64 * 1024); tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size; + tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size; for (i = 0; i < tcp_hashinfo.bhash_size; i++) { spin_lock_init(&tcp_hashinfo.bhash[i].lock); INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain); + spin_lock_init(&tcp_hashinfo.bhash2[i].lock); + INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain); } + tcp_hashinfo.pernet = false; cnt = tcp_hashinfo.ehash_mask + 1; sysctl_tcp_max_orphans = cnt / 2; diff --git a/net/ipv4/tcp_cdg.c b/net/ipv4/tcp_cdg.c index ddc7ba0554bddaa5df2fffdb61faba1f3cfbde5c..ba4d98e510e05790c15e0f572119b54179e76819 100644 --- a/net/ipv4/tcp_cdg.c +++ b/net/ipv4/tcp_cdg.c @@ -243,7 +243,7 @@ static bool tcp_cdg_backoff(struct sock *sk, u32 grad) struct cdg *ca = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); - if (prandom_u32() <= nexp_u32(grad * backoff_factor)) + if (get_random_u32() <= nexp_u32(grad * backoff_factor)) return false; if (use_ineff) { @@ -375,6 +375,7 @@ static void tcp_cdg_init(struct sock *sk) struct cdg *ca = inet_csk_ca(sk); struct tcp_sock *tp = tcp_sk(sk); + ca->gradients = NULL; /* We silently fall back to window = 1 if allocation fails. */ if (window > 1) ca->gradients = kcalloc(window, sizeof(ca->gradients[0]), @@ -388,6 +389,7 @@ static void tcp_cdg_release(struct sock *sk) struct cdg *ca = inet_csk_ca(sk); kfree(ca->gradients); + ca->gradients = NULL; } static struct tcp_congestion_ops tcp_cdg __read_mostly = { diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c index 75a1c985f49a267f93c01d3133326a569f8f3651..01b50fa791898831dc777bb3a64ad93ad57c1bfd 100644 --- a/net/ipv4/tcp_diag.c +++ b/net/ipv4/tcp_diag.c @@ -181,13 +181,21 @@ static size_t tcp_diag_get_aux_size(struct sock *sk, bool net_admin) static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, const struct inet_diag_req_v2 *r) { - inet_diag_dump_icsk(&tcp_hashinfo, skb, cb, r); + struct inet_hashinfo *hinfo; + + hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo; + + inet_diag_dump_icsk(hinfo, skb, cb, r); } static int tcp_diag_dump_one(struct netlink_callback *cb, const struct inet_diag_req_v2 *req) { - return inet_diag_dump_one_icsk(&tcp_hashinfo, cb, req); + struct inet_hashinfo *hinfo; + + hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo; + + return inet_diag_dump_one_icsk(hinfo, cb, req); } #ifdef CONFIG_INET_DIAG_DESTROY @@ -195,9 +203,13 @@ static int tcp_diag_destroy(struct sk_buff *in_skb, const struct inet_diag_req_v2 *req) { struct net *net = sock_net(in_skb->sk); - struct sock *sk = inet_diag_find_one_icsk(net, &tcp_hashinfo, req); + struct inet_hashinfo *hinfo; + struct sock *sk; int err; + hinfo = net->ipv4.tcp_death_row.hashinfo; + sk = inet_diag_find_one_icsk(net, hinfo, req); + if (IS_ERR(sk)) return PTR_ERR(sk); diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 825b216d11f52bc79a29b7a696005175a55a6804..45cc7f1ca29618e3ac1066cb49e7d6dc90e1c64d 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -272,8 +272,9 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, * The request socket is not added to the ehash * because it's been added to the accept queue directly. */ + req->timeout = tcp_timeout_init(child); inet_csk_reset_xmit_timer(child, ICSK_TIME_RETRANS, - TCP_TIMEOUT_INIT, TCP_RTO_MAX); + req->timeout, TCP_RTO_MAX); refcount_set(&req->rsk_refcnt, 2); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index ab5f0ea166f1a0535e299a9051406b5e2895f1f0..bc2ea12221f95a571f19ff3263277267372689af 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -2513,6 +2513,21 @@ static inline bool tcp_may_undo(const struct tcp_sock *tp) return tp->undo_marker && (!tp->undo_retrans || tcp_packet_delayed(tp)); } +static bool tcp_is_non_sack_preventing_reopen(struct sock *sk) +{ + struct tcp_sock *tp = tcp_sk(sk); + + if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) { + /* Hold old state until something *above* high_seq + * is ACKed. For Reno it is MUST to prevent false + * fast retransmits (RFC2582). SACK TCP is safe. */ + if (!tcp_any_retrans_done(sk)) + tp->retrans_stamp = 0; + return true; + } + return false; +} + /* People celebrate: "We love our President!" */ static bool tcp_try_undo_recovery(struct sock *sk) { @@ -2535,14 +2550,8 @@ static bool tcp_try_undo_recovery(struct sock *sk) } else if (tp->rack.reo_wnd_persist) { tp->rack.reo_wnd_persist--; } - if (tp->snd_una == tp->high_seq && tcp_is_reno(tp)) { - /* Hold old state until something *above* high_seq - * is ACKed. For Reno it is MUST to prevent false - * fast retransmits (RFC2582). SACK TCP is safe. */ - if (!tcp_any_retrans_done(sk)) - tp->retrans_stamp = 0; + if (tcp_is_non_sack_preventing_reopen(sk)) return true; - } tcp_set_ca_state(sk, TCP_CA_Open); tp->is_sack_reneg = 0; return false; @@ -2578,6 +2587,8 @@ static bool tcp_try_undo_loss(struct sock *sk, bool frto_undo) NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSPURIOUSRTOS); inet_csk(sk)->icsk_retransmits = 0; + if (tcp_is_non_sack_preventing_reopen(sk)) + return true; if (frto_undo || tcp_is_sack(tp)) { tcp_set_ca_state(sk, TCP_CA_Open); tp->is_sack_reneg = 0; @@ -3614,12 +3625,9 @@ bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, /* RFC 5961 7 [ACK Throttling] */ static void tcp_send_challenge_ack(struct sock *sk) { - /* unprotected vars, we dont care of overwrites */ - static u32 challenge_timestamp; - static unsigned int challenge_count; struct tcp_sock *tp = tcp_sk(sk); struct net *net = sock_net(sk); - u32 count, now; + u32 count, now, ack_limit; /* First check our per-socket dupack rate limit. */ if (__tcp_oow_rate_limited(net, @@ -3627,18 +3635,22 @@ static void tcp_send_challenge_ack(struct sock *sk) &tp->last_oow_ack_time)) return; + ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit); + if (ack_limit == INT_MAX) + goto send_ack; + /* Then check host-wide RFC 5961 rate limit. */ now = jiffies / HZ; - if (now != challenge_timestamp) { - u32 ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit); + if (now != READ_ONCE(net->ipv4.tcp_challenge_timestamp)) { u32 half = (ack_limit + 1) >> 1; - challenge_timestamp = now; - WRITE_ONCE(challenge_count, half + prandom_u32_max(ack_limit)); + WRITE_ONCE(net->ipv4.tcp_challenge_timestamp, now); + WRITE_ONCE(net->ipv4.tcp_challenge_count, half + prandom_u32_max(ack_limit)); } - count = READ_ONCE(challenge_count); + count = READ_ONCE(net->ipv4.tcp_challenge_count); if (count > 0) { - WRITE_ONCE(challenge_count, count - 1); + WRITE_ONCE(net->ipv4.tcp_challenge_count, count - 1); +send_ack: NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK); tcp_send_ack(sk); } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 0c83780dc9bf4293135b8044daf41090b66f8b08..7a250ef9d1b7b5eedc10dbf4f599343d0fa464d2 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -199,16 +199,18 @@ static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr, /* This will initiate an outgoing connection. */ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { + struct inet_bind_hashbucket *prev_addr_hashbucket = NULL; struct sockaddr_in *usin = (struct sockaddr_in *)uaddr; + struct inet_timewait_death_row *tcp_death_row; + __be32 daddr, nexthop, prev_sk_rcv_saddr; struct inet_sock *inet = inet_sk(sk); struct tcp_sock *tp = tcp_sk(sk); + struct ip_options_rcu *inet_opt; + struct net *net = sock_net(sk); __be16 orig_sport, orig_dport; - __be32 daddr, nexthop; struct flowi4 *fl4; struct rtable *rt; int err; - struct ip_options_rcu *inet_opt; - struct inet_timewait_death_row *tcp_death_row = sock_net(sk)->ipv4.tcp_death_row; if (addr_len < sizeof(struct sockaddr_in)) return -EINVAL; @@ -234,7 +236,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) if (IS_ERR(rt)) { err = PTR_ERR(rt); if (err == -ENETUNREACH) - IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES); + IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); return err; } @@ -246,10 +248,29 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) if (!inet_opt || !inet_opt->opt.srr) daddr = fl4->daddr; - if (!inet->inet_saddr) + tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; + + if (!inet->inet_saddr) { + if (inet_csk(sk)->icsk_bind2_hash) { + prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo, + sk, net, inet->inet_num); + prev_sk_rcv_saddr = sk->sk_rcv_saddr; + } inet->inet_saddr = fl4->saddr; + } + sk_rcv_saddr_set(sk, inet->inet_saddr); + if (prev_addr_hashbucket) { + err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk); + if (err) { + inet->inet_saddr = 0; + sk_rcv_saddr_set(sk, prev_sk_rcv_saddr); + ip_rt_put(rt); + return err; + } + } + if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) { /* Reset inherited state */ tp->rx_opt.ts_recent = 0; @@ -298,12 +319,11 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) inet->inet_daddr, inet->inet_sport, usin->sin_port)); - tp->tsoffset = secure_tcp_ts_off(sock_net(sk), - inet->inet_saddr, + tp->tsoffset = secure_tcp_ts_off(net, inet->inet_saddr, inet->inet_daddr); } - inet->inet_id = prandom_u32(); + inet->inet_id = get_random_u16(); if (tcp_fastopen_defer_connect(sk, &err)) return err; @@ -475,9 +495,9 @@ int tcp_v4_err(struct sk_buff *skb, u32 info) int err; struct net *net = dev_net(skb->dev); - sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr, - th->dest, iph->saddr, ntohs(th->source), - inet_iif(skb), 0); + sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, + iph->daddr, th->dest, iph->saddr, + ntohs(th->source), inet_iif(skb), 0); if (!sk) { __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); return -ENOENT; @@ -740,8 +760,8 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb) * Incoming packet is checked with md5 hash with finding key, * no RST generated if md5 hash doesn't match. */ - sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0, - ip_hdr(skb)->saddr, + sk1 = __inet_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, + NULL, 0, ip_hdr(skb)->saddr, th->source, ip_hdr(skb)->daddr, ntohs(th->source), dif, sdif); /* don't send rst if it can't find key */ @@ -1523,7 +1543,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, inet_csk(newsk)->icsk_ext_hdr_len = 0; if (inet_opt) inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen; - newinet->inet_id = prandom_u32(); + newinet->inet_id = get_random_u16(); /* Set ToS of the new socket based upon the value of incoming SYN. * ECT bits are set later in tcp_init_transfer(). @@ -1709,6 +1729,7 @@ EXPORT_SYMBOL(tcp_v4_do_rcv); int tcp_v4_early_demux(struct sk_buff *skb) { + struct net *net = dev_net(skb->dev); const struct iphdr *iph; const struct tcphdr *th; struct sock *sk; @@ -1725,7 +1746,7 @@ int tcp_v4_early_demux(struct sk_buff *skb) if (th->doff < sizeof(struct tcphdr) / 4) return 0; - sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo, + sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, iph->saddr, th->source, iph->daddr, ntohs(th->dest), skb->skb_iif, inet_sdif(skb)); @@ -1951,7 +1972,8 @@ int tcp_v4_rcv(struct sk_buff *skb) th = (const struct tcphdr *)skb->data; iph = ip_hdr(skb); lookup: - sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source, + sk = __inet_lookup_skb(net->ipv4.tcp_death_row.hashinfo, + skb, __tcp_hdrlen(th), th->source, th->dest, sdif, &refcounted); if (!sk) goto no_tcp_socket; @@ -2133,9 +2155,9 @@ do_time_wait: } switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) { case TCP_TW_SYN: { - struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev), - &tcp_hashinfo, skb, - __tcp_hdrlen(th), + struct sock *sk2 = inet_lookup_listener(net, + net->ipv4.tcp_death_row.hashinfo, + skb, __tcp_hdrlen(th), iph->saddr, th->source, iph->daddr, th->dest, inet_iif(skb), @@ -2285,15 +2307,16 @@ static bool seq_sk_match(struct seq_file *seq, const struct sock *sk) */ static void *listening_get_first(struct seq_file *seq) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct tcp_iter_state *st = seq->private; st->offset = 0; - for (; st->bucket <= tcp_hashinfo.lhash2_mask; st->bucket++) { + for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) { struct inet_listen_hashbucket *ilb2; struct hlist_nulls_node *node; struct sock *sk; - ilb2 = &tcp_hashinfo.lhash2[st->bucket]; + ilb2 = &hinfo->lhash2[st->bucket]; if (hlist_nulls_empty(&ilb2->nulls_head)) continue; @@ -2318,6 +2341,7 @@ static void *listening_get_next(struct seq_file *seq, void *cur) struct tcp_iter_state *st = seq->private; struct inet_listen_hashbucket *ilb2; struct hlist_nulls_node *node; + struct inet_hashinfo *hinfo; struct sock *sk = cur; ++st->num; @@ -2329,7 +2353,8 @@ static void *listening_get_next(struct seq_file *seq, void *cur) return sk; } - ilb2 = &tcp_hashinfo.lhash2[st->bucket]; + hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; + ilb2 = &hinfo->lhash2[st->bucket]; spin_unlock(&ilb2->lock); ++st->bucket; return listening_get_first(seq); @@ -2351,9 +2376,10 @@ static void *listening_get_idx(struct seq_file *seq, loff_t *pos) return rc; } -static inline bool empty_bucket(const struct tcp_iter_state *st) +static inline bool empty_bucket(struct inet_hashinfo *hinfo, + const struct tcp_iter_state *st) { - return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain); + return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain); } /* @@ -2362,20 +2388,21 @@ static inline bool empty_bucket(const struct tcp_iter_state *st) */ static void *established_get_first(struct seq_file *seq) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct tcp_iter_state *st = seq->private; st->offset = 0; - for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) { + for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) { struct sock *sk; struct hlist_nulls_node *node; - spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket); + spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket); /* Lockless fast path for the common case of empty buckets */ - if (empty_bucket(st)) + if (empty_bucket(hinfo, st)) continue; spin_lock_bh(lock); - sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) { + sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) { if (seq_sk_match(seq, sk)) return sk; } @@ -2387,9 +2414,10 @@ static void *established_get_first(struct seq_file *seq) static void *established_get_next(struct seq_file *seq, void *cur) { - struct sock *sk = cur; - struct hlist_nulls_node *node; + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct tcp_iter_state *st = seq->private; + struct hlist_nulls_node *node; + struct sock *sk = cur; ++st->num; ++st->offset; @@ -2401,7 +2429,7 @@ static void *established_get_next(struct seq_file *seq, void *cur) return sk; } - spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); + spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); ++st->bucket; return established_get_first(seq); } @@ -2439,6 +2467,7 @@ static void *tcp_get_idx(struct seq_file *seq, loff_t pos) static void *tcp_seek_last_pos(struct seq_file *seq) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct tcp_iter_state *st = seq->private; int bucket = st->bucket; int offset = st->offset; @@ -2447,7 +2476,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq) switch (st->state) { case TCP_SEQ_STATE_LISTENING: - if (st->bucket > tcp_hashinfo.lhash2_mask) + if (st->bucket > hinfo->lhash2_mask) break; st->state = TCP_SEQ_STATE_LISTENING; rc = listening_get_first(seq); @@ -2459,7 +2488,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq) st->state = TCP_SEQ_STATE_ESTABLISHED; fallthrough; case TCP_SEQ_STATE_ESTABLISHED: - if (st->bucket > tcp_hashinfo.ehash_mask) + if (st->bucket > hinfo->ehash_mask) break; rc = established_get_first(seq); while (offset-- && rc && bucket == st->bucket) @@ -2527,16 +2556,17 @@ EXPORT_SYMBOL(tcp_seq_next); void tcp_seq_stop(struct seq_file *seq, void *v) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct tcp_iter_state *st = seq->private; switch (st->state) { case TCP_SEQ_STATE_LISTENING: if (v != SEQ_START_TOKEN) - spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock); + spin_unlock(&hinfo->lhash2[st->bucket].lock); break; case TCP_SEQ_STATE_ESTABLISHED: if (v) - spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); + spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); break; } } @@ -2731,6 +2761,7 @@ static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter, static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq, struct sock *start_sk) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct bpf_tcp_iter_state *iter = seq->private; struct tcp_iter_state *st = &iter->state; struct hlist_nulls_node *node; @@ -2750,7 +2781,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq, expected++; } } - spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock); + spin_unlock(&hinfo->lhash2[st->bucket].lock); return expected; } @@ -2758,6 +2789,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq, static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, struct sock *start_sk) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct bpf_tcp_iter_state *iter = seq->private; struct tcp_iter_state *st = &iter->state; struct hlist_nulls_node *node; @@ -2777,13 +2809,14 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, expected++; } } - spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); + spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket)); return expected; } static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) { + struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo; struct bpf_tcp_iter_state *iter = seq->private; struct tcp_iter_state *st = &iter->state; unsigned int expected; @@ -2799,7 +2832,7 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) st->offset = 0; st->bucket++; if (st->state == TCP_SEQ_STATE_LISTENING && - st->bucket > tcp_hashinfo.lhash2_mask) { + st->bucket > hinfo->lhash2_mask) { st->state = TCP_SEQ_STATE_ESTABLISHED; st->bucket = 0; } @@ -3064,7 +3097,7 @@ struct proto tcp_prot = { .slab_flags = SLAB_TYPESAFE_BY_RCU, .twsk_prot = &tcp_timewait_sock_ops, .rsk_prot = &tcp_request_sock_ops, - .h.hashinfo = &tcp_hashinfo, + .h.hashinfo = NULL, .no_autobind = true, .diag_destroy = tcp_abort, }; @@ -3072,19 +3105,43 @@ EXPORT_SYMBOL(tcp_prot); static void __net_exit tcp_sk_exit(struct net *net) { - struct inet_timewait_death_row *tcp_death_row = net->ipv4.tcp_death_row; - if (net->ipv4.tcp_congestion_control) bpf_module_put(net->ipv4.tcp_congestion_control, net->ipv4.tcp_congestion_control->owner); - if (refcount_dec_and_test(&tcp_death_row->tw_refcount)) - kfree(tcp_death_row); } -static int __net_init tcp_sk_init(struct net *net) +static void __net_init tcp_set_hashinfo(struct net *net) { - int cnt; + struct inet_hashinfo *hinfo; + unsigned int ehash_entries; + struct net *old_net; + + if (net_eq(net, &init_net)) + goto fallback; + + old_net = current->nsproxy->net_ns; + ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries); + if (!ehash_entries) + goto fallback; + + ehash_entries = roundup_pow_of_two(ehash_entries); + hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries); + if (!hinfo) { + pr_warn("Failed to allocate TCP ehash (entries: %u) " + "for a netns, fallback to the global one\n", + ehash_entries); +fallback: + hinfo = &tcp_hashinfo; + ehash_entries = tcp_hashinfo.ehash_mask + 1; + } + + net->ipv4.tcp_death_row.hashinfo = hinfo; + net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2; + net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128); +} +static int __net_init tcp_sk_init(struct net *net) +{ net->ipv4.sysctl_tcp_ecn = 2; net->ipv4.sysctl_tcp_ecn_fallback = 1; @@ -3110,15 +3167,9 @@ static int __net_init tcp_sk_init(struct net *net) net->ipv4.sysctl_tcp_tw_reuse = 2; net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1; - net->ipv4.tcp_death_row = kzalloc(sizeof(struct inet_timewait_death_row), GFP_KERNEL); - if (!net->ipv4.tcp_death_row) - return -ENOMEM; - refcount_set(&net->ipv4.tcp_death_row->tw_refcount, 1); - cnt = tcp_hashinfo.ehash_mask + 1; - net->ipv4.tcp_death_row->sysctl_max_tw_buckets = cnt / 2; - net->ipv4.tcp_death_row->hashinfo = &tcp_hashinfo; + refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1); + tcp_set_hashinfo(net); - net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128); net->ipv4.sysctl_tcp_sack = 1; net->ipv4.sysctl_tcp_window_scaling = 1; net->ipv4.sysctl_tcp_timestamps = 1; @@ -3139,8 +3190,10 @@ static int __net_init tcp_sk_init(struct net *net) net->ipv4.sysctl_tcp_tso_win_divisor = 3; /* Default TSQ limit of 16 TSO segments */ net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536; - /* rfc5961 challenge ack rate limiting */ - net->ipv4.sysctl_tcp_challenge_ack_limit = 1000; + + /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */ + net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX; + net->ipv4.sysctl_tcp_min_tso_segs = 2; net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */ net->ipv4.sysctl_tcp_min_rtt_wlen = 300; @@ -3178,10 +3231,13 @@ static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list) { struct net *net; - inet_twsk_purge(&tcp_hashinfo, AF_INET); + tcp_twsk_purge(net_exit_list, AF_INET); - list_for_each_entry(net, net_exit_list, exit_list) + list_for_each_entry(net, net_exit_list, exit_list) { + inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo); + WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount)); tcp_fastopen_ctx_destroy(net); + } } static struct pernet_operations __net_initdata tcp_sk_ops = { diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c index d58e672be31c764805ca6ad120459eb2d86b4d87..82f4575f9cd90049a5ad4c7329ad1ddc28fc1aa0 100644 --- a/net/ipv4/tcp_metrics.c +++ b/net/ipv4/tcp_metrics.c @@ -969,6 +969,7 @@ static struct genl_family tcp_metrics_nl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = tcp_metrics_nl_ops, .n_small_ops = ARRAY_SIZE(tcp_metrics_nl_ops), + .resv_start_op = TCP_METRICS_CMD_DEL + 1, }; static unsigned int tcpmhash_entries; diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index cb95d88497aeae8802b0c42c83fae67352ef4d98..c375f603a16cf7ca5147f69d52abb2ccbecfcff1 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -247,10 +247,10 @@ void tcp_time_wait(struct sock *sk, int state, int timeo) { const struct inet_connection_sock *icsk = inet_csk(sk); const struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); struct inet_timewait_sock *tw; - struct inet_timewait_death_row *tcp_death_row = sock_net(sk)->ipv4.tcp_death_row; - tw = inet_twsk_alloc(sk, tcp_death_row, state); + tw = inet_twsk_alloc(sk, &net->ipv4.tcp_death_row, state); if (tw) { struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw); @@ -319,14 +319,14 @@ void tcp_time_wait(struct sock *sk, int state, int timeo) /* Linkage updates. * Note that access to tw after this point is illegal. */ - inet_twsk_hashdance(tw, sk, &tcp_hashinfo); + inet_twsk_hashdance(tw, sk, net->ipv4.tcp_death_row.hashinfo); local_bh_enable(); } else { /* Sorry, if we're out of memory, just CLOSE this * socket up. We've got bigger problems than * non-graceful socket closings. */ - NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPTIMEWAITOVERFLOW); + NET_INC_STATS(net, LINUX_MIB_TCPTIMEWAITOVERFLOW); } tcp_update_metrics(sk); @@ -347,6 +347,27 @@ void tcp_twsk_destructor(struct sock *sk) } EXPORT_SYMBOL_GPL(tcp_twsk_destructor); +void tcp_twsk_purge(struct list_head *net_exit_list, int family) +{ + bool purged_once = false; + struct net *net; + + list_for_each_entry(net, net_exit_list, exit_list) { + if (net->ipv4.tcp_death_row.hashinfo->pernet) { + /* Even if tw_refcount == 1, we must clean up kernel reqsk */ + inet_twsk_purge(net->ipv4.tcp_death_row.hashinfo, family); + } else if (!purged_once) { + /* The last refcount is decremented in tcp_sk_exit_batch() */ + if (refcount_read(&net->ipv4.tcp_death_row.tw_refcount) == 1) + continue; + + inet_twsk_purge(&tcp_hashinfo, family); + purged_once = true; + } + } +} +EXPORT_SYMBOL_GPL(tcp_twsk_purge); + /* Warning : This function is called without sk_listener being locked. * Be sure to read socket fields once, as their value could change under us. */ @@ -541,6 +562,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk, newtp->fastopen_req = NULL; RCU_INIT_POINTER(newtp->fastopen_rsk, NULL); + newtp->bpf_chg_cc_inprogress = 0; tcp_bpf_clone(sk, newsk); __TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS); diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index 30abde86db45e560680669ddfb533bcf12deacb0..45dda788938704c3f762256266d9ea29b6ded4a5 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -195,12 +195,9 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb) off = skb_gro_offset(skb); hlen = off + sizeof(*th); - th = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) { - th = skb_gro_header_slow(skb, hlen, off); - if (unlikely(!th)) - goto out; - } + th = skb_gro_header(skb, hlen, off); + if (unlikely(!th)) + goto out; thlen = th->doff * 4; if (thlen < sizeof(*th)) @@ -258,7 +255,15 @@ found: mss = skb_shinfo(p)->gso_size; - flush |= (len - 1) >= mss; + /* If skb is a GRO packet, make sure its gso_size matches prior packet mss. + * If it is a single frame, do not aggregate it if its length + * is bigger than our mss. + */ + if (unlikely(skb_is_gso(skb))) + flush |= (mss != skb_shinfo(skb)->gso_size); + else + flush |= (len - 1) >= mss; + flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq); #ifdef CONFIG_TLS_DEVICE flush |= p->decrypted ^ skb->decrypted; @@ -272,7 +277,12 @@ found: tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH); out_check_final: - flush = len < mss; + /* Force a flush if last segment is smaller than mss. */ + if (unlikely(skb_is_gso(skb))) + flush = len != NAPI_GRO_CB(skb)->count * skb_shinfo(skb)->gso_size; + else + flush = len < mss; + flush |= (__force int)(flags & (TCP_FLAG_URG | TCP_FLAG_PSH | TCP_FLAG_RST | TCP_FLAG_SYN | TCP_FLAG_FIN)); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 78b654ff421b19fc8b73d73ff2d358008856a498..c69f4d966024ce2c37dae63f7be94fe79c7dd648 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -239,7 +239,7 @@ void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss, if (wscale_ok) { /* Set window scaling on max possible window */ space = max_t(u32, space, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); - space = max_t(u32, space, sysctl_rmem_max); + space = max_t(u32, space, READ_ONCE(sysctl_rmem_max)); space = min_t(u32, space, *window_clamp); *rcv_wscale = clamp_t(int, ilog2(space) - 15, 0, TCP_MAX_WSCALE); @@ -1875,15 +1875,20 @@ static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited) const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; struct tcp_sock *tp = tcp_sk(sk); - /* Track the maximum number of outstanding packets in each - * window, and remember whether we were cwnd-limited then. + /* Track the strongest available signal of the degree to which the cwnd + * is fully utilized. If cwnd-limited then remember that fact for the + * current window. If not cwnd-limited then track the maximum number of + * outstanding packets in the current window. (If cwnd-limited then we + * chose to not update tp->max_packets_out to avoid an extra else + * clause with no functional impact.) */ - if (!before(tp->snd_una, tp->max_packets_seq) || - tp->packets_out > tp->max_packets_out || - is_cwnd_limited) { - tp->max_packets_out = tp->packets_out; - tp->max_packets_seq = tp->snd_nxt; + if (!before(tp->snd_una, tp->cwnd_usage_seq) || + is_cwnd_limited || + (!tp->is_cwnd_limited && + tp->packets_out > tp->max_packets_out)) { tp->is_cwnd_limited = is_cwnd_limited; + tp->max_packets_out = tp->packets_out; + tp->cwnd_usage_seq = tp->snd_nxt; } if (tcp_is_cwnd_limited(sk)) { diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index b4dfb82d6ecbef59967be631b3698cce5bd57b10..cb79127f45c341e13bb66f8dc61c4fa84dbd340d 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -428,7 +428,7 @@ static void tcp_fastopen_synack_timer(struct sock *sk, struct request_sock *req) if (!tp->retrans_stamp) tp->retrans_stamp = tcp_time_stamp(tp); inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS, - TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX); + req->timeout << req->num_timeout, TCP_RTO_MAX); } diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 34eda973bbf13e089ff1ae45c852f4acfba0d67d..662d717d512335bcb99f57b829402f61efebbe94 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -246,7 +246,7 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum, inet_get_local_port_range(net, &low, &high); remaining = (high - low) + 1; - rand = prandom_u32(); + rand = get_random_u32(); first = reciprocal_scale(rand, remaining) + low; /* * force rand to be an odd multiple of UDP_HTABLE_SIZE @@ -783,6 +783,8 @@ int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) */ if (tunnel) { /* ...not for tunnels though: we don't have a sending socket */ + if (udp_sk(sk)->encap_err_rcv) + udp_sk(sk)->encap_err_rcv(sk, skb, iph->ihl << 2); goto out; } if (!inet->recverr) { @@ -1596,7 +1598,7 @@ drop: } EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb); -void udp_destruct_sock(struct sock *sk) +void udp_destruct_common(struct sock *sk) { /* reclaim completely the forward allocated memory */ struct udp_sock *up = udp_sk(sk); @@ -1609,10 +1611,14 @@ void udp_destruct_sock(struct sock *sk) kfree_skb(skb); } udp_rmem_release(sk, total, 0, true); +} +EXPORT_SYMBOL_GPL(udp_destruct_common); +static void udp_destruct_sock(struct sock *sk) +{ + udp_destruct_common(sk); inet_sock_destruct(sk); } -EXPORT_SYMBOL_GPL(udp_destruct_sock); int udp_init_sock(struct sock *sk) { @@ -1620,7 +1626,6 @@ int udp_init_sock(struct sock *sk) sk->sk_destruct = udp_destruct_sock; return 0; } -EXPORT_SYMBOL_GPL(udp_init_sock); void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) { @@ -1799,41 +1804,29 @@ EXPORT_SYMBOL(__skb_recv_udp); int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { - int copied = 0; - - while (1) { - struct sk_buff *skb; - int err, used; - - skb = skb_recv_udp(sk, MSG_DONTWAIT, &err); - if (!skb) - return err; + struct sk_buff *skb; + int err, copied; - if (udp_lib_checksum_complete(skb)) { - __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, - IS_UDPLITE(sk)); - __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, - IS_UDPLITE(sk)); - atomic_inc(&sk->sk_drops); - kfree_skb(skb); - continue; - } +try_again: + skb = skb_recv_udp(sk, MSG_DONTWAIT, &err); + if (!skb) + return err; - WARN_ON(!skb_set_owner_sk_safe(skb, sk)); - used = recv_actor(sk, skb); - if (used <= 0) { - if (!copied) - copied = used; - kfree_skb(skb); - break; - } else if (used <= skb->len) { - copied += used; - } + if (udp_lib_checksum_complete(skb)) { + int is_udplite = IS_UDPLITE(sk); + struct net *net = sock_net(sk); + __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, is_udplite); + __UDP_INC_STATS(net, UDP_MIB_INERRORS, is_udplite); + atomic_inc(&sk->sk_drops); kfree_skb(skb); - break; + goto try_again; } + WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk)); + copied = recv_actor(sk, skb); + kfree_skb(skb); + return copied; } EXPORT_SYMBOL(udp_read_skb); diff --git a/net/ipv4/udp_tunnel_core.c b/net/ipv4/udp_tunnel_core.c index 8efaf8c3fe2a9ade839b684c454bc395e2ceb202..8242c8947340e86327040ecdc0429a552de4c0b4 100644 --- a/net/ipv4/udp_tunnel_core.c +++ b/net/ipv4/udp_tunnel_core.c @@ -72,6 +72,7 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock, udp_sk(sk)->encap_type = cfg->encap_type; udp_sk(sk)->encap_rcv = cfg->encap_rcv; + udp_sk(sk)->encap_err_rcv = cfg->encap_err_rcv; udp_sk(sk)->encap_err_lookup = cfg->encap_err_lookup; udp_sk(sk)->encap_destroy = cfg->encap_destroy; udp_sk(sk)->gro_receive = cfg->gro_receive; diff --git a/net/ipv4/udplite.c b/net/ipv4/udplite.c index 6e08a76ae1e7e13905fa13ea12e075b94308a8ff..e0c9cc39b81e38df3f83d22a886b2f793c7b732b 100644 --- a/net/ipv4/udplite.c +++ b/net/ipv4/udplite.c @@ -17,6 +17,14 @@ struct udp_table udplite_table __read_mostly; EXPORT_SYMBOL(udplite_table); +/* Designate sk as UDP-Lite socket */ +static int udplite_sk_init(struct sock *sk) +{ + udp_init_sock(sk); + udp_sk(sk)->pcflag = UDPLITE_BIT; + return 0; +} + static int udplite_rcv(struct sk_buff *skb) { return __udp4_lib_rcv(skb, &udplite_table, IPPROTO_UDPLITE); diff --git a/net/ipv4/xfrm4_tunnel.c b/net/ipv4/xfrm4_tunnel.c index 9d4f418f1bf81655a6a16647c032f78539b7e1e1..8489fa10658377eb0942943e537a453d781f4520 100644 --- a/net/ipv4/xfrm4_tunnel.c +++ b/net/ipv4/xfrm4_tunnel.c @@ -22,13 +22,17 @@ static int ipip_xfrm_rcv(struct xfrm_state *x, struct sk_buff *skb) return ip_hdr(skb)->protocol; } -static int ipip_init_state(struct xfrm_state *x) +static int ipip_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { - if (x->props.mode != XFRM_MODE_TUNNEL) + if (x->props.mode != XFRM_MODE_TUNNEL) { + NL_SET_ERR_MSG(extack, "IPv4 tunnel can only be used with tunnel mode"); return -EINVAL; + } - if (x->encap) + if (x->encap) { + NL_SET_ERR_MSG(extack, "IPv4 tunnel is not compatible with encapsulation"); return -EINVAL; + } x->props.header_len = sizeof(struct iphdr); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index b624e3d8c5f0a5062879d8cbfacfc37d138afc3a..417834b7169d7afc9730d2a2fe919261e77871f1 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -104,7 +104,7 @@ static inline u32 cstamp_delta(unsigned long cstamp) static inline s32 rfc3315_s14_backoff_init(s32 irt) { /* multiply 'initial retransmission time' by 0.9 .. 1.1 */ - u64 tmp = (900000 + prandom_u32() % 200001) * (u64)irt; + u64 tmp = (900000 + prandom_u32_max(200001)) * (u64)irt; do_div(tmp, 1000000); return (s32)tmp; } @@ -112,11 +112,11 @@ static inline s32 rfc3315_s14_backoff_init(s32 irt) static inline s32 rfc3315_s14_backoff_update(s32 rt, s32 mrt) { /* multiply 'retransmission timeout' by 1.9 .. 2.1 */ - u64 tmp = (1900000 + prandom_u32() % 200001) * (u64)rt; + u64 tmp = (1900000 + prandom_u32_max(200001)) * (u64)rt; do_div(tmp, 1000000); if ((s32)tmp > mrt) { /* multiply 'maximum retransmission time' by 0.9 .. 1.1 */ - tmp = (900000 + prandom_u32() % 200001) * (u64)mrt; + tmp = (900000 + prandom_u32_max(200001)) * (u64)mrt; do_div(tmp, 1000000); } return (s32)tmp; @@ -3557,11 +3557,15 @@ static int addrconf_notify(struct notifier_block *this, unsigned long event, fallthrough; case NETDEV_UP: case NETDEV_CHANGE: - if (dev->flags & IFF_SLAVE) + if (idev && idev->cnf.disable_ipv6) break; - if (idev && idev->cnf.disable_ipv6) + if (dev->flags & IFF_SLAVE) { + if (event == NETDEV_UP && !IS_ERR_OR_NULL(idev) && + dev->flags & IFF_UP && dev->flags & IFF_MULTICAST) + ipv6_mc_up(idev); break; + } if (event == NETDEV_UP) { /* restore routes for permanent addresses */ @@ -3963,7 +3967,7 @@ static void addrconf_dad_kick(struct inet6_ifaddr *ifp) if (ifp->flags & IFA_F_OPTIMISTIC) rand_num = 0; else - rand_num = prandom_u32() % (idev->cnf.rtr_solicit_delay ? : 1); + rand_num = prandom_u32_max(idev->cnf.rtr_solicit_delay ?: 1); nonce = 0; if (idev->cnf.enhanced_dad || @@ -7162,9 +7166,8 @@ static int __net_init addrconf_init_net(struct net *net) if (!dflt) goto err_alloc_dflt; - if (IS_ENABLED(CONFIG_SYSCTL) && - !net_eq(net, &init_net)) { - switch (sysctl_devconf_inherit_init_net) { + if (!net_eq(net, &init_net)) { + switch (net_inherit_devconf()) { case 1: /* copy from init_net */ memcpy(all, init_net.ipv6.devconf_all, sizeof(ipv6_devconf)); diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 2ce0c44d0081440aaf0d7a0d4de2b14775f72917..0241910049825ba6e67ac66e3569bdca4512640d 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -109,6 +109,12 @@ static __inline__ struct ipv6_pinfo *inet6_sk_generic(struct sock *sk) return (struct ipv6_pinfo *)(((u8 *)sk) + offset); } +void inet6_sock_destruct(struct sock *sk) +{ + inet6_cleanup_sock(sk); + inet_sock_destruct(sk); +} + static int inet6_create(struct net *net, struct socket *sock, int protocol, int kern) { @@ -201,7 +207,7 @@ lookup_protocol: inet->hdrincl = 1; } - sk->sk_destruct = inet_sock_destruct; + sk->sk_destruct = inet6_sock_destruct; sk->sk_family = PF_INET6; sk->sk_protocol = protocol; @@ -510,6 +516,12 @@ void inet6_destroy_sock(struct sock *sk) } EXPORT_SYMBOL_GPL(inet6_destroy_sock); +void inet6_cleanup_sock(struct sock *sk) +{ + inet6_destroy_sock(sk); +} +EXPORT_SYMBOL_GPL(inet6_cleanup_sock); + /* * This does both peername and sockname. */ @@ -1057,6 +1069,8 @@ static const struct ipv6_stub ipv6_stub_impl = { static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = { .inet6_bind = __inet6_bind, .udp6_lib_lookup = __udp6_lib_lookup, + .ipv6_setsockopt = do_ipv6_setsockopt, + .ipv6_getsockopt = do_ipv6_getsockopt, }; static int __init inet6_init(void) @@ -1070,13 +1084,13 @@ static int __init inet6_init(void) for (r = &inetsw6[0]; r < &inetsw6[SOCK_MAX]; ++r) INIT_LIST_HEAD(r); + raw_hashinfo_init(&raw_v6_hashinfo); + if (disable_ipv6_mod) { pr_info("Loaded, but administratively disabled, reboot required to enable\n"); goto out; } - raw_hashinfo_init(&raw_v6_hashinfo); - err = proto_register(&tcpv6_prot, 1); if (err) goto out; diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index b5995c1f4d7a50dd2f8766304f2bb4a4ae4f2358..5228d27162893e4f2eecac9303533f4e85596cf1 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -666,30 +666,38 @@ static int ah6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; } -static int ah6_init_state(struct xfrm_state *x) +static int ah6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { struct ah_data *ahp = NULL; struct xfrm_algo_desc *aalg_desc; struct crypto_ahash *ahash; - if (!x->aalg) + if (!x->aalg) { + NL_SET_ERR_MSG(extack, "AH requires a state with an AUTH algorithm"); goto error; + } - if (x->encap) + if (x->encap) { + NL_SET_ERR_MSG(extack, "AH is not compatible with encapsulation"); goto error; + } ahp = kzalloc(sizeof(*ahp), GFP_KERNEL); if (!ahp) return -ENOMEM; ahash = crypto_alloc_ahash(x->aalg->alg_name, 0, 0); - if (IS_ERR(ahash)) + if (IS_ERR(ahash)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } ahp->ahash = ahash; if (crypto_ahash_setkey(ahash, x->aalg->alg_key, - (x->aalg->alg_key_len + 7) / 8)) + (x->aalg->alg_key_len + 7) / 8)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } /* * Lookup the algorithm description maintained by xfrm_algo, @@ -702,9 +710,7 @@ static int ah6_init_state(struct xfrm_state *x) if (aalg_desc->uinfo.auth.icv_fullbits/8 != crypto_ahash_digestsize(ahash)) { - pr_info("AH: %s digestsize %u != %u\n", - x->aalg->alg_name, crypto_ahash_digestsize(ahash), - aalg_desc->uinfo.auth.icv_fullbits/8); + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; } @@ -721,6 +727,7 @@ static int ah6_init_state(struct xfrm_state *x) x->props.header_len += sizeof(struct ipv6hdr); break; default: + NL_SET_ERR_MSG(extack, "Invalid mode requested for AH, must be one of TRANSPORT, TUNNEL, BEET"); goto error; } x->data = ahp; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 8220923a12f7a854918dfeb3df696191e0a8a4cb..14ed868680c6acb4f91a8fe5e0ee9719f171695d 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -151,6 +151,7 @@ static void esp_free_tcp_sk(struct rcu_head *head) static struct sock *esp6_find_tcp_sk(struct xfrm_state *x) { struct xfrm_encap_tmpl *encap = x->encap; + struct net *net = xs_net(x); struct esp_tcp_sk *esk; __be16 sport, dport; struct sock *nsk; @@ -177,7 +178,7 @@ static struct sock *esp6_find_tcp_sk(struct xfrm_state *x) } spin_unlock_bh(&x->lock); - sk = __inet6_lookup_established(xs_net(x), &tcp_hashinfo, &x->id.daddr.in6, + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &x->id.daddr.in6, dport, &x->props.saddr.in6, ntohs(sport), 0, 0); if (!sk) return ERR_PTR(-ENOENT); @@ -1050,16 +1051,17 @@ static void esp6_destroy(struct xfrm_state *x) crypto_free_aead(aead); } -static int esp_init_aead(struct xfrm_state *x) +static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack) { char aead_name[CRYPTO_MAX_ALG_NAME]; struct crypto_aead *aead; int err; - err = -ENAMETOOLONG; if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", - x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) - goto error; + x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); + return -ENAMETOOLONG; + } aead = crypto_alloc_aead(aead_name, 0, 0); err = PTR_ERR(aead); @@ -1077,11 +1079,15 @@ static int esp_init_aead(struct xfrm_state *x) if (err) goto error; + return 0; + error: + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); return err; } -static int esp_init_authenc(struct xfrm_state *x) +static int esp_init_authenc(struct xfrm_state *x, + struct netlink_ext_ack *extack) { struct crypto_aead *aead; struct crypto_authenc_key_param *param; @@ -1092,10 +1098,6 @@ static int esp_init_authenc(struct xfrm_state *x) unsigned int keylen; int err; - err = -EINVAL; - if (!x->ealg) - goto error; - err = -ENAMETOOLONG; if ((x->props.flags & XFRM_STATE_ESN)) { @@ -1104,22 +1106,28 @@ static int esp_init_authenc(struct xfrm_state *x) x->geniv ?: "", x->geniv ? "(" : "", x->aalg ? x->aalg->alg_name : "digest_null", x->ealg->alg_name, - x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) + x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); goto error; + } } else { if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME, "%s%sauthenc(%s,%s)%s", x->geniv ?: "", x->geniv ? "(" : "", x->aalg ? x->aalg->alg_name : "digest_null", x->ealg->alg_name, - x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) + x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) { + NL_SET_ERR_MSG(extack, "Algorithm name is too long"); goto error; + } } aead = crypto_alloc_aead(authenc_name, 0, 0); err = PTR_ERR(aead); - if (IS_ERR(aead)) + if (IS_ERR(aead)) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto error; + } x->data = aead; @@ -1149,17 +1157,16 @@ static int esp_init_authenc(struct xfrm_state *x) err = -EINVAL; if (aalg_desc->uinfo.auth.icv_fullbits / 8 != crypto_aead_authsize(aead)) { - pr_info("ESP: %s digestsize %u != %u\n", - x->aalg->alg_name, - crypto_aead_authsize(aead), - aalg_desc->uinfo.auth.icv_fullbits / 8); + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto free_key; } err = crypto_aead_setauthsize( aead, x->aalg->alg_trunc_len / 8); - if (err) + if (err) { + NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations"); goto free_key; + } } param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8); @@ -1174,7 +1181,7 @@ error: return err; } -static int esp6_init_state(struct xfrm_state *x) +static int esp6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { struct crypto_aead *aead; u32 align; @@ -1182,10 +1189,14 @@ static int esp6_init_state(struct xfrm_state *x) x->data = NULL; - if (x->aead) - err = esp_init_aead(x); - else - err = esp_init_authenc(x); + if (x->aead) { + err = esp_init_aead(x, extack); + } else if (x->ealg) { + err = esp_init_authenc(x, extack); + } else { + NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided"); + err = -EINVAL; + } if (err) goto error; @@ -1213,6 +1224,7 @@ static int esp6_init_state(struct xfrm_state *x) switch (encap->encap_type) { default: + NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP"); err = -EINVAL; goto error; case UDP_ENCAP_ESPINUDP: diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index 3a293838a91db4846aee08260ca1ec1e290972e7..79d43548279cb1be4766ba1e6c2b761c5fe65e8c 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -145,7 +145,10 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x, struct sk_buff *skb, netdev_features_t features) { - return skb_eth_gso_segment(skb, features, htons(ETH_P_IPV6)); + __be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP) + : htons(ETH_P_IPV6); + + return skb_eth_gso_segment(skb, features, type); } static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x, diff --git a/net/ipv6/ila/ila_main.c b/net/ipv6/ila/ila_main.c index 36c58aa257e88c9a918f6664e85dce7fe7e3d246..3faf62530d6a4338686a564ecabd147b806263c9 100644 --- a/net/ipv6/ila/ila_main.c +++ b/net/ipv6/ila/ila_main.c @@ -55,6 +55,7 @@ struct genl_family ila_nl_family __ro_after_init = { .module = THIS_MODULE, .ops = ila_nl_ops, .n_ops = ARRAY_SIZE(ila_nl_ops), + .resv_start_op = ILA_CMD_FLUSH + 1, }; static __net_init int ila_init_net(struct net *net) diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 7d53d62783b1bb3f2ec4318526d06da16a7d0c28..b64b49012655eda43ba00bc2c8e58cf51827bf85 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -21,8 +21,6 @@ #include #include -extern struct inet_hashinfo tcp_hashinfo; - u32 inet6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) @@ -169,7 +167,7 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net, struct sock *sk, *reuse_sk; bool no_reuseport; - if (hashinfo != &tcp_hashinfo) + if (hashinfo != net->ipv4.tcp_death_row.hashinfo) return NULL; /* only TCP is supported */ no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c index 1098131ed90c5027480dd2fd83e77cd531ddcfe4..571f0e4d9cf3d085bf19a6497aa33623d1532aeb 100644 --- a/net/ipv6/ioam6.c +++ b/net/ipv6/ioam6.c @@ -619,6 +619,7 @@ static struct genl_family ioam6_genl_family __ro_after_init = { .parallel_ops = true, .ops = ioam6_genl_ops, .n_ops = ARRAY_SIZE(ioam6_genl_ops), + .resv_start_op = IOAM6_CMD_NS_SET_SCHEMA + 1, .module = THIS_MODULE, }; diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index ceb85c67ce3952b7142eeec29bff46a7eaf5217b..18481eb76a0a4a0e79924f4c657412d09bf914fe 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -220,7 +220,7 @@ static struct ip6_flowlabel *fl_intern(struct net *net, spin_lock_bh(&ip6_fl_lock); if (label == 0) { for (;;) { - fl->label = htonl(prandom_u32())&IPV6_FLOWLABEL_MASK; + fl->label = htonl(get_random_u32())&IPV6_FLOWLABEL_MASK; if (fl->label) { lfl = __fl_lookup(net, fl->label); if (!lfl) diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 80cb50d459e4c5f972f2cd6f941226268ff422ba..48b4ff0294f6c53e84578bca9016e0905ab6a539 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -360,7 +360,7 @@ static struct ip6_tnl *ip6gre_tunnel_locate(struct net *net, if (parms->name[0]) { if (!dev_valid_name(parms->name)) return NULL; - strlcpy(name, parms->name, IFNAMSIZ); + strscpy(name, parms->name, IFNAMSIZ); } else { strcpy(name, "ip6gre%d"); } diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index d12dba2dd5354dbb79bb80df4038dec2544cddeb..3ee345672849a8f1489e0adeb7a10b62e5860165 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -219,12 +219,9 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, off = skb_gro_offset(skb); hlen = off + sizeof(*iph); - iph = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) { - iph = skb_gro_header_slow(skb, hlen, off); - if (unlikely(!iph)) - goto out; - } + iph = skb_gro_header(skb, hlen, off); + if (unlikely(!iph)) + goto out; skb_set_network_header(skb, off); skb_gro_pull(skb, sizeof(*iph)); @@ -235,7 +232,7 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, proto = iph->nexthdr; ops = rcu_dereference(inet6_offloads[proto]); if (!ops || !ops->callbacks.gro_receive) { - __pskb_pull(skb, skb_gro_offset(skb)); + pskb_pull(skb, skb_gro_offset(skb)); skb_gro_frag0_invalidate(skb); proto = ipv6_gso_pull_exthdrs(skb, proto); skb_gro_pull(skb, -skb_transport_offset(skb)); diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index f152e51242cb6e11f33ab69150e44cd079f8e3af..e19507614f6413393d571d3b00d883b7d8b46f59 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1567,7 +1567,7 @@ emsgsize: paged = true; zc = true; } else { - uarg->zerocopy = 0; + uarg_to_msgzc(uarg)->zerocopy = 0; skb_zcopy_set(skb, uarg, &extra_uref); } } @@ -1648,10 +1648,7 @@ alloc_new_skb: (fraglen + alloc_extra < SKB_MAX_ALLOC || !(rt->dst.dev->features & NETIF_F_SG))) alloclen = fraglen; - else if (!zc) { - alloclen = min_t(int, fraglen, MAX_HEADER); - pagedlen = fraglen - alloclen; - } else { + else { alloclen = fragheaderlen + transhdrlen; pagedlen = datalen - transhdrlen; } diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 79c6a827dea9fa102f917afd5b11283428d36dcc..cc5d5e75b658f3972122d573eac88c7d59d637cb 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -293,7 +293,7 @@ static struct ip6_tnl *ip6_tnl_create(struct net *net, struct __ip6_tnl_parm *p) if (p->name[0]) { if (!dev_valid_name(p->name)) goto failed; - strlcpy(name, p->name, IFNAMSIZ); + strscpy(name, p->name, IFNAMSIZ); } else { sprintf(name, "ip6tnl%%d"); } @@ -1988,39 +1988,6 @@ static void ip6_tnl_netlink_parms(struct nlattr *data[], parms->fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]); } -static bool ip6_tnl_netlink_encap_parms(struct nlattr *data[], - struct ip_tunnel_encap *ipencap) -{ - bool ret = false; - - memset(ipencap, 0, sizeof(*ipencap)); - - if (!data) - return ret; - - if (data[IFLA_IPTUN_ENCAP_TYPE]) { - ret = true; - ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]); - } - - if (data[IFLA_IPTUN_ENCAP_FLAGS]) { - ret = true; - ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]); - } - - if (data[IFLA_IPTUN_ENCAP_SPORT]) { - ret = true; - ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]); - } - - if (data[IFLA_IPTUN_ENCAP_DPORT]) { - ret = true; - ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]); - } - - return ret; -} - static int ip6_tnl_newlink(struct net *src_net, struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) @@ -2033,7 +2000,7 @@ static int ip6_tnl_newlink(struct net *src_net, struct net_device *dev, nt = netdev_priv(dev); - if (ip6_tnl_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { err = ip6_tnl_encap_setup(nt, &ipencap); if (err < 0) return err; @@ -2070,7 +2037,7 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[], if (dev == ip6n->fb_tnl_dev) return -EINVAL; - if (ip6_tnl_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { int err = ip6_tnl_encap_setup(t, &ipencap); if (err < 0) diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index 8fe59a79e800eb76fdbbe0211ae3254cac6fded3..151337d7f67b43ed930cc14f679be95764d645bc 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -154,7 +154,7 @@ vti6_tnl_link(struct vti6_net *ip6n, struct ip6_tnl *t) { struct ip6_tnl __rcu **tp = vti6_tnl_bucket(ip6n, &t->parms); - rcu_assign_pointer(t->next , rtnl_dereference(*tp)); + rcu_assign_pointer(t->next, rtnl_dereference(*tp)); rcu_assign_pointer(*tp, t); } @@ -211,7 +211,7 @@ static struct ip6_tnl *vti6_tnl_create(struct net *net, struct __ip6_tnl_parm *p if (p->name[0]) { if (!dev_valid_name(p->name)) goto failed; - strlcpy(name, p->name, IFNAMSIZ); + strscpy(name, p->name, IFNAMSIZ); } else { sprintf(name, "ip6_vti%%d"); } diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index a9ba41648e36854fe1702a3397a36a366be4c682..facdc78a43e5c67103a31d642254b94b106193c9 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1028,8 +1028,11 @@ static void ip6mr_cache_resolve(struct net *net, struct mr_table *mrt, ((struct nlmsgerr *)nlmsg_data(nlh))->error = -EMSGSIZE; } rtnl_unicast(skb, net, NETLINK_CB(skb).portid); - } else + } else { + rcu_read_lock(); ip6_mr_forward(net, mrt, skb->dev, skb, c); + rcu_read_unlock(); + } } } @@ -1827,8 +1830,8 @@ int ip6_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, * Getsock opt support for the multicast routing system. */ -int ip6_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, - int __user *optlen) +int ip6_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval, + sockptr_t optlen) { int olr; int val; @@ -1859,16 +1862,16 @@ int ip6_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, return -ENOPROTOOPT; } - if (get_user(olr, optlen)) + if (copy_from_sockptr(&olr, optlen, sizeof(int))) return -EFAULT; olr = min_t(int, olr, sizeof(int)); if (olr < 0) return -EINVAL; - if (put_user(olr, optlen)) + if (copy_to_sockptr(optlen, &olr, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &val, olr)) + if (copy_to_sockptr(optval, &val, olr)) return -EFAULT; return 0; } diff --git a/net/ipv6/ipcomp6.c b/net/ipv6/ipcomp6.c index 15f984be35705ab7c316e47985f89fd258335c22..72d4858dec18a0b985171c7ddda70e073aa0ac9d 100644 --- a/net/ipv6/ipcomp6.c +++ b/net/ipv6/ipcomp6.c @@ -136,7 +136,8 @@ out: return err; } -static int ipcomp6_init_state(struct xfrm_state *x) +static int ipcomp6_init_state(struct xfrm_state *x, + struct netlink_ext_ack *extack) { int err = -EINVAL; @@ -148,17 +149,20 @@ static int ipcomp6_init_state(struct xfrm_state *x) x->props.header_len += sizeof(struct ipv6hdr); break; default: + NL_SET_ERR_MSG(extack, "Unsupported XFRM mode for IPcomp"); goto out; } - err = ipcomp_init_state(x); + err = ipcomp_init_state(x, extack); if (err) goto out; if (x->props.mode == XFRM_MODE_TUNNEL) { err = ipcomp6_tunnel_attach(x); - if (err) + if (err) { + NL_SET_ERR_MSG(extack, "Kernel error: failed to initialize the associated state"); goto out; + } } err = 0; diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 222f6bf220ba0d08bdde1464a1d383f819b3fe34..532f4478c88402b3967241e7399b5385d688db90 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -210,7 +210,7 @@ static int ipv6_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < GROUP_FILTER_SIZE(0)) return -EINVAL; - if (optlen > sysctl_optmem_max) + if (optlen > READ_ONCE(sysctl_optmem_max)) return -ENOBUFS; gsf = memdup_sockptr(optval, optlen); @@ -244,7 +244,7 @@ static int compat_ipv6_set_mcast_msfilter(struct sock *sk, sockptr_t optval, if (optlen < size0) return -EINVAL; - if (optlen > sysctl_optmem_max - 4) + if (optlen > READ_ONCE(sysctl_optmem_max) - 4) return -ENOBUFS; p = kmalloc(optlen + 4, GFP_KERNEL); @@ -327,7 +327,7 @@ static int ipv6_set_opt_hdr(struct sock *sk, int optname, sockptr_t optval, int err; /* hop-by-hop / destination options are privileged option */ - if (optname != IPV6_RTHDR && !ns_capable(net->user_ns, CAP_NET_RAW)) + if (optname != IPV6_RTHDR && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW)) return -EPERM; /* remove any sticky options header with a zero option @@ -391,8 +391,8 @@ sticky_done: return err; } -static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, - sockptr_t optval, unsigned int optlen) +int do_ipv6_setsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, unsigned int optlen) { struct ipv6_pinfo *np = inet6_sk(sk); struct net *net = sock_net(sk); @@ -417,7 +417,13 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (needs_rtnl) rtnl_lock(); - lock_sock(sk); + sockopt_lock_sock(sk); + + /* Another thread has converted the socket into IPv4 with + * IPV6_ADDRFORM concurrently. + */ + if (unlikely(sk->sk_family != AF_INET6)) + goto unlock; switch (optname) { @@ -425,9 +431,6 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (optlen < sizeof(int)) goto e_inval; if (val == PF_INET) { - struct ipv6_txoptions *opt; - struct sk_buff *pktopt; - if (sk->sk_type == SOCK_RAW) break; @@ -458,7 +461,6 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, break; } - fl6_free_socklist(sk); __ipv6_sock_mc_close(sk); __ipv6_sock_ac_close(sk); @@ -475,9 +477,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, &tcp_prot, 1); - /* Paired with READ_ONCE(sk->sk_prot) in net/ipv6/af_inet6.c */ + /* Paired with READ_ONCE(sk->sk_prot) in inet6_stream_ops */ WRITE_ONCE(sk->sk_prot, &tcp_prot); - icsk->icsk_af_ops = &ipv4_specific; + /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ + WRITE_ONCE(icsk->icsk_af_ops, &ipv4_specific); sk->sk_socket->ops = &inet_stream_ops; sk->sk_family = PF_INET; tcp_sync_mss(sk, icsk->icsk_pmtu_cookie); @@ -490,19 +493,19 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, prot, 1); - /* Paired with READ_ONCE(sk->sk_prot) in net/ipv6/af_inet6.c */ + /* Paired with READ_ONCE(sk->sk_prot) in inet6_dgram_ops */ WRITE_ONCE(sk->sk_prot, prot); sk->sk_socket->ops = &inet_dgram_ops; sk->sk_family = PF_INET; } - opt = xchg((__force struct ipv6_txoptions **)&np->opt, - NULL); - if (opt) { - atomic_sub(opt->tot_len, &sk->sk_omem_alloc); - txopt_put(opt); - } - pktopt = xchg(&np->pktoptions, NULL); - kfree_skb(pktopt); + + /* Disable all options not to allocate memory anymore, + * but there is still a race. See the lockless path + * in udpv6_sendmsg() and ipv6_local_rxpmtu(). + */ + np->rxopt.all = 0; + + inet6_cleanup_sock(sk); /* * ... and add it to the refcnt debug socks count @@ -634,8 +637,8 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, break; case IPV6_TRANSPARENT: - if (valbool && !ns_capable(net->user_ns, CAP_NET_RAW) && - !ns_capable(net->user_ns, CAP_NET_ADMIN)) { + if (valbool && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW) && + !sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN)) { retv = -EPERM; break; } @@ -946,7 +949,7 @@ done: case IPV6_IPSEC_POLICY: case IPV6_XFRM_POLICY: retv = -EPERM; - if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) + if (!sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN)) break; retv = xfrm_user_policy(sk, optname, optval, optlen); break; @@ -994,14 +997,15 @@ done: break; } - release_sock(sk); +unlock: + sockopt_release_sock(sk); if (needs_rtnl) rtnl_unlock(); return retv; e_inval: - release_sock(sk); + sockopt_release_sock(sk); if (needs_rtnl) rtnl_unlock(); return -EINVAL; @@ -1030,7 +1034,7 @@ int ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, EXPORT_SYMBOL(ipv6_setsockopt); static int ipv6_getsockopt_sticky(struct sock *sk, struct ipv6_txoptions *opt, - int optname, char __user *optval, int len) + int optname, sockptr_t optval, int len) { struct ipv6_opt_hdr *hdr; @@ -1058,56 +1062,53 @@ static int ipv6_getsockopt_sticky(struct sock *sk, struct ipv6_txoptions *opt, return 0; len = min_t(unsigned int, len, ipv6_optlen(hdr)); - if (copy_to_user(optval, hdr, len)) + if (copy_to_sockptr(optval, hdr, len)) return -EFAULT; return len; } -static int ipv6_get_msfilter(struct sock *sk, void __user *optval, - int __user *optlen, int len) +static int ipv6_get_msfilter(struct sock *sk, sockptr_t optval, + sockptr_t optlen, int len) { const int size0 = offsetof(struct group_filter, gf_slist_flex); - struct group_filter __user *p = optval; struct group_filter gsf; int num; int err; if (len < size0) return -EINVAL; - if (copy_from_user(&gsf, p, size0)) + if (copy_from_sockptr(&gsf, optval, size0)) return -EFAULT; if (gsf.gf_group.ss_family != AF_INET6) return -EADDRNOTAVAIL; num = gsf.gf_numsrc; - lock_sock(sk); - err = ip6_mc_msfget(sk, &gsf, p->gf_slist_flex); + sockopt_lock_sock(sk); + err = ip6_mc_msfget(sk, &gsf, optval, size0); if (!err) { if (num > gsf.gf_numsrc) num = gsf.gf_numsrc; - if (put_user(GROUP_FILTER_SIZE(num), optlen) || - copy_to_user(p, &gsf, size0)) + len = GROUP_FILTER_SIZE(num); + if (copy_to_sockptr(optlen, &len, sizeof(int)) || + copy_to_sockptr(optval, &gsf, size0)) err = -EFAULT; } - release_sock(sk); + sockopt_release_sock(sk); return err; } -static int compat_ipv6_get_msfilter(struct sock *sk, void __user *optval, - int __user *optlen) +static int compat_ipv6_get_msfilter(struct sock *sk, sockptr_t optval, + sockptr_t optlen, int len) { const int size0 = offsetof(struct compat_group_filter, gf_slist_flex); - struct compat_group_filter __user *p = optval; struct compat_group_filter gf32; struct group_filter gf; - int len, err; + int err; int num; - if (get_user(len, optlen)) - return -EFAULT; if (len < size0) return -EINVAL; - if (copy_from_user(&gf32, p, size0)) + if (copy_from_sockptr(&gf32, optval, size0)) return -EFAULT; gf.gf_interface = gf32.gf_interface; gf.gf_fmode = gf32.gf_fmode; @@ -1117,23 +1118,25 @@ static int compat_ipv6_get_msfilter(struct sock *sk, void __user *optval, if (gf.gf_group.ss_family != AF_INET6) return -EADDRNOTAVAIL; - lock_sock(sk); - err = ip6_mc_msfget(sk, &gf, p->gf_slist_flex); - release_sock(sk); + sockopt_lock_sock(sk); + err = ip6_mc_msfget(sk, &gf, optval, size0); + sockopt_release_sock(sk); if (err) return err; if (num > gf.gf_numsrc) num = gf.gf_numsrc; len = GROUP_FILTER_SIZE(num) - (sizeof(gf)-sizeof(gf32)); - if (put_user(len, optlen) || - put_user(gf.gf_fmode, &p->gf_fmode) || - put_user(gf.gf_numsrc, &p->gf_numsrc)) + if (copy_to_sockptr(optlen, &len, sizeof(int)) || + copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode), + &gf.gf_fmode, sizeof(gf32.gf_fmode)) || + copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc), + &gf.gf_numsrc, sizeof(gf32.gf_numsrc))) return -EFAULT; return 0; } -static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen, unsigned int flags) +int do_ipv6_getsockopt(struct sock *sk, int level, int optname, + sockptr_t optval, sockptr_t optlen) { struct ipv6_pinfo *np = inet6_sk(sk); int len; @@ -1142,7 +1145,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, if (ip6_mroute_opt(optname)) return ip6_mroute_getsockopt(sk, optname, optval, optlen); - if (get_user(len, optlen)) + if (copy_from_sockptr(&len, optlen, sizeof(int))) return -EFAULT; switch (optname) { case IPV6_ADDRFORM: @@ -1156,7 +1159,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, break; case MCAST_MSFILTER: if (in_compat_syscall()) - return compat_ipv6_get_msfilter(sk, optval, optlen); + return compat_ipv6_get_msfilter(sk, optval, optlen, len); return ipv6_get_msfilter(sk, optval, optlen, len); case IPV6_2292PKTOPTIONS: { @@ -1166,16 +1169,21 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, if (sk->sk_type != SOCK_STREAM) return -ENOPROTOOPT; - msg.msg_control_user = optval; + if (optval.is_kernel) { + msg.msg_control_is_user = false; + msg.msg_control = optval.kernel; + } else { + msg.msg_control_is_user = true; + msg.msg_control_user = optval.user; + } msg.msg_controllen = len; - msg.msg_flags = flags; - msg.msg_control_is_user = true; + msg.msg_flags = 0; - lock_sock(sk); + sockopt_lock_sock(sk); skb = np->pktoptions; if (skb) ip6_datagram_recv_ctl(sk, &msg, skb); - release_sock(sk); + sockopt_release_sock(sk); if (!skb) { if (np->rxopt.bits.rxinfo) { struct in6_pktinfo src_info; @@ -1212,7 +1220,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, } } len -= msg.msg_controllen; - return put_user(len, optlen); + return copy_to_sockptr(optlen, &len, sizeof(int)); } case IPV6_MTU: { @@ -1264,15 +1272,15 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, { struct ipv6_txoptions *opt; - lock_sock(sk); + sockopt_lock_sock(sk); opt = rcu_dereference_protected(np->opt, lockdep_sock_is_held(sk)); len = ipv6_getsockopt_sticky(sk, opt, optname, optval, len); - release_sock(sk); + sockopt_release_sock(sk); /* check if ipv6_getsockopt_sticky() returns err code */ if (len < 0) return len; - return put_user(len, optlen); + return copy_to_sockptr(optlen, &len, sizeof(int)); } case IPV6_RECVHOPOPTS: @@ -1326,9 +1334,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, if (!mtuinfo.ip6m_mtu) return -ENOTCONN; - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &mtuinfo, len)) + if (copy_to_sockptr(optval, &mtuinfo, len)) return -EFAULT; return 0; @@ -1405,7 +1413,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, if (len < sizeof(freq)) return -EINVAL; - if (copy_from_user(&freq, optval, sizeof(freq))) + if (copy_from_sockptr(&freq, optval, sizeof(freq))) return -EFAULT; if (freq.flr_action != IPV6_FL_A_GET) @@ -1420,9 +1428,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, if (val < 0) return val; - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &freq, len)) + if (copy_to_sockptr(optval, &freq, len)) return -EFAULT; return 0; @@ -1474,9 +1482,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname, return -ENOPROTOOPT; } len = min_t(unsigned int, sizeof(int), len); - if (put_user(len, optlen)) + if (copy_to_sockptr(optlen, &len, sizeof(int))) return -EFAULT; - if (copy_to_user(optval, &val, len)) + if (copy_to_sockptr(optval, &val, len)) return -EFAULT; return 0; } @@ -1492,7 +1500,8 @@ int ipv6_getsockopt(struct sock *sk, int level, int optname, if (level != SOL_IPV6) return -ENOPROTOOPT; - err = do_ipv6_getsockopt(sk, level, optname, optval, optlen, 0); + err = do_ipv6_getsockopt(sk, level, optname, + USER_SOCKPTR(optval), USER_SOCKPTR(optlen)); #ifdef CONFIG_NETFILTER /* we need to exclude all possible ENOPROTOOPTs except default case */ if (err == -ENOPROTOOPT && optname != IPV6_2292PKTOPTIONS) { diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 87c699d57b36699f1e739f98118376c448e6094e..7860383295d84728dacb7e5331a489c3bc4f50e0 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -580,7 +580,7 @@ done: } int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf, - struct sockaddr_storage __user *p) + sockptr_t optval, size_t ss_offset) { struct ipv6_pinfo *inet6 = inet6_sk(sk); const struct in6_addr *group; @@ -612,8 +612,7 @@ int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf, copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc; gsf->gf_numsrc = count; - - for (i = 0; i < copycount; i++, p++) { + for (i = 0; i < copycount; i++) { struct sockaddr_in6 *psin6; struct sockaddr_storage ss; @@ -621,8 +620,9 @@ int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf, memset(&ss, 0, sizeof(ss)); psin6->sin6_family = AF_INET6; psin6->sin6_addr = psl->sl_addr[i]; - if (copy_to_user(p, &ss, sizeof(ss))) + if (copy_to_sockptr_offset(optval, ss_offset, &ss, sizeof(ss))) return -EFAULT; + ss_offset += sizeof(ss); } return 0; } @@ -1050,7 +1050,7 @@ bool ipv6_chk_mcast_addr(struct net_device *dev, const struct in6_addr *group, /* called with mc_lock */ static void mld_gq_start_work(struct inet6_dev *idev) { - unsigned long tv = prandom_u32() % idev->mc_maxdelay; + unsigned long tv = prandom_u32_max(idev->mc_maxdelay); idev->mc_gq_running = 1; if (!mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2)) @@ -1068,7 +1068,7 @@ static void mld_gq_stop_work(struct inet6_dev *idev) /* called with mc_lock */ static void mld_ifc_start_work(struct inet6_dev *idev, unsigned long delay) { - unsigned long tv = prandom_u32() % delay; + unsigned long tv = prandom_u32_max(delay); if (!mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2)) in6_dev_hold(idev); @@ -1085,7 +1085,7 @@ static void mld_ifc_stop_work(struct inet6_dev *idev) /* called with mc_lock */ static void mld_dad_start_work(struct inet6_dev *idev, unsigned long delay) { - unsigned long tv = prandom_u32() % delay; + unsigned long tv = prandom_u32_max(delay); if (!mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2)) in6_dev_hold(idev); @@ -1130,7 +1130,7 @@ static void igmp6_group_queried(struct ifmcaddr6 *ma, unsigned long resptime) } if (delay >= resptime) - delay = prandom_u32() % resptime; + delay = prandom_u32_max(resptime); if (!mod_delayed_work(mld_wq, &ma->mca_work, delay)) refcount_inc(&ma->mca_refcnt); @@ -2574,7 +2574,7 @@ static void igmp6_join_group(struct ifmcaddr6 *ma) igmp6_send(&ma->mca_addr, ma->idev->dev, ICMPV6_MGM_REPORT); - delay = prandom_u32() % unsolicited_report_interval(ma->idev); + delay = prandom_u32_max(unsolicited_report_interval(ma->idev)); if (cancel_delayed_work(&ma->mca_work)) { refcount_dec(&ma->mca_refcnt); diff --git a/net/ipv6/mip6.c b/net/ipv6/mip6.c index aeb35d26e47490be2ea26e1e95ac27a9f20f7b2e..83d2a8be263fb7bdd0cbe820168b1aac9a4336b2 100644 --- a/net/ipv6/mip6.c +++ b/net/ipv6/mip6.c @@ -247,15 +247,14 @@ static int mip6_destopt_reject(struct xfrm_state *x, struct sk_buff *skb, return err; } -static int mip6_destopt_init_state(struct xfrm_state *x) +static int mip6_destopt_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { if (x->id.spi) { - pr_info("%s: spi is not 0: %u\n", __func__, x->id.spi); + NL_SET_ERR_MSG(extack, "SPI must be 0"); return -EINVAL; } if (x->props.mode != XFRM_MODE_ROUTEOPTIMIZATION) { - pr_info("%s: state's mode is not %u: %u\n", - __func__, XFRM_MODE_ROUTEOPTIMIZATION, x->props.mode); + NL_SET_ERR_MSG(extack, "XFRM mode must be XFRM_MODE_ROUTEOPTIMIZATION"); return -EINVAL; } @@ -333,15 +332,14 @@ static int mip6_rthdr_output(struct xfrm_state *x, struct sk_buff *skb) return 0; } -static int mip6_rthdr_init_state(struct xfrm_state *x) +static int mip6_rthdr_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { if (x->id.spi) { - pr_info("%s: spi is not 0: %u\n", __func__, x->id.spi); + NL_SET_ERR_MSG(extack, "SPI must be 0"); return -EINVAL; } if (x->props.mode != XFRM_MODE_ROUTEOPTIMIZATION) { - pr_info("%s: state's mode is not %u: %u\n", - __func__, XFRM_MODE_ROUTEOPTIMIZATION, x->props.mode); + NL_SET_ERR_MSG(extack, "XFRM mode must be XFRM_MODE_ROUTEOPTIMIZATION"); return -EINVAL; } diff --git a/net/ipv6/netfilter/ip6t_rpfilter.c b/net/ipv6/netfilter/ip6t_rpfilter.c index d800801a5dd27ca0527fb8b9008595813c06338a..69d86b040a6afe649f3c5d9e2bca235e95a689db 100644 --- a/net/ipv6/netfilter/ip6t_rpfilter.c +++ b/net/ipv6/netfilter/ip6t_rpfilter.c @@ -37,6 +37,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb, bool ret = false; struct flowi6 fl6 = { .flowi6_iif = LOOPBACK_IFINDEX, + .flowi6_l3mdev = l3mdev_master_ifindex_rcu(dev), .flowlabel = (* (__be32 *) iph) & IPV6_FLOWINFO_MASK, .flowi6_proto = iph->nexthdr, .daddr = iph->saddr, @@ -55,9 +56,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb, if (rpfilter_addr_linklocal(&iph->saddr)) { lookup_flags |= RT6_LOOKUP_F_IFACE; fl6.flowi6_oif = dev->ifindex; - /* Set flowi6_oif for vrf devices to lookup route in l3mdev domain. */ - } else if (netif_is_l3_master(dev) || netif_is_l3_slave(dev) || - (flags & XT_RPFILTER_LOOSE) == 0) + } else if ((flags & XT_RPFILTER_LOOSE) == 0) fl6.flowi6_oif = dev->ifindex; rt = (void *)ip6_route_lookup(net, &fl6, skb, lookup_flags); @@ -72,9 +71,7 @@ static bool rpfilter_lookup_reverse6(struct net *net, const struct sk_buff *skb, goto out; } - if (rt->rt6i_idev->dev == dev || - l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) == dev->ifindex || - (flags & XT_RPFILTER_LOOSE)) + if (rt->rt6i_idev->dev == dev || (flags & XT_RPFILTER_LOOSE)) ret = true; out: ip6_rt_put(rt); diff --git a/net/ipv6/netfilter/nf_conntrack_reasm.c b/net/ipv6/netfilter/nf_conntrack_reasm.c index 7dd3629dd19e71a6db2add2265ca49ab9cceaf63..38db0064d6613a8472ec2835afdbf80071c1fcc2 100644 --- a/net/ipv6/netfilter/nf_conntrack_reasm.c +++ b/net/ipv6/netfilter/nf_conntrack_reasm.c @@ -86,7 +86,6 @@ static int nf_ct_frag6_sysctl_register(struct net *net) table[1].extra2 = &nf_frag->fqdir->high_thresh; table[2].data = &nf_frag->fqdir->high_thresh; table[2].extra1 = &nf_frag->fqdir->low_thresh; - table[2].extra2 = &nf_frag->fqdir->high_thresh; hdr = register_net_sysctl(net, "net/netfilter", table); if (hdr == NULL) diff --git a/net/ipv6/netfilter/nf_socket_ipv6.c b/net/ipv6/netfilter/nf_socket_ipv6.c index aa5bb8789ba0b44f075ae6a8992132062762b8fa..a7690ec623259636a0c5de2067ccf980ab4e491c 100644 --- a/net/ipv6/netfilter/nf_socket_ipv6.c +++ b/net/ipv6/netfilter/nf_socket_ipv6.c @@ -83,8 +83,8 @@ nf_socket_get_sock_v6(struct net *net, struct sk_buff *skb, int doff, { switch (protocol) { case IPPROTO_TCP: - return inet6_lookup(net, &tcp_hashinfo, skb, doff, - saddr, sport, daddr, dport, + return inet6_lookup(net, net->ipv4.tcp_death_row.hashinfo, + skb, doff, saddr, sport, daddr, dport, in->ifindex); case IPPROTO_UDP: return udp6_lib_lookup(net, saddr, sport, daddr, dport, diff --git a/net/ipv6/netfilter/nf_tproxy_ipv6.c b/net/ipv6/netfilter/nf_tproxy_ipv6.c index 6bac68fb27a396fa2d80bac2085e716a82ef035f..929502e51203bc39232ab0e43f768fe5f72b7641 100644 --- a/net/ipv6/netfilter/nf_tproxy_ipv6.c +++ b/net/ipv6/netfilter/nf_tproxy_ipv6.c @@ -80,6 +80,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff, const struct net_device *in, const enum nf_tproxy_lookup_t lookup_type) { + struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo; struct sock *sk; switch (protocol) { @@ -93,7 +94,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff, switch (lookup_type) { case NF_TPROXY_LOOKUP_LISTENER: - sk = inet6_lookup_listener(net, &tcp_hashinfo, skb, + sk = inet6_lookup_listener(net, hinfo, skb, thoff + __tcp_hdrlen(hp), saddr, sport, daddr, ntohs(dport), @@ -108,9 +109,8 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff, */ break; case NF_TPROXY_LOOKUP_ESTABLISHED: - sk = __inet6_lookup_established(net, &tcp_hashinfo, - saddr, sport, daddr, ntohs(dport), - in->ifindex, 0); + sk = __inet6_lookup_established(net, hinfo, saddr, sport, daddr, + ntohs(dport), in->ifindex, 0); break; default: BUG(); diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c index 8970d0b4faeb422fbcf335d9b75fa932dd87bd85..91faac610e03dbdc659202e0404177c242a3563b 100644 --- a/net/ipv6/netfilter/nft_fib_ipv6.c +++ b/net/ipv6/netfilter/nft_fib_ipv6.c @@ -41,6 +41,8 @@ static int nft_fib6_flowi_init(struct flowi6 *fl6, const struct nft_fib *priv, if (ipv6_addr_type(&fl6->daddr) & IPV6_ADDR_LINKLOCAL) { lookup_flags |= RT6_LOOKUP_F_IFACE; fl6->flowi6_oif = get_ifindex(dev ? dev : pkt->skb->dev); + } else if (priv->flags & NFTA_FIB_F_IIF) { + fl6->flowi6_l3mdev = l3mdev_master_ifindex_rcu(dev); } if (ipv6_addr_type(&fl6->saddr) & IPV6_ADDR_UNICAST) @@ -197,7 +199,8 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, if (rt->rt6i_flags & (RTF_REJECT | RTF_ANYCAST | RTF_LOCAL)) goto put_rt_err; - if (oif && oif != rt->rt6i_idev->dev) + if (oif && oif != rt->rt6i_idev->dev && + l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) != oif->ifindex) goto put_rt_err; nft_fib_store_result(dest, priv, rt->rt6i_idev->dev); diff --git a/net/ipv6/output_core.c b/net/ipv6/output_core.c index 2880dc7d9a491682bfea59757593d07178874b56..2685c3f15e9d30e8500f32e6b1e39ce36bf3509c 100644 --- a/net/ipv6/output_core.c +++ b/net/ipv6/output_core.c @@ -18,7 +18,7 @@ static u32 __ipv6_select_ident(struct net *net, u32 id; do { - id = prandom_u32(); + id = get_random_u32(); } while (!id); return id; diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index 91b840514656901517f12757af04298c23d1af55..86c26e48d065a17bf59571c563d3efb25f21fbbf 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -20,6 +20,7 @@ #include #include #include +#include #include static void ping_v6_destroy(struct sock *sk) @@ -49,6 +50,20 @@ static int dummy_ipv6_chk_addr(struct net *net, const struct in6_addr *addr, return 0; } +static int ping_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr, + int addr_len) +{ + /* This check is replicated from __ip6_datagram_connect() and + * intended to prevent BPF program called below from accessing + * bytes that are out of the bound specified by user in addr_len. + */ + + if (addr_len < SIN6_LEN_RFC2133) + return -EINVAL; + + return BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr); +} + static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) { struct inet_sock *inet = inet_sk(sk); @@ -164,7 +179,7 @@ static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) lock_sock(sk); err = ip6_append_data(sk, ping_getfrag, &pfh, len, - 0, &ipc6, &fl6, rt, + sizeof(struct icmp6hdr), &ipc6, &fl6, rt, MSG_DONTWAIT); if (err) { @@ -191,6 +206,7 @@ struct proto pingv6_prot = { .init = ping_init_sock, .close = ping_close, .destroy = ping_v6_destroy, + .pre_connect = ping_v6_pre_connect, .connect = ip6_datagram_connect_v6_only, .disconnect = __udp_disconnect, .setsockopt = ipv6_setsockopt, diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c index 73aaabf0e966551a1b1e12b12888475a13dbf74b..29346a6eec9ffed46b00153c4a6cb0295a327ceb 100644 --- a/net/ipv6/seg6.c +++ b/net/ipv6/seg6.c @@ -191,6 +191,11 @@ static int seg6_genl_sethmac(struct sk_buff *skb, struct genl_info *info) goto out_unlock; } + if (slen > nla_len(info->attrs[SEG6_ATTR_SECRET])) { + err = -EINVAL; + goto out_unlock; + } + if (hinfo) { err = seg6_hmac_info_del(net, hmackeyid); if (err) @@ -499,6 +504,7 @@ static struct genl_family seg6_genl_family __ro_after_init = { .parallel_ops = true, .ops = seg6_genl_ops, .n_ops = ARRAY_SIZE(seg6_genl_ops), + .resv_start_op = SEG6_CMD_GET_TUNSRC + 1, .module = THIS_MODULE, }; diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index b7de5e46fdd8f867027bcc31f83da79ed4fad1b9..8370726ae7bf138a9354651fc144ef4cda187617 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -73,6 +73,55 @@ struct bpf_lwt_prog { char *name; }; +/* default length values (expressed in bits) for both Locator-Block and + * Locator-Node Function. + * + * Both SEG6_LOCAL_LCBLOCK_DBITS and SEG6_LOCAL_LCNODE_FN_DBITS *must* be: + * i) greater than 0; + * ii) evenly divisible by 8. In other terms, the lengths of the + * Locator-Block and Locator-Node Function must be byte-aligned (we can + * relax this constraint in the future if really needed). + * + * Moreover, a third condition must hold: + * iii) SEG6_LOCAL_LCBLOCK_DBITS + SEG6_LOCAL_LCNODE_FN_DBITS <= 128. + * + * The correctness of SEG6_LOCAL_LCBLOCK_DBITS and SEG6_LOCAL_LCNODE_FN_DBITS + * values are checked during the kernel compilation. If the compilation stops, + * check the value of these parameters to see if they meet conditions (i), (ii) + * and (iii). + */ +#define SEG6_LOCAL_LCBLOCK_DBITS 32 +#define SEG6_LOCAL_LCNODE_FN_DBITS 16 + +/* The following next_csid_chk_{cntr,lcblock,lcblock_fn}_bits macros can be + * used directly to check whether the lengths (in bits) of Locator-Block and + * Locator-Node Function are valid according to (i), (ii), (iii). + */ +#define next_csid_chk_cntr_bits(blen, flen) \ + ((blen) + (flen) > 128) + +#define next_csid_chk_lcblock_bits(blen) \ +({ \ + typeof(blen) __tmp = blen; \ + (!__tmp || __tmp > 120 || (__tmp & 0x07)); \ +}) + +#define next_csid_chk_lcnode_fn_bits(flen) \ + next_csid_chk_lcblock_bits(flen) + +/* Supported Flavor operations are reported in this bitmask */ +#define SEG6_LOCAL_FLV_SUPP_OPS (BIT(SEG6_LOCAL_FLV_OP_NEXT_CSID)) + +struct seg6_flavors_info { + /* Flavor operations */ + __u32 flv_ops; + + /* Locator-Block length, expressed in bits */ + __u8 lcblock_bits; + /* Locator-Node Function length, expressed in bits*/ + __u8 lcnode_func_bits; +}; + enum seg6_end_dt_mode { DT_INVALID_MODE = -EINVAL, DT_LEGACY_MODE = 0, @@ -136,6 +185,8 @@ struct seg6_local_lwt { #ifdef CONFIG_NET_L3_MASTER_DEV struct seg6_end_dt_info dt_info; #endif + struct seg6_flavors_info flv_info; + struct pcpu_seg6_local_counters __percpu *pcpu_counters; int headroom; @@ -271,8 +322,50 @@ int seg6_lookup_nexthop(struct sk_buff *skb, return seg6_lookup_any_nexthop(skb, nhaddr, tbl_id, false); } -/* regular endpoint function */ -static int input_action_end(struct sk_buff *skb, struct seg6_local_lwt *slwt) +static __u8 seg6_flv_lcblock_octects(const struct seg6_flavors_info *finfo) +{ + return finfo->lcblock_bits >> 3; +} + +static __u8 seg6_flv_lcnode_func_octects(const struct seg6_flavors_info *finfo) +{ + return finfo->lcnode_func_bits >> 3; +} + +static bool seg6_next_csid_is_arg_zero(const struct in6_addr *addr, + const struct seg6_flavors_info *finfo) +{ + __u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo); + __u8 blk_octects = seg6_flv_lcblock_octects(finfo); + __u8 arg_octects; + int i; + + arg_octects = 16 - blk_octects - fnc_octects; + for (i = 0; i < arg_octects; ++i) { + if (addr->s6_addr[blk_octects + fnc_octects + i] != 0x00) + return false; + } + + return true; +} + +/* assume that DA.Argument length > 0 */ +static void seg6_next_csid_advance_arg(struct in6_addr *addr, + const struct seg6_flavors_info *finfo) +{ + __u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo); + __u8 blk_octects = seg6_flv_lcblock_octects(finfo); + + /* advance DA.Argument */ + memmove(&addr->s6_addr[blk_octects], + &addr->s6_addr[blk_octects + fnc_octects], + 16 - blk_octects - fnc_octects); + + memset(&addr->s6_addr[16 - fnc_octects], 0x00, fnc_octects); +} + +static int input_action_end_core(struct sk_buff *skb, + struct seg6_local_lwt *slwt) { struct ipv6_sr_hdr *srh; @@ -291,6 +384,38 @@ drop: return -EINVAL; } +static int end_next_csid_core(struct sk_buff *skb, struct seg6_local_lwt *slwt) +{ + const struct seg6_flavors_info *finfo = &slwt->flv_info; + struct in6_addr *daddr = &ipv6_hdr(skb)->daddr; + + if (seg6_next_csid_is_arg_zero(daddr, finfo)) + return input_action_end_core(skb, slwt); + + /* update DA */ + seg6_next_csid_advance_arg(daddr, finfo); + + seg6_lookup_nexthop(skb, NULL, 0); + + return dst_input(skb); +} + +static bool seg6_next_csid_enabled(__u32 fops) +{ + return fops & BIT(SEG6_LOCAL_FLV_OP_NEXT_CSID); +} + +/* regular endpoint function */ +static int input_action_end(struct sk_buff *skb, struct seg6_local_lwt *slwt) +{ + const struct seg6_flavors_info *finfo = &slwt->flv_info; + + if (seg6_next_csid_enabled(finfo->flv_ops)) + return end_next_csid_core(skb, slwt); + + return input_action_end_core(skb, slwt); +} + /* regular endpoint, and forward to specified nexthop */ static int input_action_end_x(struct sk_buff *skb, struct seg6_local_lwt *slwt) { @@ -951,7 +1076,8 @@ static struct seg6_action_desc seg6_action_table[] = { { .action = SEG6_LOCAL_ACTION_END, .attrs = 0, - .optattrs = SEG6_F_LOCAL_COUNTERS, + .optattrs = SEG6_F_LOCAL_COUNTERS | + SEG6_F_ATTR(SEG6_LOCAL_FLAVORS), .input = input_action_end, }, { @@ -1132,9 +1258,11 @@ static const struct nla_policy seg6_local_policy[SEG6_LOCAL_MAX + 1] = { [SEG6_LOCAL_OIF] = { .type = NLA_U32 }, [SEG6_LOCAL_BPF] = { .type = NLA_NESTED }, [SEG6_LOCAL_COUNTERS] = { .type = NLA_NESTED }, + [SEG6_LOCAL_FLAVORS] = { .type = NLA_NESTED }, }; -static int parse_nla_srh(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_srh(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct ipv6_sr_hdr *srh; int len; @@ -1191,7 +1319,8 @@ static void destroy_attr_srh(struct seg6_local_lwt *slwt) kfree(slwt->srh); } -static int parse_nla_table(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_table(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { slwt->table = nla_get_u32(attrs[SEG6_LOCAL_TABLE]); @@ -1225,7 +1354,8 @@ seg6_end_dt_info *seg6_possible_end_dt_info(struct seg6_local_lwt *slwt) } static int parse_nla_vrftable(struct nlattr **attrs, - struct seg6_local_lwt *slwt) + struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct seg6_end_dt_info *info = seg6_possible_end_dt_info(slwt); @@ -1261,7 +1391,8 @@ static int cmp_nla_vrftable(struct seg6_local_lwt *a, struct seg6_local_lwt *b) return 0; } -static int parse_nla_nh4(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_nh4(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { memcpy(&slwt->nh4, nla_data(attrs[SEG6_LOCAL_NH4]), sizeof(struct in_addr)); @@ -1287,7 +1418,8 @@ static int cmp_nla_nh4(struct seg6_local_lwt *a, struct seg6_local_lwt *b) return memcmp(&a->nh4, &b->nh4, sizeof(struct in_addr)); } -static int parse_nla_nh6(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_nh6(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { memcpy(&slwt->nh6, nla_data(attrs[SEG6_LOCAL_NH6]), sizeof(struct in6_addr)); @@ -1313,7 +1445,8 @@ static int cmp_nla_nh6(struct seg6_local_lwt *a, struct seg6_local_lwt *b) return memcmp(&a->nh6, &b->nh6, sizeof(struct in6_addr)); } -static int parse_nla_iif(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_iif(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { slwt->iif = nla_get_u32(attrs[SEG6_LOCAL_IIF]); @@ -1336,7 +1469,8 @@ static int cmp_nla_iif(struct seg6_local_lwt *a, struct seg6_local_lwt *b) return 0; } -static int parse_nla_oif(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_oif(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { slwt->oif = nla_get_u32(attrs[SEG6_LOCAL_OIF]); @@ -1366,7 +1500,8 @@ static const struct nla_policy bpf_prog_policy[SEG6_LOCAL_BPF_PROG_MAX + 1] = { .len = MAX_PROG_NAME }, }; -static int parse_nla_bpf(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_bpf(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct nlattr *tb[SEG6_LOCAL_BPF_PROG_MAX + 1]; struct bpf_prog *p; @@ -1444,7 +1579,8 @@ nla_policy seg6_local_counters_policy[SEG6_LOCAL_CNT_MAX + 1] = { }; static int parse_nla_counters(struct nlattr **attrs, - struct seg6_local_lwt *slwt) + struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct pcpu_seg6_local_counters __percpu *pcounters; struct nlattr *tb[SEG6_LOCAL_CNT_MAX + 1]; @@ -1542,8 +1678,195 @@ static void destroy_attr_counters(struct seg6_local_lwt *slwt) free_percpu(slwt->pcpu_counters); } +static const +struct nla_policy seg6_local_flavors_policy[SEG6_LOCAL_FLV_MAX + 1] = { + [SEG6_LOCAL_FLV_OPERATION] = { .type = NLA_U32 }, + [SEG6_LOCAL_FLV_LCBLOCK_BITS] = { .type = NLA_U8 }, + [SEG6_LOCAL_FLV_LCNODE_FN_BITS] = { .type = NLA_U8 }, +}; + +/* check whether the lengths of the Locator-Block and Locator-Node Function + * are compatible with the dimension of a C-SID container. + */ +static int seg6_chk_next_csid_cfg(__u8 block_len, __u8 func_len) +{ + /* Locator-Block and Locator-Node Function cannot exceed 128 bits + * (i.e. C-SID container lenghts). + */ + if (next_csid_chk_cntr_bits(block_len, func_len)) + return -EINVAL; + + /* Locator-Block length must be greater than zero and evenly divisible + * by 8. There must be room for a Locator-Node Function, at least. + */ + if (next_csid_chk_lcblock_bits(block_len)) + return -EINVAL; + + /* Locator-Node Function length must be greater than zero and evenly + * divisible by 8. There must be room for the Locator-Block. + */ + if (next_csid_chk_lcnode_fn_bits(func_len)) + return -EINVAL; + + return 0; +} + +static int seg6_parse_nla_next_csid_cfg(struct nlattr **tb, + struct seg6_flavors_info *finfo, + struct netlink_ext_ack *extack) +{ + __u8 func_len = SEG6_LOCAL_LCNODE_FN_DBITS; + __u8 block_len = SEG6_LOCAL_LCBLOCK_DBITS; + int rc; + + if (tb[SEG6_LOCAL_FLV_LCBLOCK_BITS]) + block_len = nla_get_u8(tb[SEG6_LOCAL_FLV_LCBLOCK_BITS]); + + if (tb[SEG6_LOCAL_FLV_LCNODE_FN_BITS]) + func_len = nla_get_u8(tb[SEG6_LOCAL_FLV_LCNODE_FN_BITS]); + + rc = seg6_chk_next_csid_cfg(block_len, func_len); + if (rc < 0) { + NL_SET_ERR_MSG(extack, + "Invalid Locator Block/Node Function lengths"); + return rc; + } + + finfo->lcblock_bits = block_len; + finfo->lcnode_func_bits = func_len; + + return 0; +} + +static int parse_nla_flavors(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) +{ + struct seg6_flavors_info *finfo = &slwt->flv_info; + struct nlattr *tb[SEG6_LOCAL_FLV_MAX + 1]; + unsigned long fops; + int rc; + + rc = nla_parse_nested_deprecated(tb, SEG6_LOCAL_FLV_MAX, + attrs[SEG6_LOCAL_FLAVORS], + seg6_local_flavors_policy, NULL); + if (rc < 0) + return rc; + + /* this attribute MUST always be present since it represents the Flavor + * operation(s) to be carried out. + */ + if (!tb[SEG6_LOCAL_FLV_OPERATION]) + return -EINVAL; + + fops = nla_get_u32(tb[SEG6_LOCAL_FLV_OPERATION]); + if (fops & ~SEG6_LOCAL_FLV_SUPP_OPS) { + NL_SET_ERR_MSG(extack, "Unsupported Flavor operation(s)"); + return -EOPNOTSUPP; + } + + finfo->flv_ops = fops; + + if (seg6_next_csid_enabled(fops)) { + /* Locator-Block and Locator-Node Function lengths can be + * provided by the user space. Otherwise, default values are + * applied. + */ + rc = seg6_parse_nla_next_csid_cfg(tb, finfo, extack); + if (rc < 0) + return rc; + } + + return 0; +} + +static int seg6_fill_nla_next_csid_cfg(struct sk_buff *skb, + struct seg6_flavors_info *finfo) +{ + if (nla_put_u8(skb, SEG6_LOCAL_FLV_LCBLOCK_BITS, finfo->lcblock_bits)) + return -EMSGSIZE; + + if (nla_put_u8(skb, SEG6_LOCAL_FLV_LCNODE_FN_BITS, + finfo->lcnode_func_bits)) + return -EMSGSIZE; + + return 0; +} + +static int put_nla_flavors(struct sk_buff *skb, struct seg6_local_lwt *slwt) +{ + struct seg6_flavors_info *finfo = &slwt->flv_info; + __u32 fops = finfo->flv_ops; + struct nlattr *nest; + int rc; + + nest = nla_nest_start(skb, SEG6_LOCAL_FLAVORS); + if (!nest) + return -EMSGSIZE; + + if (nla_put_u32(skb, SEG6_LOCAL_FLV_OPERATION, fops)) { + rc = -EMSGSIZE; + goto err; + } + + if (seg6_next_csid_enabled(fops)) { + rc = seg6_fill_nla_next_csid_cfg(skb, finfo); + if (rc < 0) + goto err; + } + + return nla_nest_end(skb, nest); + +err: + nla_nest_cancel(skb, nest); + return rc; +} + +static int seg6_cmp_nla_next_csid_cfg(struct seg6_flavors_info *finfo_a, + struct seg6_flavors_info *finfo_b) +{ + if (finfo_a->lcblock_bits != finfo_b->lcblock_bits) + return 1; + + if (finfo_a->lcnode_func_bits != finfo_b->lcnode_func_bits) + return 1; + + return 0; +} + +static int cmp_nla_flavors(struct seg6_local_lwt *a, struct seg6_local_lwt *b) +{ + struct seg6_flavors_info *finfo_a = &a->flv_info; + struct seg6_flavors_info *finfo_b = &b->flv_info; + + if (finfo_a->flv_ops != finfo_b->flv_ops) + return 1; + + if (seg6_next_csid_enabled(finfo_a->flv_ops)) { + if (seg6_cmp_nla_next_csid_cfg(finfo_a, finfo_b)) + return 1; + } + + return 0; +} + +static int encap_size_flavors(struct seg6_local_lwt *slwt) +{ + struct seg6_flavors_info *finfo = &slwt->flv_info; + int nlsize; + + nlsize = nla_total_size(0) + /* nest SEG6_LOCAL_FLAVORS */ + nla_total_size(4); /* SEG6_LOCAL_FLV_OPERATION */ + + if (seg6_next_csid_enabled(finfo->flv_ops)) + nlsize += nla_total_size(1) + /* SEG6_LOCAL_FLV_LCBLOCK_BITS */ + nla_total_size(1); /* SEG6_LOCAL_FLV_LCNODE_FN_BITS */ + + return nlsize; +} + struct seg6_action_param { - int (*parse)(struct nlattr **attrs, struct seg6_local_lwt *slwt); + int (*parse)(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack); int (*put)(struct sk_buff *skb, struct seg6_local_lwt *slwt); int (*cmp)(struct seg6_local_lwt *a, struct seg6_local_lwt *b); @@ -1593,6 +1916,10 @@ static struct seg6_action_param seg6_action_params[SEG6_LOCAL_MAX + 1] = { .put = put_nla_counters, .cmp = cmp_nla_counters, .destroy = destroy_attr_counters }, + + [SEG6_LOCAL_FLAVORS] = { .parse = parse_nla_flavors, + .put = put_nla_flavors, + .cmp = cmp_nla_flavors }, }; /* call the destroy() callback (if available) for each set attribute in @@ -1636,7 +1963,8 @@ static void destroy_attrs(struct seg6_local_lwt *slwt) } static int parse_nla_optional_attrs(struct nlattr **attrs, - struct seg6_local_lwt *slwt) + struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct seg6_action_desc *desc = slwt->desc; unsigned long parsed_optattrs = 0; @@ -1652,7 +1980,7 @@ static int parse_nla_optional_attrs(struct nlattr **attrs, */ param = &seg6_action_params[i]; - err = param->parse(attrs, slwt); + err = param->parse(attrs, slwt, extack); if (err < 0) goto parse_optattrs_err; @@ -1705,7 +2033,8 @@ static void seg6_local_lwtunnel_destroy_state(struct seg6_local_lwt *slwt) ops->destroy_state(slwt); } -static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt) +static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt, + struct netlink_ext_ack *extack) { struct seg6_action_param *param; struct seg6_action_desc *desc; @@ -1749,14 +2078,14 @@ static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt) param = &seg6_action_params[i]; - err = param->parse(attrs, slwt); + err = param->parse(attrs, slwt, extack); if (err < 0) goto parse_attrs_err; } } /* parse the optional attributes, if any */ - err = parse_nla_optional_attrs(attrs, slwt); + err = parse_nla_optional_attrs(attrs, slwt, extack); if (err < 0) goto parse_attrs_err; @@ -1800,7 +2129,7 @@ static int seg6_local_build_state(struct net *net, struct nlattr *nla, slwt = seg6_local_lwtunnel(newts); slwt->action = nla_get_u32(tb[SEG6_LOCAL_ACTION]); - err = parse_nla_action(tb, slwt); + err = parse_nla_action(tb, slwt, extack); if (err < 0) goto out_free; @@ -1904,6 +2233,9 @@ static int seg6_local_get_encap_size(struct lwtunnel_state *lwt) /* SEG6_LOCAL_CNT_ERRORS */ nla_total_size_64bit(sizeof(__u64)); + if (attrs & SEG6_F_ATTR(SEG6_LOCAL_FLAVORS)) + nlsize += encap_size_flavors(slwt); + return nlsize; } @@ -1959,6 +2291,15 @@ int __init seg6_local_init(void) */ BUILD_BUG_ON(SEG6_LOCAL_MAX + 1 > BITS_PER_TYPE(unsigned long)); + /* If the default NEXT-C-SID Locator-Block/Node Function lengths (in + * bits) have been changed with invalid values, kernel build stops + * here. + */ + BUILD_BUG_ON(next_csid_chk_cntr_bits(SEG6_LOCAL_LCBLOCK_DBITS, + SEG6_LOCAL_LCNODE_FN_DBITS)); + BUILD_BUG_ON(next_csid_chk_lcblock_bits(SEG6_LOCAL_LCBLOCK_DBITS)); + BUILD_BUG_ON(next_csid_chk_lcnode_fn_bits(SEG6_LOCAL_LCNODE_FN_DBITS)); + return lwtunnel_encap_add_ops(&seg6_local_ops, LWTUNNEL_ENCAP_SEG6_LOCAL); } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 6b73b7a5f1755b8143f6285437eddd8b7cce2089..d27683e3fc971ec8f9c6ece5f92c5927f5feb789 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -254,7 +254,7 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net, if (parms->name[0]) { if (!dev_valid_name(parms->name)) goto failed; - strlcpy(name, parms->name, IFNAMSIZ); + strscpy(name, parms->name, IFNAMSIZ); } else { strcpy(name, "sit%d"); } @@ -1503,71 +1503,12 @@ static void ipip6_netlink_parms(struct nlattr *data[], if (!data) return; - if (data[IFLA_IPTUN_LINK]) - parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]); - - if (data[IFLA_IPTUN_LOCAL]) - parms->iph.saddr = nla_get_be32(data[IFLA_IPTUN_LOCAL]); - - if (data[IFLA_IPTUN_REMOTE]) - parms->iph.daddr = nla_get_be32(data[IFLA_IPTUN_REMOTE]); - - if (data[IFLA_IPTUN_TTL]) { - parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]); - if (parms->iph.ttl) - parms->iph.frag_off = htons(IP_DF); - } - - if (data[IFLA_IPTUN_TOS]) - parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]); - - if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC])) - parms->iph.frag_off = htons(IP_DF); - - if (data[IFLA_IPTUN_FLAGS]) - parms->i_flags = nla_get_be16(data[IFLA_IPTUN_FLAGS]); - - if (data[IFLA_IPTUN_PROTO]) - parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]); + ip_tunnel_netlink_parms(data, parms); if (data[IFLA_IPTUN_FWMARK]) *fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]); } -/* This function returns true when ENCAP attributes are present in the nl msg */ -static bool ipip6_netlink_encap_parms(struct nlattr *data[], - struct ip_tunnel_encap *ipencap) -{ - bool ret = false; - - memset(ipencap, 0, sizeof(*ipencap)); - - if (!data) - return ret; - - if (data[IFLA_IPTUN_ENCAP_TYPE]) { - ret = true; - ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]); - } - - if (data[IFLA_IPTUN_ENCAP_FLAGS]) { - ret = true; - ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]); - } - - if (data[IFLA_IPTUN_ENCAP_SPORT]) { - ret = true; - ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]); - } - - if (data[IFLA_IPTUN_ENCAP_DPORT]) { - ret = true; - ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]); - } - - return ret; -} - #ifdef CONFIG_IPV6_SIT_6RD /* This function returns true when 6RD attributes are present in the nl msg */ static bool ipip6_netlink_6rd_parms(struct nlattr *data[], @@ -1619,7 +1560,7 @@ static int ipip6_newlink(struct net *src_net, struct net_device *dev, nt = netdev_priv(dev); - if (ipip6_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { err = ip_tunnel_encap_setup(nt, &ipencap); if (err < 0) return err; @@ -1671,7 +1612,7 @@ static int ipip6_changelink(struct net_device *dev, struct nlattr *tb[], if (dev == sitn->fb_tunnel_dev) return -EINVAL; - if (ipip6_netlink_encap_parms(data, &ipencap)) { + if (ip_tunnel_netlink_encap_parms(data, &ipencap)) { err = ip_tunnel_encap_setup(t, &ipencap); if (err < 0) return err; diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index e54eee80ce5f343dd527deb1ff0b27600794ec4b..2a3f9296df1e505b40e925c31b0d2aa2a2327cfd 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -146,15 +146,16 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; - struct inet_sock *inet = inet_sk(sk); struct inet_connection_sock *icsk = inet_csk(sk); + struct in6_addr *saddr = NULL, *final_p, final; struct inet_timewait_death_row *tcp_death_row; struct ipv6_pinfo *np = tcp_inet6_sk(sk); + struct inet_sock *inet = inet_sk(sk); struct tcp_sock *tp = tcp_sk(sk); - struct in6_addr *saddr = NULL, *final_p, final; + struct net *net = sock_net(sk); struct ipv6_txoptions *opt; - struct flowi6 fl6; struct dst_entry *dst; + struct flowi6 fl6; int addr_type; int err; @@ -237,7 +238,8 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, sin.sin_port = usin->sin6_port; sin.sin_addr.s_addr = usin->sin6_addr.s6_addr32[3]; - icsk->icsk_af_ops = &ipv6_mapped; + /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ + WRITE_ONCE(icsk->icsk_af_ops, &ipv6_mapped); if (sk_is_mptcp(sk)) mptcpv6_handle_mapped(sk, true); sk->sk_backlog_rcv = tcp_v4_do_rcv; @@ -249,7 +251,8 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, if (err) { icsk->icsk_ext_hdr_len = exthdrlen; - icsk->icsk_af_ops = &ipv6_specific; + /* Paired with READ_ONCE() in tcp_(get|set)sockopt() */ + WRITE_ONCE(icsk->icsk_af_ops, &ipv6_specific); if (sk_is_mptcp(sk)) mptcpv6_handle_mapped(sk, false); sk->sk_backlog_rcv = tcp_v6_do_rcv; @@ -280,15 +283,33 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6)); - dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p); + dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); if (IS_ERR(dst)) { err = PTR_ERR(dst); goto failure; } + tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row; + if (!saddr) { + struct inet_bind_hashbucket *prev_addr_hashbucket = NULL; + struct in6_addr prev_v6_rcv_saddr; + + if (icsk->icsk_bind2_hash) { + prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo, + sk, net, inet->inet_num); + prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr; + } saddr = &fl6.saddr; sk->sk_v6_rcv_saddr = *saddr; + + if (prev_addr_hashbucket) { + err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk); + if (err) { + sk->sk_v6_rcv_saddr = prev_v6_rcv_saddr; + goto failure; + } + } } /* set the source address */ @@ -308,7 +329,6 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, inet->inet_dport = usin->sin6_port; tcp_set_state(sk, TCP_SYN_SENT); - tcp_death_row = sock_net(sk)->ipv4.tcp_death_row; err = inet6_hash_connect(tcp_death_row, sk); if (err) goto late_failure; @@ -322,8 +342,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, sk->sk_v6_daddr.s6_addr32, inet->inet_sport, inet->inet_dport)); - tp->tsoffset = secure_tcpv6_ts_off(sock_net(sk), - np->saddr.s6_addr32, + tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32, sk->sk_v6_daddr.s6_addr32); } @@ -386,7 +405,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, bool fatal; int err; - sk = __inet6_lookup_established(net, &tcp_hashinfo, + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &hdr->daddr, th->dest, &hdr->saddr, ntohs(th->source), skb->dev->ifindex, inet6_sdif(skb)); @@ -841,7 +860,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = { static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq, u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, struct tcp_md5sig_key *key, int rst, - u8 tclass, __be32 label, u32 priority) + u8 tclass, __be32 label, u32 priority, u32 txhash) { const struct tcphdr *th = tcp_hdr(skb); struct tcphdr *t1; @@ -932,16 +951,16 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 } if (sk) { - if (sk->sk_state == TCP_TIME_WAIT) { + if (sk->sk_state == TCP_TIME_WAIT) mark = inet_twsk(sk)->tw_mark; - /* autoflowlabel relies on buff->hash */ - skb_set_hash(buff, inet_twsk(sk)->tw_txhash, - PKT_HASH_TYPE_L4); - } else { + else mark = sk->sk_mark; - } skb_set_delivery_time(buff, tcp_transmit_time(sk), true); } + if (txhash) { + /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ + skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); + } fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; fl6.fl6_dport = t1->dest; fl6.fl6_sport = t1->source; @@ -984,6 +1003,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) __be32 label = 0; u32 priority = 0; struct net *net; + u32 txhash = 0; int oif = 0; if (th->rst) @@ -1019,11 +1039,10 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) * Incoming packet is checked with md5 hash with finding key, * no RST generated if md5 hash doesn't match. */ - sk1 = inet6_lookup_listener(net, - &tcp_hashinfo, NULL, 0, - &ipv6h->saddr, - th->source, &ipv6h->daddr, - ntohs(th->source), dif, sdif); + sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, + NULL, 0, &ipv6h->saddr, th->source, + &ipv6h->daddr, ntohs(th->source), + dif, sdif); if (!sk1) goto out; @@ -1057,10 +1076,12 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) if (np->repflow) label = ip6_flowlabel(ipv6h); priority = sk->sk_priority; + txhash = sk->sk_hash; } if (sk->sk_state == TCP_TIME_WAIT) { label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel); priority = inet_twsk(sk)->tw_priority; + txhash = inet_twsk(sk)->tw_txhash; } } else { if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET) @@ -1068,7 +1089,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb) } tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1, - ipv6_get_dsfield(ipv6h), label, priority); + ipv6_get_dsfield(ipv6h), label, priority, txhash); #ifdef CONFIG_TCP_MD5SIG out: @@ -1079,10 +1100,10 @@ out: static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq, u32 ack, u32 win, u32 tsval, u32 tsecr, int oif, struct tcp_md5sig_key *key, u8 tclass, - __be32 label, u32 priority) + __be32 label, u32 priority, u32 txhash) { tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, key, 0, - tclass, label, priority); + tclass, label, priority, txhash); } static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb) @@ -1094,7 +1115,8 @@ static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb) tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale, tcp_time_stamp_raw() + tcptw->tw_ts_offset, tcptw->tw_ts_recent, tw->tw_bound_dev_if, tcp_twsk_md5_key(tcptw), - tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority); + tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority, + tw->tw_txhash); inet_twsk_put(tw); } @@ -1121,7 +1143,8 @@ static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, tcp_time_stamp_raw() + tcp_rsk(req)->ts_off, req->ts_recent, sk->sk_bound_dev_if, tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, l3index), - ipv6_get_dsfield(ipv6_hdr(skb)), 0, sk->sk_priority); + ipv6_get_dsfield(ipv6_hdr(skb)), 0, sk->sk_priority, + tcp_rsk(req)->txhash); } @@ -1619,7 +1642,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) hdr = ipv6_hdr(skb); lookup: - sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), + sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th), th->source, th->dest, inet6_iif(skb), sdif, &refcounted); if (!sk) @@ -1794,7 +1817,7 @@ do_time_wait: { struct sock *sk2; - sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo, + sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th), &ipv6_hdr(skb)->saddr, th->source, &ipv6_hdr(skb)->daddr, @@ -1827,6 +1850,7 @@ do_time_wait: void tcp_v6_early_demux(struct sk_buff *skb) { + struct net *net = dev_net(skb->dev); const struct ipv6hdr *hdr; const struct tcphdr *th; struct sock *sk; @@ -1844,7 +1868,7 @@ void tcp_v6_early_demux(struct sk_buff *skb) return; /* Note : We use inet6_iif() here, not tcp_v6_iif() */ - sk = __inet6_lookup_established(dev_net(skb->dev), &tcp_hashinfo, + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &hdr->saddr, th->source, &hdr->daddr, ntohs(th->dest), inet6_iif(skb), inet6_sdif(skb)); @@ -2176,7 +2200,7 @@ struct proto tcpv6_prot = { .slab_flags = SLAB_TYPESAFE_BY_RCU, .twsk_prot = &tcp6_timewait_sock_ops, .rsk_prot = &tcp6_request_sock_ops, - .h.hashinfo = &tcp_hashinfo, + .h.hashinfo = NULL, .no_autobind = true, .diag_destroy = tcp_abort, }; @@ -2210,7 +2234,7 @@ static void __net_exit tcpv6_net_exit(struct net *net) static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list) { - inet_twsk_purge(&tcp_hashinfo, AF_INET6); + tcp_twsk_purge(net_exit_list, AF_INET6); } static struct pernet_operations tcpv6_net_ops = { diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 16c176e7c69a2bafa3a463c92456b05eacb0d248..8d09f0ea5b8c70df643a9ddd892624fba9d08f5f 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -56,6 +56,19 @@ #include #include "udp_impl.h" +static void udpv6_destruct_sock(struct sock *sk) +{ + udp_destruct_common(sk); + inet6_sock_destruct(sk); +} + +int udpv6_init_sock(struct sock *sk) +{ + skb_queue_head_init(&udp_sk(sk)->reader_queue); + sk->sk_destruct = udpv6_destruct_sock; + return 0; +} + static u32 udp6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, @@ -616,8 +629,11 @@ int __udp6_lib_err(struct sk_buff *skb, struct inet6_skb_parm *opt, } /* Tunnels don't have an application socket: don't pass errors back */ - if (tunnel) + if (tunnel) { + if (udp_sk(sk)->encap_err_rcv) + udp_sk(sk)->encap_err_rcv(sk, skb, offset); goto out; + } if (!np->recverr) { if (!harderr || sk->sk_state != TCP_ESTABLISHED) @@ -647,16 +663,20 @@ static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) rc = __udp_enqueue_schedule_skb(sk, skb); if (rc < 0) { int is_udplite = IS_UDPLITE(sk); + enum skb_drop_reason drop_reason; /* Note that an ENOMEM error is charged twice */ - if (rc == -ENOMEM) + if (rc == -ENOMEM) { UDP6_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, is_udplite); - else + drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF; + } else { UDP6_INC_STATS(sock_net(sk), UDP_MIB_MEMERRORS, is_udplite); + drop_reason = SKB_DROP_REASON_PROTO_MEM; + } UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); - kfree_skb(skb); + kfree_skb_reason(skb, drop_reason); return -1; } @@ -672,11 +692,14 @@ static __inline__ int udpv6_err(struct sk_buff *skb, static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) { + enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; struct udp_sock *up = udp_sk(sk); int is_udplite = IS_UDPLITE(sk); - if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) + if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) { + drop_reason = SKB_DROP_REASON_XFRM_POLICY; goto drop; + } if (static_branch_unlikely(&udpv6_encap_needed_key) && up->encap_type) { int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); @@ -735,8 +758,10 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) udp_lib_checksum_complete(skb)) goto csum_error; - if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) + if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) { + drop_reason = SKB_DROP_REASON_SOCKET_FILTER; goto drop; + } udp_csum_pull_header(skb); @@ -745,11 +770,12 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) return __udpv6_queue_rcv_skb(sk, skb); csum_error: + drop_reason = SKB_DROP_REASON_UDP_CSUM; __UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); drop: __UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); atomic_inc(&sk->sk_drops); - kfree_skb(skb); + kfree_skb_reason(skb, drop_reason); return -1; } @@ -1720,7 +1746,7 @@ struct proto udpv6_prot = { .connect = ip6_datagram_connect, .disconnect = udp_disconnect, .ioctl = udp_ioctl, - .init = udp_init_sock, + .init = udpv6_init_sock, .destroy = udpv6_destroy_sock, .setsockopt = udpv6_setsockopt, .getsockopt = udpv6_getsockopt, diff --git a/net/ipv6/udp_impl.h b/net/ipv6/udp_impl.h index 4251e49d32a0d067a282359e98eab0e8f67218fd..0590f566379d7d07dfdd1b0ae808b9d8964eb5aa 100644 --- a/net/ipv6/udp_impl.h +++ b/net/ipv6/udp_impl.h @@ -12,6 +12,7 @@ int __udp6_lib_rcv(struct sk_buff *, struct udp_table *, int); int __udp6_lib_err(struct sk_buff *, struct inet6_skb_parm *, u8, u8, int, __be32, struct udp_table *); +int udpv6_init_sock(struct sock *sk); int udp_v6_get_port(struct sock *sk, unsigned short snum); void udp_v6_rehash(struct sock *sk); diff --git a/net/ipv6/udplite.c b/net/ipv6/udplite.c index b707258562597ebddf5e0d75e6415b6f967cca33..67eaf3ca14cea71fad76d88414483c0d1d2321b9 100644 --- a/net/ipv6/udplite.c +++ b/net/ipv6/udplite.c @@ -12,6 +12,13 @@ #include #include "udp_impl.h" +static int udplitev6_sk_init(struct sock *sk) +{ + udpv6_init_sock(sk); + udp_sk(sk)->pcflag = UDPLITE_BIT; + return 0; +} + static int udplitev6_rcv(struct sk_buff *skb) { return __udp6_lib_rcv(skb, &udplite_table, IPPROTO_UDPLITE); @@ -38,7 +45,7 @@ struct proto udplitev6_prot = { .connect = ip6_datagram_connect, .disconnect = udp_disconnect, .ioctl = udp_ioctl, - .init = udplite_sk_init, + .init = udplitev6_sk_init, .destroy = udpv6_destroy_sock, .setsockopt = udpv6_setsockopt, .getsockopt = udpv6_getsockopt, diff --git a/net/ipv6/xfrm6_tunnel.c b/net/ipv6/xfrm6_tunnel.c index 2b31112c0856bd8136ea6283a601e4c392976726..1323f2f6928e2abf277e9ce7bd06025cd0049031 100644 --- a/net/ipv6/xfrm6_tunnel.c +++ b/net/ipv6/xfrm6_tunnel.c @@ -270,13 +270,17 @@ static int xfrm6_tunnel_err(struct sk_buff *skb, struct inet6_skb_parm *opt, return 0; } -static int xfrm6_tunnel_init_state(struct xfrm_state *x) +static int xfrm6_tunnel_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { - if (x->props.mode != XFRM_MODE_TUNNEL) + if (x->props.mode != XFRM_MODE_TUNNEL) { + NL_SET_ERR_MSG(extack, "IPv6 tunnel can only be used with tunnel mode"); return -EINVAL; + } - if (x->encap) + if (x->encap) { + NL_SET_ERR_MSG(extack, "IPv6 tunnel is not compatible with encapsulation"); return -EINVAL; + } x->props.header_len = sizeof(struct ipv6hdr); diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 71899e5a5a11133aded2410210092b8553cfd042..27725464ec08fe2b5f2e86202636cbc895568098 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -1412,12 +1412,6 @@ static int kcm_attach(struct socket *sock, struct socket *csock, psock->sk = csk; psock->bpf_prog = prog; - err = strp_init(&psock->strp, csk, &cb); - if (err) { - kmem_cache_free(kcm_psockp, psock); - goto out; - } - write_lock_bh(&csk->sk_callback_lock); /* Check if sk_user_data is already by KCM or someone else. @@ -1425,13 +1419,18 @@ static int kcm_attach(struct socket *sock, struct socket *csock, */ if (csk->sk_user_data) { write_unlock_bh(&csk->sk_callback_lock); - strp_stop(&psock->strp); - strp_done(&psock->strp); kmem_cache_free(kcm_psockp, psock); err = -EALREADY; goto out; } + err = strp_init(&psock->strp, csk, &cb); + if (err) { + write_unlock_bh(&csk->sk_callback_lock); + kmem_cache_free(kcm_psockp, psock); + goto out; + } + psock->save_data_ready = csk->sk_data_ready; psock->save_write_space = csk->sk_write_space; psock->save_state_change = csk->sk_state_change; @@ -1839,10 +1838,10 @@ static int kcm_release(struct socket *sock) kcm = kcm_sk(sk); mux = kcm->mux; + lock_sock(sk); sock_orphan(sk); kfree_skb(kcm->seq_skb); - lock_sock(sk); /* Purge queue under lock to avoid race condition with tx_work trying * to act when queue is nonempty. If tx_work runs after this point * it will just return. diff --git a/net/key/af_key.c b/net/key/af_key.c index fda2dcc8a3831b46e57d361b59988a78b71271d4..c85df5b958d266ce37fb27caba1bad74e8ef70a7 100644 --- a/net/key/af_key.c +++ b/net/key/af_key.c @@ -1697,9 +1697,12 @@ static int pfkey_register(struct sock *sk, struct sk_buff *skb, const struct sad pfk->registered |= (1<sadb_msg_satype); } + mutex_lock(&pfkey_mutex); xfrm_probe_algs(); supp_skb = compose_sadb_supported(hdr, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&pfkey_mutex); + if (!supp_skb) { if (hdr->sadb_msg_satype != SADB_SATYPE_UNSPEC) pfk->registered &= ~(1<sadb_msg_satype); diff --git a/net/l2tp/l2tp_eth.c b/net/l2tp/l2tp_eth.c index 6cd97c75445c83ee2b0f1a1b9806cce0a75102ac..f2ae03c404736d826fd7dc327b1567eac1c8651a 100644 --- a/net/l2tp/l2tp_eth.c +++ b/net/l2tp/l2tp_eth.c @@ -254,7 +254,7 @@ static int l2tp_eth_create(struct net *net, struct l2tp_tunnel *tunnel, int rc; if (cfg->ifname) { - strlcpy(name, cfg->ifname, IFNAMSIZ); + strscpy(name, cfg->ifname, IFNAMSIZ); name_assign_type = NET_NAME_USER; } else { strcpy(name, L2TP_ETH_DEV_NAME); @@ -314,7 +314,7 @@ static int l2tp_eth_create(struct net *net, struct l2tp_tunnel *tunnel, return rc; } - strlcpy(session->ifname, dev->name, IFNAMSIZ); + strscpy(session->ifname, dev->name, IFNAMSIZ); rcu_assign_pointer(spriv->dev, dev); rtnl_unlock(); diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index 96eb91be9238ba13413e7948df07f1084389fd83..a901fd14fe3bfe320951475f5f8a0880c3bdf865 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -989,6 +989,7 @@ static struct genl_family l2tp_nl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = l2tp_nl_ops, .n_small_ops = ARRAY_SIZE(l2tp_nl_ops), + .resv_start_op = L2TP_CMD_SESSION_GET + 1, .mcgrps = l2tp_multicast_group, .n_mcgrps = ARRAY_SIZE(l2tp_multicast_group), }; diff --git a/net/mac80211/Makefile b/net/mac80211/Makefile index af1df3a6bd55628f5e4036f192a2cffac6ce3573..b8de44da1fb87fbf5a1db982c2d8eef50acaa3c8 100644 --- a/net/mac80211/Makefile +++ b/net/mac80211/Makefile @@ -16,6 +16,7 @@ mac80211-y := \ s1g.o \ ibss.o \ iface.o \ + link.o \ rate.o \ michael.o \ tkip.o \ diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index a4f6971b7a190a9c1469994626c7048a43523505..687b4c878d4ad385b72f19474c1571bac7b3116f 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -23,6 +23,30 @@ #include "mesh.h" #include "wme.h" +static struct ieee80211_link_data * +ieee80211_link_or_deflink(struct ieee80211_sub_if_data *sdata, int link_id, + bool require_valid) +{ + struct ieee80211_link_data *link; + + if (link_id < 0) { + /* + * For keys, if sdata is not an MLD, we might not use + * the return value at all (if it's not a pairwise key), + * so in that case (require_valid==false) don't error. + */ + if (require_valid && sdata->vif.valid_links) + return ERR_PTR(-EINVAL); + + return &sdata->deflink; + } + + link = sdata_dereference(sdata->link[link_id], sdata); + if (!link) + return ERR_PTR(-ENOLINK); + return link; +} + static void ieee80211_set_mu_mimo_follow(struct ieee80211_sub_if_data *sdata, struct vif_params *params) { @@ -202,6 +226,10 @@ static int ieee80211_change_iface(struct wiphy *wiphy, if (params->use_4addr == ifmgd->use_4addr) return 0; + /* FIXME: no support for 4-addr MLO yet */ + if (sdata->vif.valid_links) + return -EOPNOTSUPP; + sdata->u.mgd.use_4addr = params->use_4addr; if (!ifmgd->associated) return 0; @@ -434,10 +462,12 @@ static int ieee80211_set_tx(struct ieee80211_sub_if_data *sdata, } static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx, bool pairwise, const u8 *mac_addr, - struct key_params *params) + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr, struct key_params *params) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + struct ieee80211_link_data *link = + ieee80211_link_or_deflink(sdata, link_id, false); struct ieee80211_local *local = sdata->local; struct sta_info *sta = NULL; struct ieee80211_key *key; @@ -446,6 +476,9 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, if (!ieee80211_sdata_running(sdata)) return -ENETDOWN; + if (IS_ERR(link)) + return PTR_ERR(link); + if (pairwise && params->mode == NL80211_KEY_SET_TX) return ieee80211_set_tx(sdata, mac_addr, key_idx); @@ -454,6 +487,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, case WLAN_CIPHER_SUITE_WEP40: case WLAN_CIPHER_SUITE_TKIP: case WLAN_CIPHER_SUITE_WEP104: + if (link_id >= 0) + return -EINVAL; if (WARN_ON_ONCE(fips_enabled)) return -EINVAL; break; @@ -466,6 +501,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, if (IS_ERR(key)) return PTR_ERR(key); + key->conf.link_id = link_id; + if (pairwise) key->conf.flags |= IEEE80211_KEY_FLAG_PAIRWISE; @@ -527,7 +564,7 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, break; } - err = ieee80211_key_link(key, sdata, sta); + err = ieee80211_key_link(key, link, sta); out_unlock: mutex_unlock(&local->sta_mtx); @@ -536,18 +573,37 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, } static struct ieee80211_key * -ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, +ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id, u8 key_idx, bool pairwise, const u8 *mac_addr) { struct ieee80211_local *local = sdata->local; + struct ieee80211_link_data *link = &sdata->deflink; struct ieee80211_key *key; - struct sta_info *sta; + + if (link_id >= 0) { + link = rcu_dereference_check(sdata->link[link_id], + lockdep_is_held(&sdata->wdev.mtx)); + if (!link) + return NULL; + } if (mac_addr) { + struct sta_info *sta; + struct link_sta_info *link_sta; + sta = sta_info_get_bss(sdata, mac_addr); if (!sta) return NULL; + if (link_id >= 0) { + link_sta = rcu_dereference_check(sta->link[link_id], + lockdep_is_held(&local->sta_mtx)); + if (!link_sta) + return NULL; + } else { + link_sta = &sta->deflink; + } + if (pairwise && key_idx < NUM_DEFAULT_KEYS) return rcu_dereference_check_key_mtx(local, sta->ptk[key_idx]); @@ -557,7 +613,7 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, NUM_DEFAULT_MGMT_KEYS + NUM_DEFAULT_BEACON_KEYS) return rcu_dereference_check_key_mtx(local, - sta->deflink.gtk[key_idx]); + link_sta->gtk[key_idx]); return NULL; } @@ -566,7 +622,7 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, return rcu_dereference_check_key_mtx(local, sdata->keys[key_idx]); - key = rcu_dereference_check_key_mtx(local, sdata->deflink.gtk[key_idx]); + key = rcu_dereference_check_key_mtx(local, link->gtk[key_idx]); if (key) return key; @@ -578,7 +634,8 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, } static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx, bool pairwise, const u8 *mac_addr) + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); struct ieee80211_local *local = sdata->local; @@ -588,7 +645,7 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev, mutex_lock(&local->sta_mtx); mutex_lock(&local->key_mtx); - key = ieee80211_lookup_key(sdata, key_idx, pairwise, mac_addr); + key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr); if (!key) { ret = -ENOENT; goto out_unlock; @@ -605,8 +662,8 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev, } static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx, bool pairwise, const u8 *mac_addr, - void *cookie, + int link_id, u8 key_idx, bool pairwise, + const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params *params)) { @@ -624,7 +681,7 @@ static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev, rcu_read_lock(); - key = ieee80211_lookup_key(sdata, key_idx, pairwise, mac_addr); + key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr); if (!key) goto out; @@ -711,34 +768,49 @@ static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev, static int ieee80211_config_default_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx, bool uni, + int link_id, u8 key_idx, bool uni, bool multi) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + struct ieee80211_link_data *link = + ieee80211_link_or_deflink(sdata, link_id, false); - ieee80211_set_default_key(sdata, key_idx, uni, multi); + if (IS_ERR(link)) + return PTR_ERR(link); + + ieee80211_set_default_key(link, key_idx, uni, multi); return 0; } static int ieee80211_config_default_mgmt_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx) + int link_id, u8 key_idx) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + struct ieee80211_link_data *link = + ieee80211_link_or_deflink(sdata, link_id, true); + + if (IS_ERR(link)) + return PTR_ERR(link); - ieee80211_set_default_mgmt_key(sdata, key_idx); + ieee80211_set_default_mgmt_key(link, key_idx); return 0; } static int ieee80211_config_default_beacon_key(struct wiphy *wiphy, struct net_device *dev, - u8 key_idx) + int link_id, u8 key_idx) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + struct ieee80211_link_data *link = + ieee80211_link_or_deflink(sdata, link_id, true); + + if (IS_ERR(link)) + return PTR_ERR(link); - ieee80211_set_default_beacon_key(sdata, key_idx); + ieee80211_set_default_beacon_key(link, key_idx); return 0; } @@ -1610,6 +1682,18 @@ static int sta_link_apply_parameters(struct ieee80211_local *local, rcu_dereference_protected(sta->link[link_id], lockdep_is_held(&local->sta_mtx)); + /* + * If there are no changes, then accept a link that doesn't exist, + * unless it's a new link. + */ + if (params->link_id < 0 && !new_link && + !params->link_mac && !params->txpwr_set && + !params->supported_rates_len && + !params->ht_capa && !params->vht_capa && + !params->he_capa && !params->eht_capa && + !params->opmode_notif_used) + return 0; + if (!link || !link_sta) return -EINVAL; @@ -1625,6 +1709,8 @@ static int sta_link_apply_parameters(struct ieee80211_local *local, params->link_mac)) { return -EINVAL; } + } else if (new_link) { + return -EINVAL; } if (params->txpwr_set) { @@ -2554,7 +2640,8 @@ static int ieee80211_set_txq_params(struct wiphy *wiphy, { struct ieee80211_local *local = wiphy_priv(wiphy); struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); - struct ieee80211_link_data *link = &sdata->deflink; + struct ieee80211_link_data *link = + ieee80211_link_or_deflink(sdata, params->link_id, true); struct ieee80211_tx_queue_params p; if (!local->ops->conf_tx) @@ -2563,6 +2650,9 @@ static int ieee80211_set_txq_params(struct wiphy *wiphy, if (local->hw.queues < IEEE80211_NUM_ACS) return -EOPNOTSUPP; + if (IS_ERR(link)) + return PTR_ERR(link); + memset(&p, 0, sizeof(p)); p.aifs = params->aifs; p.cw_max = params->cwmax; @@ -3597,9 +3687,6 @@ static int ieee80211_set_csa_beacon(struct ieee80211_sub_if_data *sdata, case NL80211_IFTYPE_MESH_POINT: { struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh; - if (params->chandef.width != sdata->vif.bss_conf.chandef.width) - return -EINVAL; - /* changes into another band are not supported */ if (sdata->vif.bss_conf.chandef.chan->band != params->chandef.chan->band) @@ -3732,7 +3819,7 @@ __ieee80211_channel_switch(struct wiphy *wiphy, struct net_device *dev, IEEE80211_QUEUE_STOP_REASON_CSA); cfg80211_ch_switch_started_notify(sdata->dev, - &sdata->deflink.csa_chandef, + &sdata->deflink.csa_chandef, 0, params->count, params->block_tx); if (changed) { @@ -4614,6 +4701,9 @@ static int ieee80211_add_intf_link(struct wiphy *wiphy, { struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev); + if (wdev->use_4addr) + return -EOPNOTSUPP; + return ieee80211_vif_set_links(sdata, wdev->valid_links); } diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index f247daa4156389dfe9ab3d723443d7ca0eec0e70..e72cf0749d49217a962d09e80821a07bdc0a0cb4 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -1799,6 +1799,12 @@ int ieee80211_link_use_channel(struct ieee80211_link_data *link, lockdep_assert_held(&local->mtx); + if (sdata->vif.active_links && + !(sdata->vif.active_links & BIT(link->link_id))) { + ieee80211_link_update_chandef(link, chandef); + return 0; + } + mutex_lock(&local->chanctx_mtx); ret = cfg80211_chandef_dfs_required(local->hw.wiphy, diff --git a/net/mac80211/debugfs_netdev.c b/net/mac80211/debugfs_netdev.c index 1e5b041a5cea53e754661fa6be6c7de52f38570a..5b014786fd2d0a8735221cc3d81075b3c3e11a7c 100644 --- a/net/mac80211/debugfs_netdev.c +++ b/net/mac80211/debugfs_netdev.c @@ -570,6 +570,30 @@ static ssize_t ieee80211_if_parse_tsf( } IEEE80211_IF_FILE_RW(tsf); +static ssize_t ieee80211_if_fmt_valid_links(const struct ieee80211_sub_if_data *sdata, + char *buf, int buflen) +{ + return snprintf(buf, buflen, "0x%x\n", sdata->vif.valid_links); +} +IEEE80211_IF_FILE_R(valid_links); + +static ssize_t ieee80211_if_fmt_active_links(const struct ieee80211_sub_if_data *sdata, + char *buf, int buflen) +{ + return snprintf(buf, buflen, "0x%x\n", sdata->vif.active_links); +} + +static ssize_t ieee80211_if_parse_active_links(struct ieee80211_sub_if_data *sdata, + const char *buf, int buflen) +{ + u16 active_links; + + if (kstrtou16(buf, 0, &active_links)) + return -EINVAL; + + return ieee80211_set_active_links(&sdata->vif, active_links) ?: buflen; +} +IEEE80211_IF_FILE_RW(active_links); #ifdef CONFIG_MAC80211_MESH IEEE80211_IF_FILE(estab_plinks, u.mesh.estab_plinks, ATOMIC); @@ -670,6 +694,8 @@ static void add_sta_files(struct ieee80211_sub_if_data *sdata) DEBUGFS_ADD_MODE(uapsd_queues, 0600); DEBUGFS_ADD_MODE(uapsd_max_sp_len, 0600); DEBUGFS_ADD_MODE(tdls_wider_bw, 0600); + DEBUGFS_ADD_MODE(valid_links, 0200); + DEBUGFS_ADD_MODE(active_links, 0600); } static void add_ap_files(struct ieee80211_sub_if_data *sdata) diff --git a/net/mac80211/driver-ops.c b/net/mac80211/driver-ops.c index 9b61dc7889c2577fb028d41aeebf7eb28b7a08ac..5392ffa182704feded6ec63a50579368b5c0d60d 100644 --- a/net/mac80211/driver-ops.c +++ b/net/mac80211/driver-ops.c @@ -192,6 +192,10 @@ int drv_conf_tx(struct ieee80211_local *local, if (!check_sdata_in_driver(sdata)) return -EIO; + if (sdata->vif.active_links && + !(sdata->vif.active_links & BIT(link->link_id))) + return 0; + if (params->cw_min == 0 || params->cw_min > params->cw_max) { /* * If we can't configure hardware anyway, don't warn. We may @@ -272,6 +276,60 @@ void drv_reset_tsf(struct ieee80211_local *local, trace_drv_return_void(local); } +int drv_assign_vif_chanctx(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx *ctx) +{ + int ret = 0; + + drv_verify_link_exists(sdata, link_conf); + if (!check_sdata_in_driver(sdata)) + return -EIO; + + if (sdata->vif.active_links && + !(sdata->vif.active_links & BIT(link_conf->link_id))) + return 0; + + trace_drv_assign_vif_chanctx(local, sdata, link_conf, ctx); + if (local->ops->assign_vif_chanctx) { + WARN_ON_ONCE(!ctx->driver_present); + ret = local->ops->assign_vif_chanctx(&local->hw, + &sdata->vif, + link_conf, + &ctx->conf); + } + trace_drv_return_int(local, ret); + + return ret; +} + +void drv_unassign_vif_chanctx(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx *ctx) +{ + might_sleep(); + + drv_verify_link_exists(sdata, link_conf); + if (!check_sdata_in_driver(sdata)) + return; + + if (sdata->vif.active_links && + !(sdata->vif.active_links & BIT(link_conf->link_id))) + return; + + trace_drv_unassign_vif_chanctx(local, sdata, link_conf, ctx); + if (local->ops->unassign_vif_chanctx) { + WARN_ON_ONCE(!ctx->driver_present); + local->ops->unassign_vif_chanctx(&local->hw, + &sdata->vif, + link_conf, + &ctx->conf); + } + trace_drv_return_void(local); +} + int drv_switch_vif_chanctx(struct ieee80211_local *local, struct ieee80211_vif_chanctx_switch *vifs, int n_vifs, enum ieee80211_chanctx_switch_mode mode) @@ -346,3 +404,117 @@ int drv_ampdu_action(struct ieee80211_local *local, return ret; } + +void drv_link_info_changed(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *info, + int link_id, u64 changed) +{ + might_sleep(); + + if (WARN_ON_ONCE(changed & (BSS_CHANGED_BEACON | + BSS_CHANGED_BEACON_ENABLED) && + sdata->vif.type != NL80211_IFTYPE_AP && + sdata->vif.type != NL80211_IFTYPE_ADHOC && + sdata->vif.type != NL80211_IFTYPE_MESH_POINT && + sdata->vif.type != NL80211_IFTYPE_OCB)) + return; + + if (WARN_ON_ONCE(sdata->vif.type == NL80211_IFTYPE_P2P_DEVICE || + sdata->vif.type == NL80211_IFTYPE_NAN || + (sdata->vif.type == NL80211_IFTYPE_MONITOR && + !sdata->vif.bss_conf.mu_mimo_owner && + !(changed & BSS_CHANGED_TXPOWER)))) + return; + + if (!check_sdata_in_driver(sdata)) + return; + + if (sdata->vif.active_links && + !(sdata->vif.active_links & BIT(link_id))) + return; + + trace_drv_link_info_changed(local, sdata, info, changed); + if (local->ops->link_info_changed) + local->ops->link_info_changed(&local->hw, &sdata->vif, + info, changed); + else if (local->ops->bss_info_changed) + local->ops->bss_info_changed(&local->hw, &sdata->vif, + info, changed); + trace_drv_return_void(local); +} + +int drv_set_key(struct ieee80211_local *local, + enum set_key_cmd cmd, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta *sta, + struct ieee80211_key_conf *key) +{ + int ret; + + might_sleep(); + + sdata = get_bss_sdata(sdata); + if (!check_sdata_in_driver(sdata)) + return -EIO; + + if (WARN_ON(key->link_id >= 0 && sdata->vif.active_links && + !(sdata->vif.active_links & BIT(key->link_id)))) + return -ENOLINK; + + trace_drv_set_key(local, cmd, sdata, sta, key); + ret = local->ops->set_key(&local->hw, cmd, &sdata->vif, sta, key); + trace_drv_return_int(local, ret); + return ret; +} + +int drv_change_vif_links(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + u16 old_links, u16 new_links, + struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]) +{ + int ret = -EOPNOTSUPP; + + might_sleep(); + + if (!check_sdata_in_driver(sdata)) + return -EIO; + + if (old_links == new_links) + return 0; + + trace_drv_change_vif_links(local, sdata, old_links, new_links); + if (local->ops->change_vif_links) + ret = local->ops->change_vif_links(&local->hw, &sdata->vif, + old_links, new_links, old); + trace_drv_return_int(local, ret); + + return ret; +} + +int drv_change_sta_links(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta *sta, + u16 old_links, u16 new_links) +{ + int ret = -EOPNOTSUPP; + + might_sleep(); + + if (!check_sdata_in_driver(sdata)) + return -EIO; + + old_links &= sdata->vif.active_links; + new_links &= sdata->vif.active_links; + + if (old_links == new_links) + return 0; + + trace_drv_change_sta_links(local, sdata, sta, old_links, new_links); + if (local->ops->change_sta_links) + ret = local->ops->change_sta_links(&local->hw, &sdata->vif, sta, + old_links, new_links); + trace_drv_return_int(local, ret); + + return ret; +} diff --git a/net/mac80211/driver-ops.h b/net/mac80211/driver-ops.h index 482f5c97a72b12751d6dc047298f65eb04a75f1f..81e40b0a3b1675a872f8f5f9adb90c626b145ab8 100644 --- a/net/mac80211/driver-ops.h +++ b/net/mac80211/driver-ops.h @@ -165,40 +165,10 @@ static inline void drv_vif_cfg_changed(struct ieee80211_local *local, trace_drv_return_void(local); } -static inline void drv_link_info_changed(struct ieee80211_local *local, - struct ieee80211_sub_if_data *sdata, - struct ieee80211_bss_conf *info, - int link_id, u64 changed) -{ - might_sleep(); - - if (WARN_ON_ONCE(changed & (BSS_CHANGED_BEACON | - BSS_CHANGED_BEACON_ENABLED) && - sdata->vif.type != NL80211_IFTYPE_AP && - sdata->vif.type != NL80211_IFTYPE_ADHOC && - sdata->vif.type != NL80211_IFTYPE_MESH_POINT && - sdata->vif.type != NL80211_IFTYPE_OCB)) - return; - - if (WARN_ON_ONCE(sdata->vif.type == NL80211_IFTYPE_P2P_DEVICE || - sdata->vif.type == NL80211_IFTYPE_NAN || - (sdata->vif.type == NL80211_IFTYPE_MONITOR && - !sdata->vif.bss_conf.mu_mimo_owner && - !(changed & BSS_CHANGED_TXPOWER)))) - return; - - if (!check_sdata_in_driver(sdata)) - return; - - trace_drv_link_info_changed(local, sdata, info, changed); - if (local->ops->link_info_changed) - local->ops->link_info_changed(&local->hw, &sdata->vif, - info, changed); - else if (local->ops->bss_info_changed) - local->ops->bss_info_changed(&local->hw, &sdata->vif, - info, changed); - trace_drv_return_void(local); -} +void drv_link_info_changed(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *info, + int link_id, u64 changed); static inline u64 drv_prepare_multicast(struct ieee80211_local *local, struct netdev_hw_addr_list *mc_list) @@ -256,25 +226,11 @@ static inline int drv_set_tim(struct ieee80211_local *local, return ret; } -static inline int drv_set_key(struct ieee80211_local *local, - enum set_key_cmd cmd, - struct ieee80211_sub_if_data *sdata, - struct ieee80211_sta *sta, - struct ieee80211_key_conf *key) -{ - int ret; - - might_sleep(); - - sdata = get_bss_sdata(sdata); - if (!check_sdata_in_driver(sdata)) - return -EIO; - - trace_drv_set_key(local, cmd, sdata, sta, key); - ret = local->ops->set_key(&local->hw, cmd, &sdata->vif, sta, key); - trace_drv_return_int(local, ret); - return ret; -} +int drv_set_key(struct ieee80211_local *local, + enum set_key_cmd cmd, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta *sta, + struct ieee80211_key_conf *key); static inline void drv_update_tkip_key(struct ieee80211_local *local, struct ieee80211_sub_if_data *sdata, @@ -945,52 +901,14 @@ static inline void drv_verify_link_exists(struct ieee80211_sub_if_data *sdata, sdata_assert_lock(sdata); } -static inline int drv_assign_vif_chanctx(struct ieee80211_local *local, - struct ieee80211_sub_if_data *sdata, - struct ieee80211_bss_conf *link_conf, - struct ieee80211_chanctx *ctx) -{ - int ret = 0; - - drv_verify_link_exists(sdata, link_conf); - if (!check_sdata_in_driver(sdata)) - return -EIO; - - trace_drv_assign_vif_chanctx(local, sdata, link_conf, ctx); - if (local->ops->assign_vif_chanctx) { - WARN_ON_ONCE(!ctx->driver_present); - ret = local->ops->assign_vif_chanctx(&local->hw, - &sdata->vif, - link_conf, - &ctx->conf); - } - trace_drv_return_int(local, ret); - - return ret; -} - -static inline void drv_unassign_vif_chanctx(struct ieee80211_local *local, - struct ieee80211_sub_if_data *sdata, - struct ieee80211_bss_conf *link_conf, - struct ieee80211_chanctx *ctx) -{ - might_sleep(); - - drv_verify_link_exists(sdata, link_conf); - if (!check_sdata_in_driver(sdata)) - return; - - trace_drv_unassign_vif_chanctx(local, sdata, link_conf, ctx); - if (local->ops->unassign_vif_chanctx) { - WARN_ON_ONCE(!ctx->driver_present); - local->ops->unassign_vif_chanctx(&local->hw, - &sdata->vif, - link_conf, - &ctx->conf); - } - trace_drv_return_void(local); -} - +int drv_assign_vif_chanctx(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx *ctx); +void drv_unassign_vif_chanctx(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_bss_conf *link_conf, + struct ieee80211_chanctx *ctx); int drv_switch_vif_chanctx(struct ieee80211_local *local, struct ieee80211_vif_chanctx_switch *vifs, int n_vifs, enum ieee80211_chanctx_switch_mode mode); @@ -1552,46 +1470,13 @@ static inline int drv_net_fill_forward_path(struct ieee80211_local *local, return ret; } -static inline int drv_change_vif_links(struct ieee80211_local *local, - struct ieee80211_sub_if_data *sdata, - u16 old_links, u16 new_links, - struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]) -{ - int ret = -EOPNOTSUPP; - - might_sleep(); - - if (!check_sdata_in_driver(sdata)) - return -EIO; - - trace_drv_change_vif_links(local, sdata, old_links, new_links); - if (local->ops->change_vif_links) - ret = local->ops->change_vif_links(&local->hw, &sdata->vif, - old_links, new_links, old); - trace_drv_return_int(local, ret); - - return ret; -} - -static inline int drv_change_sta_links(struct ieee80211_local *local, - struct ieee80211_sub_if_data *sdata, - struct ieee80211_sta *sta, - u16 old_links, u16 new_links) -{ - int ret = -EOPNOTSUPP; - - might_sleep(); - - if (!check_sdata_in_driver(sdata)) - return -EIO; - - trace_drv_change_sta_links(local, sdata, sta, old_links, new_links); - if (local->ops->change_sta_links) - ret = local->ops->change_sta_links(&local->hw, &sdata->vif, sta, - old_links, new_links); - trace_drv_return_int(local, ret); - - return ret; -} +int drv_change_vif_links(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + u16 old_links, u16 new_links, + struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]); +int drv_change_sta_links(struct ieee80211_local *local, + struct ieee80211_sub_if_data *sdata, + struct ieee80211_sta *sta, + u16 old_links, u16 new_links); #endif /* __MAC80211_DRIVER_OPS */ diff --git a/net/mac80211/eht.c b/net/mac80211/eht.c index 31e20a342f21628f92d5dbe21999abb931a157d3..18bc6b78b26796acd5b4fec2f1e76f2c2f8cd302 100644 --- a/net/mac80211/eht.c +++ b/net/mac80211/eht.c @@ -30,7 +30,9 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata, return; mcs_nss_size = ieee80211_eht_mcs_nss_size(he_cap_ie_elem, - &eht_cap_ie_elem->fixed); + &eht_cap_ie_elem->fixed, + sdata->vif.type == + NL80211_IFTYPE_STATION); eht_total_size += mcs_nss_size; diff --git a/net/mac80211/ethtool.c b/net/mac80211/ethtool.c index c2b38370bfb10ca69e76ad828f9cc5be71595902..a3830d925cc29ddfed3836d21aef74bebb4f542b 100644 --- a/net/mac80211/ethtool.c +++ b/net/mac80211/ethtool.c @@ -83,17 +83,17 @@ static void ieee80211_get_stats(struct net_device *dev, #define ADD_STA_STATS(sta) \ do { \ - data[i++] += (sta)->rx_stats.packets; \ - data[i++] += (sta)->rx_stats.bytes; \ + data[i++] += sinfo.rx_packets; \ + data[i++] += sinfo.rx_bytes; \ data[i++] += (sta)->rx_stats.num_duplicates; \ data[i++] += (sta)->rx_stats.fragments; \ - data[i++] += (sta)->rx_stats.dropped; \ + data[i++] += sinfo.rx_dropped_misc; \ \ data[i++] += sinfo.tx_packets; \ data[i++] += sinfo.tx_bytes; \ data[i++] += (sta)->status_stats.filtered; \ - data[i++] += (sta)->status_stats.retry_failed; \ - data[i++] += (sta)->status_stats.retry_count; \ + data[i++] += sinfo.tx_failed; \ + data[i++] += sinfo.tx_retries; \ } while (0) /* For Managed stations, find the single station based on BSSID diff --git a/net/mac80211/he.c b/net/mac80211/he.c index d9228fd3f77a50cb96808d0053ad6f1bd8b26d7f..729f261520c774a362d93fc484eb7a552e283117 100644 --- a/net/mac80211/he.c +++ b/net/mac80211/he.c @@ -31,25 +31,27 @@ ieee80211_update_from_he_6ghz_capa(const struct ieee80211_he_6ghz_capa *he_6ghz_ break; } - sta->sta.smps_mode = smps_mode; + link_sta->pub->smps_mode = smps_mode; } else { - sta->sta.smps_mode = IEEE80211_SMPS_OFF; + link_sta->pub->smps_mode = IEEE80211_SMPS_OFF; } switch (le16_get_bits(he_6ghz_capa->capa, IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN)) { case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454: - sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454; break; case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991: - sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991; break; case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895: default: - sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895; break; } + ieee80211_sta_recalc_aggregates(&sta->sta); + link_sta->pub->he_6ghz_capa = *he_6ghz_capa; } diff --git a/net/mac80211/ht.c b/net/mac80211/ht.c index 8c24817cd497295cb3d77e9cf7f2f52e12b0c73d..83bc41346ae7fe110d5cfa164cf8f4276579fe26 100644 --- a/net/mac80211/ht.c +++ b/net/mac80211/ht.c @@ -241,9 +241,11 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata, ht_cap.mcs.rx_highest = ht_cap_ie->mcs.rx_highest; if (ht_cap.cap & IEEE80211_HT_CAP_MAX_AMSDU) - sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_7935; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_7935; else - sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_3839; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_3839; + + ieee80211_sta_recalc_aggregates(&sta->sta); apply: changed = memcmp(&link_sta->pub->ht_cap, &ht_cap, sizeof(ht_cap)); @@ -299,12 +301,13 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata, break; } - if (smps_mode != sta->sta.smps_mode) + if (smps_mode != link_sta->pub->smps_mode) changed = true; - sta->sta.smps_mode = smps_mode; + link_sta->pub->smps_mode = smps_mode; } else { - sta->sta.smps_mode = IEEE80211_SMPS_OFF; + link_sta->pub->smps_mode = IEEE80211_SMPS_OFF; } + return changed; } diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c index d56890e3fabb3d43b1cb0c9cc5dae188f45b1ad6..9dffc30795887a74552b20a555d0695a9ffdf465 100644 --- a/net/mac80211/ibss.c +++ b/net/mac80211/ibss.c @@ -530,6 +530,10 @@ int ieee80211_ibss_finish_csa(struct ieee80211_sub_if_data *sdata) sdata_assert_lock(sdata); + /* When not connected/joined, sending CSA doesn't make sense. */ + if (ifibss->state != IEEE80211_IBSS_MLME_JOINED) + return -ENOLINK; + /* update cfg80211 bss information with the new channel */ if (!is_zero_ether_addr(ifibss->bssid)) { cbss = cfg80211_get_bss(sdata->local->hw.wiphy, @@ -1346,10 +1350,10 @@ static void ieee80211_sta_create_ibss(struct ieee80211_sub_if_data *sdata) capability, 0, true); } -static unsigned ibss_setup_channels(struct wiphy *wiphy, - struct ieee80211_channel **channels, - unsigned int channels_max, - u32 center_freq, u32 width) +static unsigned int ibss_setup_channels(struct wiphy *wiphy, + struct ieee80211_channel **channels, + unsigned int channels_max, + u32 center_freq, u32 width) { struct ieee80211_channel *chan = NULL; unsigned int n_chan = 0; diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index e192e1ec02610d2450513b2254cc56e1678ccb92..a842f2e1c230966c52e45bb6e84fbcfbac994c60 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -213,6 +213,7 @@ struct ieee80211_rx_data { struct ieee80211_sub_if_data *sdata; struct ieee80211_link_data *link; struct sta_info *sta; + struct link_sta_info *link_sta; struct ieee80211_key *key; unsigned int flags; @@ -1080,6 +1081,10 @@ struct ieee80211_sub_if_data { struct ieee80211_link_data deflink; struct ieee80211_link_data __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; + /* for ieee80211_set_active_links_async() */ + struct work_struct activate_links_work; + u16 desired_active_links; + #ifdef CONFIG_MAC80211_DEBUGFS struct { struct dentry *subdir_stations; @@ -1704,6 +1709,14 @@ struct ieee802_11_elems { /* whether a parse error occurred while retrieving these elements */ bool parse_error; + + /* + * scratch buffer that can be used for various element parsing related + * tasks, e.g., element de-fragmentation etc. + */ + size_t scratch_len; + u8 *scratch_pos; + u8 scratch[]; }; static inline struct ieee80211_local *hw_to_local( @@ -1810,6 +1823,7 @@ void ieee80211_sta_connection_lost(struct ieee80211_sub_if_data *sdata, u8 reason, bool tx); void ieee80211_mgd_setup_link(struct ieee80211_link_data *link); void ieee80211_mgd_stop_link(struct ieee80211_link_data *link); +void ieee80211_mgd_set_link_qos_params(struct ieee80211_link_data *link); /* IBSS code */ void ieee80211_ibss_notify_scan_completed(struct ieee80211_local *local); @@ -1929,9 +1943,6 @@ void ieee80211_sdata_stop(struct ieee80211_sub_if_data *sdata); int ieee80211_add_virtual_monitor(struct ieee80211_local *local); void ieee80211_del_virtual_monitor(struct ieee80211_local *local); -int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata, - u16 new_links); - bool __ieee80211_recalc_txpower(struct ieee80211_sub_if_data *sdata); void ieee80211_recalc_txpower(struct ieee80211_sub_if_data *sdata, bool update_bss); @@ -1942,6 +1953,17 @@ static inline bool ieee80211_sdata_running(struct ieee80211_sub_if_data *sdata) return test_bit(SDATA_STATE_RUNNING, &sdata->state); } +/* link handling */ +void ieee80211_link_setup(struct ieee80211_link_data *link); +void ieee80211_link_init(struct ieee80211_sub_if_data *sdata, + int link_id, + struct ieee80211_link_data *link, + struct ieee80211_bss_conf *link_conf); +void ieee80211_link_stop(struct ieee80211_link_data *link); +int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata, + u16 new_links); +void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata); + /* tx handling */ void ieee80211_clear_tx_pending(struct ieee80211_local *local); void ieee80211_tx_pending(struct tasklet_struct *t); @@ -2184,6 +2206,8 @@ static inline void ieee80211_tx_skb(struct ieee80211_sub_if_data *sdata, * for that non-transmitting BSS is returned * @link_id: the link ID to parse elements for, if a STA profile * is present in the multi-link element, or -1 to ignore + * @from_ap: frame is received from an AP (currently used only + * for EHT capabilities parsing) */ struct ieee80211_elems_parse_params { const u8 *start; @@ -2193,6 +2217,7 @@ struct ieee80211_elems_parse_params { u32 crc; struct cfg80211_bss *bss; int link_id; + bool from_ap; }; struct ieee802_11_elems * @@ -2382,6 +2407,7 @@ u8 *ieee80211_ie_build_he_cap(ieee80211_conn_flags_t disable_flags, u8 *pos, const struct ieee80211_sta_he_cap *he_cap, u8 *end); void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata, + enum ieee80211_smps_mode smps_mode, struct sk_buff *skb); u8 *ieee80211_ie_build_he_oper(u8 *pos, struct cfg80211_chan_def *chandef); int ieee80211_parse_bitrates(enum nl80211_chan_width width, @@ -2407,8 +2433,7 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info, const struct ieee80211_vht_operation *oper, const struct ieee80211_ht_operation *htop, struct cfg80211_chan_def *chandef); -void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata, - const struct ieee80211_eht_operation *eht_oper, +void ieee80211_chandef_eht_oper(const struct ieee80211_eht_operation *eht_oper, bool support_160, bool support_320, struct cfg80211_chan_def *chandef); bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, @@ -2513,7 +2538,8 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype); u8 *ieee80211_ie_build_eht_cap(u8 *pos, const struct ieee80211_sta_he_cap *he_cap, const struct ieee80211_sta_eht_cap *eht_cap, - u8 *end); + u8 *end, + bool for_ap); void ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata, diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 95b58c5cac07f48b7b140ca678636e320d1b6961..dd9ac1f7d2ea6771ac4328a36e1dece4c29b30b4 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -200,15 +200,73 @@ static int ieee80211_verify_mac(struct ieee80211_sub_if_data *sdata, u8 *addr, return ret; } +static int ieee80211_can_powered_addr_change(struct ieee80211_sub_if_data *sdata) +{ + struct ieee80211_roc_work *roc; + struct ieee80211_local *local = sdata->local; + struct ieee80211_sub_if_data *scan_sdata; + int ret = 0; + + /* To be the most flexible here we want to only limit changing the + * address if the specific interface is doing offchannel work or + * scanning. + */ + if (netif_carrier_ok(sdata->dev)) + return -EBUSY; + + mutex_lock(&local->mtx); + + /* First check no ROC work is happening on this iface */ + list_for_each_entry(roc, &local->roc_list, list) { + if (roc->sdata != sdata) + continue; + + if (roc->started) { + ret = -EBUSY; + goto unlock; + } + } + + /* And if this iface is scanning */ + if (local->scanning) { + scan_sdata = rcu_dereference_protected(local->scan_sdata, + lockdep_is_held(&local->mtx)); + if (sdata == scan_sdata) + ret = -EBUSY; + } + + switch (sdata->vif.type) { + case NL80211_IFTYPE_STATION: + case NL80211_IFTYPE_P2P_CLIENT: + /* More interface types could be added here but changing the + * address while powered makes the most sense in client modes. + */ + break; + default: + ret = -EOPNOTSUPP; + } + +unlock: + mutex_unlock(&local->mtx); + return ret; +} + static int ieee80211_change_mac(struct net_device *dev, void *addr) { struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev); + struct ieee80211_local *local = sdata->local; struct sockaddr *sa = addr; bool check_dup = true; + bool live = false; int ret; - if (ieee80211_sdata_running(sdata)) - return -EBUSY; + if (ieee80211_sdata_running(sdata)) { + ret = ieee80211_can_powered_addr_change(sdata); + if (ret) + return ret; + + live = true; + } if (sdata->vif.type == NL80211_IFTYPE_MONITOR && !(sdata->u.mntr.flags & MONITOR_FLAG_ACTIVE)) @@ -218,6 +276,8 @@ static int ieee80211_change_mac(struct net_device *dev, void *addr) if (ret) return ret; + if (live) + drv_remove_interface(local, sdata); ret = eth_mac_addr(dev, sa); if (ret == 0) { @@ -225,6 +285,12 @@ static int ieee80211_change_mac(struct net_device *dev, void *addr) ether_addr_copy(sdata->vif.bss_conf.addr, sdata->vif.addr); } + /* Regardless of eth_mac_addr() return we still want to add the + * interface back. This should not fail... + */ + if (live) + WARN_ON(drv_add_interface(local, sdata)); + return ret; } @@ -296,6 +362,11 @@ static int ieee80211_check_concurrent_iface(struct ieee80211_sub_if_data *sdata, nsdata->vif.type)) return -ENOTUNIQ; + /* No support for VLAN with MLO yet */ + if (iftype == NL80211_IFTYPE_AP_VLAN && + nsdata->wdev.use_4addr) + return -EOPNOTSUPP; + /* * can only add VLANs to enabled APs */ @@ -368,246 +439,6 @@ static int ieee80211_open(struct net_device *dev) return err; } -static void ieee80211_link_setup(struct ieee80211_link_data *link) -{ - if (link->sdata->vif.type == NL80211_IFTYPE_STATION) - ieee80211_mgd_setup_link(link); -} - -static void ieee80211_link_init(struct ieee80211_sub_if_data *sdata, - int link_id, - struct ieee80211_link_data *link, - struct ieee80211_bss_conf *link_conf) -{ - bool deflink = link_id < 0; - - if (link_id < 0) - link_id = 0; - - rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf); - rcu_assign_pointer(sdata->link[link_id], link); - - link->sdata = sdata; - link->link_id = link_id; - link->conf = link_conf; - link_conf->link_id = link_id; - - INIT_WORK(&link->csa_finalize_work, - ieee80211_csa_finalize_work); - INIT_WORK(&link->color_change_finalize_work, - ieee80211_color_change_finalize_work); - INIT_LIST_HEAD(&link->assigned_chanctx_list); - INIT_LIST_HEAD(&link->reserved_chanctx_list); - INIT_DELAYED_WORK(&link->dfs_cac_timer_work, - ieee80211_dfs_cac_timer_work); - - if (!deflink) { - switch (sdata->vif.type) { - case NL80211_IFTYPE_AP: - ether_addr_copy(link_conf->addr, - sdata->wdev.links[link_id].addr); - WARN_ON(!(sdata->wdev.valid_links & BIT(link_id))); - break; - case NL80211_IFTYPE_STATION: - break; - default: - WARN_ON(1); - } - } -} - -static void ieee80211_link_stop(struct ieee80211_link_data *link) -{ - if (link->sdata->vif.type == NL80211_IFTYPE_STATION) - ieee80211_mgd_stop_link(link); - - ieee80211_link_release_channel(link); -} - -struct link_container { - struct ieee80211_link_data data; - struct ieee80211_bss_conf conf; -}; - -static void ieee80211_free_links(struct ieee80211_sub_if_data *sdata, - struct link_container **links) -{ - unsigned int link_id; - - synchronize_rcu(); - - for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) { - if (!links[link_id]) - continue; - ieee80211_link_stop(&links[link_id]->data); - kfree(links[link_id]); - } -} - -static int ieee80211_check_dup_link_addrs(struct ieee80211_sub_if_data *sdata) -{ - unsigned int i, j; - - for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) { - struct ieee80211_link_data *link1; - - link1 = sdata_dereference(sdata->link[i], sdata); - if (!link1) - continue; - for (j = i + 1; j < IEEE80211_MLD_MAX_NUM_LINKS; j++) { - struct ieee80211_link_data *link2; - - link2 = sdata_dereference(sdata->link[j], sdata); - if (!link2) - continue; - - if (ether_addr_equal(link1->conf->addr, - link2->conf->addr)) - return -EALREADY; - } - } - - return 0; -} - -static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, - struct link_container **to_free, - u16 new_links) -{ - u16 old_links = sdata->vif.valid_links; - unsigned long add = new_links & ~old_links; - unsigned long rem = old_links & ~new_links; - unsigned int link_id; - int ret; - struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link; - struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]; - struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS]; - bool use_deflink = old_links == 0; /* set for error case */ - - sdata_assert_lock(sdata); - - memset(to_free, 0, sizeof(links)); - - if (old_links == new_links) - return 0; - - /* if there were no old links, need to clear the pointers to deflink */ - if (!old_links) - rem |= BIT(0); - - /* allocate new link structures first */ - for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { - link = kzalloc(sizeof(*link), GFP_KERNEL); - if (!link) { - ret = -ENOMEM; - goto free; - } - links[link_id] = link; - } - - /* keep track of the old pointers for the driver */ - BUILD_BUG_ON(sizeof(old) != sizeof(sdata->vif.link_conf)); - memcpy(old, sdata->vif.link_conf, sizeof(old)); - /* and for us in error cases */ - BUILD_BUG_ON(sizeof(old_data) != sizeof(sdata->link)); - memcpy(old_data, sdata->link, sizeof(old_data)); - - /* grab old links to free later */ - for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) { - if (rcu_access_pointer(sdata->link[link_id]) != &sdata->deflink) { - /* - * we must have allocated the data through this path so - * we know we can free both at the same time - */ - to_free[link_id] = container_of(rcu_access_pointer(sdata->link[link_id]), - typeof(*links[link_id]), - data); - } - - RCU_INIT_POINTER(sdata->link[link_id], NULL); - RCU_INIT_POINTER(sdata->vif.link_conf[link_id], NULL); - } - - /* link them into data structures */ - for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { - WARN_ON(!use_deflink && - rcu_access_pointer(sdata->link[link_id]) == &sdata->deflink); - - link = links[link_id]; - ieee80211_link_init(sdata, link_id, &link->data, &link->conf); - ieee80211_link_setup(&link->data); - } - - if (new_links == 0) - ieee80211_link_init(sdata, -1, &sdata->deflink, - &sdata->vif.bss_conf); - - sdata->vif.valid_links = new_links; - - ret = ieee80211_check_dup_link_addrs(sdata); - if (!ret) { - /* tell the driver */ - ret = drv_change_vif_links(sdata->local, sdata, - old_links, new_links, - old); - } - - if (ret) { - /* restore config */ - memcpy(sdata->link, old_data, sizeof(old_data)); - memcpy(sdata->vif.link_conf, old, sizeof(old)); - sdata->vif.valid_links = old_links; - /* and free (only) the newly allocated links */ - memset(to_free, 0, sizeof(links)); - goto free; - } - - /* use deflink/bss_conf again if and only if there are no more links */ - use_deflink = new_links == 0; - - goto deinit; -free: - /* if we failed during allocation, only free all */ - for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) { - kfree(links[link_id]); - links[link_id] = NULL; - } -deinit: - if (use_deflink) - ieee80211_link_init(sdata, -1, &sdata->deflink, - &sdata->vif.bss_conf); - return ret; -} - -int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata, - u16 new_links) -{ - struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS]; - int ret; - - ret = ieee80211_vif_update_links(sdata, links, new_links); - ieee80211_free_links(sdata, links); - - return ret; -} - -static void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata) -{ - struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS]; - - /* - * The locking here is different because when we free links - * in the station case we need to be able to cancel_work_sync() - * something that also takes the lock. - */ - - sdata_lock(sdata); - ieee80211_vif_update_links(sdata, links, 0); - sdata_unlock(sdata); - - ieee80211_free_links(sdata, links); -} - static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_down) { struct ieee80211_local *local = sdata->local; @@ -630,7 +461,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do /* * Stop TX on this interface first. */ - if (sdata->dev) + if (!local->ops->wake_tx_queue && sdata->dev) netif_tx_stop_all_queues(sdata->dev); ieee80211_roc_purge(local, sdata); @@ -923,6 +754,8 @@ static int ieee80211_stop(struct net_device *dev) ieee80211_stop_mbssid(sdata); } + cancel_work_sync(&sdata->activate_links_work); + wiphy_lock(sdata->local->hw.wiphy); ieee80211_do_stop(sdata, true); wiphy_unlock(sdata->local->hw.wiphy); @@ -1579,8 +1412,6 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up) sdata->vif.type != NL80211_IFTYPE_STATION); } - set_bit(SDATA_STATE_RUNNING, &sdata->state); - switch (sdata->vif.type) { case NL80211_IFTYPE_P2P_DEVICE: rcu_assign_pointer(local->p2p_sdata, sdata); @@ -1639,6 +1470,8 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up) spin_unlock_irqrestore(&local->queue_stop_reason_lock, flags); } + set_bit(SDATA_STATE_RUNNING, &sdata->state); + return 0; err_del_interface: drv_remove_interface(local, sdata); @@ -1893,6 +1726,15 @@ static void ieee80211_recalc_smps_work(struct work_struct *work) ieee80211_recalc_smps(sdata, &sdata->deflink); } +static void ieee80211_activate_links_work(struct work_struct *work) +{ + struct ieee80211_sub_if_data *sdata = + container_of(work, struct ieee80211_sub_if_data, + activate_links_work); + + ieee80211_set_active_links(&sdata->vif, sdata->desired_active_links); +} + /* * Helper function to initialise an interface to a specific type. */ @@ -1930,6 +1772,7 @@ static void ieee80211_setup_sdata(struct ieee80211_sub_if_data *sdata, skb_queue_head_init(&sdata->status_queue); INIT_WORK(&sdata->work, ieee80211_iface_work); INIT_WORK(&sdata->recalc_smps, ieee80211_recalc_smps_work); + INIT_WORK(&sdata->activate_links_work, ieee80211_activate_links_work); switch (type) { case NL80211_IFTYPE_P2P_GO: @@ -2267,7 +2110,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name, wdev = &sdata->wdev; sdata->dev = NULL; - strlcpy(sdata->name, name, IFNAMSIZ); + strscpy(sdata->name, name, IFNAMSIZ); ieee80211_assign_perm_addr(local, wdev->address, type); memcpy(sdata->vif.addr, wdev->address, ETH_ALEN); ether_addr_copy(sdata->vif.bss_conf.addr, sdata->vif.addr); @@ -2396,6 +2239,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name, sdata->u.mgd.use_4addr = params->use_4addr; ndev->features |= local->hw.netdev_features; + ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; ndev->hw_features |= ndev->features & MAC80211_SUPPORTED_FEATURES_TX; diff --git a/net/mac80211/key.c b/net/mac80211/key.c index 6befb578ed9e28cf1ef20937ab9906e30f052a0b..e8f6c1e5eabfc76db97255a7622ec67032ef6fa7 100644 --- a/net/mac80211/key.c +++ b/net/mac80211/key.c @@ -177,6 +177,10 @@ static int ieee80211_key_enable_hw_accel(struct ieee80211_key *key) } } + if (key->conf.link_id >= 0 && sdata->vif.active_links && + !(sdata->vif.active_links & BIT(key->conf.link_id))) + return 0; + ret = drv_set_key(key->local, SET_KEY, sdata, sta ? &sta->sta : NULL, &key->conf); @@ -246,6 +250,10 @@ static void ieee80211_key_disable_hw_accel(struct ieee80211_key *key) sta = key->sta; sdata = key->sdata; + if (key->conf.link_id >= 0 && sdata->vif.active_links && + !(sdata->vif.active_links & BIT(key->conf.link_id))) + return; + if (!(key->conf.flags & (IEEE80211_KEY_FLAG_GENERATE_MMIC | IEEE80211_KEY_FLAG_PUT_MIC_SPACE | IEEE80211_KEY_FLAG_RESERVE_TAILROOM))) @@ -344,9 +352,10 @@ static void ieee80211_pairwise_rekey(struct ieee80211_key *old, } } -static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, +static void __ieee80211_set_default_key(struct ieee80211_link_data *link, int idx, bool uni, bool multi) { + struct ieee80211_sub_if_data *sdata = link->sdata; struct ieee80211_key *key = NULL; assert_key_lock(sdata->local); @@ -354,7 +363,7 @@ static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, if (idx >= 0 && idx < NUM_DEFAULT_KEYS) { key = key_mtx_dereference(sdata->local, sdata->keys[idx]); if (!key) - key = key_mtx_dereference(sdata->local, sdata->deflink.gtk[idx]); + key = key_mtx_dereference(sdata->local, link->gtk[idx]); } if (uni) { @@ -365,47 +374,48 @@ static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, } if (multi) - rcu_assign_pointer(sdata->deflink.default_multicast_key, key); + rcu_assign_pointer(link->default_multicast_key, key); ieee80211_debugfs_key_update_default(sdata); } -void ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, int idx, +void ieee80211_set_default_key(struct ieee80211_link_data *link, int idx, bool uni, bool multi) { - mutex_lock(&sdata->local->key_mtx); - __ieee80211_set_default_key(sdata, idx, uni, multi); - mutex_unlock(&sdata->local->key_mtx); + mutex_lock(&link->sdata->local->key_mtx); + __ieee80211_set_default_key(link, idx, uni, multi); + mutex_unlock(&link->sdata->local->key_mtx); } static void -__ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, int idx) +__ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link, int idx) { + struct ieee80211_sub_if_data *sdata = link->sdata; struct ieee80211_key *key = NULL; assert_key_lock(sdata->local); if (idx >= NUM_DEFAULT_KEYS && idx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS) - key = key_mtx_dereference(sdata->local, - sdata->deflink.gtk[idx]); + key = key_mtx_dereference(sdata->local, link->gtk[idx]); - rcu_assign_pointer(sdata->deflink.default_mgmt_key, key); + rcu_assign_pointer(link->default_mgmt_key, key); ieee80211_debugfs_key_update_default(sdata); } -void ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, +void ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link, int idx) { - mutex_lock(&sdata->local->key_mtx); - __ieee80211_set_default_mgmt_key(sdata, idx); - mutex_unlock(&sdata->local->key_mtx); + mutex_lock(&link->sdata->local->key_mtx); + __ieee80211_set_default_mgmt_key(link, idx); + mutex_unlock(&link->sdata->local->key_mtx); } static void -__ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, int idx) +__ieee80211_set_default_beacon_key(struct ieee80211_link_data *link, int idx) { + struct ieee80211_sub_if_data *sdata = link->sdata; struct ieee80211_key *key = NULL; assert_key_lock(sdata->local); @@ -413,28 +423,30 @@ __ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, int idx) if (idx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS && idx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS + NUM_DEFAULT_BEACON_KEYS) - key = key_mtx_dereference(sdata->local, - sdata->deflink.gtk[idx]); + key = key_mtx_dereference(sdata->local, link->gtk[idx]); - rcu_assign_pointer(sdata->deflink.default_beacon_key, key); + rcu_assign_pointer(link->default_beacon_key, key); ieee80211_debugfs_key_update_default(sdata); } -void ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, +void ieee80211_set_default_beacon_key(struct ieee80211_link_data *link, int idx) { - mutex_lock(&sdata->local->key_mtx); - __ieee80211_set_default_beacon_key(sdata, idx); - mutex_unlock(&sdata->local->key_mtx); + mutex_lock(&link->sdata->local->key_mtx); + __ieee80211_set_default_beacon_key(link, idx); + mutex_unlock(&link->sdata->local->key_mtx); } static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, - struct sta_info *sta, - bool pairwise, - struct ieee80211_key *old, - struct ieee80211_key *new) + struct ieee80211_link_data *link, + struct sta_info *sta, + bool pairwise, + struct ieee80211_key *old, + struct ieee80211_key *new) { + struct link_sta_info *link_sta = sta ? &sta->deflink : NULL; + int link_id; int idx; int ret = 0; bool defunikey, defmultikey, defmgmtkey, defbeaconkey; @@ -446,13 +458,36 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, if (new) { idx = new->conf.keyidx; - list_add_tail_rcu(&new->list, &sdata->key_list); is_wep = new->conf.cipher == WLAN_CIPHER_SUITE_WEP40 || new->conf.cipher == WLAN_CIPHER_SUITE_WEP104; + link_id = new->conf.link_id; } else { idx = old->conf.keyidx; is_wep = old->conf.cipher == WLAN_CIPHER_SUITE_WEP40 || old->conf.cipher == WLAN_CIPHER_SUITE_WEP104; + link_id = old->conf.link_id; + } + + if (WARN(old && old->conf.link_id != link_id, + "old link ID %d doesn't match new link ID %d\n", + old->conf.link_id, link_id)) + return -EINVAL; + + if (link_id >= 0) { + if (!link) { + link = sdata_dereference(sdata->link[link_id], sdata); + if (!link) + return -ENOLINK; + } + + if (sta) { + link_sta = rcu_dereference_protected(sta->link[link_id], + lockdep_is_held(&sta->local->sta_mtx)); + if (!link_sta) + return -ENOLINK; + } + } else { + link = &sdata->deflink; } if ((is_wep || pairwise) && idx >= NUM_DEFAULT_KEYS) @@ -482,6 +517,9 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, if (ret) return ret; + if (new) + list_add_tail_rcu(&new->list, &sdata->key_list); + if (sta) { if (pairwise) { rcu_assign_pointer(sta->ptk[idx], new); @@ -489,7 +527,7 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, !(new->conf.flags & IEEE80211_KEY_FLAG_NO_AUTO_TX)) _ieee80211_set_tx_key(new, true); } else { - rcu_assign_pointer(sta->deflink.gtk[idx], new); + rcu_assign_pointer(link_sta->gtk[idx], new); } /* Only needed for transition from no key -> key. * Still triggers unnecessary when using Extended Key ID @@ -503,39 +541,39 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata, sdata->default_unicast_key); defmultikey = old && old == key_mtx_dereference(sdata->local, - sdata->deflink.default_multicast_key); + link->default_multicast_key); defmgmtkey = old && old == key_mtx_dereference(sdata->local, - sdata->deflink.default_mgmt_key); + link->default_mgmt_key); defbeaconkey = old && old == key_mtx_dereference(sdata->local, - sdata->deflink.default_beacon_key); + link->default_beacon_key); if (defunikey && !new) - __ieee80211_set_default_key(sdata, -1, true, false); + __ieee80211_set_default_key(link, -1, true, false); if (defmultikey && !new) - __ieee80211_set_default_key(sdata, -1, false, true); + __ieee80211_set_default_key(link, -1, false, true); if (defmgmtkey && !new) - __ieee80211_set_default_mgmt_key(sdata, -1); + __ieee80211_set_default_mgmt_key(link, -1); if (defbeaconkey && !new) - __ieee80211_set_default_beacon_key(sdata, -1); + __ieee80211_set_default_beacon_key(link, -1); if (is_wep || pairwise) rcu_assign_pointer(sdata->keys[idx], new); else - rcu_assign_pointer(sdata->deflink.gtk[idx], new); + rcu_assign_pointer(link->gtk[idx], new); if (defunikey && new) - __ieee80211_set_default_key(sdata, new->conf.keyidx, + __ieee80211_set_default_key(link, new->conf.keyidx, true, false); if (defmultikey && new) - __ieee80211_set_default_key(sdata, new->conf.keyidx, + __ieee80211_set_default_key(link, new->conf.keyidx, false, true); if (defmgmtkey && new) - __ieee80211_set_default_mgmt_key(sdata, + __ieee80211_set_default_mgmt_key(link, new->conf.keyidx); if (defbeaconkey && new) - __ieee80211_set_default_beacon_key(sdata, + __ieee80211_set_default_beacon_key(link, new->conf.keyidx); } @@ -569,6 +607,7 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len, key->conf.flags = 0; key->flags = 0; + key->conf.link_id = -1; key->conf.cipher = cipher; key->conf.keyidx = idx; key->conf.keylen = key_len; @@ -797,9 +836,10 @@ static bool ieee80211_key_identical(struct ieee80211_sub_if_data *sdata, } int ieee80211_key_link(struct ieee80211_key *key, - struct ieee80211_sub_if_data *sdata, + struct ieee80211_link_data *link, struct sta_info *sta) { + struct ieee80211_sub_if_data *sdata = link->sdata; static atomic_t key_color = ATOMIC_INIT(0); struct ieee80211_key *old_key = NULL; int idx = key->conf.keyidx; @@ -827,15 +867,26 @@ int ieee80211_key_link(struct ieee80211_key *key, (old_key && old_key->conf.cipher != key->conf.cipher)) goto out; } else if (sta) { - old_key = key_mtx_dereference(sdata->local, - sta->deflink.gtk[idx]); + struct link_sta_info *link_sta = &sta->deflink; + int link_id = key->conf.link_id; + + if (link_id >= 0) { + link_sta = rcu_dereference_protected(sta->link[link_id], + lockdep_is_held(&sta->local->sta_mtx)); + if (!link_sta) { + ret = -ENOLINK; + goto out; + } + } + + old_key = key_mtx_dereference(sdata->local, link_sta->gtk[idx]); } else { if (idx < NUM_DEFAULT_KEYS) old_key = key_mtx_dereference(sdata->local, sdata->keys[idx]); if (!old_key) old_key = key_mtx_dereference(sdata->local, - sdata->deflink.gtk[idx]); + link->gtk[idx]); } /* Non-pairwise keys must also not switch the cipher on rekey */ @@ -866,7 +917,7 @@ int ieee80211_key_link(struct ieee80211_key *key, increment_tailroom_need_count(sdata); - ret = ieee80211_key_replace(sdata, sta, pairwise, old_key, key); + ret = ieee80211_key_replace(sdata, link, sta, pairwise, old_key, key); if (!ret) { ieee80211_debugfs_key_add(key); @@ -890,9 +941,9 @@ void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom) * Replace key with nothingness if it was ever used. */ if (key->sdata) - ieee80211_key_replace(key->sdata, key->sta, - key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, - key, NULL); + ieee80211_key_replace(key->sdata, NULL, key->sta, + key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, + key, NULL); ieee80211_key_destroy(key, delay_tailroom); } @@ -1019,15 +1070,45 @@ static void ieee80211_free_keys_iface(struct ieee80211_sub_if_data *sdata, ieee80211_debugfs_key_remove_beacon_default(sdata); list_for_each_entry_safe(key, tmp, &sdata->key_list, list) { - ieee80211_key_replace(key->sdata, key->sta, - key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, - key, NULL); + ieee80211_key_replace(key->sdata, NULL, key->sta, + key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, + key, NULL); list_add_tail(&key->list, keys); } ieee80211_debugfs_key_update_default(sdata); } +void ieee80211_remove_link_keys(struct ieee80211_link_data *link, + struct list_head *keys) +{ + struct ieee80211_sub_if_data *sdata = link->sdata; + struct ieee80211_local *local = sdata->local; + struct ieee80211_key *key, *tmp; + + mutex_lock(&local->key_mtx); + list_for_each_entry_safe(key, tmp, &sdata->key_list, list) { + if (key->conf.link_id != link->link_id) + continue; + ieee80211_key_replace(key->sdata, link, key->sta, + key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, + key, NULL); + list_add_tail(&key->list, keys); + } + mutex_unlock(&local->key_mtx); +} + +void ieee80211_free_key_list(struct ieee80211_local *local, + struct list_head *keys) +{ + struct ieee80211_key *key, *tmp; + + mutex_lock(&local->key_mtx); + list_for_each_entry_safe(key, tmp, keys, list) + __ieee80211_key_destroy(key, false); + mutex_unlock(&local->key_mtx); +} + void ieee80211_free_keys(struct ieee80211_sub_if_data *sdata, bool force_synchronize) { @@ -1087,9 +1168,9 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local, key = key_mtx_dereference(local, sta->deflink.gtk[i]); if (!key) continue; - ieee80211_key_replace(key->sdata, key->sta, - key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, - key, NULL); + ieee80211_key_replace(key->sdata, NULL, key->sta, + key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, + key, NULL); __ieee80211_key_destroy(key, key->sdata->vif.type == NL80211_IFTYPE_STATION); } @@ -1098,9 +1179,9 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local, key = key_mtx_dereference(local, sta->ptk[i]); if (!key) continue; - ieee80211_key_replace(key->sdata, key->sta, - key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, - key, NULL); + ieee80211_key_replace(key->sdata, NULL, key->sta, + key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE, + key, NULL); __ieee80211_key_destroy(key, key->sdata->vif.type == NL80211_IFTYPE_STATION); } @@ -1307,7 +1388,8 @@ ieee80211_gtk_rekey_add(struct ieee80211_vif *vif, if (sdata->u.mgd.mfp != IEEE80211_MFP_DISABLED) key->conf.flags |= IEEE80211_KEY_FLAG_RX_MGMT; - err = ieee80211_key_link(key, sdata, NULL); + /* FIXME: this function needs to get a link ID */ + err = ieee80211_key_link(key, &sdata->deflink, NULL); if (err) return ERR_PTR(err); @@ -1363,3 +1445,37 @@ void ieee80211_key_replay(struct ieee80211_key_conf *keyconf) } } EXPORT_SYMBOL_GPL(ieee80211_key_replay); + +int ieee80211_key_switch_links(struct ieee80211_sub_if_data *sdata, + unsigned long del_links_mask, + unsigned long add_links_mask) +{ + struct ieee80211_key *key; + int ret; + + list_for_each_entry(key, &sdata->key_list, list) { + if (key->conf.link_id < 0 || + !(del_links_mask & BIT(key->conf.link_id))) + continue; + + /* shouldn't happen for per-link keys */ + WARN_ON(key->sta); + + ieee80211_key_disable_hw_accel(key); + } + + list_for_each_entry(key, &sdata->key_list, list) { + if (key->conf.link_id < 0 || + !(add_links_mask & BIT(key->conf.link_id))) + continue; + + /* shouldn't happen for per-link keys */ + WARN_ON(key->sta); + + ret = ieee80211_key_enable_hw_accel(key); + if (ret) + return ret; + } + + return 0; +} diff --git a/net/mac80211/key.h b/net/mac80211/key.h index e994dcea1ce3aed51bbf691fd5aa17ce7b40575d..f3df97df4b72b7b9e6b46907fa91bef64e186d55 100644 --- a/net/mac80211/key.h +++ b/net/mac80211/key.h @@ -22,6 +22,7 @@ struct ieee80211_local; struct ieee80211_sub_if_data; +struct ieee80211_link_data; struct sta_info; /** @@ -144,22 +145,29 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len, * to make it used, free old key. On failure, also free the new key. */ int ieee80211_key_link(struct ieee80211_key *key, - struct ieee80211_sub_if_data *sdata, + struct ieee80211_link_data *link, struct sta_info *sta); int ieee80211_set_tx_key(struct ieee80211_key *key); void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom); void ieee80211_key_free_unused(struct ieee80211_key *key); -void ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, int idx, +void ieee80211_set_default_key(struct ieee80211_link_data *link, int idx, bool uni, bool multi); -void ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, +void ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link, int idx); -void ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, +void ieee80211_set_default_beacon_key(struct ieee80211_link_data *link, int idx); +void ieee80211_remove_link_keys(struct ieee80211_link_data *link, + struct list_head *keys); +void ieee80211_free_key_list(struct ieee80211_local *local, + struct list_head *keys); void ieee80211_free_keys(struct ieee80211_sub_if_data *sdata, bool force_synchronize); void ieee80211_free_sta_keys(struct ieee80211_local *local, struct sta_info *sta); void ieee80211_reenable_keys(struct ieee80211_sub_if_data *sdata); +int ieee80211_key_switch_links(struct ieee80211_sub_if_data *sdata, + unsigned long del_links_mask, + unsigned long add_links_mask); #define key_mtx_dereference(local, ref) \ rcu_dereference_protected(ref, lockdep_is_held(&((local)->key_mtx))) diff --git a/net/mac80211/link.c b/net/mac80211/link.c new file mode 100644 index 0000000000000000000000000000000000000000..e309708abae8b2e708b2608d26e11b86c0476ec2 --- /dev/null +++ b/net/mac80211/link.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * MLO link handling + * + * Copyright (C) 2022 Intel Corporation + */ +#include +#include +#include +#include "ieee80211_i.h" +#include "driver-ops.h" +#include "key.h" + +void ieee80211_link_setup(struct ieee80211_link_data *link) +{ + if (link->sdata->vif.type == NL80211_IFTYPE_STATION) + ieee80211_mgd_setup_link(link); +} + +void ieee80211_link_init(struct ieee80211_sub_if_data *sdata, + int link_id, + struct ieee80211_link_data *link, + struct ieee80211_bss_conf *link_conf) +{ + bool deflink = link_id < 0; + + if (link_id < 0) + link_id = 0; + + rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf); + rcu_assign_pointer(sdata->link[link_id], link); + + link->sdata = sdata; + link->link_id = link_id; + link->conf = link_conf; + link_conf->link_id = link_id; + + INIT_WORK(&link->csa_finalize_work, + ieee80211_csa_finalize_work); + INIT_WORK(&link->color_change_finalize_work, + ieee80211_color_change_finalize_work); + INIT_LIST_HEAD(&link->assigned_chanctx_list); + INIT_LIST_HEAD(&link->reserved_chanctx_list); + INIT_DELAYED_WORK(&link->dfs_cac_timer_work, + ieee80211_dfs_cac_timer_work); + + if (!deflink) { + switch (sdata->vif.type) { + case NL80211_IFTYPE_AP: + ether_addr_copy(link_conf->addr, + sdata->wdev.links[link_id].addr); + link_conf->bssid = link_conf->addr; + WARN_ON(!(sdata->wdev.valid_links & BIT(link_id))); + break; + case NL80211_IFTYPE_STATION: + /* station sets the bssid in ieee80211_mgd_setup_link */ + break; + default: + WARN_ON(1); + } + } +} + +void ieee80211_link_stop(struct ieee80211_link_data *link) +{ + if (link->sdata->vif.type == NL80211_IFTYPE_STATION) + ieee80211_mgd_stop_link(link); + + ieee80211_link_release_channel(link); +} + +struct link_container { + struct ieee80211_link_data data; + struct ieee80211_bss_conf conf; +}; + +static void ieee80211_tear_down_links(struct ieee80211_sub_if_data *sdata, + struct link_container **links, u16 mask) +{ + struct ieee80211_link_data *link; + LIST_HEAD(keys); + unsigned int link_id; + + for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) { + if (!(mask & BIT(link_id))) + continue; + link = &links[link_id]->data; + if (link_id == 0 && !link) + link = &sdata->deflink; + if (WARN_ON(!link)) + continue; + ieee80211_remove_link_keys(link, &keys); + ieee80211_link_stop(link); + } + + synchronize_rcu(); + + ieee80211_free_key_list(sdata->local, &keys); +} + +static void ieee80211_free_links(struct ieee80211_sub_if_data *sdata, + struct link_container **links) +{ + unsigned int link_id; + + for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) + kfree(links[link_id]); +} + +static int ieee80211_check_dup_link_addrs(struct ieee80211_sub_if_data *sdata) +{ + unsigned int i, j; + + for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) { + struct ieee80211_link_data *link1; + + link1 = sdata_dereference(sdata->link[i], sdata); + if (!link1) + continue; + for (j = i + 1; j < IEEE80211_MLD_MAX_NUM_LINKS; j++) { + struct ieee80211_link_data *link2; + + link2 = sdata_dereference(sdata->link[j], sdata); + if (!link2) + continue; + + if (ether_addr_equal(link1->conf->addr, + link2->conf->addr)) + return -EALREADY; + } + } + + return 0; +} + +static void ieee80211_set_vif_links_bitmaps(struct ieee80211_sub_if_data *sdata, + u16 links) +{ + sdata->vif.valid_links = links; + + if (!links) { + sdata->vif.active_links = 0; + return; + } + + switch (sdata->vif.type) { + case NL80211_IFTYPE_AP: + /* in an AP all links are always active */ + sdata->vif.active_links = links; + break; + case NL80211_IFTYPE_STATION: + if (sdata->vif.active_links) + break; + WARN_ON(hweight16(links) > 1); + sdata->vif.active_links = links; + break; + default: + WARN_ON(1); + } +} + +static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, + struct link_container **to_free, + u16 new_links) +{ + u16 old_links = sdata->vif.valid_links; + u16 old_active = sdata->vif.active_links; + unsigned long add = new_links & ~old_links; + unsigned long rem = old_links & ~new_links; + unsigned int link_id; + int ret; + struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link; + struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]; + struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS]; + bool use_deflink = old_links == 0; /* set for error case */ + + sdata_assert_lock(sdata); + + memset(to_free, 0, sizeof(links)); + + if (old_links == new_links) + return 0; + + /* if there were no old links, need to clear the pointers to deflink */ + if (!old_links) + rem |= BIT(0); + + /* allocate new link structures first */ + for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link) { + ret = -ENOMEM; + goto free; + } + links[link_id] = link; + } + + /* keep track of the old pointers for the driver */ + BUILD_BUG_ON(sizeof(old) != sizeof(sdata->vif.link_conf)); + memcpy(old, sdata->vif.link_conf, sizeof(old)); + /* and for us in error cases */ + BUILD_BUG_ON(sizeof(old_data) != sizeof(sdata->link)); + memcpy(old_data, sdata->link, sizeof(old_data)); + + /* grab old links to free later */ + for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) { + if (rcu_access_pointer(sdata->link[link_id]) != &sdata->deflink) { + /* + * we must have allocated the data through this path so + * we know we can free both at the same time + */ + to_free[link_id] = container_of(rcu_access_pointer(sdata->link[link_id]), + typeof(*links[link_id]), + data); + } + + RCU_INIT_POINTER(sdata->link[link_id], NULL); + RCU_INIT_POINTER(sdata->vif.link_conf[link_id], NULL); + } + + /* link them into data structures */ + for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { + WARN_ON(!use_deflink && + rcu_access_pointer(sdata->link[link_id]) == &sdata->deflink); + + link = links[link_id]; + ieee80211_link_init(sdata, link_id, &link->data, &link->conf); + ieee80211_link_setup(&link->data); + } + + if (new_links == 0) + ieee80211_link_init(sdata, -1, &sdata->deflink, + &sdata->vif.bss_conf); + + ret = ieee80211_check_dup_link_addrs(sdata); + if (!ret) { + /* for keys we will not be able to undo this */ + ieee80211_tear_down_links(sdata, to_free, rem); + + ieee80211_set_vif_links_bitmaps(sdata, new_links); + + /* tell the driver */ + ret = drv_change_vif_links(sdata->local, sdata, + old_links & old_active, + new_links & sdata->vif.active_links, + old); + } + + if (ret) { + /* restore config */ + memcpy(sdata->link, old_data, sizeof(old_data)); + memcpy(sdata->vif.link_conf, old, sizeof(old)); + ieee80211_set_vif_links_bitmaps(sdata, old_links); + /* and free (only) the newly allocated links */ + memset(to_free, 0, sizeof(links)); + goto free; + } + + /* use deflink/bss_conf again if and only if there are no more links */ + use_deflink = new_links == 0; + + goto deinit; +free: + /* if we failed during allocation, only free all */ + for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) { + kfree(links[link_id]); + links[link_id] = NULL; + } +deinit: + if (use_deflink) + ieee80211_link_init(sdata, -1, &sdata->deflink, + &sdata->vif.bss_conf); + return ret; +} + +int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata, + u16 new_links) +{ + struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS]; + int ret; + + ret = ieee80211_vif_update_links(sdata, links, new_links); + ieee80211_free_links(sdata, links); + + return ret; +} + +void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata) +{ + struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS]; + + /* + * The locking here is different because when we free links + * in the station case we need to be able to cancel_work_sync() + * something that also takes the lock. + */ + + sdata_lock(sdata); + ieee80211_vif_update_links(sdata, links, 0); + sdata_unlock(sdata); + + ieee80211_free_links(sdata, links); +} + +static int _ieee80211_set_active_links(struct ieee80211_sub_if_data *sdata, + u16 active_links) +{ + struct ieee80211_bss_conf *link_confs[IEEE80211_MLD_MAX_NUM_LINKS]; + struct ieee80211_local *local = sdata->local; + u16 old_active = sdata->vif.active_links; + unsigned long rem = old_active & ~active_links; + unsigned long add = active_links & ~old_active; + struct sta_info *sta; + unsigned int link_id; + int ret, i; + + if (!ieee80211_sdata_running(sdata)) + return -ENETDOWN; + + if (sdata->vif.type != NL80211_IFTYPE_STATION) + return -EINVAL; + + /* cannot activate links that don't exist */ + if (active_links & ~sdata->vif.valid_links) + return -EINVAL; + + /* nothing to do */ + if (old_active == active_links) + return 0; + + for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) + link_confs[i] = sdata_dereference(sdata->vif.link_conf[i], + sdata); + + if (add) { + sdata->vif.active_links |= active_links; + ret = drv_change_vif_links(local, sdata, + old_active, + sdata->vif.active_links, + link_confs); + if (ret) { + sdata->vif.active_links = old_active; + return ret; + } + } + + for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) { + struct ieee80211_link_data *link; + + link = sdata_dereference(sdata->link[link_id], sdata); + + /* FIXME: kill TDLS connections on the link */ + + ieee80211_link_release_channel(link); + } + + list_for_each_entry(sta, &local->sta_list, list) { + if (sdata != sta->sdata) + continue; + ret = drv_change_sta_links(local, sdata, &sta->sta, + old_active, + old_active | active_links); + WARN_ON_ONCE(ret); + } + + ret = ieee80211_key_switch_links(sdata, rem, add); + WARN_ON_ONCE(ret); + + list_for_each_entry(sta, &local->sta_list, list) { + if (sdata != sta->sdata) + continue; + ret = drv_change_sta_links(local, sdata, &sta->sta, + old_active | active_links, + active_links); + WARN_ON_ONCE(ret); + } + + for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { + struct ieee80211_link_data *link; + + link = sdata_dereference(sdata->link[link_id], sdata); + + ret = ieee80211_link_use_channel(link, &link->conf->chandef, + IEEE80211_CHANCTX_SHARED); + WARN_ON_ONCE(ret); + + ieee80211_link_info_change_notify(sdata, link, + BSS_CHANGED_ERP_CTS_PROT | + BSS_CHANGED_ERP_PREAMBLE | + BSS_CHANGED_ERP_SLOT | + BSS_CHANGED_HT | + BSS_CHANGED_BASIC_RATES | + BSS_CHANGED_BSSID | + BSS_CHANGED_CQM | + BSS_CHANGED_QOS | + BSS_CHANGED_TXPOWER | + BSS_CHANGED_BANDWIDTH | + BSS_CHANGED_TWT | + BSS_CHANGED_HE_OBSS_PD | + BSS_CHANGED_HE_BSS_COLOR); + ieee80211_mgd_set_link_qos_params(link); + } + + old_active = sdata->vif.active_links; + sdata->vif.active_links = active_links; + + if (rem) { + ret = drv_change_vif_links(local, sdata, old_active, + active_links, link_confs); + WARN_ON_ONCE(ret); + } + + return 0; +} + +int ieee80211_set_active_links(struct ieee80211_vif *vif, u16 active_links) +{ + struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif); + struct ieee80211_local *local = sdata->local; + u16 old_active; + int ret; + + sdata_lock(sdata); + mutex_lock(&local->sta_mtx); + mutex_lock(&local->mtx); + mutex_lock(&local->key_mtx); + old_active = sdata->vif.active_links; + if (old_active & active_links) { + /* + * if there's at least one link that stays active across + * the change then switch to it (to those) first, and + * then enable the additional links + */ + ret = _ieee80211_set_active_links(sdata, + old_active & active_links); + if (!ret) + ret = _ieee80211_set_active_links(sdata, active_links); + } else { + /* otherwise switch directly */ + ret = _ieee80211_set_active_links(sdata, active_links); + } + mutex_unlock(&local->key_mtx); + mutex_unlock(&local->mtx); + mutex_unlock(&local->sta_mtx); + sdata_unlock(sdata); + + return ret; +} +EXPORT_SYMBOL_GPL(ieee80211_set_active_links); + +void ieee80211_set_active_links_async(struct ieee80211_vif *vif, + u16 active_links) +{ + struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif); + + if (!ieee80211_sdata_running(sdata)) + return; + + if (sdata->vif.type != NL80211_IFTYPE_STATION) + return; + + /* cannot activate links that don't exist */ + if (active_links & ~sdata->vif.valid_links) + return; + + /* nothing to do */ + if (sdata->vif.active_links == active_links) + return; + + sdata->desired_active_links = active_links; + schedule_work(&sdata->activate_links_work); +} +EXPORT_SYMBOL_GPL(ieee80211_set_active_links_async); diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 5b1c47ed0cc08a260713977df15db1042ed3db1f..46f3eddc23887c78598a5349191419a3993acd8b 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -699,6 +699,8 @@ struct ieee80211_hw *ieee80211_alloc_hw_nm(size_t priv_data_len, NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211_TX_STATUS); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SCAN_FREQ_KHZ); + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE); if (!ops->hw_scan) { wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN | diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c index 6991c4c479daf1ddd9fb0a2504d6cc3768a9e730..5a99b8f6e465ff6b5f29907c839f4eaabae16152 100644 --- a/net/mac80211/mesh.c +++ b/net/mac80211/mesh.c @@ -634,7 +634,7 @@ int mesh_add_he_6ghz_cap_ie(struct ieee80211_sub_if_data *sdata, if (!iftd) return 0; - ieee80211_ie_build_he_6ghz_cap(sdata, skb); + ieee80211_ie_build_he_6ghz_cap(sdata, sdata->deflink.smps_mode, skb); return 0; } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 3d4ab711f0d1ebd5d4584d36e23909b3b894bdd1..d8484cd870de58dac322714481a6fd97700b6101 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -314,7 +314,7 @@ ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata, if (eht_oper && (eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) { struct cfg80211_chan_def eht_chandef = *chandef; - ieee80211_chandef_eht_oper(sdata, eht_oper, + ieee80211_chandef_eht_oper(eht_oper, eht_chandef.width == NL80211_CHAN_WIDTH_160, false, &eht_chandef); @@ -695,6 +695,7 @@ static bool ieee80211_add_vht_ie(struct ieee80211_sub_if_data *sdata, static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb, struct ieee80211_supported_band *sband, + enum ieee80211_smps_mode smps_mode, ieee80211_conn_flags_t conn_flags) { u8 *pos, *pre_he_pos; @@ -719,7 +720,7 @@ static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata, /* trim excess if any */ skb_trim(skb, skb->len - (pre_he_pos + he_cap_size - pos)); - ieee80211_ie_build_he_6ghz_cap(sdata, skb); + ieee80211_ie_build_he_6ghz_cap(sdata, smps_mode, skb); } static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata, @@ -746,11 +747,13 @@ static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata, eht_cap_size = 2 + 1 + sizeof(eht_cap->eht_cap_elem) + ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, - &eht_cap->eht_cap_elem) + + &eht_cap->eht_cap_elem, + false) + ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], eht_cap->eht_cap_elem.phy_cap_info); pos = skb_put(skb, eht_cap_size); - ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + eht_cap_size); + ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + eht_cap_size, + false); } static void ieee80211_assoc_add_rates(struct sk_buff *skb, @@ -1098,7 +1101,7 @@ static size_t ieee80211_assoc_link_elems(struct ieee80211_sub_if_data *sdata, offset); if (!(assoc_data->link[link_id].conn_flags & IEEE80211_CONN_DISABLE_HE)) { - ieee80211_add_he_ie(sdata, skb, sband, + ieee80211_add_he_ie(sdata, skb, sband, smps_mode, assoc_data->link[link_id].conn_flags); ADD_PRESENT_EXT_ELEM(WLAN_EID_EXT_HE_CAPABILITY); } @@ -1220,14 +1223,21 @@ static void ieee80211_assoc_add_ml_elem(struct ieee80211_sub_if_data *sdata, ml_elem = skb_put(skb, sizeof(*ml_elem)); ml_elem->control = cpu_to_le16(IEEE80211_ML_CONTROL_TYPE_BASIC | - IEEE80211_MLC_BASIC_PRES_EML_CAPA | IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP); common = skb_put(skb, sizeof(*common)); common->len = sizeof(*common) + - 2 + /* EML capabilities */ 2; /* MLD capa/ops */ memcpy(common->mld_mac_addr, sdata->vif.addr, ETH_ALEN); - skb_put_data(skb, &eml_capa, sizeof(eml_capa)); + + /* add EML_CAPA only if needed, see Draft P802.11be_D2.1, 35.3.17 */ + if (eml_capa & + cpu_to_le16((IEEE80211_EML_CAP_EMLSR_SUPP | + IEEE80211_EML_CAP_EMLMR_SUPPORT))) { + common->len += 2; /* EML capabilities */ + ml_elem->control |= + cpu_to_le16(IEEE80211_MLC_BASIC_PRES_EML_CAPA); + skb_put_data(skb, &eml_capa, sizeof(eml_capa)); + } /* need indication from userspace to support this */ mld_capa_ops &= ~cpu_to_le16(IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP); skb_put_data(skb, &mld_capa_ops, sizeof(mld_capa_ops)); @@ -1536,8 +1546,9 @@ void ieee80211_send_nullfunc(struct ieee80211_local *local, struct ieee80211_hdr_3addr *nullfunc; struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; - skb = ieee80211_nullfunc_get(&local->hw, &sdata->vif, - !ieee80211_hw_check(&local->hw, DOESNT_SUPPORT_QOS_NDP)); + skb = ieee80211_nullfunc_get(&local->hw, &sdata->vif, -1, + !ieee80211_hw_check(&local->hw, + DOESNT_SUPPORT_QOS_NDP)); if (!skb) return; @@ -1902,7 +1913,7 @@ ieee80211_sta_process_chanswitch(struct ieee80211_link_data *link, IEEE80211_QUEUE_STOP_REASON_CSA); mutex_unlock(&local->mtx); - cfg80211_ch_switch_started_notify(sdata->dev, &csa_ie.chandef, + cfg80211_ch_switch_started_notify(sdata->dev, &csa_ie.chandef, 0, csa_ie.count, csa_ie.mode); if (local->ops->channel_switch) { @@ -2435,6 +2446,29 @@ static void ieee80211_sta_handle_tspec_ac_params_wk(struct work_struct *work) ieee80211_sta_handle_tspec_ac_params(sdata); } +void ieee80211_mgd_set_link_qos_params(struct ieee80211_link_data *link) +{ + struct ieee80211_sub_if_data *sdata = link->sdata; + struct ieee80211_local *local = sdata->local; + struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; + struct ieee80211_tx_queue_params *params = link->tx_conf; + u8 ac; + + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { + mlme_dbg(sdata, + "WMM AC=%d acm=%d aifs=%d cWmin=%d cWmax=%d txop=%d uapsd=%d, downgraded=%d\n", + ac, params[ac].acm, + params[ac].aifs, params[ac].cw_min, params[ac].cw_max, + params[ac].txop, params[ac].uapsd, + ifmgd->tx_tspec[ac].downgraded); + if (!ifmgd->tx_tspec[ac].downgraded && + drv_conf_tx(local, link, ac, ¶ms[ac])) + link_err(link, + "failed to set TX queue parameters for AC %d\n", + ac); + } +} + /* MLME */ static bool ieee80211_sta_wmm_params(struct ieee80211_local *local, @@ -2566,20 +2600,10 @@ ieee80211_sta_wmm_params(struct ieee80211_local *local, } } - for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) { - mlme_dbg(sdata, - "WMM AC=%d acm=%d aifs=%d cWmin=%d cWmax=%d txop=%d uapsd=%d, downgraded=%d\n", - ac, params[ac].acm, - params[ac].aifs, params[ac].cw_min, params[ac].cw_max, - params[ac].txop, params[ac].uapsd, - ifmgd->tx_tspec[ac].downgraded); + for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) link->tx_conf[ac] = params[ac]; - if (!ifmgd->tx_tspec[ac].downgraded && - drv_conf_tx(local, link, ac, ¶ms[ac])) - link_err(link, - "failed to set TX queue parameters for AC %d\n", - ac); - } + + ieee80211_mgd_set_link_qos_params(link); /* enable WMM or activate new settings */ link->conf->qos = true; @@ -3420,11 +3444,11 @@ static void ieee80211_destroy_auth_data(struct ieee80211_sub_if_data *sdata, ieee80211_link_info_change_notify(sdata, &sdata->deflink, BSS_CHANGED_BSSID); sdata->u.mgd.flags = 0; + mutex_lock(&sdata->local->mtx); ieee80211_link_release_channel(&sdata->deflink); - mutex_unlock(&sdata->local->mtx); - ieee80211_vif_set_links(sdata, 0); + mutex_unlock(&sdata->local->mtx); } cfg80211_put_bss(sdata->local->hw.wiphy, auth_data->bss); @@ -3462,10 +3486,6 @@ static void ieee80211_destroy_assoc_data(struct ieee80211_sub_if_data *sdata, sdata->u.mgd.flags = 0; sdata->vif.bss_conf.mu_mimo_owner = false; - mutex_lock(&sdata->local->mtx); - ieee80211_link_release_channel(&sdata->deflink); - mutex_unlock(&sdata->local->mtx); - if (status != ASSOC_REJECTED) { struct cfg80211_assoc_failure data = { .timeout = status == ASSOC_TIMEOUT, @@ -3484,7 +3504,10 @@ static void ieee80211_destroy_assoc_data(struct ieee80211_sub_if_data *sdata, cfg80211_assoc_failure(sdata->dev, &data); } + mutex_lock(&sdata->local->mtx); + ieee80211_link_release_channel(&sdata->deflink); ieee80211_vif_set_links(sdata, 0); + mutex_unlock(&sdata->local->mtx); } kfree(assoc_data); @@ -3905,6 +3928,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, .len = elem_len, .bss = cbss, .link_id = link == &sdata->deflink ? -1 : link->link_id, + .from_ap = true, }; bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ; bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ; @@ -4033,15 +4057,14 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link, goto out; } - sband = ieee80211_get_link_sband(link); - if (!sband) { + if (WARN_ON(!link->conf->chandef.chan)) { ret = false; goto out; } + sband = local->hw.wiphy->bands[link->conf->chandef.chan->band]; if (!(link->u.mgd.conn_flags & IEEE80211_CONN_DISABLE_HE) && (!elems->he_cap || !elems->he_operation)) { - mutex_unlock(&sdata->local->sta_mtx); sdata_info(sdata, "HE AP is missing HE capability/operation\n"); ret = false; @@ -4386,8 +4409,11 @@ ieee80211_verify_peer_he_mcs_support(struct ieee80211_sub_if_data *sdata, he_cap_elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_CAPABILITY, ies->data, ies->len); + if (!he_cap_elem) + return false; + /* invalid HE IE */ - if (!he_cap_elem || he_cap_elem->datalen < 1 + sizeof(*he_cap)) { + if (he_cap_elem->datalen < 1 + sizeof(*he_cap)) { sdata_info(sdata, "Invalid HE elem, Disable HE\n"); return false; @@ -4573,6 +4599,11 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ; bool is_5ghz = cbss->channel->band == NL80211_BAND_5GHZ; struct ieee80211_bss *bss = (void *)cbss->priv; + struct ieee80211_elems_parse_params parse_params = { + .bss = cbss, + .link_id = -1, + .from_ap = true, + }; struct ieee802_11_elems *elems; const struct cfg80211_bss_ies *ies; int ret; @@ -4582,7 +4613,9 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, rcu_read_lock(); ies = rcu_dereference(cbss->ies); - elems = ieee802_11_parse_elems(ies->data, ies->len, false, cbss); + parse_params.start = ies->data; + parse_params.len = ies->len; + elems = ieee802_11_parse_elems_full(&parse_params); if (!elems) { rcu_read_unlock(); return -ENOMEM; @@ -4646,8 +4679,6 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, } if (!elems->vht_cap_elem) { - sdata_info(sdata, - "bad VHT capabilities, disabling VHT\n"); *conn_flags |= IEEE80211_CONN_DISABLE_VHT; vht_oper = NULL; } @@ -4788,6 +4819,40 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata, return ret; } +static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies, + u8 *dtim_count, u8 *dtim_period) +{ + const u8 *tim_ie = cfg80211_find_ie(WLAN_EID_TIM, ies->data, ies->len); + const u8 *idx_ie = cfg80211_find_ie(WLAN_EID_MULTI_BSSID_IDX, ies->data, + ies->len); + const struct ieee80211_tim_ie *tim = NULL; + const struct ieee80211_bssid_index *idx; + bool valid = tim_ie && tim_ie[1] >= 2; + + if (valid) + tim = (void *)(tim_ie + 2); + + if (dtim_count) + *dtim_count = valid ? tim->dtim_count : 0; + + if (dtim_period) + *dtim_period = valid ? tim->dtim_period : 0; + + /* Check if value is overridden by non-transmitted profile */ + if (!idx_ie || idx_ie[1] < 3) + return valid; + + idx = (void *)(idx_ie + 2); + + if (dtim_count) + *dtim_count = idx->dtim_count; + + if (dtim_period) + *dtim_period = idx->dtim_period; + + return true; +} + static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, struct ieee802_11_elems *elems, @@ -4851,11 +4916,24 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata, goto out_err; if (link_id != assoc_data->assoc_link_id) { - err = ieee80211_prep_channel(sdata, link, - assoc_data->link[link_id].bss, + struct cfg80211_bss *cbss = assoc_data->link[link_id].bss; + const struct cfg80211_bss_ies *ies; + + rcu_read_lock(); + ies = rcu_dereference(cbss->ies); + ieee80211_get_dtim(ies, + &link->conf->sync_dtim_count, + &link->u.mgd.dtim_period); + link->conf->dtim_period = link->u.mgd.dtim_period ?: 1; + link->conf->beacon_int = cbss->beacon_interval; + rcu_read_unlock(); + + err = ieee80211_prep_channel(sdata, link, cbss, &link->u.mgd.conn_flags); - if (err) + if (err) { + link_info(link, "prep_channel failed\n"); goto out_err; + } } err = ieee80211_mgd_setup_link_sta(link, sta, link_sta, @@ -4937,6 +5015,11 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data; u16 capab_info, status_code, aid; + struct ieee80211_elems_parse_params parse_params = { + .bss = NULL, + .link_id = -1, + .from_ap = true, + }; struct ieee802_11_elems *elems; int ac; const u8 *elem_start; @@ -4991,7 +5074,9 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, return; elem_len = len - (elem_start - (u8 *)mgmt); - elems = ieee802_11_parse_elems(elem_start, elem_len, false, NULL); + parse_params.start = elem_start; + parse_params.len = elem_len; + elems = ieee802_11_parse_elems_full(&parse_params); if (!elems) goto notify_driver; @@ -5124,7 +5209,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, resp.req_ies = ifmgd->assoc_req_ies; resp.req_ies_len = ifmgd->assoc_req_ies_len; if (sdata->vif.valid_links) - resp.ap_mld_addr = assoc_data->ap_addr; + resp.ap_mld_addr = sdata->vif.cfg.ap_addr; cfg80211_rx_assoc_resp(sdata->dev, &resp); notify_driver: drv_mgd_complete_tx(sdata->local, sdata, &info); @@ -5356,6 +5441,10 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, u32 ncrc = 0; u8 *bssid, *variable = mgmt->u.beacon.variable; u8 deauth_buf[IEEE80211_DEAUTH_FRAME_LEN]; + struct ieee80211_elems_parse_params parse_params = { + .link_id = -1, + .from_ap = true, + }; sdata_assert_lock(sdata); @@ -5374,6 +5463,9 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, if (baselen > len) return; + parse_params.start = variable; + parse_params.len = len - baselen; + rcu_read_lock(); chanctx_conf = rcu_dereference(link->conf->chanctx_conf); if (!chanctx_conf) { @@ -5392,8 +5484,8 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, if (ifmgd->assoc_data && ifmgd->assoc_data->need_beacon && !WARN_ON(sdata->vif.valid_links) && ieee80211_rx_our_beacon(bssid, ifmgd->assoc_data->link[0].bss)) { - elems = ieee802_11_parse_elems(variable, len - baselen, false, - ifmgd->assoc_data->link[0].bss); + parse_params.bss = ifmgd->assoc_data->link[0].bss; + elems = ieee802_11_parse_elems_full(&parse_params); if (!elems) return; @@ -5459,9 +5551,10 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, */ if (!ieee80211_is_s1g_beacon(hdr->frame_control)) ncrc = crc32_be(0, (void *)&mgmt->u.beacon.beacon_int, 4); - elems = ieee802_11_parse_elems_crc(variable, len - baselen, - false, care_about_ies, ncrc, - link->u.mgd.bss); + parse_params.bss = link->u.mgd.bss; + parse_params.filter = care_about_ies; + parse_params.crc = ncrc; + elems = ieee802_11_parse_elems_full(&parse_params); if (!elems) return; ncrc = elems->crc; @@ -5590,12 +5683,16 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, mutex_lock(&local->sta_mtx); sta = sta_info_get(sdata, sdata->vif.cfg.ap_addr); - if (WARN_ON(!sta)) + if (WARN_ON(!sta)) { + mutex_unlock(&local->sta_mtx); goto free; + } link_sta = rcu_dereference_protected(sta->link[link->link_id], lockdep_is_held(&local->sta_mtx)); - if (WARN_ON(!link_sta)) + if (WARN_ON(!link_sta)) { + mutex_unlock(&local->sta_mtx); goto free; + } changed |= ieee80211_recalc_twt_req(link, link_sta, elems); @@ -5671,6 +5768,13 @@ void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata, sdata_lock(sdata); + if (rx_status->link_valid) { + link = sdata_dereference(sdata->link[rx_status->link_id], + sdata); + if (!link) + goto out; + } + switch (fc & IEEE80211_FCTL_STYPE) { case IEEE80211_STYPE_BEACON: ieee80211_rx_mgmt_beacon(link, (void *)mgmt, @@ -5747,6 +5851,7 @@ void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata, } break; } +out: sdata_unlock(sdata); } @@ -6282,6 +6387,8 @@ void ieee80211_mgd_setup_link(struct ieee80211_link_data *link) if (sdata->u.mgd.assoc_data) ether_addr_copy(link->conf->addr, sdata->u.mgd.assoc_data->link[link_id].addr); + else if (!is_valid_ether_addr(link->conf->addr)) + eth_random_addr(link->conf->addr); } /* scan finished notification */ @@ -6298,40 +6405,6 @@ void ieee80211_mlme_notify_scan_completed(struct ieee80211_local *local) rcu_read_unlock(); } -static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies, - u8 *dtim_count, u8 *dtim_period) -{ - const u8 *tim_ie = cfg80211_find_ie(WLAN_EID_TIM, ies->data, ies->len); - const u8 *idx_ie = cfg80211_find_ie(WLAN_EID_MULTI_BSSID_IDX, ies->data, - ies->len); - const struct ieee80211_tim_ie *tim = NULL; - const struct ieee80211_bssid_index *idx; - bool valid = tim_ie && tim_ie[1] >= 2; - - if (valid) - tim = (void *)(tim_ie + 2); - - if (dtim_count) - *dtim_count = valid ? tim->dtim_count : 0; - - if (dtim_period) - *dtim_period = valid ? tim->dtim_period : 0; - - /* Check if value is overridden by non-transmitted profile */ - if (!idx_ie || idx_ie[1] < 3) - return valid; - - idx = (void *)(idx_ie + 2); - - if (dtim_count) - *dtim_count = idx->dtim_count; - - if (dtim_period) - *dtim_period = idx->dtim_period; - - return true; -} - static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, struct cfg80211_bss *cbss, s8 link_id, const u8 *ap_mld_addr, bool assoc, @@ -6369,9 +6442,6 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, goto out_err; } - if (mlo && !is_valid_ether_addr(link->conf->addr)) - eth_random_addr(link->conf->addr); - if (WARN_ON(!ifmgd->auth_data && !ifmgd->assoc_data)) { err = -EINVAL; goto out_err; @@ -6509,6 +6579,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, return 0; out_err: + ieee80211_link_release_channel(&sdata->deflink); ieee80211_vif_set_links(sdata, 0); return err; } @@ -6836,22 +6907,9 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata, for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) size += req->links[i].elems_len; - if (req->ap_mld_addr) { - for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) { - if (!req->links[i].bss) - continue; - if (i == assoc_link_id) - continue; - /* - * For now, support only a single link in MLO, we - * don't have the necessary parsing of the multi- - * link element in the association response, etc. - */ - sdata_info(sdata, - "refusing MLO association with >1 links\n"); - return -EINVAL; - } - } + /* FIXME: no support for 4-addr MLO yet */ + if (sdata->u.mgd.use_4addr && req->link_id >= 0) + return -EOPNOTSUPP; assoc_data = kzalloc(size, GFP_KERNEL); if (!assoc_data) diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c index 5f27e6746762a0456ed244bbff68c9953235c710..3d91b98db0996a2e1ec6d57fb035fc2988a94b09 100644 --- a/net/mac80211/rc80211_minstrel_ht.c +++ b/net/mac80211/rc80211_minstrel_ht.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2010-2013 Felix Fietkau - * Copyright (C) 2019-2021 Intel Corporation + * Copyright (C) 2019-2022 Intel Corporation */ #include #include @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "rate.h" #include "sta_info.h" @@ -1478,7 +1479,7 @@ minstrel_ht_set_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi, * - for fallback rates, to increase chances of getting through */ if (offset > 0 || - (mi->sta->smps_mode == IEEE80211_SMPS_DYNAMIC && + (mi->sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC && group->streams > 1)) { ratetbl->rate[offset].count = ratetbl->rate[offset].count_rts; flags |= IEEE80211_TX_RC_USE_RTS_CTS; @@ -1550,6 +1551,7 @@ minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) { struct ieee80211_sta_rates *rates; int i = 0; + int max_rates = min_t(int, mp->hw->max_rates, IEEE80211_TX_RATE_TABLE_SIZE); rates = kzalloc(sizeof(*rates), GFP_ATOMIC); if (!rates) @@ -1559,16 +1561,17 @@ minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi) minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_tp_rate[0]); /* Fill up remaining, keep one entry for max_probe_rate */ - for (; i < (mp->hw->max_rates - 1); i++) + for (; i < (max_rates - 1); i++) minstrel_ht_set_rate(mp, mi, rates, i, mi->max_tp_rate[i]); - if (i < mp->hw->max_rates) + if (i < max_rates) minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_prob_rate); if (i < IEEE80211_TX_RATE_TABLE_SIZE) rates->rate[i].idx = -1; - mi->sta->max_rc_amsdu_len = minstrel_ht_get_max_amsdu_len(mi); + mi->sta->deflink.agg.max_rc_amsdu_len = minstrel_ht_get_max_amsdu_len(mi); + ieee80211_sta_recalc_aggregates(mi->sta); rate_control_set_rates(mp->hw, mi->sta, rates); } @@ -1779,7 +1782,7 @@ minstrel_ht_update_caps(void *priv, struct ieee80211_supported_band *sband, nss = minstrel_mcs_groups[i].streams; /* Mark MCS > 7 as unsupported if STA is in static SMPS mode */ - if (sta->smps_mode == IEEE80211_SMPS_STATIC && nss > 1) + if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC && nss > 1) continue; /* HT rate */ @@ -2033,7 +2036,7 @@ static void __init init_sample_table(void) memset(sample_table, 0xff, sizeof(sample_table)); for (col = 0; col < SAMPLE_COLUMNS; col++) { - prandom_bytes(rnd, sizeof(rnd)); + get_random_bytes(rnd, sizeof(rnd)); for (i = 0; i < MCS_GROUP_RATES; i++) { new_idx = (i + rnd[i]) % MCS_GROUP_RATES; while (sample_table[col][new_idx] != 0xff) diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 57df21e2170ad5247af09c1a45863c64dc4e3d34..f99416d2e14417d4a667309d57dd2df1caa6084b 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -49,7 +49,7 @@ static struct sk_buff *ieee80211_clean_skb(struct sk_buff *skb, if (present_fcs_len) __pskb_trim(skb, skb->len - present_fcs_len); - __pskb_pull(skb, rtap_space); + pskb_pull(skb, rtap_space); hdr = (void *)skb->data; fc = hdr->frame_control; @@ -74,7 +74,7 @@ static struct sk_buff *ieee80211_clean_skb(struct sk_buff *skb, memmove(skb->data + IEEE80211_HT_CTL_LEN, skb->data, hdrlen - IEEE80211_HT_CTL_LEN); - __pskb_pull(skb, IEEE80211_HT_CTL_LEN); + pskb_pull(skb, IEEE80211_HT_CTL_LEN); return skb; } @@ -215,9 +215,19 @@ ieee80211_rx_radiotap_hdrlen(struct ieee80211_local *local, } static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata, + int link_id, struct sta_info *sta, struct sk_buff *skb) { + struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); + + if (link_id >= 0) { + status->link_valid = 1; + status->link_id = link_id; + } else { + status->link_valid = 0; + } + skb_queue_tail(&sdata->skb_queue, skb); ieee80211_queue_work(&sdata->local->hw, &sdata->work); if (sta) @@ -225,11 +235,12 @@ static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata, } static void ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata, + int link_id, struct sta_info *sta, struct sk_buff *skb) { skb->protocol = 0; - __ieee80211_queue_skb_to_iface(sdata, sta, skb); + __ieee80211_queue_skb_to_iface(sdata, link_id, sta, skb); } static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata, @@ -272,7 +283,7 @@ static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata, if (!skb) return; - ieee80211_queue_skb_to_iface(sdata, NULL, skb); + ieee80211_queue_skb_to_iface(sdata, -1, NULL, skb); } /* @@ -1394,7 +1405,7 @@ static void ieee80211_rx_reorder_ampdu(struct ieee80211_rx_data *rx, /* if this mpdu is fragmented - terminate rx aggregation session */ sc = le16_to_cpu(hdr->seq_ctrl); if (sc & IEEE80211_SCTL_FRAG) { - ieee80211_queue_skb_to_iface(rx->sdata, NULL, skb); + ieee80211_queue_skb_to_iface(rx->sdata, rx->link_id, NULL, skb); return; } @@ -1441,7 +1452,7 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx) if (unlikely(ieee80211_has_retry(hdr->frame_control) && rx->sta->last_seq_ctrl[rx->seqno_idx] == hdr->seq_ctrl)) { I802_DEBUG_INC(rx->local->dot11FrameDuplicateCount); - rx->sta->deflink.rx_stats.num_duplicates++; + rx->link_sta->rx_stats.num_duplicates++; return RX_DROP_UNUSABLE; } else if (!(status->flag & RX_FLAG_AMSDU_MORE)) { rx->sta->last_seq_ctrl[rx->seqno_idx] = hdr->seq_ctrl; @@ -1720,12 +1731,13 @@ static ieee80211_rx_result debug_noinline ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx) { struct sta_info *sta = rx->sta; + struct link_sta_info *link_sta = rx->link_sta; struct sk_buff *skb = rx->skb; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; int i; - if (!sta) + if (!sta || !link_sta) return RX_CONTINUE; /* @@ -1741,47 +1753,47 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx) NL80211_IFTYPE_ADHOC); if (ether_addr_equal(bssid, rx->sdata->u.ibss.bssid) && test_sta_flag(sta, WLAN_STA_AUTHORIZED)) { - sta->deflink.rx_stats.last_rx = jiffies; + link_sta->rx_stats.last_rx = jiffies; if (ieee80211_is_data(hdr->frame_control) && !is_multicast_ether_addr(hdr->addr1)) - sta->deflink.rx_stats.last_rate = + link_sta->rx_stats.last_rate = sta_stats_encode_rate(status); } } else if (rx->sdata->vif.type == NL80211_IFTYPE_OCB) { - sta->deflink.rx_stats.last_rx = jiffies; + link_sta->rx_stats.last_rx = jiffies; } else if (!ieee80211_is_s1g_beacon(hdr->frame_control) && !is_multicast_ether_addr(hdr->addr1)) { /* * Mesh beacons will update last_rx when if they are found to * match the current local configuration when processed. */ - sta->deflink.rx_stats.last_rx = jiffies; + link_sta->rx_stats.last_rx = jiffies; if (ieee80211_is_data(hdr->frame_control)) - sta->deflink.rx_stats.last_rate = sta_stats_encode_rate(status); + link_sta->rx_stats.last_rate = sta_stats_encode_rate(status); } - sta->deflink.rx_stats.fragments++; + link_sta->rx_stats.fragments++; - u64_stats_update_begin(&rx->sta->deflink.rx_stats.syncp); - sta->deflink.rx_stats.bytes += rx->skb->len; - u64_stats_update_end(&rx->sta->deflink.rx_stats.syncp); + u64_stats_update_begin(&link_sta->rx_stats.syncp); + link_sta->rx_stats.bytes += rx->skb->len; + u64_stats_update_end(&link_sta->rx_stats.syncp); if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) { - sta->deflink.rx_stats.last_signal = status->signal; - ewma_signal_add(&sta->deflink.rx_stats_avg.signal, + link_sta->rx_stats.last_signal = status->signal; + ewma_signal_add(&link_sta->rx_stats_avg.signal, -status->signal); } if (status->chains) { - sta->deflink.rx_stats.chains = status->chains; + link_sta->rx_stats.chains = status->chains; for (i = 0; i < ARRAY_SIZE(status->chain_signal); i++) { int signal = status->chain_signal[i]; if (!(status->chains & BIT(i))) continue; - sta->deflink.rx_stats.chain_signal_last[i] = signal; - ewma_signal_add(&sta->deflink.rx_stats_avg.chain_signal[i], + link_sta->rx_stats.chain_signal_last[i] = signal; + ewma_signal_add(&link_sta->rx_stats_avg.chain_signal[i], -signal); } } @@ -1842,7 +1854,7 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx) * Update counter and free packet here to avoid * counting this as a dropped packed. */ - sta->deflink.rx_stats.packets++; + link_sta->rx_stats.packets++; dev_kfree_skb(rx->skb); return RX_QUEUED; } @@ -1854,7 +1866,6 @@ static struct ieee80211_key * ieee80211_rx_get_bigtk(struct ieee80211_rx_data *rx, int idx) { struct ieee80211_key *key = NULL; - struct ieee80211_sub_if_data *sdata = rx->sdata; int idx2; /* Make sure key gets set if either BIGTK key index is set so that @@ -1873,14 +1884,14 @@ ieee80211_rx_get_bigtk(struct ieee80211_rx_data *rx, int idx) idx2 = idx - 1; } - if (rx->sta) - key = rcu_dereference(rx->sta->deflink.gtk[idx]); + if (rx->link_sta) + key = rcu_dereference(rx->link_sta->gtk[idx]); if (!key) - key = rcu_dereference(sdata->deflink.gtk[idx]); - if (!key && rx->sta) - key = rcu_dereference(rx->sta->deflink.gtk[idx2]); + key = rcu_dereference(rx->link->gtk[idx]); + if (!key && rx->link_sta) + key = rcu_dereference(rx->link_sta->gtk[idx2]); if (!key) - key = rcu_dereference(sdata->deflink.gtk[idx2]); + key = rcu_dereference(rx->link->gtk[idx2]); return key; } @@ -1967,10 +1978,11 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) if (mmie_keyidx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS || mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS + - NUM_DEFAULT_BEACON_KEYS) { - cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev, - skb->data, - skb->len); + NUM_DEFAULT_BEACON_KEYS) { + if (rx->sdata->dev) + cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev, + skb->data, + skb->len); return RX_DROP_MONITOR; /* unexpected BIP keyidx */ } @@ -1986,15 +1998,15 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) if (mmie_keyidx < NUM_DEFAULT_KEYS || mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS) return RX_DROP_MONITOR; /* unexpected BIP keyidx */ - if (rx->sta) { + if (rx->link_sta) { if (ieee80211_is_group_privacy_action(skb) && test_sta_flag(rx->sta, WLAN_STA_MFP)) return RX_DROP_MONITOR; - rx->key = rcu_dereference(rx->sta->deflink.gtk[mmie_keyidx]); + rx->key = rcu_dereference(rx->link_sta->gtk[mmie_keyidx]); } if (!rx->key) - rx->key = rcu_dereference(rx->sdata->deflink.gtk[mmie_keyidx]); + rx->key = rcu_dereference(rx->link->gtk[mmie_keyidx]); } else if (!ieee80211_has_protected(fc)) { /* * The frame was not protected, so skip decryption. However, we @@ -2003,25 +2015,24 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) * have been expected. */ struct ieee80211_key *key = NULL; - struct ieee80211_sub_if_data *sdata = rx->sdata; int i; if (ieee80211_is_beacon(fc)) { key = ieee80211_rx_get_bigtk(rx, -1); } else if (ieee80211_is_mgmt(fc) && is_multicast_ether_addr(hdr->addr1)) { - key = rcu_dereference(rx->sdata->deflink.default_mgmt_key); + key = rcu_dereference(rx->link->default_mgmt_key); } else { - if (rx->sta) { + if (rx->link_sta) { for (i = 0; i < NUM_DEFAULT_KEYS; i++) { - key = rcu_dereference(rx->sta->deflink.gtk[i]); + key = rcu_dereference(rx->link_sta->gtk[i]); if (key) break; } } if (!key) { for (i = 0; i < NUM_DEFAULT_KEYS; i++) { - key = rcu_dereference(sdata->deflink.gtk[i]); + key = rcu_dereference(rx->link->gtk[i]); if (key) break; } @@ -2050,13 +2061,13 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) return RX_DROP_UNUSABLE; /* check per-station GTK first, if multicast packet */ - if (is_multicast_ether_addr(hdr->addr1) && rx->sta) - rx->key = rcu_dereference(rx->sta->deflink.gtk[keyidx]); + if (is_multicast_ether_addr(hdr->addr1) && rx->link_sta) + rx->key = rcu_dereference(rx->link_sta->gtk[keyidx]); /* if not found, try default key */ if (!rx->key) { if (is_multicast_ether_addr(hdr->addr1)) - rx->key = rcu_dereference(rx->sdata->deflink.gtk[keyidx]); + rx->key = rcu_dereference(rx->link->gtk[keyidx]); if (!rx->key) rx->key = rcu_dereference(rx->sdata->keys[keyidx]); @@ -2121,7 +2132,8 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx) /* either the frame has been decrypted or will be dropped */ status->flag |= RX_FLAG_DECRYPTED; - if (unlikely(ieee80211_is_beacon(fc) && result == RX_DROP_UNUSABLE)) + if (unlikely(ieee80211_is_beacon(fc) && result == RX_DROP_UNUSABLE && + rx->sdata->dev)) cfg80211_rx_unprot_mlme_mgmt(rx->sdata->dev, skb->data, skb->len); @@ -2380,7 +2392,7 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx) out: ieee80211_led_rx(rx->local); if (rx->sta) - rx->sta->deflink.rx_stats.packets++; + rx->link_sta->rx_stats.packets++; return RX_CONTINUE; } @@ -2656,9 +2668,9 @@ ieee80211_deliver_skb(struct ieee80211_rx_data *rx) * for non-QoS-data frames. Here we know it's a data * frame, so count MSDUs. */ - u64_stats_update_begin(&rx->sta->deflink.rx_stats.syncp); - rx->sta->deflink.rx_stats.msdu[rx->seqno_idx]++; - u64_stats_update_end(&rx->sta->deflink.rx_stats.syncp); + u64_stats_update_begin(&rx->link_sta->rx_stats.syncp); + rx->link_sta->rx_stats.msdu[rx->seqno_idx]++; + u64_stats_update_end(&rx->link_sta->rx_stats.syncp); } if ((sdata->vif.type == NL80211_IFTYPE_AP || @@ -3046,7 +3058,8 @@ ieee80211_rx_h_data(struct ieee80211_rx_data *rx) (tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_REQUEST || tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_RESPONSE)) { rx->skb->protocol = cpu_to_be16(ETH_P_TDLS); - __ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb); + __ieee80211_queue_skb_to_iface(sdata, rx->link_id, + rx->sta, rx->skb); return RX_QUEUED; } } @@ -3354,7 +3367,7 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx) switch (mgmt->u.action.category) { case WLAN_CATEGORY_HT: /* reject HT action frames from stations not supporting HT */ - if (!rx->sta->sta.deflink.ht_cap.ht_supported) + if (!rx->link_sta->pub->ht_cap.ht_supported) goto invalid; if (sdata->vif.type != NL80211_IFTYPE_STATION && @@ -3394,9 +3407,9 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx) } /* if no change do nothing */ - if (rx->sta->sta.smps_mode == smps_mode) + if (rx->link_sta->pub->smps_mode == smps_mode) goto handled; - rx->sta->sta.smps_mode = smps_mode; + rx->link_sta->pub->smps_mode = smps_mode; sta_opmode.smps_mode = ieee80211_smps_mode_to_smps_mode(smps_mode); sta_opmode.changed = STA_OPMODE_SMPS_MODE_CHANGED; @@ -3418,26 +3431,26 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx) struct sta_opmode_info sta_opmode = {}; /* If it doesn't support 40 MHz it can't change ... */ - if (!(rx->sta->sta.deflink.ht_cap.cap & + if (!(rx->link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40)) goto handled; if (chanwidth == IEEE80211_HT_CHANWIDTH_20MHZ) max_bw = IEEE80211_STA_RX_BW_20; else - max_bw = ieee80211_sta_cap_rx_bw(&rx->sta->deflink); + max_bw = ieee80211_sta_cap_rx_bw(rx->link_sta); /* set cur_max_bandwidth and recalc sta bw */ - rx->sta->deflink.cur_max_bandwidth = max_bw; - new_bw = ieee80211_sta_cur_vht_bw(&rx->sta->deflink); + rx->link_sta->cur_max_bandwidth = max_bw; + new_bw = ieee80211_sta_cur_vht_bw(rx->link_sta); - if (rx->sta->sta.deflink.bandwidth == new_bw) + if (rx->link_sta->pub->bandwidth == new_bw) goto handled; - rx->sta->sta.deflink.bandwidth = new_bw; + rx->link_sta->pub->bandwidth = new_bw; sband = rx->local->hw.wiphy->bands[status->band]; sta_opmode.bw = - ieee80211_sta_rx_bw_to_chan_width(&rx->sta->deflink); + ieee80211_sta_rx_bw_to_chan_width(rx->link_sta); sta_opmode.changed = STA_OPMODE_MAX_BW_CHANGED; rate_control_rate_update(local, sband, rx->sta, 0, @@ -3631,12 +3644,12 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx) handled: if (rx->sta) - rx->sta->deflink.rx_stats.packets++; + rx->link_sta->rx_stats.packets++; dev_kfree_skb(rx->skb); return RX_QUEUED; queue: - ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb); + ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb); return RX_QUEUED; } @@ -3675,7 +3688,7 @@ ieee80211_rx_h_userspace_mgmt(struct ieee80211_rx_data *rx) if (cfg80211_rx_mgmt_ext(&rx->sdata->wdev, &info)) { if (rx->sta) - rx->sta->deflink.rx_stats.packets++; + rx->link_sta->rx_stats.packets++; dev_kfree_skb(rx->skb); return RX_QUEUED; } @@ -3713,7 +3726,7 @@ ieee80211_rx_h_action_post_userspace(struct ieee80211_rx_data *rx) handled: if (rx->sta) - rx->sta->deflink.rx_stats.packets++; + rx->link_sta->rx_stats.packets++; dev_kfree_skb(rx->skb); return RX_QUEUED; } @@ -3794,7 +3807,7 @@ ieee80211_rx_h_ext(struct ieee80211_rx_data *rx) return RX_DROP_MONITOR; /* for now only beacons are ext, so queue them */ - ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb); + ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb); return RX_QUEUED; } @@ -3851,7 +3864,7 @@ ieee80211_rx_h_mgmt(struct ieee80211_rx_data *rx) return RX_DROP_MONITOR; } - ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb); + ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb); return RX_QUEUED; } @@ -3933,7 +3946,7 @@ static void ieee80211_rx_handlers_result(struct ieee80211_rx_data *rx, case RX_DROP_MONITOR: I802_DEBUG_INC(rx->sdata->local->rx_handlers_drop); if (rx->sta) - rx->sta->deflink.rx_stats.dropped++; + rx->link_sta->rx_stats.dropped++; fallthrough; case RX_CONTINUE: { struct ieee80211_rate *rate = NULL; @@ -3952,7 +3965,7 @@ static void ieee80211_rx_handlers_result(struct ieee80211_rx_data *rx, case RX_DROP_UNUSABLE: I802_DEBUG_INC(rx->sdata->local->rx_handlers_drop); if (rx->sta) - rx->sta->deflink.rx_stats.dropped++; + rx->link_sta->rx_stats.dropped++; dev_kfree_skb(rx->skb); break; case RX_QUEUED: @@ -4074,6 +4087,7 @@ void ieee80211_release_reorder_timeout(struct sta_info *sta, int tid) .link_id = -1, }; struct tid_ampdu_rx *tid_agg_rx; + u8 link_id; tid_agg_rx = rcu_dereference(sta->ampdu_mlme.tid_rx[tid]); if (!tid_agg_rx) @@ -4093,6 +4107,10 @@ void ieee80211_release_reorder_timeout(struct sta_info *sta, int tid) }; drv_event_callback(rx.local, rx.sdata, &event); } + /* FIXME: statistics won't be right with this */ + link_id = sta->sta.valid_links ? ffs(sta->sta.valid_links) - 1 : 0; + rx.link = rcu_dereference(sta->sdata->link[link_id]); + rx.link_sta = rcu_dereference(sta->link[link_id]); ieee80211_rx_handlers(&rx, &frames); } @@ -4336,6 +4354,7 @@ void ieee80211_check_fast_rx(struct sta_info *sta) .vif_type = sdata->vif.type, .control_port_protocol = sdata->control_port_protocol, }, *old, *new = NULL; + u32 offload_flags; bool set_offload = false; bool assign = false; bool offload; @@ -4451,10 +4470,10 @@ void ieee80211_check_fast_rx(struct sta_info *sta) if (assign) new = kmemdup(&fastrx, sizeof(fastrx), GFP_KERNEL); - offload = assign && - (sdata->vif.offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED); + offload_flags = get_bss_sdata(sdata)->vif.offload_flags; + offload = offload_flags & IEEE80211_OFFLOAD_DECAP_ENABLED; - if (offload) + if (assign && offload) set_offload = !test_and_set_sta_flag(sta, WLAN_STA_DECAP_OFFLOAD); else set_offload = test_and_clear_sta_flag(sta, WLAN_STA_DECAP_OFFLOAD); @@ -4508,6 +4527,15 @@ void ieee80211_check_fast_rx_iface(struct ieee80211_sub_if_data *sdata) mutex_unlock(&local->sta_mtx); } +static bool +ieee80211_rx_is_valid_sta_link_id(struct ieee80211_sta *sta, u8 link_id) +{ + if (!sta->mlo) + return false; + + return !!(sta->valid_links & BIT(link_id)); +} + static void ieee80211_rx_8023(struct ieee80211_rx_data *rx, struct ieee80211_fast_rx *fast_rx, int orig_len) @@ -4515,19 +4543,30 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx, struct ieee80211_sta_rx_stats *stats; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb); struct sta_info *sta = rx->sta; + struct link_sta_info *link_sta; struct sk_buff *skb = rx->skb; void *sa = skb->data + ETH_ALEN; void *da = skb->data; - stats = &sta->deflink.rx_stats; + if (rx->link_id >= 0) { + link_sta = rcu_dereference(sta->link[rx->link_id]); + if (WARN_ON_ONCE(!link_sta)) { + dev_kfree_skb(rx->skb); + return; + } + } else { + link_sta = &sta->deflink; + } + + stats = &link_sta->rx_stats; if (fast_rx->uses_rss) - stats = this_cpu_ptr(sta->deflink.pcpu_rx_stats); + stats = this_cpu_ptr(link_sta->pcpu_rx_stats); /* statistics part of ieee80211_rx_h_sta_process() */ if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) { stats->last_signal = status->signal; if (!fast_rx->uses_rss) - ewma_signal_add(&sta->deflink.rx_stats_avg.signal, + ewma_signal_add(&link_sta->rx_stats_avg.signal, -status->signal); } @@ -4543,7 +4582,7 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx, stats->chain_signal_last[i] = signal; if (!fast_rx->uses_rss) - ewma_signal_add(&sta->deflink.rx_stats_avg.chain_signal[i], + ewma_signal_add(&link_sta->rx_stats_avg.chain_signal[i], -signal); } } @@ -4619,7 +4658,8 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx, u8 da[ETH_ALEN]; u8 sa[ETH_ALEN]; } addrs __aligned(2); - struct ieee80211_sta_rx_stats *stats = &sta->deflink.rx_stats; + struct link_sta_info *link_sta; + struct ieee80211_sta_rx_stats *stats; /* for parallel-rx, we need to have DUP_VALIDATED, otherwise we write * to a common data structure; drivers can implement that per queue @@ -4671,7 +4711,7 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx, if (!(status->rx_flags & IEEE80211_RX_AMSDU)) { if (!pskb_may_pull(skb, snap_offs + sizeof(*payload))) - goto drop; + return false; payload = (void *)(skb->data + snap_offs); @@ -4720,8 +4760,19 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx, return true; drop: dev_kfree_skb(skb); + + if (rx->link_id >= 0) { + link_sta = rcu_dereference(sta->link[rx->link_id]); + if (!link_sta) + return true; + } else { + link_sta = &sta->deflink; + } + if (fast_rx->uses_rss) - stats = this_cpu_ptr(sta->deflink.pcpu_rx_stats); + stats = this_cpu_ptr(link_sta->pcpu_rx_stats); + else + stats = &link_sta->rx_stats; stats->dropped++; return true; @@ -4769,7 +4820,17 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx, if (!link) return true; rx->link = link; + + if (rx->sta) { + rx->link_sta = + rcu_dereference(rx->sta->link[rx->link_id]); + if (!rx->link_sta) + return true; + } } else { + if (rx->sta) + rx->link_sta = &rx->sta->deflink; + rx->link = &sdata->deflink; } @@ -4827,6 +4888,7 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw, struct list_head *list) { struct ieee80211_local *local = hw_to_local(hw); + struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_fast_rx *fast_rx; struct ieee80211_rx_data rx; @@ -4847,7 +4909,31 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw, rx.sta = container_of(pubsta, struct sta_info, sta); rx.sdata = rx.sta->sdata; - rx.link = &rx.sdata->deflink; + + if (status->link_valid && + !ieee80211_rx_is_valid_sta_link_id(pubsta, status->link_id)) + goto drop; + + /* + * TODO: Should the frame be dropped if the right link_id is not + * available? Or may be it is fine in the current form to proceed with + * the frame processing because with frame being in 802.3 format, + * link_id is used only for stats purpose and updating the stats on + * the deflink is fine? + */ + if (status->link_valid) + rx.link_id = status->link_id; + + if (rx.link_id >= 0) { + struct ieee80211_link_data *link; + + link = rcu_dereference(rx.sdata->link[rx.link_id]); + if (!link) + goto drop; + rx.link = link; + } else { + rx.link = &rx.sdata->deflink; + } fast_rx = rcu_dereference(rx.sta->fast_rx); if (!fast_rx) @@ -4877,7 +4963,19 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx, rx->sta = link_sta->sta; rx->link_id = link_sta->link_id; } else { + struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); + rx->sta = sta_info_get_bss(rx->sdata, hdr->addr2); + if (rx->sta) { + if (status->link_valid && + !ieee80211_rx_is_valid_sta_link_id(&rx->sta->sta, + status->link_id)) + return false; + + rx->link_id = status->link_valid ? status->link_id : -1; + } else { + rx->link_id = -1; + } } return ieee80211_prepare_and_rx_handle(rx, skb, consume); @@ -4893,6 +4991,7 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, struct list_head *list) { struct ieee80211_local *local = hw_to_local(hw); + struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_sub_if_data *sdata; struct ieee80211_hdr *hdr; __le16 fc; @@ -4937,10 +5036,39 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, if (ieee80211_is_data(fc)) { struct sta_info *sta, *prev_sta; + u8 link_id = status->link_id; if (pubsta) { rx.sta = container_of(pubsta, struct sta_info, sta); rx.sdata = rx.sta->sdata; + + if (status->link_valid && + !ieee80211_rx_is_valid_sta_link_id(pubsta, link_id)) + goto out; + + if (status->link_valid) + rx.link_id = status->link_id; + + /* + * In MLO connection, fetch the link_id using addr2 + * when the driver does not pass link_id in status. + * When the address translation is already performed by + * driver/hw, the valid link_id must be passed in + * status. + */ + + if (!status->link_valid && pubsta->mlo) { + struct ieee80211_hdr *hdr = (void *)skb->data; + struct link_sta_info *link_sta; + + link_sta = link_sta_info_get_bss(rx.sdata, + hdr->addr2); + if (!link_sta) + goto out; + + rx.link_id = link_sta->link_id; + } + if (ieee80211_prepare_and_rx_handle(&rx, skb, true)) return; goto out; @@ -4954,6 +5082,13 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, continue; } + if ((status->link_valid && + !ieee80211_rx_is_valid_sta_link_id(&prev_sta->sta, + link_id)) || + (!status->link_valid && prev_sta->sta.mlo)) + continue; + + rx.link_id = status->link_valid ? link_id : -1; rx.sta = prev_sta; rx.sdata = prev_sta->sdata; ieee80211_prepare_and_rx_handle(&rx, skb, false); @@ -4962,6 +5097,13 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, } if (prev_sta) { + if ((status->link_valid && + !ieee80211_rx_is_valid_sta_link_id(&prev_sta->sta, + link_id)) || + (!status->link_valid && prev_sta->sta.mlo)) + goto out; + + rx.link_id = status->link_valid ? link_id : -1; rx.sta = prev_sta; rx.sdata = prev_sta->sdata; @@ -5104,6 +5246,9 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta, } } + if (WARN_ON_ONCE(status->link_id >= IEEE80211_LINK_UNSPECIFIED)) + goto drop; + status->rx_flags = 0; kcov_remote_start_common(skb_get_kcov_handle(skb)); diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c index fa8ddf576bc1c88e31984e1fc8d746813b3f9b09..dc3cdee51e6604d8713b0c1915743dbe2212bfbc 100644 --- a/net/mac80211/scan.c +++ b/net/mac80211/scan.c @@ -469,20 +469,23 @@ static void __ieee80211_scan_completed(struct ieee80211_hw *hw, bool aborted) scan_req = rcu_dereference_protected(local->scan_req, lockdep_is_held(&local->mtx)); - if (scan_req != local->int_scan_req) { - local->scan_info.aborted = aborted; - cfg80211_scan_done(scan_req, &local->scan_info); - } RCU_INIT_POINTER(local->scan_req, NULL); RCU_INIT_POINTER(local->scan_sdata, NULL); local->scanning = 0; local->scan_chandef.chan = NULL; + synchronize_rcu(); + + if (scan_req != local->int_scan_req) { + local->scan_info.aborted = aborted; + cfg80211_scan_done(scan_req, &local->scan_info); + } + /* Set power back to normal operating levels. */ ieee80211_hw_config(local, 0); - if (!hw_scan) { + if (!hw_scan && was_scanning) { ieee80211_configure_filter(local); drv_sw_scan_complete(local, scan_sdata); ieee80211_offchannel_return(local); @@ -638,7 +641,7 @@ static void ieee80211_send_scan_probe_req(struct ieee80211_sub_if_data *sdata, if (flags & IEEE80211_PROBE_FLAG_RANDOM_SN) { struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); - u16 sn = get_random_u32(); + u16 sn = get_random_u16(); info->control.flags |= IEEE80211_TX_CTRL_NO_SEQNO; hdr->seq_ctrl = diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index cb23da9aff1e6952971b907f0e1ee25a6d744016..cebfd148bb4066911ace9f749b72c8c53a6e3941 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -274,6 +274,43 @@ link_sta_info_get_bss(struct ieee80211_sub_if_data *sdata, const u8 *addr) return NULL; } +struct ieee80211_sta * +ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw, + const u8 *addr, + const u8 *localaddr, + unsigned int *link_id) +{ + struct ieee80211_local *local = hw_to_local(hw); + struct link_sta_info *link_sta; + struct rhlist_head *tmp; + + for_each_link_sta_info(local, addr, link_sta, tmp) { + struct sta_info *sta = link_sta->sta; + struct ieee80211_link_data *link; + u8 _link_id = link_sta->link_id; + + if (!localaddr) { + if (link_id) + *link_id = _link_id; + return &sta->sta; + } + + link = rcu_dereference(sta->sdata->link[_link_id]); + if (!link) + continue; + + if (memcmp(link->conf->addr, localaddr, ETH_ALEN)) + continue; + + if (link_id) + *link_id = _link_id; + return &sta->sta; + } + + return NULL; +} +EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_link_addrs); + struct sta_info *sta_info_get_by_addrs(struct ieee80211_local *local, const u8 *sta_addr, const u8 *vif_addr) { @@ -339,6 +376,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id, sta_info_free_link(&alloc->info); kfree_rcu(alloc, rcu_head); } + + ieee80211_sta_recalc_aggregates(&sta->sta); } /** @@ -472,8 +511,12 @@ static void sta_info_add_link(struct sta_info *sta, link_info->sta = sta; link_info->link_id = link_id; link_info->pub = link_sta; + link_sta->link_id = link_id; rcu_assign_pointer(sta->link[link_id], link_info); rcu_assign_pointer(sta->sta.link[link_id], link_sta); + + link_sta->smps_mode = IEEE80211_SMPS_OFF; + link_sta->agg.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA; } static struct sta_info * @@ -494,7 +537,7 @@ __sta_info_alloc(struct ieee80211_sub_if_data *sdata, sta->sdata = sdata; if (sta_info_alloc_link(local, &sta->deflink, gfp)) - return NULL; + goto free; if (link_id >= 0) { sta_info_add_link(sta, link_id, &sta->deflink, @@ -504,6 +547,8 @@ __sta_info_alloc(struct ieee80211_sub_if_data *sdata, sta_info_add_link(sta, 0, &sta->deflink, &sta->sta.deflink); } + sta->sta.cur = &sta->sta.deflink.agg; + spin_lock_init(&sta->lock); spin_lock_init(&sta->ps_lock); INIT_WORK(&sta->drv_deliver_wk, sta_deliver_ps_frames); @@ -627,9 +672,6 @@ __sta_info_alloc(struct ieee80211_sub_if_data *sdata, } } - sta->sta.smps_mode = IEEE80211_SMPS_OFF; - sta->sta.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA; - sta->cparams.ce_threshold = CODEL_DISABLED_THRESHOLD; sta->cparams.target = MS2TIME(20); sta->cparams.interval = MS2TIME(100); @@ -2085,6 +2127,44 @@ void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid, } EXPORT_SYMBOL(ieee80211_sta_register_airtime); +void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta) +{ + struct sta_info *sta = container_of(pubsta, struct sta_info, sta); + struct ieee80211_link_sta *link_sta; + int link_id, i; + bool first = true; + + if (!pubsta->valid_links || !pubsta->mlo) { + pubsta->cur = &pubsta->deflink.agg; + return; + } + + rcu_read_lock(); + for_each_sta_active_link(&sta->sdata->vif, pubsta, link_sta, link_id) { + if (first) { + sta->cur = pubsta->deflink.agg; + first = false; + continue; + } + + sta->cur.max_amsdu_len = + min(sta->cur.max_amsdu_len, + link_sta->agg.max_amsdu_len); + sta->cur.max_rc_amsdu_len = + min(sta->cur.max_rc_amsdu_len, + link_sta->agg.max_rc_amsdu_len); + + for (i = 0; i < ARRAY_SIZE(sta->cur.max_tid_amsdu_len); i++) + sta->cur.max_tid_amsdu_len[i] = + min(sta->cur.max_tid_amsdu_len[i], + link_sta->agg.max_tid_amsdu_len[i]); + } + rcu_read_unlock(); + + pubsta->cur = &sta->cur; +} +EXPORT_SYMBOL(ieee80211_sta_recalc_aggregates); + void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local, struct sta_info *sta, u8 ac, u16 tx_airtime, bool tx_completed) @@ -2316,9 +2396,9 @@ static inline u64 sta_get_tidstats_msdu(struct ieee80211_sta_rx_stats *rxstats, u64 value; do { - start = u64_stats_fetch_begin(&rxstats->syncp); + start = u64_stats_fetch_begin_irq(&rxstats->syncp); value = rxstats->msdu[tid]; - } while (u64_stats_fetch_retry(&rxstats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&rxstats->syncp, start)); return value; } @@ -2384,9 +2464,9 @@ static inline u64 sta_get_stats_bytes(struct ieee80211_sta_rx_stats *rxstats) u64 value; do { - start = u64_stats_fetch_begin(&rxstats->syncp); + start = u64_stats_fetch_begin_irq(&rxstats->syncp); value = rxstats->bytes; - } while (u64_stats_fetch_retry(&rxstats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&rxstats->syncp, start)); return value; } @@ -2777,10 +2857,13 @@ int ieee80211_sta_activate_link(struct sta_info *sta, unsigned int link_id) sta->sta.valid_links = new_links; - if (!test_sta_flag(sta, WLAN_STA_INSERTED)) { - ret = 0; + if (!test_sta_flag(sta, WLAN_STA_INSERTED)) goto hash; - } + + /* Ensure the values are updated for the driver, + * redone by sta_remove_link on failure. + */ + ieee80211_sta_recalc_aggregates(&sta->sta); ret = drv_change_sta_links(sdata->local, sdata, &sta->sta, old_links, new_links); @@ -2799,6 +2882,7 @@ hash: void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id) { struct ieee80211_sub_if_data *sdata = sta->sdata; + u16 old_links = sta->sta.valid_links; lockdep_assert_held(&sdata->local->sta_mtx); @@ -2806,8 +2890,7 @@ void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id) if (test_sta_flag(sta, WLAN_STA_INSERTED)) drv_change_sta_links(sdata->local, sdata, &sta->sta, - sta->sta.valid_links, - sta->sta.valid_links & ~BIT(link_id)); + old_links, sta->sta.valid_links); sta_remove_link(sta, link_id, true); } @@ -2834,3 +2917,13 @@ void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta, if (val) sta->sta.max_amsdu_subframes = 4 << val; } + +#ifdef CONFIG_LOCKDEP +bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta) +{ + struct sta_info *sta = container_of(pubsta, struct sta_info, sta); + + return lockdep_is_held(&sta->local->sta_mtx); +} +EXPORT_SYMBOL(lockdep_sta_mutex_held); +#endif diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h index 2eb3a9452e075e0f89e91bfd7d679e8988560950..2517ea714dc42d080cce857073f6c4d865fb3057 100644 --- a/net/mac80211/sta_info.h +++ b/net/mac80211/sta_info.h @@ -622,6 +622,8 @@ struct link_sta_info { * @tdls_chandef: a TDLS peer can have a wider chandef that is compatible to * the BSS one. * @frags: fragment cache + * @cur: storage for aggregation data + * &struct ieee80211_sta points either here or to deflink.agg. * @deflink: This is the default link STA information, for non MLO STA all link * specific STA information is accessed through @deflink or through * link[0] which points to address of @deflink. For MLO Link STA @@ -705,6 +707,7 @@ struct sta_info { struct ieee80211_fragment_cache frags; + struct ieee80211_sta_aggregates cur; struct link_sta_info deflink; struct link_sta_info __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; diff --git a/net/mac80211/status.c b/net/mac80211/status.c index 8e77fd2e9fdffc59364e3d4b706313560845aeb9..3f9ddd7f04b6461459f1ebf875aea7a1f809b278 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -729,7 +729,7 @@ static void ieee80211_report_used_skb(struct ieee80211_local *local, if (!sdata) { skb->dev = NULL; - } else { + } else if (!dropped) { unsigned int hdr_size = ieee80211_hdrlen(hdr->frame_control); diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 45df9932d0ba15bcb375e34cf2fe8c5a9abe5bef..a364148149f9413b632e6639ec1d66f32da8aa54 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -576,6 +576,51 @@ ieee80211_tx_h_check_control_port_protocol(struct ieee80211_tx_data *tx) return TX_CONTINUE; } +static struct ieee80211_key * +ieee80211_select_link_key(struct ieee80211_tx_data *tx) +{ + struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx->skb->data; + struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb); + enum { + USE_NONE, + USE_MGMT_KEY, + USE_MCAST_KEY, + } which_key = USE_NONE; + struct ieee80211_link_data *link; + unsigned int link_id; + + if (ieee80211_is_group_privacy_action(tx->skb)) + which_key = USE_MCAST_KEY; + else if (ieee80211_is_mgmt(hdr->frame_control) && + is_multicast_ether_addr(hdr->addr1) && + ieee80211_is_robust_mgmt_frame(tx->skb)) + which_key = USE_MGMT_KEY; + else if (is_multicast_ether_addr(hdr->addr1)) + which_key = USE_MCAST_KEY; + else + return NULL; + + link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); + if (link_id == IEEE80211_LINK_UNSPECIFIED) { + link = &tx->sdata->deflink; + } else { + link = rcu_dereference(tx->sdata->link[link_id]); + if (!link) + return NULL; + } + + switch (which_key) { + case USE_NONE: + break; + case USE_MGMT_KEY: + return rcu_dereference(link->default_mgmt_key); + case USE_MCAST_KEY: + return rcu_dereference(link->default_multicast_key); + } + + return NULL; +} + static ieee80211_tx_result debug_noinline ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx) { @@ -591,16 +636,7 @@ ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx) if (tx->sta && (key = rcu_dereference(tx->sta->ptk[tx->sta->ptk_idx]))) tx->key = key; - else if (ieee80211_is_group_privacy_action(tx->skb) && - (key = rcu_dereference(tx->sdata->deflink.default_multicast_key))) - tx->key = key; - else if (ieee80211_is_mgmt(hdr->frame_control) && - is_multicast_ether_addr(hdr->addr1) && - ieee80211_is_robust_mgmt_frame(tx->skb) && - (key = rcu_dereference(tx->sdata->deflink.default_mgmt_key))) - tx->key = key; - else if (is_multicast_ether_addr(hdr->addr1) && - (key = rcu_dereference(tx->sdata->deflink.default_multicast_key))) + else if ((key = ieee80211_select_link_key(tx))) tx->key = key; else if (!is_multicast_ether_addr(hdr->addr1) && (key = rcu_dereference(tx->sdata->default_unicast_key))) @@ -2283,6 +2319,10 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, u16 len_rthdr; int hdrlen; + sdata = IEEE80211_DEV_TO_SUB_IF(dev); + if (unlikely(!ieee80211_sdata_running(sdata))) + goto fail; + memset(info, 0, sizeof(*info)); info->flags = IEEE80211_TX_CTL_REQ_TX_STATUS | IEEE80211_TX_CTL_INJECTED; @@ -2342,8 +2382,6 @@ netdev_tx_t ieee80211_monitor_start_xmit(struct sk_buff *skb, * This is necessary, for example, for old hostapd versions that * don't use nl80211-based management TX/RX. */ - sdata = IEEE80211_DEV_TO_SUB_IF(dev); - list_for_each_entry_rcu(tmp_sdata, &local->interfaces, list) { if (!ieee80211_sdata_running(tmp_sdata)) continue; @@ -2640,7 +2678,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, goto free; } memcpy(hdr.addr2, link->conf->addr, ETH_ALEN); - } else if (link_id == IEEE80211_LINK_UNSPECIFIED) { + } else if (link_id == IEEE80211_LINK_UNSPECIFIED || + (sta && sta->sta.mlo)) { memcpy(hdr.addr2, sdata->vif.addr, ETH_ALEN); } else { struct ieee80211_bss_conf *conf; @@ -3350,7 +3389,7 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata, int subframe_len = skb->len - ETH_ALEN; u8 max_subframes = sta->sta.max_amsdu_subframes; int max_frags = local->hw.max_tx_fragments; - int max_amsdu_len = sta->sta.max_amsdu_len; + int max_amsdu_len = sta->sta.cur->max_amsdu_len; int orig_truesize; u32 flow_idx; __be16 len; @@ -3376,13 +3415,13 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata, if (test_bit(IEEE80211_TXQ_NO_AMSDU, &txqi->flags)) return false; - if (sta->sta.max_rc_amsdu_len) + if (sta->sta.cur->max_rc_amsdu_len) max_amsdu_len = min_t(int, max_amsdu_len, - sta->sta.max_rc_amsdu_len); + sta->sta.cur->max_rc_amsdu_len); - if (sta->sta.max_tid_amsdu_len[tid]) + if (sta->sta.cur->max_tid_amsdu_len[tid]) max_amsdu_len = min_t(int, max_amsdu_len, - sta->sta.max_tid_amsdu_len[tid]); + sta->sta.cur->max_tid_amsdu_len[tid]); flow_idx = fq_flow_idx(fq, skb); @@ -3735,8 +3774,8 @@ begin: !test_sta_flag(tx.sta, WLAN_STA_AUTHORIZED) && (!(info->control.flags & IEEE80211_TX_CTRL_PORT_CTRL_PROTO) || - !ether_addr_equal(tx.sdata->vif.addr, - hdr->addr2)))) { + !ieee80211_is_our_addr(tx.sdata, hdr->addr2, + NULL)))) { I802_DEBUG_INC(local->tx_handlers_drop_unauth_port); ieee80211_free_txskb(&local->hw, skb); goto begin; @@ -4132,7 +4171,7 @@ void __ieee80211_subif_start_xmit(struct sk_buff *skb, struct sk_buff *next; int len = skb->len; - if (unlikely(skb->len < ETH_HLEN)) { + if (unlikely(!ieee80211_sdata_running(sdata) || skb->len < ETH_HLEN)) { kfree_skb(skb); return; } @@ -4529,7 +4568,7 @@ netdev_tx_t ieee80211_subif_start_xmit_8023(struct sk_buff *skb, struct ieee80211_key *key; struct sta_info *sta; - if (unlikely(skb->len < ETH_HLEN)) { + if (unlikely(!ieee80211_sdata_running(sdata) || skb->len < ETH_HLEN)) { kfree_skb(skb); return NETDEV_TX_OK; } @@ -5061,6 +5100,8 @@ ieee80211_beacon_get_finish(struct ieee80211_hw *hw, rate_control_get_rate(sdata, NULL, &txrc); info->control.vif = vif; + info->control.flags |= u32_encode_bits(link->link_id, + IEEE80211_TX_CTRL_MLO_LINK); info->flags |= IEEE80211_TX_CTL_CLEAR_PS_FILT | IEEE80211_TX_CTL_ASSIGN_SEQ | IEEE80211_TX_CTL_FIRST_FRAGMENT; @@ -5430,33 +5471,39 @@ EXPORT_SYMBOL(ieee80211_pspoll_get); struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - bool qos_ok) + int link_id, bool qos_ok) { + struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif); + struct ieee80211_local *local = sdata->local; + struct ieee80211_link_data *link = NULL; struct ieee80211_hdr_3addr *nullfunc; - struct ieee80211_sub_if_data *sdata; - struct ieee80211_local *local; struct sk_buff *skb; bool qos = false; if (WARN_ON(vif->type != NL80211_IFTYPE_STATION)) return NULL; - sdata = vif_to_sdata(vif); - local = sdata->local; + skb = dev_alloc_skb(local->hw.extra_tx_headroom + + sizeof(*nullfunc) + 2); + if (!skb) + return NULL; + rcu_read_lock(); if (qos_ok) { struct sta_info *sta; - rcu_read_lock(); - sta = sta_info_get(sdata, sdata->deflink.u.mgd.bssid); + sta = sta_info_get(sdata, vif->cfg.ap_addr); qos = sta && sta->sta.wme; - rcu_read_unlock(); } - skb = dev_alloc_skb(local->hw.extra_tx_headroom + - sizeof(*nullfunc) + 2); - if (!skb) - return NULL; + if (link_id >= 0) { + link = rcu_dereference(sdata->link[link_id]); + if (WARN_ON_ONCE(!link)) { + rcu_read_unlock(); + kfree_skb(skb); + return NULL; + } + } skb_reserve(skb, local->hw.extra_tx_headroom); @@ -5477,9 +5524,16 @@ struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw, skb_put_data(skb, &qoshdr, sizeof(qoshdr)); } - memcpy(nullfunc->addr1, sdata->deflink.u.mgd.bssid, ETH_ALEN); - memcpy(nullfunc->addr2, vif->addr, ETH_ALEN); - memcpy(nullfunc->addr3, sdata->deflink.u.mgd.bssid, ETH_ALEN); + if (link) { + memcpy(nullfunc->addr1, link->conf->bssid, ETH_ALEN); + memcpy(nullfunc->addr2, link->conf->addr, ETH_ALEN); + memcpy(nullfunc->addr3, link->conf->bssid, ETH_ALEN); + } else { + memcpy(nullfunc->addr1, vif->cfg.ap_addr, ETH_ALEN); + memcpy(nullfunc->addr2, vif->addr, ETH_ALEN); + memcpy(nullfunc->addr3, vif->cfg.ap_addr, ETH_ALEN); + } + rcu_read_unlock(); return skb; } @@ -5878,6 +5932,9 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev, skb_reset_network_header(skb); skb_reset_mac_header(skb); + if (local->hw.queues < IEEE80211_NUM_ACS) + goto start_xmit; + /* update QoS header to prioritize control port frames if possible, * priorization also happens for control port frames send over * AF_PACKET @@ -5885,6 +5942,7 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev, rcu_read_lock(); err = ieee80211_lookup_ra_sta(sdata, skb, &sta); if (err) { + dev_kfree_skb(skb); rcu_read_unlock(); return err; } @@ -5899,11 +5957,12 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev, * for MLO STA, the SA should be the AP MLD address, but * the link ID has been selected already */ - if (sta->sta.mlo) + if (sta && sta->sta.mlo) memcpy(ehdr->h_source, sdata->vif.addr, ETH_ALEN); } rcu_read_unlock(); +start_xmit: /* mutex lock is only needed for incrementing the cookie counter */ mutex_lock(&local->mtx); diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 53826c66372320fd8580f1f689c9ea56d6ac792a..b512cb37aafb77134b039218cf13520a5dbaa017 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -301,14 +301,14 @@ static void __ieee80211_wake_txqs(struct ieee80211_sub_if_data *sdata, int ac) local_bh_disable(); spin_lock(&fq->lock); + sdata->vif.txqs_stopped[ac] = false; + if (!test_bit(SDATA_STATE_RUNNING, &sdata->state)) goto out; if (sdata->vif.type == NL80211_IFTYPE_AP) ps = &sdata->bss->ps; - sdata->vif.txqs_stopped[ac] = false; - list_for_each_entry_rcu(sta, &local->sta_list, list) { if (sdata != sta->sdata) continue; @@ -954,9 +954,11 @@ void ieee80211_queue_delayed_work(struct ieee80211_hw *hw, } EXPORT_SYMBOL(ieee80211_queue_delayed_work); -static void ieee80211_parse_extension_element(u32 *crc, - const struct element *elem, - struct ieee802_11_elems *elems) +static void +ieee80211_parse_extension_element(u32 *crc, + const struct element *elem, + struct ieee802_11_elems *elems, + struct ieee80211_elems_parse_params *params) { const void *data = elem->data + 1; u8 len; @@ -1013,7 +1015,8 @@ static void ieee80211_parse_extension_element(u32 *crc, break; case WLAN_EID_EXT_EHT_CAPABILITY: if (ieee80211_eht_capa_size_ok(elems->he_cap, - data, len)) { + data, len, + params->from_ap)) { elems->eht_cap = data; elems->eht_cap_len = len; } @@ -1385,7 +1388,7 @@ _ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params, case WLAN_EID_EXTENSION: ieee80211_parse_extension_element(calc_crc ? &crc : NULL, - elem, elems); + elem, elems, params); break; case WLAN_EID_S1G_CAPABILITIES: if (elen >= sizeof(*elems->s1g_capab)) @@ -1442,6 +1445,8 @@ static size_t ieee802_11_find_bssid_profile(const u8 *start, size_t len, for_each_element_id(elem, WLAN_EID_MULTIPLE_BSSID, start, len) { if (elem->datalen < 2) continue; + if (elem->data[0] < 1 || elem->data[0] > 8) + continue; for_each_element(sub, elem->data + 1, elem->datalen - 1) { u8 new_bssid[ETH_ALEN]; @@ -1501,24 +1506,26 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params) const struct element *non_inherit = NULL; u8 *nontransmitted_profile; int nontransmitted_profile_len = 0; + size_t scratch_len = params->len; - elems = kzalloc(sizeof(*elems), GFP_ATOMIC); + elems = kzalloc(sizeof(*elems) + scratch_len, GFP_ATOMIC); if (!elems) return NULL; elems->ie_start = params->start; elems->total_len = params->len; - - nontransmitted_profile = kmalloc(params->len, GFP_ATOMIC); - if (nontransmitted_profile) { - nontransmitted_profile_len = - ieee802_11_find_bssid_profile(params->start, params->len, - elems, params->bss, - nontransmitted_profile); - non_inherit = - cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE, - nontransmitted_profile, - nontransmitted_profile_len); - } + elems->scratch_len = scratch_len; + elems->scratch_pos = elems->scratch; + + nontransmitted_profile = elems->scratch_pos; + nontransmitted_profile_len = + ieee802_11_find_bssid_profile(params->start, params->len, + elems, params->bss, + nontransmitted_profile); + elems->scratch_pos += nontransmitted_profile_len; + elems->scratch_len -= nontransmitted_profile_len; + non_inherit = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE, + nontransmitted_profile, + nontransmitted_profile_len); elems->crc = _ieee802_11_parse_elems_full(params, elems, non_inherit); @@ -1552,8 +1559,6 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params) offsetofend(struct ieee80211_bssid_index, dtim_count)) elems->dtim_count = elems->bssid_index->dtim_count; - kfree(nontransmitted_profile); - return elems; } @@ -2025,7 +2030,8 @@ static int ieee80211_build_preq_ies_band(struct ieee80211_sub_if_data *sdata, cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band), IEEE80211_CHAN_NO_HE | IEEE80211_CHAN_NO_EHT)) { - pos = ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, end); + pos = ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, end, + sdata->vif.type == NL80211_IFTYPE_AP); if (!pos) goto out_err; } @@ -2042,7 +2048,7 @@ static int ieee80211_build_preq_ies_band(struct ieee80211_sub_if_data *sdata, if (he_cap) { enum nl80211_iftype iftype = ieee80211_vif_type_p2p(&sdata->vif); - __le16 cap = ieee80211_get_he_6ghz_capa(sband, iftype); + __le16 cap = ieee80211_get_he_6ghz_capa(sband6, iftype); pos = ieee80211_write_he_6ghz_cap(pos, cap, end); } @@ -2526,7 +2532,6 @@ int ieee80211_reconfig(struct ieee80211_local *local) if (link) ieee80211_assign_chanctx(local, sdata, link); } - sdata_unlock(sdata); switch (sdata->vif.type) { case NL80211_IFTYPE_AP_VLAN: @@ -2545,6 +2550,7 @@ int ieee80211_reconfig(struct ieee80211_local *local) &sdata->deflink.tx_conf[i]); break; } + sdata_unlock(sdata); /* common change flags for all interface types */ changed = BSS_CHANGED_ERP_CTS_PROT | @@ -2653,23 +2659,21 @@ int ieee80211_reconfig(struct ieee80211_local *local) } /* APs are now beaconing, add back stations */ - mutex_lock(&local->sta_mtx); - list_for_each_entry(sta, &local->sta_list, list) { - enum ieee80211_sta_state state; - - if (!sta->uploaded) - continue; - - if (sta->sdata->vif.type != NL80211_IFTYPE_AP && - sta->sdata->vif.type != NL80211_IFTYPE_AP_VLAN) + list_for_each_entry(sdata, &local->interfaces, list) { + if (!ieee80211_sdata_running(sdata)) continue; - for (state = IEEE80211_STA_NOTEXIST; - state < sta->sta_state; state++) - WARN_ON(drv_sta_state(local, sta->sdata, sta, state, - state + 1)); + sdata_lock(sdata); + switch (sdata->vif.type) { + case NL80211_IFTYPE_AP_VLAN: + case NL80211_IFTYPE_AP: + ieee80211_reconfig_stations(sdata); + break; + default: + break; + } + sdata_unlock(sdata); } - mutex_unlock(&local->sta_mtx); /* add back keys */ list_for_each_entry(sdata, &local->interfaces, list) @@ -2898,7 +2902,7 @@ void ieee80211_recalc_min_chandef(struct ieee80211_sub_if_data *sdata, */ rcu_read_unlock(); - if (WARN_ON_ONCE(!chanctx_conf)) + if (!chanctx_conf) goto unlock; chanctx = container_of(chanctx_conf, struct ieee80211_chanctx, @@ -3080,6 +3084,7 @@ end: } void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata, + enum ieee80211_smps_mode smps_mode, struct sk_buff *skb) { struct ieee80211_supported_band *sband; @@ -3106,7 +3111,7 @@ void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata, cap = le16_to_cpu(iftd->he_6ghz_capa.capa); cap &= ~IEEE80211_HE_6GHZ_CAP_SM_PS; - switch (sdata->deflink.smps_mode) { + switch (smps_mode) { case IEEE80211_SMPS_AUTOMATIC: case IEEE80211_SMPS_NUM_MODES: WARN_ON(1); @@ -3507,8 +3512,7 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info, return true; } -void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata, - const struct ieee80211_eht_operation *eht_oper, +void ieee80211_chandef_eht_oper(const struct ieee80211_eht_operation *eht_oper, bool support_160, bool support_320, struct cfg80211_chan_def *chandef) { @@ -3684,7 +3688,7 @@ bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata, support_320 = eht_phy_cap & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; - ieee80211_chandef_eht_oper(sdata, eht_oper, support_160, + ieee80211_chandef_eht_oper(eht_oper, support_160, support_320, &he_chandef); } @@ -4770,6 +4774,7 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype) const struct ieee80211_sta_he_cap *he_cap; const struct ieee80211_sta_eht_cap *eht_cap; struct ieee80211_supported_band *sband; + bool is_ap; u8 n; sband = ieee80211_get_sband(sdata); @@ -4781,8 +4786,12 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype) if (!he_cap || !eht_cap) return 0; + is_ap = iftype == NL80211_IFTYPE_AP || + iftype == NL80211_IFTYPE_P2P_GO; + n = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, - &eht_cap->eht_cap_elem); + &eht_cap->eht_cap_elem, + is_ap); return 2 + 1 + sizeof(he_cap->he_cap_elem) + n + ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], @@ -4793,7 +4802,8 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype) u8 *ieee80211_ie_build_eht_cap(u8 *pos, const struct ieee80211_sta_he_cap *he_cap, const struct ieee80211_sta_eht_cap *eht_cap, - u8 *end) + u8 *end, + bool for_ap) { u8 mcs_nss_len, ppet_len; u8 ie_len; @@ -4804,7 +4814,8 @@ u8 *ieee80211_ie_build_eht_cap(u8 *pos, return orig_pos; mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, - &eht_cap->eht_cap_elem); + &eht_cap->eht_cap_elem, + for_ap); ppet_len = ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0], eht_cap->eht_cap_elem.phy_cap_info); diff --git a/net/mac80211/vht.c b/net/mac80211/vht.c index b2b09d421e8b8f0e61333927af21ea6c9787aa8c..803de58814852c90f1d353f0c9514d4be14e0c0f 100644 --- a/net/mac80211/vht.c +++ b/net/mac80211/vht.c @@ -323,16 +323,18 @@ ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata, */ switch (vht_cap->cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK) { case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454: - link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454; break; case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991: - link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991; break; case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895: default: - link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895; + link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895; break; } + + ieee80211_sta_recalc_aggregates(&link_sta->sta->sta); } /* FIXME: move this to some better location - parses HE/EHT now */ diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index 93ec2f34974899786d5290a688dbe70e7f1ed4b3..20f742b5503b449b3972a7434ca3a4865b2e9584 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -351,7 +351,7 @@ static u8 ccmp_gcmp_aad(struct sk_buff *skb, u8 *aad) * FC | A1 | A2 | A3 | SC | [A4] | [QC] */ put_unaligned_be16(len_a, &aad[0]); put_unaligned(mask_fc, (__le16 *)&aad[2]); - memcpy(&aad[4], &hdr->addr1, 3 * ETH_ALEN); + memcpy(&aad[4], &hdr->addrs, 3 * ETH_ALEN); /* Mask Seq#, leave Frag# */ aad[22] = *((u8 *) &hdr->seq_ctrl) & 0x0f; @@ -792,7 +792,7 @@ static void bip_aad(struct sk_buff *skb, u8 *aad) IEEE80211_FCTL_MOREDATA); put_unaligned(mask_fc, (__le16 *) &aad[0]); /* A1 || A2 || A3 */ - memcpy(aad + 2, &hdr->addr1, 3 * ETH_ALEN); + memcpy(aad + 2, &hdr->addrs, 3 * ETH_ALEN); } diff --git a/net/mac802154/rx.c b/net/mac802154/rx.c index b8ce84618a55b7dc082e4d1af6720ffbcda7f46c..c439125ef2b91353eee3e978bea337bdebbd6394 100644 --- a/net/mac802154/rx.c +++ b/net/mac802154/rx.c @@ -44,7 +44,7 @@ ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata, switch (mac_cb(skb)->dest.mode) { case IEEE802154_ADDR_NONE: - if (mac_cb(skb)->dest.mode != IEEE802154_ADDR_NONE) + if (hdr->source.mode != IEEE802154_ADDR_NONE) /* FIXME: check if we are PAN coordinator */ skb->pkt_type = PACKET_OTHERHOST; else diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c index c2fc2a7b252853af80515a0848ae1ee8806af120..b6b5e496fa403cc94331d3efbf1ba1dcbc405110 100644 --- a/net/mctp/af_mctp.c +++ b/net/mctp/af_mctp.c @@ -295,11 +295,12 @@ __must_hold(&net->mctp.keys_lock) mctp_dev_release_key(key->dev, key); spin_unlock_irqrestore(&key->lock, flags); - hlist_del(&key->hlist); - hlist_del(&key->sklist); - - /* unref for the lists */ - mctp_key_unref(key); + if (!hlist_unhashed(&key->hlist)) { + hlist_del_init(&key->hlist); + hlist_del_init(&key->sklist); + /* unref for the lists */ + mctp_key_unref(key); + } kfree_skb(skb); } @@ -373,9 +374,17 @@ static int mctp_ioctl_alloctag(struct mctp_sock *msk, unsigned long arg) ctl.tag = tag | MCTP_TAG_OWNER | MCTP_TAG_PREALLOC; if (copy_to_user((void __user *)arg, &ctl, sizeof(ctl))) { - spin_lock_irqsave(&key->lock, flags); - __mctp_key_remove(key, net, flags, MCTP_TRACE_KEY_DROPPED); + unsigned long fl2; + /* Unwind our key allocation: the keys list lock needs to be + * taken before the individual key locks, and we need a valid + * flags value (fl2) to pass to __mctp_key_remove, hence the + * second spin_lock_irqsave() rather than a plain spin_lock(). + */ + spin_lock_irqsave(&net->mctp.keys_lock, flags); + spin_lock_irqsave(&key->lock, fl2); + __mctp_key_remove(key, net, fl2, MCTP_TRACE_KEY_DROPPED); mctp_key_unref(key); + spin_unlock_irqrestore(&net->mctp.keys_lock, flags); return -EFAULT; } diff --git a/net/mctp/route.c b/net/mctp/route.c index 3b24b8d18b5b55d3f23b8f8af6cb7edb563dc171..2155f15a074cd10649a4e7e09ce0dba3acc66f72 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -228,12 +228,12 @@ __releases(&key->lock) if (!key->manual_alloc) { spin_lock_irqsave(&net->mctp.keys_lock, flags); - hlist_del(&key->hlist); - hlist_del(&key->sklist); + if (!hlist_unhashed(&key->hlist)) { + hlist_del_init(&key->hlist); + hlist_del_init(&key->sklist); + mctp_key_unref(key); + } spin_unlock_irqrestore(&net->mctp.keys_lock, flags); - - /* unref for the lists */ - mctp_key_unref(key); } /* and one for the local reference */ diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 35b5f806fdda178d9742572845bf6fb0fca73dbe..b52afe316dc41deac29f4bb1f6eae7c3d4ade90e 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -1079,9 +1079,9 @@ static void mpls_get_stats(struct mpls_dev *mdev, p = per_cpu_ptr(mdev->stats, i); do { - start = u64_stats_fetch_begin(&p->syncp); + start = u64_stats_fetch_begin_irq(&p->syncp); local = p->stats; - } while (u64_stats_fetch_retry(&p->syncp, start)); + } while (u64_stats_fetch_retry_irq(&p->syncp, start)); stats->rx_packets += local.rx_packets; stats->rx_bytes += local.rx_bytes; diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c index 7f9a7178043739a4d2aa1832810a3823483616b8..8df1bdb647e299344110610688618ca74ec32026 100644 --- a/net/mptcp/mptcp_diag.c +++ b/net/mptcp/mptcp_diag.c @@ -81,15 +81,18 @@ static void mptcp_diag_dump_listeners(struct sk_buff *skb, struct netlink_callba struct mptcp_diag_ctx *diag_ctx = (void *)cb->ctx; struct nlattr *bc = cb_data->inet_diag_nla_bc; struct net *net = sock_net(skb->sk); + struct inet_hashinfo *hinfo; int i; - for (i = diag_ctx->l_slot; i <= tcp_hashinfo.lhash2_mask; i++) { + hinfo = net->ipv4.tcp_death_row.hashinfo; + + for (i = diag_ctx->l_slot; i <= hinfo->lhash2_mask; i++) { struct inet_listen_hashbucket *ilb; struct hlist_nulls_node *node; struct sock *sk; int num = 0; - ilb = &tcp_hashinfo.lhash2[i]; + ilb = &hinfo->lhash2[i]; rcu_read_lock(); spin_lock(&ilb->lock); diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c index 291b5da42fdb647278dbff41dc8cec6b2df10b30..9813ed0fde9bd815b77ca734532c36778d34f4fd 100644 --- a/net/mptcp/pm_netlink.c +++ b/net/mptcp/pm_netlink.c @@ -796,7 +796,7 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk, u8 rm_id = rm_list->ids[i]; bool removed = false; - list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { + mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); int how = RCV_SHUTDOWN | SEND_SHUTDOWN; u8 id = subflow->local_id; @@ -1327,7 +1327,7 @@ static int mptcp_nl_cmd_add_addr(struct sk_buff *skb, struct genl_info *info) return -EINVAL; } - entry = kmalloc(sizeof(*entry), GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL_ACCOUNT); if (!entry) { GENL_SET_ERR_MSG(info, "can't allocate addr"); return -ENOMEM; @@ -2218,17 +2218,17 @@ static const struct genl_small_ops mptcp_pm_ops[] = { { .cmd = MPTCP_PM_CMD_ADD_ADDR, .doit = mptcp_nl_cmd_add_addr, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_DEL_ADDR, .doit = mptcp_nl_cmd_del_addr, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_FLUSH_ADDRS, .doit = mptcp_nl_cmd_flush_addrs, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_GET_ADDR, @@ -2238,7 +2238,7 @@ static const struct genl_small_ops mptcp_pm_ops[] = { { .cmd = MPTCP_PM_CMD_SET_LIMITS, .doit = mptcp_nl_cmd_set_limits, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_GET_LIMITS, @@ -2247,27 +2247,27 @@ static const struct genl_small_ops mptcp_pm_ops[] = { { .cmd = MPTCP_PM_CMD_SET_FLAGS, .doit = mptcp_nl_cmd_set_flags, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_ANNOUNCE, .doit = mptcp_nl_cmd_announce, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_REMOVE, .doit = mptcp_nl_cmd_remove, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_SUBFLOW_CREATE, .doit = mptcp_nl_cmd_sf_create, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, { .cmd = MPTCP_PM_CMD_SUBFLOW_DESTROY, .doit = mptcp_nl_cmd_sf_destroy, - .flags = GENL_ADMIN_PERM, + .flags = GENL_UNS_ADMIN_PERM, }, }; @@ -2280,6 +2280,7 @@ static struct genl_family mptcp_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = mptcp_pm_ops, .n_small_ops = ARRAY_SIZE(mptcp_pm_ops), + .resv_start_op = MPTCP_PM_CMD_SUBFLOW_DESTROY + 1, .mcgrps = mptcp_pm_mcgrps, .n_mcgrps = ARRAY_SIZE(mptcp_pm_mcgrps), }; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index da4257504fad0d504c3229dfda54511667f2f056..f599ad44ed24c4819ed7d9874def00fe0ad71e00 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -150,9 +150,15 @@ static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, to->len, MPTCP_SKB_CB(from)->end_seq); MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; - kfree_skb_partial(from, fragstolen); + + /* note the fwd memory can reach a negative value after accounting + * for the delta, but the later skb free will restore a non + * negative one + */ atomic_add(delta, &sk->sk_rmem_alloc); mptcp_rmem_charge(sk, delta); + kfree_skb_partial(from, fragstolen); + return true; } @@ -656,9 +662,9 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, skb = skb_peek(&ssk->sk_receive_queue); if (!skb) { - /* if no data is found, a racing workqueue/recvmsg - * already processed the new data, stop here or we - * can enter an infinite loop + /* With racing move_skbs_to_msk() and __mptcp_move_skbs(), + * a different CPU can have already processed the pending + * data, stop here or we can enter an infinite loop */ if (!moved) done = true; @@ -666,9 +672,9 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, } if (__mptcp_check_fallback(msk)) { - /* if we are running under the workqueue, TCP could have - * collapsed skbs between dummy map creation and now - * be sure to adjust the size + /* Under fallback skbs have no MPTCP extension and TCP could + * collapse them between the dummy map creation and the + * current dequeue. Be sure to adjust the map size. */ map_remaining = skb->len; subflow->map_data_len = skb->len; @@ -1263,7 +1269,7 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, i = skb_shinfo(skb)->nr_frags; can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); - if (!can_coalesce && i >= sysctl_max_skb_frags) { + if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) { tcp_mark_push(tcp_sk(ssk), skb); goto alloc_skb; } @@ -1538,8 +1544,9 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) struct mptcp_sendmsg_info info = { .flags = flags, }; + bool do_check_data_fin = false; struct mptcp_data_frag *dfrag; - int len, copied = 0; + int len; while ((dfrag = mptcp_send_head(sk))) { info.sent = dfrag->already_sent; @@ -1574,8 +1581,8 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags) goto out; } + do_check_data_fin = true; info.sent += ret; - copied += ret; len -= ret; mptcp_update_post_push(msk, dfrag, ret); @@ -1591,7 +1598,7 @@ out: /* ensure the rtx timer is running */ if (!mptcp_timer_pending(sk)) mptcp_reset_timer(sk); - if (copied) + if (do_check_data_fin) __mptcp_check_send_data_fin(sk); } @@ -1670,6 +1677,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) { struct mptcp_sock *msk = mptcp_sk(sk); struct page_frag *pfrag; + struct socket *ssock; size_t copied = 0; int ret = 0; long timeo; @@ -1683,14 +1691,39 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) lock_sock(sk); + ssock = __mptcp_nmpc_socket(msk); + if (unlikely(ssock && inet_sk(ssock->sk)->defer_connect)) { + struct sock *ssk = ssock->sk; + int copied_syn = 0; + + lock_sock(ssk); + + ret = tcp_sendmsg_fastopen(ssk, msg, &copied_syn, len, NULL); + copied += copied_syn; + if (ret == -EINPROGRESS && copied_syn > 0) { + /* reflect the new state on the MPTCP socket */ + inet_sk_state_store(sk, inet_sk_state_load(ssk)); + release_sock(ssk); + goto out; + } else if (ret) { + release_sock(ssk); + goto do_error; + } + release_sock(ssk); + } + timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { ret = sk_stream_wait_connect(sk, &timeo); if (ret) - goto out; + goto do_error; } + ret = -EPIPE; + if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) + goto do_error; + pfrag = sk_page_frag(sk); while (msg_data_left(msg)) { @@ -1699,11 +1732,6 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) bool dfrag_collapsed; size_t psize, offset; - if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { - ret = -EPIPE; - goto out; - } - /* reuse tail pfrag, if possible, or carve a new one from the * page allocator */ @@ -1735,7 +1763,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) if (copy_page_from_iter(dfrag->page, offset, psize, &msg->msg_iter) != psize) { ret = -EFAULT; - goto out; + goto do_error; } /* data successfully copied into the write queue */ @@ -1767,7 +1795,7 @@ wait_for_memory: __mptcp_push_pending(sk, msg->msg_flags); ret = sk_stream_wait_memory(sk, &timeo); if (ret) - goto out; + goto do_error; } if (copied) @@ -1775,7 +1803,14 @@ wait_for_memory: out: release_sock(sk); - return copied ? : ret; + return copied; + +do_error: + if (copied) + goto out; + + copied = sk_stream_error(sk, msg->msg_flags, ret); + goto out; } static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, @@ -2278,8 +2313,14 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); - if (flags & MPTCP_CF_FASTCLOSE) + if (flags & MPTCP_CF_FASTCLOSE) { + /* be sure to force the tcp_disconnect() path, + * to generate the egress reset + */ + ssk->sk_lingertime = 0; + sock_set_flag(ssk, SOCK_LINGER); subflow->send_fastclose = 1; + } need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); if (!dispose_it) { @@ -2357,7 +2398,7 @@ static void __mptcp_close_subflow(struct mptcp_sock *msk) might_sleep(); - list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { + mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (inet_sk_state_load(ssk) != TCP_CLOSE) @@ -2400,7 +2441,7 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk) mptcp_token_destroy(msk); - list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { + mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); bool slow; @@ -2412,12 +2453,31 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk) unlock_sock_fast(tcp_sk, slow); } + /* Mirror the tcp_reset() error propagation */ + switch (sk->sk_state) { + case TCP_SYN_SENT: + sk->sk_err = ECONNREFUSED; + break; + case TCP_CLOSE_WAIT: + sk->sk_err = EPIPE; + break; + case TCP_CLOSE: + return; + default: + sk->sk_err = ECONNRESET; + } + inet_sk_state_store(sk, TCP_CLOSE); sk->sk_shutdown = SHUTDOWN_MASK; smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); - mptcp_close_wake_up(sk); + /* the calling mptcp_worker will properly destroy the socket */ + if (sock_flag(sk, SOCK_DEAD)) + return; + + sk->sk_state_change(sk); + sk_error_report(sk); } static void __mptcp_retrans(struct sock *sk) @@ -2523,6 +2583,16 @@ static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) mptcp_reset_timeout(msk, 0); } +static void mptcp_do_fastclose(struct sock *sk) +{ + struct mptcp_subflow_context *subflow, *tmp; + struct mptcp_sock *msk = mptcp_sk(sk); + + mptcp_for_each_subflow_safe(msk, subflow, tmp) + __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), + subflow, MPTCP_CF_FASTCLOSE); +} + static void mptcp_worker(struct work_struct *work) { struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); @@ -2551,11 +2621,15 @@ static void mptcp_worker(struct work_struct *work) * closed, but we need the msk around to reply to incoming DATA_FIN, * even if it is orphaned and in FIN_WAIT2 state */ - if (sock_flag(sk, SOCK_DEAD) && - (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) { - inet_sk_state_store(sk, TCP_CLOSE); - __mptcp_destroy_sock(sk); - goto unlock; + if (sock_flag(sk, SOCK_DEAD)) { + if (mptcp_check_close_timeout(sk)) { + inet_sk_state_store(sk, TCP_CLOSE); + mptcp_do_fastclose(sk); + } + if (sk->sk_state == TCP_CLOSE) { + __mptcp_destroy_sock(sk); + goto unlock; + } } if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) @@ -2656,7 +2730,7 @@ static void __mptcp_clear_xmit(struct sock *sk) dfrag_clear(sk, dfrag); } -static void mptcp_cancel_work(struct sock *sk) +void mptcp_cancel_work(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); @@ -2796,13 +2870,24 @@ static void __mptcp_destroy_sock(struct sock *sk) sock_put(sk); } -static void mptcp_close(struct sock *sk, long timeout) +static __poll_t mptcp_check_readable(struct mptcp_sock *msk) +{ + /* Concurrent splices from sk_receive_queue into receive_queue will + * always show at least one non-empty queue when checked in this order. + */ + if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && + skb_queue_empty_lockless(&msk->receive_queue)) + return 0; + + return EPOLLIN | EPOLLRDNORM; +} + +bool __mptcp_close(struct sock *sk, long timeout) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); bool do_cancel_work = false; - lock_sock(sk); sk->sk_shutdown = SHUTDOWN_MASK; if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { @@ -2810,8 +2895,13 @@ static void mptcp_close(struct sock *sk, long timeout) goto cleanup; } - if (mptcp_close_state(sk)) + if (mptcp_check_readable(msk)) { + /* the msk has read data, do the MPTCP equivalent of TCP reset */ + inet_sk_state_store(sk, TCP_CLOSE); + mptcp_do_fastclose(sk); + } else if (mptcp_close_state(sk)) { __mptcp_wr_shutdown(sk); + } sk_stream_wait_close(sk, timeout); @@ -2844,6 +2934,17 @@ cleanup: } else { mptcp_reset_timeout(msk, 0); } + + return do_cancel_work; +} + +static void mptcp_close(struct sock *sk, long timeout) +{ + bool do_cancel_work; + + lock_sock(sk); + + do_cancel_work = __mptcp_close(sk, timeout); release_sock(sk); if (do_cancel_work) mptcp_cancel_work(sk); @@ -3047,7 +3148,7 @@ void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) __mptcp_clear_xmit(sk); /* join list will be eventually flushed (with rst) at sock lock release time */ - list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) + mptcp_for_each_subflow_safe(msk, subflow, tmp) __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ @@ -3519,6 +3620,7 @@ static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, do_connect: err = ssock->ops->connect(ssock, uaddr, addr_len, flags); + inet_sk(sock->sk)->defer_connect = inet_sk(ssock->sk)->defer_connect; sock->state = ssock->state; /* on successful connect, the msk state will be moved to established by @@ -3616,18 +3718,6 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, return err; } -static __poll_t mptcp_check_readable(struct mptcp_sock *msk) -{ - /* Concurrent splices from sk_receive_queue into receive_queue will - * always show at least one non-empty queue when checked in this order. - */ - if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && - skb_queue_empty_lockless(&msk->receive_queue)) - return 0; - - return EPOLLIN | EPOLLRDNORM; -} - static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) { struct sock *sk = (struct sock *)msk; @@ -3669,13 +3759,16 @@ static __poll_t mptcp_poll(struct file *file, struct socket *sock, if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { mask |= mptcp_check_readable(msk); mask |= mptcp_check_writeable(msk); + } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { + /* cf tcp_poll() note about TFO */ + mask |= EPOLLOUT | EPOLLWRNORM; } if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) mask |= EPOLLHUP; if (sk->sk_shutdown & RCV_SHUTDOWN) mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; - /* This barrier is coupled with smp_wmb() in tcp_reset() */ + /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */ smp_rmb(); if (sk->sk_err) mask |= EPOLLERR; diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 132d50833df18e704b27a1bf86c0dc20576e70ec..c0b5b4628f65018d02ffb4dcdd1f632828e1865c 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -314,6 +314,8 @@ struct mptcp_sock { #define mptcp_for_each_subflow(__msk, __subflow) \ list_for_each_entry(__subflow, &((__msk)->conn_list), node) +#define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp) \ + list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node) static inline void msk_owned_by_me(const struct mptcp_sock *msk) { @@ -612,6 +614,8 @@ void mptcp_subflow_reset(struct sock *ssk); void mptcp_subflow_queue_clean(struct sock *ssk); void mptcp_sock_graft(struct sock *sk, struct socket *parent); struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk); +bool __mptcp_close(struct sock *sk, long timeout); +void mptcp_cancel_work(struct sock *sk); bool mptcp_addresses_equal(const struct mptcp_addr_info *a, const struct mptcp_addr_info *b, bool use_port); diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index 423d3826ca1ee72e7250e6de8c616188dfa60579..c7cb68c725b2922269ff8963250db9e6a47bd340 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -559,6 +559,7 @@ static bool mptcp_supported_sockopt(int level, int optname) case TCP_NOTSENT_LOWAT: case TCP_TX_DELAY: case TCP_INQ: + case TCP_FASTOPEN_CONNECT: return true; } @@ -567,7 +568,7 @@ static bool mptcp_supported_sockopt(int level, int optname) /* TCP_REPAIR, TCP_REPAIR_QUEUE, TCP_QUEUE_SEQ, TCP_REPAIR_OPTIONS, * TCP_REPAIR_WINDOW are not supported, better avoid this mess */ - /* TCP_FASTOPEN_KEY, TCP_FASTOPEN TCP_FASTOPEN_CONNECT, TCP_FASTOPEN_NO_COOKIE, + /* TCP_FASTOPEN_KEY, TCP_FASTOPEN, TCP_FASTOPEN_NO_COOKIE, * are not supported fastopen is currently unsupported */ } @@ -768,6 +769,19 @@ static int mptcp_setsockopt_sol_tcp_defer(struct mptcp_sock *msk, sockptr_t optv return tcp_setsockopt(listener->sk, SOL_TCP, TCP_DEFER_ACCEPT, optval, optlen); } +static int mptcp_setsockopt_sol_tcp_fastopen_connect(struct mptcp_sock *msk, sockptr_t optval, + unsigned int optlen) +{ + struct socket *sock; + + /* Limit to first subflow */ + sock = __mptcp_nmpc_socket(msk); + if (!sock) + return -EINVAL; + + return tcp_setsockopt(sock->sk, SOL_TCP, TCP_FASTOPEN_CONNECT, optval, optlen); +} + static int mptcp_setsockopt_sol_tcp(struct mptcp_sock *msk, int optname, sockptr_t optval, unsigned int optlen) { @@ -796,6 +810,8 @@ static int mptcp_setsockopt_sol_tcp(struct mptcp_sock *msk, int optname, return mptcp_setsockopt_sol_tcp_nodelay(msk, optval, optlen); case TCP_DEFER_ACCEPT: return mptcp_setsockopt_sol_tcp_defer(msk, optval, optlen); + case TCP_FASTOPEN_CONNECT: + return mptcp_setsockopt_sol_tcp_fastopen_connect(msk, optval, optlen); } return -EOPNOTSUPP; @@ -1157,6 +1173,7 @@ static int mptcp_getsockopt_sol_tcp(struct mptcp_sock *msk, int optname, case TCP_INFO: case TCP_CC_INFO: case TCP_DEFER_ACCEPT: + case TCP_FASTOPEN_CONNECT: return mptcp_getsockopt_first_sf_only(msk, SOL_TCP, optname, optval, optlen); case TCP_INQ: diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index c7d49fb6e7bdbe2e38542646c8276517f279fcd6..07dd23d0fe04ac37f4cc66c0c21d4d41f50fb3f4 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -602,30 +602,6 @@ static bool subflow_hmac_valid(const struct request_sock *req, return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN); } -static void mptcp_sock_destruct(struct sock *sk) -{ - /* if new mptcp socket isn't accepted, it is free'd - * from the tcp listener sockets request queue, linked - * from req->sk. The tcp socket is released. - * This calls the ULP release function which will - * also remove the mptcp socket, via - * sock_put(ctx->conn). - * - * Problem is that the mptcp socket will be in - * ESTABLISHED state and will not have the SOCK_DEAD flag. - * Both result in warnings from inet_sock_destruct. - */ - if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { - sk->sk_state = TCP_CLOSE; - WARN_ON_ONCE(sk->sk_socket); - sock_orphan(sk); - } - - /* We don't need to clear msk->subflow, as it's still NULL at this point */ - mptcp_destroy_common(mptcp_sk(sk), 0); - inet_sock_destruct(sk); -} - static void mptcp_force_close(struct sock *sk) { /* the msk is not yet exposed to user-space */ @@ -768,7 +744,6 @@ create_child: /* new mpc subflow takes ownership of the newly * created mptcp socket */ - new_msk->sk_destruct = mptcp_sock_destruct; mptcp_sk(new_msk)->setsockopt_seq = ctx->setsockopt_seq; mptcp_pm_new_connection(mptcp_sk(new_msk), child, 1); mptcp_token_accept(subflow_req, mptcp_sk(new_msk)); @@ -1763,13 +1738,19 @@ void mptcp_subflow_queue_clean(struct sock *listener_ssk) for (msk = head; msk; msk = next) { struct sock *sk = (struct sock *)msk; - bool slow; + bool slow, do_cancel_work; + sock_hold(sk); slow = lock_sock_fast_nested(sk); next = msk->dl_next; msk->first = NULL; msk->dl_next = NULL; + + do_cancel_work = __mptcp_close(sk, 0); unlock_sock_fast(sk, slow); + if (do_cancel_work) + mptcp_cancel_work(sk); + sock_put(sk); } /* we are still under the listener msk socket lock */ diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c index c189b4c8a1823e328367c2c98ff9caf5be69286b..d27f4eccce6d79e649dc8e6d84d4113c0f5b6cd6 100644 --- a/net/ncsi/ncsi-netlink.c +++ b/net/ncsi/ncsi-netlink.c @@ -768,6 +768,7 @@ static struct genl_family ncsi_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = ncsi_ops, .n_small_ops = ARRAY_SIZE(ncsi_ops), + .resv_start_op = NCSI_CMD_SET_CHANNEL_MASK + 1, }; static int __init ncsi_init_netlink(void) diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile index 06df49ea6329773960d41604a1d052b692efc14f..0f060d10088085e35e67bdac84a6dab7ca195f5c 100644 --- a/net/netfilter/Makefile +++ b/net/netfilter/Makefile @@ -60,6 +60,12 @@ obj-$(CONFIG_NF_NAT) += nf_nat.o nf_nat-$(CONFIG_NF_NAT_REDIRECT) += nf_nat_redirect.o nf_nat-$(CONFIG_NF_NAT_MASQUERADE) += nf_nat_masquerade.o +ifeq ($(CONFIG_NF_NAT),m) +nf_nat-$(CONFIG_DEBUG_INFO_BTF_MODULES) += nf_nat_bpf.o +else ifeq ($(CONFIG_NF_NAT),y) +nf_nat-$(CONFIG_DEBUG_INFO_BTF) += nf_nat_bpf.o +endif + # NAT helpers obj-$(CONFIG_NF_NAT_AMANDA) += nf_nat_amanda.o obj-$(CONFIG_NF_NAT_FTP) += nf_nat_ftp.o diff --git a/net/netfilter/core.c b/net/netfilter/core.c index dcf752b55a52dc90e7648b2cf44496058712ec2b..5a6705a0e4ecf7e41d0100abe40b19ee18385359 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -300,12 +300,6 @@ nf_hook_entry_head(struct net *net, int pf, unsigned int hooknum, if (WARN_ON_ONCE(ARRAY_SIZE(net->nf.hooks_ipv6) <= hooknum)) return NULL; return net->nf.hooks_ipv6 + hooknum; -#if IS_ENABLED(CONFIG_DECNET) - case NFPROTO_DECNET: - if (WARN_ON_ONCE(ARRAY_SIZE(net->nf.hooks_decnet) <= hooknum)) - return NULL; - return net->nf.hooks_decnet + hooknum; -#endif default: WARN_ON_ONCE(1); return NULL; @@ -750,10 +744,6 @@ static int __net_init netfilter_net_init(struct net *net) #ifdef CONFIG_NETFILTER_FAMILY_BRIDGE __netfilter_net_init(net->nf.hooks_bridge, ARRAY_SIZE(net->nf.hooks_bridge)); #endif -#if IS_ENABLED(CONFIG_DECNET) - __netfilter_net_init(net->nf.hooks_decnet, ARRAY_SIZE(net->nf.hooks_decnet)); -#endif - #ifdef CONFIG_PROC_FS net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter", net->proc_net); diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index 16ae92054baa80b232995661ef72f5c8e6866663..e7ba5b6dd2b7c1792e76723f68553f8d174e1ae9 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -353,7 +353,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, c = kmalloc(sizeof(*c) + len + 1, GFP_ATOMIC); if (unlikely(!c)) return; - strlcpy(c->str, ext->comment, len + 1); + strscpy(c->str, ext->comment, len + 1); set->ext_size += sizeof(*c) + strlen(c->str) + 1; rcu_assign_pointer(comment->c, c); } @@ -1072,7 +1072,7 @@ static int ip_set_create(struct sk_buff *skb, const struct nfnl_info *info, if (!set) return -ENOMEM; spin_lock_init(&set->lock); - strlcpy(set->name, name, IPSET_MAXNAMELEN); + strscpy(set->name, name, IPSET_MAXNAMELEN); set->family = family; set->revision = revision; @@ -1719,11 +1719,13 @@ call_ad(struct net *net, struct sock *ctnl, struct sk_buff *skb, skb2 = nlmsg_new(payload, GFP_KERNEL); if (!skb2) return -ENOMEM; - rep = __nlmsg_put(skb2, NETLINK_CB(skb).portid, - nlh->nlmsg_seq, NLMSG_ERROR, payload, 0); + rep = nlmsg_put(skb2, NETLINK_CB(skb).portid, + nlh->nlmsg_seq, NLMSG_ERROR, payload, 0); errmsg = nlmsg_data(rep); errmsg->error = ret; - memcpy(&errmsg->msg, nlh, nlh->nlmsg_len); + unsafe_memcpy(&errmsg->msg, nlh, nlh->nlmsg_len, + /* Bounds checked by the skb layer. */); + cmdattr = (void *)&errmsg->msg + min_len; ret = nla_parse(cda, IPSET_ATTR_CMD_MAX, cmdattr, diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index fb67f1ca2495b3e5e157d72608d0b6916a49bc61..8c04bb57dd6fe3870efd75b7263f73d398c2888b 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1308,7 +1308,7 @@ void ip_vs_random_dropentry(struct netns_ipvs *ipvs) * Randomly scan 1/32 of the whole table every second */ for (idx = 0; idx < (ip_vs_conn_tab_size>>5); idx++) { - unsigned int hash = prandom_u32() & ip_vs_conn_tab_mask; + unsigned int hash = get_random_u32() & ip_vs_conn_tab_mask; hlist_for_each_entry_rcu(cp, &ip_vs_conn_tab[hash], c_list) { if (cp->ipvs != ipvs) diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index efab2b06d3732b522b9e5d07982abf0363870fff..988222fff9f025a6635b66e47ad518e67e34c182 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -2611,7 +2611,7 @@ ip_vs_copy_service(struct ip_vs_service_entry *dst, struct ip_vs_service *src) dst->addr = src->addr.ip; dst->port = src->port; dst->fwmark = src->fwmark; - strlcpy(dst->sched_name, sched_name, sizeof(dst->sched_name)); + strscpy(dst->sched_name, sched_name, sizeof(dst->sched_name)); dst->flags = src->flags; dst->timeout = src->timeout / HZ; dst->netmask = src->netmask; @@ -2805,13 +2805,13 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len) mutex_lock(&ipvs->sync_mutex); if (ipvs->sync_state & IP_VS_STATE_MASTER) { d[0].state = IP_VS_STATE_MASTER; - strlcpy(d[0].mcast_ifn, ipvs->mcfg.mcast_ifn, + strscpy(d[0].mcast_ifn, ipvs->mcfg.mcast_ifn, sizeof(d[0].mcast_ifn)); d[0].syncid = ipvs->mcfg.syncid; } if (ipvs->sync_state & IP_VS_STATE_BACKUP) { d[1].state = IP_VS_STATE_BACKUP; - strlcpy(d[1].mcast_ifn, ipvs->bcfg.mcast_ifn, + strscpy(d[1].mcast_ifn, ipvs->bcfg.mcast_ifn, sizeof(d[1].mcast_ifn)); d[1].syncid = ipvs->bcfg.syncid; } @@ -3561,7 +3561,7 @@ static int ip_vs_genl_new_daemon(struct netns_ipvs *ipvs, struct nlattr **attrs) attrs[IPVS_DAEMON_ATTR_MCAST_IFN] && attrs[IPVS_DAEMON_ATTR_SYNC_ID])) return -EINVAL; - strlcpy(c.mcast_ifn, nla_data(attrs[IPVS_DAEMON_ATTR_MCAST_IFN]), + strscpy(c.mcast_ifn, nla_data(attrs[IPVS_DAEMON_ATTR_MCAST_IFN]), sizeof(c.mcast_ifn)); c.syncid = nla_get_u32(attrs[IPVS_DAEMON_ATTR_SYNC_ID]); @@ -4005,6 +4005,7 @@ static struct genl_family ip_vs_genl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = ip_vs_genl_ops, .n_small_ops = ARRAY_SIZE(ip_vs_genl_ops), + .resv_start_op = IPVS_CMD_FLUSH + 1, }; static int __init ip_vs_genl_register(void) diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 9d43277b8b4fec1bb3da9993c33a3605f966ae38..a56fd0b5a430af283d02e25a365a49edfd0e4d65 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -1280,12 +1280,12 @@ static void set_sock_size(struct sock *sk, int mode, int val) lock_sock(sk); if (mode) { val = clamp_t(int, val, (SOCK_MIN_SNDBUF + 1) / 2, - sysctl_wmem_max); + READ_ONCE(sysctl_wmem_max)); sk->sk_sndbuf = val * 2; sk->sk_userlocks |= SOCK_SNDBUF_LOCK; } else { val = clamp_t(int, val, (SOCK_MIN_RCVBUF + 1) / 2, - sysctl_rmem_max); + READ_ONCE(sysctl_rmem_max)); sk->sk_rcvbuf = val * 2; sk->sk_userlocks |= SOCK_RCVBUF_LOCK; } diff --git a/net/netfilter/ipvs/ip_vs_twos.c b/net/netfilter/ipvs/ip_vs_twos.c index acb55d8393ef6933d003ff3ffaa21891c1aee595..f2579fc9c75bd3f9ffbe4e77fd4f36d3721e1b60 100644 --- a/net/netfilter/ipvs/ip_vs_twos.c +++ b/net/netfilter/ipvs/ip_vs_twos.c @@ -71,8 +71,8 @@ static struct ip_vs_dest *ip_vs_twos_schedule(struct ip_vs_service *svc, * from 0 to total_weight */ total_weight += 1; - rweight1 = prandom_u32() % total_weight; - rweight2 = prandom_u32() % total_weight; + rweight1 = prandom_u32_max(total_weight); + rweight2 = prandom_u32_max(total_weight); /* Pick two weighted servers */ list_for_each_entry_rcu(dest, &svc->destinations, n_list) { diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index 1cd87b28c9b05d3542d4028c28b1b5c4550fe4ee..8639e7efd0e22e5c4afd837f5191421b7be69629 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -6,12 +6,14 @@ * are exposed through to BPF programs is explicitly unstable. */ +#include #include #include +#include +#include #include #include #include -#include #include #include @@ -134,7 +136,6 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, memset(&ct->proto, 0, sizeof(ct->proto)); __nf_ct_set_timeout(ct, timeout * HZ); - ct->status |= IPS_CONFIRMED; out: if (opts->netns_id >= 0) @@ -184,14 +185,58 @@ static struct nf_conn *__bpf_nf_ct_lookup(struct net *net, return ct; } +BTF_ID_LIST(btf_nf_conn_ids) +BTF_ID(struct, nf_conn) +BTF_ID(struct, nf_conn___init) + +/* Check writes into `struct nf_conn` */ +static int _nf_conntrack_btf_struct_access(struct bpf_verifier_log *log, + const struct btf *btf, + const struct btf_type *t, int off, + int size, enum bpf_access_type atype, + u32 *next_btf_id, + enum bpf_type_flag *flag) +{ + const struct btf_type *ncit; + const struct btf_type *nct; + size_t end; + + ncit = btf_type_by_id(btf, btf_nf_conn_ids[1]); + nct = btf_type_by_id(btf, btf_nf_conn_ids[0]); + + if (t != nct && t != ncit) { + bpf_log(log, "only read is supported\n"); + return -EACCES; + } + + /* `struct nf_conn` and `struct nf_conn___init` have the same layout + * so we are safe to simply merge offset checks here + */ + switch (off) { +#if defined(CONFIG_NF_CONNTRACK_MARK) + case offsetof(struct nf_conn, mark): + end = offsetofend(struct nf_conn, mark); + break; +#endif + default: + bpf_log(log, "no write support to nf_conn at off %d\n", off); + return -EACCES; + } + + if (off + size > end) { + bpf_log(log, + "write access at off %d with size %d beyond the member of nf_conn ended at %zu\n", + off, size, end); + return -EACCES; + } + + return 0; +} + __diag_push(); __diag_ignore_all("-Wmissing-prototypes", "Global functions as their definitions will be in nf_conntrack BTF"); -struct nf_conn___init { - struct nf_conn ct; -}; - /* bpf_xdp_ct_alloc - Allocate a new CT entry * * Parameters: @@ -339,6 +384,7 @@ struct nf_conn *bpf_ct_insert_entry(struct nf_conn___init *nfct_i) struct nf_conn *nfct = (struct nf_conn *)nfct_i; int err; + nfct->status |= IPS_CONFIRMED; err = nf_conntrack_hash_check_insert(nfct); if (err < 0) { nf_conntrack_free(nfct); @@ -449,5 +495,19 @@ int register_nf_conntrack_bpf(void) int ret; ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set); - return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set); + if (!ret) { + mutex_lock(&nf_conn_btf_access_lock); + nfct_btf_struct_access = _nf_conntrack_btf_struct_access; + mutex_unlock(&nf_conn_btf_access_lock); + } + + return ret; +} + +void cleanup_nf_conntrack_bpf(void) +{ + mutex_lock(&nf_conn_btf_access_lock); + nfct_btf_struct_access = NULL; + mutex_unlock(&nf_conn_btf_access_lock); } diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 71c2f4f95d369db14335f94b5a2639116c9adab0..f97bda06d2a9064cc36f07e27dd9c04e9de2602f 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -67,6 +67,7 @@ struct conntrack_gc_work { struct delayed_work dwork; u32 next_bucket; u32 avg_timeout; + u32 count; u32 start_time; bool exiting; bool early_drop; @@ -85,10 +86,12 @@ static DEFINE_MUTEX(nf_conntrack_mutex); /* clamp timeouts to this value (TCP unacked) */ #define GC_SCAN_INTERVAL_CLAMP (300ul * HZ) -/* large initial bias so that we don't scan often just because we have - * three entries with a 1s timeout. +/* Initial bias pretending we have 100 entries at the upper bound so we don't + * wakeup often just because we have three entries with a 1s timeout while still + * allowing non-idle machines to wakeup more often when needed. */ -#define GC_SCAN_INTERVAL_INIT INT_MAX +#define GC_SCAN_INITIAL_COUNT 100 +#define GC_SCAN_INTERVAL_INIT GC_SCAN_INTERVAL_MAX #define GC_SCAN_MAX_DURATION msecs_to_jiffies(10) #define GC_SCAN_EXPIRED_MAX (64000u / HZ) @@ -1466,6 +1469,7 @@ static void gc_worker(struct work_struct *work) unsigned int expired_count = 0; unsigned long next_run; s32 delta_time; + long count; gc_work = container_of(work, struct conntrack_gc_work, dwork.work); @@ -1475,10 +1479,12 @@ static void gc_worker(struct work_struct *work) if (i == 0) { gc_work->avg_timeout = GC_SCAN_INTERVAL_INIT; + gc_work->count = GC_SCAN_INITIAL_COUNT; gc_work->start_time = start_time; } next_run = gc_work->avg_timeout; + count = gc_work->count; end_time = start_time + GC_SCAN_MAX_DURATION; @@ -1498,8 +1504,8 @@ static void gc_worker(struct work_struct *work) hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) { struct nf_conntrack_net *cnet; - unsigned long expires; struct net *net; + long expires; tmp = nf_ct_tuplehash_to_ctrack(h); @@ -1513,6 +1519,7 @@ static void gc_worker(struct work_struct *work) gc_work->next_bucket = i; gc_work->avg_timeout = next_run; + gc_work->count = count; delta_time = nfct_time_stamp - gc_work->start_time; @@ -1528,8 +1535,8 @@ static void gc_worker(struct work_struct *work) } expires = clamp(nf_ct_expires(tmp), GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_CLAMP); + expires = (expires - (long)next_run) / ++count; next_run += expires; - next_run /= 2u; if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp)) continue; @@ -1570,6 +1577,7 @@ static void gc_worker(struct work_struct *work) delta_time = nfct_time_stamp - end_time; if (delta_time > 0 && i < hashsz) { gc_work->avg_timeout = next_run; + gc_work->count = count; gc_work->next_bucket = i; next_run = 0; goto early_exit; @@ -1782,7 +1790,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, } spin_unlock_bh(&nf_conntrack_expect_lock); } - if (!exp) + if (!exp && tmpl) __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC); /* Other CPU might have obtained a pointer to this object before it was @@ -2068,10 +2076,6 @@ void nf_conntrack_alter_reply(struct nf_conn *ct, ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply; if (ct->master || (help && !hlist_empty(&help->expectations))) return; - - rcu_read_lock(); - __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC); - rcu_read_unlock(); } EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply); @@ -2512,6 +2516,7 @@ static int kill_all(struct nf_conn *i, void *data) void nf_conntrack_cleanup_start(void) { + cleanup_nf_conntrack_bpf(); conntrack_gc_work.exiting = true; } @@ -2797,7 +2802,6 @@ int nf_conntrack_init_net(struct net *net) nf_conntrack_acct_pernet_init(net); nf_conntrack_tstamp_pernet_init(net); nf_conntrack_ecache_pernet_init(net); - nf_conntrack_helper_pernet_init(net); nf_conntrack_proto_pernet_init(net); return 0; @@ -2807,10 +2811,6 @@ err_expect: return ret; } -#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \ - (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \ - IS_ENABLED(CONFIG_NF_CT_NETLINK)) - /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ int __nf_ct_change_timeout(struct nf_conn *ct, u64 timeout) @@ -2866,5 +2866,3 @@ int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status) return 0; } EXPORT_SYMBOL_GPL(nf_ct_change_status_common); - -#endif diff --git a/net/netfilter/nf_conntrack_ftp.c b/net/netfilter/nf_conntrack_ftp.c index 0d9332e9cf71a8fae7a5e7b5a0bc904863c50072..617f744a2e3a33c223ed2205559df941cbb55174 100644 --- a/net/netfilter/nf_conntrack_ftp.c +++ b/net/netfilter/nf_conntrack_ftp.c @@ -33,6 +33,7 @@ MODULE_AUTHOR("Rusty Russell "); MODULE_DESCRIPTION("ftp connection tracking helper"); MODULE_ALIAS("ip_conntrack_ftp"); MODULE_ALIAS_NFCT_HELPER(HELPER_NAME); +static DEFINE_SPINLOCK(nf_ftp_lock); #define MAX_PORTS 8 static u_int16_t ports[MAX_PORTS]; @@ -409,7 +410,8 @@ static int help(struct sk_buff *skb, } datalen = skb->len - dataoff; - spin_lock_bh(&ct->lock); + /* seqadj (nat) uses ct->lock internally, nf_nat_ftp would cause deadlock */ + spin_lock_bh(&nf_ftp_lock); fb_ptr = skb->data + dataoff; ends_in_nl = (fb_ptr[datalen - 1] == '\n'); @@ -538,7 +540,7 @@ out_update_nl: if (ends_in_nl) update_nl_seq(ct, seq, ct_ftp_info, dir, skb); out: - spin_unlock_bh(&ct->lock); + spin_unlock_bh(&nf_ftp_lock); return ret; } diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index e96b3222144471abe75925340fa5824c66545a5a..ff737a76052ed8e83e844637c0966efef7021c23 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -35,11 +35,6 @@ unsigned int nf_ct_helper_hsize __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_helper_hsize); static unsigned int nf_ct_helper_count __read_mostly; -static bool nf_ct_auto_assign_helper __read_mostly = false; -module_param_named(nf_conntrack_helper, nf_ct_auto_assign_helper, bool, 0644); -MODULE_PARM_DESC(nf_conntrack_helper, - "Enable automatic conntrack helper assignment (default 0)"); - static DEFINE_MUTEX(nf_ct_nat_helpers_mutex); static struct list_head nf_ct_nat_helpers __read_mostly; @@ -51,24 +46,6 @@ static unsigned int helper_hash(const struct nf_conntrack_tuple *tuple) (__force __u16)tuple->src.u.all) % nf_ct_helper_hsize; } -static struct nf_conntrack_helper * -__nf_ct_helper_find(const struct nf_conntrack_tuple *tuple) -{ - struct nf_conntrack_helper *helper; - struct nf_conntrack_tuple_mask mask = { .src.u.all = htons(0xFFFF) }; - unsigned int h; - - if (!nf_ct_helper_count) - return NULL; - - h = helper_hash(tuple); - hlist_for_each_entry_rcu(helper, &nf_ct_helper_hash[h], hnode) { - if (nf_ct_tuple_src_mask_cmp(tuple, &helper->tuple, &mask)) - return helper; - } - return NULL; -} - struct nf_conntrack_helper * __nf_conntrack_helper_find(const char *name, u16 l3num, u8 protonum) { @@ -209,33 +186,11 @@ nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp) } EXPORT_SYMBOL_GPL(nf_ct_helper_ext_add); -static struct nf_conntrack_helper * -nf_ct_lookup_helper(struct nf_conn *ct, struct net *net) -{ - struct nf_conntrack_net *cnet = nf_ct_pernet(net); - - if (!cnet->sysctl_auto_assign_helper) { - if (cnet->auto_assign_helper_warned) - return NULL; - if (!__nf_ct_helper_find(&ct->tuplehash[IP_CT_DIR_REPLY].tuple)) - return NULL; - pr_info("nf_conntrack: default automatic helper assignment " - "has been turned off for security reasons and CT-based " - "firewall rule not found. Use the iptables CT target " - "to attach helpers instead.\n"); - cnet->auto_assign_helper_warned = true; - return NULL; - } - - return __nf_ct_helper_find(&ct->tuplehash[IP_CT_DIR_REPLY].tuple); -} - int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, gfp_t flags) { struct nf_conntrack_helper *helper = NULL; struct nf_conn_help *help; - struct net *net = nf_ct_net(ct); /* We already got a helper explicitly attached. The function * nf_conntrack_alter_reply - in case NAT is in use - asks for looking @@ -246,23 +201,21 @@ int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, if (test_bit(IPS_HELPER_BIT, &ct->status)) return 0; - if (tmpl != NULL) { - help = nfct_help(tmpl); - if (help != NULL) { - helper = rcu_dereference(help->helper); - set_bit(IPS_HELPER_BIT, &ct->status); - } + if (WARN_ON_ONCE(!tmpl)) + return 0; + + help = nfct_help(tmpl); + if (help != NULL) { + helper = rcu_dereference(help->helper); + set_bit(IPS_HELPER_BIT, &ct->status); } help = nfct_help(ct); if (helper == NULL) { - helper = nf_ct_lookup_helper(ct, net); - if (helper == NULL) { - if (help) - RCU_INIT_POINTER(help->helper, NULL); - return 0; - } + if (help) + RCU_INIT_POINTER(help->helper, NULL); + return 0; } if (help == NULL) { @@ -545,19 +498,6 @@ void nf_nat_helper_unregister(struct nf_conntrack_nat_helper *nat) } EXPORT_SYMBOL_GPL(nf_nat_helper_unregister); -void nf_ct_set_auto_assign_helper_warned(struct net *net) -{ - nf_ct_pernet(net)->auto_assign_helper_warned = true; -} -EXPORT_SYMBOL_GPL(nf_ct_set_auto_assign_helper_warned); - -void nf_conntrack_helper_pernet_init(struct net *net) -{ - struct nf_conntrack_net *cnet = nf_ct_pernet(net); - - cnet->sysctl_auto_assign_helper = nf_ct_auto_assign_helper; -} - int nf_conntrack_helper_init(void) { nf_ct_helper_hsize = 1; /* gets rounded up to use one page */ diff --git a/net/netfilter/nf_conntrack_irc.c b/net/netfilter/nf_conntrack_irc.c index 1796c456ac98beb96753652934759a81e6980448..5703846bea3b699a8e31ed9ced9c7542362e528b 100644 --- a/net/netfilter/nf_conntrack_irc.c +++ b/net/netfilter/nf_conntrack_irc.c @@ -157,15 +157,37 @@ static int help(struct sk_buff *skb, unsigned int protoff, data = ib_ptr; data_limit = ib_ptr + datalen; - /* strlen("\1DCC SENT t AAAAAAAA P\1\n")=24 - * 5+MINMATCHLEN+strlen("t AAAAAAAA P\1\n")=14 */ - while (data < data_limit - (19 + MINMATCHLEN)) { - if (memcmp(data, "\1DCC ", 5)) { + /* Skip any whitespace */ + while (data < data_limit - 10) { + if (*data == ' ' || *data == '\r' || *data == '\n') + data++; + else + break; + } + + /* strlen("PRIVMSG x ")=10 */ + if (data < data_limit - 10) { + if (strncasecmp("PRIVMSG ", data, 8)) + goto out; + data += 8; + } + + /* strlen(" :\1DCC SENT t AAAAAAAA P\1\n")=26 + * 7+MINMATCHLEN+strlen("t AAAAAAAA P\1\n")=26 + */ + while (data < data_limit - (21 + MINMATCHLEN)) { + /* Find first " :", the start of message */ + if (memcmp(data, " :", 2)) { data++; continue; } + data += 2; + + /* then check that place only for the DCC command */ + if (memcmp(data, "\1DCC ", 5)) + goto out; data += 5; - /* we have at least (19+MINMATCHLEN)-5 bytes valid data left */ + /* we have at least (21+MINMATCHLEN)-(2+5) bytes valid data left */ iph = ip_hdr(skb); pr_debug("DCC found in master %pI4:%u %pI4:%u\n", @@ -181,7 +203,7 @@ static int help(struct sk_buff *skb, unsigned int protoff, pr_debug("DCC %s detected\n", dccprotos[i]); /* we have at least - * (19+MINMATCHLEN)-5-dccprotos[i].matchlen bytes valid + * (21+MINMATCHLEN)-7-dccprotos[i].matchlen bytes valid * data left (== 14/13 bytes) */ if (parse_dcc(data, data_limit, &dcc_ip, &dcc_port, &addr_beg_p, &addr_end_p)) { @@ -194,8 +216,9 @@ static int help(struct sk_buff *skb, unsigned int protoff, /* dcc_ip can be the internal OR external (NAT'ed) IP */ tuple = &ct->tuplehash[dir].tuple; - if (tuple->src.u3.ip != dcc_ip && - tuple->dst.u3.ip != dcc_ip) { + if ((tuple->src.u3.ip != dcc_ip && + ct->tuplehash[!dir].tuple.dst.u3.ip != dcc_ip) || + dcc_port == 0) { net_warn_ratelimited("Forged DCC command from %pI4: %pI4:%u\n", &tuple->src.u3.ip, &dcc_ip, dcc_port); diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 04169b54f2a2bef4ec165ebe322b661757a0c273..7562b215b932a0046b49ebd710a29b052e00344b 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -2298,11 +2298,6 @@ ctnetlink_create_conntrack(struct net *net, ct->status |= IPS_HELPER; RCU_INIT_POINTER(help->helper, helper); } - } else { - /* try an implicit helper assignation */ - err = __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC); - if (err < 0) - goto err2; } err = ctnetlink_setup_nat(ct, cda); diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index a63b51dceaf2cc3624e8e23862072c203e0b911f..6566310831779bbac55317361adfaf2f88b143d0 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -47,6 +47,12 @@ static const char *const tcp_conntrack_names[] = { "SYN_SENT2", }; +enum nf_ct_tcp_action { + NFCT_TCP_IGNORE, + NFCT_TCP_INVALID, + NFCT_TCP_ACCEPT, +}; + #define SECS * HZ #define MINS * 60 SECS #define HOURS * 60 MINS @@ -472,23 +478,45 @@ static void tcp_init_sender(struct ip_ct_tcp_state *sender, } } -static bool tcp_in_window(struct nf_conn *ct, - enum ip_conntrack_dir dir, - unsigned int index, - const struct sk_buff *skb, - unsigned int dataoff, - const struct tcphdr *tcph, - const struct nf_hook_state *hook_state) +__printf(6, 7) +static enum nf_ct_tcp_action nf_tcp_log_invalid(const struct sk_buff *skb, + const struct nf_conn *ct, + const struct nf_hook_state *state, + const struct ip_ct_tcp_state *sender, + enum nf_ct_tcp_action ret, + const char *fmt, ...) +{ + const struct nf_tcp_net *tn = nf_tcp_pernet(nf_ct_net(ct)); + struct va_format vaf; + va_list args; + bool be_liberal; + + be_liberal = sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || tn->tcp_be_liberal; + if (be_liberal) + return NFCT_TCP_ACCEPT; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + nf_ct_l4proto_log_invalid(skb, ct, state, "%pV", &vaf); + va_end(args); + + return ret; +} + +static enum nf_ct_tcp_action +tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, + unsigned int index, const struct sk_buff *skb, + unsigned int dataoff, const struct tcphdr *tcph, + const struct nf_hook_state *hook_state) { struct ip_ct_tcp *state = &ct->proto.tcp; - struct net *net = nf_ct_net(ct); - struct nf_tcp_net *tn = nf_tcp_pernet(net); struct ip_ct_tcp_state *sender = &state->seen[dir]; struct ip_ct_tcp_state *receiver = &state->seen[!dir]; __u32 seq, ack, sack, end, win, swin; - u16 win_raw; + bool in_recv_win, seq_ok; s32 receiver_offset; - bool res, in_recv_win; + u16 win_raw; /* * Get the required data from the packet. @@ -517,7 +545,7 @@ static bool tcp_in_window(struct nf_conn *ct, end, win); if (!tcph->ack) /* Simultaneous open */ - return true; + return NFCT_TCP_ACCEPT; } else { /* * We are in the middle of a connection, @@ -560,7 +588,7 @@ static bool tcp_in_window(struct nf_conn *ct, end, win); if (dir == IP_CT_DIR_REPLY && !tcph->ack) - return true; + return NFCT_TCP_ACCEPT; } if (!(tcph->ack)) { @@ -584,91 +612,166 @@ static bool tcp_in_window(struct nf_conn *ct, */ seq = end = sender->td_end; - /* Is the ending sequence in the receive window (if available)? */ - in_recv_win = !receiver->td_maxwin || - after(end, sender->td_end - receiver->td_maxwin - 1); - - if (before(seq, sender->td_maxend + 1) && - in_recv_win && - before(sack, receiver->td_end + 1) && - after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) { - /* - * Take into account window scaling (RFC 1323). - */ - if (!tcph->syn) - win <<= sender->td_scale; - - /* - * Update sender data. - */ - swin = win + (sack - ack); - if (sender->td_maxwin < swin) - sender->td_maxwin = swin; - if (after(end, sender->td_end)) { + seq_ok = before(seq, sender->td_maxend + 1); + if (!seq_ok) { + u32 overshot = end - sender->td_maxend + 1; + bool ack_ok; + + ack_ok = after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1); + in_recv_win = receiver->td_maxwin && + after(end, sender->td_end - receiver->td_maxwin - 1); + + if (in_recv_win && + ack_ok && + overshot <= receiver->td_maxwin && + before(sack, receiver->td_end + 1)) { + /* Work around TCPs that send more bytes than allowed by + * the receive window. + * + * If the (marked as invalid) packet is allowed to pass by + * the ruleset and the peer acks this data, then its possible + * all future packets will trigger 'ACK is over upper bound' check. + * + * Thus if only the sequence check fails then do update td_end so + * possible ACK for this data can update internal state. + */ sender->td_end = end; sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; - } - if (tcph->ack) { - if (!(sender->flags & IP_CT_TCP_FLAG_MAXACK_SET)) { - sender->td_maxack = ack; - sender->flags |= IP_CT_TCP_FLAG_MAXACK_SET; - } else if (after(ack, sender->td_maxack)) - sender->td_maxack = ack; + + return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, + "%u bytes more than expected", overshot); } - /* - * Update receiver data. - */ - if (receiver->td_maxwin != 0 && after(end, sender->td_maxend)) - receiver->td_maxwin += end - sender->td_maxend; - if (after(sack + win, receiver->td_maxend - 1)) { - receiver->td_maxend = sack + win; - if (win == 0) - receiver->td_maxend++; + return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, + "SEQ is over upper bound %u (over the window of the receiver)", + sender->td_maxend + 1); + } + + if (!before(sack, receiver->td_end + 1)) + return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, + "ACK is over upper bound %u (ACKed data not seen yet)", + receiver->td_end + 1); + + /* Is the ending sequence in the receive window (if available)? */ + in_recv_win = !receiver->td_maxwin || + after(end, sender->td_end - receiver->td_maxwin - 1); + if (!in_recv_win) + return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, + "SEQ is under lower bound %u (already ACKed data retransmitted)", + sender->td_end - receiver->td_maxwin - 1); + if (!after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) + return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, + "ignored ACK under lower bound %u (possible overly delayed)", + receiver->td_end - MAXACKWINDOW(sender) - 1); + + /* Take into account window scaling (RFC 1323). */ + if (!tcph->syn) + win <<= sender->td_scale; + + /* Update sender data. */ + swin = win + (sack - ack); + if (sender->td_maxwin < swin) + sender->td_maxwin = swin; + if (after(end, sender->td_end)) { + sender->td_end = end; + sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; + } + if (tcph->ack) { + if (!(sender->flags & IP_CT_TCP_FLAG_MAXACK_SET)) { + sender->td_maxack = ack; + sender->flags |= IP_CT_TCP_FLAG_MAXACK_SET; + } else if (after(ack, sender->td_maxack)) { + sender->td_maxack = ack; } - if (ack == receiver->td_end) - receiver->flags &= ~IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; + } - /* - * Check retransmissions. - */ - if (index == TCP_ACK_SET) { - if (state->last_dir == dir - && state->last_seq == seq - && state->last_ack == ack - && state->last_end == end - && state->last_win == win_raw) - state->retrans++; - else { - state->last_dir = dir; - state->last_seq = seq; - state->last_ack = ack; - state->last_end = end; - state->last_win = win_raw; - state->retrans = 0; - } + /* Update receiver data. */ + if (receiver->td_maxwin != 0 && after(end, sender->td_maxend)) + receiver->td_maxwin += end - sender->td_maxend; + if (after(sack + win, receiver->td_maxend - 1)) { + receiver->td_maxend = sack + win; + if (win == 0) + receiver->td_maxend++; + } + if (ack == receiver->td_end) + receiver->flags &= ~IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; + + /* Check retransmissions. */ + if (index == TCP_ACK_SET) { + if (state->last_dir == dir && + state->last_seq == seq && + state->last_ack == ack && + state->last_end == end && + state->last_win == win_raw) { + state->retrans++; + } else { + state->last_dir = dir; + state->last_seq = seq; + state->last_ack = ack; + state->last_end = end; + state->last_win = win_raw; + state->retrans = 0; } - res = true; - } else { - res = false; - if (sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || - tn->tcp_be_liberal) - res = true; - if (!res) { + } + + return NFCT_TCP_ACCEPT; +} + +static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, + enum ip_conntrack_dir dir, + int index, + const struct sk_buff *skb, + const struct nf_hook_state *hook_state) +{ + const unsigned int *timeouts; + const struct nf_tcp_net *tn; + unsigned int timeout; + u32 expires; + + if (!test_bit(IPS_ASSURED_BIT, &ct->status) || + test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) + return; + + /* We don't want to have connections hanging around in ESTABLISHED + * state for long time 'just because' conntrack deemed a FIN/RST + * out-of-window. + * + * Shrink the timeout just like when there is unacked data. + * This speeds up eviction of 'dead' connections where the + * connection and conntracks internal state are out of sync. + */ + switch (index) { + case TCP_RST_SET: + case TCP_FIN_SET: + break; + default: + return; + } + + if (ct->proto.tcp.last_dir != dir && + (ct->proto.tcp.last_index == TCP_FIN_SET || + ct->proto.tcp.last_index == TCP_RST_SET)) { + expires = nf_ct_expires(ct); + if (expires < 120 * HZ) + return; + + tn = nf_tcp_pernet(nf_ct_net(ct)); + timeouts = nf_ct_timeout_lookup(ct); + if (!timeouts) + timeouts = tn->timeouts; + + timeout = READ_ONCE(timeouts[TCP_CONNTRACK_UNACK]); + if (expires > timeout) { nf_ct_l4proto_log_invalid(skb, ct, hook_state, - "%s", - before(seq, sender->td_maxend + 1) ? - in_recv_win ? - before(sack, receiver->td_end + 1) ? - after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1) ? "BUG" - : "ACK is under the lower bound (possible overly delayed ACK)" - : "ACK is over the upper bound (ACKed data not seen yet)" - : "SEQ is under the lower bound (already ACKed data retransmitted)" - : "SEQ is over the upper bound (over the window of the receiver)"); + "packet (index %d, dir %d) response for index %d lower timeout to %u", + index, dir, ct->proto.tcp.last_index, timeout); + + WRITE_ONCE(ct->timeout, timeout + nfct_time_stamp); } + } else { + ct->proto.tcp.last_index = index; + ct->proto.tcp.last_dir = dir; } - - return res; } /* table of valid flag combinations - PUSH, ECE and CWR are always valid */ @@ -830,6 +933,7 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, struct nf_conntrack_tuple *tuple; enum tcp_conntrack new_state, old_state; unsigned int index, *timeouts; + enum nf_ct_tcp_action res; enum ip_conntrack_dir dir; const struct tcphdr *th; struct tcphdr _tcph; @@ -1095,10 +1199,18 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, break; } - if (!tcp_in_window(ct, dir, index, - skb, dataoff, th, state)) { + res = tcp_in_window(ct, dir, index, + skb, dataoff, th, state); + switch (res) { + case NFCT_TCP_IGNORE: + spin_unlock_bh(&ct->lock); + return NF_ACCEPT; + case NFCT_TCP_INVALID: + nf_tcp_handle_invalid(ct, dir, index, skb, state); spin_unlock_bh(&ct->lock); return -NF_ACCEPT; + case NFCT_TCP_ACCEPT: + break; } in_window: /* From now on we have got in-window packets */ diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index daf06f71d31cad67e020847dc745e9aa5a422c70..77f5e82d8e3fe3d678cbfaa384d71eb588df14ba 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -477,7 +477,7 @@ static int ct_sip_walk_headers(const struct nf_conn *ct, const char *dptr, return ret; if (ret == 0) break; - dataoff += *matchoff; + dataoff = *matchoff; } *in_header = 0; } @@ -489,7 +489,7 @@ static int ct_sip_walk_headers(const struct nf_conn *ct, const char *dptr, break; if (ret == 0) return ret; - dataoff += *matchoff; + dataoff = *matchoff; } if (in_header) diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c index 05895878610c06e7bba4954d661883ac2c8496b2..4ffe84c5a82cbda032fc51a0ee5b0bfb0c1cedd0 100644 --- a/net/netfilter/nf_conntrack_standalone.c +++ b/net/netfilter/nf_conntrack_standalone.c @@ -561,7 +561,6 @@ enum nf_ct_sysctl_index { NF_SYSCTL_CT_LOG_INVALID, NF_SYSCTL_CT_EXPECT_MAX, NF_SYSCTL_CT_ACCT, - NF_SYSCTL_CT_HELPER, #ifdef CONFIG_NF_CONNTRACK_EVENTS NF_SYSCTL_CT_EVENTS, #endif @@ -680,14 +679,6 @@ static struct ctl_table nf_ct_sysctl_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, - [NF_SYSCTL_CT_HELPER] = { - .procname = "nf_conntrack_helper", - .maxlen = sizeof(u8), - .mode = 0644, - .proc_handler = proc_dou8vec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, #ifdef CONFIG_NF_CONNTRACK_EVENTS [NF_SYSCTL_CT_EVENTS] = { .procname = "nf_conntrack_events", @@ -1100,7 +1091,6 @@ static int nf_conntrack_standalone_init_sysctl(struct net *net) table[NF_SYSCTL_CT_CHECKSUM].data = &net->ct.sysctl_checksum; table[NF_SYSCTL_CT_LOG_INVALID].data = &net->ct.sysctl_log_invalid; table[NF_SYSCTL_CT_ACCT].data = &net->ct.sysctl_acct; - table[NF_SYSCTL_CT_HELPER].data = &cnet->sysctl_auto_assign_helper; #ifdef CONFIG_NF_CONNTRACK_EVENTS table[NF_SYSCTL_CT_EVENTS].data = &net->ct.sysctl_events; #endif diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 765ac779bfc8f5fe89b6e75b65e4e1ecd01b9272..81c26a96c30bb68203e88cb3db674335d725fe16 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -437,12 +437,17 @@ static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table, } } +void nf_flow_table_gc_run(struct nf_flowtable *flow_table) +{ + nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL); +} + static void nf_flow_offload_work_gc(struct work_struct *work) { struct nf_flowtable *flow_table; flow_table = container_of(work, struct nf_flowtable, gc_work.work); - nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL); + nf_flow_table_gc_run(flow_table); queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ); } @@ -600,11 +605,11 @@ void nf_flow_table_free(struct nf_flowtable *flow_table) mutex_unlock(&flowtable_lock); cancel_delayed_work_sync(&flow_table->gc_work); - nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL); - nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL); nf_flow_table_offload_flush(flow_table); - if (nf_flowtable_hw_offload(flow_table)) - nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, NULL); + /* ... no more pending work after this stage ... */ + nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL); + nf_flow_table_gc_run(flow_table); + nf_flow_table_offload_flush_cleanup(flow_table); rhashtable_destroy(&flow_table->rhashtable); } EXPORT_SYMBOL_GPL(nf_flow_table_free); diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 103b6cbf257f2eb26f2eb09f6366d117f6822bf9..b04645ced89baabda0afd23d18dbb5e8b4ed793e 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -1074,6 +1074,14 @@ void nf_flow_offload_stats(struct nf_flowtable *flowtable, flow_offload_queue_work(offload); } +void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable) +{ + if (nf_flowtable_hw_offload(flowtable)) { + flush_workqueue(nf_flow_offload_del_wq); + nf_flow_table_gc_run(flowtable); + } +} + void nf_flow_table_offload_flush(struct nf_flowtable *flowtable) { if (nf_flowtable_hw_offload(flowtable)) { diff --git a/net/netfilter/nf_log.c b/net/netfilter/nf_log.c index edee7fa944c13a07c19bf7ddce84b908fe0c6836..8a29290149bd789a424012a4a1c6f161ea7be327 100644 --- a/net/netfilter/nf_log.c +++ b/net/netfilter/nf_log.c @@ -443,9 +443,9 @@ static int nf_log_proc_dostring(struct ctl_table *table, int write, mutex_lock(&nf_log_mutex); logger = nft_log_dereference(net->nf.nf_loggers[tindex]); if (!logger) - strlcpy(buf, "NONE", sizeof(buf)); + strscpy(buf, "NONE", sizeof(buf)); else - strlcpy(buf, logger->name, sizeof(buf)); + strscpy(buf, logger->name, sizeof(buf)); mutex_unlock(&nf_log_mutex); r = proc_dostring(&tmp, write, buffer, lenp, ppos); } diff --git a/net/netfilter/nf_nat_amanda.c b/net/netfilter/nf_nat_amanda.c index 3bc7e0854efe4d0265f9e442bca10a140c65ab2a..98deef6cde694349b5f952620b8f5e654e9f5f30 100644 --- a/net/netfilter/nf_nat_amanda.c +++ b/net/netfilter/nf_nat_amanda.c @@ -44,19 +44,7 @@ static unsigned int help(struct sk_buff *skb, exp->expectfn = nf_nat_follow_master; /* Try to get same port: if not, try to change it. */ - for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) { - int res; - - exp->tuple.dst.u.tcp.port = htons(port); - res = nf_ct_expect_related(exp, 0); - if (res == 0) - break; - else if (res != -EBUSY) { - port = 0; - break; - } - } - + port = nf_nat_exp_find_port(exp, ntohs(exp->saved_proto.tcp.port)); if (port == 0) { nf_ct_helper_log(skb, exp->master, "all ports in use"); return NF_DROP; diff --git a/net/netfilter/nf_nat_bpf.c b/net/netfilter/nf_nat_bpf.c new file mode 100644 index 0000000000000000000000000000000000000000..0fa5a0bbb0ffebf55f61b5ff50ec909896401889 --- /dev/null +++ b/net/netfilter/nf_nat_bpf.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Unstable NAT Helpers for XDP and TC-BPF hook + * + * These are called from the XDP and SCHED_CLS BPF programs. Note that it is + * allowed to break compatibility for these functions since the interface they + * are exposed through to BPF programs is explicitly unstable. + */ + +#include +#include +#include +#include +#include + +__diag_push(); +__diag_ignore_all("-Wmissing-prototypes", + "Global functions as their definitions will be in nf_nat BTF"); + +/* bpf_ct_set_nat_info - Set source or destination nat address + * + * Set source or destination nat address of the newly allocated + * nf_conn before insertion. This must be invoked for referenced + * PTR_TO_BTF_ID to nf_conn___init. + * + * Parameters: + * @nfct - Pointer to referenced nf_conn object, obtained using + * bpf_xdp_ct_alloc or bpf_skb_ct_alloc. + * @addr - Nat source/destination address + * @port - Nat source/destination port. Non-positive values are + * interpreted as select a random port. + * @manip - NF_NAT_MANIP_SRC or NF_NAT_MANIP_DST + */ +int bpf_ct_set_nat_info(struct nf_conn___init *nfct, + union nf_inet_addr *addr, int port, + enum nf_nat_manip_type manip) +{ + struct nf_conn *ct = (struct nf_conn *)nfct; + u16 proto = nf_ct_l3num(ct); + struct nf_nat_range2 range; + + if (proto != NFPROTO_IPV4 && proto != NFPROTO_IPV6) + return -EINVAL; + + memset(&range, 0, sizeof(struct nf_nat_range2)); + range.flags = NF_NAT_RANGE_MAP_IPS; + range.min_addr = *addr; + range.max_addr = range.min_addr; + if (port > 0) { + range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED; + range.min_proto.all = cpu_to_be16(port); + range.max_proto.all = range.min_proto.all; + } + + return nf_nat_setup_info(ct, &range, manip) == NF_DROP ? -ENOMEM : 0; +} + +__diag_pop() + +BTF_SET8_START(nf_nat_kfunc_set) +BTF_ID_FLAGS(func, bpf_ct_set_nat_info, KF_TRUSTED_ARGS) +BTF_SET8_END(nf_nat_kfunc_set) + +static const struct btf_kfunc_id_set nf_bpf_nat_kfunc_set = { + .owner = THIS_MODULE, + .set = &nf_nat_kfunc_set, +}; + +int register_nf_nat_bpf(void) +{ + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, + &nf_bpf_nat_kfunc_set); + if (ret) + return ret; + + return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, + &nf_bpf_nat_kfunc_set); +} diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 7981be526f268db13f6881bbeb3a516cd751304f..18319a6e68062bf2bdd7c1ef44a9ee5a7a5185ad 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include #include @@ -468,7 +468,7 @@ find_free_id: if (range->flags & NF_NAT_RANGE_PROTO_OFFSET) off = (ntohs(*keyptr) - ntohs(range->base_proto.all)); else - off = prandom_u32(); + off = get_random_u16(); attempts = range_size; if (attempts > max_attempts) @@ -490,7 +490,7 @@ another_round: if (attempts >= range_size || attempts < 16) return; attempts /= 2; - off = prandom_u32(); + off = get_random_u16(); goto another_round; } @@ -1152,7 +1152,7 @@ static int __init nf_nat_init(void) WARN_ON(nf_nat_hook != NULL); RCU_INIT_POINTER(nf_nat_hook, &nat_hook); - return 0; + return register_nf_nat_bpf(); } static void __exit nf_nat_cleanup(void) diff --git a/net/netfilter/nf_nat_ftp.c b/net/netfilter/nf_nat_ftp.c index aace6768a64e716611f1c83925941d8ea20a9abf..c92a436d9c486e0c70afdb6bdef128c990b7a682 100644 --- a/net/netfilter/nf_nat_ftp.c +++ b/net/netfilter/nf_nat_ftp.c @@ -86,22 +86,9 @@ static unsigned int nf_nat_ftp(struct sk_buff *skb, * this one. */ exp->expectfn = nf_nat_follow_master; - /* Try to get same port: if not, try to change it. */ - for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - port = 0; - break; - } - } - + port = nf_nat_exp_find_port(exp, ntohs(exp->saved_proto.tcp.port)); if (port == 0) { - nf_ct_helper_log(skb, ct, "all ports in use"); + nf_ct_helper_log(skb, exp->master, "all ports in use"); return NF_DROP; } diff --git a/net/netfilter/nf_nat_helper.c b/net/netfilter/nf_nat_helper.c index a263505455fcc15c39da3d01b6729afc1615e68e..a95a25196943d07c4a45a6cb127b44b75e08ba65 100644 --- a/net/netfilter/nf_nat_helper.c +++ b/net/netfilter/nf_nat_helper.c @@ -198,3 +198,34 @@ void nf_nat_follow_master(struct nf_conn *ct, nf_nat_setup_info(ct, &range, NF_NAT_MANIP_DST); } EXPORT_SYMBOL(nf_nat_follow_master); + +u16 nf_nat_exp_find_port(struct nf_conntrack_expect *exp, u16 port) +{ + static const unsigned int max_attempts = 128; + int range, attempts_left; + u16 min = port; + + range = USHRT_MAX - port; + attempts_left = range; + + if (attempts_left > max_attempts) + attempts_left = max_attempts; + + /* Try to get same port: if not, try to change it. */ + for (;;) { + int res; + + exp->tuple.dst.u.tcp.port = htons(port); + res = nf_ct_expect_related(exp, 0); + if (res == 0) + return port; + + if (res != -EBUSY || (--attempts_left < 0)) + break; + + port = min + prandom_u32_max(range); + } + + return 0; +} +EXPORT_SYMBOL_GPL(nf_nat_exp_find_port); diff --git a/net/netfilter/nf_nat_irc.c b/net/netfilter/nf_nat_irc.c index c691ab8d234cf153e96081b6281ed86639e35021..19c4fcc60c50f22abceac840f294afb262034bb0 100644 --- a/net/netfilter/nf_nat_irc.c +++ b/net/netfilter/nf_nat_irc.c @@ -48,20 +48,8 @@ static unsigned int help(struct sk_buff *skb, exp->dir = IP_CT_DIR_REPLY; exp->expectfn = nf_nat_follow_master; - /* Try to get same port: if not, try to change it. */ - for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) { - int ret; - - exp->tuple.dst.u.tcp.port = htons(port); - ret = nf_ct_expect_related(exp, 0); - if (ret == 0) - break; - else if (ret != -EBUSY) { - port = 0; - break; - } - } - + port = nf_nat_exp_find_port(exp, + ntohs(exp->saved_proto.tcp.port)); if (port == 0) { nf_ct_helper_log(skb, ct, "all ports in use"); return NF_DROP; diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index f0a735e868518f80ddcaf852432c96186053f961..cf4aeb299bdef2f481d91aeba7c7c173012ee353 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -410,19 +410,7 @@ static unsigned int nf_nat_sip_expect(struct sk_buff *skb, unsigned int protoff, exp->dir = !dir; exp->expectfn = nf_nat_sip_expected; - for (; port != 0; port++) { - int ret; - - exp->tuple.dst.u.udp.port = htons(port); - ret = nf_ct_expect_related(exp, NF_CT_EXP_F_SKIP_MASTER); - if (ret == 0) - break; - else if (ret != -EBUSY) { - port = 0; - break; - } - } - + port = nf_nat_exp_find_port(exp, port); if (port == 0) { nf_ct_helper_log(skb, ct, "all ports in use for SIP"); return NF_DROP; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 62cfb0e31c40e023f53b657cbfd4433bfaae4c74..a0653a8dfa8270788fad518829325e34d0336219 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -32,7 +32,6 @@ static LIST_HEAD(nf_tables_objects); static LIST_HEAD(nf_tables_flowtables); static LIST_HEAD(nf_tables_destroy_list); static DEFINE_SPINLOCK(nf_tables_destroy_list_lock); -static u64 table_handle; enum { NFT_VALIDATE_SKIP = 0, @@ -743,7 +742,7 @@ __printf(2, 3) int nft_request_module(struct net *net, const char *fmt, return -ENOMEM; req->done = false; - strlcpy(req->module, module_name, MODULE_NAME_LEN); + strscpy(req->module, module_name, MODULE_NAME_LEN); list_add_tail(&req->list, &nft_net->module_list); return -EAGAIN; @@ -1235,7 +1234,7 @@ static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info, INIT_LIST_HEAD(&table->flowtables); table->family = family; table->flags = flags; - table->handle = ++table_handle; + table->handle = ++nft_net->table_handle; if (table->flags & NFT_TABLE_F_OWNER) table->nlpid = NETLINK_CB(skb).portid; @@ -2167,8 +2166,10 @@ static int nft_basechain_init(struct nft_base_chain *basechain, u8 family, chain->flags |= NFT_CHAIN_BASE | flags; basechain->policy = NF_ACCEPT; if (chain->flags & NFT_CHAIN_HW_OFFLOAD && - !nft_chain_offload_support(basechain)) + !nft_chain_offload_support(basechain)) { + list_splice_init(&basechain->hook_list, &hook->list); return -EOPNOTSUPP; + } flow_block_init(&basechain->flow_block); @@ -2198,7 +2199,6 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, const struct nlattr * const *nla = ctx->nla; struct nft_table *table = ctx->table; struct nft_base_chain *basechain; - struct nft_stats __percpu *stats; struct net *net = ctx->net; char name[NFT_NAME_MAXLEN]; struct nft_rule_blob *blob; @@ -2211,6 +2211,7 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, return -EOVERFLOW; if (nla[NFTA_CHAIN_HOOK]) { + struct nft_stats __percpu *stats = NULL; struct nft_chain_hook hook; if (flags & NFT_CHAIN_BINDING) @@ -2236,15 +2237,17 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask, return PTR_ERR(stats); } rcu_assign_pointer(basechain->stats, stats); - static_branch_inc(&nft_counters_enabled); } err = nft_basechain_init(basechain, family, &hook, flags); if (err < 0) { nft_chain_release_hook(&hook); kfree(basechain); + free_percpu(stats); return err; } + if (stats) + static_branch_inc(&nft_counters_enabled); } else { if (flags & NFT_CHAIN_BASE) return -EINVAL; @@ -2574,6 +2577,9 @@ static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info, nft_ctx_init(&ctx, net, skb, info->nlh, family, table, chain, nla); if (chain != NULL) { + if (chain->flags & NFT_CHAIN_BINDING) + return -EINVAL; + if (info->nlh->nlmsg_flags & NLM_F_EXCL) { NL_SET_BAD_ATTR(extack, attr); return -EEXIST; @@ -9707,6 +9713,8 @@ static int nft_verdict_init(const struct nft_ctx *ctx, struct nft_data *data, return PTR_ERR(chain); if (nft_is_base_chain(chain)) return -EOPNOTSUPP; + if (nft_chain_is_bound(chain)) + return -EINVAL; if (desc->flags & NFT_DATA_DESC_SETELEM && chain->flags & NFT_CHAIN_BINDING) return -EINVAL; diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c index 71e29adac48b49470de25ebac25167b23a9f4514..8120aadf6a0fe1ae30ee7ec07d97d17fd28ce210 100644 --- a/net/netfilter/nfnetlink_hook.c +++ b/net/netfilter/nfnetlink_hook.c @@ -215,13 +215,6 @@ nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *de hook_head = rcu_dereference(net->nf.hooks_bridge[hook]); #endif break; -#if IS_ENABLED(CONFIG_DECNET) - case NFPROTO_DECNET: - if (hook >= ARRAY_SIZE(net->nf.hooks_decnet)) - return ERR_PTR(-EINVAL); - hook_head = rcu_dereference(net->nf.hooks_decnet[hook]); - break; -#endif #if defined(CONFIG_NETFILTER_INGRESS) || defined(CONFIG_NETFILTER_EGRESS) case NFPROTO_NETDEV: if (hook >= NF_NETDEV_NUMHOOKS) diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index 0fa2e203042728582b9860a6bdd1f798210637af..ee6840bd593378ed23d57fd501f4df3184dc2592 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -269,6 +269,7 @@ bool nf_osf_find(const struct sk_buff *skb, struct nf_osf_hdr_ctx ctx; const struct tcphdr *tcp; struct tcphdr _tcph; + bool found = false; memset(&ctx, 0, sizeof(ctx)); @@ -283,10 +284,11 @@ bool nf_osf_find(const struct sk_buff *skb, data->genre = f->genre; data->version = f->version; + found = true; break; } - return true; + return found; } EXPORT_SYMBOL_GPL(nf_osf_find); diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index b04995c3e17f120a36c1c7e1ddead14f82f22bb6..a3f01f209a53383ade9f9322601dc9e5f56deb4f 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -1089,9 +1089,6 @@ static int nft_ct_helper_obj_init(const struct nft_ctx *ctx, if (err < 0) goto err_put_helper; - /* Avoid the bogus warning, helper will be assigned after CT init */ - nf_ct_set_auto_assign_helper_warned(ctx->net); - return 0; err_put_helper: diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index 0053a697c9316364cb16f3312f30c0bafe26dfb1..adacf95b6e2bd204ea50c689f38be37642daaa18 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -51,7 +51,7 @@ static void nft_osf_eval(const struct nft_expr *expr, struct nft_regs *regs, snprintf(os_match, NFT_OSF_MAXGENRELEN, "%s:%s", data.genre, data.version); else - strlcpy(os_match, data.genre, NFT_OSF_MAXGENRELEN); + strscpy(os_match, data.genre, NFT_OSF_MAXGENRELEN); strncpy((char *)dest, os_match, NFT_OSF_MAXGENRELEN); } @@ -115,9 +115,21 @@ static int nft_osf_validate(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nft_data **data) { - return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_LOCAL_IN) | - (1 << NF_INET_PRE_ROUTING) | - (1 << NF_INET_FORWARD)); + unsigned int hooks; + + switch (ctx->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + hooks = (1 << NF_INET_LOCAL_IN) | + (1 << NF_INET_PRE_ROUTING) | + (1 << NF_INET_FORWARD); + break; + default: + return -EOPNOTSUPP; + } + + return nft_chain_validate_hooks(ctx->chain, hooks); } static bool nft_osf_reduce(struct nft_regs_track *track, diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index 2e7ac007cb30fe6c02429034d102b4f2d6df9ceb..088244f9d83833b3f40e29a9a40ab0480cf50744 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -173,10 +173,10 @@ static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = { [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 }, [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 }, [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 }, - [NFTA_PAYLOAD_OFFSET] = { .type = NLA_U32 }, - [NFTA_PAYLOAD_LEN] = { .type = NLA_U32 }, + [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX_BE(NLA_U32, 255), + [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX_BE(NLA_U32, 255), [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 }, - [NFTA_PAYLOAD_CSUM_OFFSET] = { .type = NLA_U32 }, + [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX_BE(NLA_U32, 255), [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 }, }; @@ -740,17 +740,23 @@ static int nft_payload_set_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_payload_set *priv = nft_expr_priv(expr); + u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE; + int err; priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); if (tb[NFTA_PAYLOAD_CSUM_TYPE]) - priv->csum_type = - ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE])); - if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) - priv->csum_offset = - ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_OFFSET])); + csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE])); + if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) { + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX, + &csum_offset); + if (err < 0) + return err; + + priv->csum_offset = csum_offset; + } if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) { u32 flags; @@ -761,7 +767,7 @@ static int nft_payload_set_init(const struct nft_ctx *ctx, priv->csum_flags = flags; } - switch (priv->csum_type) { + switch (csum_type) { case NFT_PAYLOAD_CSUM_NONE: case NFT_PAYLOAD_CSUM_INET: break; @@ -775,6 +781,7 @@ static int nft_payload_set_init(const struct nft_ctx *ctx, default: return -EOPNOTSUPP; } + priv->csum_type = csum_type; return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg, priv->len); @@ -833,6 +840,7 @@ nft_payload_select_ops(const struct nft_ctx *ctx, { enum nft_payload_bases base; unsigned int offset, len; + int err; if (tb[NFTA_PAYLOAD_BASE] == NULL || tb[NFTA_PAYLOAD_OFFSET] == NULL || @@ -859,8 +867,13 @@ nft_payload_select_ops(const struct nft_ctx *ctx, if (tb[NFTA_PAYLOAD_DREG] == NULL) return ERR_PTR(-EINVAL); - offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); - len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset); + if (err < 0) + return ERR_PTR(err); + + err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len); + if (err < 0) + return ERR_PTR(err); if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) && base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER) diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index a7de29137618ae2a2d7a2dccfb0a13c0c9503f8d..49a5348a6a14f08c7c67be724bbb5c1d9962e2c4 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -40,16 +40,17 @@ static noinline bool nft_sock_get_eval_cgroupv2(u32 *dest, struct sock *sk, const struct nft_pktinfo *pkt, u32 level) { struct cgroup *cgrp; + u64 cgid; if (!sk_fullsock(sk)) return false; - cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); - if (level > cgrp->level) + cgrp = cgroup_ancestor(sock_cgroup_ptr(&sk->sk_cgrp_data), level); + if (!cgrp) return false; - memcpy(dest, &cgrp->ancestor_ids[level], sizeof(u64)); - + cgid = cgroup_id(cgrp); + memcpy(dest, &cgid, sizeof(u64)); return true; } #endif diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index 68b2eed742df907863fe237860798a0814a2df17..62da25ad264bcfe9566b86680b8271d68864fa1c 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -312,6 +312,13 @@ static int nft_tproxy_dump(struct sk_buff *skb, return 0; } +static int nft_tproxy_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr, + const struct nft_data **data) +{ + return nft_chain_validate_hooks(ctx->chain, 1 << NF_INET_PRE_ROUTING); +} + static struct nft_expr_type nft_tproxy_type; static const struct nft_expr_ops nft_tproxy_ops = { .type = &nft_tproxy_type, @@ -321,6 +328,7 @@ static const struct nft_expr_ops nft_tproxy_ops = { .destroy = nft_tproxy_destroy, .dump = nft_tproxy_dump, .reduce = NFT_REDUCE_READONLY, + .validate = nft_tproxy_validate, }; static struct nft_expr_type nft_tproxy_type __read_mostly = { diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c index 5edaaded706d9cf018ad39ef64184507c99a6778..983ade4be3b39b1cb55396a0f7ca8e2e0d0c4cac 100644 --- a/net/netfilter/nft_tunnel.c +++ b/net/netfilter/nft_tunnel.c @@ -161,6 +161,7 @@ static const struct nft_expr_ops nft_tunnel_get_ops = { static struct nft_expr_type nft_tunnel_type __read_mostly = { .name = "tunnel", + .family = NFPROTO_NETDEV, .ops = &nft_tunnel_get_ops, .policy = nft_tunnel_policy, .maxattr = NFTA_TUNNEL_MAX, diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 54a489f16b171220aaffef3ea43355fd89cd91ce..470282cf3fae61884894765965105c96537dc3a2 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -766,7 +766,7 @@ void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr, msize += off; m->u.user.match_size = msize; - strlcpy(name, match->name, sizeof(name)); + strscpy(name, match->name, sizeof(name)); module_put(match->me); strncpy(m->u.user.name, name, sizeof(m->u.user.name)); @@ -1146,7 +1146,7 @@ void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr, tsize += off; t->u.user.target_size = tsize; - strlcpy(name, target->name, sizeof(name)); + strscpy(name, target->name, sizeof(name)); module_put(target->me); strncpy(t->u.user.name, name, sizeof(t->u.user.name)); @@ -1827,7 +1827,7 @@ int xt_proto_init(struct net *net, u_int8_t af) root_uid = make_kuid(net->user_ns, 0); root_gid = make_kgid(net->user_ns, 0); - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_TABLES, sizeof(buf)); proc = proc_create_net_data(buf, 0440, net->proc_net, &xt_table_seq_ops, sizeof(struct seq_net_private), @@ -1837,7 +1837,7 @@ int xt_proto_init(struct net *net, u_int8_t af) if (uid_valid(root_uid) && gid_valid(root_gid)) proc_set_user(proc, root_uid, root_gid); - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_MATCHES, sizeof(buf)); proc = proc_create_seq_private(buf, 0440, net->proc_net, &xt_match_seq_ops, sizeof(struct nf_mttg_trav), @@ -1847,7 +1847,7 @@ int xt_proto_init(struct net *net, u_int8_t af) if (uid_valid(root_uid) && gid_valid(root_gid)) proc_set_user(proc, root_uid, root_gid); - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_TARGETS, sizeof(buf)); proc = proc_create_seq_private(buf, 0440, net->proc_net, &xt_target_seq_ops, sizeof(struct nf_mttg_trav), @@ -1862,12 +1862,12 @@ int xt_proto_init(struct net *net, u_int8_t af) #ifdef CONFIG_PROC_FS out_remove_matches: - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_MATCHES, sizeof(buf)); remove_proc_entry(buf, net->proc_net); out_remove_tables: - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_TABLES, sizeof(buf)); remove_proc_entry(buf, net->proc_net); out: @@ -1881,15 +1881,15 @@ void xt_proto_fini(struct net *net, u_int8_t af) #ifdef CONFIG_PROC_FS char buf[XT_FUNCTION_MAXNAMELEN]; - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_TABLES, sizeof(buf)); remove_proc_entry(buf, net->proc_net); - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_TARGETS, sizeof(buf)); remove_proc_entry(buf, net->proc_net); - strlcpy(buf, xt_prefix[af], sizeof(buf)); + strscpy(buf, xt_prefix[af], sizeof(buf)); strlcat(buf, FORMAT_MATCHES, sizeof(buf)); remove_proc_entry(buf, net->proc_net); #endif /*CONFIG_PROC_FS*/ diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c index 8aec1b529364ae1702f38a1ce4dd69c79cd6eb5e..80f6624e23554b30090b4a86b763ad279f5e44ac 100644 --- a/net/netfilter/xt_RATEEST.c +++ b/net/netfilter/xt_RATEEST.c @@ -144,7 +144,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par) goto err1; gnet_stats_basic_sync_init(&est->bstats); - strlcpy(est->name, info->name, sizeof(est->name)); + strscpy(est->name, info->name, sizeof(est->name)); spin_lock_init(&est->lock); est->refcnt = 1; est->params.interval = info->interval; diff --git a/net/netfilter/xt_statistic.c b/net/netfilter/xt_statistic.c index 203e24ae472c2a1bd0f67eaf2ce92639a6aebca4..b26c1dcfc27b50c73a24904c599950443f51b780 100644 --- a/net/netfilter/xt_statistic.c +++ b/net/netfilter/xt_statistic.c @@ -34,7 +34,7 @@ statistic_mt(const struct sk_buff *skb, struct xt_action_param *par) switch (info->mode) { case XT_STATISTIC_MODE_RANDOM: - if ((prandom_u32() & 0x7FFFFFFF) < info->u.random.probability) + if ((get_random_u32() & 0x7FFFFFFF) < info->u.random.probability) ret = !ret; break; case XT_STATISTIC_MODE_NTH: diff --git a/net/netlabel/netlabel_calipso.c b/net/netlabel/netlabel_calipso.c index 91a19c3ea1a365a871dcf6a617e7f301bf5f899e..f1d5b8465217802a3015cae4edef70cf67b22834 100644 --- a/net/netlabel/netlabel_calipso.c +++ b/net/netlabel/netlabel_calipso.c @@ -344,6 +344,7 @@ static struct genl_family netlbl_calipso_gnl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = netlbl_calipso_ops, .n_small_ops = ARRAY_SIZE(netlbl_calipso_ops), + .resv_start_op = NLBL_CALIPSO_C_LISTALL + 1, }; /* NetLabel Generic NETLINK Protocol Functions diff --git a/net/netlabel/netlabel_cipso_v4.c b/net/netlabel/netlabel_cipso_v4.c index 894e6b8f1a8681605dbeefe41fa5a22c3e8a3b43..fa08ee75ac0635c38054629c70ed620b6c4a4def 100644 --- a/net/netlabel/netlabel_cipso_v4.c +++ b/net/netlabel/netlabel_cipso_v4.c @@ -767,6 +767,7 @@ static struct genl_family netlbl_cipsov4_gnl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = netlbl_cipsov4_ops, .n_small_ops = ARRAY_SIZE(netlbl_cipsov4_ops), + .resv_start_op = NLBL_CIPSOV4_C_LISTALL + 1, }; /* diff --git a/net/netlabel/netlabel_mgmt.c b/net/netlabel/netlabel_mgmt.c index 032b7d7b32c7658fa9bcf9d335eaaefe0cb54f7e..689eaa2afbecd73118f9ed2ec10ba2be5ce39c4f 100644 --- a/net/netlabel/netlabel_mgmt.c +++ b/net/netlabel/netlabel_mgmt.c @@ -826,6 +826,7 @@ static struct genl_family netlbl_mgmt_gnl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = netlbl_mgmt_genl_ops, .n_small_ops = ARRAY_SIZE(netlbl_mgmt_genl_ops), + .resv_start_op = NLBL_MGMT_C_VERSION + 1, }; /* diff --git a/net/netlabel/netlabel_unlabeled.c b/net/netlabel/netlabel_unlabeled.c index 0555dffd80e0533ad84cf9bb94cba37cfd0c3c61..9996883bf2b78d3ca0399fc622679e4bb04d5541 100644 --- a/net/netlabel/netlabel_unlabeled.c +++ b/net/netlabel/netlabel_unlabeled.c @@ -1374,6 +1374,7 @@ static struct genl_family netlbl_unlabel_gnl_family __ro_after_init = { .module = THIS_MODULE, .small_ops = netlbl_unlabel_genl_ops, .n_small_ops = ARRAY_SIZE(netlbl_unlabel_genl_ops), + .resv_start_op = NLBL_UNLABEL_C_STATICLISTDEF + 1, }; /* diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 0cd91f813a3bd16da13d139e779a54217b563d22..a662e8a5ff84a658e51d165e3a6da1e182ad6b38 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -2400,6 +2400,69 @@ error_free: } EXPORT_SYMBOL(__netlink_dump_start); +static size_t +netlink_ack_tlv_len(struct netlink_sock *nlk, int err, + const struct netlink_ext_ack *extack) +{ + size_t tlvlen; + + if (!extack || !(nlk->flags & NETLINK_F_EXT_ACK)) + return 0; + + tlvlen = 0; + if (extack->_msg) + tlvlen += nla_total_size(strlen(extack->_msg) + 1); + if (extack->cookie_len) + tlvlen += nla_total_size(extack->cookie_len); + + /* Following attributes are only reported as error (not warning) */ + if (!err) + return tlvlen; + + if (extack->bad_attr) + tlvlen += nla_total_size(sizeof(u32)); + if (extack->policy) + tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy); + if (extack->miss_type) + tlvlen += nla_total_size(sizeof(u32)); + if (extack->miss_nest) + tlvlen += nla_total_size(sizeof(u32)); + + return tlvlen; +} + +static void +netlink_ack_tlv_fill(struct sk_buff *in_skb, struct sk_buff *skb, + struct nlmsghdr *nlh, int err, + const struct netlink_ext_ack *extack) +{ + if (extack->_msg) + WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG, extack->_msg)); + if (extack->cookie_len) + WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE, + extack->cookie_len, extack->cookie)); + + if (!err) + return; + + if (extack->bad_attr && + !WARN_ON((u8 *)extack->bad_attr < in_skb->data || + (u8 *)extack->bad_attr >= in_skb->data + in_skb->len)) + WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS, + (u8 *)extack->bad_attr - (u8 *)nlh)); + if (extack->policy) + netlink_policy_dump_write_attr(skb, extack->policy, + NLMSGERR_ATTR_POLICY); + if (extack->miss_type) + WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_TYPE, + extack->miss_type)); + if (extack->miss_nest && + !WARN_ON((u8 *)extack->miss_nest < in_skb->data || + (u8 *)extack->miss_nest > in_skb->data + in_skb->len)) + WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_NEST, + (u8 *)extack->miss_nest - (u8 *)nlh)); +} + void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err, const struct netlink_ext_ack *extack) { @@ -2407,29 +2470,20 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err, struct nlmsghdr *rep; struct nlmsgerr *errmsg; size_t payload = sizeof(*errmsg); - size_t tlvlen = 0; struct netlink_sock *nlk = nlk_sk(NETLINK_CB(in_skb).sk); unsigned int flags = 0; - bool nlk_has_extack = nlk->flags & NETLINK_F_EXT_ACK; + size_t tlvlen; /* Error messages get the original request appened, unless the user * requests to cap the error message, and get extra error data if * requested. */ - if (nlk_has_extack && extack && extack->_msg) - tlvlen += nla_total_size(strlen(extack->_msg) + 1); - if (err && !(nlk->flags & NETLINK_F_CAP_ACK)) payload += nlmsg_len(nlh); else flags |= NLM_F_CAPPED; - if (err && nlk_has_extack && extack && extack->bad_attr) - tlvlen += nla_total_size(sizeof(u32)); - if (nlk_has_extack && extack && extack->cookie_len) - tlvlen += nla_total_size(extack->cookie_len); - if (err && nlk_has_extack && extack && extack->policy) - tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy); + tlvlen = netlink_ack_tlv_len(nlk, err, extack); if (tlvlen) flags |= NLM_F_ACK_TLVS; @@ -2440,31 +2494,16 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err, return; } - rep = __nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, - NLMSG_ERROR, payload, flags); + rep = nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, + NLMSG_ERROR, payload, flags); errmsg = nlmsg_data(rep); errmsg->error = err; - memcpy(&errmsg->msg, nlh, payload > sizeof(*errmsg) ? nlh->nlmsg_len : sizeof(*nlh)); + unsafe_memcpy(&errmsg->msg, nlh, payload > sizeof(*errmsg) + ? nlh->nlmsg_len : sizeof(*nlh), + /* Bounds checked by the skb layer. */); - if (nlk_has_extack && extack) { - if (extack->_msg) { - WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG, - extack->_msg)); - } - if (err && extack->bad_attr && - !WARN_ON((u8 *)extack->bad_attr < in_skb->data || - (u8 *)extack->bad_attr >= in_skb->data + - in_skb->len)) - WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS, - (u8 *)extack->bad_attr - - (u8 *)nlh)); - if (extack->cookie_len) - WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE, - extack->cookie_len, extack->cookie)); - if (extack->policy) - netlink_policy_dump_write_attr(skb, extack->policy, - NLMSGERR_ATTR_POLICY); - } + if (tlvlen) + netlink_ack_tlv_fill(in_skb, skb, nlh, err, extack); nlmsg_end(skb, rep); diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 57010927e20a805b003d0e31d6e0b24f0207c799..39b7c00e4cef09579adf05f6ccd394f0c6fe8eed 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -739,6 +739,36 @@ out: return err; } +static int genl_header_check(const struct genl_family *family, + struct nlmsghdr *nlh, struct genlmsghdr *hdr, + struct netlink_ext_ack *extack) +{ + u16 flags; + + /* Only for commands added after we started validating */ + if (hdr->cmd < family->resv_start_op) + return 0; + + if (hdr->reserved) { + NL_SET_ERR_MSG(extack, "genlmsghdr.reserved field is not 0"); + return -EINVAL; + } + + /* Old netlink flags have pretty loose semantics, allow only the flags + * consumed by the core where we can enforce the meaning. + */ + flags = nlh->nlmsg_flags; + if ((flags & NLM_F_DUMP) == NLM_F_DUMP) /* DUMP is 2 bits */ + flags &= ~NLM_F_DUMP; + if (flags & ~(NLM_F_REQUEST | NLM_F_ACK | NLM_F_ECHO)) { + NL_SET_ERR_MSG(extack, + "ambiguous or reserved bits set in nlmsg_flags"); + return -EINVAL; + } + + return 0; +} + static int genl_family_rcv_msg(const struct genl_family *family, struct sk_buff *skb, struct nlmsghdr *nlh, @@ -757,6 +787,9 @@ static int genl_family_rcv_msg(const struct genl_family *family, if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen)) return -EINVAL; + if (genl_header_check(family, nlh, hdr, extack)) + return -EINVAL; + if (genl_get_cmd(hdr->cmd, family, &op)) return -EOPNOTSUPP; @@ -1348,6 +1381,7 @@ static struct genl_family genl_ctrl __ro_after_init = { .module = THIS_MODULE, .ops = genl_ctrl_ops, .n_ops = ARRAY_SIZE(genl_ctrl_ops), + .resv_start_op = CTRL_CMD_GETPOLICY + 1, .mcgrps = genl_ctrl_groups, .n_mcgrps = ARRAY_SIZE(genl_ctrl_groups), .id = GENL_ID_CTRL, @@ -1362,7 +1396,7 @@ static int genl_bind(struct net *net, int group) unsigned int id; int ret = 0; - genl_lock_all(); + down_read(&cb_lock); idr_for_each_entry(&genl_fam_idr, family, id) { const struct genl_multicast_group *grp; @@ -1383,7 +1417,7 @@ static int genl_bind(struct net *net, int group) break; } - genl_unlock_all(); + up_read(&cb_lock); return ret; } diff --git a/net/nfc/hci/hcp.c b/net/nfc/hci/hcp.c index 05c60988f59af88c7fcac1dbeb6d52cda3d2abfa..4902f5064098168d07076dc7db1502a051f1c722 100644 --- a/net/nfc/hci/hcp.c +++ b/net/nfc/hci/hcp.c @@ -73,14 +73,12 @@ int nfc_hci_hcp_message_tx(struct nfc_hci_dev *hdev, u8 pipe, if (firstfrag) { firstfrag = false; packet->message.header = HCP_HEADER(type, instruction); - if (ptr) { - memcpy(packet->message.data, ptr, - data_link_len - 1); - ptr += data_link_len - 1; - } } else { - memcpy(&packet->message, ptr, data_link_len); - ptr += data_link_len; + packet->message.header = *ptr++; + } + if (ptr) { + memcpy(packet->message.data, ptr, data_link_len - 1); + ptr += data_link_len - 1; } /* This is the last fragment, set the cb bit */ diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c index 7c62417ccfd78a76937fa7cf7491b5556d5a1603..9d91087b93992bdf90f1f6a4f8960c3eed77d81b 100644 --- a/net/nfc/netlink.c +++ b/net/nfc/netlink.c @@ -1783,6 +1783,7 @@ static struct genl_family nfc_genl_family __ro_after_init = { .module = THIS_MODULE, .ops = nfc_genl_ops, .n_ops = ARRAY_SIZE(nfc_genl_ops), + .resv_start_op = NFC_CMD_DEACTIVATE_TARGET + 1, .mcgrps = nfc_genl_mcgrps, .n_mcgrps = ARRAY_SIZE(nfc_genl_mcgrps), }; diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 868db4669a2912d01433dddc41f3905a5e7bf2bf..ca3ebfdb30231dd48b22cc36d8890c5e4b39223d 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -1033,7 +1033,7 @@ static int sample(struct datapath *dp, struct sk_buff *skb, actions = nla_next(sample_arg, &rem); if ((arg->probability != U32_MAX) && - (!arg->probability || prandom_u32() > arg->probability)) { + (!arg->probability || get_random_u32() > arg->probability)) { if (last) consume_skb(skb); return 0; diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index 4e70df91d0f2a08f36c2b19860cf21924739bafb..c7b10234cf7c48e13c0cc2075dd03cba40348abb 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -1015,7 +1015,8 @@ static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key, * connections which we will commit, we may need to attach * the helper here. */ - if (info->commit && info->helper && !nfct_help(ct)) { + if (!nf_ct_is_confirmed(ct) && info->commit && + info->helper && !nfct_help(ct)) { int err = __nf_ct_try_assign_helper(ct, info->ct, GFP_ATOMIC); if (err) @@ -1982,7 +1983,8 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, } else { struct ovs_ct_limit *ct_limit; - ct_limit = kmalloc(sizeof(*ct_limit), GFP_KERNEL); + ct_limit = kmalloc(sizeof(*ct_limit), + GFP_KERNEL_ACCOUNT); if (!ct_limit) return -ENOMEM; @@ -2252,14 +2254,16 @@ exit_err: static const struct genl_small_ops ct_limit_genl_ops[] = { { .cmd = OVS_CT_LIMIT_CMD_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN - * privilege. */ + .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN + * privilege. + */ .doit = ovs_ct_limit_cmd_set, }, { .cmd = OVS_CT_LIMIT_CMD_DEL, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN - * privilege. */ + .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN + * privilege. + */ .doit = ovs_ct_limit_cmd_del, }, { .cmd = OVS_CT_LIMIT_CMD_GET, @@ -2283,6 +2287,7 @@ struct genl_family dp_ct_limit_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = ct_limit_genl_ops, .n_small_ops = ARRAY_SIZE(ct_limit_genl_ops), + .resv_start_op = OVS_CT_LIMIT_CMD_GET + 1, .mcgrps = &ovs_ct_limit_multicast_group, .n_mcgrps = 1, .module = THIS_MODULE, diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 7e8a39a356271760be552dc5715ac162efac5901..c8a9075ddd0a8cee23a392594659f652f1c386bd 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -252,10 +252,17 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) upcall.mru = OVS_CB(skb)->mru; error = ovs_dp_upcall(dp, skb, key, &upcall, 0); - if (unlikely(error)) - kfree_skb(skb); - else + switch (error) { + case 0: + case -EAGAIN: + case -ERESTARTSYS: + case -EINTR: consume_skb(skb); + break; + default: + kfree_skb(skb); + break; + } stats_counter = &stats->n_missed; goto out; } @@ -551,8 +558,9 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, out: if (err) skb_tx_error(skb); - kfree_skb(user_skb); - kfree_skb(nskb); + consume_skb(user_skb); + consume_skb(nskb); + return err; } @@ -684,6 +692,7 @@ static struct genl_family dp_packet_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = dp_packet_genl_ops, .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops), + .resv_start_op = OVS_PACKET_CMD_EXECUTE + 1, .module = THIS_MODULE, }; @@ -1501,6 +1510,7 @@ static struct genl_family dp_flow_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = dp_flow_genl_ops, .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops), + .resv_start_op = OVS_FLOW_CMD_SET + 1, .mcgrps = &ovs_dp_flow_multicast_group, .n_mcgrps = 1, .module = THIS_MODULE, @@ -1515,6 +1525,7 @@ static size_t ovs_dp_cmd_msg_size(void) msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats)); msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */ msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */ + msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */ return msgsize; } @@ -1526,7 +1537,8 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, struct ovs_header *ovs_header; struct ovs_dp_stats dp_stats; struct ovs_dp_megaflow_stats dp_megaflow_stats; - int err; + struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids); + int err, pids_len; ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family, flags, cmd); @@ -1556,6 +1568,12 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb, ovs_flow_tbl_masks_cache_size(&dp->table))) goto nla_put_failure; + if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) { + pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32); + if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids)) + goto nla_put_failure; + } + genlmsg_end(skb, ovs_header); return 0; @@ -1779,6 +1797,8 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) parms.dp = dp; parms.port_no = OVSP_LOCAL; parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID]; + parms.desired_ifindex = a[OVS_DP_ATTR_IFINDEX] + ? nla_get_u32(a[OVS_DP_ATTR_IFINDEX]) : 0; /* So far only local changes have been made, now need the lock. */ ovs_lock(); @@ -1802,7 +1822,7 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) ovs_dp_reset_user_features(skb, info); } - goto err_unlock_and_destroy_meters; + goto err_destroy_portids; } err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid, @@ -1817,6 +1837,8 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) ovs_notify(&dp_datapath_genl_family, reply, info); return 0; +err_destroy_portids: + kfree(rcu_dereference_raw(dp->upcall_portids)); err_unlock_and_destroy_meters: ovs_unlock(); ovs_meters_exit(dp); @@ -1996,6 +2018,7 @@ static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = { [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 }, [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0, PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)), + [OVS_DP_ATTR_IFINDEX] = {.type = NLA_U32 }, }; static const struct genl_small_ops dp_datapath_genl_ops[] = { @@ -2032,6 +2055,7 @@ static struct genl_family dp_datapath_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = dp_datapath_genl_ops, .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops), + .resv_start_op = OVS_DP_CMD_SET + 1, .mcgrps = &ovs_dp_datapath_multicast_group, .n_mcgrps = 1, .module = THIS_MODULE, @@ -2199,7 +2223,10 @@ static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info) if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] || !a[OVS_VPORT_ATTR_UPCALL_PID]) return -EINVAL; - if (a[OVS_VPORT_ATTR_IFINDEX]) + + parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]); + + if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL) return -EOPNOTSUPP; port_no = a[OVS_VPORT_ATTR_PORT_NO] @@ -2236,11 +2263,12 @@ restart: } parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]); - parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]); parms.options = a[OVS_VPORT_ATTR_OPTIONS]; parms.dp = dp; parms.port_no = port_no; parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID]; + parms.desired_ifindex = a[OVS_VPORT_ATTR_IFINDEX] + ? nla_get_u32(a[OVS_VPORT_ATTR_IFINDEX]) : 0; vport = new_vport(&parms); err = PTR_ERR(vport); diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 4c09cf8a0ab2dcbfe9a14cc2b41d02edad1cfef0..4a07ab094a84edab0811c2839ec5adc712d7cea8 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -3304,7 +3304,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr, /* Disallow subsequent L2.5+ set actions and mpls_pop * actions once the last MPLS label in the packet is - * is popped as there is no check here to ensure that + * popped as there is no check here to ensure that * the new eth type is valid and thus set actions could * write off the end of the packet or otherwise corrupt * it. diff --git a/net/openvswitch/meter.c b/net/openvswitch/meter.c index 04a060ac7fdf6e1f7de8fbf19880870ece8ea520..6e38f68f88c26ca6ef9e0f817efa6c42aff595bd 100644 --- a/net/openvswitch/meter.c +++ b/net/openvswitch/meter.c @@ -343,7 +343,7 @@ static struct dp_meter *dp_meter_create(struct nlattr **a) return ERR_PTR(-EINVAL); /* Allocate and set up the meter before locking anything. */ - meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL); + meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL_ACCOUNT); if (!meter) return ERR_PTR(-ENOMEM); @@ -687,9 +687,9 @@ static const struct genl_small_ops dp_meter_genl_ops[] = { }, { .cmd = OVS_METER_CMD_SET, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN - * privilege. - */ + .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN + * privilege. + */ .doit = ovs_meter_cmd_set, }, { .cmd = OVS_METER_CMD_GET, @@ -699,9 +699,9 @@ static const struct genl_small_ops dp_meter_genl_ops[] = { }, { .cmd = OVS_METER_CMD_DEL, .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, - .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN - * privilege. - */ + .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN + * privilege. + */ .doit = ovs_meter_cmd_del }, }; @@ -720,6 +720,7 @@ struct genl_family dp_meter_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = dp_meter_genl_ops, .n_small_ops = ARRAY_SIZE(dp_meter_genl_ops), + .resv_start_op = OVS_METER_CMD_GET + 1, .mcgrps = &ovs_meter_multicast_group, .n_mcgrps = 1, .module = THIS_MODULE, diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c index 5b2ee9c1c00b11675a770c8e0829d0cbb4c012eb..74c88a6baa43a4995d92bff60588e09dd38f4549 100644 --- a/net/openvswitch/vport-internal_dev.c +++ b/net/openvswitch/vport-internal_dev.c @@ -65,7 +65,7 @@ static int internal_dev_stop(struct net_device *netdev) static void internal_dev_getinfo(struct net_device *netdev, struct ethtool_drvinfo *info) { - strlcpy(info->driver, "openvswitch", sizeof(info->driver)); + strscpy(info->driver, "openvswitch", sizeof(info->driver)); } static const struct ethtool_ops internal_dev_ethtool_ops = { @@ -147,6 +147,7 @@ static struct vport *internal_dev_create(const struct vport_parms *parms) } dev_net_set(vport->dev, ovs_dp_get_net(vport->dp)); + dev->ifindex = parms->desired_ifindex; internal_dev = internal_dev_priv(vport->dev); internal_dev->vport = vport; @@ -189,7 +190,7 @@ static void internal_dev_destroy(struct vport *vport) rtnl_unlock(); } -static netdev_tx_t internal_dev_recv(struct sk_buff *skb) +static int internal_dev_recv(struct sk_buff *skb) { struct net_device *netdev = skb->dev; diff --git a/net/openvswitch/vport.h b/net/openvswitch/vport.h index 9de5030d9801c7065a4cf5478cfc3778891a6535..6ff45e8a0868d099f6ef05191ec3755d460757db 100644 --- a/net/openvswitch/vport.h +++ b/net/openvswitch/vport.h @@ -90,12 +90,14 @@ struct vport { * @type: New vport's type. * @options: %OVS_VPORT_ATTR_OPTIONS attribute from Netlink message, %NULL if * none was supplied. + * @desired_ifindex: New vport's ifindex. * @dp: New vport's datapath. * @port_no: New vport's port number. */ struct vport_parms { const char *name; enum ovs_vport_type type; + int desired_ifindex; struct nlattr *options; /* For ovs_vport_alloc(). */ @@ -130,7 +132,7 @@ struct vport_ops { int (*set_options)(struct vport *, struct nlattr *); int (*get_options)(const struct vport *, struct sk_buff *); - netdev_tx_t (*send) (struct sk_buff *skb); + int (*send)(struct sk_buff *skb); struct module *owner; struct list_head list; }; diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 5cbe07116e04e68251de863307e7194b9e07a789..6ce8dd19f33c3aafef0b9dab648a44fdd3478866 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1350,7 +1350,7 @@ static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb) if (READ_ONCE(history[i]) == rxhash) count++; - victim = prandom_u32() % ROLLOVER_HLEN; + victim = prandom_u32_max(ROLLOVER_HLEN); /* Avoid dirtying the cache line if possible */ if (READ_ONCE(history[victim]) != rxhash) @@ -1905,7 +1905,7 @@ static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev, */ spkt->spkt_family = dev->type; - strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device)); + strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device)); spkt->spkt_protocol = skb->protocol; /* @@ -3565,7 +3565,7 @@ static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr, rcu_read_lock(); dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex)); if (dev) - strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data)); + strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data)); rcu_read_unlock(); return sizeof(*uaddr); @@ -4725,37 +4725,37 @@ static struct pernet_operations packet_net_ops = { static void __exit packet_exit(void) { - unregister_netdevice_notifier(&packet_netdev_notifier); - unregister_pernet_subsys(&packet_net_ops); sock_unregister(PF_PACKET); proto_unregister(&packet_proto); + unregister_netdevice_notifier(&packet_netdev_notifier); + unregister_pernet_subsys(&packet_net_ops); } static int __init packet_init(void) { int rc; - rc = proto_register(&packet_proto, 0); - if (rc) - goto out; - rc = sock_register(&packet_family_ops); - if (rc) - goto out_proto; rc = register_pernet_subsys(&packet_net_ops); if (rc) - goto out_sock; + goto out; rc = register_netdevice_notifier(&packet_netdev_notifier); if (rc) goto out_pernet; + rc = proto_register(&packet_proto, 0); + if (rc) + goto out_notifier; + rc = sock_register(&packet_family_ops); + if (rc) + goto out_proto; return 0; -out_pernet: - unregister_pernet_subsys(&packet_net_ops); -out_sock: - sock_unregister(PF_PACKET); out_proto: proto_unregister(&packet_proto); +out_notifier: + unregister_netdevice_notifier(&packet_netdev_notifier); +out_pernet: + unregister_pernet_subsys(&packet_net_ops); out: return rc; } diff --git a/net/psample/psample.c b/net/psample/psample.c index 118d5d2a81a02308cd15a58fcc426954f111be22..81a794e36f535864869812c2003ff65d4c96efd0 100644 --- a/net/psample/psample.c +++ b/net/psample/psample.c @@ -115,6 +115,7 @@ static struct genl_family psample_nl_family __ro_after_init = { .mcgrps = psample_nl_mcgrps, .small_ops = psample_nl_ops, .n_small_ops = ARRAY_SIZE(psample_nl_ops), + .resv_start_op = PSAMPLE_CMD_GET_GROUP + 1, .n_mcgrps = ARRAY_SIZE(psample_nl_mcgrps), }; diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c index b239120dd9ca691df8a561418dd91dbdaa319a77..3ff6995244e5eb20ee1e1b2457ccb188a2c36592 100644 --- a/net/rds/af_rds.c +++ b/net/rds/af_rds.c @@ -894,7 +894,7 @@ module_exit(rds_exit); u32 rds_gen_num; -static int rds_init(void) +static int __init rds_init(void) { int ret; diff --git a/net/rds/bind.c b/net/rds/bind.c index 5b5fb4ca8d3e5523336aee389acd7ad160b28d2f..97a29172a8eec44b370ae19004f2c99610f75971 100644 --- a/net/rds/bind.c +++ b/net/rds/bind.c @@ -104,7 +104,7 @@ static int rds_add_bound(struct rds_sock *rs, const struct in6_addr *addr, return -EINVAL; last = rover; } else { - rover = max_t(u16, prandom_u32(), 2); + rover = max_t(u16, get_random_u16(), 2); last = rover - 1; } diff --git a/net/rds/message.c b/net/rds/message.c index d74be4e3f3faf5147f851d20b674d935d035594e..44dbc612ef549c0ffb94d563379e3d891c1c38cd 100644 --- a/net/rds/message.c +++ b/net/rds/message.c @@ -354,7 +354,7 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in for (i = 0; i < rm->data.op_nents; ++i) { sg_set_page(&rm->data.op_sg[i], - virt_to_page(page_addrs[i]), + virt_to_page((void *)page_addrs[i]), PAGE_SIZE, 0); } diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c index a9e4ff948a7d69a4c9ade67c482e2f7df25eebdb..d36f3f6b43510dde1ef2e98515b41c6890d76ebf 100644 --- a/net/rds/rdma_transport.c +++ b/net/rds/rdma_transport.c @@ -291,7 +291,7 @@ static void rds_rdma_listen_stop(void) #endif } -static int rds_rdma_init(void) +static int __init rds_rdma_init(void) { int ret; @@ -307,7 +307,7 @@ out: } module_init(rds_rdma_init); -static void rds_rdma_exit(void) +static void __exit rds_rdma_exit(void) { /* stop listening first to ensure no new connections are attempted */ rds_rdma_listen_stop(); diff --git a/net/rds/tcp.c b/net/rds/tcp.c index 73ee2771093d60253d3872cdd5379fac9ba8197e..4444fd82b66df7d54065893dc3d1aa02c3e2ef7c 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -166,10 +166,10 @@ void rds_tcp_reset_callbacks(struct socket *sock, */ atomic_set(&cp->cp_state, RDS_CONN_RESETTING); wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); - lock_sock(osock->sk); /* reset receive side state for rds_tcp_data_recv() for osock */ cancel_delayed_work_sync(&cp->cp_send_w); cancel_delayed_work_sync(&cp->cp_recv_w); + lock_sock(osock->sk); if (tc->t_tinc) { rds_inc_put(&tc->t_tinc->ti_inc); tc->t_tinc = NULL; @@ -712,7 +712,7 @@ static void rds_tcp_exit(void) } module_exit(rds_tcp_exit); -static int rds_tcp_init(void) +static int __init rds_tcp_init(void) { int ret; diff --git a/net/rose/rose_loopback.c b/net/rose/rose_loopback.c index 11c45c8c6c1641781dd733e93e5453c2be75340b..036d92c0ad794889e65aec901c39d97fbc1bb9f2 100644 --- a/net/rose/rose_loopback.c +++ b/net/rose/rose_loopback.c @@ -96,7 +96,8 @@ static void rose_loopback_timer(struct timer_list *unused) } if (frametype == ROSE_CALL_REQUEST) { - if (!rose_loopback_neigh->dev) { + if (!rose_loopback_neigh->dev && + !rose_loopback_neigh->loopback) { kfree_skb(skb); continue; } diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 571436064cd6f8e0bf4b714a6a1c05db488c1122..1ad0ec5afb50ce5817b3e79757240762b7034573 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -782,7 +782,6 @@ void rxrpc_delete_call_timer(struct rxrpc_call *call); */ extern const char *const rxrpc_call_states[]; extern const char *const rxrpc_call_completions[]; -extern unsigned int rxrpc_max_call_lifetime; extern struct kmem_cache *rxrpc_call_jar; struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *, unsigned long); @@ -982,6 +981,7 @@ void rxrpc_send_keepalive(struct rxrpc_peer *); /* * peer_event.c */ +void rxrpc_encap_err_rcv(struct sock *sk, struct sk_buff *skb, unsigned int udp_offset); void rxrpc_error_report(struct sock *); void rxrpc_peer_keepalive_worker(struct work_struct *); diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index f8ecad2b730e8850cc1235cd72362c22cd7f6192..2a93e7b5fbd0514facd5ebc56194b2a343be4152 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -166,7 +166,7 @@ static void rxrpc_resend(struct rxrpc_call *call, unsigned long now_j) _enter("{%d,%d}", call->tx_hard_ack, call->tx_top); now = ktime_get_real(); - max_age = ktime_sub(now, jiffies_to_usecs(call->peer->rto_j)); + max_age = ktime_sub_us(now, jiffies_to_usecs(call->peer->rto_j)); spin_lock_bh(&call->lock); diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index 84d0a41096450cbe8a7aa70441025f4d52c025a6..6401cdf7a62469dad7d2b5b4deb532dafa28634a 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -285,8 +285,10 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, _enter("%p,%lx", rx, p->user_call_ID); limiter = rxrpc_get_call_slot(p, gfp); - if (!limiter) + if (!limiter) { + release_sock(&rx->sk); return ERR_PTR(-ERESTARTSYS); + } call = rxrpc_alloc_client_call(rx, srx, gfp, debug_id); if (IS_ERR(call)) { diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c index 96ecb7356c0fedad3713b628d69f236d6e6684d5..38ea98ff426bdb0de27d2dc631aca259cc3e832b 100644 --- a/net/rxrpc/local_object.c +++ b/net/rxrpc/local_object.c @@ -137,6 +137,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net) tuncfg.encap_type = UDP_ENCAP_RXRPC; tuncfg.encap_rcv = rxrpc_input_packet; + tuncfg.encap_err_rcv = rxrpc_encap_err_rcv; tuncfg.sk_user_data = local; setup_udp_tunnel_sock(net, local->socket, &tuncfg); @@ -405,6 +406,9 @@ static void rxrpc_local_processor(struct work_struct *work) container_of(work, struct rxrpc_local, processor); bool again; + if (local->dead) + return; + trace_rxrpc_local(local->debug_id, rxrpc_local_processing, refcount_read(&local->ref), NULL); diff --git a/net/rxrpc/peer_event.c b/net/rxrpc/peer_event.c index be032850ae8ca042cd3b4ab262e6fc25928fe5a0..32561e9567fe374979c43e4d5f1ae6531e2f2f18 100644 --- a/net/rxrpc/peer_event.c +++ b/net/rxrpc/peer_event.c @@ -16,22 +16,105 @@ #include #include #include +#include #include "ar-internal.h" +static void rxrpc_adjust_mtu(struct rxrpc_peer *, unsigned int); static void rxrpc_store_error(struct rxrpc_peer *, struct sock_exterr_skb *); static void rxrpc_distribute_error(struct rxrpc_peer *, int, enum rxrpc_call_completion); /* - * Find the peer associated with an ICMP packet. + * Find the peer associated with an ICMPv4 packet. */ static struct rxrpc_peer *rxrpc_lookup_peer_icmp_rcu(struct rxrpc_local *local, - const struct sk_buff *skb, + struct sk_buff *skb, + unsigned int udp_offset, + unsigned int *info, struct sockaddr_rxrpc *srx) { - struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); + struct iphdr *ip, *ip0 = ip_hdr(skb); + struct icmphdr *icmp = icmp_hdr(skb); + struct udphdr *udp = (struct udphdr *)(skb->data + udp_offset); - _enter(""); + _enter("%u,%u,%u", ip0->protocol, icmp->type, icmp->code); + + switch (icmp->type) { + case ICMP_DEST_UNREACH: + *info = ntohs(icmp->un.frag.mtu); + fallthrough; + case ICMP_TIME_EXCEEDED: + case ICMP_PARAMETERPROB: + ip = (struct iphdr *)((void *)icmp + 8); + break; + default: + return NULL; + } + + memset(srx, 0, sizeof(*srx)); + srx->transport_type = local->srx.transport_type; + srx->transport_len = local->srx.transport_len; + srx->transport.family = local->srx.transport.family; + + /* Can we see an ICMP4 packet on an ICMP6 listening socket? and vice + * versa? + */ + switch (srx->transport.family) { + case AF_INET: + srx->transport_len = sizeof(srx->transport.sin); + srx->transport.family = AF_INET; + srx->transport.sin.sin_port = udp->dest; + memcpy(&srx->transport.sin.sin_addr, &ip->daddr, + sizeof(struct in_addr)); + break; + +#ifdef CONFIG_AF_RXRPC_IPV6 + case AF_INET6: + srx->transport_len = sizeof(srx->transport.sin); + srx->transport.family = AF_INET; + srx->transport.sin.sin_port = udp->dest; + memcpy(&srx->transport.sin.sin_addr, &ip->daddr, + sizeof(struct in_addr)); + break; +#endif + + default: + WARN_ON_ONCE(1); + return NULL; + } + + _net("ICMP {%pISp}", &srx->transport); + return rxrpc_lookup_peer_rcu(local, srx); +} + +#ifdef CONFIG_AF_RXRPC_IPV6 +/* + * Find the peer associated with an ICMPv6 packet. + */ +static struct rxrpc_peer *rxrpc_lookup_peer_icmp6_rcu(struct rxrpc_local *local, + struct sk_buff *skb, + unsigned int udp_offset, + unsigned int *info, + struct sockaddr_rxrpc *srx) +{ + struct icmp6hdr *icmp = icmp6_hdr(skb); + struct ipv6hdr *ip, *ip0 = ipv6_hdr(skb); + struct udphdr *udp = (struct udphdr *)(skb->data + udp_offset); + + _enter("%u,%u,%u", ip0->nexthdr, icmp->icmp6_type, icmp->icmp6_code); + + switch (icmp->icmp6_type) { + case ICMPV6_DEST_UNREACH: + *info = ntohl(icmp->icmp6_mtu); + fallthrough; + case ICMPV6_PKT_TOOBIG: + case ICMPV6_TIME_EXCEED: + case ICMPV6_PARAMPROB: + ip = (struct ipv6hdr *)((void *)icmp + 8); + break; + default: + return NULL; + } memset(srx, 0, sizeof(*srx)); srx->transport_type = local->srx.transport_type; @@ -41,6 +124,165 @@ static struct rxrpc_peer *rxrpc_lookup_peer_icmp_rcu(struct rxrpc_local *local, /* Can we see an ICMP4 packet on an ICMP6 listening socket? and vice * versa? */ + switch (srx->transport.family) { + case AF_INET: + _net("Rx ICMP6 on v4 sock"); + srx->transport_len = sizeof(srx->transport.sin); + srx->transport.family = AF_INET; + srx->transport.sin.sin_port = udp->dest; + memcpy(&srx->transport.sin.sin_addr, + &ip->daddr.s6_addr32[3], sizeof(struct in_addr)); + break; + case AF_INET6: + _net("Rx ICMP6"); + srx->transport.sin.sin_port = udp->dest; + memcpy(&srx->transport.sin6.sin6_addr, &ip->daddr, + sizeof(struct in6_addr)); + break; + default: + WARN_ON_ONCE(1); + return NULL; + } + + _net("ICMP {%pISp}", &srx->transport); + return rxrpc_lookup_peer_rcu(local, srx); +} +#endif /* CONFIG_AF_RXRPC_IPV6 */ + +/* + * Handle an error received on the local endpoint as a tunnel. + */ +void rxrpc_encap_err_rcv(struct sock *sk, struct sk_buff *skb, + unsigned int udp_offset) +{ + struct sock_extended_err ee; + struct sockaddr_rxrpc srx; + struct rxrpc_local *local; + struct rxrpc_peer *peer; + unsigned int info = 0; + int err; + u8 version = ip_hdr(skb)->version; + u8 type = icmp_hdr(skb)->type; + u8 code = icmp_hdr(skb)->code; + + rcu_read_lock(); + local = rcu_dereference_sk_user_data(sk); + if (unlikely(!local)) { + rcu_read_unlock(); + return; + } + + rxrpc_new_skb(skb, rxrpc_skb_received); + + switch (ip_hdr(skb)->version) { + case IPVERSION: + peer = rxrpc_lookup_peer_icmp_rcu(local, skb, udp_offset, + &info, &srx); + break; +#ifdef CONFIG_AF_RXRPC_IPV6 + case 6: + peer = rxrpc_lookup_peer_icmp6_rcu(local, skb, udp_offset, + &info, &srx); + break; +#endif + default: + rcu_read_unlock(); + return; + } + + if (peer && !rxrpc_get_peer_maybe(peer)) + peer = NULL; + if (!peer) { + rcu_read_unlock(); + return; + } + + memset(&ee, 0, sizeof(ee)); + + switch (version) { + case IPVERSION: + switch (type) { + case ICMP_DEST_UNREACH: + switch (code) { + case ICMP_FRAG_NEEDED: + rxrpc_adjust_mtu(peer, info); + rcu_read_unlock(); + rxrpc_put_peer(peer); + return; + default: + break; + } + + err = EHOSTUNREACH; + if (code <= NR_ICMP_UNREACH) { + /* Might want to do something different with + * non-fatal errors + */ + //harderr = icmp_err_convert[code].fatal; + err = icmp_err_convert[code].errno; + } + break; + + case ICMP_TIME_EXCEEDED: + err = EHOSTUNREACH; + break; + default: + err = EPROTO; + break; + } + + ee.ee_origin = SO_EE_ORIGIN_ICMP; + ee.ee_type = type; + ee.ee_code = code; + ee.ee_errno = err; + break; + +#ifdef CONFIG_AF_RXRPC_IPV6 + case 6: + switch (type) { + case ICMPV6_PKT_TOOBIG: + rxrpc_adjust_mtu(peer, info); + rcu_read_unlock(); + rxrpc_put_peer(peer); + return; + } + + icmpv6_err_convert(type, code, &err); + + if (err == EACCES) + err = EHOSTUNREACH; + + ee.ee_origin = SO_EE_ORIGIN_ICMP6; + ee.ee_type = type; + ee.ee_code = code; + ee.ee_errno = err; + break; +#endif + } + + trace_rxrpc_rx_icmp(peer, &ee, &srx); + + rxrpc_distribute_error(peer, err, RXRPC_CALL_NETWORK_ERROR); + rcu_read_unlock(); + rxrpc_put_peer(peer); +} + +/* + * Find the peer associated with a local error. + */ +static struct rxrpc_peer *rxrpc_lookup_peer_local_rcu(struct rxrpc_local *local, + const struct sk_buff *skb, + struct sockaddr_rxrpc *srx) +{ + struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); + + _enter(""); + + memset(srx, 0, sizeof(*srx)); + srx->transport_type = local->srx.transport_type; + srx->transport_len = local->srx.transport_len; + srx->transport.family = local->srx.transport.family; + switch (srx->transport.family) { case AF_INET: srx->transport_len = sizeof(srx->transport.sin); @@ -104,10 +346,8 @@ static struct rxrpc_peer *rxrpc_lookup_peer_icmp_rcu(struct rxrpc_local *local, /* * Handle an MTU/fragmentation problem. */ -static void rxrpc_adjust_mtu(struct rxrpc_peer *peer, struct sock_exterr_skb *serr) +static void rxrpc_adjust_mtu(struct rxrpc_peer *peer, unsigned int mtu) { - u32 mtu = serr->ee.ee_info; - _net("Rx ICMP Fragmentation Needed (%d)", mtu); /* wind down the local interface MTU */ @@ -148,7 +388,7 @@ void rxrpc_error_report(struct sock *sk) struct sock_exterr_skb *serr; struct sockaddr_rxrpc srx; struct rxrpc_local *local; - struct rxrpc_peer *peer; + struct rxrpc_peer *peer = NULL; struct sk_buff *skb; rcu_read_lock(); @@ -172,41 +412,20 @@ void rxrpc_error_report(struct sock *sk) } rxrpc_new_skb(skb, rxrpc_skb_received); serr = SKB_EXT_ERR(skb); - if (!skb->len && serr->ee.ee_origin == SO_EE_ORIGIN_TIMESTAMPING) { - _leave("UDP empty message"); - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); - return; - } - peer = rxrpc_lookup_peer_icmp_rcu(local, skb, &srx); - if (peer && !rxrpc_get_peer_maybe(peer)) - peer = NULL; - if (!peer) { - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); - _leave(" [no peer]"); - return; - } - - trace_rxrpc_rx_icmp(peer, &serr->ee, &srx); - - if ((serr->ee.ee_origin == SO_EE_ORIGIN_ICMP && - serr->ee.ee_type == ICMP_DEST_UNREACH && - serr->ee.ee_code == ICMP_FRAG_NEEDED)) { - rxrpc_adjust_mtu(peer, serr); - rcu_read_unlock(); - rxrpc_free_skb(skb, rxrpc_skb_freed); - rxrpc_put_peer(peer); - _leave(" [MTU update]"); - return; + if (serr->ee.ee_origin == SO_EE_ORIGIN_LOCAL) { + peer = rxrpc_lookup_peer_local_rcu(local, skb, &srx); + if (peer && !rxrpc_get_peer_maybe(peer)) + peer = NULL; + if (peer) { + trace_rxrpc_rx_icmp(peer, &serr->ee, &srx); + rxrpc_store_error(peer, serr); + } } - rxrpc_store_error(peer, serr); rcu_read_unlock(); rxrpc_free_skb(skb, rxrpc_skb_freed); rxrpc_put_peer(peer); - _leave(""); } diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c index 250f23bc1c076d363fa8d9c3f3e3142f3ef9f7f6..7e39c262fd79ef02b7f35ab584fd6a571d5a595b 100644 --- a/net/rxrpc/recvmsg.c +++ b/net/rxrpc/recvmsg.c @@ -771,46 +771,3 @@ call_complete: goto out; } EXPORT_SYMBOL(rxrpc_kernel_recv_data); - -/** - * rxrpc_kernel_get_reply_time - Get timestamp on first reply packet - * @sock: The socket that the call exists on - * @call: The call to query - * @_ts: Where to put the timestamp - * - * Retrieve the timestamp from the first DATA packet of the reply if it is - * in the ring. Returns true if successful, false if not. - */ -bool rxrpc_kernel_get_reply_time(struct socket *sock, struct rxrpc_call *call, - ktime_t *_ts) -{ - struct sk_buff *skb; - rxrpc_seq_t hard_ack, top, seq; - bool success = false; - - mutex_lock(&call->user_mutex); - - if (READ_ONCE(call->state) != RXRPC_CALL_CLIENT_RECV_REPLY) - goto out; - - hard_ack = call->rx_hard_ack; - if (hard_ack != 0) - goto out; - - seq = hard_ack + 1; - top = smp_load_acquire(&call->rx_top); - if (after(seq, top)) - goto out; - - skb = call->rxtx_buffer[seq & RXRPC_RXTX_BUFF_MASK]; - if (!skb) - goto out; - - *_ts = skb_get_ktime(skb); - success = true; - -out: - mutex_unlock(&call->user_mutex); - return success; -} -EXPORT_SYMBOL(rxrpc_kernel_get_reply_time); diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index 258917a714c813725a9873e8d7bdd654bf0485b1..78fa0524156f121f717291deeb548ec70e24ab66 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -540,7 +540,7 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, * directly into the target buffer. */ sg = _sg; - nsg = skb_shinfo(skb)->nr_frags; + nsg = skb_shinfo(skb)->nr_frags + 1; if (nsg <= 4) { nsg = 4; } else { diff --git a/net/rxrpc/sendmsg.c b/net/rxrpc/sendmsg.c index 1d38e279e2efaadc848fbf666ddf3b10816a5027..3c3a626459debb3e564ef6742d61d4fdbaada6e7 100644 --- a/net/rxrpc/sendmsg.c +++ b/net/rxrpc/sendmsg.c @@ -51,10 +51,7 @@ static int rxrpc_wait_for_tx_window_intr(struct rxrpc_sock *rx, return sock_intr_errno(*timeo); trace_rxrpc_transmit(call, rxrpc_transmit_wait); - mutex_unlock(&call->user_mutex); *timeo = schedule_timeout(*timeo); - if (mutex_lock_interruptible(&call->user_mutex) < 0) - return sock_intr_errno(*timeo); } } @@ -290,37 +287,48 @@ out: static int rxrpc_send_data(struct rxrpc_sock *rx, struct rxrpc_call *call, struct msghdr *msg, size_t len, - rxrpc_notify_end_tx_t notify_end_tx) + rxrpc_notify_end_tx_t notify_end_tx, + bool *_dropped_lock) { struct rxrpc_skb_priv *sp; struct sk_buff *skb; struct sock *sk = &rx->sk; + enum rxrpc_call_state state; long timeo; - bool more; - int ret, copied; + bool more = msg->msg_flags & MSG_MORE; + int ret, copied = 0; timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); /* this should be in poll */ sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); +reload: + ret = -EPIPE; if (sk->sk_shutdown & SEND_SHUTDOWN) - return -EPIPE; - - more = msg->msg_flags & MSG_MORE; - + goto maybe_error; + state = READ_ONCE(call->state); + ret = -ESHUTDOWN; + if (state >= RXRPC_CALL_COMPLETE) + goto maybe_error; + ret = -EPROTO; + if (state != RXRPC_CALL_CLIENT_SEND_REQUEST && + state != RXRPC_CALL_SERVER_ACK_REQUEST && + state != RXRPC_CALL_SERVER_SEND_REPLY) + goto maybe_error; + + ret = -EMSGSIZE; if (call->tx_total_len != -1) { - if (len > call->tx_total_len) - return -EMSGSIZE; - if (!more && len != call->tx_total_len) - return -EMSGSIZE; + if (len - copied > call->tx_total_len) + goto maybe_error; + if (!more && len - copied != call->tx_total_len) + goto maybe_error; } skb = call->tx_pending; call->tx_pending = NULL; rxrpc_see_skb(skb, rxrpc_skb_seen); - copied = 0; do { /* Check to see if there's a ping ACK to reply to. */ if (call->ackr_reason == RXRPC_ACK_PING_RESPONSE) @@ -331,16 +339,8 @@ static int rxrpc_send_data(struct rxrpc_sock *rx, _debug("alloc"); - if (!rxrpc_check_tx_space(call, NULL)) { - ret = -EAGAIN; - if (msg->msg_flags & MSG_DONTWAIT) - goto maybe_error; - ret = rxrpc_wait_for_tx_window(rx, call, - &timeo, - msg->msg_flags & MSG_WAITALL); - if (ret < 0) - goto maybe_error; - } + if (!rxrpc_check_tx_space(call, NULL)) + goto wait_for_space; /* Work out the maximum size of a packet. Assume that * the security header is going to be in the padded @@ -468,6 +468,27 @@ maybe_error: efault: ret = -EFAULT; goto out; + +wait_for_space: + ret = -EAGAIN; + if (msg->msg_flags & MSG_DONTWAIT) + goto maybe_error; + mutex_unlock(&call->user_mutex); + *_dropped_lock = true; + ret = rxrpc_wait_for_tx_window(rx, call, &timeo, + msg->msg_flags & MSG_WAITALL); + if (ret < 0) + goto maybe_error; + if (call->interruptibility == RXRPC_INTERRUPTIBLE) { + if (mutex_lock_interruptible(&call->user_mutex) < 0) { + ret = sock_intr_errno(timeo); + goto maybe_error; + } + } else { + mutex_lock(&call->user_mutex); + } + *_dropped_lock = false; + goto reload; } /* @@ -629,6 +650,7 @@ int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len) enum rxrpc_call_state state; struct rxrpc_call *call; unsigned long now, j; + bool dropped_lock = false; int ret; struct rxrpc_send_params p = { @@ -737,21 +759,13 @@ int rxrpc_do_sendmsg(struct rxrpc_sock *rx, struct msghdr *msg, size_t len) ret = rxrpc_send_abort_packet(call); } else if (p.command != RXRPC_CMD_SEND_DATA) { ret = -EINVAL; - } else if (rxrpc_is_client_call(call) && - state != RXRPC_CALL_CLIENT_SEND_REQUEST) { - /* request phase complete for this client call */ - ret = -EPROTO; - } else if (rxrpc_is_service_call(call) && - state != RXRPC_CALL_SERVER_ACK_REQUEST && - state != RXRPC_CALL_SERVER_SEND_REPLY) { - /* Reply phase not begun or not complete for service call. */ - ret = -EPROTO; } else { - ret = rxrpc_send_data(rx, call, msg, len, NULL); + ret = rxrpc_send_data(rx, call, msg, len, NULL, &dropped_lock); } out_put_unlock: - mutex_unlock(&call->user_mutex); + if (!dropped_lock) + mutex_unlock(&call->user_mutex); error_put: rxrpc_put_call(call, rxrpc_call_put); _leave(" = %d", ret); @@ -779,6 +793,7 @@ int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call, struct msghdr *msg, size_t len, rxrpc_notify_end_tx_t notify_end_tx) { + bool dropped_lock = false; int ret; _enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]); @@ -796,7 +811,7 @@ int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call, case RXRPC_CALL_SERVER_ACK_REQUEST: case RXRPC_CALL_SERVER_SEND_REPLY: ret = rxrpc_send_data(rxrpc_sk(sock->sk), call, msg, len, - notify_end_tx); + notify_end_tx, &dropped_lock); break; case RXRPC_CALL_COMPLETE: read_lock_bh(&call->state_lock); @@ -810,7 +825,8 @@ int rxrpc_kernel_send_data(struct socket *sock, struct rxrpc_call *call, break; } - mutex_unlock(&call->user_mutex); + if (!dropped_lock) + mutex_unlock(&call->user_mutex); _leave(" = %d", ret); return ret; } diff --git a/net/sched/act_api.c b/net/sched/act_api.c index 817065aa28331c2a9f886bdb67db55b79348a0ed..9b31a10cc63997f8a38c916d7bdf49ddf2f61c36 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -676,6 +676,31 @@ int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index) } EXPORT_SYMBOL(tcf_idr_search); +static int __tcf_generic_walker(struct net *net, struct sk_buff *skb, + struct netlink_callback *cb, int type, + const struct tc_action_ops *ops, + struct netlink_ext_ack *extack) +{ + struct tc_action_net *tn = net_generic(net, ops->net_id); + + if (unlikely(ops->walk)) + return ops->walk(net, skb, cb, type, ops, extack); + + return tcf_generic_walker(tn, skb, cb, type, ops, extack); +} + +static int __tcf_idr_search(struct net *net, + const struct tc_action_ops *ops, + struct tc_action **a, u32 index) +{ + struct tc_action_net *tn = net_generic(net, ops->net_id); + + if (unlikely(ops->lookup)) + return ops->lookup(net, a, index); + + return tcf_idr_search(tn, a, index); +} + static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) { struct tc_action *p; @@ -926,7 +951,7 @@ int tcf_register_action(struct tc_action_ops *act, struct tc_action_ops *a; int ret; - if (!act->act || !act->dump || !act->init || !act->walk || !act->lookup) + if (!act->act || !act->dump || !act->init) return -EINVAL; /* We have to register pernet ops before making the action ops visible, @@ -1638,7 +1663,7 @@ static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla, goto err_out; } err = -ENOENT; - if (ops->lookup(net, &a, index) == 0) { + if (__tcf_idr_search(net, ops, &a, index) == 0) { NL_SET_ERR_MSG(extack, "TC action with specified index not found"); goto err_mod; } @@ -1703,7 +1728,7 @@ static int tca_action_flush(struct net *net, struct nlattr *nla, goto out_module_put; } - err = ops->walk(net, skb, &dcb, RTM_DELACTION, ops, extack); + err = __tcf_generic_walker(net, skb, &dcb, RTM_DELACTION, ops, extack); if (err <= 0) { nla_nest_cancel(skb, nest); goto out_module_put; @@ -2121,7 +2146,7 @@ static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb) if (nest == NULL) goto out_module_put; - ret = a_o->walk(net, skb, cb, RTM_GETACTION, a_o, NULL); + ret = __tcf_generic_walker(net, skb, cb, RTM_GETACTION, a_o, NULL); if (ret < 0) goto out_module_put; diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index fea2d78b9ddca9cedd8d80f6c7dcff0573a496df..b79eee44e24eb4bddf456ad82fdd905c4d8cfd5f 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -29,7 +29,6 @@ struct tcf_bpf_cfg { bool is_ebpf; }; -static unsigned int bpf_net_id; static struct tc_action_ops act_bpf_ops; static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act, @@ -280,7 +279,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, bpf_net_id); + struct tc_action_net *tn = net_generic(net, act_bpf_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_ACT_BPF_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -334,7 +333,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla, is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS]; is_ebpf = tb[TCA_ACT_BPF_FD]; - if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) { + if (is_bpf == is_ebpf) { ret = -EINVAL; goto put_chain; } @@ -390,23 +389,6 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } -static int tcf_bpf_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, bpf_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_bpf_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, bpf_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -415,27 +397,25 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, - .walk = tcf_bpf_walker, - .lookup = tcf_bpf_search, .size = sizeof(struct tcf_bpf), }; static __net_init int bpf_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, bpf_net_id); + struct tc_action_net *tn = net_generic(net, act_bpf_ops.net_id); return tc_action_net_init(net, tn, &act_bpf_ops); } static void __net_exit bpf_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, bpf_net_id); + tc_action_net_exit(net_list, act_bpf_ops.net_id); } static struct pernet_operations bpf_net_ops = { .init = bpf_init_net, .exit_batch = bpf_exit_net, - .id = &bpf_net_id, + .id = &act_bpf_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c index 09e2aafc8943b04ed811f63b98fbae5f69ec5d05..66b143bb04ac94935b3999691abc1273f252e7b5 100644 --- a/net/sched/act_connmark.c +++ b/net/sched/act_connmark.c @@ -25,7 +25,6 @@ #include #include -static unsigned int connmark_net_id; static struct tc_action_ops act_connmark_ops; static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a, @@ -99,7 +98,7 @@ static int tcf_connmark_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, connmark_net_id); + struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id); struct nlattr *tb[TCA_CONNMARK_MAX + 1]; bool bind = flags & TCA_ACT_FLAGS_BIND; struct tcf_chain *goto_ch = NULL; @@ -200,23 +199,6 @@ nla_put_failure: return -1; } -static int tcf_connmark_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, connmark_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_connmark_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, connmark_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_connmark_ops = { .kind = "connmark", .id = TCA_ID_CONNMARK, @@ -224,27 +206,25 @@ static struct tc_action_ops act_connmark_ops = { .act = tcf_connmark_act, .dump = tcf_connmark_dump, .init = tcf_connmark_init, - .walk = tcf_connmark_walker, - .lookup = tcf_connmark_search, .size = sizeof(struct tcf_connmark_info), }; static __net_init int connmark_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, connmark_net_id); + struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id); return tc_action_net_init(net, tn, &act_connmark_ops); } static void __net_exit connmark_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, connmark_net_id); + tc_action_net_exit(net_list, act_connmark_ops.net_id); } static struct pernet_operations connmark_net_ops = { .init = connmark_init_net, .exit_batch = connmark_exit_net, - .id = &connmark_net_id, + .id = &act_connmark_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 22847ee009efaf36967a29b1f19ae6ab232614bc..1366adf9b9091ae0429c6d6581e6a38cbbe06335 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -37,7 +37,6 @@ static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = { [TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), }, }; -static unsigned int csum_net_id; static struct tc_action_ops act_csum_ops; static int tcf_csum_init(struct net *net, struct nlattr *nla, @@ -45,7 +44,7 @@ static int tcf_csum_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, csum_net_id); + struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct tcf_csum_params *params_new; struct nlattr *tb[TCA_CSUM_MAX + 1]; @@ -673,23 +672,6 @@ static void tcf_csum_cleanup(struct tc_action *a) kfree_rcu(params, rcu); } -static int tcf_csum_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, csum_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_csum_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, csum_net_id); - - return tcf_idr_search(tn, a, index); -} - static size_t tcf_csum_get_fill_size(const struct tc_action *act) { return nla_total_size(sizeof(struct tc_csum)); @@ -722,8 +704,6 @@ static struct tc_action_ops act_csum_ops = { .dump = tcf_csum_dump, .init = tcf_csum_init, .cleanup = tcf_csum_cleanup, - .walk = tcf_csum_walker, - .lookup = tcf_csum_search, .get_fill_size = tcf_csum_get_fill_size, .offload_act_setup = tcf_csum_offload_act_setup, .size = sizeof(struct tcf_csum), @@ -731,20 +711,20 @@ static struct tc_action_ops act_csum_ops = { static __net_init int csum_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, csum_net_id); + struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id); return tc_action_net_init(net, tn, &act_csum_ops); } static void __net_exit csum_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, csum_net_id); + tc_action_net_exit(net_list, act_csum_ops.net_id); } static struct pernet_operations csum_net_ops = { .init = csum_init_net, .exit_batch = csum_exit_net, - .id = &csum_net_id, + .id = &act_csum_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index d55afb8d14be8d25453cb16742242edaeaf9e94e..b38d91d6b249b475ffc10303ad5e2c49f9497e00 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -649,7 +649,6 @@ static void tcf_ct_flow_tables_uninit(void) } static struct tc_action_ops act_ct_ops; -static unsigned int ct_net_id; struct tc_ct_action_net { struct tc_action_net tn; /* Must be first */ @@ -697,7 +696,6 @@ drop_ct: static int tcf_ct_skb_network_trim(struct sk_buff *skb, int family) { unsigned int len; - int err; switch (family) { case NFPROTO_IPV4: @@ -711,9 +709,7 @@ static int tcf_ct_skb_network_trim(struct sk_buff *skb, int family) len = skb->len; } - err = pskb_trim_rcsum(skb, len); - - return err; + return pskb_trim_rcsum(skb, len); } static u8 tcf_ct_skb_nf_family(struct sk_buff *skb) @@ -1255,7 +1251,7 @@ static int tcf_ct_fill_params(struct net *net, struct nlattr **tb, struct netlink_ext_ack *extack) { - struct tc_ct_action_net *tn = net_generic(net, ct_net_id); + struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id); struct nf_conntrack_zone zone; struct nf_conn *tmpl; int err; @@ -1330,7 +1326,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, ct_net_id); + struct tc_action_net *tn = net_generic(net, act_ct_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct tcf_ct_params *params = NULL; struct nlattr *tb[TCA_CT_MAX + 1]; @@ -1394,7 +1390,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, err = tcf_ct_flow_table_get(net, params); if (err) - goto cleanup; + goto cleanup_params; spin_lock_bh(&c->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); @@ -1409,6 +1405,9 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla, return res; +cleanup_params: + if (params->tmpl) + nf_ct_put(params->tmpl); cleanup: if (goto_ch) tcf_chain_put_by_act(goto_ch); @@ -1558,23 +1557,6 @@ nla_put_failure: return -1; } -static int tcf_ct_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, ct_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_ct_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, ct_net_id); - - return tcf_idr_search(tn, a, index); -} - static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets, u64 drops, u64 lastuse, bool hw) { @@ -1613,8 +1595,6 @@ static struct tc_action_ops act_ct_ops = { .dump = tcf_ct_dump, .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, - .walk = tcf_ct_walker, - .lookup = tcf_ct_search, .stats_update = tcf_stats_update, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), @@ -1623,7 +1603,7 @@ static struct tc_action_ops act_ct_ops = { static __net_init int ct_init_net(struct net *net) { unsigned int n_bits = sizeof_field(struct tcf_ct_params, labels) * 8; - struct tc_ct_action_net *tn = net_generic(net, ct_net_id); + struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id); if (nf_connlabels_get(net, n_bits - 1)) { tn->labels = false; @@ -1641,20 +1621,20 @@ static void __net_exit ct_exit_net(struct list_head *net_list) rtnl_lock(); list_for_each_entry(net, net_list, exit_list) { - struct tc_ct_action_net *tn = net_generic(net, ct_net_id); + struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id); if (tn->labels) nf_connlabels_put(net); } rtnl_unlock(); - tc_action_net_exit(net_list, ct_net_id); + tc_action_net_exit(net_list, act_ct_ops.net_id); } static struct pernet_operations ct_net_ops = { .init = ct_init_net, .exit_batch = ct_exit_net, - .id = &ct_net_id, + .id = &act_ct_ops.net_id, .size = sizeof(struct tc_ct_action_net), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 0281e45987a471d731530d76c38c0ad2a85ddc44..d4102f0a9abd133092f801b3a33e6bb3f8c269f7 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -25,7 +25,6 @@ #include static struct tc_action_ops act_ctinfo_ops; -static unsigned int ctinfo_net_id; static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca, struct tcf_ctinfo_params *cp, @@ -157,7 +156,7 @@ static int tcf_ctinfo_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, ctinfo_net_id); + struct tc_action_net *tn = net_generic(net, act_ctinfo_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; u32 dscpmask = 0, dscpstatemask, index; struct nlattr *tb[TCA_CTINFO_MAX + 1]; @@ -342,23 +341,6 @@ nla_put_failure: return -1; } -static int tcf_ctinfo_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, ctinfo_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_ctinfo_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, ctinfo_net_id); - - return tcf_idr_search(tn, a, index); -} - static void tcf_ctinfo_cleanup(struct tc_action *a) { struct tcf_ctinfo *ci = to_ctinfo(a); @@ -377,27 +359,25 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, - .walk = tcf_ctinfo_walker, - .lookup = tcf_ctinfo_search, .size = sizeof(struct tcf_ctinfo), }; static __net_init int ctinfo_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, ctinfo_net_id); + struct tc_action_net *tn = net_generic(net, act_ctinfo_ops.net_id); return tc_action_net_init(net, tn, &act_ctinfo_ops); } static void __net_exit ctinfo_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, ctinfo_net_id); + tc_action_net_exit(net_list, act_ctinfo_ops.net_id); } static struct pernet_operations ctinfo_net_ops = { .init = ctinfo_init_net, .exit_batch = ctinfo_exit_net, - .id = &ctinfo_net_id, + .id = &act_ctinfo_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index ac29d10652321dc9c43614f35f0ef01b5b80c578..62d682b96b88527314387c9180df2ffc9371d160 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -19,14 +19,13 @@ #include #include -static unsigned int gact_net_id; static struct tc_action_ops act_gact_ops; #ifdef CONFIG_GACT_PROB static int gact_net_rand(struct tcf_gact *gact) { smp_rmb(); /* coupled with smp_wmb() in tcf_gact_init() */ - if (prandom_u32() % gact->tcfg_pval) + if (prandom_u32_max(gact->tcfg_pval)) return gact->tcf_action; return gact->tcfg_paction; } @@ -55,7 +54,7 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, gact_net_id); + struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_GACT_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -222,23 +221,6 @@ nla_put_failure: return -1; } -static int tcf_gact_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, gact_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_gact_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, gact_net_id); - - return tcf_idr_search(tn, a, index); -} - static size_t tcf_gact_get_fill_size(const struct tc_action *act) { size_t sz = nla_total_size(sizeof(struct tc_gact)); /* TCA_GACT_PARMS */ @@ -308,8 +290,6 @@ static struct tc_action_ops act_gact_ops = { .stats_update = tcf_gact_stats_update, .dump = tcf_gact_dump, .init = tcf_gact_init, - .walk = tcf_gact_walker, - .lookup = tcf_gact_search, .get_fill_size = tcf_gact_get_fill_size, .offload_act_setup = tcf_gact_offload_act_setup, .size = sizeof(struct tcf_gact), @@ -317,20 +297,20 @@ static struct tc_action_ops act_gact_ops = { static __net_init int gact_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, gact_net_id); + struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id); return tc_action_net_init(net, tn, &act_gact_ops); } static void __net_exit gact_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, gact_net_id); + tc_action_net_exit(net_list, act_gact_ops.net_id); } static struct pernet_operations gact_net_ops = { .init = gact_init_net, .exit_batch = gact_exit_net, - .id = &gact_net_id, + .id = &act_gact_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c index fd515527473324085781385a087d5e0f6bbd22c1..3049878e731538408142c26041fc423453ba6126 100644 --- a/net/sched/act_gate.c +++ b/net/sched/act_gate.c @@ -15,7 +15,6 @@ #include #include -static unsigned int gate_net_id; static struct tc_action_ops act_gate_ops; static ktime_t gate_get_time(struct tcf_gate *gact) @@ -298,7 +297,7 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, gate_net_id); + struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id); enum tk_offsets tk_offset = TK_OFFS_TAI; bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_GATE_MAX + 1]; @@ -565,16 +564,6 @@ nla_put_failure: return -1; } -static int tcf_gate_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, gate_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets, u64 drops, u64 lastuse, bool hw) { @@ -585,13 +574,6 @@ static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets, tm->lastuse = max_t(u64, tm->lastuse, lastuse); } -static int tcf_gate_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, gate_net_id); - - return tcf_idr_search(tn, a, index); -} - static size_t tcf_gate_get_fill_size(const struct tc_action *act) { return nla_total_size(sizeof(struct tc_gate)); @@ -654,30 +636,28 @@ static struct tc_action_ops act_gate_ops = { .dump = tcf_gate_dump, .init = tcf_gate_init, .cleanup = tcf_gate_cleanup, - .walk = tcf_gate_walker, .stats_update = tcf_gate_stats_update, .get_fill_size = tcf_gate_get_fill_size, - .lookup = tcf_gate_search, .offload_act_setup = tcf_gate_offload_act_setup, .size = sizeof(struct tcf_gate), }; static __net_init int gate_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, gate_net_id); + struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id); return tc_action_net_init(net, tn, &act_gate_ops); } static void __net_exit gate_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, gate_net_id); + tc_action_net_exit(net_list, act_gate_ops.net_id); } static struct pernet_operations gate_net_ops = { .init = gate_init_net, .exit_batch = gate_exit_net, - .id = &gate_net_id, + .id = &act_gate_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 41ba55e60b1bf153a2f42455f598e1f20aad4cab..41d63b33461dc9242009832d5cf37143fafb4a63 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -30,7 +30,6 @@ #include #include -static unsigned int ife_net_id; static int max_metacnt = IFE_META_MAX + 1; static struct tc_action_ops act_ife_ops; @@ -482,7 +481,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, ife_net_id); + struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_IFE_MAX + 1]; struct nlattr *tb2[IFE_META_MAX + 1]; @@ -878,23 +877,6 @@ static int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, return tcf_ife_decode(skb, a, res); } -static int tcf_ife_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, ife_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_ife_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, ife_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -903,27 +885,25 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, - .walk = tcf_ife_walker, - .lookup = tcf_ife_search, .size = sizeof(struct tcf_ife_info), }; static __net_init int ife_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, ife_net_id); + struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id); return tc_action_net_init(net, tn, &act_ife_ops); } static void __net_exit ife_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, ife_net_id); + tc_action_net_exit(net_list, act_ife_ops.net_id); } static struct pernet_operations ife_net_ops = { .init = ife_init_net, .exit_batch = ife_exit_net, - .id = &ife_net_id, + .id = &act_ife_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c index 2f3d507c24a1fbfcc242dc4ac21a0d12cf4b0a11..1625e103741614d6d2b871a862108ceab3f5a22a 100644 --- a/net/sched/act_ipt.c +++ b/net/sched/act_ipt.c @@ -24,10 +24,7 @@ #include -static unsigned int ipt_net_id; static struct tc_action_ops act_ipt_ops; - -static unsigned int xt_net_id; static struct tc_action_ops act_xt_ops; static int ipt_init_target(struct net *net, struct xt_entry_target *t, @@ -206,8 +203,8 @@ static int tcf_ipt_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - return __tcf_ipt_init(net, ipt_net_id, nla, est, a, &act_ipt_ops, - tp, flags); + return __tcf_ipt_init(net, act_ipt_ops.net_id, nla, est, + a, &act_ipt_ops, tp, flags); } static int tcf_xt_init(struct net *net, struct nlattr *nla, @@ -215,8 +212,8 @@ static int tcf_xt_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - return __tcf_ipt_init(net, xt_net_id, nla, est, a, &act_xt_ops, - tp, flags); + return __tcf_ipt_init(net, act_xt_ops.net_id, nla, est, + a, &act_xt_ops, tp, flags); } static int tcf_ipt_act(struct sk_buff *skb, const struct tc_action *a, @@ -316,23 +313,6 @@ nla_put_failure: return -1; } -static int tcf_ipt_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, ipt_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_ipt_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, ipt_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_ipt_ops = { .kind = "ipt", .id = TCA_ID_IPT, @@ -341,47 +321,28 @@ static struct tc_action_ops act_ipt_ops = { .dump = tcf_ipt_dump, .cleanup = tcf_ipt_release, .init = tcf_ipt_init, - .walk = tcf_ipt_walker, - .lookup = tcf_ipt_search, .size = sizeof(struct tcf_ipt), }; static __net_init int ipt_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, ipt_net_id); + struct tc_action_net *tn = net_generic(net, act_ipt_ops.net_id); return tc_action_net_init(net, tn, &act_ipt_ops); } static void __net_exit ipt_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, ipt_net_id); + tc_action_net_exit(net_list, act_ipt_ops.net_id); } static struct pernet_operations ipt_net_ops = { .init = ipt_init_net, .exit_batch = ipt_exit_net, - .id = &ipt_net_id, + .id = &act_ipt_ops.net_id, .size = sizeof(struct tc_action_net), }; -static int tcf_xt_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, xt_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_xt_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, xt_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_xt_ops = { .kind = "xt", .id = TCA_ID_XT, @@ -390,27 +351,25 @@ static struct tc_action_ops act_xt_ops = { .dump = tcf_ipt_dump, .cleanup = tcf_ipt_release, .init = tcf_xt_init, - .walk = tcf_xt_walker, - .lookup = tcf_xt_search, .size = sizeof(struct tcf_ipt), }; static __net_init int xt_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, xt_net_id); + struct tc_action_net *tn = net_generic(net, act_xt_ops.net_id); return tc_action_net_init(net, tn, &act_xt_ops); } static void __net_exit xt_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, xt_net_id); + tc_action_net_exit(net_list, act_xt_ops.net_id); } static struct pernet_operations xt_net_ops = { .init = xt_init_net, .exit_batch = xt_exit_net, - .id = &xt_net_id, + .id = &act_xt_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index a1d70cf86843ff1a1138024ceefaf012e6bb7206..b8ad6ae282c02f2c194ffa3d1e3064963d695367 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -86,7 +86,6 @@ static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = { [TCA_MIRRED_PARMS] = { .len = sizeof(struct tc_mirred) }, }; -static unsigned int mirred_net_id; static struct tc_action_ops act_mirred_ops; static int tcf_mirred_init(struct net *net, struct nlattr *nla, @@ -94,7 +93,7 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, mirred_net_id); + struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_MIRRED_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -306,8 +305,7 @@ static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a, /* let's the caller reinsert the packet, if possible */ if (use_reinsert) { - res->ingress = want_ingress; - err = tcf_mirred_forward(res->ingress, skb); + err = tcf_mirred_forward(want_ingress, skb); if (err) tcf_action_inc_overlimit_qstats(&m->common); __this_cpu_dec(mirred_rec_level); @@ -373,23 +371,6 @@ nla_put_failure: return -1; } -static int tcf_mirred_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, mirred_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_mirred_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, mirred_net_id); - - return tcf_idr_search(tn, a, index); -} - static int mirred_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { @@ -510,8 +491,6 @@ static struct tc_action_ops act_mirred_ops = { .dump = tcf_mirred_dump, .cleanup = tcf_mirred_release, .init = tcf_mirred_init, - .walk = tcf_mirred_walker, - .lookup = tcf_mirred_search, .get_fill_size = tcf_mirred_get_fill_size, .offload_act_setup = tcf_mirred_offload_act_setup, .size = sizeof(struct tcf_mirred), @@ -520,20 +499,20 @@ static struct tc_action_ops act_mirred_ops = { static __net_init int mirred_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, mirred_net_id); + struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id); return tc_action_net_init(net, tn, &act_mirred_ops); } static void __net_exit mirred_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, mirred_net_id); + tc_action_net_exit(net_list, act_mirred_ops.net_id); } static struct pernet_operations mirred_net_ops = { .init = mirred_init_net, .exit_batch = mirred_exit_net, - .id = &mirred_net_id, + .id = &act_mirred_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index adabeccb63e1d2009bc2d6418c4f4e7f6dd309b5..8ad25cc8ccd55dcbe80dfe47fb18f04e197d94d6 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -15,7 +15,6 @@ #include #include -static unsigned int mpls_net_id; static struct tc_action_ops act_mpls_ops; #define ACT_MPLS_TTL_DEFAULT 255 @@ -155,7 +154,7 @@ static int tcf_mpls_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, mpls_net_id); + struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_MPLS_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -367,23 +366,6 @@ nla_put_failure: return -EMSGSIZE; } -static int tcf_mpls_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, mpls_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_mpls_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, mpls_net_id); - - return tcf_idr_search(tn, a, index); -} - static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data, u32 *index_inc, bool bind, struct netlink_ext_ack *extack) @@ -451,28 +433,26 @@ static struct tc_action_ops act_mpls_ops = { .dump = tcf_mpls_dump, .init = tcf_mpls_init, .cleanup = tcf_mpls_cleanup, - .walk = tcf_mpls_walker, - .lookup = tcf_mpls_search, .offload_act_setup = tcf_mpls_offload_act_setup, .size = sizeof(struct tcf_mpls), }; static __net_init int mpls_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, mpls_net_id); + struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id); return tc_action_net_init(net, tn, &act_mpls_ops); } static void __net_exit mpls_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, mpls_net_id); + tc_action_net_exit(net_list, act_mpls_ops.net_id); } static struct pernet_operations mpls_net_ops = { .init = mpls_init_net, .exit_batch = mpls_exit_net, - .id = &mpls_net_id, + .id = &act_mpls_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c index 2a39b3729e8449a45f0e1630e3102d85a8a52acf..9265145f104046e419db00d66f741232e4a0f2b5 100644 --- a/net/sched/act_nat.c +++ b/net/sched/act_nat.c @@ -26,7 +26,6 @@ #include -static unsigned int nat_net_id; static struct tc_action_ops act_nat_ops; static const struct nla_policy nat_policy[TCA_NAT_MAX + 1] = { @@ -37,7 +36,7 @@ static int tcf_nat_init(struct net *net, struct nlattr *nla, struct nlattr *est, struct tc_action **a, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, nat_net_id); + struct tc_action_net *tn = net_generic(net, act_nat_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_NAT_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -289,23 +288,6 @@ nla_put_failure: return -1; } -static int tcf_nat_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, nat_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_nat_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, nat_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_nat_ops = { .kind = "nat", .id = TCA_ID_NAT, @@ -313,27 +295,25 @@ static struct tc_action_ops act_nat_ops = { .act = tcf_nat_act, .dump = tcf_nat_dump, .init = tcf_nat_init, - .walk = tcf_nat_walker, - .lookup = tcf_nat_search, .size = sizeof(struct tcf_nat), }; static __net_init int nat_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, nat_net_id); + struct tc_action_net *tn = net_generic(net, act_nat_ops.net_id); return tc_action_net_init(net, tn, &act_nat_ops); } static void __net_exit nat_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, nat_net_id); + tc_action_net_exit(net_list, act_nat_ops.net_id); } static struct pernet_operations nat_net_ops = { .init = nat_init_net, .exit_batch = nat_exit_net, - .id = &nat_net_id, + .id = &act_nat_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 823ee643371c2642facdadd835a52fd4cac3478c..94ed5857ce678419c48028c6c7704a0ec6eb5d9c 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -21,7 +21,6 @@ #include #include -static unsigned int pedit_net_id; static struct tc_action_ops act_pedit_ops; static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { @@ -139,7 +138,7 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, pedit_net_id); + struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_PEDIT_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -492,23 +491,6 @@ nla_put_failure: return -1; } -static int tcf_pedit_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, pedit_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_pedit_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, pedit_net_id); - - return tcf_idr_search(tn, a, index); -} - static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, u32 *index_inc, bool bind, struct netlink_ext_ack *extack) @@ -553,28 +535,26 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, - .walk = tcf_pedit_walker, - .lookup = tcf_pedit_search, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; static __net_init int pedit_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, pedit_net_id); + struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); return tc_action_net_init(net, tn, &act_pedit_ops); } static void __net_exit pedit_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, pedit_net_id); + tc_action_net_exit(net_list, act_pedit_ops.net_id); } static struct pernet_operations pedit_net_ops = { .init = pedit_init_net, .exit_batch = pedit_exit_net, - .id = &pedit_net_id, + .id = &act_pedit_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index b759628a47c204d49aa6caca5031451809ec99fd..0adb26e366a7be8058e6a169511f53d71bd26e78 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -22,19 +22,8 @@ /* Each policer is serialized by its individual spinlock */ -static unsigned int police_net_id; static struct tc_action_ops act_police_ops; -static int tcf_police_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, police_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = { [TCA_POLICE_RATE] = { .len = TC_RTAB_SIZE }, [TCA_POLICE_PEAKRATE] = { .len = TC_RTAB_SIZE }, @@ -58,7 +47,7 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, struct tc_police *parm; struct tcf_police *police; struct qdisc_rate_table *R_tab = NULL, *P_tab = NULL; - struct tc_action_net *tn = net_generic(net, police_net_id); + struct tc_action_net *tn = net_generic(net, act_police_ops.net_id); struct tcf_police_params *new; bool exists = false; u32 index; @@ -412,13 +401,6 @@ nla_put_failure: return -1; } -static int tcf_police_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, police_net_id); - - return tcf_idr_search(tn, a, index); -} - static int tcf_police_act_to_flow_act(int tc_act, u32 *extval, struct netlink_ext_ack *extack) { @@ -513,8 +495,6 @@ static struct tc_action_ops act_police_ops = { .act = tcf_police_act, .dump = tcf_police_dump, .init = tcf_police_init, - .walk = tcf_police_walker, - .lookup = tcf_police_search, .cleanup = tcf_police_cleanup, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), @@ -522,20 +502,20 @@ static struct tc_action_ops act_police_ops = { static __net_init int police_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, police_net_id); + struct tc_action_net *tn = net_generic(net, act_police_ops.net_id); return tc_action_net_init(net, tn, &act_police_ops); } static void __net_exit police_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, police_net_id); + tc_action_net_exit(net_list, act_police_ops.net_id); } static struct pernet_operations police_net_ops = { .init = police_init_net, .exit_batch = police_exit_net, - .id = &police_net_id, + .id = &act_police_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2f7f5e44d28c9875f397aa0eaec946222c3042a0..7a25477f5d996aa633c15f5f82a65ef45aae7203 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -23,7 +23,6 @@ #include -static unsigned int sample_net_id; static struct tc_action_ops act_sample_ops; static const struct nla_policy sample_policy[TCA_SAMPLE_MAX + 1] = { @@ -38,7 +37,7 @@ static int tcf_sample_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, sample_net_id); + struct tc_action_net *tn = net_generic(net, act_sample_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_SAMPLE_MAX + 1]; struct psample_group *psample_group; @@ -169,7 +168,7 @@ static int tcf_sample_act(struct sk_buff *skb, const struct tc_action *a, psample_group = rcu_dereference_bh(s->psample_group); /* randomly sample packets according to rate */ - if (psample_group && (prandom_u32() % s->rate == 0)) { + if (psample_group && (prandom_u32_max(s->rate) == 0)) { if (!skb_at_tc_ingress(skb)) { md.in_ifindex = skb->skb_iif; md.out_ifindex = skb->dev->ifindex; @@ -241,23 +240,6 @@ nla_put_failure: return -1; } -static int tcf_sample_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, sample_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_sample_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, sample_net_id); - - return tcf_idr_search(tn, a, index); -} - static void tcf_psample_group_put(void *priv) { struct psample_group *group = priv; @@ -321,8 +303,6 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, - .walk = tcf_sample_walker, - .lookup = tcf_sample_search, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), @@ -330,20 +310,20 @@ static struct tc_action_ops act_sample_ops = { static __net_init int sample_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, sample_net_id); + struct tc_action_net *tn = net_generic(net, act_sample_ops.net_id); return tc_action_net_init(net, tn, &act_sample_ops); } static void __net_exit sample_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, sample_net_id); + tc_action_net_exit(net_list, act_sample_ops.net_id); } static struct pernet_operations sample_net_ops = { .init = sample_init_net, .exit_batch = sample_exit_net, - .id = &sample_net_id, + .id = &act_sample_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c index 8c1d60bde93e09595bac51625b86e022e88afb58..18d3761354611a6eda6d1fac376e04e67910bdc3 100644 --- a/net/sched/act_simple.c +++ b/net/sched/act_simple.c @@ -18,7 +18,6 @@ #include #include -static unsigned int simp_net_id; static struct tc_action_ops act_simp_ops; #define SIMP_MAX_DATA 32 @@ -89,7 +88,7 @@ static int tcf_simp_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, simp_net_id); + struct tc_action_net *tn = net_generic(net, act_simp_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_DEF_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -198,23 +197,6 @@ nla_put_failure: return -1; } -static int tcf_simp_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, simp_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_simp_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, simp_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_simp_ops = { .kind = "simple", .id = TCA_ID_SIMP, @@ -223,27 +205,25 @@ static struct tc_action_ops act_simp_ops = { .dump = tcf_simp_dump, .cleanup = tcf_simp_release, .init = tcf_simp_init, - .walk = tcf_simp_walker, - .lookup = tcf_simp_search, .size = sizeof(struct tcf_defact), }; static __net_init int simp_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, simp_net_id); + struct tc_action_net *tn = net_generic(net, act_simp_ops.net_id); return tc_action_net_init(net, tn, &act_simp_ops); } static void __net_exit simp_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, simp_net_id); + tc_action_net_exit(net_list, act_simp_ops.net_id); } static struct pernet_operations simp_net_ops = { .init = simp_init_net, .exit_batch = simp_exit_net, - .id = &simp_net_id, + .id = &act_simp_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index e3bd11dfe1ca5a0478977a879984fd71e0f9d9c6..7f598784fd305e5a14562759cb290a0fa269c1e8 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -20,7 +20,6 @@ #include #include -static unsigned int skbedit_net_id; static struct tc_action_ops act_skbedit_ops; static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params, @@ -118,7 +117,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 act_flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, skbedit_net_id); + struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id); bool bind = act_flags & TCA_ACT_FLAGS_BIND; struct tcf_skbedit_params *params_new; struct nlattr *tb[TCA_SKBEDIT_MAX + 1]; @@ -347,23 +346,6 @@ static void tcf_skbedit_cleanup(struct tc_action *a) kfree_rcu(params, rcu); } -static int tcf_skbedit_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, skbedit_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_skbedit_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, skbedit_net_id); - - return tcf_idr_search(tn, a, index); -} - static size_t tcf_skbedit_get_fill_size(const struct tc_action *act) { return nla_total_size(sizeof(struct tc_skbedit)) @@ -428,29 +410,27 @@ static struct tc_action_ops act_skbedit_ops = { .dump = tcf_skbedit_dump, .init = tcf_skbedit_init, .cleanup = tcf_skbedit_cleanup, - .walk = tcf_skbedit_walker, .get_fill_size = tcf_skbedit_get_fill_size, - .lookup = tcf_skbedit_search, .offload_act_setup = tcf_skbedit_offload_act_setup, .size = sizeof(struct tcf_skbedit), }; static __net_init int skbedit_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, skbedit_net_id); + struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id); return tc_action_net_init(net, tn, &act_skbedit_ops); } static void __net_exit skbedit_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, skbedit_net_id); + tc_action_net_exit(net_list, act_skbedit_ops.net_id); } static struct pernet_operations skbedit_net_ops = { .init = skbedit_init_net, .exit_batch = skbedit_exit_net, - .id = &skbedit_net_id, + .id = &act_skbedit_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index 2083612d87803c2dd92f9124305f7b0e4fdaad4e..d98758a639340517aa6a40e20e76156cffd26178 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -19,7 +19,6 @@ #include #include -static unsigned int skbmod_net_id; static struct tc_action_ops act_skbmod_ops; static int tcf_skbmod_act(struct sk_buff *skb, const struct tc_action *a, @@ -103,7 +102,7 @@ static int tcf_skbmod_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, skbmod_net_id); + struct tc_action_net *tn = net_generic(net, act_skbmod_ops.net_id); bool ovr = flags & TCA_ACT_FLAGS_REPLACE; bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_SKBMOD_MAX + 1]; @@ -276,23 +275,6 @@ nla_put_failure: return -1; } -static int tcf_skbmod_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, skbmod_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tcf_skbmod_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, skbmod_net_id); - - return tcf_idr_search(tn, a, index); -} - static struct tc_action_ops act_skbmod_ops = { .kind = "skbmod", .id = TCA_ACT_SKBMOD, @@ -301,27 +283,25 @@ static struct tc_action_ops act_skbmod_ops = { .dump = tcf_skbmod_dump, .init = tcf_skbmod_init, .cleanup = tcf_skbmod_cleanup, - .walk = tcf_skbmod_walker, - .lookup = tcf_skbmod_search, .size = sizeof(struct tcf_skbmod), }; static __net_init int skbmod_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, skbmod_net_id); + struct tc_action_net *tn = net_generic(net, act_skbmod_ops.net_id); return tc_action_net_init(net, tn, &act_skbmod_ops); } static void __net_exit skbmod_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, skbmod_net_id); + tc_action_net_exit(net_list, act_skbmod_ops.net_id); } static struct pernet_operations skbmod_net_ops = { .init = skbmod_init_net, .exit_batch = skbmod_exit_net, - .id = &skbmod_net_id, + .id = &act_skbmod_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 856dc23cef8c855186a4f4870709d87cba0d334a..2691a3d8e4511de5058553e7b4465c8cc83d0c8a 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -20,7 +20,6 @@ #include #include -static unsigned int tunnel_key_net_id; static struct tc_action_ops act_tunnel_key_ops; static int tunnel_key_act(struct sk_buff *skb, const struct tc_action *a, @@ -358,7 +357,7 @@ static int tunnel_key_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 act_flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, tunnel_key_net_id); + struct tc_action_net *tn = net_generic(net, act_tunnel_key_ops.net_id); bool bind = act_flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_TUNNEL_KEY_MAX + 1]; struct tcf_tunnel_key_params *params_new; @@ -770,23 +769,6 @@ nla_put_failure: return -1; } -static int tunnel_key_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, tunnel_key_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - -static int tunnel_key_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, tunnel_key_net_id); - - return tcf_idr_search(tn, a, index); -} - static void tcf_tunnel_encap_put_tunnel(void *priv) { struct ip_tunnel_info *tunnel = priv; @@ -850,28 +832,26 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, - .walk = tunnel_key_walker, - .lookup = tunnel_key_search, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; static __net_init int tunnel_key_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, tunnel_key_net_id); + struct tc_action_net *tn = net_generic(net, act_tunnel_key_ops.net_id); return tc_action_net_init(net, tn, &act_tunnel_key_ops); } static void __net_exit tunnel_key_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, tunnel_key_net_id); + tc_action_net_exit(net_list, act_tunnel_key_ops.net_id); } static struct pernet_operations tunnel_key_net_ops = { .init = tunnel_key_init_net, .exit_batch = tunnel_key_exit_net, - .id = &tunnel_key_net_id, + .id = &act_tunnel_key_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c index 68b5e772386a2109d513e2ba9d19b18d1db17ca7..7b24e898a3e6b6b803659c36ee20721681bfbf16 100644 --- a/net/sched/act_vlan.c +++ b/net/sched/act_vlan.c @@ -16,7 +16,6 @@ #include #include -static unsigned int vlan_net_id; static struct tc_action_ops act_vlan_ops; static int tcf_vlan_act(struct sk_buff *skb, const struct tc_action *a, @@ -117,7 +116,7 @@ static int tcf_vlan_init(struct net *net, struct nlattr *nla, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { - struct tc_action_net *tn = net_generic(net, vlan_net_id); + struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_VLAN_MAX + 1]; struct tcf_chain *goto_ch = NULL; @@ -333,16 +332,6 @@ nla_put_failure: return -1; } -static int tcf_vlan_walker(struct net *net, struct sk_buff *skb, - struct netlink_callback *cb, int type, - const struct tc_action_ops *ops, - struct netlink_ext_ack *extack) -{ - struct tc_action_net *tn = net_generic(net, vlan_net_id); - - return tcf_generic_walker(tn, skb, cb, type, ops, extack); -} - static void tcf_vlan_stats_update(struct tc_action *a, u64 bytes, u64 packets, u64 drops, u64 lastuse, bool hw) { @@ -353,13 +342,6 @@ static void tcf_vlan_stats_update(struct tc_action *a, u64 bytes, u64 packets, tm->lastuse = max_t(u64, tm->lastuse, lastuse); } -static int tcf_vlan_search(struct net *net, struct tc_action **a, u32 index) -{ - struct tc_action_net *tn = net_generic(net, vlan_net_id); - - return tcf_idr_search(tn, a, index); -} - static size_t tcf_vlan_get_fill_size(const struct tc_action *act) { return nla_total_size(sizeof(struct tc_vlan)) @@ -438,30 +420,28 @@ static struct tc_action_ops act_vlan_ops = { .dump = tcf_vlan_dump, .init = tcf_vlan_init, .cleanup = tcf_vlan_cleanup, - .walk = tcf_vlan_walker, .stats_update = tcf_vlan_stats_update, .get_fill_size = tcf_vlan_get_fill_size, - .lookup = tcf_vlan_search, .offload_act_setup = tcf_vlan_offload_act_setup, .size = sizeof(struct tcf_vlan), }; static __net_init int vlan_init_net(struct net *net) { - struct tc_action_net *tn = net_generic(net, vlan_net_id); + struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id); return tc_action_net_init(net, tn, &act_vlan_ops); } static void __net_exit vlan_exit_net(struct list_head *net_list) { - tc_action_net_exit(net_list, vlan_net_id); + tc_action_net_exit(net_list, act_vlan_ops.net_id); } static struct pernet_operations vlan_net_ops = { .init = vlan_init_net, .exit_batch = vlan_exit_net, - .id = &vlan_net_id, + .id = &act_vlan_ops.net_id, .size = sizeof(struct tc_action_net), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 790d6809be813e82e2e612fb2ed444f27e6b3313..50566db45949b7381a46b3f4b50d830e5a745d5c 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -1977,9 +1977,6 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, bool rtnl_held = false; u32 flags; - if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - replay: tp_created = 0; @@ -2137,6 +2134,7 @@ replay: } if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { + tfilter_put(tp, fh); NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind"); err = -EINVAL; goto errout; @@ -2208,9 +2206,6 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, int err; bool rtnl_held = false; - if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) @@ -2826,10 +2821,6 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, unsigned long cl; int err; - if (n->nlmsg_type != RTM_GETCHAIN && - !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - replay: q = NULL; err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, @@ -3639,9 +3630,6 @@ int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch, if (err) return err; - if (!block_index) - return 0; - qe->info.binder_type = binder_type; qe->info.chain_head_change = tcf_chain_head_change_dflt; qe->info.chain_head_change_priv = &qe->filter_chain; diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c index 8158fc9ee1ab089e78de825a1767c660ac24757c..d229ce99e55487f396501a5d56ac98e8d58c99d9 100644 --- a/net/sched/cls_basic.c +++ b/net/sched/cls_basic.c @@ -251,15 +251,8 @@ static void basic_walk(struct tcf_proto *tp, struct tcf_walker *arg, struct basic_filter *f; list_for_each_entry(f, &head->flist, link) { - if (arg->count < arg->skip) - goto skip; - - if (arg->fn(tp, f, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, f)) break; - } -skip: - arg->count++; } } @@ -268,12 +261,7 @@ static void basic_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct basic_filter *f = fh; - if (f && f->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &f->res, base); - else - __tcf_unbind_filter(q, &f->res); - } + tc_cls_bind_class(classid, cl, q, &f->res, base); } static int basic_dump(struct net *net, struct tcf_proto *tp, void *fh, diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index c85b85a192bf2b7bc703da3f9ef3037f9a946bec..bc317b3eac124b2df292bdb61f7a2dc39ef11d51 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -635,12 +635,7 @@ static void cls_bpf_bind_class(void *fh, u32 classid, unsigned long cl, { struct cls_bpf_prog *prog = fh; - if (prog && prog->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &prog->res, base); - else - __tcf_unbind_filter(q, &prog->res); - } + tc_cls_bind_class(classid, cl, q, &prog->res, base); } static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg, @@ -650,14 +645,8 @@ static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg, struct cls_bpf_prog *prog; list_for_each_entry(prog, &head->plist, link) { - if (arg->count < arg->skip) - goto skip; - if (arg->fn(tp, prog, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, prog)) break; - } -skip: - arg->count++; } } diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c index 972303aa8edd6a753529cc4420f0a9e5ee8ebd59..014cd3de7b5dd5ee8e33bce50625ffb72879230d 100644 --- a/net/sched/cls_flow.c +++ b/net/sched/cls_flow.c @@ -683,14 +683,8 @@ static void flow_walk(struct tcf_proto *tp, struct tcf_walker *arg, struct flow_filter *f; list_for_each_entry(f, &head->filters, list) { - if (arg->count < arg->skip) - goto skip; - if (arg->fn(tp, f, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, f)) break; - } -skip: - arg->count++; } } diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c index 041d63ff809a7c6883ac295365a888fa54911a01..25bc57ee6ea1097c1a7a77703c389ab8294755af 100644 --- a/net/sched/cls_flower.c +++ b/net/sched/cls_flower.c @@ -69,6 +69,7 @@ struct fl_flow_key { struct flow_dissector_key_hash hash; struct flow_dissector_key_num_of_vlans num_of_vlans; struct flow_dissector_key_pppoe pppoe; + struct flow_dissector_key_l2tpv3 l2tpv3; } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ struct fl_flow_mask_range { @@ -712,6 +713,7 @@ static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { [TCA_FLOWER_KEY_NUM_OF_VLANS] = { .type = NLA_U8 }, [TCA_FLOWER_KEY_PPPOE_SID] = { .type = NLA_U16 }, [TCA_FLOWER_KEY_PPP_PROTO] = { .type = NLA_U16 }, + [TCA_FLOWER_KEY_L2TPV3_SID] = { .type = NLA_U32 }, }; @@ -1790,6 +1792,11 @@ static int fl_set_key(struct net *net, struct nlattr **tb, fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, sizeof(key->arp.tha)); + } else if (key->basic.ip_proto == IPPROTO_L2TP) { + fl_set_key_val(tb, &key->l2tpv3.session_id, + TCA_FLOWER_KEY_L2TPV3_SID, + &mask->l2tpv3.session_id, TCA_FLOWER_UNSPEC, + sizeof(key->l2tpv3.session_id)); } if (key->basic.ip_proto == IPPROTO_TCP || @@ -1970,6 +1977,8 @@ static void fl_init_dissector(struct flow_dissector *dissector, FLOW_DISSECTOR_KEY_NUM_OF_VLANS, num_of_vlans); FL_KEY_SET_IF_MASKED(mask, keys, cnt, FLOW_DISSECTOR_KEY_PPPOE, pppoe); + FL_KEY_SET_IF_MASKED(mask, keys, cnt, + FLOW_DISSECTOR_KEY_L2TPV3, l2tpv3); skb_flow_dissector_init(dissector, keys, cnt); } @@ -3196,6 +3205,13 @@ static int fl_dump_key(struct sk_buff *skb, struct net *net, mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, sizeof(key->arp.tha)))) goto nla_put_failure; + else if (key->basic.ip_proto == IPPROTO_L2TP && + fl_dump_key_val(skb, &key->l2tpv3.session_id, + TCA_FLOWER_KEY_L2TPV3_SID, + &mask->l2tpv3.session_id, + TCA_FLOWER_UNSPEC, + sizeof(key->l2tpv3.session_id))) + goto nla_put_failure; if ((key->basic.ip_proto == IPPROTO_TCP || key->basic.ip_proto == IPPROTO_UDP || @@ -3389,12 +3405,7 @@ static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct cls_fl_filter *f = fh; - if (f && f->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &f->res, base); - else - __tcf_unbind_filter(q, &f->res); - } + tc_cls_bind_class(classid, cl, q, &f->res, base); } static bool fl_delete_empty(struct tcf_proto *tp) diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 8654b0ce997c1cb525222f51a8531e99a3799efb..a32351da968cd7cb6fea11daf491af314093e07b 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -358,15 +358,8 @@ static void fw_walk(struct tcf_proto *tp, struct tcf_walker *arg, for (f = rtnl_dereference(head->ht[h]); f; f = rtnl_dereference(f->next)) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(tp, f, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, f)) return; - } - arg->count++; } } } @@ -423,12 +416,7 @@ static void fw_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct fw_filter *f = fh; - if (f && f->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &f->res, base); - else - __tcf_unbind_filter(q, &f->res); - } + tc_cls_bind_class(classid, cl, q, &f->res, base); } static struct tcf_proto_ops cls_fw_ops __read_mostly = { diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c index 06cf22adbab732b32371cb1ca11818492a6f565e..39a5d9c170def16d7133930d30bf23125915269e 100644 --- a/net/sched/cls_matchall.c +++ b/net/sched/cls_matchall.c @@ -313,10 +313,7 @@ static int mall_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, tc_cleanup_offload_action(&cls_mall.rule->action); kfree(cls_mall.rule); - if (err) - return err; - - return 0; + return err; } static void mall_stats_hw_filter(struct tcf_proto *tp, @@ -397,12 +394,7 @@ static void mall_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct cls_mall_head *head = fh; - if (head && head->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &head->res, base); - else - __tcf_unbind_filter(q, &head->res); - } + tc_cls_bind_class(classid, cl, q, &head->res, base); } static struct tcf_proto_ops cls_mall_ops __read_mostly = { diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index 48712bc51bda7ec737ebe79a6fea0f792bf78f5a..9e43b929d4ca4e42a960315f143f4023f8b454d6 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -488,7 +488,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, } if (opt == NULL) - return handle ? -EINVAL : 0; + return -EINVAL; err = nla_parse_nested_deprecated(tb, TCA_ROUTE4_MAX, opt, route4_policy, NULL); @@ -496,7 +496,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, return err; fold = *arg; - if (fold && handle && fold->handle != handle) + if (fold && fold->handle != handle) return -EINVAL; err = -ENOBUFS; @@ -587,15 +587,8 @@ static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg, for (f = rtnl_dereference(b->ht[h1]); f; f = rtnl_dereference(f->next)) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(tp, f, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, f)) return; - } - arg->count++; } } } @@ -656,12 +649,7 @@ static void route4_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct route4_filter *f = fh; - if (f && f->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &f->res, base); - else - __tcf_unbind_filter(q, &f->res); - } + tc_cls_bind_class(classid, cl, q, &f->res, base); } static struct tcf_proto_ops cls_route4_ops __read_mostly = { diff --git a/net/sched/cls_rsvp.h b/net/sched/cls_rsvp.h index 5cd9d6b143c44bf250ca1e53c384e3cf3fab57d2..b00a7dbd0587403b16624aa05b6d8bafacbe4b7f 100644 --- a/net/sched/cls_rsvp.h +++ b/net/sched/cls_rsvp.h @@ -671,15 +671,8 @@ static void rsvp_walk(struct tcf_proto *tp, struct tcf_walker *arg, for (f = rtnl_dereference(s->ht[h1]); f; f = rtnl_dereference(f->next)) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(tp, f, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, f)) return; - } - arg->count++; } } } @@ -740,12 +733,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct rsvp_filter *f = fh; - if (f && f->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &f->res, base); - else - __tcf_unbind_filter(q, &f->res); - } + tc_cls_bind_class(classid, cl, q, &f->res, base); } static struct tcf_proto_ops RSVP_OPS __read_mostly = { diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c index 742c7d49a9581c185b649352034d548bd7fdf4c5..1c9eeb98d826e42bc2ff2f169a36a04dac517b9d 100644 --- a/net/sched/cls_tcindex.c +++ b/net/sched/cls_tcindex.c @@ -566,13 +566,8 @@ static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker, for (i = 0; i < p->hash; i++) { if (!p->perfect[i].res.class) continue; - if (walker->count >= walker->skip) { - if (walker->fn(tp, p->perfect + i, walker) < 0) { - walker->stop = 1; - return; - } - } - walker->count++; + if (!tc_cls_stats_dump(tp, walker, p->perfect + i)) + return; } } if (!p->h) @@ -580,13 +575,8 @@ static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker, for (i = 0; i < p->hash; i++) { for (f = rtnl_dereference(p->h[i]); f; f = next) { next = rtnl_dereference(f->next); - if (walker->count >= walker->skip) { - if (walker->fn(tp, &f->result, walker) < 0) { - walker->stop = 1; - return; - } - } - walker->count++; + if (!tc_cls_stats_dump(tp, walker, &f->result)) + return; } } } @@ -701,12 +691,7 @@ static void tcindex_bind_class(void *fh, u32 classid, unsigned long cl, { struct tcindex_filter_result *r = fh; - if (r && r->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &r->res, base); - else - __tcf_unbind_filter(q, &r->res); - } + tc_cls_bind_class(classid, cl, q, &r->res, base); } static struct tcf_proto_ops cls_tcindex_ops __read_mostly = { diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 4d27300c287c46d11bf9d44f8c66eded9e734581..34d25f7a0687a7f24f49554e7187a62ae8945781 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -1040,7 +1040,11 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, } #endif - memcpy(&n->sel, s, sel_size); + unsafe_memcpy(&n->sel, s, sel_size, + /* A composite flex-array structure destination, + * which was correctly sized with struct_size(), + * bounds-checked against nla_len(), and allocated + * above. */); RCU_INIT_POINTER(n->ht_up, ht); n->handle = handle; n->fshift = s->hmask ? ffs(ntohl(s->hmask)) - 1 : 0; @@ -1125,26 +1129,16 @@ static void u32_walk(struct tcf_proto *tp, struct tcf_walker *arg, ht = rtnl_dereference(ht->next)) { if (ht->prio != tp->prio) continue; - if (arg->count >= arg->skip) { - if (arg->fn(tp, ht, arg) < 0) { - arg->stop = 1; - return; - } - } - arg->count++; + + if (!tc_cls_stats_dump(tp, arg, ht)) + return; + for (h = 0; h <= ht->divisor; h++) { for (n = rtnl_dereference(ht->ht[h]); n; n = rtnl_dereference(n->next)) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(tp, n, arg) < 0) { - arg->stop = 1; + if (!tc_cls_stats_dump(tp, arg, n)) return; - } - arg->count++; } } } @@ -1256,12 +1250,7 @@ static void u32_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct tc_u_knode *n = fh; - if (n && n->res.classid == classid) { - if (cl) - __tcf_bind_filter(q, &n->res, base); - else - __tcf_unbind_filter(q, &n->res); - } + tc_cls_bind_class(classid, cl, q, &n->res, base); } static int u32_dump(struct net *net, struct tcf_proto *tp, void *fh, diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index bf87b50837a84088ff930c1b6c5a7827c89b1150..c98af0ada706efee202a20a6bfb6f2b984106f45 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -171,7 +171,7 @@ out_einval: } EXPORT_SYMBOL(register_qdisc); -int unregister_qdisc(struct Qdisc_ops *qops) +void unregister_qdisc(struct Qdisc_ops *qops) { struct Qdisc_ops *q, **qp; int err = -ENOENT; @@ -186,7 +186,8 @@ int unregister_qdisc(struct Qdisc_ops *qops) err = 0; } write_unlock(&qdisc_mod_lock); - return err; + + WARN(err, "unregister qdisc(%s) failed\n", qops->id); } EXPORT_SYMBOL(unregister_qdisc); @@ -194,7 +195,7 @@ EXPORT_SYMBOL(unregister_qdisc); void qdisc_get_default(char *name, size_t len) { read_lock(&qdisc_mod_lock); - strlcpy(name, default_qdisc_ops->id, len); + strscpy(name, default_qdisc_ops->id, len); read_unlock(&qdisc_mod_lock); } @@ -867,6 +868,23 @@ void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, } EXPORT_SYMBOL(qdisc_offload_graft_helper); +void qdisc_offload_query_caps(struct net_device *dev, + enum tc_setup_type type, + void *caps, size_t caps_len) +{ + const struct net_device_ops *ops = dev->netdev_ops; + struct tc_query_caps_base base = { + .type = type, + .caps = caps, + }; + + memset(caps, 0, caps_len); + + if (ops->ndo_setup_tc) + ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base); +} +EXPORT_SYMBOL(qdisc_offload_query_caps); + static void qdisc_offload_graft_root(struct net_device *dev, struct Qdisc *new, struct Qdisc *old, struct netlink_ext_ack *extack) @@ -1163,7 +1181,7 @@ static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca, static struct Qdisc *qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue, - struct Qdisc *p, u32 parent, u32 handle, + u32 parent, u32 handle, struct nlattr **tca, int *errp, struct netlink_ext_ack *extack) { @@ -1424,10 +1442,6 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, struct Qdisc *p = NULL; int err; - if ((n->nlmsg_type != RTM_GETQDISC) && - !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) @@ -1508,9 +1522,6 @@ static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, struct Qdisc *q, *p; int err; - if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - replay: /* Reinit, just in case something touches this. */ err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, @@ -1640,7 +1651,7 @@ create_n_graft: } if (clid == TC_H_INGRESS) { if (dev_ingress_queue(dev)) { - q = qdisc_create(dev, dev_ingress_queue(dev), p, + q = qdisc_create(dev, dev_ingress_queue(dev), tcm->tcm_parent, tcm->tcm_parent, tca, &err, extack); } else { @@ -1657,7 +1668,7 @@ create_n_graft: else dev_queue = netdev_get_tx_queue(dev, 0); - q = qdisc_create(dev, dev_queue, p, + q = qdisc_create(dev, dev_queue, tcm->tcm_parent, tcm->tcm_handle, tca, &err, extack); } @@ -1904,7 +1915,7 @@ static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg) { struct tcf_bind_args *a = (void *)arg; - if (tp->ops->bind_class) { + if (n && tp->ops->bind_class) { struct Qdisc *q = tcf_block_q(tp->chain->block); sch_tree_lock(q); @@ -1992,10 +2003,6 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, u32 qid; int err; - if ((n->nlmsg_type != RTM_GETTCLASS) && - !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) - return -EPERM; - err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy, extack); if (err < 0) diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c index 4c8e994cf0a536a93e56a200102d49294005f291..f52255fea652beae735e514489db958a6ad1b7a1 100644 --- a/net/sched/sch_atm.c +++ b/net/sched/sch_atm.c @@ -354,12 +354,8 @@ static void atm_tc_walk(struct Qdisc *sch, struct qdisc_walker *walker) if (walker->stop) return; list_for_each_entry(flow, &p->flows, list) { - if (walker->count >= walker->skip && - walker->fn(sch, (unsigned long)flow, walker) < 0) { - walker->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)flow, walker)) break; - } - walker->count++; } } @@ -577,7 +573,6 @@ static void atm_tc_reset(struct Qdisc *sch) pr_debug("atm_tc_reset(sch %p,[qdisc %p])\n", sch, p); list_for_each_entry(flow, &p->flows, list) qdisc_reset(flow->q); - sch->q.qlen = 0; } static void atm_tc_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index a43a58a73d0961c27278b8e43ed60878706396c5..817cd0695b3502b3e0bfd744d2c81f67cc474074 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -573,7 +573,7 @@ static bool cobalt_should_drop(struct cobalt_vars *vars, /* Simple BLUE implementation. Lack of ECN is deliberate. */ if (vars->p_drop) - drop |= (prandom_u32() < vars->p_drop); + drop |= (get_random_u32() < vars->p_drop); /* Overload the drop_next field as an activity timeout */ if (!vars->count) @@ -2092,11 +2092,11 @@ retry: WARN_ON(host_load > CAKE_QUEUES); - /* The shifted prandom_u32() is a way to apply dithering to - * avoid accumulating roundoff errors + /* The get_random_u16() is a way to apply dithering to avoid + * accumulating roundoff errors */ flow->deficit += (b->flow_quantum * quantum_div[host_load] + - (prandom_u32() >> 16)) >> 16; + get_random_u16()) >> 16; list_move_tail(&flow->flowchain, &b->old_flows); goto retry; @@ -2569,9 +2569,6 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt, struct nlattr *tb[TCA_CAKE_MAX + 1]; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_CAKE_MAX, opt, cake_policy, extack); if (err < 0) @@ -3064,16 +3061,13 @@ static void cake_walk(struct Qdisc *sch, struct qdisc_walker *arg) struct cake_tin_data *b = &q->tins[q->tin_order[i]]; for (j = 0; j < CAKE_QUEUES; j++) { - if (list_empty(&b->flows[j].flowchain) || - arg->count < arg->skip) { + if (list_empty(&b->flows[j].flowchain)) { arg->count++; continue; } - if (arg->fn(sch, i * CAKE_QUEUES + j + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, i * CAKE_QUEUES + j + 1, + arg)) break; - } - arg->count++; } } } diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c index 91a0dc463c4826ebf721e76b603a9a2b7989c5b3..6568e17c4c634184493d8e5414b6671aecc2d276 100644 --- a/net/sched/sch_cbq.c +++ b/net/sched/sch_cbq.c @@ -975,7 +975,6 @@ cbq_reset(struct Qdisc *sch) cl->cpriority = cl->priority; } } - sch->q.qlen = 0; } @@ -1677,15 +1676,8 @@ static void cbq_walk(struct Qdisc *sch, struct qdisc_walker *arg) for (h = 0; h < q->clhash.hashsize; h++) { hlist_for_each_entry(cl, &q->clhash.hash[h], common.hnode) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, (unsigned long)cl, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg)) return; - } - arg->count++; } } } diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c index 459cc240eda9c44ded125519c9f86384f98a18f2..cac870eb7897305e952398108a7cb1edce71a27a 100644 --- a/net/sched/sch_cbs.c +++ b/net/sched/sch_cbs.c @@ -520,13 +520,7 @@ static unsigned long cbs_find(struct Qdisc *sch, u32 classid) static void cbs_walk(struct Qdisc *sch, struct qdisc_walker *walker) { if (!walker->stop) { - if (walker->count >= walker->skip) { - if (walker->fn(sch, 1, walker) < 0) { - walker->stop = 1; - return; - } - } - walker->count++; + tc_qdisc_stats_dump(sch, 1, walker); } } diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index 2adbd945bf15aee182bc1505ec54b9695d104d52..3ac3e5c80b6ffb035181d3270b05d7ce32fa2735 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -60,7 +60,6 @@ struct choke_sched_data { u32 forced_drop; /* Forced drops, qavg > max_thresh */ u32 forced_mark; /* Forced marks, qavg > max_thresh */ u32 pdrop; /* Drops due to queue limits */ - u32 other; /* Drops due to drop() calls */ u32 matched; /* Drops to flow match */ } stats; @@ -315,8 +314,6 @@ static void choke_reset(struct Qdisc *sch) rtnl_qdisc_drop(skb, sch); } - sch->q.qlen = 0; - sch->qstats.backlog = 0; if (q->tab) memset(q->tab, 0, (q->tab_mask + 1) * sizeof(struct sk_buff *)); q->head = q->tail = 0; @@ -466,7 +463,6 @@ static int choke_dump_stats(struct Qdisc *sch, struct gnet_dump *d) .early = q->stats.prob_drop + q->stats.forced_drop, .marked = q->stats.prob_mark + q->stats.forced_mark, .pdrop = q->stats.pdrop, - .other = q->stats.other, .matched = q->stats.matched, }; diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index 30169b3adbbb064c51b6006755d56446570f974c..d7a4874543de5d49796a353645e685d1e2e49e0a 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -138,9 +138,6 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt, unsigned int qlen, dropped = 0; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_CODEL_MAX, opt, codel_policy, NULL); if (err < 0) diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 18e4f7a0b291242731610723f38b1f1c7858beb0..e35a4e90f4e6ccfbaa033e2d9779c24fe1c983ad 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -284,15 +284,8 @@ static void drr_walk(struct Qdisc *sch, struct qdisc_walker *arg) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, (unsigned long)cl, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg)) return; - } - arg->count++; } } } @@ -441,8 +434,6 @@ static void drr_reset_qdisc(struct Qdisc *sch) qdisc_reset(cl->qdisc); } } - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void drr_destroy_qdisc(struct Qdisc *sch) diff --git a/net/sched/sch_dsmark.c b/net/sched/sch_dsmark.c index 4c100d10526996c8d26adc22666e0f5f4ecad148..401ffaf87d6221cda528dbfcbf2f0fe3d4c81585 100644 --- a/net/sched/sch_dsmark.c +++ b/net/sched/sch_dsmark.c @@ -176,16 +176,12 @@ static void dsmark_walk(struct Qdisc *sch, struct qdisc_walker *walker) return; for (i = 0; i < p->indices; i++) { - if (p->mv[i].mask == 0xff && !p->mv[i].value) - goto ignore; - if (walker->count >= walker->skip) { - if (walker->fn(sch, i + 1, walker) < 0) { - walker->stop = 1; - break; - } + if (p->mv[i].mask == 0xff && !p->mv[i].value) { + walker->count++; + continue; } -ignore: - walker->count++; + if (!tc_qdisc_stats_dump(sch, i + 1, walker)) + break; } } @@ -409,8 +405,6 @@ static void dsmark_reset(struct Qdisc *sch) pr_debug("%s(sch %p,[qdisc %p])\n", __func__, sch, p); if (p->q) qdisc_reset(p->q); - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void dsmark_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c index c48f91075b5c60ee4fe9e10186c913baf5894a69..61d1f0e32cf3561bdce63a2911ab15bd84bd3359 100644 --- a/net/sched/sch_etf.c +++ b/net/sched/sch_etf.c @@ -323,9 +323,6 @@ static int etf_enable_offload(struct net_device *dev, struct etf_sched_data *q, struct tc_etf_qopt_offload etf = { }; int err; - if (q->offload) - return 0; - if (!ops->ndo_setup_tc) { NL_SET_ERR_MSG(extack, "Specified device does not support ETF offload"); return -EOPNOTSUPP; @@ -445,9 +442,6 @@ static void etf_reset(struct Qdisc *sch) timesortedlist_clear(sch); __qdisc_reset_queue(&sch->q); - sch->qstats.backlog = 0; - sch->q.qlen = 0; - q->last = 0; } diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index d73393493553389ac77fa7d16c500ad6c7f49404..b10efeaf0629d2ab8a5bbe2b080c3ef84d52717f 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -341,15 +341,8 @@ static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (i = 0; i < q->nbands; i++) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, i + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, i + 1, arg)) break; - } - arg->count++; } } @@ -594,11 +587,6 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, unsigned int i; int err; - if (!opt) { - NL_SET_ERR_MSG(extack, "ETS options are required for this operation"); - return -EINVAL; - } - err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack); if (err < 0) return err; @@ -727,8 +715,6 @@ static void ets_qdisc_reset(struct Qdisc *sch) } for (band = 0; band < q->nbands; band++) qdisc_reset(q->classes[band].qdisc); - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void ets_qdisc_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 2fb76fc0cc31bf732e0b32a95d8e45f457acb683..48d14fb90ba02d00411e3fb51ea9fd56ffeec149 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -808,9 +808,6 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, unsigned drop_len = 0; u32 fq_log; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_FQ_MAX, opt, fq_policy, NULL); if (err < 0) diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 839e1235db053b7a8606c67484db861c4605b61e..99d318b60568237de48b5904b3198e1e8475b383 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -347,8 +347,6 @@ static void fq_codel_reset(struct Qdisc *sch) codel_vars_init(&flow->cvars); } memset(q->backlogs, 0, q->flows_cnt * sizeof(u32)); - sch->q.qlen = 0; - sch->qstats.backlog = 0; q->memory_usage = 0; } @@ -374,9 +372,6 @@ static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt, u32 quantum = 0; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_FQ_CODEL_MAX, opt, fq_codel_policy, NULL); if (err < 0) @@ -483,26 +478,24 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, if (opt) { err = fq_codel_change(sch, opt, extack); if (err) - goto init_failure; + return err; } err = tcf_block_get(&q->block, &q->filter_list, sch, extack); if (err) - goto init_failure; + return err; if (!q->flows) { q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_codel_flow), GFP_KERNEL); - if (!q->flows) { - err = -ENOMEM; - goto init_failure; - } + if (!q->flows) + return -ENOMEM; + q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL); - if (!q->backlogs) { - err = -ENOMEM; - goto alloc_failure; - } + if (!q->backlogs) + return -ENOMEM; + for (i = 0; i < q->flows_cnt; i++) { struct fq_codel_flow *flow = q->flows + i; @@ -515,13 +508,6 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, else sch->flags &= ~TCQ_F_CAN_BYPASS; return 0; - -alloc_failure: - kvfree(q->flows); - q->flows = NULL; -init_failure: - q->flows_cnt = 0; - return err; } static int fq_codel_dump(struct Qdisc *sch, struct sk_buff *skb) @@ -687,16 +673,12 @@ static void fq_codel_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (i = 0; i < q->flows_cnt; i++) { - if (list_empty(&q->flows[i].flowchain) || - arg->count < arg->skip) { + if (list_empty(&q->flows[i].flowchain)) { arg->count++; continue; } - if (arg->fn(sch, i + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, i + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index d6aba6edd16e5eab120a57c316fcb06a5d5f3442..6980796d435d9d496bbf6db773dc6d8b5857c50c 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -283,9 +283,6 @@ static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt, unsigned int num_dropped = 0; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested(tb, TCA_FQ_PIE_MAX, opt, fq_pie_policy, extack); if (err < 0) return err; @@ -521,9 +518,6 @@ static void fq_pie_reset(struct Qdisc *sch) INIT_LIST_HEAD(&flow->flowchain); pie_vars_init(&flow->vars); } - - sch->q.qlen = 0; - sch->qstats.backlog = 0; } static void fq_pie_destroy(struct Qdisc *sch) diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index d47b9689eba6a4d5828aaad6e574c62d68aa4194..a9aadc4e6858104e59b89b55418bdf51e63e0d75 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -409,7 +409,7 @@ static inline bool qdisc_restart(struct Qdisc *q, int *packets) void __qdisc_run(struct Qdisc *q) { - int quota = dev_tx_weight; + int quota = READ_ONCE(dev_tx_weight); int packets; while (qdisc_restart(q, &packets)) { @@ -941,7 +941,6 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, goto errout; __skb_queue_head_init(&sch->gso_skb); __skb_queue_head_init(&sch->skb_bad_txq); - qdisc_skb_head_init(&sch->q); gnet_stats_basic_sync_init(&sch->bstats); spin_lock_init(&sch->q.lock); @@ -1122,6 +1121,21 @@ struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, } EXPORT_SYMBOL(dev_graft_qdisc); +static void shutdown_scheduler_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_qdisc_default) +{ + struct Qdisc *qdisc = dev_queue->qdisc_sleeping; + struct Qdisc *qdisc_default = _qdisc_default; + + if (qdisc) { + rcu_assign_pointer(dev_queue->qdisc, qdisc_default); + dev_queue->qdisc_sleeping = qdisc_default; + + qdisc_put(qdisc); + } +} + static void attach_one_default_qdisc(struct net_device *dev, struct netdev_queue *dev_queue, void *_unused) @@ -1169,6 +1183,7 @@ static void attach_default_qdiscs(struct net_device *dev) if (qdisc == &noop_qdisc) { netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n", default_qdisc_ops->id, noqueue_qdisc_ops.id); + netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); dev->priv_flags |= IFF_NO_QUEUE; netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL); qdisc = txq->qdisc_sleeping; @@ -1447,21 +1462,6 @@ void dev_init_scheduler(struct net_device *dev) timer_setup(&dev->watchdog_timer, dev_watchdog, 0); } -static void shutdown_scheduler_queue(struct net_device *dev, - struct netdev_queue *dev_queue, - void *_qdisc_default) -{ - struct Qdisc *qdisc = dev_queue->qdisc_sleeping; - struct Qdisc *qdisc_default = _qdisc_default; - - if (qdisc) { - rcu_assign_pointer(dev_queue->qdisc, qdisc_default); - dev_queue->qdisc_sleeping = qdisc_default; - - qdisc_put(qdisc); - } -} - void dev_shutdown(struct net_device *dev) { netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index 1073c76d05c45ffa3d3ab4cf2d24441c80d5efcc..a661b062cca85431845ef3abf84205035d3029ae 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -648,9 +648,6 @@ static int gred_change(struct Qdisc *sch, struct nlattr *opt, u32 max_P; struct gred_sched_data *prealloc; - if (opt == NULL) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_GRED_MAX, opt, gred_policy, extack); if (err < 0) @@ -829,7 +826,6 @@ static int gred_dump(struct Qdisc *sch, struct sk_buff *skb) opt.Wlog = q->parms.Wlog; opt.Plog = q->parms.Plog; opt.Scell_log = q->parms.Scell_log; - opt.other = q->stats.other; opt.early = q->stats.prob_drop; opt.forced = q->stats.forced_drop; opt.pdrop = q->stats.pdrop; @@ -895,8 +891,6 @@ append_opt: goto nla_put_failure; if (nla_put_u32(skb, TCA_GRED_VQ_STAT_PDROP, q->stats.pdrop)) goto nla_put_failure; - if (nla_put_u32(skb, TCA_GRED_VQ_STAT_OTHER, q->stats.other)) - goto nla_put_failure; nla_nest_end(skb, vq); } @@ -914,10 +908,9 @@ static void gred_destroy(struct Qdisc *sch) struct gred_sched *table = qdisc_priv(sch); int i; - for (i = 0; i < table->DPs; i++) { - if (table->tab[i]) - gred_destroy_vq(table->tab[i]); - } + for (i = 0; i < table->DPs; i++) + gred_destroy_vq(table->tab[i]); + gred_offload(sch, TC_GRED_DESTROY); kfree(table->opt); } diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index d3979a6000e7d2edecef5cb7e50a10b26eb2736f..70b0c5873d32666fb1a5469dfd25d1a43e4f63ef 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1349,15 +1349,8 @@ hfsc_walk(struct Qdisc *sch, struct qdisc_walker *arg) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], cl_common.hnode) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, (unsigned long)cl, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg)) return; - } - arg->count++; } } } @@ -1430,7 +1423,7 @@ hfsc_change_qdisc(struct Qdisc *sch, struct nlattr *opt, struct hfsc_sched *q = qdisc_priv(sch); struct tc_hfsc_qopt *qopt; - if (opt == NULL || nla_len(opt) < sizeof(*qopt)) + if (nla_len(opt) < sizeof(*qopt)) return -EINVAL; qopt = nla_data(opt); @@ -1484,8 +1477,6 @@ hfsc_reset_qdisc(struct Qdisc *sch) } q->eligible = RB_ROOT; qdisc_watchdog_cancel(&q->watchdog); - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 420ede8753229faae172c0829b60d81e8eaa8de8..d26cd436cbe31bd37b1b651e0760845ae7ea2616 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -516,9 +516,6 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt, u32 new_quantum = q->quantum; u32 new_hhf_non_hh_weight = q->hhf_non_hh_weight; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_HHF_MAX, opt, hhf_policy, NULL); if (err < 0) diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index 23a9d6242429f1f9fadd047602108daffe3708ea..e5b4bbf3ce3d5f36edb512d4017ebd97209bb377 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -1008,8 +1008,6 @@ static void htb_reset(struct Qdisc *sch) } qdisc_watchdog_cancel(&q->watchdog); __qdisc_reset_queue(&q->direct_queue); - sch->q.qlen = 0; - sch->qstats.backlog = 0; memset(q->hlevel, 0, sizeof(q->hlevel)); memset(q->row_mask, 0, sizeof(q->row_mask)); } @@ -1104,9 +1102,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, err = qdisc_class_hash_init(&q->clhash); if (err < 0) - goto err_free_direct_qdiscs; - - qdisc_skb_head_init(&q->direct_queue); + return err; if (tb[TCA_HTB_DIRECT_QLEN]) q->direct_qlen = nla_get_u32(tb[TCA_HTB_DIRECT_QLEN]); @@ -1127,8 +1123,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, qdisc = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops, TC_H_MAKE(sch->handle, 0), extack); if (!qdisc) { - err = -ENOMEM; - goto err_free_qdiscs; + return -ENOMEM; } htb_set_lockdep_class_child(qdisc); @@ -1146,7 +1141,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, }; err = htb_offload(dev, &offload_opt); if (err) - goto err_free_qdiscs; + return err; /* Defer this assignment, so that htb_destroy skips offload-related * parts (especially calling ndo_setup_tc) on errors. @@ -1154,22 +1149,6 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt, q->offload = true; return 0; - -err_free_qdiscs: - for (ntx = 0; ntx < q->num_direct_qdiscs && q->direct_qdiscs[ntx]; - ntx++) - qdisc_put(q->direct_qdiscs[ntx]); - - qdisc_class_hash_destroy(&q->clhash); - /* Prevent use-after-free and double-free when htb_destroy gets called. - */ - q->clhash.hash = NULL; - q->clhash.hashsize = 0; - -err_free_direct_qdiscs: - kfree(q->direct_qdiscs); - q->direct_qdiscs = NULL; - return err; } static void htb_attach_offload(struct Qdisc *sch) @@ -1692,13 +1671,12 @@ static void htb_destroy(struct Qdisc *sch) qdisc_class_hash_destroy(&q->clhash); __qdisc_reset_queue(&q->direct_queue); - if (!q->offload) - return; - - offload_opt = (struct tc_htb_qopt_offload) { - .command = TC_HTB_DESTROY, - }; - htb_offload(dev, &offload_opt); + if (q->offload) { + offload_opt = (struct tc_htb_qopt_offload) { + .command = TC_HTB_DESTROY, + }; + htb_offload(dev, &offload_opt); + } if (!q->direct_qdiscs) return; @@ -2141,15 +2119,8 @@ static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, (unsigned long)cl, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg)) return; - } - arg->count++; } } } diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c index 83d2e54bf303a4353be84416d998a76f37734086..d0bc660d7401f9d0b1509ac65e641622f51a99b3 100644 --- a/net/sched/sch_mq.c +++ b/net/sched/sch_mq.c @@ -247,11 +247,8 @@ static void mq_walk(struct Qdisc *sch, struct qdisc_walker *arg) arg->count = arg->skip; for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) { - if (arg->fn(sch, ntx + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, ntx + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index b29f3453c6eafe85208c1e5fe38961ef2203000e..4c68abaa289bd0c626a9bc82fac0f17fc61a94ba 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -558,11 +558,8 @@ static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) /* Walk hierarchy with a virtual class per tc */ arg->count = arg->skip; for (ntx = arg->skip; ntx < netdev_get_num_tc(dev); ntx++) { - if (arg->fn(sch, ntx + TC_H_MIN_PRIORITY, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, ntx + TC_H_MIN_PRIORITY, arg)) return; - } - arg->count++; } /* Pad the values and skip over unused traffic classes */ diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index cd8ab90c4765d48b7b0f65ff1d68a43a9b2b7ace..75c9c860182b406e06e747455b1b7084fbab415d 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -152,7 +152,6 @@ multiq_reset(struct Qdisc *sch) for (band = 0; band < q->bands; band++) qdisc_reset(q->queues[band]); - sch->q.qlen = 0; q->curband = 0; } @@ -354,15 +353,8 @@ static void multiq_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (band = 0; band < q->bands; band++) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, band + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, band + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 5449ed114e40688ac575deb73d9c94bc877b7902..fb00ac40ecb7283a0ac85008e91ec499e66a4ce7 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -171,7 +171,7 @@ static inline struct netem_skb_cb *netem_skb_cb(struct sk_buff *skb) static void init_crandom(struct crndstate *state, unsigned long rho) { state->rho = rho; - state->last = prandom_u32(); + state->last = get_random_u32(); } /* get_crandom - correlated random number generator @@ -184,9 +184,9 @@ static u32 get_crandom(struct crndstate *state) unsigned long answer; if (!state || state->rho == 0) /* no correlation */ - return prandom_u32(); + return get_random_u32(); - value = prandom_u32(); + value = get_random_u32(); rho = (u64)state->rho + 1; answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32; state->last = answer; @@ -200,7 +200,7 @@ static u32 get_crandom(struct crndstate *state) static bool loss_4state(struct netem_sched_data *q) { struct clgstate *clg = &q->clg; - u32 rnd = prandom_u32(); + u32 rnd = get_random_u32(); /* * Makes a comparison between rnd and the transition @@ -268,15 +268,15 @@ static bool loss_gilb_ell(struct netem_sched_data *q) switch (clg->state) { case GOOD_STATE: - if (prandom_u32() < clg->a1) + if (get_random_u32() < clg->a1) clg->state = BAD_STATE; - if (prandom_u32() < clg->a4) + if (get_random_u32() < clg->a4) return true; break; case BAD_STATE: - if (prandom_u32() < clg->a2) + if (get_random_u32() < clg->a2) clg->state = GOOD_STATE; - if (prandom_u32() > clg->a3) + if (get_random_u32() > clg->a3) return true; } @@ -513,8 +513,8 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, goto finish_segs; } - skb->data[prandom_u32() % skb_headlen(skb)] ^= - 1<<(prandom_u32() % 8); + skb->data[prandom_u32_max(skb_headlen(skb))] ^= + 1<q.qlen >= sch->limit)) { @@ -632,7 +632,7 @@ static void get_slot_next(struct netem_sched_data *q, u64 now) if (!q->slot_dist) next_delay = q->slot_config.min_delay + - (prandom_u32() * + (get_random_u32() * (q->slot_config.max_delay - q->slot_config.min_delay) >> 32); else @@ -961,9 +961,6 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, int old_loss_model = CLG_RANDOM; int ret; - if (opt == NULL) - return -EINVAL; - qopt = nla_data(opt); ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt)); if (ret < 0) @@ -1254,12 +1251,8 @@ static unsigned long netem_find(struct Qdisc *sch, u32 classid) static void netem_walk(struct Qdisc *sch, struct qdisc_walker *walker) { if (!walker->stop) { - if (walker->count >= walker->skip) - if (walker->fn(sch, 1, walker) < 0) { - walker->stop = 1; - return; - } - walker->count++; + if (!tc_qdisc_stats_dump(sch, 1, walker)) + return; } } diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c index 5a457ff61acd8731a8e31ab729c7e127aa6611b5..265c238047a42f47ca18d976873857e20bebcf30 100644 --- a/net/sched/sch_pie.c +++ b/net/sched/sch_pie.c @@ -72,7 +72,7 @@ bool pie_drop_early(struct Qdisc *sch, struct pie_params *params, if (vars->accu_prob >= (MAX_PROB / 2) * 17) return true; - prandom_bytes(&rnd, 8); + get_random_bytes(&rnd, 8); if ((rnd >> BITS_PER_BYTE) < local_prob) { vars->accu_prob = 0; return true; @@ -143,9 +143,6 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt, unsigned int qlen, dropped = 0; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_PIE_MAX, opt, pie_policy, NULL); if (err < 0) diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cbc2ebca4548c927070dd2dbd7685cdb6a7ffe8f..ea8c4a7174bba0880851be8631f9234bdecc3ada 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -161,9 +161,6 @@ static int plug_change(struct Qdisc *sch, struct nlattr *opt, struct plug_sched_data *q = qdisc_priv(sch); struct tc_plug_qopt *msg; - if (opt == NULL) - return -EINVAL; - msg = nla_data(opt); if (nla_len(opt) < sizeof(*msg)) return -EINVAL; diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 3b8d7197c06bff3f346b1616db0210d6e8c5e704..fdc5ef52c3ee92c40232a5af7c68ce920b61b7e6 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -135,8 +135,6 @@ prio_reset(struct Qdisc *sch) for (prio = 0; prio < q->bands; prio++) qdisc_reset(q->queues[prio]); - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static int prio_offload(struct Qdisc *sch, struct tc_prio_qopt *qopt) @@ -187,7 +185,7 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt, return -EINVAL; qopt = nla_data(opt); - if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2) + if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < TCQ_MIN_PRIO_BANDS) return -EINVAL; for (i = 0; i <= TC_PRIO_MAX; i++) { @@ -378,15 +376,8 @@ static void prio_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (prio = 0; prio < q->bands; prio++) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, prio + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, prio + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index d4ce58c90f9fbab1a4f35e08213b11f87cce92af..cf5ebe43b3b4eb7f163da31cc565bc7faaab69e5 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -659,15 +659,8 @@ static void qfq_walk(struct Qdisc *sch, struct qdisc_walker *arg) for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, (unsigned long)cl, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg)) return; - } - arg->count++; } } } @@ -1458,8 +1451,6 @@ static void qfq_reset_qdisc(struct Qdisc *sch) qdisc_reset(cl->qdisc); } } - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void qfq_destroy_qdisc(struct Qdisc *sch) diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 40adf1f07a82dfdb8f704eecfa9a14f000213a91..a5a401f93c1a2654b6ada19177e6111079312b3b 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -176,8 +176,6 @@ static void red_reset(struct Qdisc *sch) struct red_sched_data *q = qdisc_priv(sch); qdisc_reset(q->qdisc); - sch->qstats.backlog = 0; - sch->q.qlen = 0; red_restart(&q->vars); } @@ -370,9 +368,6 @@ static int red_change(struct Qdisc *sch, struct nlattr *opt, struct nlattr *tb[TCA_RED_MAX + 1]; int err; - if (!opt) - return -EINVAL; - err = nla_parse_nested_deprecated(tb, TCA_RED_MAX, opt, red_policy, extack); if (err < 0) @@ -463,7 +458,6 @@ static int red_dump_stats(struct Qdisc *sch, struct gnet_dump *d) } st.early = q->stats.prob_drop + q->stats.forced_drop; st.pdrop = q->stats.pdrop; - st.other = q->stats.other; st.marked = q->stats.prob_mark + q->stats.forced_mark; return gnet_stats_copy_app(d, &st, sizeof(st)); @@ -522,12 +516,7 @@ static unsigned long red_find(struct Qdisc *sch, u32 classid) static void red_walk(struct Qdisc *sch, struct qdisc_walker *walker) { if (!walker->stop) { - if (walker->count >= walker->skip) - if (walker->fn(sch, 1, walker) < 0) { - walker->stop = 1; - return; - } - walker->count++; + tc_qdisc_stats_dump(sch, 1, walker); } } diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index 3d061a13d7ed2bd5294c3eddd1bd1bcdaafda464..0366a1a029a9ea8f4d08095fdf015c7594f379e1 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -135,15 +135,15 @@ static void increment_one_qlen(u32 sfbhash, u32 slot, struct sfb_sched_data *q) } } -static void increment_qlen(const struct sk_buff *skb, struct sfb_sched_data *q) +static void increment_qlen(const struct sfb_skb_cb *cb, struct sfb_sched_data *q) { u32 sfbhash; - sfbhash = sfb_hash(skb, 0); + sfbhash = cb->hashes[0]; if (sfbhash) increment_one_qlen(sfbhash, 0, q); - sfbhash = sfb_hash(skb, 1); + sfbhash = cb->hashes[1]; if (sfbhash) increment_one_qlen(sfbhash, 1, q); } @@ -281,8 +281,10 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct sfb_sched_data *q = qdisc_priv(sch); + unsigned int len = qdisc_pkt_len(skb); struct Qdisc *child = q->qdisc; struct tcf_proto *fl; + struct sfb_skb_cb cb; int i; u32 p_min = ~0; u32 minqlen = ~0; @@ -377,7 +379,7 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, goto enqueue; } - r = prandom_u32() & SFB_MAX_PROB; + r = get_random_u16() & SFB_MAX_PROB; if (unlikely(r < p_min)) { if (unlikely(p_min > SFB_MAX_PROB / 2)) { @@ -399,11 +401,12 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } enqueue: + memcpy(&cb, sfb_skb_cb(skb), sizeof(cb)); ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { - qdisc_qstats_backlog_inc(sch, skb); + sch->qstats.backlog += len; sch->q.qlen++; - increment_qlen(skb, q); + increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { q->stats.childdrop++; qdisc_qstats_drop(sch); @@ -453,8 +456,6 @@ static void sfb_reset(struct Qdisc *sch) struct sfb_sched_data *q = qdisc_priv(sch); qdisc_reset(q->qdisc); - sch->qstats.backlog = 0; - sch->q.qlen = 0; q->slot = 0; q->double_buffering = false; sfb_zero_all_buckets(q); @@ -658,12 +659,7 @@ static int sfb_delete(struct Qdisc *sch, unsigned long cl, static void sfb_walk(struct Qdisc *sch, struct qdisc_walker *walker) { if (!walker->stop) { - if (walker->count >= walker->skip) - if (walker->fn(sch, 1, walker) < 0) { - walker->stop = 1; - return; - } - walker->count++; + tc_qdisc_stats_dump(sch, 1, walker); } } diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index f8e569f79f1367563bf92793dbd2a9c0a4ce957b..abd436307d6a8492d2d297c138d3b3e0309b949f 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -888,16 +888,12 @@ static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (i = 0; i < q->divisor; i++) { - if (q->ht[i] == SFQ_EMPTY_SLOT || - arg->count < arg->skip) { + if (q->ht[i] == SFQ_EMPTY_SLOT) { arg->count++; continue; } - if (arg->fn(sch, i + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, i + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c index 7a5e4c454715617cb57c6db7de7fdaa9e6886d40..5df2dacb7b1abaecdeaa5df5f38d98dc5e2f920d 100644 --- a/net/sched/sch_skbprio.c +++ b/net/sched/sch_skbprio.c @@ -213,9 +213,6 @@ static void skbprio_reset(struct Qdisc *sch) struct skbprio_sched_data *q = qdisc_priv(sch); int prio; - sch->qstats.backlog = 0; - sch->q.qlen = 0; - for (prio = 0; prio < SKBPRIO_MAX_PRIORITY; prio++) __skb_queue_purge(&q->qdiscs[prio]); @@ -268,15 +265,8 @@ static void skbprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) return; for (i = 0; i < SKBPRIO_MAX_PRIORITY; i++) { - if (arg->count < arg->skip) { - arg->count++; - continue; - } - if (arg->fn(sch, i + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, i + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index 0b941dd63d2680d72c598369818410a91b0c576e..570389f6cdd7dbab5749dc06d886555305cbf623 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -27,7 +27,6 @@ #include static LIST_HEAD(taprio_list); -static DEFINE_SPINLOCK(taprio_list_lock); #define TAPRIO_ALL_GATES_OPEN -1 @@ -67,6 +66,7 @@ struct taprio_sched { u32 flags; enum tk_offsets tk_offset; int clockid; + bool offloaded; atomic64_t picos_per_byte; /* Using picoseconds because for 10Gbps+ * speeds it's sub-nanoseconds per byte */ @@ -78,8 +78,8 @@ struct taprio_sched { struct sched_gate_list __rcu *admin_sched; struct hrtimer advance_timer; struct list_head taprio_list; - struct sk_buff *(*dequeue)(struct Qdisc *sch); - struct sk_buff *(*peek)(struct Qdisc *sch); + u32 max_frm_len[TC_MAX_QUEUE]; /* for the fast path */ + u32 max_sdu[TC_MAX_QUEUE]; /* for dump and offloading */ u32 txtime_delay; }; @@ -417,6 +417,9 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, struct Qdisc *child, struct sk_buff **to_free) { struct taprio_sched *q = qdisc_priv(sch); + struct net_device *dev = qdisc_dev(sch); + int prio = skb->priority; + u8 tc; /* sk_flags are only safe to use on full sockets. */ if (skb->sk && sk_fullsock(skb->sk) && sock_flag(skb->sk, SOCK_TXTIME)) { @@ -428,12 +431,20 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch, return qdisc_drop(skb, sch, to_free); } + /* Devices with full offload are expected to honor this in hardware */ + tc = netdev_get_prio_tc_map(dev, prio); + if (skb->len > q->max_frm_len[tc]) + return qdisc_drop(skb, sch, to_free); + qdisc_qstats_backlog_inc(sch, skb); sch->q.qlen++; return qdisc_enqueue(skb, child, to_free); } +/* Will not be called in the full offload case, since the TX queues are + * attached to the Qdisc created using qdisc_create_dflt() + */ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { @@ -441,11 +452,6 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct Qdisc *child; int queue; - if (unlikely(FULL_OFFLOAD_IS_ENABLED(q->flags))) { - WARN_ONCE(1, "Trying to enqueue skb into the root of a taprio qdisc configured with full offload\n"); - return qdisc_drop(skb, sch, to_free); - } - queue = skb_get_queue_mapping(skb); child = q->qdiscs[queue]; @@ -454,10 +460,10 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* Large packets might not be transmitted when the transmission duration * exceeds any configured interval. Therefore, segment the skb into - * smaller chunks. Skip it for the full offload case, as the driver - * and/or the hardware is expected to handle this. + * smaller chunks. Drivers with full offload are expected to handle + * this in hardware. */ - if (skb_is_gso(skb) && !FULL_OFFLOAD_IS_ENABLED(q->flags)) { + if (skb_is_gso(skb)) { unsigned int slen = 0, numsegs = 0, len = qdisc_pkt_len(skb); netdev_features_t features = netif_skb_features(skb); struct sk_buff *segs, *nskb; @@ -491,7 +497,10 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, return taprio_enqueue_one(skb, sch, child, to_free); } -static struct sk_buff *taprio_peek_soft(struct Qdisc *sch) +/* Will not be called in the full offload case, since the TX queues are + * attached to the Qdisc created using qdisc_create_dflt() + */ +static struct sk_buff *taprio_peek(struct Qdisc *sch) { struct taprio_sched *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); @@ -535,20 +544,6 @@ static struct sk_buff *taprio_peek_soft(struct Qdisc *sch) return NULL; } -static struct sk_buff *taprio_peek_offload(struct Qdisc *sch) -{ - WARN_ONCE(1, "Trying to peek into the root of a taprio qdisc configured with full offload\n"); - - return NULL; -} - -static struct sk_buff *taprio_peek(struct Qdisc *sch) -{ - struct taprio_sched *q = qdisc_priv(sch); - - return q->peek(sch); -} - static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry) { atomic_set(&entry->budget, @@ -556,7 +551,10 @@ static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry) atomic64_read(&q->picos_per_byte))); } -static struct sk_buff *taprio_dequeue_soft(struct Qdisc *sch) +/* Will not be called in the full offload case, since the TX queues are + * attached to the Qdisc created using qdisc_create_dflt() + */ +static struct sk_buff *taprio_dequeue(struct Qdisc *sch) { struct taprio_sched *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); @@ -644,20 +642,6 @@ done: return skb; } -static struct sk_buff *taprio_dequeue_offload(struct Qdisc *sch) -{ - WARN_ONCE(1, "Trying to dequeue from the root of a taprio qdisc configured with full offload\n"); - - return NULL; -} - -static struct sk_buff *taprio_dequeue(struct Qdisc *sch) -{ - struct taprio_sched *q = qdisc_priv(sch); - - return q->dequeue(sch); -} - static bool should_restart_cycle(const struct sched_gate_list *oper, const struct sched_entry *entry) { @@ -780,6 +764,11 @@ static const struct nla_policy entry_policy[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = { [TCA_TAPRIO_SCHED_ENTRY_INTERVAL] = { .type = NLA_U32 }, }; +static const struct nla_policy taprio_tc_policy[TCA_TAPRIO_TC_ENTRY_MAX + 1] = { + [TCA_TAPRIO_TC_ENTRY_INDEX] = { .type = NLA_U32 }, + [TCA_TAPRIO_TC_ENTRY_MAX_SDU] = { .type = NLA_U32 }, +}; + static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = { [TCA_TAPRIO_ATTR_PRIOMAP] = { .len = sizeof(struct tc_mqprio_qopt) @@ -792,6 +781,7 @@ static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = { [TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION] = { .type = NLA_S64 }, [TCA_TAPRIO_ATTR_FLAGS] = { .type = NLA_U32 }, [TCA_TAPRIO_ATTR_TXTIME_DELAY] = { .type = NLA_U32 }, + [TCA_TAPRIO_ATTR_TC_ENTRY] = { .type = NLA_NESTED }, }; static int fill_sched_entry(struct taprio_sched *q, struct nlattr **tb, @@ -1098,27 +1088,20 @@ static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); - struct net_device *qdev; struct taprio_sched *q; - bool found = false; ASSERT_RTNL(); if (event != NETDEV_UP && event != NETDEV_CHANGE) return NOTIFY_DONE; - spin_lock(&taprio_list_lock); list_for_each_entry(q, &taprio_list, taprio_list) { - qdev = qdisc_dev(q->root); - if (qdev == dev) { - found = true; - break; - } - } - spin_unlock(&taprio_list_lock); + if (dev != qdisc_dev(q->root)) + continue; - if (found) taprio_set_picos_per_byte(dev, q); + break; + } return NOTIFY_DONE; } @@ -1193,16 +1176,10 @@ static void taprio_offload_config_changed(struct taprio_sched *q) { struct sched_gate_list *oper, *admin; - spin_lock(&q->current_entry_lock); - - oper = rcu_dereference_protected(q->oper_sched, - lockdep_is_held(&q->current_entry_lock)); - admin = rcu_dereference_protected(q->admin_sched, - lockdep_is_held(&q->current_entry_lock)); + oper = rtnl_dereference(q->oper_sched); + admin = rtnl_dereference(q->admin_sched); switch_schedules(q, &admin, &oper); - - spin_unlock(&q->current_entry_lock); } static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask) @@ -1255,7 +1232,8 @@ static int taprio_enable_offload(struct net_device *dev, { const struct net_device_ops *ops = dev->netdev_ops; struct tc_taprio_qopt_offload *offload; - int err = 0; + struct tc_taprio_caps caps; + int tc, err = 0; if (!ops->ndo_setup_tc) { NL_SET_ERR_MSG(extack, @@ -1263,6 +1241,19 @@ static int taprio_enable_offload(struct net_device *dev, return -EOPNOTSUPP; } + qdisc_offload_query_caps(dev, TC_SETUP_QDISC_TAPRIO, + &caps, sizeof(caps)); + + if (!caps.supports_queue_max_sdu) { + for (tc = 0; tc < TC_MAX_QUEUE; tc++) { + if (q->max_sdu[tc]) { + NL_SET_ERR_MSG_MOD(extack, + "Device does not handle queueMaxSDU"); + return -EOPNOTSUPP; + } + } + } + offload = taprio_offload_alloc(sched->num_entries); if (!offload) { NL_SET_ERR_MSG(extack, @@ -1272,6 +1263,9 @@ static int taprio_enable_offload(struct net_device *dev, offload->enable = 1; taprio_sched_to_offload(dev, sched, offload); + for (tc = 0; tc < TC_MAX_QUEUE; tc++) + offload->max_sdu[tc] = q->max_sdu[tc]; + err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload); if (err < 0) { NL_SET_ERR_MSG(extack, @@ -1279,6 +1273,8 @@ static int taprio_enable_offload(struct net_device *dev, goto done; } + q->offloaded = true; + done: taprio_offload_free(offload); @@ -1293,12 +1289,9 @@ static int taprio_disable_offload(struct net_device *dev, struct tc_taprio_qopt_offload *offload; int err; - if (!FULL_OFFLOAD_IS_ENABLED(q->flags)) + if (!q->offloaded) return 0; - if (!ops->ndo_setup_tc) - return -EOPNOTSUPP; - offload = taprio_offload_alloc(0); if (!offload) { NL_SET_ERR_MSG(extack, @@ -1314,6 +1307,8 @@ static int taprio_disable_offload(struct net_device *dev, goto out; } + q->offloaded = false; + out: taprio_offload_free(offload); @@ -1405,6 +1400,89 @@ out: return err; } +static int taprio_parse_tc_entry(struct Qdisc *sch, + struct nlattr *opt, + u32 max_sdu[TC_QOPT_MAX_QUEUE], + unsigned long *seen_tcs, + struct netlink_ext_ack *extack) +{ + struct nlattr *tb[TCA_TAPRIO_TC_ENTRY_MAX + 1] = { }; + struct net_device *dev = qdisc_dev(sch); + u32 val = 0; + int err, tc; + + err = nla_parse_nested(tb, TCA_TAPRIO_TC_ENTRY_MAX, opt, + taprio_tc_policy, extack); + if (err < 0) + return err; + + if (!tb[TCA_TAPRIO_TC_ENTRY_INDEX]) { + NL_SET_ERR_MSG_MOD(extack, "TC entry index missing"); + return -EINVAL; + } + + tc = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_INDEX]); + if (tc >= TC_QOPT_MAX_QUEUE) { + NL_SET_ERR_MSG_MOD(extack, "TC entry index out of range"); + return -ERANGE; + } + + if (*seen_tcs & BIT(tc)) { + NL_SET_ERR_MSG_MOD(extack, "Duplicate TC entry"); + return -EINVAL; + } + + *seen_tcs |= BIT(tc); + + if (tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU]) + val = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU]); + + if (val > dev->max_mtu) { + NL_SET_ERR_MSG_MOD(extack, "TC max SDU exceeds device max MTU"); + return -ERANGE; + } + + max_sdu[tc] = val; + + return 0; +} + +static int taprio_parse_tc_entries(struct Qdisc *sch, + struct nlattr *opt, + struct netlink_ext_ack *extack) +{ + struct taprio_sched *q = qdisc_priv(sch); + struct net_device *dev = qdisc_dev(sch); + u32 max_sdu[TC_QOPT_MAX_QUEUE]; + unsigned long seen_tcs = 0; + struct nlattr *n; + int tc, rem; + int err = 0; + + for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) + max_sdu[tc] = q->max_sdu[tc]; + + nla_for_each_nested(n, opt, rem) { + if (nla_type(n) != TCA_TAPRIO_ATTR_TC_ENTRY) + continue; + + err = taprio_parse_tc_entry(sch, n, max_sdu, &seen_tcs, extack); + if (err) + goto out; + } + + for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) { + q->max_sdu[tc] = max_sdu[tc]; + if (max_sdu[tc]) + q->max_frm_len[tc] = max_sdu[tc] + dev->hard_header_len; + else + q->max_frm_len[tc] = U32_MAX; /* never oversized */ + } + +out: + return err; +} + static int taprio_mqprio_cmp(const struct net_device *dev, const struct tc_mqprio_qopt *mqprio) { @@ -1483,6 +1561,10 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt, if (err < 0) return err; + err = taprio_parse_tc_entries(sch, opt, extack); + if (err) + return err; + new_admin = kzalloc(sizeof(*new_admin), GFP_KERNEL); if (!new_admin) { NL_SET_ERR_MSG(extack, "Not enough memory for a new schedule"); @@ -1490,10 +1572,8 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt, } INIT_LIST_HEAD(&new_admin->entries); - rcu_read_lock(); - oper = rcu_dereference(q->oper_sched); - admin = rcu_dereference(q->admin_sched); - rcu_read_unlock(); + oper = rtnl_dereference(q->oper_sched); + admin = rtnl_dereference(q->admin_sched); /* no changes - no new mqprio settings */ if (!taprio_mqprio_cmp(dev, mqprio)) @@ -1563,17 +1643,6 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt, q->advance_timer.function = advance_sched; } - if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { - q->dequeue = taprio_dequeue_offload; - q->peek = taprio_peek_offload; - } else { - /* Be sure to always keep the function pointers - * in a consistent state. - */ - q->dequeue = taprio_dequeue_soft; - q->peek = taprio_peek_soft; - } - err = taprio_get_start_time(sch, new_admin, &start); if (err < 0) { NL_SET_ERR_MSG(extack, "Internal error: failed get start time"); @@ -1636,19 +1705,16 @@ static void taprio_reset(struct Qdisc *sch) if (q->qdiscs[i]) qdisc_reset(q->qdiscs[i]); } - sch->qstats.backlog = 0; - sch->q.qlen = 0; } static void taprio_destroy(struct Qdisc *sch) { struct taprio_sched *q = qdisc_priv(sch); struct net_device *dev = qdisc_dev(sch); + struct sched_gate_list *oper, *admin; unsigned int i; - spin_lock(&taprio_list_lock); list_del(&q->taprio_list); - spin_unlock(&taprio_list_lock); /* Note that taprio_reset() might not be called if an error * happens in qdisc_create(), after taprio_init() has been called. @@ -1667,11 +1733,14 @@ static void taprio_destroy(struct Qdisc *sch) netdev_reset_tc(dev); - if (q->oper_sched) - call_rcu(&q->oper_sched->rcu, taprio_free_sched_cb); + oper = rtnl_dereference(q->oper_sched); + admin = rtnl_dereference(q->admin_sched); - if (q->admin_sched) - call_rcu(&q->admin_sched->rcu, taprio_free_sched_cb); + if (oper) + call_rcu(&oper->rcu, taprio_free_sched_cb); + + if (admin) + call_rcu(&admin->rcu, taprio_free_sched_cb); } static int taprio_init(struct Qdisc *sch, struct nlattr *opt, @@ -1686,9 +1755,6 @@ static int taprio_init(struct Qdisc *sch, struct nlattr *opt, hrtimer_init(&q->advance_timer, CLOCK_TAI, HRTIMER_MODE_ABS); q->advance_timer.function = advance_sched; - q->dequeue = taprio_dequeue_soft; - q->peek = taprio_peek_soft; - q->root = sch; /* We only support static clockids. Use an invalid value as default @@ -1697,15 +1763,17 @@ static int taprio_init(struct Qdisc *sch, struct nlattr *opt, q->clockid = -1; q->flags = TAPRIO_FLAGS_INVALID; - spin_lock(&taprio_list_lock); list_add(&q->taprio_list, &taprio_list); - spin_unlock(&taprio_list_lock); - if (sch->parent != TC_H_ROOT) + if (sch->parent != TC_H_ROOT) { + NL_SET_ERR_MSG_MOD(extack, "Can only be attached as root qdisc"); return -EOPNOTSUPP; + } - if (!netif_is_multiqueue(dev)) + if (!netif_is_multiqueue(dev)) { + NL_SET_ERR_MSG_MOD(extack, "Multi-queue device is required"); return -EOPNOTSUPP; + } /* pre-allocate qdisc, attachment can't fail */ q->qdiscs = kcalloc(dev->num_tx_queues, @@ -1877,6 +1945,33 @@ error_nest: return -1; } +static int taprio_dump_tc_entries(struct taprio_sched *q, struct sk_buff *skb) +{ + struct nlattr *n; + int tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) { + n = nla_nest_start(skb, TCA_TAPRIO_ATTR_TC_ENTRY); + if (!n) + return -EMSGSIZE; + + if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_INDEX, tc)) + goto nla_put_failure; + + if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_MAX_SDU, + q->max_sdu[tc])) + goto nla_put_failure; + + nla_nest_end(skb, n); + } + + return 0; + +nla_put_failure: + nla_nest_cancel(skb, n); + return -EMSGSIZE; +} + static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) { struct taprio_sched *q = qdisc_priv(sch); @@ -1886,9 +1981,8 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) struct nlattr *nest, *sched_nest; unsigned int i; - rcu_read_lock(); - oper = rcu_dereference(q->oper_sched); - admin = rcu_dereference(q->admin_sched); + oper = rtnl_dereference(q->oper_sched); + admin = rtnl_dereference(q->admin_sched); opt.num_tc = netdev_get_num_tc(dev); memcpy(opt.prio_tc_map, dev->prio_tc_map, sizeof(opt.prio_tc_map)); @@ -1916,6 +2010,9 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay)) goto options_error; + if (taprio_dump_tc_entries(q, skb)) + goto options_error; + if (oper && dump_schedule(skb, oper)) goto options_error; @@ -1932,8 +2029,6 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb) nla_nest_end(skb, sched_nest); done: - rcu_read_unlock(); - return nla_nest_end(skb, nest); admin_error: @@ -1943,7 +2038,6 @@ options_error: nla_nest_cancel(skb, nest); start_error: - rcu_read_unlock(); return -ENOSPC; } @@ -2002,11 +2096,8 @@ static void taprio_walk(struct Qdisc *sch, struct qdisc_walker *arg) arg->count = arg->skip; for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) { - if (arg->fn(sch, ntx + 1, arg) < 0) { - arg->stop = 1; + if (!tc_qdisc_stats_dump(sch, ntx + 1, arg)) break; - } - arg->count++; } } diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c index 72102277449e1bae0cb23db59878d698ee90c353..277ad11f4d6135373e0a196ed423e0beddec1470 100644 --- a/net/sched/sch_tbf.c +++ b/net/sched/sch_tbf.c @@ -330,8 +330,6 @@ static void tbf_reset(struct Qdisc *sch) struct tbf_sched_data *q = qdisc_priv(sch); qdisc_reset(q->qdisc); - sch->qstats.backlog = 0; - sch->q.qlen = 0; q->t_c = ktime_get_ns(); q->tokens = q->buffer; q->ptokens = q->mtu; @@ -356,6 +354,7 @@ static int tbf_change(struct Qdisc *sch, struct nlattr *opt, struct nlattr *tb[TCA_TBF_MAX + 1]; struct tc_tbf_qopt *qopt; struct Qdisc *child = NULL; + struct Qdisc *old = NULL; struct psched_ratecfg rate; struct psched_ratecfg peak; u64 max_size; @@ -447,7 +446,7 @@ static int tbf_change(struct Qdisc *sch, struct nlattr *opt, sch_tree_lock(sch); if (child) { qdisc_tree_flush_backlog(q->qdisc); - qdisc_put(q->qdisc); + old = q->qdisc; q->qdisc = child; } q->limit = qopt->limit; @@ -467,6 +466,7 @@ static int tbf_change(struct Qdisc *sch, struct nlattr *opt, memcpy(&q->peak, &peak, sizeof(struct psched_ratecfg)); sch_tree_unlock(sch); + qdisc_put(old); err = 0; tbf_offload_change(sch); @@ -580,12 +580,7 @@ static unsigned long tbf_find(struct Qdisc *sch, u32 classid) static void tbf_walk(struct Qdisc *sch, struct qdisc_walker *walker) { if (!walker->stop) { - if (walker->count >= walker->skip) - if (walker->fn(sch, 1, walker) < 0) { - walker->stop = 1; - return; - } - walker->count++; + tc_qdisc_stats_dump(sch, 1, walker); } } diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 6af6b95bdb6723a3d600c1defae08fd897c7a308..16f9238aa51d133f7004b97b7aa558354634b73d 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -124,7 +124,6 @@ teql_reset(struct Qdisc *sch) struct teql_sched_data *dat = qdisc_priv(sch); skb_queue_purge(&dat->q); - sch->q.qlen = 0; } static void @@ -492,7 +491,7 @@ static int __init teql_init(void) master = netdev_priv(dev); - strlcpy(master->qops.id, dev->name, IFNAMSIZ); + strscpy(master->qops.id, dev->name, IFNAMSIZ); err = register_qdisc(&master->qops); if (err) { diff --git a/net/sctp/auth.c b/net/sctp/auth.c index db6b7373d16c374f1dba808070e95309d71c90d7..34964145514e6d8c284b5a2687ac19ef1e8b4cf0 100644 --- a/net/sctp/auth.c +++ b/net/sctp/auth.c @@ -863,12 +863,17 @@ int sctp_auth_set_key(struct sctp_endpoint *ep, } list_del_init(&shkey->key_list); - sctp_auth_shkey_release(shkey); list_add(&cur_key->key_list, sh_keys); - if (asoc && asoc->active_key_id == auth_key->sca_keynumber) - sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL); + if (asoc && asoc->active_key_id == auth_key->sca_keynumber && + sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL)) { + list_del_init(&cur_key->key_list); + sctp_auth_shkey_release(cur_key); + list_add(&shkey->key_list, sh_keys); + return -ENOMEM; + } + sctp_auth_shkey_release(shkey); return 0; } @@ -902,8 +907,13 @@ int sctp_auth_set_active_key(struct sctp_endpoint *ep, return -EINVAL; if (asoc) { + __u16 active_key_id = asoc->active_key_id; + asoc->active_key_id = key_id; - sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL); + if (sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL)) { + asoc->active_key_id = active_key_id; + return -ENOMEM; + } } else ep->active_key_id = key_id; diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 171f1a35d205225cdec9dbc6d0a07f446ce92996..83628c347744b32987c256c4a49a3c6fc19e20da 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -8319,7 +8319,7 @@ static int sctp_get_port_local(struct sock *sk, union sctp_addr *addr) inet_get_local_port_range(net, &low, &high); remaining = (high - low) + 1; - rover = prandom_u32() % remaining + low; + rover = prandom_u32_max(remaining) + low; do { rover++; @@ -9448,7 +9448,7 @@ void sctp_copy_sock(struct sock *newsk, struct sock *sk, newinet->inet_rcv_saddr = inet->inet_rcv_saddr; newinet->inet_dport = htons(asoc->peer.port); newinet->pmtudisc = inet->pmtudisc; - newinet->inet_id = prandom_u32(); + newinet->inet_id = get_random_u16(); newinet->uc_ttl = inet->uc_ttl; newinet->mc_loop = 1; diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 79c1318af1fefae8e6fc2fac27434e086da5b05c..3ccbf3c201cd27851037cd302d39b5b0124ca765 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -379,6 +379,8 @@ static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, sk->sk_state = SMC_INIT; sk->sk_destruct = smc_destruct; sk->sk_protocol = protocol; + WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(net->smc.sysctl_wmem)); + WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(net->smc.sysctl_rmem)); smc = smc_sk(sk); INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work); INIT_WORK(&smc->connect_work, smc_connect_work); @@ -427,6 +429,7 @@ static int smc_bind(struct socket *sock, struct sockaddr *uaddr, goto out_rel; smc->clcsock->sk->sk_reuse = sk->sk_reuse; + smc->clcsock->sk->sk_reuseport = sk->sk_reuseport; rc = kernel_bind(smc->clcsock, uaddr, addr_len); out_rel: @@ -1855,7 +1858,6 @@ static void smc_listen_out_connected(struct smc_sock *new_smc) { struct sock *newsmcsk = &new_smc->sk; - sk_refcnt_debug_inc(newsmcsk); if (newsmcsk->sk_state == SMC_INIT) newsmcsk->sk_state = SMC_ACTIVE; @@ -3254,9 +3256,6 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol, smc->clcsock = clcsock; } - smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE); - smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE); - out: return rc; } diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index ff49a11f57b87fa79f28c3385680e954c7bda13a..e6ee797640b451e3b54a3215a66f91ebfc2c46fc 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -757,6 +757,7 @@ int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk, lnk->lgr = lgr; smc_lgr_hold(lgr); /* lgr_put in smcr_link_clear() */ lnk->link_idx = link_idx; + lnk->wr_rx_id_compl = 0; smc_ibdev_cnt_inc(lnk); smcr_copy_dev_info_to_link(lnk); atomic_set(&lnk->conn_cnt, 0); @@ -2238,7 +2239,7 @@ out: static int smcr_buf_map_usable_links(struct smc_link_group *lgr, struct smc_buf_desc *buf_desc, bool is_rmb) { - int i, rc = 0; + int i, rc = 0, cnt = 0; /* protect against parallel link reconfiguration */ mutex_lock(&lgr->llc_conf_mutex); @@ -2251,9 +2252,12 @@ static int smcr_buf_map_usable_links(struct smc_link_group *lgr, rc = -ENOMEM; goto out; } + cnt++; } out: mutex_unlock(&lgr->llc_conf_mutex); + if (!rc && !cnt) + rc = -EINVAL; return rc; } @@ -2306,10 +2310,10 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) if (is_rmb) /* use socket recv buffer size (w/o overhead) as start value */ - sk_buf_size = smc->sk.sk_rcvbuf / 2; + sk_buf_size = smc->sk.sk_rcvbuf; else /* use socket send buffer size (w/o overhead) as start value */ - sk_buf_size = smc->sk.sk_sndbuf / 2; + sk_buf_size = smc->sk.sk_sndbuf; for (bufsize_short = smc_compress_bufsize(sk_buf_size, is_smcd, is_rmb); bufsize_short >= 0; bufsize_short--) { @@ -2368,7 +2372,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) if (is_rmb) { conn->rmb_desc = buf_desc; conn->rmbe_size_short = bufsize_short; - smc->sk.sk_rcvbuf = bufsize * 2; + smc->sk.sk_rcvbuf = bufsize; atomic_set(&conn->bytes_to_rcv, 0); conn->rmbe_update_limit = smc_rmb_wnd_update_limit(buf_desc->len); @@ -2376,7 +2380,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */ } else { conn->sndbuf_desc = buf_desc; - smc->sk.sk_sndbuf = bufsize * 2; + smc->sk.sk_sndbuf = bufsize; atomic_set(&conn->sndbuf_space, bufsize); } return 0; diff --git a/net/smc/smc_core.h b/net/smc/smc_core.h index fe8b524ad846b83e6c6f4f97a05f0cbd5ee6719e..285f9bd8e232e5c54bbac82c3cdddc0bd408285e 100644 --- a/net/smc/smc_core.h +++ b/net/smc/smc_core.h @@ -115,8 +115,10 @@ struct smc_link { dma_addr_t wr_rx_dma_addr; /* DMA address of wr_rx_bufs */ dma_addr_t wr_rx_v2_dma_addr; /* DMA address of v2 rx buf*/ u64 wr_rx_id; /* seq # of last recv WR */ + u64 wr_rx_id_compl; /* seq # of last completed WR */ u32 wr_rx_cnt; /* number of WR recv buffers */ unsigned long wr_rx_tstamp; /* jiffies when last buf rx */ + wait_queue_head_t wr_rx_empty_wait; /* wait for RQ empty */ struct ib_reg_wr wr_reg; /* WR register memory region */ wait_queue_head_t wr_reg_wait; /* wait for wr_reg result */ diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index 175026ae33ae9e39f0466c006a06b2ce1725bdab..524649d0ab6520484b25f410d83b31f78793df5b 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -2127,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) init_waitqueue_head(&lgr->llc_flow_waiter); init_waitqueue_head(&lgr->llc_msg_waiter); mutex_init(&lgr->llc_conf_mutex); - lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time); + lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time); } /* called after lgr was removed from lgr_list */ diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h index 4404e52b3346fbc51855667fb4832f32a9987ef9..7e7a3162c68b3d9923a9a5ed4315ec67c4c5fb85 100644 --- a/net/smc/smc_llc.h +++ b/net/smc/smc_llc.h @@ -19,6 +19,7 @@ #define SMC_LLC_WAIT_FIRST_TIME (5 * HZ) #define SMC_LLC_WAIT_TIME (2 * HZ) +#define SMC_LLC_TESTLINK_DEFAULT_TIME (30 * HZ) enum smc_llc_reqresp { SMC_LLC_REQ, diff --git a/net/smc/smc_netlink.c b/net/smc/smc_netlink.c index c5a62f6f52ba5fc092072899916c0d30971d6d26..621c46c700739df1e6db3f13d089f6f57977a11b 100644 --- a/net/smc/smc_netlink.c +++ b/net/smc/smc_netlink.c @@ -142,7 +142,8 @@ struct genl_family smc_gen_nl_family __ro_after_init = { .netnsok = true, .module = THIS_MODULE, .ops = smc_gen_nl_ops, - .n_ops = ARRAY_SIZE(smc_gen_nl_ops) + .n_ops = ARRAY_SIZE(smc_gen_nl_ops), + .resv_start_op = SMC_NETLINK_DISABLE_HS_LIMITATION + 1, }; int __init smc_nl_init(void) diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index 4c3bf6db7038e8f71bec760c119c0ac7cd0c7eec..25fb2fd186e22f5132a1c67d2ac3d8c5658a9fe2 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -715,7 +715,8 @@ static struct genl_family smc_pnet_nl_family __ro_after_init = { .netnsok = true, .module = THIS_MODULE, .ops = smc_pnet_ops, - .n_ops = ARRAY_SIZE(smc_pnet_ops) + .n_ops = ARRAY_SIZE(smc_pnet_ops), + .resv_start_op = SMC_PNETID_FLUSH + 1, }; bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid) diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c index 0613868fdb979bc14d6a8989066503393e06cc23..b6f79fabb9d3f2256af847aff3ff7cf57ec8f4bb 100644 --- a/net/smc/smc_sysctl.c +++ b/net/smc/smc_sysctl.c @@ -16,8 +16,12 @@ #include "smc.h" #include "smc_core.h" +#include "smc_llc.h" #include "smc_sysctl.h" +static int min_sndbuf = SMC_BUF_MIN_SIZE; +static int min_rcvbuf = SMC_BUF_MIN_SIZE; + static struct ctl_table smc_table[] = { { .procname = "autocorking_size", @@ -35,6 +39,29 @@ static struct ctl_table smc_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO, }, + { + .procname = "smcr_testlink_time", + .data = &init_net.smc.sysctl_smcr_testlink_time, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_jiffies, + }, + { + .procname = "wmem", + .data = &init_net.smc.sysctl_wmem, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &min_sndbuf, + }, + { + .procname = "rmem", + .data = &init_net.smc.sysctl_rmem, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &min_rcvbuf, + }, { } }; @@ -60,6 +87,9 @@ int __net_init smc_sysctl_net_init(struct net *net) net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE; net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS; + net->smc.sysctl_smcr_testlink_time = SMC_LLC_TESTLINK_DEFAULT_TIME; + WRITE_ONCE(net->smc.sysctl_wmem, READ_ONCE(net->ipv4.sysctl_tcp_wmem[1])); + WRITE_ONCE(net->smc.sysctl_rmem, READ_ONCE(net->ipv4.sysctl_tcp_rmem[1])); return 0; diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c index 26f8f240d9e8473e9901cc1ce8c4e253651b2943..b0678a417e09d73d28c6382acc7b6e07e324c501 100644 --- a/net/smc/smc_wr.c +++ b/net/smc/smc_wr.c @@ -454,6 +454,7 @@ static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) for (i = 0; i < num; i++) { link = wc[i].qp->qp_context; + link->wr_rx_id_compl = wc[i].wr_id; if (wc[i].status == IB_WC_SUCCESS) { link->wr_rx_tstamp = jiffies; smc_wr_rx_demultiplex(&wc[i]); @@ -465,6 +466,8 @@ static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num) case IB_WC_RNR_RETRY_EXC_ERR: case IB_WC_WR_FLUSH_ERR: smcr_link_down_cond_sched(link); + if (link->wr_rx_id_compl == link->wr_rx_id) + wake_up(&link->wr_rx_empty_wait); break; default: smc_wr_rx_post(link); /* refill WR RX */ @@ -639,6 +642,7 @@ void smc_wr_free_link(struct smc_link *lnk) return; ibdev = lnk->smcibdev->ibdev; + smc_wr_drain_cq(lnk); smc_wr_wakeup_reg_wait(lnk); smc_wr_wakeup_tx_wait(lnk); @@ -889,6 +893,7 @@ int smc_wr_create_link(struct smc_link *lnk) atomic_set(&lnk->wr_tx_refcnt, 0); init_waitqueue_head(&lnk->wr_reg_wait); atomic_set(&lnk->wr_reg_refcnt, 0); + init_waitqueue_head(&lnk->wr_rx_empty_wait); return rc; dma_unmap: diff --git a/net/smc/smc_wr.h b/net/smc/smc_wr.h index a54e90a1110fdb03449e150afe9420b0aa0d93a2..45e9b894d3f8a427c88950f213128887118bee46 100644 --- a/net/smc/smc_wr.h +++ b/net/smc/smc_wr.h @@ -73,6 +73,11 @@ static inline void smc_wr_tx_link_put(struct smc_link *link) wake_up_all(&link->wr_tx_wait); } +static inline void smc_wr_drain_cq(struct smc_link *lnk) +{ + wait_event(lnk->wr_rx_empty_wait, lnk->wr_rx_id_compl == lnk->wr_rx_id); +} + static inline void smc_wr_wakeup_tx_wait(struct smc_link *lnk) { wake_up_all(&lnk->wr_tx_wait); diff --git a/net/socket.c b/net/socket.c index 9b27c5e4e5ba829c2fc31a74e3b38aa2c0b006a1..00da9ce3dba0bf336c07a320c867c4957c6e25c8 100644 --- a/net/socket.c +++ b/net/socket.c @@ -355,7 +355,7 @@ static const struct super_operations sockfs_ops = { */ static char *sockfs_dname(struct dentry *dentry, char *buffer, int buflen) { - return dynamic_dname(dentry, buffer, buflen, "socket:[%lu]", + return dynamic_dname(buffer, buflen, "socket:[%lu]", d_inode(dentry)->i_ino); } @@ -1801,7 +1801,7 @@ int __sys_listen(int fd, int backlog) sock = sockfd_lookup_light(fd, &err, &fput_needed); if (sock) { - somaxconn = sock_net(sock->sk)->core.sysctl_somaxconn; + somaxconn = READ_ONCE(sock_net(sock->sk)->core.sysctl_somaxconn); if ((unsigned int)backlog > somaxconn) backlog = somaxconn; diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index 5f96e75f9eecfdab7424ac5c421d53bb637546b4..48337687848c6654fdcfe08fe26d10a5fda12b62 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -130,8 +130,8 @@ gss_krb5_make_confounder(char *p, u32 conflen) /* initialize to random value */ if (i == 0) { - i = prandom_u32(); - i = (i << 32) | prandom_u32(); + i = get_random_u32(); + i = (i << 32) | get_random_u32(); } switch (conflen) { diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index c3c693b51c94582766e00a0e816c748512e76c47..f075a9fb5ccc6cf5a008c44ac391436689f2d030 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -677,7 +677,7 @@ static void cache_limit_defers(void) /* Consider removing either the first or the last */ if (cache_defer_cnt > DFR_MAX) { - if (prandom_u32() & 1) + if (prandom_u32_max(2)) discard = list_entry(cache_defer_list.next, struct cache_deferred_req, recent); else diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index b098e707ad4155f235046b02a355fb2e043d592a..993acf38af8704532fbc0d81f7adbadf08299deb 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -345,7 +345,7 @@ static int rpc_alloc_clid(struct rpc_clnt *clnt) { int clid; - clid = ida_simple_get(&rpc_clids, 0, 0, GFP_KERNEL); + clid = ida_alloc(&rpc_clids, GFP_KERNEL); if (clid < 0) return clid; clnt->cl_clid = clid; @@ -354,7 +354,7 @@ static int rpc_alloc_clid(struct rpc_clnt *clnt) static void rpc_free_clid(struct rpc_clnt *clnt) { - ida_simple_remove(&rpc_clids, clnt->cl_clid); + ida_free(&rpc_clids, clnt->cl_clid); } static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, @@ -873,6 +873,57 @@ void rpc_killall_tasks(struct rpc_clnt *clnt) } EXPORT_SYMBOL_GPL(rpc_killall_tasks); +/** + * rpc_cancel_tasks - try to cancel a set of RPC tasks + * @clnt: Pointer to RPC client + * @error: RPC task error value to set + * @fnmatch: Pointer to selector function + * @data: User data + * + * Uses @fnmatch to define a set of RPC tasks that are to be cancelled. + * The argument @error must be a negative error value. + */ +unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data) +{ + struct rpc_task *task; + unsigned long count = 0; + + if (list_empty(&clnt->cl_tasks)) + return 0; + /* + * Spin lock all_tasks to prevent changes... + */ + spin_lock(&clnt->cl_lock); + list_for_each_entry(task, &clnt->cl_tasks, tk_task) { + if (!RPC_IS_ACTIVATED(task)) + continue; + if (!fnmatch(task, data)) + continue; + rpc_task_try_cancel(task, error); + count++; + } + spin_unlock(&clnt->cl_lock); + return count; +} +EXPORT_SYMBOL_GPL(rpc_cancel_tasks); + +static int rpc_clnt_disconnect_xprt(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, void *dummy) +{ + if (xprt_connected(xprt)) + xprt_force_disconnect(xprt); + return 0; +} + +void rpc_clnt_disconnect(struct rpc_clnt *clnt) +{ + rpc_clnt_iterate_for_each_xprt(clnt, rpc_clnt_disconnect_xprt, NULL); +} +EXPORT_SYMBOL_GPL(rpc_clnt_disconnect); + /* * Properly shut down an RPC client, terminating all outstanding * requests. @@ -1642,7 +1693,7 @@ static void __rpc_call_rpcerror(struct rpc_task *task, int tk_status, int rpc_status) { trace_rpc_call_rpcerror(task, tk_status, rpc_status); - task->tk_rpc_status = rpc_status; + rpc_task_set_rpc_status(task, rpc_status); rpc_exit(task, tk_status); } @@ -1902,7 +1953,7 @@ call_encode(struct rpc_task *task) break; case -EKEYEXPIRED: if (!task->tk_cred_retry) { - rpc_exit(task, task->tk_status); + rpc_call_rpcerror(task, task->tk_status); } else { task->tk_action = call_refresh; task->tk_cred_retry--; @@ -2435,10 +2486,8 @@ rpc_check_timeout(struct rpc_task *task) { struct rpc_clnt *clnt = task->tk_client; - if (RPC_SIGNALLED(task)) { - rpc_call_rpcerror(task, -ERESTARTSYS); + if (RPC_SIGNALLED(task)) return; - } if (xprt_adjust_timeout(task->tk_rqstp) == 0) return; @@ -2873,6 +2922,9 @@ int rpc_clnt_test_and_add_xprt(struct rpc_clnt *clnt, task = rpc_call_null_helper(clnt, xprt, NULL, RPC_TASK_ASYNC, &rpc_cb_add_xprt_call_ops, data); + if (IS_ERR(task)) + return PTR_ERR(task); + data->xps->xps_nunique_destaddr_xprts++; rpc_put_task(task); success: diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 25b9221950ffbfa43f47498c6af563548f2eb54f..be587a308e05a5f84432dfcfa3c59e6dd0acba8b 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -65,6 +65,13 @@ gfp_t rpc_task_gfp_mask(void) } EXPORT_SYMBOL_GPL(rpc_task_gfp_mask); +bool rpc_task_set_rpc_status(struct rpc_task *task, int rpc_status) +{ + if (cmpxchg(&task->tk_rpc_status, 0, rpc_status) == 0) + return true; + return false; +} + unsigned long rpc_task_timeout(const struct rpc_task *task) { @@ -269,7 +276,7 @@ EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue); static int rpc_wait_bit_killable(struct wait_bit_key *key, int mode) { - freezable_schedule_unsafe(); + schedule(); if (signal_pending_state(mode, current)) return -ERESTARTSYS; return 0; @@ -333,14 +340,12 @@ static int rpc_complete_task(struct rpc_task *task) * to enforce taking of the wq->lock and hence avoid races with * rpc_complete_task(). */ -int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) +int rpc_wait_for_completion_task(struct rpc_task *task) { - if (action == NULL) - action = rpc_wait_bit_killable; return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, - action, TASK_KILLABLE); + rpc_wait_bit_killable, TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); } -EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); +EXPORT_SYMBOL_GPL(rpc_wait_for_completion_task); /* * Make an RPC task runnable. @@ -855,12 +860,25 @@ void rpc_signal_task(struct rpc_task *task) if (!RPC_IS_ACTIVATED(task)) return; + if (!rpc_task_set_rpc_status(task, -ERESTARTSYS)) + return; trace_rpc_task_signalled(task, task->tk_action); set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); smp_mb__after_atomic(); queue = READ_ONCE(task->tk_waitqueue); if (queue) - rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS); + rpc_wake_up_queued_task(queue, task); +} + +void rpc_task_try_cancel(struct rpc_task *task, int error) +{ + struct rpc_wait_queue *queue; + + if (!rpc_task_set_rpc_status(task, error)) + return; + queue = READ_ONCE(task->tk_waitqueue); + if (queue) + rpc_wake_up_queued_task(queue, task); } void rpc_exit(struct rpc_task *task, int status) @@ -907,10 +925,16 @@ static void __rpc_execute(struct rpc_task *task) * Perform the next FSM step or a pending callback. * * tk_action may be NULL if the task has been killed. - * In particular, note that rpc_killall_tasks may - * do this at any time, so beware when dereferencing. */ do_action = task->tk_action; + /* Tasks with an RPC error status should exit */ + if (do_action != rpc_exit_task && + (status = READ_ONCE(task->tk_rpc_status)) != 0) { + task->tk_status = status; + if (do_action != NULL) + do_action = rpc_exit_task; + } + /* Callbacks override all actions */ if (task->tk_callback) { do_action = task->tk_callback; task->tk_callback = NULL; @@ -932,14 +956,6 @@ static void __rpc_execute(struct rpc_task *task) continue; } - /* - * Signalled tasks should exit rather than sleep. - */ - if (RPC_SIGNALLED(task)) { - task->tk_rpc_status = -ERESTARTSYS; - rpc_exit(task, -ERESTARTSYS); - } - /* * The queue->lock protects against races with * rpc_make_runnable(). @@ -955,6 +971,12 @@ static void __rpc_execute(struct rpc_task *task) spin_unlock(&queue->lock); continue; } + /* Wake up any task that has an exit status */ + if (READ_ONCE(task->tk_rpc_status) != 0) { + rpc_wake_up_task_queue_locked(queue, task); + spin_unlock(&queue->lock); + continue; + } rpc_clear_running(task); spin_unlock(&queue->lock); if (task_is_async) @@ -964,7 +986,7 @@ static void __rpc_execute(struct rpc_task *task) trace_rpc_task_sync_sleep(task, task->tk_action); status = out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_QUEUED, rpc_wait_bit_killable, - TASK_KILLABLE); + TASK_KILLABLE|TASK_FREEZABLE); if (status < 0) { /* * When a sync task receives a signal, it exits with @@ -972,10 +994,7 @@ static void __rpc_execute(struct rpc_task *task) * clean up after sleeping on some queue, we don't * break the loop here, but go around once more. */ - trace_rpc_task_signalled(task, task->tk_action); - set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); - task->tk_rpc_status = -ERESTARTSYS; - rpc_exit(task, -ERESTARTSYS); + rpc_signal_task(task); } trace_rpc_task_sync_wake(task, task->tk_action); } diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 7c9a0d0b12300425a4a707f8469b6333446a26b8..149171774bc631f7fb7c2703aed64d7e4e779100 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -1205,7 +1205,7 @@ svc_generic_init_request(struct svc_rqst *rqstp, goto err_bad_proc; /* Initialize storage for argp and resp */ - memset(rqstp->rq_argp, 0, procp->pc_argsize); + memset(rqstp->rq_argp, 0, procp->pc_argzero); memset(rqstp->rq_resp, 0, procp->pc_ressize); /* Bump per-procedure stats counter */ @@ -1434,8 +1434,7 @@ svc_process(struct svc_rqst *rqstp) { struct kvec *argv = &rqstp->rq_arg.head[0]; struct kvec *resv = &rqstp->rq_res.head[0]; - struct svc_serv *serv = rqstp->rq_server; - u32 dir; + __be32 dir; #if IS_ENABLED(CONFIG_FAIL_SUNRPC) if (!fail_sunrpc.ignore_server_disconnect && @@ -1450,7 +1449,7 @@ svc_process(struct svc_rqst *rqstp) rqstp->rq_next_page = &rqstp->rq_respages[1]; resv->iov_base = page_address(rqstp->rq_respages[0]); resv->iov_len = 0; - rqstp->rq_res.pages = rqstp->rq_respages + 1; + rqstp->rq_res.pages = rqstp->rq_next_page; rqstp->rq_res.len = 0; rqstp->rq_res.page_base = 0; rqstp->rq_res.page_len = 0; @@ -1458,18 +1457,17 @@ svc_process(struct svc_rqst *rqstp) rqstp->rq_res.tail[0].iov_base = NULL; rqstp->rq_res.tail[0].iov_len = 0; - dir = svc_getnl(argv); - if (dir != 0) { - /* direction != CALL */ - svc_printk(rqstp, "bad direction %d, dropping request\n", dir); - serv->sv_stats->rpcbadfmt++; + dir = svc_getu32(argv); + if (dir != rpc_call) + goto out_baddir; + if (!svc_process_common(rqstp, argv, resv)) goto out_drop; - } - - /* Returns 1 for send, 0 for drop */ - if (likely(svc_process_common(rqstp, argv, resv))) - return svc_send(rqstp); + return svc_send(rqstp); +out_baddir: + svc_printk(rqstp, "bad direction 0x%08x, dropping request\n", + be32_to_cpu(dir)); + rqstp->rq_server->sv_stats->rpcbadfmt++; out_drop: svc_drop(rqstp); return 0; @@ -1556,8 +1554,12 @@ out: EXPORT_SYMBOL_GPL(bc_svc_process); #endif /* CONFIG_SUNRPC_BACKCHANNEL */ -/* - * Return (transport-specific) limit on the rpc payload. +/** + * svc_max_payload - Return transport-specific limit on the RPC payload + * @rqstp: RPC transaction context + * + * Returns the maximum number of payload bytes the current transport + * allows. */ u32 svc_max_payload(const struct svc_rqst *rqstp) { diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 482586c23fdd5d9587f833b233253042d06da30b..336a7c7833e4971193097e4b81416651521997bc 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -946,6 +946,28 @@ void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, } EXPORT_SYMBOL_GPL(xdr_init_encode); +/** + * xdr_init_encode_pages - Initialize an xdr_stream for encoding into pages + * @xdr: pointer to xdr_stream struct + * @buf: pointer to XDR buffer into which to encode data + * @pages: list of pages to decode into + * @rqst: pointer to controlling rpc_rqst, for debugging + * + */ +void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf, + struct page **pages, struct rpc_rqst *rqst) +{ + xdr_reset_scratch_buffer(xdr); + + xdr->buf = buf; + xdr->page_ptr = pages; + xdr->iov = NULL; + xdr->p = page_address(*pages); + xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE); + xdr->rqst = rqst; +} +EXPORT_SYMBOL_GPL(xdr_init_encode_pages); + /** * __xdr_commit_encode - Ensure all data is written to buffer * @xdr: pointer to xdr_stream @@ -1575,7 +1597,7 @@ EXPORT_SYMBOL_GPL(xdr_buf_from_iov); * * @buf and @subbuf may be pointers to the same struct xdr_buf. * - * Returns -1 if base of length are out of bounds. + * Returns -1 if base or length are out of bounds. */ int xdr_buf_subsegment(const struct xdr_buf *buf, struct xdr_buf *subbuf, unsigned int base, unsigned int len) diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index d71eec494826b269a76c887cfa82972fd805e59b..656cec2083718580f9060e9a370e88b95e5c4b4d 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -1179,11 +1179,8 @@ xprt_request_dequeue_receive_locked(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; - if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) { + if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) xprt_request_rb_remove(req->rq_xprt, req); - xdr_free_bvec(&req->rq_rcv_buf); - req->rq_private_buf.bvec = NULL; - } } /** @@ -1221,6 +1218,8 @@ void xprt_complete_rqst(struct rpc_task *task, int copied) xprt->stat.recvs++; + xdr_free_bvec(&req->rq_rcv_buf); + req->rq_private_buf.bvec = NULL; req->rq_private_buf.len = copied; /* Ensure all writes are done before we update */ /* req->rq_reply_bytes_recvd */ @@ -1453,6 +1452,7 @@ xprt_request_dequeue_xprt(struct rpc_task *task) xprt_request_dequeue_transmit_locked(task); xprt_request_dequeue_receive_locked(task); spin_unlock(&xprt->queue_lock); + xdr_free_bvec(&req->rq_rcv_buf); } } @@ -1788,7 +1788,7 @@ static int xprt_alloc_id(struct rpc_xprt *xprt) { int id; - id = ida_simple_get(&rpc_xprt_ids, 0, 0, GFP_KERNEL); + id = ida_alloc(&rpc_xprt_ids, GFP_KERNEL); if (id < 0) return id; @@ -1798,7 +1798,7 @@ static int xprt_alloc_id(struct rpc_xprt *xprt) static void xprt_free_id(struct rpc_xprt *xprt) { - ida_simple_remove(&rpc_xprt_ids, xprt->id); + ida_free(&rpc_xprt_ids, xprt->id); } struct rpc_xprt *xprt_alloc(struct net *net, size_t size, @@ -1822,10 +1822,7 @@ struct rpc_xprt *xprt_alloc(struct net *net, size_t size, goto out_free; list_add(&req->rq_list, &xprt->free); } - if (max_alloc > num_prealloc) - xprt->max_reqs = max_alloc; - else - xprt->max_reqs = num_prealloc; + xprt->max_reqs = max_t(unsigned int, max_alloc, num_prealloc); xprt->min_reqs = num_prealloc; xprt->num_reqs = num_prealloc; @@ -1868,7 +1865,7 @@ xprt_alloc_xid(struct rpc_xprt *xprt) static void xprt_init_xid(struct rpc_xprt *xprt) { - xprt->xid = prandom_u32(); + xprt->xid = get_random_u32(); } static void diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c index 685db598acbe1a5db1ea73e3f1482c10ebb042b9..701250b305dba9c1cf809738361a929dd35e523e 100644 --- a/net/sunrpc/xprtmultipath.c +++ b/net/sunrpc/xprtmultipath.c @@ -103,7 +103,7 @@ static int xprt_switch_alloc_id(struct rpc_xprt_switch *xps, gfp_t gfp_flags) { int id; - id = ida_simple_get(&rpc_xprtswitch_ids, 0, 0, gfp_flags); + id = ida_alloc(&rpc_xprtswitch_ids, gfp_flags); if (id < 0) return id; @@ -113,7 +113,7 @@ static int xprt_switch_alloc_id(struct rpc_xprt_switch *xps, gfp_t gfp_flags) static void xprt_switch_free_id(struct rpc_xprt_switch *xps) { - ida_simple_remove(&rpc_xprtswitch_ids, xps->xps_id); + ida_free(&rpc_xprtswitch_ids, xps->xps_id); } /** diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c index faba7136dd9a3b6443724972ed7582580043130e..e4d84a13c566e7d90fe15b87fe3148bb7b4a43ed 100644 --- a/net/sunrpc/xprtrdma/backchannel.c +++ b/net/sunrpc/xprtrdma/backchannel.c @@ -189,7 +189,7 @@ create_req: return NULL; size = min_t(size_t, r_xprt->rx_ep->re_inline_recv, PAGE_SIZE); - req = rpcrdma_req_create(r_xprt, size, GFP_KERNEL); + req = rpcrdma_req_create(r_xprt, size); if (!req) return NULL; if (rpcrdma_req_setup(r_xprt, req)) { diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index de0bdb6b729f8935023468f65472c7301ec4d9a1..ffbf99894970e0cd77bf3d0d606eb9eefbaf4da2 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -124,16 +124,16 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) unsigned int depth = ep->re_max_fr_depth; struct scatterlist *sg; struct ib_mr *frmr; - int rc; + + sg = kcalloc_node(depth, sizeof(*sg), XPRTRDMA_GFP_FLAGS, + ibdev_to_node(ep->re_id->device)); + if (!sg) + return -ENOMEM; frmr = ib_alloc_mr(ep->re_pd, ep->re_mrtype, depth); if (IS_ERR(frmr)) goto out_mr_err; - sg = kmalloc_array(depth, sizeof(*sg), GFP_KERNEL); - if (!sg) - goto out_list_err; - mr->mr_xprt = r_xprt; mr->mr_ibmr = frmr; mr->mr_device = NULL; @@ -146,13 +146,9 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) return 0; out_mr_err: - rc = PTR_ERR(frmr); - trace_xprtrdma_frwr_alloc(mr, rc); - return rc; - -out_list_err: - ib_dereg_mr(frmr); - return -ENOMEM; + kfree(sg); + trace_xprtrdma_frwr_alloc(mr, PTR_ERR(frmr)); + return PTR_ERR(frmr); } /** diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c index 85c8cdda98b189421a7fdf13cb720ad42f756f67..aa2227a7e552100d3a3ec5f879650cd3e49d2ad2 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c @@ -119,12 +119,12 @@ xprt_rdma_bc_allocate(struct rpc_task *task) return -EINVAL; } - page = alloc_page(RPCRDMA_DEF_GFP); + page = alloc_page(GFP_NOIO | __GFP_NOWARN); if (!page) return -ENOMEM; rqst->rq_buffer = page_address(page); - rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, RPCRDMA_DEF_GFP); + rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, GFP_NOIO | __GFP_NOWARN); if (!rqst->rq_rbuffer) { put_page(page); return -ENOMEM; diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index bcb37b51adf65878eda0910833fa85efe7f12484..10bb2b929c6d70749b10d12d54824275add1aaa4 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -494,8 +494,7 @@ xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) xprt_reconnect_backoff(xprt, RPCRDMA_INIT_REEST_TO); } trace_xprtrdma_op_connect(r_xprt, delay); - queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker, - delay); + queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay); } /** diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 2fbe9aaeec349a6c27bceb0b7c070f5b0e4363b7..44b87e4274b428483fd6382c04e869e0e4af0e05 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -76,8 +76,7 @@ static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt); static void rpcrdma_ep_get(struct rpcrdma_ep *ep); static int rpcrdma_ep_put(struct rpcrdma_ep *ep); static struct rpcrdma_regbuf * -rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction, - gfp_t flags); +rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction); static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb); static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb); @@ -373,7 +372,7 @@ static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt) struct rpcrdma_ep *ep; int rc; - ep = kzalloc(sizeof(*ep), GFP_KERNEL); + ep = kzalloc(sizeof(*ep), XPRTRDMA_GFP_FLAGS); if (!ep) return -ENOTCONN; ep->re_xprt = &r_xprt->rx_xprt; @@ -606,7 +605,7 @@ static struct rpcrdma_sendctx *rpcrdma_sendctx_create(struct rpcrdma_ep *ep) struct rpcrdma_sendctx *sc; sc = kzalloc(struct_size(sc, sc_sges, ep->re_attr.cap.max_send_sge), - GFP_KERNEL); + XPRTRDMA_GFP_FLAGS); if (!sc) return NULL; @@ -629,7 +628,7 @@ static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt) * Sends are posted. */ i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS; - buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), GFP_KERNEL); + buf->rb_sc_ctxs = kcalloc(i, sizeof(sc), XPRTRDMA_GFP_FLAGS); if (!buf->rb_sc_ctxs) return -ENOMEM; @@ -740,13 +739,16 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_ep *ep = r_xprt->rx_ep; + struct ib_device *device = ep->re_id->device; unsigned int count; + /* Try to allocate enough to perform one full-sized I/O */ for (count = 0; count < ep->re_max_rdma_segs; count++) { struct rpcrdma_mr *mr; int rc; - mr = kzalloc(sizeof(*mr), GFP_KERNEL); + mr = kzalloc_node(sizeof(*mr), XPRTRDMA_GFP_FLAGS, + ibdev_to_node(device)); if (!mr) break; @@ -791,38 +793,33 @@ void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt) /* If there is no underlying connection, it's no use * to wake the refresh worker. */ - if (ep->re_connect_status == 1) { - /* The work is scheduled on a WQ_MEM_RECLAIM - * workqueue in order to prevent MR allocation - * from recursing into NFS during direct reclaim. - */ - queue_work(xprtiod_workqueue, &buf->rb_refresh_worker); - } + if (ep->re_connect_status != 1) + return; + queue_work(system_highpri_wq, &buf->rb_refresh_worker); } /** * rpcrdma_req_create - Allocate an rpcrdma_req object * @r_xprt: controlling r_xprt * @size: initial size, in bytes, of send and receive buffers - * @flags: GFP flags passed to memory allocators * * Returns an allocated and fully initialized rpcrdma_req or NULL. */ -struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size, - gfp_t flags) +struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, + size_t size) { struct rpcrdma_buffer *buffer = &r_xprt->rx_buf; struct rpcrdma_req *req; - req = kzalloc(sizeof(*req), flags); + req = kzalloc(sizeof(*req), XPRTRDMA_GFP_FLAGS); if (req == NULL) goto out1; - req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE, flags); + req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE); if (!req->rl_sendbuf) goto out2; - req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE, flags); + req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE); if (!req->rl_recvbuf) goto out3; @@ -858,7 +855,7 @@ int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz; maxhdrsize *= sizeof(__be32); rb = rpcrdma_regbuf_alloc(__roundup_pow_of_two(maxhdrsize), - DMA_TO_DEVICE, GFP_KERNEL); + DMA_TO_DEVICE); if (!rb) goto out; @@ -929,12 +926,12 @@ struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, struct rpcrdma_buffer *buf = &r_xprt->rx_buf; struct rpcrdma_rep *rep; - rep = kzalloc(sizeof(*rep), GFP_KERNEL); + rep = kzalloc(sizeof(*rep), XPRTRDMA_GFP_FLAGS); if (rep == NULL) goto out; rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv, - DMA_FROM_DEVICE, GFP_KERNEL); + DMA_FROM_DEVICE); if (!rep->rr_rdmabuf) goto out_free; @@ -1064,8 +1061,8 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) { struct rpcrdma_req *req; - req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE * 2, - GFP_KERNEL); + req = rpcrdma_req_create(r_xprt, + RPCRDMA_V1_DEF_INLINE_SIZE * 2); if (!req) goto out; list_add(&req->rl_list, &buf->rb_send_bufs); @@ -1235,15 +1232,14 @@ void rpcrdma_buffer_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) * or Replies they may be registered externally via frwr_map. */ static struct rpcrdma_regbuf * -rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction, - gfp_t flags) +rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction) { struct rpcrdma_regbuf *rb; - rb = kmalloc(sizeof(*rb), flags); + rb = kmalloc(sizeof(*rb), XPRTRDMA_GFP_FLAGS); if (!rb) return NULL; - rb->rg_data = kmalloc(size, flags); + rb->rg_data = kmalloc(size, XPRTRDMA_GFP_FLAGS); if (!rb->rg_data) { kfree(rb); return NULL; diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index c79f92eeda762714c8ab66db59d4f2ac1b0f3aaf..5e5ff6784ef5ffe58d82bd1dd66ed8b6d7813c86 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -149,7 +149,11 @@ static inline void *rdmab_data(const struct rpcrdma_regbuf *rb) return rb->rg_data; } -#define RPCRDMA_DEF_GFP (GFP_NOIO | __GFP_NOWARN) +/* Do not use emergency memory reserves, and fail quickly if memory + * cannot be allocated easily. These flags may be used wherever there + * is robust logic to handle a failure to allocate. + */ +#define XPRTRDMA_GFP_FLAGS (__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN) /* To ensure a transport can always make forward progress, * the number of RDMA segments allowed in header chunk lists @@ -467,8 +471,8 @@ void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp); /* * Buffer calls - xprtrdma/verbs.c */ -struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size, - gfp_t flags); +struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, + size_t size); int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req); void rpcrdma_req_destroy(struct rpcrdma_req *req); int rpcrdma_buffer_create(struct rpcrdma_xprt *); diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index e976007f4fd00f2c6238c9756370dd3231f2c98a..915b9902f673b2ace31daaf86ce532bc0b6523f1 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -261,7 +261,7 @@ static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) switch (sap->sa_family) { case AF_LOCAL: sun = xs_addr_un(xprt); - strlcpy(buf, sun->sun_path, sizeof(buf)); + strscpy(buf, sun->sun_path, sizeof(buf)); xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); break; @@ -1619,7 +1619,7 @@ static int xs_get_random_port(void) if (max < min) return -EADDRINUSE; range = max - min + 1; - rand = (unsigned short) prandom_u32() % range; + rand = prandom_u32_max(range); return rand + min; } @@ -1978,8 +1978,7 @@ static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) * we'll need to figure out how to pass a namespace to * connect. */ - task->tk_rpc_status = -ENOTCONN; - rpc_exit(task, -ENOTCONN); + rpc_task_set_rpc_status(task, -ENOTCONN); goto out_wake; } ret = xs_local_setup_socket(transport); diff --git a/net/tipc/monitor.c b/net/tipc/monitor.c index 2f4d23238a7e33a6ff22e87fca6ee0281040b338..9618e4429f0fe848d2dce8b2e5b4ba0e46a52b1c 100644 --- a/net/tipc/monitor.c +++ b/net/tipc/monitor.c @@ -160,7 +160,7 @@ static void map_set(u64 *up_map, int i, unsigned int v) static int map_get(u64 up_map, int i) { - return (up_map & (1 << i)) >> i; + return (up_map & (1ULL << i)) >> i; } static struct tipc_peer *peer_prev(struct tipc_peer *peer) diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c index 8267b751a526a4dc30062e5e9b38da92bc97266e..190b49c5cbc3ec5029ffbf45455a971ab56b5d6c 100644 --- a/net/tipc/name_distr.c +++ b/net/tipc/name_distr.c @@ -41,14 +41,6 @@ int sysctl_tipc_named_timeout __read_mostly = 2000; -struct distr_queue_item { - struct distr_item i; - u32 dtype; - u32 node; - unsigned long expires; - struct list_head next; -}; - /** * publ_to_item - add publication info to a publication message * @p: publication info diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c index c447cb5f879e70bbce4feca8822292f2600e648b..e8fd257c0e6888b18164f4a63a68ad358ed0459e 100644 --- a/net/tipc/netlink.c +++ b/net/tipc/netlink.c @@ -294,6 +294,7 @@ struct genl_family tipc_genl_family __ro_after_init = { .module = THIS_MODULE, .ops = tipc_genl_v2_ops, .n_ops = ARRAY_SIZE(tipc_genl_v2_ops), + .resv_start_op = TIPC_NL_ADDR_LEGACY_GET + 1, }; int __init tipc_netlink_start(void) diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c index 0749df80454d4fb8075abc0020584b7f5d1a4662..fc68733673ba6ddca574f67253ede5e340235104 100644 --- a/net/tipc/netlink_compat.c +++ b/net/tipc/netlink_compat.c @@ -1357,6 +1357,7 @@ static struct genl_family tipc_genl_compat_family __ro_after_init = { .module = THIS_MODULE, .small_ops = tipc_genl_compat_ops, .n_small_ops = ARRAY_SIZE(tipc_genl_compat_ops), + .resv_start_op = TIPC_GENL_CMD + 1, }; int __init tipc_netlink_compat_start(void) diff --git a/net/tipc/socket.c b/net/tipc/socket.c index f1c3b8eb4b3d3356070f77f7065591bd23681480..e902b01ea3cb189ac560e531bd25c7068fe9e322 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -3010,7 +3010,7 @@ static int tipc_sk_insert(struct tipc_sock *tsk) struct net *net = sock_net(sk); struct tipc_net *tn = net_generic(net, tipc_net_id); u32 remaining = (TIPC_MAX_PORT - TIPC_MIN_PORT) + 1; - u32 portid = prandom_u32() % remaining + TIPC_MIN_PORT; + u32 portid = prandom_u32_max(remaining) + TIPC_MIN_PORT; while (remaining--) { portid++; diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 0f983e5f7dde80a6b310101b6c399a019fcf347f..a03d66046ca328bc885f527d3068c1c88d3ce7c1 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -902,17 +902,28 @@ static void tls_device_core_ctrl_rx_resync(struct tls_context *tls_ctx, } static int -tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx) +tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx) { + struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx); + const struct tls_cipher_size_desc *cipher_sz; int err, offset, copy, data_len, pos; struct sk_buff *skb, *skb_iter; struct scatterlist sg[1]; struct strp_msg *rxm; char *orig_buf, *buf; + switch (tls_ctx->crypto_recv.info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: + case TLS_CIPHER_AES_GCM_256: + break; + default: + return -EINVAL; + } + cipher_sz = &tls_cipher_size_desc[tls_ctx->crypto_recv.info.cipher_type]; + rxm = strp_msg(tls_strp_msg(sw_ctx)); - orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + - TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation); + orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_sz->iv, + sk->sk_allocation); if (!orig_buf) return -ENOMEM; buf = orig_buf; @@ -927,10 +938,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx) sg_init_table(sg, 1); sg_set_buf(&sg[0], buf, - rxm->full_len + TLS_HEADER_SIZE + - TLS_CIPHER_AES_GCM_128_IV_SIZE); - err = skb_copy_bits(skb, offset, buf, - TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE); + rxm->full_len + TLS_HEADER_SIZE + cipher_sz->iv); + err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_sz->iv); if (err) goto free_buf; @@ -941,7 +950,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx) else err = 0; - data_len = rxm->full_len - TLS_CIPHER_AES_GCM_128_TAG_SIZE; + data_len = rxm->full_len - cipher_sz->tag; if (skb_pagelen(skb) > offset) { copy = min_t(int, skb_pagelen(skb) - offset, data_len); @@ -1024,7 +1033,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) * likely have initial fragments decrypted, and final ones not * decrypted. We need to reencrypt that single SKB. */ - return tls_device_reencrypt(sk, sw_ctx); + return tls_device_reencrypt(sk, tls_ctx); } /* Return immediately if the record is either entirely plaintext or @@ -1041,7 +1050,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx) } ctx->resync_nh_reset = 1; - return tls_device_reencrypt(sk, sw_ctx); + return tls_device_reencrypt(sk, tls_ctx); } static void tls_device_attach(struct tls_context *ctx, struct sock *sk, @@ -1062,9 +1071,9 @@ static void tls_device_attach(struct tls_context *ctx, struct sock *sk, int tls_set_device_offload(struct sock *sk, struct tls_context *ctx) { - u16 nonce_size, tag_size, iv_size, rec_seq_size, salt_size; struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_prot_info *prot = &tls_ctx->prot_info; + const struct tls_cipher_size_desc *cipher_sz; struct tls_record_info *start_marker_record; struct tls_offload_context_tx *offload_ctx; struct tls_crypto_info *crypto_info; @@ -1099,44 +1108,44 @@ int tls_set_device_offload(struct sock *sk, struct tls_context *ctx) switch (crypto_info->cipher_type) { case TLS_CIPHER_AES_GCM_128: - nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; - tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; - iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; - rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; - salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; rec_seq = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; break; + case TLS_CIPHER_AES_GCM_256: + iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv; + rec_seq = + ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq; + break; default: rc = -EINVAL; goto release_netdev; } + cipher_sz = &tls_cipher_size_desc[crypto_info->cipher_type]; /* Sanity-check the rec_seq_size for stack allocations */ - if (rec_seq_size > TLS_MAX_REC_SEQ_SIZE) { + if (cipher_sz->rec_seq > TLS_MAX_REC_SEQ_SIZE) { rc = -EINVAL; goto release_netdev; } prot->version = crypto_info->version; prot->cipher_type = crypto_info->cipher_type; - prot->prepend_size = TLS_HEADER_SIZE + nonce_size; - prot->tag_size = tag_size; + prot->prepend_size = TLS_HEADER_SIZE + cipher_sz->iv; + prot->tag_size = cipher_sz->tag; prot->overhead_size = prot->prepend_size + prot->tag_size; - prot->iv_size = iv_size; - prot->salt_size = salt_size; - ctx->tx.iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE, - GFP_KERNEL); + prot->iv_size = cipher_sz->iv; + prot->salt_size = cipher_sz->salt; + ctx->tx.iv = kmalloc(cipher_sz->iv + cipher_sz->salt, GFP_KERNEL); if (!ctx->tx.iv) { rc = -ENOMEM; goto release_netdev; } - memcpy(ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size); + memcpy(ctx->tx.iv + cipher_sz->salt, iv, cipher_sz->iv); - prot->rec_seq_size = rec_seq_size; - ctx->tx.rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL); + prot->rec_seq_size = cipher_sz->rec_seq; + ctx->tx.rec_seq = kmemdup(rec_seq, cipher_sz->rec_seq, GFP_KERNEL); if (!ctx->tx.rec_seq) { rc = -ENOMEM; goto free_iv; diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c index 7dfc8023e0f1a9c7bfdcfb910ef1284f55a5cb2d..cdb391a8754b5b65d1ef1b7e6716bc7180f793db 100644 --- a/net/tls/tls_device_fallback.c +++ b/net/tls/tls_device_fallback.c @@ -54,13 +54,25 @@ static int tls_enc_record(struct aead_request *aead_req, struct scatter_walk *out, int *in_len, struct tls_prot_info *prot) { - unsigned char buf[TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE]; + unsigned char buf[TLS_HEADER_SIZE + MAX_IV_SIZE]; + const struct tls_cipher_size_desc *cipher_sz; struct scatterlist sg_in[3]; struct scatterlist sg_out[3]; + unsigned int buf_size; u16 len; int rc; - len = min_t(int, *in_len, ARRAY_SIZE(buf)); + switch (prot->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + case TLS_CIPHER_AES_GCM_256: + break; + default: + return -EINVAL; + } + cipher_sz = &tls_cipher_size_desc[prot->cipher_type]; + + buf_size = TLS_HEADER_SIZE + cipher_sz->iv; + len = min_t(int, *in_len, buf_size); scatterwalk_copychunks(buf, in, len, 0); scatterwalk_copychunks(buf, out, len, 1); @@ -73,13 +85,11 @@ static int tls_enc_record(struct aead_request *aead_req, scatterwalk_pagedone(out, 1, 1); len = buf[4] | (buf[3] << 8); - len -= TLS_CIPHER_AES_GCM_128_IV_SIZE; + len -= cipher_sz->iv; - tls_make_aad(aad, len - TLS_CIPHER_AES_GCM_128_TAG_SIZE, - (char *)&rcd_sn, buf[0], prot); + tls_make_aad(aad, len - cipher_sz->tag, (char *)&rcd_sn, buf[0], prot); - memcpy(iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, buf + TLS_HEADER_SIZE, - TLS_CIPHER_AES_GCM_128_IV_SIZE); + memcpy(iv + cipher_sz->salt, buf + TLS_HEADER_SIZE, cipher_sz->iv); sg_init_table(sg_in, ARRAY_SIZE(sg_in)); sg_init_table(sg_out, ARRAY_SIZE(sg_out)); @@ -90,7 +100,7 @@ static int tls_enc_record(struct aead_request *aead_req, *in_len -= len; if (*in_len < 0) { - *in_len += TLS_CIPHER_AES_GCM_128_TAG_SIZE; + *in_len += cipher_sz->tag; /* the input buffer doesn't contain the entire record. * trim len accordingly. The resulting authentication tag * will contain garbage, but we don't care, so we won't @@ -111,7 +121,7 @@ static int tls_enc_record(struct aead_request *aead_req, scatterwalk_pagedone(out, 1, 1); } - len -= TLS_CIPHER_AES_GCM_128_TAG_SIZE; + len -= cipher_sz->tag; aead_request_set_crypt(aead_req, sg_in, sg_out, len, iv); rc = crypto_aead_encrypt(aead_req); @@ -299,11 +309,14 @@ static void fill_sg_out(struct scatterlist sg_out[3], void *buf, int sync_size, void *dummy_buf) { + const struct tls_cipher_size_desc *cipher_sz = + &tls_cipher_size_desc[tls_ctx->crypto_send.info.cipher_type]; + sg_set_buf(&sg_out[0], dummy_buf, sync_size); sg_set_buf(&sg_out[1], nskb->data + tcp_payload_offset, payload_len); /* Add room for authentication tag produced by crypto */ dummy_buf += sync_size; - sg_set_buf(&sg_out[2], dummy_buf, TLS_CIPHER_AES_GCM_128_TAG_SIZE); + sg_set_buf(&sg_out[2], dummy_buf, cipher_sz->tag); } static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx, @@ -315,7 +328,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx, struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx); int tcp_payload_offset = skb_tcp_all_headers(skb); int payload_len = skb->len - tcp_payload_offset; - void *buf, *iv, *aad, *dummy_buf; + const struct tls_cipher_size_desc *cipher_sz; + void *buf, *iv, *aad, *dummy_buf, *salt; struct aead_request *aead_req; struct sk_buff *nskb = NULL; int buf_len; @@ -324,20 +338,26 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx, if (!aead_req) return NULL; - buf_len = TLS_CIPHER_AES_GCM_128_SALT_SIZE + - TLS_CIPHER_AES_GCM_128_IV_SIZE + - TLS_AAD_SPACE_SIZE + - sync_size + - TLS_CIPHER_AES_GCM_128_TAG_SIZE; + switch (tls_ctx->crypto_send.info.cipher_type) { + case TLS_CIPHER_AES_GCM_128: + salt = tls_ctx->crypto_send.aes_gcm_128.salt; + break; + case TLS_CIPHER_AES_GCM_256: + salt = tls_ctx->crypto_send.aes_gcm_256.salt; + break; + default: + return NULL; + } + cipher_sz = &tls_cipher_size_desc[tls_ctx->crypto_send.info.cipher_type]; + buf_len = cipher_sz->salt + cipher_sz->iv + TLS_AAD_SPACE_SIZE + + sync_size + cipher_sz->tag; buf = kmalloc(buf_len, GFP_ATOMIC); if (!buf) goto free_req; iv = buf; - memcpy(iv, tls_ctx->crypto_send.aes_gcm_128.salt, - TLS_CIPHER_AES_GCM_128_SALT_SIZE); - aad = buf + TLS_CIPHER_AES_GCM_128_SALT_SIZE + - TLS_CIPHER_AES_GCM_128_IV_SIZE; + memcpy(iv, salt, cipher_sz->salt); + aad = buf + cipher_sz->salt + cipher_sz->iv; dummy_buf = aad + TLS_AAD_SPACE_SIZE; nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC); @@ -451,6 +471,7 @@ int tls_sw_fallback_init(struct sock *sk, struct tls_offload_context_tx *offload_ctx, struct tls_crypto_info *crypto_info) { + const struct tls_cipher_size_desc *cipher_sz; const u8 *key; int rc; @@ -463,15 +484,23 @@ int tls_sw_fallback_init(struct sock *sk, goto err_out; } - key = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->key; + switch (crypto_info->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + key = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->key; + break; + case TLS_CIPHER_AES_GCM_256: + key = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->key; + break; + default: + return -EINVAL; + } + cipher_sz = &tls_cipher_size_desc[crypto_info->cipher_type]; - rc = crypto_aead_setkey(offload_ctx->aead_send, key, - TLS_CIPHER_AES_GCM_128_KEY_SIZE); + rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_sz->key); if (rc) goto free_aead; - rc = crypto_aead_setauthsize(offload_ctx->aead_send, - TLS_CIPHER_AES_GCM_128_TAG_SIZE); + rc = crypto_aead_setauthsize(offload_ctx->aead_send, cipher_sz->tag); if (rc) goto free_aead; diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index 08ddf9d837ae31864bdcc8dfba44977a19179fe2..3735cb00905dfc7b0a12343308c4fae72f1bb839 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -58,6 +58,23 @@ enum { TLS_NUM_PROTS, }; +#define CIPHER_SIZE_DESC(cipher) [cipher] = { \ + .iv = cipher ## _IV_SIZE, \ + .key = cipher ## _KEY_SIZE, \ + .salt = cipher ## _SALT_SIZE, \ + .tag = cipher ## _TAG_SIZE, \ + .rec_seq = cipher ## _REC_SEQ_SIZE, \ +} + +const struct tls_cipher_size_desc tls_cipher_size_desc[] = { + CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_128), + CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_256), + CIPHER_SIZE_DESC(TLS_CIPHER_AES_CCM_128), + CIPHER_SIZE_DESC(TLS_CIPHER_CHACHA20_POLY1305), + CIPHER_SIZE_DESC(TLS_CIPHER_SM4_GCM), + CIPHER_SIZE_DESC(TLS_CIPHER_SM4_CCM), +}; + static const struct proto *saved_tcpv6_prot; static DEFINE_MUTEX(tcpv6_prot_mutex); static const struct proto *saved_tcpv4_prot; @@ -507,6 +524,54 @@ static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval, rc = -EFAULT; break; } + case TLS_CIPHER_ARIA_GCM_128: { + struct tls12_crypto_info_aria_gcm_128 * + crypto_info_aria_gcm_128 = + container_of(crypto_info, + struct tls12_crypto_info_aria_gcm_128, + info); + + if (len != sizeof(*crypto_info_aria_gcm_128)) { + rc = -EINVAL; + goto out; + } + lock_sock(sk); + memcpy(crypto_info_aria_gcm_128->iv, + cctx->iv + TLS_CIPHER_ARIA_GCM_128_SALT_SIZE, + TLS_CIPHER_ARIA_GCM_128_IV_SIZE); + memcpy(crypto_info_aria_gcm_128->rec_seq, cctx->rec_seq, + TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE); + release_sock(sk); + if (copy_to_user(optval, + crypto_info_aria_gcm_128, + sizeof(*crypto_info_aria_gcm_128))) + rc = -EFAULT; + break; + } + case TLS_CIPHER_ARIA_GCM_256: { + struct tls12_crypto_info_aria_gcm_256 * + crypto_info_aria_gcm_256 = + container_of(crypto_info, + struct tls12_crypto_info_aria_gcm_256, + info); + + if (len != sizeof(*crypto_info_aria_gcm_256)) { + rc = -EINVAL; + goto out; + } + lock_sock(sk); + memcpy(crypto_info_aria_gcm_256->iv, + cctx->iv + TLS_CIPHER_ARIA_GCM_256_SALT_SIZE, + TLS_CIPHER_ARIA_GCM_256_IV_SIZE); + memcpy(crypto_info_aria_gcm_256->rec_seq, cctx->rec_seq, + TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE); + release_sock(sk); + if (copy_to_user(optval, + crypto_info_aria_gcm_256, + sizeof(*crypto_info_aria_gcm_256))) + rc = -EFAULT; + break; + } default: rc = -EINVAL; } @@ -668,6 +733,20 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, case TLS_CIPHER_SM4_CCM: optsize = sizeof(struct tls12_crypto_info_sm4_ccm); break; + case TLS_CIPHER_ARIA_GCM_128: + if (crypto_info->version != TLS_1_2_VERSION) { + rc = -EINVAL; + goto err_crypto_info; + } + optsize = sizeof(struct tls12_crypto_info_aria_gcm_128); + break; + case TLS_CIPHER_ARIA_GCM_256: + if (crypto_info->version != TLS_1_2_VERSION) { + rc = -EINVAL; + goto err_crypto_info; + } + optsize = sizeof(struct tls12_crypto_info_aria_gcm_256); + break; default: rc = -EINVAL; goto err_crypto_info; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index fe27241cd13fcfc7bfab366dbf9dd6c1c0366bdf..264cf367e26566281851de2ee8e2eb7e5808268c 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -2629,6 +2629,40 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) cipher_name = "ccm(sm4)"; break; } + case TLS_CIPHER_ARIA_GCM_128: { + struct tls12_crypto_info_aria_gcm_128 *aria_gcm_128_info; + + aria_gcm_128_info = (void *)crypto_info; + nonce_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE; + tag_size = TLS_CIPHER_ARIA_GCM_128_TAG_SIZE; + iv_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE; + iv = aria_gcm_128_info->iv; + rec_seq_size = TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE; + rec_seq = aria_gcm_128_info->rec_seq; + keysize = TLS_CIPHER_ARIA_GCM_128_KEY_SIZE; + key = aria_gcm_128_info->key; + salt = aria_gcm_128_info->salt; + salt_size = TLS_CIPHER_ARIA_GCM_128_SALT_SIZE; + cipher_name = "gcm(aria)"; + break; + } + case TLS_CIPHER_ARIA_GCM_256: { + struct tls12_crypto_info_aria_gcm_256 *gcm_256_info; + + gcm_256_info = (void *)crypto_info; + nonce_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE; + tag_size = TLS_CIPHER_ARIA_GCM_256_TAG_SIZE; + iv_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE; + iv = gcm_256_info->iv; + rec_seq_size = TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE; + rec_seq = gcm_256_info->rec_seq; + keysize = TLS_CIPHER_ARIA_GCM_256_KEY_SIZE; + key = gcm_256_info->key; + salt = gcm_256_info->salt; + salt_size = TLS_CIPHER_ARIA_GCM_256_SALT_SIZE; + cipher_name = "gcm(aria)"; + break; + } default: rc = -EINVAL; goto free_priv; diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index bf338b782fc4c49c0f59fffe7fbf02f4ec9a2229..b3545fc680979573c672ca232381c66aa14de6fa 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -569,12 +569,6 @@ static void unix_sock_destructor(struct sock *sk) skb_queue_purge(&sk->sk_receive_queue); -#if IS_ENABLED(CONFIG_AF_UNIX_OOB) - if (u->oob_skb) { - kfree_skb(u->oob_skb); - u->oob_skb = NULL; - } -#endif DEBUG_NET_WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc)); DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk)); DEBUG_NET_WARN_ON_ONCE(sk->sk_socket); @@ -620,6 +614,13 @@ static void unix_release_sock(struct sock *sk, int embrion) unix_state_unlock(sk); +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (u->oob_skb) { + kfree_skb(u->oob_skb); + u->oob_skb = NULL; + } +#endif + wake_up_interruptible_all(&u->peer_wait); if (skpair != NULL) { @@ -785,15 +786,45 @@ static int unix_set_peek_off(struct sock *sk, int val) } #ifdef CONFIG_PROC_FS +static int unix_count_nr_fds(struct sock *sk) +{ + struct sk_buff *skb; + struct unix_sock *u; + int nr_fds = 0; + + spin_lock(&sk->sk_receive_queue.lock); + skb = skb_peek(&sk->sk_receive_queue); + while (skb) { + u = unix_sk(skb->sk); + nr_fds += atomic_read(&u->scm_stat.nr_fds); + skb = skb_peek_next(skb, &sk->sk_receive_queue); + } + spin_unlock(&sk->sk_receive_queue.lock); + + return nr_fds; +} + static void unix_show_fdinfo(struct seq_file *m, struct socket *sock) { struct sock *sk = sock->sk; struct unix_sock *u; + int nr_fds; if (sk) { - u = unix_sk(sock->sk); - seq_printf(m, "scm_fds: %u\n", - atomic_read(&u->scm_stat.nr_fds)); + u = unix_sk(sk); + if (sock->type == SOCK_DGRAM) { + nr_fds = atomic_read(&u->scm_stat.nr_fds); + goto out_print; + } + + unix_state_lock(sk); + if (sk->sk_state != TCP_LISTEN) + nr_fds = atomic_read(&u->scm_stat.nr_fds); + else + nr_fds = unix_count_nr_fds(sk); + unix_state_unlock(sk); +out_print: + seq_printf(m, "scm_fds: %u\n", nr_fds); } } #else @@ -1116,7 +1147,7 @@ static int unix_autobind(struct sock *sk) addr->name->sun_family = AF_UNIX; refcount_set(&addr->refcnt, 1); - ordernum = prandom_u32(); + ordernum = get_random_u32(); lastnum = ordernum & 0xFFFFF; retry: ordernum = (ordernum + 1) & 0xFFFFF; @@ -2506,32 +2537,18 @@ static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t si static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor) { - int copied = 0; - - while (1) { - struct unix_sock *u = unix_sk(sk); - struct sk_buff *skb; - int used, err; - - mutex_lock(&u->iolock); - skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err); - mutex_unlock(&u->iolock); - if (!skb) - return err; + struct unix_sock *u = unix_sk(sk); + struct sk_buff *skb; + int err, copied; - used = recv_actor(sk, skb); - if (used <= 0) { - if (!copied) - copied = used; - kfree_skb(skb); - break; - } else if (used <= skb->len) { - copied += used; - } + mutex_lock(&u->iolock); + skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err); + mutex_unlock(&u->iolock); + if (!skb) + return err; - kfree_skb(skb); - break; - } + copied = recv_actor(sk, skb); + kfree_skb(skb); return copied; } @@ -2543,13 +2560,14 @@ static long unix_stream_data_wait(struct sock *sk, long timeo, struct sk_buff *last, unsigned int last_len, bool freezable) { + unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE; struct sk_buff *tail; DEFINE_WAIT(wait); unix_state_lock(sk); for (;;) { - prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); + prepare_to_wait(sk_sleep(sk), &wait, state); tail = skb_peek_tail(&sk->sk_receive_queue); if (tail != last || @@ -2562,10 +2580,7 @@ static long unix_stream_data_wait(struct sock *sk, long timeo, sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); unix_state_unlock(sk); - if (freezable) - timeo = freezable_schedule_timeout(timeo); - else - timeo = schedule_timeout(timeo); + timeo = schedule_timeout(timeo); unix_state_lock(sk); if (sock_flag(sk, SOCK_DEAD)) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index d45d5366115a769b21bfc1db5a67f7d53c3fa9b8..dc27635403932154f3dec069c2e10d2ae365d8cb 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -204,6 +204,7 @@ void wait_for_unix_gc(void) /* The external entry point: unix_gc() */ void unix_gc(void) { + struct sk_buff *next_skb, *skb; struct unix_sock *u; struct unix_sock *next; struct sk_buff_head hitlist; @@ -297,11 +298,30 @@ void unix_gc(void) spin_unlock(&unix_gc_lock); + /* We need io_uring to clean its registered files, ignore all io_uring + * originated skbs. It's fine as io_uring doesn't keep references to + * other io_uring instances and so killing all other files in the cycle + * will put all io_uring references forcing it to go through normal + * release.path eventually putting registered files. + */ + skb_queue_walk_safe(&hitlist, skb, next_skb) { + if (skb->scm_io_uring) { + __skb_unlink(skb, &hitlist); + skb_queue_tail(&skb->sk->sk_receive_queue, skb); + } + } + /* Here we are. Hitlist is filled. Die. */ __skb_queue_purge(&hitlist); spin_lock(&unix_gc_lock); + /* There could be io_uring registered files, just push them back to + * the inflight list + */ + list_for_each_entry_safe(u, next, &gc_candidates, link) + list_move_tail(&u->link, &gc_inflight_list); + /* All candidates should have been detached by now. */ BUG_ON(!list_empty(&gc_candidates)); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index b4ee163154a683ef22568c6c05825fa2ad14a768..ee418701cdee902ee7fea08767f51be055b87ad6 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -882,6 +882,16 @@ s64 vsock_stream_has_space(struct vsock_sock *vsk) } EXPORT_SYMBOL_GPL(vsock_stream_has_space); +void vsock_data_ready(struct sock *sk) +{ + struct vsock_sock *vsk = vsock_sk(sk); + + if (vsock_stream_has_data(vsk) >= sk->sk_rcvlowat || + sock_flag(sk, SOCK_DONE)) + sk->sk_data_ready(sk); +} +EXPORT_SYMBOL_GPL(vsock_data_ready); + static int vsock_release(struct socket *sock) { __vsock_release(sock->sk, 0); @@ -1066,8 +1076,9 @@ static __poll_t vsock_poll(struct file *file, struct socket *sock, if (transport && transport->stream_is_active(vsk) && !(sk->sk_shutdown & RCV_SHUTDOWN)) { bool data_ready_now = false; + int target = sock_rcvlowat(sk, 0, INT_MAX); int ret = transport->notify_poll_in( - vsk, 1, &data_ready_now); + vsk, target, &data_ready_now); if (ret < 0) { mask |= EPOLLERR; } else { @@ -2137,6 +2148,25 @@ out: return err; } +static int vsock_set_rcvlowat(struct sock *sk, int val) +{ + const struct vsock_transport *transport; + struct vsock_sock *vsk; + + vsk = vsock_sk(sk); + + if (val > vsk->buffer_size) + return -EINVAL; + + transport = vsk->transport; + + if (transport && transport->set_rcvlowat) + return transport->set_rcvlowat(vsk, val); + + WRITE_ONCE(sk->sk_rcvlowat, val ? : 1); + return 0; +} + static const struct proto_ops vsock_stream_ops = { .family = PF_VSOCK, .owner = THIS_MODULE, @@ -2156,6 +2186,7 @@ static const struct proto_ops vsock_stream_ops = { .recvmsg = vsock_connectible_recvmsg, .mmap = sock_no_mmap, .sendpage = sock_no_sendpage, + .set_rcvlowat = vsock_set_rcvlowat, }; static const struct proto_ops vsock_seqpacket_ops = { diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index fd98229e3db30988b983761b9c809ce059d6d9b2..59c3e2697069045ae1227f0ea1d1064823148517 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -815,6 +815,12 @@ int hvs_notify_send_post_enqueue(struct vsock_sock *vsk, ssize_t written, return 0; } +static +int hvs_set_rcvlowat(struct vsock_sock *vsk, int val) +{ + return -EOPNOTSUPP; +} + static struct vsock_transport hvs_transport = { .module = THIS_MODULE, @@ -850,6 +856,7 @@ static struct vsock_transport hvs_transport = { .notify_send_pre_enqueue = hvs_notify_send_pre_enqueue, .notify_send_post_enqueue = hvs_notify_send_post_enqueue, + .set_rcvlowat = hvs_set_rcvlowat }; static bool hvs_check_transport(struct vsock_sock *vsk) diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index ec2c2afbf0d060b4f6335a7616c8deba0a95fde6..a9980e9b93040613d7a3863c1cff472e3dd63270 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -634,10 +634,7 @@ virtio_transport_notify_poll_in(struct vsock_sock *vsk, size_t target, bool *data_ready_now) { - if (vsock_stream_has_data(vsk)) - *data_ready_now = true; - else - *data_ready_now = false; + *data_ready_now = vsock_stream_has_data(vsk) >= target; return 0; } @@ -1084,7 +1081,7 @@ virtio_transport_recv_connected(struct sock *sk, switch (le16_to_cpu(pkt->hdr.op)) { case VIRTIO_VSOCK_OP_RW: virtio_transport_recv_enqueue(vsk, pkt); - sk->sk_data_ready(sk); + vsock_data_ready(sk); return err; case VIRTIO_VSOCK_OP_CREDIT_REQUEST: virtio_transport_send_credit_update(vsk); @@ -1342,7 +1339,7 @@ EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) { - kfree(pkt->buf); + kvfree(pkt->buf); kfree(pkt); } EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index b14f0ed7427bc4f35307fc28de6e4908ae659448..842c94286d316af337fa2fa9f3a598284bf2f306 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -951,7 +951,7 @@ static int vmci_transport_recv_listen(struct sock *sk, * for ourself or any previous connection requests that we received. * If it's the latter, we try to find a socket in our list of pending * connections and, if we do, call the appropriate handler for the - * state that that socket is in. Otherwise we try to service the + * state that socket is in. Otherwise we try to service the * connection request. */ pending = vmci_transport_get_pending(sk, pkt); diff --git a/net/vmw_vsock/vmci_transport_notify.c b/net/vmw_vsock/vmci_transport_notify.c index d69fc4b595ad42b2784698ea1e8e2b7b60632343..7c3a7db134b2820033c5d475731b0ab2b84382e5 100644 --- a/net/vmw_vsock/vmci_transport_notify.c +++ b/net/vmw_vsock/vmci_transport_notify.c @@ -307,7 +307,7 @@ vmci_transport_handle_wrote(struct sock *sk, struct vsock_sock *vsk = vsock_sk(sk); PKT_FIELD(vsk, sent_waiting_read) = false; #endif - sk->sk_data_ready(sk); + vsock_data_ready(sk); } static void vmci_transport_notify_pkt_socket_init(struct sock *sk) @@ -340,12 +340,12 @@ vmci_transport_notify_pkt_poll_in(struct sock *sk, { struct vsock_sock *vsk = vsock_sk(sk); - if (vsock_stream_has_data(vsk)) { + if (vsock_stream_has_data(vsk) >= target) { *data_ready_now = true; } else { - /* We can't read right now because there is nothing in the - * queue. Ask for notifications when there is something to - * read. + /* We can't read right now because there is not enough data + * in the queue. Ask for notifications when there is something + * to read. */ if (sk->sk_state == TCP_ESTABLISHED) { if (!send_waiting_read(sk, 1)) diff --git a/net/vmw_vsock/vmci_transport_notify_qstate.c b/net/vmw_vsock/vmci_transport_notify_qstate.c index 0f36d7c45db3933f25add3a3e15f6f5fb8c242f7..e96a88d850a86a9ffbd422674b080ef324dc4799 100644 --- a/net/vmw_vsock/vmci_transport_notify_qstate.c +++ b/net/vmw_vsock/vmci_transport_notify_qstate.c @@ -84,7 +84,7 @@ vmci_transport_handle_wrote(struct sock *sk, bool bottom_half, struct sockaddr_vm *dst, struct sockaddr_vm *src) { - sk->sk_data_ready(sk); + vsock_data_ready(sk); } static void vsock_block_update_write_window(struct sock *sk) @@ -161,12 +161,12 @@ vmci_transport_notify_pkt_poll_in(struct sock *sk, { struct vsock_sock *vsk = vsock_sk(sk); - if (vsock_stream_has_data(vsk)) { + if (vsock_stream_has_data(vsk) >= target) { *data_ready_now = true; } else { - /* We can't read right now because there is nothing in the - * queue. Ask for notifications when there is something to - * read. + /* We can't read right now because there is not enough data + * in the queue. Ask for notifications when there is something + * to read. */ if (sk->sk_state == TCP_ESTABLISHED) vsock_block_update_write_window(sk); @@ -282,7 +282,7 @@ vmci_transport_notify_pkt_recv_post_dequeue( /* See the comment in * vmci_transport_notify_pkt_send_post_enqueue(). */ - sk->sk_data_ready(sk); + vsock_data_ready(sk); } return err; diff --git a/net/wireless/core.c b/net/wireless/core.c index eefd6d8ff46518d5875d86ef704b3463132d9e86..5b0c4d5b80cf5f61c006ac17eea29ceb481d6742 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -860,6 +860,9 @@ int wiphy_register(struct wiphy *wiphy) for (i = 0; i < sband->n_iftype_data; i++) { const struct ieee80211_sband_iftype_data *iftd; + bool has_ap, has_non_ap; + u32 ap_bits = BIT(NL80211_IFTYPE_AP) | + BIT(NL80211_IFTYPE_P2P_GO); iftd = &sband->iftype_data[i]; @@ -879,6 +882,19 @@ int wiphy_register(struct wiphy *wiphy) else have_he = have_he && iftd->he_cap.has_he; + + has_ap = iftd->types_mask & ap_bits; + has_non_ap = iftd->types_mask & ~ap_bits; + + /* + * For EHT 20 MHz STA, the capabilities format differs + * but to simplify, don't check 20 MHz but rather check + * only if AP and non-AP were mentioned at the same time, + * reject if so. + */ + if (WARN_ON(iftd->eht_cap.has_eht && + has_ap && has_non_ap)) + return -EINVAL; } if (WARN_ON(!have_he && band == NL80211_BAND_6GHZ)) diff --git a/net/wireless/debugfs.c b/net/wireless/debugfs.c index aab43469a2f0417096e2a5e7689c88236f88943e..0878b162890af7c57073991468415a87dc9edc16 100644 --- a/net/wireless/debugfs.c +++ b/net/wireless/debugfs.c @@ -65,9 +65,10 @@ static ssize_t ht40allow_map_read(struct file *file, { struct wiphy *wiphy = file->private_data; char *buf; - unsigned int offset = 0, buf_size = PAGE_SIZE, i, r; + unsigned int offset = 0, buf_size = PAGE_SIZE, i; enum nl80211_band band; struct ieee80211_supported_band *sband; + ssize_t r; buf = kzalloc(buf_size, GFP_KERNEL); if (!buf) diff --git a/net/wireless/ibss.c b/net/wireless/ibss.c index 4935f94d1acc8b98c61078e2dca8a6dcff3ffd8c..edd062f104f482a9d26ccb3ca0a45277c1883cc3 100644 --- a/net/wireless/ibss.c +++ b/net/wireless/ibss.c @@ -171,7 +171,7 @@ static void __cfg80211_clear_ibss(struct net_device *dev, bool nowext) */ if (rdev->ops->del_key) for (i = 0; i < 6; i++) - rdev_del_key(rdev, dev, i, false, NULL); + rdev_del_key(rdev, dev, -1, i, false, NULL); if (wdev->u.ibss.current_bss) { cfg80211_unhold_bss(wdev->u.ibss.current_bss); diff --git a/net/wireless/lib80211_crypt_ccmp.c b/net/wireless/lib80211_crypt_ccmp.c index 6a5f08f7491eba108cd2ffd1d0e20f7b6b5ee5f5..cca5e1cf089ec0a3d25594925d5fc2d8520cbab4 100644 --- a/net/wireless/lib80211_crypt_ccmp.c +++ b/net/wireless/lib80211_crypt_ccmp.c @@ -136,7 +136,7 @@ static int ccmp_init_iv_and_aad(const struct ieee80211_hdr *hdr, pos = (u8 *) hdr; aad[0] = pos[0] & 0x8f; aad[1] = pos[1] & 0xc7; - memcpy(aad + 2, hdr->addr1, 3 * ETH_ALEN); + memcpy(aad + 2, &hdr->addrs, 3 * ETH_ALEN); pos = (u8 *) & hdr->seq_ctrl; aad[20] = pos[0] & 0x0f; aad[21] = 0; /* all bits masked */ diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 2705e3ee8fc4e83eebc4d9a43dd23b070cace22b..597c5223651460c2f45613751912dd56f807c2da 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -1545,7 +1545,6 @@ static int nl80211_key_allowed(struct wireless_dev *wdev) return -ENOLINK; case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_P2P_CLIENT: - /* for MLO, require driver validation of the link ID */ if (wdev->connected) return 0; return -ENOLINK; @@ -1821,10 +1820,15 @@ nl80211_send_iftype_data(struct sk_buff *msg, if (eht_cap->has_eht && he_cap->has_he) { u8 mcs_nss_size, ppe_thresh_size; u16 ppe_thres_hdr; + bool is_ap; + + is_ap = iftdata->types_mask & BIT(NL80211_IFTYPE_AP) || + iftdata->types_mask & BIT(NL80211_IFTYPE_P2P_GO); mcs_nss_size = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem, - &eht_cap->eht_cap_elem); + &eht_cap->eht_cap_elem, + is_ap); ppe_thres_hdr = get_unaligned_le16(&eht_cap->eht_ppe_thres[0]); ppe_thresh_size = @@ -3476,8 +3480,21 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info) if (result) goto out; - result = rdev_set_txq_params(rdev, netdev, - &txq_params); + txq_params.link_id = + nl80211_link_id_or_invalid(info->attrs); + + wdev_lock(netdev->ieee80211_ptr); + if (txq_params.link_id >= 0 && + !(netdev->ieee80211_ptr->valid_links & + BIT(txq_params.link_id))) + result = -ENOLINK; + else if (txq_params.link_id >= 0 && + !netdev->ieee80211_ptr->valid_links) + result = -EINVAL; + else + result = rdev_set_txq_params(rdev, netdev, + &txq_params); + wdev_unlock(netdev->ieee80211_ptr); if (result) goto out; } @@ -3848,12 +3865,19 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag for_each_valid_link(wdev, link_id) { struct nlattr *link = nla_nest_start(msg, link_id + 1); + struct cfg80211_chan_def chandef = {}; + int ret; if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id)) goto nla_put_failure; if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, wdev->links[link_id].addr)) goto nla_put_failure; + + ret = rdev_get_channel(rdev, wdev, link_id, &chandef); + if (ret == 0 && nl80211_send_chandef(msg, &chandef)) + goto nla_put_failure; + nla_nest_end(msg, link); } @@ -4320,6 +4344,38 @@ static int nl80211_set_noack_map(struct sk_buff *skb, struct genl_info *info) return rdev_set_noack_map(rdev, dev, noack_map); } +static int nl80211_validate_key_link_id(struct genl_info *info, + struct wireless_dev *wdev, + int link_id, bool pairwise) +{ + if (pairwise) { + if (link_id != -1) { + GENL_SET_ERR_MSG(info, + "link ID not allowed for pairwise key"); + return -EINVAL; + } + + return 0; + } + + if (wdev->valid_links) { + if (link_id == -1) { + GENL_SET_ERR_MSG(info, + "link ID must for MLO group key"); + return -EINVAL; + } + if (!(wdev->valid_links & BIT(link_id))) { + GENL_SET_ERR_MSG(info, "invalid link ID for MLO group key"); + return -EINVAL; + } + } else if (link_id != -1) { + GENL_SET_ERR_MSG(info, "link ID not allowed for non-MLO group key"); + return -EINVAL; + } + + return 0; +} + struct get_key_cookie { struct sk_buff *msg; int error; @@ -4381,13 +4437,15 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info) void *hdr; struct sk_buff *msg; bool bigtk_support = false; + int link_id = nl80211_link_id_or_invalid(info->attrs); + struct wireless_dev *wdev = dev->ieee80211_ptr; if (wiphy_ext_feature_isset(&rdev->wiphy, NL80211_EXT_FEATURE_BEACON_PROTECTION)) bigtk_support = true; - if ((dev->ieee80211_ptr->iftype == NL80211_IFTYPE_STATION || - dev->ieee80211_ptr->iftype == NL80211_IFTYPE_P2P_CLIENT) && + if ((wdev->iftype == NL80211_IFTYPE_STATION || + wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) && wiphy_ext_feature_isset(&rdev->wiphy, NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT)) bigtk_support = true; @@ -4439,8 +4497,12 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info) nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr)) goto nla_put_failure; - err = rdev_get_key(rdev, dev, key_idx, pairwise, mac_addr, &cookie, - get_key_callback); + err = nl80211_validate_key_link_id(info, wdev, link_id, pairwise); + if (err) + goto free_msg; + + err = rdev_get_key(rdev, dev, link_id, key_idx, pairwise, mac_addr, + &cookie, get_key_callback); if (err) goto free_msg; @@ -4464,6 +4526,8 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) struct key_parse key; int err; struct net_device *dev = info->user_ptr[1]; + int link_id = nl80211_link_id_or_invalid(info->attrs); + struct wireless_dev *wdev = dev->ieee80211_ptr; err = nl80211_parse_key(info, &key); if (err) @@ -4479,7 +4543,7 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) !(key.p.mode == NL80211_KEY_SET_TX)) return -EINVAL; - wdev_lock(dev->ieee80211_ptr); + wdev_lock(wdev); if (key.def) { if (!rdev->ops->set_default_key) { @@ -4487,18 +4551,22 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) goto out; } - err = nl80211_key_allowed(dev->ieee80211_ptr); + err = nl80211_key_allowed(wdev); if (err) goto out; - err = rdev_set_default_key(rdev, dev, key.idx, - key.def_uni, key.def_multi); + err = nl80211_validate_key_link_id(info, wdev, link_id, false); + if (err) + goto out; + + err = rdev_set_default_key(rdev, dev, link_id, key.idx, + key.def_uni, key.def_multi); if (err) goto out; #ifdef CONFIG_CFG80211_WEXT - dev->ieee80211_ptr->wext.default_key = key.idx; + wdev->wext.default_key = key.idx; #endif } else if (key.defmgmt) { if (key.def_uni || !key.def_multi) { @@ -4511,16 +4579,20 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) goto out; } - err = nl80211_key_allowed(dev->ieee80211_ptr); + err = nl80211_key_allowed(wdev); if (err) goto out; - err = rdev_set_default_mgmt_key(rdev, dev, key.idx); + err = nl80211_validate_key_link_id(info, wdev, link_id, false); + if (err) + goto out; + + err = rdev_set_default_mgmt_key(rdev, dev, link_id, key.idx); if (err) goto out; #ifdef CONFIG_CFG80211_WEXT - dev->ieee80211_ptr->wext.default_mgmt_key = key.idx; + wdev->wext.default_mgmt_key = key.idx; #endif } else if (key.defbeacon) { if (key.def_uni || !key.def_multi) { @@ -4533,11 +4605,15 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) goto out; } - err = nl80211_key_allowed(dev->ieee80211_ptr); + err = nl80211_key_allowed(wdev); + if (err) + goto out; + + err = nl80211_validate_key_link_id(info, wdev, link_id, false); if (err) goto out; - err = rdev_set_default_beacon_key(rdev, dev, key.idx); + err = rdev_set_default_beacon_key(rdev, dev, link_id, key.idx); if (err) goto out; } else if (key.p.mode == NL80211_KEY_SET_TX && @@ -4553,14 +4629,18 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info) goto out; } - err = rdev_add_key(rdev, dev, key.idx, + err = nl80211_validate_key_link_id(info, wdev, link_id, true); + if (err) + goto out; + + err = rdev_add_key(rdev, dev, link_id, key.idx, NL80211_KEYTYPE_PAIRWISE, mac_addr, &key.p); } else { err = -EINVAL; } out: - wdev_unlock(dev->ieee80211_ptr); + wdev_unlock(wdev); return err; } @@ -4572,6 +4652,8 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info) struct net_device *dev = info->user_ptr[1]; struct key_parse key; const u8 *mac_addr = NULL; + int link_id = nl80211_link_id_or_invalid(info->attrs); + struct wireless_dev *wdev = dev->ieee80211_ptr; err = nl80211_parse_key(info, &key); if (err) @@ -4613,18 +4695,23 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info) return -EINVAL; } - wdev_lock(dev->ieee80211_ptr); - err = nl80211_key_allowed(dev->ieee80211_ptr); + wdev_lock(wdev); + err = nl80211_key_allowed(wdev); if (err) GENL_SET_ERR_MSG(info, "key not allowed"); + + if (!err) + err = nl80211_validate_key_link_id(info, wdev, link_id, + key.type == NL80211_KEYTYPE_PAIRWISE); + if (!err) { - err = rdev_add_key(rdev, dev, key.idx, + err = rdev_add_key(rdev, dev, link_id, key.idx, key.type == NL80211_KEYTYPE_PAIRWISE, mac_addr, &key.p); if (err) GENL_SET_ERR_MSG(info, "key addition failed"); } - wdev_unlock(dev->ieee80211_ptr); + wdev_unlock(wdev); return err; } @@ -4636,6 +4723,8 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info) struct net_device *dev = info->user_ptr[1]; u8 *mac_addr = NULL; struct key_parse key; + int link_id = nl80211_link_id_or_invalid(info->attrs); + struct wireless_dev *wdev = dev->ieee80211_ptr; err = nl80211_parse_key(info, &key); if (err) @@ -4663,27 +4752,31 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info) if (!rdev->ops->del_key) return -EOPNOTSUPP; - wdev_lock(dev->ieee80211_ptr); - err = nl80211_key_allowed(dev->ieee80211_ptr); + wdev_lock(wdev); + err = nl80211_key_allowed(wdev); if (key.type == NL80211_KEYTYPE_GROUP && mac_addr && !(rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN)) err = -ENOENT; if (!err) - err = rdev_del_key(rdev, dev, key.idx, + err = nl80211_validate_key_link_id(info, wdev, link_id, + key.type == NL80211_KEYTYPE_PAIRWISE); + + if (!err) + err = rdev_del_key(rdev, dev, link_id, key.idx, key.type == NL80211_KEYTYPE_PAIRWISE, mac_addr); #ifdef CONFIG_CFG80211_WEXT if (!err) { - if (key.idx == dev->ieee80211_ptr->wext.default_key) - dev->ieee80211_ptr->wext.default_key = -1; - else if (key.idx == dev->ieee80211_ptr->wext.default_mgmt_key) - dev->ieee80211_ptr->wext.default_mgmt_key = -1; + if (key.idx == wdev->wext.default_key) + wdev->wext.default_key = -1; + else if (key.idx == wdev->wext.default_mgmt_key) + wdev->wext.default_mgmt_key = -1; } #endif - wdev_unlock(dev->ieee80211_ptr); + wdev_unlock(wdev); return err; } @@ -5587,7 +5680,7 @@ static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params) params->eht_cap = (void *)(cap->data + 1); if (!ieee80211_eht_capa_size_ok((const u8 *)params->he_cap, (const u8 *)params->eht_cap, - cap->datalen - 1)) + cap->datalen - 1, true)) return -EINVAL; } cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_OPERATION, ies, ies_len); @@ -6816,7 +6909,8 @@ static int nl80211_set_station_tdls(struct genl_info *info, if (!ieee80211_eht_capa_size_ok((const u8 *)params->link_sta_params.he_capa, (const u8 *)params->link_sta_params.eht_capa, - params->link_sta_params.eht_capa_len)) + params->link_sta_params.eht_capa_len, + false)) return -EINVAL; } } @@ -7127,7 +7221,8 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info) if (!ieee80211_eht_capa_size_ok((const u8 *)params.link_sta_params.he_capa, (const u8 *)params.link_sta_params.eht_capa, - params.link_sta_params.eht_capa_len)) + params.link_sta_params.eht_capa_len, + false)) return -EINVAL; } } @@ -10087,8 +10182,10 @@ static int nl80211_send_bss(struct sk_buff *msg, struct netlink_callback *cb, (nla_put_u32(msg, NL80211_BSS_STATUS, NL80211_BSS_STATUS_ASSOCIATED) || (wdev->valid_links && - nla_put_u8(msg, NL80211_BSS_MLO_LINK_ID, - link_id)))) + (nla_put_u8(msg, NL80211_BSS_MLO_LINK_ID, + link_id) || + nla_put(msg, NL80211_BSS_MLD_ADDR, ETH_ALEN, + wdev->u.client.connected_addr))))) goto nla_put_failure; } break; @@ -11184,7 +11281,6 @@ static int nl80211_set_mcast_rate(struct sk_buff *skb, struct genl_info *info) struct net_device *dev = info->user_ptr[1]; int mcast_rate[NUM_NL80211_BANDS]; u32 nla_rate; - int err; if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_ADHOC && dev->ieee80211_ptr->iftype != NL80211_IFTYPE_MESH_POINT && @@ -11203,9 +11299,7 @@ static int nl80211_set_mcast_rate(struct sk_buff *skb, struct genl_info *info) if (!nl80211_parse_mcast_rate(rdev, mcast_rate, nla_rate)) return -EINVAL; - err = rdev_set_mcast_rate(rdev, dev, mcast_rate); - - return err; + return rdev_set_mcast_rate(rdev, dev, mcast_rate); } static struct sk_buff * @@ -13171,7 +13265,9 @@ static int nl80211_parse_wowlan_tcp(struct cfg80211_registered_device *rdev, wake_mask_size); if (tok) { cfg->tokens_size = tokens_size; - memcpy(&cfg->payload_tok, tok, sizeof(*tok) + tokens_size); + cfg->payload_tok = *tok; + memcpy(cfg->payload_tok.token_stream, tok->token_stream, + tokens_size); } trig->tcp = cfg; @@ -15887,7 +15983,8 @@ nl80211_add_mod_link_station(struct sk_buff *skb, struct genl_info *info, if (!ieee80211_eht_capa_size_ok((const u8 *)params.he_capa, (const u8 *)params.eht_capa, - params.eht_capa_len)) + params.eht_capa_len, + false)) return -EINVAL; } } @@ -17141,6 +17238,7 @@ static struct genl_family nl80211_fam __ro_after_init = { .n_ops = ARRAY_SIZE(nl80211_ops), .small_ops = nl80211_small_ops, .n_small_ops = ARRAY_SIZE(nl80211_small_ops), + .resv_start_op = NL80211_CMD_REMOVE_LINK_STA + 1, .mcgrps = nl80211_mcgrps, .n_mcgrps = ARRAY_SIZE(nl80211_mcgrps), .parallel_ops = true, @@ -18836,11 +18934,13 @@ EXPORT_SYMBOL(cfg80211_pmksa_candidate_notify); static void nl80211_ch_switch_notify(struct cfg80211_registered_device *rdev, struct net_device *netdev, + unsigned int link_id, struct cfg80211_chan_def *chandef, gfp_t gfp, enum nl80211_commands notif, u8 count, bool quiet) { + struct wireless_dev *wdev = netdev->ieee80211_ptr; struct sk_buff *msg; void *hdr; @@ -18857,6 +18957,10 @@ static void nl80211_ch_switch_notify(struct cfg80211_registered_device *rdev, if (nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex)) goto nla_put_failure; + if (wdev->valid_links && + nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id)) + goto nla_put_failure; + if (nl80211_send_chandef(msg, chandef)) goto nla_put_failure; @@ -18916,22 +19020,26 @@ void cfg80211_ch_switch_notify(struct net_device *dev, cfg80211_sched_dfs_chan_update(rdev); - nl80211_ch_switch_notify(rdev, dev, chandef, GFP_KERNEL, + nl80211_ch_switch_notify(rdev, dev, link_id, chandef, GFP_KERNEL, NL80211_CMD_CH_SWITCH_NOTIFY, 0, false); } EXPORT_SYMBOL(cfg80211_ch_switch_notify); void cfg80211_ch_switch_started_notify(struct net_device *dev, struct cfg80211_chan_def *chandef, - u8 count, bool quiet) + unsigned int link_id, u8 count, + bool quiet) { struct wireless_dev *wdev = dev->ieee80211_ptr; struct wiphy *wiphy = wdev->wiphy; struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); - trace_cfg80211_ch_switch_started_notify(dev, chandef); + ASSERT_WDEV_LOCK(wdev); + WARN_INVALID_LINK_ID(wdev, link_id); + + trace_cfg80211_ch_switch_started_notify(dev, chandef, link_id); - nl80211_ch_switch_notify(rdev, dev, chandef, GFP_KERNEL, + nl80211_ch_switch_notify(rdev, dev, link_id, chandef, GFP_KERNEL, NL80211_CMD_CH_SWITCH_STARTED_NOTIFY, count, quiet); } diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index 40915a82da739011fabf4bd00213e7c8b6ceaf3f..13b209a8db2879aaf2ee0d8400246e3a1ea3d1db 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -77,65 +77,69 @@ rdev_change_virtual_intf(struct cfg80211_registered_device *rdev, } static inline int rdev_add_key(struct cfg80211_registered_device *rdev, - struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr, + struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr, struct key_params *params) { int ret; - trace_rdev_add_key(&rdev->wiphy, netdev, key_index, pairwise, + trace_rdev_add_key(&rdev->wiphy, netdev, link_id, key_index, pairwise, mac_addr, params->mode); - ret = rdev->ops->add_key(&rdev->wiphy, netdev, key_index, pairwise, - mac_addr, params); + ret = rdev->ops->add_key(&rdev->wiphy, netdev, link_id, key_index, + pairwise, mac_addr, params); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } static inline int rdev_get_key(struct cfg80211_registered_device *rdev, struct net_device *netdev, - u8 key_index, bool pairwise, const u8 *mac_addr, void *cookie, + int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, + void *cookie, void (*callback)(void *cookie, struct key_params*)) { int ret; - trace_rdev_get_key(&rdev->wiphy, netdev, key_index, pairwise, mac_addr); - ret = rdev->ops->get_key(&rdev->wiphy, netdev, key_index, pairwise, - mac_addr, cookie, callback); + trace_rdev_get_key(&rdev->wiphy, netdev, link_id, key_index, pairwise, + mac_addr); + ret = rdev->ops->get_key(&rdev->wiphy, netdev, link_id, key_index, + pairwise, mac_addr, cookie, callback); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } static inline int rdev_del_key(struct cfg80211_registered_device *rdev, - struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr) + struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr) { int ret; - trace_rdev_del_key(&rdev->wiphy, netdev, key_index, pairwise, mac_addr); - ret = rdev->ops->del_key(&rdev->wiphy, netdev, key_index, pairwise, - mac_addr); + trace_rdev_del_key(&rdev->wiphy, netdev, link_id, key_index, pairwise, + mac_addr); + ret = rdev->ops->del_key(&rdev->wiphy, netdev, link_id, key_index, + pairwise, mac_addr); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } static inline int rdev_set_default_key(struct cfg80211_registered_device *rdev, - struct net_device *netdev, u8 key_index, bool unicast, - bool multicast) + struct net_device *netdev, int link_id, u8 key_index, + bool unicast, bool multicast) { int ret; - trace_rdev_set_default_key(&rdev->wiphy, netdev, key_index, + trace_rdev_set_default_key(&rdev->wiphy, netdev, link_id, key_index, unicast, multicast); - ret = rdev->ops->set_default_key(&rdev->wiphy, netdev, key_index, - unicast, multicast); + ret = rdev->ops->set_default_key(&rdev->wiphy, netdev, link_id, + key_index, unicast, multicast); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } static inline int rdev_set_default_mgmt_key(struct cfg80211_registered_device *rdev, - struct net_device *netdev, u8 key_index) + struct net_device *netdev, int link_id, u8 key_index) { int ret; - trace_rdev_set_default_mgmt_key(&rdev->wiphy, netdev, key_index); - ret = rdev->ops->set_default_mgmt_key(&rdev->wiphy, netdev, + trace_rdev_set_default_mgmt_key(&rdev->wiphy, netdev, link_id, + key_index); + ret = rdev->ops->set_default_mgmt_key(&rdev->wiphy, netdev, link_id, key_index); trace_rdev_return_int(&rdev->wiphy, ret); return ret; @@ -143,13 +147,15 @@ rdev_set_default_mgmt_key(struct cfg80211_registered_device *rdev, static inline int rdev_set_default_beacon_key(struct cfg80211_registered_device *rdev, - struct net_device *netdev, u8 key_index) + struct net_device *netdev, int link_id, + u8 key_index) { int ret; - trace_rdev_set_default_beacon_key(&rdev->wiphy, netdev, key_index); - ret = rdev->ops->set_default_beacon_key(&rdev->wiphy, netdev, - key_index); + trace_rdev_set_default_beacon_key(&rdev->wiphy, netdev, link_id, + key_index); + ret = rdev->ops->set_default_beacon_key(&rdev->wiphy, netdev, link_id, + key_index); trace_rdev_return_int(&rdev->wiphy, ret); return ret; } diff --git a/net/wireless/reg.c b/net/wireless/reg.c index c7383ede794fc4e6b35915c5121a819edc49b007..d5c7a5aa68532365984ac3b30ef0e79b0a090702 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -2389,6 +2389,10 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev) switch (iftype) { case NL80211_IFTYPE_AP: case NL80211_IFTYPE_P2P_GO: + if (!wdev->links[link].ap.beacon_interval) + continue; + chandef = wdev->links[link].ap.chandef; + break; case NL80211_IFTYPE_MESH_POINT: if (!wdev->u.mesh.beacon_interval) continue; diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 0134e5d5c81a4f9297fa7d8d5fdc45f0e6ae21eb..806a5f1330ff566a29c375cfc17218e78e252a09 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -143,18 +143,12 @@ static inline void bss_ref_get(struct cfg80211_registered_device *rdev, lockdep_assert_held(&rdev->bss_lock); bss->refcount++; - if (bss->pub.hidden_beacon_bss) { - bss = container_of(bss->pub.hidden_beacon_bss, - struct cfg80211_internal_bss, - pub); - bss->refcount++; - } - if (bss->pub.transmitted_bss) { - bss = container_of(bss->pub.transmitted_bss, - struct cfg80211_internal_bss, - pub); - bss->refcount++; - } + + if (bss->pub.hidden_beacon_bss) + bss_from_pub(bss->pub.hidden_beacon_bss)->refcount++; + + if (bss->pub.transmitted_bss) + bss_from_pub(bss->pub.transmitted_bss)->refcount++; } static inline void bss_ref_put(struct cfg80211_registered_device *rdev, @@ -304,7 +298,8 @@ static size_t cfg80211_gen_new_ie(const u8 *ie, size_t ielen, tmp_old = cfg80211_find_ie(WLAN_EID_SSID, ie, ielen); tmp_old = (tmp_old) ? tmp_old + tmp_old[1] + 2 : ie; - while (tmp_old + tmp_old[1] + 2 - ie <= ielen) { + while (tmp_old + 2 - ie <= ielen && + tmp_old + tmp_old[1] + 2 - ie <= ielen) { if (tmp_old[0] == 0) { tmp_old++; continue; @@ -364,7 +359,8 @@ static size_t cfg80211_gen_new_ie(const u8 *ie, size_t ielen, * copied to new ie, skip ssid, capability, bssid-index ie */ tmp_new = sub_copy; - while (tmp_new + tmp_new[1] + 2 - sub_copy <= subie_len) { + while (tmp_new + 2 - sub_copy <= subie_len && + tmp_new + tmp_new[1] + 2 - sub_copy <= subie_len) { if (!(tmp_new[0] == WLAN_EID_NON_TX_BSSID_CAP || tmp_new[0] == WLAN_EID_SSID)) { memcpy(pos, tmp_new, tmp_new[1] + 2); @@ -427,6 +423,15 @@ cfg80211_add_nontrans_list(struct cfg80211_bss *trans_bss, rcu_read_unlock(); + /* + * This is a bit weird - it's not on the list, but already on another + * one! The only way that could happen is if there's some BSSID/SSID + * shared by multiple APs in their multi-BSSID profiles, potentially + * with hidden SSID mixed in ... ignore it. + */ + if (!list_empty(&nontrans_bss->nontrans_list)) + return -EINVAL; + /* add to the list */ list_add_tail(&nontrans_bss->nontrans_list, &trans_bss->nontrans_list); return 0; @@ -540,7 +545,7 @@ static int cfg80211_parse_ap_info(struct cfg80211_colocated_ap *entry, memcpy(entry->bssid, pos, ETH_ALEN); pos += ETH_ALEN; - if (length == IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM) { + if (length >= IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM) { memcpy(&entry->short_ssid, pos, sizeof(entry->short_ssid)); entry->short_ssid_valid = true; @@ -1602,6 +1607,23 @@ struct cfg80211_non_tx_bss { u8 bssid_index; }; +static void cfg80211_update_hidden_bsses(struct cfg80211_internal_bss *known, + const struct cfg80211_bss_ies *new_ies, + const struct cfg80211_bss_ies *old_ies) +{ + struct cfg80211_internal_bss *bss; + + /* Assign beacon IEs to all sub entries */ + list_for_each_entry(bss, &known->hidden_list, hidden_list) { + const struct cfg80211_bss_ies *ies; + + ies = rcu_access_pointer(bss->pub.beacon_ies); + WARN_ON(ies != old_ies); + + rcu_assign_pointer(bss->pub.beacon_ies, new_ies); + } +} + static bool cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, struct cfg80211_internal_bss *known, @@ -1625,7 +1647,6 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); } else if (rcu_access_pointer(new->pub.beacon_ies)) { const struct cfg80211_bss_ies *old; - struct cfg80211_internal_bss *bss; if (known->pub.hidden_beacon_bss && !list_empty(&known->hidden_list)) { @@ -1653,16 +1674,7 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, if (old == rcu_access_pointer(known->pub.ies)) rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies); - /* Assign beacon IEs to all sub entries */ - list_for_each_entry(bss, &known->hidden_list, hidden_list) { - const struct cfg80211_bss_ies *ies; - - ies = rcu_access_pointer(bss->pub.beacon_ies); - WARN_ON(ies != old); - - rcu_assign_pointer(bss->pub.beacon_ies, - new->pub.beacon_ies); - } + cfg80211_update_hidden_bsses(known, new->pub.beacon_ies, old); if (old) kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); @@ -1739,6 +1751,8 @@ cfg80211_bss_update(struct cfg80211_registered_device *rdev, new->refcount = 1; INIT_LIST_HEAD(&new->hidden_list); INIT_LIST_HEAD(&new->pub.nontrans_list); + /* we'll set this later if it was non-NULL */ + new->pub.transmitted_bss = NULL; if (rcu_access_pointer(tmp->pub.proberesp_ies)) { hidden = rb_find_bss(rdev, tmp, BSS_CMP_HIDE_ZLEN); @@ -2021,10 +2035,15 @@ cfg80211_inform_single_bss_data(struct wiphy *wiphy, spin_lock_bh(&rdev->bss_lock); if (cfg80211_add_nontrans_list(non_tx_data->tx_bss, &res->pub)) { - if (__cfg80211_unlink_bss(rdev, res)) + if (__cfg80211_unlink_bss(rdev, res)) { rdev->bss_generation++; + res = NULL; + } } spin_unlock_bh(&rdev->bss_lock); + + if (!res) + return NULL; } trace_cfg80211_return_bss(&res->pub); @@ -2143,6 +2162,8 @@ static void cfg80211_parse_mbssid_data(struct wiphy *wiphy, for_each_element_id(elem, WLAN_EID_MULTIPLE_BSSID, ie, ielen) { if (elem->datalen < 4) continue; + if (elem->data[0] < 1 || (int)elem->data[0] > 8) + continue; for_each_element(sub, elem->data + 1, elem->datalen - 1) { u8 profile_len; @@ -2279,7 +2300,7 @@ cfg80211_update_notlisted_nontrans(struct wiphy *wiphy, size_t new_ie_len; struct cfg80211_bss_ies *new_ies; const struct cfg80211_bss_ies *old; - u8 cpy_len; + size_t cpy_len; lockdep_assert_held(&wiphy_to_rdev(wiphy)->bss_lock); @@ -2346,6 +2367,8 @@ cfg80211_update_notlisted_nontrans(struct wiphy *wiphy, } else { old = rcu_access_pointer(nontrans_bss->beacon_ies); rcu_assign_pointer(nontrans_bss->beacon_ies, new_ies); + cfg80211_update_hidden_bsses(bss_from_pub(nontrans_bss), + new_ies, old); rcu_assign_pointer(nontrans_bss->ies, new_ies); if (old) kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); diff --git a/net/wireless/sme.c b/net/wireless/sme.c index 27fb2a0c405242aae74c67724326a61ebc92089f..d513536617bd9ea5161f0bfa0681ab01817d588d 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -747,6 +747,9 @@ void __cfg80211_connect_result(struct net_device *dev, if (WARN_ON(!cr->links[link].addr)) goto out; } + + if (WARN_ON(wdev->connect_keys)) + goto out; } wdev->unprot_beacon_reported = 0; @@ -1325,7 +1328,7 @@ void __cfg80211_disconnected(struct net_device *dev, const u8 *ie, NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT)) max_key_idx = 7; for (i = 0; i <= max_key_idx; i++) - rdev_del_key(rdev, dev, i, false, NULL); + rdev_del_key(rdev, dev, -1, i, false, NULL); } rdev_set_qos_map(rdev, dev, NULL); diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 10b2fd9bacb55028b5f2652a1b0bee8beb146477..a405c3edbc47ec47f46d10f50c44959cc8ab759b 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -434,13 +434,14 @@ TRACE_EVENT(rdev_change_virtual_intf, ); DECLARE_EVENT_CLASS(key_handle, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr), - TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr), + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr), + TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY MAC_ENTRY(mac_addr) + __field(int, link_id) __field(u8, key_index) __field(bool, pairwise) ), @@ -448,34 +449,38 @@ DECLARE_EVENT_CLASS(key_handle, WIPHY_ASSIGN; NETDEV_ASSIGN; MAC_ASSIGN(mac_addr, mac_addr); + __entry->link_id = link_id; __entry->key_index = key_index; __entry->pairwise = pairwise; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key_index: %u, pairwise: %s, mac addr: " MAC_PR_FMT, - WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index, - BOOL_TO_STR(__entry->pairwise), MAC_PR_ARG(mac_addr)) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, " + "key_index: %u, pairwise: %s, mac addr: " MAC_PR_FMT, + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id, + __entry->key_index, BOOL_TO_STR(__entry->pairwise), + MAC_PR_ARG(mac_addr)) ); DEFINE_EVENT(key_handle, rdev_get_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr), - TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr) + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr), + TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr) ); DEFINE_EVENT(key_handle, rdev_del_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr), - TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr) + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr), + TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr) ); TRACE_EVENT(rdev_add_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool pairwise, const u8 *mac_addr, u8 mode), - TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr, mode), + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool pairwise, const u8 *mac_addr, u8 mode), + TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr, mode), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY MAC_ENTRY(mac_addr) + __field(int, link_id) __field(u8, key_index) __field(bool, pairwise) __field(u8, mode) @@ -484,24 +489,27 @@ TRACE_EVENT(rdev_add_key, WIPHY_ASSIGN; NETDEV_ASSIGN; MAC_ASSIGN(mac_addr, mac_addr); + __entry->link_id = link_id; __entry->key_index = key_index; __entry->pairwise = pairwise; __entry->mode = mode; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key_index: %u, " - "mode: %u, pairwise: %s, mac addr: " MAC_PR_FMT, - WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index, - __entry->mode, BOOL_TO_STR(__entry->pairwise), - MAC_PR_ARG(mac_addr)) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, " + "key_index: %u, mode: %u, pairwise: %s, " + "mac addr: " MAC_PR_FMT, + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id, + __entry->key_index, __entry->mode, + BOOL_TO_STR(__entry->pairwise), MAC_PR_ARG(mac_addr)) ); TRACE_EVENT(rdev_set_default_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index, - bool unicast, bool multicast), - TP_ARGS(wiphy, netdev, key_index, unicast, multicast), + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index, bool unicast, bool multicast), + TP_ARGS(wiphy, netdev, link_id, key_index, unicast, multicast), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY + __field(int, link_id) __field(u8, key_index) __field(bool, unicast) __field(bool, multicast) @@ -509,48 +517,58 @@ TRACE_EVENT(rdev_set_default_key, TP_fast_assign( WIPHY_ASSIGN; NETDEV_ASSIGN; + __entry->link_id = link_id; __entry->key_index = key_index; __entry->unicast = unicast; __entry->multicast = multicast; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u, unicast: %s, multicast: %s", - WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index, - BOOL_TO_STR(__entry->unicast), + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, " + "key index: %u, unicast: %s, multicast: %s", + WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id, + __entry->key_index, BOOL_TO_STR(__entry->unicast), BOOL_TO_STR(__entry->multicast)) ); TRACE_EVENT(rdev_set_default_mgmt_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index), - TP_ARGS(wiphy, netdev, key_index), + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index), + TP_ARGS(wiphy, netdev, link_id, key_index), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY + __field(int, link_id) __field(u8, key_index) ), TP_fast_assign( WIPHY_ASSIGN; NETDEV_ASSIGN; + __entry->link_id = link_id; __entry->key_index = key_index; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u", - WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, " + "key index: %u", WIPHY_PR_ARG, NETDEV_PR_ARG, + __entry->link_id, __entry->key_index) ); TRACE_EVENT(rdev_set_default_beacon_key, - TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index), - TP_ARGS(wiphy, netdev, key_index), + TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id, + u8 key_index), + TP_ARGS(wiphy, netdev, link_id, key_index), TP_STRUCT__entry( WIPHY_ENTRY NETDEV_ENTRY + __field(int, link_id) __field(u8, key_index) ), TP_fast_assign( WIPHY_ASSIGN; NETDEV_ASSIGN; + __entry->link_id = link_id; __entry->key_index = key_index; ), - TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u", - WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index) + TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, " + "key index: %u", WIPHY_PR_ARG, NETDEV_PR_ARG, + __entry->link_id, __entry->key_index) ); TRACE_EVENT(rdev_start_ap, @@ -3245,18 +3263,21 @@ TRACE_EVENT(cfg80211_ch_switch_notify, TRACE_EVENT(cfg80211_ch_switch_started_notify, TP_PROTO(struct net_device *netdev, - struct cfg80211_chan_def *chandef), - TP_ARGS(netdev, chandef), + struct cfg80211_chan_def *chandef, + unsigned int link_id), + TP_ARGS(netdev, chandef, link_id), TP_STRUCT__entry( NETDEV_ENTRY CHAN_DEF_ENTRY + __field(unsigned int, link_id) ), TP_fast_assign( NETDEV_ASSIGN; CHAN_DEF_ASSIGN(chandef); + __entry->link_id = link_id; ), - TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT, - NETDEV_PR_ARG, CHAN_DEF_PR_ARG) + TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT ", link:%d", + NETDEV_PR_ARG, CHAN_DEF_PR_ARG, __entry->link_id) ); TRACE_EVENT(cfg80211_radar_event, diff --git a/net/wireless/util.c b/net/wireless/util.c index 2c127951764a86b30b411f128ad399e9aded4459..1f285b51502866439da976109764fbe4362bee60 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -559,7 +559,7 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, return -1; hdrlen = ieee80211_hdrlen(hdr->frame_control) + data_offset; - if (skb->len < hdrlen + 8) + if (skb->len < hdrlen) return -1; /* convert IEEE 802.11 header + possible LLC headers into Ethernet @@ -574,8 +574,9 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, memcpy(tmp.h_dest, ieee80211_get_DA(hdr), ETH_ALEN); memcpy(tmp.h_source, ieee80211_get_SA(hdr), ETH_ALEN); - if (iftype == NL80211_IFTYPE_MESH_POINT) - skb_copy_bits(skb, hdrlen, &mesh_flags, 1); + if (iftype == NL80211_IFTYPE_MESH_POINT && + skb_copy_bits(skb, hdrlen, &mesh_flags, 1) < 0) + return -1; mesh_flags &= MESH_FLAGS_AE; @@ -595,11 +596,12 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, if (iftype == NL80211_IFTYPE_MESH_POINT) { if (mesh_flags == MESH_FLAGS_AE_A4) return -1; - if (mesh_flags == MESH_FLAGS_AE_A5_A6) { - skb_copy_bits(skb, hdrlen + - offsetof(struct ieee80211s_hdr, eaddr1), - tmp.h_dest, 2 * ETH_ALEN); - } + if (mesh_flags == MESH_FLAGS_AE_A5_A6 && + skb_copy_bits(skb, hdrlen + + offsetof(struct ieee80211s_hdr, eaddr1), + tmp.h_dest, 2 * ETH_ALEN) < 0) + return -1; + hdrlen += __ieee80211_get_mesh_hdrlen(mesh_flags); } break; @@ -613,10 +615,11 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, if (iftype == NL80211_IFTYPE_MESH_POINT) { if (mesh_flags == MESH_FLAGS_AE_A5_A6) return -1; - if (mesh_flags == MESH_FLAGS_AE_A4) - skb_copy_bits(skb, hdrlen + - offsetof(struct ieee80211s_hdr, eaddr1), - tmp.h_source, ETH_ALEN); + if (mesh_flags == MESH_FLAGS_AE_A4 && + skb_copy_bits(skb, hdrlen + + offsetof(struct ieee80211s_hdr, eaddr1), + tmp.h_source, ETH_ALEN) < 0) + return -1; hdrlen += __ieee80211_get_mesh_hdrlen(mesh_flags); } break; @@ -628,16 +631,15 @@ int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, break; } - skb_copy_bits(skb, hdrlen, &payload, sizeof(payload)); - tmp.h_proto = payload.proto; - - if (likely((!is_amsdu && ether_addr_equal(payload.hdr, rfc1042_header) && - tmp.h_proto != htons(ETH_P_AARP) && - tmp.h_proto != htons(ETH_P_IPX)) || - ether_addr_equal(payload.hdr, bridge_tunnel_header))) { + if (likely(skb_copy_bits(skb, hdrlen, &payload, sizeof(payload)) == 0 && + ((!is_amsdu && ether_addr_equal(payload.hdr, rfc1042_header) && + payload.proto != htons(ETH_P_AARP) && + payload.proto != htons(ETH_P_IPX)) || + ether_addr_equal(payload.hdr, bridge_tunnel_header)))) { /* remove RFC1042 or Bridge-Tunnel encapsulation and * replace EtherType */ hdrlen += ETH_ALEN + 2; + tmp.h_proto = payload.proto; skb_postpull_rcsum(skb, &payload, ETH_ALEN + 2); } else { tmp.h_proto = htons(skb->len - hdrlen); @@ -935,13 +937,13 @@ void cfg80211_upload_connect_keys(struct wireless_dev *wdev) for (i = 0; i < CFG80211_MAX_WEP_KEYS; i++) { if (!wdev->connect_keys->params[i].cipher) continue; - if (rdev_add_key(rdev, dev, i, false, NULL, + if (rdev_add_key(rdev, dev, -1, i, false, NULL, &wdev->connect_keys->params[i])) { netdev_err(dev, "failed to set key %d\n", i); continue; } if (wdev->connect_keys->def == i && - rdev_set_default_key(rdev, dev, i, true, true)) { + rdev_set_default_key(rdev, dev, -1, i, true, true)) { netdev_err(dev, "failed to set defkey %d\n", i); continue; } @@ -1361,7 +1363,7 @@ static u32 cfg80211_calculate_bitrate_he(struct rate_info *rate) 25599, /* 4.166666... */ 17067, /* 2.777777... */ 12801, /* 2.083333... */ - 11769, /* 1.851851... */ + 11377, /* 1.851725... */ 10239, /* 1.666666... */ 8532, /* 1.388888... */ 7680, /* 1.250000... */ @@ -1444,7 +1446,7 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) 25599, /* 4.166666... */ 17067, /* 2.777777... */ 12801, /* 2.083333... */ - 11769, /* 1.851851... */ + 11377, /* 1.851725... */ 10239, /* 1.666666... */ 8532, /* 1.388888... */ 7680, /* 1.250000... */ diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c index a9767bfe73300701c041dd11b983de2d4918cf5b..ddf340bfa07ac9e251c14a9c68c3d36234bdfbed 100644 --- a/net/wireless/wext-compat.c +++ b/net/wireless/wext-compat.c @@ -470,7 +470,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, !(rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN)) err = -ENOENT; else - err = rdev_del_key(rdev, dev, idx, pairwise, + err = rdev_del_key(rdev, dev, -1, idx, pairwise, addr); } wdev->wext.connect.privacy = false; @@ -509,7 +509,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, if (wdev->connected || (wdev->iftype == NL80211_IFTYPE_ADHOC && wdev->u.ibss.current_bss)) - err = rdev_add_key(rdev, dev, idx, pairwise, addr, params); + err = rdev_add_key(rdev, dev, -1, idx, pairwise, addr, params); else if (params->cipher != WLAN_CIPHER_SUITE_WEP40 && params->cipher != WLAN_CIPHER_SUITE_WEP104) return -EINVAL; @@ -546,7 +546,8 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, __cfg80211_leave_ibss(rdev, wdev->netdev, true); rejoin = true; } - err = rdev_set_default_key(rdev, dev, idx, true, true); + err = rdev_set_default_key(rdev, dev, -1, idx, true, + true); } if (!err) { wdev->wext.default_key = idx; @@ -561,7 +562,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev, if (wdev->connected || (wdev->iftype == NL80211_IFTYPE_ADHOC && wdev->u.ibss.current_bss)) - err = rdev_set_default_mgmt_key(rdev, dev, idx); + err = rdev_set_default_mgmt_key(rdev, dev, -1, idx); if (!err) wdev->wext.default_mgmt_key = idx; return err; @@ -632,7 +633,7 @@ static int cfg80211_wext_siwencode(struct net_device *dev, if (wdev->connected || (wdev->iftype == NL80211_IFTYPE_ADHOC && wdev->u.ibss.current_bss)) - err = rdev_set_default_key(rdev, dev, idx, true, + err = rdev_set_default_key(rdev, dev, -1, idx, true, true); if (!err) wdev->wext.default_key = idx; @@ -685,6 +686,13 @@ static int cfg80211_wext_siwencodeext(struct net_device *dev, !rdev->ops->set_default_key) return -EOPNOTSUPP; + wdev_lock(wdev); + if (wdev->valid_links) { + wdev_unlock(wdev); + return -EOPNOTSUPP; + } + wdev_unlock(wdev); + switch (ext->alg) { case IW_ENCODE_ALG_NONE: remove = true; diff --git a/net/wireless/wext-core.c b/net/wireless/wext-core.c index 76a80a41615befda05f5e62a8b5f2ba1a2253d04..fe8765c4075d370598ed164e4c1896c2bfc37676 100644 --- a/net/wireless/wext-core.c +++ b/net/wireless/wext-core.c @@ -468,6 +468,7 @@ void wireless_send_event(struct net_device * dev, struct __compat_iw_event *compat_event; struct compat_iw_point compat_wrqu; struct sk_buff *compskb; + int ptr_len; #endif /* @@ -582,6 +583,9 @@ void wireless_send_event(struct net_device * dev, nlmsg_end(skb, nlh); #ifdef CONFIG_COMPAT hdr_len = compat_event_type_size[descr->header_type]; + + /* ptr_len is remaining size in event header apart from LCP */ + ptr_len = hdr_len - IW_EV_COMPAT_LCP_LEN; event_len = hdr_len + extra_len; compskb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC); @@ -612,16 +616,15 @@ void wireless_send_event(struct net_device * dev, if (descr->header_type == IW_HEADER_TYPE_POINT) { compat_wrqu.length = wrqu->data.length; compat_wrqu.flags = wrqu->data.flags; - memcpy(&compat_event->pointer, - ((char *) &compat_wrqu) + IW_EV_COMPAT_POINT_OFF, - hdr_len - IW_EV_COMPAT_LCP_LEN); + memcpy(compat_event->ptr_bytes, + ((char *)&compat_wrqu) + IW_EV_COMPAT_POINT_OFF, + ptr_len); if (extra_len) - memcpy(((char *) compat_event) + hdr_len, - extra, extra_len); + memcpy(&compat_event->ptr_bytes[ptr_len], + extra, extra_len); } else { /* extra_len must be zero, so no if (extra) needed */ - memcpy(&compat_event->pointer, wrqu, - hdr_len - IW_EV_COMPAT_LCP_LEN); + memcpy(compat_event->ptr_bytes, wrqu, ptr_len); } nlmsg_end(compskb, nlh); diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 869b9b9b9fad6268f11e4b8b7f2cf18ec19c7feb..4681e8e8ad943605b61aa6be685e6c76a70b4c0f 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -19,8 +19,6 @@ #include "xdp_umem.h" #include "xsk_queue.h" -#define XDP_UMEM_MIN_CHUNK_SIZE 2048 - static DEFINE_IDA(umem_ida); static void xdp_umem_unpin_pages(struct xdp_umem *umem) diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 5b4ce6ba1bc7fc8aeddd33a394db6d10e1e6e95f..9f0561b67c12e9079338ed44fa6442c608792950 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -355,16 +355,15 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entr return nb_pkts; } -u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries) +u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts) { struct xdp_sock *xs; - u32 nb_pkts; rcu_read_lock(); if (!list_is_singular(&pool->xsk_tx_list)) { /* Fallback to the non-batched version */ rcu_read_unlock(); - return xsk_tx_peek_release_fallback(pool, max_entries); + return xsk_tx_peek_release_fallback(pool, nb_pkts); } xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list); @@ -373,12 +372,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries) goto out; } - max_entries = xskq_cons_nb_entries(xs->tx, max_entries); - nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, max_entries); - if (!nb_pkts) { - xs->tx->queue_empty_descs++; - goto out; - } + nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts); /* This is the backpressure mechanism for the Tx path. Try to * reserve space in the completion queue for all packets, but @@ -386,12 +380,18 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries) * packets. This avoids having to implement any buffering in * the Tx path. */ - nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts); + nb_pkts = xskq_prod_nb_free(pool->cq, nb_pkts); if (!nb_pkts) goto out; - xskq_cons_release_n(xs->tx, max_entries); + nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts); + if (!nb_pkts) { + xs->tx->queue_empty_descs++; + goto out; + } + __xskq_cons_release(xs->tx); + xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts); xs->sk.sk_write_space(&xs->sk); out: @@ -954,8 +954,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len) goto out_unlock; } - err = xp_assign_dev_shared(xs->pool, umem_xs->umem, - dev, qid); + err = xp_assign_dev_shared(xs->pool, umem_xs, dev, + qid); if (err) { xp_destroy(xs->pool); xs->pool = NULL; diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index f70112176b7c1f59c2210b27d02ec74edc51a5e0..ed6c71826d31f9212cc1ca3eb65afa58e69e4759 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -212,17 +212,18 @@ err_unreg_pool: return err; } -int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem, +int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, struct net_device *dev, u16 queue_id) { u16 flags; + struct xdp_umem *umem = umem_xs->umem; /* One fill and completion ring required for each queue id. */ if (!pool->fq || !pool->cq) return -EINVAL; flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY; - if (pool->uses_need_wakeup) + if (umem_xs->pool->uses_need_wakeup) flags |= XDP_USE_NEED_WAKEUP; return xp_assign_dev(pool, dev, queue_id, flags); @@ -379,6 +380,16 @@ static void xp_check_dma_contiguity(struct xsk_dma_map *dma_map) static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map) { + if (!pool->unaligned) { + u32 i; + + for (i = 0; i < pool->heads_cnt; i++) { + struct xdp_buff_xsk *xskb = &pool->heads[i]; + + xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, xskb->orig_addr); + } + } + pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL); if (!pool->dma_pages) return -ENOMEM; @@ -428,12 +439,6 @@ int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev, if (pool->unaligned) xp_check_dma_contiguity(dma_map); - else - for (i = 0; i < pool->heads_cnt; i++) { - struct xdp_buff_xsk *xskb = &pool->heads[i]; - - xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, xskb->orig_addr); - } err = xp_init_dma_info(pool, dma_map); if (err) { diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index fb20bf7207cfb5bb90743b4a19b3c56812f29d10..c6fb6b763658255873e1263337b09c980ce88079 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -205,6 +205,11 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q, return false; } +static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt) +{ + q->cached_cons += cnt; +} + static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, u32 max) { @@ -226,6 +231,8 @@ static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff cached_cons++; } + /* Release valid plus any invalid entries */ + xskq_cons_release_n(q, cached_cons - q->cached_cons); return nb_entries; } @@ -291,11 +298,6 @@ static inline void xskq_cons_release(struct xsk_queue *q) q->cached_cons++; } -static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt) -{ - q->cached_cons += cnt; -} - static inline u32 xskq_cons_present_entries(struct xsk_queue *q) { /* No barriers needed since data is not accessed */ @@ -350,21 +352,17 @@ static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr) return 0; } -static inline u32 xskq_prod_reserve_addr_batch(struct xsk_queue *q, struct xdp_desc *descs, - u32 max) +static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_desc *descs, + u32 nb_entries) { struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring; - u32 nb_entries, i, cached_prod; - - nb_entries = xskq_prod_nb_free(q, max); + u32 i, cached_prod; /* A, matches D */ cached_prod = q->cached_prod; for (i = 0; i < nb_entries; i++) ring->desc[cached_prod++ & q->ring_mask] = descs[i].addr; q->cached_prod = cached_prod; - - return nb_entries; } static inline int xskq_prod_reserve_desc(struct xsk_queue *q, diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index 82d14eea1b5ad06a481137867ca070a465463b9a..29a540dcb5a71030a1d65e830e8a48ba3dc630e1 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -91,7 +91,7 @@ static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb) } /* remove header, leave non-ESP marker/SPI */ - if (!__pskb_pull(skb, rxm->offset + 2)) { + if (!pskb_pull(skb, rxm->offset + 2)) { XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR); kfree_skb(skb); return; @@ -168,7 +168,7 @@ int espintcp_queue_out(struct sock *sk, struct sk_buff *skb) { struct espintcp_ctx *ctx = espintcp_getctx(sk); - if (skb_queue_len(&ctx->out_queue) >= netdev_max_backlog) + if (skb_queue_len(&ctx->out_queue) >= READ_ONCE(netdev_max_backlog)) return -ENOBUFS; __skb_queue_tail(&ctx->out_queue, skb); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 637ca8838436861cbaf093cfd3914394d26e14d6..5f5aafd418af0e2537792dc642813f0dbed64c88 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -207,7 +207,8 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur EXPORT_SYMBOL_GPL(validate_xmit_xfrm); int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, - struct xfrm_user_offload *xuo) + struct xfrm_user_offload *xuo, + struct netlink_ext_ack *extack) { int err; struct dst_entry *dst; @@ -216,15 +217,21 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, xfrm_address_t *saddr; xfrm_address_t *daddr; - if (!x->type_offload) + if (!x->type_offload) { + NL_SET_ERR_MSG(extack, "Type doesn't support offload"); return -EINVAL; + } /* We don't yet support UDP encapsulation and TFC padding. */ - if (x->encap || x->tfcpad) + if (x->encap || x->tfcpad) { + NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded"); return -EINVAL; + } - if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) + if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) { + NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request"); return -EINVAL; + } dev = dev_get_by_index(net, xuo->ifindex); if (!dev) { @@ -256,6 +263,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, if (x->props.flags & XFRM_STATE_ESN && !dev->xfrmdev_ops->xdo_dev_state_advance_esn) { + NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN"); xso->dev = NULL; dev_put(dev); return -EINVAL; @@ -277,8 +285,10 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, xso->real_dev = NULL; netdev_put(dev, &xso->dev_tracker); - if (err != -EOPNOTSUPP) + if (err != -EOPNOTSUPP) { + NL_SET_ERR_MSG(extack, "Device failed to offload this state"); return err; + } } return 0; diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c index 144238a50f3d4e1efc99eb8e893650e60f1b3bda..97074f6f2bdee947b7691be123389d68eca9d828 100644 --- a/net/xfrm/xfrm_input.c +++ b/net/xfrm/xfrm_input.c @@ -20,11 +20,13 @@ #include #include #include +#include #include "xfrm_inout.h" struct xfrm_trans_tasklet { - struct tasklet_struct tasklet; + struct work_struct work; + spinlock_t queue_lock; struct sk_buff_head queue; }; @@ -669,7 +671,6 @@ resume: x->curlft.bytes += skb->len; x->curlft.packets++; - x->curlft.use_time = ktime_get_real_seconds(); spin_unlock(&x->lock); @@ -720,7 +721,8 @@ resume: sp = skb_sec_path(skb); if (sp) sp->olen = 0; - skb_dst_drop(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); gro_cells_receive(&gro_cells, skb); return 0; } else { @@ -738,7 +740,8 @@ resume: sp = skb_sec_path(skb); if (sp) sp->olen = 0; - skb_dst_drop(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); gro_cells_receive(&gro_cells, skb); return err; } @@ -761,18 +764,22 @@ int xfrm_input_resume(struct sk_buff *skb, int nexthdr) } EXPORT_SYMBOL(xfrm_input_resume); -static void xfrm_trans_reinject(struct tasklet_struct *t) +static void xfrm_trans_reinject(struct work_struct *work) { - struct xfrm_trans_tasklet *trans = from_tasklet(trans, t, tasklet); + struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work); struct sk_buff_head queue; struct sk_buff *skb; __skb_queue_head_init(&queue); + spin_lock_bh(&trans->queue_lock); skb_queue_splice_init(&trans->queue, &queue); + spin_unlock_bh(&trans->queue_lock); + local_bh_disable(); while ((skb = __skb_dequeue(&queue))) XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net, NULL, skb); + local_bh_enable(); } int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, @@ -783,15 +790,17 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, trans = this_cpu_ptr(&xfrm_trans_tasklet); - if (skb_queue_len(&trans->queue) >= netdev_max_backlog) + if (skb_queue_len(&trans->queue) >= READ_ONCE(netdev_max_backlog)) return -ENOBUFS; BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb)); XFRM_TRANS_SKB_CB(skb)->finish = finish; XFRM_TRANS_SKB_CB(skb)->net = net; + spin_lock_bh(&trans->queue_lock); __skb_queue_tail(&trans->queue, skb); - tasklet_schedule(&trans->tasklet); + spin_unlock_bh(&trans->queue_lock); + schedule_work(&trans->work); return 0; } EXPORT_SYMBOL(xfrm_trans_queue_net); @@ -818,7 +827,8 @@ void __init xfrm_input_init(void) struct xfrm_trans_tasklet *trans; trans = &per_cpu(xfrm_trans_tasklet, i); + spin_lock_init(&trans->queue_lock); __skb_queue_head_init(&trans->queue); - tasklet_setup(&trans->tasklet, xfrm_trans_reinject); + INIT_WORK(&trans->work, xfrm_trans_reinject); } } diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c index 5113fa0fbceed12cb74649840adb13dba70d44ff..5a67b120c4dbd4754b2842bf8e845fa675823810 100644 --- a/net/xfrm/xfrm_interface.c +++ b/net/xfrm/xfrm_interface.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -56,6 +57,89 @@ static const struct net_device_ops xfrmi_netdev_ops; struct xfrmi_net { /* lists for storing interfaces in use */ struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE]; + struct xfrm_if __rcu *collect_md_xfrmi; +}; + +static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = { + [LWT_XFRM_IF_ID] = NLA_POLICY_MIN(NLA_U32, 1), + [LWT_XFRM_LINK] = NLA_POLICY_MIN(NLA_U32, 1), +}; + +static void xfrmi_destroy_state(struct lwtunnel_state *lwt) +{ +} + +static int xfrmi_build_state(struct net *net, struct nlattr *nla, + unsigned int family, const void *cfg, + struct lwtunnel_state **ts, + struct netlink_ext_ack *extack) +{ + struct nlattr *tb[LWT_XFRM_MAX + 1]; + struct lwtunnel_state *new_state; + struct xfrm_md_info *info; + int ret; + + ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack); + if (ret < 0) + return ret; + + if (!tb[LWT_XFRM_IF_ID]) { + NL_SET_ERR_MSG(extack, "if_id must be set"); + return -EINVAL; + } + + new_state = lwtunnel_state_alloc(sizeof(*info)); + if (!new_state) { + NL_SET_ERR_MSG(extack, "failed to create encap info"); + return -ENOMEM; + } + + new_state->type = LWTUNNEL_ENCAP_XFRM; + + info = lwt_xfrm_info(new_state); + + info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]); + + if (tb[LWT_XFRM_LINK]) + info->link = nla_get_u32(tb[LWT_XFRM_LINK]); + + *ts = new_state; + return 0; +} + +static int xfrmi_fill_encap_info(struct sk_buff *skb, + struct lwtunnel_state *lwt) +{ + struct xfrm_md_info *info = lwt_xfrm_info(lwt); + + if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) || + (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link))) + return -EMSGSIZE; + + return 0; +} + +static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate) +{ + return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */ + nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */ +} + +static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b) +{ + struct xfrm_md_info *a_info = lwt_xfrm_info(a); + struct xfrm_md_info *b_info = lwt_xfrm_info(b); + + return memcmp(a_info, b_info, sizeof(*a_info)); +} + +static const struct lwtunnel_encap_ops xfrmi_encap_ops = { + .build_state = xfrmi_build_state, + .destroy_state = xfrmi_destroy_state, + .fill_encap = xfrmi_fill_encap_info, + .get_encap_size = xfrmi_encap_nlsize, + .cmp_encap = xfrmi_encap_cmp, + .owner = THIS_MODULE, }; #define for_each_xfrmi_rcu(start, xi) \ @@ -77,17 +161,23 @@ static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x) return xi; } + xi = rcu_dereference(xfrmn->collect_md_xfrmi); + if (xi && (xi->dev->flags & IFF_UP)) + return xi; + return NULL; } -static struct xfrm_if *xfrmi_decode_session(struct sk_buff *skb, - unsigned short family) +static bool xfrmi_decode_session(struct sk_buff *skb, + unsigned short family, + struct xfrm_if_decode_session_result *res) { struct net_device *dev; + struct xfrm_if *xi; int ifindex = 0; if (!secpath_exists(skb) || !skb->dev) - return NULL; + return false; switch (family) { case AF_INET6: @@ -107,11 +197,18 @@ static struct xfrm_if *xfrmi_decode_session(struct sk_buff *skb, } if (!dev || !(dev->flags & IFF_UP)) - return NULL; + return false; if (dev->netdev_ops != &xfrmi_netdev_ops) - return NULL; + return false; - return netdev_priv(dev); + xi = netdev_priv(dev); + res->net = xi->net; + + if (xi->p.collect_md) + res->if_id = xfrm_input_state(skb)->if_id; + else + res->if_id = xi->p.if_id; + return true; } static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi) @@ -157,7 +254,10 @@ static int xfrmi_create(struct net_device *dev) if (err < 0) goto out; - xfrmi_link(xfrmn, xi); + if (xi->p.collect_md) + rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi); + else + xfrmi_link(xfrmn, xi); return 0; @@ -185,7 +285,10 @@ static void xfrmi_dev_uninit(struct net_device *dev) struct xfrm_if *xi = netdev_priv(dev); struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id); - xfrmi_unlink(xfrmn, xi); + if (xi->p.collect_md) + RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL); + else + xfrmi_unlink(xfrmn, xi); } static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet) @@ -214,6 +317,7 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err) struct xfrm_state *x; struct xfrm_if *xi; bool xnet; + int link; if (err && !secpath_exists(skb)) return 0; @@ -224,6 +328,7 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err) if (!xi) return 1; + link = skb->dev->ifindex; dev = xi->dev; skb->dev = dev; @@ -254,6 +359,17 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err) } xfrmi_scrub_packet(skb, xnet); + if (xi->p.collect_md) { + struct metadata_dst *md_dst; + + md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC); + if (!md_dst) + return -ENOMEM; + + md_dst->u.xfrm_info.if_id = x->if_id; + md_dst->u.xfrm_info.link = link; + skb_dst_set(skb, (struct dst_entry *)md_dst); + } dev_sw_netstats_rx_add(dev, skb->len); return 0; @@ -269,10 +385,23 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) struct net_device *tdev; struct xfrm_state *x; int err = -1; + u32 if_id; int mtu; + if (xi->p.collect_md) { + struct xfrm_md_info *md_info = skb_xfrm_md_info(skb); + + if (unlikely(!md_info)) + return -EINVAL; + + if_id = md_info->if_id; + fl->flowi_oif = md_info->link; + } else { + if_id = xi->p.if_id; + } + dst_hold(dst); - dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, xi->p.if_id); + dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id); if (IS_ERR(dst)) { err = PTR_ERR(dst); dst = NULL; @@ -283,7 +412,7 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl) if (!x) goto tx_err_link_failure; - if (x->if_id != xi->p.if_id) + if (x->if_id != if_id) goto tx_err_link_failure; tdev = dst->dev; @@ -633,6 +762,9 @@ static void xfrmi_netlink_parms(struct nlattr *data[], if (data[IFLA_XFRM_IF_ID]) parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]); + + if (data[IFLA_XFRM_COLLECT_METADATA]) + parms->collect_md = true; } static int xfrmi_newlink(struct net *src_net, struct net_device *dev, @@ -645,14 +777,27 @@ static int xfrmi_newlink(struct net *src_net, struct net_device *dev, int err; xfrmi_netlink_parms(data, &p); - if (!p.if_id) { - NL_SET_ERR_MSG(extack, "if_id must be non zero"); - return -EINVAL; - } + if (p.collect_md) { + struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id); - xi = xfrmi_locate(net, &p); - if (xi) - return -EEXIST; + if (p.link || p.if_id) { + NL_SET_ERR_MSG(extack, "link and if_id must be zero"); + return -EINVAL; + } + + if (rtnl_dereference(xfrmn->collect_md_xfrmi)) + return -EEXIST; + + } else { + if (!p.if_id) { + NL_SET_ERR_MSG(extack, "if_id must be non zero"); + return -EINVAL; + } + + xi = xfrmi_locate(net, &p); + if (xi) + return -EEXIST; + } xi = netdev_priv(dev); xi->p = p; @@ -682,12 +827,22 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[], return -EINVAL; } + if (p.collect_md) { + NL_SET_ERR_MSG(extack, "collect_md can't be changed"); + return -EINVAL; + } + xi = xfrmi_locate(net, &p); if (!xi) { xi = netdev_priv(dev); } else { if (xi->dev != dev) return -EEXIST; + if (xi->p.collect_md) { + NL_SET_ERR_MSG(extack, + "device can't be changed to collect_md"); + return -EINVAL; + } } return xfrmi_update(xi, &p); @@ -700,6 +855,8 @@ static size_t xfrmi_get_size(const struct net_device *dev) nla_total_size(4) + /* IFLA_XFRM_IF_ID */ nla_total_size(4) + + /* IFLA_XFRM_COLLECT_METADATA */ + nla_total_size(0) + 0; } @@ -709,7 +866,8 @@ static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev) struct xfrm_if_parms *parm = &xi->p; if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) || - nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id)) + nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) || + (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA))) goto nla_put_failure; return 0; @@ -725,8 +883,10 @@ static struct net *xfrmi_get_link_net(const struct net_device *dev) } static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = { - [IFLA_XFRM_LINK] = { .type = NLA_U32 }, - [IFLA_XFRM_IF_ID] = { .type = NLA_U32 }, + [IFLA_XFRM_UNSPEC] = { .strict_start_type = IFLA_XFRM_COLLECT_METADATA }, + [IFLA_XFRM_LINK] = { .type = NLA_U32 }, + [IFLA_XFRM_IF_ID] = { .type = NLA_U32 }, + [IFLA_XFRM_COLLECT_METADATA] = { .type = NLA_FLAG }, }; static struct rtnl_link_ops xfrmi_link_ops __read_mostly = { @@ -762,6 +922,9 @@ static void __net_exit xfrmi_exit_batch_net(struct list_head *net_exit_list) xip = &xi->next) unregister_netdevice_queue(xi->dev, &list); } + xi = rtnl_dereference(xfrmn->collect_md_xfrmi); + if (xi) + unregister_netdevice_queue(xi->dev, &list); } unregister_netdevice_many(&list); rtnl_unlock(); @@ -999,6 +1162,8 @@ static int __init xfrmi_init(void) if (err < 0) goto rtnl_link_failed; + lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); + xfrm_if_register_cb(&xfrm_if_cb); return err; @@ -1017,6 +1182,7 @@ pernet_dev_failed: static void __exit xfrmi_fini(void) { xfrm_if_unregister_cb(); + lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM); rtnl_link_unregister(&xfrmi_link_ops); xfrmi4_fini(); xfrmi6_fini(); diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c index cb40ff0ff28da274222b6a6b4971dec354c147e2..80143360bf0955420b97ebe3f7538c1fda14b217 100644 --- a/net/xfrm/xfrm_ipcomp.c +++ b/net/xfrm/xfrm_ipcomp.c @@ -203,6 +203,7 @@ static void ipcomp_free_scratches(void) vfree(*per_cpu_ptr(scratches, i)); free_percpu(scratches); + ipcomp_scratches = NULL; } static void * __percpu *ipcomp_alloc_scratches(void) @@ -325,18 +326,22 @@ void ipcomp_destroy(struct xfrm_state *x) } EXPORT_SYMBOL_GPL(ipcomp_destroy); -int ipcomp_init_state(struct xfrm_state *x) +int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) { int err; struct ipcomp_data *ipcd; struct xfrm_algo_desc *calg_desc; err = -EINVAL; - if (!x->calg) + if (!x->calg) { + NL_SET_ERR_MSG(extack, "Missing required compression algorithm"); goto out; + } - if (x->encap) + if (x->encap) { + NL_SET_ERR_MSG(extack, "IPComp is not compatible with encapsulation"); goto out; + } err = -ENOMEM; ipcd = kzalloc(sizeof(*ipcd), GFP_KERNEL); diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index 555ab35cd119a71ba5828e90c3fc297383194d6e..9a5e79a38c6797e86648178090601471c2e68584 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -534,7 +534,6 @@ static int xfrm_output_one(struct sk_buff *skb, int err) x->curlft.bytes += skb->len; x->curlft.packets++; - x->curlft.use_time = ktime_get_real_seconds(); spin_unlock_bh(&x->lock); diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index f1a0bab920a5580a507932974a86f96c08113759..e392d8d05e0ca23aa57097c6ede9dfde69f8cc5a 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -1889,7 +1889,7 @@ EXPORT_SYMBOL(xfrm_policy_walk_done); */ static int xfrm_policy_match(const struct xfrm_policy *pol, const struct flowi *fl, - u8 type, u16 family, int dir, u32 if_id) + u8 type, u16 family, u32 if_id) { const struct xfrm_selector *sel = &pol->selector; int ret = -ESRCH; @@ -2014,7 +2014,7 @@ static struct xfrm_policy * __xfrm_policy_eval_candidates(struct hlist_head *chain, struct xfrm_policy *prefer, const struct flowi *fl, - u8 type, u16 family, int dir, u32 if_id) + u8 type, u16 family, u32 if_id) { u32 priority = prefer ? prefer->priority : ~0u; struct xfrm_policy *pol; @@ -2028,7 +2028,7 @@ __xfrm_policy_eval_candidates(struct hlist_head *chain, if (pol->priority > priority) break; - err = xfrm_policy_match(pol, fl, type, family, dir, if_id); + err = xfrm_policy_match(pol, fl, type, family, if_id); if (err) { if (err != -ESRCH) return ERR_PTR(err); @@ -2053,7 +2053,7 @@ static struct xfrm_policy * xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand, struct xfrm_policy *prefer, const struct flowi *fl, - u8 type, u16 family, int dir, u32 if_id) + u8 type, u16 family, u32 if_id) { struct xfrm_policy *tmp; int i; @@ -2061,8 +2061,7 @@ xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand, for (i = 0; i < ARRAY_SIZE(cand->res); i++) { tmp = __xfrm_policy_eval_candidates(cand->res[i], prefer, - fl, type, family, dir, - if_id); + fl, type, family, if_id); if (!tmp) continue; @@ -2101,7 +2100,7 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, ret = NULL; hlist_for_each_entry_rcu(pol, chain, bydst) { - err = xfrm_policy_match(pol, fl, type, family, dir, if_id); + err = xfrm_policy_match(pol, fl, type, family, if_id); if (err) { if (err == -ESRCH) continue; @@ -2120,7 +2119,7 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type, goto skip_inexact; pol = xfrm_policy_eval_candidates(&cand, ret, fl, type, - family, dir, if_id); + family, if_id); if (pol) { ret = pol; if (IS_ERR(pol)) @@ -3162,7 +3161,7 @@ ok: return dst; nopol: - if (!(dst_orig->dev->flags & IFF_LOOPBACK) && + if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) && net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) { err = -EPERM; goto error; @@ -3516,17 +3515,17 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, int xerr_idx = -1; const struct xfrm_if_cb *ifcb; struct sec_path *sp; - struct xfrm_if *xi; u32 if_id = 0; rcu_read_lock(); ifcb = xfrm_if_get_cb(); if (ifcb) { - xi = ifcb->decode_session(skb, family); - if (xi) { - if_id = xi->p.if_id; - net = xi->net; + struct xfrm_if_decode_session_result r; + + if (ifcb->decode_session(skb, family, &r)) { + if_id = r.if_id; + net = r.net; } } rcu_read_unlock(); @@ -3599,6 +3598,7 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb, if (pols[1]) { if (IS_ERR(pols[1])) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR); + xfrm_pol_put(pols[0]); return 0; } pols[1]->curlft.use_time = ktime_get_real_seconds(); diff --git a/net/xfrm/xfrm_replay.c b/net/xfrm/xfrm_replay.c index 9277d81b344cbb0b7910b2189a4aa1f7037384ec..9f4d42eb090f154457902e86f85ac1b269c76625 100644 --- a/net/xfrm/xfrm_replay.c +++ b/net/xfrm/xfrm_replay.c @@ -766,18 +766,22 @@ int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb) } #endif -int xfrm_init_replay(struct xfrm_state *x) +int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack) { struct xfrm_replay_state_esn *replay_esn = x->replay_esn; if (replay_esn) { if (replay_esn->replay_window > - replay_esn->bmp_len * sizeof(__u32) * 8) + replay_esn->bmp_len * sizeof(__u32) * 8) { + NL_SET_ERR_MSG(extack, "ESN replay window is too large for the chosen bitmap size"); return -EINVAL; + } if (x->props.flags & XFRM_STATE_ESN) { - if (replay_esn->replay_window == 0) + if (replay_esn->replay_window == 0) { + NL_SET_ERR_MSG(extack, "ESN replay window must be > 0"); return -EINVAL; + } x->repl_mode = XFRM_REPLAY_MODE_ESN; } else { x->repl_mode = XFRM_REPLAY_MODE_BMP; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 52e60e607f8ad5ff446ab4cc6ccba9d14acb43c7..3d2fe7712ac5b4191830d9eb272f55b4e035b949 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1592,6 +1592,7 @@ static struct xfrm_state *xfrm_state_clone(struct xfrm_state *orig, x->replay = orig->replay; x->preplay = orig->preplay; x->mapping_maxage = orig->mapping_maxage; + x->lastused = orig->lastused; x->new_mapping = 0; x->new_mapping_sport = 0; @@ -2071,7 +2072,7 @@ int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high) } else { u32 spi = 0; for (h = 0; h < high-low+1; h++) { - spi = low + prandom_u32()%(high-low+1); + spi = low + prandom_u32_max(high - low + 1); x0 = xfrm_state_lookup(net, mark, &x->id.daddr, htonl(spi), x->id.proto, x->props.family); if (x0 == NULL) { newspi = htonl(spi); @@ -2610,7 +2611,8 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu) } EXPORT_SYMBOL_GPL(xfrm_state_mtu); -int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) +int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload, + struct netlink_ext_ack *extack) { const struct xfrm_mode *inner_mode; const struct xfrm_mode *outer_mode; @@ -2625,12 +2627,16 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) if (x->sel.family != AF_UNSPEC) { inner_mode = xfrm_get_mode(x->props.mode, x->sel.family); - if (inner_mode == NULL) + if (inner_mode == NULL) { + NL_SET_ERR_MSG(extack, "Requested mode not found"); goto error; + } if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) && - family != x->sel.family) + family != x->sel.family) { + NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family"); goto error; + } x->inner_mode = *inner_mode; } else { @@ -2638,11 +2644,15 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) int iafamily = AF_INET; inner_mode = xfrm_get_mode(x->props.mode, x->props.family); - if (inner_mode == NULL) + if (inner_mode == NULL) { + NL_SET_ERR_MSG(extack, "Requested mode not found"); goto error; + } - if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) + if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) { + NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate an AF_UNSPEC selector"); goto error; + } x->inner_mode = *inner_mode; @@ -2657,24 +2667,27 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload) } x->type = xfrm_get_type(x->id.proto, family); - if (x->type == NULL) + if (x->type == NULL) { + NL_SET_ERR_MSG(extack, "Requested type not found"); goto error; + } x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload); - err = x->type->init_state(x); + err = x->type->init_state(x, extack); if (err) goto error; outer_mode = xfrm_get_mode(x->props.mode, family); if (!outer_mode) { + NL_SET_ERR_MSG(extack, "Requested mode not found"); err = -EPROTONOSUPPORT; goto error; } x->outer_mode = *outer_mode; if (init_replay) { - err = xfrm_init_replay(x); + err = xfrm_init_replay(x, extack); if (err) goto error; } @@ -2689,7 +2702,7 @@ int xfrm_init_state(struct xfrm_state *x) { int err; - err = __xfrm_init_state(x, true, false); + err = __xfrm_init_state(x, true, false, NULL); if (!err) x->km.state = XFRM_STATE_VALID; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 2ff017117730322301d11ccef7d1959706ab3c68..e73f9efc54c1272dc4da0fb4413bb154276de907 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -35,7 +35,8 @@ #endif #include -static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type) +static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type, + struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[type]; struct xfrm_algo *algp; @@ -44,8 +45,10 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type) return 0; algp = nla_data(rt); - if (nla_len(rt) < (int)xfrm_alg_len(algp)) + if (nla_len(rt) < (int)xfrm_alg_len(algp)) { + NL_SET_ERR_MSG(extack, "Invalid AUTH/CRYPT/COMP attribute length"); return -EINVAL; + } switch (type) { case XFRMA_ALG_AUTH: @@ -54,6 +57,7 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type) break; default: + NL_SET_ERR_MSG(extack, "Invalid algorithm attribute type"); return -EINVAL; } @@ -61,7 +65,8 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type) return 0; } -static int verify_auth_trunc(struct nlattr **attrs) +static int verify_auth_trunc(struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_ALG_AUTH_TRUNC]; struct xfrm_algo_auth *algp; @@ -70,14 +75,16 @@ static int verify_auth_trunc(struct nlattr **attrs) return 0; algp = nla_data(rt); - if (nla_len(rt) < (int)xfrm_alg_auth_len(algp)) + if (nla_len(rt) < (int)xfrm_alg_auth_len(algp)) { + NL_SET_ERR_MSG(extack, "Invalid AUTH_TRUNC attribute length"); return -EINVAL; + } algp->alg_name[sizeof(algp->alg_name) - 1] = '\0'; return 0; } -static int verify_aead(struct nlattr **attrs) +static int verify_aead(struct nlattr **attrs, struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_ALG_AEAD]; struct xfrm_algo_aead *algp; @@ -86,8 +93,10 @@ static int verify_aead(struct nlattr **attrs) return 0; algp = nla_data(rt); - if (nla_len(rt) < (int)aead_len(algp)) + if (nla_len(rt) < (int)aead_len(algp)) { + NL_SET_ERR_MSG(extack, "Invalid AEAD attribute length"); return -EINVAL; + } algp->alg_name[sizeof(algp->alg_name) - 1] = '\0'; return 0; @@ -102,7 +111,7 @@ static void verify_one_addr(struct nlattr **attrs, enum xfrm_attr_type_t type, *addrp = nla_data(rt); } -static inline int verify_sec_ctx_len(struct nlattr **attrs) +static inline int verify_sec_ctx_len(struct nlattr **attrs, struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_SEC_CTX]; struct xfrm_user_sec_ctx *uctx; @@ -112,42 +121,59 @@ static inline int verify_sec_ctx_len(struct nlattr **attrs) uctx = nla_data(rt); if (uctx->len > nla_len(rt) || - uctx->len != (sizeof(struct xfrm_user_sec_ctx) + uctx->ctx_len)) + uctx->len != (sizeof(struct xfrm_user_sec_ctx) + uctx->ctx_len)) { + NL_SET_ERR_MSG(extack, "Invalid security context length"); return -EINVAL; + } return 0; } static inline int verify_replay(struct xfrm_usersa_info *p, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_REPLAY_ESN_VAL]; struct xfrm_replay_state_esn *rs; - if (!rt) - return (p->flags & XFRM_STATE_ESN) ? -EINVAL : 0; + if (!rt) { + if (p->flags & XFRM_STATE_ESN) { + NL_SET_ERR_MSG(extack, "Missing required attribute for ESN"); + return -EINVAL; + } + return 0; + } rs = nla_data(rt); - if (rs->bmp_len > XFRMA_REPLAY_ESN_MAX / sizeof(rs->bmp[0]) / 8) + if (rs->bmp_len > XFRMA_REPLAY_ESN_MAX / sizeof(rs->bmp[0]) / 8) { + NL_SET_ERR_MSG(extack, "ESN bitmap length must be <= 128"); return -EINVAL; + } if (nla_len(rt) < (int)xfrm_replay_state_esn_len(rs) && - nla_len(rt) != sizeof(*rs)) + nla_len(rt) != sizeof(*rs)) { + NL_SET_ERR_MSG(extack, "ESN attribute is too short to fit the full bitmap length"); return -EINVAL; + } /* As only ESP and AH support ESN feature. */ - if ((p->id.proto != IPPROTO_ESP) && (p->id.proto != IPPROTO_AH)) + if ((p->id.proto != IPPROTO_ESP) && (p->id.proto != IPPROTO_AH)) { + NL_SET_ERR_MSG(extack, "ESN only supported for ESP and AH"); return -EINVAL; + } - if (p->replay_window != 0) + if (p->replay_window != 0) { + NL_SET_ERR_MSG(extack, "ESN not compatible with legacy replay_window"); return -EINVAL; + } return 0; } static int verify_newsa_info(struct xfrm_usersa_info *p, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { int err; @@ -161,10 +187,12 @@ static int verify_newsa_info(struct xfrm_usersa_info *p, break; #else err = -EAFNOSUPPORT; + NL_SET_ERR_MSG(extack, "IPv6 support disabled"); goto out; #endif default: + NL_SET_ERR_MSG(extack, "Invalid address family"); goto out; } @@ -173,65 +201,98 @@ static int verify_newsa_info(struct xfrm_usersa_info *p, break; case AF_INET: - if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) + if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) { + NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)"); goto out; + } break; case AF_INET6: #if IS_ENABLED(CONFIG_IPV6) - if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) + if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) { + NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)"); goto out; + } break; #else + NL_SET_ERR_MSG(extack, "IPv6 support disabled"); err = -EAFNOSUPPORT; goto out; #endif default: + NL_SET_ERR_MSG(extack, "Invalid address family in selector"); goto out; } err = -EINVAL; switch (p->id.proto) { case IPPROTO_AH: - if ((!attrs[XFRMA_ALG_AUTH] && - !attrs[XFRMA_ALG_AUTH_TRUNC]) || - attrs[XFRMA_ALG_AEAD] || + if (!attrs[XFRMA_ALG_AUTH] && + !attrs[XFRMA_ALG_AUTH_TRUNC]) { + NL_SET_ERR_MSG(extack, "Missing required attribute for AH: AUTH_TRUNC or AUTH"); + goto out; + } + + if (attrs[XFRMA_ALG_AEAD] || attrs[XFRMA_ALG_CRYPT] || attrs[XFRMA_ALG_COMP] || - attrs[XFRMA_TFCPAD]) + attrs[XFRMA_TFCPAD]) { + NL_SET_ERR_MSG(extack, "Invalid attributes for AH: AEAD, CRYPT, COMP, TFCPAD"); goto out; + } break; case IPPROTO_ESP: - if (attrs[XFRMA_ALG_COMP]) + if (attrs[XFRMA_ALG_COMP]) { + NL_SET_ERR_MSG(extack, "Invalid attribute for ESP: COMP"); goto out; + } + if (!attrs[XFRMA_ALG_AUTH] && !attrs[XFRMA_ALG_AUTH_TRUNC] && !attrs[XFRMA_ALG_CRYPT] && - !attrs[XFRMA_ALG_AEAD]) + !attrs[XFRMA_ALG_AEAD]) { + NL_SET_ERR_MSG(extack, "Missing required attribute for ESP: at least one of AUTH, AUTH_TRUNC, CRYPT, AEAD"); goto out; + } + if ((attrs[XFRMA_ALG_AUTH] || attrs[XFRMA_ALG_AUTH_TRUNC] || attrs[XFRMA_ALG_CRYPT]) && - attrs[XFRMA_ALG_AEAD]) + attrs[XFRMA_ALG_AEAD]) { + NL_SET_ERR_MSG(extack, "Invalid attribute combination for ESP: AEAD can't be used with AUTH, AUTH_TRUNC, CRYPT"); goto out; + } + if (attrs[XFRMA_TFCPAD] && - p->mode != XFRM_MODE_TUNNEL) + p->mode != XFRM_MODE_TUNNEL) { + NL_SET_ERR_MSG(extack, "TFC padding can only be used in tunnel mode"); goto out; + } break; case IPPROTO_COMP: - if (!attrs[XFRMA_ALG_COMP] || - attrs[XFRMA_ALG_AEAD] || + if (!attrs[XFRMA_ALG_COMP]) { + NL_SET_ERR_MSG(extack, "Missing required attribute for COMP: COMP"); + goto out; + } + + if (attrs[XFRMA_ALG_AEAD] || attrs[XFRMA_ALG_AUTH] || attrs[XFRMA_ALG_AUTH_TRUNC] || attrs[XFRMA_ALG_CRYPT] || - attrs[XFRMA_TFCPAD] || - (ntohl(p->id.spi) >= 0x10000)) + attrs[XFRMA_TFCPAD]) { + NL_SET_ERR_MSG(extack, "Invalid attributes for COMP: AEAD, AUTH, AUTH_TRUNC, CRYPT, TFCPAD"); goto out; + } + + if (ntohl(p->id.spi) >= 0x10000) { + NL_SET_ERR_MSG(extack, "SPI is too large for COMP (must be < 0x10000)"); + goto out; + } break; #if IS_ENABLED(CONFIG_IPV6) @@ -244,29 +305,36 @@ static int verify_newsa_info(struct xfrm_usersa_info *p, attrs[XFRMA_ALG_CRYPT] || attrs[XFRMA_ENCAP] || attrs[XFRMA_SEC_CTX] || - attrs[XFRMA_TFCPAD] || - !attrs[XFRMA_COADDR]) + attrs[XFRMA_TFCPAD]) { + NL_SET_ERR_MSG(extack, "Invalid attributes for DSTOPTS/ROUTING"); goto out; + } + + if (!attrs[XFRMA_COADDR]) { + NL_SET_ERR_MSG(extack, "Missing required COADDR attribute for DSTOPTS/ROUTING"); + goto out; + } break; #endif default: + NL_SET_ERR_MSG(extack, "Unsupported protocol"); goto out; } - if ((err = verify_aead(attrs))) + if ((err = verify_aead(attrs, extack))) goto out; - if ((err = verify_auth_trunc(attrs))) + if ((err = verify_auth_trunc(attrs, extack))) goto out; - if ((err = verify_one_alg(attrs, XFRMA_ALG_AUTH))) + if ((err = verify_one_alg(attrs, XFRMA_ALG_AUTH, extack))) goto out; - if ((err = verify_one_alg(attrs, XFRMA_ALG_CRYPT))) + if ((err = verify_one_alg(attrs, XFRMA_ALG_CRYPT, extack))) goto out; - if ((err = verify_one_alg(attrs, XFRMA_ALG_COMP))) + if ((err = verify_one_alg(attrs, XFRMA_ALG_COMP, extack))) goto out; - if ((err = verify_sec_ctx_len(attrs))) + if ((err = verify_sec_ctx_len(attrs, extack))) goto out; - if ((err = verify_replay(p, attrs))) + if ((err = verify_replay(p, attrs, extack))) goto out; err = -EINVAL; @@ -278,14 +346,19 @@ static int verify_newsa_info(struct xfrm_usersa_info *p, break; default: + NL_SET_ERR_MSG(extack, "Unsupported mode"); goto out; } err = 0; - if (attrs[XFRMA_MTIMER_THRESH]) - if (!attrs[XFRMA_ENCAP]) + if (attrs[XFRMA_MTIMER_THRESH]) { + if (!attrs[XFRMA_ENCAP]) { + NL_SET_ERR_MSG(extack, "MTIMER_THRESH attribute can only be set on ENCAP states"); err = -EINVAL; + goto out; + } + } out: return err; @@ -293,7 +366,7 @@ out: static int attach_one_algo(struct xfrm_algo **algpp, u8 *props, struct xfrm_algo_desc *(*get_byname)(const char *, int), - struct nlattr *rta) + struct nlattr *rta, struct netlink_ext_ack *extack) { struct xfrm_algo *p, *ualg; struct xfrm_algo_desc *algo; @@ -304,8 +377,10 @@ static int attach_one_algo(struct xfrm_algo **algpp, u8 *props, ualg = nla_data(rta); algo = get_byname(ualg->alg_name, 1); - if (!algo) + if (!algo) { + NL_SET_ERR_MSG(extack, "Requested COMP algorithm not found"); return -ENOSYS; + } *props = algo->desc.sadb_alg_id; p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL); @@ -317,7 +392,8 @@ static int attach_one_algo(struct xfrm_algo **algpp, u8 *props, return 0; } -static int attach_crypt(struct xfrm_state *x, struct nlattr *rta) +static int attach_crypt(struct xfrm_state *x, struct nlattr *rta, + struct netlink_ext_ack *extack) { struct xfrm_algo *p, *ualg; struct xfrm_algo_desc *algo; @@ -328,8 +404,10 @@ static int attach_crypt(struct xfrm_state *x, struct nlattr *rta) ualg = nla_data(rta); algo = xfrm_ealg_get_byname(ualg->alg_name, 1); - if (!algo) + if (!algo) { + NL_SET_ERR_MSG(extack, "Requested CRYPT algorithm not found"); return -ENOSYS; + } x->props.ealgo = algo->desc.sadb_alg_id; p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL); @@ -343,7 +421,7 @@ static int attach_crypt(struct xfrm_state *x, struct nlattr *rta) } static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props, - struct nlattr *rta) + struct nlattr *rta, struct netlink_ext_ack *extack) { struct xfrm_algo *ualg; struct xfrm_algo_auth *p; @@ -355,8 +433,10 @@ static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props, ualg = nla_data(rta); algo = xfrm_aalg_get_byname(ualg->alg_name, 1); - if (!algo) + if (!algo) { + NL_SET_ERR_MSG(extack, "Requested AUTH algorithm not found"); return -ENOSYS; + } *props = algo->desc.sadb_alg_id; p = kmalloc(sizeof(*p) + (ualg->alg_key_len + 7) / 8, GFP_KERNEL); @@ -373,7 +453,7 @@ static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props, } static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props, - struct nlattr *rta) + struct nlattr *rta, struct netlink_ext_ack *extack) { struct xfrm_algo_auth *p, *ualg; struct xfrm_algo_desc *algo; @@ -384,10 +464,14 @@ static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props, ualg = nla_data(rta); algo = xfrm_aalg_get_byname(ualg->alg_name, 1); - if (!algo) + if (!algo) { + NL_SET_ERR_MSG(extack, "Requested AUTH_TRUNC algorithm not found"); return -ENOSYS; - if (ualg->alg_trunc_len > algo->uinfo.auth.icv_fullbits) + } + if (ualg->alg_trunc_len > algo->uinfo.auth.icv_fullbits) { + NL_SET_ERR_MSG(extack, "Invalid length requested for truncated ICV"); return -EINVAL; + } *props = algo->desc.sadb_alg_id; p = kmemdup(ualg, xfrm_alg_auth_len(ualg), GFP_KERNEL); @@ -402,7 +486,8 @@ static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props, return 0; } -static int attach_aead(struct xfrm_state *x, struct nlattr *rta) +static int attach_aead(struct xfrm_state *x, struct nlattr *rta, + struct netlink_ext_ack *extack) { struct xfrm_algo_aead *p, *ualg; struct xfrm_algo_desc *algo; @@ -413,8 +498,10 @@ static int attach_aead(struct xfrm_state *x, struct nlattr *rta) ualg = nla_data(rta); algo = xfrm_aead_get_byname(ualg->alg_name, ualg->alg_icv_len, 1); - if (!algo) + if (!algo) { + NL_SET_ERR_MSG(extack, "Requested AEAD algorithm not found"); return -ENOSYS; + } x->props.ealgo = algo->desc.sadb_alg_id; p = kmemdup(ualg, aead_len(ualg), GFP_KERNEL); @@ -579,7 +666,8 @@ static void xfrm_smark_init(struct nlattr **attrs, struct xfrm_mark *m) static struct xfrm_state *xfrm_state_construct(struct net *net, struct xfrm_usersa_info *p, struct nlattr **attrs, - int *errp) + int *errp, + struct netlink_ext_ack *extack) { struct xfrm_state *x = xfrm_state_alloc(net); int err = -ENOMEM; @@ -606,21 +694,21 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, if (attrs[XFRMA_SA_EXTRA_FLAGS]) x->props.extra_flags = nla_get_u32(attrs[XFRMA_SA_EXTRA_FLAGS]); - if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD]))) + if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD], extack))) goto error; if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo, - attrs[XFRMA_ALG_AUTH_TRUNC]))) + attrs[XFRMA_ALG_AUTH_TRUNC], extack))) goto error; if (!x->props.aalgo) { if ((err = attach_auth(&x->aalg, &x->props.aalgo, - attrs[XFRMA_ALG_AUTH]))) + attrs[XFRMA_ALG_AUTH], extack))) goto error; } - if ((err = attach_crypt(x, attrs[XFRMA_ALG_CRYPT]))) + if ((err = attach_crypt(x, attrs[XFRMA_ALG_CRYPT], extack))) goto error; if ((err = attach_one_algo(&x->calg, &x->props.calgo, xfrm_calg_get_byname, - attrs[XFRMA_ALG_COMP]))) + attrs[XFRMA_ALG_COMP], extack))) goto error; if (attrs[XFRMA_TFCPAD]) @@ -633,7 +721,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, if (attrs[XFRMA_IF_ID]) x->if_id = nla_get_u32(attrs[XFRMA_IF_ID]); - err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV]); + err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV], extack); if (err) goto error; @@ -653,7 +741,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, /* sysctl_xfrm_aevent_etime is in 100ms units */ x->replay_maxage = (net->xfrm.sysctl_aevent_etime*HZ)/XFRM_AE_ETH_M; - if ((err = xfrm_init_replay(x))) + if ((err = xfrm_init_replay(x, extack))) goto error; /* override default values from above */ @@ -662,7 +750,8 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, /* configure the hardware if offload is requested */ if (attrs[XFRMA_OFFLOAD_DEV]) { err = xfrm_dev_state_add(net, x, - nla_data(attrs[XFRMA_OFFLOAD_DEV])); + nla_data(attrs[XFRMA_OFFLOAD_DEV]), + extack); if (err) goto error; } @@ -678,7 +767,7 @@ error_no_put: } static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_usersa_info *p = nlmsg_data(nlh); @@ -686,11 +775,11 @@ static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh, int err; struct km_event c; - err = verify_newsa_info(p, attrs); + err = verify_newsa_info(p, attrs, extack); if (err) return err; - x = xfrm_state_construct(net, p, attrs, &err); + x = xfrm_state_construct(net, p, attrs, &err, extack); if (!x) return err; @@ -757,7 +846,7 @@ static struct xfrm_state *xfrm_user_state_lookup(struct net *net, } static int xfrm_del_sa(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_state *x; @@ -1254,7 +1343,8 @@ static int build_spdinfo(struct sk_buff *skb, struct net *net, } static int xfrm_set_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrmu_spdhthresh *thresh4 = NULL; @@ -1299,7 +1389,8 @@ static int xfrm_set_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_get_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct sk_buff *r_skb; @@ -1358,7 +1449,8 @@ static int build_sadinfo(struct sk_buff *skb, struct net *net, } static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct sk_buff *r_skb; @@ -1378,7 +1470,7 @@ static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_get_sa(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_usersa_id *p = nlmsg_data(nlh); @@ -1402,7 +1494,8 @@ out_noput: } static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_state *x; @@ -1477,7 +1570,7 @@ out_noput: return err; } -static int verify_policy_dir(u8 dir) +static int verify_policy_dir(u8 dir, struct netlink_ext_ack *extack) { switch (dir) { case XFRM_POLICY_IN: @@ -1486,13 +1579,14 @@ static int verify_policy_dir(u8 dir) break; default: + NL_SET_ERR_MSG(extack, "Invalid policy direction"); return -EINVAL; } return 0; } -static int verify_policy_type(u8 type) +static int verify_policy_type(u8 type, struct netlink_ext_ack *extack) { switch (type) { case XFRM_POLICY_TYPE_MAIN: @@ -1502,13 +1596,15 @@ static int verify_policy_type(u8 type) break; default: + NL_SET_ERR_MSG(extack, "Invalid policy type"); return -EINVAL; } return 0; } -static int verify_newpolicy_info(struct xfrm_userpolicy_info *p) +static int verify_newpolicy_info(struct xfrm_userpolicy_info *p, + struct netlink_ext_ack *extack) { int ret; @@ -1520,6 +1616,7 @@ static int verify_newpolicy_info(struct xfrm_userpolicy_info *p) break; default: + NL_SET_ERR_MSG(extack, "Invalid policy share"); return -EINVAL; } @@ -1529,35 +1626,44 @@ static int verify_newpolicy_info(struct xfrm_userpolicy_info *p) break; default: + NL_SET_ERR_MSG(extack, "Invalid policy action"); return -EINVAL; } switch (p->sel.family) { case AF_INET: - if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) + if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) { + NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)"); return -EINVAL; + } break; case AF_INET6: #if IS_ENABLED(CONFIG_IPV6) - if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) + if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) { + NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)"); return -EINVAL; + } break; #else + NL_SET_ERR_MSG(extack, "IPv6 support disabled"); return -EAFNOSUPPORT; #endif default: + NL_SET_ERR_MSG(extack, "Invalid selector family"); return -EINVAL; } - ret = verify_policy_dir(p->dir); + ret = verify_policy_dir(p->dir, extack); if (ret) return ret; - if (p->index && (xfrm_policy_id2dir(p->index) != p->dir)) + if (p->index && (xfrm_policy_id2dir(p->index) != p->dir)) { + NL_SET_ERR_MSG(extack, "Policy index doesn't match direction"); return -EINVAL; + } return 0; } @@ -1599,13 +1705,16 @@ static void copy_templates(struct xfrm_policy *xp, struct xfrm_user_tmpl *ut, } } -static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family) +static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family, + struct netlink_ext_ack *extack) { u16 prev_family; int i; - if (nr > XFRM_MAX_DEPTH) + if (nr > XFRM_MAX_DEPTH) { + NL_SET_ERR_MSG(extack, "Template count must be <= XFRM_MAX_DEPTH (" __stringify(XFRM_MAX_DEPTH) ")"); return -EINVAL; + } prev_family = family; @@ -1625,12 +1734,16 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family) case XFRM_MODE_BEET: break; default: - if (ut[i].family != prev_family) + if (ut[i].family != prev_family) { + NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change"); return -EINVAL; + } break; } - if (ut[i].mode >= XFRM_MODE_MAX) + if (ut[i].mode >= XFRM_MODE_MAX) { + NL_SET_ERR_MSG(extack, "Mode in template must be < XFRM_MODE_MAX (" __stringify(XFRM_MODE_MAX) ")"); return -EINVAL; + } prev_family = ut[i].family; @@ -1642,17 +1755,21 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family) break; #endif default: + NL_SET_ERR_MSG(extack, "Invalid family in template"); return -EINVAL; } - if (!xfrm_id_proto_valid(ut[i].id.proto)) + if (!xfrm_id_proto_valid(ut[i].id.proto)) { + NL_SET_ERR_MSG(extack, "Invalid XFRM protocol in template"); return -EINVAL; + } } return 0; } -static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs) +static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_TMPL]; @@ -1663,7 +1780,7 @@ static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs) int nr = nla_len(rt) / sizeof(*utmpl); int err; - err = validate_tmpl(nr, utmpl, pol->family); + err = validate_tmpl(nr, utmpl, pol->family, extack); if (err) return err; @@ -1672,7 +1789,8 @@ static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs) return 0; } -static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs) +static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct nlattr *rt = attrs[XFRMA_POLICY_TYPE]; struct xfrm_userpolicy_type *upt; @@ -1684,7 +1802,7 @@ static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs) type = upt->type; } - err = verify_policy_type(type); + err = verify_policy_type(type, extack); if (err) return err; @@ -1719,7 +1837,11 @@ static void copy_to_user_policy(struct xfrm_policy *xp, struct xfrm_userpolicy_i p->share = XFRM_SHARE_ANY; /* XXX xp->share */ } -static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_userpolicy_info *p, struct nlattr **attrs, int *errp) +static struct xfrm_policy *xfrm_policy_construct(struct net *net, + struct xfrm_userpolicy_info *p, + struct nlattr **attrs, + int *errp, + struct netlink_ext_ack *extack) { struct xfrm_policy *xp = xfrm_policy_alloc(net, GFP_KERNEL); int err; @@ -1731,11 +1853,11 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_us copy_from_user_policy(xp, p); - err = copy_from_user_policy_type(&xp->type, attrs); + err = copy_from_user_policy_type(&xp->type, attrs, extack); if (err) goto error; - if (!(err = copy_from_user_tmpl(xp, attrs))) + if (!(err = copy_from_user_tmpl(xp, attrs, extack))) err = copy_from_user_sec_ctx(xp, attrs); if (err) goto error; @@ -1754,7 +1876,8 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_us } static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_userpolicy_info *p = nlmsg_data(nlh); @@ -1763,14 +1886,14 @@ static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh, int err; int excl; - err = verify_newpolicy_info(p); + err = verify_newpolicy_info(p, extack); if (err) return err; - err = verify_sec_ctx_len(attrs); + err = verify_sec_ctx_len(attrs, extack); if (err) return err; - xp = xfrm_policy_construct(net, p, attrs, &err); + xp = xfrm_policy_construct(net, p, attrs, &err, extack); if (!xp) return err; @@ -2015,7 +2138,7 @@ static bool xfrm_userpolicy_is_valid(__u8 policy) } static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_userpolicy_default *up = nlmsg_data(nlh); @@ -2036,7 +2159,7 @@ static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct sk_buff *r_skb; struct nlmsghdr *r_nlh; @@ -2066,7 +2189,8 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_policy *xp; @@ -2081,11 +2205,11 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, p = nlmsg_data(nlh); delete = nlh->nlmsg_type == XFRM_MSG_DELPOLICY; - err = copy_from_user_policy_type(&type, attrs); + err = copy_from_user_policy_type(&type, attrs, extack); if (err) return err; - err = verify_policy_dir(p->dir); + err = verify_policy_dir(p->dir, extack); if (err) return err; @@ -2101,7 +2225,7 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh, struct nlattr *rt = attrs[XFRMA_SEC_CTX]; struct xfrm_sec_ctx *ctx; - err = verify_sec_ctx_len(attrs); + err = verify_sec_ctx_len(attrs, extack); if (err) return err; @@ -2149,7 +2273,8 @@ out: } static int xfrm_flush_sa(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct km_event c; @@ -2249,7 +2374,7 @@ out_cancel: } static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_state *x; @@ -2293,7 +2418,7 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_new_ae(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_state *x; @@ -2344,14 +2469,15 @@ out: } static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct km_event c; u8 type = XFRM_POLICY_TYPE_MAIN; int err; - err = copy_from_user_policy_type(&type, attrs); + err = copy_from_user_policy_type(&type, attrs, extack); if (err) return err; @@ -2372,7 +2498,8 @@ static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh, } static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_policy *xp; @@ -2383,11 +2510,11 @@ static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh, struct xfrm_mark m; u32 if_id = 0; - err = copy_from_user_policy_type(&type, attrs); + err = copy_from_user_policy_type(&type, attrs, extack); if (err) return err; - err = verify_policy_dir(p->dir); + err = verify_policy_dir(p->dir, extack); if (err) return err; @@ -2403,7 +2530,7 @@ static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh, struct nlattr *rt = attrs[XFRMA_SEC_CTX]; struct xfrm_sec_ctx *ctx; - err = verify_sec_ctx_len(attrs); + err = verify_sec_ctx_len(attrs, extack); if (err) return err; @@ -2438,7 +2565,8 @@ out: } static int xfrm_add_sa_expire(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_state *x; @@ -2472,7 +2600,8 @@ out: } static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, + struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct xfrm_policy *xp; @@ -2490,15 +2619,15 @@ static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh, xfrm_mark_get(attrs, &mark); - err = verify_newpolicy_info(&ua->policy); + err = verify_newpolicy_info(&ua->policy, extack); if (err) goto free_state; - err = verify_sec_ctx_len(attrs); + err = verify_sec_ctx_len(attrs, extack); if (err) goto free_state; /* build an XP */ - xp = xfrm_policy_construct(net, &ua->policy, attrs, &err); + xp = xfrm_policy_construct(net, &ua->policy, attrs, &err, extack); if (!xp) goto free_state; @@ -2577,7 +2706,7 @@ static int copy_from_user_migrate(struct xfrm_migrate *ma, } static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { struct xfrm_userpolicy_id *pi = nlmsg_data(nlh); struct xfrm_migrate m[XFRM_MAX_DEPTH]; @@ -2594,7 +2723,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh, kmp = attrs[XFRMA_KMADDRESS] ? &km : NULL; - err = copy_from_user_policy_type(&type, attrs); + err = copy_from_user_policy_type(&type, attrs, extack); if (err) return err; @@ -2623,7 +2752,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh, } #else static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh, - struct nlattr **attrs) + struct nlattr **attrs, struct netlink_ext_ack *extack) { return -ENOPROTOOPT; } @@ -2819,7 +2948,8 @@ static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = { }; static const struct xfrm_link { - int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **); + int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **, + struct netlink_ext_ack *); int (*start)(struct netlink_callback *); int (*dump)(struct sk_buff *, struct netlink_callback *); int (*done)(struct netlink_callback *); @@ -2921,7 +3051,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, goto err; } - err = link->doit(skb, nlh, attrs); + err = link->doit(skb, nlh, attrs, extack); /* We need to free skb allocated in xfrm_alloc_compat() before * returning from this function, because consume_skb() won't take @@ -3272,11 +3402,11 @@ static struct xfrm_policy *xfrm_compile_policy(struct sock *sk, int opt, *dir = -EINVAL; if (len < sizeof(*p) || - verify_newpolicy_info(p)) + verify_newpolicy_info(p, NULL)) return NULL; nr = ((len - sizeof(*p)) / sizeof(*ut)); - if (validate_tmpl(nr, ut, p->sel.family)) + if (validate_tmpl(nr, ut, p->sel.family, NULL)) return NULL; if (p->dir > XFRM_POLICY_OUT) diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9bd1af8e05a16363e670db910b9e872b78cfb2ab --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 + +target.json +bindings_generated.rs +bindings_helpers_generated.rs +exports_*_generated.h +doc/ +test/ diff --git a/rust/Makefile b/rust/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7700d3853404e5553c8f06b862608de7a89fae80 --- /dev/null +++ b/rust/Makefile @@ -0,0 +1,381 @@ +# SPDX-License-Identifier: GPL-2.0 + +always-$(CONFIG_RUST) += target.json +no-clean-files += target.json + +obj-$(CONFIG_RUST) += core.o compiler_builtins.o +always-$(CONFIG_RUST) += exports_core_generated.h + +# Missing prototypes are expected in the helpers since these are exported +# for Rust only, thus there is no header nor prototypes. +obj-$(CONFIG_RUST) += helpers.o +CFLAGS_REMOVE_helpers.o = -Wmissing-prototypes -Wmissing-declarations + +always-$(CONFIG_RUST) += libmacros.so +no-clean-files += libmacros.so + +always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs +obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o +always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \ + exports_kernel_generated.h + +obj-$(CONFIG_RUST) += exports.o + +# Avoids running `$(RUSTC)` for the sysroot when it may not be available. +ifdef CONFIG_RUST + +# `$(rust_flags)` is passed in case the user added `--sysroot`. +rustc_sysroot := $(shell $(RUSTC) $(rust_flags) --print sysroot) +rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2) +RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library + +ifeq ($(quiet),silent_) +cargo_quiet=-q +rust_test_quiet=-q +rustdoc_test_quiet=--test-args -q +else ifeq ($(quiet),quiet_) +rust_test_quiet=-q +rustdoc_test_quiet=--test-args -q +else +cargo_quiet=--verbose +endif + +core-cfgs = \ + --cfg no_fp_fmt_parse + +alloc-cfgs = \ + --cfg no_fmt \ + --cfg no_global_oom_handling \ + --cfg no_macros \ + --cfg no_rc \ + --cfg no_str \ + --cfg no_string \ + --cfg no_sync \ + --cfg no_thin + +quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $< + cmd_rustdoc = \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \ + $(rustc_target_flags) -L$(objtree)/$(obj) \ + --output $(objtree)/$(obj)/doc \ + --crate-name $(subst rustdoc-,,$@) \ + @$(objtree)/include/generated/rustc_cfg $< + +# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute +# can be used to specify a custom logo. However: +# - The given value is used as-is, thus it cannot be relative or a local file +# (unlike the non-custom case) since the generated docs have subfolders. +# - It requires adding it to every crate. +# - It requires changing `core` which comes from the sysroot. +# +# Using `-Zcrate-attr` would solve the last two points, but not the first. +# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new +# command-like flags to solve the issue. Meanwhile, we use the non-custom case +# and then retouch the generated files. +rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \ + rustdoc-alloc rustdoc-kernel + $(Q)cp $(srctree)/Documentation/images/logo.svg $(objtree)/$(obj)/doc + $(Q)cp $(srctree)/Documentation/images/COPYING-logo $(objtree)/$(obj)/doc + $(Q)find $(objtree)/$(obj)/doc -name '*.html' -type f -print0 | xargs -0 sed -Ei \ + -e 's:rust-logo\.svg:logo.svg:g' \ + -e 's:rust-logo\.png:logo.svg:g' \ + -e 's:favicon\.svg:logo.svg:g' \ + -e 's:::g' + $(Q)echo '.logo-container > img { object-fit: contain; }' \ + >> $(objtree)/$(obj)/doc/rustdoc.css + +rustdoc-macros: private rustdoc_host = yes +rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \ + --extern proc_macro +rustdoc-macros: $(src)/macros/lib.rs FORCE + $(call if_changed,rustdoc) + +rustdoc-core: private rustc_target_flags = $(core-cfgs) +rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE + $(call if_changed,rustdoc) + +rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE + $(call if_changed,rustdoc) + +# We need to allow `rustdoc::broken_intra_doc_links` because some +# `no_global_oom_handling` functions refer to non-`no_global_oom_handling` +# functions. Ideally `rustdoc` would have a way to distinguish broken links +# due to things that are "configured out" vs. entirely non-existing ones. +rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \ + -Arustdoc::broken_intra_doc_links +rustdoc-alloc: $(src)/alloc/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE + $(call if_changed,rustdoc) + +rustdoc-kernel: private rustc_target_flags = --extern alloc \ + --extern macros=$(objtree)/$(obj)/libmacros.so \ + --extern bindings +rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \ + rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \ + $(obj)/bindings.o FORCE + $(call if_changed,rustdoc) + +quiet_cmd_rustc_test_library = RUSTC TL $< + cmd_rustc_test_library = \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC) $(rust_common_flags) \ + @$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \ + --crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \ + --out-dir $(objtree)/$(obj)/test --cfg testlib \ + --sysroot $(objtree)/$(obj)/test/sysroot \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $< + +rusttestlib-macros: private rustc_target_flags = --extern proc_macro +rusttestlib-macros: private rustc_test_library_proc = yes +rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE + $(call if_changed,rustc_test_library) + +rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE + $(call if_changed,rustc_test_library) + +quiet_cmd_rustdoc_test = RUSTDOC T $< + cmd_rustdoc_test = \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTDOC) --test $(rust_common_flags) \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) $(rustdoc_test_target_flags) \ + --sysroot $(objtree)/$(obj)/test/sysroot $(rustdoc_test_quiet) \ + -L$(objtree)/$(obj)/test --output $(objtree)/$(obj)/doc \ + --crate-name $(subst rusttest-,,$@) $< + +# We cannot use `-Zpanic-abort-tests` because some tests are dynamic, +# so for the moment we skip `-Cpanic=abort`. +quiet_cmd_rustc_test = RUSTC T $< + cmd_rustc_test = \ + OBJTREE=$(abspath $(objtree)) \ + $(RUSTC) --test $(rust_common_flags) \ + @$(objtree)/include/generated/rustc_cfg \ + $(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \ + --sysroot $(objtree)/$(obj)/test/sysroot \ + -L$(objtree)/$(obj)/test \ + --crate-name $(subst rusttest-,,$@) $<; \ + $(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \ + $(rustc_test_run_flags) + +rusttest: rusttest-macros rusttest-kernel + +# This prepares a custom sysroot with our custom `alloc` instead of +# the standard one. +# +# This requires several hacks: +# - Unlike `core` and `alloc`, `std` depends on more than a dozen crates, +# including third-party crates that need to be downloaded, plus custom +# `build.rs` steps. Thus hardcoding things here is not maintainable. +# - `cargo` knows how to build the standard library, but it is an unstable +# feature so far (`-Zbuild-std`). +# - `cargo` only considers the use case of building the standard library +# to use it in a given package. Thus we need to create a dummy package +# and pick the generated libraries from there. +# - Since we only keep a subset of upstream `alloc` in-tree, we need +# to recreate it on the fly by putting our sources on top. +# - The usual ways of modifying the dependency graph in `cargo` do not seem +# to apply for the `-Zbuild-std` steps, thus we have to mislead it +# by modifying the sources in the sysroot. +# - To avoid messing with the user's Rust installation, we create a clone +# of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std` +# steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag. +# +# In the future, we hope to avoid the whole ordeal by either: +# - Making the `test` crate not depend on `std` (either improving upstream +# or having our own custom crate). +# - Making the tests run in kernel space (requires the previous point). +# - Making `std` and friends be more like a "normal" crate, so that +# `-Zbuild-std` and related hacks are not needed. +quiet_cmd_rustsysroot = RUSTSYSROOT + cmd_rustsysroot = \ + rm -rf $(objtree)/$(obj)/test; \ + mkdir -p $(objtree)/$(obj)/test; \ + cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \ + cp -r $(srctree)/$(src)/alloc/* \ + $(objtree)/$(obj)/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \ + echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \ + echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \ + >> $(objtree)/$(obj)/test/rustc_sysroot; \ + chmod u+x $(objtree)/$(obj)/test/rustc_sysroot; \ + $(CARGO) -q new $(objtree)/$(obj)/test/dummy; \ + RUSTC=$(objtree)/$(obj)/test/rustc_sysroot $(CARGO) $(cargo_quiet) \ + test -Zbuild-std --target $(rustc_host_target) \ + --manifest-path $(objtree)/$(obj)/test/dummy/Cargo.toml; \ + rm $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \ + cp $(objtree)/$(obj)/test/dummy/target/$(rustc_host_target)/debug/deps/* \ + $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib + +rusttest-prepare: FORCE + $(call if_changed,rustsysroot) + +rusttest-macros: private rustc_target_flags = --extern proc_macro +rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro +rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE + $(call if_changed,rustc_test) + $(call if_changed,rustdoc_test) + +rusttest-kernel: private rustc_target_flags = --extern alloc \ + --extern macros --extern bindings +rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ + rusttestlib-macros rusttestlib-bindings FORCE + $(call if_changed,rustc_test) + $(call if_changed,rustc_test_library) + +filechk_rust_target = $(objtree)/scripts/generate_rust_target < $< + +$(obj)/target.json: $(objtree)/include/config/auto.conf FORCE + $(call filechk,rust_target) + +ifdef CONFIG_CC_IS_CLANG +bindgen_c_flags = $(c_flags) +else +# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC +# plugin backend and/or the Clang driver would be perfectly compatible with GCC. +# +# For the moment, here we are tweaking the flags on the fly. This is a hack, +# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT` +# if we end up using one of those structs). +bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ + -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \ + -mindirect-branch=thunk-extern -mindirect-branch-register \ + -mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \ + -mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \ + -mno-pointers-to-nested-functions -mno-string \ + -mno-strict-align -mstrict-align \ + -fconserve-stack -falign-jumps=% -falign-loops=% \ + -femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \ + -fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \ + -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \ + -fzero-call-used-regs=% -fno-stack-clash-protection \ + -fno-inline-functions-called-once \ + --param=% --param asan-% + +# Derived from `scripts/Makefile.clang`. +BINDGEN_TARGET_x86 := x86_64-linux-gnu +BINDGEN_TARGET := $(BINDGEN_TARGET_$(SRCARCH)) + +# All warnings are inhibited since GCC builds are very experimental, +# many GCC warnings are not supported by Clang, they may only appear in +# some configurations, with new GCC versions, etc. +bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET) + +bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \ + $(bindgen_extra_c_flags) +endif + +ifdef CONFIG_LTO +bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags)) +else +bindgen_c_flags_lto = $(bindgen_c_flags) +endif + +bindgen_c_flags_final = $(bindgen_c_flags_lto) -D__BINDGEN__ + +quiet_cmd_bindgen = BINDGEN $@ + cmd_bindgen = \ + $(BINDGEN) $< $(bindgen_target_flags) \ + --use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \ + --no-debug '.*' \ + --size_t-is-usize -o $@ -- $(bindgen_c_flags_final) -DMODULE \ + $(bindgen_target_cflags) $(bindgen_target_extra) + +$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \ + $(shell grep -v '^\#\|^$$' $(srctree)/$(src)/bindgen_parameters) +$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \ + $(src)/bindgen_parameters FORCE + $(call if_changed_dep,bindgen) + +# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn +# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here +# given it is `libclang`; but for consistency, future Clang changes and/or +# a potential future GCC backend for `bindgen`, we disable it too. +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \ + --blacklist-type '.*' --whitelist-var '' \ + --whitelist-function 'rust_helper_.*' +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \ + -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations +$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \ + sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n pub fn \1/g' $@ +$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE + $(call if_changed_dep,bindgen) + +quiet_cmd_exports = EXPORTS $@ + cmd_exports = \ + $(NM) -p --defined-only $< \ + | grep -E ' (T|R|D) ' | cut -d ' ' -f 3 \ + | xargs -Isymbol \ + echo 'EXPORT_SYMBOL_RUST_GPL(symbol);' > $@ + +$(obj)/exports_core_generated.h: $(obj)/core.o FORCE + $(call if_changed,exports) + +$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE + $(call if_changed,exports) + +$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE + $(call if_changed,exports) + +$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE + $(call if_changed,exports) + +quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ + cmd_rustc_procmacro = \ + $(RUSTC_OR_CLIPPY) $(rust_common_flags) \ + --emit=dep-info,link --extern proc_macro \ + --crate-type proc-macro --out-dir $(objtree)/$(obj) \ + --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \ + mv $(objtree)/$(obj)/$(patsubst lib%.so,%,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) + +# Procedural macros can only be used with the `rustc` that compiled it. +# Therefore, to get `libmacros.so` automatically recompiled when the compiler +# version changes, we add `core.o` as a dependency (even if it is not needed). +$(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE + $(call if_changed_dep,rustc_procmacro) + +quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ + cmd_rustc_library = \ + OBJTREE=$(abspath $(objtree)) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \ + --emit=dep-info,obj,metadata --crate-type rlib \ + --out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ + mv $(objtree)/$(obj)/$(patsubst %.o,%,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) \ + $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) + +rust-analyzer: + $(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \ + $(RUST_LIB_SRC) > $(objtree)/rust-project.json + +$(obj)/core.o: private skip_clippy = 1 +$(obj)/core.o: private skip_flags = -Dunreachable_pub +$(obj)/core.o: private rustc_target_flags = $(core-cfgs) +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs $(obj)/target.json FORCE + $(call if_changed_dep,rustc_library) + +$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' +$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE + $(call if_changed_dep,rustc_library) + +$(obj)/alloc.o: private skip_clippy = 1 +$(obj)/alloc.o: private skip_flags = -Dunreachable_pub +$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs) +$(obj)/alloc.o: $(src)/alloc/lib.rs $(obj)/compiler_builtins.o FORCE + $(call if_changed_dep,rustc_library) + +$(obj)/bindings.o: $(src)/bindings/lib.rs \ + $(obj)/compiler_builtins.o \ + $(obj)/bindings/bindings_generated.rs \ + $(obj)/bindings/bindings_helpers_generated.rs FORCE + $(call if_changed_dep,rustc_library) + +$(obj)/kernel.o: private rustc_target_flags = --extern alloc \ + --extern macros --extern bindings +$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o \ + $(obj)/libmacros.so $(obj)/bindings.o FORCE + $(call if_changed_dep,rustc_library) + +endif # CONFIG_RUST diff --git a/rust/alloc/README.md b/rust/alloc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c89c753720b5092a208a838e7afe99996dac03bb --- /dev/null +++ b/rust/alloc/README.md @@ -0,0 +1,33 @@ +# `alloc` + +These source files come from the Rust standard library, hosted in +the repository, licensed under +"Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +see . + +Please note that these files should be kept as close as possible to +upstream. In general, only additions should be performed (e.g. new +methods). Eventually, changes should make it into upstream so that, +at some point, this fork can be dropped from the kernel tree. + + +## Rationale + +On one hand, kernel folks wanted to keep `alloc` in-tree to have more +freedom in both workflow and actual features if actually needed +(e.g. receiver types if we ended up using them), which is reasonable. + +On the other hand, Rust folks wanted to keep `alloc` as close as +upstream as possible and avoid as much divergence as possible, which +is also reasonable. + +We agreed on a middle-ground: we would keep a subset of `alloc` +in-tree that would be as small and as close as possible to upstream. +Then, upstream can start adding the functions that we add to `alloc` +etc., until we reach a point where the kernel already knows exactly +what it needs in `alloc` and all the new methods are merged into +upstream, so that we can drop `alloc` from the kernel tree and go back +to using the upstream one. + +By doing this, the kernel can go a bit faster now, and Rust can +slowly incorporate and discuss the changes as needed. diff --git a/rust/alloc/alloc.rs b/rust/alloc/alloc.rs new file mode 100644 index 0000000000000000000000000000000000000000..ca224a54177077b53c25ee7a7b404e800451ef70 --- /dev/null +++ b/rust/alloc/alloc.rs @@ -0,0 +1,440 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Memory allocation APIs + +#![stable(feature = "alloc_module", since = "1.28.0")] + +#[cfg(not(test))] +use core::intrinsics; +use core::intrinsics::{min_align_of_val, size_of_val}; + +use core::ptr::Unique; +#[cfg(not(test))] +use core::ptr::{self, NonNull}; + +#[stable(feature = "alloc_module", since = "1.28.0")] +#[doc(inline)] +pub use core::alloc::*; + +use core::marker::Destruct; + +#[cfg(test)] +mod tests; + +extern "Rust" { + // These are the magic symbols to call the global allocator. rustc generates + // them to call `__rg_alloc` etc. if there is a `#[global_allocator]` attribute + // (the code expanding that attribute macro generates those functions), or to call + // the default implementations in libstd (`__rdl_alloc` etc. in `library/std/src/alloc.rs`) + // otherwise. + // The rustc fork of LLVM also special-cases these function names to be able to optimize them + // like `malloc`, `realloc`, and `free`, respectively. + #[rustc_allocator] + #[rustc_allocator_nounwind] + fn __rust_alloc(size: usize, align: usize) -> *mut u8; + #[rustc_allocator_nounwind] + fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize); + #[rustc_allocator_nounwind] + fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8; + #[rustc_allocator_nounwind] + fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8; +} + +/// The global memory allocator. +/// +/// This type implements the [`Allocator`] trait by forwarding calls +/// to the allocator registered with the `#[global_allocator]` attribute +/// if there is one, or the `std` crate’s default. +/// +/// Note: while this type is unstable, the functionality it provides can be +/// accessed through the [free functions in `alloc`](self#functions). +#[unstable(feature = "allocator_api", issue = "32838")] +#[derive(Copy, Clone, Default, Debug)] +#[cfg(not(test))] +pub struct Global; + +#[cfg(test)] +pub use std::alloc::Global; + +/// Allocate memory with the global allocator. +/// +/// This function forwards calls to the [`GlobalAlloc::alloc`] method +/// of the allocator registered with the `#[global_allocator]` attribute +/// if there is one, or the `std` crate’s default. +/// +/// This function is expected to be deprecated in favor of the `alloc` method +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. +/// +/// # Safety +/// +/// See [`GlobalAlloc::alloc`]. +/// +/// # Examples +/// +/// ``` +/// use std::alloc::{alloc, dealloc, Layout}; +/// +/// unsafe { +/// let layout = Layout::new::(); +/// let ptr = alloc(layout); +/// +/// *(ptr as *mut u16) = 42; +/// assert_eq!(*(ptr as *mut u16), 42); +/// +/// dealloc(ptr, layout); +/// } +/// ``` +#[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] +#[inline] +pub unsafe fn alloc(layout: Layout) -> *mut u8 { + unsafe { __rust_alloc(layout.size(), layout.align()) } +} + +/// Deallocate memory with the global allocator. +/// +/// This function forwards calls to the [`GlobalAlloc::dealloc`] method +/// of the allocator registered with the `#[global_allocator]` attribute +/// if there is one, or the `std` crate’s default. +/// +/// This function is expected to be deprecated in favor of the `dealloc` method +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. +/// +/// # Safety +/// +/// See [`GlobalAlloc::dealloc`]. +#[stable(feature = "global_alloc", since = "1.28.0")] +#[inline] +pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { + unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) } +} + +/// Reallocate memory with the global allocator. +/// +/// This function forwards calls to the [`GlobalAlloc::realloc`] method +/// of the allocator registered with the `#[global_allocator]` attribute +/// if there is one, or the `std` crate’s default. +/// +/// This function is expected to be deprecated in favor of the `realloc` method +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. +/// +/// # Safety +/// +/// See [`GlobalAlloc::realloc`]. +#[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] +#[inline] +pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + unsafe { __rust_realloc(ptr, layout.size(), layout.align(), new_size) } +} + +/// Allocate zero-initialized memory with the global allocator. +/// +/// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method +/// of the allocator registered with the `#[global_allocator]` attribute +/// if there is one, or the `std` crate’s default. +/// +/// This function is expected to be deprecated in favor of the `alloc_zeroed` method +/// of the [`Global`] type when it and the [`Allocator`] trait become stable. +/// +/// # Safety +/// +/// See [`GlobalAlloc::alloc_zeroed`]. +/// +/// # Examples +/// +/// ``` +/// use std::alloc::{alloc_zeroed, dealloc, Layout}; +/// +/// unsafe { +/// let layout = Layout::new::(); +/// let ptr = alloc_zeroed(layout); +/// +/// assert_eq!(*(ptr as *mut u16), 0); +/// +/// dealloc(ptr, layout); +/// } +/// ``` +#[stable(feature = "global_alloc", since = "1.28.0")] +#[must_use = "losing the pointer will leak memory"] +#[inline] +pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { + unsafe { __rust_alloc_zeroed(layout.size(), layout.align()) } +} + +#[cfg(not(test))] +impl Global { + #[inline] + fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result, AllocError> { + match layout.size() { + 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), + // SAFETY: `layout` is non-zero in size, + size => unsafe { + let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) }; + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, size)) + }, + } + } + + // SAFETY: Same as `Allocator::grow` + #[inline] + unsafe fn grow_impl( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + zeroed: bool, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() >= old_layout.size(), + "`new_layout.size()` must be greater than or equal to `old_layout.size()`" + ); + + match old_layout.size() { + 0 => self.alloc_impl(new_layout, zeroed), + + // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size` + // as required by safety conditions. Other conditions must be upheld by the caller + old_size if old_layout.align() == new_layout.align() => unsafe { + let new_size = new_layout.size(); + + // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. + intrinsics::assume(new_size >= old_layout.size()); + + let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + if zeroed { + raw_ptr.add(old_size).write_bytes(0, new_size - old_size); + } + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`, + // both the old and new memory allocation are valid for reads and writes for `old_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + old_size => unsafe { + let new_ptr = self.alloc_impl(new_layout, zeroed)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size); + self.deallocate(ptr, old_layout); + Ok(new_ptr) + }, + } + } +} + +#[unstable(feature = "allocator_api", issue = "32838")] +#[cfg(not(test))] +unsafe impl Allocator for Global { + #[inline] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, false) + } + + #[inline] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + self.alloc_impl(layout, true) + } + + #[inline] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + if layout.size() != 0 { + // SAFETY: `layout` is non-zero in size, + // other conditions must be upheld by the caller + unsafe { dealloc(ptr.as_ptr(), layout) } + } + } + + #[inline] + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, false) } + } + + #[inline] + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + // SAFETY: all conditions must be upheld by the caller + unsafe { self.grow_impl(ptr, old_layout, new_layout, true) } + } + + #[inline] + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!( + new_layout.size() <= old_layout.size(), + "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" + ); + + match new_layout.size() { + // SAFETY: conditions must be upheld by the caller + 0 => unsafe { + self.deallocate(ptr, old_layout); + Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) + }, + + // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller + new_size if old_layout.align() == new_layout.align() => unsafe { + // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. + intrinsics::assume(new_size <= old_layout.size()); + + let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); + let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; + Ok(NonNull::slice_from_raw_parts(ptr, new_size)) + }, + + // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`, + // both the old and new memory allocation are valid for reads and writes for `new_size` + // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap + // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract + // for `dealloc` must be upheld by the caller. + new_size => unsafe { + let new_ptr = self.allocate(new_layout)?; + ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size); + self.deallocate(ptr, old_layout); + Ok(new_ptr) + }, + } + } +} + +/// The allocator for unique pointers. +#[cfg(all(not(no_global_oom_handling), not(test)))] +#[lang = "exchange_malloc"] +#[inline] +unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { + let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + match Global.allocate(layout) { + Ok(ptr) => ptr.as_mut_ptr(), + Err(_) => handle_alloc_error(layout), + } +} + +#[cfg_attr(not(test), lang = "box_free")] +#[inline] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +// This signature has to be the same as `Box`, otherwise an ICE will happen. +// When an additional parameter to `Box` is added (like `A: Allocator`), this has to be added here as +// well. +// For example if `Box` is changed to `struct Box(Unique, A)`, +// this function has to be changed to `fn box_free(Unique, A)` as well. +pub(crate) const unsafe fn box_free( + ptr: Unique, + alloc: A, +) { + unsafe { + let size = size_of_val(ptr.as_ref()); + let align = min_align_of_val(ptr.as_ref()); + let layout = Layout::from_size_align_unchecked(size, align); + alloc.deallocate(From::from(ptr.cast()), layout) + } +} + +// # Allocation error handler + +#[cfg(not(no_global_oom_handling))] +extern "Rust" { + // This is the magic symbol to call the global alloc error handler. rustc generates + // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the + // default implementations below (`__rdl_oom`) otherwise. + fn __rust_alloc_error_handler(size: usize, align: usize) -> !; +} + +/// Abort on memory allocation error or failure. +/// +/// Callers of memory allocation APIs wishing to abort computation +/// in response to an allocation error are encouraged to call this function, +/// rather than directly invoking `panic!` or similar. +/// +/// The default behavior of this function is to print a message to standard error +/// and abort the process. +/// It can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`]. +/// +/// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html +/// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html +#[stable(feature = "global_alloc", since = "1.28.0")] +#[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")] +#[cfg(all(not(no_global_oom_handling), not(test)))] +#[cold] +pub const fn handle_alloc_error(layout: Layout) -> ! { + const fn ct_error(_: Layout) -> ! { + panic!("allocation failed"); + } + + fn rt_error(layout: Layout) -> ! { + unsafe { + __rust_alloc_error_handler(layout.size(), layout.align()); + } + } + + unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) } +} + +// For alloc test `std::alloc::handle_alloc_error` can be used directly. +#[cfg(all(not(no_global_oom_handling), test))] +pub use std::alloc::handle_alloc_error; + +#[cfg(all(not(no_global_oom_handling), not(test)))] +#[doc(hidden)] +#[allow(unused_attributes)] +#[unstable(feature = "alloc_internals", issue = "none")] +pub mod __alloc_error_handler { + use crate::alloc::Layout; + + // called via generated `__rust_alloc_error_handler` + + // if there is no `#[alloc_error_handler]` + #[rustc_std_internal_symbol] + pub unsafe extern "C-unwind" fn __rdl_oom(size: usize, _align: usize) -> ! { + panic!("memory allocation of {size} bytes failed") + } + + // if there is an `#[alloc_error_handler]` + #[rustc_std_internal_symbol] + pub unsafe extern "C-unwind" fn __rg_oom(size: usize, align: usize) -> ! { + let layout = unsafe { Layout::from_size_align_unchecked(size, align) }; + extern "Rust" { + #[lang = "oom"] + fn oom_impl(layout: Layout) -> !; + } + unsafe { oom_impl(layout) } + } +} + +/// Specialize clones into pre-allocated, uninitialized memory. +/// Used by `Box::clone` and `Rc`/`Arc::make_mut`. +pub(crate) trait WriteCloneIntoRaw: Sized { + unsafe fn write_clone_into_raw(&self, target: *mut Self); +} + +impl WriteCloneIntoRaw for T { + #[inline] + default unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // Having allocated *first* may allow the optimizer to create + // the cloned value in-place, skipping the local and move. + unsafe { target.write(self.clone()) }; + } +} + +impl WriteCloneIntoRaw for T { + #[inline] + unsafe fn write_clone_into_raw(&self, target: *mut Self) { + // We can always copy in-place, without ever involving a local value. + unsafe { target.copy_from_nonoverlapping(self, 1) }; + } +} diff --git a/rust/alloc/borrow.rs b/rust/alloc/borrow.rs new file mode 100644 index 0000000000000000000000000000000000000000..dde4957200d4065957eea8e76dfe9e912b24b3bb --- /dev/null +++ b/rust/alloc/borrow.rs @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A module for working with borrowed data. + +#![stable(feature = "rust1", since = "1.0.0")] + +use core::cmp::Ordering; +use core::hash::{Hash, Hasher}; +use core::ops::Deref; +#[cfg(not(no_global_oom_handling))] +use core::ops::{Add, AddAssign}; + +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::borrow::{Borrow, BorrowMut}; + +use core::fmt; +#[cfg(not(no_global_oom_handling))] +use crate::string::String; + +use Cow::*; + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, B: ?Sized> Borrow for Cow<'a, B> +where + B: ToOwned, + ::Owned: 'a, +{ + fn borrow(&self) -> &B { + &**self + } +} + +/// A generalization of `Clone` to borrowed data. +/// +/// Some types make it possible to go from borrowed to owned, usually by +/// implementing the `Clone` trait. But `Clone` works only for going from `&T` +/// to `T`. The `ToOwned` trait generalizes `Clone` to construct owned data +/// from any borrow of a given type. +#[cfg_attr(not(test), rustc_diagnostic_item = "ToOwned")] +#[stable(feature = "rust1", since = "1.0.0")] +pub trait ToOwned { + /// The resulting type after obtaining ownership. + #[stable(feature = "rust1", since = "1.0.0")] + type Owned: Borrow; + + /// Creates owned data from borrowed data, usually by cloning. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// let s: &str = "a"; + /// let ss: String = s.to_owned(); + /// + /// let v: &[i32] = &[1, 2]; + /// let vv: Vec = v.to_owned(); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use = "cloning is often expensive and is not expected to have side effects"] + fn to_owned(&self) -> Self::Owned; + + /// Uses borrowed data to replace owned data, usually by cloning. + /// + /// This is borrow-generalized version of `Clone::clone_from`. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// # #![feature(toowned_clone_into)] + /// let mut s: String = String::new(); + /// "hello".clone_into(&mut s); + /// + /// let mut v: Vec = Vec::new(); + /// [1, 2][..].clone_into(&mut v); + /// ``` + #[unstable(feature = "toowned_clone_into", reason = "recently added", issue = "41263")] + fn clone_into(&self, target: &mut Self::Owned) { + *target = self.to_owned(); + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ToOwned for T +where + T: Clone, +{ + type Owned = T; + fn to_owned(&self) -> T { + self.clone() + } + + fn clone_into(&self, target: &mut T) { + target.clone_from(self); + } +} + +/// A clone-on-write smart pointer. +/// +/// The type `Cow` is a smart pointer providing clone-on-write functionality: it +/// can enclose and provide immutable access to borrowed data, and clone the +/// data lazily when mutation or ownership is required. The type is designed to +/// work with general borrowed data via the `Borrow` trait. +/// +/// `Cow` implements `Deref`, which means that you can call +/// non-mutating methods directly on the data it encloses. If mutation +/// is desired, `to_mut` will obtain a mutable reference to an owned +/// value, cloning if necessary. +/// +/// If you need reference-counting pointers, note that +/// [`Rc::make_mut`][crate::rc::Rc::make_mut] and +/// [`Arc::make_mut`][crate::sync::Arc::make_mut] can provide clone-on-write +/// functionality as well. +/// +/// # Examples +/// +/// ``` +/// use std::borrow::Cow; +/// +/// fn abs_all(input: &mut Cow<[i32]>) { +/// for i in 0..input.len() { +/// let v = input[i]; +/// if v < 0 { +/// // Clones into a vector if not already owned. +/// input.to_mut()[i] = -v; +/// } +/// } +/// } +/// +/// // No clone occurs because `input` doesn't need to be mutated. +/// let slice = [0, 1, 2]; +/// let mut input = Cow::from(&slice[..]); +/// abs_all(&mut input); +/// +/// // Clone occurs because `input` needs to be mutated. +/// let slice = [-1, 0, 1]; +/// let mut input = Cow::from(&slice[..]); +/// abs_all(&mut input); +/// +/// // No clone occurs because `input` is already owned. +/// let mut input = Cow::from(vec![-1, 0, 1]); +/// abs_all(&mut input); +/// ``` +/// +/// Another example showing how to keep `Cow` in a struct: +/// +/// ``` +/// use std::borrow::Cow; +/// +/// struct Items<'a, X: 'a> where [X]: ToOwned> { +/// values: Cow<'a, [X]>, +/// } +/// +/// impl<'a, X: Clone + 'a> Items<'a, X> where [X]: ToOwned> { +/// fn new(v: Cow<'a, [X]>) -> Self { +/// Items { values: v } +/// } +/// } +/// +/// // Creates a container from borrowed values of a slice +/// let readonly = [1, 2]; +/// let borrowed = Items::new((&readonly[..]).into()); +/// match borrowed { +/// Items { values: Cow::Borrowed(b) } => println!("borrowed {b:?}"), +/// _ => panic!("expect borrowed value"), +/// } +/// +/// let mut clone_on_write = borrowed; +/// // Mutates the data from slice into owned vec and pushes a new value on top +/// clone_on_write.values.to_mut().push(3); +/// println!("clone_on_write = {:?}", clone_on_write.values); +/// +/// // The data was mutated. Let's check it out. +/// match clone_on_write { +/// Items { values: Cow::Owned(_) } => println!("clone_on_write contains owned data"), +/// _ => panic!("expect owned data"), +/// } +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Cow")] +pub enum Cow<'a, B: ?Sized + 'a> +where + B: ToOwned, +{ + /// Borrowed data. + #[stable(feature = "rust1", since = "1.0.0")] + Borrowed(#[stable(feature = "rust1", since = "1.0.0")] &'a B), + + /// Owned data. + #[stable(feature = "rust1", since = "1.0.0")] + Owned(#[stable(feature = "rust1", since = "1.0.0")] ::Owned), +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Cow<'_, B> { + fn clone(&self) -> Self { + match *self { + Borrowed(b) => Borrowed(b), + Owned(ref o) => { + let b: &B = o.borrow(); + Owned(b.to_owned()) + } + } + } + + fn clone_from(&mut self, source: &Self) { + match (self, source) { + (&mut Owned(ref mut dest), &Owned(ref o)) => o.borrow().clone_into(dest), + (t, s) => *t = s.clone(), + } + } +} + +impl Cow<'_, B> { + /// Returns true if the data is borrowed, i.e. if `to_mut` would require additional work. + /// + /// # Examples + /// + /// ``` + /// #![feature(cow_is_borrowed)] + /// use std::borrow::Cow; + /// + /// let cow = Cow::Borrowed("moo"); + /// assert!(cow.is_borrowed()); + /// + /// let bull: Cow<'_, str> = Cow::Owned("...moo?".to_string()); + /// assert!(!bull.is_borrowed()); + /// ``` + #[unstable(feature = "cow_is_borrowed", issue = "65143")] + #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")] + pub const fn is_borrowed(&self) -> bool { + match *self { + Borrowed(_) => true, + Owned(_) => false, + } + } + + /// Returns true if the data is owned, i.e. if `to_mut` would be a no-op. + /// + /// # Examples + /// + /// ``` + /// #![feature(cow_is_borrowed)] + /// use std::borrow::Cow; + /// + /// let cow: Cow<'_, str> = Cow::Owned("moo".to_string()); + /// assert!(cow.is_owned()); + /// + /// let bull = Cow::Borrowed("...moo?"); + /// assert!(!bull.is_owned()); + /// ``` + #[unstable(feature = "cow_is_borrowed", issue = "65143")] + #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")] + pub const fn is_owned(&self) -> bool { + !self.is_borrowed() + } + + /// Acquires a mutable reference to the owned form of the data. + /// + /// Clones the data if it is not already owned. + /// + /// # Examples + /// + /// ``` + /// use std::borrow::Cow; + /// + /// let mut cow = Cow::Borrowed("foo"); + /// cow.to_mut().make_ascii_uppercase(); + /// + /// assert_eq!( + /// cow, + /// Cow::Owned(String::from("FOO")) as Cow + /// ); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn to_mut(&mut self) -> &mut ::Owned { + match *self { + Borrowed(borrowed) => { + *self = Owned(borrowed.to_owned()); + match *self { + Borrowed(..) => unreachable!(), + Owned(ref mut owned) => owned, + } + } + Owned(ref mut owned) => owned, + } + } + + /// Extracts the owned data. + /// + /// Clones the data if it is not already owned. + /// + /// # Examples + /// + /// Calling `into_owned` on a `Cow::Borrowed` returns a clone of the borrowed data: + /// + /// ``` + /// use std::borrow::Cow; + /// + /// let s = "Hello world!"; + /// let cow = Cow::Borrowed(s); + /// + /// assert_eq!( + /// cow.into_owned(), + /// String::from(s) + /// ); + /// ``` + /// + /// Calling `into_owned` on a `Cow::Owned` returns the owned data. The data is moved out of the + /// `Cow` without being cloned. + /// + /// ``` + /// use std::borrow::Cow; + /// + /// let s = "Hello world!"; + /// let cow: Cow = Cow::Owned(String::from(s)); + /// + /// assert_eq!( + /// cow.into_owned(), + /// String::from(s) + /// ); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_owned(self) -> ::Owned { + match self { + Borrowed(borrowed) => borrowed.to_owned(), + Owned(owned) => owned, + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_deref", issue = "88955")] +impl const Deref for Cow<'_, B> +where + B::Owned: ~const Borrow, +{ + type Target = B; + + fn deref(&self) -> &B { + match *self { + Borrowed(borrowed) => borrowed, + Owned(ref owned) => owned.borrow(), + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for Cow<'_, B> where B: Eq + ToOwned {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Cow<'_, B> +where + B: Ord + ToOwned, +{ + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + Ord::cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, 'b, B: ?Sized, C: ?Sized> PartialEq> for Cow<'a, B> +where + B: PartialEq + ToOwned, + C: ToOwned, +{ + #[inline] + fn eq(&self, other: &Cow<'b, C>) -> bool { + PartialEq::eq(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, B: ?Sized> PartialOrd for Cow<'a, B> +where + B: PartialOrd + ToOwned, +{ + #[inline] + fn partial_cmp(&self, other: &Cow<'a, B>) -> Option { + PartialOrd::partial_cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Cow<'_, B> +where + B: fmt::Debug + ToOwned, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Borrowed(ref b) => fmt::Debug::fmt(b, f), + Owned(ref o) => fmt::Debug::fmt(o, f), + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for Cow<'_, B> +where + B: fmt::Display + ToOwned, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Borrowed(ref b) => fmt::Display::fmt(b, f), + Owned(ref o) => fmt::Display::fmt(o, f), + } + } +} + +#[stable(feature = "default", since = "1.11.0")] +impl Default for Cow<'_, B> +where + B: ToOwned, +{ + /// Creates an owned Cow<'a, B> with the default value for the contained owned value. + fn default() -> Self { + Owned(::Owned::default()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for Cow<'_, B> +where + B: Hash + ToOwned, +{ + #[inline] + fn hash(&self, state: &mut H) { + Hash::hash(&**self, state) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRef for Cow<'_, T> { + fn as_ref(&self) -> &T { + self + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_add", since = "1.14.0")] +impl<'a> Add<&'a str> for Cow<'a, str> { + type Output = Cow<'a, str>; + + #[inline] + fn add(mut self, rhs: &'a str) -> Self::Output { + self += rhs; + self + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_add", since = "1.14.0")] +impl<'a> Add> for Cow<'a, str> { + type Output = Cow<'a, str>; + + #[inline] + fn add(mut self, rhs: Cow<'a, str>) -> Self::Output { + self += rhs; + self + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_add", since = "1.14.0")] +impl<'a> AddAssign<&'a str> for Cow<'a, str> { + fn add_assign(&mut self, rhs: &'a str) { + if self.is_empty() { + *self = Cow::Borrowed(rhs) + } else if !rhs.is_empty() { + if let Cow::Borrowed(lhs) = *self { + let mut s = String::with_capacity(lhs.len() + rhs.len()); + s.push_str(lhs); + *self = Cow::Owned(s); + } + self.to_mut().push_str(rhs); + } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "cow_add", since = "1.14.0")] +impl<'a> AddAssign> for Cow<'a, str> { + fn add_assign(&mut self, rhs: Cow<'a, str>) { + if self.is_empty() { + *self = rhs + } else if !rhs.is_empty() { + if let Cow::Borrowed(lhs) = *self { + let mut s = String::with_capacity(lhs.len() + rhs.len()); + s.push_str(lhs); + *self = Cow::Owned(s); + } + self.to_mut().push_str(&rhs); + } + } +} diff --git a/rust/alloc/boxed.rs b/rust/alloc/boxed.rs new file mode 100644 index 0000000000000000000000000000000000000000..dcfe87b14f3a644484a83d1cf1cfde54d12fddbd --- /dev/null +++ b/rust/alloc/boxed.rs @@ -0,0 +1,2028 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A pointer type for heap allocation. +//! +//! [`Box`], casually referred to as a 'box', provides the simplest form of +//! heap allocation in Rust. Boxes provide ownership for this allocation, and +//! drop their contents when they go out of scope. Boxes also ensure that they +//! never allocate more than `isize::MAX` bytes. +//! +//! # Examples +//! +//! Move a value from the stack to the heap by creating a [`Box`]: +//! +//! ``` +//! let val: u8 = 5; +//! let boxed: Box = Box::new(val); +//! ``` +//! +//! Move a value from a [`Box`] back to the stack by [dereferencing]: +//! +//! ``` +//! let boxed: Box = Box::new(5); +//! let val: u8 = *boxed; +//! ``` +//! +//! Creating a recursive data structure: +//! +//! ``` +//! #[derive(Debug)] +//! enum List { +//! Cons(T, Box>), +//! Nil, +//! } +//! +//! let list: List = List::Cons(1, Box::new(List::Cons(2, Box::new(List::Nil)))); +//! println!("{list:?}"); +//! ``` +//! +//! This will print `Cons(1, Cons(2, Nil))`. +//! +//! Recursive structures must be boxed, because if the definition of `Cons` +//! looked like this: +//! +//! ```compile_fail,E0072 +//! # enum List { +//! Cons(T, List), +//! # } +//! ``` +//! +//! It wouldn't work. This is because the size of a `List` depends on how many +//! elements are in the list, and so we don't know how much memory to allocate +//! for a `Cons`. By introducing a [`Box`], which has a defined size, we know how +//! big `Cons` needs to be. +//! +//! # Memory layout +//! +//! For non-zero-sized values, a [`Box`] will use the [`Global`] allocator for +//! its allocation. It is valid to convert both ways between a [`Box`] and a +//! raw pointer allocated with the [`Global`] allocator, given that the +//! [`Layout`] used with the allocator is correct for the type. More precisely, +//! a `value: *mut T` that has been allocated with the [`Global`] allocator +//! with `Layout::for_value(&*value)` may be converted into a box using +//! [`Box::::from_raw(value)`]. Conversely, the memory backing a `value: *mut +//! T` obtained from [`Box::::into_raw`] may be deallocated using the +//! [`Global`] allocator with [`Layout::for_value(&*value)`]. +//! +//! For zero-sized values, the `Box` pointer still has to be [valid] for reads +//! and writes and sufficiently aligned. In particular, casting any aligned +//! non-zero integer literal to a raw pointer produces a valid pointer, but a +//! pointer pointing into previously allocated memory that since got freed is +//! not valid. The recommended way to build a Box to a ZST if `Box::new` cannot +//! be used is to use [`ptr::NonNull::dangling`]. +//! +//! So long as `T: Sized`, a `Box` is guaranteed to be represented +//! as a single pointer and is also ABI-compatible with C pointers +//! (i.e. the C type `T*`). This means that if you have extern "C" +//! Rust functions that will be called from C, you can define those +//! Rust functions using `Box` types, and use `T*` as corresponding +//! type on the C side. As an example, consider this C header which +//! declares functions that create and destroy some kind of `Foo` +//! value: +//! +//! ```c +//! /* C header */ +//! +//! /* Returns ownership to the caller */ +//! struct Foo* foo_new(void); +//! +//! /* Takes ownership from the caller; no-op when invoked with null */ +//! void foo_delete(struct Foo*); +//! ``` +//! +//! These two functions might be implemented in Rust as follows. Here, the +//! `struct Foo*` type from C is translated to `Box`, which captures +//! the ownership constraints. Note also that the nullable argument to +//! `foo_delete` is represented in Rust as `Option>`, since `Box` +//! cannot be null. +//! +//! ``` +//! #[repr(C)] +//! pub struct Foo; +//! +//! #[no_mangle] +//! pub extern "C" fn foo_new() -> Box { +//! Box::new(Foo) +//! } +//! +//! #[no_mangle] +//! pub extern "C" fn foo_delete(_: Option>) {} +//! ``` +//! +//! Even though `Box` has the same representation and C ABI as a C pointer, +//! this does not mean that you can convert an arbitrary `T*` into a `Box` +//! and expect things to work. `Box` values will always be fully aligned, +//! non-null pointers. Moreover, the destructor for `Box` will attempt to +//! free the value with the global allocator. In general, the best practice +//! is to only use `Box` for pointers that originated from the global +//! allocator. +//! +//! **Important.** At least at present, you should avoid using +//! `Box` types for functions that are defined in C but invoked +//! from Rust. In those cases, you should directly mirror the C types +//! as closely as possible. Using types like `Box` where the C +//! definition is just using `T*` can lead to undefined behavior, as +//! described in [rust-lang/unsafe-code-guidelines#198][ucg#198]. +//! +//! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198 +//! [dereferencing]: core::ops::Deref +//! [`Box::::from_raw(value)`]: Box::from_raw +//! [`Global`]: crate::alloc::Global +//! [`Layout`]: crate::alloc::Layout +//! [`Layout::for_value(&*value)`]: crate::alloc::Layout::for_value +//! [valid]: ptr#safety + +#![stable(feature = "rust1", since = "1.0.0")] + +use core::any::Any; +use core::async_iter::AsyncIterator; +use core::borrow; +use core::cmp::Ordering; +use core::convert::{From, TryFrom}; +use core::fmt; +use core::future::Future; +use core::hash::{Hash, Hasher}; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::iter::{FusedIterator, Iterator}; +use core::marker::{Destruct, Unpin, Unsize}; +use core::mem; +use core::ops::{ + CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Generator, GeneratorState, Receiver, +}; +use core::pin::Pin; +use core::ptr::{self, Unique}; +use core::task::{Context, Poll}; + +#[cfg(not(no_global_oom_handling))] +use crate::alloc::{handle_alloc_error, WriteCloneIntoRaw}; +use crate::alloc::{AllocError, Allocator, Global, Layout}; +#[cfg(not(no_global_oom_handling))] +use crate::borrow::Cow; +use crate::raw_vec::RawVec; +#[cfg(not(no_global_oom_handling))] +use crate::str::from_boxed_utf8_unchecked; +#[cfg(not(no_global_oom_handling))] +use crate::vec::Vec; + +#[cfg(not(no_thin))] +#[unstable(feature = "thin_box", issue = "92791")] +pub use thin::ThinBox; + +#[cfg(not(no_thin))] +mod thin; + +/// A pointer type for heap allocation. +/// +/// See the [module-level documentation](../../std/boxed/index.html) for more. +#[lang = "owned_box"] +#[fundamental] +#[stable(feature = "rust1", since = "1.0.0")] +// The declaration of the `Box` struct must be kept in sync with the +// `alloc::alloc::box_free` function or ICEs will happen. See the comment +// on `box_free` for more details. +pub struct Box< + T: ?Sized, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +>(Unique, A); + +impl Box { + /// Allocates memory on the heap and then places `x` into it. + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// let five = Box::new(5); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline(always)] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub fn new(x: T) -> Self { + box x + } + + /// Constructs a new box with uninitialized contents. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let mut five = Box::::new_uninit(); + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5) + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + #[inline] + pub fn new_uninit() -> Box> { + Self::new_uninit_in(Global) + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let zero = Box::::new_zeroed(); + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[inline] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed() -> Box> { + Self::new_zeroed_in(Global) + } + + /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then + /// `x` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "pin", since = "1.33.0")] + #[must_use] + #[inline(always)] + pub fn pin(x: T) -> Pin> { + (box x).into() + } + + /// Allocates memory on the heap then places `x` into it, + /// returning an error if the allocation fails + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// let five = Box::try_new(5)?; + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new(x: T) -> Result { + Self::try_new_in(x, Global) + } + + /// Constructs a new box with uninitialized contents on the heap, + /// returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let mut five = Box::::try_new_uninit()?; + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[inline] + pub fn try_new_uninit() -> Result>, AllocError> { + Box::try_new_uninit_in(Global) + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes on the heap + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let zero = Box::::try_new_zeroed()?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[inline] + pub fn try_new_zeroed() -> Result>, AllocError> { + Box::try_new_zeroed_in(Global) + } +} + +impl Box { + /// Allocates memory in the given allocator then places `x` into it. + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let five = Box::new_in(5, System); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[must_use] + #[inline] + pub const fn new_in(x: T, alloc: A) -> Self + where + A: ~const Allocator + ~const Destruct, + { + let mut boxed = Self::new_uninit_in(alloc); + unsafe { + boxed.as_mut_ptr().write(x); + boxed.assume_init() + } + } + + /// Allocates memory in the given allocator then places `x` into it, + /// returning an error if the allocation fails + /// + /// This doesn't actually allocate if `T` is zero-sized. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let five = Box::try_new_in(5, System)?; + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn try_new_in(x: T, alloc: A) -> Result + where + T: ~const Destruct, + A: ~const Allocator + ~const Destruct, + { + let mut boxed = Self::try_new_uninit_in(alloc)?; + unsafe { + boxed.as_mut_ptr().write(x); + Ok(boxed.assume_init()) + } + } + + /// Constructs a new box with uninitialized contents in the provided allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut five = Box::::new_uninit_in(System); + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[cfg(not(no_global_oom_handling))] + #[must_use] + // #[unstable(feature = "new_uninit", issue = "63291")] + pub const fn new_uninit_in(alloc: A) -> Box, A> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. + // That would make code size bigger. + match Box::try_new_uninit_in(alloc) { + Ok(m) => m, + Err(_) => handle_alloc_error(layout), + } + } + + /// Constructs a new box with uninitialized contents in the provided allocator, + /// returning an error if the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut five = Box::::try_new_uninit_in(System)?; + /// + /// let five = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn try_new_uninit_in(alloc: A) -> Result, A>, AllocError> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + let ptr = alloc.allocate(layout)?.cast(); + unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes in the provided allocator. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let zero = Box::::new_zeroed_in(System); + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[cfg(not(no_global_oom_handling))] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub const fn new_zeroed_in(alloc: A) -> Box, A> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable. + // That would make code size bigger. + match Box::try_new_zeroed_in(alloc) { + Ok(m) => m, + Err(_) => handle_alloc_error(layout), + } + } + + /// Constructs a new `Box` with uninitialized contents, with the memory + /// being filled with `0` bytes in the provided allocator, + /// returning an error if the allocation fails, + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let zero = Box::::try_new_zeroed_in(System)?; + /// let zero = unsafe { zero.assume_init() }; + /// + /// assert_eq!(*zero, 0); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn try_new_zeroed_in(alloc: A) -> Result, A>, AllocError> + where + A: ~const Allocator + ~const Destruct, + { + let layout = Layout::new::>(); + let ptr = alloc.allocate_zeroed(layout)?.cast(); + unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) } + } + + /// Constructs a new `Pin>`. If `T` does not implement `Unpin`, then + /// `x` will be pinned in memory and unable to be moved. + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[must_use] + #[inline(always)] + pub const fn pin_in(x: T, alloc: A) -> Pin + where + A: 'static + ~const Allocator + ~const Destruct, + { + Self::into_pin(Self::new_in(x, alloc)) + } + + /// Converts a `Box` into a `Box<[T]>` + /// + /// This conversion does not allocate on the heap and happens in place. + #[unstable(feature = "box_into_boxed_slice", issue = "71582")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn into_boxed_slice(boxed: Self) -> Box<[T], A> { + let (raw, alloc) = Box::into_raw_with_allocator(boxed); + unsafe { Box::from_raw_in(raw as *mut [T; 1], alloc) } + } + + /// Consumes the `Box`, returning the wrapped value. + /// + /// # Examples + /// + /// ``` + /// #![feature(box_into_inner)] + /// + /// let c = Box::new(5); + /// + /// assert_eq!(Box::into_inner(c), 5); + /// ``` + #[unstable(feature = "box_into_inner", issue = "80437")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn into_inner(boxed: Self) -> T + where + Self: ~const Destruct, + { + *boxed + } +} + +impl Box<[T]> { + /// Constructs a new boxed slice with uninitialized contents. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let mut values = Box::<[u32]>::new_uninit_slice(3); + /// + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]) + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit]> { + unsafe { RawVec::with_capacity(len).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents, with the memory + /// being filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let values = Box::<[u32]>::new_zeroed_slice(3); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit]> { + unsafe { RawVec::with_capacity_zeroed(len).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents. Returns an error if + /// the allocation fails + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let mut values = Box::<[u32]>::try_new_uninit_slice(3)?; + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new_uninit_slice(len: usize) -> Result]>, AllocError> { + unsafe { + let layout = match Layout::array::>(len) { + Ok(l) => l, + Err(_) => return Err(AllocError), + }; + let ptr = Global.allocate(layout)?; + Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) + } + } + + /// Constructs a new boxed slice with uninitialized contents, with the memory + /// being filled with `0` bytes. Returns an error if the allocation fails + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// let values = Box::<[u32]>::try_new_zeroed_slice(3)?; + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]); + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn try_new_zeroed_slice(len: usize) -> Result]>, AllocError> { + unsafe { + let layout = match Layout::array::>(len) { + Ok(l) => l, + Err(_) => return Err(AllocError), + }; + let ptr = Global.allocate_zeroed(layout)?; + Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len)) + } + } +} + +impl Box<[T], A> { + /// Constructs a new boxed slice with uninitialized contents in the provided allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let mut values = Box::<[u32], _>::new_uninit_slice_in(3, System); + /// + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]) + /// ``` + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { + unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) } + } + + /// Constructs a new boxed slice with uninitialized contents in the provided allocator, + /// with the memory being filled with `0` bytes. + /// + /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage + /// of this method. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, new_uninit)] + /// + /// use std::alloc::System; + /// + /// let values = Box::<[u32], _>::new_zeroed_slice_in(3, System); + /// let values = unsafe { values.assume_init() }; + /// + /// assert_eq!(*values, [0, 0, 0]) + /// ``` + /// + /// [zeroed]: mem::MaybeUninit::zeroed + #[cfg(not(no_global_oom_handling))] + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "new_uninit", issue = "63291")] + #[must_use] + pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit], A> { + unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) } + } +} + +impl Box, A> { + /// Converts to `Box`. + /// + /// # Safety + /// + /// As with [`MaybeUninit::assume_init`], + /// it is up to the caller to guarantee that the value + /// really is in an initialized state. + /// Calling this when the content is not yet fully initialized + /// causes immediate undefined behavior. + /// + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let mut five = Box::::new_uninit(); + /// + /// let five: Box = unsafe { + /// // Deferred initialization: + /// five.as_mut_ptr().write(5); + /// + /// five.assume_init() + /// }; + /// + /// assert_eq!(*five, 5) + /// ``` + #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const unsafe fn assume_init(self) -> Box { + let (raw, alloc) = Box::into_raw_with_allocator(self); + unsafe { Box::from_raw_in(raw as *mut T, alloc) } + } + + /// Writes the value and converts to `Box`. + /// + /// This method converts the box similarly to [`Box::assume_init`] but + /// writes `value` into it before conversion thus guaranteeing safety. + /// In some scenarios use of this method may improve performance because + /// the compiler may be able to optimize copying from stack. + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let big_box = Box::<[usize; 1024]>::new_uninit(); + /// + /// let mut array = [0; 1024]; + /// for (i, place) in array.iter_mut().enumerate() { + /// *place = i; + /// } + /// + /// // The optimizer may be able to elide this copy, so previous code writes + /// // to heap directly. + /// let big_box = Box::write(big_box, array); + /// + /// for (i, x) in big_box.iter().enumerate() { + /// assert_eq!(*x, i); + /// } + /// ``` + #[unstable(feature = "new_uninit", issue = "63291")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn write(mut boxed: Self, value: T) -> Box { + unsafe { + (*boxed).write(value); + boxed.assume_init() + } + } +} + +impl Box<[mem::MaybeUninit], A> { + /// Converts to `Box<[T], A>`. + /// + /// # Safety + /// + /// As with [`MaybeUninit::assume_init`], + /// it is up to the caller to guarantee that the values + /// really are in an initialized state. + /// Calling this when the content is not yet fully initialized + /// causes immediate undefined behavior. + /// + /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init + /// + /// # Examples + /// + /// ``` + /// #![feature(new_uninit)] + /// + /// let mut values = Box::<[u32]>::new_uninit_slice(3); + /// + /// let values = unsafe { + /// // Deferred initialization: + /// values[0].as_mut_ptr().write(1); + /// values[1].as_mut_ptr().write(2); + /// values[2].as_mut_ptr().write(3); + /// + /// values.assume_init() + /// }; + /// + /// assert_eq!(*values, [1, 2, 3]) + /// ``` + #[unstable(feature = "new_uninit", issue = "63291")] + #[inline] + pub unsafe fn assume_init(self) -> Box<[T], A> { + let (raw, alloc) = Box::into_raw_with_allocator(self); + unsafe { Box::from_raw_in(raw as *mut [T], alloc) } + } +} + +impl Box { + /// Constructs a box from a raw pointer. + /// + /// After calling this function, the raw pointer is owned by the + /// resulting `Box`. Specifically, the `Box` destructor will call + /// the destructor of `T` and free the allocated memory. For this + /// to be safe, the memory must have been allocated in accordance + /// with the [memory layout] used by `Box` . + /// + /// # Safety + /// + /// This function is unsafe because improper use may lead to + /// memory problems. For example, a double-free may occur if the + /// function is called twice on the same raw pointer. + /// + /// The safety conditions are described in the [memory layout] section. + /// + /// # Examples + /// + /// Recreate a `Box` which was previously converted to a raw pointer + /// using [`Box::into_raw`]: + /// ``` + /// let x = Box::new(5); + /// let ptr = Box::into_raw(x); + /// let x = unsafe { Box::from_raw(ptr) }; + /// ``` + /// Manually create a `Box` from scratch by using the global allocator: + /// ``` + /// use std::alloc::{alloc, Layout}; + /// + /// unsafe { + /// let ptr = alloc(Layout::new::()) as *mut i32; + /// // In general .write is required to avoid attempting to destruct + /// // the (uninitialized) previous contents of `ptr`, though for this + /// // simple example `*ptr = 5` would have worked as well. + /// ptr.write(5); + /// let x = Box::from_raw(ptr); + /// } + /// ``` + /// + /// [memory layout]: self#memory-layout + /// [`Layout`]: crate::Layout + #[stable(feature = "box_raw", since = "1.4.0")] + #[inline] + pub unsafe fn from_raw(raw: *mut T) -> Self { + unsafe { Self::from_raw_in(raw, Global) } + } +} + +impl Box { + /// Constructs a box from a raw pointer in the given allocator. + /// + /// After calling this function, the raw pointer is owned by the + /// resulting `Box`. Specifically, the `Box` destructor will call + /// the destructor of `T` and free the allocated memory. For this + /// to be safe, the memory must have been allocated in accordance + /// with the [memory layout] used by `Box` . + /// + /// # Safety + /// + /// This function is unsafe because improper use may lead to + /// memory problems. For example, a double-free may occur if the + /// function is called twice on the same raw pointer. + /// + /// + /// # Examples + /// + /// Recreate a `Box` which was previously converted to a raw pointer + /// using [`Box::into_raw_with_allocator`]: + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let x = Box::new_in(5, System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; + /// ``` + /// Manually create a `Box` from scratch by using the system allocator: + /// ``` + /// #![feature(allocator_api, slice_ptr_get)] + /// + /// use std::alloc::{Allocator, Layout, System}; + /// + /// unsafe { + /// let ptr = System.allocate(Layout::new::())?.as_mut_ptr() as *mut i32; + /// // In general .write is required to avoid attempting to destruct + /// // the (uninitialized) previous contents of `ptr`, though for this + /// // simple example `*ptr = 5` would have worked as well. + /// ptr.write(5); + /// let x = Box::from_raw_in(ptr, System); + /// } + /// # Ok::<(), std::alloc::AllocError>(()) + /// ``` + /// + /// [memory layout]: self#memory-layout + /// [`Layout`]: crate::Layout + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const unsafe fn from_raw_in(raw: *mut T, alloc: A) -> Self { + Box(unsafe { Unique::new_unchecked(raw) }, alloc) + } + + /// Consumes the `Box`, returning a wrapped raw pointer. + /// + /// The pointer will be properly aligned and non-null. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Box`. In particular, the + /// caller should properly destroy `T` and release the memory, taking + /// into account the [memory layout] used by `Box`. The easiest way to + /// do this is to convert the raw pointer back into a `Box` with the + /// [`Box::from_raw`] function, allowing the `Box` destructor to perform + /// the cleanup. + /// + /// Note: this is an associated function, which means that you have + /// to call it as `Box::into_raw(b)` instead of `b.into_raw()`. This + /// is so that there is no conflict with a method on the inner type. + /// + /// # Examples + /// Converting the raw pointer back into a `Box` with [`Box::from_raw`] + /// for automatic cleanup: + /// ``` + /// let x = Box::new(String::from("Hello")); + /// let ptr = Box::into_raw(x); + /// let x = unsafe { Box::from_raw(ptr) }; + /// ``` + /// Manual cleanup by explicitly running the destructor and deallocating + /// the memory: + /// ``` + /// use std::alloc::{dealloc, Layout}; + /// use std::ptr; + /// + /// let x = Box::new(String::from("Hello")); + /// let p = Box::into_raw(x); + /// unsafe { + /// ptr::drop_in_place(p); + /// dealloc(p as *mut u8, Layout::new::()); + /// } + /// ``` + /// + /// [memory layout]: self#memory-layout + #[stable(feature = "box_raw", since = "1.4.0")] + #[inline] + pub fn into_raw(b: Self) -> *mut T { + Self::into_raw_with_allocator(b).0 + } + + /// Consumes the `Box`, returning a wrapped raw pointer and the allocator. + /// + /// The pointer will be properly aligned and non-null. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Box`. In particular, the + /// caller should properly destroy `T` and release the memory, taking + /// into account the [memory layout] used by `Box`. The easiest way to + /// do this is to convert the raw pointer back into a `Box` with the + /// [`Box::from_raw_in`] function, allowing the `Box` destructor to perform + /// the cleanup. + /// + /// Note: this is an associated function, which means that you have + /// to call it as `Box::into_raw_with_allocator(b)` instead of `b.into_raw_with_allocator()`. This + /// is so that there is no conflict with a method on the inner type. + /// + /// # Examples + /// Converting the raw pointer back into a `Box` with [`Box::from_raw_in`] + /// for automatic cleanup: + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let x = Box::new_in(String::from("Hello"), System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// let x = unsafe { Box::from_raw_in(ptr, alloc) }; + /// ``` + /// Manual cleanup by explicitly running the destructor and deallocating + /// the memory: + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::{Allocator, Layout, System}; + /// use std::ptr::{self, NonNull}; + /// + /// let x = Box::new_in(String::from("Hello"), System); + /// let (ptr, alloc) = Box::into_raw_with_allocator(x); + /// unsafe { + /// ptr::drop_in_place(ptr); + /// let non_null = NonNull::new_unchecked(ptr); + /// alloc.deallocate(non_null.cast(), Layout::new::()); + /// } + /// ``` + /// + /// [memory layout]: self#memory-layout + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn into_raw_with_allocator(b: Self) -> (*mut T, A) { + let (leaked, alloc) = Box::into_unique(b); + (leaked.as_ptr(), alloc) + } + + #[unstable( + feature = "ptr_internals", + issue = "none", + reason = "use `Box::leak(b).into()` or `Unique::from(Box::leak(b))` instead" + )] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + #[doc(hidden)] + pub const fn into_unique(b: Self) -> (Unique, A) { + // Box is recognized as a "unique pointer" by Stacked Borrows, but internally it is a + // raw pointer for the type system. Turning it directly into a raw pointer would not be + // recognized as "releasing" the unique pointer to permit aliased raw accesses, + // so all raw pointer methods have to go through `Box::leak`. Turning *that* to a raw pointer + // behaves correctly. + let alloc = unsafe { ptr::read(&b.1) }; + (Unique::from(Box::leak(b)), alloc) + } + + /// Returns a reference to the underlying allocator. + /// + /// Note: this is an associated function, which means that you have + /// to call it as `Box::allocator(&b)` instead of `b.allocator()`. This + /// is so that there is no conflict with a method on the inner type. + #[unstable(feature = "allocator_api", issue = "32838")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn allocator(b: &Self) -> &A { + &b.1 + } + + /// Consumes and leaks the `Box`, returning a mutable reference, + /// `&'a mut T`. Note that the type `T` must outlive the chosen lifetime + /// `'a`. If the type has only static references, or none at all, then this + /// may be chosen to be `'static`. + /// + /// This function is mainly useful for data that lives for the remainder of + /// the program's life. Dropping the returned reference will cause a memory + /// leak. If this is not acceptable, the reference should first be wrapped + /// with the [`Box::from_raw`] function producing a `Box`. This `Box` can + /// then be dropped which will properly destroy `T` and release the + /// allocated memory. + /// + /// Note: this is an associated function, which means that you have + /// to call it as `Box::leak(b)` instead of `b.leak()`. This + /// is so that there is no conflict with a method on the inner type. + /// + /// # Examples + /// + /// Simple usage: + /// + /// ``` + /// let x = Box::new(41); + /// let static_ref: &'static mut usize = Box::leak(x); + /// *static_ref += 1; + /// assert_eq!(*static_ref, 42); + /// ``` + /// + /// Unsized data: + /// + /// ``` + /// let x = vec![1, 2, 3].into_boxed_slice(); + /// let static_ref = Box::leak(x); + /// static_ref[0] = 4; + /// assert_eq!(*static_ref, [4, 2, 3]); + /// ``` + #[stable(feature = "box_leak", since = "1.26.0")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + #[inline] + pub const fn leak<'a>(b: Self) -> &'a mut T + where + A: 'a, + { + unsafe { &mut *mem::ManuallyDrop::new(b).0.as_ptr() } + } + + /// Converts a `Box` into a `Pin>` + /// + /// This conversion does not allocate on the heap and happens in place. + /// + /// This is also available via [`From`]. + #[unstable(feature = "box_into_pin", issue = "62370")] + #[rustc_const_unstable(feature = "const_box", issue = "92521")] + pub const fn into_pin(boxed: Self) -> Pin + where + A: 'static, + { + // It's not possible to move or replace the insides of a `Pin>` + // when `T: !Unpin`, so it's safe to pin it directly without any + // additional requirements. + unsafe { Pin::new_unchecked(boxed) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box { + fn drop(&mut self) { + // FIXME: Do nothing, drop is currently performed by compiler. + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Default for Box { + /// Creates a `Box`, with the `Default` value for T. + fn default() -> Self { + box T::default() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Box<[T]> { + fn default() -> Self { + let ptr: Unique<[T]> = Unique::<[T; 0]>::dangling(); + Box(ptr, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "default_box_extra", since = "1.17.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Box { + fn default() -> Self { + // SAFETY: This is the same as `Unique::cast` but with an unsized `U = str`. + let ptr: Unique = unsafe { + let bytes: Unique<[u8]> = Unique::<[u8; 0]>::dangling(); + Unique::new_unchecked(bytes.as_ptr() as *mut str) + }; + Box(ptr, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Box { + /// Returns a new box with a `clone()` of this box's contents. + /// + /// # Examples + /// + /// ``` + /// let x = Box::new(5); + /// let y = x.clone(); + /// + /// // The value is the same + /// assert_eq!(x, y); + /// + /// // But they are unique objects + /// assert_ne!(&*x as *const i32, &*y as *const i32); + /// ``` + #[inline] + fn clone(&self) -> Self { + // Pre-allocate memory to allow writing the cloned value directly. + let mut boxed = Self::new_uninit_in(self.1.clone()); + unsafe { + (**self).write_clone_into_raw(boxed.as_mut_ptr()); + boxed.assume_init() + } + } + + /// Copies `source`'s contents into `self` without creating a new allocation. + /// + /// # Examples + /// + /// ``` + /// let x = Box::new(5); + /// let mut y = Box::new(10); + /// let yp: *const i32 = &*y; + /// + /// y.clone_from(&x); + /// + /// // The value is the same + /// assert_eq!(x, y); + /// + /// // And no allocation occurred + /// assert_eq!(yp, &*y); + /// ``` + #[inline] + fn clone_from(&mut self, source: &Self) { + (**self).clone_from(&(**source)); + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_slice_clone", since = "1.3.0")] +impl Clone for Box { + fn clone(&self) -> Self { + // this makes a copy of the data + let buf: Box<[u8]> = self.as_bytes().into(); + unsafe { from_boxed_utf8_unchecked(buf) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialEq for Box { + #[inline] + fn eq(&self, other: &Self) -> bool { + PartialEq::eq(&**self, &**other) + } + #[inline] + fn ne(&self, other: &Self) -> bool { + PartialEq::ne(&**self, &**other) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for Box { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + PartialOrd::partial_cmp(&**self, &**other) + } + #[inline] + fn lt(&self, other: &Self) -> bool { + PartialOrd::lt(&**self, &**other) + } + #[inline] + fn le(&self, other: &Self) -> bool { + PartialOrd::le(&**self, &**other) + } + #[inline] + fn ge(&self, other: &Self) -> bool { + PartialOrd::ge(&**self, &**other) + } + #[inline] + fn gt(&self, other: &Self) -> bool { + PartialOrd::gt(&**self, &**other) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Box { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + Ord::cmp(&**self, &**other) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for Box {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for Box { + fn hash(&self, state: &mut H) { + (**self).hash(state); + } +} + +#[stable(feature = "indirect_hasher_impl", since = "1.22.0")] +impl Hasher for Box { + fn finish(&self) -> u64 { + (**self).finish() + } + fn write(&mut self, bytes: &[u8]) { + (**self).write(bytes) + } + fn write_u8(&mut self, i: u8) { + (**self).write_u8(i) + } + fn write_u16(&mut self, i: u16) { + (**self).write_u16(i) + } + fn write_u32(&mut self, i: u32) { + (**self).write_u32(i) + } + fn write_u64(&mut self, i: u64) { + (**self).write_u64(i) + } + fn write_u128(&mut self, i: u128) { + (**self).write_u128(i) + } + fn write_usize(&mut self, i: usize) { + (**self).write_usize(i) + } + fn write_i8(&mut self, i: i8) { + (**self).write_i8(i) + } + fn write_i16(&mut self, i: i16) { + (**self).write_i16(i) + } + fn write_i32(&mut self, i: i32) { + (**self).write_i32(i) + } + fn write_i64(&mut self, i: i64) { + (**self).write_i64(i) + } + fn write_i128(&mut self, i: i128) { + (**self).write_i128(i) + } + fn write_isize(&mut self, i: isize) { + (**self).write_isize(i) + } + fn write_length_prefix(&mut self, len: usize) { + (**self).write_length_prefix(len) + } + fn write_str(&mut self, s: &str) { + (**self).write_str(s) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "from_for_ptrs", since = "1.6.0")] +impl From for Box { + /// Converts a `T` into a `Box` + /// + /// The conversion allocates on the heap and moves `t` + /// from the stack into it. + /// + /// # Examples + /// + /// ```rust + /// let x = 5; + /// let boxed = Box::new(5); + /// + /// assert_eq!(Box::from(x), boxed); + /// ``` + fn from(t: T) -> Self { + Box::new(t) + } +} + +#[stable(feature = "pin", since = "1.33.0")] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const From> for Pin> +where + A: 'static, +{ + /// Converts a `Box` into a `Pin>` + /// + /// This conversion does not allocate on the heap and happens in place. + fn from(boxed: Box) -> Self { + Box::into_pin(boxed) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_slice", since = "1.17.0")] +impl From<&[T]> for Box<[T]> { + /// Converts a `&[T]` into a `Box<[T]>` + /// + /// This conversion allocates on the heap + /// and performs a copy of `slice`. + /// + /// # Examples + /// ```rust + /// // create a &[u8] which will be used to create a Box<[u8]> + /// let slice: &[u8] = &[104, 101, 108, 108, 111]; + /// let boxed_slice: Box<[u8]> = Box::from(slice); + /// + /// println!("{boxed_slice:?}"); + /// ``` + fn from(slice: &[T]) -> Box<[T]> { + let len = slice.len(); + let buf = RawVec::with_capacity(len); + unsafe { + ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len); + buf.into_box(slice.len()).assume_init() + } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box<[T]> { + /// Converts a `Cow<'_, [T]>` into a `Box<[T]>` + /// + /// When `cow` is the `Cow::Borrowed` variant, this + /// conversion allocates on the heap and copies the + /// underlying slice. Otherwise, it will try to reuse the owned + /// `Vec`'s allocation. + #[inline] + fn from(cow: Cow<'_, [T]>) -> Box<[T]> { + match cow { + Cow::Borrowed(slice) => Box::from(slice), + Cow::Owned(slice) => Box::from(slice), + } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_slice", since = "1.17.0")] +impl From<&str> for Box { + /// Converts a `&str` into a `Box` + /// + /// This conversion allocates on the heap + /// and performs a copy of `s`. + /// + /// # Examples + /// + /// ```rust + /// let boxed: Box = Box::from("hello"); + /// println!("{boxed}"); + /// ``` + #[inline] + fn from(s: &str) -> Box { + unsafe { from_boxed_utf8_unchecked(Box::from(s.as_bytes())) } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_cow", since = "1.45.0")] +impl From> for Box { + /// Converts a `Cow<'_, str>` into a `Box` + /// + /// When `cow` is the `Cow::Borrowed` variant, this + /// conversion allocates on the heap and copies the + /// underlying `str`. Otherwise, it will try to reuse the owned + /// `String`'s allocation. + /// + /// # Examples + /// + /// ```rust + /// use std::borrow::Cow; + /// + /// let unboxed = Cow::Borrowed("hello"); + /// let boxed: Box = Box::from(unboxed); + /// println!("{boxed}"); + /// ``` + /// + /// ```rust + /// # use std::borrow::Cow; + /// let unboxed = Cow::Owned("hello".to_string()); + /// let boxed: Box = Box::from(unboxed); + /// println!("{boxed}"); + /// ``` + #[inline] + fn from(cow: Cow<'_, str>) -> Box { + match cow { + Cow::Borrowed(s) => Box::from(s), + Cow::Owned(s) => Box::from(s), + } + } +} + +#[stable(feature = "boxed_str_conv", since = "1.19.0")] +impl From> for Box<[u8], A> { + /// Converts a `Box` into a `Box<[u8]>` + /// + /// This conversion does not allocate on the heap and happens in place. + /// + /// # Examples + /// ```rust + /// // create a Box which will be used to create a Box<[u8]> + /// let boxed: Box = Box::from("hello"); + /// let boxed_str: Box<[u8]> = Box::from(boxed); + /// + /// // create a &[u8] which will be used to create a Box<[u8]> + /// let slice: &[u8] = &[104, 101, 108, 108, 111]; + /// let boxed_slice = Box::from(slice); + /// + /// assert_eq!(boxed_slice, boxed_str); + /// ``` + #[inline] + fn from(s: Box) -> Self { + let (raw, alloc) = Box::into_raw_with_allocator(s); + unsafe { Box::from_raw_in(raw as *mut [u8], alloc) } + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_from_array", since = "1.45.0")] +impl From<[T; N]> for Box<[T]> { + /// Converts a `[T; N]` into a `Box<[T]>` + /// + /// This conversion moves the array to newly heap-allocated memory. + /// + /// # Examples + /// + /// ```rust + /// let boxed: Box<[u8]> = Box::from([4, 2]); + /// println!("{boxed:?}"); + /// ``` + fn from(array: [T; N]) -> Box<[T]> { + box array + } +} + +#[stable(feature = "boxed_slice_try_from", since = "1.43.0")] +impl TryFrom> for Box<[T; N]> { + type Error = Box<[T]>; + + /// Attempts to convert a `Box<[T]>` into a `Box<[T; N]>`. + /// + /// The conversion occurs in-place and does not require a + /// new memory allocation. + /// + /// # Errors + /// + /// Returns the old `Box<[T]>` in the `Err` variant if + /// `boxed_slice.len()` does not equal `N`. + fn try_from(boxed_slice: Box<[T]>) -> Result { + if boxed_slice.len() == N { + Ok(unsafe { Box::from_raw(Box::into_raw(boxed_slice) as *mut [T; N]) }) + } else { + Err(boxed_slice) + } + } +} + +impl Box { + /// Attempt to downcast the box to a concrete type. + /// + /// # Examples + /// + /// ``` + /// use std::any::Any; + /// + /// fn print_if_string(value: Box) { + /// if let Ok(string) = value.downcast::() { + /// println!("String ({}): {}", string.len(), string); + /// } + /// } + /// + /// let my_string = "Hello World".to_string(); + /// print_if_string(Box::new(my_string)); + /// print_if_string(Box::new(0i8)); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } + } + + /// Downcasts the box to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut dyn Any, _) = Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) + } + } +} + +impl Box { + /// Attempt to downcast the box to a concrete type. + /// + /// # Examples + /// + /// ``` + /// use std::any::Any; + /// + /// fn print_if_string(value: Box) { + /// if let Ok(string) = value.downcast::() { + /// println!("String ({}): {}", string.len(), string); + /// } + /// } + /// + /// let my_string = "Hello World".to_string(); + /// print_if_string(Box::new(my_string)); + /// print_if_string(Box::new(0i8)); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } + } + + /// Downcasts the box to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut (dyn Any + Send), _) = Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) + } + } +} + +impl Box { + /// Attempt to downcast the box to a concrete type. + /// + /// # Examples + /// + /// ``` + /// use std::any::Any; + /// + /// fn print_if_string(value: Box) { + /// if let Ok(string) = value.downcast::() { + /// println!("String ({}): {}", string.len(), string); + /// } + /// } + /// + /// let my_string = "Hello World".to_string(); + /// print_if_string(Box::new(my_string)); + /// print_if_string(Box::new(0i8)); + /// ``` + #[inline] + #[stable(feature = "box_send_sync_any_downcast", since = "1.51.0")] + pub fn downcast(self) -> Result, Self> { + if self.is::() { unsafe { Ok(self.downcast_unchecked::()) } } else { Err(self) } + } + + /// Downcasts the box to a concrete type. + /// + /// For a safe alternative see [`downcast`]. + /// + /// # Examples + /// + /// ``` + /// #![feature(downcast_unchecked)] + /// + /// use std::any::Any; + /// + /// let x: Box = Box::new(1_usize); + /// + /// unsafe { + /// assert_eq!(*x.downcast_unchecked::(), 1); + /// } + /// ``` + /// + /// # Safety + /// + /// The contained value must be of type `T`. Calling this method + /// with the incorrect type is *undefined behavior*. + /// + /// [`downcast`]: Self::downcast + #[inline] + #[unstable(feature = "downcast_unchecked", issue = "90850")] + pub unsafe fn downcast_unchecked(self) -> Box { + debug_assert!(self.is::()); + unsafe { + let (raw, alloc): (*mut (dyn Any + Send + Sync), _) = + Box::into_raw_with_allocator(self); + Box::from_raw_in(raw as *mut T, alloc) + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Display for Box { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&**self, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Box { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Pointer for Box { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // It's not possible to extract the inner Uniq directly from the Box, + // instead we cast it to a *const which aliases the Unique + let ptr: *const T = &**self; + fmt::Pointer::fmt(&ptr, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const Deref for Box { + type Target = T; + + fn deref(&self) -> &T { + &**self + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const DerefMut for Box { + fn deref_mut(&mut self) -> &mut T { + &mut **self + } +} + +#[unstable(feature = "receiver_trait", issue = "none")] +impl Receiver for Box {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for Box { + type Item = I::Item; + fn next(&mut self) -> Option { + (**self).next() + } + fn size_hint(&self) -> (usize, Option) { + (**self).size_hint() + } + fn nth(&mut self, n: usize) -> Option { + (**self).nth(n) + } + fn last(self) -> Option { + BoxIter::last(self) + } +} + +trait BoxIter { + type Item; + fn last(self) -> Option; +} + +impl BoxIter for Box { + type Item = I::Item; + default fn last(self) -> Option { + #[inline] + fn some(_: Option, x: T) -> Option { + Some(x) + } + + self.fold(None, some) + } +} + +/// Specialization for sized `I`s that uses `I`s implementation of `last()` +/// instead of the default. +#[stable(feature = "rust1", since = "1.0.0")] +impl BoxIter for Box { + fn last(self) -> Option { + (*self).last() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for Box { + fn next_back(&mut self) -> Option { + (**self).next_back() + } + fn nth_back(&mut self, n: usize) -> Option { + (**self).nth_back(n) + } +} +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for Box { + fn len(&self) -> usize { + (**self).len() + } + fn is_empty(&self) -> bool { + (**self).is_empty() + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Box {} + +#[stable(feature = "boxed_closure_impls", since = "1.35.0")] +impl + ?Sized, A: Allocator> FnOnce for Box { + type Output = >::Output; + + extern "rust-call" fn call_once(self, args: Args) -> Self::Output { + >::call_once(*self, args) + } +} + +#[stable(feature = "boxed_closure_impls", since = "1.35.0")] +impl + ?Sized, A: Allocator> FnMut for Box { + extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output { + >::call_mut(self, args) + } +} + +#[stable(feature = "boxed_closure_impls", since = "1.35.0")] +impl + ?Sized, A: Allocator> Fn for Box { + extern "rust-call" fn call(&self, args: Args) -> Self::Output { + >::call(self, args) + } +} + +#[unstable(feature = "coerce_unsized", issue = "27732")] +impl, U: ?Sized, A: Allocator> CoerceUnsized> for Box {} + +#[unstable(feature = "dispatch_from_dyn", issue = "none")] +impl, U: ?Sized> DispatchFromDyn> for Box {} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "boxed_slice_from_iter", since = "1.32.0")] +impl FromIterator for Box<[I]> { + fn from_iter>(iter: T) -> Self { + iter.into_iter().collect::>().into_boxed_slice() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "box_slice_clone", since = "1.3.0")] +impl Clone for Box<[T], A> { + fn clone(&self) -> Self { + let alloc = Box::allocator(self).clone(); + self.to_vec_in(alloc).into_boxed_slice() + } + + fn clone_from(&mut self, other: &Self) { + if self.len() == other.len() { + self.clone_from_slice(&other); + } else { + *self = other.clone(); + } + } +} + +#[stable(feature = "box_borrow", since = "1.1.0")] +impl borrow::Borrow for Box { + fn borrow(&self) -> &T { + &**self + } +} + +#[stable(feature = "box_borrow", since = "1.1.0")] +impl borrow::BorrowMut for Box { + fn borrow_mut(&mut self) -> &mut T { + &mut **self + } +} + +#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] +impl AsRef for Box { + fn as_ref(&self) -> &T { + &**self + } +} + +#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")] +impl AsMut for Box { + fn as_mut(&mut self) -> &mut T { + &mut **self + } +} + +/* Nota bene + * + * We could have chosen not to add this impl, and instead have written a + * function of Pin> to Pin. Such a function would not be sound, + * because Box implements Unpin even when T does not, as a result of + * this impl. + * + * We chose this API instead of the alternative for a few reasons: + * - Logically, it is helpful to understand pinning in regard to the + * memory region being pointed to. For this reason none of the + * standard library pointer types support projecting through a pin + * (Box is the only pointer type in std for which this would be + * safe.) + * - It is in practice very useful to have Box be unconditionally + * Unpin because of trait objects, for which the structural auto + * trait functionality does not apply (e.g., Box would + * otherwise not be Unpin). + * + * Another type with the same semantics as Box but only a conditional + * implementation of `Unpin` (where `T: Unpin`) would be valid/safe, and + * could have a method to project a Pin from it. + */ +#[stable(feature = "pin", since = "1.33.0")] +#[rustc_const_unstable(feature = "const_box", issue = "92521")] +impl const Unpin for Box where A: 'static {} + +#[unstable(feature = "generator_trait", issue = "43122")] +impl + Unpin, R, A: Allocator> Generator for Box +where + A: 'static, +{ + type Yield = G::Yield; + type Return = G::Return; + + fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState { + G::resume(Pin::new(&mut *self), arg) + } +} + +#[unstable(feature = "generator_trait", issue = "43122")] +impl, R, A: Allocator> Generator for Pin> +where + A: 'static, +{ + type Yield = G::Yield; + type Return = G::Return; + + fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState { + G::resume((*self).as_mut(), arg) + } +} + +#[stable(feature = "futures_api", since = "1.36.0")] +impl Future for Box +where + A: 'static, +{ + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + F::poll(Pin::new(&mut *self), cx) + } +} + +#[unstable(feature = "async_iterator", issue = "79024")] +impl AsyncIterator for Box { + type Item = S::Item; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut **self).poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + (**self).size_hint() + } +} diff --git a/rust/alloc/collections/mod.rs b/rust/alloc/collections/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..1eec265b28f807ed1e42d66f4fe7041a57af7067 --- /dev/null +++ b/rust/alloc/collections/mod.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! Collection types. + +#![stable(feature = "rust1", since = "1.0.0")] + +#[cfg(not(no_global_oom_handling))] +pub mod binary_heap; +#[cfg(not(no_global_oom_handling))] +mod btree; +#[cfg(not(no_global_oom_handling))] +pub mod linked_list; +#[cfg(not(no_global_oom_handling))] +pub mod vec_deque; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +pub mod btree_map { + //! An ordered map based on a B-Tree. + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::btree::map::*; +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +pub mod btree_set { + //! An ordered set based on a B-Tree. + #[stable(feature = "rust1", since = "1.0.0")] + pub use super::btree::set::*; +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(no_inline)] +pub use binary_heap::BinaryHeap; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(no_inline)] +pub use btree_map::BTreeMap; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(no_inline)] +pub use btree_set::BTreeSet; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(no_inline)] +pub use linked_list::LinkedList; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +#[doc(no_inline)] +pub use vec_deque::VecDeque; + +use crate::alloc::{Layout, LayoutError}; +use core::fmt::Display; + +/// The error type for `try_reserve` methods. +#[derive(Clone, PartialEq, Eq, Debug)] +#[stable(feature = "try_reserve", since = "1.57.0")] +pub struct TryReserveError { + kind: TryReserveErrorKind, +} + +impl TryReserveError { + /// Details about the allocation that caused the error + #[inline] + #[must_use] + #[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" + )] + pub fn kind(&self) -> TryReserveErrorKind { + self.kind.clone() + } +} + +/// Details of the allocation that caused a `TryReserveError` +#[derive(Clone, PartialEq, Eq, Debug)] +#[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" +)] +pub enum TryReserveErrorKind { + /// Error due to the computed capacity exceeding the collection's maximum + /// (usually `isize::MAX` bytes). + CapacityOverflow, + + /// The memory allocator returned an error + AllocError { + /// The layout of allocation request that failed + layout: Layout, + + #[doc(hidden)] + #[unstable( + feature = "container_error_extra", + issue = "none", + reason = "\ + Enable exposing the allocator’s custom error value \ + if an associated type is added in the future: \ + https://github.com/rust-lang/wg-allocators/issues/23" + )] + non_exhaustive: (), + }, +} + +#[unstable( + feature = "try_reserve_kind", + reason = "Uncertain how much info should be exposed", + issue = "48043" +)] +impl From for TryReserveError { + #[inline] + fn from(kind: TryReserveErrorKind) -> Self { + Self { kind } + } +} + +#[unstable(feature = "try_reserve_kind", reason = "new API", issue = "48043")] +impl From for TryReserveErrorKind { + /// Always evaluates to [`TryReserveErrorKind::CapacityOverflow`]. + #[inline] + fn from(_: LayoutError) -> Self { + TryReserveErrorKind::CapacityOverflow + } +} + +#[stable(feature = "try_reserve", since = "1.57.0")] +impl Display for TryReserveError { + fn fmt( + &self, + fmt: &mut core::fmt::Formatter<'_>, + ) -> core::result::Result<(), core::fmt::Error> { + fmt.write_str("memory allocation failed")?; + let reason = match self.kind { + TryReserveErrorKind::CapacityOverflow => { + " because the computed capacity exceeded the collection's maximum" + } + TryReserveErrorKind::AllocError { .. } => { + " because the memory allocator returned a error" + } + }; + fmt.write_str(reason) + } +} + +/// An intermediate trait for specialization of `Extend`. +#[doc(hidden)] +trait SpecExtend { + /// Extends `self` with the contents of the given iterator. + fn spec_extend(&mut self, iter: I); +} diff --git a/rust/alloc/lib.rs b/rust/alloc/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..233bcd5e46544c1f60e0ee72ad2281541be686d2 --- /dev/null +++ b/rust/alloc/lib.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! # The Rust core allocation and collections library +//! +//! This library provides smart pointers and collections for managing +//! heap-allocated values. +//! +//! This library, like libcore, normally doesn’t need to be used directly +//! since its contents are re-exported in the [`std` crate](../std/index.html). +//! Crates that use the `#![no_std]` attribute however will typically +//! not depend on `std`, so they’d use this crate instead. +//! +//! ## Boxed values +//! +//! The [`Box`] type is a smart pointer type. There can only be one owner of a +//! [`Box`], and the owner can decide to mutate the contents, which live on the +//! heap. +//! +//! This type can be sent among threads efficiently as the size of a `Box` value +//! is the same as that of a pointer. Tree-like data structures are often built +//! with boxes because each node often has only one owner, the parent. +//! +//! ## Reference counted pointers +//! +//! The [`Rc`] type is a non-threadsafe reference-counted pointer type intended +//! for sharing memory within a thread. An [`Rc`] pointer wraps a type, `T`, and +//! only allows access to `&T`, a shared reference. +//! +//! This type is useful when inherited mutability (such as using [`Box`]) is too +//! constraining for an application, and is often paired with the [`Cell`] or +//! [`RefCell`] types in order to allow mutation. +//! +//! ## Atomically reference counted pointers +//! +//! The [`Arc`] type is the threadsafe equivalent of the [`Rc`] type. It +//! provides all the same functionality of [`Rc`], except it requires that the +//! contained type `T` is shareable. Additionally, [`Arc`][`Arc`] is itself +//! sendable while [`Rc`][`Rc`] is not. +//! +//! This type allows for shared access to the contained data, and is often +//! paired with synchronization primitives such as mutexes to allow mutation of +//! shared resources. +//! +//! ## Collections +//! +//! Implementations of the most common general purpose data structures are +//! defined in this library. They are re-exported through the +//! [standard collections library](../std/collections/index.html). +//! +//! ## Heap interfaces +//! +//! The [`alloc`](alloc/index.html) module defines the low-level interface to the +//! default global allocator. It is not compatible with the libc allocator API. +//! +//! [`Arc`]: sync +//! [`Box`]: boxed +//! [`Cell`]: core::cell +//! [`Rc`]: rc +//! [`RefCell`]: core::cell + +// To run liballoc tests without x.py without ending up with two copies of liballoc, Miri needs to be +// able to "empty" this crate. See . +// rustc itself never sets the feature, so this line has no affect there. +#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))] +#![allow(unused_attributes)] +#![stable(feature = "alloc", since = "1.36.0")] +#![doc( + html_playground_url = "https://play.rust-lang.org/", + issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/", + test(no_crate_inject, attr(allow(unused_variables), deny(warnings))) +)] +#![doc(cfg_hide( + not(test), + not(any(test, bootstrap)), + any(not(feature = "miri-test-libstd"), test, doctest), + no_global_oom_handling, + not(no_global_oom_handling), + target_has_atomic = "ptr" +))] +#![no_std] +#![needs_allocator] +// +// Lints: +#![deny(unsafe_op_in_unsafe_fn)] +#![warn(deprecated_in_future)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] +#![allow(explicit_outlives_requirements)] +// +// Library features: +#![cfg_attr(not(no_global_oom_handling), feature(alloc_c_string))] +#![feature(alloc_layout_extra)] +#![feature(allocator_api)] +#![feature(array_chunks)] +#![feature(array_methods)] +#![feature(array_windows)] +#![feature(assert_matches)] +#![feature(async_iterator)] +#![feature(coerce_unsized)] +#![cfg_attr(not(no_global_oom_handling), feature(const_alloc_error))] +#![feature(const_box)] +#![cfg_attr(not(no_global_oom_handling), feature(const_btree_new))] +#![feature(const_cow_is_borrowed)] +#![feature(const_convert)] +#![feature(const_size_of_val)] +#![feature(const_align_of_val)] +#![feature(const_ptr_read)] +#![feature(const_maybe_uninit_write)] +#![feature(const_maybe_uninit_as_mut_ptr)] +#![feature(const_refs_to_cell)] +#![feature(core_c_str)] +#![feature(core_intrinsics)] +#![feature(core_ffi_c)] +#![feature(const_eval_select)] +#![feature(const_pin)] +#![feature(cstr_from_bytes_until_nul)] +#![feature(dispatch_from_dyn)] +#![feature(exact_size_is_empty)] +#![feature(extend_one)] +#![feature(fmt_internals)] +#![feature(fn_traits)] +#![feature(hasher_prefixfree_extras)] +#![feature(inplace_iteration)] +#![feature(iter_advance_by)] +#![feature(layout_for_ptr)] +#![feature(maybe_uninit_slice)] +#![cfg_attr(test, feature(new_uninit))] +#![feature(nonnull_slice_from_raw_parts)] +#![feature(pattern)] +#![feature(ptr_internals)] +#![feature(ptr_metadata)] +#![feature(ptr_sub_ptr)] +#![feature(receiver_trait)] +#![feature(set_ptr_value)] +#![feature(slice_group_by)] +#![feature(slice_ptr_get)] +#![feature(slice_ptr_len)] +#![feature(slice_range)] +#![feature(str_internals)] +#![feature(strict_provenance)] +#![feature(trusted_len)] +#![feature(trusted_random_access)] +#![feature(try_trait_v2)] +#![feature(unchecked_math)] +#![feature(unicode_internals)] +#![feature(unsize)] +// +// Language features: +#![feature(allocator_internals)] +#![feature(allow_internal_unstable)] +#![feature(associated_type_bounds)] +#![feature(box_syntax)] +#![feature(cfg_sanitize)] +#![feature(const_deref)] +#![feature(const_mut_refs)] +#![feature(const_ptr_write)] +#![feature(const_precise_live_drops)] +#![feature(const_trait_impl)] +#![feature(const_try)] +#![feature(dropck_eyepatch)] +#![feature(exclusive_range_pattern)] +#![feature(fundamental)] +#![cfg_attr(not(test), feature(generator_trait))] +#![feature(hashmap_internals)] +#![feature(lang_items)] +#![feature(let_else)] +#![feature(min_specialization)] +#![feature(negative_impls)] +#![feature(never_type)] +#![feature(nll)] // Not necessary, but here to test the `nll` feature. +#![feature(rustc_allow_const_fn_unstable)] +#![feature(rustc_attrs)] +#![feature(slice_internals)] +#![feature(staged_api)] +#![cfg_attr(test, feature(test))] +#![feature(unboxed_closures)] +#![feature(unsized_fn_params)] +#![feature(c_unwind)] +// +// Rustdoc features: +#![feature(doc_cfg)] +#![feature(doc_cfg_hide)] +// Technically, this is a bug in rustdoc: rustdoc sees the documentation on `#[lang = slice_alloc]` +// blocks is for `&[T]`, which also has documentation using this feature in `core`, and gets mad +// that the feature-gate isn't enabled. Ideally, it wouldn't check for the feature gate for docs +// from other crates, but since this can only appear for lang items, it doesn't seem worth fixing. +#![feature(intra_doc_pointers)] + +// Allow testing this library +#[cfg(test)] +#[macro_use] +extern crate std; +#[cfg(test)] +extern crate test; + +// Module with internal macros used by other modules (needs to be included before other modules). +#[cfg(not(no_macros))] +#[macro_use] +mod macros; + +mod raw_vec; + +// Heaps provided for low-level allocation strategies + +pub mod alloc; + +// Primitive types using the heaps above + +// Need to conditionally define the mod from `boxed.rs` to avoid +// duplicating the lang-items when building in test cfg; but also need +// to allow code to have `use boxed::Box;` declarations. +#[cfg(not(test))] +pub mod boxed; +#[cfg(test)] +mod boxed { + pub use std::boxed::Box; +} +pub mod borrow; +pub mod collections; +#[cfg(not(no_global_oom_handling))] +pub mod ffi; +#[cfg(not(no_fmt))] +pub mod fmt; +#[cfg(not(no_rc))] +pub mod rc; +pub mod slice; +#[cfg(not(no_str))] +pub mod str; +#[cfg(not(no_string))] +pub mod string; +#[cfg(not(no_sync))] +#[cfg(target_has_atomic = "ptr")] +pub mod sync; +#[cfg(all(not(no_global_oom_handling), target_has_atomic = "ptr"))] +pub mod task; +#[cfg(test)] +mod tests; +pub mod vec; + +#[doc(hidden)] +#[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")] +pub mod __export { + pub use core::format_args; +} diff --git a/rust/alloc/raw_vec.rs b/rust/alloc/raw_vec.rs new file mode 100644 index 0000000000000000000000000000000000000000..daf5f2da7168e2365c03298f64319b0ed2278b80 --- /dev/null +++ b/rust/alloc/raw_vec.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#![unstable(feature = "raw_vec_internals", reason = "unstable const warnings", issue = "none")] + +use core::alloc::LayoutError; +use core::cmp; +use core::intrinsics; +use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::ops::Drop; +use core::ptr::{self, NonNull, Unique}; +use core::slice; + +#[cfg(not(no_global_oom_handling))] +use crate::alloc::handle_alloc_error; +use crate::alloc::{Allocator, Global, Layout}; +use crate::boxed::Box; +use crate::collections::TryReserveError; +use crate::collections::TryReserveErrorKind::*; + +#[cfg(test)] +mod tests; + +#[cfg(not(no_global_oom_handling))] +enum AllocInit { + /// The contents of the new memory are uninitialized. + Uninitialized, + /// The new memory is guaranteed to be zeroed. + Zeroed, +} + +/// A low-level utility for more ergonomically allocating, reallocating, and deallocating +/// a buffer of memory on the heap without having to worry about all the corner cases +/// involved. This type is excellent for building your own data structures like Vec and VecDeque. +/// In particular: +/// +/// * Produces `Unique::dangling()` on zero-sized types. +/// * Produces `Unique::dangling()` on zero-length allocations. +/// * Avoids freeing `Unique::dangling()`. +/// * Catches all overflows in capacity computations (promotes them to "capacity overflow" panics). +/// * Guards against 32-bit systems allocating more than isize::MAX bytes. +/// * Guards against overflowing your length. +/// * Calls `handle_alloc_error` for fallible allocations. +/// * Contains a `ptr::Unique` and thus endows the user with all related benefits. +/// * Uses the excess returned from the allocator to use the largest available capacity. +/// +/// This type does not in anyway inspect the memory that it manages. When dropped it *will* +/// free its memory, but it *won't* try to drop its contents. It is up to the user of `RawVec` +/// to handle the actual things *stored* inside of a `RawVec`. +/// +/// Note that the excess of a zero-sized types is always infinite, so `capacity()` always returns +/// `usize::MAX`. This means that you need to be careful when round-tripping this type with a +/// `Box<[T]>`, since `capacity()` won't yield the length. +#[allow(missing_debug_implementations)] +pub(crate) struct RawVec { + ptr: Unique, + cap: usize, + alloc: A, +} + +impl RawVec { + /// HACK(Centril): This exists because stable `const fn` can only call stable `const fn`, so + /// they cannot call `Self::new()`. + /// + /// If you change `RawVec::new` or dependencies, please take care to not introduce anything + /// that would truly const-call something unstable. + pub const NEW: Self = Self::new(); + + /// Creates the biggest possible `RawVec` (on the system heap) + /// without allocating. If `T` has positive size, then this makes a + /// `RawVec` with capacity `0`. If `T` is zero-sized, then it makes a + /// `RawVec` with capacity `usize::MAX`. Useful for implementing + /// delayed allocation. + #[must_use] + pub const fn new() -> Self { + Self::new_in(Global) + } + + /// Creates a `RawVec` (on the system heap) with exactly the + /// capacity and alignment requirements for a `[T; capacity]`. This is + /// equivalent to calling `RawVec::new` when `capacity` is `0` or `T` is + /// zero-sized. Note that if `T` is zero-sized this means you will + /// *not* get a `RawVec` with the requested capacity. + /// + /// # Panics + /// + /// Panics if the requested capacity exceeds `isize::MAX` bytes. + /// + /// # Aborts + /// + /// Aborts on OOM. + #[cfg(not(any(no_global_oom_handling, test)))] + #[must_use] + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_in(capacity, Global) + } + + /// Like `with_capacity`, but guarantees the buffer is zeroed. + #[cfg(not(any(no_global_oom_handling, test)))] + #[must_use] + #[inline] + pub fn with_capacity_zeroed(capacity: usize) -> Self { + Self::with_capacity_zeroed_in(capacity, Global) + } +} + +impl RawVec { + // Tiny Vecs are dumb. Skip to: + // - 8 if the element size is 1, because any heap allocators is likely + // to round up a request of less than 8 bytes to at least 8 bytes. + // - 4 if elements are moderate-sized (<= 1 KiB). + // - 1 otherwise, to avoid wasting too much space for very short Vecs. + pub(crate) const MIN_NON_ZERO_CAP: usize = if mem::size_of::() == 1 { + 8 + } else if mem::size_of::() <= 1024 { + 4 + } else { + 1 + }; + + /// Like `new`, but parameterized over the choice of allocator for + /// the returned `RawVec`. + pub const fn new_in(alloc: A) -> Self { + // `cap: 0` means "unallocated". zero-sized types are ignored. + Self { ptr: Unique::dangling(), cap: 0, alloc } + } + + /// Like `with_capacity`, but parameterized over the choice of + /// allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Self::allocate_in(capacity, AllocInit::Uninitialized, alloc) + } + + /// Like `with_capacity_zeroed`, but parameterized over the choice + /// of allocator for the returned `RawVec`. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { + Self::allocate_in(capacity, AllocInit::Zeroed, alloc) + } + + /// Converts the entire buffer into `Box<[MaybeUninit]>` with the specified `len`. + /// + /// Note that this will correctly reconstitute any `cap` changes + /// that may have been performed. (See description of type for details.) + /// + /// # Safety + /// + /// * `len` must be greater than or equal to the most recently requested capacity, and + /// * `len` must be less than or equal to `self.capacity()`. + /// + /// Note, that the requested capacity and `self.capacity()` could differ, as + /// an allocator could overallocate and return a greater memory block than requested. + pub unsafe fn into_box(self, len: usize) -> Box<[MaybeUninit], A> { + // Sanity-check one half of the safety requirement (we cannot check the other half). + debug_assert!( + len <= self.capacity(), + "`len` must be smaller than or equal to `self.capacity()`" + ); + + let me = ManuallyDrop::new(self); + unsafe { + let slice = slice::from_raw_parts_mut(me.ptr() as *mut MaybeUninit, len); + Box::from_raw_in(slice, ptr::read(&me.alloc)) + } + } + + #[cfg(not(no_global_oom_handling))] + fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self { + // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. + if mem::size_of::() == 0 || capacity == 0 { + Self::new_in(alloc) + } else { + // We avoid `unwrap_or_else` here because it bloats the amount of + // LLVM IR generated. + let layout = match Layout::array::(capacity) { + Ok(layout) => layout, + Err(_) => capacity_overflow(), + }; + match alloc_guard(layout.size()) { + Ok(_) => {} + Err(_) => capacity_overflow(), + } + let result = match init { + AllocInit::Uninitialized => alloc.allocate(layout), + AllocInit::Zeroed => alloc.allocate_zeroed(layout), + }; + let ptr = match result { + Ok(ptr) => ptr, + Err(_) => handle_alloc_error(layout), + }; + + // Allocators currently return a `NonNull<[u8]>` whose length + // matches the size requested. If that ever changes, the capacity + // here should change to `ptr.len() / mem::size_of::()`. + Self { + ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }, + cap: capacity, + alloc, + } + } + } + + /// Reconstitutes a `RawVec` from a pointer, capacity, and allocator. + /// + /// # Safety + /// + /// The `ptr` must be allocated (via the given allocator `alloc`), and with the given + /// `capacity`. + /// The `capacity` cannot exceed `isize::MAX` for sized types. (only a concern on 32-bit + /// systems). ZST vectors may have a capacity up to `usize::MAX`. + /// If the `ptr` and `capacity` come from a `RawVec` created via `alloc`, then this is + /// guaranteed. + #[inline] + pub unsafe fn from_raw_parts_in(ptr: *mut T, capacity: usize, alloc: A) -> Self { + Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap: capacity, alloc } + } + + /// Gets a raw pointer to the start of the allocation. Note that this is + /// `Unique::dangling()` if `capacity == 0` or `T` is zero-sized. In the former case, you must + /// be careful. + #[inline] + pub fn ptr(&self) -> *mut T { + self.ptr.as_ptr() + } + + /// Gets the capacity of the allocation. + /// + /// This will always be `usize::MAX` if `T` is zero-sized. + #[inline(always)] + pub fn capacity(&self) -> usize { + if mem::size_of::() == 0 { usize::MAX } else { self.cap } + } + + /// Returns a shared reference to the allocator backing this `RawVec`. + pub fn allocator(&self) -> &A { + &self.alloc + } + + fn current_memory(&self) -> Option<(NonNull, Layout)> { + if mem::size_of::() == 0 || self.cap == 0 { + None + } else { + // We have an allocated chunk of memory, so we can bypass runtime + // checks to get our current layout. + unsafe { + let layout = Layout::array::(self.cap).unwrap_unchecked(); + Some((self.ptr.cast().into(), layout)) + } + } + } + + /// Ensures that the buffer contains at least enough space to hold `len + + /// additional` elements. If it doesn't already have enough capacity, will + /// reallocate enough space plus comfortable slack space to get amortized + /// *O*(1) behavior. Will limit this behavior if it would needlessly cause + /// itself to panic. + /// + /// If `len` exceeds `self.capacity()`, this may fail to actually allocate + /// the requested space. This is not really unsafe, but the unsafe + /// code *you* write that relies on the behavior of this function may break. + /// + /// This is ideal for implementing a bulk-push operation like `extend`. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Aborts + /// + /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn reserve(&mut self, len: usize, additional: usize) { + // Callers expect this function to be very cheap when there is already sufficient capacity. + // Therefore, we move all the resizing and error-handling logic from grow_amortized and + // handle_reserve behind a call, while making sure that this function is likely to be + // inlined as just a comparison and a call if the comparison fails. + #[cold] + fn do_reserve_and_handle( + slf: &mut RawVec, + len: usize, + additional: usize, + ) { + handle_reserve(slf.grow_amortized(len, additional)); + } + + if self.needs_to_grow(len, additional) { + do_reserve_and_handle(self, len, additional); + } + } + + /// A specialized version of `reserve()` used only by the hot and + /// oft-instantiated `Vec::push()`, which does its own capacity check. + #[cfg(not(no_global_oom_handling))] + #[inline(never)] + pub fn reserve_for_push(&mut self, len: usize) { + handle_reserve(self.grow_amortized(len, 1)); + } + + /// The same as `reserve`, but returns on errors instead of panicking or aborting. + pub fn try_reserve(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + if self.needs_to_grow(len, additional) { + self.grow_amortized(len, additional) + } else { + Ok(()) + } + } + + /// The same as `reserve_for_push`, but returns on errors instead of panicking or aborting. + #[inline(never)] + pub fn try_reserve_for_push(&mut self, len: usize) -> Result<(), TryReserveError> { + self.grow_amortized(len, 1) + } + + /// Ensures that the buffer contains at least enough space to hold `len + + /// additional` elements. If it doesn't already, will reallocate the + /// minimum possible amount of memory necessary. Generally this will be + /// exactly the amount of memory necessary, but in principle the allocator + /// is free to give back more than we asked for. + /// + /// If `len` exceeds `self.capacity()`, this may fail to actually allocate + /// the requested space. This is not really unsafe, but the unsafe code + /// *you* write that relies on the behavior of this function may break. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Aborts + /// + /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] + pub fn reserve_exact(&mut self, len: usize, additional: usize) { + handle_reserve(self.try_reserve_exact(len, additional)); + } + + /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. + pub fn try_reserve_exact( + &mut self, + len: usize, + additional: usize, + ) -> Result<(), TryReserveError> { + if self.needs_to_grow(len, additional) { self.grow_exact(len, additional) } else { Ok(()) } + } + + /// Shrinks the buffer down to the specified capacity. If the given amount + /// is 0, actually completely deallocates. + /// + /// # Panics + /// + /// Panics if the given amount is *larger* than the current capacity. + /// + /// # Aborts + /// + /// Aborts on OOM. + #[cfg(not(no_global_oom_handling))] + pub fn shrink_to_fit(&mut self, cap: usize) { + handle_reserve(self.shrink(cap)); + } +} + +impl RawVec { + /// Returns if the buffer needs to grow to fulfill the needed extra capacity. + /// Mainly used to make inlining reserve-calls possible without inlining `grow`. + fn needs_to_grow(&self, len: usize, additional: usize) -> bool { + additional > self.capacity().wrapping_sub(len) + } + + fn set_ptr_and_cap(&mut self, ptr: NonNull<[u8]>, cap: usize) { + // Allocators currently return a `NonNull<[u8]>` whose length matches + // the size requested. If that ever changes, the capacity here should + // change to `ptr.len() / mem::size_of::()`. + self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) }; + self.cap = cap; + } + + // This method is usually instantiated many times. So we want it to be as + // small as possible, to improve compile times. But we also want as much of + // its contents to be statically computable as possible, to make the + // generated code run faster. Therefore, this method is carefully written + // so that all of the code that depends on `T` is within it, while as much + // of the code that doesn't depend on `T` as possible is in functions that + // are non-generic over `T`. + fn grow_amortized(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + // This is ensured by the calling contexts. + debug_assert!(additional > 0); + + if mem::size_of::() == 0 { + // Since we return a capacity of `usize::MAX` when `elem_size` is + // 0, getting to here necessarily means the `RawVec` is overfull. + return Err(CapacityOverflow.into()); + } + + // Nothing we can really do about these checks, sadly. + let required_cap = len.checked_add(additional).ok_or(CapacityOverflow)?; + + // This guarantees exponential growth. The doubling cannot overflow + // because `cap <= isize::MAX` and the type of `cap` is `usize`. + let cap = cmp::max(self.cap * 2, required_cap); + let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap); + + let new_layout = Layout::array::(cap); + + // `finish_grow` is non-generic over `T`. + let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; + self.set_ptr_and_cap(ptr, cap); + Ok(()) + } + + // The constraints on this method are much the same as those on + // `grow_amortized`, but this method is usually instantiated less often so + // it's less critical. + fn grow_exact(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> { + if mem::size_of::() == 0 { + // Since we return a capacity of `usize::MAX` when the type size is + // 0, getting to here necessarily means the `RawVec` is overfull. + return Err(CapacityOverflow.into()); + } + + let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; + let new_layout = Layout::array::(cap); + + // `finish_grow` is non-generic over `T`. + let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?; + self.set_ptr_and_cap(ptr, cap); + Ok(()) + } + + #[allow(dead_code)] + fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> { + assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity"); + + let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; + + let ptr = unsafe { + // `Layout::array` cannot overflow here because it would have + // overflowed earlier when capacity was larger. + let new_layout = Layout::array::(cap).unwrap_unchecked(); + self.alloc + .shrink(ptr, layout, new_layout) + .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? + }; + self.set_ptr_and_cap(ptr, cap); + Ok(()) + } +} + +// This function is outside `RawVec` to minimize compile times. See the comment +// above `RawVec::grow_amortized` for details. (The `A` parameter isn't +// significant, because the number of different `A` types seen in practice is +// much smaller than the number of `T` types.) +#[inline(never)] +fn finish_grow( + new_layout: Result, + current_memory: Option<(NonNull, Layout)>, + alloc: &mut A, +) -> Result, TryReserveError> +where + A: Allocator, +{ + // Check for the error here to minimize the size of `RawVec::grow_*`. + let new_layout = new_layout.map_err(|_| CapacityOverflow)?; + + alloc_guard(new_layout.size())?; + + let memory = if let Some((ptr, old_layout)) = current_memory { + debug_assert_eq!(old_layout.align(), new_layout.align()); + unsafe { + // The allocator checks for alignment equality + intrinsics::assume(old_layout.align() == new_layout.align()); + alloc.grow(ptr, old_layout, new_layout) + } + } else { + alloc.allocate(new_layout) + }; + + memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into()) +} + +unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec { + /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. + fn drop(&mut self) { + if let Some((ptr, layout)) = self.current_memory() { + unsafe { self.alloc.deallocate(ptr, layout) } + } + } +} + +// Central function for reserve error handling. +#[cfg(not(no_global_oom_handling))] +#[inline] +fn handle_reserve(result: Result<(), TryReserveError>) { + match result.map_err(|e| e.kind()) { + Err(CapacityOverflow) => capacity_overflow(), + Err(AllocError { layout, .. }) => handle_alloc_error(layout), + Ok(()) => { /* yay */ } + } +} + +// We need to guarantee the following: +// * We don't ever allocate `> isize::MAX` byte-size objects. +// * We don't overflow `usize::MAX` and actually allocate too little. +// +// On 64-bit we just need to check for overflow since trying to allocate +// `> isize::MAX` bytes will surely fail. On 32-bit and 16-bit we need to add +// an extra guard for this in case we're running on a platform which can use +// all 4GB in user-space, e.g., PAE or x32. + +#[inline] +fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> { + if usize::BITS < 64 && alloc_size > isize::MAX as usize { + Err(CapacityOverflow.into()) + } else { + Ok(()) + } +} + +// One central function responsible for reporting capacity overflows. This'll +// ensure that the code generation related to these panics is minimal as there's +// only one location which panics rather than a bunch throughout the module. +#[cfg(not(no_global_oom_handling))] +fn capacity_overflow() -> ! { + panic!("capacity overflow"); +} diff --git a/rust/alloc/slice.rs b/rust/alloc/slice.rs new file mode 100644 index 0000000000000000000000000000000000000000..e444e97fa14538b5fab07dab4ab1438c86c56821 --- /dev/null +++ b/rust/alloc/slice.rs @@ -0,0 +1,1204 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A dynamically-sized view into a contiguous sequence, `[T]`. +//! +//! *[See also the slice primitive type](slice).* +//! +//! Slices are a view into a block of memory represented as a pointer and a +//! length. +//! +//! ``` +//! // slicing a Vec +//! let vec = vec![1, 2, 3]; +//! let int_slice = &vec[..]; +//! // coercing an array to a slice +//! let str_slice: &[&str] = &["one", "two", "three"]; +//! ``` +//! +//! Slices are either mutable or shared. The shared slice type is `&[T]`, +//! while the mutable slice type is `&mut [T]`, where `T` represents the element +//! type. For example, you can mutate the block of memory that a mutable slice +//! points to: +//! +//! ``` +//! let x = &mut [1, 2, 3]; +//! x[1] = 7; +//! assert_eq!(x, &[1, 7, 3]); +//! ``` +//! +//! Here are some of the things this module contains: +//! +//! ## Structs +//! +//! There are several structs that are useful for slices, such as [`Iter`], which +//! represents iteration over a slice. +//! +//! ## Trait Implementations +//! +//! There are several implementations of common traits for slices. Some examples +//! include: +//! +//! * [`Clone`] +//! * [`Eq`], [`Ord`] - for slices whose element type are [`Eq`] or [`Ord`]. +//! * [`Hash`] - for slices whose element type is [`Hash`]. +//! +//! ## Iteration +//! +//! The slices implement `IntoIterator`. The iterator yields references to the +//! slice elements. +//! +//! ``` +//! let numbers = &[0, 1, 2]; +//! for n in numbers { +//! println!("{n} is a number!"); +//! } +//! ``` +//! +//! The mutable slice yields mutable references to the elements: +//! +//! ``` +//! let mut scores = [7, 8, 9]; +//! for score in &mut scores[..] { +//! *score += 1; +//! } +//! ``` +//! +//! This iterator yields mutable references to the slice's elements, so while +//! the element type of the slice is `i32`, the element type of the iterator is +//! `&mut i32`. +//! +//! * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default +//! iterators. +//! * Further methods that return iterators are [`.split`], [`.splitn`], +//! [`.chunks`], [`.windows`] and more. +//! +//! [`Hash`]: core::hash::Hash +//! [`.iter`]: slice::iter +//! [`.iter_mut`]: slice::iter_mut +//! [`.split`]: slice::split +//! [`.splitn`]: slice::splitn +//! [`.chunks`]: slice::chunks +//! [`.windows`]: slice::windows +#![stable(feature = "rust1", since = "1.0.0")] +// Many of the usings in this module are only used in the test configuration. +// It's cleaner to just turn off the unused_imports warning than to fix them. +#![cfg_attr(test, allow(unused_imports, dead_code))] + +use core::borrow::{Borrow, BorrowMut}; +#[cfg(not(no_global_oom_handling))] +use core::cmp::Ordering::{self, Less}; +#[cfg(not(no_global_oom_handling))] +use core::mem; +#[cfg(not(no_global_oom_handling))] +use core::mem::size_of; +#[cfg(not(no_global_oom_handling))] +use core::ptr; + +use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] +use crate::alloc::Global; +#[cfg(not(no_global_oom_handling))] +use crate::borrow::ToOwned; +use crate::boxed::Box; +use crate::vec::Vec; + +#[unstable(feature = "slice_range", issue = "76393")] +pub use core::slice::range; +#[unstable(feature = "array_chunks", issue = "74985")] +pub use core::slice::ArrayChunks; +#[unstable(feature = "array_chunks", issue = "74985")] +pub use core::slice::ArrayChunksMut; +#[unstable(feature = "array_windows", issue = "75027")] +pub use core::slice::ArrayWindows; +#[stable(feature = "inherent_ascii_escape", since = "1.60.0")] +pub use core::slice::EscapeAscii; +#[stable(feature = "slice_get_slice", since = "1.28.0")] +pub use core::slice::SliceIndex; +#[stable(feature = "from_ref", since = "1.28.0")] +pub use core::slice::{from_mut, from_ref}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::slice::{from_raw_parts, from_raw_parts_mut}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::slice::{Chunks, Windows}; +#[stable(feature = "chunks_exact", since = "1.31.0")] +pub use core::slice::{ChunksExact, ChunksExactMut}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::slice::{ChunksMut, Split, SplitMut}; +#[unstable(feature = "slice_group_by", issue = "80552")] +pub use core::slice::{GroupBy, GroupByMut}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::slice::{Iter, IterMut}; +#[stable(feature = "rchunks", since = "1.31.0")] +pub use core::slice::{RChunks, RChunksExact, RChunksExactMut, RChunksMut}; +#[stable(feature = "slice_rsplit", since = "1.27.0")] +pub use core::slice::{RSplit, RSplitMut}; +#[stable(feature = "rust1", since = "1.0.0")] +pub use core::slice::{RSplitN, RSplitNMut, SplitN, SplitNMut}; +#[stable(feature = "split_inclusive", since = "1.51.0")] +pub use core::slice::{SplitInclusive, SplitInclusiveMut}; + +//////////////////////////////////////////////////////////////////////////////// +// Basic slice extension methods +//////////////////////////////////////////////////////////////////////////////// + +// HACK(japaric) needed for the implementation of `vec!` macro during testing +// N.B., see the `hack` module in this file for more details. +#[cfg(test)] +pub use hack::into_vec; + +// HACK(japaric) needed for the implementation of `Vec::clone` during testing +// N.B., see the `hack` module in this file for more details. +#[cfg(test)] +pub use hack::to_vec; + +// HACK(japaric): With cfg(test) `impl [T]` is not available, these three +// functions are actually methods that are in `impl [T]` but not in +// `core::slice::SliceExt` - we need to supply these functions for the +// `test_permutations` test +pub(crate) mod hack { + use core::alloc::Allocator; + + use crate::boxed::Box; + use crate::vec::Vec; + + // We shouldn't add inline attribute to this since this is used in + // `vec!` macro mostly and causes perf regression. See #71204 for + // discussion and perf results. + pub fn into_vec(b: Box<[T], A>) -> Vec { + unsafe { + let len = b.len(); + let (b, alloc) = Box::into_raw_with_allocator(b); + Vec::from_raw_parts_in(b as *mut T, len, len, alloc) + } + } + + #[cfg(not(no_global_oom_handling))] + #[inline] + pub fn to_vec(s: &[T], alloc: A) -> Vec { + T::to_vec(s, alloc) + } + + #[cfg(not(no_global_oom_handling))] + pub trait ConvertVec { + fn to_vec(s: &[Self], alloc: A) -> Vec + where + Self: Sized; + } + + #[cfg(not(no_global_oom_handling))] + impl ConvertVec for T { + #[inline] + default fn to_vec(s: &[Self], alloc: A) -> Vec { + struct DropGuard<'a, T, A: Allocator> { + vec: &'a mut Vec, + num_init: usize, + } + impl<'a, T, A: Allocator> Drop for DropGuard<'a, T, A> { + #[inline] + fn drop(&mut self) { + // SAFETY: + // items were marked initialized in the loop below + unsafe { + self.vec.set_len(self.num_init); + } + } + } + let mut vec = Vec::with_capacity_in(s.len(), alloc); + let mut guard = DropGuard { vec: &mut vec, num_init: 0 }; + let slots = guard.vec.spare_capacity_mut(); + // .take(slots.len()) is necessary for LLVM to remove bounds checks + // and has better codegen than zip. + for (i, b) in s.iter().enumerate().take(slots.len()) { + guard.num_init = i; + slots[i].write(b.clone()); + } + core::mem::forget(guard); + // SAFETY: + // the vec was allocated and initialized above to at least this length. + unsafe { + vec.set_len(s.len()); + } + vec + } + } + + #[cfg(not(no_global_oom_handling))] + impl ConvertVec for T { + #[inline] + fn to_vec(s: &[Self], alloc: A) -> Vec { + let mut v = Vec::with_capacity_in(s.len(), alloc); + // SAFETY: + // allocated above with the capacity of `s`, and initialize to `s.len()` in + // ptr::copy_to_non_overlapping below. + unsafe { + s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), s.len()); + v.set_len(s.len()); + } + v + } + } +} + +#[cfg(not(test))] +impl [T] { + /// Sorts the slice. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`sort_unstable`](slice::sort_unstable). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive, iterative merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage half the size of `self`, but for short slices a + /// non-allocating insertion sort is used instead. + /// + /// # Examples + /// + /// ``` + /// let mut v = [-5, 4, 1, -3, 2]; + /// + /// v.sort(); + /// assert!(v == [-5, -3, 1, 2, 4]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn sort(&mut self) + where + T: Ord, + { + merge_sort(self, |a, b| a.lt(b)); + } + + /// Sorts the slice with a comparator function. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case. + /// + /// The comparator function must define a total ordering for the elements in the slice. If + /// the ordering is not total, the order of the elements is unspecified. An order is a + /// total order if it is (for all `a`, `b` and `c`): + /// + /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and + /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`. + /// + /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use + /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`. + /// + /// ``` + /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0]; + /// floats.sort_by(|a, b| a.partial_cmp(b).unwrap()); + /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]); + /// ``` + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`sort_unstable_by`](slice::sort_unstable_by). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive, iterative merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage half the size of `self`, but for short slices a + /// non-allocating insertion sort is used instead. + /// + /// # Examples + /// + /// ``` + /// let mut v = [5, 4, 1, 3, 2]; + /// v.sort_by(|a, b| a.cmp(b)); + /// assert!(v == [1, 2, 3, 4, 5]); + /// + /// // reverse sorting + /// v.sort_by(|a, b| b.cmp(a)); + /// assert!(v == [5, 4, 3, 2, 1]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn sort_by(&mut self, mut compare: F) + where + F: FnMut(&T, &T) -> Ordering, + { + merge_sort(self, |a, b| compare(a, b) == Less); + } + + /// Sorts the slice with a key extraction function. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). + /// + /// For expensive key functions (e.g. functions that are not simple property accesses or + /// basic operations), [`sort_by_cached_key`](slice::sort_by_cached_key) is likely to be + /// significantly faster, as it does not recompute element keys. + /// + /// When applicable, unstable sorting is preferred because it is generally faster than stable + /// sorting and it doesn't allocate auxiliary memory. + /// See [`sort_unstable_by_key`](slice::sort_unstable_by_key). + /// + /// # Current implementation + /// + /// The current algorithm is an adaptive, iterative merge sort inspired by + /// [timsort](https://en.wikipedia.org/wiki/Timsort). + /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of + /// two or more sorted sequences concatenated one after another. + /// + /// Also, it allocates temporary storage half the size of `self`, but for short slices a + /// non-allocating insertion sort is used instead. + /// + /// # Examples + /// + /// ``` + /// let mut v = [-5i32, 4, 1, -3, 2]; + /// + /// v.sort_by_key(|k| k.abs()); + /// assert!(v == [1, 2, -3, 4, -5]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[stable(feature = "slice_sort_by_key", since = "1.7.0")] + #[inline] + pub fn sort_by_key(&mut self, mut f: F) + where + F: FnMut(&T) -> K, + K: Ord, + { + merge_sort(self, |a, b| f(a).lt(&f(b))); + } + + /// Sorts the slice with a key extraction function. + /// + /// During sorting, the key function is called at most once per element, by using + /// temporary storage to remember the results of key evaluation. + /// The order of calls to the key function is unspecified and may change in future versions + /// of the standard library. + /// + /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* + *n* \* log(*n*)) + /// worst-case, where the key function is *O*(*m*). + /// + /// For simple key functions (e.g., functions that are property accesses or + /// basic operations), [`sort_by_key`](slice::sort_by_key) is likely to be + /// faster. + /// + /// # Current implementation + /// + /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters, + /// which combines the fast average case of randomized quicksort with the fast worst case of + /// heapsort, while achieving linear time on slices with certain patterns. It uses some + /// randomization to avoid degenerate cases, but with a fixed seed to always provide + /// deterministic behavior. + /// + /// In the worst case, the algorithm allocates temporary storage in a `Vec<(K, usize)>` the + /// length of the slice. + /// + /// # Examples + /// + /// ``` + /// let mut v = [-5i32, 4, 32, -3, 2]; + /// + /// v.sort_by_cached_key(|k| k.to_string()); + /// assert!(v == [-3, -5, 2, 32, 4]); + /// ``` + /// + /// [pdqsort]: https://github.com/orlp/pdqsort + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[stable(feature = "slice_sort_by_cached_key", since = "1.34.0")] + #[inline] + pub fn sort_by_cached_key(&mut self, f: F) + where + F: FnMut(&T) -> K, + K: Ord, + { + // Helper macro for indexing our vector by the smallest possible type, to reduce allocation. + macro_rules! sort_by_key { + ($t:ty, $slice:ident, $f:ident) => {{ + let mut indices: Vec<_> = + $slice.iter().map($f).enumerate().map(|(i, k)| (k, i as $t)).collect(); + // The elements of `indices` are unique, as they are indexed, so any sort will be + // stable with respect to the original slice. We use `sort_unstable` here because + // it requires less memory allocation. + indices.sort_unstable(); + for i in 0..$slice.len() { + let mut index = indices[i].1; + while (index as usize) < i { + index = indices[index as usize].1; + } + indices[i].1 = index; + $slice.swap(i, index as usize); + } + }}; + } + + let sz_u8 = mem::size_of::<(K, u8)>(); + let sz_u16 = mem::size_of::<(K, u16)>(); + let sz_u32 = mem::size_of::<(K, u32)>(); + let sz_usize = mem::size_of::<(K, usize)>(); + + let len = self.len(); + if len < 2 { + return; + } + if sz_u8 < sz_u16 && len <= (u8::MAX as usize) { + return sort_by_key!(u8, self, f); + } + if sz_u16 < sz_u32 && len <= (u16::MAX as usize) { + return sort_by_key!(u16, self, f); + } + if sz_u32 < sz_usize && len <= (u32::MAX as usize) { + return sort_by_key!(u32, self, f); + } + sort_by_key!(usize, self, f) + } + + /// Copies `self` into a new `Vec`. + /// + /// # Examples + /// + /// ``` + /// let s = [10, 40, 30]; + /// let x = s.to_vec(); + /// // Here, `s` and `x` can be modified independently. + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[rustc_conversion_suggestion] + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn to_vec(&self) -> Vec + where + T: Clone, + { + self.to_vec_in(Global) + } + + /// Copies `self` into a new `Vec` with an allocator. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let s = [10, 40, 30]; + /// let x = s.to_vec_in(System); + /// // Here, `s` and `x` can be modified independently. + /// ``` + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn to_vec_in(&self, alloc: A) -> Vec + where + T: Clone, + { + // N.B., see the `hack` module in this file for more details. + hack::to_vec(self, alloc) + } + + /// Converts `self` into a vector without clones or allocation. + /// + /// The resulting vector can be converted back into a box via + /// `Vec`'s `into_boxed_slice` method. + /// + /// # Examples + /// + /// ``` + /// let s: Box<[i32]> = Box::new([10, 40, 30]); + /// let x = s.into_vec(); + /// // `s` cannot be used anymore because it has been converted into `x`. + /// + /// assert_eq!(x, vec![10, 40, 30]); + /// ``` + #[rustc_allow_incoherent_impl] + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn into_vec(self: Box) -> Vec { + // N.B., see the `hack` module in this file for more details. + hack::into_vec(self) + } + + /// Creates a vector by repeating a slice `n` times. + /// + /// # Panics + /// + /// This function will panic if the capacity would overflow. + /// + /// # Examples + /// + /// Basic usage: + /// + /// ``` + /// assert_eq!([1, 2].repeat(3), vec![1, 2, 1, 2, 1, 2]); + /// ``` + /// + /// A panic upon overflow: + /// + /// ```should_panic + /// // this will panic at runtime + /// b"0123456789abcdef".repeat(usize::MAX); + /// ``` + #[rustc_allow_incoherent_impl] + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "repeat_generic_slice", since = "1.40.0")] + pub fn repeat(&self, n: usize) -> Vec + where + T: Copy, + { + if n == 0 { + return Vec::new(); + } + + // If `n` is larger than zero, it can be split as + // `n = 2^expn + rem (2^expn > rem, expn >= 0, rem >= 0)`. + // `2^expn` is the number represented by the leftmost '1' bit of `n`, + // and `rem` is the remaining part of `n`. + + // Using `Vec` to access `set_len()`. + let capacity = self.len().checked_mul(n).expect("capacity overflow"); + let mut buf = Vec::with_capacity(capacity); + + // `2^expn` repetition is done by doubling `buf` `expn`-times. + buf.extend(self); + { + let mut m = n >> 1; + // If `m > 0`, there are remaining bits up to the leftmost '1'. + while m > 0 { + // `buf.extend(buf)`: + unsafe { + ptr::copy_nonoverlapping( + buf.as_ptr(), + (buf.as_mut_ptr() as *mut T).add(buf.len()), + buf.len(), + ); + // `buf` has capacity of `self.len() * n`. + let buf_len = buf.len(); + buf.set_len(buf_len * 2); + } + + m >>= 1; + } + } + + // `rem` (`= n - 2^expn`) repetition is done by copying + // first `rem` repetitions from `buf` itself. + let rem_len = capacity - buf.len(); // `self.len() * rem` + if rem_len > 0 { + // `buf.extend(buf[0 .. rem_len])`: + unsafe { + // This is non-overlapping since `2^expn > rem`. + ptr::copy_nonoverlapping( + buf.as_ptr(), + (buf.as_mut_ptr() as *mut T).add(buf.len()), + rem_len, + ); + // `buf.len() + rem_len` equals to `buf.capacity()` (`= self.len() * n`). + buf.set_len(capacity); + } + } + buf + } + + /// Flattens a slice of `T` into a single value `Self::Output`. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(["hello", "world"].concat(), "helloworld"); + /// assert_eq!([[1, 2], [3, 4]].concat(), [1, 2, 3, 4]); + /// ``` + #[rustc_allow_incoherent_impl] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn concat(&self) -> >::Output + where + Self: Concat, + { + Concat::concat(self) + } + + /// Flattens a slice of `T` into a single value `Self::Output`, placing a + /// given separator between each. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(["hello", "world"].join(" "), "hello world"); + /// assert_eq!([[1, 2], [3, 4]].join(&0), [1, 2, 0, 3, 4]); + /// assert_eq!([[1, 2], [3, 4]].join(&[0, 0][..]), [1, 2, 0, 0, 3, 4]); + /// ``` + #[rustc_allow_incoherent_impl] + #[stable(feature = "rename_connect_to_join", since = "1.3.0")] + pub fn join(&self, sep: Separator) -> >::Output + where + Self: Join, + { + Join::join(self, sep) + } + + /// Flattens a slice of `T` into a single value `Self::Output`, placing a + /// given separator between each. + /// + /// # Examples + /// + /// ``` + /// # #![allow(deprecated)] + /// assert_eq!(["hello", "world"].connect(" "), "hello world"); + /// assert_eq!([[1, 2], [3, 4]].connect(&0), [1, 2, 0, 3, 4]); + /// ``` + #[rustc_allow_incoherent_impl] + #[stable(feature = "rust1", since = "1.0.0")] + #[deprecated(since = "1.3.0", note = "renamed to join")] + pub fn connect(&self, sep: Separator) -> >::Output + where + Self: Join, + { + Join::join(self, sep) + } +} + +#[cfg(not(test))] +impl [u8] { + /// Returns a vector containing a copy of this slice where each byte + /// is mapped to its ASCII upper case equivalent. + /// + /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z', + /// but non-ASCII letters are unchanged. + /// + /// To uppercase the value in-place, use [`make_ascii_uppercase`]. + /// + /// [`make_ascii_uppercase`]: slice::make_ascii_uppercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the uppercase bytes as a new Vec, \ + without modifying the original"] + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] + pub fn to_ascii_uppercase(&self) -> Vec { + let mut me = self.to_vec(); + me.make_ascii_uppercase(); + me + } + + /// Returns a vector containing a copy of this slice where each byte + /// is mapped to its ASCII lower case equivalent. + /// + /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z', + /// but non-ASCII letters are unchanged. + /// + /// To lowercase the value in-place, use [`make_ascii_lowercase`]. + /// + /// [`make_ascii_lowercase`]: slice::make_ascii_lowercase + #[cfg(not(no_global_oom_handling))] + #[rustc_allow_incoherent_impl] + #[must_use = "this returns the lowercase bytes as a new Vec, \ + without modifying the original"] + #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")] + #[inline] + pub fn to_ascii_lowercase(&self) -> Vec { + let mut me = self.to_vec(); + me.make_ascii_lowercase(); + me + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Extension traits for slices over specific kinds of data +//////////////////////////////////////////////////////////////////////////////// + +/// Helper trait for [`[T]::concat`](slice::concat). +/// +/// Note: the `Item` type parameter is not used in this trait, +/// but it allows impls to be more generic. +/// Without it, we get this error: +/// +/// ```error +/// error[E0207]: the type parameter `T` is not constrained by the impl trait, self type, or predica +/// --> src/liballoc/slice.rs:608:6 +/// | +/// 608 | impl> Concat for [V] { +/// | ^ unconstrained type parameter +/// ``` +/// +/// This is because there could exist `V` types with multiple `Borrow<[_]>` impls, +/// such that multiple `T` types would apply: +/// +/// ``` +/// # #[allow(dead_code)] +/// pub struct Foo(Vec, Vec); +/// +/// impl std::borrow::Borrow<[u32]> for Foo { +/// fn borrow(&self) -> &[u32] { &self.0 } +/// } +/// +/// impl std::borrow::Borrow<[String]> for Foo { +/// fn borrow(&self) -> &[String] { &self.1 } +/// } +/// ``` +#[unstable(feature = "slice_concat_trait", issue = "27747")] +pub trait Concat { + #[unstable(feature = "slice_concat_trait", issue = "27747")] + /// The resulting type after concatenation + type Output; + + /// Implementation of [`[T]::concat`](slice::concat) + #[unstable(feature = "slice_concat_trait", issue = "27747")] + fn concat(slice: &Self) -> Self::Output; +} + +/// Helper trait for [`[T]::join`](slice::join) +#[unstable(feature = "slice_concat_trait", issue = "27747")] +pub trait Join { + #[unstable(feature = "slice_concat_trait", issue = "27747")] + /// The resulting type after concatenation + type Output; + + /// Implementation of [`[T]::join`](slice::join) + #[unstable(feature = "slice_concat_trait", issue = "27747")] + fn join(slice: &Self, sep: Separator) -> Self::Output; +} + +#[cfg(not(no_global_oom_handling))] +#[unstable(feature = "slice_concat_ext", issue = "27747")] +impl> Concat for [V] { + type Output = Vec; + + fn concat(slice: &Self) -> Vec { + let size = slice.iter().map(|slice| slice.borrow().len()).sum(); + let mut result = Vec::with_capacity(size); + for v in slice { + result.extend_from_slice(v.borrow()) + } + result + } +} + +#[cfg(not(no_global_oom_handling))] +#[unstable(feature = "slice_concat_ext", issue = "27747")] +impl> Join<&T> for [V] { + type Output = Vec; + + fn join(slice: &Self, sep: &T) -> Vec { + let mut iter = slice.iter(); + let first = match iter.next() { + Some(first) => first, + None => return vec![], + }; + let size = slice.iter().map(|v| v.borrow().len()).sum::() + slice.len() - 1; + let mut result = Vec::with_capacity(size); + result.extend_from_slice(first.borrow()); + + for v in iter { + result.push(sep.clone()); + result.extend_from_slice(v.borrow()) + } + result + } +} + +#[cfg(not(no_global_oom_handling))] +#[unstable(feature = "slice_concat_ext", issue = "27747")] +impl> Join<&[T]> for [V] { + type Output = Vec; + + fn join(slice: &Self, sep: &[T]) -> Vec { + let mut iter = slice.iter(); + let first = match iter.next() { + Some(first) => first, + None => return vec![], + }; + let size = + slice.iter().map(|v| v.borrow().len()).sum::() + sep.len() * (slice.len() - 1); + let mut result = Vec::with_capacity(size); + result.extend_from_slice(first.borrow()); + + for v in iter { + result.extend_from_slice(sep); + result.extend_from_slice(v.borrow()) + } + result + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Standard trait implementations for slices +//////////////////////////////////////////////////////////////////////////////// + +#[stable(feature = "rust1", since = "1.0.0")] +impl Borrow<[T]> for Vec { + fn borrow(&self) -> &[T] { + &self[..] + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl BorrowMut<[T]> for Vec { + fn borrow_mut(&mut self) -> &mut [T] { + &mut self[..] + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl ToOwned for [T] { + type Owned = Vec; + #[cfg(not(test))] + fn to_owned(&self) -> Vec { + self.to_vec() + } + + #[cfg(test)] + fn to_owned(&self) -> Vec { + hack::to_vec(self, Global) + } + + fn clone_into(&self, target: &mut Vec) { + // drop anything in target that will not be overwritten + target.truncate(self.len()); + + // target.len <= self.len due to the truncate above, so the + // slices here are always in-bounds. + let (init, tail) = self.split_at(target.len()); + + // reuse the contained values' allocations/resources. + target.clone_from_slice(init); + target.extend_from_slice(tail); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Sorting +//////////////////////////////////////////////////////////////////////////////// + +/// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted. +/// +/// This is the integral subroutine of insertion sort. +#[cfg(not(no_global_oom_handling))] +fn insert_head(v: &mut [T], is_less: &mut F) +where + F: FnMut(&T, &T) -> bool, +{ + if v.len() >= 2 && is_less(&v[1], &v[0]) { + unsafe { + // There are three ways to implement insertion here: + // + // 1. Swap adjacent elements until the first one gets to its final destination. + // However, this way we copy data around more than is necessary. If elements are big + // structures (costly to copy), this method will be slow. + // + // 2. Iterate until the right place for the first element is found. Then shift the + // elements succeeding it to make room for it and finally place it into the + // remaining hole. This is a good method. + // + // 3. Copy the first element into a temporary variable. Iterate until the right place + // for it is found. As we go along, copy every traversed element into the slot + // preceding it. Finally, copy data from the temporary variable into the remaining + // hole. This method is very good. Benchmarks demonstrated slightly better + // performance than with the 2nd method. + // + // All methods were benchmarked, and the 3rd showed best results. So we chose that one. + let tmp = mem::ManuallyDrop::new(ptr::read(&v[0])); + + // Intermediate state of the insertion process is always tracked by `hole`, which + // serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` in the end. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and + // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it + // initially held exactly once. + let mut hole = InsertionHole { src: &*tmp, dest: &mut v[1] }; + ptr::copy_nonoverlapping(&v[1], &mut v[0], 1); + + for i in 2..v.len() { + if !is_less(&v[i], &*tmp) { + break; + } + ptr::copy_nonoverlapping(&v[i], &mut v[i - 1], 1); + hole.dest = &mut v[i]; + } + // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`. + } + } + + // When dropped, copies from `src` into `dest`. + struct InsertionHole { + src: *const T, + dest: *mut T, + } + + impl Drop for InsertionHole { + fn drop(&mut self) { + unsafe { + ptr::copy_nonoverlapping(self.src, self.dest, 1); + } + } + } +} + +/// Merges non-decreasing runs `v[..mid]` and `v[mid..]` using `buf` as temporary storage, and +/// stores the result into `v[..]`. +/// +/// # Safety +/// +/// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough +/// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type. +#[cfg(not(no_global_oom_handling))] +unsafe fn merge(v: &mut [T], mid: usize, buf: *mut T, is_less: &mut F) +where + F: FnMut(&T, &T) -> bool, +{ + let len = v.len(); + let v = v.as_mut_ptr(); + let (v_mid, v_end) = unsafe { (v.add(mid), v.add(len)) }; + + // The merge process first copies the shorter run into `buf`. Then it traces the newly copied + // run and the longer run forwards (or backwards), comparing their next unconsumed elements and + // copying the lesser (or greater) one into `v`. + // + // As soon as the shorter run is fully consumed, the process is done. If the longer run gets + // consumed first, then we must copy whatever is left of the shorter run into the remaining + // hole in `v`. + // + // Intermediate state of the process is always tracked by `hole`, which serves two purposes: + // 1. Protects integrity of `v` from panics in `is_less`. + // 2. Fills the remaining hole in `v` if the longer run gets consumed first. + // + // Panic safety: + // + // If `is_less` panics at any point during the process, `hole` will get dropped and fill the + // hole in `v` with the unconsumed range in `buf`, thus ensuring that `v` still holds every + // object it initially held exactly once. + let mut hole; + + if mid <= len - mid { + // The left run is shorter. + unsafe { + ptr::copy_nonoverlapping(v, buf, mid); + hole = MergeHole { start: buf, end: buf.add(mid), dest: v }; + } + + // Initially, these pointers point to the beginnings of their arrays. + let left = &mut hole.start; + let mut right = v_mid; + let out = &mut hole.dest; + + while *left < hole.end && right < v_end { + // Consume the lesser side. + // If equal, prefer the left run to maintain stability. + unsafe { + let to_copy = if is_less(&*right, &**left) { + get_and_increment(&mut right) + } else { + get_and_increment(left) + }; + ptr::copy_nonoverlapping(to_copy, get_and_increment(out), 1); + } + } + } else { + // The right run is shorter. + unsafe { + ptr::copy_nonoverlapping(v_mid, buf, len - mid); + hole = MergeHole { start: buf, end: buf.add(len - mid), dest: v_mid }; + } + + // Initially, these pointers point past the ends of their arrays. + let left = &mut hole.dest; + let right = &mut hole.end; + let mut out = v_end; + + while v < *left && buf < *right { + // Consume the greater side. + // If equal, prefer the right run to maintain stability. + unsafe { + let to_copy = if is_less(&*right.offset(-1), &*left.offset(-1)) { + decrement_and_get(left) + } else { + decrement_and_get(right) + }; + ptr::copy_nonoverlapping(to_copy, decrement_and_get(&mut out), 1); + } + } + } + // Finally, `hole` gets dropped. If the shorter run was not fully consumed, whatever remains of + // it will now be copied into the hole in `v`. + + unsafe fn get_and_increment(ptr: &mut *mut T) -> *mut T { + let old = *ptr; + *ptr = unsafe { ptr.offset(1) }; + old + } + + unsafe fn decrement_and_get(ptr: &mut *mut T) -> *mut T { + *ptr = unsafe { ptr.offset(-1) }; + *ptr + } + + // When dropped, copies the range `start..end` into `dest..`. + struct MergeHole { + start: *mut T, + end: *mut T, + dest: *mut T, + } + + impl Drop for MergeHole { + fn drop(&mut self) { + // `T` is not a zero-sized type, and these are pointers into a slice's elements. + unsafe { + let len = self.end.sub_ptr(self.start); + ptr::copy_nonoverlapping(self.start, self.dest, len); + } + } + } +} + +/// This merge sort borrows some (but not all) ideas from TimSort, which is described in detail +/// [here](https://github.com/python/cpython/blob/main/Objects/listsort.txt). +/// +/// The algorithm identifies strictly descending and non-descending subsequences, which are called +/// natural runs. There is a stack of pending runs yet to be merged. Each newly found run is pushed +/// onto the stack, and then some pairs of adjacent runs are merged until these two invariants are +/// satisfied: +/// +/// 1. for every `i` in `1..runs.len()`: `runs[i - 1].len > runs[i].len` +/// 2. for every `i` in `2..runs.len()`: `runs[i - 2].len > runs[i - 1].len + runs[i].len` +/// +/// The invariants ensure that the total running time is *O*(*n* \* log(*n*)) worst-case. +#[cfg(not(no_global_oom_handling))] +fn merge_sort(v: &mut [T], mut is_less: F) +where + F: FnMut(&T, &T) -> bool, +{ + // Slices of up to this length get sorted using insertion sort. + const MAX_INSERTION: usize = 20; + // Very short runs are extended using insertion sort to span at least this many elements. + const MIN_RUN: usize = 10; + + // Sorting has no meaningful behavior on zero-sized types. + if size_of::() == 0 { + return; + } + + let len = v.len(); + + // Short arrays get sorted in-place via insertion sort to avoid allocations. + if len <= MAX_INSERTION { + if len >= 2 { + for i in (0..len - 1).rev() { + insert_head(&mut v[i..], &mut is_less); + } + } + return; + } + + // Allocate a buffer to use as scratch memory. We keep the length 0 so we can keep in it + // shallow copies of the contents of `v` without risking the dtors running on copies if + // `is_less` panics. When merging two sorted runs, this buffer holds a copy of the shorter run, + // which will always have length at most `len / 2`. + let mut buf = Vec::with_capacity(len / 2); + + // In order to identify natural runs in `v`, we traverse it backwards. That might seem like a + // strange decision, but consider the fact that merges more often go in the opposite direction + // (forwards). According to benchmarks, merging forwards is slightly faster than merging + // backwards. To conclude, identifying runs by traversing backwards improves performance. + let mut runs = vec![]; + let mut end = len; + while end > 0 { + // Find the next natural run, and reverse it if it's strictly descending. + let mut start = end - 1; + if start > 0 { + start -= 1; + unsafe { + if is_less(v.get_unchecked(start + 1), v.get_unchecked(start)) { + while start > 0 && is_less(v.get_unchecked(start), v.get_unchecked(start - 1)) { + start -= 1; + } + v[start..end].reverse(); + } else { + while start > 0 && !is_less(v.get_unchecked(start), v.get_unchecked(start - 1)) + { + start -= 1; + } + } + } + } + + // Insert some more elements into the run if it's too short. Insertion sort is faster than + // merge sort on short sequences, so this significantly improves performance. + while start > 0 && end - start < MIN_RUN { + start -= 1; + insert_head(&mut v[start..end], &mut is_less); + } + + // Push this run onto the stack. + runs.push(Run { start, len: end - start }); + end = start; + + // Merge some pairs of adjacent runs to satisfy the invariants. + while let Some(r) = collapse(&runs) { + let left = runs[r + 1]; + let right = runs[r]; + unsafe { + merge( + &mut v[left.start..right.start + right.len], + left.len, + buf.as_mut_ptr(), + &mut is_less, + ); + } + runs[r] = Run { start: left.start, len: left.len + right.len }; + runs.remove(r + 1); + } + } + + // Finally, exactly one run must remain in the stack. + debug_assert!(runs.len() == 1 && runs[0].start == 0 && runs[0].len == len); + + // Examines the stack of runs and identifies the next pair of runs to merge. More specifically, + // if `Some(r)` is returned, that means `runs[r]` and `runs[r + 1]` must be merged next. If the + // algorithm should continue building a new run instead, `None` is returned. + // + // TimSort is infamous for its buggy implementations, as described here: + // http://envisage-project.eu/timsort-specification-and-verification/ + // + // The gist of the story is: we must enforce the invariants on the top four runs on the stack. + // Enforcing them on just top three is not sufficient to ensure that the invariants will still + // hold for *all* runs in the stack. + // + // This function correctly checks invariants for the top four runs. Additionally, if the top + // run starts at index 0, it will always demand a merge operation until the stack is fully + // collapsed, in order to complete the sort. + #[inline] + fn collapse(runs: &[Run]) -> Option { + let n = runs.len(); + if n >= 2 + && (runs[n - 1].start == 0 + || runs[n - 2].len <= runs[n - 1].len + || (n >= 3 && runs[n - 3].len <= runs[n - 2].len + runs[n - 1].len) + || (n >= 4 && runs[n - 4].len <= runs[n - 3].len + runs[n - 2].len)) + { + if n >= 3 && runs[n - 3].len < runs[n - 1].len { Some(n - 3) } else { Some(n - 2) } + } else { + None + } + } + + #[derive(Clone, Copy)] + struct Run { + start: usize, + len: usize, + } +} diff --git a/rust/alloc/vec/drain.rs b/rust/alloc/vec/drain.rs new file mode 100644 index 0000000000000000000000000000000000000000..b6a5f98e4fcd95b9d05ecb38a978ef5866b87f39 --- /dev/null +++ b/rust/alloc/vec/drain.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::alloc::{Allocator, Global}; +use core::fmt; +use core::iter::{FusedIterator, TrustedLen}; +use core::mem; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +use super::Vec; + +/// A draining iterator for `Vec`. +/// +/// This `struct` is created by [`Vec::drain`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::Drain<_> = v.drain(..); +/// ``` +#[stable(feature = "drain", since = "1.6.0")] +pub struct Drain< + 'a, + T: 'a, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +#[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Drain<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[must_use] + #[inline] + pub fn allocator(&self) -> &A { + unsafe { self.vec.as_ref().allocator() } + } +} + +#[stable(feature = "vec_drain_as_slice", since = "1.46.0")] +impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +#[stable(feature = "drain", since = "1.6.0")] +unsafe impl Sync for Drain<'_, T, A> {} +#[stable(feature = "drain", since = "1.6.0")] +unsafe impl Send for Drain<'_, T, A> {} + +#[stable(feature = "drain", since = "1.6.0")] +impl Iterator for Drain<'_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter.next().map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl DoubleEndedIterator for Drain<'_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter.next_back().map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl Drop for Drain<'_, T, A> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); + + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::replace(&mut self.iter, (&mut []).iter()); + let drop_len = iter.len(); + + let mut vec = self.vec; + + if mem::size_of::() == 0 { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // vec::Splice modifies vec::Drain fields and may grow the vec which would invalidate + // the iterator's internal pointers. Creating a reference to deallocated memory + // is invalid even when it is zero-length + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + let drop_offset = drop_ptr.sub_ptr(vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +#[stable(feature = "drain", since = "1.6.0")] +impl ExactSizeIterator for Drain<'_, T, A> { + fn is_empty(&self) -> bool { + self.iter.is_empty() + } +} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for Drain<'_, T, A> {} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, T, A> {} diff --git a/rust/alloc/vec/drain_filter.rs b/rust/alloc/vec/drain_filter.rs new file mode 100644 index 0000000000000000000000000000000000000000..b04fce041622f39593e828185a950fc6ca313416 --- /dev/null +++ b/rust/alloc/vec/drain_filter.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::alloc::{Allocator, Global}; +use core::ptr::{self}; +use core::slice::{self}; + +use super::Vec; + +/// An iterator which uses a closure to determine if an element should be removed. +/// +/// This struct is created by [`Vec::drain_filter`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// #![feature(drain_filter)] +/// +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::DrainFilter<_, _> = v.drain_filter(|x| *x % 2 == 0); +/// ``` +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +#[derive(Debug)] +pub struct DrainFilter< + 'a, + T, + F, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> where + F: FnMut(&mut T) -> bool, +{ + pub(super) vec: &'a mut Vec, + /// The index of the item that will be inspected by the next call to `next`. + pub(super) idx: usize, + /// The number of items that have been drained (removed) thus far. + pub(super) del: usize, + /// The original length of `vec` prior to draining. + pub(super) old_len: usize, + /// The filter test predicate. + pub(super) pred: F, + /// A flag that indicates a panic has occurred in the filter test predicate. + /// This is used as a hint in the drop implementation to prevent consumption + /// of the remainder of the `DrainFilter`. Any unprocessed items will be + /// backshifted in the `vec`, but no further items will be dropped or + /// tested by the filter predicate. + pub(super) panic_flag: bool, +} + +impl DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + self.vec.allocator() + } +} + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +impl Iterator for DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + type Item = T; + + fn next(&mut self) -> Option { + unsafe { + while self.idx < self.old_len { + let i = self.idx; + let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len); + self.panic_flag = true; + let drained = (self.pred)(&mut v[i]); + self.panic_flag = false; + // Update the index *after* the predicate is called. If the index + // is updated prior and the predicate panics, the element at this + // index would be leaked. + self.idx += 1; + if drained { + self.del += 1; + return Some(ptr::read(&v[i])); + } else if self.del > 0 { + let del = self.del; + let src: *const T = &v[i]; + let dst: *mut T = &mut v[i - del]; + ptr::copy_nonoverlapping(src, dst, 1); + } + } + None + } + } + + fn size_hint(&self) -> (usize, Option) { + (0, Some(self.old_len - self.idx)) + } +} + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +impl Drop for DrainFilter<'_, T, F, A> +where + F: FnMut(&mut T) -> bool, +{ + fn drop(&mut self) { + struct BackshiftOnDrop<'a, 'b, T, F, A: Allocator> + where + F: FnMut(&mut T) -> bool, + { + drain: &'b mut DrainFilter<'a, T, F, A>, + } + + impl<'a, 'b, T, F, A: Allocator> Drop for BackshiftOnDrop<'a, 'b, T, F, A> + where + F: FnMut(&mut T) -> bool, + { + fn drop(&mut self) { + unsafe { + if self.drain.idx < self.drain.old_len && self.drain.del > 0 { + // This is a pretty messed up state, and there isn't really an + // obviously right thing to do. We don't want to keep trying + // to execute `pred`, so we just backshift all the unprocessed + // elements and tell the vec that they still exist. The backshift + // is required to prevent a double-drop of the last successfully + // drained item prior to a panic in the predicate. + let ptr = self.drain.vec.as_mut_ptr(); + let src = ptr.add(self.drain.idx); + let dst = src.sub(self.drain.del); + let tail_len = self.drain.old_len - self.drain.idx; + src.copy_to(dst, tail_len); + } + self.drain.vec.set_len(self.drain.old_len - self.drain.del); + } + } + } + + let backshift = BackshiftOnDrop { drain: self }; + + // Attempt to consume any remaining elements if the filter predicate + // has not yet panicked. We'll backshift any remaining elements + // whether we've already panicked or if the consumption here panics. + if !backshift.drain.panic_flag { + backshift.drain.for_each(drop); + } + } +} diff --git a/rust/alloc/vec/into_iter.rs b/rust/alloc/vec/into_iter.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7a50e76691e804850ef75d7bc6a00e25d971080 --- /dev/null +++ b/rust/alloc/vec/into_iter.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#[cfg(not(no_global_oom_handling))] +use super::AsVecIntoIter; +use crate::alloc::{Allocator, Global}; +use crate::raw_vec::RawVec; +use core::fmt; +use core::intrinsics::arith_offset; +use core::iter::{ + FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccessNoCoerce, +}; +use core::marker::PhantomData; +use core::mem::{self, ManuallyDrop}; +#[cfg(not(no_global_oom_handling))] +use core::ops::Deref; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +/// An iterator that moves out of a vector. +/// +/// This `struct` is created by the `into_iter` method on [`Vec`](super::Vec) +/// (provided by the [`IntoIterator`] trait). +/// +/// # Example +/// +/// ``` +/// let v = vec![0, 1, 2]; +/// let iter: std::vec::IntoIter<_> = v.into_iter(); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_insignificant_dtor] +pub struct IntoIter< + T, + #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global, +> { + pub(super) buf: NonNull, + pub(super) phantom: PhantomData, + pub(super) cap: usize, + // the drop impl reconstructs a RawVec from buf, cap and alloc + // to avoid dropping the allocator twice we need to wrap it into ManuallyDrop + pub(super) alloc: ManuallyDrop, + pub(super) ptr: *const T, + pub(super) end: *const T, +} + +#[stable(feature = "vec_intoiter_debug", since = "1.13.0")] +impl fmt::Debug for IntoIter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("IntoIter").field(&self.as_slice()).finish() + } +} + +impl IntoIter { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let vec = vec!['a', 'b', 'c']; + /// let mut into_iter = vec.into_iter(); + /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); + /// let _ = into_iter.next().unwrap(); + /// assert_eq!(into_iter.as_slice(), &['b', 'c']); + /// ``` + #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] + pub fn as_slice(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.ptr, self.len()) } + } + + /// Returns the remaining items of this iterator as a mutable slice. + /// + /// # Examples + /// + /// ``` + /// let vec = vec!['a', 'b', 'c']; + /// let mut into_iter = vec.into_iter(); + /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']); + /// into_iter.as_mut_slice()[2] = 'z'; + /// assert_eq!(into_iter.next().unwrap(), 'a'); + /// assert_eq!(into_iter.next().unwrap(), 'b'); + /// assert_eq!(into_iter.next().unwrap(), 'z'); + /// ``` + #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { &mut *self.as_raw_mut_slice() } + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + &self.alloc + } + + fn as_raw_mut_slice(&mut self) -> *mut [T] { + ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len()) + } + + /// Drops remaining elements and relinquishes the backing allocation. + /// + /// This is roughly equivalent to the following, but more efficient + /// + /// ``` + /// # let mut into_iter = Vec::::with_capacity(10).into_iter(); + /// (&mut into_iter).for_each(core::mem::drop); + /// unsafe { core::ptr::write(&mut into_iter, Vec::new().into_iter()); } + /// ``` + /// + /// This method is used by in-place iteration, refer to the vec::in_place_collect + /// documentation for an overview. + #[cfg(not(no_global_oom_handling))] + pub(super) fn forget_allocation_drop_remaining(&mut self) { + let remaining = self.as_raw_mut_slice(); + + // overwrite the individual fields instead of creating a new + // struct and then overwriting &mut self. + // this creates less assembly + self.cap = 0; + self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) }; + self.ptr = self.buf.as_ptr(); + self.end = self.buf.as_ptr(); + + unsafe { + ptr::drop_in_place(remaining); + } + } + + /// Forgets to Drop the remaining elements while still allowing the backing allocation to be freed. + #[allow(dead_code)] + pub(crate) fn forget_remaining_elements(&mut self) { + self.ptr = self.end; + } +} + +#[stable(feature = "vec_intoiter_as_ref", since = "1.46.0")] +impl AsRef<[T]> for IntoIter { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Send for IntoIter {} +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl Sync for IntoIter {} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Iterator for IntoIter { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + if self.ptr as *const _ == self.end { + None + } else if mem::size_of::() == 0 { + // purposefully don't use 'ptr.offset' because for + // vectors with 0-size elements this would return the + // same pointer. + self.ptr = unsafe { arith_offset(self.ptr as *const i8, 1) as *mut T }; + + // Make up a value of this ZST. + Some(unsafe { mem::zeroed() }) + } else { + let old = self.ptr; + self.ptr = unsafe { self.ptr.offset(1) }; + + Some(unsafe { ptr::read(old) }) + } + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let exact = if mem::size_of::() == 0 { + self.end.addr().wrapping_sub(self.ptr.addr()) + } else { + unsafe { self.end.sub_ptr(self.ptr) } + }; + (exact, Some(exact)) + } + + #[inline] + fn advance_by(&mut self, n: usize) -> Result<(), usize> { + let step_size = self.len().min(n); + let to_drop = ptr::slice_from_raw_parts_mut(self.ptr as *mut T, step_size); + if mem::size_of::() == 0 { + // SAFETY: due to unchecked casts of unsigned amounts to signed offsets the wraparound + // effectively results in unsigned pointers representing positions 0..usize::MAX, + // which is valid for ZSTs. + self.ptr = unsafe { arith_offset(self.ptr as *const i8, step_size as isize) as *mut T } + } else { + // SAFETY: the min() above ensures that step_size is in bounds + self.ptr = unsafe { self.ptr.add(step_size) }; + } + // SAFETY: the min() above ensures that step_size is in bounds + unsafe { + ptr::drop_in_place(to_drop); + } + if step_size < n { + return Err(step_size); + } + Ok(()) + } + + #[inline] + fn count(self) -> usize { + self.len() + } + + unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> Self::Item + where + Self: TrustedRandomAccessNoCoerce, + { + // SAFETY: the caller must guarantee that `i` is in bounds of the + // `Vec`, so `i` cannot overflow an `isize`, and the `self.ptr.add(i)` + // is guaranteed to pointer to an element of the `Vec` and + // thus guaranteed to be valid to dereference. + // + // Also note the implementation of `Self: TrustedRandomAccess` requires + // that `T: Copy` so reading elements from the buffer doesn't invalidate + // them for `Drop`. + unsafe { + if mem::size_of::() == 0 { mem::zeroed() } else { ptr::read(self.ptr.add(i)) } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl DoubleEndedIterator for IntoIter { + #[inline] + fn next_back(&mut self) -> Option { + if self.end == self.ptr { + None + } else if mem::size_of::() == 0 { + // See above for why 'ptr.offset' isn't used + self.end = unsafe { arith_offset(self.end as *const i8, -1) as *mut T }; + + // Make up a value of this ZST. + Some(unsafe { mem::zeroed() }) + } else { + self.end = unsafe { self.end.offset(-1) }; + + Some(unsafe { ptr::read(self.end) }) + } + } + + #[inline] + fn advance_back_by(&mut self, n: usize) -> Result<(), usize> { + let step_size = self.len().min(n); + if mem::size_of::() == 0 { + // SAFETY: same as for advance_by() + self.end = unsafe { + arith_offset(self.end as *const i8, step_size.wrapping_neg() as isize) as *mut T + } + } else { + // SAFETY: same as for advance_by() + self.end = unsafe { self.end.offset(step_size.wrapping_neg() as isize) }; + } + let to_drop = ptr::slice_from_raw_parts_mut(self.end as *mut T, step_size); + // SAFETY: same as for advance_by() + unsafe { + ptr::drop_in_place(to_drop); + } + if step_size < n { + return Err(step_size); + } + Ok(()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ExactSizeIterator for IntoIter { + fn is_empty(&self) -> bool { + self.ptr == self.end + } +} + +#[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for IntoIter {} + +#[unstable(feature = "trusted_len", issue = "37572")] +unsafe impl TrustedLen for IntoIter {} + +#[doc(hidden)] +#[unstable(issue = "none", feature = "std_internals")] +#[rustc_unsafe_specialization_marker] +pub trait NonDrop {} + +// T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr +// and thus we can't implement drop-handling +#[unstable(issue = "none", feature = "std_internals")] +impl NonDrop for T {} + +#[doc(hidden)] +#[unstable(issue = "none", feature = "std_internals")] +// TrustedRandomAccess (without NoCoerce) must not be implemented because +// subtypes/supertypes of `T` might not be `NonDrop` +unsafe impl TrustedRandomAccessNoCoerce for IntoIter +where + T: NonDrop, +{ + const MAY_HAVE_SIDE_EFFECT: bool = false; +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_into_iter_clone", since = "1.8.0")] +impl Clone for IntoIter { + #[cfg(not(test))] + fn clone(&self) -> Self { + self.as_slice().to_vec_in(self.alloc.deref().clone()).into_iter() + } + #[cfg(test)] + fn clone(&self) -> Self { + crate::slice::to_vec(self.as_slice(), self.alloc.deref().clone()).into_iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T, A: Allocator> Drop for IntoIter { + fn drop(&mut self) { + struct DropGuard<'a, T, A: Allocator>(&'a mut IntoIter); + + impl Drop for DropGuard<'_, T, A> { + fn drop(&mut self) { + unsafe { + // `IntoIter::alloc` is not used anymore after this and will be dropped by RawVec + let alloc = ManuallyDrop::take(&mut self.0.alloc); + // RawVec handles deallocation + let _ = RawVec::from_raw_parts_in(self.0.buf.as_ptr(), self.0.cap, alloc); + } + } + } + + let guard = DropGuard(self); + // destroy the remaining elements + unsafe { + ptr::drop_in_place(guard.0.as_raw_mut_slice()); + } + // now `guard` will be dropped and do the rest + } +} + +// In addition to the SAFETY invariants of the following three unsafe traits +// also refer to the vec::in_place_collect module documentation to get an overview +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl InPlaceIterable for IntoIter {} + +#[unstable(issue = "none", feature = "inplace_iteration")] +#[doc(hidden)] +unsafe impl SourceIter for IntoIter { + type Source = Self; + + #[inline] + unsafe fn as_inner(&mut self) -> &mut Self::Source { + self + } +} + +#[cfg(not(no_global_oom_handling))] +unsafe impl AsVecIntoIter for IntoIter { + type Item = T; + + fn as_into_iter(&mut self) -> &mut IntoIter { + self + } +} diff --git a/rust/alloc/vec/is_zero.rs b/rust/alloc/vec/is_zero.rs new file mode 100644 index 0000000000000000000000000000000000000000..377f3d17277755044c4a3cb051717395c7486f87 --- /dev/null +++ b/rust/alloc/vec/is_zero.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::boxed::Box; + +#[rustc_specialization_trait] +pub(super) unsafe trait IsZero { + /// Whether this value's representation is all zeros + fn is_zero(&self) -> bool; +} + +macro_rules! impl_is_zero { + ($t:ty, $is_zero:expr) => { + unsafe impl IsZero for $t { + #[inline] + fn is_zero(&self) -> bool { + $is_zero(*self) + } + } + }; +} + +impl_is_zero!(i16, |x| x == 0); +impl_is_zero!(i32, |x| x == 0); +impl_is_zero!(i64, |x| x == 0); +impl_is_zero!(i128, |x| x == 0); +impl_is_zero!(isize, |x| x == 0); + +impl_is_zero!(u16, |x| x == 0); +impl_is_zero!(u32, |x| x == 0); +impl_is_zero!(u64, |x| x == 0); +impl_is_zero!(u128, |x| x == 0); +impl_is_zero!(usize, |x| x == 0); + +impl_is_zero!(bool, |x| x == false); +impl_is_zero!(char, |x| x == '\0'); + +impl_is_zero!(f32, |x: f32| x.to_bits() == 0); +impl_is_zero!(f64, |x: f64| x.to_bits() == 0); + +unsafe impl IsZero for *const T { + #[inline] + fn is_zero(&self) -> bool { + (*self).is_null() + } +} + +unsafe impl IsZero for *mut T { + #[inline] + fn is_zero(&self) -> bool { + (*self).is_null() + } +} + +unsafe impl IsZero for [T; N] { + #[inline] + fn is_zero(&self) -> bool { + // Because this is generated as a runtime check, it's not obvious that + // it's worth doing if the array is really long. The threshold here + // is largely arbitrary, but was picked because as of 2022-05-01 LLVM + // can const-fold the check in `vec![[0; 32]; n]` but not in + // `vec![[0; 64]; n]`: https://godbolt.org/z/WTzjzfs5b + // Feel free to tweak if you have better evidence. + + N <= 32 && self.iter().all(IsZero::is_zero) + } +} + +// `Option<&T>` and `Option>` are guaranteed to represent `None` as null. +// For fat pointers, the bytes that would be the pointer metadata in the `Some` +// variant are padding in the `None` variant, so ignoring them and +// zero-initializing instead is ok. +// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of +// `SpecFromElem`. + +unsafe impl IsZero for Option<&T> { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } +} + +unsafe impl IsZero for Option> { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } +} + +// `Option` and similar have a representation guarantee that +// they're the same size as the corresponding `u32` type, as well as a guarantee +// that transmuting between `NonZeroU32` and `Option` works. +// While the documentation officially makes it UB to transmute from `None`, +// we're the standard library so we can make extra inferences, and we know that +// the only niche available to represent `None` is the one that's all zeros. + +macro_rules! impl_is_zero_option_of_nonzero { + ($($t:ident,)+) => {$( + unsafe impl IsZero for Option { + #[inline] + fn is_zero(&self) -> bool { + self.is_none() + } + } + )+}; +} + +impl_is_zero_option_of_nonzero!( + NonZeroU8, + NonZeroU16, + NonZeroU32, + NonZeroU64, + NonZeroU128, + NonZeroI8, + NonZeroI16, + NonZeroI32, + NonZeroI64, + NonZeroI128, + NonZeroUsize, + NonZeroIsize, +); diff --git a/rust/alloc/vec/mod.rs b/rust/alloc/vec/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..540787804cc29720cebc407de89f2765fe9a2c09 --- /dev/null +++ b/rust/alloc/vec/mod.rs @@ -0,0 +1,3140 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! A contiguous growable array type with heap-allocated contents, written +//! `Vec`. +//! +//! Vectors have *O*(1) indexing, amortized *O*(1) push (to the end) and +//! *O*(1) pop (from the end). +//! +//! Vectors ensure they never allocate more than `isize::MAX` bytes. +//! +//! # Examples +//! +//! You can explicitly create a [`Vec`] with [`Vec::new`]: +//! +//! ``` +//! let v: Vec = Vec::new(); +//! ``` +//! +//! ...or by using the [`vec!`] macro: +//! +//! ``` +//! let v: Vec = vec![]; +//! +//! let v = vec![1, 2, 3, 4, 5]; +//! +//! let v = vec![0; 10]; // ten zeroes +//! ``` +//! +//! You can [`push`] values onto the end of a vector (which will grow the vector +//! as needed): +//! +//! ``` +//! let mut v = vec![1, 2]; +//! +//! v.push(3); +//! ``` +//! +//! Popping values works in much the same way: +//! +//! ``` +//! let mut v = vec![1, 2]; +//! +//! let two = v.pop(); +//! ``` +//! +//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits): +//! +//! ``` +//! let mut v = vec![1, 2, 3]; +//! let three = v[2]; +//! v[1] = v[1] + 5; +//! ``` +//! +//! [`push`]: Vec::push + +#![stable(feature = "rust1", since = "1.0.0")] + +#[cfg(not(no_global_oom_handling))] +use core::cmp; +use core::cmp::Ordering; +use core::convert::TryFrom; +use core::fmt; +use core::hash::{Hash, Hasher}; +use core::intrinsics::{arith_offset, assume}; +use core::iter; +#[cfg(not(no_global_oom_handling))] +use core::iter::FromIterator; +use core::marker::PhantomData; +use core::mem::{self, ManuallyDrop, MaybeUninit}; +use core::ops::{self, Index, IndexMut, Range, RangeBounds}; +use core::ptr::{self, NonNull}; +use core::slice::{self, SliceIndex}; + +use crate::alloc::{Allocator, Global}; +use crate::borrow::{Cow, ToOwned}; +use crate::boxed::Box; +use crate::collections::TryReserveError; +use crate::raw_vec::RawVec; + +#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] +pub use self::drain_filter::DrainFilter; + +mod drain_filter; + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_splice", since = "1.21.0")] +pub use self::splice::Splice; + +#[cfg(not(no_global_oom_handling))] +mod splice; + +#[stable(feature = "drain", since = "1.6.0")] +pub use self::drain::Drain; + +mod drain; + +#[cfg(not(no_global_oom_handling))] +mod cow; + +#[cfg(not(no_global_oom_handling))] +pub(crate) use self::in_place_collect::AsVecIntoIter; +#[stable(feature = "rust1", since = "1.0.0")] +pub use self::into_iter::IntoIter; + +mod into_iter; + +#[cfg(not(no_global_oom_handling))] +use self::is_zero::IsZero; + +mod is_zero; + +#[cfg(not(no_global_oom_handling))] +mod in_place_collect; + +mod partial_eq; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_elem::SpecFromElem; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_elem; + +#[cfg(not(no_global_oom_handling))] +use self::set_len_on_drop::SetLenOnDrop; + +#[cfg(not(no_global_oom_handling))] +mod set_len_on_drop; + +#[cfg(not(no_global_oom_handling))] +use self::in_place_drop::InPlaceDrop; + +#[cfg(not(no_global_oom_handling))] +mod in_place_drop; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_iter_nested::SpecFromIterNested; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_iter_nested; + +#[cfg(not(no_global_oom_handling))] +use self::spec_from_iter::SpecFromIter; + +#[cfg(not(no_global_oom_handling))] +mod spec_from_iter; + +#[cfg(not(no_global_oom_handling))] +use self::spec_extend::SpecExtend; + +#[cfg(not(no_global_oom_handling))] +mod spec_extend; + +/// A contiguous growable array type, written as `Vec`, short for 'vector'. +/// +/// # Examples +/// +/// ``` +/// let mut vec = Vec::new(); +/// vec.push(1); +/// vec.push(2); +/// +/// assert_eq!(vec.len(), 2); +/// assert_eq!(vec[0], 1); +/// +/// assert_eq!(vec.pop(), Some(2)); +/// assert_eq!(vec.len(), 1); +/// +/// vec[0] = 7; +/// assert_eq!(vec[0], 7); +/// +/// vec.extend([1, 2, 3].iter().copied()); +/// +/// for x in &vec { +/// println!("{x}"); +/// } +/// assert_eq!(vec, [7, 1, 2, 3]); +/// ``` +/// +/// The [`vec!`] macro is provided for convenient initialization: +/// +/// ``` +/// let mut vec1 = vec![1, 2, 3]; +/// vec1.push(4); +/// let vec2 = Vec::from([1, 2, 3, 4]); +/// assert_eq!(vec1, vec2); +/// ``` +/// +/// It can also initialize each element of a `Vec` with a given value. +/// This may be more efficient than performing allocation and initialization +/// in separate steps, especially when initializing a vector of zeros: +/// +/// ``` +/// let vec = vec![0; 5]; +/// assert_eq!(vec, [0, 0, 0, 0, 0]); +/// +/// // The following is equivalent, but potentially slower: +/// let mut vec = Vec::with_capacity(5); +/// vec.resize(5, 0); +/// assert_eq!(vec, [0, 0, 0, 0, 0]); +/// ``` +/// +/// For more information, see +/// [Capacity and Reallocation](#capacity-and-reallocation). +/// +/// Use a `Vec` as an efficient stack: +/// +/// ``` +/// let mut stack = Vec::new(); +/// +/// stack.push(1); +/// stack.push(2); +/// stack.push(3); +/// +/// while let Some(top) = stack.pop() { +/// // Prints 3, 2, 1 +/// println!("{top}"); +/// } +/// ``` +/// +/// # Indexing +/// +/// The `Vec` type allows to access values by index, because it implements the +/// [`Index`] trait. An example will be more explicit: +/// +/// ``` +/// let v = vec![0, 2, 4, 6]; +/// println!("{}", v[1]); // it will display '2' +/// ``` +/// +/// However be careful: if you try to access an index which isn't in the `Vec`, +/// your software will panic! You cannot do this: +/// +/// ```should_panic +/// let v = vec![0, 2, 4, 6]; +/// println!("{}", v[6]); // it will panic! +/// ``` +/// +/// Use [`get`] and [`get_mut`] if you want to check whether the index is in +/// the `Vec`. +/// +/// # Slicing +/// +/// A `Vec` can be mutable. On the other hand, slices are read-only objects. +/// To get a [slice][prim@slice], use [`&`]. Example: +/// +/// ``` +/// fn read_slice(slice: &[usize]) { +/// // ... +/// } +/// +/// let v = vec![0, 1]; +/// read_slice(&v); +/// +/// // ... and that's all! +/// // you can also do it like this: +/// let u: &[usize] = &v; +/// // or like this: +/// let u: &[_] = &v; +/// ``` +/// +/// In Rust, it's more common to pass slices as arguments rather than vectors +/// when you just want to provide read access. The same goes for [`String`] and +/// [`&str`]. +/// +/// # Capacity and reallocation +/// +/// The capacity of a vector is the amount of space allocated for any future +/// elements that will be added onto the vector. This is not to be confused with +/// the *length* of a vector, which specifies the number of actual elements +/// within the vector. If a vector's length exceeds its capacity, its capacity +/// will automatically be increased, but its elements will have to be +/// reallocated. +/// +/// For example, a vector with capacity 10 and length 0 would be an empty vector +/// with space for 10 more elements. Pushing 10 or fewer elements onto the +/// vector will not change its capacity or cause reallocation to occur. However, +/// if the vector's length is increased to 11, it will have to reallocate, which +/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity`] +/// whenever possible to specify how big the vector is expected to get. +/// +/// # Guarantees +/// +/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees +/// about its design. This ensures that it's as low-overhead as possible in +/// the general case, and can be correctly manipulated in primitive ways +/// by unsafe code. Note that these guarantees refer to an unqualified `Vec`. +/// If additional type parameters are added (e.g., to support custom allocators), +/// overriding their defaults may change the behavior. +/// +/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length) +/// triplet. No more, no less. The order of these fields is completely +/// unspecified, and you should use the appropriate methods to modify these. +/// The pointer will never be null, so this type is null-pointer-optimized. +/// +/// However, the pointer might not actually point to allocated memory. In particular, +/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`], +/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`] +/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized +/// types inside a `Vec`, it will not allocate space for them. *Note that in this case +/// the `Vec` might not report a [`capacity`] of 0*. `Vec` will allocate if and only +/// if [mem::size_of::\]\() * [capacity]\() > 0. In general, `Vec`'s allocation +/// details are very subtle --- if you intend to allocate memory using a `Vec` +/// and use it for something else (either to pass to unsafe code, or to build your +/// own memory-backed collection), be sure to deallocate this memory by using +/// `from_raw_parts` to recover the `Vec` and then dropping it. +/// +/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap +/// (as defined by the allocator Rust is configured to use by default), and its +/// pointer points to [`len`] initialized, contiguous elements in order (what +/// you would see if you coerced it to a slice), followed by [capacity] - [len] +/// logically uninitialized, contiguous elements. +/// +/// A vector containing the elements `'a'` and `'b'` with capacity 4 can be +/// visualized as below. The top part is the `Vec` struct, it contains a +/// pointer to the head of the allocation in the heap, length and capacity. +/// The bottom part is the allocation on the heap, a contiguous memory block. +/// +/// ```text +/// ptr len capacity +/// +--------+--------+--------+ +/// | 0x0123 | 2 | 4 | +/// +--------+--------+--------+ +/// | +/// v +/// Heap +--------+--------+--------+--------+ +/// | 'a' | 'b' | uninit | uninit | +/// +--------+--------+--------+--------+ +/// ``` +/// +/// - **uninit** represents memory that is not initialized, see [`MaybeUninit`]. +/// - Note: the ABI is not stable and `Vec` makes no guarantees about its memory +/// layout (including the order of fields). +/// +/// `Vec` will never perform a "small optimization" where elements are actually +/// stored on the stack for two reasons: +/// +/// * It would make it more difficult for unsafe code to correctly manipulate +/// a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were +/// only moved, and it would be more difficult to determine if a `Vec` had +/// actually allocated memory. +/// +/// * It would penalize the general case, incurring an additional branch +/// on every access. +/// +/// `Vec` will never automatically shrink itself, even if completely empty. This +/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec` +/// and then filling it back up to the same [`len`] should incur no calls to +/// the allocator. If you wish to free up unused memory, use +/// [`shrink_to_fit`] or [`shrink_to`]. +/// +/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is +/// sufficient. [`push`] and [`insert`] *will* (re)allocate if +/// [len] == [capacity]. That is, the reported capacity is completely +/// accurate, and can be relied on. It can even be used to manually free the memory +/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even +/// when not necessary. +/// +/// `Vec` does not guarantee any particular growth strategy when reallocating +/// when full, nor when [`reserve`] is called. The current strategy is basic +/// and it may prove desirable to use a non-constant growth factor. Whatever +/// strategy is used will of course guarantee *O*(1) amortized [`push`]. +/// +/// `vec![x; n]`, `vec![a, b, c, d]`, and +/// [`Vec::with_capacity(n)`][`Vec::with_capacity`], will all produce a `Vec` +/// with exactly the requested capacity. If [len] == [capacity], +/// (as is the case for the [`vec!`] macro), then a `Vec` can be converted to +/// and from a [`Box<[T]>`][owned slice] without reallocating or moving the elements. +/// +/// `Vec` will not specifically overwrite any data that is removed from it, +/// but also won't specifically preserve it. Its uninitialized memory is +/// scratch space that it may use however it wants. It will generally just do +/// whatever is most efficient or otherwise easy to implement. Do not rely on +/// removed data to be erased for security purposes. Even if you drop a `Vec`, its +/// buffer may simply be reused by another allocation. Even if you zero a `Vec`'s memory +/// first, that might not actually happen because the optimizer does not consider +/// this a side-effect that must be preserved. There is one case which we will +/// not break, however: using `unsafe` code to write to the excess capacity, +/// and then increasing the length to match, is always valid. +/// +/// Currently, `Vec` does not guarantee the order in which elements are dropped. +/// The order has changed in the past and may change again. +/// +/// [`get`]: ../../std/vec/struct.Vec.html#method.get +/// [`get_mut`]: ../../std/vec/struct.Vec.html#method.get_mut +/// [`String`]: crate::string::String +/// [`&str`]: type@str +/// [`shrink_to_fit`]: Vec::shrink_to_fit +/// [`shrink_to`]: Vec::shrink_to +/// [capacity]: Vec::capacity +/// [`capacity`]: Vec::capacity +/// [mem::size_of::\]: core::mem::size_of +/// [len]: Vec::len +/// [`len`]: Vec::len +/// [`push`]: Vec::push +/// [`insert`]: Vec::insert +/// [`reserve`]: Vec::reserve +/// [`MaybeUninit`]: core::mem::MaybeUninit +/// [owned slice]: Box +#[stable(feature = "rust1", since = "1.0.0")] +#[cfg_attr(not(test), rustc_diagnostic_item = "Vec")] +#[rustc_insignificant_dtor] +pub struct Vec { + buf: RawVec, + len: usize, +} + +//////////////////////////////////////////////////////////////////////////////// +// Inherent methods +//////////////////////////////////////////////////////////////////////////////// + +impl Vec { + /// Constructs a new, empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. + /// + /// # Examples + /// + /// ``` + /// # #![allow(unused_mut)] + /// let mut vec: Vec = Vec::new(); + /// ``` + #[inline] + #[rustc_const_stable(feature = "const_vec_new", since = "1.39.0")] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub const fn new() -> Self { + Vec { buf: RawVec::NEW, len: 0 } + } + + /// Constructs a new, empty `Vec` with the specified capacity. + /// + /// The vector will be able to hold exactly `capacity` elements without + /// reallocating. If `capacity` is 0, the vector will not allocate. + /// + /// It is important to note that although the returned vector has the + /// *capacity* specified, the vector will have a zero *length*. For an + /// explanation of the difference between length and capacity, see + /// *[Capacity and reallocation]*. + /// + /// [Capacity and reallocation]: #capacity-and-reallocation + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// + /// // The vector contains no items, even though it has capacity for more + /// assert_eq!(vec.len(), 0); + /// assert_eq!(vec.capacity(), 10); + /// + /// // These are all done without reallocating... + /// for i in 0..10 { + /// vec.push(i); + /// } + /// assert_eq!(vec.len(), 10); + /// assert_eq!(vec.capacity(), 10); + /// + /// // ...but this may make the vector reallocate + /// vec.push(11); + /// assert_eq!(vec.len(), 11); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + #[must_use] + pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_in(capacity, Global) + } + + /// Creates a `Vec` directly from the raw components of another vector. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren't + /// checked: + /// + /// * `ptr` needs to have been previously allocated via [`String`]/`Vec` + /// (at least, it's highly likely to be incorrect if it wasn't). + /// * `T` needs to have the same alignment as what `ptr` was allocated with. + /// (`T` having a less strict alignment is not sufficient, the alignment really + /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be + /// allocated and deallocated with the same layout.) + /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs + /// to be the same size as the pointer was allocated with. (Because similar to + /// alignment, [`dealloc`] must be called with the same layout `size`.) + /// * `length` needs to be less than or equal to `capacity`. + /// + /// Violating these may cause problems like corrupting the allocator's + /// internal data structures. For example it is normally **not** safe + /// to build a `Vec` from a pointer to a C `char` array with length + /// `size_t`, doing so is only safe if the array was initially allocated by + /// a `Vec` or `String`. + /// It's also not safe to build one from a `Vec` and its length, because + /// the allocator cares about the alignment, and these two types have different + /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after + /// turning it into a `Vec` it'll be deallocated with alignment 1. To avoid + /// these issues, it is often preferable to do casting/transmuting using + /// [`slice::from_raw_parts`] instead. + /// + /// The ownership of `ptr` is effectively transferred to the + /// `Vec` which may then deallocate, reallocate or change the + /// contents of memory pointed to by the pointer at will. Ensure + /// that nothing else uses the pointer after calling this + /// function. + /// + /// [`String`]: crate::string::String + /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// + /// # Examples + /// + /// ``` + /// use std::ptr; + /// use std::mem; + /// + /// let v = vec![1, 2, 3]; + /// + // FIXME Update this when vec_into_raw_parts is stabilized + /// // Prevent running `v`'s destructor so we are in complete control + /// // of the allocation. + /// let mut v = mem::ManuallyDrop::new(v); + /// + /// // Pull out the various important pieces of information about `v` + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// + /// unsafe { + /// // Overwrite memory with 4, 5, 6 + /// for i in 0..len as isize { + /// ptr::write(p.offset(i), 4 + i); + /// } + /// + /// // Put everything back together into a Vec + /// let rebuilt = Vec::from_raw_parts(p, len, cap); + /// assert_eq!(rebuilt, [4, 5, 6]); + /// } + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self { + unsafe { Self::from_raw_parts_in(ptr, length, capacity, Global) } + } +} + +impl Vec { + /// Constructs a new, empty `Vec`. + /// + /// The vector will not allocate until elements are pushed onto it. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// # #[allow(unused_mut)] + /// let mut vec: Vec = Vec::new_in(System); + /// ``` + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub const fn new_in(alloc: A) -> Self { + Vec { buf: RawVec::new_in(alloc), len: 0 } + } + + /// Constructs a new, empty `Vec` with the specified capacity with the provided + /// allocator. + /// + /// The vector will be able to hold exactly `capacity` elements without + /// reallocating. If `capacity` is 0, the vector will not allocate. + /// + /// It is important to note that although the returned vector has the + /// *capacity* specified, the vector will have a zero *length*. For an + /// explanation of the difference between length and capacity, see + /// *[Capacity and reallocation]*. + /// + /// [Capacity and reallocation]: #capacity-and-reallocation + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// let mut vec = Vec::with_capacity_in(10, System); + /// + /// // The vector contains no items, even though it has capacity for more + /// assert_eq!(vec.len(), 0); + /// assert_eq!(vec.capacity(), 10); + /// + /// // These are all done without reallocating... + /// for i in 0..10 { + /// vec.push(i); + /// } + /// assert_eq!(vec.len(), 10); + /// assert_eq!(vec.capacity(), 10); + /// + /// // ...but this may make the vector reallocate + /// vec.push(11); + /// assert_eq!(vec.len(), 11); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub fn with_capacity_in(capacity: usize, alloc: A) -> Self { + Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 } + } + + /// Creates a `Vec` directly from the raw components of another vector. + /// + /// # Safety + /// + /// This is highly unsafe, due to the number of invariants that aren't + /// checked: + /// + /// * `ptr` needs to have been previously allocated via [`String`]/`Vec` + /// (at least, it's highly likely to be incorrect if it wasn't). + /// * `T` needs to have the same size and alignment as what `ptr` was allocated with. + /// (`T` having a less strict alignment is not sufficient, the alignment really + /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be + /// allocated and deallocated with the same layout.) + /// * `length` needs to be less than or equal to `capacity`. + /// * `capacity` needs to be the capacity that the pointer was allocated with. + /// + /// Violating these may cause problems like corrupting the allocator's + /// internal data structures. For example it is **not** safe + /// to build a `Vec` from a pointer to a C `char` array with length `size_t`. + /// It's also not safe to build one from a `Vec` and its length, because + /// the allocator cares about the alignment, and these two types have different + /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after + /// turning it into a `Vec` it'll be deallocated with alignment 1. + /// + /// The ownership of `ptr` is effectively transferred to the + /// `Vec` which may then deallocate, reallocate or change the + /// contents of memory pointed to by the pointer at will. Ensure + /// that nothing else uses the pointer after calling this + /// function. + /// + /// [`String`]: crate::string::String + /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api)] + /// + /// use std::alloc::System; + /// + /// use std::ptr; + /// use std::mem; + /// + /// let mut v = Vec::with_capacity_in(3, System); + /// v.push(1); + /// v.push(2); + /// v.push(3); + /// + // FIXME Update this when vec_into_raw_parts is stabilized + /// // Prevent running `v`'s destructor so we are in complete control + /// // of the allocation. + /// let mut v = mem::ManuallyDrop::new(v); + /// + /// // Pull out the various important pieces of information about `v` + /// let p = v.as_mut_ptr(); + /// let len = v.len(); + /// let cap = v.capacity(); + /// let alloc = v.allocator(); + /// + /// unsafe { + /// // Overwrite memory with 4, 5, 6 + /// for i in 0..len as isize { + /// ptr::write(p.offset(i), 4 + i); + /// } + /// + /// // Put everything back together into a Vec + /// let rebuilt = Vec::from_raw_parts_in(p, len, cap, alloc.clone()); + /// assert_eq!(rebuilt, [4, 5, 6]); + /// } + /// ``` + #[inline] + #[unstable(feature = "allocator_api", issue = "32838")] + pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self { + unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } } + } + + /// Decomposes a `Vec` into its raw components. + /// + /// Returns the raw pointer to the underlying data, the length of + /// the vector (in elements), and the allocated capacity of the + /// data (in elements). These are the same arguments in the same + /// order as the arguments to [`from_raw_parts`]. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Vec`. The only way to do + /// this is to convert the raw pointer, length, and capacity back + /// into a `Vec` with the [`from_raw_parts`] function, allowing + /// the destructor to perform the cleanup. + /// + /// [`from_raw_parts`]: Vec::from_raw_parts + /// + /// # Examples + /// + /// ``` + /// #![feature(vec_into_raw_parts)] + /// let v: Vec = vec![-1, 0, 1]; + /// + /// let (ptr, len, cap) = v.into_raw_parts(); + /// + /// let rebuilt = unsafe { + /// // We can now make changes to the components, such as + /// // transmuting the raw pointer to a compatible type. + /// let ptr = ptr as *mut u32; + /// + /// Vec::from_raw_parts(ptr, len, cap) + /// }; + /// assert_eq!(rebuilt, [4294967295, 0, 1]); + /// ``` + #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] + pub fn into_raw_parts(self) -> (*mut T, usize, usize) { + let mut me = ManuallyDrop::new(self); + (me.as_mut_ptr(), me.len(), me.capacity()) + } + + /// Decomposes a `Vec` into its raw components. + /// + /// Returns the raw pointer to the underlying data, the length of the vector (in elements), + /// the allocated capacity of the data (in elements), and the allocator. These are the same + /// arguments in the same order as the arguments to [`from_raw_parts_in`]. + /// + /// After calling this function, the caller is responsible for the + /// memory previously managed by the `Vec`. The only way to do + /// this is to convert the raw pointer, length, and capacity back + /// into a `Vec` with the [`from_raw_parts_in`] function, allowing + /// the destructor to perform the cleanup. + /// + /// [`from_raw_parts_in`]: Vec::from_raw_parts_in + /// + /// # Examples + /// + /// ``` + /// #![feature(allocator_api, vec_into_raw_parts)] + /// + /// use std::alloc::System; + /// + /// let mut v: Vec = Vec::new_in(System); + /// v.push(-1); + /// v.push(0); + /// v.push(1); + /// + /// let (ptr, len, cap, alloc) = v.into_raw_parts_with_alloc(); + /// + /// let rebuilt = unsafe { + /// // We can now make changes to the components, such as + /// // transmuting the raw pointer to a compatible type. + /// let ptr = ptr as *mut u32; + /// + /// Vec::from_raw_parts_in(ptr, len, cap, alloc) + /// }; + /// assert_eq!(rebuilt, [4294967295, 0, 1]); + /// ``` + #[unstable(feature = "allocator_api", issue = "32838")] + // #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")] + pub fn into_raw_parts_with_alloc(self) -> (*mut T, usize, usize, A) { + let mut me = ManuallyDrop::new(self); + let len = me.len(); + let capacity = me.capacity(); + let ptr = me.as_mut_ptr(); + let alloc = unsafe { ptr::read(me.allocator()) }; + (ptr, len, capacity, alloc) + } + + /// Returns the number of elements the vector can hold without + /// reallocating. + /// + /// # Examples + /// + /// ``` + /// let vec: Vec = Vec::with_capacity(10); + /// assert_eq!(vec.capacity(), 10); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn capacity(&self) -> usize { + self.buf.capacity() + } + + /// Reserves capacity for at least `additional` more elements to be inserted + /// in the given `Vec`. The collection may reserve more space to avoid + /// frequent reallocations. After calling `reserve`, capacity will be + /// greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.reserve(10); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve(&mut self, additional: usize) { + self.buf.reserve(self.len, additional); + } + + /// Reserves the minimum capacity for exactly `additional` more elements to + /// be inserted in the given `Vec`. After calling `reserve_exact`, + /// capacity will be greater than or equal to `self.len() + additional`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`reserve`] if future insertions are expected. + /// + /// [`reserve`]: Vec::reserve + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.reserve_exact(10); + /// assert!(vec.capacity() >= 11); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn reserve_exact(&mut self, additional: usize) { + self.buf.reserve_exact(self.len, additional); + } + + /// Tries to reserve capacity for at least `additional` more elements to be inserted + /// in the given `Vec`. The collection may reserve more space to avoid + /// frequent reallocations. After calling `try_reserve`, capacity will be + /// greater than or equal to `self.len() + additional`. Does nothing if + /// capacity is already sufficient. + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = Vec::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.buf.try_reserve(self.len, additional) + } + + /// Tries to reserve the minimum capacity for exactly `additional` + /// elements to be inserted in the given `Vec`. After calling + /// `try_reserve_exact`, capacity will be greater than or equal to + /// `self.len() + additional` if it returns `Ok(())`. + /// Does nothing if the capacity is already sufficient. + /// + /// Note that the allocator may give the collection more space than it + /// requests. Therefore, capacity can not be relied upon to be precisely + /// minimal. Prefer [`try_reserve`] if future insertions are expected. + /// + /// [`try_reserve`]: Vec::try_reserve + /// + /// # Errors + /// + /// If the capacity overflows, or the allocator reports a failure, then an error + /// is returned. + /// + /// # Examples + /// + /// ``` + /// use std::collections::TryReserveError; + /// + /// fn process_data(data: &[u32]) -> Result, TryReserveError> { + /// let mut output = Vec::new(); + /// + /// // Pre-reserve the memory, exiting if we can't + /// output.try_reserve_exact(data.len())?; + /// + /// // Now we know this can't OOM in the middle of our complex work + /// output.extend(data.iter().map(|&val| { + /// val * 2 + 5 // very complicated + /// })); + /// + /// Ok(output) + /// } + /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?"); + /// ``` + #[stable(feature = "try_reserve", since = "1.57.0")] + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { + self.buf.try_reserve_exact(self.len, additional) + } + + /// Shrinks the capacity of the vector as much as possible. + /// + /// It will drop down as close as possible to the length but the allocator + /// may still inform the vector that there is space for a few more elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// assert_eq!(vec.capacity(), 10); + /// vec.shrink_to_fit(); + /// assert!(vec.capacity() >= 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn shrink_to_fit(&mut self) { + // The capacity is never less than the length, and there's nothing to do when + // they are equal, so we can avoid the panic case in `RawVec::shrink_to_fit` + // by only calling it with a greater capacity. + if self.capacity() > self.len { + self.buf.shrink_to_fit(self.len); + } + } + + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length + /// and the supplied value. + /// + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// # Examples + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// assert_eq!(vec.capacity(), 10); + /// vec.shrink_to(4); + /// assert!(vec.capacity() >= 4); + /// vec.shrink_to(0); + /// assert!(vec.capacity() >= 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "shrink_to", since = "1.56.0")] + pub fn shrink_to(&mut self, min_capacity: usize) { + if self.capacity() > min_capacity { + self.buf.shrink_to_fit(cmp::max(self.len, min_capacity)); + } + } + + /// Converts the vector into [`Box<[T]>`][owned slice]. + /// + /// Note that this will drop any excess capacity. + /// + /// [owned slice]: Box + /// + /// # Examples + /// + /// ``` + /// let v = vec![1, 2, 3]; + /// + /// let slice = v.into_boxed_slice(); + /// ``` + /// + /// Any excess capacity is removed: + /// + /// ``` + /// let mut vec = Vec::with_capacity(10); + /// vec.extend([1, 2, 3]); + /// + /// assert_eq!(vec.capacity(), 10); + /// let slice = vec.into_boxed_slice(); + /// assert_eq!(slice.into_vec().capacity(), 3); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn into_boxed_slice(mut self) -> Box<[T], A> { + unsafe { + self.shrink_to_fit(); + let me = ManuallyDrop::new(self); + let buf = ptr::read(&me.buf); + let len = me.len(); + buf.into_box(len).assume_init() + } + } + + /// Shortens the vector, keeping the first `len` elements and dropping + /// the rest. + /// + /// If `len` is greater than the vector's current length, this has no + /// effect. + /// + /// The [`drain`] method can emulate `truncate`, but causes the excess + /// elements to be returned instead of dropped. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// Truncating a five element vector to two elements: + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4, 5]; + /// vec.truncate(2); + /// assert_eq!(vec, [1, 2]); + /// ``` + /// + /// No truncation occurs when `len` is greater than the vector's current + /// length: + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.truncate(8); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + /// + /// Truncating when `len == 0` is equivalent to calling the [`clear`] + /// method. + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.truncate(0); + /// assert_eq!(vec, []); + /// ``` + /// + /// [`clear`]: Vec::clear + /// [`drain`]: Vec::drain + #[stable(feature = "rust1", since = "1.0.0")] + pub fn truncate(&mut self, len: usize) { + // This is safe because: + // + // * the slice passed to `drop_in_place` is valid; the `len > self.len` + // case avoids creating an invalid slice, and + // * the `len` of the vector is shrunk before calling `drop_in_place`, + // such that no value will be dropped twice in case `drop_in_place` + // were to panic once (if it panics twice, the program aborts). + unsafe { + // Note: It's intentional that this is `>` and not `>=`. + // Changing it to `>=` has negative performance + // implications in some cases. See #78884 for more. + if len > self.len { + return; + } + let remaining_len = self.len - len; + let s = ptr::slice_from_raw_parts_mut(self.as_mut_ptr().add(len), remaining_len); + self.len = len; + ptr::drop_in_place(s); + } + } + + /// Extracts a slice containing the entire vector. + /// + /// Equivalent to `&s[..]`. + /// + /// # Examples + /// + /// ``` + /// use std::io::{self, Write}; + /// let buffer = vec![1, 2, 3, 5, 8]; + /// io::sink().write(buffer.as_slice()).unwrap(); + /// ``` + #[inline] + #[stable(feature = "vec_as_slice", since = "1.7.0")] + pub fn as_slice(&self) -> &[T] { + self + } + + /// Extracts a mutable slice of the entire vector. + /// + /// Equivalent to `&mut s[..]`. + /// + /// # Examples + /// + /// ``` + /// use std::io::{self, Read}; + /// let mut buffer = vec![0; 3]; + /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap(); + /// ``` + #[inline] + #[stable(feature = "vec_as_slice", since = "1.7.0")] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self + } + + /// Returns a raw pointer to the vector's buffer. + /// + /// The caller must ensure that the vector outlives the pointer this + /// function returns, or else it will end up pointing to garbage. + /// Modifying the vector may cause its buffer to be reallocated, + /// which would also make any pointers to it invalid. + /// + /// The caller must also ensure that the memory the pointer (non-transitively) points to + /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer + /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`]. + /// + /// # Examples + /// + /// ``` + /// let x = vec![1, 2, 4]; + /// let x_ptr = x.as_ptr(); + /// + /// unsafe { + /// for i in 0..x.len() { + /// assert_eq!(*x_ptr.add(i), 1 << i); + /// } + /// } + /// ``` + /// + /// [`as_mut_ptr`]: Vec::as_mut_ptr + #[stable(feature = "vec_as_ptr", since = "1.37.0")] + #[inline] + pub fn as_ptr(&self) -> *const T { + // We shadow the slice method of the same name to avoid going through + // `deref`, which creates an intermediate reference. + let ptr = self.buf.ptr(); + unsafe { + assume(!ptr.is_null()); + } + ptr + } + + /// Returns an unsafe mutable pointer to the vector's buffer. + /// + /// The caller must ensure that the vector outlives the pointer this + /// function returns, or else it will end up pointing to garbage. + /// Modifying the vector may cause its buffer to be reallocated, + /// which would also make any pointers to it invalid. + /// + /// # Examples + /// + /// ``` + /// // Allocate vector big enough for 4 elements. + /// let size = 4; + /// let mut x: Vec = Vec::with_capacity(size); + /// let x_ptr = x.as_mut_ptr(); + /// + /// // Initialize elements via raw pointer writes, then set length. + /// unsafe { + /// for i in 0..size { + /// *x_ptr.add(i) = i as i32; + /// } + /// x.set_len(size); + /// } + /// assert_eq!(&*x, &[0, 1, 2, 3]); + /// ``` + #[stable(feature = "vec_as_ptr", since = "1.37.0")] + #[inline] + pub fn as_mut_ptr(&mut self) -> *mut T { + // We shadow the slice method of the same name to avoid going through + // `deref_mut`, which creates an intermediate reference. + let ptr = self.buf.ptr(); + unsafe { + assume(!ptr.is_null()); + } + ptr + } + + /// Returns a reference to the underlying allocator. + #[unstable(feature = "allocator_api", issue = "32838")] + #[inline] + pub fn allocator(&self) -> &A { + self.buf.allocator() + } + + /// Forces the length of the vector to `new_len`. + /// + /// This is a low-level operation that maintains none of the normal + /// invariants of the type. Normally changing the length of a vector + /// is done using one of the safe operations instead, such as + /// [`truncate`], [`resize`], [`extend`], or [`clear`]. + /// + /// [`truncate`]: Vec::truncate + /// [`resize`]: Vec::resize + /// [`extend`]: Extend::extend + /// [`clear`]: Vec::clear + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`capacity()`]. + /// - The elements at `old_len..new_len` must be initialized. + /// + /// [`capacity()`]: Vec::capacity + /// + /// # Examples + /// + /// This method can be useful for situations in which the vector + /// is serving as a buffer for other code, particularly over FFI: + /// + /// ```no_run + /// # #![allow(dead_code)] + /// # // This is just a minimal skeleton for the doc example; + /// # // don't use this as a starting point for a real library. + /// # pub struct StreamWrapper { strm: *mut std::ffi::c_void } + /// # const Z_OK: i32 = 0; + /// # extern "C" { + /// # fn deflateGetDictionary( + /// # strm: *mut std::ffi::c_void, + /// # dictionary: *mut u8, + /// # dictLength: *mut usize, + /// # ) -> i32; + /// # } + /// # impl StreamWrapper { + /// pub fn get_dictionary(&self) -> Option> { + /// // Per the FFI method's docs, "32768 bytes is always enough". + /// let mut dict = Vec::with_capacity(32_768); + /// let mut dict_length = 0; + /// // SAFETY: When `deflateGetDictionary` returns `Z_OK`, it holds that: + /// // 1. `dict_length` elements were initialized. + /// // 2. `dict_length` <= the capacity (32_768) + /// // which makes `set_len` safe to call. + /// unsafe { + /// // Make the FFI call... + /// let r = deflateGetDictionary(self.strm, dict.as_mut_ptr(), &mut dict_length); + /// if r == Z_OK { + /// // ...and update the length to what was initialized. + /// dict.set_len(dict_length); + /// Some(dict) + /// } else { + /// None + /// } + /// } + /// } + /// # } + /// ``` + /// + /// While the following example is sound, there is a memory leak since + /// the inner vectors were not freed prior to the `set_len` call: + /// + /// ``` + /// let mut vec = vec![vec![1, 0, 0], + /// vec![0, 1, 0], + /// vec![0, 0, 1]]; + /// // SAFETY: + /// // 1. `old_len..0` is empty so no elements need to be initialized. + /// // 2. `0 <= capacity` always holds whatever `capacity` is. + /// unsafe { + /// vec.set_len(0); + /// } + /// ``` + /// + /// Normally, here, one would use [`clear`] instead to correctly drop + /// the contents and thus not leak memory. + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + + self.len = new_len; + } + + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// This does not preserve ordering, but is *O*(1). + /// If you need to preserve the element order, use [`remove`] instead. + /// + /// [`remove`]: Vec::remove + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec!["foo", "bar", "baz", "qux"]; + /// + /// assert_eq!(v.swap_remove(1), "bar"); + /// assert_eq!(v, ["foo", "qux", "baz"]); + /// + /// assert_eq!(v.swap_remove(0), "foo"); + /// assert_eq!(v, ["baz", "qux"]); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn swap_remove(&mut self, index: usize) -> T { + #[cold] + #[inline(never)] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("swap_remove index (is {index}) should be < len (is {len})"); + } + + let len = self.len(); + if index >= len { + assert_failed(index, len); + } + unsafe { + // We replace self[index] with the last element. Note that if the + // bounds check above succeeds there must be a last element (which + // can be self[index] itself). + let value = ptr::read(self.as_ptr().add(index)); + let base_ptr = self.as_mut_ptr(); + ptr::copy(base_ptr.add(len - 1), base_ptr.add(index), 1); + self.set_len(len - 1); + value + } + } + + /// Inserts an element at position `index` within the vector, shifting all + /// elements after it to the right. + /// + /// # Panics + /// + /// Panics if `index > len`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.insert(1, 4); + /// assert_eq!(vec, [1, 4, 2, 3]); + /// vec.insert(4, 5); + /// assert_eq!(vec, [1, 4, 2, 3, 5]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn insert(&mut self, index: usize, element: T) { + #[cold] + #[inline(never)] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("insertion index (is {index}) should be <= len (is {len})"); + } + + let len = self.len(); + if index > len { + assert_failed(index, len); + } + + // space for the new element + if len == self.buf.capacity() { + self.reserve(1); + } + + unsafe { + // infallible + // The spot to put the new value + { + let p = self.as_mut_ptr().add(index); + // Shift everything over to make space. (Duplicating the + // `index`th element into two consecutive places.) + ptr::copy(p, p.offset(1), len - index); + // Write it in, overwriting the first copy of the `index`th + // element. + ptr::write(p, element); + } + self.set_len(len + 1); + } + } + + /// Removes and returns the element at position `index` within the vector, + /// shifting all elements after it to the left. + /// + /// Note: Because this shifts over the remaining elements, it has a + /// worst-case performance of *O*(*n*). If you don't need the order of elements + /// to be preserved, use [`swap_remove`] instead. If you'd like to remove + /// elements from the beginning of the `Vec`, consider using + /// [`VecDeque::pop_front`] instead. + /// + /// [`swap_remove`]: Vec::swap_remove + /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// assert_eq!(v.remove(1), 2); + /// assert_eq!(v, [1, 3]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[track_caller] + pub fn remove(&mut self, index: usize) -> T { + #[cold] + #[inline(never)] + #[track_caller] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("removal index (is {index}) should be < len (is {len})"); + } + + let len = self.len(); + if index >= len { + assert_failed(index, len); + } + unsafe { + // infallible + let ret; + { + // the place we are taking from. + let ptr = self.as_mut_ptr().add(index); + // copy it out, unsafely having a copy of the value on + // the stack and in the vector at the same time. + ret = ptr::read(ptr); + + // Shift everything down to fill in that spot. + ptr::copy(ptr.offset(1), ptr, len - index - 1); + } + self.set_len(len - 1); + ret + } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.retain(|&x| x % 2 == 0); + /// assert_eq!(vec, [2, 4]); + /// ``` + /// + /// Because the elements are visited exactly once in the original order, + /// external state may be used to decide which elements to keep. + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4, 5]; + /// let keep = [false, true, true, false, true]; + /// let mut iter = keep.iter(); + /// vec.retain(|_| *iter.next().unwrap()); + /// assert_eq!(vec, [2, 3, 5]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.retain_mut(|elem| f(elem)); + } + + /// Retains only the elements specified by the predicate, passing a mutable reference to it. + /// + /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.retain_mut(|x| if *x > 3 { + /// false + /// } else { + /// *x += 1; + /// true + /// }); + /// assert_eq!(vec, [2, 3, 4]); + /// ``` + #[stable(feature = "vec_retain_mut", since = "1.61.0")] + pub fn retain_mut(&mut self, mut f: F) + where + F: FnMut(&mut T) -> bool, + { + let original_len = self.len(); + // Avoid double drop if the drop guard is not executed, + // since we may make some holes during the process. + unsafe { self.set_len(0) }; + + // Vec: [Kept, Kept, Hole, Hole, Hole, Hole, Unchecked, Unchecked] + // |<- processed len ->| ^- next to check + // |<- deleted cnt ->| + // |<- original_len ->| + // Kept: Elements which predicate returns true on. + // Hole: Moved or dropped element slot. + // Unchecked: Unchecked valid elements. + // + // This drop guard will be invoked when predicate or `drop` of element panicked. + // It shifts unchecked elements to cover holes and `set_len` to the correct length. + // In cases when predicate and `drop` never panick, it will be optimized out. + struct BackshiftOnDrop<'a, T, A: Allocator> { + v: &'a mut Vec, + processed_len: usize, + deleted_cnt: usize, + original_len: usize, + } + + impl Drop for BackshiftOnDrop<'_, T, A> { + fn drop(&mut self) { + if self.deleted_cnt > 0 { + // SAFETY: Trailing unchecked items must be valid since we never touch them. + unsafe { + ptr::copy( + self.v.as_ptr().add(self.processed_len), + self.v.as_mut_ptr().add(self.processed_len - self.deleted_cnt), + self.original_len - self.processed_len, + ); + } + } + // SAFETY: After filling holes, all items are in contiguous memory. + unsafe { + self.v.set_len(self.original_len - self.deleted_cnt); + } + } + } + + let mut g = BackshiftOnDrop { v: self, processed_len: 0, deleted_cnt: 0, original_len }; + + fn process_loop( + original_len: usize, + f: &mut F, + g: &mut BackshiftOnDrop<'_, T, A>, + ) where + F: FnMut(&mut T) -> bool, + { + while g.processed_len != original_len { + // SAFETY: Unchecked element must be valid. + let cur = unsafe { &mut *g.v.as_mut_ptr().add(g.processed_len) }; + if !f(cur) { + // Advance early to avoid double drop if `drop_in_place` panicked. + g.processed_len += 1; + g.deleted_cnt += 1; + // SAFETY: We never touch this element again after dropped. + unsafe { ptr::drop_in_place(cur) }; + // We already advanced the counter. + if DELETED { + continue; + } else { + break; + } + } + if DELETED { + // SAFETY: `deleted_cnt` > 0, so the hole slot must not overlap with current element. + // We use copy for move, and never touch this element again. + unsafe { + let hole_slot = g.v.as_mut_ptr().add(g.processed_len - g.deleted_cnt); + ptr::copy_nonoverlapping(cur, hole_slot, 1); + } + } + g.processed_len += 1; + } + } + + // Stage 1: Nothing was deleted. + process_loop::(original_len, &mut f, &mut g); + + // Stage 2: Some elements were deleted. + process_loop::(original_len, &mut f, &mut g); + + // All item are processed. This can be optimized to `set_len` by LLVM. + drop(g); + } + + /// Removes all but the first of consecutive elements in the vector that resolve to the same + /// key. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![10, 20, 21, 30, 20]; + /// + /// vec.dedup_by_key(|i| *i / 10); + /// + /// assert_eq!(vec, [10, 20, 30, 20]); + /// ``` + #[stable(feature = "dedup_by", since = "1.16.0")] + #[inline] + pub fn dedup_by_key(&mut self, mut key: F) + where + F: FnMut(&mut T) -> K, + K: PartialEq, + { + self.dedup_by(|a, b| key(a) == key(b)) + } + + /// Removes all but the first of consecutive elements in the vector satisfying a given equality + /// relation. + /// + /// The `same_bucket` function is passed references to two elements from the vector and + /// must determine if the elements compare equal. The elements are passed in opposite order + /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!["foo", "bar", "Bar", "baz", "bar"]; + /// + /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b)); + /// + /// assert_eq!(vec, ["foo", "bar", "baz", "bar"]); + /// ``` + #[stable(feature = "dedup_by", since = "1.16.0")] + pub fn dedup_by(&mut self, mut same_bucket: F) + where + F: FnMut(&mut T, &mut T) -> bool, + { + let len = self.len(); + if len <= 1 { + return; + } + + /* INVARIANT: vec.len() > read >= write > write-1 >= 0 */ + struct FillGapOnDrop<'a, T, A: core::alloc::Allocator> { + /* Offset of the element we want to check if it is duplicate */ + read: usize, + + /* Offset of the place where we want to place the non-duplicate + * when we find it. */ + write: usize, + + /* The Vec that would need correction if `same_bucket` panicked */ + vec: &'a mut Vec, + } + + impl<'a, T, A: core::alloc::Allocator> Drop for FillGapOnDrop<'a, T, A> { + fn drop(&mut self) { + /* This code gets executed when `same_bucket` panics */ + + /* SAFETY: invariant guarantees that `read - write` + * and `len - read` never overflow and that the copy is always + * in-bounds. */ + unsafe { + let ptr = self.vec.as_mut_ptr(); + let len = self.vec.len(); + + /* How many items were left when `same_bucket` panicked. + * Basically vec[read..].len() */ + let items_left = len.wrapping_sub(self.read); + + /* Pointer to first item in vec[write..write+items_left] slice */ + let dropped_ptr = ptr.add(self.write); + /* Pointer to first item in vec[read..] slice */ + let valid_ptr = ptr.add(self.read); + + /* Copy `vec[read..]` to `vec[write..write+items_left]`. + * The slices can overlap, so `copy_nonoverlapping` cannot be used */ + ptr::copy(valid_ptr, dropped_ptr, items_left); + + /* How many items have been already dropped + * Basically vec[read..write].len() */ + let dropped = self.read.wrapping_sub(self.write); + + self.vec.set_len(len - dropped); + } + } + } + + let mut gap = FillGapOnDrop { read: 1, write: 1, vec: self }; + let ptr = gap.vec.as_mut_ptr(); + + /* Drop items while going through Vec, it should be more efficient than + * doing slice partition_dedup + truncate */ + + /* SAFETY: Because of the invariant, read_ptr, prev_ptr and write_ptr + * are always in-bounds and read_ptr never aliases prev_ptr */ + unsafe { + while gap.read < len { + let read_ptr = ptr.add(gap.read); + let prev_ptr = ptr.add(gap.write.wrapping_sub(1)); + + if same_bucket(&mut *read_ptr, &mut *prev_ptr) { + // Increase `gap.read` now since the drop may panic. + gap.read += 1; + /* We have found duplicate, drop it in-place */ + ptr::drop_in_place(read_ptr); + } else { + let write_ptr = ptr.add(gap.write); + + /* Because `read_ptr` can be equal to `write_ptr`, we either + * have to use `copy` or conditional `copy_nonoverlapping`. + * Looks like the first option is faster. */ + ptr::copy(read_ptr, write_ptr, 1); + + /* We have filled that place, so go further */ + gap.write += 1; + gap.read += 1; + } + } + + /* Technically we could let `gap` clean up with its Drop, but + * when `same_bucket` is guaranteed to not panic, this bloats a little + * the codegen, so we just do it manually */ + gap.vec.set_len(gap.write); + mem::forget(gap); + } + } + + /// Appends an element to the back of a collection. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `isize::MAX` bytes. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2]; + /// vec.push(3); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn push(&mut self, value: T) { + // This will panic or abort if we would allocate > isize::MAX bytes + // or if the length increment would overflow for zero-sized types. + if self.len == self.buf.capacity() { + self.buf.reserve_for_push(self.len); + } + unsafe { + let end = self.as_mut_ptr().add(self.len); + ptr::write(end, value); + self.len += 1; + } + } + + /// Tries to append an element to the back of a collection. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2]; + /// vec.try_push(3).unwrap(); + /// assert_eq!(vec, [1, 2, 3]); + /// ``` + #[inline] + #[stable(feature = "kernel", since = "1.0.0")] + pub fn try_push(&mut self, value: T) -> Result<(), TryReserveError> { + if self.len == self.buf.capacity() { + self.buf.try_reserve_for_push(self.len)?; + } + unsafe { + let end = self.as_mut_ptr().add(self.len); + ptr::write(end, value); + self.len += 1; + } + Ok(()) + } + + /// Removes the last element from a vector and returns it, or [`None`] if it + /// is empty. + /// + /// If you'd like to pop the first element, consider using + /// [`VecDeque::pop_front`] instead. + /// + /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// assert_eq!(vec.pop(), Some(3)); + /// assert_eq!(vec, [1, 2]); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn pop(&mut self) -> Option { + if self.len == 0 { + None + } else { + unsafe { + self.len -= 1; + Some(ptr::read(self.as_ptr().add(self.len()))) + } + } + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + /// + /// # Panics + /// + /// Panics if the number of elements in the vector overflows a `usize`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// let mut vec2 = vec![4, 5, 6]; + /// vec.append(&mut vec2); + /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]); + /// assert_eq!(vec2, []); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "append", since = "1.4.0")] + pub fn append(&mut self, other: &mut Self) { + unsafe { + self.append_elements(other.as_slice() as _); + other.set_len(0); + } + } + + /// Appends elements to `self` from other buffer. + #[cfg(not(no_global_oom_handling))] + #[inline] + unsafe fn append_elements(&mut self, other: *const [T]) { + let count = unsafe { (*other).len() }; + self.reserve(count); + let len = self.len(); + unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) }; + self.len += count; + } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v, &[1]); + /// assert_eq!(u, &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v, &[]); + /// ``` + #[stable(feature = "drain", since = "1.6.0")] + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + // Memory safety + // + // When the Drain is first created, it shortens the length of + // the source vector to make sure no uninitialized or moved-from elements + // are accessible at all if the Drain's destructor never gets to run. + // + // Drain will ptr::read out the values to remove. + // When finished, remaining tail of the vec is copied back to cover + // the hole, and the vector length is restored to the new length. + // + let len = self.len(); + let Range { start, end } = slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + // Use the borrow in the IterMut to indicate borrowing behavior of the + // whole Drain iterator (like &mut T). + let range_slice = slice::from_raw_parts_mut(self.as_mut_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + + /// Clears the vector, removing all values. + /// + /// Note that this method has no effect on the allocated capacity + /// of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// + /// v.clear(); + /// + /// assert!(v.is_empty()); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn clear(&mut self) { + let elems: *mut [T] = self.as_mut_slice(); + + // SAFETY: + // - `elems` comes directly from `as_mut_slice` and is therefore valid. + // - Setting `self.len` before calling `drop_in_place` means that, + // if an element's `Drop` impl panics, the vector's `Drop` impl will + // do nothing (leaking the rest of the elements) instead of dropping + // some twice. + unsafe { + self.len = 0; + ptr::drop_in_place(elems); + } + } + + /// Returns the number of elements in the vector, also referred to + /// as its 'length'. + /// + /// # Examples + /// + /// ``` + /// let a = vec![1, 2, 3]; + /// assert_eq!(a.len(), 3); + /// ``` + #[inline] + #[stable(feature = "rust1", since = "1.0.0")] + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the vector contains no elements. + /// + /// # Examples + /// + /// ``` + /// let mut v = Vec::new(); + /// assert!(v.is_empty()); + /// + /// v.push(1); + /// assert!(!v.is_empty()); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Splits the collection into two at the given index. + /// + /// Returns a newly allocated vector containing the elements in the range + /// `[at, len)`. After the call, the original vector will be left containing + /// the elements `[0, at)` with its previous capacity unchanged. + /// + /// # Panics + /// + /// Panics if `at > len`. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// let vec2 = vec.split_off(1); + /// assert_eq!(vec, [1]); + /// assert_eq!(vec2, [2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[must_use = "use `.truncate()` if you don't need the other half"] + #[stable(feature = "split_off", since = "1.4.0")] + pub fn split_off(&mut self, at: usize) -> Self + where + A: Clone, + { + #[cold] + #[inline(never)] + fn assert_failed(at: usize, len: usize) -> ! { + panic!("`at` split index (is {at}) should be <= len (is {len})"); + } + + if at > self.len() { + assert_failed(at, self.len()); + } + + if at == 0 { + // the new vector can take over the original buffer and avoid the copy + return mem::replace( + self, + Vec::with_capacity_in(self.capacity(), self.allocator().clone()), + ); + } + + let other_len = self.len - at; + let mut other = Vec::with_capacity_in(other_len, self.allocator().clone()); + + // Unsafely `set_len` and copy items to `other`. + unsafe { + self.set_len(at); + other.set_len(other_len); + + ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len()); + } + other + } + + /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the `Vec` is extended by the + /// difference, with each additional slot filled with the result of + /// calling the closure `f`. The return values from `f` will end up + /// in the `Vec` in the order they have been generated. + /// + /// If `new_len` is less than `len`, the `Vec` is simply truncated. + /// + /// This method uses a closure to create new values on every push. If + /// you'd rather [`Clone`] a given value, use [`Vec::resize`]. If you + /// want to use the [`Default`] trait to generate values, you can + /// pass [`Default::default`] as the second argument. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 3]; + /// vec.resize_with(5, Default::default); + /// assert_eq!(vec, [1, 2, 3, 0, 0]); + /// + /// let mut vec = vec![]; + /// let mut p = 1; + /// vec.resize_with(4, || { p *= 2; p }); + /// assert_eq!(vec, [2, 4, 8, 16]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_resize_with", since = "1.33.0")] + pub fn resize_with(&mut self, new_len: usize, f: F) + where + F: FnMut() -> T, + { + let len = self.len(); + if new_len > len { + self.extend_with(new_len - len, ExtendFunc(f)); + } else { + self.truncate(new_len); + } + } + + /// Consumes and leaks the `Vec`, returning a mutable reference to the contents, + /// `&'a mut [T]`. Note that the type `T` must outlive the chosen lifetime + /// `'a`. If the type has only static references, or none at all, then this + /// may be chosen to be `'static`. + /// + /// As of Rust 1.57, this method does not reallocate or shrink the `Vec`, + /// so the leaked allocation may include unused capacity that is not part + /// of the returned slice. + /// + /// This function is mainly useful for data that lives for the remainder of + /// the program's life. Dropping the returned reference will cause a memory + /// leak. + /// + /// # Examples + /// + /// Simple usage: + /// + /// ``` + /// let x = vec![1, 2, 3]; + /// let static_ref: &'static mut [usize] = x.leak(); + /// static_ref[0] += 1; + /// assert_eq!(static_ref, &[2, 2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_leak", since = "1.47.0")] + #[inline] + pub fn leak<'a>(self) -> &'a mut [T] + where + A: 'a, + { + let mut me = ManuallyDrop::new(self); + unsafe { slice::from_raw_parts_mut(me.as_mut_ptr(), me.len) } + } + + /// Returns the remaining spare capacity of the vector as a slice of + /// `MaybeUninit`. + /// + /// The returned slice can be used to fill the vector with data (e.g. by + /// reading from a file) before marking the data as initialized using the + /// [`set_len`] method. + /// + /// [`set_len`]: Vec::set_len + /// + /// # Examples + /// + /// ``` + /// // Allocate vector big enough for 10 elements. + /// let mut v = Vec::with_capacity(10); + /// + /// // Fill in the first 3 elements. + /// let uninit = v.spare_capacity_mut(); + /// uninit[0].write(0); + /// uninit[1].write(1); + /// uninit[2].write(2); + /// + /// // Mark the first 3 elements of the vector as being initialized. + /// unsafe { + /// v.set_len(3); + /// } + /// + /// assert_eq!(&v, &[0, 1, 2]); + /// ``` + #[stable(feature = "vec_spare_capacity", since = "1.60.0")] + #[inline] + pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + // Note: + // This method is not implemented in terms of `split_at_spare_mut`, + // to prevent invalidation of pointers to the buffer. + unsafe { + slice::from_raw_parts_mut( + self.as_mut_ptr().add(self.len) as *mut MaybeUninit, + self.buf.capacity() - self.len, + ) + } + } + + /// Returns vector content as a slice of `T`, along with the remaining spare + /// capacity of the vector as a slice of `MaybeUninit`. + /// + /// The returned spare capacity slice can be used to fill the vector with data + /// (e.g. by reading from a file) before marking the data as initialized using + /// the [`set_len`] method. + /// + /// [`set_len`]: Vec::set_len + /// + /// Note that this is a low-level API, which should be used with care for + /// optimization purposes. If you need to append data to a `Vec` + /// you can use [`push`], [`extend`], [`extend_from_slice`], + /// [`extend_from_within`], [`insert`], [`append`], [`resize`] or + /// [`resize_with`], depending on your exact needs. + /// + /// [`push`]: Vec::push + /// [`extend`]: Vec::extend + /// [`extend_from_slice`]: Vec::extend_from_slice + /// [`extend_from_within`]: Vec::extend_from_within + /// [`insert`]: Vec::insert + /// [`append`]: Vec::append + /// [`resize`]: Vec::resize + /// [`resize_with`]: Vec::resize_with + /// + /// # Examples + /// + /// ``` + /// #![feature(vec_split_at_spare)] + /// + /// let mut v = vec![1, 1, 2]; + /// + /// // Reserve additional space big enough for 10 elements. + /// v.reserve(10); + /// + /// let (init, uninit) = v.split_at_spare_mut(); + /// let sum = init.iter().copied().sum::(); + /// + /// // Fill in the next 4 elements. + /// uninit[0].write(sum); + /// uninit[1].write(sum * 2); + /// uninit[2].write(sum * 3); + /// uninit[3].write(sum * 4); + /// + /// // Mark the 4 elements of the vector as being initialized. + /// unsafe { + /// let len = v.len(); + /// v.set_len(len + 4); + /// } + /// + /// assert_eq!(&v, &[1, 1, 2, 4, 8, 12, 16]); + /// ``` + #[unstable(feature = "vec_split_at_spare", issue = "81944")] + #[inline] + pub fn split_at_spare_mut(&mut self) -> (&mut [T], &mut [MaybeUninit]) { + // SAFETY: + // - len is ignored and so never changed + let (init, spare, _) = unsafe { self.split_at_spare_mut_with_len() }; + (init, spare) + } + + /// Safety: changing returned .2 (&mut usize) is considered the same as calling `.set_len(_)`. + /// + /// This method provides unique access to all vec parts at once in `extend_from_within`. + unsafe fn split_at_spare_mut_with_len( + &mut self, + ) -> (&mut [T], &mut [MaybeUninit], &mut usize) { + let ptr = self.as_mut_ptr(); + // SAFETY: + // - `ptr` is guaranteed to be valid for `self.len` elements + // - but the allocation extends out to `self.buf.capacity()` elements, possibly + // uninitialized + let spare_ptr = unsafe { ptr.add(self.len) }; + let spare_ptr = spare_ptr.cast::>(); + let spare_len = self.buf.capacity() - self.len; + + // SAFETY: + // - `ptr` is guaranteed to be valid for `self.len` elements + // - `spare_ptr` is pointing one element past the buffer, so it doesn't overlap with `initialized` + unsafe { + let initialized = slice::from_raw_parts_mut(ptr, self.len); + let spare = slice::from_raw_parts_mut(spare_ptr, spare_len); + + (initialized, spare, &mut self.len) + } + } +} + +impl Vec { + /// Resizes the `Vec` in-place so that `len` is equal to `new_len`. + /// + /// If `new_len` is greater than `len`, the `Vec` is extended by the + /// difference, with each additional slot filled with `value`. + /// If `new_len` is less than `len`, the `Vec` is simply truncated. + /// + /// This method requires `T` to implement [`Clone`], + /// in order to be able to clone the passed value. + /// If you need more flexibility (or want to rely on [`Default`] instead of + /// [`Clone`]), use [`Vec::resize_with`]. + /// If you only need to resize to a smaller size, use [`Vec::truncate`]. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!["hello"]; + /// vec.resize(3, "world"); + /// assert_eq!(vec, ["hello", "world", "world"]); + /// + /// let mut vec = vec![1, 2, 3, 4]; + /// vec.resize(2, 0); + /// assert_eq!(vec, [1, 2]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_resize", since = "1.5.0")] + pub fn resize(&mut self, new_len: usize, value: T) { + let len = self.len(); + + if new_len > len { + self.extend_with(new_len - len, ExtendElement(value)) + } else { + self.truncate(new_len); + } + } + + /// Clones and appends all elements in a slice to the `Vec`. + /// + /// Iterates over the slice `other`, clones each element, and then appends + /// it to this `Vec`. The `other` slice is traversed in-order. + /// + /// Note that this function is same as [`extend`] except that it is + /// specialized to work with slices instead. If and when Rust gets + /// specialization this function will likely be deprecated (but still + /// available). + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1]; + /// vec.extend_from_slice(&[2, 3, 4]); + /// assert_eq!(vec, [1, 2, 3, 4]); + /// ``` + /// + /// [`extend`]: Vec::extend + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_extend_from_slice", since = "1.6.0")] + pub fn extend_from_slice(&mut self, other: &[T]) { + self.spec_extend(other.iter()) + } + + /// Copies elements from `src` range to the end of the vector. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![0, 1, 2, 3, 4]; + /// + /// vec.extend_from_within(2..); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4]); + /// + /// vec.extend_from_within(..2); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1]); + /// + /// vec.extend_from_within(4..8); + /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1, 4, 2, 3, 4]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[stable(feature = "vec_extend_from_within", since = "1.53.0")] + pub fn extend_from_within(&mut self, src: R) + where + R: RangeBounds, + { + let range = slice::range(src, ..self.len()); + self.reserve(range.len()); + + // SAFETY: + // - `slice::range` guarantees that the given range is valid for indexing self + unsafe { + self.spec_extend_from_within(range); + } + } +} + +impl Vec<[T; N], A> { + /// Takes a `Vec<[T; N]>` and flattens it into a `Vec`. + /// + /// # Panics + /// + /// Panics if the length of the resulting vector would overflow a `usize`. + /// + /// This is only possible when flattening a vector of arrays of zero-sized + /// types, and thus tends to be irrelevant in practice. If + /// `size_of::() > 0`, this will never panic. + /// + /// # Examples + /// + /// ``` + /// #![feature(slice_flatten)] + /// + /// let mut vec = vec![[1, 2, 3], [4, 5, 6], [7, 8, 9]]; + /// assert_eq!(vec.pop(), Some([7, 8, 9])); + /// + /// let mut flattened = vec.into_flattened(); + /// assert_eq!(flattened.pop(), Some(6)); + /// ``` + #[unstable(feature = "slice_flatten", issue = "95629")] + pub fn into_flattened(self) -> Vec { + let (ptr, len, cap, alloc) = self.into_raw_parts_with_alloc(); + let (new_len, new_cap) = if mem::size_of::() == 0 { + (len.checked_mul(N).expect("vec len overflow"), usize::MAX) + } else { + // SAFETY: + // - `cap * N` cannot overflow because the allocation is already in + // the address space. + // - Each `[T; N]` has `N` valid elements, so there are `len * N` + // valid elements in the allocation. + unsafe { (len.unchecked_mul(N), cap.unchecked_mul(N)) } + }; + // SAFETY: + // - `ptr` was allocated by `self` + // - `ptr` is well-aligned because `[T; N]` has the same alignment as `T`. + // - `new_cap` refers to the same sized allocation as `cap` because + // `new_cap * size_of::()` == `cap * size_of::<[T; N]>()` + // - `len` <= `cap`, so `len * N` <= `cap * N`. + unsafe { Vec::::from_raw_parts_in(ptr.cast(), new_len, new_cap, alloc) } + } +} + +// This code generalizes `extend_with_{element,default}`. +trait ExtendWith { + fn next(&mut self) -> T; + fn last(self) -> T; +} + +struct ExtendElement(T); +impl ExtendWith for ExtendElement { + fn next(&mut self) -> T { + self.0.clone() + } + fn last(self) -> T { + self.0 + } +} + +struct ExtendFunc(F); +impl T> ExtendWith for ExtendFunc { + fn next(&mut self) -> T { + (self.0)() + } + fn last(mut self) -> T { + (self.0)() + } +} + +impl Vec { + #[cfg(not(no_global_oom_handling))] + /// Extend the vector by `n` values, using the given generator. + fn extend_with>(&mut self, n: usize, mut value: E) { + self.reserve(n); + + unsafe { + let mut ptr = self.as_mut_ptr().add(self.len()); + // Use SetLenOnDrop to work around bug where compiler + // might not realize the store through `ptr` through self.set_len() + // don't alias. + let mut local_len = SetLenOnDrop::new(&mut self.len); + + // Write all elements except the last one + for _ in 1..n { + ptr::write(ptr, value.next()); + ptr = ptr.offset(1); + // Increment the length in every step in case next() panics + local_len.increment_len(1); + } + + if n > 0 { + // We can write the last element directly without cloning needlessly + ptr::write(ptr, value.last()); + local_len.increment_len(1); + } + + // len set by scope guard + } + } +} + +impl Vec { + /// Removes consecutive repeated elements in the vector according to the + /// [`PartialEq`] trait implementation. + /// + /// If the vector is sorted, this removes all duplicates. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec![1, 2, 2, 3, 2]; + /// + /// vec.dedup(); + /// + /// assert_eq!(vec, [1, 2, 3, 2]); + /// ``` + #[stable(feature = "rust1", since = "1.0.0")] + #[inline] + pub fn dedup(&mut self) { + self.dedup_by(|a, b| a == b) + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Internal methods and functions +//////////////////////////////////////////////////////////////////////////////// + +#[doc(hidden)] +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +pub fn from_elem(elem: T, n: usize) -> Vec { + ::from_elem(elem, n, Global) +} + +#[doc(hidden)] +#[cfg(not(no_global_oom_handling))] +#[unstable(feature = "allocator_api", issue = "32838")] +pub fn from_elem_in(elem: T, n: usize, alloc: A) -> Vec { + ::from_elem(elem, n, alloc) +} + +trait ExtendFromWithinSpec { + /// # Safety + /// + /// - `src` needs to be valid index + /// - `self.capacity() - self.len()` must be `>= src.len()` + unsafe fn spec_extend_from_within(&mut self, src: Range); +} + +impl ExtendFromWithinSpec for Vec { + default unsafe fn spec_extend_from_within(&mut self, src: Range) { + // SAFETY: + // - len is increased only after initializing elements + let (this, spare, len) = unsafe { self.split_at_spare_mut_with_len() }; + + // SAFETY: + // - caller guaratees that src is a valid index + let to_clone = unsafe { this.get_unchecked(src) }; + + iter::zip(to_clone, spare) + .map(|(src, dst)| dst.write(src.clone())) + // Note: + // - Element was just initialized with `MaybeUninit::write`, so it's ok to increase len + // - len is increased after each element to prevent leaks (see issue #82533) + .for_each(|_| *len += 1); + } +} + +impl ExtendFromWithinSpec for Vec { + unsafe fn spec_extend_from_within(&mut self, src: Range) { + let count = src.len(); + { + let (init, spare) = self.split_at_spare_mut(); + + // SAFETY: + // - caller guaratees that `src` is a valid index + let source = unsafe { init.get_unchecked(src) }; + + // SAFETY: + // - Both pointers are created from unique slice references (`&mut [_]`) + // so they are valid and do not overlap. + // - Elements are :Copy so it's OK to to copy them, without doing + // anything with the original values + // - `count` is equal to the len of `source`, so source is valid for + // `count` reads + // - `.reserve(count)` guarantees that `spare.len() >= count` so spare + // is valid for `count` writes + unsafe { ptr::copy_nonoverlapping(source.as_ptr(), spare.as_mut_ptr() as _, count) }; + } + + // SAFETY: + // - The elements were just initialized by `copy_nonoverlapping` + self.len += count; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Common trait implementations for Vec +//////////////////////////////////////////////////////////////////////////////// + +#[stable(feature = "rust1", since = "1.0.0")] +impl ops::Deref for Vec { + type Target = [T]; + + fn deref(&self) -> &[T] { + unsafe { slice::from_raw_parts(self.as_ptr(), self.len) } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl ops::DerefMut for Vec { + fn deref_mut(&mut self) -> &mut [T] { + unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) } + } +} + +#[cfg(not(no_global_oom_handling))] +trait SpecCloneFrom { + fn clone_from(this: &mut Self, other: &Self); +} + +#[cfg(not(no_global_oom_handling))] +impl SpecCloneFrom for Vec { + default fn clone_from(this: &mut Self, other: &Self) { + // drop anything that will not be overwritten + this.truncate(other.len()); + + // self.len <= other.len due to the truncate above, so the + // slices here are always in-bounds. + let (init, tail) = other.split_at(this.len()); + + // reuse the contained values' allocations/resources. + this.clone_from_slice(init); + this.extend_from_slice(tail); + } +} + +#[cfg(not(no_global_oom_handling))] +impl SpecCloneFrom for Vec { + fn clone_from(this: &mut Self, other: &Self) { + this.clear(); + this.extend_from_slice(other); + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Clone for Vec { + #[cfg(not(test))] + fn clone(&self) -> Self { + let alloc = self.allocator().clone(); + <[T]>::to_vec_in(&**self, alloc) + } + + // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is + // required for this method definition, is not available. Instead use the + // `slice::to_vec` function which is only available with cfg(test) + // NB see the slice::hack module in slice.rs for more information + #[cfg(test)] + fn clone(&self) -> Self { + let alloc = self.allocator().clone(); + crate::slice::to_vec(&**self, alloc) + } + + fn clone_from(&mut self, other: &Self) { + SpecCloneFrom::clone_from(self, other) + } +} + +/// The hash of a vector is the same as that of the corresponding slice, +/// as required by the `core::borrow::Borrow` implementation. +/// +/// ``` +/// #![feature(build_hasher_simple_hash_one)] +/// use std::hash::BuildHasher; +/// +/// let b = std::collections::hash_map::RandomState::new(); +/// let v: Vec = vec![0xa8, 0x3c, 0x09]; +/// let s: &[u8] = &[0xa8, 0x3c, 0x09]; +/// assert_eq!(b.hash_one(v), b.hash_one(s)); +/// ``` +#[stable(feature = "rust1", since = "1.0.0")] +impl Hash for Vec { + #[inline] + fn hash(&self, state: &mut H) { + Hash::hash(&**self, state) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_on_unimplemented( + message = "vector indices are of type `usize` or ranges of `usize`", + label = "vector indices are of type `usize` or ranges of `usize`" +)] +impl, A: Allocator> Index for Vec { + type Output = I::Output; + + #[inline] + fn index(&self, index: I) -> &Self::Output { + Index::index(&**self, index) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_on_unimplemented( + message = "vector indices are of type `usize` or ranges of `usize`", + label = "vector indices are of type `usize` or ranges of `usize`" +)] +impl, A: Allocator> IndexMut for Vec { + #[inline] + fn index_mut(&mut self, index: I) -> &mut Self::Output { + IndexMut::index_mut(&mut **self, index) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl FromIterator for Vec { + #[inline] + fn from_iter>(iter: I) -> Vec { + >::from_iter(iter.into_iter()) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl IntoIterator for Vec { + type Item = T; + type IntoIter = IntoIter; + + /// Creates a consuming iterator, that is, one that moves each value out of + /// the vector (from start to end). The vector cannot be used after calling + /// this. + /// + /// # Examples + /// + /// ``` + /// let v = vec!["a".to_string(), "b".to_string()]; + /// for s in v.into_iter() { + /// // s has type String, not &String + /// println!("{s}"); + /// } + /// ``` + #[inline] + fn into_iter(self) -> IntoIter { + unsafe { + let mut me = ManuallyDrop::new(self); + let alloc = ManuallyDrop::new(ptr::read(me.allocator())); + let begin = me.as_mut_ptr(); + let end = if mem::size_of::() == 0 { + arith_offset(begin as *const i8, me.len() as isize) as *const T + } else { + begin.add(me.len()) as *const T + }; + let cap = me.buf.capacity(); + IntoIter { + buf: NonNull::new_unchecked(begin), + phantom: PhantomData, + cap, + alloc, + ptr: begin, + end, + } + } + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a Vec { + type Item = &'a T; + type IntoIter = slice::Iter<'a, T>; + + fn into_iter(self) -> slice::Iter<'a, T> { + self.iter() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec { + type Item = &'a mut T; + type IntoIter = slice::IterMut<'a, T>; + + fn into_iter(self) -> slice::IterMut<'a, T> { + self.iter_mut() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl Extend for Vec { + #[inline] + fn extend>(&mut self, iter: I) { + >::spec_extend(self, iter.into_iter()) + } + + #[inline] + fn extend_one(&mut self, item: T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +impl Vec { + // leaf method to which various SpecFrom/SpecExtend implementations delegate when + // they have no further optimizations to apply + #[cfg(not(no_global_oom_handling))] + fn extend_desugared>(&mut self, mut iterator: I) { + // This is the case for a general iterator. + // + // This function should be the moral equivalent of: + // + // for item in iterator { + // self.push(item); + // } + while let Some(element) = iterator.next() { + let len = self.len(); + if len == self.capacity() { + let (lower, _) = iterator.size_hint(); + self.reserve(lower.saturating_add(1)); + } + unsafe { + ptr::write(self.as_mut_ptr().add(len), element); + // Since next() executes user code which can panic we have to bump the length + // after each step. + // NB can't overflow since we would have had to alloc the address space + self.set_len(len + 1); + } + } + } + + /// Creates a splicing iterator that replaces the specified range in the vector + /// with the given `replace_with` iterator and yields the removed items. + /// `replace_with` does not need to be the same length as `range`. + /// + /// `range` is removed even if the iterator is not consumed until the end. + /// + /// It is unspecified how many elements are removed from the vector + /// if the `Splice` value is leaked. + /// + /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. + /// + /// This is optimal if: + /// + /// * The tail (elements in the vector after `range`) is empty, + /// * or `replace_with` yields fewer or equal elements than `range`’s length + /// * or the lower bound of its `size_hint()` is exact. + /// + /// Otherwise, a temporary vector is allocated and the tail is moved twice. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3, 4]; + /// let new = [7, 8, 9]; + /// let u: Vec<_> = v.splice(1..3, new).collect(); + /// assert_eq!(v, &[1, 7, 8, 9, 4]); + /// assert_eq!(u, &[2, 3]); + /// ``` + #[cfg(not(no_global_oom_handling))] + #[inline] + #[stable(feature = "vec_splice", since = "1.21.0")] + pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, A> + where + R: RangeBounds, + I: IntoIterator, + { + Splice { drain: self.drain(range), replace_with: replace_with.into_iter() } + } + + /// Creates an iterator which uses a closure to determine if an element should be removed. + /// + /// If the closure returns true, then the element is removed and yielded. + /// If the closure returns false, the element will remain in the vector and will not be yielded + /// by the iterator. + /// + /// Using this method is equivalent to the following code: + /// + /// ``` + /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 }; + /// # let mut vec = vec![1, 2, 3, 4, 5, 6]; + /// let mut i = 0; + /// while i < vec.len() { + /// if some_predicate(&mut vec[i]) { + /// let val = vec.remove(i); + /// // your code here + /// } else { + /// i += 1; + /// } + /// } + /// + /// # assert_eq!(vec, vec![1, 4, 5]); + /// ``` + /// + /// But `drain_filter` is easier to use. `drain_filter` is also more efficient, + /// because it can backshift the elements of the array in bulk. + /// + /// Note that `drain_filter` also lets you mutate every element in the filter closure, + /// regardless of whether you choose to keep or remove it. + /// + /// # Examples + /// + /// Splitting an array into evens and odds, reusing the original allocation: + /// + /// ``` + /// #![feature(drain_filter)] + /// let mut numbers = vec![1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]; + /// + /// let evens = numbers.drain_filter(|x| *x % 2 == 0).collect::>(); + /// let odds = numbers; + /// + /// assert_eq!(evens, vec![2, 4, 6, 8, 14]); + /// assert_eq!(odds, vec![1, 3, 5, 9, 11, 13, 15]); + /// ``` + #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")] + pub fn drain_filter(&mut self, filter: F) -> DrainFilter<'_, T, F, A> + where + F: FnMut(&mut T) -> bool, + { + let old_len = self.len(); + + // Guard against us getting leaked (leak amplification) + unsafe { + self.set_len(0); + } + + DrainFilter { vec: self, idx: 0, del: 0, old_len, pred: filter, panic_flag: false } + } +} + +/// Extend implementation that copies elements out of references before pushing them onto the Vec. +/// +/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to +/// append the entire slice at once. +/// +/// [`copy_from_slice`]: slice::copy_from_slice +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "extend_ref", since = "1.2.0")] +impl<'a, T: Copy + 'a, A: Allocator + 'a> Extend<&'a T> for Vec { + fn extend>(&mut self, iter: I) { + self.spec_extend(iter.into_iter()) + } + + #[inline] + fn extend_one(&mut self, &item: &'a T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +/// Implements comparison of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl PartialOrd for Vec { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + PartialOrd::partial_cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl Eq for Vec {} + +/// Implements ordering of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison). +#[stable(feature = "rust1", since = "1.0.0")] +impl Ord for Vec { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + Ord::cmp(&**self, &**other) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +unsafe impl<#[may_dangle] T, A: Allocator> Drop for Vec { + fn drop(&mut self) { + unsafe { + // use drop for [T] + // use a raw slice to refer to the elements of the vector as weakest necessary type; + // could avoid questions of validity in certain cases + ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.as_mut_ptr(), self.len)) + } + // RawVec handles deallocation + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")] +impl const Default for Vec { + /// Creates an empty `Vec`. + fn default() -> Vec { + Vec::new() + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl fmt::Debug for Vec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&**self, f) + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRef> for Vec { + fn as_ref(&self) -> &Vec { + self + } +} + +#[stable(feature = "vec_as_mut", since = "1.5.0")] +impl AsMut> for Vec { + fn as_mut(&mut self) -> &mut Vec { + self + } +} + +#[stable(feature = "rust1", since = "1.0.0")] +impl AsRef<[T]> for Vec { + fn as_ref(&self) -> &[T] { + self + } +} + +#[stable(feature = "vec_as_mut", since = "1.5.0")] +impl AsMut<[T]> for Vec { + fn as_mut(&mut self) -> &mut [T] { + self + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From<&[T]> for Vec { + /// Allocate a `Vec` and fill it by cloning `s`'s items. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from(&[1, 2, 3][..]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: &[T]) -> Vec { + s.to_vec() + } + #[cfg(test)] + fn from(s: &[T]) -> Vec { + crate::slice::to_vec(s, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_from_mut", since = "1.19.0")] +impl From<&mut [T]> for Vec { + /// Allocate a `Vec` and fill it by cloning `s`'s items. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from(&mut [1, 2, 3][..]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: &mut [T]) -> Vec { + s.to_vec() + } + #[cfg(test)] + fn from(s: &mut [T]) -> Vec { + crate::slice::to_vec(s, Global) + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "vec_from_array", since = "1.44.0")] +impl From<[T; N]> for Vec { + /// Allocate a `Vec` and move `s`'s items into it. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from([1, 2, 3]), vec![1, 2, 3]); + /// ``` + #[cfg(not(test))] + fn from(s: [T; N]) -> Vec { + <[T]>::into_vec(box s) + } + + #[cfg(test)] + fn from(s: [T; N]) -> Vec { + crate::slice::into_vec(box s) + } +} + +#[stable(feature = "vec_from_cow_slice", since = "1.14.0")] +impl<'a, T> From> for Vec +where + [T]: ToOwned>, +{ + /// Convert a clone-on-write slice into a vector. + /// + /// If `s` already owns a `Vec`, it will be returned directly. + /// If `s` is borrowing a slice, a new `Vec` will be allocated and + /// filled by cloning `s`'s items into it. + /// + /// # Examples + /// + /// ``` + /// # use std::borrow::Cow; + /// let o: Cow<[i32]> = Cow::Owned(vec![1, 2, 3]); + /// let b: Cow<[i32]> = Cow::Borrowed(&[1, 2, 3]); + /// assert_eq!(Vec::from(o), Vec::from(b)); + /// ``` + fn from(s: Cow<'a, [T]>) -> Vec { + s.into_owned() + } +} + +// note: test pulls in libstd, which causes errors here +#[cfg(not(test))] +#[stable(feature = "vec_from_box", since = "1.18.0")] +impl From> for Vec { + /// Convert a boxed slice into a vector by transferring ownership of + /// the existing heap allocation. + /// + /// # Examples + /// + /// ``` + /// let b: Box<[i32]> = vec![1, 2, 3].into_boxed_slice(); + /// assert_eq!(Vec::from(b), vec![1, 2, 3]); + /// ``` + fn from(s: Box<[T], A>) -> Self { + s.into_vec() + } +} + +// note: test pulls in libstd, which causes errors here +#[cfg(not(no_global_oom_handling))] +#[cfg(not(test))] +#[stable(feature = "box_from_vec", since = "1.20.0")] +impl From> for Box<[T], A> { + /// Convert a vector into a boxed slice. + /// + /// If `v` has excess capacity, its items will be moved into a + /// newly-allocated buffer with exactly the right capacity. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Box::from(vec![1, 2, 3]), vec![1, 2, 3].into_boxed_slice()); + /// ``` + fn from(v: Vec) -> Self { + v.into_boxed_slice() + } +} + +#[cfg(not(no_global_oom_handling))] +#[stable(feature = "rust1", since = "1.0.0")] +impl From<&str> for Vec { + /// Allocate a `Vec` and fill it with a UTF-8 string. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(Vec::from("123"), vec![b'1', b'2', b'3']); + /// ``` + fn from(s: &str) -> Vec { + From::from(s.as_bytes()) + } +} + +#[stable(feature = "array_try_from_vec", since = "1.48.0")] +impl TryFrom> for [T; N] { + type Error = Vec; + + /// Gets the entire contents of the `Vec` as an array, + /// if its size exactly matches that of the requested array. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); + /// assert_eq!(>::new().try_into(), Ok([])); + /// ``` + /// + /// If the length doesn't match, the input comes back in `Err`: + /// ``` + /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); + /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// ``` + /// + /// If you're fine with just getting a prefix of the `Vec`, + /// you can call [`.truncate(N)`](Vec::truncate) first. + /// ``` + /// let mut v = String::from("hello world").into_bytes(); + /// v.sort(); + /// v.truncate(2); + /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// assert_eq!(a, b' '); + /// assert_eq!(b, b'd'); + /// ``` + fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { + if vec.len() != N { + return Err(vec); + } + + // SAFETY: `.set_len(0)` is always sound. + unsafe { vec.set_len(0) }; + + // SAFETY: A `Vec`'s pointer is always aligned properly, and + // the alignment the array needs is the same as the items. + // We checked earlier that we have sufficient items. + // The items will not double-drop as the `set_len` + // tells the `Vec` not to also drop them. + let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; + Ok(array) + } +} diff --git a/rust/alloc/vec/partial_eq.rs b/rust/alloc/vec/partial_eq.rs new file mode 100644 index 0000000000000000000000000000000000000000..10ad4e492287b9e4dfa0c70fa4957752c1548ccb --- /dev/null +++ b/rust/alloc/vec/partial_eq.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +use crate::alloc::Allocator; +#[cfg(not(no_global_oom_handling))] +use crate::borrow::Cow; + +use super::Vec; + +macro_rules! __impl_slice_eq1 { + ([$($vars:tt)*] $lhs:ty, $rhs:ty $(where $ty:ty: $bound:ident)?, #[$stability:meta]) => { + #[$stability] + impl PartialEq<$rhs> for $lhs + where + T: PartialEq, + $($ty: $bound)? + { + #[inline] + fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] } + #[inline] + fn ne(&self, other: &$rhs) -> bool { self[..] != other[..] } + } + } +} + +__impl_slice_eq1! { [A1: Allocator, A2: Allocator] Vec, Vec, #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, &[U], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, &mut [U], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator] &[T], Vec, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } +__impl_slice_eq1! { [A: Allocator] &mut [T], Vec, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] } +__impl_slice_eq1! { [A: Allocator] Vec, [U], #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } +__impl_slice_eq1! { [A: Allocator] [T], Vec, #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [A: Allocator] Cow<'_, [T]>, Vec where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [] Cow<'_, [T]>, &[U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +#[cfg(not(no_global_oom_handling))] +__impl_slice_eq1! { [] Cow<'_, [T]>, &mut [U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator, const N: usize] Vec, [U; N], #[stable(feature = "rust1", since = "1.0.0")] } +__impl_slice_eq1! { [A: Allocator, const N: usize] Vec, &[U; N], #[stable(feature = "rust1", since = "1.0.0")] } + +// NOTE: some less important impls are omitted to reduce code bloat +// FIXME(Centril): Reconsider this? +//__impl_slice_eq1! { [const N: usize] Vec, &mut [B; N], } +//__impl_slice_eq1! { [const N: usize] [A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] &[A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] &mut [A; N], Vec, } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, [B; N], } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &[B; N], } +//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &mut [B; N], } diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters new file mode 100644 index 0000000000000000000000000000000000000000..be4963bf720304da3d9d72128983555ed44c4c62 --- /dev/null +++ b/rust/bindgen_parameters @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 + +--opaque-type xregs_state +--opaque-type desc_struct +--opaque-type arch_lbr_state +--opaque-type local_apic + +# Packed type cannot transitively contain a `#[repr(align)]` type. +--opaque-type x86_msi_data +--opaque-type x86_msi_addr_lo + +# `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2, +# commit 2aed6b021680 ("context: Escape the try keyword properly"). +--opaque-type kunit_try_catch + +# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust +# warning. We don't need to peek into it anyway. +--opaque-type spinlock + +# `seccomp`'s comment gets understood as a doctest +--no-doc-comments diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h new file mode 100644 index 0000000000000000000000000000000000000000..c48bc284214ab1fb9afa74326b871ba8566dee1e --- /dev/null +++ b/rust/bindings/bindings_helper.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Header that contains the code (mostly headers) for which Rust bindings + * will be automatically generated by `bindgen`. + * + * Sorted alphabetically. + */ + +#include + +/* `bindgen` gets confused at certain things. */ +const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; +const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..6c50ee62c56b6ee2d210b963b5c26d615134e884 --- /dev/null +++ b/rust/bindings/lib.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bindings. +//! +//! Imports the generated bindings by `bindgen`. +//! +//! This crate may not be directly used. If you need a kernel C API that is +//! not ported or wrapped in the `kernel` crate, then do so first instead of +//! using this crate. + +#![no_std] +#![feature(core_ffi_c)] +// See . +#![cfg_attr(test, allow(deref_nullptr))] +#![cfg_attr(test, allow(unaligned_references))] +#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] +#![allow( + clippy::all, + missing_docs, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + improper_ctypes, + unreachable_pub, + unsafe_op_in_unsafe_fn +)] + +mod bindings_raw { + // Use glob import here to expose all helpers. + // Symbols defined within the module will take precedence to the glob import. + pub use super::bindings_helper::*; + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_generated.rs" + )); +} + +// When both a directly exposed symbol and a helper exists for the same function, +// the directly exposed symbol is preferred and the helper becomes dead code, so +// ignore the warning here. +#[allow(dead_code)] +mod bindings_helper { + // Import the generated bindings for types. + include!(concat!( + env!("OBJTREE"), + "/rust/bindings/bindings_helpers_generated.rs" + )); +} + +pub use bindings_raw::*; + +pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL; +pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO; diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8f39a3e685514f3d49d27c185fff0f7a7a97172 --- /dev/null +++ b/rust/compiler_builtins.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Our own `compiler_builtins`. +//! +//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`]. +//! Since we do not need the vast majority of them, we avoid the dependency +//! by providing this file. +//! +//! At the moment, some builtins are required that should not be. For instance, +//! [`core`] has 128-bit integers functionality which we should not be compiling +//! in. We will work with upstream [`core`] to provide feature flags to disable +//! the parts we do not need. For the moment, we define them to [`panic!`] at +//! runtime for simplicity to catch mistakes, instead of performing surgery +//! on `core.o`. +//! +//! In any case, all these symbols are weakened to ensure we do not override +//! those that may be provided by the rest of the kernel. +//! +//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins +//! [`compiler-rt`]: https://compiler-rt.llvm.org/ + +#![feature(compiler_builtins)] +#![compiler_builtins] +#![no_builtins] +#![no_std] + +macro_rules! define_panicking_intrinsics( + ($reason: tt, { $($ident: ident, )* }) => { + $( + #[doc(hidden)] + #[no_mangle] + pub extern "C" fn $ident() { + panic!($reason); + } + )* + } +); + +define_panicking_intrinsics!("`f32` should not be used", { + __eqsf2, + __gesf2, + __lesf2, + __nesf2, + __unordsf2, +}); + +define_panicking_intrinsics!("`f64` should not be used", { + __unorddf2, +}); + +define_panicking_intrinsics!("`i128` should not be used", { + __ashrti3, + __muloti4, + __multi3, +}); + +define_panicking_intrinsics!("`u128` should not be used", { + __ashlti3, + __lshrti3, + __udivmodti4, + __udivti3, + __umodti3, +}); diff --git a/rust/exports.c b/rust/exports.c new file mode 100644 index 0000000000000000000000000000000000000000..bb7cc64cecd0ddce7e4b727581d1f842b5e79a84 --- /dev/null +++ b/rust/exports.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * A hack to export Rust symbols for loadable modules without having to redo + * the entire `include/linux/export.h` logic in Rust. + * + * This requires the Rust's new/future `v0` mangling scheme because the default + * one ("legacy") uses invalid characters for C identifiers (thus we cannot use + * the `EXPORT_SYMBOL_*` macros). + * + * All symbols are exported as GPL-only to guarantee no GPL-only feature is + * accidentally exposed. + */ + +#include + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) + +#include "exports_core_generated.h" +#include "exports_alloc_generated.h" +#include "exports_bindings_generated.h" +#include "exports_kernel_generated.h" diff --git a/rust/helpers.c b/rust/helpers.c new file mode 100644 index 0000000000000000000000000000000000000000..b4f15eee2ffdbb7e382715bb1384d45640753c05 --- /dev/null +++ b/rust/helpers.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions + * cannot be called either. This file explicitly creates functions ("helpers") + * that wrap those so that they can be called from Rust. + * + * Even though Rust kernel modules should never use directly the bindings, some + * of these helpers need to be exported because Rust generics and inlined + * functions may not get their code generated in the crate where they are + * defined. Other helpers, called from non-inline functions, may not be + * exported, in principle. However, in general, the Rust compiler does not + * guarantee codegen will be performed for a non-inline function either. + * Therefore, this file exports all the helpers. In the future, this may be + * revisited to reduce the number of exports after the compiler is informed + * about the places codegen is required. + * + * All symbols are exported as GPL-only to guarantee no GPL-only feature is + * accidentally exposed. + */ + +#include +#include + +__noreturn void rust_helper_BUG(void) +{ + BUG(); +} +EXPORT_SYMBOL_GPL(rust_helper_BUG); + +/* + * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type + * as the Rust `usize` type, so we can use it in contexts where Rust + * expects a `usize` like slice (array) indices. `usize` is defined to be + * the same as C's `uintptr_t` type (can hold any pointer) but not + * necessarily the same as `size_t` (can hold the size of any single + * object). Most modern platforms use the same concrete integer type for + * both of them, but in case we find ourselves on a platform where + * that's not true, fail early instead of risking ABI or + * integer-overflow issues. + * + * If your platform fails this assertion, it means that you are in + * danger of integer-overflow bugs (even if you attempt to remove + * `--size_t-is-usize`). It may be easiest to change the kernel ABI on + * your platform such that `size_t` matches `uintptr_t` (i.e., to increase + * `size_t`, because `uintptr_t` has to be at least as big as `size_t`). + */ +static_assert( + sizeof(size_t) == sizeof(uintptr_t) && + __alignof__(size_t) == __alignof__(uintptr_t), + "Rust code expects C `size_t` to match Rust `usize`" +); diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs new file mode 100644 index 0000000000000000000000000000000000000000..397a3dd57a9b132f9982e213868bc558ed843f27 --- /dev/null +++ b/rust/kernel/allocator.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Allocator support. + +use core::alloc::{GlobalAlloc, Layout}; +use core::ptr; + +use crate::bindings; + +struct KernelAllocator; + +unsafe impl GlobalAlloc for KernelAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // `krealloc()` is used instead of `kmalloc()` because the latter is + // an inline function and cannot be bound to as a result. + unsafe { bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8 } + } + + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + unsafe { + bindings::kfree(ptr as *const core::ffi::c_void); + } + } +} + +#[global_allocator] +static ALLOCATOR: KernelAllocator = KernelAllocator; + +// `rustc` only generates these for some crate types. Even then, we would need +// to extract the object file that has them from the archive. For the moment, +// let's generate them ourselves instead. +// +// Note that `#[no_mangle]` implies exported too, nowadays. +#[no_mangle] +fn __rust_alloc(size: usize, _align: usize) -> *mut u8 { + unsafe { bindings::krealloc(core::ptr::null(), size, bindings::GFP_KERNEL) as *mut u8 } +} + +#[no_mangle] +fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) { + unsafe { bindings::kfree(ptr as *const core::ffi::c_void) }; +} + +#[no_mangle] +fn __rust_realloc(ptr: *mut u8, _old_size: usize, _align: usize, new_size: usize) -> *mut u8 { + unsafe { + bindings::krealloc( + ptr as *const core::ffi::c_void, + new_size, + bindings::GFP_KERNEL, + ) as *mut u8 + } +} + +#[no_mangle] +fn __rust_alloc_zeroed(size: usize, _align: usize) -> *mut u8 { + unsafe { + bindings::krealloc( + core::ptr::null(), + size, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + ) as *mut u8 + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs new file mode 100644 index 0000000000000000000000000000000000000000..466b2a8fe569b60994294d3e1ca35de511631246 --- /dev/null +++ b/rust/kernel/error.rs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel errors. +//! +//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h) + +use alloc::collections::TryReserveError; + +/// Contains the C-compatible error codes. +pub mod code { + /// Out of memory. + pub const ENOMEM: super::Error = super::Error(-(crate::bindings::ENOMEM as i32)); +} + +/// Generic integer kernel error. +/// +/// The kernel defines a set of integer generic error codes based on C and +/// POSIX ones. These codes may have a more specific meaning in some contexts. +/// +/// # Invariants +/// +/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`). +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Error(core::ffi::c_int); + +impl Error { + /// Returns the kernel error code. + pub fn to_kernel_errno(self) -> core::ffi::c_int { + self.0 + } +} + +impl From for Error { + fn from(_: TryReserveError) -> Error { + code::ENOMEM + } +} + +/// A [`Result`] with an [`Error`] error type. +/// +/// To be used as the return type for functions that may fail. +/// +/// # Error codes in C and Rust +/// +/// In C, it is common that functions indicate success or failure through +/// their return value; modifying or returning extra data through non-`const` +/// pointer parameters. In particular, in the kernel, functions that may fail +/// typically return an `int` that represents a generic error code. We model +/// those as [`Error`]. +/// +/// In Rust, it is idiomatic to model functions that may fail as returning +/// a [`Result`]. Since in the kernel many functions return an error code, +/// [`Result`] is a type alias for a [`core::result::Result`] that uses +/// [`Error`] as its error type. +/// +/// Note that even if a function does not return anything when it succeeds, +/// it should still be modeled as returning a `Result` rather than +/// just an [`Error`]. +pub type Result = core::result::Result; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..abd46261d3855b635ad45d33a6f52f62143434ac --- /dev/null +++ b/rust/kernel/lib.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` crate. +//! +//! This crate contains the kernel APIs that have been ported or wrapped for +//! usage by Rust code in the kernel and is shared by all of them. +//! +//! In other words, all the rest of the Rust code in the kernel (e.g. kernel +//! modules written in Rust) depends on [`core`], [`alloc`] and this crate. +//! +//! If you need a kernel C API that is not ported or wrapped yet here, then +//! do so first instead of bypassing this crate. + +#![no_std] +#![feature(core_ffi_c)] + +// Ensure conditional compilation based on the kernel configuration works; +// otherwise we may silently break things like initcall handling. +#[cfg(not(CONFIG_RUST))] +compile_error!("Missing kernel configuration for conditional compilation"); + +#[cfg(not(test))] +#[cfg(not(testlib))] +mod allocator; +pub mod error; +pub mod prelude; +pub mod print; +pub mod str; + +#[doc(hidden)] +pub use bindings; +pub use macros; + +/// Prefix to appear before log messages printed from within the `kernel` crate. +const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; + +/// The top level entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait Module: Sized + Sync { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init(module: &'static ThisModule) -> error::Result; +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: `include/linux/export.h` +pub struct ThisModule(*mut bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { + ThisModule(ptr) + } +} + +#[cfg(not(any(testlib, test)))] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + pr_emerg!("{}\n", info); + // SAFETY: FFI call. + unsafe { bindings::BUG() }; + // Bindgen currently does not recognize `__noreturn` so `BUG` returns `()` + // instead of `!`. See . + loop {} +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs new file mode 100644 index 0000000000000000000000000000000000000000..495e2225072685fac6476bcbf2723962a88cc87f --- /dev/null +++ b/rust/kernel/prelude.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` prelude. +//! +//! These are the most common items used by Rust code in the kernel, +//! intended to be imported by all Rust code, for convenience. +//! +//! # Examples +//! +//! ``` +//! use kernel::prelude::*; +//! ``` + +pub use super::{ + error::{Error, Result}, + pr_emerg, pr_info, ThisModule, +}; +pub use alloc::{boxed::Box, vec::Vec}; +pub use core::pin::Pin; +pub use macros::module; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs new file mode 100644 index 0000000000000000000000000000000000000000..55db5a1ba75215a6d2e14e96f6714870f5988420 --- /dev/null +++ b/rust/kernel/print.rs @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Printing facilities. +//! +//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h) +//! +//! Reference: + +use core::{ + ffi::{c_char, c_void}, + fmt, +}; + +use crate::str::RawFormatter; + +#[cfg(CONFIG_PRINTK)] +use crate::bindings; + +// Called from `vsprintf` with format specifier `%pA`. +#[no_mangle] +unsafe fn rust_fmt_argument(buf: *mut c_char, end: *mut c_char, ptr: *const c_void) -> *mut c_char { + use fmt::Write; + // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`. + let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) }; + let _ = w.write_fmt(unsafe { *(ptr as *const fmt::Arguments<'_>) }); + w.pos().cast() +} + +/// Format strings. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +pub mod format_strings { + use crate::bindings; + + /// The length we copy from the `KERN_*` kernel prefixes. + const LENGTH_PREFIX: usize = 2; + + /// The length of the fixed format strings. + pub const LENGTH: usize = 10; + + /// Generates a fixed format string for the kernel's [`_printk`]. + /// + /// The format string is always the same for a given level, i.e. for a + /// given `prefix`, which are the kernel's `KERN_*` constants. + /// + /// [`_printk`]: ../../../../include/linux/printk.h + const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { + // Ensure the `KERN_*` macros are what we expect. + assert!(prefix[0] == b'\x01'); + if is_cont { + assert!(prefix[1] == b'c'); + } else { + assert!(prefix[1] >= b'0' && prefix[1] <= b'7'); + } + assert!(prefix[2] == b'\x00'); + + let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont { + b"%pA\0\0\0\0\0" + } else { + b"%s: %pA\0" + }; + + [ + prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5], + suffix[6], suffix[7], + ] + } + + // Generate the format strings at compile-time. + // + // This avoids the compiler generating the contents on the fly in the stack. + // + // Furthermore, `static` instead of `const` is used to share the strings + // for all the kernel. + pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG); + pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO); +} + +/// Prints a message via the kernel's [`_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// # Safety +/// +/// The format string must be one of the ones in [`format_strings`], and +/// the module name must be null-terminated. +/// +/// [`_printk`]: ../../../../include/linux/_printk.h +#[doc(hidden)] +#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))] +pub unsafe fn call_printk( + format_string: &[u8; format_strings::LENGTH], + module_name: &[u8], + args: fmt::Arguments<'_>, +) { + // `_printk` does not seem to fail in any path. + #[cfg(CONFIG_PRINTK)] + unsafe { + bindings::_printk( + format_string.as_ptr() as _, + module_name.as_ptr(), + &args as *const _ as *const c_void, + ); + } +} + +/// Performs formatting and forwards the string to [`call_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +#[cfg(not(testlib))] +#[macro_export] +#[allow(clippy::crate_in_macro_def)] +macro_rules! print_macro ( + // The non-continuation cases (most of them, e.g. `INFO`). + ($format_string:path, $($arg:tt)+) => ( + // SAFETY: This hidden macro should only be called by the documented + // printing macros which ensure the format string is one of the fixed + // ones. All `__LOG_PREFIX`s are null-terminated as they are generated + // by the `module!` proc macro or fixed values defined in a kernel + // crate. + unsafe { + $crate::print::call_printk( + &$format_string, + crate::__LOG_PREFIX, + format_args!($($arg)+), + ); + } + ); +); + +/// Stub for doctests +#[cfg(testlib)] +#[macro_export] +macro_rules! print_macro ( + ($format_string:path, $e:expr, $($arg:tt)+) => ( + () + ); +); + +// We could use a macro to generate these macros. However, doing so ends +// up being a bit ugly: it requires the dollar token trick to escape `$` as +// well as playing with the `doc` attribute. Furthermore, they cannot be easily +// imported in the prelude due to [1]. So, for the moment, we just write them +// manually, like in the C side; while keeping most of the logic in another +// macro, i.e. [`print_macro`]. +// +// [1]: https://github.com/rust-lang/rust/issues/52234 + +/// Prints an emergency-level message (level 0). +/// +/// Use this level if the system is unusable. +/// +/// Equivalent to the kernel's [`pr_emerg`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// `alloc::format!` for information about the formatting syntax. +/// +/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_emerg!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_emerg ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::EMERG, $($arg)*) + ) +); + +/// Prints an info-level message (level 6). +/// +/// Use this level for informational messages. +/// +/// Equivalent to the kernel's [`pr_info`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// `alloc::format!` for information about the formatting syntax. +/// +/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_info!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_info ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::INFO, $($arg)*) + ) +); diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs new file mode 100644 index 0000000000000000000000000000000000000000..e45ff220ae50f81c040056609ca57e568b979340 --- /dev/null +++ b/rust/kernel/str.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! String representations. + +use core::fmt; + +/// Allows formatting of [`fmt::Arguments`] into a raw buffer. +/// +/// It does not fail if callers write past the end of the buffer so that they can calculate the +/// size required to fit everything. +/// +/// # Invariants +/// +/// The memory region between `pos` (inclusive) and `end` (exclusive) is valid for writes if `pos` +/// is less than `end`. +pub(crate) struct RawFormatter { + // Use `usize` to use `saturating_*` functions. + #[allow(dead_code)] + beg: usize, + pos: usize, + end: usize, +} + +impl RawFormatter { + /// Creates a new instance of [`RawFormatter`] with the given buffer pointers. + /// + /// # Safety + /// + /// If `pos` is less than `end`, then the region between `pos` (inclusive) and `end` + /// (exclusive) must be valid for writes for the lifetime of the returned [`RawFormatter`]. + pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self { + // INVARIANT: The safety requierments guarantee the type invariants. + Self { + beg: pos as _, + pos: pos as _, + end: end as _, + } + } + + /// Returns the current insert position. + /// + /// N.B. It may point to invalid memory. + pub(crate) fn pos(&self) -> *mut u8 { + self.pos as _ + } +} + +impl fmt::Write for RawFormatter { + fn write_str(&mut self, s: &str) -> fmt::Result { + // `pos` value after writing `len` bytes. This does not have to be bounded by `end`, but we + // don't want it to wrap around to 0. + let pos_new = self.pos.saturating_add(s.len()); + + // Amount that we can copy. `saturating_sub` ensures we get 0 if `pos` goes past `end`. + let len_to_copy = core::cmp::min(pos_new, self.end).saturating_sub(self.pos); + + if len_to_copy > 0 { + // SAFETY: If `len_to_copy` is non-zero, then we know `pos` has not gone past `end` + // yet, so it is valid for write per the type invariants. + unsafe { + core::ptr::copy_nonoverlapping( + s.as_bytes().as_ptr(), + self.pos as *mut u8, + len_to_copy, + ) + }; + } + + self.pos = pos_new; + Ok(()) + } +} diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs new file mode 100644 index 0000000000000000000000000000000000000000..cdc7dc6135d2afe27a289a5cb69ee5fb8551f186 --- /dev/null +++ b/rust/macros/helpers.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 + +use proc_macro::{token_stream, TokenTree}; + +pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option { + if let Some(TokenTree::Ident(ident)) = it.next() { + Some(ident.to_string()) + } else { + None + } +} + +pub(crate) fn try_literal(it: &mut token_stream::IntoIter) -> Option { + if let Some(TokenTree::Literal(literal)) = it.next() { + Some(literal.to_string()) + } else { + None + } +} + +pub(crate) fn try_byte_string(it: &mut token_stream::IntoIter) -> Option { + try_literal(it).and_then(|byte_string| { + if byte_string.starts_with("b\"") && byte_string.ends_with('\"') { + Some(byte_string[2..byte_string.len() - 1].to_string()) + } else { + None + } + }) +} + +pub(crate) fn expect_ident(it: &mut token_stream::IntoIter) -> String { + try_ident(it).expect("Expected Ident") +} + +pub(crate) fn expect_punct(it: &mut token_stream::IntoIter) -> char { + if let TokenTree::Punct(punct) = it.next().expect("Reached end of token stream for Punct") { + punct.as_char() + } else { + panic!("Expected Punct"); + } +} + +pub(crate) fn expect_byte_string(it: &mut token_stream::IntoIter) -> String { + try_byte_string(it).expect("Expected byte string") +} + +pub(crate) fn expect_end(it: &mut token_stream::IntoIter) { + if it.next().is_some() { + panic!("Expected end"); + } +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..91764bfb1f893add8230f947234454f0db922d38 --- /dev/null +++ b/rust/macros/lib.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Crate for all kernel procedural macros. + +mod helpers; +mod module; + +use proc_macro::TokenStream; + +/// Declares a kernel module. +/// +/// The `type` argument should be a type which implements the [`Module`] +/// trait. Also accepts various forms of kernel metadata. +/// +/// C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) +/// +/// [`Module`]: ../kernel/trait.Module.html +/// +/// # Examples +/// +/// ```ignore +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyModule, +/// name: b"my_kernel_module", +/// author: b"Rust for Linux Contributors", +/// description: b"My very own kernel module!", +/// license: b"GPL", +/// params: { +/// my_i32: i32 { +/// default: 42, +/// permissions: 0o000, +/// description: b"Example of i32", +/// }, +/// writeable_i32: i32 { +/// default: 42, +/// permissions: 0o644, +/// description: b"Example of i32", +/// }, +/// }, +/// } +/// +/// struct MyModule; +/// +/// impl kernel::Module for MyModule { +/// fn init() -> Result { +/// // If the parameter is writeable, then the kparam lock must be +/// // taken to read the parameter: +/// { +/// let lock = THIS_MODULE.kernel_param_lock(); +/// pr_info!("i32 param is: {}\n", writeable_i32.read(&lock)); +/// } +/// // If the parameter is read only, it can be read without locking +/// // the kernel parameters: +/// pr_info!("i32 param is: {}\n", my_i32.read()); +/// Ok(Self) +/// } +/// } +/// ``` +/// +/// # Supported argument types +/// - `type`: type which implements the [`Module`] trait (required). +/// - `name`: byte array of the name of the kernel module (required). +/// - `author`: byte array of the author of the kernel module. +/// - `description`: byte array of the description of the kernel module. +/// - `license`: byte array of the license of the kernel module (required). +/// - `alias`: byte array of alias name of the kernel module. +#[proc_macro] +pub fn module(ts: TokenStream) -> TokenStream { + module::module(ts) +} diff --git a/rust/macros/module.rs b/rust/macros/module.rs new file mode 100644 index 0000000000000000000000000000000000000000..186a5b8be23cd69900d25d8c2fe822c5b81411d2 --- /dev/null +++ b/rust/macros/module.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0 + +use crate::helpers::*; +use proc_macro::{token_stream, Literal, TokenStream, TokenTree}; +use std::fmt::Write; + +struct ModInfoBuilder<'a> { + module: &'a str, + counter: usize, + buffer: String, +} + +impl<'a> ModInfoBuilder<'a> { + fn new(module: &'a str) -> Self { + ModInfoBuilder { + module, + counter: 0, + buffer: String::new(), + } + } + + fn emit_base(&mut self, field: &str, content: &str, builtin: bool) { + let string = if builtin { + // Built-in modules prefix their modinfo strings by `module.`. + format!( + "{module}.{field}={content}\0", + module = self.module, + field = field, + content = content + ) + } else { + // Loadable modules' modinfo strings go as-is. + format!("{field}={content}\0", field = field, content = content) + }; + + write!( + &mut self.buffer, + " + {cfg} + #[doc(hidden)] + #[link_section = \".modinfo\"] + #[used] + pub static __{module}_{counter}: [u8; {length}] = *{string}; + ", + cfg = if builtin { + "#[cfg(not(MODULE))]" + } else { + "#[cfg(MODULE)]" + }, + module = self.module.to_uppercase(), + counter = self.counter, + length = string.len(), + string = Literal::byte_string(string.as_bytes()), + ) + .unwrap(); + + self.counter += 1; + } + + fn emit_only_builtin(&mut self, field: &str, content: &str) { + self.emit_base(field, content, true) + } + + fn emit_only_loadable(&mut self, field: &str, content: &str) { + self.emit_base(field, content, false) + } + + fn emit(&mut self, field: &str, content: &str) { + self.emit_only_builtin(field, content); + self.emit_only_loadable(field, content); + } +} + +#[derive(Debug, Default)] +struct ModuleInfo { + type_: String, + license: String, + name: String, + author: Option, + description: Option, + alias: Option, +} + +impl ModuleInfo { + fn parse(it: &mut token_stream::IntoIter) -> Self { + let mut info = ModuleInfo::default(); + + const EXPECTED_KEYS: &[&str] = + &["type", "name", "author", "description", "license", "alias"]; + const REQUIRED_KEYS: &[&str] = &["type", "name", "license"]; + let mut seen_keys = Vec::new(); + + loop { + let key = match it.next() { + Some(TokenTree::Ident(ident)) => ident.to_string(), + Some(_) => panic!("Expected Ident or end"), + None => break, + }; + + if seen_keys.contains(&key) { + panic!( + "Duplicated key \"{}\". Keys can only be specified once.", + key + ); + } + + assert_eq!(expect_punct(it), ':'); + + match key.as_str() { + "type" => info.type_ = expect_ident(it), + "name" => info.name = expect_byte_string(it), + "author" => info.author = Some(expect_byte_string(it)), + "description" => info.description = Some(expect_byte_string(it)), + "license" => info.license = expect_byte_string(it), + "alias" => info.alias = Some(expect_byte_string(it)), + _ => panic!( + "Unknown key \"{}\". Valid keys are: {:?}.", + key, EXPECTED_KEYS + ), + } + + assert_eq!(expect_punct(it), ','); + + seen_keys.push(key); + } + + expect_end(it); + + for key in REQUIRED_KEYS { + if !seen_keys.iter().any(|e| e == key) { + panic!("Missing required key \"{}\".", key); + } + } + + let mut ordered_keys: Vec<&str> = Vec::new(); + for key in EXPECTED_KEYS { + if seen_keys.iter().any(|e| e == key) { + ordered_keys.push(key); + } + } + + if seen_keys != ordered_keys { + panic!( + "Keys are not ordered as expected. Order them like: {:?}.", + ordered_keys + ); + } + + info + } +} + +pub(crate) fn module(ts: TokenStream) -> TokenStream { + let mut it = ts.into_iter(); + + let info = ModuleInfo::parse(&mut it); + + let mut modinfo = ModInfoBuilder::new(info.name.as_ref()); + if let Some(author) = info.author { + modinfo.emit("author", &author); + } + if let Some(description) = info.description { + modinfo.emit("description", &description); + } + modinfo.emit("license", &info.license); + if let Some(alias) = info.alias { + modinfo.emit("alias", &alias); + } + + // Built-in modules also export the `file` modinfo string. + let file = + std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); + modinfo.emit_only_builtin("file", &file); + + format!( + " + /// The module name. + /// + /// Used by the printing macros, e.g. [`info!`]. + const __LOG_PREFIX: &[u8] = b\"{name}\\0\"; + + /// The \"Rust loadable module\" mark, for `scripts/is_rust_module.sh`. + // + // This may be best done another way later on, e.g. as a new modinfo + // key or a new section. For the moment, keep it simple. + #[cfg(MODULE)] + #[doc(hidden)] + #[used] + static __IS_RUST_MODULE: () = (); + + static mut __MOD: Option<{type_}> = None; + + // SAFETY: `__this_module` is constructed by the kernel at load time and will not be + // freed until the module is unloaded. + #[cfg(MODULE)] + static THIS_MODULE: kernel::ThisModule = unsafe {{ + kernel::ThisModule::from_ptr(&kernel::bindings::__this_module as *const _ as *mut _) + }}; + #[cfg(not(MODULE))] + static THIS_MODULE: kernel::ThisModule = unsafe {{ + kernel::ThisModule::from_ptr(core::ptr::null_mut()) + }}; + + // Loadable modules need to export the `{{init,cleanup}}_module` identifiers. + #[cfg(MODULE)] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn init_module() -> core::ffi::c_int {{ + __init() + }} + + #[cfg(MODULE)] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn cleanup_module() {{ + __exit() + }} + + // Built-in modules are initialized through an initcall pointer + // and the identifiers need to be unique. + #[cfg(not(MODULE))] + #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] + #[doc(hidden)] + #[link_section = \"{initcall_section}\"] + #[used] + pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init; + + #[cfg(not(MODULE))] + #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] + core::arch::global_asm!( + r#\".section \"{initcall_section}\", \"a\" + __{name}_initcall: + .long __{name}_init - . + .previous + \"# + ); + + #[cfg(not(MODULE))] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{ + __init() + }} + + #[cfg(not(MODULE))] + #[doc(hidden)] + #[no_mangle] + pub extern \"C\" fn __{name}_exit() {{ + __exit() + }} + + fn __init() -> core::ffi::c_int {{ + match <{type_} as kernel::Module>::init(&THIS_MODULE) {{ + Ok(m) => {{ + unsafe {{ + __MOD = Some(m); + }} + return 0; + }} + Err(e) => {{ + return e.to_kernel_errno(); + }} + }} + }} + + fn __exit() {{ + unsafe {{ + // Invokes `drop()` on `__MOD`, which should be used for cleanup. + __MOD = None; + }} + }} + + {modinfo} + ", + type_ = info.type_, + name = info.name, + modinfo = modinfo.buffer, + initcall_section = ".initcall6.init" + ) + .parse() + .expect("Error parsing formatted string into token stream.") +} diff --git a/samples/Kconfig b/samples/Kconfig index 470ee3baf2e16db4c5f4534eb8036d4c3c4450d5..0d81c00289ee3684af5db35756f4e0ceb294344e 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -263,6 +263,8 @@ config SAMPLE_CORESIGHT_SYSCFG This demonstrates how a user may create their own CoreSight configurations and easily load them into the system at runtime. +source "samples/rust/Kconfig" + endif # SAMPLES config HAVE_SAMPLE_FTRACE_DIRECT diff --git a/samples/Makefile b/samples/Makefile index 701e912ab5afeeb89b6995e98ffbc5d90bef98a7..9832ef3f8fcbaf293a861ece56c594559e6cb965 100644 --- a/samples/Makefile +++ b/samples/Makefile @@ -35,3 +35,4 @@ subdir-$(CONFIG_SAMPLE_WATCH_QUEUE) += watch_queue obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak/ obj-$(CONFIG_SAMPLE_CORESIGHT_SYSCFG) += coresight/ obj-$(CONFIG_SAMPLE_FPROBE) += fprobe/ +obj-$(CONFIG_SAMPLES_RUST) += rust/ diff --git a/samples/bpf/map_perf_test_kern.c b/samples/bpf/map_perf_test_kern.c index 8773f22b6a983cd51c1625270c3f930d6ac6d34b..7342c5b2f27873f7a620f3b4c841459f0c853c5a 100644 --- a/samples/bpf/map_perf_test_kern.c +++ b/samples/bpf/map_perf_test_kern.c @@ -108,11 +108,14 @@ int stress_hmap(struct pt_regs *ctx) u32 key = bpf_get_current_pid_tgid(); long init_val = 1; long *value; + int i; - bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY); - value = bpf_map_lookup_elem(&hash_map, &key); - if (value) - bpf_map_delete_elem(&hash_map, &key); + for (i = 0; i < 10; i++) { + bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY); + value = bpf_map_lookup_elem(&hash_map, &key); + if (value) + bpf_map_delete_elem(&hash_map, &key); + } return 0; } @@ -123,11 +126,14 @@ int stress_percpu_hmap(struct pt_regs *ctx) u32 key = bpf_get_current_pid_tgid(); long init_val = 1; long *value; + int i; - bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY); - value = bpf_map_lookup_elem(&percpu_hash_map, &key); - if (value) - bpf_map_delete_elem(&percpu_hash_map, &key); + for (i = 0; i < 10; i++) { + bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY); + value = bpf_map_lookup_elem(&percpu_hash_map, &key); + if (value) + bpf_map_delete_elem(&percpu_hash_map, &key); + } return 0; } @@ -137,11 +143,14 @@ int stress_hmap_alloc(struct pt_regs *ctx) u32 key = bpf_get_current_pid_tgid(); long init_val = 1; long *value; + int i; - bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY); - value = bpf_map_lookup_elem(&hash_map_alloc, &key); - if (value) - bpf_map_delete_elem(&hash_map_alloc, &key); + for (i = 0; i < 10; i++) { + bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY); + value = bpf_map_lookup_elem(&hash_map_alloc, &key); + if (value) + bpf_map_delete_elem(&hash_map_alloc, &key); + } return 0; } @@ -151,11 +160,14 @@ int stress_percpu_hmap_alloc(struct pt_regs *ctx) u32 key = bpf_get_current_pid_tgid(); long init_val = 1; long *value; + int i; - bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY); - value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key); - if (value) - bpf_map_delete_elem(&percpu_hash_map_alloc, &key); + for (i = 0; i < 10; i++) { + bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY); + value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key); + if (value) + bpf_map_delete_elem(&percpu_hash_map_alloc, &key); + } return 0; } diff --git a/samples/bpf/map_perf_test_user.c b/samples/bpf/map_perf_test_user.c index b6fc174ab1f2148a2f99ca05be9625c70fc90af7..1bb53f4b29e11efc39458713c4871ac4c0ccecd4 100644 --- a/samples/bpf/map_perf_test_user.c +++ b/samples/bpf/map_perf_test_user.c @@ -72,7 +72,7 @@ static int test_flags = ~0; static uint32_t num_map_entries; static uint32_t inner_lru_hash_size; static int lru_hash_lookup_test_entries = 32; -static uint32_t max_cnt = 1000000; +static uint32_t max_cnt = 10000; static int check_test_flags(enum test_type t) { diff --git a/samples/bpf/task_fd_query_kern.c b/samples/bpf/task_fd_query_kern.c index c821294e17742b36f2dc5262506cab193b813ba3..186ac0a79c0af98e11cff73ba8e1c0db07079e7f 100644 --- a/samples/bpf/task_fd_query_kern.c +++ b/samples/bpf/task_fd_query_kern.c @@ -10,7 +10,7 @@ int bpf_prog1(struct pt_regs *ctx) return 0; } -SEC("kretprobe/blk_account_io_done") +SEC("kretprobe/__blk_account_io_done") int bpf_prog2(struct pt_regs *ctx) { return 0; diff --git a/samples/bpf/task_fd_query_user.c b/samples/bpf/task_fd_query_user.c index 424718c0872c57436221ea04cadd5679c19028f6..a33d74bd3a4b7b009db7d646c21b5d5ddde98bdd 100644 --- a/samples/bpf/task_fd_query_user.c +++ b/samples/bpf/task_fd_query_user.c @@ -348,7 +348,7 @@ int main(int argc, char **argv) /* test two functions in the corresponding *_kern.c file */ CHECK_AND_RET(test_debug_fs_kprobe(0, "blk_mq_start_request", BPF_FD_TYPE_KPROBE)); - CHECK_AND_RET(test_debug_fs_kprobe(1, "blk_account_io_done", + CHECK_AND_RET(test_debug_fs_kprobe(1, "__blk_account_io_done", BPF_FD_TYPE_KRETPROBE)); /* test nondebug fs kprobe */ diff --git a/samples/bpf/tracex3_kern.c b/samples/bpf/tracex3_kern.c index 710a4410b2fb81b82eefff2ad9dc1d84829326bd..bde6591cb20c548b061ed24cfdf4d34dc2ca5b62 100644 --- a/samples/bpf/tracex3_kern.c +++ b/samples/bpf/tracex3_kern.c @@ -49,7 +49,7 @@ struct { __uint(max_entries, SLOTS); } lat_map SEC(".maps"); -SEC("kprobe/blk_account_io_done") +SEC("kprobe/__blk_account_io_done") int bpf_prog2(struct pt_regs *ctx) { long rq = PT_REGS_PARM1(ctx); diff --git a/samples/bpf/xdp_router_ipv4_user.c b/samples/bpf/xdp_router_ipv4_user.c index 294fc15ad1cb24800a6223e5f8addd3ec9e5264e..683913bbf279763c420d7e70730a6ce5309885f4 100644 --- a/samples/bpf/xdp_router_ipv4_user.c +++ b/samples/bpf/xdp_router_ipv4_user.c @@ -209,7 +209,7 @@ static void read_route(struct nlmsghdr *nh, int nll) /* Rereading the route table to check if * there is an entry with the same * prefix but a different metric as the - * deleted enty. + * deleted entry. */ get_route_table(AF_INET); } else if (prefix_key->data[0] == diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c index 3e404e51ec6489225ccb1f4e429f4a09a8dc84d9..f29bb3c72230785cd6edba16ee82582751b0377f 100644 --- a/samples/landlock/sandboxer.c +++ b/samples/landlock/sandboxer.c @@ -162,11 +162,10 @@ out_free_name: LANDLOCK_ACCESS_FS_MAKE_SYM | \ LANDLOCK_ACCESS_FS_REFER) -#define ACCESS_ABI_2 ( \ - LANDLOCK_ACCESS_FS_REFER) - /* clang-format on */ +#define LANDLOCK_ABI_LAST 2 + int main(const int argc, char *const argv[], char *const *const envp) { const char *cmd_path; @@ -196,8 +195,12 @@ int main(const int argc, char *const argv[], char *const *const envp) "\nexample:\n" "%s=\"/bin:/lib:/usr:/proc:/etc:/dev/urandom\" " "%s=\"/dev/null:/dev/full:/dev/zero:/dev/pts:/tmp\" " - "%s bash -i\n", + "%s bash -i\n\n", ENV_FS_RO_NAME, ENV_FS_RW_NAME, argv[0]); + fprintf(stderr, + "This sandboxer can use Landlock features " + "up to ABI version %d.\n", + LANDLOCK_ABI_LAST); return 1; } @@ -225,12 +228,30 @@ int main(const int argc, char *const argv[], char *const *const envp) } return 1; } + /* Best-effort security. */ - if (abi < 2) { - ruleset_attr.handled_access_fs &= ~ACCESS_ABI_2; - access_fs_ro &= ~ACCESS_ABI_2; - access_fs_rw &= ~ACCESS_ABI_2; + switch (abi) { + case 1: + /* Removes LANDLOCK_ACCESS_FS_REFER for ABI < 2 */ + ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_REFER; + + fprintf(stderr, + "Hint: You should update the running kernel " + "to leverage Landlock features " + "provided by ABI version %d (instead of %d).\n", + LANDLOCK_ABI_LAST, abi); + __attribute__((fallthrough)); + case LANDLOCK_ABI_LAST: + break; + default: + fprintf(stderr, + "Hint: You should update this sandboxer " + "to leverage Landlock features " + "provided by ABI version %d (instead of %d).\n", + abi, LANDLOCK_ABI_LAST); } + access_fs_ro &= ruleset_attr.handled_access_fs; + access_fs_rw &= ruleset_attr.handled_access_fs; ruleset_fd = landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); diff --git a/samples/qmi/qmi_sample_client.c b/samples/qmi/qmi_sample_client.c index 78fcedbd25e2634ecd77d3970b449da07fab5c30..c045e3d2432660b85085ae5d3ad55a159e6e8773 100644 --- a/samples/qmi/qmi_sample_client.c +++ b/samples/qmi/qmi_sample_client.c @@ -42,7 +42,7 @@ struct test_name_type_v01 { char name[TEST_MAX_NAME_SIZE_V01]; }; -static struct qmi_elem_info test_name_type_v01_ei[] = { +static const struct qmi_elem_info test_name_type_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -71,7 +71,7 @@ struct test_ping_req_msg_v01 { struct test_name_type_v01 client_name; }; -static struct qmi_elem_info test_ping_req_msg_v01_ei[] = { +static const struct qmi_elem_info test_ping_req_msg_v01_ei[] = { { .data_type = QMI_UNSIGNED_1_BYTE, .elem_len = 4, @@ -113,7 +113,7 @@ struct test_ping_resp_msg_v01 { struct test_name_type_v01 service_name; }; -static struct qmi_elem_info test_ping_resp_msg_v01_ei[] = { +static const struct qmi_elem_info test_ping_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, @@ -172,7 +172,7 @@ struct test_data_req_msg_v01 { struct test_name_type_v01 client_name; }; -static struct qmi_elem_info test_data_req_msg_v01_ei[] = { +static const struct qmi_elem_info test_data_req_msg_v01_ei[] = { { .data_type = QMI_DATA_LEN, .elem_len = 1, @@ -224,7 +224,7 @@ struct test_data_resp_msg_v01 { struct test_name_type_v01 service_name; }; -static struct qmi_elem_info test_data_resp_msg_v01_ei[] = { +static const struct qmi_elem_info test_data_resp_msg_v01_ei[] = { { .data_type = QMI_STRUCT, .elem_len = 1, diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..841e0906e9439473b8e365dad917ca89b5e05668 --- /dev/null +++ b/samples/rust/Kconfig @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0 + +menuconfig SAMPLES_RUST + bool "Rust samples" + depends on RUST + help + You can build sample Rust kernel code here. + + If unsure, say N. + +if SAMPLES_RUST + +config SAMPLE_RUST_MINIMAL + tristate "Minimal" + help + This option builds the Rust minimal module sample. + + To compile this as a module, choose M here: + the module will be called rust_minimal. + + If unsure, say N. + +config SAMPLE_RUST_HOSTPROGS + bool "Host programs" + help + This option builds the Rust host program samples. + + If unsure, say N. + +endif # SAMPLES_RUST diff --git a/samples/rust/Makefile b/samples/rust/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..1daba5f8658af96027a2500d3355959acf08182a --- /dev/null +++ b/samples/rust/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SAMPLE_RUST_MINIMAL) += rust_minimal.o + +subdir-$(CONFIG_SAMPLE_RUST_HOSTPROGS) += hostprogs diff --git a/samples/rust/hostprogs/.gitignore b/samples/rust/hostprogs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a6c173da5048dc1d6907b7983952467cdf5af64a --- /dev/null +++ b/samples/rust/hostprogs/.gitignore @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +single diff --git a/samples/rust/hostprogs/Makefile b/samples/rust/hostprogs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..8ddcbd7416db5d7425a27455d9223ce013e2dd42 --- /dev/null +++ b/samples/rust/hostprogs/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +hostprogs-always-y := single + +single-rust := y diff --git a/samples/rust/hostprogs/a.rs b/samples/rust/hostprogs/a.rs new file mode 100644 index 0000000000000000000000000000000000000000..f7a4a3d0f4e0b53bf3f61a2ba46bdff5a4139a6f --- /dev/null +++ b/samples/rust/hostprogs/a.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust single host program sample: module `a`. + +pub(crate) fn f(x: i32) { + println!("The number is {}.", x); +} diff --git a/samples/rust/hostprogs/b.rs b/samples/rust/hostprogs/b.rs new file mode 100644 index 0000000000000000000000000000000000000000..c1675890648fd9940c624c485749488731301e91 --- /dev/null +++ b/samples/rust/hostprogs/b.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust single host program sample: module `b`. + +pub(crate) const CONSTANT: i32 = 42; diff --git a/samples/rust/hostprogs/single.rs b/samples/rust/hostprogs/single.rs new file mode 100644 index 0000000000000000000000000000000000000000..8c48a119339a88587904cfcf9c1c7bade525cf01 --- /dev/null +++ b/samples/rust/hostprogs/single.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust single host program sample. + +mod a; +mod b; + +fn main() { + println!("Hello world!"); + + a::f(b::CONSTANT); +} diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs new file mode 100644 index 0000000000000000000000000000000000000000..54ad17685742683cd782ba2d885edf2ce77490e5 --- /dev/null +++ b/samples/rust/rust_minimal.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust minimal sample. + +use kernel::prelude::*; + +module! { + type: RustMinimal, + name: b"rust_minimal", + author: b"Rust for Linux Contributors", + description: b"Rust minimal sample", + license: b"GPL", +} + +struct RustMinimal { + numbers: Vec, +} + +impl kernel::Module for RustMinimal { + fn init(_module: &'static ThisModule) -> Result { + pr_info!("Rust minimal sample (init)\n"); + pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); + + let mut numbers = Vec::new(); + numbers.try_push(72)?; + numbers.try_push(108)?; + numbers.try_push(200)?; + + Ok(RustMinimal { numbers }) + } +} + +impl Drop for RustMinimal { + fn drop(&mut self) { + pr_info!("My numbers are {:?}\n", self.numbers); + pr_info!("Rust minimal sample (exit)\n"); + } +} diff --git a/samples/user_events/example.c b/samples/user_events/example.c index 4f5778e441c0b503ed20bb282992bdac96c14c6f..d06dc24156ec03360b6342836af6f9b89b71cfe6 100644 --- a/samples/user_events/example.c +++ b/samples/user_events/example.c @@ -12,13 +12,21 @@ #include #include #include +#include +#include #include +#if __BITS_PER_LONG == 64 +#define endian_swap(x) htole64(x) +#else +#define endian_swap(x) htole32(x) +#endif + /* Assumes debugfs is mounted */ const char *data_file = "/sys/kernel/debug/tracing/user_events_data"; const char *status_file = "/sys/kernel/debug/tracing/user_events_status"; -static int event_status(char **status) +static int event_status(long **status) { int fd = open(status_file, O_RDONLY); @@ -33,7 +41,8 @@ static int event_status(char **status) return 0; } -static int event_reg(int fd, const char *command, int *status, int *write) +static int event_reg(int fd, const char *command, long *index, long *mask, + int *write) { struct user_reg reg = {0}; @@ -43,7 +52,8 @@ static int event_reg(int fd, const char *command, int *status, int *write) if (ioctl(fd, DIAG_IOCSREG, ®) == -1) return -1; - *status = reg.status_index; + *index = reg.status_bit / __BITS_PER_LONG; + *mask = endian_swap(1L << (reg.status_bit % __BITS_PER_LONG)); *write = reg.write_index; return 0; @@ -51,8 +61,9 @@ static int event_reg(int fd, const char *command, int *status, int *write) int main(int argc, char **argv) { - int data_fd, status, write; - char *status_page; + int data_fd, write; + long index, mask; + long *status_page; struct iovec io[2]; __u32 count = 0; @@ -61,7 +72,7 @@ int main(int argc, char **argv) data_fd = open(data_file, O_RDWR); - if (event_reg(data_fd, "test u32 count", &status, &write) == -1) + if (event_reg(data_fd, "test u32 count", &index, &mask, &write) == -1) return errno; /* Setup iovec */ @@ -75,7 +86,7 @@ ask: getchar(); /* Check if anyone is listening */ - if (status_page[status]) { + if (status_page[index] & mask) { /* Yep, trace out our data */ writev(data_fd, (const struct iovec *)io, 2); diff --git a/samples/vfio-mdev/mbochs.c b/samples/vfio-mdev/mbochs.c index 344c2901a82bf49ab0413a1c9d46bd03b4055b5f..117a8d799f711205e550d512c5ee41bcbbe8f698 100644 --- a/samples/vfio-mdev/mbochs.c +++ b/samples/vfio-mdev/mbochs.c @@ -21,7 +21,6 @@ */ #include #include -#include #include #include #include @@ -100,35 +99,44 @@ MODULE_PARM_DESC(mem, "megabytes available to " MBOCHS_NAME " devices"); #define MBOCHS_TYPE_2 "medium" #define MBOCHS_TYPE_3 "large" -static const struct mbochs_type { - const char *name; +static struct mbochs_type { + struct mdev_type type; u32 mbytes; u32 max_x; u32 max_y; } mbochs_types[] = { { - .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_1, + .type.sysfs_name = MBOCHS_TYPE_1, + .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_1, .mbytes = 4, .max_x = 800, .max_y = 600, }, { - .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_2, + .type.sysfs_name = MBOCHS_TYPE_2, + .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_2, .mbytes = 16, .max_x = 1920, .max_y = 1440, }, { - .name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_3, + .type.sysfs_name = MBOCHS_TYPE_3, + .type.pretty_name = MBOCHS_CLASS_NAME "-" MBOCHS_TYPE_3, .mbytes = 64, .max_x = 0, .max_y = 0, }, }; +static struct mdev_type *mbochs_mdev_types[] = { + &mbochs_types[0].type, + &mbochs_types[1].type, + &mbochs_types[2].type, +}; static dev_t mbochs_devt; static struct class *mbochs_class; static struct cdev mbochs_cdev; static struct device mbochs_dev; +static struct mdev_parent mbochs_parent; static atomic_t mbochs_avail_mbytes; static const struct vfio_device_ops mbochs_dev_ops; @@ -505,13 +513,14 @@ static int mbochs_reset(struct mdev_state *mdev_state) return 0; } -static int mbochs_probe(struct mdev_device *mdev) +static int mbochs_init_dev(struct vfio_device *vdev) { + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + struct mdev_device *mdev = to_mdev_device(vdev->dev); + struct mbochs_type *type = + container_of(mdev->type, struct mbochs_type, type); int avail_mbytes = atomic_read(&mbochs_avail_mbytes); - const struct mbochs_type *type = - &mbochs_types[mdev_get_type_group_id(mdev)]; - struct device *dev = mdev_dev(mdev); - struct mdev_state *mdev_state; int ret = -ENOMEM; do { @@ -520,14 +529,9 @@ static int mbochs_probe(struct mdev_device *mdev) } while (!atomic_try_cmpxchg(&mbochs_avail_mbytes, &avail_mbytes, avail_mbytes - type->mbytes)); - mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL); - if (mdev_state == NULL) - goto err_avail; - vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mbochs_dev_ops); - mdev_state->vconfig = kzalloc(MBOCHS_CONFIG_SPACE_SIZE, GFP_KERNEL); - if (mdev_state->vconfig == NULL) - goto err_mem; + if (!mdev_state->vconfig) + goto err_avail; mdev_state->memsize = type->mbytes * 1024 * 1024; mdev_state->pagecount = mdev_state->memsize >> PAGE_SHIFT; @@ -535,10 +539,7 @@ static int mbochs_probe(struct mdev_device *mdev) sizeof(struct page *), GFP_KERNEL); if (!mdev_state->pages) - goto err_mem; - - dev_info(dev, "%s: %s, %d MB, %ld pages\n", __func__, - type->name, type->mbytes, mdev_state->pagecount); + goto err_vconfig; mutex_init(&mdev_state->ops_lock); mdev_state->mdev = mdev; @@ -553,19 +554,47 @@ static int mbochs_probe(struct mdev_device *mdev) mbochs_create_config_space(mdev_state); mbochs_reset(mdev_state); + dev_info(vdev->dev, "%s: %s, %d MB, %ld pages\n", __func__, + type->type.pretty_name, type->mbytes, mdev_state->pagecount); + return 0; + +err_vconfig: + kfree(mdev_state->vconfig); +err_avail: + atomic_add(type->mbytes, &mbochs_avail_mbytes); + return ret; +} + +static int mbochs_probe(struct mdev_device *mdev) +{ + struct mdev_state *mdev_state; + int ret = -ENOMEM; + + mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev, + &mbochs_dev_ops); + if (IS_ERR(mdev_state)) + return PTR_ERR(mdev_state); + ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev); if (ret) - goto err_mem; + goto err_put_vdev; dev_set_drvdata(&mdev->dev, mdev_state); return 0; -err_mem: - vfio_uninit_group_dev(&mdev_state->vdev); + +err_put_vdev: + vfio_put_device(&mdev_state->vdev); + return ret; +} + +static void mbochs_release_dev(struct vfio_device *vdev) +{ + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + + atomic_add(mdev_state->type->mbytes, &mbochs_avail_mbytes); kfree(mdev_state->pages); kfree(mdev_state->vconfig); - kfree(mdev_state); -err_avail: - atomic_add(type->mbytes, &mbochs_avail_mbytes); - return ret; + vfio_free_device(vdev); } static void mbochs_remove(struct mdev_device *mdev) @@ -573,11 +602,7 @@ static void mbochs_remove(struct mdev_device *mdev) struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev); vfio_unregister_group_dev(&mdev_state->vdev); - vfio_uninit_group_dev(&mdev_state->vdev); - atomic_add(mdev_state->type->mbytes, &mbochs_avail_mbytes); - kfree(mdev_state->pages); - kfree(mdev_state->vconfig); - kfree(mdev_state); + vfio_put_device(&mdev_state->vdev); } static ssize_t mbochs_read(struct vfio_device *vdev, char __user *buf, @@ -1325,78 +1350,27 @@ static const struct attribute_group *mdev_dev_groups[] = { NULL, }; -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) +static ssize_t mbochs_show_description(struct mdev_type *mtype, char *buf) { - const struct mbochs_type *type = - &mbochs_types[mtype_get_type_group_id(mtype)]; - - return sprintf(buf, "%s\n", type->name); -} -static MDEV_TYPE_ATTR_RO(name); - -static ssize_t description_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - const struct mbochs_type *type = - &mbochs_types[mtype_get_type_group_id(mtype)]; + struct mbochs_type *type = + container_of(mtype, struct mbochs_type, type); return sprintf(buf, "virtual display, %d MB video memory\n", type ? type->mbytes : 0); } -static MDEV_TYPE_ATTR_RO(description); -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) +static unsigned int mbochs_get_available(struct mdev_type *mtype) { - const struct mbochs_type *type = - &mbochs_types[mtype_get_type_group_id(mtype)]; - int count = atomic_read(&mbochs_avail_mbytes) / type->mbytes; - - return sprintf(buf, "%d\n", count); -} -static MDEV_TYPE_ATTR_RO(available_instances); + struct mbochs_type *type = + container_of(mtype, struct mbochs_type, type); -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); + return atomic_read(&mbochs_avail_mbytes) / type->mbytes; } -static MDEV_TYPE_ATTR_RO(device_api); - -static struct attribute *mdev_types_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_description.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_available_instances.attr, - NULL, -}; - -static struct attribute_group mdev_type_group1 = { - .name = MBOCHS_TYPE_1, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group mdev_type_group2 = { - .name = MBOCHS_TYPE_2, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group mdev_type_group3 = { - .name = MBOCHS_TYPE_3, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group *mdev_type_groups[] = { - &mdev_type_group1, - &mdev_type_group2, - &mdev_type_group3, - NULL, -}; static const struct vfio_device_ops mbochs_dev_ops = { .close_device = mbochs_close_device, + .init = mbochs_init_dev, + .release = mbochs_release_dev, .read = mbochs_read, .write = mbochs_write, .ioctl = mbochs_ioctl, @@ -1404,6 +1378,7 @@ static const struct vfio_device_ops mbochs_dev_ops = { }; static struct mdev_driver mbochs_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, .driver = { .name = "mbochs", .owner = THIS_MODULE, @@ -1412,7 +1387,8 @@ static struct mdev_driver mbochs_driver = { }, .probe = mbochs_probe, .remove = mbochs_remove, - .supported_type_groups = mdev_type_groups, + .get_available = mbochs_get_available, + .show_description = mbochs_show_description, }; static const struct file_operations vd_fops = { @@ -1457,7 +1433,9 @@ static int __init mbochs_dev_init(void) if (ret) goto err_class; - ret = mdev_register_device(&mbochs_dev, &mbochs_driver); + ret = mdev_register_parent(&mbochs_parent, &mbochs_dev, &mbochs_driver, + mbochs_mdev_types, + ARRAY_SIZE(mbochs_mdev_types)); if (ret) goto err_device; @@ -1478,7 +1456,7 @@ err_cdev: static void __exit mbochs_dev_exit(void) { mbochs_dev.bus = NULL; - mdev_unregister_device(&mbochs_dev); + mdev_unregister_parent(&mbochs_parent); device_unregister(&mbochs_dev); mdev_unregister_driver(&mbochs_driver); diff --git a/samples/vfio-mdev/mdpy.c b/samples/vfio-mdev/mdpy.c index e8c46eb2e246858e571b8cc50613c0f2dca46b7b..946e8cfde6fdd649cb01b8eea63cdc0ae76a5848 100644 --- a/samples/vfio-mdev/mdpy.c +++ b/samples/vfio-mdev/mdpy.c @@ -17,7 +17,6 @@ */ #include #include -#include #include #include #include @@ -43,36 +42,34 @@ MODULE_LICENSE("GPL v2"); -static int max_devices = 4; -module_param_named(count, max_devices, int, 0444); -MODULE_PARM_DESC(count, "number of " MDPY_NAME " devices"); - - #define MDPY_TYPE_1 "vga" #define MDPY_TYPE_2 "xga" #define MDPY_TYPE_3 "hd" -static const struct mdpy_type { - const char *name; +static struct mdpy_type { + struct mdev_type type; u32 format; u32 bytepp; u32 width; u32 height; } mdpy_types[] = { { - .name = MDPY_CLASS_NAME "-" MDPY_TYPE_1, + .type.sysfs_name = MDPY_TYPE_1, + .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_1, .format = DRM_FORMAT_XRGB8888, .bytepp = 4, .width = 640, .height = 480, }, { - .name = MDPY_CLASS_NAME "-" MDPY_TYPE_2, + .type.sysfs_name = MDPY_TYPE_2, + .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_2, .format = DRM_FORMAT_XRGB8888, .bytepp = 4, .width = 1024, .height = 768, }, { - .name = MDPY_CLASS_NAME "-" MDPY_TYPE_3, + .type.sysfs_name = MDPY_TYPE_3, + .type.pretty_name = MDPY_CLASS_NAME "-" MDPY_TYPE_3, .format = DRM_FORMAT_XRGB8888, .bytepp = 4, .width = 1920, @@ -80,11 +77,17 @@ static const struct mdpy_type { }, }; +static struct mdev_type *mdpy_mdev_types[] = { + &mdpy_types[0].type, + &mdpy_types[1].type, + &mdpy_types[2].type, +}; + static dev_t mdpy_devt; static struct class *mdpy_class; static struct cdev mdpy_cdev; static struct device mdpy_dev; -static u32 mdpy_count; +static struct mdev_parent mdpy_parent; static const struct vfio_device_ops mdpy_dev_ops; /* State of each mdev device */ @@ -216,61 +219,71 @@ static int mdpy_reset(struct mdev_state *mdev_state) return 0; } -static int mdpy_probe(struct mdev_device *mdev) +static int mdpy_init_dev(struct vfio_device *vdev) { + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + struct mdev_device *mdev = to_mdev_device(vdev->dev); const struct mdpy_type *type = - &mdpy_types[mdev_get_type_group_id(mdev)]; - struct device *dev = mdev_dev(mdev); - struct mdev_state *mdev_state; + container_of(mdev->type, struct mdpy_type, type); u32 fbsize; - int ret; - - if (mdpy_count >= max_devices) - return -ENOMEM; - - mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL); - if (mdev_state == NULL) - return -ENOMEM; - vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mdpy_dev_ops); + int ret = -ENOMEM; mdev_state->vconfig = kzalloc(MDPY_CONFIG_SPACE_SIZE, GFP_KERNEL); - if (mdev_state->vconfig == NULL) { - ret = -ENOMEM; - goto err_state; - } + if (!mdev_state->vconfig) + return ret; fbsize = roundup_pow_of_two(type->width * type->height * type->bytepp); mdev_state->memblk = vmalloc_user(fbsize); - if (!mdev_state->memblk) { - ret = -ENOMEM; - goto err_vconfig; - } - dev_info(dev, "%s: %s (%dx%d)\n", __func__, type->name, type->width, - type->height); + if (!mdev_state->memblk) + goto out_vconfig; mutex_init(&mdev_state->ops_lock); mdev_state->mdev = mdev; - mdev_state->type = type; + mdev_state->type = type; mdev_state->memsize = fbsize; mdpy_create_config_space(mdev_state); mdpy_reset(mdev_state); - mdpy_count++; + dev_info(vdev->dev, "%s: %s (%dx%d)\n", __func__, type->type.pretty_name, + type->width, type->height); + return 0; + +out_vconfig: + kfree(mdev_state->vconfig); + return ret; +} + +static int mdpy_probe(struct mdev_device *mdev) +{ + struct mdev_state *mdev_state; + int ret; + + mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev, + &mdpy_dev_ops); + if (IS_ERR(mdev_state)) + return PTR_ERR(mdev_state); ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev); if (ret) - goto err_mem; + goto err_put_vdev; dev_set_drvdata(&mdev->dev, mdev_state); return 0; -err_mem: + +err_put_vdev: + vfio_put_device(&mdev_state->vdev); + return ret; +} + +static void mdpy_release_dev(struct vfio_device *vdev) +{ + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + vfree(mdev_state->memblk); -err_vconfig: kfree(mdev_state->vconfig); -err_state: - vfio_uninit_group_dev(&mdev_state->vdev); - kfree(mdev_state); - return ret; + vfio_free_device(vdev); } static void mdpy_remove(struct mdev_device *mdev) @@ -280,12 +293,7 @@ static void mdpy_remove(struct mdev_device *mdev) dev_info(&mdev->dev, "%s\n", __func__); vfio_unregister_group_dev(&mdev_state->vdev); - vfree(mdev_state->memblk); - kfree(mdev_state->vconfig); - vfio_uninit_group_dev(&mdev_state->vdev); - kfree(mdev_state); - - mdpy_count--; + vfio_put_device(&mdev_state->vdev); } static ssize_t mdpy_read(struct vfio_device *vdev, char __user *buf, @@ -641,73 +649,17 @@ static const struct attribute_group *mdev_dev_groups[] = { NULL, }; -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - const struct mdpy_type *type = - &mdpy_types[mtype_get_type_group_id(mtype)]; - - return sprintf(buf, "%s\n", type->name); -} -static MDEV_TYPE_ATTR_RO(name); - -static ssize_t description_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) +static ssize_t mdpy_show_description(struct mdev_type *mtype, char *buf) { - const struct mdpy_type *type = - &mdpy_types[mtype_get_type_group_id(mtype)]; + struct mdpy_type *type = container_of(mtype, struct mdpy_type, type); return sprintf(buf, "virtual display, %dx%d framebuffer\n", type->width, type->height); } -static MDEV_TYPE_ATTR_RO(description); - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) -{ - return sprintf(buf, "%d\n", max_devices - mdpy_count); -} -static MDEV_TYPE_ATTR_RO(available_instances); - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} -static MDEV_TYPE_ATTR_RO(device_api); - -static struct attribute *mdev_types_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_description.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_available_instances.attr, - NULL, -}; - -static struct attribute_group mdev_type_group1 = { - .name = MDPY_TYPE_1, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group mdev_type_group2 = { - .name = MDPY_TYPE_2, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group mdev_type_group3 = { - .name = MDPY_TYPE_3, - .attrs = mdev_types_attrs, -}; - -static struct attribute_group *mdev_type_groups[] = { - &mdev_type_group1, - &mdev_type_group2, - &mdev_type_group3, - NULL, -}; static const struct vfio_device_ops mdpy_dev_ops = { + .init = mdpy_init_dev, + .release = mdpy_release_dev, .read = mdpy_read, .write = mdpy_write, .ioctl = mdpy_ioctl, @@ -715,6 +667,8 @@ static const struct vfio_device_ops mdpy_dev_ops = { }; static struct mdev_driver mdpy_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, + .max_instances = 4, .driver = { .name = "mdpy", .owner = THIS_MODULE, @@ -723,7 +677,7 @@ static struct mdev_driver mdpy_driver = { }, .probe = mdpy_probe, .remove = mdpy_remove, - .supported_type_groups = mdev_type_groups, + .show_description = mdpy_show_description, }; static const struct file_operations vd_fops = { @@ -766,7 +720,9 @@ static int __init mdpy_dev_init(void) if (ret) goto err_class; - ret = mdev_register_device(&mdpy_dev, &mdpy_driver); + ret = mdev_register_parent(&mdpy_parent, &mdpy_dev, &mdpy_driver, + mdpy_mdev_types, + ARRAY_SIZE(mdpy_mdev_types)); if (ret) goto err_device; @@ -787,7 +743,7 @@ err_cdev: static void __exit mdpy_dev_exit(void) { mdpy_dev.bus = NULL; - mdev_unregister_device(&mdpy_dev); + mdev_unregister_parent(&mdpy_parent); device_unregister(&mdpy_dev); mdev_unregister_driver(&mdpy_driver); @@ -797,5 +753,8 @@ static void __exit mdpy_dev_exit(void) mdpy_class = NULL; } +module_param_named(count, mdpy_driver.max_instances, int, 0444); +MODULE_PARM_DESC(count, "number of " MDPY_NAME " devices"); + module_init(mdpy_dev_init) module_exit(mdpy_dev_exit) diff --git a/samples/vfio-mdev/mtty.c b/samples/vfio-mdev/mtty.c index f42a59ed2e3fec1609d1383a944e26d579bcfeb4..e72085fc1376318a7fc0dc4f6db724dd4dd80092 100644 --- a/samples/vfio-mdev/mtty.c +++ b/samples/vfio-mdev/mtty.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -20,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -74,6 +72,7 @@ static struct mtty_dev { struct cdev vd_cdev; struct idr vd_idr; struct device dev; + struct mdev_parent parent; } mtty_dev; struct mdev_region_info { @@ -144,6 +143,21 @@ struct mdev_state { int nr_ports; }; +static struct mtty_type { + struct mdev_type type; + int nr_ports; +} mtty_types[2] = { + { .nr_ports = 1, .type.sysfs_name = "1", + .type.pretty_name = "Single port serial" }, + { .nr_ports = 2, .type.sysfs_name = "2", + .type.pretty_name = "Dual port serial" }, +}; + +static struct mdev_type *mtty_mdev_types[] = { + &mtty_types[0].type, + &mtty_types[1].type, +}; + static atomic_t mdev_avail_ports = ATOMIC_INIT(MAX_MTTYS); static const struct file_operations vd_fops = { @@ -703,71 +717,82 @@ accessfailed: return ret; } -static int mtty_probe(struct mdev_device *mdev) +static int mtty_init_dev(struct vfio_device *vdev) { - struct mdev_state *mdev_state; - int nr_ports = mdev_get_type_group_id(mdev) + 1; + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + struct mdev_device *mdev = to_mdev_device(vdev->dev); + struct mtty_type *type = + container_of(mdev->type, struct mtty_type, type); int avail_ports = atomic_read(&mdev_avail_ports); int ret; do { - if (avail_ports < nr_ports) + if (avail_ports < type->nr_ports) return -ENOSPC; } while (!atomic_try_cmpxchg(&mdev_avail_ports, - &avail_ports, avail_ports - nr_ports)); + &avail_ports, + avail_ports - type->nr_ports)); - mdev_state = kzalloc(sizeof(struct mdev_state), GFP_KERNEL); - if (mdev_state == NULL) { - ret = -ENOMEM; - goto err_nr_ports; - } - - vfio_init_group_dev(&mdev_state->vdev, &mdev->dev, &mtty_dev_ops); - - mdev_state->nr_ports = nr_ports; + mdev_state->nr_ports = type->nr_ports; mdev_state->irq_index = -1; mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE; mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE; mutex_init(&mdev_state->rxtx_lock); - mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL); - if (mdev_state->vconfig == NULL) { + mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL); + if (!mdev_state->vconfig) { ret = -ENOMEM; - goto err_state; + goto err_nr_ports; } mutex_init(&mdev_state->ops_lock); mdev_state->mdev = mdev; - mtty_create_config_space(mdev_state); + return 0; + +err_nr_ports: + atomic_add(type->nr_ports, &mdev_avail_ports); + return ret; +} + +static int mtty_probe(struct mdev_device *mdev) +{ + struct mdev_state *mdev_state; + int ret; + + mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev, + &mtty_dev_ops); + if (IS_ERR(mdev_state)) + return PTR_ERR(mdev_state); ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev); if (ret) - goto err_vconfig; + goto err_put_vdev; dev_set_drvdata(&mdev->dev, mdev_state); return 0; -err_vconfig: - kfree(mdev_state->vconfig); -err_state: - vfio_uninit_group_dev(&mdev_state->vdev); - kfree(mdev_state); -err_nr_ports: - atomic_add(nr_ports, &mdev_avail_ports); +err_put_vdev: + vfio_put_device(&mdev_state->vdev); return ret; } +static void mtty_release_dev(struct vfio_device *vdev) +{ + struct mdev_state *mdev_state = + container_of(vdev, struct mdev_state, vdev); + + atomic_add(mdev_state->nr_ports, &mdev_avail_ports); + kfree(mdev_state->vconfig); + vfio_free_device(vdev); +} + static void mtty_remove(struct mdev_device *mdev) { struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev); - int nr_ports = mdev_state->nr_ports; vfio_unregister_group_dev(&mdev_state->vdev); - - kfree(mdev_state->vconfig); - vfio_uninit_group_dev(&mdev_state->vdev); - kfree(mdev_state); - atomic_add(nr_ports, &mdev_avail_ports); + vfio_put_device(&mdev_state->vdev); } static int mtty_reset(struct mdev_state *mdev_state) @@ -1231,68 +1256,24 @@ static const struct attribute_group *mdev_dev_groups[] = { NULL, }; -static ssize_t name_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - static const char *name_str[2] = { "Single port serial", - "Dual port serial" }; - - return sysfs_emit(buf, "%s\n", - name_str[mtype_get_type_group_id(mtype)]); -} - -static MDEV_TYPE_ATTR_RO(name); - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) +static unsigned int mtty_get_available(struct mdev_type *mtype) { - unsigned int ports = mtype_get_type_group_id(mtype) + 1; + struct mtty_type *type = container_of(mtype, struct mtty_type, type); - return sprintf(buf, "%d\n", atomic_read(&mdev_avail_ports) / ports); + return atomic_read(&mdev_avail_ports) / type->nr_ports; } -static MDEV_TYPE_ATTR_RO(available_instances); - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} - -static MDEV_TYPE_ATTR_RO(device_api); - -static struct attribute *mdev_types_attrs[] = { - &mdev_type_attr_name.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_available_instances.attr, - NULL, -}; - -static struct attribute_group mdev_type_group1 = { - .name = "1", - .attrs = mdev_types_attrs, -}; - -static struct attribute_group mdev_type_group2 = { - .name = "2", - .attrs = mdev_types_attrs, -}; - -static struct attribute_group *mdev_type_groups[] = { - &mdev_type_group1, - &mdev_type_group2, - NULL, -}; - static const struct vfio_device_ops mtty_dev_ops = { .name = "vfio-mtty", + .init = mtty_init_dev, + .release = mtty_release_dev, .read = mtty_read, .write = mtty_write, .ioctl = mtty_ioctl, }; static struct mdev_driver mtty_driver = { + .device_api = VFIO_DEVICE_API_PCI_STRING, .driver = { .name = "mtty", .owner = THIS_MODULE, @@ -1301,7 +1282,7 @@ static struct mdev_driver mtty_driver = { }, .probe = mtty_probe, .remove = mtty_remove, - .supported_type_groups = mdev_type_groups, + .get_available = mtty_get_available, }; static void mtty_device_release(struct device *dev) @@ -1352,7 +1333,9 @@ static int __init mtty_dev_init(void) if (ret) goto err_class; - ret = mdev_register_device(&mtty_dev.dev, &mtty_driver); + ret = mdev_register_parent(&mtty_dev.parent, &mtty_dev.dev, + &mtty_driver, mtty_mdev_types, + ARRAY_SIZE(mtty_mdev_types)); if (ret) goto err_device; return 0; @@ -1372,7 +1355,7 @@ err_cdev: static void __exit mtty_dev_exit(void) { mtty_dev.dev.bus = NULL; - mdev_unregister_device(&mtty_dev.dev); + mdev_unregister_parent(&mtty_dev.parent); device_unregister(&mtty_dev.dev); idr_destroy(&mtty_dev.vd_idr); diff --git a/scripts/.gitignore b/scripts/.gitignore index eed308bef604a13acba7c50d44bea2ce9f213bdd..b7aec8eb1bd443dbd4dd189868f321e329fbc4be 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only /asn1_compiler /bin2c +/generate_rust_target /insert-sys-cert /kallsyms /module.lds diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index ece44b735061348a385070ba782f797abf5d4b80..2bc08ace38a3b56a0bf95d3fd2b739c406d9dd3a 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -100,8 +100,29 @@ echo-cmd = $(if $($(quiet)cmd_$(1)),\ quiet_redirect := silent_redirect := exec >/dev/null; +# Delete the target on interruption +# +# GNU Make automatically deletes the target if it has already been changed by +# the interrupted recipe. So, you can safely stop the build by Ctrl-C (Make +# will delete incomplete targets), and resume it later. +# +# However, this does not work when the stderr is piped to another program, like +# $ make >&2 | tee log +# Make dies with SIGPIPE before cleaning the targets. +# +# To address it, we clean the target in signal traps. +# +# Make deletes the target when it catches SIGHUP, SIGINT, SIGQUIT, SIGTERM. +# So, we cover them, and also SIGPIPE just in case. +# +# Of course, this is unneeded for phony targets. +delete-on-interrupt = \ + $(if $(filter-out $(PHONY), $@), \ + $(foreach sig, HUP INT QUIT TERM PIPE, \ + trap 'rm -f $@; trap - $(sig); kill -s $(sig) $$$$' $(sig);)) + # printing commands -cmd = @set -e; $(echo-cmd) $($(quiet)redirect) $(cmd_$(1)) +cmd = @set -e; $(echo-cmd) $($(quiet)redirect) $(delete-on-interrupt) $(cmd_$(1)) ### # if_changed - execute command if any prerequisite is newer than diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include index a0ccceb22cf8b5ccfae079f6aea5499a7cdfae0a..274125307ebd75f32f9a65c8ea02106769b30257 100644 --- a/scripts/Kconfig.include +++ b/scripts/Kconfig.include @@ -36,12 +36,12 @@ ld-option = $(success,$(LD) -v $(1)) as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler -o /dev/null -) # check if $(CC) and $(LD) exist -$(error-if,$(failure,command -v $(CC)),compiler '$(CC)' not found) +$(error-if,$(failure,command -v $(CC)),C compiler '$(CC)' not found) $(error-if,$(failure,command -v $(LD)),linker '$(LD)' not found) -# Get the compiler name, version, and error out if it is not supported. +# Get the C compiler name, version, and error out if it is not supported. cc-info := $(shell,$(srctree)/scripts/cc-version.sh $(CC)) -$(error-if,$(success,test -z "$(cc-info)"),Sorry$(comma) this compiler is not supported.) +$(error-if,$(success,test -z "$(cc-info)"),Sorry$(comma) this C compiler is not supported.) cc-name := $(shell,set -- $(cc-info) && echo $1) cc-version := $(shell,set -- $(cc-info) && echo $2) diff --git a/scripts/Makefile b/scripts/Makefile index f084f08ed176e2a2b4f70ed533f95a69465f7dfd..1575af84d557bdbf56e8c7b7772c54d2e2a3ab3e 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -10,6 +10,9 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert +hostprogs-always-$(CONFIG_RUST) += generate_rust_target + +generate_rust_target-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include HOSTLDLIBS_sorttable = -lpthread diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 784f46d41959b635bb1581f8bbfeee8fa47d3024..41f3602fc8de75b49a733b8e0d8359cf114edcc6 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -5,8 +5,8 @@ src := $(obj) -PHONY := __build -__build: +PHONY := $(obj)/ +$(obj)/: # Init all relevant variables used in kbuild files so # 1) they have correct type @@ -26,6 +26,7 @@ EXTRA_CPPFLAGS := EXTRA_LDFLAGS := asflags-y := ccflags-y := +rustflags-y := cppflags-y := ldflags-y := @@ -139,7 +140,7 @@ $(obj)/%.symtypes : $(src)/%.c FORCE # LLVM assembly # Generate .ll files from .c quiet_cmd_cc_ll_c = CC $(quiet_modtag) $@ - cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -o $@ $< + cmd_cc_ll_c = $(CC) $(c_flags) -emit-llvm -S -fno-discard-value-names -o $@ $< $(obj)/%.ll: $(src)/%.c FORCE $(call if_changed_dep,cc_ll_c) @@ -271,6 +272,65 @@ quiet_cmd_cc_lst_c = MKLST $@ $(obj)/%.lst: $(src)/%.c FORCE $(call if_changed_dep,cc_lst_c) +# Compile Rust sources (.rs) +# --------------------------------------------------------------------------- + +rust_allowed_features := core_ffi_c + +rust_common_cmd = \ + RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ + -Zallow-features=$(rust_allowed_features) \ + -Zcrate-attr=no_std \ + -Zcrate-attr='feature($(rust_allowed_features))' \ + --extern alloc --extern kernel \ + --crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \ + --crate-name $(basename $(notdir $@)) + +rust_handle_depfile = \ + mv $(obj)/$(basename $(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) + +# `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit +# will be used. We explicitly request `-Ccodegen-units=1` in any case, and +# the compiler shows a warning if it is not 1. However, if we ever stop +# requesting it explicitly and we start using some other `--emit` that does not +# imply it (and for which codegen is performed), then we would be out of sync, +# i.e. the outputs we would get for the different single targets (e.g. `.ll`) +# would not match each other. + +quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ + cmd_rustc_o_rs = \ + $(rust_common_cmd) --emit=dep-info,obj $<; \ + $(rust_handle_depfile) + +$(obj)/%.o: $(src)/%.rs FORCE + $(call if_changed_dep,rustc_o_rs) + +quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ + cmd_rustc_rsi_rs = \ + $(rust_common_cmd) --emit=dep-info -Zunpretty=expanded $< >$@; \ + command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \ + $(rust_handle_depfile) + +$(obj)/%.rsi: $(src)/%.rs FORCE + $(call if_changed_dep,rustc_rsi_rs) + +quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ + cmd_rustc_s_rs = \ + $(rust_common_cmd) --emit=dep-info,asm $<; \ + $(rust_handle_depfile) + +$(obj)/%.s: $(src)/%.rs FORCE + $(call if_changed_dep,rustc_s_rs) + +quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ + cmd_rustc_ll_rs = \ + $(rust_common_cmd) --emit=dep-info,llvm-ir $<; \ + $(rust_handle_depfile) + +$(obj)/%.ll: $(src)/%.rs FORCE + $(call if_changed_dep,rustc_ll_rs) + # Compile assembler sources (.S) # --------------------------------------------------------------------------- @@ -323,7 +383,7 @@ $(obj)/%.o: $(src)/%.S FORCE targets += $(filter-out $(subdir-builtin), $(real-obj-y)) targets += $(filter-out $(subdir-modorder), $(real-obj-m)) -targets += $(real-dtb-y) $(lib-y) $(always-y) $(MAKECMDGOALS) +targets += $(real-dtb-y) $(lib-y) $(always-y) # Linker scripts preprocessor (.lds.S -> .lds) # --------------------------------------------------------------------------- @@ -374,7 +434,7 @@ $(obj)/built-in.a: $(real-obj-y) FORCE cmd_modules_order = { $(foreach m, $(real-prereqs), \ $(if $(filter %/modules.order, $m), cat $m, echo $(patsubst %.o,%.ko,$m));) :; } \ - | $(AWK) '!x[$$0]++' - > $@ + > $@ $(obj)/modules.order: $(obj-m) FORCE $(call if_changed,modules_order) @@ -400,8 +460,6 @@ $(multi-obj-m): %.o: %.mod FORCE $(call if_changed_rule,ld_multi_m) $(call multi_depend, $(multi-obj-m), .o, -objs -y -m) -targets := $(filter-out $(PHONY), $(targets)) - # Add intermediate targets: # When building objects with specific suffix patterns, add intermediate # targets that the final targets are derived from. @@ -420,52 +478,29 @@ targets += $(call intermediate_targets, .asn1.o, .asn1.c .asn1.h) \ # Build # --------------------------------------------------------------------------- -ifdef single-build - -KBUILD_SINGLE_TARGETS := $(filter $(obj)/%, $(KBUILD_SINGLE_TARGETS)) - -curdir-single := $(sort $(foreach x, $(KBUILD_SINGLE_TARGETS), \ - $(if $(filter $(x) $(basename $(x)).o, $(targets)), $(x)))) - -# Handle single targets without any rule: show "Nothing to be done for ..." or -# "No rule to make target ..." depending on whether the target exists. -unknown-single := $(filter-out $(addsuffix /%, $(subdir-ym)), \ - $(filter-out $(curdir-single), $(KBUILD_SINGLE_TARGETS))) - -single-subdirs := $(foreach d, $(subdir-ym), \ - $(if $(filter $(d)/%, $(KBUILD_SINGLE_TARGETS)), $(d))) - -__build: $(curdir-single) $(single-subdirs) -ifneq ($(unknown-single),) - $(Q)$(MAKE) -f /dev/null $(unknown-single) -endif +$(obj)/: $(if $(KBUILD_BUILTIN), $(targets-for-builtin)) \ + $(if $(KBUILD_MODULES), $(targets-for-modules)) \ + $(subdir-ym) $(always-y) @: -ifeq ($(curdir-single),) -# Nothing to do in this directory. Do not include any .*.cmd file for speed-up -targets := -else -targets += $(curdir-single) -endif +# Single targets +# --------------------------------------------------------------------------- -else +single-subdirs := $(foreach d, $(subdir-ym), $(if $(filter $d/%, $(MAKECMDGOALS)), $d)) +single-subdir-goals := $(filter $(addsuffix /%, $(single-subdirs)), $(MAKECMDGOALS)) -__build: $(if $(KBUILD_BUILTIN), $(targets-for-builtin)) \ - $(if $(KBUILD_MODULES), $(targets-for-modules)) \ - $(subdir-ym) $(always-y) +$(single-subdir-goals): $(single-subdirs) @: -endif - # Descending # --------------------------------------------------------------------------- PHONY += $(subdir-ym) $(subdir-ym): $(Q)$(MAKE) $(build)=$@ \ - $(if $(filter $@/, $(KBUILD_SINGLE_TARGETS)),single-build=) \ need-builtin=$(if $(filter $@/built-in.a, $(subdir-builtin)),1) \ - need-modorder=$(if $(filter $@/modules.order, $(subdir-modorder)),1) + need-modorder=$(if $(filter $@/modules.order, $(subdir-modorder)),1) \ + $(filter $@/%, $(single-subdir-goals)) # Add FORCE to the prequisites of a target to force it to be always rebuilt. # --------------------------------------------------------------------------- @@ -474,6 +509,9 @@ PHONY += FORCE FORCE: +targets += $(filter-out $(single-subdir-goals), $(MAKECMDGOALS)) +targets := $(filter-out $(PHONY), $(targets)) + # Read all saved command lines and dependencies for the $(targets) we # may be building above, using $(if_changed{,_dep}). As an # optimization, we don't need to read them if the target does not diff --git a/scripts/Makefile.compiler b/scripts/Makefile.compiler index 94d0d40cddb3d614facd505e46d4d83cbc7f7e10..20d353dcabfbc50b0419936724c57a4a6f2f22e3 100644 --- a/scripts/Makefile.compiler +++ b/scripts/Makefile.compiler @@ -61,9 +61,13 @@ cc-option-yn = $(call try-run,\ cc-disable-warning = $(call try-run,\ $(CC) -Werror $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -W$(strip $(1)) -c -x c /dev/null -o "$$TMP",-Wno-$(strip $(1))) -# cc-ifversion -# Usage: EXTRA_CFLAGS += $(call cc-ifversion, -lt, 0402, -O1) -cc-ifversion = $(shell [ $(CONFIG_GCC_VERSION)0 $(1) $(2)000 ] && echo $(3) || echo $(4)) +# gcc-min-version +# Usage: cflags-$(call gcc-min-version, 70100) += -foo +gcc-min-version = $(shell [ $(CONFIG_GCC_VERSION)0 -ge $(1)0 ] && echo y) + +# clang-min-version +# Usage: cflags-$(call clang-min-version, 110000) += -foo +clang-min-version = $(shell [ $(CONFIG_CLANG_VERSION)0 -ge $(1)0 ] && echo y) # ld-option # Usage: KBUILD_LDFLAGS += $(call ld-option, -X, -Y) diff --git a/scripts/Makefile.debug b/scripts/Makefile.debug index 9f39b0130551f3d35182df243a3fc9de38a1e264..332c486f705f75edb63fd06f412c3563733e3814 100644 --- a/scripts/Makefile.debug +++ b/scripts/Makefile.debug @@ -1,26 +1,30 @@ DEBUG_CFLAGS := +DEBUG_RUSTFLAGS := + +debug-flags-y := -g ifdef CONFIG_DEBUG_INFO_SPLIT DEBUG_CFLAGS += -gsplit-dwarf -else -DEBUG_CFLAGS += -g -endif - -ifndef CONFIG_AS_IS_LLVM -KBUILD_AFLAGS += -Wa,-gdwarf-2 endif -ifndef CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT -dwarf-version-$(CONFIG_DEBUG_INFO_DWARF4) := 4 -dwarf-version-$(CONFIG_DEBUG_INFO_DWARF5) := 5 -DEBUG_CFLAGS += -gdwarf-$(dwarf-version-y) +debug-flags-$(CONFIG_DEBUG_INFO_DWARF4) += -gdwarf-4 +debug-flags-$(CONFIG_DEBUG_INFO_DWARF5) += -gdwarf-5 +ifeq ($(CONFIG_CC_IS_CLANG)$(CONFIG_AS_IS_GNU),yy) +# Clang does not pass -g or -gdwarf-* option down to GAS. +# Add -Wa, prefix to explicitly specify the flags. +KBUILD_AFLAGS += $(addprefix -Wa$(comma), $(debug-flags-y)) endif +DEBUG_CFLAGS += $(debug-flags-y) +KBUILD_AFLAGS += $(debug-flags-y) ifdef CONFIG_DEBUG_INFO_REDUCED DEBUG_CFLAGS += -fno-var-tracking +DEBUG_RUSTFLAGS += -Cdebuginfo=1 ifdef CONFIG_CC_IS_GCC DEBUG_CFLAGS += -femit-struct-debug-baseonly endif +else +DEBUG_RUSTFLAGS += -Cdebuginfo=2 endif ifdef CONFIG_DEBUG_INFO_COMPRESSED @@ -29,5 +33,8 @@ KBUILD_AFLAGS += -gz=zlib KBUILD_LDFLAGS += --compress-debug-sections=zlib endif -KBUILD_CFLAGS += $(DEBUG_CFLAGS) +KBUILD_CFLAGS += $(DEBUG_CFLAGS) export DEBUG_CFLAGS + +KBUILD_RUSTFLAGS += $(DEBUG_RUSTFLAGS) +export DEBUG_RUSTFLAGS diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn index 0621c39a3955674dc9dda8e54e605203800bcaaa..6bbba36c596957cab633b51795294af773d8af61 100644 --- a/scripts/Makefile.extrawarn +++ b/scripts/Makefile.extrawarn @@ -47,11 +47,24 @@ else ifdef CONFIG_CC_IS_CLANG KBUILD_CFLAGS += -Wno-initializer-overrides +# Clang before clang-16 would warn on default argument promotions. +ifneq ($(call clang-min-version, 160000),y) +# Disable -Wformat KBUILD_CFLAGS += -Wno-format +# Then re-enable flags that were part of the -Wformat group that aren't +# problematic. +KBUILD_CFLAGS += -Wformat-extra-args -Wformat-invalid-specifier +KBUILD_CFLAGS += -Wformat-zero-length -Wnonnull +# Requires clang-12+. +ifeq ($(call clang-min-version, 120000),y) +KBUILD_CFLAGS += -Wformat-insufficient-args +endif +endif KBUILD_CFLAGS += -Wno-sign-compare KBUILD_CFLAGS += $(call cc-disable-warning, pointer-to-enum-cast) KBUILD_CFLAGS += -Wno-tautological-constant-out-of-range-compare KBUILD_CFLAGS += $(call cc-disable-warning, unaligned-access) +KBUILD_CFLAGS += $(call cc-disable-warning, cast-function-type-strict) endif endif diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 278b4d6ac9454d9cc5f3fc43a77881dd3d9bcbfe..da133780b7518dbcc028dee3c940e857e15a3284 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -22,6 +22,8 @@ $(obj)/%.tab.c $(obj)/%.tab.h: $(src)/%.y FORCE # to preprocess a data file. # # Both C and C++ are supported, but preferred language is C for such utilities. +# Rust is also supported, but it may only be used in scenarios where a Rust +# toolchain is required to be available (e.g. when `CONFIG_RUST` is enabled). # # Sample syntax (see Documentation/kbuild/makefiles.rst for reference) # hostprogs := bin2hex @@ -37,15 +39,20 @@ $(obj)/%.tab.c $(obj)/%.tab.h: $(src)/%.y FORCE # qconf-objs := menu.o # Will compile qconf as a C++ program, and menu as a C program. # They are linked as C++ code to the executable qconf +# +# hostprogs := target +# target-rust := y +# Will compile `target` as a Rust program, using `target.rs` as the crate root. +# The crate may consist of several source files. # C code # Executables compiled from a single .c file host-csingle := $(foreach m,$(hostprogs), \ - $(if $($(m)-objs)$($(m)-cxxobjs),,$(m))) + $(if $($(m)-objs)$($(m)-cxxobjs)$($(m)-rust),,$(m))) # C executables linked based on several .o files host-cmulti := $(foreach m,$(hostprogs),\ - $(if $($(m)-cxxobjs),,$(if $($(m)-objs),$(m)))) + $(if $($(m)-cxxobjs)$($(m)-rust),,$(if $($(m)-objs),$(m)))) # Object (.o) files compiled from .c files host-cobjs := $(sort $(foreach m,$(hostprogs),$($(m)-objs))) @@ -58,11 +65,17 @@ host-cxxmulti := $(foreach m,$(hostprogs),$(if $($(m)-cxxobjs),$(m))) # C++ Object (.o) files compiled from .cc files host-cxxobjs := $(sort $(foreach m,$(host-cxxmulti),$($(m)-cxxobjs))) +# Rust code +# Executables compiled from a single Rust crate (which may consist of +# one or more .rs files) +host-rust := $(foreach m,$(hostprogs),$(if $($(m)-rust),$(m))) + host-csingle := $(addprefix $(obj)/,$(host-csingle)) host-cmulti := $(addprefix $(obj)/,$(host-cmulti)) host-cobjs := $(addprefix $(obj)/,$(host-cobjs)) host-cxxmulti := $(addprefix $(obj)/,$(host-cxxmulti)) host-cxxobjs := $(addprefix $(obj)/,$(host-cxxobjs)) +host-rust := $(addprefix $(obj)/,$(host-rust)) ##### # Handle options to gcc. Support building with separate output directory @@ -71,6 +84,8 @@ _hostc_flags = $(KBUILD_HOSTCFLAGS) $(HOST_EXTRACFLAGS) \ $(HOSTCFLAGS_$(target-stem).o) _hostcxx_flags = $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) +_hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ + $(HOSTRUSTFLAGS_$(target-stem)) # $(objtree)/$(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) @@ -82,6 +97,7 @@ endif hostc_flags = -Wp,-MMD,$(depfile) $(_hostc_flags) hostcxx_flags = -Wp,-MMD,$(depfile) $(_hostcxx_flags) +hostrust_flags = $(_hostrust_flags) ##### # Compile programs on the host @@ -128,5 +144,17 @@ quiet_cmd_host-cxxobjs = HOSTCXX $@ $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE $(call if_changed_dep,host-cxxobjs) +# Create executable from a single Rust crate (which may consist of +# one or more `.rs` files) +# host-rust -> Executable +quiet_cmd_host-rust = HOSTRUSTC $@ + cmd_host-rust = \ + $(HOSTRUSTC) $(hostrust_flags) --emit=dep-info,link \ + --out-dir=$(obj)/ $<; \ + mv $(obj)/$(target-stem).d $(depfile); \ + sed -i '/^\#/d' $(depfile) +$(host-rust): $(obj)/%: $(src)/%.rs FORCE + $(call if_changed_dep,host-rust) + targets += $(host-csingle) $(host-cmulti) $(host-cobjs) \ - $(host-cxxmulti) $(host-cxxobjs) + $(host-cxxmulti) $(host-cxxobjs) $(host-rust) diff --git a/scripts/Makefile.kmsan b/scripts/Makefile.kmsan new file mode 100644 index 0000000000000000000000000000000000000000..b5b0aa61322ecaedaca55e3d6d0860775ec633fc --- /dev/null +++ b/scripts/Makefile.kmsan @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +kmsan-cflags := -fsanitize=kernel-memory + +ifdef CONFIG_KMSAN_CHECK_PARAM_RETVAL +kmsan-cflags += -fsanitize-memory-param-retval +endif + +export CFLAGS_KMSAN := $(kmsan-cflags) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 3fb6a99e78c473128c7d4785fbdf073a05435b9a..3aa384cec76b8b5b9b0bb72791be2d5cc7f6ed1e 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -8,6 +8,7 @@ ldflags-y += $(EXTRA_LDFLAGS) # flags that take effect in current and sub directories KBUILD_AFLAGS += $(subdir-asflags-y) KBUILD_CFLAGS += $(subdir-ccflags-y) +KBUILD_RUSTFLAGS += $(subdir-rustflags-y) # Figure out what we need to build from the various variables # =========================================================================== @@ -89,6 +90,7 @@ always-y += $(dtb-y) # Add subdir path +ifneq ($(obj),.) extra-y := $(addprefix $(obj)/,$(extra-y)) always-y := $(addprefix $(obj)/,$(always-y)) targets := $(addprefix $(obj)/,$(targets)) @@ -100,6 +102,7 @@ multi-obj-m := $(addprefix $(obj)/, $(multi-obj-m)) multi-dtb-y := $(addprefix $(obj)/, $(multi-dtb-y)) real-dtb-y := $(addprefix $(obj)/, $(real-dtb-y)) subdir-ym := $(addprefix $(obj)/,$(subdir-ym)) +endif # Finds the multi-part object the current object will be linked into. # If the object belongs to two or more multi-part objects, list them all. @@ -128,6 +131,10 @@ _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \ $(filter-out $(ccflags-remove-y), \ $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(ccflags-y)) \ $(CFLAGS_$(target-stem).o)) +_rust_flags = $(filter-out $(RUSTFLAGS_REMOVE_$(target-stem).o), \ + $(filter-out $(rustflags-remove-y), \ + $(KBUILD_RUSTFLAGS) $(rustflags-y)) \ + $(RUSTFLAGS_$(target-stem).o)) _a_flags = $(filter-out $(AFLAGS_REMOVE_$(target-stem).o), \ $(filter-out $(asflags-remove-y), \ $(KBUILD_CPPFLAGS) $(KBUILD_AFLAGS) $(asflags-y)) \ @@ -157,6 +164,15 @@ _c_flags += $(if $(patsubst n%,, \ endif endif +ifeq ($(CONFIG_KMSAN),y) +_c_flags += $(if $(patsubst n%,, \ + $(KMSAN_SANITIZE_$(basetarget).o)$(KMSAN_SANITIZE)y), \ + $(CFLAGS_KMSAN)) +_c_flags += $(if $(patsubst n%,, \ + $(KMSAN_ENABLE_CHECKS_$(basetarget).o)$(KMSAN_ENABLE_CHECKS)y), \ + , -mllvm -msan-disable-checks=1) +endif + ifeq ($(CONFIG_UBSAN),y) _c_flags += $(if $(patsubst n%,, \ $(UBSAN_SANITIZE_$(basetarget).o)$(UBSAN_SANITIZE)$(CONFIG_UBSAN_SANITIZE_ALL)), \ @@ -202,6 +218,11 @@ modkern_cflags = \ $(KBUILD_CFLAGS_MODULE) $(CFLAGS_MODULE), \ $(KBUILD_CFLAGS_KERNEL) $(CFLAGS_KERNEL) $(modfile_flags)) +modkern_rustflags = \ + $(if $(part-of-module), \ + $(KBUILD_RUSTFLAGS_MODULE) $(RUSTFLAGS_MODULE), \ + $(KBUILD_RUSTFLAGS_KERNEL) $(RUSTFLAGS_KERNEL)) + modkern_aflags = $(if $(part-of-module), \ $(KBUILD_AFLAGS_MODULE) $(AFLAGS_MODULE), \ $(KBUILD_AFLAGS_KERNEL) $(AFLAGS_KERNEL)) @@ -211,6 +232,8 @@ c_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(_c_flags) $(modkern_cflags) \ $(basename_flags) $(modname_flags) +rust_flags = $(_rust_flags) $(modkern_rustflags) @$(objtree)/include/generated/rustc_cfg + a_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(_a_flags) $(modkern_aflags) @@ -229,25 +252,26 @@ ifdef CONFIG_OBJTOOL objtool := $(objtree)/tools/objtool/objtool -objtool_args = \ - $(if $(CONFIG_HAVE_JUMP_LABEL_HACK), --hacks=jump_label) \ - $(if $(CONFIG_HAVE_NOINSTR_HACK), --hacks=noinstr) \ - $(if $(CONFIG_X86_KERNEL_IBT), --ibt) \ - $(if $(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL), --mcount) \ - $(if $(CONFIG_UNWINDER_ORC), --orc) \ - $(if $(CONFIG_RETPOLINE), --retpoline) \ - $(if $(CONFIG_RETHUNK), --rethunk) \ - $(if $(CONFIG_SLS), --sls) \ - $(if $(CONFIG_STACK_VALIDATION), --stackval) \ - $(if $(CONFIG_HAVE_STATIC_CALL_INLINE), --static-call) \ - $(if $(CONFIG_HAVE_UACCESS_VALIDATION), --uaccess) \ +objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK) += --hacks=jump_label +objtool-args-$(CONFIG_HAVE_NOINSTR_HACK) += --hacks=noinstr +objtool-args-$(CONFIG_X86_KERNEL_IBT) += --ibt +objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL) += --mcount +objtool-args-$(CONFIG_UNWINDER_ORC) += --orc +objtool-args-$(CONFIG_RETPOLINE) += --retpoline +objtool-args-$(CONFIG_RETHUNK) += --rethunk +objtool-args-$(CONFIG_SLS) += --sls +objtool-args-$(CONFIG_STACK_VALIDATION) += --stackval +objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE) += --static-call +objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess +objtool-args-$(CONFIG_GCOV_KERNEL) += --no-unreachable + +objtool-args = $(objtool-args-y) \ $(if $(delay-objtool), --link) \ - $(if $(part-of-module), --module) \ - $(if $(CONFIG_GCOV_KERNEL), --no-unreachable) + $(if $(part-of-module), --module) delay-objtool := $(or $(CONFIG_LTO_CLANG),$(CONFIG_X86_KERNEL_IBT)) -cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool_args) $@) +cmd_objtool = $(if $(objtool-enabled), ; $(objtool) $(objtool-args) $@) cmd_gen_objtooldep = $(if $(objtool-enabled), { echo ; echo '$@: $$(wildcard $(objtool))' ; } >> $(dot-target).cmd) endif # CONFIG_OBJTOOL @@ -371,17 +395,15 @@ DT_CHECKER_FLAGS ?= $(if $(DT_SCHEMA_FILES),-l $(DT_SCHEMA_FILES),-m) DT_BINDING_DIR := Documentation/devicetree/bindings DT_TMP_SCHEMA := $(objtree)/$(DT_BINDING_DIR)/processed-schema.json -quiet_cmd_dtb_check = CHECK $@ - cmd_dtb_check = $(DT_CHECKER) $(DT_CHECKER_FLAGS) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ || true +quiet_cmd_dtb = DTC_CHK $@ + cmd_dtb = $(cmd_dtc) ; $(DT_CHECKER) $(DT_CHECKER_FLAGS) -u $(srctree)/$(DT_BINDING_DIR) -p $(DT_TMP_SCHEMA) $@ || true +else +quiet_cmd_dtb = $(quiet_cmd_dtc) + cmd_dtb = $(cmd_dtc) endif -define rule_dtc - $(call cmd_and_fixdep,dtc) - $(call cmd,dtb_check) -endef - $(obj)/%.dtb: $(src)/%.dts $(DTC) $(DT_TMP_SCHEMA) FORCE - $(call if_changed_rule,dtc) + $(call if_changed_dep,dtb) $(obj)/%.dtbo: $(src)/%.dts $(DTC) FORCE $(call if_changed_dep,dtc) diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index 35100e981f4a78b8220d7ffd895e11f695d9ad46..25bedd83644b0b7789912baff4b667b0eb2b7988 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -39,11 +39,13 @@ quiet_cmd_ld_ko_o = LD [M] $@ quiet_cmd_btf_ko = BTF [M] $@ cmd_btf_ko = \ - if [ -f vmlinux ]; then \ + if [ ! -f vmlinux ]; then \ + printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \ + elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then \ + printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \ + else \ LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \ $(RESOLVE_BTFIDS) -b vmlinux $@; \ - else \ - printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \ fi; # Same as newer-prereqs, but allows to exclude specified extra dependencies @@ -55,7 +57,7 @@ if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \ printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:) # Re-generate module BTFs if either module's .ko or vmlinux changed -$(modules): %.ko: %.o %.mod.o scripts/module.lds $(if $(KBUILD_BUILTIN),vmlinux) FORCE +$(modules): %.ko: %.o %.mod.o scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),vmlinux) FORCE +$(call if_changed_except,ld_ko_o,vmlinux) ifdef CONFIG_DEBUG_INFO_BTF_MODULES +$(if $(newer-prereqs),$(call cmd,btf_ko)) diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index 911606496341ca66a34365dcd29f5cc0d09ff0dd..8489a3402eb8cb6fd56b5e2faa926cc0d9fa5b51 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -32,49 +32,58 @@ # Step 4 is solely used to allow module versioning in external modules, # where the CRC of each module is retrieved from the Module.symvers file. -# KBUILD_MODPOST_NOFINAL can be set to skip the final link of modules. -# This is solely useful to speed up test compiles - PHONY := __modpost __modpost: include include/config/auto.conf include $(srctree)/scripts/Kbuild.include -MODPOST = scripts/mod/modpost \ +modpost-args = \ $(if $(CONFIG_MODVERSIONS),-m) \ $(if $(CONFIG_MODULE_SRCVERSION_ALL),-a) \ $(if $(CONFIG_SECTION_MISMATCH_WARN_ONLY),,-E) \ + $(if $(KBUILD_NSDEPS),-d $(MODULES_NSDEPS)) \ + $(if $(CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS)$(KBUILD_NSDEPS),-N) \ -o $@ -ifdef MODPOST_VMLINUX - -quiet_cmd_modpost = MODPOST $@ - cmd_modpost = $(MODPOST) $< - -vmlinux.symvers: vmlinux.o - $(call cmd,modpost) - -__modpost: vmlinux.symvers - -else +# 'make -i -k' ignores compile errors, and builds as many modules as possible. +ifneq ($(findstring i,$(filter-out --%,$(MAKEFLAGS))),) +modpost-args += -n +endif ifeq ($(KBUILD_EXTMOD),) -input-symdump := vmlinux.symvers -output-symdump := modules-only.symvers - -quiet_cmd_cat = GEN $@ - cmd_cat = cat $(real-prereqs) > $@ +# Generate the list of in-tree objects in vmlinux +# --------------------------------------------------------------------------- -ifneq ($(wildcard vmlinux.symvers),) - -__modpost: Module.symvers -Module.symvers: vmlinux.symvers modules-only.symvers FORCE - $(call if_changed,cat) - -targets += Module.symvers +# This is used to retrieve symbol versions generated by genksyms. +ifdef CONFIG_MODVERSIONS +vmlinux.symvers Module.symvers: .vmlinux.objs +endif +# Ignore libgcc.a +# Some architectures do '$(CC) --print-libgcc-file-name' to borrow libgcc.a +# from the toolchain, but there is no EXPORT_SYMBOL in it. + +quiet_cmd_vmlinux_objs = GEN $@ + cmd_vmlinux_objs = \ + for f in $(real-prereqs); do \ + case $${f} in \ + *libgcc.a) ;; \ + *) $(AR) t $${f} ;; \ + esac \ + done > $@ + +targets += .vmlinux.objs +.vmlinux.objs: vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE + $(call if_changed,vmlinux_objs) + +vmlinux.o-if-present := $(wildcard vmlinux.o) +output-symdump := vmlinux.symvers + +ifdef KBUILD_MODULES +output-symdump := $(if $(vmlinux.o-if-present), Module.symvers, modules-only.symvers) +missing-input := $(filter-out $(vmlinux.o-if-present),vmlinux.o) endif else @@ -86,54 +95,37 @@ src := $(obj) # Include the module's Makefile to find KBUILD_EXTRA_SYMBOLS include $(or $(wildcard $(src)/Kbuild), $(src)/Makefile) -# modpost option for external modules -MODPOST += -e - -input-symdump := Module.symvers $(KBUILD_EXTRA_SYMBOLS) +module.symvers-if-present := $(wildcard Module.symvers) output-symdump := $(KBUILD_EXTMOD)/Module.symvers +missing-input := $(filter-out $(module.symvers-if-present), Module.symvers) -endif - -existing-input-symdump := $(wildcard $(input-symdump)) +modpost-args += -e $(addprefix -i ,$(module.symvers-if-present) $(KBUILD_EXTRA_SYMBOLS)) -# modpost options for modules (both in-kernel and external) -MODPOST += \ - $(addprefix -i ,$(existing-input-symdump)) \ - $(if $(KBUILD_NSDEPS),-d $(MODULES_NSDEPS)) \ - $(if $(CONFIG_MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS)$(KBUILD_NSDEPS),-N) +endif # ($(KBUILD_EXTMOD),) -# 'make -i -k' ignores compile errors, and builds as many modules as possible. -ifneq ($(findstring i,$(filter-out --%,$(MAKEFLAGS))),) -MODPOST += -n +ifneq ($(KBUILD_MODPOST_WARN)$(missing-input),) +modpost-args += -w endif -# Clear VPATH to not search for *.symvers in $(srctree). Check only $(objtree). -VPATH := -$(input-symdump): - @echo >&2 'WARNING: Symbol version dump "$@" is missing.' - @echo >&2 ' Modules may not have dependencies or modversions.' - @echo >&2 ' You may get many unresolved symbol warnings.' +modorder-if-needed := $(if $(KBUILD_MODULES), $(MODORDER)) -# KBUILD_MODPOST_WARN can be set to avoid error out in case of undefined symbols -ifneq ($(KBUILD_MODPOST_WARN)$(filter-out $(existing-input-symdump), $(input-symdump)),) -MODPOST += -w -endif +MODPOST = scripts/mod/modpost # Read out modules.order to pass in modpost. # Otherwise, allmodconfig would fail with "Argument list too long". quiet_cmd_modpost = MODPOST $@ - cmd_modpost = sed 's/ko$$/o/' $< | $(MODPOST) -T - - -$(output-symdump): $(MODORDER) $(input-symdump) FORCE - $(call if_changed,modpost) + cmd_modpost = \ + $(if $(missing-input), \ + echo >&2 "WARNING: $(missing-input) is missing."; \ + echo >&2 " Modules may not have dependencies or modversions."; \ + echo >&2 " You may get many unresolved symbol warnings.";) \ + sed 's/ko$$/o/' $(or $(modorder-if-needed), /dev/null) | $(MODPOST) $(modpost-args) -T - $(vmlinux.o-if-present) targets += $(output-symdump) +$(output-symdump): $(modorder-if-needed) $(vmlinux.o-if-present) $(moudle.symvers-if-present) $(MODPOST) FORCE + $(call if_changed,modpost) __modpost: $(output-symdump) -ifneq ($(KBUILD_MODPOST_NOFINAL),1) - $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modfinal -endif - PHONY += FORCE FORCE: @@ -141,6 +133,4 @@ existing-targets := $(wildcard $(sort $(targets))) -include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd) -endif - .PHONY: $(PHONY) diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 5017f6b2da809c2af4e199bd86b4e91762c719ff..8bbcced67c221e1e14e34a34005e5b2e993e678d 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -29,7 +29,10 @@ KDEB_SOURCENAME ?= linux-upstream KBUILD_PKG_ROOTCMD ?="fakeroot -u" export KDEB_SOURCENAME # Include only those top-level files that are needed by make, plus the GPL copy -TAR_CONTENT := $(KBUILD_ALLDIRS) .config .scmversion Makefile \ +TAR_CONTENT := Documentation LICENSES arch block certs crypto drivers fs \ + include init io_uring ipc kernel lib mm net samples scripts \ + security sound tools usr virt \ + .config .scmversion Makefile \ Kbuild Kconfig COPYING $(wildcard localversion*) MKSPEC := $(srctree)/scripts/package/mkspec diff --git a/scripts/Makefile.vmlinux b/scripts/Makefile.vmlinux index 7a63abf22399936ce4f7e5a078e8e4b27bba850e..49946cb968440c6fecd63b8076e1a46a87d80aee 100644 --- a/scripts/Makefile.vmlinux +++ b/scripts/Makefile.vmlinux @@ -1,18 +1,37 @@ # SPDX-License-Identifier: GPL-2.0-only +PHONY := __default +__default: vmlinux + include include/config/auto.conf include $(srctree)/scripts/Kbuild.include # for c_flags include $(srctree)/scripts/Makefile.lib +targets := + quiet_cmd_cc_o_c = CC $@ cmd_cc_o_c = $(CC) $(c_flags) -c -o $@ $< %.o: %.c FORCE $(call if_changed_dep,cc_o_c) -targets := $(MAKECMDGOALS) +ifdef CONFIG_MODULES +targets += .vmlinux.export.o +vmlinux: .vmlinux.export.o +endif + +ARCH_POSTLINK := $(wildcard $(srctree)/arch/$(SRCARCH)/Makefile.postlink) + +# Final link of vmlinux with optional arch pass after final link +cmd_link_vmlinux = \ + $< "$(LD)" "$(KBUILD_LDFLAGS)" "$(LDFLAGS_vmlinux)"; \ + $(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true) + +targets += vmlinux +vmlinux: scripts/link-vmlinux.sh vmlinux.o $(KBUILD_LDS) FORCE + +$(call if_changed_dep,link_vmlinux) # Add FORCE to the prequisites of a target to force it to be always rebuilt. # --------------------------------------------------------------------------- diff --git a/scripts/Makefile.vmlinux_o b/scripts/Makefile.vmlinux_o index 84019814f33f09323bcb9b50e8c3b1cae19c1696..0edfdb40364b8c8c523554015e65d68de2877ee7 100644 --- a/scripts/Makefile.vmlinux_o +++ b/scripts/Makefile.vmlinux_o @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only PHONY := __default -__default: vmlinux.o +__default: vmlinux.o modules.builtin.modinfo modules.builtin include include/config/auto.conf include $(srctree)/scripts/Kbuild.include @@ -18,7 +18,7 @@ quiet_cmd_gen_initcalls_lds = GEN $@ $(PERL) $(real-prereqs) > $@ .tmp_initcalls.lds: $(srctree)/scripts/generate_initcall_order.pl \ - $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS) FORCE + vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE $(call if_changed,gen_initcalls_lds) targets := .tmp_initcalls.lds @@ -35,18 +35,11 @@ endif objtool-enabled := $(or $(delay-objtool),$(CONFIG_NOINSTR_VALIDATION)) -# Reuse objtool_args defined in scripts/Makefile.lib if LTO or IBT is enabled. -# -# Add some more flags as needed. -# --no-unreachable and --link might be added twice, but it is fine. -# -# Expand objtool_args to a simple variable to avoid circular reference. +vmlinux-objtool-args-$(delay-objtool) += $(objtool-args-y) +vmlinux-objtool-args-$(CONFIG_GCOV_KERNEL) += --no-unreachable +vmlinux-objtool-args-$(CONFIG_NOINSTR_VALIDATION) += --noinstr $(if $(CONFIG_CPU_UNRET_ENTRY), --unret) -objtool_args := \ - $(if $(delay-objtool),$(objtool_args)) \ - $(if $(CONFIG_NOINSTR_VALIDATION), --noinstr $(if $(CONFIG_CPU_UNRET_ENTRY), --unret)) \ - $(if $(CONFIG_GCOV_KERNEL), --no-unreachable) \ - --link +objtool-args = $(vmlinux-objtool-args-y) --link # Link of vmlinux.o used for section mismatch analysis # --------------------------------------------------------------------------- @@ -55,7 +48,7 @@ quiet_cmd_ld_vmlinux.o = LD $@ cmd_ld_vmlinux.o = \ $(LD) ${KBUILD_LDFLAGS} -r -o $@ \ $(addprefix -T , $(initcalls-lds)) \ - --whole-archive $(KBUILD_VMLINUX_OBJS) --no-whole-archive \ + --whole-archive vmlinux.a --no-whole-archive \ --start-group $(KBUILD_VMLINUX_LIBS) --end-group \ $(cmd_objtool) @@ -64,11 +57,35 @@ define rule_ld_vmlinux.o $(call cmd,gen_objtooldep) endef -vmlinux.o: $(initcalls-lds) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS) FORCE +vmlinux.o: $(initcalls-lds) vmlinux.a $(KBUILD_VMLINUX_LIBS) FORCE $(call if_changed_rule,ld_vmlinux.o) targets += vmlinux.o +# module.builtin.modinfo +# --------------------------------------------------------------------------- + +OBJCOPYFLAGS_modules.builtin.modinfo := -j .modinfo -O binary + +targets += modules.builtin.modinfo +modules.builtin.modinfo: vmlinux.o FORCE + $(call if_changed,objcopy) + +# module.builtin +# --------------------------------------------------------------------------- + +# The second line aids cases where multiple modules share the same object. + +quiet_cmd_modules_builtin = GEN $@ + cmd_modules_builtin = \ + tr '\0' '\n' < $< | \ + sed -n 's/^[[:alnum:]:_]*\.file=//p' | \ + tr ' ' '\n' | uniq | sed -e 's:^:kernel/:' -e 's/$$/.ko/' > $@ + +targets += modules.builtin +modules.builtin: modules.builtin.modinfo FORCE + $(call if_changed,modules_builtin) + # Add FORCE to the prequisites of a target to force it to be always rebuilt. # --------------------------------------------------------------------------- diff --git a/scripts/asn1_compiler.c b/scripts/asn1_compiler.c index adabd41452640991ed86d1557ac32012042ce4e0..71d4a7c87900890a04b3451450cbaab4be5f2ee7 100644 --- a/scripts/asn1_compiler.c +++ b/scripts/asn1_compiler.c @@ -832,7 +832,7 @@ static void parse(void) static struct element *element_list; -static struct element *alloc_elem(struct token *type) +static struct element *alloc_elem(void) { struct element *e = calloc(1, sizeof(*e)); if (!e) { @@ -860,7 +860,7 @@ static struct element *parse_type(struct token **_cursor, struct token *end, char *p; int labelled = 0, implicit = 0; - top = element = alloc_elem(cursor); + top = element = alloc_elem(); element->class = ASN1_UNIV; element->method = ASN1_PRIM; element->tag = token_to_tag[cursor->token_type]; @@ -939,7 +939,7 @@ static struct element *parse_type(struct token **_cursor, struct token *end, if (!implicit) element->method |= ASN1_CONS; element->compound = implicit ? TAG_OVERRIDE : SEQUENCE; - element->children = alloc_elem(cursor); + element->children = alloc_elem(); element = element->children; element->class = ASN1_UNIV; element->method = ASN1_PRIM; diff --git a/scripts/atomic/check-atomics.sh b/scripts/atomic/check-atomics.sh deleted file mode 100755 index 0e7bab3eb0d11b808301187258e7d12b7ee2050f..0000000000000000000000000000000000000000 --- a/scripts/atomic/check-atomics.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# -# Check if atomic headers are up-to-date - -ATOMICDIR=$(dirname $0) -ATOMICTBL=${ATOMICDIR}/atomics.tbl -LINUXDIR=${ATOMICDIR}/../.. - -echo '' | sha1sum - > /dev/null 2>&1 -if [ $? -ne 0 ]; then - printf "sha1sum not available, skipping atomic header checks.\n" - exit 0 -fi - -cat </dev/null #if defined(__clang__) @@ -32,7 +32,7 @@ get_canonical_version() # $@ instead of $1 because multiple words might be given, e.g. CC="ccache gcc". orig_args="$@" -set -- $(get_compiler_info "$@") +set -- $(get_c_compiler_info "$@") name=$1 @@ -52,7 +52,7 @@ ICC) min_version=$($min_tool_version icc) ;; *) - echo "$orig_args: unknown compiler" >&2 + echo "$orig_args: unknown C compiler" >&2 exit 1 ;; esac @@ -62,7 +62,7 @@ min_cversion=$(get_canonical_version $min_version) if [ "$cversion" -lt "$min_cversion" ]; then echo >&2 "***" - echo >&2 "*** Compiler is too old." + echo >&2 "*** C compiler is too old." echo >&2 "*** Your $name version: $version" echo >&2 "*** Minimum $name version: $min_version" echo >&2 "***" diff --git a/scripts/check-local-export b/scripts/check-local-export index 6ccc2f4674166c2b22d22cf04e9d0c34224a5151..f90b5a9c67b357fa94b43ffacc3335d82d5bfc90 100755 --- a/scripts/check-local-export +++ b/scripts/check-local-export @@ -1,25 +1,14 @@ -#!/usr/bin/env bash +#!/bin/sh # SPDX-License-Identifier: GPL-2.0-only # # Copyright (C) 2022 Masahiro Yamada +# Copyright (C) 2022 Owen Rafferty # # Exit with error if a local exported symbol is found. # EXPORT_SYMBOL should be used for global symbols. set -e - -# catch errors from ${NM} -set -o pipefail - -# Run the last element of a pipeline in the current shell. -# Without this, the while-loop would be executed in a subshell, and -# the changes made to 'symbol_types' and 'export_symbols' would be lost. -shopt -s lastpipe - -declare -A symbol_types -declare -a export_symbols - -exit_code=0 +pid=$$ # If there is no symbol in the object, ${NM} (both GNU nm and llvm-nm) shows # 'no symbols' diagnostic (but exits with 0). It is harmless and hidden by @@ -29,43 +18,53 @@ exit_code=0 # TODO: # Use --quiet instead of 2>/dev/null when we upgrade the minimum version of # binutils to 2.37, llvm to 13.0.0. -# Then, the following line will be really simple: -# ${NM} --quiet ${1} | +# Then, the following line will be simpler: +# { ${NM} --quiet ${1} || kill 0; } | + +{ ${NM} ${1} 2>/dev/null || { echo "${0}: ${NM} failed" >&2; kill $pid; } } | +${AWK} -v "file=${1}" ' +BEGIN { + i = 0 +} + +# Skip the line if the number of fields is less than 3. +# +# case 1) +# For undefined symbols, the first field (value) is empty. +# The outout looks like this: +# " U _printk" +# It is unneeded to record undefined symbols. +# +# case 2) +# For Clang LTO, llvm-nm outputs a line with type t but empty name: +# "---------------- t" +!length($3) { + next +} -{ ${NM} ${1} 2>/dev/null || { echo "${0}: ${NM} failed" >&2; false; } } | -while read value type name -do - # Skip the line if the number of fields is less than 3. - # - # case 1) - # For undefined symbols, the first field (value) is empty. - # The outout looks like this: - # " U _printk" - # It is unneeded to record undefined symbols. - # - # case 2) - # For Clang LTO, llvm-nm outputs a line with type 't' but empty name: - # "---------------- t" - if [[ -z ${name} ]]; then - continue - fi +# save (name, type) in the associative array +{ symbol_types[$3]=$2 } - # save (name, type) in the associative array - symbol_types[${name}]=${type} +# append the exported symbol to the array +($3 ~ /^__ksymtab_/) { + export_symbols[i] = $3 + sub(/^__ksymtab_/, "", export_symbols[i]) + i++ +} - # append the exported symbol to the array - if [[ ${name} == __ksymtab_* ]]; then - export_symbols+=(${name#__ksymtab_}) - fi -done +END { + exit_code = 0 + for (j = 0; j < i; ++j) { + name = export_symbols[j] + # nm(3) says "If lowercase, the symbol is usually local" + if (symbol_types[name] ~ /[a-z]/) { + printf "%s: error: local symbol %s was exported\n", + file, name | "cat 1>&2" + exit_code = 1 + } + } -for name in "${export_symbols[@]}" -do - # nm(3) says "If lowercase, the symbol is usually local" - if [[ ${symbol_types[$name]} =~ [a-z] ]]; then - echo "$@: error: local symbol '${name}' was exported" >&2 - exit_code=1 - fi -done + exit exit_code +}' -exit ${exit_code} +exit $? diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 79e759aac543b8dea36fe405df70f017e423d3e8..1e5e66ae5a52202f99c3d93f71ad81b239056217 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -576,10 +576,14 @@ our $typeKernelTypedefs = qr{(?x: (?:__)?(?:u|s|be|le)(?:8|16|32|64)| atomic_t )}; +our $typeStdioTypedefs = qr{(?x: + FILE +)}; our $typeTypedefs = qr{(?x: $typeC99Typedefs\b| $typeOtherOSTypedefs\b| - $typeKernelTypedefs\b + $typeKernelTypedefs\b| + $typeStdioTypedefs\b )}; our $zero_initializer = qr{(?:(?:0[xX])?0+$Int_type?|NULL|false)\b}; @@ -807,6 +811,8 @@ our %deprecated_apis = ( "rcu_barrier_sched" => "rcu_barrier", "get_state_synchronize_sched" => "get_state_synchronize_rcu", "cond_synchronize_sched" => "cond_synchronize_rcu", + "kmap" => "kmap_local_page", + "kmap_atomic" => "kmap_local_page", ); #Create a search pattern for all these strings to speed up a loop below @@ -3140,6 +3146,50 @@ sub process { } } +# Check Fixes: styles is correct + if (!$in_header_lines && + $line =~ /^\s*fixes:?\s*(?:commit\s*)?[0-9a-f]{5,}\b/i) { + my $orig_commit = ""; + my $id = "0123456789ab"; + my $title = "commit title"; + my $tag_case = 1; + my $tag_space = 1; + my $id_length = 1; + my $id_case = 1; + my $title_has_quotes = 0; + + if ($line =~ /(\s*fixes:?)\s+([0-9a-f]{5,})\s+($balanced_parens)/i) { + my $tag = $1; + $orig_commit = $2; + $title = $3; + + $tag_case = 0 if $tag eq "Fixes:"; + $tag_space = 0 if ($line =~ /^fixes:? [0-9a-f]{5,} ($balanced_parens)/i); + + $id_length = 0 if ($orig_commit =~ /^[0-9a-f]{12}$/i); + $id_case = 0 if ($orig_commit !~ /[A-F]/); + + # Always strip leading/trailing parens then double quotes if existing + $title = substr($title, 1, -1); + if ($title =~ /^".*"$/) { + $title = substr($title, 1, -1); + $title_has_quotes = 1; + } + } + + my ($cid, $ctitle) = git_commit_info($orig_commit, $id, + $title); + + if ($ctitle ne $title || $tag_case || $tag_space || + $id_length || $id_case || !$title_has_quotes) { + if (WARN("BAD_FIXES_TAG", + "Please use correct Fixes: style 'Fixes: <12 chars of sha1> (\"\")' - ie: 'Fixes: $cid (\"$ctitle\")'\n" . $herecurr) && + $fix) { + $fixed[$fixlinenr] = "Fixes: $cid (\"$ctitle\")"; + } + } + } + # Check email subject for common tools that don't need to be mentioned if ($in_header_lines && $line =~ /^Subject:.*\b(?:checkpatch|sparse|smatch)\b[^:]/i) { @@ -3616,7 +3666,7 @@ sub process { my $comment = ""; if ($realfile =~ /\.(h|s|S)$/) { $comment = '/*'; - } elsif ($realfile =~ /\.(c|dts|dtsi)$/) { + } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; @@ -3664,7 +3714,7 @@ sub process { } # check we are in a valid source file if not then ignore this hunk - next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/); + next if ($realfile !~ /\.(h|c|rs|s|S|sh|dtsi|dts)$/); # check for using SPDX-License-Identifier on the wrong line number if ($realline != $checklicenseline && @@ -3751,7 +3801,7 @@ sub process { if ($realfile =~ /\.S$/ && $line =~ /^\+\s*(?:[A-Z]+_)?SYM_[A-Z]+_(?:START|END)(?:_[A-Z_]+)?\s*\(\s*\.L/) { WARN("AVOID_L_PREFIX", - "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/asm-annotations.rst\n" . $herecurr); + "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/core-api/asm-annotations.rst\n" . $herecurr); } # check we are in a valid source file C or perl if not then ignore this hunk @@ -4695,12 +4745,12 @@ sub process { } } -# avoid BUG() or BUG_ON() - if ($line =~ /\b(?:BUG|BUG_ON)\b/) { +# do not use BUG() or variants + if ($line =~ /\b(?!AA_|BUILD_|DCCP_|IDA_|KVM_|RWLOCK_|snd_|SPIN_)(?:[a-zA-Z_]*_)?BUG(?:_ON)?(?:_[A-Z_]+)?\s*\(/) { my $msg_level = \&WARN; $msg_level = \&CHK if ($file); &{$msg_level}("AVOID_BUG", - "Avoid crashing the kernel - try using WARN_ON & recovery code rather than BUG() or BUG_ON()\n" . $herecurr); + "Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants\n" . $herecurr); } # avoid LINUX_VERSION_CODE @@ -6783,15 +6833,19 @@ sub process { } if ($bad_specifier ne "") { my $stat_real = get_stat_real($linenr, $lc); + my $msg_level = \&WARN; my $ext_type = "Invalid"; my $use = ""; if ($bad_specifier =~ /p[Ff]/) { $use = " - use %pS instead"; $use =~ s/pS/ps/ if ($bad_specifier =~ /pf/); + } elsif ($bad_specifier =~ /pA/) { + $use = " - '%pA' is only intended to be used from Rust code"; + $msg_level = \&ERROR; } - WARN("VSPRINTF_POINTER_EXTENSION", - "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); + &{$msg_level}("VSPRINTF_POINTER_EXTENSION", + "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n"); } } } diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index 47da25b3ba7d206b7000847597c1519dd9dd3155..d800b2c0af9771644f51bc0d8fb60d831b0b61f0 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -109,20 +109,6 @@ def to_cmdfile(path): return os.path.join(dir, '.' + base + '.cmd') -def cmdfiles_for_o(obj): - """Generate the iterator of .cmd files associated with the object - - Yield the .cmd file used to build the given object - - Args: - obj: The object path - - Yields: - The path to .cmd file - """ - yield to_cmdfile(obj) - - def cmdfiles_for_a(archive, ar): """Generate the iterator of .cmd files associated with the archive. @@ -211,13 +197,10 @@ def main(): for path in paths: # If 'path' is a directory, handle all .cmd files under it. # Otherwise, handle .cmd files associated with the file. - # Most of built-in objects are linked via archives (built-in.a or lib.a) - # but some objects are linked to vmlinux directly. + # built-in objects are linked via vmlinux.a # Modules are listed in modules.order. if os.path.isdir(path): cmdfiles = cmdfiles_in_dir(path) - elif path.endswith('.o'): - cmdfiles = cmdfiles_for_o(path) elif path.endswith('.a'): cmdfiles = cmdfiles_for_a(path, ar) elif path.endswith('modules.order'): diff --git a/scripts/clang-tools/run-clang-tools.py b/scripts/clang-tools/run-clang-tools.py index 1337cedca096ddfdc7c9b16cd56774e7d005f89b..56f2ec8f0f40a707df4265657664e4347bbf7bda 100755 --- a/scripts/clang-tools/run-clang-tools.py +++ b/scripts/clang-tools/run-clang-tools.py @@ -12,7 +12,6 @@ compile_commands.json. import argparse import json import multiprocessing -import os import subprocess import sys @@ -46,13 +45,14 @@ def init(l, a): def run_analysis(entry): # Disable all checks, then re-enable the ones we want - checks = "-checks=-*," + checks = [] + checks.append("-checks=-*") if args.type == "clang-tidy": - checks += "linuxkernel-*" + checks.append("linuxkernel-*") else: - checks += "clang-analyzer-*" - checks += ",-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling" - p = subprocess.run(["clang-tidy", "-p", args.path, checks, entry["file"]], + checks.append("clang-analyzer-*") + checks.append("-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling") + p = subprocess.run(["clang-tidy", "-p", args.path, ",".join(checks), entry["file"]], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=entry["directory"]) diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh index 7075e26ab2c4f420f521f89c2550c1aa6691f4b5..564c5632e1a243f2c3ebed14370ce12b4e99d64b 100755 --- a/scripts/decode_stacktrace.sh +++ b/scripts/decode_stacktrace.sh @@ -8,6 +8,14 @@ usage() { echo " $0 -r <release> | <vmlinux> [<base path>|auto] [<modules path>]" } +# Try to find a Rust demangler +if type llvm-cxxfilt >/dev/null 2>&1 ; then + cppfilt=llvm-cxxfilt +elif type c++filt >/dev/null 2>&1 ; then + cppfilt=c++filt + cppfilt_opts=-i +fi + if [[ $1 == "-r" ]] ; then vmlinux="" basepath="auto" @@ -180,6 +188,12 @@ parse_symbol() { # In the case of inlines, move everything to same line code=${code//$'\n'/' '} + # Demangle if the name looks like a Rust symbol and if + # we got a Rust demangler + if [[ $name =~ ^_R && $cppfilt != "" ]] ; then + name=$("$cppfilt" "$cppfilt_opts" "$name") + fi + # Replace old address with pretty line numbers symbol="$segment$name ($code)" } diff --git a/scripts/decodecode b/scripts/decodecode index c711a196511c666edfda6ef9cc0dbe1875c5592a..b28fd26865617193defb80b8051e794a77e2ccba 100755 --- a/scripts/decodecode +++ b/scripts/decodecode @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # SPDX-License-Identifier: GPL-2.0 # Disassemble the Code: line in Linux oopses # usage: decodecode < oops.file @@ -8,6 +8,8 @@ # AFLAGS=--32 decodecode < 386.oops # PC=hex - the PC (program counter) the oops points to +faultlinenum=1 + cleanup() { rm -f $T $T.s $T.o $T.oo $T.aa $T.dis exit 1 @@ -102,28 +104,125 @@ disas() { grep -v "/tmp\|Disassembly\|\.text\|^$" > $t.dis 2>&1 } +# Match the maximum number of opcode bytes from @op_bytes contained within +# @opline +# +# Params: +# @op_bytes: The string of bytes from the Code: line +# @opline: The disassembled line coming from objdump +# +# Returns: +# The max number of opcode bytes from the beginning of @op_bytes which match +# the opcode bytes in the objdump line. +get_substr_opcode_bytes_num() +{ + local op_bytes=$1 + local opline=$2 + + local retval=0 + substr="" + + for opc in $op_bytes; + do + substr+="$opc" + + # return if opcode bytes do not match @opline anymore + if ! echo $opline | grep -q "$substr"; + then + break + fi + + # add trailing space + substr+=" " + retval=$((retval+1)) + done + + return $retval +} + +# Return the line number in objdump output to where the IP marker in the Code: +# line points to +# +# Params: +# @all_code: code in bytes without the marker +# @dis_file: disassembled file +# @ip_byte: The byte to which the IP points to +get_faultlinenum() +{ + local all_code="$1" + local dis_file="$2" + + # num bytes including IP byte + local num_bytes_ip=$(( $3 + 1 * $width )) + + # Add the two header lines (we're counting from 1). + local retval=3 + + # remove marker + all_code=$(echo $all_code | sed -e 's/[<>()]//g') + + while read line + do + get_substr_opcode_bytes_num "$all_code" "$line" + ate_opcodes=$? + + if ! (( $ate_opcodes )); then + continue + fi + + num_bytes_ip=$((num_bytes_ip - ($ate_opcodes * $width) )) + if (( $num_bytes_ip <= 0 )); then + break + fi + + # Delete matched opcode bytes from all_code. For that, compute + # how many chars those opcodes are represented by and include + # trailing space. + # + # a byte is 2 chars, ate_opcodes is also the number of trailing + # spaces + del_chars=$(( ($ate_opcodes * $width * 2) + $ate_opcodes )) + + all_code=$(echo $all_code | sed -e "s!^.\{$del_chars\}!!") + + let "retval+=1" + + done < $dis_file + + return $retval +} + marker=`expr index "$code" "\<"` if [ $marker -eq 0 ]; then marker=`expr index "$code" "\("` fi - touch $T.oo if [ $marker -ne 0 ]; then - # 2 opcode bytes and a single space - pc_sub=$(( $marker / 3 )) + # How many bytes to subtract from the program counter + # in order to get to the beginning virtual address of the + # Code: + pc_sub=$(( (($marker - 1) / (2 * $width + 1)) * $width )) echo All code >> $T.oo echo ======== >> $T.oo beforemark=`echo "$code"` echo -n " .$type 0x" > $T.s + echo $beforemark | sed -e 's/ /,0x/g; s/[<>()]//g' >> $T.s + disas $T $pc_sub + cat $T.dis >> $T.oo - rm -f $T.o $T.s $T.dis -# and fix code at-and-after marker + get_faultlinenum "$code" "$T.dis" $pc_sub + faultlinenum=$? + + # and fix code at-and-after marker code=`echo "$code" | cut -c$((${marker} + 1))-` + + rm -f $T.o $T.s $T.dis fi + echo Code starting with the faulting instruction > $T.aa echo =========================================== >> $T.aa code=`echo $code | sed -e 's/\r//;s/ [<(]/ /;s/[>)] / /;s/ /,0x/g; s/[>)]$//'` @@ -132,15 +231,6 @@ echo $code >> $T.s disas $T 0 cat $T.dis >> $T.aa -# (lines of whole $T.oo) - (lines of $T.aa, i.e. "Code starting") + 3, -# i.e. the title + the "===..=" line (sed is counting from 1, 0 address is -# special) -faultlinenum=$(( $(wc -l $T.oo | cut -d" " -f1) - \ - $(wc -l $T.aa | cut -d" " -f1) + 3)) - -faultline=`cat $T.dis | head -1 | cut -d":" -f2-` -faultline=`echo "$faultline" | sed -e 's/\[/\\\[/g; s/\]/\\\]/g'` - cat $T.oo | sed -e "${faultlinenum}s/^\([^:]*:\)\(.*\)/\1\*\2\t\t<-- trapping instruction/" echo cat $T.aa diff --git a/scripts/dtc/dt-extract-compatibles b/scripts/dtc/dt-extract-compatibles new file mode 100755 index 0000000000000000000000000000000000000000..a1119762ed086a0eaf1ff9b3670ba0b49090e45c --- /dev/null +++ b/scripts/dtc/dt-extract-compatibles @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only + +import os +import glob +import re +import argparse + + +def parse_of_declare_macros(data): + """ Find all compatible strings in OF_DECLARE() style macros """ + compat_list = [] + # CPU_METHOD_OF_DECLARE does not have a compatible string + for m in re.finditer(r'(?<!CPU_METHOD_)(IRQCHIP|OF)_(DECLARE|MATCH)(_DRIVER)?\(.*?\)', data): + try: + compat = re.search(r'"(.*?)"', m[0])[1] + except: + # Fails on compatible strings in #define, so just skip + continue + compat_list += [compat] + + return compat_list + + +def parse_of_device_id(data): + """ Find all compatible strings in of_device_id structs """ + compat_list = [] + for m in re.finditer(r'of_device_id\s+[a-zA-Z0-9_]+\[\]\s*=\s*({.*?);', data): + compat_list += re.findall(r'\.compatible\s+=\s+"([a-zA-Z0-9_\-,]+)"', m[1]) + + return compat_list + + +def parse_compatibles(file): + with open(file, 'r', encoding='utf-8') as f: + data = f.read().replace('\n', '') + + compat_list = parse_of_declare_macros(data) + compat_list += parse_of_device_id(data) + + return compat_list + +def print_compat(filename, compatibles): + if not compatibles: + return + if show_filename: + compat_str = ' '.join(compatibles) + print(filename + ": compatible(s): " + compat_str) + else: + print(*compatibles, sep='\n') + +show_filename = False + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("cfile", type=str, nargs='*', help="C source files or directories to parse") + ap.add_argument('-H', '--with-filename', help="Print filename with compatibles", action="store_true") + args = ap.parse_args() + + show_filename = args.with_filename + + for f in args.cfile: + if os.path.isdir(f): + for filename in glob.iglob(f + "/**/*.c", recursive=True): + compat_list = parse_compatibles(filename) + print_compat(filename, compat_list) + else: + compat_list = parse_compatibles(f) + print_compat(f, compat_list) diff --git a/scripts/extract-ikconfig b/scripts/extract-ikconfig index 3b42f255e2baa85738240c6bd847e4fe21a378f5..8df33e7d6daa3122856aa10b54995f56d058da30 100755 --- a/scripts/extract-ikconfig +++ b/scripts/extract-ikconfig @@ -62,6 +62,7 @@ try_decompress 'BZh' xy bunzip2 try_decompress '\135\0\0\0' xxx unlzma try_decompress '\211\114\132' xy 'lzop -d' try_decompress '\002\041\114\030' xyy 'lz4 -d -l' +try_decompress '\050\265\057\375' xxx unzstd # Bail out: echo "$me: Cannot find kernel config." >&2 diff --git a/scripts/gcc-ld b/scripts/gcc-ld deleted file mode 100755 index 997b818c3962646b38c06747197d38e6aa1fa514..0000000000000000000000000000000000000000 --- a/scripts/gcc-ld +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# run gcc with ld options -# used as a wrapper to execute link time optimizations -# yes virginia, this is not pretty - -ARGS="-nostdlib" - -while [ "$1" != "" ] ; do - case "$1" in - -save-temps|-m32|-m64) N="$1" ;; - -r) N="$1" ;; - -[Wg]*) N="$1" ;; - -[olv]|-[Ofd]*|-nostdlib) N="$1" ;; - --end-group|--start-group) - N="-Wl,$1" ;; - -[RTFGhIezcbyYu]*|\ ---script|--defsym|-init|-Map|--oformat|-rpath|\ --rpath-link|--sort-section|--section-start|-Tbss|-Tdata|-Ttext|\ ---version-script|--dynamic-list|--version-exports-symbol|--wrap|-m) - A="$1" ; shift ; N="-Wl,$A,$1" ;; - -[m]*) N="$1" ;; - -*) N="-Wl,$1" ;; - *) N="$1" ;; - esac - ARGS="$ARGS $N" - shift -done - -exec $CC $ARGS diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py new file mode 100755 index 0000000000000000000000000000000000000000..75bb611bd7516cd12f5948a2194420c18d2e27cb --- /dev/null +++ b/scripts/generate_rust_analyzer.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""generate_rust_analyzer - Generates the `rust-project.json` file for `rust-analyzer`. +""" + +import argparse +import json +import logging +import pathlib +import sys + +def generate_crates(srctree, objtree, sysroot_src): + # Generate the configuration list. + cfg = [] + with open(objtree / "include" / "generated" / "rustc_cfg") as fd: + for line in fd: + line = line.replace("--cfg=", "") + line = line.replace("\n", "") + cfg.append(line) + + # Now fill the crates list -- dependencies need to come first. + # + # Avoid O(n^2) iterations by keeping a map of indexes. + crates = [] + crates_indexes = {} + + def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False): + crates_indexes[display_name] = len(crates) + crates.append({ + "display_name": display_name, + "root_module": str(root_module), + "is_workspace_member": is_workspace_member, + "is_proc_macro": is_proc_macro, + "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps], + "cfg": cfg, + "edition": "2021", + "env": { + "RUST_MODFILE": "This is only for rust-analyzer" + } + }) + + # First, the ones in `rust/` since they are a bit special. + append_crate( + "core", + sysroot_src / "core" / "src" / "lib.rs", + [], + is_workspace_member=False, + ) + + append_crate( + "compiler_builtins", + srctree / "rust" / "compiler_builtins.rs", + [], + ) + + append_crate( + "alloc", + srctree / "rust" / "alloc" / "lib.rs", + ["core", "compiler_builtins"], + ) + + append_crate( + "macros", + srctree / "rust" / "macros" / "lib.rs", + [], + is_proc_macro=True, + ) + crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so" + + append_crate( + "bindings", + srctree / "rust"/ "bindings" / "lib.rs", + ["core"], + cfg=cfg, + ) + crates[-1]["env"]["OBJTREE"] = str(objtree.resolve(True)) + + append_crate( + "kernel", + srctree / "rust" / "kernel" / "lib.rs", + ["core", "alloc", "macros", "bindings"], + cfg=cfg, + ) + crates[-1]["source"] = { + "include_dirs": [ + str(srctree / "rust" / "kernel"), + str(objtree / "rust") + ], + "exclude_dirs": [], + } + + # Then, the rest outside of `rust/`. + # + # We explicitly mention the top-level folders we want to cover. + for folder in ("samples", "drivers"): + for path in (srctree / folder).rglob("*.rs"): + logging.info("Checking %s", path) + name = path.name.replace(".rs", "") + + # Skip those that are not crate roots. + if f"{name}.o" not in open(path.parent / "Makefile").read(): + continue + + logging.info("Adding %s", name) + append_crate( + name, + path, + ["core", "alloc", "kernel"], + cfg=cfg, + ) + + return crates + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument("srctree", type=pathlib.Path) + parser.add_argument("objtree", type=pathlib.Path) + parser.add_argument("sysroot_src", type=pathlib.Path) + args = parser.parse_args() + + logging.basicConfig( + format="[%(asctime)s] [%(levelname)s] %(message)s", + level=logging.INFO if args.verbose else logging.WARNING + ) + + rust_project = { + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src), + "sysroot_src": str(args.sysroot_src), + } + + json.dump(rust_project, sys.stdout, sort_keys=True, indent=4) + +if __name__ == "__main__": + main() diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs new file mode 100644 index 0000000000000000000000000000000000000000..3c6cbe2b278d302ebd6375e900dbe4875765805f --- /dev/null +++ b/scripts/generate_rust_target.rs @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The custom target specification file generator for `rustc`. +//! +//! To configure a target from scratch, a JSON-encoded file has to be passed +//! to `rustc` (introduced in [RFC 131]). These options and the file itself are +//! unstable. Eventually, `rustc` should provide a way to do this in a stable +//! manner. For instance, via command-line arguments. Therefore, this file +//! should avoid using keys which can be set via `-C` or `-Z` options. +//! +//! [RFC 131]: https://rust-lang.github.io/rfcs/0131-target-specification.html + +use std::{ + collections::HashMap, + fmt::{Display, Formatter, Result}, + io::BufRead, +}; + +enum Value { + Boolean(bool), + Number(i32), + String(String), + Object(Object), +} + +type Object = Vec<(String, Value)>; + +/// Minimal "almost JSON" generator (e.g. no `null`s, no arrays, no escaping), +/// enough for this purpose. +impl Display for Value { + fn fmt(&self, formatter: &mut Formatter<'_>) -> Result { + match self { + Value::Boolean(boolean) => write!(formatter, "{}", boolean), + Value::Number(number) => write!(formatter, "{}", number), + Value::String(string) => write!(formatter, "\"{}\"", string), + Value::Object(object) => { + formatter.write_str("{")?; + if let [ref rest @ .., ref last] = object[..] { + for (key, value) in rest { + write!(formatter, "\"{}\": {},", key, value)?; + } + write!(formatter, "\"{}\": {}", last.0, last.1)?; + } + formatter.write_str("}") + } + } + } +} + +struct TargetSpec(Object); + +impl TargetSpec { + fn new() -> TargetSpec { + TargetSpec(Vec::new()) + } +} + +trait Push<T> { + fn push(&mut self, key: &str, value: T); +} + +impl Push<bool> for TargetSpec { + fn push(&mut self, key: &str, value: bool) { + self.0.push((key.to_string(), Value::Boolean(value))); + } +} + +impl Push<i32> for TargetSpec { + fn push(&mut self, key: &str, value: i32) { + self.0.push((key.to_string(), Value::Number(value))); + } +} + +impl Push<String> for TargetSpec { + fn push(&mut self, key: &str, value: String) { + self.0.push((key.to_string(), Value::String(value))); + } +} + +impl Push<&str> for TargetSpec { + fn push(&mut self, key: &str, value: &str) { + self.push(key, value.to_string()); + } +} + +impl Push<Object> for TargetSpec { + fn push(&mut self, key: &str, value: Object) { + self.0.push((key.to_string(), Value::Object(value))); + } +} + +impl Display for TargetSpec { + fn fmt(&self, formatter: &mut Formatter<'_>) -> Result { + // We add some newlines for clarity. + formatter.write_str("{\n")?; + if let [ref rest @ .., ref last] = self.0[..] { + for (key, value) in rest { + write!(formatter, " \"{}\": {},\n", key, value)?; + } + write!(formatter, " \"{}\": {}\n", last.0, last.1)?; + } + formatter.write_str("}") + } +} + +struct KernelConfig(HashMap<String, String>); + +impl KernelConfig { + /// Parses `include/config/auto.conf` from `stdin`. + fn from_stdin() -> KernelConfig { + let mut result = HashMap::new(); + + let stdin = std::io::stdin(); + let mut handle = stdin.lock(); + let mut line = String::new(); + + loop { + line.clear(); + + if handle.read_line(&mut line).unwrap() == 0 { + break; + } + + if line.starts_with('#') { + continue; + } + + let (key, value) = line.split_once('=').expect("Missing `=` in line."); + result.insert(key.to_string(), value.trim_end_matches('\n').to_string()); + } + + KernelConfig(result) + } + + /// Does the option exist in the configuration (any value)? + /// + /// The argument must be passed without the `CONFIG_` prefix. + /// This avoids repetition and it also avoids `fixdep` making us + /// depend on it. + fn has(&self, option: &str) -> bool { + let option = "CONFIG_".to_owned() + option; + self.0.contains_key(&option) + } +} + +fn main() { + let cfg = KernelConfig::from_stdin(); + let mut ts = TargetSpec::new(); + + // `llvm-target`s are taken from `scripts/Makefile.clang`. + if cfg.has("X86_64") { + ts.push("arch", "x86_64"); + ts.push( + "data-layout", + "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", + ); + let mut features = "-3dnow,-3dnowa,-mmx,+soft-float".to_string(); + if cfg.has("RETPOLINE") { + features += ",+retpoline-external-thunk"; + } + ts.push("features", features); + ts.push("llvm-target", "x86_64-linux-gnu"); + ts.push("target-pointer-width", "64"); + } else { + panic!("Unsupported architecture"); + } + + ts.push("emit-debug-gdb-scripts", false); + ts.push("frame-pointer", "may-omit"); + ts.push( + "stack-probes", + vec![("kind".to_string(), Value::String("none".to_string()))], + ); + + // Everything else is LE, whether `CPU_LITTLE_ENDIAN` is declared or not + // (e.g. x86). It is also `rustc`'s default. + if cfg.has("CPU_BIG_ENDIAN") { + ts.push("target-endian", "big"); + } + + println!("{}", ts); +} diff --git a/scripts/head-object-list.txt b/scripts/head-object-list.txt new file mode 100644 index 0000000000000000000000000000000000000000..b16326a92c458abbeafe1f3d73113e9065de74c3 --- /dev/null +++ b/scripts/head-object-list.txt @@ -0,0 +1,53 @@ +# Head objects +# +# The objects listed here are placed at the head of vmlinux. A typical use-case +# is an object that contains the entry point. This is kept for compatibility +# with head-y, which Kbuild used to support. +# +# A counter approach is to control the section placement by the linker script. +# The code marked as __HEAD goes into the ".head.text" section, which is placed +# before the normal ".text" section. +# +# If you can achieve the correct code ordering by linker script, please delete +# the entry from this file. +# +arch/alpha/kernel/head.o +arch/arc/kernel/head.o +arch/arm/kernel/head-nommu.o +arch/arm/kernel/head.o +arch/arm64/kernel/head.o +arch/csky/kernel/head.o +arch/hexagon/kernel/head.o +arch/ia64/kernel/head.o +arch/loongarch/kernel/head.o +arch/m68k/68000/head.o +arch/m68k/coldfire/head.o +arch/m68k/kernel/head.o +arch/m68k/kernel/sun3-head.o +arch/microblaze/kernel/head.o +arch/mips/kernel/head.o +arch/nios2/kernel/head.o +arch/openrisc/kernel/head.o +arch/parisc/kernel/head.o +arch/powerpc/kernel/head_40x.o +arch/powerpc/kernel/head_44x.o +arch/powerpc/kernel/head_64.o +arch/powerpc/kernel/head_8xx.o +arch/powerpc/kernel/head_85xx.o +arch/powerpc/kernel/head_book3s_32.o +arch/powerpc/kernel/entry_64.o +arch/powerpc/kernel/fpu.o +arch/powerpc/kernel/vector.o +arch/powerpc/kernel/prom_init.o +arch/riscv/kernel/head.o +arch/s390/kernel/head64.o +arch/sh/kernel/head_32.o +arch/sparc/kernel/head_32.o +arch/sparc/kernel/head_64.o +arch/x86/kernel/head_32.o +arch/x86/kernel/head_64.o +arch/x86/kernel/head32.o +arch/x86/kernel/head64.o +arch/x86/kernel/ebda.o +arch/x86/kernel/platform-quirks.o +arch/xtensa/kernel/head.o diff --git a/scripts/is_rust_module.sh b/scripts/is_rust_module.sh new file mode 100755 index 0000000000000000000000000000000000000000..28b3831a7593f42b12b4b3e8a67733d9a1c0fe44 --- /dev/null +++ b/scripts/is_rust_module.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# is_rust_module.sh module.ko +# +# Returns `0` if `module.ko` is a Rust module, `1` otherwise. + +set -e + +# Using the `16_` prefix ensures other symbols with the same substring +# are not picked up (even if it would be unlikely). The last part is +# used just in case LLVM decides to use the `.` suffix. +# +# In the future, checking for the `.comment` section may be another +# option, see https://github.com/rust-lang/rust/pull/97550. +${NM} "$*" | grep -qE '^[0-9a-fA-F]+ r _R[^[:space:]]+16___IS_RUST_MODULE[^[:space:]]*$' diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index f18e6dfc68c58398c6f1bfa15e43d5f3101a39a7..03fa07ad45d95b8e6ff64820fc4d34c2bc607214 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -18,6 +18,7 @@ * */ +#include <getopt.h> #include <stdbool.h> #include <stdio.h> #include <stdlib.h> @@ -27,7 +28,23 @@ #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0])) -#define KSYM_NAME_LEN 128 +#define _stringify_1(x) #x +#define _stringify(x) _stringify_1(x) + +#define KSYM_NAME_LEN 512 + +/* + * A substantially bigger size than the current maximum. + * + * It cannot be defined as an expression because it gets stringified + * for the fscanf() format string. Therefore, a _Static_assert() is + * used instead to maintain the relationship with KSYM_NAME_LEN. + */ +#define KSYM_NAME_LEN_BUFFER 2048 +_Static_assert( + KSYM_NAME_LEN_BUFFER == KSYM_NAME_LEN * 4, + "Please keep KSYM_NAME_LEN_BUFFER in sync with KSYM_NAME_LEN" +); struct sym_entry { unsigned long long addr; @@ -71,7 +88,7 @@ static unsigned char best_table_len[256]; static void usage(void) { fprintf(stderr, "Usage: kallsyms [--all-symbols] [--absolute-percpu] " - "[--base-relative] < in.map > out.S\n"); + "[--base-relative] in.map > out.S\n"); exit(1); } @@ -107,9 +124,6 @@ static bool is_ignored_symbol(const char *name, char type) /* Symbol names that begin with the following are ignored.*/ static const char * const ignored_prefixes[] = { - "$", /* local symbols for ARM, MIPS, etc. */ - ".L", /* local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. */ - "__crc_", /* modversions */ "__efistub_", /* arm64 EFI stub namespace */ "__kvm_nvhe_$", /* arm64 local symbols in non-VHE KVM namespace */ "__kvm_nvhe_.L", /* arm64 local symbols in non-VHE KVM namespace */ @@ -119,6 +133,7 @@ static bool is_ignored_symbol(const char *name, char type) "__ThumbV7PILongThunk_", "__LA25Thunk_", /* mips lld */ "__microLA25Thunk_", + "__kcfi_typeid_", /* CFI type identifiers */ NULL }; @@ -198,15 +213,15 @@ static void check_symbol_range(const char *sym, unsigned long long addr, static struct sym_entry *read_symbol(FILE *in) { - char name[500], type; + char name[KSYM_NAME_LEN_BUFFER+1], type; unsigned long long addr; unsigned int len; struct sym_entry *sym; int rc; - rc = fscanf(in, "%llx %c %499s\n", &addr, &type, name); + rc = fscanf(in, "%llx %c %" _stringify(KSYM_NAME_LEN_BUFFER) "s\n", &addr, &type, name); if (rc != 3) { - if (rc != EOF && fgets(name, 500, in) == NULL) + if (rc != EOF && fgets(name, ARRAY_SIZE(name), in) == NULL) fprintf(stderr, "Read error or end of file.\n"); return NULL; } @@ -313,12 +328,19 @@ static void shrink_table(void) } } -static void read_map(FILE *in) +static void read_map(const char *in) { + FILE *fp; struct sym_entry *sym; - while (!feof(in)) { - sym = read_symbol(in); + fp = fopen(in, "r"); + if (!fp) { + perror(in); + exit(1); + } + + while (!feof(fp)) { + sym = read_symbol(fp); if (!sym) continue; @@ -329,12 +351,15 @@ static void read_map(FILE *in) table = realloc(table, sizeof(*table) * table_size); if (!table) { fprintf(stderr, "out of memory\n"); + fclose(fp); exit (1); } } table[table_cnt++] = sym; } + + fclose(fp); } static void output_label(const char *label) @@ -471,12 +496,35 @@ static void write_src(void) if ((i & 0xFF) == 0) markers[i >> 8] = off; - printf("\t.byte 0x%02x", table[i]->len); + /* There cannot be any symbol of length zero. */ + if (table[i]->len == 0) { + fprintf(stderr, "kallsyms failure: " + "unexpected zero symbol length\n"); + exit(EXIT_FAILURE); + } + + /* Only lengths that fit in up-to-two-byte ULEB128 are supported. */ + if (table[i]->len > 0x3FFF) { + fprintf(stderr, "kallsyms failure: " + "unexpected huge symbol length\n"); + exit(EXIT_FAILURE); + } + + /* Encode length with ULEB128. */ + if (table[i]->len <= 0x7F) { + /* Most symbols use a single byte for the length. */ + printf("\t.byte 0x%02x", table[i]->len); + off += table[i]->len + 1; + } else { + /* "Big" symbols use two bytes. */ + printf("\t.byte 0x%02x, 0x%02x", + (table[i]->len & 0x7F) | 0x80, + (table[i]->len >> 7) & 0x7F); + off += table[i]->len + 2; + } for (k = 0; k < table[i]->len; k++) printf(", 0x%02x", table[i]->sym[k]); printf("\n"); - - off += table[i]->len + 1; } printf("\n"); @@ -765,22 +813,26 @@ static void record_relative_base(void) int main(int argc, char **argv) { - if (argc >= 2) { - int i; - for (i = 1; i < argc; i++) { - if(strcmp(argv[i], "--all-symbols") == 0) - all_symbols = 1; - else if (strcmp(argv[i], "--absolute-percpu") == 0) - absolute_percpu = 1; - else if (strcmp(argv[i], "--base-relative") == 0) - base_relative = 1; - else - usage(); - } - } else if (argc != 1) + while (1) { + static struct option long_options[] = { + {"all-symbols", no_argument, &all_symbols, 1}, + {"absolute-percpu", no_argument, &absolute_percpu, 1}, + {"base-relative", no_argument, &base_relative, 1}, + {}, + }; + + int c = getopt_long(argc, argv, "", long_options, NULL); + + if (c == -1) + break; + if (c != 0) + usage(); + } + + if (optind >= argc) usage(); - read_map(stdin); + read_map(argv[optind]); shrink_table(); if (absolute_percpu) make_percpus_absolute(); diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c index 4178065ca27fd626476bd84216992c877a6f54b9..33d19e419908b8315603f04db41189ed9c506a0c 100644 --- a/scripts/kconfig/conf.c +++ b/scripts/kconfig/conf.c @@ -551,7 +551,7 @@ static int conf_choice(struct menu *menu) print_help(child); continue; } - sym_set_choice_value(sym, child->sym); + sym_set_tristate_value(child->sym, yes); for (child = child->list; child; child = child->next) { indent += 2; conf(child); diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index c4340c90e172f875739dd19a8939d812cf66c978..b7c9f1dd5e4229df71e99ae9585424155ac101f0 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -216,6 +216,13 @@ static const char *conf_get_autoheader_name(void) return name ? name : "include/generated/autoconf.h"; } +static const char *conf_get_rustccfg_name(void) +{ + char *name = getenv("KCONFIG_RUSTCCFG"); + + return name ? name : "include/generated/rustc_cfg"; +} + static int conf_set_sym_val(struct symbol *sym, int def, int def_flags, char *p) { char *p2; @@ -605,6 +612,9 @@ static const struct comment_style comment_style_c = { static void conf_write_heading(FILE *fp, const struct comment_style *cs) { + if (!cs) + return; + fprintf(fp, "%s\n", cs->prefix); fprintf(fp, "%s Automatically generated file; DO NOT EDIT.\n", @@ -745,6 +755,65 @@ static void print_symbol_for_c(FILE *fp, struct symbol *sym) free(escaped); } +static void print_symbol_for_rustccfg(FILE *fp, struct symbol *sym) +{ + const char *val; + const char *val_prefix = ""; + char *val_prefixed = NULL; + size_t val_prefixed_len; + char *escaped = NULL; + + if (sym->type == S_UNKNOWN) + return; + + val = sym_get_string_value(sym); + + switch (sym->type) { + case S_BOOLEAN: + case S_TRISTATE: + /* + * We do not care about disabled ones, i.e. no need for + * what otherwise are "comments" in other printers. + */ + if (*val == 'n') + return; + + /* + * To have similar functionality to the C macro `IS_ENABLED()` + * we provide an empty `--cfg CONFIG_X` here in both `y` + * and `m` cases. + * + * Then, the common `fprintf()` below will also give us + * a `--cfg CONFIG_X="y"` or `--cfg CONFIG_X="m"`, which can + * be used as the equivalent of `IS_BUILTIN()`/`IS_MODULE()`. + */ + fprintf(fp, "--cfg=%s%s\n", CONFIG_, sym->name); + break; + case S_HEX: + if (val[0] != '0' || (val[1] != 'x' && val[1] != 'X')) + val_prefix = "0x"; + break; + default: + break; + } + + if (strlen(val_prefix) > 0) { + val_prefixed_len = strlen(val) + strlen(val_prefix) + 1; + val_prefixed = xmalloc(val_prefixed_len); + snprintf(val_prefixed, val_prefixed_len, "%s%s", val_prefix, val); + val = val_prefixed; + } + + /* All values get escaped: the `--cfg` option only takes strings */ + escaped = escape_string_value(val); + val = escaped; + + fprintf(fp, "--cfg=%s%s=%s\n", CONFIG_, sym->name, val); + + free(escaped); + free(val_prefixed); +} + /* * Write out a minimal config. * All values that has default values are skipped as this is redundant. @@ -1132,6 +1201,12 @@ int conf_write_autoconf(int overwrite) if (ret) return ret; + ret = __conf_write_autoconf(conf_get_rustccfg_name(), + print_symbol_for_rustccfg, + NULL); + if (ret) + return ret; + /* * Create include/config/auto.conf. This must be the last step because * Kbuild has a dependency on auto.conf and this marks the successful diff --git a/scripts/kconfig/lkc.h b/scripts/kconfig/lkc.h index fa8c010aa6834718679a7a1bfe208c16c5b31ea4..6ac2eabe109d2659232e774681dae6323529c00e 100644 --- a/scripts/kconfig/lkc.h +++ b/scripts/kconfig/lkc.h @@ -98,7 +98,6 @@ bool menu_is_empty(struct menu *menu); bool menu_is_visible(struct menu *menu); bool menu_has_prompt(struct menu *menu); const char *menu_get_prompt(struct menu *menu); -struct menu *menu_get_root_menu(struct menu *menu); struct menu *menu_get_parent_menu(struct menu *menu); bool menu_has_help(struct menu *menu); const char *menu_get_help(struct menu *menu); @@ -124,11 +123,6 @@ static inline struct symbol *sym_get_choice_value(struct symbol *sym) return (struct symbol *)sym->curr.val; } -static inline bool sym_set_choice_value(struct symbol *ch, struct symbol *chval) -{ - return sym_set_tristate_value(chval, yes); -} - static inline bool sym_is_choice(struct symbol *sym) { return sym->flags & SYMBOL_CHOICE ? true : false; diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index 3d6f7cba88464b30c21aea07c148424250c23455..62b6313f51c8ba88d9def3cefa8b712fd5b6b3e1 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -661,11 +661,6 @@ const char *menu_get_prompt(struct menu *menu) return NULL; } -struct menu *menu_get_root_menu(struct menu *menu) -{ - return &rootmenu; -} - struct menu *menu_get_parent_menu(struct menu *menu) { enum prop_type type; diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index eecc1863e556b95acce0222fed00760708f5c60b..918470d768e9c7d9a1fbbdda6aea85ff59002e1a 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -3,17 +3,15 @@ # # link vmlinux # -# vmlinux is linked from the objects selected by $(KBUILD_VMLINUX_OBJS) and -# $(KBUILD_VMLINUX_LIBS). Most are built-in.a files from top-level directories -# in the kernel tree, others are specified in arch/$(ARCH)/Makefile. +# vmlinux is linked from the objects in vmlinux.a and $(KBUILD_VMLINUX_LIBS). +# vmlinux.a contains objects that are linked unconditionally. # $(KBUILD_VMLINUX_LIBS) are archives which are linked conditionally # (not within --whole-archive), and do not require symbol indexes added. # # vmlinux # ^ # | -# +--< $(KBUILD_VMLINUX_OBJS) -# | +--< init/built-in.a drivers/built-in.a mm/built-in.a + more +# +--< vmlinux.a # | # +--< $(KBUILD_VMLINUX_LIBS) # | +--< lib/lib.a + more @@ -67,7 +65,7 @@ vmlinux_link() objs=vmlinux.o libs= else - objs="${KBUILD_VMLINUX_OBJS}" + objs=vmlinux.a libs="${KBUILD_VMLINUX_LIBS}" fi @@ -75,6 +73,8 @@ vmlinux_link() objs="${objs} .vmlinux.export.o" fi + objs="${objs} init/version-timestamp.o" + if [ "${SRCARCH}" = "um" ]; then wl=-Wl, ld="${CC}" @@ -157,7 +157,7 @@ kallsyms() fi info KSYMS ${2} - ${NM} -n ${1} | scripts/kallsyms ${kallsymopt} > ${2} + scripts/kallsyms ${kallsymopt} ${1} > ${2} } # Perform one step in kallsyms generation, including temporary linking of @@ -170,7 +170,8 @@ kallsyms_step() kallsyms_S=${kallsyms_vmlinux}.S vmlinux_link ${kallsyms_vmlinux} "${kallsymso_prev}" ${btf_vmlinux_bin_o} - kallsyms ${kallsyms_vmlinux} ${kallsyms_S} + mksysmap ${kallsyms_vmlinux} ${kallsyms_vmlinux}.syms + kallsyms ${kallsyms_vmlinux}.syms ${kallsyms_S} info AS ${kallsyms_S} ${CC} ${NOSTDINC_FLAGS} ${LINUXINCLUDE} ${KBUILD_CPPFLAGS} \ @@ -182,6 +183,7 @@ kallsyms_step() # See mksymap for additional details mksysmap() { + info NM ${2} ${CONFIG_SHELL} "${srctree}/scripts/mksysmap" ${1} ${2} } @@ -197,8 +199,6 @@ cleanup() rm -f System.map rm -f vmlinux rm -f vmlinux.map - rm -f .vmlinux.objs - rm -f .vmlinux.export.c } # Use "make V=1" to debug this script @@ -213,52 +213,7 @@ if [ "$1" = "clean" ]; then exit 0 fi -# Update version -info GEN .version -if [ -r .version ]; then - VERSION=$(expr 0$(cat .version) + 1) - echo $VERSION > .version -else - rm -f .version - echo 1 > .version -fi; - -# final build of init/ -${MAKE} -f "${srctree}/scripts/Makefile.build" obj=init need-builtin=1 - -#link vmlinux.o -${MAKE} -f "${srctree}/scripts/Makefile.vmlinux_o" - -# Generate the list of in-tree objects in vmlinux -# -# This is used to retrieve symbol versions generated by genksyms. -for f in ${KBUILD_VMLINUX_OBJS} ${KBUILD_VMLINUX_LIBS}; do - case ${f} in - *libgcc.a) - # Some architectures do '$(CC) --print-libgcc-file-name' to - # borrow libgcc.a from the toolchain. - # There is no EXPORT_SYMBOL in external objects. Ignore this. - ;; - *.a) - ${AR} t ${f} ;; - *) - echo ${f} ;; - esac -done > .vmlinux.objs - -# modpost vmlinux.o to check for section mismatches -${MAKE} -f "${srctree}/scripts/Makefile.modpost" MODPOST_VMLINUX=1 - -info MODINFO modules.builtin.modinfo -${OBJCOPY} -j .modinfo -O binary vmlinux.o modules.builtin.modinfo -info GEN modules.builtin -# The second line aids cases where multiple modules share the same object. -tr '\0' '\n' < modules.builtin.modinfo | sed -n 's/^[[:alnum:]:_]*\.file=//p' | - tr ' ' '\n' | uniq | sed -e 's:^:kernel/:' -e 's/$/.ko/' > modules.builtin - -if is_enabled CONFIG_MODULES; then - ${MAKE} -f "${srctree}/scripts/Makefile.vmlinux" .vmlinux.export.o -fi +${MAKE} -f "${srctree}/scripts/Makefile.build" obj=init init/version-timestamp.o btf_vmlinux_bin_o="" if is_enabled CONFIG_DEBUG_INFO_BTF; then @@ -318,7 +273,6 @@ if is_enabled CONFIG_DEBUG_INFO_BTF && is_enabled CONFIG_BPF; then ${RESOLVE_BTFIDS} vmlinux fi -info SYSMAP System.map mksysmap vmlinux System.map if is_enabled CONFIG_BUILDTIME_TABLE_SORT; then @@ -331,9 +285,7 @@ fi # step a (see comment above) if is_enabled CONFIG_KALLSYMS; then - mksysmap ${kallsyms_vmlinux} .tmp_System.map - - if ! cmp -s System.map .tmp_System.map; then + if ! cmp -s System.map ${kallsyms_vmlinux}.syms; then echo >&2 Inconsistent kallsyms data echo >&2 Try "make KALLSYMS_EXTRA_PASS=1" as a workaround exit 1 diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 250925aab101c8eeda2728f78538e81992dda38e..b6593eac5003cfae2ddfce5ac50ab6155058377e 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -30,6 +30,12 @@ llvm) echo 11.0.0 fi ;; +rustc) + echo 1.62.0 + ;; +bindgen) + echo 0.56.0 + ;; *) echo "$1: unknown tool" >&2 exit 1 diff --git a/scripts/mkcompile_h b/scripts/mkcompile_h index ca40a5258c8778214c93f32ae241b1274d0cceaf..2596f78e52ef95683a11110085878233d564b40a 100755 --- a/scripts/mkcompile_h +++ b/scripts/mkcompile_h @@ -1,33 +1,10 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 -TARGET=$1 -ARCH=$2 -SMP=$3 -PREEMPT=$4 -PREEMPT_DYNAMIC=$5 -PREEMPT_RT=$6 -CC_VERSION="$7" -LD=$8 +UTS_MACHINE=$1 +CC_VERSION="$2" +LD=$3 -# Do not expand names -set -f - -# Fix the language to get consistent output -LC_ALL=C -export LC_ALL - -if [ -z "$KBUILD_BUILD_VERSION" ]; then - VERSION=$(cat .version 2>/dev/null || echo 1) -else - VERSION=$KBUILD_BUILD_VERSION -fi - -if [ -z "$KBUILD_BUILD_TIMESTAMP" ]; then - TIMESTAMP=`date` -else - TIMESTAMP=$KBUILD_BUILD_TIMESTAMP -fi if test -z "$KBUILD_BUILD_USER"; then LINUX_COMPILE_BY=$(whoami | sed 's/\\/\\\\/') else @@ -39,63 +16,12 @@ else LINUX_COMPILE_HOST=$KBUILD_BUILD_HOST fi -UTS_VERSION="#$VERSION" -CONFIG_FLAGS="" -if [ -n "$SMP" ] ; then CONFIG_FLAGS="SMP"; fi - -if [ -n "$PREEMPT_RT" ] ; then - CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT_RT" -elif [ -n "$PREEMPT_DYNAMIC" ] ; then - CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT_DYNAMIC" -elif [ -n "$PREEMPT" ] ; then - CONFIG_FLAGS="$CONFIG_FLAGS PREEMPT" -fi - -# Truncate to maximum length -UTS_LEN=64 -UTS_VERSION="$(echo $UTS_VERSION $CONFIG_FLAGS $TIMESTAMP | cut -b -$UTS_LEN)" - -# Generate a temporary compile.h +LD_VERSION=$(LC_ALL=C $LD -v | head -n1 | + sed -e 's/(compatible with [^)]*)//' -e 's/[[:space:]]*$//') -{ echo /\* This file is auto generated, version $VERSION \*/ - if [ -n "$CONFIG_FLAGS" ] ; then echo "/* $CONFIG_FLAGS */"; fi - - echo \#define UTS_MACHINE \"$ARCH\" - - echo \#define UTS_VERSION \"$UTS_VERSION\" - - printf '#define LINUX_COMPILE_BY "%s"\n' "$LINUX_COMPILE_BY" - echo \#define LINUX_COMPILE_HOST \"$LINUX_COMPILE_HOST\" - - LD_VERSION=$($LD -v | head -n1 | sed 's/(compatible with [^)]*)//' \ - | sed 's/[[:space:]]*$//') - printf '#define LINUX_COMPILER "%s"\n' "$CC_VERSION, $LD_VERSION" -} > .tmpcompile - -# Only replace the real compile.h if the new one is different, -# in order to preserve the timestamp and avoid unnecessary -# recompilations. -# We don't consider the file changed if only the date/time changed, -# unless KBUILD_BUILD_TIMESTAMP was explicitly set (e.g. for -# reproducible builds with that value referring to a commit timestamp). -# A kernel config change will increase the generation number, thus -# causing compile.h to be updated (including date/time) due to the -# changed comment in the -# first line. - -if [ -z "$KBUILD_BUILD_TIMESTAMP" ]; then - IGNORE_PATTERN="UTS_VERSION" -else - IGNORE_PATTERN="NOT_A_PATTERN_TO_BE_MATCHED" -fi - -if [ -r $TARGET ] && \ - grep -v $IGNORE_PATTERN $TARGET > .tmpver.1 && \ - grep -v $IGNORE_PATTERN .tmpcompile > .tmpver.2 && \ - cmp -s .tmpver.1 .tmpver.2; then - rm -f .tmpcompile -else - echo " UPD $TARGET" - mv -f .tmpcompile $TARGET -fi -rm -f .tmpver.1 .tmpver.2 +cat <<EOF +#define UTS_MACHINE "${UTS_MACHINE}" +#define LINUX_COMPILE_BY "${LINUX_COMPILE_BY}" +#define LINUX_COMPILE_HOST "${LINUX_COMPILE_HOST}" +#define LINUX_COMPILER "${CC_VERSION}, ${LD_VERSION}" +EOF diff --git a/scripts/mksysmap b/scripts/mksysmap index 9aa23d15862a02b0ae86646b0cb6d7f84f13c5f8..16a08b8ef2f83d5b9a16e4af4a998f1ae6dd17df 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -37,8 +37,24 @@ # readprofile starts reading symbols when _stext is found, and # continue until it finds a symbol which is not either of 'T', 't', -# 'W' or 'w'. __crc_ are 'A' and placed in the middle -# so we just ignore them to let readprofile continue to work. -# (At least sparc64 has __crc_ in the middle). +# 'W' or 'w'. +# +# Ignored prefixes: +# $ - local symbols for ARM, MIPS, etc. +# .L - local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. +# __crc_ - modversions +# __kstrtab_ - EXPORT_SYMBOL (symbol name) +# __kstrtabns_ - EXPORT_SYMBOL (namespace) +# +# Ignored symbols: +# L0 - for LoongArch? -$NM -n $1 | grep -v '\( [aNUw] \)\|\(__crc_\)\|\( \$[adt]\)\|\( \.L\)' > $2 +$NM -n $1 | grep -v \ + -e ' [aNUw] ' \ + -e ' \$' \ + -e ' \.L' \ + -e ' __crc_' \ + -e ' __kstrtab_' \ + -e ' __kstrtabns_' \ + -e ' L0$' \ +> $2 diff --git a/scripts/module.lds.S b/scripts/module.lds.S index 3a3aa2354ed86749d5399fcd3a7c718fe973a043..da4bddd261717bf336c1e0efdb5338f738fc7cad 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -3,20 +3,10 @@ * Archs are free to supply their own linker scripts. ld will * combine them automatically. */ -#ifdef CONFIG_CFI_CLANG -# include <asm/page.h> -# define ALIGN_CFI ALIGN(PAGE_SIZE) -# define SANITIZER_DISCARDS *(.eh_frame) -#else -# define ALIGN_CFI -# define SANITIZER_DISCARDS -#endif - SECTIONS { /DISCARD/ : { *(.discard) *(.discard.*) - SANITIZER_DISCARDS } __ksymtab 0 : { *(SORT(___ksymtab+*)) } @@ -33,6 +23,10 @@ SECTIONS { __patchable_function_entries : { *(__patchable_function_entries) } +#ifdef CONFIG_ARCH_USES_CFI_TRAPS + __kcfi_traps : { KEEP(*(.kcfi_traps)) } +#endif + #ifdef CONFIG_LTO_CLANG /* * With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and @@ -53,15 +47,6 @@ SECTIONS { *(.rodata .rodata.[0-9a-zA-Z_]*) *(.rodata..L*) } - - /* - * With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning - * of the .text section, and is aligned to PAGE_SIZE. - */ - .text : ALIGN_CFI { - *(.text.__cfi_check) - *(.text .text.[0-9a-zA-Z_]* .text..L.cfi*) - } #endif } diff --git a/scripts/package/mkspec b/scripts/package/mkspec index 8fa7c5b8a1a15d4ed004f3cebd5fd0ded82f2441..70392fd2fd29c19534cb1f0054ee2e0551a2f7ec 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -88,17 +88,15 @@ $S mkdir -p %{buildroot}/boot %ifarch ia64 mkdir -p %{buildroot}/boot/efi - cp \$($MAKE image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE + cp \$($MAKE -s image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE ln -s efi/vmlinuz-$KERNELRELEASE %{buildroot}/boot/ %else - cp \$($MAKE image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE + cp \$($MAKE -s image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE %endif $M $MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install $MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE cp .config %{buildroot}/boot/config-$KERNELRELEASE - bzip2 -9 --keep vmlinux - mv vmlinux.bz2 %{buildroot}/boot/vmlinux-$KERNELRELEASE.bz2 $S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/build $S$M rm -f %{buildroot}/lib/modules/$KERNELRELEASE/source $S$M mkdir -p %{buildroot}/usr/src/kernels/$KERNELRELEASE diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh new file mode 100755 index 0000000000000000000000000000000000000000..aebbf19139709da1b739f83570960e186d49951d --- /dev/null +++ b/scripts/rust_is_available.sh @@ -0,0 +1,160 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# Tests whether a suitable Rust toolchain is available. +# +# Pass `-v` for human output and more checks (as warnings). + +set -e + +min_tool_version=$(dirname $0)/min-tool-version.sh + +# Convert the version string x.y.z to a canonical up-to-7-digits form. +# +# Note that this function uses one more digit (compared to other +# instances in other version scripts) to give a bit more space to +# `rustc` since it will reach 1.100.0 in late 2026. +get_canonical_version() +{ + IFS=. + set -- $1 + echo $((100000 * $1 + 100 * $2 + $3)) +} + +# Check that the Rust compiler exists. +if ! command -v "$RUSTC" >/dev/null; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' could not be found." + echo >&2 "***" + fi + exit 1 +fi + +# Check that the Rust bindings generator exists. +if ! command -v "$BINDGEN" >/dev/null; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." + echo >&2 "***" + fi + exit 1 +fi + +# Check that the Rust compiler version is suitable. +# +# Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_compiler_version=$( \ + LC_ALL=C "$RUSTC" --version 2>/dev/null \ + | head -n 1 \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ +) +rust_compiler_min_version=$($min_tool_version rustc) +rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) +rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version) +if [ "$rust_compiler_cversion" -lt "$rust_compiler_min_cversion" ]; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' is too old." + echo >&2 "*** Your version: $rust_compiler_version" + echo >&2 "*** Minimum version: $rust_compiler_min_version" + echo >&2 "***" + fi + exit 1 +fi +if [ "$1" = -v ] && [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' is too new. This may or may not work." + echo >&2 "*** Your version: $rust_compiler_version" + echo >&2 "*** Expected version: $rust_compiler_min_version" + echo >&2 "***" +fi + +# Check that the Rust bindings generator is suitable. +# +# Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_bindings_generator_version=$( \ + LC_ALL=C "$BINDGEN" --version 2>/dev/null \ + | head -n 1 \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ +) +rust_bindings_generator_min_version=$($min_tool_version bindgen) +rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) +rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version) +if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cversion" ]; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." + echo >&2 "*** Your version: $rust_bindings_generator_version" + echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" + echo >&2 "***" + fi + exit 1 +fi +if [ "$1" = -v ] && [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' is too new. This may or may not work." + echo >&2 "*** Your version: $rust_bindings_generator_version" + echo >&2 "*** Expected version: $rust_bindings_generator_min_version" + echo >&2 "***" +fi + +# Check that the `libclang` used by the Rust bindings generator is suitable. +bindgen_libclang_version=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null \ + | grep -F 'clang version ' \ + | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + | head -n 1 \ +) +bindgen_libclang_min_version=$($min_tool_version llvm) +bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) +bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version) +if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." + echo >&2 "*** Your version: $bindgen_libclang_version" + echo >&2 "*** Minimum version: $bindgen_libclang_min_version" + echo >&2 "***" + fi + exit 1 +fi + +# If the C compiler is Clang, then we can also check whether its version +# matches the `libclang` version used by the Rust bindings generator. +# +# In the future, we might be able to perform a full version check, see +# https://github.com/rust-lang/rust-bindgen/issues/2138. +if [ "$1" = -v ]; then + cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') + if [ "$cc_name" = Clang ]; then + clang_version=$( \ + LC_ALL=C "$CC" --version 2>/dev/null \ + | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' + ) + if [ "$clang_version" != "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" + echo >&2 "*** version does not match Clang's. This may be a problem." + echo >&2 "*** libclang version: $bindgen_libclang_version" + echo >&2 "*** Clang version: $clang_version" + echo >&2 "***" + fi + fi +fi + +# Check that the source code for the `core` standard library exists. +# +# `$KRUSTFLAGS` is passed in case the user added `--sysroot`. +rustc_sysroot=$("$RUSTC" $KRUSTFLAGS --print sysroot) +rustc_src=${RUST_LIB_SRC:-"$rustc_sysroot/lib/rustlib/src/rust/library"} +rustc_src_core="$rustc_src/core/src/lib.rs" +if [ ! -e "$rustc_src_core" ]; then + if [ "$1" = -v ]; then + echo >&2 "***" + echo >&2 "*** Source code for the 'core' standard library could not be found" + echo >&2 "*** at '$rustc_src_core'." + echo >&2 "***" + fi + exit 1 +fi diff --git a/scripts/rust_is_available_bindgen_libclang.h b/scripts/rust_is_available_bindgen_libclang.h new file mode 100644 index 0000000000000000000000000000000000000000..0ef6db10d67413466375de3faddcf15ea98c1b56 --- /dev/null +++ b/scripts/rust_is_available_bindgen_libclang.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#pragma message("clang version " __clang_version__) diff --git a/scripts/selinux/install_policy.sh b/scripts/selinux/install_policy.sh index 2dccf141241d76e1620c68b2c32149f025c9f218..24086793b0d8d40db6d2498b4120a73f0ff549c2 100755 --- a/scripts/selinux/install_policy.sh +++ b/scripts/selinux/install_policy.sh @@ -31,8 +31,7 @@ fi if selinuxenabled; then echo "SELinux is already enabled" echo "This prevents safely relabeling all files." - echo "Boot with selinux=0 on the kernel command-line or" - echo "SELINUX=disabled in /etc/selinux/config." + echo "Boot with selinux=0 on the kernel command-line." exit 1 fi @@ -78,7 +77,7 @@ cd /etc/selinux/dummy/contexts/files $SF -F file_contexts / mounts=`cat /proc/$$/mounts | \ - egrep "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ + grep -E "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \ awk '{ print $2 '}` $SF -F file_contexts $mounts diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening index bd2aabb2c60f926f9484c88e81ecabfbe4a898d8..d766b7d0ffd138592862e9706413ac584ee990da 100644 --- a/security/Kconfig.hardening +++ b/security/Kconfig.hardening @@ -22,11 +22,17 @@ menu "Memory initialization" config CC_HAS_AUTO_VAR_INIT_PATTERN def_bool $(cc-option,-ftrivial-auto-var-init=pattern) -config CC_HAS_AUTO_VAR_INIT_ZERO - # GCC ignores the -enable flag, so we can test for the feature with - # a single invocation using the flag, but drop it as appropriate in - # the Makefile, depending on the presence of Clang. +config CC_HAS_AUTO_VAR_INIT_ZERO_BARE + def_bool $(cc-option,-ftrivial-auto-var-init=zero) + +config CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER + # Clang 16 and later warn about using the -enable flag, but it + # is required before then. def_bool $(cc-option,-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang) + depends on !CC_HAS_AUTO_VAR_INIT_ZERO_BARE + +config CC_HAS_AUTO_VAR_INIT_ZERO + def_bool CC_HAS_AUTO_VAR_INIT_ZERO_BARE || CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER choice prompt "Initialize kernel stack variables at function entry" @@ -106,6 +112,7 @@ choice config INIT_STACK_ALL_PATTERN bool "pattern-init everything (strongest)" depends on CC_HAS_AUTO_VAR_INIT_PATTERN + depends on !KMSAN help Initializes everything on the stack (including padding) with a specific debug value. This is intended to eliminate @@ -124,6 +131,7 @@ choice config INIT_STACK_ALL_ZERO bool "zero-init everything (strongest and safest)" depends on CC_HAS_AUTO_VAR_INIT_ZERO + depends on !KMSAN help Initializes everything on the stack (including padding) with a zero value. This is intended to eliminate all @@ -218,6 +226,7 @@ config STACKLEAK_RUNTIME_DISABLE config INIT_ON_ALLOC_DEFAULT_ON bool "Enable heap memory zeroing on allocation by default" + depends on !KMSAN help This has the effect of setting "init_on_alloc=1" on the kernel command line. This can be disabled with "init_on_alloc=0". @@ -230,6 +239,7 @@ config INIT_ON_ALLOC_DEFAULT_ON config INIT_ON_FREE_DEFAULT_ON bool "Enable heap memory zeroing on free by default" + depends on !KMSAN help This has the effect of setting "init_on_free=1" on the kernel command line. This can be disabled with "init_on_free=0". diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index e29cade7b6627f2a697e8d1f64b6ede60b4159d3..f56070270c69db19350f868286dc43469a3d35ca 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -614,7 +614,7 @@ static int apparmor_sb_pivotroot(const struct path *old_path, return error; } -static int apparmor_getprocattr(struct task_struct *task, char *name, +static int apparmor_getprocattr(struct task_struct *task, const char *name, char **value) { int error = -ENOENT; diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 2e6fb6e2ffd2fe5e62022c80127f6c26bfbcfb3d..23d484e05e6f2f59975e0cbdd4e1ea9c479142cd 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -457,10 +457,21 @@ static int evm_xattr_acl_change(struct user_namespace *mnt_userns, int rc; /* - * user_ns is not relevant here, ACL_USER/ACL_GROUP don't have impact - * on the inode mode (see posix_acl_equiv_mode()). + * An earlier comment here mentioned that the idmappings for + * ACL_{GROUP,USER} don't matter since EVM is only interested in the + * mode stored as part of POSIX ACLs. Nonetheless, if it must translate + * from the uapi POSIX ACL representation to the VFS internal POSIX ACL + * representation it should do so correctly. There's no guarantee that + * we won't change POSIX ACLs in a way that ACL_{GROUP,USER} matters + * for the mode at some point and it's difficult to keep track of all + * the LSM and integrity modules and what they do to POSIX ACLs. + * + * Frankly, EVM shouldn't try to interpret the uapi struct for POSIX + * ACLs it received. It requires knowledge that only the VFS is + * guaranteed to have. */ - acl = posix_acl_from_xattr(&init_user_ns, xattr_value, xattr_value_len); + acl = vfs_set_acl_prepare(mnt_userns, i_user_ns(inode), + xattr_value, xattr_value_len); if (IS_ERR_OR_NULL(acl)) return 1; diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index bde74fcecee38e9207d601c7b47bb1b814856bba..3e0fbbd995342fcf80be6d5254bfe1bd78e01edc 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -750,22 +750,26 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, const struct evm_ima_xattr_data *xvalue = xattr_value; int digsig = 0; int result; + int err; result = ima_protect_xattr(dentry, xattr_name, xattr_value, xattr_value_len); if (result == 1) { if (!xattr_value_len || (xvalue->type >= IMA_XATTR_LAST)) return -EINVAL; + + err = validate_hash_algo(dentry, xvalue, xattr_value_len); + if (err) + return err; + digsig = (xvalue->type == EVM_IMA_XATTR_DIGSIG); } else if (!strcmp(xattr_name, XATTR_NAME_EVM) && xattr_value_len > 0) { digsig = (xvalue->type == EVM_XATTR_PORTABLE_DIGSIG); } if (result == 1 || evm_revalidate_status(xattr_name)) { - result = validate_hash_algo(dentry, xvalue, xattr_value_len); - if (result) - return result; - ima_reset_appraise_flags(d_backing_inode(dentry), digsig); + if (result == 1) + result = 0; } return result; } diff --git a/security/integrity/platform_certs/load_uefi.c b/security/integrity/platform_certs/load_uefi.c index 093894a640dca6f4592268077428832df0449976..b78753d27d8ea60cda66279c80304b04c88fc2e6 100644 --- a/security/integrity/platform_certs/load_uefi.c +++ b/security/integrity/platform_certs/load_uefi.c @@ -31,7 +31,7 @@ static const struct dmi_system_id uefi_skip_cert[] = { { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir8,1") }, { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir8,2") }, { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir9,1") }, - { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacMini8,1") }, + { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "Macmini8,1") }, { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacPro7,1") }, { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "iMac20,1") }, { UEFI_QUIRK_SKIP_CERT("Apple Inc.", "iMac20,2") }, diff --git a/security/keys/internal.h b/security/keys/internal.h index 9b9cf3b6fcbb4d5e57e57100055cee2772d1db40..3c1e7122076b9ebc89c8479302d51387dbb16d02 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -165,8 +165,6 @@ extern struct key *request_key_and_link(struct key_type *type, extern bool lookup_user_key_possessed(const struct key *key, const struct key_match_data *match_data); -#define KEY_LOOKUP_CREATE 0x01 -#define KEY_LOOKUP_PARTIAL 0x02 extern long join_session_keyring(const char *name); extern void key_change_session_keyring(struct callback_head *twork); diff --git a/security/keys/keyring.c b/security/keys/keyring.c index 5e6a907607530e2097ab5a4eba4f19b4d8426f2a..4448758f643a57bcbcb4a7654eccf15932ff78c3 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -79,7 +79,7 @@ static void keyring_revoke(struct key *keyring); static void keyring_destroy(struct key *keyring); static void keyring_describe(const struct key *keyring, struct seq_file *m); static long keyring_read(const struct key *keyring, - char __user *buffer, size_t buflen); + char *buffer, size_t buflen); struct key_type key_type_keyring = { .name = "keyring", diff --git a/security/landlock/fs.c b/security/landlock/fs.c index ec5a6247cd3e7569e08c72dbcd6db4339d53ca7a..64ed7665455fedeb4a35a638cbc9e840b124004d 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -149,6 +149,16 @@ retry: LANDLOCK_ACCESS_FS_READ_FILE) /* clang-format on */ +/* + * All access rights that are denied by default whether they are handled or not + * by a ruleset/layer. This must be ORed with all ruleset->fs_access_masks[] + * entries when we need to get the absolute handled access masks. + */ +/* clang-format off */ +#define ACCESS_INITIALLY_DENIED ( \ + LANDLOCK_ACCESS_FS_REFER) +/* clang-format on */ + /* * @path: Should have been checked by get_path_from_fd(). */ @@ -167,7 +177,9 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, return -EINVAL; /* Transforms relative access rights to absolute ones. */ - access_rights |= LANDLOCK_MASK_ACCESS_FS & ~ruleset->fs_access_masks[0]; + access_rights |= + LANDLOCK_MASK_ACCESS_FS & + ~(ruleset->fs_access_masks[0] | ACCESS_INITIALLY_DENIED); object = get_inode_object(d_backing_inode(path->dentry)); if (IS_ERR(object)) return PTR_ERR(object); @@ -277,23 +289,12 @@ static inline bool is_nouser_or_private(const struct dentry *dentry) static inline access_mask_t get_handled_accesses(const struct landlock_ruleset *const domain) { - access_mask_t access_dom = 0; - unsigned long access_bit; - - for (access_bit = 0; access_bit < LANDLOCK_NUM_ACCESS_FS; - access_bit++) { - size_t layer_level; + access_mask_t access_dom = ACCESS_INITIALLY_DENIED; + size_t layer_level; - for (layer_level = 0; layer_level < domain->num_layers; - layer_level++) { - if (domain->fs_access_masks[layer_level] & - BIT_ULL(access_bit)) { - access_dom |= BIT_ULL(access_bit); - break; - } - } - } - return access_dom; + for (layer_level = 0; layer_level < domain->num_layers; layer_level++) + access_dom |= domain->fs_access_masks[layer_level]; + return access_dom & LANDLOCK_MASK_ACCESS_FS; } static inline access_mask_t @@ -316,8 +317,13 @@ init_layer_masks(const struct landlock_ruleset *const domain, for_each_set_bit(access_bit, &access_req, ARRAY_SIZE(*layer_masks)) { - if (domain->fs_access_masks[layer_level] & - BIT_ULL(access_bit)) { + /* + * Artificially handles all initially denied by default + * access rights. + */ + if (BIT_ULL(access_bit) & + (domain->fs_access_masks[layer_level] | + ACCESS_INITIALLY_DENIED)) { (*layer_masks)[access_bit] |= BIT_ULL(layer_level); handled_accesses |= BIT_ULL(access_bit); @@ -706,7 +712,7 @@ static inline access_mask_t maybe_remove(const struct dentry *const dentry) * allowed accesses in @layer_masks_dom. * * This is similar to check_access_path_dual() but much simpler because it only - * handles walking on the same mount point and only check one set of accesses. + * handles walking on the same mount point and only checks one set of accesses. * * Returns: * - true if all the domain access rights are allowed for @dir; @@ -857,10 +863,6 @@ static int current_check_refer_path(struct dentry *const old_dentry, NULL, NULL); } - /* Backward compatibility: no reparenting support. */ - if (!(get_handled_accesses(dom) & LANDLOCK_ACCESS_FS_REFER)) - return -EXDEV; - access_request_parent1 |= LANDLOCK_ACCESS_FS_REFER; access_request_parent2 |= LANDLOCK_ACCESS_FS_REFER; diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index 735a0865ea113512488cc1ec988c0690c3bc0bd6..2ca0ccbd905aeb46d5fdc84ffc5fe6f800cdcc76 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -149,10 +149,10 @@ static const struct file_operations ruleset_fops = { * * Possible returned errors are: * - * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; - * - EINVAL: unknown @flags, or unknown access, or too small @size; - * - E2BIG or EFAULT: @attr or @size inconsistencies; - * - ENOMSG: empty &landlock_ruleset_attr.handled_access_fs. + * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; + * - %EINVAL: unknown @flags, or unknown access, or too small @size; + * - %E2BIG or %EFAULT: @attr or @size inconsistencies; + * - %ENOMSG: empty &landlock_ruleset_attr.handled_access_fs. */ SYSCALL_DEFINE3(landlock_create_ruleset, const struct landlock_ruleset_attr __user *const, attr, @@ -280,7 +280,7 @@ out_fdput: * @ruleset_fd: File descriptor tied to the ruleset that should be extended * with the new rule. * @rule_type: Identify the structure type pointed to by @rule_attr (only - * LANDLOCK_RULE_PATH_BENEATH for now). + * %LANDLOCK_RULE_PATH_BENEATH for now). * @rule_attr: Pointer to a rule (only of type &struct * landlock_path_beneath_attr for now). * @flags: Must be 0. @@ -290,17 +290,17 @@ out_fdput: * * Possible returned errors are: * - * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; - * - EINVAL: @flags is not 0, or inconsistent access in the rule (i.e. + * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; + * - %EINVAL: @flags is not 0, or inconsistent access in the rule (i.e. * &landlock_path_beneath_attr.allowed_access is not a subset of the * ruleset handled accesses); - * - ENOMSG: Empty accesses (e.g. &landlock_path_beneath_attr.allowed_access); - * - EBADF: @ruleset_fd is not a file descriptor for the current thread, or a + * - %ENOMSG: Empty accesses (e.g. &landlock_path_beneath_attr.allowed_access); + * - %EBADF: @ruleset_fd is not a file descriptor for the current thread, or a * member of @rule_attr is not a file descriptor as expected; - * - EBADFD: @ruleset_fd is not a ruleset file descriptor, or a member of + * - %EBADFD: @ruleset_fd is not a ruleset file descriptor, or a member of * @rule_attr is not the expected file descriptor type; - * - EPERM: @ruleset_fd has no write access to the underlying ruleset; - * - EFAULT: @rule_attr inconsistency. + * - %EPERM: @ruleset_fd has no write access to the underlying ruleset; + * - %EFAULT: @rule_attr inconsistency. */ SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd, const enum landlock_rule_type, rule_type, @@ -378,20 +378,20 @@ out_put_ruleset: * @flags: Must be 0. * * This system call enables to enforce a Landlock ruleset on the current - * thread. Enforcing a ruleset requires that the task has CAP_SYS_ADMIN in its + * thread. Enforcing a ruleset requires that the task has %CAP_SYS_ADMIN in its * namespace or is running with no_new_privs. This avoids scenarios where * unprivileged tasks can affect the behavior of privileged children. * * Possible returned errors are: * - * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; - * - EINVAL: @flags is not 0. - * - EBADF: @ruleset_fd is not a file descriptor for the current thread; - * - EBADFD: @ruleset_fd is not a ruleset file descriptor; - * - EPERM: @ruleset_fd has no read access to the underlying ruleset, or the + * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time; + * - %EINVAL: @flags is not 0. + * - %EBADF: @ruleset_fd is not a file descriptor for the current thread; + * - %EBADFD: @ruleset_fd is not a ruleset file descriptor; + * - %EPERM: @ruleset_fd has no read access to the underlying ruleset, or the * current thread is not running with no_new_privs, or it doesn't have - * CAP_SYS_ADMIN in its namespace. - * - E2BIG: The maximum number of stacked rulesets is reached for the current + * %CAP_SYS_ADMIN in its namespace. + * - %E2BIG: The maximum number of stacked rulesets is reached for the current * thread. */ SYSCALL_DEFINE2(landlock_restrict_self, const int, ruleset_fd, const __u32, diff --git a/security/loadpin/Kconfig b/security/loadpin/Kconfig index 70e7985b2561c2ac0b6f3f99cf319caa5f03c783..6724eaba3d36471239e814bfafccad48fa0c045b 100644 --- a/security/loadpin/Kconfig +++ b/security/loadpin/Kconfig @@ -33,4 +33,9 @@ config SECURITY_LOADPIN_VERITY on the LoadPin securityfs entry 'dm-verity'. The ioctl expects a file descriptor of a file with verity digests as parameter. The file must be located on the pinned root and - contain a comma separated list of digests. + start with the line: + + # LOADPIN_TRUSTED_VERITY_ROOT_DIGESTS + + This is followed by the verity digests, with one digest per + line. diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c index 44521582dcba2464de36fa4119293df724dd1c51..de41621f4998e37fcdce46c1fba88e458e8fd9c8 100644 --- a/security/loadpin/loadpin.c +++ b/security/loadpin/loadpin.c @@ -21,6 +21,8 @@ #include <linux/dm-verity-loadpin.h> #include <uapi/linux/loadpin.h> +#define VERITY_DIGEST_FILE_HEADER "# LOADPIN_TRUSTED_VERITY_ROOT_DIGESTS" + static void report_load(const char *origin, struct file *file, char *operation) { char *cmdline, *pathname; @@ -292,9 +294,21 @@ static int read_trusted_verity_root_digests(unsigned int fd) p = strim(data); while ((d = strsep(&p, "\n")) != NULL) { - int len = strlen(d); + int len; struct dm_verity_loadpin_trusted_root_digest *trd; + if (d == data) { + /* first line, validate header */ + if (strcmp(d, VERITY_DIGEST_FILE_HEADER)) { + rc = -EPROTO; + goto err; + } + + continue; + } + + len = strlen(d); + if (len % 2) { rc = -EPROTO; goto err; diff --git a/security/lockdown/lockdown.c b/security/lockdown/lockdown.c index 87cbdc64d272c72409bcb6f8f5e6a740ef95764a..a79b985e917ee642f1b9a7a8350b8a9fc26ff6c8 100644 --- a/security/lockdown/lockdown.c +++ b/security/lockdown/lockdown.c @@ -63,7 +63,7 @@ static int lockdown_is_locked_down(enum lockdown_reason what) if (kernel_locked_down >= what) { if (lockdown_reasons[what]) - pr_notice("Lockdown: %s: %s is restricted; see man kernel_lockdown.7\n", + pr_notice_ratelimited("Lockdown: %s: %s is restricted; see man kernel_lockdown.7\n", current->comm, lockdown_reasons[what]); return -EPERM; } diff --git a/security/lsm_audit.c b/security/lsm_audit.c index 78a278f28e49ab0a59ffffbc81bd61bcc4bc28de..75cc3f8d2a4245198e44f579ef3b60798f5837f6 100644 --- a/security/lsm_audit.c +++ b/security/lsm_audit.c @@ -44,9 +44,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb, struct iphdr *ih; ih = ip_hdr(skb); - if (ih == NULL) - return -EINVAL; - ad->u.net->v4info.saddr = ih->saddr; ad->u.net->v4info.daddr = ih->daddr; @@ -59,8 +56,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb, switch (ih->protocol) { case IPPROTO_TCP: { struct tcphdr *th = tcp_hdr(skb); - if (th == NULL) - break; ad->u.net->sport = th->source; ad->u.net->dport = th->dest; @@ -68,8 +63,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb, } case IPPROTO_UDP: { struct udphdr *uh = udp_hdr(skb); - if (uh == NULL) - break; ad->u.net->sport = uh->source; ad->u.net->dport = uh->dest; @@ -77,8 +70,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb, } case IPPROTO_DCCP: { struct dccp_hdr *dh = dccp_hdr(skb); - if (dh == NULL) - break; ad->u.net->sport = dh->dccph_sport; ad->u.net->dport = dh->dccph_dport; @@ -86,8 +77,7 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb, } case IPPROTO_SCTP: { struct sctphdr *sh = sctp_hdr(skb); - if (sh == NULL) - break; + ad->u.net->sport = sh->source; ad->u.net->dport = sh->dest; break; @@ -115,8 +105,6 @@ int ipv6_skb_to_auditdata(struct sk_buff *skb, __be16 frag_off; ip6 = ipv6_hdr(skb); - if (ip6 == NULL) - return -EINVAL; ad->u.net->v6info.saddr = ip6->saddr; ad->u.net->v6info.daddr = ip6->daddr; /* IPv6 can have several extension header before the Transport header diff --git a/security/security.c b/security/security.c index 14d30fec8a0031d2a2ed32dbd4e61fe1966e1dfb..79d82cb6e4696c84873e8bf59eabf8dd87a40abe 100644 --- a/security/security.c +++ b/security/security.c @@ -52,6 +52,7 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX+1] = { [LOCKDOWN_IOPORT] = "raw io port access", [LOCKDOWN_MSR] = "raw MSR access", [LOCKDOWN_ACPI_TABLES] = "modifying ACPI tables", + [LOCKDOWN_DEVICE_TREE] = "modifying device tree contents", [LOCKDOWN_PCMCIA_CIS] = "direct PCMCIA CIS storage", [LOCKDOWN_TIOCSSERIAL] = "reconfiguration of serial port IO", [LOCKDOWN_MODULE_PARAMETERS] = "unsafe module parameters", @@ -60,6 +61,7 @@ const char *const lockdown_reasons[LOCKDOWN_CONFIDENTIALITY_MAX+1] = { [LOCKDOWN_XMON_WR] = "xmon write access", [LOCKDOWN_BPF_WRITE_USER] = "use of bpf to write user RAM", [LOCKDOWN_DBG_WRITE_KERNEL] = "use of kgdb/kdb to write kernel RAM", + [LOCKDOWN_RTAS_ERROR_INJECTION] = "RTAS error injection", [LOCKDOWN_INTEGRITY_MAX] = "integrity", [LOCKDOWN_KCORE] = "/proc/kcore access", [LOCKDOWN_KPROBES] = "use of kprobes", @@ -1909,6 +1911,11 @@ void security_task_to_inode(struct task_struct *p, struct inode *inode) call_void_hook(task_to_inode, p, inode); } +int security_create_user_ns(const struct cred *cred) +{ + return call_int_hook(userns_create, 0, cred); +} + int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag) { return call_int_hook(ipc_permission, 0, ipcp, flag); @@ -2057,8 +2064,8 @@ void security_d_instantiate(struct dentry *dentry, struct inode *inode) } EXPORT_SYMBOL(security_d_instantiate); -int security_getprocattr(struct task_struct *p, const char *lsm, char *name, - char **value) +int security_getprocattr(struct task_struct *p, const char *lsm, + const char *name, char **value) { struct security_hook_list *hp; @@ -2660,4 +2667,8 @@ int security_uring_sqpoll(void) { return call_int_hook(uring_sqpoll, 0); } +int security_uring_cmd(struct io_uring_cmd *ioucmd) +{ + return call_int_hook(uring_cmd, 0, ioucmd); +} #endif /* CONFIG_IO_URING */ diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 79573504783bb7f34f698ee9ef1b76c0f063c06a..f553c370397eeb16c2dc7509d97f733add15cfba 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -91,6 +91,7 @@ #include <uapi/linux/mount.h> #include <linux/fsnotify.h> #include <linux/fanotify.h> +#include <linux/io_uring.h> #include "avc.h" #include "objsec.h" @@ -4221,6 +4222,14 @@ static void selinux_task_to_inode(struct task_struct *p, spin_unlock(&isec->lock); } +static int selinux_userns_create(const struct cred *cred) +{ + u32 sid = current_sid(); + + return avc_has_perm(&selinux_state, sid, sid, SECCLASS_USER_NAMESPACE, + USER_NAMESPACE__CREATE, NULL); +} + /* Returns error only if unable to parse addresses */ static int selinux_parse_skb_ipv4(struct sk_buff *skb, struct common_audit_data *ad, u8 *proto) @@ -5986,7 +5995,6 @@ static int selinux_msg_queue_alloc_security(struct kern_ipc_perm *msq) struct ipc_security_struct *isec; struct common_audit_data ad; u32 sid = current_sid(); - int rc; isec = selinux_ipc(msq); ipc_init_security(isec, SECCLASS_MSGQ); @@ -5994,10 +6002,9 @@ static int selinux_msg_queue_alloc_security(struct kern_ipc_perm *msq) ad.type = LSM_AUDIT_DATA_IPC; ad.u.ipc_id = msq->key; - rc = avc_has_perm(&selinux_state, - sid, isec->sid, SECCLASS_MSGQ, - MSGQ__CREATE, &ad); - return rc; + return avc_has_perm(&selinux_state, + sid, isec->sid, SECCLASS_MSGQ, + MSGQ__CREATE, &ad); } static int selinux_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg) @@ -6125,7 +6132,6 @@ static int selinux_shm_alloc_security(struct kern_ipc_perm *shp) struct ipc_security_struct *isec; struct common_audit_data ad; u32 sid = current_sid(); - int rc; isec = selinux_ipc(shp); ipc_init_security(isec, SECCLASS_SHM); @@ -6133,10 +6139,9 @@ static int selinux_shm_alloc_security(struct kern_ipc_perm *shp) ad.type = LSM_AUDIT_DATA_IPC; ad.u.ipc_id = shp->key; - rc = avc_has_perm(&selinux_state, - sid, isec->sid, SECCLASS_SHM, - SHM__CREATE, &ad); - return rc; + return avc_has_perm(&selinux_state, + sid, isec->sid, SECCLASS_SHM, + SHM__CREATE, &ad); } static int selinux_shm_associate(struct kern_ipc_perm *shp, int shmflg) @@ -6210,7 +6215,6 @@ static int selinux_sem_alloc_security(struct kern_ipc_perm *sma) struct ipc_security_struct *isec; struct common_audit_data ad; u32 sid = current_sid(); - int rc; isec = selinux_ipc(sma); ipc_init_security(isec, SECCLASS_SEM); @@ -6218,10 +6222,9 @@ static int selinux_sem_alloc_security(struct kern_ipc_perm *sma) ad.type = LSM_AUDIT_DATA_IPC; ad.u.ipc_id = sma->key; - rc = avc_has_perm(&selinux_state, - sid, isec->sid, SECCLASS_SEM, - SEM__CREATE, &ad); - return rc; + return avc_has_perm(&selinux_state, + sid, isec->sid, SECCLASS_SEM, + SEM__CREATE, &ad); } static int selinux_sem_associate(struct kern_ipc_perm *sma, int semflg) @@ -6327,7 +6330,7 @@ static void selinux_d_instantiate(struct dentry *dentry, struct inode *inode) } static int selinux_getprocattr(struct task_struct *p, - char *name, char **value) + const char *name, char **value) { const struct task_security_struct *__tsec; u32 sid; @@ -6987,6 +6990,28 @@ static int selinux_uring_sqpoll(void) return avc_has_perm(&selinux_state, sid, sid, SECCLASS_IO_URING, IO_URING__SQPOLL, NULL); } + +/** + * selinux_uring_cmd - check if IORING_OP_URING_CMD is allowed + * @ioucmd: the io_uring command structure + * + * Check to see if the current domain is allowed to execute an + * IORING_OP_URING_CMD against the device/file specified in @ioucmd. + * + */ +static int selinux_uring_cmd(struct io_uring_cmd *ioucmd) +{ + struct file *file = ioucmd->file; + struct inode *inode = file_inode(file); + struct inode_security_struct *isec = selinux_inode(inode); + struct common_audit_data ad; + + ad.type = LSM_AUDIT_DATA_FILE; + ad.u.file = file; + + return avc_has_perm(&selinux_state, current_sid(), isec->sid, + SECCLASS_IO_URING, IO_URING__CMD, &ad); +} #endif /* CONFIG_IO_URING */ /* @@ -7111,6 +7136,7 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(task_movememory, selinux_task_movememory), LSM_HOOK_INIT(task_kill, selinux_task_kill), LSM_HOOK_INIT(task_to_inode, selinux_task_to_inode), + LSM_HOOK_INIT(userns_create, selinux_userns_create), LSM_HOOK_INIT(ipc_permission, selinux_ipc_permission), LSM_HOOK_INIT(ipc_getsecid, selinux_ipc_getsecid), @@ -7231,6 +7257,7 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { #ifdef CONFIG_IO_URING LSM_HOOK_INIT(uring_override_creds, selinux_uring_override_creds), LSM_HOOK_INIT(uring_sqpoll, selinux_uring_sqpoll), + LSM_HOOK_INIT(uring_cmd, selinux_uring_cmd), #endif /* diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h index ff757ae5f25379ee7fb49887abc0524715a7661b..a3c380775d410c513e256450a08a6c22ccc3749a 100644 --- a/security/selinux/include/classmap.h +++ b/security/selinux/include/classmap.h @@ -253,7 +253,9 @@ const struct security_class_mapping secclass_map[] = { { "anon_inode", { COMMON_FILE_PERMS, NULL } }, { "io_uring", - { "override_creds", "sqpoll", NULL } }, + { "override_creds", "sqpoll", "cmd", NULL } }, + { "user_namespace", + { "create", NULL } }, { NULL } }; diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c index 8fcdd494af273f402a073d948cb6d6879c98cc12..a00d191394365b50fd59fcdda75ba8f1343ec1f9 100644 --- a/security/selinux/selinuxfs.c +++ b/security/selinux/selinuxfs.c @@ -71,7 +71,7 @@ struct selinux_fs_info { struct dentry *bool_dir; unsigned int bool_num; char **bool_pending_names; - unsigned int *bool_pending_values; + int *bool_pending_values; struct dentry *class_dir; unsigned long last_class_ino; bool policy_opened; @@ -356,7 +356,7 @@ static const struct file_operations sel_policyvers_ops = { /* declaration for sel_write_load */ static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_dir, unsigned int *bool_num, char ***bool_pending_names, - unsigned int **bool_pending_values); + int **bool_pending_values); static int sel_make_classes(struct selinux_policy *newpolicy, struct dentry *class_dir, unsigned long *last_class_ino); @@ -527,7 +527,7 @@ static const struct file_operations sel_policy_ops = { }; static void sel_remove_old_bool_data(unsigned int bool_num, char **bool_names, - unsigned int *bool_values) + int *bool_values) { u32 i; @@ -545,7 +545,7 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi, struct dentry *tmp_parent, *tmp_bool_dir, *tmp_class_dir, *old_dentry; unsigned int tmp_bool_num, old_bool_num; char **tmp_bool_names, **old_bool_names; - unsigned int *tmp_bool_values, *old_bool_values; + int *tmp_bool_values, *old_bool_values; unsigned long tmp_ino = fsi->last_ino; /* Don't increment last_ino in this function */ tmp_parent = sel_make_disconnected_dir(fsi->sb, &tmp_ino); @@ -1423,7 +1423,7 @@ static void sel_remove_entries(struct dentry *de) static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_dir, unsigned int *bool_num, char ***bool_pending_names, - unsigned int **bool_pending_values) + int **bool_pending_values) { int ret; ssize_t len; @@ -1917,7 +1917,6 @@ static int sel_make_class_dir_entries(struct selinux_policy *newpolicy, struct selinux_fs_info *fsi = sb->s_fs_info; struct dentry *dentry = NULL; struct inode *inode = NULL; - int rc; dentry = d_alloc_name(dir, "index"); if (!dentry) @@ -1937,9 +1936,7 @@ static int sel_make_class_dir_entries(struct selinux_policy *newpolicy, if (IS_ERR(dentry)) return PTR_ERR(dentry); - rc = sel_make_perm_files(newpolicy, classname, index, dentry); - - return rc; + return sel_make_perm_files(newpolicy, classname, index, dentry); } static int sel_make_classes(struct selinux_policy *newpolicy, diff --git a/security/selinux/ss/context.h b/security/selinux/ss/context.h index 62990aa1ec9e7178a046a1de5067f35a61629983..eda32c3d4c0a85c40cd98f09e5db62305fd1398d 100644 --- a/security/selinux/ss/context.h +++ b/security/selinux/ss/context.h @@ -38,7 +38,7 @@ static inline void mls_context_init(struct context *c) memset(&c->range, 0, sizeof(c->range)); } -static inline int mls_context_cpy(struct context *dst, struct context *src) +static inline int mls_context_cpy(struct context *dst, const struct context *src) { int rc; @@ -58,7 +58,7 @@ out: /* * Sets both levels in the MLS range of 'dst' to the low level of 'src'. */ -static inline int mls_context_cpy_low(struct context *dst, struct context *src) +static inline int mls_context_cpy_low(struct context *dst, const struct context *src) { int rc; @@ -78,7 +78,7 @@ out: /* * Sets both levels in the MLS range of 'dst' to the high level of 'src'. */ -static inline int mls_context_cpy_high(struct context *dst, struct context *src) +static inline int mls_context_cpy_high(struct context *dst, const struct context *src) { int rc; @@ -97,9 +97,10 @@ out: static inline int mls_context_glblub(struct context *dst, - struct context *c1, struct context *c2) + const struct context *c1, const struct context *c2) { - struct mls_range *dr = &dst->range, *r1 = &c1->range, *r2 = &c2->range; + struct mls_range *dr = &dst->range; + const struct mls_range *r1 = &c1->range, *r2 = &c2->range; int rc = 0; if (r1->level[1].sens < r2->level[0].sens || @@ -127,7 +128,7 @@ out: return rc; } -static inline int mls_context_cmp(struct context *c1, struct context *c2) +static inline int mls_context_cmp(const struct context *c1, const struct context *c2) { return ((c1->range.level[0].sens == c2->range.level[0].sens) && ebitmap_cmp(&c1->range.level[0].cat, &c2->range.level[0].cat) && @@ -147,7 +148,7 @@ static inline void context_init(struct context *c) memset(c, 0, sizeof(*c)); } -static inline int context_cpy(struct context *dst, struct context *src) +static inline int context_cpy(struct context *dst, const struct context *src) { int rc; @@ -180,7 +181,7 @@ static inline void context_destroy(struct context *c) mls_context_destroy(c); } -static inline int context_cmp(struct context *c1, struct context *c2) +static inline int context_cmp(const struct context *c1, const struct context *c2) { if (c1->len && c2->len) return (c1->len == c2->len && !strcmp(c1->str, c2->str)); diff --git a/security/selinux/ss/ebitmap.c b/security/selinux/ss/ebitmap.c index abde349c8321728306b9e677a6d569e6fca4e7e9..d31b87be9a1ed2ec48c7922689f8962e5f05ecd4 100644 --- a/security/selinux/ss/ebitmap.c +++ b/security/selinux/ss/ebitmap.c @@ -28,9 +28,9 @@ static struct kmem_cache *ebitmap_node_cachep __ro_after_init; -int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2) +int ebitmap_cmp(const struct ebitmap *e1, const struct ebitmap *e2) { - struct ebitmap_node *n1, *n2; + const struct ebitmap_node *n1, *n2; if (e1->highbit != e2->highbit) return 0; @@ -50,9 +50,10 @@ int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2) return 1; } -int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src) +int ebitmap_cpy(struct ebitmap *dst, const struct ebitmap *src) { - struct ebitmap_node *n, *new, *prev; + struct ebitmap_node *new, *prev; + const struct ebitmap_node *n; ebitmap_init(dst); n = src->node; @@ -78,7 +79,7 @@ int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src) return 0; } -int ebitmap_and(struct ebitmap *dst, struct ebitmap *e1, struct ebitmap *e2) +int ebitmap_and(struct ebitmap *dst, const struct ebitmap *e1, const struct ebitmap *e2) { struct ebitmap_node *n; int bit, rc; @@ -217,9 +218,9 @@ netlbl_import_failure: * if last_e2bit is non-zero, the highest set bit in e2 cannot exceed * last_e2bit. */ -int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit) +int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2, u32 last_e2bit) { - struct ebitmap_node *n1, *n2; + const struct ebitmap_node *n1, *n2; int i; if (e1->highbit < e2->highbit) @@ -258,9 +259,9 @@ int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit) return 1; } -int ebitmap_get_bit(struct ebitmap *e, unsigned long bit) +int ebitmap_get_bit(const struct ebitmap *e, unsigned long bit) { - struct ebitmap_node *n; + const struct ebitmap_node *n; if (e->highbit < bit) return 0; @@ -467,7 +468,7 @@ bad: goto out; } -int ebitmap_write(struct ebitmap *e, void *fp) +int ebitmap_write(const struct ebitmap *e, void *fp) { struct ebitmap_node *n; u32 count; diff --git a/security/selinux/ss/ebitmap.h b/security/selinux/ss/ebitmap.h index 58eb822f11eefa6ac00cb8d0558d9aa8a772dce0..e5b57dc3fc53136585a5f0eb6e2b743c7090c11e 100644 --- a/security/selinux/ss/ebitmap.h +++ b/security/selinux/ss/ebitmap.h @@ -44,7 +44,7 @@ struct ebitmap { #define ebitmap_length(e) ((e)->highbit) -static inline unsigned int ebitmap_start_positive(struct ebitmap *e, +static inline unsigned int ebitmap_start_positive(const struct ebitmap *e, struct ebitmap_node **n) { unsigned int ofs; @@ -62,7 +62,7 @@ static inline void ebitmap_init(struct ebitmap *e) memset(e, 0, sizeof(*e)); } -static inline unsigned int ebitmap_next_positive(struct ebitmap *e, +static inline unsigned int ebitmap_next_positive(const struct ebitmap *e, struct ebitmap_node **n, unsigned int bit) { @@ -85,7 +85,7 @@ static inline unsigned int ebitmap_next_positive(struct ebitmap *e, #define EBITMAP_NODE_OFFSET(node, bit) \ (((bit) - (node)->startbit) % EBITMAP_UNIT_SIZE) -static inline int ebitmap_node_get_bit(struct ebitmap_node *n, +static inline int ebitmap_node_get_bit(const struct ebitmap_node *n, unsigned int bit) { unsigned int index = EBITMAP_NODE_INDEX(n, bit); @@ -122,15 +122,15 @@ static inline void ebitmap_node_clr_bit(struct ebitmap_node *n, (bit) < ebitmap_length(e); \ (bit) = ebitmap_next_positive(e, &(n), bit)) \ -int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2); -int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src); -int ebitmap_and(struct ebitmap *dst, struct ebitmap *e1, struct ebitmap *e2); -int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit); -int ebitmap_get_bit(struct ebitmap *e, unsigned long bit); +int ebitmap_cmp(const struct ebitmap *e1, const struct ebitmap *e2); +int ebitmap_cpy(struct ebitmap *dst, const struct ebitmap *src); +int ebitmap_and(struct ebitmap *dst, const struct ebitmap *e1, const struct ebitmap *e2); +int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2, u32 last_e2bit); +int ebitmap_get_bit(const struct ebitmap *e, unsigned long bit); int ebitmap_set_bit(struct ebitmap *e, unsigned long bit, int value); void ebitmap_destroy(struct ebitmap *e); int ebitmap_read(struct ebitmap *e, void *fp); -int ebitmap_write(struct ebitmap *e, void *fp); +int ebitmap_write(const struct ebitmap *e, void *fp); u32 ebitmap_hash(const struct ebitmap *e, u32 hash); #ifdef CONFIG_NETLABEL diff --git a/security/selinux/ss/mls_types.h b/security/selinux/ss/mls_types.h index 068e0d7809db93cce811462635a4e5bd0caa2873..7d48d5e52233b53599053387109297ff71e85892 100644 --- a/security/selinux/ss/mls_types.h +++ b/security/selinux/ss/mls_types.h @@ -27,13 +27,13 @@ struct mls_range { struct mls_level level[2]; /* low == level[0], high == level[1] */ }; -static inline int mls_level_eq(struct mls_level *l1, struct mls_level *l2) +static inline int mls_level_eq(const struct mls_level *l1, const struct mls_level *l2) { return ((l1->sens == l2->sens) && ebitmap_cmp(&l1->cat, &l2->cat)); } -static inline int mls_level_dom(struct mls_level *l1, struct mls_level *l2) +static inline int mls_level_dom(const struct mls_level *l1, const struct mls_level *l2) { return ((l1->sens >= l2->sens) && ebitmap_contains(&l1->cat, &l2->cat, 0)); diff --git a/security/smack/smack.h b/security/smack/smack.h index fc837dcebf96e05ac38dfbb09678409901b5f398..e2239be7bd60a02836e6a31bbc7bbc888fe33aa8 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -180,15 +180,6 @@ struct smack_known_list_elem { struct smack_known *smk_label; }; -/* Super block security struct flags for mount options */ -#define FSDEFAULT_MNT 0x01 -#define FSFLOOR_MNT 0x02 -#define FSHAT_MNT 0x04 -#define FSROOT_MNT 0x08 -#define FSTRANS_MNT 0x10 - -#define NUM_SMK_MNT_OPTS 5 - enum { Opt_error = -1, Opt_fsdefault = 0, diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 001831458fa2c895283c1b01e4ec77b58047e827..b6306d71c90889a13f687144941583ccb5f33807 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -42,6 +42,7 @@ #include <linux/fs_context.h> #include <linux/fs_parser.h> #include <linux/watch_queue.h> +#include <linux/io_uring.h> #include "smack.h" #define TRANS_TRUE "TRUE" @@ -496,13 +497,11 @@ static int smack_ptrace_access_check(struct task_struct *ctp, unsigned int mode) */ static int smack_ptrace_traceme(struct task_struct *ptp) { - int rc; struct smack_known *skp; skp = smk_of_task(smack_cred(current_cred())); - rc = smk_ptrace_rule_check(ptp, skp, PTRACE_MODE_ATTACH, __func__); - return rc; + return smk_ptrace_rule_check(ptp, skp, PTRACE_MODE_ATTACH, __func__); } /** @@ -2278,6 +2277,21 @@ static void smack_sk_free_security(struct sock *sk) kfree(sk->sk_security); } +/** + * smack_sk_clone_security - Copy security context + * @sk: the old socket + * @newsk: the new socket + * + * Copy the security context of the old socket pointer to the cloned + */ +static void smack_sk_clone_security(const struct sock *sk, struct sock *newsk) +{ + struct socket_smack *ssp_old = sk->sk_security; + struct socket_smack *ssp_new = newsk->sk_security; + + *ssp_new = *ssp_old; +} + /** * smack_ipv4host_label - check host based restrictions * @sip: the object end @@ -3479,7 +3493,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode) * * Returns the length of the smack label or an error code */ -static int smack_getprocattr(struct task_struct *p, char *name, char **value) +static int smack_getprocattr(struct task_struct *p, const char *name, char **value) { struct smack_known *skp = smk_of_task_struct_obj(p); char *cp; @@ -4732,6 +4746,36 @@ static int smack_uring_sqpoll(void) return -EPERM; } +/** + * smack_uring_cmd - check on file operations for io_uring + * @ioucmd: the command in question + * + * Make a best guess about whether a io_uring "command" should + * be allowed. Use the same logic used for determining if the + * file could be opened for read in the absence of better criteria. + */ +static int smack_uring_cmd(struct io_uring_cmd *ioucmd) +{ + struct file *file = ioucmd->file; + struct smk_audit_info ad; + struct task_smack *tsp; + struct inode *inode; + int rc; + + if (!file) + return -EINVAL; + + tsp = smack_cred(file->f_cred); + inode = file_inode(file); + + smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_PATH); + smk_ad_setfield_u_fs_path(&ad, file->f_path); + rc = smk_tskacc(tsp, smk_of_inode(inode), MAY_READ, &ad); + rc = smk_bu_credfile(file->f_cred, file, MAY_READ, rc); + + return rc; +} + #endif /* CONFIG_IO_URING */ struct lsm_blob_sizes smack_blob_sizes __lsm_ro_after_init = { @@ -4851,6 +4895,7 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(socket_getpeersec_dgram, smack_socket_getpeersec_dgram), LSM_HOOK_INIT(sk_alloc_security, smack_sk_alloc_security), LSM_HOOK_INIT(sk_free_security, smack_sk_free_security), + LSM_HOOK_INIT(sk_clone_security, smack_sk_clone_security), LSM_HOOK_INIT(sock_graft, smack_sock_graft), LSM_HOOK_INIT(inet_conn_request, smack_inet_conn_request), LSM_HOOK_INIT(inet_csk_clone, smack_inet_csk_clone), @@ -4889,6 +4934,7 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { #ifdef CONFIG_IO_URING LSM_HOOK_INIT(uring_override_creds, smack_uring_override_creds), LSM_HOOK_INIT(uring_sqpoll, smack_uring_sqpoll), + LSM_HOOK_INIT(uring_cmd, smack_uring_cmd), #endif }; diff --git a/security/tomoyo/audit.c b/security/tomoyo/audit.c index 023bedd9dfa30e448847da3f9d3a80decfc8b087..7cf8fdbb29bfd8642d437aee1b28b730d8a62654 100644 --- a/security/tomoyo/audit.c +++ b/security/tomoyo/audit.c @@ -423,7 +423,7 @@ void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf((char *) &len, 1, fmt, args) + 1; + len = vsnprintf(NULL, 0, fmt, args) + 1; va_end(args); va_start(args, fmt); tomoyo_write_log2(r, len, fmt, args); diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index ff17abc96e5c1f330273518096a759f18083f02d..f4cd9b58b20547ec29bdfef8a77421b0cb5b596f 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -2057,7 +2057,7 @@ int tomoyo_supervisor(struct tomoyo_request_info *r, const char *fmt, ...) bool quota_exceeded = false; va_start(args, fmt); - len = vsnprintf((char *) &len, 1, fmt, args) + 1; + len = vsnprintf(NULL, 0, fmt, args) + 1; va_end(args); /* Write /sys/kernel/security/tomoyo/audit. */ va_start(args, fmt); diff --git a/security/tomoyo/file.c b/security/tomoyo/file.c index 1e6077568fdeccddcc3a7242d325f0827aa537ee..8f3b90b6e03d24d3ba8a258a4958a09735dc5b13 100644 --- a/security/tomoyo/file.c +++ b/security/tomoyo/file.c @@ -717,7 +717,7 @@ int tomoyo_path_number_perm(const u8 type, const struct path *path, int idx; if (tomoyo_init_request_info(&r, NULL, tomoyo_pn2mac[type]) - == TOMOYO_CONFIG_DISABLED || !path->dentry) + == TOMOYO_CONFIG_DISABLED) return 0; idx = tomoyo_read_lock(); if (!tomoyo_get_realpath(&buf, path)) diff --git a/security/tomoyo/realpath.c b/security/tomoyo/realpath.c index df479898041657a618a5bd86f2ed579ff82adda7..1c483ee7f93d8219ab312b4a27547bba539706f1 100644 --- a/security/tomoyo/realpath.c +++ b/security/tomoyo/realpath.c @@ -240,11 +240,8 @@ char *tomoyo_realpath_from_path(const struct path *path) char *name = NULL; unsigned int buf_len = PAGE_SIZE / 2; struct dentry *dentry = path->dentry; - struct super_block *sb; + struct super_block *sb = dentry->d_sb; - if (!dentry) - return NULL; - sb = dentry->d_sb; while (1) { char *pos; struct inode *inode; @@ -264,10 +261,8 @@ char *tomoyo_realpath_from_path(const struct path *path) inode = d_backing_inode(sb->s_root); /* * Get local name for filesystems without rename() operation - * or dentry without vfsmount. */ - if (!path->mnt || - (!inode->i_op->rename && + if ((!inode->i_op->rename && !(sb->s_type->fs_flags & FS_REQUIRES_DEV))) pos = tomoyo_get_local_path(path->dentry, buf, buf_len - 1); diff --git a/sound/aoa/codecs/onyx.c b/sound/aoa/codecs/onyx.c index 1abee841cc451f89d0936b46678c3bcda87e5860..2d0f904aba00f873959546e921b95500519902f4 100644 --- a/sound/aoa/codecs/onyx.c +++ b/sound/aoa/codecs/onyx.c @@ -1029,7 +1029,7 @@ static int onyx_i2c_probe(struct i2c_client *client, return -ENODEV; } -static int onyx_i2c_remove(struct i2c_client *client) +static void onyx_i2c_remove(struct i2c_client *client) { struct onyx *onyx = i2c_get_clientdata(client); @@ -1037,7 +1037,6 @@ static int onyx_i2c_remove(struct i2c_client *client) of_node_put(onyx->codec.node); kfree(onyx->codec_info); kfree(onyx); - return 0; } static const struct i2c_device_id onyx_i2c_id[] = { diff --git a/sound/aoa/codecs/tas.c b/sound/aoa/codecs/tas.c index ab19a37e2a68e69d02837f9b692dd47084f10b9a..ab89475b7715ffbdfd5070a19b0460ea22df3a5c 100644 --- a/sound/aoa/codecs/tas.c +++ b/sound/aoa/codecs/tas.c @@ -912,7 +912,7 @@ static int tas_i2c_probe(struct i2c_client *client, return -EINVAL; } -static int tas_i2c_remove(struct i2c_client *client) +static void tas_i2c_remove(struct i2c_client *client) { struct tas *tas = i2c_get_clientdata(client); u8 tmp = TAS_ACR_ANALOG_PDOWN; @@ -925,7 +925,6 @@ static int tas_i2c_remove(struct i2c_client *client) mutex_destroy(&tas->mtx); kfree(tas); - return 0; } static const struct i2c_device_id tas_i2c_id[] = { diff --git a/sound/core/control.c b/sound/core/control.c index f3e893715369f0c759bca83e1184cff63e075398..a7271927d875f3b55d9f4885195f0e0549b16078 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -385,14 +385,14 @@ static bool elem_id_matches(const struct snd_kcontrol *kctl, #define MULTIPLIER 37 static unsigned long get_ctl_id_hash(const struct snd_ctl_elem_id *id) { + int i; unsigned long h; - const unsigned char *p; h = id->iface; h = MULTIPLIER * h + id->device; h = MULTIPLIER * h + id->subdevice; - for (p = id->name; *p; p++) - h = MULTIPLIER * h + *p; + for (i = 0; i < SNDRV_CTL_ELEM_ID_NAME_MAXLEN && id->name[i]; i++) + h = MULTIPLIER * h + id->name[i]; h = MULTIPLIER * h + id->index; h &= LONG_MAX; return h; diff --git a/sound/core/init.c b/sound/core/init.c index 193dae361fac39d7d9f3a6a1cd413944eb0e6a2a..5377f94eb2111f844e1c439926f1a27bfb728fd4 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -178,10 +178,8 @@ int snd_card_new(struct device *parent, int idx, const char *xid, return -ENOMEM; err = snd_card_init(card, parent, idx, xid, module, extra_size); - if (err < 0) { - kfree(card); - return err; - } + if (err < 0) + return err; /* card is freed by error handler */ *card_ret = card; return 0; @@ -233,7 +231,7 @@ int snd_devm_card_new(struct device *parent, int idx, const char *xid, card->managed = true; err = snd_card_init(card, parent, idx, xid, module, extra_size); if (err < 0) { - devres_free(card); + devres_free(card); /* in managed mode, we need to free manually */ return err; } @@ -297,6 +295,8 @@ static int snd_card_init(struct snd_card *card, struct device *parent, mutex_unlock(&snd_card_mutex); dev_err(parent, "cannot find the slot for index %d (range 0-%i), error: %d\n", idx, snd_ecards_limit - 1, err); + if (!card->managed) + kfree(card); /* manually free here, as no destructor called */ return err; } set_bit(idx, snd_cards_lock); /* lock it */ diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c index d3885cb02270e005ba74b06a82f440c85216e028..03cffe7713667783f08039d474766313e36181b1 100644 --- a/sound/core/memalloc.c +++ b/sound/core/memalloc.c @@ -18,17 +18,17 @@ #include <sound/memalloc.h> #include "memalloc_local.h" +#define DEFAULT_GFP \ + (GFP_KERNEL | \ + __GFP_COMP | /* compound page lets parts be mapped */ \ + __GFP_RETRY_MAYFAIL | /* don't trigger OOM-killer */ \ + __GFP_NOWARN) /* no stack trace print - this call is non-critical */ + static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab); -/* a cast to gfp flag from the dev pointer; for CONTINUOUS and VMALLOC types */ -static inline gfp_t snd_mem_get_gfp_flags(const struct snd_dma_buffer *dmab, - gfp_t default_gfp) -{ - if (!dmab->dev.dev) - return default_gfp; - else - return (__force gfp_t)(unsigned long)dmab->dev.dev; -} +#ifdef CONFIG_SND_DMA_SGBUF +static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size); +#endif static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size) { @@ -277,19 +277,54 @@ EXPORT_SYMBOL(snd_sgbuf_get_chunk_size); /* * Continuous pages allocator */ -static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size) +static void *do_alloc_pages(struct device *dev, size_t size, dma_addr_t *addr, + bool wc) { - gfp_t gfp = snd_mem_get_gfp_flags(dmab, GFP_KERNEL); - void *p = alloc_pages_exact(size, gfp); + void *p; + gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; - if (p) - dmab->addr = page_to_phys(virt_to_page(p)); + again: + p = alloc_pages_exact(size, gfp); + if (!p) + return NULL; + *addr = page_to_phys(virt_to_page(p)); + if (!dev) + return p; + if ((*addr + size - 1) & ~dev->coherent_dma_mask) { + if (IS_ENABLED(CONFIG_ZONE_DMA32) && !(gfp & GFP_DMA32)) { + gfp |= GFP_DMA32; + goto again; + } + if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { + gfp = (gfp & ~GFP_DMA32) | GFP_DMA; + goto again; + } + } +#ifdef CONFIG_X86 + if (wc) + set_memory_wc((unsigned long)(p), size >> PAGE_SHIFT); +#endif return p; } +static void do_free_pages(void *p, size_t size, bool wc) +{ +#ifdef CONFIG_X86 + if (wc) + set_memory_wb((unsigned long)(p), size >> PAGE_SHIFT); +#endif + free_pages_exact(p, size); +} + + +static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size) +{ + return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, false); +} + static void snd_dma_continuous_free(struct snd_dma_buffer *dmab) { - free_pages_exact(dmab->area, dmab->bytes); + do_free_pages(dmab->area, dmab->bytes, false); } static int snd_dma_continuous_mmap(struct snd_dma_buffer *dmab, @@ -312,9 +347,7 @@ static const struct snd_malloc_ops snd_dma_continuous_ops = { */ static void *snd_dma_vmalloc_alloc(struct snd_dma_buffer *dmab, size_t size) { - gfp_t gfp = snd_mem_get_gfp_flags(dmab, GFP_KERNEL | __GFP_HIGHMEM); - - return __vmalloc(size, gfp); + return vmalloc(size); } static void snd_dma_vmalloc_free(struct snd_dma_buffer *dmab) @@ -428,12 +461,6 @@ static const struct snd_malloc_ops snd_dma_iram_ops = { }; #endif /* CONFIG_GENERIC_ALLOCATOR */ -#define DEFAULT_GFP \ - (GFP_KERNEL | \ - __GFP_COMP | /* compound page lets parts be mapped */ \ - __GFP_NORETRY | /* don't trigger OOM-killer */ \ - __GFP_NOWARN) /* no stack trace print - this call is non-critical */ - /* * Coherent device pages allocator */ @@ -463,6 +490,25 @@ static const struct snd_malloc_ops snd_dma_dev_ops = { /* * Write-combined pages */ +/* x86-specific allocations */ +#ifdef CONFIG_SND_DMA_SGBUF +static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size) +{ + return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, true); +} + +static void snd_dma_wc_free(struct snd_dma_buffer *dmab) +{ + do_free_pages(dmab->area, dmab->bytes, true); +} + +static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab, + struct vm_area_struct *area) +{ + area->vm_page_prot = pgprot_writecombine(area->vm_page_prot); + return snd_dma_continuous_mmap(dmab, area); +} +#else static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size) { return dma_alloc_wc(dmab->dev.dev, size, &dmab->addr, DEFAULT_GFP); @@ -479,6 +525,7 @@ static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab, return dma_mmap_wc(dmab->dev.dev, area, dmab->area, dmab->addr, dmab->bytes); } +#endif /* CONFIG_SND_DMA_SGBUF */ static const struct snd_malloc_ops snd_dma_wc_ops = { .alloc = snd_dma_wc_alloc, @@ -486,10 +533,6 @@ static const struct snd_malloc_ops snd_dma_wc_ops = { .mmap = snd_dma_wc_mmap, }; -#ifdef CONFIG_SND_DMA_SGBUF -static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size); -#endif - /* * Non-contiguous pages allocator */ @@ -515,10 +558,13 @@ static void *snd_dma_noncontig_alloc(struct snd_dma_buffer *dmab, size_t size) dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, sg_dma_address(sgt->sgl)); p = dma_vmap_noncontiguous(dmab->dev.dev, size, sgt); - if (p) + if (p) { dmab->private_data = sgt; - else + /* store the first page address for convenience */ + dmab->addr = snd_sgbuf_get_addr(dmab, 0); + } else { dma_free_noncontiguous(dmab->dev.dev, size, sgt, dmab->dev.dir); + } return p; } @@ -679,14 +725,11 @@ struct snd_dma_sg_fallback { static void __snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab, struct snd_dma_sg_fallback *sgbuf) { + bool wc = dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK; size_t i; - if (sgbuf->count && dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK) - set_pages_array_wb(sgbuf->pages, sgbuf->count); for (i = 0; i < sgbuf->count && sgbuf->pages[i]; i++) - dma_free_coherent(dmab->dev.dev, PAGE_SIZE, - page_address(sgbuf->pages[i]), - sgbuf->addrs[i]); + do_free_pages(page_address(sgbuf->pages[i]), PAGE_SIZE, wc); kvfree(sgbuf->pages); kvfree(sgbuf->addrs); kfree(sgbuf); @@ -698,6 +741,7 @@ static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size) struct page **pages; size_t i, count; void *p; + bool wc = dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK; sgbuf = kzalloc(sizeof(*sgbuf), GFP_KERNEL); if (!sgbuf) @@ -712,19 +756,18 @@ static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size) goto error; for (i = 0; i < count; sgbuf->count++, i++) { - p = dma_alloc_coherent(dmab->dev.dev, PAGE_SIZE, - &sgbuf->addrs[i], DEFAULT_GFP); + p = do_alloc_pages(dmab->dev.dev, PAGE_SIZE, &sgbuf->addrs[i], wc); if (!p) goto error; sgbuf->pages[i] = virt_to_page(p); } - if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK) - set_pages_array_wc(pages, count); p = vmap(pages, count, VM_MAP, PAGE_KERNEL); if (!p) goto error; dmab->private_data = sgbuf; + /* store the first page address for convenience */ + dmab->addr = snd_sgbuf_get_addr(dmab, 0); return p; error: diff --git a/sound/core/memalloc_local.h b/sound/core/memalloc_local.h index a6f3a87194daa0619101a60cc965a99795aa18cd..8b19f3a68a4ba82dd714a01ead1c78b436d72229 100644 --- a/sound/core/memalloc_local.h +++ b/sound/core/memalloc_local.h @@ -13,8 +13,4 @@ struct snd_malloc_ops { void (*sync)(struct snd_dma_buffer *dmab, enum snd_dma_sync_mode mode); }; -#ifdef CONFIG_SND_DMA_SGBUF -extern const struct snd_malloc_ops snd_dma_sg_ops; -#endif - #endif /* __MEMALLOC_LOCAL_H */ diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index 90c3a367d7de9acde49b816bbd4853e55eba2497..ac2efeb63a39637d0399c3273491bb3924c9ddde 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -1237,12 +1237,12 @@ snd_pcm_sframes_t snd_pcm_oss_write3(struct snd_pcm_substream *substream, const struct snd_pcm_runtime *runtime = substream->runtime; int ret; while (1) { - if (runtime->status->state == SNDRV_PCM_STATE_XRUN || - runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) { + if (runtime->state == SNDRV_PCM_STATE_XRUN || + runtime->state == SNDRV_PCM_STATE_SUSPENDED) { #ifdef OSS_DEBUG pcm_dbg(substream->pcm, "pcm_oss: write: recovering from %s\n", - runtime->status->state == SNDRV_PCM_STATE_XRUN ? + runtime->state == SNDRV_PCM_STATE_XRUN ? "XRUN" : "SUSPEND"); #endif ret = snd_pcm_oss_prepare(substream); @@ -1257,7 +1257,7 @@ snd_pcm_sframes_t snd_pcm_oss_write3(struct snd_pcm_substream *substream, const break; /* test, if we can't store new data, because the stream */ /* has not been started */ - if (runtime->status->state == SNDRV_PCM_STATE_PREPARED) + if (runtime->state == SNDRV_PCM_STATE_PREPARED) return -EAGAIN; } return ret; @@ -1269,18 +1269,18 @@ snd_pcm_sframes_t snd_pcm_oss_read3(struct snd_pcm_substream *substream, char *p snd_pcm_sframes_t delay; int ret; while (1) { - if (runtime->status->state == SNDRV_PCM_STATE_XRUN || - runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) { + if (runtime->state == SNDRV_PCM_STATE_XRUN || + runtime->state == SNDRV_PCM_STATE_SUSPENDED) { #ifdef OSS_DEBUG pcm_dbg(substream->pcm, "pcm_oss: read: recovering from %s\n", - runtime->status->state == SNDRV_PCM_STATE_XRUN ? + runtime->state == SNDRV_PCM_STATE_XRUN ? "XRUN" : "SUSPEND"); #endif ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DRAIN, NULL); if (ret < 0) break; - } else if (runtime->status->state == SNDRV_PCM_STATE_SETUP) { + } else if (runtime->state == SNDRV_PCM_STATE_SETUP) { ret = snd_pcm_oss_prepare(substream); if (ret < 0) break; @@ -1293,7 +1293,7 @@ snd_pcm_sframes_t snd_pcm_oss_read3(struct snd_pcm_substream *substream, char *p frames, in_kernel); mutex_lock(&runtime->oss.params_lock); if (ret == -EPIPE) { - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) { + if (runtime->state == SNDRV_PCM_STATE_DRAINING) { ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DROP, NULL); if (ret < 0) break; @@ -1312,12 +1312,12 @@ snd_pcm_sframes_t snd_pcm_oss_writev3(struct snd_pcm_substream *substream, void struct snd_pcm_runtime *runtime = substream->runtime; int ret; while (1) { - if (runtime->status->state == SNDRV_PCM_STATE_XRUN || - runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) { + if (runtime->state == SNDRV_PCM_STATE_XRUN || + runtime->state == SNDRV_PCM_STATE_SUSPENDED) { #ifdef OSS_DEBUG pcm_dbg(substream->pcm, "pcm_oss: writev: recovering from %s\n", - runtime->status->state == SNDRV_PCM_STATE_XRUN ? + runtime->state == SNDRV_PCM_STATE_XRUN ? "XRUN" : "SUSPEND"); #endif ret = snd_pcm_oss_prepare(substream); @@ -1330,7 +1330,7 @@ snd_pcm_sframes_t snd_pcm_oss_writev3(struct snd_pcm_substream *substream, void /* test, if we can't store new data, because the stream */ /* has not been started */ - if (runtime->status->state == SNDRV_PCM_STATE_PREPARED) + if (runtime->state == SNDRV_PCM_STATE_PREPARED) return -EAGAIN; } return ret; @@ -1341,18 +1341,18 @@ snd_pcm_sframes_t snd_pcm_oss_readv3(struct snd_pcm_substream *substream, void * struct snd_pcm_runtime *runtime = substream->runtime; int ret; while (1) { - if (runtime->status->state == SNDRV_PCM_STATE_XRUN || - runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) { + if (runtime->state == SNDRV_PCM_STATE_XRUN || + runtime->state == SNDRV_PCM_STATE_SUSPENDED) { #ifdef OSS_DEBUG pcm_dbg(substream->pcm, "pcm_oss: readv: recovering from %s\n", - runtime->status->state == SNDRV_PCM_STATE_XRUN ? + runtime->state == SNDRV_PCM_STATE_XRUN ? "XRUN" : "SUSPEND"); #endif ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DRAIN, NULL); if (ret < 0) break; - } else if (runtime->status->state == SNDRV_PCM_STATE_SETUP) { + } else if (runtime->state == SNDRV_PCM_STATE_SETUP) { ret = snd_pcm_oss_prepare(substream); if (ret < 0) break; @@ -1635,7 +1635,7 @@ static int snd_pcm_oss_sync1(struct snd_pcm_substream *substream, size_t size) result = 0; set_current_state(TASK_INTERRUPTIBLE); snd_pcm_stream_lock_irq(substream); - state = runtime->status->state; + state = runtime->state; snd_pcm_stream_unlock_irq(substream); if (state != SNDRV_PCM_STATE_RUNNING) { set_current_state(TASK_RUNNING); @@ -1672,14 +1672,14 @@ static int snd_pcm_oss_sync(struct snd_pcm_oss_file *pcm_oss_file) runtime = substream->runtime; if (atomic_read(&substream->mmap_count)) goto __direct; - err = snd_pcm_oss_make_ready(substream); - if (err < 0) - return err; atomic_inc(&runtime->oss.rw_ref); if (mutex_lock_interruptible(&runtime->oss.params_lock)) { atomic_dec(&runtime->oss.rw_ref); return -ERESTARTSYS; } + err = snd_pcm_oss_make_ready_locked(substream); + if (err < 0) + goto unlock; format = snd_pcm_oss_format_from(runtime->oss.format); width = snd_pcm_format_physical_width(format); if (runtime->oss.buffer_used > 0) { @@ -2854,8 +2854,8 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait) struct snd_pcm_runtime *runtime = psubstream->runtime; poll_wait(file, &runtime->sleep, wait); snd_pcm_stream_lock_irq(psubstream); - if (runtime->status->state != SNDRV_PCM_STATE_DRAINING && - (runtime->status->state != SNDRV_PCM_STATE_RUNNING || + if (runtime->state != SNDRV_PCM_STATE_DRAINING && + (runtime->state != SNDRV_PCM_STATE_RUNNING || snd_pcm_oss_playback_ready(psubstream))) mask |= EPOLLOUT | EPOLLWRNORM; snd_pcm_stream_unlock_irq(psubstream); @@ -2865,7 +2865,7 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait) snd_pcm_state_t ostate; poll_wait(file, &runtime->sleep, wait); snd_pcm_stream_lock_irq(csubstream); - ostate = runtime->status->state; + ostate = runtime->state; if (ostate != SNDRV_PCM_STATE_RUNNING || snd_pcm_oss_capture_ready(csubstream)) mask |= EPOLLIN | EPOLLRDNORM; diff --git a/sound/core/pcm.c b/sound/core/pcm.c index 82925709fa12ad38d19d3b7a1d47e96b0616c8cb..9d95e37311230ddd11b76937106d1dffc47d1a9f 100644 --- a/sound/core/pcm.c +++ b/sound/core/pcm.c @@ -387,7 +387,7 @@ static void snd_pcm_substream_proc_hw_params_read(struct snd_info_entry *entry, snd_iprintf(buffer, "closed\n"); goto unlock; } - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (runtime->state == SNDRV_PCM_STATE_OPEN) { snd_iprintf(buffer, "no setup\n"); goto unlock; } @@ -424,7 +424,7 @@ static void snd_pcm_substream_proc_sw_params_read(struct snd_info_entry *entry, snd_iprintf(buffer, "closed\n"); goto unlock; } - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (runtime->state == SNDRV_PCM_STATE_OPEN) { snd_iprintf(buffer, "no setup\n"); goto unlock; } @@ -970,7 +970,7 @@ int snd_pcm_attach_substream(struct snd_pcm *pcm, int stream, init_waitqueue_head(&runtime->sleep); init_waitqueue_head(&runtime->tsleep); - runtime->status->state = SNDRV_PCM_STATE_OPEN; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_OPEN); mutex_init(&runtime->buffer_mutex); atomic_set(&runtime->buffer_accessing, 0); @@ -1112,7 +1112,8 @@ static int snd_pcm_dev_disconnect(struct snd_device *device) if (snd_pcm_running(substream)) snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); /* to be sure, set the state unconditionally */ - substream->runtime->status->state = SNDRV_PCM_STATE_DISCONNECTED; + __snd_pcm_set_state(substream->runtime, + SNDRV_PCM_STATE_DISCONNECTED); wake_up(&substream->runtime->sleep); wake_up(&substream->runtime->tsleep); } diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c index 917c5b4f19d7875c029a324b14e30d84f547a18b..42c2ada8e888737e97d59ab374b30aef29da6410 100644 --- a/sound/core/pcm_compat.c +++ b/sound/core/pcm_compat.c @@ -295,7 +295,7 @@ static int snd_pcm_ioctl_xferi_compat(struct snd_pcm_substream *substream, return -ENOTTY; if (substream->stream != dir) return -EINVAL; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (get_user(buf, &data32->buf) || @@ -341,7 +341,7 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream, return -ENOTTY; if (substream->stream != dir) return -EINVAL; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; ch = substream->runtime->channels; diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c index 5b2ca028f5aabdf2d6a915733cfa75f1da096299..494ec0c207fad10ec7e9d68a01565f951524ee39 100644 --- a/sound/core/pcm_dmaengine.c +++ b/sound/core/pcm_dmaengine.c @@ -133,12 +133,14 @@ EXPORT_SYMBOL_GPL(snd_dmaengine_pcm_set_config_from_dai_data); static void dmaengine_pcm_dma_complete(void *arg) { + unsigned int new_pos; struct snd_pcm_substream *substream = arg; struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); - prtd->pos += snd_pcm_lib_period_bytes(substream); - if (prtd->pos >= snd_pcm_lib_buffer_bytes(substream)) - prtd->pos = 0; + new_pos = prtd->pos + snd_pcm_lib_period_bytes(substream); + if (new_pos >= snd_pcm_lib_buffer_bytes(substream)) + new_pos = 0; + prtd->pos = new_pos; snd_pcm_period_elapsed(substream); } diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index 40751e5aff09f13f665e0fed717210a2bdad1086..8b6aeb8a78f7d3ac2819cda7b2781e1ec5035067 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -186,7 +186,7 @@ int snd_pcm_update_state(struct snd_pcm_substream *substream, avail = snd_pcm_avail(substream); if (avail > runtime->avail_max) runtime->avail_max = avail; - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) { + if (runtime->state == SNDRV_PCM_STATE_DRAINING) { if (avail >= runtime->buffer_size) { snd_pcm_drain_done(substream); return -EPIPE; @@ -1911,7 +1911,7 @@ static int wait_for_avail(struct snd_pcm_substream *substream, snd_pcm_stream_lock_irq(substream); set_current_state(TASK_INTERRUPTIBLE); - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_SUSPENDED: err = -ESTRPIPE; goto _endloop; @@ -2099,14 +2099,14 @@ static int pcm_sanity_check(struct snd_pcm_substream *substream) runtime = substream->runtime; if (snd_BUG_ON(!substream->ops->copy_user && !runtime->dma_area)) return -EINVAL; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; return 0; } static int pcm_accessible_state(struct snd_pcm_runtime *runtime) { - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_PREPARED: case SNDRV_PCM_STATE_RUNNING: case SNDRV_PCM_STATE_PAUSED: @@ -2225,7 +2225,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, goto _end_unlock; runtime->twake = runtime->control->avail_min ? : 1; - if (runtime->status->state == SNDRV_PCM_STATE_RUNNING) + if (runtime->state == SNDRV_PCM_STATE_RUNNING) snd_pcm_update_hw_ptr(substream); /* @@ -2233,7 +2233,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, * thread may start capture */ if (!is_playback && - runtime->status->state == SNDRV_PCM_STATE_PREPARED && + runtime->state == SNDRV_PCM_STATE_PREPARED && size >= runtime->start_threshold) { err = snd_pcm_start(substream); if (err < 0) @@ -2247,7 +2247,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, snd_pcm_uframes_t cont; if (!avail) { if (!is_playback && - runtime->status->state == SNDRV_PCM_STATE_DRAINING) { + runtime->state == SNDRV_PCM_STATE_DRAINING) { snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP); goto _end_unlock; } @@ -2303,7 +2303,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream, xfer += frames; avail -= frames; if (is_playback && - runtime->status->state == SNDRV_PCM_STATE_PREPARED && + runtime->state == SNDRV_PCM_STATE_PREPARED && snd_pcm_playback_hw_avail(runtime) >= (snd_pcm_sframes_t)runtime->start_threshold) { err = snd_pcm_start(substream); if (err < 0) diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index ad0541e9e88808424aee75cdfcdc4f4d4098a02b..33769ca78cc8f9e47fb0348c79049c2170809eed 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -595,8 +595,8 @@ static void snd_pcm_set_state(struct snd_pcm_substream *substream, snd_pcm_state_t state) { snd_pcm_stream_lock_irq(substream); - if (substream->runtime->status->state != SNDRV_PCM_STATE_DISCONNECTED) - substream->runtime->status->state = state; + if (substream->runtime->state != SNDRV_PCM_STATE_DISCONNECTED) + __snd_pcm_set_state(substream->runtime, state); snd_pcm_stream_unlock_irq(substream); } @@ -724,7 +724,7 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream, if (err < 0) return err; snd_pcm_stream_lock_irq(substream); - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_OPEN: case SNDRV_PCM_STATE_SETUP: case SNDRV_PCM_STATE_PREPARED: @@ -889,7 +889,7 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) if (result < 0) return result; snd_pcm_stream_lock_irq(substream); - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_SETUP: case SNDRV_PCM_STATE_PREPARED: if (atomic_read(&substream->mmap_count)) @@ -920,7 +920,7 @@ static int snd_pcm_sw_params(struct snd_pcm_substream *substream, return -ENXIO; runtime = substream->runtime; snd_pcm_stream_lock_irq(substream); - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (runtime->state == SNDRV_PCM_STATE_OPEN) { snd_pcm_stream_unlock_irq(substream); return -EBADFD; } @@ -1013,8 +1013,8 @@ int snd_pcm_status64(struct snd_pcm_substream *substream, } else runtime->audio_tstamp_report.valid = 1; - status->state = runtime->status->state; - status->suspended_state = runtime->status->suspended_state; + status->state = runtime->state; + status->suspended_state = runtime->suspended_state; if (status->state == SNDRV_PCM_STATE_OPEN) goto _end; status->trigger_tstamp_sec = runtime->trigger_tstamp.tv_sec; @@ -1148,7 +1148,7 @@ static int snd_pcm_channel_info(struct snd_pcm_substream *substream, channel = info->channel; runtime = substream->runtime; snd_pcm_stream_lock_irq(substream); - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (runtime->state == SNDRV_PCM_STATE_OPEN) { snd_pcm_stream_unlock_irq(substream); return -EBADFD; } @@ -1411,7 +1411,7 @@ static int snd_pcm_pre_start(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - if (runtime->status->state != SNDRV_PCM_STATE_PREPARED) + if (runtime->state != SNDRV_PCM_STATE_PREPARED) return -EBADFD; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && !snd_pcm_playback_data(substream)) @@ -1444,7 +1444,7 @@ static void snd_pcm_post_start(struct snd_pcm_substream *substream, runtime->hw_ptr_jiffies = jiffies; runtime->hw_ptr_buffer_jiffies = (runtime->buffer_size * HZ) / runtime->rate; - runtime->status->state = state; + __snd_pcm_set_state(runtime, state); if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && runtime->silence_size > 0) snd_pcm_playback_silence(substream, ULONG_MAX); @@ -1485,7 +1485,7 @@ static int snd_pcm_pre_stop(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; runtime->trigger_master = substream; return 0; @@ -1506,9 +1506,9 @@ static void snd_pcm_post_stop(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - if (runtime->status->state != state) { + if (runtime->state != state) { snd_pcm_trigger_tstamp(substream); - runtime->status->state = state; + __snd_pcm_set_state(runtime, state); snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MSTOP); } wake_up(&runtime->sleep); @@ -1584,9 +1584,9 @@ static int snd_pcm_pre_pause(struct snd_pcm_substream *substream, if (!(runtime->info & SNDRV_PCM_INFO_PAUSE)) return -ENOSYS; if (pause_pushed(state)) { - if (runtime->status->state != SNDRV_PCM_STATE_RUNNING) + if (runtime->state != SNDRV_PCM_STATE_RUNNING) return -EBADFD; - } else if (runtime->status->state != SNDRV_PCM_STATE_PAUSED) + } else if (runtime->state != SNDRV_PCM_STATE_PAUSED) return -EBADFD; runtime->trigger_master = substream; return 0; @@ -1628,12 +1628,12 @@ static void snd_pcm_post_pause(struct snd_pcm_substream *substream, struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_trigger_tstamp(substream); if (pause_pushed(state)) { - runtime->status->state = SNDRV_PCM_STATE_PAUSED; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_PAUSED); snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MPAUSE); wake_up(&runtime->sleep); wake_up(&runtime->tsleep); } else { - runtime->status->state = SNDRV_PCM_STATE_RUNNING; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_RUNNING); snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MCONTINUE); } } @@ -1668,7 +1668,7 @@ static int snd_pcm_pre_suspend(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_SUSPENDED: return -EBUSY; /* unresumable PCM state; return -EBUSY for skipping suspend */ @@ -1699,8 +1699,9 @@ static void snd_pcm_post_suspend(struct snd_pcm_substream *substream, { struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_trigger_tstamp(substream); - runtime->status->suspended_state = runtime->status->state; - runtime->status->state = SNDRV_PCM_STATE_SUSPENDED; + runtime->suspended_state = runtime->state; + runtime->status->suspended_state = runtime->suspended_state; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SUSPENDED); snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MSUSPEND); wake_up(&runtime->sleep); wake_up(&runtime->tsleep); @@ -1791,8 +1792,8 @@ static int snd_pcm_do_resume(struct snd_pcm_substream *substream, if (runtime->trigger_master != substream) return 0; /* DMA not running previously? */ - if (runtime->status->suspended_state != SNDRV_PCM_STATE_RUNNING && - (runtime->status->suspended_state != SNDRV_PCM_STATE_DRAINING || + if (runtime->suspended_state != SNDRV_PCM_STATE_RUNNING && + (runtime->suspended_state != SNDRV_PCM_STATE_DRAINING || substream->stream != SNDRV_PCM_STREAM_PLAYBACK)) return 0; return substream->ops->trigger(substream, SNDRV_PCM_TRIGGER_RESUME); @@ -1811,7 +1812,7 @@ static void snd_pcm_post_resume(struct snd_pcm_substream *substream, { struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_trigger_tstamp(substream); - runtime->status->state = runtime->status->suspended_state; + __snd_pcm_set_state(runtime, runtime->suspended_state); snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MRESUME); } @@ -1848,7 +1849,7 @@ static int snd_pcm_xrun(struct snd_pcm_substream *substream) int result; snd_pcm_stream_lock_irq(substream); - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_XRUN: result = 0; /* already there */ break; @@ -1871,7 +1872,7 @@ static int snd_pcm_pre_reset(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_RUNNING: case SNDRV_PCM_STATE_PREPARED: case SNDRV_PCM_STATE_PAUSED: @@ -1933,8 +1934,8 @@ static int snd_pcm_pre_prepare(struct snd_pcm_substream *substream, struct snd_pcm_runtime *runtime = substream->runtime; int f_flags = (__force int)state; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; if (snd_pcm_running(substream)) return -EBUSY; @@ -1985,7 +1986,7 @@ static int snd_pcm_prepare(struct snd_pcm_substream *substream, f_flags = substream->f_flags; snd_pcm_stream_lock_irq(substream); - switch (substream->runtime->status->state) { + switch (substream->runtime->state) { case SNDRV_PCM_STATE_PAUSED: snd_pcm_pause(substream, false); fallthrough; @@ -2009,7 +2010,7 @@ static int snd_pcm_pre_drain_init(struct snd_pcm_substream *substream, snd_pcm_state_t state) { struct snd_pcm_runtime *runtime = substream->runtime; - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_OPEN: case SNDRV_PCM_STATE_DISCONNECTED: case SNDRV_PCM_STATE_SUSPENDED: @@ -2024,28 +2025,28 @@ static int snd_pcm_do_drain_init(struct snd_pcm_substream *substream, { struct snd_pcm_runtime *runtime = substream->runtime; if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_PREPARED: /* start playback stream if possible */ if (! snd_pcm_playback_empty(substream)) { snd_pcm_do_start(substream, SNDRV_PCM_STATE_DRAINING); snd_pcm_post_start(substream, SNDRV_PCM_STATE_DRAINING); } else { - runtime->status->state = SNDRV_PCM_STATE_SETUP; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SETUP); } break; case SNDRV_PCM_STATE_RUNNING: - runtime->status->state = SNDRV_PCM_STATE_DRAINING; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_DRAINING); break; case SNDRV_PCM_STATE_XRUN: - runtime->status->state = SNDRV_PCM_STATE_SETUP; + __snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SETUP); break; default: break; } } else { /* stop running stream */ - if (runtime->status->state == SNDRV_PCM_STATE_RUNNING) { + if (runtime->state == SNDRV_PCM_STATE_RUNNING) { snd_pcm_state_t new_state; new_state = snd_pcm_capture_avail(runtime) > 0 ? @@ -2055,7 +2056,7 @@ static int snd_pcm_do_drain_init(struct snd_pcm_substream *substream, } } - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING && + if (runtime->state == SNDRV_PCM_STATE_DRAINING && runtime->trigger_master == substream && (runtime->hw.info & SNDRV_PCM_INFO_DRAIN_TRIGGER)) return substream->ops->trigger(substream, @@ -2096,7 +2097,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, card = substream->pcm->card; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (file) { @@ -2107,7 +2108,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, snd_pcm_stream_lock_irq(substream); /* resume pause */ - if (runtime->status->state == SNDRV_PCM_STATE_PAUSED) + if (runtime->state == SNDRV_PCM_STATE_PAUSED) snd_pcm_pause(substream, false); /* pre-start/stop - all running streams are changed to DRAINING state */ @@ -2135,7 +2136,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, if (s->stream != SNDRV_PCM_STREAM_PLAYBACK) continue; runtime = s->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) { + if (runtime->state == SNDRV_PCM_STATE_DRAINING) { to_check = runtime; break; } @@ -2174,7 +2175,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream, break; } if (tout == 0) { - if (substream->runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) + if (substream->runtime->state == SNDRV_PCM_STATE_SUSPENDED) result = -ESTRPIPE; else { dev_dbg(substream->pcm->card->dev, @@ -2206,13 +2207,13 @@ static int snd_pcm_drop(struct snd_pcm_substream *substream) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; snd_pcm_stream_lock_irq(substream); /* resume pause */ - if (runtime->status->state == SNDRV_PCM_STATE_PAUSED) + if (runtime->state == SNDRV_PCM_STATE_PAUSED) snd_pcm_pause(substream, false); snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP); @@ -2275,8 +2276,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) snd_pcm_group_init(group); down_write(&snd_pcm_link_rwsem); - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN || - substream->runtime->status->state != substream1->runtime->status->state || + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN || + substream->runtime->state != substream1->runtime->state || substream->pcm->nonatomic != substream1->pcm->nonatomic) { res = -EBADFD; goto _end; @@ -2700,7 +2701,7 @@ void snd_pcm_release_substream(struct snd_pcm_substream *substream) snd_pcm_drop(substream); if (substream->hw_opened) { - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) do_hw_free(substream); substream->ops->close(substream); substream->hw_opened = 0; @@ -2904,7 +2905,7 @@ static int snd_pcm_release(struct inode *inode, struct file *file) */ static int do_pcm_hwsync(struct snd_pcm_substream *substream) { - switch (substream->runtime->status->state) { + switch (substream->runtime->state) { case SNDRV_PCM_STATE_DRAINING: if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) return -EBADFD; @@ -3203,7 +3204,7 @@ static int snd_pcm_xferi_frames_ioctl(struct snd_pcm_substream *substream, struct snd_pcm_runtime *runtime = substream->runtime; snd_pcm_sframes_t result; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (put_user(0, &_xferi->result)) return -EFAULT; @@ -3226,7 +3227,7 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream, void *bufs; snd_pcm_sframes_t result; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (runtime->channels > 128) return -EINVAL; @@ -3290,7 +3291,7 @@ static int snd_pcm_common_ioctl(struct file *file, if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; - if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; res = snd_power_wait(substream->pcm->card); @@ -3421,7 +3422,7 @@ int snd_pcm_kernel_ioctl(struct snd_pcm_substream *substream, snd_pcm_uframes_t *frames = arg; snd_pcm_sframes_t result; - if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; switch (cmd) { @@ -3466,8 +3467,8 @@ static ssize_t snd_pcm_read(struct file *file, char __user *buf, size_t count, if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; if (!frame_aligned(runtime, count)) return -EINVAL; @@ -3491,8 +3492,8 @@ static ssize_t snd_pcm_write(struct file *file, const char __user *buf, if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; if (!frame_aligned(runtime, count)) return -EINVAL; @@ -3518,8 +3519,8 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to) if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; if (!iter_is_iovec(to)) return -EINVAL; @@ -3555,8 +3556,8 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from) if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN || - runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_OPEN || + runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; if (!iter_is_iovec(from)) return -EINVAL; @@ -3595,7 +3596,7 @@ static __poll_t snd_pcm_poll(struct file *file, poll_table *wait) return ok | EPOLLERR; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return ok | EPOLLERR; poll_wait(file, &runtime->sleep, wait); @@ -3603,7 +3604,7 @@ static __poll_t snd_pcm_poll(struct file *file, poll_table *wait) mask = 0; snd_pcm_stream_lock_irq(substream); avail = snd_pcm_avail(substream); - switch (runtime->status->state) { + switch (runtime->state) { case SNDRV_PCM_STATE_RUNNING: case SNDRV_PCM_STATE_PREPARED: case SNDRV_PCM_STATE_PAUSED: @@ -3667,6 +3668,7 @@ static int snd_pcm_mmap_status(struct snd_pcm_substream *substream, struct file area->vm_ops = &snd_pcm_vm_ops_status; area->vm_private_data = substream; area->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; + area->vm_flags &= ~(VM_WRITE | VM_MAYWRITE); return 0; } @@ -3874,7 +3876,7 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file, return -EINVAL; } runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_OPEN) + if (runtime->state == SNDRV_PCM_STATE_OPEN) return -EBADFD; if (!(runtime->info & SNDRV_PCM_INFO_MMAP)) return -ENXIO; @@ -3911,7 +3913,7 @@ static int snd_pcm_mmap(struct file *file, struct vm_area_struct *area) substream = pcm_file->substream; if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; - if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; offset = area->vm_pgoff << PAGE_SHIFT; @@ -3949,7 +3951,7 @@ static int snd_pcm_fasync(int fd, struct file * file, int on) if (PCM_RUNTIME_CHECK(substream)) return -ENXIO; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED) + if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED) return -EBADFD; return snd_fasync_helper(fd, file, on, &runtime->fasync); } diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c index 6963d5a487b3278dbaa8cbf029a9dde9b3966066..d8edb60550724262a09c2d8dcf6a7bced561d477 100644 --- a/sound/core/rawmidi.c +++ b/sound/core/rawmidi.c @@ -1899,10 +1899,8 @@ static int snd_rawmidi_free(struct snd_rawmidi *rmidi) snd_info_free_entry(rmidi->proc_entry); rmidi->proc_entry = NULL; - mutex_lock(®ister_mutex); if (rmidi->ops && rmidi->ops->dev_unregister) rmidi->ops->dev_unregister(rmidi); - mutex_unlock(®ister_mutex); snd_rawmidi_free_substreams(&rmidi->streams[SNDRV_RAWMIDI_STREAM_INPUT]); snd_rawmidi_free_substreams(&rmidi->streams[SNDRV_RAWMIDI_STREAM_OUTPUT]); diff --git a/sound/core/seq/oss/seq_oss_midi.c b/sound/core/seq/oss/seq_oss_midi.c index 1e3bf086f86715a5f03beefa2bf1ecc6fd12efdf..07efb38f58ac128da66453785627d8f6daf29656 100644 --- a/sound/core/seq/oss/seq_oss_midi.c +++ b/sound/core/seq/oss/seq_oss_midi.c @@ -270,7 +270,9 @@ snd_seq_oss_midi_clear_all(void) void snd_seq_oss_midi_setup(struct seq_oss_devinfo *dp) { + spin_lock_irq(®ister_lock); dp->max_mididev = max_midi_devs; + spin_unlock_irq(®ister_lock); } /* diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 2e9d695d336c9faac010c6bfdddc08445a96a8aa..2d707afa1ef1cd3a0e367df865f142f10f344387 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -121,13 +121,13 @@ struct snd_seq_client *snd_seq_client_use_ptr(int clientid) spin_unlock_irqrestore(&clients_lock, flags); #ifdef CONFIG_MODULES if (!in_interrupt()) { - static char client_requested[SNDRV_SEQ_GLOBAL_CLIENTS]; - static char card_requested[SNDRV_CARDS]; + static DECLARE_BITMAP(client_requested, SNDRV_SEQ_GLOBAL_CLIENTS); + static DECLARE_BITMAP(card_requested, SNDRV_CARDS); + if (clientid < SNDRV_SEQ_GLOBAL_CLIENTS) { int idx; - if (!client_requested[clientid]) { - client_requested[clientid] = 1; + if (!test_and_set_bit(clientid, client_requested)) { for (idx = 0; idx < 15; idx++) { if (seq_client_load[idx] < 0) break; @@ -142,10 +142,8 @@ struct snd_seq_client *snd_seq_client_use_ptr(int clientid) int card = (clientid - SNDRV_SEQ_GLOBAL_CLIENTS) / SNDRV_SEQ_CLIENTS_PER_CARD; if (card < snd_ecards_limit) { - if (! card_requested[card]) { - card_requested[card] = 1; + if (!test_and_set_bit(card, card_requested)) snd_request_card(card); - } snd_seq_device_load_drivers(); } } diff --git a/sound/core/sound_oss.c b/sound/core/sound_oss.c index 7ed0a2a9103524a39bb0cb091454bfcc08b49155..2751bf2ff61bca8dfd54125489c5ab047c45bb9b 100644 --- a/sound/core/sound_oss.c +++ b/sound/core/sound_oss.c @@ -162,7 +162,6 @@ int snd_unregister_oss_device(int type, struct snd_card *card, int dev) mutex_unlock(&sound_oss_mutex); return -ENOENT; } - unregister_sound_special(minor); switch (SNDRV_MINOR_OSS_DEVICE(minor)) { case SNDRV_MINOR_OSS_PCM: track2 = SNDRV_MINOR_OSS(cidx, SNDRV_MINOR_OSS_AUDIO); @@ -174,12 +173,18 @@ int snd_unregister_oss_device(int type, struct snd_card *card, int dev) track2 = SNDRV_MINOR_OSS(cidx, SNDRV_MINOR_OSS_DMMIDI1); break; } - if (track2 >= 0) { - unregister_sound_special(track2); + if (track2 >= 0) snd_oss_minors[track2] = NULL; - } snd_oss_minors[minor] = NULL; mutex_unlock(&sound_oss_mutex); + + /* call unregister_sound_special() outside sound_oss_mutex; + * otherwise may deadlock, as it can trigger the release of a card + */ + unregister_sound_special(minor); + if (track2 >= 0) + unregister_sound_special(track2); + kfree(mptr); return 0; } diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index 9b4a7cdb103ad8c142bd5fe4a4e73914a21df553..a38e602b4fc6099f7fbff7471ee9f4b55c13b3ed 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -535,7 +535,7 @@ static void copy_play_buf(struct loopback_pcm *play, /* check if playback is draining, trim the capture copy size * when our pointer is at the end of playback ring buffer */ - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING && + if (runtime->state == SNDRV_PCM_STATE_DRAINING && snd_pcm_playback_hw_avail(runtime) < runtime->buffer_size) { snd_pcm_uframes_t appl_ptr, appl_ptr1, diff; appl_ptr = appl_ptr1 = runtime->control->appl_ptr; @@ -605,17 +605,18 @@ static unsigned int loopback_jiffies_timer_pos_update cable->streams[SNDRV_PCM_STREAM_PLAYBACK]; struct loopback_pcm *dpcm_capt = cable->streams[SNDRV_PCM_STREAM_CAPTURE]; - unsigned long delta_play = 0, delta_capt = 0; + unsigned long delta_play = 0, delta_capt = 0, cur_jiffies; unsigned int running, count1, count2; + cur_jiffies = jiffies; running = cable->running ^ cable->pause; if (running & (1 << SNDRV_PCM_STREAM_PLAYBACK)) { - delta_play = jiffies - dpcm_play->last_jiffies; + delta_play = cur_jiffies - dpcm_play->last_jiffies; dpcm_play->last_jiffies += delta_play; } if (running & (1 << SNDRV_PCM_STREAM_CAPTURE)) { - delta_capt = jiffies - dpcm_capt->last_jiffies; + delta_capt = cur_jiffies - dpcm_capt->last_jiffies; dpcm_capt->last_jiffies += delta_capt; } @@ -729,7 +730,7 @@ static void loopback_snd_timer_period_elapsed(struct loopback_cable *cable, if (event == SNDRV_TIMER_EVENT_MSTOP) { if (!dpcm_play || - dpcm_play->substream->runtime->status->state != + dpcm_play->substream->runtime->state != SNDRV_PCM_STATE_DRAINING) { spin_unlock_irqrestore(&cable->lock, flags); return; diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c index 2a7fc49c1a7c5e68409f630549ced4f8ac70faca..9c17b49a2ae1c98d06a2f8ec9d434b638e698948 100644 --- a/sound/drivers/dummy.c +++ b/sound/drivers/dummy.c @@ -42,6 +42,8 @@ MODULE_LICENSE("GPL"); #define USE_CHANNELS_MAX 2 #define USE_PERIODS_MIN 1 #define USE_PERIODS_MAX 1024 +#define USE_MIXER_VOLUME_LEVEL_MIN -50 +#define USE_MIXER_VOLUME_LEVEL_MAX 100 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX; /* Index 0-MAX */ static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR; /* ID for this card */ @@ -50,6 +52,8 @@ static char *model[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = NULL}; static int pcm_devs[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 1}; static int pcm_substreams[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 8}; //static int midi_devs[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 2}; +static int mixer_volume_level_min = USE_MIXER_VOLUME_LEVEL_MIN; +static int mixer_volume_level_max = USE_MIXER_VOLUME_LEVEL_MAX; #ifdef CONFIG_HIGH_RES_TIMERS static bool hrtimer = 1; #endif @@ -69,6 +73,10 @@ module_param_array(pcm_substreams, int, NULL, 0444); MODULE_PARM_DESC(pcm_substreams, "PCM substreams # (1-128) for dummy driver."); //module_param_array(midi_devs, int, NULL, 0444); //MODULE_PARM_DESC(midi_devs, "MIDI devices # (0-2) for dummy driver."); +module_param(mixer_volume_level_min, int, 0444); +MODULE_PARM_DESC(mixer_volume_level_min, "Minimum mixer volume level for dummy driver. Default: -50"); +module_param(mixer_volume_level_max, int, 0444); +MODULE_PARM_DESC(mixer_volume_level_max, "Maximum mixer volume level for dummy driver. Default: 100"); module_param(fake_buffer, bool, 0444); MODULE_PARM_DESC(fake_buffer, "Fake buffer allocations."); #ifdef CONFIG_HIGH_RES_TIMERS @@ -296,7 +304,7 @@ static void dummy_systimer_callback(struct timer_list *t) struct dummy_systimer_pcm *dpcm = from_timer(dpcm, t, timer); unsigned long flags; int elapsed = 0; - + spin_lock_irqsave(&dpcm->lock, flags); dummy_systimer_update(dpcm); dummy_systimer_rearm(dpcm); @@ -713,11 +721,11 @@ static int snd_dummy_volume_info(struct snd_kcontrol *kcontrol, { uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; uinfo->count = 2; - uinfo->value.integer.min = -50; - uinfo->value.integer.max = 100; + uinfo->value.integer.min = mixer_volume_level_min; + uinfo->value.integer.max = mixer_volume_level_max; return 0; } - + static int snd_dummy_volume_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { @@ -739,15 +747,15 @@ static int snd_dummy_volume_put(struct snd_kcontrol *kcontrol, int left, right; left = ucontrol->value.integer.value[0]; - if (left < -50) - left = -50; - if (left > 100) - left = 100; + if (left < mixer_volume_level_min) + left = mixer_volume_level_min; + if (left > mixer_volume_level_max) + left = mixer_volume_level_max; right = ucontrol->value.integer.value[1]; - if (right < -50) - right = -50; - if (right > 100) - right = 100; + if (right < mixer_volume_level_min) + right = mixer_volume_level_min; + if (right > mixer_volume_level_max) + right = mixer_volume_level_max; spin_lock_irq(&dummy->mixer_lock); change = dummy->mixer_volume[addr][0] != left || dummy->mixer_volume[addr][1] != right; @@ -766,7 +774,7 @@ static const DECLARE_TLV_DB_SCALE(db_scale_dummy, -4500, 30, 0); .private_value = addr } #define snd_dummy_capsrc_info snd_ctl_boolean_stereo_info - + static int snd_dummy_capsrc_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { @@ -1076,6 +1084,12 @@ static int snd_dummy_probe(struct platform_device *devptr) dummy->pcm_hw.channels_max = m->channels_max; } + if (mixer_volume_level_min > mixer_volume_level_max) { + pr_warn("snd-dummy: Invalid mixer volume level: min=%d, max=%d. Fall back to default value.\n", + mixer_volume_level_min, mixer_volume_level_max); + mixer_volume_level_min = USE_MIXER_VOLUME_LEVEL_MIN; + mixer_volume_level_max = USE_MIXER_VOLUME_LEVEL_MAX; + } err = snd_card_dummy_new_mixer(dummy); if (err < 0) return err; @@ -1100,7 +1114,7 @@ static int snd_dummy_suspend(struct device *pdev) snd_power_change_state(card, SNDRV_CTL_POWER_D3hot); return 0; } - + static int snd_dummy_resume(struct device *pdev) { struct snd_card *card = dev_get_drvdata(pdev); diff --git a/sound/drivers/vx/vx_pcm.c b/sound/drivers/vx/vx_pcm.c index 3924f5283745e4b78da4dc54ac41625ddb5441bc..ceaeb257003bcc6165027b7222d7e332d07914ed 100644 --- a/sound/drivers/vx/vx_pcm.c +++ b/sound/drivers/vx/vx_pcm.c @@ -1215,8 +1215,7 @@ int snd_vx_pcm_new(struct vx_core *chip) if (ins) snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &vx_pcm_capture_ops); snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, - snd_dma_continuous_data(GFP_KERNEL | GFP_DMA32), - 0, 0); + NULL, 0, 0); pcm->private_data = chip; pcm->private_free = snd_vx_pcm_free; diff --git a/sound/firewire/bebob/bebob_pcm.c b/sound/firewire/bebob/bebob_pcm.c index f8d9a2041264d9c102824218d02f139e45a65636..ce49eef0fcbaae7e39df210408c1215028bf0703 100644 --- a/sound/firewire/bebob/bebob_pcm.c +++ b/sound/firewire/bebob/bebob_pcm.c @@ -214,7 +214,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_bebob *bebob = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -236,7 +236,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&bebob->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) bebob->substreams_counter--; snd_bebob_stream_stop_duplex(bebob); diff --git a/sound/firewire/dice/dice-harman.c b/sound/firewire/dice/dice-harman.c index a8ca00c397e84dfb4459322cbd46735ddd25656a..212ae77dfca2752a3bf095f42b9a5ac592bb4d68 100644 --- a/sound/firewire/dice/dice-harman.c +++ b/sound/firewire/dice/dice-harman.c @@ -2,8 +2,6 @@ // dice-harman.c - a part of driver for DICE based devices // // Copyright (c) 2021 Takashi Sakamoto -// -// Licensed under the terms of the GNU General Public License, version 2. #include "dice.h" diff --git a/sound/firewire/dice/dice-pcm.c b/sound/firewire/dice/dice-pcm.c index a69ca1111b0332f5386f154d6a2d4998d23c55b6..d64366217d572d78a3aea031432203609a589a1d 100644 --- a/sound/firewire/dice/dice-pcm.c +++ b/sound/firewire/dice/dice-pcm.c @@ -266,7 +266,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_dice *dice = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int events_per_period = params_period_size(hw_params); unsigned int events_per_buffer = params_buffer_size(hw_params); @@ -293,7 +293,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&dice->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --dice->substreams_counter; snd_dice_stream_stop_duplex(dice); diff --git a/sound/firewire/dice/dice-presonus.c b/sound/firewire/dice/dice-presonus.c index 503f462a83f4e100b18a1dbdb9410a58d033d96f..967cc3119a64752ae7d500b72d78f98da50dd771 100644 --- a/sound/firewire/dice/dice-presonus.c +++ b/sound/firewire/dice/dice-presonus.c @@ -2,8 +2,6 @@ // dice-presonus.c - a part of driver for DICE based devices // // Copyright (c) 2019 Takashi Sakamoto -// -// Licensed under the terms of the GNU General Public License, version 2. #include "dice.h" diff --git a/sound/firewire/digi00x/digi00x-pcm.c b/sound/firewire/digi00x/digi00x-pcm.c index b7f6eda09f9f7a444530b190f273c02b05319ff3..3bd1575c9d9c1f9095308564fb72414abf03973c 100644 --- a/sound/firewire/digi00x/digi00x-pcm.c +++ b/sound/firewire/digi00x/digi00x-pcm.c @@ -190,7 +190,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_dg00x *dg00x = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -212,7 +212,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&dg00x->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --dg00x->substreams_counter; snd_dg00x_stream_stop_duplex(dg00x); diff --git a/sound/firewire/fireface/ff-pcm.c b/sound/firewire/fireface/ff-pcm.c index f978cc2fed7dd5cec0fa99753f32a5f26d5fa794..ec915671a79b3f6db870254c86c4c182ce520473 100644 --- a/sound/firewire/fireface/ff-pcm.c +++ b/sound/firewire/fireface/ff-pcm.c @@ -230,7 +230,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_ff *ff = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -252,7 +252,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&ff->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --ff->substreams_counter; snd_ff_stream_stop_duplex(ff); diff --git a/sound/firewire/fireface/ff-protocol-former.c b/sound/firewire/fireface/ff-protocol-former.c index bf44cad7985e080697563bdf56a0df34eba37196..8900ffe517ed3fb01ca7b6ebde10707914e8cdc9 100644 --- a/sound/firewire/fireface/ff-protocol-former.c +++ b/sound/firewire/fireface/ff-protocol-former.c @@ -2,8 +2,6 @@ // ff-protocol-former.c - a part of driver for RME Fireface series // // Copyright (c) 2019 Takashi Sakamoto -// -// Licensed under the terms of the GNU General Public License, version 2. #include <linux/delay.h> diff --git a/sound/firewire/fireface/ff-protocol-latter.c b/sound/firewire/fireface/ff-protocol-latter.c index 7ddb7b97f02db7409e70acb984aca330e536ed74..76c3eab36d4e07ad34939d7c2e4c8786118e73f1 100644 --- a/sound/firewire/fireface/ff-protocol-latter.c +++ b/sound/firewire/fireface/ff-protocol-latter.c @@ -1,9 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 -// ff-protocol-latter - a part of driver for RME Fireface series +// ff-protocol-latter.c - a part of driver for RME Fireface series // // Copyright (c) 2019 Takashi Sakamoto -// -// Licensed under the terms of the GNU General Public License, version 2. #include <linux/delay.h> diff --git a/sound/firewire/fireworks/fireworks_pcm.c b/sound/firewire/fireworks/fireworks_pcm.c index a0d5db1d8eb2371f211b6b506648a5eab4936575..c3c21860b245bcc9d78f6b1daa021ba96de8fca4 100644 --- a/sound/firewire/fireworks/fireworks_pcm.c +++ b/sound/firewire/fireworks/fireworks_pcm.c @@ -250,7 +250,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_efw *efw = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -272,7 +272,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&efw->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --efw->substreams_counter; snd_efw_stream_stop_duplex(efw); diff --git a/sound/firewire/motu/motu-pcm.c b/sound/firewire/motu/motu-pcm.c index 8e143737126388079baec57a3faca94c5df69ecf..d410c2efbde577b93a3c2e60973276992a713358 100644 --- a/sound/firewire/motu/motu-pcm.c +++ b/sound/firewire/motu/motu-pcm.c @@ -210,7 +210,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_motu *motu = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -232,7 +232,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&motu->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --motu->substreams_counter; snd_motu_stream_stop_duplex(motu); diff --git a/sound/firewire/motu/motu-protocol-v1.c b/sound/firewire/motu/motu-protocol-v1.c index f1d6a326dc07c73fca6013bad68e622411909394..e811629f167b847b169567eba70fb66bc966ba32 100644 --- a/sound/firewire/motu/motu-protocol-v1.c +++ b/sound/firewire/motu/motu-protocol-v1.c @@ -1,10 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only - // motu-protocol-v1.c - a part of driver for MOTU FireWire series // // Copyright (c) 2021 Takashi Sakamoto <o-takashi@sakamocchi.jp> -// -// Licensed under the terms of the GNU General Public License, version 2. #include "motu.h" diff --git a/sound/firewire/oxfw/oxfw-pcm.c b/sound/firewire/oxfw/oxfw-pcm.c index 2dfa7e179cb6a56e7c1c7f976f67f6142e21f0a5..5f43a0b826d2eeddc9f62155b87c279f4ccb0760 100644 --- a/sound/firewire/oxfw/oxfw-pcm.c +++ b/sound/firewire/oxfw/oxfw-pcm.c @@ -239,7 +239,7 @@ static int pcm_capture_hw_params(struct snd_pcm_substream *substream, struct snd_oxfw *oxfw = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int channels = params_channels(hw_params); unsigned int frames_per_period = params_period_size(hw_params); @@ -262,7 +262,7 @@ static int pcm_playback_hw_params(struct snd_pcm_substream *substream, struct snd_oxfw *oxfw = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int channels = params_channels(hw_params); unsigned int frames_per_period = params_period_size(hw_params); @@ -286,7 +286,7 @@ static int pcm_capture_hw_free(struct snd_pcm_substream *substream) mutex_lock(&oxfw->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --oxfw->substreams_count; snd_oxfw_stream_stop_duplex(oxfw); @@ -301,7 +301,7 @@ static int pcm_playback_hw_free(struct snd_pcm_substream *substream) mutex_lock(&oxfw->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --oxfw->substreams_count; snd_oxfw_stream_stop_duplex(oxfw); diff --git a/sound/firewire/tascam/tascam-pcm.c b/sound/firewire/tascam/tascam-pcm.c index 36c1353f249489adf5721b68610933065212e8fc..f6da571707ac2ca2bf81bc75f8a35912e9eadcc3 100644 --- a/sound/firewire/tascam/tascam-pcm.c +++ b/sound/firewire/tascam/tascam-pcm.c @@ -119,7 +119,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream, struct snd_tscm *tscm = substream->private_data; int err = 0; - if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) { + if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) { unsigned int rate = params_rate(hw_params); unsigned int frames_per_period = params_period_size(hw_params); unsigned int frames_per_buffer = params_buffer_size(hw_params); @@ -141,7 +141,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream) mutex_lock(&tscm->mutex); - if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN) + if (substream->runtime->state != SNDRV_PCM_STATE_OPEN) --tscm->substreams_counter; snd_tscm_stream_stop_duplex(tscm); diff --git a/sound/hda/ext/hdac_ext_bus.c b/sound/hda/ext/hdac_ext_bus.c index 765c40a6ccbadabc4ba5b1227f0ea1e901b3a75f..6004ea1c373e106d61c4d379578b395e81048c64 100644 --- a/sound/hda/ext/hdac_ext_bus.c +++ b/sound/hda/ext/hdac_ext_bus.c @@ -60,59 +60,6 @@ void snd_hdac_ext_bus_exit(struct hdac_bus *bus) } EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_exit); -static void default_release(struct device *dev) -{ - snd_hdac_ext_bus_device_exit(dev_to_hdac_dev(dev)); -} - -/** - * snd_hdac_ext_bus_device_init - initialize the HDA extended codec base device - * @bus: hdac bus to attach to - * @addr: codec address - * @hdev: hdac device to init - * @type: codec type (HDAC_DEV_*) to use for this device - * - * Returns zero for success or a negative error code. - */ -int snd_hdac_ext_bus_device_init(struct hdac_bus *bus, int addr, - struct hdac_device *hdev, int type) -{ - char name[15]; - int ret; - - hdev->bus = bus; - - snprintf(name, sizeof(name), "ehdaudio%dD%d", bus->idx, addr); - - ret = snd_hdac_device_init(hdev, bus, name, addr); - if (ret < 0) { - dev_err(bus->dev, "device init failed for hdac device\n"); - return ret; - } - hdev->type = type; - hdev->dev.release = default_release; - - ret = snd_hdac_device_register(hdev); - if (ret) { - dev_err(bus->dev, "failed to register hdac device\n"); - snd_hdac_ext_bus_device_exit(hdev); - return ret; - } - - return 0; -} -EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_device_init); - -/** - * snd_hdac_ext_bus_device_exit - clean up a HD-audio extended codec base device - * @hdev: hdac device to clean up - */ -void snd_hdac_ext_bus_device_exit(struct hdac_device *hdev) -{ - snd_hdac_device_exit(hdev); -} -EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_device_exit); - /** * snd_hdac_ext_bus_device_remove - remove HD-audio extended codec base devices * diff --git a/sound/hda/ext/hdac_ext_controller.c b/sound/hda/ext/hdac_ext_controller.c index a42f66f561f57f0cb15332bc6aa6e1d36a9ed79a..80876b9a87f46021ba115b53b3940846ea9ac0d9 100644 --- a/sound/hda/ext/hdac_ext_controller.c +++ b/sound/hda/ext/hdac_ext_controller.c @@ -170,7 +170,7 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable) { int timeout; u32 val; - int mask = (1 << AZX_MLCTL_CPA_SHIFT); + int mask = (1 << AZX_ML_LCTL_CPA_SHIFT); udelay(3); timeout = 150; @@ -178,10 +178,10 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable) do { val = readl(link->ml_addr + AZX_REG_ML_LCTL); if (enable) { - if (((val & mask) >> AZX_MLCTL_CPA_SHIFT)) + if (((val & mask) >> AZX_ML_LCTL_CPA_SHIFT)) return 0; } else { - if (!((val & mask) >> AZX_MLCTL_CPA_SHIFT)) + if (!((val & mask) >> AZX_ML_LCTL_CPA_SHIFT)) return 0; } udelay(3); @@ -197,7 +197,7 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable) int snd_hdac_ext_bus_link_power_up(struct hdac_ext_link *link) { snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL, - AZX_MLCTL_SPA, AZX_MLCTL_SPA); + AZX_ML_LCTL_SPA, AZX_ML_LCTL_SPA); return check_hdac_link_power_active(link, true); } @@ -209,7 +209,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_link_power_up); */ int snd_hdac_ext_bus_link_power_down(struct hdac_ext_link *link) { - snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL, AZX_MLCTL_SPA, 0); + snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL, AZX_ML_LCTL_SPA, 0); return check_hdac_link_power_active(link, false); } @@ -226,7 +226,7 @@ int snd_hdac_ext_bus_link_power_up_all(struct hdac_bus *bus) list_for_each_entry(hlink, &bus->hlink_list, list) { snd_hdac_updatel(hlink->ml_addr, AZX_REG_ML_LCTL, - AZX_MLCTL_SPA, AZX_MLCTL_SPA); + AZX_ML_LCTL_SPA, AZX_ML_LCTL_SPA); ret = check_hdac_link_power_active(hlink, true); if (ret < 0) return ret; @@ -247,7 +247,7 @@ int snd_hdac_ext_bus_link_power_down_all(struct hdac_bus *bus) list_for_each_entry(hlink, &bus->hlink_list, list) { snd_hdac_updatel(hlink->ml_addr, AZX_REG_ML_LCTL, - AZX_MLCTL_SPA, 0); + AZX_ML_LCTL_SPA, 0); ret = check_hdac_link_power_active(hlink, false); if (ret < 0) return ret; @@ -281,7 +281,7 @@ int snd_hdac_ext_bus_link_get(struct hdac_bus *bus, * clear the register to invalidate all the output streams */ snd_hdac_updatew(link->ml_addr, AZX_REG_ML_LOSIDV, - ML_LOSIDV_STREAM_MASK, 0); + AZX_ML_LOSIDV_STREAM_MASK, 0); /* * wait for 521usec for codec to report status * HDA spec section 4.3 - Codec Discovery diff --git a/sound/hda/ext/hdac_ext_stream.c b/sound/hda/ext/hdac_ext_stream.c index d2b5724b463ffbe857ae7406f9022a0e9bddde06..70f3ad71aaf0dcf557b8c1bf75d47babce3a9d96 100644 --- a/sound/hda/ext/hdac_ext_stream.c +++ b/sound/hda/ext/hdac_ext_stream.c @@ -26,9 +26,9 @@ * initialize the stream, if ppcap is enabled then init those and then * invoke hdac stream initialization routine */ -void snd_hdac_ext_stream_init(struct hdac_bus *bus, - struct hdac_ext_stream *hext_stream, - int idx, int direction, int tag) +static void snd_hdac_ext_stream_init(struct hdac_bus *bus, + struct hdac_ext_stream *hext_stream, + int idx, int direction, int tag) { if (bus->ppcap) { hext_stream->pphc_addr = bus->ppcap + AZX_PPHC_BASE + @@ -56,7 +56,6 @@ void snd_hdac_ext_stream_init(struct hdac_bus *bus, hext_stream->decoupled = false; snd_hdac_stream_init(bus, &hext_stream->hstream, idx, direction, tag); } -EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_init); /** * snd_hdac_ext_stream_init_all - create and initialize the stream objects @@ -88,11 +87,11 @@ int snd_hdac_ext_stream_init_all(struct hdac_bus *bus, int start_idx, EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_init_all); /** - * snd_hdac_stream_free_all - free hdac extended stream objects + * snd_hdac_ext_stream_free_all - free hdac extended stream objects * * @bus: HD-audio core bus */ -void snd_hdac_stream_free_all(struct hdac_bus *bus) +void snd_hdac_ext_stream_free_all(struct hdac_bus *bus) { struct hdac_stream *s, *_s; struct hdac_ext_stream *hext_stream; @@ -104,7 +103,7 @@ void snd_hdac_stream_free_all(struct hdac_bus *bus) kfree(hext_stream); } } -EXPORT_SYMBOL_GPL(snd_hdac_stream_free_all); +EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_free_all); void snd_hdac_ext_stream_decouple_locked(struct hdac_bus *bus, struct hdac_ext_stream *hext_stream, @@ -268,19 +267,15 @@ hdac_ext_link_stream_assign(struct hdac_bus *bus, if (hstream->direction != substream->stream) continue; - /* check if decoupled stream and not in use is available */ - if (hext_stream->decoupled && !hext_stream->link_locked) { - res = hext_stream; - break; - } - + /* check if link stream is available */ if (!hext_stream->link_locked) { - snd_hdac_ext_stream_decouple_locked(bus, hext_stream, true); res = hext_stream; break; } + } if (res) { + snd_hdac_ext_stream_decouple_locked(bus, res, true); res->link_locked = 1; res->link_substream = substream; } @@ -309,13 +304,12 @@ hdac_ext_host_stream_assign(struct hdac_bus *bus, continue; if (!hstream->opened) { - if (!hext_stream->decoupled) - snd_hdac_ext_stream_decouple_locked(bus, hext_stream, true); res = hext_stream; break; } } if (res) { + snd_hdac_ext_stream_decouple_locked(bus, res, true); res->hstream.opened = 1; res->hstream.running = 0; res->hstream.substream = substream; @@ -388,15 +382,17 @@ void snd_hdac_ext_stream_release(struct hdac_ext_stream *hext_stream, int type) case HDAC_EXT_STREAM_TYPE_HOST: spin_lock_irq(&bus->reg_lock); - if (hext_stream->decoupled && !hext_stream->link_locked) + /* couple link only if not in use */ + if (!hext_stream->link_locked) snd_hdac_ext_stream_decouple_locked(bus, hext_stream, false); + snd_hdac_stream_release_locked(&hext_stream->hstream); spin_unlock_irq(&bus->reg_lock); - snd_hdac_stream_release(&hext_stream->hstream); break; case HDAC_EXT_STREAM_TYPE_LINK: spin_lock_irq(&bus->reg_lock); - if (hext_stream->decoupled && !hext_stream->hstream.opened) + /* couple host only if not in use */ + if (!hext_stream->hstream.opened) snd_hdac_ext_stream_decouple_locked(bus, hext_stream, false); hext_stream->link_locked = 0; hext_stream->link_substream = NULL; diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c index f3582012d22f3a7e859dee33950f58d2f0d7cd16..1b8be39c38a96bffdfc984b9b4113a375d88e71d 100644 --- a/sound/hda/hdac_stream.c +++ b/sound/hda/hdac_stream.c @@ -13,6 +13,39 @@ #include <sound/hda_register.h> #include "trace.h" +/* + * the hdac_stream library is intended to be used with the following + * transitions. The states are not formally defined in the code but loosely + * inspired by boolean variables. Note that the 'prepared' field is not used + * in this library but by the callers during the hw_params/prepare transitions + * + * | + * stream_init() | + * v + * +--+-------+ + * | unused | + * +--+----+--+ + * | ^ + * stream_assign() | | stream_release() + * v | + * +--+----+--+ + * | opened | + * +--+----+--+ + * | ^ + * stream_reset() | | + * stream_setup() | | stream_cleanup() + * v | + * +--+----+--+ + * | prepared | + * +--+----+--+ + * | ^ + * stream_start() | | stream_stop() + * v | + * +--+----+--+ + * | running | + * +----------+ + */ + /** * snd_hdac_get_stream_stripe_ctl - get stripe control value * @bus: HD-audio core bus @@ -112,10 +145,10 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start) EXPORT_SYMBOL_GPL(snd_hdac_stream_start); /** - * snd_hdac_stream_clear - stop a stream DMA + * snd_hdac_stream_clear - helper to clear stream registers and stop DMA transfers * @azx_dev: HD-audio core stream to stop */ -void snd_hdac_stream_clear(struct hdac_stream *azx_dev) +static void snd_hdac_stream_clear(struct hdac_stream *azx_dev) { snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_DMA_START | SD_INT_MASK, 0); @@ -124,7 +157,6 @@ void snd_hdac_stream_clear(struct hdac_stream *azx_dev) snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0); azx_dev->running = false; } -EXPORT_SYMBOL_GPL(snd_hdac_stream_clear); /** * snd_hdac_stream_stop - stop a stream @@ -142,17 +174,28 @@ void snd_hdac_stream_stop(struct hdac_stream *azx_dev) } EXPORT_SYMBOL_GPL(snd_hdac_stream_stop); +/** + * snd_hdac_stop_streams - stop all streams + * @bus: HD-audio core bus + */ +void snd_hdac_stop_streams(struct hdac_bus *bus) +{ + struct hdac_stream *stream; + + list_for_each_entry(stream, &bus->stream_list, list) + snd_hdac_stream_stop(stream); +} +EXPORT_SYMBOL_GPL(snd_hdac_stop_streams); + /** * snd_hdac_stop_streams_and_chip - stop all streams and chip if running * @bus: HD-audio core bus */ void snd_hdac_stop_streams_and_chip(struct hdac_bus *bus) { - struct hdac_stream *stream; if (bus->chip_init) { - list_for_each_entry(stream, &bus->stream_list, list) - snd_hdac_stream_stop(stream); + snd_hdac_stop_streams(bus); snd_hdac_bus_stop_chip(bus); } } @@ -165,7 +208,6 @@ EXPORT_SYMBOL_GPL(snd_hdac_stop_streams_and_chip); void snd_hdac_stream_reset(struct hdac_stream *azx_dev) { unsigned char val; - int timeout; int dma_run_state; snd_hdac_stream_clear(azx_dev); @@ -173,30 +215,17 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev) dma_run_state = snd_hdac_stream_readb(azx_dev, SD_CTL) & SD_CTL_DMA_START; snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET); - udelay(3); - timeout = 300; - do { - val = snd_hdac_stream_readb(azx_dev, SD_CTL) & - SD_CTL_STREAM_RESET; - if (val) - break; - } while (--timeout); + + /* wait for hardware to report that the stream entered reset */ + snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, (val & SD_CTL_STREAM_RESET), 3, 300); if (azx_dev->bus->dma_stop_delay && dma_run_state) udelay(azx_dev->bus->dma_stop_delay); - val &= ~SD_CTL_STREAM_RESET; - snd_hdac_stream_writeb(azx_dev, SD_CTL, val); - udelay(3); + snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0); - timeout = 300; - /* waiting for hardware to report that the stream is out of reset */ - do { - val = snd_hdac_stream_readb(azx_dev, SD_CTL) & - SD_CTL_STREAM_RESET; - if (!val) - break; - } while (--timeout); + /* wait for hardware to report that the stream is out of reset */ + snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, !(val & SD_CTL_STREAM_RESET), 3, 300); /* reset first position - may not be synced with hw at this time */ if (azx_dev->posbuf) @@ -336,6 +365,21 @@ struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus, } EXPORT_SYMBOL_GPL(snd_hdac_stream_assign); +/** + * snd_hdac_stream_release_locked - release the assigned stream + * @azx_dev: HD-audio core stream to release + * + * Release the stream that has been assigned by snd_hdac_stream_assign(). + * The bus->reg_lock needs to be taken at a higher level + */ +void snd_hdac_stream_release_locked(struct hdac_stream *azx_dev) +{ + azx_dev->opened = 0; + azx_dev->running = 0; + azx_dev->substream = NULL; +} +EXPORT_SYMBOL_GPL(snd_hdac_stream_release_locked); + /** * snd_hdac_stream_release - release the assigned stream * @azx_dev: HD-audio core stream to release @@ -347,9 +391,7 @@ void snd_hdac_stream_release(struct hdac_stream *azx_dev) struct hdac_bus *bus = azx_dev->bus; spin_lock_irq(&bus->reg_lock); - azx_dev->opened = 0; - azx_dev->running = 0; - azx_dev->substream = NULL; + snd_hdac_stream_release_locked(azx_dev); spin_unlock_irq(&bus->reg_lock); } EXPORT_SYMBOL_GPL(snd_hdac_stream_release); diff --git a/sound/hda/hdmi_chmap.c b/sound/hda/hdmi_chmap.c index aad5c4bf4d3441754b0ec1d5d046b438b1ee6c5e..5d8e1d944b0afb2128c8770583f947eb3d3df85e 100644 --- a/sound/hda/hdmi_chmap.c +++ b/sound/hda/hdmi_chmap.c @@ -774,7 +774,7 @@ static int hdmi_chmap_ctl_put(struct snd_kcontrol *kcontrol, substream = snd_pcm_chmap_substream(info, ctl_idx); if (!substream || !substream->runtime) return 0; /* just for avoiding error from alsactl restore */ - switch (substream->runtime->status->state) { + switch (substream->runtime->state) { case SNDRV_PCM_STATE_OPEN: case SNDRV_PCM_STATE_SETUP: break; diff --git a/sound/hda/intel-dsp-config.c b/sound/hda/intel-dsp-config.c index d84ffdf47210ebbbb9165c3f4f5b400266ad3e79..b9eb3208f2888001b478c756efc66f56d12db253 100644 --- a/sound/hda/intel-dsp-config.c +++ b/sound/hda/intel-dsp-config.c @@ -427,6 +427,11 @@ static const struct config_entry config_table[] = { .device = 0x51cd, }, /* Alderlake-PS */ + { + .flags = FLAG_SOF, + .device = 0x51c9, + .codec_hid = &essx_83x6, + }, { .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE, .device = 0x51c9, @@ -450,6 +455,16 @@ static const struct config_entry config_table[] = { .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE, .device = 0x51cb, }, + /* RaptorLake-M */ + { + .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE, + .device = 0x51ce, + }, + /* RaptorLake-PX */ + { + .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE, + .device = 0x51cf, + }, #endif }; diff --git a/sound/hda/intel-nhlt.c b/sound/hda/intel-nhlt.c index 9db5ccd9aa2db12fcbea22c974e4dfb6f5f53b7e..2c4dfc0b7e342cb7eddde5e89ef64f91508847d6 100644 --- a/sound/hda/intel-nhlt.c +++ b/sound/hda/intel-nhlt.c @@ -55,16 +55,22 @@ int intel_nhlt_get_dmic_geo(struct device *dev, struct nhlt_acpi_table *nhlt) /* find max number of channels based on format_configuration */ if (fmt_configs->fmt_count) { + struct nhlt_fmt_cfg *fmt_cfg = fmt_configs->fmt_config; + dev_dbg(dev, "found %d format definitions\n", fmt_configs->fmt_count); for (i = 0; i < fmt_configs->fmt_count; i++) { struct wav_fmt_ext *fmt_ext; - fmt_ext = &fmt_configs->fmt_config[i].fmt_ext; + fmt_ext = &fmt_cfg->fmt_ext; if (fmt_ext->fmt.channels > max_ch) max_ch = fmt_ext->fmt.channels; + + /* Move to the next nhlt_fmt_cfg */ + fmt_cfg = (struct nhlt_fmt_cfg *)(fmt_cfg->config.caps + + fmt_cfg->config.size); } dev_dbg(dev, "max channels found %d\n", max_ch); } else { @@ -151,6 +157,85 @@ int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8 device_type) } EXPORT_SYMBOL(intel_nhlt_ssp_endpoint_mask); +#define SSP_BLOB_V1_0_SIZE 84 +#define SSP_BLOB_V1_0_MDIVC_OFFSET 19 /* offset in u32 */ + +#define SSP_BLOB_V1_5_SIZE 96 +#define SSP_BLOB_V1_5_MDIVC_OFFSET 21 /* offset in u32 */ +#define SSP_BLOB_VER_1_5 0xEE000105 + +#define SSP_BLOB_V2_0_SIZE 88 +#define SSP_BLOB_V2_0_MDIVC_OFFSET 20 /* offset in u32 */ +#define SSP_BLOB_VER_2_0 0xEE000200 + +int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num) +{ + struct nhlt_endpoint *epnt; + struct nhlt_fmt *fmt; + struct nhlt_fmt_cfg *cfg; + int mclk_mask = 0; + int i, j; + + if (!nhlt) + return 0; + + epnt = (struct nhlt_endpoint *)nhlt->desc; + for (i = 0; i < nhlt->endpoint_count; i++) { + + /* we only care about endpoints connected to an audio codec over SSP */ + if (epnt->linktype == NHLT_LINK_SSP && + epnt->device_type == NHLT_DEVICE_I2S && + epnt->virtual_bus_id == ssp_num) { + + fmt = (struct nhlt_fmt *)(epnt->config.caps + epnt->config.size); + cfg = fmt->fmt_config; + + /* + * In theory all formats should use the same MCLK but it doesn't hurt to + * double-check that the configuration is consistent + */ + for (j = 0; j < fmt->fmt_count; j++) { + u32 *blob; + int mdivc_offset; + int size; + + /* first check we have enough data to read the blob type */ + if (cfg->config.size < 8) + return -EINVAL; + + blob = (u32 *)cfg->config.caps; + + if (blob[1] == SSP_BLOB_VER_2_0) { + mdivc_offset = SSP_BLOB_V2_0_MDIVC_OFFSET; + size = SSP_BLOB_V2_0_SIZE; + } else if (blob[1] == SSP_BLOB_VER_1_5) { + mdivc_offset = SSP_BLOB_V1_5_MDIVC_OFFSET; + size = SSP_BLOB_V1_5_SIZE; + } else { + mdivc_offset = SSP_BLOB_V1_0_MDIVC_OFFSET; + size = SSP_BLOB_V1_0_SIZE; + } + + /* make sure we have enough data for the fixed part of the blob */ + if (cfg->config.size < size) + return -EINVAL; + + mclk_mask |= blob[mdivc_offset] & GENMASK(1, 0); + + cfg = (struct nhlt_fmt_cfg *)(cfg->config.caps + cfg->config.size); + } + } + epnt = (struct nhlt_endpoint *)((u8 *)epnt + epnt->length); + } + + /* make sure only one MCLK is used */ + if (hweight_long(mclk_mask) != 1) + return -EINVAL; + + return mclk_mask; +} +EXPORT_SYMBOL(intel_nhlt_ssp_mclk_mask); + static struct nhlt_specific_cfg * nhlt_get_specific_cfg(struct device *dev, struct nhlt_fmt *fmt, u8 num_ch, u32 rate, u8 vbps, u8 bps) diff --git a/sound/isa/es18xx.c b/sound/isa/es18xx.c index 3fcd168480b64bb90bf8f50d1ac0c69ae98b8d71..0a32845b1017a6463b4471869e8acc693f51a5d5 100644 --- a/sound/isa/es18xx.c +++ b/sound/isa/es18xx.c @@ -1344,11 +1344,8 @@ ES18XX_SINGLE("GPO1 Switch", 0, ES18XX_PM, 1, 1, ES18XX_FL_PMPORT), static int snd_es18xx_config_read(struct snd_es18xx *chip, unsigned char reg) { - int data; - outb(reg, chip->ctrl_port); - data = inb(chip->ctrl_port + 1); - return data; + return inb(chip->ctrl_port + 1); } static void snd_es18xx_config_write(struct snd_es18xx *chip, diff --git a/sound/isa/sb/emu8000_pcm.c b/sound/isa/sb/emu8000_pcm.c index f8d90a1e989b0925797d1dfc24849840b36b39e5..c8afc4347c54d4070e3ecaa8077d10088d170731 100644 --- a/sound/isa/sb/emu8000_pcm.c +++ b/sound/isa/sb/emu8000_pcm.c @@ -236,7 +236,7 @@ static int emu8k_pcm_open(struct snd_pcm_substream *subs) /* use timer to update periods.. (specified in msec) */ snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_PERIOD_TIME, - (1000000 + HZ - 1) / HZ, UINT_MAX); + DIV_ROUND_UP(1000000, HZ), UINT_MAX); return 0; } diff --git a/sound/oss/dmasound/dmasound.h b/sound/oss/dmasound/dmasound.h index ad8ce6a1c25c7db3439817d57a7cb55758fa8ab2..f065840c0efb437faeec4f615541e736043dae20 100644 --- a/sound/oss/dmasound/dmasound.h +++ b/sound/oss/dmasound/dmasound.h @@ -250,7 +250,4 @@ extern int dmasound_catchRadius; #define SW_INPUT_VOLUME_SCALE 4 #define SW_INPUT_VOLUME_DEFAULT (128 / SW_INPUT_VOLUME_SCALE) -extern int expand_read_bal; /* Balance factor for reading */ -extern uint software_input_volume; /* software implemented recording volume! */ - #endif /* _dmasound_h_ */ diff --git a/sound/pci/asihpi/asihpi.c b/sound/pci/asihpi/asihpi.c index 5e1f9f10051bcb01d78451ddfd39079d93665f45..8de43aaa10aac953667f70f94cd3066fdf58138c 100644 --- a/sound/pci/asihpi/asihpi.c +++ b/sound/pci/asihpi/asihpi.c @@ -632,7 +632,7 @@ static int snd_card_asihpi_trigger(struct snd_pcm_substream *substream, /*? workaround linked streams don't transition to SETUP 20070706*/ - s->runtime->status->state = SNDRV_PCM_STATE_SETUP; + __snd_pcm_set_state(s->runtime, SNDRV_PCM_STATE_SETUP); if (card->support_grouping) { snd_printdd("%d group\n", s->number); diff --git a/sound/pci/asihpi/hpifunc.c b/sound/pci/asihpi/hpifunc.c index 1de05383126aabf9ef7c6286f37158ff0e56d494..24047fafef5187678e5f9dbd740b9817e500a7da 100644 --- a/sound/pci/asihpi/hpifunc.c +++ b/sound/pci/asihpi/hpifunc.c @@ -2020,7 +2020,6 @@ u16 hpi_meter_get_peak(u32 h_control, short an_peakdB[HPI_MAX_CHANNELS] HPI_CONTROL_GET_STATE); if (hpi_handle_indexes(h_control, &hm.adapter_index, &hm.obj_index)) return HPI_ERROR_INVALID_HANDLE; - hm.obj_index = hm.obj_index; hm.u.c.attribute = HPI_METER_PEAK; hpi_send_recv(&hm, &hr); diff --git a/sound/pci/asihpi/hpimsgx.c b/sound/pci/asihpi/hpimsgx.c index f7427f8eb6303e0d63c27940a9c96fb1f4c4e835..d0caef2994818efd1fe7162320075400cd94d525 100644 --- a/sound/pci/asihpi/hpimsgx.c +++ b/sound/pci/asihpi/hpimsgx.c @@ -93,11 +93,6 @@ static void HPIMSGX__cleanup(u16 adapter_index, void *h_owner); #pragma pack(push, 1) #endif -struct hpi_subsys_response { - struct hpi_response_header h; - struct hpi_subsys_res s; -}; - struct hpi_adapter_response { struct hpi_response_header h; struct hpi_adapter_res a; diff --git a/sound/pci/emu10k1/emupcm.c b/sound/pci/emu10k1/emupcm.c index b2701a4452d86dd15123b272b38c2c2b7b35dcb7..48af77ae8020f5a5c2ecca1c2bd974cc186f1779 100644 --- a/sound/pci/emu10k1/emupcm.c +++ b/sound/pci/emu10k1/emupcm.c @@ -124,7 +124,7 @@ static int snd_emu10k1_pcm_channel_alloc(struct snd_emu10k1_pcm * epcm, int voic epcm->voices[0]->epcm = epcm; if (voices > 1) { for (i = 1; i < voices; i++) { - epcm->voices[i] = &epcm->emu->voices[epcm->voices[0]->number + i]; + epcm->voices[i] = &epcm->emu->voices[(epcm->voices[0]->number + i) % NUM_G]; epcm->voices[i]->epcm = epcm; } } diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c index 15e2a0009080ee1ea1a22d518b7f95a729c645bf..e5f0549bf06d0b79ee5b1ad87f16aa76a6a83a7d 100644 --- a/sound/pci/hda/cs35l41_hda.c +++ b/sound/pci/hda/cs35l41_hda.c @@ -91,20 +91,18 @@ static const struct reg_sequence cs35l41_hda_mute[] = { { CS35L41_AMP_DIG_VOL_CTRL, 0x0000A678 }, // AMP_VOL_PCM Mute }; -static int cs35l41_control_add(struct cs_dsp_coeff_ctl *cs_ctl) +static void cs35l41_add_controls(struct cs35l41_hda *cs35l41) { - struct cs35l41_hda *cs35l41 = container_of(cs_ctl->dsp, struct cs35l41_hda, cs_dsp); struct hda_cs_dsp_ctl_info info; info.device_name = cs35l41->amp_name; info.fw_type = cs35l41->firmware_type; info.card = cs35l41->codec->card; - return hda_cs_dsp_control_add(cs_ctl, &info); + hda_cs_dsp_add_controls(&cs35l41->cs_dsp, &info); } static const struct cs_dsp_client_ops client_ops = { - .control_add = cs35l41_control_add, .control_remove = hda_cs_dsp_control_remove, }; @@ -435,6 +433,8 @@ static int cs35l41_init_dsp(struct cs35l41_hda *cs35l41) if (ret) goto err_release; + cs35l41_add_controls(cs35l41); + ret = cs35l41_save_calibration(cs35l41); err_release: @@ -461,9 +461,12 @@ static void cs35l41_remove_dsp(struct cs35l41_hda *cs35l41) struct cs_dsp *dsp = &cs35l41->cs_dsp; cancel_work_sync(&cs35l41->fw_load_work); + + mutex_lock(&cs35l41->fw_mutex); cs35l41_shutdown_dsp(cs35l41); cs_dsp_remove(dsp); cs35l41->halo_initialized = false; + mutex_unlock(&cs35l41->fw_mutex); } /* Protection release cycle to get the speaker out of Safe-Mode */ @@ -487,10 +490,10 @@ static void cs35l41_hda_playback_hook(struct device *dev, int action) struct regmap *reg = cs35l41->regmap; int ret = 0; - mutex_lock(&cs35l41->fw_mutex); - switch (action) { case HDA_GEN_PCM_ACT_OPEN: + pm_runtime_get_sync(dev); + mutex_lock(&cs35l41->fw_mutex); cs35l41->playback_started = true; if (cs35l41->firmware_running) { regmap_multi_reg_write(reg, cs35l41_hda_config_dsp, @@ -508,15 +511,21 @@ static void cs35l41_hda_playback_hook(struct device *dev, int action) CS35L41_AMP_EN_MASK, 1 << CS35L41_AMP_EN_SHIFT); if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST) regmap_write(reg, CS35L41_GPIO1_CTRL1, 0x00008001); + mutex_unlock(&cs35l41->fw_mutex); break; case HDA_GEN_PCM_ACT_PREPARE: + mutex_lock(&cs35l41->fw_mutex); ret = cs35l41_global_enable(reg, cs35l41->hw_cfg.bst_type, 1); + mutex_unlock(&cs35l41->fw_mutex); break; case HDA_GEN_PCM_ACT_CLEANUP: + mutex_lock(&cs35l41->fw_mutex); regmap_multi_reg_write(reg, cs35l41_hda_mute, ARRAY_SIZE(cs35l41_hda_mute)); ret = cs35l41_global_enable(reg, cs35l41->hw_cfg.bst_type, 0); + mutex_unlock(&cs35l41->fw_mutex); break; case HDA_GEN_PCM_ACT_CLOSE: + mutex_lock(&cs35l41->fw_mutex); ret = regmap_update_bits(reg, CS35L41_PWR_CTRL2, CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT); if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST) @@ -530,14 +539,16 @@ static void cs35l41_hda_playback_hook(struct device *dev, int action) } cs35l41_irq_release(cs35l41); cs35l41->playback_started = false; + mutex_unlock(&cs35l41->fw_mutex); + + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); break; default: dev_warn(cs35l41->dev, "Playback action not supported: %d\n", action); break; } - mutex_unlock(&cs35l41->fw_mutex); - if (ret) dev_err(cs35l41->dev, "Regmap access fail: %d\n", ret); } @@ -562,45 +573,148 @@ static int cs35l41_hda_channel_map(struct device *dev, unsigned int tx_num, unsi rx_slot); } +static void cs35l41_ready_for_reset(struct cs35l41_hda *cs35l41) +{ + mutex_lock(&cs35l41->fw_mutex); + if (cs35l41->firmware_running) { + + regcache_cache_only(cs35l41->regmap, false); + + cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap); + cs35l41_shutdown_dsp(cs35l41); + cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type); + + regcache_cache_only(cs35l41->regmap, true); + regcache_mark_dirty(cs35l41->regmap); + } + mutex_unlock(&cs35l41->fw_mutex); +} + +static int cs35l41_system_suspend(struct device *dev) +{ + struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev); + int ret; + + dev_dbg(cs35l41->dev, "System Suspend\n"); + + if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) { + dev_err(cs35l41->dev, "System Suspend not supported\n"); + return -EINVAL; + } + + ret = pm_runtime_force_suspend(dev); + if (ret) + return ret; + + /* Shutdown DSP before system suspend */ + cs35l41_ready_for_reset(cs35l41); + + /* + * Reset GPIO may be shared, so cannot reset here. + * However beyond this point, amps may be powered down. + */ + return 0; +} + +static int cs35l41_system_resume(struct device *dev) +{ + struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev); + int ret; + + dev_dbg(cs35l41->dev, "System Resume\n"); + + if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) { + dev_err(cs35l41->dev, "System Resume not supported\n"); + return -EINVAL; + } + + if (cs35l41->reset_gpio) { + usleep_range(2000, 2100); + gpiod_set_value_cansleep(cs35l41->reset_gpio, 1); + } + + usleep_range(2000, 2100); + + ret = pm_runtime_force_resume(dev); + + mutex_lock(&cs35l41->fw_mutex); + if (!ret && cs35l41->request_fw_load && !cs35l41->fw_request_ongoing) { + cs35l41->fw_request_ongoing = true; + schedule_work(&cs35l41->fw_load_work); + } + mutex_unlock(&cs35l41->fw_mutex); + + return ret; +} + static int cs35l41_runtime_suspend(struct device *dev) { struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev); + int ret = 0; - dev_dbg(cs35l41->dev, "Suspend\n"); + dev_dbg(cs35l41->dev, "Runtime Suspend\n"); - if (!cs35l41->firmware_running) + if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) { + dev_dbg(cs35l41->dev, "Runtime Suspend not supported\n"); return 0; + } - if (cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap, cs35l41->hw_cfg.bst_type) < 0) - return 0; + mutex_lock(&cs35l41->fw_mutex); + + if (cs35l41->playback_started) { + regmap_multi_reg_write(cs35l41->regmap, cs35l41_hda_mute, + ARRAY_SIZE(cs35l41_hda_mute)); + cs35l41_global_enable(cs35l41->regmap, cs35l41->hw_cfg.bst_type, 0); + regmap_update_bits(cs35l41->regmap, CS35L41_PWR_CTRL2, + CS35L41_AMP_EN_MASK, 0 << CS35L41_AMP_EN_SHIFT); + if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST) + regmap_write(cs35l41->regmap, CS35L41_GPIO1_CTRL1, 0x00000001); + regmap_update_bits(cs35l41->regmap, CS35L41_PWR_CTRL2, + CS35L41_VMON_EN_MASK | CS35L41_IMON_EN_MASK, + 0 << CS35L41_VMON_EN_SHIFT | 0 << CS35L41_IMON_EN_SHIFT); + cs35l41->playback_started = false; + } + + if (cs35l41->firmware_running) { + ret = cs35l41_enter_hibernate(cs35l41->dev, cs35l41->regmap, + cs35l41->hw_cfg.bst_type); + if (ret) + goto err; + } else { + cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type); + } regcache_cache_only(cs35l41->regmap, true); regcache_mark_dirty(cs35l41->regmap); - return 0; +err: + mutex_unlock(&cs35l41->fw_mutex); + + return ret; } static int cs35l41_runtime_resume(struct device *dev) { struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev); - int ret; + int ret = 0; - dev_dbg(cs35l41->dev, "Resume.\n"); + dev_dbg(cs35l41->dev, "Runtime Resume\n"); if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST_NO_VSPK_SWITCH) { - dev_dbg(cs35l41->dev, "System does not support Resume\n"); + dev_dbg(cs35l41->dev, "Runtime Resume not supported\n"); return 0; } - if (!cs35l41->firmware_running) - return 0; + mutex_lock(&cs35l41->fw_mutex); regcache_cache_only(cs35l41->regmap, false); - ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap); - if (ret) { - regcache_cache_only(cs35l41->regmap, true); - return ret; + if (cs35l41->firmware_running) { + ret = cs35l41_exit_hibernate(cs35l41->dev, cs35l41->regmap); + if (ret) { + dev_warn(cs35l41->dev, "Unable to exit Hibernate."); + goto err; + } } /* Test key needs to be unlocked to allow the OTP settings to re-apply */ @@ -609,26 +723,16 @@ static int cs35l41_runtime_resume(struct device *dev) cs35l41_test_key_lock(cs35l41->dev, cs35l41->regmap); if (ret) { dev_err(cs35l41->dev, "Failed to restore register cache: %d\n", ret); - return ret; + goto err; } if (cs35l41->hw_cfg.bst_type == CS35L41_EXT_BOOST) cs35l41_init_boost(cs35l41->dev, cs35l41->regmap, &cs35l41->hw_cfg); - return 0; -} - -static int cs35l41_hda_suspend_hook(struct device *dev) -{ - dev_dbg(dev, "Request Suspend\n"); - pm_runtime_mark_last_busy(dev); - return pm_runtime_put_autosuspend(dev); -} +err: + mutex_unlock(&cs35l41->fw_mutex); -static int cs35l41_hda_resume_hook(struct device *dev) -{ - dev_dbg(dev, "Request Resume\n"); - return pm_runtime_get_sync(dev); + return ret; } static int cs35l41_smart_amp(struct cs35l41_hda *cs35l41) @@ -678,8 +782,6 @@ clean_dsp: static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load) { - pm_runtime_get_sync(cs35l41->dev); - if (cs35l41->firmware_running && !load) { dev_dbg(cs35l41->dev, "Unloading Firmware\n"); cs35l41_shutdown_dsp(cs35l41); @@ -689,9 +791,6 @@ static void cs35l41_load_firmware(struct cs35l41_hda *cs35l41, bool load) } else { dev_dbg(cs35l41->dev, "Unable to Load firmware.\n"); } - - pm_runtime_mark_last_busy(cs35l41->dev); - pm_runtime_put_autosuspend(cs35l41->dev); } static int cs35l41_fw_load_ctl_get(struct snd_kcontrol *kcontrol, @@ -707,16 +806,21 @@ static void cs35l41_fw_load_work(struct work_struct *work) { struct cs35l41_hda *cs35l41 = container_of(work, struct cs35l41_hda, fw_load_work); + pm_runtime_get_sync(cs35l41->dev); + mutex_lock(&cs35l41->fw_mutex); /* Recheck if playback is ongoing, mutex will block playback during firmware loading */ if (cs35l41->playback_started) - dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback\n"); + dev_err(cs35l41->dev, "Cannot Load/Unload firmware during Playback. Retrying...\n"); else cs35l41_load_firmware(cs35l41, cs35l41->request_fw_load); cs35l41->fw_request_ongoing = false; mutex_unlock(&cs35l41->fw_mutex); + + pm_runtime_mark_last_busy(cs35l41->dev); + pm_runtime_put_autosuspend(cs35l41->dev); } static int cs35l41_fw_load_ctl_put(struct snd_kcontrol *kcontrol, @@ -840,10 +944,12 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas pm_runtime_get_sync(dev); + mutex_lock(&cs35l41->fw_mutex); + comps->dev = dev; if (!cs35l41->acpi_subsystem_id) - cs35l41->acpi_subsystem_id = devm_kasprintf(dev, GFP_KERNEL, "%.8x", - comps->codec->core.subsystem_id); + cs35l41->acpi_subsystem_id = kasprintf(GFP_KERNEL, "%.8x", + comps->codec->core.subsystem_id); cs35l41->codec = comps->codec; strscpy(comps->name, dev_name(dev), sizeof(comps->name)); @@ -852,10 +958,8 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas if (firmware_autostart) { dev_dbg(cs35l41->dev, "Firmware Autostart.\n"); cs35l41->request_fw_load = true; - mutex_lock(&cs35l41->fw_mutex); if (cs35l41_smart_amp(cs35l41) < 0) dev_warn(cs35l41->dev, "Cannot Run Firmware, reverting to dsp bypass...\n"); - mutex_unlock(&cs35l41->fw_mutex); } else { dev_dbg(cs35l41->dev, "Firmware Autostart is disabled.\n"); } @@ -863,8 +967,8 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas ret = cs35l41_create_controls(cs35l41); comps->playback_hook = cs35l41_hda_playback_hook; - comps->suspend_hook = cs35l41_hda_suspend_hook; - comps->resume_hook = cs35l41_hda_resume_hook; + + mutex_unlock(&cs35l41->fw_mutex); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); @@ -1048,36 +1152,6 @@ static int cs35l41_hda_apply_properties(struct cs35l41_hda *cs35l41) return cs35l41_hda_channel_map(cs35l41->dev, 0, NULL, 1, &hw_cfg->spk_pos); } -static int cs35l41_get_acpi_sub_string(struct device *dev, struct acpi_device *adev, - const char **subsysid) -{ - struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; - union acpi_object *obj; - acpi_status status; - int ret = 0; - - status = acpi_evaluate_object(adev->handle, "_SUB", NULL, &buffer); - if (ACPI_SUCCESS(status)) { - obj = buffer.pointer; - if (obj->type == ACPI_TYPE_STRING) { - *subsysid = devm_kstrdup(dev, obj->string.pointer, GFP_KERNEL); - if (*subsysid == NULL) { - dev_err(dev, "Cannot allocate Subsystem ID"); - ret = -ENOMEM; - } - } else { - dev_warn(dev, "Warning ACPI _SUB did not return a string\n"); - ret = -ENODEV; - } - acpi_os_free(buffer.pointer); - } else { - dev_dbg(dev, "Warning ACPI _SUB failed: %#x\n", status); - ret = -ENODEV; - } - - return ret; -} - static int cs35l41_get_speaker_id(struct device *dev, int amp_index, int num_amps, int fixed_gpio_id) { @@ -1154,7 +1228,6 @@ static int cs35l41_no_acpi_dsd(struct cs35l41_hda *cs35l41, struct device *physd hw_cfg->gpio2.func = CS35L41_INTERRUPT; hw_cfg->gpio2.valid = true; hw_cfg->valid = true; - put_device(physdev); if (strncmp(hid, "CLSA0100", 8) == 0) { hw_cfg->bst_type = CS35L41_EXT_BOOST_NO_VSPK_SWITCH; @@ -1183,6 +1256,7 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i u32 values[HDA_MAX_COMPONENTS]; struct acpi_device *adev; struct device *physdev; + const char *sub; char *property; size_t nval; int i, ret; @@ -1196,17 +1270,17 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i physdev = get_device(acpi_get_first_physical_node(adev)); acpi_dev_put(adev); - ret = cs35l41_get_acpi_sub_string(cs35l41->dev, adev, &cs35l41->acpi_subsystem_id); - if (ret) - dev_info(cs35l41->dev, "No Subsystem ID found in ACPI: %d", ret); - else - dev_dbg(cs35l41->dev, "Subsystem ID %s found", cs35l41->acpi_subsystem_id); + sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev)); + if (IS_ERR(sub)) + sub = NULL; + cs35l41->acpi_subsystem_id = sub; property = "cirrus,dev-index"; ret = device_property_count_u32(physdev, property); - if (ret <= 0) - return cs35l41_no_acpi_dsd(cs35l41, physdev, id, hid); - + if (ret <= 0) { + ret = cs35l41_no_acpi_dsd(cs35l41, physdev, id, hid); + goto err_put_physdev; + } if (ret > ARRAY_SIZE(values)) { ret = -EINVAL; goto err; @@ -1295,8 +1369,9 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i return 0; err: - put_device(physdev); dev_err(cs35l41->dev, "Failed property %s: %d\n", property, ret); +err_put_physdev: + put_device(physdev); return ret; } @@ -1433,6 +1508,7 @@ err: if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type)) gpiod_set_value_cansleep(cs35l41->reset_gpio, 0); gpiod_put(cs35l41->reset_gpio); + kfree(cs35l41->acpi_subsystem_id); return ret; } @@ -1455,11 +1531,13 @@ void cs35l41_hda_remove(struct device *dev) if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type)) gpiod_set_value_cansleep(cs35l41->reset_gpio, 0); gpiod_put(cs35l41->reset_gpio); + kfree(cs35l41->acpi_subsystem_id); } EXPORT_SYMBOL_NS_GPL(cs35l41_hda_remove, SND_HDA_SCODEC_CS35L41); const struct dev_pm_ops cs35l41_hda_pm_ops = { RUNTIME_PM_OPS(cs35l41_runtime_suspend, cs35l41_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(cs35l41_system_suspend, cs35l41_system_resume) }; EXPORT_SYMBOL_NS_GPL(cs35l41_hda_pm_ops, SND_HDA_SCODEC_CS35L41); diff --git a/sound/pci/hda/cs35l41_hda_i2c.c b/sound/pci/hda/cs35l41_hda_i2c.c index 5baacfde4f16b7e522c0866880975a779ede98e2..5a6252d9b9e1b1949064f99e70ce440297f1fba4 100644 --- a/sound/pci/hda/cs35l41_hda_i2c.c +++ b/sound/pci/hda/cs35l41_hda_i2c.c @@ -33,11 +33,9 @@ static int cs35l41_hda_i2c_probe(struct i2c_client *clt, const struct i2c_device devm_regmap_init_i2c(clt, &cs35l41_regmap_i2c)); } -static int cs35l41_hda_i2c_remove(struct i2c_client *clt) +static void cs35l41_hda_i2c_remove(struct i2c_client *clt) { cs35l41_hda_remove(&clt->dev); - - return 0; } static const struct i2c_device_id cs35l41_hda_i2c_id[] = { diff --git a/sound/pci/hda/hda_beep.c b/sound/pci/hda/hda_beep.c index 53a2b89f8983ce4f1e2733d62836f68f09668429..e63621bcb21427b6ecc1e5516f37ca38ac1cef2f 100644 --- a/sound/pci/hda/hda_beep.c +++ b/sound/pci/hda/hda_beep.c @@ -118,6 +118,12 @@ static int snd_hda_beep_event(struct input_dev *dev, unsigned int type, return 0; } +static void turn_on_beep(struct hda_beep *beep) +{ + if (beep->keep_power_at_enable) + snd_hda_power_up_pm(beep->codec); +} + static void turn_off_beep(struct hda_beep *beep) { cancel_work_sync(&beep->beep_work); @@ -125,6 +131,8 @@ static void turn_off_beep(struct hda_beep *beep) /* turn off beep */ generate_tone(beep, 0); } + if (beep->keep_power_at_enable) + snd_hda_power_down_pm(beep->codec); } /** @@ -140,7 +148,9 @@ int snd_hda_enable_beep_device(struct hda_codec *codec, int enable) enable = !!enable; if (beep->enabled != enable) { beep->enabled = enable; - if (!enable) + if (enable) + turn_on_beep(beep); + else turn_off_beep(beep); return 1; } @@ -167,7 +177,8 @@ static int beep_dev_disconnect(struct snd_device *device) input_unregister_device(beep->dev); else input_free_device(beep->dev); - turn_off_beep(beep); + if (beep->enabled) + turn_off_beep(beep); return 0; } diff --git a/sound/pci/hda/hda_beep.h b/sound/pci/hda/hda_beep.h index a25358a4807abfbc581e352b8ce63811da4f172b..db76e3ddba65445bc8e47257436d6f6cb51df142 100644 --- a/sound/pci/hda/hda_beep.h +++ b/sound/pci/hda/hda_beep.h @@ -25,6 +25,7 @@ struct hda_beep { unsigned int enabled:1; unsigned int linear_tone:1; /* linear tone for IDT/STAC codec */ unsigned int playing:1; + unsigned int keep_power_at_enable:1; /* set by driver */ struct work_struct beep_work; /* scheduled task for beep event */ struct mutex mutex; void (*power_hook)(struct hda_beep *beep, bool on); diff --git a/sound/pci/hda/hda_bind.c b/sound/pci/hda/hda_bind.c index cae9a975cbcca809bdd59993e1343eebd4c6ad76..1a868dd9dc4b6886a8dda83514052910e52014cc 100644 --- a/sound/pci/hda/hda_bind.c +++ b/sound/pci/hda/hda_bind.c @@ -157,10 +157,10 @@ static int hda_codec_driver_remove(struct device *dev) return codec->bus->core.ext_ops->hdev_detach(&codec->core); } - refcount_dec(&codec->pcm_ref); snd_hda_codec_disconnect_pcms(codec); snd_hda_jack_tbl_disconnect(codec); - wait_event(codec->remove_sleep, !refcount_read(&codec->pcm_ref)); + if (!refcount_dec_and_test(&codec->pcm_ref)) + wait_event(codec->remove_sleep, !refcount_read(&codec->pcm_ref)); snd_power_sync_ref(codec->bus->card); if (codec->patch_ops.free) diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c index 384426d7e9ddc7c94e6bae2fa3d1565f1b5b41ee..b4d1e658c556031c54a9970c7e9df310d49c383f 100644 --- a/sound/pci/hda/hda_codec.c +++ b/sound/pci/hda/hda_codec.c @@ -883,13 +883,7 @@ static void snd_hda_codec_dev_release(struct device *dev) snd_hda_sysfs_clear(codec); kfree(codec->modelname); kfree(codec->wcaps); - - /* - * In the case of ASoC HD-audio, hda_codec is device managed. - * It will be freed when the ASoC device is removed. - */ - if (codec->core.type == HDA_DEV_LEGACY) - kfree(codec); + kfree(codec); } #define DEV_NAME_LEN 31 @@ -931,8 +925,28 @@ snd_hda_codec_device_init(struct hda_bus *bus, unsigned int codec_addr, } codec->bus = bus; + codec->depop_delay = -1; + codec->fixup_id = HDA_FIXUP_ID_NOT_SET; + codec->core.dev.release = snd_hda_codec_dev_release; + codec->core.exec_verb = codec_exec_verb; codec->core.type = HDA_DEV_LEGACY; + mutex_init(&codec->spdif_mutex); + mutex_init(&codec->control_mutex); + snd_array_init(&codec->mixers, sizeof(struct hda_nid_item), 32); + snd_array_init(&codec->nids, sizeof(struct hda_nid_item), 32); + snd_array_init(&codec->init_pins, sizeof(struct hda_pincfg), 16); + snd_array_init(&codec->driver_pins, sizeof(struct hda_pincfg), 16); + snd_array_init(&codec->cvt_setups, sizeof(struct hda_cvt_setup), 8); + snd_array_init(&codec->spdif_out, sizeof(struct hda_spdif_out), 16); + snd_array_init(&codec->jacktbl, sizeof(struct hda_jack_tbl), 16); + snd_array_init(&codec->verbs, sizeof(struct hda_verb *), 8); + INIT_LIST_HEAD(&codec->conn_list); + INIT_LIST_HEAD(&codec->pcm_list_head); + INIT_DELAYED_WORK(&codec->jackpoll_work, hda_jackpoll_work); + refcount_set(&codec->pcm_ref, 1); + init_waitqueue_head(&codec->remove_sleep); + return codec; } EXPORT_SYMBOL_GPL(snd_hda_codec_device_init); @@ -985,29 +999,8 @@ int snd_hda_codec_device_new(struct hda_bus *bus, struct snd_card *card, if (snd_BUG_ON(codec_addr > HDA_MAX_CODEC_ADDRESS)) return -EINVAL; - codec->core.dev.release = snd_hda_codec_dev_release; - codec->core.exec_verb = codec_exec_verb; - codec->card = card; codec->addr = codec_addr; - mutex_init(&codec->spdif_mutex); - mutex_init(&codec->control_mutex); - snd_array_init(&codec->mixers, sizeof(struct hda_nid_item), 32); - snd_array_init(&codec->nids, sizeof(struct hda_nid_item), 32); - snd_array_init(&codec->init_pins, sizeof(struct hda_pincfg), 16); - snd_array_init(&codec->driver_pins, sizeof(struct hda_pincfg), 16); - snd_array_init(&codec->cvt_setups, sizeof(struct hda_cvt_setup), 8); - snd_array_init(&codec->spdif_out, sizeof(struct hda_spdif_out), 16); - snd_array_init(&codec->jacktbl, sizeof(struct hda_jack_tbl), 16); - snd_array_init(&codec->verbs, sizeof(struct hda_verb *), 8); - INIT_LIST_HEAD(&codec->conn_list); - INIT_LIST_HEAD(&codec->pcm_list_head); - refcount_set(&codec->pcm_ref, 1); - init_waitqueue_head(&codec->remove_sleep); - - INIT_DELAYED_WORK(&codec->jackpoll_work, hda_jackpoll_work); - codec->depop_delay = -1; - codec->fixup_id = HDA_FIXUP_ID_NOT_SET; #ifdef CONFIG_PM codec->power_jiffies = jiffies; diff --git a/sound/pci/hda/hda_component.h b/sound/pci/hda/hda_component.h index 1223621bd62ca8280bb1d03d679b9f3f25f39f0e..534e845b9cd1da5deb9b3d0ad96410f37dbb057c 100644 --- a/sound/pci/hda/hda_component.h +++ b/sound/pci/hda/hda_component.h @@ -16,6 +16,4 @@ struct hda_component { char name[HDA_MAX_NAME_SIZE]; struct hda_codec *codec; void (*playback_hook)(struct device *dev, int action); - int (*suspend_hook)(struct device *dev); - int (*resume_hook)(struct device *dev); }; diff --git a/sound/pci/hda/hda_controller.c b/sound/pci/hda/hda_controller.c index 75dcb14ff20adbd8bd0111930f68e8c6f168a335..0ff286b7b66bee3965012e69074baba863479a70 100644 --- a/sound/pci/hda/hda_controller.c +++ b/sound/pci/hda/hda_controller.c @@ -1033,10 +1033,8 @@ EXPORT_SYMBOL_GPL(azx_init_chip); void azx_stop_all_streams(struct azx *chip) { struct hdac_bus *bus = azx_bus(chip); - struct hdac_stream *s; - list_for_each_entry(s, &bus->stream_list, list) - snd_hdac_stream_stop(s); + snd_hdac_stop_streams(bus); } EXPORT_SYMBOL_GPL(azx_stop_all_streams); diff --git a/sound/pci/hda/hda_cs_dsp_ctl.c b/sound/pci/hda/hda_cs_dsp_ctl.c index 89ee549cb7d5011e2e7817e28cbae6d5711dd06b..1622a22f96f6a6b2492a00284c8dc80adcf0e835 100644 --- a/sound/pci/hda/hda_cs_dsp_ctl.c +++ b/sound/pci/hda/hda_cs_dsp_ctl.c @@ -97,7 +97,7 @@ static unsigned int wmfw_convert_flags(unsigned int in) return out; } -static int hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char *name) +static void hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char *name) { struct cs_dsp_coeff_ctl *cs_ctl = ctl->cs_ctl; struct snd_kcontrol_new kcontrol = {0}; @@ -107,7 +107,7 @@ static int hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char if (cs_ctl->len > ADSP_MAX_STD_CTRL_SIZE) { dev_err(cs_ctl->dsp->dev, "KControl %s: length %zu exceeds maximum %d\n", name, cs_ctl->len, ADSP_MAX_STD_CTRL_SIZE); - return -EINVAL; + return; } kcontrol.name = name; @@ -120,24 +120,21 @@ static int hda_cs_dsp_add_kcontrol(struct hda_cs_dsp_coeff_ctl *ctl, const char /* Save ctl inside private_data, ctl is owned by cs_dsp, * and will be freed when cs_dsp removes the control */ kctl = snd_ctl_new1(&kcontrol, (void *)ctl); - if (!kctl) { - ret = -ENOMEM; - return ret; - } + if (!kctl) + return; ret = snd_ctl_add(ctl->card, kctl); if (ret) { dev_err(cs_ctl->dsp->dev, "Failed to add KControl %s = %d\n", kcontrol.name, ret); - return ret; + return; } dev_dbg(cs_ctl->dsp->dev, "Added KControl: %s\n", kcontrol.name); ctl->kctl = kctl; - - return 0; } -int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ctl_info *info) +static void hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, + const struct hda_cs_dsp_ctl_info *info) { struct cs_dsp *cs_dsp = cs_ctl->dsp; char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; @@ -145,13 +142,10 @@ int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ct const char *region_name; int ret; - if (cs_ctl->flags & WMFW_CTL_FLAG_SYS) - return 0; - region_name = cs_dsp_mem_region_name(cs_ctl->alg_region.type); if (!region_name) { - dev_err(cs_dsp->dev, "Unknown region type: %d\n", cs_ctl->alg_region.type); - return -EINVAL; + dev_warn(cs_dsp->dev, "Unknown region type: %d\n", cs_ctl->alg_region.type); + return; } ret = scnprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s %s %.12s %x", info->device_name, @@ -171,22 +165,39 @@ int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ct ctl = kzalloc(sizeof(*ctl), GFP_KERNEL); if (!ctl) - return -ENOMEM; + return; ctl->cs_ctl = cs_ctl; ctl->card = info->card; cs_ctl->priv = ctl; - ret = hda_cs_dsp_add_kcontrol(ctl, name); - if (ret) { - dev_err(cs_dsp->dev, "Error (%d) adding control %s\n", ret, name); - kfree(ctl); - return ret; - } + hda_cs_dsp_add_kcontrol(ctl, name); +} - return 0; +void hda_cs_dsp_add_controls(struct cs_dsp *dsp, const struct hda_cs_dsp_ctl_info *info) +{ + struct cs_dsp_coeff_ctl *cs_ctl; + + /* + * pwr_lock would cause mutex inversion with ALSA control lock compared + * to the get/put functions. + * It is safe to walk the list without holding a mutex because entries + * are persistent and only cs_dsp_power_up() or cs_dsp_remove() can + * change the list. + */ + lockdep_assert_not_held(&dsp->pwr_lock); + + list_for_each_entry(cs_ctl, &dsp->ctl_list, list) { + if (cs_ctl->flags & WMFW_CTL_FLAG_SYS) + continue; + + if (cs_ctl->priv) + continue; + + hda_cs_dsp_control_add(cs_ctl, info); + } } -EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_control_add, SND_HDA_CS_DSP_CONTROLS); +EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_add_controls, SND_HDA_CS_DSP_CONTROLS); void hda_cs_dsp_control_remove(struct cs_dsp_coeff_ctl *cs_ctl) { @@ -203,19 +214,18 @@ int hda_cs_dsp_write_ctl(struct cs_dsp *dsp, const char *name, int type, struct hda_cs_dsp_coeff_ctl *ctl; int ret; + mutex_lock(&dsp->pwr_lock); cs_ctl = cs_dsp_get_ctl(dsp, name, type, alg); - if (!cs_ctl) - return -EINVAL; - - ctl = cs_ctl->priv; - ret = cs_dsp_coeff_write_ctrl(cs_ctl, 0, buf, len); + mutex_unlock(&dsp->pwr_lock); if (ret) return ret; if (cs_ctl->flags & WMFW_CTL_FLAG_SYS) return 0; + ctl = cs_ctl->priv; + snd_ctl_notify(ctl->card, SNDRV_CTL_EVENT_MASK_VALUE, &ctl->kctl->id); return 0; @@ -225,13 +235,14 @@ EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_write_ctl, SND_HDA_CS_DSP_CONTROLS); int hda_cs_dsp_read_ctl(struct cs_dsp *dsp, const char *name, int type, unsigned int alg, void *buf, size_t len) { - struct cs_dsp_coeff_ctl *cs_ctl; + int ret; - cs_ctl = cs_dsp_get_ctl(dsp, name, type, alg); - if (!cs_ctl) - return -EINVAL; + mutex_lock(&dsp->pwr_lock); + ret = cs_dsp_coeff_read_ctrl(cs_dsp_get_ctl(dsp, name, type, alg), 0, buf, len); + mutex_unlock(&dsp->pwr_lock); + + return ret; - return cs_dsp_coeff_read_ctrl(cs_ctl, 0, buf, len); } EXPORT_SYMBOL_NS_GPL(hda_cs_dsp_read_ctl, SND_HDA_CS_DSP_CONTROLS); diff --git a/sound/pci/hda/hda_cs_dsp_ctl.h b/sound/pci/hda/hda_cs_dsp_ctl.h index 4babc69cf2f0c86a00c30ddd56188b4a04823dca..2cf93359c4f230903028c986ff31c92378df68d3 100644 --- a/sound/pci/hda/hda_cs_dsp_ctl.h +++ b/sound/pci/hda/hda_cs_dsp_ctl.h @@ -29,7 +29,7 @@ struct hda_cs_dsp_ctl_info { extern const char * const hda_cs_dsp_fw_ids[HDA_CS_DSP_NUM_FW]; -int hda_cs_dsp_control_add(struct cs_dsp_coeff_ctl *cs_ctl, struct hda_cs_dsp_ctl_info *info); +void hda_cs_dsp_add_controls(struct cs_dsp *dsp, const struct hda_cs_dsp_ctl_info *info); void hda_cs_dsp_control_remove(struct cs_dsp_coeff_ctl *cs_ctl); int hda_cs_dsp_write_ctl(struct cs_dsp *dsp, const char *name, int type, unsigned int alg, const void *buf, size_t len); diff --git a/sound/pci/hda/hda_eld.c b/sound/pci/hda/hda_eld.c index 9e97443795f837323aef902fa3f16970eb3d9413..1d108ed5c6f2518a33db6e78ddaff2301319dbff 100644 --- a/sound/pci/hda/hda_eld.c +++ b/sound/pci/hda/hda_eld.c @@ -440,7 +440,8 @@ static void hdmi_print_sad_info(int i, struct cea_sad *a, } void snd_hdmi_print_eld_info(struct hdmi_eld *eld, - struct snd_info_buffer *buffer) + struct snd_info_buffer *buffer, + hda_nid_t pin_nid, int dev_id, hda_nid_t cvt_nid) { struct parsed_hdmi_eld *e = &eld->info; char buf[SND_PRINT_CHANNEL_ALLOCATION_ADVISED_BUFSIZE]; @@ -462,6 +463,9 @@ void snd_hdmi_print_eld_info(struct hdmi_eld *eld, snd_iprintf(buffer, "monitor_present\t\t%d\n", eld->monitor_present); snd_iprintf(buffer, "eld_valid\t\t%d\n", eld->eld_valid); + snd_iprintf(buffer, "codec_pin_nid\t\t0x%x\n", pin_nid); + snd_iprintf(buffer, "codec_dev_id\t\t0x%x\n", dev_id); + snd_iprintf(buffer, "codec_cvt_nid\t\t0x%x\n", cvt_nid); if (!eld->eld_valid) return; snd_iprintf(buffer, "monitor_name\t\t%s\n", e->monitor_name); diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index a77165bd92a983c35bfa56e6f0ca4407b1ff7aa1..6ff19dd0d10c261703ad21f9d612403f5f48de59 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -86,9 +86,6 @@ enum { #define INTEL_SCH_HDA_DEVC 0x78 #define INTEL_SCH_HDA_DEVC_NOSNOOP (0x1<<11) -/* Define VIA HD Audio Device ID*/ -#define VIA_HDAC_DEVICE_ID 0x3288 - /* max number of SDs */ /* ICH, ATI and VIA have 4 playback and 4 capture */ #define ICH6_NUM_CAPTURE 4 @@ -102,10 +99,6 @@ enum { #define ATIHDMI_NUM_CAPTURE 0 #define ATIHDMI_NUM_PLAYBACK 8 -/* TERA has 4 playback and 3 capture */ -#define TERA_NUM_CAPTURE 3 -#define TERA_NUM_PLAYBACK 4 - static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX; static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR; @@ -496,14 +489,14 @@ static int intel_ml_lctl_set_power(struct azx *chip, int state) * If other links are enabled for stream, they need similar fix */ val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); - val &= ~AZX_MLCTL_SPA; - val |= state << AZX_MLCTL_SPA_SHIFT; + val &= ~AZX_ML_LCTL_SPA; + val |= state << AZX_ML_LCTL_SPA_SHIFT; writel(val, bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); /* wait for CPA */ timeout = 50; while (timeout) { if (((readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL)) & - AZX_MLCTL_CPA) == (state << AZX_MLCTL_CPA_SHIFT)) + AZX_ML_LCTL_CPA) == (state << AZX_ML_LCTL_CPA_SHIFT)) return 0; timeout--; udelay(10); @@ -521,15 +514,15 @@ static void intel_init_lctl(struct azx *chip) /* 0. check lctl register value is correct or not */ val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); /* if SCF is already set, let's use it */ - if ((val & ML_LCTL_SCF_MASK) != 0) + if ((val & AZX_ML_LCTL_SCF) != 0) return; /* * Before operating on SPA, CPA must match SPA. * Any deviation may result in undefined behavior. */ - if (((val & AZX_MLCTL_SPA) >> AZX_MLCTL_SPA_SHIFT) != - ((val & AZX_MLCTL_CPA) >> AZX_MLCTL_CPA_SHIFT)) + if (((val & AZX_ML_LCTL_SPA) >> AZX_ML_LCTL_SPA_SHIFT) != + ((val & AZX_ML_LCTL_CPA) >> AZX_ML_LCTL_CPA_SHIFT)) return; /* 1. turn link down: set SPA to 0 and wait CPA to 0 */ @@ -539,7 +532,7 @@ static void intel_init_lctl(struct azx *chip) goto set_spa; /* 2. update SCF to select a properly audio clock*/ - val &= ~ML_LCTL_SCF_MASK; + val &= ~AZX_ML_LCTL_SCF; val |= intel_get_lctl_scf(chip); writel(val, bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); @@ -1817,7 +1810,7 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci, /* use the non-cached pages in non-snoop mode */ if (!azx_snoop(chip)) - azx_bus(chip)->dma_type = SNDRV_DMA_TYPE_DEV_WC; + azx_bus(chip)->dma_type = SNDRV_DMA_TYPE_DEV_WC_SG; if (chip->driver_type == AZX_DRIVER_NVIDIA) { dev_dbg(chip->card->dev, "Enable delay in RIRB handling\n"); @@ -2550,9 +2543,12 @@ static const struct pci_device_id azx_ids[] = { /* 5 Series/3400 */ { PCI_DEVICE(0x8086, 0x3b56), .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_NOPM }, + { PCI_DEVICE(0x8086, 0x3b57), + .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_NOPM }, /* Poulsbo */ { PCI_DEVICE(0x8086, 0x811b), - .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE }, + .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE | + AZX_DCAPS_POSFIX_LPIB }, /* Oaktrail */ { PCI_DEVICE(0x8086, 0x080a), .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE }, diff --git a/sound/pci/hda/hda_local.h b/sound/pci/hda/hda_local.h index 682dca2057dbe39b3a3b2e7105f5f3acb1bec296..53a5a62b78fa9823d1658839c5fe810f451f8f60 100644 --- a/sound/pci/hda/hda_local.h +++ b/sound/pci/hda/hda_local.h @@ -712,7 +712,8 @@ int snd_hdmi_get_eld_ati(struct hda_codec *codec, hda_nid_t nid, #ifdef CONFIG_SND_PROC_FS void snd_hdmi_print_eld_info(struct hdmi_eld *eld, - struct snd_info_buffer *buffer); + struct snd_info_buffer *buffer, + hda_nid_t pin_nid, int dev_id, hda_nid_t cvt_nid); void snd_hdmi_write_eld_info(struct hdmi_eld *eld, struct snd_info_buffer *buffer); #endif diff --git a/sound/pci/hda/hda_sysfs.c b/sound/pci/hda/hda_sysfs.c index bf951c10ae61b0819676e9134237002dda9cba04..69ebc37a4d6f3af470e9c4a2ed27de95af7dc42f 100644 --- a/sound/pci/hda/hda_sysfs.c +++ b/sound/pci/hda/hda_sysfs.c @@ -375,8 +375,6 @@ static ssize_t user_pin_configs_show(struct device *dev, return pin_configs_show(codec, &codec->user_pins, buf); } -#define MAX_PIN_CONFIGS 32 - static int parse_user_pin_configs(struct hda_codec *codec, const char *buf) { int nid, cfg, err; diff --git a/sound/pci/hda/hda_tegra.c b/sound/pci/hda/hda_tegra.c index 7debb2c76aa62b778cec1d2c110694d4a57c91dc..976a112c7d0061c932c82310b376bacf381fc2c8 100644 --- a/sound/pci/hda/hda_tegra.c +++ b/sound/pci/hda/hda_tegra.c @@ -474,7 +474,8 @@ MODULE_DEVICE_TABLE(of, hda_tegra_match); static int hda_tegra_probe(struct platform_device *pdev) { const unsigned int driver_flags = AZX_DCAPS_CORBRP_SELF_CLEAR | - AZX_DCAPS_PM_RUNTIME; + AZX_DCAPS_PM_RUNTIME | + AZX_DCAPS_4K_BDLE_BOUNDARY; struct snd_card *card; struct azx *chip; struct hda_tegra *hda; diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c index 208933792787dd27d480db2b4b75fb050360c000..9580fe00cbd92c3ff5eb7308559bef33b3829d9e 100644 --- a/sound/pci/hda/patch_ca0132.c +++ b/sound/pci/hda/patch_ca0132.c @@ -2962,7 +2962,6 @@ static int dsp_allocate_ports_format(struct hda_codec *codec, const unsigned short fmt, unsigned int *port_map) { - int status; unsigned int num_chans; unsigned int sample_rate_div = ((get_hdafmt_rate(fmt) >> 0) & 3) + 1; @@ -2976,9 +2975,7 @@ static int dsp_allocate_ports_format(struct hda_codec *codec, num_chans = get_hdafmt_chs(fmt) + 1; - status = dsp_allocate_ports(codec, num_chans, rate_multi, port_map); - - return status; + return dsp_allocate_ports(codec, num_chans, rate_multi, port_map); } /* diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c index 6c209cd26c0cab1ed9e835e118e6bcfaf37a7f05..21edf7a619f07d97afa7e057527ec6cf058ede04 100644 --- a/sound/pci/hda/patch_hdmi.c +++ b/sound/pci/hda/patch_hdmi.c @@ -53,7 +53,8 @@ MODULE_PARM_DESC(enable_all_pins, "Forcibly enable all pins"); struct hdmi_spec_per_cvt { hda_nid_t cvt_nid; - int assigned; + bool assigned; /* the stream has been assigned */ + bool silent_stream; /* silent stream activated */ unsigned int channels_min; unsigned int channels_max; u32 rates; @@ -150,7 +151,7 @@ struct hdmi_spec { */ int dev_num; struct snd_array pins; /* struct hdmi_spec_per_pin */ - struct hdmi_pcm pcm_rec[16]; + struct hdmi_pcm pcm_rec[8]; struct mutex pcm_lock; struct mutex bind_lock; /* for audio component binding */ /* pcm_bitmap means which pcms have been assigned to pins*/ @@ -166,10 +167,10 @@ struct hdmi_spec { struct hdmi_ops ops; bool dyn_pin_out; - bool dyn_pcm_assign; - bool dyn_pcm_no_legacy; /* hdmi interrupt trigger control flag for Nvidia codec */ bool hdmi_intr_trig_ctrl; + bool nv_dp_workaround; /* workaround DP audio infoframe for Nvidia */ + bool intel_hsw_fixup; /* apply Intel platform-specific fixups */ /* * Non-generic VIA/NVIDIA specific @@ -229,7 +230,7 @@ struct dp_audio_infoframe { union audio_infoframe { struct hdmi_audio_infoframe hdmi; struct dp_audio_infoframe dp; - u8 bytes[0]; + DECLARE_FLEX_ARRAY(u8, bytes); }; /* @@ -493,7 +494,8 @@ static void print_eld_info(struct snd_info_entry *entry, struct hdmi_spec_per_pin *per_pin = entry->private_data; mutex_lock(&per_pin->lock); - snd_hdmi_print_eld_info(&per_pin->sink_eld, buffer); + snd_hdmi_print_eld_info(&per_pin->sink_eld, buffer, per_pin->pin_nid, + per_pin->dev_id, per_pin->cvt_nid); mutex_unlock(&per_pin->lock); } @@ -679,15 +681,24 @@ static void hdmi_pin_setup_infoframe(struct hda_codec *codec, int ca, int active_channels, int conn_type) { + struct hdmi_spec *spec = codec->spec; union audio_infoframe ai; memset(&ai, 0, sizeof(ai)); - if (conn_type == 0) { /* HDMI */ + if ((conn_type == 0) || /* HDMI */ + /* Nvidia DisplayPort: Nvidia HW expects same layout as HDMI */ + (conn_type == 1 && spec->nv_dp_workaround)) { struct hdmi_audio_infoframe *hdmi_ai = &ai.hdmi; - hdmi_ai->type = 0x84; - hdmi_ai->ver = 0x01; - hdmi_ai->len = 0x0a; + if (conn_type == 0) { /* HDMI */ + hdmi_ai->type = 0x84; + hdmi_ai->ver = 0x01; + hdmi_ai->len = 0x0a; + } else {/* Nvidia DP */ + hdmi_ai->type = 0x84; + hdmi_ai->ver = 0x1b; + hdmi_ai->len = 0x11 << 2; + } hdmi_ai->CC02_CT47 = active_channels - 1; hdmi_ai->CA = ca; hdmi_checksum_audio_infoframe(hdmi_ai); @@ -977,7 +988,8 @@ static int hdmi_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid, * of the pin. */ static int hdmi_choose_cvt(struct hda_codec *codec, - int pin_idx, int *cvt_id) + int pin_idx, int *cvt_id, + bool silent) { struct hdmi_spec *spec = codec->spec; struct hdmi_spec_per_pin *per_pin; @@ -992,6 +1004,9 @@ static int hdmi_choose_cvt(struct hda_codec *codec, if (per_pin && per_pin->silent_stream) { cvt_idx = cvt_nid_to_cvt_index(codec, per_pin->cvt_nid); + per_cvt = get_cvt(spec, cvt_idx); + if (per_cvt->assigned && !silent) + return -EBUSY; if (cvt_id) *cvt_id = cvt_idx; return 0; @@ -1002,7 +1017,7 @@ static int hdmi_choose_cvt(struct hda_codec *codec, per_cvt = get_cvt(spec, cvt_idx); /* Must not already be assigned */ - if (per_cvt->assigned) + if (per_cvt->assigned || per_cvt->silent_stream) continue; if (per_pin == NULL) break; @@ -1171,9 +1186,7 @@ static void pin_cvt_fixup(struct hda_codec *codec, spec->ops.pin_cvt_fixup(codec, per_pin, cvt_nid); } -/* called in hdmi_pcm_open when no pin is assigned to the PCM - * in dyn_pcm_assign mode. - */ +/* called in hdmi_pcm_open when no pin is assigned to the PCM */ static int hdmi_pcm_open_no_pin(struct hda_pcm_stream *hinfo, struct hda_codec *codec, struct snd_pcm_substream *substream) @@ -1188,12 +1201,12 @@ static int hdmi_pcm_open_no_pin(struct hda_pcm_stream *hinfo, if (pcm_idx < 0) return -EINVAL; - err = hdmi_choose_cvt(codec, -1, &cvt_idx); + err = hdmi_choose_cvt(codec, -1, &cvt_idx, false); if (err) return err; per_cvt = get_cvt(spec, cvt_idx); - per_cvt->assigned = 1; + per_cvt->assigned = true; hinfo->nid = per_cvt->cvt_nid; pin_cvt_fixup(codec, NULL, per_cvt->cvt_nid); @@ -1241,28 +1254,21 @@ static int hdmi_pcm_open(struct hda_pcm_stream *hinfo, mutex_lock(&spec->pcm_lock); pin_idx = hinfo_to_pin_index(codec, hinfo); - if (!spec->dyn_pcm_assign) { - if (snd_BUG_ON(pin_idx < 0)) { - err = -EINVAL; - goto unlock; - } - } else { - /* no pin is assigned to the PCM - * PA need pcm open successfully when probe - */ - if (pin_idx < 0) { - err = hdmi_pcm_open_no_pin(hinfo, codec, substream); - goto unlock; - } + /* no pin is assigned to the PCM + * PA need pcm open successfully when probe + */ + if (pin_idx < 0) { + err = hdmi_pcm_open_no_pin(hinfo, codec, substream); + goto unlock; } - err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx); + err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx, false); if (err < 0) goto unlock; per_cvt = get_cvt(spec, cvt_idx); /* Claim converter */ - per_cvt->assigned = 1; + per_cvt->assigned = true; set_bit(pcm_idx, &spec->pcm_in_use); per_pin = get_pin(spec, pin_idx); @@ -1296,7 +1302,7 @@ static int hdmi_pcm_open(struct hda_pcm_stream *hinfo, snd_hdmi_eld_update_pcm_info(&eld->info, hinfo); if (hinfo->channels_min > hinfo->channels_max || !hinfo->rates || !hinfo->formats) { - per_cvt->assigned = 0; + per_cvt->assigned = false; hinfo->nid = 0; snd_hda_spdif_ctls_unassign(codec, pcm_idx); err = -ENODEV; @@ -1358,43 +1364,6 @@ static int hdmi_find_pcm_slot(struct hdmi_spec *spec, { int i; - /* on the new machines, try to assign the pcm slot dynamically, - * not use the preferred fixed map (legacy way) anymore. - */ - if (spec->dyn_pcm_no_legacy) - goto last_try; - - /* - * generic_hdmi_build_pcms() may allocate extra PCMs on some - * platforms (with maximum of 'num_nids + dev_num - 1') - * - * The per_pin of pin_nid_idx=n and dev_id=m prefers to get pcm-n - * if m==0. This guarantees that dynamic pcm assignments are compatible - * with the legacy static per_pin-pcm assignment that existed in the - * days before DP-MST. - * - * Intel DP-MST prefers this legacy behavior for compatibility, too. - * - * per_pin of m!=0 prefers to get pcm=(num_nids + (m - 1)). - */ - - if (per_pin->dev_id == 0 || spec->intel_hsw_fixup) { - if (!test_bit(per_pin->pin_nid_idx, &spec->pcm_bitmap)) - return per_pin->pin_nid_idx; - } else { - i = spec->num_nids + (per_pin->dev_id - 1); - if (i < spec->pcm_used && !(test_bit(i, &spec->pcm_bitmap))) - return i; - } - - /* have a second try; check the area over num_nids */ - for (i = spec->num_nids; i < spec->pcm_used; i++) { - if (!test_bit(i, &spec->pcm_bitmap)) - return i; - } - - last_try: - /* the last try; check the empty slots in pins */ for (i = 0; i < spec->pcm_used; i++) { if (!test_bit(i, &spec->pcm_bitmap)) return i; @@ -1456,10 +1425,9 @@ static void hdmi_pcm_setup_pin(struct hdmi_spec *spec, int mux_idx; bool non_pcm; - if (per_pin->pcm_idx >= 0 && per_pin->pcm_idx < spec->pcm_used) - pcm = get_pcm_rec(spec, per_pin->pcm_idx); - else + if (per_pin->pcm_idx < 0 || per_pin->pcm_idx >= spec->pcm_used) return; + pcm = get_pcm_rec(spec, per_pin->pcm_idx); if (!pcm->pcm) return; if (!test_bit(per_pin->pcm_idx, &spec->pcm_in_use)) @@ -1557,14 +1525,12 @@ static void update_eld(struct hda_codec *codec, */ pcm_jack = pin_idx_to_pcm_jack(codec, per_pin); - if (spec->dyn_pcm_assign) { - if (eld->eld_valid) { - hdmi_attach_hda_pcm(spec, per_pin); - hdmi_pcm_setup_pin(spec, per_pin); - } else { - hdmi_pcm_reset_pin(spec, per_pin); - hdmi_detach_hda_pcm(spec, per_pin); - } + if (eld->eld_valid) { + hdmi_attach_hda_pcm(spec, per_pin); + hdmi_pcm_setup_pin(spec, per_pin); + } else { + hdmi_pcm_reset_pin(spec, per_pin); + hdmi_detach_hda_pcm(spec, per_pin); } /* if pcm_idx == -1, it means this is in monitor connection event * we can get the correct pcm_idx now. @@ -1748,14 +1714,14 @@ static void silent_stream_enable(struct hda_codec *codec, } pin_idx = pin_id_to_pin_index(codec, per_pin->pin_nid, per_pin->dev_id); - err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx); + err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx, true); if (err) { codec_err(codec, "hdmi: no free converter to enable silent mode\n"); goto unlock_out; } per_cvt = get_cvt(spec, cvt_idx); - per_cvt->assigned = 1; + per_cvt->silent_stream = true; per_pin->cvt_nid = per_cvt->cvt_nid; per_pin->silent_stream = true; @@ -1815,7 +1781,7 @@ static void silent_stream_disable(struct hda_codec *codec, cvt_idx = cvt_nid_to_cvt_index(codec, per_pin->cvt_nid); if (cvt_idx >= 0 && cvt_idx < spec->num_cvts) { per_cvt = get_cvt(spec, cvt_idx); - per_cvt->assigned = 0; + per_cvt->silent_stream = false; } if (spec->silent_stream_type == SILENT_STREAM_I915) { @@ -1926,7 +1892,7 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid) * structures based on worst case. */ dev_num = spec->dev_num; - } else if (spec->dyn_pcm_assign && codec->dp_mst) { + } else if (codec->dp_mst) { dev_num = snd_hda_get_num_devices(codec, pin_nid) + 1; /* * spec->dev_num is the maxinum number of device entries @@ -1951,13 +1917,8 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid) if (!per_pin) return -ENOMEM; - if (spec->dyn_pcm_assign) { - per_pin->pcm = NULL; - per_pin->pcm_idx = -1; - } else { - per_pin->pcm = get_hdmi_pcm(spec, pin_idx); - per_pin->pcm_idx = pin_idx; - } + per_pin->pcm = NULL; + per_pin->pcm_idx = -1; per_pin->pin_nid = pin_nid; per_pin->pin_nid_idx = spec->num_nids; per_pin->dev_id = i; @@ -1966,6 +1927,8 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid) err = hdmi_read_pin_conn(codec, pin_idx); if (err < 0) return err; + if (!is_jack_detectable(codec, pin_nid)) + codec_warn(codec, "HDMI: pin NID 0x%x - jack not detectable\n", pin_nid); spec->num_pins++; } spec->num_nids++; @@ -2113,10 +2076,9 @@ static int generic_hdmi_playback_pcm_prepare(struct hda_pcm_stream *hinfo, mutex_lock(&spec->pcm_lock); pin_idx = hinfo_to_pin_index(codec, hinfo); - if (spec->dyn_pcm_assign && pin_idx < 0) { - /* when dyn_pcm_assign and pcm is not bound to a pin - * skip pin setup and return 0 to make audio playback - * be ongoing + if (pin_idx < 0) { + /* when pcm is not bound to a pin skip pin setup and return 0 + * to make audio playback be ongoing */ pin_cvt_fixup(codec, NULL, cvt_nid); snd_hda_codec_setup_stream(codec, cvt_nid, @@ -2211,7 +2173,7 @@ static int hdmi_pcm_close(struct hda_pcm_stream *hinfo, goto unlock; } per_cvt = get_cvt(spec, cvt_idx); - per_cvt->assigned = 0; + per_cvt->assigned = false; hinfo->nid = 0; azx_stream(get_azx_dev(substream))->stripe = 0; @@ -2219,7 +2181,7 @@ static int hdmi_pcm_close(struct hda_pcm_stream *hinfo, snd_hda_spdif_ctls_unassign(codec, pcm_idx); clear_bit(pcm_idx, &spec->pcm_in_use); pin_idx = hinfo_to_pin_index(codec, hinfo); - if (spec->dyn_pcm_assign && pin_idx < 0) + if (pin_idx < 0) goto unlock; if (snd_BUG_ON(pin_idx < 0)) { @@ -2317,21 +2279,8 @@ static int generic_hdmi_build_pcms(struct hda_codec *codec) struct hdmi_spec *spec = codec->spec; int idx, pcm_num; - /* - * for non-mst mode, pcm number is the same as before - * for DP MST mode without extra PCM, pcm number is same - * for DP MST mode with extra PCMs, pcm number is - * (nid number + dev_num - 1) - * dev_num is the device entry number in a pin - */ - - if (spec->dyn_pcm_no_legacy && codec->mst_no_extra_pcms) - pcm_num = spec->num_cvts; - else if (codec->mst_no_extra_pcms) - pcm_num = spec->num_nids; - else - pcm_num = spec->num_nids + spec->dev_num - 1; - + /* limit the PCM devices to the codec converters */ + pcm_num = spec->num_cvts; codec_dbg(codec, "hdmi: pcm_num set to %d\n", pcm_num); for (idx = 0; idx < pcm_num; idx++) { @@ -2350,8 +2299,8 @@ static int generic_hdmi_build_pcms(struct hda_codec *codec) pstr = &info->stream[SNDRV_PCM_STREAM_PLAYBACK]; pstr->substreams = 1; pstr->ops = generic_ops; - /* pcm number is less than 16 */ - if (spec->pcm_used >= 16) + /* pcm number is less than pcm_rec array size */ + if (spec->pcm_used >= ARRAY_SIZE(spec->pcm_rec)) break; /* other pstr fields are set in open */ } @@ -2370,17 +2319,12 @@ static int generic_hdmi_build_jack(struct hda_codec *codec, int pcm_idx) { char hdmi_str[32] = "HDMI/DP"; struct hdmi_spec *spec = codec->spec; - struct hdmi_spec_per_pin *per_pin = get_pin(spec, pcm_idx); struct snd_jack *jack; int pcmdev = get_pcm_rec(spec, pcm_idx)->device; int err; if (pcmdev > 0) sprintf(hdmi_str + strlen(hdmi_str), ",pcm=%d", pcmdev); - if (!spec->dyn_pcm_assign && - !is_jack_detectable(codec, per_pin->pin_nid)) - strncat(hdmi_str, " Phantom", - sizeof(hdmi_str) - strlen(hdmi_str) - 1); err = snd_jack_new(codec->card, hdmi_str, SND_JACK_AVOUT, &jack, true, false); @@ -2413,18 +2357,9 @@ static int generic_hdmi_build_controls(struct hda_codec *codec) /* create the spdif for each pcm * pin will be bound when monitor is connected */ - if (spec->dyn_pcm_assign) - err = snd_hda_create_dig_out_ctls(codec, + err = snd_hda_create_dig_out_ctls(codec, 0, spec->cvt_nids[0], HDA_PCM_TYPE_HDMI); - else { - struct hdmi_spec_per_pin *per_pin = - get_pin(spec, pcm_idx); - err = snd_hda_create_dig_out_ctls(codec, - per_pin->pin_nid, - per_pin->mux_nids[0], - HDA_PCM_TYPE_HDMI); - } if (err < 0) return err; snd_hda_spdif_ctls_unassign(codec, pcm_idx); @@ -2544,11 +2479,7 @@ static void generic_hdmi_free(struct hda_codec *codec) for (pcm_idx = 0; pcm_idx < spec->pcm_used; pcm_idx++) { if (spec->pcm_rec[pcm_idx].jack == NULL) continue; - if (spec->dyn_pcm_assign) - snd_device_free(codec->card, - spec->pcm_rec[pcm_idx].jack); - else - spec->pcm_rec[pcm_idx].jack = NULL; + snd_device_free(codec->card, spec->pcm_rec[pcm_idx].jack); } generic_spec_free(codec); @@ -2735,9 +2666,6 @@ static void generic_acomp_pin_eld_notify(void *audio_ptr, int port, int dev_id) */ if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) return; - /* ditto during suspend/resume process itself */ - if (snd_hdac_is_in_pm(&codec->core)) - return; check_presence_and_report(codec, pin_nid, dev_id); } @@ -2921,9 +2849,6 @@ static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe) */ if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND) return; - /* ditto during suspend/resume process itself */ - if (snd_hdac_is_in_pm(&codec->core)) - return; snd_hdac_i915_set_bclk(&codec->bus->core); check_presence_and_report(codec, pin_nid, dev_id); @@ -3028,7 +2953,6 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, return err; spec = codec->spec; codec->dp_mst = true; - spec->dyn_pcm_assign = true; spec->vendor_nid = vendor_nid; spec->port_map = port_map; spec->port_num = port_num; @@ -3092,17 +3016,9 @@ static int patch_i915_tgl_hdmi(struct hda_codec *codec) * the index indicate the port number. */ static const int map[] = {0x4, 0x6, 0x8, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf}; - int ret; - - ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4, - enable_silent_stream); - if (!ret) { - struct hdmi_spec *spec = codec->spec; - spec->dyn_pcm_no_legacy = true; - } - - return ret; + return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4, + enable_silent_stream); } static int patch_i915_adlp_hdmi(struct hda_codec *codec) @@ -3617,6 +3533,7 @@ static int patch_nvhdmi_2ch(struct hda_codec *codec) spec->pcm_playback.rates = SUPPORTED_RATES; spec->pcm_playback.maxbps = SUPPORTED_MAXBPS; spec->pcm_playback.formats = SUPPORTED_FORMATS; + spec->nv_dp_workaround = true; return 0; } @@ -3741,7 +3658,6 @@ static int patch_nvhdmi(struct hda_codec *codec) codec->dp_mst = true; spec = codec->spec; - spec->dyn_pcm_assign = true; err = hdmi_parse_codec(codec); if (err < 0) { @@ -3756,6 +3672,7 @@ static int patch_nvhdmi(struct hda_codec *codec) spec->chmap.ops.chmap_cea_alloc_validate_get_type = nvhdmi_chmap_cea_alloc_validate_get_type; spec->chmap.ops.chmap_validate = nvhdmi_chmap_validate; + spec->nv_dp_workaround = true; codec->link_down_at_suspend = 1; @@ -3779,6 +3696,7 @@ static int patch_nvhdmi_legacy(struct hda_codec *codec) spec->chmap.ops.chmap_cea_alloc_validate_get_type = nvhdmi_chmap_cea_alloc_validate_get_type; spec->chmap.ops.chmap_validate = nvhdmi_chmap_validate; + spec->nv_dp_workaround = true; codec->link_down_at_suspend = 1; @@ -3984,6 +3902,7 @@ static int tegra_hdmi_init(struct hda_codec *codec) generic_hdmi_init_per_pins(codec); + codec->depop_delay = 10; codec->patch_ops.build_pcms = tegra_hdmi_build_pcms; spec->chmap.ops.chmap_cea_alloc_validate_get_type = nvhdmi_chmap_cea_alloc_validate_get_type; @@ -3992,6 +3911,7 @@ static int tegra_hdmi_init(struct hda_codec *codec) spec->chmap.ops.chmap_cea_alloc_validate_get_type = nvhdmi_chmap_cea_alloc_validate_get_type; spec->chmap.ops.chmap_validate = nvhdmi_chmap_validate; + spec->nv_dp_workaround = true; return 0; } @@ -4017,10 +3937,8 @@ static int patch_tegra234_hdmi(struct hda_codec *codec) return err; codec->dp_mst = true; - codec->mst_no_extra_pcms = true; spec = codec->spec; spec->dyn_pin_out = true; - spec->dyn_pcm_assign = true; spec->hdmi_intr_trig_ctrl = true; return tegra_hdmi_init(codec); diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 47e72cf76608eb5e6297e2186c1a8f1fe14d322a..e6c4bb5fa041a179956dad4b3b7bf93b9752e92f 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -18,6 +18,7 @@ #include <linux/module.h> #include <linux/input.h> #include <linux/leds.h> +#include <linux/ctype.h> #include <sound/core.h> #include <sound/jack.h> #include <sound/hda_codec.h> @@ -4021,22 +4022,16 @@ static void alc5505_dsp_init(struct hda_codec *codec) static int alc269_suspend(struct hda_codec *codec) { struct alc_spec *spec = codec->spec; - int i; if (spec->has_alc5505_dsp) alc5505_dsp_suspend(codec); - for (i = 0; i < HDA_MAX_COMPONENTS; i++) - if (spec->comps[i].suspend_hook) - spec->comps[i].suspend_hook(spec->comps[i].dev); - return alc_suspend(codec); } static int alc269_resume(struct hda_codec *codec) { struct alc_spec *spec = codec->spec; - int i; if (spec->codec_variant == ALC269_TYPE_ALC269VB) alc269vb_toggle_power_output(codec, 0); @@ -4067,10 +4062,6 @@ static int alc269_resume(struct hda_codec *codec) if (spec->has_alc5505_dsp) alc5505_dsp_resume(codec); - for (i = 0; i < HDA_MAX_COMPONENTS; i++) - if (spec->comps[i].resume_hook) - spec->comps[i].resume_hook(spec->comps[i].dev); - return 0; } #endif /* CONFIG_PM */ @@ -4700,6 +4691,48 @@ static void alc236_fixup_hp_mute_led_micmute_vref(struct hda_codec *codec, alc236_fixup_hp_micmute_led_vref(codec, fix, action); } +static inline void alc298_samsung_write_coef_pack(struct hda_codec *codec, + const unsigned short coefs[2]) +{ + alc_write_coef_idx(codec, 0x23, coefs[0]); + alc_write_coef_idx(codec, 0x25, coefs[1]); + alc_write_coef_idx(codec, 0x26, 0xb011); +} + +struct alc298_samsung_amp_desc { + unsigned char nid; + unsigned short init_seq[2][2]; +}; + +static void alc298_fixup_samsung_amp(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + int i, j; + static const unsigned short init_seq[][2] = { + { 0x19, 0x00 }, { 0x20, 0xc0 }, { 0x22, 0x44 }, { 0x23, 0x08 }, + { 0x24, 0x85 }, { 0x25, 0x41 }, { 0x35, 0x40 }, { 0x36, 0x01 }, + { 0x38, 0x81 }, { 0x3a, 0x03 }, { 0x3b, 0x81 }, { 0x40, 0x3e }, + { 0x41, 0x07 }, { 0x400, 0x1 } + }; + static const struct alc298_samsung_amp_desc amps[] = { + { 0x3a, { { 0x18, 0x1 }, { 0x26, 0x0 } } }, + { 0x39, { { 0x18, 0x2 }, { 0x26, 0x1 } } } + }; + + if (action != HDA_FIXUP_ACT_INIT) + return; + + for (i = 0; i < ARRAY_SIZE(amps); i++) { + alc_write_coef_idx(codec, 0x22, amps[i].nid); + + for (j = 0; j < ARRAY_SIZE(amps[i].init_seq); j++) + alc298_samsung_write_coef_pack(codec, amps[i].init_seq[j]); + + for (j = 0; j < ARRAY_SIZE(init_seq); j++) + alc298_samsung_write_coef_pack(codec, init_seq[j]); + } +} + #if IS_REACHABLE(CONFIG_INPUT) static void gpio2_mic_hotkey_event(struct hda_codec *codec, struct hda_jack_callback *event) @@ -6621,19 +6654,12 @@ static int comp_bind(struct device *dev) { struct hda_codec *cdc = dev_to_hda_codec(dev); struct alc_spec *spec = cdc->spec; - int ret, i; + int ret; ret = component_bind_all(dev, spec->comps); if (ret) return ret; - if (snd_hdac_is_power_on(&cdc->core)) { - codec_dbg(cdc, "Resuming after bind.\n"); - for (i = 0; i < HDA_MAX_COMPONENTS; i++) - if (spec->comps[i].resume_hook) - spec->comps[i].resume_hook(spec->comps[i].dev); - } - return 0; } @@ -6662,23 +6688,51 @@ static void comp_generic_playback_hook(struct hda_pcm_stream *hinfo, struct hda_ } } +struct cs35l41_dev_name { + const char *bus; + const char *hid; + int index; +}; + +/* match the device name in a slightly relaxed manner */ +static int comp_match_cs35l41_dev_name(struct device *dev, void *data) +{ + struct cs35l41_dev_name *p = data; + const char *d = dev_name(dev); + int n = strlen(p->bus); + char tmp[32]; + + /* check the bus name */ + if (strncmp(d, p->bus, n)) + return 0; + /* skip the bus number */ + if (isdigit(d[n])) + n++; + /* the rest must be exact matching */ + snprintf(tmp, sizeof(tmp), "-%s:00-cs35l41-hda.%d", p->hid, p->index); + return !strcmp(d + n, tmp); +} + static void cs35l41_generic_fixup(struct hda_codec *cdc, int action, const char *bus, const char *hid, int count) { struct device *dev = hda_codec_dev(cdc); struct alc_spec *spec = cdc->spec; - char *name; + struct cs35l41_dev_name *rec; int ret, i; switch (action) { case HDA_FIXUP_ACT_PRE_PROBE: for (i = 0; i < count; i++) { - name = devm_kasprintf(dev, GFP_KERNEL, - "%s-%s:00-cs35l41-hda.%d", bus, hid, i); - if (!name) + rec = devm_kmalloc(dev, sizeof(*rec), GFP_KERNEL); + if (!rec) return; + rec->bus = bus; + rec->hid = hid; + rec->index = i; spec->comps[i].codec = cdc; - component_match_add(dev, &spec->match, component_compare_dev_name, name); + component_match_add(dev, &spec->match, + comp_match_cs35l41_dev_name, rec); } ret = component_master_add_with_match(dev, &comp_master_ops, spec->match); if (ret) @@ -6696,12 +6750,12 @@ static void cs35l41_fixup_i2c_two(struct hda_codec *cdc, const struct hda_fixup static void cs35l41_fixup_spi_two(struct hda_codec *codec, const struct hda_fixup *fix, int action) { - cs35l41_generic_fixup(codec, action, "spi0", "CSC3551", 2); + cs35l41_generic_fixup(codec, action, "spi", "CSC3551", 2); } static void cs35l41_fixup_spi_four(struct hda_codec *codec, const struct hda_fixup *fix, int action) { - cs35l41_generic_fixup(codec, action, "spi0", "CSC3551", 4); + cs35l41_generic_fixup(codec, action, "spi", "CSC3551", 4); } static void alc287_fixup_legion_16achg6_speakers(struct hda_codec *cdc, const struct hda_fixup *fix, @@ -7025,11 +7079,14 @@ enum { ALC294_FIXUP_ASUS_GU502_HP, ALC294_FIXUP_ASUS_GU502_PINS, ALC294_FIXUP_ASUS_GU502_VERBS, + ALC294_FIXUP_ASUS_G513_PINS, + ALC285_FIXUP_ASUS_G533Z_PINS, ALC285_FIXUP_HP_GPIO_LED, ALC285_FIXUP_HP_MUTE_LED, ALC236_FIXUP_HP_GPIO_LED, ALC236_FIXUP_HP_MUTE_LED, ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF, + ALC298_FIXUP_SAMSUNG_AMP, ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, ALC295_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -8362,6 +8419,26 @@ static const struct hda_fixup alc269_fixups[] = { [ALC294_FIXUP_ASUS_GU502_HP] = { .type = HDA_FIXUP_FUNC, .v.func = alc294_fixup_gu502_hp, + }, + [ALC294_FIXUP_ASUS_G513_PINS] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x19, 0x03a11050 }, /* front HP mic */ + { 0x1a, 0x03a11c30 }, /* rear external mic */ + { 0x21, 0x03211420 }, /* front HP out */ + { } + }, + }, + [ALC285_FIXUP_ASUS_G533Z_PINS] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x14, 0x90170152 }, /* Speaker Surround Playback Switch */ + { 0x19, 0x03a19020 }, /* Mic Boost Volume */ + { 0x1a, 0x03a11c30 }, /* Mic Boost Volume */ + { 0x1e, 0x90170151 }, /* Rear jack, IN OUT EAPD Detect */ + { 0x21, 0x03211420 }, + { } + }, }, [ALC294_FIXUP_ASUS_COEF_1B] = { .type = HDA_FIXUP_VERBS, @@ -8396,6 +8473,12 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc236_fixup_hp_mute_led_micmute_vref, }, + [ALC298_FIXUP_SAMSUNG_AMP] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc298_fixup_samsung_amp, + .chained = true, + .chain_id = ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET + }, [ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET] = { .type = HDA_FIXUP_VERBS, .v.verbs = (const struct hda_verb[]) { @@ -9116,6 +9199,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1028, 0x0a9d, "Dell Latitude 5430", ALC269_FIXUP_DELL4_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0a9e, "Dell Latitude 5430", ALC269_FIXUP_DELL4_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x0b19, "Dell XPS 15 9520", ALC289_FIXUP_DUAL_SPK), + SND_PCI_QUIRK(0x1028, 0x0b1a, "Dell Precision 5570", ALC289_FIXUP_DUAL_SPK), SND_PCI_QUIRK(0x1028, 0x164a, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1028, 0x164b, "Dell", ALC293_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x1586, "HP", ALC269_FIXUP_HP_MUTE_LED_MIC2), @@ -9243,6 +9327,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8896, "HP EliteBook 855 G8 Notebook PC", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x896e, "HP EliteBook x360 830 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8971, "HP EliteBook 830 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8972, "HP EliteBook 840 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), @@ -9269,6 +9354,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8aa3, "HP ProBook 450 G9 (MB 8AA1)", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8aa8, "HP EliteBook 640 G9 (MB 8AA6)", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8aab, "HP EliteBook 650 G9 (MB 8AA9)", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8abb, "HP ZBook Firefly 14 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8ad1, "HP EliteBook 840 14 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8ad2, "HP EliteBook 860 16 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC), @@ -9290,10 +9376,11 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x13b0, "ASUS Z550SA", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x1427, "Asus Zenbook UX31E", ALC269VB_FIXUP_ASUS_ZENBOOK), SND_PCI_QUIRK(0x1043, 0x1517, "Asus Zenbook UX31A", ALC269VB_FIXUP_ASUS_ZENBOOK_UX31A), + SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK), + SND_PCI_QUIRK(0x1043, 0x16b2, "ASUS GU603", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x16e3, "ASUS UX50", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x1740, "ASUS UX430UA", ALC295_FIXUP_ASUS_DACS), SND_PCI_QUIRK(0x1043, 0x17d1, "ASUS UX431FL", ALC294_FIXUP_ASUS_DUAL_SPK), - SND_PCI_QUIRK(0x1043, 0x1662, "ASUS GV301QH", ALC294_FIXUP_ASUS_DUAL_SPK), SND_PCI_QUIRK(0x1043, 0x1881, "ASUS Zephyrus S/M", ALC294_FIXUP_ASUS_GX502_PINS), SND_PCI_QUIRK(0x1043, 0x18b1, "Asus MJ401TA", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x18f1, "Asus FX505DT", ALC256_FIXUP_ASUS_HEADSET_MIC), @@ -9309,14 +9396,17 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1b13, "Asus U41SV", ALC269_FIXUP_INV_DMIC), SND_PCI_QUIRK(0x1043, 0x1bbd, "ASUS Z550MA", ALC255_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1c23, "Asus X55U", ALC269_FIXUP_LIMIT_INT_MIC_BOOST), + SND_PCI_QUIRK(0x1043, 0x1c92, "ASUS ROG Strix G15", ALC285_FIXUP_ASUS_G533Z_PINS), SND_PCI_QUIRK(0x1043, 0x1ccd, "ASUS X555UB", ALC256_FIXUP_ASUS_MIC), + SND_PCI_QUIRK(0x1043, 0x1d42, "ASUS Zephyrus G14 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1d4e, "ASUS TM420", ALC256_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1e11, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA502), SND_PCI_QUIRK(0x1043, 0x1e51, "ASUS Zephyrus M15", ALC294_FIXUP_ASUS_GU502_PINS), + SND_PCI_QUIRK(0x1043, 0x1e5e, "ASUS ROG Strix G513", ALC294_FIXUP_ASUS_G513_PINS), SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401), + SND_PCI_QUIRK(0x1043, 0x1c52, "ASUS Zephyrus G15 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401), - SND_PCI_QUIRK(0x1043, 0x1d42, "ASUS Zephyrus G14 2022", ALC289_FIXUP_ASUS_GA401), - SND_PCI_QUIRK(0x1043, 0x16b2, "ASUS GU603", ALC289_FIXUP_ASUS_GA401), + SND_PCI_QUIRK(0x1043, 0x1f92, "ASUS ROG Flow X16", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x3030, "ASUS ZN270IE", ALC256_FIXUP_ASUS_AIO_GPIO2), SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x1043, 0x834a, "ASUS S101", ALC269_FIXUP_STEREO_DMIC), @@ -9338,17 +9428,18 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x10ec, 0x10f2, "Intel Reference board", ALC700_FIXUP_INTEL_REFERENCE), SND_PCI_QUIRK(0x10ec, 0x118c, "Medion EE4254 MD62100", ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE), SND_PCI_QUIRK(0x10ec, 0x1230, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), + SND_PCI_QUIRK(0x10ec, 0x124c, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x10ec, 0x1252, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x10ec, 0x1254, "Intel Reference board", ALC295_FIXUP_CHROME_BOOK), SND_PCI_QUIRK(0x10f7, 0x8338, "Panasonic CF-SZ6", ALC269_FIXUP_HEADSET_MODE), SND_PCI_QUIRK(0x144d, 0xc109, "Samsung Ativ book 9 (NP900X3G)", ALC269_FIXUP_INV_DMIC), - SND_PCI_QUIRK(0x144d, 0xc169, "Samsung Notebook 9 Pen (NP930SBE-K01US)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), - SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), - SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), - SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), + SND_PCI_QUIRK(0x144d, 0xc169, "Samsung Notebook 9 Pen (NP930SBE-K01US)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc176, "Samsung Notebook 9 Pro (NP930MBE-K04US)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc189, "Samsung Galaxy Flex Book (NT950QCG-X716)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc18a, "Samsung Galaxy Book Ion (NP930XCJ-K01US)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc740, "Samsung Ativ book 8 (NP870Z5G)", ALC269_FIXUP_ATIV_BOOK_8), - SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), - SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), + SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_AMP), + SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc832, "Samsung Galaxy Book Flex Alpha (NP730QCJ)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x1458, 0xfa53, "Gigabyte BXBT-2807", ALC283_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC), @@ -9520,6 +9611,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x9e54, "LENOVO NB", ALC269_FIXUP_LENOVO_EAPD), SND_PCI_QUIRK(0x1849, 0x1233, "ASRock NUC Box 1100", ALC233_FIXUP_NO_AUDIO_JACK), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), + SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20), SND_PCI_QUIRK(0x1b35, 0x1236, "CZC TMI", ALC269_FIXUP_CZC_TMI), SND_PCI_QUIRK(0x1b35, 0x1237, "CZC L101", ALC269_FIXUP_CZC_L101), @@ -9716,7 +9808,7 @@ static const struct hda_model_fixup alc269_fixup_models[] = { {.id = ALC299_FIXUP_PREDATOR_SPK, .name = "predator-spk"}, {.id = ALC298_FIXUP_HUAWEI_MBX_STEREO, .name = "huawei-mbx-stereo"}, {.id = ALC256_FIXUP_MEDION_HEADSET_NO_PRESENCE, .name = "alc256-medion-headset"}, - {.id = ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc298-samsung-headphone"}, + {.id = ALC298_FIXUP_SAMSUNG_AMP, .name = "alc298-samsung-amp"}, {.id = ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET, .name = "alc256-samsung-headphone"}, {.id = ALC255_FIXUP_XIAOMI_HEADSET_MIC, .name = "alc255-xiaomi-headset"}, {.id = ALC274_FIXUP_HP_MIC, .name = "alc274-hp-mic-detect"}, diff --git a/sound/pci/hda/patch_sigmatel.c b/sound/pci/hda/patch_sigmatel.c index 61df4d33c48ffa95dc5e6adb11e48ff364984c1f..a794a01a68ca60e55bcd0ab653a0db3e6c3ffcc5 100644 --- a/sound/pci/hda/patch_sigmatel.c +++ b/sound/pci/hda/patch_sigmatel.c @@ -209,6 +209,7 @@ struct sigmatel_spec { /* beep widgets */ hda_nid_t anabeep_nid; + bool beep_power_on; /* SPDIF-out mux */ const char * const *spdif_labels; @@ -4310,6 +4311,8 @@ static int stac_parse_auto_config(struct hda_codec *codec) if (codec->beep) { /* IDT/STAC codecs have linear beep tone parameter */ codec->beep->linear_tone = spec->linear_tone_beep; + /* keep power up while beep is enabled */ + codec->beep->keep_power_at_enable = 1; /* if no beep switch is available, make its own one */ caps = query_amp_caps(codec, nid, HDA_OUTPUT); if (!(caps & AC_AMPCAP_MUTE)) { diff --git a/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c b/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c index dfc4295b69c413a5164a932d98f9f38313aa8126..aaa82ec36540d0b7bfec6ca897d990d09ec03143 100644 --- a/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c +++ b/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c @@ -257,8 +257,7 @@ int snd_pdacf_pcm_new(struct snd_pdacf *chip) return err; snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &pdacf_pcm_capture_ops); - snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, - snd_dma_continuous_data(GFP_KERNEL | GFP_DMA32), + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); pcm->private_data = chip; diff --git a/sound/ppc/keywest.c b/sound/ppc/keywest.c index 6e5daae18f9d0edb813016292b2a1809d9e3aa9b..80e5108157efcd113f6a313a529ee45e121a8447 100644 --- a/sound/ppc/keywest.c +++ b/sound/ppc/keywest.c @@ -71,14 +71,12 @@ static int keywest_attach_adapter(struct i2c_adapter *adapter) return 0; } -static int keywest_remove(struct i2c_client *client) +static void keywest_remove(struct i2c_client *client) { if (! keywest_ctx) - return 0; + return; if (client == keywest_ctx->client) keywest_ctx->client = NULL; - - return 0; } diff --git a/sound/ppc/tumbler.c b/sound/ppc/tumbler.c index c65e74d7cd0a6a5419d66c34b0754d857b478bfc..f3f8ad7c3df8dc5af611b172e63bd306db966eb2 100644 --- a/sound/ppc/tumbler.c +++ b/sound/ppc/tumbler.c @@ -1060,8 +1060,7 @@ static struct device_node *find_audio_device(const char *name) if (! gpiop) return NULL; - for (np = of_get_next_child(gpiop, NULL); np; - np = of_get_next_child(gpiop, np)) { + for_each_child_of_node(gpiop, np) { const char *property = of_get_property(np, "audio-gpio", NULL); if (property && strcmp(property, name) == 0) break; @@ -1080,8 +1079,7 @@ static struct device_node *find_compatible_audio_device(const char *name) if (!gpiop) return NULL; - for (np = of_get_next_child(gpiop, NULL); np; - np = of_get_next_child(gpiop, np)) { + for_each_child_of_node(gpiop, np) { if (of_device_is_compatible(np, name)) break; } diff --git a/sound/soc/Kconfig b/sound/soc/Kconfig index 7d4747b6bab247c2ab6450840f2ff7c49080ac00..848fbae26c3bf665aa52b41886de40d33d50742c 100644 --- a/sound/soc/Kconfig +++ b/sound/soc/Kconfig @@ -68,6 +68,7 @@ config SND_SOC_ACPI # All the supported SoCs source "sound/soc/adi/Kconfig" source "sound/soc/amd/Kconfig" +source "sound/soc/apple/Kconfig" source "sound/soc/atmel/Kconfig" source "sound/soc/au1x/Kconfig" source "sound/soc/bcm/Kconfig" diff --git a/sound/soc/Makefile b/sound/soc/Makefile index 453181ef6c94c90dfc57a666632e348e4a1a0381..507eaed1d6a139c8baaae18da94d436af54132f8 100644 --- a/sound/soc/Makefile +++ b/sound/soc/Makefile @@ -34,6 +34,7 @@ obj-$(CONFIG_SND_SOC_ACPI) += snd-soc-acpi.o obj-$(CONFIG_SND_SOC) += snd-soc-core.o obj-$(CONFIG_SND_SOC) += codecs/ obj-$(CONFIG_SND_SOC) += generic/ +obj-$(CONFIG_SND_SOC) += apple/ obj-$(CONFIG_SND_SOC) += adi/ obj-$(CONFIG_SND_SOC) += amd/ obj-$(CONFIG_SND_SOC) += atmel/ diff --git a/sound/soc/amd/Kconfig b/sound/soc/amd/Kconfig index 08f5289dac5408cca3ba08878d26aa8d40838b97..150786279257d683f5a7323db926f024422149e8 100644 --- a/sound/soc/amd/Kconfig +++ b/sound/soc/amd/Kconfig @@ -122,8 +122,29 @@ config SND_SOC_AMD_RPL_ACP6x tristate "AMD Audio Coprocessor-v6.2 RPL support" depends on X86 && PCI help - This option enables Audio Coprocessor i.e ACP v6.2 support on + This option enables Audio Coprocessor i.e. ACP v6.2 support on AMD RPL platform. By enabling this flag build will be triggered for ACP PCI driver. Say m if you have such a device. If unsure select "N". + +config SND_SOC_AMD_PS + tristate "AMD Audio Coprocessor-v6.2 Pink Sardine support" + depends on X86 && PCI && ACPI + help + This option enables Audio Coprocessor i.e ACP v6.2 support on + AMD Pink sardine platform. By enabling this flag build will be + triggered for ACP PCI driver, ACP PDM DMA driver. + Say m if you have such a device. + If unsure select "N". + +config SND_SOC_AMD_PS_MACH + tristate "AMD PINK SARDINE support for DMIC" + select SND_SOC_DMIC + depends on SND_SOC_AMD_PS + help + This option enables machine driver for Pink Sardine platform + using dmic. ACP IP has PDM Decoder block with DMA controller. + DMIC can be connected directly to ACP IP. + Say m if you have such a device. + If unsure select "N". diff --git a/sound/soc/amd/Makefile b/sound/soc/amd/Makefile index 0592e7c5c40711a316cededfe36541339a45d947..82e1cf864a40957a44ff056de1b91abb2346bb86 100644 --- a/sound/soc/amd/Makefile +++ b/sound/soc/amd/Makefile @@ -18,3 +18,4 @@ obj-$(CONFIG_SND_SOC_AMD_ACP6x) += yc/ obj-$(CONFIG_SND_SOC_AMD_ACP_COMMON) += acp/ obj-$(CONFIG_SND_AMD_ACP_CONFIG) += snd-acp-config.o obj-$(CONFIG_SND_SOC_AMD_RPL_ACP6x) += rpl/ +obj-$(CONFIG_SND_SOC_AMD_PS) += ps/ diff --git a/sound/soc/amd/acp/acp-i2s.c b/sound/soc/amd/acp/acp-i2s.c index 393f729ef561beeb8c7f7262395f28dc31df678d..ac416572db0d3a5b8e1db41cb962f165bf009a66 100644 --- a/sound/soc/amd/acp/acp-i2s.c +++ b/sound/soc/amd/acp/acp-i2s.c @@ -25,6 +25,65 @@ #define DRV_NAME "acp_i2s_playcap" +static int acp_i2s_set_fmt(struct snd_soc_dai *cpu_dai, + unsigned int fmt) +{ + struct acp_dev_data *adata = snd_soc_dai_get_drvdata(cpu_dai); + int mode; + + mode = fmt & SND_SOC_DAIFMT_FORMAT_MASK; + switch (mode) { + case SND_SOC_DAIFMT_I2S: + adata->tdm_mode = TDM_DISABLE; + break; + case SND_SOC_DAIFMT_DSP_A: + adata->tdm_mode = TDM_ENABLE; + break; + default: + return -EINVAL; + } + return 0; +} + +static int acp_i2s_set_tdm_slot(struct snd_soc_dai *dai, u32 tx_mask, u32 rx_mask, + int slots, int slot_width) +{ + struct device *dev = dai->component->dev; + struct acp_dev_data *adata = snd_soc_dai_get_drvdata(dai); + struct acp_stream *stream; + int slot_len; + + switch (slot_width) { + case SLOT_WIDTH_8: + slot_len = 8; + break; + case SLOT_WIDTH_16: + slot_len = 16; + break; + case SLOT_WIDTH_24: + slot_len = 24; + break; + case SLOT_WIDTH_32: + slot_len = 0; + break; + default: + dev_err(dev, "Unsupported bitdepth %d\n", slot_width); + return -EINVAL; + } + + spin_lock_irq(&adata->acp_lock); + list_for_each_entry(stream, &adata->stream_list, list) { + if (tx_mask && stream->dir == SNDRV_PCM_STREAM_PLAYBACK) + adata->tdm_tx_fmt[stream->dai_id - 1] = + FRM_LEN | (slots << 15) | (slot_len << 18); + else if (rx_mask && stream->dir == SNDRV_PCM_STREAM_CAPTURE) + adata->tdm_rx_fmt[stream->dai_id - 1] = + FRM_LEN | (slots << 15) | (slot_len << 18); + } + spin_unlock_irq(&adata->acp_lock); + return 0; +} + static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) { @@ -33,7 +92,7 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_ struct acp_resource *rsrc; u32 val; u32 xfer_resolution; - u32 reg_val; + u32 reg_val, fmt_reg, tdm_fmt; u32 lrclk_div_val, bclk_div_val; adata = snd_soc_dai_get_drvdata(dai); @@ -62,12 +121,15 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_ switch (dai->driver->id) { case I2S_BT_INSTANCE: reg_val = ACP_BTTDM_ITER; + fmt_reg = ACP_BTTDM_TXFRMT; break; case I2S_SP_INSTANCE: reg_val = ACP_I2STDM_ITER; + fmt_reg = ACP_I2STDM_TXFRMT; break; case I2S_HS_INSTANCE: reg_val = ACP_HSTDM_ITER; + fmt_reg = ACP_HSTDM_TXFRMT; break; default: dev_err(dev, "Invalid dai id %x\n", dai->driver->id); @@ -77,12 +139,15 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_ switch (dai->driver->id) { case I2S_BT_INSTANCE: reg_val = ACP_BTTDM_IRER; + fmt_reg = ACP_BTTDM_RXFRMT; break; case I2S_SP_INSTANCE: reg_val = ACP_I2STDM_IRER; + fmt_reg = ACP_I2STDM_RXFRMT; break; case I2S_HS_INSTANCE: reg_val = ACP_HSTDM_IRER; + fmt_reg = ACP_HSTDM_RXFRMT; break; default: dev_err(dev, "Invalid dai id %x\n", dai->driver->id); @@ -95,6 +160,16 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_ val = val | (xfer_resolution << 3); writel(val, adata->acp_base + reg_val); + if (adata->tdm_mode) { + val = readl(adata->acp_base + reg_val); + writel(val | BIT(1), adata->acp_base + reg_val); + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + tdm_fmt = adata->tdm_tx_fmt[dai->driver->id - 1]; + else + tdm_fmt = adata->tdm_rx_fmt[dai->driver->id - 1]; + writel(tdm_fmt, adata->acp_base + fmt_reg); + } + if (rsrc->soc_mclk) { switch (params_format(params)) { case SNDRV_PCM_FORMAT_S16_LE: @@ -443,6 +518,7 @@ static int acp_i2s_startup(struct snd_pcm_substream *substream, struct snd_soc_d stream->id = dai->driver->id + dir; stream->dai_id = dai->driver->id; stream->irq_bit = irq_bit; + stream->dir = substream->stream; return 0; } @@ -452,6 +528,8 @@ const struct snd_soc_dai_ops asoc_acp_cpu_dai_ops = { .hw_params = acp_i2s_hwparams, .prepare = acp_i2s_prepare, .trigger = acp_i2s_trigger, + .set_fmt = acp_i2s_set_fmt, + .set_tdm_slot = acp_i2s_set_tdm_slot, }; EXPORT_SYMBOL_NS_GPL(asoc_acp_cpu_dai_ops, SND_SOC_ACP_COMMON); diff --git a/sound/soc/amd/acp/acp-mach-common.c b/sound/soc/amd/acp/acp-mach-common.c index f0c49127aad11178770e2d186b8be62f50ea916d..4c69cb6e34006696044f585281931bfaa8e6f24e 100644 --- a/sound/soc/amd/acp/acp-mach-common.c +++ b/sound/soc/amd/acp/acp-mach-common.c @@ -584,7 +584,7 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card) if (drv_data->dmic_cpu_id) num_links++; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_links, GFP_KERNEL); + links = devm_kcalloc(dev, num_links, sizeof(struct snd_soc_dai_link), GFP_KERNEL); if (!links) return -ENOMEM; @@ -749,7 +749,7 @@ int acp_legacy_dai_links_create(struct snd_soc_card *card) if (drv_data->dmic_cpu_id) num_links++; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_links, GFP_KERNEL); + links = devm_kcalloc(dev, num_links, sizeof(struct snd_soc_dai_link), GFP_KERNEL); if (!links) return -ENOMEM; diff --git a/sound/soc/amd/acp/acp-pci.c b/sound/soc/amd/acp/acp-pci.c index 2c8e960cc9a6a96fa4e9ad1eeb13075fcdec96fb..a0c84cd07fdea5899ed563c3d931fda2ac7e04cd 100644 --- a/sound/soc/amd/acp/acp-pci.c +++ b/sound/soc/amd/acp/acp-pci.c @@ -62,10 +62,9 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id if (!chip) return -ENOMEM; - if (pci_enable_device(pci)) { - dev_err(&pci->dev, "pci_enable_device failed\n"); - return -ENODEV; - } + if (pci_enable_device(pci)) + return dev_err_probe(&pci->dev, -ENODEV, + "pci_enable_device failed\n"); ret = pci_request_regions(pci, "AMD ACP3x audio"); if (ret < 0) { @@ -105,14 +104,13 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id chip->base = devm_ioremap(&pci->dev, addr, pci_resource_len(pci, 0)); if (!chip->base) { ret = -ENOMEM; - goto release_regions; + goto unregister_dmic_dev; } - res = devm_kzalloc(&pci->dev, sizeof(struct resource) * num_res, GFP_KERNEL); + res = devm_kcalloc(&pci->dev, num_res, sizeof(struct resource), GFP_KERNEL); if (!res) { - platform_device_unregister(dmic_dev); ret = -ENOMEM; - goto release_regions; + goto unregister_dmic_dev; } for (i = 0; i < num_res; i++, res_acp++) { @@ -139,13 +137,14 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id pdev = platform_device_register_full(&pdevinfo); if (IS_ERR(pdev)) { dev_err(&pci->dev, "cannot register %s device\n", pdevinfo.name); - platform_device_unregister(dmic_dev); ret = PTR_ERR(pdev); - goto release_regions; + goto unregister_dmic_dev; } return ret; +unregister_dmic_dev: + platform_device_unregister(dmic_dev); release_regions: pci_release_regions(pci); disable_pci: diff --git a/sound/soc/amd/acp/acp-platform.c b/sound/soc/amd/acp/acp-platform.c index f561d39b33e2adb8638f36aada844ed609554c3e..85a81add4ef9fd5eb7dd5a5006c19e8e987dc416 100644 --- a/sound/soc/amd/acp/acp-platform.c +++ b/sound/soc/amd/acp/acp-platform.c @@ -94,7 +94,7 @@ static irqreturn_t i2s_irq_handler(int irq, void *data) struct acp_resource *rsrc = adata->rsrc; struct acp_stream *stream; u16 i2s_flag = 0; - u32 ext_intr_stat, ext_intr_stat1, i; + u32 ext_intr_stat, ext_intr_stat1; if (!adata) return IRQ_NONE; @@ -104,25 +104,24 @@ static irqreturn_t i2s_irq_handler(int irq, void *data) ext_intr_stat = readl(ACP_EXTERNAL_INTR_STAT(adata, rsrc->irqp_used)); - for (i = 0; i < ACP_MAX_STREAM; i++) { - stream = adata->stream[i]; - if (stream && (ext_intr_stat & stream->irq_bit)) { + spin_lock(&adata->acp_lock); + list_for_each_entry(stream, &adata->stream_list, list) { + if (ext_intr_stat & stream->irq_bit) { writel(stream->irq_bit, ACP_EXTERNAL_INTR_STAT(adata, rsrc->irqp_used)); snd_pcm_period_elapsed(stream->substream); i2s_flag = 1; - break; } if (adata->rsrc->no_of_ctrls == 2) { - if (stream && (ext_intr_stat1 & stream->irq_bit)) { + if (ext_intr_stat1 & stream->irq_bit) { writel(stream->irq_bit, ACP_EXTERNAL_INTR_STAT(adata, (rsrc->irqp_used - 1))); snd_pcm_period_elapsed(stream->substream); i2s_flag = 1; - break; } } } + spin_unlock(&adata->acp_lock); if (i2s_flag) return IRQ_HANDLED; @@ -146,9 +145,8 @@ static void config_pte_for_stream(struct acp_dev_data *adata, struct acp_stream writel(0x01, adata->acp_base + ACPAXI2AXI_ATU_CTRL); } -static void config_acp_dma(struct acp_dev_data *adata, int cpu_id, int size) +static void config_acp_dma(struct acp_dev_data *adata, struct acp_stream *stream, int size) { - struct acp_stream *stream = adata->stream[cpu_id]; struct snd_pcm_substream *substream = stream->substream; struct acp_resource *rsrc = adata->rsrc; dma_addr_t addr = substream->dma_buffer.addr; @@ -174,13 +172,10 @@ static void config_acp_dma(struct acp_dev_data *adata, int cpu_id, int size) static int acp_dma_open(struct snd_soc_component *component, struct snd_pcm_substream *substream) { - struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream); - struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0); struct snd_pcm_runtime *runtime = substream->runtime; struct device *dev = component->dev; struct acp_dev_data *adata = dev_get_drvdata(dev); struct acp_stream *stream; - int stream_id = cpu_dai->driver->id * 2 + substream->stream; int ret; stream = kzalloc(sizeof(*stream), GFP_KERNEL); @@ -188,7 +183,10 @@ static int acp_dma_open(struct snd_soc_component *component, struct snd_pcm_subs return -ENOMEM; stream->substream = substream; - adata->stream[stream_id] = stream; + + spin_lock_irq(&adata->acp_lock); + list_add_tail(&stream->list, &adata->stream_list); + spin_unlock_irq(&adata->acp_lock); if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) runtime->hw = acp_pcm_hardware_playback; @@ -212,16 +210,13 @@ static int acp_dma_hw_params(struct snd_soc_component *component, struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { - struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream); struct acp_dev_data *adata = snd_soc_component_get_drvdata(component); - struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0); struct acp_stream *stream = substream->runtime->private_data; - int stream_id = cpu_dai->driver->id * 2 + substream->stream; u64 size = params_buffer_bytes(params); /* Configure ACP DMA block with params */ config_pte_for_stream(adata, stream); - config_acp_dma(adata, stream_id, size); + config_acp_dma(adata, stream, size); return 0; } @@ -261,16 +256,15 @@ static int acp_dma_new(struct snd_soc_component *component, static int acp_dma_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { - struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream); - struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0); struct device *dev = component->dev; struct acp_dev_data *adata = dev_get_drvdata(dev); - struct acp_stream *stream; - int stream_id = cpu_dai->driver->id * 2 + substream->stream; + struct acp_stream *stream = substream->runtime->private_data; - stream = adata->stream[stream_id]; + /* Remove entry from list */ + spin_lock_irq(&adata->acp_lock); + list_del(&stream->list); + spin_unlock_irq(&adata->acp_lock); kfree(stream); - adata->stream[stream_id] = NULL; return 0; } @@ -305,6 +299,10 @@ int acp_platform_register(struct device *dev) dev_err(dev, "Fail to register acp i2s component\n"); return status; } + + INIT_LIST_HEAD(&adata->stream_list); + spin_lock_init(&adata->acp_lock); + return 0; } EXPORT_SYMBOL_NS_GPL(acp_platform_register, SND_SOC_ACP_COMMON); diff --git a/sound/soc/amd/acp/amd.h b/sound/soc/amd/acp/amd.h index af9603724a68c5d81c4d9634df29cd534c4db746..5f2119f422715dab2f52aac3395922072bee996a 100644 --- a/sound/soc/amd/acp/amd.h +++ b/sound/soc/amd/acp/amd.h @@ -21,9 +21,9 @@ #define ACP3X_DEV 3 #define ACP6X_DEV 6 -#define I2S_SP_INSTANCE 0x00 -#define I2S_BT_INSTANCE 0x01 -#define DMIC_INSTANCE 0x02 +#define DMIC_INSTANCE 0x00 +#define I2S_SP_INSTANCE 0x01 +#define I2S_BT_INSTANCE 0x02 #define I2S_HS_INSTANCE 0x03 #define MEM_WINDOW_START 0x4080000 @@ -84,6 +84,14 @@ #define ACP_MAX_STREAM 8 +#define TDM_ENABLE 1 +#define TDM_DISABLE 0 + +#define SLOT_WIDTH_8 0x8 +#define SLOT_WIDTH_16 0x10 +#define SLOT_WIDTH_24 0x18 +#define SLOT_WIDTH_32 0x20 + struct acp_chip_info { char *name; /* Platform name */ unsigned int acp_rev; /* ACP Revision id */ @@ -91,10 +99,12 @@ struct acp_chip_info { }; struct acp_stream { + struct list_head list; struct snd_pcm_substream *substream; int irq_bit; int dai_id; int id; + int dir; u64 bytescount; u32 reg_offset; u32 pte_offset; @@ -119,11 +129,13 @@ struct acp_dev_data { void __iomem *acp_base; unsigned int i2s_irq; + bool tdm_mode; /* SOC specific dais */ struct snd_soc_dai_driver *dai_driver; int num_dai; - struct acp_stream *stream[ACP_MAX_STREAM]; + struct list_head stream_list; + spinlock_t acp_lock; struct snd_soc_acpi_mach *machines; struct platform_device *mach_dev; @@ -132,6 +144,8 @@ struct acp_dev_data { u32 lrclk_div; struct acp_resource *rsrc; + u32 tdm_tx_fmt[3]; + u32 tdm_rx_fmt[3]; }; union acp_i2stdm_mstrclkgen { diff --git a/sound/soc/amd/ps/Makefile b/sound/soc/amd/ps/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..383973a12f6a4e9546d58d13e5699b338471bfc1 --- /dev/null +++ b/sound/soc/amd/ps/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Pink Sardine platform Support +snd-pci-ps-objs := pci-ps.o +snd-ps-pdm-dma-objs := ps-pdm-dma.o +snd-soc-ps-mach-objs := ps-mach.o + +obj-$(CONFIG_SND_SOC_AMD_PS) += snd-pci-ps.o +obj-$(CONFIG_SND_SOC_AMD_PS) += snd-ps-pdm-dma.o +obj-$(CONFIG_SND_SOC_AMD_PS_MACH) += snd-soc-ps-mach.o diff --git a/sound/soc/amd/ps/acp62.h b/sound/soc/amd/ps/acp62.h new file mode 100644 index 0000000000000000000000000000000000000000..8b30aefa4cd0563461f8cfef14246f507b04b26f --- /dev/null +++ b/sound/soc/amd/ps/acp62.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * AMD ALSA SoC PDM Driver + * + * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. + */ + +#include <sound/acp62_chip_offset_byte.h> + +#define ACP_DEVICE_ID 0x15E2 +#define ACP6x_REG_START 0x1240000 +#define ACP6x_REG_END 0x1250200 +#define ACP6x_DEVS 3 +#define ACP6x_PDM_MODE 1 + +#define ACP_SOFT_RESET_SOFTRESET_AUDDONE_MASK 0x00010001 +#define ACP_PGFSM_CNTL_POWER_ON_MASK 1 +#define ACP_PGFSM_CNTL_POWER_OFF_MASK 0 +#define ACP_PGFSM_STATUS_MASK 3 +#define ACP_POWERED_ON 0 +#define ACP_POWER_ON_IN_PROGRESS 1 +#define ACP_POWERED_OFF 2 +#define ACP_POWER_OFF_IN_PROGRESS 3 + +#define ACP_ERROR_MASK 0x20000000 +#define ACP_EXT_INTR_STAT_CLEAR_MASK 0xFFFFFFFF +#define PDM_DMA_STAT 0x10 + +#define PDM_DMA_INTR_MASK 0x10000 +#define ACP_ERROR_STAT 29 +#define PDM_DECIMATION_FACTOR 2 +#define ACP_PDM_CLK_FREQ_MASK 7 +#define ACP_WOV_MISC_CTRL_MASK 0x10 +#define ACP_PDM_ENABLE 1 +#define ACP_PDM_DISABLE 0 +#define ACP_PDM_DMA_EN_STATUS 2 +#define TWO_CH 2 +#define DELAY_US 5 +#define ACP_COUNTER 20000 + +#define ACP_SRAM_PTE_OFFSET 0x03800000 +#define PAGE_SIZE_4K_ENABLE 2 +#define PDM_PTE_OFFSET 0 +#define PDM_MEM_WINDOW_START 0x4000000 + +#define CAPTURE_MIN_NUM_PERIODS 4 +#define CAPTURE_MAX_NUM_PERIODS 4 +#define CAPTURE_MAX_PERIOD_SIZE 8192 +#define CAPTURE_MIN_PERIOD_SIZE 4096 + +#define MAX_BUFFER (CAPTURE_MAX_PERIOD_SIZE * CAPTURE_MAX_NUM_PERIODS) +#define MIN_BUFFER MAX_BUFFER + +/* time in ms for runtime suspend delay */ +#define ACP_SUSPEND_DELAY_MS 2000 + +enum acp_config { + ACP_CONFIG_0 = 0, + ACP_CONFIG_1, + ACP_CONFIG_2, + ACP_CONFIG_3, + ACP_CONFIG_4, + ACP_CONFIG_5, + ACP_CONFIG_6, + ACP_CONFIG_7, + ACP_CONFIG_8, + ACP_CONFIG_9, + ACP_CONFIG_10, + ACP_CONFIG_11, + ACP_CONFIG_12, + ACP_CONFIG_13, + ACP_CONFIG_14, + ACP_CONFIG_15, +}; + +struct pdm_stream_instance { + u16 num_pages; + u16 channels; + dma_addr_t dma_addr; + u64 bytescount; + void __iomem *acp62_base; +}; + +struct pdm_dev_data { + u32 pdm_irq; + void __iomem *acp62_base; + struct snd_pcm_substream *capture_stream; +}; + +static inline u32 acp62_readl(void __iomem *base_addr) +{ + return readl(base_addr); +} + +static inline void acp62_writel(u32 val, void __iomem *base_addr) +{ + writel(val, base_addr); +} diff --git a/sound/soc/amd/ps/pci-ps.c b/sound/soc/amd/ps/pci-ps.c new file mode 100644 index 0000000000000000000000000000000000000000..dff2e2376bbf965814665a5be2143f0aca919f47 --- /dev/null +++ b/sound/soc/amd/ps/pci-ps.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * AMD Pink Sardine ACP PCI Driver + * + * Copyright 2022 Advanced Micro Devices, Inc. + */ + +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/platform_device.h> +#include <linux/acpi.h> +#include <linux/interrupt.h> +#include <sound/pcm_params.h> +#include <linux/pm_runtime.h> + +#include "acp62.h" + +struct acp62_dev_data { + void __iomem *acp62_base; + struct resource *res; + bool acp62_audio_mode; + struct platform_device *pdev[ACP6x_DEVS]; +}; + +static int acp62_power_on(void __iomem *acp_base) +{ + u32 val; + int timeout; + + val = acp62_readl(acp_base + ACP_PGFSM_STATUS); + + if (!val) + return val; + + if ((val & ACP_PGFSM_STATUS_MASK) != ACP_POWER_ON_IN_PROGRESS) + acp62_writel(ACP_PGFSM_CNTL_POWER_ON_MASK, acp_base + ACP_PGFSM_CONTROL); + timeout = 0; + while (++timeout < 500) { + val = acp62_readl(acp_base + ACP_PGFSM_STATUS); + if (!val) + return 0; + udelay(1); + } + return -ETIMEDOUT; +} + +static int acp62_reset(void __iomem *acp_base) +{ + u32 val; + int timeout; + + acp62_writel(1, acp_base + ACP_SOFT_RESET); + timeout = 0; + while (++timeout < 500) { + val = acp62_readl(acp_base + ACP_SOFT_RESET); + if (val & ACP_SOFT_RESET_SOFTRESET_AUDDONE_MASK) + break; + cpu_relax(); + } + acp62_writel(0, acp_base + ACP_SOFT_RESET); + timeout = 0; + while (++timeout < 500) { + val = acp62_readl(acp_base + ACP_SOFT_RESET); + if (!val) + return 0; + cpu_relax(); + } + return -ETIMEDOUT; +} + +static void acp62_enable_interrupts(void __iomem *acp_base) +{ + acp62_writel(1, acp_base + ACP_EXTERNAL_INTR_ENB); +} + +static void acp62_disable_interrupts(void __iomem *acp_base) +{ + acp62_writel(ACP_EXT_INTR_STAT_CLEAR_MASK, acp_base + + ACP_EXTERNAL_INTR_STAT); + acp62_writel(0, acp_base + ACP_EXTERNAL_INTR_CNTL); + acp62_writel(0, acp_base + ACP_EXTERNAL_INTR_ENB); +} + +static int acp62_init(void __iomem *acp_base, struct device *dev) +{ + int ret; + + ret = acp62_power_on(acp_base); + if (ret) { + dev_err(dev, "ACP power on failed\n"); + return ret; + } + acp62_writel(0x01, acp_base + ACP_CONTROL); + ret = acp62_reset(acp_base); + if (ret) { + dev_err(dev, "ACP reset failed\n"); + return ret; + } + acp62_writel(0x03, acp_base + ACP_CLKMUX_SEL); + acp62_enable_interrupts(acp_base); + return 0; +} + +static int acp62_deinit(void __iomem *acp_base, struct device *dev) +{ + int ret; + + acp62_disable_interrupts(acp_base); + ret = acp62_reset(acp_base); + if (ret) { + dev_err(dev, "ACP reset failed\n"); + return ret; + } + acp62_writel(0, acp_base + ACP_CLKMUX_SEL); + acp62_writel(0, acp_base + ACP_CONTROL); + return 0; +} + +static irqreturn_t acp62_irq_handler(int irq, void *dev_id) +{ + struct acp62_dev_data *adata; + struct pdm_dev_data *ps_pdm_data; + u32 val; + + adata = dev_id; + if (!adata) + return IRQ_NONE; + + val = acp62_readl(adata->acp62_base + ACP_EXTERNAL_INTR_STAT); + if (val & BIT(PDM_DMA_STAT)) { + ps_pdm_data = dev_get_drvdata(&adata->pdev[0]->dev); + acp62_writel(BIT(PDM_DMA_STAT), adata->acp62_base + ACP_EXTERNAL_INTR_STAT); + if (ps_pdm_data->capture_stream) + snd_pcm_period_elapsed(ps_pdm_data->capture_stream); + return IRQ_HANDLED; + } + return IRQ_NONE; +} + +static int snd_acp62_probe(struct pci_dev *pci, + const struct pci_device_id *pci_id) +{ + struct acp62_dev_data *adata; + struct platform_device_info pdevinfo[ACP6x_DEVS]; + int index, ret; + int val = 0x00; + struct acpi_device *adev; + const union acpi_object *obj; + u32 addr; + unsigned int irqflags; + + irqflags = IRQF_SHARED; + /* Pink Sardine device check */ + switch (pci->revision) { + case 0x63: + break; + default: + dev_dbg(&pci->dev, "acp62 pci device not found\n"); + return -ENODEV; + } + if (pci_enable_device(pci)) { + dev_err(&pci->dev, "pci_enable_device failed\n"); + return -ENODEV; + } + + ret = pci_request_regions(pci, "AMD ACP6.2 audio"); + if (ret < 0) { + dev_err(&pci->dev, "pci_request_regions failed\n"); + goto disable_pci; + } + adata = devm_kzalloc(&pci->dev, sizeof(struct acp62_dev_data), + GFP_KERNEL); + if (!adata) { + ret = -ENOMEM; + goto release_regions; + } + + addr = pci_resource_start(pci, 0); + adata->acp62_base = devm_ioremap(&pci->dev, addr, + pci_resource_len(pci, 0)); + if (!adata->acp62_base) { + ret = -ENOMEM; + goto release_regions; + } + pci_set_master(pci); + pci_set_drvdata(pci, adata); + ret = acp62_init(adata->acp62_base, &pci->dev); + if (ret) + goto release_regions; + val = acp62_readl(adata->acp62_base + ACP_PIN_CONFIG); + switch (val) { + case ACP_CONFIG_0: + case ACP_CONFIG_1: + case ACP_CONFIG_2: + case ACP_CONFIG_3: + case ACP_CONFIG_9: + case ACP_CONFIG_15: + dev_info(&pci->dev, "Audio Mode %d\n", val); + break; + default: + + /* Checking DMIC hardware*/ + adev = acpi_find_child_device(ACPI_COMPANION(&pci->dev), 0x02, 0); + + if (!adev) + break; + + if (!acpi_dev_get_property(adev, "acp-audio-device-type", + ACPI_TYPE_INTEGER, &obj) && + obj->integer.value == 2) { + adata->res = devm_kzalloc(&pci->dev, sizeof(struct resource), GFP_KERNEL); + if (!adata->res) { + ret = -ENOMEM; + goto de_init; + } + + adata->res->name = "acp_iomem"; + adata->res->flags = IORESOURCE_MEM; + adata->res->start = addr; + adata->res->end = addr + (ACP6x_REG_END - ACP6x_REG_START); + adata->acp62_audio_mode = ACP6x_PDM_MODE; + + memset(&pdevinfo, 0, sizeof(pdevinfo)); + pdevinfo[0].name = "acp_ps_pdm_dma"; + pdevinfo[0].id = 0; + pdevinfo[0].parent = &pci->dev; + pdevinfo[0].num_res = 1; + pdevinfo[0].res = adata->res; + + pdevinfo[1].name = "dmic-codec"; + pdevinfo[1].id = 0; + pdevinfo[1].parent = &pci->dev; + + pdevinfo[2].name = "acp_ps_mach"; + pdevinfo[2].id = 0; + pdevinfo[2].parent = &pci->dev; + + for (index = 0; index < ACP6x_DEVS; index++) { + adata->pdev[index] = + platform_device_register_full(&pdevinfo[index]); + + if (IS_ERR(adata->pdev[index])) { + dev_err(&pci->dev, + "cannot register %s device\n", + pdevinfo[index].name); + ret = PTR_ERR(adata->pdev[index]); + goto unregister_devs; + } + ret = devm_request_irq(&pci->dev, pci->irq, acp62_irq_handler, + irqflags, "ACP_PCI_IRQ", adata); + if (ret) { + dev_err(&pci->dev, "ACP PCI IRQ request failed\n"); + goto unregister_devs; + } + } + } + break; + } + pm_runtime_set_autosuspend_delay(&pci->dev, ACP_SUSPEND_DELAY_MS); + pm_runtime_use_autosuspend(&pci->dev); + pm_runtime_put_noidle(&pci->dev); + pm_runtime_allow(&pci->dev); + return 0; +unregister_devs: + for (--index; index >= 0; index--) + platform_device_unregister(adata->pdev[index]); +de_init: + if (acp62_deinit(adata->acp62_base, &pci->dev)) + dev_err(&pci->dev, "ACP de-init failed\n"); +release_regions: + pci_release_regions(pci); +disable_pci: + pci_disable_device(pci); + + return ret; +} + +static int __maybe_unused snd_acp62_suspend(struct device *dev) +{ + struct acp62_dev_data *adata; + int ret; + + adata = dev_get_drvdata(dev); + ret = acp62_deinit(adata->acp62_base, dev); + if (ret) + dev_err(dev, "ACP de-init failed\n"); + return ret; +} + +static int __maybe_unused snd_acp62_resume(struct device *dev) +{ + struct acp62_dev_data *adata; + int ret; + + adata = dev_get_drvdata(dev); + ret = acp62_init(adata->acp62_base, dev); + if (ret) + dev_err(dev, "ACP init failed\n"); + return ret; +} + +static const struct dev_pm_ops acp62_pm_ops = { + SET_RUNTIME_PM_OPS(snd_acp62_suspend, snd_acp62_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(snd_acp62_suspend, snd_acp62_resume) +}; + +static void snd_acp62_remove(struct pci_dev *pci) +{ + struct acp62_dev_data *adata; + int ret, index; + + adata = pci_get_drvdata(pci); + if (adata->acp62_audio_mode == ACP6x_PDM_MODE) { + for (index = 0; index < ACP6x_DEVS; index++) + platform_device_unregister(adata->pdev[index]); + } + ret = acp62_deinit(adata->acp62_base, &pci->dev); + if (ret) + dev_err(&pci->dev, "ACP de-init failed\n"); + pm_runtime_forbid(&pci->dev); + pm_runtime_get_noresume(&pci->dev); + pci_release_regions(pci); + pci_disable_device(pci); +} + +static const struct pci_device_id snd_acp62_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_DEVICE_ID), + .class = PCI_CLASS_MULTIMEDIA_OTHER << 8, + .class_mask = 0xffffff }, + { 0, }, +}; +MODULE_DEVICE_TABLE(pci, snd_acp62_ids); + +static struct pci_driver ps_acp62_driver = { + .name = KBUILD_MODNAME, + .id_table = snd_acp62_ids, + .probe = snd_acp62_probe, + .remove = snd_acp62_remove, + .driver = { + .pm = &acp62_pm_ops, + } +}; + +module_pci_driver(ps_acp62_driver); + +MODULE_AUTHOR("Vijendar.Mukunda@amd.com"); +MODULE_AUTHOR("Syed.SabaKareem@amd.com"); +MODULE_DESCRIPTION("AMD ACP Pink Sardine PCI driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/amd/ps/ps-mach.c b/sound/soc/amd/ps/ps-mach.c new file mode 100644 index 0000000000000000000000000000000000000000..b3e97093481d241aa4bf1638a41d9b69645f5900 --- /dev/null +++ b/sound/soc/amd/ps/ps-mach.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Machine driver for AMD Pink Sardine platform using DMIC + * + * Copyright 2022 Advanced Micro Devices, Inc. + */ + +#include <sound/soc.h> +#include <sound/soc-dapm.h> +#include <linux/module.h> +#include <sound/pcm.h> +#include <sound/pcm_params.h> +#include <linux/io.h> +#include <linux/dmi.h> + +#include "acp62.h" + +#define DRV_NAME "acp_ps_mach" + +SND_SOC_DAILINK_DEF(acp62_pdm, + DAILINK_COMP_ARRAY(COMP_CPU("acp_ps_pdm_dma.0"))); + +SND_SOC_DAILINK_DEF(dmic_codec, + DAILINK_COMP_ARRAY(COMP_CODEC("dmic-codec.0", + "dmic-hifi"))); + +SND_SOC_DAILINK_DEF(pdm_platform, + DAILINK_COMP_ARRAY(COMP_PLATFORM("acp_ps_pdm_dma.0"))); + +static struct snd_soc_dai_link acp62_dai_pdm[] = { + { + .name = "acp62-dmic-capture", + .stream_name = "DMIC capture", + .capture_only = 1, + SND_SOC_DAILINK_REG(acp62_pdm, dmic_codec, pdm_platform), + }, +}; + +static struct snd_soc_card acp62_card = { + .name = "acp62", + .owner = THIS_MODULE, + .dai_link = acp62_dai_pdm, + .num_links = 1, +}; + +static int acp62_probe(struct platform_device *pdev) +{ + struct acp62_pdm *machine = NULL; + struct snd_soc_card *card; + int ret; + + platform_set_drvdata(pdev, &acp62_card); + card = platform_get_drvdata(pdev); + acp62_card.dev = &pdev->dev; + + snd_soc_card_set_drvdata(card, machine); + ret = devm_snd_soc_register_card(&pdev->dev, card); + if (ret) { + return dev_err_probe(&pdev->dev, ret, + "snd_soc_register_card(%s) failed\n", + card->name); + } + + return 0; +} + +static struct platform_driver acp62_mach_driver = { + .driver = { + .name = "acp_ps_mach", + .pm = &snd_soc_pm_ops, + }, + .probe = acp62_probe, +}; + +module_platform_driver(acp62_mach_driver); + +MODULE_AUTHOR("Syed.SabaKareem@amd.com"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:" DRV_NAME); diff --git a/sound/soc/amd/ps/ps-pdm-dma.c b/sound/soc/amd/ps/ps-pdm-dma.c new file mode 100644 index 0000000000000000000000000000000000000000..b207b726cd82999de3fbc95b6bfed48cff7192f4 --- /dev/null +++ b/sound/soc/amd/ps/ps-pdm-dma.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * AMD ALSA SoC Pink Sardine PDM Driver + * + * Copyright 2022 Advanced Micro Devices, Inc. + */ + +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/err.h> +#include <linux/io.h> +#include <sound/pcm_params.h> +#include <sound/soc.h> +#include <sound/soc-dai.h> +#include <linux/pm_runtime.h> + +#include "acp62.h" + +#define DRV_NAME "acp_ps_pdm_dma" + +static const struct snd_pcm_hardware acp62_pdm_hardware_capture = { + .info = SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_BLOCK_TRANSFER | + SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .buffer_bytes_max = CAPTURE_MAX_NUM_PERIODS * CAPTURE_MAX_PERIOD_SIZE, + .period_bytes_min = CAPTURE_MIN_PERIOD_SIZE, + .period_bytes_max = CAPTURE_MAX_PERIOD_SIZE, + .periods_min = CAPTURE_MIN_NUM_PERIODS, + .periods_max = CAPTURE_MAX_NUM_PERIODS, +}; + +static void acp62_init_pdm_ring_buffer(u32 physical_addr, u32 buffer_size, + u32 watermark_size, void __iomem *acp_base) +{ + acp62_writel(physical_addr, acp_base + ACP_WOV_RX_RINGBUFADDR); + acp62_writel(buffer_size, acp_base + ACP_WOV_RX_RINGBUFSIZE); + acp62_writel(watermark_size, acp_base + ACP_WOV_RX_INTR_WATERMARK_SIZE); + acp62_writel(0x01, acp_base + ACPAXI2AXI_ATU_CTRL); +} + +static void acp62_enable_pdm_clock(void __iomem *acp_base) +{ + u32 pdm_clk_enable, pdm_ctrl; + + pdm_clk_enable = ACP_PDM_CLK_FREQ_MASK; + pdm_ctrl = 0x00; + + acp62_writel(pdm_clk_enable, acp_base + ACP_WOV_CLK_CTRL); + pdm_ctrl = acp62_readl(acp_base + ACP_WOV_MISC_CTRL); + pdm_ctrl |= ACP_WOV_MISC_CTRL_MASK; + acp62_writel(pdm_ctrl, acp_base + ACP_WOV_MISC_CTRL); +} + +static void acp62_enable_pdm_interrupts(void __iomem *acp_base) +{ + u32 ext_int_ctrl; + + ext_int_ctrl = acp62_readl(acp_base + ACP_EXTERNAL_INTR_CNTL); + ext_int_ctrl |= PDM_DMA_INTR_MASK; + acp62_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL); +} + +static void acp62_disable_pdm_interrupts(void __iomem *acp_base) +{ + u32 ext_int_ctrl; + + ext_int_ctrl = acp62_readl(acp_base + ACP_EXTERNAL_INTR_CNTL); + ext_int_ctrl &= ~PDM_DMA_INTR_MASK; + acp62_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL); +} + +static bool acp62_check_pdm_dma_status(void __iomem *acp_base) +{ + bool pdm_dma_status; + u32 pdm_enable, pdm_dma_enable; + + pdm_dma_status = false; + pdm_enable = acp62_readl(acp_base + ACP_WOV_PDM_ENABLE); + pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE); + if ((pdm_enable & ACP_PDM_ENABLE) && (pdm_dma_enable & ACP_PDM_DMA_EN_STATUS)) + pdm_dma_status = true; + + return pdm_dma_status; +} + +static int acp62_start_pdm_dma(void __iomem *acp_base) +{ + u32 pdm_enable; + u32 pdm_dma_enable; + int timeout; + + pdm_enable = 0x01; + pdm_dma_enable = 0x01; + + acp62_enable_pdm_clock(acp_base); + acp62_writel(pdm_enable, acp_base + ACP_WOV_PDM_ENABLE); + acp62_writel(pdm_dma_enable, acp_base + ACP_WOV_PDM_DMA_ENABLE); + timeout = 0; + while (++timeout < ACP_COUNTER) { + pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE); + if ((pdm_dma_enable & 0x02) == ACP_PDM_DMA_EN_STATUS) + return 0; + udelay(DELAY_US); + } + return -ETIMEDOUT; +} + +static int acp62_stop_pdm_dma(void __iomem *acp_base) +{ + u32 pdm_enable, pdm_dma_enable; + int timeout; + + pdm_enable = 0x00; + pdm_dma_enable = 0x00; + + pdm_enable = acp62_readl(acp_base + ACP_WOV_PDM_ENABLE); + pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE); + if (pdm_dma_enable & 0x01) { + pdm_dma_enable = 0x02; + acp62_writel(pdm_dma_enable, acp_base + ACP_WOV_PDM_DMA_ENABLE); + timeout = 0; + while (++timeout < ACP_COUNTER) { + pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE); + if ((pdm_dma_enable & 0x02) == 0x00) + break; + udelay(DELAY_US); + } + if (timeout == ACP_COUNTER) + return -ETIMEDOUT; + } + if (pdm_enable == ACP_PDM_ENABLE) { + pdm_enable = ACP_PDM_DISABLE; + acp62_writel(pdm_enable, acp_base + ACP_WOV_PDM_ENABLE); + } + acp62_writel(0x01, acp_base + ACP_WOV_PDM_FIFO_FLUSH); + return 0; +} + +static void acp62_config_dma(struct pdm_stream_instance *rtd, int direction) +{ + u16 page_idx; + u32 low, high, val; + dma_addr_t addr; + + addr = rtd->dma_addr; + val = PDM_PTE_OFFSET; + + /* Group Enable */ + acp62_writel(ACP_SRAM_PTE_OFFSET | BIT(31), rtd->acp62_base + + ACPAXI2AXI_ATU_BASE_ADDR_GRP_1); + acp62_writel(PAGE_SIZE_4K_ENABLE, rtd->acp62_base + + ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1); + for (page_idx = 0; page_idx < rtd->num_pages; page_idx++) { + /* Load the low address of page int ACP SRAM through SRBM */ + low = lower_32_bits(addr); + high = upper_32_bits(addr); + + acp62_writel(low, rtd->acp62_base + ACP_SCRATCH_REG_0 + val); + high |= BIT(31); + acp62_writel(high, rtd->acp62_base + ACP_SCRATCH_REG_0 + val + 4); + val += 8; + addr += PAGE_SIZE; + } +} + +static int acp62_pdm_dma_open(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_pcm_runtime *runtime; + struct pdm_dev_data *adata; + struct pdm_stream_instance *pdm_data; + int ret; + + runtime = substream->runtime; + adata = dev_get_drvdata(component->dev); + pdm_data = kzalloc(sizeof(*pdm_data), GFP_KERNEL); + if (!pdm_data) + return -EINVAL; + + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + runtime->hw = acp62_pdm_hardware_capture; + + ret = snd_pcm_hw_constraint_integer(runtime, + SNDRV_PCM_HW_PARAM_PERIODS); + if (ret < 0) { + dev_err(component->dev, "set integer constraint failed\n"); + kfree(pdm_data); + return ret; + } + + acp62_enable_pdm_interrupts(adata->acp62_base); + + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) + adata->capture_stream = substream; + + pdm_data->acp62_base = adata->acp62_base; + runtime->private_data = pdm_data; + return ret; +} + +static int acp62_pdm_dma_hw_params(struct snd_soc_component *component, + struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct pdm_stream_instance *rtd; + size_t size, period_bytes; + + rtd = substream->runtime->private_data; + if (!rtd) + return -EINVAL; + size = params_buffer_bytes(params); + period_bytes = params_period_bytes(params); + rtd->dma_addr = substream->runtime->dma_addr; + rtd->num_pages = (PAGE_ALIGN(size) >> PAGE_SHIFT); + acp62_config_dma(rtd, substream->stream); + acp62_init_pdm_ring_buffer(PDM_MEM_WINDOW_START, size, + period_bytes, rtd->acp62_base); + return 0; +} + +static u64 acp62_pdm_get_byte_count(struct pdm_stream_instance *rtd, + int direction) +{ + u32 high, low; + u64 byte_count; + + high = acp62_readl(rtd->acp62_base + ACP_WOV_RX_LINEARPOSITIONCNTR_HIGH); + byte_count = high; + low = acp62_readl(rtd->acp62_base + ACP_WOV_RX_LINEARPOSITIONCNTR_LOW); + byte_count = (byte_count << 32) | low; + return byte_count; +} + +static snd_pcm_uframes_t acp62_pdm_dma_pointer(struct snd_soc_component *comp, + struct snd_pcm_substream *stream) +{ + struct pdm_stream_instance *rtd; + u32 pos, buffersize; + u64 bytescount; + + rtd = stream->runtime->private_data; + buffersize = frames_to_bytes(stream->runtime, + stream->runtime->buffer_size); + bytescount = acp62_pdm_get_byte_count(rtd, stream->stream); + if (bytescount > rtd->bytescount) + bytescount -= rtd->bytescount; + pos = do_div(bytescount, buffersize); + return bytes_to_frames(stream->runtime, pos); +} + +static int acp62_pdm_dma_new(struct snd_soc_component *component, + struct snd_soc_pcm_runtime *rtd) +{ + struct device *parent = component->dev->parent; + + snd_pcm_set_managed_buffer_all(rtd->pcm, SNDRV_DMA_TYPE_DEV, + parent, MIN_BUFFER, MAX_BUFFER); + return 0; +} + +static int acp62_pdm_dma_close(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct pdm_dev_data *adata = dev_get_drvdata(component->dev); + struct snd_pcm_runtime *runtime = substream->runtime; + + acp62_disable_pdm_interrupts(adata->acp62_base); + adata->capture_stream = NULL; + kfree(runtime->private_data); + return 0; +} + +static int acp62_pdm_dai_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) +{ + struct pdm_stream_instance *rtd; + int ret; + bool pdm_status; + unsigned int ch_mask; + + rtd = substream->runtime->private_data; + ret = 0; + switch (substream->runtime->channels) { + case TWO_CH: + ch_mask = 0x00; + break; + default: + return -EINVAL; + } + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + acp62_writel(ch_mask, rtd->acp62_base + ACP_WOV_PDM_NO_OF_CHANNELS); + acp62_writel(PDM_DECIMATION_FACTOR, rtd->acp62_base + + ACP_WOV_PDM_DECIMATION_FACTOR); + rtd->bytescount = acp62_pdm_get_byte_count(rtd, substream->stream); + pdm_status = acp62_check_pdm_dma_status(rtd->acp62_base); + if (!pdm_status) + ret = acp62_start_pdm_dma(rtd->acp62_base); + break; + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + pdm_status = acp62_check_pdm_dma_status(rtd->acp62_base); + if (pdm_status) + ret = acp62_stop_pdm_dma(rtd->acp62_base); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static const struct snd_soc_dai_ops acp62_pdm_dai_ops = { + .trigger = acp62_pdm_dai_trigger, +}; + +static struct snd_soc_dai_driver acp62_pdm_dai_driver = { + .name = "acp_ps_pdm_dma.0", + .capture = { + .rates = SNDRV_PCM_RATE_48000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 2, + .rate_min = 48000, + .rate_max = 48000, + }, + .ops = &acp62_pdm_dai_ops, +}; + +static const struct snd_soc_component_driver acp62_pdm_component = { + .name = DRV_NAME, + .open = acp62_pdm_dma_open, + .close = acp62_pdm_dma_close, + .hw_params = acp62_pdm_dma_hw_params, + .pointer = acp62_pdm_dma_pointer, + .pcm_construct = acp62_pdm_dma_new, +}; + +static int acp62_pdm_audio_probe(struct platform_device *pdev) +{ + struct resource *res; + struct pdm_dev_data *adata; + int status; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "IORESOURCE_MEM FAILED\n"); + return -ENODEV; + } + + adata = devm_kzalloc(&pdev->dev, sizeof(*adata), GFP_KERNEL); + if (!adata) + return -ENOMEM; + + adata->acp62_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!adata->acp62_base) + return -ENOMEM; + + adata->capture_stream = NULL; + + dev_set_drvdata(&pdev->dev, adata); + status = devm_snd_soc_register_component(&pdev->dev, + &acp62_pdm_component, + &acp62_pdm_dai_driver, 1); + if (status) { + dev_err(&pdev->dev, "Fail to register acp pdm dai\n"); + + return -ENODEV; + } + pm_runtime_set_autosuspend_delay(&pdev->dev, ACP_SUSPEND_DELAY_MS); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_allow(&pdev->dev); + return 0; +} + +static int acp62_pdm_audio_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + return 0; +} + +static int __maybe_unused acp62_pdm_resume(struct device *dev) +{ + struct pdm_dev_data *adata; + struct snd_pcm_runtime *runtime; + struct pdm_stream_instance *rtd; + u32 period_bytes, buffer_len; + + adata = dev_get_drvdata(dev); + if (adata->capture_stream && adata->capture_stream->runtime) { + runtime = adata->capture_stream->runtime; + rtd = runtime->private_data; + period_bytes = frames_to_bytes(runtime, runtime->period_size); + buffer_len = frames_to_bytes(runtime, runtime->buffer_size); + acp62_config_dma(rtd, SNDRV_PCM_STREAM_CAPTURE); + acp62_init_pdm_ring_buffer(PDM_MEM_WINDOW_START, buffer_len, + period_bytes, adata->acp62_base); + } + acp62_enable_pdm_interrupts(adata->acp62_base); + return 0; +} + +static int __maybe_unused acp62_pdm_suspend(struct device *dev) +{ + struct pdm_dev_data *adata; + + adata = dev_get_drvdata(dev); + acp62_disable_pdm_interrupts(adata->acp62_base); + return 0; +} + +static int __maybe_unused acp62_pdm_runtime_resume(struct device *dev) +{ + struct pdm_dev_data *adata; + + adata = dev_get_drvdata(dev); + acp62_enable_pdm_interrupts(adata->acp62_base); + return 0; +} + +static const struct dev_pm_ops acp62_pdm_pm_ops = { + SET_RUNTIME_PM_OPS(acp62_pdm_suspend, acp62_pdm_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(acp62_pdm_suspend, acp62_pdm_resume) +}; + +static struct platform_driver acp62_pdm_dma_driver = { + .probe = acp62_pdm_audio_probe, + .remove = acp62_pdm_audio_remove, + .driver = { + .name = "acp_ps_pdm_dma", + .pm = &acp62_pdm_pm_ops, + }, +}; + +module_platform_driver(acp62_pdm_dma_driver); + +MODULE_AUTHOR("Syed.SabaKareem@amd.com"); +MODULE_DESCRIPTION("AMD PINK SARDINE PDM Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:" DRV_NAME); diff --git a/sound/soc/amd/vangogh/acp5x-mach.c b/sound/soc/amd/vangogh/acp5x-mach.c index af3737ef970768976a2ab0c8f480b7bac4665f0f..eebf2650ad272415f1bdf5866985630b21fe8dcc 100644 --- a/sound/soc/amd/vangogh/acp5x-mach.c +++ b/sound/soc/amd/vangogh/acp5x-mach.c @@ -172,7 +172,7 @@ static int acp5x_cs35l41_hw_params(struct snd_pcm_substream *substream, struct snd_soc_card *card = rtd->card; struct snd_soc_dai *codec_dai; int ret, i; - unsigned int num_codecs = rtd->num_codecs; + unsigned int num_codecs = rtd->dai_link->num_codecs; unsigned int bclk_val; ret = 0; diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index e0b24e1daef3d56a85d71070c1c778a7e704fcff..2cb50d5cf1a9a48e98343485e30247c13cce760b 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -171,6 +171,20 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21J6"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "82"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "UM5302TA"), + } + }, {} }; diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..793f7782e0d721a1778a82e3515665c3f9f14ede --- /dev/null +++ b/sound/soc/apple/Kconfig @@ -0,0 +1,8 @@ +config SND_SOC_APPLE_MCA + tristate "Apple Silicon MCA driver" + depends on ARCH_APPLE || COMPILE_TEST + select SND_DMAENGINE_PCM + default ARCH_APPLE + help + This option enables an ASoC platform driver for MCA peripherals found + on Apple Silicon SoCs. diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7a30bf452817e0d41d70d0df385662ebcbea1d7a --- /dev/null +++ b/sound/soc/apple/Makefile @@ -0,0 +1,3 @@ +snd-soc-apple-mca-objs := mca.o + +obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c new file mode 100644 index 0000000000000000000000000000000000000000..24381c42eb54c15e50fad471c382f33b078e0e12 --- /dev/null +++ b/sound/soc/apple/mca.c @@ -0,0 +1,1174 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Apple SoCs MCA driver +// +// Copyright (C) The Asahi Linux Contributors +// +// The MCA peripheral is made up of a number of identical units called clusters. +// Each cluster has its separate clock parent, SYNC signal generator, carries +// four SERDES units and has a dedicated I2S port on the SoC's periphery. +// +// The clusters can operate independently, or can be combined together in a +// configurable manner. We mostly treat them as self-contained independent +// units and don't configure any cross-cluster connections except for the I2S +// ports. The I2S ports can be routed to any of the clusters (irrespective +// of their native cluster). We map this onto ASoC's (DPCM) notion of backend +// and frontend DAIs. The 'cluster guts' are frontends which are dynamically +// routed to backend I2S ports. +// +// DAI references in devicetree are resolved to backends. The routing between +// frontends and backends is determined by the machine driver in the DAPM paths +// it supplies. + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_clk.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include <sound/core.h> +#include <sound/pcm.h> +#include <sound/pcm_params.h> +#include <sound/soc.h> +#include <sound/dmaengine_pcm.h> + +#define USE_RXB_FOR_CAPTURE + +/* Relative to cluster base */ +#define REG_STATUS 0x0 +#define STATUS_MCLK_EN BIT(0) +#define REG_MCLK_CONF 0x4 +#define MCLK_CONF_DIV GENMASK(11, 8) + +#define REG_SYNCGEN_STATUS 0x100 +#define SYNCGEN_STATUS_EN BIT(0) +#define REG_SYNCGEN_MCLK_SEL 0x104 +#define SYNCGEN_MCLK_SEL GENMASK(3, 0) +#define REG_SYNCGEN_HI_PERIOD 0x108 +#define REG_SYNCGEN_LO_PERIOD 0x10c + +#define REG_PORT_ENABLES 0x600 +#define PORT_ENABLES_CLOCKS GENMASK(2, 1) +#define PORT_ENABLES_TX_DATA BIT(3) +#define REG_PORT_CLOCK_SEL 0x604 +#define PORT_CLOCK_SEL GENMASK(11, 8) +#define REG_PORT_DATA_SEL 0x608 +#define PORT_DATA_SEL_TXA(cl) (1 << ((cl)*2)) +#define PORT_DATA_SEL_TXB(cl) (2 << ((cl)*2)) + +#define REG_INTSTATE 0x700 +#define REG_INTMASK 0x704 + +/* Bases of serdes units (relative to cluster) */ +#define CLUSTER_RXA_OFF 0x200 +#define CLUSTER_TXA_OFF 0x300 +#define CLUSTER_RXB_OFF 0x400 +#define CLUSTER_TXB_OFF 0x500 + +#define CLUSTER_TX_OFF CLUSTER_TXA_OFF + +#ifndef USE_RXB_FOR_CAPTURE +#define CLUSTER_RX_OFF CLUSTER_RXA_OFF +#else +#define CLUSTER_RX_OFF CLUSTER_RXB_OFF +#endif + +/* Relative to serdes unit base */ +#define REG_SERDES_STATUS 0x00 +#define SERDES_STATUS_EN BIT(0) +#define SERDES_STATUS_RST BIT(1) +#define REG_TX_SERDES_CONF 0x04 +#define REG_RX_SERDES_CONF 0x08 +#define SERDES_CONF_NCHANS GENMASK(3, 0) +#define SERDES_CONF_WIDTH_MASK GENMASK(8, 4) +#define SERDES_CONF_WIDTH_16BIT 0x40 +#define SERDES_CONF_WIDTH_20BIT 0x80 +#define SERDES_CONF_WIDTH_24BIT 0xc0 +#define SERDES_CONF_WIDTH_32BIT 0x100 +#define SERDES_CONF_BCLK_POL 0x400 +#define SERDES_CONF_LSB_FIRST 0x800 +#define SERDES_CONF_UNK1 BIT(12) +#define SERDES_CONF_UNK2 BIT(13) +#define SERDES_CONF_UNK3 BIT(14) +#define SERDES_CONF_NO_DATA_FEEDBACK BIT(15) +#define SERDES_CONF_SYNC_SEL GENMASK(18, 16) +#define SERDES_CONF_SOME_RST BIT(19) +#define REG_TX_SERDES_BITSTART 0x08 +#define REG_RX_SERDES_BITSTART 0x0c +#define REG_TX_SERDES_SLOTMASK 0x0c +#define REG_RX_SERDES_SLOTMASK 0x10 +#define REG_RX_SERDES_PORT 0x04 + +/* Relative to switch base */ +#define REG_DMA_ADAPTER_A(cl) (0x8000 * (cl)) +#define REG_DMA_ADAPTER_B(cl) (0x8000 * (cl) + 0x4000) +#define DMA_ADAPTER_TX_LSB_PAD GENMASK(4, 0) +#define DMA_ADAPTER_TX_NCHANS GENMASK(6, 5) +#define DMA_ADAPTER_RX_MSB_PAD GENMASK(12, 8) +#define DMA_ADAPTER_RX_NCHANS GENMASK(14, 13) +#define DMA_ADAPTER_NCHANS GENMASK(22, 20) + +#define SWITCH_STRIDE 0x8000 +#define CLUSTER_STRIDE 0x4000 + +#define MAX_NCLUSTERS 6 + +#define APPLE_MCA_FMTBITS (SNDRV_PCM_FMTBIT_S16_LE | \ + SNDRV_PCM_FMTBIT_S24_LE | \ + SNDRV_PCM_FMTBIT_S32_LE) + +struct mca_cluster { + int no; + __iomem void *base; + struct mca_data *host; + struct device *pd_dev; + struct clk *clk_parent; + struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1]; + + bool port_started[SNDRV_PCM_STREAM_LAST + 1]; + int port_driver; /* The cluster driving this cluster's port */ + + bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1]; + struct device_link *pd_link; + + unsigned int bclk_ratio; + + /* Masks etc. picked up via the set_tdm_slot method */ + int tdm_slots; + int tdm_slot_width; + unsigned int tdm_tx_mask; + unsigned int tdm_rx_mask; +}; + +struct mca_data { + struct device *dev; + + __iomem void *switch_base; + + struct device *pd_dev; + struct reset_control *rstc; + struct device_link *pd_link; + + /* Mutex for accessing port_driver of foreign clusters */ + struct mutex port_mutex; + + int nclusters; + struct mca_cluster clusters[]; +}; + +static void mca_modify(struct mca_cluster *cl, int regoffset, u32 mask, u32 val) +{ + __iomem void *ptr = cl->base + regoffset; + u32 newval; + + newval = (val & mask) | (readl_relaxed(ptr) & ~mask); + writel_relaxed(newval, ptr); +} + +/* + * Get the cluster of FE or BE DAI + */ +static struct mca_cluster *mca_dai_to_cluster(struct snd_soc_dai *dai) +{ + struct mca_data *mca = snd_soc_dai_get_drvdata(dai); + /* + * FE DAIs are 0 ... nclusters - 1 + * BE DAIs are nclusters ... 2*nclusters - 1 + */ + int cluster_no = dai->id % mca->nclusters; + + return &mca->clusters[cluster_no]; +} + +/* called before PCM trigger */ +static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + int serdes_unit = is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF; + int serdes_conf = + serdes_unit + (is_tx ? REG_TX_SERDES_CONF : REG_RX_SERDES_CONF); + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + mca_modify(cl, serdes_unit + REG_SERDES_STATUS, + SERDES_STATUS_EN | SERDES_STATUS_RST, + SERDES_STATUS_RST); + mca_modify(cl, serdes_conf, SERDES_CONF_SOME_RST, + SERDES_CONF_SOME_RST); + readl_relaxed(cl->base + serdes_conf); + mca_modify(cl, serdes_conf, SERDES_STATUS_RST, 0); + WARN_ON(readl_relaxed(cl->base + REG_SERDES_STATUS) & + SERDES_STATUS_RST); + break; + default: + break; + } +} + +static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + int serdes_unit = is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + mca_modify(cl, serdes_unit + REG_SERDES_STATUS, + SERDES_STATUS_EN | SERDES_STATUS_RST, + SERDES_STATUS_EN); + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + mca_modify(cl, serdes_unit + REG_SERDES_STATUS, + SERDES_STATUS_EN, 0); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int mca_fe_enable_clocks(struct mca_cluster *cl) +{ + struct mca_data *mca = cl->host; + int ret; + + ret = clk_prepare_enable(cl->clk_parent); + if (ret) { + dev_err(mca->dev, + "cluster %d: unable to enable clock parent: %d\n", + cl->no, ret); + return ret; + } + + /* + * We can't power up the device earlier than this because + * the power state driver would error out on seeing the device + * as clock-gated. + */ + cl->pd_link = device_link_add(mca->dev, cl->pd_dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!cl->pd_link) { + dev_err(mca->dev, + "cluster %d: unable to prop-up power domain\n", cl->no); + clk_disable_unprepare(cl->clk_parent); + return -EINVAL; + } + + writel_relaxed(cl->no + 1, cl->base + REG_SYNCGEN_MCLK_SEL); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, + SYNCGEN_STATUS_EN); + mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, STATUS_MCLK_EN); + + return 0; +} + +static void mca_fe_disable_clocks(struct mca_cluster *cl) +{ + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); + mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, 0); + + device_link_del(cl->pd_link); + clk_disable_unprepare(cl->clk_parent); +} + +static bool mca_fe_clocks_in_use(struct mca_cluster *cl) +{ + struct mca_data *mca = cl->host; + struct mca_cluster *be_cl; + int stream, i; + + mutex_lock(&mca->port_mutex); + for (i = 0; i < mca->nclusters; i++) { + be_cl = &mca->clusters[i]; + + if (be_cl->port_driver != cl->no) + continue; + + for_each_pcm_streams(stream) { + if (be_cl->clocks_in_use[stream]) { + mutex_unlock(&mca->port_mutex); + return true; + } + } + } + mutex_unlock(&mca->port_mutex); + return false; +} + +static int mca_be_prepare(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_data *mca = cl->host; + struct mca_cluster *fe_cl; + int ret; + + if (cl->port_driver < 0) + return -EINVAL; + + fe_cl = &mca->clusters[cl->port_driver]; + + /* + * Typically the CODECs we are paired with will require clocks + * to be present at time of unmute with the 'mute_stream' op + * or at time of DAPM widget power-up. We need to enable clocks + * here at the latest (frontend prepare would be too late). + */ + if (!mca_fe_clocks_in_use(fe_cl)) { + ret = mca_fe_enable_clocks(fe_cl); + if (ret < 0) + return ret; + } + + cl->clocks_in_use[substream->stream] = true; + + return 0; +} + +static int mca_be_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_data *mca = cl->host; + struct mca_cluster *fe_cl; + + if (cl->port_driver < 0) + return -EINVAL; + + /* + * We are operating on a foreign cluster here, but since we + * belong to the same PCM, accesses should have been + * synchronized at ASoC level. + */ + fe_cl = &mca->clusters[cl->port_driver]; + if (!mca_fe_clocks_in_use(fe_cl)) + return 0; /* Nothing to do */ + + cl->clocks_in_use[substream->stream] = false; + + if (!mca_fe_clocks_in_use(fe_cl)) + mca_fe_disable_clocks(fe_cl); + + return 0; +} + +static unsigned int mca_crop_mask(unsigned int mask, int nchans) +{ + while (hweight32(mask) > nchans) + mask &= ~(1 << __fls(mask)); + + return mask; +} + +static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, + unsigned int mask, int slots, int nchans, + int slot_width, bool is_tx, int port) +{ + __iomem void *serdes_base = cl->base + serdes_unit; + u32 serdes_conf, serdes_conf_mask; + + serdes_conf_mask = SERDES_CONF_WIDTH_MASK | SERDES_CONF_NCHANS; + serdes_conf = FIELD_PREP(SERDES_CONF_NCHANS, max(slots, 1) - 1); + switch (slot_width) { + case 16: + serdes_conf |= SERDES_CONF_WIDTH_16BIT; + break; + case 20: + serdes_conf |= SERDES_CONF_WIDTH_20BIT; + break; + case 24: + serdes_conf |= SERDES_CONF_WIDTH_24BIT; + break; + case 32: + serdes_conf |= SERDES_CONF_WIDTH_32BIT; + break; + default: + goto err; + } + + serdes_conf_mask |= SERDES_CONF_SYNC_SEL; + serdes_conf |= FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1); + + if (is_tx) { + serdes_conf_mask |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 | + SERDES_CONF_UNK3; + serdes_conf |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 | + SERDES_CONF_UNK3; + } else { + serdes_conf_mask |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 | + SERDES_CONF_UNK3 | + SERDES_CONF_NO_DATA_FEEDBACK; + serdes_conf |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 | + SERDES_CONF_NO_DATA_FEEDBACK; + } + + mca_modify(cl, + serdes_unit + + (is_tx ? REG_TX_SERDES_CONF : REG_RX_SERDES_CONF), + serdes_conf_mask, serdes_conf); + + if (is_tx) { + writel_relaxed(0xffffffff, + serdes_base + REG_TX_SERDES_SLOTMASK); + writel_relaxed(~((u32)mca_crop_mask(mask, nchans)), + serdes_base + REG_TX_SERDES_SLOTMASK + 0x4); + writel_relaxed(0xffffffff, + serdes_base + REG_TX_SERDES_SLOTMASK + 0x8); + writel_relaxed(~((u32)mask), + serdes_base + REG_TX_SERDES_SLOTMASK + 0xc); + } else { + writel_relaxed(0xffffffff, + serdes_base + REG_RX_SERDES_SLOTMASK); + writel_relaxed(~((u32)mca_crop_mask(mask, nchans)), + serdes_base + REG_RX_SERDES_SLOTMASK + 0x4); + writel_relaxed(1 << port, + serdes_base + REG_RX_SERDES_PORT); + } + + return 0; + +err: + dev_err(cl->host->dev, + "unsupported SERDES configuration requested (mask=0x%x slots=%d slot_width=%d)\n", + mask, slots, slot_width); + return -EINVAL; +} + +static int mca_fe_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, + unsigned int rx_mask, int slots, int slot_width) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + + cl->tdm_slots = slots; + cl->tdm_slot_width = slot_width; + cl->tdm_tx_mask = tx_mask; + cl->tdm_rx_mask = rx_mask; + + return 0; +} + +static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_data *mca = cl->host; + bool fpol_inv = false; + u32 serdes_conf = 0; + u32 bitstart; + + if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) != + SND_SOC_DAIFMT_BP_FP) + goto err; + + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + fpol_inv = 0; + bitstart = 1; + break; + case SND_SOC_DAIFMT_LEFT_J: + fpol_inv = 1; + bitstart = 0; + break; + default: + goto err; + } + + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_NB_IF: + case SND_SOC_DAIFMT_IB_IF: + fpol_inv ^= 1; + break; + } + + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_NB_NF: + case SND_SOC_DAIFMT_NB_IF: + serdes_conf |= SERDES_CONF_BCLK_POL; + break; + } + + if (!fpol_inv) + goto err; + + mca_modify(cl, CLUSTER_TX_OFF + REG_TX_SERDES_CONF, + SERDES_CONF_BCLK_POL, serdes_conf); + mca_modify(cl, CLUSTER_RX_OFF + REG_RX_SERDES_CONF, + SERDES_CONF_BCLK_POL, serdes_conf); + writel_relaxed(bitstart, + cl->base + CLUSTER_TX_OFF + REG_TX_SERDES_BITSTART); + writel_relaxed(bitstart, + cl->base + CLUSTER_RX_OFF + REG_RX_SERDES_BITSTART); + + return 0; + +err: + dev_err(mca->dev, "unsupported DAI format (0x%x) requested\n", fmt); + return -EINVAL; +} + +static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + + cl->bclk_ratio = ratio; + + return 0; +} + +static int mca_fe_get_port(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *fe = asoc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *be; + struct snd_soc_dpcm *dpcm; + + be = NULL; + for_each_dpcm_be(fe, substream->stream, dpcm) { + be = dpcm->be; + break; + } + + if (!be) + return -EINVAL; + + return mca_dai_to_cluster(asoc_rtd_to_cpu(be, 0))->no; +} + +static int mca_fe_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_data *mca = cl->host; + struct device *dev = mca->dev; + unsigned int samp_rate = params_rate(params); + bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + bool refine_tdm = false; + unsigned long bclk_ratio; + unsigned int tdm_slots, tdm_slot_width, tdm_mask; + u32 regval, pad; + int ret, port, nchans_ceiled; + + if (!cl->tdm_slot_width) { + /* + * We were not given TDM settings from above, set initial + * guesses which will later be refined. + */ + tdm_slot_width = params_width(params); + tdm_slots = params_channels(params); + refine_tdm = true; + } else { + tdm_slot_width = cl->tdm_slot_width; + tdm_slots = cl->tdm_slots; + tdm_mask = is_tx ? cl->tdm_tx_mask : cl->tdm_rx_mask; + } + + if (cl->bclk_ratio) + bclk_ratio = cl->bclk_ratio; + else + bclk_ratio = tdm_slot_width * tdm_slots; + + if (refine_tdm) { + int nchannels = params_channels(params); + + if (nchannels > 2) { + dev_err(dev, "missing TDM for stream with two or more channels\n"); + return -EINVAL; + } + + if ((bclk_ratio % nchannels) != 0) { + dev_err(dev, "BCLK ratio (%ld) not divisible by no. of channels (%d)\n", + bclk_ratio, nchannels); + return -EINVAL; + } + + tdm_slot_width = bclk_ratio / nchannels; + + if (tdm_slot_width > 32 && nchannels == 1) + tdm_slot_width = 32; + + if (tdm_slot_width < params_width(params)) { + dev_err(dev, "TDM slots too narrow (tdm=%d params=%d)\n", + tdm_slot_width, params_width(params)); + return -EINVAL; + } + + tdm_mask = (1 << tdm_slots) - 1; + } + + port = mca_fe_get_port(substream); + if (port < 0) + return port; + + ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF, + tdm_mask, tdm_slots, params_channels(params), + tdm_slot_width, is_tx, port); + if (ret) + return ret; + + pad = 32 - params_width(params); + + /* + * TODO: Here the register semantics aren't clear. + */ + nchans_ceiled = min_t(int, params_channels(params), 4); + regval = FIELD_PREP(DMA_ADAPTER_NCHANS, nchans_ceiled) | + FIELD_PREP(DMA_ADAPTER_TX_NCHANS, 0x2) | + FIELD_PREP(DMA_ADAPTER_RX_NCHANS, 0x2) | + FIELD_PREP(DMA_ADAPTER_TX_LSB_PAD, pad) | + FIELD_PREP(DMA_ADAPTER_RX_MSB_PAD, pad); + +#ifndef USE_RXB_FOR_CAPTURE + writel_relaxed(regval, mca->switch_base + REG_DMA_ADAPTER_A(cl->no)); +#else + if (is_tx) + writel_relaxed(regval, + mca->switch_base + REG_DMA_ADAPTER_A(cl->no)); + else + writel_relaxed(regval, + mca->switch_base + REG_DMA_ADAPTER_B(cl->no)); +#endif + + if (!mca_fe_clocks_in_use(cl)) { + /* + * Set up FSYNC duty cycle as even as possible. + */ + writel_relaxed((bclk_ratio / 2) - 1, + cl->base + REG_SYNCGEN_HI_PERIOD); + writel_relaxed(((bclk_ratio + 1) / 2) - 1, + cl->base + REG_SYNCGEN_LO_PERIOD); + writel_relaxed(FIELD_PREP(MCLK_CONF_DIV, 0x1), + cl->base + REG_MCLK_CONF); + + ret = clk_set_rate(cl->clk_parent, bclk_ratio * samp_rate); + if (ret) { + dev_err(mca->dev, "cluster %d: unable to set clock parent: %d\n", + cl->no, ret); + return ret; + } + } + + return 0; +} + +static const struct snd_soc_dai_ops mca_fe_ops = { + .set_fmt = mca_fe_set_fmt, + .set_bclk_ratio = mca_set_bclk_ratio, + .set_tdm_slot = mca_fe_set_tdm_slot, + .hw_params = mca_fe_hw_params, + .trigger = mca_fe_trigger, +}; + +static bool mca_be_started(struct mca_cluster *cl) +{ + int stream; + + for_each_pcm_streams(stream) + if (cl->port_started[stream]) + return true; + return false; +} + +static int mca_be_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_pcm_runtime *be = asoc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe; + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; + struct mca_data *mca = cl->host; + struct snd_soc_dpcm *dpcm; + + fe = NULL; + + for_each_dpcm_fe(be, substream->stream, dpcm) { + if (fe && dpcm->fe != fe) { + dev_err(mca->dev, "many FE per one BE unsupported\n"); + return -EINVAL; + } + + fe = dpcm->fe; + } + + if (!fe) + return -EINVAL; + + fe_cl = mca_dai_to_cluster(asoc_rtd_to_cpu(fe, 0)); + + if (mca_be_started(cl)) { + /* + * Port is already started in the other direction. + * Make sure there isn't a conflict with another cluster + * driving the port. + */ + if (cl->port_driver != fe_cl->no) + return -EINVAL; + + cl->port_started[substream->stream] = true; + return 0; + } + + writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA, + cl->base + REG_PORT_ENABLES); + writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1), + cl->base + REG_PORT_CLOCK_SEL); + writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), + cl->base + REG_PORT_DATA_SEL); + mutex_lock(&mca->port_mutex); + cl->port_driver = fe_cl->no; + mutex_unlock(&mca->port_mutex); + cl->port_started[substream->stream] = true; + + return 0; +} + +static void mca_be_shutdown(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_data *mca = cl->host; + + cl->port_started[substream->stream] = false; + + if (!mca_be_started(cl)) { + /* + * Were we the last direction to shutdown? + * Turn off the lights. + */ + writel_relaxed(0, cl->base + REG_PORT_ENABLES); + writel_relaxed(0, cl->base + REG_PORT_DATA_SEL); + mutex_lock(&mca->port_mutex); + cl->port_driver = -1; + mutex_unlock(&mca->port_mutex); + } +} + +static const struct snd_soc_dai_ops mca_be_ops = { + .prepare = mca_be_prepare, + .hw_free = mca_be_hw_free, + .startup = mca_be_startup, + .shutdown = mca_be_shutdown, +}; + +static int mca_set_runtime_hwparams(struct snd_soc_component *component, + struct snd_pcm_substream *substream, + struct dma_chan *chan) +{ + struct device *dma_dev = chan->device->dev; + struct snd_dmaengine_dai_dma_data dma_data = {}; + int ret; + + struct snd_pcm_hardware hw; + + memset(&hw, 0, sizeof(hw)); + + hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED; + hw.periods_min = 2; + hw.periods_max = UINT_MAX; + hw.period_bytes_min = 256; + hw.period_bytes_max = dma_get_max_seg_size(dma_dev); + hw.buffer_bytes_max = SIZE_MAX; + hw.fifo_size = 16; + + ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data, + &hw, chan); + + if (ret) + return ret; + + return snd_soc_set_runtime_hwparams(substream, &hw); +} + +static int mca_pcm_open(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0)); + struct dma_chan *chan = cl->dma_chans[substream->stream]; + int ret; + + if (rtd->dai_link->no_pcm) + return 0; + + ret = mca_set_runtime_hwparams(component, substream, chan); + if (ret) + return ret; + + return snd_dmaengine_pcm_open(substream, chan); +} + +static int mca_hw_params(struct snd_soc_component *component, + struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); + struct dma_slave_config slave_config; + int ret; + + if (rtd->dai_link->no_pcm) + return 0; + + memset(&slave_config, 0, sizeof(slave_config)); + ret = snd_hwparams_to_dma_slave_config(substream, params, + &slave_config); + if (ret < 0) + return ret; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + slave_config.dst_port_window_size = + min_t(u32, params_channels(params), 4); + else + slave_config.src_port_window_size = + min_t(u32, params_channels(params), 4); + + return dmaengine_slave_config(chan, &slave_config); +} + +static int mca_close(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + + if (rtd->dai_link->no_pcm) + return 0; + + return snd_dmaengine_pcm_close(substream); +} + +static int mca_trigger(struct snd_soc_component *component, + struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + + if (rtd->dai_link->no_pcm) + return 0; + + /* + * Before we do the PCM trigger proper, insert an opportunity + * to reset the frontend's SERDES. + */ + mca_fe_early_trigger(substream, cmd, asoc_rtd_to_cpu(rtd, 0)); + + return snd_dmaengine_pcm_trigger(substream, cmd); +} + +static snd_pcm_uframes_t mca_pointer(struct snd_soc_component *component, + struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + + if (rtd->dai_link->no_pcm) + return -ENOTSUPP; + + return snd_dmaengine_pcm_pointer(substream); +} + +static struct dma_chan *mca_request_dma_channel(struct mca_cluster *cl, unsigned int stream) +{ + bool is_tx = (stream == SNDRV_PCM_STREAM_PLAYBACK); +#ifndef USE_RXB_FOR_CAPTURE + char *name = devm_kasprintf(cl->host->dev, GFP_KERNEL, + is_tx ? "tx%da" : "rx%da", cl->no); +#else + char *name = devm_kasprintf(cl->host->dev, GFP_KERNEL, + is_tx ? "tx%da" : "rx%db", cl->no); +#endif + return of_dma_request_slave_channel(cl->host->dev->of_node, name); + +} + +static void mca_pcm_free(struct snd_soc_component *component, + struct snd_pcm *pcm) +{ + struct snd_soc_pcm_runtime *rtd = snd_pcm_chip(pcm); + struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0)); + unsigned int i; + + if (rtd->dai_link->no_pcm) + return; + + for_each_pcm_streams(i) { + struct snd_pcm_substream *substream = + rtd->pcm->streams[i].substream; + + if (!substream || !cl->dma_chans[i]) + continue; + + dma_release_channel(cl->dma_chans[i]); + cl->dma_chans[i] = NULL; + } +} + + +static int mca_pcm_new(struct snd_soc_component *component, + struct snd_soc_pcm_runtime *rtd) +{ + struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0)); + unsigned int i; + + if (rtd->dai_link->no_pcm) + return 0; + + for_each_pcm_streams(i) { + struct snd_pcm_substream *substream = + rtd->pcm->streams[i].substream; + struct dma_chan *chan; + + if (!substream) + continue; + + chan = mca_request_dma_channel(cl, i); + + if (IS_ERR_OR_NULL(chan)) { + dev_err(component->dev, "unable to obtain DMA channel (stream %d cluster %d): %pe\n", + i, cl->no, chan); + mca_pcm_free(component, rtd->pcm); + return -EINVAL; + } + + cl->dma_chans[i] = chan; + snd_pcm_set_managed_buffer(substream, SNDRV_DMA_TYPE_DEV_IRAM, + chan->device->dev, 512 * 1024 * 6, + SIZE_MAX); + } + + return 0; +} + +static const struct snd_soc_component_driver mca_component = { + .name = "apple-mca", + .open = mca_pcm_open, + .close = mca_close, + .hw_params = mca_hw_params, + .trigger = mca_trigger, + .pointer = mca_pointer, + .pcm_construct = mca_pcm_new, + .pcm_destruct = mca_pcm_free, +}; + +static void apple_mca_release(struct mca_data *mca) +{ + int i; + + for (i = 0; i < mca->nclusters; i++) { + struct mca_cluster *cl = &mca->clusters[i]; + + if (!IS_ERR_OR_NULL(cl->clk_parent)) + clk_put(cl->clk_parent); + + if (!IS_ERR_OR_NULL(cl->pd_dev)) + dev_pm_domain_detach(cl->pd_dev, true); + } + + if (mca->pd_link) + device_link_del(mca->pd_link); + + if (!IS_ERR_OR_NULL(mca->pd_dev)) + dev_pm_domain_detach(mca->pd_dev, true); + + reset_control_rearm(mca->rstc); +} + +static int apple_mca_probe(struct platform_device *pdev) +{ + struct mca_data *mca; + struct mca_cluster *clusters; + struct snd_soc_dai_driver *dai_drivers; + struct resource *res; + void __iomem *base; + int nclusters; + int ret, i; + + base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(base)) + return PTR_ERR(base); + + if (resource_size(res) < CLUSTER_STRIDE) + return -EINVAL; + nclusters = (resource_size(res) - CLUSTER_STRIDE) / CLUSTER_STRIDE + 1; + + mca = devm_kzalloc(&pdev->dev, struct_size(mca, clusters, nclusters), + GFP_KERNEL); + if (!mca) + return -ENOMEM; + mca->dev = &pdev->dev; + mca->nclusters = nclusters; + mutex_init(&mca->port_mutex); + platform_set_drvdata(pdev, mca); + clusters = mca->clusters; + + mca->switch_base = + devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(mca->switch_base)) + return PTR_ERR(mca->switch_base); + + mca->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL); + if (IS_ERR(mca->rstc)) + return PTR_ERR(mca->rstc); + + dai_drivers = devm_kzalloc( + &pdev->dev, sizeof(*dai_drivers) * 2 * nclusters, GFP_KERNEL); + if (!dai_drivers) + return -ENOMEM; + + mca->pd_dev = dev_pm_domain_attach_by_id(&pdev->dev, 0); + if (IS_ERR(mca->pd_dev)) + return -EINVAL; + + mca->pd_link = device_link_add(&pdev->dev, mca->pd_dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!mca->pd_link) { + ret = -EINVAL; + /* Prevent an unbalanced reset rearm */ + mca->rstc = NULL; + goto err_release; + } + + reset_control_reset(mca->rstc); + + for (i = 0; i < nclusters; i++) { + struct mca_cluster *cl = &clusters[i]; + struct snd_soc_dai_driver *fe = + &dai_drivers[mca->nclusters + i]; + struct snd_soc_dai_driver *be = &dai_drivers[i]; + + cl->host = mca; + cl->no = i; + cl->base = base + CLUSTER_STRIDE * i; + cl->port_driver = -1; + cl->clk_parent = of_clk_get(pdev->dev.of_node, i); + if (IS_ERR(cl->clk_parent)) { + dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n", + i, PTR_ERR(cl->clk_parent)); + ret = PTR_ERR(cl->clk_parent); + goto err_release; + } + cl->pd_dev = dev_pm_domain_attach_by_id(&pdev->dev, i + 1); + if (IS_ERR(cl->pd_dev)) { + dev_err(&pdev->dev, + "unable to obtain cluster %d PD: %ld\n", i, + PTR_ERR(cl->pd_dev)); + ret = PTR_ERR(cl->pd_dev); + goto err_release; + } + + fe->id = i; + fe->name = + devm_kasprintf(&pdev->dev, GFP_KERNEL, "mca-pcm-%d", i); + if (!fe->name) { + ret = -ENOMEM; + goto err_release; + } + fe->ops = &mca_fe_ops; + fe->playback.channels_min = 1; + fe->playback.channels_max = 32; + fe->playback.rates = SNDRV_PCM_RATE_8000_192000; + fe->playback.formats = APPLE_MCA_FMTBITS; + fe->capture.channels_min = 1; + fe->capture.channels_max = 32; + fe->capture.rates = SNDRV_PCM_RATE_8000_192000; + fe->capture.formats = APPLE_MCA_FMTBITS; + fe->symmetric_rate = 1; + + fe->playback.stream_name = + devm_kasprintf(&pdev->dev, GFP_KERNEL, "PCM%d TX", i); + fe->capture.stream_name = + devm_kasprintf(&pdev->dev, GFP_KERNEL, "PCM%d RX", i); + + if (!fe->playback.stream_name || !fe->capture.stream_name) { + ret = -ENOMEM; + goto err_release; + } + + be->id = i + nclusters; + be->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "mca-i2s-%d", i); + if (!be->name) { + ret = -ENOMEM; + goto err_release; + } + be->ops = &mca_be_ops; + be->playback.channels_min = 1; + be->playback.channels_max = 32; + be->playback.rates = SNDRV_PCM_RATE_8000_192000; + be->playback.formats = APPLE_MCA_FMTBITS; + be->capture.channels_min = 1; + be->capture.channels_max = 32; + be->capture.rates = SNDRV_PCM_RATE_8000_192000; + be->capture.formats = APPLE_MCA_FMTBITS; + + be->playback.stream_name = + devm_kasprintf(&pdev->dev, GFP_KERNEL, "I2S%d TX", i); + be->capture.stream_name = + devm_kasprintf(&pdev->dev, GFP_KERNEL, "I2S%d RX", i); + if (!be->playback.stream_name || !be->capture.stream_name) { + ret = -ENOMEM; + goto err_release; + } + } + + ret = snd_soc_register_component(&pdev->dev, &mca_component, + dai_drivers, nclusters * 2); + if (ret) { + dev_err(&pdev->dev, "unable to register ASoC component: %d\n", + ret); + goto err_release; + } + + return 0; + +err_release: + apple_mca_release(mca); + return ret; +} + +static int apple_mca_remove(struct platform_device *pdev) +{ + struct mca_data *mca = platform_get_drvdata(pdev); + + snd_soc_unregister_component(&pdev->dev); + apple_mca_release(mca); + return 0; +} + +static const struct of_device_id apple_mca_of_match[] = { + { .compatible = "apple,mca", }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_mca_of_match); + +static struct platform_driver apple_mca_driver = { + .driver = { + .name = "apple-mca", + .of_match_table = apple_mca_of_match, + }, + .probe = apple_mca_probe, + .remove = apple_mca_remove, +}; +module_platform_driver(apple_mca_driver); + +MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>"); +MODULE_DESCRIPTION("ASoC Apple MCA driver"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/atmel/atmel_ssc_dai.c b/sound/soc/atmel/atmel_ssc_dai.c index e868b7e028d6cee698dfde1264a6cf8317652103..3763454436c15756960701bea651fa7db9e35fe8 100644 --- a/sound/soc/atmel/atmel_ssc_dai.c +++ b/sound/soc/atmel/atmel_ssc_dai.c @@ -891,7 +891,6 @@ static int asoc_ssc_init(struct device *dev) int atmel_ssc_set_audio(int ssc_id) { struct ssc_device *ssc; - int ret; /* If we can grab the SSC briefly to parent the DAI device off it */ ssc = ssc_request(ssc_id); @@ -903,9 +902,7 @@ int atmel_ssc_set_audio(int ssc_id) ssc_info[ssc_id].ssc = ssc; } - ret = asoc_ssc_init(&ssc->pdev->dev); - - return ret; + return asoc_ssc_init(&ssc->pdev->dev); } EXPORT_SYMBOL_GPL(atmel_ssc_set_audio); diff --git a/sound/soc/atmel/mchp-spdiftx.c b/sound/soc/atmel/mchp-spdiftx.c index 4850a177803dbee786c34a5aa93793cbc917d167..ab2d7a791f39ce3dc742fe325e786d75d98c2943 100644 --- a/sound/soc/atmel/mchp-spdiftx.c +++ b/sound/soc/atmel/mchp-spdiftx.c @@ -196,7 +196,7 @@ struct mchp_spdiftx_dev { struct clk *pclk; struct clk *gclk; unsigned int fmt; - int gclk_enabled:1; + unsigned int gclk_enabled:1; }; static inline int mchp_spdiftx_is_running(struct mchp_spdiftx_dev *dev) diff --git a/sound/soc/atmel/sam9g20_wm8731.c b/sound/soc/atmel/sam9g20_wm8731.c index 4d25fb61c6527fd3d71b874951f09d1c750b4fc2..1430642c8433ab11d35aa953bd08c879cff7a21a 100644 --- a/sound/soc/atmel/sam9g20_wm8731.c +++ b/sound/soc/atmel/sam9g20_wm8731.c @@ -172,7 +172,7 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev) ret = snd_soc_register_card(card); if (ret) { dev_err_probe(&pdev->dev, ret, - "snd_soc_register_card() failed: %d\n", ret); + "snd_soc_register_card() failed\n"); goto err; } diff --git a/sound/soc/bcm/bcm2835-i2s.c b/sound/soc/bcm/bcm2835-i2s.c index f4d84774dac72851551456f2981f332ddbd0c1ff..85f705afcdbbbb34e12fe56cbc466b58ca65eba1 100644 --- a/sound/soc/bcm/bcm2835-i2s.c +++ b/sound/soc/bcm/bcm2835-i2s.c @@ -841,14 +841,9 @@ static int bcm2835_i2s_probe(struct platform_device *pdev) /* get the clock */ dev->clk_prepared = false; dev->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(dev->clk)) { - ret = PTR_ERR(dev->clk); - if (ret == -EPROBE_DEFER) - dev_dbg(&pdev->dev, "could not get clk: %d\n", ret); - else - dev_err(&pdev->dev, "could not get clk: %d\n", ret); - return ret; - } + if (IS_ERR(dev->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk), + "could not get clk\n"); /* Request ioarea */ base = devm_platform_ioremap_resource(pdev, 0); diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index d16b4efb88a77b2c9a91ebc6f97db86d4321cb5a..e3b90c425fafebbe769e8f875c42ec0af1a73d34 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -98,6 +98,7 @@ config SND_SOC_ALL_CODECS imply SND_SOC_DA9055 imply SND_SOC_DMIC imply SND_SOC_ES8316 + imply SND_SOC_ES8326 imply SND_SOC_ES8328_SPI imply SND_SOC_ES8328_I2C imply SND_SOC_ES7134 @@ -205,6 +206,7 @@ config SND_SOC_ALL_CODECS imply SND_SOC_SIMPLE_AMPLIFIER imply SND_SOC_SIMPLE_MUX imply SND_SOC_SPDIF + imply SND_SOC_SRC4XXX_I2C imply SND_SOC_SSM2305 imply SND_SOC_SSM2518 imply SND_SOC_SSM2602_SPI @@ -608,7 +610,7 @@ config SND_SOC_BT_SCO config SND_SOC_CPCAP tristate "Motorola CPCAP codec" - depends on MFD_CPCAP + depends on MFD_CPCAP || COMPILE_TEST config SND_SOC_CQ0093VC tristate @@ -690,9 +692,15 @@ config SND_SOC_CS35L45_I2C Enable support for Cirrus Logic CS35L45 smart speaker amplifier with I2C control. +config SND_SOC_CS42L42_CORE + tristate + config SND_SOC_CS42L42 - tristate "Cirrus Logic CS42L42 CODEC" + tristate "Cirrus Logic CS42L42 CODEC (I2C)" depends on I2C + select REGMAP + select REGMAP_I2C + select SND_SOC_CS42L42_CORE config SND_SOC_CS42L51 tristate @@ -714,6 +722,13 @@ config SND_SOC_CS42L73 tristate "Cirrus Logic CS42L73 CODEC" depends on I2C +config SND_SOC_CS42L83 + tristate "Cirrus Logic CS42L83 CODEC" + depends on I2C + select REGMAP + select REGMAP_I2C + select SND_SOC_CS42L42_CORE + config SND_SOC_CS4234 tristate "Cirrus Logic CS4234 CODEC" depends on I2C @@ -913,6 +928,10 @@ config SND_SOC_ES8316 tristate "Everest Semi ES8316 CODEC" depends on I2C +config SND_SOC_ES8326 + tristate "Everest Semi ES8326 CODEC" + depends on I2C + config SND_SOC_ES8328 tristate @@ -966,7 +985,7 @@ config SND_SOC_LM49453 config SND_SOC_LOCHNAGAR_SC tristate "Lochnagar Sound Card" - depends on MFD_LOCHNAGAR + depends on MFD_LOCHNAGAR || COMPILE_TEST help This driver support the sound card functionality of the Cirrus Logic Lochnagar audio development board. @@ -1191,8 +1210,7 @@ config SND_SOC_RK3328 config SND_SOC_RK817 tristate "Rockchip RK817 audio CODEC" - depends on MFD_RK808 - select REGMAP_I2C + depends on MFD_RK808 || COMPILE_TEST config SND_SOC_RL6231 tristate @@ -1471,6 +1489,18 @@ config SND_SOC_SIMPLE_MUX config SND_SOC_SPDIF tristate "S/PDIF CODEC" +config SND_SOC_SRC4XXX_I2C + tristate "Texas Instruments SRC4XXX DIR/DIT and SRC codecs" + depends on I2C + select SND_SOC_SRC4XXX + help + Enable support for the TI SRC4XXX family of codecs. These include the + scr4392 which has digital receivers, transmitters, and + a sample rate converter, including numerous ports. + +config SND_SOC_SRC4XXX + tristate + config SND_SOC_SSM2305 tristate "Analog Devices SSM2305 Class-D Amplifier" help @@ -1726,8 +1756,10 @@ config SND_SOC_WCD_MBHC config SND_SOC_WCD934X tristate "WCD9340/WCD9341 Codec" depends on COMMON_CLK + depends on SLIMBUS + select REGMAP_SLIMBUS select SND_SOC_WCD_MBHC - depends on MFD_WCD934X + depends on MFD_WCD934X || COMPILE_TEST help The WCD9340/9341 is a audio codec IC Integrated in Qualcomm SoCs like SDM845. diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index 92fd441d426a83f4f2c2454b6eaa4a59208e07fe..9170ee1447dda2ef90dbedecaf117c41338ff3b7 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -65,11 +65,13 @@ snd-soc-cs35l45-objs := cs35l45.o snd-soc-cs35l45-spi-objs := cs35l45-spi.o snd-soc-cs35l45-i2c-objs := cs35l45-i2c.o snd-soc-cs42l42-objs := cs42l42.o +snd-soc-cs42l42-i2c-objs := cs42l42-i2c.o snd-soc-cs42l51-objs := cs42l51.o snd-soc-cs42l51-i2c-objs := cs42l51-i2c.o snd-soc-cs42l52-objs := cs42l52.o snd-soc-cs42l56-objs := cs42l56.o snd-soc-cs42l73-objs := cs42l73.o +snd-soc-cs42l83-i2c-objs := cs42l83-i2c.o snd-soc-cs4234-objs := cs4234.o snd-soc-cs4265-objs := cs4265.o snd-soc-cs4270-objs := cs4270.o @@ -100,6 +102,7 @@ snd-soc-dmic-objs := dmic.o snd-soc-es7134-objs := es7134.o snd-soc-es7241-objs := es7241.o snd-soc-es8316-objs := es8316.o +snd-soc-es8326-objs := es8326.o snd-soc-es8328-objs := es8328.o snd-soc-es8328-i2c-objs := es8328-i2c.o snd-soc-es8328-spi-objs := es8328-spi.o @@ -231,6 +234,8 @@ snd-soc-sigmadsp-regmap-objs := sigmadsp-regmap.o snd-soc-si476x-objs := si476x.o snd-soc-spdif-tx-objs := spdif_transmitter.o snd-soc-spdif-rx-objs := spdif_receiver.o +snd-soc-src4xxx-objs := src4xxx.o +snd-soc-src4xxx-i2c-objs := src4xxx-i2c.o snd-soc-ssm2305-objs := ssm2305.o snd-soc-ssm2518-objs := ssm2518.o snd-soc-ssm2602-objs := ssm2602.o @@ -419,12 +424,14 @@ obj-$(CONFIG_SND_SOC_CS35L45_TABLES) += snd-soc-cs35l45-tables.o obj-$(CONFIG_SND_SOC_CS35L45) += snd-soc-cs35l45.o obj-$(CONFIG_SND_SOC_CS35L45_SPI) += snd-soc-cs35l45-spi.o obj-$(CONFIG_SND_SOC_CS35L45_I2C) += snd-soc-cs35l45-i2c.o -obj-$(CONFIG_SND_SOC_CS42L42) += snd-soc-cs42l42.o +obj-$(CONFIG_SND_SOC_CS42L42_CORE) += snd-soc-cs42l42.o +obj-$(CONFIG_SND_SOC_CS42L42) += snd-soc-cs42l42-i2c.o obj-$(CONFIG_SND_SOC_CS42L51) += snd-soc-cs42l51.o obj-$(CONFIG_SND_SOC_CS42L51_I2C) += snd-soc-cs42l51-i2c.o obj-$(CONFIG_SND_SOC_CS42L52) += snd-soc-cs42l52.o obj-$(CONFIG_SND_SOC_CS42L56) += snd-soc-cs42l56.o obj-$(CONFIG_SND_SOC_CS42L73) += snd-soc-cs42l73.o +obj-$(CONFIG_SND_SOC_CS42L83) += snd-soc-cs42l83-i2c.o obj-$(CONFIG_SND_SOC_CS4234) += snd-soc-cs4234.o obj-$(CONFIG_SND_SOC_CS4265) += snd-soc-cs4265.o obj-$(CONFIG_SND_SOC_CS4270) += snd-soc-cs4270.o @@ -455,6 +462,7 @@ obj-$(CONFIG_SND_SOC_DMIC) += snd-soc-dmic.o obj-$(CONFIG_SND_SOC_ES7134) += snd-soc-es7134.o obj-$(CONFIG_SND_SOC_ES7241) += snd-soc-es7241.o obj-$(CONFIG_SND_SOC_ES8316) += snd-soc-es8316.o +obj-$(CONFIG_SND_SOC_ES8326) += snd-soc-es8326.o obj-$(CONFIG_SND_SOC_ES8328) += snd-soc-es8328.o obj-$(CONFIG_SND_SOC_ES8328_I2C)+= snd-soc-es8328-i2c.o obj-$(CONFIG_SND_SOC_ES8328_SPI)+= snd-soc-es8328-spi.o @@ -579,6 +587,8 @@ obj-$(CONFIG_SND_SOC_SIGMADSP_I2C) += snd-soc-sigmadsp-i2c.o obj-$(CONFIG_SND_SOC_SIGMADSP_REGMAP) += snd-soc-sigmadsp-regmap.o obj-$(CONFIG_SND_SOC_SI476X) += snd-soc-si476x.o obj-$(CONFIG_SND_SOC_SPDIF) += snd-soc-spdif-rx.o snd-soc-spdif-tx.o +obj-$(CONFIG_SND_SOC_SRC4XXX) += snd-soc-src4xxx.o +obj-$(CONFIG_SND_SOC_SRC4XXX_I2C) += snd-soc-src4xxx-i2c.o obj-$(CONFIG_SND_SOC_SSM2305) += snd-soc-ssm2305.o obj-$(CONFIG_SND_SOC_SSM2518) += snd-soc-ssm2518.o obj-$(CONFIG_SND_SOC_SSM2602) += snd-soc-ssm2602.o diff --git a/sound/soc/codecs/adau1761-i2c.c b/sound/soc/codecs/adau1761-i2c.c index 0683caf86aea0199ec603a14b64c4b9253968e1c..0cefff49569c2ea1dec86d01ae5eba2da93e6b83 100644 --- a/sound/soc/codecs/adau1761-i2c.c +++ b/sound/soc/codecs/adau1761-i2c.c @@ -30,10 +30,9 @@ static int adau1761_i2c_probe(struct i2c_client *client) id->driver_data, NULL); } -static int adau1761_i2c_remove(struct i2c_client *client) +static void adau1761_i2c_remove(struct i2c_client *client) { adau17x1_remove(&client->dev); - return 0; } static const struct i2c_device_id adau1761_i2c_ids[] = { diff --git a/sound/soc/codecs/adau1781-i2c.c b/sound/soc/codecs/adau1781-i2c.c index e046de0ebcc7986928ccf2ac5e22d6b1ae397ff0..39021b8cfb626f7f9f3745e6312af912d7acef92 100644 --- a/sound/soc/codecs/adau1781-i2c.c +++ b/sound/soc/codecs/adau1781-i2c.c @@ -30,10 +30,9 @@ static int adau1781_i2c_probe(struct i2c_client *client) id->driver_data, NULL); } -static int adau1781_i2c_remove(struct i2c_client *client) +static void adau1781_i2c_remove(struct i2c_client *client) { adau17x1_remove(&client->dev); - return 0; } static const struct i2c_device_id adau1781_i2c_ids[] = { diff --git a/sound/soc/codecs/ak4375.c b/sound/soc/codecs/ak4375.c index 1ed004ba7cd2333c728c48d6a239946e90fa086f..573389e402f836c4d34025b21391f637722ffe3f 100644 --- a/sound/soc/codecs/ak4375.c +++ b/sound/soc/codecs/ak4375.c @@ -580,11 +580,9 @@ static int ak4375_i2c_probe(struct i2c_client *i2c) return 0; } -static int ak4375_i2c_remove(struct i2c_client *i2c) +static void ak4375_i2c_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); - - return 0; } static const struct of_device_id ak4375_of_match[] = { diff --git a/sound/soc/codecs/ak4458.c b/sound/soc/codecs/ak4458.c index ea33cc83c86c2da05233ffe8ebdbeb003837846a..1db73552c7466b8b9128a7a18f86706fc83ec25d 100644 --- a/sound/soc/codecs/ak4458.c +++ b/sound/soc/codecs/ak4458.c @@ -447,6 +447,13 @@ static int ak4458_hw_params(struct snd_pcm_substream *substream, snd_soc_component_update_bits(component, AK4458_0B_CONTROL7, AK4458_DCHAIN_MASK, dchn); + if (ak4458->drvdata->type == AK4497) { + ret = snd_soc_component_update_bits(component, AK4458_09_DSD2, + 0x4, (ak4458->dsd_path << 2)); + if (ret < 0) + return ret; + } + ret = ak4458_rstn_control(component, 0); if (ret) return ret; @@ -629,48 +636,6 @@ static void ak4458_reset(struct ak4458_priv *ak4458, bool active) } } -static int ak4458_init(struct snd_soc_component *component) -{ - struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component); - int ret; - - /* External Mute ON */ - if (ak4458->mute_gpiod) - gpiod_set_value_cansleep(ak4458->mute_gpiod, 1); - - ak4458_reset(ak4458, false); - - ret = snd_soc_component_update_bits(component, AK4458_00_CONTROL1, - 0x80, 0x80); /* ACKS bit = 1; 10000000 */ - if (ret < 0) - return ret; - - if (ak4458->drvdata->type == AK4497) { - ret = snd_soc_component_update_bits(component, AK4458_09_DSD2, - 0x4, (ak4458->dsd_path << 2)); - if (ret < 0) - return ret; - } - - return ak4458_rstn_control(component, 1); -} - -static int ak4458_probe(struct snd_soc_component *component) -{ - struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component); - - ak4458->fs = 48000; - - return ak4458_init(component); -} - -static void ak4458_remove(struct snd_soc_component *component) -{ - struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component); - - ak4458_reset(ak4458, true); -} - #ifdef CONFIG_PM static int __maybe_unused ak4458_runtime_suspend(struct device *dev) { @@ -714,8 +679,6 @@ static int __maybe_unused ak4458_runtime_resume(struct device *dev) #endif /* CONFIG_PM */ static const struct snd_soc_component_driver soc_codec_dev_ak4458 = { - .probe = ak4458_probe, - .remove = ak4458_remove, .controls = ak4458_snd_controls, .num_controls = ARRAY_SIZE(ak4458_snd_controls), .dapm_widgets = ak4458_dapm_widgets, @@ -728,8 +691,6 @@ static const struct snd_soc_component_driver soc_codec_dev_ak4458 = { }; static const struct snd_soc_component_driver soc_codec_dev_ak4497 = { - .probe = ak4458_probe, - .remove = ak4458_remove, .controls = ak4497_snd_controls, .num_controls = ARRAY_SIZE(ak4497_snd_controls), .dapm_widgets = ak4497_dapm_widgets, @@ -820,15 +781,17 @@ static int ak4458_i2c_probe(struct i2c_client *i2c) pm_runtime_enable(&i2c->dev); regcache_cache_only(ak4458->regmap, true); + ak4458_reset(ak4458, false); return 0; } -static int ak4458_i2c_remove(struct i2c_client *i2c) +static void ak4458_i2c_remove(struct i2c_client *i2c) { - pm_runtime_disable(&i2c->dev); + struct ak4458_priv *ak4458 = i2c_get_clientdata(i2c); - return 0; + ak4458_reset(ak4458, true); + pm_runtime_disable(&i2c->dev); } static const struct of_device_id ak4458_of_match[] = { diff --git a/sound/soc/codecs/ak4641.c b/sound/soc/codecs/ak4641.c index 88851e94b045219832b9b4d3e4b9c21d28028769..0d3ee195b3cc2b0811a86b813fc925a9e136d2d5 100644 --- a/sound/soc/codecs/ak4641.c +++ b/sound/soc/codecs/ak4641.c @@ -604,7 +604,7 @@ err_out: return ret; } -static int ak4641_i2c_remove(struct i2c_client *i2c) +static void ak4641_i2c_remove(struct i2c_client *i2c) { struct ak4641_platform_data *pdata = i2c->dev.platform_data; @@ -616,8 +616,6 @@ static int ak4641_i2c_remove(struct i2c_client *i2c) if (gpio_is_valid(pdata->gpio_npdn)) gpio_free(pdata->gpio_npdn); } - - return 0; } static const struct i2c_device_id ak4641_i2c_id[] = { diff --git a/sound/soc/codecs/ak5558.c b/sound/soc/codecs/ak5558.c index 887d2c04d647a11a40efafb5536efc1bc553e710..60abcffe6a0ce5806357cfbddc5d76b567d2481f 100644 --- a/sound/soc/codecs/ak5558.c +++ b/sound/soc/codecs/ak5558.c @@ -479,11 +479,9 @@ static int ak5558_i2c_probe(struct i2c_client *i2c) return 0; } -static int ak5558_i2c_remove(struct i2c_client *i2c) +static void ak5558_i2c_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); - - return 0; } static const struct of_device_id ak5558_i2c_dt_ids[] __maybe_unused = { diff --git a/sound/soc/codecs/cs35l32.c b/sound/soc/codecs/cs35l32.c index 8ff6f66be86fa92dfd3483080e9c85d2bbf2498d..dc7a58d680762763ea169b039dae0bc49a7db625 100644 --- a/sound/soc/codecs/cs35l32.c +++ b/sound/soc/codecs/cs35l32.c @@ -497,14 +497,12 @@ err_supplies: return ret; } -static int cs35l32_i2c_remove(struct i2c_client *i2c_client) +static void cs35l32_i2c_remove(struct i2c_client *i2c_client) { struct cs35l32_private *cs35l32 = i2c_get_clientdata(i2c_client); /* Hold down reset */ gpiod_set_value_cansleep(cs35l32->reset_gpio, 0); - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c index 082025fa0370c06942a21fe12218f1060579f8dc..15e79168d256e425fcf0727dd2fab04227a23e24 100644 --- a/sound/soc/codecs/cs35l33.c +++ b/sound/soc/codecs/cs35l33.c @@ -1250,7 +1250,7 @@ err_enable: return ret; } -static int cs35l33_i2c_remove(struct i2c_client *client) +static void cs35l33_i2c_remove(struct i2c_client *client) { struct cs35l33_private *cs35l33 = i2c_get_clientdata(client); @@ -1259,8 +1259,6 @@ static int cs35l33_i2c_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); regulator_bulk_disable(cs35l33->num_core_supplies, cs35l33->core_supplies); - - return 0; } static const struct of_device_id cs35l33_of_match[] = { diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c index 472ac982779be55811b40837a3016dc4eadfe2b0..b3f98023e6a758c106b03f76ccb8ea3edc2c8351 100644 --- a/sound/soc/codecs/cs35l34.c +++ b/sound/soc/codecs/cs35l34.c @@ -1128,7 +1128,7 @@ err_regulator: return ret; } -static int cs35l34_i2c_remove(struct i2c_client *client) +static void cs35l34_i2c_remove(struct i2c_client *client) { struct cs35l34_private *cs35l34 = i2c_get_clientdata(client); @@ -1137,8 +1137,6 @@ static int cs35l34_i2c_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); regulator_bulk_disable(cs35l34->num_core_supplies, cs35l34->core_supplies); - - return 0; } static int __maybe_unused cs35l34_runtime_resume(struct device *dev) diff --git a/sound/soc/codecs/cs35l35.c b/sound/soc/codecs/cs35l35.c index 714a759dca21bf160b5daee0cba7f26bddfb1e01..947a440a3a47a729f3e002cedb57f9f50eee213c 100644 --- a/sound/soc/codecs/cs35l35.c +++ b/sound/soc/codecs/cs35l35.c @@ -1627,14 +1627,12 @@ err: return ret; } -static int cs35l35_i2c_remove(struct i2c_client *i2c_client) +static void cs35l35_i2c_remove(struct i2c_client *i2c_client) { struct cs35l35_private *cs35l35 = i2c_get_clientdata(i2c_client); regulator_bulk_disable(cs35l35->num_supplies, cs35l35->supplies); gpiod_set_value_cansleep(cs35l35->reset_gpio, 0); - - return 0; } static const struct of_device_id cs35l35_of_match[] = { diff --git a/sound/soc/codecs/cs35l36.c b/sound/soc/codecs/cs35l36.c index 4dc13e6f4874cbf85ffa7ed8e86e95788d8855f0..31ae752e242f89189c93ad690bd0ffeadaef208d 100644 --- a/sound/soc/codecs/cs35l36.c +++ b/sound/soc/codecs/cs35l36.c @@ -1910,7 +1910,7 @@ err_disable_regs: return ret; } -static int cs35l36_i2c_remove(struct i2c_client *client) +static void cs35l36_i2c_remove(struct i2c_client *client) { struct cs35l36_private *cs35l36 = i2c_get_clientdata(client); @@ -1924,8 +1924,6 @@ static int cs35l36_i2c_remove(struct i2c_client *client) gpiod_set_value_cansleep(cs35l36->reset_gpio, 0); regulator_bulk_disable(cs35l36->num_supplies, cs35l36->supplies); - - return 0; } static const struct of_device_id cs35l36_of_match[] = { {.compatible = "cirrus,cs35l36"}, diff --git a/sound/soc/codecs/cs35l41-i2c.c b/sound/soc/codecs/cs35l41-i2c.c index 37c703c08fd592ba373236a54035e3f7f12fa4c1..3676b596f60bec5b42359efa58231d5124c11d87 100644 --- a/sound/soc/codecs/cs35l41-i2c.c +++ b/sound/soc/codecs/cs35l41-i2c.c @@ -56,13 +56,11 @@ static int cs35l41_i2c_probe(struct i2c_client *client) return cs35l41_probe(cs35l41, hw_cfg); } -static int cs35l41_i2c_remove(struct i2c_client *client) +static void cs35l41_i2c_remove(struct i2c_client *client) { struct cs35l41_private *cs35l41 = i2c_get_clientdata(client); cs35l41_remove(cs35l41); - - return 0; } #ifdef CONFIG_OF diff --git a/sound/soc/codecs/cs35l45-i2c.c b/sound/soc/codecs/cs35l45-i2c.c index 06c2ddffb9c53c2d69bf34da9bcb7295b7e1f5f2..39d28641429e4d1eb29ed68324b203550f7f3898 100644 --- a/sound/soc/codecs/cs35l45-i2c.c +++ b/sound/soc/codecs/cs35l45-i2c.c @@ -36,13 +36,11 @@ static int cs35l45_i2c_probe(struct i2c_client *client) return cs35l45_probe(cs35l45); } -static int cs35l45_i2c_remove(struct i2c_client *client) +static void cs35l45_i2c_remove(struct i2c_client *client) { struct cs35l45_private *cs35l45 = i2c_get_clientdata(client); cs35l45_remove(cs35l45); - - return 0; } static const struct of_device_id cs35l45_of_match[] = { diff --git a/sound/soc/codecs/cs4234.c b/sound/soc/codecs/cs4234.c index b49a3cf21ebe275138a07129a7a6dfa993ff44c9..dee1a6662c2e1874869d743b964374a6fded850e 100644 --- a/sound/soc/codecs/cs4234.c +++ b/sound/soc/codecs/cs4234.c @@ -850,7 +850,7 @@ fail_shutdown: return ret; } -static int cs4234_i2c_remove(struct i2c_client *i2c_client) +static void cs4234_i2c_remove(struct i2c_client *i2c_client) { struct cs4234 *cs4234 = i2c_get_clientdata(i2c_client); struct device *dev = &i2c_client->dev; @@ -858,8 +858,6 @@ static int cs4234_i2c_remove(struct i2c_client *i2c_client) snd_soc_unregister_component(dev); pm_runtime_disable(dev); cs4234_shutdown(cs4234); - - return 0; } static int __maybe_unused cs4234_runtime_resume(struct device *dev) diff --git a/sound/soc/codecs/cs4265.c b/sound/soc/codecs/cs4265.c index 76c19802d5fe1ea03c16dc9ec9c54112095d7ac2..3573363b7e3122447de13a171cd2e4546ab2e83c 100644 --- a/sound/soc/codecs/cs4265.c +++ b/sound/soc/codecs/cs4265.c @@ -623,14 +623,12 @@ static int cs4265_i2c_probe(struct i2c_client *i2c_client) ARRAY_SIZE(cs4265_dai)); } -static int cs4265_i2c_remove(struct i2c_client *i2c) +static void cs4265_i2c_remove(struct i2c_client *i2c) { struct cs4265_private *cs4265 = i2c_get_clientdata(i2c); if (cs4265->reset_gpio) gpiod_set_value_cansleep(cs4265->reset_gpio, 0); - - return 0; } static const struct of_device_id cs4265_of_match[] = { diff --git a/sound/soc/codecs/cs4270.c b/sound/soc/codecs/cs4270.c index ba67e43edf35139ac96985ab720996abc320761d..1b640d8232badc67a8a4bdfc79cc1c8638f46fe3 100644 --- a/sound/soc/codecs/cs4270.c +++ b/sound/soc/codecs/cs4270.c @@ -650,13 +650,11 @@ static const struct regmap_config cs4270_regmap = { * This function puts the chip into low power mode when the i2c device * is removed. */ -static int cs4270_i2c_remove(struct i2c_client *i2c_client) +static void cs4270_i2c_remove(struct i2c_client *i2c_client) { struct cs4270_private *cs4270 = i2c_get_clientdata(i2c_client); gpiod_set_value_cansleep(cs4270->reset_gpio, 0); - - return 0; } /** diff --git a/sound/soc/codecs/cs42l42-i2c.c b/sound/soc/codecs/cs42l42-i2c.c new file mode 100644 index 0000000000000000000000000000000000000000..67b253287dafec53d7b414739fbcb457f11842c6 --- /dev/null +++ b/sound/soc/codecs/cs42l42-i2c.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * cs42l42-i2c.c -- CS42L42 ALSA SoC audio driver for I2C + * + * Copyright 2016, 2022 Cirrus Logic, Inc. + */ + +#include <linux/i2c.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "cs42l42.h" + +static int cs42l42_i2c_probe(struct i2c_client *i2c_client) +{ + struct device *dev = &i2c_client->dev; + struct cs42l42_private *cs42l42; + struct regmap *regmap; + int ret; + + cs42l42 = devm_kzalloc(dev, sizeof(*cs42l42), GFP_KERNEL); + if (!cs42l42) + return -ENOMEM; + + regmap = devm_regmap_init_i2c(i2c_client, &cs42l42_regmap); + if (IS_ERR(regmap)) + return dev_err_probe(&i2c_client->dev, PTR_ERR(regmap), + "regmap_init() failed\n"); + + cs42l42->devid = CS42L42_CHIP_ID; + cs42l42->dev = dev; + cs42l42->regmap = regmap; + cs42l42->irq = i2c_client->irq; + + ret = cs42l42_common_probe(cs42l42, &cs42l42_soc_component, &cs42l42_dai); + if (ret) + return ret; + + return cs42l42_init(cs42l42); +} + +static void cs42l42_i2c_remove(struct i2c_client *i2c_client) +{ + struct cs42l42_private *cs42l42 = dev_get_drvdata(&i2c_client->dev); + + cs42l42_common_remove(cs42l42); +} + +static int __maybe_unused cs42l42_i2c_resume(struct device *dev) +{ + int ret; + + ret = cs42l42_resume(dev); + if (ret) + return ret; + + cs42l42_resume_restore(dev); + + return 0; +} + +static const struct dev_pm_ops cs42l42_i2c_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l42_i2c_resume) +}; + +static const struct of_device_id __maybe_unused cs42l42_of_match[] = { + { .compatible = "cirrus,cs42l42", }, + {} +}; +MODULE_DEVICE_TABLE(of, cs42l42_of_match); + +static const struct acpi_device_id __maybe_unused cs42l42_acpi_match[] = { + {"10134242", 0,}, + {} +}; +MODULE_DEVICE_TABLE(acpi, cs42l42_acpi_match); + +static const struct i2c_device_id cs42l42_id[] = { + {"cs42l42", 0}, + {} +}; + +MODULE_DEVICE_TABLE(i2c, cs42l42_id); + +static struct i2c_driver cs42l42_i2c_driver = { + .driver = { + .name = "cs42l42", + .pm = &cs42l42_i2c_pm_ops, + .of_match_table = of_match_ptr(cs42l42_of_match), + .acpi_match_table = ACPI_PTR(cs42l42_acpi_match), + }, + .id_table = cs42l42_id, + .probe_new = cs42l42_i2c_probe, + .remove = cs42l42_i2c_remove, +}; + +module_i2c_driver(cs42l42_i2c_driver); + +MODULE_DESCRIPTION("ASoC CS42L42 I2C driver"); +MODULE_AUTHOR("Richard Fitzgerald <rf@opensource.cirrus.com>"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(SND_SOC_CS42L42_CORE); diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index d545a593a2516603979ddf6c731cd80e0c64d2d6..2fefbcf7bd130a9bbd201bb00bae0170cfde7c63 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -12,10 +12,9 @@ #include <linux/module.h> #include <linux/moduleparam.h> #include <linux/version.h> -#include <linux/kernel.h> +#include <linux/types.h> #include <linux/init.h> #include <linux/delay.h> -#include <linux/i2c.h> #include <linux/gpio.h> #include <linux/regmap.h> #include <linux/slab.h> @@ -37,6 +36,14 @@ #include "cs42l42.h" #include "cirrus_legacy.h" +static const char * const cs42l42_supply_names[] = { + "VA", + "VP", + "VCP", + "VD_FILT", + "VL", +}; + static const struct reg_default cs42l42_reg_defaults[] = { { CS42L42_FRZ_CTL, 0x00 }, { CS42L42_SRC_CTL, 0x10 }, @@ -164,7 +171,7 @@ static const struct reg_default cs42l42_reg_defaults[] = { { CS42L42_ASP_RX_DAI1_CH2_BIT_LSB, 0x00 }, }; -static bool cs42l42_readable_register(struct device *dev, unsigned int reg) +bool cs42l42_readable_register(struct device *dev, unsigned int reg) { switch (reg) { case CS42L42_PAGE_REGISTER: @@ -323,8 +330,9 @@ static bool cs42l42_readable_register(struct device *dev, unsigned int reg) return false; } } +EXPORT_SYMBOL_NS_GPL(cs42l42_readable_register, SND_SOC_CS42L42_CORE); -static bool cs42l42_volatile_register(struct device *dev, unsigned int reg) +bool cs42l42_volatile_register(struct device *dev, unsigned int reg) { switch (reg) { case CS42L42_DEVID_AB: @@ -355,8 +363,9 @@ static bool cs42l42_volatile_register(struct device *dev, unsigned int reg) return false; } } +EXPORT_SYMBOL_NS_GPL(cs42l42_volatile_register, SND_SOC_CS42L42_CORE); -static const struct regmap_range_cfg cs42l42_page_range = { +const struct regmap_range_cfg cs42l42_page_range = { .name = "Pages", .range_min = 0, .range_max = CS42L42_MAX_REGISTER, @@ -366,8 +375,9 @@ static const struct regmap_range_cfg cs42l42_page_range = { .window_start = 0, .window_len = 256, }; +EXPORT_SYMBOL_NS_GPL(cs42l42_page_range, SND_SOC_CS42L42_CORE); -static const struct regmap_config cs42l42_regmap = { +const struct regmap_config cs42l42_regmap = { .reg_bits = 8, .val_bits = 8, @@ -385,6 +395,7 @@ static const struct regmap_config cs42l42_regmap = { .use_single_read = true, .use_single_write = true, }; +EXPORT_SYMBOL_NS_GPL(cs42l42_regmap, SND_SOC_CS42L42_CORE); static DECLARE_TLV_DB_SCALE(adc_tlv, -9700, 100, true); static DECLARE_TLV_DB_SCALE(mixer_tlv, -6300, 100, true); @@ -395,7 +406,7 @@ static int cs42l42_slow_start_put(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); u8 val; - /* all bits of SLOW_START_EN much change together */ + /* all bits of SLOW_START_EN must change together */ switch (ucontrol->value.integer.value[0]) { case 0: val = 0; @@ -571,7 +582,7 @@ static int cs42l42_set_jack(struct snd_soc_component *component, struct snd_soc_ return 0; } -static const struct snd_soc_component_driver soc_component_dev_cs42l42 = { +const struct snd_soc_component_driver cs42l42_soc_component = { .set_jack = cs42l42_set_jack, .dapm_widgets = cs42l42_dapm_widgets, .num_dapm_widgets = ARRAY_SIZE(cs42l42_dapm_widgets), @@ -582,6 +593,7 @@ static const struct snd_soc_component_driver soc_component_dev_cs42l42 = { .idle_bias_on = 1, .endianness = 1, }; +EXPORT_SYMBOL_NS_GPL(cs42l42_soc_component, SND_SOC_CS42L42_CORE); /* Switch to SCLK. Atomic delay after the write to allow the switch to complete. */ static const struct reg_sequence cs42l42_to_sclk_seq[] = { @@ -639,18 +651,12 @@ static const struct cs42l42_pll_params pll_ratio_table[] = { { 24576000, 1, 0x03, 0x40, 0x000000, 0x03, 0x10, 12288000, 128, 1} }; -static int cs42l42_pll_config(struct snd_soc_component *component) +static int cs42l42_pll_config(struct snd_soc_component *component, unsigned int clk) { struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component); int i; - u32 clk; u32 fsync; - if (!cs42l42->sclk) - clk = cs42l42->bclk; - else - clk = cs42l42->sclk; - /* Don't reconfigure if there is an audio stream running */ if (cs42l42->stream_use) { if (pll_ratio_table[cs42l42->pll_config].sclk == clk) @@ -885,22 +891,30 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream, struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component); unsigned int channels = params_channels(params); unsigned int width = (params_width(params) / 8) - 1; + unsigned int slot_width = 0; unsigned int val = 0; + unsigned int bclk; int ret; cs42l42->srate = params_rate(params); - cs42l42->bclk = snd_soc_params_to_bclk(params); - /* I2S frame always has 2 channels even for mono audio */ - if (channels == 1) - cs42l42->bclk *= 2; + if (cs42l42->bclk_ratio) { + /* machine driver has set the BCLK/samp-rate ratio */ + bclk = cs42l42->bclk_ratio * params_rate(params); + } else if (cs42l42->sclk) { + /* machine driver has set the SCLK */ + bclk = cs42l42->sclk; + } else { + /* + * Assume 24-bit samples are in 32-bit slots, to prevent SCLK being + * more than assumed (which would result in overclocking). + */ + if (params_width(params) == 24) + slot_width = 32; - /* - * Assume 24-bit samples are in 32-bit slots, to prevent SCLK being - * more than assumed (which would result in overclocking). - */ - if (params_width(params) == 24) - cs42l42->bclk = (cs42l42->bclk / 3) * 4; + /* I2S frame always has multiple of 2 channels */ + bclk = snd_soc_tdm_params_to_bclk(params, slot_width, 0, 2); + } switch (substream->stream) { case SNDRV_PCM_STREAM_CAPTURE: @@ -940,7 +954,7 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream, break; } - ret = cs42l42_pll_config(component); + ret = cs42l42_pll_config(component, bclk); if (ret) return ret; @@ -973,6 +987,17 @@ static int cs42l42_set_sysclk(struct snd_soc_dai *dai, return -EINVAL; } +static int cs42l42_set_bclk_ratio(struct snd_soc_dai *dai, + unsigned int bclk_ratio) +{ + struct snd_soc_component *component = dai->component; + struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component); + + cs42l42->bclk_ratio = bclk_ratio; + + return 0; +} + static int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream) { struct snd_soc_component *component = dai->component; @@ -1076,10 +1101,11 @@ static const struct snd_soc_dai_ops cs42l42_ops = { .hw_params = cs42l42_pcm_hw_params, .set_fmt = cs42l42_set_dai_fmt, .set_sysclk = cs42l42_set_sysclk, + .set_bclk_ratio = cs42l42_set_bclk_ratio, .mute_stream = cs42l42_mute_stream, }; -static struct snd_soc_dai_driver cs42l42_dai = { +struct snd_soc_dai_driver cs42l42_dai = { .name = "cs42l42", .playback = { .stream_name = "Playback", @@ -1099,6 +1125,7 @@ static struct snd_soc_dai_driver cs42l42_dai = { .symmetric_sample_bits = 1, .ops = &cs42l42_ops, }; +EXPORT_SYMBOL_NS_GPL(cs42l42_dai, SND_SOC_CS42L42_CORE); static void cs42l42_manual_hs_type_detect(struct cs42l42_private *cs42l42) { @@ -1172,14 +1199,11 @@ static void cs42l42_manual_hs_type_detect(struct cs42l42_private *cs42l42) cs42l42->hs_type = CS42L42_PLUG_OMTP; hs_det_sw = CS42L42_HSDET_SW_TYPE2; break; - case CS42L42_HSDET_COMP_TYPE3: + /* Detect Type 3 and Type 4 Headsets as Headphones */ + default: cs42l42->hs_type = CS42L42_PLUG_HEADPHONE; hs_det_sw = CS42L42_HSDET_SW_TYPE3; break; - default: - cs42l42->hs_type = CS42L42_PLUG_INVALID; - hs_det_sw = CS42L42_HSDET_SW_TYPE4; - break; } } @@ -1617,10 +1641,9 @@ static irqreturn_t cs42l42_irq_thread(int irq, void *data) unsigned int current_plug_status; unsigned int current_button_status; unsigned int i; - int report = 0; mutex_lock(&cs42l42->irq_lock); - if (cs42l42->suspended) { + if (cs42l42->suspended || !cs42l42->init_done) { mutex_unlock(&cs42l42->irq_lock); return IRQ_NONE; } @@ -1711,13 +1734,15 @@ static irqreturn_t cs42l42_irq_thread(int irq, void *data) if (current_button_status & CS42L42_M_DETECT_TF_MASK) { dev_dbg(cs42l42->dev, "Button released\n"); - report = 0; + snd_soc_jack_report(cs42l42->jack, 0, + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3); } else if (current_button_status & CS42L42_M_DETECT_FT_MASK) { - report = cs42l42_handle_button_press(cs42l42); - + snd_soc_jack_report(cs42l42->jack, + cs42l42_handle_button_press(cs42l42), + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3); } - snd_soc_jack_report(cs42l42->jack, report, SND_JACK_BTN_0 | SND_JACK_BTN_1 | - SND_JACK_BTN_2 | SND_JACK_BTN_3); } } @@ -2093,7 +2118,7 @@ static const struct reg_sequence __maybe_unused cs42l42_shutdown_seq[] = { REG_SEQ0(CS42L42_PWR_CTL1, 0xFF) }; -static int __maybe_unused cs42l42_suspend(struct device *dev) +int cs42l42_suspend(struct device *dev) { struct cs42l42_private *cs42l42 = dev_get_drvdata(dev); unsigned int reg; @@ -2153,8 +2178,9 @@ static int __maybe_unused cs42l42_suspend(struct device *dev) return 0; } +EXPORT_SYMBOL_NS_GPL(cs42l42_suspend, SND_SOC_CS42L42_CORE); -static int __maybe_unused cs42l42_resume(struct device *dev) +int cs42l42_resume(struct device *dev) { struct cs42l42_private *cs42l42 = dev_get_drvdata(dev); int ret; @@ -2176,6 +2202,16 @@ static int __maybe_unused cs42l42_resume(struct device *dev) gpiod_set_value_cansleep(cs42l42->reset_gpio, 1); usleep_range(CS42L42_BOOT_TIME_US, CS42L42_BOOT_TIME_US * 2); + dev_dbg(dev, "System resume powered up\n"); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cs42l42_resume, SND_SOC_CS42L42_CORE); + +void cs42l42_resume_restore(struct device *dev) +{ + struct cs42l42_private *cs42l42 = dev_get_drvdata(dev); + regcache_cache_only(cs42l42->regmap, false); regcache_mark_dirty(cs42l42->regmap); @@ -2188,40 +2224,40 @@ static int __maybe_unused cs42l42_resume(struct device *dev) mutex_unlock(&cs42l42->irq_lock); dev_dbg(dev, "System resumed\n"); +} +EXPORT_SYMBOL_NS_GPL(cs42l42_resume_restore, SND_SOC_CS42L42_CORE); + +static int __maybe_unused cs42l42_i2c_resume(struct device *dev) +{ + int ret; + + ret = cs42l42_resume(dev); + if (ret) + return ret; + + cs42l42_resume_restore(dev); return 0; } -static int cs42l42_i2c_probe(struct i2c_client *i2c_client) +int cs42l42_common_probe(struct cs42l42_private *cs42l42, + const struct snd_soc_component_driver *component_drv, + struct snd_soc_dai_driver *dai) { - struct cs42l42_private *cs42l42; - int ret, i, devid; - unsigned int reg; - - cs42l42 = devm_kzalloc(&i2c_client->dev, sizeof(struct cs42l42_private), - GFP_KERNEL); - if (!cs42l42) - return -ENOMEM; + int ret, i; - cs42l42->dev = &i2c_client->dev; - i2c_set_clientdata(i2c_client, cs42l42); + dev_set_drvdata(cs42l42->dev, cs42l42); mutex_init(&cs42l42->irq_lock); - cs42l42->regmap = devm_regmap_init_i2c(i2c_client, &cs42l42_regmap); - if (IS_ERR(cs42l42->regmap)) { - ret = PTR_ERR(cs42l42->regmap); - dev_err(&i2c_client->dev, "regmap_init() failed: %d\n", ret); - return ret; - } - + BUILD_BUG_ON(ARRAY_SIZE(cs42l42_supply_names) != ARRAY_SIZE(cs42l42->supplies)); for (i = 0; i < ARRAY_SIZE(cs42l42->supplies); i++) cs42l42->supplies[i].supply = cs42l42_supply_names[i]; - ret = devm_regulator_bulk_get(&i2c_client->dev, + ret = devm_regulator_bulk_get(cs42l42->dev, ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies); if (ret != 0) { - dev_err(&i2c_client->dev, + dev_err(cs42l42->dev, "Failed to request supplies: %d\n", ret); return ret; } @@ -2229,13 +2265,13 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client) ret = regulator_bulk_enable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies); if (ret != 0) { - dev_err(&i2c_client->dev, + dev_err(cs42l42->dev, "Failed to enable supplies: %d\n", ret); return ret; } /* Reset the Device */ - cs42l42->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, + cs42l42->reset_gpio = devm_gpiod_get_optional(cs42l42->dev, "reset", GPIOD_OUT_LOW); if (IS_ERR(cs42l42->reset_gpio)) { ret = PTR_ERR(cs42l42->reset_gpio); @@ -2243,50 +2279,74 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client) } if (cs42l42->reset_gpio) { - dev_dbg(&i2c_client->dev, "Found reset GPIO\n"); + dev_dbg(cs42l42->dev, "Found reset GPIO\n"); gpiod_set_value_cansleep(cs42l42->reset_gpio, 1); } usleep_range(CS42L42_BOOT_TIME_US, CS42L42_BOOT_TIME_US * 2); /* Request IRQ if one was specified */ - if (i2c_client->irq) { - ret = request_threaded_irq(i2c_client->irq, + if (cs42l42->irq) { + ret = request_threaded_irq(cs42l42->irq, NULL, cs42l42_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, "cs42l42", cs42l42); - if (ret == -EPROBE_DEFER) { - goto err_disable_noirq; - } else if (ret != 0) { - dev_err(&i2c_client->dev, - "Failed to request IRQ: %d\n", ret); + if (ret) { + dev_err_probe(cs42l42->dev, ret, + "Failed to request IRQ\n"); goto err_disable_noirq; } } + /* Register codec now so it can EPROBE_DEFER */ + ret = devm_snd_soc_register_component(cs42l42->dev, component_drv, dai, 1); + if (ret < 0) + goto err; + + return 0; + +err: + if (cs42l42->irq) + free_irq(cs42l42->irq, cs42l42); + +err_disable_noirq: + gpiod_set_value_cansleep(cs42l42->reset_gpio, 0); +err_disable_noreset: + regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies); + + return ret; +} +EXPORT_SYMBOL_NS_GPL(cs42l42_common_probe, SND_SOC_CS42L42_CORE); + +int cs42l42_init(struct cs42l42_private *cs42l42) +{ + unsigned int reg; + int devid, ret; + /* initialize codec */ devid = cirrus_read_device_id(cs42l42->regmap, CS42L42_DEVID_AB); if (devid < 0) { ret = devid; - dev_err(&i2c_client->dev, "Failed to read device ID: %d\n", ret); + dev_err(cs42l42->dev, "Failed to read device ID: %d\n", ret); goto err_disable; } - if (devid != CS42L42_CHIP_ID) { + if (devid != cs42l42->devid) { ret = -ENODEV; - dev_err(&i2c_client->dev, - "CS42L42 Device ID (%X). Expected %X\n", - devid, CS42L42_CHIP_ID); + dev_err(cs42l42->dev, + "CS42L%x Device ID (%X). Expected %X\n", + cs42l42->devid & 0xff, devid, cs42l42->devid); goto err_disable; } ret = regmap_read(cs42l42->regmap, CS42L42_REVID, ®); if (ret < 0) { - dev_err(&i2c_client->dev, "Get Revision ID failed\n"); + dev_err(cs42l42->dev, "Get Revision ID failed\n"); goto err_shutdown; } - dev_info(&i2c_client->dev, - "Cirrus Logic CS42L42, Revision: %02X\n", reg & 0xFF); + dev_info(cs42l42->dev, + "Cirrus Logic CS42L%x, Revision: %02X\n", + cs42l42->devid & 0xff, reg & 0xFF); /* Power up the codec */ regmap_update_bits(cs42l42->regmap, CS42L42_PWR_CTL1, @@ -2305,22 +2365,22 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client) (1 << CS42L42_ADC_PDN_SHIFT) | (0 << CS42L42_PDN_ALL_SHIFT)); - ret = cs42l42_handle_device_data(&i2c_client->dev, cs42l42); + ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42); if (ret != 0) goto err_shutdown; /* Setup headset detection */ cs42l42_setup_hs_type_detect(cs42l42); + /* + * Set init_done before unmasking interrupts so any triggered + * immediately will be handled. + */ + cs42l42->init_done = true; + /* Mask/Unmask Interrupts */ cs42l42_set_interrupt_masks(cs42l42); - /* Register codec for machine driver */ - ret = devm_snd_soc_register_component(&i2c_client->dev, - &soc_component_dev_cs42l42, &cs42l42_dai, 1); - if (ret < 0) - goto err_shutdown; - return 0; err_shutdown: @@ -2329,78 +2389,35 @@ err_shutdown: regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff); err_disable: - if (i2c_client->irq) - free_irq(i2c_client->irq, cs42l42); + if (cs42l42->irq) + free_irq(cs42l42->irq, cs42l42); -err_disable_noirq: gpiod_set_value_cansleep(cs42l42->reset_gpio, 0); -err_disable_noreset: regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies); return ret; } +EXPORT_SYMBOL_NS_GPL(cs42l42_init, SND_SOC_CS42L42_CORE); -static int cs42l42_i2c_remove(struct i2c_client *i2c_client) +void cs42l42_common_remove(struct cs42l42_private *cs42l42) { - struct cs42l42_private *cs42l42 = i2c_get_clientdata(i2c_client); - - if (i2c_client->irq) - free_irq(i2c_client->irq, cs42l42); + if (cs42l42->irq) + free_irq(cs42l42->irq, cs42l42); /* * The driver might not have control of reset and power supplies, * so ensure that the chip internals are powered down. */ - regmap_write(cs42l42->regmap, CS42L42_CODEC_INT_MASK, 0xff); - regmap_write(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK, 0xff); - regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff); + if (cs42l42->init_done) { + regmap_write(cs42l42->regmap, CS42L42_CODEC_INT_MASK, 0xff); + regmap_write(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK, 0xff); + regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff); + } gpiod_set_value_cansleep(cs42l42->reset_gpio, 0); regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies); - - return 0; } - -static const struct dev_pm_ops cs42l42_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l42_resume) -}; - -#ifdef CONFIG_OF -static const struct of_device_id cs42l42_of_match[] = { - { .compatible = "cirrus,cs42l42", }, - {} -}; -MODULE_DEVICE_TABLE(of, cs42l42_of_match); -#endif - -#ifdef CONFIG_ACPI -static const struct acpi_device_id cs42l42_acpi_match[] = { - {"10134242", 0,}, - {} -}; -MODULE_DEVICE_TABLE(acpi, cs42l42_acpi_match); -#endif - -static const struct i2c_device_id cs42l42_id[] = { - {"cs42l42", 0}, - {} -}; - -MODULE_DEVICE_TABLE(i2c, cs42l42_id); - -static struct i2c_driver cs42l42_i2c_driver = { - .driver = { - .name = "cs42l42", - .pm = &cs42l42_pm_ops, - .of_match_table = of_match_ptr(cs42l42_of_match), - .acpi_match_table = ACPI_PTR(cs42l42_acpi_match), - }, - .id_table = cs42l42_id, - .probe_new = cs42l42_i2c_probe, - .remove = cs42l42_i2c_remove, -}; - -module_i2c_driver(cs42l42_i2c_driver); +EXPORT_SYMBOL_NS_GPL(cs42l42_common_remove, SND_SOC_CS42L42_CORE); MODULE_DESCRIPTION("ASoC CS42L42 driver"); MODULE_AUTHOR("James Schulman, Cirrus Logic Inc, <james.schulman@cirrus.com>"); diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h index 5f50970375d4f4a84347c308b4c662bd66605c40..a721366641127b33b5094667165d9efd2e27c91a 100644 --- a/sound/soc/codecs/cs42l42.h +++ b/sound/soc/codecs/cs42l42.h @@ -12,17 +12,16 @@ #ifndef __CS42L42_H__ #define __CS42L42_H__ +#include <dt-bindings/sound/cs42l42.h> +#include <linux/device.h> +#include <linux/gpio.h> #include <linux/mutex.h> +#include <linux/regmap.h> +#include <linux/regulator/consumer.h> #include <sound/jack.h> #include <sound/cs42l42.h> - -static const char *const cs42l42_supply_names[CS42L42_NUM_SUPPLIES] = { - "VA", - "VP", - "VCP", - "VD_FILT", - "VL", -}; +#include <sound/soc-component.h> +#include <sound/soc-dai.h> struct cs42l42_private { struct regmap *regmap; @@ -32,9 +31,11 @@ struct cs42l42_private { struct completion pdn_done; struct snd_soc_jack *jack; struct mutex irq_lock; + int devid; + int irq; int pll_config; - int bclk; u32 sclk; + u32 bclk_ratio; u32 srate; u8 plug_state; u8 hs_type; @@ -50,6 +51,24 @@ struct cs42l42_private { u8 stream_use; bool hp_adc_up_pending; bool suspended; + bool init_done; }; +extern const struct regmap_range_cfg cs42l42_page_range; +extern const struct regmap_config cs42l42_regmap; +extern const struct snd_soc_component_driver cs42l42_soc_component; +extern struct snd_soc_dai_driver cs42l42_dai; + +bool cs42l42_readable_register(struct device *dev, unsigned int reg); +bool cs42l42_volatile_register(struct device *dev, unsigned int reg); + +int cs42l42_suspend(struct device *dev); +int cs42l42_resume(struct device *dev); +void cs42l42_resume_restore(struct device *dev); +int cs42l42_common_probe(struct cs42l42_private *cs42l42, + const struct snd_soc_component_driver *component_drv, + struct snd_soc_dai_driver *dai); +int cs42l42_init(struct cs42l42_private *cs42l42); +void cs42l42_common_remove(struct cs42l42_private *cs42l42); + #endif /* __CS42L42_H__ */ diff --git a/sound/soc/codecs/cs42l51-i2c.c b/sound/soc/codecs/cs42l51-i2c.c index 3613fb12d623bfd07f08491332ca7de6b981a420..85238339fbcab574cd2ca93a2dd0968a0abac073 100644 --- a/sound/soc/codecs/cs42l51-i2c.c +++ b/sound/soc/codecs/cs42l51-i2c.c @@ -28,11 +28,9 @@ static int cs42l51_i2c_probe(struct i2c_client *i2c) return cs42l51_probe(&i2c->dev, devm_regmap_init_i2c(i2c, &config)); } -static int cs42l51_i2c_remove(struct i2c_client *i2c) +static void cs42l51_i2c_remove(struct i2c_client *i2c) { cs42l51_remove(&i2c->dev); - - return 0; } static const struct dev_pm_ops cs42l51_pm_ops = { diff --git a/sound/soc/codecs/cs42l56.c b/sound/soc/codecs/cs42l56.c index 03e2540a0ba1264cbce4244139a1649094acacd7..26066682c983e6d34d65c70ef1af5df3e98633d2 100644 --- a/sound/soc/codecs/cs42l56.c +++ b/sound/soc/codecs/cs42l56.c @@ -1320,13 +1320,12 @@ err_enable: return ret; } -static int cs42l56_i2c_remove(struct i2c_client *client) +static void cs42l56_i2c_remove(struct i2c_client *client) { struct cs42l56_private *cs42l56 = i2c_get_clientdata(client); regulator_bulk_disable(ARRAY_SIZE(cs42l56->supplies), cs42l56->supplies); - return 0; } static const struct of_device_id cs42l56_of_match[] = { diff --git a/sound/soc/codecs/cs42l83-i2c.c b/sound/soc/codecs/cs42l83-i2c.c new file mode 100644 index 0000000000000000000000000000000000000000..f90d43996a51662619e9bde981ce443558a06dec --- /dev/null +++ b/sound/soc/codecs/cs42l83-i2c.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * cs42l83-i2c.c -- CS42L83 ALSA SoC audio driver for I2C + * + * Based on cs42l42-i2c.c: + * Copyright 2016, 2022 Cirrus Logic, Inc. + */ + +#include <linux/i2c.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include "cs42l42.h" + +static const struct reg_default cs42l83_reg_defaults[] = { + { CS42L42_FRZ_CTL, 0x00 }, + { CS42L42_SRC_CTL, 0x10 }, + { CS42L42_MCLK_CTL, 0x00 }, /* <- only deviation from CS42L42 */ + { CS42L42_SFTRAMP_RATE, 0xA4 }, + { CS42L42_SLOW_START_ENABLE, 0x70 }, + { CS42L42_I2C_DEBOUNCE, 0x88 }, + { CS42L42_I2C_STRETCH, 0x03 }, + { CS42L42_I2C_TIMEOUT, 0xB7 }, + { CS42L42_PWR_CTL1, 0xFF }, + { CS42L42_PWR_CTL2, 0x84 }, + { CS42L42_PWR_CTL3, 0x20 }, + { CS42L42_RSENSE_CTL1, 0x40 }, + { CS42L42_RSENSE_CTL2, 0x00 }, + { CS42L42_OSC_SWITCH, 0x00 }, + { CS42L42_RSENSE_CTL3, 0x1B }, + { CS42L42_TSENSE_CTL, 0x1B }, + { CS42L42_TSRS_INT_DISABLE, 0x00 }, + { CS42L42_HSDET_CTL1, 0x77 }, + { CS42L42_HSDET_CTL2, 0x00 }, + { CS42L42_HS_SWITCH_CTL, 0xF3 }, + { CS42L42_HS_CLAMP_DISABLE, 0x00 }, + { CS42L42_MCLK_SRC_SEL, 0x00 }, + { CS42L42_SPDIF_CLK_CFG, 0x00 }, + { CS42L42_FSYNC_PW_LOWER, 0x00 }, + { CS42L42_FSYNC_PW_UPPER, 0x00 }, + { CS42L42_FSYNC_P_LOWER, 0xF9 }, + { CS42L42_FSYNC_P_UPPER, 0x00 }, + { CS42L42_ASP_CLK_CFG, 0x00 }, + { CS42L42_ASP_FRM_CFG, 0x10 }, + { CS42L42_FS_RATE_EN, 0x00 }, + { CS42L42_IN_ASRC_CLK, 0x00 }, + { CS42L42_OUT_ASRC_CLK, 0x00 }, + { CS42L42_PLL_DIV_CFG1, 0x00 }, + { CS42L42_ADC_OVFL_INT_MASK, 0x01 }, + { CS42L42_MIXER_INT_MASK, 0x0F }, + { CS42L42_SRC_INT_MASK, 0x0F }, + { CS42L42_ASP_RX_INT_MASK, 0x1F }, + { CS42L42_ASP_TX_INT_MASK, 0x0F }, + { CS42L42_CODEC_INT_MASK, 0x03 }, + { CS42L42_SRCPL_INT_MASK, 0x7F }, + { CS42L42_VPMON_INT_MASK, 0x01 }, + { CS42L42_PLL_LOCK_INT_MASK, 0x01 }, + { CS42L42_TSRS_PLUG_INT_MASK, 0x0F }, + { CS42L42_PLL_CTL1, 0x00 }, + { CS42L42_PLL_DIV_FRAC0, 0x00 }, + { CS42L42_PLL_DIV_FRAC1, 0x00 }, + { CS42L42_PLL_DIV_FRAC2, 0x00 }, + { CS42L42_PLL_DIV_INT, 0x40 }, + { CS42L42_PLL_CTL3, 0x10 }, + { CS42L42_PLL_CAL_RATIO, 0x80 }, + { CS42L42_PLL_CTL4, 0x03 }, + { CS42L42_LOAD_DET_EN, 0x00 }, + { CS42L42_HSBIAS_SC_AUTOCTL, 0x03 }, + { CS42L42_WAKE_CTL, 0xC0 }, + { CS42L42_ADC_DISABLE_MUTE, 0x00 }, + { CS42L42_TIPSENSE_CTL, 0x02 }, + { CS42L42_MISC_DET_CTL, 0x03 }, + { CS42L42_MIC_DET_CTL1, 0x1F }, + { CS42L42_MIC_DET_CTL2, 0x2F }, + { CS42L42_DET_INT1_MASK, 0xE0 }, + { CS42L42_DET_INT2_MASK, 0xFF }, + { CS42L42_HS_BIAS_CTL, 0xC2 }, + { CS42L42_ADC_CTL, 0x00 }, + { CS42L42_ADC_VOLUME, 0x00 }, + { CS42L42_ADC_WNF_HPF_CTL, 0x71 }, + { CS42L42_DAC_CTL1, 0x00 }, + { CS42L42_DAC_CTL2, 0x02 }, + { CS42L42_HP_CTL, 0x0D }, + { CS42L42_CLASSH_CTL, 0x07 }, + { CS42L42_MIXER_CHA_VOL, 0x3F }, + { CS42L42_MIXER_ADC_VOL, 0x3F }, + { CS42L42_MIXER_CHB_VOL, 0x3F }, + { CS42L42_EQ_COEF_IN0, 0x00 }, + { CS42L42_EQ_COEF_IN1, 0x00 }, + { CS42L42_EQ_COEF_IN2, 0x00 }, + { CS42L42_EQ_COEF_IN3, 0x00 }, + { CS42L42_EQ_COEF_RW, 0x00 }, + { CS42L42_EQ_COEF_OUT0, 0x00 }, + { CS42L42_EQ_COEF_OUT1, 0x00 }, + { CS42L42_EQ_COEF_OUT2, 0x00 }, + { CS42L42_EQ_COEF_OUT3, 0x00 }, + { CS42L42_EQ_INIT_STAT, 0x00 }, + { CS42L42_EQ_START_FILT, 0x00 }, + { CS42L42_EQ_MUTE_CTL, 0x00 }, + { CS42L42_SP_RX_CH_SEL, 0x04 }, + { CS42L42_SP_RX_ISOC_CTL, 0x04 }, + { CS42L42_SP_RX_FS, 0x8C }, + { CS42l42_SPDIF_CH_SEL, 0x0E }, + { CS42L42_SP_TX_ISOC_CTL, 0x04 }, + { CS42L42_SP_TX_FS, 0xCC }, + { CS42L42_SPDIF_SW_CTL1, 0x3F }, + { CS42L42_SRC_SDIN_FS, 0x40 }, + { CS42L42_SRC_SDOUT_FS, 0x40 }, + { CS42L42_SPDIF_CTL1, 0x01 }, + { CS42L42_SPDIF_CTL2, 0x00 }, + { CS42L42_SPDIF_CTL3, 0x00 }, + { CS42L42_SPDIF_CTL4, 0x42 }, + { CS42L42_ASP_TX_SZ_EN, 0x00 }, + { CS42L42_ASP_TX_CH_EN, 0x00 }, + { CS42L42_ASP_TX_CH_AP_RES, 0x0F }, + { CS42L42_ASP_TX_CH1_BIT_MSB, 0x00 }, + { CS42L42_ASP_TX_CH1_BIT_LSB, 0x00 }, + { CS42L42_ASP_TX_HIZ_DLY_CFG, 0x00 }, + { CS42L42_ASP_TX_CH2_BIT_MSB, 0x00 }, + { CS42L42_ASP_TX_CH2_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_EN, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH1_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI0_CH1_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH1_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH2_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI0_CH2_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH2_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH3_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI0_CH3_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH3_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH4_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI0_CH4_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI0_CH4_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI1_CH1_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI1_CH1_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI1_CH1_BIT_LSB, 0x00 }, + { CS42L42_ASP_RX_DAI1_CH2_AP_RES, 0x03 }, + { CS42L42_ASP_RX_DAI1_CH2_BIT_MSB, 0x00 }, + { CS42L42_ASP_RX_DAI1_CH2_BIT_LSB, 0x00 }, +}; + +/* + * This is all the same as for CS42L42 but we + * replace the on-reset register defaults. + */ +const struct regmap_config cs42l83_regmap = { + .reg_bits = 8, + .val_bits = 8, + + .readable_reg = cs42l42_readable_register, + .volatile_reg = cs42l42_volatile_register, + + .ranges = &cs42l42_page_range, + .num_ranges = 1, + + .max_register = CS42L42_MAX_REGISTER, + .reg_defaults = cs42l83_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(cs42l83_reg_defaults), + .cache_type = REGCACHE_RBTREE, + + .use_single_read = true, + .use_single_write = true, +}; + +static int cs42l83_i2c_probe(struct i2c_client *i2c_client) +{ + struct device *dev = &i2c_client->dev; + struct cs42l42_private *cs42l83; + struct regmap *regmap; + int ret; + + cs42l83 = devm_kzalloc(dev, sizeof(*cs42l83), GFP_KERNEL); + if (!cs42l83) + return -ENOMEM; + + regmap = devm_regmap_init_i2c(i2c_client, &cs42l83_regmap); + if (IS_ERR(regmap)) + return dev_err_probe(&i2c_client->dev, PTR_ERR(regmap), + "regmap_init() failed\n"); + + cs42l83->devid = CS42L83_CHIP_ID; + cs42l83->dev = dev; + cs42l83->regmap = regmap; + cs42l83->irq = i2c_client->irq; + + ret = cs42l42_common_probe(cs42l83, &cs42l42_soc_component, &cs42l42_dai); + if (ret) + return ret; + + return cs42l42_init(cs42l83); +} + +static void cs42l83_i2c_remove(struct i2c_client *i2c_client) +{ + struct cs42l42_private *cs42l83 = dev_get_drvdata(&i2c_client->dev); + + cs42l42_common_remove(cs42l83); +} + +static int __maybe_unused cs42l83_i2c_resume(struct device *dev) +{ + int ret; + + ret = cs42l42_resume(dev); + if (ret) + return ret; + + cs42l42_resume_restore(dev); + + return 0; +} + +static const struct dev_pm_ops cs42l83_i2c_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l83_i2c_resume) +}; + +static const struct of_device_id __maybe_unused cs42l83_of_match[] = { + { .compatible = "cirrus,cs42l83", }, + {} +}; +MODULE_DEVICE_TABLE(of, cs42l83_of_match); + +static struct i2c_driver cs42l83_i2c_driver = { + .driver = { + .name = "cs42l83", + .pm = &cs42l83_i2c_pm_ops, + .of_match_table = of_match_ptr(cs42l83_of_match), + }, + .probe_new = cs42l83_i2c_probe, + .remove = cs42l83_i2c_remove, +}; + +module_i2c_driver(cs42l83_i2c_driver); + +MODULE_DESCRIPTION("ASoC CS42L83 I2C driver"); +MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(SND_SOC_CS42L42_CORE); diff --git a/sound/soc/codecs/cs42xx8-i2c.c b/sound/soc/codecs/cs42xx8-i2c.c index cb06a06d48b024ac316ed929747eac93f7867ee0..bd80e9fc907fd48fb67a6d88d7ea3ff3befaf2de 100644 --- a/sound/soc/codecs/cs42xx8-i2c.c +++ b/sound/soc/codecs/cs42xx8-i2c.c @@ -30,11 +30,9 @@ static int cs42xx8_i2c_probe(struct i2c_client *i2c) return 0; } -static int cs42xx8_i2c_remove(struct i2c_client *i2c) +static void cs42xx8_i2c_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); - - return 0; } static struct i2c_device_id cs42xx8_i2c_id[] = { diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c index ca4d47cc9c915013488c85e81a68535e284710c0..db39abb2a31b5969739d035af11916080e4f10bd 100644 --- a/sound/soc/codecs/cs43130.c +++ b/sound/soc/codecs/cs43130.c @@ -1666,10 +1666,9 @@ static int cs43130_show_dc(struct device *dev, char *buf, u8 ch) struct cs43130_private *cs43130 = i2c_get_clientdata(client); if (!cs43130->hpload_done) - return scnprintf(buf, PAGE_SIZE, "NO_HPLOAD\n"); + return sysfs_emit(buf, "NO_HPLOAD\n"); else - return scnprintf(buf, PAGE_SIZE, "%u\n", - cs43130->hpload_dc[ch]); + return sysfs_emit(buf, "%u\n", cs43130->hpload_dc[ch]); } static ssize_t hpload_dc_l_show(struct device *dev, @@ -1705,8 +1704,8 @@ static int cs43130_show_ac(struct device *dev, char *buf, u8 ch) if (cs43130->hpload_done && cs43130->ac_meas) { for (i = 0; i < ARRAY_SIZE(cs43130_ac_freq); i++) { - tmp = scnprintf(buf + j, PAGE_SIZE - j, "%u\n", - cs43130->hpload_ac[i][ch]); + tmp = sysfs_emit_at(buf, j, "%u\n", + cs43130->hpload_ac[i][ch]); if (!tmp) break; @@ -1715,7 +1714,7 @@ static int cs43130_show_ac(struct device *dev, char *buf, u8 ch) return j; } else { - return scnprintf(buf, PAGE_SIZE, "NO_HPLOAD\n"); + return sysfs_emit(buf, "NO_HPLOAD\n"); } } @@ -2583,7 +2582,7 @@ err_supplies: return ret; } -static int cs43130_i2c_remove(struct i2c_client *client) +static void cs43130_i2c_remove(struct i2c_client *client) { struct cs43130_private *cs43130 = i2c_get_clientdata(client); @@ -2610,8 +2609,6 @@ static int cs43130_i2c_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); regulator_bulk_disable(CS43130_NUM_SUPPLIES, cs43130->supplies); - - return 0; } static int __maybe_unused cs43130_runtime_suspend(struct device *dev) diff --git a/sound/soc/codecs/cs4349.c b/sound/soc/codecs/cs4349.c index f7c5c2fd430462cae969c34afa0c29f410ede1b4..ba94ffd0a7e4e8125d360906de6dd1e61aab82e2 100644 --- a/sound/soc/codecs/cs4349.c +++ b/sound/soc/codecs/cs4349.c @@ -305,14 +305,12 @@ static int cs4349_i2c_probe(struct i2c_client *client) &cs4349_dai, 1); } -static int cs4349_i2c_remove(struct i2c_client *client) +static void cs4349_i2c_remove(struct i2c_client *client) { struct cs4349_private *cs4349 = i2c_get_clientdata(client); /* Hold down reset */ gpiod_set_value_cansleep(cs4349->reset_gpio, 0); - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/cs53l30.c b/sound/soc/codecs/cs53l30.c index 8796d8e84b7a26fe1da146a86e9c67ebf786c748..69db0013d2436e92682d308bb7d353c07c03a036 100644 --- a/sound/soc/codecs/cs53l30.c +++ b/sound/soc/codecs/cs53l30.c @@ -1043,7 +1043,7 @@ error_supplies: return ret; } -static int cs53l30_i2c_remove(struct i2c_client *client) +static void cs53l30_i2c_remove(struct i2c_client *client) { struct cs53l30_private *cs53l30 = i2c_get_clientdata(client); @@ -1052,8 +1052,6 @@ static int cs53l30_i2c_remove(struct i2c_client *client) regulator_bulk_disable(ARRAY_SIZE(cs53l30->supplies), cs53l30->supplies); - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/cx2072x.c b/sound/soc/codecs/cx2072x.c index b6667e8a6099946bf81bb0a6e7f121bd6bbd9fde..5deceaa89282a90ac496ecc9eb03bef791bfc6ea 100644 --- a/sound/soc/codecs/cx2072x.c +++ b/sound/soc/codecs/cx2072x.c @@ -1673,10 +1673,9 @@ static int cx2072x_i2c_probe(struct i2c_client *i2c) return 0; } -static int cx2072x_i2c_remove(struct i2c_client *i2c) +static void cx2072x_i2c_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); - return 0; } static const struct i2c_device_id cx2072x_i2c_id[] = { diff --git a/sound/soc/codecs/da7219.c b/sound/soc/codecs/da7219.c index 50ecf30e6136ae7a1c6030f8c9754fefcc1f9a85..4746c8700451281d6bdd3f570d3c892acc5b3055 100644 --- a/sound/soc/codecs/da7219.c +++ b/sound/soc/codecs/da7219.c @@ -2196,6 +2196,7 @@ static int da7219_register_dai_clks(struct snd_soc_component *component) dai_clk_lookup = clkdev_hw_create(dai_clk_hw, init.name, "%s", dev_name(dev)); if (!dai_clk_lookup) { + clk_hw_unregister(dai_clk_hw); ret = -ENOMEM; goto err; } else { @@ -2217,12 +2218,12 @@ static int da7219_register_dai_clks(struct snd_soc_component *component) return 0; err: - do { + while (--i >= 0) { if (da7219->dai_clks_lookup[i]) clkdev_drop(da7219->dai_clks_lookup[i]); clk_hw_unregister(&da7219->dai_clks_hw[i]); - } while (i-- > 0); + } if (np) kfree(da7219->clk_hw_data); diff --git a/sound/soc/codecs/es8316.c b/sound/soc/codecs/es8316.c index de7185f73e1e726dc73df15b096b21623d183d9e..056c3082fe02caedb944a70f0f74d564cc760dbe 100644 --- a/sound/soc/codecs/es8316.c +++ b/sound/soc/codecs/es8316.c @@ -767,9 +767,31 @@ static void es8316_remove(struct snd_soc_component *component) clk_disable_unprepare(es8316->mclk); } +static int es8316_resume(struct snd_soc_component *component) +{ + struct es8316_priv *es8316 = snd_soc_component_get_drvdata(component); + + regcache_cache_only(es8316->regmap, false); + regcache_sync(es8316->regmap); + + return 0; +} + +static int es8316_suspend(struct snd_soc_component *component) +{ + struct es8316_priv *es8316 = snd_soc_component_get_drvdata(component); + + regcache_cache_only(es8316->regmap, true); + regcache_mark_dirty(es8316->regmap); + + return 0; +} + static const struct snd_soc_component_driver soc_component_dev_es8316 = { .probe = es8316_probe, .remove = es8316_remove, + .resume = es8316_resume, + .suspend = es8316_suspend, .set_jack = es8316_set_jack, .controls = es8316_snd_controls, .num_controls = ARRAY_SIZE(es8316_snd_controls), @@ -793,6 +815,8 @@ static const struct regmap_access_table es8316_volatile_table = { static const struct regmap_config es8316_regmap = { .reg_bits = 8, .val_bits = 8, + .use_single_read = true, + .use_single_write = true, .max_register = 0x53, .volatile_table = &es8316_volatile_table, .cache_type = REGCACHE_RBTREE, diff --git a/sound/soc/codecs/es8326.c b/sound/soc/codecs/es8326.c new file mode 100755 index 0000000000000000000000000000000000000000..87c1cc16592bb8a4ea57c2c179aa4e46305bdef8 --- /dev/null +++ b/sound/soc/codecs/es8326.c @@ -0,0 +1,905 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// es8326.c -- es8326 ALSA SoC audio driver +// Copyright Everest Semiconductor Co., Ltd +// +// Authors: David Yang <yangxiaohua@everest-semi.com> +// + +#include <linux/clk.h> +#include <linux/i2c.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/module.h> +#include <sound/jack.h> +#include <sound/pcm_params.h> +#include <sound/soc.h> +#include <sound/soc-dapm.h> +#include <sound/tlv.h> +#include "es8326.h" + +struct es8326_priv { + struct clk *mclk; + struct i2c_client *i2c; + struct regmap *regmap; + struct snd_soc_component *component; + struct delayed_work jack_detect_work; + struct delayed_work button_press_work; + struct snd_soc_jack *jack; + int irq; + /* The lock protects the situation that an irq is generated + * while enabling or disabling or during an irq. + */ + struct mutex lock; + u8 mic1_src; + u8 mic2_src; + u8 jack_pol; + u8 interrupt_src; + u8 interrupt_clk; + bool jd_inverted; + unsigned int sysclk; +}; + +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(dac_vol_tlv, -9550, 50, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_vol_tlv, -9550, 50, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_analog_pga_tlv, 0, 300, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_pga_tlv, 0, 600, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(softramp_rate, 0, 100, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(drc_target_tlv, -3200, 200, 0); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(drc_recovery_tlv, -125, 250, 0); + +static const char *const winsize[] = { + "0.25db/2 LRCK", + "0.25db/4 LRCK", + "0.25db/8 LRCK", + "0.25db/16 LRCK", + "0.25db/32 LRCK", + "0.25db/64 LRCK", + "0.25db/128 LRCK", + "0.25db/256 LRCK", + "0.25db/512 LRCK", + "0.25db/1024 LRCK", + "0.25db/2048 LRCK", + "0.25db/4096 LRCK", + "0.25db/8192 LRCK", + "0.25db/16384 LRCK", + "0.25db/32768 LRCK", + "0.25db/65536 LRCK", +}; + +static const char *const dacpol_txt[] = { + "Normal", "R Invert", "L Invert", "L + R Invert" }; + +static const struct soc_enum dacpol = + SOC_ENUM_SINGLE(ES8326_DAC_DSM, 4, 4, dacpol_txt); +static const struct soc_enum alc_winsize = + SOC_ENUM_SINGLE(ES8326_ADC_RAMPRATE, 4, 16, winsize); +static const struct soc_enum drc_winsize = + SOC_ENUM_SINGLE(ES8326_DRC_WINSIZE, 4, 16, winsize); + +static const struct snd_kcontrol_new es8326_snd_controls[] = { + SOC_SINGLE_TLV("DAC Playback Volume", ES8326_DAC_VOL, 0, 0xbf, 0, dac_vol_tlv), + SOC_ENUM("Playback Polarity", dacpol), + SOC_SINGLE_TLV("DAC Ramp Rate", ES8326_DAC_RAMPRATE, 0, 0x0f, 0, softramp_rate), + SOC_SINGLE_TLV("DRC Recovery Level", ES8326_DRC_RECOVERY, 0, 4, 0, drc_recovery_tlv), + SOC_ENUM("DRC Winsize", drc_winsize), + SOC_SINGLE_TLV("DRC Target Level", ES8326_DRC_WINSIZE, 0, 0x0f, 0, drc_target_tlv), + + SOC_DOUBLE_R_TLV("ADC Capture Volume", ES8326_ADC1_VOL, ES8326_ADC2_VOL, 0, 0xff, 0, + adc_vol_tlv), + SOC_DOUBLE_TLV("ADC PGA Volume", ES8326_ADC_SCALE, 4, 0, 5, 0, adc_pga_tlv), + SOC_SINGLE_TLV("ADC PGA Gain Volume", ES8326_PGAGAIN, 0, 10, 0, adc_analog_pga_tlv), + SOC_SINGLE_TLV("ADC Ramp Rate", ES8326_ADC_RAMPRATE, 0, 0x0f, 0, softramp_rate), + SOC_SINGLE("ALC Capture Switch", ES8326_ALC_RECOVERY, 3, 1, 0), + SOC_SINGLE_TLV("ALC Capture Recovery Level", ES8326_ALC_LEVEL, + 0, 4, 0, drc_recovery_tlv), + SOC_ENUM("ALC Capture Winsize", alc_winsize), + SOC_SINGLE_TLV("ALC Capture Target Level", ES8326_ALC_LEVEL, + 0, 0x0f, 0, drc_target_tlv), + +}; + +static const struct snd_soc_dapm_widget es8326_dapm_widgets[] = { + SND_SOC_DAPM_INPUT("MIC1"), + SND_SOC_DAPM_INPUT("MIC2"), + SND_SOC_DAPM_INPUT("MIC3"), + SND_SOC_DAPM_INPUT("MIC4"), + + SND_SOC_DAPM_ADC("ADC L", NULL, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_ADC("ADC R", NULL, SND_SOC_NOPM, 0, 0), + + /* Digital Interface */ + SND_SOC_DAPM_AIF_OUT("I2S OUT", "I2S1 Capture", 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("I2S IN", "I2S1 Playback", 0, SND_SOC_NOPM, 0, 0), + + /* ADC Digital Mute */ + SND_SOC_DAPM_PGA("ADC L1", ES8326_ADC_MUTE, 0, 1, NULL, 0), + SND_SOC_DAPM_PGA("ADC R1", ES8326_ADC_MUTE, 1, 1, NULL, 0), + SND_SOC_DAPM_PGA("ADC L2", ES8326_ADC_MUTE, 2, 1, NULL, 0), + SND_SOC_DAPM_PGA("ADC R2", ES8326_ADC_MUTE, 3, 1, NULL, 0), + + /* Analog Power Supply*/ + SND_SOC_DAPM_DAC("Right DAC", NULL, ES8326_ANA_PDN, 0, 1), + SND_SOC_DAPM_DAC("Left DAC", NULL, ES8326_ANA_PDN, 1, 1), + SND_SOC_DAPM_SUPPLY("Analog Power", ES8326_ANA_PDN, 7, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("IBias Power", ES8326_ANA_PDN, 6, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("ADC Vref", ES8326_ANA_PDN, 5, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("DAC Vref", ES8326_ANA_PDN, 4, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("Vref Power", ES8326_ANA_PDN, 3, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("MICBIAS1", ES8326_ANA_MICBIAS, 2, 0, NULL, 0), + SND_SOC_DAPM_SUPPLY("MICBIAS2", ES8326_ANA_MICBIAS, 3, 0, NULL, 0), + + SND_SOC_DAPM_PGA("LHPMIX", ES8326_DAC2HPMIX, 7, 0, NULL, 0), + SND_SOC_DAPM_PGA("RHPMIX", ES8326_DAC2HPMIX, 3, 0, NULL, 0), + + /* Headphone Charge Pump and Output */ + SND_SOC_DAPM_SUPPLY("HPOR Cal", ES8326_HP_CAL, 7, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("HPOL Cal", ES8326_HP_CAL, 3, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("Headphone Charge Pump", ES8326_HP_DRIVER, + 3, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("Headphone Driver Bias", ES8326_HP_DRIVER, + 2, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("Headphone LDO", ES8326_HP_DRIVER, + 1, 1, NULL, 0), + SND_SOC_DAPM_SUPPLY("Headphone Reference", ES8326_HP_DRIVER, + 0, 1, NULL, 0), + SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOR Supply", ES8326_HP_CAL, + ES8326_HPOR_SHIFT, 7, 7, 0), + SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOL Supply", ES8326_HP_CAL, + 0, 7, 7, 0), + + SND_SOC_DAPM_OUTPUT("HPOL"), + SND_SOC_DAPM_OUTPUT("HPOR"), +}; + +static const struct snd_soc_dapm_route es8326_dapm_routes[] = { + {"ADC L1", NULL, "MIC1"}, + {"ADC R1", NULL, "MIC2"}, + {"ADC L2", NULL, "MIC3"}, + {"ADC R2", NULL, "MIC4"}, + + {"ADC L", NULL, "ADC L1"}, + {"ADC R", NULL, "ADC R1"}, + {"ADC L", NULL, "ADC L2"}, + {"ADC R", NULL, "ADC R2"}, + + {"I2S OUT", NULL, "ADC L"}, + {"I2S OUT", NULL, "ADC R"}, + + {"I2S OUT", NULL, "Analog Power"}, + {"I2S OUT", NULL, "ADC Vref"}, + {"I2S OUT", NULL, "Vref Power"}, + {"I2S OUT", NULL, "IBias Power"}, + {"I2S IN", NULL, "Analog Power"}, + {"I2S IN", NULL, "DAC Vref"}, + {"I2S IN", NULL, "Vref Power"}, + {"I2S IN", NULL, "IBias Power"}, + + {"Right DAC", NULL, "I2S IN"}, + {"Left DAC", NULL, "I2S IN"}, + + {"LHPMIX", NULL, "Left DAC"}, + {"RHPMIX", NULL, "Right DAC"}, + + {"HPOR", NULL, "HPOR Cal"}, + {"HPOL", NULL, "HPOL Cal"}, + {"HPOR", NULL, "HPOR Supply"}, + {"HPOL", NULL, "HPOL Supply"}, + {"HPOL", NULL, "Headphone Charge Pump"}, + {"HPOR", NULL, "Headphone Charge Pump"}, + {"HPOL", NULL, "Headphone Driver Bias"}, + {"HPOR", NULL, "Headphone Driver Bias"}, + {"HPOL", NULL, "Headphone LDO"}, + {"HPOR", NULL, "Headphone LDO"}, + {"HPOL", NULL, "Headphone Reference"}, + {"HPOR", NULL, "Headphone Reference"}, + + {"HPOL", NULL, "LHPMIX"}, + {"HPOR", NULL, "RHPMIX"}, +}; + +static const struct regmap_range es8326_volatile_ranges[] = { + regmap_reg_range(ES8326_HP_DETECT, ES8326_HP_DETECT), +}; + +static const struct regmap_access_table es8326_volatile_table = { + .yes_ranges = es8326_volatile_ranges, + .n_yes_ranges = ARRAY_SIZE(es8326_volatile_ranges), +}; + +static const struct regmap_config es8326_regmap_config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = 0xff, + .volatile_table = &es8326_volatile_table, + .cache_type = REGCACHE_RBTREE, +}; + +struct _coeff_div { + u16 fs; + u32 rate; + u32 mclk; + u8 reg4; + u8 reg5; + u8 reg6; + u8 reg7; + u8 reg8; + u8 reg9; + u8 rega; + u8 regb; +}; + +/* codec hifi mclk clock divider coefficients */ +/* {ratio, LRCK, MCLK, REG04, REG05, REG06, REG07, REG08, REG09, REG10, REG11} */ +static const struct _coeff_div coeff_div[] = { + {32, 8000, 256000, 0x60, 0x00, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {32, 16000, 512000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F}, + {32, 44100, 1411200, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {32, 48000, 1536000, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {36, 8000, 288000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47}, + {36, 16000, 576000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47}, + {48, 8000, 384000, 0x60, 0x02, 0x1F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {48, 16000, 768000, 0x20, 0x02, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x3F}, + {48, 48000, 2304000, 0x00, 0x02, 0x0D, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {64, 8000, 512000, 0x60, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {64, 16000, 1024000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F}, + + {64, 44100, 2822400, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {64, 48000, 3072000, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {72, 8000, 576000, 0x20, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x23, 0x47}, + {72, 16000, 1152000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x23, 0x47}, + {96, 8000, 768000, 0x60, 0x02, 0x1D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {96, 16000, 1536000, 0x20, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F}, + {100, 48000, 4800000, 0x04, 0x04, 0x3F, 0x6D, 0x38, 0x08, 0x4f, 0x1f}, + {125, 48000, 6000000, 0x04, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27}, + {128, 8000, 1024000, 0x60, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {128, 16000, 2048000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + + {128, 44100, 5644800, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {128, 48000, 6144000, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {144, 8000, 1152000, 0x20, 0x00, 0x03, 0x35, 0x0A, 0x1B, 0x23, 0x47}, + {144, 16000, 2304000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x23, 0x47}, + {192, 8000, 1536000, 0x60, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {192, 16000, 3072000, 0x20, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F}, + {200, 48000, 9600000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {250, 48000, 12000000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27}, + {256, 8000, 2048000, 0x60, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {256, 16000, 4096000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + + {256, 44100, 11289600, 0x00, 0x00, 0x10, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {256, 48000, 12288000, 0x00, 0x00, 0x30, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {288, 8000, 2304000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x23, 0x47}, + {384, 8000, 3072000, 0x60, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x7F}, + {384, 16000, 6144000, 0x20, 0x02, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + {384, 48000, 18432000, 0x00, 0x02, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {400, 48000, 19200000, 0x09, 0x04, 0x0f, 0x6d, 0x3a, 0x0A, 0x4F, 0x1F}, + {500, 48000, 24000000, 0x18, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {512, 8000, 4096000, 0x60, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {512, 16000, 8192000, 0x20, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + + {512, 44100, 22579200, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {512, 48000, 24576000, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {768, 8000, 6144000, 0x60, 0x02, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {768, 16000, 12288000, 0x20, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + {800, 48000, 38400000, 0x00, 0x18, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F}, + {1024, 8000, 8192000, 0x60, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {1024, 16000, 16384000, 0x20, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + {1152, 16000, 18432000, 0x20, 0x08, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + {1536, 8000, 12288000, 0x60, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + + {1536, 16000, 24576000, 0x20, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F}, + {1625, 8000, 13000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27}, + {1625, 16000, 26000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27}, + {2048, 8000, 16384000, 0x60, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {2304, 8000, 18432000, 0x40, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x5F}, + {3072, 8000, 24576000, 0x60, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F}, + {3250, 8000, 26000000, 0x0C, 0x18, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27}, + +}; + +static inline int get_coeff(int mclk, int rate) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(coeff_div); i++) { + if (coeff_div[i].rate == rate && coeff_div[i].mclk == mclk) + return i; + } + + return -EINVAL; +} + +static int es8326_set_dai_sysclk(struct snd_soc_dai *codec_dai, + int clk_id, unsigned int freq, int dir) +{ + struct snd_soc_component *codec = codec_dai->component; + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(codec); + + es8326->sysclk = freq; + + return 0; +} + +static int es8326_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) +{ + struct snd_soc_component *component = codec_dai->component; + u8 iface = 0; + + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_CBC_CFP: + snd_soc_component_update_bits(component, ES8326_RESET, + ES8326_MASTER_MODE_EN, ES8326_MASTER_MODE_EN); + break; + case SND_SOC_DAIFMT_CBC_CFC: + break; + default: + return -EINVAL; + } + + /* interface format */ + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + break; + case SND_SOC_DAIFMT_RIGHT_J: + dev_err(component->dev, "Codec driver does not support right justified\n"); + return -EINVAL; + case SND_SOC_DAIFMT_LEFT_J: + iface |= ES8326_DAIFMT_LEFT_J; + break; + case SND_SOC_DAIFMT_DSP_A: + iface |= ES8326_DAIFMT_DSP_A; + break; + case SND_SOC_DAIFMT_DSP_B: + iface |= ES8326_DAIFMT_DSP_B; + break; + default: + return -EINVAL; + } + + snd_soc_component_update_bits(component, ES8326_FMT, ES8326_DAIFMT_MASK, iface); + + return 0; +} + +static int es8326_pcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + u8 srate = 0; + int coeff; + + coeff = get_coeff(es8326->sysclk, params_rate(params)); + /* bit size */ + switch (params_format(params)) { + case SNDRV_PCM_FORMAT_S16_LE: + srate |= ES8326_S16_LE; + break; + case SNDRV_PCM_FORMAT_S20_3LE: + srate |= ES8326_S20_3_LE; + break; + case SNDRV_PCM_FORMAT_S18_3LE: + srate |= ES8326_S18_LE; + break; + case SNDRV_PCM_FORMAT_S24_LE: + srate |= ES8326_S24_LE; + break; + case SNDRV_PCM_FORMAT_S32_LE: + srate |= ES8326_S32_LE; + break; + default: + return -EINVAL; + } + + /* set iface & srate */ + snd_soc_component_update_bits(component, ES8326_FMT, ES8326_DATA_LEN_MASK, srate); + + if (coeff >= 0) { + regmap_write(es8326->regmap, ES8326_CLK_DIV1, + coeff_div[coeff].reg4); + regmap_write(es8326->regmap, ES8326_CLK_DIV2, + coeff_div[coeff].reg5); + regmap_write(es8326->regmap, ES8326_CLK_DLL, + coeff_div[coeff].reg6); + regmap_write(es8326->regmap, ES8326_CLK_MUX, + coeff_div[coeff].reg7); + regmap_write(es8326->regmap, ES8326_CLK_ADC_SEL, + coeff_div[coeff].reg8); + regmap_write(es8326->regmap, ES8326_CLK_DAC_SEL, + coeff_div[coeff].reg9); + regmap_write(es8326->regmap, ES8326_CLK_ADC_OSR, + coeff_div[coeff].rega); + regmap_write(es8326->regmap, ES8326_CLK_DAC_OSR, + coeff_div[coeff].regb); + } else { + dev_warn(component->dev, "Clock coefficients do not match"); + } + + return 0; +} + +static int es8326_set_bias_level(struct snd_soc_component *codec, + enum snd_soc_bias_level level) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(codec); + int ret; + + switch (level) { + case SND_SOC_BIAS_ON: + ret = clk_prepare_enable(es8326->mclk); + if (ret) + return ret; + regmap_write(es8326->regmap, ES8326_RESET, ES8326_PWRUP_SEQ_EN); + regmap_write(es8326->regmap, ES8326_INTOUT_IO, 0x45); + regmap_write(es8326->regmap, ES8326_SDINOUT1_IO, + (ES8326_IO_DMIC_CLK << ES8326_SDINOUT1_SHIFT)); + regmap_write(es8326->regmap, ES8326_SDINOUT23_IO, ES8326_IO_INPUT); + regmap_write(es8326->regmap, ES8326_CLK_RESAMPLE, 0x05); + regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x02); + regmap_write(es8326->regmap, ES8326_PGA_PDN, 0x40); + regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0xAA); + regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_ON); + break; + case SND_SOC_BIAS_PREPARE: + break; + case SND_SOC_BIAS_STANDBY: + break; + case SND_SOC_BIAS_OFF: + clk_disable_unprepare(es8326->mclk); + regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0x11); + regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_OFF); + regmap_write(es8326->regmap, ES8326_PGA_PDN, 0xF8); + regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x00); + regmap_write(es8326->regmap, ES8326_INT_SOURCE, 0x08); + regmap_write(es8326->regmap, ES8326_SDINOUT1_IO, ES8326_IO_INPUT); + regmap_write(es8326->regmap, ES8326_SDINOUT23_IO, ES8326_IO_INPUT); + regmap_write(es8326->regmap, ES8326_RESET, + ES8326_CODEC_RESET | ES8326_PWRUP_SEQ_EN); + break; + } + + return 0; +} + +#define es8326_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\ + SNDRV_PCM_FMTBIT_S24_LE) + +static const struct snd_soc_dai_ops es8326_ops = { + .hw_params = es8326_pcm_hw_params, + .set_fmt = es8326_set_dai_fmt, + .set_sysclk = es8326_set_dai_sysclk, +}; + +static struct snd_soc_dai_driver es8326_dai = { + .name = "ES8326 HiFi", + .playback = { + .stream_name = "Playback", + .channels_min = 1, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = es8326_FORMATS, + }, + .capture = { + .stream_name = "Capture", + .channels_min = 1, + .channels_max = 2, + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = es8326_FORMATS, + }, + .ops = &es8326_ops, + .symmetric_rate = 1, +}; + +static void es8326_enable_micbias(struct snd_soc_component *component) +{ + struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); + + snd_soc_dapm_mutex_lock(dapm); + snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS1"); + snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS2"); + snd_soc_dapm_sync_unlocked(dapm); + snd_soc_dapm_mutex_unlock(dapm); +} + +static void es8326_disable_micbias(struct snd_soc_component *component) +{ + struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); + + snd_soc_dapm_mutex_lock(dapm); + snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS1"); + snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS2"); + snd_soc_dapm_sync_unlocked(dapm); + snd_soc_dapm_mutex_unlock(dapm); +} + +/* + * For button detection, set the following in soundcard + * snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE); + * snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOLUMEUP); + * snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN); + */ +static void es8326_jack_button_handler(struct work_struct *work) +{ + struct es8326_priv *es8326 = + container_of(work, struct es8326_priv, button_press_work.work); + struct snd_soc_component *comp = es8326->component; + unsigned int iface; + static int button_to_report, press_count; + static int prev_button, cur_button; + + if (!(es8326->jack->status & SND_JACK_HEADSET)) /* Jack unplugged */ + return; + + mutex_lock(&es8326->lock); + iface = snd_soc_component_read(comp, ES8326_HP_DETECT); + switch (iface) { + case 0x93: + /* pause button detected */ + cur_button = SND_JACK_BTN_0; + break; + case 0x6f: + /* button volume up */ + cur_button = SND_JACK_BTN_1; + break; + case 0x27: + /* button volume down */ + cur_button = SND_JACK_BTN_2; + break; + case 0x1e: + /* button released or not pressed */ + cur_button = 0; + break; + default: + break; + } + + if ((prev_button == cur_button) && (cur_button != 0)) { + press_count++; + if (press_count > 10) { + /* report a press every 500ms */ + snd_soc_jack_report(es8326->jack, cur_button, + SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2); + press_count = 0; + } + button_to_report = cur_button; + queue_delayed_work(system_wq, &es8326->button_press_work, + msecs_to_jiffies(50)); + } else if (prev_button != cur_button) { + /* mismatch, detect again */ + prev_button = cur_button; + queue_delayed_work(system_wq, &es8326->button_press_work, + msecs_to_jiffies(50)); + } else { + /* released or no pressed */ + if (button_to_report != 0) { + snd_soc_jack_report(es8326->jack, button_to_report, + SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2); + snd_soc_jack_report(es8326->jack, 0, + SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2); + button_to_report = 0; + } + } + mutex_unlock(&es8326->lock); +} + +static void es8326_jack_detect_handler(struct work_struct *work) +{ + struct es8326_priv *es8326 = + container_of(work, struct es8326_priv, jack_detect_work.work); + struct snd_soc_component *comp = es8326->component; + unsigned int iface; + + mutex_lock(&es8326->lock); + iface = snd_soc_component_read(comp, ES8326_HP_DETECT); + dev_dbg(comp->dev, "gpio flag %#04x", iface); + if ((iface & ES8326_HPINSERT_FLAG) == 0) { + /* Jack unplugged or spurious IRQ */ + dev_dbg(comp->dev, "No headset detected"); + if (es8326->jack->status & SND_JACK_HEADPHONE) { + snd_soc_jack_report(es8326->jack, 0, SND_JACK_HEADSET); + snd_soc_component_write(comp, ES8326_ADC1_SRC, es8326->mic2_src); + es8326_disable_micbias(comp); + } + } else if ((iface & ES8326_HPINSERT_FLAG) == ES8326_HPINSERT_FLAG) { + if (es8326->jack->status & SND_JACK_HEADSET) { + /* detect button */ + queue_delayed_work(system_wq, &es8326->button_press_work, 10); + } else { + if ((iface & ES8326_HPBUTTON_FLAG) == 0x00) { + dev_dbg(comp->dev, "Headset detected"); + snd_soc_jack_report(es8326->jack, + SND_JACK_HEADSET, SND_JACK_HEADSET); + snd_soc_component_write(comp, + ES8326_ADC1_SRC, es8326->mic1_src); + } else { + dev_dbg(comp->dev, "Headphone detected"); + snd_soc_jack_report(es8326->jack, + SND_JACK_HEADPHONE, SND_JACK_HEADSET); + } + } + } + mutex_unlock(&es8326->lock); +} + +static irqreturn_t es8326_irq(int irq, void *dev_id) +{ + struct es8326_priv *es8326 = dev_id; + struct snd_soc_component *comp = es8326->component; + + if (!es8326->jack) + goto out; + + es8326_enable_micbias(comp); + + if (es8326->jack->status & SND_JACK_HEADSET) + queue_delayed_work(system_wq, &es8326->jack_detect_work, + msecs_to_jiffies(10)); + else + queue_delayed_work(system_wq, &es8326->jack_detect_work, + msecs_to_jiffies(300)); + +out: + return IRQ_HANDLED; +} + +static int es8326_resume(struct snd_soc_component *component) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + unsigned int reg; + + regcache_cache_only(es8326->regmap, false); + regcache_sync(es8326->regmap); + + regmap_write(es8326->regmap, ES8326_CLK_CTL, ES8326_CLK_ON); + /* Two channel ADC */ + regmap_write(es8326->regmap, ES8326_PULLUP_CTL, 0x02); + regmap_write(es8326->regmap, ES8326_CLK_INV, 0x00); + regmap_write(es8326->regmap, ES8326_CLK_DIV_CPC, 0x1F); + regmap_write(es8326->regmap, ES8326_CLK_VMIDS1, 0xC8); + regmap_write(es8326->regmap, ES8326_CLK_VMIDS2, 0x88); + regmap_write(es8326->regmap, ES8326_CLK_CAL_TIME, 0x20); + regmap_write(es8326->regmap, ES8326_SYS_BIAS, 0x08); + regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0x22); + regmap_write(es8326->regmap, ES8326_ADC1_SRC, es8326->mic1_src); + regmap_write(es8326->regmap, ES8326_ADC2_SRC, es8326->mic2_src); + regmap_write(es8326->regmap, ES8326_HPJACK_TIMER, 0x88); + regmap_write(es8326->regmap, ES8326_HP_DET, + ES8326_HP_DET_SRC_PIN9 | es8326->jack_pol); + regmap_write(es8326->regmap, ES8326_INT_SOURCE, es8326->interrupt_src); + regmap_write(es8326->regmap, ES8326_INTOUT_IO, es8326->interrupt_clk); + regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_ON); + snd_soc_component_update_bits(component, ES8326_PGAGAIN, + ES8326_MIC_SEL_MASK, ES8326_MIC1_SEL); + + regmap_read(es8326->regmap, ES8326_CHIP_VERSION, ®); + if ((reg & ES8326_VERSION_B) == 1) { + regmap_write(es8326->regmap, ES8326_ANA_MICBIAS, 0xDD); + regmap_write(es8326->regmap, ES8326_ANA_VSEL, 0x7F); + regmap_write(es8326->regmap, ES8326_VMIDLOW, 0x0F); + /* enable button detect */ + regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xA0); + } + + es8326_irq(es8326->irq, es8326); + return 0; +} + +static int es8326_suspend(struct snd_soc_component *component) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + + cancel_delayed_work_sync(&es8326->jack_detect_work); + es8326_disable_micbias(component); + + regmap_write(es8326->regmap, ES8326_CLK_CTL, ES8326_CLK_OFF); + regcache_cache_only(es8326->regmap, true); + regcache_mark_dirty(es8326->regmap); + + return 0; +} + +static int es8326_probe(struct snd_soc_component *component) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + int ret; + + es8326->component = component; + es8326->jd_inverted = device_property_read_bool(component->dev, + "everest,jack-detect-inverted"); + + ret = device_property_read_u8(component->dev, "everest,mic1-src", &es8326->mic1_src); + if (ret != 0) { + dev_dbg(component->dev, "mic1-src return %d", ret); + es8326->mic1_src = ES8326_ADC_AMIC; + } + dev_dbg(component->dev, "mic1-src %x", es8326->mic1_src); + + ret = device_property_read_u8(component->dev, "everest,mic2-src", &es8326->mic2_src); + if (ret != 0) { + dev_dbg(component->dev, "mic2-src return %d", ret); + es8326->mic2_src = ES8326_ADC_DMIC; + } + dev_dbg(component->dev, "mic2-src %x", es8326->mic2_src); + + ret = device_property_read_u8(component->dev, "everest,jack-pol", &es8326->jack_pol); + if (ret != 0) { + dev_dbg(component->dev, "jack-pol return %d", ret); + es8326->jack_pol = ES8326_HP_DET_BUTTON_POL | ES8326_HP_TYPE_OMTP; + } + dev_dbg(component->dev, "jack-pol %x", es8326->jack_pol); + + ret = device_property_read_u8(component->dev, "everest,interrupt-src", &es8326->jack_pol); + if (ret != 0) { + dev_dbg(component->dev, "interrupt-src return %d", ret); + es8326->interrupt_src = ES8326_HP_DET_SRC_PIN9; + } + dev_dbg(component->dev, "interrupt-src %x", es8326->interrupt_src); + + ret = device_property_read_u8(component->dev, "everest,interrupt-clk", &es8326->jack_pol); + if (ret != 0) { + dev_dbg(component->dev, "interrupt-clk return %d", ret); + es8326->interrupt_clk = 0x45; + } + dev_dbg(component->dev, "interrupt-clk %x", es8326->interrupt_clk); + + es8326_resume(component); + return 0; +} + +static void es8326_enable_jack_detect(struct snd_soc_component *component, + struct snd_soc_jack *jack) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + + mutex_lock(&es8326->lock); + if (es8326->jd_inverted) + snd_soc_component_update_bits(component, ES8326_HP_DET, + ES8326_HP_DET_JACK_POL, ~es8326->jack_pol); + es8326->jack = jack; + + mutex_unlock(&es8326->lock); + es8326_irq(es8326->irq, es8326); +} + +static void es8326_disable_jack_detect(struct snd_soc_component *component) +{ + struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component); + + dev_dbg(component->dev, "Enter into %s\n", __func__); + if (!es8326->jack) + return; /* Already disabled (or never enabled) */ + cancel_delayed_work_sync(&es8326->jack_detect_work); + + mutex_lock(&es8326->lock); + if (es8326->jack->status & SND_JACK_MICROPHONE) { + es8326_disable_micbias(component); + snd_soc_jack_report(es8326->jack, 0, SND_JACK_HEADSET); + } + es8326->jack = NULL; + mutex_unlock(&es8326->lock); +} + +static int es8326_set_jack(struct snd_soc_component *component, + struct snd_soc_jack *jack, void *data) +{ + if (jack) + es8326_enable_jack_detect(component, jack); + else + es8326_disable_jack_detect(component); + + return 0; +} + +static void es8326_remove(struct snd_soc_component *component) +{ + es8326_disable_jack_detect(component); + es8326_set_bias_level(component, SND_SOC_BIAS_OFF); +} + +static const struct snd_soc_component_driver soc_component_dev_es8326 = { + .probe = es8326_probe, + .remove = es8326_remove, + .resume = es8326_resume, + .suspend = es8326_suspend, + .set_bias_level = es8326_set_bias_level, + .set_jack = es8326_set_jack, + .dapm_widgets = es8326_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(es8326_dapm_widgets), + .dapm_routes = es8326_dapm_routes, + .num_dapm_routes = ARRAY_SIZE(es8326_dapm_routes), + .controls = es8326_snd_controls, + .num_controls = ARRAY_SIZE(es8326_snd_controls), + .use_pmdown_time = 1, + .endianness = 1, +}; + +static int es8326_i2c_probe(struct i2c_client *i2c, + const struct i2c_device_id *id) +{ + struct es8326_priv *es8326; + int ret; + + es8326 = devm_kzalloc(&i2c->dev, sizeof(struct es8326_priv), GFP_KERNEL); + if (!es8326) + return -ENOMEM; + + i2c_set_clientdata(i2c, es8326); + es8326->i2c = i2c; + mutex_init(&es8326->lock); + es8326->regmap = devm_regmap_init_i2c(i2c, &es8326_regmap_config); + if (IS_ERR(es8326->regmap)) { + ret = PTR_ERR(es8326->regmap); + dev_err(&i2c->dev, "Failed to init regmap: %d\n", ret); + return ret; + } + + es8326->irq = i2c->irq; + INIT_DELAYED_WORK(&es8326->jack_detect_work, + es8326_jack_detect_handler); + INIT_DELAYED_WORK(&es8326->button_press_work, + es8326_jack_button_handler); + /* ES8316 is level-based while ES8326 is edge-based */ + ret = devm_request_threaded_irq(&i2c->dev, es8326->irq, NULL, es8326_irq, + IRQF_TRIGGER_RISING | IRQF_ONESHOT, + "es8326", es8326); + if (ret) { + dev_warn(&i2c->dev, "Failed to request IRQ: %d: %d\n", + es8326->irq, ret); + es8326->irq = -ENXIO; + } + + es8326->mclk = devm_clk_get_optional(&i2c->dev, "mclk"); + if (IS_ERR(es8326->mclk)) { + dev_err(&i2c->dev, "unable to get mclk\n"); + return PTR_ERR(es8326->mclk); + } + if (!es8326->mclk) + dev_warn(&i2c->dev, "assuming static mclk\n"); + + ret = clk_prepare_enable(es8326->mclk); + if (ret) { + dev_err(&i2c->dev, "unable to enable mclk\n"); + return ret; + } + return devm_snd_soc_register_component(&i2c->dev, + &soc_component_dev_es8326, + &es8326_dai, 1); +} + +static const struct i2c_device_id es8326_i2c_id[] = { + {"es8326", 0 }, + {} +}; +MODULE_DEVICE_TABLE(i2c, es8326_i2c_id); + +#ifdef CONFIG_OF +static const struct of_device_id es8326_of_match[] = { + { .compatible = "everest,es8326", }, + {} +}; +MODULE_DEVICE_TABLE(of, es8326_of_match); +#endif + +#ifdef CONFIG_ACPI +static const struct acpi_device_id es8326_acpi_match[] = { + {"ESSX8326", 0}, + {}, +}; +MODULE_DEVICE_TABLE(acpi, es8326_acpi_match); +#endif + +static struct i2c_driver es8326_i2c_driver = { + .driver = { + .name = "es8326", + .acpi_match_table = ACPI_PTR(es8326_acpi_match), + .of_match_table = of_match_ptr(es8326_of_match), + }, + .probe = es8326_i2c_probe, + .id_table = es8326_i2c_id, +}; +module_i2c_driver(es8326_i2c_driver); + +MODULE_DESCRIPTION("ASoC es8326 driver"); +MODULE_AUTHOR("David Yang <yangxiaohua@everest-semi.com>"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/codecs/es8326.h b/sound/soc/codecs/es8326.h new file mode 100755 index 0000000000000000000000000000000000000000..8e5ffe5ee10da3b6cbdbccea3ac726e46c9ac38f --- /dev/null +++ b/sound/soc/codecs/es8326.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * es8326.h -- es8326 ALSA SoC audio driver + * Copyright Everest Semiconductor Co.,Ltd + * + * Authors: David Yang <yangxiaohua@everest-semi.com> + */ + +#ifndef _ES8326_H +#define _ES8326_H + +#define CONFIG_HHTECH_MINIPMP 1 + +/* ES8326 register space */ +#define ES8326_RESET 0x00 +#define ES8326_CLK_CTL 0x01 +#define ES8326_CLK_INV 0x02 +#define ES8326_CLK_RESAMPLE 0x03 +#define ES8326_CLK_DIV1 0x04 +#define ES8326_CLK_DIV2 0x05 +#define ES8326_CLK_DLL 0x06 +#define ES8326_CLK_MUX 0x07 +#define ES8326_CLK_ADC_SEL 0x08 +#define ES8326_CLK_DAC_SEL 0x09 +#define ES8326_CLK_ADC_OSR 0x0a +#define ES8326_CLK_DAC_OSR 0x0b +#define ES8326_CLK_DIV_CPC 0x0c +#define ES8326_CLK_DIV_BCLK 0x0d +#define ES8326_CLK_TRI 0x0e +#define ES8326_CLK_DIV_LRCK 0x0f +#define ES8326_CLK_VMIDS1 0x10 +#define ES8326_CLK_VMIDS2 0x11 +#define ES8326_CLK_CAL_TIME 0x12 +#define ES8326_FMT 0x13 + +#define ES8326_DAC_MUTE 0x14 +#define ES8326_ADC_MUTE 0x15 +#define ES8326_ANA_PDN 0x16 +#define ES8326_PGA_PDN 0x17 +#define ES8326_VMIDSEL 0x18 +#define ES8326_ANA_LP 0x19 +#define ES8326_ANA_DMS 0x1a +#define ES8326_ANA_MICBIAS 0x1b +#define ES8326_ANA_VSEL 0x1c +#define ES8326_SYS_BIAS 0x1d +#define ES8326_BIAS_SW1 0x1e +#define ES8326_BIAS_SW2 0x1f +#define ES8326_BIAS_SW3 0x20 +#define ES8326_BIAS_SW4 0x21 +#define ES8326_VMIDLOW 0x22 +#define ES8326_PGAGAIN 0x23 +#define ES8326_HP_DRIVER 0x24 +#define ES8326_DAC2HPMIX 0x25 +#define ES8326_HP_VOL 0x26 +#define ES8326_HP_CAL 0x27 +#define ES8326_HP_DRIVER_REF 0x28 +#define ES8326_ADC_SCALE 0x29 +#define ES8326_ADC1_SRC 0x2a +#define ES8326_ADC2_SRC 0x2b +#define ES8326_ADC1_VOL 0x2c +#define ES8326_ADC2_VOL 0x2d +#define ES8326_ADC_RAMPRATE 0x2e +#define ES8326_ALC_RECOVERY 0x32 +#define ES8326_ALC_LEVEL 0x33 +#define ES8326_ADC_HPFS1 0x34 +#define ES8326_ADC_HPFS2 0x35 +#define ES8326_ADC_EQ 0x36 +#define ES8326_HP_OFFSET_CAL 0x4A +#define ES8326_HPL_OFFSET_INI 0x4B +#define ES8326_HPR_OFFSET_INI 0x4C +#define ES8326_DAC_DSM 0x4D +#define ES8326_DAC_RAMPRATE 0x4E +#define ES8326_DAC_VPPSCALE 0x4F +#define ES8326_DAC_VOL 0x50 +#define ES8326_DRC_RECOVERY 0x53 +#define ES8326_DRC_WINSIZE 0x54 +#define ES8326_HPJACK_TIMER 0x56 +#define ES8326_HP_DET 0x57 +#define ES8326_INT_SOURCE 0x58 +#define ES8326_INTOUT_IO 0x59 +#define ES8326_SDINOUT1_IO 0x5A +#define ES8326_SDINOUT23_IO 0x5B +#define ES8326_JACK_PULSE 0x5C + +#define ES8326_PULLUP_CTL 0xF9 +#define ES8326_HP_DETECT 0xFB +#define ES8326_CHIP_ID1 0xFD +#define ES8326_CHIP_ID2 0xFE +#define ES8326_CHIP_VERSION 0xFF + +/* ES8326_RESET */ +#define ES8326_CSM_ON (1 << 7) +#define ES8326_MASTER_MODE_EN (1 << 6) +#define ES8326_PWRUP_SEQ_EN (1 << 5) +#define ES8326_CODEC_RESET (0x0f << 0) +#define ES8326_CSM_OFF (0 << 7) + +/* ES8326_CLK_CTL */ +#define ES8326_CLK_ON (0x7f << 0) +#define ES8326_CLK_OFF (0 << 0) + +/* ES8326_CLK_INV */ +#define ES8326_BCLK_AS_MCLK (1 << 3) + +/* ES8326_FMT */ +#define ES8326_S24_LE (0 << 2) +#define ES8326_S20_3_LE (1 << 2) +#define ES8326_S18_LE (2 << 2) +#define ES8326_S16_LE (3 << 2) +#define ES8326_S32_LE (4 << 2) +#define ES8326_DATA_LEN_MASK (7 << 2) + +#define ES8326_DAIFMT_MASK ((1 << 5) | (3 << 0)) +#define ES8326_DAIFMT_I2S 0 +#define ES8326_DAIFMT_LEFT_J (1 << 0) +#define ES8326_DAIFMT_DSP_A (3 << 0) +#define ES8326_DAIFMT_DSP_B ((1 << 5) | (3 << 0)) + +/* ES8326_PGAGAIN */ +#define ES8326_MIC_SEL_MASK (3 << 4) +#define ES8326_MIC1_SEL (1 << 4) +#define ES8326_MIC2_SEL (1 << 5) + +/* ES8326_HP_CAL */ +#define ES8326_HPOR_SHIFT 4 + +/* ES8326_ADC1_SRC */ +#define ES8326_ADC1_SHIFT 0 +#define ES8326_ADC2_SHIFT 4 +#define ES8326_ADC_SRC_ANA 0 +#define ES8326_ADC_SRC_ANA_INV_SW0 1 +#define ES8326_ADC_SRC_ANA_INV_SW1 2 +#define ES8326_ADC_SRC_DMIC_MCLK 3 +#define ES8326_ADC_SRC_DMIC_SDIN2 4 +#define ES8326_ADC_SRC_DMIC_SDIN2_INV 5 +#define ES8326_ADC_SRC_DMIC_SDIN3 6 +#define ES8326_ADC_SRC_DMIC_SDIN3_INV 7 + +#define ES8326_ADC_AMIC ((ES8326_ADC_SRC_ANA_INV_SW1 << ES8326_ADC2_SHIFT) \ + | (ES8326_ADC_SRC_ANA_INV_SW1 << ES8326_ADC1_SHIFT)) +#define ES8326_ADC_DMIC ((ES8326_ADC_SRC_DMIC_SDIN2 << ES8326_ADC2_SHIFT) \ + | (ES8326_ADC_SRC_DMIC_SDIN2 << ES8326_ADC1_SHIFT)) +/* ES8326_ADC2_SRC */ +#define ES8326_ADC3_SHIFT 0 +#define ES8326_ADC4_SHIFT 3 + +/* ES8326_HP_DET */ +#define ES8326_HP_DET_SRC_PIN27 (1 << 5) +#define ES8326_HP_DET_SRC_PIN9 (1 << 4) +#define ES8326_HP_DET_JACK_POL (1 << 3) +#define ES8326_HP_DET_BUTTON_POL (1 << 2) +#define ES8326_HP_TYPE_OMTP (3 << 0) +#define ES8326_HP_TYPE_CTIA (2 << 0) +#define ES8326_HP_TYPE_AUTO (1 << 0) +#define ES8326_HP_TYPE_AUTO_INV (0 << 0) + +/* ES8326_SDINOUT1_IO */ +#define ES8326_IO_INPUT (0 << 0) +#define ES8326_IO_SDIN_SLOT0 (1 << 0) +#define ES8326_IO_SDIN_SLOT1 (2 << 0) +#define ES8326_IO_SDIN_SLOT2 (3 << 0) +#define ES8326_IO_SDIN_SLOT7 (8 << 0) +#define ES8326_IO_DMIC_CLK (9 << 0) +#define ES8326_IO_DMIC_CLK_INV (0x0a << 0) +#define ES8326_IO_SDOUT2 (0x0b << 0) +#define ES8326_IO_LOW (0x0e << 0) +#define ES8326_IO_HIGH (0x0f << 0) +#define ES8326_ADC2DAC (1 << 3) +#define ES8326_SDINOUT1_SHIFT 4 + +/* ES8326_SDINOUT23_IO */ +#define ES8326_SDINOUT2_SHIFT 4 +#define ES8326_SDINOUT3_SHIFT 0 + +/* ES8326_HP_DETECT */ +#define ES8326_HPINSERT_FLAG (1 << 1) +#define ES8326_HPBUTTON_FLAG (1 << 0) + +/* ES8326_CHIP_VERSION 0xFF */ +#define ES8326_VERSION_B (1 << 0) + +#endif diff --git a/sound/soc/codecs/hda.c b/sound/soc/codecs/hda.c index ad20a3dff9b7ebf4a9f5eecd394b9b2b4895b525..61e8e9be6b8d7c22dc9b7ad893dcff1a61ce4c36 100644 --- a/sound/soc/codecs/hda.c +++ b/sound/soc/codecs/hda.c @@ -224,9 +224,6 @@ static int hda_codec_probe(struct snd_soc_component *component) goto err; } - /* configure codec for 1:1 PCM:DAI mapping */ - codec->mst_no_extra_pcms = 1; - ret = snd_hda_codec_parse_pcms(codec); if (ret < 0) { dev_err(&hdev->dev, "unable to map pcms to dai %d\n", ret); diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c index 8debcee592247b70f80a7448e9416c88e71c59d4..8af434e14bfba2228d6cde172d139f9dee98bd37 100644 --- a/sound/soc/codecs/hdac_hda.c +++ b/sound/soc/codecs/hdac_hda.c @@ -246,7 +246,7 @@ static int hdac_hda_dai_hw_free(struct snd_pcm_substream *substream, return -EINVAL; hda_stream = &pcm->stream[substream->stream]; - snd_hda_codec_cleanup(&hda_pvt->codec, hda_stream, substream); + snd_hda_codec_cleanup(hda_pvt->codec, hda_stream, substream); return 0; } @@ -264,7 +264,7 @@ static int hdac_hda_dai_prepare(struct snd_pcm_substream *substream, int ret = 0; hda_pvt = snd_soc_component_get_drvdata(component); - hdev = &hda_pvt->codec.core; + hdev = &hda_pvt->codec->core; pcm = snd_soc_find_pcm_from_dai(hda_pvt, dai); if (!pcm) return -EINVAL; @@ -274,7 +274,7 @@ static int hdac_hda_dai_prepare(struct snd_pcm_substream *substream, stream = hda_pvt->pcm[dai->id].stream_tag[substream->stream]; format_val = hda_pvt->pcm[dai->id].format_val[substream->stream]; - ret = snd_hda_codec_prepare(&hda_pvt->codec, hda_stream, + ret = snd_hda_codec_prepare(hda_pvt->codec, hda_stream, stream, format_val, substream); if (ret < 0) dev_err(&hdev->dev, "codec prepare failed %d\n", ret); @@ -299,7 +299,7 @@ static int hdac_hda_dai_open(struct snd_pcm_substream *substream, hda_stream = &pcm->stream[substream->stream]; - return hda_stream->ops.open(hda_stream, &hda_pvt->codec, substream); + return hda_stream->ops.open(hda_stream, hda_pvt->codec, substream); } static void hdac_hda_dai_close(struct snd_pcm_substream *substream, @@ -317,7 +317,7 @@ static void hdac_hda_dai_close(struct snd_pcm_substream *substream, hda_stream = &pcm->stream[substream->stream]; - hda_stream->ops.close(hda_stream, &hda_pvt->codec, substream); + hda_stream->ops.close(hda_stream, hda_pvt->codec, substream); snd_hda_codec_pcm_put(pcm); } @@ -325,7 +325,7 @@ static void hdac_hda_dai_close(struct snd_pcm_substream *substream, static struct hda_pcm *snd_soc_find_pcm_from_dai(struct hdac_hda_priv *hda_pvt, struct snd_soc_dai *dai) { - struct hda_codec *hcodec = &hda_pvt->codec; + struct hda_codec *hcodec = hda_pvt->codec; struct hda_pcm *cpcm; const char *pcm_name; @@ -394,8 +394,8 @@ static int hdac_hda_codec_probe(struct snd_soc_component *component) snd_soc_component_get_drvdata(component); struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); - struct hdac_device *hdev = &hda_pvt->codec.core; - struct hda_codec *hcodec = &hda_pvt->codec; + struct hdac_device *hdev = &hda_pvt->codec->core; + struct hda_codec *hcodec = hda_pvt->codec; struct hdac_ext_link *hlink; hda_codec_patch_t patch; int ret; @@ -461,9 +461,6 @@ static int hdac_hda_codec_probe(struct snd_soc_component *component) dev_dbg(&hdev->dev, "no patch file found\n"); } - /* configure codec for 1:1 PCM:DAI mapping */ - hcodec->mst_no_extra_pcms = 1; - ret = snd_hda_codec_parse_pcms(hcodec); if (ret < 0) { dev_err(&hdev->dev, "unable to map pcms to dai %d\n", ret); @@ -515,8 +512,8 @@ static void hdac_hda_codec_remove(struct snd_soc_component *component) { struct hdac_hda_priv *hda_pvt = snd_soc_component_get_drvdata(component); - struct hdac_device *hdev = &hda_pvt->codec.core; - struct hda_codec *codec = &hda_pvt->codec; + struct hdac_device *hdev = &hda_pvt->codec->core; + struct hda_codec *codec = hda_pvt->codec; struct hdac_ext_link *hlink = NULL; hlink = snd_hdac_ext_bus_get_link(hdev->bus, dev_name(&hdev->dev)); @@ -584,7 +581,6 @@ static const struct snd_soc_component_driver hdac_hda_codec = { static int hdac_hda_dev_probe(struct hdac_device *hdev) { struct hdac_ext_link *hlink; - struct hdac_hda_priv *hda_pvt; int ret; /* hold the ref while we probe */ @@ -595,10 +591,6 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev) } snd_hdac_ext_bus_link_get(hdev->bus, hlink); - hda_pvt = hdac_to_hda_priv(hdev); - if (!hda_pvt) - return -ENOMEM; - /* ASoC specific initialization */ ret = devm_snd_soc_register_component(&hdev->dev, &hdac_hda_codec, hdac_hda_dais, @@ -608,7 +600,6 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev) return ret; } - dev_set_drvdata(&hdev->dev, hda_pvt); snd_hdac_ext_bus_link_put(hdev->bus, hlink); return ret; diff --git a/sound/soc/codecs/hdac_hda.h b/sound/soc/codecs/hdac_hda.h index d0efc5e254ae9db2da7887e50a3733aa3f9163df..fc19c34ca00e5c55f1eb21a064f4fe841fef806b 100644 --- a/sound/soc/codecs/hdac_hda.h +++ b/sound/soc/codecs/hdac_hda.h @@ -23,7 +23,7 @@ struct hdac_hda_pcm { }; struct hdac_hda_priv { - struct hda_codec codec; + struct hda_codec *codec; struct hdac_hda_pcm pcm[HDAC_LAST_DAI_ID]; bool need_display_power; }; diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c index 5679102de91f8d2510bc570bf51f979ff0b47384..0b1cdb2d60498e37ca239923a535402a6ac558aa 100644 --- a/sound/soc/codecs/hdmi-codec.c +++ b/sound/soc/codecs/hdmi-codec.c @@ -20,10 +20,6 @@ #define HDMI_CODEC_CHMAP_IDX_UNKNOWN -1 -struct hdmi_codec_channel_map_table { - unsigned char map; /* ALSA API channel map position */ -}; - /* * CEA speaker placement for HDMI 1.4: * @@ -827,7 +823,7 @@ static int hdmi_dai_probe(struct snd_soc_dai *dai) if (ret) return ret; - daifmt = kzalloc(sizeof(*daifmt), GFP_KERNEL); + daifmt = devm_kzalloc(dai->dev, sizeof(*daifmt), GFP_KERNEL); if (!daifmt) return -ENOMEM; @@ -894,17 +890,10 @@ static int hdmi_dai_spdif_probe(struct snd_soc_dai *dai) return 0; } -static int hdmi_codec_dai_remove(struct snd_soc_dai *dai) -{ - kfree(dai->playback_dma_data); - return 0; -} - static const struct snd_soc_dai_driver hdmi_i2s_dai = { .name = "i2s-hifi", .id = DAI_ID_I2S, .probe = hdmi_dai_probe, - .remove = hdmi_codec_dai_remove, .playback = { .stream_name = "I2S Playback", .channels_min = 2, @@ -929,7 +918,6 @@ static const struct snd_soc_dai_driver hdmi_spdif_dai = { .name = "spdif-hifi", .id = DAI_ID_SPDIF, .probe = hdmi_dai_spdif_probe, - .remove = hdmi_codec_dai_remove, .playback = { .stream_name = "SPDIF Playback", .channels_min = 2, diff --git a/sound/soc/codecs/lpass-rx-macro.c b/sound/soc/codecs/lpass-rx-macro.c index 3143f9cd7277e4c81294ffe3014c95c866e1d32f..a9ef9d5ffcc5ca1e37c718c374bdf6035e8655a8 100644 --- a/sound/soc/codecs/lpass-rx-macro.c +++ b/sound/soc/codecs/lpass-rx-macro.c @@ -596,7 +596,6 @@ struct rx_macro { int rx_port_value[RX_MACRO_PORTS_MAX]; u16 prim_int_users[INTERP_MAX]; int rx_mclk_users; - bool reset_swr; int clsh_users; int rx_mclk_cnt; bool is_ear_mode_on; @@ -3442,18 +3441,15 @@ static int swclk_gate_enable(struct clk_hw *hw) } rx_macro_mclk_enable(rx, true); - if (rx->reset_swr) - regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL, - CDC_RX_SWR_RESET_MASK, - CDC_RX_SWR_RESET); + regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL, + CDC_RX_SWR_RESET_MASK, + CDC_RX_SWR_RESET); regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL, CDC_RX_SWR_CLK_EN_MASK, 1); - if (rx->reset_swr) - regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL, - CDC_RX_SWR_RESET_MASK, 0); - rx->reset_swr = false; + regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL, + CDC_RX_SWR_RESET_MASK, 0); return 0; } @@ -3579,7 +3575,6 @@ static int rx_macro_probe(struct platform_device *pdev) dev_set_drvdata(dev, rx); - rx->reset_swr = true; rx->dev = dev; /* set MCLK and NPL rates */ @@ -3659,6 +3654,8 @@ static int rx_macro_remove(struct platform_device *pdev) static const struct of_device_id rx_macro_dt_match[] = { { .compatible = "qcom,sc7280-lpass-rx-macro" }, { .compatible = "qcom,sm8250-lpass-rx-macro" }, + { .compatible = "qcom,sm8450-lpass-rx-macro" }, + { .compatible = "qcom,sc8280xp-lpass-rx-macro" }, { } }; MODULE_DEVICE_TABLE(of, rx_macro_dt_match); @@ -3701,7 +3698,6 @@ static int __maybe_unused rx_macro_runtime_resume(struct device *dev) } regcache_cache_only(rx->regmap, false); regcache_sync(rx->regmap); - rx->reset_swr = true; return 0; err_fsgen: diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c index 55503ba480bb6350fb482f12e24c77581d0bb16a..ee15cf6b98bba19f0175997f8ad42d32e516593f 100644 --- a/sound/soc/codecs/lpass-tx-macro.c +++ b/sound/soc/codecs/lpass-tx-macro.c @@ -259,7 +259,7 @@ struct tx_macro { struct tx_mute_work tx_mute_dwork[NUM_DECIMATORS]; unsigned long active_ch_mask[TX_MACRO_MAX_DAIS]; unsigned long active_ch_cnt[TX_MACRO_MAX_DAIS]; - unsigned long active_decimator[TX_MACRO_MAX_DAIS]; + int active_decimator[TX_MACRO_MAX_DAIS]; struct regmap *regmap; struct clk *mclk; struct clk *npl; @@ -268,7 +268,6 @@ struct tx_macro { struct clk *fsgen; struct clk_hw hw; bool dec_active[NUM_DECIMATORS]; - bool reset_swr; int tx_mclk_users; u16 dmic_clk_div; bool bcs_enable; @@ -823,17 +822,23 @@ static int tx_macro_tx_mixer_put(struct snd_kcontrol *kcontrol, struct tx_macro *tx = snd_soc_component_get_drvdata(component); if (enable) { + if (tx->active_decimator[dai_id] == dec_id) + return 0; + set_bit(dec_id, &tx->active_ch_mask[dai_id]); tx->active_ch_cnt[dai_id]++; tx->active_decimator[dai_id] = dec_id; } else { + if (tx->active_decimator[dai_id] == -1) + return 0; + tx->active_ch_cnt[dai_id]--; clear_bit(dec_id, &tx->active_ch_mask[dai_id]); tx->active_decimator[dai_id] = -1; } snd_soc_dapm_mixer_update_power(widget->dapm, kcontrol, enable, update); - return 0; + return 1; } static int tx_macro_enable_dec(struct snd_soc_dapm_widget *w, @@ -1019,9 +1024,12 @@ static int tx_macro_dec_mode_put(struct snd_kcontrol *kcontrol, int path = e->shift_l; struct tx_macro *tx = snd_soc_component_get_drvdata(component); + if (tx->dec_mode[path] == value) + return 0; + tx->dec_mode[path] = value; - return 0; + return 1; } static int tx_macro_get_bcs(struct snd_kcontrol *kcontrol, @@ -1118,6 +1126,10 @@ static int tx_macro_digital_mute(struct snd_soc_dai *dai, int mute, int stream) struct tx_macro *tx = snd_soc_component_get_drvdata(component); u16 decimator; + /* active decimator not set yet */ + if (tx->active_decimator[dai->id] == -1) + return 0; + decimator = tx->active_decimator[dai->id]; if (mute) @@ -1702,18 +1714,14 @@ static int swclk_gate_enable(struct clk_hw *hw) } tx_macro_mclk_enable(tx, true); - if (tx->reset_swr) - regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL, - CDC_TX_SWR_RESET_MASK, - CDC_TX_SWR_RESET_ENABLE); + regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL, + CDC_TX_SWR_RESET_MASK, CDC_TX_SWR_RESET_ENABLE); regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL, CDC_TX_SWR_CLK_EN_MASK, CDC_TX_SWR_CLK_ENABLE); - if (tx->reset_swr) - regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL, - CDC_TX_SWR_RESET_MASK, 0x0); - tx->reset_swr = false; + regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL, + CDC_TX_SWR_RESET_MASK, 0x0); return 0; } @@ -1855,7 +1863,6 @@ static int tx_macro_probe(struct platform_device *pdev) dev_set_drvdata(dev, tx); - tx->reset_swr = true; tx->dev = dev; /* set MCLK and NPL rates */ @@ -1970,7 +1977,6 @@ static int __maybe_unused tx_macro_runtime_resume(struct device *dev) regcache_cache_only(tx->regmap, false); regcache_sync(tx->regmap); - tx->reset_swr = true; return 0; err_fsgen: @@ -1988,6 +1994,8 @@ static const struct dev_pm_ops tx_macro_pm_ops = { static const struct of_device_id tx_macro_dt_match[] = { { .compatible = "qcom,sc7280-lpass-tx-macro" }, { .compatible = "qcom,sm8250-lpass-tx-macro" }, + { .compatible = "qcom,sm8450-lpass-tx-macro" }, + { .compatible = "qcom,sc8280xp-lpass-tx-macro" }, { } }; MODULE_DEVICE_TABLE(of, tx_macro_dt_match); diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c index 1ea10dc70748a1cf66a0fb39043d1e734b3dfb0c..b0b6cf29cba30ce32d38557ab27a5df14b490b41 100644 --- a/sound/soc/codecs/lpass-va-macro.c +++ b/sound/soc/codecs/lpass-va-macro.c @@ -23,7 +23,12 @@ #define CDC_VA_MCLK_CONTROL_EN BIT(0) #define CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL (0x0004) #define CDC_VA_FS_CONTROL_EN BIT(0) +#define CDC_VA_FS_COUNTER_CLR BIT(1) #define CDC_VA_CLK_RST_CTRL_SWR_CONTROL (0x0008) +#define CDC_VA_SWR_RESET_MASK BIT(1) +#define CDC_VA_SWR_RESET_ENABLE BIT(1) +#define CDC_VA_SWR_CLK_EN_MASK BIT(0) +#define CDC_VA_SWR_CLK_ENABLE BIT(0) #define CDC_VA_TOP_CSR_TOP_CFG0 (0x0080) #define CDC_VA_FS_BROADCAST_EN BIT(1) #define CDC_VA_TOP_CSR_DMIC0_CTL (0x0084) @@ -65,6 +70,8 @@ #define CDC_VA_TOP_CSR_SWR_MIC_CTL0 (0x00D0) #define CDC_VA_TOP_CSR_SWR_MIC_CTL1 (0x00D4) #define CDC_VA_TOP_CSR_SWR_MIC_CTL2 (0x00D8) +#define CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK (0xEE) +#define CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1 (0xCC) #define CDC_VA_TOP_CSR_SWR_CTRL (0x00DC) #define CDC_VA_INP_MUX_ADC_MUX0_CFG0 (0x0100) #define CDC_VA_INP_MUX_ADC_MUX0_CFG1 (0x0104) @@ -193,6 +200,7 @@ struct va_macro { unsigned long active_ch_mask[VA_MACRO_MAX_DAIS]; unsigned long active_ch_cnt[VA_MACRO_MAX_DAIS]; u16 dmic_clk_div; + bool has_swr_master; int dec_mode[VA_MACRO_NUM_DECIMATORS]; struct regmap *regmap; @@ -215,6 +223,18 @@ struct va_macro { #define to_va_macro(_hw) container_of(_hw, struct va_macro, hw) +struct va_macro_data { + bool has_swr_master; +}; + +static const struct va_macro_data sm8250_va_data = { + .has_swr_master = false, +}; + +static const struct va_macro_data sm8450_va_data = { + .has_swr_master = true, +}; + static bool va_is_volatile_register(struct device *dev, unsigned int reg) { switch (reg) { @@ -324,6 +344,9 @@ static bool va_is_rw_register(struct device *dev, unsigned int reg) case CDC_VA_TOP_CSR_DMIC2_CTL: case CDC_VA_TOP_CSR_DMIC3_CTL: case CDC_VA_TOP_CSR_DMIC_CFG: + case CDC_VA_TOP_CSR_SWR_MIC_CTL0: + case CDC_VA_TOP_CSR_SWR_MIC_CTL1: + case CDC_VA_TOP_CSR_SWR_MIC_CTL2: case CDC_VA_TOP_CSR_DEBUG_BUS: case CDC_VA_TOP_CSR_DEBUG_EN: case CDC_VA_TOP_CSR_TX_I2S_CTL: @@ -423,9 +446,12 @@ static int va_clk_rsc_fs_gen_request(struct va_macro *va, bool enable) regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_MCLK_CONTROL, CDC_VA_MCLK_CONTROL_EN, CDC_VA_MCLK_CONTROL_EN); - + /* clear the fs counter */ regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL, - CDC_VA_FS_CONTROL_EN, + CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR, + CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR); + regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL, + CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR, CDC_VA_FS_CONTROL_EN); regmap_update_bits(regmap, CDC_VA_TOP_CSR_TOP_CFG0, @@ -1302,12 +1328,36 @@ static const struct snd_soc_component_driver va_macro_component_drv = { static int fsgen_gate_enable(struct clk_hw *hw) { - return va_macro_mclk_enable(to_va_macro(hw), true); + struct va_macro *va = to_va_macro(hw); + struct regmap *regmap = va->regmap; + int ret; + + ret = va_macro_mclk_enable(va, true); + if (!va->has_swr_master) + return ret; + + regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, + CDC_VA_SWR_RESET_MASK, CDC_VA_SWR_RESET_ENABLE); + + regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, + CDC_VA_SWR_CLK_EN_MASK, + CDC_VA_SWR_CLK_ENABLE); + regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, + CDC_VA_SWR_RESET_MASK, 0x0); + + return ret; } static void fsgen_gate_disable(struct clk_hw *hw) { - va_macro_mclk_enable(to_va_macro(hw), false); + struct va_macro *va = to_va_macro(hw); + struct regmap *regmap = va->regmap; + + if (va->has_swr_master) + regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL, + CDC_VA_SWR_CLK_EN_MASK, 0x0); + + va_macro_mclk_enable(va, false); } static int fsgen_gate_is_enabled(struct clk_hw *hw) @@ -1401,6 +1451,7 @@ undefined_rate: static int va_macro_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + const struct va_macro_data *data; struct va_macro *va; void __iomem *base; u32 sample_rate = 0; @@ -1455,6 +1506,9 @@ static int va_macro_probe(struct platform_device *pdev) dev_set_drvdata(dev, va); + data = of_device_get_match_data(dev); + va->has_swr_master = data->has_swr_master; + /* mclk rate */ clk_set_rate(va->mclk, 2 * VA_MACRO_MCLK_FREQ); @@ -1480,6 +1534,20 @@ static int va_macro_probe(struct platform_device *pdev) goto err_clkout; } + if (va->has_swr_master) { + /* Set default CLK div to 1 */ + regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL0, + CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK, + CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1); + regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL1, + CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK, + CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1); + regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL2, + CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK, + CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1); + + } + ret = devm_snd_soc_register_component(dev, &va_macro_component_drv, va_macro_dais, ARRAY_SIZE(va_macro_dais)); @@ -1554,8 +1622,10 @@ static const struct dev_pm_ops va_macro_pm_ops = { }; static const struct of_device_id va_macro_dt_match[] = { - { .compatible = "qcom,sc7280-lpass-va-macro" }, - { .compatible = "qcom,sm8250-lpass-va-macro" }, + { .compatible = "qcom,sc7280-lpass-va-macro", .data = &sm8250_va_data }, + { .compatible = "qcom,sm8250-lpass-va-macro", .data = &sm8250_va_data }, + { .compatible = "qcom,sm8450-lpass-va-macro", .data = &sm8450_va_data }, + { .compatible = "qcom,sc8280xp-lpass-va-macro", .data = &sm8450_va_data }, {} }; MODULE_DEVICE_TABLE(of, va_macro_dt_match); diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c index 27da6c6c3c5a049222e5f1b491dd133868c3e4e2..5e0abefe7ccedc10f2ac0bafe5313ca4d3fc4566 100644 --- a/sound/soc/codecs/lpass-wsa-macro.c +++ b/sound/soc/codecs/lpass-wsa-macro.c @@ -338,7 +338,6 @@ struct wsa_macro { int ec_hq[WSA_MACRO_RX1 + 1]; u16 prim_int_users[WSA_MACRO_RX1 + 1]; u16 wsa_mclk_users; - bool reset_swr; unsigned long active_ch_mask[WSA_MACRO_MAX_DAIS]; unsigned long active_ch_cnt[WSA_MACRO_MAX_DAIS]; int rx_port_value[WSA_MACRO_RX_MAX]; @@ -2271,23 +2270,16 @@ static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable) wsa_macro_mclk_enable(wsa, true); /* reset swr ip */ - if (wsa->reset_swr) - regmap_update_bits(regmap, - CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, - CDC_WSA_SWR_RST_EN_MASK, - CDC_WSA_SWR_RST_ENABLE); + regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, + CDC_WSA_SWR_RST_EN_MASK, CDC_WSA_SWR_RST_ENABLE); regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, CDC_WSA_SWR_CLK_EN_MASK, CDC_WSA_SWR_CLK_ENABLE); /* Bring out of reset */ - if (wsa->reset_swr) - regmap_update_bits(regmap, - CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, - CDC_WSA_SWR_RST_EN_MASK, - CDC_WSA_SWR_RST_DISABLE); - wsa->reset_swr = false; + regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, + CDC_WSA_SWR_RST_EN_MASK, CDC_WSA_SWR_RST_DISABLE); } else { regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL, CDC_WSA_SWR_CLK_EN_MASK, 0); @@ -2431,7 +2423,6 @@ static int wsa_macro_probe(struct platform_device *pdev) dev_set_drvdata(dev, wsa); - wsa->reset_swr = true; wsa->dev = dev; /* set MCLK and NPL rates */ @@ -2561,6 +2552,8 @@ static const struct dev_pm_ops wsa_macro_pm_ops = { static const struct of_device_id wsa_macro_dt_match[] = { {.compatible = "qcom,sc7280-lpass-wsa-macro"}, {.compatible = "qcom,sm8250-lpass-wsa-macro"}, + {.compatible = "qcom,sm8450-lpass-wsa-macro"}, + {.compatible = "qcom,sc8280xp-lpass-wsa-macro" }, {} }; MODULE_DEVICE_TABLE(of, wsa_macro_dt_match); diff --git a/sound/soc/codecs/max98088.c b/sound/soc/codecs/max98088.c index 5435a49604cf1e5136570b26e8352bf6bdf6c308..405ec16be2b6ac925f3f2e65380ef3c312a5113b 100644 --- a/sound/soc/codecs/max98088.c +++ b/sound/soc/codecs/max98088.c @@ -474,6 +474,9 @@ static const struct snd_kcontrol_new max98088_snd_controls[] = { max98088_mic2pre_get, max98088_mic2pre_set, max98088_micboost_tlv), + SOC_SINGLE("Noise Gate Threshold", M98088_REG_40_MICAGC_THRESH, + 4, 15, 0), + SOC_SINGLE("INA Volume", M98088_REG_37_LVL_INA, 0, 7, 1), SOC_SINGLE("INB Volume", M98088_REG_38_LVL_INB, 0, 7, 1), @@ -1746,7 +1749,6 @@ MODULE_DEVICE_TABLE(i2c, max98088_i2c_id); static int max98088_i2c_probe(struct i2c_client *i2c) { struct max98088_priv *max98088; - int ret; const struct i2c_device_id *id; max98088 = devm_kzalloc(&i2c->dev, sizeof(struct max98088_priv), @@ -1769,9 +1771,8 @@ static int max98088_i2c_probe(struct i2c_client *i2c) i2c_set_clientdata(i2c, max98088); max98088->pdata = i2c->dev.platform_data; - ret = devm_snd_soc_register_component(&i2c->dev, &soc_component_dev_max98088, + return devm_snd_soc_register_component(&i2c->dev, &soc_component_dev_max98088, &max98088_dai[0], 2); - return ret; } #if defined(CONFIG_OF) diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c index 142083b13ac3be0ae4383b5d48917eb5090fa107..06ed2a938108e02bee965ad3e66b6978b556bbc3 100644 --- a/sound/soc/codecs/max98090.c +++ b/sound/soc/codecs/max98090.c @@ -2615,11 +2615,9 @@ static void max98090_i2c_shutdown(struct i2c_client *i2c) msleep(40); } -static int max98090_i2c_remove(struct i2c_client *client) +static void max98090_i2c_remove(struct i2c_client *client) { max98090_i2c_shutdown(client); - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/max98373-sdw.c b/sound/soc/codecs/max98373-sdw.c index 97b64477dde671632e28d9b2c12984ff8c6db08e..899965b19d12da10ae5e9d725500b57501d96b91 100644 --- a/sound/soc/codecs/max98373-sdw.c +++ b/sound/soc/codecs/max98373-sdw.c @@ -281,6 +281,8 @@ static __maybe_unused int max98373_resume(struct device *dev) msecs_to_jiffies(MAX98373_PROBE_TIMEOUT)); if (!time) { dev_err(dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index 5c08166a8dc62a6116161f2237d17637c80e2d56..7a5260ff8d6b0472e923d80c71a3df6387c9043e 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -161,8 +161,6 @@ static struct reg_default max98390_reg_defaults[] = { {MAX98390_R23FF_GLOBAL_EN, 0x00}, }; -static int max98390_dsm_calibrate(struct snd_soc_component *component); - static int max98390_dai_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) { struct snd_soc_component *component = codec_dai->component; @@ -635,10 +633,48 @@ static int max98390_dsm_calib_get(struct snd_kcontrol *kcontrol, static int max98390_dsm_calib_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { - struct snd_soc_component *component = - snd_soc_kcontrol_component(kcontrol); + struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); + struct max98390_priv *max98390 = snd_soc_component_get_drvdata(component); + struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); + unsigned int rdc, rdc_cal_result, rdc_integer, rdc_factor, temp, val; + + snd_soc_dapm_mutex_lock(dapm); + + regmap_read(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, &val); + if (!val) { + /* Enable the codec for the duration of calibration readout */ + regmap_update_bits(max98390->regmap, MAX98390_R203A_AMP_EN, + MAX98390_AMP_EN_MASK, 1); + regmap_update_bits(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, + MAX98390_GLOBAL_EN_MASK, 1); + } + + regmap_read(max98390->regmap, THERMAL_RDC_RD_BACK_BYTE1, &rdc); + regmap_read(max98390->regmap, THERMAL_RDC_RD_BACK_BYTE0, &rdc_cal_result); + regmap_read(max98390->regmap, MAX98390_MEAS_ADC_CH2_READ, &temp); + + if (!val) { + /* Disable the codec if it was disabled */ + regmap_update_bits(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, + MAX98390_GLOBAL_EN_MASK, 0); + regmap_update_bits(max98390->regmap, MAX98390_R203A_AMP_EN, + MAX98390_AMP_EN_MASK, 0); + } + + snd_soc_dapm_mutex_unlock(dapm); - max98390_dsm_calibrate(component); + rdc_cal_result |= (rdc << 8) & 0x0000FFFF; + if (rdc_cal_result) + max98390->ref_rdc_value = 268435456U / rdc_cal_result; + + max98390->ambient_temp_value = temp * 52 - 1188; + + rdc_integer = rdc_cal_result * 937 / 65536; + rdc_factor = ((rdc_cal_result * 937 * 100) / 65536) - (rdc_integer * 100); + + dev_info(component->dev, + "rdc resistance about %d.%02d ohm, reg=0x%X temp reg=0x%X\n", + rdc_integer, rdc_factor, rdc_cal_result, temp); return 0; } @@ -819,40 +855,6 @@ err: return ret; } -static int max98390_dsm_calibrate(struct snd_soc_component *component) -{ - unsigned int rdc, rdc_cal_result, temp; - unsigned int rdc_integer, rdc_factor; - struct max98390_priv *max98390 = - snd_soc_component_get_drvdata(component); - - regmap_write(max98390->regmap, MAX98390_R203A_AMP_EN, 0x81); - regmap_write(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, 0x01); - - regmap_read(max98390->regmap, - THERMAL_RDC_RD_BACK_BYTE1, &rdc); - regmap_read(max98390->regmap, - THERMAL_RDC_RD_BACK_BYTE0, &rdc_cal_result); - rdc_cal_result |= (rdc << 8) & 0x0000FFFF; - if (rdc_cal_result) - max98390->ref_rdc_value = 268435456U / rdc_cal_result; - - regmap_read(max98390->regmap, MAX98390_MEAS_ADC_CH2_READ, &temp); - max98390->ambient_temp_value = temp * 52 - 1188; - - rdc_integer = rdc_cal_result * 937 / 65536; - rdc_factor = ((rdc_cal_result * 937 * 100) / 65536) - - (rdc_integer * 100); - - dev_info(component->dev, "rdc resistance about %d.%02d ohm, reg=0x%X temp reg=0x%X\n", - rdc_integer, rdc_factor, rdc_cal_result, temp); - - regmap_write(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, 0x00); - regmap_write(max98390->regmap, MAX98390_R203A_AMP_EN, 0x80); - - return 0; -} - static void max98390_init_regs(struct snd_soc_component *component) { struct max98390_priv *max98390 = diff --git a/sound/soc/codecs/max98396.c b/sound/soc/codecs/max98396.c index 364b4b7ee033ffc17a1ae45ae6e70d7439fa37cb..a7b6a23f2cd86c206c608b8e249ba121c65e2783 100644 --- a/sound/soc/codecs/max98396.c +++ b/sound/soc/codecs/max98396.c @@ -1483,6 +1483,87 @@ static int max98396_probe(struct snd_soc_component *component) MAX98396_CLK_MON_AUTO_RESTART_MASK, MAX98396_CLK_MON_AUTO_RESTART_MASK); + regmap_update_bits(max98396->regmap, + MAX98396_R203F_ENABLE_CTRLS, + MAX98396_CTRL_DMON_STUCK_EN_MASK, + max98396->dmon_stuck_enable ? + MAX98396_CTRL_DMON_STUCK_EN_MASK : 0); + + regmap_update_bits(max98396->regmap, + MAX98396_R203F_ENABLE_CTRLS, + MAX98396_CTRL_DMON_MAG_EN_MASK, + max98396->dmon_mag_enable ? + MAX98396_CTRL_DMON_MAG_EN_MASK : 0); + + switch (max98396->dmon_duration) { + case 64: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_DURATION_MASK, 0); + break; + case 256: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_DURATION_MASK, 1); + break; + case 1024: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_DURATION_MASK, 2); + break; + case 4096: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_DURATION_MASK, 3); + break; + default: + dev_err(component->dev, "Invalid DMON duration %d\n", + max98396->dmon_duration); + } + + switch (max98396->dmon_stuck_threshold) { + case 15: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_STUCK_THRESH_MASK, + 0 << MAX98396_DMON_STUCK_THRESH_SHIFT); + break; + case 13: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_STUCK_THRESH_MASK, + 1 << MAX98396_DMON_STUCK_THRESH_SHIFT); + break; + case 22: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_STUCK_THRESH_MASK, + 2 << MAX98396_DMON_STUCK_THRESH_SHIFT); + break; + case 9: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_STUCK_THRESH_MASK, + 3 << MAX98396_DMON_STUCK_THRESH_SHIFT); + break; + default: + dev_err(component->dev, "Invalid DMON stuck threshold %d\n", + max98396->dmon_stuck_threshold); + } + + switch (max98396->dmon_mag_threshold) { + case 2 ... 5: + regmap_update_bits(max98396->regmap, + MAX98396_R2039_DATA_MON_CTRL, + MAX98396_DMON_STUCK_THRESH_MASK, + (5 - max98396->dmon_mag_threshold) + << MAX98396_DMON_MAG_THRESH_SHIFT); + break; + default: + dev_err(component->dev, "Invalid DMON magnitude threshold %d\n", + max98396->dmon_mag_threshold); + } + /* Speaker Amplifier PCM RX Enable by default */ regmap_update_bits(max98396->regmap, MAX98396_R205E_PCM_RX_EN, @@ -1614,6 +1695,27 @@ static void max98396_read_device_property(struct device *dev, max98396->bypass_slot = value & 0xF; else max98396->bypass_slot = 0; + + max98396->dmon_stuck_enable = + device_property_read_bool(dev, "adi,dmon-stuck-enable"); + + if (!device_property_read_u32(dev, "adi,dmon-stuck-threshold-bits", &value)) + max98396->dmon_stuck_threshold = value; + else + max98396->dmon_stuck_threshold = 15; + + max98396->dmon_mag_enable = + device_property_read_bool(dev, "adi,dmon-magnitude-enable"); + + if (!device_property_read_u32(dev, "adi,dmon-magnitude-threshold-bits", &value)) + max98396->dmon_mag_threshold = value; + else + max98396->dmon_mag_threshold = 5; + + if (!device_property_read_u32(dev, "adi,dmon-duration-ms", &value)) + max98396->dmon_duration = value; + else + max98396->dmon_duration = 64; } static void max98396_core_supplies_disable(void *priv) diff --git a/sound/soc/codecs/max98396.h b/sound/soc/codecs/max98396.h index 7278c779989a21f2a4309c97037fde4167a92e63..d396aa3e698b0b843bcb9c4871770fe5a161972e 100644 --- a/sound/soc/codecs/max98396.h +++ b/sound/soc/codecs/max98396.h @@ -212,8 +212,17 @@ #define MAX98396_CLK_MON_AUTO_RESTART_MASK (0x1 << 0) #define MAX98396_CLK_MON_AUTO_RESTART_SHIFT (0) +/* MAX98396_R2039_DATA_MON_CTRL */ +#define MAX98396_DMON_MAG_THRESH_SHIFT (4) +#define MAX98396_DMON_MAG_THRESH_MASK (0x3 << MAX98396_DMON_MAG_THRESH_SHIFT) +#define MAX98396_DMON_STUCK_THRESH_SHIFT (2) +#define MAX98396_DMON_STUCK_THRESH_MASK (0x3 << MAX98396_DMON_STUCK_THRESH_SHIFT) +#define MAX98396_DMON_DURATION_MASK (0x3) + /* MAX98396_R203F_ENABLE_CTRLS */ #define MAX98396_CTRL_CMON_EN_SHIFT (0) +#define MAX98396_CTRL_DMON_STUCK_EN_MASK (0x1 << 1) +#define MAX98396_CTRL_DMON_MAG_EN_MASK (0x1 << 2) /* MAX98396_R2041_PCM_MODE_CFG */ #define MAX98396_PCM_MODE_CFG_FORMAT_MASK (0x7 << 3) @@ -305,6 +314,11 @@ struct max98396_priv { unsigned int i_slot; unsigned int spkfb_slot; unsigned int bypass_slot; + bool dmon_stuck_enable; + unsigned int dmon_stuck_threshold; + bool dmon_mag_enable; + unsigned int dmon_mag_threshold; + unsigned int dmon_duration; bool interleave_mode; bool tdm_mode; int tdm_max_samplerate; diff --git a/sound/soc/codecs/max9860.c b/sound/soc/codecs/max9860.c index 771b3dcd6cc322cd2baa8852a06fb368c755a08a..9611ab1e79e50a121f94a7ec43f9b654605f9f8d 100644 --- a/sound/soc/codecs/max9860.c +++ b/sound/soc/codecs/max9860.c @@ -701,14 +701,13 @@ err_regulator: return ret; } -static int max9860_remove(struct i2c_client *i2c) +static void max9860_remove(struct i2c_client *i2c) { struct device *dev = &i2c->dev; struct max9860_priv *max9860 = dev_get_drvdata(dev); pm_runtime_disable(dev); regulator_disable(max9860->dvddio); - return 0; } static const struct i2c_device_id max9860_i2c_id[] = { diff --git a/sound/soc/codecs/max98927.c b/sound/soc/codecs/max98927.c index 9cce7c0f01424816aa8537233f161df5a48bb33e..331d3e1d735c9feed3ddee7ab9b9d4a0a6fa375e 100644 --- a/sound/soc/codecs/max98927.c +++ b/sound/soc/codecs/max98927.c @@ -934,15 +934,13 @@ static int max98927_i2c_probe(struct i2c_client *i2c) return ret; } -static int max98927_i2c_remove(struct i2c_client *i2c) +static void max98927_i2c_remove(struct i2c_client *i2c) { struct max98927_priv *max98927 = i2c_get_clientdata(i2c); if (max98927->reset_gpio) { gpiod_set_value_cansleep(max98927->reset_gpio, 1); } - - return 0; } static const struct i2c_device_id max98927_i2c_id[] = { diff --git a/sound/soc/codecs/mt6359-accdet.c b/sound/soc/codecs/mt6359-accdet.c index c190628e290566c26e4a56ca702818d7a2f30965..7f624854948c7c696f9df91de04ba893133ed475 100644 --- a/sound/soc/codecs/mt6359-accdet.c +++ b/sound/soc/codecs/mt6359-accdet.c @@ -965,7 +965,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev) mutex_init(&priv->res_lock); priv->accdet_irq = platform_get_irq(pdev, 0); - if (priv->accdet_irq) { + if (priv->accdet_irq >= 0) { ret = devm_request_threaded_irq(&pdev->dev, priv->accdet_irq, NULL, mt6359_accdet_irq, IRQF_TRIGGER_HIGH | IRQF_ONESHOT, @@ -979,7 +979,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev) if (priv->caps & ACCDET_PMIC_EINT0) { priv->accdet_eint0 = platform_get_irq(pdev, 1); - if (priv->accdet_eint0) { + if (priv->accdet_eint0 >= 0) { ret = devm_request_threaded_irq(&pdev->dev, priv->accdet_eint0, NULL, mt6359_accdet_irq, @@ -994,7 +994,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev) } } else if (priv->caps & ACCDET_PMIC_EINT1) { priv->accdet_eint1 = platform_get_irq(pdev, 2); - if (priv->accdet_eint1) { + if (priv->accdet_eint1 >= 0) { ret = devm_request_threaded_irq(&pdev->dev, priv->accdet_eint1, NULL, mt6359_accdet_irq, diff --git a/sound/soc/codecs/mt6660.c b/sound/soc/codecs/mt6660.c index ba11555796ad880099be259b956729c384b5a89f..554c33e8b62f4d57bafbebf572770f286d989500 100644 --- a/sound/soc/codecs/mt6660.c +++ b/sound/soc/codecs/mt6660.c @@ -503,27 +503,30 @@ static int mt6660_i2c_probe(struct i2c_client *client) dev_err(chip->dev, "read chip revision fail\n"); goto probe_fail; } - pm_runtime_set_active(chip->dev); - pm_runtime_enable(chip->dev); ret = devm_snd_soc_register_component(chip->dev, &mt6660_component_driver, &mt6660_codec_dai, 1); + if (!ret) { + pm_runtime_set_active(chip->dev); + pm_runtime_enable(chip->dev); + } + return ret; + probe_fail: _mt6660_chip_power_on(chip, 0); mutex_destroy(&chip->io_lock); return ret; } -static int mt6660_i2c_remove(struct i2c_client *client) +static void mt6660_i2c_remove(struct i2c_client *client) { struct mt6660_chip *chip = i2c_get_clientdata(client); pm_runtime_disable(chip->dev); pm_runtime_set_suspended(chip->dev); mutex_destroy(&chip->io_lock); - return 0; } static int __maybe_unused mt6660_i2c_runtime_suspend(struct device *dev) diff --git a/sound/soc/codecs/nau8540.c b/sound/soc/codecs/nau8540.c index 58f70a02f18aad83654420d8bd843bf4a1e37dc9..0626d5694c2244003bdcee52ed0fc9d7b4e985fe 100644 --- a/sound/soc/codecs/nau8540.c +++ b/sound/soc/codecs/nau8540.c @@ -357,17 +357,32 @@ static const struct snd_soc_dapm_route nau8540_dapm_routes[] = { {"AIFTX", NULL, "Digital CH4 Mux"}, }; -static int nau8540_clock_check(struct nau8540 *nau8540, int rate, int osr) +static const struct nau8540_osr_attr * +nau8540_get_osr(struct nau8540 *nau8540) { + unsigned int osr; + + regmap_read(nau8540->regmap, NAU8540_REG_ADC_SAMPLE_RATE, &osr); + osr &= NAU8540_ADC_OSR_MASK; if (osr >= ARRAY_SIZE(osr_adc_sel)) - return -EINVAL; + return NULL; + return &osr_adc_sel[osr]; +} + +static int nau8540_dai_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct nau8540 *nau8540 = snd_soc_component_get_drvdata(component); + const struct nau8540_osr_attr *osr; - if (rate * osr > CLK_ADC_MAX) { - dev_err(nau8540->dev, "exceed the maximum frequency of CLK_ADC\n"); + osr = nau8540_get_osr(nau8540); + if (!osr || !osr->osr) return -EINVAL; - } - return 0; + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, CLK_ADC_MAX / osr->osr); } static int nau8540_hw_params(struct snd_pcm_substream *substream, @@ -375,7 +390,8 @@ static int nau8540_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct nau8540 *nau8540 = snd_soc_component_get_drvdata(component); - unsigned int val_len = 0, osr; + unsigned int val_len = 0; + const struct nau8540_osr_attr *osr; /* CLK_ADC = OSR * FS * ADC clock frequency is defined as Over Sampling Rate (OSR) @@ -383,13 +399,14 @@ static int nau8540_hw_params(struct snd_pcm_substream *substream, * values must be selected such that the maximum frequency is less * than 6.144 MHz. */ - regmap_read(nau8540->regmap, NAU8540_REG_ADC_SAMPLE_RATE, &osr); - osr &= NAU8540_ADC_OSR_MASK; - if (nau8540_clock_check(nau8540, params_rate(params), osr)) + osr = nau8540_get_osr(nau8540); + if (!osr || !osr->osr) + return -EINVAL; + if (params_rate(params) * osr->osr > CLK_ADC_MAX) return -EINVAL; regmap_update_bits(nau8540->regmap, NAU8540_REG_CLOCK_SRC, NAU8540_CLK_ADC_SRC_MASK, - osr_adc_sel[osr].clk_src << NAU8540_CLK_ADC_SRC_SFT); + osr->clk_src << NAU8540_CLK_ADC_SRC_SFT); switch (params_width(params)) { case 16: @@ -515,6 +532,7 @@ static int nau8540_set_tdm_slot(struct snd_soc_dai *dai, static const struct snd_soc_dai_ops nau8540_dai_ops = { + .startup = nau8540_dai_startup, .hw_params = nau8540_hw_params, .set_fmt = nau8540_set_fmt, .set_tdm_slot = nau8540_set_tdm_slot, diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index 2d21339932e651bdf24bf3b6a6ebf6bef5bfde40..4a72b94e8410424023fdef82365e413d0dc256f7 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -670,28 +670,40 @@ static const struct snd_soc_dapm_route nau8821_dapm_routes[] = { {"HPOR", NULL, "Class G"}, }; -static int nau8821_clock_check(struct nau8821 *nau8821, - int stream, int rate, int osr) +static const struct nau8821_osr_attr * +nau8821_get_osr(struct nau8821 *nau8821, int stream) { - int osrate = 0; + unsigned int osr; if (stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_read(nau8821->regmap, NAU8821_R2C_DAC_CTRL1, &osr); + osr &= NAU8821_DAC_OVERSAMPLE_MASK; if (osr >= ARRAY_SIZE(osr_dac_sel)) - return -EINVAL; - osrate = osr_dac_sel[osr].osr; + return NULL; + return &osr_dac_sel[osr]; } else { + regmap_read(nau8821->regmap, NAU8821_R2B_ADC_RATE, &osr); + osr &= NAU8821_ADC_SYNC_DOWN_MASK; if (osr >= ARRAY_SIZE(osr_adc_sel)) - return -EINVAL; - osrate = osr_adc_sel[osr].osr; + return NULL; + return &osr_adc_sel[osr]; } +} - if (!osrate || rate * osrate > CLK_DA_AD_MAX) { - dev_err(nau8821->dev, - "exceed the maximum frequency of CLK_ADC or CLK_DAC"); +static int nau8821_dai_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct nau8821 *nau8821 = snd_soc_component_get_drvdata(component); + const struct nau8821_osr_attr *osr; + + osr = nau8821_get_osr(nau8821, substream->stream); + if (!osr || !osr->osr) return -EINVAL; - } - return 0; + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, CLK_DA_AD_MAX / osr->osr); } static int nau8821_hw_params(struct snd_pcm_substream *substream, @@ -699,7 +711,8 @@ static int nau8821_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct nau8821 *nau8821 = snd_soc_component_get_drvdata(component); - unsigned int val_len = 0, osr, ctrl_val, bclk_fs, clk_div; + unsigned int val_len = 0, ctrl_val, bclk_fs, clk_div; + const struct nau8821_osr_attr *osr; nau8821->fs = params_rate(params); /* CLK_DAC or CLK_ADC = OSR * FS @@ -708,27 +721,19 @@ static int nau8821_hw_params(struct snd_pcm_substream *substream, * values must be selected such that the maximum frequency is less * than 6.144 MHz. */ - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - regmap_read(nau8821->regmap, NAU8821_R2C_DAC_CTRL1, &osr); - osr &= NAU8821_DAC_OVERSAMPLE_MASK; - if (nau8821_clock_check(nau8821, substream->stream, - nau8821->fs, osr)) { - return -EINVAL; - } + osr = nau8821_get_osr(nau8821, substream->stream); + if (!osr || !osr->osr) + return -EINVAL; + if (nau8821->fs * osr->osr > CLK_DA_AD_MAX) + return -EINVAL; + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) regmap_update_bits(nau8821->regmap, NAU8821_R03_CLK_DIVIDER, NAU8821_CLK_DAC_SRC_MASK, - osr_dac_sel[osr].clk_src << NAU8821_CLK_DAC_SRC_SFT); - } else { - regmap_read(nau8821->regmap, NAU8821_R2B_ADC_RATE, &osr); - osr &= NAU8821_ADC_SYNC_DOWN_MASK; - if (nau8821_clock_check(nau8821, substream->stream, - nau8821->fs, osr)) { - return -EINVAL; - } + osr->clk_src << NAU8821_CLK_DAC_SRC_SFT); + else regmap_update_bits(nau8821->regmap, NAU8821_R03_CLK_DIVIDER, NAU8821_CLK_ADC_SRC_MASK, - osr_adc_sel[osr].clk_src << NAU8821_CLK_ADC_SRC_SFT); - } + osr->clk_src << NAU8821_CLK_ADC_SRC_SFT); /* make BCLK and LRC divde configuration if the codec as master. */ regmap_read(nau8821->regmap, NAU8821_R1D_I2S_PCM_CTRL2, &ctrl_val); @@ -843,6 +848,7 @@ static int nau8821_digital_mute(struct snd_soc_dai *dai, int mute, } static const struct snd_soc_dai_ops nau8821_dai_ops = { + .startup = nau8821_dai_startup, .hw_params = nau8821_hw_params, .set_fmt = nau8821_set_dai_fmt, .mute_stream = nau8821_digital_mute, diff --git a/sound/soc/codecs/nau8824.c b/sound/soc/codecs/nau8824.c index ad54d70f7d8e75d4eda4f527548852d60a132363..4f19fd9b65d119d7c1eb148bc85056f88e2867fd 100644 --- a/sound/soc/codecs/nau8824.c +++ b/sound/soc/codecs/nau8824.c @@ -901,7 +901,10 @@ static void nau8824_jdet_work(struct work_struct *work) NAU8824_IRQ_KEY_RELEASE_DIS | NAU8824_IRQ_KEY_SHORT_PRESS_DIS, 0); - nau8824_sema_release(nau8824); + if (nau8824->resume_lock) { + nau8824_sema_release(nau8824); + nau8824->resume_lock = false; + } } static void nau8824_setup_auto_irq(struct nau8824 *nau8824) @@ -966,7 +969,10 @@ static irqreturn_t nau8824_interrupt(int irq, void *data) /* release semaphore held after resume, * and cancel jack detection */ - nau8824_sema_release(nau8824); + if (nau8824->resume_lock) { + nau8824_sema_release(nau8824); + nau8824->resume_lock = false; + } cancel_work_sync(&nau8824->jdet_work); } else if (active_irq & NAU8824_KEY_SHORT_PRESS_IRQ) { int key_status, button_pressed; @@ -1014,27 +1020,42 @@ static irqreturn_t nau8824_interrupt(int irq, void *data) return IRQ_HANDLED; } -static int nau8824_clock_check(struct nau8824 *nau8824, - int stream, int rate, int osr) +static const struct nau8824_osr_attr * +nau8824_get_osr(struct nau8824 *nau8824, int stream) { - int osrate; + unsigned int osr; if (stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_read(nau8824->regmap, + NAU8824_REG_DAC_FILTER_CTRL_1, &osr); + osr &= NAU8824_DAC_OVERSAMPLE_MASK; if (osr >= ARRAY_SIZE(osr_dac_sel)) - return -EINVAL; - osrate = osr_dac_sel[osr].osr; + return NULL; + return &osr_dac_sel[osr]; } else { + regmap_read(nau8824->regmap, + NAU8824_REG_ADC_FILTER_CTRL, &osr); + osr &= NAU8824_ADC_SYNC_DOWN_MASK; if (osr >= ARRAY_SIZE(osr_adc_sel)) - return -EINVAL; - osrate = osr_adc_sel[osr].osr; + return NULL; + return &osr_adc_sel[osr]; } +} + +static int nau8824_dai_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct nau8824 *nau8824 = snd_soc_component_get_drvdata(component); + const struct nau8824_osr_attr *osr; - if (!osrate || rate * osr > CLK_DA_AD_MAX) { - dev_err(nau8824->dev, "exceed the maximum frequency of CLK_ADC or CLK_DAC\n"); + osr = nau8824_get_osr(nau8824, substream->stream); + if (!osr || !osr->osr) return -EINVAL; - } - return 0; + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, CLK_DA_AD_MAX / osr->osr); } static int nau8824_hw_params(struct snd_pcm_substream *substream, @@ -1042,7 +1063,9 @@ static int nau8824_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct nau8824 *nau8824 = snd_soc_component_get_drvdata(component); - unsigned int val_len = 0, osr, ctrl_val, bclk_fs, bclk_div; + unsigned int val_len = 0, ctrl_val, bclk_fs, bclk_div; + const struct nau8824_osr_attr *osr; + int err = -EINVAL; nau8824_sema_acquire(nau8824, HZ); @@ -1053,27 +1076,19 @@ static int nau8824_hw_params(struct snd_pcm_substream *substream, * than 6.144 MHz. */ nau8824->fs = params_rate(params); - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - regmap_read(nau8824->regmap, - NAU8824_REG_DAC_FILTER_CTRL_1, &osr); - osr &= NAU8824_DAC_OVERSAMPLE_MASK; - if (nau8824_clock_check(nau8824, substream->stream, - nau8824->fs, osr)) - return -EINVAL; + osr = nau8824_get_osr(nau8824, substream->stream); + if (!osr || !osr->osr) + goto error; + if (nau8824->fs * osr->osr > CLK_DA_AD_MAX) + goto error; + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) regmap_update_bits(nau8824->regmap, NAU8824_REG_CLK_DIVIDER, NAU8824_CLK_DAC_SRC_MASK, - osr_dac_sel[osr].clk_src << NAU8824_CLK_DAC_SRC_SFT); - } else { - regmap_read(nau8824->regmap, - NAU8824_REG_ADC_FILTER_CTRL, &osr); - osr &= NAU8824_ADC_SYNC_DOWN_MASK; - if (nau8824_clock_check(nau8824, substream->stream, - nau8824->fs, osr)) - return -EINVAL; + osr->clk_src << NAU8824_CLK_DAC_SRC_SFT); + else regmap_update_bits(nau8824->regmap, NAU8824_REG_CLK_DIVIDER, NAU8824_CLK_ADC_SRC_MASK, - osr_adc_sel[osr].clk_src << NAU8824_CLK_ADC_SRC_SFT); - } + osr->clk_src << NAU8824_CLK_ADC_SRC_SFT); /* make BCLK and LRC divde configuration if the codec as master. */ regmap_read(nau8824->regmap, @@ -1090,7 +1105,7 @@ static int nau8824_hw_params(struct snd_pcm_substream *substream, else if (bclk_fs <= 256) bclk_div = 0; else - return -EINVAL; + goto error; regmap_update_bits(nau8824->regmap, NAU8824_REG_PORT0_I2S_PCM_CTRL_2, NAU8824_I2S_LRC_DIV_MASK | NAU8824_I2S_BLK_DIV_MASK, @@ -1111,15 +1126,17 @@ static int nau8824_hw_params(struct snd_pcm_substream *substream, val_len |= NAU8824_I2S_DL_32; break; default: - return -EINVAL; + goto error; } regmap_update_bits(nau8824->regmap, NAU8824_REG_PORT0_I2S_PCM_CTRL_1, NAU8824_I2S_DL_MASK, val_len); + err = 0; + error: nau8824_sema_release(nau8824); - return 0; + return err; } static int nau8824_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) @@ -1128,8 +1145,6 @@ static int nau8824_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) struct nau8824 *nau8824 = snd_soc_component_get_drvdata(component); unsigned int ctrl1_val = 0, ctrl2_val = 0; - nau8824_sema_acquire(nau8824, HZ); - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { case SND_SOC_DAIFMT_CBM_CFM: ctrl2_val |= NAU8824_I2S_MS_MASTER; @@ -1171,6 +1186,8 @@ static int nau8824_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) return -EINVAL; } + nau8824_sema_acquire(nau8824, HZ); + regmap_update_bits(nau8824->regmap, NAU8824_REG_PORT0_I2S_PCM_CTRL_1, NAU8824_I2S_DF_MASK | NAU8824_I2S_BP_MASK | NAU8824_I2S_PCMB_EN, ctrl1_val); @@ -1513,6 +1530,7 @@ static int __maybe_unused nau8824_suspend(struct snd_soc_component *component) static int __maybe_unused nau8824_resume(struct snd_soc_component *component) { struct nau8824 *nau8824 = snd_soc_component_get_drvdata(component); + int ret; regcache_cache_only(nau8824->regmap, false); regcache_sync(nau8824->regmap); @@ -1520,7 +1538,10 @@ static int __maybe_unused nau8824_resume(struct snd_soc_component *component) /* Hold semaphore to postpone playback happening * until jack detection done. */ - nau8824_sema_acquire(nau8824, 0); + nau8824->resume_lock = true; + ret = nau8824_sema_acquire(nau8824, 0); + if (ret) + nau8824->resume_lock = false; enable_irq(nau8824->irq); } @@ -1547,6 +1568,7 @@ static const struct snd_soc_component_driver nau8824_component_driver = { }; static const struct snd_soc_dai_ops nau8824_dai_ops = { + .startup = nau8824_dai_startup, .hw_params = nau8824_hw_params, .set_fmt = nau8824_set_fmt, .set_tdm_slot = nau8824_set_tdm_slot, @@ -1928,6 +1950,7 @@ static int nau8824_i2c_probe(struct i2c_client *i2c) nau8824->regmap = devm_regmap_init_i2c(i2c, &nau8824_regmap_config); if (IS_ERR(nau8824->regmap)) return PTR_ERR(nau8824->regmap); + nau8824->resume_lock = false; nau8824->dev = dev; nau8824->irq = i2c->irq; sema_init(&nau8824->jd_sem, 1); diff --git a/sound/soc/codecs/nau8824.h b/sound/soc/codecs/nau8824.h index de4bae8281d01d6bb030e247315bd5dcf48a00ef..5fcfc43dfc85587c3353b566a7f8beb12838d693 100644 --- a/sound/soc/codecs/nau8824.h +++ b/sound/soc/codecs/nau8824.h @@ -436,6 +436,7 @@ struct nau8824 { struct semaphore jd_sem; int fs; int irq; + int resume_lock; int micbias_voltage; int vref_impedance; int jkdet_polarity; diff --git a/sound/soc/codecs/nau8825.c b/sound/soc/codecs/nau8825.c index 54ef7b0fa87860ec7d6199f884dd7a5aa255492c..3eac7c92df887cad3ca02b05fc91e47179100e11 100644 --- a/sound/soc/codecs/nau8825.c +++ b/sound/soc/codecs/nau8825.c @@ -1247,27 +1247,42 @@ static const struct snd_soc_dapm_route nau8825_dapm_routes[] = { {"HPOR", NULL, "Class G"}, }; -static int nau8825_clock_check(struct nau8825 *nau8825, - int stream, int rate, int osr) +static const struct nau8825_osr_attr * +nau8825_get_osr(struct nau8825 *nau8825, int stream) { - int osrate; + unsigned int osr; if (stream == SNDRV_PCM_STREAM_PLAYBACK) { + regmap_read(nau8825->regmap, + NAU8825_REG_DAC_CTRL1, &osr); + osr &= NAU8825_DAC_OVERSAMPLE_MASK; if (osr >= ARRAY_SIZE(osr_dac_sel)) - return -EINVAL; - osrate = osr_dac_sel[osr].osr; + return NULL; + return &osr_dac_sel[osr]; } else { + regmap_read(nau8825->regmap, + NAU8825_REG_ADC_RATE, &osr); + osr &= NAU8825_ADC_SYNC_DOWN_MASK; if (osr >= ARRAY_SIZE(osr_adc_sel)) - return -EINVAL; - osrate = osr_adc_sel[osr].osr; + return NULL; + return &osr_adc_sel[osr]; } +} + +static int nau8825_dai_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct nau8825 *nau8825 = snd_soc_component_get_drvdata(component); + const struct nau8825_osr_attr *osr; - if (!osrate || rate * osr > CLK_DA_AD_MAX) { - dev_err(nau8825->dev, "exceed the maximum frequency of CLK_ADC or CLK_DAC\n"); + osr = nau8825_get_osr(nau8825, substream->stream); + if (!osr || !osr->osr) return -EINVAL; - } - return 0; + return snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, CLK_DA_AD_MAX / osr->osr); } static int nau8825_hw_params(struct snd_pcm_substream *substream, @@ -1276,7 +1291,9 @@ static int nau8825_hw_params(struct snd_pcm_substream *substream, { struct snd_soc_component *component = dai->component; struct nau8825 *nau8825 = snd_soc_component_get_drvdata(component); - unsigned int val_len = 0, osr, ctrl_val, bclk_fs, bclk_div; + unsigned int val_len = 0, ctrl_val, bclk_fs, bclk_div; + const struct nau8825_osr_attr *osr; + int err = -EINVAL; nau8825_sema_acquire(nau8825, 3 * HZ); @@ -1286,29 +1303,19 @@ static int nau8825_hw_params(struct snd_pcm_substream *substream, * values must be selected such that the maximum frequency is less * than 6.144 MHz. */ - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - regmap_read(nau8825->regmap, NAU8825_REG_DAC_CTRL1, &osr); - osr &= NAU8825_DAC_OVERSAMPLE_MASK; - if (nau8825_clock_check(nau8825, substream->stream, - params_rate(params), osr)) { - nau8825_sema_release(nau8825); - return -EINVAL; - } + osr = nau8825_get_osr(nau8825, substream->stream); + if (!osr || !osr->osr) + goto error; + if (params_rate(params) * osr->osr > CLK_DA_AD_MAX) + goto error; + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) regmap_update_bits(nau8825->regmap, NAU8825_REG_CLK_DIVIDER, NAU8825_CLK_DAC_SRC_MASK, - osr_dac_sel[osr].clk_src << NAU8825_CLK_DAC_SRC_SFT); - } else { - regmap_read(nau8825->regmap, NAU8825_REG_ADC_RATE, &osr); - osr &= NAU8825_ADC_SYNC_DOWN_MASK; - if (nau8825_clock_check(nau8825, substream->stream, - params_rate(params), osr)) { - nau8825_sema_release(nau8825); - return -EINVAL; - } + osr->clk_src << NAU8825_CLK_DAC_SRC_SFT); + else regmap_update_bits(nau8825->regmap, NAU8825_REG_CLK_DIVIDER, NAU8825_CLK_ADC_SRC_MASK, - osr_adc_sel[osr].clk_src << NAU8825_CLK_ADC_SRC_SFT); - } + osr->clk_src << NAU8825_CLK_ADC_SRC_SFT); /* make BCLK and LRC divde configuration if the codec as master. */ regmap_read(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL2, &ctrl_val); @@ -1321,10 +1328,8 @@ static int nau8825_hw_params(struct snd_pcm_substream *substream, bclk_div = 1; else if (bclk_fs <= 128) bclk_div = 0; - else { - nau8825_sema_release(nau8825); - return -EINVAL; - } + else + goto error; regmap_update_bits(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL2, NAU8825_I2S_LRC_DIV_MASK | NAU8825_I2S_BLK_DIV_MASK, ((bclk_div + 1) << NAU8825_I2S_LRC_DIV_SFT) | bclk_div); @@ -1344,17 +1349,18 @@ static int nau8825_hw_params(struct snd_pcm_substream *substream, val_len |= NAU8825_I2S_DL_32; break; default: - nau8825_sema_release(nau8825); - return -EINVAL; + goto error; } regmap_update_bits(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL1, NAU8825_I2S_DL_MASK, val_len); + err = 0; + error: /* Release the semaphore. */ nau8825_sema_release(nau8825); - return 0; + return err; } static int nau8825_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) @@ -1419,9 +1425,107 @@ static int nau8825_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt) return 0; } +/** + * nau8825_set_tdm_slot - configure DAI TDM. + * @dai: DAI + * @tx_mask: bitmask representing active TX slots. + * @rx_mask: bitmask representing active RX slots. + * @slots: Number of slots in use. + * @slot_width: Width in bits for each slot. + * + * Configures a DAI for TDM operation. Support TDM 4/8 slots. + * The limitation is DAC and ADC need shift 4 slots at 8 slots mode. + */ +static int nau8825_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, + unsigned int rx_mask, int slots, int slot_width) +{ + struct snd_soc_component *component = dai->component; + struct nau8825 *nau8825 = snd_soc_component_get_drvdata(component); + unsigned int ctrl_val = 0, ctrl_offset = 0, value = 0, dac_s, adc_s; + + if (slots != 4 && slots != 8) { + dev_err(nau8825->dev, "Only support 4 or 8 slots!\n"); + return -EINVAL; + } + + /* The driver is limited to 1-channel for ADC, and 2-channel for DAC on TDM mode */ + if (hweight_long((unsigned long) tx_mask) != 1 || + hweight_long((unsigned long) rx_mask) != 2) { + dev_err(nau8825->dev, + "The limitation is 1-channel for ADC, and 2-channel for DAC on TDM mode.\n"); + return -EINVAL; + } + + if (((tx_mask & 0xf) && (tx_mask & 0xf0)) || + ((rx_mask & 0xf) && (rx_mask & 0xf0)) || + ((tx_mask & 0xf) && (rx_mask & 0xf0)) || + ((rx_mask & 0xf) && (tx_mask & 0xf0))) { + dev_err(nau8825->dev, + "Slot assignment of DAC and ADC need to set same interval.\n"); + return -EINVAL; + } + + /* The offset of fixed 4 slots for 8 slots support */ + if (rx_mask & 0xf0) { + regmap_update_bits(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL2, + NAU8825_I2S_PCM_TS_EN_MASK, NAU8825_I2S_PCM_TS_EN); + regmap_read(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL1, &value); + ctrl_val |= NAU8825_TDM_OFFSET_EN; + ctrl_offset = 4 * slot_width; + if (!(value & NAU8825_I2S_PCMB_MASK)) + ctrl_offset += 1; + dac_s = (rx_mask & 0xf0) >> 4; + adc_s = fls((tx_mask & 0xf0) >> 4); + } else { + dac_s = rx_mask & 0xf; + adc_s = fls(tx_mask & 0xf); + } + + ctrl_val |= NAU8825_TDM_MODE; + + switch (dac_s) { + case 0x3: + ctrl_val |= 1 << NAU8825_TDM_DACR_RX_SFT; + break; + case 0x5: + ctrl_val |= 2 << NAU8825_TDM_DACR_RX_SFT; + break; + case 0x6: + ctrl_val |= 1 << NAU8825_TDM_DACL_RX_SFT; + ctrl_val |= 2 << NAU8825_TDM_DACR_RX_SFT; + break; + case 0x9: + ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT; + break; + case 0xa: + ctrl_val |= 1 << NAU8825_TDM_DACL_RX_SFT; + ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT; + break; + case 0xc: + ctrl_val |= 2 << NAU8825_TDM_DACL_RX_SFT; + ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT; + break; + default: + return -EINVAL; + } + + ctrl_val |= adc_s - 1; + + regmap_update_bits(nau8825->regmap, NAU8825_REG_TDM_CTRL, + NAU8825_TDM_MODE | NAU8825_TDM_OFFSET_EN | + NAU8825_TDM_DACL_RX_MASK | NAU8825_TDM_DACR_RX_MASK | + NAU8825_TDM_TX_MASK, ctrl_val); + regmap_update_bits(nau8825->regmap, NAU8825_REG_LEFT_TIME_SLOT, + NAU8825_TSLOT_L0_MASK, ctrl_offset); + + return 0; +} + static const struct snd_soc_dai_ops nau8825_dai_ops = { + .startup = nau8825_dai_startup, .hw_params = nau8825_hw_params, .set_fmt = nau8825_set_dai_fmt, + .set_tdm_slot = nau8825_set_tdm_slot, }; #define NAU8825_RATES SNDRV_PCM_RATE_8000_192000 @@ -1976,6 +2080,10 @@ static void nau8825_init_regs(struct nau8825 *nau8825) /* Disable short Frame Sync detection logic */ regmap_update_bits(regmap, NAU8825_REG_LEFT_TIME_SLOT, NAU8825_DIS_FS_SHORT_DET, NAU8825_DIS_FS_SHORT_DET); + /* ADCDAT IO drive strength control */ + regmap_update_bits(regmap, NAU8825_REG_CHARGE_PUMP, + NAU8825_ADCOUT_DS_MASK, + nau8825->adcout_ds << NAU8825_ADCOUT_DS_SFT); } static const struct regmap_config nau8825_regmap_config = { @@ -2514,6 +2622,7 @@ static void nau8825_print_device_properties(struct nau8825 *nau8825) nau8825->jack_eject_debounce); dev_dbg(dev, "crosstalk-enable: %d\n", nau8825->xtalk_enable); + dev_dbg(dev, "adcout-drive-strong: %d\n", nau8825->adcout_ds); } static int nau8825_read_device_properties(struct device *dev, @@ -2580,6 +2689,7 @@ static int nau8825_read_device_properties(struct device *dev, nau8825->jack_eject_debounce = 0; nau8825->xtalk_enable = device_property_read_bool(dev, "nuvoton,crosstalk-enable"); + nau8825->adcout_ds = device_property_read_bool(dev, "nuvoton,adcout-drive-strong"); nau8825->mclk = devm_clk_get(dev, "mclk"); if (PTR_ERR(nau8825->mclk) == -EPROBE_DEFER) { @@ -2668,10 +2778,8 @@ static int nau8825_i2c_probe(struct i2c_client *i2c) &nau8825_dai, 1); } -static int nau8825_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void nau8825_i2c_remove(struct i2c_client *client) +{} static const struct i2c_device_id nau8825_i2c_ids[] = { { "nau8825", 0 }, diff --git a/sound/soc/codecs/nau8825.h b/sound/soc/codecs/nau8825.h index 887bbff03ec6682663bd037522db4fc8a790a25d..d84191a7beb21b09367ab1a2bb7073e1a7712690 100644 --- a/sound/soc/codecs/nau8825.h +++ b/sound/soc/codecs/nau8825.h @@ -225,6 +225,15 @@ #define NAU8825_JKDET_PULL_EN (1 << 9) /* 0 - enable pull, 1 - disable */ #define NAU8825_JKDET_OUTPUT_EN (1 << 8) /* 0 - enable input, 1 - enable output */ +/* TDM_CTRL (0x1b) */ +#define NAU8825_TDM_MODE (0x1 << 15) +#define NAU8825_TDM_OFFSET_EN (0x1 << 14) +#define NAU8825_TDM_DACL_RX_SFT 6 +#define NAU8825_TDM_DACL_RX_MASK (0x3 << NAU8825_TDM_DACL_RX_SFT) +#define NAU8825_TDM_DACR_RX_SFT 4 +#define NAU8825_TDM_DACR_RX_MASK (0x3 << NAU8825_TDM_DACR_RX_SFT) +#define NAU8825_TDM_TX_MASK 0x3 + /* I2S_PCM_CTRL1 (0x1c) */ #define NAU8825_I2S_BP_SFT 7 #define NAU8825_I2S_BP_MASK (1 << NAU8825_I2S_BP_SFT) @@ -249,6 +258,9 @@ #define NAU8825_I2S_TRISTATE (1 << 15) /* 0 - normal mode, 1 - Hi-Z output */ #define NAU8825_I2S_LRC_DIV_SFT 12 #define NAU8825_I2S_LRC_DIV_MASK (0x3 << NAU8825_I2S_LRC_DIV_SFT) +#define NAU8825_I2S_PCM_TS_EN_SFT 10 +#define NAU8825_I2S_PCM_TS_EN_MASK (1 << NAU8825_I2S_PCM_TS_EN_SFT) +#define NAU8825_I2S_PCM_TS_EN (1 << NAU8825_I2S_PCM_TS_EN_SFT) #define NAU8825_I2S_MS_SFT 3 #define NAU8825_I2S_MS_MASK (1 << NAU8825_I2S_MS_SFT) #define NAU8825_I2S_MS_MASTER (1 << NAU8825_I2S_MS_SFT) @@ -259,6 +271,8 @@ #define NAU8825_FS_ERR_CMP_SEL_SFT 14 #define NAU8825_FS_ERR_CMP_SEL_MASK (0x3 << NAU8825_FS_ERR_CMP_SEL_SFT) #define NAU8825_DIS_FS_SHORT_DET (1 << 13) +#define NAU8825_TSLOT_L0_MASK 0x3ff +#define NAU8825_TSLOT_R0_MASK 0x3ff /* BIQ_CTRL (0x20) */ #define NAU8825_BIQ_WRT_SFT 4 @@ -418,6 +432,8 @@ #define NAU8825_POWERUP_HP_DRV_L (1 << 0) /* CHARGE_PUMP (0x80) */ +#define NAU8825_ADCOUT_DS_SFT 12 +#define NAU8825_ADCOUT_DS_MASK (1 << NAU8825_ADCOUT_DS_SFT) #define NAU8825_JAMNODCLOW (1 << 10) #define NAU8825_POWER_DOWN_DACR (1 << 9) #define NAU8825_POWER_DOWN_DACL (1 << 8) @@ -477,6 +493,7 @@ struct nau8825 { int imp_rms[NAU8825_XTALK_IMM]; int xtalk_enable; bool xtalk_baktab_initialized; /* True if initialized. */ + bool adcout_ds; }; int nau8825_enable_jack_detect(struct snd_soc_component *component, diff --git a/sound/soc/codecs/pcm1789-i2c.c b/sound/soc/codecs/pcm1789-i2c.c index 1d2f7480a6e4410562122a8d5362fafc49465f0c..fafe0dcbe4eaf23a1643da2d62d881293a50cf15 100644 --- a/sound/soc/codecs/pcm1789-i2c.c +++ b/sound/soc/codecs/pcm1789-i2c.c @@ -27,11 +27,9 @@ static int pcm1789_i2c_probe(struct i2c_client *client) return pcm1789_common_init(&client->dev, regmap); } -static int pcm1789_i2c_remove(struct i2c_client *client) +static void pcm1789_i2c_remove(struct i2c_client *client) { pcm1789_common_exit(&client->dev); - - return 0; } #ifdef CONFIG_OF diff --git a/sound/soc/codecs/pcm3168a-i2c.c b/sound/soc/codecs/pcm3168a-i2c.c index c0fa0dc80e8fad6ebe620b028ce2b3fe8a9f1005..a0eec82e9872a11088fcf18810cf59099d67d86b 100644 --- a/sound/soc/codecs/pcm3168a-i2c.c +++ b/sound/soc/codecs/pcm3168a-i2c.c @@ -26,11 +26,9 @@ static int pcm3168a_i2c_probe(struct i2c_client *i2c) return pcm3168a_probe(&i2c->dev, regmap); } -static int pcm3168a_i2c_remove(struct i2c_client *i2c) +static void pcm3168a_i2c_remove(struct i2c_client *i2c) { pcm3168a_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id pcm3168a_i2c_id[] = { diff --git a/sound/soc/codecs/pcm512x-i2c.c b/sound/soc/codecs/pcm512x-i2c.c index 81754e141a557e4a59b499b402dd380d21bdc6c3..9dfbbe8f4a0b2efd9664d36d3005fa237bca2ba8 100644 --- a/sound/soc/codecs/pcm512x-i2c.c +++ b/sound/soc/codecs/pcm512x-i2c.c @@ -29,10 +29,9 @@ static int pcm512x_i2c_probe(struct i2c_client *i2c) return pcm512x_probe(&i2c->dev, regmap); } -static int pcm512x_i2c_remove(struct i2c_client *i2c) +static void pcm512x_i2c_remove(struct i2c_client *i2c) { pcm512x_remove(&i2c->dev); - return 0; } static const struct i2c_device_id pcm512x_i2c_id[] = { diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c index 0be6e72ff5a9a97880e2c2adab2aa286298406fb..5c29416aa781c457cd7f539df9cf0dde57440080 100644 --- a/sound/soc/codecs/rt1308-sdw.c +++ b/sound/soc/codecs/rt1308-sdw.c @@ -749,6 +749,8 @@ static int __maybe_unused rt1308_dev_resume(struct device *dev) msecs_to_jiffies(RT1308_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c index e53396606a1c6f8af317dbf2904427137027f9ff..ed0a114363621026cafdf1e4f9841ff89507a4e2 100644 --- a/sound/soc/codecs/rt1316-sdw.c +++ b/sound/soc/codecs/rt1316-sdw.c @@ -734,6 +734,8 @@ static int __maybe_unused rt1316_dev_resume(struct device *dev) msecs_to_jiffies(RT1316_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt274.c b/sound/soc/codecs/rt274.c index f2c50b11e4d0c734a462fc9a2bf729e0454f1e56..4667bf7561b13ac4db7746872174b539ea42585a 100644 --- a/sound/soc/codecs/rt274.c +++ b/sound/soc/codecs/rt274.c @@ -1204,14 +1204,12 @@ static int rt274_i2c_probe(struct i2c_client *i2c) return ret; } -static int rt274_i2c_remove(struct i2c_client *i2c) +static void rt274_i2c_remove(struct i2c_client *i2c) { struct rt274_priv *rt274 = i2c_get_clientdata(i2c); if (i2c->irq) free_irq(i2c->irq, rt274); - - return 0; } diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c index c4f7c4c2d7939b15bde36ffc04c4681b6335f41f..ceb56647e369e31f21e5000dd95e5dae55927f7c 100644 --- a/sound/soc/codecs/rt286.c +++ b/sound/soc/codecs/rt286.c @@ -1249,14 +1249,12 @@ static int rt286_i2c_probe(struct i2c_client *i2c) return ret; } -static int rt286_i2c_remove(struct i2c_client *i2c) +static void rt286_i2c_remove(struct i2c_client *i2c) { struct rt286_priv *rt286 = i2c_get_clientdata(i2c); if (i2c->irq) free_irq(i2c->irq, rt286); - - return 0; } diff --git a/sound/soc/codecs/rt298.c b/sound/soc/codecs/rt298.c index b0b53d4f07df91b565c43cc470af063fe1aa8479..a2ce52dafea84e1da40a43fa0d2c492d3c22f196 100644 --- a/sound/soc/codecs/rt298.c +++ b/sound/soc/codecs/rt298.c @@ -1290,14 +1290,12 @@ static int rt298_i2c_probe(struct i2c_client *i2c) return ret; } -static int rt298_i2c_remove(struct i2c_client *i2c) +static void rt298_i2c_remove(struct i2c_client *i2c) { struct rt298_priv *rt298 = i2c_get_clientdata(i2c); if (i2c->irq) free_irq(i2c->irq, rt298); - - return 0; } diff --git a/sound/soc/codecs/rt5616.c b/sound/soc/codecs/rt5616.c index 970d6c4a358e01c26e150e3d7f7594f7d6899ecc..948abde10463e50c92f376b1cb3d3ddbeb896b97 100644 --- a/sound/soc/codecs/rt5616.c +++ b/sound/soc/codecs/rt5616.c @@ -1388,10 +1388,8 @@ static int rt5616_i2c_probe(struct i2c_client *i2c) rt5616_dai, ARRAY_SIZE(rt5616_dai)); } -static int rt5616_i2c_remove(struct i2c_client *i2c) -{ - return 0; -} +static void rt5616_i2c_remove(struct i2c_client *i2c) +{} static void rt5616_i2c_shutdown(struct i2c_client *client) { diff --git a/sound/soc/codecs/rt5631.c b/sound/soc/codecs/rt5631.c index 957f6b19beec931ca6d084f3860d0fc6c7adb6a3..55c232413e2b3e9f478010e1c16cfd834b7880c1 100644 --- a/sound/soc/codecs/rt5631.c +++ b/sound/soc/codecs/rt5631.c @@ -1720,10 +1720,8 @@ static int rt5631_i2c_probe(struct i2c_client *i2c) return ret; } -static int rt5631_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void rt5631_i2c_remove(struct i2c_client *client) +{} static struct i2c_driver rt5631_i2c_driver = { .driver = { diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c index 5a844329800f0ccb7209af1800322d0e2c96f244..0f8e6dd214b0ded89171e9951ff09c7e4a999c7e 100644 --- a/sound/soc/codecs/rt5640.c +++ b/sound/soc/codecs/rt5640.c @@ -2494,7 +2494,7 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component, /* Select JD-source */ snd_soc_component_update_bits(component, RT5640_JD_CTRL, - RT5640_JD_MASK, rt5640->jd_src); + RT5640_JD_MASK, rt5640->jd_src << RT5640_JD_SFT); /* Selecting GPIO01 as an interrupt */ snd_soc_component_update_bits(component, RT5640_GPIO_CTRL1, @@ -2504,12 +2504,8 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component, snd_soc_component_update_bits(component, RT5640_GPIO_CTRL3, RT5640_GP1_PF_MASK, RT5640_GP1_PF_OUT); - /* Enabling jd2 in general control 1 */ snd_soc_component_write(component, RT5640_DUMMY1, 0x3f41); - /* Enabling jd2 in general control 2 */ - snd_soc_component_write(component, RT5640_DUMMY2, 0x4001); - rt5640_set_ovcd_params(component); /* @@ -2518,12 +2514,25 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component, * pin 0/1 instead of it being stuck to 1. So we invert the JD polarity * on systems where the hardware does not already do this. */ - if (rt5640->jd_inverted) - snd_soc_component_write(component, RT5640_IRQ_CTRL1, - RT5640_IRQ_JD_NOR); - else - snd_soc_component_write(component, RT5640_IRQ_CTRL1, - RT5640_IRQ_JD_NOR | RT5640_JD_P_INV); + if (rt5640->jd_inverted) { + if (rt5640->jd_src == RT5640_JD_SRC_JD1_IN4P) + snd_soc_component_write(component, RT5640_IRQ_CTRL1, + RT5640_IRQ_JD_NOR); + else if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N) + snd_soc_component_update_bits(component, RT5640_DUMMY2, + RT5640_IRQ_JD2_MASK | RT5640_JD2_MASK, + RT5640_IRQ_JD2_NOR | RT5640_JD2_EN); + } else { + if (rt5640->jd_src == RT5640_JD_SRC_JD1_IN4P) + snd_soc_component_write(component, RT5640_IRQ_CTRL1, + RT5640_IRQ_JD_NOR | RT5640_JD_P_INV); + else if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N) + snd_soc_component_update_bits(component, RT5640_DUMMY2, + RT5640_IRQ_JD2_MASK | RT5640_JD2_P_MASK | + RT5640_JD2_MASK, + RT5640_IRQ_JD2_NOR | RT5640_JD2_P_INV | + RT5640_JD2_EN); + } rt5640->jack = jack; if (rt5640->jack->status & SND_JACK_MICROPHONE) { @@ -2725,10 +2734,8 @@ static int rt5640_probe(struct snd_soc_component *component) if (device_property_read_u32(component->dev, "realtek,jack-detect-source", &val) == 0) { - if (val <= RT5640_JD_SRC_GPIO4) - rt5640->jd_src = val << RT5640_JD_SFT; - else if (val == RT5640_JD_SRC_HDA_HEADER) - rt5640->jd_src = RT5640_JD_SRC_HDA_HEADER; + if (val <= RT5640_JD_SRC_HDA_HEADER) + rt5640->jd_src = val; else dev_warn(component->dev, "Warning: Invalid jack-detect-source value: %d, leaving jack-detect disabled\n", val); @@ -2809,12 +2816,31 @@ static int rt5640_resume(struct snd_soc_component *component) regcache_sync(rt5640->regmap); if (rt5640->jack) { - if (rt5640->jd_src == RT5640_JD_SRC_HDA_HEADER) + if (rt5640->jd_src == RT5640_JD_SRC_HDA_HEADER) { snd_soc_component_update_bits(component, RT5640_DUMMY2, 0x1100, 0x1100); - else - snd_soc_component_write(component, RT5640_DUMMY2, - 0x4001); + } else { + if (rt5640->jd_inverted) { + if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N) + snd_soc_component_update_bits( + component, RT5640_DUMMY2, + RT5640_IRQ_JD2_MASK | + RT5640_JD2_MASK, + RT5640_IRQ_JD2_NOR | + RT5640_JD2_EN); + + } else { + if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N) + snd_soc_component_update_bits( + component, RT5640_DUMMY2, + RT5640_IRQ_JD2_MASK | + RT5640_JD2_P_MASK | + RT5640_JD2_MASK, + RT5640_IRQ_JD2_NOR | + RT5640_JD2_P_INV | + RT5640_JD2_EN); + } + } queue_delayed_work(system_long_wq, &rt5640->jack_work, 0); } diff --git a/sound/soc/codecs/rt5640.h b/sound/soc/codecs/rt5640.h index 505c93514051a49cbafe625e54fda0a1db44990a..f58b88e3325b5c3de692548463562c49e5daebc8 100644 --- a/sound/soc/codecs/rt5640.h +++ b/sound/soc/codecs/rt5640.h @@ -1984,6 +1984,20 @@ #define RT5640_M_MONO_ADC_R_SFT 12 #define RT5640_MCLK_DET (0x1 << 11) +/* General Control 1 (0xfb) */ +#define RT5640_IRQ_JD2_MASK (0x1 << 12) +#define RT5640_IRQ_JD2_SFT 12 +#define RT5640_IRQ_JD2_BP (0x0 << 12) +#define RT5640_IRQ_JD2_NOR (0x1 << 12) +#define RT5640_JD2_P_MASK (0x1 << 10) +#define RT5640_JD2_P_SFT 10 +#define RT5640_JD2_P_NOR (0x0 << 10) +#define RT5640_JD2_P_INV (0x1 << 10) +#define RT5640_JD2_MASK (0x1 << 8) +#define RT5640_JD2_SFT 8 +#define RT5640_JD2_DIS (0x0 << 8) +#define RT5640_JD2_EN (0x1 << 8) + /* Codec Private Register definition */ /* MIC Over current threshold scale factor (0x15) */ diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 8635bc6567dcecae372a120a98925e2dd692e55f..620ecbfa4a7a8379837e390f43fb880b3a0c7e64 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -4145,7 +4145,7 @@ err_enable: return ret; } -static int rt5645_i2c_remove(struct i2c_client *i2c) +static void rt5645_i2c_remove(struct i2c_client *i2c) { struct rt5645_priv *rt5645 = i2c_get_clientdata(i2c); @@ -4162,8 +4162,6 @@ static int rt5645_i2c_remove(struct i2c_client *i2c) cancel_delayed_work_sync(&rt5645->rcclock_work); regulator_bulk_disable(ARRAY_SIZE(rt5645->supplies), rt5645->supplies); - - return 0; } static void rt5645_i2c_shutdown(struct i2c_client *i2c) diff --git a/sound/soc/codecs/rt5663.c b/sound/soc/codecs/rt5663.c index ca981b374b0c8538ca450eec114ed9db28008abe..f73751dbde30c7440bf6ed78ff194df6d9b37e19 100644 --- a/sound/soc/codecs/rt5663.c +++ b/sound/soc/codecs/rt5663.c @@ -3710,7 +3710,7 @@ err_enable: return ret; } -static int rt5663_i2c_remove(struct i2c_client *i2c) +static void rt5663_i2c_remove(struct i2c_client *i2c) { struct rt5663_priv *rt5663 = i2c_get_clientdata(i2c); @@ -3718,8 +3718,6 @@ static int rt5663_i2c_remove(struct i2c_client *i2c) free_irq(i2c->irq, rt5663); regulator_bulk_disable(ARRAY_SIZE(rt5663->supplies), rt5663->supplies); - - return 0; } static void rt5663_i2c_shutdown(struct i2c_client *client) diff --git a/sound/soc/codecs/rt5670.c b/sound/soc/codecs/rt5670.c index 60dbfa2a54f1b86a073ec4721016d88c844f315e..ebac6caeb40ada0e1047fc6ea4ff07f75919dd4f 100644 --- a/sound/soc/codecs/rt5670.c +++ b/sound/soc/codecs/rt5670.c @@ -3320,11 +3320,9 @@ err: return ret; } -static int rt5670_i2c_remove(struct i2c_client *i2c) +static void rt5670_i2c_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); - - return 0; } static struct i2c_driver rt5670_i2c_driver = { diff --git a/sound/soc/codecs/rt5677.c b/sound/soc/codecs/rt5677.c index 31a2dd0aafb64f0bca2be5a1eec3d9530e966d50..c26395f42d8e7de927c8bc4323e25a19975744db 100644 --- a/sound/soc/codecs/rt5677.c +++ b/sound/soc/codecs/rt5677.c @@ -5693,11 +5693,9 @@ static int rt5677_i2c_probe(struct i2c_client *i2c) rt5677_dai, ARRAY_SIZE(rt5677_dai)); } -static int rt5677_i2c_remove(struct i2c_client *i2c) +static void rt5677_i2c_remove(struct i2c_client *i2c) { rt5677_free_gpio(i2c); - - return 0; } static struct i2c_driver rt5677_i2c_driver = { diff --git a/sound/soc/codecs/rt5682-i2c.c b/sound/soc/codecs/rt5682-i2c.c index 3f72f60934362d4885e47bedeac9f7d0e580da67..2935c1bb81f3fbc16c3cd7e56bc7d05c04693423 100644 --- a/sound/soc/codecs/rt5682-i2c.c +++ b/sound/soc/codecs/rt5682-i2c.c @@ -302,11 +302,9 @@ static void rt5682_i2c_shutdown(struct i2c_client *client) rt5682_reset(rt5682); } -static int rt5682_i2c_remove(struct i2c_client *client) +static void rt5682_i2c_remove(struct i2c_client *client) { rt5682_i2c_shutdown(client); - - return 0; } static const struct of_device_id rt5682_of_match[] = { diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c index f04e18c32489ddef2b9f1fc32f49bc56e869688a..c1a94229dc7e3c06784cf9c9cb2e9e4042eb285d 100644 --- a/sound/soc/codecs/rt5682-sdw.c +++ b/sound/soc/codecs/rt5682-sdw.c @@ -793,6 +793,8 @@ static int __maybe_unused rt5682_dev_resume(struct device *dev) msecs_to_jiffies(RT5682_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt5682s.c b/sound/soc/codecs/rt5682s.c index eb47e7cd485aa11f39528d7b9008e9b4ef9741a9..466a37f3500cd452bef6e542b80a22b40aa3a16d 100644 --- a/sound/soc/codecs/rt5682s.c +++ b/sound/soc/codecs/rt5682s.c @@ -739,6 +739,7 @@ static void rt5682s_disable_push_button_irq(struct snd_soc_component *component) */ static int rt5682s_headset_detect(struct snd_soc_component *component, int jack_insert) { + struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); unsigned int val, count; int jack_type = 0; @@ -805,12 +806,10 @@ static int rt5682s_headset_detect(struct snd_soc_component *component, int jack_ snd_soc_component_update_bits(component, RT5682S_CBJ_CTRL_1, RT5682S_TRIG_JD_MASK, RT5682S_TRIG_JD_LOW); - if (!snd_soc_dapm_get_pin_status(&component->dapm, "MICBIAS")) - snd_soc_component_update_bits(component, - RT5682S_PWR_ANLG_1, RT5682S_PWR_MB, 0); - if (!snd_soc_dapm_get_pin_status(&component->dapm, "Vref2")) + if (!rt5682s->wclk_enabled) { snd_soc_component_update_bits(component, - RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2, 0); + RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2 | RT5682S_PWR_MB, 0); + } snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3, RT5682S_PWR_CBJ, 0); @@ -845,6 +844,7 @@ static void rt5682s_jack_detect_handler(struct work_struct *work) snd_soc_dapm_mutex_lock(dapm); mutex_lock(&rt5682s->calibrate_mutex); + mutex_lock(&rt5682s->wclk_mutex); val = snd_soc_component_read(rt5682s->component, RT5682S_AJD1_CTRL) & RT5682S_JDH_RS_MASK; @@ -900,6 +900,7 @@ static void rt5682s_jack_detect_handler(struct work_struct *work) rt5682s->irq_work_delay_time = 50; } + mutex_unlock(&rt5682s->wclk_mutex); mutex_unlock(&rt5682s->calibrate_mutex); snd_soc_dapm_mutex_unlock(dapm); @@ -1154,29 +1155,52 @@ static int set_dmic_clk(struct snd_soc_dapm_widget *w, return 0; } -static int set_filter_clk(struct snd_soc_dapm_widget *w, + +static int rt5682s_set_pllb_power(struct rt5682s_priv *rt5682s, int on) +{ + struct snd_soc_component *component = rt5682s->component; + + if (on) { + snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3, + RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB | RT5682S_PWR_PLLB, + RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB | RT5682S_PWR_PLLB); + snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3, + RT5682S_RSTB_PLLB, RT5682S_RSTB_PLLB); + } else { + snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3, + RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB | + RT5682S_RSTB_PLLB | RT5682S_PWR_PLLB, 0); + } + + return 0; +} + +static int set_pllb_event(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) { struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); - int ref, val, reg, idx; - static const int div_f[] = {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48}; - static const int div_o[] = {1, 2, 4, 6, 8, 12, 16, 24, 32, 48}; + int on = 0; - val = snd_soc_component_read(component, RT5682S_GPIO_CTRL_1) - & RT5682S_GP4_PIN_MASK; + if (rt5682s->wclk_enabled) + return 0; - if (w->shift == RT5682S_PWR_ADC_S1F_BIT && val == RT5682S_GP4_PIN_ADCDAT2) - ref = 256 * rt5682s->lrck[RT5682S_AIF2]; - else - ref = 256 * rt5682s->lrck[RT5682S_AIF1]; + if (SND_SOC_DAPM_EVENT_ON(event)) + on = 1; - idx = rt5682s_div_sel(rt5682s, ref, div_f, ARRAY_SIZE(div_f)); + rt5682s_set_pllb_power(rt5682s, on); - if (w->shift == RT5682S_PWR_ADC_S1F_BIT) - reg = RT5682S_PLL_TRACK_3; - else - reg = RT5682S_PLL_TRACK_2; + return 0; +} + +static void rt5682s_set_filter_clk(struct rt5682s_priv *rt5682s, int reg, int ref) +{ + struct snd_soc_component *component = rt5682s->component; + int idx; + static const int div_f[] = {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48}; + static const int div_o[] = {1, 2, 4, 6, 8, 12, 16, 24, 32, 48}; + + idx = rt5682s_div_sel(rt5682s, ref, div_f, ARRAY_SIZE(div_f)); snd_soc_component_update_bits(component, reg, RT5682S_FILTER_CLK_DIV_MASK, idx << RT5682S_FILTER_CLK_DIV_SFT); @@ -1190,6 +1214,29 @@ static int set_filter_clk(struct snd_soc_dapm_widget *w, snd_soc_component_update_bits(component, RT5682S_ADDA_CLK_1, RT5682S_ADC_OSR_MASK | RT5682S_DAC_OSR_MASK, (idx << RT5682S_ADC_OSR_SFT) | (idx << RT5682S_DAC_OSR_SFT)); +} + +static int set_filter_clk(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, int event) +{ + struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); + struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); + int ref, reg, val; + + val = snd_soc_component_read(component, RT5682S_GPIO_CTRL_1) + & RT5682S_GP4_PIN_MASK; + + if (w->shift == RT5682S_PWR_ADC_S1F_BIT && val == RT5682S_GP4_PIN_ADCDAT2) + ref = 256 * rt5682s->lrck[RT5682S_AIF2]; + else + ref = 256 * rt5682s->lrck[RT5682S_AIF1]; + + if (w->shift == RT5682S_PWR_ADC_S1F_BIT) + reg = RT5682S_PLL_TRACK_3; + else + reg = RT5682S_PLL_TRACK_2; + + rt5682s_set_filter_clk(rt5682s, reg, ref); return 0; } @@ -1218,13 +1265,9 @@ static int set_dmic_power(struct snd_soc_dapm_widget *w, break; case SND_SOC_DAPM_POST_PMD: - if (!rt5682s->jack_type) { - if (!snd_soc_dapm_get_pin_status(w->dapm, "MICBIAS")) - snd_soc_component_update_bits(component, - RT5682S_PWR_ANLG_1, RT5682S_PWR_MB, 0); - if (!snd_soc_dapm_get_pin_status(w->dapm, "Vref2")) - snd_soc_component_update_bits(component, - RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2, 0); + if (!rt5682s->jack_type && !rt5682s->wclk_enabled) { + snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1, + RT5682S_PWR_VREF2 | RT5682S_PWR_MB, 0); } break; } @@ -1232,41 +1275,58 @@ static int set_dmic_power(struct snd_soc_dapm_widget *w, return 0; } -static int set_i2s_clk(struct snd_soc_dapm_widget *w, - struct snd_kcontrol *kcontrol, int event) +static void rt5682s_set_i2s(struct rt5682s_priv *rt5682s, int id, int on) { - struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); - struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); - int pre_div, id; - unsigned int reg, mask, sft; - - if (event != SND_SOC_DAPM_PRE_PMU) - return 0; - - if (w->shift == RT5682S_PWR_I2S2_BIT) { - id = RT5682S_AIF2; - reg = RT5682S_I2S2_M_CLK_CTRL_1; - mask = RT5682S_I2S2_M_D_MASK; - sft = RT5682S_I2S2_M_D_SFT; + struct snd_soc_component *component = rt5682s->component; + int pre_div; + unsigned int p_reg, p_mask, p_sft; + unsigned int c_reg, c_mask, c_sft; + + if (id == RT5682S_AIF1) { + c_reg = RT5682S_ADDA_CLK_1; + c_mask = RT5682S_I2S_M_D_MASK; + c_sft = RT5682S_I2S_M_D_SFT; + p_reg = RT5682S_PWR_DIG_1; + p_mask = RT5682S_PWR_I2S1; + p_sft = RT5682S_PWR_I2S1_BIT; } else { - id = RT5682S_AIF1; - reg = RT5682S_ADDA_CLK_1; - mask = RT5682S_I2S_M_D_MASK; - sft = RT5682S_I2S_M_D_SFT; + c_reg = RT5682S_I2S2_M_CLK_CTRL_1; + c_mask = RT5682S_I2S2_M_D_MASK; + c_sft = RT5682S_I2S2_M_D_SFT; + p_reg = RT5682S_PWR_DIG_1; + p_mask = RT5682S_PWR_I2S2; + p_sft = RT5682S_PWR_I2S2_BIT; } - if (!rt5682s->master[id]) - return 0; + if (on && rt5682s->master[id]) { + pre_div = get_clk_info(rt5682s->sysclk, rt5682s->lrck[id]); + if (pre_div < 0) { + dev_err(component->dev, "get pre_div failed\n"); + return; + } - pre_div = get_clk_info(rt5682s->sysclk, rt5682s->lrck[id]); - if (pre_div < 0) { - dev_err(component->dev, "get pre_div failed\n"); - return -EINVAL; + dev_dbg(component->dev, "lrck is %dHz and pre_div is %d for iis %d master\n", + rt5682s->lrck[id], pre_div, id); + snd_soc_component_update_bits(component, c_reg, c_mask, pre_div << c_sft); } - dev_dbg(component->dev, "lrck is %dHz and pre_div is %d for iis %d master\n", - rt5682s->lrck[id], pre_div, id); - snd_soc_component_update_bits(component, reg, mask, pre_div << sft); + snd_soc_component_update_bits(component, p_reg, p_mask, on << p_sft); +} + +static int set_i2s_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, int event) +{ + struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); + struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); + int on = 0; + + if (SND_SOC_DAPM_EVENT_ON(event)) + on = 1; + + if (!strcmp(w->name, "I2S1") && !rt5682s->wclk_enabled) + rt5682s_set_i2s(rt5682s, RT5682S_AIF1, on); + else if (!strcmp(w->name, "I2S2")) + rt5682s_set_i2s(rt5682s, RT5682S_AIF2, on); return 0; } @@ -1615,26 +1675,18 @@ static const struct snd_soc_dapm_widget rt5682s_dapm_widgets[] = { RT5682S_PWR_LDO_MB2_BIT, 0, NULL, 0), SND_SOC_DAPM_SUPPLY("LDO", RT5682S_PWR_ANLG_3, RT5682S_PWR_LDO_BIT, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY("Vref2", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY("MICBIAS", SND_SOC_NOPM, 0, 0, NULL, 0), /* PLL Powers */ SND_SOC_DAPM_SUPPLY_S("PLLA_LDO", 0, RT5682S_PWR_ANLG_3, RT5682S_PWR_LDO_PLLA_BIT, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY_S("PLLB_LDO", 0, RT5682S_PWR_ANLG_3, - RT5682S_PWR_LDO_PLLB_BIT, 0, NULL, 0), SND_SOC_DAPM_SUPPLY_S("PLLA_BIAS", 0, RT5682S_PWR_ANLG_3, RT5682S_PWR_BIAS_PLLA_BIT, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY_S("PLLB_BIAS", 0, RT5682S_PWR_ANLG_3, - RT5682S_PWR_BIAS_PLLB_BIT, 0, NULL, 0), SND_SOC_DAPM_SUPPLY_S("PLLA", 0, RT5682S_PWR_ANLG_3, RT5682S_PWR_PLLA_BIT, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY_S("PLLB", 0, RT5682S_PWR_ANLG_3, - RT5682S_PWR_PLLB_BIT, 0, set_filter_clk, SND_SOC_DAPM_PRE_PMU), SND_SOC_DAPM_SUPPLY_S("PLLA_RST", 1, RT5682S_PWR_ANLG_3, RT5682S_RSTB_PLLA_BIT, 0, NULL, 0), - SND_SOC_DAPM_SUPPLY_S("PLLB_RST", 1, RT5682S_PWR_ANLG_3, - RT5682S_RSTB_PLLB_BIT, 0, NULL, 0), + SND_SOC_DAPM_SUPPLY("PLLB", SND_SOC_NOPM, 0, 0, + set_pllb_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD), /* ASRC */ SND_SOC_DAPM_SUPPLY_S("DAC STO1 ASRC", 1, RT5682S_PLL_TRACK_1, @@ -1720,10 +1772,10 @@ static const struct snd_soc_dapm_widget rt5682s_dapm_widgets[] = { SND_SOC_DAPM_PGA("Stereo1 ADC MIX", SND_SOC_NOPM, 0, 0, NULL, 0), /* Digital Interface */ - SND_SOC_DAPM_SUPPLY("I2S1", RT5682S_PWR_DIG_1, RT5682S_PWR_I2S1_BIT, - 0, set_i2s_clk, SND_SOC_DAPM_PRE_PMU), - SND_SOC_DAPM_SUPPLY("I2S2", RT5682S_PWR_DIG_1, RT5682S_PWR_I2S2_BIT, - 0, set_i2s_clk, SND_SOC_DAPM_PRE_PMU), + SND_SOC_DAPM_SUPPLY("I2S1", SND_SOC_NOPM, 0, 0, + set_i2s_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD), + SND_SOC_DAPM_SUPPLY("I2S2", SND_SOC_NOPM, 0, 0, + set_i2s_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD), SND_SOC_DAPM_PGA("IF1 DAC1 L", SND_SOC_NOPM, 0, 0, NULL, 0), SND_SOC_DAPM_PGA("IF1 DAC1 R", SND_SOC_NOPM, 0, 0, NULL, 0), @@ -1801,9 +1853,6 @@ static const struct snd_soc_dapm_route rt5682s_dapm_routes[] = { {"PLLA", NULL, "PLLA_LDO"}, {"PLLA", NULL, "PLLA_BIAS"}, {"PLLA", NULL, "PLLA_RST"}, - {"PLLB", NULL, "PLLB_LDO"}, - {"PLLB", NULL, "PLLB_BIAS"}, - {"PLLB", NULL, "PLLB_RST"}, /*ASRC*/ {"ADC Stereo1 Filter", NULL, "ADC STO1 ASRC", is_using_asrc}, @@ -2431,12 +2480,15 @@ static int rt5682s_set_bias_level(struct snd_soc_component *component, RT5682S_PWR_LDO, RT5682S_PWR_LDO); break; case SND_SOC_BIAS_STANDBY: - regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, - RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL); + if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF) + regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, + RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL); break; case SND_SOC_BIAS_OFF: - regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, - RT5682S_DIG_GATE_CTRL | RT5682S_PWR_LDO, 0); + regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, RT5682S_PWR_LDO, 0); + if (!rt5682s->wclk_enabled) + regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, + RT5682S_DIG_GATE_CTRL, 0); break; case SND_SOC_BIAS_ON: break; @@ -2464,30 +2516,34 @@ static int rt5682s_wclk_prepare(struct clk_hw *hw) struct rt5682s_priv *rt5682s = container_of(hw, struct rt5682s_priv, dai_clks_hw[RT5682S_DAI_WCLK_IDX]); struct snd_soc_component *component = rt5682s->component; - struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); + int ref, reg; if (!rt5682s_clk_check(rt5682s)) return -EINVAL; - snd_soc_dapm_mutex_lock(dapm); - - snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS"); - snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1, - RT5682S_PWR_MB, RT5682S_PWR_MB); + mutex_lock(&rt5682s->wclk_mutex); - snd_soc_dapm_force_enable_pin_unlocked(dapm, "Vref2"); snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1, - RT5682S_PWR_VREF2 | RT5682S_PWR_FV2, RT5682S_PWR_VREF2); + RT5682S_PWR_VREF2 | RT5682S_PWR_FV2 | RT5682S_PWR_MB, + RT5682S_PWR_VREF2 | RT5682S_PWR_MB); usleep_range(15000, 20000); snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1, RT5682S_PWR_FV2, RT5682S_PWR_FV2); - snd_soc_dapm_force_enable_pin_unlocked(dapm, "I2S1"); - /* Only need to power PLLB due to the rate set restriction */ - snd_soc_dapm_force_enable_pin_unlocked(dapm, "PLLB"); - snd_soc_dapm_sync_unlocked(dapm); + /* Set and power on I2S1 */ + snd_soc_component_update_bits(component, RT5682S_PWR_DIG_1, + RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL); + rt5682s_set_i2s(rt5682s, RT5682S_AIF1, 1); - snd_soc_dapm_mutex_unlock(dapm); + /* Only need to power on PLLB due to the rate set restriction */ + reg = RT5682S_PLL_TRACK_2; + ref = 256 * rt5682s->lrck[RT5682S_AIF1]; + rt5682s_set_filter_clk(rt5682s, reg, ref); + rt5682s_set_pllb_power(rt5682s, 1); + + rt5682s->wclk_enabled = 1; + + mutex_unlock(&rt5682s->wclk_mutex); return 0; } @@ -2497,24 +2553,27 @@ static void rt5682s_wclk_unprepare(struct clk_hw *hw) struct rt5682s_priv *rt5682s = container_of(hw, struct rt5682s_priv, dai_clks_hw[RT5682S_DAI_WCLK_IDX]); struct snd_soc_component *component = rt5682s->component; - struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component); if (!rt5682s_clk_check(rt5682s)) return; - snd_soc_dapm_mutex_lock(dapm); + mutex_lock(&rt5682s->wclk_mutex); - snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS"); - snd_soc_dapm_disable_pin_unlocked(dapm, "Vref2"); if (!rt5682s->jack_type) snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2 | RT5682S_PWR_FV2 | RT5682S_PWR_MB, 0); - snd_soc_dapm_disable_pin_unlocked(dapm, "I2S1"); - snd_soc_dapm_disable_pin_unlocked(dapm, "PLLB"); - snd_soc_dapm_sync_unlocked(dapm); + /* Power down I2S1 */ + rt5682s_set_i2s(rt5682s, RT5682S_AIF1, 0); + snd_soc_component_update_bits(component, RT5682S_PWR_DIG_1, + RT5682S_DIG_GATE_CTRL, 0); - snd_soc_dapm_mutex_unlock(dapm); + /* Power down PLLB */ + rt5682s_set_pllb_power(rt5682s, 0); + + rt5682s->wclk_enabled = 0; + + mutex_unlock(&rt5682s->wclk_mutex); } static unsigned long rt5682s_wclk_recalc_rate(struct clk_hw *hw, @@ -2805,19 +2864,10 @@ static inline int rt5682s_dai_probe_clks(struct snd_soc_component *component) static int rt5682s_probe(struct snd_soc_component *component) { struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component); - struct snd_soc_dapm_context *dapm = &component->dapm; - int ret; rt5682s->component = component; - ret = rt5682s_dai_probe_clks(component); - if (ret) - return ret; - - snd_soc_dapm_disable_pin(dapm, "MICBIAS"); - snd_soc_dapm_disable_pin(dapm, "Vref2"); - snd_soc_dapm_sync(dapm); - return 0; + return rt5682s_dai_probe_clks(component); } static void rt5682s_remove(struct snd_soc_component *component) @@ -3113,6 +3163,7 @@ static int rt5682s_i2c_probe(struct i2c_client *i2c) mutex_init(&rt5682s->calibrate_mutex); mutex_init(&rt5682s->sar_mutex); + mutex_init(&rt5682s->wclk_mutex); rt5682s_calibrate(rt5682s); regmap_update_bits(rt5682s->regmap, RT5682S_MICBIAS_2, @@ -3194,11 +3245,9 @@ static void rt5682s_i2c_shutdown(struct i2c_client *client) rt5682s_reset(rt5682s); } -static int rt5682s_i2c_remove(struct i2c_client *client) +static void rt5682s_i2c_remove(struct i2c_client *client) { rt5682s_i2c_shutdown(client); - - return 0; } static const struct of_device_id rt5682s_of_match[] = { diff --git a/sound/soc/codecs/rt5682s.h b/sound/soc/codecs/rt5682s.h index 7353831c73dd518c3b8b655f58c03f13f4b2eba2..824dc6543c182429ac21ca3aee5ecfd65558c747 100644 --- a/sound/soc/codecs/rt5682s.h +++ b/sound/soc/codecs/rt5682s.h @@ -1450,6 +1450,7 @@ struct rt5682s_priv { struct delayed_work jd_check_work; struct mutex calibrate_mutex; struct mutex sar_mutex; + struct mutex wclk_mutex; #ifdef CONFIG_COMMON_CLK struct clk_hw dai_clks_hw[RT5682S_DAI_NUM_CLKS]; @@ -1469,6 +1470,7 @@ struct rt5682s_priv { int jack_type; int irq_work_delay_time; + int wclk_enabled; }; int rt5682s_sel_asrc_clk_src(struct snd_soc_component *component, diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c index f7439e40ca8b543dff73f464c2a326b4707dd127..96fc5f36d0d05aaf65638c1349958ffb42e99bf1 100644 --- a/sound/soc/codecs/rt700-sdw.c +++ b/sound/soc/codecs/rt700-sdw.c @@ -542,6 +542,8 @@ static int __maybe_unused rt700_dev_resume(struct device *dev) msecs_to_jiffies(RT700_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt711-sdca-sdw.c b/sound/soc/codecs/rt711-sdca-sdw.c index a085b2f530aa1d268135f3f259ef3b4a72c0c7b8..4120842fe699022440b6f39b57ce8b7ebab5482a 100644 --- a/sound/soc/codecs/rt711-sdca-sdw.c +++ b/sound/soc/codecs/rt711-sdca-sdw.c @@ -449,6 +449,8 @@ static int __maybe_unused rt711_sdca_dev_resume(struct device *dev) msecs_to_jiffies(RT711_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt715-sdca-sdw.c b/sound/soc/codecs/rt715-sdca-sdw.c index 13e731d166753ac117f59249a88388b297468df3..3f981a9e7fb67b54caee8290ef4d3eccf0537800 100644 --- a/sound/soc/codecs/rt715-sdca-sdw.c +++ b/sound/soc/codecs/rt715-sdca-sdw.c @@ -244,6 +244,8 @@ static int __maybe_unused rt715_dev_resume(struct device *dev) msecs_to_jiffies(RT715_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Enumeration not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt715-sdw.c b/sound/soc/codecs/rt715-sdw.c index b047bf87a100c590241fe4c01ae2e64004e1be79..4e61e16470eda2b6fe40b7bfe304b0ff4160f960 100644 --- a/sound/soc/codecs/rt715-sdw.c +++ b/sound/soc/codecs/rt715-sdw.c @@ -562,6 +562,8 @@ static int __maybe_unused rt715_dev_resume(struct device *dev) msecs_to_jiffies(RT715_PROBE_TIMEOUT)); if (!time) { dev_err(&slave->dev, "Initialization not complete, timed out\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; } diff --git a/sound/soc/codecs/rt9120.c b/sound/soc/codecs/rt9120.c index da495bdc841582dcac4f859d3b5ab718ded69bfa..644300e88b4c5e32c24992d2afe96a0e7bb216ce 100644 --- a/sound/soc/codecs/rt9120.c +++ b/sound/soc/codecs/rt9120.c @@ -572,11 +572,10 @@ static int rt9120_probe(struct i2c_client *i2c) &rt9120_dai, 1); } -static int rt9120_remove(struct i2c_client *i2c) +static void rt9120_remove(struct i2c_client *i2c) { pm_runtime_disable(&i2c->dev); pm_runtime_set_suspended(&i2c->dev); - return 0; } static int __maybe_unused rt9120_runtime_suspend(struct device *dev) diff --git a/sound/soc/codecs/sgtl5000.c b/sound/soc/codecs/sgtl5000.c index 3fafd9fc5cfd61bdc3159b68db0c450dc32bd08d..4b2135eba74d93a001f9ea2947fa6e509e776a95 100644 --- a/sound/soc/codecs/sgtl5000.c +++ b/sound/soc/codecs/sgtl5000.c @@ -1790,7 +1790,7 @@ disable_regs: return ret; } -static int sgtl5000_i2c_remove(struct i2c_client *client) +static void sgtl5000_i2c_remove(struct i2c_client *client) { struct sgtl5000_priv *sgtl5000 = i2c_get_clientdata(client); @@ -1800,8 +1800,6 @@ static int sgtl5000_i2c_remove(struct i2c_client *client) clk_disable_unprepare(sgtl5000->mclk); regulator_bulk_disable(sgtl5000->num_supplies, sgtl5000->supplies); regulator_bulk_free(sgtl5000->num_supplies, sgtl5000->supplies); - - return 0; } static void sgtl5000_i2c_shutdown(struct i2c_client *client) diff --git a/sound/soc/codecs/sigmadsp.c b/sound/soc/codecs/sigmadsp.c index b992216aee55f495c6cc580eaea56bf0f597615c..3047a6fbb3805bc2c248b8a2478efee9941f6037 100644 --- a/sound/soc/codecs/sigmadsp.c +++ b/sound/soc/codecs/sigmadsp.c @@ -227,13 +227,11 @@ static int sigma_fw_load_control(struct sigmadsp *sigmadsp, if (!ctrl) return -ENOMEM; - name = kzalloc(name_len + 1, GFP_KERNEL); + name = kmemdup_nul(ctrl_chunk->name, name_len, GFP_KERNEL); if (!name) { ret = -ENOMEM; goto err_free_ctrl; } - memcpy(name, ctrl_chunk->name, name_len); - name[name_len] = '\0'; ctrl->name = name; /* diff --git a/sound/soc/codecs/src4xxx-i2c.c b/sound/soc/codecs/src4xxx-i2c.c new file mode 100644 index 0000000000000000000000000000000000000000..43daa9dc8ab58dda6794ed79122cd5b5a58c4b13 --- /dev/null +++ b/sound/soc/codecs/src4xxx-i2c.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Driver for SRC4XXX codecs +// +// Copyright 2021-2022 Deqx Pty Ltd +// Author: Matt Flax <flatmax@flatmax.com> + +#include <linux/i2c.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/regmap.h> + +#include "src4xxx.h" + +static int src4xxx_i2c_probe(struct i2c_client *i2c, + const struct i2c_device_id *id) +{ + return src4xxx_probe(&i2c->dev, + devm_regmap_init_i2c(i2c, &src4xxx_regmap_config), NULL); +} + +static const struct i2c_device_id src4xxx_i2c_ids[] = { + { "src4392", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, src4xxx_i2c_ids); + +static const struct of_device_id src4xxx_of_match[] = { + { .compatible = "ti,src4392", }, + { } +}; +MODULE_DEVICE_TABLE(of, src4xxx_of_match); + + +static struct i2c_driver src4xxx_i2c_driver = { + .driver = { + .name = "src4xxx", + .of_match_table = of_match_ptr(src4xxx_of_match), + }, + .probe = src4xxx_i2c_probe, + .id_table = src4xxx_i2c_ids, +}; +module_i2c_driver(src4xxx_i2c_driver); + +MODULE_DESCRIPTION("ASoC SRC4392 CODEC I2C driver"); +MODULE_AUTHOR("Matt Flax <flatmax@flatmax.com>"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/codecs/src4xxx.c b/sound/soc/codecs/src4xxx.c new file mode 100644 index 0000000000000000000000000000000000000000..db4e280dd055eef6a53deebfa4fded028845e0d6 --- /dev/null +++ b/sound/soc/codecs/src4xxx.c @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// TI SRC4xxx Audio Codec driver +// +// Copyright 2021-2022 Deqx Pty Ltd +// Author: Matt Flax <flatmax@flatmax.com> + +#include <linux/module.h> + +#include <sound/soc.h> +#include <sound/tlv.h> + +#include "src4xxx.h" + +struct src4xxx { + struct regmap *regmap; + bool master[2]; + int mclk_hz; + struct device *dev; +}; + +enum {SRC4XXX_PORTA, SRC4XXX_PORTB}; + +/* SRC attenuation */ +static const DECLARE_TLV_DB_SCALE(src_tlv, -12750, 50, 0); + +static const struct snd_kcontrol_new src4xxx_controls[] = { + SOC_DOUBLE_R_TLV("SRC Volume", + SRC4XXX_SCR_CTL_30, SRC4XXX_SCR_CTL_31, 0, 255, 1, src_tlv), +}; + +/* I2S port control */ +static const char * const port_out_src_text[] = { + "loopback", "other_port", "DIR", "SRC" +}; +static SOC_ENUM_SINGLE_DECL(porta_out_src_enum, SRC4XXX_PORTA_CTL_03, 4, + port_out_src_text); +static SOC_ENUM_SINGLE_DECL(portb_out_src_enum, SRC4XXX_PORTB_CTL_05, 4, + port_out_src_text); +static const struct snd_kcontrol_new porta_out_control = + SOC_DAPM_ENUM("Port A source select", porta_out_src_enum); +static const struct snd_kcontrol_new portb_out_control = + SOC_DAPM_ENUM("Port B source select", portb_out_src_enum); + +/* Digital audio transmitter control */ +static const char * const dit_mux_text[] = {"Port A", "Port B", "DIR", "SRC"}; +static SOC_ENUM_SINGLE_DECL(dit_mux_enum, SRC4XXX_TX_CTL_07, 3, dit_mux_text); +static const struct snd_kcontrol_new dit_mux_control = + SOC_DAPM_ENUM("DIT source", dit_mux_enum); + +/* SRC control */ +static const char * const src_in_text[] = {"Port A", "Port B", "DIR"}; +static SOC_ENUM_SINGLE_DECL(src_in_enum, SRC4XXX_SCR_CTL_2D, 0, src_in_text); +static const struct snd_kcontrol_new src_in_control = + SOC_DAPM_ENUM("SRC source select", src_in_enum); + +/* DIR control */ +static const char * const dir_in_text[] = {"Ch 1", "Ch 2", "Ch 3", "Ch 4"}; +static SOC_ENUM_SINGLE_DECL(dir_in_enum, SRC4XXX_RCV_CTL_0D, 0, dir_in_text); +static const struct snd_kcontrol_new dir_in_control = + SOC_DAPM_ENUM("Digital Input", dir_in_enum); + +static const struct snd_soc_dapm_widget src4xxx_dapm_widgets[] = { + SND_SOC_DAPM_INPUT("loopback_A"), + SND_SOC_DAPM_INPUT("other_port_A"), + SND_SOC_DAPM_INPUT("DIR_A"), + SND_SOC_DAPM_INPUT("SRC_A"), + SND_SOC_DAPM_MUX("Port A source", + SND_SOC_NOPM, 0, 0, &porta_out_control), + + SND_SOC_DAPM_INPUT("loopback_B"), + SND_SOC_DAPM_INPUT("other_port_B"), + SND_SOC_DAPM_INPUT("DIR_B"), + SND_SOC_DAPM_INPUT("SRC_B"), + SND_SOC_DAPM_MUX("Port B source", + SND_SOC_NOPM, 0, 0, &portb_out_control), + + SND_SOC_DAPM_INPUT("Port_A"), + SND_SOC_DAPM_INPUT("Port_B"), + SND_SOC_DAPM_INPUT("DIR_"), + + /* Digital audio receivers and transmitters */ + SND_SOC_DAPM_OUTPUT("DIR_OUT"), + SND_SOC_DAPM_OUTPUT("SRC_OUT"), + SND_SOC_DAPM_MUX("DIT Out Src", SRC4XXX_PWR_RST_01, + SRC4XXX_ENABLE_DIT_SHIFT, 1, &dit_mux_control), + + /* Audio Interface */ + SND_SOC_DAPM_AIF_IN("AIF_A_RX", "Playback A", 0, + SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_A_SHIFT, 1), + SND_SOC_DAPM_AIF_OUT("AIF_A_TX", "Capture A", 0, + SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_A_SHIFT, 1), + SND_SOC_DAPM_AIF_IN("AIF_B_RX", "Playback B", 0, + SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_B_SHIFT, 1), + SND_SOC_DAPM_AIF_OUT("AIF_B_TX", "Capture B", 0, + SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_B_SHIFT, 1), + + SND_SOC_DAPM_MUX("SRC source", SND_SOC_NOPM, 0, 0, &src_in_control), + + SND_SOC_DAPM_INPUT("MCLK"), + SND_SOC_DAPM_INPUT("RXMCLKI"), + SND_SOC_DAPM_INPUT("RXMCLKO"), + + SND_SOC_DAPM_INPUT("RX1"), + SND_SOC_DAPM_INPUT("RX2"), + SND_SOC_DAPM_INPUT("RX3"), + SND_SOC_DAPM_INPUT("RX4"), + SND_SOC_DAPM_MUX("Digital Input", SRC4XXX_PWR_RST_01, + SRC4XXX_ENABLE_DIR_SHIFT, 1, &dir_in_control), +}; + +static const struct snd_soc_dapm_route src4xxx_audio_routes[] = { + /* I2S Input to Output Routing */ + {"Port A source", "loopback", "loopback_A"}, + {"Port A source", "other_port", "other_port_A"}, + {"Port A source", "DIR", "DIR_A"}, + {"Port A source", "SRC", "SRC_A"}, + {"Port B source", "loopback", "loopback_B"}, + {"Port B source", "other_port", "other_port_B"}, + {"Port B source", "DIR", "DIR_B"}, + {"Port B source", "SRC", "SRC_B"}, + /* DIT muxing */ + {"DIT Out Src", "Port A", "Capture A"}, + {"DIT Out Src", "Port B", "Capture B"}, + {"DIT Out Src", "DIR", "DIR_OUT"}, + {"DIT Out Src", "SRC", "SRC_OUT"}, + + /* SRC input selection */ + {"SRC source", "Port A", "Port_A"}, + {"SRC source", "Port B", "Port_B"}, + {"SRC source", "DIR", "DIR_"}, + /* SRC mclk selection */ + {"SRC mclk source", "Master (MCLK)", "MCLK"}, + {"SRC mclk source", "Master (RXCLKI)", "RXMCLKI"}, + {"SRC mclk source", "Recovered receiver clk", "RXMCLKO"}, + /* DIR input selection */ + {"Digital Input", "Ch 1", "RX1"}, + {"Digital Input", "Ch 2", "RX2"}, + {"Digital Input", "Ch 3", "RX3"}, + {"Digital Input", "Ch 4", "RX4"}, +}; + + +static const struct snd_soc_component_driver src4xxx_driver = { + .controls = src4xxx_controls, + .num_controls = ARRAY_SIZE(src4xxx_controls), + + .dapm_widgets = src4xxx_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(src4xxx_dapm_widgets), + .dapm_routes = src4xxx_audio_routes, + .num_dapm_routes = ARRAY_SIZE(src4xxx_audio_routes), +}; + +static int src4xxx_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) +{ + struct snd_soc_component *component = dai->component; + struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component); + unsigned int ctrl; + + switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { + case SND_SOC_DAIFMT_CBM_CFM: + ctrl = SRC4XXX_BUS_MASTER; + src4xxx->master[dai->id] = true; + break; + case SND_SOC_DAIFMT_CBS_CFS: + ctrl = 0; + src4xxx->master[dai->id] = false; + break; + default: + return -EINVAL; + break; + } + + switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_I2S: + ctrl |= SRC4XXX_BUS_I2S; + break; + case SND_SOC_DAIFMT_LEFT_J: + ctrl |= SRC4XXX_BUS_LEFT_J; + break; + case SND_SOC_DAIFMT_RIGHT_J: + ctrl |= SRC4XXX_BUS_RIGHT_J_24; + break; + default: + return -EINVAL; + break; + } + + switch (fmt & SND_SOC_DAIFMT_INV_MASK) { + case SND_SOC_DAIFMT_NB_NF: + break; + default: + return -EINVAL; + break; + } + + regmap_update_bits(src4xxx->regmap, SRC4XXX_BUS_FMT(dai->id), + SRC4XXX_BUS_FMT_MS_MASK, ctrl); + + return 0; +} + +static int src4xxx_set_mclk_hz(struct snd_soc_dai *codec_dai, + int clk_id, unsigned int freq, int dir) +{ + struct snd_soc_component *component = codec_dai->component; + struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component); + + dev_info(component->dev, "changing mclk rate from %d to %d Hz\n", + src4xxx->mclk_hz, freq); + src4xxx->mclk_hz = freq; + + return 0; +} + +static int src4xxx_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct snd_soc_component *component = dai->component; + struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component); + unsigned int mclk_div; + int val, pj, jd, d; + int reg; + int ret; + + switch (dai->id) { + case SRC4XXX_PORTB: + reg = SRC4XXX_PORTB_CTL_06; + break; + default: + reg = SRC4XXX_PORTA_CTL_04; + break; + } + + if (src4xxx->master[dai->id]) { + mclk_div = src4xxx->mclk_hz/params_rate(params); + if (src4xxx->mclk_hz != mclk_div*params_rate(params)) { + dev_err(component->dev, + "mclk %d / rate %d has a remainder.\n", + src4xxx->mclk_hz, params_rate(params)); + return -EINVAL; + } + + val = ((int)mclk_div - 128) / 128; + if ((val < 0) | (val > 3)) { + dev_err(component->dev, + "div register setting %d is out of range\n", + val); + dev_err(component->dev, + "unsupported sample rate %d Hz for the master clock of %d Hz\n", + params_rate(params), src4xxx->mclk_hz); + return -EINVAL; + } + + /* set the TX DIV */ + ret = regmap_update_bits(src4xxx->regmap, + SRC4XXX_TX_CTL_07, SRC4XXX_TX_MCLK_DIV_MASK, + val<<SRC4XXX_TX_MCLK_DIV_SHIFT); + if (ret) { + dev_err(component->dev, + "Couldn't set the TX's div register to %d << %d = 0x%x\n", + val, SRC4XXX_TX_MCLK_DIV_SHIFT, + val<<SRC4XXX_TX_MCLK_DIV_SHIFT); + return ret; + } + + /* set the PLL for the digital receiver */ + switch (src4xxx->mclk_hz) { + case 24576000: + pj = 0x22; + jd = 0x00; + d = 0x00; + break; + case 22579200: + pj = 0x22; + jd = 0x1b; + d = 0xa3; + break; + default: + /* don't error out here, + * other parts of the chip are still functional + * Dummy initialize variables to avoid + * -Wsometimes-uninitialized from clang. + */ + dev_info(component->dev, + "Couldn't set the RCV PLL as this master clock rate is unknown. Chosen regmap values may not match real world values.\n"); + pj = 0x0; + jd = 0xff; + d = 0xff; + break; + } + ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_0F, pj); + if (ret < 0) + dev_err(component->dev, + "Failed to update PLL register 0x%x\n", + SRC4XXX_RCV_PLL_0F); + ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_10, jd); + if (ret < 0) + dev_err(component->dev, + "Failed to update PLL register 0x%x\n", + SRC4XXX_RCV_PLL_10); + ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_11, d); + if (ret < 0) + dev_err(component->dev, + "Failed to update PLL register 0x%x\n", + SRC4XXX_RCV_PLL_11); + + ret = regmap_update_bits(src4xxx->regmap, + SRC4XXX_TX_CTL_07, SRC4XXX_TX_MCLK_DIV_MASK, + val<<SRC4XXX_TX_MCLK_DIV_SHIFT); + if (ret < 0) { + dev_err(component->dev, + "Couldn't set the TX's div register to %d << %d = 0x%x\n", + val, SRC4XXX_TX_MCLK_DIV_SHIFT, + val<<SRC4XXX_TX_MCLK_DIV_SHIFT); + return ret; + } + + return regmap_update_bits(src4xxx->regmap, reg, + SRC4XXX_MCLK_DIV_MASK, val); + } else { + dev_info(dai->dev, "not setting up MCLK as not master\n"); + } + + return 0; +}; + +static const struct snd_soc_dai_ops src4xxx_dai_ops = { + .hw_params = src4xxx_hw_params, + .set_sysclk = src4xxx_set_mclk_hz, + .set_fmt = src4xxx_set_dai_fmt, +}; + +#define SRC4XXX_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S32_LE) +#define SRC4XXX_RATES (SNDRV_PCM_RATE_44100|SNDRV_PCM_RATE_48000|\ + SNDRV_PCM_RATE_88200|\ + SNDRV_PCM_RATE_96000|\ + SNDRV_PCM_RATE_176400|\ + SNDRV_PCM_RATE_192000) + +static struct snd_soc_dai_driver src4xxx_dai_driver[] = { + { + .id = SRC4XXX_PORTA, + .name = "src4xxx-portA", + .playback = { + .stream_name = "Playback A", + .channels_min = 2, + .channels_max = 2, + .rates = SRC4XXX_RATES, + .formats = SRC4XXX_FORMATS, + }, + .capture = { + .stream_name = "Capture A", + .channels_min = 2, + .channels_max = 2, + .rates = SRC4XXX_RATES, + .formats = SRC4XXX_FORMATS, + }, + .ops = &src4xxx_dai_ops, + }, + { + .id = SRC4XXX_PORTB, + .name = "src4xxx-portB", + .playback = { + .stream_name = "Playback B", + .channels_min = 2, + .channels_max = 2, + .rates = SRC4XXX_RATES, + .formats = SRC4XXX_FORMATS, + }, + .capture = { + .stream_name = "Capture B", + .channels_min = 2, + .channels_max = 2, + .rates = SRC4XXX_RATES, + .formats = SRC4XXX_FORMATS, + }, + .ops = &src4xxx_dai_ops, + }, +}; + +static const struct reg_default src4xxx_reg_defaults[] = { + { SRC4XXX_PWR_RST_01, 0x00 }, /* all powered down intially */ + { SRC4XXX_PORTA_CTL_03, 0x00 }, + { SRC4XXX_PORTA_CTL_04, 0x00 }, + { SRC4XXX_PORTB_CTL_05, 0x00 }, + { SRC4XXX_PORTB_CTL_06, 0x00 }, + { SRC4XXX_TX_CTL_07, 0x00 }, + { SRC4XXX_TX_CTL_08, 0x00 }, + { SRC4XXX_TX_CTL_09, 0x00 }, + { SRC4XXX_SRC_DIT_IRQ_MSK_0B, 0x00 }, + { SRC4XXX_SRC_DIT_IRQ_MODE_0C, 0x00 }, + { SRC4XXX_RCV_CTL_0D, 0x00 }, + { SRC4XXX_RCV_CTL_0E, 0x00 }, + { SRC4XXX_RCV_PLL_0F, 0x00 }, /* not spec. in the datasheet */ + { SRC4XXX_RCV_PLL_10, 0xff }, /* not spec. in the datasheet */ + { SRC4XXX_RCV_PLL_11, 0xff }, /* not spec. in the datasheet */ + { SRC4XXX_RVC_IRQ_MSK_16, 0x00 }, + { SRC4XXX_RVC_IRQ_MSK_17, 0x00 }, + { SRC4XXX_RVC_IRQ_MODE_18, 0x00 }, + { SRC4XXX_RVC_IRQ_MODE_19, 0x00 }, + { SRC4XXX_RVC_IRQ_MODE_1A, 0x00 }, + { SRC4XXX_GPIO_1_1B, 0x00 }, + { SRC4XXX_GPIO_2_1C, 0x00 }, + { SRC4XXX_GPIO_3_1D, 0x00 }, + { SRC4XXX_GPIO_4_1E, 0x00 }, + { SRC4XXX_SCR_CTL_2D, 0x00 }, + { SRC4XXX_SCR_CTL_2E, 0x00 }, + { SRC4XXX_SCR_CTL_2F, 0x00 }, + { SRC4XXX_SCR_CTL_30, 0x00 }, + { SRC4XXX_SCR_CTL_31, 0x00 }, +}; + +int src4xxx_probe(struct device *dev, struct regmap *regmap, + void (*switch_mode)(struct device *dev)) +{ + struct src4xxx *src4xxx; + int ret; + + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + src4xxx = devm_kzalloc(dev, sizeof(*src4xxx), GFP_KERNEL); + if (!src4xxx) + return -ENOMEM; + + src4xxx->regmap = regmap; + src4xxx->dev = dev; + src4xxx->mclk_hz = 0; /* mclk has not been configured yet */ + dev_set_drvdata(dev, src4xxx); + + ret = regmap_write(regmap, SRC4XXX_PWR_RST_01, SRC4XXX_RESET); + if (ret < 0) + dev_err(dev, "Failed to issue reset: %d\n", ret); + usleep_range(1, 500); /* sleep for more then 500 ns */ + ret = regmap_write(regmap, SRC4XXX_PWR_RST_01, SRC4XXX_POWER_DOWN); + if (ret < 0) + dev_err(dev, "Failed to decommission reset: %d\n", ret); + usleep_range(500, 1000); /* sleep for 500 us or more */ + + ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_PWR_RST_01, + SRC4XXX_POWER_ENABLE, SRC4XXX_POWER_ENABLE); + if (ret < 0) + dev_err(dev, "Failed to port A and B : %d\n", ret); + + /* set receiver to use master clock (rcv mclk is most likely jittery) */ + ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_RCV_CTL_0D, + SRC4XXX_RXCLK_MCLK, SRC4XXX_RXCLK_MCLK); + if (ret < 0) + dev_err(dev, + "Failed to enable mclk as the PLL1 DIR reference : %d\n", ret); + + /* default to leaving the PLL2 running on loss of lock, divide by 8 */ + ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_RCV_CTL_0E, + SRC4XXX_PLL2_DIV_8 | SRC4XXX_REC_MCLK_EN | SRC4XXX_PLL2_LOL, + SRC4XXX_PLL2_DIV_8 | SRC4XXX_REC_MCLK_EN | SRC4XXX_PLL2_LOL); + if (ret < 0) + dev_err(dev, "Failed to enable mclk rec and div : %d\n", ret); + + ret = devm_snd_soc_register_component(dev, &src4xxx_driver, + src4xxx_dai_driver, ARRAY_SIZE(src4xxx_dai_driver)); + if (ret == 0) + dev_info(dev, "src4392 probe ok %d\n", ret); + return ret; +} +EXPORT_SYMBOL_GPL(src4xxx_probe); + +static bool src4xxx_volatile_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case SRC4XXX_RES_00: + case SRC4XXX_GLOBAL_ITR_STS_02: + case SRC4XXX_SRC_DIT_STS_0A: + case SRC4XXX_NON_AUDIO_D_12: + case SRC4XXX_RVC_STS_13: + case SRC4XXX_RVC_STS_14: + case SRC4XXX_RVC_STS_15: + case SRC4XXX_SUB_CODE_1F: + case SRC4XXX_SUB_CODE_20: + case SRC4XXX_SUB_CODE_21: + case SRC4XXX_SUB_CODE_22: + case SRC4XXX_SUB_CODE_23: + case SRC4XXX_SUB_CODE_24: + case SRC4XXX_SUB_CODE_25: + case SRC4XXX_SUB_CODE_26: + case SRC4XXX_SUB_CODE_27: + case SRC4XXX_SUB_CODE_28: + case SRC4XXX_PC_PREAMBLE_HI_29: + case SRC4XXX_PC_PREAMBLE_LO_2A: + case SRC4XXX_PD_PREAMBLE_HI_2B: + case SRC4XXX_PC_PREAMBLE_LO_2C: + case SRC4XXX_IO_RATIO_32: + case SRC4XXX_IO_RATIO_33: + return true; + } + + if (reg > SRC4XXX_IO_RATIO_33 && reg < SRC4XXX_PAGE_SEL_7F) + return true; + + return false; +} + +const struct regmap_config src4xxx_regmap_config = { + .val_bits = 8, + .reg_bits = 8, + .max_register = SRC4XXX_IO_RATIO_33, + + .reg_defaults = src4xxx_reg_defaults, + .num_reg_defaults = ARRAY_SIZE(src4xxx_reg_defaults), + .volatile_reg = src4xxx_volatile_register, + .cache_type = REGCACHE_RBTREE, +}; +EXPORT_SYMBOL_GPL(src4xxx_regmap_config); + +MODULE_DESCRIPTION("ASoC SRC4XXX CODEC driver"); +MODULE_AUTHOR("Matt Flax <flatmax@flatmax.com>"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/codecs/src4xxx.h b/sound/soc/codecs/src4xxx.h new file mode 100644 index 0000000000000000000000000000000000000000..5bf778fb9945e1dc312f7ee9309d251d753094a2 --- /dev/null +++ b/sound/soc/codecs/src4xxx.h @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// src4xxx.h -- SRC4XXX ALSA SoC audio driver +// +// Copyright 2021-2022 Deqx Pty Ltd +// Author: Matt R Flax <flatmax@flatmax.com> + +#ifndef __SRC4XXX_H__ +#define __SRC4XXX_H__ + +#define SRC4XXX_RES_00 0x00 +#define SRC4XXX_PWR_RST_01 0x01 +#define SRC4XXX_RESET 0x80 +#define SRC4XXX_POWER_DOWN 0x00 +#define SRC4XXX_POWER_ENABLE 0x20 +#define SRC4XXX_ENABLE_SRC 0x1 +#define SRC4XXX_ENABLE_SRC_SHIFT 0 +#define SRC4XXX_ENABLE_DIR 0x2 +#define SRC4XXX_ENABLE_DIR_SHIFT 1 +#define SRC4XXX_ENABLE_DIT 0x4 +#define SRC4XXX_ENABLE_DIT_SHIFT 2 +#define SRC4XXX_ENABLE_PORT_B 0x8 +#define SRC4XXX_ENABLE_PORT_B_SHIFT 3 +#define SRC4XXX_ENABLE_PORT_A 0x10 +#define SRC4XXX_ENABLE_PORT_A_SHIFT 4 + +#define SRC4XXX_PORTA_CTL_03 0x03 +#define SRC4XXX_BUS_MASTER 0x8 +#define SRC4XXX_BUS_LEFT_J 0x0 +#define SRC4XXX_BUS_I2S 0x1 +#define SRC4XXX_BUS_RIGHT_J_16 0x4 +#define SRC4XXX_BUS_RIGHT_J_18 0x5 +#define SRC4XXX_BUS_RIGHT_J_20 0x6 +#define SRC4XXX_BUS_RIGHT_J_24 0x7 +#define SRC4XXX_BUS_FMT_MS_MASK 0xf + +#define SRC4XXX_PORTA_CTL_04 0x04 +#define SRC4XXX_MCLK_DIV_MASK 0x3 + +#define SRC4XXX_BUS_FMT(id) (SRC4XXX_PORTA_CTL_03+2*id) +#define SRC4XXX_BUS_CLK(id) (SRC4XXX_PORTA_CTL_04+2*id) + +#define SRC4XXX_PORTB_CTL_05 0x05 +#define SRC4XXX_PORTB_CTL_06 0x06 + +#define SRC4XXX_TX_CTL_07 0x07 +#define SRC4XXX_TX_MCLK_DIV_MASK 0x60 +#define SRC4XXX_TX_MCLK_DIV_SHIFT 5 + +#define SRC4XXX_TX_CTL_08 0x08 +#define SRC4XXX_TX_CTL_09 0x09 +#define SRC4XXX_SRC_DIT_IRQ_MSK_0B 0x0B +#define SRC4XXX_SRC_BTI_EN 0x01 +#define SRC4XXX_SRC_TSLIP_EN 0x02 +#define SRC4XXX_SRC_DIT_IRQ_MODE_0C 0x0C +#define SRC4XXX_RCV_CTL_0D 0x0D +#define SRC4XXX_RXCLK_RXCKI 0x0 +#define SRC4XXX_RXCLK_MCLK 0x8 +#define SRC4XXX_RCV_CTL_0E 0x0E +#define SRC4XXX_REC_MCLK_EN 0x1 +#define SRC4XXX_PLL2_DIV_0 (0x0<<1) +#define SRC4XXX_PLL2_DIV_2 (0x1<<1) +#define SRC4XXX_PLL2_DIV_4 (0x2<<1) +#define SRC4XXX_PLL2_DIV_8 (0x3<<1) +#define SRC4XXX_PLL2_LOL 0x8 +#define SRC4XXX_RCV_PLL_0F 0x0F +#define SRC4XXX_RCV_PLL_10 0x10 +#define SRC4XXX_RCV_PLL_11 0x11 +#define SRC4XXX_RVC_IRQ_MSK_16 0x16 +#define SRC4XXX_RVC_IRQ_MSK_17 0x17 +#define SRC4XXX_RVC_IRQ_MODE_18 0x18 +#define SRC4XXX_RVC_IRQ_MODE_19 0x19 +#define SRC4XXX_RVC_IRQ_MODE_1A 0x1A +#define SRC4XXX_GPIO_1_1B 0x1B +#define SRC4XXX_GPIO_2_1C 0x1C +#define SRC4XXX_GPIO_3_1D 0x1D +#define SRC4XXX_GPIO_4_1E 0x1E +#define SRC4XXX_SCR_CTL_2D 0x2D +#define SRC4XXX_SCR_CTL_2E 0x2E +#define SRC4XXX_SCR_CTL_2F 0x2F +#define SRC4XXX_SCR_CTL_30 0x30 +#define SRC4XXX_SCR_CTL_31 0x31 +#define SRC4XXX_PAGE_SEL_7F 0x7F + +// read only registers +#define SRC4XXX_GLOBAL_ITR_STS_02 0x02 +#define SRC4XXX_SRC_DIT_STS_0A 0x0A +#define SRC4XXX_NON_AUDIO_D_12 0x12 +#define SRC4XXX_RVC_STS_13 0x13 +#define SRC4XXX_RVC_STS_14 0x14 +#define SRC4XXX_RVC_STS_15 0x15 +#define SRC4XXX_SUB_CODE_1F 0x1F +#define SRC4XXX_SUB_CODE_20 0x20 +#define SRC4XXX_SUB_CODE_21 0x21 +#define SRC4XXX_SUB_CODE_22 0x22 +#define SRC4XXX_SUB_CODE_23 0x23 +#define SRC4XXX_SUB_CODE_24 0x24 +#define SRC4XXX_SUB_CODE_25 0x25 +#define SRC4XXX_SUB_CODE_26 0x26 +#define SRC4XXX_SUB_CODE_27 0x27 +#define SRC4XXX_SUB_CODE_28 0x28 +#define SRC4XXX_PC_PREAMBLE_HI_29 0x29 +#define SRC4XXX_PC_PREAMBLE_LO_2A 0x2A +#define SRC4XXX_PD_PREAMBLE_HI_2B 0x2B +#define SRC4XXX_PC_PREAMBLE_LO_2C 0x2C +#define SRC4XXX_IO_RATIO_32 0x32 +#define SRC4XXX_IO_RATIO_33 0x33 + +int src4xxx_probe(struct device *dev, struct regmap *regmap, + void (*switch_mode)(struct device *dev)); +extern const struct regmap_config src4xxx_regmap_config; + +#endif /* __SRC4XXX_H__ */ diff --git a/sound/soc/codecs/ssm2518.c b/sound/soc/codecs/ssm2518.c index 6d8847848299003a95b1a8813185fef3eabdddca..22cb3b7c828305f42e0125685b66d6140b0e74b3 100644 --- a/sound/soc/codecs/ssm2518.c +++ b/sound/soc/codecs/ssm2518.c @@ -6,14 +6,13 @@ * Author: Lars-Peter Clausen <lars@metafoo.de> */ +#include <linux/err.h> #include <linux/module.h> #include <linux/init.h> #include <linux/i2c.h> #include <linux/regmap.h> #include <linux/slab.h> -#include <linux/gpio.h> -#include <linux/of_gpio.h> -#include <linux/platform_data/ssm2518.h> +#include <linux/gpio/consumer.h> #include <sound/core.h> #include <sound/pcm.h> #include <sound/pcm_params.h> @@ -114,7 +113,7 @@ struct ssm2518 { unsigned int sysclk; const struct snd_pcm_hw_constraint_list *constraints; - int enable_gpio; + struct gpio_desc *enable_gpio; }; static const struct reg_default ssm2518_reg_defaults[] = { @@ -483,8 +482,8 @@ static int ssm2518_set_power(struct ssm2518 *ssm2518, bool enable) regcache_mark_dirty(ssm2518->regmap); } - if (gpio_is_valid(ssm2518->enable_gpio)) - gpio_set_value(ssm2518->enable_gpio, enable); + if (ssm2518->enable_gpio) + gpiod_set_value_cansleep(ssm2518->enable_gpio, enable); regcache_cache_only(ssm2518->regmap, !enable); @@ -736,7 +735,6 @@ static const struct regmap_config ssm2518_regmap_config = { static int ssm2518_i2c_probe(struct i2c_client *i2c) { - struct ssm2518_platform_data *pdata = i2c->dev.platform_data; struct ssm2518 *ssm2518; int ret; @@ -744,22 +742,14 @@ static int ssm2518_i2c_probe(struct i2c_client *i2c) if (ssm2518 == NULL) return -ENOMEM; - if (pdata) { - ssm2518->enable_gpio = pdata->enable_gpio; - } else if (i2c->dev.of_node) { - ssm2518->enable_gpio = of_get_gpio(i2c->dev.of_node, 0); - if (ssm2518->enable_gpio < 0 && ssm2518->enable_gpio != -ENOENT) - return ssm2518->enable_gpio; - } else { - ssm2518->enable_gpio = -1; - } + /* Start with enabling the chip */ + ssm2518->enable_gpio = devm_gpiod_get_optional(&i2c->dev, NULL, + GPIOD_OUT_HIGH); + ret = PTR_ERR_OR_ZERO(ssm2518->enable_gpio); + if (ret) + return ret; - if (gpio_is_valid(ssm2518->enable_gpio)) { - ret = devm_gpio_request_one(&i2c->dev, ssm2518->enable_gpio, - GPIOF_OUT_INIT_HIGH, "SSM2518 nSD"); - if (ret) - return ret; - } + gpiod_set_consumer_name(ssm2518->enable_gpio, "SSM2518 nSD"); i2c_set_clientdata(i2c, ssm2518); diff --git a/sound/soc/codecs/sta350.c b/sound/soc/codecs/sta350.c index 7b2c5b57d5d4529ad98cd0f595abe4ab4e177ef5..9ed13aeb3cbdc6a10d8a106d399d017ff5ed54c4 100644 --- a/sound/soc/codecs/sta350.c +++ b/sound/soc/codecs/sta350.c @@ -1242,10 +1242,8 @@ static int sta350_i2c_probe(struct i2c_client *i2c) return ret; } -static int sta350_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void sta350_i2c_remove(struct i2c_client *client) +{} static const struct i2c_device_id sta350_i2c_id[] = { { "sta350", 0 }, diff --git a/sound/soc/codecs/sti-sas.c b/sound/soc/codecs/sti-sas.c index f076878908eec5ec1f07e18fce715a8441edd09b..99545bcb2ba9edba86c223ea80ea29ffd174d4ca 100644 --- a/sound/soc/codecs/sti-sas.c +++ b/sound/soc/codecs/sti-sas.c @@ -96,11 +96,8 @@ static int sti_sas_write_reg(void *context, unsigned int reg, unsigned int value) { struct sti_sas_data *drvdata = context; - int status; - - status = regmap_write(drvdata->dac.regmap, reg, value); - return status; + return regmap_write(drvdata->dac.regmap, reg, value); } static int sti_sas_init_sas_registers(struct snd_soc_component *component, @@ -385,11 +382,8 @@ static int sti_sas_resume(struct snd_soc_component *component) static int sti_sas_component_probe(struct snd_soc_component *component) { struct sti_sas_data *drvdata = dev_get_drvdata(component->dev); - int ret; - ret = sti_sas_init_sas_registers(component, drvdata); - - return ret; + return sti_sas_init_sas_registers(component, drvdata); } static struct snd_soc_component_driver sti_sas_driver = { diff --git a/sound/soc/codecs/tas2552.c b/sound/soc/codecs/tas2552.c index 8bd667da876725944fcce9cc74a1d42dc5a2e516..59a4ea5f6e30575dad2f8fb8262a7995965f3aab 100644 --- a/sound/soc/codecs/tas2552.c +++ b/sound/soc/codecs/tas2552.c @@ -736,10 +736,9 @@ static int tas2552_probe(struct i2c_client *client) return ret; } -static int tas2552_i2c_remove(struct i2c_client *client) +static void tas2552_i2c_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); - return 0; } static const struct i2c_device_id tas2552_id[] = { diff --git a/sound/soc/codecs/tas2562.c b/sound/soc/codecs/tas2562.c index dc088a1c67213173ffe3f76752f37fe129b129c8..b486d0bd86c991c76ad47905c5fe4d7da996460d 100644 --- a/sound/soc/codecs/tas2562.c +++ b/sound/soc/codecs/tas2562.c @@ -54,6 +54,8 @@ struct tas2562_data { int i_sense_slot; int volume_lvl; int model_id; + bool dac_powered; + bool unmuted; }; enum tas256x_model { @@ -63,39 +65,6 @@ enum tas256x_model { TAS2110, }; -static int tas2562_set_bias_level(struct snd_soc_component *component, - enum snd_soc_bias_level level) -{ - struct tas2562_data *tas2562 = - snd_soc_component_get_drvdata(component); - - switch (level) { - case SND_SOC_BIAS_ON: - snd_soc_component_update_bits(component, - TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, TAS2562_ACTIVE); - break; - case SND_SOC_BIAS_STANDBY: - case SND_SOC_BIAS_PREPARE: - snd_soc_component_update_bits(component, - TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, TAS2562_MUTE); - break; - case SND_SOC_BIAS_OFF: - snd_soc_component_update_bits(component, - TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, TAS2562_SHUTDOWN); - break; - - default: - dev_err(tas2562->dev, - "wrong power level setting %d\n", level); - return -EINVAL; - } - - return 0; -} - static int tas2562_set_samplerate(struct tas2562_data *tas2562, int samplerate) { int samp_rate; @@ -384,30 +353,43 @@ static int tas2562_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) return 0; } +static int tas2562_update_pwr_ctrl(struct tas2562_data *tas2562) +{ + struct snd_soc_component *component = tas2562->component; + unsigned int val; + int ret; + + if (tas2562->dac_powered) + val = tas2562->unmuted ? + TAS2562_ACTIVE : TAS2562_MUTE; + else + val = TAS2562_SHUTDOWN; + + ret = snd_soc_component_update_bits(component, TAS2562_PWR_CTRL, + TAS2562_MODE_MASK, val); + if (ret < 0) + return ret; + + return 0; +} + static int tas2562_mute(struct snd_soc_dai *dai, int mute, int direction) { - struct snd_soc_component *component = dai->component; + struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(dai->component); - return snd_soc_component_update_bits(component, TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, - mute ? TAS2562_MUTE : 0); + tas2562->unmuted = !mute; + return tas2562_update_pwr_ctrl(tas2562); } static int tas2562_codec_probe(struct snd_soc_component *component) { struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); - int ret; tas2562->component = component; if (tas2562->sdz_gpio) gpiod_set_value_cansleep(tas2562->sdz_gpio, 1); - ret = snd_soc_component_update_bits(component, TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, TAS2562_MUTE); - if (ret < 0) - return ret; - return 0; } @@ -457,35 +439,23 @@ static int tas2562_dac_event(struct snd_soc_dapm_widget *w, struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); - int ret; + int ret = 0; switch (event) { case SND_SOC_DAPM_POST_PMU: - ret = snd_soc_component_update_bits(component, - TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, - TAS2562_MUTE); - if (ret) - goto end; + tas2562->dac_powered = true; + ret = tas2562_update_pwr_ctrl(tas2562); break; case SND_SOC_DAPM_PRE_PMD: - ret = snd_soc_component_update_bits(component, - TAS2562_PWR_CTRL, - TAS2562_MODE_MASK, - TAS2562_SHUTDOWN); - if (ret) - goto end; + tas2562->dac_powered = false; + ret = tas2562_update_pwr_ctrl(tas2562); break; default: dev_err(tas2562->dev, "Not supported evevt\n"); return -EINVAL; } -end: - if (ret < 0) - return ret; - - return 0; + return ret; } static int tas2562_volume_control_get(struct snd_kcontrol *kcontrol, @@ -579,7 +549,6 @@ static const struct snd_soc_component_driver soc_component_dev_tas2110 = { .probe = tas2562_codec_probe, .suspend = tas2562_suspend, .resume = tas2562_resume, - .set_bias_level = tas2562_set_bias_level, .controls = tas2562_snd_controls, .num_controls = ARRAY_SIZE(tas2562_snd_controls), .dapm_widgets = tas2110_dapm_widgets, @@ -618,7 +587,6 @@ static const struct snd_soc_component_driver soc_component_dev_tas2562 = { .probe = tas2562_codec_probe, .suspend = tas2562_suspend, .resume = tas2562_resume, - .set_bias_level = tas2562_set_bias_level, .controls = tas2562_snd_controls, .num_controls = ARRAY_SIZE(tas2562_snd_controls), .dapm_widgets = tas2562_dapm_widgets, diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 846d9d3ecc9de1136ae3f5e25d5e5a041e9bbb4c..51b87a936179863ddd131bb17077d6efbaa7d02f 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -31,11 +31,66 @@ struct tas2764_priv { struct gpio_desc *sdz_gpio; struct regmap *regmap; struct device *dev; + int irq; int v_sense_slot; int i_sense_slot; + + bool dac_powered; + bool unmuted; +}; + +static const char *tas2764_int_ltch0_msgs[8] = { + "fault: over temperature", /* INT_LTCH0 & BIT(0) */ + "fault: over current", + "fault: bad TDM clock", + "limiter active", + "fault: PVDD below limiter inflection point", + "fault: limiter max attenuation", + "fault: BOP infinite hold", + "fault: BOP mute", /* INT_LTCH0 & BIT(7) */ +}; + +static const unsigned int tas2764_int_readout_regs[6] = { + TAS2764_INT_LTCH0, + TAS2764_INT_LTCH1, + TAS2764_INT_LTCH1_0, + TAS2764_INT_LTCH2, + TAS2764_INT_LTCH3, + TAS2764_INT_LTCH4, }; +static irqreturn_t tas2764_irq(int irq, void *data) +{ + struct tas2764_priv *tas2764 = data; + u8 latched[6] = {0, 0, 0, 0, 0, 0}; + int ret = IRQ_NONE; + int i; + + for (i = 0; i < ARRAY_SIZE(latched); i++) + latched[i] = snd_soc_component_read(tas2764->component, + tas2764_int_readout_regs[i]); + + for (i = 0; i < 8; i++) { + if (latched[0] & BIT(i)) { + dev_crit_ratelimited(tas2764->dev, "%s\n", + tas2764_int_ltch0_msgs[i]); + ret = IRQ_HANDLED; + } + } + + if (latched[0]) { + dev_err_ratelimited(tas2764->dev, "other context to the fault: %02x,%02x,%02x,%02x,%02x", + latched[1], latched[2], latched[3], latched[4], latched[5]); + snd_soc_component_update_bits(tas2764->component, + TAS2764_INT_CLK_CFG, + TAS2764_INT_CLK_CFG_IRQZ_CLR, + TAS2764_INT_CLK_CFG_IRQZ_CLR); + } + + return ret; +} + static void tas2764_reset(struct tas2764_priv *tas2764) { if (tas2764->reset_gpio) { @@ -50,34 +105,22 @@ static void tas2764_reset(struct tas2764_priv *tas2764) usleep_range(1000, 2000); } -static int tas2764_set_bias_level(struct snd_soc_component *component, - enum snd_soc_bias_level level) +static int tas2764_update_pwr_ctrl(struct tas2764_priv *tas2764) { - struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + struct snd_soc_component *component = tas2764->component; + unsigned int val; + int ret; - switch (level) { - case SND_SOC_BIAS_ON: - snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_ACTIVE); - break; - case SND_SOC_BIAS_STANDBY: - case SND_SOC_BIAS_PREPARE: - snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_MUTE); - break; - case SND_SOC_BIAS_OFF: - snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_SHUTDOWN); - break; + if (tas2764->dac_powered) + val = tas2764->unmuted ? + TAS2764_PWR_CTRL_ACTIVE : TAS2764_PWR_CTRL_MUTE; + else + val = TAS2764_PWR_CTRL_SHUTDOWN; - default: - dev_err(tas2764->dev, - "wrong power level setting %d\n", level); - return -EINVAL; - } + ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, + TAS2764_PWR_CTRL_MASK, val); + if (ret < 0) + return ret; return 0; } @@ -114,9 +157,7 @@ static int tas2764_codec_resume(struct snd_soc_component *component) usleep_range(1000, 2000); } - ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_ACTIVE); + ret = tas2764_update_pwr_ctrl(tas2764); if (ret < 0) return ret; @@ -150,14 +191,12 @@ static int tas2764_dac_event(struct snd_soc_dapm_widget *w, switch (event) { case SND_SOC_DAPM_POST_PMU: - ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_MUTE); + tas2764->dac_powered = true; + ret = tas2764_update_pwr_ctrl(tas2764); break; case SND_SOC_DAPM_PRE_PMD: - ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_SHUTDOWN); + tas2764->dac_powered = false; + ret = tas2764_update_pwr_ctrl(tas2764); break; default: dev_err(tas2764->dev, "Unsupported event\n"); @@ -202,17 +241,11 @@ static const struct snd_soc_dapm_route tas2764_audio_map[] = { static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction) { - struct snd_soc_component *component = dai->component; - int ret; - - ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - mute ? TAS2764_PWR_CTRL_MUTE : 0); + struct tas2764_priv *tas2764 = + snd_soc_component_get_drvdata(dai->component); - if (ret < 0) - return ret; - - return 0; + tas2764->unmuted = !mute; + return tas2764_update_pwr_ctrl(tas2764); } static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) @@ -485,7 +518,7 @@ static struct snd_soc_dai_driver tas2764_dai_driver[] = { .id = 0, .playback = { .stream_name = "ASI1 Playback", - .channels_min = 2, + .channels_min = 1, .channels_max = 2, .rates = TAS2764_RATES, .formats = TAS2764_FORMATS, @@ -516,6 +549,34 @@ static int tas2764_codec_probe(struct snd_soc_component *component) tas2764_reset(tas2764); + if (tas2764->irq) { + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK0, 0xff); + if (ret < 0) + return ret; + + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK1, 0xff); + if (ret < 0) + return ret; + + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK2, 0xff); + if (ret < 0) + return ret; + + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK3, 0xff); + if (ret < 0) + return ret; + + ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK4, 0xff); + if (ret < 0) + return ret; + + ret = devm_request_threaded_irq(tas2764->dev, tas2764->irq, NULL, tas2764_irq, + IRQF_ONESHOT | IRQF_SHARED | IRQF_TRIGGER_LOW, + "tas2764", tas2764); + if (ret) + dev_warn(tas2764->dev, "failed to request IRQ: %d\n", ret); + } + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, TAS2764_TDM_CFG5_VSNS_ENABLE, 0); if (ret < 0) @@ -526,30 +587,33 @@ static int tas2764_codec_probe(struct snd_soc_component *component) if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, - TAS2764_PWR_CTRL_MASK, - TAS2764_PWR_CTRL_MUTE); - if (ret < 0) - return ret; - return 0; } static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2764_playback_volume, -10050, 50, 1); +static const char * const tas2764_hpf_texts[] = { + "Disabled", "2 Hz", "50 Hz", "100 Hz", "200 Hz", + "400 Hz", "800 Hz" +}; + +static SOC_ENUM_SINGLE_DECL( + tas2764_hpf_enum, TAS2764_DC_BLK0, + TAS2764_DC_BLK0_HPF_FREQ_PB_SHIFT, tas2764_hpf_texts); + static const struct snd_kcontrol_new tas2764_snd_controls[] = { SOC_SINGLE_TLV("Speaker Volume", TAS2764_DVC, 0, TAS2764_DVC_MAX, 1, tas2764_playback_volume), SOC_SINGLE_TLV("Amp Gain Volume", TAS2764_CHNL_0, 1, 0x14, 0, tas2764_digital_tlv), + SOC_ENUM("HPF Corner Frequency", tas2764_hpf_enum), }; static const struct snd_soc_component_driver soc_component_driver_tas2764 = { .probe = tas2764_codec_probe, .suspend = tas2764_codec_suspend, .resume = tas2764_codec_resume, - .set_bias_level = tas2764_set_bias_level, .controls = tas2764_snd_controls, .num_controls = ARRAY_SIZE(tas2764_snd_controls), .dapm_widgets = tas2764_dapm_widgets, @@ -585,9 +649,21 @@ static const struct regmap_range_cfg tas2764_regmap_ranges[] = { }, }; +static bool tas2764_volatile_register(struct device *dev, unsigned int reg) +{ + switch (reg) { + case TAS2764_INT_LTCH0 ... TAS2764_INT_LTCH4: + case TAS2764_INT_CLK_CFG: + return true; + default: + return false; + } +} + static const struct regmap_config tas2764_i2c_regmap = { .reg_bits = 8, .val_bits = 8, + .volatile_reg = tas2764_volatile_register, .reg_defaults = tas2764_reg_defaults, .num_reg_defaults = ARRAY_SIZE(tas2764_reg_defaults), .cache_type = REGCACHE_RBTREE, @@ -641,6 +717,7 @@ static int tas2764_i2c_probe(struct i2c_client *client) return -ENOMEM; tas2764->dev = &client->dev; + tas2764->irq = client->irq; i2c_set_clientdata(client, tas2764); dev_set_drvdata(&client->dev, tas2764); diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index f015f22a083b56078277627628571223f08de60c..168af772a898ffd1c5e96c50df77ff6225f704cd 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -33,6 +33,10 @@ #define TAS2764_VSENSE_POWER_EN 3 #define TAS2764_ISENSE_POWER_EN 4 +/* DC Blocker Control */ +#define TAS2764_DC_BLK0 TAS2764_REG(0x0, 0x04) +#define TAS2764_DC_BLK0_HPF_FREQ_PB_SHIFT 0 + /* Digital Volume Control */ #define TAS2764_DVC TAS2764_REG(0X0, 0x1a) #define TAS2764_DVC_MAX 0xc9 @@ -87,4 +91,23 @@ #define TAS2764_TDM_CFG6_ISNS_ENABLE BIT(6) #define TAS2764_TDM_CFG6_50_MASK GENMASK(5, 0) +/* Interrupt Masks */ +#define TAS2764_INT_MASK0 TAS2764_REG(0x0, 0x3b) +#define TAS2764_INT_MASK1 TAS2764_REG(0x0, 0x3c) +#define TAS2764_INT_MASK2 TAS2764_REG(0x0, 0x40) +#define TAS2764_INT_MASK3 TAS2764_REG(0x0, 0x41) +#define TAS2764_INT_MASK4 TAS2764_REG(0x0, 0x3d) + +/* Latched Fault Registers */ +#define TAS2764_INT_LTCH0 TAS2764_REG(0x0, 0x49) +#define TAS2764_INT_LTCH1 TAS2764_REG(0x0, 0x4a) +#define TAS2764_INT_LTCH1_0 TAS2764_REG(0x0, 0x4b) +#define TAS2764_INT_LTCH2 TAS2764_REG(0x0, 0x4f) +#define TAS2764_INT_LTCH3 TAS2764_REG(0x0, 0x50) +#define TAS2764_INT_LTCH4 TAS2764_REG(0x0, 0x51) + +/* Clock/IRQ Settings */ +#define TAS2764_INT_CLK_CFG TAS2764_REG(0x0, 0x5c) +#define TAS2764_INT_CLK_CFG_IRQZ_CLR BIT(2) + #endif /* __TAS2764__ */ diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index bb653b6641466c4c974d1bc26c4fd43fe415e0cf..b6765235a4b3d8e000bcbeef735fdcfc26c3d99c 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -495,6 +495,8 @@ static struct snd_soc_dai_driver tas2770_dai_driver[] = { }, }; +static const struct regmap_config tas2770_i2c_regmap; + static int tas2770_codec_probe(struct snd_soc_component *component) { struct tas2770_priv *tas2770 = @@ -508,6 +510,7 @@ static int tas2770_codec_probe(struct snd_soc_component *component) } tas2770_reset(tas2770); + regmap_reinit_cache(tas2770->regmap, &tas2770_i2c_regmap); return 0; } diff --git a/sound/soc/codecs/tas5086.c b/sound/soc/codecs/tas5086.c index a864984225bc46c86914f75d14403939015480c3..22143cc5afa706f76df0796427c35df126d9d34c 100644 --- a/sound/soc/codecs/tas5086.c +++ b/sound/soc/codecs/tas5086.c @@ -981,10 +981,8 @@ static int tas5086_i2c_probe(struct i2c_client *i2c) return ret; } -static int tas5086_i2c_remove(struct i2c_client *i2c) -{ - return 0; -} +static void tas5086_i2c_remove(struct i2c_client *i2c) +{} static struct i2c_driver tas5086_i2c_driver = { .driver = { diff --git a/sound/soc/codecs/tas571x.c b/sound/soc/codecs/tas571x.c index 4e7f20db57c4dda92678fa358528cf63a6eef4ab..84ec1b5276465134de1ab8cdff00ad08043a5dd5 100644 --- a/sound/soc/codecs/tas571x.c +++ b/sound/soc/codecs/tas571x.c @@ -884,13 +884,11 @@ disable_regs: return ret; } -static int tas571x_i2c_remove(struct i2c_client *client) +static void tas571x_i2c_remove(struct i2c_client *client) { struct tas571x_private *priv = i2c_get_clientdata(client); regulator_bulk_disable(priv->chip->num_supply_names, priv->supplies); - - return 0; } static const struct of_device_id tas571x_of_match[] __maybe_unused = { diff --git a/sound/soc/codecs/tas5805m.c b/sound/soc/codecs/tas5805m.c index b1bb614534f7437860ecf4b825e6e1b64c4c2ffe..beb4ec629a03c553ec78d476286e48d5873c5be2 100644 --- a/sound/soc/codecs/tas5805m.c +++ b/sound/soc/codecs/tas5805m.c @@ -522,7 +522,7 @@ static int tas5805m_i2c_probe(struct i2c_client *i2c) return 0; } -static int tas5805m_i2c_remove(struct i2c_client *i2c) +static void tas5805m_i2c_remove(struct i2c_client *i2c) { struct device *dev = &i2c->dev; struct tas5805m_priv *tas5805m = dev_get_drvdata(dev); @@ -531,7 +531,6 @@ static int tas5805m_i2c_remove(struct i2c_client *i2c) gpiod_set_value(tas5805m->gpio_pdn_n, 0); usleep_range(10000, 15000); regulator_disable(tas5805m->pvdd); - return 0; } static const struct i2c_device_id tas5805m_i2c_id[] = { diff --git a/sound/soc/codecs/tas6424.c b/sound/soc/codecs/tas6424.c index 63d2983c3fcf4dd64839f8f07d8caf8a0f46ca00..f8ff69fa2549c9f3de436c18dde73fca9514b27e 100644 --- a/sound/soc/codecs/tas6424.c +++ b/sound/soc/codecs/tas6424.c @@ -774,7 +774,7 @@ disable_regs: return ret; } -static int tas6424_i2c_remove(struct i2c_client *client) +static void tas6424_i2c_remove(struct i2c_client *client) { struct device *dev = &client->dev; struct tas6424_data *tas6424 = dev_get_drvdata(dev); @@ -790,8 +790,6 @@ static int tas6424_i2c_remove(struct i2c_client *client) tas6424->supplies); if (ret < 0) dev_err(dev, "unable to disable supplies: %d\n", ret); - - return 0; } static const struct i2c_device_id tas6424_i2c_ids[] = { diff --git a/sound/soc/codecs/tfa989x.c b/sound/soc/codecs/tfa989x.c index 1c27429b9af646d61daa4f81f4dcfe3d1a67fae8..b853507e65a846e9361d0294ed798052eb747c20 100644 --- a/sound/soc/codecs/tfa989x.c +++ b/sound/soc/codecs/tfa989x.c @@ -193,7 +193,7 @@ static int tfa9890_init(struct regmap *regmap) { int ret; - /* unhide keys to allow updating them */ + /* temporarily allow access to hidden registers */ ret = regmap_write(regmap, TFA989X_HIDE_UNHIDE_KEY, 0x5a6b); if (ret) return ret; @@ -203,7 +203,7 @@ static int tfa9890_init(struct regmap *regmap) if (ret) return ret; - /* hide keys again */ + /* hide registers again */ ret = regmap_write(regmap, TFA989X_HIDE_UNHIDE_KEY, 0x0000); if (ret) return ret; diff --git a/sound/soc/codecs/tlv320adc3xxx.c b/sound/soc/codecs/tlv320adc3xxx.c index 748998e48af97abebb3d417ff8d57b1af64798c7..baab320ef98879d73b4d14c4fddc9d0c688e46e5 100644 --- a/sound/soc/codecs/tlv320adc3xxx.c +++ b/sound/soc/codecs/tlv320adc3xxx.c @@ -1426,7 +1426,7 @@ err_unprepare_mclk: return ret; } -static int __exit adc3xxx_i2c_remove(struct i2c_client *client) +static void __exit adc3xxx_i2c_remove(struct i2c_client *client) { struct adc3xxx *adc3xxx = i2c_get_clientdata(client); @@ -1434,7 +1434,6 @@ static int __exit adc3xxx_i2c_remove(struct i2c_client *client) clk_disable_unprepare(adc3xxx->mclk); adc3xxx_free_gpio(adc3xxx); snd_soc_unregister_component(&client->dev); - return 0; } static const struct of_device_id tlv320adc3xxx_of_match[] = { diff --git a/sound/soc/codecs/tlv320adcx140.c b/sound/soc/codecs/tlv320adcx140.c index 2844a9d2bc4a03776d687c623a9302f117fd1f03..91a22d927915841545a0ed042b0bc91c337b04bf 100644 --- a/sound/soc/codecs/tlv320adcx140.c +++ b/sound/soc/codecs/tlv320adcx140.c @@ -31,6 +31,7 @@ struct adcx140_priv { struct device *dev; bool micbias_vg; + bool phase_calib_on; unsigned int dai_fmt; unsigned int slot_width; @@ -592,6 +593,52 @@ static const struct snd_soc_dapm_route adcx140_audio_map[] = { {"MIC4M Input Mux", "Digital", "MIC4M"}, }; +#define ADCX140_PHASE_CALIB_SWITCH(xname) {\ + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \ + .access = SNDRV_CTL_ELEM_ACCESS_READWRITE,\ + .info = adcx140_phase_calib_info, \ + .get = adcx140_phase_calib_get, \ + .put = adcx140_phase_calib_put} + +static int adcx140_phase_calib_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = 1; + return 0; +} + +static int adcx140_phase_calib_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *value) +{ + struct snd_soc_component *codec = + snd_soc_kcontrol_component(kcontrol); + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(codec); + + value->value.integer.value[0] = adcx140->phase_calib_on ? 1 : 0; + + + return 0; +} + +static int adcx140_phase_calib_put(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *value) +{ + struct snd_soc_component *codec + = snd_soc_kcontrol_component(kcontrol); + struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(codec); + + bool v = value->value.integer.value[0] ? true : false; + + if (adcx140->phase_calib_on != v) { + adcx140->phase_calib_on = v; + return 1; + } + return 0; +} + static const struct snd_kcontrol_new adcx140_snd_controls[] = { SOC_SINGLE_TLV("Analog CH1 Mic Gain Volume", ADCX140_CH1_CFG1, 2, 42, 0, adc_tlv), @@ -628,6 +675,7 @@ static const struct snd_kcontrol_new adcx140_snd_controls[] = { 0, 0xff, 0, dig_vol_tlv), SOC_SINGLE_TLV("Digital CH8 Out Volume", ADCX140_CH8_CFG2, 0, 0xff, 0, dig_vol_tlv), + ADCX140_PHASE_CALIB_SWITCH("Phase Calibration Switch"), }; static int adcx140_reset(struct adcx140_priv *adcx140) @@ -653,6 +701,8 @@ static int adcx140_reset(struct adcx140_priv *adcx140) static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state) { int pwr_ctrl = 0; + int ret = 0; + struct snd_soc_component *component = adcx140->component; if (power_state) pwr_ctrl = ADCX140_PWR_CFG_ADC_PDZ | ADCX140_PWR_CFG_PLL_PDZ; @@ -660,6 +710,14 @@ static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state) if (adcx140->micbias_vg && power_state) pwr_ctrl |= ADCX140_PWR_CFG_BIAS_PDZ; + if (pwr_ctrl) { + ret = regmap_write(adcx140->regmap, ADCX140_PHASE_CALIB, + adcx140->phase_calib_on ? 0x00 : 0x40); + if (ret) + dev_err(component->dev, "%s: register write error %d\n", + __func__, ret); + } + regmap_update_bits(adcx140->regmap, ADCX140_PWR_CFG, ADCX140_PWR_CTRL_MSK, pwr_ctrl); } @@ -1095,6 +1153,7 @@ static int adcx140_i2c_probe(struct i2c_client *i2c) if (!adcx140) return -ENOMEM; + adcx140->phase_calib_on = false; adcx140->dev = &i2c->dev; adcx140->gpio_reset = devm_gpiod_get_optional(adcx140->dev, diff --git a/sound/soc/codecs/tlv320adcx140.h b/sound/soc/codecs/tlv320adcx140.h index d7d4e3a88b5c7422b3575ea0dabbf0a4e3b51caf..fd80fac8b32749bdb7d48cfc96afb93dcacc7eaa 100644 --- a/sound/soc/codecs/tlv320adcx140.h +++ b/sound/soc/codecs/tlv320adcx140.h @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 -// TLV320ADCX104 Sound driver +// TLV320ADCX140 Sound driver // Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com/ #ifndef _TLV320ADCX140_H @@ -90,6 +90,7 @@ #define ADCX140_PWR_CFG 0x75 #define ADCX140_DEV_STS0 0x76 #define ADCX140_DEV_STS1 0x77 +#define ADCX140_PHASE_CALIB 0X7b #define ADCX140_RESET BIT(0) diff --git a/sound/soc/codecs/tlv320aic26.c b/sound/soc/codecs/tlv320aic26.c index 8bae4b47506880ed66db7753985eb25bc5641ff7..e5dfb3d752a3af4cc98f1564e685860448c84626 100644 --- a/sound/soc/codecs/tlv320aic26.c +++ b/sound/soc/codecs/tlv320aic26.c @@ -271,7 +271,7 @@ static ssize_t keyclick_show(struct device *dev, freq = (125 << ((val >> 8) & 0x7)) >> 1; len = 2 * (1 + ((val >> 4) & 0xf)); - return sprintf(buf, "amp=%x freq=%iHz len=%iclks\n", amp, freq, len); + return sysfs_emit(buf, "amp=%x freq=%iHz len=%iclks\n", amp, freq, len); } /* Any write to the keyclick attribute will trigger the keyclick event */ diff --git a/sound/soc/codecs/tlv320aic32x4-i2c.c b/sound/soc/codecs/tlv320aic32x4-i2c.c index 0645239901b1a94b410b1c979af3a129060cebba..d1e543ca3521e6b4db5dc23c71834d364e20be8c 100644 --- a/sound/soc/codecs/tlv320aic32x4-i2c.c +++ b/sound/soc/codecs/tlv320aic32x4-i2c.c @@ -45,11 +45,9 @@ static int aic32x4_i2c_probe(struct i2c_client *i2c) return aic32x4_probe(&i2c->dev, regmap); } -static int aic32x4_i2c_remove(struct i2c_client *i2c) +static void aic32x4_i2c_remove(struct i2c_client *i2c) { aic32x4_remove(&i2c->dev); - - return 0; } static const struct i2c_device_id aic32x4_i2c_id[] = { diff --git a/sound/soc/codecs/tlv320aic3x-i2c.c b/sound/soc/codecs/tlv320aic3x-i2c.c index 7bd9ce08bb7bf9bbc92574ac0588144b59a81c49..d7e94d564dbf7e5dd9a76f81598bcaa94bccd413 100644 --- a/sound/soc/codecs/tlv320aic3x-i2c.c +++ b/sound/soc/codecs/tlv320aic3x-i2c.c @@ -41,11 +41,9 @@ static int aic3x_i2c_probe(struct i2c_client *i2c) return aic3x_probe(&i2c->dev, regmap, id->driver_data); } -static int aic3x_i2c_remove(struct i2c_client *i2c) +static void aic3x_i2c_remove(struct i2c_client *i2c) { aic3x_remove(&i2c->dev); - - return 0; } static const struct of_device_id aic3x_of_id[] = { diff --git a/sound/soc/codecs/tlv320dac33.c b/sound/soc/codecs/tlv320dac33.c index 17ae3b1d96fb4964542235e06fba5908d4c88f6f..16ce3ef1134b8bcec05b05714cb1ebc353710a8b 100644 --- a/sound/soc/codecs/tlv320dac33.c +++ b/sound/soc/codecs/tlv320dac33.c @@ -1536,7 +1536,7 @@ err_gpio: return ret; } -static int dac33_i2c_remove(struct i2c_client *client) +static void dac33_i2c_remove(struct i2c_client *client) { struct tlv320dac33_priv *dac33 = i2c_get_clientdata(client); @@ -1545,8 +1545,6 @@ static int dac33_i2c_remove(struct i2c_client *client) if (dac33->power_gpio >= 0) gpio_free(dac33->power_gpio); - - return 0; } static const struct i2c_device_id tlv320dac33_i2c_id[] = { diff --git a/sound/soc/codecs/ts3a227e.c b/sound/soc/codecs/ts3a227e.c index d8ab0810fceba005ed20bea1673ba5647aba33c8..2305a472d13214b157616fbdabf23568fda913b2 100644 --- a/sound/soc/codecs/ts3a227e.c +++ b/sound/soc/codecs/ts3a227e.c @@ -78,12 +78,20 @@ static const int ts3a227e_buttons[] = { #define ADC_COMPLETE_INT_DISABLE 0x04 #define INTB_DISABLE 0x08 +/* TS3A227E_REG_SETTING_1 0x4 */ +#define DEBOUNCE_INSERTION_SETTING_SFT (0) +#define DEBOUNCE_INSERTION_SETTING_MASK (0x7 << DEBOUNCE_PRESS_SETTING_SFT) + /* TS3A227E_REG_SETTING_2 0x05 */ #define KP_ENABLE 0x04 /* TS3A227E_REG_SETTING_3 0x06 */ -#define MICBIAS_SETTING_SFT (3) +#define MICBIAS_SETTING_SFT 3 #define MICBIAS_SETTING_MASK (0x7 << MICBIAS_SETTING_SFT) +#define DEBOUNCE_RELEASE_SETTING_SFT 2 +#define DEBOUNCE_RELEASE_SETTING_MASK (0x1 << DEBOUNCE_RELEASE_SETTING_SFT) +#define DEBOUNCE_PRESS_SETTING_SFT 0 +#define DEBOUNCE_PRESS_SETTING_MASK (0x3 << DEBOUNCE_PRESS_SETTING_SFT) /* TS3A227E_REG_ACCESSORY_STATUS 0x0b */ #define TYPE_3_POLE 0x01 @@ -136,7 +144,7 @@ static bool ts3a227e_volatile_reg(struct device *dev, unsigned int reg) { switch (reg) { case TS3A227E_REG_INTERRUPT ... TS3A227E_REG_INTERRUPT_DISABLE: - case TS3A227E_REG_SETTING_2: + case TS3A227E_REG_SETTING_1 ... TS3A227E_REG_SETTING_2: case TS3A227E_REG_SWITCH_STATUS_1 ... TS3A227E_REG_ADC_OUTPUT: return true; default: @@ -269,14 +277,55 @@ static const struct regmap_config ts3a227e_regmap_config = { static int ts3a227e_parse_device_property(struct ts3a227e *ts3a227e, struct device *dev) { - u32 micbias; + u32 value; + u32 value_ms; + u32 setting3_value = 0; + u32 setting3_mask = 0; int err; - err = device_property_read_u32(dev, "ti,micbias", &micbias); + err = device_property_read_u32(dev, "ti,micbias", &value); + if (!err) { + setting3_mask = MICBIAS_SETTING_MASK; + setting3_value = (value << MICBIAS_SETTING_SFT) & + MICBIAS_SETTING_MASK; + } + + err = device_property_read_u32(dev, "ti,debounce-release-ms", + &value_ms); if (!err) { + value = (value_ms > 10); + setting3_mask |= DEBOUNCE_RELEASE_SETTING_MASK; + setting3_value |= (value << DEBOUNCE_RELEASE_SETTING_SFT) & + DEBOUNCE_RELEASE_SETTING_MASK; + } + + err = device_property_read_u32(dev, "ti,debounce-press-ms", &value_ms); + if (!err) { + value = (value_ms + 20) / 40; + if (value > 3) + value = 3; + setting3_mask |= DEBOUNCE_PRESS_SETTING_MASK; + setting3_value |= (value << DEBOUNCE_PRESS_SETTING_SFT) & + DEBOUNCE_PRESS_SETTING_MASK; + } + + if (setting3_mask) regmap_update_bits(ts3a227e->regmap, TS3A227E_REG_SETTING_3, - MICBIAS_SETTING_MASK, - (micbias & 0x07) << MICBIAS_SETTING_SFT); + setting3_mask, setting3_value); + + err = device_property_read_u32(dev, "ti,debounce-insertion-ms", + &value_ms); + if (!err) { + if (value_ms < 165) + value = (value_ms + 15) / 30; + else if (value_ms < 1500) + value = 6; + else + value = 7; + regmap_update_bits(ts3a227e->regmap, TS3A227E_REG_SETTING_1, + DEBOUNCE_INSERTION_SETTING_MASK, + (value << DEBOUNCE_INSERTION_SETTING_SFT) & + DEBOUNCE_INSERTION_SETTING_MASK); } return 0; diff --git a/sound/soc/codecs/uda134x.c b/sound/soc/codecs/uda134x.c index 2db3d8a60c7a0cc23827262248812a8d79731321..1a62bec94005602ae4edd742219cf18c36bfeecf 100644 --- a/sound/soc/codecs/uda134x.c +++ b/sound/soc/codecs/uda134x.c @@ -450,7 +450,7 @@ static int uda134x_soc_probe(struct snd_soc_component *component) struct uda134x_priv *uda134x = snd_soc_component_get_drvdata(component); struct uda134x_platform_data *pd = uda134x->pd; const struct snd_soc_dapm_widget *widgets; - unsigned num_widgets; + unsigned int num_widgets; int ret; printk(KERN_INFO "UDA134X SoC Audio Codec\n"); diff --git a/sound/soc/codecs/wcd-mbhc-v2.c b/sound/soc/codecs/wcd-mbhc-v2.c index 98baef594bf31688e7855c2276f78e6a11609d82..1911750f7445c8fb50ba3c39dd8877ee7c0e6475 100644 --- a/sound/soc/codecs/wcd-mbhc-v2.c +++ b/sound/soc/codecs/wcd-mbhc-v2.c @@ -714,11 +714,12 @@ static int wcd_mbhc_initialise(struct wcd_mbhc *mbhc) struct snd_soc_component *component = mbhc->component; int ret; - ret = pm_runtime_resume_and_get(component->dev); + ret = pm_runtime_get_sync(component->dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(component->dev, - "pm_runtime_resume_and_get failed in %s, ret %d\n", + "pm_runtime_get_sync failed in %s, ret %d\n", __func__, ret); + pm_runtime_put_noidle(component->dev); return ret; } @@ -1096,11 +1097,12 @@ static void wcd_correct_swch_plug(struct work_struct *work) mbhc = container_of(work, struct wcd_mbhc, correct_plug_swch); component = mbhc->component; - ret = pm_runtime_resume_and_get(component->dev); + ret = pm_runtime_get_sync(component->dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(component->dev, - "pm_runtime_resume_and_get failed in %s, ret %d\n", + "pm_runtime_get_sync failed in %s, ret %d\n", __func__, ret); + pm_runtime_put_noidle(component->dev); return; } micbias_mv = wcd_mbhc_get_micbias(mbhc); @@ -1189,7 +1191,7 @@ correct_plug_type: pt_gnd_mic_swap_cnt = 0; plug_type = wcd_mbhc_get_plug_from_adc(mbhc, output_mv); continue; - } else if (cross_conn < 0) /* Error */ + } else /* Error if (cross_conn < 0) */ continue; if (pt_gnd_mic_swap_cnt == GND_MIC_SWAP_THRESHOLD) { diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c index beeeb35e803215517cb90a1112325b3819a242b0..d2548fdf9ae562378d8dca44403d3652df9a9e3b 100644 --- a/sound/soc/codecs/wcd9335.c +++ b/sound/soc/codecs/wcd9335.c @@ -1821,12 +1821,10 @@ static int wcd9335_set_decimator_rate(struct snd_soc_dai *dai, tx_port_reg = WCD9335_CDC_IF_ROUTER_TX_MUX_CFG3; shift = 0; shift_val = 0x0F; - } else if (tx_port == 13) { + } else /* (tx_port == 13) */ { tx_port_reg = WCD9335_CDC_IF_ROUTER_TX_MUX_CFG3; shift = 4; shift_val = 0x03; - } else { - return -EINVAL; } tx_mux_sel = snd_soc_component_read(comp, tx_port_reg) & @@ -1974,8 +1972,8 @@ static int wcd9335_trigger(struct snd_pcm_substream *substream, int cmd, case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - slim_stream_unprepare(dai_data->sruntime); slim_stream_disable(dai_data->sruntime); + slim_stream_unprepare(dai_data->sruntime); break; default: break; @@ -5013,16 +5011,22 @@ static const struct regmap_irq wcd9335_codec_irqs[] = { }, }; +static const unsigned int wcd9335_config_regs[] = { + WCD9335_INTR_LEVEL0, +}; + static const struct regmap_irq_chip wcd9335_regmap_irq1_chip = { .name = "wcd9335_pin1_irq", .status_base = WCD9335_INTR_PIN1_STATUS0, .mask_base = WCD9335_INTR_PIN1_MASK0, .ack_base = WCD9335_INTR_PIN1_CLEAR0, - .type_base = WCD9335_INTR_LEVEL0, - .num_type_reg = 4, .num_regs = 4, .irqs = wcd9335_codec_irqs, .num_irqs = ARRAY_SIZE(wcd9335_codec_irqs), + .config_base = wcd9335_config_regs, + .num_config_bases = ARRAY_SIZE(wcd9335_config_regs), + .num_config_regs = 4, + .set_type_config = regmap_irq_set_type_config_simple, }; static int wcd9335_parse_dt(struct wcd9335_codec *wcd) diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c index f56907d0942db55ce6392d3f704fe4cb9f558588..28175c746b9ae2687ecd2553c506517588cf7908 100644 --- a/sound/soc/codecs/wcd934x.c +++ b/sound/soc/codecs/wcd934x.c @@ -1913,8 +1913,8 @@ static int wcd934x_trigger(struct snd_pcm_substream *substream, int cmd, case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - slim_stream_unprepare(dai_data->sruntime); slim_stream_disable(dai_data->sruntime); + slim_stream_unprepare(dai_data->sruntime); break; default: break; diff --git a/sound/soc/codecs/wcd938x.c b/sound/soc/codecs/wcd938x.c index 781ae569be29036ba2c64423d6cc7c401d8b6989..aca06a4026f3e3680926b9261387fe811fd57cab 100644 --- a/sound/soc/codecs/wcd938x.c +++ b/sound/soc/codecs/wcd938x.c @@ -1298,7 +1298,6 @@ static struct regmap_irq_chip wcd938x_regmap_irq_chip = { .num_regs = 3, .status_base = WCD938X_DIGITAL_INTR_STATUS_0, .mask_base = WCD938X_DIGITAL_INTR_MASK_0, - .type_base = WCD938X_DIGITAL_INTR_LEVEL_0, .ack_base = WCD938X_DIGITAL_INTR_CLEAR_0, .use_ack = 1, .runtime_pm = true, diff --git a/sound/soc/codecs/wm1250-ev1.c b/sound/soc/codecs/wm1250-ev1.c index 98343626078b6e3c48f881f9d2f4789861adafd1..0064a607ec68b2e6aa3391610817a440418ee003 100644 --- a/sound/soc/codecs/wm1250-ev1.c +++ b/sound/soc/codecs/wm1250-ev1.c @@ -228,11 +228,9 @@ static int wm1250_ev1_probe(struct i2c_client *i2c) return 0; } -static int wm1250_ev1_remove(struct i2c_client *i2c) +static void wm1250_ev1_remove(struct i2c_client *i2c) { wm1250_ev1_free(i2c); - - return 0; } static const struct i2c_device_id wm1250_ev1_i2c_id[] = { diff --git a/sound/soc/codecs/wm2200.c b/sound/soc/codecs/wm2200.c index 7b4e162a298c0dd03e5e6be9b137c2c274127481..0a65afa44a59bc86fb2f7e636518806fa93b420e 100644 --- a/sound/soc/codecs/wm2200.c +++ b/sound/soc/codecs/wm2200.c @@ -2414,7 +2414,7 @@ err_enable: return ret; } -static int wm2200_i2c_remove(struct i2c_client *i2c) +static void wm2200_i2c_remove(struct i2c_client *i2c) { struct wm2200_priv *wm2200 = i2c_get_clientdata(i2c); @@ -2427,8 +2427,6 @@ static int wm2200_i2c_remove(struct i2c_client *i2c) gpio_set_value_cansleep(wm2200->pdata.ldo_ena, 0); regulator_bulk_disable(ARRAY_SIZE(wm2200->core_supplies), wm2200->core_supplies); - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/wm5100.c b/sound/soc/codecs/wm5100.c index 35a85ce6b4648f9c5bb9d00cf72fcdb9ae6ff1b8..3b09d4a1684f3144b5f799050bd0549bae495f6f 100644 --- a/sound/soc/codecs/wm5100.c +++ b/sound/soc/codecs/wm5100.c @@ -2635,7 +2635,7 @@ err: return ret; } -static int wm5100_i2c_remove(struct i2c_client *i2c) +static void wm5100_i2c_remove(struct i2c_client *i2c) { struct wm5100_priv *wm5100 = i2c_get_clientdata(i2c); @@ -2651,8 +2651,6 @@ static int wm5100_i2c_remove(struct i2c_client *i2c) gpio_set_value_cansleep(wm5100->pdata.ldo_ena, 0); gpio_free(wm5100->pdata.ldo_ena); } - - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c index af7d324e335258909354e15918d4531876bb7c78..c09c9ac51b3e76dc7584da6b0402ebb527994daa 100644 --- a/sound/soc/codecs/wm5102.c +++ b/sound/soc/codecs/wm5102.c @@ -2099,9 +2099,6 @@ static int wm5102_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm5102_digital_vu[i], WM5102_DIG_VU, WM5102_DIG_VU); - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, "ADSP2 Compressed IRQ", wm5102_adsp2_irq, wm5102); @@ -2134,6 +2131,9 @@ static int wm5102_probe(struct platform_device *pdev) goto err_spk_irqs; } + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + return ret; err_spk_irqs: diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c index f3f4a10bf0f7c82c3a3f28103f07b20f6c54871e..fc634c995834d87d83ca6400bac314f6f2556392 100644 --- a/sound/soc/codecs/wm5110.c +++ b/sound/soc/codecs/wm5110.c @@ -2457,9 +2457,6 @@ static int wm5110_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm5110_digital_vu[i], WM5110_DIG_VU, WM5110_DIG_VU); - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, "ADSP2 Compressed IRQ", wm5110_adsp2_irq, wm5110); @@ -2492,6 +2489,9 @@ static int wm5110_probe(struct platform_device *pdev) goto err_spk_irqs; } + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + return ret; err_spk_irqs: diff --git a/sound/soc/codecs/wm8804-i2c.c b/sound/soc/codecs/wm8804-i2c.c index 04dc9fb5afb4ee61210faa0267b2a531bceddc05..3ce1a39d76eb58218d0cb2126fe538eab95aadd0 100644 --- a/sound/soc/codecs/wm8804-i2c.c +++ b/sound/soc/codecs/wm8804-i2c.c @@ -25,10 +25,9 @@ static int wm8804_i2c_probe(struct i2c_client *i2c) return wm8804_probe(&i2c->dev, regmap); } -static int wm8804_i2c_remove(struct i2c_client *i2c) +static void wm8804_i2c_remove(struct i2c_client *i2c) { wm8804_remove(&i2c->dev); - return 0; } static const struct i2c_device_id wm8804_i2c_id[] = { diff --git a/sound/soc/codecs/wm8900.c b/sound/soc/codecs/wm8900.c index d6420df3505d524ffc8b7a655f69662d3b8a97c0..03bbd85ebdf40c51694bd5167224e9546cc854c4 100644 --- a/sound/soc/codecs/wm8900.c +++ b/sound/soc/codecs/wm8900.c @@ -1282,10 +1282,8 @@ static int wm8900_i2c_probe(struct i2c_client *i2c) return ret; } -static int wm8900_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void wm8900_i2c_remove(struct i2c_client *client) +{} static const struct i2c_device_id wm8900_i2c_id[] = { { "wm8900", 0 }, diff --git a/sound/soc/codecs/wm8903.c b/sound/soc/codecs/wm8903.c index 54e0a7628cd57b1cff1e967058a2212c723d298a..41346e5ec5ad401e23e013c375fce2476114f38d 100644 --- a/sound/soc/codecs/wm8903.c +++ b/sound/soc/codecs/wm8903.c @@ -2182,7 +2182,7 @@ err: return ret; } -static int wm8903_i2c_remove(struct i2c_client *client) +static void wm8903_i2c_remove(struct i2c_client *client) { struct wm8903_priv *wm8903 = i2c_get_clientdata(client); @@ -2191,8 +2191,6 @@ static int wm8903_i2c_remove(struct i2c_client *client) if (client->irq) free_irq(client->irq, wm8903); wm8903_free_gpio(wm8903); - - return 0; } static const struct of_device_id wm8903_of_match[] = { diff --git a/sound/soc/codecs/wm8960.c b/sound/soc/codecs/wm8960.c index 37956516d9976a13cac0f6a71914d42dfa79c4ea..0d167238a369a66094d7a643cd126ff9c68ce49c 100644 --- a/sound/soc/codecs/wm8960.c +++ b/sound/soc/codecs/wm8960.c @@ -1486,10 +1486,8 @@ static int wm8960_i2c_probe(struct i2c_client *i2c) return ret; } -static int wm8960_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void wm8960_i2c_remove(struct i2c_client *client) +{} static const struct i2c_device_id wm8960_i2c_id[] = { { "wm8960", 0 }, diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index 398c448ea8540de5f8a5911414f42dd657312bb6..81049664387e147b44551501fd9bfc1a4ce26f0b 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -3778,10 +3778,9 @@ err: return ret; } -static int wm8962_i2c_remove(struct i2c_client *client) +static void wm8962_i2c_remove(struct i2c_client *client) { pm_runtime_disable(&client->dev); - return 0; } #ifdef CONFIG_PM diff --git a/sound/soc/codecs/wm8993.c b/sound/soc/codecs/wm8993.c index 8db98b5a06bf4c6214802a17358cbd229fe6a6bb..22a47acbc6d1571591f1020b623f7429032b2cd2 100644 --- a/sound/soc/codecs/wm8993.c +++ b/sound/soc/codecs/wm8993.c @@ -1722,15 +1722,13 @@ err_enable: return ret; } -static int wm8993_i2c_remove(struct i2c_client *i2c) +static void wm8993_i2c_remove(struct i2c_client *i2c) { struct wm8993_priv *wm8993 = i2c_get_clientdata(i2c); if (i2c->irq) free_irq(i2c->irq, wm8993); regulator_bulk_disable(ARRAY_SIZE(wm8993->supplies), wm8993->supplies); - - return 0; } static const struct i2c_device_id wm8993_i2c_id[] = { diff --git a/sound/soc/codecs/wm8996.c b/sound/soc/codecs/wm8996.c index 17f307a3104670164fb9959e26cecfa1b7c85356..b52ed89d631a7f4ae84dea93b0eb37814a19ead3 100644 --- a/sound/soc/codecs/wm8996.c +++ b/sound/soc/codecs/wm8996.c @@ -3065,7 +3065,7 @@ err: return ret; } -static int wm8996_i2c_remove(struct i2c_client *client) +static void wm8996_i2c_remove(struct i2c_client *client) { struct wm8996_priv *wm8996 = i2c_get_clientdata(client); @@ -3074,8 +3074,6 @@ static int wm8996_i2c_remove(struct i2c_client *client) gpio_set_value_cansleep(wm8996->pdata.ldo_ena, 0); gpio_free(wm8996->pdata.ldo_ena); } - - return 0; } static const struct i2c_device_id wm8996_i2c_id[] = { diff --git a/sound/soc/codecs/wm8997.c b/sound/soc/codecs/wm8997.c index 210ad662fc26d6d554b2b16462ab6a6353790f7e..77136a521605936d31468cca02c0a96258d0e5f1 100644 --- a/sound/soc/codecs/wm8997.c +++ b/sound/soc/codecs/wm8997.c @@ -1161,9 +1161,6 @@ static int wm8997_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm8997_digital_vu[i], WM8997_DIG_VU, WM8997_DIG_VU); - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - arizona_init_common(arizona); ret = arizona_init_vol_limit(arizona); @@ -1182,6 +1179,9 @@ static int wm8997_probe(struct platform_device *pdev) goto err_spk_irqs; } + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + return ret; err_spk_irqs: diff --git a/sound/soc/codecs/wm9081.c b/sound/soc/codecs/wm9081.c index d5151877d0fa218b68512e2473ae9bc09eba7e31..513ec0ba81bb8d36711f23ae42de715136bfbb5f 100644 --- a/sound/soc/codecs/wm9081.c +++ b/sound/soc/codecs/wm9081.c @@ -1356,10 +1356,8 @@ static int wm9081_i2c_probe(struct i2c_client *i2c) return 0; } -static int wm9081_i2c_remove(struct i2c_client *client) -{ - return 0; -} +static void wm9081_i2c_remove(struct i2c_client *client) +{} static const struct i2c_device_id wm9081_i2c_id[] = { { "wm9081", 0 }, diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c index cfaa45ede916ae42bef79e54203ff66a50bd1e40..8a2e9771bb50ee4e4dd94f75788a70952ca66918 100644 --- a/sound/soc/codecs/wm_adsp.c +++ b/sound/soc/codecs/wm_adsp.c @@ -1602,7 +1602,9 @@ static int wm_adsp_buffer_init(struct wm_adsp *dsp) if (list_empty(&dsp->buffer_list)) { /* Fall back to legacy support */ ret = wm_adsp_buffer_parse_legacy(dsp); - if (ret) + if (ret == -ENODEV) + adsp_info(dsp, "Legacy support not available\n"); + else if (ret) adsp_warn(dsp, "Failed to parse legacy: %d\n", ret); } diff --git a/sound/soc/codecs/wsa883x.c b/sound/soc/codecs/wsa883x.c index 63e1d7aa6137987d36be80c0ffcf77a8317f69e9..c7b10bbfba7eaf1e1fe6425b35160184097ea1c7 100644 --- a/sound/soc/codecs/wsa883x.c +++ b/sound/soc/codecs/wsa883x.c @@ -415,7 +415,6 @@ #define WSA883X_NUM_REGISTERS (WSA883X_EMEM_63 + 1) #define WSA883X_MAX_REGISTER (WSA883X_NUM_REGISTERS - 1) -#define WSA883X_PROBE_TIMEOUT 1000 #define WSA883X_VERSION_1_0 0 #define WSA883X_VERSION_1_1 1 @@ -1409,6 +1408,7 @@ static int wsa883x_probe(struct sdw_slave *pdev, wsa883x->sconfig.type = SDW_STREAM_PDM; pdev->prop.sink_ports = GENMASK(WSA883X_MAX_SWR_PORTS, 0); + pdev->prop.simple_clk_stop_capable = true; pdev->prop.sink_dpn_prop = wsa_sink_dpn_prop; pdev->prop.scp_int1_mask = SDW_SCP_INT1_BUS_CLASH | SDW_SCP_INT1_PARITY; gpiod_direction_output(wsa883x->sd_n, 1); @@ -1440,43 +1440,17 @@ err: static int __maybe_unused wsa883x_runtime_suspend(struct device *dev) { struct regmap *regmap = dev_get_regmap(dev, NULL); - struct wsa883x_priv *wsa883x = dev_get_drvdata(dev); - - gpiod_direction_output(wsa883x->sd_n, 0); regcache_cache_only(regmap, true); regcache_mark_dirty(regmap); - regulator_disable(wsa883x->vdd); return 0; } static int __maybe_unused wsa883x_runtime_resume(struct device *dev) { - struct sdw_slave *slave = dev_to_sdw_dev(dev); struct regmap *regmap = dev_get_regmap(dev, NULL); - struct wsa883x_priv *wsa883x = dev_get_drvdata(dev); - unsigned long time; - int ret; - - ret = regulator_enable(wsa883x->vdd); - if (ret) { - dev_err(dev, "Failed to enable vdd regulator (%d)\n", ret); - return ret; - } - - gpiod_direction_output(wsa883x->sd_n, 1); - - time = wait_for_completion_timeout(&slave->initialization_complete, - msecs_to_jiffies(WSA883X_PROBE_TIMEOUT)); - if (!time) { - dev_err(dev, "Initialization not complete, timed out\n"); - gpiod_direction_output(wsa883x->sd_n, 0); - regulator_disable(wsa883x->vdd); - return -ETIMEDOUT; - } - usleep_range(20000, 20010); regcache_cache_only(regmap, false); regcache_sync(regmap); diff --git a/sound/soc/fsl/eukrea-tlv320.c b/sound/soc/fsl/eukrea-tlv320.c index 8b61582753c868f30291b8843b9bd1c0bd55cb55..9af4c4a35eb16610a92e48975e6f5a1121a1e9c7 100644 --- a/sound/soc/fsl/eukrea-tlv320.c +++ b/sound/soc/fsl/eukrea-tlv320.c @@ -86,7 +86,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev) int ret; int int_port = 0, ext_port; struct device_node *np = pdev->dev.of_node; - struct device_node *ssi_np = NULL, *codec_np = NULL; + struct device_node *ssi_np = NULL, *codec_np = NULL, *tmp_np = NULL; eukrea_tlv320.dev = &pdev->dev; if (np) { @@ -143,7 +143,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev) } if (machine_is_eukrea_cpuimx27() || - of_find_compatible_node(NULL, NULL, "fsl,imx21-audmux")) { + (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx21-audmux"))) { imx_audmux_v1_configure_port(MX27_AUDMUX_HPCR1_SSI0, IMX_AUDMUX_V1_PCR_SYN | IMX_AUDMUX_V1_PCR_TFSDIR | @@ -158,10 +158,11 @@ static int eukrea_tlv320_probe(struct platform_device *pdev) IMX_AUDMUX_V1_PCR_SYN | IMX_AUDMUX_V1_PCR_RXDSEL(MX27_AUDMUX_HPCR1_SSI0) ); + of_node_put(tmp_np); } else if (machine_is_eukrea_cpuimx25sd() || machine_is_eukrea_cpuimx35sd() || machine_is_eukrea_cpuimx51sd() || - of_find_compatible_node(NULL, NULL, "fsl,imx31-audmux")) { + (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx31-audmux"))) { if (!np) ext_port = machine_is_eukrea_cpuimx25sd() ? 4 : 3; @@ -178,6 +179,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev) IMX_AUDMUX_V2_PTCR_SYN, IMX_AUDMUX_V2_PDCR_RXDSEL(int_port) ); + of_node_put(tmp_np); } else { if (np) { /* The eukrea,asoc-tlv320 driver was explicitly diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c index aa5edf32d988955ba08178e1d96518b2e5406213..936aef5d2767c5a1d47a8b66758995b9da38c914 100644 --- a/sound/soc/fsl/fsl_asrc.c +++ b/sound/soc/fsl/fsl_asrc.c @@ -20,6 +20,7 @@ #define IDEAL_RATIO_DECIMAL_DEPTH 26 #define DIVIDER_NUM 64 +#define INIT_RETRY_NUM 50 #define pair_err(fmt, ...) \ dev_err(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__) @@ -27,6 +28,9 @@ #define pair_dbg(fmt, ...) \ dev_dbg(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__) +#define pair_warn(fmt, ...) \ + dev_warn(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__) + /* Corresponding to process_option */ static unsigned int supported_asrc_rate[] = { 5512, 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000, @@ -579,7 +583,7 @@ static void fsl_asrc_start_pair(struct fsl_asrc_pair *pair) { struct fsl_asrc *asrc = pair->asrc; enum asrc_pair_index index = pair->index; - int reg, retry = 10, i; + int reg, retry = INIT_RETRY_NUM, i; /* Enable the current pair */ regmap_update_bits(asrc->regmap, REG_ASRCTR, @@ -592,6 +596,10 @@ static void fsl_asrc_start_pair(struct fsl_asrc_pair *pair) reg &= ASRCFG_INIRQi_MASK(index); } while (!reg && --retry); + /* NOTE: Doesn't treat initialization timeout as an error */ + if (!retry) + pair_warn("initialization isn't finished\n"); + /* Make the input fifo to ASRC STALL level */ regmap_read(asrc->regmap, REG_ASRCNCR, ®); for (i = 0; i < pair->channels * 4; i++) @@ -1257,6 +1265,7 @@ static int fsl_asrc_runtime_resume(struct device *dev) { struct fsl_asrc *asrc = dev_get_drvdata(dev); struct fsl_asrc_priv *asrc_priv = asrc->private; + int reg, retry = INIT_RETRY_NUM; int i, ret; u32 asrctr; @@ -1295,6 +1304,24 @@ static int fsl_asrc_runtime_resume(struct device *dev) regmap_update_bits(asrc->regmap, REG_ASRCTR, ASRCTR_ASRCEi_ALL_MASK, asrctr); + /* Wait for status of initialization for all enabled pairs */ + do { + udelay(5); + regmap_read(asrc->regmap, REG_ASRCFG, ®); + reg = (reg >> ASRCFG_INIRQi_SHIFT(0)) & 0x7; + } while ((reg != ((asrctr >> ASRCTR_ASRCEi_SHIFT(0)) & 0x7)) && --retry); + + /* + * NOTE: Doesn't treat initialization timeout as an error + * Some of the pairs may success, then still can continue. + */ + if (!retry) { + for (i = ASRC_PAIR_A; i < ASRC_PAIR_MAX_NUM; i++) { + if ((asrctr & ASRCTR_ASRCEi_MASK(i)) && !(reg & (1 << i))) + dev_warn(dev, "Pair %c initialization isn't finished\n", 'A' + i); + } + } + return 0; disable_asrck_clk: diff --git a/sound/soc/fsl/fsl_asrc_dma.c b/sound/soc/fsl/fsl_asrc_dma.c index 12ddf2320f2db6de1fd4edfd0ada335d3445dcc3..3b81a465814a1f777ed0820a853fbd93eceaf018 100644 --- a/sound/soc/fsl/fsl_asrc_dma.c +++ b/sound/soc/fsl/fsl_asrc_dma.c @@ -139,7 +139,7 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, struct dma_chan *tmp_chan = NULL, *be_chan = NULL; struct snd_soc_component *component_be = NULL; struct fsl_asrc *asrc = pair->asrc; - struct dma_slave_config config_fe, config_be; + struct dma_slave_config config_fe = {}, config_be = {}; struct sdma_peripheral_config audio_config; enum asrc_pair_index index = pair->index; struct device *dev = component->dev; @@ -183,7 +183,6 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component, return -EINVAL; } - memset(&config_fe, 0, sizeof(config_fe)); ret = snd_dmaengine_pcm_prepare_slave_config(substream, params, &config_fe); if (ret) { dev_err(dev, "failed to prepare DMA config for Front-End\n"); diff --git a/sound/soc/fsl/fsl_aud2htx.c b/sound/soc/fsl/fsl_aud2htx.c index 873295f59ad7b9dde877ce1f990bb098f77385ca..1e421d9a03fbe45ec48865bd748d1dfaeba48b98 100644 --- a/sound/soc/fsl/fsl_aud2htx.c +++ b/sound/soc/fsl/fsl_aud2htx.c @@ -234,18 +234,26 @@ static int fsl_aud2htx_probe(struct platform_device *pdev) regcache_cache_only(aud2htx->regmap, true); + /* + * Register platform component before registering cpu dai for there + * is not defer probe for platform component in snd_soc_add_pcm_runtime(). + */ + ret = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0); + if (ret) { + dev_err(&pdev->dev, "failed to pcm register\n"); + pm_runtime_disable(&pdev->dev); + return ret; + } + ret = devm_snd_soc_register_component(&pdev->dev, &fsl_aud2htx_component, &fsl_aud2htx_dai, 1); if (ret) { dev_err(&pdev->dev, "failed to register ASoC DAI\n"); + pm_runtime_disable(&pdev->dev); return ret; } - ret = imx_pcm_dma_init(pdev); - if (ret) - dev_err(&pdev->dev, "failed to init imx pcm dma: %d\n", ret); - return ret; } diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index 43857b7a81c94b68e77658710abb4ac7f6b70a7a..672148dd4b234febc98e4c7133bc09fda7d5ecad 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -199,18 +199,10 @@ static int fsl_audmix_put_out_src(struct snd_kcontrol *kcontrol, static const struct snd_kcontrol_new fsl_audmix_snd_controls[] = { /* FSL_AUDMIX_CTR controls */ - { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, - .name = "Mixing Clock Source", - .info = snd_soc_info_enum_double, - .access = SNDRV_CTL_ELEM_ACCESS_WRITE, - .put = fsl_audmix_put_mix_clk_src, - .private_value = (unsigned long)&fsl_audmix_enum[0] }, - { .iface = SNDRV_CTL_ELEM_IFACE_MIXER, - .name = "Output Source", - .info = snd_soc_info_enum_double, - .access = SNDRV_CTL_ELEM_ACCESS_WRITE, - .put = fsl_audmix_put_out_src, - .private_value = (unsigned long)&fsl_audmix_enum[1] }, + SOC_ENUM_EXT("Mixing Clock Source", fsl_audmix_enum[0], + snd_soc_get_enum_double, fsl_audmix_put_mix_clk_src), + SOC_ENUM_EXT("Output Source", fsl_audmix_enum[1], + snd_soc_get_enum_double, fsl_audmix_put_out_src), SOC_ENUM("Output Width", fsl_audmix_enum[2]), SOC_ENUM("Frame Rate Diff Error", fsl_audmix_enum[3]), SOC_ENUM("Clock Freq Diff Error", fsl_audmix_enum[4]), diff --git a/sound/soc/fsl/fsl_mqs.c b/sound/soc/fsl/fsl_mqs.c index c1e2f671191b5fb0a60e3b7ec425c3cbf56035a0..4922e6795b73f02a8d53a33f0fad079e4ccd0739 100644 --- a/sound/soc/fsl/fsl_mqs.c +++ b/sound/soc/fsl/fsl_mqs.c @@ -122,7 +122,7 @@ static int fsl_mqs_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) } switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { - case SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_CBC_CFC: break; default: return -EINVAL; diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c index 7523bb944b216da4c601e8c5332683fa669fa8cd..81f89f6767a2dd105fd0bd0aca724eed0ae49d2e 100644 --- a/sound/soc/fsl/fsl_sai.c +++ b/sound/soc/fsl/fsl_sai.c @@ -114,11 +114,8 @@ static irqreturn_t fsl_sai_isr(int irq, void *devid) if (flags & FSL_SAI_CSR_SEF) dev_dbg(dev, "isr: Tx Frame sync error detected\n"); - if (flags & FSL_SAI_CSR_FEF) { + if (flags & FSL_SAI_CSR_FEF) dev_dbg(dev, "isr: Transmit underrun detected\n"); - /* FIFO reset for safety */ - xcsr |= FSL_SAI_CSR_FR; - } if (flags & FSL_SAI_CSR_FWF) dev_dbg(dev, "isr: Enabled transmit FIFO is empty\n"); @@ -148,11 +145,8 @@ irq_rx: if (flags & FSL_SAI_CSR_SEF) dev_dbg(dev, "isr: Rx Frame sync error detected\n"); - if (flags & FSL_SAI_CSR_FEF) { + if (flags & FSL_SAI_CSR_FEF) dev_dbg(dev, "isr: Receive overflow detected\n"); - /* FIFO reset for safety */ - xcsr |= FSL_SAI_CSR_FR; - } if (flags & FSL_SAI_CSR_FWF) dev_dbg(dev, "isr: Enabled receive FIFO is full\n"); @@ -533,14 +527,17 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream, u32 slot_width = word_width; int adir = tx ? RX : TX; u32 pins, bclk; + u32 watermark; int ret, i; - if (sai->slots) - slots = sai->slots; - if (sai->slot_width) slot_width = sai->slot_width; + if (sai->slots) + slots = sai->slots; + else if (sai->bclk_ratio) + slots = sai->bclk_ratio / slot_width; + pins = DIV_ROUND_UP(channels, slots); /* @@ -625,7 +622,15 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream, FSL_SAI_CR5_FBT_MASK, val_cr5); } - if (hweight8(dl_cfg[dl_cfg_idx].mask[tx]) <= 1) + /* + * Combine mode has limation: + * - Can't used for singel dataline/FIFO case except the FIFO0 + * - Can't used for multi dataline/FIFO case except the enabled FIFOs + * are successive and start from FIFO0 + * + * So for common usage, all multi fifo case disable the combine mode. + */ + if (hweight8(dl_cfg[dl_cfg_idx].mask[tx]) <= 1 || sai->is_multi_fifo_dma) regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs), FSL_SAI_CR4_FCOMB_MASK, 0); else @@ -636,6 +641,26 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream, dma_params->addr = sai->res->start + FSL_SAI_xDR0(tx) + dl_cfg[dl_cfg_idx].start_off[tx] * 0x4; + if (sai->is_multi_fifo_dma) { + sai->audio_config[tx].words_per_fifo = min(slots, channels); + if (tx) { + sai->audio_config[tx].n_fifos_dst = pins; + sai->audio_config[tx].stride_fifos_dst = dl_cfg[dl_cfg_idx].next_off[tx]; + } else { + sai->audio_config[tx].n_fifos_src = pins; + sai->audio_config[tx].stride_fifos_src = dl_cfg[dl_cfg_idx].next_off[tx]; + } + dma_params->maxburst = sai->audio_config[tx].words_per_fifo * pins; + dma_params->peripheral_config = &sai->audio_config[tx]; + dma_params->peripheral_size = sizeof(sai->audio_config[tx]); + + watermark = tx ? (sai->soc_data->fifo_depth - dma_params->maxburst) : + (dma_params->maxburst - 1); + regmap_update_bits(sai->regmap, FSL_SAI_xCR1(tx, ofs), + FSL_SAI_CR1_RFW_MASK(sai->soc_data->fifo_depth), + watermark); + } + /* Find a proper tcre setting */ for (i = 0; i < sai->soc_data->pins; i++) { trce_mask = (1 << (i + 1)) - 1; @@ -1263,6 +1288,7 @@ static int fsl_sai_probe(struct platform_device *pdev) char tmp[8]; int irq, ret, i; int index; + u32 dmas[4]; sai = devm_kzalloc(dev, sizeof(*sai), GFP_KERNEL); if (!sai) @@ -1306,7 +1332,7 @@ static int fsl_sai_probe(struct platform_device *pdev) sai->mclk_clk[i] = devm_clk_get(dev, tmp); if (IS_ERR(sai->mclk_clk[i])) { dev_err(dev, "failed to get mclk%d clock: %ld\n", - i + 1, PTR_ERR(sai->mclk_clk[i])); + i, PTR_ERR(sai->mclk_clk[i])); sai->mclk_clk[i] = NULL; } } @@ -1319,6 +1345,11 @@ static int fsl_sai_probe(struct platform_device *pdev) fsl_asoc_get_pll_clocks(&pdev->dev, &sai->pll8k_clk, &sai->pll11k_clk); + /* Use Multi FIFO mode depending on the support from SDMA script */ + ret = of_property_read_u32_array(np, "dmas", dmas, 4); + if (!sai->soc_data->use_edma && !ret && dmas[2] == IMX_DMATYPE_MULTI_SAI) + sai->is_multi_fifo_dma = true; + /* read dataline mask for rx and tx*/ ret = fsl_sai_read_dlcfg(sai); if (ret < 0) { diff --git a/sound/soc/fsl/fsl_sai.h b/sound/soc/fsl/fsl_sai.h index 17956b5731dc35d73066c292a7056a647bd16236..697f6690068c813e9ee49cc57a9e671416dd06c1 100644 --- a/sound/soc/fsl/fsl_sai.h +++ b/sound/soc/fsl/fsl_sai.h @@ -6,6 +6,7 @@ #ifndef __FSL_SAI_H #define __FSL_SAI_H +#include <linux/dma/imx-dma.h> #include <sound/dmaengine_pcm.h> #define FSL_SAI_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\ @@ -281,6 +282,7 @@ struct fsl_sai { bool is_lsb_first; bool is_dsp_mode; bool is_pdm_mode; + bool is_multi_fifo_dma; bool synchronous[2]; struct fsl_sai_dl_cfg *dl_cfg; unsigned int dl_cfg_cnt; @@ -300,6 +302,7 @@ struct fsl_sai { struct pm_qos_request pm_qos_req; struct pinctrl *pinctrl; struct pinctrl_state *pins_state; + struct sdma_peripheral_config audio_config[2]; }; #define TX 1 diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c index 7fc1c96929bb841a282b3b728254d06adf8b3e90..275aba8e0c46959cc6d4afa77469801b41a7bb66 100644 --- a/sound/soc/fsl/fsl_spdif.c +++ b/sound/soc/fsl/fsl_spdif.c @@ -44,6 +44,8 @@ static u8 srpc_dpll_locked[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0xa, 0xb }; #define DEFAULT_RXCLK_SRC 1 +#define RX_SAMPLE_RATE_KCONTROL "RX Sample Rate" + /** * struct fsl_spdif_soc_data: soc specific data * @@ -98,6 +100,8 @@ struct spdif_mixer_control { * @soc: SPDIF soc data * @fsl_spdif_control: SPDIF control data * @cpu_dai_drv: cpu dai driver + * @snd_card: sound card pointer + * @rxrate_kcontrol: kcontrol for RX Sample Rate * @pdev: platform device pointer * @regmap: regmap handler * @dpll_locked: dpll lock flag @@ -122,6 +126,8 @@ struct fsl_spdif_priv { const struct fsl_spdif_soc_data *soc; struct spdif_mixer_control fsl_spdif_control; struct snd_soc_dai_driver cpu_dai_drv; + struct snd_card *snd_card; + struct snd_kcontrol *rxrate_kcontrol; struct platform_device *pdev; struct regmap *regmap; bool dpll_locked; @@ -226,6 +232,12 @@ static void spdif_irq_dpll_lock(struct fsl_spdif_priv *spdif_priv) locked ? "locked" : "loss lock"); spdif_priv->dpll_locked = locked ? true : false; + + if (spdif_priv->snd_card && spdif_priv->rxrate_kcontrol) { + snd_ctl_notify(spdif_priv->snd_card, + SNDRV_CTL_EVENT_MASK_VALUE, + &spdif_priv->rxrate_kcontrol->id); + } } /* Receiver found illegal symbol interrupt handler */ @@ -1197,7 +1209,7 @@ static struct snd_kcontrol_new fsl_spdif_ctrls[] = { /* DPLL lock info get controller */ { .iface = SNDRV_CTL_ELEM_IFACE_PCM, - .name = "RX Sample Rate", + .name = RX_SAMPLE_RATE_KCONTROL, .access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, .info = fsl_spdif_rxrate_info, @@ -1251,6 +1263,13 @@ static int fsl_spdif_dai_probe(struct snd_soc_dai *dai) snd_soc_add_dai_controls(dai, fsl_spdif_ctrls_rcm, ARRAY_SIZE(fsl_spdif_ctrls_rcm)); + spdif_private->snd_card = dai->component->card->snd_card; + spdif_private->rxrate_kcontrol = snd_soc_card_get_kcontrol(dai->component->card, + RX_SAMPLE_RATE_KCONTROL); + if (!spdif_private->rxrate_kcontrol) + dev_err(&spdif_private->pdev->dev, "failed to get %s kcontrol\n", + RX_SAMPLE_RATE_KCONTROL); + /*Clear the val bit for Tx*/ regmap_update_bits(spdif_private->regmap, REG_SPDIF_SCR, SCR_VAL_MASK, SCR_VAL_CLEAR); diff --git a/sound/soc/fsl/fsl_utils.c b/sound/soc/fsl/fsl_utils.c index d0fc430f7033dbaa015175d1247f1d23596c4a93..a5ab27c2f711c849efb8a7e4668007ce8ac0f596 100644 --- a/sound/soc/fsl/fsl_utils.c +++ b/sound/soc/fsl/fsl_utils.c @@ -124,7 +124,7 @@ void fsl_asoc_reparent_pll_clocks(struct device *dev, struct clk *clk, { struct clk *p, *pll = NULL, *npll = NULL; bool reparent = false; - int ret = 0; + int ret; if (!clk || !pll8k_clk || !pll11k_clk) return; diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c index 14be29530fb5dfb47db422be2d356aa4f0f67a18..3f128ced418098521355fad94f02350041736256 100644 --- a/sound/soc/fsl/imx-card.c +++ b/sound/soc/fsl/imx-card.c @@ -698,6 +698,10 @@ static int imx_card_parse_of(struct imx_card_data *data) of_node_put(cpu); of_node_put(codec); of_node_put(platform); + + cpu = NULL; + codec = NULL; + platform = NULL; } return 0; diff --git a/sound/soc/fsl/imx-pcm.h b/sound/soc/fsl/imx-pcm.h index 06b25f4b26b6f4a0963fbe7799d0f64a273f605a..ac5f57c3cc55cd79b42fa0e353b68bc4160c9fad 100644 --- a/sound/soc/fsl/imx-pcm.h +++ b/sound/soc/fsl/imx-pcm.h @@ -18,15 +18,6 @@ #define IMX_DEFAULT_DMABUF_SIZE (64 * 1024) -static inline void -imx_pcm_dma_params_init_data(struct imx_dma_data *dma_data, - int dma, enum sdma_peripheral_type peripheral_type) -{ - dma_data->dma_request = dma; - dma_data->priority = DMA_PRIO_HIGH; - dma_data->peripheral_type = peripheral_type; -} - struct imx_pcm_fiq_params { int irq; void __iomem *base; diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c index 2e117311e582fa99d2eca326b3e7739e6c45eed0..4d99f4858a14f51b33882f4acc89c3bbd06dbfec 100644 --- a/sound/soc/fsl/imx-rpmsg.c +++ b/sound/soc/fsl/imx-rpmsg.c @@ -19,6 +19,7 @@ struct imx_rpmsg { struct snd_soc_dai_link dai; struct snd_soc_card card; + unsigned long sysclk; }; static const struct snd_soc_dapm_widget imx_rpmsg_dapm_widgets[] = { @@ -28,6 +29,27 @@ static const struct snd_soc_dapm_widget imx_rpmsg_dapm_widgets[] = { SND_SOC_DAPM_MIC("Main MIC", NULL), }; +static int imx_rpmsg_late_probe(struct snd_soc_card *card) +{ + struct imx_rpmsg *data = snd_soc_card_get_drvdata(card); + struct snd_soc_pcm_runtime *rtd = list_first_entry(&card->rtd_list, + struct snd_soc_pcm_runtime, list); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct device *dev = card->dev; + int ret; + + if (!data->sysclk) + return 0; + + ret = snd_soc_dai_set_sysclk(codec_dai, 0, data->sysclk, SND_SOC_CLOCK_IN); + if (ret && ret != -ENOTSUPP) { + dev_err(dev, "failed to set sysclk in %s\n", __func__); + return ret; + } + + return 0; +} + static int imx_rpmsg_probe(struct platform_device *pdev) { struct snd_soc_dai_link_component *dlc; @@ -72,12 +94,18 @@ static int imx_rpmsg_probe(struct platform_device *pdev) data->dai.codecs->dai_name = "snd-soc-dummy-dai"; data->dai.codecs->name = "snd-soc-dummy"; } else { + struct clk *clk; + data->dai.codecs->of_node = args.np; ret = snd_soc_get_dai_name(&args, &data->dai.codecs->dai_name); if (ret) { dev_err(&pdev->dev, "Unable to get codec_dai_name\n"); goto fail; } + + clk = devm_get_clk_from_child(&pdev->dev, args.np, NULL); + if (!IS_ERR(clk)) + data->sysclk = clk_get_rate(clk); } data->dai.cpus->dai_name = dev_name(&rpmsg_pdev->dev); @@ -103,6 +131,7 @@ static int imx_rpmsg_probe(struct platform_device *pdev) data->card.owner = THIS_MODULE; data->card.dapm_widgets = imx_rpmsg_dapm_widgets; data->card.num_dapm_widgets = ARRAY_SIZE(imx_rpmsg_dapm_widgets); + data->card.late_probe = imx_rpmsg_late_probe; /* * Inoder to use common api to get card name and audio routing. * Use parent of_node for this device, revert it after finishing using diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c index 4a29e314fa9530beac9b4352ed77e7f5953fdabd..bef16833c487e27eaf6ff7a2716e88a1d5cd5356 100644 --- a/sound/soc/generic/simple-card-utils.c +++ b/sound/soc/generic/simple-card-utils.c @@ -9,12 +9,38 @@ #include <linux/gpio/consumer.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_gpio.h> #include <linux/of_graph.h> #include <sound/jack.h> #include <sound/pcm_params.h> #include <sound/simple_card_utils.h> +static void asoc_simple_fixup_sample_fmt(struct asoc_simple_data *data, + struct snd_pcm_hw_params *params) +{ + int i; + struct snd_mask *mask = hw_param_mask(params, + SNDRV_PCM_HW_PARAM_FORMAT); + struct { + char *fmt; + u32 val; + } of_sample_fmt_table[] = { + { "s8", SNDRV_PCM_FORMAT_S8}, + { "s16_le", SNDRV_PCM_FORMAT_S16_LE}, + { "s24_le", SNDRV_PCM_FORMAT_S24_LE}, + { "s24_3le", SNDRV_PCM_FORMAT_S24_3LE}, + { "s32_le", SNDRV_PCM_FORMAT_S32_LE}, + }; + + for (i = 0; i < ARRAY_SIZE(of_sample_fmt_table); i++) { + if (!strcmp(data->convert_sample_format, + of_sample_fmt_table[i].fmt)) { + snd_mask_none(mask); + snd_mask_set(mask, of_sample_fmt_table[i].val); + break; + } + } +} + void asoc_simple_convert_fixup(struct asoc_simple_data *data, struct snd_pcm_hw_params *params) { @@ -30,6 +56,9 @@ void asoc_simple_convert_fixup(struct asoc_simple_data *data, if (data->convert_channels) channels->min = channels->max = data->convert_channels; + + if (data->convert_sample_format) + asoc_simple_fixup_sample_fmt(data, params); } EXPORT_SYMBOL_GPL(asoc_simple_convert_fixup); @@ -49,6 +78,10 @@ void asoc_simple_parse_convert(struct device_node *np, /* channels transfer */ snprintf(prop, sizeof(prop), "%s%s", prefix, "convert-channels"); of_property_read_u32(np, prop, &data->convert_channels); + + /* convert sample format */ + snprintf(prop, sizeof(prop), "%s%s", prefix, "convert-sample-format"); + of_property_read_string(np, prop, &data->convert_sample_format); } EXPORT_SYMBOL_GPL(asoc_simple_parse_convert); @@ -695,12 +728,12 @@ int asoc_simple_init_jack(struct snd_soc_card *card, char *pin) { struct device *dev = card->dev; - enum of_gpio_flags flags; + struct gpio_desc *desc; char prop[128]; char *pin_name; char *gpio_name; int mask; - int det; + int error; if (!prefix) prefix = ""; @@ -708,36 +741,39 @@ int asoc_simple_init_jack(struct snd_soc_card *card, sjack->gpio.gpio = -ENOENT; if (is_hp) { - snprintf(prop, sizeof(prop), "%shp-det-gpio", prefix); + snprintf(prop, sizeof(prop), "%shp-det", prefix); pin_name = pin ? pin : "Headphones"; gpio_name = "Headphone detection"; mask = SND_JACK_HEADPHONE; } else { - snprintf(prop, sizeof(prop), "%smic-det-gpio", prefix); + snprintf(prop, sizeof(prop), "%smic-det", prefix); pin_name = pin ? pin : "Mic Jack"; gpio_name = "Mic detection"; mask = SND_JACK_MICROPHONE; } - det = of_get_named_gpio_flags(dev->of_node, prop, 0, &flags); - if (det == -EPROBE_DEFER) - return -EPROBE_DEFER; + desc = gpiod_get_optional(dev, prop, GPIOD_IN); + error = PTR_ERR_OR_ZERO(desc); + if (error) + return error; + + if (desc) { + error = gpiod_set_consumer_name(desc, gpio_name); + if (error) + return error; - if (gpio_is_valid(det)) { sjack->pin.pin = pin_name; sjack->pin.mask = mask; sjack->gpio.name = gpio_name; sjack->gpio.report = mask; - sjack->gpio.gpio = det; - sjack->gpio.invert = !!(flags & OF_GPIO_ACTIVE_LOW); + sjack->gpio.desc = desc; sjack->gpio.debounce_time = 150; snd_soc_card_jack_new_pins(card, pin_name, mask, &sjack->jack, &sjack->pin, 1); - snd_soc_jack_add_gpios(&sjack->jack, 1, - &sjack->gpio); + snd_soc_jack_add_gpios(&sjack->jack, 1, &sjack->gpio); } return 0; diff --git a/sound/soc/intel/Kconfig b/sound/soc/intel/Kconfig index ded903f95b67c5c16a950580d5401749bf40a86f..d2ca710ac3fa4ffd7eeb6faf3c8c73695e7ff936 100644 --- a/sound/soc/intel/Kconfig +++ b/sound/soc/intel/Kconfig @@ -23,7 +23,7 @@ config SND_SOC_INTEL_CATPT depends on ACPI || COMPILE_TEST depends on DMADEVICES && SND_DMA_SGBUF select DW_DMAC_CORE - select SND_SOC_ACPI_INTEL_MATCH + select SND_SOC_ACPI if ACPI select WANT_DEV_COREDUMP select SND_INTEL_DSP_CONFIG help diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c index a56dd48c045f3e9330e97ab5573266c59fba6436..c75616a5fd0abc150b9efc3c17b38932e9c82916 100644 --- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c +++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c @@ -676,10 +676,9 @@ static int sst_soc_pcm_new(struct snd_soc_component *component, if (dai->driver->playback.channels_min || dai->driver->capture.channels_min) { - snd_pcm_set_managed_buffer_all(pcm, - SNDRV_DMA_TYPE_CONTINUOUS, - snd_dma_continuous_data(GFP_DMA), - SST_MIN_BUFFER, SST_MAX_BUFFER); + snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV, + pcm->card->dev, + SST_MIN_BUFFER, SST_MAX_BUFFER); } return 0; } diff --git a/sound/soc/intel/atom/sst/sst.c b/sound/soc/intel/atom/sst/sst.c index 160b50f479fb4f3a1171f0785a705a76fba8dc80..a0d29510d2bc486cbaeb7f3a9bae4287136e2aeb 100644 --- a/sound/soc/intel/atom/sst/sst.c +++ b/sound/soc/intel/atom/sst/sst.c @@ -242,11 +242,11 @@ static ssize_t firmware_version_show(struct device *dev, if (ctx->fw_version.type == 0 && ctx->fw_version.major == 0 && ctx->fw_version.minor == 0 && ctx->fw_version.build == 0) - return sprintf(buf, "FW not yet loaded\n"); + return sysfs_emit(buf, "FW not yet loaded\n"); else - return sprintf(buf, "v%02x.%02x.%02x.%02x\n", - ctx->fw_version.type, ctx->fw_version.major, - ctx->fw_version.minor, ctx->fw_version.build); + return sysfs_emit(buf, "v%02x.%02x.%02x.%02x\n", + ctx->fw_version.type, ctx->fw_version.major, + ctx->fw_version.minor, ctx->fw_version.build); } diff --git a/sound/soc/intel/avs/boards/hdaudio.c b/sound/soc/intel/avs/boards/hdaudio.c index d2fc41d39448d4c5e85b9ba163bd440f0a592e19..073663ba140d0fe358255de8b5b06b4ef0032a6d 100644 --- a/sound/soc/intel/avs/boards/hdaudio.c +++ b/sound/soc/intel/avs/boards/hdaudio.c @@ -42,6 +42,7 @@ static int avs_create_dai_links(struct device *dev, struct hda_codec *codec, int dl[i].dpcm_capture = 1; dl[i].platforms = platform; dl[i].num_platforms = 1; + dl[i].ignore_pmdown_time = 1; dl[i].codecs = devm_kzalloc(dev, sizeof(*dl->codecs), GFP_KERNEL); dl[i].cpus = devm_kzalloc(dev, sizeof(*dl->cpus), GFP_KERNEL); diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index c50c20fd681a165f3afc6a13726663d0ec2bfa99..bb0719c58ca49673fe6e9a029549a86130233998 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -466,7 +466,7 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) err_acquire_irq: snd_hdac_bus_free_stream_pages(bus); - snd_hdac_stream_free_all(bus); + snd_hdac_ext_stream_free_all(bus); err_init_streams: iounmap(adev->dsp_ba); err_remap_bar4: @@ -502,7 +502,7 @@ static void avs_pci_remove(struct pci_dev *pci) snd_hda_codec_unregister(hdac_to_hda_codec(hdev)); snd_hdac_bus_free_stream_pages(bus); - snd_hdac_stream_free_all(bus); + snd_hdac_ext_stream_free_all(bus); /* reverse ml_capabilities */ snd_hdac_link_free_all(bus); snd_hdac_ext_bus_exit(bus); diff --git a/sound/soc/intel/boards/Makefile b/sound/soc/intel/boards/Makefile index eea1e26acfdaa206eef3f9d938d2a8d47294479d..53458e748191ae7942c0a8a43e1d268ddd0e2025 100644 --- a/sound/soc/intel/boards/Makefile +++ b/sound/soc/intel/boards/Makefile @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only -snd-soc-sst-haswell-objs := hsw_rt5640.o +snd-soc-hsw-rt5640-objs := hsw_rt5640.o snd-soc-sst-bdw-rt5650-mach-objs := bdw-rt5650.o snd-soc-sst-bdw-rt5677-mach-objs := bdw-rt5677.o -snd-soc-sst-broadwell-objs := bdw_rt286.o +snd-soc-bdw-rt286-objs := bdw_rt286.o snd-soc-sst-bxt-da7219_max98357a-objs := bxt_da7219_max98357a.o snd-soc-sst-bxt-rt298-objs := bxt_rt298.o snd-soc-sst-sof-pcm512x-objs := sof_pcm512x.o @@ -47,13 +47,13 @@ obj-$(CONFIG_SND_SOC_INTEL_SOF_RT5682_MACH) += snd-soc-sof_rt5682.o obj-$(CONFIG_SND_SOC_INTEL_SOF_CS42L42_MACH) += snd-soc-sof_cs42l42.o obj-$(CONFIG_SND_SOC_INTEL_SOF_ES8336_MACH) += snd-soc-sof_es8336.o obj-$(CONFIG_SND_SOC_INTEL_SOF_NAU8825_MACH) += snd-soc-sof_nau8825.o -obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-sst-haswell.o +obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-hsw-rt5640.o obj-$(CONFIG_SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON) += snd-soc-sst-bxt-da7219_max98357a.o obj-$(CONFIG_SND_SOC_INTEL_BXT_RT298_MACH) += snd-soc-sst-bxt-rt298.o obj-$(CONFIG_SND_SOC_INTEL_SOF_PCM512x_MACH) += snd-soc-sst-sof-pcm512x.o obj-$(CONFIG_SND_SOC_INTEL_SOF_WM8804_MACH) += snd-soc-sst-sof-wm8804.o obj-$(CONFIG_SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH) += snd-soc-sst-glk-rt5682_max98357a.o -obj-$(CONFIG_SND_SOC_INTEL_BROADWELL_MACH) += snd-soc-sst-broadwell.o +obj-$(CONFIG_SND_SOC_INTEL_BROADWELL_MACH) += snd-soc-bdw-rt286.o obj-$(CONFIG_SND_SOC_INTEL_BDW_RT5650_MACH) += snd-soc-sst-bdw-rt5650-mach.o obj-$(CONFIG_SND_SOC_INTEL_BDW_RT5677_MACH) += snd-soc-sst-bdw-rt5677-mach.o obj-$(CONFIG_SND_SOC_INTEL_BYTCR_RT5640_MACH) += snd-soc-sst-bytcr-rt5640.o diff --git a/sound/soc/intel/boards/hda_dsp_common.c b/sound/soc/intel/boards/hda_dsp_common.c index 83c7dfbccd9d4575812705b9f1dd12881eb88444..04b7d4f7f9e240fbae42ecf3437350e5bc796a15 100644 --- a/sound/soc/intel/boards/hda_dsp_common.c +++ b/sound/soc/intel/boards/hda_dsp_common.c @@ -54,7 +54,7 @@ int hda_dsp_hdmi_build_controls(struct snd_soc_card *card, return -EINVAL; hda_pvt = snd_soc_component_get_drvdata(comp); - hcodec = &hda_pvt->codec; + hcodec = hda_pvt->codec; list_for_each_entry(hpcm, &hcodec->pcm_list_head, list) { spcm = hda_dsp_hdmi_pcm_handle(card, i); diff --git a/sound/soc/intel/boards/skl_hda_dsp_generic.c b/sound/soc/intel/boards/skl_hda_dsp_generic.c index 81144efb4b44e4bcc826f653c32e336d40972687..879ebba5283229165797f8d0781b17e3047c1b82 100644 --- a/sound/soc/intel/boards/skl_hda_dsp_generic.c +++ b/sound/soc/intel/boards/skl_hda_dsp_generic.c @@ -190,7 +190,7 @@ static void skl_set_hda_codec_autosuspend_delay(struct snd_soc_card *card) * all codecs are on the same bus, so it's sufficient * to look up only the first one */ - snd_hda_set_power_save(hda_pvt->codec.bus, + snd_hda_set_power_save(hda_pvt->codec->bus, HDA_CODEC_AUTOSUSPEND_DELAY_MS); break; } diff --git a/sound/soc/intel/boards/sof_cirrus_common.c b/sound/soc/intel/boards/sof_cirrus_common.c index f4192df962d60faf89ca1029ee055d028d434875..6e39eda77385b3de8271094a1af464ddb57a270e 100644 --- a/sound/soc/intel/boards/sof_cirrus_common.c +++ b/sound/soc/intel/boards/sof_cirrus_common.c @@ -10,6 +10,9 @@ #include "../../codecs/cs35l41.h" #include "sof_cirrus_common.h" +#define CS35L41_HID "CSC3541" +#define CS35L41_MAX_AMPS 4 + /* * Cirrus Logic CS35L41/CS35L53 */ @@ -35,50 +38,12 @@ static const struct snd_soc_dapm_route cs35l41_dapm_routes[] = { {"TR Spk", NULL, "TR SPK"}, }; -static struct snd_soc_dai_link_component cs35l41_components[] = { - { - .name = CS35L41_DEV0_NAME, - .dai_name = CS35L41_CODEC_DAI, - }, - { - .name = CS35L41_DEV1_NAME, - .dai_name = CS35L41_CODEC_DAI, - }, - { - .name = CS35L41_DEV2_NAME, - .dai_name = CS35L41_CODEC_DAI, - }, - { - .name = CS35L41_DEV3_NAME, - .dai_name = CS35L41_CODEC_DAI, - }, -}; +static struct snd_soc_dai_link_component cs35l41_components[CS35L41_MAX_AMPS]; /* * Mapping between ACPI instance id and speaker position. - * - * Four speakers: - * 0: Tweeter left, 1: Woofer left - * 2: Tweeter right, 3: Woofer right */ -static struct snd_soc_codec_conf cs35l41_codec_conf[] = { - { - .dlc = COMP_CODEC_CONF(CS35L41_DEV0_NAME), - .name_prefix = "TL", - }, - { - .dlc = COMP_CODEC_CONF(CS35L41_DEV1_NAME), - .name_prefix = "WL", - }, - { - .dlc = COMP_CODEC_CONF(CS35L41_DEV2_NAME), - .name_prefix = "TR", - }, - { - .dlc = COMP_CODEC_CONF(CS35L41_DEV3_NAME), - .name_prefix = "WR", - }, -}; +static struct snd_soc_codec_conf cs35l41_codec_conf[CS35L41_MAX_AMPS]; static int cs35l41_init(struct snd_soc_pcm_runtime *rtd) { @@ -117,10 +82,10 @@ static int cs35l41_init(struct snd_soc_pcm_runtime *rtd) static const struct { unsigned int rx[2]; } cs35l41_channel_map[] = { - {.rx = {0, 1}}, /* TL */ {.rx = {0, 1}}, /* WL */ - {.rx = {1, 0}}, /* TR */ {.rx = {1, 0}}, /* WR */ + {.rx = {0, 1}}, /* TL */ + {.rx = {1, 0}}, /* TR */ }; static int cs35l41_hw_params(struct snd_pcm_substream *substream, @@ -175,10 +140,51 @@ static const struct snd_soc_ops cs35l41_ops = { .hw_params = cs35l41_hw_params, }; +static const char * const cs35l41_name_prefixes[] = { "WL", "WR", "TL", "TR" }; + +/* + * Expected UIDs are integers (stored as strings). + * UID Mapping is fixed: + * UID 0x0 -> WL + * UID 0x1 -> WR + * UID 0x2 -> TL + * UID 0x3 -> TR + * Note: If there are less than 4 Amps, UIDs still map to WL/WR/TL/TR. Dynamic code will only create + * dai links for UIDs which exist, and ignore non-existant ones. Only 2 or 4 amps are expected. + * Return number of codecs found. + */ +static int cs35l41_compute_codec_conf(void) +{ + const char * const uid_strings[] = { "0", "1", "2", "3" }; + unsigned int uid, sz = 0; + struct acpi_device *adev; + struct device *physdev; + + for (uid = 0; uid < CS35L41_MAX_AMPS; uid++) { + adev = acpi_dev_get_first_match_dev(CS35L41_HID, uid_strings[uid], -1); + if (!adev) { + pr_devel("Cannot find match for HID %s UID %u (%s)\n", CS35L41_HID, uid, + cs35l41_name_prefixes[uid]); + continue; + } + physdev = get_device(acpi_get_first_physical_node(adev)); + cs35l41_components[sz].name = dev_name(physdev); + cs35l41_components[sz].dai_name = CS35L41_CODEC_DAI; + cs35l41_codec_conf[sz].dlc.name = dev_name(physdev); + cs35l41_codec_conf[sz].name_prefix = cs35l41_name_prefixes[uid]; + acpi_dev_put(adev); + sz++; + } + + if (sz != 2 && sz != 4) + pr_warn("Invalid number of cs35l41 amps found: %d, expected 2 or 4\n", sz); + return sz; +} + void cs35l41_set_dai_link(struct snd_soc_dai_link *link) { + link->num_codecs = cs35l41_compute_codec_conf(); link->codecs = cs35l41_components; - link->num_codecs = ARRAY_SIZE(cs35l41_components); link->init = cs35l41_init; link->ops = &cs35l41_ops; } diff --git a/sound/soc/intel/boards/sof_cs42l42.c b/sound/soc/intel/boards/sof_cs42l42.c index 85ffd065895d3bc0d3df5638e844ab066f7f702c..e38bd2831e6ac77a6e7486ea47889113e0246223 100644 --- a/sound/soc/intel/boards/sof_cs42l42.c +++ b/sound/soc/intel/boards/sof_cs42l42.c @@ -445,9 +445,9 @@ static int create_hdmi_dai_links(struct device *dev, if (hdmi_num <= 0) return 0; - idisp_components = devm_kzalloc(dev, - sizeof(struct snd_soc_dai_link_component) * - hdmi_num, GFP_KERNEL); + idisp_components = devm_kcalloc(dev, + hdmi_num, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); if (!idisp_components) goto devm_err; @@ -543,10 +543,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, struct snd_soc_dai_link *links; int ret, id = 0, link_seq; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * - sof_audio_card_cs42l42.num_links, GFP_KERNEL); - cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) * - sof_audio_card_cs42l42.num_links, GFP_KERNEL); + links = devm_kcalloc(dev, sof_audio_card_cs42l42.num_links, + sizeof(struct snd_soc_dai_link), GFP_KERNEL); + cpus = devm_kcalloc(dev, sof_audio_card_cs42l42.num_links, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); if (!links || !cpus) goto devm_err; diff --git a/sound/soc/intel/boards/sof_da7219_max98373.c b/sound/soc/intel/boards/sof_da7219_max98373.c index 34cf849a8344e9382a01eacfad7ca279de43ee71..e048e789e633d9f5367f469d054cf7630468c8f5 100644 --- a/sound/soc/intel/boards/sof_da7219_max98373.c +++ b/sound/soc/intel/boards/sof_da7219_max98373.c @@ -195,7 +195,7 @@ static int ssp1_hw_params(struct snd_pcm_substream *substream, struct snd_soc_pcm_runtime *runtime = asoc_substream_to_rtd(substream); int ret, j; - for (j = 0; j < runtime->num_codecs; j++) { + for (j = 0; j < runtime->dai_link->num_codecs; j++) { struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(runtime, j); if (!strcmp(codec_dai->component->name, MAXIM_DEV0_NAME)) { diff --git a/sound/soc/intel/boards/sof_es8336.c b/sound/soc/intel/boards/sof_es8336.c index 606cc3242a60fbfe9f9d166713a23bf9ec9060a2..fbb42e54947a8c9547c034cf2ccdfe998d50b8d5 100644 --- a/sound/soc/intel/boards/sof_es8336.c +++ b/sound/soc/intel/boards/sof_es8336.c @@ -481,9 +481,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, /* HDMI */ if (hdmi_num > 0) { - idisp_components = devm_kzalloc(dev, - sizeof(struct snd_soc_dai_link_component) * - hdmi_num, GFP_KERNEL); + idisp_components = devm_kcalloc(dev, + hdmi_num, + sizeof(struct snd_soc_dai_link_component), + GFP_KERNEL); if (!idisp_components) goto devm_err; } diff --git a/sound/soc/intel/boards/sof_nau8825.c b/sound/soc/intel/boards/sof_nau8825.c index 8d7e5ba9e51627521e7f0ce122dbb829560df0cf..5585c217f78d3a8ac0196ccd8e3977697df6d4e6 100644 --- a/sound/soc/intel/boards/sof_nau8825.c +++ b/sound/soc/intel/boards/sof_nau8825.c @@ -355,10 +355,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, struct snd_soc_dai_link *links; int i, id = 0; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * - sof_audio_card_nau8825.num_links, GFP_KERNEL); - cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) * - sof_audio_card_nau8825.num_links, GFP_KERNEL); + links = devm_kcalloc(dev, sof_audio_card_nau8825.num_links, + sizeof(struct snd_soc_dai_link), GFP_KERNEL); + cpus = devm_kcalloc(dev, sof_audio_card_nau8825.num_links, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); if (!links || !cpus) goto devm_err; @@ -421,9 +421,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, /* HDMI */ if (hdmi_num > 0) { - idisp_components = devm_kzalloc(dev, - sizeof(struct snd_soc_dai_link_component) * - hdmi_num, GFP_KERNEL); + idisp_components = devm_kcalloc(dev, + hdmi_num, + sizeof(struct snd_soc_dai_link_component), + GFP_KERNEL); if (!idisp_components) goto devm_err; } diff --git a/sound/soc/intel/boards/sof_realtek_common.c b/sound/soc/intel/boards/sof_realtek_common.c index b9643ca2e2f228cf220e322ab3ca44d08c93edd8..ff2851fc8930a2126d8e6976bb8c14706ab864b7 100644 --- a/sound/soc/intel/boards/sof_realtek_common.c +++ b/sound/soc/intel/boards/sof_realtek_common.c @@ -253,63 +253,70 @@ EXPORT_SYMBOL_NS(sof_rt1015p_codec_conf, SND_SOC_INTEL_SOF_REALTEK_COMMON); * RT1015 audio amplifier */ +static const struct { + unsigned int tx; + unsigned int rx; +} rt1015_tdm_mask[] = { + {.tx = 0x0, .rx = 0x1}, + {.tx = 0x0, .rx = 0x2}, +}; + static int rt1015_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai_link *dai_link = rtd->dai_link; struct snd_soc_dai *codec_dai; - int i, fs = 64, ret; + int i, clk_freq, ret; - for_each_rtd_codec_dais(rtd, i, codec_dai) { - ret = snd_soc_dai_set_pll(codec_dai, 0, RT1015_PLL_S_BCLK, - params_rate(params) * fs, - params_rate(params) * 256); - if (ret) - return ret; + clk_freq = sof_dai_get_bclk(rtd); - ret = snd_soc_dai_set_sysclk(codec_dai, RT1015_SCLK_S_PLL, - params_rate(params) * 256, - SND_SOC_CLOCK_IN); - if (ret) - return ret; + if (clk_freq <= 0) { + dev_err(rtd->dev, "fail to get bclk freq, ret %d\n", clk_freq); + return -EINVAL; } - return 0; -} - -static int rt1015_hw_params_pll_and_tdm(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params) -{ - struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); - struct snd_soc_dai *codec_dai; - int i, fs = 100, ret; - for_each_rtd_codec_dais(rtd, i, codec_dai) { ret = snd_soc_dai_set_pll(codec_dai, 0, RT1015_PLL_S_BCLK, - params_rate(params) * fs, + clk_freq, params_rate(params) * 256); - if (ret) + if (ret) { + dev_err(codec_dai->dev, "fail to set pll, ret %d\n", + ret); return ret; + } ret = snd_soc_dai_set_sysclk(codec_dai, RT1015_SCLK_S_PLL, params_rate(params) * 256, SND_SOC_CLOCK_IN); - if (ret) + if (ret) { + dev_err(codec_dai->dev, "fail to set sysclk, ret %d\n", + ret); return ret; - } - /* rx slot 1 for RT1015_DEV0_NAME */ - ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0), - 0x0, 0x1, 4, 24); - if (ret) - return ret; + } - /* rx slot 2 for RT1015_DEV1_NAME */ - ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 1), - 0x0, 0x2, 4, 24); - if (ret) - return ret; + switch (dai_link->dai_fmt & SND_SOC_DAIFMT_FORMAT_MASK) { + case SND_SOC_DAIFMT_DSP_A: + case SND_SOC_DAIFMT_DSP_B: + /* 4-slot TDM */ + ret = snd_soc_dai_set_tdm_slot(codec_dai, + rt1015_tdm_mask[i].tx, + rt1015_tdm_mask[i].rx, + 4, + params_width(params)); + if (ret < 0) { + dev_err(codec_dai->dev, "fail to set tdm slot, ret %d\n", + ret); + return ret; + } + break; + default: + dev_dbg(codec_dai->dev, "codec is in I2S mode\n"); + break; + } + } - return 0; + return ret; } static struct snd_soc_ops rt1015_ops = { @@ -351,15 +358,12 @@ void sof_rt1015_codec_conf(struct snd_soc_card *card) } EXPORT_SYMBOL_NS(sof_rt1015_codec_conf, SND_SOC_INTEL_SOF_REALTEK_COMMON); -void sof_rt1015_dai_link(struct snd_soc_dai_link *link, unsigned int fs) +void sof_rt1015_dai_link(struct snd_soc_dai_link *link) { link->codecs = rt1015_components; link->num_codecs = ARRAY_SIZE(rt1015_components); link->init = speaker_codec_init_lr; link->ops = &rt1015_ops; - - if (fs == 100) - rt1015_ops.hw_params = rt1015_hw_params_pll_and_tdm; } EXPORT_SYMBOL_NS(sof_rt1015_dai_link, SND_SOC_INTEL_SOF_REALTEK_COMMON); diff --git a/sound/soc/intel/boards/sof_realtek_common.h b/sound/soc/intel/boards/sof_realtek_common.h index 77844342109003cc678c4f08fd7101f919446046..3ae99d8239e0c068df3c18a00ba6e83e79a8593e 100644 --- a/sound/soc/intel/boards/sof_realtek_common.h +++ b/sound/soc/intel/boards/sof_realtek_common.h @@ -32,7 +32,7 @@ void sof_rt1015p_codec_conf(struct snd_soc_card *card); #define RT1015_DEV0_NAME "i2c-10EC1015:00" #define RT1015_DEV1_NAME "i2c-10EC1015:01" -void sof_rt1015_dai_link(struct snd_soc_dai_link *link, unsigned int fs); +void sof_rt1015_dai_link(struct snd_soc_dai_link *link); void sof_rt1015_codec_conf(struct snd_soc_card *card); #define RT1308_CODEC_DAI "rt1308-aif" diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c index 045965312245b4bb5ace04ab3ef2e940b3703f79..2d0986824b3d7a45d3a08877eb6bf91403c3796d 100644 --- a/sound/soc/intel/boards/sof_rt5682.c +++ b/sound/soc/intel/boards/sof_rt5682.c @@ -46,7 +46,6 @@ ((quirk << SOF_RT5682_NUM_HDMIDEV_SHIFT) & SOF_RT5682_NUM_HDMIDEV_MASK) #define SOF_RT1011_SPEAKER_AMP_PRESENT BIT(13) #define SOF_RT1015_SPEAKER_AMP_PRESENT BIT(14) -#define SOF_RT1015_SPEAKER_AMP_100FS BIT(15) #define SOF_RT1015P_SPEAKER_AMP_PRESENT BIT(16) #define SOF_MAX98373_SPEAKER_AMP_PRESENT BIT(17) #define SOF_MAX98360A_SPEAKER_AMP_PRESENT BIT(18) @@ -132,7 +131,6 @@ static const struct dmi_system_id sof_rt5682_quirk_table[] = { SOF_RT5682_SSP_CODEC(0) | SOF_SPEAKER_AMP_PRESENT | SOF_RT1015_SPEAKER_AMP_PRESENT | - SOF_RT1015_SPEAKER_AMP_100FS | SOF_RT5682_SSP_AMP(1)), }, { @@ -600,10 +598,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, struct snd_soc_dai_link *links; int i, id = 0; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * - sof_audio_card_rt5682.num_links, GFP_KERNEL); - cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) * - sof_audio_card_rt5682.num_links, GFP_KERNEL); + links = devm_kcalloc(dev, sof_audio_card_rt5682.num_links, + sizeof(struct snd_soc_dai_link), GFP_KERNEL); + cpus = devm_kcalloc(dev, sof_audio_card_rt5682.num_links, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); if (!links || !cpus) goto devm_err; @@ -687,9 +685,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, /* HDMI */ if (hdmi_num > 0) { - idisp_components = devm_kzalloc(dev, - sizeof(struct snd_soc_dai_link_component) * - hdmi_num, GFP_KERNEL); + idisp_components = devm_kcalloc(dev, + hdmi_num, + sizeof(struct snd_soc_dai_link_component), + GFP_KERNEL); if (!idisp_components) goto devm_err; } @@ -739,8 +738,7 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, links[id].id = id; if (sof_rt5682_quirk & SOF_RT1015_SPEAKER_AMP_PRESENT) { - sof_rt1015_dai_link(&links[id], (sof_rt5682_quirk & - SOF_RT1015_SPEAKER_AMP_100FS) ? 100 : 64); + sof_rt1015_dai_link(&links[id]); } else if (sof_rt5682_quirk & SOF_RT1015P_SPEAKER_AMP_PRESENT) { sof_rt1015p_dai_link(&links[id]); } else if (sof_rt5682_quirk & SOF_RT1019_SPEAKER_AMP_PRESENT) { @@ -1010,7 +1008,6 @@ static const struct platform_device_id board_ids[] = { SOF_RT5682_SSP_CODEC(0) | SOF_SPEAKER_AMP_PRESENT | SOF_RT1015_SPEAKER_AMP_PRESENT | - SOF_RT1015_SPEAKER_AMP_100FS | SOF_RT5682_SSP_AMP(1)), }, { diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index a49bfaab6b21fa8c41cd63bea0cc5c933d47d6e7..2ff30b40a1e4cf94a3e53624a551893f7d080864 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -266,6 +266,16 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { .driver_data = (void *)(SOF_SDW_TGL_HDMI | SOF_SDW_FOUR_SPK), }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0AFF") + }, + .driver_data = (void *)(SOF_SDW_TGL_HDMI | + RT711_JD2 | + SOF_SDW_FOUR_SPK), + }, { .callback = sof_sdw_quirk_cb, .matches = { diff --git a/sound/soc/intel/boards/sof_ssp_amp.c b/sound/soc/intel/boards/sof_ssp_amp.c index 4a762e002ac7527be32d5298037191489b392ad8..94d25aeb6e7cea1603ec55be7be24903b81e5d8d 100644 --- a/sound/soc/intel/boards/sof_ssp_amp.c +++ b/sound/soc/intel/boards/sof_ssp_amp.c @@ -210,10 +210,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, struct snd_soc_dai_link *links; int i, id = 0; - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * - sof_ssp_amp_card.num_links, GFP_KERNEL); - cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) * - sof_ssp_amp_card.num_links, GFP_KERNEL); + links = devm_kcalloc(dev, sof_ssp_amp_card.num_links, + sizeof(struct snd_soc_dai_link), GFP_KERNEL); + cpus = devm_kcalloc(dev, sof_ssp_amp_card.num_links, + sizeof(struct snd_soc_dai_link_component), GFP_KERNEL); if (!links || !cpus) return NULL; @@ -306,9 +306,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev, if (sof_ssp_amp_quirk & SOF_HDMI_PLAYBACK_PRESENT) { /* HDMI */ if (hdmi_num > 0) { - idisp_components = devm_kzalloc(dev, - sizeof(struct snd_soc_dai_link_component) * - hdmi_num, GFP_KERNEL); + idisp_components = devm_kcalloc(dev, + hdmi_num, + sizeof(struct snd_soc_dai_link_component), + GFP_KERNEL); if (!idisp_components) goto devm_err; } diff --git a/sound/soc/intel/catpt/device.c b/sound/soc/intel/catpt/device.c index d48a71d2cf1ecaeb043d8224794c631ebec01ee3..d5d08bd766c702b66d69a40f8cda0cf308bb180c 100644 --- a/sound/soc/intel/catpt/device.c +++ b/sound/soc/intel/catpt/device.c @@ -22,7 +22,6 @@ #include <sound/intel-dsp-config.h> #include <sound/soc.h> #include <sound/soc-acpi.h> -#include <sound/soc-acpi-intel-match.h> #include "core.h" #include "registers.h" @@ -310,8 +309,36 @@ static int catpt_acpi_remove(struct platform_device *pdev) return 0; } +static struct snd_soc_acpi_mach lpt_machines[] = { + { + .id = "INT33CA", + .drv_name = "hsw_rt5640", + }, + {} +}; + +static struct snd_soc_acpi_mach wpt_machines[] = { + { + .id = "INT33CA", + .drv_name = "hsw_rt5640", + }, + { + .id = "INT343A", + .drv_name = "bdw_rt286", + }, + { + .id = "10EC5650", + .drv_name = "bdw-rt5650", + }, + { + .id = "RT5677CE", + .drv_name = "bdw-rt5677", + }, + {} +}; + static struct catpt_spec lpt_desc = { - .machines = snd_soc_acpi_intel_haswell_machines, + .machines = lpt_machines, .core_id = 0x01, .host_dram_offset = 0x000000, .host_iram_offset = 0x080000, @@ -326,7 +353,7 @@ static struct catpt_spec lpt_desc = { }; static struct catpt_spec wpt_desc = { - .machines = snd_soc_acpi_intel_broadwell_machines, + .machines = wpt_machines, .core_id = 0x02, .host_dram_offset = 0x000000, .host_iram_offset = 0x0A0000, diff --git a/sound/soc/intel/catpt/sysfs.c b/sound/soc/intel/catpt/sysfs.c index 1bdbcc04dc71285199d0a06c843cafcbe25d306f..9b6d2d93a2e7e507041ba7d02b5941227ae4a432 100644 --- a/sound/soc/intel/catpt/sysfs.c +++ b/sound/soc/intel/catpt/sysfs.c @@ -27,8 +27,8 @@ static ssize_t fw_version_show(struct device *dev, if (ret) return CATPT_IPC_ERROR(ret); - return sprintf(buf, "%d.%d.%d.%d\n", version.type, version.major, - version.minor, version.build); + return sysfs_emit(buf, "%d.%d.%d.%d\n", version.type, version.major, + version.minor, version.build); } static DEVICE_ATTR_RO(fw_version); @@ -37,7 +37,7 @@ static ssize_t fw_info_show(struct device *dev, { struct catpt_dev *cdev = dev_get_drvdata(dev); - return sprintf(buf, "%s\n", cdev->ipc.config.fw_info); + return sysfs_emit(buf, "%s\n", cdev->ipc.config.fw_info); } static DEVICE_ATTR_RO(fw_info); diff --git a/sound/soc/intel/common/Makefile b/sound/soc/intel/common/Makefile index 8ca8f872ec80ccb050a3e7e119553baae5395eed..41054cf09ec9dfceb4879b8d90e79ca44f0f4986 100644 --- a/sound/soc/intel/common/Makefile +++ b/sound/soc/intel/common/Makefile @@ -9,7 +9,7 @@ snd-soc-acpi-intel-match-objs := soc-acpi-intel-byt-match.o soc-acpi-intel-cht-m soc-acpi-intel-cml-match.o soc-acpi-intel-icl-match.o \ soc-acpi-intel-tgl-match.o soc-acpi-intel-ehl-match.o \ soc-acpi-intel-jsl-match.o soc-acpi-intel-adl-match.o \ - soc-acpi-intel-mtl-match.o \ + soc-acpi-intel-rpl-match.o soc-acpi-intel-mtl-match.o \ soc-acpi-intel-hda-match.o \ soc-acpi-intel-sdw-mockup-match.o diff --git a/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c b/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c index cbcb649604e57975f56cf8a4610fb7eaa94addcc..6daf60b1edf1b33d61515dbd9b4a6f82c96f4d09 100644 --- a/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c @@ -9,40 +9,25 @@ #include <sound/soc-acpi.h> #include <sound/soc-acpi-intel-match.h> -struct snd_soc_acpi_mach snd_soc_acpi_intel_haswell_machines[] = { - { - .id = "INT33CA", - .drv_name = "hsw_rt5640", - .fw_filename = "intel/IntcSST1.bin", - .sof_tplg_filename = "sof-hsw.tplg", - }, - {} -}; -EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_haswell_machines); - struct snd_soc_acpi_mach snd_soc_acpi_intel_broadwell_machines[] = { { .id = "INT343A", .drv_name = "bdw_rt286", - .fw_filename = "intel/IntcSST2.bin", .sof_tplg_filename = "sof-bdw-rt286.tplg", }, { .id = "10EC5650", .drv_name = "bdw-rt5650", - .fw_filename = "intel/IntcSST2.bin", .sof_tplg_filename = "sof-bdw-rt5650.tplg", }, { .id = "RT5677CE", .drv_name = "bdw-rt5677", - .fw_filename = "intel/IntcSST2.bin", .sof_tplg_filename = "sof-bdw-rt5677.tplg", }, { .id = "INT33CA", .drv_name = "hsw_rt5640", - .fw_filename = "intel/IntcSST2.bin", .sof_tplg_filename = "sof-bdw-rt5640.tplg", }, {} diff --git a/sound/soc/intel/common/soc-acpi-intel-rpl-match.c b/sound/soc/intel/common/soc-acpi-intel-rpl-match.c new file mode 100644 index 0000000000000000000000000000000000000000..9ccf7370157b404908297a043c81eaaab845b5d2 --- /dev/null +++ b/sound/soc/intel/common/soc-acpi-intel-rpl-match.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * soc-apci-intel-rpl-match.c - tables and support for RPL ACPI enumeration. + * + * Copyright (c) 2022 Intel Corporation. + */ + +#include <sound/soc-acpi.h> +#include <sound/soc-acpi-intel-match.h> + +static const struct snd_soc_acpi_endpoint single_endpoint = { + .num = 0, + .aggregated = 0, + .group_position = 0, + .group_id = 0, +}; + +static const struct snd_soc_acpi_endpoint spk_l_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 0, + .group_id = 1, +}; + +static const struct snd_soc_acpi_endpoint spk_r_endpoint = { + .num = 0, + .aggregated = 1, + .group_position = 1, + .group_id = 1, +}; + +static const struct snd_soc_acpi_adr_device rt711_0_adr[] = { + { + .adr = 0x000020025D071100ull, + .num_endpoints = 1, + .endpoints = &single_endpoint, + .name_prefix = "rt711" + } +}; + +static const struct snd_soc_acpi_link_adr rpl_rvp[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(rt711_0_adr), + .adr_d = rt711_0_adr, + }, + {} +}; + +static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = { + { + .adr = 0x000030025D071101ull, + .num_endpoints = 1, + .endpoints = &single_endpoint, + .name_prefix = "rt711" + } +}; + +static const struct snd_soc_acpi_adr_device rt1316_1_group1_adr[] = { + { + .adr = 0x000131025D131601ull, /* unique ID is set for some reason */ + .num_endpoints = 1, + .endpoints = &spk_l_endpoint, + .name_prefix = "rt1316-1" + } +}; + +static const struct snd_soc_acpi_adr_device rt1316_3_group1_adr[] = { + { + .adr = 0x000330025D131601ull, + .num_endpoints = 1, + .endpoints = &spk_r_endpoint, + .name_prefix = "rt1316-2" + } +}; + +static const struct snd_soc_acpi_adr_device rt714_2_adr[] = { + { + .adr = 0x000230025D071401ull, + .num_endpoints = 1, + .endpoints = &single_endpoint, + .name_prefix = "rt714" + } +}; + +static const struct snd_soc_acpi_link_adr rpl_sdca_3_in_1[] = { + { + .mask = BIT(0), + .num_adr = ARRAY_SIZE(rt711_sdca_0_adr), + .adr_d = rt711_sdca_0_adr, + }, + { + .mask = BIT(1), + .num_adr = ARRAY_SIZE(rt1316_1_group1_adr), + .adr_d = rt1316_1_group1_adr, + }, + { + .mask = BIT(2), + .num_adr = ARRAY_SIZE(rt714_2_adr), + .adr_d = rt714_2_adr, + }, + { + .mask = BIT(3), + .num_adr = ARRAY_SIZE(rt1316_3_group1_adr), + .adr_d = rt1316_3_group1_adr, + }, + {} +}; + +struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_machines[] = { + {}, +}; +EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_rpl_machines); + +/* this table is used when there is no I2S codec present */ +struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_sdw_machines[] = { + { + .link_mask = 0xF, /* 4 active links required */ + .links = rpl_sdca_3_in_1, + .drv_name = "sof_sdw", + .sof_tplg_filename = "sof-rpl-rt711-l0-rt1316-l13-rt714-l2.tplg", + }, + { + .link_mask = 0x1, /* link0 required */ + .links = rpl_rvp, + .drv_name = "sof_sdw", + .sof_tplg_filename = "sof-rpl-rt711.tplg", + }, + {}, +}; +EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_rpl_sdw_machines); diff --git a/sound/soc/intel/skylake/skl-nhlt.c b/sound/soc/intel/skylake/skl-nhlt.c index deb7b820325e7f9673072713b5213f3967a91467..e617b4c335a4335dbcfb8de2e120c63171032b3a 100644 --- a/sound/soc/intel/skylake/skl-nhlt.c +++ b/sound/soc/intel/skylake/skl-nhlt.c @@ -61,7 +61,7 @@ static ssize_t platform_id_show(struct device *dev, nhlt->header.oem_revision); skl_nhlt_trim_space(platform_id); - return sprintf(buf, "%s\n", platform_id); + return sysfs_emit(buf, "%s\n", platform_id); } static DEVICE_ATTR_RO(platform_id); diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c index 9d72ebd812af9958ba62e046ddd6fa831ead63d1..1015716f93361082f0ae9788754ed8cc57b26b24 100644 --- a/sound/soc/intel/skylake/skl-pcm.c +++ b/sound/soc/intel/skylake/skl-pcm.c @@ -275,7 +275,7 @@ static int skl_pcm_prepare(struct snd_pcm_substream *substream, * calls prepare another time, reset the FW pipe to clean state */ if (mconfig && - (substream->runtime->status->state == SNDRV_PCM_STATE_XRUN || + (substream->runtime->state == SNDRV_PCM_STATE_XRUN || mconfig->pipe->state == SKL_PIPE_CREATED || mconfig->pipe->state == SKL_PIPE_PAUSED)) { @@ -593,7 +593,7 @@ static int skl_link_pcm_prepare(struct snd_pcm_substream *substream, /* In case of XRUN recovery, reset the FW pipe to clean state */ mconfig = skl_tplg_be_get_cpr_module(dai, substream->stream); if (mconfig && !mconfig->pipe->passthru && - (substream->runtime->status->state == SNDRV_PCM_STATE_XRUN)) + (substream->runtime->state == SNDRV_PCM_STATE_XRUN)) skl_reset_pipe(skl, mconfig->pipe); return 0; diff --git a/sound/soc/intel/skylake/skl-topology.h b/sound/soc/intel/skylake/skl-topology.h index a5bccf2fcd888ae4a60a8133c3c43e9cecb057b9..017ac0ef324ddd65a20e4c3436485cdac1c57b99 100644 --- a/sound/soc/intel/skylake/skl-topology.h +++ b/sound/soc/intel/skylake/skl-topology.h @@ -233,8 +233,8 @@ struct skl_uuid_inst_map { struct skl_kpb_params { u32 num_modules; union { - struct skl_mod_inst_map map[0]; - struct skl_uuid_inst_map map_uuid[0]; + DECLARE_FLEX_ARRAY(struct skl_mod_inst_map, map); + DECLARE_FLEX_ARRAY(struct skl_uuid_inst_map, map_uuid); } u; }; diff --git a/sound/soc/intel/skylake/skl.c b/sound/soc/intel/skylake/skl.c index aeca58246fc77b03c4e3d21e539dc451da73c1d2..bbba2df33aaf0b198e829af1f41a8a3191bde448 100644 --- a/sound/soc/intel/skylake/skl.c +++ b/sound/soc/intel/skylake/skl.c @@ -444,7 +444,7 @@ static int skl_free(struct hdac_bus *bus) if (bus->irq >= 0) free_irq(bus->irq, (void *)bus); snd_hdac_bus_free_stream_pages(bus); - snd_hdac_stream_free_all(bus); + snd_hdac_ext_stream_free_all(bus); snd_hdac_link_free_all(bus); if (bus->remap_addr) @@ -689,6 +689,35 @@ static void load_codec_module(struct hda_codec *codec) #endif /* CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC */ +static void skl_codec_device_exit(struct device *dev) +{ + snd_hdac_device_exit(dev_to_hdac_dev(dev)); +} + +static struct hda_codec *skl_codec_device_init(struct hdac_bus *bus, int addr) +{ + struct hda_codec *codec; + int ret; + + codec = snd_hda_codec_device_init(to_hda_bus(bus), addr, "ehdaudio%dD%d", bus->idx, addr); + if (IS_ERR(codec)) { + dev_err(bus->dev, "device init failed for hdac device\n"); + return codec; + } + + codec->core.type = HDA_DEV_ASOC; + codec->core.dev.release = skl_codec_device_exit; + + ret = snd_hdac_device_register(&codec->core); + if (ret) { + dev_err(bus->dev, "failed to register hdac device\n"); + snd_hdac_device_exit(&codec->core); + return ERR_PTR(ret); + } + + return codec; +} + /* * Probe the given codec address */ @@ -697,12 +726,11 @@ static int probe_codec(struct hdac_bus *bus, int addr) unsigned int cmd = (addr << 28) | (AC_NODE_ROOT << 20) | (AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID; unsigned int res = -1; - struct skl_dev *skl = bus_to_skl(bus); #if IS_ENABLED(CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC) + struct skl_dev *skl = bus_to_skl(bus); struct hdac_hda_priv *hda_codec; - int err; #endif - struct hdac_device *hdev; + struct hda_codec *codec; mutex_lock(&bus->cmd_mutex); snd_hdac_bus_send_cmd(bus, cmd); @@ -718,25 +746,22 @@ static int probe_codec(struct hdac_bus *bus, int addr) if (!hda_codec) return -ENOMEM; - hda_codec->codec.bus = skl_to_hbus(skl); - hdev = &hda_codec->codec.core; + codec = skl_codec_device_init(bus, addr); + if (IS_ERR(codec)) + return PTR_ERR(codec); - err = snd_hdac_ext_bus_device_init(bus, addr, hdev, HDA_DEV_ASOC); - if (err < 0) - return err; + hda_codec->codec = codec; + dev_set_drvdata(&codec->core.dev, hda_codec); /* use legacy bus only for HDA codecs, idisp uses ext bus */ if ((res & 0xFFFF0000) != IDISP_INTEL_VENDOR_ID) { - hdev->type = HDA_DEV_LEGACY; - load_codec_module(&hda_codec->codec); + codec->core.type = HDA_DEV_LEGACY; + load_codec_module(hda_codec->codec); } return 0; #else - hdev = devm_kzalloc(&skl->pci->dev, sizeof(*hdev), GFP_KERNEL); - if (!hdev) - return -ENOMEM; - - return snd_hdac_ext_bus_device_init(bus, addr, hdev, HDA_DEV_ASOC); + codec = skl_codec_device_init(bus, addr); + return PTR_ERR_OR_ZERO(codec); #endif /* CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC */ } @@ -1127,7 +1152,6 @@ static void skl_remove(struct pci_dev *pci) if (skl->nhlt) intel_nhlt_free(skl->nhlt); skl_free(bus); - dev_set_drvdata(&pci->dev, NULL); } /* PCI IDs */ diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-common.h b/sound/soc/mediatek/mt8183/mt8183-afe-common.h index b220e7a7db7e09be6ee11208f27d08dfea26de28..40ab48c1566c8cac33af206f9caae784855fab27 100644 --- a/sound/soc/mediatek/mt8183/mt8183-afe-common.h +++ b/sound/soc/mediatek/mt8183/mt8183-afe-common.h @@ -99,6 +99,9 @@ unsigned int mt8183_general_rate_transform(struct device *dev, unsigned int mt8183_rate_transform(struct device *dev, unsigned int rate, int aud_blk); +int mt8183_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name); + /* dai register */ int mt8183_dai_adda_register(struct mtk_base_afe *afe); int mt8183_dai_pcm_register(struct mtk_base_afe *afe); diff --git a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c index b33cc9a73ed1cc284e1dce3c1d0ed16a2f6a26c5..9f22d39398189235ddf5544a88e839a3dad1ebac 100644 --- a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c +++ b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c @@ -17,6 +17,7 @@ #include "../../codecs/da7219-aad.h" #include "../../codecs/da7219.h" #include "../../codecs/rt1015.h" +#include "../common/mtk-afe-platform-driver.h" #include "mt8183-afe-common.h" #define DA7219_CODEC_DAI "da7219-hifi" @@ -372,6 +373,36 @@ static int mt8183_da7219_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd) &priv->hdmi_jack, NULL); } +static int mt8183_bt_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + int ret; + + ret = mt8183_dai_i2s_set_share(afe, "I2S5", "I2S0"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + return 0; +} + +static int mt8183_da7219_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + int ret; + + ret = mt8183_dai_i2s_set_share(afe, "I2S2", "I2S3"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + return 0; +} + static struct snd_soc_dai_link mt8183_da7219_dai_links[] = { /* FE */ { @@ -500,6 +531,7 @@ static struct snd_soc_dai_link mt8183_da7219_dai_links[] = { .ignore_suspend = 1, .be_hw_params_fixup = mt8183_i2s_hw_params_fixup, .ops = &mt8183_da7219_i2s_ops, + .init = &mt8183_da7219_init, SND_SOC_DAILINK_REG(i2s2), }, { @@ -515,6 +547,7 @@ static struct snd_soc_dai_link mt8183_da7219_dai_links[] = { .ignore_suspend = 1, .be_hw_params_fixup = mt8183_i2s_hw_params_fixup, .ops = &mt8183_mt6358_i2s_ops, + .init = &mt8183_bt_init, SND_SOC_DAILINK_REG(i2s5), }, { diff --git a/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c b/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c index 138591d71ebda11ddbcfc9d09fcc31beaccbadd0..6a9ace4180d3475bc6308a5a4df129afe7945308 100644 --- a/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c +++ b/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c @@ -43,7 +43,6 @@ struct mtk_afe_i2s_priv { int rate; /* for determine which apll to use */ int low_jitter_en; - const char *share_property_name; int share_i2s_id; int mclk_id; @@ -977,54 +976,55 @@ static const struct mtk_afe_i2s_priv mt8183_i2s_priv[DAI_I2S_NUM] = { [DAI_I2S0] = { .id = MT8183_DAI_I2S_0, .mclk_id = MT8183_I2S0_MCK, - .share_property_name = "i2s0-share", .share_i2s_id = -1, }, [DAI_I2S1] = { .id = MT8183_DAI_I2S_1, .mclk_id = MT8183_I2S1_MCK, - .share_property_name = "i2s1-share", .share_i2s_id = -1, }, [DAI_I2S2] = { .id = MT8183_DAI_I2S_2, .mclk_id = MT8183_I2S2_MCK, - .share_property_name = "i2s2-share", .share_i2s_id = -1, }, [DAI_I2S3] = { .id = MT8183_DAI_I2S_3, .mclk_id = MT8183_I2S3_MCK, - .share_property_name = "i2s3-share", .share_i2s_id = -1, }, [DAI_I2S5] = { .id = MT8183_DAI_I2S_5, .mclk_id = MT8183_I2S5_MCK, - .share_property_name = "i2s5-share", .share_i2s_id = -1, }, }; -static int mt8183_dai_i2s_get_share(struct mtk_base_afe *afe) +/** + * mt8183_dai_i2s_set_share() - Set up I2S ports to share a single clock. + * @afe: Pointer to &struct mtk_base_afe + * @main_i2s_name: The name of the I2S port that will provide the clock + * @secondary_i2s_name: The name of the I2S port that will use this clock + */ +int mt8183_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name) { - struct mt8183_afe_private *afe_priv = afe->platform_priv; - const struct device_node *of_node = afe->dev->of_node; - const char *of_str; - const char *property_name; - struct mtk_afe_i2s_priv *i2s_priv; - int i; + struct mtk_afe_i2s_priv *secondary_i2s_priv; + int main_i2s_id; - for (i = 0; i < DAI_I2S_NUM; i++) { - i2s_priv = afe_priv->dai_priv[mt8183_i2s_priv[i].id]; - property_name = mt8183_i2s_priv[i].share_property_name; - if (of_property_read_string(of_node, property_name, &of_str)) - continue; - i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str); - } + secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name); + if (!secondary_i2s_priv) + return -EINVAL; + + main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name); + if (main_i2s_id < 0) + return main_i2s_id; + + secondary_i2s_priv->share_i2s_id = main_i2s_id; return 0; } +EXPORT_SYMBOL_GPL(mt8183_dai_i2s_set_share); static int mt8183_dai_i2s_set_priv(struct mtk_base_afe *afe) { @@ -1074,10 +1074,5 @@ int mt8183_dai_i2s_register(struct mtk_base_afe *afe) if (ret) return ret; - /* parse share i2s */ - ret = mt8183_dai_i2s_get_share(afe); - if (ret) - return ret; - return 0; } diff --git a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c index ab157db7833501d8a90baa6ceb634d67c1351e3a..a8608522367792a27db1b3db755179428cb9e09c 100644 --- a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c +++ b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c @@ -15,6 +15,7 @@ #include "../../codecs/rt1015.h" #include "../../codecs/ts3a227e.h" +#include "../common/mtk-afe-platform-driver.h" #include "mt8183-afe-common.h" #define RT1015_CODEC_DAI "rt1015-aif" @@ -391,6 +392,36 @@ mt8183_mt6358_ts3a227_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd) &priv->hdmi_jack, NULL); } +static int mt8183_bt_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + int ret; + + ret = mt8183_dai_i2s_set_share(afe, "I2S5", "I2S0"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + return 0; +} + +static int mt8183_i2s2_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + int ret; + + ret = mt8183_dai_i2s_set_share(afe, "I2S2", "I2S3"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + return 0; +} + static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = { /* FE */ { @@ -527,6 +558,7 @@ static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = { .ignore_suspend = 1, .be_hw_params_fixup = mt8183_i2s_hw_params_fixup, .ops = &mt8183_mt6358_i2s_ops, + .init = &mt8183_i2s2_init, SND_SOC_DAILINK_REG(i2s2), }, { @@ -541,6 +573,7 @@ static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = { .dpcm_playback = 1, .ignore_suspend = 1, .ops = &mt8183_mt6358_i2s_ops, + .init = &mt8183_bt_init, SND_SOC_DAILINK_REG(i2s5), }, { diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-common.h b/sound/soc/mediatek/mt8186/mt8186-afe-common.h index b8f03e1b7e49da3f734593c2f6eec274f7fa9481..d592585209955a2dc35108680a5c4c5869dd6ee0 100644 --- a/sound/soc/mediatek/mt8186/mt8186-afe-common.h +++ b/sound/soc/mediatek/mt8186/mt8186-afe-common.h @@ -189,6 +189,9 @@ unsigned int mt8186_rate_transform(struct device *dev, unsigned int mt8186_tdm_relatch_rate_transform(struct device *dev, unsigned int rate); +int mt8186_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name); + int mt8186_dai_set_priv(struct mtk_base_afe *afe, int id, int priv_size, const void *priv_data); diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c b/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c index 274c0c8ec2f29f621652845e6310ffe750b8a8fd..eda913fa147af9906ba420c0bd84d7ed2cef382b 100644 --- a/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c +++ b/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c @@ -170,25 +170,25 @@ static int mt8186_afe_gpio_adda_ul(struct device *dev, bool enable) if (enable) { ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_CLK_MISO_ON); if (ret) { - dev_err(dev, "%s(), MISO CLK ON slect fail!\n", __func__); + dev_err(dev, "%s(), MISO CLK ON select fail!\n", __func__); return ret; } ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_DAT_MISO_ON); if (ret) { - dev_err(dev, "%s(), MISO DAT ON slect fail!\n", __func__); + dev_err(dev, "%s(), MISO DAT ON select fail!\n", __func__); return ret; } } else { ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_DAT_MISO_OFF); if (ret) { - dev_err(dev, "%s(), MISO DAT OFF slect fail!\n", __func__); + dev_err(dev, "%s(), MISO DAT OFF select fail!\n", __func__); return ret; } ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_CLK_MISO_OFF); if (ret) { - dev_err(dev, "%s(), MISO CLK OFF slect fail!\n", __func__); + dev_err(dev, "%s(), MISO CLK OFF select fail!\n", __func__); return ret; } } diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c index eb729ab00f5aa79cf08dd3601f80d86472ca450b..d7e94e6a19c704f031b18ee0a8a1190faa298f60 100644 --- a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c +++ b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c @@ -1359,6 +1359,9 @@ static const struct snd_soc_dapm_widget mt8186_memif_widgets[] = { SND_SOC_DAPM_MUX("UL5_IN_MUX", SND_SOC_NOPM, 0, 0, &ul5_in_mux_control), + SND_SOC_DAPM_MIXER("DSP_DL1_VIRT", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER("DSP_DL2_VIRT", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_INPUT("UL1_VIRTUAL_INPUT"), SND_SOC_DAPM_INPUT("UL2_VIRTUAL_INPUT"), SND_SOC_DAPM_INPUT("UL3_VIRTUAL_INPUT"), diff --git a/sound/soc/mediatek/mt8186/mt8186-dai-adda.c b/sound/soc/mediatek/mt8186/mt8186-dai-adda.c index 266704556f37d923ecb827ba7e1963a64abfa0f9..094402470dc2383e8045a1991223f6c749a23156 100644 --- a/sound/soc/mediatek/mt8186/mt8186-dai-adda.c +++ b/sound/soc/mediatek/mt8186/mt8186-dai-adda.c @@ -271,9 +271,6 @@ static int mtk_adda_ul_event(struct snd_soc_dapm_widget *w, /* should delayed 1/fs(smallest is 8k) = 125us before afe off */ usleep_range(125, 135); mt8186_afe_gpio_request(afe->dev, false, MT8186_DAI_ADDA, 1); - - /* reset dmic */ - afe_priv->mtkaif_dmic = 0; break; default: break; diff --git a/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c b/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c index ec79e2f2a54d4fdbe2da117c06da64c8cffab477..f07181be4370d948b13c47edd9413003885a9727 100644 --- a/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c +++ b/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c @@ -44,7 +44,6 @@ struct mtk_afe_i2s_priv { int low_jitter_en; int master; /* only i2s0 has slave mode*/ - const char *share_property_name; int share_i2s_id; int mclk_id; @@ -658,9 +657,15 @@ static const struct snd_soc_dapm_route mtk_dai_i2s_routes[] = { {"I2S1_CH1", "DL1_CH1 Switch", "DL1"}, {"I2S1_CH2", "DL1_CH2 Switch", "DL1"}, + {"I2S1_CH1", "DL1_CH1 Switch", "DSP_DL1_VIRT"}, + {"I2S1_CH2", "DL1_CH2 Switch", "DSP_DL1_VIRT"}, + {"I2S1_CH1", "DL2_CH1 Switch", "DL2"}, {"I2S1_CH2", "DL2_CH2 Switch", "DL2"}, + {"I2S1_CH1", "DL2_CH1 Switch", "DSP_DL2_VIRT"}, + {"I2S1_CH2", "DL2_CH2 Switch", "DSP_DL2_VIRT"}, + {"I2S1_CH1", "DL3_CH1 Switch", "DL3"}, {"I2S1_CH2", "DL3_CH2 Switch", "DL3"}, @@ -728,9 +733,15 @@ static const struct snd_soc_dapm_route mtk_dai_i2s_routes[] = { {"I2S3_CH1", "DL1_CH1 Switch", "DL1"}, {"I2S3_CH2", "DL1_CH2 Switch", "DL1"}, + {"I2S3_CH1", "DL1_CH1 Switch", "DSP_DL1_VIRT"}, + {"I2S3_CH2", "DL1_CH2 Switch", "DSP_DL1_VIRT"}, + {"I2S3_CH1", "DL2_CH1 Switch", "DL2"}, {"I2S3_CH2", "DL2_CH2 Switch", "DL2"}, + {"I2S3_CH1", "DL2_CH1 Switch", "DSP_DL2_VIRT"}, + {"I2S3_CH2", "DL2_CH2 Switch", "DSP_DL2_VIRT"}, + {"I2S3_CH1", "DL3_CH1 Switch", "DL3"}, {"I2S3_CH2", "DL3_CH2 Switch", "DL3"}, @@ -968,7 +979,7 @@ static int mtk_dai_i2s_config(struct mtk_base_afe *afe, } /* set share i2s */ - if (i2s_priv && i2s_priv->share_i2s_id >= 0) { + if (i2s_priv->share_i2s_id >= 0) { ret = mtk_dai_i2s_config(afe, params, i2s_priv->share_i2s_id); if (ret) return ret; @@ -1128,49 +1139,51 @@ static const struct mtk_afe_i2s_priv mt8186_i2s_priv[DAI_I2S_NUM] = { [DAI_I2S0] = { .id = MT8186_DAI_I2S_0, .mclk_id = MT8186_I2S0_MCK, - .share_property_name = "i2s0-share", .share_i2s_id = -1, }, [DAI_I2S1] = { .id = MT8186_DAI_I2S_1, .mclk_id = MT8186_I2S1_MCK, - .share_property_name = "i2s1-share", .share_i2s_id = -1, }, [DAI_I2S2] = { .id = MT8186_DAI_I2S_2, .mclk_id = MT8186_I2S2_MCK, - .share_property_name = "i2s2-share", .share_i2s_id = -1, }, [DAI_I2S3] = { .id = MT8186_DAI_I2S_3, /* clock gate naming is hf_faud_i2s4_m_ck*/ .mclk_id = MT8186_I2S4_MCK, - .share_property_name = "i2s3-share", .share_i2s_id = -1, } }; -static int mt8186_dai_i2s_get_share(struct mtk_base_afe *afe) +/** + * mt8186_dai_i2s_set_share() - Set up I2S ports to share a single clock. + * @afe: Pointer to &struct mtk_base_afe + * @main_i2s_name: The name of the I2S port that will provide the clock + * @secondary_i2s_name: The name of the I2S port that will use this clock + */ +int mt8186_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name) { - struct mt8186_afe_private *afe_priv = afe->platform_priv; - const struct device_node *of_node = afe->dev->of_node; - const char *of_str; - const char *property_name; - struct mtk_afe_i2s_priv *i2s_priv; - int i; + struct mtk_afe_i2s_priv *secondary_i2s_priv; + int main_i2s_id; - for (i = 0; i < DAI_I2S_NUM; i++) { - i2s_priv = afe_priv->dai_priv[mt8186_i2s_priv[i].id]; - property_name = mt8186_i2s_priv[i].share_property_name; - if (of_property_read_string(of_node, property_name, &of_str)) - continue; - i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str); - } + secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name); + if (!secondary_i2s_priv) + return -EINVAL; + + main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name); + if (main_i2s_id < 0) + return main_i2s_id; + + secondary_i2s_priv->share_i2s_id = main_i2s_id; return 0; } +EXPORT_SYMBOL_GPL(mt8186_dai_i2s_set_share); static int mt8186_dai_i2s_set_priv(struct mtk_base_afe *afe) { @@ -1214,10 +1227,5 @@ int mt8186_dai_i2s_register(struct mtk_base_afe *afe) if (ret) return ret; - /* parse share i2s */ - ret = mt8186_dai_i2s_get_share(afe); - if (ret) - return ret; - return 0; } diff --git a/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c b/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c index 387f25cad80954f166b0e8185dc2fd9a0919efbe..cfca6bdee834546d27a5220676f577b328f3e3a7 100644 --- a/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c +++ b/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c @@ -18,6 +18,8 @@ #include "../../codecs/da7219.h" #include "../../codecs/mt6358.h" #include "../common/mtk-afe-platform-driver.h" +#include "../common/mtk-dsp-sof-common.h" +#include "../common/mtk-soc-card.h" #include "mt8186-afe-common.h" #include "mt8186-afe-clk.h" #include "mt8186-afe-gpio.h" @@ -26,10 +28,27 @@ #define DA7219_CODEC_DAI "da7219-hifi" #define DA7219_DEV_NAME "da7219.5-001a" +#define SOF_DMA_DL1 "SOF_DMA_DL1" +#define SOF_DMA_DL2 "SOF_DMA_DL2" +#define SOF_DMA_UL1 "SOF_DMA_UL1" +#define SOF_DMA_UL2 "SOF_DMA_UL2" + struct mt8186_mt6366_da7219_max98357_priv { struct snd_soc_jack headset_jack, hdmi_jack; }; +/* Headset jack detection DAPM pins */ +static struct snd_soc_jack_pin mt8186_jack_pins[] = { + { + .pin = "Headphones", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + static struct snd_soc_codec_conf mt8186_mt6366_da7219_max98357_codec_conf[] = { { .dlc = COMP_CODEC_CONF("mt6358-sound"), @@ -47,19 +66,30 @@ static struct snd_soc_codec_conf mt8186_mt6366_da7219_max98357_codec_conf[] = { static int mt8186_da7219_init(struct snd_soc_pcm_runtime *rtd) { - struct mt8186_mt6366_da7219_max98357_priv *priv = + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8186_mt6366_da7219_max98357_priv *priv = soc_card_data->mach_priv; struct snd_soc_jack *jack = &priv->headset_jack; struct snd_soc_component *cmpnt_codec = asoc_rtd_to_codec(rtd, 0)->component; int ret; + ret = mt8186_dai_i2s_set_share(afe, "I2S1", "I2S0"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + /* Enable Headset and 4 Buttons Jack detection */ - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3 | SND_JACK_LINEOUT, - jack); + jack, mt8186_jack_pins, + ARRAY_SIZE(mt8186_jack_pins)); if (ret) { dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret); return ret; @@ -152,12 +182,22 @@ static const struct snd_soc_ops mt8186_da7219_i2s_ops = { static int mt8186_mt6366_da7219_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd) { + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); struct snd_soc_component *cmpnt_codec = asoc_rtd_to_codec(rtd, 0)->component; - struct mt8186_mt6366_da7219_max98357_priv *priv = + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8186_mt6366_da7219_max98357_priv *priv = soc_card_data->mach_priv; int ret; + ret = mt8186_dai_i2s_set_share(afe, "I2S3", "I2S2"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + ret = snd_soc_card_jack_new(rtd->card, "HDMI Jack", SND_JACK_LINEOUT, &priv->hdmi_jack); if (ret) { dev_err(rtd->dev, "HDMI Jack creation failed: %d\n", ret); @@ -201,6 +241,24 @@ static int mt8186_anx7625_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, return mt8186_hw_params_fixup(rtd, params, SNDRV_PCM_FORMAT_S24_LE); } +/* fixup the BE DAI link to match any values from topology */ +static int mt8186_sof_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, + struct snd_pcm_hw_params *params) +{ + int ret; + + ret = mtk_sof_dai_link_fixup(rtd, params); + + if (!strcmp(rtd->dai_link->name, "I2S0") || + !strcmp(rtd->dai_link->name, "I2S1") || + !strcmp(rtd->dai_link->name, "I2S2")) + mt8186_i2s_hw_params_fixup(rtd, params); + else if (!strcmp(rtd->dai_link->name, "I2S3")) + mt8186_anx7625_i2s_hw_params_fixup(rtd, params); + + return ret; +} + static int mt8186_mt6366_da7219_max98357_playback_startup(struct snd_pcm_substream *substream) { static const unsigned int rates[] = { @@ -474,6 +532,33 @@ SND_SOC_DAILINK_DEFS(hostless_src_aaudio, DAILINK_COMP_ARRAY(COMP_CPU("Hostless SRC AAudio DAI")), DAILINK_COMP_ARRAY(COMP_DUMMY()), DAILINK_COMP_ARRAY(COMP_EMPTY())); +SND_SOC_DAILINK_DEFS(AFE_SOF_DL1, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL1")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_DL2, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL2")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_UL1, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL1")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_UL2, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL2")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static const struct sof_conn_stream g_sof_conn_streams[] = { + { "I2S1", "AFE_SOF_DL1", SOF_DMA_DL1, SNDRV_PCM_STREAM_PLAYBACK}, + { "I2S3", "AFE_SOF_DL2", SOF_DMA_DL2, SNDRV_PCM_STREAM_PLAYBACK}, + { "Primary Codec", "AFE_SOF_UL1", SOF_DMA_UL1, SNDRV_PCM_STREAM_CAPTURE}, + { "I2S0", "AFE_SOF_UL2", SOF_DMA_UL2, SNDRV_PCM_STREAM_CAPTURE}, +}; + static struct snd_soc_dai_link mt8186_mt6366_da7219_max98357_dai_links[] = { /* Front End DAI links */ { @@ -848,30 +933,75 @@ static struct snd_soc_dai_link mt8186_mt6366_da7219_max98357_dai_links[] = { .ignore_suspend = 1, SND_SOC_DAILINK_REG(hostless_ul6), }, + /* SOF BE */ + { + .name = "AFE_SOF_DL1", + .no_pcm = 1, + .dpcm_playback = 1, + SND_SOC_DAILINK_REG(AFE_SOF_DL1), + }, + { + .name = "AFE_SOF_DL2", + .no_pcm = 1, + .dpcm_playback = 1, + SND_SOC_DAILINK_REG(AFE_SOF_DL2), + }, + { + .name = "AFE_SOF_UL1", + .no_pcm = 1, + .dpcm_capture = 1, + SND_SOC_DAILINK_REG(AFE_SOF_UL1), + }, + { + .name = "AFE_SOF_UL2", + .no_pcm = 1, + .dpcm_capture = 1, + SND_SOC_DAILINK_REG(AFE_SOF_UL2), + }, }; static const struct snd_soc_dapm_widget mt8186_mt6366_da7219_max98357_widgets[] = { SND_SOC_DAPM_SPK("Speakers", NULL), + SND_SOC_DAPM_HP("Headphones", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), SND_SOC_DAPM_OUTPUT("HDMI1"), + SND_SOC_DAPM_MIXER(SOF_DMA_DL1, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_DL2, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_UL1, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_UL2, SND_SOC_NOPM, 0, 0, NULL, 0), }; static const struct snd_soc_dapm_route mt8186_mt6366_da7219_max98357_routes[] = { /* SPK */ { "Speakers", NULL, "Speaker"}, + /* Headset */ + { "Headphones", NULL, "HPL" }, + { "Headphones", NULL, "HPR" }, + { "MIC", NULL, "Headset Mic" }, /* HDMI */ { "HDMI1", NULL, "TX"}, + /* SOF Uplink */ + {SOF_DMA_UL1, NULL, "UL1_CH1"}, + {SOF_DMA_UL1, NULL, "UL1_CH2"}, + {SOF_DMA_UL2, NULL, "UL2_CH1"}, + {SOF_DMA_UL2, NULL, "UL2_CH2"}, + /* SOF Downlink */ + {"DSP_DL1_VIRT", NULL, SOF_DMA_DL1}, + {"DSP_DL2_VIRT", NULL, SOF_DMA_DL2}, }; static const struct snd_kcontrol_new mt8186_mt6366_da7219_max98357_controls[] = { SOC_DAPM_PIN_SWITCH("Speakers"), + SOC_DAPM_PIN_SWITCH("Headphones"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), SOC_DAPM_PIN_SWITCH("HDMI1"), }; static struct snd_soc_card mt8186_mt6366_da7219_max98357_soc_card = { - .name = "mt8186_mt6366_da7219_max98357", + .name = "mt8186_da7219_max98357", .owner = THIS_MODULE, .dai_link = mt8186_mt6366_da7219_max98357_dai_links, .num_links = ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links), @@ -889,8 +1019,10 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev) { struct snd_soc_card *card; struct snd_soc_dai_link *dai_link; - struct mt8186_mt6366_da7219_max98357_priv *priv; - struct device_node *platform_node, *headset_codec, *playback_codec; + struct mtk_soc_card_data *soc_card_data; + struct mt8186_mt6366_da7219_max98357_priv *mach_priv; + struct device_node *platform_node, *headset_codec, *playback_codec, *adsp_node; + int sof_on = 0; int ret, i; card = (struct snd_soc_card *)device_get_match_data(&pdev->dev); @@ -898,11 +1030,60 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev) return -EINVAL; card->dev = &pdev->dev; + soc_card_data = devm_kzalloc(&pdev->dev, sizeof(*soc_card_data), GFP_KERNEL); + if (!soc_card_data) + return -ENOMEM; + mach_priv = devm_kzalloc(&pdev->dev, sizeof(*mach_priv), GFP_KERNEL); + if (!mach_priv) + return -ENOMEM; + + soc_card_data->mach_priv = mach_priv; + + adsp_node = of_parse_phandle(pdev->dev.of_node, "mediatek,adsp", 0); + if (adsp_node) { + struct mtk_sof_priv *sof_priv; + + sof_priv = devm_kzalloc(&pdev->dev, sizeof(*sof_priv), GFP_KERNEL); + if (!sof_priv) { + ret = -ENOMEM; + goto err_adsp_node; + } + sof_priv->conn_streams = g_sof_conn_streams; + sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams); + sof_priv->sof_dai_link_fixup = mt8186_sof_dai_link_fixup; + soc_card_data->sof_priv = sof_priv; + card->probe = mtk_sof_card_probe; + card->late_probe = mtk_sof_card_late_probe; + if (!card->topology_shortname_created) { + snprintf(card->topology_shortname, 32, "sof-%s", card->name); + card->topology_shortname_created = true; + } + card->name = card->topology_shortname; + sof_on = 1; + } else { + dev_info(&pdev->dev, "Probe without adsp\n"); + } + + if (of_property_read_bool(pdev->dev.of_node, "mediatek,dai-link")) { + ret = mtk_sof_dailink_parse_of(card, pdev->dev.of_node, + "mediatek,dai-link", + mt8186_mt6366_da7219_max98357_dai_links, + ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links)); + if (ret) { + dev_dbg(&pdev->dev, "Parse dai-link fail\n"); + goto err_adsp_node; + } + } else { + if (!sof_on) + card->num_links = ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links) + - ARRAY_SIZE(g_sof_conn_streams); + } + platform_node = of_parse_phandle(pdev->dev.of_node, "mediatek,platform", 0); if (!platform_node) { ret = -EINVAL; dev_err_probe(&pdev->dev, ret, "Property 'platform' missing or invalid\n"); - return ret; + goto err_platform_node; } playback_codec = of_get_child_by_name(pdev->dev.of_node, "playback-codecs"); @@ -941,17 +1122,14 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev) goto err_probe; } - if (!dai_link->platforms->name) - dai_link->platforms->of_node = platform_node; - } + if (!strncmp(dai_link->name, "AFE_SOF", strlen("AFE_SOF")) && sof_on) + dai_link->platforms->of_node = adsp_node; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) { - ret = -ENOMEM; - goto err_probe; + if (!dai_link->platforms->name && !dai_link->platforms->of_node) + dai_link->platforms->of_node = platform_node; } - snd_soc_card_set_drvdata(card, priv); + snd_soc_card_set_drvdata(card, soc_card_data); ret = mt8186_afe_gpio_init(&pdev->dev); if (ret) { @@ -969,6 +1147,9 @@ err_headset_codec: of_node_put(playback_codec); err_playback_codec: of_node_put(platform_node); +err_platform_node: +err_adsp_node: + of_node_put(adsp_node); return ret; } diff --git a/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c b/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c index 891146fd6c2bb2201ee9e23a53588bb63fdf8ada..2414c5b77233c0ee347ffe34a2a38da21722bd02 100644 --- a/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c +++ b/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c @@ -19,6 +19,8 @@ #include "../../codecs/mt6358.h" #include "../../codecs/rt5682.h" #include "../common/mtk-afe-platform-driver.h" +#include "../common/mtk-dsp-sof-common.h" +#include "../common/mtk-soc-card.h" #include "mt8186-afe-common.h" #include "mt8186-afe-clk.h" #include "mt8186-afe-gpio.h" @@ -30,10 +32,27 @@ #define RT5682S_CODEC_DAI "rt5682s-aif1" #define RT5682S_DEV0_NAME "rt5682s.5-001a" +#define SOF_DMA_DL1 "SOF_DMA_DL1" +#define SOF_DMA_DL2 "SOF_DMA_DL2" +#define SOF_DMA_UL1 "SOF_DMA_UL1" +#define SOF_DMA_UL2 "SOF_DMA_UL2" + struct mt8186_mt6366_rt1019_rt5682s_priv { struct snd_soc_jack headset_jack, hdmi_jack; }; +/* Headset jack detection DAPM pins */ +static struct snd_soc_jack_pin mt8186_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + static struct snd_soc_codec_conf mt8186_mt6366_rt1019_rt5682s_codec_conf[] = { { .dlc = COMP_CODEC_CONF("mt6358-sound"), @@ -51,18 +70,29 @@ static struct snd_soc_codec_conf mt8186_mt6366_rt1019_rt5682s_codec_conf[] = { static int mt8186_rt5682s_init(struct snd_soc_pcm_runtime *rtd) { - struct mt8186_mt6366_rt1019_rt5682s_priv *priv = + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8186_mt6366_rt1019_rt5682s_priv *priv = soc_card_data->mach_priv; struct snd_soc_jack *jack = &priv->headset_jack; struct snd_soc_component *cmpnt_codec = asoc_rtd_to_codec(rtd, 0)->component; int ret; - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + ret = mt8186_dai_i2s_set_share(afe, "I2S1", "I2S0"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3, - jack); + jack, mt8186_jack_pins, + ARRAY_SIZE(mt8186_jack_pins)); if (ret) { dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret); return ret; @@ -128,12 +158,22 @@ static const struct snd_soc_ops mt8186_rt5682s_i2s_ops = { static int mt8186_mt6366_rt1019_rt5682s_hdmi_init(struct snd_soc_pcm_runtime *rtd) { + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); struct snd_soc_component *cmpnt_codec = asoc_rtd_to_codec(rtd, 0)->component; - struct mt8186_mt6366_rt1019_rt5682s_priv *priv = + struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card); + struct mt8186_mt6366_rt1019_rt5682s_priv *priv = soc_card_data->mach_priv; int ret; + ret = mt8186_dai_i2s_set_share(afe, "I2S3", "I2S2"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + ret = snd_soc_card_jack_new(rtd->card, "HDMI Jack", SND_JACK_LINEOUT, &priv->hdmi_jack); if (ret) { dev_err(rtd->dev, "HDMI Jack creation failed: %d\n", ret); @@ -177,6 +217,24 @@ static int mt8186_it6505_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, return mt8186_hw_params_fixup(rtd, params, SNDRV_PCM_FORMAT_S32_LE); } +/* fixup the BE DAI link to match any values from topology */ +static int mt8186_sof_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, + struct snd_pcm_hw_params *params) +{ + int ret; + + ret = mtk_sof_dai_link_fixup(rtd, params); + + if (!strcmp(rtd->dai_link->name, "I2S0") || + !strcmp(rtd->dai_link->name, "I2S1") || + !strcmp(rtd->dai_link->name, "I2S2")) + mt8186_i2s_hw_params_fixup(rtd, params); + else if (!strcmp(rtd->dai_link->name, "I2S3")) + mt8186_it6505_i2s_hw_params_fixup(rtd, params); + + return ret; +} + static int mt8186_mt6366_rt1019_rt5682s_playback_startup(struct snd_pcm_substream *substream) { static const unsigned int rates[] = { @@ -450,6 +508,33 @@ SND_SOC_DAILINK_DEFS(hostless_src_aaudio, DAILINK_COMP_ARRAY(COMP_CPU("Hostless SRC AAudio DAI")), DAILINK_COMP_ARRAY(COMP_DUMMY()), DAILINK_COMP_ARRAY(COMP_EMPTY())); +SND_SOC_DAILINK_DEFS(AFE_SOF_DL1, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL1")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_DL2, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL2")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_UL1, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL1")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(AFE_SOF_UL2, + DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL2")), + DAILINK_COMP_ARRAY(COMP_DUMMY()), + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static const struct sof_conn_stream g_sof_conn_streams[] = { + { "I2S1", "AFE_SOF_DL1", SOF_DMA_DL1, SNDRV_PCM_STREAM_PLAYBACK}, + { "I2S3", "AFE_SOF_DL2", SOF_DMA_DL2, SNDRV_PCM_STREAM_PLAYBACK}, + { "Primary Codec", "AFE_SOF_UL1", SOF_DMA_UL1, SNDRV_PCM_STREAM_CAPTURE}, + { "I2S0", "AFE_SOF_UL2", SOF_DMA_UL2, SNDRV_PCM_STREAM_CAPTURE}, +}; + static struct snd_soc_dai_link mt8186_mt6366_rt1019_rt5682s_dai_links[] = { /* Front End DAI links */ { @@ -824,30 +909,75 @@ static struct snd_soc_dai_link mt8186_mt6366_rt1019_rt5682s_dai_links[] = { .ignore_suspend = 1, SND_SOC_DAILINK_REG(hostless_ul6), }, + /* SOF BE */ + { + .name = "AFE_SOF_DL1", + .no_pcm = 1, + .dpcm_playback = 1, + SND_SOC_DAILINK_REG(AFE_SOF_DL1), + }, + { + .name = "AFE_SOF_DL2", + .no_pcm = 1, + .dpcm_playback = 1, + SND_SOC_DAILINK_REG(AFE_SOF_DL2), + }, + { + .name = "AFE_SOF_UL1", + .no_pcm = 1, + .dpcm_capture = 1, + SND_SOC_DAILINK_REG(AFE_SOF_UL1), + }, + { + .name = "AFE_SOF_UL2", + .no_pcm = 1, + .dpcm_capture = 1, + SND_SOC_DAILINK_REG(AFE_SOF_UL2), + }, }; static const struct snd_soc_dapm_widget mt8186_mt6366_rt1019_rt5682s_widgets[] = { SND_SOC_DAPM_SPK("Speakers", NULL), + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), SND_SOC_DAPM_OUTPUT("HDMI1"), + SND_SOC_DAPM_MIXER(SOF_DMA_DL1, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_DL2, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_UL1, SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER(SOF_DMA_UL2, SND_SOC_NOPM, 0, 0, NULL, 0), }; static const struct snd_soc_dapm_route mt8186_mt6366_rt1019_rt5682s_routes[] = { /* SPK */ { "Speakers", NULL, "Speaker" }, + /* Headset */ + { "Headphone", NULL, "HPOL" }, + { "Headphone", NULL, "HPOR" }, + { "IN1P", NULL, "Headset Mic" }, /* HDMI */ { "HDMI1", NULL, "TX" }, + /* SOF Uplink */ + {SOF_DMA_UL1, NULL, "UL1_CH1"}, + {SOF_DMA_UL1, NULL, "UL1_CH2"}, + {SOF_DMA_UL2, NULL, "UL2_CH1"}, + {SOF_DMA_UL2, NULL, "UL2_CH2"}, + /* SOF Downlink */ + {"DSP_DL1_VIRT", NULL, SOF_DMA_DL1}, + {"DSP_DL2_VIRT", NULL, SOF_DMA_DL2}, }; static const struct snd_kcontrol_new mt8186_mt6366_rt1019_rt5682s_controls[] = { SOC_DAPM_PIN_SWITCH("Speakers"), + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), SOC_DAPM_PIN_SWITCH("HDMI1"), }; static struct snd_soc_card mt8186_mt6366_rt1019_rt5682s_soc_card = { - .name = "mt8186_mt6366_rt1019_rt5682s", + .name = "mt8186_rt1019_rt5682s", .owner = THIS_MODULE, .dai_link = mt8186_mt6366_rt1019_rt5682s_dai_links, .num_links = ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links), @@ -865,8 +995,10 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev) { struct snd_soc_card *card; struct snd_soc_dai_link *dai_link; - struct mt8186_mt6366_rt1019_rt5682s_priv *priv; - struct device_node *platform_node, *headset_codec, *playback_codec; + struct mtk_soc_card_data *soc_card_data; + struct mt8186_mt6366_rt1019_rt5682s_priv *mach_priv; + struct device_node *platform_node, *headset_codec, *playback_codec, *adsp_node; + int sof_on = 0; int ret, i; card = (struct snd_soc_card *)device_get_match_data(&pdev->dev); @@ -874,11 +1006,60 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev) return -EINVAL; card->dev = &pdev->dev; + soc_card_data = devm_kzalloc(&pdev->dev, sizeof(*soc_card_data), GFP_KERNEL); + if (!soc_card_data) + return -ENOMEM; + mach_priv = devm_kzalloc(&pdev->dev, sizeof(*mach_priv), GFP_KERNEL); + if (!mach_priv) + return -ENOMEM; + + soc_card_data->mach_priv = mach_priv; + + adsp_node = of_parse_phandle(pdev->dev.of_node, "mediatek,adsp", 0); + if (adsp_node) { + struct mtk_sof_priv *sof_priv; + + sof_priv = devm_kzalloc(&pdev->dev, sizeof(*sof_priv), GFP_KERNEL); + if (!sof_priv) { + ret = -ENOMEM; + goto err_adsp_node; + } + sof_priv->conn_streams = g_sof_conn_streams; + sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams); + sof_priv->sof_dai_link_fixup = mt8186_sof_dai_link_fixup; + soc_card_data->sof_priv = sof_priv; + card->probe = mtk_sof_card_probe; + card->late_probe = mtk_sof_card_late_probe; + if (!card->topology_shortname_created) { + snprintf(card->topology_shortname, 32, "sof-%s", card->name); + card->topology_shortname_created = true; + } + card->name = card->topology_shortname; + sof_on = 1; + } else { + dev_info(&pdev->dev, "Probe without adsp\n"); + } + + if (of_property_read_bool(pdev->dev.of_node, "mediatek,dai-link")) { + ret = mtk_sof_dailink_parse_of(card, pdev->dev.of_node, + "mediatek,dai-link", + mt8186_mt6366_rt1019_rt5682s_dai_links, + ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links)); + if (ret) { + dev_dbg(&pdev->dev, "Parse dai-link fail\n"); + goto err_adsp_node; + } + } else { + if (!sof_on) + card->num_links = ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links) + - ARRAY_SIZE(g_sof_conn_streams); + } + platform_node = of_parse_phandle(pdev->dev.of_node, "mediatek,platform", 0); if (!platform_node) { ret = -EINVAL; dev_err_probe(&pdev->dev, ret, "Property 'platform' missing or invalid\n"); - return ret; + goto err_platform_node; } playback_codec = of_get_child_by_name(pdev->dev.of_node, "playback-codecs"); @@ -917,17 +1098,14 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev) goto err_probe; } - if (!dai_link->platforms->name) - dai_link->platforms->of_node = platform_node; - } + if (!strncmp(dai_link->name, "AFE_SOF", strlen("AFE_SOF")) && sof_on) + dai_link->platforms->of_node = adsp_node; - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) { - ret = -ENOMEM; - goto err_probe; + if (!dai_link->platforms->name && !dai_link->platforms->of_node) + dai_link->platforms->of_node = platform_node; } - snd_soc_card_set_drvdata(card, priv); + snd_soc_card_set_drvdata(card, soc_card_data); ret = mt8186_afe_gpio_init(&pdev->dev); if (ret) { @@ -945,6 +1123,9 @@ err_headset_codec: of_node_put(playback_codec); err_playback_codec: of_node_put(platform_node); +err_platform_node: +err_adsp_node: + of_node_put(adsp_node); return ret; } diff --git a/sound/soc/mediatek/mt8192/mt8192-afe-common.h b/sound/soc/mediatek/mt8192/mt8192-afe-common.h index d55eff46cc7feb008894af2a71426ef7c4b93757..ad461dcb6ee1a9ab6f5d0dd53cf644eef5f693b9 100644 --- a/sound/soc/mediatek/mt8192/mt8192-afe-common.h +++ b/sound/soc/mediatek/mt8192/mt8192-afe-common.h @@ -159,6 +159,9 @@ int mt8192_dai_src_register(struct mtk_base_afe *afe); int mt8192_dai_pcm_register(struct mtk_base_afe *afe); int mt8192_dai_tdm_register(struct mtk_base_afe *afe); +int mt8192_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name); + unsigned int mt8192_general_rate_transform(struct device *dev, unsigned int rate); unsigned int mt8192_rate_transform(struct device *dev, diff --git a/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c b/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c index 5b29340f9516c9cbe04747fe665d82c8d80a760c..ea516d63d94dd7153b387754b5afd6e12360c0f3 100644 --- a/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c +++ b/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c @@ -45,7 +45,6 @@ struct mtk_afe_i2s_priv { int rate; /* for determine which apll to use */ int low_jitter_en; - const char *share_property_name; int share_i2s_id; int mclk_id; @@ -1984,78 +1983,75 @@ static const struct mtk_afe_i2s_priv mt8192_i2s_priv[DAI_I2S_NUM] = { [DAI_I2S0] = { .id = MT8192_DAI_I2S_0, .mclk_id = MT8192_I2S0_MCK, - .share_property_name = "i2s0-share", .share_i2s_id = -1, }, [DAI_I2S1] = { .id = MT8192_DAI_I2S_1, .mclk_id = MT8192_I2S1_MCK, - .share_property_name = "i2s1-share", .share_i2s_id = -1, }, [DAI_I2S2] = { .id = MT8192_DAI_I2S_2, .mclk_id = MT8192_I2S2_MCK, - .share_property_name = "i2s2-share", .share_i2s_id = -1, }, [DAI_I2S3] = { .id = MT8192_DAI_I2S_3, .mclk_id = MT8192_I2S3_MCK, - .share_property_name = "i2s3-share", .share_i2s_id = -1, }, [DAI_I2S5] = { .id = MT8192_DAI_I2S_5, .mclk_id = MT8192_I2S5_MCK, - .share_property_name = "i2s5-share", .share_i2s_id = -1, }, [DAI_I2S6] = { .id = MT8192_DAI_I2S_6, .mclk_id = MT8192_I2S6_MCK, - .share_property_name = "i2s6-share", .share_i2s_id = -1, }, [DAI_I2S7] = { .id = MT8192_DAI_I2S_7, .mclk_id = MT8192_I2S7_MCK, - .share_property_name = "i2s7-share", .share_i2s_id = -1, }, [DAI_I2S8] = { .id = MT8192_DAI_I2S_8, .mclk_id = MT8192_I2S8_MCK, - .share_property_name = "i2s8-share", .share_i2s_id = -1, }, [DAI_I2S9] = { .id = MT8192_DAI_I2S_9, .mclk_id = MT8192_I2S9_MCK, - .share_property_name = "i2s9-share", .share_i2s_id = -1, }, }; -static int mt8192_dai_i2s_get_share(struct mtk_base_afe *afe) +/** + * mt8192_dai_i2s_set_share() - Set up I2S ports to share a single clock. + * @afe: Pointer to &struct mtk_base_afe + * @main_i2s_name: The name of the I2S port that will provide the clock + * @secondary_i2s_name: The name of the I2S port that will use this clock + */ +int mt8192_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name, + const char *secondary_i2s_name) { - struct mt8192_afe_private *afe_priv = afe->platform_priv; - const struct device_node *of_node = afe->dev->of_node; - const char *of_str; - const char *property_name; - struct mtk_afe_i2s_priv *i2s_priv; - int i; + struct mtk_afe_i2s_priv *secondary_i2s_priv; + int main_i2s_id; - for (i = 0; i < DAI_I2S_NUM; i++) { - i2s_priv = afe_priv->dai_priv[mt8192_i2s_priv[i].id]; - property_name = mt8192_i2s_priv[i].share_property_name; - if (of_property_read_string(of_node, property_name, &of_str)) - continue; - i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str); - } + secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name); + if (!secondary_i2s_priv) + return -EINVAL; + + main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name); + if (main_i2s_id < 0) + return main_i2s_id; + + secondary_i2s_priv->share_i2s_id = main_i2s_id; return 0; } +EXPORT_SYMBOL_GPL(mt8192_dai_i2s_set_share); static int mt8192_dai_i2s_set_priv(struct mtk_base_afe *afe) { @@ -2101,10 +2097,5 @@ int mt8192_dai_i2s_register(struct mtk_base_afe *afe) if (ret) return ret; - /* parse share i2s */ - ret = mt8192_dai_i2s_get_share(afe); - if (ret) - return ret; - return 0; } diff --git a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c index d0f9d66627b15e5cf9ba763c5cf5d149720d0972..b93c3237ef2d6934b3462cdc57b41df1e5a8764c 100644 --- a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c +++ b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c @@ -24,6 +24,8 @@ #include "mt8192-afe-clk.h" #include "mt8192-afe-gpio.h" +#define DRIVER_NAME "mt8192_mt6359" + #define RT1015_CODEC_DAI "rt1015-aif" #define RT1015_DEV0_NAME "rt1015.1-0028" #define RT1015_DEV1_NAME "rt1015.1-0029" @@ -41,6 +43,18 @@ struct mt8192_mt6359_priv { struct snd_soc_jack hdmi_jack; }; +/* Headset jack detection DAPM pins */ +static struct snd_soc_jack_pin mt8192_jack_pins[] = { + { + .pin = "Headphone Jack", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + static int mt8192_rt1015_i2s_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { @@ -311,17 +325,27 @@ static int mt8192_mt6359_init(struct snd_soc_pcm_runtime *rtd) static int mt8192_rt5682_init(struct snd_soc_pcm_runtime *rtd) { + struct snd_soc_component *cmpnt_afe = + snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME); + struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe); struct snd_soc_component *cmpnt_codec = asoc_rtd_to_codec(rtd, 0)->component; struct mt8192_mt6359_priv *priv = snd_soc_card_get_drvdata(rtd->card); struct snd_soc_jack *jack = &priv->headset_jack; int ret; - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + ret = mt8192_dai_i2s_set_share(afe, "I2S8", "I2S9"); + if (ret) { + dev_err(rtd->dev, "Failed to set up shared clocks\n"); + return ret; + } + + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3, - jack); + jack, mt8192_jack_pins, + ARRAY_SIZE(mt8192_jack_pins)); if (ret) { dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret); return ret; @@ -1048,6 +1072,7 @@ static struct snd_soc_codec_conf rt1015_amp_conf[] = { static struct snd_soc_card mt8192_mt6359_rt1015_rt5682_card = { .name = RT1015_RT5682_CARD_NAME, + .driver_name = DRIVER_NAME, .owner = THIS_MODULE, .dai_link = mt8192_mt6359_dai_links, .num_links = ARRAY_SIZE(mt8192_mt6359_dai_links), @@ -1083,6 +1108,7 @@ static const struct snd_kcontrol_new mt8192_mt6359_rt1015p_rt5682x_controls[] = }; static struct snd_soc_card mt8192_mt6359_rt1015p_rt5682x_card = { + .driver_name = DRIVER_NAME, .owner = THIS_MODULE, .dai_link = mt8192_mt6359_dai_links, .num_links = ARRAY_SIZE(mt8192_mt6359_dai_links), @@ -1244,7 +1270,7 @@ static const struct dev_pm_ops mt8192_mt6359_pm_ops = { static struct platform_driver mt8192_mt6359_driver = { .driver = { - .name = "mt8192_mt6359", + .name = DRIVER_NAME, #ifdef CONFIG_OF .of_match_table = mt8192_mt6359_dt_match, #endif diff --git a/sound/soc/mediatek/mt8195/mt8195-afe-clk.c b/sound/soc/mediatek/mt8195/mt8195-afe-clk.c index 2ee3872c83c36f58063385e599ed44d33f76c6ca..9ca2cb8c8a9c2387c71f597225c57fac6e20f92e 100644 --- a/sound/soc/mediatek/mt8195/mt8195-afe-clk.c +++ b/sound/soc/mediatek/mt8195/mt8195-afe-clk.c @@ -117,7 +117,7 @@ static struct mt8195_afe_tuner_cfg mt8195_afe_tuner_cfgs[MT8195_AUD_PLL_NUM] = { .upper_bound_reg = AFE_APLL_TUNER_CFG, .upper_bound_shift = 8, .upper_bound_maskbit = 0xff, - .upper_bound_default = 0x2, + .upper_bound_default = 0x3, }, [MT8195_AUD_PLL2] = { .id = MT8195_AUD_PLL2, @@ -135,7 +135,7 @@ static struct mt8195_afe_tuner_cfg mt8195_afe_tuner_cfgs[MT8195_AUD_PLL_NUM] = { .upper_bound_reg = AFE_APLL_TUNER_CFG1, .upper_bound_shift = 8, .upper_bound_maskbit = 0xff, - .upper_bound_default = 0x2, + .upper_bound_default = 0x3, }, [MT8195_AUD_PLL3] = { .id = MT8195_AUD_PLL3, diff --git a/sound/soc/mediatek/mt8195/mt8195-mt6359.c b/sound/soc/mediatek/mt8195/mt8195-mt6359.c index c530e3fc27e43b7b2cec24a48b5803f4c3e8940d..480ed3e08d5bf9ee31074fc3d58f865910bdcf82 100644 --- a/sound/soc/mediatek/mt8195/mt8195-mt6359.c +++ b/sound/soc/mediatek/mt8195/mt8195-mt6359.c @@ -63,6 +63,18 @@ struct mt8195_mt6359_priv { struct clk *i2so1_mclk; }; +/* Headset jack detection DAPM pins */ +static struct snd_soc_jack_pin mt8195_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + static const struct snd_soc_dapm_widget mt8195_mt6359_widgets[] = { SND_SOC_DAPM_HP("Headphone", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), @@ -563,11 +575,12 @@ static int mt8195_rt5682_init(struct snd_soc_pcm_runtime *rtd) priv->i2so1_mclk = afe_priv->clk[MT8195_CLK_TOP_APLL12_DIV2]; - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3, - jack); + jack, mt8195_jack_pins, + ARRAY_SIZE(mt8195_jack_pins)); if (ret) { dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret); return ret; @@ -1383,7 +1396,13 @@ static int mt8195_mt6359_dev_probe(struct platform_device *pdev) sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams); sof_priv->sof_dai_link_fixup = mt8195_dai_link_fixup; soc_card_data->sof_priv = sof_priv; + card->probe = mtk_sof_card_probe; card->late_probe = mtk_sof_card_late_probe; + if (!card->topology_shortname_created) { + snprintf(card->topology_shortname, 32, "sof-%s", card->name); + card->topology_shortname_created = true; + } + card->name = card->topology_shortname; sof_on = 1; } @@ -1526,16 +1545,11 @@ static const struct of_device_id mt8195_mt6359_dt_match[] = { {}, }; -static const struct dev_pm_ops mt8195_mt6359_pm_ops = { - .poweroff = snd_soc_poweroff, - .restore = snd_soc_resume, -}; - static struct platform_driver mt8195_mt6359_driver = { .driver = { .name = "mt8195_mt6359", .of_match_table = mt8195_mt6359_dt_match, - .pm = &mt8195_mt6359_pm_ops, + .pm = &snd_soc_pm_ops, }, .probe = mt8195_mt6359_dev_probe, }; diff --git a/sound/soc/qcom/Kconfig b/sound/soc/qcom/Kconfig index 750653404ba34056070ebadd1ff19e1a917c0aad..d0e59e07b1fc2f273c13174849fa1c27360c09b2 100644 --- a/sound/soc/qcom/Kconfig +++ b/sound/soc/qcom/Kconfig @@ -173,6 +173,17 @@ config SND_SOC_SM8250 SM8250 SoC-based systems. Say Y if you want to use audio device on this SoCs. +config SND_SOC_SC8280XP + tristate "SoC Machine driver for SC8280XP boards" + depends on QCOM_APR && SOUNDWIRE + depends on COMMON_CLK + select SND_SOC_QDSP6 + select SND_SOC_QCOM_COMMON + help + To add support for audio on Qualcomm Technologies Inc. + SC8280XP SoC-based systems. + Say Y if you want to use audio device on this SoCs. + config SND_SOC_SC7180 tristate "SoC Machine driver for SC7180 boards" depends on I2C && GPIOLIB diff --git a/sound/soc/qcom/Makefile b/sound/soc/qcom/Makefile index 8b7b876899a8dcfabffd4db1bdf2164a7e6768b5..8b97172cf990fb0d825944214372c31ced9a56d3 100644 --- a/sound/soc/qcom/Makefile +++ b/sound/soc/qcom/Makefile @@ -26,6 +26,7 @@ snd-soc-sc7180-objs := sc7180.o snd-soc-sc7280-objs := sc7280.o snd-soc-sdm845-objs := sdm845.o snd-soc-sm8250-objs := sm8250.o +snd-soc-sc8280xp-objs := sc8280xp.o snd-soc-qcom-common-objs := common.o obj-$(CONFIG_SND_SOC_STORM) += snd-soc-storm.o @@ -33,6 +34,7 @@ obj-$(CONFIG_SND_SOC_APQ8016_SBC) += snd-soc-apq8016-sbc.o obj-$(CONFIG_SND_SOC_MSM8996) += snd-soc-apq8096.o obj-$(CONFIG_SND_SOC_SC7180) += snd-soc-sc7180.o obj-$(CONFIG_SND_SOC_SC7280) += snd-soc-sc7280.o +obj-$(CONFIG_SND_SOC_SC8280XP) += snd-soc-sc8280xp.o obj-$(CONFIG_SND_SOC_SDM845) += snd-soc-sdm845.o obj-$(CONFIG_SND_SOC_SM8250) += snd-soc-sm8250.o obj-$(CONFIG_SND_SOC_QCOM_COMMON) += snd-soc-qcom-common.o diff --git a/sound/soc/qcom/common.c b/sound/soc/qcom/common.c index c407684ce1a225b1ab807772048d1402b03a6210..69dd3b504e209fd01e0bee2886d3504c12e86e4e 100644 --- a/sound/soc/qcom/common.c +++ b/sound/soc/qcom/common.c @@ -3,6 +3,9 @@ // Copyright (c) 2018, The Linux Foundation. All rights reserved. #include <linux/module.h> +#include <sound/jack.h> +#include <linux/input-event-codes.h> +#include "qdsp6/q6afe.h" #include "common.h" int qcom_snd_parse_of(struct snd_soc_card *card) @@ -175,6 +178,174 @@ err_put_np: of_node_put(np); return ret; } -EXPORT_SYMBOL(qcom_snd_parse_of); +EXPORT_SYMBOL_GPL(qcom_snd_parse_of); +#if IS_ENABLED(CONFIG_SOUNDWIRE) +int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *sruntime, + bool *stream_prepared) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + int ret; + + if (!sruntime) + return 0; + + switch (cpu_dai->id) { + case WSA_CODEC_DMA_RX_0: + case WSA_CODEC_DMA_RX_1: + case RX_CODEC_DMA_RX_0: + case RX_CODEC_DMA_RX_1: + case TX_CODEC_DMA_TX_0: + case TX_CODEC_DMA_TX_1: + case TX_CODEC_DMA_TX_2: + case TX_CODEC_DMA_TX_3: + break; + default: + return 0; + } + + if (*stream_prepared) { + sdw_disable_stream(sruntime); + sdw_deprepare_stream(sruntime); + *stream_prepared = false; + } + + ret = sdw_prepare_stream(sruntime); + if (ret) + return ret; + + /** + * NOTE: there is a strict hw requirement about the ordering of port + * enables and actual WSA881x PA enable. PA enable should only happen + * after soundwire ports are enabled if not DC on the line is + * accumulated resulting in Click/Pop Noise + * PA enable/mute are handled as part of codec DAPM and digital mute. + */ + + ret = sdw_enable_stream(sruntime); + if (ret) { + sdw_deprepare_stream(sruntime); + return ret; + } + *stream_prepared = true; + + return ret; +} +EXPORT_SYMBOL_GPL(qcom_snd_sdw_prepare); + +int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct sdw_stream_runtime **psruntime) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *codec_dai; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sdw_stream_runtime *sruntime; + int i; + + switch (cpu_dai->id) { + case WSA_CODEC_DMA_RX_0: + case RX_CODEC_DMA_RX_0: + case RX_CODEC_DMA_RX_1: + case TX_CODEC_DMA_TX_0: + case TX_CODEC_DMA_TX_1: + case TX_CODEC_DMA_TX_2: + case TX_CODEC_DMA_TX_3: + for_each_rtd_codec_dais(rtd, i, codec_dai) { + sruntime = snd_soc_dai_get_stream(codec_dai, substream->stream); + if (sruntime != ERR_PTR(-ENOTSUPP)) + *psruntime = sruntime; + } + break; + } + + return 0; + +} +EXPORT_SYMBOL_GPL(qcom_snd_sdw_hw_params); + +int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *sruntime, bool *stream_prepared) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + + switch (cpu_dai->id) { + case WSA_CODEC_DMA_RX_0: + case WSA_CODEC_DMA_RX_1: + case RX_CODEC_DMA_RX_0: + case RX_CODEC_DMA_RX_1: + case TX_CODEC_DMA_TX_0: + case TX_CODEC_DMA_TX_1: + case TX_CODEC_DMA_TX_2: + case TX_CODEC_DMA_TX_3: + if (sruntime && *stream_prepared) { + sdw_disable_stream(sruntime); + sdw_deprepare_stream(sruntime); + *stream_prepared = false; + } + break; + default: + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(qcom_snd_sdw_hw_free); +#endif + +int qcom_snd_wcd_jack_setup(struct snd_soc_pcm_runtime *rtd, + struct snd_soc_jack *jack, bool *jack_setup) +{ + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct snd_soc_card *card = rtd->card; + int rval, i; + + if (!*jack_setup) { + rval = snd_soc_card_jack_new(card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_LINEOUT | + SND_JACK_MECHANICAL | + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3 | + SND_JACK_BTN_4 | SND_JACK_BTN_5, + jack); + + if (rval < 0) { + dev_err(card->dev, "Unable to add Headphone Jack\n"); + return rval; + } + + snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_MEDIA); + snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOICECOMMAND); + snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEUP); + snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN); + *jack_setup = true; + } + + switch (cpu_dai->id) { + case TX_CODEC_DMA_TX_0: + case TX_CODEC_DMA_TX_1: + case TX_CODEC_DMA_TX_2: + case TX_CODEC_DMA_TX_3: + for_each_rtd_codec_dais(rtd, i, codec_dai) { + rval = snd_soc_component_set_jack(codec_dai->component, + jack, NULL); + if (rval != 0 && rval != -ENOTSUPP) { + dev_warn(card->dev, "Failed to set jack: %d\n", rval); + return rval; + } + } + + break; + default: + break; + } + + + return 0; +} +EXPORT_SYMBOL_GPL(qcom_snd_wcd_jack_setup); MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/qcom/common.h b/sound/soc/qcom/common.h index f05c05b12bd74a00e5b5bcc3e71f30d6c92b875f..c5472a642de082910bd1f46aeea79a5646f841ad 100644 --- a/sound/soc/qcom/common.h +++ b/sound/soc/qcom/common.h @@ -5,7 +5,42 @@ #define __QCOM_SND_COMMON_H__ #include <sound/soc.h> +#include <linux/soundwire/sdw.h> int qcom_snd_parse_of(struct snd_soc_card *card); +int qcom_snd_wcd_jack_setup(struct snd_soc_pcm_runtime *rtd, + struct snd_soc_jack *jack, bool *jack_setup); +#if IS_ENABLED(CONFIG_SOUNDWIRE) +int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *runtime, + bool *stream_prepared); +int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct sdw_stream_runtime **psruntime); +int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *sruntime, + bool *stream_prepared); +#else +static inline int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *runtime, + bool *stream_prepared) +{ + return -ENOTSUPP; +} + +static inline int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct sdw_stream_runtime **psruntime) +{ + return -ENOTSUPP; +} + +static inline int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream, + struct sdw_stream_runtime *sruntime, + bool *stream_prepared) +{ + return -ENOTSUPP; +} +#endif #endif diff --git a/sound/soc/qcom/qdsp6/q6prm-clocks.c b/sound/soc/qcom/qdsp6/q6prm-clocks.c index a26cda5140c17e3e7247ab27aa29d5c2987dcbc3..73b0cbac73d4f3b26b13a166467ddeb44fbdeb4c 100644 --- a/sound/soc/qcom/qdsp6/q6prm-clocks.c +++ b/sound/soc/qcom/qdsp6/q6prm-clocks.c @@ -50,6 +50,15 @@ static const struct q6dsp_clk_init q6prm_clks[] = { Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_MCLK), Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_NPL_MCLK), Q6PRM_CLK(LPASS_CLK_ID_VA_CORE_2X_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_2X_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_TX_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_TX_2X_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_TX_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_TX_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK), + Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK), Q6DSP_VOTE_CLK(LPASS_HW_MACRO_VOTE, Q6PRM_HW_CORE_ID_LPASS, "LPASS_HW_MACRO"), Q6DSP_VOTE_CLK(LPASS_HW_DCODEC_VOTE, Q6PRM_HW_CORE_ID_DCODEC, diff --git a/sound/soc/qcom/qdsp6/q6prm.h b/sound/soc/qcom/qdsp6/q6prm.h index fea4d1954bc1c031e9078178e18246534e6e8712..a988a32086fe105e32c4fd713b8e0a3d17b83bca 100644 --- a/sound/soc/qcom/qdsp6/q6prm.h +++ b/sound/soc/qcom/qdsp6/q6prm.h @@ -64,6 +64,25 @@ #define Q6PRM_LPASS_CLK_ID_RX_CORE_MCLK 0x30e #define Q6PRM_LPASS_CLK_ID_RX_CORE_NPL_MCLK 0x30f +/* Clock ID for MCLK for WSA2 core */ +#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_MCLK 0x310 +/* Clock ID for NPL MCLK for WSA2 core */ +#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_2X_MCLK 0x311 +/* Clock ID for RX Core TX MCLK */ +#define Q6PRM_LPASS_CLK_ID_RX_CORE_TX_MCLK 0x312 +/* Clock ID for RX CORE TX 2X MCLK */ +#define Q6PRM_LPASS_CLK_ID_RX_CORE_TX_2X_MCLK 0x313 +/* Clock ID for WSA core TX MCLK */ +#define Q6PRM_LPASS_CLK_ID_WSA_CORE_TX_MCLK 0x314 +/* Clock ID for WSA core TX 2X MCLK */ +#define Q6PRM_LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK 0x315 +/* Clock ID for WSA2 core TX MCLK */ +#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_TX_MCLK 0x316 +/* Clock ID for WSA2 core TX 2X MCLK */ +#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK 0x317 +/* Clock ID for RX CORE MCLK2 2X MCLK */ +#define Q6PRM_LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK 0x318 + #define Q6PRM_LPASS_CLK_SRC_INTERNAL 1 #define Q6PRM_LPASS_CLK_ROOT_DEFAULT 0 #define Q6PRM_HW_CORE_ID_LPASS 1 diff --git a/sound/soc/qcom/sc8280xp.c b/sound/soc/qcom/sc8280xp.c new file mode 100644 index 0000000000000000000000000000000000000000..ade44ad7c585a3d1a5a7c593eddf2c08c89730db --- /dev/null +++ b/sound/soc/qcom/sc8280xp.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2022, Linaro Limited + +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of_device.h> +#include <sound/soc.h> +#include <sound/soc-dapm.h> +#include <sound/pcm.h> +#include <linux/soundwire/sdw.h> +#include <sound/jack.h> +#include <linux/input-event-codes.h> +#include "qdsp6/q6afe.h" +#include "common.h" + +#define DRIVER_NAME "sc8280xp" + +struct sc8280xp_snd_data { + bool stream_prepared[AFE_PORT_MAX]; + struct snd_soc_card *card; + struct sdw_stream_runtime *sruntime[AFE_PORT_MAX]; + struct snd_soc_jack jack; + bool jack_setup; +}; + +static int sc8280xp_snd_init(struct snd_soc_pcm_runtime *rtd) +{ + struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card); + + return qcom_snd_wcd_jack_setup(rtd, &data->jack, &data->jack_setup); +} + +static int sc8280xp_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_interval *rate = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + struct snd_interval *channels = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_CHANNELS); + + rate->min = rate->max = 48000; + channels->min = 2; + channels->max = 2; + switch (cpu_dai->id) { + case TX_CODEC_DMA_TX_0: + case TX_CODEC_DMA_TX_1: + case TX_CODEC_DMA_TX_2: + case TX_CODEC_DMA_TX_3: + channels->min = 1; + break; + default: + break; + } + + + return 0; +} + +static int sc8280xp_snd_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sc8280xp_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card); + + return qcom_snd_sdw_hw_params(substream, params, &pdata->sruntime[cpu_dai->id]); +} + +static int sc8280xp_snd_prepare(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card); + struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + + return qcom_snd_sdw_prepare(substream, sruntime, + &data->stream_prepared[cpu_dai->id]); +} + +static int sc8280xp_snd_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; + + return qcom_snd_sdw_hw_free(substream, sruntime, + &data->stream_prepared[cpu_dai->id]); +} + +static const struct snd_soc_ops sc8280xp_be_ops = { + .hw_params = sc8280xp_snd_hw_params, + .hw_free = sc8280xp_snd_hw_free, + .prepare = sc8280xp_snd_prepare, +}; + +static void sc8280xp_add_be_ops(struct snd_soc_card *card) +{ + struct snd_soc_dai_link *link; + int i; + + for_each_card_prelinks(card, i, link) { + if (link->no_pcm == 1) { + link->init = sc8280xp_snd_init; + link->be_hw_params_fixup = sc8280xp_be_hw_params_fixup; + link->ops = &sc8280xp_be_ops; + } + } +} + +static int sc8280xp_platform_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card; + struct sc8280xp_snd_data *data; + struct device *dev = &pdev->dev; + int ret; + + card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL); + if (!card) + return -ENOMEM; + card->owner = THIS_MODULE; + /* Allocate the private data */ + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + card->dev = dev; + dev_set_drvdata(dev, card); + snd_soc_card_set_drvdata(card, data); + ret = qcom_snd_parse_of(card); + if (ret) + return ret; + + card->driver_name = DRIVER_NAME; + sc8280xp_add_be_ops(card); + return devm_snd_soc_register_card(dev, card); +} + +static const struct of_device_id snd_sc8280xp_dt_match[] = { + {.compatible = "qcom,sc8280xp-sndcard",}, + {} +}; + +MODULE_DEVICE_TABLE(of, snd_sc8280xp_dt_match); + +static struct platform_driver snd_sc8280xp_driver = { + .probe = sc8280xp_platform_probe, + .driver = { + .name = "snd-sc8280xp", + .of_match_table = snd_sc8280xp_dt_match, + }, +}; +module_platform_driver(snd_sc8280xp_driver); +MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org"); +MODULE_DESCRIPTION("SC8280XP ASoC Machine Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/sound/soc/qcom/sm8250.c b/sound/soc/qcom/sm8250.c index ce4a5713386a3681f6b17ce7ad3c59436554323c..8dbe9ef41b1c9e463b94a491e7ebb2e33ce1dd2f 100644 --- a/sound/soc/qcom/sm8250.c +++ b/sound/soc/qcom/sm8250.c @@ -27,57 +27,8 @@ struct sm8250_snd_data { static int sm8250_snd_init(struct snd_soc_pcm_runtime *rtd) { struct sm8250_snd_data *data = snd_soc_card_get_drvdata(rtd->card); - struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); - struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); - struct snd_soc_card *card = rtd->card; - int rval, i; - - if (!data->jack_setup) { - struct snd_jack *jack; - - rval = snd_soc_card_jack_new(card, "Headset Jack", - SND_JACK_HEADSET | SND_JACK_LINEOUT | - SND_JACK_MECHANICAL | - SND_JACK_BTN_0 | SND_JACK_BTN_1 | - SND_JACK_BTN_2 | SND_JACK_BTN_3 | - SND_JACK_BTN_4 | SND_JACK_BTN_5, - &data->jack); - - if (rval < 0) { - dev_err(card->dev, "Unable to add Headphone Jack\n"); - return rval; - } - - jack = data->jack.jack; - - snd_jack_set_key(jack, SND_JACK_BTN_0, KEY_MEDIA); - snd_jack_set_key(jack, SND_JACK_BTN_1, KEY_VOICECOMMAND); - snd_jack_set_key(jack, SND_JACK_BTN_2, KEY_VOLUMEUP); - snd_jack_set_key(jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN); - data->jack_setup = true; - } - - switch (cpu_dai->id) { - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - for_each_rtd_codec_dais(rtd, i, codec_dai) { - rval = snd_soc_component_set_jack(codec_dai->component, - &data->jack, NULL); - if (rval != 0 && rval != -ENOTSUPP) { - dev_warn(card->dev, "Failed to set jack: %d\n", rval); - return rval; - } - } - - break; - default: - break; - } - - return 0; + return qcom_snd_wcd_jack_setup(rtd, &data->jack, &data->jack_setup); } static int sm8250_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd, @@ -121,92 +72,21 @@ static int sm8250_snd_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params) { struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *codec_dai; struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct sm8250_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card); - struct sdw_stream_runtime *sruntime; - int i; - - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - for_each_rtd_codec_dais(rtd, i, codec_dai) { - sruntime = snd_soc_dai_get_stream(codec_dai, - substream->stream); - if (sruntime != ERR_PTR(-ENOTSUPP)) - pdata->sruntime[cpu_dai->id] = sruntime; - } - break; - } - - return 0; + return qcom_snd_sdw_hw_params(substream, params, &pdata->sruntime[cpu_dai->id]); } -static int sm8250_snd_wsa_dma_prepare(struct snd_pcm_substream *substream) +static int sm8250_snd_prepare(struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = substream->private_data; struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct sm8250_snd_data *data = snd_soc_card_get_drvdata(rtd->card); struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; - int ret; - - if (!sruntime) - return 0; - if (data->stream_prepared[cpu_dai->id]) { - sdw_disable_stream(sruntime); - sdw_deprepare_stream(sruntime); - data->stream_prepared[cpu_dai->id] = false; - } - - ret = sdw_prepare_stream(sruntime); - if (ret) - return ret; - - /** - * NOTE: there is a strict hw requirement about the ordering of port - * enables and actual WSA881x PA enable. PA enable should only happen - * after soundwire ports are enabled if not DC on the line is - * accumulated resulting in Click/Pop Noise - * PA enable/mute are handled as part of codec DAPM and digital mute. - */ - - ret = sdw_enable_stream(sruntime); - if (ret) { - sdw_deprepare_stream(sruntime); - return ret; - } - data->stream_prepared[cpu_dai->id] = true; - - return ret; -} - -static int sm8250_snd_prepare(struct snd_pcm_substream *substream) -{ - struct snd_soc_pcm_runtime *rtd = substream->private_data; - struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); - - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case WSA_CODEC_DMA_RX_1: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - return sm8250_snd_wsa_dma_prepare(substream); - default: - break; - } - - return 0; + return qcom_snd_sdw_prepare(substream, sruntime, + &data->stream_prepared[cpu_dai->id]); } static int sm8250_snd_hw_free(struct snd_pcm_substream *substream) @@ -216,26 +96,8 @@ static int sm8250_snd_hw_free(struct snd_pcm_substream *substream) struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id]; - switch (cpu_dai->id) { - case WSA_CODEC_DMA_RX_0: - case WSA_CODEC_DMA_RX_1: - case RX_CODEC_DMA_RX_0: - case RX_CODEC_DMA_RX_1: - case TX_CODEC_DMA_TX_0: - case TX_CODEC_DMA_TX_1: - case TX_CODEC_DMA_TX_2: - case TX_CODEC_DMA_TX_3: - if (sruntime && data->stream_prepared[cpu_dai->id]) { - sdw_disable_stream(sruntime); - sdw_deprepare_stream(sruntime); - data->stream_prepared[cpu_dai->id] = false; - } - break; - default: - break; - } - - return 0; + return qcom_snd_sdw_hw_free(substream, sruntime, + &data->stream_prepared[cpu_dai->id]); } static const struct snd_soc_ops sm8250_be_ops = { @@ -270,6 +132,7 @@ static int sm8250_platform_probe(struct platform_device *pdev) if (!card) return -ENOMEM; + card->owner = THIS_MODULE; /* Allocate the private data */ data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) diff --git a/sound/soc/rockchip/rockchip_i2s.c b/sound/soc/rockchip/rockchip_i2s.c index f5f3540a9e18613e78522a0480ea6e496ccb3d44..a8758ad68442daba7bdbe22dce06851e0c6fa96f 100644 --- a/sound/soc/rockchip/rockchip_i2s.c +++ b/sound/soc/rockchip/rockchip_i2s.c @@ -126,7 +126,6 @@ static inline struct rk_i2s_dev *to_info(struct snd_soc_dai *dai) static int rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on) { unsigned int val = 0; - int retry = 10; int ret = 0; spin_lock(&i2s->lock); @@ -163,18 +162,14 @@ static int rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on) I2S_CLR_TXC | I2S_CLR_RXC); if (ret < 0) goto end; - regmap_read(i2s->regmap, I2S_CLR, &val); - - /* Should wait for clear operation to finish */ - while (val) { - regmap_read(i2s->regmap, I2S_CLR, &val); - retry--; - if (!retry) { - dev_warn(i2s->dev, "fail to clear\n"); - ret = -EBUSY; - break; - } - } + ret = regmap_read_poll_timeout_atomic(i2s->regmap, + I2S_CLR, + val, + val == 0, + 20, + 200); + if (ret < 0) + dev_warn(i2s->dev, "fail to clear: %d\n", ret); } } end: @@ -188,7 +183,6 @@ end: static int rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on) { unsigned int val = 0; - int retry = 10; int ret = 0; spin_lock(&i2s->lock); @@ -226,17 +220,14 @@ static int rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on) I2S_CLR_TXC | I2S_CLR_RXC); if (ret < 0) goto end; - regmap_read(i2s->regmap, I2S_CLR, &val); - /* Should wait for clear operation to finish */ - while (val) { - regmap_read(i2s->regmap, I2S_CLR, &val); - retry--; - if (!retry) { - dev_warn(i2s->dev, "fail to clear\n"); - ret = -EBUSY; - break; - } - } + ret = regmap_read_poll_timeout_atomic(i2s->regmap, + I2S_CLR, + val, + val == 0, + 20, + 200); + if (ret < 0) + dev_warn(i2s->dev, "fail to clear: %d\n", ret); } } end: diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c index e7d52d27132ea1bf160b10f27230a383026845fc..0fbbf3b02c09570fb531b86acb402868808ed5bf 100644 --- a/sound/soc/samsung/aries_wm8994.c +++ b/sound/soc/samsung/aries_wm8994.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ #include <linux/extcon.h> #include <linux/iio/consumer.h> -#include <linux/iio/iio.h> #include <linux/input-event-codes.h> #include <linux/mfd/wm8994/registers.h> #include <linux/module.h> @@ -543,6 +542,7 @@ static int aries_audio_probe(struct platform_device *pdev) struct aries_wm8994_data *priv; struct snd_soc_dai_link *dai_link; const struct of_device_id *match; + enum iio_chan_type channel_type; int ret, i; if (!np) @@ -594,7 +594,11 @@ static int aries_audio_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(priv->adc), "Failed to get ADC channel"); - if (priv->adc->channel->type != IIO_VOLTAGE) + ret = iio_get_channel_type(priv->adc, &channel_type); + if (ret) + return dev_err_probe(dev, ret, + "Failed to get ADC channel type"); + if (channel_type != IIO_VOLTAGE) return -EINVAL; priv->gpio_headset_key = devm_gpiod_get(dev, "headset-key", diff --git a/sound/soc/samsung/odroid.c b/sound/soc/samsung/odroid.c index 4ff12e2e704fecb10c6e4f6cec313c8b4da27319..1e0fefa89ad5e2196915efc2087ed59a8c75e6af 100644 --- a/sound/soc/samsung/odroid.c +++ b/sound/soc/samsung/odroid.c @@ -97,7 +97,7 @@ static int odroid_card_be_hw_params(struct snd_pcm_substream *substream, if (ret < 0) return ret; - if (rtd->num_codecs > 1) { + if (rtd->dai_link->num_codecs > 1) { struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 1); ret = snd_soc_dai_set_sysclk(codec_dai, 0, rclk_freq, diff --git a/sound/soc/sh/rcar/ctu.c b/sound/soc/sh/rcar/ctu.c index 6156445bcb69a3e0297517986eb2e83ec6fd7876..e39eb2ac7e9554afd56d2a94dc32a1803ef03b2c 100644 --- a/sound/soc/sh/rcar/ctu.c +++ b/sound/soc/sh/rcar/ctu.c @@ -171,7 +171,11 @@ static int rsnd_ctu_init(struct rsnd_mod *mod, struct rsnd_dai_stream *io, struct rsnd_priv *priv) { - rsnd_mod_power_on(mod); + int ret; + + ret = rsnd_mod_power_on(mod); + if (ret < 0) + return ret; rsnd_ctu_activation(mod); diff --git a/sound/soc/sh/rcar/dvc.c b/sound/soc/sh/rcar/dvc.c index 5137e03a9d7c71f2000c093b518711aa77673d51..16befcbc312cb7de8f863af54465832e2aff1804 100644 --- a/sound/soc/sh/rcar/dvc.c +++ b/sound/soc/sh/rcar/dvc.c @@ -186,7 +186,11 @@ static int rsnd_dvc_init(struct rsnd_mod *mod, struct rsnd_dai_stream *io, struct rsnd_priv *priv) { - rsnd_mod_power_on(mod); + int ret; + + ret = rsnd_mod_power_on(mod); + if (ret < 0) + return ret; rsnd_dvc_activation(mod); diff --git a/sound/soc/sh/rcar/mix.c b/sound/soc/sh/rcar/mix.c index 3572c2c5686c7973d06b32f7b0a7fb471f868a8d..1de0e085804cc247f139e4f83bec86343e4df45b 100644 --- a/sound/soc/sh/rcar/mix.c +++ b/sound/soc/sh/rcar/mix.c @@ -146,7 +146,11 @@ static int rsnd_mix_init(struct rsnd_mod *mod, struct rsnd_dai_stream *io, struct rsnd_priv *priv) { - rsnd_mod_power_on(mod); + int ret; + + ret = rsnd_mod_power_on(mod); + if (ret < 0) + return ret; rsnd_mix_activation(mod); diff --git a/sound/soc/sh/rcar/src.c b/sound/soc/sh/rcar/src.c index 0ea84ae57c6acbb0b23eed19eef33486e4066918..f832165e46bc04dc413086dc7bd8f57647f98a88 100644 --- a/sound/soc/sh/rcar/src.c +++ b/sound/soc/sh/rcar/src.c @@ -463,11 +463,14 @@ static int rsnd_src_init(struct rsnd_mod *mod, struct rsnd_priv *priv) { struct rsnd_src *src = rsnd_mod_to_src(mod); + int ret; /* reset sync convert_rate */ src->sync.val = 0; - rsnd_mod_power_on(mod); + ret = rsnd_mod_power_on(mod); + if (ret < 0) + return ret; rsnd_src_activation(mod); diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c index 43c5e27dc5c869716b861dadf0d65c21d6019187..7ade6c5ed96ff239b8ab91fa67c41a6f0b59008c 100644 --- a/sound/soc/sh/rcar/ssi.c +++ b/sound/soc/sh/rcar/ssi.c @@ -480,7 +480,9 @@ static int rsnd_ssi_init(struct rsnd_mod *mod, ssi->usrcnt++; - rsnd_mod_power_on(mod); + ret = rsnd_mod_power_on(mod); + if (ret < 0) + return ret; rsnd_ssi_config_init(mod, io); diff --git a/sound/soc/sh/rz-ssi.c b/sound/soc/sh/rz-ssi.c index 7ace0c0db5b154b457415ece608296cb0e63f094..5d6bae33ae34ca64ce079a771c2624ae7274a14b 100644 --- a/sound/soc/sh/rz-ssi.c +++ b/sound/soc/sh/rz-ssi.c @@ -598,7 +598,7 @@ static int rz_ssi_dma_transfer(struct rz_ssi_priv *ssi, return -EINVAL; runtime = substream->runtime; - if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) + if (runtime->state == SNDRV_PCM_STATE_DRAINING) /* * Stream is ending, so do not queue up any more DMA * transfers otherwise we play partial sound clips diff --git a/sound/soc/soc-ac97.c b/sound/soc/soc-ac97.c index 5f49e3dec3fc164998c56ec36f9891a5a0944ccd..32c5be61e2ec8269a0229a1dc5152081d116d4f6 100644 --- a/sound/soc/soc-ac97.c +++ b/sound/soc/soc-ac97.c @@ -57,7 +57,7 @@ static inline struct snd_soc_component *gpio_to_component(struct gpio_chip *chip return gpio_priv->component; } -static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned offset) +static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned int offset) { if (offset >= AC97_NUM_GPIOS) return -EINVAL; @@ -66,7 +66,7 @@ static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned offset) } static int snd_soc_ac97_gpio_direction_in(struct gpio_chip *chip, - unsigned offset) + unsigned int offset) { struct snd_soc_component *component = gpio_to_component(chip); @@ -75,7 +75,7 @@ static int snd_soc_ac97_gpio_direction_in(struct gpio_chip *chip, 1 << offset, 1 << offset); } -static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned offset) +static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned int offset) { struct snd_soc_component *component = gpio_to_component(chip); int ret; @@ -88,7 +88,7 @@ static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned offset) return !!(ret & (1 << offset)); } -static void snd_soc_ac97_gpio_set(struct gpio_chip *chip, unsigned offset, +static void snd_soc_ac97_gpio_set(struct gpio_chip *chip, unsigned int offset, int value) { struct snd_ac97_gpio_priv *gpio_priv = gpiochip_get_data(chip); diff --git a/sound/soc/soc-component.c b/sound/soc/soc-component.c index e12f8244242b9afd96bc3c85da56d5b293896a8d..659b9ade415869d2806058846259211ba78388c2 100644 --- a/sound/soc/soc-component.c +++ b/sound/soc/soc-component.c @@ -1213,11 +1213,9 @@ int snd_soc_pcm_component_pm_runtime_get(struct snd_soc_pcm_runtime *rtd, int i; for_each_rtd_components(rtd, i, component) { - int ret = pm_runtime_get_sync(component->dev); - if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(component->dev); + int ret = pm_runtime_resume_and_get(component->dev); + if (ret < 0 && ret != -EACCES) return soc_component_ret(component, ret); - } /* mark stream if succeeded */ soc_component_mark_push(component, stream, pm); } diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c index e9dd25894dc0fd97636a960300a821d2877863d0..870f13e1d389c8c3afc4eb419791a83fbe228a36 100644 --- a/sound/soc/soc-compress.c +++ b/sound/soc/soc-compress.c @@ -560,8 +560,8 @@ int snd_soc_new_compress(struct snd_soc_pcm_runtime *rtd, int num) BUILD_BUG_ON((int)SNDRV_PCM_STREAM_PLAYBACK != (int)SND_COMPRESS_PLAYBACK); BUILD_BUG_ON((int)SNDRV_PCM_STREAM_CAPTURE != (int)SND_COMPRESS_CAPTURE); - if (rtd->num_cpus > 1 || - rtd->num_codecs > 1) { + if (rtd->dai_link->num_cpus > 1 || + rtd->dai_link->num_codecs > 1) { dev_err(rtd->card->dev, "Compress ASoC: Multi CPU/Codec not supported\n"); return -EINVAL; diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index e824ff1a9fc0961e9ef35f2ff6cd11c0b022c5c8..12a82f5a3ff64ae2aedddb16164fb8d6075e445e 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -72,7 +72,7 @@ static ssize_t pmdown_time_show(struct device *dev, { struct snd_soc_pcm_runtime *rtd = dev_get_drvdata(dev); - return sprintf(buf, "%ld\n", rtd->pmdown_time); + return sysfs_emit(buf, "%ld\n", rtd->pmdown_time); } static ssize_t pmdown_time_store(struct device *dev, @@ -107,7 +107,7 @@ static umode_t soc_dev_attr_is_visible(struct kobject *kobj, if (attr == &dev_attr_pmdown_time.attr) return attr->mode; /* always visible */ - return rtd->num_codecs ? attr->mode : 0; /* enabled only with codec */ + return rtd->dai_link->num_codecs ? attr->mode : 0; /* enabled only with codec */ } static const struct attribute_group soc_dapm_dev_group = { @@ -482,11 +482,10 @@ static struct snd_soc_pcm_runtime *soc_new_pcm_runtime( * asoc_rtd_to_cpu() * asoc_rtd_to_codec() */ - rtd->num_cpus = dai_link->num_cpus; - rtd->num_codecs = dai_link->num_codecs; rtd->card = card; rtd->dai_link = dai_link; rtd->num = card->num_rtd++; + rtd->pmdown_time = pmdown_time; /* default power off timeout */ /* see for_each_card_rtds */ list_add_tail(&rtd->list, &card->rtd_list); @@ -1247,9 +1246,6 @@ static int soc_init_pcm_runtime(struct snd_soc_card *card, struct snd_soc_component *component; int ret, num, i; - /* set default power off timeout */ - rtd->pmdown_time = pmdown_time; - /* do machine specific initialization */ ret = snd_soc_link_init(rtd); if (ret < 0) @@ -1840,21 +1836,22 @@ match: } } -#define soc_setup_card_name(name, name1, name2, norm) \ - __soc_setup_card_name(name, sizeof(name), name1, name2, norm) -static void __soc_setup_card_name(char *name, int len, - const char *name1, const char *name2, - int normalization) +#define soc_setup_card_name(card, name, name1, name2) \ + __soc_setup_card_name(card, name, sizeof(name), name1, name2) +static void __soc_setup_card_name(struct snd_soc_card *card, + char *name, int len, + const char *name1, const char *name2) { + const char *src = name1 ? name1 : name2; int i; - snprintf(name, len, "%s", name1 ? name1 : name2); + snprintf(name, len, "%s", src); - if (!normalization) + if (name != card->snd_card->driver) return; /* - * Name normalization + * Name normalization (driver field) * * The driver name is somewhat special, as it's used as a key for * searches in the user-space. @@ -1874,6 +1871,14 @@ static void __soc_setup_card_name(char *name, int len, break; } } + + /* + * The driver field should contain a valid string from the user view. + * The wrapping usually does not work so well here. Set a smaller string + * in the specific ASoC driver. + */ + if (strlen(src) > len - 1) + dev_err(card->dev, "ASoC: driver name too long '%s' -> '%s'\n", src, name); } static void soc_cleanup_card_resources(struct snd_soc_card *card) @@ -2041,12 +2046,12 @@ static int snd_soc_bind_card(struct snd_soc_card *card) /* try to set some sane longname if DMI is available */ snd_soc_set_dmi_name(card, NULL); - soc_setup_card_name(card->snd_card->shortname, - card->name, NULL, 0); - soc_setup_card_name(card->snd_card->longname, - card->long_name, card->name, 0); - soc_setup_card_name(card->snd_card->driver, - card->driver_name, card->name, 1); + soc_setup_card_name(card, card->snd_card->shortname, + card->name, NULL); + soc_setup_card_name(card, card->snd_card->longname, + card->long_name, card->name); + soc_setup_card_name(card, card->snd_card->driver, + card->driver_name, card->name); if (card->components) { /* the current implementation of snd_component_add() accepts */ diff --git a/sound/soc/soc-dai.c b/sound/soc/soc-dai.c index d530e8c2b77b1a83cc6c4559cd9c47e5ef30758d..49752af0e205d69600e2fe9ae895cec104141d5c 100644 --- a/sound/soc/soc-dai.c +++ b/sound/soc/soc-dai.c @@ -124,7 +124,7 @@ EXPORT_SYMBOL_GPL(snd_soc_dai_set_pll); */ int snd_soc_dai_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) { - int ret = -EINVAL; + int ret = -ENOTSUPP; if (dai->driver->ops && dai->driver->ops->set_bclk_ratio) diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index b05231414c1d0f6d8e8be7ee8bacdf2c4f70fa56..d515e7a78ea86892b543a4dc3c46ffe0dea1bef2 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -2386,11 +2386,10 @@ int snd_soc_dapm_mixer_update_power(struct snd_soc_dapm_context *dapm, EXPORT_SYMBOL_GPL(snd_soc_dapm_mixer_update_power); static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt, - char *buf) + char *buf, int count) { struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(cmpnt); struct snd_soc_dapm_widget *w; - int count = 0; char *state = "not set"; /* card won't be set for the dummy component, as a spot fix @@ -2423,7 +2422,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt, case snd_soc_dapm_pinctrl: case snd_soc_dapm_clock_supply: if (w->name) - count += sprintf(buf + count, "%s: %s\n", + count += sysfs_emit_at(buf, count, "%s: %s\n", w->name, w->power ? "On":"Off"); break; default: @@ -2445,7 +2444,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt, state = "Off"; break; } - count += sprintf(buf + count, "PM State: %s\n", state); + count += sysfs_emit_at(buf, count, "PM State: %s\n", state); return count; } @@ -2463,7 +2462,7 @@ static ssize_t dapm_widget_show(struct device *dev, for_each_rtd_codec_dais(rtd, i, codec_dai) { struct snd_soc_component *cmpnt = codec_dai->component; - count += dapm_widget_show_component(cmpnt, buf + count); + count = dapm_widget_show_component(cmpnt, buf, count); } mutex_unlock(&rtd->card->dapm_mutex); @@ -3631,10 +3630,18 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, enum snd_soc_dapm_direction dir; struct snd_soc_dapm_widget *w; const char *prefix; - int ret; + int ret = -ENOMEM; if ((w = dapm_cnew_widget(widget)) == NULL) - return ERR_PTR(-ENOMEM); + goto cnew_failed; + + prefix = soc_dapm_prefix(dapm); + if (prefix) + w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, widget->name); + else + w->name = kstrdup_const(widget->name, GFP_KERNEL); + if (!w->name) + goto name_failed; switch (w->id) { case snd_soc_dapm_regulator_supply: @@ -3673,17 +3680,6 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, break; } - prefix = soc_dapm_prefix(dapm); - if (prefix) - w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, widget->name); - else - w->name = kstrdup_const(widget->name, GFP_KERNEL); - if (w->name == NULL) { - kfree_const(w->sname); - kfree(w); - return ERR_PTR(-ENOMEM); - } - switch (w->id) { case snd_soc_dapm_mic: w->is_ep = SND_SOC_DAPM_EP_SOURCE; @@ -3768,12 +3764,13 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, return w; request_failed: - if (ret != -EPROBE_DEFER) - dev_err(dapm->dev, "ASoC: Failed to request %s: %d\n", - w->name, ret); - + dev_err_probe(dapm->dev, ret, "ASoC: Failed to request %s\n", + w->name); + kfree_const(w->name); +name_failed: kfree_const(w->sname); kfree(w); +cnew_failed: return ERR_PTR(ret); } @@ -3844,6 +3841,15 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w, unsigned int fmt; int ret = 0; + /* + * NOTE + * + * snd_pcm_hw_params is quite large (608 bytes on arm64) and is + * starting to get a bit excessive for allocation on the stack, + * especially when you're building with some of the KASAN type + * stuff that increases stack usage. + * So, we use kzalloc()/kfree() for params in this function. + */ params = kzalloc(sizeof(*params), GFP_KERNEL); if (!params) return -ENOMEM; @@ -3886,23 +3892,22 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w, * necessary */ config = rtd->dai_link->params + rtd->params_select; - if (WARN_ON(!config)) { + if (!config) { dev_err(w->dapm->dev, "ASoC: link config missing\n"); ret = -EINVAL; goto out; } /* Be a little careful as we don't want to overflow the mask array */ - if (config->formats) { - fmt = ffs(config->formats) - 1; - } else { - dev_warn(w->dapm->dev, "ASoC: Invalid format %llx specified\n", - config->formats); + if (!config->formats) { + dev_warn(w->dapm->dev, "ASoC: Invalid format was specified\n"); ret = -EINVAL; goto out; } + fmt = ffs(config->formats) - 1; + snd_mask_set(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT), fmt); hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE)->min = config->rate_min; @@ -3941,7 +3946,9 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w, runtime->rate = params_rate(params); out: + /* see above NOTE */ kfree(params); + return ret; } @@ -4354,6 +4361,7 @@ static void dapm_connect_dai_pair(struct snd_soc_card *card, struct snd_soc_dapm_widget *dai, *codec, *playback_cpu, *capture_cpu; struct snd_pcm_substream *substream; struct snd_pcm_str *streams = rtd->pcm->streams; + int stream; if (dai_link->params) { playback_cpu = cpu_dai->capture_widget; @@ -4364,37 +4372,39 @@ static void dapm_connect_dai_pair(struct snd_soc_card *card, } /* connect BE DAI playback if widgets are valid */ + stream = SNDRV_PCM_STREAM_PLAYBACK; codec = codec_dai->playback_widget; if (playback_cpu && codec) { - if (dai_link->params && !rtd->playback_widget) { - substream = streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (dai_link->params && !rtd->c2c_widget[stream]) { + substream = streams[stream].substream; dai = snd_soc_dapm_new_dai(card, substream, "playback"); if (IS_ERR(dai)) goto capture; - rtd->playback_widget = dai; + rtd->c2c_widget[stream] = dai; } dapm_connect_dai_routes(&card->dapm, cpu_dai, playback_cpu, - rtd->playback_widget, + rtd->c2c_widget[stream], codec_dai, codec); } capture: /* connect BE DAI capture if widgets are valid */ + stream = SNDRV_PCM_STREAM_CAPTURE; codec = codec_dai->capture_widget; if (codec && capture_cpu) { - if (dai_link->params && !rtd->capture_widget) { - substream = streams[SNDRV_PCM_STREAM_CAPTURE].substream; + if (dai_link->params && !rtd->c2c_widget[stream]) { + substream = streams[stream].substream; dai = snd_soc_dapm_new_dai(card, substream, "capture"); if (IS_ERR(dai)) return; - rtd->capture_widget = dai; + rtd->c2c_widget[stream] = dai; } dapm_connect_dai_routes(&card->dapm, codec_dai, codec, - rtd->capture_widget, + rtd->c2c_widget[stream], cpu_dai, capture_cpu); } } @@ -4452,11 +4462,11 @@ void snd_soc_dapm_connect_dai_link_widgets(struct snd_soc_card *card) if (rtd->dai_link->dynamic) continue; - if (rtd->num_cpus == 1) { + if (rtd->dai_link->num_cpus == 1) { for_each_rtd_codec_dais(rtd, i, codec_dai) dapm_connect_dai_pair(card, rtd, codec_dai, asoc_rtd_to_cpu(rtd, 0)); - } else if (rtd->num_codecs == rtd->num_cpus) { + } else if (rtd->dai_link->num_codecs == rtd->dai_link->num_cpus) { for_each_rtd_codec_dais(rtd, i, codec_dai) dapm_connect_dai_pair(card, rtd, codec_dai, asoc_rtd_to_cpu(rtd, i)); diff --git a/sound/soc/soc-generic-dmaengine-pcm.c b/sound/soc/soc-generic-dmaengine-pcm.c index 87858462bba99697eed264f28ec117656e7c2421..3b99f619e37eb915c34b122e09e2fa66c196d93b 100644 --- a/sound/soc/soc-generic-dmaengine-pcm.c +++ b/sound/soc/soc-generic-dmaengine-pcm.c @@ -54,7 +54,7 @@ int snd_dmaengine_pcm_prepare_slave_config(struct snd_pcm_substream *substream, struct snd_dmaengine_dai_dma_data *dma_data; int ret; - if (rtd->num_cpus > 1) { + if (rtd->dai_link->num_cpus > 1) { dev_err(rtd->dev, "%s doesn't support Multi CPU yet\n", __func__); return -EINVAL; @@ -105,7 +105,7 @@ dmaengine_pcm_set_runtime_hwparams(struct snd_soc_component *component, struct snd_dmaengine_dai_dma_data *dma_data; struct snd_pcm_hardware hw; - if (rtd->num_cpus > 1) { + if (rtd->dai_link->num_cpus > 1) { dev_err(rtd->dev, "%s doesn't support Multi CPU yet\n", __func__); return -EINVAL; @@ -179,7 +179,7 @@ static struct dma_chan *dmaengine_pcm_compat_request_channel( struct dmaengine_pcm *pcm = soc_component_to_pcm(component); struct snd_dmaengine_dai_dma_data *dma_data; - if (rtd->num_cpus > 1) { + if (rtd->dai_link->num_cpus > 1) { dev_err(rtd->dev, "%s doesn't support Multi CPU yet\n", __func__); return NULL; diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index 4f60c0a833110fc41bc0424783660645c88cfb3e..fb87d6d2340853ecf8b1f4116a18be7f6a5cb9f8 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -27,6 +27,28 @@ #include <sound/soc-link.h> #include <sound/initval.h> +#define soc_pcm_ret(rtd, ret) _soc_pcm_ret(rtd, __func__, ret) +static inline int _soc_pcm_ret(struct snd_soc_pcm_runtime *rtd, + const char *func, int ret) +{ + /* Positive, Zero values are not errors */ + if (ret >= 0) + return ret; + + /* Negative values might be errors */ + switch (ret) { + case -EPROBE_DEFER: + case -ENOTSUPP: + break; + default: + dev_err(rtd->dev, + "ASoC: error at %s on %s: %d\n", + func, rtd->dai_link->name, ret); + } + + return ret; +} + static inline void snd_soc_dpcm_mutex_lock(struct snd_soc_pcm_runtime *rtd) { mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass); @@ -62,11 +84,11 @@ static inline void snd_soc_dpcm_stream_unlock_irq(struct snd_soc_pcm_runtime *rt static inline const char *soc_cpu_dai_name(struct snd_soc_pcm_runtime *rtd) { - return (rtd)->num_cpus == 1 ? asoc_rtd_to_cpu(rtd, 0)->name : "multicpu"; + return (rtd)->dai_link->num_cpus == 1 ? asoc_rtd_to_cpu(rtd, 0)->name : "multicpu"; } static inline const char *soc_codec_dai_name(struct snd_soc_pcm_runtime *rtd) { - return (rtd)->num_codecs == 1 ? asoc_rtd_to_codec(rtd, 0)->name : "multicodec"; + return (rtd)->dai_link->num_codecs == 1 ? asoc_rtd_to_codec(rtd, 0)->name : "multicodec"; } #ifdef CONFIG_DEBUG_FS @@ -163,7 +185,7 @@ static ssize_t dpcm_state_read_file(struct file *file, char __user *user_buf, int stream; char *buf; - if (fe->num_cpus > 1) { + if (fe->dai_link->num_cpus > 1) { dev_err(fe->dev, "%s doesn't support Multi CPU yet\n", __func__); return -EINVAL; @@ -615,7 +637,7 @@ int snd_soc_runtime_calc_hw(struct snd_soc_pcm_runtime *rtd, * connected to CPU DAI(s), use CPU DAI's directly and let * channel allocation be fixed up later */ - if (rtd->num_codecs > 1) { + if (rtd->dai_link->num_codecs > 1) { hw->channels_min = cpu_chan_min; hw->channels_max = cpu_chan_max; } @@ -723,7 +745,7 @@ static int soc_pcm_close(struct snd_pcm_substream *substream) struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); snd_soc_dpcm_mutex_lock(rtd); - soc_pcm_clean(rtd, substream, 0); + __soc_pcm_close(rtd, substream); snd_soc_dpcm_mutex_unlock(rtd); return 0; } @@ -832,12 +854,10 @@ dynamic: snd_soc_runtime_activate(rtd, substream->stream); ret = 0; err: - if (ret < 0) { + if (ret < 0) soc_pcm_clean(rtd, substream, 1); - dev_err(rtd->dev, "%s() failed (%d)", __func__, ret); - } - return ret; + return soc_pcm_ret(rtd, ret); } /* PCM open ops for non-DPCM streams */ @@ -852,16 +872,6 @@ static int soc_pcm_open(struct snd_pcm_substream *substream) return ret; } -static void codec2codec_close_delayed_work(struct snd_soc_pcm_runtime *rtd) -{ - /* - * Currently nothing to do for c2c links - * Since c2c links are internal nodes in the DAPM graph and - * don't interface with the outside world or application layer - * we don't have to do any special handling on close. - */ -} - /* * Called by ALSA when the PCM substream is prepared, can set format, sample * rate, etc. This function is non atomic and can be called multiple times, @@ -901,10 +911,7 @@ static int __soc_pcm_prepare(struct snd_soc_pcm_runtime *rtd, snd_soc_dai_digital_mute(dai, 0, substream->stream); out: - if (ret < 0) - dev_err(rtd->dev, "ASoC: %s() failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(rtd, ret); } /* PCM prepare ops for non-DPCM streams */ @@ -1070,12 +1077,10 @@ static int __soc_pcm_hw_params(struct snd_soc_pcm_runtime *rtd, ret = snd_soc_pcm_component_hw_params(substream, params); out: - if (ret < 0) { + if (ret < 0) soc_pcm_hw_clean(rtd, substream, 1); - dev_err(rtd->dev, "ASoC: %s() failed (%d)\n", __func__, ret); - } - return ret; + return soc_pcm_ret(rtd, ret); } /* hw_params PCM ops for non-DPCM streams */ @@ -1374,7 +1379,7 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe, struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(fe, 0); int paths; - if (fe->num_cpus > 1) { + if (fe->dai_link->num_cpus > 1) { dev_err(fe->dev, "%s doesn't support Multi CPU yet\n", __func__); return -EINVAL; @@ -1453,6 +1458,10 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, struct snd_soc_dapm_widget *widget; int i, new = 0, err; + /* don't connect if FE is not running */ + if (!fe->dpcm[stream].runtime && !fe->fe_compr) + return new; + /* Create any new FE <--> BE connections */ for_each_dapm_widgets(list, i, widget) { @@ -1477,10 +1486,6 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream, continue; } - /* don't connect if FE is not running */ - if (!fe->dpcm[stream].runtime && !fe->fe_compr) - continue; - /* * Filter for systems with 'component_chaining' enabled. * This helps to avoid unnecessary re-configuration of an @@ -1637,10 +1642,7 @@ int dpcm_be_dai_startup(struct snd_soc_pcm_runtime *fe, int stream) unwind: dpcm_be_dai_startup_rollback(fe, stream, dpcm); - dev_err(fe->dev, "ASoC: %s() failed at %s (%d)\n", - __func__, be->dai_link->name, err); - - return err; + return soc_pcm_ret(fe, err); } static void dpcm_runtime_setup_fe(struct snd_pcm_substream *substream) @@ -1749,7 +1751,7 @@ static void dpcm_runtime_setup_be_chan(struct snd_pcm_substream *substream) * chan min/max cannot be enforced if there are multiple CODEC * DAIs connected to a single CPU DAI, use CPU DAI's directly */ - if (be->num_codecs == 1) { + if (be->dai_link->num_codecs == 1) { struct snd_soc_pcm_stream *codec_stream = snd_soc_dai_get_pcm_stream( asoc_rtd_to_codec(be, 0), stream); @@ -1840,10 +1842,7 @@ static int dpcm_apply_symmetry(struct snd_pcm_substream *fe_substream, } } error: - if (err < 0) - dev_err(fe->dev, "ASoC: %s failed (%d)\n", __func__, err); - - return err; + return soc_pcm_ret(fe, err); } static int dpcm_fe_dai_startup(struct snd_pcm_substream *fe_substream) @@ -1880,10 +1879,7 @@ unwind: be_err: dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); - if (ret < 0) - dev_err(fe->dev, "%s() failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(fe, ret); } static int dpcm_fe_dai_shutdown(struct snd_pcm_substream *substream) @@ -2082,10 +2078,7 @@ out: dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); snd_soc_dpcm_mutex_unlock(fe); - if (ret < 0) - dev_err(fe->dev, "ASoC: %s failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(fe, ret); } int dpcm_be_dai_trigger(struct snd_soc_pcm_runtime *fe, int stream, @@ -2254,10 +2247,7 @@ next: if (ret) break; } - if (ret < 0) - dev_err(fe->dev, "ASoC: %s() failed at %s (%d)\n", - __func__, be->dai_link->name, ret); - return ret; + return soc_pcm_ret(fe, ret); } EXPORT_SYMBOL_GPL(dpcm_be_dai_trigger); @@ -2428,10 +2418,7 @@ int dpcm_be_dai_prepare(struct snd_soc_pcm_runtime *fe, int stream) be->dpcm[stream].state = SND_SOC_DPCM_STATE_PREPARE; } - if (ret < 0) - dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(fe, ret); } static int dpcm_fe_dai_prepare(struct snd_pcm_substream *substream) @@ -2468,10 +2455,7 @@ out: dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO); snd_soc_dpcm_mutex_unlock(fe); - if (ret < 0) - dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(fe, ret); } static int dpcm_run_update_shutdown(struct snd_soc_pcm_runtime *fe, int stream) @@ -2504,10 +2488,7 @@ static int dpcm_run_update_shutdown(struct snd_soc_pcm_runtime *fe, int stream) /* run the stream event for each BE */ dpcm_dapm_stream_event(fe, stream, SND_SOC_DAPM_STREAM_NOP); - if (err < 0) - dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, err); - - return err; + return soc_pcm_ret(fe, err); } static int dpcm_run_update_startup(struct snd_soc_pcm_runtime *fe, int stream) @@ -2597,10 +2578,7 @@ disconnect: dpcm->state = SND_SOC_DPCM_LINK_STATE_FREE; } - if (ret < 0) - dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret); - - return ret; + return soc_pcm_ret(fe, ret); } static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new) @@ -2612,7 +2590,7 @@ static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new) if (!fe->dai_link->dynamic) return 0; - if (fe->num_cpus > 1) { + if (fe->dai_link->num_cpus > 1) { dev_err(fe->dev, "%s doesn't support Multi CPU yet\n", __func__); return -EINVAL; @@ -2756,7 +2734,7 @@ static int soc_get_playback_capture(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *cpu_dai; int i; - if (rtd->dai_link->dynamic && rtd->num_cpus > 1) { + if (rtd->dai_link->dynamic && rtd->dai_link->num_cpus > 1) { dev_err(rtd->dev, "DPCM doesn't support Multi CPU for Front-Ends yet\n"); return -EINVAL; @@ -2808,9 +2786,9 @@ static int soc_get_playback_capture(struct snd_soc_pcm_runtime *rtd, SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK; for_each_rtd_codec_dais(rtd, i, codec_dai) { - if (rtd->num_cpus == 1) { + if (rtd->dai_link->num_cpus == 1) { cpu_dai = asoc_rtd_to_cpu(rtd, 0); - } else if (rtd->num_cpus == rtd->num_codecs) { + } else if (rtd->dai_link->num_cpus == rtd->dai_link->num_codecs) { cpu_dai = asoc_rtd_to_cpu(rtd, i); } else { dev_err(rtd->card->dev, @@ -2899,14 +2877,19 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) return ret; /* DAPM dai link stream work */ - if (rtd->dai_link->params) - rtd->close_delayed_work_func = codec2codec_close_delayed_work; - else + /* + * Currently nothing to do for c2c links + * Since c2c links are internal nodes in the DAPM graph and + * don't interface with the outside world or application layer + * we don't have to do any special handling on close. + */ + if (!rtd->dai_link->params) rtd->close_delayed_work_func = snd_soc_close_delayed_work; rtd->pcm = pcm; pcm->nonatomic = rtd->dai_link->nonatomic; pcm->private_data = rtd; + pcm->no_device_suspend = true; if (rtd->dai_link->no_pcm || rtd->dai_link->params) { if (playback) @@ -2961,8 +2944,6 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) ret = snd_soc_pcm_component_new(rtd); if (ret < 0) return ret; - - pcm->no_device_suspend = true; out: dev_dbg(rtd->card->dev, "%s <-> %s mapping ok\n", soc_codec_dai_name(rtd), soc_cpu_dai_name(rtd)); diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c index b101db85446ff02a9fca8c4c322dc8924618485f..c3be24b2fac5560a94293ef4f83872b0704b64ff 100644 --- a/sound/soc/soc-topology.c +++ b/sound/soc/soc-topology.c @@ -1755,6 +1755,7 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg, /* enable DPCM */ link->dynamic = 1; + link->ignore_pmdown_time = 1; link->dpcm_playback = le32_to_cpu(pcm->playback); link->dpcm_capture = le32_to_cpu(pcm->capture); if (pcm->flag_mask) diff --git a/sound/soc/soc-utils-test.c b/sound/soc/soc-utils-test.c index 5ad8e23af49ade31bd2f9a9541117e0c483062e8..616d2c926dd181f10733ce5e274953e45761fe4c 100644 --- a/sound/soc/soc-utils-test.c +++ b/sound/soc/soc-utils-test.c @@ -170,8 +170,54 @@ static void test_tdm_params_to_bclk(struct kunit *test) } } +static void test_snd_soc_params_to_bclk_one(struct kunit *test, + unsigned int rate, snd_pcm_format_t fmt, + unsigned int channels, + unsigned int expected_bclk) +{ + struct snd_pcm_hw_params params; + int got_bclk; + + _snd_pcm_hw_params_any(¶ms); + snd_mask_none(hw_param_mask(¶ms, SNDRV_PCM_HW_PARAM_FORMAT)); + hw_param_interval(¶ms, SNDRV_PCM_HW_PARAM_RATE)->min = rate; + hw_param_interval(¶ms, SNDRV_PCM_HW_PARAM_RATE)->max = rate; + hw_param_interval(¶ms, SNDRV_PCM_HW_PARAM_CHANNELS)->min = channels; + hw_param_interval(¶ms, SNDRV_PCM_HW_PARAM_CHANNELS)->max = channels; + params_set_format(¶ms, fmt); + + got_bclk = snd_soc_params_to_bclk(¶ms); + pr_debug("%s: r=%u sb=%u ch=%u expected=%u got=%d\n", + __func__, + rate, params_width(¶ms), channels, expected_bclk, got_bclk); + KUNIT_ASSERT_EQ(test, expected_bclk, (unsigned int)got_bclk); +} + +static void test_snd_soc_params_to_bclk(struct kunit *test) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tdm_params_to_bclk_cases); ++i) { + /* + * snd_soc_params_to_bclk() is all the test cases where + * snd_pcm_hw_params values are not overridden. + */ + if (tdm_params_to_bclk_cases[i].tdm_width | + tdm_params_to_bclk_cases[i].tdm_slots | + tdm_params_to_bclk_cases[i].slot_multiple) + continue; + + test_snd_soc_params_to_bclk_one(test, + tdm_params_to_bclk_cases[i].rate, + tdm_params_to_bclk_cases[i].fmt, + tdm_params_to_bclk_cases[i].channels, + tdm_params_to_bclk_cases[i].bclk); + } +} + static struct kunit_case soc_utils_test_cases[] = { KUNIT_CASE(test_tdm_params_to_bclk), + KUNIT_CASE(test_snd_soc_params_to_bclk), {} }; diff --git a/sound/soc/soc-utils.c b/sound/soc/soc-utils.c index 70c380c0ac7b69b2d022bd35ce087ebc0df51d81..a3b6df2378b40476b99c186ccf1961c3015fdb72 100644 --- a/sound/soc/soc-utils.c +++ b/sound/soc/soc-utils.c @@ -56,23 +56,24 @@ EXPORT_SYMBOL_GPL(snd_soc_params_to_bclk); /** * snd_soc_tdm_params_to_bclk - calculate bclk from params and tdm slot info. * - * Calculate the bclk from the params sample rate and the tdm slot count and - * tdm slot width. Either or both of tdm_width and tdm_slots can be 0. + * Calculate the bclk from the params sample rate, the tdm slot count and the + * tdm slot width. Optionally round-up the slot count to a given multiple. + * Either or both of tdm_width and tdm_slots can be 0. * - * If tdm_width == 0 and tdm_slots > 0: the params_width will be used. - * If tdm_width > 0 and tdm_slots == 0: the params_channels will be used - * as the slot count. - * Both tdm_width and tdm_slots are 0: this is equivalent to calling - * snd_soc_params_to_bclk(). + * If tdm_width == 0: use params_width() as the slot width. + * If tdm_slots == 0: use params_channels() as the slot count. * - * If slot_multiple > 1 the slot count (or params_channels if tdm_slots == 0) - * will be rounded up to a multiple of this value. This is mainly useful for + * If slot_multiple > 1 the slot count (or params_channels() if tdm_slots == 0) + * will be rounded up to a multiple of slot_multiple. This is mainly useful for * I2S mode, which has a left and right phase so the number of slots is always * a multiple of 2. * + * If tdm_width == 0 && tdm_slots == 0 && slot_multiple < 2, this is equivalent + * to calling snd_soc_params_to_bclk(). + * * @params: Pointer to struct_pcm_hw_params. - * @tdm_width: Width in bits of the tdm slots. - * @tdm_slots: Number of tdm slots per frame. + * @tdm_width: Width in bits of the tdm slots. Must be >= 0. + * @tdm_slots: Number of tdm slots per frame. Must be >= 0. * @slot_multiple: If >1 roundup slot count to a multiple of this value. * * Return: bclk frequency in Hz, else a negative error code if params format diff --git a/sound/soc/sof/Kconfig b/sound/soc/sof/Kconfig index e90f173d067c9aba54b4e8e99933eca47913b003..37f7df5fde175cef88525ba40f617020ae83cf56 100644 --- a/sound/soc/sof/Kconfig +++ b/sound/soc/sof/Kconfig @@ -196,6 +196,7 @@ config SND_SOC_SOF_DEBUG_ENABLE_FIRMWARE_TRACE config SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST tristate "SOF enable IPC flood test" + depends on SND_SOC_SOF select SND_SOC_SOF_CLIENT help This option enables a separate client device for IPC flood test @@ -214,6 +215,7 @@ config SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST_NUM config SND_SOC_SOF_DEBUG_IPC_MSG_INJECTOR tristate "SOF enable IPC message injector" + depends on SND_SOC_SOF select SND_SOC_SOF_CLIENT help This option enables the IPC message injector which can be used to send diff --git a/sound/soc/sof/Makefile b/sound/soc/sof/Makefile index 9a74ed116ed92a2263aaeed7ab0ab165428e1e88..eab7cc53f71a437ada5b506c2b9508208bc1cbdb 100644 --- a/sound/soc/sof/Makefile +++ b/sound/soc/sof/Makefile @@ -9,7 +9,8 @@ snd-sof-objs += ipc3.o ipc3-loader.o ipc3-topology.o ipc3-control.o ipc3-pcm.o\ ipc3-dtrace.o endif ifneq ($(CONFIG_SND_SOC_SOF_INTEL_IPC4),) -snd-sof-objs += ipc4.o ipc4-loader.o ipc4-topology.o ipc4-control.o ipc4-pcm.o +snd-sof-objs += ipc4.o ipc4-loader.o ipc4-topology.o ipc4-control.o ipc4-pcm.o\ + ipc4-mtrace.o endif # SOF client support diff --git a/sound/soc/sof/amd/Kconfig b/sound/soc/sof/amd/Kconfig index 190c85d57047bfc54fc07f26bd5c21e5e3e401d0..a305ea6efea9617b6752a4ef538efe8bd154c329 100644 --- a/sound/soc/sof/amd/Kconfig +++ b/sound/soc/sof/amd/Kconfig @@ -31,4 +31,14 @@ config SND_SOC_SOF_AMD_RENOIR select SND_SOC_SOF_AMD_COMMON help Select this option for SOF support on AMD Renoir platform + +config SND_SOC_SOF_AMD_REMBRANDT + tristate "SOF support for REMBRANDT" + depends on SND_SOC_SOF_PCI + select SND_SOC_SOF_AMD_COMMON + help + Select this option for SOF support on AMD Rembrandt platform + Say Y if you want to enable SOF on Rembrandt. + If unsure select "N". + endif diff --git a/sound/soc/sof/amd/Makefile b/sound/soc/sof/amd/Makefile index 7b9f1a0af3c87e9ff3bc0315ad951c74ee9d6f69..5626d13b3e69c4136c457211123d62b2d089bf6a 100644 --- a/sound/soc/sof/amd/Makefile +++ b/sound/soc/sof/amd/Makefile @@ -4,8 +4,10 @@ # # Copyright(c) 2021 Advanced Micro Devices, Inc. All rights reserved. -snd-sof-amd-acp-objs := acp.o acp-loader.o acp-ipc.o acp-pcm.o acp-stream.o acp-trace.o +snd-sof-amd-acp-objs := acp.o acp-loader.o acp-ipc.o acp-pcm.o acp-stream.o acp-trace.o acp-common.o snd-sof-amd-renoir-objs := pci-rn.o renoir.o +snd-sof-amd-rembrandt-objs := pci-rmb.o rembrandt.o obj-$(CONFIG_SND_SOC_SOF_AMD_COMMON) += snd-sof-amd-acp.o obj-$(CONFIG_SND_SOC_SOF_AMD_RENOIR) +=snd-sof-amd-renoir.o +obj-$(CONFIG_SND_SOC_SOF_AMD_REMBRANDT) +=snd-sof-amd-rembrandt.o diff --git a/sound/soc/sof/amd/acp-common.c b/sound/soc/sof/amd/acp-common.c new file mode 100644 index 0000000000000000000000000000000000000000..27b95187356e5f1d6c1ea90fc3cb2b4536ea9d96 --- /dev/null +++ b/sound/soc/sof/amd/acp-common.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2022 Advanced Micro Devices, Inc. +// +// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com> +// V sujith kumar Reddy <Vsujithkumar.Reddy@amd.com> + +/* ACP-specific Common code */ + +#include "../sof-priv.h" +#include "../sof-audio.h" +#include "../ops.h" +#include "../sof-audio.h" +#include "acp.h" +#include "acp-dsp-offset.h" + +int acp_dai_probe(struct snd_soc_dai *dai) +{ + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component); + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int val; + + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->i2s_pin_config_offset); + if (val != desc->i2s_mode) { + dev_err(sdev->dev, "I2S Mode is not supported (I2S_PIN_CONFIG: %#x)\n", val); + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_NS(acp_dai_probe, SND_SOC_SOF_AMD_COMMON); + +struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev) +{ + struct snd_sof_pdata *sof_pdata = sdev->pdata; + const struct sof_dev_desc *desc = sof_pdata->desc; + struct snd_soc_acpi_mach *mach; + + mach = snd_soc_acpi_find_machine(desc->machines); + if (!mach) { + dev_warn(sdev->dev, "No matching ASoC machine driver found\n"); + return NULL; + } + + sof_pdata->tplg_filename = mach->sof_tplg_filename; + sof_pdata->fw_filename = mach->fw_filename; + + return mach; +} + +/* AMD Common DSP ops */ +struct snd_sof_dsp_ops sof_acp_common_ops = { + /* probe and remove */ + .probe = amd_sof_acp_probe, + .remove = amd_sof_acp_remove, + + /* Register IO */ + .write = sof_io_write, + .read = sof_io_read, + + /* Block IO */ + .block_read = acp_dsp_block_read, + .block_write = acp_dsp_block_write, + + /*Firmware loading */ + .load_firmware = snd_sof_load_firmware_memcpy, + .pre_fw_run = acp_dsp_pre_fw_run, + .get_bar_index = acp_get_bar_index, + + /* DSP core boot */ + .run = acp_sof_dsp_run, + + /*IPC */ + .send_msg = acp_sof_ipc_send_msg, + .ipc_msg_data = acp_sof_ipc_msg_data, + .get_mailbox_offset = acp_sof_ipc_get_mailbox_offset, + .get_window_offset = acp_sof_ipc_get_window_offset, + .irq_thread = acp_sof_ipc_irq_thread, + + /* stream callbacks */ + .pcm_open = acp_pcm_open, + .pcm_close = acp_pcm_close, + .pcm_hw_params = acp_pcm_hw_params, + + .hw_info = SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_PAUSE | + SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, + + /* Machine driver callbacks */ + .machine_select = amd_sof_machine_select, + .machine_register = sof_machine_register, + .machine_unregister = sof_machine_unregister, + + /* Trace Logger */ + .trace_init = acp_sof_trace_init, + .trace_release = acp_sof_trace_release, + + /* PM */ + .suspend = amd_sof_acp_suspend, + .resume = amd_sof_acp_resume, +}; +EXPORT_SYMBOL_NS(sof_acp_common_ops, SND_SOC_SOF_AMD_COMMON); + +MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON); +MODULE_DESCRIPTION("ACP SOF COMMON Driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/amd/acp-dsp-offset.h b/sound/soc/sof/amd/acp-dsp-offset.h index 56cefd4a84fcdf3f624304d4239c0edeb984fffb..de5726251dc6102e3714b7b649240a27c071cf60 100644 --- a/sound/soc/sof/amd/acp-dsp-offset.h +++ b/sound/soc/sof/amd/acp-dsp-offset.h @@ -48,22 +48,29 @@ #define ACP_SOFT_RESET 0x1000 #define ACP_CONTROL 0x1004 -#define ACP_I2S_PIN_CONFIG 0x1400 +#define ACP3X_I2S_PIN_CONFIG 0x1400 +#define ACP6X_I2S_PIN_CONFIG 0x1440 -/* Registers from ACP_PGFSM block */ -#define ACP_PGFSM_CONTROL 0x141C -#define ACP_PGFSM_STATUS 0x1420 -#define ACP_CLKMUX_SEL 0x1424 +/* Registers offsets from ACP_PGFSM block */ +#define ACP3X_PGFSM_BASE 0x141C +#define ACP6X_PGFSM_BASE 0x1024 +#define PGFSM_CONTROL_OFFSET 0x0 +#define PGFSM_STATUS_OFFSET 0x4 +#define ACP3X_CLKMUX_SEL 0x1424 +#define ACP6X_CLKMUX_SEL 0x102C /* Registers from ACP_INTR block */ -#define ACP_EXTERNAL_INTR_ENB 0x1800 -#define ACP_EXTERNAL_INTR_CNTL 0x1804 -#define ACP_EXTERNAL_INTR_STAT 0x1808 -#define ACP_DSP_SW_INTR_CNTL 0x1814 -#define ACP_DSP_SW_INTR_STAT 0x1818 -#define ACP_SW_INTR_TRIG 0x181C +#define ACP3X_EXT_INTR_STAT 0x1808 +#define ACP6X_EXT_INTR_STAT 0x1A0C + +#define ACP3X_DSP_SW_INTR_BASE 0x1814 +#define ACP6X_DSP_SW_INTR_BASE 0x1808 +#define DSP_SW_INTR_CNTL_OFFSET 0x0 +#define DSP_SW_INTR_STAT_OFFSET 0x4 +#define DSP_SW_INTR_TRIG_OFFSET 0x8 #define ACP_ERROR_STATUS 0x18C4 -#define ACP_AXI2DAGB_SEM_0 0x1880 +#define ACP3X_AXI2DAGB_SEM_0 0x1880 +#define ACP6X_AXI2DAGB_SEM_0 0x1874 /* Registers from ACP_SHA block */ #define ACP_SHA_DSP_FW_QUALIFIER 0x1C70 @@ -77,5 +84,5 @@ #define ACP_SHA_PSP_ACK 0x1C74 #define ACP_SCRATCH_REG_0 0x10000 - +#define ACP6X_DSP_FUSION_RUNSTALL 0x0644 #endif diff --git a/sound/soc/sof/amd/acp-ipc.c b/sound/soc/sof/amd/acp-ipc.c index e1842f03708336ec895dd0b4402d1d9149e14048..dd030566e37259912cc4b8815753fd2f2819d5a8 100644 --- a/sound/soc/sof/amd/acp-ipc.c +++ b/sound/soc/sof/amd/acp-ipc.c @@ -30,30 +30,36 @@ EXPORT_SYMBOL_NS(acp_mailbox_read, SND_SOC_SOF_AMD_COMMON); static void acpbus_trigger_host_to_dsp_swintr(struct acp_dev_data *adata) { struct snd_sof_dev *sdev = adata->dev; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); u32 swintr_trigger; - swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_SW_INTR_TRIG); + swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->dsp_intr_base + + DSP_SW_INTR_TRIG_OFFSET); swintr_trigger |= 0x01; - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SW_INTR_TRIG, swintr_trigger); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_TRIG_OFFSET, + swintr_trigger); } static void acp_ipc_host_msg_set(struct snd_sof_dev *sdev) { - unsigned int host_msg = offsetof(struct scratch_ipc_conf, sof_host_msg_write); + unsigned int host_msg = sdev->debug_box.offset + + offsetof(struct scratch_ipc_conf, sof_host_msg_write); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + host_msg, 1); } static void acp_dsp_ipc_host_done(struct snd_sof_dev *sdev) { - unsigned int dsp_msg = offsetof(struct scratch_ipc_conf, sof_dsp_msg_write); + unsigned int dsp_msg = sdev->debug_box.offset + + offsetof(struct scratch_ipc_conf, sof_dsp_msg_write); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_msg, 0); } static void acp_dsp_ipc_dsp_done(struct snd_sof_dev *sdev) { - unsigned int dsp_ack = offsetof(struct scratch_ipc_conf, sof_dsp_ack_write); + unsigned int dsp_ack = sdev->debug_box.offset + + offsetof(struct scratch_ipc_conf, sof_dsp_ack_write); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_ack, 0); } @@ -61,10 +67,11 @@ static void acp_dsp_ipc_dsp_done(struct snd_sof_dev *sdev) int acp_sof_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) { struct acp_dev_data *adata = sdev->pdata->hw_pdata; - unsigned int offset = offsetof(struct scratch_ipc_conf, sof_in_box); + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int offset = sdev->host_box.offset; unsigned int count = ACP_HW_SEM_RETRY_COUNT; - while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0)) { + while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset)) { /* Wait until acquired HW Semaphore Lock or timeout*/ count--; if (!count) { @@ -80,7 +87,7 @@ int acp_sof_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) acpbus_trigger_host_to_dsp_swintr(adata); /* Unlock or Release HW Semaphore */ - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0, 0x0); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset, 0x0); return 0; } @@ -91,7 +98,7 @@ static void acp_dsp_ipc_get_reply(struct snd_sof_dev *sdev) struct snd_sof_ipc_msg *msg = sdev->msg; struct sof_ipc_reply reply; struct sof_ipc_cmd_hdr *hdr; - unsigned int offset = offsetof(struct scratch_ipc_conf, sof_in_box); + unsigned int offset = sdev->host_box.offset; int ret = 0; /* @@ -141,11 +148,19 @@ out: irqreturn_t acp_sof_ipc_irq_thread(int irq, void *context) { struct snd_sof_dev *sdev = context; - unsigned int dsp_msg_write = offsetof(struct scratch_ipc_conf, sof_dsp_msg_write); - unsigned int dsp_ack_write = offsetof(struct scratch_ipc_conf, sof_dsp_ack_write); + unsigned int dsp_msg_write = sdev->debug_box.offset + + offsetof(struct scratch_ipc_conf, sof_dsp_msg_write); + unsigned int dsp_ack_write = sdev->debug_box.offset + + offsetof(struct scratch_ipc_conf, sof_dsp_ack_write); bool ipc_irq = false; int dsp_msg, dsp_ack; + if (sdev->first_boot && sdev->fw_state != SOF_FW_BOOT_COMPLETE) { + snd_sof_ipc_msgs_rx(sdev); + acp_dsp_ipc_host_done(sdev); + return IRQ_HANDLED; + } + dsp_msg = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_msg_write); if (dsp_msg) { snd_sof_ipc_msgs_rx(sdev); @@ -175,7 +190,7 @@ EXPORT_SYMBOL_NS(acp_sof_ipc_irq_thread, SND_SOC_SOF_AMD_COMMON); int acp_sof_ipc_msg_data(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, void *p, size_t sz) { - unsigned int offset = offsetof(struct scratch_ipc_conf, sof_out_box); + unsigned int offset = sdev->dsp_box.offset; if (!substream || !sdev->stream_box.size) acp_mailbox_read(sdev, offset, p, sz); @@ -186,8 +201,16 @@ EXPORT_SYMBOL_NS(acp_sof_ipc_msg_data, SND_SOC_SOF_AMD_COMMON); int acp_sof_ipc_get_mailbox_offset(struct snd_sof_dev *sdev) { - return ACP_SCRATCH_MEMORY_ADDRESS; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + + return desc->sram_pte_offset; } EXPORT_SYMBOL_NS(acp_sof_ipc_get_mailbox_offset, SND_SOC_SOF_AMD_COMMON); +int acp_sof_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id) +{ + return 0; +} +EXPORT_SYMBOL_NS(acp_sof_ipc_get_window_offset, SND_SOC_SOF_AMD_COMMON); + MODULE_DESCRIPTION("AMD ACP sof-ipc driver"); diff --git a/sound/soc/sof/amd/acp-loader.c b/sound/soc/sof/amd/acp-loader.c index 7ca51e0f3b1b19bedf0be5b11913ce6b0213fc99..d1e74baf5d8bcaa754f37e705ce5fbe97eef8705 100644 --- a/sound/soc/sof/amd/acp-loader.c +++ b/sound/soc/sof/amd/acp-loader.c @@ -30,9 +30,10 @@ int acp_dsp_block_read(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_type, u32 offset, void *dest, size_t size) { + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); switch (blk_type) { case SOF_FW_BLK_TYPE_SRAM: - offset = offset - ACP_SCRATCH_MEMORY_ADDRESS; + offset = offset - desc->sram_pte_offset; memcpy_from_scratch(sdev, offset, dest, size); break; default: @@ -49,6 +50,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t { struct snd_sof_pdata *plat_data = sdev->pdata; struct pci_dev *pci = to_pci_dev(sdev->dev); + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); struct acp_dev_data *adata; void *dest; u32 dma_size, page_count; @@ -84,7 +86,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t adata->fw_data_bin_size = size + offset; break; case SOF_FW_BLK_TYPE_SRAM: - offset = offset - ACP_SCRATCH_MEMORY_ADDRESS; + offset = offset - desc->sram_pte_offset; memcpy_to_scratch(sdev, offset, src, size); return 0; default: @@ -105,14 +107,13 @@ EXPORT_SYMBOL_NS(acp_get_bar_index, SND_SOC_SOF_AMD_COMMON); static void configure_pte_for_fw_loading(int type, int num_pages, struct acp_dev_data *adata) { - struct snd_sof_dev *sdev; + struct snd_sof_dev *sdev = adata->dev; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); unsigned int low, high; dma_addr_t addr; u16 page_idx; u32 offset; - sdev = adata->dev; - switch (type) { case FW_BIN: offset = FW_BIN_PTE_OFFSET; @@ -129,7 +130,7 @@ static void configure_pte_for_fw_loading(int type, int num_pages, struct acp_dev /* Group Enable */ snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACPAXI2AXI_ATU_BASE_ADDR_GRP_1, - ACP_SRAM_PTE_OFFSET | BIT(31)); + desc->sram_pte_offset | BIT(31)); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1, PAGE_SIZE_4K_ENABLE); @@ -197,12 +198,19 @@ EXPORT_SYMBOL_NS(acp_dsp_pre_fw_run, SND_SOC_SOF_AMD_COMMON); int acp_sof_dsp_run(struct snd_sof_dev *sdev) { + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); int val; snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DSP0_RUNSTALL, ACP_DSP_RUN); val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP0_RUNSTALL); dev_dbg(sdev->dev, "ACP_DSP0_RUNSTALL : 0x%0x\n", val); + /* Some platforms won't support fusion DSP,keep offset zero for no support */ + if (desc->fusion_dsp_offset) { + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->fusion_dsp_offset, ACP_DSP_RUN); + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->fusion_dsp_offset); + dev_dbg(sdev->dev, "ACP_DSP0_FUSION_RUNSTALL : 0x%0x\n", val); + } return 0; } EXPORT_SYMBOL_NS(acp_sof_dsp_run, SND_SOC_SOF_AMD_COMMON); diff --git a/sound/soc/sof/amd/acp-pcm.c b/sound/soc/sof/amd/acp-pcm.c index 0ba8ae46bd76b1b5aa1eedf05e63f7d08850d4c9..727c3a784a204f105c185c704b23c7c95933c9a2 100644 --- a/sound/soc/sof/amd/acp-pcm.c +++ b/sound/soc/sof/amd/acp-pcm.c @@ -42,7 +42,8 @@ int acp_pcm_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substr /* write buffer size of stream in scratch memory */ - buf_offset = offsetof(struct scratch_reg_conf, buf_size); + buf_offset = sdev->debug_box.offset + + offsetof(struct scratch_reg_conf, buf_size); index = stream->stream_tag - 1; buf_offset = buf_offset + index * 4; diff --git a/sound/soc/sof/amd/acp-stream.c b/sound/soc/sof/amd/acp-stream.c index b3ca4a90dbf8039b72e75afb5aeacc6aeb856310..6f40ef7ba85ec26330e2ccb0a66c35e5776d3e68 100644 --- a/sound/soc/sof/amd/acp-stream.c +++ b/sound/soc/sof/amd/acp-stream.c @@ -26,6 +26,7 @@ int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *stream) { + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); unsigned int pte_reg, pte_size, phy_addr_offset, index; int stream_tag = stream->stream_tag; u32 low, high, offset, reg_val; @@ -88,7 +89,8 @@ int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *strea /* write phy_addr in scratch memory */ - phy_addr_offset = offsetof(struct scratch_reg_conf, reg_offset); + phy_addr_offset = sdev->debug_box.offset + + offsetof(struct scratch_reg_conf, reg_offset); index = stream_tag - 1; phy_addr_offset = phy_addr_offset + index * 4; @@ -96,7 +98,8 @@ int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *strea phy_addr_offset, stream->reg_offset); /* Group Enable */ - reg_val = ACP_SRAM_PTE_OFFSET + offset; + offset = offset + sdev->debug_box.offset; + reg_val = desc->sram_pte_offset + offset; snd_sof_dsp_write(sdev, ACP_DSP_BAR, pte_reg, reg_val | BIT(31)); snd_sof_dsp_write(sdev, ACP_DSP_BAR, pte_size, PAGE_SIZE_4K_ENABLE); diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c index c40d2900dd36d33cb05ce51d91f87942e1c77116..36966643e36ab5d37a08f7b20b5c4cbed653eb8b 100644 --- a/sound/soc/sof/amd/acp.c +++ b/sound/soc/sof/amd/acp.c @@ -39,9 +39,11 @@ static int smn_read(struct pci_dev *dev, u32 smn_addr, u32 *data) static void init_dma_descriptor(struct acp_dev_data *adata) { struct snd_sof_dev *sdev = adata->dev; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); unsigned int addr; - addr = ACP_SRAM_PTE_OFFSET + offsetof(struct scratch_reg_conf, dma_desc); + addr = desc->sram_pte_offset + sdev->debug_box.offset + + offsetof(struct scratch_reg_conf, dma_desc); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DMA_DESC_BASE_ADDR, addr); snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DMA_DESC_MAX_NUM_DSCR, ACP_MAX_DESC_CNT); @@ -53,8 +55,9 @@ static void configure_dma_descriptor(struct acp_dev_data *adata, unsigned short struct snd_sof_dev *sdev = adata->dev; unsigned int offset; - offset = ACP_SCRATCH_REG_0 + offsetof(struct scratch_reg_conf, dma_desc) + - idx * sizeof(struct dma_descriptor); + offset = ACP_SCRATCH_REG_0 + sdev->debug_box.offset + + offsetof(struct scratch_reg_conf, dma_desc) + + idx * sizeof(struct dma_descriptor); snd_sof_dsp_write(sdev, ACP_DSP_BAR, offset, dscr_info->src_addr); snd_sof_dsp_write(sdev, ACP_DSP_BAR, offset + 0x4, dscr_info->dest_addr); @@ -300,8 +303,9 @@ void memcpy_to_scratch(struct snd_sof_dev *sdev, u32 offset, unsigned int *src, static int acp_memory_init(struct snd_sof_dev *sdev) { struct acp_dev_data *adata = sdev->pdata->hw_pdata; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); - snd_sof_dsp_update_bits(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_CNTL, + snd_sof_dsp_update_bits(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_CNTL_OFFSET, ACP_DSP_INTR_EN_MASK, ACP_DSP_INTR_EN_MASK); init_dma_descriptor(adata); @@ -311,18 +315,20 @@ static int acp_memory_init(struct snd_sof_dev *sdev) static irqreturn_t acp_irq_thread(int irq, void *context) { struct snd_sof_dev *sdev = context; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int base = desc->dsp_intr_base; unsigned int val, count = ACP_HW_SEM_RETRY_COUNT; - val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_EXTERNAL_INTR_STAT); + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->ext_intr_stat); if (val & ACP_SHA_STAT) { /* Clear SHA interrupt raised by PSP */ - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_EXTERNAL_INTR_STAT, val); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, val); return IRQ_HANDLED; } - val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT); + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET); if (val & ACP_DSP_TO_HOST_IRQ) { - while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0)) { + while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset)) { /* Wait until acquired HW Semaphore lock or timeout */ count--; if (!count) { @@ -333,10 +339,10 @@ static irqreturn_t acp_irq_thread(int irq, void *context) sof_ops(sdev)->irq_thread(irq, sdev); val |= ACP_DSP_TO_HOST_IRQ; - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT, val); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET, val); /* Unlock or Release HW Semaphore */ - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0, 0x0); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset, 0x0); return IRQ_HANDLED; } @@ -347,9 +353,11 @@ static irqreturn_t acp_irq_thread(int irq, void *context) static irqreturn_t acp_irq_handler(int irq, void *dev_id) { struct snd_sof_dev *sdev = dev_id; + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int base = desc->dsp_intr_base; unsigned int val; - val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT); + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET); if (val) return IRQ_WAKE_THREAD; @@ -358,20 +366,22 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id) static int acp_power_on(struct snd_sof_dev *sdev) { + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); + unsigned int base = desc->pgfsm_base; unsigned int val; int ret; - val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_PGFSM_STATUS); + val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET); if (val == ACP_POWERED_ON) return 0; if (val & ACP_PGFSM_STATUS_MASK) - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_PGFSM_CONTROL, + snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + PGFSM_CONTROL_OFFSET, ACP_PGFSM_CNTL_POWER_ON_MASK); - ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, ACP_PGFSM_STATUS, val, !val, - ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US); + ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET, val, + !val, ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US); if (ret < 0) dev_err(sdev->dev, "timeout in ACP_PGFSM_STATUS read\n"); @@ -437,6 +447,7 @@ EXPORT_SYMBOL_NS(amd_sof_acp_suspend, SND_SOC_SOF_AMD_COMMON); int amd_sof_acp_resume(struct snd_sof_dev *sdev) { + const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata); int ret; ret = acp_init(sdev); @@ -445,7 +456,7 @@ int amd_sof_acp_resume(struct snd_sof_dev *sdev) return ret; } - snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CLKMUX_SEL, 0x03); + snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_clkmux_sel, 0x03); ret = acp_memory_init(sdev); @@ -507,6 +518,15 @@ int amd_sof_acp_probe(struct snd_sof_dev *sdev) return ret; } + sdev->dsp_box.offset = 0; + sdev->dsp_box.size = BOX_SIZE_512; + + sdev->host_box.offset = sdev->dsp_box.offset + sdev->dsp_box.size; + sdev->host_box.size = BOX_SIZE_512; + + sdev->debug_box.offset = sdev->host_box.offset + sdev->host_box.size; + sdev->debug_box.size = BOX_SIZE_1024; + acp_memory_init(sdev); acp_dsp_stream_init(sdev); diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h index 4c42b8fd6abf185c278285fd0fbcece4be7da661..dd3c072d01721737bc67190eeccf1cb3fdb29e93 100644 --- a/sound/soc/sof/amd/acp.h +++ b/sound/soc/sof/amd/acp.h @@ -30,7 +30,8 @@ #define ACP_SOFT_RESET_DONE_MASK 0x00010001 #define ACP_DSP_INTR_EN_MASK 0x00000001 -#define ACP_SRAM_PTE_OFFSET 0x02050000 +#define ACP3X_SRAM_PTE_OFFSET 0x02050000 +#define ACP6X_SRAM_PTE_OFFSET 0x03800000 #define PAGE_SIZE_4K_ENABLE 0x2 #define ACP_PAGE_SIZE 0x1000 #define ACP_DMA_CH_RUN 0x02 @@ -45,7 +46,7 @@ #define ACPBUS_REG_BASE_OFFSET ACP_DMA_CNTL_0 #define ACP_DEFAULT_DRAM_LENGTH 0x00080000 -#define ACP_SCRATCH_MEMORY_ADDRESS 0x02050000 +#define ACP3X_SCRATCH_MEMORY_ADDRESS 0x02050000 #define ACP_SYSTEM_MEMORY_WINDOW 0x4000000 #define ACP_IRAM_BASE_ADDRESS 0x000000 #define ACP_DATA_RAM_BASE_ADDRESS 0x01000000 @@ -54,6 +55,7 @@ #define ACP_DSP_TO_HOST_IRQ 0x04 #define HOST_BRIDGE_CZN 0x1630 +#define HOST_BRIDGE_RMB 0x14B5 #define ACP_SHA_STAT 0x8000 #define ACP_PSP_TIMEOUT_COUNTER 5 #define ACP_EXT_INTR_ERROR_STAT 0x20000000 @@ -64,6 +66,9 @@ #define MBOX_READY_MASK 0x80000000 #define MBOX_STATUS_MASK 0xFFFF +#define BOX_SIZE_512 0x200 +#define BOX_SIZE_1024 0x400 + struct acp_atu_grp_pte { u32 low; u32 high; @@ -88,10 +93,6 @@ struct dma_descriptor { /* Scratch memory structure for communication b/w host and dsp */ struct scratch_ipc_conf { - /* DSP mailbox */ - u8 sof_out_box[512]; - /* Host mailbox */ - u8 sof_in_box[512]; /* Debug memory */ u8 sof_debug_box[1024]; /* Exception memory*/ @@ -139,6 +140,20 @@ struct acp_dsp_stream { unsigned int reg_offset; }; +struct sof_amd_acp_desc { + unsigned int rev; + unsigned int host_bridge_id; + unsigned int i2s_mode; + u32 pgfsm_base; + u32 ext_intr_stat; + u32 dsp_intr_base; + u32 sram_pte_offset; + u32 i2s_pin_config_offset; + u32 hw_semaphore_offset; + u32 acp_clkmux_sel; + u32 fusion_dsp_offset; +}; + /* Common device data struct for ACP devices */ struct acp_dev_data { struct snd_sof_dev *dev; @@ -206,8 +221,15 @@ int acp_pcm_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substr struct snd_pcm_hw_params *params, struct snd_sof_platform_stream_params *platform_params); +extern struct snd_sof_dsp_ops sof_acp_common_ops; + extern struct snd_sof_dsp_ops sof_renoir_ops; +int sof_renoir_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_rembrandt_ops; +int sof_rembrandt_ops_init(struct snd_sof_dev *sdev); +int acp_dai_probe(struct snd_soc_dai *dai); +struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev); /* Machine configuration */ int snd_amd_acp_find_config(struct pci_dev *pci); @@ -220,10 +242,6 @@ int acp_sof_trace_release(struct snd_sof_dev *sdev); int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state); int amd_sof_acp_resume(struct snd_sof_dev *sdev); -struct sof_amd_acp_desc { - unsigned int host_bridge_id; -}; - static inline const struct sof_amd_acp_desc *get_chip_info(struct snd_sof_pdata *pdata) { const struct sof_dev_desc *desc = pdata->desc; diff --git a/sound/soc/sof/amd/pci-rmb.c b/sound/soc/sof/amd/pci-rmb.c new file mode 100644 index 0000000000000000000000000000000000000000..4e1de462b431b8ba251cfc27f888d127f26d6f2c --- /dev/null +++ b/sound/soc/sof/amd/pci-rmb.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2022 Advanced Micro Devices, Inc. All rights reserved. +// +// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com> + +/*. + * PCI interface for Rembrandt ACP device + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <sound/sof.h> +#include <sound/soc-acpi.h> + +#include "../ops.h" +#include "../sof-pci-dev.h" +#include "../../amd/mach-config.h" +#include "acp.h" +#include "acp-dsp-offset.h" + +#define ACP6x_REG_START 0x1240000 +#define ACP6x_REG_END 0x125C000 + +static struct platform_device *dmic_dev; +static struct platform_device *pdev; + +static const struct resource rembrandt_res[] = { + { + .start = 0, + .end = ACP6x_REG_END - ACP6x_REG_START, + .name = "acp_mem", + .flags = IORESOURCE_MEM, + }, + { + .start = 0, + .end = 0, + .name = "acp_dai_irq", + .flags = IORESOURCE_IRQ, + }, +}; + +static const struct sof_amd_acp_desc rembrandt_chip_info = { + .rev = 6, + .host_bridge_id = HOST_BRIDGE_RMB, + .i2s_mode = 0x0a, + .pgfsm_base = ACP6X_PGFSM_BASE, + .ext_intr_stat = ACP6X_EXT_INTR_STAT, + .dsp_intr_base = ACP6X_DSP_SW_INTR_BASE, + .sram_pte_offset = ACP6X_SRAM_PTE_OFFSET, + .i2s_pin_config_offset = ACP6X_I2S_PIN_CONFIG, + .hw_semaphore_offset = ACP6X_AXI2DAGB_SEM_0, + .acp_clkmux_sel = ACP6X_CLKMUX_SEL, + .fusion_dsp_offset = ACP6X_DSP_FUSION_RUNSTALL, +}; + +static const struct sof_dev_desc rembrandt_desc = { + .machines = snd_soc_acpi_amd_rmb_sof_machines, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &rembrandt_chip_info, + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "amd/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "amd/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-rmb.ri", + }, + .nocodec_tplg_filename = "sof-acp.tplg", + .ops = &sof_rembrandt_ops, + .ops_init = sof_rembrandt_ops_init, +}; + +static int acp_pci_rmb_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) +{ + struct platform_device_info pdevinfo; + struct device *dev = &pci->dev; + const struct resource *res_i2s; + struct resource *res; + unsigned int flag, i, addr; + int ret; + + flag = snd_amd_acp_find_config(pci); + if (flag != FLAG_AMD_SOF && flag != FLAG_AMD_SOF_ONLY_DMIC) + return -ENODEV; + + ret = sof_pci_probe(pci, pci_id); + if (ret != 0) + return ret; + + dmic_dev = platform_device_register_data(dev, "dmic-codec", PLATFORM_DEVID_NONE, NULL, 0); + if (IS_ERR(dmic_dev)) { + dev_err(dev, "failed to create DMIC device\n"); + sof_pci_remove(pci); + return PTR_ERR(dmic_dev); + } + + /* Register platform device only if flag set to FLAG_AMD_SOF_ONLY_DMIC */ + if (flag != FLAG_AMD_SOF_ONLY_DMIC) + return 0; + + addr = pci_resource_start(pci, 0); + res = devm_kzalloc(&pci->dev, sizeof(struct resource) * ARRAY_SIZE(rembrandt_res), + GFP_KERNEL); + if (!res) { + platform_device_unregister(dmic_dev); + sof_pci_remove(pci); + return -ENOMEM; + } + + res_i2s = rembrandt_res; + for (i = 0; i < ARRAY_SIZE(rembrandt_res); i++, res_i2s++) { + res[i].name = res_i2s->name; + res[i].flags = res_i2s->flags; + res[i].start = addr + res_i2s->start; + res[i].end = addr + res_i2s->end; + if (res_i2s->flags == IORESOURCE_IRQ) { + res[i].start = pci->irq; + res[i].end = res[i].start; + } + } + + memset(&pdevinfo, 0, sizeof(pdevinfo)); + + /* + * We have common PCI driver probe for ACP device but we have to support I2S without SOF + * for some distributions. Register platform device that will be used to support non dsp + * ACP's audio ends points on some machines. + */ + pdevinfo.name = "acp_asoc_rembrandt"; + pdevinfo.id = 0; + pdevinfo.parent = &pci->dev; + pdevinfo.num_res = ARRAY_SIZE(rembrandt_res); + pdevinfo.res = &res[0]; + + pdev = platform_device_register_full(&pdevinfo); + if (IS_ERR(pdev)) { + dev_err(&pci->dev, "cannot register %s device\n", pdevinfo.name); + platform_device_unregister(dmic_dev); + sof_pci_remove(pci); + ret = PTR_ERR(pdev); + } + + return ret; +}; + +static void acp_pci_rmb_remove(struct pci_dev *pci) +{ + if (dmic_dev) + platform_device_unregister(dmic_dev); + if (pdev) + platform_device_unregister(pdev); + + sof_pci_remove(pci); +} + +/* PCI IDs */ +static const struct pci_device_id rmb_pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_PCI_DEV_ID), + .driver_data = (unsigned long)&rembrandt_desc}, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, rmb_pci_ids); + +/* pci_driver definition */ +static struct pci_driver snd_sof_pci_amd_rmb_driver = { + .name = KBUILD_MODNAME, + .id_table = rmb_pci_ids, + .probe = acp_pci_rmb_probe, + .remove = acp_pci_rmb_remove, +}; +module_pci_driver(snd_sof_pci_amd_rmb_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON); +MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV); diff --git a/sound/soc/sof/amd/pci-rn.c b/sound/soc/sof/amd/pci-rn.c index 3a7fed25a22674f9c014d0d4c9ad9e3440d69e3d..fca40b261671bd901b5b51f526c0d4d11e072082 100644 --- a/sound/soc/sof/amd/pci-rn.c +++ b/sound/soc/sof/amd/pci-rn.c @@ -21,6 +21,7 @@ #include "../sof-pci-dev.h" #include "../../amd/mach-config.h" #include "acp.h" +#include "acp-dsp-offset.h" #define ACP3x_REG_START 0x1240000 #define ACP3x_REG_END 0x125C000 @@ -44,7 +45,16 @@ static const struct resource renoir_res[] = { }; static const struct sof_amd_acp_desc renoir_chip_info = { + .rev = 3, .host_bridge_id = HOST_BRIDGE_CZN, + .i2s_mode = 0x04, + .pgfsm_base = ACP3X_PGFSM_BASE, + .ext_intr_stat = ACP3X_EXT_INTR_STAT, + .dsp_intr_base = ACP3X_DSP_SW_INTR_BASE, + .sram_pte_offset = ACP3X_SRAM_PTE_OFFSET, + .i2s_pin_config_offset = ACP3X_I2S_PIN_CONFIG, + .hw_semaphore_offset = ACP3X_AXI2DAGB_SEM_0, + .acp_clkmux_sel = ACP3X_CLKMUX_SEL, }; static const struct sof_dev_desc renoir_desc = { @@ -68,6 +78,7 @@ static const struct sof_dev_desc renoir_desc = { }, .nocodec_tplg_filename = "sof-acp.tplg", .ops = &sof_renoir_ops, + .ops_init = sof_renoir_ops_init, }; static int acp_pci_rn_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) diff --git a/sound/soc/sof/amd/rembrandt.c b/sound/soc/sof/amd/rembrandt.c new file mode 100644 index 0000000000000000000000000000000000000000..dcb64a23e1219d2a248a2761e079439c5c92e1f6 --- /dev/null +++ b/sound/soc/sof/amd/rembrandt.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2022 Advanced Micro Devices, Inc. +// +// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com> + +/* + * Hardware interface for Audio DSP on Rembrandt platform + */ + +#include <linux/platform_device.h> +#include <linux/module.h> + +#include "../ops.h" +#include "../sof-audio.h" +#include "acp.h" +#include "acp-dsp-offset.h" + +#define I2S_HS_INSTANCE 0 +#define I2S_BT_INSTANCE 1 +#define I2S_SP_INSTANCE 2 +#define PDM_DMIC_INSTANCE 3 + +static struct snd_soc_dai_driver rembrandt_sof_dai[] = { + [I2S_HS_INSTANCE] = { + .id = I2S_HS_INSTANCE, + .name = "acp-sof-hs", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S HS controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + .probe = &acp_dai_probe, + }, + + [I2S_BT_INSTANCE] = { + .id = I2S_BT_INSTANCE, + .name = "acp-sof-bt", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S BT controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + .probe = &acp_dai_probe, + }, + + [I2S_SP_INSTANCE] = { + .id = I2S_SP_INSTANCE, + .name = "acp-sof-sp", + .playback = { + .rates = SNDRV_PCM_RATE_8000_96000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 8, + .rate_min = 8000, + .rate_max = 96000, + }, + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 | + SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE, + /* Supporting only stereo for I2S SP controller capture */ + .channels_min = 2, + .channels_max = 2, + .rate_min = 8000, + .rate_max = 48000, + }, + .probe = &acp_dai_probe, + }, + + [PDM_DMIC_INSTANCE] = { + .id = PDM_DMIC_INSTANCE, + .name = "acp-sof-dmic", + .capture = { + .rates = SNDRV_PCM_RATE_8000_48000, + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels_min = 2, + .channels_max = 4, + .rate_min = 8000, + .rate_max = 48000, + }, + }, +}; + +/* Rembrandt ops */ +struct snd_sof_dsp_ops sof_rembrandt_ops; +EXPORT_SYMBOL_NS(sof_rembrandt_ops, SND_SOC_SOF_AMD_COMMON); + +int sof_rembrandt_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_rembrandt_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops)); + + sof_rembrandt_ops.drv = rembrandt_sof_dai; + sof_rembrandt_ops.num_drv = ARRAY_SIZE(rembrandt_sof_dai); + + return 0; +} + +MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON); +MODULE_DESCRIPTION("REMBRANDT SOF Driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/amd/renoir.c b/sound/soc/sof/amd/renoir.c index 9261c8bc2236d2a27d13fc3f436b8426022cfaa7..6ea8727f977e75a472c24236e6750f67affb6aad 100644 --- a/sound/soc/sof/amd/renoir.c +++ b/sound/soc/sof/amd/renoir.c @@ -23,22 +23,6 @@ #define I2S_SP_INSTANCE 1 #define PDM_DMIC_INSTANCE 2 -#define I2S_MODE 0x04 - -static int renoir_dai_probe(struct snd_soc_dai *dai) -{ - struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component); - unsigned int val; - - val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_I2S_PIN_CONFIG); - if (val != I2S_MODE) { - dev_err(sdev->dev, "I2S Mode is not supported (I2S_PIN_CONFIG: %#x)\n", val); - return -EINVAL; - } - - return 0; -} - static struct snd_soc_dai_driver renoir_sof_dai[] = { [I2S_BT_INSTANCE] = { .id = I2S_BT_INSTANCE, @@ -62,7 +46,7 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = { .rate_min = 8000, .rate_max = 48000, }, - .probe = &renoir_dai_probe, + .probe = &acp_dai_probe, }, [I2S_SP_INSTANCE] = { @@ -87,7 +71,7 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = { .rate_min = 8000, .rate_max = 48000, }, - .probe = &renoir_dai_probe, + .probe = &acp_dai_probe, }, [PDM_DMIC_INSTANCE] = { @@ -104,82 +88,21 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = { }, }; -static struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev) -{ - struct snd_sof_pdata *sof_pdata = sdev->pdata; - const struct sof_dev_desc *desc = sof_pdata->desc; - struct snd_soc_acpi_mach *mach; +/* Renoir ops */ +struct snd_sof_dsp_ops sof_renoir_ops; +EXPORT_SYMBOL_NS(sof_renoir_ops, SND_SOC_SOF_AMD_COMMON); - mach = snd_soc_acpi_find_machine(desc->machines); - if (!mach) { - dev_warn(sdev->dev, "No matching ASoC machine driver found\n"); - return NULL; - } +int sof_renoir_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_renoir_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops)); - sof_pdata->tplg_filename = mach->sof_tplg_filename; - sof_pdata->fw_filename = mach->fw_filename; + sof_renoir_ops.drv = renoir_sof_dai; + sof_renoir_ops.num_drv = ARRAY_SIZE(renoir_sof_dai); - return mach; + return 0; } -/* AMD Renoir DSP ops */ -struct snd_sof_dsp_ops sof_renoir_ops = { - /* probe and remove */ - .probe = amd_sof_acp_probe, - .remove = amd_sof_acp_remove, - - /* Register IO */ - .write = sof_io_write, - .read = sof_io_read, - - /* Block IO */ - .block_read = acp_dsp_block_read, - .block_write = acp_dsp_block_write, - - /*Firmware loading */ - .load_firmware = snd_sof_load_firmware_memcpy, - .pre_fw_run = acp_dsp_pre_fw_run, - .get_bar_index = acp_get_bar_index, - - /* DSP core boot */ - .run = acp_sof_dsp_run, - - /*IPC */ - .send_msg = acp_sof_ipc_send_msg, - .ipc_msg_data = acp_sof_ipc_msg_data, - .get_mailbox_offset = acp_sof_ipc_get_mailbox_offset, - .irq_thread = acp_sof_ipc_irq_thread, - - /* DAI drivers */ - .drv = renoir_sof_dai, - .num_drv = ARRAY_SIZE(renoir_sof_dai), - - /* stream callbacks */ - .pcm_open = acp_pcm_open, - .pcm_close = acp_pcm_close, - .pcm_hw_params = acp_pcm_hw_params, - - .hw_info = SNDRV_PCM_INFO_MMAP | - SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE | - SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, - - /* Machine driver callbacks */ - .machine_select = amd_sof_machine_select, - .machine_register = sof_machine_register, - .machine_unregister = sof_machine_unregister, - - /* Trace Logger */ - .trace_init = acp_sof_trace_init, - .trace_release = acp_sof_trace_release, - - /* PM */ - .suspend = amd_sof_acp_suspend, - .resume = amd_sof_acp_resume, -}; -EXPORT_SYMBOL(sof_renoir_ops); - MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON); MODULE_DESCRIPTION("RENOIR SOF Driver"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c index 67139e15f862a114b0319ec88ddca489eac365a5..8e1a9ba111ad592cfca814ecd0c00d6c348ccddb 100644 --- a/sound/soc/sof/compress.c +++ b/sound/soc/sof/compress.c @@ -11,20 +11,20 @@ #include "sof-priv.h" #include "sof-utils.h" -static void sof_set_transferred_bytes(struct snd_compr_tstamp *tstamp, +static void sof_set_transferred_bytes(struct sof_compr_stream *sstream, u64 host_pos, u64 buffer_size) { u64 prev_pos; unsigned int copied; - div64_u64_rem(tstamp->copied_total, buffer_size, &prev_pos); + div64_u64_rem(sstream->copied_total, buffer_size, &prev_pos); if (host_pos < prev_pos) copied = (buffer_size - prev_pos) + host_pos; else copied = host_pos - prev_pos; - tstamp->copied_total += copied; + sstream->copied_total += copied; } static void snd_sof_compr_fragment_elapsed_work(struct work_struct *work) @@ -49,7 +49,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream) struct snd_soc_pcm_runtime *rtd; struct snd_compr_runtime *crtd; struct snd_soc_component *component; - struct snd_compr_tstamp *tstamp; + struct sof_compr_stream *sstream; struct snd_sof_pcm *spcm; if (!cstream) @@ -57,7 +57,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream) rtd = cstream->private_data; crtd = cstream->runtime; - tstamp = crtd->private_data; + sstream = crtd->private_data; component = snd_soc_rtdcom_lookup(rtd, SOF_AUDIO_PCM_DRV_NAME); spcm = snd_sof_find_spcm_dai(component, rtd); @@ -67,7 +67,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream) return; } - sof_set_transferred_bytes(tstamp, spcm->stream[cstream->direction].posn.host_posn, + sof_set_transferred_bytes(sstream, spcm->stream[cstream->direction].posn.host_posn, crtd->buffer_size); /* use the same workqueue-based solution as for PCM, cf. snd_sof_pcm_elapsed */ @@ -96,24 +96,24 @@ static int sof_compr_open(struct snd_soc_component *component, { struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *crtd = cstream->runtime; - struct snd_compr_tstamp *tstamp; + struct sof_compr_stream *sstream; struct snd_sof_pcm *spcm; int dir; - tstamp = kzalloc(sizeof(*tstamp), GFP_KERNEL); - if (!tstamp) + sstream = kzalloc(sizeof(*sstream), GFP_KERNEL); + if (!sstream) return -ENOMEM; spcm = snd_sof_find_spcm_dai(component, rtd); if (!spcm) { - kfree(tstamp); + kfree(sstream); return -EINVAL; } dir = cstream->direction; if (spcm->stream[dir].cstream) { - kfree(tstamp); + kfree(sstream); return -EBUSY; } @@ -122,7 +122,7 @@ static int sof_compr_open(struct snd_soc_component *component, spcm->stream[dir].posn.dai_posn = 0; spcm->prepared[dir] = false; - crtd->private_data = tstamp; + crtd->private_data = sstream; return 0; } @@ -131,7 +131,7 @@ static int sof_compr_free(struct snd_soc_component *component, struct snd_compr_stream *cstream) { struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component); - struct snd_compr_tstamp *tstamp = cstream->runtime->private_data; + struct sof_compr_stream *sstream = cstream->runtime->private_data; struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct sof_ipc_stream stream; struct sof_ipc_reply reply; @@ -155,7 +155,7 @@ static int sof_compr_free(struct snd_soc_component *component, cancel_work_sync(&spcm->stream[cstream->direction].period_elapsed_work); spcm->stream[cstream->direction].cstream = NULL; - kfree(tstamp); + kfree(sstream); return ret; } @@ -169,7 +169,7 @@ static int sof_compr_set_params(struct snd_soc_component *component, struct sof_ipc_pcm_params_reply ipc_params_reply; struct sof_ipc_fw_ready *ready = &sdev->fw_ready; struct sof_ipc_fw_version *v = &ready->version; - struct snd_compr_tstamp *tstamp; + struct sof_compr_stream *sstream; struct sof_ipc_pcm_params *pcm; struct snd_sof_pcm *spcm; size_t ext_data_size; @@ -184,7 +184,7 @@ static int sof_compr_set_params(struct snd_soc_component *component, return -EINVAL; } - tstamp = crtd->private_data; + sstream = crtd->private_data; spcm = snd_sof_find_spcm_dai(component, rtd); @@ -237,8 +237,9 @@ static int sof_compr_set_params(struct snd_soc_component *component, goto out; } - tstamp->byte_offset = sdev->stream_box.offset + ipc_params_reply.posn_offset; - tstamp->sampling_rate = params->codec.sample_rate; + sstream->sampling_rate = params->codec.sample_rate; + sstream->channels = params->codec.ch_out; + sstream->sample_container_bytes = pcm->params.sample_container_bytes; spcm->prepared[cstream->direction] = true; @@ -296,18 +297,13 @@ static int sof_compr_trigger(struct snd_soc_component *component, &reply, sizeof(reply)); } -static int sof_compr_copy(struct snd_soc_component *component, - struct snd_compr_stream *cstream, - char __user *buf, size_t count) +static int sof_compr_copy_playback(struct snd_compr_runtime *rtd, + char __user *buf, size_t count) { - struct snd_compr_runtime *rtd = cstream->runtime; - unsigned int offset, n; void *ptr; + unsigned int offset, n; int ret; - if (count > rtd->buffer_size) - count = rtd->buffer_size; - div_u64_rem(rtd->total_bytes_available, rtd->buffer_size, &offset); ptr = rtd->dma_area + offset; n = rtd->buffer_size - offset; @@ -322,14 +318,58 @@ static int sof_compr_copy(struct snd_soc_component *component, return count - ret; } +static int sof_compr_copy_capture(struct snd_compr_runtime *rtd, + char __user *buf, size_t count) +{ + void *ptr; + unsigned int offset, n; + int ret; + + div_u64_rem(rtd->total_bytes_transferred, rtd->buffer_size, &offset); + ptr = rtd->dma_area + offset; + n = rtd->buffer_size - offset; + + if (count < n) { + ret = copy_to_user(buf, ptr, count); + } else { + ret = copy_to_user(buf, ptr, n); + ret += copy_to_user(buf + n, rtd->dma_area, count - n); + } + + return count - ret; +} + +static int sof_compr_copy(struct snd_soc_component *component, + struct snd_compr_stream *cstream, + char __user *buf, size_t count) +{ + struct snd_compr_runtime *rtd = cstream->runtime; + + if (count > rtd->buffer_size) + count = rtd->buffer_size; + + if (cstream->direction == SND_COMPRESS_PLAYBACK) + return sof_compr_copy_playback(rtd, buf, count); + else + return sof_compr_copy_capture(rtd, buf, count); +} + static int sof_compr_pointer(struct snd_soc_component *component, struct snd_compr_stream *cstream, struct snd_compr_tstamp *tstamp) { - struct snd_compr_tstamp *pstamp = cstream->runtime->private_data; + struct snd_sof_pcm *spcm; + struct snd_soc_pcm_runtime *rtd = cstream->private_data; + struct sof_compr_stream *sstream = cstream->runtime->private_data; + + spcm = snd_sof_find_spcm_dai(component, rtd); + if (!spcm) + return -EINVAL; - tstamp->sampling_rate = pstamp->sampling_rate; - tstamp->copied_total = pstamp->copied_total; + tstamp->sampling_rate = sstream->sampling_rate; + tstamp->copied_total = sstream->copied_total; + tstamp->pcm_io_frames = div_u64(spcm->stream[cstream->direction].posn.dai_posn, + sstream->channels * sstream->sample_container_bytes); return 0; } diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c index c99b5e6c026c10dca012f656335a8e307ad934ca..3e6141d03770fc1fc386583409a89f8796e4dabb 100644 --- a/sound/soc/sof/core.c +++ b/sound/soc/sof/core.c @@ -15,6 +15,9 @@ #include "sof-priv.h" #include "ops.h" +#define CREATE_TRACE_POINTS +#include <trace/events/sof.h> + /* see SOF_DBG_ flags */ static int sof_core_debug = IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_FIRMWARE_TRACE); module_param_named(sof_debug, sof_core_debug, int, 0444); diff --git a/sound/soc/sof/imx/Kconfig b/sound/soc/sof/imx/Kconfig index cc6e695f913a121b4fe2b6b33fc9e5ea7cbd7563..4751b04d5e6fe3a24d25578c5422776170e44aec 100644 --- a/sound/soc/sof/imx/Kconfig +++ b/sound/soc/sof/imx/Kconfig @@ -41,4 +41,13 @@ config SND_SOC_SOF_IMX8M Say Y if you have such a device. If unsure select "N". +config SND_SOC_SOF_IMX8ULP + tristate "SOF support for i.MX8ULP" + depends on IMX_DSP + select SND_SOC_SOF_IMX_COMMON + help + This adds support for Sound Open Firmware for NXP i.MX8ULP platforms. + Say Y if you have such a device. + If unsure select "N". + endif ## SND_SOC_SOF_IMX_TOPLEVEL diff --git a/sound/soc/sof/imx/Makefile b/sound/soc/sof/imx/Makefile index dba93c3466ec558a9ae19271297ecfe0b52485fb..798b43a415bf96d44df38fa0bec0b6582cb1819c 100644 --- a/sound/soc/sof/imx/Makefile +++ b/sound/soc/sof/imx/Makefile @@ -1,9 +1,11 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) snd-sof-imx8-objs := imx8.o snd-sof-imx8m-objs := imx8m.o +snd-sof-imx8ulp-objs := imx8ulp.o snd-sof-imx-common-objs := imx-common.o obj-$(CONFIG_SND_SOC_SOF_IMX8) += snd-sof-imx8.o obj-$(CONFIG_SND_SOC_SOF_IMX8M) += snd-sof-imx8m.o +obj-$(CONFIG_SND_SOC_SOF_IMX8ULP) += snd-sof-imx8ulp.o obj-$(CONFIG_SND_SOC_SOF_IMX_COMMON) += imx-common.o diff --git a/sound/soc/sof/imx/imx8ulp.c b/sound/soc/sof/imx/imx8ulp.c new file mode 100644 index 0000000000000000000000000000000000000000..4a562c9856e93c1bcd111f70e68c3513b43699bc --- /dev/null +++ b/sound/soc/sof/imx/imx8ulp.c @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// Copyright 2021-2022 NXP +// +// Author: Peng Zhang <peng.zhang_8@nxp.com> +// +// Hardware interface for audio DSP on i.MX8ULP + +#include <linux/arm-smccc.h> +#include <linux/clk.h> +#include <linux/firmware.h> +#include <linux/firmware/imx/dsp.h> +#include <linux/firmware/imx/ipc.h> +#include <linux/firmware/imx/svc/misc.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/of_reserved_mem.h> + +#include <sound/sof.h> +#include <sound/sof/xtensa.h> + +#include "../ops.h" +#include "../sof-of-dev.h" +#include "imx-common.h" + +#define FSL_SIP_HIFI_XRDC 0xc200000e + +/* SIM Domain register */ +#define SYSCTRL0 0x8 +#define EXECUTE_BIT BIT(13) +#define RESET_BIT BIT(16) +#define HIFI4_CLK_BIT BIT(17) +#define PB_CLK_BIT BIT(18) +#define PLAT_CLK_BIT BIT(19) +#define DEBUG_LOGIC_BIT BIT(25) + +#define MBOX_OFFSET 0x800000 +#define MBOX_SIZE 0x1000 + +static struct clk_bulk_data imx8ulp_dsp_clks[] = { + { .id = "core" }, + { .id = "ipg" }, + { .id = "ocram" }, + { .id = "mu" }, +}; + +struct imx8ulp_priv { + struct device *dev; + struct snd_sof_dev *sdev; + + /* DSP IPC handler */ + struct imx_dsp_ipc *dsp_ipc; + struct platform_device *ipc_dev; + + struct regmap *regmap; + struct imx_clocks *clks; +}; + +static void imx8ulp_sim_lpav_start(struct imx8ulp_priv *priv) +{ + /* Controls the HiFi4 DSP Reset: 1 in reset, 0 out of reset */ + regmap_update_bits(priv->regmap, SYSCTRL0, RESET_BIT, 0); + + /* Reset HiFi4 DSP Debug logic: 1 debug reset, 0 out of reset*/ + regmap_update_bits(priv->regmap, SYSCTRL0, DEBUG_LOGIC_BIT, 0); + + /* Stall HIFI4 DSP Execution: 1 stall, 0 run */ + regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, 0); +} + +static int imx8ulp_get_mailbox_offset(struct snd_sof_dev *sdev) +{ + return MBOX_OFFSET; +} + +static int imx8ulp_get_window_offset(struct snd_sof_dev *sdev, u32 id) +{ + return MBOX_OFFSET; +} + +static void imx8ulp_dsp_handle_reply(struct imx_dsp_ipc *ipc) +{ + struct imx8ulp_priv *priv = imx_dsp_get_data(ipc); + unsigned long flags; + + spin_lock_irqsave(&priv->sdev->ipc_lock, flags); + + snd_sof_ipc_process_reply(priv->sdev, 0); + + spin_unlock_irqrestore(&priv->sdev->ipc_lock, flags); +} + +static void imx8ulp_dsp_handle_request(struct imx_dsp_ipc *ipc) +{ + struct imx8ulp_priv *priv = imx_dsp_get_data(ipc); + u32 p; /* panic code */ + + /* Read the message from the debug box. */ + sof_mailbox_read(priv->sdev, priv->sdev->debug_box.offset + 4, &p, sizeof(p)); + + /* Check to see if the message is a panic code (0x0dead***) */ + if ((p & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) + snd_sof_dsp_panic(priv->sdev, p, true); + else + snd_sof_ipc_msgs_rx(priv->sdev); +} + +static struct imx_dsp_ops dsp_ops = { + .handle_reply = imx8ulp_dsp_handle_reply, + .handle_request = imx8ulp_dsp_handle_request, +}; + +static int imx8ulp_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) +{ + struct imx8ulp_priv *priv = sdev->pdata->hw_pdata; + + sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data, + msg->msg_size); + imx_dsp_ring_doorbell(priv->dsp_ipc, 0); + + return 0; +} + +static int imx8ulp_run(struct snd_sof_dev *sdev) +{ + struct imx8ulp_priv *priv = sdev->pdata->hw_pdata; + + imx8ulp_sim_lpav_start(priv); + + return 0; +} + +static int imx8ulp_reset(struct snd_sof_dev *sdev) +{ + struct imx8ulp_priv *priv = sdev->pdata->hw_pdata; + struct arm_smccc_res smc_resource; + + /* HiFi4 Platform Clock Enable: 1 enabled, 0 disabled */ + regmap_update_bits(priv->regmap, SYSCTRL0, PLAT_CLK_BIT, PLAT_CLK_BIT); + + /* HiFi4 PBCLK clock enable: 1 enabled, 0 disabled */ + regmap_update_bits(priv->regmap, SYSCTRL0, PB_CLK_BIT, PB_CLK_BIT); + + /* HiFi4 Clock Enable: 1 enabled, 0 disabled */ + regmap_update_bits(priv->regmap, SYSCTRL0, HIFI4_CLK_BIT, HIFI4_CLK_BIT); + + regmap_update_bits(priv->regmap, SYSCTRL0, RESET_BIT, RESET_BIT); + usleep_range(1, 2); + + /* Stall HIFI4 DSP Execution: 1 stall, 0 not stall */ + regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, EXECUTE_BIT); + usleep_range(1, 2); + + arm_smccc_smc(FSL_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &smc_resource); + + return 0; +} + +static int imx8ulp_probe(struct snd_sof_dev *sdev) +{ + struct platform_device *pdev = + container_of(sdev->dev, struct platform_device, dev); + struct device_node *np = pdev->dev.of_node; + struct device_node *res_node; + struct resource *mmio; + struct imx8ulp_priv *priv; + struct resource res; + u32 base, size; + int ret = 0; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->clks = devm_kzalloc(&pdev->dev, sizeof(*priv->clks), GFP_KERNEL); + if (!priv->clks) + return -ENOMEM; + + sdev->num_cores = 1; + sdev->pdata->hw_pdata = priv; + priv->dev = sdev->dev; + priv->sdev = sdev; + + /* System integration module(SIM) control dsp configuration */ + priv->regmap = syscon_regmap_lookup_by_phandle(np, "fsl,dsp-ctrl"); + if (IS_ERR(priv->regmap)) + return PTR_ERR(priv->regmap); + + priv->ipc_dev = platform_device_register_data(sdev->dev, "imx-dsp", + PLATFORM_DEVID_NONE, + pdev, sizeof(*pdev)); + if (IS_ERR(priv->ipc_dev)) + return PTR_ERR(priv->ipc_dev); + + priv->dsp_ipc = dev_get_drvdata(&priv->ipc_dev->dev); + if (!priv->dsp_ipc) { + /* DSP IPC driver not probed yet, try later */ + ret = -EPROBE_DEFER; + dev_err(sdev->dev, "Failed to get drvdata\n"); + goto exit_pdev_unregister; + } + + imx_dsp_set_data(priv->dsp_ipc, priv); + priv->dsp_ipc->ops = &dsp_ops; + + /* DSP base */ + mmio = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (mmio) { + base = mmio->start; + size = resource_size(mmio); + } else { + dev_err(sdev->dev, "error: failed to get DSP base at idx 0\n"); + ret = -EINVAL; + goto exit_pdev_unregister; + } + + sdev->bar[SOF_FW_BLK_TYPE_IRAM] = devm_ioremap(sdev->dev, base, size); + if (!sdev->bar[SOF_FW_BLK_TYPE_IRAM]) { + dev_err(sdev->dev, "failed to ioremap base 0x%x size 0x%x\n", + base, size); + ret = -ENODEV; + goto exit_pdev_unregister; + } + sdev->mmio_bar = SOF_FW_BLK_TYPE_IRAM; + + res_node = of_parse_phandle(np, "memory-reserved", 0); + if (!res_node) { + dev_err(&pdev->dev, "failed to get memory region node\n"); + ret = -ENODEV; + goto exit_pdev_unregister; + } + + ret = of_address_to_resource(res_node, 0, &res); + of_node_put(res_node); + if (ret) { + dev_err(&pdev->dev, "failed to get reserved region address\n"); + goto exit_pdev_unregister; + } + + sdev->bar[SOF_FW_BLK_TYPE_SRAM] = devm_ioremap_wc(sdev->dev, res.start, + resource_size(&res)); + if (!sdev->bar[SOF_FW_BLK_TYPE_SRAM]) { + dev_err(sdev->dev, "failed to ioremap mem 0x%x size 0x%x\n", + base, size); + ret = -ENOMEM; + goto exit_pdev_unregister; + } + sdev->mailbox_bar = SOF_FW_BLK_TYPE_SRAM; + + /* set default mailbox offset for FW ready message */ + sdev->dsp_box.offset = MBOX_OFFSET; + + ret = of_reserved_mem_device_init(sdev->dev); + if (ret) { + dev_err(&pdev->dev, "failed to init reserved memory region %d\n", ret); + goto exit_pdev_unregister; + } + + priv->clks->dsp_clks = imx8ulp_dsp_clks; + priv->clks->num_dsp_clks = ARRAY_SIZE(imx8ulp_dsp_clks); + + ret = imx8_parse_clocks(sdev, priv->clks); + if (ret < 0) + goto exit_pdev_unregister; + + ret = imx8_enable_clocks(sdev, priv->clks); + if (ret < 0) + goto exit_pdev_unregister; + + return 0; + +exit_pdev_unregister: + platform_device_unregister(priv->ipc_dev); + + return ret; +} + +static int imx8ulp_remove(struct snd_sof_dev *sdev) +{ + struct imx8ulp_priv *priv = sdev->pdata->hw_pdata; + + imx8_disable_clocks(sdev, priv->clks); + platform_device_unregister(priv->ipc_dev); + + return 0; +} + +/* on i.MX8 there is 1 to 1 match between type and BAR idx */ +static int imx8ulp_get_bar_index(struct snd_sof_dev *sdev, u32 type) +{ + return type; +} + +static int imx8ulp_suspend(struct snd_sof_dev *sdev) +{ + int i; + struct imx8ulp_priv *priv = (struct imx8ulp_priv *)sdev->pdata->hw_pdata; + + /*Stall DSP, release in .run() */ + regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, EXECUTE_BIT); + + for (i = 0; i < DSP_MU_CHAN_NUM; i++) + imx_dsp_free_channel(priv->dsp_ipc, i); + + imx8_disable_clocks(sdev, priv->clks); + + return 0; +} + +static int imx8ulp_resume(struct snd_sof_dev *sdev) +{ + struct imx8ulp_priv *priv = (struct imx8ulp_priv *)sdev->pdata->hw_pdata; + int i; + + imx8_enable_clocks(sdev, priv->clks); + + for (i = 0; i < DSP_MU_CHAN_NUM; i++) + imx_dsp_request_channel(priv->dsp_ipc, i); + + return 0; +} + +static int imx8ulp_dsp_runtime_resume(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_dsp_state = { + .state = SOF_DSP_PM_D0, + .substate = 0, + }; + + imx8ulp_resume(sdev); + + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); +} + +static int imx8ulp_dsp_runtime_suspend(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_dsp_state = { + .state = SOF_DSP_PM_D3, + .substate = 0, + }; + + imx8ulp_suspend(sdev); + + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); +} + +static int imx8ulp_dsp_suspend(struct snd_sof_dev *sdev, unsigned int target_state) +{ + const struct sof_dsp_power_state target_dsp_state = { + .state = target_state, + .substate = 0, + }; + + if (!pm_runtime_suspended(sdev->dev)) + imx8ulp_suspend(sdev); + + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); +} + +static int imx8ulp_dsp_resume(struct snd_sof_dev *sdev) +{ + const struct sof_dsp_power_state target_dsp_state = { + .state = SOF_DSP_PM_D0, + .substate = 0, + }; + + imx8ulp_resume(sdev); + + if (pm_runtime_suspended(sdev->dev)) { + pm_runtime_disable(sdev->dev); + pm_runtime_set_active(sdev->dev); + pm_runtime_mark_last_busy(sdev->dev); + pm_runtime_enable(sdev->dev); + pm_runtime_idle(sdev->dev); + } + + return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); +} + +static struct snd_soc_dai_driver imx8ulp_dai[] = { + { + .name = "sai5", + .playback = { + .channels_min = 1, + .channels_max = 32, + }, + .capture = { + .channels_min = 1, + .channels_max = 32, + }, + }, + { + .name = "sai6", + .playback = { + .channels_min = 1, + .channels_max = 32, + }, + .capture = { + .channels_min = 1, + .channels_max = 32, + }, + }, +}; + +static int imx8ulp_dsp_set_power_state(struct snd_sof_dev *sdev, + const struct sof_dsp_power_state *target_state) +{ + sdev->dsp_power_state = *target_state; + + return 0; +} + +/* i.MX8 ops */ +static struct snd_sof_dsp_ops sof_imx8ulp_ops = { + /* probe and remove */ + .probe = imx8ulp_probe, + .remove = imx8ulp_remove, + /* DSP core boot */ + .run = imx8ulp_run, + .reset = imx8ulp_reset, + + /* Block IO */ + .block_read = sof_block_read, + .block_write = sof_block_write, + + /* Module IO */ + .read64 = sof_io_read64, + + /* Mailbox IO */ + .mailbox_read = sof_mailbox_read, + .mailbox_write = sof_mailbox_write, + + /* ipc */ + .send_msg = imx8ulp_send_msg, + .get_mailbox_offset = imx8ulp_get_mailbox_offset, + .get_window_offset = imx8ulp_get_window_offset, + + .ipc_msg_data = sof_ipc_msg_data, + .set_stream_data_offset = sof_set_stream_data_offset, + + /* stream callbacks */ + .pcm_open = sof_stream_pcm_open, + .pcm_close = sof_stream_pcm_close, + + /* module loading */ + .get_bar_index = imx8ulp_get_bar_index, + /* firmware loading */ + .load_firmware = snd_sof_load_firmware_memcpy, + + /* Debug information */ + .dbg_dump = imx8_dump, + + /* Firmware ops */ + .dsp_arch_ops = &sof_xtensa_arch_ops, + + /* DAI drivers */ + .drv = imx8ulp_dai, + .num_drv = ARRAY_SIZE(imx8ulp_dai), + + /* ALSA HW info flags */ + .hw_info = SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED | + SNDRV_PCM_INFO_PAUSE | + SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, + + /* PM */ + .runtime_suspend = imx8ulp_dsp_runtime_suspend, + .runtime_resume = imx8ulp_dsp_runtime_resume, + + .suspend = imx8ulp_dsp_suspend, + .resume = imx8ulp_dsp_resume, + + .set_power_state = imx8ulp_dsp_set_power_state, +}; + +static struct sof_dev_desc sof_of_imx8ulp_desc = { + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "imx/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "imx/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-imx8ulp.ri", + }, + .nocodec_tplg_filename = "sof-imx8ulp-nocodec.tplg", + .ops = &sof_imx8ulp_ops, +}; + +static const struct of_device_id sof_of_imx8ulp_ids[] = { + { .compatible = "fsl,imx8ulp-dsp", .data = &sof_of_imx8ulp_desc}, + { } +}; +MODULE_DEVICE_TABLE(of, sof_of_imx8ulp_ids); + +/* DT driver definition */ +static struct platform_driver snd_sof_of_imx8ulp_driver = { + .probe = sof_of_probe, + .remove = sof_of_remove, + .driver = { + .name = "sof-audio-of-imx8ulp", + .pm = &sof_of_pm, + .of_match_table = sof_of_imx8ulp_ids, + }, +}; +module_platform_driver(snd_sof_of_imx8ulp_driver); + +MODULE_IMPORT_NS(SND_SOC_SOF_XTENSA); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index 3f54678e810baacfdaf8a9ee054b4eb0c7b1406e..7af495fb61256a5ea5610a21596301134aa9e1b9 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -95,6 +95,31 @@ config SND_SOC_SOF_MERRIFIELD Say Y if you have such a device. If unsure select "N". +config SND_SOC_SOF_INTEL_SKL + tristate + select SND_SOC_SOF_HDA_COMMON + select SND_SOC_SOF_INTEL_IPC4 + +config SND_SOC_SOF_SKYLAKE + tristate "SOF support for SkyLake" + default SND_SOC_SOF_PCI + select SND_SOC_SOF_INTEL_SKL + help + This adds support for the Intel(R) platforms using the SkyLake processors. + Say Y if you have such a device. + If unsure select "N". + This is intended only for developers and not a recommend option for distros. + +config SND_SOC_SOF_KABYLAKE + tristate "SOF support for KabyLake" + default SND_SOC_SOF_PCI + select SND_SOC_SOF_INTEL_SKL + help + This adds support for the Intel(R) platforms using the KabyLake processors. + Say Y if you have such a device. + If unsure select "N". + This is intended only for developers and not a recommend option for distros. + config SND_SOC_SOF_INTEL_APL tristate select SND_SOC_SOF_HDA_COMMON diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile index a079159bb2f024e3c58184f8ee6cf0552caa16a9..8b8ea03617850d4a5187fb0eb70dc6d2f6bc49e7 100644 --- a/sound/soc/sof/intel/Makefile +++ b/sound/soc/sof/intel/Makefile @@ -6,7 +6,9 @@ snd-sof-acpi-intel-bdw-objs := bdw.o snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \ hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \ hda-dai.o hda-bus.o \ + skl.o hda-loader-skl.o \ apl.o cnl.o tgl.o icl.o mtl.o hda-common-ops.o + snd-sof-intel-hda-common-$(CONFIG_SND_SOC_SOF_HDA_PROBES) += hda-probes.o snd-sof-intel-hda-objs := hda-codec.o @@ -20,6 +22,7 @@ obj-$(CONFIG_SND_SOC_SOF_HDA_COMMON) += snd-sof-intel-hda-common.o obj-$(CONFIG_SND_SOC_SOF_HDA) += snd-sof-intel-hda.o snd-sof-pci-intel-tng-objs := pci-tng.o +snd-sof-pci-intel-skl-objs := pci-skl.o snd-sof-pci-intel-apl-objs := pci-apl.o snd-sof-pci-intel-cnl-objs := pci-cnl.o snd-sof-pci-intel-icl-objs := pci-icl.o @@ -27,6 +30,7 @@ snd-sof-pci-intel-tgl-objs := pci-tgl.o snd-sof-pci-intel-mtl-objs := pci-mtl.o obj-$(CONFIG_SND_SOC_SOF_MERRIFIELD) += snd-sof-pci-intel-tng.o +obj-$(CONFIG_SND_SOC_SOF_INTEL_SKL) += snd-sof-pci-intel-skl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_APL) += snd-sof-pci-intel-apl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_CNL) += snd-sof-pci-intel-cnl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_ICL) += snd-sof-pci-intel-icl.o diff --git a/sound/soc/sof/intel/apl.c b/sound/soc/sof/intel/apl.c index 084c245a952288a66cc3ef3e8c7fb874de9adeeb..1549ca7587a4c5e8c45b07fd249a4d44f8de225e 100644 --- a/sound/soc/sof/intel/apl.c +++ b/sound/soc/sof/intel/apl.c @@ -45,6 +45,9 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev) /* ipc */ sof_apl_ops.send_msg = hda_dsp_ipc_send_msg; + + /* debug */ + sof_apl_ops.ipc_dump = hda_ipc_dump; } if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { @@ -57,11 +60,16 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev) ipc4_data = sdev->private; ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_5; + /* doorbell */ sof_apl_ops.irq_thread = hda_dsp_ipc4_irq_thread; /* ipc */ sof_apl_ops.send_msg = hda_dsp_ipc4_send_msg; + + /* debug */ + sof_apl_ops.ipc_dump = hda_ipc4_dump; } /* set DAI driver ops */ @@ -70,7 +78,6 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev) /* debug */ sof_apl_ops.debug_map = apl_dsp_debugfs; sof_apl_ops.debug_map_count = ARRAY_SIZE(apl_dsp_debugfs); - sof_apl_ops.ipc_dump = hda_ipc_dump; /* firmware run */ sof_apl_ops.run = hda_dsp_cl_boot_firmware; @@ -102,6 +109,8 @@ const struct sof_intel_dsp_desc apl_chip_info = { .quirks = SOF_INTEL_PROCEN_FMT_QUIRK, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_1_5_PLUS, }; EXPORT_SYMBOL_NS(apl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/cnl.c b/sound/soc/sof/intel/cnl.c index a064453f0bc3e58d92be7617c452d8e91e323079..19d0b1909bfd6a3505ae36f7ff6f42e9caf18447 100644 --- a/sound/soc/sof/intel/cnl.c +++ b/sound/soc/sof/intel/cnl.c @@ -17,6 +17,7 @@ #include <sound/sof/ext_manifest4.h> #include <sound/sof/ipc4/header.h> +#include <trace/events/sof_intel.h> #include "../ipc4-priv.h" #include "../ops.h" #include "hda.h" @@ -121,9 +122,7 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context) msg_ext = hipci & CNL_DSP_REG_HIPCIDR_MSG_MASK; msg = hipcida & CNL_DSP_REG_HIPCIDA_MSG_MASK; - dev_vdbg(sdev->dev, - "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n", - msg, msg_ext); + trace_sof_intel_ipc_firmware_response(sdev, msg, msg_ext); /* mask Done interrupt */ snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, @@ -153,9 +152,7 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context) msg = hipctdr & CNL_DSP_REG_HIPCTDR_MSG_MASK; msg_ext = hipctdd & CNL_DSP_REG_HIPCTDD_MSG_MASK; - dev_vdbg(sdev->dev, - "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n", - msg, msg_ext); + trace_sof_intel_ipc_firmware_initiated(sdev, msg, msg_ext); /* handle messages from DSP */ if ((hipctdr & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) { @@ -335,6 +332,27 @@ void cnl_ipc_dump(struct snd_sof_dev *sdev) hipcida, hipctdr, hipcctl); } +void cnl_ipc4_dump(struct snd_sof_dev *sdev) +{ + u32 hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl; + + hda_ipc_irq_dump(sdev); + + hipcidr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR); + hipcidd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDD); + hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDA); + hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDR); + hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDD); + hipctda = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDA); + hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCCTL); + + /* dump the IPC regs */ + /* TODO: parse the raw msg */ + dev_err(sdev->dev, + "Host IPC initiator: %#x|%#x|%#x, target: %#x|%#x|%#x, ctl: %#x\n", + hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl); +} + /* cannonlake ops */ struct snd_sof_dsp_ops sof_cnl_ops; EXPORT_SYMBOL_NS(sof_cnl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -354,6 +372,9 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev) /* ipc */ sof_cnl_ops.send_msg = cnl_ipc_send_msg; + + /* debug */ + sof_cnl_ops.ipc_dump = cnl_ipc_dump; } if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { @@ -366,11 +387,16 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev) ipc4_data = sdev->private; ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_8; + /* doorbell */ sof_cnl_ops.irq_thread = cnl_ipc4_irq_thread; /* ipc */ sof_cnl_ops.send_msg = cnl_ipc4_send_msg; + + /* debug */ + sof_cnl_ops.ipc_dump = cnl_ipc4_dump; } /* set DAI driver ops */ @@ -379,7 +405,6 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev) /* debug */ sof_cnl_ops.debug_map = cnl_dsp_debugfs; sof_cnl_ops.debug_map_count = ARRAY_SIZE(cnl_dsp_debugfs); - sof_cnl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ sof_cnl_ops.post_fw_run = hda_dsp_post_fw_run; @@ -413,6 +438,8 @@ const struct sof_intel_dsp_desc cnl_chip_info = { .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_1_8, }; EXPORT_SYMBOL_NS(cnl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -443,6 +470,8 @@ const struct sof_intel_dsp_desc jsl_chip_info = { .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_0, }; EXPORT_SYMBOL_NS(jsl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/hda-codec.c b/sound/soc/sof/intel/hda-codec.c index 2f3f4a733d9e69094431457dee6d6b0c2c8450ef..1e9afc48394c7bcc0e8c86e2829b683b4845111d 100644 --- a/sound/soc/sof/intel/hda-codec.c +++ b/sound/soc/sof/intel/hda-codec.c @@ -109,17 +109,45 @@ EXPORT_SYMBOL_NS(hda_codec_jack_check, SND_SOC_SOF_HDA_AUDIO_CODEC); #define is_generic_config(x) 0 #endif +static void hda_codec_device_exit(struct device *dev) +{ + snd_hdac_device_exit(dev_to_hdac_dev(dev)); +} + +static struct hda_codec *hda_codec_device_init(struct hdac_bus *bus, int addr, int type) +{ + struct hda_codec *codec; + int ret; + + codec = snd_hda_codec_device_init(to_hda_bus(bus), addr, "ehdaudio%dD%d", bus->idx, addr); + if (IS_ERR(codec)) { + dev_err(bus->dev, "device init failed for hdac device\n"); + return codec; + } + + codec->core.type = type; + codec->core.dev.release = hda_codec_device_exit; + + ret = snd_hdac_device_register(&codec->core); + if (ret) { + dev_err(bus->dev, "failed to register hdac device\n"); + snd_hdac_device_exit(&codec->core); + return ERR_PTR(ret); + } + + return codec; +} + /* probe individual codec */ static int hda_codec_probe(struct snd_sof_dev *sdev, int address, bool hda_codec_use_common_hdmi) { #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC) struct hdac_hda_priv *hda_priv; - struct hda_codec *codec; int type = HDA_DEV_LEGACY; #endif struct hda_bus *hbus = sof_to_hbus(sdev); - struct hdac_device *hdev; + struct hda_codec *codec; u32 hda_cmd = (address << 28) | (AC_NODE_ROOT << 20) | (AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID; u32 resp = -1; @@ -142,20 +170,20 @@ static int hda_codec_probe(struct snd_sof_dev *sdev, int address, if (!hda_priv) return -ENOMEM; - hda_priv->codec.bus = hbus; - hdev = &hda_priv->codec.core; - codec = &hda_priv->codec; - /* only probe ASoC codec drivers for HDAC-HDMI */ if (!hda_codec_use_common_hdmi && (resp & 0xFFFF0000) == IDISP_VID_INTEL) type = HDA_DEV_ASOC; - ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev, type); + codec = hda_codec_device_init(&hbus->core, address, type); + ret = PTR_ERR_OR_ZERO(codec); if (ret < 0) return ret; + hda_priv->codec = codec; + dev_set_drvdata(&codec->core.dev, hda_priv); + if ((resp & 0xFFFF0000) == IDISP_VID_INTEL) { - if (!hdev->bus->audio_component) { + if (!hbus->core.audio_component) { dev_dbg(sdev->dev, "iDisp hw present but no driver\n"); ret = -ENOENT; @@ -181,15 +209,12 @@ static int hda_codec_probe(struct snd_sof_dev *sdev, int address, out: if (ret < 0) { - snd_hdac_device_unregister(hdev); - put_device(&hdev->dev); + snd_hdac_device_unregister(&codec->core); + put_device(&codec->core.dev); } #else - hdev = devm_kzalloc(sdev->dev, sizeof(*hdev), GFP_KERNEL); - if (!hdev) - return -ENOMEM; - - ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev, HDA_DEV_ASOC); + codec = hda_codec_device_init(&hbus->core, address, HDA_DEV_ASOC); + ret = PTR_ERR_OR_ZERO(codec); #endif return ret; diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c index eddfd77ad90f4e2250fc12e7cc8ad9603c308c46..3c76f843454b6fc7fb4f50fd73c515fd7e27cc5b 100644 --- a/sound/soc/sof/intel/hda-dsp.c +++ b/sound/soc/sof/intel/hda-dsp.c @@ -18,6 +18,7 @@ #include <linux/module.h> #include <sound/hdaudio_ext.h> #include <sound/hda_register.h> +#include <trace/events/sof_intel.h> #include "../sof-audio.h" #include "../ops.h" #include "hda.h" @@ -113,7 +114,7 @@ static int hda_dsp_core_reset_leave(struct snd_sof_dev *sdev, unsigned int core_ return ret; } -static int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask) +int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask) { /* stall core */ snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR, @@ -125,7 +126,7 @@ static int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_ return hda_dsp_core_reset_enter(sdev, core_mask); } -static bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask) +bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask) { int val; bool is_enable; @@ -397,8 +398,7 @@ static int hda_dsp_update_d0i3c_register(struct snd_sof_dev *sdev, u8 value) return ret; } - dev_vdbg(bus->dev, "D0I3C updated, register = 0x%x\n", - snd_hdac_chip_readb(bus, VS_D0I3C)); + trace_sof_intel_D0I3C_updated(sdev, snd_hdac_chip_readb(bus, VS_D0I3C)); return 0; } @@ -620,14 +620,18 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) /* * The memory used for IMR boot loses its content in deeper than S3 state * We must not try IMR boot on next power up (as it will fail). + * + * In case of firmware crash or boot failure set the skip_imr_boot to true + * as well in order to try to re-load the firmware to do a 'cold' boot. */ - if (sdev->system_suspend_target > SOF_SUSPEND_S3) + if (sdev->system_suspend_target > SOF_SUSPEND_S3 || + sdev->fw_state == SOF_FW_CRASHED || + sdev->fw_state == SOF_FW_BOOT_FAILED) hda->skip_imr_boot = true; - hda_sdw_int_enable(sdev, false); - - /* disable IPC interrupts */ - hda_dsp_ipc_int_disable(sdev); + ret = chip->disable_interrupts(sdev); + if (ret < 0) + return ret; #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) hda_codec_jack_wake_enable(sdev, runtime_suspend); @@ -636,11 +640,9 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) snd_hdac_ext_bus_link_power_down_all(bus); #endif - /* power down DSP */ - ret = hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask); + ret = chip->power_down_dsp(sdev); if (ret < 0) { - dev_err(sdev->dev, - "error: failed to power down core during suspend\n"); + dev_err(sdev->dev, "failed to power down DSP during suspend\n"); return ret; } @@ -984,3 +986,11 @@ power_down: return ret; } + +int hda_dsp_disable_interrupts(struct snd_sof_dev *sdev) +{ + hda_sdw_int_enable(sdev, false); + hda_dsp_ipc_int_disable(sdev); + + return 0; +} diff --git a/sound/soc/sof/intel/hda-ipc.c b/sound/soc/sof/intel/hda-ipc.c index 65e688f749eaf16955e29d2f7bd52273de024d99..9b3667c705e4728c7d73ee292c86eae44c22979a 100644 --- a/sound/soc/sof/intel/hda-ipc.c +++ b/sound/soc/sof/intel/hda-ipc.c @@ -16,6 +16,7 @@ */ #include <sound/sof/ipc4/header.h> +#include <trace/events/sof_intel.h> #include "../ops.h" #include "hda.h" @@ -212,9 +213,7 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) msg = hipci & HDA_DSP_REG_HIPCI_MSG_MASK; msg_ext = hipcie & HDA_DSP_REG_HIPCIE_MSG_MASK; - dev_vdbg(sdev->dev, - "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n", - msg, msg_ext); + trace_sof_intel_ipc_firmware_response(sdev, msg, msg_ext); /* mask Done interrupt */ snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, @@ -255,9 +254,7 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) msg = hipct & HDA_DSP_REG_HIPCT_MSG_MASK; msg_ext = hipcte & HDA_DSP_REG_HIPCTE_MSG_MASK; - dev_vdbg(sdev->dev, - "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n", - msg, msg_ext); + trace_sof_intel_ipc_firmware_initiated(sdev, msg, msg_ext); /* mask BUSY interrupt */ snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, @@ -307,12 +304,13 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) /* Check if an IPC IRQ occurred */ bool hda_dsp_check_ipc_irq(struct snd_sof_dev *sdev) { + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; bool ret = false; u32 irq_status; /* store status */ irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIS); - dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status); + trace_sof_intel_hda_irq_ipc_check(sdev, irq_status); /* invalid message ? */ if (irq_status == 0xffffffff) @@ -322,6 +320,13 @@ bool hda_dsp_check_ipc_irq(struct snd_sof_dev *sdev) if (irq_status & HDA_DSP_ADSPIS_IPC) ret = true; + /* CLDMA message ? */ + if (irq_status & HDA_DSP_ADSPIS_CL_DMA) { + hda->code_loading = 0; + wake_up(&hda->waitq); + ret = false; + } + out: return ret; } diff --git a/sound/soc/sof/intel/hda-ipc.h b/sound/soc/sof/intel/hda-ipc.h index 10fbca5939dba63dd4f70cb97adab1670aaa5671..8ec5e9f6f8d7a4afb7b2f792d66e0eb1a707b5d3 100644 --- a/sound/soc/sof/intel/hda-ipc.h +++ b/sound/soc/sof/intel/hda-ipc.h @@ -51,5 +51,6 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context); int cnl_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); void cnl_ipc_dump(struct snd_sof_dev *sdev); +void cnl_ipc4_dump(struct snd_sof_dev *sdev); #endif diff --git a/sound/soc/sof/intel/hda-loader-skl.c b/sound/soc/sof/intel/hda-loader-skl.c new file mode 100644 index 0000000000000000000000000000000000000000..0193fb3964a0580f75e006a7494a7d5f1e83af30 --- /dev/null +++ b/sound/soc/sof/intel/hda-loader-skl.c @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2018-2022 Intel Corporation. All rights reserved. +// + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/firmware.h> +#include <linux/fs.h> +#include <linux/interrupt.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <sound/hdaudio_ext.h> +#include <sound/sof.h> +#include <sound/pcm_params.h> + +#include "../sof-priv.h" +#include "../ops.h" +#include "hda.h" + +#define HDA_SKL_WAIT_TIMEOUT 500 /* 500 msec */ +#define HDA_SKL_CLDMA_MAX_BUFFER_SIZE (32 * PAGE_SIZE) + +/* Stream Reset */ +#define HDA_CL_SD_CTL_SRST_SHIFT 0 +#define HDA_CL_SD_CTL_SRST(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_SRST_SHIFT) + +/* Stream Run */ +#define HDA_CL_SD_CTL_RUN_SHIFT 1 +#define HDA_CL_SD_CTL_RUN(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_RUN_SHIFT) + +/* Interrupt On Completion Enable */ +#define HDA_CL_SD_CTL_IOCE_SHIFT 2 +#define HDA_CL_SD_CTL_IOCE(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_IOCE_SHIFT) + +/* FIFO Error Interrupt Enable */ +#define HDA_CL_SD_CTL_FEIE_SHIFT 3 +#define HDA_CL_SD_CTL_FEIE(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_FEIE_SHIFT) + +/* Descriptor Error Interrupt Enable */ +#define HDA_CL_SD_CTL_DEIE_SHIFT 4 +#define HDA_CL_SD_CTL_DEIE(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_DEIE_SHIFT) + +/* FIFO Limit Change */ +#define HDA_CL_SD_CTL_FIFOLC_SHIFT 5 +#define HDA_CL_SD_CTL_FIFOLC(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_FIFOLC_SHIFT) + +/* Stripe Control */ +#define HDA_CL_SD_CTL_STRIPE_SHIFT 16 +#define HDA_CL_SD_CTL_STRIPE(x) (((x) & 0x3) << \ + HDA_CL_SD_CTL_STRIPE_SHIFT) + +/* Traffic Priority */ +#define HDA_CL_SD_CTL_TP_SHIFT 18 +#define HDA_CL_SD_CTL_TP(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_TP_SHIFT) + +/* Bidirectional Direction Control */ +#define HDA_CL_SD_CTL_DIR_SHIFT 19 +#define HDA_CL_SD_CTL_DIR(x) (((x) & 0x1) << \ + HDA_CL_SD_CTL_DIR_SHIFT) + +/* Stream Number */ +#define HDA_CL_SD_CTL_STRM_SHIFT 20 +#define HDA_CL_SD_CTL_STRM(x) (((x) & 0xf) << \ + HDA_CL_SD_CTL_STRM_SHIFT) + +#define HDA_CL_SD_CTL_INT(x) \ + (HDA_CL_SD_CTL_IOCE(x) | \ + HDA_CL_SD_CTL_FEIE(x) | \ + HDA_CL_SD_CTL_DEIE(x)) + +#define HDA_CL_SD_CTL_INT_MASK \ + (HDA_CL_SD_CTL_IOCE(1) | \ + HDA_CL_SD_CTL_FEIE(1) | \ + HDA_CL_SD_CTL_DEIE(1)) + +#define DMA_ADDRESS_128_BITS_ALIGNMENT 7 +#define BDL_ALIGN(x) ((x) >> DMA_ADDRESS_128_BITS_ALIGNMENT) + +/* Buffer Descriptor List Lower Base Address */ +#define HDA_CL_SD_BDLPLBA_SHIFT 7 +#define HDA_CL_SD_BDLPLBA_MASK GENMASK(31, 7) +#define HDA_CL_SD_BDLPLBA(x) \ + ((BDL_ALIGN(lower_32_bits(x)) << HDA_CL_SD_BDLPLBA_SHIFT) & \ + HDA_CL_SD_BDLPLBA_MASK) + +/* Buffer Descriptor List Upper Base Address */ +#define HDA_CL_SD_BDLPUBA(x) \ + (upper_32_bits(x)) + +/* Software Position in Buffer Enable */ +#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT 0 +#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK \ + (1 << HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT) + +#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(x) \ + (((x) << HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT) & \ + HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK) + +#define HDA_CL_DMA_SD_INT_COMPLETE 0x4 + +static int cl_skl_cldma_setup_bdle(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab_data, + __le32 **bdlp, int size, int with_ioc) +{ + phys_addr_t addr = virt_to_phys(dmab_data->area); + __le32 *bdl = *bdlp; + + /* + * This code is simplified by using one fragment of physical memory and assuming + * all the code fits. This could be improved with scatter-gather but the firmware + * size is limited by DSP memory anyways + */ + bdl[0] = cpu_to_le32(lower_32_bits(addr)); + bdl[1] = cpu_to_le32(upper_32_bits(addr)); + bdl[2] = cpu_to_le32(size); + bdl[3] = (!with_ioc) ? 0 : cpu_to_le32(0x01); + + return 1; /* one fragment */ +} + +static void cl_skl_cldma_stream_run(struct snd_sof_dev *sdev, bool enable) +{ + int sd_offset = SOF_HDA_ADSP_LOADER_BASE; + unsigned char val; + int retries; + u32 run = enable ? 0x1 : 0; + + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL, + HDA_CL_SD_CTL_RUN(1), HDA_CL_SD_CTL_RUN(run)); + + retries = 300; + do { + udelay(3); + + /* waiting for hardware to report the stream Run bit set */ + val = snd_sof_dsp_read(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL); + val &= HDA_CL_SD_CTL_RUN(1); + if (enable && val) + break; + else if (!enable && !val) + break; + } while (--retries); + + if (retries == 0) + dev_err(sdev->dev, "%s: failed to set Run bit=%d enable=%d\n", + __func__, val, enable); +} + +static void cl_skl_cldma_stream_clear(struct snd_sof_dev *sdev) +{ + int sd_offset = SOF_HDA_ADSP_LOADER_BASE; + + /* make sure Run bit is cleared before setting stream register */ + cl_skl_cldma_stream_run(sdev, 0); + + /* Disable the Interrupt On Completion, FIFO Error Interrupt, + * Descriptor Error Interrupt and set the cldma stream number to 0. + */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL, + HDA_CL_SD_CTL_INT_MASK, HDA_CL_SD_CTL_INT(0)); + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL, + HDA_CL_SD_CTL_STRM(0xf), HDA_CL_SD_CTL_STRM(0)); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL, HDA_CL_SD_BDLPLBA(0)); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU, 0); + + /* Set the Cyclic Buffer Length to 0. */ + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL, 0); + /* Set the Last Valid Index. */ + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI, 0); +} + +static void cl_skl_cldma_setup_spb(struct snd_sof_dev *sdev, + unsigned int size, bool enable) +{ + int sd_offset = SOF_DSP_REG_CL_SPBFIFO; + + if (enable) + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL, + HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK, + HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(1)); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPIB, size); +} + +static void cl_skl_cldma_set_intr(struct snd_sof_dev *sdev, bool enable) +{ + u32 val = enable ? HDA_DSP_ADSPIC_CL_DMA : 0; + + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC, + HDA_DSP_ADSPIC_CL_DMA, val); +} + +static void cl_skl_cldma_cleanup_spb(struct snd_sof_dev *sdev) +{ + int sd_offset = SOF_DSP_REG_CL_SPBFIFO; + + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL, + HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK, + HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(0)); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPIB, 0); +} + +static void cl_skl_cldma_setup_controller(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab_bdl, + unsigned int max_size, u32 count) +{ + int sd_offset = SOF_HDA_ADSP_LOADER_BASE; + + /* Clear the stream first and then set it. */ + cl_skl_cldma_stream_clear(sdev); + + /* setting the stream register */ + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL, + HDA_CL_SD_BDLPLBA(dmab_bdl->addr)); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU, + HDA_CL_SD_BDLPUBA(dmab_bdl->addr)); + + /* Set the Cyclic Buffer Length. */ + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL, max_size); + /* Set the Last Valid Index. */ + snd_sof_dsp_write(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI, count - 1); + + /* Set the Interrupt On Completion, FIFO Error Interrupt, + * Descriptor Error Interrupt and the cldma stream number. + */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL, + HDA_CL_SD_CTL_INT_MASK, HDA_CL_SD_CTL_INT(1)); + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL, + HDA_CL_SD_CTL_STRM(0xf), + HDA_CL_SD_CTL_STRM(1)); +} + +static int cl_stream_prepare_skl(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab, + struct snd_dma_buffer *dmab_bdl) + +{ + unsigned int bufsize = HDA_SKL_CLDMA_MAX_BUFFER_SIZE; + __le32 *bdl; + int frags; + int ret; + + ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev, bufsize, dmab); + if (ret < 0) { + dev_err(sdev->dev, "%s: failed to alloc fw buffer: %x\n", __func__, ret); + return ret; + } + + ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev, bufsize, dmab_bdl); + if (ret < 0) { + dev_err(sdev->dev, "%s: failed to alloc blde: %x\n", __func__, ret); + snd_dma_free_pages(dmab); + return ret; + } + + bdl = (__le32 *)dmab_bdl->area; + frags = cl_skl_cldma_setup_bdle(sdev, dmab, &bdl, bufsize, 1); + cl_skl_cldma_setup_controller(sdev, dmab_bdl, bufsize, frags); + + return ret; +} + +static void cl_cleanup_skl(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab, + struct snd_dma_buffer *dmab_bdl) +{ + cl_skl_cldma_cleanup_spb(sdev); + cl_skl_cldma_stream_clear(sdev); + snd_dma_free_pages(dmab); + snd_dma_free_pages(dmab_bdl); +} + +static int cl_dsp_init_skl(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab, + struct snd_dma_buffer *dmab_bdl) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + unsigned int status; + u32 flags; + int ret; + + /* check if the init_core is already enabled, if yes, reset and make it run, + * if not, powerdown and enable it again. + */ + if (hda_dsp_core_is_enabled(sdev, chip->init_core_mask)) { + /* if enabled, reset it, and run the init_core. */ + ret = hda_dsp_core_stall_reset(sdev, chip->init_core_mask); + if (ret < 0) + goto err; + + ret = hda_dsp_core_run(sdev, chip->init_core_mask); + if (ret < 0) { + dev_err(sdev->dev, "%s: dsp core start failed %d\n", __func__, ret); + goto err; + } + } else { + /* if not enabled, power down it first and then powerup and run + * the init_core. + */ + ret = hda_dsp_core_reset_power_down(sdev, chip->init_core_mask); + if (ret < 0) { + dev_err(sdev->dev, "%s: dsp core0 disable fail: %d\n", __func__, ret); + goto err; + } + ret = hda_dsp_enable_core(sdev, chip->init_core_mask); + if (ret < 0) { + dev_err(sdev->dev, "%s: dsp core0 enable fail: %d\n", __func__, ret); + goto err; + } + } + + /* prepare DMA for code loader stream */ + ret = cl_stream_prepare_skl(sdev, dmab, dmab_bdl); + if (ret < 0) { + dev_err(sdev->dev, "%s: dma prepare fw loading err: %x\n", __func__, ret); + return ret; + } + + /* enable the interrupt */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC, + HDA_DSP_ADSPIC_IPC, HDA_DSP_ADSPIC_IPC); + + /* enable IPC DONE interrupt */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl, + HDA_DSP_REG_HIPCCTL_DONE, + HDA_DSP_REG_HIPCCTL_DONE); + + /* enable IPC BUSY interrupt */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl, + HDA_DSP_REG_HIPCCTL_BUSY, + HDA_DSP_REG_HIPCCTL_BUSY); + + /* polling the ROM init status information. */ + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, + chip->rom_status_reg, status, + (FSR_TO_STATE_CODE(status) + == FSR_STATE_INIT_DONE), + HDA_DSP_REG_POLL_INTERVAL_US, + chip->rom_init_timeout * + USEC_PER_MSEC); + if (ret < 0) + goto err; + + return ret; + +err: + flags = SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX; + + snd_sof_dsp_dbg_dump(sdev, "Boot failed\n", flags); + cl_cleanup_skl(sdev, dmab, dmab_bdl); + hda_dsp_core_reset_power_down(sdev, chip->init_core_mask); + return ret; +} + +static void cl_skl_cldma_fill_buffer(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab, + unsigned int bufsize, + unsigned int copysize, + const void *curr_pos, + bool intr_enable) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + + /* copy the image into the buffer with the maximum buffer size. */ + unsigned int size = (bufsize == copysize) ? bufsize : copysize; + + memcpy(dmab->area, curr_pos, size); + + /* Set the wait condition for every load. */ + hda->code_loading = 1; + + /* Set the interrupt. */ + if (intr_enable) + cl_skl_cldma_set_intr(sdev, true); + + /* Set the SPB. */ + cl_skl_cldma_setup_spb(sdev, size, true); + + /* Trigger the code loading stream. */ + cl_skl_cldma_stream_run(sdev, true); +} + +static int cl_skl_cldma_wait_interruptible(struct snd_sof_dev *sdev, + bool intr_wait) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + int sd_offset = SOF_HDA_ADSP_LOADER_BASE; + u8 cl_dma_intr_status; + + /* + * Wait for CLDMA interrupt to inform the binary segment transfer is + * complete. + */ + if (!wait_event_timeout(hda->waitq, !hda->code_loading, + msecs_to_jiffies(HDA_SKL_WAIT_TIMEOUT))) { + dev_err(sdev->dev, "cldma copy timeout\n"); + dev_err(sdev->dev, "ROM code=%#x: FW status=%#x\n", + snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_ROM_ERROR), + snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg)); + return -EIO; + } + + /* now check DMA interrupt status */ + cl_dma_intr_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, + sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS); + + if (!(cl_dma_intr_status & HDA_CL_DMA_SD_INT_COMPLETE)) { + dev_err(sdev->dev, "cldma copy failed\n"); + return -EIO; + } + + dev_dbg(sdev->dev, "cldma buffer copy complete\n"); + return 0; +} + +static int +cl_skl_cldma_copy_to_buf(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab, + const void *bin, + u32 total_size, u32 bufsize) +{ + unsigned int bytes_left = total_size; + const void *curr_pos = bin; + int ret; + + if (total_size <= 0) + return -EINVAL; + + while (bytes_left > 0) { + if (bytes_left > bufsize) { + dev_dbg(sdev->dev, "cldma copy %#x bytes\n", bufsize); + + cl_skl_cldma_fill_buffer(sdev, dmab, bufsize, bufsize, curr_pos, true); + + ret = cl_skl_cldma_wait_interruptible(sdev, false); + if (ret < 0) { + dev_err(sdev->dev, "%s: fw failed to load. %#x bytes remaining\n", + __func__, bytes_left); + return ret; + } + + bytes_left -= bufsize; + curr_pos += bufsize; + } else { + dev_dbg(sdev->dev, "cldma copy %#x bytes\n", bytes_left); + + cl_skl_cldma_set_intr(sdev, false); + cl_skl_cldma_fill_buffer(sdev, dmab, bufsize, bytes_left, curr_pos, false); + return 0; + } + } + + return bytes_left; +} + +static int cl_copy_fw_skl(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmab) + +{ + struct snd_sof_pdata *plat_data = sdev->pdata; + const struct firmware *fw = plat_data->fw; + struct firmware stripped_firmware; + unsigned int bufsize = HDA_SKL_CLDMA_MAX_BUFFER_SIZE; + int ret; + + stripped_firmware.data = plat_data->fw->data + plat_data->fw_offset; + stripped_firmware.size = plat_data->fw->size - plat_data->fw_offset; + + dev_dbg(sdev->dev, "firmware size: %#zx buffer size %#x\n", fw->size, bufsize); + + ret = cl_skl_cldma_copy_to_buf(sdev, dmab, stripped_firmware.data, + stripped_firmware.size, bufsize); + if (ret < 0) + dev_err(sdev->dev, "%s: fw copy failed %d\n", __func__, ret); + + return ret; +} + +int hda_dsp_cl_boot_firmware_skl(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + struct snd_dma_buffer dmab_bdl; + struct snd_dma_buffer dmab; + unsigned int reg; + u32 flags; + int ret; + + ret = cl_dsp_init_skl(sdev, &dmab, &dmab_bdl); + + /* retry enabling core and ROM load. seemed to help */ + if (ret < 0) { + ret = cl_dsp_init_skl(sdev, &dmab, &dmab_bdl); + if (ret < 0) { + dev_err(sdev->dev, "Error code=%#x: FW status=%#x\n", + snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_ROM_ERROR), + snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg)); + dev_err(sdev->dev, "Core En/ROM load fail:%d\n", ret); + return ret; + } + } + + dev_dbg(sdev->dev, "ROM init successful\n"); + + /* at this point DSP ROM has been initialized and should be ready for + * code loading and firmware boot + */ + ret = cl_copy_fw_skl(sdev, &dmab); + if (ret < 0) { + dev_err(sdev->dev, "%s: load firmware failed : %d\n", __func__, ret); + goto err; + } + + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, + chip->rom_status_reg, reg, + (FSR_TO_STATE_CODE(reg) + == FSR_STATE_ROM_BASEFW_ENTERED), + HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_BASEFW_TIMEOUT_US); + + dev_dbg(sdev->dev, "Firmware download successful, booting...\n"); + + cl_skl_cldma_stream_run(sdev, false); + cl_cleanup_skl(sdev, &dmab, &dmab_bdl); + + if (!ret) + return chip->init_core_mask; + + return ret; + +err: + flags = SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX; + + snd_sof_dsp_dbg_dump(sdev, "Boot failed\n", flags); + + /* power down DSP */ + hda_dsp_core_reset_power_down(sdev, chip->init_core_mask); + cl_skl_cldma_stream_run(sdev, false); + cl_cleanup_skl(sdev, &dmab, &dmab_bdl); + + dev_err(sdev->dev, "%s: load fw failed err: %d\n", __func__, ret); + return ret; +} diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c index eb22eb3f6fee17fc38f53f77e8e1383a22fd2fcf..98812d51b31c8eb212735d99efd44237e47db9ba 100644 --- a/sound/soc/sof/intel/hda-loader.c +++ b/sound/soc/sof/intel/hda-loader.c @@ -177,14 +177,13 @@ int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot) * - IMR boot: wait for ROM firmware entered (firmware booted up from IMR) */ if (imr_boot) - target_status = HDA_DSP_ROM_FW_ENTERED; + target_status = FSR_STATE_FW_ENTERED; else - target_status = HDA_DSP_ROM_INIT; + target_status = FSR_STATE_INIT_DONE; ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, chip->rom_status_reg, status, - ((status & HDA_DSP_ROM_STS_MASK) - == target_status), + (FSR_TO_STATE_CODE(status) == target_status), HDA_DSP_REG_POLL_INTERVAL_US, chip->rom_init_timeout * USEC_PER_MSEC); @@ -292,8 +291,7 @@ int hda_cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream status = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, chip->rom_status_reg, reg, - ((reg & HDA_DSP_ROM_STS_MASK) - == HDA_DSP_ROM_FW_ENTERED), + (FSR_TO_STATE_CODE(reg) == FSR_STATE_FW_ENTERED), HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_BASEFW_TIMEOUT_US); diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c index 6888e0a4665d2d06be1d75719f37983c5d99c83f..0a9c80216a8c29d32878026e31f7169b33830b82 100644 --- a/sound/soc/sof/intel/hda-pcm.c +++ b/sound/soc/sof/intel/hda-pcm.c @@ -18,6 +18,7 @@ #include <linux/moduleparam.h> #include <sound/hda_register.h> #include <sound/pcm_params.h> +#include <trace/events/sof_intel.h> #include "../sof-audio.h" #include "../ops.h" #include "hda.h" @@ -196,8 +197,7 @@ snd_pcm_uframes_t hda_dsp_pcm_pointer(struct snd_sof_dev *sdev, found: pos = bytes_to_frames(substream->runtime, pos); - dev_vdbg(sdev->dev, "PCM: stream %d dir %d position %lu\n", - hstream->index, substream->stream, pos); + trace_sof_intel_hda_dsp_pcm(sdev, hstream, substream, pos); return pos; } diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c index b58662faa4aad530a80b848d3f5ee00ed66a8612..be60e7785da940cd0bbb5ef1b45d30be23a1f4ff 100644 --- a/sound/soc/sof/intel/hda-stream.c +++ b/sound/soc/sof/intel/hda-stream.c @@ -19,6 +19,7 @@ #include <sound/hdaudio_ext.h> #include <sound/hda_register.h> #include <sound/sof.h> +#include <trace/events/sof_intel.h> #include "../ops.h" #include "../sof-audio.h" #include "hda.h" @@ -93,9 +94,6 @@ static int hda_setup_bdle(struct snd_sof_dev *sdev, bdl++; hstream->frags++; offset += chunk; - - dev_vdbg(sdev->dev, "bdl, frags:%d, chunk size:0x%x;\n", - hstream->frags, chunk); } *bdlp = bdl; @@ -700,7 +698,7 @@ bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev) spin_lock_irq(&bus->reg_lock); status = snd_hdac_chip_readl(bus, INTSTS); - dev_vdbg(bus->dev, "stream irq, INTSTS status: 0x%x\n", status); + trace_sof_intel_hda_dsp_check_stream_irq(sdev, status); /* if Register inaccessible, ignore it.*/ if (status != 0xffffffff) @@ -739,8 +737,7 @@ static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status) if (status & BIT(s->index) && s->opened) { sd_status = snd_hdac_stream_readb(s, SD_STS); - dev_vdbg(bus->dev, "stream %d status 0x%x\n", - s->index, sd_status); + trace_sof_intel_hda_dsp_stream_status(bus->dev, s, sd_status); snd_hdac_stream_writeb(s, SD_STS, sd_status); diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 6d4ecbe14adf31583f1bfa73742f426bb5a618ef..1188ec51816bd38217fd5b8a84231a75f2a3e3d3 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -31,6 +31,9 @@ #include "../ops.h" #include "hda.h" +#define CREATE_TRACE_POINTS +#include <trace/events/sof_intel.h> + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) #include <sound/soc-acpi-intel-match.h> #endif @@ -376,6 +379,10 @@ static int dmic_num_override = -1; module_param_named(dmic_num, dmic_num_override, int, 0444); MODULE_PARM_DESC(dmic_num, "SOF HDA DMIC number"); +static int mclk_id_override = -1; +module_param_named(mclk_id, mclk_id_override, int, 0444); +MODULE_PARM_DESC(mclk_id, "SOF SSP mclk_id"); + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) static bool hda_codec_use_common_hdmi = IS_ENABLED(CONFIG_SND_HDA_CODEC_HDMI); module_param_named(use_common_hdmi, hda_codec_use_common_hdmi, bool, 0444); @@ -591,7 +598,8 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags) /* print ROM/FW status */ hda_dsp_get_state(sdev, level); - if (flags & SOF_DBG_DUMP_REGS) { + /* The firmware register dump only available with IPC3 */ + if (flags & SOF_DBG_DUMP_REGS && sdev->pdata->ipc_type == SOF_IPC) { u32 status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_STATUS); u32 panic = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_TRACEP); @@ -655,6 +663,24 @@ void hda_ipc_dump(struct snd_sof_dev *sdev) hipcie, hipct, hipcctl); } +void hda_ipc4_dump(struct snd_sof_dev *sdev) +{ + u32 hipci, hipcie, hipct, hipcte, hipcctl; + + hda_ipc_irq_dump(sdev); + + hipci = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCI); + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCIE); + hipct = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCT); + hipcte = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCTE); + hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL); + + /* dump the IPC regs */ + /* TODO: parse the raw msg */ + dev_err(sdev->dev, "Host IPC initiator: %#x|%#x, target: %#x|%#x, ctl: %#x\n", + hipci, hipcie, hipct, hipcte, hipcctl); +} + static int hda_init(struct snd_sof_dev *sdev) { struct hda_bus *hbus; @@ -749,6 +775,18 @@ static int check_nhlt_ssp_mask(struct snd_sof_dev *sdev) return ssp_mask; } +static int check_nhlt_ssp_mclk_mask(struct snd_sof_dev *sdev, int ssp_num) +{ + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; + struct nhlt_acpi_table *nhlt; + + nhlt = hdev->nhlt; + if (!nhlt) + return 0; + + return intel_nhlt_ssp_mclk_mask(nhlt, ssp_num); +} + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) || IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE) static const char *fixup_tplg_name(struct snd_sof_dev *sdev, @@ -938,17 +976,25 @@ static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context) struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; /* deal with streams and controller first */ - if (hda_dsp_check_stream_irq(sdev)) + if (hda_dsp_check_stream_irq(sdev)) { + trace_sof_intel_hda_irq(sdev, "stream"); hda_dsp_stream_threaded_handler(irq, sdev); + } - if (hda_check_ipc_irq(sdev)) + if (hda_check_ipc_irq(sdev)) { + trace_sof_intel_hda_irq(sdev, "ipc"); sof_ops(sdev)->irq_thread(irq, sdev); + } - if (hda_dsp_check_sdw_irq(sdev)) + if (hda_dsp_check_sdw_irq(sdev)) { + trace_sof_intel_hda_irq(sdev, "sdw"); hda_dsp_sdw_thread(irq, hdev->sdw); + } - if (hda_sdw_check_wakeen_irq(sdev)) + if (hda_sdw_check_wakeen_irq(sdev)) { + trace_sof_intel_hda_irq(sdev, "wakeen"); hda_sdw_process_wakeen(sdev); + } hda_check_for_state_change(sdev); @@ -1109,6 +1155,8 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) INIT_DELAYED_WORK(&hdev->d0i3_work, hda_dsp_d0i3_work); + init_waitqueue_head(&hdev->waitq); + hdev->nhlt = intel_nhlt_init(sdev->dev); return 0; @@ -1162,9 +1210,9 @@ int hda_dsp_remove(struct snd_sof_dev *sdev) snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL, SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN, 0); - /* disable cores */ - if (chip) - hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask); + /* no need to check for error as the DSP will be disabled anyway */ + if (chip && chip->power_down_dsp) + chip->power_down_dsp(sdev); /* disable DSP */ snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL, @@ -1190,6 +1238,14 @@ int hda_dsp_remove(struct snd_sof_dev *sdev) return 0; } +int hda_power_down_dsp(struct snd_sof_dev *sdev) +{ + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; + + return hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask); +} + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) static void hda_generic_machine_select(struct snd_sof_dev *sdev, struct snd_soc_acpi_mach **mach) @@ -1529,6 +1585,7 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) mach->mach_params.i2s_link_mask) { const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata); int ssp_num; + int mclk_mask; if (hweight_long(mach->mach_params.i2s_link_mask) > 1 && !(mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_SSP_MSB)) @@ -1553,6 +1610,21 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) sof_pdata->tplg_filename = tplg_filename; add_extension = true; + + mclk_mask = check_nhlt_ssp_mclk_mask(sdev, ssp_num); + + if (mclk_mask < 0) { + dev_err(sdev->dev, "Invalid MCLK configuration\n"); + return NULL; + } + + dev_dbg(sdev->dev, "MCLK mask %#x found in NHLT\n", mclk_mask); + + if (mclk_mask) { + dev_info(sdev->dev, "Overriding topology with MCLK mask %#x from NHLT\n", mclk_mask); + sdev->mclk_id_override = true; + sdev->mclk_id_quirk = (mclk_mask & BIT(0)) ? 0 : 1; + } } if (tplg_fixup && add_extension) { @@ -1565,6 +1637,13 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) sof_pdata->tplg_filename = tplg_filename; } + + /* check if mclk_id should be modified from topology defaults */ + if (mclk_id_override >= 0) { + dev_info(sdev->dev, "Overriding topology with MCLK %d from kernel_parameter\n", mclk_id_override); + sdev->mclk_id_override = true; + sdev->mclk_id_quirk = mclk_id_override; + } } /* diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 5ef3e8775e364172b2dd0f69e057e3480872250e..2ab3c3840b926925803d39a89b1b9a07dff76047 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -229,6 +229,7 @@ #define FSR_STATE_ROM_GET_LOAD_OFFSET 0x7 #define FSR_STATE_ROM_FETCH_ROM_EXT 0x8 #define FSR_STATE_ROM_FETCH_ROM_EXT_DONE 0x9 +#define FSR_STATE_ROM_BASEFW_ENTERED 0xf /* SKL */ /* (ROM) CSE states */ #define FSR_STATE_ROM_CSE_IMR_REQUEST 0x10 @@ -251,12 +252,6 @@ #define FSR_STATE_BRINGUP_FW_ENTERED FSR_STATE_FW_ENTERED /* ROM status/error values */ -#define HDA_DSP_ROM_STS_MASK GENMASK(23, 0) -#define HDA_DSP_ROM_INIT 0x1 -#define HDA_DSP_ROM_FW_MANIFEST_LOADED 0x3 -#define HDA_DSP_ROM_FW_FW_LOADED 0x4 -#define HDA_DSP_ROM_FW_ENTERED 0x5 -#define HDA_DSP_ROM_RFW_START 0xf #define HDA_DSP_ROM_CSE_ERROR 40 #define HDA_DSP_ROM_CSE_WRONG_RESPONSE 41 #define HDA_DSP_ROM_IMR_TO_SMALL 42 @@ -424,6 +419,7 @@ #endif /* Intel HD Audio SRAM Window 0*/ +#define HDA_DSP_SRAM_REG_ROM_STATUS_SKL 0x8000 #define HDA_ADSP_SRAM0_BASE_SKL 0x8000 /* Firmware status window */ @@ -441,6 +437,8 @@ #define APL_SSP_COUNT 6 #define CNL_SSP_COUNT 3 #define ICL_SSP_COUNT 6 +#define TGL_SSP_COUNT 3 +#define MTL_SSP_COUNT 3 /* SSP Registers */ #define SSP_SSC1_OFFSET 0x4 @@ -518,6 +516,9 @@ struct sof_intel_hda_dev { /* FW clock config, 0:HPRO, 1:LPRO */ bool clk_config_lpro; + wait_queue_head_t waitq; + bool code_loading; + /* Intel NHLT information */ struct nhlt_acpi_table *nhlt; }; @@ -566,9 +567,11 @@ int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_reset_power_down(struct snd_sof_dev *sdev, unsigned int core_mask); +int hda_power_down_dsp(struct snd_sof_dev *sdev); int hda_dsp_core_get(struct snd_sof_dev *sdev, int core); void hda_dsp_ipc_int_enable(struct snd_sof_dev *sdev); void hda_dsp_ipc_int_disable(struct snd_sof_dev *sdev); +bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_set_power_state(struct snd_sof_dev *sdev, const struct sof_dsp_power_state *target_state); @@ -584,6 +587,7 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags); void hda_ipc_dump(struct snd_sof_dev *sdev); void hda_ipc_irq_dump(struct snd_sof_dev *sdev); void hda_dsp_d0i3_work(struct work_struct *work); +int hda_dsp_disable_interrupts(struct snd_sof_dev *sdev); /* * DSP PCM Operations. @@ -773,6 +777,8 @@ int hda_dsp_dais_suspend(struct snd_sof_dev *sdev); */ extern struct snd_sof_dsp_ops sof_hda_common_ops; +extern struct snd_sof_dsp_ops sof_skl_ops; +int sof_skl_ops_init(struct snd_sof_dev *sdev); extern struct snd_sof_dsp_ops sof_apl_ops; int sof_apl_ops_init(struct snd_sof_dev *sdev); extern struct snd_sof_dsp_ops sof_cnl_ops; @@ -784,6 +790,7 @@ int sof_icl_ops_init(struct snd_sof_dev *sdev); extern struct snd_sof_dsp_ops sof_mtl_ops; int sof_mtl_ops_init(struct snd_sof_dev *sdev); +extern const struct sof_intel_dsp_desc skl_chip_info; extern const struct sof_intel_dsp_desc apl_chip_info; extern const struct sof_intel_dsp_desc cnl_chip_info; extern const struct sof_intel_dsp_desc icl_chip_info; @@ -837,11 +844,16 @@ extern int sof_hda_position_quirk; void hda_set_dai_drv_ops(struct snd_sof_dev *sdev, struct snd_sof_dsp_ops *ops); void hda_ops_free(struct snd_sof_dev *sdev); +/* SKL/KBL */ +int hda_dsp_cl_boot_firmware_skl(struct snd_sof_dev *sdev); +int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask); + /* IPC4 */ irqreturn_t cnl_ipc4_irq_thread(int irq, void *context); int cnl_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); irqreturn_t hda_dsp_ipc4_irq_thread(int irq, void *context); int hda_dsp_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); +void hda_ipc4_dump(struct snd_sof_dev *sdev); extern struct sdw_intel_ops sdw_callback; #endif diff --git a/sound/soc/sof/intel/icl.c b/sound/soc/sof/intel/icl.c index 4e37b7fe06274db5356be18e5056a5170eba82ca..6d5877108a3dcffb999f1751f7260ef816b84872 100644 --- a/sound/soc/sof/intel/icl.c +++ b/sound/soc/sof/intel/icl.c @@ -13,6 +13,7 @@ #include <linux/kconfig.h> #include <linux/export.h> #include <linux/bits.h> +#include "../ipc4-priv.h" #include "../ops.h" #include "hda.h" #include "hda-ipc.h" @@ -106,16 +107,42 @@ int sof_icl_ops_init(struct snd_sof_dev *sdev) /* probe/remove/shutdown */ sof_icl_ops.shutdown = hda_dsp_shutdown; - /* doorbell */ - sof_icl_ops.irq_thread = cnl_ipc_irq_thread; + if (sdev->pdata->ipc_type == SOF_IPC) { + /* doorbell */ + sof_icl_ops.irq_thread = cnl_ipc_irq_thread; - /* ipc */ - sof_icl_ops.send_msg = cnl_ipc_send_msg; + /* ipc */ + sof_icl_ops.send_msg = cnl_ipc_send_msg; + + /* debug */ + sof_icl_ops.ipc_dump = cnl_ipc_dump; + } + + if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_fw_data *ipc4_data; + + sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; + + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2; + + /* doorbell */ + sof_icl_ops.irq_thread = cnl_ipc4_irq_thread; + + /* ipc */ + sof_icl_ops.send_msg = cnl_ipc4_send_msg; + + /* debug */ + sof_icl_ops.ipc_dump = cnl_ipc4_dump; + } /* debug */ sof_icl_ops.debug_map = icl_dsp_debugfs; sof_icl_ops.debug_map_count = ARRAY_SIZE(icl_dsp_debugfs); - sof_icl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ sof_icl_ops.post_fw_run = icl_dsp_post_fw_run; @@ -153,6 +180,8 @@ const struct sof_intel_dsp_desc icl_chip_info = { .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_0, }; EXPORT_SYMBOL_NS(icl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/mtl.c b/sound/soc/sof/intel/mtl.c index 96239ebb1eedb16827dcd0077e78977fec754f85..10298532816feaff62f209d70fb916cf6a4a0ab3 100644 --- a/sound/soc/sof/intel/mtl.c +++ b/sound/soc/sof/intel/mtl.c @@ -11,6 +11,7 @@ #include <linux/firmware.h> #include <sound/sof/ipc4/header.h> +#include <trace/events/sof_intel.h> #include "../ipc4-priv.h" #include "../ops.h" #include "hda.h" @@ -63,7 +64,7 @@ static bool mtl_dsp_check_ipc_irq(struct snd_sof_dev *sdev) hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK; irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr + MTL_DSP_IRQSTS); - dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status); + trace_sof_intel_hda_irq_ipc_check(sdev, irq_status); if (irq_status != U32_MAX && (irq_status & MTL_DSP_IRQSTS_IPC)) return true; @@ -143,7 +144,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev) /* check if operation was successful */ host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK; - irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten, (irqinten & host_ipc) == host_ipc, HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); @@ -158,7 +158,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev) /* check if operation was successful */ host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK; - hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie, (hipcie & host_ipc) == host_ipc, HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); @@ -170,7 +169,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev) snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, MTL_DSP_REG_HfSNDWIE_IE_MASK, MTL_DSP_REG_HfSNDWIE_IE_MASK); host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK; - hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie, (hipcie & host_ipc) == host_ipc, HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); @@ -198,7 +196,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev) /* check if operation was successful */ host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK; - irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten, (irqinten & host_ipc) == 0, HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); @@ -212,7 +209,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev) /* check if operation was successful */ host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK; - hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE); ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie, (hipcie & host_ipc) == 0, HDA_DSP_REG_POLL_INTERVAL_US, @@ -227,7 +223,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev) snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, MTL_DSP_REG_HfSNDWIE_IE_MASK, 0); host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK; - hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE); ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie, (hipcie & host_ipc) == 0, HDA_DSP_REG_POLL_INTERVAL_US, @@ -259,7 +254,6 @@ static int mtl_dsp_pre_fw_run(struct snd_sof_dev *sdev) /* poll with timeout to check if operation successful */ cpa = MTL_HFDSSCS_CPA_MASK; - dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs, (dsphfdsscs & cpa) == cpa, HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US); @@ -276,7 +270,6 @@ static int mtl_dsp_pre_fw_run(struct snd_sof_dev *sdev) /* poll with timeout to check if operation successful */ pgs = MTL_HFPWRSTS_DSPHPXPGS_MASK; - dsphfpwrsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFPWRSTS); ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFPWRSTS, dsphfpwrsts, (dsphfpwrsts & pgs) == pgs, HDA_DSP_REG_POLL_INTERVAL_US, @@ -405,6 +398,33 @@ static int mtl_dsp_core_power_down(struct snd_sof_dev *sdev, int core) return ret; } +static int mtl_power_down_dsp(struct snd_sof_dev *sdev) +{ + u32 dsphfdsscs, cpa; + int ret; + + /* first power down core */ + ret = mtl_dsp_core_power_down(sdev, SOF_DSP_PRIMARY_CORE); + if (ret) { + dev_err(sdev->dev, "mtl dsp power down error, %d\n", ret); + return ret; + } + + /* Set the DSP subsystem power down */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS, + MTL_HFDSSCS_SPA_MASK, 0); + + /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ + usleep_range(1000, 1010); + + /* poll with timeout to check if operation successful */ + cpa = MTL_HFDSSCS_CPA_MASK; + dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS); + return snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs, + (dsphfdsscs & cpa) == 0, HDA_DSP_REG_POLL_INTERVAL_US, + HDA_DSP_RESET_TIMEOUT_US); +} + static int mtl_dsp_cl_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; @@ -551,166 +571,27 @@ static int mtl_dsp_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id) return MTL_SRAM_WINDOW_OFFSET(id); } -static int mtl_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) -{ - struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; - const struct sof_intel_dsp_desc *chip = hda->desc; -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - struct hdac_bus *bus = sof_to_bus(sdev); -#endif - u32 dsphfdsscs; - u32 cpa; - int ret; - int i; - - mtl_disable_ipc_interrupts(sdev); - ret = mtl_disable_interrupts(sdev); - if (ret) - return ret; - -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - hda_codec_jack_wake_enable(sdev, runtime_suspend); - /* power down all hda link */ - snd_hdac_ext_bus_link_power_down_all(bus); -#endif - snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFPWRCTL, - MTL_HFPWRCTL_WPDSPHPXPG, 0); - - /* Set the DSP subsystem power down */ - snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS, - MTL_HFDSSCS_SPA_MASK, 0); - - /* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */ - usleep_range(1000, 1010); - - /* poll with timeout to check if operation successful */ - cpa = MTL_HFDSSCS_CPA_MASK; - dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS); - ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs, - (dsphfdsscs & cpa) == 0, HDA_DSP_REG_POLL_INTERVAL_US, - HDA_DSP_RESET_TIMEOUT_US); - if (ret < 0) - dev_err(sdev->dev, "failed to disable DSP subsystem\n"); - - /* reset ref counts for all cores */ - for (i = 0; i < chip->cores_num; i++) - sdev->dsp_core_ref_count[i] = 0; - - /* TODO: need to reset controller? */ - - /* display codec can be powered off after link reset */ - hda_codec_i915_display_power(sdev, false); - - return 0; -} - -static int mtl_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state) -{ - const struct sof_dsp_power_state target_dsp_state = { - .state = target_state, - .substate = target_state == SOF_DSP_PM_D0 ? - SOF_HDA_DSP_PM_D0I3 : 0, - }; - int ret; - - ret = mtl_suspend(sdev, false); - if (ret < 0) - return ret; - - return snd_sof_dsp_set_power_state(sdev, &target_dsp_state); -} - -static int mtl_dsp_runtime_suspend(struct snd_sof_dev *sdev) -{ - const struct sof_dsp_power_state target_state = { - .state = SOF_DSP_PM_D3, - }; - int ret; - - ret = mtl_suspend(sdev, true); - if (ret < 0) - return ret; - - return snd_sof_dsp_set_power_state(sdev, &target_state); -} - -static int mtl_resume(struct snd_sof_dev *sdev, bool runtime_resume) -{ -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - struct hdac_bus *bus = sof_to_bus(sdev); - struct hdac_ext_link *hlink = NULL; -#endif - - /* display codec must be powered before link reset */ - hda_codec_i915_display_power(sdev, true); - -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - /* check jack status */ - if (runtime_resume) { - hda_codec_jack_wake_enable(sdev, false); - if (sdev->system_suspend_target == SOF_SUSPEND_NONE) - hda_codec_jack_check(sdev); - } - - /* turn off the links that were off before suspend */ - list_for_each_entry(hlink, &bus->hlink_list, list) { - if (!hlink->ref_count) - snd_hdac_ext_bus_link_power_down(hlink); - } - - /* check dma status and clean up CORB/RIRB buffers */ - if (!bus->cmd_dma_state) - snd_hdac_bus_stop_cmd_io(bus); -#endif - - return 0; -} - -static int mtl_dsp_resume(struct snd_sof_dev *sdev) -{ - const struct sof_dsp_power_state target_state = { - .state = SOF_DSP_PM_D0, - .substate = SOF_HDA_DSP_PM_D0I0, - }; - int ret; - - ret = mtl_resume(sdev, false); - if (ret < 0) - return ret; - - return snd_sof_dsp_set_power_state(sdev, &target_state); -} - -static int mtl_dsp_runtime_resume(struct snd_sof_dev *sdev) -{ - const struct sof_dsp_power_state target_state = { - .state = SOF_DSP_PM_D0, - }; - int ret; - - ret = mtl_resume(sdev, true); - if (ret < 0) - return ret; - - return snd_sof_dsp_set_power_state(sdev, &target_state); -} - static void mtl_ipc_dump(struct snd_sof_dev *sdev) { - u32 hipcctl; - u32 hipcida; - u32 hipctdr; + u32 hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl; - /* read IPC status */ + hipcidr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDR); + hipcidd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDDY); hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDA); - hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL); hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDR); + hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDDY); + hipctda = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDA); + hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL); - /* dump the IPC regs */ - /* TODO: parse the raw msg */ dev_err(sdev->dev, - "error: host status 0x%8.8x dsp status 0x%8.8x mask 0x%8.8x\n", - hipcida, hipctdr, hipcctl); + "Host IPC initiator: %#x|%#x|%#x, target: %#x|%#x|%#x, ctl: %#x\n", + hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl); +} + +static int mtl_dsp_disable_interrupts(struct snd_sof_dev *sdev) +{ + mtl_disable_ipc_interrupts(sdev); + return mtl_disable_interrupts(sdev); } /* Meteorlake ops */ @@ -751,12 +632,6 @@ int sof_mtl_ops_init(struct snd_sof_dev *sdev) /* dsp core get/put */ /* TODO: add core_get and core_put */ - /* PM */ - sof_mtl_ops.suspend = mtl_dsp_suspend; - sof_mtl_ops.resume = mtl_dsp_resume; - sof_mtl_ops.runtime_suspend = mtl_dsp_runtime_suspend; - sof_mtl_ops.runtime_resume = mtl_dsp_runtime_resume; - sdev->private = devm_kzalloc(sdev->dev, sizeof(struct sof_ipc4_fw_data), GFP_KERNEL); if (!sdev->private) return -ENOMEM; @@ -764,6 +639,8 @@ int sof_mtl_ops_init(struct snd_sof_dev *sdev) ipc4_data = sdev->private; ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2; + /* set DAI ops */ hda_set_dai_drv_ops(sdev, &sof_mtl_ops); @@ -782,13 +659,15 @@ const struct sof_intel_dsp_desc mtl_chip_info = { .ipc_ctl = MTL_DSP_REG_HFIPCXCTL, .rom_status_reg = MTL_DSP_ROM_STS, .rom_init_timeout = 300, - .ssp_count = ICL_SSP_COUNT, + .ssp_count = MTL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE_ACE, .sdw_alh_base = SDW_ALH_BASE_ACE, .check_sdw_irq = mtl_dsp_check_sdw_irq, .check_ipc_irq = mtl_dsp_check_ipc_irq, .cl_init = mtl_dsp_cl_init, + .power_down_dsp = mtl_power_down_dsp, + .disable_interrupts = mtl_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_ACE_1_0, }; EXPORT_SYMBOL_NS(mtl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/pci-skl.c b/sound/soc/sof/intel/pci-skl.c new file mode 100644 index 0000000000000000000000000000000000000000..3a99dc444f92ea6b46014a4faaf9463fe4421c29 --- /dev/null +++ b/sound/soc/sof/intel/pci-skl.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2018-2022 Intel Corporation. All rights reserved. +// + +#include <linux/module.h> +#include <linux/pci.h> +#include <sound/soc-acpi.h> +#include <sound/soc-acpi-intel-match.h> +#include <sound/sof.h> +#include "../ops.h" +#include "../sof-pci-dev.h" + +/* platform specific devices */ +#include "hda.h" + +static struct sof_dev_desc skl_desc = { + .machines = snd_soc_acpi_intel_skl_machines, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .chip_info = &skl_chip_info, + .irqindex_host_ipc = -1, + .ipc_supported_mask = BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_INTEL_IPC4, + .default_fw_path = { + [SOF_INTEL_IPC4] = "intel/avs/skl", + }, + .default_tplg_path = { + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-skl-nocodec.tplg", + .ops = &sof_skl_ops, + .ops_init = sof_skl_ops_init, +}; + +static struct sof_dev_desc kbl_desc = { + .machines = snd_soc_acpi_intel_kbl_machines, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .chip_info = &skl_chip_info, + .irqindex_host_ipc = -1, + .ipc_supported_mask = BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_INTEL_IPC4, + .default_fw_path = { + [SOF_INTEL_IPC4] = "intel/avs/kbl", + }, + .default_tplg_path = { + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-kbl-nocodec.tplg", + .ops = &sof_skl_ops, + .ops_init = sof_skl_ops_init, +}; + +/* PCI IDs */ +static const struct pci_device_id sof_pci_ids[] = { + /* Sunrise Point-LP */ + { PCI_DEVICE(0x8086, 0x9d70), .driver_data = (unsigned long)&skl_desc}, + /* KBL */ + { PCI_DEVICE(0x8086, 0x9d71), .driver_data = (unsigned long)&kbl_desc}, + { 0, } +}; +MODULE_DEVICE_TABLE(pci, sof_pci_ids); + +/* pci_driver definition */ +static struct pci_driver snd_sof_pci_intel_skl_driver = { + .name = "sof-audio-pci-intel-skl", + .id_table = sof_pci_ids, + .probe = hda_pci_intel_probe, + .remove = sof_pci_remove, + .shutdown = sof_pci_shutdown, + .driver = { + .pm = &sof_pci_pm, + }, +}; +module_pci_driver(snd_sof_pci_intel_skl_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_IMPORT_NS(SND_SOC_SOF_INTEL_HDA_COMMON); +MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV); diff --git a/sound/soc/sof/intel/pci-tgl.c b/sound/soc/sof/intel/pci-tgl.c index ccc44ba3ad94d767123b7b873e973600b91f92c5..2d63cc236a68e8d9cc99d325b7fcf1d5b10f4e9f 100644 --- a/sound/soc/sof/intel/pci-tgl.c +++ b/sound/soc/sof/intel/pci-tgl.c @@ -159,6 +159,62 @@ static const struct sof_dev_desc adl_desc = { .ops_init = sof_tgl_ops_init, }; +static const struct sof_dev_desc rpls_desc = { + .machines = snd_soc_acpi_intel_rpl_machines, + .alt_machines = snd_soc_acpi_intel_rpl_sdw_machines, + .use_acpi_target_states = true, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &adls_chip_info, + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/rpl-s", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-rpl-s.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-rpl-nocodec.tplg", + .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, +}; + +static const struct sof_dev_desc rpl_desc = { + .machines = snd_soc_acpi_intel_rpl_machines, + .alt_machines = snd_soc_acpi_intel_rpl_sdw_machines, + .use_acpi_target_states = true, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &tgl_chip_info, + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/rpl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-rpl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-rpl-nocodec.tplg", + .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, +}; + /* PCI IDs */ static const struct pci_device_id sof_pci_ids[] = { { PCI_DEVICE(0x8086, 0xa0c8), /* TGL-LP */ @@ -172,19 +228,23 @@ static const struct pci_device_id sof_pci_ids[] = { { PCI_DEVICE(0x8086, 0x7ad0), /* ADL-S */ .driver_data = (unsigned long)&adls_desc}, { PCI_DEVICE(0x8086, 0x7a50), /* RPL-S */ - .driver_data = (unsigned long)&adls_desc}, + .driver_data = (unsigned long)&rpls_desc}, { PCI_DEVICE(0x8086, 0x51c8), /* ADL-P */ .driver_data = (unsigned long)&adl_desc}, - { PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */ - .driver_data = (unsigned long)&adl_desc}, { PCI_DEVICE(0x8086, 0x51c9), /* ADL-PS */ .driver_data = (unsigned long)&adl_desc}, { PCI_DEVICE(0x8086, 0x51ca), /* RPL-P */ - .driver_data = (unsigned long)&adl_desc}, + .driver_data = (unsigned long)&rpl_desc}, { PCI_DEVICE(0x8086, 0x51cb), /* RPL-P */ - .driver_data = (unsigned long)&adl_desc}, + .driver_data = (unsigned long)&rpl_desc}, { PCI_DEVICE(0x8086, 0x51cc), /* ADL-M */ .driver_data = (unsigned long)&adl_desc}, + { PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */ + .driver_data = (unsigned long)&adl_desc}, + { PCI_DEVICE(0x8086, 0x51ce), /* RPL-M */ + .driver_data = (unsigned long)&rpl_desc}, + { PCI_DEVICE(0x8086, 0x51cf), /* RPL-PX */ + .driver_data = (unsigned long)&rpl_desc}, { PCI_DEVICE(0x8086, 0x54c8), /* ADL-N */ .driver_data = (unsigned long)&adl_desc}, { 0, } diff --git a/sound/soc/sof/intel/shim.h b/sound/soc/sof/intel/shim.h index 638159bee864515e93c75829cff7561b4c36c814..3ceba5c39317d66573cca91b1d2be78f1d0eccbf 100644 --- a/sound/soc/sof/intel/shim.h +++ b/sound/soc/sof/intel/shim.h @@ -186,6 +186,8 @@ struct sof_intel_dsp_desc { enum sof_intel_hw_ip_version hw_ip_version; bool (*check_sdw_irq)(struct snd_sof_dev *sdev); bool (*check_ipc_irq)(struct snd_sof_dev *sdev); + int (*power_down_dsp)(struct snd_sof_dev *sdev); + int (*disable_interrupts)(struct snd_sof_dev *sdev); int (*cl_init)(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot); }; diff --git a/sound/soc/sof/intel/skl.c b/sound/soc/sof/intel/skl.c new file mode 100644 index 0000000000000000000000000000000000000000..13efdb94d071d2b3b780a7f856b22fbc3d5b502c --- /dev/null +++ b/sound/soc/sof/intel/skl.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +// +// This file is provided under a dual BSD/GPLv2 license. When using or +// redistributing this file, you may do so under either license. +// +// Copyright(c) 2018-2022 Intel Corporation. All rights reserved. +// + +/* + * Hardware interface for audio DSP on Skylake and Kabylake. + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/firmware.h> +#include <linux/fs.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/pci.h> +#include <linux/pm_runtime.h> +#include <sound/hdaudio_ext.h> +#include <sound/pcm_params.h> +#include <sound/sof.h> +#include <sound/sof/ext_manifest4.h> + +#include "../sof-priv.h" +#include "../ipc4-priv.h" +#include "../ops.h" +#include "hda.h" +#include "../sof-audio.h" + +#define SRAM_MEMORY_WINDOW_BASE 0x8000 + +static const __maybe_unused struct snd_sof_debugfs_map skl_dsp_debugfs[] = { + {"hda", HDA_DSP_HDA_BAR, 0, 0x4000}, + {"pp", HDA_DSP_PP_BAR, 0, 0x1000}, + {"dsp", HDA_DSP_BAR, 0, 0x10000}, +}; + +static int skl_dsp_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id) +{ + return SRAM_MEMORY_WINDOW_BASE + (0x2000 * id); +} + +static int skl_dsp_ipc_get_mailbox_offset(struct snd_sof_dev *sdev) +{ + return SRAM_MEMORY_WINDOW_BASE + 0x1000; +} + +/* skylake ops */ +struct snd_sof_dsp_ops sof_skl_ops; +EXPORT_SYMBOL_NS(sof_skl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_skl_ops_init(struct snd_sof_dev *sdev) +{ + struct sof_ipc4_fw_data *ipc4_data; + + /* common defaults */ + memcpy(&sof_skl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + + /* probe/remove/shutdown */ + sof_skl_ops.shutdown = hda_dsp_shutdown; + + sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; + + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET_CAVS_1_5; + + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_5; + + sof_skl_ops.get_window_offset = skl_dsp_ipc_get_window_offset; + sof_skl_ops.get_mailbox_offset = skl_dsp_ipc_get_mailbox_offset; + + /* doorbell */ + sof_skl_ops.irq_thread = hda_dsp_ipc4_irq_thread; + + /* ipc */ + sof_skl_ops.send_msg = hda_dsp_ipc4_send_msg; + + /* set DAI driver ops */ + hda_set_dai_drv_ops(sdev, &sof_skl_ops); + + /* debug */ + sof_skl_ops.debug_map = skl_dsp_debugfs; + sof_skl_ops.debug_map_count = ARRAY_SIZE(skl_dsp_debugfs); + sof_skl_ops.ipc_dump = hda_ipc4_dump; + + /* firmware run */ + sof_skl_ops.run = hda_dsp_cl_boot_firmware_skl; + + /* pre/post fw run */ + sof_skl_ops.post_fw_run = hda_dsp_post_fw_run; + + return 0; +}; +EXPORT_SYMBOL_NS(sof_skl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); + +const struct sof_intel_dsp_desc skl_chip_info = { + .cores_num = 2, + .init_core_mask = 1, + .host_managed_cores_mask = GENMASK(1, 0), + .ipc_req = HDA_DSP_REG_HIPCI, + .ipc_req_mask = HDA_DSP_REG_HIPCI_BUSY, + .ipc_ack = HDA_DSP_REG_HIPCIE, + .ipc_ack_mask = HDA_DSP_REG_HIPCIE_DONE, + .ipc_ctl = HDA_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS_SKL, + .rom_init_timeout = 300, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, + .hw_ip_version = SOF_INTEL_CAVS_1_5, +}; +EXPORT_SYMBOL_NS(skl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/tgl.c b/sound/soc/sof/intel/tgl.c index 6dfb4786c7824620556aa564b6f9243e5e0fb034..9ae2890e9dac491e2243cbc1e934e06eeca0d100 100644 --- a/sound/soc/sof/intel/tgl.c +++ b/sound/soc/sof/intel/tgl.c @@ -68,6 +68,9 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev) /* ipc */ sof_tgl_ops.send_msg = cnl_ipc_send_msg; + + /* debug */ + sof_tgl_ops.ipc_dump = cnl_ipc_dump; } if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { @@ -80,11 +83,16 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev) ipc4_data = sdev->private; ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2; + /* doorbell */ sof_tgl_ops.irq_thread = cnl_ipc4_irq_thread; /* ipc */ sof_tgl_ops.send_msg = cnl_ipc4_send_msg; + + /* debug */ + sof_tgl_ops.ipc_dump = cnl_ipc4_dump; } /* set DAI driver ops */ @@ -93,7 +101,6 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev) /* debug */ sof_tgl_ops.debug_map = tgl_dsp_debugfs; sof_tgl_ops.debug_map_count = ARRAY_SIZE(tgl_dsp_debugfs); - sof_tgl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ sof_tgl_ops.post_fw_run = hda_dsp_post_fw_run; @@ -121,13 +128,15 @@ const struct sof_intel_dsp_desc tgl_chip_info = { .ipc_ctl = CNL_DSP_REG_HIPCCTL, .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, - .ssp_count = ICL_SSP_COUNT, + .ssp_count = TGL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(tgl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -144,13 +153,15 @@ const struct sof_intel_dsp_desc tglh_chip_info = { .ipc_ctl = CNL_DSP_REG_HIPCCTL, .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, - .ssp_count = ICL_SSP_COUNT, + .ssp_count = TGL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(tglh_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -167,13 +178,15 @@ const struct sof_intel_dsp_desc ehl_chip_info = { .ipc_ctl = CNL_DSP_REG_HIPCCTL, .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, - .ssp_count = ICL_SSP_COUNT, + .ssp_count = TGL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(ehl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -190,13 +203,15 @@ const struct sof_intel_dsp_desc adls_chip_info = { .ipc_ctl = CNL_DSP_REG_HIPCCTL, .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, - .ssp_count = ICL_SSP_COUNT, + .ssp_count = TGL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, .check_ipc_irq = hda_dsp_check_ipc_irq, .cl_init = cl_dsp_init, + .power_down_dsp = hda_power_down_dsp, + .disable_interrupts = hda_dsp_disable_interrupts, .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(adls_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c index 9c6a84bdeca75301aa6ee56a55d4b35672a57971..dad57bef38f6d03273d5528fc855eb7856d67d44 100644 --- a/sound/soc/sof/ipc3-pcm.c +++ b/sound/soc/sof/ipc3-pcm.c @@ -346,6 +346,15 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, dev_dbg(component->dev, "AMD_SP channels_min: %d channels_max: %d\n", channels->min, channels->max); break; + case SOF_DAI_AMD_HS: + rate->min = private->dai_config->acphs.fsync_rate; + rate->max = private->dai_config->acphs.fsync_rate; + channels->min = private->dai_config->acphs.tdm_slots; + channels->max = private->dai_config->acphs.tdm_slots; + + dev_dbg(component->dev, + "AMD_HS channel_max: %d rate_max: %d\n", channels->max, rate->max); + break; case SOF_DAI_AMD_DMIC: rate->min = private->dai_config->acpdmic.pdm_rate; rate->max = private->dai_config->acpdmic.pdm_rate; diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c index 65923e7a5976f21e0df5c99c4799d54c5a190967..c148715aa0f91c0b80709cde99ab8a8745729159 100644 --- a/sound/soc/sof/ipc3-topology.c +++ b/sound/soc/sof/ipc3-topology.c @@ -1217,6 +1217,36 @@ static int sof_link_acp_sp_load(struct snd_soc_component *scomp, struct snd_sof_ return 0; } +static int sof_link_acp_hs_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink, + struct sof_ipc_dai_config *config, struct snd_sof_dai *dai) +{ + struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs; + struct sof_dai_private_data *private = dai->private; + u32 size = sizeof(*config); + + /* Configures the DAI hardware format and inverted clocks */ + sof_dai_set_format(hw_config, config); + + /* init IPC */ + memset(&config->acphs, 0, sizeof(config->acphs)); + config->hdr.size = size; + + config->acphs.fsync_rate = le32_to_cpu(hw_config->fsync_rate); + config->acphs.tdm_slots = le32_to_cpu(hw_config->tdm_slots); + + dev_info(scomp->dev, "ACP_HS config ACP%d channel %d rate %d\n", + config->dai_index, config->acphs.tdm_slots, + config->acphs.fsync_rate); + + dai->number_configs = 1; + dai->current_config = 0; + private->dai_config = kmemdup(config, size, GFP_KERNEL); + if (!private->dai_config) + return -ENOMEM; + + return 0; +} + static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink, struct sof_ipc_dai_config *config, struct snd_sof_dai *dai) { @@ -1249,6 +1279,7 @@ static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai static int sof_link_ssp_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink, struct sof_ipc_dai_config *config, struct snd_sof_dai *dai) { + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs; struct sof_dai_private_data *private = dai->private; u32 size = sizeof(*config); @@ -1273,6 +1304,12 @@ static int sof_link_ssp_load(struct snd_soc_component *scomp, struct snd_sof_dai config[i].hdr.size = size; + if (sdev->mclk_id_override) { + dev_dbg(scomp->dev, "tplg: overriding topology mclk_id %d by quirk %d\n", + config[i].ssp.mclk_id, sdev->mclk_id_quirk); + config[i].ssp.mclk_id = sdev->mclk_id_quirk; + } + /* copy differentiating hw configs to ipc structs */ config[i].ssp.mclk_rate = le32_to_cpu(hw_config[i].mclk_rate); config[i].ssp.bclk_rate = le32_to_cpu(hw_config[i].bclk_rate); @@ -1510,6 +1547,9 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget) case SOF_DAI_AMD_SP: ret = sof_link_acp_sp_load(scomp, slink, config, dai); break; + case SOF_DAI_AMD_HS: + ret = sof_link_acp_hs_load(scomp, slink, config, dai); + break; case SOF_DAI_AMD_DMIC: ret = sof_link_acp_dmic_load(scomp, slink, config, dai); break; diff --git a/sound/soc/sof/ipc3.c b/sound/soc/sof/ipc3.c index 82fa320253beffc49e3497007daa117e6b7cd246..b28af3a48b7075f7625ff72f8983baf61188066f 100644 --- a/sound/soc/sof/ipc3.c +++ b/sound/soc/sof/ipc3.c @@ -9,6 +9,7 @@ #include <sound/sof/stream.h> #include <sound/sof/control.h> +#include <trace/events/sof.h> #include "sof-priv.h" #include "sof-audio.h" #include "ipc3-priv.h" @@ -23,7 +24,7 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd) u8 *str2 = NULL; u32 glb; u32 type; - bool vdbg = false; + bool is_sof_ipc_stream_position = false; glb = cmd & SOF_GLB_TYPE_MASK; type = cmd & SOF_CMD_TYPE_MASK; @@ -118,7 +119,7 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd) case SOF_IPC_STREAM_TRIG_XRUN: str2 = "TRIG_XRUN"; break; case SOF_IPC_STREAM_POSITION: - vdbg = true; + is_sof_ipc_stream_position = true; str2 = "POSITION"; break; case SOF_IPC_STREAM_VORBIS_PARAMS: str2 = "VORBIS_PARAMS"; break; @@ -206,8 +207,8 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd) } if (str2) { - if (vdbg) - dev_vdbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); + if (is_sof_ipc_stream_position) + trace_sof_stream_position_ipc_rx(dev); else dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); } else { @@ -852,8 +853,7 @@ static void ipc3_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id) return; } - dev_vdbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n", - posn.host_posn, posn.dai_posn, posn.wallclock); + trace_sof_ipc3_period_elapsed_position(sdev, &posn); memcpy(&stream->posn, &posn, sizeof(posn)); diff --git a/sound/soc/sof/ipc4-loader.c b/sound/soc/sof/ipc4-loader.c index 9fadae8fd011420d21a4cd252ca47ffe2d052c53..e635ae515fa9fd7f574a2d415450ccfb42ac52c5 100644 --- a/sound/soc/sof/ipc4-loader.c +++ b/sound/soc/sof/ipc4-loader.c @@ -8,6 +8,7 @@ #include <linux/firmware.h> #include <sound/sof/ext_manifest4.h> #include <sound/sof/ipc4/header.h> +#include <trace/events/sof.h> #include "ipc4-priv.h" #include "sof-audio.h" #include "sof-priv.h" @@ -40,6 +41,17 @@ static size_t sof_ipc4_fw_parse_ext_man(struct snd_sof_dev *sdev) ext_man_hdr = (struct sof_ext_manifest4_hdr *)fw->data; + /* + * At the start of the firmware image we must have an extended manifest. + * Verify that the magic number is correct. + */ + if (ext_man_hdr->id != SOF_EXT_MAN4_MAGIC_NUMBER) { + dev_err(sdev->dev, + "Unexpected extended manifest magic number: %#x\n", + ext_man_hdr->id); + return -EINVAL; + } + fw_hdr_offset = ipc4_data->manifest_fw_hdr_offset; if (!fw_hdr_offset) return -EINVAL; @@ -146,6 +158,7 @@ static int sof_ipc4_validate_firmware(struct snd_sof_dev *sdev) static int sof_ipc4_query_fw_configuration(struct snd_sof_dev *sdev) { + struct sof_ipc4_fw_data *ipc4_data = sdev->private; const struct sof_ipc_ops *iops = sdev->ipc->ops; struct sof_ipc4_fw_version *fw_ver; struct sof_ipc4_tuple *tuple; @@ -182,13 +195,14 @@ static int sof_ipc4_query_fw_configuration(struct snd_sof_dev *sdev) fw_ver->build); break; case SOF_IPC4_FW_CFG_DL_MAILBOX_BYTES: - dev_vdbg(sdev->dev, "DL mailbox size: %u\n", *tuple->value); + trace_sof_ipc4_fw_config(sdev, "DL mailbox size", *tuple->value); break; case SOF_IPC4_FW_CFG_UL_MAILBOX_BYTES: - dev_vdbg(sdev->dev, "UL mailbox size: %u\n", *tuple->value); + trace_sof_ipc4_fw_config(sdev, "UL mailbox size", *tuple->value); break; case SOF_IPC4_FW_CFG_TRACE_LOG_BYTES: - dev_vdbg(sdev->dev, "Trace log size: %u\n", *tuple->value); + trace_sof_ipc4_fw_config(sdev, "Trace log size", *tuple->value); + ipc4_data->mtrace_log_bytes = *tuple->value; break; default: break; diff --git a/sound/soc/sof/ipc4-mtrace.c b/sound/soc/sof/ipc4-mtrace.c new file mode 100644 index 0000000000000000000000000000000000000000..9c7080041d0828caa122f0a37c09a461cf03981e --- /dev/null +++ b/sound/soc/sof/ipc4-mtrace.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright(c) 2022 Intel Corporation. All rights reserved. + +#include <linux/debugfs.h> +#include <linux/sched/signal.h> +#include <sound/sof/ipc4/header.h> +#include "sof-priv.h" +#include "ipc4-priv.h" + +/* + * debug info window is organized in 16 (equal sized) pages: + * + * ------------------------ + * | Page0 - descriptors | + * ------------------------ + * | Page1 - slot0 | + * ------------------------ + * | Page2 - slot1 | + * ------------------------ + * | ... | + * ------------------------ + * | Page14 - slot13 | + * ------------------------ + * | Page15 - slot14 | + * ------------------------ + * + * The slot size == page size + * + * The first page contains descriptors for the remaining 15 cores + * The slot descriptor is: + * u32 res_id; + * u32 type; + * u32 vma; + * + * Log buffer slots have the following layout: + * u32 host_read_ptr; + * u32 dsp_write_ptr; + * u8 buffer[]; + * + * The two pointers are offsets within the buffer. + */ + +#define SOF_MTRACE_DESCRIPTOR_SIZE 12 /* 3 x u32 */ + +#define FW_EPOCH_DELTA 11644473600LL + +#define INVALID_SLOT_OFFSET 0xffffffff +#define MAX_ALLOWED_LIBRARIES 16 +#define MAX_MTRACE_SLOTS 15 + +#define SOF_MTRACE_PAGE_SIZE 0x1000 +#define SOF_MTRACE_SLOT_SIZE SOF_MTRACE_PAGE_SIZE + +/* debug log slot types */ +#define SOF_MTRACE_SLOT_UNUSED 0x00000000 +#define SOF_MTRACE_SLOT_CRITICAL_LOG 0x54524300 /* byte 0: core ID */ +#define SOF_MTRACE_SLOT_DEBUG_LOG 0x474f4c00 /* byte 0: core ID */ +#define SOF_MTRACE_SLOT_GDB_STUB 0x42444700 +#define SOF_MTRACE_SLOT_TELEMETRY 0x4c455400 +#define SOF_MTRACE_SLOT_BROKEN 0x44414544 + /* for debug and critical types */ +#define SOF_MTRACE_SLOT_CORE_MASK GENMASK(7, 0) +#define SOF_MTRACE_SLOT_TYPE_MASK GENMASK(31, 8) + +#define DEFAULT_AGING_TIMER_PERIOD_MS 0x100 +#define DEFAULT_FIFO_FULL_TIMER_PERIOD_MS 0x1000 + +/* ipc4 log level and source definitions for logs_priorities_mask */ +#define SOF_MTRACE_LOG_LEVEL_CRITICAL BIT(0) +#define SOF_MTRACE_LOG_LEVEL_ERROR BIT(1) +#define SOF_MTRACE_LOG_LEVEL_WARNING BIT(2) +#define SOF_MTRACE_LOG_LEVEL_INFO BIT(3) +#define SOF_MTRACE_LOG_LEVEL_VERBOSE BIT(4) +#define SOF_MTRACE_LOG_SOURCE_INFRA BIT(5) /* log source 0 */ +#define SOF_MTRACE_LOG_SOURCE_HAL BIT(6) +#define SOF_MTRACE_LOG_SOURCE_MODULE BIT(7) +#define SOF_MTRACE_LOG_SOURCE_AUDIO BIT(8) +#define SOF_MTRACE_LOG_SOURCE_SCHEDULER BIT(9) +#define SOF_MTRACE_LOG_SOURCE_ULP_INFRA BIT(10) +#define SOF_MTRACE_LOG_SOURCE_ULP_MODULE BIT(11) +#define SOF_MTRACE_LOG_SOURCE_VISION BIT(12) /* log source 7 */ +#define DEFAULT_LOGS_PRIORITIES_MASK (SOF_MTRACE_LOG_LEVEL_CRITICAL | \ + SOF_MTRACE_LOG_LEVEL_ERROR | \ + SOF_MTRACE_LOG_LEVEL_WARNING | \ + SOF_MTRACE_LOG_LEVEL_INFO | \ + SOF_MTRACE_LOG_SOURCE_INFRA | \ + SOF_MTRACE_LOG_SOURCE_HAL | \ + SOF_MTRACE_LOG_SOURCE_MODULE | \ + SOF_MTRACE_LOG_SOURCE_AUDIO) + +struct sof_log_state_info { + u32 aging_timer_period; + u32 fifo_full_timer_period; + u32 enable; + u32 logs_priorities_mask[MAX_ALLOWED_LIBRARIES]; +} __packed; + +enum sof_mtrace_state { + SOF_MTRACE_DISABLED, + SOF_MTRACE_INITIALIZING, + SOF_MTRACE_ENABLED, +}; + +struct sof_mtrace_core_data { + struct snd_sof_dev *sdev; + + int id; + u32 slot_offset; + void *log_buffer; + u32 host_read_ptr; + u32 dsp_write_ptr; + /* pos update IPC arrived before the slot offset is known, queried */ + bool delayed_pos_update; + wait_queue_head_t trace_sleep; +}; + +struct sof_mtrace_priv { + struct snd_sof_dev *sdev; + enum sof_mtrace_state mtrace_state; + struct sof_log_state_info state_info; + + struct sof_mtrace_core_data cores[]; +}; + +static int sof_ipc4_mtrace_dfs_open(struct inode *inode, struct file *file) +{ + struct sof_mtrace_core_data *core_data = inode->i_private; + int ret; + + ret = debugfs_file_get(file->f_path.dentry); + if (unlikely(ret)) + return ret; + + core_data->log_buffer = kmalloc(SOF_MTRACE_SLOT_SIZE, GFP_KERNEL); + if (!core_data->log_buffer) { + debugfs_file_put(file->f_path.dentry); + return -ENOMEM; + } + + ret = simple_open(inode, file); + if (ret) { + kfree(core_data->log_buffer); + debugfs_file_put(file->f_path.dentry); + } + + return ret; +} + +static bool sof_wait_mtrace_avail(struct sof_mtrace_core_data *core_data) +{ + wait_queue_entry_t wait; + + /* data immediately available */ + if (core_data->host_read_ptr != core_data->dsp_write_ptr) + return true; + + /* wait for available trace data from FW */ + init_waitqueue_entry(&wait, current); + set_current_state(TASK_INTERRUPTIBLE); + add_wait_queue(&core_data->trace_sleep, &wait); + + if (!signal_pending(current)) { + /* set timeout to max value, no error code */ + schedule_timeout(MAX_SCHEDULE_TIMEOUT); + } + remove_wait_queue(&core_data->trace_sleep, &wait); + + if (core_data->host_read_ptr != core_data->dsp_write_ptr) + return true; + + return false; +} + +static ssize_t sof_ipc4_mtrace_dfs_read(struct file *file, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct sof_mtrace_core_data *core_data = file->private_data; + u32 log_buffer_offset, log_buffer_size, read_ptr, write_ptr; + struct snd_sof_dev *sdev = core_data->sdev; + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + void *log_buffer = core_data->log_buffer; + loff_t lpos = *ppos; + u32 avail; + int ret; + + /* check pos and count */ + if (lpos < 0) + return -EINVAL; + if (!count || count < sizeof(avail)) + return 0; + + /* get available count based on current host offset */ + if (!sof_wait_mtrace_avail(core_data)) { + /* No data available */ + avail = 0; + if (copy_to_user(buffer, &avail, sizeof(avail))) + return -EFAULT; + + return 0; + } + + if (core_data->slot_offset == INVALID_SLOT_OFFSET) + return 0; + + /* The log data buffer starts after the two pointer in the slot */ + log_buffer_offset = core_data->slot_offset + (sizeof(u32) * 2); + /* The log data size excludes the pointers */ + log_buffer_size = SOF_MTRACE_SLOT_SIZE - (sizeof(u32) * 2); + + read_ptr = core_data->host_read_ptr; + write_ptr = core_data->dsp_write_ptr; + + if (read_ptr < write_ptr) + avail = write_ptr - read_ptr; + else + avail = log_buffer_size - read_ptr + write_ptr; + + if (!avail) + return 0; + + if (avail > log_buffer_size) + avail = log_buffer_size; + + /* Need space for the initial u32 of the avail */ + if (avail > count - sizeof(avail)) + avail = count - sizeof(avail); + + if (sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS)) + dev_dbg(sdev->dev, + "core%d, host read: %#x, dsp write: %#x, avail: %#x\n", + core_data->id, read_ptr, write_ptr, avail); + + if (read_ptr < write_ptr) { + /* Read data between read pointer and write pointer */ + sof_mailbox_read(sdev, log_buffer_offset + read_ptr, log_buffer, avail); + } else { + /* read from read pointer to end of the slot */ + sof_mailbox_read(sdev, log_buffer_offset + read_ptr, log_buffer, + avail - write_ptr); + /* read from slot start to write pointer */ + if (write_ptr) + sof_mailbox_read(sdev, log_buffer_offset, + (u8 *)(log_buffer) + avail - write_ptr, + write_ptr); + } + + /* first write the number of bytes we have gathered */ + ret = copy_to_user(buffer, &avail, sizeof(avail)); + if (ret) + return -EFAULT; + + /* Followed by the data itself */ + ret = copy_to_user(buffer + sizeof(avail), log_buffer, avail); + if (ret) + return -EFAULT; + + /* Update the host_read_ptr in the slot for this core */ + read_ptr += avail; + if (read_ptr >= log_buffer_size) + read_ptr -= log_buffer_size; + sof_mailbox_write(sdev, core_data->slot_offset, &read_ptr, sizeof(read_ptr)); + + /* Only update the host_read_ptr if mtrace is enabled */ + if (priv->mtrace_state != SOF_MTRACE_DISABLED) + core_data->host_read_ptr = read_ptr; + + /* + * Ask for a new buffer from user space for the next chunk, not + * streaming due to the heading number of bytes value. + */ + *ppos += count; + + return count; +} + +static int sof_ipc4_mtrace_dfs_release(struct inode *inode, struct file *file) +{ + struct sof_mtrace_core_data *core_data = inode->i_private; + + debugfs_file_put(file->f_path.dentry); + + kfree(core_data->log_buffer); + + return 0; +} + +static const struct file_operations sof_dfs_mtrace_fops = { + .open = sof_ipc4_mtrace_dfs_open, + .read = sof_ipc4_mtrace_dfs_read, + .llseek = default_llseek, + .release = sof_ipc4_mtrace_dfs_release, + + .owner = THIS_MODULE, +}; + +static ssize_t sof_ipc4_priority_mask_dfs_read(struct file *file, char __user *to, + size_t count, loff_t *ppos) +{ + struct sof_mtrace_priv *priv = file->private_data; + int i, ret, offset, remaining; + char *buf; + + /* + * one entry (14 char + new line = 15): + * " 0: 000001ef" + * + * 16 * 15 + 1 = 241 + */ + buf = kzalloc(241, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + for (i = 0; i < MAX_ALLOWED_LIBRARIES; i++) { + offset = strlen(buf); + remaining = 241 - offset; + snprintf(buf + offset, remaining, "%2d: 0x%08x\n", i, + priv->state_info.logs_priorities_mask[i]); + } + + ret = simple_read_from_buffer(to, count, ppos, buf, strlen(buf)); + + kfree(buf); + return ret; +} + +static ssize_t sof_ipc4_priority_mask_dfs_write(struct file *file, + const char __user *from, + size_t count, loff_t *ppos) +{ + struct sof_mtrace_priv *priv = file->private_data; + int id, ret; + char *buf; + u32 mask; + + /* + * To update Nth mask entry, write: + * "N,0x1234" or "N,1234" to the debugfs file + * The mask will be interpreted as hexadecimal number + */ + buf = memdup_user_nul(from, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = sscanf(buf, "%d,0x%x", &id, &mask); + if (ret != 2) { + ret = sscanf(buf, "%d,%x", &id, &mask); + if (ret != 2) { + ret = -EINVAL; + goto out; + } + } + + if (id >= MAX_ALLOWED_LIBRARIES) { + ret = -EINVAL; + goto out; + } + + priv->state_info.logs_priorities_mask[id] = mask; + ret = count; + +out: + kfree(buf); + return ret; +} + +static const struct file_operations sof_dfs_priority_mask_fops = { + .open = simple_open, + .read = sof_ipc4_priority_mask_dfs_read, + .write = sof_ipc4_priority_mask_dfs_write, + .llseek = default_llseek, + + .owner = THIS_MODULE, +}; + +static int mtrace_debugfs_create(struct snd_sof_dev *sdev) +{ + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + struct dentry *dfs_root; + char dfs_name[100]; + int i; + + dfs_root = debugfs_create_dir("mtrace", sdev->debugfs_root); + if (IS_ERR_OR_NULL(dfs_root)) + return 0; + + /* Create files for the logging parameters */ + debugfs_create_u32("aging_timer_period", 0644, dfs_root, + &priv->state_info.aging_timer_period); + debugfs_create_u32("fifo_full_timer_period", 0644, dfs_root, + &priv->state_info.fifo_full_timer_period); + debugfs_create_file("logs_priorities_mask", 0644, dfs_root, priv, + &sof_dfs_priority_mask_fops); + + /* Separate log files per core */ + for (i = 0; i < sdev->num_cores; i++) { + snprintf(dfs_name, sizeof(dfs_name), "core%d", i); + debugfs_create_file(dfs_name, 0444, dfs_root, &priv->cores[i], + &sof_dfs_mtrace_fops); + } + + return 0; +} + +static int ipc4_mtrace_enable(struct snd_sof_dev *sdev) +{ + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + const struct sof_ipc_ops *iops = sdev->ipc->ops; + struct sof_ipc4_msg msg; + u64 system_time; + ktime_t kt; + int ret; + + if (priv->mtrace_state != SOF_MTRACE_DISABLED) + return 0; + + msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG); + msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST); + msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID); + msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID); + msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_SYSTEM_TIME); + + /* The system time is in usec, UTC, epoch is 1601-01-01 00:00:00 */ + kt = ktime_add_us(ktime_get_real(), FW_EPOCH_DELTA * USEC_PER_SEC); + system_time = ktime_to_us(kt); + msg.data_size = sizeof(system_time); + msg.data_ptr = &system_time; + ret = iops->set_get_data(sdev, &msg, msg.data_size, true); + if (ret) + return ret; + + msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_ENABLE_LOGS); + + priv->state_info.enable = 1; + + msg.data_size = sizeof(priv->state_info); + msg.data_ptr = &priv->state_info; + + priv->mtrace_state = SOF_MTRACE_INITIALIZING; + ret = iops->set_get_data(sdev, &msg, msg.data_size, true); + if (ret) { + priv->mtrace_state = SOF_MTRACE_DISABLED; + return ret; + } + + priv->mtrace_state = SOF_MTRACE_ENABLED; + + return 0; +} + +static void ipc4_mtrace_disable(struct snd_sof_dev *sdev) +{ + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + const struct sof_ipc_ops *iops = sdev->ipc->ops; + struct sof_ipc4_msg msg; + int i; + + if (priv->mtrace_state == SOF_MTRACE_DISABLED) + return; + + msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG); + msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST); + msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID); + msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID); + msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_ENABLE_LOGS); + + priv->state_info.enable = 0; + + msg.data_size = sizeof(priv->state_info); + msg.data_ptr = &priv->state_info; + iops->set_get_data(sdev, &msg, msg.data_size, true); + + priv->mtrace_state = SOF_MTRACE_DISABLED; + + for (i = 0; i < sdev->num_cores; i++) { + struct sof_mtrace_core_data *core_data = &priv->cores[i]; + + core_data->host_read_ptr = 0; + core_data->dsp_write_ptr = 0; + wake_up(&core_data->trace_sleep); + } +} + +/* + * Each DSP core logs to a dedicated slot. + * Parse the slot descriptors at debug_box offset to find the debug log slots + * and map them to cores. + * There are 15 slots and therefore 15 descriptors to check (MAX_MTRACE_SLOTS) + */ +static void sof_mtrace_find_core_slots(struct snd_sof_dev *sdev) +{ + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + struct sof_mtrace_core_data *core_data; + u32 slot_desc_type_offset, type, core; + int i; + + for (i = 0; i < MAX_MTRACE_SLOTS; i++) { + /* The type is the second u32 in the slot descriptor */ + slot_desc_type_offset = sdev->debug_box.offset; + slot_desc_type_offset += SOF_MTRACE_DESCRIPTOR_SIZE * i + sizeof(u32); + sof_mailbox_read(sdev, slot_desc_type_offset, &type, sizeof(type)); + + if ((type & SOF_MTRACE_SLOT_TYPE_MASK) == SOF_MTRACE_SLOT_DEBUG_LOG) { + core = type & SOF_MTRACE_SLOT_CORE_MASK; + + if (core >= sdev->num_cores) { + dev_dbg(sdev->dev, "core%u is invalid for slot%d\n", + core, i); + continue; + } + + core_data = &priv->cores[core]; + /* + * The area reserved for descriptors have the same size + * as a slot. + * In other words: slot0 starts at + * debug_box + SOF_MTRACE_SLOT_SIZE offset + */ + core_data->slot_offset = sdev->debug_box.offset; + core_data->slot_offset += SOF_MTRACE_SLOT_SIZE * (i + 1); + dev_dbg(sdev->dev, "slot%d is used for core%u\n", i, core); + if (core_data->delayed_pos_update) { + sof_ipc4_mtrace_update_pos(sdev, core); + core_data->delayed_pos_update = false; + } + } else if (type) { + dev_dbg(sdev->dev, "slot%d is not a log slot (%#x)\n", i, type); + } + } +} + +static int ipc4_mtrace_init(struct snd_sof_dev *sdev) +{ + struct sof_ipc4_fw_data *ipc4_data = sdev->private; + struct sof_mtrace_priv *priv; + int i, ret; + + if (sdev->fw_trace_data) { + dev_err(sdev->dev, "fw_trace_data has been already allocated\n"); + return -EBUSY; + } + + if (!ipc4_data->mtrace_log_bytes || + ipc4_data->mtrace_type != SOF_IPC4_MTRACE_INTEL_CAVS_2) { + sdev->fw_trace_is_supported = false; + return 0; + } + + priv = devm_kzalloc(sdev->dev, struct_size(priv, cores, sdev->num_cores), + GFP_KERNEL); + if (!priv) + return -ENOMEM; + + sdev->fw_trace_data = priv; + + /* Set initial values for mtrace parameters */ + priv->state_info.aging_timer_period = DEFAULT_AGING_TIMER_PERIOD_MS; + priv->state_info.fifo_full_timer_period = DEFAULT_FIFO_FULL_TIMER_PERIOD_MS; + /* Only enable basefw logs initially (index 0 is always basefw) */ + priv->state_info.logs_priorities_mask[0] = DEFAULT_LOGS_PRIORITIES_MASK; + + for (i = 0; i < sdev->num_cores; i++) { + struct sof_mtrace_core_data *core_data = &priv->cores[i]; + + init_waitqueue_head(&core_data->trace_sleep); + core_data->sdev = sdev; + core_data->id = i; + } + + ret = ipc4_mtrace_enable(sdev); + if (ret) { + /* + * Mark firmware tracing as not supported and return 0 to not + * block the whole audio stack + */ + sdev->fw_trace_is_supported = false; + dev_dbg(sdev->dev, "initialization failed, fw tracing is disabled\n"); + return 0; + } + + sof_mtrace_find_core_slots(sdev); + + ret = mtrace_debugfs_create(sdev); + if (ret) + ipc4_mtrace_disable(sdev); + + return ret; +} + +static void ipc4_mtrace_free(struct snd_sof_dev *sdev) +{ + ipc4_mtrace_disable(sdev); +} + +int sof_ipc4_mtrace_update_pos(struct snd_sof_dev *sdev, int core) +{ + struct sof_mtrace_priv *priv = sdev->fw_trace_data; + struct sof_mtrace_core_data *core_data; + + if (!sdev->fw_trace_is_supported || + priv->mtrace_state == SOF_MTRACE_DISABLED) + return 0; + + if (core >= sdev->num_cores) + return -EINVAL; + + core_data = &priv->cores[core]; + + if (core_data->slot_offset == INVALID_SLOT_OFFSET) { + core_data->delayed_pos_update = true; + return 0; + } + + /* Read out the dsp_write_ptr from the slot for this core */ + sof_mailbox_read(sdev, core_data->slot_offset + sizeof(u32), + &core_data->dsp_write_ptr, 4); + core_data->dsp_write_ptr -= core_data->dsp_write_ptr % 4; + + if (sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS)) + dev_dbg(sdev->dev, "core%d, host read: %#x, dsp write: %#x", + core, core_data->host_read_ptr, core_data->dsp_write_ptr); + + wake_up(&core_data->trace_sleep); + + return 0; +} + +static int ipc4_mtrace_resume(struct snd_sof_dev *sdev) +{ + return ipc4_mtrace_enable(sdev); +} + +static void ipc4_mtrace_suspend(struct snd_sof_dev *sdev, pm_message_t pm_state) +{ + ipc4_mtrace_disable(sdev); +} + +const struct sof_ipc_fw_tracing_ops ipc4_mtrace_ops = { + .init = ipc4_mtrace_init, + .free = ipc4_mtrace_free, + .suspend = ipc4_mtrace_suspend, + .resume = ipc4_mtrace_resume, +}; diff --git a/sound/soc/sof/ipc4-priv.h b/sound/soc/sof/ipc4-priv.h index 9492fe1796c2f1f13613ea9b8bdc6fe279cc671a..e3b8484a2f1fa8a6389d5598c4de66a2ab127207 100644 --- a/sound/soc/sof/ipc4-priv.h +++ b/sound/soc/sof/ipc4-priv.h @@ -13,18 +13,33 @@ #include <sound/sof/ext_manifest4.h> #include "sof-priv.h" +/* The DSP window indices are fixed */ +#define SOF_IPC4_OUTBOX_WINDOW_IDX 1 +#define SOF_IPC4_DEBUG_WINDOW_IDX 2 + +enum sof_ipc4_mtrace_type { + SOF_IPC4_MTRACE_NOT_AVAILABLE = 0, + SOF_IPC4_MTRACE_INTEL_CAVS_1_5, + SOF_IPC4_MTRACE_INTEL_CAVS_1_8, + SOF_IPC4_MTRACE_INTEL_CAVS_2, +}; + /** * struct sof_ipc4_fw_data - IPC4-specific data * @manifest_fw_hdr_offset: FW header offset in the manifest * @num_fw_modules : Number of modules in base FW * @fw_modules: Array of base FW modules * @nhlt: NHLT table either from the BIOS or the topology manifest + * @mtrace_type: mtrace type supported on the booted platform + * @mtrace_log_bytes: log bytes as reported by the firmware via fw_config reply */ struct sof_ipc4_fw_data { u32 manifest_fw_hdr_offset; int num_fw_modules; void *fw_modules; void *nhlt; + enum sof_ipc4_mtrace_type mtrace_type; + u32 mtrace_log_bytes; }; /** @@ -45,7 +60,8 @@ extern const struct sof_ipc_fw_loader_ops ipc4_loader_ops; extern const struct sof_ipc_tplg_ops ipc4_tplg_ops; extern const struct sof_ipc_tplg_control_ops tplg_ipc4_control_ops; extern const struct sof_ipc_pcm_ops ipc4_pcm_ops; +extern const struct sof_ipc_fw_tracing_ops ipc4_mtrace_ops; int sof_ipc4_set_pipeline_state(struct snd_sof_dev *sdev, u32 id, u32 state); - +int sof_ipc4_mtrace_update_pos(struct snd_sof_dev *sdev, int core); #endif diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index af072b484a6072eb1b83823ff94db3dac4cc56ce..a81af5f73a4b4c096141c3dbf19fa469c2b8ca41 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -331,7 +331,7 @@ static int sof_ipc4_widget_setup_msg(struct snd_sof_widget *swidget, struct sof_ msg->extension = SOF_IPC4_MOD_EXT_PPL_ID(swidget->pipeline_id); msg->extension |= SOF_IPC4_MOD_EXT_CORE_ID(swidget->core); - type = fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP ? 1 : 0; + type = (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP) ? 1 : 0; msg->extension |= SOF_IPC4_MOD_EXT_DOMAIN(type); return 0; @@ -771,7 +771,7 @@ static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget) goto err; ret = sof_update_ipc_object(scomp, src, SOF_SRC_TOKENS, swidget->tuples, - swidget->num_tuples, sizeof(src), 1); + swidget->num_tuples, sizeof(*src), 1); if (ret) { dev_err(scomp->dev, "Parsing SRC tokens failed\n"); goto err; @@ -1251,7 +1251,7 @@ sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget, if (blob->alh_cfg.count > 1) { int group_id; - group_id = ida_alloc_max(&alh_group_ida, ALH_MULTI_GTW_COUNT, + group_id = ida_alloc_max(&alh_group_ida, ALH_MULTI_GTW_COUNT - 1, GFP_KERNEL); if (group_id < 0) @@ -1447,7 +1447,6 @@ static int sof_ipc4_control_setup(struct snd_sof_dev *sdev, struct snd_sof_contr static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) { - struct snd_sof_widget *pipe_widget = swidget->pipe_widget; struct sof_ipc4_pipeline *pipeline; struct sof_ipc4_msg *msg; void *ipc_data = NULL; @@ -1530,7 +1529,7 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget swidget->widget->name); return ret; } - pipeline = pipe_widget->private; + msg->primary &= ~SOF_IPC4_MOD_INSTANCE_MASK; msg->primary |= SOF_IPC4_MOD_INSTANCE(swidget->instance_id); @@ -1544,9 +1543,16 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget msg->data_ptr = ipc_data; ret = sof_ipc_tx_message(sdev->ipc, msg, ipc_size, NULL, 0); - if (ret < 0) + if (ret < 0) { dev_err(sdev->dev, "failed to create module %s\n", swidget->widget->name); + if (swidget->id != snd_soc_dapm_scheduler) { + struct sof_ipc4_fw_module *fw_module = swidget->module_info; + + ida_free(&fw_module->m_ida, swidget->instance_id); + } + } + return ret; } diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c index 432b812bdf9c68810188a2ae775584d087956397..6eaa18e27e5af8f6fcb5bfc856fe5d368862c91d 100644 --- a/sound/soc/sof/ipc4.c +++ b/sound/soc/sof/ipc4.c @@ -205,6 +205,11 @@ static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_ms /* Notification message */ u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary); + /* Do not print log buffer notification if not desired */ + if (notif == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS && + !sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS)) + return; + if (notif < SOF_IPC4_NOTIFY_TYPE_LAST) str2 = ipc4_dbg_notification_type[notif]; if (!str2) @@ -234,6 +239,13 @@ static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_ms static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg, bool data_size_valid) { + /* Do not print log buffer notification if not desired */ + if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS) && + !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) && + SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION && + SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS) + return; + if (data_size_valid && msg->data_size) dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text, msg->primary, msg->extension, msg->data_size); @@ -283,6 +295,7 @@ static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data) if (ret == 0) { dev_err(sdev->dev, "ipc timed out for %#x|%#x\n", ipc4_msg->primary, ipc4_msg->extension); + snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout"); return -ETIMEDOUT; } @@ -525,7 +538,7 @@ static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg return inbox_offset; } inbox_size = SOF_IPC4_MSG_MAX_SIZE; - outbox_offset = snd_sof_dsp_get_window_offset(sdev, 1); + outbox_offset = snd_sof_dsp_get_window_offset(sdev, SOF_IPC4_OUTBOX_WINDOW_IDX); outbox_size = SOF_IPC4_MSG_MAX_SIZE; sdev->dsp_box.offset = inbox_offset; @@ -533,10 +546,14 @@ static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg sdev->host_box.offset = outbox_offset; sdev->host_box.size = outbox_size; + sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev, + SOF_IPC4_DEBUG_WINDOW_IDX); + dev_dbg(sdev->dev, "mailbox upstream 0x%x - size 0x%x\n", inbox_offset, inbox_size); dev_dbg(sdev->dev, "mailbox downstream 0x%x - size 0x%x\n", outbox_offset, outbox_size); + dev_dbg(sdev->dev, "debug box 0x%x\n", sdev->debug_box.offset); return sof_ipc4_init_msg_memory(sdev); } @@ -573,6 +590,9 @@ static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev) case SOF_IPC4_NOTIFY_RESOURCE_EVENT: data_size = sizeof(struct sof_ipc4_notify_resource_data); break; + case SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS: + sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary)); + break; default: dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n", ipc4_msg->primary, ipc4_msg->extension); @@ -646,4 +666,5 @@ const struct sof_ipc_ops ipc4_ops = { .fw_loader = &ipc4_loader_ops, .tplg = &ipc4_tplg_ops, .pcm = &ipc4_pcm_ops, + .fw_tracing = &ipc4_mtrace_ops, }; diff --git a/sound/soc/sof/mediatek/mt8186/mt8186.c b/sound/soc/sof/mediatek/mt8186/mt8186.c index e006532caf2f0528da6a472aa9ddedf0e030d22b..181189e00e020a78484cc23672d4c7f4cc711a83 100644 --- a/sound/soc/sof/mediatek/mt8186/mt8186.c +++ b/sound/soc/sof/mediatek/mt8186/mt8186.c @@ -460,14 +460,79 @@ static int mt8186_get_bar_index(struct snd_sof_dev *sdev, u32 type) return type; } -static int mt8186_ipc_msg_data(struct snd_sof_dev *sdev, - struct snd_pcm_substream *substream, - void *p, size_t sz) +static int mt8186_pcm_hw_params(struct snd_sof_dev *sdev, + struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_sof_platform_stream_params *platform_params) { - sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz); + platform_params->cont_update_posn = 1; + return 0; } +static snd_pcm_uframes_t mt8186_pcm_pointer(struct snd_sof_dev *sdev, + struct snd_pcm_substream *substream) +{ + int ret; + snd_pcm_uframes_t pos; + struct snd_sof_pcm *spcm; + struct sof_ipc_stream_posn posn; + struct snd_sof_pcm_stream *stream; + struct snd_soc_component *scomp = sdev->component; + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + + spcm = snd_sof_find_spcm_dai(scomp, rtd); + if (!spcm) { + dev_warn_ratelimited(sdev->dev, "warn: can't find PCM with DAI ID %d\n", + rtd->dai_link->id); + return 0; + } + + stream = &spcm->stream[substream->stream]; + ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); + if (ret < 0) { + dev_warn(sdev->dev, "failed to read stream position: %d\n", ret); + return 0; + } + + memcpy(&stream->posn, &posn, sizeof(posn)); + pos = spcm->stream[substream->stream].posn.host_posn; + pos = bytes_to_frames(substream->runtime, pos); + + return pos; +} + +static struct snd_soc_dai_driver mt8186_dai[] = { +{ + .name = "SOF_DL1", + .playback = { + .channels_min = 1, + .channels_max = 2, + }, +}, +{ + .name = "SOF_DL2", + .playback = { + .channels_min = 1, + .channels_max = 2, + }, +}, +{ + .name = "SOF_UL1", + .capture = { + .channels_min = 1, + .channels_max = 2, + }, +}, +{ + .name = "SOF_UL2", + .capture = { + .channels_min = 1, + .channels_max = 2, + }, +}, +}; + /* mt8186 ops */ static struct snd_sof_dsp_ops sof_mt8186_ops = { /* probe and remove */ @@ -481,6 +546,10 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = { .block_read = sof_block_read, .block_write = sof_block_write, + /* Mailbox IO */ + .mailbox_read = sof_mailbox_read, + .mailbox_write = sof_mailbox_write, + /* Register IO */ .write = sof_io_write, .read = sof_io_read, @@ -491,18 +560,28 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = { .send_msg = mt8186_send_msg, .get_mailbox_offset = mt8186_get_mailbox_offset, .get_window_offset = mt8186_get_window_offset, - .ipc_msg_data = mt8186_ipc_msg_data, + .ipc_msg_data = sof_ipc_msg_data, .set_stream_data_offset = sof_set_stream_data_offset, /* misc */ .get_bar_index = mt8186_get_bar_index, + /* stream callbacks */ + .pcm_open = sof_stream_pcm_open, + .pcm_hw_params = mt8186_pcm_hw_params, + .pcm_pointer = mt8186_pcm_pointer, + .pcm_close = sof_stream_pcm_close, + /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, /* Firmware ops */ .dsp_arch_ops = &sof_xtensa_arch_ops, + /* DAI drivers */ + .drv = mt8186_dai, + .num_drv = ARRAY_SIZE(mt8186_dai), + /* PM */ .suspend = mt8186_dsp_suspend, .resume = mt8186_dsp_resume, @@ -515,7 +594,16 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = { SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, }; +static struct snd_sof_of_mach sof_mt8186_machs[] = { + { + .compatible = "mediatek,mt8186", + .sof_tplg_filename = "sof-mt8186.tplg", + }, + {} +}; + static const struct sof_dev_desc sof_of_mt8186_desc = { + .of_machines = sof_mt8186_machs, .ipc_supported_mask = BIT(SOF_IPC), .ipc_default = SOF_IPC, .default_fw_path = { diff --git a/sound/soc/sof/mediatek/mt8195/mt8195.c b/sound/soc/sof/mediatek/mt8195/mt8195.c index 9c146015cd1b7bd728d3b5e04e7aeb4edc3a5771..3c81e84fcecfa211c295bf66fe42cb2b0ba2fafe 100644 --- a/sound/soc/sof/mediatek/mt8195/mt8195.c +++ b/sound/soc/sof/mediatek/mt8195/mt8195.c @@ -496,14 +496,48 @@ static int mt8195_get_bar_index(struct snd_sof_dev *sdev, u32 type) return type; } -static int mt8195_ipc_msg_data(struct snd_sof_dev *sdev, - struct snd_pcm_substream *substream, - void *p, size_t sz) +static int mt8195_pcm_hw_params(struct snd_sof_dev *sdev, + struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_sof_platform_stream_params *platform_params) { - sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz); + platform_params->cont_update_posn = 1; + return 0; } +static snd_pcm_uframes_t mt8195_pcm_pointer(struct snd_sof_dev *sdev, + struct snd_pcm_substream *substream) +{ + int ret; + snd_pcm_uframes_t pos; + struct snd_sof_pcm *spcm; + struct sof_ipc_stream_posn posn; + struct snd_sof_pcm_stream *stream; + struct snd_soc_component *scomp = sdev->component; + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + + spcm = snd_sof_find_spcm_dai(scomp, rtd); + if (!spcm) { + dev_warn_ratelimited(sdev->dev, "warn: can't find PCM with DAI ID %d\n", + rtd->dai_link->id); + return 0; + } + + stream = &spcm->stream[substream->stream]; + ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); + if (ret < 0) { + dev_warn(sdev->dev, "failed to read stream position: %d\n", ret); + return 0; + } + + memcpy(&stream->posn, &posn, sizeof(posn)); + pos = spcm->stream[substream->stream].posn.host_posn; + pos = bytes_to_frames(substream->runtime, pos); + + return pos; +} + static void mt8195_adsp_dump(struct snd_sof_dev *sdev, u32 flags) { u32 dbg_pc, dbg_data, dbg_bus0, dbg_bus1, dbg_inst; @@ -574,6 +608,10 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = { .block_read = sof_block_read, .block_write = sof_block_write, + /* Mailbox IO */ + .mailbox_read = sof_mailbox_read, + .mailbox_write = sof_mailbox_write, + /* Register IO */ .write = sof_io_write, .read = sof_io_read, @@ -584,12 +622,18 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = { .send_msg = mt8195_send_msg, .get_mailbox_offset = mt8195_get_mailbox_offset, .get_window_offset = mt8195_get_window_offset, - .ipc_msg_data = mt8195_ipc_msg_data, + .ipc_msg_data = sof_ipc_msg_data, .set_stream_data_offset = sof_set_stream_data_offset, /* misc */ .get_bar_index = mt8195_get_bar_index, + /* stream callbacks */ + .pcm_open = sof_stream_pcm_open, + .pcm_hw_params = mt8195_pcm_hw_params, + .pcm_pointer = mt8195_pcm_pointer, + .pcm_close = sof_stream_pcm_close, + /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -615,7 +659,20 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = { SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, }; +static struct snd_sof_of_mach sof_mt8195_machs[] = { + { + .compatible = "google,tomato", + .sof_tplg_filename = "sof-mt8195-mt6359-rt1019-rt5682-dts.tplg" + }, { + .compatible = "mediatek,mt8195", + .sof_tplg_filename = "sof-mt8195.tplg" + }, { + /* sentinel */ + } +}; + static const struct sof_dev_desc sof_of_mt8195_desc = { + .of_machines = sof_mt8195_machs, .ipc_supported_mask = BIT(SOF_IPC), .ipc_default = SOF_IPC, .default_fw_path = { @@ -652,4 +709,5 @@ static struct platform_driver snd_sof_of_mt8195_driver = { module_platform_driver(snd_sof_of_mt8195_driver); MODULE_IMPORT_NS(SND_SOC_SOF_XTENSA); +MODULE_IMPORT_NS(SND_SOC_SOF_MTK_COMMON); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c index 356497fe4f4ce619e81816ea94d893749cc58d4f..3537805070adadabc93dca4b7d379cb012b5b653 100644 --- a/sound/soc/sof/nocodec.c +++ b/sound/soc/sof/nocodec.c @@ -32,7 +32,7 @@ static int sof_nocodec_bes_setup(struct device *dev, /* set up BE dai_links */ for (i = 0; i < link_num; i++) { - dlc = devm_kzalloc(dev, 3 * sizeof(*dlc), GFP_KERNEL); + dlc = devm_kcalloc(dev, 3, sizeof(*dlc), GFP_KERNEL); if (!dlc) return -ENOMEM; @@ -78,7 +78,7 @@ static int sof_nocodec_setup(struct device *dev, struct snd_soc_dai_link *links; /* create dummy BE dai_links */ - links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_dai_drivers, GFP_KERNEL); + links = devm_kcalloc(dev, num_dai_drivers, sizeof(struct snd_soc_dai_link), GFP_KERNEL); if (!links) return -ENOMEM; diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c index 6cb6a432be5e99ffca60825daceefb0e000e9be4..14571b821ecac57396a4971abc622bd23a4249b4 100644 --- a/sound/soc/sof/pcm.c +++ b/sound/soc/sof/pcm.c @@ -13,6 +13,8 @@ #include <linux/pm_runtime.h> #include <sound/pcm_params.h> #include <sound/sof.h> +#include <trace/events/sof.h> +#include "sof-of-dev.h" #include "sof-priv.h" #include "sof-audio.h" #include "sof-utils.h" @@ -383,9 +385,7 @@ static snd_pcm_uframes_t sof_pcm_pointer(struct snd_soc_component *component, dai = bytes_to_frames(substream->runtime, spcm->stream[substream->stream].posn.dai_posn); - dev_vdbg(component->dev, - "PCM: stream %d dir %d DMA position %lu DAI position %lu\n", - spcm->pcm.pcm_id, substream->stream, host, dai); + trace_sof_pcm_pointer_position(sdev, spcm, substream, host, dai); return host; } @@ -655,7 +655,12 @@ void snd_sof_new_platform_drv(struct snd_sof_dev *sdev) struct snd_sof_pdata *plat_data = sdev->pdata; const char *drv_name; - drv_name = plat_data->machine->drv_name; + if (plat_data->machine) + drv_name = plat_data->machine->drv_name; + else if (plat_data->of_machine) + drv_name = plat_data->of_machine->drv_name; + else + drv_name = NULL; pd->name = "sof-audio-component"; pd->probe = sof_pcm_probe; diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c index 28976098a89e712d1203af958c524de4e4d2d63f..62092e2d609c728dbe16d641c63a220e9316b33c 100644 --- a/sound/soc/sof/sof-audio.c +++ b/sound/soc/sof/sof-audio.c @@ -9,7 +9,9 @@ // #include <linux/bitfield.h> +#include <trace/events/sof.h> #include "sof-audio.h" +#include "sof-of-dev.h" #include "ops.h" static void sof_reset_route_setup_status(struct snd_sof_dev *sdev, struct snd_sof_widget *widget) @@ -35,6 +37,8 @@ int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) if (!swidget->private) return 0; + trace_sof_widget_free(swidget); + /* only free when use_count is 0 */ if (--swidget->use_count) return 0; @@ -85,6 +89,8 @@ int sof_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) if (!swidget->private) return 0; + trace_sof_widget_setup(swidget); + /* widget already set up */ if (++swidget->use_count > 1) return 0; @@ -265,14 +271,16 @@ sof_unprepare_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widg struct snd_sof_widget *swidget = widget->dobj.private; struct snd_soc_dapm_path *p; - if (!widget_ops[widget->id].ipc_unprepare || !swidget->prepared) - goto sink_unprepare; + /* return if the widget is in use or if it is already unprepared */ + if (!swidget->prepared || swidget->use_count > 1) + return; + + if (widget_ops[widget->id].ipc_unprepare) + /* unprepare the source widget */ + widget_ops[widget->id].ipc_unprepare(swidget); - /* unprepare the source widget */ - widget_ops[widget->id].ipc_unprepare(swidget); swidget->prepared = false; -sink_unprepare: /* unprepare all widgets in the sink paths */ snd_soc_dapm_widget_for_each_sink_path(widget, p) { if (!p->walking && p->sink->dobj.private) { @@ -784,6 +792,28 @@ int sof_dai_get_bclk(struct snd_soc_pcm_runtime *rtd) } EXPORT_SYMBOL(sof_dai_get_bclk); +static struct snd_sof_of_mach *sof_of_machine_select(struct snd_sof_dev *sdev) +{ + struct snd_sof_pdata *sof_pdata = sdev->pdata; + const struct sof_dev_desc *desc = sof_pdata->desc; + struct snd_sof_of_mach *mach = desc->of_machines; + + if (!mach) + return NULL; + + for (; mach->compatible; mach++) { + if (of_machine_is_compatible(mach->compatible)) { + sof_pdata->tplg_filename = mach->sof_tplg_filename; + if (mach->fw_filename) + sof_pdata->fw_filename = mach->fw_filename; + + return mach; + } + } + + return NULL; +} + /* * SOF Driver enumeration. */ @@ -794,6 +824,7 @@ int sof_machine_check(struct snd_sof_dev *sdev) struct snd_soc_acpi_mach *mach; if (!IS_ENABLED(CONFIG_SND_SOC_SOF_FORCE_NOCODEC_MODE)) { + const struct snd_sof_of_mach *of_mach; /* find machine */ mach = snd_sof_machine_select(sdev); @@ -803,6 +834,12 @@ int sof_machine_check(struct snd_sof_dev *sdev) return 0; } + of_mach = sof_of_machine_select(sdev); + if (of_mach) { + sof_pdata->of_machine = of_mach; + return 0; + } + if (!IS_ENABLED(CONFIG_SND_SOC_SOF_NOCODEC)) { dev_err(sdev->dev, "error: no matching ASoC machine driver found - aborting probe\n"); return -ENODEV; diff --git a/sound/soc/sof/sof-client-probes.c b/sound/soc/sof/sof-client-probes.c index eb246b8234614cde8f18b6ab8f6a3cbba5f64ebc..ddeabbb5580e1c74c5cbb6de99b797b051ab775f 100644 --- a/sound/soc/sof/sof-client-probes.c +++ b/sound/soc/sof/sof-client-probes.c @@ -12,6 +12,8 @@ #include <linux/debugfs.h> #include <linux/module.h> #include <linux/pm_runtime.h> +#include <linux/string_helpers.h> + #include <sound/soc.h> #include <sound/sof/header.h> #include "sof-client.h" @@ -410,79 +412,6 @@ static const struct snd_compress_ops sof_probes_compressed_ops = { .copy = sof_probes_compr_copy, }; -/** - * strsplit_u32 - Split string into sequence of u32 tokens - * @buf: String to split into tokens. - * @delim: String containing delimiter characters. - * @tkns: Returned u32 sequence pointer. - * @num_tkns: Returned number of tokens obtained. - */ -static int strsplit_u32(char *buf, const char *delim, u32 **tkns, size_t *num_tkns) -{ - char *s; - u32 *data, *tmp; - size_t count = 0; - size_t cap = 32; - int ret = 0; - - *tkns = NULL; - *num_tkns = 0; - data = kcalloc(cap, sizeof(*data), GFP_KERNEL); - if (!data) - return -ENOMEM; - - while ((s = strsep(&buf, delim)) != NULL) { - ret = kstrtouint(s, 0, data + count); - if (ret) - goto exit; - if (++count >= cap) { - cap *= 2; - tmp = krealloc(data, cap * sizeof(*data), GFP_KERNEL); - if (!tmp) { - ret = -ENOMEM; - goto exit; - } - data = tmp; - } - } - - if (!count) - goto exit; - *tkns = kmemdup(data, count * sizeof(*data), GFP_KERNEL); - if (!(*tkns)) { - ret = -ENOMEM; - goto exit; - } - *num_tkns = count; - -exit: - kfree(data); - return ret; -} - -static int tokenize_input(const char __user *from, size_t count, - loff_t *ppos, u32 **tkns, size_t *num_tkns) -{ - char *buf; - int ret; - - buf = kmalloc(count + 1, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ret = simple_write_to_buffer(buf, count, ppos, from, count); - if (ret != count) { - ret = ret >= 0 ? -EIO : ret; - goto exit; - } - - buf[count] = '\0'; - ret = strsplit_u32(buf, ",", tkns, num_tkns); -exit: - kfree(buf); - return ret; -} - static ssize_t sof_probes_dfs_points_read(struct file *file, char __user *to, size_t count, loff_t *ppos) { @@ -548,8 +477,8 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from, struct sof_probes_priv *priv = cdev->data; struct device *dev = &cdev->auxdev.dev; struct sof_probe_point_desc *desc; - size_t num_tkns, bytes; - u32 *tkns; + u32 num_elems, *array; + size_t bytes; int ret, err; if (priv->extractor_stream_tag == SOF_PROBES_INVALID_NODE_ID) { @@ -557,16 +486,18 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from, return -ENOENT; } - ret = tokenize_input(from, count, ppos, &tkns, &num_tkns); + ret = parse_int_array_user(from, count, (int **)&array); if (ret < 0) return ret; - bytes = sizeof(*tkns) * num_tkns; - if (!num_tkns || (bytes % sizeof(*desc))) { + + num_elems = *array; + bytes = sizeof(*array) * num_elems; + if (bytes % sizeof(*desc)) { ret = -EINVAL; goto exit; } - desc = (struct sof_probe_point_desc *)tkns; + desc = (struct sof_probe_point_desc *)&array[1]; ret = pm_runtime_resume_and_get(dev); if (ret < 0 && ret != -EACCES) { @@ -583,7 +514,7 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from, if (err < 0) dev_err_ratelimited(dev, "debugfs write failed to idle %d\n", err); exit: - kfree(tkns); + kfree(array); return ret; } @@ -603,22 +534,17 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from, struct sof_client_dev *cdev = file->private_data; struct sof_probes_priv *priv = cdev->data; struct device *dev = &cdev->auxdev.dev; - size_t num_tkns; - u32 *tkns; int ret, err; + u32 *array; if (priv->extractor_stream_tag == SOF_PROBES_INVALID_NODE_ID) { dev_warn(dev, "no extractor stream running\n"); return -ENOENT; } - ret = tokenize_input(from, count, ppos, &tkns, &num_tkns); + ret = parse_int_array_user(from, count, (int **)&array); if (ret < 0) return ret; - if (!num_tkns) { - ret = -EINVAL; - goto exit; - } ret = pm_runtime_resume_and_get(dev); if (ret < 0) { @@ -626,7 +552,7 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from, goto exit; } - ret = sof_probes_points_remove(cdev, tkns, num_tkns); + ret = sof_probes_points_remove(cdev, &array[1], array[0]); if (!ret) ret = count; @@ -635,7 +561,7 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from, if (err < 0) dev_err_ratelimited(dev, "debugfs write failed to idle %d\n", err); exit: - kfree(tkns); + kfree(array); return ret; } diff --git a/sound/soc/sof/sof-of-dev.h b/sound/soc/sof/sof-of-dev.h index fd950a222ba484d6bee778b4c881b59c079f928d..2948b3a0d9fef918ffd4e7d393dbc166b8bbfad1 100644 --- a/sound/soc/sof/sof-of-dev.h +++ b/sound/soc/sof/sof-of-dev.h @@ -9,6 +9,13 @@ #ifndef __SOUND_SOC_SOF_OF_H #define __SOUND_SOC_SOF_OF_H +struct snd_sof_of_mach { + const char *compatible; + const char *drv_name; + const char *fw_filename; + const char *sof_tplg_filename; +}; + extern const struct dev_pm_ops sof_of_pm; int sof_of_probe(struct platform_device *pdev); diff --git a/sound/soc/sof/sof-pci-dev.c b/sound/soc/sof/sof-pci-dev.c index d627092b399d793428da037c59d9dde81dce8ec6..643fd1036d60b2d8122d847c3725de99814529c1 100644 --- a/sound/soc/sof/sof-pci-dev.c +++ b/sound/soc/sof/sof-pci-dev.c @@ -138,7 +138,7 @@ static const struct dmi_system_id community_key_platforms[] = { .ident = "Google Chromebooks", .callback = chromebook_use_community_key, .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Google"), + DMI_MATCH(DMI_PRODUCT_FAMILY, "Google"), } }, {}, diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index 823583086279b3bd092bd392d79c946d4d99f76a..de08825915b35d637f4d0de9335d31b0352ffe70 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -105,6 +105,13 @@ enum sof_debugfs_access_type { SOF_DEBUGFS_ACCESS_D0_ONLY, }; +struct sof_compr_stream { + u64 copied_total; + u32 sampling_rate; + u16 channels; + u16 sample_container_bytes; +}; + struct snd_sof_dev; struct snd_sof_ipc_msg; struct snd_sof_ipc; @@ -594,6 +601,10 @@ struct snd_sof_dev { /* to protect the ipc_rx_handler_list and dsp_state_handler_list list */ struct mutex client_event_handler_mutex; + /* quirks to override topology values */ + bool mclk_id_override; + u16 mclk_id_quirk; /* same size as in IPC3 definitions */ + void *private; /* core does not touch this */ }; diff --git a/sound/soc/sof/sof-utils.c b/sound/soc/sof/sof-utils.c index a3300ecee0628290199f955f26dc110ae27a1242..b6345a7345af7bb2e713958f879a43be60a4cfd7 100644 --- a/sound/soc/sof/sof-utils.c +++ b/sound/soc/sof/sof-utils.c @@ -45,8 +45,6 @@ int snd_sof_create_page_table(struct device *dev, u32 pfn = snd_sgbuf_get_addr(dmab, i * PAGE_SIZE) >> PAGE_SHIFT; u8 *pg_table; - dev_vdbg(dev, "pfn i %i idx %d pfn %x\n", i, idx, pfn); - pg_table = (u8 *)(page_table + idx); /* diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 9273a70fec25c5bf439b8fd43b67ed92c92fc6e9..38855dd60617d5185a0fcf36f03de3ae21d0b709 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -287,6 +287,7 @@ static const struct sof_dai_types sof_dais[] = { {"ACP", SOF_DAI_AMD_BT}, {"ACPSP", SOF_DAI_AMD_SP}, {"ACPDMIC", SOF_DAI_AMD_DMIC}, + {"ACPHS", SOF_DAI_AMD_HS}, {"AFE", SOF_DAI_MEDIATEK_AFE}, }; @@ -1011,9 +1012,6 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, } list_for_each_entry(rtd, &card->rtd_list, list) { - dev_vdbg(scomp->dev, "tplg: check widget: %s stream: %s dai stream: %s\n", - w->name, w->sname, rtd->dai_link->stream_name); - /* does stream match DAI link ? */ if (!rtd->dai_link->stream_name || strcmp(w->sname, rtd->dai_link->stream_name)) @@ -1032,7 +1030,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, break; } } - if (i == rtd->num_cpus) { + if (i == rtd->dai_link->num_cpus) { dev_err(scomp->dev, "error: can't find BE for DAI %s\n", w->name); @@ -1054,7 +1052,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, break; } } - if (i == rtd->num_cpus) { + if (i == rtd->dai_link->num_cpus) { dev_err(scomp->dev, "error: can't find BE for DAI %s\n", w->name); @@ -1537,9 +1535,6 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index, stream = SNDRV_PCM_STREAM_PLAYBACK; - dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: playback d0i3:%d\n", - spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible); - caps = &spcm->pcm.caps[stream]; /* allocate playback page table buffer */ @@ -1567,9 +1562,6 @@ capture: if (!spcm->pcm.capture) return ret; - dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: capture d0i3:%d\n", - spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible); - caps = &spcm->pcm.caps[stream]; /* allocate capture page table buffer */ diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c index 04f2912e14181519c67f20c15e8d23a4c29b7f4d..643fc8a170184445380349243ba31ede1bdcb0fb 100644 --- a/sound/soc/stm/stm32_adfsdm.c +++ b/sound/soc/stm/stm32_adfsdm.c @@ -335,8 +335,6 @@ static int stm32_adfsdm_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, priv); - pm_runtime_enable(&pdev->dev); - ret = devm_snd_soc_register_component(&pdev->dev, &stm32_adfsdm_dai_component, &priv->dai_drv, 1); @@ -366,9 +364,13 @@ static int stm32_adfsdm_probe(struct platform_device *pdev) #endif ret = snd_soc_add_component(component, NULL, 0); - if (ret < 0) + if (ret < 0) { dev_err(&pdev->dev, "%s: Failed to register PCM platform\n", __func__); + return ret; + } + + pm_runtime_enable(&pdev->dev); return ret; } diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c index 6aafe793eec441d2198322700529d950cca47260..ce7f6942308f9df6aa5a8313f11e3ae537b4046c 100644 --- a/sound/soc/stm/stm32_i2s.c +++ b/sound/soc/stm/stm32_i2s.c @@ -1136,8 +1136,6 @@ static int stm32_i2s_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(i2s->regmap), "Regmap init error\n"); - pm_runtime_enable(&pdev->dev); - ret = snd_dmaengine_pcm_register(&pdev->dev, &stm32_i2s_pcm_config, 0); if (ret) return dev_err_probe(&pdev->dev, ret, "PCM DMA register error\n"); @@ -1180,6 +1178,8 @@ static int stm32_i2s_probe(struct platform_device *pdev) FIELD_GET(I2S_VERR_MIN_MASK, val)); } + pm_runtime_enable(&pdev->dev); + return ret; error: diff --git a/sound/soc/stm/stm32_spdifrx.c b/sound/soc/stm/stm32_spdifrx.c index 0f7146756717670cb5c6f8b240a5b0e9549a6933..d399c906bb921a767b73d19e3cdf41ae9728336b 100644 --- a/sound/soc/stm/stm32_spdifrx.c +++ b/sound/soc/stm/stm32_spdifrx.c @@ -1002,8 +1002,6 @@ static int stm32_spdifrx_probe(struct platform_device *pdev) udelay(2); reset_control_deassert(rst); - pm_runtime_enable(&pdev->dev); - pcm_config = &stm32_spdifrx_pcm_config; ret = snd_dmaengine_pcm_register(&pdev->dev, pcm_config, 0); if (ret) @@ -1036,6 +1034,8 @@ static int stm32_spdifrx_probe(struct platform_device *pdev) FIELD_GET(SPDIFRX_VERR_MIN_MASK, ver)); } + pm_runtime_enable(&pdev->dev); + return ret; error: diff --git a/sound/soc/sunxi/Kconfig b/sound/soc/sunxi/Kconfig index ddcaaa98d3cb33a17e3277c86e4402a0b49c99e3..1f18f016acbb8feb0170e3a659a50db4d2b5e02c 100644 --- a/sound/soc/sunxi/Kconfig +++ b/sound/soc/sunxi/Kconfig @@ -56,6 +56,13 @@ config SND_SUN4I_SPDIF Say Y or M to add support for the S/PDIF audio block in the Allwinner A10 and affiliated SoCs. +config SND_SUN50I_DMIC + tristate "Allwinner H6 DMIC Support" + select SND_SOC_GENERIC_DMAENGINE_PCM + help + Say Y or M to add support for the DMIC audio block in the Allwinner + H6 and affiliated SoCs. + config SND_SUN8I_ADDA_PR_REGMAP tristate select REGMAP diff --git a/sound/soc/sunxi/Makefile b/sound/soc/sunxi/Makefile index a86be340a076108bcbd6ea426484b9578f443eaf..4483fe9c94ef504406675f65db8a67458135881e 100644 --- a/sound/soc/sunxi/Makefile +++ b/sound/soc/sunxi/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_SND_SUN8I_CODEC_ANALOG) += sun8i-codec-analog.o obj-$(CONFIG_SND_SUN50I_CODEC_ANALOG) += sun50i-codec-analog.o obj-$(CONFIG_SND_SUN8I_CODEC) += sun8i-codec.o obj-$(CONFIG_SND_SUN8I_ADDA_PR_REGMAP) += sun8i-adda-pr-regmap.o +obj-$(CONFIG_SND_SUN50I_DMIC) += sun50i-dmic.o diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c index 830beb38bf1563cbd5fd23125f2bf5b555c1932f..835dc34043670ba2e76bb5c6642739cde7a30c27 100644 --- a/sound/soc/sunxi/sun4i-codec.c +++ b/sound/soc/sunxi/sun4i-codec.c @@ -1232,6 +1232,9 @@ static const struct snd_soc_component_driver sun8i_a23_codec_codec = { static const struct snd_soc_component_driver sun4i_codec_component = { .name = "sun4i-codec", .legacy_dai_naming = 1, +#ifdef CONFIG_DEBUG_FS + .debugfs_prefix = "cpu", +#endif }; #define SUN4I_CODEC_RATES SNDRV_PCM_RATE_CONTINUOUS @@ -1804,7 +1807,7 @@ static int sun4i_codec_probe(struct platform_device *pdev) ret = snd_soc_register_card(card); if (ret) { - dev_err(&pdev->dev, "Failed to register our card\n"); + dev_err_probe(&pdev->dev, ret, "Failed to register our card\n"); goto err_assert_reset; } diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c new file mode 100644 index 0000000000000000000000000000000000000000..86cff5a5b1bd67edcc3bb176954c5c712ebeffb2 --- /dev/null +++ b/sound/soc/sunxi/sun50i-dmic.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// +// This driver supports the DMIC in Allwinner's H6 SoCs. +// +// Copyright 2021 Ban Tao <fengzheng923@gmail.com> + +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/of_device.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <sound/dmaengine_pcm.h> +#include <sound/pcm_params.h> +#include <sound/soc.h> + +#define SUN50I_DMIC_EN_CTL (0x00) + #define SUN50I_DMIC_EN_CTL_GLOBE BIT(8) + #define SUN50I_DMIC_EN_CTL_CHAN(v) ((v) << 0) + #define SUN50I_DMIC_EN_CTL_CHAN_MASK GENMASK(7, 0) +#define SUN50I_DMIC_SR (0x04) + #define SUN50I_DMIC_SR_SAMPLE_RATE(v) ((v) << 0) + #define SUN50I_DMIC_SR_SAMPLE_RATE_MASK GENMASK(2, 0) +#define SUN50I_DMIC_CTL (0x08) + #define SUN50I_DMIC_CTL_OVERSAMPLE_RATE BIT(0) +#define SUN50I_DMIC_DATA (0x10) +#define SUN50I_DMIC_INTC (0x14) + #define SUN50I_DMIC_FIFO_DRQ_EN BIT(2) +#define SUN50I_DMIC_INT_STA (0x18) + #define SUN50I_DMIC_INT_STA_OVERRUN_IRQ_PENDING BIT(1) + #define SUN50I_DMIC_INT_STA_DATA_IRQ_PENDING BIT(0) +#define SUN50I_DMIC_RXFIFO_CTL (0x1c) + #define SUN50I_DMIC_RXFIFO_CTL_FLUSH BIT(31) + #define SUN50I_DMIC_RXFIFO_CTL_MODE_MASK BIT(9) + #define SUN50I_DMIC_RXFIFO_CTL_MODE_LSB (0 << 9) + #define SUN50I_DMIC_RXFIFO_CTL_MODE_MSB (1 << 9) + #define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK BIT(8) + #define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_16 (0 << 8) + #define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_24 (1 << 8) +#define SUN50I_DMIC_CH_NUM (0x24) + #define SUN50I_DMIC_CH_NUM_N(v) ((v) << 0) + #define SUN50I_DMIC_CH_NUM_N_MASK GENMASK(2, 0) +#define SUN50I_DMIC_CNT (0x2c) + #define SUN50I_DMIC_CNT_N (1 << 0) +#define SUN50I_DMIC_HPF_CTRL (0x38) +#define SUN50I_DMIC_VERSION (0x50) + +struct sun50i_dmic_dev { + struct clk *dmic_clk; + struct clk *bus_clk; + struct reset_control *rst; + struct regmap *regmap; + struct snd_dmaengine_dai_dma_data dma_params_rx; +}; + +struct dmic_rate { + unsigned int samplerate; + unsigned int rate_bit; +}; + +static const struct dmic_rate dmic_rate_s[] = { + {48000, 0x0}, + {44100, 0x0}, + {32000, 0x1}, + {24000, 0x2}, + {22050, 0x2}, + {16000, 0x3}, + {12000, 0x4}, + {11025, 0x4}, + {8000, 0x5}, +}; + +static int sun50i_dmic_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *cpu_dai) +{ + struct snd_soc_pcm_runtime *rtd = substream->private_data; + struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0)); + + /* only support capture */ + if (substream->stream != SNDRV_PCM_STREAM_CAPTURE) + return -EINVAL; + + regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL, + SUN50I_DMIC_RXFIFO_CTL_FLUSH, + SUN50I_DMIC_RXFIFO_CTL_FLUSH); + regmap_write(host->regmap, SUN50I_DMIC_CNT, SUN50I_DMIC_CNT_N); + + return 0; +} + +static int sun50i_dmic_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *cpu_dai) +{ + int i = 0; + unsigned long rate = params_rate(params); + unsigned int mclk = 0; + unsigned int channels = params_channels(params); + unsigned int chan_en = (1 << channels) - 1; + struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(cpu_dai); + + /* DMIC num is N+1 */ + regmap_update_bits(host->regmap, SUN50I_DMIC_CH_NUM, + SUN50I_DMIC_CH_NUM_N_MASK, + SUN50I_DMIC_CH_NUM_N(channels - 1)); + regmap_write(host->regmap, SUN50I_DMIC_HPF_CTRL, chan_en); + regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL, + SUN50I_DMIC_EN_CTL_CHAN_MASK, + SUN50I_DMIC_EN_CTL_CHAN(chan_en)); + + switch (params_format(params)) { + case SNDRV_PCM_FORMAT_S16_LE: + regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL, + SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK, + SUN50I_DMIC_RXFIFO_CTL_SAMPLE_16); + break; + case SNDRV_PCM_FORMAT_S24_LE: + regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL, + SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK, + SUN50I_DMIC_RXFIFO_CTL_SAMPLE_24); + break; + default: + dev_err(cpu_dai->dev, "Invalid format!\n"); + return -EINVAL; + } + /* The hardware supports FIFO mode 1 for 24-bit samples */ + regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL, + SUN50I_DMIC_RXFIFO_CTL_MODE_MASK, + SUN50I_DMIC_RXFIFO_CTL_MODE_MSB); + + switch (rate) { + case 11025: + case 22050: + case 44100: + mclk = 22579200; + break; + case 8000: + case 12000: + case 16000: + case 24000: + case 32000: + case 48000: + mclk = 24576000; + break; + default: + dev_err(cpu_dai->dev, "Invalid rate!\n"); + return -EINVAL; + } + + if (clk_set_rate(host->dmic_clk, mclk)) { + dev_err(cpu_dai->dev, "mclk : %u not support\n", mclk); + return -EINVAL; + } + + for (i = 0; i < ARRAY_SIZE(dmic_rate_s); i++) { + if (dmic_rate_s[i].samplerate == rate) { + regmap_update_bits(host->regmap, SUN50I_DMIC_SR, + SUN50I_DMIC_SR_SAMPLE_RATE_MASK, + SUN50I_DMIC_SR_SAMPLE_RATE(dmic_rate_s[i].rate_bit)); + break; + } + } + + switch (params_physical_width(params)) { + case 16: + host->dma_params_rx.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; + break; + case 32: + host->dma_params_rx.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + break; + default: + dev_err(cpu_dai->dev, "Unsupported physical sample width: %d\n", + params_physical_width(params)); + return -EINVAL; + } + + /* oversamplerate adjust */ + if (params_rate(params) >= 24000) + regmap_update_bits(host->regmap, SUN50I_DMIC_CTL, + SUN50I_DMIC_CTL_OVERSAMPLE_RATE, + SUN50I_DMIC_CTL_OVERSAMPLE_RATE); + else + regmap_update_bits(host->regmap, SUN50I_DMIC_CTL, + SUN50I_DMIC_CTL_OVERSAMPLE_RATE, 0); + + return 0; +} + +static int sun50i_dmic_trigger(struct snd_pcm_substream *substream, int cmd, + struct snd_soc_dai *dai) +{ + int ret = 0; + struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(dai); + + if (substream->stream != SNDRV_PCM_STREAM_CAPTURE) + return -EINVAL; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + /* DRQ ENABLE */ + regmap_update_bits(host->regmap, SUN50I_DMIC_INTC, + SUN50I_DMIC_FIFO_DRQ_EN, + SUN50I_DMIC_FIFO_DRQ_EN); + /* Global enable */ + regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL, + SUN50I_DMIC_EN_CTL_GLOBE, + SUN50I_DMIC_EN_CTL_GLOBE); + break; + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + /* DRQ DISABLE */ + regmap_update_bits(host->regmap, SUN50I_DMIC_INTC, + SUN50I_DMIC_FIFO_DRQ_EN, 0); + /* Global disable */ + regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL, + SUN50I_DMIC_EN_CTL_GLOBE, 0); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +static int sun50i_dmic_soc_dai_probe(struct snd_soc_dai *dai) +{ + struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(dai); + + snd_soc_dai_init_dma_data(dai, NULL, &host->dma_params_rx); + + return 0; +} + +static const struct snd_soc_dai_ops sun50i_dmic_dai_ops = { + .startup = sun50i_dmic_startup, + .trigger = sun50i_dmic_trigger, + .hw_params = sun50i_dmic_hw_params, +}; + +static const struct regmap_config sun50i_dmic_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = SUN50I_DMIC_VERSION, + .cache_type = REGCACHE_NONE, +}; + +#define SUN50I_DMIC_RATES (SNDRV_PCM_RATE_8000_48000) +#define SUN50I_DMIC_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE) + +static struct snd_soc_dai_driver sun50i_dmic_dai = { + .capture = { + .channels_min = 1, + .channels_max = 8, + .rates = SUN50I_DMIC_RATES, + .formats = SUN50I_DMIC_FORMATS, + .sig_bits = 21, + }, + .probe = sun50i_dmic_soc_dai_probe, + .ops = &sun50i_dmic_dai_ops, + .name = "dmic", +}; + +static const struct of_device_id sun50i_dmic_of_match[] = { + { + .compatible = "allwinner,sun50i-h6-dmic", + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, sun50i_dmic_of_match); + +static const struct snd_soc_component_driver sun50i_dmic_component = { + .name = "sun50i-dmic", +}; + +static int sun50i_dmic_runtime_suspend(struct device *dev) +{ + struct sun50i_dmic_dev *host = dev_get_drvdata(dev); + + clk_disable_unprepare(host->dmic_clk); + clk_disable_unprepare(host->bus_clk); + + return 0; +} + +static int sun50i_dmic_runtime_resume(struct device *dev) +{ + struct sun50i_dmic_dev *host = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(host->dmic_clk); + if (ret) + return ret; + + ret = clk_prepare_enable(host->bus_clk); + if (ret) { + clk_disable_unprepare(host->dmic_clk); + return ret; + } + + return 0; +} + +static int sun50i_dmic_probe(struct platform_device *pdev) +{ + struct sun50i_dmic_dev *host; + struct resource *res; + int ret; + void __iomem *base; + + host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + /* Get the addresses */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return dev_err_probe(&pdev->dev, PTR_ERR(base), + "get resource failed.\n"); + + host->regmap = devm_regmap_init_mmio(&pdev->dev, base, + &sun50i_dmic_regmap_config); + + /* Clocks */ + host->bus_clk = devm_clk_get(&pdev->dev, "bus"); + if (IS_ERR(host->bus_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->bus_clk), + "failed to get bus clock.\n"); + + host->dmic_clk = devm_clk_get(&pdev->dev, "mod"); + if (IS_ERR(host->dmic_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->dmic_clk), + "failed to get dmic clock.\n"); + + host->dma_params_rx.addr = res->start + SUN50I_DMIC_DATA; + host->dma_params_rx.maxburst = 8; + + platform_set_drvdata(pdev, host); + + host->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); + if (IS_ERR(host->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(host->rst), + "Failed to get reset.\n"); + reset_control_deassert(host->rst); + + ret = devm_snd_soc_register_component(&pdev->dev, &sun50i_dmic_component, + &sun50i_dmic_dai, 1); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "failed to register component.\n"); + + pm_runtime_enable(&pdev->dev); + if (!pm_runtime_enabled(&pdev->dev)) { + ret = sun50i_dmic_runtime_resume(&pdev->dev); + if (ret) + goto err_disable_runtime_pm; + } + + ret = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0); + if (ret) + goto err_suspend; + + return 0; +err_suspend: + if (!pm_runtime_status_suspended(&pdev->dev)) + sun50i_dmic_runtime_suspend(&pdev->dev); +err_disable_runtime_pm: + pm_runtime_disable(&pdev->dev); + return ret; +} + +static int sun50i_dmic_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + sun50i_dmic_runtime_suspend(&pdev->dev); + + return 0; +} + +static const struct dev_pm_ops sun50i_dmic_pm = { + SET_RUNTIME_PM_OPS(sun50i_dmic_runtime_suspend, + sun50i_dmic_runtime_resume, NULL) +}; + +static struct platform_driver sun50i_dmic_driver = { + .driver = { + .name = "sun50i-dmic", + .of_match_table = of_match_ptr(sun50i_dmic_of_match), + .pm = &sun50i_dmic_pm, + }, + .probe = sun50i_dmic_probe, + .remove = sun50i_dmic_remove, +}; + +module_platform_driver(sun50i_dmic_driver); + +MODULE_DESCRIPTION("Allwinner sun50i DMIC SoC Interface"); +MODULE_AUTHOR("Ban Tao <fengzheng923@gmail.com>"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:sun50i-dmic"); diff --git a/sound/soc/ti/omap-mcbsp-st.c b/sound/soc/ti/omap-mcbsp-st.c index 7e8179cae92ebaaf334dbd25a055a9714a03ec1a..8163f453bf36e54ffa0cb37a3316be8c7fe5b1c8 100644 --- a/sound/soc/ti/omap-mcbsp-st.c +++ b/sound/soc/ti/omap-mcbsp-st.c @@ -244,10 +244,10 @@ static ssize_t st_taps_show(struct device *dev, spin_lock_irq(&mcbsp->lock); for (i = 0; i < st_data->nr_taps; i++) - status += sprintf(&buf[status], (i ? ", %d" : "%d"), - st_data->taps[i]); + status += sysfs_emit_at(buf, status, (i ? ", %d" : "%d"), + st_data->taps[i]); if (i) - status += sprintf(&buf[status], "\n"); + status += sysfs_emit_at(buf, status, "\n"); spin_unlock_irq(&mcbsp->lock); return status; diff --git a/sound/soc/ti/omap-mcbsp.c b/sound/soc/ti/omap-mcbsp.c index c4ac1f30b9fe45519fc38026ac5bada4cc0efc53..7c539a41a6a346b4ec1d56476b211dc85be5ffde 100644 --- a/sound/soc/ti/omap-mcbsp.c +++ b/sound/soc/ti/omap-mcbsp.c @@ -517,7 +517,7 @@ static ssize_t prop##_show(struct device *dev, \ { \ struct omap_mcbsp *mcbsp = dev_get_drvdata(dev); \ \ - return sprintf(buf, "%u\n", mcbsp->prop); \ + return sysfs_emit(buf, "%u\n", mcbsp->prop); \ } \ \ static ssize_t prop##_store(struct device *dev, \ @@ -560,11 +560,11 @@ static ssize_t dma_op_mode_show(struct device *dev, for (s = &dma_op_modes[i]; i < ARRAY_SIZE(dma_op_modes); s++, i++) { if (dma_op_mode == i) - len += sprintf(buf + len, "[%s] ", *s); + len += sysfs_emit_at(buf, len, "[%s] ", *s); else - len += sprintf(buf + len, "%s ", *s); + len += sysfs_emit_at(buf, len, "%s ", *s); } - len += sprintf(buf + len, "\n"); + len += sysfs_emit_at(buf, len, "\n"); return len; } @@ -614,7 +614,7 @@ static int omap_mcbsp_init(struct platform_device *pdev) { struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev); struct resource *res; - int ret = 0; + int ret; spin_lock_init(&mcbsp->lock); mcbsp->free = true; diff --git a/sound/usb/card.c b/sound/usb/card.c index d356743de2ff9b720505ae1e856c4beccb4f7b0a..a5ed11ea11456e9b28723d60c880c14e84f3430d 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -690,7 +690,7 @@ static bool get_alias_id(struct usb_device *dev, unsigned int *id) return false; } -static bool check_delayed_register_option(struct snd_usb_audio *chip, int iface) +static int check_delayed_register_option(struct snd_usb_audio *chip) { int i; unsigned int id, inum; @@ -699,14 +699,31 @@ static bool check_delayed_register_option(struct snd_usb_audio *chip, int iface) if (delayed_register[i] && sscanf(delayed_register[i], "%x:%x", &id, &inum) == 2 && id == chip->usb_id) - return inum != iface; + return inum; } - return false; + return -1; } static const struct usb_device_id usb_audio_ids[]; /* defined below */ +/* look for the last interface that matches with our ids and remember it */ +static void find_last_interface(struct snd_usb_audio *chip) +{ + struct usb_host_config *config = chip->dev->actconfig; + struct usb_interface *intf; + int i; + + if (!config) + return; + for (i = 0; i < config->desc.bNumInterfaces; i++) { + intf = config->interface[i]; + if (usb_match_id(intf, usb_audio_ids)) + chip->last_iface = intf->altsetting[0].desc.bInterfaceNumber; + } + usb_audio_dbg(chip, "Found last interface = %d\n", chip->last_iface); +} + /* look for the corresponding quirk */ static const struct snd_usb_audio_quirk * get_alias_quirk(struct usb_device *dev, unsigned int id) @@ -813,6 +830,7 @@ static int usb_audio_probe(struct usb_interface *intf, err = -ENODEV; goto __error; } + find_last_interface(chip); } if (chip->num_interfaces >= MAX_CARD_INTERFACES) { @@ -862,11 +880,11 @@ static int usb_audio_probe(struct usb_interface *intf, chip->need_delayed_register = false; /* clear again */ } - /* we are allowed to call snd_card_register() many times, but first - * check to see if a device needs to skip it or do anything special + /* register card if we reach to the last interface or to the specified + * one given via option */ - if (!snd_usb_registration_quirk(chip, ifnum) && - !check_delayed_register_option(chip, ifnum)) { + if (check_delayed_register_option(chip) == ifnum || + usb_interface_claimed(usb_ifnum_to_if(dev, chip->last_iface))) { err = snd_card_register(chip->card); if (err < 0) goto __error; diff --git a/sound/usb/card.h b/sound/usb/card.h index ca75f2206170fd379789893820159dfc34422a91..40061550105ac45a73d5877976b85042f2f4da61 100644 --- a/sound/usb/card.h +++ b/sound/usb/card.h @@ -129,7 +129,8 @@ struct snd_usb_endpoint { in a stream */ bool implicit_fb_sync; /* syncs with implicit feedback */ bool lowlatency_playback; /* low-latency playback mode */ - bool need_setup; /* (re-)need for configure? */ + bool need_setup; /* (re-)need for hw_params? */ + bool need_prepare; /* (re-)need for prepare? */ /* for hw constraints */ const struct audioformat *cur_audiofmt; diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 0d7b73bf7945063cda7bd50d81c65327dde5ef2a..d0b8d61d1d22b2705562637e36ec6d4205663ab4 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -32,6 +32,7 @@ struct snd_usb_iface_ref { unsigned char iface; bool need_setup; int opened; + int altset; struct list_head list; }; @@ -39,7 +40,9 @@ struct snd_usb_iface_ref { struct snd_usb_clock_ref { unsigned char clock; atomic_t locked; + int opened; int rate; + bool need_setup; struct list_head list; }; @@ -93,12 +96,13 @@ static inline unsigned get_usb_high_speed_rate(unsigned int rate) */ static void release_urb_ctx(struct snd_urb_ctx *u) { - if (u->buffer_size) + if (u->urb && u->buffer_size) usb_free_coherent(u->ep->chip->dev, u->buffer_size, u->urb->transfer_buffer, u->urb->transfer_dma); usb_free_urb(u->urb); u->urb = NULL; + u->buffer_size = 0; } static const char *usb_error_string(int err) @@ -758,7 +762,8 @@ bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip, * The endpoint needs to be closed via snd_usb_endpoint_close() later. * * Note that this function doesn't configure the endpoint. The substream - * needs to set it up later via snd_usb_endpoint_configure(). + * needs to set it up later via snd_usb_endpoint_set_params() and + * snd_usb_endpoint_prepare(). */ struct snd_usb_endpoint * snd_usb_endpoint_open(struct snd_usb_audio *chip, @@ -801,6 +806,7 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip, ep = NULL; goto unlock; } + ep->clock_ref->opened++; } ep->cur_audiofmt = fp; @@ -818,6 +824,7 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip, ep->implicit_fb_sync = fp->implicit_fb; ep->need_setup = true; + ep->need_prepare = true; usb_audio_dbg(chip, " channels=%d, rate=%d, format=%s, period_bytes=%d, periods=%d, implicit_fb=%d\n", ep->cur_channels, ep->cur_rate, @@ -894,6 +901,9 @@ static int endpoint_set_interface(struct snd_usb_audio *chip, int altset = set ? ep->altsetting : 0; int err; + if (ep->iface_ref->altset == altset) + return 0; + usb_audio_dbg(chip, "Setting usb interface %d:%d for EP 0x%x\n", ep->iface, altset, ep->ep_num); err = usb_set_interface(chip->dev, ep->iface, altset); @@ -905,6 +915,7 @@ static int endpoint_set_interface(struct snd_usb_audio *chip, if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY) msleep(50); + ep->iface_ref->altset = altset; return 0; } @@ -924,6 +935,10 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip, endpoint_set_interface(chip, ep, false); if (!--ep->opened) { + if (ep->clock_ref) { + if (!--ep->clock_ref->opened) + ep->clock_ref->rate = 0; + } ep->iface = 0; ep->altsetting = 0; ep->cur_audiofmt = NULL; @@ -938,7 +953,7 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip, /* Prepare for suspening EP, called from the main suspend handler */ void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep) { - ep->need_setup = true; + ep->need_prepare = true; if (ep->iface_ref) ep->iface_ref->need_setup = true; if (ep->clock_ref) @@ -1259,6 +1274,7 @@ static int sync_ep_set_params(struct snd_usb_endpoint *ep) if (!ep->syncbuf) return -ENOMEM; + ep->nurbs = SYNC_URBS; for (i = 0; i < SYNC_URBS; i++) { struct snd_urb_ctx *u = &ep->urb[i]; u->index = i; @@ -1278,8 +1294,6 @@ static int sync_ep_set_params(struct snd_usb_endpoint *ep) u->urb->complete = snd_complete_urb; } - ep->nurbs = SYNC_URBS; - return 0; out_of_memory: @@ -1287,23 +1301,51 @@ out_of_memory: return -ENOMEM; } +/* update the rate of the referred clock; return the actual rate */ +static int update_clock_ref_rate(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep) +{ + struct snd_usb_clock_ref *clock = ep->clock_ref; + int rate = ep->cur_rate; + + if (!clock || clock->rate == rate) + return rate; + if (clock->rate) { + if (atomic_read(&clock->locked)) + return clock->rate; + if (clock->rate != rate) { + usb_audio_err(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n", + clock->rate, rate, ep->ep_num); + return clock->rate; + } + } + clock->rate = rate; + clock->need_setup = true; + return rate; +} + /* * snd_usb_endpoint_set_params: configure an snd_usb_endpoint * + * It's called either from hw_params callback. * Determine the number of URBs to be used on this endpoint. * An endpoint must be configured before it can be started. * An endpoint that is already running can not be reconfigured. */ -static int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, - struct snd_usb_endpoint *ep) +int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep) { const struct audioformat *fmt = ep->cur_audiofmt; - int err; + int err = 0; + + mutex_lock(&chip->mutex); + if (!ep->need_setup) + goto unlock; /* release old buffers, if any */ err = release_urbs(ep, false); if (err < 0) - return err; + goto unlock; ep->datainterval = fmt->datainterval; ep->maxpacksize = fmt->maxpacksize; @@ -1341,55 +1383,60 @@ static int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, usb_audio_dbg(chip, "Set up %d URBS, ret=%d\n", ep->nurbs, err); if (err < 0) - return err; + goto unlock; /* some unit conversions in runtime */ ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes; ep->curframesize = ep->curpacksize / ep->cur_frame_bytes; - return 0; + err = update_clock_ref_rate(chip, ep); + if (err >= 0) { + ep->need_setup = false; + err = 0; + } + + unlock: + mutex_unlock(&chip->mutex); + return err; } static int init_sample_rate(struct snd_usb_audio *chip, struct snd_usb_endpoint *ep) { struct snd_usb_clock_ref *clock = ep->clock_ref; - int err; + int rate, err; - if (clock) { - if (atomic_read(&clock->locked)) - return 0; - if (clock->rate == ep->cur_rate) - return 0; - if (clock->rate && clock->rate != ep->cur_rate) { - usb_audio_dbg(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n", - clock->rate, ep->cur_rate, ep->ep_num); - return -EINVAL; - } - } + rate = update_clock_ref_rate(chip, ep); + if (rate < 0) + return rate; + if (clock && !clock->need_setup) + return 0; - err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, ep->cur_rate); - if (err < 0) + err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, rate); + if (err < 0) { + if (clock) + clock->rate = 0; /* reset rate */ return err; + } if (clock) - clock->rate = ep->cur_rate; + clock->need_setup = false; return 0; } /* - * snd_usb_endpoint_configure: Configure the endpoint + * snd_usb_endpoint_prepare: Prepare the endpoint * * This function sets up the EP to be fully usable state. - * It's called either from hw_params or prepare callback. + * It's called either from prepare callback. * The function checks need_setup flag, and performs nothing unless needed, * so it's safe to call this multiple times. * * This returns zero if unchanged, 1 if the configuration has changed, * or a negative error code. */ -int snd_usb_endpoint_configure(struct snd_usb_audio *chip, - struct snd_usb_endpoint *ep) +int snd_usb_endpoint_prepare(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep) { bool iface_first; int err = 0; @@ -1397,7 +1444,7 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, mutex_lock(&chip->mutex); if (WARN_ON(!ep->iface_ref)) goto unlock; - if (!ep->need_setup) + if (!ep->need_prepare) goto unlock; /* If the interface has been already set up, just set EP parameters */ @@ -1410,9 +1457,6 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, if (err < 0) goto unlock; } - err = snd_usb_endpoint_set_params(chip, ep); - if (err < 0) - goto unlock; goto done; } @@ -1440,10 +1484,6 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, if (err < 0) goto unlock; - err = snd_usb_endpoint_set_params(chip, ep); - if (err < 0) - goto unlock; - err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt); if (err < 0) goto unlock; @@ -1458,7 +1498,7 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, ep->iface_ref->need_setup = false; done: - ep->need_setup = false; + ep->need_prepare = false; err = 1; unlock: @@ -1631,8 +1671,7 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending) WRITE_ONCE(ep->sync_source->sync_sink, NULL); stop_urbs(ep, false, keep_pending); if (ep->clock_ref) - if (!atomic_dec_return(&ep->clock_ref->locked)) - ep->clock_ref->rate = 0; + atomic_dec(&ep->clock_ref->locked); } } diff --git a/sound/usb/endpoint.h b/sound/usb/endpoint.h index 6a9af04cf175afbe31836b98e928fc5ab06c8f1e..e67ea28faa54f689c99991cffc1f8f481278a136 100644 --- a/sound/usb/endpoint.h +++ b/sound/usb/endpoint.h @@ -17,8 +17,10 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip, bool is_sync_ep); void snd_usb_endpoint_close(struct snd_usb_audio *chip, struct snd_usb_endpoint *ep); -int snd_usb_endpoint_configure(struct snd_usb_audio *chip, - struct snd_usb_endpoint *ep); +int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep); +int snd_usb_endpoint_prepare(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep); int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock); bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip, diff --git a/sound/usb/line6/driver.h b/sound/usb/line6/driver.h index ecf3a2b39c7eb44129e6acc82363d4e2b728e44b..dbb1d90d36475e27fcdf10379becf33c19bc7058 100644 --- a/sound/usb/line6/driver.h +++ b/sound/usb/line6/driver.h @@ -193,8 +193,6 @@ extern int line6_send_raw_message_async(struct usb_line6 *line6, const char *buffer, int size); extern int line6_send_sysex_message(struct usb_line6 *line6, const char *buffer, int size); -extern ssize_t line6_set_raw(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count); extern int line6_version_request_async(struct usb_line6 *line6); extern int line6_write_data(struct usb_line6 *line6, unsigned address, void *data, unsigned datalen); diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index d45d1d7e666447db9862b55dcec0d93152dc6c38..8ed165f036a016c985d7499909319538a7cb0a93 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -433,35 +433,6 @@ static void close_endpoints(struct snd_usb_audio *chip, } } -static int configure_endpoints(struct snd_usb_audio *chip, - struct snd_usb_substream *subs) -{ - int err; - - if (subs->data_endpoint->need_setup) { - /* stop any running stream beforehand */ - if (stop_endpoints(subs, false)) - sync_pending_stops(subs); - if (subs->sync_endpoint) { - err = snd_usb_endpoint_configure(chip, subs->sync_endpoint); - if (err < 0) - return err; - } - err = snd_usb_endpoint_configure(chip, subs->data_endpoint); - if (err < 0) - return err; - snd_usb_set_format_quirk(subs, subs->cur_audiofmt); - } else { - if (subs->sync_endpoint) { - err = snd_usb_endpoint_configure(chip, subs->sync_endpoint); - if (err < 0) - return err; - } - } - - return 0; -} - /* * hw_params callback * @@ -551,7 +522,16 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream, subs->cur_audiofmt = fmt; mutex_unlock(&chip->mutex); - ret = configure_endpoints(chip, subs); + if (!subs->data_endpoint->need_setup) + goto unlock; + + if (subs->sync_endpoint) { + ret = snd_usb_endpoint_set_params(chip, subs->sync_endpoint); + if (ret < 0) + goto unlock; + } + + ret = snd_usb_endpoint_set_params(chip, subs->data_endpoint); unlock: if (ret < 0) @@ -634,9 +614,18 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream) goto unlock; } - ret = configure_endpoints(chip, subs); + if (subs->sync_endpoint) { + ret = snd_usb_endpoint_prepare(chip, subs->sync_endpoint); + if (ret < 0) + goto unlock; + } + + ret = snd_usb_endpoint_prepare(chip, subs->data_endpoint); if (ret < 0) goto unlock; + else if (ret > 0) + snd_usb_set_format_quirk(subs, subs->cur_audiofmt); + ret = 0; /* reset the pointer */ subs->buffer_bytes = frames_to_bytes(runtime, runtime->buffer_size); @@ -1406,7 +1395,7 @@ static int prepare_playback_urb(struct snd_usb_substream *subs, transfer_done = subs->transfer_done; if (subs->lowlatency_playback && - runtime->status->state != SNDRV_PCM_STATE_DRAINING) { + runtime->state != SNDRV_PCM_STATE_DRAINING) { unsigned int hwptr = subs->hwptr_done / stride; /* calculate the byte offset-in-buffer of the appl_ptr */ @@ -1594,7 +1583,7 @@ static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substrea return 0; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: - stop_endpoints(subs, substream->runtime->status->state == SNDRV_PCM_STATE_DRAINING); + stop_endpoints(subs, substream->runtime->state == SNDRV_PCM_STATE_DRAINING); snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, NULL, NULL); subs->running = 0; diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index f93201a830b5a7404483c048364a8bf31067d772..06dfdd45cff8c83a5258ec76bf92aab8e9144ada 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2985,6 +2985,82 @@ YAMAHA_DEVICE(0x7010, "UB99"), } } }, +/* DIGIDESIGN MBOX 3 */ +{ + USB_DEVICE(0x0dba, 0x5000), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "Digidesign", + .product_name = "Mbox 3", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = (const struct snd_usb_audio_quirk[]) { + { + .ifnum = 0, + .type = QUIRK_IGNORE_INTERFACE + }, + { + .ifnum = 1, + .type = QUIRK_IGNORE_INTERFACE + }, + { + .ifnum = 2, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 4, + .iface = 2, + .altsetting = 1, + .altset_idx = 1, + .attributes = 0x00, + .endpoint = 0x01, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { + 48000 + } + } + }, + { + .ifnum = 3, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S24_3LE, + .channels = 4, + .iface = 3, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x81, + .attributes = 0x00, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .maxpacksize = 0x009c, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { + 48000 + } + } + }, + { + .ifnum = 4, + .type = QUIRK_MIDI_FIXED_ENDPOINT, + .data = &(const struct snd_usb_midi_endpoint_info) { + .out_cables = 0x0001, + .in_cables = 0x0001 + } + }, + { + .ifnum = -1 + } + } + } +}, { /* Tascam US122 MKII - playback-only support */ USB_DEVICE_VENDOR_SPEC(0x0644, 0x8021), diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 168fd802d70bd058fe641d94bb739efe41ed963b..eadac586bcc83783ff12a26b79256bcd62fdd05d 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1020,6 +1020,304 @@ static int snd_usb_axefx3_boot_quirk(struct usb_device *dev) return 0; } +static void mbox3_setup_48_24_magic(struct usb_device *dev) +{ + /* The Mbox 3 is "little endian" */ + /* max volume is: 0x0000. */ + /* min volume is: 0x0080 (shown in little endian form) */ + + + /* Load 48000Hz rate into buffer */ + u8 com_buff[4] = {0x80, 0xbb, 0x00, 0x00}; + + /* Set 48000Hz sample rate */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 0x01, 0x21, 0x0100, 0x0001, &com_buff, 4); //Is this really needed? + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 0x01, 0x21, 0x0100, 0x8101, &com_buff, 4); + + /* Deactivate Tuner */ + /* on = 0x01*/ + /* off = 0x00*/ + com_buff[0] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 0x01, 0x21, 0x0003, 0x2001, &com_buff, 1); + + /* Set clock source to Internal (as opposed to S/PDIF) */ + com_buff[0] = 0x01; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0100, 0x8001, &com_buff, 1); + + /* Mute the hardware loopbacks to start the device in a known state. */ + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* Analogue input 1 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0110, 0x4001, &com_buff, 2); + /* Analogue input 1 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0111, 0x4001, &com_buff, 2); + /* Analogue input 2 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0114, 0x4001, &com_buff, 2); + /* Analogue input 2 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0115, 0x4001, &com_buff, 2); + /* Analogue input 3 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0118, 0x4001, &com_buff, 2); + /* Analogue input 3 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0119, 0x4001, &com_buff, 2); + /* Analogue input 4 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011c, 0x4001, &com_buff, 2); + /* Analogue input 4 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011d, 0x4001, &com_buff, 2); + + /* Set software sends to output */ + com_buff[0] = 0x00; + com_buff[1] = 0x00; + /* Analogue software return 1 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0100, 0x4001, &com_buff, 2); + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* Analogue software return 1 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0101, 0x4001, &com_buff, 2); + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* Analogue software return 2 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0104, 0x4001, &com_buff, 2); + com_buff[0] = 0x00; + com_buff[1] = 0x00; + /* Analogue software return 2 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0105, 0x4001, &com_buff, 2); + + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* Analogue software return 3 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0108, 0x4001, &com_buff, 2); + /* Analogue software return 3 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0109, 0x4001, &com_buff, 2); + /* Analogue software return 4 left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010c, 0x4001, &com_buff, 2); + /* Analogue software return 4 right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010d, 0x4001, &com_buff, 2); + + /* Return to muting sends */ + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* Analogue fx return left channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0120, 0x4001, &com_buff, 2); + /* Analogue fx return right channel: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0121, 0x4001, &com_buff, 2); + + /* Analogue software input 1 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0100, 0x4201, &com_buff, 2); + /* Analogue software input 2 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0101, 0x4201, &com_buff, 2); + /* Analogue software input 3 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0102, 0x4201, &com_buff, 2); + /* Analogue software input 4 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0103, 0x4201, &com_buff, 2); + /* Analogue input 1 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0104, 0x4201, &com_buff, 2); + /* Analogue input 2 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0105, 0x4201, &com_buff, 2); + /* Analogue input 3 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0106, 0x4201, &com_buff, 2); + /* Analogue input 4 fx send: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0107, 0x4201, &com_buff, 2); + + /* Toggle allowing host control */ + com_buff[0] = 0x02; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 3, 0x21, 0x0000, 0x2001, &com_buff, 1); + + /* Do not dim fx returns */ + com_buff[0] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 3, 0x21, 0x0002, 0x2001, &com_buff, 1); + + /* Do not set fx returns to mono */ + com_buff[0] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 3, 0x21, 0x0001, 0x2001, &com_buff, 1); + + /* Mute the S/PDIF hardware loopback + * same odd volume logic here as above + */ + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* S/PDIF hardware input 1 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0112, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 1 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0113, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 2 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0116, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 2 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0117, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 3 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011a, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 3 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011b, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 4 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011e, 0x4001, &com_buff, 2); + /* S/PDIF hardware input 4 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x011f, 0x4001, &com_buff, 2); + /* S/PDIF software return 1 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0102, 0x4001, &com_buff, 2); + /* S/PDIF software return 1 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0103, 0x4001, &com_buff, 2); + /* S/PDIF software return 2 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0106, 0x4001, &com_buff, 2); + /* S/PDIF software return 2 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0107, 0x4001, &com_buff, 2); + + com_buff[0] = 0x00; + com_buff[1] = 0x00; + /* S/PDIF software return 3 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010a, 0x4001, &com_buff, 2); + + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* S/PDIF software return 3 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010b, 0x4001, &com_buff, 2); + /* S/PDIF software return 4 left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010e, 0x4001, &com_buff, 2); + + com_buff[0] = 0x00; + com_buff[1] = 0x00; + /* S/PDIF software return 4 right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x010f, 0x4001, &com_buff, 2); + + com_buff[0] = 0x00; + com_buff[1] = 0x80; + /* S/PDIF fx returns left channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0122, 0x4001, &com_buff, 2); + /* S/PDIF fx returns right channel */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0123, 0x4001, &com_buff, 2); + + /* Set the dropdown "Effect" to the first option */ + /* Room1 = 0x00 */ + /* Room2 = 0x01 */ + /* Room3 = 0x02 */ + /* Hall 1 = 0x03 */ + /* Hall 2 = 0x04 */ + /* Plate = 0x05 */ + /* Delay = 0x06 */ + /* Echo = 0x07 */ + com_buff[0] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0200, 0x4301, &com_buff, 1); /* max is 0xff */ + /* min is 0x00 */ + + + /* Set the effect duration to 0 */ + /* max is 0xffff */ + /* min is 0x0000 */ + com_buff[0] = 0x00; + com_buff[1] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0400, 0x4301, &com_buff, 2); + + /* Set the effect volume and feedback to 0 */ + /* max is 0xff */ + /* min is 0x00 */ + com_buff[0] = 0x00; + /* feedback: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0500, 0x4301, &com_buff, 1); + /* volume: */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 1, 0x21, 0x0300, 0x4301, &com_buff, 1); + + /* Set soft button hold duration */ + /* 0x03 = 250ms */ + /* 0x05 = 500ms DEFAULT */ + /* 0x08 = 750ms */ + /* 0x0a = 1sec */ + com_buff[0] = 0x05; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 3, 0x21, 0x0005, 0x2001, &com_buff, 1); + + /* Use dim LEDs for button of state */ + com_buff[0] = 0x00; + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), + 3, 0x21, 0x0004, 0x2001, &com_buff, 1); +} + +#define MBOX3_DESCRIPTOR_SIZE 464 + +static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) +{ + struct usb_host_config *config = dev->actconfig; + int err; + int descriptor_size; + + descriptor_size = le16_to_cpu(get_cfg_desc(config)->wTotalLength); + + if (descriptor_size != MBOX3_DESCRIPTOR_SIZE) { + dev_err(&dev->dev, "Invalid descriptor size=%d.\n", descriptor_size); + return -ENODEV; + } + + dev_dbg(&dev->dev, "device initialised!\n"); + + err = usb_get_descriptor(dev, USB_DT_DEVICE, 0, + &dev->descriptor, sizeof(dev->descriptor)); + config = dev->actconfig; + if (err < 0) + dev_dbg(&dev->dev, "error usb_get_descriptor: %d\n", err); + + err = usb_reset_configuration(dev); + if (err < 0) + dev_dbg(&dev->dev, "error usb_reset_configuration: %d\n", err); + dev_dbg(&dev->dev, "mbox3_boot: new boot length = %d\n", + le16_to_cpu(get_cfg_desc(config)->wTotalLength)); + + mbox3_setup_48_24_magic(dev); + dev_info(&dev->dev, "Digidesign Mbox 3: 24bit 48kHz"); + + return 0; /* Successful boot */ +} #define MICROBOOK_BUF_SIZE 128 @@ -1324,6 +1622,10 @@ int snd_usb_apply_boot_quirk(struct usb_device *dev, case USB_ID(0x0dba, 0x3000): /* Digidesign Mbox 2 */ return snd_usb_mbox2_boot_quirk(dev); + case USB_ID(0x0dba, 0x5000): + /* Digidesign Mbox 3 */ + return snd_usb_mbox3_boot_quirk(dev); + case USB_ID(0x1235, 0x0010): /* Focusrite Novation Saffire 6 USB */ case USB_ID(0x1235, 0x0018): /* Focusrite Novation Twitch */ @@ -1728,48 +2030,6 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip, } } -/* - * registration quirk: - * the registration is skipped if a device matches with the given ID, - * unless the interface reaches to the defined one. This is for delaying - * the registration until the last known interface, so that the card and - * devices appear at the same time. - */ - -struct registration_quirk { - unsigned int usb_id; /* composed via USB_ID() */ - unsigned int interface; /* the interface to trigger register */ -}; - -#define REG_QUIRK_ENTRY(vendor, product, iface) \ - { .usb_id = USB_ID(vendor, product), .interface = (iface) } - -static const struct registration_quirk registration_quirks[] = { - REG_QUIRK_ENTRY(0x0951, 0x16d8, 2), /* Kingston HyperX AMP */ - REG_QUIRK_ENTRY(0x0951, 0x16ed, 2), /* Kingston HyperX Cloud Alpha S */ - REG_QUIRK_ENTRY(0x0951, 0x16ea, 2), /* Kingston HyperX Cloud Flight S */ - REG_QUIRK_ENTRY(0x0ecb, 0x1f46, 2), /* JBL Quantum 600 */ - REG_QUIRK_ENTRY(0x0ecb, 0x1f47, 2), /* JBL Quantum 800 */ - REG_QUIRK_ENTRY(0x0ecb, 0x1f4c, 2), /* JBL Quantum 400 */ - REG_QUIRK_ENTRY(0x0ecb, 0x2039, 2), /* JBL Quantum 400 */ - REG_QUIRK_ENTRY(0x0ecb, 0x203c, 2), /* JBL Quantum 600 */ - REG_QUIRK_ENTRY(0x0ecb, 0x203e, 2), /* JBL Quantum 800 */ - { 0 } /* terminator */ -}; - -/* return true if skipping registration */ -bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface) -{ - const struct registration_quirk *q; - - for (q = registration_quirks; q->usb_id; q++) - if (chip->usb_id == q->usb_id) - return iface != q->interface; - - /* Register as normal */ - return false; -} - /* * driver behavior quirk flags */ @@ -1903,6 +2163,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x21b4, 0x0081, /* AudioQuest DragonFly */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x2522, 0x0007, /* LH Labs Geek Out HD Audio 1V5 */ + QUIRK_FLAG_SET_IFACE_FIRST), DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */ diff --git a/sound/usb/quirks.h b/sound/usb/quirks.h index 31abb7cb01a526d59aeaa78d2e46d16c686083c7..f9bfd5ac7bab01717de3a76227482a128bf73165 100644 --- a/sound/usb/quirks.h +++ b/sound/usb/quirks.h @@ -48,8 +48,6 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip, struct audioformat *fp, int stream); -bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface); - void snd_usb_init_quirk_flags(struct snd_usb_audio *chip); #endif /* __USBAUDIO_QUIRKS_H */ diff --git a/sound/usb/stream.c b/sound/usb/stream.c index ceb93d798182cfba087848a86609dd23bfbd7ce9..f75601ca2d525688b63396066e91543db4132911 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -495,6 +495,10 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, return 0; } } + + if (chip->card->registered) + chip->need_delayed_register = true; + /* look for an empty stream */ list_for_each_entry(as, &chip->pcm_list, list) { if (as->fmt_type != fp->fmt_type) @@ -502,9 +506,6 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, subs = &as->substream[stream]; if (subs->ep_num) continue; - if (snd_device_get_state(chip->card, as->pcm) != - SNDRV_DEV_BUILD) - chip->need_delayed_register = true; err = snd_pcm_new_stream(as->pcm, stream, 1); if (err < 0) return err; @@ -1105,7 +1106,7 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, * Dallas DS4201 workaround: It presents 5 altsettings, but the last * one misses syncpipe, and does not produce any sound. */ - if (chip->usb_id == USB_ID(0x04fa, 0x4201)) + if (chip->usb_id == USB_ID(0x04fa, 0x4201) && num >= 4) num = 4; for (i = 0; i < num; i++) { @@ -1221,12 +1222,6 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, if (err < 0) return err; } - - /* try to set the interface... */ - usb_set_interface(chip->dev, iface_no, 0); - snd_usb_init_pitch(chip, fp); - snd_usb_init_sample_rate(chip, fp, fp->rate_max); - usb_set_interface(chip->dev, iface_no, altno); } return 0; } diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index ffbb4b0d09a07e40e382ad37ef46ba276643cc84..2c6575029b1cd7fde6f1125a8085d06e3be35034 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -37,6 +37,7 @@ struct snd_usb_audio { unsigned int quirk_flags; unsigned int need_delayed_register:1; /* warn for delayed registration */ int num_interfaces; + int last_iface; int num_suspended_intf; int sample_rate_read_error; diff --git a/sound/usb/usx2y/usbusx2yaudio.c b/sound/usb/usx2y/usbusx2yaudio.c index 9cd5e3aae4f77da68a06c66c719557161285b699..5197599e7aa61bb9df5a775b88ee05df4798f647 100644 --- a/sound/usb/usx2y/usbusx2yaudio.c +++ b/sound/usb/usx2y/usbusx2yaudio.c @@ -822,8 +822,7 @@ static int snd_usx2y_pcm_hw_free(struct snd_pcm_substream *substream) usx2y_urbs_release(subs); if (!cap_subs->pcm_substream || !cap_subs->pcm_substream->runtime || - !cap_subs->pcm_substream->runtime->status || - cap_subs->pcm_substream->runtime->status->state < SNDRV_PCM_STATE_PREPARED) { + cap_subs->pcm_substream->runtime->state < SNDRV_PCM_STATE_PREPARED) { atomic_set(&cap_subs->state, STATE_STOPPED); usx2y_urbs_release(cap_subs); } diff --git a/sound/usb/usx2y/usx2yhwdeppcm.c b/sound/usb/usx2y/usx2yhwdeppcm.c index 240349b644f38fc41f16d3737d35361bc43856b4..767a227d54da491b533ab2bb76d1c6d5653df35a 100644 --- a/sound/usb/usx2y/usx2yhwdeppcm.c +++ b/sound/usb/usx2y/usx2yhwdeppcm.c @@ -374,8 +374,7 @@ static int snd_usx2y_usbpcm_hw_free(struct snd_pcm_substream *substream) usx2y_usbpcm_urbs_release(subs); if (!cap_subs->pcm_substream || !cap_subs->pcm_substream->runtime || - !cap_subs->pcm_substream->runtime->status || - cap_subs->pcm_substream->runtime->status->state < SNDRV_PCM_STATE_PREPARED) { + cap_subs->pcm_substream->runtime->state < SNDRV_PCM_STATE_PREPARED) { atomic_set(&cap_subs->state, STATE_STOPPED); if (cap_subs2) atomic_set(&cap_subs2->state, STATE_STOPPED); diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 3bb134355874c8bf12e81da517aa661dbb60ea6c..316917b9870704de245f002cb4261c46f8a2fea4 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -75,9 +75,11 @@ struct kvm_regs { /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */ #define KVM_ARM_DEVICE_TYPE_SHIFT 0 -#define KVM_ARM_DEVICE_TYPE_MASK (0xffff << KVM_ARM_DEVICE_TYPE_SHIFT) +#define KVM_ARM_DEVICE_TYPE_MASK GENMASK(KVM_ARM_DEVICE_TYPE_SHIFT + 15, \ + KVM_ARM_DEVICE_TYPE_SHIFT) #define KVM_ARM_DEVICE_ID_SHIFT 16 -#define KVM_ARM_DEVICE_ID_MASK (0xffff << KVM_ARM_DEVICE_ID_SHIFT) +#define KVM_ARM_DEVICE_ID_MASK GENMASK(KVM_ARM_DEVICE_ID_SHIFT + 15, \ + KVM_ARM_DEVICE_ID_SHIFT) /* Supported device IDs */ #define KVM_ARM_DEVICE_VGIC_V2 0 diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd-ibs.h index 9a3312e12e2ed99bd00940d96837711f2ca4b086..93807b437e4deb85b2d96e61df2bfd0b0262e711 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd-ibs.h @@ -6,6 +6,22 @@ #include "msr-index.h" +/* IBS_OP_DATA2 DataSrc */ +#define IBS_DATA_SRC_LOC_CACHE 2 +#define IBS_DATA_SRC_DRAM 3 +#define IBS_DATA_SRC_REM_CACHE 4 +#define IBS_DATA_SRC_IO 7 + +/* IBS_OP_DATA2 DataSrc Extension */ +#define IBS_DATA_SRC_EXT_LOC_CACHE 1 +#define IBS_DATA_SRC_EXT_NEAR_CCX_CACHE 2 +#define IBS_DATA_SRC_EXT_DRAM 3 +#define IBS_DATA_SRC_EXT_FAR_CCX_CACHE 5 +#define IBS_DATA_SRC_EXT_PMEM 6 +#define IBS_DATA_SRC_EXT_IO 7 +#define IBS_DATA_SRC_EXT_EXT_MEM 8 +#define IBS_DATA_SRC_EXT_PEER_AGENT_MEM 12 + /* * IBS Hardware MSRs */ diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 235dc85c91c3e372980b8b428e9713869115ead4..ef4775c6db01c128ab112a3d741c1b38ef7cdf64 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -457,7 +457,8 @@ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ #define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ -#define X86_BUG_RETBLEED X86_BUG(26) /* CPU is affected by RETBleed */ -#define X86_BUG_EIBRS_PBRSB X86_BUG(27) /* EIBRS is vulnerable to Post Barrier RSB Predictions */ +#define X86_BUG_MMIO_UNKNOWN X86_BUG(26) /* CPU is too old and its MMIO Stale Data status is unknown */ +#define X86_BUG_RETBLEED X86_BUG(27) /* CPU is affected by RETBleed */ +#define X86_BUG_EIBRS_PBRSB X86_BUG(28) /* EIBRS is vulnerable to Post Barrier RSB Predictions */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 6674bdb096f346d940e353e3c4df7491fd5e0779..10ac52705892a1680415d144877574a2457f0344 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -155,6 +155,11 @@ * Return Stack Buffer Predictions. */ +#define ARCH_CAP_XAPIC_DISABLE BIT(21) /* + * IA32_XAPIC_DISABLE_STATUS MSR + * supported + */ + #define MSR_IA32_FLUSH_CMD 0x0000010b #define L1D_FLUSH BIT(0) /* * Writeback and invalidate the @@ -585,6 +590,9 @@ #define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 #define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 +/* AMD Last Branch Record MSRs */ +#define MSR_AMD64_LBR_SELECT 0xc000010e + /* Fam 17h MSRs */ #define MSR_F17H_IRPERF 0xc00000e9 @@ -756,6 +764,8 @@ #define MSR_AMD_DBG_EXTN_CFG 0xc000010f #define MSR_AMD_SAMP_BR_FROM 0xc0010300 +#define DBG_EXTN_CFG_LBRV2EN BIT_ULL(6) + #define MSR_IA32_MPERF 0x000000e7 #define MSR_IA32_APERF 0x000000e8 @@ -1054,4 +1064,12 @@ #define MSR_IA32_HW_FEEDBACK_PTR 0x17d0 #define MSR_IA32_HW_FEEDBACK_CONFIG 0x17d1 +/* x2APIC locked status */ +#define MSR_IA32_XAPIC_DISABLE_STATUS 0xBD +#define LEGACY_XAPIC_DISABLED BIT(0) /* + * x2APIC mode is locked and + * disabling x2APIC will cause + * a #GP + */ + #endif /* _ASM_X86_MSR_INDEX_H */ diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst index 7c188a598444c463c2ea6d06711788b9810f0337..7f3b67a8b48f3c6ca968b3b92232fc25eee887d8 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-map.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst @@ -55,7 +55,7 @@ MAP COMMANDS | | **devmap** | **devmap_hash** | **sockmap** | **cpumap** | **xskmap** | **sockhash** | | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage** | | **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage** -| | **task_storage** | **bloom_filter** } +| | **task_storage** | **bloom_filter** | **user_ringbuf** } DESCRIPTION =========== diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 0744bd1150be72a32c2ed9bbcef937319261c5ef..68a70ac03c80ac9aa8fd8020f750f0ed6641a1fd 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -43,11 +43,6 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_ENUM64] = "ENUM64", }; -struct btf_attach_point { - __u32 obj_id; - __u32 btf_id; -}; - static const char *btf_int_enc_str(__u8 encoding) { switch (encoding) { @@ -640,10 +635,9 @@ static int do_dump(int argc, char **argv) btf = btf__parse_split(*argv, base ?: base_btf); err = libbpf_get_error(btf); - if (err) { - btf = NULL; + if (!btf) { p_err("failed to load BTF from %s: %s", - *argv, strerror(err)); + *argv, strerror(errno)); goto done; } NEXT_ARG(); @@ -688,8 +682,8 @@ static int do_dump(int argc, char **argv) btf = btf__load_from_kernel_by_id_split(btf_id, base_btf); err = libbpf_get_error(btf); - if (err) { - p_err("get btf by id (%u): %s", btf_id, strerror(err)); + if (!btf) { + p_err("get btf by id (%u): %s", btf_id, strerror(errno)); goto done; } } @@ -825,7 +819,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, u32_as_hash_field(id)); if (err) { p_err("failed to append entry to hashmap for BTF ID %u, object ID %u: %s", - btf_id, id, strerror(errno)); + btf_id, id, strerror(-err)); goto err_free; } } diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 125798b0bc5dd691ccd5b64fa9b231b52d7f50a3..19924b6ce796f6489abec2b1259bdebfc0504f99 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -452,7 +452,7 @@ static int btf_dumper_int(const struct btf_type *t, __u8 bit_offset, *(char *)data); break; case BTF_INT_BOOL: - jsonw_bool(jw, *(int *)data); + jsonw_bool(jw, *(bool *)data); break; default: /* shouldn't happen */ diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index cced668fb2a3cab82fb0bcead7b71f7188e24e37..b46a998d8f8dfe7970aca944539b90f3441dddcd 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c @@ -136,8 +136,8 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, jsonw_string_field(json_wtr, "attach_type", attach_type_str); else jsonw_uint_field(json_wtr, "attach_type", attach_type); - jsonw_string_field(json_wtr, "attach_flags", - attach_flags_str); + if (!(query_flags & BPF_F_QUERY_EFFECTIVE)) + jsonw_string_field(json_wtr, "attach_flags", attach_flags_str); jsonw_string_field(json_wtr, "name", prog_name); if (attach_btf_name) jsonw_string_field(json_wtr, "attach_btf_name", attach_btf_name); @@ -150,7 +150,10 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, printf("%-15s", attach_type_str); else printf("type %-10u", attach_type); - printf(" %-15s %-15s", attach_flags_str, prog_name); + if (query_flags & BPF_F_QUERY_EFFECTIVE) + printf(" %-15s", prog_name); + else + printf(" %-15s %-15s", attach_flags_str, prog_name); if (attach_btf_name) printf(" %-15s", attach_btf_name); else if (info.attach_btf_id) @@ -195,6 +198,32 @@ static int cgroup_has_attached_progs(int cgroup_fd) return no_prog ? 0 : 1; } + +static int show_effective_bpf_progs(int cgroup_fd, enum bpf_attach_type type, + int level) +{ + LIBBPF_OPTS(bpf_prog_query_opts, p); + __u32 prog_ids[1024] = {0}; + __u32 iter; + int ret; + + p.query_flags = query_flags; + p.prog_cnt = ARRAY_SIZE(prog_ids); + p.prog_ids = prog_ids; + + ret = bpf_prog_query_opts(cgroup_fd, type, &p); + if (ret) + return ret; + + if (p.prog_cnt == 0) + return 0; + + for (iter = 0; iter < p.prog_cnt; iter++) + show_bpf_prog(prog_ids[iter], type, NULL, level); + + return 0; +} + static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, int level) { @@ -245,6 +274,14 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, return 0; } +static int show_bpf_progs(int cgroup_fd, enum bpf_attach_type type, + int level) +{ + return query_flags & BPF_F_QUERY_EFFECTIVE ? + show_effective_bpf_progs(cgroup_fd, type, level) : + show_attached_bpf_progs(cgroup_fd, type, level); +} + static int do_show(int argc, char **argv) { enum bpf_attach_type type; @@ -292,6 +329,8 @@ static int do_show(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); + else if (query_flags & BPF_F_QUERY_EFFECTIVE) + printf("%-8s %-15s %-15s\n", "ID", "AttachType", "Name"); else printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType", "AttachFlags", "Name"); @@ -304,7 +343,7 @@ static int do_show(int argc, char **argv) * If we were able to get the show for at least one * attach type, let's return 0. */ - if (show_attached_bpf_progs(cgroup_fd, type, 0) == 0) + if (show_bpf_progs(cgroup_fd, type, 0) == 0) ret = 0; } @@ -362,7 +401,7 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb, btf_vmlinux = libbpf_find_kernel_btf(); for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) - show_attached_bpf_progs(cgroup_fd, type, ftw->level); + show_bpf_progs(cgroup_fd, type, ftw->level); if (errno == EINVAL) /* Last attach type does not support query. @@ -436,6 +475,11 @@ static int do_show_tree(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); + else if (query_flags & BPF_F_QUERY_EFFECTIVE) + printf("%s\n" + "%-8s %-15s %-15s\n", + "CgroupPath", + "ID", "AttachType", "Name"); else printf("%s\n" "%-8s %-15s %-15s %-15s\n", diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 067e9ea59e3b086565d8d2537bb5a89e69c0ed0c..8727765add888390c4e85f9aa4962e9651b8334a 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -722,6 +722,7 @@ print_all_levels(__maybe_unused enum libbpf_print_level level, static int prog_fd_by_nametag(void *nametag, int **fds, bool tag) { + char prog_name[MAX_PROG_FULL_NAME]; unsigned int id = 0; int fd, nb_fds = 0; void *tmp; @@ -754,12 +755,20 @@ static int prog_fd_by_nametag(void *nametag, int **fds, bool tag) goto err_close_fd; } - if ((tag && memcmp(nametag, info.tag, BPF_TAG_SIZE)) || - (!tag && strncmp(nametag, info.name, BPF_OBJ_NAME_LEN))) { + if (tag && memcmp(nametag, info.tag, BPF_TAG_SIZE)) { close(fd); continue; } + if (!tag) { + get_prog_full_name(&info, fd, prog_name, + sizeof(prog_name)); + if (strncmp(nametag, prog_name, sizeof(prog_name))) { + close(fd); + continue; + } + } + if (nb_fds > 0) { tmp = realloc(*fds, (nb_fds + 1) * sizeof(int)); if (!tmp) { @@ -820,7 +829,7 @@ int prog_parse_fds(int *argc, char ***argv, int **fds) NEXT_ARGP(); name = **argv; - if (strlen(name) > BPF_OBJ_NAME_LEN - 1) { + if (strlen(name) > MAX_PROG_FULL_NAME - 1) { p_err("can't parse name"); return -1; } diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 7ecabf7947fb09a54b2f744f7331bb1a39bc1fcc..36cf0f1517c9406a06f5d2fa57a937ff8b49b86d 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -1147,7 +1147,7 @@ exit_free: return res; #else /* Detection assumes user has specific privileges. - * We do not use libpcap so let's approximate, and restrict usage to + * We do not use libcap so let's approximate, and restrict usage to * root user only. */ if (geteuid()) { diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 7070dcffa822461ecde1fef4b86be4012dcb725c..cf8b4e525c889cf17bc1a61bed1c0768816b76d5 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -1594,14 +1594,14 @@ static int do_object(int argc, char **argv) err = bpf_linker__add_file(linker, file, NULL); if (err) { - p_err("failed to link '%s': %s (%d)", file, strerror(err), err); + p_err("failed to link '%s': %s (%d)", file, strerror(errno), errno); goto out; } } err = bpf_linker__finalize(linker); if (err) { - p_err("failed to finalize ELF file: %s (%d)", strerror(err), err); + p_err("failed to finalize ELF file: %s (%d)", strerror(errno), errno); goto out; } diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 7a20931c325059f733b85d24230c3f0fefdd629b..2863639706dd40c04866d00736eec23987b667ed 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -83,6 +83,36 @@ static bool is_iter_map_target(const char *target_name) strcmp(target_name, "bpf_sk_storage_map") == 0; } +static bool is_iter_cgroup_target(const char *target_name) +{ + return strcmp(target_name, "cgroup") == 0; +} + +static const char *cgroup_order_string(__u32 order) +{ + switch (order) { + case BPF_CGROUP_ITER_ORDER_UNSPEC: + return "order_unspec"; + case BPF_CGROUP_ITER_SELF_ONLY: + return "self_only"; + case BPF_CGROUP_ITER_DESCENDANTS_PRE: + return "descendants_pre"; + case BPF_CGROUP_ITER_DESCENDANTS_POST: + return "descendants_post"; + case BPF_CGROUP_ITER_ANCESTORS_UP: + return "ancestors_up"; + default: /* won't happen */ + return "unknown"; + } +} + +static bool is_iter_task_target(const char *target_name) +{ + return strcmp(target_name, "task") == 0 || + strcmp(target_name, "task_file") == 0 || + strcmp(target_name, "task_vma") == 0; +} + static void show_iter_json(struct bpf_link_info *info, json_writer_t *wtr) { const char *target_name = u64_to_ptr(info->iter.target_name); @@ -91,6 +121,18 @@ static void show_iter_json(struct bpf_link_info *info, json_writer_t *wtr) if (is_iter_map_target(target_name)) jsonw_uint_field(wtr, "map_id", info->iter.map.map_id); + else if (is_iter_task_target(target_name)) { + if (info->iter.task.tid) + jsonw_uint_field(wtr, "tid", info->iter.task.tid); + else if (info->iter.task.pid) + jsonw_uint_field(wtr, "pid", info->iter.task.pid); + } + + if (is_iter_cgroup_target(target_name)) { + jsonw_lluint_field(wtr, "cgroup_id", info->iter.cgroup.cgroup_id); + jsonw_string_field(wtr, "order", + cgroup_order_string(info->iter.cgroup.order)); + } } static int get_prog_info(int prog_id, struct bpf_prog_info *info) @@ -208,6 +250,18 @@ static void show_iter_plain(struct bpf_link_info *info) if (is_iter_map_target(target_name)) printf("map_id %u ", info->iter.map.map_id); + else if (is_iter_task_target(target_name)) { + if (info->iter.task.tid) + printf("tid %u ", info->iter.task.tid); + else if (info->iter.task.pid) + printf("pid %u ", info->iter.task.pid); + } + + if (is_iter_cgroup_target(target_name)) { + printf("cgroup_id %llu ", info->iter.cgroup.cgroup_id); + printf("order %s ", + cgroup_order_string(info->iter.cgroup.order)); + } } static int show_link_close_plain(int fd, struct bpf_link_info *info) diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 451cefc2d0da988aca77b767d99f872b05e8f83c..ccd7457f92bff92b077bad755a7699d65e39dcae 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -435,6 +435,16 @@ int main(int argc, char **argv) setlinebuf(stdout); +#ifdef USE_LIBCAP + /* Libcap < 2.63 hooks before main() to compute the number of + * capabilities of the running kernel, and doing so it calls prctl() + * which may fail and set errno to non-zero. + * Let's reset errno to make sure this does not interfere with the + * batch mode. + */ + errno = 0; +#endif + last_do_help = do_help; pretty_output = false; json_output = false; diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 38b6bc9c26c3bb0d95ec6ceb4c791f54998039c5..9a6ca9f311338ddffa25278b1c8d87838068253e 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -1459,7 +1459,7 @@ static int do_help(int argc, char **argv) " devmap | devmap_hash | sockmap | cpumap | xskmap | sockhash |\n" " cgroup_storage | reuseport_sockarray | percpu_cgroup_storage |\n" " queue | stack | sk_storage | struct_ops | ringbuf | inode_storage |\n" - " task_storage | bloom_filter }\n" + " task_storage | bloom_filter | user_ringbuf }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-n|--nomount} }\n" "", diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c index 6b0c410152defa3ef9e325e74dfa9102147c17bd..21d7d447e1f3b2db3740629efc877f52306cb141 100644 --- a/tools/bpf/bpftool/map_perf_ring.c +++ b/tools/bpf/bpftool/map_perf_ring.c @@ -29,13 +29,6 @@ static volatile bool stop; -struct event_ring_info { - int fd; - int key; - unsigned int cpu; - void *mem; -}; - struct perf_event_sample { struct perf_event_header header; __u64 time; @@ -195,10 +188,9 @@ int do_event_pipe(int argc, char **argv) opts.map_keys = &ctx.idx; pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, print_bpf_output, &ctx, &opts); - err = libbpf_get_error(pb); - if (err) { + if (!pb) { p_err("failed to create perf buffer: %s (%d)", - strerror(err), err); + strerror(errno), errno); goto err_close_map; } @@ -213,7 +205,7 @@ int do_event_pipe(int argc, char **argv) err = perf_buffer__poll(pb, 200); if (err < 0 && err != -EINTR) { p_err("perf buffer polling failed: %s (%d)", - strerror(err), err); + strerror(errno), errno); goto err_close_pb; } } diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index fc6ce0b2535ad6522e5ec94b3329b8d09d1aa404..57619f240b5604f8a76f9abaa95288dca64d1df5 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -137,6 +137,12 @@ FEATURE_DISPLAY ?= \ libaio \ libzstd +# +# Declare group members of a feature to display the logical OR of the detection +# result instead of each member result. +# +FEATURE_GROUP_MEMBERS-libbfd = libbfd-liberty libbfd-liberty-z + # Set FEATURE_CHECK_(C|LD)FLAGS-all for all FEATURE_TESTS features. # If in the future we need per-feature checks/flags for features not # mentioned in this list we need to refactor this ;-). @@ -177,19 +183,28 @@ endif # # Print the result of the feature test: # -feature_print_status = $(eval $(feature_print_status_code)) $(info $(MSG)) +feature_print_status = $(eval $(feature_print_status_code)) + +feature_group = $(eval $(feature_gen_group)) $(GROUP) + +define feature_gen_group + GROUP := $(1) + ifneq ($(feature_verbose),1) + GROUP += $(FEATURE_GROUP_MEMBERS-$(1)) + endif +endef define feature_print_status_code - ifeq ($(feature-$(1)), 1) - MSG = $(shell printf '...%30s: [ \033[32mon\033[m ]' $(1)) + ifneq (,$(filter 1,$(foreach feat,$(call feature_group,$(feat)),$(feature-$(feat))))) + MSG = $(shell printf '...%40s: [ \033[32mon\033[m ]' $(1)) else - MSG = $(shell printf '...%30s: [ \033[31mOFF\033[m ]' $(1)) + MSG = $(shell printf '...%40s: [ \033[31mOFF\033[m ]' $(1)) endif endef -feature_print_text = $(eval $(feature_print_text_code)) $(info $(MSG)) +feature_print_text = $(eval $(feature_print_text_code)) define feature_print_text_code - MSG = $(shell printf '...%30s: %s' $(1) $(2)) + MSG = $(shell printf '...%40s: %s' $(1) $(2)) endef # @@ -244,24 +259,29 @@ ifeq ($(VF),1) feature_verbose := 1 endif +ifneq ($(feature_verbose),1) + # + # Determine the features to omit from the displayed message, as only the + # logical OR of the detection result will be shown. + # + FEATURE_OMIT := $(foreach feat,$(FEATURE_DISPLAY),$(FEATURE_GROUP_MEMBERS-$(feat))) +endif + feature_display_entries = $(eval $(feature_display_entries_code)) define feature_display_entries_code ifeq ($(feature_display),1) - $(info ) - $(info Auto-detecting system features:) - $(foreach feat,$(FEATURE_DISPLAY),$(call feature_print_status,$(feat),)) - ifneq ($(feature_verbose),1) - $(info ) - endif + $$(info ) + $$(info Auto-detecting system features:) + $(foreach feat,$(filter-out $(FEATURE_OMIT),$(FEATURE_DISPLAY)),$(call feature_print_status,$(feat),) $$(info $(MSG))) endif ifeq ($(feature_verbose),1) - TMP := $(filter-out $(FEATURE_DISPLAY),$(FEATURE_TESTS)) - $(foreach feat,$(TMP),$(call feature_print_status,$(feat),)) - $(info ) + $(eval TMP := $(filter-out $(FEATURE_DISPLAY),$(FEATURE_TESTS))) + $(foreach feat,$(TMP),$(call feature_print_status,$(feat),) $$(info $(MSG))) endif endef ifeq ($(FEATURE_DISPLAY_DEFERRED),) $(call feature_display_entries) + $(info ) endif diff --git a/tools/cgroup/iocost_monitor.py b/tools/cgroup/iocost_monitor.py index c4ff907c078b5ed923c9c17252341838e11260e9..0dbbc67400fcb2612abe6a994f900f037301cc9b 100644 --- a/tools/cgroup/iocost_monitor.py +++ b/tools/cgroup/iocost_monitor.py @@ -61,6 +61,11 @@ autop_names = { } class BlkgIterator: + def __init__(self, root_blkcg, q_id, include_dying=False): + self.include_dying = include_dying + self.blkgs = [] + self.walk(root_blkcg, q_id, '') + def blkcg_name(blkcg): return blkcg.css.cgroup.kn.name.string_().decode('utf-8') @@ -82,11 +87,6 @@ class BlkgIterator: blkcg.css.children.address_of_(), 'css.sibling'): self.walk(c, q_id, path) - def __init__(self, root_blkcg, q_id, include_dying=False): - self.include_dying = include_dying - self.blkgs = [] - self.walk(root_blkcg, q_id, '') - def __iter__(self): return iter(self.blkgs) diff --git a/tools/debugging/kernel-chktaint b/tools/debugging/kernel-chktaint index f1af27ce9f200f32c4811d2be30a74056057306c..279be06332be990754490662ee00fa4257cecbda 100755 --- a/tools/debugging/kernel-chktaint +++ b/tools/debugging/kernel-chktaint @@ -187,6 +187,7 @@ else echo " * auxiliary taint, defined for and used by distros (#16)" fi + T=`expr $T / 2` if [ `expr $T % 2` -eq 0 ]; then addout " " @@ -195,6 +196,14 @@ else echo " * kernel was built with the struct randomization plugin (#17)" fi +T=`expr $T / 2` +if [ `expr $T % 2` -eq 0 ]; then + addout " " +else + addout "N" + echo " * an in-kernel test (such as a KUnit test) has been run (#18)" +fi + echo "For a more detailed explanation of the various taint flags see" echo " Documentation/admin-guide/tainted-kernels.rst in the Linux kernel sources" echo " or https://kernel.org/doc/html/latest/admin-guide/tainted-kernels.html" diff --git a/tools/hv/hv_kvp_daemon.c b/tools/hv/hv_kvp_daemon.c index 1e6fd6ca513bdf446e2b56816de69a6afc1054eb..27f5e7dfc2f761ad2d2ec9ff8fba996ef046d319 100644 --- a/tools/hv/hv_kvp_daemon.c +++ b/tools/hv/hv_kvp_daemon.c @@ -44,7 +44,7 @@ /* * KVP protocol: The user mode component first registers with the - * the kernel component. Subsequently, the kernel component requests, data + * kernel component. Subsequently, the kernel component requests, data * for the specified keys. In response to this message the user mode component * fills in the value corresponding to the specified key. We overload the * sequence field in the cn_msg header to define our KVP message types. @@ -772,11 +772,11 @@ static int kvp_process_ip_address(void *addrp, const char *str; if (family == AF_INET) { - addr = (struct sockaddr_in *)addrp; + addr = addrp; str = inet_ntop(family, &addr->sin_addr, tmp, 50); addr_length = INET_ADDRSTRLEN; } else { - addr6 = (struct sockaddr_in6 *)addrp; + addr6 = addrp; str = inet_ntop(family, &addr6->sin6_addr.s6_addr, tmp, 50); addr_length = INET6_ADDRSTRLEN; } diff --git a/tools/iio/iio_event_monitor.c b/tools/iio/iio_event_monitor.c index 2f4581658859979e82ecac61db23dc0c6b35bab7..0a5c2bb60030b93b7c67f2afbf87dde2139ac7c3 100644 --- a/tools/iio/iio_event_monitor.c +++ b/tools/iio/iio_event_monitor.c @@ -69,12 +69,15 @@ static const char * const iio_ev_type_text[] = { [IIO_EV_TYPE_MAG_ADAPTIVE] = "mag_adaptive", [IIO_EV_TYPE_CHANGE] = "change", [IIO_EV_TYPE_MAG_REFERENCED] = "mag_referenced", + [IIO_EV_TYPE_GESTURE] = "gesture", }; static const char * const iio_ev_dir_text[] = { [IIO_EV_DIR_EITHER] = "either", [IIO_EV_DIR_RISING] = "rising", - [IIO_EV_DIR_FALLING] = "falling" + [IIO_EV_DIR_FALLING] = "falling", + [IIO_EV_DIR_SINGLETAP] = "singletap", + [IIO_EV_DIR_DOUBLETAP] = "doubletap", }; static const char * const iio_modifier_names[] = { @@ -122,6 +125,12 @@ static const char * const iio_modifier_names[] = { [IIO_MOD_PM4] = "pm4", [IIO_MOD_PM10] = "pm10", [IIO_MOD_O2] = "o2", + [IIO_MOD_LINEAR_X] = "linear_x", + [IIO_MOD_LINEAR_Y] = "linear_y", + [IIO_MOD_LINEAR_Z] = "linear_z", + [IIO_MOD_PITCH] = "pitch", + [IIO_MOD_YAW] = "yaw", + [IIO_MOD_ROLL] = "roll", }; static bool event_is_known(struct iio_event_data *event) @@ -227,6 +236,7 @@ static bool event_is_known(struct iio_event_data *event) case IIO_EV_TYPE_THRESH_ADAPTIVE: case IIO_EV_TYPE_MAG_ADAPTIVE: case IIO_EV_TYPE_CHANGE: + case IIO_EV_TYPE_GESTURE: break; default: return false; @@ -236,6 +246,8 @@ static bool event_is_known(struct iio_event_data *event) case IIO_EV_DIR_EITHER: case IIO_EV_DIR_RISING: case IIO_EV_DIR_FALLING: + case IIO_EV_DIR_SINGLETAP: + case IIO_EV_DIR_DOUBLETAP: case IIO_EV_DIR_NONE: break; default: diff --git a/tools/include/asm-generic/hugetlb_encode.h b/tools/include/asm-generic/hugetlb_encode.h index 4f3d5aaa11f531164beab5a47bed8478e5f17546..de687009bfe5394f139b57f05562eb327e0101af 100644 --- a/tools/include/asm-generic/hugetlb_encode.h +++ b/tools/include/asm-generic/hugetlb_encode.h @@ -20,18 +20,18 @@ #define HUGETLB_FLAG_ENCODE_SHIFT 26 #define HUGETLB_FLAG_ENCODE_MASK 0x3f -#define HUGETLB_FLAG_ENCODE_16KB (14 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_64KB (16 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_512KB (19 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_1MB (20 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_2MB (21 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_8MB (23 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_16MB (24 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_32MB (25 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_256MB (28 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_512MB (29 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_1GB (30 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_2GB (31 << HUGETLB_FLAG_ENCODE_SHIFT) -#define HUGETLB_FLAG_ENCODE_16GB (34 << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16KB (14U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_64KB (16U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_512KB (19U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_1MB (20U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_2MB (21U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_8MB (23U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16MB (24U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_32MB (25U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_256MB (28U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_512MB (29U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_1GB (30U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_2GB (31U << HUGETLB_FLAG_ENCODE_SHIFT) +#define HUGETLB_FLAG_ENCODE_16GB (34U << HUGETLB_FLAG_ENCODE_SHIFT) #endif /* _ASM_GENERIC_HUGETLB_ENCODE_H_ */ diff --git a/tools/include/linux/compiler_types.h b/tools/include/linux/compiler_types.h index 24ae3054f304f27432cf9cdc14503a86d480b9dc..1bdd834bdd57198059c91222036314403191cdbc 100644 --- a/tools/include/linux/compiler_types.h +++ b/tools/include/linux/compiler_types.h @@ -36,4 +36,8 @@ #include <linux/compiler-gcc.h> #endif +#ifndef asm_volatile_goto +#define asm_volatile_goto(x...) asm goto(x) +#endif + #endif /* __LINUX_COMPILER_TYPES_H */ diff --git a/tools/include/linux/find.h b/tools/include/linux/find.h index 47e2bd6c51745d4942f4121dc58004f4d94c0aa5..38c0a542b0e2f20a6852305f4cb2d2fff307f7c6 100644 --- a/tools/include/linux/find.h +++ b/tools/include/linux/find.h @@ -8,21 +8,23 @@ #include <linux/bitops.h> -extern unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le); +unsigned long _find_next_bit(const unsigned long *addr1, unsigned long nbits, + unsigned long start); +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start); +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start); extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size); extern unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size); extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size); -extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size); #ifndef find_next_bit /** * find_next_bit - find the next set bit in a memory region * @addr: The address to base the search on - * @offset: The bitnumber to start searching at * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at * * Returns the bit number for the next set bit * If no bits are set, returns @size. @@ -41,7 +43,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, return val ? __ffs(val) : size; } - return _find_next_bit(addr, NULL, size, offset, 0UL, 0); + return _find_next_bit(addr, size, offset); } #endif @@ -50,8 +52,8 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size, * find_next_and_bit - find the next set bit in both memory regions * @addr1: The first address to base the search on * @addr2: The second address to base the search on - * @offset: The bitnumber to start searching at * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at * * Returns the bit number for the next set bit * If no bits are set, returns @size. @@ -71,7 +73,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1, return val ? __ffs(val) : size; } - return _find_next_bit(addr1, addr2, size, offset, 0UL, 0); + return _find_next_and_bit(addr1, addr2, size, offset); } #endif @@ -79,8 +81,8 @@ unsigned long find_next_and_bit(const unsigned long *addr1, /** * find_next_zero_bit - find the next cleared bit in a memory region * @addr: The address to base the search on - * @offset: The bitnumber to start searching at * @size: The bitmap size in bits + * @offset: The bitnumber to start searching at * * Returns the bit number of the next zero bit * If no bits are zero, returns @size. @@ -99,7 +101,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, return val == ~0UL ? size : ffz(val); } - return _find_next_bit(addr, NULL, size, offset, ~0UL, 0); + return _find_next_zero_bit(addr, size, offset); } #endif @@ -172,43 +174,4 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size) } #endif -#ifndef find_last_bit -/** - * find_last_bit - find the last set bit in a memory region - * @addr: The address to start the search at - * @size: The number of bits to search - * - * Returns the bit number of the last set bit, or size. - */ -static inline -unsigned long find_last_bit(const unsigned long *addr, unsigned long size) -{ - if (small_const_nbits(size)) { - unsigned long val = *addr & GENMASK(size - 1, 0); - - return val ? __fls(val) : size; - } - - return _find_last_bit(addr, size); -} -#endif - -/** - * find_next_clump8 - find next 8-bit clump with set bits in a memory region - * @clump: location to store copy of found clump - * @addr: address to base the search on - * @size: bitmap size in number of bits - * @offset: bit offset at which to start searching - * - * Returns the bit offset for the next set clump; the found clump value is - * copied to the location pointed by @clump. If no bits are set, returns @size. - */ -extern unsigned long find_next_clump8(unsigned long *clump, - const unsigned long *addr, - unsigned long size, unsigned long offset); - -#define find_first_clump8(clump, bits, size) \ - find_next_clump8((clump), (bits), (size), 0) - - #endif /*__LINUX_FIND_H_ */ diff --git a/tools/include/linux/gfp.h b/tools/include/linux/gfp.h index b238dbc9eb858675f701456da7cf6e1f52365197..6a10ff5f5be9070cce7130e62f723e4ecb0a1994 100644 --- a/tools/include/linux/gfp.h +++ b/tools/include/linux/gfp.h @@ -3,26 +3,7 @@ #define _TOOLS_INCLUDE_LINUX_GFP_H #include <linux/types.h> - -#define __GFP_BITS_SHIFT 26 -#define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) - -#define __GFP_HIGH 0x20u -#define __GFP_IO 0x40u -#define __GFP_FS 0x80u -#define __GFP_NOWARN 0x200u -#define __GFP_ZERO 0x8000u -#define __GFP_ATOMIC 0x80000u -#define __GFP_ACCOUNT 0x100000u -#define __GFP_DIRECT_RECLAIM 0x400000u -#define __GFP_KSWAPD_RECLAIM 0x2000000u - -#define __GFP_RECLAIM (__GFP_DIRECT_RECLAIM | __GFP_KSWAPD_RECLAIM) - -#define GFP_ZONEMASK 0x0fu -#define GFP_ATOMIC (__GFP_HIGH | __GFP_ATOMIC | __GFP_KSWAPD_RECLAIM) -#define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) -#define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) +#include <linux/gfp_types.h> static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h new file mode 100644 index 0000000000000000000000000000000000000000..5f9f1ed190a0bf937122973277e02fbb8734ba48 --- /dev/null +++ b/tools/include/linux/gfp_types.h @@ -0,0 +1 @@ +#include "../../../include/linux/gfp_types.h" diff --git a/tools/include/linux/kallsyms.h b/tools/include/linux/kallsyms.h index efb6c3f5f2a9a53a58e436fe120dd7b4d9880d8a..5a37ccbec54fbc603108ad9346c0351c8a3c56a8 100644 --- a/tools/include/linux/kallsyms.h +++ b/tools/include/linux/kallsyms.h @@ -6,7 +6,7 @@ #include <stdio.h> #include <unistd.h> -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 struct module; diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h index 0616409513eb7947409f23045dd6395a9b68ce0e..311759ea25e9214b31cd8afc3b64cea1213972de 100644 --- a/tools/include/linux/slab.h +++ b/tools/include/linux/slab.h @@ -41,4 +41,8 @@ struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, unsigned int align, unsigned int flags, void (*ctor)(void *)); +void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); +int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, + void **list); + #endif /* _TOOLS_SLAB_H */ diff --git a/tools/include/nolibc/arch-riscv.h b/tools/include/nolibc/arch-riscv.h index 95e2b792492572b69acbc4f1a18ea844ea9d7eb0..ba04771cb3a3418fca083b85b4450ed5cf4b6046 100644 --- a/tools/include/nolibc/arch-riscv.h +++ b/tools/include/nolibc/arch-riscv.h @@ -190,7 +190,7 @@ __asm__ (".section .text\n" ".option norelax\n" "lla gp, __global_pointer$\n" ".option pop\n" - "ld a0, 0(sp)\n" // argc (a0) was in the stack + "lw a0, 0(sp)\n" // argc (a0) was in the stack "add a1, sp, "SZREG"\n" // argv (a1) = sp "slli a2, a0, "PTRLOG"\n" // envp (a2) = SZREG*argc ... "add a2, a2, "SZREG"\n" // + SZREG (skip null) diff --git a/tools/include/nolibc/sys.h b/tools/include/nolibc/sys.h index 08491070387bc03ba24ba155d68a70583724c058..ce3ee03aa6794b484e98c93ba4f4addca62e06d0 100644 --- a/tools/include/nolibc/sys.h +++ b/tools/include/nolibc/sys.h @@ -692,12 +692,12 @@ void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd, { #ifndef my_syscall6 /* Function not implemented. */ - return -ENOSYS; + return (void *)-ENOSYS; #else int n; -#if defined(__i386__) +#if defined(__NR_mmap2) n = __NR_mmap2; offset >>= 12; #else diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h index 6c1aa92a92e4411946335cbb9724b75d8efa987a..6ce1f1ceb432c64599f706b86e74a12581c2a54e 100644 --- a/tools/include/uapi/asm-generic/mman-common.h +++ b/tools/include/uapi/asm-generic/mman-common.h @@ -77,6 +77,8 @@ #define MADV_DONTNEED_LOCKED 24 /* like DONTNEED, but drop locked pages too */ +#define MADV_COLLAPSE 25 /* Synchronous hugepage collapse */ + /* compatibility flags */ #define MAP_FILE 0 diff --git a/tools/include/uapi/asm/errno.h b/tools/include/uapi/asm/errno.h index d30439b4b8ab45c7a851891e69ca6ebea007a319..869379f91fe487ad3611966b09bda47462da0b70 100644 --- a/tools/include/uapi/asm/errno.h +++ b/tools/include/uapi/asm/errno.h @@ -9,8 +9,8 @@ #include "../../../arch/alpha/include/uapi/asm/errno.h" #elif defined(__mips__) #include "../../../arch/mips/include/uapi/asm/errno.h" -#elif defined(__xtensa__) -#include "../../../arch/xtensa/include/uapi/asm/errno.h" +#elif defined(__hppa__) +#include "../../../arch/parisc/include/uapi/asm/errno.h" #else #include <asm-generic/errno.h> #endif diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 59a217ca2dfd3e6f3ad53a3c0eb448af6664afda..51b9aa640ad2af58bebe1b846685e75bf6f9011f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -87,10 +87,35 @@ struct bpf_cgroup_storage_key { __u32 attach_type; /* program attach type (enum bpf_attach_type) */ }; +enum bpf_cgroup_iter_order { + BPF_CGROUP_ITER_ORDER_UNSPEC = 0, + BPF_CGROUP_ITER_SELF_ONLY, /* process only a single object. */ + BPF_CGROUP_ITER_DESCENDANTS_PRE, /* walk descendants in pre-order. */ + BPF_CGROUP_ITER_DESCENDANTS_POST, /* walk descendants in post-order. */ + BPF_CGROUP_ITER_ANCESTORS_UP, /* walk ancestors upward. */ +}; + union bpf_iter_link_info { struct { __u32 map_fd; } map; + struct { + enum bpf_cgroup_iter_order order; + + /* At most one of cgroup_fd and cgroup_id can be non-zero. If + * both are zero, the walk starts from the default cgroup v2 + * root. For walking v1 hierarchy, one should always explicitly + * specify cgroup_fd. + */ + __u32 cgroup_fd; + __u64 cgroup_id; + } cgroup; + /* Parameters of task iterators. */ + struct { + __u32 tid; + __u32 pid; + __u32 pid_fd; + } task; }; /* BPF syscall commands, see bpf(2) man-page for more details. */ @@ -909,6 +934,7 @@ enum bpf_map_type { BPF_MAP_TYPE_INODE_STORAGE, BPF_MAP_TYPE_TASK_STORAGE, BPF_MAP_TYPE_BLOOM_FILTER, + BPF_MAP_TYPE_USER_RINGBUF, }; /* Note that tracing related programs such as @@ -1233,7 +1259,7 @@ enum { /* Query effective (directly attached + inherited from ancestor cgroups) * programs that will be executed for events within a cgroup. - * attach_flags with this flag are returned only for directly attached programs. + * attach_flags with this flag are always returned 0. */ #define BPF_F_QUERY_EFFECTIVE (1U << 0) @@ -1432,7 +1458,10 @@ union bpf_attr { __u32 attach_flags; __aligned_u64 prog_ids; __u32 prog_cnt; - __aligned_u64 prog_attach_flags; /* output: per-program attach_flags */ + /* output: per-program attach_flags. + * not allowed to be set during effective query. + */ + __aligned_u64 prog_attach_flags; } query; struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ @@ -2573,10 +2602,12 @@ union bpf_attr { * There are two supported modes at this time: * * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer - * (room space is added or removed below the layer 2 header). + * (room space is added or removed between the layer 2 and + * layer 3 headers). * * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer - * (room space is added or removed below the layer 3 header). + * (room space is added or removed between the layer 3 and + * layer 4 headers). * * The following flags are supported at this time: * @@ -3008,8 +3039,18 @@ union bpf_attr { * **BPF_F_USER_STACK** * Collect a user space stack instead of a kernel stack. * **BPF_F_USER_BUILD_ID** - * Collect buildid+offset instead of ips for user stack, - * only valid if **BPF_F_USER_STACK** is also specified. + * Collect (build_id, file_offset) instead of ips for user + * stack, only valid if **BPF_F_USER_STACK** is also + * specified. + * + * *file_offset* is an offset relative to the beginning + * of the executable or shared object file backing the vma + * which the *ip* falls in. It is *not* an offset relative + * to that object's base address. Accordingly, it must be + * adjusted by adding (sh_addr - sh_offset), where + * sh_{addr,offset} correspond to the executable section + * containing *file_offset* in the object, for comparisons + * to symbols' st_value to be valid. * * **bpf_get_stack**\ () can collect up to * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject @@ -4425,7 +4466,7 @@ union bpf_attr { * * **-EEXIST** if the option already exists. * - * **-EFAULT** on failrue to parse the existing header options. + * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. @@ -4634,7 +4675,7 @@ union bpf_attr { * a *map* with *task* as the **key**. From this * perspective, the usage is not much different from * **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this - * helper enforces the key must be an task_struct and the map must also + * helper enforces the key must be a task_struct and the map must also * be a **BPF_MAP_TYPE_TASK_STORAGE**. * * Underneath, the value is stored locally at *task* instead of @@ -4692,7 +4733,7 @@ union bpf_attr { * * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size) * Description - * Returns the stored IMA hash of the *inode* (if it's avaialable). + * Returns the stored IMA hash of the *inode* (if it's available). * If the hash is larger than *size*, then only *size* * bytes will be copied to *dst* * Return @@ -4716,12 +4757,12 @@ union bpf_attr { * * The argument *len_diff* can be used for querying with a planned * size change. This allows to check MTU prior to changing packet - * ctx. Providing an *len_diff* adjustment that is larger than the + * ctx. Providing a *len_diff* adjustment that is larger than the * actual packet size (resulting in negative packet size) will in - * principle not exceed the MTU, why it is not considered a - * failure. Other BPF-helpers are needed for performing the - * planned size change, why the responsability for catch a negative - * packet size belong in those helpers. + * principle not exceed the MTU, which is why it is not considered + * a failure. Other BPF helpers are needed for performing the + * planned size change; therefore the responsibility for catching + * a negative packet size belongs in those helpers. * * Specifying *ifindex* zero means the MTU check is performed * against the current net device. This is practical if this isn't @@ -4919,6 +4960,7 @@ union bpf_attr { * Get address of the traced function (for tracing and kprobe programs). * Return * Address of the traced function. + * 0 for kprobes placed within the function (not at the entry). * * u64 bpf_get_attach_cookie(void *ctx) * Description @@ -5048,12 +5090,12 @@ union bpf_attr { * * long bpf_get_func_arg(void *ctx, u32 n, u64 *value) * Description - * Get **n**-th argument (zero based) of the traced function (for tracing programs) + * Get **n**-th argument register (zero based) of the traced function (for tracing programs) * returned in **value**. * * Return * 0 on success. - * **-EINVAL** if n >= arguments count of traced function. + * **-EINVAL** if n >= argument register count of traced function. * * long bpf_get_func_ret(void *ctx, u64 *value) * Description @@ -5066,24 +5108,37 @@ union bpf_attr { * * long bpf_get_func_arg_cnt(void *ctx) * Description - * Get number of arguments of the traced function (for tracing programs). + * Get number of registers of the traced function (for tracing programs) where + * function arguments are stored in these registers. * * Return - * The number of arguments of the traced function. + * The number of argument registers of the traced function. * * int bpf_get_retval(void) * Description - * Get the syscall's return value that will be returned to userspace. + * Get the BPF program's return value that will be returned to the upper layers. * - * This helper is currently supported by cgroup programs only. + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. * Return - * The syscall's return value. + * The BPF program's return value. * * int bpf_set_retval(int retval) * Description - * Set the syscall's return value that will be returned to userspace. + * Set the BPF program's return value that will be returned to the upper layers. + * + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. + * + * Note that there is the following corner case where the program exports an error + * via bpf_set_retval but signals success via 'return 1': + * + * bpf_set_retval(-EPERM); + * return 1; + * + * In this case, the BPF program's return value will use helper's -EPERM. This + * still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case. * - * This helper is currently supported by cgroup programs only. * Return * 0 on success, or a negative error in case of failure. * @@ -5331,6 +5386,55 @@ union bpf_attr { * **-EACCES** if the SYN cookie is not valid. * * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. + * + * u64 bpf_ktime_get_tai_ns(void) + * Description + * A nonsettable system-wide clock derived from wall-clock time but + * ignoring leap seconds. This clock does not experience + * discontinuities and backwards jumps caused by NTP inserting leap + * seconds as CLOCK_REALTIME does. + * + * See: **clock_gettime**\ (**CLOCK_TAI**) + * Return + * Current *ktime*. + * + * long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags) + * Description + * Drain samples from the specified user ring buffer, and invoke + * the provided callback for each such sample: + * + * long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx); + * + * If **callback_fn** returns 0, the helper will continue to try + * and drain the next sample, up to a maximum of + * BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, + * the helper will skip the rest of the samples and return. Other + * return values are not used now, and will be rejected by the + * verifier. + * Return + * The number of drained samples if no error was encountered while + * draining samples, or 0 if no samples were present in the ring + * buffer. If a user-space producer was epoll-waiting on this map, + * and at least one sample was drained, they will receive an event + * notification notifying them of available space in the ring + * buffer. If the BPF_RB_NO_WAKEUP flag is passed to this + * function, no wakeup notification will be sent. If the + * BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will + * be sent even if no sample was drained. + * + * On failure, the returned value is one of the following: + * + * **-EBUSY** if the ring buffer is contended, and another calling + * context was concurrently draining the ring buffer. + * + * **-EINVAL** if user-space is not properly tracking the ring + * buffer due to the producer position not being aligned to 8 + * bytes, a sample not being aligned to 8 bytes, or the producer + * position not matching the advertised length of a sample. + * + * **-E2BIG** if user-space has tried to publish a sample which is + * larger than the size of the ring buffer, or which cannot fit + * within a struct bpf_dynptr. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5541,6 +5645,8 @@ union bpf_attr { FN(tcp_raw_gen_syncookie_ipv6), \ FN(tcp_raw_check_syncookie_ipv4), \ FN(tcp_raw_check_syncookie_ipv6), \ + FN(ktime_get_tai_ns), \ + FN(user_ringbuf_drain), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -5603,6 +5709,11 @@ enum { BPF_F_SEQ_NUMBER = (1ULL << 3), }; +/* BPF_FUNC_skb_get_tunnel_key flags. */ +enum { + BPF_F_TUNINFO_FLAGS = (1ULL << 4), +}; + /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and * BPF_FUNC_perf_event_read_value flags. */ @@ -5792,7 +5903,10 @@ struct bpf_tunnel_key { }; __u8 tunnel_tos; __u8 tunnel_ttl; - __u16 tunnel_ext; /* Padding, future use. */ + union { + __u16 tunnel_ext; /* compat */ + __be16 tunnel_flags; + }; __u32 tunnel_label; union { __u32 local_ipv4; @@ -5836,6 +5950,11 @@ enum bpf_ret_code { * represented by BPF_REDIRECT above). */ BPF_LWT_REROUTE = 128, + /* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR + * to indicate that no custom dissection was performed, and + * fallback to standard dissector is requested. + */ + BPF_FLOW_DISSECTOR_CONTINUE = 129, }; struct bpf_sock { @@ -6134,11 +6253,26 @@ struct bpf_link_info { struct { __aligned_u64 target_name; /* in/out: target_name buffer ptr */ __u32 target_name_len; /* in/out: target_name buffer len */ + + /* If the iter specific field is 32 bits, it can be put + * in the first or second union. Otherwise it should be + * put in the second union. + */ union { struct { __u32 map_id; } map; }; + union { + struct { + __u64 cgroup_id; + __u32 order; + } cgroup; + struct { + __u32 tid; + __u32 pid; + } task; + }; } iter; struct { __u32 netns_ino; diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h index 581ed4bdc06219ee7c42516ba48b436b8abcf6c8..ea6defacc1a7d22d4fafdc1fb95d732054f158b1 100644 --- a/tools/include/uapi/linux/perf_event.h +++ b/tools/include/uapi/linux/perf_event.h @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift { PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */ + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privilege mode */ + PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */ }; @@ -233,6 +235,8 @@ enum perf_branch_sample_type { PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT, + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT, + PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT, }; @@ -253,9 +257,37 @@ enum { PERF_BR_COND_RET = 10, /* conditional function return */ PERF_BR_ERET = 11, /* exception return */ PERF_BR_IRQ = 12, /* irq */ + PERF_BR_SERROR = 13, /* system error */ + PERF_BR_NO_TX = 14, /* not in transaction */ + PERF_BR_EXTEND_ABI = 15, /* extend ABI */ PERF_BR_MAX, }; +enum { + PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */ + PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */ + PERF_BR_NEW_FAULT_INST = 2, /* Inst fault */ + PERF_BR_NEW_ARCH_1 = 3, /* Architecture specific */ + PERF_BR_NEW_ARCH_2 = 4, /* Architecture specific */ + PERF_BR_NEW_ARCH_3 = 5, /* Architecture specific */ + PERF_BR_NEW_ARCH_4 = 6, /* Architecture specific */ + PERF_BR_NEW_ARCH_5 = 7, /* Architecture specific */ + PERF_BR_NEW_MAX, +}; + +enum { + PERF_BR_PRIV_UNKNOWN = 0, + PERF_BR_PRIV_USER = 1, + PERF_BR_PRIV_KERNEL = 2, + PERF_BR_PRIV_HV = 3, +}; + +#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1 +#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2 +#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3 +#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4 +#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5 + #define PERF_SAMPLE_BRANCH_PLM_ALL \ (PERF_SAMPLE_BRANCH_USER|\ PERF_SAMPLE_BRANCH_KERNEL|\ @@ -1295,7 +1327,9 @@ union perf_mem_data_src { #define PERF_MEM_LVLNUM_L2 0x02 /* L2 */ #define PERF_MEM_LVLNUM_L3 0x03 /* L3 */ #define PERF_MEM_LVLNUM_L4 0x04 /* L4 */ -/* 5-0xa available */ +/* 5-0x8 available */ +#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */ +#define PERF_MEM_LVLNUM_IO 0x0a /* I/O */ #define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */ #define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */ #define PERF_MEM_LVLNUM_RAM 0x0d /* RAM */ @@ -1373,7 +1407,9 @@ struct perf_branch_entry { abort:1, /* transaction abort */ cycles:16, /* cycle count to last branch */ type:4, /* branch type */ - reserved:40; + new_type:4, /* additional branch type */ + priv:3, /* privilege level */ + reserved:33; }; union perf_sample_weight { diff --git a/tools/include/uapi/linux/tc_act/tc_bpf.h b/tools/include/uapi/linux/tc_act/tc_bpf.h index 653c4f94f76e371cf08283010db13615f5d1dfc0..fe6c8f8f3e8c60961135609d5bce024979064ccc 100644 --- a/tools/include/uapi/linux/tc_act/tc_bpf.h +++ b/tools/include/uapi/linux/tc_act/tc_bpf.h @@ -1,11 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ /* * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. */ #ifndef __LINUX_TC_BPF_H diff --git a/tools/lib/api/fd/array.h b/tools/lib/api/fd/array.h index 60ad197c8ee94dc2a4b8bfd0bf7a95f0ab1d10c8..5c01f7b05dfb1739183865566c46e3c486dd3683 100644 --- a/tools/lib/api/fd/array.h +++ b/tools/lib/api/fd/array.h @@ -31,8 +31,9 @@ struct fdarray { }; enum fdarray_flags { - fdarray_flag__default = 0x00000000, - fdarray_flag__nonfilterable = 0x00000001 + fdarray_flag__default = 0x00000000, + fdarray_flag__nonfilterable = 0x00000001, + fdarray_flag__non_perf_event = 0x00000002, }; void fdarray__init(struct fdarray *fda, int nr_autogrow); diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index efcc06dafbd991ba317bd9d147775e6286560b9f..1d49a035283653943a50b05d1f7c4a506367c7da 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -84,9 +84,7 @@ static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr, return ensure_good_fd(fd); } -#define PROG_LOAD_ATTEMPTS 5 - -static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) +int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) { int fd; @@ -107,7 +105,7 @@ static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int */ int probe_memcg_account(void) { - const size_t prog_load_attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); + const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd); struct bpf_insn insns[] = { BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns), BPF_EXIT_INSN(), @@ -117,13 +115,13 @@ int probe_memcg_account(void) int prog_fd; /* attempt loading freplace trying to use custom BTF */ - memset(&attr, 0, prog_load_attr_sz); + memset(&attr, 0, attr_sz); attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; attr.insns = ptr_to_u64(insns); attr.insn_cnt = insn_cnt; attr.license = ptr_to_u64("GPL"); - prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, prog_load_attr_sz); + prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz); if (prog_fd >= 0) { close(prog_fd); return 1; @@ -183,7 +181,7 @@ int bpf_map_create(enum bpf_map_type map_type, return libbpf_err(-EINVAL); attr.map_type = map_type; - if (map_name) + if (map_name && kernel_supports(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name)); attr.key_size = key_size; attr.value_size = value_size; @@ -234,6 +232,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insn_cnt, const struct bpf_prog_load_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, fd_array); void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; @@ -253,7 +252,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, if (attempts == 0) attempts = PROG_LOAD_ATTEMPTS; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.prog_type = prog_type; attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); @@ -263,7 +262,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); attr.kern_version = OPTS_GET(opts, kern_version, 0); - if (prog_name) + if (prog_name && kernel_supports(NULL, FEAT_PROG_NAME)) libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name)); attr.license = ptr_to_u64(license); @@ -316,7 +315,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, attr.log_level = log_level; } - fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); + fd = sys_bpf_prog_load(&attr, attr_sz, attempts); if (fd >= 0) return fd; @@ -356,7 +355,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, break; } - fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); + fd = sys_bpf_prog_load(&attr, attr_sz, attempts); if (fd >= 0) goto done; } @@ -370,7 +369,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type, attr.log_size = log_size; attr.log_level = 1; - fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); + fd = sys_bpf_prog_load(&attr, attr_sz, attempts); } done: /* free() doesn't affect errno, so we don't need to restore it */ @@ -382,127 +381,136 @@ done: int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; - ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_elem(int fd, const void *key, void *value) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); - ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; - ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); - ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.value = ptr_to_u64(value); attr.flags = flags; - ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_delete_elem(int fd, const void *key) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); - ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags) { + const size_t attr_sz = offsetofend(union bpf_attr, flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.flags = flags; - ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_get_next_key(int fd, const void *key, void *next_key) { + const size_t attr_sz = offsetofend(union bpf_attr, next_key); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; attr.key = ptr_to_u64(key); attr.next_key = ptr_to_u64(next_key); - ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_map_freeze(int fd) { + const size_t attr_sz = offsetofend(union bpf_attr, map_fd); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_fd = fd; - ret = sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr)); + ret = sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz); return libbpf_err_errno(ret); } @@ -511,13 +519,14 @@ static int bpf_map_batch_common(int cmd, int fd, void *in_batch, __u32 *count, const struct bpf_map_batch_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, batch); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_map_batch_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.batch.map_fd = fd; attr.batch.in_batch = ptr_to_u64(in_batch); attr.batch.out_batch = ptr_to_u64(out_batch); @@ -527,7 +536,7 @@ static int bpf_map_batch_common(int cmd, int fd, void *in_batch, attr.batch.elem_flags = OPTS_GET(opts, elem_flags, 0); attr.batch.flags = OPTS_GET(opts, flags, 0); - ret = sys_bpf(cmd, &attr, sizeof(attr)); + ret = sys_bpf(cmd, &attr, attr_sz); *count = attr.batch.count; return libbpf_err_errno(ret); @@ -566,14 +575,15 @@ int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *co int bpf_obj_pin(int fd, const char *pathname) { + const size_t attr_sz = offsetofend(union bpf_attr, file_flags); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.pathname = ptr_to_u64((void *)pathname); attr.bpf_fd = fd; - ret = sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr)); + ret = sys_bpf(BPF_OBJ_PIN, &attr, attr_sz); return libbpf_err_errno(ret); } @@ -584,17 +594,18 @@ int bpf_obj_get(const char *pathname) int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, file_flags); union bpf_attr attr; int fd; if (!OPTS_VALID(opts, bpf_obj_get_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.pathname = ptr_to_u64((void *)pathname); attr.file_flags = OPTS_GET(opts, file_flags, 0); - fd = sys_bpf_fd(BPF_OBJ_GET, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_OBJ_GET, &attr, attr_sz); return libbpf_err_errno(fd); } @@ -612,52 +623,50 @@ int bpf_prog_attach_opts(int prog_fd, int target_fd, enum bpf_attach_type type, const struct bpf_prog_attach_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_prog_attach_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.target_fd = target_fd; attr.attach_bpf_fd = prog_fd; attr.attach_type = type; attr.attach_flags = OPTS_GET(opts, flags, 0); attr.replace_bpf_fd = OPTS_GET(opts, replace_prog_fd, 0); - ret = sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz); return libbpf_err_errno(ret); } -__attribute__((alias("bpf_prog_attach_opts"))) -int bpf_prog_attach_xattr(int prog_fd, int target_fd, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts); - int bpf_prog_detach(int target_fd, enum bpf_attach_type type) { + const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.target_fd = target_fd; attr.attach_type = type; - ret = sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type) { + const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.target_fd = target_fd; attr.attach_bpf_fd = prog_fd; attr.attach_type = type; - ret = sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz); return libbpf_err_errno(ret); } @@ -665,6 +674,7 @@ int bpf_link_create(int prog_fd, int target_fd, enum bpf_attach_type attach_type, const struct bpf_link_create_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, link_create); __u32 target_btf_id, iter_info_len; union bpf_attr attr; int fd, err; @@ -683,7 +693,7 @@ int bpf_link_create(int prog_fd, int target_fd, return libbpf_err(-EINVAL); } - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.link_create.prog_fd = prog_fd; attr.link_create.target_fd = target_fd; attr.link_create.attach_type = attach_type; @@ -727,7 +737,7 @@ int bpf_link_create(int prog_fd, int target_fd, break; } proceed: - fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, attr_sz); if (fd >= 0) return fd; /* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry @@ -763,44 +773,47 @@ proceed: int bpf_link_detach(int link_fd) { + const size_t attr_sz = offsetofend(union bpf_attr, link_detach); union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.link_detach.link_fd = link_fd; - ret = sys_bpf(BPF_LINK_DETACH, &attr, sizeof(attr)); + ret = sys_bpf(BPF_LINK_DETACH, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_link_update(int link_fd, int new_prog_fd, const struct bpf_link_update_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, link_update); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_link_update_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.link_update.link_fd = link_fd; attr.link_update.new_prog_fd = new_prog_fd; attr.link_update.flags = OPTS_GET(opts, flags, 0); attr.link_update.old_prog_fd = OPTS_GET(opts, old_prog_fd, 0); - ret = sys_bpf(BPF_LINK_UPDATE, &attr, sizeof(attr)); + ret = sys_bpf(BPF_LINK_UPDATE, &attr, attr_sz); return libbpf_err_errno(ret); } int bpf_iter_create(int link_fd) { + const size_t attr_sz = offsetofend(union bpf_attr, iter_create); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.iter_create.link_fd = link_fd; - fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, attr_sz); return libbpf_err_errno(fd); } @@ -808,13 +821,14 @@ int bpf_prog_query_opts(int target_fd, enum bpf_attach_type type, struct bpf_prog_query_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, query); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_prog_query_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.query.target_fd = target_fd; attr.query.attach_type = type; @@ -823,7 +837,7 @@ int bpf_prog_query_opts(int target_fd, attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); - ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz); OPTS_SET(opts, attach_flags, attr.query.attach_flags); OPTS_SET(opts, prog_cnt, attr.query.prog_cnt); @@ -852,13 +866,14 @@ int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, test); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_test_run_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.test.prog_fd = prog_fd; attr.test.batch_size = OPTS_GET(opts, batch_size, 0); attr.test.cpu = OPTS_GET(opts, cpu, 0); @@ -874,7 +889,7 @@ int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts) attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL)); attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL)); - ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, attr_sz); OPTS_SET(opts, data_size_out, attr.test.data_size_out); OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out); @@ -886,13 +901,14 @@ int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts) static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd) { + const size_t attr_sz = offsetofend(union bpf_attr, open_flags); union bpf_attr attr; int err; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.start_id = start_id; - err = sys_bpf(cmd, &attr, sizeof(attr)); + err = sys_bpf(cmd, &attr, attr_sz); if (!err) *next_id = attr.next_id; @@ -921,80 +937,84 @@ int bpf_link_get_next_id(__u32 start_id, __u32 *next_id) int bpf_prog_get_fd_by_id(__u32 id) { + const size_t attr_sz = offsetofend(union bpf_attr, open_flags); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.prog_id = id; - fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, attr_sz); return libbpf_err_errno(fd); } int bpf_map_get_fd_by_id(__u32 id) { + const size_t attr_sz = offsetofend(union bpf_attr, open_flags); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.map_id = id; - fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); return libbpf_err_errno(fd); } int bpf_btf_get_fd_by_id(__u32 id) { + const size_t attr_sz = offsetofend(union bpf_attr, open_flags); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.btf_id = id; - fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz); return libbpf_err_errno(fd); } int bpf_link_get_fd_by_id(__u32 id) { + const size_t attr_sz = offsetofend(union bpf_attr, open_flags); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.link_id = id; - fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, attr_sz); return libbpf_err_errno(fd); } int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len) { + const size_t attr_sz = offsetofend(union bpf_attr, info); union bpf_attr attr; int err; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.info.bpf_fd = bpf_fd; attr.info.info_len = *info_len; attr.info.info = ptr_to_u64(info); - err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)); - + err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz); if (!err) *info_len = attr.info.info_len; - return libbpf_err_errno(err); } int bpf_raw_tracepoint_open(const char *name, int prog_fd) { + const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.raw_tracepoint.name = ptr_to_u64(name); attr.raw_tracepoint.prog_fd = prog_fd; - fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz); return libbpf_err_errno(fd); } @@ -1050,16 +1070,18 @@ int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, __u64 *probe_addr) { - union bpf_attr attr = {}; + const size_t attr_sz = offsetofend(union bpf_attr, task_fd_query); + union bpf_attr attr; int err; + memset(&attr, 0, attr_sz); attr.task_fd_query.pid = pid; attr.task_fd_query.fd = fd; attr.task_fd_query.flags = flags; attr.task_fd_query.buf = ptr_to_u64(buf); attr.task_fd_query.buf_len = *buf_len; - err = sys_bpf(BPF_TASK_FD_QUERY, &attr, sizeof(attr)); + err = sys_bpf(BPF_TASK_FD_QUERY, &attr, attr_sz); *buf_len = attr.task_fd_query.buf_len; *prog_id = attr.task_fd_query.prog_id; @@ -1072,30 +1094,32 @@ int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, int bpf_enable_stats(enum bpf_stats_type type) { + const size_t attr_sz = offsetofend(union bpf_attr, enable_stats); union bpf_attr attr; int fd; - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.enable_stats.type = type; - fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, sizeof(attr)); + fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, attr_sz); return libbpf_err_errno(fd); } int bpf_prog_bind_map(int prog_fd, int map_fd, const struct bpf_prog_bind_opts *opts) { + const size_t attr_sz = offsetofend(union bpf_attr, prog_bind_map); union bpf_attr attr; int ret; if (!OPTS_VALID(opts, bpf_prog_bind_opts)) return libbpf_err(-EINVAL); - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, attr_sz); attr.prog_bind_map.prog_fd = prog_fd; attr.prog_bind_map.map_fd = map_fd; attr.prog_bind_map.flags = OPTS_GET(opts, flags, 0); - ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, sizeof(attr)); + ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz); return libbpf_err_errno(ret); } diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 7349b16b8e2f0817ad5cb2b1c48f2e0a2f462c0e..d37c4fe2849d22163243451f855ff351fa52e806 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -160,18 +160,6 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) } #endif -/* - * Helper structure used by eBPF C program - * to describe BPF map attributes to libbpf loader - */ -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; -} __attribute__((deprecated("use BTF-defined maps in .maps section"))); - enum libbpf_pin_type { LIBBPF_PIN_NONE, /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index 43ca3aff2292ff47db34f552dc1fcf726e8e18f0..2972dc25ff72226b9938106c950a2cd000b393d5 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -426,7 +426,7 @@ struct pt_regs; */ #define BPF_PROG(name, args...) \ name(unsigned long long *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(unsigned long long *ctx, ##args); \ typeof(name(0)) name(unsigned long long *ctx) \ { \ @@ -435,9 +435,116 @@ typeof(name(0)) name(unsigned long long *ctx) \ return ____##name(___bpf_ctx_cast(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(unsigned long long *ctx, ##args) +#ifndef ___bpf_nth2 +#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ + _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N +#endif +#ifndef ___bpf_narg2 +#define ___bpf_narg2(...) \ + ___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \ + 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0) +#endif + +#define ___bpf_treg_cnt(t) \ + __builtin_choose_expr(sizeof(t) == 1, 1, \ + __builtin_choose_expr(sizeof(t) == 2, 1, \ + __builtin_choose_expr(sizeof(t) == 4, 1, \ + __builtin_choose_expr(sizeof(t) == 8, 1, \ + __builtin_choose_expr(sizeof(t) == 16, 2, \ + (void)0))))) + +#define ___bpf_reg_cnt0() (0) +#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) + +#define ___bpf_union_arg(t, x, n) \ + __builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \ + (void)0))))) + +#define ___bpf_ctx_arg0(n, args...) +#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) +#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) +#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) +#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) +#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) +#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) +#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) +#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) +#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) +#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) +#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) +#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) +#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) + +#define ___bpf_ctx_decl0() +#define ___bpf_ctx_decl1(t, x) , t x +#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) +#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) +#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) +#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) +#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) +#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) +#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) +#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) +#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) +#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) +#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) +#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) + +/* + * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct + * arguments. Since each struct argument might take one or two u64 values + * in the trampoline stack, argument type size is needed to place proper number + * of u64 values for each argument. Therefore, BPF_PROG2 has different + * syntax from BPF_PROG. For example, for the following BPF_PROG syntax: + * + * int BPF_PROG(test2, int a, int b) { ... } + * + * the corresponding BPF_PROG2 syntax is: + * + * int BPF_PROG2(test2, int, a, int, b) { ... } + * + * where type and the corresponding argument name are separated by comma. + * + * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger + * than 8 bytes: + * + * int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b, + * int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret) + * { + * // access a, b, c, d, e, and ret directly + * ... + * } + */ +#define BPF_PROG2(name, args...) \ +name(unsigned long long *ctx); \ +static __always_inline typeof(name(0)) \ +____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \ +typeof(name(0)) name(unsigned long long *ctx) \ +{ \ + return ____##name(ctx ___bpf_ctx_arg(args)); \ +} \ +static __always_inline typeof(name(0)) \ +____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) + struct pt_regs; #define ___bpf_kprobe_args0() ctx @@ -460,7 +567,7 @@ struct pt_regs; */ #define BPF_KPROBE(name, args...) \ name(struct pt_regs *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -469,7 +576,7 @@ typeof(name(0)) name(struct pt_regs *ctx) \ return ____##name(___bpf_kprobe_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) #define ___bpf_kretprobe_args0() ctx @@ -484,7 +591,7 @@ ____##name(struct pt_regs *ctx, ##args) */ #define BPF_KRETPROBE(name, args...) \ name(struct pt_regs *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -540,7 +647,7 @@ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) #define BPF_KSYSCALL(name, args...) \ name(struct pt_regs *ctx); \ extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -555,7 +662,7 @@ typeof(name(0)) name(struct pt_regs *ctx) \ return ____##name(___bpf_syscall_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) #define BPF_KPROBE_SYSCALL BPF_KSYSCALL diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 2d14f1a52d7aa8780f1a6696246165486270e7d6..d88647da2c7fc599780a250cae309632ba9e2fb8 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -1225,8 +1225,6 @@ int btf__load_into_kernel(struct btf *btf) return btf_load_into_kernel(btf, NULL, 0, 0); } -int btf__load(struct btf *) __attribute__((alias("btf__load_into_kernel"))); - int btf__fd(const struct btf *btf) { return btf->fd; @@ -4644,20 +4642,17 @@ static int btf_dedup_remap_types(struct btf_dedup *d) */ struct btf *btf__load_vmlinux_btf(void) { - struct { - const char *path_fmt; - bool raw_btf; - } locations[] = { + const char *locations[] = { /* try canonical vmlinux BTF through sysfs first */ - { "/sys/kernel/btf/vmlinux", true /* raw BTF */ }, - /* fall back to trying to find vmlinux ELF on disk otherwise */ - { "/boot/vmlinux-%1$s" }, - { "/lib/modules/%1$s/vmlinux-%1$s" }, - { "/lib/modules/%1$s/build/vmlinux" }, - { "/usr/lib/modules/%1$s/kernel/vmlinux" }, - { "/usr/lib/debug/boot/vmlinux-%1$s" }, - { "/usr/lib/debug/boot/vmlinux-%1$s.debug" }, - { "/usr/lib/debug/lib/modules/%1$s/vmlinux" }, + "/sys/kernel/btf/vmlinux", + /* fall back to trying to find vmlinux on disk otherwise */ + "/boot/vmlinux-%1$s", + "/lib/modules/%1$s/vmlinux-%1$s", + "/lib/modules/%1$s/build/vmlinux", + "/usr/lib/modules/%1$s/kernel/vmlinux", + "/usr/lib/debug/boot/vmlinux-%1$s", + "/usr/lib/debug/boot/vmlinux-%1$s.debug", + "/usr/lib/debug/lib/modules/%1$s/vmlinux", }; char path[PATH_MAX + 1]; struct utsname buf; @@ -4667,15 +4662,12 @@ struct btf *btf__load_vmlinux_btf(void) uname(&buf); for (i = 0; i < ARRAY_SIZE(locations); i++) { - snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release); + snprintf(path, PATH_MAX, locations[i], buf.release); - if (access(path, R_OK)) + if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS)) continue; - if (locations[i].raw_btf) - btf = btf__parse_raw(path); - else - btf = btf__parse_elf(path, NULL); + btf = btf__parse(path, NULL); err = libbpf_get_error(btf); pr_debug("loading kernel BTF '%s': %d\n", path, err); if (err) diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 583760df83b42081fc74a7a134e6042c587de6b9..8e6880d91c8408261f2fc2071a4376ca279829a1 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -116,7 +116,6 @@ LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_b LIBBPF_API struct btf *btf__load_vmlinux_btf(void); LIBBPF_API struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf); -LIBBPF_API struct btf *libbpf_find_kernel_btf(void); LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id); LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf); @@ -487,6 +486,8 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t) return (struct btf_enum *)(t + 1); } +struct btf_enum64; + static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) { return (struct btf_enum64 *)(t + 1); @@ -494,7 +495,28 @@ static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) static inline __u64 btf_enum64_value(const struct btf_enum64 *e) { - return ((__u64)e->val_hi32 << 32) | e->val_lo32; + /* struct btf_enum64 is introduced in Linux 6.0, which is very + * bleeding-edge. Here we are avoiding relying on struct btf_enum64 + * definition coming from kernel UAPI headers to support wider range + * of system-wide kernel headers. + * + * Given this header can be also included from C++ applications, that + * further restricts C tricks we can use (like using compatible + * anonymous struct). So just treat struct btf_enum64 as + * a three-element array of u32 and access second (lo32) and third + * (hi32) elements directly. + * + * For reference, here is a struct btf_enum64 definition: + * + * const struct btf_enum64 { + * __u32 name_off; + * __u32 val_lo32; + * __u32 val_hi32; + * }; + */ + const __u32 *e64 = (const __u32 *)e; + + return ((__u64)e64[2] << 32) | e64[1]; } static inline struct btf_member *btf_members(const struct btf_type *t) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 627edb5bb6def42d18008b67290d9a96ca880c8e..4221f73a74d016e2923a9529a58d481d4b4932b6 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -2385,7 +2385,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->indent_lvl = OPTS_GET(opts, indent_level, 0); /* default indent string is a tab */ - if (!opts->indent_str) + if (!OPTS_GET(opts, indent_str, NULL)) d->typed_dump->indent_str[0] = '\t'; else libbpf_strlcpy(d->typed_dump->indent_str, opts->indent_str, diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 50d41815f431a7498ac92f17b3fa41332d3e9c85..184ce1684dcd4c46b0cf4525f0b80047c5446692 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -163,6 +163,7 @@ static const char * const map_type_name[] = { [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", + [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", }; static const char * const prog_type_name[] = { @@ -223,13 +224,18 @@ __printf(2, 3) void libbpf_print(enum libbpf_print_level level, const char *format, ...) { va_list args; + int old_errno; if (!__libbpf_pr) return; + old_errno = errno; + va_start(args, format); __libbpf_pr(level, format, args); va_end(args); + + errno = old_errno; } static void pr_perm_msg(int err) @@ -412,6 +418,7 @@ struct bpf_program { int fd; bool autoload; + bool autoattach; bool mark_btf_static; enum bpf_prog_type type; enum bpf_attach_type expected_attach_type; @@ -591,7 +598,6 @@ struct elf_state { size_t strtabidx; struct elf_sec_desc *secs; int sec_cnt; - int maps_shndx; int btf_maps_shndx; __u32 btf_maps_sec_btf_id; int text_shndx; @@ -751,6 +757,8 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, prog->autoload = true; } + prog->autoattach = true; + /* inherit object's log_level */ prog->log_level = obj->log_level; @@ -876,7 +884,7 @@ __u32 get_kernel_version(void) __u32 major, minor, patch; struct utsname info; - if (access(ubuntu_kver_file, R_OK) == 0) { + if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) { FILE *f; f = fopen(ubuntu_kver_file, "r"); @@ -1272,7 +1280,6 @@ static struct bpf_object *bpf_object__new(const char *path, */ obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; - obj->efile.maps_shndx = -1; obj->efile.btf_maps_shndx = -1; obj->efile.st_ops_shndx = -1; obj->kconfig_map_idx = -1; @@ -1642,6 +1649,10 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj) for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { sec_desc = &obj->efile.secs[sec_idx]; + /* Skip recognized sections with size 0. */ + if (!sec_desc->data || sec_desc->data->d_size == 0) + continue; + switch (sec_desc->sec_type) { case SEC_DATA: sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); @@ -2086,19 +2097,30 @@ static bool get_map_field_int(const char *map_name, const struct btf *btf, return true; } +static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) +{ + int len; + + len = snprintf(buf, buf_sz, "%s/%s", path, name); + if (len < 0) + return -EINVAL; + if (len >= buf_sz) + return -ENAMETOOLONG; + + return 0; +} + static int build_map_pin_path(struct bpf_map *map, const char *path) { char buf[PATH_MAX]; - int len; + int err; if (!path) path = "/sys/fs/bpf"; - len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); - if (len < 0) - return -EINVAL; - else if (len >= PATH_MAX) - return -ENAMETOOLONG; + err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); + if (err) + return err; return bpf_map__set_pin_path(map, buf); } @@ -2362,6 +2384,12 @@ static size_t adjust_ringbuf_sz(size_t sz) return sz; } +static bool map_is_ringbuf(const struct bpf_map *map) +{ + return map->def.type == BPF_MAP_TYPE_RINGBUF || + map->def.type == BPF_MAP_TYPE_USER_RINGBUF; +} + static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) { map->def.type = def->map_type; @@ -2376,7 +2404,7 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def map->btf_value_type_id = def->value_type_id; /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ - if (map->def.type == BPF_MAP_TYPE_RINGBUF) + if (map_is_ringbuf(map)) map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); if (def->parts & MAP_DEF_MAP_TYPE) @@ -3359,7 +3387,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (err) return err; } else if (strcmp(name, "maps") == 0) { - obj->efile.maps_shndx = idx; + pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); + return -ENOTSUP; } else if (strcmp(name, MAPS_ELF_SEC) == 0) { obj->efile.btf_maps_shndx = idx; } else if (strcmp(name, BTF_ELF_SEC) == 0) { @@ -3891,8 +3920,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, int shndx) { - return shndx == obj->efile.maps_shndx || - shndx == obj->efile.btf_maps_shndx; + return shndx == obj->efile.btf_maps_shndx; } static enum libbpf_map_type @@ -4277,11 +4305,12 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) int bpf_map__reuse_fd(struct bpf_map *map, int fd) { - struct bpf_map_info info = {}; + struct bpf_map_info info; __u32 len = sizeof(info), name_len; int new_fd, err; char *new_name; + memset(&info, 0, len); err = bpf_obj_get_info_by_fd(fd, &info, &len); if (err && errno == EINVAL) err = bpf_get_map_info_from_fdinfo(fd, &info); @@ -4358,7 +4387,7 @@ int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) map->def.max_entries = max_entries; /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ - if (map->def.type == BPF_MAP_TYPE_RINGBUF) + if (map_is_ringbuf(map)) map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); return 0; @@ -4408,14 +4437,23 @@ static int probe_fd(int fd) static int probe_kern_prog_name(void) { + const size_t attr_sz = offsetofend(union bpf_attr, prog_name); struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - int ret, insn_cnt = ARRAY_SIZE(insns); + union bpf_attr attr; + int ret; + + memset(&attr, 0, attr_sz); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.license = ptr_to_u64("GPL"); + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = (__u32)ARRAY_SIZE(insns); + libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); /* make sure loading with name works */ - ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "test", "GPL", insns, insn_cnt, NULL); + ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); return probe_fd(ret); } @@ -4430,7 +4468,7 @@ static int probe_kern_global_data(void) }; int ret, map, insn_cnt = ARRAY_SIZE(insns); - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), 32, 1, NULL); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL); if (map < 0) { ret = -errno; cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); @@ -4563,7 +4601,7 @@ static int probe_kern_array_mmap(void) LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); int fd; - fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), sizeof(int), 1, &opts); + fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); return probe_fd(fd); } @@ -4610,7 +4648,7 @@ static int probe_prog_bind_map(void) }; int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); - map = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), 32, 1, NULL); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL); if (map < 0) { ret = -errno; cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); @@ -4814,13 +4852,12 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) { - struct bpf_map_info map_info = {}; + struct bpf_map_info map_info; char msg[STRERR_BUFSIZE]; - __u32 map_info_len; + __u32 map_info_len = sizeof(map_info); int err; - map_info_len = sizeof(map_info); - + memset(&map_info, 0, map_info_len); err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len); if (err && errno == EINVAL) err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); @@ -7244,8 +7281,6 @@ static int bpf_object_unload(struct bpf_object *obj) return 0; } -int bpf_object__unload(struct bpf_object *obj) __attribute__((alias("bpf_object_unload"))); - static int bpf_object__sanitize_maps(struct bpf_object *obj) { struct bpf_map *m; @@ -7944,17 +7979,9 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path) continue; if (path) { - int len; - - len = snprintf(buf, PATH_MAX, "%s/%s", path, - bpf_map__name(map)); - if (len < 0) { - err = -EINVAL; - goto err_unpin_maps; - } else if (len >= PATH_MAX) { - err = -ENAMETOOLONG; + err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); + if (err) goto err_unpin_maps; - } sanitize_pin_path(buf); pin_path = buf; } else if (!map->pin_path) { @@ -7992,14 +8019,9 @@ int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) char buf[PATH_MAX]; if (path) { - int len; - - len = snprintf(buf, PATH_MAX, "%s/%s", path, - bpf_map__name(map)); - if (len < 0) - return libbpf_err(-EINVAL); - else if (len >= PATH_MAX) - return libbpf_err(-ENAMETOOLONG); + err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); + if (err) + return libbpf_err(err); sanitize_pin_path(buf); pin_path = buf; } else if (!map->pin_path) { @@ -8017,6 +8039,7 @@ int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) int bpf_object__pin_programs(struct bpf_object *obj, const char *path) { struct bpf_program *prog; + char buf[PATH_MAX]; int err; if (!obj) @@ -8028,17 +8051,9 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) } bpf_object__for_each_program(prog, obj) { - char buf[PATH_MAX]; - int len; - - len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); - if (len < 0) { - err = -EINVAL; - goto err_unpin_programs; - } else if (len >= PATH_MAX) { - err = -ENAMETOOLONG; + err = pathname_concat(buf, sizeof(buf), path, prog->name); + if (err) goto err_unpin_programs; - } err = bpf_program__pin(prog, buf); if (err) @@ -8049,13 +8064,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) err_unpin_programs: while ((prog = bpf_object__prev_program(obj, prog))) { - char buf[PATH_MAX]; - int len; - - len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); - if (len < 0) - continue; - else if (len >= PATH_MAX) + if (pathname_concat(buf, sizeof(buf), path, prog->name)) continue; bpf_program__unpin(prog, buf); @@ -8074,13 +8083,10 @@ int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) bpf_object__for_each_program(prog, obj) { char buf[PATH_MAX]; - int len; - len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); - if (len < 0) - return libbpf_err(-EINVAL); - else if (len >= PATH_MAX) - return libbpf_err(-ENAMETOOLONG); + err = pathname_concat(buf, sizeof(buf), path, prog->name); + if (err) + return libbpf_err(err); err = bpf_program__unpin(prog, buf); if (err) @@ -8298,6 +8304,16 @@ int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) return 0; } +bool bpf_program__autoattach(const struct bpf_program *prog) +{ + return prog->autoattach; +} + +void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) +{ + prog->autoattach = autoattach; +} + const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) { return prog->insns; @@ -8978,11 +8994,12 @@ int libbpf_find_vmlinux_btf_id(const char *name, static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) { - struct bpf_prog_info info = {}; + struct bpf_prog_info info; __u32 info_len = sizeof(info); struct btf *btf; int err; + memset(&info, 0, info_len); err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len); if (err) { pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n", @@ -9056,11 +9073,15 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac int err = 0; /* BPF program's BTF ID */ - if (attach_prog_fd) { + if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { + if (!attach_prog_fd) { + pr_warn("prog '%s': attach program FD is not set\n", prog->name); + return -EINVAL; + } err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); if (err < 0) { - pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n", - attach_prog_fd, attach_name, err); + pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n", + prog->name, attach_prog_fd, attach_name, err); return err; } *btf_obj_fd = 0; @@ -9077,7 +9098,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); } if (err) { - pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err); + pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n", + prog->name, attach_name, err); return err; } return 0; @@ -9810,13 +9832,16 @@ static int determine_uprobe_retprobe_bit(void) static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, uint64_t offset, int pid, size_t ref_ctr_off) { - struct perf_event_attr attr = {}; + const size_t attr_sz = sizeof(struct perf_event_attr); + struct perf_event_attr attr; char errmsg[STRERR_BUFSIZE]; int type, pfd; if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) return -EINVAL; + memset(&attr, 0, attr_sz); + type = uprobe ? determine_uprobe_perf_type() : determine_kprobe_perf_type(); if (type < 0) { @@ -9837,7 +9862,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, } attr.config |= 1 << bit; } - attr.size = sizeof(attr); + attr.size = attr_sz; attr.type = type; attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ @@ -9879,7 +9904,7 @@ static bool use_debugfs(void) static int has_debugfs = -1; if (has_debugfs < 0) - has_debugfs = access(DEBUGFS, F_OK) == 0; + has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; return has_debugfs == 1; } @@ -9936,7 +9961,8 @@ static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retpro static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, const char *kfunc_name, size_t offset, int pid) { - struct perf_event_attr attr = {}; + const size_t attr_sz = sizeof(struct perf_event_attr); + struct perf_event_attr attr; char errmsg[STRERR_BUFSIZE]; int type, pfd, err; @@ -9955,7 +9981,9 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); goto err_clean_legacy; } - attr.size = sizeof(attr); + + memset(&attr, 0, attr_sz); + attr.size = attr_sz; attr.config = type; attr.type = PERF_TYPE_TRACEPOINT; @@ -10412,6 +10440,7 @@ static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retpro static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, const char *binary_path, size_t offset, int pid) { + const size_t attr_sz = sizeof(struct perf_event_attr); struct perf_event_attr attr; int type, pfd, err; @@ -10429,8 +10458,8 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, goto err_clean_legacy; } - memset(&attr, 0, sizeof(attr)); - attr.size = sizeof(attr); + memset(&attr, 0, attr_sz); + attr.size = attr_sz; attr.config = type; attr.type = PERF_TYPE_TRACEPOINT; @@ -10662,15 +10691,17 @@ static const char *arch_specific_lib_paths(void) static int resolve_full_path(const char *file, char *result, size_t result_sz) { const char *search_paths[3] = {}; - int i; + int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + perm = R_OK; } else { search_paths[0] = getenv("PATH"); search_paths[1] = "/usr/bin:/usr/sbin"; + perm = R_OK | X_OK; } for (i = 0; i < ARRAY_SIZE(search_paths); i++) { @@ -10689,8 +10720,8 @@ static int resolve_full_path(const char *file, char *result, size_t result_sz) if (!seg_len) continue; snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); - /* ensure it is an executable file/link */ - if (access(result, R_OK | X_OK) < 0) + /* ensure it has required permissions */ + if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) continue; pr_debug("resolved '%s' to '%s'\n", file, result); return 0; @@ -10967,7 +10998,8 @@ static int determine_tracepoint_id(const char *tp_category, static int perf_event_open_tracepoint(const char *tp_category, const char *tp_name) { - struct perf_event_attr attr = {}; + const size_t attr_sz = sizeof(struct perf_event_attr); + struct perf_event_attr attr; char errmsg[STRERR_BUFSIZE]; int tp_id, pfd, err; @@ -10979,8 +11011,9 @@ static int perf_event_open_tracepoint(const char *tp_category, return tp_id; } + memset(&attr, 0, attr_sz); attr.type = PERF_TYPE_TRACEPOINT; - attr.size = sizeof(attr); + attr.size = attr_sz; attr.config = tp_id; pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, @@ -11600,12 +11633,15 @@ struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, void *ctx, const struct perf_buffer_opts *opts) { + const size_t attr_sz = sizeof(struct perf_event_attr); struct perf_buffer_params p = {}; - struct perf_event_attr attr = {}; + struct perf_event_attr attr; if (!OPTS_VALID(opts, perf_buffer_opts)) return libbpf_err_ptr(-EINVAL); + memset(&attr, 0, attr_sz); + attr.size = attr_sz; attr.config = PERF_COUNT_SW_BPF_OUTPUT; attr.type = PERF_TYPE_SOFTWARE; attr.sample_type = PERF_SAMPLE_RAW; @@ -12328,7 +12364,7 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) struct bpf_program *prog = *s->progs[i].prog; struct bpf_link **link = s->progs[i].link; - if (!prog->autoload) + if (!prog->autoload || !prog->autoattach) continue; /* auto-attaching not supported for this program */ diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 61493c4cddac11c2b216b76a94414ae48c9f9c89..eee883f007f9e24cb4c742422b3697b3fae8183c 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -118,7 +118,9 @@ struct bpf_object_open_opts { * auto-pinned to that path on load; defaults to "/sys/fs/bpf". */ const char *pin_root_path; - long :0; + + __u32 :32; /* stub out now removed attach_prog_fd */ + /* Additional kernel config content that augments and overrides * system Kconfig for CONFIG_xxx externs. */ @@ -260,6 +262,8 @@ LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog); LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog); LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload); +LIBBPF_API bool bpf_program__autoattach(const struct bpf_program *prog); +LIBBPF_API void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach); struct bpf_insn; @@ -1009,6 +1013,7 @@ LIBBPF_API int bpf_tc_query(const struct bpf_tc_hook *hook, /* Ring buffer APIs */ struct ring_buffer; +struct user_ring_buffer; typedef int (*ring_buffer_sample_fn)(void *ctx, void *data, size_t size); @@ -1028,6 +1033,112 @@ LIBBPF_API int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms); LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb); LIBBPF_API int ring_buffer__epoll_fd(const struct ring_buffer *rb); +struct user_ring_buffer_opts { + size_t sz; /* size of this struct, for forward/backward compatibility */ +}; + +#define user_ring_buffer_opts__last_field sz + +/* @brief **user_ring_buffer__new()** creates a new instance of a user ring + * buffer. + * + * @param map_fd A file descriptor to a BPF_MAP_TYPE_USER_RINGBUF map. + * @param opts Options for how the ring buffer should be created. + * @return A user ring buffer on success; NULL and errno being set on a + * failure. + */ +LIBBPF_API struct user_ring_buffer * +user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts); + +/* @brief **user_ring_buffer__reserve()** reserves a pointer to a sample in the + * user ring buffer. + * @param rb A pointer to a user ring buffer. + * @param size The size of the sample, in bytes. + * @return A pointer to an 8-byte aligned reserved region of the user ring + * buffer; NULL, and errno being set if a sample could not be reserved. + * + * This function is *not* thread safe, and callers must synchronize accessing + * this function if there are multiple producers. If a size is requested that + * is larger than the size of the entire ring buffer, errno will be set to + * E2BIG and NULL is returned. If the ring buffer could accommodate the size, + * but currently does not have enough space, errno is set to ENOSPC and NULL is + * returned. + * + * After initializing the sample, callers must invoke + * **user_ring_buffer__submit()** to post the sample to the kernel. Otherwise, + * the sample must be freed with **user_ring_buffer__discard()**. + */ +LIBBPF_API void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size); + +/* @brief **user_ring_buffer__reserve_blocking()** reserves a record in the + * ring buffer, possibly blocking for up to @timeout_ms until a sample becomes + * available. + * @param rb The user ring buffer. + * @param size The size of the sample, in bytes. + * @param timeout_ms The amount of time, in milliseconds, for which the caller + * should block when waiting for a sample. -1 causes the caller to block + * indefinitely. + * @return A pointer to an 8-byte aligned reserved region of the user ring + * buffer; NULL, and errno being set if a sample could not be reserved. + * + * This function is *not* thread safe, and callers must synchronize + * accessing this function if there are multiple producers + * + * If **timeout_ms** is -1, the function will block indefinitely until a sample + * becomes available. Otherwise, **timeout_ms** must be non-negative, or errno + * is set to EINVAL, and NULL is returned. If **timeout_ms** is 0, no blocking + * will occur and the function will return immediately after attempting to + * reserve a sample. + * + * If **size** is larger than the size of the entire ring buffer, errno is set + * to E2BIG and NULL is returned. If the ring buffer could accommodate + * **size**, but currently does not have enough space, the caller will block + * until at most **timeout_ms** has elapsed. If insufficient space is available + * at that time, errno is set to ENOSPC, and NULL is returned. + * + * The kernel guarantees that it will wake up this thread to check if + * sufficient space is available in the ring buffer at least once per + * invocation of the **bpf_ringbuf_drain()** helper function, provided that at + * least one sample is consumed, and the BPF program did not invoke the + * function with BPF_RB_NO_WAKEUP. A wakeup may occur sooner than that, but the + * kernel does not guarantee this. If the helper function is invoked with + * BPF_RB_FORCE_WAKEUP, a wakeup event will be sent even if no sample is + * consumed. + * + * When a sample of size **size** is found within **timeout_ms**, a pointer to + * the sample is returned. After initializing the sample, callers must invoke + * **user_ring_buffer__submit()** to post the sample to the ring buffer. + * Otherwise, the sample must be freed with **user_ring_buffer__discard()**. + */ +LIBBPF_API void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, + __u32 size, + int timeout_ms); + +/* @brief **user_ring_buffer__submit()** submits a previously reserved sample + * into the ring buffer. + * @param rb The user ring buffer. + * @param sample A reserved sample. + * + * It is not necessary to synchronize amongst multiple producers when invoking + * this function. + */ +LIBBPF_API void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample); + +/* @brief **user_ring_buffer__discard()** discards a previously reserved sample. + * @param rb The user ring buffer. + * @param sample A reserved sample. + * + * It is not necessary to synchronize amongst multiple producers when invoking + * this function. + */ +LIBBPF_API void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample); + +/* @brief **user_ring_buffer__free()** frees a ring buffer that was previously + * created with **user_ring_buffer__new()**. + * @param rb The user ring buffer being freed. + */ +LIBBPF_API void user_ring_buffer__free(struct user_ring_buffer *rb); + /* Perf buffer APIs */ struct perf_buffer; diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 119e6e1ea7f114d4ac2f19b068c5e94cc6046da8..c1d6aa7c82b6d6ba449667e2466d486a8c08b84a 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -358,6 +358,8 @@ LIBBPF_1.0.0 { bpf_obj_get_opts; bpf_prog_query_opts; bpf_program__attach_ksyscall; + bpf_program__autoattach; + bpf_program__set_autoattach; btf__add_enum64; btf__add_enum64_value; libbpf_bpf_attach_type_str; @@ -366,3 +368,13 @@ LIBBPF_1.0.0 { libbpf_bpf_prog_type_str; perf_buffer__buffer; }; + +LIBBPF_1.1.0 { + global: + user_ring_buffer__discard; + user_ring_buffer__free; + user_ring_buffer__new; + user_ring_buffer__reserve; + user_ring_buffer__reserve_blocking; + user_ring_buffer__submit; +} LIBBPF_1.0.0; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4135ae0a2bc3c6feb7276e1773b04303caeec8a3..377642ff51fce597329e771eb73b9785844f3d17 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -573,4 +573,7 @@ static inline bool is_pow_of_2(size_t x) return x && (x & (x - 1)) == 0; } +#define PROG_LOAD_ATTEMPTS 5 +int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts); + #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ diff --git a/tools/lib/bpf/libbpf_legacy.h b/tools/lib/bpf/libbpf_legacy.h index 5b7e0155db6a5a9b8358100a98172632935fb333..1e1be467bede6db7821e2d41f0cef8eab56b03b8 100644 --- a/tools/lib/bpf/libbpf_legacy.h +++ b/tools/lib/bpf/libbpf_legacy.h @@ -125,6 +125,8 @@ struct bpf_map; struct btf; struct btf_ext; +LIBBPF_API struct btf *libbpf_find_kernel_btf(void); + LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); LIBBPF_API enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map); diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 0b5398786bf3d38118932b0776976cbe2dee558f..f3a8e8e74eb89dc2a06aa30daee37b341558836a 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -193,7 +193,7 @@ static int probe_map_create(enum bpf_map_type map_type) LIBBPF_OPTS(bpf_map_create_opts, opts); int key_size, value_size, max_entries; __u32 btf_key_type_id = 0, btf_value_type_id = 0; - int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err; + int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err = 0; key_size = sizeof(__u32); value_size = sizeof(__u32); @@ -231,6 +231,7 @@ static int probe_map_create(enum bpf_map_type map_type) return btf_fd; break; case BPF_MAP_TYPE_RINGBUF: + case BPF_MAP_TYPE_USER_RINGBUF: key_size = 0; value_size = 0; max_entries = 4096; diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h index 2fb2f4290080d279005005c65c4baa868b1b1be5..e944f5bce7286120aff0b776e496aaca5fc139ec 100644 --- a/tools/lib/bpf/libbpf_version.h +++ b/tools/lib/bpf/libbpf_version.h @@ -4,6 +4,6 @@ #define __LIBBPF_VERSION_H #define LIBBPF_MAJOR_VERSION 1 -#define LIBBPF_MINOR_VERSION 0 +#define LIBBPF_MINOR_VERSION 1 #endif /* __LIBBPF_VERSION_H */ diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index 6c013168032df853a9d2ffe9aadd7450c5a2ffe8..35104580870c0f0ae90e6e4e5a9602e8d67f7f84 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -587,11 +587,12 @@ static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd) { - struct bpf_prog_info info = {}; + struct bpf_prog_info info; __u32 info_len = sizeof(info); char name[256]; int len, ret; + memset(&info, 0, info_len); ret = bpf_obj_get_info_by_fd(fd, &info, &info_len); if (ret < 0) return ret; diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c index f57e77a6e40fdfb9d0b972f651e75617e6d5abe4..3900d052ed19ea651c314fb9205e89f495fa71c1 100644 --- a/tools/lib/bpf/nlattr.c +++ b/tools/lib/bpf/nlattr.c @@ -32,7 +32,7 @@ static struct nlattr *nla_next(const struct nlattr *nla, int *remaining) static int nla_ok(const struct nlattr *nla, int remaining) { - return remaining >= sizeof(*nla) && + return remaining >= (int)sizeof(*nla) && nla->nla_len >= sizeof(*nla) && nla->nla_len <= remaining; } diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c index 8bc117bcc7bcd2eb1839d34f8648aeae9845a49b..d285171d4b69ac68b006f2e051e55f6b4ca08062 100644 --- a/tools/lib/bpf/ringbuf.c +++ b/tools/lib/bpf/ringbuf.c @@ -16,6 +16,7 @@ #include <asm/barrier.h> #include <sys/mman.h> #include <sys/epoll.h> +#include <time.h> #include "libbpf.h" #include "libbpf_internal.h" @@ -39,6 +40,23 @@ struct ring_buffer { int ring_cnt; }; +struct user_ring_buffer { + struct epoll_event event; + unsigned long *consumer_pos; + unsigned long *producer_pos; + void *data; + unsigned long mask; + size_t page_size; + int map_fd; + int epoll_fd; +}; + +/* 8-byte ring buffer header structure */ +struct ringbuf_hdr { + __u32 len; + __u32 pad; +}; + static void ringbuf_unmap_ring(struct ring_buffer *rb, struct ring *r) { if (r->consumer_pos) { @@ -300,3 +318,256 @@ int ring_buffer__epoll_fd(const struct ring_buffer *rb) { return rb->epoll_fd; } + +static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb) +{ + if (rb->consumer_pos) { + munmap(rb->consumer_pos, rb->page_size); + rb->consumer_pos = NULL; + } + if (rb->producer_pos) { + munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1)); + rb->producer_pos = NULL; + } +} + +void user_ring_buffer__free(struct user_ring_buffer *rb) +{ + if (!rb) + return; + + user_ringbuf_unmap_ring(rb); + + if (rb->epoll_fd >= 0) + close(rb->epoll_fd); + + free(rb); +} + +static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) +{ + struct bpf_map_info info; + __u32 len = sizeof(info); + void *tmp; + struct epoll_event *rb_epoll; + int err; + + memset(&info, 0, sizeof(info)); + + err = bpf_obj_get_info_by_fd(map_fd, &info, &len); + if (err) { + err = -errno; + pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err); + return err; + } + + if (info.type != BPF_MAP_TYPE_USER_RINGBUF) { + pr_warn("user ringbuf: map fd=%d is not BPF_MAP_TYPE_USER_RINGBUF\n", map_fd); + return -EINVAL; + } + + rb->map_fd = map_fd; + rb->mask = info.max_entries - 1; + + /* Map read-only consumer page */ + tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0); + if (tmp == MAP_FAILED) { + err = -errno; + pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n", + map_fd, err); + return err; + } + rb->consumer_pos = tmp; + + /* Map read-write the producer page and data pages. We map the data + * region as twice the total size of the ring buffer to allow the + * simple reading and writing of samples that wrap around the end of + * the buffer. See the kernel implementation for details. + */ + tmp = mmap(NULL, rb->page_size + 2 * info.max_entries, + PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, rb->page_size); + if (tmp == MAP_FAILED) { + err = -errno; + pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n", + map_fd, err); + return err; + } + + rb->producer_pos = tmp; + rb->data = tmp + rb->page_size; + + rb_epoll = &rb->event; + rb_epoll->events = EPOLLOUT; + if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) { + err = -errno; + pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err); + return err; + } + + return 0; +} + +struct user_ring_buffer * +user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts) +{ + struct user_ring_buffer *rb; + int err; + + if (!OPTS_VALID(opts, user_ring_buffer_opts)) + return errno = EINVAL, NULL; + + rb = calloc(1, sizeof(*rb)); + if (!rb) + return errno = ENOMEM, NULL; + + rb->page_size = getpagesize(); + + rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if (rb->epoll_fd < 0) { + err = -errno; + pr_warn("user ringbuf: failed to create epoll instance: %d\n", err); + goto err_out; + } + + err = user_ringbuf_map(rb, map_fd); + if (err) + goto err_out; + + return rb; + +err_out: + user_ring_buffer__free(rb); + return errno = -err, NULL; +} + +static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard) +{ + __u32 new_len; + struct ringbuf_hdr *hdr; + uintptr_t hdr_offset; + + hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ; + hdr = rb->data + (hdr_offset & rb->mask); + + new_len = hdr->len & ~BPF_RINGBUF_BUSY_BIT; + if (discard) + new_len |= BPF_RINGBUF_DISCARD_BIT; + + /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in + * the kernel. + */ + __atomic_exchange_n(&hdr->len, new_len, __ATOMIC_ACQ_REL); +} + +void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample) +{ + user_ringbuf_commit(rb, sample, true); +} + +void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample) +{ + user_ringbuf_commit(rb, sample, false); +} + +void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size) +{ + __u32 avail_size, total_size, max_size; + /* 64-bit to avoid overflow in case of extreme application behavior */ + __u64 cons_pos, prod_pos; + struct ringbuf_hdr *hdr; + + /* Synchronizes with smp_store_release() in __bpf_user_ringbuf_peek() in + * the kernel. + */ + cons_pos = smp_load_acquire(rb->consumer_pos); + /* Synchronizes with smp_store_release() in user_ringbuf_commit() */ + prod_pos = smp_load_acquire(rb->producer_pos); + + max_size = rb->mask + 1; + avail_size = max_size - (prod_pos - cons_pos); + /* Round up total size to a multiple of 8. */ + total_size = (size + BPF_RINGBUF_HDR_SZ + 7) / 8 * 8; + + if (total_size > max_size) + return errno = E2BIG, NULL; + + if (avail_size < total_size) + return errno = ENOSPC, NULL; + + hdr = rb->data + (prod_pos & rb->mask); + hdr->len = size | BPF_RINGBUF_BUSY_BIT; + hdr->pad = 0; + + /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in + * the kernel. + */ + smp_store_release(rb->producer_pos, prod_pos + total_size); + + return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask); +} + +static __u64 ns_elapsed_timespec(const struct timespec *start, const struct timespec *end) +{ + __u64 start_ns, end_ns, ns_per_s = 1000000000; + + start_ns = (__u64)start->tv_sec * ns_per_s + start->tv_nsec; + end_ns = (__u64)end->tv_sec * ns_per_s + end->tv_nsec; + + return end_ns - start_ns; +} + +void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms) +{ + void *sample; + int err, ms_remaining = timeout_ms; + struct timespec start; + + if (timeout_ms < 0 && timeout_ms != -1) + return errno = EINVAL, NULL; + + if (timeout_ms != -1) { + err = clock_gettime(CLOCK_MONOTONIC, &start); + if (err) + return NULL; + } + + do { + int cnt, ms_elapsed; + struct timespec curr; + __u64 ns_per_ms = 1000000; + + sample = user_ring_buffer__reserve(rb, size); + if (sample) + return sample; + else if (errno != ENOSPC) + return NULL; + + /* The kernel guarantees at least one event notification + * delivery whenever at least one sample is drained from the + * ring buffer in an invocation to bpf_ringbuf_drain(). Other + * additional events may be delivered at any time, but only one + * event is guaranteed per bpf_ringbuf_drain() invocation, + * provided that a sample is drained, and the BPF program did + * not pass BPF_RB_NO_WAKEUP to bpf_ringbuf_drain(). If + * BPF_RB_FORCE_WAKEUP is passed to bpf_ringbuf_drain(), a + * wakeup event will be delivered even if no samples are + * drained. + */ + cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining); + if (cnt < 0) + return NULL; + + if (timeout_ms == -1) + continue; + + err = clock_gettime(CLOCK_MONOTONIC, &curr); + if (err) + return NULL; + + ms_elapsed = ns_elapsed_timespec(&start, &curr) / ns_per_ms; + ms_remaining = timeout_ms - ms_elapsed; + } while (ms_remaining > 0); + + /* Try one more time to reserve a sample after the specified timeout has elapsed. */ + return user_ring_buffer__reserve(rb, size); +} diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index 70adf7b119b99e995907df417924fd08d3e4cc62..1e82ab06c3ebf4d32eb0e257a1f647ee4c6d0cd3 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -251,6 +251,29 @@ static inline int skel_map_update_elem(int fd, const void *key, return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz); } +static inline int skel_map_delete_elem(int fd, const void *key) +{ + const size_t attr_sz = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + attr.map_fd = fd; + attr.key = (long)key; + + return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz); +} + +static inline int skel_map_get_fd_by_id(__u32 id) +{ + const size_t attr_sz = offsetofend(union bpf_attr, flags); + union bpf_attr attr; + + memset(&attr, 0, attr_sz); + attr.map_id = id; + + return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz); +} + static inline int skel_raw_tracepoint_open(const char *name, int prog_fd) { const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd); @@ -285,6 +308,8 @@ static inline int skel_link_create(int prog_fd, int target_fd, static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) { + const size_t prog_load_attr_sz = offsetofend(union bpf_attr, fd_array); + const size_t test_run_attr_sz = offsetofend(union bpf_attr, test); int map_fd = -1, prog_fd = -1, key = 0, err; union bpf_attr attr; @@ -302,7 +327,7 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) goto out; } - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, prog_load_attr_sz); attr.prog_type = BPF_PROG_TYPE_SYSCALL; attr.insns = (long) opts->insns; attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); @@ -313,18 +338,18 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) attr.log_size = opts->ctx->log_size; attr.log_buf = opts->ctx->log_buf; attr.prog_flags = BPF_F_SLEEPABLE; - err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); + err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz); if (prog_fd < 0) { opts->errstr = "failed to load loader prog"; set_err; goto out; } - memset(&attr, 0, sizeof(attr)); + memset(&attr, 0, test_run_attr_sz); attr.test.prog_fd = prog_fd; attr.test.ctx_in = (long) opts->ctx; attr.test.ctx_size_in = opts->ctx->sz; - err = skel_sys_bpf(BPF_PROG_RUN, &attr, sizeof(attr)); + err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz); if (err < 0 || (int)attr.test.retval < 0) { opts->errstr = "failed to execute loader prog"; if (err < 0) { diff --git a/tools/lib/bpf/usdt.bpf.h b/tools/lib/bpf/usdt.bpf.h index 4f2adc0bd6ca379b6b3cf79bcd9de19671472339..fdfd235e52c426fbbbe748e033680eb1f8064465 100644 --- a/tools/lib/bpf/usdt.bpf.h +++ b/tools/lib/bpf/usdt.bpf.h @@ -232,7 +232,7 @@ long bpf_usdt_cookie(struct pt_regs *ctx) */ #define BPF_USDT(name, args...) \ name(struct pt_regs *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -241,7 +241,7 @@ typeof(name(0)) name(struct pt_regs *ctx) \ return ____##name(___bpf_usdt_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) #endif /* __USDT_BPF_H__ */ diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index d18e37982344c7c85b4a31d896f580d14927c892..e83b497c2245443473539a5a25c4ca98f13257fc 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -282,7 +282,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) * If this is not supported, USDTs with semaphores will not be supported. * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") */ - man->has_sema_refcnt = access(ref_ctr_sysfs_path, F_OK) == 0; + man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0; return man; } diff --git a/tools/lib/find_bit.c b/tools/lib/find_bit.c index ba4b8d94e004823219594a09347be0d416823db0..6a3dc167d30ed16210a7b8dc0396c911393d04e4 100644 --- a/tools/lib/find_bit.c +++ b/tools/lib/find_bit.c @@ -18,66 +18,54 @@ #include <linux/bitmap.h> #include <linux/kernel.h> -#if !defined(find_next_bit) || !defined(find_next_zero_bit) || \ - !defined(find_next_and_bit) - /* - * This is a common helper function for find_next_bit, find_next_zero_bit, and - * find_next_and_bit. The differences are: - * - The "invert" argument, which is XORed with each fetched word before - * searching it for one bits. - * - The optional "addr2", which is anded with "addr1" if present. + * Common helper for find_bit() function family + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) + * @size: The bitmap size in bits */ -unsigned long _find_next_bit(const unsigned long *addr1, - const unsigned long *addr2, unsigned long nbits, - unsigned long start, unsigned long invert, unsigned long le) -{ - unsigned long tmp, mask; - (void) le; - - if (unlikely(start >= nbits)) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - - /* Handle 1st word. */ - mask = BITMAP_FIRST_WORD_MASK(start); - - /* - * Due to the lack of swab() in tools, and the fact that it doesn't - * need little-endian support, just comment it out - */ -#if (0) - if (le) - mask = swab(mask); -#endif - - tmp &= mask; +#define FIND_FIRST_BIT(FETCH, MUNGE, size) \ +({ \ + unsigned long idx, val, sz = (size); \ + \ + for (idx = 0; idx * BITS_PER_LONG < sz; idx++) { \ + val = (FETCH); \ + if (val) { \ + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(val)), sz); \ + break; \ + } \ + } \ + \ + sz; \ +}) - start = round_down(start, BITS_PER_LONG); - - while (!tmp) { - start += BITS_PER_LONG; - if (start >= nbits) - return nbits; - - tmp = addr1[start / BITS_PER_LONG]; - if (addr2) - tmp &= addr2[start / BITS_PER_LONG]; - tmp ^= invert; - } - -#if (0) - if (le) - tmp = swab(tmp); -#endif - - return min(start + __ffs(tmp), nbits); -} -#endif +/* + * Common helper for find_next_bit() function family + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) + * @size: The bitmap size in bits + * @start: The bitnumber to start searching at + */ +#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ +({ \ + unsigned long mask, idx, tmp, sz = (size), __start = (start); \ + \ + if (unlikely(__start >= sz)) \ + goto out; \ + \ + mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ + idx = __start / BITS_PER_LONG; \ + \ + for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ + if ((idx + 1) * BITS_PER_LONG >= sz) \ + goto out; \ + idx++; \ + } \ + \ + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ +out: \ + sz; \ +}) #ifndef find_first_bit /* @@ -85,14 +73,7 @@ unsigned long _find_next_bit(const unsigned long *addr1, */ unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx]) - return min(idx * BITS_PER_LONG + __ffs(addr[idx]), size); - } - - return size; + return FIND_FIRST_BIT(addr[idx], /* nop */, size); } #endif @@ -104,15 +85,7 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size) { - unsigned long idx, val; - - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - val = addr1[idx] & addr2[idx]; - if (val) - return min(idx * BITS_PER_LONG + __ffs(val), size); - } - - return size; + return FIND_FIRST_BIT(addr1[idx] & addr2[idx], /* nop */, size); } #endif @@ -122,13 +95,29 @@ unsigned long _find_first_and_bit(const unsigned long *addr1, */ unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size) { - unsigned long idx; + return FIND_FIRST_BIT(~addr[idx], /* nop */, size); +} +#endif - for (idx = 0; idx * BITS_PER_LONG < size; idx++) { - if (addr[idx] != ~0UL) - return min(idx * BITS_PER_LONG + ffz(addr[idx]), size); - } +#ifndef find_next_bit +unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); +} +#endif - return size; +#ifndef find_next_and_bit +unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long *addr2, + unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr1[idx] & addr2[idx], /* nop */, nbits, start); +} +#endif + +#ifndef find_next_zero_bit +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start) +{ + return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); } #endif diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c index e6c98a6e3908e20f8c68946f677b7161540f0e47..61b637f29b82785b066f2e14a2f8159d186eac9a 100644 --- a/tools/lib/perf/evlist.c +++ b/tools/lib/perf/evlist.c @@ -40,11 +40,11 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, * We already have cpus for evsel (via PMU sysfs) so * keep it, if there's no target cpu list defined. */ - if (!evsel->own_cpus || - (!evsel->system_wide && evlist->has_user_cpus) || - (!evsel->system_wide && - !evsel->requires_cpu && - perf_cpu_map__empty(evlist->user_requested_cpus))) { + if (evsel->system_wide) { + perf_cpu_map__put(evsel->cpus); + evsel->cpus = perf_cpu_map__new(NULL); + } else if (!evsel->own_cpus || evlist->has_user_cpus || + (!evsel->requires_cpu && perf_cpu_map__empty(evlist->user_requested_cpus))) { perf_cpu_map__put(evsel->cpus); evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus); } else if (evsel->cpus != evsel->own_cpus) { @@ -52,7 +52,10 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, evsel->cpus = perf_cpu_map__get(evsel->own_cpus); } - if (!evsel->system_wide) { + if (evsel->system_wide) { + perf_thread_map__put(evsel->threads); + evsel->threads = perf_thread_map__new_dummy(); + } else { perf_thread_map__put(evsel->threads); evsel->threads = perf_thread_map__get(evlist->threads); } @@ -64,9 +67,7 @@ static void perf_evlist__propagate_maps(struct perf_evlist *evlist) { struct perf_evsel *evsel; - /* Recomputing all_cpus, so start with a blank slate. */ - perf_cpu_map__put(evlist->all_cpus); - evlist->all_cpus = NULL; + evlist->needs_map_propagation = true; perf_evlist__for_each_evsel(evlist, evsel) __perf_evlist__propagate_maps(evlist, evsel); @@ -78,7 +79,9 @@ void perf_evlist__add(struct perf_evlist *evlist, evsel->idx = evlist->nr_entries; list_add_tail(&evsel->node, &evlist->entries); evlist->nr_entries += 1; - __perf_evlist__propagate_maps(evlist, evsel); + + if (evlist->needs_map_propagation) + __perf_evlist__propagate_maps(evlist, evsel); } void perf_evlist__remove(struct perf_evlist *evlist, @@ -174,9 +177,6 @@ void perf_evlist__set_maps(struct perf_evlist *evlist, evlist->threads = perf_thread_map__get(threads); } - if (!evlist->all_cpus && cpus) - evlist->all_cpus = perf_cpu_map__get(cpus); - perf_evlist__propagate_maps(evlist); } @@ -441,6 +441,7 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, perf_evlist__for_each_entry(evlist, evsel) { bool overwrite = evsel->attr.write_backward; + enum fdarray_flags flgs; struct perf_mmap *map; int *output, fd, cpu; @@ -486,6 +487,8 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, if (ops->idx) ops->idx(evlist, evsel, mp, idx); + /* Debug message used by test scripts */ + pr_debug("idx %d: mmapping fd %d\n", idx, *output); if (ops->mmap(map, mp, *output, evlist_cpu) < 0) return -1; @@ -494,6 +497,8 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, if (!idx) perf_evlist__set_mmap_first(evlist, map, overwrite); } else { + /* Debug message used by test scripts */ + pr_debug("idx %d: set output fd %d -> %d\n", idx, fd, *output); if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0) return -1; @@ -502,8 +507,8 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, revent = !overwrite ? POLLIN : 0; - if (!evsel->system_wide && - perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) { + flgs = evsel->system_wide ? fdarray_flag__nonfilterable : fdarray_flag__default; + if (perf_evlist__add_pollfd(evlist, fd, map, revent, flgs) < 0) { perf_mmap__put(map); return -1; } @@ -519,6 +524,48 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, return 0; } +static int +mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, + struct perf_mmap_param *mp) +{ + int nr_threads = perf_thread_map__nr(evlist->threads); + int nr_cpus = perf_cpu_map__nr(evlist->all_cpus); + int cpu, thread, idx = 0; + int nr_mmaps = 0; + + pr_debug("%s: nr cpu values (may include -1) %d nr threads %d\n", + __func__, nr_cpus, nr_threads); + + /* per-thread mmaps */ + for (thread = 0; thread < nr_threads; thread++, idx++) { + int output = -1; + int output_overwrite = -1; + + if (mmap_per_evsel(evlist, ops, idx, mp, 0, thread, &output, + &output_overwrite, &nr_mmaps)) + goto out_unmap; + } + + /* system-wide mmaps i.e. per-cpu */ + for (cpu = 1; cpu < nr_cpus; cpu++, idx++) { + int output = -1; + int output_overwrite = -1; + + if (mmap_per_evsel(evlist, ops, idx, mp, cpu, 0, &output, + &output_overwrite, &nr_mmaps)) + goto out_unmap; + } + + if (nr_mmaps != evlist->nr_mmaps) + pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps); + + return 0; + +out_unmap: + perf_evlist__munmap(evlist); + return -1; +} + static int mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, struct perf_mmap_param *mp) @@ -528,6 +575,8 @@ mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, int nr_mmaps = 0; int cpu, thread; + pr_debug("%s: nr cpu values %d nr threads %d\n", __func__, nr_cpus, nr_threads); + for (cpu = 0; cpu < nr_cpus; cpu++) { int output = -1; int output_overwrite = -1; @@ -569,6 +618,7 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, struct perf_mmap_param *mp) { + const struct perf_cpu_map *cpus = evlist->all_cpus; struct perf_evsel *evsel; if (!ops || !ops->get || !ops->mmap) @@ -588,6 +638,9 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist, if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0) return -ENOMEM; + if (perf_cpu_map__empty(cpus)) + return mmap_per_thread(evlist, ops, mp); + return mmap_per_cpu(evlist, ops, mp); } diff --git a/tools/lib/perf/evsel.c b/tools/lib/perf/evsel.c index 8ce5bbd096666cb9e8ba23d075be99c6719a30f6..8b51b008a81f142129069bc351c86e6aa2804ed8 100644 --- a/tools/lib/perf/evsel.c +++ b/tools/lib/perf/evsel.c @@ -515,9 +515,6 @@ int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads) if (ncpus == 0 || nthreads == 0) return 0; - if (evsel->system_wide) - nthreads = 1; - evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id)); if (evsel->sample_id == NULL) return -ENOMEM; diff --git a/tools/lib/perf/include/internal/evlist.h b/tools/lib/perf/include/internal/evlist.h index 6f89aec3e6084d4b1035cd74cbc06bd7a45e320f..850f07070036c787286590870b7cf328216533ee 100644 --- a/tools/lib/perf/include/internal/evlist.h +++ b/tools/lib/perf/include/internal/evlist.h @@ -19,6 +19,7 @@ struct perf_evlist { int nr_entries; int nr_groups; bool has_user_cpus; + bool needs_map_propagation; /** * The cpus passed from the command line or all online CPUs by * default. diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index 93bf93a59c99b600cb30d758b7f3acbd4d9955f6..ad47d7b31046c7c96e4663e0de8f4cfa21f5d7e4 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -6,7 +6,6 @@ #include <linux/types.h> #include <linux/limits.h> #include <linux/bpf.h> -#include <linux/compiler.h> #include <sys/types.h> /* pid_t */ #define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem)) @@ -97,7 +96,7 @@ struct perf_record_throttle { }; #ifndef KSYM_NAME_LEN -#define KSYM_NAME_LEN 256 +#define KSYM_NAME_LEN 512 #endif struct perf_record_ksymbol { @@ -153,6 +152,7 @@ struct perf_record_header_attr { enum { PERF_CPU_MAP__CPUS = 0, PERF_CPU_MAP__MASK = 1, + PERF_CPU_MAP__RANGE_CPUS = 2, }; /* @@ -195,7 +195,18 @@ struct perf_record_mask_cpu_map64 { #pragma GCC diagnostic ignored "-Wpacked" #pragma GCC diagnostic ignored "-Wattributes" -struct __packed perf_record_cpu_map_data { +/* + * An encoding of a CPU map for a range starting at start_cpu through to + * end_cpu. If any_cpu is 1, an any CPU (-1) value (aka dummy value) is present. + */ +struct perf_record_range_cpu_map { + __u8 any_cpu; + __u8 __pad; + __u16 start_cpu; + __u16 end_cpu; +}; + +struct perf_record_cpu_map_data { __u16 type; union { /* Used when type == PERF_CPU_MAP__CPUS. */ @@ -204,8 +215,10 @@ struct __packed perf_record_cpu_map_data { struct perf_record_mask_cpu_map32 mask32_data; /* Used when type == PERF_CPU_MAP__MASK and long_size == 8. */ struct perf_record_mask_cpu_map64 mask64_data; + /* Used when type == PERF_CPU_MAP__RANGE_CPUS. */ + struct perf_record_range_cpu_map range_cpu_data; }; -}; +} __attribute__((packed)); #pragma GCC diagnostic pop @@ -233,7 +246,16 @@ struct perf_record_event_update { struct perf_event_header header; __u64 type; __u64 id; - char data[]; + union { + /* Used when type == PERF_EVENT_UPDATE__SCALE. */ + struct perf_record_event_update_scale scale; + /* Used when type == PERF_EVENT_UPDATE__UNIT. */ + char unit[0]; + /* Used when type == PERF_EVENT_UPDATE__NAME. */ + char name[0]; + /* Used when type == PERF_EVENT_UPDATE__CPUS. */ + struct perf_record_event_update_cpus cpus; + }; }; #define MAX_EVENT_NAME 64 diff --git a/tools/lib/subcmd/exec-cmd.c b/tools/lib/subcmd/exec-cmd.c index 33e94fb8398677e41c95b0fc2c59a5e5326c52c8..5dbea456973e1e21fb59c09874e8fb1623d31719 100644 --- a/tools/lib/subcmd/exec-cmd.c +++ b/tools/lib/subcmd/exec-cmd.c @@ -24,6 +24,9 @@ void exec_cmd_init(const char *exec_name, const char *prefix, subcmd_config.prefix = prefix; subcmd_config.exec_path = exec_path; subcmd_config.exec_path_env = exec_path_env; + + /* Setup environment variable for invoked shell script. */ + setenv("PREFIX", prefix, 1); } #define is_dir_sep(c) ((c) == '/') diff --git a/tools/lib/symbol/kallsyms.h b/tools/lib/symbol/kallsyms.h index 72ab9870454baf15052fabb233ebcc04b27f94e3..542f9b059c3bd2502e8713fa995137e66b8503a5 100644 --- a/tools/lib/symbol/kallsyms.h +++ b/tools/lib/symbol/kallsyms.h @@ -7,7 +7,7 @@ #include <linux/types.h> #ifndef KSYM_NAME_LEN -#define KSYM_NAME_LEN 256 +#define KSYM_NAME_LEN 512 #endif static inline u8 kallsyms2elf_binding(char type) diff --git a/tools/memory-model/Documentation/litmus-tests.txt b/tools/memory-model/Documentation/litmus-tests.txt index 8a9d5d2787f9e9af30c82e873d8d4eabda28f597..26554b1c5575e086578d1af42318bc461cb3587c 100644 --- a/tools/memory-model/Documentation/litmus-tests.txt +++ b/tools/memory-model/Documentation/litmus-tests.txt @@ -946,22 +946,39 @@ Limitations of the Linux-kernel memory model (LKMM) include: carrying a dependency, then the compiler can break that dependency by substituting a constant of that value. - Conversely, LKMM sometimes doesn't recognize that a particular - optimization is not allowed, and as a result, thinks that a - dependency is not present (because the optimization would break it). - The memory model misses some pretty obvious control dependencies - because of this limitation. A simple example is: + Conversely, LKMM will sometimes overestimate the amount of + reordering compilers and CPUs can carry out, leading it to miss + some pretty obvious cases of ordering. A simple example is: r1 = READ_ONCE(x); if (r1 == 0) smp_mb(); WRITE_ONCE(y, 1); - There is a control dependency from the READ_ONCE to the WRITE_ONCE, - even when r1 is nonzero, but LKMM doesn't realize this and thinks - that the write may execute before the read if r1 != 0. (Yes, that - doesn't make sense if you think about it, but the memory model's - intelligence is limited.) + The WRITE_ONCE() does not depend on the READ_ONCE(), and as a + result, LKMM does not claim ordering. However, even though no + dependency is present, the WRITE_ONCE() will not be executed before + the READ_ONCE(). There are two reasons for this: + + The presence of the smp_mb() in one of the branches + prevents the compiler from moving the WRITE_ONCE() + up before the "if" statement, since the compiler has + to assume that r1 will sometimes be 0 (but see the + comment below); + + CPUs do not execute stores before po-earlier conditional + branches, even in cases where the store occurs after the + two arms of the branch have recombined. + + It is clear that it is not dangerous in the slightest for LKMM to + make weaker guarantees than architectures. In fact, it is + desirable, as it gives compilers room for making optimizations. + For instance, suppose that a 0 value in r1 would trigger undefined + behavior elsewhere. Then a clever compiler might deduce that r1 + can never be 0 in the if condition. As a result, said clever + compiler might deem it safe to optimize away the smp_mb(), + eliminating the branch and any ordering an architecture would + guarantee otherwise. 2. Multiple access sizes for a single variable are not supported, and neither are misaligned or partially overlapping accesses. diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index c260006106be7cfdb62e1c7e541231fdab6bc465..1c253b4b7ce00b0cc4824b63d57cea8095a2ba87 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -635,6 +635,12 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec *type = INSN_CONTEXT_SWITCH; break; + case 0xe0: /* loopne */ + case 0xe1: /* loope */ + case 0xe2: /* loop */ + *type = INSN_JUMP_CONDITIONAL; + break; + case 0xe8: *type = INSN_CALL; /* diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 91678252a9b6740b2463181bd8094d796f38bcb0..43ec14c29a60c1d63243bf2d0bfeee6552130bae 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -162,32 +162,34 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, /* * Unfortunately these have to be hard coded because the noreturn - * attribute isn't provided in ELF data. + * attribute isn't provided in ELF data. Keep 'em sorted. */ static const char * const global_noreturns[] = { + "__invalid_creds", + "__module_put_and_kthread_exit", + "__reiserfs_panic", "__stack_chk_fail", - "panic", + "__ubsan_handle_builtin_unreachable", + "cpu_bringup_and_idle", + "cpu_startup_entry", "do_exit", + "do_group_exit", "do_task_dead", - "kthread_exit", - "make_task_dead", - "__module_put_and_kthread_exit", + "ex_handler_msr_mce", + "fortify_panic", "kthread_complete_and_exit", - "__reiserfs_panic", + "kthread_exit", + "kunit_try_catch_throw", "lbug_with_loc", - "fortify_panic", - "usercopy_abort", "machine_real_restart", + "make_task_dead", + "panic", "rewind_stack_and_make_dead", - "kunit_try_catch_throw", - "xen_start_kernel", - "cpu_bringup_and_idle", - "do_group_exit", + "sev_es_terminate", + "snp_abort", "stop_this_cpu", - "__invalid_creds", - "cpu_startup_entry", - "__ubsan_handle_builtin_unreachable", - "ex_handler_msr_mce", + "usercopy_abort", + "xen_start_kernel", }; if (!func) @@ -1060,6 +1062,26 @@ static const char *uaccess_safe_builtin[] = { "__sanitizer_cov_trace_cmp4", "__sanitizer_cov_trace_cmp8", "__sanitizer_cov_trace_switch", + /* KMSAN */ + "kmsan_copy_to_user", + "kmsan_report", + "kmsan_unpoison_entry_regs", + "kmsan_unpoison_memory", + "__msan_chain_origin", + "__msan_get_context_state", + "__msan_instrument_asm_store", + "__msan_metadata_ptr_for_load_1", + "__msan_metadata_ptr_for_load_2", + "__msan_metadata_ptr_for_load_4", + "__msan_metadata_ptr_for_load_8", + "__msan_metadata_ptr_for_load_n", + "__msan_metadata_ptr_for_store_1", + "__msan_metadata_ptr_for_store_2", + "__msan_metadata_ptr_for_store_4", + "__msan_metadata_ptr_for_store_8", + "__msan_metadata_ptr_for_store_n", + "__msan_poison_alloca", + "__msan_warning", /* UBSAN */ "ubsan_type_mismatch_common", "__ubsan_handle_type_mismatch", @@ -1071,6 +1093,9 @@ static const char *uaccess_safe_builtin[] = { "copy_mc_fragile_handle_tail", "copy_mc_enhanced_fast_string", "ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */ + "clear_user_erms", + "clear_user_rep_good", + "clear_user_original", NULL }; @@ -2102,9 +2127,6 @@ static int read_noendbr_hints(struct objtool_file *file) return -1; } - if (insn->type == INSN_ENDBR) - WARN_FUNC("ANNOTATE_NOENDBR on ENDBR", insn->sec, insn->offset); - insn->noendbr = 1; } @@ -2233,7 +2255,7 @@ static int read_intra_function_calls(struct objtool_file *file) */ insn->type = INSN_JUMP_UNCONDITIONAL; - dest_off = insn->offset + insn->len + insn->immediate; + dest_off = arch_jump_destination(insn); insn->jump_dest = find_insn(file, insn->sec, dest_off); if (!insn->jump_dest) { WARN_FUNC("can't find call dest at %s+0x%lx", @@ -3314,6 +3336,10 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, next_insn = next_insn_to_validate(file, insn); if (func && insn->func && func != insn->func->pfunc) { + /* Ignore KCFI type preambles, which always fall through */ + if (!strncmp(func->name, "__cfi_", 6)) + return 0; + WARN("%s() falls through to next function %s()", func->name, insn->func->name); return 1; @@ -4111,7 +4137,9 @@ static int validate_ibt(struct objtool_file *file) !strcmp(sec->name, "__bug_table") || !strcmp(sec->name, "__ex_table") || !strcmp(sec->name, "__jump_table") || - !strcmp(sec->name, "__mcount_loc")) + !strcmp(sec->name, "__mcount_loc") || + !strcmp(sec->name, ".kcfi_traps") || + strstr(sec->name, "__patchable_function_entries")) continue; list_for_each_entry(reloc, &sec->reloc->reloc_list, list) diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index c25e957c1e520b8105516c6deb96578870ddb495..7e24b09b1163ad085b9ba7b3dceae0e3ff507502 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -619,6 +619,11 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab, Elf64_Xword entsize = symtab->sh.sh_entsize; int max_idx, idx = sym->idx; Elf_Scn *s, *t = NULL; + bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE && + sym->sym.st_shndx != SHN_XINDEX; + + if (is_special_shndx) + shndx = sym->sym.st_shndx; s = elf_getscn(elf->elf, symtab->idx); if (!s) { @@ -704,7 +709,7 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab, } /* setup extended section index magic and write the symbol */ - if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) { + if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) { sym->sym.st_shndx = shndx; if (!shndx_data) shndx = 0; diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index 4b9c71faa01ad21baf8e50b0d4a0cfb6a851f8fc..a653311d96938954dca67d723f96e71d998d4729 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -15,13 +15,14 @@ perf*.1 perf*.xml perf*.html common-cmds.h -perf.data -perf.data.old +perf*.data +perf*.data.old output.svg perf-archive perf-iostat tags TAGS +stats-*.csv cscope* config.mak config.mak.autogen @@ -29,6 +30,7 @@ config.mak.autogen *-flex.* *.pyc *.pyo +*.stdout .config-detected util/intel-pt-decoder/inat-tables.c arch/*/include/generated/ diff --git a/tools/perf/Documentation/intel-hybrid.txt b/tools/perf/Documentation/intel-hybrid.txt index c9302096dc461e82bf9a3a3d2c7e6430f1417675..e7a776ad25d719fa6af1115f2e75f48e8ae63948 100644 --- a/tools/perf/Documentation/intel-hybrid.txt +++ b/tools/perf/Documentation/intel-hybrid.txt @@ -21,11 +21,6 @@ cat /sys/devices/cpu_atom/cpus It indicates cpu0-cpu15 are core cpus and cpu16-cpu23 are atom cpus. -Quickstart - -List hybrid event ------------------ - As before, use perf-list to list the symbolic event. perf list @@ -40,7 +35,6 @@ the event is belong to. Same event name but with different pmu can be supported. Enable hybrid event with a specific pmu ---------------------------------------- To enable a core only event or atom only event, following syntax is supported: @@ -53,7 +47,6 @@ For example, count the 'cycles' event on core cpus. perf stat -e cpu_core/cycles/ Create two events for one hardware event automatically ------------------------------------------------------- When creating one event and the event is available on both atom and core, two events are created automatically. One is for atom, the other is for @@ -132,7 +125,6 @@ For perf-stat result, it displays two events: The first 'cycles' is core event, the second 'cycles' is atom event. Thread mode example: --------------------- perf-stat reports the scaled counts for hybrid event and with a percentage displayed. The percentage is the event's running time/enabling time. @@ -176,14 +168,12 @@ perf_event_attr: 604,097,080 cpu_atom/cycles/ (99.57%) perf-record: ------------- If there is no '-e' specified in perf record, on hybrid platform, it creates two default 'cycles' and adds them to event list. One is for core, the other is for atom. perf-stat: ----------- If there is no '-e' specified in perf stat, on hybrid platform, besides of software events, following events are created and diff --git a/tools/perf/Documentation/itrace.txt b/tools/perf/Documentation/itrace.txt index 6b189669c450e7030f8cf371dfe8bbfdabda539f..0916bbfe64cb7873d7433055f352518f208b6e5b 100644 --- a/tools/perf/Documentation/itrace.txt +++ b/tools/perf/Documentation/itrace.txt @@ -64,6 +64,7 @@ debug messages will or will not be logged. Each flag must be preceded by either '+' or '-'. The flags are: a all perf events + e output only on errors (size configurable - see linkperf:perf-config[1]) o output to stdout If supported, the 'q' option may be repeated to increase the effect. diff --git a/tools/perf/Documentation/perf-arm-coresight.txt b/tools/perf/Documentation/perf-arm-coresight.txt new file mode 100644 index 0000000000000000000000000000000000000000..c117fc50a2a95606fd8cba425cb254338e2910ad --- /dev/null +++ b/tools/perf/Documentation/perf-arm-coresight.txt @@ -0,0 +1,5 @@ +Arm CoreSight Support +===================== + +For full documentation, see Documentation/trace/coresight/coresight-perf.rst +in the kernel tree. diff --git a/tools/perf/Documentation/perf-c2c.txt b/tools/perf/Documentation/perf-c2c.txt index f1f7ae6b08d1e389a57280e8d00d77ca5dcbf864..5c5eb2def83e409c631d012423c1d524613002a1 100644 --- a/tools/perf/Documentation/perf-c2c.txt +++ b/tools/perf/Documentation/perf-c2c.txt @@ -19,9 +19,10 @@ C2C stands for Cache To Cache. The perf c2c tool provides means for Shared Data C2C/HITM analysis. It allows you to track down the cacheline contentions. -On x86, the tool is based on load latency and precise store facility events +On Intel, the tool is based on load latency and precise store facility events provided by Intel CPUs. On PowerPC, the tool uses random instruction sampling -with thresholding feature. +with thresholding feature. On AMD, the tool uses IBS op pmu (due to hardware +limitations, perf c2c is not supported on Zen3 cpus). These events provide: - memory address of the access @@ -49,7 +50,8 @@ RECORD OPTIONS -l:: --ldlat:: - Configure mem-loads latency. (x86 only) + Configure mem-loads latency. Supported on Intel and Arm64 processors + only. Ignored on other archs. -k:: --all-kernel:: @@ -135,11 +137,15 @@ Following perf record options are configured by default: -W,-d,--phys-data,--sample-cpu Unless specified otherwise with '-e' option, following events are monitored by -default on x86: +default on Intel: cpu/mem-loads,ldlat=30/P cpu/mem-stores/P +following on AMD: + + ibs_op// + and following on PowerPC: cpu/mem-loads/ diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt index 0420e71698ee4f9234eda381d7633805216848ca..39c890ead2dc0045a018268fcd5afc006e1ad5b4 100644 --- a/tools/perf/Documentation/perf-config.txt +++ b/tools/perf/Documentation/perf-config.txt @@ -729,6 +729,13 @@ auxtrace.*:: If the directory does not exist or has the wrong file type, the current directory is used. +itrace.*:: + + debug-log-buffer-size:: + Log size in bytes to output when using the option --itrace=d+e + Refer 'itrace' option of linkperf:perf-script[1] or + linkperf:perf-report[1]. The default is 16384. + daemon.*:: daemon.base:: diff --git a/tools/perf/Documentation/perf-inject.txt b/tools/perf/Documentation/perf-inject.txt index ffc293fdf61df3d0c9e85977dcaa99033e0ff271..c972032f4ca0d248f9dc473b4d5d09023944bf46 100644 --- a/tools/perf/Documentation/perf-inject.txt +++ b/tools/perf/Documentation/perf-inject.txt @@ -25,10 +25,17 @@ OPTIONS ------- -b:: --build-ids:: - Inject build-ids into the output stream + Inject build-ids of DSOs hit by samples into the output stream. + This means it needs to process all SAMPLE records to find the DSOs. ---buildid-all: - Inject build-ids of all DSOs into the output stream +--buildid-all:: + Inject build-ids of all DSOs into the output stream regardless of hits + and skip SAMPLE processing. + +--known-build-ids=:: + Override build-ids to inject using these comma-separated pairs of + build-id and path. Understands file://filename to read these pairs + from a file, which can be generated with perf buildid-list. -v:: --verbose:: diff --git a/tools/perf/Documentation/perf-intel-pt.txt b/tools/perf/Documentation/perf-intel-pt.txt index 3dc3f0ccbd5130f2571dc9ffe4681044eb4dcabc..92464a5d7eafdee9ebe75b3653080f1bd97592d4 100644 --- a/tools/perf/Documentation/perf-intel-pt.txt +++ b/tools/perf/Documentation/perf-intel-pt.txt @@ -943,12 +943,15 @@ event packets are recorded only if the "pwr_evt" config term was used. Refer to the config terms section above. The power events record information about C-state changes, whereas CBR is indicative of CPU frequency. perf script "event,synth" fields display information like this: + cbr: cbr: 22 freq: 2189 MHz (200%) mwait: hints: 0x60 extensions: 0x1 pwre: hw: 0 cstate: 2 sub-cstate: 0 exstop: ip: 1 pwrx: deepest cstate: 2 last cstate: 2 wake reason: 0x4 + Where: + "cbr" includes the frequency and the percentage of maximum non-turbo "mwait" shows mwait hints and extensions "pwre" shows C-state transitions (to a C-state deeper than C0) and @@ -956,6 +959,7 @@ Where: "exstop" indicates execution stopped and whether the IP was recorded exactly, "pwrx" indicates return to C0 + For more details refer to the Intel 64 and IA-32 Architectures Software Developer Manuals. @@ -969,8 +973,10 @@ are quite important. Users must know if what they are seeing is a complete picture or not. The "e" option may be followed by flags which affect what errors will or will not be reported. Each flag must be preceded by either '+' or '-'. The flags supported by Intel PT are: + -o Suppress overflow errors -l Suppress trace data lost errors + For example, for errors but not overflow or data lost errors: --itrace=e-o-l @@ -980,11 +986,16 @@ decoded packets and instructions. Note that this option slows down the decoder and that the resulting file may be very large. The "d" option may be followed by flags which affect what debug messages will or will not be logged. Each flag must be preceded by either '+' or '-'. The flags support by Intel PT are: + -a Suppress logging of perf events +a Log all perf events + +e Output only on decoding errors (size configurable) +o Output to stdout instead of "intel_pt.log" + By default, logged perf events are filtered by any specified time ranges, but -flag +a overrides that. +flag +a overrides that. The +e flag can be useful for analyzing errors. By +default, the log size in that case is 16384 bytes, but can be altered by +linkperf:perf-config[1] e.g. perf config itrace.debug-log-buffer-size=30000 In addition, the period of the "instructions" event can be specified. e.g. diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt index 193c5d8b8db924d166c60c85901c880f3da207b6..3b1e16563b795188a7dec58b9f1b562705027081 100644 --- a/tools/perf/Documentation/perf-lock.txt +++ b/tools/perf/Documentation/perf-lock.txt @@ -40,6 +40,10 @@ COMMON OPTIONS --verbose:: Be more verbose (show symbol address, etc). +-q:: +--quiet:: + Do not show any message. (Suppress -v) + -D:: --dump-raw-trace:: Dump raw trace in ASCII. @@ -94,6 +98,11 @@ REPORT OPTIONS EventManager_De 1845 1 636 futex-default-S 1609 0 0 +-E:: +--entries=<value>:: + Display this many entries. + + INFO OPTIONS ------------ @@ -105,6 +114,7 @@ INFO OPTIONS --map:: dump map of lock instances (address:name table) + CONTENTION OPTIONS -------------- @@ -148,6 +158,16 @@ CONTENTION OPTIONS --map-nr-entries:: Maximum number of BPF map entries (default: 10240). +--max-stack:: + Maximum stack depth when collecting lock contention (default: 8). + +--stack-skip + Number of stack depth to skip when finding a lock caller (default: 3). + +-E:: +--entries=<value>:: + Display this many entries. + SEE ALSO -------- diff --git a/tools/perf/Documentation/perf-mem.txt b/tools/perf/Documentation/perf-mem.txt index 66177511c5c4b381b4a8fe144884afe2080e0e71..005c95580b1e659a7cdf47778bb60e1ed0d2489b 100644 --- a/tools/perf/Documentation/perf-mem.txt +++ b/tools/perf/Documentation/perf-mem.txt @@ -85,7 +85,8 @@ RECORD OPTIONS Be more verbose (show counter open errors, etc) --ldlat <n>:: - Specify desired latency for loads event. (x86 only) + Specify desired latency for loads event. Supported on Intel and Arm64 + processors only. Ignored on other archs. In addition, for report all perf report options are valid, and for record all perf record options. diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt index 099817ef5150d7eee7c172e1b056694ea93a58d4..e41ae950fdc3b682e87b1f6d233021b782a0bdf7 100644 --- a/tools/perf/Documentation/perf-record.txt +++ b/tools/perf/Documentation/perf-record.txt @@ -397,6 +397,10 @@ following filters are defined: - abort_tx: only when the target is a hardware transaction abort - cond: conditional branches - save_type: save branch type during sampling in case binary is not available later + For the platforms with Intel Arch LBR support (12th-Gen+ client or + 4th-Gen Xeon+ server), the save branch type is unconditionally enabled + when the taken branch stack sampling is enabled. + - priv: save privilege state during sampling in case binary is not available later + The option requires at least one branch type among any, any_call, any_ret, ind_call, cond. @@ -407,6 +411,7 @@ is enabled for all the sampling events. The sampled branch type is the same for The various filters must be specified as a comma separated list: --branch-filter any_ret,u,k Note that this feature may not be available on all processors. +-W:: --weight:: Enable weightened sampling. An additional weight is recorded per sample and can be displayed with the weight and local_weight sort keys. This currently works for TSX @@ -430,8 +435,10 @@ if combined with -a or -C options. -D:: --delay=:: After starting the program, wait msecs before measuring (-1: start with events -disabled). This is useful to filter out the startup phase of the program, which -is often very different. +disabled), or enable events only for specified ranges of msecs (e.g. +-D 10-20,30-40 means wait 10 msecs, enable for 10 msecs, wait 10 msecs, enable +for 10 msecs, then stop). Note, delaying enabling of events is useful to filter +out the startup phase of the program, which is often very different. -I:: --intr-regs:: @@ -757,8 +764,6 @@ events in data directory files. Option specified with no or empty value defaults to CPU layout. Masks defined or provided by the option value are filtered through the mask provided by -C option. -include::intel-hybrid.txt[] - --debuginfod[=URLs]:: Specify debuginfod URL to be used when cacheing perf.data binaries, it follows the same syntax as the DEBUGINFOD_URLS variable, like: @@ -778,6 +783,8 @@ include::intel-hybrid.txt[] only, as of now. So the applications built without the frame pointer might see bogus addresses. +include::intel-hybrid.txt[] + SEE ALSO -------- linkperf:perf-stat[1], linkperf:perf-list[1], linkperf:perf-intel-pt[1] diff --git a/tools/perf/Documentation/perf-report.txt b/tools/perf/Documentation/perf-report.txt index 24efc0583c939414f93ba77dd75f3131478b6602..4533db2ee56bb31b53400ec523fb19cdc9288d76 100644 --- a/tools/perf/Documentation/perf-report.txt +++ b/tools/perf/Documentation/perf-report.txt @@ -73,7 +73,7 @@ OPTIONS Sort histogram entries by given key(s) - multiple keys can be specified in CSV format. Following sort keys are available: pid, comm, dso, symbol, parent, cpu, socket, srcline, weight, - local_weight, cgroup_id. + local_weight, cgroup_id, addr. Each key has following meaning: @@ -114,6 +114,7 @@ OPTIONS - local_ins_lat: Local instruction latency version - p_stage_cyc: On powerpc, this presents the number of cycles spent in a pipeline stage. And currently supported only on powerpc. + - addr: (Full) virtual address of the sampled instruction By default, comm, dso and symbol keys are used. (i.e. --sort comm,dso,symbol) diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 0661a1cf98556ed38f068c981c7f58c61f8119c2..6fd4b1384b975880c299e4263c8276d28ea0552d 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -19,6 +19,11 @@ detected_var = $(shell echo "$(1)=$($(1))" >> $(OUTPUT).config-detected) CFLAGS := $(EXTRA_CFLAGS) $(filter-out -Wnested-externs,$(EXTRA_WARNINGS)) HOSTCFLAGS := $(filter-out -Wnested-externs,$(EXTRA_WARNINGS)) +# Enabled Wthread-safety analysis for clang builds. +ifeq ($(CC_NO_CLANG), 0) + CFLAGS += -Wthread-safety +endif + include $(srctree)/tools/scripts/Makefile.arch $(call detected_var,SRCARCH) @@ -265,7 +270,7 @@ endif # defined. get-executable-or-default fails with an error if the first argument is supplied but # doesn't exist. override PYTHON_CONFIG := $(call get-executable-or-default,PYTHON_CONFIG,$(PYTHON_AUTO)) -override PYTHON := $(call get-executable-or-default,PYTHON,$(subst -config,,$(PYTHON_AUTO))) +override PYTHON := $(call get-executable-or-default,PYTHON,$(subst -config,,$(PYTHON_CONFIG))) grep-libs = $(filter -l%,$(1)) strip-libs = $(filter-out -l%,$(1)) @@ -1291,6 +1296,8 @@ perf_examples_instdir_SQ = $(subst ','\'',$(perf_examples_instdir)) STRACE_GROUPS_INSTDIR_SQ = $(subst ','\'',$(STRACE_GROUPS_INSTDIR)) tip_instdir_SQ = $(subst ','\'',$(tip_instdir)) +export perfexec_instdir_SQ + # If we install to $(HOME) we keep the traceevent default: # $(HOME)/.traceevent/plugins # Otherwise we install plugins into the global $(libdir). @@ -1301,14 +1308,18 @@ endif print_var = $(eval $(print_var_code)) $(info $(MSG)) define print_var_code - MSG = $(shell printf '...%30s: %s' $(1) $($(1))) + MSG = $(shell printf '...%40s: %s' $(1) $($(1))) endef +ifeq ($(feature_display),1) + $(call feature_display_entries) +endif + ifeq ($(VF),1) # Display EXTRA features which are detected manualy # from here with feature_check call and thus cannot # be partof global state output. - $(foreach feat,$(FEATURE_TESTS_EXTRA),$(call feature_print_status,$(feat),)) + $(foreach feat,$(FEATURE_TESTS_EXTRA),$(call feature_print_status,$(feat),) $(info $(MSG))) $(call print_var,prefix) $(call print_var,bindir) $(call print_var,libdir) @@ -1318,11 +1329,12 @@ ifeq ($(VF),1) $(call print_var,JDIR) ifeq ($(dwarf-post-unwind),1) - $(call feature_print_text,"DWARF post unwind library", $(dwarf-post-unwind-text)) + $(call feature_print_text,"DWARF post unwind library", $(dwarf-post-unwind-text)) $(info $(MSG)) endif - $(info ) endif +$(info ) + $(call detected_var,bindir_SQ) $(call detected_var,PYTHON_WORD) ifneq ($(OUTPUT),) @@ -1352,7 +1364,3 @@ endif # tests. $(shell rm -f $(FEATURE_DUMP_FILENAME)) $(foreach feat,$(FEATURE_TESTS),$(shell echo "$(call feature_assign,$(feat))" >> $(FEATURE_DUMP_FILENAME))) - -ifeq ($(feature_display),1) - $(call feature_display_entries) -endif diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf index e5921b3471535d45c49f957c9b787733ddc69a8d..a432e59afc42af92c18cace1941636050e7d6aa9 100644 --- a/tools/perf/Makefile.perf +++ b/tools/perf/Makefile.perf @@ -629,7 +629,16 @@ sync_file_range_tbls := $(srctree)/tools/perf/trace/beauty/sync_file_range.sh $(sync_file_range_arrays): $(linux_uapi_dir)/fs.h $(sync_file_range_tbls) $(Q)$(SHELL) '$(sync_file_range_tbls)' $(linux_uapi_dir) > $@ -all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) +TESTS_CORESIGHT_DIR := $(srctree)/tools/perf/tests/shell/coresight + +tests-coresight-targets: FORCE + $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR) + +tests-coresight-targets-clean: + $(call QUIET_CLEAN, coresight) + $(Q)$(MAKE) -C $(TESTS_CORESIGHT_DIR) O=$(OUTPUT) clean >/dev/null + +all: shell_compatibility_test $(ALL_PROGRAMS) $(LANG_BINDINGS) $(OTHER_PROGRAMS) tests-coresight-targets # Create python binding output directory if not already present _dummy := $(shell [ -d '$(OUTPUT)python' ] || mkdir -p '$(OUTPUT)python') @@ -954,11 +963,11 @@ ifndef NO_LIBBPF $(call QUIET_INSTALL, bpf-headers) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf/linux'; \ - $(INSTALL) include/bpf/*.h -t '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf'; \ - $(INSTALL) include/bpf/linux/*.h -t '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf/linux' + $(INSTALL) include/bpf/*.h -m 644 -t '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf'; \ + $(INSTALL) include/bpf/linux/*.h -m 644 -t '$(DESTDIR_SQ)$(perf_include_instdir_SQ)/bpf/linux' $(call QUIET_INSTALL, bpf-examples) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perf_examples_instdir_SQ)/bpf'; \ - $(INSTALL) examples/bpf/*.c -t '$(DESTDIR_SQ)$(perf_examples_instdir_SQ)/bpf' + $(INSTALL) examples/bpf/*.c -m 644 -t '$(DESTDIR_SQ)$(perf_examples_instdir_SQ)/bpf' endif $(call QUIET_INSTALL, perf-archive) \ $(INSTALL) $(OUTPUT)perf-archive -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)' @@ -967,13 +976,13 @@ endif ifndef NO_LIBAUDIT $(call QUIET_INSTALL, strace/groups) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(STRACE_GROUPS_INSTDIR_SQ)'; \ - $(INSTALL) trace/strace/groups/* -t '$(DESTDIR_SQ)$(STRACE_GROUPS_INSTDIR_SQ)' + $(INSTALL) trace/strace/groups/* -m 644 -t '$(DESTDIR_SQ)$(STRACE_GROUPS_INSTDIR_SQ)' endif ifndef NO_LIBPERL $(call QUIET_INSTALL, perl-scripts) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl/Perf-Trace-Util/lib/Perf/Trace'; \ - $(INSTALL) scripts/perl/Perf-Trace-Util/lib/Perf/Trace/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl/Perf-Trace-Util/lib/Perf/Trace'; \ - $(INSTALL) scripts/perl/*.pl -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl'; \ + $(INSTALL) scripts/perl/Perf-Trace-Util/lib/Perf/Trace/* -m 644 -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl/Perf-Trace-Util/lib/Perf/Trace'; \ + $(INSTALL) scripts/perl/*.pl -m 644 -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl/bin'; \ $(INSTALL) scripts/perl/bin/* -t '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/scripts/perl/bin' endif @@ -990,23 +999,26 @@ endif $(INSTALL) $(DLFILTERS) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/dlfilters'; $(call QUIET_INSTALL, perf_completion-script) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(sysconfdir_SQ)/bash_completion.d'; \ - $(INSTALL) perf-completion.sh '$(DESTDIR_SQ)$(sysconfdir_SQ)/bash_completion.d/perf' + $(INSTALL) perf-completion.sh -m 644 '$(DESTDIR_SQ)$(sysconfdir_SQ)/bash_completion.d/perf' $(call QUIET_INSTALL, perf-tip) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(tip_instdir_SQ)'; \ - $(INSTALL) Documentation/tips.txt -t '$(DESTDIR_SQ)$(tip_instdir_SQ)' + $(INSTALL) Documentation/tips.txt -m 644 -t '$(DESTDIR_SQ)$(tip_instdir_SQ)' install-tests: all install-gtk $(call QUIET_INSTALL, tests) \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ - $(INSTALL) tests/attr.py '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ + $(INSTALL) tests/attr.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ $(INSTALL) tests/pe-file.exe* '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/attr'; \ - $(INSTALL) tests/attr/* '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/attr'; \ + $(INSTALL) tests/attr/* -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/attr'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \ $(INSTALL) tests/shell/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ - $(INSTALL) tests/shell/lib/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ - $(INSTALL) tests/shell/lib/*.py '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib' + $(INSTALL) tests/shell/lib/*.sh -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ + $(INSTALL) tests/shell/lib/*.py -m 644 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight' ; \ + $(INSTALL) tests/shell/coresight/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/coresight' + $(Q)$(MAKE) -C tests/shell/coresight install-tests install-bin: install-tools install-tests install-traceevent-plugins @@ -1077,7 +1089,7 @@ endif # BUILD_BPF_SKEL bpf-skel-clean: $(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TMP_OUT) $(SKELETONS) -clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBPERF)-clean fixdep-clean python-clean bpf-skel-clean +clean:: $(LIBTRACEEVENT)-clean $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBPERF)-clean fixdep-clean python-clean bpf-skel-clean tests-coresight-targets-clean $(call QUIET_CLEAN, core-objs) $(RM) $(LIBPERF_A) $(OUTPUT)perf-archive $(OUTPUT)perf-iostat $(LANG_BINDINGS) $(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete $(Q)$(RM) $(OUTPUT).config-detected diff --git a/tools/perf/arch/arm/util/auxtrace.c b/tools/perf/arch/arm/util/auxtrace.c index 5fc6a2a3dbc5fd77215047d7d2c5e8df92f165c4..deeb163999cebc0f961c47ebba34b2bd1d8518d1 100644 --- a/tools/perf/arch/arm/util/auxtrace.c +++ b/tools/perf/arch/arm/util/auxtrace.c @@ -4,9 +4,11 @@ * Author: Mathieu Poirier <mathieu.poirier@linaro.org> */ +#include <dirent.h> #include <stdbool.h> #include <linux/coresight-pmu.h> #include <linux/zalloc.h> +#include <api/fs/fs.h> #include "../../../util/auxtrace.h" #include "../../../util/debug.h" @@ -14,6 +16,7 @@ #include "../../../util/pmu.h" #include "cs-etm.h" #include "arm-spe.h" +#include "hisi-ptt.h" static struct perf_pmu **find_all_arm_spe_pmus(int *nr_spes, int *err) { @@ -50,42 +53,114 @@ static struct perf_pmu **find_all_arm_spe_pmus(int *nr_spes, int *err) return arm_spe_pmus; } +static struct perf_pmu **find_all_hisi_ptt_pmus(int *nr_ptts, int *err) +{ + const char *sysfs = sysfs__mountpoint(); + struct perf_pmu **hisi_ptt_pmus = NULL; + struct dirent *dent; + char path[PATH_MAX]; + DIR *dir = NULL; + int idx = 0; + + snprintf(path, PATH_MAX, "%s" EVENT_SOURCE_DEVICE_PATH, sysfs); + dir = opendir(path); + if (!dir) { + pr_err("can't read directory '%s'\n", EVENT_SOURCE_DEVICE_PATH); + *err = -EINVAL; + return NULL; + } + + while ((dent = readdir(dir))) { + if (strstr(dent->d_name, HISI_PTT_PMU_NAME)) + (*nr_ptts)++; + } + + if (!(*nr_ptts)) + goto out; + + hisi_ptt_pmus = zalloc(sizeof(struct perf_pmu *) * (*nr_ptts)); + if (!hisi_ptt_pmus) { + pr_err("hisi_ptt alloc failed\n"); + *err = -ENOMEM; + goto out; + } + + rewinddir(dir); + while ((dent = readdir(dir))) { + if (strstr(dent->d_name, HISI_PTT_PMU_NAME) && idx < *nr_ptts) { + hisi_ptt_pmus[idx] = perf_pmu__find(dent->d_name); + if (hisi_ptt_pmus[idx]) + idx++; + } + } + +out: + closedir(dir); + return hisi_ptt_pmus; +} + +static struct perf_pmu *find_pmu_for_event(struct perf_pmu **pmus, + int pmu_nr, struct evsel *evsel) +{ + int i; + + if (!pmus) + return NULL; + + for (i = 0; i < pmu_nr; i++) { + if (evsel->core.attr.type == pmus[i]->type) + return pmus[i]; + } + + return NULL; +} + struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, int *err) { - struct perf_pmu *cs_etm_pmu; + struct perf_pmu *cs_etm_pmu = NULL; + struct perf_pmu **arm_spe_pmus = NULL; + struct perf_pmu **hisi_ptt_pmus = NULL; struct evsel *evsel; - bool found_etm = false; + struct perf_pmu *found_etm = NULL; struct perf_pmu *found_spe = NULL; - struct perf_pmu **arm_spe_pmus = NULL; + struct perf_pmu *found_ptt = NULL; + int auxtrace_event_cnt = 0; int nr_spes = 0; - int i = 0; + int nr_ptts = 0; if (!evlist) return NULL; cs_etm_pmu = perf_pmu__find(CORESIGHT_ETM_PMU_NAME); arm_spe_pmus = find_all_arm_spe_pmus(&nr_spes, err); + hisi_ptt_pmus = find_all_hisi_ptt_pmus(&nr_ptts, err); evlist__for_each_entry(evlist, evsel) { - if (cs_etm_pmu && - evsel->core.attr.type == cs_etm_pmu->type) - found_etm = true; - - if (!nr_spes || found_spe) - continue; - - for (i = 0; i < nr_spes; i++) { - if (evsel->core.attr.type == arm_spe_pmus[i]->type) { - found_spe = arm_spe_pmus[i]; - break; - } - } + if (cs_etm_pmu && !found_etm) + found_etm = find_pmu_for_event(&cs_etm_pmu, 1, evsel); + + if (arm_spe_pmus && !found_spe) + found_spe = find_pmu_for_event(arm_spe_pmus, nr_spes, evsel); + + if (hisi_ptt_pmus && !found_ptt) + found_ptt = find_pmu_for_event(hisi_ptt_pmus, nr_ptts, evsel); } + free(arm_spe_pmus); + free(hisi_ptt_pmus); + + if (found_etm) + auxtrace_event_cnt++; - if (found_etm && found_spe) { - pr_err("Concurrent ARM Coresight ETM and SPE operation not currently supported\n"); + if (found_spe) + auxtrace_event_cnt++; + + if (found_ptt) + auxtrace_event_cnt++; + + if (auxtrace_event_cnt > 1) { + pr_err("Concurrent AUX trace operation not currently supported\n"); *err = -EOPNOTSUPP; return NULL; } @@ -96,6 +171,9 @@ struct auxtrace_record #if defined(__aarch64__) if (found_spe) return arm_spe_recording_init(err, found_spe); + + if (found_ptt) + return hisi_ptt_recording_init(err, found_ptt); #endif /* diff --git a/tools/perf/arch/arm/util/pmu.c b/tools/perf/arch/arm/util/pmu.c index b8b23b9dc5987a4dc118ee7bc45b162d08a57111..887c8addc4916848e9eadd29240dc0f440e8b7aa 100644 --- a/tools/perf/arch/arm/util/pmu.c +++ b/tools/perf/arch/arm/util/pmu.c @@ -10,6 +10,7 @@ #include <linux/string.h> #include "arm-spe.h" +#include "hisi-ptt.h" #include "../../../util/pmu.h" struct perf_event_attr @@ -22,6 +23,8 @@ struct perf_event_attr #if defined(__aarch64__) } else if (strstarts(pmu->name, ARM_SPE_PMU_NAME)) { return arm_spe_pmu_default_config(pmu); + } else if (strstarts(pmu->name, HISI_PTT_PMU_NAME)) { + pmu->selectable = true; #endif } diff --git a/tools/perf/arch/arm64/annotate/instructions.c b/tools/perf/arch/arm64/annotate/instructions.c index 037e292ecd8eb0a6c3bb883bcb3c352f92a75491..4af0c3a0f86ee5911b723763cae3f2529b09d6dd 100644 --- a/tools/perf/arch/arm64/annotate/instructions.c +++ b/tools/perf/arch/arm64/annotate/instructions.c @@ -102,7 +102,7 @@ static int arm64__annotate_init(struct arch *arch, char *cpuid __maybe_unused) if (err) goto out_free_arm; /* b, b.cond, br, cbz/cbnz, tbz/tbnz */ - err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|le|ls|lt|mi|ne|pl)?n?z?$", + err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|hs|le|lo|ls|lt|mi|ne|pl|vc|vs)?n?z?$", REG_EXTENDED); if (err) goto out_free_call; diff --git a/tools/perf/arch/arm64/util/Build b/tools/perf/arch/arm64/util/Build index 9fcb4e68add930af4d7c4df4539f319925f3b927..337aa9bdf905de3ece8145d23f993f9d73a8b30e 100644 --- a/tools/perf/arch/arm64/util/Build +++ b/tools/perf/arch/arm64/util/Build @@ -11,4 +11,4 @@ perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o perf-$(CONFIG_AUXTRACE) += ../../arm/util/pmu.o \ ../../arm/util/auxtrace.o \ ../../arm/util/cs-etm.o \ - arm-spe.o mem-events.o + arm-spe.o mem-events.o hisi-ptt.o diff --git a/tools/perf/arch/arm64/util/hisi-ptt.c b/tools/perf/arch/arm64/util/hisi-ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..ba97c8a562a0261b53df54b525c77744290bc6ce --- /dev/null +++ b/tools/perf/arch/arm64/util/hisi-ptt.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/bitops.h> +#include <linux/log2.h> +#include <linux/zalloc.h> +#include <time.h> + +#include <internal/lib.h> // page_size +#include "../../../util/auxtrace.h" +#include "../../../util/cpumap.h" +#include "../../../util/debug.h" +#include "../../../util/event.h" +#include "../../../util/evlist.h" +#include "../../../util/evsel.h" +#include "../../../util/hisi-ptt.h" +#include "../../../util/pmu.h" +#include "../../../util/record.h" +#include "../../../util/session.h" +#include "../../../util/tsc.h" + +#define KiB(x) ((x) * 1024) +#define MiB(x) ((x) * 1024 * 1024) + +struct hisi_ptt_recording { + struct auxtrace_record itr; + struct perf_pmu *hisi_ptt_pmu; + struct evlist *evlist; +}; + +static size_t +hisi_ptt_info_priv_size(struct auxtrace_record *itr __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return HISI_PTT_AUXTRACE_PRIV_SIZE; +} + +static int hisi_ptt_info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu; + + if (priv_size != HISI_PTT_AUXTRACE_PRIV_SIZE) + return -EINVAL; + + if (!session->evlist->core.nr_mmaps) + return -EINVAL; + + auxtrace_info->type = PERF_AUXTRACE_HISI_PTT; + auxtrace_info->priv[0] = hisi_ptt_pmu->type; + + return 0; +} + +static int hisi_ptt_set_auxtrace_mmap_page(struct record_opts *opts) +{ + bool privileged = perf_event_paranoid_check(-1); + + if (!opts->full_auxtrace) + return 0; + + if (opts->full_auxtrace && !opts->auxtrace_mmap_pages) { + if (privileged) { + opts->auxtrace_mmap_pages = MiB(16) / page_size; + } else { + opts->auxtrace_mmap_pages = KiB(128) / page_size; + if (opts->mmap_pages == UINT_MAX) + opts->mmap_pages = KiB(256) / page_size; + } + } + + /* Validate auxtrace_mmap_pages */ + if (opts->auxtrace_mmap_pages) { + size_t sz = opts->auxtrace_mmap_pages * (size_t)page_size; + size_t min_sz = KiB(8); + + if (sz < min_sz || !is_power_of_2(sz)) { + pr_err("Invalid mmap size for HISI PTT: must be at least %zuKiB and a power of 2\n", + min_sz / 1024); + return -EINVAL; + } + } + + return 0; +} + +static int hisi_ptt_recording_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + struct perf_pmu *hisi_ptt_pmu = pttr->hisi_ptt_pmu; + struct evsel *evsel, *hisi_ptt_evsel = NULL; + struct evsel *tracking_evsel; + int err; + + pttr->evlist = evlist; + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == hisi_ptt_pmu->type) { + if (hisi_ptt_evsel) { + pr_err("There may be only one " HISI_PTT_PMU_NAME "x event\n"); + return -EINVAL; + } + evsel->core.attr.freq = 0; + evsel->core.attr.sample_period = 1; + evsel->needs_auxtrace_mmap = true; + hisi_ptt_evsel = evsel; + opts->full_auxtrace = true; + } + } + + err = hisi_ptt_set_auxtrace_mmap_page(opts); + if (err) + return err; + /* + * To obtain the auxtrace buffer file descriptor, the auxtrace event + * must come first. + */ + evlist__to_front(evlist, hisi_ptt_evsel); + evsel__set_sample_bit(hisi_ptt_evsel, TIME); + + /* Add dummy event to keep tracking */ + err = parse_event(evlist, "dummy:u"); + if (err) + return err; + + tracking_evsel = evlist__last(evlist); + evlist__set_tracking_event(evlist, tracking_evsel); + + tracking_evsel->core.attr.freq = 0; + tracking_evsel->core.attr.sample_period = 1; + evsel__set_sample_bit(tracking_evsel, TIME); + + return 0; +} + +static u64 hisi_ptt_reference(struct auxtrace_record *itr __maybe_unused) +{ + return rdtsc(); +} + +static void hisi_ptt_recording_free(struct auxtrace_record *itr) +{ + struct hisi_ptt_recording *pttr = + container_of(itr, struct hisi_ptt_recording, itr); + + free(pttr); +} + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu) +{ + struct hisi_ptt_recording *pttr; + + if (!hisi_ptt_pmu) { + *err = -ENODEV; + return NULL; + } + + pttr = zalloc(sizeof(*pttr)); + if (!pttr) { + *err = -ENOMEM; + return NULL; + } + + pttr->hisi_ptt_pmu = hisi_ptt_pmu; + pttr->itr.pmu = hisi_ptt_pmu; + pttr->itr.recording_options = hisi_ptt_recording_options; + pttr->itr.info_priv_size = hisi_ptt_info_priv_size; + pttr->itr.info_fill = hisi_ptt_info_fill; + pttr->itr.free = hisi_ptt_recording_free; + pttr->itr.reference = hisi_ptt_reference; + pttr->itr.read_finish = auxtrace_record__read_finish; + pttr->itr.alignment = 0; + + *err = 0; + return &pttr->itr; +} diff --git a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl index 2600b4237292c78c44363159a02eaca9a2dd88c8..2bca64f96164af9d4ebc3335feab2372891af2a1 100644 --- a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl +++ b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl @@ -110,7 +110,7 @@ 79 common settimeofday sys_settimeofday compat_sys_settimeofday 80 common getgroups sys_getgroups 81 common setgroups sys_setgroups -82 32 select ppc_select sys_ni_syscall +82 32 select sys_old_select compat_sys_old_select 82 64 select sys_ni_syscall 82 spu select sys_ni_syscall 83 common symlink sys_symlink @@ -178,9 +178,9 @@ 133 common fchdir sys_fchdir 134 common bdflush sys_ni_syscall 135 common sysfs sys_sysfs -136 32 personality sys_personality ppc64_personality -136 64 personality ppc64_personality -136 spu personality ppc64_personality +136 32 personality sys_personality compat_sys_ppc64_personality +136 64 personality sys_ppc64_personality +136 spu personality sys_ppc64_personality 137 common afs_syscall sys_ni_syscall 138 common setfsuid sys_setfsuid 139 common setfsgid sys_setfsgid @@ -228,8 +228,8 @@ 176 64 rt_sigtimedwait sys_rt_sigtimedwait 177 nospu rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo 178 nospu rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend -179 common pread64 sys_pread64 compat_sys_pread64 -180 common pwrite64 sys_pwrite64 compat_sys_pwrite64 +179 common pread64 sys_pread64 compat_sys_ppc_pread64 +180 common pwrite64 sys_pwrite64 compat_sys_ppc_pwrite64 181 common chown sys_chown 182 common getcwd sys_getcwd 183 common capget sys_capget @@ -242,10 +242,10 @@ 188 common putpmsg sys_ni_syscall 189 nospu vfork sys_vfork 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit -191 common readahead sys_readahead compat_sys_readahead +191 common readahead sys_readahead compat_sys_ppc_readahead 192 32 mmap2 sys_mmap2 compat_sys_mmap2 -193 32 truncate64 sys_truncate64 compat_sys_truncate64 -194 32 ftruncate64 sys_ftruncate64 compat_sys_ftruncate64 +193 32 truncate64 sys_truncate64 compat_sys_ppc_truncate64 +194 32 ftruncate64 sys_ftruncate64 compat_sys_ppc_ftruncate64 195 32 stat64 sys_stat64 196 32 lstat64 sys_lstat64 197 32 fstat64 sys_fstat64 @@ -288,7 +288,7 @@ 230 common io_submit sys_io_submit compat_sys_io_submit 231 common io_cancel sys_io_cancel 232 nospu set_tid_address sys_set_tid_address -233 common fadvise64 sys_fadvise64 ppc32_fadvise64 +233 common fadvise64 sys_fadvise64 compat_sys_ppc32_fadvise64 234 nospu exit_group sys_exit_group 235 nospu lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie 236 common epoll_create sys_epoll_create @@ -323,7 +323,7 @@ 251 spu utimes sys_utimes 252 common statfs64 sys_statfs64 compat_sys_statfs64 253 common fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 -254 32 fadvise64_64 ppc_fadvise64_64 +254 32 fadvise64_64 sys_ppc_fadvise64_64 254 spu fadvise64_64 sys_ni_syscall 255 common rtas sys_rtas 256 32 sys_debug_setcontext sys_debug_setcontext sys_ni_syscall @@ -390,7 +390,7 @@ 305 common signalfd sys_signalfd compat_sys_signalfd 306 common timerfd_create sys_timerfd_create 307 common eventfd sys_eventfd -308 common sync_file_range2 sys_sync_file_range2 compat_sys_sync_file_range2 +308 common sync_file_range2 sys_sync_file_range2 compat_sys_ppc_sync_file_range2 309 nospu fallocate sys_fallocate compat_sys_fallocate 310 nospu subpage_prot sys_subpage_prot 311 32 timerfd_settime sys_timerfd_settime32 diff --git a/tools/perf/arch/x86/util/intel-pt.c b/tools/perf/arch/x86/util/intel-pt.c index 13933020a79eb182d26916108d1ca4505582348e..af102f471e9f453d512e61cc920887ef4061b8c1 100644 --- a/tools/perf/arch/x86/util/intel-pt.c +++ b/tools/perf/arch/x86/util/intel-pt.c @@ -11,6 +11,7 @@ #include <linux/bitops.h> #include <linux/log2.h> #include <linux/zalloc.h> +#include <linux/err.h> #include <cpuid.h> #include "../../../util/session.h" @@ -426,20 +427,14 @@ static int intel_pt_track_switches(struct evlist *evlist) if (!evlist__can_select_event(evlist, sched_switch)) return -EPERM; - err = parse_event(evlist, sched_switch); - if (err) { - pr_debug2("%s: failed to parse %s, error %d\n", + evsel = evlist__add_sched_switch(evlist, true); + if (IS_ERR(evsel)) { + err = PTR_ERR(evsel); + pr_debug2("%s: failed to create %s, error = %d\n", __func__, sched_switch, err); return err; } - evsel = evlist__last(evlist); - - evsel__set_sample_bit(evsel, CPU); - evsel__set_sample_bit(evsel, TIME); - - evsel->core.system_wide = true; - evsel->no_aux_samples = true; evsel->immediate = true; return 0; @@ -871,7 +866,7 @@ static int intel_pt_recording_options(struct auxtrace_record *itr, * User space tasks can migrate between CPUs, so when tracing * selected CPUs, sideband for all CPUs is still needed. */ - need_system_wide_tracking = evlist->core.has_user_cpus && + need_system_wide_tracking = opts->target.cpu_list && !intel_pt_evsel->core.attr.exclude_user; tracking_evsel = evlist__add_aux_dummy(evlist, need_system_wide_tracking); diff --git a/tools/perf/arch/x86/util/mem-events.c b/tools/perf/arch/x86/util/mem-events.c index 5214370ca4e488a43226ff6caabdb48240968ae2..f683ac702247cdbcf4433bbc32c0feb26f793df8 100644 --- a/tools/perf/arch/x86/util/mem-events.c +++ b/tools/perf/arch/x86/util/mem-events.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include "util/pmu.h" +#include "util/env.h" #include "map_symbol.h" #include "mem-events.h" +#include "linux/string.h" static char mem_loads_name[100]; static bool mem_loads_name__init; @@ -12,18 +14,43 @@ static char mem_stores_name[100]; #define E(t, n, s) { .tag = t, .name = n, .sysfs_name = s } -static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { +static struct perf_mem_event perf_mem_events_intel[PERF_MEM_EVENTS__MAX] = { E("ldlat-loads", "%s/mem-loads,ldlat=%u/P", "%s/events/mem-loads"), E("ldlat-stores", "%s/mem-stores/P", "%s/events/mem-stores"), E(NULL, NULL, NULL), }; +static struct perf_mem_event perf_mem_events_amd[PERF_MEM_EVENTS__MAX] = { + E(NULL, NULL, NULL), + E(NULL, NULL, NULL), + E("mem-ldst", "ibs_op//", "ibs_op"), +}; + +static int perf_mem_is_amd_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + + perf_env__cpuid(&env); + if (env.cpuid && strstarts(env.cpuid, "AuthenticAMD")) + return 1; + return -1; +} + struct perf_mem_event *perf_mem_events__ptr(int i) { + /* 0: Uninitialized, 1: Yes, -1: No */ + static int is_amd; + if (i >= PERF_MEM_EVENTS__MAX) return NULL; - return &perf_mem_events[i]; + if (!is_amd) + is_amd = perf_mem_is_amd_cpu(); + + if (is_amd == 1) + return &perf_mem_events_amd[i]; + + return &perf_mem_events_intel[i]; } bool is_mem_loads_aux_event(struct evsel *leader) diff --git a/tools/perf/bench/epoll-ctl.c b/tools/perf/bench/epoll-ctl.c index 4256dc5d6236d4aeef59b02cc2cbcfa3dec4c26b..521d1ff97b069acdeace710bf1eed0d0a051746c 100644 --- a/tools/perf/bench/epoll-ctl.c +++ b/tools/perf/bench/epoll-ctl.c @@ -23,6 +23,7 @@ #include <sys/eventfd.h> #include <perf/cpumap.h> +#include "../util/mutex.h" #include "../util/stat.h" #include <subcmd/parse-options.h> #include "bench.h" @@ -58,10 +59,10 @@ static unsigned int nested = 0; /* amount of fds to monitor, per thread */ static unsigned int nfds = 64; -static pthread_mutex_t thread_lock; +static struct mutex thread_lock; static unsigned int threads_starting; static struct stats all_stats[EPOLL_NR_OPS]; -static pthread_cond_t thread_parent, thread_worker; +static struct cond thread_parent, thread_worker; struct worker { int tid; @@ -174,12 +175,12 @@ static void *workerfn(void *arg) struct timespec ts = { .tv_sec = 0, .tv_nsec = 250 }; - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); /* Let 'em loose */ do { @@ -367,9 +368,9 @@ int bench_epoll_ctl(int argc, const char **argv) for (i = 0; i < EPOLL_NR_OPS; i++) init_stats(&all_stats[i]); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); threads_starting = nthreads; @@ -377,11 +378,11 @@ int bench_epoll_ctl(int argc, const char **argv) do_threads(worker, cpu); - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); sleep(nsecs); toggle_done(0, NULL, NULL); @@ -394,9 +395,9 @@ int bench_epoll_ctl(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); for (i = 0; i < nthreads; i++) { unsigned long t[EPOLL_NR_OPS]; diff --git a/tools/perf/bench/epoll-wait.c b/tools/perf/bench/epoll-wait.c index 2728b0140853fd6a3e89b90317e0944b0140df40..c1cdf03c075dc72910523e1299d300794e2453d1 100644 --- a/tools/perf/bench/epoll-wait.c +++ b/tools/perf/bench/epoll-wait.c @@ -79,6 +79,7 @@ #include <perf/cpumap.h> #include "../util/stat.h" +#include "../util/mutex.h" #include <subcmd/parse-options.h> #include "bench.h" @@ -109,10 +110,10 @@ static bool multiq; /* use an epoll instance per thread */ /* amount of fds to monitor, per thread */ static unsigned int nfds = 64; -static pthread_mutex_t thread_lock; +static struct mutex thread_lock; static unsigned int threads_starting; static struct stats throughput_stats; -static pthread_cond_t thread_parent, thread_worker; +static struct cond thread_parent, thread_worker; struct worker { int tid; @@ -189,12 +190,12 @@ static void *workerfn(void *arg) int to = nonblocking? 0 : -1; int efd = multiq ? w->epollfd : epollfd; - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); do { /* @@ -485,9 +486,9 @@ int bench_epoll_wait(int argc, const char **argv) getpid(), nthreads, oneshot ? " (EPOLLONESHOT semantics)": "", nfds, nsecs); init_stats(&throughput_stats); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); threads_starting = nthreads; @@ -495,11 +496,11 @@ int bench_epoll_wait(int argc, const char **argv) do_threads(worker, cpu); - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); /* * At this point the workers should be blocked waiting for read events @@ -522,9 +523,9 @@ int bench_epoll_wait(int argc, const char **argv) err(EXIT_FAILURE, "pthread_join"); /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); /* sort the array back before reporting */ if (randomize) diff --git a/tools/perf/bench/futex-hash.c b/tools/perf/bench/futex-hash.c index f05db4cf983d6e0c8e32ab7920a6a0865002c870..2005a3fa3026799d1cfcd246cc956ac4e528c97b 100644 --- a/tools/perf/bench/futex-hash.c +++ b/tools/perf/bench/futex-hash.c @@ -23,6 +23,7 @@ #include <sys/mman.h> #include <perf/cpumap.h> +#include "../util/mutex.h" #include "../util/stat.h" #include <subcmd/parse-options.h> #include "bench.h" @@ -34,10 +35,10 @@ static bool done = false; static int futex_flag = 0; struct timeval bench__start, bench__end, bench__runtime; -static pthread_mutex_t thread_lock; +static struct mutex thread_lock; static unsigned int threads_starting; static struct stats throughput_stats; -static pthread_cond_t thread_parent, thread_worker; +static struct cond thread_parent, thread_worker; struct worker { int tid; @@ -73,12 +74,12 @@ static void *workerfn(void *arg) unsigned int i; unsigned long ops = w->ops; /* avoid cacheline bouncing */ - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); do { for (i = 0; i < params.nfutexes; i++, ops++) { @@ -165,9 +166,9 @@ int bench_futex_hash(int argc, const char **argv) getpid(), params.nthreads, params.nfutexes, params.fshared ? "shared":"private", params.runtime); init_stats(&throughput_stats); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); threads_starting = params.nthreads; pthread_attr_init(&thread_attr); @@ -203,11 +204,11 @@ int bench_futex_hash(int argc, const char **argv) CPU_FREE(cpuset); pthread_attr_destroy(&thread_attr); - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); sleep(params.runtime); toggle_done(0, NULL, NULL); @@ -219,9 +220,9 @@ int bench_futex_hash(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); for (i = 0; i < params.nthreads; i++) { unsigned long t = bench__runtime.tv_sec > 0 ? diff --git a/tools/perf/bench/futex-lock-pi.c b/tools/perf/bench/futex-lock-pi.c index 0abb3f7ee24f78ece65b33c69e2f95461f250d35..2d04179497270a712645ca115a7ac245cc5de56c 100644 --- a/tools/perf/bench/futex-lock-pi.c +++ b/tools/perf/bench/futex-lock-pi.c @@ -8,6 +8,7 @@ #include <pthread.h> #include <signal.h> +#include "../util/mutex.h" #include "../util/stat.h" #include <subcmd/parse-options.h> #include <linux/compiler.h> @@ -34,10 +35,10 @@ static u_int32_t global_futex = 0; static struct worker *worker; static bool done = false; static int futex_flag = 0; -static pthread_mutex_t thread_lock; +static struct mutex thread_lock; static unsigned int threads_starting; static struct stats throughput_stats; -static pthread_cond_t thread_parent, thread_worker; +static struct cond thread_parent, thread_worker; static struct bench_futex_parameters params = { .runtime = 10, @@ -83,12 +84,12 @@ static void *workerfn(void *arg) struct worker *w = (struct worker *) arg; unsigned long ops = w->ops; - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); do { int ret; @@ -197,9 +198,9 @@ int bench_futex_lock_pi(int argc, const char **argv) getpid(), params.nthreads, params.runtime); init_stats(&throughput_stats); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); threads_starting = params.nthreads; pthread_attr_init(&thread_attr); @@ -208,11 +209,11 @@ int bench_futex_lock_pi(int argc, const char **argv) create_threads(worker, thread_attr, cpu); pthread_attr_destroy(&thread_attr); - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); sleep(params.runtime); toggle_done(0, NULL, NULL); @@ -224,9 +225,9 @@ int bench_futex_lock_pi(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); for (i = 0; i < params.nthreads; i++) { unsigned long t = bench__runtime.tv_sec > 0 ? diff --git a/tools/perf/bench/futex-requeue.c b/tools/perf/bench/futex-requeue.c index b6faabfafb8eed33d6046c4d74c7c842f23aca17..69ad896f556c93aacfc1206c6d3747f96dd17ded 100644 --- a/tools/perf/bench/futex-requeue.c +++ b/tools/perf/bench/futex-requeue.c @@ -15,6 +15,7 @@ #include <pthread.h> #include <signal.h> +#include "../util/mutex.h" #include "../util/stat.h" #include <subcmd/parse-options.h> #include <linux/compiler.h> @@ -34,8 +35,8 @@ static u_int32_t futex1 = 0, futex2 = 0; static pthread_t *worker; static bool done = false; -static pthread_mutex_t thread_lock; -static pthread_cond_t thread_parent, thread_worker; +static struct mutex thread_lock; +static struct cond thread_parent, thread_worker; static struct stats requeuetime_stats, requeued_stats; static unsigned int threads_starting; static int futex_flag = 0; @@ -82,12 +83,12 @@ static void *workerfn(void *arg __maybe_unused) { int ret; - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); while (1) { if (!params.pi) { @@ -209,9 +210,9 @@ int bench_futex_requeue(int argc, const char **argv) init_stats(&requeued_stats); init_stats(&requeuetime_stats); pthread_attr_init(&thread_attr); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); for (j = 0; j < bench_repeat && !done; j++) { unsigned int nrequeued = 0, wakeups = 0; @@ -221,11 +222,11 @@ int bench_futex_requeue(int argc, const char **argv) block_threads(worker, thread_attr, cpu); /* make sure all threads are already blocked */ - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); usleep(100000); @@ -297,9 +298,9 @@ int bench_futex_requeue(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); pthread_attr_destroy(&thread_attr); print_summary(); diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c index e47f46a3a47e934db6aa875cfb16e6c1482241bf..6682e49d0ee03a9e124e7001193360cb308529de 100644 --- a/tools/perf/bench/futex-wake-parallel.c +++ b/tools/perf/bench/futex-wake-parallel.c @@ -10,6 +10,7 @@ #include "bench.h" #include <linux/compiler.h> #include "../util/debug.h" +#include "../util/mutex.h" #ifndef HAVE_PTHREAD_BARRIER int bench_futex_wake_parallel(int argc __maybe_unused, const char **argv __maybe_unused) @@ -49,8 +50,8 @@ static u_int32_t futex = 0; static pthread_t *blocked_worker; static bool done = false; -static pthread_mutex_t thread_lock; -static pthread_cond_t thread_parent, thread_worker; +static struct mutex thread_lock; +static struct cond thread_parent, thread_worker; static pthread_barrier_t barrier; static struct stats waketime_stats, wakeup_stats; static unsigned int threads_starting; @@ -125,12 +126,12 @@ static void wakeup_threads(struct thread_data *td, pthread_attr_t thread_attr) static void *blocked_workerfn(void *arg __maybe_unused) { - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); while (1) { /* handle spurious wakeups */ if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR) @@ -294,9 +295,9 @@ int bench_futex_wake_parallel(int argc, const char **argv) init_stats(&waketime_stats); pthread_attr_init(&thread_attr); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); for (j = 0; j < bench_repeat && !done; j++) { waking_worker = calloc(params.nwakes, sizeof(*waking_worker)); @@ -307,11 +308,11 @@ int bench_futex_wake_parallel(int argc, const char **argv) block_threads(blocked_worker, thread_attr, cpu); /* make sure all threads are already blocked */ - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); usleep(100000); @@ -332,9 +333,9 @@ int bench_futex_wake_parallel(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); pthread_attr_destroy(&thread_attr); print_summary(); diff --git a/tools/perf/bench/futex-wake.c b/tools/perf/bench/futex-wake.c index 201a3555f09a2053fa2e30176faaae25e7003555..9ecab6620a8752b54cf37e2f3e3ff896e7759009 100644 --- a/tools/perf/bench/futex-wake.c +++ b/tools/perf/bench/futex-wake.c @@ -14,6 +14,7 @@ #include <pthread.h> #include <signal.h> +#include "../util/mutex.h" #include "../util/stat.h" #include <subcmd/parse-options.h> #include <linux/compiler.h> @@ -34,8 +35,8 @@ static u_int32_t futex1 = 0; static pthread_t *worker; static bool done = false; -static pthread_mutex_t thread_lock; -static pthread_cond_t thread_parent, thread_worker; +static struct mutex thread_lock; +static struct cond thread_parent, thread_worker; static struct stats waketime_stats, wakeup_stats; static unsigned int threads_starting; static int futex_flag = 0; @@ -65,12 +66,12 @@ static const char * const bench_futex_wake_usage[] = { static void *workerfn(void *arg __maybe_unused) { - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); threads_starting--; if (!threads_starting) - pthread_cond_signal(&thread_parent); - pthread_cond_wait(&thread_worker, &thread_lock); - pthread_mutex_unlock(&thread_lock); + cond_signal(&thread_parent); + cond_wait(&thread_worker, &thread_lock); + mutex_unlock(&thread_lock); while (1) { if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR) @@ -178,9 +179,9 @@ int bench_futex_wake(int argc, const char **argv) init_stats(&wakeup_stats); init_stats(&waketime_stats); pthread_attr_init(&thread_attr); - pthread_mutex_init(&thread_lock, NULL); - pthread_cond_init(&thread_parent, NULL); - pthread_cond_init(&thread_worker, NULL); + mutex_init(&thread_lock); + cond_init(&thread_parent); + cond_init(&thread_worker); for (j = 0; j < bench_repeat && !done; j++) { unsigned int nwoken = 0; @@ -190,11 +191,11 @@ int bench_futex_wake(int argc, const char **argv) block_threads(worker, thread_attr, cpu); /* make sure all threads are already blocked */ - pthread_mutex_lock(&thread_lock); + mutex_lock(&thread_lock); while (threads_starting) - pthread_cond_wait(&thread_parent, &thread_lock); - pthread_cond_broadcast(&thread_worker); - pthread_mutex_unlock(&thread_lock); + cond_wait(&thread_parent, &thread_lock); + cond_broadcast(&thread_worker); + mutex_unlock(&thread_lock); usleep(100000); @@ -224,9 +225,9 @@ int bench_futex_wake(int argc, const char **argv) } /* cleanup & report results */ - pthread_cond_destroy(&thread_parent); - pthread_cond_destroy(&thread_worker); - pthread_mutex_destroy(&thread_lock); + cond_destroy(&thread_parent); + cond_destroy(&thread_worker); + mutex_destroy(&thread_lock); pthread_attr_destroy(&thread_attr); print_summary(); diff --git a/tools/perf/bench/numa.c b/tools/perf/bench/numa.c index 20eed1e53f8092836b35051d392d7bd98b737539..e78dedf9e682c68a09c81d5eb45a3616195d021a 100644 --- a/tools/perf/bench/numa.c +++ b/tools/perf/bench/numa.c @@ -6,8 +6,6 @@ */ #include <inttypes.h> -/* For the CLR_() macros */ -#include <pthread.h> #include <subcmd/parse-options.h> #include "../util/cloexec.h" @@ -35,6 +33,7 @@ #include <linux/zalloc.h> #include "../util/header.h" +#include "../util/mutex.h" #include <numa.h> #include <numaif.h> @@ -67,7 +66,7 @@ struct thread_data { u64 system_time_ns; u64 user_time_ns; double speed_gbs; - pthread_mutex_t *process_lock; + struct mutex *process_lock; }; /* Parameters set by options: */ @@ -137,16 +136,16 @@ struct params { struct global_info { u8 *data; - pthread_mutex_t startup_mutex; - pthread_cond_t startup_cond; + struct mutex startup_mutex; + struct cond startup_cond; int nr_tasks_started; - pthread_mutex_t start_work_mutex; - pthread_cond_t start_work_cond; + struct mutex start_work_mutex; + struct cond start_work_cond; int nr_tasks_working; bool start_work; - pthread_mutex_t stop_work_mutex; + struct mutex stop_work_mutex; u64 bytes_done; struct thread_data *threads; @@ -524,30 +523,6 @@ static void * setup_private_data(ssize_t bytes) return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0, g->p.thp, g->p.init_random); } -/* - * Return a process-shared (global) mutex: - */ -static void init_global_mutex(pthread_mutex_t *mutex) -{ - pthread_mutexattr_t attr; - - pthread_mutexattr_init(&attr); - pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); - pthread_mutex_init(mutex, &attr); -} - -/* - * Return a process-shared (global) condition variable: - */ -static void init_global_cond(pthread_cond_t *cond) -{ - pthread_condattr_t attr; - - pthread_condattr_init(&attr); - pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED); - pthread_cond_init(cond, &attr); -} - static int parse_cpu_list(const char *arg) { p0.cpu_list_str = strdup(arg); @@ -1220,22 +1195,22 @@ static void *worker_thread(void *__tdata) } if (g->p.serialize_startup) { - pthread_mutex_lock(&g->startup_mutex); + mutex_lock(&g->startup_mutex); g->nr_tasks_started++; /* The last thread wakes the main process. */ if (g->nr_tasks_started == g->p.nr_tasks) - pthread_cond_signal(&g->startup_cond); + cond_signal(&g->startup_cond); - pthread_mutex_unlock(&g->startup_mutex); + mutex_unlock(&g->startup_mutex); /* Here we will wait for the main process to start us all at once: */ - pthread_mutex_lock(&g->start_work_mutex); + mutex_lock(&g->start_work_mutex); g->start_work = false; g->nr_tasks_working++; while (!g->start_work) - pthread_cond_wait(&g->start_work_cond, &g->start_work_mutex); + cond_wait(&g->start_work_cond, &g->start_work_mutex); - pthread_mutex_unlock(&g->start_work_mutex); + mutex_unlock(&g->start_work_mutex); } gettimeofday(&start0, NULL); @@ -1254,17 +1229,17 @@ static void *worker_thread(void *__tdata) val += do_work(thread_data, g->p.bytes_thread, 0, 1, l, val); if (g->p.sleep_usecs) { - pthread_mutex_lock(td->process_lock); + mutex_lock(td->process_lock); usleep(g->p.sleep_usecs); - pthread_mutex_unlock(td->process_lock); + mutex_unlock(td->process_lock); } /* * Amount of work to be done under a process-global lock: */ if (g->p.bytes_process_locked) { - pthread_mutex_lock(td->process_lock); + mutex_lock(td->process_lock); val += do_work(process_data, g->p.bytes_process_locked, thread_nr, g->p.nr_threads, l, val); - pthread_mutex_unlock(td->process_lock); + mutex_unlock(td->process_lock); } work_done = g->p.bytes_global + g->p.bytes_process + @@ -1361,9 +1336,9 @@ static void *worker_thread(void *__tdata) free_data(thread_data, g->p.bytes_thread); - pthread_mutex_lock(&g->stop_work_mutex); + mutex_lock(&g->stop_work_mutex); g->bytes_done += bytes_done; - pthread_mutex_unlock(&g->stop_work_mutex); + mutex_unlock(&g->stop_work_mutex); return NULL; } @@ -1373,7 +1348,7 @@ static void *worker_thread(void *__tdata) */ static void worker_process(int process_nr) { - pthread_mutex_t process_lock; + struct mutex process_lock; struct thread_data *td; pthread_t *pthreads; u8 *process_data; @@ -1381,7 +1356,7 @@ static void worker_process(int process_nr) int ret; int t; - pthread_mutex_init(&process_lock, NULL); + mutex_init(&process_lock); set_taskname("process %d", process_nr); /* @@ -1540,11 +1515,11 @@ static int init(void) g->data = setup_shared_data(g->p.bytes_global); /* Startup serialization: */ - init_global_mutex(&g->start_work_mutex); - init_global_cond(&g->start_work_cond); - init_global_mutex(&g->startup_mutex); - init_global_cond(&g->startup_cond); - init_global_mutex(&g->stop_work_mutex); + mutex_init_pshared(&g->start_work_mutex); + cond_init_pshared(&g->start_work_cond); + mutex_init_pshared(&g->startup_mutex); + cond_init_pshared(&g->startup_cond); + mutex_init_pshared(&g->stop_work_mutex); init_thread_data(); @@ -1633,17 +1608,17 @@ static int __bench_numa(const char *name) * Wait for all the threads to start up. The last thread will * signal this process. */ - pthread_mutex_lock(&g->startup_mutex); + mutex_lock(&g->startup_mutex); while (g->nr_tasks_started != g->p.nr_tasks) - pthread_cond_wait(&g->startup_cond, &g->startup_mutex); + cond_wait(&g->startup_cond, &g->startup_mutex); - pthread_mutex_unlock(&g->startup_mutex); + mutex_unlock(&g->startup_mutex); /* Wait for all threads to be at the start_work_cond. */ while (!threads_ready) { - pthread_mutex_lock(&g->start_work_mutex); + mutex_lock(&g->start_work_mutex); threads_ready = (g->nr_tasks_working == g->p.nr_tasks); - pthread_mutex_unlock(&g->start_work_mutex); + mutex_unlock(&g->start_work_mutex); if (!threads_ready) usleep(1); } @@ -1661,10 +1636,10 @@ static int __bench_numa(const char *name) start = stop; /* Start all threads running. */ - pthread_mutex_lock(&g->start_work_mutex); + mutex_lock(&g->start_work_mutex); g->start_work = true; - pthread_mutex_unlock(&g->start_work_mutex); - pthread_cond_broadcast(&g->start_work_cond); + mutex_unlock(&g->start_work_mutex); + cond_broadcast(&g->start_work_cond); } else { gettimeofday(&start, NULL); } diff --git a/tools/perf/builtin-c2c.c b/tools/perf/builtin-c2c.c index 653e13b5037ec074fa4e51a0df367b64c6d40ae5..a9190458d2d50015cbe997ed9cab63b8b51d984a 100644 --- a/tools/perf/builtin-c2c.c +++ b/tools/perf/builtin-c2c.c @@ -146,15 +146,15 @@ static void *c2c_he_zalloc(size_t size) c2c_he->cpuset = bitmap_zalloc(c2c.cpus_cnt); if (!c2c_he->cpuset) - return NULL; + goto out_free; c2c_he->nodeset = bitmap_zalloc(c2c.nodes_cnt); if (!c2c_he->nodeset) - return NULL; + goto out_free; c2c_he->node_stats = zalloc(c2c.nodes_cnt * sizeof(*c2c_he->node_stats)); if (!c2c_he->node_stats) - return NULL; + goto out_free; init_stats(&c2c_he->cstats.lcl_hitm); init_stats(&c2c_he->cstats.rmt_hitm); @@ -163,6 +163,12 @@ static void *c2c_he_zalloc(size_t size) init_stats(&c2c_he->cstats.load); return &c2c_he->he; + +out_free: + free(c2c_he->nodeset); + free(c2c_he->cpuset); + free(c2c_he); + return NULL; } static void c2c_he_free(void *he) @@ -673,28 +679,35 @@ STAT_FN(ld_l2hit) STAT_FN(ld_llchit) STAT_FN(rmt_hit) -static uint64_t total_records(struct c2c_stats *stats) +static uint64_t get_load_llc_misses(struct c2c_stats *stats) { - uint64_t lclmiss, ldcnt, total; - - lclmiss = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hitm + - stats->rmt_hit; + return stats->lcl_dram + + stats->rmt_dram + + stats->rmt_hitm + + stats->rmt_hit; +} - ldcnt = lclmiss + - stats->ld_fbhit + - stats->ld_l1hit + - stats->ld_l2hit + - stats->ld_llchit + - stats->lcl_hitm; +static uint64_t get_load_cache_hits(struct c2c_stats *stats) +{ + return stats->ld_fbhit + + stats->ld_l1hit + + stats->ld_l2hit + + stats->ld_llchit + + stats->lcl_hitm; +} - total = ldcnt + - stats->st_l1hit + - stats->st_l1miss + - stats->st_na; +static uint64_t get_stores(struct c2c_stats *stats) +{ + return stats->st_l1hit + + stats->st_l1miss + + stats->st_na; +} - return total; +static uint64_t total_records(struct c2c_stats *stats) +{ + return get_load_llc_misses(stats) + + get_load_cache_hits(stats) + + get_stores(stats); } static int @@ -731,21 +744,8 @@ tot_recs_cmp(struct perf_hpp_fmt *fmt __maybe_unused, static uint64_t total_loads(struct c2c_stats *stats) { - uint64_t lclmiss, ldcnt; - - lclmiss = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hitm + - stats->rmt_hit; - - ldcnt = lclmiss + - stats->ld_fbhit + - stats->ld_l1hit + - stats->ld_l2hit + - stats->ld_llchit + - stats->lcl_hitm; - - return ldcnt; + return get_load_llc_misses(stats) + + get_load_cache_hits(stats); } static int @@ -2370,10 +2370,7 @@ static void print_c2c__display_stats(FILE *out) int llc_misses; struct c2c_stats *stats = &c2c.hists.stats; - llc_misses = stats->lcl_dram + - stats->rmt_dram + - stats->rmt_hit + - stats->rmt_hitm; + llc_misses = get_load_llc_misses(stats); fprintf(out, "=================================================\n"); fprintf(out, " Trace Event Information \n"); @@ -3284,6 +3281,7 @@ static int perf_c2c__record(int argc, const char **argv) */ if (e->tag) { e->record = true; + rec_argv[i++] = "-W"; } else { e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); e->record = true; diff --git a/tools/perf/builtin-inject.c b/tools/perf/builtin-inject.c index 2a0f992ca0be75a1cb8752d634ab5c0721c60e29..e254f18986f7cfe3cda758fd693cc0fd974da552 100644 --- a/tools/perf/builtin-inject.c +++ b/tools/perf/builtin-inject.c @@ -21,6 +21,7 @@ #include "util/data.h" #include "util/auxtrace.h" #include "util/jit.h" +#include "util/string2.h" #include "util/symbol.h" #include "util/synthetic-events.h" #include "util/thread.h" @@ -38,6 +39,7 @@ #include <linux/string.h> #include <linux/zalloc.h> #include <linux/hash.h> +#include <ctype.h> #include <errno.h> #include <signal.h> #include <inttypes.h> @@ -123,6 +125,7 @@ struct perf_inject { char event_copy[PERF_SAMPLE_MAX_SIZE]; struct perf_file_section secs[HEADER_FEAT_BITS]; struct guest_session guest_session; + struct strlist *known_build_ids; }; struct event_entry { @@ -433,8 +436,10 @@ static struct dso *findnew_dso(int pid, int tid, const char *filename, } if (dso) { + mutex_lock(&dso->lock); nsinfo__put(dso->nsinfo); dso->nsinfo = nsi; + mutex_unlock(&dso->lock); } else nsinfo__put(nsi); @@ -617,6 +622,7 @@ static int dso__read_build_id(struct dso *dso) if (dso->has_build_id) return 0; + mutex_lock(&dso->lock); nsinfo__mountns_enter(dso->nsinfo, &nsc); if (filename__read_build_id(dso->long_name, &dso->bid) > 0) dso->has_build_id = true; @@ -630,13 +636,78 @@ static int dso__read_build_id(struct dso *dso) free(new_name); } nsinfo__mountns_exit(&nsc); + mutex_unlock(&dso->lock); return dso->has_build_id ? 0 : -1; } +static struct strlist *perf_inject__parse_known_build_ids( + const char *known_build_ids_string) +{ + struct str_node *pos, *tmp; + struct strlist *known_build_ids; + int bid_len; + + known_build_ids = strlist__new(known_build_ids_string, NULL); + if (known_build_ids == NULL) + return NULL; + strlist__for_each_entry_safe(pos, tmp, known_build_ids) { + const char *build_id, *dso_name; + + build_id = skip_spaces(pos->s); + dso_name = strchr(build_id, ' '); + if (dso_name == NULL) { + strlist__remove(known_build_ids, pos); + continue; + } + bid_len = dso_name - pos->s; + dso_name = skip_spaces(dso_name); + if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) { + strlist__remove(known_build_ids, pos); + continue; + } + for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) { + if (!isxdigit(build_id[2 * ix]) || + !isxdigit(build_id[2 * ix + 1])) { + strlist__remove(known_build_ids, pos); + break; + } + } + } + return known_build_ids; +} + +static bool perf_inject__lookup_known_build_id(struct perf_inject *inject, + struct dso *dso) +{ + struct str_node *pos; + int bid_len; + + strlist__for_each_entry(pos, inject->known_build_ids) { + const char *build_id, *dso_name; + + build_id = skip_spaces(pos->s); + dso_name = strchr(build_id, ' '); + bid_len = dso_name - pos->s; + dso_name = skip_spaces(dso_name); + if (strcmp(dso->long_name, dso_name)) + continue; + for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) { + dso->bid.data[ix] = (hex(build_id[2 * ix]) << 4 | + hex(build_id[2 * ix + 1])); + } + dso->bid.size = bid_len / 2; + dso->has_build_id = 1; + return true; + } + return false; +} + static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool, struct machine *machine, u8 cpumode, u32 flags) { + struct perf_inject *inject = container_of(tool, struct perf_inject, + tool); int err; if (is_anon_memory(dso->long_name) || flags & MAP_HUGETLB) @@ -644,6 +715,10 @@ static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool, if (is_no_dso_memory(dso->long_name)) return 0; + if (inject->known_build_ids != NULL && + perf_inject__lookup_known_build_id(inject, dso)) + return 1; + if (dso__read_build_id(dso) < 0) { pr_debug("no build_id found for %s\n", dso->long_name); return -1; @@ -2112,12 +2187,16 @@ int cmd_inject(int argc, const char **argv) }; int ret; bool repipe = true; + const char *known_build_ids = NULL; struct option options[] = { OPT_BOOLEAN('b', "build-ids", &inject.build_ids, "Inject build-ids into the output stream"), OPT_BOOLEAN(0, "buildid-all", &inject.build_id_all, "Inject build-ids of all DSOs into the output stream"), + OPT_STRING(0, "known-build-ids", &known_build_ids, + "buildid path [,buildid path...]", + "build-ids to use for given paths"), OPT_STRING('i', "input", &inject.input_name, "file", "input file name"), OPT_STRING('o', "output", &inject.output.path, "file", @@ -2257,6 +2336,15 @@ int cmd_inject(int argc, const char **argv) */ inject.tool.ordered_events = true; inject.tool.ordering_requires_timestamps = true; + if (known_build_ids != NULL) { + inject.known_build_ids = + perf_inject__parse_known_build_ids(known_build_ids); + + if (inject.known_build_ids == NULL) { + pr_err("Couldn't parse known build ids.\n"); + goto out_delete; + } + } } if (inject.sched_stat) { @@ -2285,6 +2373,7 @@ int cmd_inject(int argc, const char **argv) guest_session__exit(&inject.guest_session); out_delete: + strlist__delete(inject.known_build_ids); zstd_fini(&(inject.session->zstd_data)); perf_session__delete(inject.session); out_close_output: diff --git a/tools/perf/builtin-list.c b/tools/perf/builtin-list.c index 744dd35205847a97c75315c30dfd9f3e28e00af3..58e1ec1654ef451f9c6c738181cd3711f5416484 100644 --- a/tools/perf/builtin-list.c +++ b/tools/perf/builtin-list.c @@ -60,7 +60,7 @@ int cmd_list(int argc, const char **argv) setup_pager(); if (!raw_dump && pager_in_use()) - printf("\nList of pre-defined events (to be used in -e):\n\n"); + printf("\nList of pre-defined events (to be used in -e or -M):\n\n"); if (hybrid_type) { pmu_name = perf_pmu__hybrid_type_to_pmu(hybrid_type); diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index dd11d3471baf3b1f87512681eabc6f6e24f14d98..9722d4ab2e55732e248abab72cc0c403d11ac4b9 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -28,7 +28,6 @@ #include <sys/types.h> #include <sys/prctl.h> #include <semaphore.h> -#include <pthread.h> #include <math.h> #include <limits.h> @@ -57,6 +56,9 @@ static bool combine_locks; static bool show_thread_stats; static bool use_bpf; static unsigned long bpf_map_entries = 10240; +static int max_stack_depth = CONTENTION_STACK_DEPTH; +static int stack_skip = CONTENTION_STACK_SKIP; +static int print_nr_entries = INT_MAX / 2; static enum { LOCK_AGGR_ADDR, @@ -561,29 +563,50 @@ enum acquire_flags { READ_LOCK = 2, }; -static int report_lock_acquire_event(struct evsel *evsel, - struct perf_sample *sample) +static int get_key_by_aggr_mode_simple(u64 *key, u64 addr, u32 tid) { - struct lock_stat *ls; - struct thread_stat *ts; - struct lock_seq_stat *seq; - const char *name = evsel__strval(evsel, sample, "name"); - u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); - int flag = evsel__intval(evsel, sample, "flags"); - u64 key; - switch (aggr_mode) { case LOCK_AGGR_ADDR: - key = addr; + *key = addr; break; case LOCK_AGGR_TASK: - key = sample->tid; + *key = tid; break; case LOCK_AGGR_CALLER: default: pr_err("Invalid aggregation mode: %d\n", aggr_mode); return -EINVAL; } + return 0; +} + +static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample); + +static int get_key_by_aggr_mode(u64 *key, u64 addr, struct evsel *evsel, + struct perf_sample *sample) +{ + if (aggr_mode == LOCK_AGGR_CALLER) { + *key = callchain_id(evsel, sample); + return 0; + } + return get_key_by_aggr_mode_simple(key, addr, sample->tid); +} + +static int report_lock_acquire_event(struct evsel *evsel, + struct perf_sample *sample) +{ + struct lock_stat *ls; + struct thread_stat *ts; + struct lock_seq_stat *seq; + const char *name = evsel__strval(evsel, sample, "name"); + u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); + int flag = evsel__intval(evsel, sample, "flags"); + u64 key; + int ret; + + ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid); + if (ret < 0) + return ret; ls = lock_stat_findnew(key, name, 0); if (!ls) @@ -654,19 +677,11 @@ static int report_lock_acquired_event(struct evsel *evsel, const char *name = evsel__strval(evsel, sample, "name"); u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); u64 key; + int ret; - switch (aggr_mode) { - case LOCK_AGGR_ADDR: - key = addr; - break; - case LOCK_AGGR_TASK: - key = sample->tid; - break; - case LOCK_AGGR_CALLER: - default: - pr_err("Invalid aggregation mode: %d\n", aggr_mode); - return -EINVAL; - } + ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid); + if (ret < 0) + return ret; ls = lock_stat_findnew(key, name, 0); if (!ls) @@ -727,19 +742,11 @@ static int report_lock_contended_event(struct evsel *evsel, const char *name = evsel__strval(evsel, sample, "name"); u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); u64 key; + int ret; - switch (aggr_mode) { - case LOCK_AGGR_ADDR: - key = addr; - break; - case LOCK_AGGR_TASK: - key = sample->tid; - break; - case LOCK_AGGR_CALLER: - default: - pr_err("Invalid aggregation mode: %d\n", aggr_mode); - return -EINVAL; - } + ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid); + if (ret < 0) + return ret; ls = lock_stat_findnew(key, name, 0); if (!ls) @@ -793,19 +800,11 @@ static int report_lock_release_event(struct evsel *evsel, const char *name = evsel__strval(evsel, sample, "name"); u64 addr = evsel__intval(evsel, sample, "lockdep_addr"); u64 key; + int ret; - switch (aggr_mode) { - case LOCK_AGGR_ADDR: - key = addr; - break; - case LOCK_AGGR_TASK: - key = sample->tid; - break; - case LOCK_AGGR_CALLER: - default: - pr_err("Invalid aggregation mode: %d\n", aggr_mode); - return -EINVAL; - } + ret = get_key_by_aggr_mode_simple(&key, addr, sample->tid); + if (ret < 0) + return ret; ls = lock_stat_findnew(key, name, 0); if (!ls) @@ -903,6 +902,23 @@ bool is_lock_function(struct machine *machine, u64 addr) return false; } +static int get_symbol_name_offset(struct map *map, struct symbol *sym, u64 ip, + char *buf, int size) +{ + u64 offset; + + if (map == NULL || sym == NULL) { + buf[0] = '\0'; + return 0; + } + + offset = map->map_ip(map, ip) - sym->start; + + if (offset) + return scnprintf(buf, size, "%s+%#lx", sym->name, offset); + else + return strlcpy(buf, sym->name, size); +} static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sample, char *buf, int size) { @@ -923,7 +939,7 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl /* use caller function name from the callchain */ ret = thread__resolve_callchain(thread, cursor, evsel, sample, - NULL, NULL, CONTENTION_STACK_DEPTH); + NULL, NULL, max_stack_depth); if (ret != 0) { thread__put(thread); return -1; @@ -940,20 +956,13 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl break; /* skip first few entries - for lock functions */ - if (++skip <= CONTENTION_STACK_SKIP) + if (++skip <= stack_skip) goto next; sym = node->ms.sym; if (sym && !is_lock_function(machine, node->ip)) { - struct map *map = node->ms.map; - u64 offset; - - offset = map->map_ip(map, node->ip) - sym->start; - - if (offset) - scnprintf(buf, size, "%s+%#lx", sym->name, offset); - else - strlcpy(buf, sym->name, size); + get_symbol_name_offset(node->ms.map, sym, node->ip, + buf, size); return 0; } @@ -978,7 +987,7 @@ static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample) /* use caller function name from the callchain */ ret = thread__resolve_callchain(thread, cursor, evsel, sample, - NULL, NULL, CONTENTION_STACK_DEPTH); + NULL, NULL, max_stack_depth); thread__put(thread); if (ret != 0) @@ -994,7 +1003,7 @@ static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample) break; /* skip first few entries - for lock functions */ - if (++skip <= CONTENTION_STACK_SKIP) + if (++skip <= stack_skip) goto next; if (node->ms.sym && is_lock_function(machine, node->ip)) @@ -1008,6 +1017,27 @@ next: return hash; } +static u64 *get_callstack(struct perf_sample *sample, int max_stack) +{ + u64 *callstack; + u64 i; + int c; + + callstack = calloc(max_stack, sizeof(*callstack)); + if (callstack == NULL) + return NULL; + + for (i = 0, c = 0; i < sample->callchain->nr && c < max_stack; i++) { + u64 ip = sample->callchain->ips[i]; + + if (ip >= PERF_CONTEXT_MAX) + continue; + + callstack[c++] = ip; + } + return callstack; +} + static int report_lock_contention_begin_event(struct evsel *evsel, struct perf_sample *sample) { @@ -1016,21 +1046,11 @@ static int report_lock_contention_begin_event(struct evsel *evsel, struct lock_seq_stat *seq; u64 addr = evsel__intval(evsel, sample, "lock_addr"); u64 key; + int ret; - switch (aggr_mode) { - case LOCK_AGGR_ADDR: - key = addr; - break; - case LOCK_AGGR_TASK: - key = sample->tid; - break; - case LOCK_AGGR_CALLER: - key = callchain_id(evsel, sample); - break; - default: - pr_err("Invalid aggregation mode: %d\n", aggr_mode); - return -EINVAL; - } + ret = get_key_by_aggr_mode(&key, addr, evsel, sample); + if (ret < 0) + return ret; ls = lock_stat_find(key); if (!ls) { @@ -1044,6 +1064,12 @@ static int report_lock_contention_begin_event(struct evsel *evsel, ls = lock_stat_findnew(key, caller, flags); if (!ls) return -ENOMEM; + + if (aggr_mode == LOCK_AGGR_CALLER && verbose) { + ls->callstack = get_callstack(sample, max_stack_depth); + if (ls->callstack == NULL) + return -ENOMEM; + } } ts = thread_stat_findnew(sample->tid); @@ -1099,21 +1125,11 @@ static int report_lock_contention_end_event(struct evsel *evsel, u64 contended_term; u64 addr = evsel__intval(evsel, sample, "lock_addr"); u64 key; + int ret; - switch (aggr_mode) { - case LOCK_AGGR_ADDR: - key = addr; - break; - case LOCK_AGGR_TASK: - key = sample->tid; - break; - case LOCK_AGGR_CALLER: - key = callchain_id(evsel, sample); - break; - default: - pr_err("Invalid aggregation mode: %d\n", aggr_mode); - return -EINVAL; - } + ret = get_key_by_aggr_mode(&key, addr, evsel, sample); + if (ret < 0) + return ret; ls = lock_stat_find(key); if (!ls) @@ -1234,7 +1250,7 @@ static void print_bad_events(int bad, int total) for (i = 0; i < BROKEN_MAX; i++) broken += bad_hist[i]; - if (broken == 0 && !verbose) + if (quiet || (broken == 0 && !verbose)) return; pr_info("\n=== output for debug===\n\n"); @@ -1251,14 +1267,16 @@ static void print_result(void) struct lock_stat *st; struct lock_key *key; char cut_name[20]; - int bad, total; + int bad, total, printed; - pr_info("%20s ", "Name"); - list_for_each_entry(key, &lock_keys, list) - pr_info("%*s ", key->len, key->header); - pr_info("\n\n"); + if (!quiet) { + pr_info("%20s ", "Name"); + list_for_each_entry(key, &lock_keys, list) + pr_info("%*s ", key->len, key->header); + pr_info("\n\n"); + } - bad = total = 0; + bad = total = printed = 0; while ((st = pop_from_result())) { total++; if (st->broken) @@ -1296,6 +1314,9 @@ static void print_result(void) pr_info(" "); } pr_info("\n"); + + if (++printed >= print_nr_entries) + break; } print_bad_events(bad, total); @@ -1457,21 +1478,23 @@ static void sort_contention_result(void) sort_result(); } -static void print_contention_result(void) +static void print_contention_result(struct lock_contention *con) { struct lock_stat *st; struct lock_key *key; - int bad, total; + int bad, total, printed; - list_for_each_entry(key, &lock_keys, list) - pr_info("%*s ", key->len, key->header); + if (!quiet) { + list_for_each_entry(key, &lock_keys, list) + pr_info("%*s ", key->len, key->header); - if (show_thread_stats) - pr_info(" %10s %s\n\n", "pid", "comm"); - else - pr_info(" %10s %s\n\n", "type", "caller"); + if (show_thread_stats) + pr_info(" %10s %s\n\n", "pid", "comm"); + else + pr_info(" %10s %s\n\n", "type", "caller"); + } - bad = total = 0; + bad = total = printed = 0; if (use_bpf) bad = bad_hist[BROKEN_CONTENDED]; @@ -1492,10 +1515,30 @@ static void print_contention_result(void) /* st->addr contains tid of thread */ t = perf_session__findnew(session, pid); pr_info(" %10d %s\n", pid, thread__comm_str(t)); - continue; + goto next; } pr_info(" %10s %s\n", get_type_str(st), st->name); + if (verbose) { + struct map *kmap; + struct symbol *sym; + char buf[128]; + u64 ip; + + for (int i = 0; i < max_stack_depth; i++) { + if (!st->callstack || !st->callstack[i]) + break; + + ip = st->callstack[i]; + sym = machine__find_kernel_symbol(con->machine, ip, &kmap); + get_symbol_name_offset(kmap, sym, ip, buf, sizeof(buf)); + pr_info("\t\t\t%#lx %s\n", (unsigned long)ip, buf); + } + } + +next: + if (++printed >= print_nr_entries) + break; } print_bad_events(bad, total); @@ -1603,6 +1646,8 @@ static int __cmd_contention(int argc, const char **argv) .target = &target, .result = &lockhash_table[0], .map_nr_entries = bpf_map_entries, + .max_stack = max_stack_depth, + .stack_skip = stack_skip, }; session = perf_session__new(use_bpf ? NULL : &data, &eops); @@ -1611,6 +1656,8 @@ static int __cmd_contention(int argc, const char **argv) return PTR_ERR(session); } + con.machine = &session->machines.host; + /* for lock function check */ symbol_conf.sort_by_name = true; symbol__init(&session->header.env); @@ -1629,8 +1676,6 @@ static int __cmd_contention(int argc, const char **argv) signal(SIGCHLD, sighandler); signal(SIGTERM, sighandler); - con.machine = &session->machines.host; - con.evlist = evlist__new(); if (con.evlist == NULL) { err = -ENOMEM; @@ -1702,7 +1747,7 @@ static int __cmd_contention(int argc, const char **argv) setup_pager(); sort_contention_result(); - print_contention_result(); + print_contention_result(&con); out_delete: evlist__delete(con.evlist); @@ -1824,6 +1869,7 @@ int cmd_lock(int argc, const char **argv) "file", "vmlinux pathname"), OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file", "kallsyms pathname"), + OPT_BOOLEAN('q', "quiet", &quiet, "Do not show any message"), OPT_END() }; @@ -1845,6 +1891,7 @@ int cmd_lock(int argc, const char **argv) "combine locks in the same class"), OPT_BOOLEAN('t', "threads", &show_thread_stats, "show per-thread lock stats"), + OPT_INTEGER('E', "entries", &print_nr_entries, "display this many functions"), OPT_PARENT(lock_options) }; @@ -1866,6 +1913,13 @@ int cmd_lock(int argc, const char **argv) "Trace on existing thread id (exclusive to --pid)"), OPT_CALLBACK(0, "map-nr-entries", &bpf_map_entries, "num", "Max number of BPF map entries", parse_map_entry), + OPT_INTEGER(0, "max-stack", &max_stack_depth, + "Set the maximum stack depth when collecting lock contention, " + "Default: " __stringify(CONTENTION_STACK_DEPTH)), + OPT_INTEGER(0, "stack-skip", &stack_skip, + "Set the number of stack depth to skip when finding a lock caller, " + "Default: " __stringify(CONTENTION_STACK_SKIP)), + OPT_INTEGER('E', "entries", &print_nr_entries, "display this many functions"), OPT_PARENT(lock_options) }; @@ -1874,8 +1928,7 @@ int cmd_lock(int argc, const char **argv) NULL }; const char *const lock_subcommands[] = { "record", "report", "script", - "info", "contention", - "contention", NULL }; + "info", "contention", NULL }; const char *lock_usage[] = { NULL, NULL diff --git a/tools/perf/builtin-mem.c b/tools/perf/builtin-mem.c index 9e435fd2350326a628f9311f85be3cb33871e5f2..923fb8316fdae832941630da7d50e96395f0f7ee 100644 --- a/tools/perf/builtin-mem.c +++ b/tools/perf/builtin-mem.c @@ -97,6 +97,9 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) else rec_argc = argc + 9 * perf_pmu__hybrid_pmu_num(); + if (mem->cpu_list) + rec_argc += 2; + rec_argv = calloc(rec_argc + 1, sizeof(char *)); if (!rec_argv) return -1; @@ -122,6 +125,7 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) (mem->operation & MEM_OPERATION_LOAD) && (mem->operation & MEM_OPERATION_STORE)) { e->record = true; + rec_argv[i++] = "-W"; } else { if (mem->operation & MEM_OPERATION_LOAD) { e = perf_mem_events__ptr(PERF_MEM_EVENTS__LOAD); @@ -158,6 +162,11 @@ static int __cmd_record(int argc, const char **argv, struct perf_mem *mem) if (all_kernel) rec_argv[i++] = "--all-kernel"; + if (mem->cpu_list) { + rec_argv[i++] = "-C"; + rec_argv[i++] = mem->cpu_list; + } + for (j = 0; j < argc; j++, i++) rec_argv[i] = argv[j]; diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 4713f0f3a6cf15add5416e3ad5fc2f6e919f2a89..52d254b1530c98be0855e1b1d4adb81748641b95 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -10,6 +10,7 @@ #include "util/build-id.h" #include <subcmd/parse-options.h> +#include <internal/xyarray.h> #include "util/parse-events.h" #include "util/config.h" @@ -21,6 +22,7 @@ #include "util/evsel.h" #include "util/debug.h" #include "util/mmap.h" +#include "util/mutex.h" #include "util/target.h" #include "util/session.h" #include "util/tool.h" @@ -143,6 +145,11 @@ static const char *thread_spec_tags[THREAD_SPEC__MAX] = { "undefined", "cpu", "core", "package", "numa", "user" }; +struct pollfd_index_map { + int evlist_pollfd_index; + int thread_pollfd_index; +}; + struct record { struct perf_tool tool; struct record_opts opts; @@ -171,6 +178,9 @@ struct record { int nr_threads; struct thread_mask *thread_masks; struct record_thread *thread_data; + struct pollfd_index_map *index_map; + size_t index_map_sz; + size_t index_map_cnt; }; static volatile int done; @@ -608,17 +618,18 @@ static int process_synthesized_event(struct perf_tool *tool, return record__write(rec, NULL, event, event->header.size); } +static struct mutex synth_lock; + static int process_locked_synthesized_event(struct perf_tool *tool, union perf_event *event, struct perf_sample *sample __maybe_unused, struct machine *machine __maybe_unused) { - static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER; int ret; - pthread_mutex_lock(&synth_lock); + mutex_lock(&synth_lock); ret = process_synthesized_event(tool, event, sample, machine); - pthread_mutex_unlock(&synth_lock); + mutex_unlock(&synth_lock); return ret; } @@ -1074,6 +1085,70 @@ static void record__free_thread_data(struct record *rec) zfree(&rec->thread_data); } +static int record__map_thread_evlist_pollfd_indexes(struct record *rec, + int evlist_pollfd_index, + int thread_pollfd_index) +{ + size_t x = rec->index_map_cnt; + + if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL)) + return -ENOMEM; + rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index; + rec->index_map[x].thread_pollfd_index = thread_pollfd_index; + rec->index_map_cnt += 1; + return 0; +} + +static int record__update_evlist_pollfd_from_thread(struct record *rec, + struct evlist *evlist, + struct record_thread *thread_data) +{ + struct pollfd *e_entries = evlist->core.pollfd.entries; + struct pollfd *t_entries = thread_data->pollfd.entries; + int err = 0; + size_t i; + + for (i = 0; i < rec->index_map_cnt; i++) { + int e_pos = rec->index_map[i].evlist_pollfd_index; + int t_pos = rec->index_map[i].thread_pollfd_index; + + if (e_entries[e_pos].fd != t_entries[t_pos].fd || + e_entries[e_pos].events != t_entries[t_pos].events) { + pr_err("Thread and evlist pollfd index mismatch\n"); + err = -EINVAL; + continue; + } + e_entries[e_pos].revents = t_entries[t_pos].revents; + } + return err; +} + +static int record__dup_non_perf_events(struct record *rec, + struct evlist *evlist, + struct record_thread *thread_data) +{ + struct fdarray *fda = &evlist->core.pollfd; + int i, ret; + + for (i = 0; i < fda->nr; i++) { + if (!(fda->priv[i].flags & fdarray_flag__non_perf_event)) + continue; + ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda); + if (ret < 0) { + pr_err("Failed to duplicate descriptor in main thread pollfd\n"); + return ret; + } + pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n", + thread_data, ret, fda->entries[i].fd); + ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret); + if (ret < 0) { + pr_err("Failed to map thread and evlist pollfd indexes\n"); + return ret; + } + } + return 0; +} + static int record__alloc_thread_data(struct record *rec, struct evlist *evlist) { int t, ret; @@ -1121,18 +1196,12 @@ static int record__alloc_thread_data(struct record *rec, struct evlist *evlist) thread_data[t].pipes.msg[0]); } else { thread_data[t].tid = gettid(); - if (evlist->ctl_fd.pos == -1) - continue; - ret = fdarray__dup_entry_from(&thread_data[t].pollfd, evlist->ctl_fd.pos, - &evlist->core.pollfd); - if (ret < 0) { - pr_err("Failed to duplicate descriptor in main thread pollfd\n"); + + ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]); + if (ret < 0) goto out_free; - } - thread_data[t].ctlfd_pos = ret; - pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n", - thread_data, thread_data[t].ctlfd_pos, - evlist->core.pollfd.entries[evlist->ctl_fd.pos].fd); + + thread_data[t].ctlfd_pos = -1; /* Not used */ } } @@ -1784,6 +1853,74 @@ record__switch_output(struct record *rec, bool at_exit) return fd; } +static void __record__read_lost_samples(struct record *rec, struct evsel *evsel, + struct perf_record_lost_samples *lost, + int cpu_idx, int thread_idx) +{ + struct perf_counts_values count; + struct perf_sample_id *sid; + struct perf_sample sample = {}; + int id_hdr_size; + + if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) { + pr_err("read LOST count failed\n"); + return; + } + + if (count.lost == 0) + return; + + lost->lost = count.lost; + if (evsel->core.ids) { + sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx); + sample.id = sid->id; + } + + id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1), + evsel->core.attr.sample_type, &sample); + lost->header.size = sizeof(*lost) + id_hdr_size; + record__write(rec, NULL, lost, lost->header.size); +} + +static void record__read_lost_samples(struct record *rec) +{ + struct perf_session *session = rec->session; + struct perf_record_lost_samples *lost; + struct evsel *evsel; + + /* there was an error during record__open */ + if (session->evlist == NULL) + return; + + lost = zalloc(PERF_SAMPLE_MAX_SIZE); + if (lost == NULL) { + pr_debug("Memory allocation failed\n"); + return; + } + + lost->header.type = PERF_RECORD_LOST_SAMPLES; + + evlist__for_each_entry(session->evlist, evsel) { + struct xyarray *xy = evsel->core.sample_id; + + if (xy == NULL || evsel->core.fd == NULL) + continue; + if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) || + xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) { + pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n"); + continue; + } + + for (int x = 0; x < xyarray__max_x(xy); x++) { + for (int y = 0; y < xyarray__max_y(xy); y++) { + __record__read_lost_samples(rec, evsel, lost, x, y); + } + } + } + free(lost); + +} + static volatile int workload_exec_errno; /* @@ -1906,17 +2043,22 @@ static int record__synthesize(struct record *rec, bool tail) err = perf_event__synthesize_bpf_events(session, process_synthesized_event, machine, opts); - if (err < 0) + if (err < 0) { pr_warning("Couldn't synthesize bpf events.\n"); + err = 0; + } if (rec->opts.synth & PERF_SYNTH_CGROUP) { err = perf_event__synthesize_cgroups(tool, process_synthesized_event, machine); - if (err < 0) + if (err < 0) { pr_warning("Couldn't synthesize cgroup events.\n"); + err = 0; + } } if (rec->opts.nr_threads_synthesize > 1) { + mutex_init(&synth_lock); perf_set_multithreaded(); f = process_locked_synthesized_event; } @@ -1930,8 +2072,10 @@ static int record__synthesize(struct record *rec, bool tail) rec->opts.nr_threads_synthesize); } - if (rec->opts.nr_threads_synthesize > 1) + if (rec->opts.nr_threads_synthesize > 1) { perf_set_singlethreaded(); + mutex_destroy(&synth_lock); + } out: return err; @@ -2290,10 +2434,14 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) record__uniquify_name(rec); + /* Debug message used by test scripts */ + pr_debug3("perf record opening and mmapping events\n"); if (record__open(rec) != 0) { err = -1; goto out_free_threads; } + /* Debug message used by test scripts */ + pr_debug3("perf record done opening and mmapping events\n"); session->header.env.comp_mmap_len = session->evlist->core.mmap_len; if (rec->opts.kcore) { @@ -2432,6 +2580,14 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } + err = event_enable_timer__start(rec->evlist->eet); + if (err) + goto out_child; + + /* Debug message used by test scripts */ + pr_debug3("perf record has started\n"); + fflush(stderr); + trigger_ready(&auxtrace_snapshot_trigger); trigger_ready(&switch_output_trigger); perf_hooks__invoke_record_start(); @@ -2530,8 +2686,9 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) record__thread_munmap_filtered, NULL) == 0) draining = true; - evlist__ctlfd_update(rec->evlist, - &thread->pollfd.entries[thread->ctlfd_pos]); + err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread); + if (err) + goto out_child; } if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) { @@ -2554,6 +2711,14 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) } } + err = event_enable_timer__process(rec->evlist->eet); + if (err < 0) + goto out_child; + if (err) { + err = 0; + done = 1; + } + /* * When perf is starting the traced process, at the end events * die with the process and we wait for that. Thus no need to @@ -2626,6 +2791,7 @@ out_free_threads: if (rec->off_cpu) rec->bytes_written += off_cpu_write(rec->session); + record__read_lost_samples(rec); record__synthesize(rec, true); /* this will be recalculated during process_buildids() */ rec->samples = 0; @@ -2775,6 +2941,12 @@ static int perf_record_config(const char *var, const char *value, void *cb) return 0; } +static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset) +{ + struct record *rec = (struct record *)opt->value; + + return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset); +} static int record__parse_affinity(const struct option *opt, const char *str, int unset) { @@ -3236,8 +3408,10 @@ static struct option __record_options[] = { OPT_CALLBACK('G', "cgroup", &record.evlist, "name", "monitor event in cgroup name only", parse_cgroups), - OPT_INTEGER('D', "delay", &record.opts.initial_delay, - "ms to wait before starting measurement after program start (-1: start with events disabled)"), + OPT_CALLBACK('D', "delay", &record, "ms", + "ms to wait before starting measurement after program start (-1: start with events disabled), " + "or ranges of time to enable events e.g. '-D 10-20,30-40'", + record__parse_event_enable_time), OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"), OPT_STRING('u', "uid", &record.opts.target.uid_str, "user", "user to profile"), @@ -3358,16 +3532,24 @@ static struct option __record_options[] = { struct option *record_options = __record_options; -static void record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus) +static int record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus) { struct perf_cpu cpu; int idx; if (cpu_map__is_dummy(cpus)) - return; + return 0; - perf_cpu_map__for_each_cpu(cpu, idx, cpus) + perf_cpu_map__for_each_cpu(cpu, idx, cpus) { + if (cpu.cpu == -1) + continue; + /* Return ENODEV is input cpu is greater than max cpu */ + if ((unsigned long)cpu.cpu > mask->nbits) + return -ENODEV; set_bit(cpu.cpu, mask->bits); + } + + return 0; } static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const char *mask_spec) @@ -3379,7 +3561,9 @@ static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const cha return -ENOMEM; bitmap_zero(mask->bits, mask->nbits); - record__mmap_cpu_mask_init(mask, cpus); + if (record__mmap_cpu_mask_init(mask, cpus)) + return -ENODEV; + perf_cpu_map__put(cpus); return 0; @@ -3461,7 +3645,12 @@ static int record__init_thread_masks_spec(struct record *rec, struct perf_cpu_ma pr_err("Failed to allocate CPUs mask\n"); return ret; } - record__mmap_cpu_mask_init(&cpus_mask, cpus); + + ret = record__mmap_cpu_mask_init(&cpus_mask, cpus); + if (ret) { + pr_err("Failed to init cpu mask\n"); + goto out_free_cpu_mask; + } ret = record__thread_mask_alloc(&full_mask, cpu__max_cpu().cpu); if (ret) { @@ -3702,7 +3891,8 @@ static int record__init_thread_default_masks(struct record *rec, struct perf_cpu if (ret) return ret; - record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus); + if (record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus)) + return -ENODEV; rec->nr_threads = 1; diff --git a/tools/perf/builtin-report.c b/tools/perf/builtin-report.c index 91ed41cc7d884d7436501211a573f9a8824fac1c..8361890176c23584ede91f5da7900ab0bc2aa23e 100644 --- a/tools/perf/builtin-report.c +++ b/tools/perf/builtin-report.c @@ -752,6 +752,22 @@ static int count_sample_event(struct perf_tool *tool __maybe_unused, return 0; } +static int count_lost_samples_event(struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine __maybe_unused) +{ + struct report *rep = container_of(tool, struct report, tool); + struct evsel *evsel; + + evsel = evlist__id2evsel(rep->session->evlist, sample->id); + if (evsel) { + hists__inc_nr_lost_samples(evsel__hists(evsel), + event->lost_samples.lost); + } + return 0; +} + static int process_attr(struct perf_tool *tool __maybe_unused, union perf_event *event, struct evlist **pevlist); @@ -761,6 +777,7 @@ static void stats_setup(struct report *rep) memset(&rep->tool, 0, sizeof(rep->tool)); rep->tool.attr = process_attr; rep->tool.sample = count_sample_event; + rep->tool.lost_samples = count_lost_samples_event; rep->tool.no_warn = true; } diff --git a/tools/perf/builtin-sched.c b/tools/perf/builtin-sched.c index 2f6cd1b8b66273fd7a80c584026c486e1dfa543f..f93737eef07ba0fcda59cccd415963459cb1715d 100644 --- a/tools/perf/builtin-sched.c +++ b/tools/perf/builtin-sched.c @@ -7,6 +7,7 @@ #include "util/evlist.h" #include "util/evsel.h" #include "util/evsel_fprintf.h" +#include "util/mutex.h" #include "util/symbol.h" #include "util/thread.h" #include "util/header.h" @@ -184,8 +185,8 @@ struct perf_sched { struct task_desc **pid_to_task; struct task_desc **tasks; const struct trace_sched_handler *tp_handler; - pthread_mutex_t start_work_mutex; - pthread_mutex_t work_done_wait_mutex; + struct mutex start_work_mutex; + struct mutex work_done_wait_mutex; int profile_cpu; /* * Track the current task - that way we can know whether there's any @@ -245,6 +246,7 @@ struct perf_sched { const char *time_str; struct perf_time_interval ptime; struct perf_time_interval hist_time; + volatile bool thread_funcs_exit; }; /* per thread run time data */ @@ -632,35 +634,34 @@ static void *thread_func(void *ctx) prctl(PR_SET_NAME, comm2); if (fd < 0) return NULL; -again: - ret = sem_post(&this_task->ready_for_work); - BUG_ON(ret); - ret = pthread_mutex_lock(&sched->start_work_mutex); - BUG_ON(ret); - ret = pthread_mutex_unlock(&sched->start_work_mutex); - BUG_ON(ret); - cpu_usage_0 = get_cpu_usage_nsec_self(fd); + while (!sched->thread_funcs_exit) { + ret = sem_post(&this_task->ready_for_work); + BUG_ON(ret); + mutex_lock(&sched->start_work_mutex); + mutex_unlock(&sched->start_work_mutex); - for (i = 0; i < this_task->nr_events; i++) { - this_task->curr_event = i; - perf_sched__process_event(sched, this_task->atoms[i]); - } + cpu_usage_0 = get_cpu_usage_nsec_self(fd); - cpu_usage_1 = get_cpu_usage_nsec_self(fd); - this_task->cpu_usage = cpu_usage_1 - cpu_usage_0; - ret = sem_post(&this_task->work_done_sem); - BUG_ON(ret); + for (i = 0; i < this_task->nr_events; i++) { + this_task->curr_event = i; + perf_sched__process_event(sched, this_task->atoms[i]); + } - ret = pthread_mutex_lock(&sched->work_done_wait_mutex); - BUG_ON(ret); - ret = pthread_mutex_unlock(&sched->work_done_wait_mutex); - BUG_ON(ret); + cpu_usage_1 = get_cpu_usage_nsec_self(fd); + this_task->cpu_usage = cpu_usage_1 - cpu_usage_0; + ret = sem_post(&this_task->work_done_sem); + BUG_ON(ret); - goto again; + mutex_lock(&sched->work_done_wait_mutex); + mutex_unlock(&sched->work_done_wait_mutex); + } + return NULL; } static void create_tasks(struct perf_sched *sched) + EXCLUSIVE_LOCK_FUNCTION(sched->start_work_mutex) + EXCLUSIVE_LOCK_FUNCTION(sched->work_done_wait_mutex) { struct task_desc *task; pthread_attr_t attr; @@ -672,10 +673,8 @@ static void create_tasks(struct perf_sched *sched) err = pthread_attr_setstacksize(&attr, (size_t) max(16 * 1024, (int)PTHREAD_STACK_MIN)); BUG_ON(err); - err = pthread_mutex_lock(&sched->start_work_mutex); - BUG_ON(err); - err = pthread_mutex_lock(&sched->work_done_wait_mutex); - BUG_ON(err); + mutex_lock(&sched->start_work_mutex); + mutex_lock(&sched->work_done_wait_mutex); for (i = 0; i < sched->nr_tasks; i++) { struct sched_thread_parms *parms = malloc(sizeof(*parms)); BUG_ON(parms == NULL); @@ -691,7 +690,30 @@ static void create_tasks(struct perf_sched *sched) } } +static void destroy_tasks(struct perf_sched *sched) + UNLOCK_FUNCTION(sched->start_work_mutex) + UNLOCK_FUNCTION(sched->work_done_wait_mutex) +{ + struct task_desc *task; + unsigned long i; + int err; + + mutex_unlock(&sched->start_work_mutex); + mutex_unlock(&sched->work_done_wait_mutex); + /* Get rid of threads so they won't be upset by mutex destrunction */ + for (i = 0; i < sched->nr_tasks; i++) { + task = sched->tasks[i]; + err = pthread_join(task->thread, NULL); + BUG_ON(err); + sem_destroy(&task->sleep_sem); + sem_destroy(&task->ready_for_work); + sem_destroy(&task->work_done_sem); + } +} + static void wait_for_tasks(struct perf_sched *sched) + EXCLUSIVE_LOCKS_REQUIRED(sched->work_done_wait_mutex) + EXCLUSIVE_LOCKS_REQUIRED(sched->start_work_mutex) { u64 cpu_usage_0, cpu_usage_1; struct task_desc *task; @@ -699,7 +721,7 @@ static void wait_for_tasks(struct perf_sched *sched) sched->start_time = get_nsecs(); sched->cpu_usage = 0; - pthread_mutex_unlock(&sched->work_done_wait_mutex); + mutex_unlock(&sched->work_done_wait_mutex); for (i = 0; i < sched->nr_tasks; i++) { task = sched->tasks[i]; @@ -707,12 +729,11 @@ static void wait_for_tasks(struct perf_sched *sched) BUG_ON(ret); sem_init(&task->ready_for_work, 0, 0); } - ret = pthread_mutex_lock(&sched->work_done_wait_mutex); - BUG_ON(ret); + mutex_lock(&sched->work_done_wait_mutex); cpu_usage_0 = get_cpu_usage_nsec_parent(); - pthread_mutex_unlock(&sched->start_work_mutex); + mutex_unlock(&sched->start_work_mutex); for (i = 0; i < sched->nr_tasks; i++) { task = sched->tasks[i]; @@ -734,8 +755,7 @@ static void wait_for_tasks(struct perf_sched *sched) sched->runavg_parent_cpu_usage = (sched->runavg_parent_cpu_usage * (sched->replay_repeat - 1) + sched->parent_cpu_usage)/sched->replay_repeat; - ret = pthread_mutex_lock(&sched->start_work_mutex); - BUG_ON(ret); + mutex_lock(&sched->start_work_mutex); for (i = 0; i < sched->nr_tasks; i++) { task = sched->tasks[i]; @@ -745,6 +765,8 @@ static void wait_for_tasks(struct perf_sched *sched) } static void run_one_test(struct perf_sched *sched) + EXCLUSIVE_LOCKS_REQUIRED(sched->work_done_wait_mutex) + EXCLUSIVE_LOCKS_REQUIRED(sched->start_work_mutex) { u64 T0, T1, delta, avg_delta, fluct; @@ -3316,11 +3338,14 @@ static int perf_sched__replay(struct perf_sched *sched) print_task_traces(sched); add_cross_task_wakeups(sched); + sched->thread_funcs_exit = false; create_tasks(sched); printf("------------------------------------------------------------\n"); for (i = 0; i < sched->replay_repeat; i++) run_one_test(sched); + sched->thread_funcs_exit = true; + destroy_tasks(sched); return 0; } @@ -3355,7 +3380,8 @@ static bool schedstat_events_exposed(void) static int __cmd_record(int argc, const char **argv) { unsigned int rec_argc, i, j; - const char **rec_argv; + char **rec_argv; + const char **rec_argv_copy; const char * const record_args[] = { "record", "-a", @@ -3384,6 +3410,7 @@ static int __cmd_record(int argc, const char **argv) ARRAY_SIZE(schedstat_args) : 0; struct tep_event *waking_event; + int ret; /* * +2 for either "-e", "sched:sched_wakeup" or @@ -3391,14 +3418,18 @@ static int __cmd_record(int argc, const char **argv) */ rec_argc = ARRAY_SIZE(record_args) + 2 + schedstat_argc + argc - 1; rec_argv = calloc(rec_argc + 1, sizeof(char *)); - if (rec_argv == NULL) return -ENOMEM; + rec_argv_copy = calloc(rec_argc + 1, sizeof(char *)); + if (rec_argv_copy == NULL) { + free(rec_argv); + return -ENOMEM; + } for (i = 0; i < ARRAY_SIZE(record_args); i++) rec_argv[i] = strdup(record_args[i]); - rec_argv[i++] = "-e"; + rec_argv[i++] = strdup("-e"); waking_event = trace_event__tp_format("sched", "sched_waking"); if (!IS_ERR(waking_event)) rec_argv[i++] = strdup("sched:sched_waking"); @@ -3409,11 +3440,19 @@ static int __cmd_record(int argc, const char **argv) rec_argv[i++] = strdup(schedstat_args[j]); for (j = 1; j < (unsigned int)argc; j++, i++) - rec_argv[i] = argv[j]; + rec_argv[i] = strdup(argv[j]); BUG_ON(i != rec_argc); - return cmd_record(i, rec_argv); + memcpy(rec_argv_copy, rec_argv, sizeof(char *) * rec_argc); + ret = cmd_record(rec_argc, rec_argv_copy); + + for (i = 0; i < rec_argc; i++) + free(rec_argv[i]); + free(rec_argv); + free(rec_argv_copy); + + return ret; } int cmd_sched(int argc, const char **argv) @@ -3430,8 +3469,6 @@ int cmd_sched(int argc, const char **argv) }, .cmp_pid = LIST_HEAD_INIT(sched.cmp_pid), .sort_list = LIST_HEAD_INIT(sched.sort_list), - .start_work_mutex = PTHREAD_MUTEX_INITIALIZER, - .work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER, .sort_order = default_sort_order, .replay_repeat = 10, .profile_cpu = -1, @@ -3545,8 +3582,10 @@ int cmd_sched(int argc, const char **argv) .fork_event = replay_fork_event, }; unsigned int i; - int ret; + int ret = 0; + mutex_init(&sched.start_work_mutex); + mutex_init(&sched.work_done_wait_mutex); for (i = 0; i < ARRAY_SIZE(sched.curr_pid); i++) sched.curr_pid[i] = -1; @@ -3558,11 +3597,10 @@ int cmd_sched(int argc, const char **argv) /* * Aliased to 'perf script' for now: */ - if (!strcmp(argv[0], "script")) - return cmd_script(argc, argv); - - if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) { - return __cmd_record(argc, argv); + if (!strcmp(argv[0], "script")) { + ret = cmd_script(argc, argv); + } else if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) { + ret = __cmd_record(argc, argv); } else if (strlen(argv[0]) > 2 && strstarts("latency", argv[0])) { sched.tp_handler = &lat_ops; if (argc > 1) { @@ -3571,7 +3609,7 @@ int cmd_sched(int argc, const char **argv) usage_with_options(latency_usage, latency_options); } setup_sorting(&sched, latency_options, latency_usage); - return perf_sched__lat(&sched); + ret = perf_sched__lat(&sched); } else if (!strcmp(argv[0], "map")) { if (argc) { argc = parse_options(argc, argv, map_options, map_usage, 0); @@ -3580,7 +3618,7 @@ int cmd_sched(int argc, const char **argv) } sched.tp_handler = &map_ops; setup_sorting(&sched, latency_options, latency_usage); - return perf_sched__map(&sched); + ret = perf_sched__map(&sched); } else if (strlen(argv[0]) > 2 && strstarts("replay", argv[0])) { sched.tp_handler = &replay_ops; if (argc) { @@ -3588,7 +3626,7 @@ int cmd_sched(int argc, const char **argv) if (argc) usage_with_options(replay_usage, replay_options); } - return perf_sched__replay(&sched); + ret = perf_sched__replay(&sched); } else if (!strcmp(argv[0], "timehist")) { if (argc) { argc = parse_options(argc, argv, timehist_options, @@ -3604,16 +3642,21 @@ int cmd_sched(int argc, const char **argv) parse_options_usage(NULL, timehist_options, "w", true); if (sched.show_next) parse_options_usage(NULL, timehist_options, "n", true); - return -EINVAL; + ret = -EINVAL; + goto out; } ret = symbol__validate_sym_arguments(); if (ret) - return ret; + goto out; - return perf_sched__timehist(&sched); + ret = perf_sched__timehist(&sched); } else { usage_with_options(sched_usage, sched_options); } - return 0; +out: + mutex_destroy(&sched.start_work_mutex); + mutex_destroy(&sched.work_done_wait_mutex); + + return ret; } diff --git a/tools/perf/builtin-script.c b/tools/perf/builtin-script.c index 13580a9c50b8d6d539e9f15d1305946a5639018a..7ca238277d835ff9afdde232ceec648e174882da 100644 --- a/tools/perf/builtin-script.c +++ b/tools/perf/builtin-script.c @@ -445,6 +445,9 @@ static int evsel__check_attr(struct evsel *evsel, struct perf_session *session) struct perf_event_attr *attr = &evsel->core.attr; bool allow_user_set; + if (evsel__is_dummy_event(evsel)) + return 0; + if (perf_header__has_feat(&session->header, HEADER_STAT)) return 0; @@ -566,6 +569,8 @@ static struct evsel *find_first_output_type(struct evlist *evlist, struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { + if (evsel__is_dummy_event(evsel)) + continue; if (output_type(evsel->core.attr.type) == (int)type) return evsel; } @@ -877,7 +882,7 @@ static int print_bstack_flags(FILE *fp, struct branch_entry *br) br->flags.in_tx ? 'X' : '-', br->flags.abort ? 'A' : '-', br->flags.cycles, - br->flags.type ? branch_type_name(br->flags.type) : "-"); + get_branch_type(br)); } static int perf_sample__fprintf_brstack(struct perf_sample *sample, @@ -2238,9 +2243,6 @@ static void __process_stat(struct evsel *counter, u64 tstamp) struct perf_cpu cpu; static int header_printed; - if (counter->core.system_wide) - nthreads = 1; - if (!header_printed) { printf("%3s %8s %15s %15s %15s %15s %s\n", "CPU", "THREAD", "VAL", "ENA", "RUN", "TIME", "EVENT"); @@ -3844,9 +3846,10 @@ int cmd_script(int argc, const char **argv) "Valid types: hw,sw,trace,raw,synth. " "Fields: comm,tid,pid,time,cpu,event,trace,ip,sym,dso," "addr,symoff,srcline,period,iregs,uregs,brstack," - "brstacksym,flags,bpf-output,brstackinsn,brstackinsnlen,brstackoff," - "callindent,insn,insnlen,synth,phys_addr,metric,misc,ipc,tod," - "data_page_size,code_page_size,ins_lat", + "brstacksym,flags,data_src,weight,bpf-output,brstackinsn," + "brstackinsnlen,brstackoff,callindent,insn,insnlen,synth," + "phys_addr,metric,misc,srccode,ipc,tod,data_page_size," + "code_page_size,ins_lat", parse_output_fields), OPT_BOOLEAN('a', "all-cpus", &system_wide, "system-wide collection from all CPUs"), diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 7fb81a44672d76e116b67207e1d80d69afafefcd..265b051579726aad7ff6a01722b80aef208004db 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -191,6 +191,7 @@ static bool append_file; static bool interval_count; static const char *output_name; static int output_fd; +static char *metrics; struct perf_stat { bool record; @@ -291,13 +292,8 @@ static inline void diff_timespec(struct timespec *r, struct timespec *a, static void perf_stat__reset_stats(void) { - int i; - evlist__reset_stats(evsel_list); perf_stat__reset_shadow_stats(); - - for (i = 0; i < stat_config.stats_num; i++) - perf_stat__reset_shadow_per_stat(&stat_config.stats[i]); } static int process_synthesized_event(struct perf_tool *tool __maybe_unused, @@ -488,46 +484,6 @@ static void read_counters(struct timespec *rs) } } -static int runtime_stat_new(struct perf_stat_config *config, int nthreads) -{ - int i; - - config->stats = calloc(nthreads, sizeof(struct runtime_stat)); - if (!config->stats) - return -1; - - config->stats_num = nthreads; - - for (i = 0; i < nthreads; i++) - runtime_stat__init(&config->stats[i]); - - return 0; -} - -static void runtime_stat_delete(struct perf_stat_config *config) -{ - int i; - - if (!config->stats) - return; - - for (i = 0; i < config->stats_num; i++) - runtime_stat__exit(&config->stats[i]); - - zfree(&config->stats); -} - -static void runtime_stat_reset(struct perf_stat_config *config) -{ - int i; - - if (!config->stats) - return; - - for (i = 0; i < config->stats_num; i++) - perf_stat__reset_shadow_per_stat(&config->stats[i]); -} - static void process_interval(void) { struct timespec ts, rs; @@ -536,7 +492,6 @@ static void process_interval(void) diff_timespec(&rs, &ts, &ref_time); perf_stat__reset_shadow_per_stat(&rt_stat); - runtime_stat_reset(&stat_config); read_counters(&rs); if (STAT_RECORD) { @@ -661,9 +616,7 @@ static void process_evlist(struct evlist *evlist, unsigned int interval) if (evlist__ctlfd_process(evlist, &cmd) > 0) { switch (cmd) { case EVLIST_CTL_CMD_ENABLE: - if (interval) - process_interval(); - break; + __fallthrough; case EVLIST_CTL_CMD_DISABLE: if (interval) process_interval(); @@ -826,6 +779,7 @@ static int __run_perf_stat(int argc, const char **argv, int run_idx) } evlist__for_each_entry(evsel_list, counter) { + counter->reset_group = false; if (bpf_counter__load(counter, &target)) return -1; if (!evsel__is_bpf(counter)) @@ -900,8 +854,6 @@ try_again: evlist__for_each_cpu(evlist_cpu_itr, evsel_list, affinity) { counter = evlist_cpu_itr.evsel; - if (!counter->reset_group && !counter->errored) - continue; if (!counter->reset_group) continue; try_again_reset: @@ -1016,7 +968,6 @@ try_again_reset: evlist__copy_prev_raw_counts(evsel_list); evlist__reset_prev_raw_counts(evsel_list); - runtime_stat_reset(&stat_config); perf_stat__reset_shadow_per_stat(&rt_stat); } else { update_stats(&walltime_nsecs_stats, t1 - t0); @@ -1147,14 +1098,23 @@ static int enable_metric_only(const struct option *opt __maybe_unused, return 0; } -static int parse_metric_groups(const struct option *opt, +static int append_metric_groups(const struct option *opt __maybe_unused, const char *str, int unset __maybe_unused) { - return metricgroup__parse_groups(opt, str, - stat_config.metric_no_group, - stat_config.metric_no_merge, - &stat_config.metric_events); + if (metrics) { + char *tmp; + + if (asprintf(&tmp, "%s,%s", metrics, str) < 0) + return -ENOMEM; + free(metrics); + metrics = tmp; + } else { + metrics = strdup(str); + if (!metrics) + return -ENOMEM; + } + return 0; } static int parse_control_option(const struct option *opt, @@ -1298,7 +1258,7 @@ static struct option stat_options[] = { "measure SMI cost"), OPT_CALLBACK('M', "metrics", &evsel_list, "metric/metric group list", "monitor specified metrics or metric groups (separated by ,)", - parse_metric_groups), + append_metric_groups), OPT_BOOLEAN_FLAG(0, "all-kernel", &stat_config.all_kernel, "Configure all used events to run in kernel space.", PARSE_OPT_EXCLUSIVE), @@ -1791,11 +1751,11 @@ static int add_default_attributes(void) * on an architecture test for such a metric name. */ if (metricgroup__has_metric("transaction")) { - struct option opt = { .value = &evsel_list }; - - return metricgroup__parse_groups(&opt, "transaction", + return metricgroup__parse_groups(evsel_list, "transaction", stat_config.metric_no_group, - stat_config.metric_no_merge, + stat_config.metric_no_merge, + stat_config.user_requested_cpu_list, + stat_config.system_wide, &stat_config.metric_events); } @@ -1931,6 +1891,9 @@ setup_metrics: free(str); } + if (!stat_config.topdown_level) + stat_config.topdown_level = TOPDOWN_MAX_LEVEL; + if (!evsel_list->core.nr_entries) { if (target__has_cpu(&target)) default_attrs0[0].config = PERF_COUNT_SW_CPU_CLOCK; @@ -1947,8 +1910,6 @@ setup_metrics: } if (evlist__add_default_attrs(evsel_list, default_attrs1) < 0) return -1; - - stat_config.topdown_level = TOPDOWN_MAX_LEVEL; /* Platform specific attrs */ if (evlist__add_default_attrs(evsel_list, default_null_attrs) < 0) return -1; @@ -2181,6 +2142,8 @@ static int __cmd_report(int argc, const char **argv) input_name = "perf.data"; } + perf_stat__init_shadow_stats(); + perf_stat.data.path = input_name; perf_stat.data.mode = PERF_DATA_MODE_READ; @@ -2260,8 +2223,6 @@ int cmd_stat(int argc, const char **argv) argc = parse_options_subcommand(argc, argv, stat_options, stat_subcommands, (const char **) stat_usage, PARSE_OPT_STOP_AT_NON_OPTION); - perf_stat__collect_metric_expr(evsel_list); - perf_stat__init_shadow_stats(); if (stat_config.csv_sep) { stat_config.csv_output = true; @@ -2428,6 +2389,34 @@ int cmd_stat(int argc, const char **argv) target.system_wide = true; } + if ((stat_config.aggr_mode == AGGR_THREAD) && (target.system_wide)) + target.per_thread = true; + + stat_config.system_wide = target.system_wide; + if (target.cpu_list) { + stat_config.user_requested_cpu_list = strdup(target.cpu_list); + if (!stat_config.user_requested_cpu_list) { + status = -ENOMEM; + goto out; + } + } + + /* + * Metric parsing needs to be delayed as metrics may optimize events + * knowing the target is system-wide. + */ + if (metrics) { + metricgroup__parse_groups(evsel_list, metrics, + stat_config.metric_no_group, + stat_config.metric_no_merge, + stat_config.user_requested_cpu_list, + stat_config.system_wide, + &stat_config.metric_events); + zfree(&metrics); + } + perf_stat__collect_metric_expr(evsel_list); + perf_stat__init_shadow_stats(); + if (add_default_attributes()) goto out; @@ -2447,9 +2436,6 @@ int cmd_stat(int argc, const char **argv) } } - if ((stat_config.aggr_mode == AGGR_THREAD) && (target.system_wide)) - target.per_thread = true; - if (evlist__fix_hybrid_cpus(evsel_list, target.cpu_list)) { pr_err("failed to use cpu list %s\n", target.cpu_list); goto out; @@ -2477,12 +2463,6 @@ int cmd_stat(int argc, const char **argv) */ if (stat_config.aggr_mode == AGGR_THREAD) { thread_map__read_comms(evsel_list->core.threads); - if (target.system_wide) { - if (runtime_stat_new(&stat_config, - perf_thread_map__nr(evsel_list->core.threads))) { - goto out; - } - } } if (stat_config.aggr_mode == AGGR_NODE) @@ -2615,6 +2595,7 @@ out: iostat_release(evsel_list); zfree(&stat_config.walltime_run); + zfree(&stat_config.user_requested_cpu_list); if (smi_cost && smi_reset) sysfs__write_int(FREEZE_ON_SMI_PATH, 0); @@ -2622,7 +2603,6 @@ out: evlist__delete(evsel_list); metricgroup__rblist_exit(&stat_config.metric_events); - runtime_stat_delete(&stat_config); evlist__close_control(stat_config.ctl_fd, stat_config.ctl_fd_ack, &stat_config.ctl_fd_close); return status; diff --git a/tools/perf/builtin-timechart.c b/tools/perf/builtin-timechart.c index e2e9ad929bafaeb3df8de6fa3568348461694d82..c36296bb7637ef6272fc0a0c583bb2ddcf7639d0 100644 --- a/tools/perf/builtin-timechart.c +++ b/tools/perf/builtin-timechart.c @@ -215,6 +215,19 @@ static struct per_pid *find_create_pid(struct timechart *tchart, int pid) return cursor; } +static struct per_pidcomm *create_pidcomm(struct per_pid *p) +{ + struct per_pidcomm *c; + + c = zalloc(sizeof(*c)); + if (!c) + return NULL; + p->current = c; + c->next = p->all; + p->all = c; + return c; +} + static void pid_set_comm(struct timechart *tchart, int pid, char *comm) { struct per_pid *p; @@ -233,12 +246,9 @@ static void pid_set_comm(struct timechart *tchart, int pid, char *comm) } c = c->next; } - c = zalloc(sizeof(*c)); + c = create_pidcomm(p); assert(c != NULL); c->comm = strdup(comm); - p->current = c; - c->next = p->all; - p->all = c; } static void pid_fork(struct timechart *tchart, int pid, int ppid, u64 timestamp) @@ -277,11 +287,8 @@ static void pid_put_sample(struct timechart *tchart, int pid, int type, p = find_create_pid(tchart, pid); c = p->current; if (!c) { - c = zalloc(sizeof(*c)); + c = create_pidcomm(p); assert(c != NULL); - p->current = c; - c->next = p->all; - p->all = c; } sample = zalloc(sizeof(*sample)); @@ -369,16 +376,13 @@ static void c_state_end(struct timechart *tchart, int cpu, u64 timestamp) tchart->power_events = pwr; } -static void p_state_change(struct timechart *tchart, int cpu, u64 timestamp, u64 new_freq) +static struct power_event *p_state_end(struct timechart *tchart, int cpu, + u64 timestamp) { - struct power_event *pwr; - - if (new_freq > 8000000) /* detect invalid data */ - return; + struct power_event *pwr = zalloc(sizeof(*pwr)); - pwr = zalloc(sizeof(*pwr)); if (!pwr) - return; + return NULL; pwr->state = cpus_pstate_state[cpu]; pwr->start_time = cpus_pstate_start_times[cpu]; @@ -386,11 +390,23 @@ static void p_state_change(struct timechart *tchart, int cpu, u64 timestamp, u64 pwr->cpu = cpu; pwr->type = PSTATE; pwr->next = tchart->power_events; - if (!pwr->start_time) pwr->start_time = tchart->first_time; tchart->power_events = pwr; + return pwr; +} + +static void p_state_change(struct timechart *tchart, int cpu, u64 timestamp, u64 new_freq) +{ + struct power_event *pwr; + + if (new_freq > 8000000) /* detect invalid data */ + return; + + pwr = p_state_end(tchart, cpu, timestamp); + if (!pwr) + return; cpus_pstate_state[cpu] = new_freq; cpus_pstate_start_times[cpu] = timestamp; @@ -698,22 +714,12 @@ static void end_sample_processing(struct timechart *tchart) #endif /* P state */ - pwr = zalloc(sizeof(*pwr)); + pwr = p_state_end(tchart, cpu, tchart->last_time); if (!pwr) return; - pwr->state = cpus_pstate_state[cpu]; - pwr->start_time = cpus_pstate_start_times[cpu]; - pwr->end_time = tchart->last_time; - pwr->cpu = cpu; - pwr->type = PSTATE; - pwr->next = tchart->power_events; - - if (!pwr->start_time) - pwr->start_time = tchart->first_time; if (!pwr->state) pwr->state = tchart->min_freq; - tchart->power_events = pwr; } } @@ -726,12 +732,9 @@ static int pid_begin_io_sample(struct timechart *tchart, int pid, int type, struct io_sample *prev; if (!c) { - c = zalloc(sizeof(*c)); + c = create_pidcomm(p); if (!c) return -ENOMEM; - p->current = c; - c->next = p->all; - p->all = c; } prev = c->io_samples; diff --git a/tools/perf/builtin-top.c b/tools/perf/builtin-top.c index fd8fd913c533cfe73b1e517d120fc1e95d0ba18b..4b3ff7687236e450550879a6f55d551da9507a09 100644 --- a/tools/perf/builtin-top.c +++ b/tools/perf/builtin-top.c @@ -136,10 +136,10 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) } notes = symbol__annotation(sym); - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); if (!symbol__hists(sym, top->evlist->core.nr_entries)) { - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); pr_err("Not enough memory for annotating '%s' symbol!\n", sym->name); sleep(1); @@ -155,7 +155,7 @@ static int perf_top__parse_source(struct perf_top *top, struct hist_entry *he) pr_err("Couldn't annotate %s: %s\n", sym->name, msg); } - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); return err; } @@ -196,6 +196,7 @@ static void perf_top__record_precise_ip(struct perf_top *top, struct hist_entry *he, struct perf_sample *sample, struct evsel *evsel, u64 ip) + EXCLUSIVE_LOCKS_REQUIRED(he->hists->lock) { struct annotation *notes; struct symbol *sym = he->ms.sym; @@ -208,19 +209,19 @@ static void perf_top__record_precise_ip(struct perf_top *top, notes = symbol__annotation(sym); - if (pthread_mutex_trylock(¬es->lock)) + if (!mutex_trylock(¬es->lock)) return; err = hist_entry__inc_addr_samples(he, sample, evsel, ip); - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); if (unlikely(err)) { /* * This function is now called with he->hists->lock held. * Release it before going to sleep. */ - pthread_mutex_unlock(&he->hists->lock); + mutex_unlock(&he->hists->lock); if (err == -ERANGE && !he->ms.map->erange_warned) ui__warn_map_erange(he->ms.map, sym, ip); @@ -230,7 +231,7 @@ static void perf_top__record_precise_ip(struct perf_top *top, sleep(1); } - pthread_mutex_lock(&he->hists->lock); + mutex_lock(&he->hists->lock); } } @@ -250,7 +251,7 @@ static void perf_top__show_details(struct perf_top *top) symbol = he->ms.sym; notes = symbol__annotation(symbol); - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); symbol__calc_percent(symbol, evsel); @@ -271,7 +272,7 @@ static void perf_top__show_details(struct perf_top *top) if (more != 0) printf("%d lines not displayed, maybe increase display entries [e]\n", more); out_unlock: - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); } static void perf_top__resort_hists(struct perf_top *t) @@ -724,13 +725,13 @@ repeat: static int hist_iter__top_callback(struct hist_entry_iter *iter, struct addr_location *al, bool single, void *arg) + EXCLUSIVE_LOCKS_REQUIRED(iter->he->hists->lock) { struct perf_top *top = arg; - struct hist_entry *he = iter->he; struct evsel *evsel = iter->evsel; if (perf_hpp_list.sym && single) - perf_top__record_precise_ip(top, he, iter->sample, evsel, al->addr); + perf_top__record_precise_ip(top, iter->he, iter->sample, evsel, al->addr); hist__account_cycles(iter->sample->branch_stack, al, iter->sample, !(top->record_opts.branch_stack & PERF_SAMPLE_BRANCH_ANY), @@ -836,12 +837,12 @@ static void perf_event__process_sample(struct perf_tool *tool, else iter.ops = &hist_iter_normal; - pthread_mutex_lock(&hists->lock); + mutex_lock(&hists->lock); if (hist_entry_iter__add(&iter, &al, top->max_stack, top) < 0) pr_err("Problem incrementing symbol period, skipping event\n"); - pthread_mutex_unlock(&hists->lock); + mutex_unlock(&hists->lock); } addr_location__put(&al); @@ -893,10 +894,10 @@ static void perf_top__mmap_read_idx(struct perf_top *top, int idx) perf_mmap__consume(&md->core); if (top->qe.rotate) { - pthread_mutex_lock(&top->qe.mutex); + mutex_lock(&top->qe.mutex); top->qe.rotate = false; - pthread_cond_signal(&top->qe.cond); - pthread_mutex_unlock(&top->qe.mutex); + cond_signal(&top->qe.cond); + mutex_unlock(&top->qe.mutex); } } @@ -1100,10 +1101,10 @@ static void *process_thread(void *arg) out = rotate_queues(top); - pthread_mutex_lock(&top->qe.mutex); + mutex_lock(&top->qe.mutex); top->qe.rotate = true; - pthread_cond_wait(&top->qe.cond, &top->qe.mutex); - pthread_mutex_unlock(&top->qe.mutex); + cond_wait(&top->qe.cond, &top->qe.mutex); + mutex_unlock(&top->qe.mutex); if (ordered_events__flush(out, OE_FLUSH__TOP)) pr_err("failed to process events\n"); @@ -1217,8 +1218,8 @@ static void init_process_thread(struct perf_top *top) ordered_events__set_copy_on_queue(&top->qe.data[0], true); ordered_events__set_copy_on_queue(&top->qe.data[1], true); top->qe.in = &top->qe.data[0]; - pthread_mutex_init(&top->qe.mutex, NULL); - pthread_cond_init(&top->qe.cond, NULL); + mutex_init(&top->qe.mutex); + cond_init(&top->qe.cond); } static int __cmd_top(struct perf_top *top) @@ -1349,7 +1350,7 @@ static int __cmd_top(struct perf_top *top) out_join: pthread_join(thread, NULL); out_join_thread: - pthread_cond_signal(&top->qe.cond); + cond_signal(&top->qe.cond); pthread_join(thread_process, NULL); return ret; } @@ -1706,6 +1707,7 @@ int cmd_top(int argc, const char **argv) if (evlist__create_maps(top.evlist, target) < 0) { ui__error("Couldn't create thread/CPU maps: %s\n", errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf))); + status = -errno; goto out_delete_evlist; } @@ -1758,11 +1760,13 @@ int cmd_top(int argc, const char **argv) if (top.sb_evlist == NULL) { pr_err("Couldn't create side band evlist.\n."); + status = -EINVAL; goto out_delete_evlist; } if (evlist__add_bpf_sb_event(top.sb_evlist, &perf_env)) { pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n."); + status = -EINVAL; goto out_delete_evlist; } } diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 0bd9d01c0df9dcfb53f94fa6eb4957818c1ef47c..d3c757769b96598f843bb68572571413e23ce93d 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -615,11 +615,8 @@ bool strarray__strtoul_flags(struct strarray *sa, char *bf, size_t size, u64 *re if (isalpha(*tok) || *tok == '_') { if (!strarray__strtoul(sa, tok, toklen, &val)) return false; - } else { - bool is_hexa = tok[0] == 0 && (tok[1] = 'x' || tok[1] == 'X'); - - val = strtoul(tok, NULL, is_hexa ? 16 : 0); - } + } else + val = strtoul(tok, NULL, 0); *ret |= (1 << (val - 1)); @@ -2173,13 +2170,10 @@ static void thread__update_stats(struct thread *thread, struct thread_trace *ttr stats = inode->priv; if (stats == NULL) { - stats = malloc(sizeof(*stats)); + stats = zalloc(sizeof(*stats)); if (stats == NULL) return; - stats->nr_failures = 0; - stats->max_errno = 0; - stats->errnos = NULL; init_stats(&stats->stats); inode->priv = stats; } @@ -2762,11 +2756,7 @@ static size_t trace__fprintf_tp_fields(struct trace *trace, struct evsel *evsel, printed += scnprintf(bf + printed, size - printed, "%s", printed ? ", " : ""); - /* - * XXX Perhaps we should have a show_tp_arg_names, - * leaving show_arg_names just for syscalls? - */ - if (1 || trace->show_arg_names) + if (trace->show_arg_names) printed += scnprintf(bf + printed, size - printed, "%s: ", field->name); printed += syscall_arg_fmt__scnprintf_val(arg, bf + printed, size - printed, &syscall_arg, val); diff --git a/tools/perf/dlfilters/dlfilter-show-cycles.c b/tools/perf/dlfilters/dlfilter-show-cycles.c index 9eccc97bff82f64108545ca6a5eedb59a70aa6e8..6d47298ebe9f61fc1fc1c0a799a0f4ec5bd2f9b2 100644 --- a/tools/perf/dlfilters/dlfilter-show-cycles.c +++ b/tools/perf/dlfilters/dlfilter-show-cycles.c @@ -98,9 +98,9 @@ int filter_event_early(void *data, const struct perf_dlfilter_sample *sample, vo static void print_vals(__u64 cycles, __u64 delta) { if (delta) - printf("%10llu %10llu ", cycles, delta); + printf("%10llu %10llu ", (unsigned long long)cycles, (unsigned long long)delta); else - printf("%10llu %10s ", cycles, ""); + printf("%10llu %10s ", (unsigned long long)cycles, ""); } int filter_event(void *data, const struct perf_dlfilter_sample *sample, void *ctx) diff --git a/tools/perf/perf.c b/tools/perf/perf.c index c21b3973641a06d08bb52002ed0c17036def5df1..7af135dea1cd86fecc542a24debe799fb8e627d8 100644 --- a/tools/perf/perf.c +++ b/tools/perf/perf.c @@ -99,10 +99,16 @@ struct pager_config { int val; }; +static bool same_cmd_with_prefix(const char *var, struct pager_config *c, + const char *header) +{ + return (strstarts(var, header) && !strcmp(var + strlen(header), c->cmd)); +} + static int pager_command_config(const char *var, const char *value, void *data) { struct pager_config *c = data; - if (strstarts(var, "pager.") && !strcmp(var + 6, c->cmd)) + if (same_cmd_with_prefix(var, c, "pager.")) c->val = perf_config_bool(var, value); return 0; } @@ -121,9 +127,9 @@ static int check_pager_config(const char *cmd) static int browser_command_config(const char *var, const char *value, void *data) { struct pager_config *c = data; - if (strstarts(var, "tui.") && !strcmp(var + 4, c->cmd)) + if (same_cmd_with_prefix(var, c, "tui.")) c->val = perf_config_bool(var, value); - if (strstarts(var, "gtk.") && !strcmp(var + 4, c->cmd)) + if (same_cmd_with_prefix(var, c, "gtk.")) c->val = perf_config_bool(var, value) ? 2 : 0; return 0; } diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/branch.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/branch.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/branch.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/branch.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/bus.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/bus.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/bus.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/bus.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/cache.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/cache.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/cache.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/cache.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/dpu.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/dpu.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/dpu.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/dpu.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/exception.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/exception.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/exception.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/exception.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/ifu.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/ifu.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/ifu.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/ifu.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/instruction.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/instruction.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/instruction.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/instruction.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/memory.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/memory.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/memory.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/memory.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a65/pipeline.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/pipeline.json similarity index 100% rename from tools/perf/pmu-events/arch/arm64/arm/cortex-a65/pipeline.json rename to tools/perf/pmu-events/arch/arm64/arm/cortex-a65-e1/pipeline.json diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/memory.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/memory.json index 20a929e7728d42e52d2095ffcf3b775e808846d3..5bed2514b245efac97859ff31983d76830156899 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/memory.json +++ b/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/memory.json @@ -3,6 +3,9 @@ "PublicDescription": "This event counts memory accesses due to load or store instructions. This event counts the sum of MEM_ACCESS_RD and MEM_ACCESS_WR.", "ArchStdEvent": "MEM_ACCESS" }, + { + "ArchStdEvent": "REMOTE_ACCESS" + }, { "ArchStdEvent": "MEM_ACCESS_RD" }, diff --git a/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/other.json b/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/other.json deleted file mode 100644 index 20d8365756c5fe1a98b3b40f5acfc9b612fd04e9..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/cortex-a76-n1/other.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "ArchStdEvent": "REMOTE_ACCESS" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/branch.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/branch.json deleted file mode 100644 index 2f2d137f5f55ad194b104681f37ce061596fb9f1..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/branch.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "ArchStdEvent": "BR_MIS_PRED" - }, - { - "ArchStdEvent": "BR_PRED" - }, - { - "ArchStdEvent": "BR_IMMED_SPEC" - }, - { - "ArchStdEvent": "BR_RETURN_SPEC" - }, - { - "ArchStdEvent": "BR_INDIRECT_SPEC" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/bus.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/bus.json deleted file mode 100644 index 75d850b781acd2673f851035ece47f67fac03041..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/bus.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "ArchStdEvent": "CPU_CYCLES" - }, - { - "ArchStdEvent": "BUS_ACCESS" - }, - { - "ArchStdEvent": "BUS_CYCLES" - }, - { - "ArchStdEvent": "BUS_ACCESS_RD" - }, - { - "ArchStdEvent": "BUS_ACCESS_WR" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/cache.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/cache.json deleted file mode 100644 index 3ad15e3a93a91546c337d2b81cf777199ea66922..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/cache.json +++ /dev/null @@ -1,107 +0,0 @@ -[ - { - "ArchStdEvent": "L1I_CACHE_REFILL" - }, - { - "ArchStdEvent": "L1I_TLB_REFILL" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L1D_CACHE" - }, - { - "ArchStdEvent": "L1D_TLB_REFILL" - }, - { - "ArchStdEvent": "L1I_CACHE" - }, - { - "ArchStdEvent": "L1D_CACHE_WB" - }, - { - "ArchStdEvent": "L2D_CACHE" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L2D_CACHE_WB" - }, - { - "ArchStdEvent": "L1D_CACHE_ALLOCATE" - }, - { - "ArchStdEvent": "L2D_CACHE_ALLOCATE" - }, - { - "ArchStdEvent": "L1D_TLB" - }, - { - "ArchStdEvent": "L1I_TLB" - }, - { - "ArchStdEvent": "L3D_CACHE_ALLOCATE" - }, - { - "ArchStdEvent": "L3D_CACHE_REFILL" - }, - { - "ArchStdEvent": "L3D_CACHE" - }, - { - "ArchStdEvent": "L2D_TLB_REFILL" - }, - { - "ArchStdEvent": "L2D_TLB" - }, - { - "ArchStdEvent": "DTLB_WALK" - }, - { - "ArchStdEvent": "ITLB_WALK" - }, - { - "ArchStdEvent": "LL_CACHE_RD" - }, - { - "ArchStdEvent": "LL_CACHE_MISS_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_WR" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_RD" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_WR" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_INNER" - }, - { - "ArchStdEvent": "L1D_CACHE_REFILL_OUTER" - }, - { - "ArchStdEvent": "L2D_CACHE_RD" - }, - { - "ArchStdEvent": "L2D_CACHE_WR" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL_RD" - }, - { - "ArchStdEvent": "L2D_CACHE_REFILL_WR" - }, - { - "ArchStdEvent": "L3D_CACHE_RD" - }, - { - "ArchStdEvent": "L3D_CACHE_REFILL_RD" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/exception.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/exception.json deleted file mode 100644 index 27c3fe9c831ae5c24565456381aa8f8f949b1435..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/exception.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "ArchStdEvent": "EXC_TAKEN" - }, - { - "ArchStdEvent": "MEMORY_ERROR" - }, - { - "ArchStdEvent": "EXC_IRQ" - }, - { - "ArchStdEvent": "EXC_FIQ" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/instruction.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/instruction.json deleted file mode 100644 index 6c3b8f772e7fb933d2ea41f183c63d3781457f0f..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/instruction.json +++ /dev/null @@ -1,65 +0,0 @@ -[ - { - "ArchStdEvent": "SW_INCR" - }, - { - "ArchStdEvent": "LD_RETIRED" - }, - { - "ArchStdEvent": "ST_RETIRED" - }, - { - "ArchStdEvent": "INST_RETIRED" - }, - { - "ArchStdEvent": "EXC_RETURN" - }, - { - "ArchStdEvent": "CID_WRITE_RETIRED" - }, - { - "ArchStdEvent": "PC_WRITE_RETIRED" - }, - { - "ArchStdEvent": "BR_IMMED_RETIRED" - }, - { - "ArchStdEvent": "BR_RETURN_RETIRED" - }, - { - "ArchStdEvent": "INST_SPEC" - }, - { - "ArchStdEvent": "TTBR_WRITE_RETIRED" - }, - { - "ArchStdEvent": "BR_RETIRED" - }, - { - "ArchStdEvent": "BR_MIS_PRED_RETIRED" - }, - { - "ArchStdEvent": "LD_SPEC" - }, - { - "ArchStdEvent": "ST_SPEC" - }, - { - "ArchStdEvent": "LDST_SPEC" - }, - { - "ArchStdEvent": "DP_SPEC" - }, - { - "ArchStdEvent": "ASE_SPEC" - }, - { - "ArchStdEvent": "VFP_SPEC" - }, - { - "ArchStdEvent": "CRYPTO_SPEC" - }, - { - "ArchStdEvent": "ISB_SPEC" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/memory.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/memory.json deleted file mode 100644 index 78ed6dfcedc1bca8a968c96bc1fcdcbd4d0a0824..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/memory.json +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "ArchStdEvent": "MEM_ACCESS" - }, - { - "ArchStdEvent": "REMOTE_ACCESS_RD" - }, - { - "ArchStdEvent": "MEM_ACCESS_RD" - }, - { - "ArchStdEvent": "MEM_ACCESS_WR" - }, - { - "ArchStdEvent": "UNALIGNED_LD_SPEC" - }, - { - "ArchStdEvent": "UNALIGNED_ST_SPEC" - }, - { - "ArchStdEvent": "UNALIGNED_LDST_SPEC" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/pipeline.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/pipeline.json deleted file mode 100644 index eeac798d403a0df02f008c942e22dce446c28125..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/pipeline.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - { - "ArchStdEvent": "STALL_FRONTEND" - }, - { - "ArchStdEvent": "STALL_BACKEND" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/spe.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/spe.json deleted file mode 100644 index 20f2165c85fec581226bc153571e13809e18104b..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-e1/spe.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "ArchStdEvent": "SAMPLE_POP" - }, - { - "ArchStdEvent": "SAMPLE_FEED" - }, - { - "ArchStdEvent": "SAMPLE_FILTRATE" - }, - { - "ArchStdEvent": "SAMPLE_COLLISION" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/memory.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/memory.json index e522113aeb961a4f02ee0a41209307d2b8478fb6..7b2b21ac150f523e04f1397a53565a288c1e6f47 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/memory.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/memory.json @@ -2,6 +2,9 @@ { "ArchStdEvent": "MEM_ACCESS" }, + { + "ArchStdEvent": "REMOTE_ACCESS" + }, { "ArchStdEvent": "MEM_ACCESS_RD" }, diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/other.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/other.json deleted file mode 100644 index 20d8365756c5fe1a98b3b40f5acfc9b612fd04e9..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-n2/other.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "ArchStdEvent": "REMOTE_ACCESS" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json index 25825e14c535b3aa89b28824ca5f3b8d32a7e74f..e29b88fb7f24a943664ca176ef65d692871730ea 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/instruction.json @@ -85,5 +85,35 @@ }, { "ArchStdEvent": "RC_ST_SPEC" + }, + { + "ArchStdEvent": "ASE_INST_SPEC" + }, + { + "ArchStdEvent": "SVE_INST_SPEC" + }, + { + "ArchStdEvent": "SVE_PRED_SPEC" + }, + { + "ArchStdEvent": "SVE_PRED_EMPTY_SPEC" + }, + { + "ArchStdEvent": "SVE_PRED_FULL_SPEC" + }, + { + "ArchStdEvent": "SVE_PRED_PARTIAL_SPEC" + }, + { + "ArchStdEvent": "SVE_LDFF_SPEC" + }, + { + "ArchStdEvent": "SVE_LDFF_FAULT_SPEC" + }, + { + "ArchStdEvent": "FP_SCALE_OPS_SPEC" + }, + { + "ArchStdEvent": "FP_FIXED_OPS_SPEC" } ] diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json index e3d08f1f7c92c1fc430079ad806788072d88664f..5aff6e93c1adb1a278e026675fc47ef5aca071ba 100644 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json +++ b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/memory.json @@ -2,6 +2,9 @@ { "ArchStdEvent": "MEM_ACCESS" }, + { + "ArchStdEvent": "REMOTE_ACCESS" + }, { "ArchStdEvent": "MEM_ACCESS_RD" }, diff --git a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/other.json b/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/other.json deleted file mode 100644 index 20d8365756c5fe1a98b3b40f5acfc9b612fd04e9..0000000000000000000000000000000000000000 --- a/tools/perf/pmu-events/arch/arm64/arm/neoverse-v1/other.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "ArchStdEvent": "REMOTE_ACCESS" - } -] diff --git a/tools/perf/pmu-events/arch/arm64/mapfile.csv b/tools/perf/pmu-events/arch/arm64/mapfile.csv index 406f6edd4e12cfe318740bacef47ea8efed67d7b..ad502d00f460736b42abb5d9622c2d2447679a59 100644 --- a/tools/perf/pmu-events/arch/arm64/mapfile.csv +++ b/tools/perf/pmu-events/arch/arm64/mapfile.csv @@ -17,7 +17,8 @@ 0x00000000420f1000,v1,arm/cortex-a53,core 0x00000000410fd040,v1,arm/cortex-a35,core 0x00000000410fd050,v1,arm/cortex-a55,core -0x00000000410fd060,v1,arm/cortex-a65,core +0x00000000410fd060,v1,arm/cortex-a65-e1,core +0x00000000410fd4a0,v1,arm/cortex-a65-e1,core 0x00000000410fd070,v1,arm/cortex-a57-a72,core 0x00000000410fd080,v1,arm/cortex-a57-a72,core 0x00000000410fd090,v1,arm/cortex-a73,core @@ -34,7 +35,6 @@ 0x00000000410fd470,v1,arm/cortex-a710,core 0x00000000410fd480,v1,arm/cortex-x2,core 0x00000000410fd490,v1,arm/neoverse-n2,core -0x00000000410fd4a0,v1,arm/neoverse-e1,core 0x00000000420f5160,v1,cavium/thunderx2,core 0x00000000430f0af0,v1,cavium/thunderx2,core 0x00000000460f0010,v1,fujitsu/a64fx,core diff --git a/tools/perf/pmu-events/arch/test/test_soc/cpu/metrics.json b/tools/perf/pmu-events/arch/test/test_soc/cpu/metrics.json index 42d9b5242fd7d4bf17277772e323f2b7f55487d0..70ec8caaaf6f0ec48560c9fa58b9258aadf8a88a 100644 --- a/tools/perf/pmu-events/arch/test/test_soc/cpu/metrics.json +++ b/tools/perf/pmu-events/arch/test/test_soc/cpu/metrics.json @@ -34,15 +34,15 @@ "MetricName": "DCache_L2_All_Miss" }, { - "MetricExpr": "dcache_l2_all_hits + dcache_l2_all_miss", + "MetricExpr": "DCache_L2_All_Hits + DCache_L2_All_Miss", "MetricName": "DCache_L2_All" }, { - "MetricExpr": "d_ratio(dcache_l2_all_hits, dcache_l2_all)", + "MetricExpr": "d_ratio(DCache_L2_All_Hits, DCache_L2_All)", "MetricName": "DCache_L2_Hits" }, { - "MetricExpr": "d_ratio(dcache_l2_all_miss, dcache_l2_all)", + "MetricExpr": "d_ratio(DCache_L2_All_Miss, DCache_L2_All)", "MetricName": "DCache_L2_Misses" }, { diff --git a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json index 095dd8c7f16197d5181013e0c1149a6c062ac3b3..e06d26ad51385ead7e383ee04ae3cdd89ff7a83d 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/adl-metrics.json @@ -1,22 +1,852 @@ [ + { + "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", + "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "(topdown\\-fetch\\-lat / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS)", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE_DATA.STALLS / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_TAG.STALLS / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(tma_branch_mispredicts / tma_bad_speculation) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (tma_branch_mispredicts / tma_bad_speculation)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "INT_MISC.UNKNOWN_BRANCH_CYCLES / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "DECODE.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu_core@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu_core@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit", + "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_lsd", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", + "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "topdown\\-br\\-mispredict / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", + "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "topdown\\-mem\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(7 * cpu_core@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - MEMORY_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((25 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (24 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(24 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "(XQ.FULL_CYCLES + L1D_PEND_MISS.L2_STALLS) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu_core@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((MEM_STORE_RETIRED.L2_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(28 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores", + "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_streaming_stores", + "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * cpu_core@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(cpu_core@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu_core@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu_core@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "cpu_core@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS) / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "CPU_CLK_UNHALTED.PAUSE / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: CPU_CLK_UNHALTED.PAUSE_INST", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to LFENCE Instructions.", + "MetricExpr": "13 * MISC2_RETIRED.LFENCE / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_memory_fence", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "160 * ASSISTS.SSE_AVX_MIX / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5_11 + UOPS_DISPATCHED.PORT_6) / (5 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10", + "MetricExpr": "UOPS_DISPATCHED.PORT_2_3_10 / (3 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", + "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_int_vector_128b + tma_int_vector_256b + tma_shuffles", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_int_operations", + "PublicDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents 128-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.", + "MetricExpr": "(INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_int_vector_128b", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents 256-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.", + "MetricExpr": "(INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_int_vector_256b", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents Shuffle (cross \"vector lane\" data transfers) uops fraction the CPU has retired.", + "MetricExpr": "INT_VEC_RETIRED.SHUFFLES / (tma_retiring * SLOTS)", + "MetricGroup": "HPC;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_shuffles", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_UOP_RETIRED.ANY / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.MACRO_FUSED / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fused_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused", + "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED) / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_non_fused_branches", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_int_operations + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "topdown\\-heavy\\-ops / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences. Sample with: UOPS_RETIRED.HEAVY", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "UOPS_RETIRED.MS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * cpu_core@ASSISTS.ANY\\,umask\\=0x1B@ / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults", + "MetricExpr": "99 * ASSISTS.PAGE_FAULT / SLOTS", + "MetricGroup": "TopdownL5;tma_assists_group", + "MetricName": "tma_page_faults", + "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults. A Page Fault may apply on first application access to a memory page. Note operating system handling of page faults accounts for the majority of its cost.", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists", + "MetricExpr": "30 * ASSISTS.FP / SLOTS", + "MetricGroup": "HPC;TopdownL5;tma_assists_group", + "MetricName": "tma_fp_assists", + "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists. FP Assist may apply when working with very small floating point values (so-called denormals).", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops as a result of handing SSE to AVX* or AVX* to SSE transition Assists. ", + "MetricExpr": "63 * ASSISTS.SSE_AVX_MIX / SLOTS", + "MetricGroup": "HPC;TopdownL5;tma_assists_group", + "MetricName": "tma_avx_assists", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources. Sample with: FRONTEND_RETIRED.MS_FLOWS", + "ScaleUnit": "100%", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", + "MetricGroup": "Mem;MemoryBW;Offcore", + "MetricName": "Memory_Bandwidth", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))", + "MetricGroup": "Mem;MemoryLat;Offcore", + "MetricName": "Memory_Latency", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ", + "MetricGroup": "Mem;MemoryTLB;Offcore", + "MetricName": "Memory_Data_TLBs", + "Unit": "cpu_core" + }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead", "Unit": "cpu_core" }, + { + "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", + "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", + "MetricName": "Big_Code", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", + "MetricGroup": "Fed;FetchBW;Frontend", + "MetricName": "Instruction_Fetch_BW", + "Unit": "cpu_core" + }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC", "Unit": "cpu_core" }, + { + "BriefDescription": "Uops Per Instruction", + "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;Ret;Retire", + "MetricName": "UPI", + "Unit": "cpu_core" + }, + { + "BriefDescription": "Instruction per taken branch", + "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN", + "MetricGroup": "Branches;Fed;FetchBW", + "MetricName": "UpTB", + "Unit": "cpu_core" + }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI", "Unit": "cpu_core" }, @@ -30,14 +860,14 @@ { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", "MetricExpr": "TOPDOWN.SLOTS", - "MetricGroup": "TmaL1", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS", "Unit": "cpu_core" }, { "BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor", - "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1", - "MetricGroup": "SMT;TmaL1", + "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1", + "MetricGroup": "SMT;tma_L1_group", "MetricName": "Slots_Utilization", "Unit": "cpu_core" }, @@ -51,21 +881,21 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC", "Unit": "cpu_core" }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc", "Unit": "cpu_core" }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5 ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common).", @@ -73,11 +903,18 @@ }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP", "Unit": "cpu_core" }, + { + "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", + "MetricGroup": "Cor;SMT", + "MetricName": "Core_Bound_Likely", + "Unit": "cpu_core" + }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", "MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED", @@ -129,14 +966,14 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP", "Unit": "cpu_core" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW.", @@ -160,7 +997,7 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting.", @@ -168,7 +1005,7 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting.", @@ -182,12 +1019,19 @@ "Unit": "cpu_core" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions", "Unit": "cpu_core" }, + { + "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.", + "MetricExpr": "(tma_retiring * SLOTS) / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@", + "MetricGroup": "Pipeline;Ret", + "MetricName": "Retire", + "Unit": "cpu_core" + }, { "BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions", "MetricExpr": "INST_RETIRED.REP_ITERATION / cpu_core@UOPS_RETIRED.SLOTS\\,cmask\\=1@", @@ -237,6 +1081,13 @@ "MetricName": "DSB_Switch_Cost", "Unit": "cpu_core" }, + { + "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))", + "MetricGroup": "DSBmiss;Fed", + "MetricName": "DSB_Misses", + "Unit": "cpu_core" + }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -251,6 +1102,13 @@ "MetricName": "IpMispredict", "Unit": "cpu_core" }, + { + "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricGroup": "Bad;BrMispredicts", + "MetricName": "Branch_Misprediction_Cost", + "Unit": "cpu_core" + }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -267,7 +1125,7 @@ }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet", "Unit": "cpu_core" @@ -281,7 +1139,7 @@ }, { "BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)", - "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )", + "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)", "MetricGroup": "Bad;Branches", "MetricName": "Other_Branches", "Unit": "cpu_core" @@ -296,77 +1154,77 @@ { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP", "Unit": "cpu_core" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI", "Unit": "cpu_core" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load", "Unit": "cpu_core" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI", "Unit": "cpu_core" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All", "Unit": "cpu_core" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load", "Unit": "cpu_core" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All", "Unit": "cpu_core" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load", "Unit": "cpu_core" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI", "Unit": "cpu_core" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI", "Unit": "cpu_core" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 4 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (4 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization", "Unit": "cpu_core" @@ -401,28 +1259,28 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T", "Unit": "cpu_core" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T", "Unit": "cpu_core" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T", "Unit": "cpu_core" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T", "Unit": "cpu_core" @@ -436,14 +1294,14 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency", "Unit": "cpu_core" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine.", @@ -451,7 +1309,7 @@ }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization", "Unit": "cpu_core" @@ -479,7 +1337,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use", "Unit": "cpu_core" @@ -500,41 +1358,408 @@ }, { "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to frontend stalls.", - "MetricExpr": "TOPDOWN_FE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE)", + "MetricExpr": "TOPDOWN_FE_BOUND.ALL / SLOTS", "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", + "MetricName": "tma_frontend_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.", + "MetricExpr": "TOPDOWN_FE_BOUND.FRONTEND_LATENCY / SLOTS", + "MetricGroup": "TopdownL2;tma_frontend_bound_group", + "MetricName": "tma_frontend_latency", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to instruction cache misses.", + "MetricExpr": "TOPDOWN_FE_BOUND.ICACHE / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_latency_group", + "MetricName": "tma_icache", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to Instruction Table Lookaside Buffer (ITLB) misses.", + "MetricExpr": "TOPDOWN_FE_BOUND.ITLB / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_latency_group", + "MetricName": "tma_itlb", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to BACLEARS, which occurs when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend", + "MetricExpr": "TOPDOWN_FE_BOUND.BRANCH_DETECT / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_latency_group", + "MetricName": "tma_branch_detect", + "PublicDescription": "Counts the number of issue slots that were not delivered by the frontend due to BACLEARS, which occurs when the Branch Target Buffer (BTB) prediction or lack thereof, was corrected by a later branch predictor in the frontend. Includes BACLEARS due to all branch types including conditional and unconditional jumps, returns, and indirect branches.", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to BTCLEARS, which occurs when the Branch Target Buffer (BTB) predicts a taken branch.", + "MetricExpr": "TOPDOWN_FE_BOUND.BRANCH_RESTEER / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_latency_group", + "MetricName": "tma_branch_resteer", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to frontend bandwidth restrictions due to decode, predecode, cisc, and other limitations.", + "MetricExpr": "TOPDOWN_FE_BOUND.FRONTEND_BANDWIDTH / SLOTS", + "MetricGroup": "TopdownL2;tma_frontend_bound_group", + "MetricName": "tma_frontend_bandwidth", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to the microcode sequencer (MS).", + "MetricExpr": "TOPDOWN_FE_BOUND.CISC / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group", + "MetricName": "tma_cisc", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to decode stalls.", + "MetricExpr": "TOPDOWN_FE_BOUND.DECODE / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group", + "MetricName": "tma_decode", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to wrong predecodes.", + "MetricExpr": "TOPDOWN_FE_BOUND.PREDECODE / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group", + "MetricName": "tma_predecode", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not delivered by the frontend due to other common frontend stalls not categorized.", + "MetricExpr": "TOPDOWN_FE_BOUND.OTHER / SLOTS", + "MetricGroup": "TopdownL3;tma_frontend_bandwidth_group", + "MetricName": "tma_other_fb", + "ScaleUnit": "100%", "Unit": "cpu_atom" }, { "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear", - "MetricExpr": "TOPDOWN_BAD_SPECULATION.ALL / (5 * CPU_CLK_UNHALTED.CORE)", + "MetricExpr": "(SLOTS - (TOPDOWN_FE_BOUND.ALL + TOPDOWN_BE_BOUND.ALL + TOPDOWN_RETIRING.ALL)) / SLOTS", "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", + "MetricName": "tma_bad_speculation", "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a mispredicted jump or a machine clear. Only issue slots wasted due to fast nukes such as memory ordering nukes are counted. Other nukes are not accounted for. Counts all issue slots blocked during this recovery window including relevant microcode flows and while uops are not yet available in the instruction queue (IQ). Also includes the issue slots that were consumed by the backend but were thrown away because they were younger than the mispredict or machine clear.", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to branch mispredicts.", + "MetricExpr": "TOPDOWN_BAD_SPECULATION.MISPREDICT / SLOTS", + "MetricGroup": "TopdownL2;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend because allocation is stalled due to a machine clear (nuke) of any kind including memory ordering and memory disambiguation.", + "MetricExpr": "TOPDOWN_BAD_SPECULATION.MACHINE_CLEARS / SLOTS", + "MetricGroup": "TopdownL2;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to a machine clear (slow nuke).", + "MetricExpr": "TOPDOWN_BAD_SPECULATION.NUKE / SLOTS", + "MetricGroup": "TopdownL3;tma_machine_clears_group", + "MetricName": "tma_nuke", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to SMC. ", + "MetricExpr": "tma_nuke * (MACHINE_CLEARS.SMC / MACHINE_CLEARS.SLOW)", + "MetricGroup": "TopdownL4;tma_nuke_group", + "MetricName": "tma_smc", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to memory ordering. ", + "MetricExpr": "tma_nuke * (MACHINE_CLEARS.MEMORY_ORDERING / MACHINE_CLEARS.SLOW)", + "MetricGroup": "TopdownL4;tma_nuke_group", + "MetricName": "tma_memory_ordering", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to FP assists. ", + "MetricExpr": "tma_nuke * (MACHINE_CLEARS.FP_ASSIST / MACHINE_CLEARS.SLOW)", + "MetricGroup": "TopdownL4;tma_nuke_group", + "MetricName": "tma_fp_assist", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to memory disambiguation. ", + "MetricExpr": "tma_nuke * (MACHINE_CLEARS.DISAMBIGUATION / MACHINE_CLEARS.SLOW)", + "MetricGroup": "TopdownL4;tma_nuke_group", + "MetricName": "tma_disambiguation", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of machine clears relative to the number of nuke slots due to page faults. ", + "MetricExpr": "tma_nuke * (MACHINE_CLEARS.PAGE_FAULT / MACHINE_CLEARS.SLOW)", + "MetricGroup": "TopdownL4;tma_nuke_group", + "MetricName": "tma_page_fault", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to a machine clear classified as a fast nuke due to memory ordering, memory disambiguation and memory renaming.", + "MetricExpr": "TOPDOWN_BAD_SPECULATION.FASTNUKE / SLOTS", + "MetricGroup": "TopdownL3;tma_machine_clears_group", + "MetricName": "tma_fast_nuke", + "ScaleUnit": "100%", "Unit": "cpu_atom" }, { "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "TOPDOWN_BE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE)", + "MetricExpr": "TOPDOWN_BE_BOUND.ALL / SLOTS", "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", + "MetricName": "tma_backend_bound", "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that uops must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. The rest of these subevents count backend stalls, in cycles, due to an outstanding request which is memory bound vs core bound. The subevents are not slot based events and therefore can not be precisely added or subtracted from the Backend_Bound_Aux subevents which are slot based.", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles due to backend bound stalls that are core execution bound and not attributed to outstanding demand load or store stalls. ", + "MetricExpr": "max(0, tma_backend_bound - tma_load_store_bound)", + "MetricGroup": "TopdownL2;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to stores or loads. ", + "MetricExpr": "min((TOPDOWN_BE_BOUND.ALL / SLOTS), (LD_HEAD.ANY_AT_RET / CLKS) + tma_store_bound)", + "MetricGroup": "TopdownL2;tma_backend_bound_group", + "MetricName": "tma_load_store_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to store buffer full.", + "MetricExpr": "tma_mem_scheduler * (MEM_SCHEDULER_BLOCK.ST_BUF / MEM_SCHEDULER_BLOCK.ALL)", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_store_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a load block.", + "MetricExpr": "LD_HEAD.L1_BOUND_AT_RET / CLKS", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_l1_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a store forward block.", + "MetricExpr": "LD_HEAD.ST_ADDR_AT_RET / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a first level TLB miss.", + "MetricExpr": "LD_HEAD.DTLB_MISS_AT_RET / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_stlb_hit", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a second level TLB miss requiring a page walk.", + "MetricExpr": "LD_HEAD.PGWALK_AT_RET / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_stlb_miss", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles that the oldest load of the load buffer is stalled at retirement due to a number of other load blocks.", + "MetricExpr": "LD_HEAD.OTHER_AT_RET / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_other_l1", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles a core is stalled due to a demand load which hit in the L2 Cache.", + "MetricExpr": "(MEM_BOUND_STALLS.LOAD_L2_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_L2_HIT / MEM_BOUND_STALLS.LOAD)", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_l2_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles a core is stalled due to a demand load which hit in the Last Level Cache (LLC) or other core with HITE/F/M.", + "MetricExpr": "(MEM_BOUND_STALLS.LOAD_LLC_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_LLC_HIT / MEM_BOUND_STALLS.LOAD)", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_l3_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load miss which hit in DRAM or MMIO (Non-DRAM).", + "MetricExpr": "(MEM_BOUND_STALLS.LOAD_DRAM_HIT / CLKS) - (MEM_BOUND_STALLS_AT_RET_CORRECTION * MEM_BOUND_STALLS.LOAD_DRAM_HIT / MEM_BOUND_STALLS.LOAD)", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_dram_bound", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles the core is stalled due to a demand load miss which hits in the L2, LLC, DRAM or MMIO (Non-DRAM) but could not be correctly attributed or cycles in which the load miss is waiting on a request buffer.", + "MetricExpr": "max(0, tma_load_store_bound - (tma_store_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_dram_bound))", + "MetricGroup": "TopdownL3;tma_load_store_bound_group", + "MetricName": "tma_other_load_store", + "ScaleUnit": "100%", "Unit": "cpu_atom" }, { "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls", - "MetricExpr": "(TOPDOWN_BE_BOUND.ALL / (5 * CPU_CLK_UNHALTED.CORE))", + "MetricExpr": "tma_backend_bound", "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound_Aux", + "MetricName": "tma_backend_bound_aux", "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that UOPS must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. All of these subevents count backend stalls, in slots, due to a resource limitation. These are not cycle based events and therefore can not be precisely added or subtracted from the Backend_Bound subevents which are cycle based. These subevents are supplementary to Backend_Bound and can be used to analyze results from a resource perspective at allocation. ", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls", + "MetricExpr": "tma_backend_bound", + "MetricGroup": "TopdownL2;tma_backend_bound_aux_group", + "MetricName": "tma_resource_bound", + "PublicDescription": "Counts the total number of issue slots that were not consumed by the backend due to backend stalls. Note that uops must be available for consumption in order for this event to count. If a uop is not available (IQ is empty), this event will not count. ", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to memory reservation stalls in which a scheduler is not able to accept uops.", + "MetricExpr": "TOPDOWN_BE_BOUND.MEM_SCHEDULER / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_mem_scheduler", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to store buffer full", + "MetricExpr": "tma_mem_scheduler * (MEM_SCHEDULER_BLOCK.ST_BUF / MEM_SCHEDULER_BLOCK.ALL)", + "MetricGroup": "TopdownL4;tma_mem_scheduler_group", + "MetricName": "tma_st_buffer", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to load buffer full", + "MetricExpr": "tma_mem_scheduler * MEM_SCHEDULER_BLOCK.LD_BUF / MEM_SCHEDULER_BLOCK.ALL", + "MetricGroup": "TopdownL4;tma_mem_scheduler_group", + "MetricName": "tma_ld_buffer", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cycles, relative to the number of mem_scheduler slots, in which uops are blocked due to RSV full relative ", + "MetricExpr": "tma_mem_scheduler * MEM_SCHEDULER_BLOCK.RSV / MEM_SCHEDULER_BLOCK.ALL", + "MetricGroup": "TopdownL4;tma_mem_scheduler_group", + "MetricName": "tma_rsv", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to IEC or FPC RAT stalls, which can be due to FIQ or IEC reservation stalls in which the integer, floating point or SIMD scheduler is not able to accept uops.", + "MetricExpr": "TOPDOWN_BE_BOUND.NON_MEM_SCHEDULER / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_non_mem_scheduler", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to the physical register file unable to accept an entry (marble stalls).", + "MetricExpr": "TOPDOWN_BE_BOUND.REGISTER / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_register", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to the reorder buffer being full (ROB stalls).", + "MetricExpr": "TOPDOWN_BE_BOUND.REORDER_BUFFER / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_reorder_buffer", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to certain allocation restrictions.", + "MetricExpr": "TOPDOWN_BE_BOUND.ALLOC_RESTRICTIONS / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_alloc_restriction", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of issue slots that were not consumed by the backend due to scoreboards from the instruction queue (IQ), jump execution unit (JEU), or microcode sequencer (MS).", + "MetricExpr": "TOPDOWN_BE_BOUND.SERIALIZATION / SLOTS", + "MetricGroup": "TopdownL3;tma_resource_bound_group", + "MetricName": "tma_serialization", + "ScaleUnit": "100%", "Unit": "cpu_atom" }, { "BriefDescription": "Counts the numer of issue slots that result in retirement slots. ", - "MetricExpr": "TOPDOWN_RETIRING.ALL / (5 * CPU_CLK_UNHALTED.CORE)", + "MetricExpr": "TOPDOWN_RETIRING.ALL / SLOTS", "MetricGroup": "TopdownL1", - "MetricName": "Retiring", + "MetricName": "tma_retiring", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops that are not from the microsequencer. ", + "MetricExpr": "(TOPDOWN_RETIRING.ALL - UOPS_RETIRED.MS) / SLOTS", + "MetricGroup": "TopdownL2;tma_retiring_group", + "MetricName": "tma_base", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of floating point operations per uop with all default weighting.", + "MetricExpr": "UOPS_RETIRED.FPDIV / SLOTS", + "MetricGroup": "TopdownL3;tma_base_group", + "MetricName": "tma_fp_uops", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops retired excluding ms and fp div uops.", + "MetricExpr": "(TOPDOWN_RETIRING.ALL - UOPS_RETIRED.MS - UOPS_RETIRED.FPDIV) / SLOTS", + "MetricGroup": "TopdownL3;tma_base_group", + "MetricName": "tma_other_ret", + "ScaleUnit": "100%", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS)", + "MetricExpr": "UOPS_RETIRED.MS / SLOTS", + "MetricGroup": "TopdownL2;tma_retiring_group", + "MetricName": "tma_ms_uops", + "PublicDescription": "Counts the number of uops that are from the complex flows issued by the micro-sequencer (MS). This includes uops from flows due to complex instructions, faults, assists, and inserted flows.", + "ScaleUnit": "100%", "Unit": "cpu_atom" }, { @@ -551,19 +1776,19 @@ }, { "BriefDescription": "", - "MetricExpr": "5 * CPU_CLK_UNHALTED.CORE", + "MetricExpr": "5 * CLKS", "MetricName": "SLOTS", "Unit": "cpu_atom" }, { "BriefDescription": "Instructions Per Cycle", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.CORE", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricName": "IPC", "Unit": "cpu_atom" }, { "BriefDescription": "Cycles Per Instruction", - "MetricExpr": "CPU_CLK_UNHALTED.CORE / INST_RETIRED.ANY", + "MetricExpr": "CLKS / INST_RETIRED.ANY", "MetricName": "CPI", "Unit": "cpu_atom" }, @@ -623,7 +1848,7 @@ }, { "BriefDescription": "Instructions per Far Branch", - "MetricExpr": "INST_RETIRED.ANY / ( BR_INST_RETIRED.FAR_BRANCH / 2 )", + "MetricExpr": "INST_RETIRED.ANY / (BR_INST_RETIRED.FAR_BRANCH / 2)", "MetricName": "IpFarBranch", "Unit": "cpu_atom" }, @@ -665,7 +1890,7 @@ }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.CORE / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricName": "Turbo_Utilization", "Unit": "cpu_atom" }, @@ -681,12 +1906,6 @@ "MetricName": "CPU_Utilization", "Unit": "cpu_atom" }, - { - "BriefDescription": "Estimated Pause cost. In percent", - "MetricExpr": "100 * SERIALIZATION.NON_C01_MS_SCB / (5 * CPU_CLK_UNHALTED.CORE)", - "MetricName": "Estimated_Pause_Cost", - "Unit": "cpu_atom" - }, { "BriefDescription": "Cycle cost per L2 hit", "MetricExpr": "MEM_BOUND_STALLS.LOAD_L2_HIT / MEM_LOAD_UOPS_RETIRED.L2_HIT", @@ -707,19 +1926,19 @@ }, { "BriefDescription": "Percent of instruction miss cost that hit in the L2", - "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_L2_HIT / ( MEM_BOUND_STALLS.IFETCH )", + "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_L2_HIT / (MEM_BOUND_STALLS.IFETCH)", "MetricName": "Inst_Miss_Cost_L2Hit_Percent", "Unit": "cpu_atom" }, { "BriefDescription": "Percent of instruction miss cost that hit in the L3", - "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_LLC_HIT / ( MEM_BOUND_STALLS.IFETCH )", + "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_LLC_HIT / (MEM_BOUND_STALLS.IFETCH)", "MetricName": "Inst_Miss_Cost_L3Hit_Percent", "Unit": "cpu_atom" }, { "BriefDescription": "Percent of instruction miss cost that hit in DRAM", - "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_DRAM_HIT / ( MEM_BOUND_STALLS.IFETCH )", + "MetricExpr": "100 * MEM_BOUND_STALLS.IFETCH_DRAM_HIT / (MEM_BOUND_STALLS.IFETCH)", "MetricName": "Inst_Miss_Cost_DRAMHit_Percent", "Unit": "cpu_atom" }, diff --git a/tools/perf/pmu-events/arch/x86/alderlake/cache.json b/tools/perf/pmu-events/arch/x86/alderlake/cache.json index 887dce4dfebacefc5d54ae0e13b4afe3692cd065..2cc62d2779d200cdde5499a9085bc6cb4f43b678 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/cache.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/cache.json @@ -1,4 +1,28 @@ [ + { + "BriefDescription": "Counts the number of cacheable memory requests that miss in the LLC. Counts on a per core basis.", + "CollectPEBSRecord": "2", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.MISS", + "PEBScounters": "0,1,2,3,4,5", + "SampleAfterValue": "200003", + "Speculative": "1", + "UMask": "0x41", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts the number of cacheable memory requests that access the LLC. Counts on a per core basis.", + "CollectPEBSRecord": "2", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x2e", + "EventName": "LONGEST_LAT_CACHE.REFERENCE", + "PEBScounters": "0,1,2,3,4,5", + "SampleAfterValue": "200003", + "Speculative": "1", + "UMask": "0x4f", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of cycles the core is stalled due to an instruction cache or TLB miss which hit in the L2, LLC, DRAM or MMIO (Non-DRAM).", "CollectPEBSRecord": "2", @@ -210,8 +234,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 128 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_128", @@ -219,7 +243,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x80", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -227,8 +251,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 16 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_16", @@ -236,7 +260,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x10", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -244,8 +268,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 256 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_256", @@ -253,7 +277,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x100", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -261,8 +285,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 32 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_32", @@ -270,7 +294,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x20", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -278,8 +302,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 4 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_4", @@ -287,7 +311,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x4", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -295,8 +319,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 512 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_512", @@ -304,7 +328,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x200", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -312,8 +336,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 64 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_64", @@ -321,7 +345,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x40", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -329,8 +353,8 @@ }, { "BriefDescription": "Counts the number of tagged loads with an instruction latency that exceeds or equals the threshold of 8 cycles as defined in MEC_CR_PEBS_LD_LAT_THRESHOLD (3F6H). Only counts with PEBS enabled.", - "CollectPEBSRecord": "3", - "Counter": "0,1,2,3,4,5", + "CollectPEBSRecord": "2", + "Counter": "0,1", "Data_LA": "1", "EventCode": "0xd0", "EventName": "MEM_UOPS_RETIRED.LOAD_LATENCY_GT_8", @@ -338,7 +362,7 @@ "MSRIndex": "0x3F6", "MSRValue": "0x8", "PEBS": "2", - "PEBScounters": "0,1,2,3,4,5", + "PEBScounters": "0,1", "SampleAfterValue": "1000003", "TakenAlone": "1", "UMask": "0x5", @@ -359,7 +383,7 @@ }, { "BriefDescription": "Counts the number of stores uops retired. Counts with or without PEBS enabled.", - "CollectPEBSRecord": "3", + "CollectPEBSRecord": "2", "Counter": "0,1,2,3,4,5", "Data_LA": "1", "EventCode": "0xd0", @@ -371,6 +395,61 @@ "UMask": "0x6", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F803C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, but no data was forwarded.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_NO_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x4003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand data reads that were supplied by the L3 cache where a snoop was sent, the snoop hit, and non-modified data was forwarded.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x8003C0001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were supplied by the L3 cache.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_RFO.L3_HIT", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F803C0002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were supplied by the L3 cache where a snoop was sent, the snoop hit, and modified data was forwarded.", "Counter": "0,1,2,3,4,5", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/frontend.json b/tools/perf/pmu-events/arch/x86/alderlake/frontend.json index 2cfa70b2d5e1f47e7555ab6f954669bb73f2e5d1..da1a7ba0e5681d43794ee93b27018b477976457d 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/frontend.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/frontend.json @@ -47,6 +47,18 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Cycles the Microcode Sequencer is busy.", + "CollectPEBSRecord": "2", + "Counter": "0,1,2,3", + "EventCode": "0x87", + "EventName": "DECODE.MS_BUSY", + "PEBScounters": "0,1,2,3", + "SampleAfterValue": "500009", + "Speculative": "1", + "UMask": "0x2", + "Unit": "cpu_core" + }, { "BriefDescription": "DSB-to-MITE switch true penalty cycles.", "CollectPEBSRecord": "2", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/memory.json b/tools/perf/pmu-events/arch/x86/alderlake/memory.json index 586fb961e46dd62d48916c08a1086310ca1a1e23..f894e4a0212b2412a59635ce6f379c57c2dfecab 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/memory.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/memory.json @@ -82,6 +82,17 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts demand data reads that were not supplied by the L3 cache.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_DATA_RD.L3_MISS_LOCAL", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F84400001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", "Counter": "0,1,2,3,4,5", @@ -93,6 +104,17 @@ "UMask": "0x1", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts demand reads for ownership (RFO) and software prefetches for exclusive ownership (PREFETCHW) that were not supplied by the L3 cache.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.DEMAND_RFO.L3_MISS_LOCAL", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x3F84400002", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Execution stalls while L3 cache miss demand load is outstanding.", "CollectPEBSRecord": "2", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/other.json b/tools/perf/pmu-events/arch/x86/alderlake/other.json index 67a9c13cc71daa7c1a3c31d937dbb4f9acccd32d..c49d8ce273100505bb10115ec843b6c8ef9b5aef 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/other.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/other.json @@ -1,4 +1,15 @@ [ + { + "BriefDescription": "Counts modified writebacks from L1 cache and L2 cache that have any type of response.", + "Counter": "0,1,2,3,4,5", + "EventCode": "0xB7", + "EventName": "OCR.COREWB_M.ANY_RESPONSE", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x10008", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts demand data reads that have any type of response.", "Counter": "0,1,2,3,4,5", @@ -103,6 +114,17 @@ "UMask": "0x1", "Unit": "cpu_core" }, + { + "BriefDescription": "Counts demand data reads that were supplied by DRAM.", + "Counter": "0,1,2,3,4,5,6,7", + "EventCode": "0x2A,0x2B", + "EventName": "OCR.DEMAND_DATA_RD.DRAM", + "MSRIndex": "0x1a6,0x1a7", + "MSRValue": "0x184000001", + "SampleAfterValue": "100003", + "UMask": "0x1", + "Unit": "cpu_core" + }, { "BriefDescription": "Counts demand read for ownership (RFO) requests and software prefetches for exclusive ownership (PREFETCHW) that have any type of response.", "Counter": "0,1,2,3,4,5,6,7", diff --git a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json index d02e078a90c94b60a000b33987c984bf85630b3b..1a137f7f8b7e8eb3f1c6f3a9ce09cf1766c129db 100644 --- a/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/alderlake/pipeline.json @@ -330,6 +330,18 @@ "UMask": "0x3", "Unit": "cpu_atom" }, + { + "BriefDescription": "Counts the number of unhalted reference clock cycles at TSC frequency.", + "CollectPEBSRecord": "2", + "Counter": "0,1,2,3,4,5", + "EventCode": "0x3c", + "EventName": "CPU_CLK_UNHALTED.REF_TSC_P", + "PEBScounters": "0,1,2,3,4,5", + "SampleAfterValue": "2000003", + "Speculative": "1", + "UMask": "0x1", + "Unit": "cpu_atom" + }, { "BriefDescription": "Counts the number of unhalted core clock cycles. (Fixed event)", "CollectPEBSRecord": "2", @@ -874,7 +886,7 @@ "PEBScounters": "0,1,2,3,4,5,6,7", "SampleAfterValue": "100003", "Speculative": "1", - "UMask": "0x1f", + "UMask": "0x1b", "Unit": "cpu_core" }, { diff --git a/tools/perf/pmu-events/arch/x86/broadwell/bdw-metrics.json b/tools/perf/pmu-events/arch/x86/broadwell/bdw-metrics.json index d65afe3d0b062070f77da9dcda4c62b425168442..c220b1cf1740d803bef5b2eb9c4eb78cee64a796 100644 --- a/tools/perf/pmu-events/arch/x86/broadwell/bdw-metrics.json +++ b/tools/perf/pmu-events/arch/x86/broadwell/bdw-metrics.json @@ -1,64 +1,552 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. ", + "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. ", + "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -76,8 +564,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -88,16 +576,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -107,51 +589,32 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -193,13 +656,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -220,22 +683,22 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -252,7 +715,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -264,83 +727,71 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "(cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * (DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED)) / CORE_CLKS", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -361,19 +812,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -391,26 +842,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -428,7 +879,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json index b6fdf5ba2c9ae3a14a72d2096b5180dd7a2f0204..5a074cf7c77da654edbb7b547357a0f46bdaff17 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json +++ b/tools/perf/pmu-events/arch/x86/broadwellde/bdwde-metrics.json @@ -1,64 +1,556 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU)", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads)", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads)", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data)", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address)", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -76,8 +568,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -88,16 +580,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -107,51 +593,32 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -193,13 +660,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -220,22 +687,22 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -252,7 +719,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -264,83 +731,71 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * (( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) if #core_wide < 1 else ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD) )", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -361,19 +816,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -391,26 +846,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -428,33 +883,21 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { - "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )", - "MetricGroup": "Mem;MemoryLat;SoC", - "MetricName": "MEM_Read_Latency" - }, - { - "BriefDescription": "Average number of parallel data read requests to external memory. Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182\\,thresh\\=1@", - "MetricGroup": "Mem;MemoryBW;SoC", - "MetricName": "MEM_Parallel_Reads" - }, - { - "BriefDescription": "Socket actual clocks when any core is active on that socket", - "MetricExpr": "cbox_0@event\\=0x0@", - "MetricGroup": "SoC", - "MetricName": "Socket_CLKS" + "BriefDescription": "Average latency of all requests to external memory (in Uncore cycles)", + "MetricExpr": "UNC_ARB_TRK_OCCUPANCY.ALL / arb@event\\=0x81\\,umask\\=0x1@", + "MetricGroup": "Mem;SoC", + "MetricName": "MEM_Request_Latency" }, { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" + "BriefDescription": "Average number of parallel requests to external memory. Accounts for all requests", + "MetricExpr": "UNC_ARB_TRK_OCCUPANCY.ALL / arb@event\\=0x81\\,umask\\=0x1@", + "MetricGroup": "Mem;SoC", + "MetricName": "MEM_Parallel_Requests" }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", diff --git a/tools/perf/pmu-events/arch/x86/broadwellx/bdx-metrics.json b/tools/perf/pmu-events/arch/x86/broadwellx/bdx-metrics.json index a3a15ee5284177cce463de290437cf2ef84f18b1..e89fa536ca030724d9392c51962bce11f99c79b4 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellx/bdx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/broadwellx/bdx-metrics.json @@ -1,64 +1,576 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * ITLB_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. ", + "MetricExpr": "BR_MISP_RETIRED.ALL_BRANCHES * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. ", + "MetricExpr": "MACHINE_CLEARS.COUNT * tma_branch_resteers / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY)", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "tma_branch_resteers - tma_mispredicts_resteers - tma_clears_resteers", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB) / (CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + cpu@DTLB_LOAD_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_LOAD_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_MISS / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "310 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "(200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + cpu@DTLB_STORE_MISSES.WALK_DURATION\\,cmask\\=1@ + 7 * DTLB_STORE_MISSES.WALK_COMPLETED) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (CYCLE_ACTIVITY.STALLS_TOTAL - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -74,6 +586,12 @@ "MetricGroup": "Branches;Fed;FetchBW", "MetricName": "UpTB" }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -82,16 +600,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -101,51 +613,32 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -187,13 +680,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -214,22 +707,22 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -246,7 +739,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -258,83 +751,71 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (BR_MISP_RETIRED.ALL_BRANCHES * (12 * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / CPU_CLK_UNHALTED.THREAD) / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY )) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * (DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED)) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION + 7 * ( DTLB_STORE_MISSES.WALK_COMPLETED + DTLB_LOAD_MISSES.WALK_COMPLETED + ITLB_MISSES.WALK_COMPLETED ) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -355,19 +836,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -385,26 +866,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -422,13 +903,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )", + "MetricExpr": "1000000000 * (cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -444,12 +925,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -498,20 +973,19 @@ "MetricGroup": "Power", "MetricName": "C7_Pkg_Residency" }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" + }, { "BriefDescription": "CPU operating frequency (in GHz)", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000", + "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", "MetricExpr": "MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY", @@ -530,7 +1004,7 @@ "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { @@ -558,7 +1032,7 @@ "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { @@ -591,21 +1065,21 @@ }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds", - "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds", - "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds", - "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) ) ) * duration_time", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests", "ScaleUnit": "1ns" @@ -640,21 +1114,21 @@ }, { "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", - "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )", + "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", - "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )", + "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Uncore operating frequency in GHz", - "MetricExpr": "UNC_C_CLOCKTICKS / ( source_count(UNC_C_CLOCKTICKS) * #num_packages ) / 1000000000", + "MetricExpr": "( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "uncore_frequency", "ScaleUnit": "1GHz" @@ -663,7 +1137,7 @@ "BriefDescription": "Intel(R) Quick Path Interconnect (QPI) data transmit bandwidth (MB/sec)", "MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "qpi_data_transmit_bw_only_data", + "MetricName": "qpi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { @@ -691,245 +1165,42 @@ "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", "MetricExpr": "(( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ + cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x180\\,filter_tid\\=0x3e@ ) * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_decoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from loop stream detector(LSD) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( LSD.UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_loop_stream_detector_lsd", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1;PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ICACHE.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ( 14 * ITLB_MISSES.STLB_HIT + cpu@ITLB_MISSES.WALK_DURATION\\,cmask\\=0x1@ + 7 * ITLB_MISSES.WALK_COMPLETED ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( ( 12 ) * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "100 * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB ) / ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) * CYCLE_ACTIVITY.STALLS_L2_MISS / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( 1 - ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) ) * CYCLE_ACTIVITY.STALLS_L2_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_dram_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( RESOURCE_STALLS.SB / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "100 * ( ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + RESOURCE_STALLS.SB ) / ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( ARITH.FPU_DIV_ACTIVE / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", - "MetricExpr": "100 * ( ( ( ( CYCLE_ACTIVITY.STALLS_TOTAL + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - ( UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) - RESOURCE_STALLS.SB - CYCLE_ACTIVITY.STALLS_MEM_ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_ports_utilization_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", - "MetricExpr": "100 * ( ( INST_RETIRED.X87 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / INST_RETIRED.ANY ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )", - "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fp_arith_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_microcode_sequencer_percent", + "MetricName": "percent_uops_delivered_from_loop_stream_detector", "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-cache.json b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-cache.json index abee6f773c1fd23938db405940fa3cffb7edf410..449fa723d0aa7bc10ee7d8c915720b205eb02188 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-cache.json +++ b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-cache.json @@ -947,21 +947,19 @@ "Unit": "CBO" }, { - "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches. Derived from unc_c_tor_inserts.miss_opcode", + "BriefDescription": "TOR Inserts; Miss Opcode Match", "Counter": "0,1,2,3", "EventCode": "0x35", - "EventName": "LLC_MISSES.DATA_READ", - "Filter": "filter_opc=0x182", + "EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0x3", "Unit": "CBO" }, { - "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches", + "BriefDescription": "LLC misses - demand and prefetch data reads - excludes LLC prefetches. Derived from unc_c_tor_inserts.miss_opcode", "Counter": "0,1,2,3", "EventCode": "0x35", - "EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE", + "EventName": "LLC_MISSES.DATA_READ", "Filter": "filter_opc=0x182", "PerPkg": "1", "ScaleUnit": "64Bytes", diff --git a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json index 071ce45620d24f2e465b7fbe31d8dc584577649e..cb1916f5260744bff814697f5e8fc5205dd9f1d4 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-interconnect.json @@ -685,36 +685,34 @@ "Unit": "QPI LL" }, { - "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data", + "BriefDescription": "Flits Transferred - Group 0; Data Tx Flits", "Counter": "0,1,2,3", - "EventName": "QPI_DATA_BANDWIDTH_TX", + "EventName": "UNC_Q_TxL_FLITS_G0.DATA", "PerPkg": "1", - "ScaleUnit": "8Bytes", "UMask": "0x2", "Unit": "QPI LL" }, { - "BriefDescription": "Number of data flits transmitted ", + "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data", "Counter": "0,1,2,3", - "EventName": "UNC_Q_TxL_FLITS_G0.DATA", + "EventName": "QPI_DATA_BANDWIDTH_TX", "PerPkg": "1", "ScaleUnit": "8Bytes", "UMask": "0x2", "Unit": "QPI LL" }, { - "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data", + "BriefDescription": "Flits Transferred - Group 0; Non-Data protocol Tx Flits", "Counter": "0,1,2,3", - "EventName": "QPI_CTL_BANDWIDTH_TX", + "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA", "PerPkg": "1", - "ScaleUnit": "8Bytes", "UMask": "0x4", "Unit": "QPI LL" }, { - "BriefDescription": "Number of non data (control) flits transmitted ", + "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data", "Counter": "0,1,2,3", - "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA", + "EventName": "QPI_CTL_BANDWIDTH_TX", "PerPkg": "1", "ScaleUnit": "8Bytes", "UMask": "0x4", diff --git a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-memory.json b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-memory.json index 302e956a82ed156f8a73ae315dfddcd70eb07f4c..05fab7d2723ea252d133368907990690faf4b0c9 100644 --- a/tools/perf/pmu-events/arch/x86/broadwellx/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/broadwellx/uncore-memory.json @@ -72,20 +72,19 @@ "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM Reads (RD_CAS + Underfills)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_READ", + "EventName": "UNC_M_CAS_COUNT.RD", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0x3", "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller", + "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.RD", + "EventName": "LLC_MISSES.MEM_READ", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0x3", @@ -110,20 +109,19 @@ "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM WR_CAS (both Modes)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_WRITE", + "EventName": "UNC_M_CAS_COUNT.WR", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0xC", "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller", + "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.WR", + "EventName": "LLC_MISSES.MEM_WRITE", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0xC", diff --git a/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json b/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json index 46613504b816ba413c4fde6dc3fdfaa36df4e4be..81de1149297daf43547fd2b297d6f3e677e25ecd 100644 --- a/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/cascadelakex/clx-metrics.json @@ -1,148 +1,742 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "9 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (11 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OCR.DEMAND_DATA_RD.L3_HIT.HIT_OTHER_CORE_FWD))) + (44 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OCR.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OCR.DEMAND_DATA_RD.L3_HIT.HIT_OTHER_CORE_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(17 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound) - tma_pmm_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "(59.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "(127 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "((89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a", + "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)", + "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_pmm_bound", + "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 11 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "((110 * Average_Frequency) * (OCR.DEMAND_RFO.L3_MISS.REMOTE_HITM + OCR.PF_L2_RFO.L3_MISS.REMOTE_HITM) + (47.5 * Average_Frequency) * (OCR.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE + OCR.PF_L2_RFO.L3_HIT.HITM_OTHER_CORE)) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "40 * ROB_MISC_EVENTS.PAUSE_INST / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" }, { - "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts", - "MetricName": "Mispredictions" + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions", + "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fused_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused", + "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_non_fused_branches", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT", - "MetricName": "Mispredictions_SMT" + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" }, { "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "Memory_Bandwidth" }, - { - "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", - "MetricGroup": "Mem;MemoryBW;Offcore_SMT", - "MetricName": "Memory_Bandwidth_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))", "MetricGroup": "Mem;MemoryLat;Offcore", "MetricName": "Memory_Latency" }, - { - "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) - ( ( ( 1 - ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) / ( ( 19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + 10 * ( (MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) ) ) + ( 25 * ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) + 33 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) ) ) ) ) ) ) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) if ( 1000000 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )", - "MetricGroup": "Mem;MemoryLat;Offcore_SMT", - "MetricName": "Memory_Latency_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ", "MetricGroup": "Mem;MemoryTLB;Offcore", "MetricName": "Memory_Data_TLBs" }, - { - "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", - "MetricGroup": "Mem;MemoryTLB;Offcore_SMT", - "MetricName": "Memory_Data_TLBs_SMT" - }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, - { - "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "Ret_SMT", - "MetricName": "Branching_Overhead_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT", - "MetricName": "Big_Code_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", "MetricGroup": "Fed;FetchBW;Frontend", "MetricName": "Instruction_Fetch_BW" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))", - "MetricGroup": "Fed;FetchBW;Frontend_SMT", - "MetricName": "Instruction_Fetch_BW_SMT" - }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -158,6 +752,12 @@ "MetricGroup": "Branches;Fed;FetchBW", "MetricName": "UpTB" }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -166,16 +766,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -185,63 +779,38 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", "MetricGroup": "Cor;SMT", "MetricName": "Core_Bound_Likely" }, - { - "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0", - "MetricGroup": "Cor;SMT_SMT", - "MetricName": "Core_Bound_Likely_SMT" - }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -283,13 +852,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -310,21 +879,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -336,9 +905,9 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -373,16 +942,10 @@ }, { "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))", "MetricGroup": "DSBmiss;Fed", "MetricName": "DSB_Misses" }, - { - "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )", - "MetricGroup": "DSBmiss;Fed_SMT", - "MetricName": "DSB_Misses_SMT" - }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -397,16 +960,10 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -415,101 +972,95 @@ }, { "BriefDescription": "Fraction of branches that are taken conditionals", - "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches;CodeGen;PGO", "MetricName": "Cond_TK" }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, { "BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps", - "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "Jump" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -536,37 +1087,37 @@ }, { "BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)", - "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_Silent_PKI" }, { "BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction", - "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_NonSilent_PKI" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -578,68 +1129,47 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License0_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License0_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License1_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License1_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License2_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License2_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -657,13 +1187,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@ ) / ( cha_0@event\\=0x0@ / duration_time )", + "MetricExpr": "1000000000 * (cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -675,38 +1205,38 @@ }, { "BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": "( 1000000000 * ( imc@event\\=0xe0\\,umask\\=0x1@ / imc@event\\=0xe3@ ) / imc_0@event\\=0x0@ )", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": "(1000000000 * (imc@event\\=0xe0\\,umask\\=0x1@ / imc@event\\=0xe3@) / imc_0@event\\=0x0@)", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_PMM_Read_Latency" }, { "BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": "1000000000 * ( UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS ) / imc_0@event\\=0x0@", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": "1000000000 * (UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS) / imc_0@event\\=0x0@", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_DRAM_Read_Latency" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]", - "MetricExpr": "( ( 64 * imc@event\\=0xe3@ / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * imc@event\\=0xe3@ / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Read_BW" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]", - "MetricExpr": "( ( 64 * imc@event\\=0xe7@ / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * imc@event\\=0xe7@ / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]", - "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3) * 4 / 1000000000 / duration_time", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]", - "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3 ) * 4 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3) * 4 / 1000000000 / duration_time", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Read_BW" }, { @@ -715,12 +1245,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -770,26 +1294,18 @@ "MetricName": "C7_Pkg_Residency" }, { - "BriefDescription": "Percentage of time spent in the active CPU power state C0", - "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC", - "MetricGroup": "", - "MetricName": "cpu_utilization_percent", - "ScaleUnit": "1%" + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" }, { "BriefDescription": "CPU operating frequency (in GHz)", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000", + "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", "MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY", @@ -808,7 +1324,7 @@ "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { @@ -836,7 +1352,7 @@ "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { @@ -869,21 +1385,21 @@ }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds", - "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests", "ScaleUnit": "1ns" @@ -892,54 +1408,54 @@ "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.", "MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "itlb_2nd_level_mpi", + "MetricName": "itlb_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.", "MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "itlb_2nd_level_large_page_mpi", + "MetricName": "itlb_large_page_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_load_mpi", + "MetricName": "dtlb_load_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for 2 megabyte page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the Data Translation Lookaside Buffer (DTLB) and further levels of TLB.", "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_2mb_large_page_load_mpi", + "MetricName": "dtlb_2mb_large_page_load_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", "MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_store_mpi", + "MetricName": "dtlb_store_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Uncore operating frequency in GHz", - "MetricExpr": "UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000", + "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "uncore_frequency", "ScaleUnit": "1GHz" @@ -948,7 +1464,7 @@ "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)", "MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "upi_data_transmit_bw_only_data", + "MetricName": "upi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { @@ -997,35 +1513,35 @@ "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", "MetricExpr": "(( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", "MetricExpr": "(( UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART0 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART1 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART2 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART3 ) * 4 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_decoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { @@ -1050,255 +1566,10 @@ "ScaleUnit": "1MB/s" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1;PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 9 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "100 * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_dram_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", - "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_pmm_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "100 * ( ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", - "MetricExpr": "100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_ports_utilization_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )", - "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fp_arith_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_memory_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fused_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_non_fused_branches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_nop_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", - "MetricExpr": "100 * ( max( 0 , ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) - ( ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) ) ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_other_light_ops_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_few_uops_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_microcode_sequencer_percent", + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.", + "MetricExpr": "100 * ( ( LSD.CYCLES_ACTIVE - LSD.CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", + "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group", + "MetricName": "tma_lsd", "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-memory.json b/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-memory.json index 6facfb244cd32ac1dd9c03517b78c216dce8ddc8..326b674045c6855821aecec9125ffb8ed2128006 100644 --- a/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-memory.json @@ -27,20 +27,19 @@ "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", + "BriefDescription": "All DRAM Read CAS Commands issued (including underfills)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_READ", + "EventName": "UNC_M_CAS_COUNT.RD", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0x3", "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller", + "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.RD", + "EventName": "LLC_MISSES.MEM_READ", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0x3", @@ -56,20 +55,19 @@ "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", + "BriefDescription": "All DRAM Write CAS commands issued", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_WRITE", + "EventName": "UNC_M_CAS_COUNT.WR", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0xC", "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller", + "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.WR", + "EventName": "LLC_MISSES.MEM_WRITE", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0xC", diff --git a/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-other.json b/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-other.json index a29bba230f4961df54f25ba00f1fc28acdad47a5..e10530c21ef8b997a80283cbd5c99d87d23f0d38 100644 --- a/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-other.json +++ b/tools/perf/pmu-events/arch/x86/cascadelakex/uncore-other.json @@ -1477,7 +1477,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x01", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1489,7 +1488,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x02", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1501,7 +1499,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x04", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1513,7 +1510,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x08", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1584,7 +1580,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x01", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1596,7 +1591,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x02", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1608,7 +1602,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x04", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1620,7 +1613,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x08", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -2254,7 +2246,7 @@ "Unit": "UPI LL" }, { - "BriefDescription": "FLITs received which bypassed the Slot0 Receive Buffer", + "BriefDescription": "FLITs received which bypassed the Slot0 Recieve Buffer", "Counter": "0,1,2,3", "EventCode": "0x31", "EventName": "UNC_UPI_RxL_BYPASSED.SLOT2", diff --git a/tools/perf/pmu-events/arch/x86/haswell/cache.json b/tools/perf/pmu-events/arch/x86/haswell/cache.json index 3b0f3a2642469b3e01f3e4bb0237725ea5e3917c..719b8e622f59663bdf9dc5790471e7a2a1e496b6 100644 --- a/tools/perf/pmu-events/arch/x86/haswell/cache.json +++ b/tools/perf/pmu-events/arch/x86/haswell/cache.json @@ -20,7 +20,7 @@ "UMask": "0x2" }, { - "BriefDescription": "L1D miss oustandings duration in cycles", + "BriefDescription": "L1D miss outstanding duration in cycles", "Counter": "2", "CounterHTOff": "2", "EventCode": "0x48", @@ -655,7 +655,7 @@ "UMask": "0x8" }, { - "BriefDescription": "Cacheable and noncachaeble code read requests", + "BriefDescription": "Cacheable and noncacheable code read requests", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0xB0", diff --git a/tools/perf/pmu-events/arch/x86/haswell/frontend.json b/tools/perf/pmu-events/arch/x86/haswell/frontend.json index c45a09abe5d3fd70c5b052851fd28023eb33a0a7..18a993297108cdd702fe83a2aeb144726c60b477 100644 --- a/tools/perf/pmu-events/arch/x86/haswell/frontend.json +++ b/tools/perf/pmu-events/arch/x86/haswell/frontend.json @@ -161,7 +161,7 @@ "UMask": "0x4" }, { - "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -172,7 +172,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -182,7 +182,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.", + "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -193,7 +193,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -203,7 +203,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -224,7 +224,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", diff --git a/tools/perf/pmu-events/arch/x86/haswell/hsw-metrics.json b/tools/perf/pmu-events/arch/x86/haswell/hsw-metrics.json index 75dc6dd9a7bcb115c5c996146a1e4c2801d113ce..6cb6603efbd8fc859df5c5234b1077d946a19b1e 100644 --- a/tools/perf/pmu-events/arch/x86/haswell/hsw-metrics.json +++ b/tools/perf/pmu-events/arch/x86/haswell/hsw-metrics.json @@ -1,64 +1,490 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.REQUEST_FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.L3_MISS))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "10 * ARITH.DIVIDER_UOPS / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -76,8 +502,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -88,37 +514,25 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "( UOPS_EXECUTED.CORE / 2 / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@) ) if #SMT_on else UOPS_EXECUTED.CORE / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)", + "MetricExpr": "(UOPS_EXECUTED.CORE / 2 / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)) if #SMT_on else UOPS_EXECUTED.CORE / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -159,9 +573,9 @@ "MetricName": "BpTkBranch" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -172,7 +586,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -184,47 +598,41 @@ }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -245,19 +653,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -275,19 +683,19 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -305,7 +713,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/haswellx/cache.json b/tools/perf/pmu-events/arch/x86/haswellx/cache.json index 7557a203a1b6633c3dd36f74e0046bb88e8207d0..427c949bed6eda1feb88666db4351f3be457a8df 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/cache.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/cache.json @@ -691,7 +691,7 @@ "UMask": "0x8" }, { - "BriefDescription": "Cacheable and noncachaeble code read requests", + "BriefDescription": "Cacheable and noncacheable code read requests", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0xB0", diff --git a/tools/perf/pmu-events/arch/x86/haswellx/frontend.json b/tools/perf/pmu-events/arch/x86/haswellx/frontend.json index c45a09abe5d3fd70c5b052851fd28023eb33a0a7..18a993297108cdd702fe83a2aeb144726c60b477 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/frontend.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/frontend.json @@ -161,7 +161,7 @@ "UMask": "0x4" }, { - "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -172,7 +172,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -182,7 +182,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.", + "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", @@ -193,7 +193,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -203,7 +203,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -224,7 +224,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", diff --git a/tools/perf/pmu-events/arch/x86/haswellx/hsx-metrics.json b/tools/perf/pmu-events/arch/x86/haswellx/hsx-metrics.json index d31d76db9d84d90dbc9a91a0201df0429888fe61..2cd86750986af5a159a1da147a570e388e5d825e 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/hsx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/hsx-metrics.json @@ -1,64 +1,514 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(8 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.REQUEST_FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.L3_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.L3_HIT / (MEM_LOAD_UOPS_RETIRED.L3_HIT + 7 * MEM_LOAD_UOPS_RETIRED.L3_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "310 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "(200 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.L3_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_L3_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(8 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "10 * ARITH.DIVIDER_UOPS / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "INST_RETIRED.X87 * UPI / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -74,6 +524,12 @@ "MetricGroup": "Branches;Fed;FetchBW", "MetricName": "UpTB" }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -82,37 +538,25 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "( UOPS_EXECUTED.CORE / 2 / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@) ) if #SMT_on else UOPS_EXECUTED.CORE / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)", + "MetricExpr": "(UOPS_EXECUTED.CORE / 2 / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)) if #SMT_on else UOPS_EXECUTED.CORE / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -153,9 +597,9 @@ "MetricName": "BpTkBranch" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -166,7 +610,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -178,47 +622,41 @@ }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -239,19 +677,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -269,19 +707,19 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -299,13 +737,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@ ) / ( cbox_0@event\\=0x0@ / duration_time )", + "MetricExpr": "1000000000 * (cbox@event\\=0x36\\,umask\\=0x3\\,filter_opc\\=0x182@ / cbox@event\\=0x35\\,umask\\=0x3\\,filter_opc\\=0x182@) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -321,12 +759,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -375,403 +807,234 @@ "MetricGroup": "Power", "MetricName": "C7_Pkg_Residency" }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" + }, { "BriefDescription": "CPU operating frequency (in GHz)", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000", + "MetricExpr": "(( CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC * #SYSTEM_TSC_FREQ ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": " CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY ", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", - "MetricExpr": " MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY ", + "MetricExpr": "MEM_UOPS_RETIRED.ALL_LOADS / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "loads_per_instr", "ScaleUnit": "1per_instr" }, { "BriefDescription": "The ratio of number of completed memory store instructions to the total number completed instructions", - "MetricExpr": " MEM_UOPS_RETIRED.ALL_STORES / INST_RETIRED.ANY ", + "MetricExpr": "MEM_UOPS_RETIRED.ALL_STORES / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "stores_per_instr", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", - "MetricExpr": " L1D.REPLACEMENT / INST_RETIRED.ANY ", + "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of demand load requests hitting in L1 data cache to the total number of completed instructions", - "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L1_HIT / INST_RETIRED.ANY ", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L1_HIT / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "l1d_demand_data_read_hits_per_instr", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of code read requests missing in L1 instruction cache (includes prefetches) to the total number of completed instructions", - "MetricExpr": " L2_RQSTS.ALL_CODE_RD / INST_RETIRED.ANY ", + "MetricExpr": "L2_RQSTS.ALL_CODE_RD / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "l1_i_code_read_misses_with_prefetches_per_instr", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed demand load requests hitting in L2 cache to the total number of completed instructions", - "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L2_HIT / INST_RETIRED.ANY ", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_HIT / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "l2_demand_data_read_hits_per_instr", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", - "MetricExpr": " L2_LINES_IN.ALL / INST_RETIRED.ANY ", + "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed data read request missing L2 cache to the total number of completed instructions", - "MetricExpr": " MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY ", + "MetricExpr": "MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "l2_demand_data_read_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of code read request missing L2 cache to the total number of completed instructions", - "MetricExpr": " L2_RQSTS.CODE_RD_MISS / INST_RETIRED.ANY ", + "MetricExpr": "L2_RQSTS.CODE_RD_MISS / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "l2_demand_code_mpi", "ScaleUnit": "1per_instr" }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency", + "ScaleUnit": "1ns" + }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests", + "ScaleUnit": "1ns" + }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds", + "MetricExpr": "( 1000000000 * ( cbox@UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ ) / ( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests", + "ScaleUnit": "1ns" + }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.", - "MetricExpr": " ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ", + "MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "itlb_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.", - "MetricExpr": " ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY ", + "MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "itlb_large_page_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", - "MetricExpr": " DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "dtlb_load_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", - "MetricExpr": " DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY ", + "MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "dtlb_store_mpi", "ScaleUnit": "1per_instr" }, + { + "BriefDescription": "Uncore operating frequency in GHz", + "MetricExpr": "( UNC_C_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time", + "MetricGroup": "", + "MetricName": "uncore_frequency", + "ScaleUnit": "1GHz" + }, { "BriefDescription": "Intel(R) Quick Path Interconnect (QPI) data transmit bandwidth (MB/sec)", - "MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time", + "MetricExpr": "( UNC_Q_TxL_FLITS_G0.DATA * 8 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "qpi_data_transmit_bw_only_data", + "MetricName": "qpi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { "BriefDescription": "DDR memory read bandwidth (MB/sec)", - "MetricExpr": "( UNC_M_CAS_COUNT.RD * 64 / 1000000) / duration_time", + "MetricExpr": "( UNC_M_CAS_COUNT.RD * 64 / 1000000) / duration_time", "MetricGroup": "", "MetricName": "memory_bandwidth_read", "ScaleUnit": "1MB/s" }, { "BriefDescription": "DDR memory write bandwidth (MB/sec)", - "MetricExpr": "( UNC_M_CAS_COUNT.WR * 64 / 1000000) / duration_time", + "MetricExpr": "( UNC_M_CAS_COUNT.WR * 64 / 1000000) / duration_time", "MetricGroup": "", "MetricName": "memory_bandwidth_write", "ScaleUnit": "1MB/s" }, { "BriefDescription": "DDR memory bandwidth (MB/sec)", - "MetricExpr": "(( UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR ) * 64 / 1000000) / duration_time", + "MetricExpr": "(( UNC_M_CAS_COUNT.RD + UNC_M_CAS_COUNT.WR ) * 64 / 1000000) / duration_time", "MetricGroup": "", "MetricName": "memory_bandwidth_total", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", - "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time", + "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x19e@ * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", - "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ * 64 / 1000000) / duration_time", + "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.OPCODE\\,filter_opc\\=0x1c8\\,filter_tid\\=0x3e@ * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", - "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )", + "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_frodecoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", - "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )", + "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_frolegacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", - "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )", + "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_fromicrocode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from loop stream detector(LSD) as a percent of total uops delivered to Instruction Decode Queue", - "MetricExpr": "100 * ( UOPS_ISSUED.ANY - IDQ.MITE_UOPS - IDQ.MS_UOPS - IDQ.DSB_UOPS ) / UOPS_ISSUED.ANY ", + "MetricExpr": "100 * ( UOPS_ISSUED.ANY - IDQ.MITE_UOPS - IDQ.MS_UOPS - IDQ.DSB_UOPS ) / UOPS_ISSUED.ANY", "MetricGroup": "", - "MetricName": "percent_uops_delivered_froloop_streadetector_lsd", + "MetricName": "percent_uops_delivered_from_loop_stream_detector", "ScaleUnit": "1%" }, { "BriefDescription": "Ratio of number of data read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", - "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x192@ ) / INST_RETIRED.ANY ", + "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x192@ ) / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "llc_data_read_mpi_demand_plus_prefetch", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", - "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x181@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x191@ ) / INST_RETIRED.ANY ", + "MetricExpr": "( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x181@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x191@ ) / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "llc_code_read_mpi_demand_plus_prefetch", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", - "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )", + "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", - "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_OPCODE\\,filter_opc\\=0x182@ )", - "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1, PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Frontend, TmaL2", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ICACHE.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot, FetchLat, IcMiss", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ( 14 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot, FetchLat, MemoryTLB", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( ( 12 ) * ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss, FetchLat", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat, MicroSeq", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "FetchBW, Frontend, TmaL2", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSBmiss, FetchBW", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSB, FetchBW", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "BadSpec, BrMispredicts, TmaL2", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "BadSpec, MachineClears, TmaL2", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "100 * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "100 * ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) + RESOURCE_STALLS.SB ) / ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) )", - "MetricGroup": "Backend, TmaL2", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) - CYCLE_ACTIVITY.STALLS_L1D_PENDING ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) * CYCLE_ACTIVITY.STALLS_L2_PENDING / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses, MemoryBound, TmaL3mem", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( 1 - ( MEM_LOAD_UOPS_RETIRED.L3_HIT / ( MEM_LOAD_UOPS_RETIRED.L3_HIT + ( 7 ) * MEM_LOAD_UOPS_RETIRED.L3_MISS ) ) ) * CYCLE_ACTIVITY.STALLS_L2_PENDING / ( CPU_CLK_UNHALTED.THREAD ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound, TmaL3mem", - "MetricName": "tma_drabound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( RESOURCE_STALLS.SB / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound, TmaL3mem", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "100 * ( ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) - ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) + RESOURCE_STALLS.SB ) / ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) ) * ( 1 - ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) ) ) )", - "MetricGroup": "Backend, TmaL2, Compute", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( 10 * ARITH.DIVIDER_UOPS / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) )", + "MetricExpr": "100 * cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ / ( cbox@UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE\\,filter_opc\\=0x182@ + cbox@UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE\\,filter_opc\\=0x182@ )", "MetricGroup": "", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", - "MetricExpr": "100 * ( ( ( ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) ) / 2 - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) if #SMT_on else ( ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.CYCLES_NO_EXECUTE ) ) + cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x1@ - ( cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x3@ if ( ( INST_RETIRED.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) > 1.8 ) else cpu@UOPS_EXECUTED.CORE\\,cmask\\=0x2@ ) - ( RS_EVENTS.EMPTY_CYCLES if ( ( ( 4 ) * ( min( CPU_CLK_UNHALTED.THREAD , IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) > 0.1 ) else 0 ) + RESOURCE_STALLS.SB ) ) - RESOURCE_STALLS.SB - ( min( CPU_CLK_UNHALTED.THREAD , CYCLE_ACTIVITY.STALLS_LDM_PENDING ) ) ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "PortsUtil", - "MetricName": "tma_ports_utilization_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "Retire, TmaL2", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Retire, TmaL2", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "MicroSeq", - "MetricName": "tma_microcode_sequencer_percent", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json index 3e48ff3516b0f65de241d25ba587b261276bc627..eb0a05fbb7048b100ac3bbb679c16eb1e4332c33 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/uncore-interconnect.json @@ -981,36 +981,34 @@ "Unit": "QPI LL" }, { - "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data", + "BriefDescription": "Flits Transferred - Group 0; Data Tx Flits", "Counter": "0,1,2,3", - "EventName": "QPI_DATA_BANDWIDTH_TX", + "EventName": "UNC_Q_TxL_FLITS_G0.DATA", "PerPkg": "1", - "ScaleUnit": "8Bytes", "UMask": "0x2", "Unit": "QPI LL" }, { - "BriefDescription": "Number of data flits transmitted ", + "BriefDescription": "Number of data flits transmitted . Derived from unc_q_txl_flits_g0.data", "Counter": "0,1,2,3", - "EventName": "UNC_Q_TxL_FLITS_G0.DATA", + "EventName": "QPI_DATA_BANDWIDTH_TX", "PerPkg": "1", "ScaleUnit": "8Bytes", "UMask": "0x2", "Unit": "QPI LL" }, { - "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data", + "BriefDescription": "Flits Transferred - Group 0; Non-Data protocol Tx Flits", "Counter": "0,1,2,3", - "EventName": "QPI_CTL_BANDWIDTH_TX", + "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA", "PerPkg": "1", - "ScaleUnit": "8Bytes", "UMask": "0x4", "Unit": "QPI LL" }, { - "BriefDescription": "Number of non data (control) flits transmitted ", + "BriefDescription": "Number of non data (control) flits transmitted . Derived from unc_q_txl_flits_g0.non_data", "Counter": "0,1,2,3", - "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA", + "EventName": "QPI_CTL_BANDWIDTH_TX", "PerPkg": "1", "ScaleUnit": "8Bytes", "UMask": "0x4", diff --git a/tools/perf/pmu-events/arch/x86/haswellx/uncore-memory.json b/tools/perf/pmu-events/arch/x86/haswellx/uncore-memory.json index db3418db312e1328bf371fbba27ff61ad39fe73b..c003daa9ed8cf2a9a3207e4eaf14d01e8c68eb0b 100644 --- a/tools/perf/pmu-events/arch/x86/haswellx/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/haswellx/uncore-memory.json @@ -72,20 +72,19 @@ "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM Reads (RD_CAS + Underfills)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_READ", + "EventName": "UNC_M_CAS_COUNT.RD", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0x3", "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller", + "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.RD", + "EventName": "LLC_MISSES.MEM_READ", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0x3", @@ -110,20 +109,19 @@ "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", + "BriefDescription": "DRAM RD_CAS and WR_CAS Commands.; All DRAM WR_CAS (both Modes)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_WRITE", + "EventName": "UNC_M_CAS_COUNT.WR", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0xC", "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller", + "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.WR", + "EventName": "LLC_MISSES.MEM_WRITE", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0xC", diff --git a/tools/perf/pmu-events/arch/x86/icelake/cache.json b/tools/perf/pmu-events/arch/x86/icelake/cache.json index b4f28f24ee63df364f2cbc67cc75863432867d5a..0f6b918484d50a011d388e36e15312ed20bcc3ad 100644 --- a/tools/perf/pmu-events/arch/x86/icelake/cache.json +++ b/tools/perf/pmu-events/arch/x86/icelake/cache.json @@ -18,13 +18,13 @@ "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL", "PEBScounters": "0,1,2,3", - "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", "SampleAfterValue": "1000003", "Speculative": "1", "UMask": "0x2" }, { - "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.", + "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.", "CollectPEBSRecord": "2", "Counter": "0,1,2,3", "CounterMask": "1", @@ -32,7 +32,7 @@ "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL_PERIODS", "PEBScounters": "0,1,2,3", - "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", "SampleAfterValue": "1000003", "Speculative": "1", "UMask": "0x2" diff --git a/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json b/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json index f0356d66a9271412f0314259319b5cc4f7aac264..3b5ef09eb8efc874ca46842694fe0043ffcb46ca 100644 --- a/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/icelake/icl-metrics.json @@ -1,26 +1,716 @@ [ + { + "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", + "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "10 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline", + "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_mite_4wide", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit", + "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_lsd", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", + "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", + "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((29 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM + (23.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(23.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(32.5 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores", + "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_streaming_stores", + "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "140 * MISC_RETIRED.PAUSE_INST / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", + "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.", + "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_branch_instructions", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * ASSISTS.ANY / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" + }, + { + "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", + "MetricGroup": "Mem;MemoryBW;Offcore", + "MetricName": "Memory_Bandwidth" + }, + { + "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))", + "MetricGroup": "Mem;MemoryLat;Offcore", + "MetricName": "Memory_Latency" + }, + { + "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ", + "MetricGroup": "Mem;MemoryTLB;Offcore", + "MetricName": "Memory_Data_TLBs" + }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, + { + "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", + "MetricGroup": "Fed;FetchBW;Frontend", + "MetricName": "Instruction_Fetch_BW" + }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, + { + "BriefDescription": "Uops Per Instruction", + "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;Ret;Retire", + "MetricName": "UPI" + }, + { + "BriefDescription": "Instruction per taken branch", + "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN", + "MetricGroup": "Branches;Fed;FetchBW", + "MetricName": "UpTB" + }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -32,13 +722,13 @@ { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", "MetricExpr": "TOPDOWN.SLOTS", - "MetricGroup": "TmaL1", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, { "BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor", - "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1", - "MetricGroup": "SMT;TmaL1", + "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1", + "MetricGroup": "SMT;tma_L1_group", "MetricName": "Slots_Utilization" }, { @@ -50,29 +740,35 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, + { + "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", + "MetricGroup": "Cor;SMT", + "MetricName": "Core_Bound_Likely" + }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", "MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED", @@ -117,13 +813,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -144,21 +840,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -170,11 +866,17 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, + { + "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.", + "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@", + "MetricGroup": "Pipeline;Ret", + "MetricName": "Retire" + }, { "BriefDescription": "", "MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@", @@ -205,6 +907,12 @@ "MetricGroup": "DSBmiss", "MetricName": "DSB_Switch_Cost" }, + { + "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))", + "MetricGroup": "DSBmiss;Fed", + "MetricName": "DSB_Misses" + }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -217,6 +925,12 @@ "MetricGroup": "Bad;BadSpec;BrMispredicts", "MetricName": "IpMispredict" }, + { + "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricGroup": "Bad;BrMispredicts", + "MetricName": "Branch_Misprediction_Cost" + }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -231,7 +945,7 @@ }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, @@ -243,74 +957,74 @@ }, { "BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)", - "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )", + "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)", "MetricGroup": "Bad;Branches", "MetricName": "Other_Branches" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( ( OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD ) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricExpr": "1000 * ((OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS) / Instructions", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, @@ -340,25 +1054,25 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -370,40 +1084,40 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License0_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License1_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License2_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions." @@ -428,7 +1142,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/icelake/pipeline.json b/tools/perf/pmu-events/arch/x86/icelake/pipeline.json index a017a47270506ba52e817a2284f035c249cb3675..c74a7369cff35c42d3b2b3cb1d33bd228423f39a 100644 --- a/tools/perf/pmu-events/arch/x86/icelake/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/icelake/pipeline.json @@ -167,7 +167,7 @@ "UMask": "0x10" }, { - "BriefDescription": "number of branch instructions retired that were mispredicted and taken. Non PEBS", + "BriefDescription": "number of branch instructions retired that were mispredicted and taken.", "CollectPEBSRecord": "2", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0xc5", diff --git a/tools/perf/pmu-events/arch/x86/icelakex/cache.json b/tools/perf/pmu-events/arch/x86/icelakex/cache.json index 775190bdd0632c5e4b18484b5fb7cd2b4a20ffcc..e4035b3e55caacb95e6163630409556fcf7d2fab 100644 --- a/tools/perf/pmu-events/arch/x86/icelakex/cache.json +++ b/tools/perf/pmu-events/arch/x86/icelakex/cache.json @@ -18,13 +18,13 @@ "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL", "PEBScounters": "0,1,2,3", - "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "PublicDescription": "Counts number of cycles a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", "SampleAfterValue": "1000003", "Speculative": "1", "UMask": "0x2" }, { - "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.", + "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.", "CollectPEBSRecord": "2", "Counter": "0,1,2,3", "CounterMask": "1", @@ -32,7 +32,7 @@ "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL_PERIODS", "PEBScounters": "0,1,2,3", - "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", "SampleAfterValue": "1000003", "Speculative": "1", "UMask": "0x2" diff --git a/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json b/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json index e905458b34b8d486d7e589e349727e5b543612d1..b52afc34a169497bd69c628eb4fb40fe15545c49 100644 --- a/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/icelakex/icx-metrics.json @@ -1,22 +1,742 @@ [ + { + "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", + "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "10 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline", + "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_mite_4wide", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", + "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", + "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (43.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(43.5 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(19 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound) - tma_pmm_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "(43.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "(108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "((97 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (97 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a", + "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)", + "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_pmm_bound", + "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(48 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores", + "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_streaming_stores", + "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "37 * MISC_RETIRED.PAUSE_INST / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", + "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.", + "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_branch_instructions", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * ASSISTS.ANY / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" + }, + { + "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", + "MetricGroup": "Mem;MemoryBW;Offcore", + "MetricName": "Memory_Bandwidth" + }, + { + "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))", + "MetricGroup": "Mem;MemoryLat;Offcore", + "MetricName": "Memory_Latency" + }, + { + "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ", + "MetricGroup": "Mem;MemoryTLB;Offcore", + "MetricName": "Memory_Data_TLBs" + }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, + { + "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", + "MetricGroup": "Fed;FetchBW;Frontend", + "MetricName": "Instruction_Fetch_BW" + }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, + { + "BriefDescription": "Uops Per Instruction", + "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;Ret;Retire", + "MetricName": "UPI" + }, + { + "BriefDescription": "Instruction per taken branch", + "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN", + "MetricGroup": "Branches;Fed;FetchBW", + "MetricName": "UpTB" + }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -26,13 +746,13 @@ { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", "MetricExpr": "TOPDOWN.SLOTS", - "MetricGroup": "TmaL1", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, { "BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor", - "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1", - "MetricGroup": "SMT;TmaL1", + "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1", + "MetricGroup": "SMT;tma_L1_group", "MetricName": "Slots_Utilization" }, { @@ -44,29 +764,35 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, + { + "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", + "MetricGroup": "Cor;SMT", + "MetricName": "Core_Bound_Likely" + }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", "MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED", @@ -111,13 +837,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -138,21 +864,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -164,11 +890,17 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, + { + "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.", + "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@", + "MetricGroup": "Pipeline;Ret", + "MetricName": "Retire" + }, { "BriefDescription": "", "MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@", @@ -193,6 +925,12 @@ "MetricGroup": "DSBmiss", "MetricName": "DSB_Switch_Cost" }, + { + "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))", + "MetricGroup": "DSBmiss;Fed", + "MetricName": "DSB_Misses" + }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -205,6 +943,12 @@ "MetricGroup": "Bad;BadSpec;BrMispredicts", "MetricName": "IpMispredict" }, + { + "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricGroup": "Bad;BrMispredicts", + "MetricName": "Branch_Misprediction_Cost" + }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -219,7 +963,7 @@ }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, @@ -231,74 +975,74 @@ }, { "BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)", - "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )", + "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)", "MetricGroup": "Bad;Branches", "MetricName": "Other_Branches" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( ( OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD ) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricExpr": "1000 * ((OFFCORE_REQUESTS.ALL_DATA_RD - OFFCORE_REQUESTS.DEMAND_DATA_RD) + L2_RQSTS.ALL_DEMAND_MISS + L2_RQSTS.SWPF_MISS) / Instructions", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, @@ -328,37 +1072,37 @@ }, { "BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)", - "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_Silent_PKI" }, { "BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction", - "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_NonSilent_PKI" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -370,40 +1114,40 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License0_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License1_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License2_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions." @@ -428,13 +1172,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( cha_0@event\\=0x0@ / duration_time )", + "MetricExpr": "1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -446,38 +1190,38 @@ }, { "BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / cha_0@event\\=0x0@ )", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": "(1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM) / cha_0@event\\=0x0@)", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_PMM_Read_Latency" }, { "BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": " 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / cha_0@event\\=0x0@", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": " 1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR) / cha_0@event\\=0x0@", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_DRAM_Read_Latency" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]", - "MetricExpr": "( ( 64 * imc@event\\=0xe3@ / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * imc@event\\=0xe3@ / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Read_BW" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]", - "MetricExpr": "( ( 64 * imc@event\\=0xe7@ / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * imc@event\\=0xe7@ / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]", "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]", - "MetricExpr": "( UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR ) * 64 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricExpr": "(UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR) * 64 / 1000000000 / duration_time", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Read_BW" }, { @@ -486,12 +1230,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -523,11 +1261,10 @@ "MetricName": "C6_Pkg_Residency" }, { - "BriefDescription": "Percentage of time spent in the active CPU power state C0", - "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC", - "MetricGroup": "", - "MetricName": "cpu_utilization_percent", - "ScaleUnit": "1%" + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" }, { "BriefDescription": "CPU operating frequency (in GHz)", @@ -536,13 +1273,6 @@ "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", "MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY", @@ -561,7 +1291,7 @@ "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { @@ -589,7 +1319,7 @@ "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { @@ -615,42 +1345,42 @@ }, { "BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", - "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD ) / INST_RETIRED.ANY", + "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD + UNC_CHA_TOR_INSERTS.IA_MISS_CRD_PREF ) / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "llc_code_read_mpi_demand_plus_prefetch", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to local memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency_for_local_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to remote memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency_for_remote_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to Intel(R) Optane(TM) Persistent Memory(PMEM) in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_to_pmem_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to DRAM in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_to_dram_latency", "ScaleUnit": "1ns" @@ -694,14 +1424,14 @@ "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" }, { @@ -715,7 +1445,7 @@ "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)", "MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "upi_data_transmit_bw_only_data", + "MetricName": "upi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { @@ -764,35 +1494,35 @@ "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", "MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_HIT_PCIRDCUR + UNC_CHA_TOR_INSERTS.IO_MISS_PCIRDCUR ) * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", "MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_HIT_ITOM + UNC_CHA_TOR_INSERTS.IO_MISS_ITOM + UNC_CHA_TOR_INSERTS.IO_HIT_ITOMCACHENEAR + UNC_CHA_TOR_INSERTS.IO_MISS_ITOMCACHENEAR ) * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_decoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { @@ -824,241 +1554,10 @@ "ScaleUnit": "1MB/s" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) )", - "MetricGroup": "TmaL1;PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( ( 5 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / ( slots ) )", - "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ICACHE_16B.IFDATA_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 10 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 3 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( max( 0 , ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) - ( ( ( 5 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / ( slots ) ) ) )", - "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )", - "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )", - "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) )", - "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( max( 0 , ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) ) ) )", - "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "100 * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) )", - "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_dram_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", - "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + L1D_PEND_MISS.FB_FULL_PERIODS ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_pmm_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "100 * ( max( 0 , ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) + ( ( 5 ) * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( slots ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "( 100 * ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "100 * ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", - "MetricExpr": "100 * ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) )", - "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fp_arith_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_memory_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * BR_INST_RETIRED.ALL_BRANCHES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_branch_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_nop_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", - "MetricExpr": "100 * ( max( 0 , ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * BR_INST_RETIRED.ALL_BRANCHES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_other_light_ops_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "100 * ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", - "MetricExpr": "100 * ( ( ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=0x1@ ) / IDQ.MITE_UOPS ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) ) )", - "MetricGroup": "TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_few_uops_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( slots ) )", - "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_microcode_sequencer_percent", + "BriefDescription": "%", + "MetricExpr": "100 * ( ( LSD.CYCLES_ACTIVE - LSD.CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )", + "MetricGroup": "FetchBW;LSD;TopdownL3;tma_L3_group;tma_fetch_bandwidth_group", + "MetricName": "tma_lsd", "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/icelakex/pipeline.json b/tools/perf/pmu-events/arch/x86/icelakex/pipeline.json index 396868f700040c6c00e13e0edec4bb198a4da859..52fba238bf1fdc14facfbb4891a154d962afd37e 100644 --- a/tools/perf/pmu-events/arch/x86/icelakex/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/icelakex/pipeline.json @@ -167,7 +167,7 @@ "UMask": "0x10" }, { - "BriefDescription": "number of branch instructions retired that were mispredicted and taken. Non PEBS", + "BriefDescription": "number of branch instructions retired that were mispredicted and taken.", "CollectPEBSRecord": "2", "Counter": "0,1,2,3,4,5,6,7", "EventCode": "0xc5", diff --git a/tools/perf/pmu-events/arch/x86/icelakex/uncore-other.json b/tools/perf/pmu-events/arch/x86/icelakex/uncore-other.json index 7783aa2ef5d18e097e35ca35455b0c9f0d6b854e..03e99b8aed93e02e0616511b4a149d92590a8092 100644 --- a/tools/perf/pmu-events/arch/x86/icelakex/uncore-other.json +++ b/tools/perf/pmu-events/arch/x86/icelakex/uncore-other.json @@ -11779,7 +11779,7 @@ "Unit": "M3UPI" }, { - "BriefDescription": "Flit Gen - Header 1 : Acumullate", + "BriefDescription": "Flit Gen - Header 1 : Accumulate", "Counter": "0,1,2,3", "CounterType": "PGMABLE", "EventCode": "0x51", diff --git a/tools/perf/pmu-events/arch/x86/ivybridge/ivb-metrics.json b/tools/perf/pmu-events/arch/x86/ivybridge/ivb-metrics.json index 3f48e75f8a86d9547a860f6cd734b1413e3bf6ca..63db3397af0f91714955e42120f6298b4238c5a0 100644 --- a/tools/perf/pmu-events/arch/x86/ivybridge/ivb-metrics.json +++ b/tools/perf/pmu-events/arch/x86/ivybridge/ivb-metrics.json @@ -1,64 +1,500 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFETCH_STALL / CLKS - tma_itlb_misses", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "13 * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) + 43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "29 * (MEM_LOAD_UOPS_RETIRED.LLC_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_RETIRED.LLC_MISS))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5) / (3 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -76,8 +512,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -88,16 +524,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -107,37 +537,25 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -179,15 +597,15 @@ }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "1 / ( ((FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD) + ((FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD) )", + "MetricExpr": "1 / (tma_fp_scalar + tma_fp_vector)", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -204,7 +622,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -216,47 +634,41 @@ }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.LLC_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -277,19 +689,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -307,26 +719,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -344,7 +756,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/cache.json b/tools/perf/pmu-events/arch/x86/ivytown/cache.json index 27576d53b34728d4589d855053a03b174a37115b..d95b98c83914382bfda98cceebc6b9f0ff306d49 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/cache.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/cache.json @@ -21,7 +21,7 @@ "UMask": "0x2" }, { - "BriefDescription": "L1D miss oustandings duration in cycles", + "BriefDescription": "L1D miss outstanding duration in cycles", "Counter": "2", "CounterHTOff": "2", "EventCode": "0x48", @@ -658,7 +658,7 @@ "UMask": "0x8" }, { - "BriefDescription": "Cacheable and noncachaeble code read requests", + "BriefDescription": "Cacheable and noncacheable code read requests", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0xB0", diff --git a/tools/perf/pmu-events/arch/x86/ivytown/floating-point.json b/tools/perf/pmu-events/arch/x86/ivytown/floating-point.json index 4c2ac010cf55df92d4102a418b5abaddc5d23406..88891cba54ec8f1115f9e95a68e04cf745a50426 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/floating-point.json @@ -91,7 +91,7 @@ "UMask": "0x20" }, { - "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULsand IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s", + "BriefDescription": "Number of FP Computational Uops Executed this cycle. The number of FADD, FSUB, FCOM, FMULs, integer MULs and IMULs, FDIVs, FPREMs, FSQRTS, integer DIVs, and IDIVs. This event does not distinguish an FADD used in the middle of a transcendental flow from a s", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x10", diff --git a/tools/perf/pmu-events/arch/x86/ivytown/frontend.json b/tools/perf/pmu-events/arch/x86/ivytown/frontend.json index 2b1a82dd86abc04a50b4ce4b34684ff9c5a857b1..0a295c4e093dd8df535318d70d6ef650537c87d5 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/frontend.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/frontend.json @@ -176,41 +176,41 @@ "UMask": "0x4" }, { - "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", "EventCode": "0x79", "EventName": "IDQ.MS_CYCLES", - "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Cycles when uops are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x30" }, { - "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", "EventCode": "0x79", "EventName": "IDQ.MS_DSB_CYCLES", - "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Cycles when uops initiated by Decode Stream Buffer (DSB) are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x10" }, { - "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "CounterMask": "1", "EdgeDetect": "1", "EventCode": "0x79", "EventName": "IDQ.MS_DSB_OCCUR", - "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequenser (MS) is busy.", + "PublicDescription": "Deliveries to Instruction Decode Queue (IDQ) initiated by Decode Stream Buffer (DSB) while Microcode Sequencer (MS) is busy.", "SampleAfterValue": "2000003", "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by Decode Stream Buffer (DSB) that are being delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -220,7 +220,7 @@ "UMask": "0x10" }, { - "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops initiated by MITE and delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", @@ -242,7 +242,7 @@ "UMask": "0x30" }, { - "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequenser (MS) is busy", + "BriefDescription": "Uops delivered to Instruction Decode Queue (IDQ) while Microcode Sequencer (MS) is busy", "Counter": "0,1,2,3", "CounterHTOff": "0,1,2,3,4,5,6,7", "EventCode": "0x79", diff --git a/tools/perf/pmu-events/arch/x86/ivytown/ivt-metrics.json b/tools/perf/pmu-events/arch/x86/ivytown/ivt-metrics.json index 19c7f3b41102d201bf6a00504797790aa6ac94ca..99a45c8d8ceeb3b982bbed731586524fb21831e7 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/ivt-metrics.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/ivt-metrics.json @@ -1,64 +1,524 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", + "MetricExpr": "ICACHE.IFETCH_STALL / CLKS - tma_itlb_misses", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING) - CYCLE_ACTIVITY.STALLS_L1D_PENDING) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_UOPS_RETIRED.L1_HIT_PS;MEM_LOAD_UOPS_RETIRED.HIT_LFB_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_UOPS_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "13 * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_UOPS_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L1D_PENDING - CYCLE_ACTIVITY.STALLS_L2_PENDING) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "(60 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) + 43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "43 * (MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "41 * (MEM_LOAD_UOPS_RETIRED.LLC_HIT * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "200 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "310 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "(200 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD))) + 180 * (MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD * (1 + mem_load_uops_retired.hit_lfb / ((MEM_LOAD_UOPS_RETIRED.L2_HIT + MEM_LOAD_UOPS_RETIRED.LLC_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM + MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS) + MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM + MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD)))) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_UOPS_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES))) + (1 - (MEM_UOPS_RETIRED.LOCK_LOADS / MEM_UOPS_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(200 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_MISS.REMOTE_HITM + 60 * OFFCORE_RESPONSE.DEMAND_RFO.LLC_HIT.HITM_OTHER_CORE) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "2 * MEM_UOPS_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_UOPS_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * DTLB_STORE_MISSES.STLB_HIT + DTLB_STORE_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_UOPS_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) + UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC if (IPC > 1.8) else UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_LDM_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,inv\\,cmask\\=1@) / 2 if #SMT_on else (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_EXECUTE) - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else 0) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC - UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(cpu@UOPS_EXECUTED.CORE\\,cmask\\=2@ - cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@) / 2 if #SMT_on else (UOPS_EXECUTED.CYCLES_GE_2_UOPS_EXEC - UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "((cpu@UOPS_EXECUTED.CORE\\,cmask\\=3@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_3_UOPS_EXEC) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5) / (3 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * OTHER_ASSISTS.ANY_WB_ASSIST / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -76,8 +536,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -88,16 +548,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -107,37 +561,25 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((cpu@UOPS_EXECUTED.CORE\\,cmask\\=1@ / 2) if #SMT_on else UOPS_EXECUTED.CYCLES_GE_1_UOP_EXEC)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -179,15 +621,15 @@ }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "1 / ( ((FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_EXECUTED.THREAD) + ((FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_EXECUTED.THREAD) )", + "MetricExpr": "1 / (tma_fp_scalar + tma_fp_vector)", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -204,7 +646,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -216,47 +658,41 @@ }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_UOPS_RETIRED.L1_MISS + mem_load_uops_retired.hit_lfb)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_UOPS_RETIRED.LLC_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "(ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION) / CORE_CLKS", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_DURATION + DTLB_LOAD_MISSES.WALK_DURATION + DTLB_STORE_MISSES.WALK_DURATION ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -277,19 +713,19 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, @@ -307,26 +743,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -344,7 +780,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, @@ -354,12 +790,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -407,5 +837,11 @@ "MetricExpr": "(cstate_pkg@c7\\-residency@ / msr@tsc@) * 100", "MetricGroup": "Power", "MetricName": "C7_Pkg_Residency" + }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" } ] diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-cache.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-cache.json index 93e07385eeec70be03db118d5592949fa949eb6a..c118ff54c30eb4b2e3fba39efd9238bbd0b9b5b7 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-cache.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-cache.json @@ -61,7 +61,7 @@ "EventCode": "0x34", "EventName": "UNC_C_LLC_LOOKUP.WRITE", "PerPkg": "1", - "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set filter mask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CBoGlCtrl[22:17] bits correspond to [M'FMESI] state.; Writeback transactions from L2 to the LLC This includes all write transactions -- both Cachable and UC.", + "PublicDescription": "Counts the number of times the LLC was accessed - this includes code, data, prefetches and hints coming from L2. This has numerous filters available. Note the non-standard filtering equation. This event will count requests that lookup the cache multiple times with multiple increments. One must ALWAYS set filter mask bit 0 and select a state or states to match. Otherwise, the event will count nothing. CBoGlCtrl[22:17] bits correspond to [M'FMESI] state.; Writeback transactions from L2 to the LLC This includes all write transactions -- both Cacheable and UC.", "UMask": "0x5", "Unit": "CBO" }, @@ -999,7 +999,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.ALL", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR. This includes requests that reside in the TOR for a short time, such as LLC Hits that do not need to snoop cores or requests that get rejected and have to be retried through one of the ingress queues. The TOR is more commonly a bottleneck in skews with smaller core counts, where the ratio of RTIDs to TOR entries is larger. Note that there are reserved TOR entries for various request types, so it is possible that a given request type be blocked with an occupancy that is less than 20. Also note that generally requests will not be able to arbitrate into the TOR pipeline if there are no available TOR slots.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR. This includes requests that reside in the TOR for a short time, such as LLC Hits that do not need to snoop cores or requests that get rejected and have to be retried through one of the ingress queues. The TOR is more commonly a bottleneck in skews with smaller core counts, where the ratio of RTIDs to TOR entries is larger. Note that there are reserved TOR entries for various request types, so it is possible that a given request type be blocked with an occupancy that is less than 20. Also note that generally requests will not be able to arbitrate into the TOR pipeline if there are no available TOR slots.", "UMask": "0x8", "Unit": "CBO" }, @@ -1009,7 +1009,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.EVICTION", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Eviction transactions inserted into the TOR. Evictions can be quick, such as when the line is in the F, S, or E states and no core valid bits are set. They can also be longer if either CV bits are set (so the cores need to be snooped) and/or if there is a HitM (in which case it is necessary to write the request out to memory).", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Eviction transactions inserted into the TOR. Evictions can be quick, such as when the line is in the F, S, or E states and no core valid bits are set. They can also be longer if either CV bits are set (so the cores need to be snooped) and/or if there is a HitM (in which case it is necessary to write the request out to memory).", "UMask": "0x4", "Unit": "CBO" }, @@ -1019,7 +1019,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.LOCAL", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisfied by locally HOMed memory.", "UMask": "0x28", "Unit": "CBO" }, @@ -1029,7 +1029,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.LOCAL_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisifed by an opcode, inserted into the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisfied by an opcode, inserted into the TOR that are satisfied by locally HOMed memory.", "UMask": "0x21", "Unit": "CBO" }, @@ -1039,7 +1039,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.MISS_LOCAL", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisfied by locally HOMed memory.", "UMask": "0x2A", "Unit": "CBO" }, @@ -1049,7 +1049,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.MISS_LOCAL_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisifed by an opcode, inserted into the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisfied by an opcode, inserted into the TOR that are satisfied by locally HOMed memory.", "UMask": "0x23", "Unit": "CBO" }, @@ -1059,7 +1059,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.MISS_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match an opcode.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match an opcode.", "UMask": "0x3", "Unit": "CBO" }, @@ -1069,7 +1069,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.MISS_REMOTE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x8A", "Unit": "CBO" }, @@ -1079,7 +1079,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.MISS_REMOTE_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisifed by an opcode, inserted into the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions, satisfied by an opcode, inserted into the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x83", "Unit": "CBO" }, @@ -1089,7 +1089,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_ALL", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched (matches an RTID destination) transactions inserted into the TOR. The NID is programmed in Cn_MSR_PMON_BOX_FILTER.nid. In conjunction with STATE = I, it is possible to monitor misses to specific NIDs in the system.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched (matches an RTID destination) transactions inserted into the TOR. The NID is programmed in Cn_MSR_PMON_BOX_FILTER.nid. In conjunction with STATE = I, it is possible to monitor misses to specific NIDs in the system.", "UMask": "0x48", "Unit": "CBO" }, @@ -1099,7 +1099,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_EVICTION", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched eviction transactions inserted into the TOR.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched eviction transactions inserted into the TOR.", "UMask": "0x44", "Unit": "CBO" }, @@ -1109,7 +1109,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_MISS_ALL", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched miss requests that were inserted into the TOR.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All NID matched miss requests that were inserted into the TOR.", "UMask": "0x4A", "Unit": "CBO" }, @@ -1119,7 +1119,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_MISS_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match a NID and an opcode.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Miss transactions inserted into the TOR that match a NID and an opcode.", "UMask": "0x43", "Unit": "CBO" }, @@ -1129,7 +1129,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match a NID and an opcode.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match a NID and an opcode.", "UMask": "0x41", "Unit": "CBO" }, @@ -1139,7 +1139,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.NID_WB", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched write transactions inserted into the TOR.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; NID matched write transactions inserted into the TOR.", "UMask": "0x50", "Unit": "CBO" }, @@ -1149,7 +1149,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match an opcode (matched by Cn_MSR_PMON_BOX_FILTER.opc)", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Transactions inserted into the TOR that match an opcode (matched by Cn_MSR_PMON_BOX_FILTER.opc)", "UMask": "0x1", "Unit": "CBO" }, @@ -1159,7 +1159,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.REMOTE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions inserted into the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x88", "Unit": "CBO" }, @@ -1169,7 +1169,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.REMOTE_OPCODE", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisifed by an opcode, inserted into the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; All transactions, satisfied by an opcode, inserted into the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x81", "Unit": "CBO" }, @@ -1179,7 +1179,7 @@ "EventCode": "0x35", "EventName": "UNC_C_TOR_INSERTS.WB", "PerPkg": "1", - "PublicDescription": "Counts the number of entries successfuly inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Write transactions inserted into the TOR. This does not include RFO, but actual operations that contain data being sent from the core.", + "PublicDescription": "Counts the number of entries successfully inserted into the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182).; Write transactions inserted into the TOR. This does not include RFO, but actual operations that contain data being sent from the core.", "UMask": "0x10", "Unit": "CBO" }, @@ -1215,7 +1215,7 @@ "EventCode": "0x36", "EventName": "UNC_C_TOR_OCCUPANCY.LOCAL_OPCODE", "PerPkg": "1", - "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisifed by an opcode, in the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisfied by an opcode, in the TOR that are satisfied by locally HOMed memory.", "UMask": "0x21", "Unit": "CBO" }, @@ -1242,7 +1242,7 @@ "EventCode": "0x36", "EventName": "UNC_C_TOR_OCCUPANCY.MISS_LOCAL_OPCODE", "PerPkg": "1", - "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisifed by an opcode, in the TOR that are satisifed by locally HOMed memory.", + "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisfied by an opcode, in the TOR that are satisfied by locally HOMed memory.", "UMask": "0x23", "Unit": "CBO" }, @@ -1269,7 +1269,7 @@ "EventCode": "0x36", "EventName": "UNC_C_TOR_OCCUPANCY.MISS_REMOTE_OPCODE", "PerPkg": "1", - "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisifed by an opcode, in the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding Miss transactions, satisfied by an opcode, in the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x83", "Unit": "CBO" }, @@ -1350,7 +1350,7 @@ "EventCode": "0x36", "EventName": "UNC_C_TOR_OCCUPANCY.REMOTE_OPCODE", "PerPkg": "1", - "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisifed by an opcode, in the TOR that are satisifed by remote caches or remote memory.", + "PublicDescription": "For each cycle, this event accumulates the number of valid entries in the TOR that match qualifications specified by the subevent. There are a number of subevent 'filters' but only a subset of the subevent combinations are valid. Subevents that require an opcode or NID match require the Cn_MSR_PMON_BOX_FILTER.{opc, nid} field to be set. If, for example, one wanted to count DRD Local Misses, one should select MISS_OPC_MATCH and set Cn_MSR_PMON_BOX_FILTER.opc to DRD (0x182); Number of outstanding transactions, satisfied by an opcode, in the TOR that are satisfied by remote caches or remote memory.", "UMask": "0x81", "Unit": "CBO" }, @@ -1446,7 +1446,7 @@ "EventCode": "0x2", "EventName": "UNC_C_TxR_INSERTS.BL_CORE", "PerPkg": "1", - "PublicDescription": "Number of allocations into the Cbo Egress. The Egress is used to queue up requests destined for the ring.; Ring transactions from the Corebo destined for the BL ring. This is commonly used for transfering writeback data to the cache.", + "PublicDescription": "Number of allocations into the Cbo Egress. The Egress is used to queue up requests destined for the ring.; Ring transactions from the Corebo destined for the BL ring. This is commonly used for transferring writeback data to the cache.", "UMask": "0x40", "Unit": "CBO" }, @@ -1692,7 +1692,7 @@ "EventCode": "0xb", "EventName": "UNC_H_CONFLICT_CYCLES.LAST", "PerPkg": "1", - "PublicDescription": "Count every last conflictor in conflict chain. Can be used to compute the average conflict chain length as (#Ackcnflts/#LastConflictor)+1. This can be used to give a feel for the conflict chain lenghts while analyzing lock kernels.", + "PublicDescription": "Count every last conflictor in conflict chain. Can be used to compute the average conflict chain length as (#Ackcnflts/#LastConflictor)+1. This can be used to give a feel for the conflict chain lengths while analyzing lock kernels.", "UMask": "0x4", "Unit": "HA" }, @@ -1729,7 +1729,7 @@ "EventCode": "0x41", "EventName": "UNC_H_DIRECTORY_LAT_OPT", "PerPkg": "1", - "PublicDescription": "Directory Latency Optimization Data Return Path Taken. When directory mode is enabled and the directory retuned for a read is Dir=I, then data can be returned using a faster path if certain conditions are met (credits, free pipeline, etc).", + "PublicDescription": "Directory Latency Optimization Data Return Path Taken. When directory mode is enabled and the directory returned for a read is Dir=I, then data can be returned using a faster path if certain conditions are met (credits, free pipeline, etc).", "Unit": "HA" }, { @@ -2686,7 +2686,7 @@ "EventCode": "0x21", "EventName": "UNC_H_SNOOP_RESP.RSPSFWD", "PerPkg": "1", - "PublicDescription": "Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1.; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currentl copy. This is common for data and code reads that hit in a remote socket in E or F state.", + "PublicDescription": "Counts the total number of RspI snoop responses received. Whenever a snoops are issued, one or more snoop responses will be returned depending on the topology of the system. In systems larger than 2s, when multiple snoops are returned this will count all the snoops that are received. For example, if 3 snoops were issued and returned RspI, RspS, and RspSFwd; then each of these sub-events would increment by 1.; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currently copy. This is common for data and code reads that hit in a remote socket in E or F state.", "UMask": "0x8", "Unit": "HA" }, @@ -2766,7 +2766,7 @@ "EventCode": "0x60", "EventName": "UNC_H_SNP_RESP_RECV_LOCAL.RSPSFWD", "PerPkg": "1", - "PublicDescription": "Number of snoop responses received for a Local request; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currentl copy. This is common for data and code reads that hit in a remote socket in E or F state.", + "PublicDescription": "Number of snoop responses received for a Local request; Filters for a snoop response of RspSFwd. This is returned when a remote caching agent forwards data but holds on to its currently copy. This is common for data and code reads that hit in a remote socket in E or F state.", "UMask": "0x8", "Unit": "HA" }, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json index b3b1a08d4acf5b1efb50851402cb836f309d3e28..10ea4afeffc1397d529de7f651d779146d0491fb 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-interconnect.json @@ -24,7 +24,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits. Had there been enough credits, the spawn would have worked as the RBT bit was set and the RBT tag matched.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits. Had there been enough credits, the spawn would have worked as the RBT bit was set and the RBT tag matched.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -34,7 +34,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_MISS", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and there weren't enough Egress credits. The valid bit was set.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and there weren't enough Egress credits. The valid bit was set.", "UMask": "0x20", "Unit": "QPI LL" }, @@ -44,7 +44,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_RBT", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits AND the RBT bit was not set, but the RBT tag matched.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because there were not enough Egress credits AND the RBT bit was not set, but the RBT tag matched.", "UMask": "0x8", "Unit": "QPI LL" }, @@ -54,7 +54,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_CREDITS_RBT_MISS", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match, the valid bit was not set and there weren't enough Egress credits.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match, the valid bit was not set and there weren't enough Egress credits.", "UMask": "0x80", "Unit": "QPI LL" }, @@ -64,7 +64,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_MISS", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match although the valid bit was set and there were enough Egress credits.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match although the valid bit was set and there were enough Egress credits.", "UMask": "0x10", "Unit": "QPI LL" }, @@ -74,7 +74,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_RBT_HIT", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the route-back table (RBT) specified that the transaction should not trigger a direct2core tranaction. This is common for IO transactions. There were enough Egress credits and the RBT tag matched but the valid bit was not set.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the route-back table (RBT) specified that the transaction should not trigger a direct2core transaction. This is common for IO transactions. There were enough Egress credits and the RBT tag matched but the valid bit was not set.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -84,7 +84,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.FAILURE_RBT_MISS", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and the valid bit was not set although there were enough Egress credits.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn failed because the RBT tag did not match and the valid bit was not set although there were enough Egress credits.", "UMask": "0x40", "Unit": "QPI LL" }, @@ -94,7 +94,7 @@ "EventCode": "0x13", "EventName": "UNC_Q_DIRECT2CORE.SUCCESS_RBT_HIT", "PerPkg": "1", - "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exlusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn was successful. There were sufficient credits, the RBT valid bit was set and there was an RBT tag match. The message was marked to spawn direct2core.", + "PublicDescription": "Counts the number of DRS packets that we attempted to do direct2core on. There are 4 mutually exclusive filters. Filter [0] can be used to get successful spawns, while [1:3] provide the different failure cases. Note that this does not count packets that are not candidates for Direct2Core. The only candidates for Direct2Core are DRS packets destined for Cbos.; The spawn was successful. There were sufficient credits, the RBT valid bit was set and there was an RBT tag match. The message was marked to spawn direct2core.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -131,7 +131,7 @@ "EventCode": "0x9", "EventName": "UNC_Q_RxL_BYPASSED", "PerPkg": "1", - "PublicDescription": "Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transfered, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.", + "PublicDescription": "Counts the number of times that an incoming flit was able to bypass the flit buffer and pass directly across the BGF and into the Egress. This is a latency optimization, and should generally be the common case. If this value is less than the number of flits transferred, it implies that there was queueing getting onto the ring, and thus the transactions saw higher latency.", "Unit": "QPI LL" }, { @@ -443,7 +443,7 @@ "EventCode": "0x1", "EventName": "UNC_Q_RxL_FLITS_G0.DATA", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flitsreceived over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.", + "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits received over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -453,7 +453,7 @@ "EventCode": "0x1", "EventName": "UNC_Q_RxL_FLITS_G0.IDLE", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of flits received over QPI that do not hold protocol payload. When QPI is not in a power saving state, it continuously transmits flits across the link. When there are no protocol flits to send, it will send IDLE and NULL flits across. These flits sometimes do carry a payload, such as credit returns, but are generall not considered part of the QPI bandwidth.", + "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of flits received over QPI that do not hold protocol payload. When QPI is not in a power saving state, it continuously transmits flits across the link. When there are no protocol flits to send, it will send IDLE and NULL flits across. These flits sometimes do carry a payload, such as credit returns, but are generally not considered part of the QPI bandwidth.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -463,7 +463,7 @@ "EventCode": "0x1", "EventName": "UNC_Q_RxL_FLITS_G0.NON_DATA", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits received across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.", + "PublicDescription": "Counts the number of flits received from the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits received across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -474,7 +474,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.DRS", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data.", "UMask": "0x18", "Unit": "QPI LL" }, @@ -485,7 +485,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.DRS_DATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).", "UMask": "0x8", "Unit": "QPI LL" }, @@ -496,7 +496,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.DRS_NONDATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits received over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits received over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.", "UMask": "0x10", "Unit": "QPI LL" }, @@ -507,7 +507,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.HOM", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits received over QPI on the home channel.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits received over QPI on the home channel.", "UMask": "0x6", "Unit": "QPI LL" }, @@ -518,7 +518,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.HOM_NONREQ", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits received over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits received over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -529,7 +529,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.HOM_REQ", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request received over QPI on the home channel. This basically counts the number of remote memory requests received over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request received over QPI on the home channel. This basically counts the number of remote memory requests received over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -540,7 +540,7 @@ "EventName": "UNC_Q_RxL_FLITS_G1.SNP", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits received over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are received on the home channel.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits received over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are received on the home channel.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -551,7 +551,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NCB", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.", "UMask": "0xC", "Unit": "QPI LL" }, @@ -562,7 +562,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NCB_DATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -573,7 +573,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NCB_NONDATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.", "UMask": "0x8", "Unit": "QPI LL" }, @@ -584,7 +584,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NCS", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits received over QPI. This includes extended headers.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits received over QPI. This includes extended headers.", "UMask": "0x10", "Unit": "QPI LL" }, @@ -595,7 +595,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NDR_AD", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -606,7 +606,7 @@ "EventName": "UNC_Q_RxL_FLITS_G2.NDR_AK", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.", + "PublicDescription": "Counts the number of flits received from the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits received over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -1227,7 +1227,7 @@ "Counter": "0,1,2,3", "EventName": "UNC_Q_TxL_FLITS_G0.DATA", "PerPkg": "1", - "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits transmitted over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of data flits transmitted over QPI. Each flit contains 64b of data. This includes both DRS and NCB data flits (coherent and non-coherent). This can be used to calculate the data bandwidth of the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This does not include the header flits that go in data packets.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -1236,7 +1236,7 @@ "Counter": "0,1,2,3", "EventName": "UNC_Q_TxL_FLITS_G0.NON_DATA", "PerPkg": "1", - "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits transmitted across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. It includes filters for Idle, protocol, and Data Flits. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time (for L0) or 4B instead of 8B for L0p.; Number of non-NULL non-data flits transmitted across QPI. This basically tracks the protocol overhead on the QPI link. One can get a good picture of the QPI-link characteristics by evaluating the protocol flits, data flits, and idle/null flits. This includes the header flits for data packets.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -1246,7 +1246,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.DRS", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency.", "UMask": "0x18", "Unit": "QPI LL" }, @@ -1256,7 +1256,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.DRS_DATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of data flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the data flits (not the header).", "UMask": "0x8", "Unit": "QPI LL" }, @@ -1266,7 +1266,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.DRS_NONDATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of protocol flits transmitted over QPI on the DRS (Data Response) channel. DRS flits are used to transmit data with coherency. This does not count data flits transmitted over the NCB channel which transmits non-coherent data. This includes only the header flits (not the data). This includes extended headers.", "UMask": "0x10", "Unit": "QPI LL" }, @@ -1276,7 +1276,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.HOM", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits transmitted over QPI on the home channel.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of flits transmitted over QPI on the home channel.", "UMask": "0x6", "Unit": "QPI LL" }, @@ -1286,7 +1286,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.HOM_NONREQ", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits transmitted over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of non-request flits transmitted over QPI on the home channel. These are most commonly snoop responses, and this event can be used as a proxy for that.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -1296,7 +1296,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.HOM_REQ", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request transmitted over QPI on the home channel. This basically counts the number of remote memory requests transmitted over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of data request transmitted over QPI on the home channel. This basically counts the number of remote memory requests transmitted over QPI. In conjunction with the local read count in the Home Agent, one can calculate the number of LLC Misses.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -1306,7 +1306,7 @@ "EventName": "UNC_Q_TxL_FLITS_G1.SNP", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits transmitted over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are transmitted on the home channel.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for SNP, HOM, and DRS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the number of snoop request flits transmitted over QPI. These requests are contained in the snoop channel. This does not include snoop responses, which are transmitted on the home channel.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -1317,7 +1317,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NCB", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass flits. These packets are generally used to transmit non-coherent data across QPI.", "UMask": "0xC", "Unit": "QPI LL" }, @@ -1328,7 +1328,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NCB_DATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not te NCB headers.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass data flits. These flits are generally used to transmit non-coherent data across QPI. This does not include a count of the DRS (coherent) data flits. This only counts the data flits, not the NCB headers.", "UMask": "0x4", "Unit": "QPI LL" }, @@ -1339,7 +1339,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NCB_NONDATA", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of Non-Coherent Bypass non-data flits. These packets are generally used to transmit non-coherent data across QPI, and the flits counted here are for headers and other non-data flits. This includes extended headers.", "UMask": "0x8", "Unit": "QPI LL" }, @@ -1350,7 +1350,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NCS", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits transmitted over QPI. This includes extended headers.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Number of NCS (non-coherent standard) flits transmitted over QPI. This includes extended headers.", "UMask": "0x10", "Unit": "QPI LL" }, @@ -1361,7 +1361,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NDR_AD", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets to the local socket which use the AK ring.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -1372,7 +1372,7 @@ "EventName": "UNC_Q_TxL_FLITS_G2.NDR_AK", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Counts the number of flits trasmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transfering a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.", + "PublicDescription": "Counts the number of flits transmitted across the QPI Link. This is one of three groups that allow us to track flits. It includes filters for NDR, NCB, and NCS message classes. Each flit is made up of 80 bits of information (in addition to some ECC data). In full-width (L0) mode, flits are made up of four fits, each of which contains 20 bits of data (along with some additional ECC data). In half-width (L0p) mode, the fits are only 10 bits, and therefore it takes twice as many fits to transmit a flit. When one talks about QPI speed (for example, 8.0 GT/s), the transfers here refer to fits. Therefore, in L0, the system will transfer 1 flit at the rate of 1/4th the QPI speed. One can calculate the bandwidth of the link by taking: flits*80b/time. Note that this is not the same as data bandwidth. For example, when we are transferring a 64B cacheline across QPI, we will break it into 9 flits -- 1 with header information and 8 with 64 bits of actual data and an additional 16 bits of other information. To calculate data bandwidth, one should therefore do: data flits * 8B / time.; Counts the total number of flits transmitted over the NDR (Non-Data Response) channel. This channel is used to send a variety of protocol flits including grants and completions. This is only for NDR packets destined for Route-thru to a remote socket.", "UMask": "0x2", "Unit": "QPI LL" }, @@ -1511,7 +1511,7 @@ "EventName": "UNC_Q_TxR_AD_SNP_CREDIT_OCCUPANCY.VN0", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO fro Snoop messages on AD.", + "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO for Snoop messages on AD.", "UMask": "0x1", "Unit": "QPI LL" }, @@ -1522,7 +1522,7 @@ "EventName": "UNC_Q_TxR_AD_SNP_CREDIT_OCCUPANCY.VN1", "ExtSel": "1", "PerPkg": "1", - "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO fro Snoop messages on AD.", + "PublicDescription": "Occupancy event that tracks the number of link layer credits into the R3 (for transactions across the BGF) available in each cycle. Flow Control FIFO for Snoop messages on AD.", "UMask": "0x2", "Unit": "QPI LL" }, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-memory.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-memory.json index 63b49b712c6217d2c98b17368729d7f6b8df7f36..ed60ebca35cb88d2fba3dd12db69f2e2a0e94853 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-memory.json @@ -188,7 +188,7 @@ "EventCode": "0x9", "EventName": "UNC_M_ECC_CORRECTABLE_ERRORS", "PerPkg": "1", - "PublicDescription": "Counts the number of ECC errors detected and corrected by the iMC on this channel. This counter is only useful with ECC DRAM devices. This count will increment one time for each correction regardless of the number of bits corrected. The iMC can correct up to 4 bit errors in independent channel mode and 8 bit erros in lockstep mode.", + "PublicDescription": "Counts the number of ECC errors detected and corrected by the iMC on this channel. This counter is only useful with ECC DRAM devices. This count will increment one time for each correction regardless of the number of bits corrected. The iMC can correct up to 4 bit errors in independent channel mode and 8 bit errors in lockstep mode.", "Unit": "iMC" }, { diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-other.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-other.json index af289aa6c98ea54c499d047a2c832c625635fd70..6c7ddf642fc381a6739c557f7be45b016d817785 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-other.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-other.json @@ -2097,7 +2097,7 @@ "EventCode": "0x33", "EventName": "UNC_R3_VNA_CREDITS_ACQUIRED", "PerPkg": "1", - "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.", + "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.", "Unit": "R3QPI" }, { @@ -2106,7 +2106,7 @@ "EventCode": "0x33", "EventName": "UNC_R3_VNA_CREDITS_ACQUIRED.AD", "PerPkg": "1", - "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.", + "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.", "UMask": "0x1", "Unit": "R3QPI" }, @@ -2116,7 +2116,7 @@ "EventCode": "0x33", "EventName": "UNC_R3_VNA_CREDITS_ACQUIRED.BL", "PerPkg": "1", - "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credts from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transfered). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transfered in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.", + "PublicDescription": "Number of QPI VNA Credit acquisitions. This event can be used in conjunction with the VNA In-Use Accumulator to calculate the average lifetime of a credit holder. VNA credits are used by all message classes in order to communicate across QPI. If a packet is unable to acquire credits, it will then attempt to use credits from the VN0 pool. Note that a single packet may require multiple flit buffers (i.e. when data is being transferred). Therefore, this event will increment by the number of credits acquired in each cycle. Filtering based on message class is not provided. One can count the number of packets transferred in a given message class using an qfclk event.; Filter for the Home (HOM) message class. HOM is generally used to send requests, request responses, and snoop responses.", "UMask": "0x4", "Unit": "R3QPI" }, diff --git a/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json b/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json index 0ba63a97ddfa79d4f107c7d19d15e998406df88c..74c87217d75c90256c0c418ae99e6ab35f4a64fc 100644 --- a/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json +++ b/tools/perf/pmu-events/arch/x86/ivytown/uncore-power.json @@ -601,7 +601,7 @@ "EventCode": "0x80", "EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C0", "PerPkg": "1", - "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", "Unit": "PCU" }, { @@ -610,7 +610,7 @@ "EventCode": "0x80", "EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C3", "PerPkg": "1", - "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", "Unit": "PCU" }, { @@ -619,7 +619,7 @@ "EventCode": "0x80", "EventName": "UNC_P_POWER_STATE_OCCUPANCY.CORES_C6", "PerPkg": "1", - "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with threshholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", + "PublicDescription": "This is an occupancy event that tracks the number of cores that are in the chosen C-State. It can be used by itself to get the average number of cores in that C-state with thresholding to generate histograms, or with other PCU events and occupancy triggering to capture other details.", "Unit": "PCU" }, { @@ -637,7 +637,7 @@ "EventCode": "0x9", "EventName": "UNC_P_PROCHOT_INTERNAL_CYCLES", "PerPkg": "1", - "PublicDescription": "Counts the number of cycles that we are in Interal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.", + "PublicDescription": "Counts the number of cycles that we are in Internal PROCHOT mode. This mode is triggered when a sensor on the die determines that we are too hot and must throttle to avoid damaging the chip.", "Unit": "PCU" }, { diff --git a/tools/perf/pmu-events/arch/x86/jaketown/jkt-metrics.json b/tools/perf/pmu-events/arch/x86/jaketown/jkt-metrics.json index c0fbb4f31241bcadf9b74eae0e035e56a2b4eff0..554f87c03c05f3af157b88b3f0676892c0bd9cc1 100644 --- a/tools/perf/pmu-events/arch/x86/jaketown/jkt-metrics.json +++ b/tools/perf/pmu-events/arch/x86/jaketown/jkt-metrics.json @@ -1,64 +1,247 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -70,8 +253,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -82,16 +265,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_DISPATCHED.THREAD / UOPS_ISSUED.ANY", @@ -101,44 +278,32 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_DISPATCHED.THREAD / (( cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)", + "MetricExpr": "UOPS_DISPATCHED.THREAD / ((cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -149,7 +314,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -161,26 +326,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -198,7 +363,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, @@ -208,12 +373,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cbox_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -261,5 +420,11 @@ "MetricExpr": "(cstate_pkg@c7\\-residency@ / msr@tsc@) * 100", "MetricGroup": "Power", "MetricName": "C7_Pkg_Residency" + }, + { + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" } ] diff --git a/tools/perf/pmu-events/arch/x86/mapfile.csv b/tools/perf/pmu-events/arch/x86/mapfile.csv index 7f2d777fd97f7ae6cf63c6f4671ba25d7e89745a..5e609b8767903db6b24021ea7229713cf704c9b7 100644 --- a/tools/perf/pmu-events/arch/x86/mapfile.csv +++ b/tools/perf/pmu-events/arch/x86/mapfile.csv @@ -1,27 +1,27 @@ Family-model,Version,Filename,EventType -GenuineIntel-6-9[7A],v1.13,alderlake,core +GenuineIntel-6-(97|9A|B7|BA|BE|BF),v1.15,alderlake,core GenuineIntel-6-(1C|26|27|35|36),v4,bonnell,core GenuineIntel-6-(3D|47),v26,broadwell,core GenuineIntel-6-56,v23,broadwellde,core GenuineIntel-6-4F,v19,broadwellx,core GenuineIntel-6-55-[56789ABCDEF],v1.16,cascadelakex,core -GenuineIntel-6-96,v1.03,elkhartlake,core +GenuineIntel-6-9[6C],v1.03,elkhartlake,core GenuineIntel-6-5[CF],v13,goldmont,core GenuineIntel-6-7A,v1.01,goldmontplus,core -GenuineIntel-6-(3C|45|46),v31,haswell,core -GenuineIntel-6-3F,v25,haswellx,core -GenuineIntel-6-(7D|7E|A7),v1.14,icelake,core -GenuineIntel-6-6[AC],v1.15,icelakex,core +GenuineIntel-6-(3C|45|46),v32,haswell,core +GenuineIntel-6-3F,v26,haswellx,core +GenuineIntel-6-(7D|7E|A7),v1.15,icelake,core +GenuineIntel-6-6[AC],v1.16,icelakex,core GenuineIntel-6-3A,v22,ivybridge,core -GenuineIntel-6-3E,v21,ivytown,core +GenuineIntel-6-3E,v22,ivytown,core GenuineIntel-6-2D,v21,jaketown,core GenuineIntel-6-(57|85),v9,knightslanding,core GenuineIntel-6-AA,v1.00,meteorlake,core GenuineIntel-6-1[AEF],v3,nehalemep,core GenuineIntel-6-2E,v3,nehalemex,core GenuineIntel-6-2A,v17,sandybridge,core -GenuineIntel-6-8F,v1.04,sapphirerapids,core -GenuineIntel-6-(37|4C|4D),v14,silvermont,core +GenuineIntel-6-8F,v1.06,sapphirerapids,core +GenuineIntel-6-(37|4A|4C|4D|5A),v14,silvermont,core GenuineIntel-6-(4E|5E|8E|9E|A5|A6),v53,skylake,core GenuineIntel-6-55-[01234],v1.28,skylakex,core GenuineIntel-6-86,v1.20,snowridgex,core diff --git a/tools/perf/pmu-events/arch/x86/sandybridge/snb-metrics.json b/tools/perf/pmu-events/arch/x86/sandybridge/snb-metrics.json index ae7ed267b2a2226036c80cce9c2147ac69771606..5d5a6d6f3bdab3cc554457287ca5b1ca58182178 100644 --- a/tools/perf/pmu-events/arch/x86/sandybridge/snb-metrics.json +++ b/tools/perf/pmu-events/arch/x86/sandybridge/snb-metrics.json @@ -1,64 +1,247 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * min(CPU_CLK_UNHALTED.THREAD, IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: RS_EVENTS.EMPTY_END", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "(12 * ITLB_MISSES.STLB_HIT + ITLB_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: ITLB_MISSES.WALK_COMPLETED", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "12 * (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT + BACLEARS.ANY) / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - (tma_frontend_bound + tma_bad_speculation + tma_retiring)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING) + RESOURCE_STALLS.SB) / (min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "(7 * DTLB_LOAD_MISSES.STLB_HIT + DTLB_LOAD_MISSES.WALK_DURATION) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_UOPS_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS)) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(1 - (MEM_LOAD_UOPS_RETIRED.LLC_HIT / (MEM_LOAD_UOPS_RETIRED.LLC_HIT + 7 * MEM_LOAD_UOPS_MISC_RETIRED.LLC_MISS))) * CYCLE_ACTIVITY.STALLS_L2_PENDING / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_UOPS_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=6@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "RESOURCE_STALLS.SB / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_UOPS_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.FPU_DIV_ACTIVE / CORE_CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "((min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.CYCLES_NO_DISPATCH) + cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=1@ - cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=3@ if (IPC > 1.8) else cpu@UOPS_DISPATCHED.THREAD\\,cmask\\=2@ - RS_EVENTS.EMPTY_CYCLES if (tma_fetch_latency > 0.1) else RESOURCE_STALLS.SB) - RESOURCE_STALLS.SB - min(CPU_CLK_UNHALTED.THREAD, CYCLE_ACTIVITY.STALLS_L1D_PENDING)) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS * FP_COMP_OPS_EXE.X87 / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) / UOPS_DISPATCHED.THREAD", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -70,8 +253,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -82,16 +265,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_DISPATCHED.THREAD / UOPS_ISSUED.ANY", @@ -101,44 +278,32 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_DISPATCHED.THREAD / (( cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2 ) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)", + "MetricExpr": "UOPS_DISPATCHED.THREAD / ((cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@ / 2) if #SMT_on else cpu@UOPS_DISPATCHED.CORE\\,cmask\\=1@)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -149,7 +314,7 @@ }, { "BriefDescription": "Fraction of Uops delivered by the DSB (aka Decoded ICache; or Uop Cache)", - "MetricExpr": "IDQ.DSB_UOPS / (( IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS ) )", + "MetricExpr": "IDQ.DSB_UOPS / ((IDQ.DSB_UOPS + LSD.UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS))", "MetricGroup": "DSB;Fed;FetchBW", "MetricName": "DSB_Coverage" }, @@ -161,26 +326,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE ) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * ( FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE ) + 8 * SIMD_FP_256.PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_COMP_OPS_EXE.SSE_SCALAR_SINGLE + FP_COMP_OPS_EXE.SSE_SCALAR_DOUBLE) + 2 * FP_COMP_OPS_EXE.SSE_PACKED_DOUBLE + 4 * (FP_COMP_OPS_EXE.SSE_PACKED_SINGLE + SIMD_FP_256.PACKED_DOUBLE) + 8 * SIMD_FP_256.PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -198,7 +363,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json index 348476ce8107f68f73bcedf8968d6de9331dd9c6..c05c741e22db19e5c3b69010350e33e64d151c08 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/cache.json @@ -35,7 +35,7 @@ "UMask": "0x2" }, { - "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability.", + "BriefDescription": "Number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability.", "CollectPEBSRecord": "2", "Counter": "0,1,2,3", "CounterMask": "1", @@ -43,7 +43,7 @@ "EventCode": "0x48", "EventName": "L1D_PEND_MISS.FB_FULL_PERIODS", "PEBScounters": "0,1,2,3", - "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailablability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", + "PublicDescription": "Counts number of phases a demand request has waited due to L1D Fill Buffer (FB) unavailability. Demand requests include cacheable/uncacheable demand load, store, lock or SW prefetch accesses.", "SampleAfterValue": "1000003", "Speculative": "1", "UMask": "0x2" diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json index 44ecf38ad970e3120cf2547498fe19bf6584c14d..ff0d47ce8e9a1d353a22541065f2d97aaa808530 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/frontend.json @@ -11,6 +11,17 @@ "Speculative": "1", "UMask": "0x1" }, + { + "BriefDescription": "Cycles the Microcode Sequencer is busy.", + "CollectPEBSRecord": "2", + "Counter": "0,1,2,3", + "EventCode": "0x87", + "EventName": "DECODE.MS_BUSY", + "PEBScounters": "0,1,2,3", + "SampleAfterValue": "500009", + "Speculative": "1", + "UMask": "0x2" + }, { "BriefDescription": "DSB-to-MITE switch true penalty cycles.", "CollectPEBSRecord": "2", diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/pipeline.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/pipeline.json index df4f3d714e6e06f4632062d2040a1f245aec5fce..b2f0d9393d3ca98b71e1378eaa7c7b8f8154e161 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/pipeline.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/pipeline.json @@ -80,10 +80,10 @@ "EventCode": "0xc1", "EventName": "ASSISTS.ANY", "PEBScounters": "0,1,2,3,4,5,6,7", - "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware Examples include AD (page Access Dirty), FP and AVX related assists.", + "PublicDescription": "Counts the number of occurrences where a microcode assist is invoked by hardware. Examples include AD (page Access Dirty), FP and AVX related assists.", "SampleAfterValue": "100003", "Speculative": "1", - "UMask": "0x1f" + "UMask": "0x1b" }, { "BriefDescription": "All branch instructions retired.", diff --git a/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json b/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json index e194dfc5c25b5febc3d963a333d150232b1e79fd..9ec42a68c160a5d55e81060d4f401dec44e913df 100644 --- a/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json +++ b/tools/perf/pmu-events/arch/x86/sapphirerapids/spr-metrics.json @@ -1,16 +1,818 @@ [ + { + "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", + "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "(topdown\\-fetch\\-lat / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS)", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE_DATA.STALLS / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_TAG.STALLS / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(tma_branch_mispredicts / tma_bad_speculation) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (tma_branch_mispredicts / tma_bad_speculation)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "INT_MISC.UNKNOWN_BRANCH_CYCLES / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: FRONTEND_RETIRED.UNKNOWN_BRANCH", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "DECODE.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: FRONTEND_RETIRED.MS_FLOWS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", + "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "topdown\\-br\\-mispredict / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: TOPDOWN.BR_MISPREDICT_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", + "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "topdown\\-mem\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - MEMORY_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((25 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (24 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(24 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(9 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "(XQ.FULL_CYCLES + L1D_PEND_MISS.L2_STALLS) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "((MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS) - tma_pmm_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to memory bandwidth Allocation feature (RDT's memory bandwidth throttling).", + "MetricExpr": "INT_MISC.MBA_STALLS / CLKS", + "MetricGroup": "MemoryBW;Offcore;Server;TopdownL5;tma_mem_bandwidth_group", + "MetricName": "tma_mba_stalls", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "(54.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "(119 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "((108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (108 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a", + "MetricExpr": "(((1 - ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) / ((19 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 10 * ((MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + (MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))) + (25 * (MEM_LOAD_RETIRED.LOCAL_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + 33 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))))))) * (MEMORY_ACTIVITY.STALLS_L3_MISS / CLKS)) if (1000000 * (MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM) > MEM_LOAD_RETIRED.L1_MISS) else 0)", + "MetricGroup": "MemoryBound;Server;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_pmm_bound", + "PublicDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((MEM_STORE_RETIRED.L2_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(28 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores", + "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_streaming_stores", + "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS) / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "CPU_CLK_UNHALTED.PAUSE / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: CPU_CLK_UNHALTED.PAUSE_INST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to LFENCE Instructions.", + "MetricExpr": "13 * MISC2_RETIRED.LFENCE / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_memory_fence", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "160 * ASSISTS.SSE_AVX_MIX / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the Advanced Matrix Extensions (AMX) execution engine was busy with tile (arithmetic) operations", + "MetricExpr": "EXE.AMX_BUSY / CORE_CLKS", + "MetricGroup": "Compute;HPC;Server;TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_amx_busy", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5_11 + UOPS_DISPATCHED.PORT_6) / (5 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3_10", + "MetricExpr": "UOPS_DISPATCHED.PORT_2_3_10 / (3 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", + "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector + tma_fp_amx", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) matrix uops fraction the CPU has retired (aggregated across all supported FP datatypes in AMX engine)", + "MetricExpr": "cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=1@ / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;HPC;Pipeline;Server;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_amx", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) matrix uops fraction the CPU has retired (aggregated across all supported FP datatypes in AMX engine). Refer to AMX_Busy and GFLOPs metrics for actual AMX utilization and FP performance, resp.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_int_vector_128b + tma_int_vector_256b + tma_shuffles", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_int_operations", + "PublicDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents 128-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.", + "MetricExpr": "(INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_int_vector_128b", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents 256-bit vector Integer ADD/SUB/SAD or VNNI (Vector Neural Network Instructions) uops fraction the CPU has retired.", + "MetricExpr": "(INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;IntVector;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_int_vector_256b", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic Integer (Int) matrix uops fraction the CPU has retired (aggregated across all supported Int datatypes in AMX engine)", + "MetricExpr": "cpu@AMX_OPS_RETIRED.INT8\\,cmask\\=1@ / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;HPC;IntVector;Pipeline;Server;TopdownL4;tma_int_operations_group", + "MetricName": "tma_int_amx", + "PublicDescription": "This metric approximates arithmetic Integer (Int) matrix uops fraction the CPU has retired (aggregated across all supported Int datatypes in AMX engine). Refer to AMX_Busy and TIOPs metrics for actual AMX utilization and Int performance, resp.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Shuffle (cross \"vector lane\" data transfers) uops fraction the CPU has retired.", + "MetricExpr": "INT_VEC_RETIRED.SHUFFLES / (tma_retiring * SLOTS)", + "MetricGroup": "HPC;Pipeline;TopdownL4;tma_int_operations_group", + "MetricName": "tma_shuffles", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_UOP_RETIRED.ANY / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.MACRO_FUSED / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fused_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused", + "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED) / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_non_fused_branches", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_int_operations + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "topdown\\-heavy\\-ops / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences. Sample with: UOPS_RETIRED.HEAVY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "UOPS_RETIRED.MS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: UOPS_RETIRED.MS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * cpu@ASSISTS.ANY\\,umask\\=0x1B@ / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults", + "MetricExpr": "99 * ASSISTS.PAGE_FAULT / SLOTS", + "MetricGroup": "TopdownL5;tma_assists_group", + "MetricName": "tma_page_faults", + "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Page Faults. A Page Fault may apply on first application access to a memory page. Note operating system handling of page faults accounts for the majority of its cost.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists", + "MetricExpr": "30 * ASSISTS.FP / SLOTS", + "MetricGroup": "HPC;TopdownL5;tma_assists_group", + "MetricName": "tma_fp_assists", + "PublicDescription": "This metric roughly estimates fraction of slots the CPU retired uops as a result of handing Floating Point (FP) Assists. FP Assist may apply when working with very small floating point values (so-called denormals).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops as a result of handing SSE to AVX* or AVX* to SSE transition Assists. ", + "MetricExpr": "63 * ASSISTS.SSE_AVX_MIX / SLOTS", + "MetricGroup": "HPC;TopdownL5;tma_assists_group", + "MetricName": "tma_avx_assists", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources. Sample with: FRONTEND_RETIRED.MS_FLOWS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" + }, + { + "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_fb_full / (tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", + "MetricGroup": "Mem;MemoryBW;Offcore", + "MetricName": "Memory_Bandwidth" + }, + { + "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)))", + "MetricGroup": "Mem;MemoryLat;Offcore", + "MetricName": "Memory_Latency" + }, + { + "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_pmm_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ", + "MetricGroup": "Mem;MemoryTLB;Offcore", + "MetricName": "Memory_Data_TLBs" + }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, + { + "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", + "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", + "MetricName": "Big_Code" + }, + { + "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", + "MetricGroup": "Fed;FetchBW;Frontend", + "MetricName": "Instruction_Fetch_BW" + }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, + { + "BriefDescription": "Uops Per Instruction", + "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;Ret;Retire", + "MetricName": "UPI" + }, + { + "BriefDescription": "Instruction per taken branch", + "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN", + "MetricGroup": "Branches;Fed;FetchBW", + "MetricName": "UpTB" + }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -20,13 +822,13 @@ { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", "MetricExpr": "TOPDOWN.SLOTS", - "MetricGroup": "TmaL1", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, { "BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor", - "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1", - "MetricGroup": "SMT;TmaL1", + "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1", + "MetricGroup": "SMT;tma_L1_group", "MetricName": "Slots_Utilization" }, { @@ -38,29 +840,35 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 ) / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5 ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(FP_ARITH_DISPATCHED.PORT_0 + FP_ARITH_DISPATCHED.PORT_1 + FP_ARITH_DISPATCHED.PORT_5) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, + { + "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", + "MetricGroup": "Cor;SMT", + "MetricName": "Core_Bound_Likely" + }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", "MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED", @@ -105,13 +913,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -132,21 +940,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.128B_PACKED_HALF)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.256B_PACKED_HALF)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.512B_PACKED_HALF)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -161,7 +969,7 @@ { "BriefDescription": "Instructions per Integer Arithmetic AMX operation (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / AMX_OPS_RETIRED.INT8", - "MetricGroup": "IntVector;InsType;Server", + "MetricGroup": "InsType;IntVector;Server", "MetricName": "IpArith_AMX_Int8", "PublicDescription": "Instructions per Integer Arithmetic AMX operation (lower number means higher occurrence rate). Operations factored per matrices' sizes of the AMX instructions." }, @@ -172,11 +980,17 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, + { + "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.", + "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@", + "MetricGroup": "Pipeline;Ret", + "MetricName": "Retire" + }, { "BriefDescription": "Estimated fraction of retirement-cycles dealing with repeat instructions", "MetricExpr": "INST_RETIRED.REP_ITERATION / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@", @@ -213,6 +1027,12 @@ "MetricGroup": "DSBmiss", "MetricName": "DSB_Switch_Cost" }, + { + "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))", + "MetricGroup": "DSBmiss;Fed", + "MetricName": "DSB_Misses" + }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -225,6 +1045,12 @@ "MetricGroup": "Bad;BadSpec;BrMispredicts", "MetricName": "IpMispredict" }, + { + "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricGroup": "Bad;BrMispredicts", + "MetricName": "Branch_Misprediction_Cost" + }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -239,7 +1065,7 @@ }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, @@ -251,7 +1077,7 @@ }, { "BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)", - "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )", + "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)", "MetricGroup": "Bad;Branches", "MetricName": "Other_Branches" }, @@ -264,67 +1090,67 @@ { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 4 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (4 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, @@ -354,37 +1180,37 @@ }, { "BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)", - "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_Silent_PKI" }, { "BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction", - "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_NonSilent_PKI" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -396,26 +1222,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF ) + 2 * ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF ) + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * ( FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16 ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR_HALF) + 2 * (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED2.COMPLEX_SCALAR_HALF) + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED2.128B_PACKED_HALF + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * (FP_ARITH_INST_RETIRED2.256B_PACKED_HALF + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) + 32 * FP_ARITH_INST_RETIRED2.512B_PACKED_HALF + 4 * AMX_OPS_RETIRED.BF16) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Tera Integer (matrix) Operations Per Second", - "MetricExpr": "( 8 * AMX_OPS_RETIRED.INT8 / 1000000000000 ) / duration_time", + "MetricExpr": "(8 * AMX_OPS_RETIRED.INT8 / 1e12) / duration_time", "MetricGroup": "Cor;HPC;IntVector;Server", "MetricName": "TIOPS" }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, @@ -439,13 +1265,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( uncore_cha_0@event\\=0x1@ / duration_time )", + "MetricExpr": "1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -457,32 +1283,32 @@ }, { "BriefDescription": "Average latency of data read request to external 3D X-Point memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / uncore_cha_0@event\\=0x1@ )", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": "(1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM) / uncore_cha_0@event\\=0x1@)", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_PMM_Read_Latency" }, { "BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": " 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / uncore_cha_0@event\\=0x1@", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": " 1000000000 * (UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR) / uncore_cha_0@event\\=0x1@", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_DRAM_Read_Latency" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for reads [GB / sec]", - "MetricExpr": "( ( 64 * UNC_M_PMM_RPQ_INSERTS / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * UNC_M_PMM_RPQ_INSERTS / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Read_BW" }, { "BriefDescription": "Average 3DXP Memory Bandwidth Use for Writes [GB / sec]", - "MetricExpr": "( ( 64 * UNC_M_PMM_WPQ_INSERTS / 1000000000 ) / duration_time )", - "MetricGroup": "Mem;MemoryBW;SoC;Server", + "MetricExpr": "((64 * UNC_M_PMM_WPQ_INSERTS / 1000000000) / duration_time)", + "MetricGroup": "Mem;MemoryBW;Server;SoC", "MetricName": "PMM_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]", "MetricExpr": "UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Write_BW" }, { @@ -491,12 +1317,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "uncore_cha_0@event\\=0x1@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -528,11 +1348,10 @@ "MetricName": "C6_Pkg_Residency" }, { - "BriefDescription": "Percentage of time spent in the active CPU power state C0", - "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC", - "MetricGroup": "", - "MetricName": "cpu_utilization_percent", - "ScaleUnit": "1%" + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" }, { "BriefDescription": "CPU operating frequency (in GHz)", @@ -541,13 +1360,6 @@ "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", "MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY", @@ -566,7 +1378,7 @@ "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { @@ -594,7 +1406,7 @@ "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { @@ -620,42 +1432,42 @@ }, { "BriefDescription": "Ratio of number of code read requests missing last level core cache (includes demand w/ prefetches) to the total number of completed instructions", - "MetricExpr": "( UNC_CHA_TOR_INSERTS.IA_MISS_CRD ) / INST_RETIRED.ANY", + "MetricExpr": "UNC_CHA_TOR_INSERTS.IA_MISS_CRD / INST_RETIRED.ANY", "MetricGroup": "", "MetricName": "llc_code_read_mpi_demand_plus_prefetch", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD / UNC_CHA_TOR_INSERTS.IA_MISS_DRD ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to local memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_LOCAL) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency_for_local_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to remote memory in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_REMOTE) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_latency_for_remote_requests", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to Intel(R) Optane(TM) Persistent Memory(PMEM) in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PMM ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_PMM) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_to_pmem_latency", "ScaleUnit": "1ns" }, { "BriefDescription": "Average latency of a last level cache (LLC) demand data read miss (read memory access) addressed to DRAM in nano seconds", - "MetricExpr": "( ( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time )", + "MetricExpr": "( 1000000000 * ( UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR / UNC_CHA_TOR_INSERTS.IA_MISS_DRD_DDR ) / ( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_TOR_OCCUPANCY.IA_MISS_DRD_DDR) * #num_packages ) ) ) * duration_time", "MetricGroup": "", "MetricName": "llc_demand_data_read_miss_to_dram_latency", "ScaleUnit": "1ns" @@ -699,14 +1511,14 @@ "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE ) / ( UNC_CHA_TOR_INSERTS.IA_MISS_DRD_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_LOCAL + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_REMOTE + UNC_CHA_TOR_INSERTS.IA_MISS_DRD_PREF_REMOTE )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" }, { @@ -720,7 +1532,7 @@ "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)", "MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "upi_data_transmit_bw_only_data", + "MetricName": "upi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { @@ -769,35 +1581,35 @@ "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", "MetricExpr": "( UNC_CHA_TOR_INSERTS.IO_PCIRDCUR * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", "MetricExpr": "(( UNC_CHA_TOR_INSERTS.IO_ITOM + UNC_CHA_TOR_INSERTS.IO_ITOMCACHENEAR ) * 64 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.DSB_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_decoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MITE_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MS_UOPS / ( IDQ.DSB_UOPS + IDQ.MITE_UOPS + IDQ.MS_UOPS + LSD.UOPS ) )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { @@ -827,264 +1639,5 @@ "MetricGroup": "", "MetricName": "llc_miss_remote_memory_bandwidth_write", "ScaleUnit": "1MB/s" - }, - { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) )", - "MetricGroup": "TmaL1;PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( topdown\\-fetch\\-lat / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) )", - "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ICACHE_DATA.STALLS / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ICACHE_TAG.STALLS / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( INT_MISC.UNKNOWN_BRANCH_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( DECODE.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 3 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( max( 0 , ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) - ( ( topdown\\-fetch\\-lat / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) ) ) )", - "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )", - "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK ) / ( CPU_CLK_UNHALTED.DISTRIBUTED ) / 2 )", - "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "( 100 * ( topdown\\-br\\-mispredict / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( max( 0 , ( max( 1 - ( ( topdown\\-fe\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) - INT_MISC.UOP_DROPPING / ( slots ) ) + ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) , 0 ) ) - ( topdown\\-br\\-mispredict / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) )", - "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "( 100 * ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "( 100 * ( topdown\\-mem\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( EXE_ACTIVITY.BOUND_ON_LOADS - MEMORY_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( MEMORY_ACTIVITY.STALLS_L1D_MISS - MEMORY_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( MEMORY_ACTIVITY.STALLS_L2_MISS - MEMORY_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) - ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) ) ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_dram_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric roughly estimates (based on idle latencies) how often the CPU was stalled on accesses to external 3D-Xpoint (Crystal Ridge, a.k.a. IXP) memory by loads, PMM stands for Persistent Memory Module. ", - "MetricExpr": "100 * ( min( ( ( ( ( 1 - ( ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) / ( ( 19 * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + 10 * ( ( MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) + ( 25 * ( ( MEM_LOAD_RETIRED.LOCAL_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) + 33 * ( ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) ) ) ) ) ) ) * ( MEMORY_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) ) ) if ( ( 1000000 ) * ( MEM_LOAD_L3_MISS_RETIRED.REMOTE_PMM + MEM_LOAD_RETIRED.LOCAL_PMM ) > MEM_LOAD_RETIRED.L1_MISS ) else 0 ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;Server;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_pmm_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "( 100 * ( max( 0 , ( topdown\\-be\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-mem\\-bound / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) ) + ( 0 * slots )", - "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", - "MetricExpr": "( 100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@ ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - EXE_ACTIVITY.BOUND_ON_LOADS ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * cpu@EXE_ACTIVITY.2_PORTS_UTIL\\,umask\\=0xc@ ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) + ( 0 * slots )", - "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_ports_utilization_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "( 100 * ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "( 100 * ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) ) + ( 0 * slots )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", - "MetricExpr": "100 * ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) + ( cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=0x1@ / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) )", - "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fp_arith_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall Integer (Int) select operations fraction the CPU has executed (retired). Vector/Matrix Int operations and shuffles are counted. Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain.", - "MetricExpr": "100 * ( ( ( INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( INT_VEC_RETIRED.SHUFFLES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_int_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * MEM_UOP_RETIRED.ANY / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_memory_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.MACRO_FUSED / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fused_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_non_fused_branches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.", - "MetricExpr": "100 * ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_nop_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", - "MetricExpr": "100 * ( max( 0 , ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) - ( ( ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE + FP_ARITH_INST_RETIRED2.SCALAR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE + FP_ARITH_INST_RETIRED2.VECTOR ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) , ( 1 ) ) ) + ( cpu@AMX_OPS_RETIRED.BF16\\,cmask\\=0x1@ / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) + ( ( ( INT_VEC_RETIRED.ADD_128 + INT_VEC_RETIRED.VNNI_128 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( INT_VEC_RETIRED.ADD_256 + INT_VEC_RETIRED.MUL_256 + INT_VEC_RETIRED.VNNI_256 ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( INT_VEC_RETIRED.SHUFFLES / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * MEM_UOP_RETIRED.ANY / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.MACRO_FUSED / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - INST_RETIRED.MACRO_FUSED ) / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) + ( ( max( 0 , ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) ) * INST_RETIRED.NOP / ( ( topdown\\-retiring / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) * ( slots ) ) ) ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_other_light_ops_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "( 100 * ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) ) + ( 0 * slots )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", - "MetricExpr": "100 * ( ( topdown\\-heavy\\-ops / ( topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound ) ) - ( UOPS_RETIRED.MS / ( slots ) ) )", - "MetricGroup": "TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_few_uops_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( UOPS_RETIRED.MS / ( slots ) )", - "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_microcode_sequencer_percent", - "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json b/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json index 73fa72d3dcb158a41767c5661079543356577bcb..f138b9836b514f5e31af9ff0d6c4e6d25c7a5da2 100644 --- a/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/skylake/skl-metrics.json @@ -1,148 +1,694 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "9 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (9 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((18.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM + (16.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(16.5 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(6.5 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 9 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(22 * Average_Frequency) * OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" }, { - "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts", - "MetricName": "Mispredictions" + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions", + "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fused_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused", + "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_non_fused_branches", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT", - "MetricName": "Mispredictions_SMT" + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" }, { "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "Memory_Bandwidth" }, - { - "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", - "MetricGroup": "Mem;MemoryBW;Offcore_SMT", - "MetricName": "Memory_Bandwidth_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (10 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))", "MetricGroup": "Mem;MemoryLat;Offcore", "MetricName": "Memory_Latency" }, - { - "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (10 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )", - "MetricGroup": "Mem;MemoryLat;Offcore_SMT", - "MetricName": "Memory_Latency_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ", "MetricGroup": "Mem;MemoryTLB;Offcore", "MetricName": "Memory_Data_TLBs" }, - { - "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", - "MetricGroup": "Mem;MemoryTLB;Offcore_SMT", - "MetricName": "Memory_Data_TLBs_SMT" - }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, - { - "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "Ret_SMT", - "MetricName": "Branching_Overhead_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT", - "MetricName": "Big_Code_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", "MetricGroup": "Fed;FetchBW;Frontend", "MetricName": "Instruction_Fetch_BW" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))", - "MetricGroup": "Fed;FetchBW;Frontend_SMT", - "MetricName": "Instruction_Fetch_BW_SMT" - }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -160,8 +706,8 @@ }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -172,16 +718,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -191,63 +731,38 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", "MetricGroup": "Cor;SMT", "MetricName": "Core_Bound_Likely" }, - { - "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0", - "MetricGroup": "Cor;SMT_SMT", - "MetricName": "Core_Bound_Likely_SMT" - }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -289,13 +804,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -316,14 +831,14 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -335,9 +850,9 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -372,16 +887,10 @@ }, { "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))", "MetricGroup": "DSBmiss;Fed", "MetricName": "DSB_Misses" }, - { - "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )", - "MetricGroup": "DSBmiss;Fed_SMT", - "MetricName": "DSB_Misses_SMT" - }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -396,16 +905,10 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -414,101 +917,95 @@ }, { "BriefDescription": "Fraction of branches that are taken conditionals", - "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches;CodeGen;PGO", "MetricName": "Cond_TK" }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, { "BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps", - "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "Jump" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -535,25 +1032,25 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -565,26 +1062,26 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -602,7 +1099,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json index 6a6764e1504b2b9d1cd89548940e7912cc795245..bc8e42554096cfb4c0afd6faa1a5eb428a5ad8d3 100644 --- a/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json +++ b/tools/perf/pmu-events/arch/x86/skylakex/skx-metrics.json @@ -1,148 +1,726 @@ [ { "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Frontend_Bound", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound." + "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Frontend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "(ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@) / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "9 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "2 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "tma_frontend_bound - tma_fetch_latency", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Bad_Speculation", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example." + "MetricExpr": "(UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Bad_Speculation_SMT", - "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "tma_bad_speculation - tma_branch_mispredicts", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", - "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Backend_Bound", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound." + "MetricExpr": "1 - tma_frontend_bound - (UOPS_ISSUED.ANY + 4 * ((INT_MISC.RECOVERY_CYCLES_ANY / 2) if #SMT_on else INT_MISC.RECOVERY_CYCLES)) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(12 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (11 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "Load_Miss_Real_Latency * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (44 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(44 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT + MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM * (1 - (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE / (OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(17 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "((OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2) if #SMT_on else OFFCORE_REQUESTS_BUFFER.SQ_FULL) / CORE_CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory", + "MetricExpr": "(59.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_local_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from local memory. Caching will improve the latency and increase performance. Sample with: MEM_LOAD_L3_MISS_RETIRED.LOCAL_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory", + "MetricExpr": "(127 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_dram", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote memory. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_DRAM_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues", + "MetricExpr": "((89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM + (89.5 * Average_Frequency) * MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Server;Snoop;TopdownL5;tma_mem_latency_group", + "MetricName": "tma_remote_cache", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling loads from remote cache in other sockets including synchronizations issues. This is caused often due to non-optimal NUMA allocations. #link to NUMA article Sample with: MEM_LOAD_L3_MISS_RETIRED.REMOTE_HITM_PS;MEM_LOAD_L3_MISS_RETIRED.REMOTE_FWD_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 11 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "((110 * Average_Frequency) * (OFFCORE_RESPONSE.DEMAND_RFO.L3_MISS.REMOTE_HITM + OFFCORE_RESPONSE.PF_L2_RFO.L3_MISS.REMOTE_HITM) + (47.5 * Average_Frequency) * (OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.HITM_OTHER_CORE + OFFCORE_RESPONSE.PF_L2_RFO.L3_HIT.HITM_OTHER_CORE)) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_HITM_PS;OFFCORE_RESPONSE.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "tma_backend_bound - tma_memory_bound", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_NONE / 2 if #SMT_on else CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "PARTIAL_RAT_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: PARTIAL_RAT_STALLS.SCOREBOARD", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Backend_Bound_SMT", - "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_1 - UOPS_EXECUTED.CORE_CYCLES_GE_2) / 2 if #SMT_on else EXE_ACTIVITY.1_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "((UOPS_EXECUTED.CORE_CYCLES_GE_2 - UOPS_EXECUTED.CORE_CYCLES_GE_3) / 2 if #SMT_on else EXE_ACTIVITY.2_PORTS_UTIL) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise).", + "MetricExpr": "(UOPS_EXECUTED.CORE_CYCLES_GE_3 / 2 if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_3) / CORE_CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_0 + UOPS_DISPATCHED_PORT.PORT_1 + UOPS_DISPATCHED_PORT.PORT_5 + UOPS_DISPATCHED_PORT.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED_PORT.PORT_0", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_1", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED_PORT.PORT_6", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "(UOPS_DISPATCHED_PORT.PORT_2 + UOPS_DISPATCHED_PORT.PORT_3 + UOPS_DISPATCHED_PORT.PORT_7 - UOPS_DISPATCHED_PORT.PORT_4) / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 2 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_2", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_2 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_2", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 3 ([SNB+]Loads and Store-address; [ICL+] Loads) Sample with: UOPS_DISPATCHED_PORT.PORT_3", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_3 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_load_op_utilization_group", + "MetricName": "tma_port_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 4 (Store-data) Sample with: UOPS_DISPATCHED_PORT.PORT_4", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_4 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_4", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 7 ([HSW+]simple Store-address) Sample with: UOPS_DISPATCHED_PORT.PORT_7", + "MetricExpr": "UOPS_DISPATCHED_PORT.PORT_7 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_store_op_utilization_group", + "MetricName": "tma_port_7", + "ScaleUnit": "100%" }, { "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "TopdownL1", - "MetricName": "Retiring", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. " + "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.RETIRE_SLOTS", + "ScaleUnit": "100%" }, { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))", - "MetricGroup": "TopdownL1_SMT", - "MetricName": "Retiring_SMT", - "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. SMT version; use when SMT is enabled and measuring per logical CPU." + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "tma_retiring - tma_heavy_operations", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" }, { - "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts", - "MetricName": "Mispredictions" + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions", + "MetricExpr": "tma_light_operations * UOPS_RETIRED.MACRO_FUSED / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fused_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused", + "MetricExpr": "tma_light_operations * (BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED) / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_non_fused_branches", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / UOPS_RETIRED.RETIRE_SLOTS", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_fused_instructions + tma_non_fused_branches + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY) / SLOTS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "(UOPS_RETIRED.RETIRE_SLOTS / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * (FP_ASSIST.ANY + OTHER_ASSISTS.ANY) / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: OTHER_ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" }, { "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", - "MetricExpr": "100 * ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) )", - "MetricGroup": "Bad;BadSpec;BrMispredicts_SMT", - "MetricName": "Mispredictions_SMT" + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" }, { "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (OFFCORE_REQUESTS_BUFFER.SQ_FULL / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "Memory_Bandwidth" }, - { - "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( OFFCORE_REQUESTS_BUFFER.SQ_FULL / 2 ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) ) + ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( ((L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )) * cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ / CPU_CLK_UNHALTED.THREAD) / #(max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) ", - "MetricGroup": "Mem;MemoryBW;Offcore_SMT", - "MetricName": "Memory_Bandwidth_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) )", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))", "MetricGroup": "Mem;MemoryLat;Offcore", "MetricName": "Memory_Latency" }, - { - "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( CPU_CLK_UNHALTED.THREAD , OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD ) / CPU_CLK_UNHALTED.THREAD - (min( CPU_CLK_UNHALTED.THREAD , cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@ ) / CPU_CLK_UNHALTED.THREAD)) / #(CYCLE_ACTIVITY.STALLS_L3_MISS / CPU_CLK_UNHALTED.THREAD + (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD) - (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD))) ) + ( (( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( (20.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) - (3.5 * ((CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time)) ) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CPU_CLK_UNHALTED.THREAD) / #(( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / CPU_CLK_UNHALTED.THREAD) ) + ( (( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) / ( (MEM_LOAD_RETIRED.L2_HIT * ( 1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) )) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=1@ ) ) * (( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / CPU_CLK_UNHALTED.THREAD)) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) )", - "MetricGroup": "Mem;MemoryLat;Offcore_SMT", - "MetricName": "Memory_Latency_SMT" - }, { "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / CPU_CLK_UNHALTED.THREAD) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency))) ", "MetricGroup": "Mem;MemoryTLB;Offcore", "MetricName": "Memory_Data_TLBs" }, - { - "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", - "MetricExpr": "100 * ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * ( ( (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) / ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (min( 9 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE , max( CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS , 0 ) ) / CPU_CLK_UNHALTED.THREAD) / (max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / CPU_CLK_UNHALTED.THREAD , 0 )) ) + ( (EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) / #((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) ) * ( (( 9 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / #(EXE_ACTIVITY.BOUND_ON_STORES / CPU_CLK_UNHALTED.THREAD) ) ) ", - "MetricGroup": "Mem;MemoryTLB;Offcore_SMT", - "MetricName": "Memory_Data_TLBs_SMT" - }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * ((BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, - { - "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.CONDITIONAL + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "Ret_SMT", - "MetricName": "Branching_Overhead_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))", - "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB_SMT", - "MetricName": "Big_Code_SMT" - }, { "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)))", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", "MetricGroup": "Fed;FetchBW;Frontend", "MetricName": "Instruction_Fetch_BW" }, - { - "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", - "MetricExpr": "100 * ( (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) - (100 * (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=1\\,edge@ ) / CPU_CLK_UNHALTED.THREAD) + (9 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))", - "MetricGroup": "Fed;FetchBW;Frontend_SMT", - "MetricName": "Instruction_Fetch_BW_SMT" - }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, @@ -158,6 +736,12 @@ "MetricGroup": "Branches;Fed;FetchBW", "MetricName": "UpTB" }, + { + "BriefDescription": "Cycles Per Instruction (per Logical Processor)", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", + "MetricName": "CPI" + }, { "BriefDescription": "Per-Logical Processor actual clocks when the Logical Processor is active.", "MetricExpr": "CPU_CLK_UNHALTED.THREAD", @@ -166,16 +750,10 @@ }, { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "TmaL1", + "MetricExpr": "4 * CORE_CLKS", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, - { - "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", - "MetricExpr": "4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "TmaL1_SMT", - "MetricName": "SLOTS_SMT" - }, { "BriefDescription": "The ratio of Executed- by Issued-Uops", "MetricExpr": "UOPS_EXECUTED.THREAD / UOPS_ISSUED.ANY", @@ -185,63 +763,38 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, - { - "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;SMT;TmaL1_SMT", - "MetricName": "CoreIPC_SMT" - }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.THREAD", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, - { - "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Ret;Flops_SMT", - "MetricName": "FLOPc_SMT" - }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, - { - "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Cor;Flops;HPC_SMT", - "MetricName": "FP_Arith_Utilization_SMT", - "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common). SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, { "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - ( UOPS_ISSUED.ANY + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD)))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * CPU_CLK_UNHALTED.THREAD)) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if 0 > 0.5 else 0", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", "MetricGroup": "Cor;SMT", "MetricName": "Core_Bound_Likely" }, - { - "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", - "MetricExpr": "( 1 - ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) / ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) if ((1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ((( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * (1 - (IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - ( UOPS_ISSUED.ANY + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))))) < ((EXE_ACTIVITY.EXE_BOUND_0_PORTS + (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL)) / CPU_CLK_UNHALTED.THREAD if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else (EXE_ACTIVITY.1_PORTS_UTIL + (UOPS_RETIRED.RETIRE_SLOTS / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * EXE_ACTIVITY.2_PORTS_UTIL) / CPU_CLK_UNHALTED.THREAD) else 1 ) if (1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 )) > 0.5 else 0", - "MetricGroup": "Cor;SMT_SMT", - "MetricName": "Core_Bound_Likely_SMT" - }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", - "MetricExpr": "( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "((CPU_CLK_UNHALTED.THREAD / 2) * (1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK)) if #core_wide < 1 else (CPU_CLK_UNHALTED.THREAD_ANY / 2) if #SMT_on else CLKS", "MetricGroup": "SMT", "MetricName": "CORE_CLKS" }, @@ -283,13 +836,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -310,21 +863,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -336,9 +889,9 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, { @@ -373,16 +926,10 @@ }, { "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / CPU_CLK_UNHALTED.THREAD / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD))) )", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_mite))", "MetricGroup": "DSBmiss;Fed", "MetricName": "DSB_Misses" }, - { - "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", - "MetricExpr": "100 * ( (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * (DSB2MITE_SWITCHES.PENALTY_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) + ((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) * (( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) / 2) / #((IDQ_UOPS_NOT_DELIVERED.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) - (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) )", - "MetricGroup": "DSBmiss;Fed_SMT", - "MetricName": "DSB_Misses_SMT" - }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -397,16 +944,10 @@ }, { "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * INT_MISC.RECOVERY_CYCLES ) / (4 * CPU_CLK_UNHALTED.THREAD))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * CPU_CLK_UNHALTED.THREAD)) ) * (4 * CPU_CLK_UNHALTED.THREAD) / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;BrMispredicts", "MetricName": "Branch_Misprediction_Cost" }, - { - "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", - "MetricExpr": " ( ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * (( UOPS_ISSUED.ANY - UOPS_RETIRED.RETIRE_SLOTS + 4 * ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) ) / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )))) + (4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) * ((BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT )) * INT_MISC.CLEAR_RESTEER_CYCLES / CPU_CLK_UNHALTED.THREAD) / #(4 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ))) ) * (4 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )) / BR_MISP_RETIRED.ALL_BRANCHES", - "MetricGroup": "Bad;BrMispredicts_SMT", - "MetricName": "Branch_Misprediction_Cost_SMT" - }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.NOT_TAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -415,101 +956,95 @@ }, { "BriefDescription": "Fraction of branches that are taken conditionals", - "MetricExpr": "( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches;CodeGen;PGO", "MetricName": "Cond_TK" }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, { "BriefDescription": "Fraction of branches that are unconditional (direct or indirect) jumps", - "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - ( BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN ) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_TAKEN - (BR_INST_RETIRED.CONDITIONAL - BR_INST_RETIRED.NOT_TAKEN) - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "Jump" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.THREAD )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, - { - "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING + EPT.WALK_PENDING ) / ( 2 * ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) ) )", - "MetricGroup": "Mem;MemoryTLB_SMT", - "MetricName": "Page_Walks_Utilization_SMT" - }, { "BriefDescription": "Average per-core data fill bandwidth to the L1 data cache [GB / sec]", "MetricExpr": "64 * L1D.REPLACEMENT / 1000000000 / duration_time", @@ -536,37 +1071,37 @@ }, { "BriefDescription": "Rate of silent evictions from the L2 cache per Kilo instruction where the evicted lines are dropped (no writeback to L3 or memory)", - "MetricExpr": "1000 * L2_LINES_OUT.SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_Silent_PKI" }, { "BriefDescription": "Rate of non silent evictions from the L2 cache per Kilo instruction", - "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / INST_RETIRED.ANY", + "MetricExpr": "1000 * L2_LINES_OUT.NON_SILENT / Instructions", "MetricGroup": "L2Evicts;Mem;Server", "MetricName": "L2_Evictions_NonSilent_PKI" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -578,68 +1113,47 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License0_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License0_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License1_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License1_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / CORE_CLKS if #SMT_on else CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License2_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions." }, - { - "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). SMT version; use when SMT is enabled and measuring per logical CPU.", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / 2 / ( ( CPU_CLK_UNHALTED.THREAD / 2 ) * ( 1 + CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / CPU_CLK_UNHALTED.REF_XCLK ) )", - "MetricGroup": "Power_SMT", - "MetricName": "Power_License2_Utilization_SMT", - "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions. SMT version; use when SMT is enabled and measuring per logical CPU." - }, { "BriefDescription": "Fraction of cycles where both hardware Logical Processors were active", - "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / ( CPU_CLK_UNHALTED.REF_XCLK_ANY / 2 ) if #SMT_on else 0", + "MetricExpr": "1 - CPU_CLK_UNHALTED.ONE_THREAD_ACTIVE / (CPU_CLK_UNHALTED.REF_XCLK_ANY / 2) if #SMT_on else 0", "MetricGroup": "SMT", "MetricName": "SMT_2T_Utilization" }, @@ -657,13 +1171,13 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "( 64 * ( uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@ ) / 1000000000 ) / duration_time", + "MetricExpr": "(64 * (uncore_imc@cas_count_read@ + uncore_imc@cas_count_write@) / 1000000000) / duration_time", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, { "BriefDescription": "Average latency of data read request to external memory (in nanoseconds). Accounts for demand loads and L1/L2 prefetches", - "MetricExpr": "1000000000 * ( cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@ ) / ( cha_0@event\\=0x0@ / duration_time )", + "MetricExpr": "1000000000 * (cha@event\\=0x36\\,umask\\=0x21\\,config\\=0x40433@ / cha@event\\=0x35\\,umask\\=0x21\\,config\\=0x40433@) / (Socket_CLKS / duration_time)", "MetricGroup": "Mem;MemoryLat;SoC", "MetricName": "MEM_Read_Latency" }, @@ -675,20 +1189,20 @@ }, { "BriefDescription": "Average latency of data read request to external DRAM memory [in nanoseconds]. Accounts for demand loads and L1/L2 data-read prefetches", - "MetricExpr": "1000000000 * ( UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS ) / imc_0@event\\=0x0@", - "MetricGroup": "Mem;MemoryLat;SoC;Server", + "MetricExpr": "1000000000 * (UNC_M_RPQ_OCCUPANCY / UNC_M_RPQ_INSERTS) / imc_0@event\\=0x0@", + "MetricGroup": "Mem;MemoryLat;Server;SoC", "MetricName": "MEM_DRAM_Read_Latency" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Writes [GB / sec]", - "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3) * 4 / 1000000000 / duration_time", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Write_BW" }, { "BriefDescription": "Average IO (network or disk) Bandwidth Use for Reads [GB / sec]", - "MetricExpr": "( UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3 ) * 4 / 1000000000 / duration_time", - "MetricGroup": "IoBW;Mem;SoC;Server", + "MetricExpr": "(UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_WRITE.PART3) * 4 / 1000000000 / duration_time", + "MetricGroup": "IoBW;Mem;Server;SoC", "MetricName": "IO_Read_BW" }, { @@ -697,12 +1211,6 @@ "MetricGroup": "SoC", "MetricName": "Socket_CLKS" }, - { - "BriefDescription": "Uncore frequency per die [GHZ]", - "MetricExpr": "cha_0@event\\=0x0@ / #num_dies / duration_time / 1000000000", - "MetricGroup": "SoC", - "MetricName": "UNCORE_FREQ" - }, { "BriefDescription": "Instructions per Far Branch ( Far Branches apply upon transition from application to operating system, handling interrupts, exceptions) [lower number means higher occurrence rate]", "MetricExpr": "INST_RETIRED.ANY / BR_INST_RETIRED.FAR_BRANCH:u", @@ -752,11 +1260,10 @@ "MetricName": "C7_Pkg_Residency" }, { - "BriefDescription": "Percentage of time spent in the active CPU power state C0", - "MetricExpr": "100 * CPU_CLK_UNHALTED.REF_TSC / TSC", - "MetricGroup": "", - "MetricName": "cpu_utilization_percent", - "ScaleUnit": "1%" + "BriefDescription": "Uncore frequency per die [GHZ]", + "MetricExpr": "Socket_CLKS / #num_dies / duration_time / 1000000000", + "MetricGroup": "SoC", + "MetricName": "UNCORE_FREQ" }, { "BriefDescription": "CPU operating frequency (in GHz)", @@ -765,13 +1272,6 @@ "MetricName": "cpu_operating_frequency", "ScaleUnit": "1GHz" }, - { - "BriefDescription": "Cycles per instruction retired; indicating how much time each executed instruction took; in units of cycles.", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / INST_RETIRED.ANY", - "MetricGroup": "", - "MetricName": "cpi", - "ScaleUnit": "1per_instr" - }, { "BriefDescription": "The ratio of number of completed memory load instructions to the total number completed instructions", "MetricExpr": "MEM_INST_RETIRED.ALL_LOADS / INST_RETIRED.ANY", @@ -790,7 +1290,7 @@ "BriefDescription": "Ratio of number of requests missing L1 data cache (includes data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L1D.REPLACEMENT / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l1d_mpi_includes_data_plus_rfo_with_prefetches", + "MetricName": "l1d_mpi", "ScaleUnit": "1per_instr" }, { @@ -818,7 +1318,7 @@ "BriefDescription": "Ratio of number of requests missing L2 cache (includes code+data+rfo w/ prefetches) to the total number of completed instructions", "MetricExpr": "L2_LINES_IN.ALL / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "l2_mpi_includes_code_plus_data_plus_rfo_with_prefetches", + "MetricName": "l2_mpi", "ScaleUnit": "1per_instr" }, { @@ -849,58 +1349,79 @@ "MetricName": "llc_code_read_mpi_demand_plus_prefetch", "ScaleUnit": "1per_instr" }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) in nano seconds", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043300000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043300000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency", + "ScaleUnit": "1ns" + }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to local memory in nano seconds", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043200000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_local_requests", + "ScaleUnit": "1ns" + }, + { + "BriefDescription": "Average latency of a last level cache (LLC) demand and prefetch data read miss (read memory access) addressed to remote memory in nano seconds", + "MetricExpr": "( 1000000000 * ( cha@unc_cha_tor_occupancy.ia_miss\\,config1\\=0x4043100000000@ / cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ ) / ( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) ) ) * duration_time", + "MetricGroup": "", + "MetricName": "llc_data_read_demand_plus_prefetch_miss_latency_for_remote_requests", + "ScaleUnit": "1ns" + }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the ITLB (Instruction TLB) and further levels of TLB.", "MetricExpr": "ITLB_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "itlb_2nd_level_mpi", + "MetricName": "itlb_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for 2 megabyte and 4 megabyte page sizes) caused by a code fetch to the total number of completed instructions. This implies it missed in the Instruction Translation Lookaside Buffer (ITLB) and further levels of TLB.", "MetricExpr": "ITLB_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "itlb_2nd_level_large_page_mpi", + "MetricName": "itlb_large_page_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_load_mpi", + "MetricName": "dtlb_load_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for 2 megabyte page sizes) caused by demand data loads to the total number of completed instructions. This implies it missed in the Data Translation Lookaside Buffer (DTLB) and further levels of TLB.", "MetricExpr": "DTLB_LOAD_MISSES.WALK_COMPLETED_2M_4M / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_2mb_large_page_load_mpi", + "MetricName": "dtlb_2mb_large_page_load_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Ratio of number of completed page walks (for all page sizes) caused by demand data stores to the total number of completed instructions. This implies it missed in the DTLB and further levels of TLB.", "MetricExpr": "DTLB_STORE_MISSES.WALK_COMPLETED / INST_RETIRED.ANY", "MetricGroup": "", - "MetricName": "dtlb_2nd_level_store_mpi", + "MetricName": "dtlb_store_mpi", "ScaleUnit": "1per_instr" }, { "BriefDescription": "Memory read that miss the last level cache (LLC) addressed to local DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_local_dram", + "MetricName": "numa_reads_addressed_to_local_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Memory reads that miss the last level cache (LLC) addressed to remote DRAM as a percentage of total memory read accesses, does not include LLC prefetches.", "MetricExpr": "100 * cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ / ( cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043200000000@ + cha@unc_cha_tor_inserts.ia_miss\\,config1\\=0x4043100000000@ )", "MetricGroup": "", - "MetricName": "numa_percent_reads_addressed_to_remote_dram", + "MetricName": "numa_reads_addressed_to_remote_dram", "ScaleUnit": "1%" }, { "BriefDescription": "Uncore operating frequency in GHz", - "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( source_count(UNC_CHA_CLOCKTICKS) * #num_packages ) / 1000000000) / duration_time", + "MetricExpr": "( UNC_CHA_CLOCKTICKS / ( #num_cores / #num_packages * #num_packages ) / 1000000000) / duration_time", "MetricGroup": "", "MetricName": "uncore_frequency", "ScaleUnit": "1GHz" @@ -909,7 +1430,7 @@ "BriefDescription": "Intel(R) Ultra Path Interconnect (UPI) data transmit bandwidth (MB/sec)", "MetricExpr": "( UNC_UPI_TxL_FLITS.ALL_DATA * (64 / 9.0) / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "upi_data_transmit_bw_only_data", + "MetricName": "upi_data_transmit_bw", "ScaleUnit": "1MB/s" }, { @@ -937,35 +1458,35 @@ "BriefDescription": "Bandwidth of IO reads that are initiated by end device controllers that are requesting memory from the CPU.", "MetricExpr": "(( UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART0 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART1 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART2 + UNC_IIO_DATA_REQ_OF_CPU.MEM_READ.PART3 ) * 4 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_read", + "MetricName": "io_bandwidth_disk_or_network_writes", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Bandwidth of IO writes that are initiated by end device controllers that are writing memory to the CPU.", "MetricExpr": "(( UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART0 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART1 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART2 + UNC_IIO_PAYLOAD_BYTES_IN.MEM_WRITE.PART3 ) * 4 / 1000000) / duration_time", "MetricGroup": "", - "MetricName": "io_bandwidth_write", + "MetricName": "io_bandwidth_disk_or_network_reads", "ScaleUnit": "1MB/s" }, { "BriefDescription": "Uops delivered from decoded instruction cache (decoded stream buffer or DSB) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.DSB_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_decoded_icache_dsb", + "MetricName": "percent_uops_delivered_from_decoded_icache", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from legacy decode pipeline (Micro-instruction Translation Engine or MITE) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MITE_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline_mite", + "MetricName": "percent_uops_delivered_from_legacy_decode_pipeline", "ScaleUnit": "1%" }, { "BriefDescription": "Uops delivered from microcode sequencer (MS) as a percent of total uops delivered to Instruction Decode Queue", "MetricExpr": "100 * ( IDQ.MS_UOPS / UOPS_ISSUED.ANY )", "MetricGroup": "", - "MetricName": "percent_uops_delivered_from_microcode_sequencer_ms", + "MetricName": "percent_uops_delivered_from_microcode_sequencer", "ScaleUnit": "1%" }, { @@ -988,250 +1509,5 @@ "MetricGroup": "", "MetricName": "llc_miss_remote_memory_bandwidth_read", "ScaleUnit": "1MB/s" - }, - { - "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound.", - "MetricExpr": "100 * ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1;PGO", - "MetricName": "tma_frontend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period.", - "MetricExpr": "100 * ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_latency_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses.", - "MetricExpr": "100 * ( ( ICACHE_16B.IFDATA_STALL + 2 * cpu@ICACHE_16B.IFDATA_STALL\\,cmask\\=0x1\\,edge\\=0x1@ ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;IcMiss;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_icache_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses.", - "MetricExpr": "100 * ( ICACHE_64B.IFTAG_STALL / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_itlb_misses_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings.", - "MetricExpr": "100 * ( INT_MISC.CLEAR_RESTEER_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) + ( ( 9 ) * BACLEARS.ANY / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_branch_resteers_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty.", - "MetricExpr": "100 * ( DSB2MITE_SWITCHES.PENALTY_CYCLES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "DSBmiss;FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_dsb_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", - "MetricExpr": "100 * ( ILD_STALL.LCP / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_lcp_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals.", - "MetricExpr": "100 * ( ( 2 ) * IDQ.MS_SWITCHES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "FetchLat;MicroSeq;TmaL3;m_tma_fetch_latency_percent", - "MetricName": "tma_ms_switches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend.", - "MetricExpr": "100 * ( ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( 4 ) * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "FetchBW;Frontend;TmaL2;m_tma_frontend_bound_percent", - "MetricName": "tma_fetch_bandwidth_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck.", - "MetricExpr": "100 * ( ( IDQ.ALL_MITE_CYCLES_ANY_UOPS - IDQ.ALL_MITE_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSBmiss;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_mite_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", - "MetricExpr": "100 * ( ( IDQ.ALL_DSB_CYCLES_ANY_UOPS - IDQ.ALL_DSB_CYCLES_4_UOPS ) / ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) / 2 )", - "MetricGroup": "DSB;FetchBW;TmaL3;m_tma_fetch_bandwidth_percent", - "MetricName": "tma_dsb_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", - "MetricExpr": "100 * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_bad_speculation_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path.", - "MetricExpr": "100 * ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "BadSpec;BrMispredicts;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_branch_mispredicts_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes.", - "MetricExpr": "100 * ( ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( BR_MISP_RETIRED.ALL_BRANCHES / ( BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT ) ) * ( ( UOPS_ISSUED.ANY - ( UOPS_RETIRED.RETIRE_SLOTS ) + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "BadSpec;MachineClears;TmaL2;m_tma_bad_speculation_percent", - "MetricName": "tma_machine_clears_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound.", - "MetricExpr": "100 * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_backend_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", - "MetricExpr": "100 * ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;m_tma_backend_bound_percent", - "MetricName": "tma_memory_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache.", - "MetricExpr": "100 * ( max( ( CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) , 0 ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l1_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l2_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance.", - "MetricExpr": "100 * ( ( CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_l3_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance.", - "MetricExpr": "100 * ( min( ( ( CYCLE_ACTIVITY.STALLS_L3_MISS / ( CPU_CLK_UNHALTED.THREAD ) + ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) - ( ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) / ( ( MEM_LOAD_RETIRED.L2_HIT * ( 1 + ( MEM_LOAD_RETIRED.FB_HIT / ( MEM_LOAD_RETIRED.L1_MISS ) ) ) ) + cpu@L1D_PEND_MISS.FB_FULL\\,cmask\\=0x1@ ) ) * ( ( CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS ) / ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) , ( 1 ) ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_dram_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck.", - "MetricExpr": "100 * ( EXE_ACTIVITY.BOUND_ON_STORES / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "MemoryBound;TmaL3mem;TmaL3;m_tma_memory_bound_percent", - "MetricName": "tma_store_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", - "MetricExpr": "100 * ( ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES ) / ( CYCLE_ACTIVITY.STALLS_TOTAL + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) + EXE_ACTIVITY.BOUND_ON_STORES ) ) * ( 1 - ( IDQ_UOPS_NOT_DELIVERED.CORE / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( UOPS_ISSUED.ANY + ( 4 ) * ( ( INT_MISC.RECOVERY_CYCLES_ANY / 2 ) if #SMT_on else INT_MISC.RECOVERY_CYCLES ) ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) )", - "MetricGroup": "Backend;TmaL2;Compute;m_tma_backend_bound_percent", - "MetricName": "tma_core_bound_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication.", - "MetricExpr": "100 * ( ARITH.DIVIDER_ACTIVE / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_divider_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", - "MetricExpr": "100 * ( ( EXE_ACTIVITY.EXE_BOUND_0_PORTS + ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) ) / ( CPU_CLK_UNHALTED.THREAD ) if ( ARITH.DIVIDER_ACTIVE < ( CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY ) ) else ( EXE_ACTIVITY.1_PORTS_UTIL + ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * EXE_ACTIVITY.2_PORTS_UTIL ) / ( CPU_CLK_UNHALTED.THREAD ) )", - "MetricGroup": "PortsUtil;TmaL3;m_tma_core_bound_percent", - "MetricName": "tma_ports_utilization_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. ", - "MetricExpr": "100 * ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "TmaL1", - "MetricName": "tma_retiring_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_light_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) )", - "MetricGroup": "HPC;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fp_arith_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_memory_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring fused instructions -- where one uop can represent multiple contiguous instructions. The instruction pairs of CMP+JCC or DEC+JCC are commonly used examples.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_fused_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions that were not fused. Non-conditional branches like direct JMP or CALL would count here. Can be used to examine fusible conditional jumps that were not fused.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_non_fused_branches_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_nop_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", - "MetricExpr": "100 * ( max( 0 , ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) - ( ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD ) + ( ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( min( ( ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) , ( 1 ) ) ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * UOPS_RETIRED.MACRO_FUSED / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * ( BR_INST_RETIRED.ALL_BRANCHES - UOPS_RETIRED.MACRO_FUSED ) / ( UOPS_RETIRED.RETIRE_SLOTS ) ) + ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) ) * INST_RETIRED.NOP / ( UOPS_RETIRED.RETIRE_SLOTS ) ) ) ) )", - "MetricGroup": "Pipeline;TmaL3;m_tma_light_operations_percent", - "MetricName": "tma_other_light_ops_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "Retire;TmaL2;m_tma_retiring_percent", - "MetricName": "tma_heavy_operations_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", - "MetricExpr": "100 * ( ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) + UOPS_RETIRED.MACRO_FUSED - INST_RETIRED.ANY ) / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) - ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) ) )", - "MetricGroup": "TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_few_uops_instructions_percent", - "ScaleUnit": "1%" - }, - { - "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided.", - "MetricExpr": "100 * ( ( ( UOPS_RETIRED.RETIRE_SLOTS ) / UOPS_ISSUED.ANY ) * IDQ.MS_UOPS / ( ( 4 ) * ( ( CPU_CLK_UNHALTED.THREAD_ANY / 2 ) if #SMT_on else ( CPU_CLK_UNHALTED.THREAD ) ) ) )", - "MetricGroup": "MicroSeq;TmaL3;m_tma_heavy_operations_percent", - "MetricName": "tma_microcode_sequencer_percent", - "ScaleUnit": "1%" } ] diff --git a/tools/perf/pmu-events/arch/x86/skylakex/uncore-memory.json b/tools/perf/pmu-events/arch/x86/skylakex/uncore-memory.json index 0746fcf2ebd979c21bd42d13a44a1061288c0fae..62941146e39670ae0206f9124f49b40cccb35edd 100644 --- a/tools/perf/pmu-events/arch/x86/skylakex/uncore-memory.json +++ b/tools/perf/pmu-events/arch/x86/skylakex/uncore-memory.json @@ -27,20 +27,19 @@ "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", + "BriefDescription": "All DRAM Read CAS Commands issued (including underfills)", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_READ", + "EventName": "UNC_M_CAS_COUNT.RD", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0x3", "Unit": "iMC" }, { - "BriefDescription": "read requests to memory controller", + "BriefDescription": "read requests to memory controller. Derived from unc_m_cas_count.rd", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.RD", + "EventName": "LLC_MISSES.MEM_READ", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0x3", @@ -56,20 +55,19 @@ "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", + "BriefDescription": "All DRAM Write CAS commands issued", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "LLC_MISSES.MEM_WRITE", + "EventName": "UNC_M_CAS_COUNT.WR", "PerPkg": "1", - "ScaleUnit": "64Bytes", "UMask": "0xC", "Unit": "iMC" }, { - "BriefDescription": "write requests to memory controller", + "BriefDescription": "write requests to memory controller. Derived from unc_m_cas_count.wr", "Counter": "0,1,2,3", "EventCode": "0x4", - "EventName": "UNC_M_CAS_COUNT.WR", + "EventName": "LLC_MISSES.MEM_WRITE", "PerPkg": "1", "ScaleUnit": "64Bytes", "UMask": "0xC", diff --git a/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json b/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json index f55aeadc630f2fd5cd68c281ee3b3543db39dd19..0d106fe7aae3599f3af92fa079c7f352e82845a1 100644 --- a/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json +++ b/tools/perf/pmu-events/arch/x86/skylakex/uncore-other.json @@ -1089,7 +1089,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x01", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1101,7 +1100,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x02", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1113,7 +1111,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x04", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1125,7 +1122,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x08", - "ScaleUnit": "4Bytes", "UMask": "0x01", "Unit": "IIO" }, @@ -1196,7 +1192,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x01", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1208,7 +1203,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x02", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1220,7 +1214,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x04", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1232,7 +1225,6 @@ "FCMask": "0x07", "PerPkg": "1", "PortMask": "0x08", - "ScaleUnit": "4Bytes", "UMask": "0x04", "Unit": "IIO" }, @@ -1974,20 +1966,19 @@ "Unit": "UPI LL" }, { - "BriefDescription": "UPI interconnect send bandwidth for payload. Derived from unc_upi_txl_flits.all_data", + "BriefDescription": "Valid data FLITs transmitted via any slot", "Counter": "0,1,2,3", "EventCode": "0x2", - "EventName": "UPI_DATA_BANDWIDTH_TX", + "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA", "PerPkg": "1", - "ScaleUnit": "7.11E-06Bytes", - "UMask": "0xf", + "UMask": "0x0F", "Unit": "UPI LL" }, { - "BriefDescription": "UPI interconnect send bandwidth for payload", + "BriefDescription": "UPI interconnect send bandwidth for payload. Derived from unc_upi_txl_flits.all_data", "Counter": "0,1,2,3", "EventCode": "0x2", - "EventName": "UNC_UPI_TxL_FLITS.ALL_DATA", + "EventName": "UPI_DATA_BANDWIDTH_TX", "PerPkg": "1", "ScaleUnit": "7.11E-06Bytes", "UMask": "0xf", diff --git a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json index 03c97bd74ad94ee7a3462ef9528f1f902d9872b8..79b8b101b68fc85623ed60a560f04f05c0e45e84 100644 --- a/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json +++ b/tools/perf/pmu-events/arch/x86/tigerlake/tgl-metrics.json @@ -1,26 +1,716 @@ [ + { + "BriefDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend", + "MetricExpr": "topdown\\-fe\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) - INT_MISC.UOP_DROPPING / SLOTS", + "MetricGroup": "PGO;TopdownL1;tma_L1_group", + "MetricName": "tma_frontend_bound", + "PublicDescription": "This category represents fraction of slots where the processor's Frontend undersupplies its Backend. Frontend denotes the first part of the processor core responsible to fetch operations that are executed later on by the Backend part. Within the Frontend; a branch predictor predicts the next address to fetch; cache-lines are fetched from the memory subsystem; parsed into instructions; and lastly decoded into micro-operations (uops). Ideally the Frontend can issue Machine_Width uops every cycle to the Backend. Frontend Bound denotes unutilized issue-slots when there is no Backend stall; i.e. bubbles where Frontend delivered no uops while Backend could have accepted them. For example; stalls due to instruction-cache misses would be categorized under Frontend Bound. Sample with: FRONTEND_RETIRED.LATENCY_GE_4_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues", + "MetricExpr": "(5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING) / SLOTS", + "MetricGroup": "Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_latency", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend latency issues. For example; instruction-cache misses; iTLB misses or fetch stalls after a branch misprediction are categorized under Frontend Latency. In such cases; the Frontend eventually delivers no uops for some period. Sample with: FRONTEND_RETIRED.LATENCY_GE_16_PS;FRONTEND_RETIRED.LATENCY_GE_8_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses", + "MetricExpr": "ICACHE_16B.IFDATA_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;IcMiss;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_icache_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to instruction cache misses. Sample with: FRONTEND_RETIRED.L2_MISS_PS;FRONTEND_RETIRED.L1I_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses", + "MetricExpr": "ICACHE_64B.IFTAG_STALL / CLKS", + "MetricGroup": "BigFoot;FetchLat;MemoryTLB;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_itlb_misses", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Instruction TLB (ITLB) misses. Sample with: FRONTEND_RETIRED.STLB_MISS_PS;FRONTEND_RETIRED.ITLB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers", + "MetricExpr": "INT_MISC.CLEAR_RESTEER_CYCLES / CLKS + tma_unknown_branches", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_branch_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers. Branch Resteers estimates the Frontend delay in fetching operations from corrected path; following all sorts of miss-predicted branches. For example; branchy code with lots of miss-predictions might get categorized under Branch Resteers. Note the value of this node may overlap with its siblings. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_mispredicts_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Branch Misprediction at execution stage. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears", + "MetricExpr": "(1 - (BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT))) * INT_MISC.CLEAR_RESTEER_CYCLES / CLKS", + "MetricGroup": "BadSpec;MachineClears;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_clears_resteers", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to Branch Resteers as a result of Machine Clears. Sample with: INT_MISC.CLEAR_RESTEER_CYCLES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears", + "MetricExpr": "10 * BACLEARS.ANY / CLKS", + "MetricGroup": "BigFoot;FetchLat;TopdownL4;tma_branch_resteers_group", + "MetricName": "tma_unknown_branches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to new branch address clears. These are fetched branches the Branch Prediction Unit was unable to recognize (First fetch or hitting BPU capacity limit). Sample with: BACLEARS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines", + "MetricExpr": "DSB2MITE_SWITCHES.PENALTY_CYCLES / CLKS", + "MetricGroup": "DSBmiss;FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_dsb_switches", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to switches from DSB to MITE pipelines. The DSB (decoded i-cache) is a Uop Cache where the front-end directly delivers Uops (micro operations) avoiding heavy x86 decoding. The DSB pipeline has shorter latency and delivered higher bandwidth than the MITE (legacy instruction decode pipeline). Switching between the two pipelines can cause penalties hence this metric measures the exposed penalty. Sample with: FRONTEND_RETIRED.DSB_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs)", + "MetricExpr": "ILD_STALL.LCP / CLKS", + "MetricGroup": "FetchLat;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_lcp", + "PublicDescription": "This metric represents fraction of cycles CPU was stalled due to Length Changing Prefixes (LCPs). Using proper compiler flags or Intel Compiler by default will certainly avoid this. #Link: Optimization Guide about LCP BKMs.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS)", + "MetricExpr": "3 * IDQ.MS_SWITCHES / CLKS", + "MetricGroup": "FetchLat;MicroSeq;TopdownL3;tma_fetch_latency_group", + "MetricName": "tma_ms_switches", + "PublicDescription": "This metric estimates the fraction of cycles when the CPU was stalled due to switches of uop delivery to the Microcode Sequencer (MS). Commonly used instructions are optimized for delivery by the DSB (decoded i-cache) or MITE (legacy instruction decode) pipelines. Certain operations cannot be handled natively by the execution pipeline; and must be performed by microcode (small programs injected into the execution stream). Switching to the MS too often can negatively impact performance. The MS is designated to deliver long uop flows required by CISC instructions like CPUID; or uncommon conditions like Floating Point Assists when dealing with Denormals. Sample with: IDQ.MS_SWITCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues", + "MetricExpr": "max(0, tma_frontend_bound - tma_fetch_latency)", + "MetricGroup": "FetchBW;Frontend;TopdownL2;tma_L2_group;tma_frontend_bound_group", + "MetricName": "tma_fetch_bandwidth", + "PublicDescription": "This metric represents fraction of slots the CPU was stalled due to Frontend bandwidth issues. For example; inefficiencies at the instruction decoders; or restrictions for caching in the DSB (decoded uops cache) are categorized under Fetch Bandwidth. In such cases; the Frontend typically delivers suboptimal amount of uops to the Backend. Sample with: FRONTEND_RETIRED.LATENCY_GE_2_BUBBLES_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_1_PS;FRONTEND_RETIRED.LATENCY_GE_2_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline)", + "MetricExpr": "(IDQ.MITE_CYCLES_ANY - IDQ.MITE_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSBmiss;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_mite", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to the MITE pipeline (the legacy decode pipeline). This pipeline is used for code that was not pre-cached in the DSB or LSD. For example; inefficiencies due to asymmetric decoders; use of long immediate or LCP can manifest as MITE fetch bandwidth bottleneck. Sample with: FRONTEND_RETIRED.ANY_DSB_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where decoder-0 was the only active decoder", + "MetricExpr": "(cpu@INST_DECODED.DECODERS\\,cmask\\=1@ - cpu@INST_DECODED.DECODERS\\,cmask\\=2@) / CORE_CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_decoder0_alone", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where (only) 4 uops were delivered by the MITE pipeline", + "MetricExpr": "(cpu@IDQ.MITE_UOPS\\,cmask\\=4@ - cpu@IDQ.MITE_UOPS\\,cmask\\=5@) / CLKS", + "MetricGroup": "DSBmiss;FetchBW;TopdownL4;tma_mite_group", + "MetricName": "tma_mite_4wide", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline", + "MetricExpr": "(IDQ.DSB_CYCLES_ANY - IDQ.DSB_CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "DSB;FetchBW;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_dsb", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to DSB (decoded uop cache) fetch pipeline. For example; inefficient utilization of the DSB cache structure or bank conflict when reading from it; are categorized here.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit", + "MetricExpr": "(LSD.CYCLES_ACTIVE - LSD.CYCLES_OK) / CORE_CLKS / 2", + "MetricGroup": "FetchBW;LSD;TopdownL3;tma_fetch_bandwidth_group", + "MetricName": "tma_lsd", + "PublicDescription": "This metric represents Core fraction of cycles in which CPU was likely limited due to LSD (Loop Stream Detector) unit. LSD typically does well sustaining Uop supply. However; in some rare cases; optimal uop-delivery could not be reached for small loops whose size (in terms of number of uops) does not suit well the LSD structure.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots wasted due to incorrect speculations", + "MetricExpr": "max(1 - (tma_frontend_bound + tma_backend_bound + tma_retiring), 0)", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_bad_speculation", + "PublicDescription": "This category represents fraction of slots wasted due to incorrect speculations. This include slots used to issue uops that do not eventually get retired and slots for which the issue-pipeline was blocked due to recovery from earlier incorrect speculation. For example; wasted work due to miss-predicted branches are categorized under Bad Speculation category. Incorrect data speculation followed by Memory Ordering Nukes is another example.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction", + "MetricExpr": "(BR_MISP_RETIRED.ALL_BRANCHES / (BR_MISP_RETIRED.ALL_BRANCHES + MACHINE_CLEARS.COUNT)) * tma_bad_speculation", + "MetricGroup": "BadSpec;BrMispredicts;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_branch_mispredicts", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Branch Misprediction. These slots are either wasted by uops fetched from an incorrectly speculated program path; or stalls when the out-of-order part of the machine needs to recover its state from a speculative path. Sample with: BR_MISP_RETIRED.ALL_BRANCHES", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears", + "MetricExpr": "max(0, tma_bad_speculation - tma_branch_mispredicts)", + "MetricGroup": "BadSpec;MachineClears;TopdownL2;tma_L2_group;tma_bad_speculation_group", + "MetricName": "tma_machine_clears", + "PublicDescription": "This metric represents fraction of slots the CPU has wasted due to Machine Clears. These slots are either wasted by uops fetched prior to the clear; or stalls the out-of-order portion of the machine needs to recover its state after the clear. For example; this can happen due to memory ordering Nukes (e.g. Memory Disambiguation) or Self-Modifying-Code (SMC) nukes. Sample with: MACHINE_CLEARS.COUNT", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend", + "MetricExpr": "topdown\\-be\\-bound / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + (5 * cpu@INT_MISC.RECOVERY_CYCLES\\,cmask\\=1\\,edge@) / SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_backend_bound", + "PublicDescription": "This category represents fraction of slots where no uops are being delivered due to a lack of required resources for accepting new uops in the Backend. Backend is the portion of the processor core where the out-of-order scheduler dispatches ready uops into their respective execution units; and once completed these uops get retired according to program order. For example; stalls due to data-cache misses or stalls due to the divider unit being overloaded are both categorized under Backend Bound. Backend Bound is further divided into two main categories: Memory Bound and Core Bound. Sample with: TOPDOWN.BACKEND_BOUND_SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck", + "MetricExpr": "((CYCLE_ACTIVITY.STALLS_MEM_ANY + EXE_ACTIVITY.BOUND_ON_STORES) / (CYCLE_ACTIVITY.STALLS_TOTAL + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) + EXE_ACTIVITY.BOUND_ON_STORES)) * tma_backend_bound", + "MetricGroup": "Backend;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_memory_bound", + "PublicDescription": "This metric represents fraction of slots the Memory subsystem within the Backend was a bottleneck. Memory Bound estimates fraction of slots where pipeline is likely stalled due to demand load or store instructions. This accounts mainly for (1) non-completed in-flight memory demand loads which coincides with execution units starvation; in addition to (2) cases where stores could impose backpressure on the pipeline when many of them get buffered at the same time (less common out of the two).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache", + "MetricExpr": "max((CYCLE_ACTIVITY.STALLS_MEM_ANY - CYCLE_ACTIVITY.STALLS_L1D_MISS) / CLKS, 0)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l1_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled without loads missing the L1 data cache. The L1 data cache typically has the shortest latency. However; in certain cases like loads blocked on older stores; a load might suffer due to high latency even though it is being satisfied by the L1. Another example is loads who miss in the TLB. These cases are characterized by execution unit stalls; while some non-completed demand load lives in the machine without having that demand load missing the L1 cache. Sample with: MEM_LOAD_RETIRED.L1_HIT_PS;MEM_LOAD_RETIRED.FB_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses", + "MetricExpr": "min(7 * cpu@DTLB_LOAD_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_LOAD_MISSES.WALK_ACTIVE, max(CYCLE_ACTIVITY.CYCLES_MEM_ANY - CYCLE_ACTIVITY.CYCLES_L1D_MISS, 0)) / CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_dtlb_load", + "PublicDescription": "This metric roughly estimates the fraction of cycles where the Data TLB (DTLB) was missed by load accesses. TLBs (Translation Look-aside Buffers) are processor caches for recently used entries out of the Page Tables that are used to map virtual- to physical-addresses by the operating system. This metric approximates the potential delay of demand loads missing the first-level data TLB (assuming worst case scenario with back to back misses to different pages). This includes hitting in the second-level TLB (STLB) as well as performing a hardware page walk on an STLB miss. Sample with: MEM_INST_RETIRED.STLB_MISS_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the (first level) DTLB was missed by load accesses, that later on hit in second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_load - tma_load_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the Second-level TLB (STLB) was missed by load accesses, performing a hardware page walk", + "MetricExpr": "DTLB_LOAD_MISSES.WALK_ACTIVE / CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_load_group", + "MetricName": "tma_load_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores", + "MetricExpr": "13 * LD_BLOCKS.STORE_FORWARD / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_store_fwd_blk", + "PublicDescription": "This metric roughly estimates fraction of cycles when the memory subsystem had loads blocked since they could not forward data from earlier (in program order) overlapping stores. To streamline memory operations in the pipeline; a load can avoid waiting for memory if a prior in-flight store is writing the data that the load wants to read (store forwarding process). However; in some cases the load may be blocked for a significant time pending the store forward. For example; when the prior store is writing a smaller region than the load is reading.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations", + "MetricExpr": "(16 * max(0, MEM_INST_RETIRED.LOCK_LOADS - L2_RQSTS.ALL_RFO) + (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES) * (10 * L2_RQSTS.RFO_HIT + min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO))) / CLKS", + "MetricGroup": "Offcore;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_lock_latency", + "PublicDescription": "This metric represents fraction of cycles the CPU spent handling cache misses due to lock operations. Due to the microarchitecture handling of locks; they are classified as L1_Bound regardless of what memory source satisfied them. Sample with: MEM_INST_RETIRED.LOCK_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary", + "MetricExpr": "Load_Miss_Real_Latency * LD_BLOCKS.NO_SR / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_split_loads", + "PublicDescription": "This metric estimates fraction of cycles handling memory load split accesses - load that cross 64-byte cache line boundary. Sample with: MEM_INST_RETIRED.SPLIT_LOADS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset", + "MetricExpr": "LD_BLOCKS_PARTIAL.ADDRESS_ALIAS / CLKS", + "MetricGroup": "TopdownL4;tma_l1_bound_group", + "MetricName": "tma_4k_aliasing", + "PublicDescription": "This metric estimates how often memory load accesses were aliased by preceding stores (in program order) with a 4K address offset. False match is possible; which incur a few cycles load re-issue. However; the short re-issue duration is often hidden by the out-of-order core and HW optimizations; hence a user may safely ignore a high value of this metric unless it manages to propagate up into parent nodes of the hierarchy (e.g. to L1_Bound).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed", + "MetricExpr": "L1D_PEND_MISS.FB_FULL / CLKS", + "MetricGroup": "MemoryBW;TopdownL4;tma_l1_bound_group", + "MetricName": "tma_fb_full", + "PublicDescription": "This metric does a *rough estimation* of how often L1D Fill Buffer unavailability limited additional L1D miss memory access requests to proceed. The higher the metric value; the deeper the memory hierarchy level the misses are satisfied from (metric values >1 are valid). Often it hints on approaching bandwidth limits (to L2 cache; L3 cache or external memory).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads", + "MetricExpr": "((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) / ((MEM_LOAD_RETIRED.L2_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS))) + L1D_PEND_MISS.FB_FULL_PERIODS)) * ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS)", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l2_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to L2 cache accesses by loads. Avoiding cache misses (i.e. L1 misses/L2 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L2_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L2_MISS - CYCLE_ACTIVITY.STALLS_L3_MISS) / CLKS", + "MetricGroup": "CacheMisses;MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_l3_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled due to loads accesses to L3 cache or contended with a sibling Core. Avoiding cache misses (i.e. L2 misses/L3 hits) can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses", + "MetricExpr": "((49 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD))) + (48 * Average_Frequency) * MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_contested_accesses", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to contested accesses. Contested accesses occur when data written by one Logical Processor are read by another Logical Processor on a different Physical Core. Examples of contested accesses include synchronizations such as locks; true data sharing such as modified locked variables; and false sharing. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD;MEM_LOAD_L3_HIT_RETIRED.XSNP_MISS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses", + "MetricExpr": "(48 * Average_Frequency) * (MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD + MEM_LOAD_L3_HIT_RETIRED.XSNP_FWD * (1 - (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM / (OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HITM + OCR.DEMAND_DATA_RD.L3_HIT.SNOOP_HIT_WITH_FWD)))) * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "Offcore;Snoop;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_data_sharing", + "PublicDescription": "This metric estimates fraction of cycles while the memory subsystem was handling synchronizations due to data-sharing accesses. Data shared by multiple Logical Processors (even just read shared) may cause increased access latency due to cache coherency. Excessive data sharing can drastically harm multithreaded performance. Sample with: MEM_LOAD_L3_HIT_RETIRED.XSNP_NO_FWD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited)", + "MetricExpr": "(17.5 * Average_Frequency) * MEM_LOAD_RETIRED.L3_HIT * (1 + (MEM_LOAD_RETIRED.FB_HIT / MEM_LOAD_RETIRED.L1_MISS) / 2) / CLKS", + "MetricGroup": "MemoryLat;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_l3_hit_latency", + "PublicDescription": "This metric represents fraction of cycles with demand load accesses that hit the L3 cache under unloaded scenarios (possibly L3 latency limited). Avoiding private cache misses (i.e. L2 misses/L3 hits) will improve the latency; reduce contention with sibling physical cores and increase performance. Note the value of this node may overlap with its siblings. Sample with: MEM_LOAD_RETIRED.L3_HIT_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors)", + "MetricExpr": "L1D_PEND_MISS.L2_STALL / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_l3_bound_group", + "MetricName": "tma_sq_full", + "PublicDescription": "This metric measures fraction of cycles where the Super Queue (SQ) was full taking into account all request-types and both hardware SMT threads (Logical Processors). The Super Queue is used for requests to access the L2 cache or to go out to the Uncore.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads", + "MetricExpr": "(CYCLE_ACTIVITY.STALLS_L3_MISS / CLKS + ((CYCLE_ACTIVITY.STALLS_L1D_MISS - CYCLE_ACTIVITY.STALLS_L2_MISS) / CLKS) - tma_l2_bound)", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_dram_bound", + "PublicDescription": "This metric estimates how often the CPU was stalled on accesses to external memory (DRAM) by loads. Better caching can improve the latency and increase performance. Sample with: MEM_LOAD_RETIRED.L3_MISS_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, cpu@OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD\\,cmask\\=4@) / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_bandwidth", + "PublicDescription": "This metric estimates fraction of cycles where the core's performance was likely hurt due to approaching bandwidth limits of external memory (DRAM). The underlying heuristic assumes that a similar off-core traffic is generated by all IA cores. This metric does not aggregate non-data-read requests by this logical processor; requests from other IA Logical Processors/Physical Cores/sockets; or other non-IA devices like GPU; hence the maximum external memory bandwidth limits may or may not be approached when this metric is flagged (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM)", + "MetricExpr": "min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DATA_RD) / CLKS - tma_mem_bandwidth", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_dram_bound_group", + "MetricName": "tma_mem_latency", + "PublicDescription": "This metric estimates fraction of cycles where the performance was likely hurt due to latency from external memory (DRAM). This metric does not aggregate requests from other Logical Processors/Physical Cores/sockets (see Uncore counters for that).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write", + "MetricExpr": "EXE_ACTIVITY.BOUND_ON_STORES / CLKS", + "MetricGroup": "MemoryBound;TmaL3mem;TopdownL3;tma_memory_bound_group", + "MetricName": "tma_store_bound", + "PublicDescription": "This metric estimates how often CPU was stalled due to RFO store memory accesses; RFO store issue a read-for-ownership request before the write. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should RFO stores be a bottleneck. Sample with: MEM_INST_RETIRED.ALL_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses", + "MetricExpr": "((L2_RQSTS.RFO_HIT * 10 * (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES))) + (1 - (MEM_INST_RETIRED.LOCK_LOADS / MEM_INST_RETIRED.ALL_STORES)) * min(CPU_CLK_UNHALTED.THREAD, OFFCORE_REQUESTS_OUTSTANDING.CYCLES_WITH_DEMAND_RFO)) / CLKS", + "MetricGroup": "MemoryLat;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_store_latency", + "PublicDescription": "This metric estimates fraction of cycles the CPU spent handling L1D store misses. Store accesses usually less impact out-of-order core performance; however; holding resources for longer time can lead into undesired implications (e.g. contention on L1D fill-buffer entries - see FB_Full)", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing", + "MetricExpr": "(54 * Average_Frequency) * OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM / CLKS", + "MetricGroup": "DataSharing;Offcore;Snoop;TopdownL4;tma_store_bound_group", + "MetricName": "tma_false_sharing", + "PublicDescription": "This metric roughly estimates how often CPU was handling synchronizations due to False Sharing. False Sharing is a multithreading hiccup; where multiple Logical Processors contend on different data-elements mapped into the same cache line. Sample with: OCR.DEMAND_RFO.L3_HIT.SNOOP_HITM", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents rate of split store accesses", + "MetricExpr": "MEM_INST_RETIRED.SPLIT_STORES / CORE_CLKS", + "MetricGroup": "TopdownL4;tma_store_bound_group", + "MetricName": "tma_split_stores", + "PublicDescription": "This metric represents rate of split store accesses. Consider aligning your data to the 64-byte cache line granularity. Sample with: MEM_INST_RETIRED.SPLIT_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores", + "MetricExpr": "9 * OCR.STREAMING_WR.ANY_RESPONSE / CLKS", + "MetricGroup": "MemoryBW;Offcore;TopdownL4;tma_store_bound_group", + "MetricName": "tma_streaming_stores", + "PublicDescription": "This metric estimates how often CPU was stalled due to Streaming store memory accesses; Streaming store optimize out a read request required by RFO stores. Even though store accesses do not typically stall out-of-order CPUs; there are few cases where stores can lead to actual stalls. This metric will be flagged should Streaming stores be a bottleneck. Sample with: OCR.STREAMING_WR.ANY_RESPONSE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses", + "MetricExpr": "(7 * cpu@DTLB_STORE_MISSES.STLB_HIT\\,cmask\\=1@ + DTLB_STORE_MISSES.WALK_ACTIVE) / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL4;tma_store_bound_group", + "MetricName": "tma_dtlb_store", + "PublicDescription": "This metric roughly estimates the fraction of cycles spent handling first-level data TLB store misses. As with ordinary data caching; focus on improving data locality and reducing working-set size to reduce DTLB overhead. Additionally; consider using profile-guided optimization (PGO) to collocate frequently-used data on the same page. Try using larger page sizes for large amounts of frequently-used data. Sample with: MEM_INST_RETIRED.STLB_MISS_STORES_PS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric roughly estimates the fraction of cycles where the TLB was missed by store accesses, hitting in the second-level TLB (STLB)", + "MetricExpr": "tma_dtlb_store - tma_store_stlb_miss", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_hit", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates the fraction of cycles where the STLB was missed by store accesses, performing a hardware page walk", + "MetricExpr": "DTLB_STORE_MISSES.WALK_ACTIVE / CORE_CLKS", + "MetricGroup": "MemoryTLB;TopdownL5;tma_dtlb_store_group", + "MetricName": "tma_store_stlb_miss", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck", + "MetricExpr": "max(0, tma_backend_bound - tma_memory_bound)", + "MetricGroup": "Backend;Compute;TopdownL2;tma_L2_group;tma_backend_bound_group", + "MetricName": "tma_core_bound", + "PublicDescription": "This metric represents fraction of slots where Core non-memory issues were of a bottleneck. Shortage in hardware compute resources; or dependencies in software's instructions are both categorized under Core Bound. Hence it may indicate the machine ran out of an out-of-order resource; certain execution units are overloaded or dependencies in program's data- or instruction-flow are limiting the performance (e.g. FP-chained long-latency arithmetic operations).", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the Divider unit was active", + "MetricExpr": "ARITH.DIVIDER_ACTIVE / CLKS", + "MetricGroup": "TopdownL3;tma_core_bound_group", + "MetricName": "tma_divider", + "PublicDescription": "This metric represents fraction of cycles where the Divider unit was active. Divide and square root instructions are performed by the Divider unit and can take considerably longer latency than integer or Floating Point addition; subtraction; or multiplication. Sample with: ARITH.DIVIDER_ACTIVE", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related)", + "MetricExpr": "(cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ + (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL)) / CLKS if (ARITH.DIVIDER_ACTIVE < (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY)) else (EXE_ACTIVITY.1_PORTS_UTIL + tma_retiring * EXE_ACTIVITY.2_PORTS_UTIL) / CLKS", + "MetricGroup": "PortsUtil;TopdownL3;tma_core_bound_group", + "MetricName": "tma_ports_utilization", + "PublicDescription": "This metric estimates fraction of cycles the CPU performance was potentially limited due to Core computation issues (non divider-related). Two distinct categories can be attributed into this metric: (1) heavy data-dependency among contiguous instructions would manifest in this metric - such cases are often referred to as low Instruction Level Parallelism (ILP). (2) Contention on some hardware execution unit other than Divider. For example; when there are too many multiply operations.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "cpu@EXE_ACTIVITY.3_PORTS_UTIL\\,umask\\=0x80@ / CLKS + tma_serializing_operation * (CYCLE_ACTIVITY.STALLS_TOTAL - CYCLE_ACTIVITY.STALLS_MEM_ANY) / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_0", + "PublicDescription": "This metric represents fraction of cycles CPU executed no uops on any execution port (Logical Processor cycles since ICL, Physical Core cycles otherwise). Long-latency instructions like divides may contribute to this metric.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations", + "MetricExpr": "RESOURCE_STALLS.SCOREBOARD / CLKS", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_serializing_operation", + "PublicDescription": "This metric represents fraction of cycles the CPU issue-pipeline was stalled due to serializing operations. Instructions like CPUID; WRMSR or LFENCE serialize the out-of-order execution which may limit performance. Sample with: RESOURCE_STALLS.SCOREBOARD", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions", + "MetricExpr": "140 * MISC_RETIRED.PAUSE_INST / CLKS", + "MetricGroup": "TopdownL6;tma_serializing_operation_group", + "MetricName": "tma_slow_pause", + "PublicDescription": "This metric represents fraction of cycles the CPU was stalled due to PAUSE Instructions. Sample with: MISC_RETIRED.PAUSE_INST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued", + "MetricExpr": "CLKS * UOPS_ISSUED.VECTOR_WIDTH_MISMATCH / UOPS_ISSUED.ANY", + "MetricGroup": "TopdownL5;tma_ports_utilized_0_group", + "MetricName": "tma_mixing_vectors", + "PublicDescription": "The Mixing_Vectors metric gives the percentage of injected blend uops out of all uops issued. Usually a Mixing_Vectors over 5% is worth investigating. Read more in Appendix B1 of the Optimizations Guide for this topic.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.1_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_1", + "PublicDescription": "This metric represents fraction of cycles where the CPU executed total of 1 uop per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). This can be due to heavy data-dependency among software instructions; or over oversubscribing a particular hardware resource. In some other cases with high 1_Port_Utilized and L1_Bound; this metric can point to L1 data-cache latency bottleneck that may not necessarily manifest with complete execution starvation (due to the short L1 latency e.g. walking a linked list) - looking at the assembly can be helpful. Sample with: EXE_ACTIVITY.1_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "EXE_ACTIVITY.2_PORTS_UTIL / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_2", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 2 uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Loop Vectorization -most compilers feature auto-Vectorization options today- reduces pressure on the execution ports as multiple elements are calculated with same uop. Sample with: EXE_ACTIVITY.2_PORTS_UTIL", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise)", + "MetricExpr": "UOPS_EXECUTED.CYCLES_GE_3 / CLKS", + "MetricGroup": "PortsUtil;TopdownL4;tma_ports_utilization_group", + "MetricName": "tma_ports_utilized_3m", + "PublicDescription": "This metric represents fraction of cycles CPU executed total of 3 or more uops per cycle on all execution ports (Logical Processor cycles since ICL, Physical Core cycles otherwise). Sample with: UOPS_EXECUTED.CYCLES_GE_3", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution ports for ALU operations.", + "MetricExpr": "(UOPS_DISPATCHED.PORT_0 + UOPS_DISPATCHED.PORT_1 + UOPS_DISPATCHED.PORT_5 + UOPS_DISPATCHED.PORT_6) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_alu_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 0 ([SNB+] ALU; [HSW+] ALU and 2nd branch) Sample with: UOPS_DISPATCHED.PORT_0", + "MetricExpr": "UOPS_DISPATCHED.PORT_0 / CORE_CLKS", + "MetricGroup": "Compute;TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_0", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 1 (ALU) Sample with: UOPS_DISPATCHED.PORT_1", + "MetricExpr": "UOPS_DISPATCHED.PORT_1 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_1", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 5 ([SNB+] Branches and ALU; [HSW+] ALU) Sample with: UOPS_DISPATCHED.PORT_5", + "MetricExpr": "UOPS_DISPATCHED.PORT_5 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_5", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port 6 ([HSW+]Primary Branch and simple ALU) Sample with: UOPS_DISPATCHED.PORT_6", + "MetricExpr": "UOPS_DISPATCHED.PORT_6 / CORE_CLKS", + "MetricGroup": "TopdownL6;tma_alu_op_utilization_group", + "MetricName": "tma_port_6", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Load operations Sample with: UOPS_DISPATCHED.PORT_2_3", + "MetricExpr": "UOPS_DISPATCHED.PORT_2_3 / (2 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_load_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents Core fraction of cycles CPU dispatched uops on execution port for Store operations Sample with: UOPS_DISPATCHED.PORT_7_8", + "MetricExpr": "(UOPS_DISPATCHED.PORT_4_9 + UOPS_DISPATCHED.PORT_7_8) / (4 * CORE_CLKS)", + "MetricGroup": "TopdownL5;tma_ports_utilized_3m_group", + "MetricName": "tma_store_op_utilization", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired", + "MetricExpr": "topdown\\-retiring / (topdown\\-fe\\-bound + topdown\\-bad\\-spec + topdown\\-retiring + topdown\\-be\\-bound) + 0*SLOTS", + "MetricGroup": "TopdownL1;tma_L1_group", + "MetricName": "tma_retiring", + "PublicDescription": "This category represents fraction of slots utilized by useful work i.e. issued uops that eventually get retired. Ideally; all pipeline slots would be attributed to the Retiring category. Retiring of 100% would indicate the maximum Pipeline_Width throughput was achieved. Maximizing Retiring typically increases the Instructions-per-cycle (see IPC metric). Note that a high Retiring value does not necessary mean there is no room for more performance. For example; Heavy-operations or Microcode Assists are categorized under Retiring. They often indicate suboptimal performance and can often be optimized or avoided. Sample with: UOPS_RETIRED.SLOTS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation)", + "MetricExpr": "max(0, tma_retiring - tma_heavy_operations)", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_light_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring light-weight operations -- instructions that require no more than one uop (micro-operation). This correlates with total number of instructions used by the program. A uops-per-instruction (see UPI metric) ratio of 1 or less should be expected for decently optimized software running on Intel Core/Xeon products. While this often indicates efficient X86 instructions were executed; high value does not necessarily mean better performance cannot be achieved. Sample with: INST_RETIRED.PREC_DIST", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired)", + "MetricExpr": "tma_x87_use + tma_fp_scalar + tma_fp_vector", + "MetricGroup": "HPC;TopdownL3;tma_light_operations_group", + "MetricName": "tma_fp_arith", + "PublicDescription": "This metric represents overall arithmetic floating-point (FP) operations fraction the CPU has executed (retired). Note this metric's value may exceed its parent due to use of \"Uops\" CountDomain and FMA double-counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric serves as an approximation of legacy x87 usage", + "MetricExpr": "tma_retiring * UOPS_EXECUTED.X87 / UOPS_EXECUTED.THREAD", + "MetricGroup": "Compute;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_x87_use", + "PublicDescription": "This metric serves as an approximation of legacy x87 usage. It accounts for instructions beyond X87 FP arithmetic operations; hence may be used as a thermometer to avoid X87 high usage and preferably upgrade to modern ISA. See Tip under Tuning Hint.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired", + "MetricExpr": "(FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_scalar", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) scalar uops fraction the CPU has retired. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL4;tma_fp_arith_group", + "MetricName": "tma_fp_vector", + "PublicDescription": "This metric approximates arithmetic floating-point (FP) vector uops fraction the CPU has retired aggregated across all vector widths. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_128b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 128-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_256b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 256-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors", + "MetricExpr": "(FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / (tma_retiring * SLOTS)", + "MetricGroup": "Compute;Flops;TopdownL5;tma_fp_vector_group", + "MetricName": "tma_fp_vector_512b", + "PublicDescription": "This metric approximates arithmetic FP vector uops fraction the CPU has retired for 512-bit wide vectors. May overcount due to FMA double counting.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring memory operations -- uops for memory load or store accesses.", + "MetricExpr": "tma_light_operations * MEM_INST_RETIRED.ANY / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_memory_operations", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring branch instructions.", + "MetricExpr": "tma_light_operations * BR_INST_RETIRED.ALL_BRANCHES / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_branch_instructions", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions", + "MetricExpr": "tma_light_operations * INST_RETIRED.NOP / (tma_retiring * SLOTS)", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_nop_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring NOP (no op) instructions. Compilers often use NOPs for certain address alignments - e.g. start address of a function or loop body. Sample with: INST_RETIRED.NOP", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents the remaining light uops fraction the CPU has executed - remaining means not covered by other sibling nodes. May undercount due to FMA double counting", + "MetricExpr": "max(0, tma_light_operations - (tma_fp_arith + tma_memory_operations + tma_branch_instructions + tma_nop_instructions))", + "MetricGroup": "Pipeline;TopdownL3;tma_light_operations_group", + "MetricName": "tma_other_light_ops", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences", + "MetricExpr": "tma_microcode_sequencer + tma_retiring * (UOPS_DECODED.DEC0 - cpu@UOPS_DECODED.DEC0\\,cmask\\=1@) / IDQ.MITE_UOPS", + "MetricGroup": "Retire;TopdownL2;tma_L2_group;tma_retiring_group", + "MetricName": "tma_heavy_operations", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring heavy-weight operations -- instructions that require two or more uops or microcoded sequences. This highly-correlates with the uop length of these instructions/sequences.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops", + "MetricExpr": "tma_heavy_operations - tma_microcode_sequencer", + "MetricGroup": "TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_few_uops_instructions", + "PublicDescription": "This metric represents fraction of slots where the CPU was retiring instructions that that are decoder into two or up to ([SNB+] four; [ADL+] five) uops. This highly-correlates with the number of uops in such instructions.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit", + "MetricExpr": "((tma_retiring * SLOTS) / UOPS_ISSUED.ANY) * IDQ.MS_UOPS / SLOTS", + "MetricGroup": "MicroSeq;TopdownL3;tma_heavy_operations_group", + "MetricName": "tma_microcode_sequencer", + "PublicDescription": "This metric represents fraction of slots the CPU was retiring uops fetched by the Microcode Sequencer (MS) unit. The MS is used for CISC instructions not supported by the default decoders (like repeat move strings; or CPUID); or by microcode assists used to address some operation modes (like in Floating Point assists). These cases can often be avoided. Sample with: IDQ.MS_UOPS", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists", + "MetricExpr": "100 * ASSISTS.ANY / SLOTS", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_assists", + "PublicDescription": "This metric estimates fraction of slots the CPU retired uops delivered by the Microcode_Sequencer as a result of Assists. Assists are long sequences of uops that are required in certain corner-cases for operations that cannot be handled natively by the execution pipeline. For example; when working with very small floating point values (so-called Denormals); the FP units are not set up to perform these operations natively. Instead; a sequence of instructions to perform the computation on the Denormals is injected into the pipeline. Since these microcode sequences might be dozens of uops long; Assists can be extremely deleterious to performance and they can be avoided in many cases. Sample with: ASSISTS.ANY", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction", + "MetricExpr": "max(0, tma_microcode_sequencer - tma_assists)", + "MetricGroup": "TopdownL4;tma_microcode_sequencer_group", + "MetricName": "tma_cisc", + "PublicDescription": "This metric estimates fraction of cycles the CPU retired uops originated from CISC (complex instruction set computer) instruction. A CISC instruction has multiple uops that are required to perform the instruction's functionality as in the case of read-modify-write as an example. Since these instructions require multiple uops they may or may not imply sub-optimal use of machine resources.", + "ScaleUnit": "100%" + }, + { + "BriefDescription": "Total pipeline cost of Branch Misprediction related bottlenecks", + "MetricExpr": "100 * (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches))", + "MetricGroup": "Bad;BadSpec;BrMispredicts", + "MetricName": "Mispredictions" + }, + { + "BriefDescription": "Total pipeline cost of (external) Memory Bandwidth related bottlenecks", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_bandwidth / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_sq_full / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full))) + (tma_l1_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_fb_full / (tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) ", + "MetricGroup": "Mem;MemoryBW;Offcore", + "MetricName": "Memory_Bandwidth" + }, + { + "BriefDescription": "Total pipeline cost of Memory Latency related bottlenecks (external memory and off-core caches)", + "MetricExpr": "100 * tma_memory_bound * ((tma_dram_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_mem_latency / (tma_mem_bandwidth + tma_mem_latency)) + (tma_l3_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_l3_hit_latency / (tma_contested_accesses + tma_data_sharing + tma_l3_hit_latency + tma_sq_full)) + (tma_l2_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)))", + "MetricGroup": "Mem;MemoryLat;Offcore", + "MetricName": "Memory_Latency" + }, + { + "BriefDescription": "Total pipeline cost of Memory Address Translation related bottlenecks (data-side TLBs)", + "MetricExpr": "100 * tma_memory_bound * ((tma_l1_bound / max(tma_memory_bound, tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_load / max(tma_l1_bound, tma_4k_aliasing + tma_dtlb_load + tma_fb_full + tma_lock_latency + tma_split_loads + tma_store_fwd_blk)) + (tma_store_bound / (tma_dram_bound + tma_l1_bound + tma_l2_bound + tma_l3_bound + tma_store_bound)) * (tma_dtlb_store / (tma_dtlb_store + tma_false_sharing + tma_split_stores + tma_store_latency + tma_streaming_stores))) ", + "MetricGroup": "Mem;MemoryTLB;Offcore", + "MetricName": "Memory_Data_TLBs" + }, { "BriefDescription": "Total pipeline cost of branch related instructions (used for program control-flow including function calls)", - "MetricExpr": "100 * (( BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * ((BR_INST_RETIRED.COND + 3 * BR_INST_RETIRED.NEAR_CALL + (BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL)) / SLOTS)", "MetricGroup": "Ret", "MetricName": "Branching_Overhead" }, { "BriefDescription": "Total pipeline cost of instruction fetch related bottlenecks by large code footprint programs (i-side cache; TLB and BTB misses)", - "MetricExpr": "100 * (( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS) * ( (ICACHE_64B.IFTAG_STALL / CPU_CLK_UNHALTED.THREAD) + (ICACHE_16B.IFDATA_STALL / CPU_CLK_UNHALTED.THREAD) + (10 * BACLEARS.ANY / CPU_CLK_UNHALTED.THREAD) ) / #(( 5 * IDQ_UOPS_NOT_DELIVERED.CYCLES_0_UOPS_DELIV.CORE - INT_MISC.UOP_DROPPING ) / TOPDOWN.SLOTS)", + "MetricExpr": "100 * tma_fetch_latency * (tma_itlb_misses + tma_icache_misses + tma_unknown_branches) / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)", "MetricGroup": "BigFoot;Fed;Frontend;IcMiss;MemoryTLB", "MetricName": "Big_Code" }, + { + "BriefDescription": "Total pipeline cost of instruction fetch bandwidth related bottlenecks", + "MetricExpr": "100 * (tma_frontend_bound - tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) - Big_Code", + "MetricGroup": "Fed;FetchBW;Frontend", + "MetricName": "Instruction_Fetch_BW" + }, { "BriefDescription": "Instructions Per Cycle (per Logical Processor)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD", + "MetricExpr": "INST_RETIRED.ANY / CLKS", "MetricGroup": "Ret;Summary", "MetricName": "IPC" }, + { + "BriefDescription": "Uops Per Instruction", + "MetricExpr": "(tma_retiring * SLOTS) / INST_RETIRED.ANY", + "MetricGroup": "Pipeline;Ret;Retire", + "MetricName": "UPI" + }, + { + "BriefDescription": "Instruction per taken branch", + "MetricExpr": "(tma_retiring * SLOTS) / BR_INST_RETIRED.NEAR_TAKEN", + "MetricGroup": "Branches;Fed;FetchBW", + "MetricName": "UpTB" + }, { "BriefDescription": "Cycles Per Instruction (per Logical Processor)", - "MetricExpr": "1 / (INST_RETIRED.ANY / CPU_CLK_UNHALTED.THREAD)", - "MetricGroup": "Pipeline;Mem", + "MetricExpr": "1 / IPC", + "MetricGroup": "Mem;Pipeline", "MetricName": "CPI" }, { @@ -32,13 +722,13 @@ { "BriefDescription": "Total issue-pipeline slots (per-Physical Core till ICL; per-Logical Processor ICL onward)", "MetricExpr": "TOPDOWN.SLOTS", - "MetricGroup": "TmaL1", + "MetricGroup": "tma_L1_group", "MetricName": "SLOTS" }, { "BriefDescription": "Fraction of Physical Core issue-slots utilized by this Logical Processor", - "MetricExpr": "TOPDOWN.SLOTS / ( TOPDOWN.SLOTS / 2 ) if #SMT_on else 1", - "MetricGroup": "SMT;TmaL1", + "MetricExpr": "SLOTS / (TOPDOWN.SLOTS / 2) if #SMT_on else 1", + "MetricGroup": "SMT;tma_L1_group", "MetricName": "Slots_Utilization" }, { @@ -50,29 +740,35 @@ }, { "BriefDescription": "Instructions Per Cycle across hyper-threads (per physical core)", - "MetricExpr": "INST_RETIRED.ANY / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;SMT;TmaL1", + "MetricExpr": "INST_RETIRED.ANY / CORE_CLKS", + "MetricGroup": "Ret;SMT;tma_L1_group", "MetricName": "CoreIPC" }, { "BriefDescription": "Floating Point Operations Per Cycle", - "MetricExpr": "( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / CPU_CLK_UNHALTED.DISTRIBUTED", - "MetricGroup": "Ret;Flops", + "MetricExpr": "(1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / CORE_CLKS", + "MetricGroup": "Flops;Ret", "MetricName": "FLOPc" }, { "BriefDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width)", - "MetricExpr": "( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)) / (2 * CORE_CLKS)", "MetricGroup": "Cor;Flops;HPC", "MetricName": "FP_Arith_Utilization", "PublicDescription": "Actual per-core usage of the Floating Point non-X87 execution units (regardless of precision or vector-width). Values > 1 are possible due to ([BDW+] Fused-Multiply Add (FMA) counting - common; [ADL+] use all of ADD/MUL/FMA in Scalar or 128/256-bit vectors - less common)." }, { "BriefDescription": "Instruction-Level-Parallelism (average number of uops executed when there is execution) per-core", - "MetricExpr": "UOPS_EXECUTED.THREAD / (( UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2 ) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", + "MetricExpr": "UOPS_EXECUTED.THREAD / ((UOPS_EXECUTED.CORE_CYCLES_GE_1 / 2) if #SMT_on else UOPS_EXECUTED.CORE_CYCLES_GE_1)", "MetricGroup": "Backend;Cor;Pipeline;PortsUtil", "MetricName": "ILP" }, + { + "BriefDescription": "Probability of Core Bound bottleneck hidden by SMT-profiling artifacts", + "MetricExpr": "(1 - tma_core_bound / tma_ports_utilization if tma_core_bound < tma_ports_utilization else 1) if SMT_2T_Utilization > 0.5 else 0", + "MetricGroup": "Cor;SMT", + "MetricName": "Core_Bound_Likely" + }, { "BriefDescription": "Core actual clocks when any Logical Processor is active on the Physical Core", "MetricExpr": "CPU_CLK_UNHALTED.DISTRIBUTED", @@ -117,13 +813,13 @@ }, { "BriefDescription": "Instructions per Floating Point (FP) Operation (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;InsType", "MetricName": "IpFLOP" }, { "BriefDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) )", + "MetricExpr": "INST_RETIRED.ANY / ((FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE))", "MetricGroup": "Flops;InsType", "MetricName": "IpArith", "PublicDescription": "Instructions per FP Arithmetic instruction (lower number means higher occurrence rate). May undercount due to FMA double counting. Approximated prior to BDW." @@ -144,21 +840,21 @@ }, { "BriefDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX128", "PublicDescription": "Instructions per FP Arithmetic AVX/SSE 128-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX256", "PublicDescription": "Instructions per FP Arithmetic AVX* 256-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." }, { "BriefDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate)", - "MetricExpr": "INST_RETIRED.ANY / ( FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE )", + "MetricExpr": "INST_RETIRED.ANY / (FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE + FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE)", "MetricGroup": "Flops;FpVector;InsType", "MetricName": "IpArith_AVX512", "PublicDescription": "Instructions per FP Arithmetic AVX 512-bit instruction (lower number means higher occurrence rate). May undercount due to FMA double counting." @@ -170,11 +866,17 @@ "MetricName": "IpSWPF" }, { - "BriefDescription": "Total number of retired Instructions, Sample with: INST_RETIRED.PREC_DIST", + "BriefDescription": "Total number of retired Instructions Sample with: INST_RETIRED.PREC_DIST", "MetricExpr": "INST_RETIRED.ANY", - "MetricGroup": "Summary;TmaL1", + "MetricGroup": "Summary;tma_L1_group", "MetricName": "Instructions" }, + { + "BriefDescription": "Average number of Uops retired in cycles where at least one uop has retired.", + "MetricExpr": "(tma_retiring * SLOTS) / cpu@UOPS_RETIRED.SLOTS\\,cmask\\=1@", + "MetricGroup": "Pipeline;Ret", + "MetricName": "Retire" + }, { "BriefDescription": "", "MetricExpr": "UOPS_EXECUTED.THREAD / cpu@UOPS_EXECUTED.THREAD\\,cmask\\=1@", @@ -205,6 +907,12 @@ "MetricGroup": "DSBmiss", "MetricName": "DSB_Switch_Cost" }, + { + "BriefDescription": "Total penalty related to DSB (uop cache) misses - subset of the Instruction_Fetch_BW Bottleneck.", + "MetricExpr": "100 * (tma_fetch_latency * tma_dsb_switches / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches) + tma_fetch_bandwidth * tma_mite / (tma_dsb + tma_lsd + tma_mite))", + "MetricGroup": "DSBmiss;Fed", + "MetricName": "DSB_Misses" + }, { "BriefDescription": "Number of Instructions per non-speculative DSB miss (lower number means higher occurrence rate)", "MetricExpr": "INST_RETIRED.ANY / FRONTEND_RETIRED.ANY_DSB_MISS", @@ -217,6 +925,12 @@ "MetricGroup": "Bad;BadSpec;BrMispredicts", "MetricName": "IpMispredict" }, + { + "BriefDescription": "Branch Misprediction Cost: Fraction of TMA slots wasted per non-speculative branch misprediction (retired JEClear)", + "MetricExpr": " (tma_branch_mispredicts + tma_fetch_latency * tma_mispredicts_resteers / (tma_branch_resteers + tma_dsb_switches + tma_icache_misses + tma_itlb_misses + tma_lcp + tma_ms_switches)) * SLOTS / BR_MISP_RETIRED.ALL_BRANCHES", + "MetricGroup": "Bad;BrMispredicts", + "MetricName": "Branch_Misprediction_Cost" + }, { "BriefDescription": "Fraction of branches that are non-taken conditionals", "MetricExpr": "BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES", @@ -231,7 +945,7 @@ }, { "BriefDescription": "Fraction of branches that are CALL or RET", - "MetricExpr": "( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES", + "MetricExpr": "(BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN) / BR_INST_RETIRED.ALL_BRANCHES", "MetricGroup": "Bad;Branches", "MetricName": "CallRet" }, @@ -243,80 +957,80 @@ }, { "BriefDescription": "Fraction of branches of other types (not individually covered by other metrics in Info.Branches group)", - "MetricExpr": "1 - ( (BR_INST_RETIRED.COND_NTAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (BR_INST_RETIRED.COND_TAKEN / BR_INST_RETIRED.ALL_BRANCHES) + (( BR_INST_RETIRED.NEAR_CALL + BR_INST_RETIRED.NEAR_RETURN ) / BR_INST_RETIRED.ALL_BRANCHES) + ((BR_INST_RETIRED.NEAR_TAKEN - BR_INST_RETIRED.COND_TAKEN - 2 * BR_INST_RETIRED.NEAR_CALL) / BR_INST_RETIRED.ALL_BRANCHES) )", + "MetricExpr": "1 - (Cond_NT + Cond_TK + CallRet + Jump)", "MetricGroup": "Bad;Branches", "MetricName": "Other_Branches" }, { "BriefDescription": "Actual Average Latency for L1 data-cache miss demand load operations (in core cycles)", - "MetricExpr": "L1D_PEND_MISS.PENDING / ( MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT )", + "MetricExpr": "L1D_PEND_MISS.PENDING / (MEM_LOAD_RETIRED.L1_MISS + MEM_LOAD_RETIRED.FB_HIT)", "MetricGroup": "Mem;MemoryBound;MemoryLat", "MetricName": "Load_Miss_Real_Latency" }, { "BriefDescription": "Memory-Level-Parallelism (average number of L1 miss demand load when there is at least one such miss. Per-Logical Processor)", "MetricExpr": "L1D_PEND_MISS.PENDING / L1D_PEND_MISS.PENDING_CYCLES", - "MetricGroup": "Mem;MemoryBound;MemoryBW", + "MetricGroup": "Mem;MemoryBW;MemoryBound", "MetricName": "MLP" }, { "BriefDescription": "L1 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L1_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI" }, { "BriefDescription": "L1 cache true misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.ALL_DEMAND_DATA_RD / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L1MPKI_Load" }, { "BriefDescription": "L2 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L2_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;Backend;CacheMisses", + "MetricGroup": "Backend;CacheMisses;Mem", "MetricName": "L2MPKI" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all request types (including speculative)", "MetricExpr": "1000 * L2_RQSTS.MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses;Offcore", + "MetricGroup": "CacheMisses;Mem;Offcore", "MetricName": "L2MPKI_All" }, { "BriefDescription": "L2 cache ([RKL+] true) misses per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2MPKI_Load" }, { "BriefDescription": "L2 cache hits per kilo instruction for all request types (including speculative)", - "MetricExpr": "1000 * ( L2_RQSTS.REFERENCES - L2_RQSTS.MISS ) / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricExpr": "1000 * (L2_RQSTS.REFERENCES - L2_RQSTS.MISS) / INST_RETIRED.ANY", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_All" }, { "BriefDescription": "L2 cache hits per kilo instruction for all demand loads (including speculative)", "MetricExpr": "1000 * L2_RQSTS.DEMAND_DATA_RD_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L2HPKI_Load" }, { "BriefDescription": "L3 cache true misses per kilo instruction for retired demand loads", "MetricExpr": "1000 * MEM_LOAD_RETIRED.L3_MISS / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "L3MPKI" }, { "BriefDescription": "Fill Buffer (FB) hits per kilo instructions for retired demand loads (L1D misses that merge into ongoing miss-handling entries)", "MetricExpr": "1000 * MEM_LOAD_RETIRED.FB_HIT / INST_RETIRED.ANY", - "MetricGroup": "Mem;CacheMisses", + "MetricGroup": "CacheMisses;Mem", "MetricName": "FB_HPKI" }, { "BriefDescription": "Utilization of the core's Page Walker(s) serving STLB misses triggered by instruction/Load/Store accesses", "MetricConstraint": "NO_NMI_WATCHDOG", - "MetricExpr": "( ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING ) / ( 2 * CPU_CLK_UNHALTED.DISTRIBUTED )", + "MetricExpr": "(ITLB_MISSES.WALK_PENDING + DTLB_LOAD_MISSES.WALK_PENDING + DTLB_STORE_MISSES.WALK_PENDING) / (2 * CORE_CLKS)", "MetricGroup": "Mem;MemoryTLB", "MetricName": "Page_Walks_Utilization" }, @@ -346,25 +1060,25 @@ }, { "BriefDescription": "Average per-thread data fill bandwidth to the L1 data cache [GB / sec]", - "MetricExpr": "(64 * L1D.REPLACEMENT / 1000000000 / duration_time)", + "MetricExpr": "L1D_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L1D_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L2 cache [GB / sec]", - "MetricExpr": "(64 * L2_LINES_IN.ALL / 1000000000 / duration_time)", + "MetricExpr": "L2_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L2_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data fill bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * LONGEST_LAT_CACHE.MISS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Fill_BW", "MetricGroup": "Mem;MemoryBW", "MetricName": "L3_Cache_Fill_BW_1T" }, { "BriefDescription": "Average per-thread data access bandwidth to the L3 cache [GB / sec]", - "MetricExpr": "(64 * OFFCORE_REQUESTS.ALL_REQUESTS / 1000000000 / duration_time)", + "MetricExpr": "L3_Cache_Access_BW", "MetricGroup": "Mem;MemoryBW;Offcore", "MetricName": "L3_Cache_Access_BW_1T" }, @@ -376,40 +1090,40 @@ }, { "BriefDescription": "Measured Average Frequency for unhalted processors [GHz]", - "MetricExpr": "(CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC) * msr@tsc@ / 1000000000 / duration_time", - "MetricGroup": "Summary;Power", + "MetricExpr": "Turbo_Utilization * msr@tsc@ / 1000000000 / duration_time", + "MetricGroup": "Power;Summary", "MetricName": "Average_Frequency" }, { "BriefDescription": "Giga Floating Point Operations Per Second", - "MetricExpr": "( ( 1 * ( FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE ) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * ( FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE ) + 8 * ( FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE ) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE ) / 1000000000 ) / duration_time", + "MetricExpr": "((1 * (FP_ARITH_INST_RETIRED.SCALAR_SINGLE + FP_ARITH_INST_RETIRED.SCALAR_DOUBLE) + 2 * FP_ARITH_INST_RETIRED.128B_PACKED_DOUBLE + 4 * (FP_ARITH_INST_RETIRED.128B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.256B_PACKED_DOUBLE) + 8 * (FP_ARITH_INST_RETIRED.256B_PACKED_SINGLE + FP_ARITH_INST_RETIRED.512B_PACKED_DOUBLE) + 16 * FP_ARITH_INST_RETIRED.512B_PACKED_SINGLE) / 1000000000) / duration_time", "MetricGroup": "Cor;Flops;HPC", "MetricName": "GFLOPs", "PublicDescription": "Giga Floating Point Operations Per Second. Aggregate across all supported options of: FP precisions, scalar and vector instructions, vector-width and AMX engine." }, { "BriefDescription": "Average Frequency Utilization relative nominal frequency", - "MetricExpr": "CPU_CLK_UNHALTED.THREAD / CPU_CLK_UNHALTED.REF_TSC", + "MetricExpr": "CLKS / CPU_CLK_UNHALTED.REF_TSC", "MetricGroup": "Power", "MetricName": "Turbo_Utilization" }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0", - "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL0_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License0_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for baseline license level 0. This includes non-AVX codes, SSE, AVX 128-bit, and low-current AVX 256-bit codes." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1", - "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL1_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License1_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 1. This includes high current AVX 256-bit instructions as well as low current AVX 512-bit instructions." }, { "BriefDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX)", - "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CPU_CLK_UNHALTED.DISTRIBUTED", + "MetricExpr": "CORE_POWER.LVL2_TURBO_LICENSE / CORE_CLKS", "MetricGroup": "Power", "MetricName": "Power_License2_Utilization", "PublicDescription": "Fraction of Core cycles where the core was running with power-delivery for license level 2 (introduced in SKX). This includes high current AVX 512-bit instructions." @@ -434,7 +1148,7 @@ }, { "BriefDescription": "Average external Memory Bandwidth Use for reads and writes [GB / sec]", - "MetricExpr": "64 * ( arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@ ) / 1000000 / duration_time / 1000", + "MetricExpr": "64 * (arb@event\\=0x81\\,umask\\=0x1@ + arb@event\\=0x84\\,umask\\=0x1@) / 1000000 / duration_time / 1000", "MetricGroup": "HPC;Mem;MemoryBW;SoC", "MetricName": "DRAM_BW_Use" }, diff --git a/tools/perf/pmu-events/empty-pmu-events.c b/tools/perf/pmu-events/empty-pmu-events.c index 5ed8c0aa48175d040081bc2411f9faf3d2f095de..480e8f0d30c836425ed547dcdad63aa2ec6411c6 100644 --- a/tools/perf/pmu-events/empty-pmu-events.c +++ b/tools/perf/pmu-events/empty-pmu-events.c @@ -142,15 +142,15 @@ static const struct pmu_event pme_test_soc_cpu[] = { .metric_name = "DCache_L2_All_Miss", }, { - .metric_expr = "dcache_l2_all_hits + dcache_l2_all_miss", + .metric_expr = "DCache_L2_All_Hits + DCache_L2_All_Miss", .metric_name = "DCache_L2_All", }, { - .metric_expr = "d_ratio(dcache_l2_all_hits, dcache_l2_all)", + .metric_expr = "d_ratio(DCache_L2_All_Hits, DCache_L2_All)", .metric_name = "DCache_L2_Hits", }, { - .metric_expr = "d_ratio(dcache_l2_all_miss, dcache_l2_all)", + .metric_expr = "d_ratio(DCache_L2_All_Miss, DCache_L2_All)", .metric_name = "DCache_L2_Misses", }, { diff --git a/tools/perf/tests/attr/base-record b/tools/perf/tests/attr/base-record index 8c10955eff939603063f5caf02c946e010d19365..3ef07a12aa14246dcc33abc626ab6feaca7b471e 100644 --- a/tools/perf/tests/attr/base-record +++ b/tools/perf/tests/attr/base-record @@ -9,7 +9,7 @@ size=128 config=0 sample_period=* sample_type=263 -read_format=0|4 +read_format=0|4|20 disabled=1 inherit=1 pinned=0 diff --git a/tools/perf/tests/attr/system-wide-dummy b/tools/perf/tests/attr/system-wide-dummy index 86a15dd359d93179519603c12cbba37757f64765..8fec06eda5f906d8497a74b782c2832204d0286b 100644 --- a/tools/perf/tests/attr/system-wide-dummy +++ b/tools/perf/tests/attr/system-wide-dummy @@ -11,7 +11,7 @@ size=128 config=9 sample_period=4000 sample_type=455 -read_format=4 +read_format=4|20 # Event will be enabled right away. disabled=0 inherit=1 diff --git a/tools/perf/tests/attr/test-record-group b/tools/perf/tests/attr/test-record-group index 14ee60fd3f410f53b131f7bf5f65dd461ffdddc9..6c1cff8aae8b8b219b4f6ced4ec6ef601c04da15 100644 --- a/tools/perf/tests/attr/test-record-group +++ b/tools/perf/tests/attr/test-record-group @@ -7,14 +7,14 @@ ret = 1 fd=1 group_fd=-1 sample_type=327 -read_format=4 +read_format=4|20 [event-2:base-record] fd=2 group_fd=1 config=1 sample_type=327 -read_format=4 +read_format=4|20 mmap=0 comm=0 task=0 diff --git a/tools/perf/tests/attr/test-record-group-sampling b/tools/perf/tests/attr/test-record-group-sampling index 300b9f7e6d6938f9cc04879df757497d09ff26d2..97e7e64a38f07f682d05b1d8a6257e92bdc07867 100644 --- a/tools/perf/tests/attr/test-record-group-sampling +++ b/tools/perf/tests/attr/test-record-group-sampling @@ -7,7 +7,7 @@ ret = 1 fd=1 group_fd=-1 sample_type=343 -read_format=12 +read_format=12|28 inherit=0 [event-2:base-record] @@ -21,8 +21,8 @@ config=3 # default | PERF_SAMPLE_READ sample_type=343 -# PERF_FORMAT_ID | PERF_FORMAT_GROUP -read_format=12 +# PERF_FORMAT_ID | PERF_FORMAT_GROUP | PERF_FORMAT_LOST +read_format=12|28 task=0 mmap=0 comm=0 diff --git a/tools/perf/tests/attr/test-record-group1 b/tools/perf/tests/attr/test-record-group1 index 3ffe246e02283970178bc65e87c3e2656299fa0c..eeb1db392bc9ceac695e4eea30d57d739d09a52e 100644 --- a/tools/perf/tests/attr/test-record-group1 +++ b/tools/perf/tests/attr/test-record-group1 @@ -7,7 +7,7 @@ ret = 1 fd=1 group_fd=-1 sample_type=327 -read_format=4 +read_format=4|20 [event-2:base-record] fd=2 @@ -15,7 +15,7 @@ group_fd=1 type=0 config=1 sample_type=327 -read_format=4 +read_format=4|20 mmap=0 comm=0 task=0 diff --git a/tools/perf/tests/attr/test-record-group2 b/tools/perf/tests/attr/test-record-group2 index 6b9f8d182ce10ef741b4c332b32da071af24f983..cebdaa8e64e474715035629e5b6d32cd54518ec3 100644 --- a/tools/perf/tests/attr/test-record-group2 +++ b/tools/perf/tests/attr/test-record-group2 @@ -9,7 +9,7 @@ group_fd=-1 config=0|1 sample_period=1234000 sample_type=87 -read_format=12 +read_format=12|28 inherit=0 freq=0 @@ -19,7 +19,7 @@ group_fd=1 config=0|1 sample_period=6789000 sample_type=87 -read_format=12 +read_format=12|28 disabled=0 inherit=0 mmap=0 diff --git a/tools/perf/tests/cpumap.c b/tools/perf/tests/cpumap.c index 7ea150cdc137d3a121118021f15d87b19c6db537..7c873c6ae3eb97d6c8ce80b449c398c66979e3f8 100644 --- a/tools/perf/tests/cpumap.c +++ b/tools/perf/tests/cpumap.c @@ -19,7 +19,6 @@ static int process_event_mask(struct perf_tool *tool __maybe_unused, struct perf_record_cpu_map *map_event = &event->cpu_map; struct perf_record_cpu_map_data *data; struct perf_cpu_map *map; - int i; unsigned int long_size; data = &map_event->data; @@ -32,16 +31,17 @@ static int process_event_mask(struct perf_tool *tool __maybe_unused, TEST_ASSERT_VAL("wrong nr", data->mask32_data.nr == 1); - for (i = 0; i < 20; i++) { + TEST_ASSERT_VAL("wrong cpu", perf_record_cpu_map_data__test_bit(0, data)); + TEST_ASSERT_VAL("wrong cpu", !perf_record_cpu_map_data__test_bit(1, data)); + for (int i = 2; i <= 20; i++) TEST_ASSERT_VAL("wrong cpu", perf_record_cpu_map_data__test_bit(i, data)); - } map = cpu_map__new_data(data); TEST_ASSERT_VAL("wrong nr", perf_cpu_map__nr(map) == 20); - for (i = 0; i < 20; i++) { - TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, i).cpu == i); - } + TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, 0).cpu == 0); + for (int i = 2; i <= 20; i++) + TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, i - 1).cpu == i); perf_cpu_map__put(map); return 0; @@ -73,25 +73,59 @@ static int process_event_cpus(struct perf_tool *tool __maybe_unused, return 0; } +static int process_event_range_cpus(struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + struct perf_record_cpu_map *map_event = &event->cpu_map; + struct perf_record_cpu_map_data *data; + struct perf_cpu_map *map; + + data = &map_event->data; + + TEST_ASSERT_VAL("wrong type", data->type == PERF_CPU_MAP__RANGE_CPUS); + + TEST_ASSERT_VAL("wrong any_cpu", data->range_cpu_data.any_cpu == 0); + TEST_ASSERT_VAL("wrong start_cpu", data->range_cpu_data.start_cpu == 1); + TEST_ASSERT_VAL("wrong end_cpu", data->range_cpu_data.end_cpu == 256); + + map = cpu_map__new_data(data); + TEST_ASSERT_VAL("wrong nr", perf_cpu_map__nr(map) == 256); + TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__cpu(map, 0).cpu == 1); + TEST_ASSERT_VAL("wrong cpu", perf_cpu_map__max(map).cpu == 256); + TEST_ASSERT_VAL("wrong refcnt", refcount_read(&map->refcnt) == 1); + perf_cpu_map__put(map); + return 0; +} + static int test__cpu_map_synthesize(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { struct perf_cpu_map *cpus; - /* This one is better stores in mask. */ - cpus = perf_cpu_map__new("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19"); + /* This one is better stored in a mask. */ + cpus = perf_cpu_map__new("0,2-20"); TEST_ASSERT_VAL("failed to synthesize map", !perf_event__synthesize_cpu_map(NULL, cpus, process_event_mask, NULL)); perf_cpu_map__put(cpus); - /* This one is better stores in cpu values. */ + /* This one is better stored in cpu values. */ cpus = perf_cpu_map__new("1,256"); TEST_ASSERT_VAL("failed to synthesize map", !perf_event__synthesize_cpu_map(NULL, cpus, process_event_cpus, NULL)); + perf_cpu_map__put(cpus); + + /* This one is better stored as a range. */ + cpus = perf_cpu_map__new("1-256"); + + TEST_ASSERT_VAL("failed to synthesize map", + !perf_event__synthesize_cpu_map(NULL, cpus, process_event_range_cpus, NULL)); + perf_cpu_map__put(cpus); return 0; } diff --git a/tools/perf/tests/event_update.c b/tools/perf/tests/event_update.c index 78db4d704e76c1327817f80dc852abe2867ecf7e..d093a9b878d1353efbe2f546e92b6766f1b158f7 100644 --- a/tools/perf/tests/event_update.c +++ b/tools/perf/tests/event_update.c @@ -21,7 +21,7 @@ static int process_event_unit(struct perf_tool *tool __maybe_unused, TEST_ASSERT_VAL("wrong id", ev->id == 123); TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__UNIT); - TEST_ASSERT_VAL("wrong unit", !strcmp(ev->data, "KRAVA")); + TEST_ASSERT_VAL("wrong unit", !strcmp(ev->unit, "KRAVA")); return 0; } @@ -31,13 +31,10 @@ static int process_event_scale(struct perf_tool *tool __maybe_unused, struct machine *machine __maybe_unused) { struct perf_record_event_update *ev = (struct perf_record_event_update *)event; - struct perf_record_event_update_scale *ev_data; - - ev_data = (struct perf_record_event_update_scale *)ev->data; TEST_ASSERT_VAL("wrong id", ev->id == 123); TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__SCALE); - TEST_ASSERT_VAL("wrong scale", ev_data->scale == 0.123); + TEST_ASSERT_VAL("wrong scale", ev->scale.scale == 0.123); return 0; } @@ -56,7 +53,7 @@ static int process_event_name(struct perf_tool *tool, TEST_ASSERT_VAL("wrong id", ev->id == 123); TEST_ASSERT_VAL("wrong id", ev->type == PERF_EVENT_UPDATE__NAME); - TEST_ASSERT_VAL("wrong name", !strcmp(ev->data, tmp->name)); + TEST_ASSERT_VAL("wrong name", !strcmp(ev->name, tmp->name)); return 0; } @@ -66,12 +63,9 @@ static int process_event_cpus(struct perf_tool *tool __maybe_unused, struct machine *machine __maybe_unused) { struct perf_record_event_update *ev = (struct perf_record_event_update *)event; - struct perf_record_event_update_cpus *ev_data; struct perf_cpu_map *map; - ev_data = (struct perf_record_event_update_cpus *) ev->data; - - map = cpu_map__new_data(&ev_data->cpus); + map = cpu_map__new_data(&ev->cpus.cpus); TEST_ASSERT_VAL("wrong id", ev->id == 123); TEST_ASSERT_VAL("wrong type", ev->type == PERF_EVENT_UPDATE__CPUS); diff --git a/tools/perf/tests/expr.c b/tools/perf/tests/expr.c index 2efe9e3a63b8b47496e05c94945ae35d78810a21..6512f5e22045a53f5be21ca3f30fff349f57186c 100644 --- a/tools/perf/tests/expr.c +++ b/tools/perf/tests/expr.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include "util/cputopo.h" #include "util/debug.h" #include "util/expr.h" #include "util/header.h" @@ -94,6 +95,10 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u ret |= test(ctx, "min(1,2) + 1", 2); ret |= test(ctx, "max(1,2) + 1", 3); ret |= test(ctx, "1+1 if 3*4 else 0", 2); + ret |= test(ctx, "100 if 1 else 200 if 1 else 300", 100); + ret |= test(ctx, "100 if 0 else 200 if 1 else 300", 200); + ret |= test(ctx, "100 if 1 else 200 if 0 else 300", 100); + ret |= test(ctx, "100 if 0 else 200 if 0 else 300", 300); ret |= test(ctx, "1.1 + 2.1", 3.2); ret |= test(ctx, ".1 + 2.", 2.1); ret |= test(ctx, "d_ratio(1, 2)", 0.5); @@ -133,7 +138,7 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u (void **)&val_ptr)); expr__ctx_clear(ctx); - ctx->runtime = 3; + ctx->sctx.runtime = 3; TEST_ASSERT_VAL("find ids", expr__find_ids("EVENT1\\,param\\=?@ + EVENT2\\,param\\=?@", NULL, ctx) == 0); @@ -154,15 +159,33 @@ static int test__expr(struct test_suite *t __maybe_unused, int subtest __maybe_u (void **)&val_ptr)); /* Only EVENT1 or EVENT2 need be measured depending on the value of smt_on. */ - expr__ctx_clear(ctx); - TEST_ASSERT_VAL("find ids", - expr__find_ids("EVENT1 if #smt_on else EVENT2", - NULL, ctx) == 0); - TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1); - TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids, - smt_on() ? "EVENT1" : "EVENT2", - (void **)&val_ptr)); + { + struct cpu_topology *topology = cpu_topology__new(); + bool smton = smt_on(topology); + bool corewide = core_wide(/*system_wide=*/false, + /*user_requested_cpus=*/false, + topology); + + cpu_topology__delete(topology); + expr__ctx_clear(ctx); + TEST_ASSERT_VAL("find ids", + expr__find_ids("EVENT1 if #smt_on else EVENT2", + NULL, ctx) == 0); + TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1); + TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids, + smton ? "EVENT1" : "EVENT2", + (void **)&val_ptr)); + + expr__ctx_clear(ctx); + TEST_ASSERT_VAL("find ids", + expr__find_ids("EVENT1 if #core_wide else EVENT2", + NULL, ctx) == 0); + TEST_ASSERT_VAL("find ids", hashmap__size(ctx->ids) == 1); + TEST_ASSERT_VAL("find ids", hashmap__find(ctx->ids, + corewide ? "EVENT1" : "EVENT2", + (void **)&val_ptr)); + } /* The expression is a constant 1.0 without needing to evaluate EVENT1. */ expr__ctx_clear(ctx); TEST_ASSERT_VAL("find ids", diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c index dfb6173b2a827f304e8faf7d2e4e96bbceaad766..8322fc2295fa049a11c4cf4bff1f79e8f1d99519 100644 --- a/tools/perf/tests/mmap-basic.c +++ b/tools/perf/tests/mmap-basic.c @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include <errno.h> #include <inttypes.h> -/* For the CLR_() macros */ -#include <pthread.h> #include <stdlib.h> #include <perf/cpumap.h> @@ -114,8 +112,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest for (i = 0; i < nsyscalls; ++i) for (j = 0; j < expected_nr_events[i]; ++j) { - int foo = syscalls[i](); - ++foo; + syscalls[i](); } md = &evlist->mmap[0]; diff --git a/tools/perf/tests/openat-syscall-all-cpus.c b/tools/perf/tests/openat-syscall-all-cpus.c index 90828ae03ef51e94207b206a139ac20df1555a88..f3275be83a3382ee1437ea671274f47fad60eca5 100644 --- a/tools/perf/tests/openat-syscall-all-cpus.c +++ b/tools/perf/tests/openat-syscall-all-cpus.c @@ -2,7 +2,7 @@ #include <errno.h> #include <inttypes.h> /* For the CPU_* macros */ -#include <pthread.h> +#include <sched.h> #include <sys/types.h> #include <sys/stat.h> diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c index 6a001fcfed68e5172e9fcc817e3064fd761102b7..7aa946aa886de501b3599ba9ede0b2bf0ac4972e 100644 --- a/tools/perf/tests/perf-record.c +++ b/tools/perf/tests/perf-record.c @@ -2,8 +2,6 @@ #include <errno.h> #include <inttypes.h> #include <linux/string.h> -/* For the CLR_() macros */ -#include <pthread.h> #include <sched.h> #include <perf/mmap.h> @@ -332,7 +330,7 @@ out_delete_evlist: out: if (err == -EACCES) return TEST_SKIP; - if (err < 0) + if (err < 0 || errs != 0) return TEST_FAIL; return TEST_OK; } diff --git a/tools/perf/tests/shell/coresight/Makefile b/tools/perf/tests/shell/coresight/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b070e779703e9fbd70f88c826172b2150ee3d302 --- /dev/null +++ b/tools/perf/tests/shell/coresight/Makefile @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +include ../../../../../tools/scripts/Makefile.include +include ../../../../../tools/scripts/Makefile.arch +include ../../../../../tools/scripts/utilities.mak + +SUBDIRS = \ + asm_pure_loop \ + memcpy_thread \ + thread_loop \ + unroll_loop_thread + +all: $(SUBDIRS) +$(SUBDIRS): + @$(MAKE) -C $@ >/dev/null + +INSTALLDIRS = $(SUBDIRS:%=install-%) + +install-tests: $(INSTALLDIRS) +$(INSTALLDIRS): + @$(MAKE) -C $(@:install-%=%) install-tests >/dev/null + +CLEANDIRS = $(SUBDIRS:%=clean-%) + +clean: $(CLEANDIRS) +$(CLEANDIRS): + $(call QUIET_CLEAN, test-$(@:clean-%=%)) $(Q)$(MAKE) -C $(@:clean-%=%) clean >/dev/null + +.PHONY: all clean $(SUBDIRS) $(CLEANDIRS) $(INSTALLDIRS) diff --git a/tools/perf/tests/shell/coresight/Makefile.miniconfig b/tools/perf/tests/shell/coresight/Makefile.miniconfig new file mode 100644 index 0000000000000000000000000000000000000000..5f72a9cb43f36baa7f233e295e4afe566a713021 --- /dev/null +++ b/tools/perf/tests/shell/coresight/Makefile.miniconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +ifndef DESTDIR +prefix ?= $(HOME) +endif + +DESTDIR_SQ = $(subst ','\'',$(DESTDIR)) +INSTALL = install +INSTDIR_SUB = tests/shell/coresight + +include ../../../../../scripts/Makefile.include +include ../../../../../scripts/Makefile.arch +include ../../../../../scripts/utilities.mak diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop.sh b/tools/perf/tests/shell/coresight/asm_pure_loop.sh new file mode 100755 index 0000000000000000000000000000000000000000..569e9d46162bc319cad7f292660c5257fd5041bc --- /dev/null +++ b/tools/perf/tests/shell/coresight/asm_pure_loop.sh @@ -0,0 +1,18 @@ +#!/bin/sh -e +# CoreSight / ASM Pure Loop + +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +TEST="asm_pure_loop" +. $(dirname $0)/../lib/coresight.sh +ARGS="" +DATV="out" +DATA="$DATD/perf-$TEST-$DATV.data" + +perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS + +perf_dump_aux_verify "$DATA" 10 10 10 + +err=$? +exit $err diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore b/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..468673ac32e8720c24863c792df5e760daf3ba58 --- /dev/null +++ b/tools/perf/tests/shell/coresight/asm_pure_loop/.gitignore @@ -0,0 +1 @@ +asm_pure_loop diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile b/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..206849e92bc935d8be818648ecc91a8bfa16555d --- /dev/null +++ b/tools/perf/tests/shell/coresight/asm_pure_loop/Makefile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +include ../Makefile.miniconfig + +# Binary to produce +BIN=asm_pure_loop +# Any linking/libraries needed for the binary - empty if none needed +LIB= + +all: $(BIN) + +$(BIN): $(BIN).S +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Build line - this is raw asm with no libc to have an always exact binary + $(Q)$(CC) $(BIN).S -nostdlib -static -o $(BIN) $(LIB) +endif +endif + +install-tests: all +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Install the test tool in the right place + $(call QUIET_INSTALL, tests) \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ + $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' +endif +endif + +clean: + $(Q)$(RM) -f $(BIN) + +.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S b/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S new file mode 100644 index 0000000000000000000000000000000000000000..75cf084a927d3d561aad87d6f8589ef7c072190e --- /dev/null +++ b/tools/perf/tests/shell/coresight/asm_pure_loop/asm_pure_loop.S @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Tamas Zsoldos <tamas.zsoldos@arm.com>, 2021 */ + +.globl _start +_start: + mov x0, 0x0000ffff + mov x1, xzr +loop: + nop + nop + cbnz x1, noskip + nop + nop + adrp x2, skip + add x2, x2, :lo12:skip + br x2 + nop + nop +noskip: + nop + nop +skip: + sub x0, x0, 1 + cbnz x0, loop + + mov x0, #0 + mov x8, #93 // __NR_exit syscall + svc #0 diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore b/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..f8217e56091ef38272c1fcf5e5fc5c4b62a25af7 --- /dev/null +++ b/tools/perf/tests/shell/coresight/memcpy_thread/.gitignore @@ -0,0 +1 @@ +memcpy_thread diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/Makefile b/tools/perf/tests/shell/coresight/memcpy_thread/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..2db637eb2c261dfc389b39d5680332cbf4673ad8 --- /dev/null +++ b/tools/perf/tests/shell/coresight/memcpy_thread/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +include ../Makefile.miniconfig + +# Binary to produce +BIN=memcpy_thread +# Any linking/libraries needed for the binary - empty if none needed +LIB=-pthread + +all: $(BIN) + +$(BIN): $(BIN).c +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Build line + $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) +endif +endif + +install-tests: all +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Install the test tool in the right place + $(call QUIET_INSTALL, tests) \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ + $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' +endif +endif + +clean: + $(Q)$(RM) -f $(BIN) + +.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c b/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c new file mode 100644 index 0000000000000000000000000000000000000000..a7e169d1bf645e302af1c9e3a10544ffcd0af76f --- /dev/null +++ b/tools/perf/tests/shell/coresight/memcpy_thread/memcpy_thread.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +// Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <pthread.h> + +struct args { + unsigned long loops; + unsigned long size; + pthread_t th; + void *ret; +}; + +static void *thrfn(void *arg) +{ + struct args *a = arg; + unsigned long i, len = a->loops; + unsigned char *src, *dst; + + src = malloc(a->size * 1024); + dst = malloc(a->size * 1024); + if ((!src) || (!dst)) { + printf("ERR: Can't allocate memory\n"); + exit(1); + } + for (i = 0; i < len; i++) + memcpy(dst, src, a->size * 1024); +} + +static pthread_t new_thr(void *(*fn) (void *arg), void *arg) +{ + pthread_t t; + pthread_attr_t attr; + + pthread_attr_init(&attr); + pthread_create(&t, &attr, fn, arg); + return t; +} + +int main(int argc, char **argv) +{ + unsigned long i, len, size, thr; + pthread_t threads[256]; + struct args args[256]; + long long v; + + if (argc < 4) { + printf("ERR: %s [copysize Kb] [numthreads] [numloops (hundreds)]\n", argv[0]); + exit(1); + } + + v = atoll(argv[1]); + if ((v < 1) || (v > (1024 * 1024))) { + printf("ERR: max memory 1GB (1048576 KB)\n"); + exit(1); + } + size = v; + thr = atol(argv[2]); + if ((thr < 1) || (thr > 256)) { + printf("ERR: threads 1-256\n"); + exit(1); + } + v = atoll(argv[3]); + if ((v < 1) || (v > 40000000000ll)) { + printf("ERR: loops 1-40000000000 (hundreds)\n"); + exit(1); + } + len = v * 100; + for (i = 0; i < thr; i++) { + args[i].loops = len; + args[i].size = size; + args[i].th = new_thr(thrfn, &(args[i])); + } + for (i = 0; i < thr; i++) + pthread_join(args[i].th, &(args[i].ret)); + return 0; +} diff --git a/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh new file mode 100755 index 0000000000000000000000000000000000000000..d21ba8545938d3e72a608a1e7170f0811d67b11a --- /dev/null +++ b/tools/perf/tests/shell/coresight/memcpy_thread_16k_10.sh @@ -0,0 +1,18 @@ +#!/bin/sh -e +# CoreSight / Memcpy 16k 10 Threads + +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +TEST="memcpy_thread" +. $(dirname $0)/../lib/coresight.sh +ARGS="16 10 1" +DATV="16k_10" +DATA="$DATD/perf-$TEST-$DATV.data" + +perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS + +perf_dump_aux_verify "$DATA" 10 10 10 + +err=$? +exit $err diff --git a/tools/perf/tests/shell/coresight/thread_loop/.gitignore b/tools/perf/tests/shell/coresight/thread_loop/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6d4c33eaa9e89317735daabde5a334824bb20365 --- /dev/null +++ b/tools/perf/tests/shell/coresight/thread_loop/.gitignore @@ -0,0 +1 @@ +thread_loop diff --git a/tools/perf/tests/shell/coresight/thread_loop/Makefile b/tools/perf/tests/shell/coresight/thread_loop/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..ea846c038e7acbf6a25e19efc73dbf00ad5e2148 --- /dev/null +++ b/tools/perf/tests/shell/coresight/thread_loop/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +include ../Makefile.miniconfig + +# Binary to produce +BIN=thread_loop +# Any linking/libraries needed for the binary - empty if none needed +LIB=-pthread + +all: $(BIN) + +$(BIN): $(BIN).c +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Build line + $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) +endif +endif + +install-tests: all +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Install the test tool in the right place + $(call QUIET_INSTALL, tests) \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ + $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' +endif +endif + +clean: + $(Q)$(RM) -f $(BIN) + +.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c b/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c new file mode 100644 index 0000000000000000000000000000000000000000..c0158fac7d0b0b47ebfc5dac1a8e3aa72d781981 --- /dev/null +++ b/tools/perf/tests/shell/coresight/thread_loop/thread_loop.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +// Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +// define this for gettid() +#define _GNU_SOURCE + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <pthread.h> +#include <sys/syscall.h> +#ifndef SYS_gettid +// gettid is 178 on arm64 +# define SYS_gettid 178 +#endif +#define gettid() syscall(SYS_gettid) + +struct args { + unsigned int loops; + pthread_t th; + void *ret; +}; + +static void *thrfn(void *arg) +{ + struct args *a = arg; + int i = 0, len = a->loops; + + if (getenv("SHOW_TID")) { + unsigned long long tid = gettid(); + + printf("%llu\n", tid); + } + asm volatile( + "loop:\n" + "add %[i], %[i], #1\n" + "cmp %[i], %[len]\n" + "blt loop\n" + : /* out */ + : /* in */ [i] "r" (i), [len] "r" (len) + : /* clobber */ + ); + return (void *)(long)i; +} + +static pthread_t new_thr(void *(*fn) (void *arg), void *arg) +{ + pthread_t t; + pthread_attr_t attr; + + pthread_attr_init(&attr); + pthread_create(&t, &attr, fn, arg); + return t; +} + +int main(int argc, char **argv) +{ + unsigned int i, len, thr; + pthread_t threads[256]; + struct args args[256]; + + if (argc < 3) { + printf("ERR: %s [numthreads] [numloops (millions)]\n", argv[0]); + exit(1); + } + + thr = atoi(argv[1]); + if ((thr < 1) || (thr > 256)) { + printf("ERR: threads 1-256\n"); + exit(1); + } + len = atoi(argv[2]); + if ((len < 1) || (len > 4000)) { + printf("ERR: max loops 4000 (millions)\n"); + exit(1); + } + len *= 1000000; + for (i = 0; i < thr; i++) { + args[i].loops = len; + args[i].th = new_thr(thrfn, &(args[i])); + } + for (i = 0; i < thr; i++) + pthread_join(args[i].th, &(args[i].ret)); + return 0; +} diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c13636fc778552684017ffc39f417e0f5c74633 --- /dev/null +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_10.sh @@ -0,0 +1,19 @@ +#!/bin/sh -e +# CoreSight / Thread Loop 10 Threads - Check TID + +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +TEST="thread_loop" +. $(dirname $0)/../lib/coresight.sh +ARGS="10 1" +DATV="check-tid-10th" +DATA="$DATD/perf-$TEST-$DATV.data" +STDO="$DATD/perf-$TEST-$DATV.stdout" + +SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO + +perf_dump_aux_tid_verify "$DATA" "$STDO" + +err=$? +exit $err diff --git a/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh new file mode 100755 index 0000000000000000000000000000000000000000..a067145af43ce7835e64d7e126162f75a19b665f --- /dev/null +++ b/tools/perf/tests/shell/coresight/thread_loop_check_tid_2.sh @@ -0,0 +1,19 @@ +#!/bin/sh -e +# CoreSight / Thread Loop 2 Threads - Check TID + +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +TEST="thread_loop" +. $(dirname $0)/../lib/coresight.sh +ARGS="2 20" +DATV="check-tid-2th" +DATA="$DATD/perf-$TEST-$DATV.data" +STDO="$DATD/perf-$TEST-$DATV.stdout" + +SHOW_TID=1 perf record -s $PERFRECOPT -o "$DATA" "$BIN" $ARGS > $STDO + +perf_dump_aux_tid_verify "$DATA" "$STDO" + +err=$? +exit $err diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore b/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2cb4e996dbf3a40c33f524f5355304ab97989cac --- /dev/null +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread/.gitignore @@ -0,0 +1 @@ +unroll_loop_thread diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile b/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..6264c4e3abd1539538c80db72e40f9fc5756b6af --- /dev/null +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +include ../Makefile.miniconfig + +# Binary to produce +BIN=unroll_loop_thread +# Any linking/libraries needed for the binary - empty if none needed +LIB=-pthread + +all: $(BIN) + +$(BIN): $(BIN).c +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Build line + $(Q)$(CC) $(BIN).c -o $(BIN) $(LIB) +endif +endif + +install-tests: all +ifdef CORESIGHT +ifeq ($(ARCH),arm64) +# Install the test tool in the right place + $(call QUIET_INSTALL, tests) \ + $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)'; \ + $(INSTALL) $(BIN) '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/$(INSTDIR_SUB)/$(BIN)/$(BIN)' +endif +endif + +clean: + $(Q)$(RM) -f $(BIN) + +.PHONY: all clean install-tests diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c b/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c new file mode 100644 index 0000000000000000000000000000000000000000..8f6d384208ed971debc09d296006e7408a790b2b --- /dev/null +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread/unroll_loop_thread.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +// Carsten Haitzler <carsten.haitzler@arm.com>, 2021 +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <pthread.h> + +struct args { + pthread_t th; + unsigned int in; + void *ret; +}; + +static void *thrfn(void *arg) +{ + struct args *a = arg; + unsigned int i, in = a->in; + + for (i = 0; i < 10000; i++) { + asm volatile ( +// force an unroll of thia add instruction so we can test long runs of code +#define SNIP1 "add %[in], %[in], #1\n" +// 10 +#define SNIP2 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 SNIP1 +// 100 +#define SNIP3 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 SNIP2 +// 1000 +#define SNIP4 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 SNIP3 +// 10000 +#define SNIP5 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 SNIP4 +// 100000 + SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 SNIP5 + : /* out */ + : /* in */ [in] "r" (in) + : /* clobber */ + ); + } +} + +static pthread_t new_thr(void *(*fn) (void *arg), void *arg) +{ + pthread_t t; + pthread_attr_t attr; + + pthread_attr_init(&attr); + pthread_create(&t, &attr, fn, arg); + return t; +} + +int main(int argc, char **argv) +{ + unsigned int i, thr; + pthread_t threads[256]; + struct args args[256]; + + if (argc < 2) { + printf("ERR: %s [numthreads]\n", argv[0]); + exit(1); + } + + thr = atoi(argv[1]); + if ((thr > 256) || (thr < 1)) { + printf("ERR: threads 1-256\n"); + exit(1); + } + for (i = 0; i < thr; i++) { + args[i].in = rand(); + args[i].th = new_thr(thrfn, &(args[i])); + } + for (i = 0; i < thr; i++) + pthread_join(args[i].th, &(args[i].ret)); + return 0; +} diff --git a/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh new file mode 100755 index 0000000000000000000000000000000000000000..f48c85230b1553539bda38066e98004c492f2fef --- /dev/null +++ b/tools/perf/tests/shell/coresight/unroll_loop_thread_10.sh @@ -0,0 +1,18 @@ +#!/bin/sh -e +# CoreSight / Unroll Loop Thread 10 + +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +TEST="unroll_loop_thread" +. $(dirname $0)/../lib/coresight.sh +ARGS="10" +DATV="10" +DATA="$DATD/perf-$TEST-$DATV.data" + +perf record $PERFRECOPT -o "$DATA" "$BIN" $ARGS + +perf_dump_aux_verify "$DATA" 10 10 10 + +err=$? +exit $err diff --git a/tools/perf/tests/shell/lib/coresight.sh b/tools/perf/tests/shell/lib/coresight.sh new file mode 100644 index 0000000000000000000000000000000000000000..45a1477256b64742b29bd762d8aa4c381bb14d7e --- /dev/null +++ b/tools/perf/tests/shell/lib/coresight.sh @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: GPL-2.0 +# Carsten Haitzler <carsten.haitzler@arm.com>, 2021 + +# This is sourced from a driver script so no need for #!/bin... etc. at the +# top - the assumption below is that it runs as part of sourcing after the +# test sets up some basic env vars to say what it is. + +# This currently works with ETMv4 / ETF not any other packet types at thi +# point. This will need changes if that changes. + +# perf record options for the perf tests to use +PERFRECMEM="-m ,16M" +PERFRECOPT="$PERFRECMEM -e cs_etm//u" + +TOOLS=$(dirname $0) +DIR="$TOOLS/$TEST" +BIN="$DIR/$TEST" +# If the test tool/binary does not exist and is executable then skip the test +if ! test -x "$BIN"; then exit 2; fi +DATD="." +# If the data dir env is set then make the data dir use that instead of ./ +if test -n "$PERF_TEST_CORESIGHT_DATADIR"; then + DATD="$PERF_TEST_CORESIGHT_DATADIR"; +fi +# If the stat dir env is set then make the data dir use that instead of ./ +STATD="." +if test -n "$PERF_TEST_CORESIGHT_STATDIR"; then + STATD="$PERF_TEST_CORESIGHT_STATDIR"; +fi + +# Called if the test fails - error code 1 +err() { + echo "$1" + exit 1 +} + +# Check that some statistics from our perf +check_val_min() { + STATF="$4" + if test "$2" -lt "$3"; then + echo ", FAILED" >> "$STATF" + err "Sanity check number of $1 is too low ($2 < $3)" + fi +} + +perf_dump_aux_verify() { + # Some basic checking that the AUX chunk contains some sensible data + # to see that we are recording something and at least a minimum + # amount of it. We should almost always see Fn packets in just about + # anything but certainly we will see some trace info and async + # packets + DUMP="$DATD/perf-tmp-aux-dump.txt" + perf report --stdio --dump -i "$1" | \ + grep -o -e I_ATOM_F -e I_ASYNC -e I_TRACE_INFO > "$DUMP" + # Simply count how many of these packets we find to see that we are + # producing a reasonable amount of data - exact checks are not sane + # as this is a lossy process where we may lose some blocks and the + # compiler may produce different code depending on the compiler and + # optimization options, so this is rough just to see if we're + # either missing almost all the data or all of it + ATOM_FX_NUM=`grep I_ATOM_F "$DUMP" | wc -l` + ASYNC_NUM=`grep I_ASYNC "$DUMP" | wc -l` + TRACE_INFO_NUM=`grep I_TRACE_INFO "$DUMP" | wc -l` + rm -f "$DUMP" + + # Arguments provide minimums for a pass + CHECK_FX_MIN="$2" + CHECK_ASYNC_MIN="$3" + CHECK_TRACE_INFO_MIN="$4" + + # Write out statistics, so over time you can track results to see if + # there is a pattern - for example we have less "noisy" results that + # produce more consistent amounts of data each run, to see if over + # time any techinques to minimize data loss are having an effect or + # not + STATF="$STATD/stats-$TEST-$DATV.csv" + if ! test -f "$STATF"; then + echo "ATOM Fx Count, Minimum, ASYNC Count, Minimum, TRACE INFO Count, Minimum" > "$STATF" + fi + echo -n "$ATOM_FX_NUM, $CHECK_FX_MIN, $ASYNC_NUM, $CHECK_ASYNC_MIN, $TRACE_INFO_NUM, $CHECK_TRACE_INFO_MIN" >> "$STATF" + + # Actually check to see if we passed or failed. + check_val_min "ATOM_FX" "$ATOM_FX_NUM" "$CHECK_FX_MIN" "$STATF" + check_val_min "ASYNC" "$ASYNC_NUM" "$CHECK_ASYNC_MIN" "$STATF" + check_val_min "TRACE_INFO" "$TRACE_INFO_NUM" "$CHECK_TRACE_INFO_MIN" "$STATF" + echo ", Ok" >> "$STATF" +} + +perf_dump_aux_tid_verify() { + # Specifically crafted test will produce a list of Tread ID's to + # stdout that need to be checked to see that they have had trace + # info collected in AUX blocks in the perf data. This will go + # through all the TID's that are listed as CID=0xabcdef and see + # that all the Thread IDs the test tool reports are in the perf + # data AUX chunks + + # The TID test tools will print a TID per stdout line that are being + # tested + TIDS=`cat "$2"` + # Scan the perf report to find the TIDs that are actually CID in hex + # and build a list of the ones found + FOUND_TIDS=`perf report --stdio --dump -i "$1" | \ + grep -o "CID=0x[0-9a-z]\+" | sed 's/CID=//g' | \ + uniq | sort | uniq` + # No CID=xxx found - maybe your kernel is reporting these as + # VMID=xxx so look there + if test -z "$FOUND_TIDS"; then + FOUND_TIDS=`perf report --stdio --dump -i "$1" | \ + grep -o "VMID=0x[0-9a-z]\+" | sed 's/VMID=//g' | \ + uniq | sort | uniq` + fi + + # Iterate over the list of TIDs that the test says it has and find + # them in the TIDs found in the perf report + MISSING="" + for TID2 in $TIDS; do + FOUND="" + for TIDHEX in $FOUND_TIDS; do + TID=`printf "%i" $TIDHEX` + if test "$TID" -eq "$TID2"; then + FOUND="y" + break + fi + done + if test -z "$FOUND"; then + MISSING="$MISSING $TID" + fi + done + if test -n "$MISSING"; then + err "Thread IDs $MISSING not found in perf AUX data" + fi +} diff --git a/tools/perf/tests/shell/lib/probe_vfs_getname.sh b/tools/perf/tests/shell/lib/probe_vfs_getname.sh index 5b17d916c5558abc7242aef5e8b6d82d6a52dbe4..b616d42bd19d437db7cfcbed76a883aa8c694ef0 100644 --- a/tools/perf/tests/shell/lib/probe_vfs_getname.sh +++ b/tools/perf/tests/shell/lib/probe_vfs_getname.sh @@ -19,6 +19,6 @@ add_probe_vfs_getname() { } skip_if_no_debuginfo() { - add_probe_vfs_getname -v 2>&1 | egrep -q "^(Failed to find the path for the kernel|Debuginfo-analysis is not supported)" && return 2 + add_probe_vfs_getname -v 2>&1 | egrep -q "^(Failed to find the path for the kernel|Debuginfo-analysis is not supported)|(file has no debug information)" && return 2 return 1 } diff --git a/tools/perf/tests/shell/lib/waiting.sh b/tools/perf/tests/shell/lib/waiting.sh new file mode 100644 index 0000000000000000000000000000000000000000..e7a39134a68e8c0fc762d3233c8fab611ad19799 --- /dev/null +++ b/tools/perf/tests/shell/lib/waiting.sh @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: GPL-2.0 + +tenths=date\ +%s%1N + +# Wait for PID $1 to have $2 number of threads started +# Time out after $3 tenths of a second or 5 seconds if $3 is "" +wait_for_threads() +{ + tm_out=$3 ; [ -n "${tm_out}" ] || tm_out=50 + start_time=$($tenths) + while [ -e "/proc/$1/task" ] ; do + th_cnt=$(find "/proc/$1/task" -mindepth 1 -maxdepth 1 -printf x | wc -c) + if [ "${th_cnt}" -ge "$2" ] ; then + return 0 + fi + # Wait at most tm_out tenths of a second + if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then + echo "PID $1 does not have $2 threads" + return 1 + fi + done + return 1 +} + +# Wait for perf record -vvv 2>$2 with PID $1 to start by looking at file $2 +# It depends on capturing perf record debug message "perf record has started" +# Time out after $3 tenths of a second or 5 seconds if $3 is "" +wait_for_perf_to_start() +{ + tm_out=$3 ; [ -n "${tm_out}" ] || tm_out=50 + echo "Waiting for \"perf record has started\" message" + start_time=$($tenths) + while [ -e "/proc/$1" ] ; do + if grep -q "perf record has started" "$2" ; then + echo OK + break + fi + # Wait at most tm_out tenths of a second + if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then + echo "perf recording did not start" + return 1 + fi + done + return 0 +} + +# Wait for process PID %1 to exit +# Time out after $2 tenths of a second or 5 seconds if $2 is "" +wait_for_process_to_exit() +{ + tm_out=$2 ; [ -n "${tm_out}" ] || tm_out=50 + start_time=$($tenths) + while [ -e "/proc/$1" ] ; do + # Wait at most tm_out tenths of a second + if [ $(($($tenths) - start_time)) -ge $tm_out ] ; then + echo "PID $1 did not exit as expected" + return 1 + fi + done + return 0 +} + +# Check if PID $1 is still running after $2 tenths of a second +# or 0.3 seconds if $2 is "" +is_running() +{ + tm_out=$2 ; [ -n "${tm_out}" ] || tm_out=3 + start_time=$($tenths) + while [ -e "/proc/$1" ] ; do + # Check for at least tm_out tenths of a second + if [ $(($($tenths) - start_time)) -gt $tm_out ] ; then + return 0 + fi + done + echo "PID $1 exited prematurely" + return 1 +} diff --git a/tools/perf/tests/shell/lock_contention.sh b/tools/perf/tests/shell/lock_contention.sh new file mode 100755 index 0000000000000000000000000000000000000000..04bf604e3c6f8807b75ad5e15e8366b590f1f4cb --- /dev/null +++ b/tools/perf/tests/shell/lock_contention.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# kernel lock contention analysis test +# SPDX-License-Identifier: GPL-2.0 + +set -e + +err=0 +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +result=$(mktemp /tmp/__perf_test.result.XXXXX) + +cleanup() { + rm -f ${perfdata} + rm -f ${result} + trap - exit term int +} + +trap_cleanup() { + cleanup + exit ${err} +} +trap trap_cleanup exit term int + +check() { + if [ `id -u` != 0 ]; then + echo "[Skip] No root permission" + err=2 + exit + fi + + if ! perf list | grep -q lock:contention_begin; then + echo "[Skip] No lock contention tracepoints" + err=2 + exit + fi +} + +test_record() +{ + echo "Testing perf lock record and perf lock contention" + perf lock record -o ${perfdata} -- perf bench sched messaging > /dev/null 2>&1 + # the output goes to the stderr and we expect only 1 output (-E 1) + perf lock contention -i ${perfdata} -E 1 -q 2> ${result} + if [ $(cat "${result}" | wc -l) != "1" ]; then + echo "[Fail] Recorded result count is not 1:" $(cat "${result}" | wc -l) + err=1 + exit + fi +} + +test_bpf() +{ + echo "Testing perf lock contention --use-bpf" + + if ! perf lock con -b true > /dev/null 2>&1 ; then + echo "[Skip] No BPF support" + exit + fi + + # the perf lock contention output goes to the stderr + perf lock con -a -b -E 1 -q -- perf bench sched messaging > /dev/null 2> ${result} + if [ $(cat "${result}" | wc -l) != "1" ]; then + echo "[Fail] BPF result count is not 1:" $(cat "${result}" | wc -l) + err=1 + exit + fi +} + +check + +test_record +test_bpf + +exit ${err} diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 00c7285ce1ac6445f5ab38f8eb045683fde8184c..301f95427159dd40e7df0d0bdafd323af6d1198d 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -61,7 +61,7 @@ test_register_capture() { echo "Register capture test [Skipped missing registers]" return fi - if ! perf record -o - --intr-regs=di,r8,dx,cx -e cpu/br_inst_retired.near_call/p \ + if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call:p \ -c 1000 --per-thread true 2> /dev/null \ | perf script -F ip,sym,iregs -i - 2> /dev/null \ | egrep -q "DI:" diff --git a/tools/perf/tests/shell/stat+csv_output.sh b/tools/perf/tests/shell/stat+csv_output.sh index eb5196f58190e71db2c1f83c4872a6f0a7a87dca..b7f050aa6210c92177c7a80a9b2bbf3e08fd2f4f 100755 --- a/tools/perf/tests/shell/stat+csv_output.sh +++ b/tools/perf/tests/shell/stat+csv_output.sh @@ -6,6 +6,8 @@ set -e +skip_test=0 + function commachecker() { local -i cnt=0 @@ -156,14 +158,47 @@ check_per_socket() echo "[Success]" } +# The perf stat options for per-socket, per-core, per-die +# and -A ( no_aggr mode ) uses the info fetched from this +# directory: "/sys/devices/system/cpu/cpu*/topology". For +# example, socket value is fetched from "physical_package_id" +# file in topology directory. +# Reference: cpu__get_topology_int in util/cpumap.c +# If the platform doesn't expose topology information, values +# will be set to -1. For example, incase of pSeries platform +# of powerpc, value for "physical_package_id" is restricted +# and set to -1. Check here validates the socket-id read from +# topology file before proceeding further + +FILE_LOC="/sys/devices/system/cpu/cpu*/topology/" +FILE_NAME="physical_package_id" + +check_for_topology() +{ + if ! ParanoidAndNotRoot 0 + then + socket_file=`ls $FILE_LOC/$FILE_NAME | head -n 1` + [ -z $socket_file ] && return 0 + socket_id=`cat $socket_file` + [ $socket_id == -1 ] && skip_test=1 + return 0 + fi +} + +check_for_topology check_no_args check_system_wide -check_system_wide_no_aggr check_interval check_event -check_per_core check_per_thread -check_per_die check_per_node -check_per_socket +if [ $skip_test -ne 1 ] +then + check_system_wide_no_aggr + check_per_core + check_per_die + check_per_socket +else + echo "[Skip] Skipping tests for system_wide_no_aggr, per_core, per_die and per_socket since socket id exposed via topology is invalid" +fi exit 0 diff --git a/tools/perf/tests/shell/stat+json_output.sh b/tools/perf/tests/shell/stat+json_output.sh index ea8714a360512a87309b3bb6acdc8ba82717da49..2c4212c641edef5e7050f997980623984610a2a7 100755 --- a/tools/perf/tests/shell/stat+json_output.sh +++ b/tools/perf/tests/shell/stat+json_output.sh @@ -6,6 +6,8 @@ set -e +skip_test=0 + pythonchecker=$(dirname $0)/lib/perf_json_output_lint.py if [ "x$PYTHON" == "x" ] then @@ -134,14 +136,47 @@ check_per_socket() echo "[Success]" } +# The perf stat options for per-socket, per-core, per-die +# and -A ( no_aggr mode ) uses the info fetched from this +# directory: "/sys/devices/system/cpu/cpu*/topology". For +# example, socket value is fetched from "physical_package_id" +# file in topology directory. +# Reference: cpu__get_topology_int in util/cpumap.c +# If the platform doesn't expose topology information, values +# will be set to -1. For example, incase of pSeries platform +# of powerpc, value for "physical_package_id" is restricted +# and set to -1. Check here validates the socket-id read from +# topology file before proceeding further + +FILE_LOC="/sys/devices/system/cpu/cpu*/topology/" +FILE_NAME="physical_package_id" + +check_for_topology() +{ + if ! ParanoidAndNotRoot 0 + then + socket_file=`ls $FILE_LOC/$FILE_NAME | head -n 1` + [ -z $socket_file ] && return 0 + socket_id=`cat $socket_file` + [ $socket_id == -1 ] && skip_test=1 + return 0 + fi +} + +check_for_topology check_no_args check_system_wide -check_system_wide_no_aggr check_interval check_event -check_per_core check_per_thread -check_per_die check_per_node -check_per_socket +if [ $skip_test -ne 1 ] +then + check_system_wide_no_aggr + check_per_core + check_per_die + check_per_socket +else + echo "[Skip] Skipping tests for system_wide_no_aggr, per_core, per_die and per_socket since socket id exposed via topology is invalid" +fi exit 0 diff --git a/tools/perf/tests/shell/stat.sh b/tools/perf/tests/shell/stat.sh index 9313ef2739e07d12afbd18a750205ee6d3036622..26a51b48aee464e3b18938736872f3aa6f00cf70 100755 --- a/tools/perf/tests/shell/stat.sh +++ b/tools/perf/tests/shell/stat.sh @@ -28,6 +28,24 @@ test_stat_record_report() { echo "stat record and report test [Success]" } +test_stat_repeat_weak_groups() { + echo "stat repeat weak groups test" + if ! perf stat -e '{cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles}' \ + true 2>&1 | grep -q 'seconds time elapsed' + then + echo "stat repeat weak groups test [Skipped event parsing failed]" + return + fi + if ! perf stat -r2 -e '{cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles,cycles}:W' \ + true > /dev/null 2>&1 + then + echo "stat repeat weak groups test [Failed]" + err=1 + return + fi + echo "stat repeat weak groups test [Success]" +} + test_topdown_groups() { # Topdown events must be grouped with the slots event first. Test that # parse-events reorders this. @@ -75,6 +93,7 @@ test_topdown_weak_groups() { test_default_stat test_stat_record_report +test_stat_repeat_weak_groups test_topdown_groups test_topdown_weak_groups exit $err diff --git a/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh b/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh new file mode 100755 index 0000000000000000000000000000000000000000..d724855d097c21a2bb23f6df16ba70c7f33dcb75 --- /dev/null +++ b/tools/perf/tests/shell/stat_bpf_counters_cgrp.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# perf stat --bpf-counters --for-each-cgroup test +# SPDX-License-Identifier: GPL-2.0 + +set -e + +test_cgroups= +if [ "$1" = "-v" ]; then + verbose="1" +fi + +# skip if --bpf-counters --for-each-cgroup is not supported +check_bpf_counter() +{ + if ! perf stat -a --bpf-counters --for-each-cgroup / true > /dev/null 2>&1; then + if [ "${verbose}" = "1" ]; then + echo "Skipping: --bpf-counters --for-each-cgroup not supported" + perf --no-pager stat -a --bpf-counters --for-each-cgroup / true || true + fi + exit 2 + fi +} + +# find two cgroups to measure +find_cgroups() +{ + # try usual systemd slices first + if [ -d /sys/fs/cgroup/system.slice -a -d /sys/fs/cgroup/user.slice ]; then + test_cgroups="system.slice,user.slice" + return + fi + + # try root and self cgroups + local self_cgrp=$(grep perf_event /proc/self/cgroup | cut -d: -f3) + if [ -z ${self_cgrp} ]; then + # cgroup v2 doesn't specify perf_event + self_cgrp=$(grep ^0: /proc/self/cgroup | cut -d: -f3) + fi + + if [ -z ${self_cgrp} ]; then + test_cgroups="/" + else + test_cgroups="/,${self_cgrp}" + fi +} + +# As cgroup events are cpu-wide, we cannot simply compare the result. +# Just check if it runs without failure and has non-zero results. +check_system_wide_counted() +{ + local output + + output=$(perf stat -a --bpf-counters --for-each-cgroup ${test_cgroups} -e cpu-clock -x, sleep 1 2>&1) + if echo ${output} | grep -q -F "<not "; then + echo "Some system-wide events are not counted" + if [ "${verbose}" = "1" ]; then + echo ${output} + fi + exit 1 + fi +} + +check_cpu_list_counted() +{ + local output + + output=$(perf stat -C 1 --bpf-counters --for-each-cgroup ${test_cgroups} -e cpu-clock -x, taskset -c 1 sleep 1 2>&1) + if echo ${output} | grep -q -F "<not "; then + echo "Some CPU events are not counted" + if [ "${verbose}" = "1" ]; then + echo ${output} + fi + exit 1 + fi +} + +check_bpf_counter +find_cgroups + +check_system_wide_counted +check_cpu_list_counted + +exit 0 diff --git a/tools/perf/tests/shell/test_arm_coresight.sh b/tools/perf/tests/shell/test_arm_coresight.sh index e4cb4f1806ffa6b4b99e336abe08441e78ef7797..daad786cf48d97e78288ba633a84a9c205511cc5 100755 --- a/tools/perf/tests/shell/test_arm_coresight.sh +++ b/tools/perf/tests/shell/test_arm_coresight.sh @@ -70,7 +70,7 @@ perf_report_instruction_samples() { # 68.12% touch libc-2.27.so [.] _dl_addr # 5.80% touch libc-2.27.so [.] getenv # 4.35% touch ld-2.27.so [.] _dl_fixup - perf report --itrace=i1000i --stdio -i ${perfdata} 2>&1 | \ + perf report --itrace=i20i --stdio -i ${perfdata} 2>&1 | \ egrep " +[0-9]+\.[0-9]+% +$1" > /dev/null 2>&1 } diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index c644f94a6500210538f75b133c78db1c208ecd51..ec801cffae6bcd359c26aebb54ced13d74ba7586 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -12,7 +12,8 @@ if ! [ -x "$(command -v cc)" ]; then fi # skip the test if the hardware doesn't support branch stack sampling -perf record -b -o- -B true > /dev/null 2>&1 || exit 2 +# and if the architecture doesn't support filter types: any,save_type,u +perf record -b -o- -B --branch-filter any,save_type,u true > /dev/null 2>&1 || exit 2 TMPDIR=$(mktemp -d /tmp/__perf_test.program.XXXXX) diff --git a/tools/perf/tests/shell/test_data_symbol.sh b/tools/perf/tests/shell/test_data_symbol.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd6eb54d235d8344796c1594834a464cddc58c0a --- /dev/null +++ b/tools/perf/tests/shell/test_data_symbol.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Test data symbol + +# SPDX-License-Identifier: GPL-2.0 +# Leo Yan <leo.yan@linaro.org>, 2022 + +skip_if_no_mem_event() { + perf mem record -e list 2>&1 | egrep -q 'available' && return 0 + return 2 +} + +skip_if_no_mem_event || exit 2 + +# skip if there's no compiler +if ! [ -x "$(command -v cc)" ]; then + echo "skip: no compiler, install gcc" + exit 2 +fi + +TEST_PROGRAM=$(mktemp /tmp/__perf_test.program.XXXXX) +PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX) + +check_result() { + # The memory report format is as below: + # 99.92% ... [.] buf1+0x38 + result=$(perf mem report -i ${PERF_DATA} -s symbol_daddr -q 2>&1 | + awk '/buf1/ { print $4 }') + + # Testing is failed if has no any sample for "buf1" + [ -z "$result" ] && return 1 + + while IFS= read -r line; do + # The "data1" and "data2" fields in structure "buf1" have + # offset "0x0" and "0x38", returns failure if detect any + # other offset value. + if [ "$line" != "buf1+0x0" ] && [ "$line" != "buf1+0x38" ]; then + return 1 + fi + done <<< "$result" + + return 0 +} + +cleanup_files() +{ + echo "Cleaning up files..." + rm -f ${PERF_DATA} + rm -f ${TEST_PROGRAM} +} + +trap cleanup_files exit term int + +# compile test program +echo "Compiling test program..." +cat << EOF | cc -o ${TEST_PROGRAM} -x c - +typedef struct _buf { + char data1; + char reserved[55]; + char data2; +} buf __attribute__((aligned(64))); + +static buf buf1; + +int main(void) { + for (;;) { + buf1.data1++; + buf1.data2 += buf1.data1; + } + return 0; +} +EOF + +echo "Recording workload..." + +# perf mem/c2c internally uses IBS PMU on AMD CPU which doesn't support +# user/kernel filtering and per-process monitoring, spin program on +# specific CPU and test in per-CPU mode. +is_amd=$(egrep -c 'vendor_id.*AuthenticAMD' /proc/cpuinfo) +if (($is_amd >= 1)); then + perf mem record -o ${PERF_DATA} -C 0 -- taskset -c 0 $TEST_PROGRAM & +else + perf mem record --all-user -o ${PERF_DATA} -- $TEST_PROGRAM & +fi + +PERFPID=$! + +sleep 1 + +kill $PERFPID +wait $PERFPID + +check_result +exit $? diff --git a/tools/perf/tests/shell/test_intel_pt.sh b/tools/perf/tests/shell/test_intel_pt.sh index a3298643884d9de295c45aacffdec158ec3aacc4..4c0aabbe33bdfaf05df66747a7091edc5ddb7651 100755 --- a/tools/perf/tests/shell/test_intel_pt.sh +++ b/tools/perf/tests/shell/test_intel_pt.sh @@ -7,32 +7,116 @@ set -e # Skip if no Intel PT perf list | grep -q 'intel_pt//' || exit 2 +shelldir=$(dirname "$0") +. "${shelldir}"/lib/waiting.sh + skip_cnt=0 ok_cnt=0 err_cnt=0 -tmpfile=`mktemp` -perfdatafile=`mktemp` +temp_dir=$(mktemp -d /tmp/perf-test-intel-pt-sh.XXXXXXXXXX) + +tmpfile="${temp_dir}/tmp-perf.data" +perfdatafile="${temp_dir}/test-perf.data" +outfile="${temp_dir}/test-out.txt" +errfile="${temp_dir}/test-err.txt" +workload="${temp_dir}/workload" +awkscript="${temp_dir}/awkscript" +jitdump_workload="${temp_dir}/jitdump_workload" +maxbrstack="${temp_dir}/maxbrstack.py" + +cleanup() +{ + trap - EXIT TERM INT + sane=$(echo "${temp_dir}" | cut -b 1-26) + if [ "${sane}" = "/tmp/perf-test-intel-pt-sh" ] ; then + echo "--- Cleaning up ---" + rm -f "${temp_dir}/"* + rmdir "${temp_dir}" + fi +} + +trap_cleanup() +{ + cleanup + exit 1 +} + +trap trap_cleanup EXIT TERM INT + +# perf record for testing without decoding +perf_record_no_decode() +{ + # Options to speed up recording: no post-processing, no build-id cache update, + # and no BPF events. + perf record -B -N --no-bpf-event "$@" +} + +# perf record for testing should not need BPF events +perf_record_no_bpf() +{ + # Options for no BPF events + perf record --no-bpf-event "$@" +} + +have_workload=false +cat << _end_of_file_ | /usr/bin/cc -o "${workload}" -xc - -pthread && have_workload=true +#include <time.h> +#include <pthread.h> + +void work(void) { + struct timespec tm = { + .tv_nsec = 1000000, + }; + int i; + + /* Run for about 30 seconds */ + for (i = 0; i < 30000; i++) + nanosleep(&tm, NULL); +} + +void *threadfunc(void *arg) { + work(); + return NULL; +} + +int main(void) { + pthread_t th; + + pthread_create(&th, NULL, threadfunc, NULL); + work(); + pthread_join(th, NULL); + return 0; +} +_end_of_file_ can_cpu_wide() { - perf record -o ${tmpfile} -B -N --no-bpf-event -e dummy:u -C $1 true 2>&1 >/dev/null || return 2 + echo "Checking for CPU-wide recording on CPU $1" + if ! perf_record_no_decode -o "${tmpfile}" -e dummy:u -C "$1" true >/dev/null 2>&1 ; then + echo "No so skipping" + return 2 + fi + echo OK return 0 } test_system_wide_side_band() { + echo "--- Test system-wide sideband ---" + # Need CPU 0 and CPU 1 can_cpu_wide 0 || return $? can_cpu_wide 1 || return $? # Record on CPU 0 a task running on CPU 1 - perf record -B -N --no-bpf-event -o ${perfdatafile} -e intel_pt//u -C 0 -- taskset --cpu-list 1 uname + perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u -C 0 -- taskset --cpu-list 1 uname # Should get MMAP events from CPU 1 because they can be needed to decode - mmap_cnt=`perf script -i ${perfdatafile} --no-itrace --show-mmap-events -C 1 2>/dev/null | grep MMAP | wc -l` + mmap_cnt=$(perf script -i "${perfdatafile}" --no-itrace --show-mmap-events -C 1 2>/dev/null | grep -c MMAP) - if [ ${mmap_cnt} -gt 0 ] ; then + if [ "${mmap_cnt}" -gt 0 ] ; then + echo OK return 0 fi @@ -40,25 +124,527 @@ test_system_wide_side_band() return 1 } +can_kernel() +{ + if [ -z "${can_kernel_trace}" ] ; then + can_kernel_trace=0 + perf_record_no_decode -o "${tmpfile}" -e dummy:k true >/dev/null 2>&1 && can_kernel_trace=1 + fi + if [ ${can_kernel_trace} -eq 0 ] ; then + echo "SKIP: no kernel tracing" + return 2 + fi + return 0 +} + +test_per_thread() +{ + k="$1" + desc="$2" + + echo "--- Test per-thread ${desc}recording ---" + + if ! $have_workload ; then + echo "No workload, so skipping" + return 2 + fi + + if [ "${k}" = "k" ] ; then + can_kernel || return 2 + fi + + cat <<- "_end_of_file_" > "${awkscript}" + BEGIN { + s = "[ ]*" + u = s"[0-9]+"s + d = s"[0-9-]+"s + x = s"[0-9a-fA-FxX]+"s + mmapping = "idx"u": mmapping fd"u + set_output = "idx"u": set output fd"u"->"u + perf_event_open = "sys_perf_event_open: pid"d"cpu"d"group_fd"d"flags"x"="u + } + + /perf record opening and mmapping events/ { + if (!done) + active = 1 + } + + /perf record done opening and mmapping events/ { + active = 0 + done = 1 + } + + $0 ~ perf_event_open && active { + match($0, perf_event_open) + $0 = substr($0, RSTART, RLENGTH) + pid = $3 + cpu = $5 + fd = $11 + print "pid " pid " cpu " cpu " fd " fd " : " $0 + fd_array[fd] = fd + pid_array[fd] = pid + cpu_array[fd] = cpu + } + + $0 ~ mmapping && active { + match($0, mmapping) + $0 = substr($0, RSTART, RLENGTH) + fd = $5 + print "fd " fd " : " $0 + if (fd in fd_array) { + mmap_array[fd] = 1 + } else { + print "Unknown fd " fd + exit 1 + } + } + + $0 ~ set_output && active { + match($0, set_output) + $0 = substr($0, RSTART, RLENGTH) + fd = $6 + fd_to = $8 + print "fd " fd " fd_to " fd_to " : " $0 + if (fd in fd_array) { + if (fd_to in fd_array) { + set_output_array[fd] = fd_to + } else { + print "Unknown fd " fd_to + exit 1 + } + } else { + print "Unknown fd " fd + exit 1 + } + } + + END { + print "Checking " length(fd_array) " fds" + for (fd in fd_array) { + if (fd in mmap_array) { + pid = pid_array[fd] + if (pid != -1) { + if (pid in pids) { + print "More than 1 mmap for PID " pid + exit 1 + } + pids[pid] = 1 + } + cpu = cpu_array[fd] + if (cpu != -1) { + if (cpu in cpus) { + print "More than 1 mmap for CPU " cpu + exit 1 + } + cpus[cpu] = 1 + } + } else if (!(fd in set_output_array)) { + print "No mmap for fd " fd + exit 1 + } + } + n = length(pids) + if (n != thread_cnt) { + print "Expected " thread_cnt " per-thread mmaps - found " n + exit 1 + } + } + _end_of_file_ + + $workload & + w1=$! + $workload & + w2=$! + echo "Workload PIDs are $w1 and $w2" + wait_for_threads ${w1} 2 + wait_for_threads ${w2} 2 + + perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u"${k}" -vvv --per-thread -p "${w1},${w2}" 2>"${errfile}" >"${outfile}" & + ppid=$! + echo "perf PID is $ppid" + wait_for_perf_to_start ${ppid} "${errfile}" || return 1 + + kill ${w1} + wait_for_process_to_exit ${w1} || return 1 + is_running ${ppid} || return 1 + + kill ${w2} + wait_for_process_to_exit ${w2} || return 1 + wait_for_process_to_exit ${ppid} || return 1 + + awk -v thread_cnt=4 -f "${awkscript}" "${errfile}" || return 1 + + echo OK + return 0 +} + +test_jitdump() +{ + echo "--- Test tracing self-modifying code that uses jitdump ---" + + script_path=$(realpath "$0") + script_dir=$(dirname "$script_path") + jitdump_incl_dir="${script_dir}/../../util" + jitdump_h="${jitdump_incl_dir}/jitdump.h" + + if [ ! -e "${jitdump_h}" ] ; then + echo "SKIP: Include file jitdump.h not found" + return 2 + fi + + if [ -z "${have_jitdump_workload}" ] ; then + have_jitdump_workload=false + # Create a workload that uses self-modifying code and generates its own jitdump file + cat <<- "_end_of_file_" | /usr/bin/cc -o "${jitdump_workload}" -I "${jitdump_incl_dir}" -xc - -pthread && have_jitdump_workload=true + #define _GNU_SOURCE + #include <sys/mman.h> + #include <sys/types.h> + #include <stddef.h> + #include <stdio.h> + #include <stdint.h> + #include <unistd.h> + #include <string.h> + + #include "jitdump.h" + + #define CHK_BYTE 0x5a + + static inline uint64_t rdtsc(void) + { + unsigned int low, high; + + asm volatile("rdtsc" : "=a" (low), "=d" (high)); + + return low | ((uint64_t)high) << 32; + } + + static FILE *open_jitdump(void) + { + struct jitheader header = { + .magic = JITHEADER_MAGIC, + .version = JITHEADER_VERSION, + .total_size = sizeof(header), + .pid = getpid(), + .timestamp = rdtsc(), + .flags = JITDUMP_FLAGS_ARCH_TIMESTAMP, + }; + char filename[256]; + FILE *f; + void *m; + + snprintf(filename, sizeof(filename), "jit-%d.dump", getpid()); + f = fopen(filename, "w+"); + if (!f) + goto err; + /* Create an MMAP event for the jitdump file. That is how perf tool finds it. */ + m = mmap(0, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, fileno(f), 0); + if (m == MAP_FAILED) + goto err_close; + munmap(m, 4096); + if (fwrite(&header,sizeof(header),1,f) != 1) + goto err_close; + return f; + + err_close: + fclose(f); + err: + return NULL; + } + + static int write_jitdump(FILE *f, void *addr, const uint8_t *dat, size_t sz, uint64_t *idx) + { + struct jr_code_load rec = { + .p.id = JIT_CODE_LOAD, + .p.total_size = sizeof(rec) + sz, + .p.timestamp = rdtsc(), + .pid = getpid(), + .tid = gettid(), + .vma = (unsigned long)addr, + .code_addr = (unsigned long)addr, + .code_size = sz, + .code_index = ++*idx, + }; + + if (fwrite(&rec,sizeof(rec),1,f) != 1 || + fwrite(dat, sz, 1, f) != 1) + return -1; + return 0; + } + + static void close_jitdump(FILE *f) + { + fclose(f); + } + + int main() + { + /* Get a memory page to store executable code */ + void *addr = mmap(0, 4096, PROT_WRITE | PROT_EXEC, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + /* Code to execute: mov CHK_BYTE, %eax ; ret */ + uint8_t dat[] = {0xb8, CHK_BYTE, 0x00, 0x00, 0x00, 0xc3}; + FILE *f = open_jitdump(); + uint64_t idx = 0; + int ret = 1; + + if (!f) + return 1; + /* Copy executable code to executable memory page */ + memcpy(addr, dat, sizeof(dat)); + /* Record it in the jitdump file */ + if (write_jitdump(f, addr, dat, sizeof(dat), &idx)) + goto out_close; + /* Call it */ + ret = ((int (*)(void))addr)() - CHK_BYTE; + out_close: + close_jitdump(f); + return ret; + } + _end_of_file_ + fi + + if ! $have_jitdump_workload ; then + echo "SKIP: No jitdump workload" + return 2 + fi + + # Change to temp_dir so jitdump collateral files go there + cd "${temp_dir}" + perf_record_no_bpf -o "${tmpfile}" -e intel_pt//u "${jitdump_workload}" + perf inject -i "${tmpfile}" -o "${perfdatafile}" --jit + decode_br_cnt=$(perf script -i "${perfdatafile}" --itrace=b | wc -l) + # Note that overflow and lost errors are suppressed for the error count + decode_err_cnt=$(perf script -i "${perfdatafile}" --itrace=e-o-l | grep -ci error) + cd - + # Should be thousands of branches + if [ "${decode_br_cnt}" -lt 1000 ] ; then + echo "Decode failed, only ${decode_br_cnt} branches" + return 1 + fi + # Should be no errors + if [ "${decode_err_cnt}" -ne 0 ] ; then + echo "Decode failed, ${decode_err_cnt} errors" + perf script -i "${perfdatafile}" --itrace=e-o-l --show-mmap-events | cat + return 1 + fi + + echo OK + return 0 +} + +test_packet_filter() +{ + echo "--- Test with MTC and TSC disabled ---" + # Disable MTC and TSC + perf_record_no_decode -o "${perfdatafile}" -e intel_pt/mtc=0,tsc=0/u uname + # Should not get MTC packet + mtc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "MTC 0x") + if [ "${mtc_cnt}" -ne 0 ] ; then + echo "Failed to filter with mtc=0" + return 1 + fi + # Should not get TSC package + tsc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TSC 0x") + if [ "${tsc_cnt}" -ne 0 ] ; then + echo "Failed to filter with tsc=0" + return 1 + fi + echo OK + return 0 +} + +test_disable_branch() +{ + echo "--- Test with branches disabled ---" + # Disable branch + perf_record_no_decode -o "${perfdatafile}" -e intel_pt/branch=0/u uname + # Should not get branch related packets + tnt_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TNT 0x") + tip_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "TIP 0x") + fup_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "FUP 0x") + if [ "${tnt_cnt}" -ne 0 ] || [ "${tip_cnt}" -ne 0 ] || [ "${fup_cnt}" -ne 0 ] ; then + echo "Failed to disable branches" + return 1 + fi + echo OK + return 0 +} + +test_time_cyc() +{ + echo "--- Test with/without CYC ---" + # Check if CYC is supported + cyc=$(cat /sys/bus/event_source/devices/intel_pt/caps/psb_cyc) + if [ "${cyc}" != "1" ] ; then + echo "SKIP: CYC is not supported" + return 2 + fi + # Enable CYC + perf_record_no_decode -o "${perfdatafile}" -e intel_pt/cyc/u uname + # should get CYC packets + cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x") + if [ "${cyc_cnt}" = "0" ] ; then + echo "Failed to get CYC packet" + return 1 + fi + # Without CYC + perf_record_no_decode -o "${perfdatafile}" -e intel_pt//u uname + # Should not get CYC packets + cyc_cnt=$(perf script -i "${perfdatafile}" -D 2>/dev/null | grep -c "CYC 0x") + if [ "${cyc_cnt}" -gt 0 ] ; then + echo "Still get CYC packet without cyc" + return 1 + fi + echo OK + return 0 +} + +test_sample() +{ + echo "--- Test recording with sample mode ---" + # Check if recording with sample mode is working + if ! perf_record_no_decode -o "${perfdatafile}" --aux-sample=8192 -e '{intel_pt//u,branch-misses:u}' uname ; then + echo "perf record failed with --aux-sample" + return 1 + fi + echo OK + return 0 +} + +test_kernel_trace() +{ + echo "--- Test with kernel trace ---" + # Check if recording with kernel trace is working + can_kernel || return 2 + if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt//k -m1,128 uname ; then + echo "perf record failed with intel_pt//k" + return 1 + fi + echo OK + return 0 +} + +test_virtual_lbr() +{ + echo "--- Test virtual LBR ---" + + # Python script to determine the maximum size of branch stacks + cat << "_end_of_file_" > "${maxbrstack}" +from __future__ import print_function + +bmax = 0 + +def process_event(param_dict): + if "brstack" in param_dict: + brstack = param_dict["brstack"] + n = len(brstack) + global bmax + if n > bmax: + bmax = n + +def trace_end(): + print("max brstack", bmax) +_end_of_file_ + + # Check if virtual lbr is working + perf_record_no_bpf -o "${perfdatafile}" --aux-sample -e '{intel_pt//,cycles}:u' uname + times_val=$(perf script -i "${perfdatafile}" --itrace=L -s "${maxbrstack}" 2>/dev/null | grep "max brstack " | cut -d " " -f 3) + case "${times_val}" in + [0-9]*) ;; + *) times_val=0;; + esac + if [ "${times_val}" -lt 2 ] ; then + echo "Failed with virtual lbr" + return 1 + fi + echo OK + return 0 +} + +test_power_event() +{ + echo "--- Test power events ---" + # Check if power events are supported + power_event=$(cat /sys/bus/event_source/devices/intel_pt/caps/power_event_trace) + if [ "${power_event}" != "1" ] ; then + echo "SKIP: power_event_trace is not supported" + return 2 + fi + if ! perf_record_no_decode -o "${perfdatafile}" -a -e intel_pt/pwr_evt/u uname ; then + echo "perf record failed with pwr_evt" + return 1 + fi + echo OK + return 0 +} + +test_no_tnt() +{ + echo "--- Test with TNT packets disabled ---" + # Check if TNT disable is supported + notnt=$(cat /sys/bus/event_source/devices/intel_pt/caps/tnt_disable) + if [ "${notnt}" != "1" ] ; then + echo "SKIP: tnt_disable is not supported" + return 2 + fi + perf_record_no_decode -o "${perfdatafile}" -e intel_pt/notnt/u uname + # Should be no TNT packets + tnt_cnt=$(perf script -i "${perfdatafile}" -D | grep -c TNT) + if [ "${tnt_cnt}" -ne 0 ] ; then + echo "TNT packets still there after notnt" + return 1 + fi + echo OK + return 0 +} + +test_event_trace() +{ + echo "--- Test with event_trace ---" + # Check if event_trace is supported + event_trace=$(cat /sys/bus/event_source/devices/intel_pt/caps/event_trace) + if [ "${event_trace}" != 1 ] ; then + echo "SKIP: event_trace is not supported" + return 2 + fi + if ! perf_record_no_decode -o "${perfdatafile}" -e intel_pt/event/u uname ; then + echo "perf record failed with event trace" + return 1 + fi + echo OK + return 0 +} + count_result() { - if [ $1 -eq 2 ] ; then - skip_cnt=`expr ${skip_cnt} \+ 1` + if [ "$1" -eq 2 ] ; then + skip_cnt=$((skip_cnt + 1)) return fi - if [ $1 -eq 0 ] ; then - ok_cnt=`expr ${ok_cnt} \+ 1` + if [ "$1" -eq 0 ] ; then + ok_cnt=$((ok_cnt + 1)) return fi - err_cnt=`expr ${err_cnt} \+ 1` + err_cnt=$((err_cnt + 1)) } -test_system_wide_side_band +ret=0 +test_system_wide_side_band || ret=$? ; count_result $ret ; ret=0 +test_per_thread "" "" || ret=$? ; count_result $ret ; ret=0 +test_per_thread "k" "(incl. kernel) " || ret=$? ; count_result $ret ; ret=0 +test_jitdump || ret=$? ; count_result $ret ; ret=0 +test_packet_filter || ret=$? ; count_result $ret ; ret=0 +test_disable_branch || ret=$? ; count_result $ret ; ret=0 +test_time_cyc || ret=$? ; count_result $ret ; ret=0 +test_sample || ret=$? ; count_result $ret ; ret=0 +test_kernel_trace || ret=$? ; count_result $ret ; ret=0 +test_virtual_lbr || ret=$? ; count_result $ret ; ret=0 +test_power_event || ret=$? ; count_result $ret ; ret=0 +test_no_tnt || ret=$? ; count_result $ret ; ret=0 +test_event_trace || ret=$? ; count_result $ret ; ret=0 -count_result $? +cleanup -rm -f ${tmpfile} -rm -f ${perfdatafile} +echo "--- Done ---" if [ ${err_cnt} -gt 0 ] ; then exit 1 diff --git a/tools/perf/tests/shell/test_java_symbol.sh b/tools/perf/tests/shell/test_java_symbol.sh new file mode 100755 index 0000000000000000000000000000000000000000..f221225808a38939079bdb9cb9524b26cba6971a --- /dev/null +++ b/tools/perf/tests/shell/test_java_symbol.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Test java symbol + +# SPDX-License-Identifier: GPL-2.0 +# Leo Yan <leo.yan@linaro.org>, 2022 + +# skip if there's no jshell +if ! [ -x "$(command -v jshell)" ]; then + echo "skip: no jshell, install JDK" + exit 2 +fi + +PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +PERF_INJ_DATA=$(mktemp /tmp/__perf_test.perf.data.inj.XXXXX) + +cleanup_files() +{ + echo "Cleaning up files..." + rm -f ${PERF_DATA} + rm -f ${PERF_INJ_DATA} +} + +trap cleanup_files exit term int + +if [ -e "$PWD/tools/perf/libperf-jvmti.so" ]; then + LIBJVMTI=$PWD/tools/perf/libperf-jvmti.so +elif [ -e "$PWD/libperf-jvmti.so" ]; then + LIBJVMTI=$PWD/libperf-jvmti.so +elif [ -e "$PREFIX/lib64/libperf-jvmti.so" ]; then + LIBJVMTI=$PREFIX/lib64/libperf-jvmti.so +elif [ -e "$PREFIX/lib/libperf-jvmti.so" ]; then + LIBJVMTI=$PREFIX/lib/libperf-jvmti.so +elif [ -e "/usr/lib/linux-tools-$(uname -a | awk '{ print $3 }' | sed -r 's/-generic//')/libperf-jvmti.so" ]; then + LIBJVMTI=/usr/lib/linux-tools-$(uname -a | awk '{ print $3 }' | sed -r 's/-generic//')/libperf-jvmti.so +else + echo "Fail to find libperf-jvmti.so" + # JVMTI is a build option, skip the test if fail to find lib + exit 2 +fi + +cat <<EOF | perf record -k 1 -o $PERF_DATA jshell -s -J-agentpath:$LIBJVMTI +int fib(int x) { + return x > 1 ? fib(x - 2) + fib(x - 1) : 1; +} + +int q = 0; + +for (int i = 0; i < 10; i++) + q += fib(i); + +System.out.println(q); +EOF + +if [ $? -ne 0 ]; then + echo "Fail to record for java program" + exit 1 +fi + +if ! perf inject -i $PERF_DATA -o $PERF_INJ_DATA -j; then + echo "Fail to inject samples" + exit 1 +fi + +# Below is an example of the instruction samples reporting: +# 8.18% jshell jitted-50116-29.so [.] Interpreter +# 0.75% Thread-1 jitted-83602-1670.so [.] jdk.internal.jimage.BasicImageReader.getString(int) +perf report --stdio -i ${PERF_INJ_DATA} 2>&1 | \ + egrep " +[0-9]+\.[0-9]+% .* (Interpreter|jdk\.internal).*" > /dev/null 2>&1 + +if [ $? -ne 0 ]; then + echo "Fail to find java symbols" + exit 1 +fi + +exit 0 diff --git a/tools/perf/tests/sigtrap.c b/tools/perf/tests/sigtrap.c index e32ece90e164a2ab38c5bd484351f1e6f0965e3f..1de7478ec1894d7799d723f09f9384937710bf40 100644 --- a/tools/perf/tests/sigtrap.c +++ b/tools/perf/tests/sigtrap.c @@ -54,6 +54,63 @@ static struct perf_event_attr make_event_attr(void) return attr; } +#ifdef HAVE_BPF_SKEL +#include <bpf/btf.h> + +static bool attr_has_sigtrap(void) +{ + bool ret = false; + struct btf *btf; + const struct btf_type *t; + const struct btf_member *m; + const char *name; + int i, id; + + btf = btf__load_vmlinux_btf(); + if (btf == NULL) { + /* should be an old kernel */ + return false; + } + + id = btf__find_by_name_kind(btf, "perf_event_attr", BTF_KIND_STRUCT); + if (id < 0) + goto out; + + t = btf__type_by_id(btf, id); + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + name = btf__name_by_offset(btf, m->name_off); + if (!strcmp(name, "sigtrap")) { + ret = true; + break; + } + } +out: + btf__free(btf); + return ret; +} +#else /* !HAVE_BPF_SKEL */ +static bool attr_has_sigtrap(void) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_DUMMY, + .size = sizeof(attr), + .remove_on_exec = 1, /* Required by sigtrap. */ + .sigtrap = 1, /* Request synchronous SIGTRAP on event. */ + }; + int fd; + bool ret = false; + + fd = sys_perf_event_open(&attr, 0, -1, -1, perf_event_open_cloexec_flag()); + if (fd >= 0) { + ret = true; + close(fd); + } + + return ret; +} +#endif /* HAVE_BPF_SKEL */ + static void sigtrap_handler(int signum __maybe_unused, siginfo_t *info, void *ucontext __maybe_unused) { @@ -139,7 +196,13 @@ static int test__sigtrap(struct test_suite *test __maybe_unused, int subtest __m fd = sys_perf_event_open(&attr, 0, -1, -1, perf_event_open_cloexec_flag()); if (fd < 0) { - pr_debug("FAILED sys_perf_event_open(): %s\n", str_error_r(errno, sbuf, sizeof(sbuf))); + if (attr_has_sigtrap()) { + pr_debug("FAILED sys_perf_event_open(): %s\n", + str_error_r(errno, sbuf, sizeof(sbuf))); + } else { + pr_debug("perf_event_attr doesn't have sigtrap\n"); + ret = TEST_SKIP; + } goto out_restore_sigaction; } diff --git a/tools/perf/tests/switch-tracking.c b/tools/perf/tests/switch-tracking.c index 2d46af9ef93573495fefb200292ef0b7bf22f6eb..87f565c7f650de70a944c0b1625bb9bf6e090515 100644 --- a/tools/perf/tests/switch-tracking.c +++ b/tools/perf/tests/switch-tracking.c @@ -6,6 +6,7 @@ #include <time.h> #include <stdlib.h> #include <linux/zalloc.h> +#include <linux/err.h> #include <perf/cpumap.h> #include <perf/evlist.h> #include <perf/mmap.h> @@ -398,19 +399,13 @@ static int test__switch_tracking(struct test_suite *test __maybe_unused, int sub goto out; } - err = parse_event(evlist, sched_switch); - if (err) { - pr_debug("Failed to parse event %s\n", sched_switch); + switch_evsel = evlist__add_sched_switch(evlist, true); + if (IS_ERR(switch_evsel)) { + err = PTR_ERR(switch_evsel); + pr_debug("Failed to create event %s\n", sched_switch); goto out_err; } - switch_evsel = evlist__last(evlist); - - evsel__set_sample_bit(switch_evsel, CPU); - evsel__set_sample_bit(switch_evsel, TIME); - - switch_evsel->core.system_wide = true; - switch_evsel->no_aux_samples = true; switch_evsel->immediate = true; /* Test moving an event to the front */ diff --git a/tools/perf/tests/topology.c b/tools/perf/tests/topology.c index 0b4f61b6cc6b8d8845ecafd589c7110826fd5d7e..c4630cfc80ea25337fd4ea4e9473238ab877320e 100644 --- a/tools/perf/tests/topology.c +++ b/tools/perf/tests/topology.c @@ -147,7 +147,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) TEST_ASSERT_VAL("Cpu map - Die ID doesn't match", session->header.env.cpu[perf_cpu_map__cpu(map, i).cpu].die_id == id.die); TEST_ASSERT_VAL("Cpu map - Node ID is set", id.node == -1); - TEST_ASSERT_VAL("Cpu map - Thread is set", id.thread == -1); + TEST_ASSERT_VAL("Cpu map - Thread IDX is set", id.thread_idx == -1); } // Test that core ID contains socket, die and core @@ -163,7 +163,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) TEST_ASSERT_VAL("Core map - Die ID doesn't match", session->header.env.cpu[perf_cpu_map__cpu(map, i).cpu].die_id == id.die); TEST_ASSERT_VAL("Core map - Node ID is set", id.node == -1); - TEST_ASSERT_VAL("Core map - Thread is set", id.thread == -1); + TEST_ASSERT_VAL("Core map - Thread IDX is set", id.thread_idx == -1); } // Test that die ID contains socket and die @@ -179,7 +179,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) TEST_ASSERT_VAL("Die map - Node ID is set", id.node == -1); TEST_ASSERT_VAL("Die map - Core is set", id.core == -1); TEST_ASSERT_VAL("Die map - CPU is set", id.cpu.cpu == -1); - TEST_ASSERT_VAL("Die map - Thread is set", id.thread == -1); + TEST_ASSERT_VAL("Die map - Thread IDX is set", id.thread_idx == -1); } // Test that socket ID contains only socket @@ -193,7 +193,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) TEST_ASSERT_VAL("Socket map - Die ID is set", id.die == -1); TEST_ASSERT_VAL("Socket map - Core is set", id.core == -1); TEST_ASSERT_VAL("Socket map - CPU is set", id.cpu.cpu == -1); - TEST_ASSERT_VAL("Socket map - Thread is set", id.thread == -1); + TEST_ASSERT_VAL("Socket map - Thread IDX is set", id.thread_idx == -1); } // Test that node ID contains only node @@ -205,7 +205,7 @@ static int check_cpu_topology(char *path, struct perf_cpu_map *map) TEST_ASSERT_VAL("Node map - Die ID is set", id.die == -1); TEST_ASSERT_VAL("Node map - Core is set", id.core == -1); TEST_ASSERT_VAL("Node map - CPU is set", id.cpu.cpu == -1); - TEST_ASSERT_VAL("Node map - Thread is set", id.thread == -1); + TEST_ASSERT_VAL("Node map - Thread IDX is set", id.thread_idx == -1); } perf_session__delete(session); diff --git a/tools/perf/tests/vmlinux-kallsyms.c b/tools/perf/tests/vmlinux-kallsyms.c index 4fd8d703ff19ee7076f941b1c7f354f02ff3f096..8ab035b55875c7152c70f3a23864c1f3adeb2bfd 100644 --- a/tools/perf/tests/vmlinux-kallsyms.c +++ b/tools/perf/tests/vmlinux-kallsyms.c @@ -43,10 +43,11 @@ static bool is_ignored_symbol(const char *name, char type) /* Symbol names that begin with the following are ignored.*/ static const char * const ignored_prefixes[] = { "$", /* local symbols for ARM, MIPS, etc. */ - ".LASANPC", /* s390 kasan local symbols */ + ".L", /* local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. */ "__crc_", /* modversions */ "__efistub_", /* arm64 EFI stub namespace */ - "__kvm_nvhe_", /* arm64 non-VHE KVM namespace */ + "__kvm_nvhe_$", /* arm64 local symbols in non-VHE KVM namespace */ + "__kvm_nvhe_.L", /* arm64 local symbols in non-VHE KVM namespace */ "__AArch64ADRPThunk_", /* arm64 lld */ "__ARMV5PILongThunk_", /* arm lld */ "__ARMV7PILongThunk_", diff --git a/tools/perf/tests/wp.c b/tools/perf/tests/wp.c index 9d4c45184e715daec8c78ac10189efcc6d456b1e..56455da30341b4ff597d86f71cc41f098d5a0d61 100644 --- a/tools/perf/tests/wp.c +++ b/tools/perf/tests/wp.c @@ -2,7 +2,9 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include <errno.h> #include <sys/ioctl.h> +#include <linux/compiler.h> #include <linux/hw_breakpoint.h> #include <linux/kernel.h> #include "tests.h" @@ -137,8 +139,7 @@ static int test__wp_rw(struct test_suite *test __maybe_unused, #endif } -static int test__wp_modify(struct test_suite *test __maybe_unused, - int subtest __maybe_unused) +static int test__wp_modify(struct test_suite *test __maybe_unused, int subtest __maybe_unused) { #if defined(__s390x__) return TEST_SKIP; @@ -160,6 +161,11 @@ static int test__wp_modify(struct test_suite *test __maybe_unused, new_attr.disabled = 1; ret = ioctl(fd, PERF_EVENT_IOC_MODIFY_ATTRIBUTES, &new_attr); if (ret < 0) { + if (errno == ENOTTY) { + test->test_cases[subtest].skip_reason = "missing kernel support"; + ret = TEST_SKIP; + } + pr_debug("ioctl(PERF_EVENT_IOC_MODIFY_ATTRIBUTES) failed\n"); close(fd); return ret; diff --git a/tools/perf/ui/browser.c b/tools/perf/ui/browser.c index fa5bd5c20e96b011320d0b0dfdf99c095c98c4d2..78fb01d6ad63f9bf277ff40be41782fa566840f2 100644 --- a/tools/perf/ui/browser.c +++ b/tools/perf/ui/browser.c @@ -268,9 +268,9 @@ void __ui_browser__show_title(struct ui_browser *browser, const char *title) void ui_browser__show_title(struct ui_browser *browser, const char *title) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); __ui_browser__show_title(browser, title); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } int ui_browser__show(struct ui_browser *browser, const char *title, @@ -284,7 +284,7 @@ int ui_browser__show(struct ui_browser *browser, const char *title, browser->refresh_dimensions(browser); - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); __ui_browser__show_title(browser, title); browser->title = title; @@ -295,16 +295,16 @@ int ui_browser__show(struct ui_browser *browser, const char *title, va_end(ap); if (err > 0) ui_helpline__push(browser->helpline); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); return err ? 0 : -1; } void ui_browser__hide(struct ui_browser *browser) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); ui_helpline__pop(); zfree(&browser->helpline); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } static void ui_browser__scrollbar_set(struct ui_browser *browser) @@ -352,9 +352,9 @@ static int __ui_browser__refresh(struct ui_browser *browser) int ui_browser__refresh(struct ui_browser *browser) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); __ui_browser__refresh(browser); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); return 0; } @@ -390,10 +390,10 @@ int ui_browser__run(struct ui_browser *browser, int delay_secs) while (1) { off_t offset; - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); err = __ui_browser__refresh(browser); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); if (err < 0) break; diff --git a/tools/perf/ui/browsers/annotate.c b/tools/perf/ui/browsers/annotate.c index 44ba900828f6ce2679e49fc2a1d6241af609a57e..c03fa76c02ffe4d92bbe3e8d0a88497516251b1e 100644 --- a/tools/perf/ui/browsers/annotate.c +++ b/tools/perf/ui/browsers/annotate.c @@ -8,22 +8,17 @@ #include "../../util/hist.h" #include "../../util/sort.h" #include "../../util/map.h" +#include "../../util/mutex.h" #include "../../util/symbol.h" #include "../../util/evsel.h" #include "../../util/evlist.h" #include <inttypes.h> -#include <pthread.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/zalloc.h> #include <sys/ttydefaults.h> #include <asm/bug.h> -struct disasm_line_samples { - double percent; - struct sym_hist_entry he; -}; - struct arch; struct annotate_browser { @@ -319,7 +314,7 @@ static void annotate_browser__calc_percent(struct annotate_browser *browser, browser->entries = RB_ROOT; - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); symbol__calc_percent(sym, evsel); @@ -348,7 +343,7 @@ static void annotate_browser__calc_percent(struct annotate_browser *browser, } disasm_rb_tree__insert(browser, &pos->al); } - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); browser->curr_hot = rb_last(&browser->entries); } @@ -474,10 +469,10 @@ static bool annotate_browser__callq(struct annotate_browser *browser, } notes = symbol__annotation(dl->ops.target.sym); - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); if (!symbol__hists(dl->ops.target.sym, evsel->evlist->core.nr_entries)) { - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); ui__warning("Not enough memory for annotating '%s' symbol!\n", dl->ops.target.sym->name); return true; @@ -486,7 +481,7 @@ static bool annotate_browser__callq(struct annotate_browser *browser, target_ms.maps = ms->maps; target_ms.map = ms->map; target_ms.sym = dl->ops.target.sym; - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); symbol__tui_annotate(&target_ms, evsel, hbt, browser->opts); sym_title(ms->sym, ms->map, title, sizeof(title), browser->opts->percent_type); ui_browser__show_title(&browser->b, title); @@ -805,7 +800,8 @@ static int annotate_browser__run(struct annotate_browser *browser, "r Run available scripts\n" "p Toggle percent type [local/global]\n" "b Toggle percent base [period/hits]\n" - "? Search string backwards\n"); + "? Search string backwards\n" + "f Toggle showing offsets to full address\n"); continue; case 'r': script_browse(NULL, NULL); @@ -912,6 +908,9 @@ show_sup_ins: hists__scnprintf_title(hists, title, sizeof(title)); annotate_browser__show(&browser->b, title, help); continue; + case 'f': + annotation__toggle_full_addr(notes, ms); + continue; case K_LEFT: case K_ESC: case 'q': diff --git a/tools/perf/ui/setup.c b/tools/perf/ui/setup.c index 700335cde618061c55163af693c99722babcb687..25ded88801a3d1ac16fb1a79b131fd2cf741fd8c 100644 --- a/tools/perf/ui/setup.c +++ b/tools/perf/ui/setup.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0 -#include <pthread.h> #include <dlfcn.h> #include <unistd.h> @@ -8,7 +7,7 @@ #include "../util/hist.h" #include "ui.h" -pthread_mutex_t ui__lock = PTHREAD_MUTEX_INITIALIZER; +struct mutex ui__lock; void *perf_gtk_handle; int use_browser = -1; @@ -76,6 +75,7 @@ int stdio__config_color(const struct option *opt __maybe_unused, void setup_browser(bool fallback_to_pager) { + mutex_init(&ui__lock); if (use_browser < 2 && (!isatty(1) || dump_trace)) use_browser = 0; @@ -118,4 +118,5 @@ void exit_browser(bool wait_for_ok) default: break; } + mutex_destroy(&ui__lock); } diff --git a/tools/perf/ui/tui/helpline.c b/tools/perf/ui/tui/helpline.c index 298d6af82fddd5d47fa27cf7217caab44368f4d5..db4952f5990bd6854d62131fe381631ca9e6e5cf 100644 --- a/tools/perf/ui/tui/helpline.c +++ b/tools/perf/ui/tui/helpline.c @@ -2,7 +2,6 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <pthread.h> #include <linux/kernel.h> #include <linux/string.h> @@ -33,7 +32,7 @@ static int tui_helpline__show(const char *format, va_list ap) int ret; static int backlog; - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); ret = vscnprintf(ui_helpline__last_msg + backlog, sizeof(ui_helpline__last_msg) - backlog, format, ap); backlog += ret; @@ -45,7 +44,7 @@ static int tui_helpline__show(const char *format, va_list ap) SLsmg_refresh(); backlog = 0; } - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); return ret; } diff --git a/tools/perf/ui/tui/progress.c b/tools/perf/ui/tui/progress.c index 3d74af5a7ece6f3a52ebbaa21ce15d1dbfc7f127..71b6c8d9474fb6c04646897dad6e54e108d03783 100644 --- a/tools/perf/ui/tui/progress.c +++ b/tools/perf/ui/tui/progress.c @@ -45,7 +45,7 @@ static void tui_progress__update(struct ui_progress *p) } ui__refresh_dimensions(false); - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); y = SLtt_Screen_Rows / 2 - 2; SLsmg_set_color(0); SLsmg_draw_box(y, 0, 3, SLtt_Screen_Cols); @@ -56,7 +56,7 @@ static void tui_progress__update(struct ui_progress *p) bar = ((SLtt_Screen_Cols - 2) * p->curr) / p->total; SLsmg_fill_region(y, 1, 1, bar, ' '); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } static void tui_progress__finish(void) @@ -67,12 +67,12 @@ static void tui_progress__finish(void) return; ui__refresh_dimensions(false); - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); y = SLtt_Screen_Rows / 2 - 2; SLsmg_set_color(0); SLsmg_fill_region(y, 0, 3, SLtt_Screen_Cols, ' '); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } static struct ui_progress_ops tui_progress__ops = { diff --git a/tools/perf/ui/tui/setup.c b/tools/perf/ui/tui/setup.c index b1be59b4e2a4f600212d33b95fd77ec954c4516f..a3b8c397c24d5bd5f609ad1f5bb49dcfe96a2426 100644 --- a/tools/perf/ui/tui/setup.c +++ b/tools/perf/ui/tui/setup.c @@ -29,10 +29,10 @@ void ui__refresh_dimensions(bool force) { if (force || ui__need_resize) { ui__need_resize = 0; - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); SLtt_get_screen_size(); SLsmg_reinit_smg(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } } @@ -170,10 +170,10 @@ void ui__exit(bool wait_for_ok) "Press any key...", 0); SLtt_set_cursor_visibility(1); - if (!pthread_mutex_trylock(&ui__lock)) { + if (mutex_trylock(&ui__lock)) { SLsmg_refresh(); SLsmg_reset_smg(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } SLang_reset_tty(); perf_error__unregister(&perf_tui_eops); diff --git a/tools/perf/ui/tui/util.c b/tools/perf/ui/tui/util.c index 0f562e2cb1e881181976ad1e1bbad77aa939c580..3c5174854ac8b2b46106dda46aaa0f57556777d5 100644 --- a/tools/perf/ui/tui/util.c +++ b/tools/perf/ui/tui/util.c @@ -95,7 +95,7 @@ int ui_browser__input_window(const char *title, const char *text, char *input, t = sep + 1; } - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); max_len += 2; nr_lines += 8; @@ -125,17 +125,17 @@ int ui_browser__input_window(const char *title, const char *text, char *input, SLsmg_write_nstring((char *)exit_msg, max_len); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); x += 2; len = 0; key = ui__getch(delay_secs); while (key != K_TIMER && key != K_ENTER && key != K_ESC) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); if (key == K_BKSPC) { if (len == 0) { - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); goto next_key; } SLsmg_gotorc(y, x + --len); @@ -147,7 +147,7 @@ int ui_browser__input_window(const char *title, const char *text, char *input, } SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); /* XXX more graceful overflow handling needed */ if (len == sizeof(buf) - 1) { @@ -215,19 +215,19 @@ void __ui__info_window(const char *title, const char *text, const char *exit_msg void ui__info_window(const char *title, const char *text) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); __ui__info_window(title, text, NULL); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); } int ui__question_window(const char *title, const char *text, const char *exit_msg, int delay_secs) { - pthread_mutex_lock(&ui__lock); + mutex_lock(&ui__lock); __ui__info_window(title, text, exit_msg); SLsmg_refresh(); - pthread_mutex_unlock(&ui__lock); + mutex_unlock(&ui__lock); return ui__getch(delay_secs); } diff --git a/tools/perf/ui/ui.h b/tools/perf/ui/ui.h index 9b6fdf06e1d2f99052eeac9a5d59e496bf7a5f54..99f8d2fe9bc5584c9b4b302a9edc41228d1327fe 100644 --- a/tools/perf/ui/ui.h +++ b/tools/perf/ui/ui.h @@ -2,11 +2,11 @@ #ifndef _PERF_UI_H_ #define _PERF_UI_H_ 1 -#include <pthread.h> +#include "../util/mutex.h" #include <stdbool.h> #include <linux/compiler.h> -extern pthread_mutex_t ui__lock; +extern struct mutex ui__lock; extern void *perf_gtk_handle; extern int use_browser; diff --git a/tools/perf/util/Build b/tools/perf/util/Build index 9dfae1bda9cc3ef747d52a09a00c7ab3676b0899..e315ecaec32330b63118a8d0687e9d7e0ba6d5ef 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -118,6 +118,8 @@ perf-$(CONFIG_AUXTRACE) += intel-pt.o perf-$(CONFIG_AUXTRACE) += intel-bts.o perf-$(CONFIG_AUXTRACE) += arm-spe.o perf-$(CONFIG_AUXTRACE) += arm-spe-decoder/ +perf-$(CONFIG_AUXTRACE) += hisi-ptt.o +perf-$(CONFIG_AUXTRACE) += hisi-ptt-decoder/ perf-$(CONFIG_AUXTRACE) += s390-cpumsf.o ifdef CONFIG_LIBOPENCSD @@ -143,6 +145,7 @@ perf-y += branch.o perf-y += mem2node.o perf-y += clockid.o perf-y += list_sort.o +perf-y += mutex.o perf-$(CONFIG_LIBBPF) += bpf-loader.o perf-$(CONFIG_LIBBPF) += bpf_map.o @@ -269,7 +272,7 @@ CFLAGS_expr-flex.o += $(flex_flags) bison_flags := -DYYENABLE_NLS=0 BISON_GE_35 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\)/\1\2/g') \>\= 35) ifeq ($(BISON_GE_35),1) - bison_flags += -Wno-unused-parameter -Wno-nested-externs -Wno-implicit-function-declaration -Wno-switch-enum + bison_flags += -Wno-unused-parameter -Wno-nested-externs -Wno-implicit-function-declaration -Wno-switch-enum -Wno-unused-but-set-variable -Wno-unknown-warning-option else bison_flags += -w endif diff --git a/tools/perf/util/PERF-VERSION-GEN b/tools/perf/util/PERF-VERSION-GEN index 0ee5af529238a174b26645042405d1dfcaf4f65d..3cc42821d9b397e379a007f98a110bf385a03034 100755 --- a/tools/perf/util/PERF-VERSION-GEN +++ b/tools/perf/util/PERF-VERSION-GEN @@ -11,7 +11,8 @@ LF=' ' # -# Always try first to get the version from the kernel Makefile +# Use version from kernel Makefile unless not in a git repository and +# PERF-VERSION-FILE exists # CID= TAG= @@ -19,9 +20,14 @@ if test -d ../../.git -o -f ../../.git then TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) CID=$(git log -1 --abbrev=12 --pretty=format:"%h" 2>/dev/null) && CID="-g$CID" -else +elif test -f ../../PERF-VERSION-FILE +then TAG=$(cut -d' ' -f3 ../../PERF-VERSION-FILE | sed -e 's/\"//g') fi +if test -z "$TAG" +then + TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) +fi VN="$TAG$CID" if test -n "$CID" diff --git a/tools/perf/util/affinity.c b/tools/perf/util/affinity.c index 4d216c0dc4259b9ef812b41a793f911f74e44e9f..4ee96b3c755b73d5bdddaad55ae290254329cec1 100644 --- a/tools/perf/util/affinity.c +++ b/tools/perf/util/affinity.c @@ -49,8 +49,14 @@ void affinity__set(struct affinity *a, int cpu) { int cpu_set_size = get_cpu_set_size(); - if (cpu == -1) + /* + * Return: + * - if cpu is -1 + * - restrict out of bound access to sched_cpus + */ + if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) return; + a->changed = true; set_bit(cpu, a->sched_cpus); /* diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index 2c6a485c3de5d9ebf5be2ade40ced867cf6ac7d1..db475e44f42fae8d87e4e18b0f8afefb0c383526 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -35,7 +35,6 @@ #include "arch/common.h" #include "namespaces.h" #include <regex.h> -#include <pthread.h> #include <linux/bitops.h> #include <linux/kernel.h> #include <linux/string.h> @@ -821,7 +820,7 @@ void symbol__annotate_zero_histograms(struct symbol *sym) { struct annotation *notes = symbol__annotation(sym); - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); if (notes->src != NULL) { memset(notes->src->histograms, 0, notes->src->nr_histograms * notes->src->sizeof_sym_hist); @@ -829,7 +828,7 @@ void symbol__annotate_zero_histograms(struct symbol *sym) memset(notes->src->cycles_hist, 0, symbol__size(sym) * sizeof(struct cyc_hist)); } - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); } static int __symbol__account_cycles(struct cyc_hist *ch, @@ -1086,7 +1085,7 @@ void annotation__compute_ipc(struct annotation *notes, size_t size) notes->hit_insn = 0; notes->cover_insn = 0; - pthread_mutex_lock(¬es->lock); + mutex_lock(¬es->lock); for (offset = size - 1; offset >= 0; --offset) { struct cyc_hist *ch; @@ -1105,7 +1104,7 @@ void annotation__compute_ipc(struct annotation *notes, size_t size) notes->have_cycles = true; } } - pthread_mutex_unlock(¬es->lock); + mutex_unlock(¬es->lock); } int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample, @@ -1258,13 +1257,13 @@ int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool r void annotation__init(struct annotation *notes) { - pthread_mutex_init(¬es->lock, NULL); + mutex_init(¬es->lock); } void annotation__exit(struct annotation *notes) { annotated_source__delete(notes->src); - pthread_mutex_destroy(¬es->lock); + mutex_destroy(¬es->lock); } static void annotation_line__add(struct annotation_line *al, struct list_head *head) @@ -1698,6 +1697,7 @@ fallback: */ __symbol__join_symfs(filename, filename_size, dso->long_name); + mutex_lock(&dso->lock); if (access(filename, R_OK) && errno == ENOENT && dso->nsinfo) { char *new_name = filename_with_chroot(dso->nsinfo->pid, filename); @@ -1706,6 +1706,7 @@ fallback: free(new_name); } } + mutex_unlock(&dso->lock); } free(build_id_path); @@ -2238,7 +2239,10 @@ int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, } args.ms = *ms; - notes->start = map__rip_2objdump(ms->map, sym->start); + if (notes->options && notes->options->full_addr) + notes->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->start = map__rip_2objdump(ms->map, ms->sym->start); return symbol__disassemble(sym, &args); } @@ -2761,6 +2765,8 @@ void annotation__update_column_widths(struct annotation *notes) { if (notes->options->use_offset) notes->widths.target = notes->widths.min_addr; + else if (notes->options->full_addr) + notes->widths.target = BITS_PER_LONG / 4; else notes->widths.target = notes->widths.max_addr; @@ -2770,6 +2776,18 @@ void annotation__update_column_widths(struct annotation *notes) notes->widths.addr += notes->widths.jumps + 1; } +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms) +{ + notes->options->full_addr = !notes->options->full_addr; + + if (notes->options->full_addr) + notes->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->start = map__rip_2objdump(ms->map, ms->sym->start); + + annotation__update_column_widths(notes); +} + static void annotation__calc_lines(struct annotation *notes, struct map *map, struct rb_root *root, struct annotation_options *opts) diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h index 986f2bbe4870ae552f80221e2685c879570e82ae..8934072c39e6beefffff0c30048b3bf9962c7d78 100644 --- a/tools/perf/util/annotate.h +++ b/tools/perf/util/annotate.h @@ -8,9 +8,9 @@ #include <linux/types.h> #include <linux/list.h> #include <linux/rbtree.h> -#include <pthread.h> #include <asm/bug.h> #include "symbol_conf.h" +#include "mutex.h" #include "spark.h" struct hist_browser_timer; @@ -88,7 +88,8 @@ struct annotation_options { show_nr_jumps, show_minmax_cycle, show_asm_raw, - annotate_src; + annotate_src, + full_addr; u8 offset_level; int min_pcnt; int max_lines; @@ -273,7 +274,7 @@ struct annotated_source { }; struct annotation { - pthread_mutex_t lock; + struct mutex lock; u64 max_coverage; u64 start; u64 hit_cycles; @@ -325,6 +326,7 @@ void annotation__compute_ipc(struct annotation *notes, size_t size); void annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym); void annotation__update_column_widths(struct annotation *notes); void annotation__init_column_widths(struct annotation *notes, struct symbol *sym); +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms); static inline struct sym_hist *annotated_source__histogram(struct annotated_source *src, int idx) { diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 22dcfe07e886f905dd5eb4805923af3f4b08f210..906476a839e1f589bd414be19328e5b4d07dccda 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -498,7 +498,7 @@ static void arm_spe__synth_data_source_generic(const struct arm_spe_record *reco static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr) { union perf_mem_data_src data_src = { 0 }; - bool is_neoverse = is_midr_in_range(midr, neoverse_spe); + bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe); if (record->op == ARM_SPE_LD) data_src.mem_op = PERF_MEM_OP_LOAD; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 6edab8a16de6af1e0d27f4475947d2c563687369..60d8beb662aa3eaaa38f493633a84d0a0bf0f615 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -26,6 +26,7 @@ #include <linux/list.h> #include <linux/zalloc.h> +#include "config.h" #include "evlist.h" #include "dso.h" #include "map.h" @@ -51,6 +52,7 @@ #include "intel-pt.h" #include "intel-bts.h" #include "arm-spe.h" +#include "hisi-ptt.h" #include "s390-cpumsf.h" #include "util/mmap.h" @@ -1319,6 +1321,9 @@ int perf_event__process_auxtrace_info(struct perf_session *session, case PERF_AUXTRACE_S390_CPUMSF: err = s390_cpumsf_process_auxtrace_info(event, session); break; + case PERF_AUXTRACE_HISI_PTT: + err = hisi_ptt_process_auxtrace_info(event, session); + break; case PERF_AUXTRACE_UNKNOWN: default: return -EINVAL; @@ -1434,6 +1439,16 @@ static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *m } } +#define ITRACE_DFLT_LOG_ON_ERROR_SZ 16384 + +static unsigned int itrace_log_on_error_size(void) +{ + unsigned int sz = 0; + + perf_config_scan("itrace.debug-log-buffer-size", "%u", &sz); + return sz ?: ITRACE_DFLT_LOG_ON_ERROR_SZ; +} + /* * Please check tools/perf/Documentation/perf-script.txt for information * about the options parsed here, which is introduced after this cset, @@ -1532,6 +1547,8 @@ int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, if (get_flags(&p, &synth_opts->log_plus_flags, &synth_opts->log_minus_flags)) goto out_err; + if (synth_opts->log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR) + synth_opts->log_on_error_size = itrace_log_on_error_size(); break; case 'c': synth_opts->branches = true; diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h index 6a4fbfd34c6ba41333a40c9eb670965634925ef0..6a0f9b98f059b62b3790cb899b18e487fe35c2fa 100644 --- a/tools/perf/util/auxtrace.h +++ b/tools/perf/util/auxtrace.h @@ -48,6 +48,7 @@ enum auxtrace_type { PERF_AUXTRACE_CS_ETM, PERF_AUXTRACE_ARM_SPE, PERF_AUXTRACE_S390_CPUMSF, + PERF_AUXTRACE_HISI_PTT, }; enum itrace_period_type { @@ -60,6 +61,7 @@ enum itrace_period_type { #define AUXTRACE_ERR_FLG_DATA_LOST (1 << ('l' - 'a')) #define AUXTRACE_LOG_FLG_ALL_PERF_EVTS (1 << ('a' - 'a')) +#define AUXTRACE_LOG_FLG_ON_ERROR (1 << ('e' - 'a')) #define AUXTRACE_LOG_FLG_USE_STDOUT (1 << ('o' - 'a')) /** @@ -110,6 +112,7 @@ enum itrace_period_type { * @log_plus_flags: flags to affect what is logged * @log_minus_flags: flags to affect what is logged * @quick: quicker (less detailed) decoding + * @log_on_error_size: size of log to keep for outputting log only on errors */ struct itrace_synth_opts { bool set; @@ -155,6 +158,7 @@ struct itrace_synth_opts { unsigned int log_plus_flags; unsigned int log_minus_flags; unsigned int quick; + unsigned int log_on_error_size; }; /** diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h index 144a8a24cc695c3e9326860442a328f84749d624..1bcbd4fb6c669d76065255ad196addaa37cf85aa 100644 --- a/tools/perf/util/bpf-event.h +++ b/tools/perf/util/bpf-event.h @@ -4,7 +4,6 @@ #include <linux/compiler.h> #include <linux/rbtree.h> -#include <pthread.h> #include <api/fd/array.h> #include <stdio.h> diff --git a/tools/perf/util/bpf-loader.c b/tools/perf/util/bpf-loader.c index e2052f4fed33babd9602fc92edcc99cd7a1cfc31..d657594894cf6d1a225965e7476c7feb6af57cb7 100644 --- a/tools/perf/util/bpf-loader.c +++ b/tools/perf/util/bpf-loader.c @@ -27,7 +27,11 @@ #include "util.h" #include "llvm-utils.h" #include "c++/clang-c.h" -#include "hashmap.h" +#ifdef HAVE_LIBBPF_SUPPORT +#include <bpf/hashmap.h> +#else +#include "util/hashmap.h" +#endif #include "asm/bug.h" #include <internal/xyarray.h> diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c index 63b9db657442510a6e8a4cc69266ad4aa4b49bde..3c2df7522f6fcbdc63a918a6d322cd2c3afbcf56 100644 --- a/tools/perf/util/bpf_counter_cgroup.c +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -95,7 +95,7 @@ static int bperf_load_program(struct evlist *evlist) perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, - FD(cgrp_switch, cpu.cpu)); + FD(cgrp_switch, i)); if (IS_ERR(link)) { pr_err("Failed to attach cgroup program\n"); err = PTR_ERR(link); @@ -115,15 +115,15 @@ static int bperf_load_program(struct evlist *evlist) evsel->cgrp = NULL; /* open single copy of the events w/o cgroup */ - err = evsel__open_per_cpu(evsel, evlist->core.all_cpus, -1); + err = evsel__open_per_cpu(evsel, evsel->core.cpus, -1); if (err) { pr_err("Failed to open first cgroup events\n"); goto out; } map_fd = bpf_map__fd(skel->maps.events); - perf_cpu_map__for_each_cpu(cpu, j, evlist->core.all_cpus) { - int fd = FD(evsel, cpu.cpu); + perf_cpu_map__for_each_cpu(cpu, j, evsel->core.cpus) { + int fd = FD(evsel, j); __u32 idx = evsel->core.idx * total_cpus + cpu.cpu; err = bpf_map_update_elem(map_fd, &idx, &fd, @@ -269,7 +269,7 @@ static int bperf_cgrp__read(struct evsel *evsel) goto out; } - perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { + perf_cpu_map__for_each_cpu(cpu, i, evsel->core.cpus) { counts = perf_counts(evsel->counts, i, 0); counts->val = values[cpu.cpu].counter; counts->ena = values[cpu.cpu].enabled; diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c index c591a66733ef5bdf834f1cf85e0430ddd246986d..fc4d613cb979ab364659753265409319ebb8f79a 100644 --- a/tools/perf/util/bpf_lock_contention.c +++ b/tools/perf/util/bpf_lock_contention.c @@ -8,17 +8,13 @@ #include "util/thread_map.h" #include "util/lock-contention.h" #include <linux/zalloc.h> +#include <linux/string.h> #include <bpf/bpf.h> #include "bpf_skel/lock_contention.skel.h" static struct lock_contention_bpf *skel; -/* should be same as bpf_skel/lock_contention.bpf.c */ -struct lock_contention_key { - s32 stack_id; -}; - struct lock_contention_data { u64 total_time; u64 min_time; @@ -40,6 +36,7 @@ int lock_contention_prepare(struct lock_contention *con) return -1; } + bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64)); bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries); bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries); @@ -91,6 +88,8 @@ int lock_contention_prepare(struct lock_contention *con) bpf_map_update_elem(fd, &pid, &val, BPF_ANY); } + skel->bss->stack_skip = con->stack_skip; + lock_contention_bpf__attach(skel); return 0; } @@ -114,7 +113,7 @@ int lock_contention_read(struct lock_contention *con) struct lock_contention_data data; struct lock_stat *st; struct machine *machine = con->machine; - u64 stack_trace[CONTENTION_STACK_DEPTH]; + u64 stack_trace[con->max_stack]; fd = bpf_map__fd(skel->maps.lock_stat); stack = bpf_map__fd(skel->maps.stacks); @@ -125,7 +124,7 @@ int lock_contention_read(struct lock_contention *con) while (!bpf_map_get_next_key(fd, &prev_key, &key)) { struct map *kmap; struct symbol *sym; - int idx; + int idx = 0; bpf_map_lookup_elem(fd, &key, &data); st = zalloc(sizeof(*st)); @@ -144,10 +143,9 @@ int lock_contention_read(struct lock_contention *con) bpf_map_lookup_elem(stack, &key, stack_trace); - /* skip BPF + lock internal functions */ - idx = CONTENTION_STACK_SKIP; + /* skip lock internal functions */ while (is_lock_function(machine, stack_trace[idx]) && - idx < CONTENTION_STACK_DEPTH - 1) + idx < con->max_stack - 1) idx++; st->addr = stack_trace[idx]; @@ -171,6 +169,14 @@ int lock_contention_read(struct lock_contention *con) return -1; } + if (verbose) { + st->callstack = memdup(stack_trace, sizeof(stack_trace)); + if (st->callstack == NULL) { + free(st); + return -1; + } + } + hlist_add_head(&st->hash_entry, con->result); prev_key = key; } diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c index 292c430768b525afdde468eca9b6ecea40955517..6a438e0102c5a2cbc724ca70cbc3a4e2dd96a474 100644 --- a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c +++ b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c @@ -43,11 +43,39 @@ struct { __uint(value_size, sizeof(struct bpf_perf_event_value)); } cgrp_readings SEC(".maps"); +/* new kernel cgroup definition */ +struct cgroup___new { + int level; + struct cgroup *ancestors[]; +} __attribute__((preserve_access_index)); + +/* old kernel cgroup definition */ +struct cgroup___old { + int level; + u64 ancestor_ids[]; +} __attribute__((preserve_access_index)); + const volatile __u32 num_events = 1; const volatile __u32 num_cpus = 1; int enabled = 0; int use_cgroup_v2 = 0; +int perf_subsys_id = -1; + +static inline __u64 get_cgroup_v1_ancestor_id(struct cgroup *cgrp, int level) +{ + /* recast pointer to capture new type for compiler */ + struct cgroup___new *cgrp_new = (void *)cgrp; + + if (bpf_core_field_exists(cgrp_new->ancestors)) { + return BPF_CORE_READ(cgrp_new, ancestors[level], kn, id); + } else { + /* recast pointer to capture old type for compiler */ + struct cgroup___old *cgrp_old = (void *)cgrp; + + return BPF_CORE_READ(cgrp_old, ancestor_ids[level]); + } +} static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) { @@ -58,7 +86,15 @@ static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) int level; int cnt; - cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_event_cgrp_id], cgroup); + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_subsys_id], cgroup); level = BPF_CORE_READ(cgrp, level); for (cnt = 0; i < MAX_LEVELS; i++) { @@ -68,7 +104,7 @@ static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) break; // convert cgroup-id to a map index - cgrp_id = BPF_CORE_READ(cgrp, ancestor_ids[i]); + cgrp_id = get_cgroup_v1_ancestor_id(cgrp, i); elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); if (!elem) continue; @@ -176,7 +212,7 @@ static int bperf_cgroup_count(void) } // This will be attached to cgroup-switches event for each cpu -SEC("perf_events") +SEC("perf_event") int BPF_PROG(on_cgrp_switch) { return bperf_cgroup_count(); diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c index 9e8b94eb632046f492acbcf12cc21ade64abbe2f..1bb8628e7c9f0c57663a1acebad958d793d1f007 100644 --- a/tools/perf/util/bpf_skel/lock_contention.bpf.c +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -72,9 +72,10 @@ struct { int enabled; int has_cpu; int has_task; +int stack_skip; /* error stat */ -unsigned long lost; +int lost; static inline int can_record(void) { @@ -117,7 +118,7 @@ int contention_begin(u64 *ctx) pelem->timestamp = bpf_ktime_get_ns(); pelem->lock = (__u64)ctx[0]; pelem->flags = (__u32)ctx[1]; - pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP); + pelem->stack_id = bpf_get_stackid(ctx, &stacks, BPF_F_FAST_STACK_CMP | stack_skip); if (pelem->stack_id < 0) lost++; diff --git a/tools/perf/util/bpf_skel/off_cpu.bpf.c b/tools/perf/util/bpf_skel/off_cpu.bpf.c index c4ba2bcf179f44c1069825a54e7c01b3c6ffcb9c..38e3b287dbb225885f24d31acebff75d24e5dab1 100644 --- a/tools/perf/util/bpf_skel/off_cpu.bpf.c +++ b/tools/perf/util/bpf_skel/off_cpu.bpf.c @@ -94,6 +94,8 @@ const volatile bool has_prev_state = false; const volatile bool needs_cgroup = false; const volatile bool uses_cgroup_v1 = false; +int perf_subsys_id = -1; + /* * Old kernel used to call it task_struct->state and now it's '__state'. * Use BPF CO-RE "ignored suffix rule" to deal with it like below: @@ -119,11 +121,19 @@ static inline __u64 get_cgroup_id(struct task_struct *t) { struct cgroup *cgrp; - if (uses_cgroup_v1) - cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_event_cgrp_id], cgroup); - else - cgrp = BPF_CORE_READ(t, cgroups, dfl_cgrp); + if (!uses_cgroup_v1) + return BPF_CORE_READ(t, cgroups, dfl_cgrp, kn, id); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_subsys_id], cgroup); return BPF_CORE_READ(cgrp, kn, id); } diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c index a9a909db8cc7fd9f6aa37426b53945db32e9721f..6d38238481d327d00b5e13fa6171f387e4d50d16 100644 --- a/tools/perf/util/branch.c +++ b/tools/perf/util/branch.c @@ -21,7 +21,10 @@ void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, if (flags->type == PERF_BR_UNKNOWN || from == 0) return; - st->counts[flags->type]++; + if (flags->type == PERF_BR_EXTEND_ABI) + st->new_counts[flags->new_type]++; + else + st->counts[flags->type]++; if (flags->type == PERF_BR_COND) { if (to > from) @@ -36,6 +39,38 @@ void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, st->cross_4k++; } +const char *branch_new_type_name(int new_type) +{ + const char *branch_new_names[PERF_BR_NEW_MAX] = { + "FAULT_ALGN", + "FAULT_DATA", + "FAULT_INST", +/* + * TODO: This switch should happen on 'session->header.env.arch' + * instead, because an arm64 platform perf recording could be + * opened for analysis on other platforms as well. + */ +#ifdef __aarch64__ + "ARM64_FIQ", + "ARM64_DEBUG_HALT", + "ARM64_DEBUG_EXIT", + "ARM64_DEBUG_INST", + "ARM64_DEBUG_DATA" +#else + "ARCH_1", + "ARCH_2", + "ARCH_3", + "ARCH_4", + "ARCH_5" +#endif + }; + + if (new_type >= 0 && new_type < PERF_BR_NEW_MAX) + return branch_new_names[new_type]; + + return NULL; +} + const char *branch_type_name(int type) { const char *branch_names[PERF_BR_MAX] = { @@ -51,7 +86,10 @@ const char *branch_type_name(int type) "COND_CALL", "COND_RET", "ERET", - "IRQ" + "IRQ", + "SERROR", + "NO_TX", + "", // Needed for PERF_BR_EXTEND_ABI that ends up triggering some compiler warnings about NULL deref }; if (type >= 0 && type < PERF_BR_MAX) @@ -60,6 +98,17 @@ const char *branch_type_name(int type) return NULL; } +const char *get_branch_type(struct branch_entry *e) +{ + if (e->flags.type == PERF_BR_UNKNOWN) + return ""; + + if (e->flags.type == PERF_BR_EXTEND_ABI) + return branch_new_type_name(e->flags.new_type); + + return branch_type_name(e->flags.type); +} + void branch_type_stat_display(FILE *fp, struct branch_type_stat *st) { u64 total = 0; @@ -106,6 +155,15 @@ void branch_type_stat_display(FILE *fp, struct branch_type_stat *st) 100.0 * (double)st->counts[i] / (double)total); } + + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + fprintf(fp, "\n%8s: %5.1f%%", + branch_new_type_name(i), + 100.0 * + (double)st->new_counts[i] / (double)total); + } + } static int count_str_scnprintf(int idx, const char *str, char *bf, int size) @@ -121,6 +179,9 @@ int branch_type_str(struct branch_type_stat *st, char *bf, int size) for (i = 0; i < PERF_BR_MAX; i++) total += st->counts[i]; + for (i = 0; i < PERF_BR_NEW_MAX; i++) + total += st->new_counts[i]; + if (total == 0) return 0; @@ -138,6 +199,11 @@ int branch_type_str(struct branch_type_stat *st, char *bf, int size) printed += count_str_scnprintf(j++, branch_type_name(i), bf + printed, size - printed); } + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + printed += count_str_scnprintf(j++, branch_new_type_name(i), bf + printed, size - printed); + } + if (st->cross_4k > 0) printed += count_str_scnprintf(j++, "CROSS_4K", bf + printed, size - printed); diff --git a/tools/perf/util/branch.h b/tools/perf/util/branch.h index 17b2ccc61094bf264bf7e663784ad2b78598f7ca..f838b23db18046256d35db5c826cca793b4b5b78 100644 --- a/tools/perf/util/branch.h +++ b/tools/perf/util/branch.h @@ -24,7 +24,9 @@ struct branch_flags { u64 abort:1; u64 cycles:16; u64 type:4; - u64 reserved:40; + u64 new_type:4; + u64 priv:3; + u64 reserved:33; }; }; }; @@ -72,6 +74,7 @@ static inline struct branch_entry *perf_sample__branch_entries(struct perf_sampl struct branch_type_stat { bool branch_to; u64 counts[PERF_BR_MAX]; + u64 new_counts[PERF_BR_NEW_MAX]; u64 cond_fwd; u64 cond_bwd; u64 cross_4k; @@ -82,6 +85,8 @@ void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, u64 from, u64 to); const char *branch_type_name(int type); +const char *branch_new_type_name(int new_type); +const char *get_branch_type(struct branch_entry *e); void branch_type_stat_display(FILE *fp, struct branch_type_stat *st); int branch_type_str(struct branch_type_stat *st, char *bf, int bfsize); diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c index ec18ed5caf3ece94fa84a4fec15124c10c1ad226..a839b30c981b7b07bcbd2993970bea98131938b8 100644 --- a/tools/perf/util/build-id.c +++ b/tools/perf/util/build-id.c @@ -898,11 +898,15 @@ static int filename__read_build_id_ns(const char *filename, static bool dso__build_id_mismatch(struct dso *dso, const char *name) { struct build_id bid; + bool ret = false; - if (filename__read_build_id_ns(name, &bid, dso->nsinfo) < 0) - return false; + mutex_lock(&dso->lock); + if (filename__read_build_id_ns(name, &bid, dso->nsinfo) >= 0) + ret = !dso__build_id_equal(dso, &bid); - return !dso__build_id_equal(dso, &bid); + mutex_unlock(&dso->lock); + + return ret; } static int dso__cache_build_id(struct dso *dso, struct machine *machine, @@ -941,8 +945,10 @@ static int dso__cache_build_id(struct dso *dso, struct machine *machine, if (!is_kallsyms && dso__build_id_mismatch(dso, name)) goto out_free; + mutex_lock(&dso->lock); ret = build_id_cache__add_b(&dso->bid, name, dso->nsinfo, is_kallsyms, is_vdso, proper_name, root_dir); + mutex_unlock(&dso->lock); out_free: free(allocated_name); return ret; diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c index 7e663673f79f96351acd14e50d1004969873db6f..a093a15f048fa84e227bbc72464b462592007a4b 100644 --- a/tools/perf/util/callchain.c +++ b/tools/perf/util/callchain.c @@ -1307,24 +1307,16 @@ int callchain_branch_counts(struct callchain_root *root, static int count_pri64_printf(int idx, const char *str, u64 value, char *bf, int bfsize) { - int printed; - - printed = scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value); - - return printed; + return scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value); } static int count_float_printf(int idx, const char *str, float value, char *bf, int bfsize, float threshold) { - int printed; - if (threshold != 0.0 && value < threshold) return 0; - printed = scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value); - - return printed; + return scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value); } static int branch_to_str(char *bf, int bfsize, diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c index 60ce5908c6640022c472acc6d0611195a4c93284..3f2ae19a1dd40372128d5f5fcb0c51e3371653a4 100644 --- a/tools/perf/util/config.c +++ b/tools/perf/util/config.c @@ -908,3 +908,34 @@ void set_buildid_dir(const char *dir) /* for communicating with external commands */ setenv("PERF_BUILDID_DIR", buildid_dir, 1); } + +struct perf_config_scan_data { + const char *name; + const char *fmt; + va_list args; + int ret; +}; + +static int perf_config_scan_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->ret = vsscanf(value, d->fmt, d->args); + + return 0; +} + +int perf_config_scan(const char *name, const char *fmt, ...) +{ + struct perf_config_scan_data d = { + .name = name, + .fmt = fmt, + }; + + va_start(d.args, fmt); + perf_config(perf_config_scan_cb, &d); + va_end(d.args); + + return d.ret; +} diff --git a/tools/perf/util/config.h b/tools/perf/util/config.h index 2fd77aaff4d249d3d21aa196cd2db10420778953..2e5e808928a55a8bc8f9aeab7ae19e33df2a4aaf 100644 --- a/tools/perf/util/config.h +++ b/tools/perf/util/config.h @@ -29,6 +29,7 @@ typedef int (*config_fn_t)(const char *, const char *, void *); int perf_default_config(const char *, const char *, void *); int perf_config(config_fn_t fn, void *); +int perf_config_scan(const char *name, const char *fmt, ...) __scanf(2, 3); int perf_config_set(struct perf_config_set *set, config_fn_t fn, void *data); int perf_config_int(int *dest, const char *, const char *); diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c index ae43fb88f444e89004f65126e30801877d0ffa33..8486ca3bec75ff22d664cdd68336f73549fdfef1 100644 --- a/tools/perf/util/cpumap.c +++ b/tools/perf/util/cpumap.c @@ -112,12 +112,39 @@ static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_ } +static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) +{ + struct perf_cpu_map *map; + unsigned int i = 0; + + map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - + data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); + if (!map) + return NULL; + + if (data->range_cpu_data.any_cpu) + map->map[i++].cpu = -1; + + for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; + i++, cpu++) + map->map[i].cpu = cpu; + + return map; +} + struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) { - if (data->type == PERF_CPU_MAP__CPUS) + switch (data->type) { + case PERF_CPU_MAP__CPUS: return cpu_map__from_entries(data); - else + case PERF_CPU_MAP__MASK: return cpu_map__from_mask(data); + case PERF_CPU_MAP__RANGE_CPUS: + return cpu_map__from_range(data); + default: + pr_err("cpu_map__new_data unknown type %d\n", data->type); + return NULL; + } } size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) @@ -202,7 +229,7 @@ static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) else if (a->core != b->core) return a->core - b->core; else - return a->thread - b->thread; + return a->thread_idx - b->thread_idx; } struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, @@ -640,7 +667,7 @@ const struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) { - return a->thread == b->thread && + return a->thread_idx == b->thread_idx && a->node == b->node && a->socket == b->socket && a->die == b->die && @@ -650,7 +677,7 @@ bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) { - return a->thread == -1 && + return a->thread_idx == -1 && a->node == -1 && a->socket == -1 && a->die == -1 && @@ -661,7 +688,7 @@ bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) struct aggr_cpu_id aggr_cpu_id__empty(void) { struct aggr_cpu_id ret = { - .thread = -1, + .thread_idx = -1, .node = -1, .socket = -1, .die = -1, diff --git a/tools/perf/util/cpumap.h b/tools/perf/util/cpumap.h index fa8a5acdcae128c2d829a80f2d92d4755d4f5cc3..4a6d029576eeb0a16190a2080a168dbe7608b302 100644 --- a/tools/perf/util/cpumap.h +++ b/tools/perf/util/cpumap.h @@ -10,7 +10,7 @@ /** Identify where counts are aggregated, -1 implies not to aggregate. */ struct aggr_cpu_id { /** A value in the range 0 to number of threads. */ - int thread; + int thread_idx; /** The numa node X as read from /sys/devices/system/node/nodeX. */ int node; /** diff --git a/tools/perf/util/cputopo.c b/tools/perf/util/cputopo.c index d275d843c1550dd27e25e3a6af0e219b929f2849..1a3ff64491581d51a9f61d233224446ffa7044c5 100644 --- a/tools/perf/util/cputopo.c +++ b/tools/perf/util/cputopo.c @@ -157,6 +157,67 @@ void cpu_topology__delete(struct cpu_topology *tp) free(tp); } +bool cpu_topology__smt_on(const struct cpu_topology *topology) +{ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *cpu_list = topology->core_cpus_list[i]; + + /* + * If there is a need to separate siblings in a core then SMT is + * enabled. + */ + if (strchr(cpu_list, ',') || strchr(cpu_list, '-')) + return true; + } + return false; +} + +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list) +{ + struct perf_cpu_map *user_requested_cpus; + + /* + * If user_requested_cpu_list is empty then all CPUs are recorded and so + * core_wide is true. + */ + if (!user_requested_cpu_list) + return true; + + user_requested_cpus = perf_cpu_map__new(user_requested_cpu_list); + /* Check that every user requested CPU is the complete set of SMT threads on a core. */ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *core_cpu_list = topology->core_cpus_list[i]; + struct perf_cpu_map *core_cpus = perf_cpu_map__new(core_cpu_list); + struct perf_cpu cpu; + int idx; + bool has_first, first = true; + + perf_cpu_map__for_each_cpu(cpu, idx, core_cpus) { + if (first) { + has_first = perf_cpu_map__has(user_requested_cpus, cpu); + first = false; + } else { + /* + * If the first core CPU is user requested then + * all subsequent CPUs in the core must be user + * requested too. If the first CPU isn't user + * requested then none of the others must be + * too. + */ + if (perf_cpu_map__has(user_requested_cpus, cpu) != has_first) { + perf_cpu_map__put(core_cpus); + perf_cpu_map__put(user_requested_cpus); + return false; + } + } + } + perf_cpu_map__put(core_cpus); + } + perf_cpu_map__put(user_requested_cpus); + return true; +} + static bool has_die_topology(void) { char filename[MAXPATHLEN]; diff --git a/tools/perf/util/cputopo.h b/tools/perf/util/cputopo.h index 854e18f9041e8eb72ae2189366f3a0114c6b859c..969e5920a00e4d2e0c145e01ea9d78e4624ce4f4 100644 --- a/tools/perf/util/cputopo.h +++ b/tools/perf/util/cputopo.h @@ -58,6 +58,11 @@ struct hybrid_topology { struct cpu_topology *cpu_topology__new(void); void cpu_topology__delete(struct cpu_topology *tp); +/* Determine from the core list whether SMT was enabled. */ +bool cpu_topology__smt_on(const struct cpu_topology *topology); +/* Are the sets of SMT siblings all enabled or all disabled in user_requested_cpus. */ +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list); struct numa_topology *numa_topology__new(void); void numa_topology__delete(struct numa_topology *tp); diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 5ac13958d1bde35b7337bb017f2d44487ac8054a..f1a14c0ad26d539a16a7e6bbcf064e85aa8e1d86 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -501,6 +501,7 @@ static int __open_dso(struct dso *dso, struct machine *machine) if (!name) return -ENOMEM; + mutex_lock(&dso->lock); if (machine) root_dir = machine->root_dir; @@ -541,6 +542,7 @@ static int __open_dso(struct dso *dso, struct machine *machine) unlink(name); out: + mutex_unlock(&dso->lock); free(name); return fd; } @@ -559,8 +561,11 @@ static int open_dso(struct dso *dso, struct machine *machine) int fd; struct nscookie nsc; - if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE) + if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE) { + mutex_lock(&dso->lock); nsinfo__mountns_enter(dso->nsinfo, &nsc); + mutex_unlock(&dso->lock); + } fd = __open_dso(dso, machine); if (dso->binary_type != DSO_BINARY_TYPE__BUILD_ID_CACHE) nsinfo__mountns_exit(&nsc); @@ -795,7 +800,7 @@ dso_cache__free(struct dso *dso) struct rb_root *root = &dso->data.cache; struct rb_node *next = rb_first(root); - pthread_mutex_lock(&dso->lock); + mutex_lock(&dso->lock); while (next) { struct dso_cache *cache; @@ -804,7 +809,7 @@ dso_cache__free(struct dso *dso) rb_erase(&cache->rb_node, root); free(cache); } - pthread_mutex_unlock(&dso->lock); + mutex_unlock(&dso->lock); } static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset) @@ -841,7 +846,7 @@ dso_cache__insert(struct dso *dso, struct dso_cache *new) struct dso_cache *cache; u64 offset = new->offset; - pthread_mutex_lock(&dso->lock); + mutex_lock(&dso->lock); while (*p != NULL) { u64 end; @@ -862,7 +867,7 @@ dso_cache__insert(struct dso *dso, struct dso_cache *new) cache = NULL; out: - pthread_mutex_unlock(&dso->lock); + mutex_unlock(&dso->lock); return cache; } @@ -1297,7 +1302,7 @@ struct dso *dso__new_id(const char *name, struct dso_id *id) dso->root = NULL; INIT_LIST_HEAD(&dso->node); INIT_LIST_HEAD(&dso->data.open_entry); - pthread_mutex_init(&dso->lock, NULL); + mutex_init(&dso->lock); refcount_set(&dso->refcnt, 1); } @@ -1336,7 +1341,7 @@ void dso__delete(struct dso *dso) dso__free_a2l(dso); zfree(&dso->symsrc_filename); nsinfo__zput(dso->nsinfo); - pthread_mutex_destroy(&dso->lock); + mutex_destroy(&dso->lock); free(dso); } diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index 66981c7a9a18d6577a3ad9436711e1996c8f3194..58d94175e7148049b5a7a4f5346ed2cbe8b594cb 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -2,7 +2,6 @@ #ifndef __PERF_DSO #define __PERF_DSO -#include <pthread.h> #include <linux/refcount.h> #include <linux/types.h> #include <linux/rbtree.h> @@ -11,6 +10,7 @@ #include <stdio.h> #include <linux/bitops.h> #include "build-id.h" +#include "mutex.h" struct machine; struct map; @@ -145,7 +145,7 @@ struct dso_cache { struct auxtrace_cache; struct dso { - pthread_mutex_t lock; + struct mutex lock; struct list_head node; struct rb_node rb_node; /* rbtree node sorted by long name */ struct rb_root *root; /* root of rbtree that rb_node is in */ diff --git a/tools/perf/util/events_stats.h b/tools/perf/util/events_stats.h index 040ab9d0a8037ded3ea8e56ce1dd8da38956e7ef..8fecc9fbaecc49ca929934d07449811e69d9649e 100644 --- a/tools/perf/util/events_stats.h +++ b/tools/perf/util/events_stats.h @@ -47,6 +47,7 @@ struct hists_stats { u64 total_non_filtered_period; u32 nr_samples; u32 nr_non_filtered_samples; + u32 nr_lost_samples; }; void events_stats__inc(struct events_stats *stats, u32 type); diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 48167f3941a65a2a32d9bdafda09630a69e59d48..6612b00949e7081454fe8cfd56e45f621df6fd4c 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -15,6 +15,7 @@ #include "target.h" #include "evlist.h" #include "evsel.h" +#include "record.h" #include "debug.h" #include "units.h" #include "bpf_counter.h" @@ -40,12 +41,14 @@ #include <sys/ioctl.h> #include <sys/mman.h> #include <sys/prctl.h> +#include <sys/timerfd.h> #include <linux/bitops.h> #include <linux/hash.h> #include <linux/log2.h> #include <linux/err.h> #include <linux/string.h> +#include <linux/time64.h> #include <linux/zalloc.h> #include <perf/evlist.h> #include <perf/evsel.h> @@ -147,6 +150,7 @@ static void evlist__purge(struct evlist *evlist) void evlist__exit(struct evlist *evlist) { + event_enable_timer__exit(&evlist->eet); zfree(&evlist->mmap); zfree(&evlist->overwrite_mmap); perf_evlist__exit(&evlist->core); @@ -264,28 +268,6 @@ int evlist__add_dummy(struct evlist *evlist) return 0; } -static void evlist__add_on_all_cpus(struct evlist *evlist, struct evsel *evsel) -{ - evsel->core.system_wide = true; - - /* - * All CPUs. - * - * Note perf_event_open() does not accept CPUs that are not online, so - * in fact this CPU list will include only all online CPUs. - */ - perf_cpu_map__put(evsel->core.own_cpus); - evsel->core.own_cpus = perf_cpu_map__new(NULL); - perf_cpu_map__put(evsel->core.cpus); - evsel->core.cpus = perf_cpu_map__get(evsel->core.own_cpus); - - /* No threads */ - perf_thread_map__put(evsel->core.threads); - evsel->core.threads = perf_thread_map__new_dummy(); - - evlist__add(evlist, evsel); -} - struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide) { struct evsel *evsel = evlist__dummy_event(evlist); @@ -298,17 +280,31 @@ struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide) evsel->core.attr.exclude_hv = 1; evsel->core.attr.freq = 0; evsel->core.attr.sample_period = 1; + evsel->core.system_wide = system_wide; evsel->no_aux_samples = true; evsel->name = strdup("dummy:u"); - if (system_wide) - evlist__add_on_all_cpus(evlist, evsel); - else - evlist__add(evlist, evsel); - + evlist__add(evlist, evsel); return evsel; } +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel = evsel__newtp_idx("sched", "sched_switch", 0); + + if (IS_ERR(evsel)) + return evsel; + + evsel__set_sample_bit(evsel, CPU); + evsel__set_sample_bit(evsel, TIME); + + evsel->core.system_wide = system_wide; + evsel->no_aux_samples = true; + + evlist__add(evlist, evsel); + return evsel; +}; + int evlist__add_attrs(struct evlist *evlist, struct perf_event_attr *attrs, size_t nr_attrs) { struct evsel *evsel, *n; @@ -480,7 +476,7 @@ static int evlist__is_enabled(struct evlist *evlist) return false; } -static void __evlist__disable(struct evlist *evlist, char *evsel_name) +static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { struct evsel *pos; struct evlist_cpu_iterator evlist_cpu_itr; @@ -502,6 +498,8 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name) continue; if (pos->disabled || !evsel__is_group_leader(pos) || !pos->core.fd) continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; if (pos->immediate) has_imm = true; if (pos->immediate != imm) @@ -518,6 +516,8 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name) continue; if (!evsel__is_group_leader(pos) || !pos->core.fd) continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; pos->disabled = true; } @@ -533,15 +533,20 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name) void evlist__disable(struct evlist *evlist) { - __evlist__disable(evlist, NULL); + __evlist__disable(evlist, NULL, false); +} + +void evlist__disable_non_dummy(struct evlist *evlist) +{ + __evlist__disable(evlist, NULL, true); } void evlist__disable_evsel(struct evlist *evlist, char *evsel_name) { - __evlist__disable(evlist, evsel_name); + __evlist__disable(evlist, evsel_name, false); } -static void __evlist__enable(struct evlist *evlist, char *evsel_name) +static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy) { struct evsel *pos; struct evlist_cpu_iterator evlist_cpu_itr; @@ -560,6 +565,8 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name) continue; if (!evsel__is_group_leader(pos) || !pos->core.fd) continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; evsel__enable_cpu(pos, evlist_cpu_itr.cpu_map_idx); } affinity__cleanup(affinity); @@ -568,6 +575,8 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name) continue; if (!evsel__is_group_leader(pos) || !pos->core.fd) continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; pos->disabled = false; } @@ -581,12 +590,17 @@ static void __evlist__enable(struct evlist *evlist, char *evsel_name) void evlist__enable(struct evlist *evlist) { - __evlist__enable(evlist, NULL); + __evlist__enable(evlist, NULL, false); +} + +void evlist__enable_non_dummy(struct evlist *evlist) +{ + __evlist__enable(evlist, NULL, true); } void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) { - __evlist__enable(evlist, evsel_name); + __evlist__enable(evlist, evsel_name, false); } void evlist__toggle_enable(struct evlist *evlist) @@ -608,7 +622,8 @@ int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) { return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, - fdarray_flag__nonfilterable); + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event); } #endif @@ -1897,7 +1912,8 @@ int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) } evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, - fdarray_flag__nonfilterable); + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event); if (evlist->ctl_fd.pos < 0) { evlist->ctl_fd.pos = -1; pr_err("Failed to add ctl fd entry: %m\n"); @@ -2147,20 +2163,234 @@ int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) return err; } -int evlist__ctlfd_update(struct evlist *evlist, struct pollfd *update) +/** + * struct event_enable_time - perf record -D/--delay single time range. + * @start: start of time range to enable events in milliseconds + * @end: end of time range to enable events in milliseconds + * + * N.B. this structure is also accessed as an array of int. + */ +struct event_enable_time { + int start; + int end; +}; + +static int parse_event_enable_time(const char *str, struct event_enable_time *range, bool first) { - int ctlfd_pos = evlist->ctl_fd.pos; - struct pollfd *entries = evlist->core.pollfd.entries; + const char *fmt = first ? "%u - %u %n" : " , %u - %u %n"; + int ret, start, end, n; - if (!evlist__ctlfd_initialized(evlist)) + ret = sscanf(str, fmt, &start, &end, &n); + if (ret != 2 || end <= start) + return -EINVAL; + if (range) { + range->start = start; + range->end = end; + } + return n; +} + +static ssize_t parse_event_enable_times(const char *str, struct event_enable_time *range) +{ + int incr = !!range; + bool first = true; + ssize_t ret, cnt; + + for (cnt = 0; *str; cnt++) { + ret = parse_event_enable_time(str, range, first); + if (ret < 0) + return ret; + /* Check no overlap */ + if (!first && range && range->start <= range[-1].end) + return -EINVAL; + str += ret; + range += incr; + first = false; + } + return cnt; +} + +/** + * struct event_enable_timer - control structure for perf record -D/--delay. + * @evlist: event list + * @times: time ranges that events are enabled (N.B. this is also accessed as an + * array of int) + * @times_cnt: number of time ranges + * @timerfd: timer file descriptor + * @pollfd_pos: position in @evlist array of file descriptors to poll (fdarray) + * @times_step: current position in (int *)@times)[], + * refer event_enable_timer__process() + * + * Note, this structure is only used when there are time ranges, not when there + * is only an initial delay. + */ +struct event_enable_timer { + struct evlist *evlist; + struct event_enable_time *times; + size_t times_cnt; + int timerfd; + int pollfd_pos; + size_t times_step; +}; + +static int str_to_delay(const char *str) +{ + char *endptr; + long d; + + d = strtol(str, &endptr, 10); + if (*endptr || d > INT_MAX || d < -1) return 0; + return d; +} - if (entries[ctlfd_pos].fd != update->fd || - entries[ctlfd_pos].events != update->events) - return -1; +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset) +{ + enum fdarray_flags flags = fdarray_flag__nonfilterable | fdarray_flag__non_perf_event; + struct event_enable_timer *eet; + ssize_t times_cnt; + ssize_t ret; + int err; + + if (unset) + return 0; + + opts->initial_delay = str_to_delay(str); + if (opts->initial_delay) + return 0; + + ret = parse_event_enable_times(str, NULL); + if (ret < 0) + return ret; + + times_cnt = ret; + if (times_cnt == 0) + return -EINVAL; + + eet = zalloc(sizeof(*eet)); + if (!eet) + return -ENOMEM; + + eet->times = calloc(times_cnt, sizeof(*eet->times)); + if (!eet->times) { + err = -ENOMEM; + goto free_eet; + } + + if (parse_event_enable_times(str, eet->times) != times_cnt) { + err = -EINVAL; + goto free_eet_times; + } + + eet->times_cnt = times_cnt; + + eet->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + if (eet->timerfd == -1) { + err = -errno; + pr_err("timerfd_create failed: %s\n", strerror(errno)); + goto free_eet_times; + } + + eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags); + if (eet->pollfd_pos < 0) { + err = eet->pollfd_pos; + goto close_timerfd; + } + + eet->evlist = evlist; + evlist->eet = eet; + opts->initial_delay = eet->times[0].start; - entries[ctlfd_pos].revents = update->revents; return 0; + +close_timerfd: + close(eet->timerfd); +free_eet_times: + free(eet->times); +free_eet: + free(eet); + return err; +} + +static int event_enable_timer__set_timer(struct event_enable_timer *eet, int ms) +{ + struct itimerspec its = { + .it_value.tv_sec = ms / MSEC_PER_SEC, + .it_value.tv_nsec = (ms % MSEC_PER_SEC) * NSEC_PER_MSEC, + }; + int err = 0; + + if (timerfd_settime(eet->timerfd, 0, &its, NULL) < 0) { + err = -errno; + pr_err("timerfd_settime failed: %s\n", strerror(errno)); + } + return err; +} + +int event_enable_timer__start(struct event_enable_timer *eet) +{ + int ms; + + if (!eet) + return 0; + + ms = eet->times[0].end - eet->times[0].start; + eet->times_step = 1; + + return event_enable_timer__set_timer(eet, ms); +} + +int event_enable_timer__process(struct event_enable_timer *eet) +{ + struct pollfd *entries; + short revents; + + if (!eet) + return 0; + + entries = eet->evlist->core.pollfd.entries; + revents = entries[eet->pollfd_pos].revents; + entries[eet->pollfd_pos].revents = 0; + + if (revents & POLLIN) { + size_t step = eet->times_step; + size_t pos = step / 2; + + if (step & 1) { + evlist__disable_non_dummy(eet->evlist); + pr_info(EVLIST_DISABLED_MSG); + if (pos >= eet->times_cnt - 1) { + /* Disarm timer */ + event_enable_timer__set_timer(eet, 0); + return 1; /* Stop */ + } + } else { + evlist__enable_non_dummy(eet->evlist); + pr_info(EVLIST_ENABLED_MSG); + } + + step += 1; + pos = step / 2; + + if (pos < eet->times_cnt) { + int *times = (int *)eet->times; /* Accessing 'times' as array of int */ + int ms = times[step] - times[step - 1]; + + eet->times_step = step; + return event_enable_timer__set_timer(eet, ms); + } + } + + return 0; +} + +void event_enable_timer__exit(struct event_enable_timer **ep) +{ + if (!ep || !*ep) + return; + free((*ep)->times); + zfree(ep); } struct evsel *evlist__find_evsel(struct evlist *evlist, int idx) diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index 351ba2887a796b4b4653ad414fe6b4ac42774888..16734c6756b3cd79eb38002cc322190cfcceb02c 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -48,6 +48,8 @@ enum bkw_mmap_state { BKW_MMAP_EMPTY, }; +struct event_enable_timer; + struct evlist { struct perf_evlist core; bool enabled; @@ -79,6 +81,7 @@ struct evlist { int ack; /* ack file descriptor for control commands */ int pos; /* index at evlist core object to check signals */ } ctl_fd; + struct event_enable_timer *eet; }; struct evsel_str_handler { @@ -124,6 +127,7 @@ static inline struct evsel *evlist__add_dummy_on_all_cpus(struct evlist *evlist) { return evlist__add_aux_dummy(evlist, true); } +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide); int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, evsel__sb_cb_t cb, void *data); @@ -205,6 +209,8 @@ void evlist__enable(struct evlist *evlist); void evlist__toggle_enable(struct evlist *evlist); void evlist__disable_evsel(struct evlist *evlist, char *evsel_name); void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); +void evlist__disable_non_dummy(struct evlist *evlist); +void evlist__enable_non_dummy(struct evlist *evlist); void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); @@ -418,13 +424,18 @@ void evlist__close_control(int ctl_fd, int ctl_fd_ack, bool *ctl_fd_close); int evlist__initialize_ctlfd(struct evlist *evlist, int ctl_fd, int ctl_fd_ack); int evlist__finalize_ctlfd(struct evlist *evlist); bool evlist__ctlfd_initialized(struct evlist *evlist); -int evlist__ctlfd_update(struct evlist *evlist, struct pollfd *update); int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd); int evlist__ctlfd_ack(struct evlist *evlist); #define EVLIST_ENABLED_MSG "Events enabled\n" #define EVLIST_DISABLED_MSG "Events disabled\n" +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset); +int event_enable_timer__start(struct event_enable_timer *eet); +void event_enable_timer__exit(struct event_enable_timer **ep); +int event_enable_timer__process(struct event_enable_timer *eet); + struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); int evlist__scnprintf_evsels(struct evlist *evlist, size_t size, char *bf); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 18c3eb864d5587a016221de5e1dae094c7b6be9d..76605fde35078e7c32cfb87125cb8a5bb7010f87 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -46,7 +46,11 @@ #include "string2.h" #include "memswap.h" #include "util.h" -#include "hashmap.h" +#ifdef HAVE_LIBBPF_SUPPORT +#include <bpf/hashmap.h> +#else +#include "util/hashmap.h" +#endif #include "pmu-hybrid.h" #include "off_cpu.h" #include "../perf-sys.h" @@ -1157,6 +1161,7 @@ void evsel__config(struct evsel *evsel, struct record_opts *opts, attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1; attr->inherit = !opts->no_inherit; attr->write_backward = opts->overwrite ? 1 : 0; + attr->read_format = PERF_FORMAT_LOST; evsel__set_sample_bit(evsel, IP); evsel__set_sample_bit(evsel, TID); @@ -1808,7 +1813,7 @@ static struct perf_thread_map *empty_thread_map; static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, struct perf_thread_map *threads) { - int nthreads; + int nthreads = perf_thread_map__nr(threads); if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) || (perf_missing_features.aux_output && evsel->core.attr.aux_output)) @@ -1834,11 +1839,6 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, threads = empty_thread_map; } - if (evsel->core.system_wide) - nthreads = 1; - else - nthreads = threads->nr; - if (evsel->core.fd == NULL && perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) return -ENOMEM; @@ -1852,6 +1852,8 @@ static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, static void evsel__disable_missing_features(struct evsel *evsel) { + if (perf_missing_features.read_lost) + evsel->core.attr.read_format &= ~PERF_FORMAT_LOST; if (perf_missing_features.weight_struct) { evsel__set_sample_bit(evsel, WEIGHT); evsel__reset_sample_bit(evsel, WEIGHT_STRUCT); @@ -1903,7 +1905,12 @@ bool evsel__detect_missing_features(struct evsel *evsel) * Must probe features in the order they were added to the * perf_event_attr interface. */ - if (!perf_missing_features.weight_struct && + if (!perf_missing_features.read_lost && + (evsel->core.attr.read_format & PERF_FORMAT_LOST)) { + perf_missing_features.read_lost = true; + pr_debug2("switching off PERF_FORMAT_LOST support\n"); + return true; + } else if (!perf_missing_features.weight_struct && (evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) { perf_missing_features.weight_struct = true; pr_debug2("switching off weight struct support\n"); @@ -2049,10 +2056,7 @@ static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, if (threads == NULL) threads = empty_thread_map; - if (evsel->core.system_wide) - nthreads = 1; - else - nthreads = threads->nr; + nthreads = perf_thread_map__nr(threads); if (evsel->cgrp) pid = evsel->cgrp->fd; @@ -2077,6 +2081,7 @@ retry_open: test_attr__ready(); + /* Debug message used by test scripts */ pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx", pid, perf_cpu_map__cpu(cpus, idx).cpu, group_fd, evsel->open_flags); @@ -2102,6 +2107,7 @@ retry_open: fd, group_fd, evsel->open_flags); } + /* Debug message used by test scripts */ pr_debug2_peo(" = %d\n", fd); if (evsel->bpf_fd >= 0) { diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index d927713b513e4a88c788f638f239bc6a0648c79d..989865e16aadd8a7917d4589e8e150b23e56cf6d 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -188,6 +188,7 @@ struct perf_missing_features { bool data_page_size; bool code_page_size; bool weight_struct; + bool read_lost; }; extern struct perf_missing_features perf_missing_features; diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index c15a9852fa419160df31a08856cc9ac366063441..aaacf514dc09c54dc046c51acc9946ff2568cc2b 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -182,7 +182,7 @@ int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) { struct expr_id_data *data_ptr = NULL, *old_data = NULL; char *old_key = NULL; - char *name, *p; + char *name; int ret; data_ptr = zalloc(sizeof(*data_ptr)); @@ -195,15 +195,6 @@ int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) return -ENOMEM; } - /* - * The jevents tool converts all metric expressions - * to lowercase, including metric references, hence - * we need to add lowercase name for metric, so it's - * properly found. - */ - for (p = name; *p; p++) - *p = tolower(*p); - /* * Intentionally passing just const char pointers, * originally from 'struct pmu_event' object. @@ -310,7 +301,9 @@ struct expr_parse_ctx *expr__ctx_new(void) free(ctx); return NULL; } - ctx->runtime = 0; + ctx->sctx.user_requested_cpu_list = NULL; + ctx->sctx.runtime = 0; + ctx->sctx.system_wide = false; return ctx; } @@ -332,6 +325,10 @@ void expr__ctx_free(struct expr_parse_ctx *ctx) struct hashmap_entry *cur; size_t bkt; + if (!ctx) + return; + + free(ctx->sctx.user_requested_cpu_list); hashmap__for_each_entry(ctx->ids, cur, bkt) { free((char *)cur->key); free(cur->value); @@ -344,16 +341,13 @@ static int __expr__parse(double *val, struct expr_parse_ctx *ctx, const char *expr, bool compute_ids) { - struct expr_scanner_ctx scanner_ctx = { - .runtime = ctx->runtime, - }; YY_BUFFER_STATE buffer; void *scanner; int ret; pr_debug2("parsing metric: %s\n", expr); - ret = expr_lex_init_extra(&scanner_ctx, &scanner); + ret = expr_lex_init_extra(&ctx->sctx, &scanner); if (ret) return ret; @@ -410,16 +404,11 @@ double arch_get_tsc_freq(void) } #endif -double expr__get_literal(const char *literal) +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) { static struct cpu_topology *topology; double result = NAN; - if (!strcasecmp("#smt_on", literal)) { - result = smt_on() > 0 ? 1.0 : 0.0; - goto out; - } - if (!strcmp("#num_cpus", literal)) { result = cpu__max_present_cpu().cpu; goto out; @@ -443,6 +432,15 @@ double expr__get_literal(const char *literal) goto out; } } + if (!strcasecmp("#smt_on", literal)) { + result = smt_on(topology) ? 1.0 : 0.0; + goto out; + } + if (!strcmp("#core_wide", literal)) { + result = core_wide(ctx->system_wide, ctx->user_requested_cpu_list, topology) + ? 1.0 : 0.0; + goto out; + } if (!strcmp("#num_packages", literal)) { result = topology->package_cpus_lists; goto out; diff --git a/tools/perf/util/expr.h b/tools/perf/util/expr.h index bd2116983bbb52d1ef765294a5e35a2f6818cf3d..d6c1668dc1a08c38438f2a10895efdda5b3585ab 100644 --- a/tools/perf/util/expr.h +++ b/tools/perf/util/expr.h @@ -2,28 +2,27 @@ #ifndef PARSE_CTX_H #define PARSE_CTX_H 1 -// There are fixes that need to land upstream before we can use libbpf's headers, -// for now use our copy unconditionally, since the data structures at this point -// are exactly the same, no problem. -//#ifdef HAVE_LIBBPF_SUPPORT -//#include <bpf/hashmap.h> -//#else +#ifdef HAVE_LIBBPF_SUPPORT +#include <bpf/hashmap.h> +#else #include "util/hashmap.h" -//#endif +#endif struct metric_ref; +struct expr_scanner_ctx { + char *user_requested_cpu_list; + int runtime; + bool system_wide; +}; + struct expr_parse_ctx { struct hashmap *ids; - int runtime; + struct expr_scanner_ctx sctx; }; struct expr_id_data; -struct expr_scanner_ctx { - int runtime; -}; - struct hashmap *ids__new(void); void ids__free(struct hashmap *ids); int ids__insert(struct hashmap *ids, const char *id); @@ -58,6 +57,6 @@ int expr__find_ids(const char *expr, const char *one, double expr_id_data__value(const struct expr_id_data *data); double expr_id_data__source_count(const struct expr_id_data *data); -double expr__get_literal(const char *literal); +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx); #endif diff --git a/tools/perf/util/expr.l b/tools/perf/util/expr.l index 4dc8edbfd9cea223c43f39cb8cee9efb92858e73..0168a96373309b712002fa91e40d6efe061813c0 100644 --- a/tools/perf/util/expr.l +++ b/tools/perf/util/expr.l @@ -79,11 +79,11 @@ static int str(yyscan_t scanner, int token, int runtime) return token; } -static int literal(yyscan_t scanner) +static int literal(yyscan_t scanner, const struct expr_scanner_ctx *sctx) { YYSTYPE *yylval = expr_get_lval(scanner); - yylval->num = expr__get_literal(expr_get_text(scanner)); + yylval->num = expr__get_literal(expr_get_text(scanner), sctx); if (isnan(yylval->num)) return EXPR_ERROR; @@ -108,7 +108,7 @@ min { return MIN; } if { return IF; } else { return ELSE; } source_count { return SOURCE_COUNT; } -{literal} { return literal(yyscanner); } +{literal} { return literal(yyscanner, sctx); } {number} { return value(yyscanner); } {symbol} { return str(yyscanner, ID, sctx->runtime); } "|" { return '|'; } diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y index a30b825adb7ba6995f871346127784be69d3574f..635e562350c5c6079c55ae4649fbda73b2ebfaff 100644 --- a/tools/perf/util/expr.y +++ b/tools/perf/util/expr.y @@ -156,7 +156,7 @@ start: if_expr } ; -if_expr: expr IF expr ELSE expr +if_expr: expr IF expr ELSE if_expr { if (fpclassify($3.val) == FP_ZERO) { /* diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c index 953338b9e887e26f943831903e72c6df1c18927d..fefc72066c4e8ee1e85068180a752c12cfb15d23 100644 --- a/tools/perf/util/genelf.c +++ b/tools/perf/util/genelf.c @@ -30,10 +30,6 @@ #define BUILD_ID_URANDOM /* different uuid for each run */ -// FIXME, remove this and fix the deprecation warnings before its removed and -// We'll break for good here... -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - #ifdef HAVE_LIBCRYPTO_SUPPORT #define BUILD_ID_MD5 @@ -45,6 +41,7 @@ #endif #ifdef BUILD_ID_MD5 +#include <openssl/evp.h> #include <openssl/md5.h> #endif #endif @@ -142,15 +139,20 @@ gen_build_id(struct buildid_note *note, static void gen_build_id(struct buildid_note *note, unsigned long load_addr, const void *code, size_t csize) { - MD5_CTX context; + EVP_MD_CTX *mdctx; if (sizeof(note->build_id) < 16) errx(1, "build_id too small for MD5"); - MD5_Init(&context); - MD5_Update(&context, &load_addr, sizeof(load_addr)); - MD5_Update(&context, code, csize); - MD5_Final((unsigned char *)note->build_id, &context); + mdctx = EVP_MD_CTX_new(); + if (!mdctx) + errx(2, "failed to create EVP_MD_CTX"); + + EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); + EVP_DigestUpdate(mdctx, &load_addr, sizeof(load_addr)); + EVP_DigestUpdate(mdctx, code, csize); + EVP_DigestFinal_ex(mdctx, (unsigned char *)note->build_id, NULL); + EVP_MD_CTX_free(mdctx); } #endif @@ -251,6 +253,7 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, Elf_Data *d; Elf_Scn *scn; Elf_Ehdr *ehdr; + Elf_Phdr *phdr; Elf_Shdr *shdr; uint64_t eh_frame_base_offset; char *strsym = NULL; @@ -285,6 +288,19 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, ehdr->e_version = EV_CURRENT; ehdr->e_shstrndx= unwinding ? 4 : 2; /* shdr index for section name */ + /* + * setup program header + */ + phdr = elf_newphdr(e, 1); + phdr[0].p_type = PT_LOAD; + phdr[0].p_offset = 0; + phdr[0].p_vaddr = 0; + phdr[0].p_paddr = 0; + phdr[0].p_filesz = csize; + phdr[0].p_memsz = csize; + phdr[0].p_flags = PF_X | PF_R; + phdr[0].p_align = 8; + /* * setup text section */ @@ -329,6 +345,7 @@ jit_write_elf(int fd, uint64_t load_addr, const char *sym, eh_frame_base_offset); if (retval) goto error; + retval = -1; } /* diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h index ae138afe6c56345a7def862fdc9ae50771616125..6af062d1c4522c78a391bc39bbd5cb6266c1b4be 100644 --- a/tools/perf/util/genelf.h +++ b/tools/perf/util/genelf.h @@ -2,6 +2,8 @@ #ifndef __GENELF_H__ #define __GENELF_H__ +#include <linux/math.h> + /* genelf.c */ int jit_write_elf(int fd, uint64_t code_addr, const char *sym, const void *code, int csize, void *debug, int nr_debug_entries, @@ -53,8 +55,10 @@ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_ent #if GEN_ELF_CLASS == ELFCLASS64 #define elf_newehdr elf64_newehdr +#define elf_newphdr elf64_newphdr #define elf_getshdr elf64_getshdr #define Elf_Ehdr Elf64_Ehdr +#define Elf_Phdr Elf64_Phdr #define Elf_Shdr Elf64_Shdr #define Elf_Sym Elf64_Sym #define ELF_ST_TYPE(a) ELF64_ST_TYPE(a) @@ -62,8 +66,10 @@ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_ent #define ELF_ST_VIS(a) ELF64_ST_VISIBILITY(a) #else #define elf_newehdr elf32_newehdr +#define elf_newphdr elf32_newphdr #define elf_getshdr elf32_getshdr #define Elf_Ehdr Elf32_Ehdr +#define Elf_Phdr Elf32_Phdr #define Elf_Shdr Elf32_Shdr #define Elf_Sym Elf32_Sym #define ELF_ST_TYPE(a) ELF32_ST_TYPE(a) @@ -72,6 +78,6 @@ int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_ent #endif /* The .text section is directly after the ELF header */ -#define GEN_ELF_TEXT_OFFSET sizeof(Elf_Ehdr) +#define GEN_ELF_TEXT_OFFSET round_up(sizeof(Elf_Ehdr) + sizeof(Elf_Phdr), 16) #endif diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index c30c29c514105bfd4c2c108bbfd4ea2564b1aa9a..98dfaf84bd13798f69b25c87a8b67e289225a2a0 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -4295,8 +4295,6 @@ out: size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) { struct perf_record_event_update *ev = &event->event_update; - struct perf_record_event_update_scale *ev_scale; - struct perf_record_event_update_cpus *ev_cpus; struct perf_cpu_map *map; size_t ret; @@ -4304,20 +4302,18 @@ size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) switch (ev->type) { case PERF_EVENT_UPDATE__SCALE: - ev_scale = (struct perf_record_event_update_scale *)ev->data; - ret += fprintf(fp, "... scale: %f\n", ev_scale->scale); + ret += fprintf(fp, "... scale: %f\n", ev->scale.scale); break; case PERF_EVENT_UPDATE__UNIT: - ret += fprintf(fp, "... unit: %s\n", ev->data); + ret += fprintf(fp, "... unit: %s\n", ev->unit); break; case PERF_EVENT_UPDATE__NAME: - ret += fprintf(fp, "... name: %s\n", ev->data); + ret += fprintf(fp, "... name: %s\n", ev->name); break; case PERF_EVENT_UPDATE__CPUS: - ev_cpus = (struct perf_record_event_update_cpus *)ev->data; ret += fprintf(fp, "... "); - map = cpu_map__new_data(&ev_cpus->cpus); + map = cpu_map__new_data(&ev->cpus.cpus); if (map) ret += cpu_map__fprintf(map, fp); else @@ -4374,8 +4370,6 @@ int perf_event__process_event_update(struct perf_tool *tool __maybe_unused, struct evlist **pevlist) { struct perf_record_event_update *ev = &event->event_update; - struct perf_record_event_update_scale *ev_scale; - struct perf_record_event_update_cpus *ev_cpus; struct evlist *evlist; struct evsel *evsel; struct perf_cpu_map *map; @@ -4395,19 +4389,17 @@ int perf_event__process_event_update(struct perf_tool *tool __maybe_unused, switch (ev->type) { case PERF_EVENT_UPDATE__UNIT: free((char *)evsel->unit); - evsel->unit = strdup(ev->data); + evsel->unit = strdup(ev->unit); break; case PERF_EVENT_UPDATE__NAME: free(evsel->name); - evsel->name = strdup(ev->data); + evsel->name = strdup(ev->name); break; case PERF_EVENT_UPDATE__SCALE: - ev_scale = (struct perf_record_event_update_scale *)ev->data; - evsel->scale = ev_scale->scale; + evsel->scale = ev->scale.scale; break; case PERF_EVENT_UPDATE__CPUS: - ev_cpus = (struct perf_record_event_update_cpus *)ev->data; - map = cpu_map__new_data(&ev_cpus->cpus); + map = cpu_map__new_data(&ev->cpus.cpus); if (map) { perf_cpu_map__put(evsel->core.own_cpus); evsel->core.own_cpus = map; diff --git a/tools/perf/util/hisi-ptt-decoder/Build b/tools/perf/util/hisi-ptt-decoder/Build new file mode 100644 index 0000000000000000000000000000000000000000..db3db8b7503322965046204cf2bcd1800572a2c8 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/Build @@ -0,0 +1 @@ +perf-$(CONFIG_AUXTRACE) += hisi-ptt-pkt-decoder.o diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c new file mode 100644 index 0000000000000000000000000000000000000000..a17c423a526ddf798ee0cbc60c6030540e780510 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <endian.h> +#include <byteswap.h> +#include <linux/bitops.h> +#include <stdarg.h> + +#include "../color.h" +#include "hisi-ptt-pkt-decoder.h" + +/* + * For 8DW format, the bit[31:11] of DW0 is always 0x1fffff, which can be + * used to distinguish the data format. + * 8DW format is like: + * bits [ 31:11 ][ 10:0 ] + * |---------------------------------------|-------------------| + * DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + * DW1 [ Prefix ] + * DW2 [ Header DW0 ] + * DW3 [ Header DW1 ] + * DW4 [ Header DW2 ] + * DW5 [ Header DW3 ] + * DW6 [ Reserved (0x0) ] + * DW7 [ Time ] + * + * 4DW format is like: + * bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + * |-----|---------|---|---|---|---|-------------|-------------| + * DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + * DW1 [ Header DW1 ] + * DW2 [ Header DW2 ] + * DW3 [ Header DW3 ] + */ + +enum hisi_ptt_8dw_pkt_field_type { + HISI_PTT_8DW_CHK_AND_RSV0, + HISI_PTT_8DW_PREFIX, + HISI_PTT_8DW_HEAD0, + HISI_PTT_8DW_HEAD1, + HISI_PTT_8DW_HEAD2, + HISI_PTT_8DW_HEAD3, + HISI_PTT_8DW_RSV1, + HISI_PTT_8DW_TIME, + HISI_PTT_8DW_TYPE_MAX +}; + +enum hisi_ptt_4dw_pkt_field_type { + HISI_PTT_4DW_HEAD1, + HISI_PTT_4DW_HEAD2, + HISI_PTT_4DW_HEAD3, + HISI_PTT_4DW_TYPE_MAX +}; + +static const char * const hisi_ptt_8dw_pkt_field_name[] = { + [HISI_PTT_8DW_PREFIX] = "Prefix", + [HISI_PTT_8DW_HEAD0] = "Header DW0", + [HISI_PTT_8DW_HEAD1] = "Header DW1", + [HISI_PTT_8DW_HEAD2] = "Header DW2", + [HISI_PTT_8DW_HEAD3] = "Header DW3", + [HISI_PTT_8DW_TIME] = "Time" +}; + +static const char * const hisi_ptt_4dw_pkt_field_name[] = { + [HISI_PTT_4DW_HEAD1] = "Header DW1", + [HISI_PTT_4DW_HEAD2] = "Header DW2", + [HISI_PTT_4DW_HEAD3] = "Header DW3", +}; + +union hisi_ptt_4dw { + struct { + uint32_t format : 2; + uint32_t type : 5; + uint32_t t9 : 1; + uint32_t t8 : 1; + uint32_t th : 1; + uint32_t so : 1; + uint32_t len : 10; + uint32_t time : 11; + }; + uint32_t value; +}; + +static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) +{ + const char *color = PERF_COLOR_BLUE; + int i; + + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + color_fprintf(stdout, color, " %s\n", desc); +} + +static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + for (i = 0; i < HISI_PTT_8DW_TYPE_MAX; i++) { + /* Do not show 8DW check field and reserved fields */ + if (i == HISI_PTT_8DW_CHK_AND_RSV0 || i == HISI_PTT_8DW_RSV1) { + pos += HISI_PTT_FIELD_LENTH; + continue; + } + + hisi_ptt_print_pkt(buf, pos, hisi_ptt_8dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_8DW_PKT]; +} + +static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) +{ + const char *color = PERF_COLOR_BLUE; + union hisi_ptt_4dw dw0; + int i; + + dw0.value = *(uint32_t *)(buf + pos); + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + + color_fprintf(stdout, color, + " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", + "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9, + "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length", + dw0.len, "Time", dw0.time); +} + +static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + hisi_ptt_4dw_print_dw0(buf, pos); + pos += HISI_PTT_FIELD_LENTH; + + for (i = 0; i < HISI_PTT_4DW_TYPE_MAX; i++) { + hisi_ptt_print_pkt(buf, pos, hisi_ptt_4dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_4DW_PKT]; +} + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type) +{ + if (type == HISI_PTT_8DW_PKT) + return hisi_ptt_8dw_kpt_desc(buf, pos); + + return hisi_ptt_4dw_kpt_desc(buf, pos); +} diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h new file mode 100644 index 0000000000000000000000000000000000000000..e78f1b5bc836e91b861f65ad9c5f90c9270a2fb6 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__HISI_PTT_PKT_DECODER_H__ +#define INCLUDE__HISI_PTT_PKT_DECODER_H__ + +#include <stddef.h> +#include <stdint.h> + +#define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) +#define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) +#define HISI_PTT_MAX_SPACE_LEN 10 +#define HISI_PTT_FIELD_LENTH 4 + +enum hisi_ptt_pkt_type { + HISI_PTT_4DW_PKT, + HISI_PTT_8DW_PKT, + HISI_PTT_PKT_MAX +}; + +static int hisi_ptt_pkt_size[] = { + [HISI_PTT_4DW_PKT] = 16, + [HISI_PTT_8DW_PKT] = 32, +}; + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type); + +#endif diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c new file mode 100644 index 0000000000000000000000000000000000000000..45b614bb73bfa321ac66c6c63e935506748ff0be --- /dev/null +++ b/tools/perf/util/hisi-ptt.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include <byteswap.h> +#include <endian.h> +#include <errno.h> +#include <inttypes.h> +#include <linux/bitops.h> +#include <linux/kernel.h> +#include <linux/log2.h> +#include <linux/types.h> +#include <linux/zalloc.h> +#include <stdlib.h> +#include <unistd.h> + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evsel.h" +#include "hisi-ptt.h" +#include "hisi-ptt-decoder/hisi-ptt-pkt-decoder.h" +#include "machine.h" +#include "session.h" +#include "tool.h" +#include <internal/lib.h> + +struct hisi_ptt { + struct auxtrace auxtrace; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; +}; + +struct hisi_ptt_queue { + struct hisi_ptt *ptt; + struct auxtrace_buffer *buffer; +}; + +static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf) +{ + uint32_t head = *(uint32_t *)buf; + + if ((HISI_PTT_8DW_CHECK_MASK & head) == HISI_PTT_IS_8DW_PKT) + return HISI_PTT_8DW_PKT; + + return HISI_PTT_4DW_PKT; +} + +static void hisi_ptt_dump(struct hisi_ptt *ptt __maybe_unused, + unsigned char *buf, size_t len) +{ + const char *color = PERF_COLOR_BLUE; + enum hisi_ptt_pkt_type type; + size_t pos = 0; + int pkt_len; + + type = hisi_ptt_check_packet_type(buf); + len = round_down(len, hisi_ptt_pkt_size[type]); + color_fprintf(stdout, color, ". ... HISI PTT data: size %zu bytes\n", + len); + + while (len > 0) { + pkt_len = hisi_ptt_pkt_desc(buf, pos, type); + if (!pkt_len) + color_fprintf(stdout, color, " Bad packet!\n"); + + pos += pkt_len; + len -= pkt_len; + } +} + +static void hisi_ptt_dump_event(struct hisi_ptt *ptt, unsigned char *buf, + size_t len) +{ + printf(".\n"); + + hisi_ptt_dump(ptt, buf, len); +} + +static int hisi_ptt_process_event(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static int hisi_ptt_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + struct perf_tool *tool __maybe_unused) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + int fd = perf_data__fd(session->data); + int size = event->auxtrace.size; + void *data = malloc(size); + off_t data_offset; + int err; + + if (!data) + return -errno; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = readn(fd, data, size); + if (err != (ssize_t)size) { + free(data); + return -errno; + } + + if (dump_trace) + hisi_ptt_dump_event(ptt, data, size); + + return 0; +} + +static int hisi_ptt_flush(struct perf_session *session __maybe_unused, + struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static void hisi_ptt_free_events(struct perf_session *session __maybe_unused) +{ +} + +static void hisi_ptt_free(struct perf_session *session) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + + session->auxtrace = NULL; + free(ptt); +} + +static bool hisi_ptt_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, auxtrace); + + return evsel->core.attr.type == ptt->pmu_type; +} + +static void hisi_ptt_print_info(__u64 type) +{ + if (!dump_trace) + return; + + fprintf(stdout, " PMU Type %" PRId64 "\n", (s64) type); +} + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct hisi_ptt *ptt; + + if (auxtrace_info->header.size < HISI_PTT_AUXTRACE_PRIV_SIZE + + sizeof(struct perf_record_auxtrace_info)) + return -EINVAL; + + ptt = zalloc(sizeof(*ptt)); + if (!ptt) + return -ENOMEM; + + ptt->session = session; + ptt->machine = &session->machines.host; /* No kvm support */ + ptt->auxtrace_type = auxtrace_info->type; + ptt->pmu_type = auxtrace_info->priv[0]; + + ptt->auxtrace.process_event = hisi_ptt_process_event; + ptt->auxtrace.process_auxtrace_event = hisi_ptt_process_auxtrace_event; + ptt->auxtrace.flush_events = hisi_ptt_flush; + ptt->auxtrace.free_events = hisi_ptt_free_events; + ptt->auxtrace.free = hisi_ptt_free; + ptt->auxtrace.evsel_is_auxtrace = hisi_ptt_evsel_is_auxtrace; + session->auxtrace = &ptt->auxtrace; + + hisi_ptt_print_info(auxtrace_info->priv[0]); + + return 0; +} diff --git a/tools/perf/util/hisi-ptt.h b/tools/perf/util/hisi-ptt.h new file mode 100644 index 0000000000000000000000000000000000000000..2db9b405621483dc6faebbbb49643049e3f33e60 --- /dev/null +++ b/tools/perf/util/hisi-ptt.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__PERF_HISI_PTT_H__ +#define INCLUDE__PERF_HISI_PTT_H__ + +#define HISI_PTT_PMU_NAME "hisi_ptt" +#define HISI_PTT_AUXTRACE_PRIV_SIZE sizeof(u64) + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu); + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +#endif diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c index 1c085ab565340c95e67b1a9b4826e4f39f76d947..17a05e943b44b5f7234719e5e3c87f82ff58ec67 100644 --- a/tools/perf/util/hist.c +++ b/tools/perf/util/hist.c @@ -215,6 +215,7 @@ void hists__calc_col_len(struct hists *hists, struct hist_entry *h) hists__new_col_len(hists, HISTC_GLOBAL_INS_LAT, 13); hists__new_col_len(hists, HISTC_LOCAL_P_STAGE_CYC, 13); hists__new_col_len(hists, HISTC_GLOBAL_P_STAGE_CYC, 13); + hists__new_col_len(hists, HISTC_ADDR, BITS_PER_LONG / 4 + 2); if (symbol_conf.nanosecs) hists__new_col_len(hists, HISTC_TIME, 16); @@ -1622,13 +1623,13 @@ struct rb_root_cached *hists__get_rotate_entries_in(struct hists *hists) { struct rb_root_cached *root; - pthread_mutex_lock(&hists->lock); + mutex_lock(&hists->lock); root = hists->entries_in; if (++hists->entries_in > &hists->entries_in_array[1]) hists->entries_in = &hists->entries_in_array[0]; - pthread_mutex_unlock(&hists->lock); + mutex_unlock(&hists->lock); return root; } @@ -2335,6 +2336,11 @@ void hists__inc_nr_samples(struct hists *hists, bool filtered) hists->stats.nr_non_filtered_samples++; } +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost) +{ + hists->stats.nr_lost_samples += lost; +} + static struct hist_entry *hists__add_dummy_entry(struct hists *hists, struct hist_entry *pair) { @@ -2678,12 +2684,16 @@ size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp, evlist__for_each_entry(evlist, pos) { struct hists *hists = evsel__hists(pos); - if (skip_empty && !hists->stats.nr_samples) + if (skip_empty && !hists->stats.nr_samples && !hists->stats.nr_lost_samples) continue; ret += fprintf(fp, "%s stats:\n", evsel__name(pos)); - ret += fprintf(fp, "%16s events: %10d\n", - "SAMPLE", hists->stats.nr_samples); + if (hists->stats.nr_samples) + ret += fprintf(fp, "%16s events: %10d\n", + "SAMPLE", hists->stats.nr_samples); + if (hists->stats.nr_lost_samples) + ret += fprintf(fp, "%16s events: %10d\n", + "LOST_SAMPLES", hists->stats.nr_lost_samples); } return ret; @@ -2805,7 +2815,7 @@ int __hists__init(struct hists *hists, struct perf_hpp_list *hpp_list) hists->entries_in = &hists->entries_in_array[0]; hists->entries_collapsed = RB_ROOT_CACHED; hists->entries = RB_ROOT_CACHED; - pthread_mutex_init(&hists->lock, NULL); + mutex_init(&hists->lock); hists->socket_filter = -1; hists->hpp_list = hpp_list; INIT_LIST_HEAD(&hists->hpp_formats); diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index 7ed4648d2fc2fe2ccb12f903b462be1e9c194215..ebd8a8f783ee6b688aa409d8e26fac8ab117416f 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -4,10 +4,10 @@ #include <linux/rbtree.h> #include <linux/types.h> -#include <pthread.h> #include "evsel.h" #include "color.h" #include "events_stats.h" +#include "mutex.h" struct hist_entry; struct hist_entry_ops; @@ -79,6 +79,7 @@ enum hist_column { HISTC_GLOBAL_P_STAGE_CYC, HISTC_ADDR_FROM, HISTC_ADDR_TO, + HISTC_ADDR, HISTC_NR_COLS, /* Last entry */ }; @@ -98,7 +99,7 @@ struct hists { const struct dso *dso_filter; const char *uid_filter_str; const char *symbol_filter_str; - pthread_mutex_t lock; + struct mutex lock; struct hists_stats stats; u64 event_stream; u16 col_len[HISTC_NR_COLS]; @@ -201,6 +202,7 @@ void hists__reset_stats(struct hists *hists); void hists__inc_stats(struct hists *hists, struct hist_entry *h); void hists__inc_nr_events(struct hists *hists); void hists__inc_nr_samples(struct hists *hists, bool filtered); +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost); size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows, int max_cols, float min_pcnt, FILE *fp, diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-log.c b/tools/perf/util/intel-pt-decoder/intel-pt-log.c index 5f5dfc8753f3361686118df9b8793ece34dcf3e2..ef55d6232cf0c2260d6f4e2bf292b53e6597073e 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-log.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-log.c @@ -5,12 +5,16 @@ */ #include <stdio.h> +#include <stdlib.h> #include <stdint.h> #include <inttypes.h> #include <stdarg.h> #include <stdbool.h> #include <string.h> +#include <linux/zalloc.h> +#include <linux/kernel.h> + #include "intel-pt-log.h" #include "intel-pt-insn-decoder.h" @@ -18,18 +22,33 @@ #define MAX_LOG_NAME 256 +#define DFLT_BUF_SZ (16 * 1024) + +struct log_buf { + char *buf; + size_t buf_sz; + size_t head; + bool wrapped; + FILE *backend; +}; + static FILE *f; static char log_name[MAX_LOG_NAME]; bool intel_pt_enable_logging; +static bool intel_pt_dump_log_on_error; +static unsigned int intel_pt_log_on_error_size; +static struct log_buf log_buf; void *intel_pt_log_fp(void) { return f; } -void intel_pt_log_enable(void) +void intel_pt_log_enable(bool dump_log_on_error, unsigned int log_on_error_size) { intel_pt_enable_logging = true; + intel_pt_dump_log_on_error = dump_log_on_error; + intel_pt_log_on_error_size = log_on_error_size; } void intel_pt_log_disable(void) @@ -74,6 +93,100 @@ static void intel_pt_print_no_data(uint64_t pos, int indent) fprintf(f, " "); } +static ssize_t log_buf__write(void *cookie, const char *buf, size_t size) +{ + struct log_buf *b = cookie; + size_t sz = size; + + if (!b->buf) + return size; + + while (sz) { + size_t space = b->buf_sz - b->head; + size_t n = min(space, sz); + + memcpy(b->buf + b->head, buf, n); + sz -= n; + buf += n; + b->head += n; + if (sz && b->head >= b->buf_sz) { + b->head = 0; + b->wrapped = true; + } + } + return size; +} + +static int log_buf__close(void *cookie) +{ + struct log_buf *b = cookie; + + zfree(&b->buf); + return 0; +} + +static FILE *log_buf__open(struct log_buf *b, FILE *backend, unsigned int sz) +{ + cookie_io_functions_t fns = { + .write = log_buf__write, + .close = log_buf__close, + }; + FILE *file; + + memset(b, 0, sizeof(*b)); + b->buf_sz = sz; + b->buf = malloc(b->buf_sz); + b->backend = backend; + file = fopencookie(b, "a", fns); + if (!file) + zfree(&b->buf); + return file; +} + +static bool remove_first_line(const char **p, size_t *n) +{ + for (; *n && **p != '\n'; ++*p, --*n) + ; + if (*n) { + *p += 1; + *n -= 1; + return true; + } + return false; +} + +static void write_lines(const char *p, size_t n, FILE *fp, bool *remove_first) +{ + if (*remove_first) + *remove_first = !remove_first_line(&p, &n); + fwrite(p, n, 1, fp); +} + +static void log_buf__dump(struct log_buf *b) +{ + bool remove_first = false; + + if (!b->buf) + return; + + fflush(f); /* Could update b->head and b->wrapped */ + fprintf(b->backend, "Dumping debug log buffer\n"); + if (b->wrapped) { + remove_first = true; + write_lines(b->buf + b->head, b->buf_sz - b->head, b->backend, &remove_first); + } + write_lines(b->buf, b->head, b->backend, &remove_first); + fprintf(b->backend, "End of debug log buffer dump\n"); + + b->head = 0; + b->wrapped = false; +} + +void intel_pt_log_dump_buf(void) +{ + log_buf__dump(&log_buf); +} + static int intel_pt_log_open(void) { if (!intel_pt_enable_logging) @@ -86,6 +199,8 @@ static int intel_pt_log_open(void) f = fopen(log_name, "w+"); else f = stdout; + if (f && intel_pt_dump_log_on_error) + f = log_buf__open(&log_buf, f, intel_pt_log_on_error_size); if (!f) { intel_pt_enable_logging = false; return -1; diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-log.h b/tools/perf/util/intel-pt-decoder/intel-pt-log.h index d900aab24b211af539e0c018cc9c1fed1c02c88a..354d7d23fc8173d2968158886ecada2293896a04 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-log.h +++ b/tools/perf/util/intel-pt-decoder/intel-pt-log.h @@ -14,9 +14,10 @@ struct intel_pt_pkt; void *intel_pt_log_fp(void); -void intel_pt_log_enable(void); +void intel_pt_log_enable(bool dump_log_on_error, unsigned int log_on_error_size); void intel_pt_log_disable(void); void intel_pt_log_set_name(const char *name); +void intel_pt_log_dump_buf(void); void __intel_pt_log_packet(const struct intel_pt_pkt *packet, int pkt_len, uint64_t pos, const unsigned char *buf); diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index d5e9fc8106dd83b262d0c9c7e21ffa9f4238fa27..e3548ddef2545e25b787fc599c2b4a15df908331 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -842,7 +842,8 @@ static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn, offset, buf, INTEL_PT_INSN_BUF_SZ); if (len <= 0) { - intel_pt_log("ERROR: failed to read at %" PRIu64 " ", offset); + intel_pt_log("ERROR: failed to read at offset %#" PRIx64 " ", + offset); if (intel_pt_enable_logging) dso__fprintf(al.map->dso, intel_pt_log_fp()); return -EINVAL; @@ -2418,6 +2419,8 @@ static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu, pid_t pid, pid_t tid, u64 ip, u64 timestamp, pid_t machine_pid, int vcpu) { + bool dump_log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR; + bool log_on_stdout = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_USE_STDOUT; union perf_event event; char msg[MAX_AUXTRACE_ERROR_MSG]; int err; @@ -2437,6 +2440,16 @@ static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu, code, cpu, pid, tid, ip, msg, timestamp, machine_pid, vcpu); + if (intel_pt_enable_logging && !log_on_stdout) { + FILE *fp = intel_pt_log_fp(); + + if (fp) + perf_event__fprintf_auxtrace_error(&event, fp); + } + + if (code != INTEL_PT_ERR_LOST && dump_log_on_error) + intel_pt_log_dump_buf(); + err = perf_session__deliver_synth_event(pt->session, &event, NULL); if (err) pr_err("Intel Processor Trace: failed to deliver error event, error %d\n", @@ -4033,6 +4046,7 @@ static const char * const intel_pt_info_fmts[] = { [INTEL_PT_SNAPSHOT_MODE] = " Snapshot mode %"PRId64"\n", [INTEL_PT_PER_CPU_MMAPS] = " Per-cpu maps %"PRId64"\n", [INTEL_PT_MTC_BIT] = " MTC bit %#"PRIx64"\n", + [INTEL_PT_MTC_FREQ_BITS] = " MTC freq bits %#"PRIx64"\n", [INTEL_PT_TSC_CTC_N] = " TSC:CTC numerator %"PRIu64"\n", [INTEL_PT_TSC_CTC_D] = " TSC:CTC denominator %"PRIu64"\n", [INTEL_PT_CYC_BIT] = " CYC bit %#"PRIx64"\n", @@ -4047,8 +4061,12 @@ static void intel_pt_print_info(__u64 *arr, int start, int finish) if (!dump_trace) return; - for (i = start; i <= finish; i++) - fprintf(stdout, intel_pt_info_fmts[i], arr[i]); + for (i = start; i <= finish; i++) { + const char *fmt = intel_pt_info_fmts[i]; + + if (fmt) + fprintf(stdout, fmt, arr[i]); + } } static void intel_pt_print_info_str(const char *name, const char *str) @@ -4271,8 +4289,12 @@ int intel_pt_process_auxtrace_info(union perf_event *event, goto err_delete_thread; } - if (pt->synth_opts.log) - intel_pt_log_enable(); + if (pt->synth_opts.log) { + bool log_on_error = pt->synth_opts.log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR; + unsigned int log_on_error_size = pt->synth_opts.log_on_error_size; + + intel_pt_log_enable(log_on_error, log_on_error_size); + } /* Maximum non-turbo ratio is TSC freq / 100 MHz */ if (pt->tc.time_mult) { diff --git a/tools/perf/util/jitdump.c b/tools/perf/util/jitdump.c index 4e663220370441de887b888bd35db7c2dbad831e..0e033278fa1274b1bf6c735ab1dae7ea506f97dc 100644 --- a/tools/perf/util/jitdump.c +++ b/tools/perf/util/jitdump.c @@ -56,13 +56,6 @@ struct jit_buf_desc { char dir[PATH_MAX]; }; -struct debug_line_info { - unsigned long vma; - unsigned int lineno; - /* The filename format is unspecified, absolute path, relative etc. */ - char const filename[]; -}; - struct jit_tool { struct perf_tool tool; struct perf_data output; diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h index 2146efc33396ee4ec75c150cbaf6b3898f569aa6..b8cb8830b7bc596d37778ffd0cd47fd37538f63d 100644 --- a/tools/perf/util/lock-contention.h +++ b/tools/perf/util/lock-contention.h @@ -11,6 +11,7 @@ struct lock_stat { u64 addr; /* address of lockdep_map, used as ID */ char *name; /* for strcpy(), we cannot use const */ + u64 *callstack; unsigned int nr_acquire; unsigned int nr_acquired; @@ -113,7 +114,9 @@ struct lock_contention { struct machine *machine; struct hlist_head *result; unsigned long map_nr_entries; - unsigned long lost; + int lost; + int max_stack; + int stack_skip; }; #ifdef HAVE_BPF_SKEL diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 2a16cae28407422d56bdc85d12c25cffa608224d..76316e459c3de9c42fd141dcac01d706cf7cc041 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -1128,10 +1128,6 @@ static struct dso *machine__get_kernel(struct machine *machine) return kernel; } -struct process_args { - u64 start; -}; - void machine__get_kallsyms_filename(struct machine *machine, char *buf, size_t bufsz) { diff --git a/tools/perf/util/map.c b/tools/perf/util/map.c index e0aa4a25458383db0a48c5621052697d2d10358d..f3a3d9b3a40dafea5e8eaf6c2c70cc36bf54f6a9 100644 --- a/tools/perf/util/map.c +++ b/tools/perf/util/map.c @@ -181,7 +181,10 @@ struct map *map__new(struct machine *machine, u64 start, u64 len, if (!(prot & PROT_EXEC)) dso__set_loaded(dso); } + mutex_lock(&dso->lock); + nsinfo__put(dso->nsinfo); dso->nsinfo = nsi; + mutex_unlock(&dso->lock); if (build_id__is_defined(bid)) { dso__set_build_id(dso, bid); diff --git a/tools/perf/util/mem-events.c b/tools/perf/util/mem-events.c index 764883183519e04905b26483b422f9532363a392..b3a91093069a57154437173897141e32f6e173d2 100644 --- a/tools/perf/util/mem-events.c +++ b/tools/perf/util/mem-events.c @@ -156,11 +156,12 @@ void perf_mem_events__list(void) for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { struct perf_mem_event *e = perf_mem_events__ptr(j); - fprintf(stderr, "%-13s%-*s%s\n", - e->tag ?: "", - verbose > 0 ? 25 : 0, - verbose > 0 ? perf_mem_events__name(j, NULL) : "", - e->supported ? ": available" : ""); + fprintf(stderr, "%-*s%-*s%s", + e->tag ? 13 : 0, + e->tag ? : "", + e->tag && verbose > 0 ? 25 : 0, + e->tag && verbose > 0 ? perf_mem_events__name(j, NULL) : "", + e->supported ? ": available\n" : ""); } } @@ -281,7 +282,7 @@ static const char * const mem_lvl[] = { "HIT", "MISS", "L1", - "LFB", + "LFB/MAB", "L2", "L3", "Local RAM", @@ -294,8 +295,10 @@ static const char * const mem_lvl[] = { }; static const char * const mem_lvlnum[] = { + [PERF_MEM_LVLNUM_CXL] = "CXL", + [PERF_MEM_LVLNUM_IO] = "I/O", [PERF_MEM_LVLNUM_ANY_CACHE] = "Any cache", - [PERF_MEM_LVLNUM_LFB] = "LFB", + [PERF_MEM_LVLNUM_LFB] = "LFB/MAB", [PERF_MEM_LVLNUM_RAM] = "RAM", [PERF_MEM_LVLNUM_PMEM] = "PMEM", [PERF_MEM_LVLNUM_NA] = "N/A", diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 464475fd6b9a3a799e0e3d72ed56c727bb6a6113..4c98ac29ee13feb7710bf1dd95ad49e40ca75ade 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -22,6 +22,7 @@ #include <linux/list_sort.h> #include <linux/string.h> #include <linux/zalloc.h> +#include <perf/cpumap.h> #include <subcmd/parse-options.h> #include <api/fs/fs.h> #include "util.h" @@ -108,17 +109,6 @@ void metricgroup__rblist_exit(struct rblist *metric_events) rblist__exit(metric_events); } -/* - * A node in the list of referenced metrics. metric_expr - * is held as a convenience to avoid a search through the - * metric list. - */ -struct metric_ref_node { - const char *metric_name; - const char *metric_expr; - struct list_head list; -}; - /** * The metric under construction. The data held here will be placed in a * metric_expr. @@ -189,10 +179,24 @@ static bool metricgroup__has_constraint(const struct pmu_event *pe) return false; } +static void metric__free(struct metric *m) +{ + if (!m) + return; + + free(m->metric_refs); + expr__ctx_free(m->pctx); + free((char *)m->modifier); + evlist__delete(m->evlist); + free(m); +} + static struct metric *metric__new(const struct pmu_event *pe, const char *modifier, bool metric_no_group, - int runtime) + int runtime, + const char *user_requested_cpu_list, + bool system_wide) { struct metric *m; @@ -201,35 +205,34 @@ static struct metric *metric__new(const struct pmu_event *pe, return NULL; m->pctx = expr__ctx_new(); - if (!m->pctx) { - free(m); - return NULL; - } + if (!m->pctx) + goto out_err; m->metric_name = pe->metric_name; - m->modifier = modifier ? strdup(modifier) : NULL; - if (modifier && !m->modifier) { - expr__ctx_free(m->pctx); - free(m); - return NULL; + m->modifier = NULL; + if (modifier) { + m->modifier = strdup(modifier); + if (!m->modifier) + goto out_err; } m->metric_expr = pe->metric_expr; m->metric_unit = pe->unit; - m->pctx->runtime = runtime; + m->pctx->sctx.user_requested_cpu_list = NULL; + if (user_requested_cpu_list) { + m->pctx->sctx.user_requested_cpu_list = strdup(user_requested_cpu_list); + if (!m->pctx->sctx.user_requested_cpu_list) + goto out_err; + } + m->pctx->sctx.runtime = runtime; + m->pctx->sctx.system_wide = system_wide; m->has_constraint = metric_no_group || metricgroup__has_constraint(pe); m->metric_refs = NULL; m->evlist = NULL; return m; -} - -static void metric__free(struct metric *m) -{ - free(m->metric_refs); - expr__ctx_free(m->pctx); - free((char *)m->modifier); - evlist__delete(m->evlist); - free(m); +out_err: + metric__free(m); + return NULL; } static bool contains_metric_id(struct evsel **metric_events, int num_events, @@ -874,6 +877,8 @@ struct metricgroup_add_iter_data { int *ret; bool *has_match; bool metric_no_group; + const char *user_requested_cpu_list; + bool system_wide; struct metric *root_metric; const struct visited_metric *visited; const struct pmu_events_table *table; @@ -887,6 +892,8 @@ static int add_metric(struct list_head *metric_list, const struct pmu_event *pe, const char *modifier, bool metric_no_group, + const char *user_requested_cpu_list, + bool system_wide, struct metric *root_metric, const struct visited_metric *visited, const struct pmu_events_table *table); @@ -899,6 +906,8 @@ static int add_metric(struct list_head *metric_list, * @metric_no_group: Should events written to events be grouped "{}" or * global. Grouping is the default but due to multiplexing the * user may override. + * @user_requested_cpu_list: Command line specified CPUs to record on. + * @system_wide: Are events for all processes recorded. * @root_metric: Metrics may reference other metrics to form a tree. In this * case the root_metric holds all the IDs and a list of referenced * metrics. When adding a root this argument is NULL. @@ -910,6 +919,8 @@ static int add_metric(struct list_head *metric_list, static int resolve_metric(struct list_head *metric_list, const char *modifier, bool metric_no_group, + const char *user_requested_cpu_list, + bool system_wide, struct metric *root_metric, const struct visited_metric *visited, const struct pmu_events_table *table) @@ -956,7 +967,8 @@ static int resolve_metric(struct list_head *metric_list, */ for (i = 0; i < pending_cnt; i++) { ret = add_metric(metric_list, &pending[i].pe, modifier, metric_no_group, - root_metric, visited, table); + user_requested_cpu_list, system_wide, root_metric, visited, + table); if (ret) break; } @@ -974,6 +986,8 @@ static int resolve_metric(struct list_head *metric_list, * global. Grouping is the default but due to multiplexing the * user may override. * @runtime: A special argument for the parser only known at runtime. + * @user_requested_cpu_list: Command line specified CPUs to record on. + * @system_wide: Are events for all processes recorded. * @root_metric: Metrics may reference other metrics to form a tree. In this * case the root_metric holds all the IDs and a list of referenced * metrics. When adding a root this argument is NULL. @@ -987,6 +1001,8 @@ static int __add_metric(struct list_head *metric_list, const char *modifier, bool metric_no_group, int runtime, + const char *user_requested_cpu_list, + bool system_wide, struct metric *root_metric, const struct visited_metric *visited, const struct pmu_events_table *table) @@ -1011,7 +1027,8 @@ static int __add_metric(struct list_head *metric_list, * This metric is the root of a tree and may reference other * metrics that are added recursively. */ - root_metric = metric__new(pe, modifier, metric_no_group, runtime); + root_metric = metric__new(pe, modifier, metric_no_group, runtime, + user_requested_cpu_list, system_wide); if (!root_metric) return -ENOMEM; @@ -1060,8 +1077,9 @@ static int __add_metric(struct list_head *metric_list, ret = -EINVAL; } else { /* Resolve referenced metrics. */ - ret = resolve_metric(metric_list, modifier, metric_no_group, root_metric, - &visited_node, table); + ret = resolve_metric(metric_list, modifier, metric_no_group, + user_requested_cpu_list, system_wide, + root_metric, &visited_node, table); } if (ret) { @@ -1109,6 +1127,8 @@ static int add_metric(struct list_head *metric_list, const struct pmu_event *pe, const char *modifier, bool metric_no_group, + const char *user_requested_cpu_list, + bool system_wide, struct metric *root_metric, const struct visited_metric *visited, const struct pmu_events_table *table) @@ -1119,7 +1139,8 @@ static int add_metric(struct list_head *metric_list, if (!strstr(pe->metric_expr, "?")) { ret = __add_metric(metric_list, pe, modifier, metric_no_group, 0, - root_metric, visited, table); + user_requested_cpu_list, system_wide, root_metric, + visited, table); } else { int j, count; @@ -1132,7 +1153,8 @@ static int add_metric(struct list_head *metric_list, for (j = 0; j < count && !ret; j++) ret = __add_metric(metric_list, pe, modifier, metric_no_group, j, - root_metric, visited, table); + user_requested_cpu_list, system_wide, + root_metric, visited, table); } return ret; @@ -1149,6 +1171,7 @@ static int metricgroup__add_metric_sys_event_iter(const struct pmu_event *pe, return 0; ret = add_metric(d->metric_list, pe, d->modifier, d->metric_no_group, + d->user_requested_cpu_list, d->system_wide, d->root_metric, d->visited, d->table); if (ret) goto out; @@ -1191,7 +1214,9 @@ struct metricgroup__add_metric_data { struct list_head *list; const char *metric_name; const char *modifier; + const char *user_requested_cpu_list; bool metric_no_group; + bool system_wide; bool has_match; }; @@ -1208,8 +1233,8 @@ static int metricgroup__add_metric_callback(const struct pmu_event *pe, data->has_match = true; ret = add_metric(data->list, pe, data->modifier, data->metric_no_group, - /*root_metric=*/NULL, - /*visited_metrics=*/NULL, table); + data->user_requested_cpu_list, data->system_wide, + /*root_metric=*/NULL, /*visited_metrics=*/NULL, table); } return ret; } @@ -1223,12 +1248,16 @@ static int metricgroup__add_metric_callback(const struct pmu_event *pe, * @metric_no_group: Should events written to events be grouped "{}" or * global. Grouping is the default but due to multiplexing the * user may override. + * @user_requested_cpu_list: Command line specified CPUs to record on. + * @system_wide: Are events for all processes recorded. * @metric_list: The list that the metric or metric group are added to. * @table: The table that is searched for metrics, most commonly the table for the * architecture perf is running upon. */ static int metricgroup__add_metric(const char *metric_name, const char *modifier, bool metric_no_group, + const char *user_requested_cpu_list, + bool system_wide, struct list_head *metric_list, const struct pmu_events_table *table) { @@ -1242,6 +1271,8 @@ static int metricgroup__add_metric(const char *metric_name, const char *modifier .metric_name = metric_name, .modifier = modifier, .metric_no_group = metric_no_group, + .user_requested_cpu_list = user_requested_cpu_list, + .system_wide = system_wide, .has_match = false, }; /* @@ -1263,6 +1294,8 @@ static int metricgroup__add_metric(const char *metric_name, const char *modifier .metric_name = metric_name, .modifier = modifier, .metric_no_group = metric_no_group, + .user_requested_cpu_list = user_requested_cpu_list, + .system_wide = system_wide, .has_match = &has_match, .ret = &ret, .table = table, @@ -1293,12 +1326,15 @@ out: * @metric_no_group: Should events written to events be grouped "{}" or * global. Grouping is the default but due to multiplexing the * user may override. + * @user_requested_cpu_list: Command line specified CPUs to record on. + * @system_wide: Are events for all processes recorded. * @metric_list: The list that metrics are added to. * @table: The table that is searched for metrics, most commonly the table for the * architecture perf is running upon. */ static int metricgroup__add_metric_list(const char *list, bool metric_no_group, - struct list_head *metric_list, + const char *user_requested_cpu_list, + bool system_wide, struct list_head *metric_list, const struct pmu_events_table *table) { char *list_itr, *list_copy, *metric_name, *modifier; @@ -1315,8 +1351,8 @@ static int metricgroup__add_metric_list(const char *list, bool metric_no_group, *modifier++ = '\0'; ret = metricgroup__add_metric(metric_name, modifier, - metric_no_group, metric_list, - table); + metric_no_group, user_requested_cpu_list, + system_wide, metric_list, table); if (ret == -EINVAL) pr_err("Cannot find metric or group `%s'\n", metric_name); @@ -1505,6 +1541,8 @@ err_out: static int parse_groups(struct evlist *perf_evlist, const char *str, bool metric_no_group, bool metric_no_merge, + const char *user_requested_cpu_list, + bool system_wide, struct perf_pmu *fake_pmu, struct rblist *metric_events_list, const struct pmu_events_table *table) @@ -1518,7 +1556,8 @@ static int parse_groups(struct evlist *perf_evlist, const char *str, if (metric_events_list->nr_entries == 0) metricgroup__rblist_init(metric_events_list); ret = metricgroup__add_metric_list(str, metric_no_group, - &metric_list, table); + user_requested_cpu_list, + system_wide, &metric_list, table); if (ret) goto out; @@ -1626,7 +1665,7 @@ static int parse_groups(struct evlist *perf_evlist, const char *str, } expr->metric_unit = m->metric_unit; expr->metric_events = metric_events; - expr->runtime = m->pctx->runtime; + expr->runtime = m->pctx->sctx.runtime; list_add(&expr->nd, &me->head); } @@ -1646,17 +1685,22 @@ out: return ret; } -int metricgroup__parse_groups(const struct option *opt, +int metricgroup__parse_groups(struct evlist *perf_evlist, const char *str, bool metric_no_group, bool metric_no_merge, + const char *user_requested_cpu_list, + bool system_wide, struct rblist *metric_events) { - struct evlist *perf_evlist = *(struct evlist **)opt->value; const struct pmu_events_table *table = pmu_events_table__find(); - return parse_groups(perf_evlist, str, metric_no_group, - metric_no_merge, NULL, metric_events, table); + if (!table) + return -EINVAL; + + return parse_groups(perf_evlist, str, metric_no_group, metric_no_merge, + user_requested_cpu_list, system_wide, + /*fake_pmu=*/NULL, metric_events, table); } int metricgroup__parse_groups_test(struct evlist *evlist, @@ -1666,8 +1710,10 @@ int metricgroup__parse_groups_test(struct evlist *evlist, bool metric_no_merge, struct rblist *metric_events) { - return parse_groups(evlist, str, metric_no_group, - metric_no_merge, &perf_pmu__fake, metric_events, table); + return parse_groups(evlist, str, metric_no_group, metric_no_merge, + /*user_requested_cpu_list=*/NULL, + /*system_wide=*/false, + &perf_pmu__fake, metric_events, table); } static int metricgroup__has_metric_callback(const struct pmu_event *pe, @@ -1700,7 +1746,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, struct rblist *new_metric_events, struct rblist *old_metric_events) { - unsigned i; + unsigned int i; for (i = 0; i < rblist__nr_entries(old_metric_events); i++) { struct rb_node *nd; diff --git a/tools/perf/util/metricgroup.h b/tools/perf/util/metricgroup.h index 016b3b1a289a62dcaf22c8ebf6fb8eb22a71c69f..732d3a0d33341d8090e87ed57d018cd9fc87864e 100644 --- a/tools/perf/util/metricgroup.h +++ b/tools/perf/util/metricgroup.h @@ -64,10 +64,12 @@ struct metric_expr { struct metric_event *metricgroup__lookup(struct rblist *metric_events, struct evsel *evsel, bool create); -int metricgroup__parse_groups(const struct option *opt, +int metricgroup__parse_groups(struct evlist *perf_evlist, const char *str, bool metric_no_group, bool metric_no_merge, + const char *user_requested_cpu_list, + bool system_wide, struct rblist *metric_events); int metricgroup__parse_groups_test(struct evlist *evlist, const struct pmu_events_table *table, diff --git a/tools/perf/util/mmap.h b/tools/perf/util/mmap.h index cd8b0777473b3cdf4ff5c40eeb9ec4d3d5a16b5a..cd4ccec7f361761b1bdff734dea35da7ea263f8e 100644 --- a/tools/perf/util/mmap.h +++ b/tools/perf/util/mmap.h @@ -9,7 +9,6 @@ #include <linux/bitops.h> #include <perf/cpumap.h> #include <stdbool.h> -#include <pthread.h> // for cpu_set_t #ifdef HAVE_AIO_SUPPORT #include <aio.h> #endif diff --git a/tools/perf/util/mutex.c b/tools/perf/util/mutex.c new file mode 100644 index 0000000000000000000000000000000000000000..bca7f0717f355e16ade273a16c2620a4254957c7 --- /dev/null +++ b/tools/perf/util/mutex.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "mutex.h" + +#include "debug.h" +#include <linux/string.h> +#include <errno.h> + +static void check_err(const char *fn, int err) +{ + char sbuf[STRERR_BUFSIZE]; + + if (err == 0) + return; + + pr_err("%s error: '%s'\n", fn, str_error_r(err, sbuf, sizeof(sbuf))); +} + +#define CHECK_ERR(err) check_err(__func__, err) + +static void __mutex_init(struct mutex *mtx, bool pshared) +{ + pthread_mutexattr_t attr; + + CHECK_ERR(pthread_mutexattr_init(&attr)); + +#ifndef NDEBUG + /* In normal builds enable error checking, such as recursive usage. */ + CHECK_ERR(pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK)); +#endif + if (pshared) + CHECK_ERR(pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)); + + CHECK_ERR(pthread_mutex_init(&mtx->lock, &attr)); + CHECK_ERR(pthread_mutexattr_destroy(&attr)); +} + +void mutex_init(struct mutex *mtx) +{ + __mutex_init(mtx, /*pshared=*/false); +} + +void mutex_init_pshared(struct mutex *mtx) +{ + __mutex_init(mtx, /*pshared=*/true); +} + +void mutex_destroy(struct mutex *mtx) +{ + CHECK_ERR(pthread_mutex_destroy(&mtx->lock)); +} + +void mutex_lock(struct mutex *mtx) + NO_THREAD_SAFETY_ANALYSIS +{ + CHECK_ERR(pthread_mutex_lock(&mtx->lock)); +} + +void mutex_unlock(struct mutex *mtx) + NO_THREAD_SAFETY_ANALYSIS +{ + CHECK_ERR(pthread_mutex_unlock(&mtx->lock)); +} + +bool mutex_trylock(struct mutex *mtx) +{ + int ret = pthread_mutex_trylock(&mtx->lock); + + if (ret == 0) + return true; /* Lock acquired. */ + + if (ret == EBUSY) + return false; /* Lock busy. */ + + /* Print error. */ + CHECK_ERR(ret); + return false; +} + +static void __cond_init(struct cond *cnd, bool pshared) +{ + pthread_condattr_t attr; + + CHECK_ERR(pthread_condattr_init(&attr)); + if (pshared) + CHECK_ERR(pthread_condattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)); + + CHECK_ERR(pthread_cond_init(&cnd->cond, &attr)); + CHECK_ERR(pthread_condattr_destroy(&attr)); +} + +void cond_init(struct cond *cnd) +{ + __cond_init(cnd, /*pshared=*/false); +} + +void cond_init_pshared(struct cond *cnd) +{ + __cond_init(cnd, /*pshared=*/true); +} + +void cond_destroy(struct cond *cnd) +{ + CHECK_ERR(pthread_cond_destroy(&cnd->cond)); +} + +void cond_wait(struct cond *cnd, struct mutex *mtx) +{ + CHECK_ERR(pthread_cond_wait(&cnd->cond, &mtx->lock)); +} + +void cond_signal(struct cond *cnd) +{ + CHECK_ERR(pthread_cond_signal(&cnd->cond)); +} + +void cond_broadcast(struct cond *cnd) +{ + CHECK_ERR(pthread_cond_broadcast(&cnd->cond)); +} diff --git a/tools/perf/util/mutex.h b/tools/perf/util/mutex.h new file mode 100644 index 0000000000000000000000000000000000000000..40661120caccbc931015f82cdecf92c69e674007 --- /dev/null +++ b/tools/perf/util/mutex.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_MUTEX_H +#define __PERF_MUTEX_H + +#include <pthread.h> +#include <stdbool.h> + +/* + * A function-like feature checking macro that is a wrapper around + * `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a + * nonzero constant integer if the attribute is supported or 0 if not. + */ +#ifdef __has_attribute +#define HAVE_ATTRIBUTE(x) __has_attribute(x) +#else +#define HAVE_ATTRIBUTE(x) 0 +#endif + +#if HAVE_ATTRIBUTE(guarded_by) && HAVE_ATTRIBUTE(pt_guarded_by) && \ + HAVE_ATTRIBUTE(lockable) && HAVE_ATTRIBUTE(exclusive_lock_function) && \ + HAVE_ATTRIBUTE(exclusive_trylock_function) && HAVE_ATTRIBUTE(exclusive_locks_required) && \ + HAVE_ATTRIBUTE(no_thread_safety_analysis) + +/* Documents if a shared field or global variable needs to be protected by a mutex. */ +#define GUARDED_BY(x) __attribute__((guarded_by(x))) + +/* + * Documents if the memory location pointed to by a pointer should be guarded by + * a mutex when dereferencing the pointer. + */ +#define PT_GUARDED_BY(x) __attribute__((pt_guarded_by(x))) + +/* Documents if a type is a lockable type. */ +#define LOCKABLE __attribute__((lockable)) + +/* Documents functions that acquire a lock in the body of a function, and do not release it. */ +#define EXCLUSIVE_LOCK_FUNCTION(...) __attribute__((exclusive_lock_function(__VA_ARGS__))) + +/* + * Documents functions that expect a lock to be held on entry to the function, + * and release it in the body of the function. + */ +#define UNLOCK_FUNCTION(...) __attribute__((unlock_function(__VA_ARGS__))) + +/* Documents functions that try to acquire a lock, and return success or failure. */ +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + __attribute__((exclusive_trylock_function(__VA_ARGS__))) + +/* Documents a function that expects a mutex to be held prior to entry. */ +#define EXCLUSIVE_LOCKS_REQUIRED(...) __attribute__((exclusive_locks_required(__VA_ARGS__))) + +/* Turns off thread safety checking within the body of a particular function. */ +#define NO_THREAD_SAFETY_ANALYSIS __attribute__((no_thread_safety_analysis)) + +#else + +#define GUARDED_BY(x) +#define PT_GUARDED_BY(x) +#define LOCKABLE +#define EXCLUSIVE_LOCK_FUNCTION(...) +#define UNLOCK_FUNCTION(...) +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) +#define EXCLUSIVE_LOCKS_REQUIRED(...) +#define NO_THREAD_SAFETY_ANALYSIS + +#endif + +/* + * A wrapper around the mutex implementation that allows perf to error check + * usage, etc. + */ +struct LOCKABLE mutex { + pthread_mutex_t lock; +}; + +/* A wrapper around the condition variable implementation. */ +struct cond { + pthread_cond_t cond; +}; + +/* Default initialize the mtx struct. */ +void mutex_init(struct mutex *mtx); +/* + * Initialize the mtx struct and set the process-shared rather than default + * process-private attribute. + */ +void mutex_init_pshared(struct mutex *mtx); +void mutex_destroy(struct mutex *mtx); + +void mutex_lock(struct mutex *mtx) EXCLUSIVE_LOCK_FUNCTION(*mtx); +void mutex_unlock(struct mutex *mtx) UNLOCK_FUNCTION(*mtx); +/* Tries to acquire the lock and returns true on success. */ +bool mutex_trylock(struct mutex *mtx) EXCLUSIVE_TRYLOCK_FUNCTION(true, *mtx); + +/* Default initialize the cond struct. */ +void cond_init(struct cond *cnd); +/* + * Initialize the cond struct and specify the process-shared rather than default + * process-private attribute. + */ +void cond_init_pshared(struct cond *cnd); +void cond_destroy(struct cond *cnd); + +void cond_wait(struct cond *cnd, struct mutex *mtx) EXCLUSIVE_LOCKS_REQUIRED(mtx); +void cond_signal(struct cond *cnd); +void cond_broadcast(struct cond *cnd); + +#endif /* __PERF_MUTEX_H */ diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c index bb4aa88c50a8276fd091a126873187775ce9f3d0..00588b9db474e81e77bff1d380c9405040eeae1d 100644 --- a/tools/perf/util/parse-branch-options.c +++ b/tools/perf/util/parse-branch-options.c @@ -32,6 +32,7 @@ static const struct branch_mode branch_modes[] = { BRANCH_OPT("call", PERF_SAMPLE_BRANCH_CALL), BRANCH_OPT("save_type", PERF_SAMPLE_BRANCH_TYPE_SAVE), BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK), + BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE), BRANCH_END }; diff --git a/tools/perf/util/parse-events-hybrid.c b/tools/perf/util/parse-events-hybrid.c index 284f8eabd3b9abfa51b26d15553faf0b1ffb5936..7c9f9150bad50271ee70adc957cf2dea3f1623b8 100644 --- a/tools/perf/util/parse-events-hybrid.c +++ b/tools/perf/util/parse-events-hybrid.c @@ -33,7 +33,8 @@ static void config_hybrid_attr(struct perf_event_attr *attr, * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied. */ attr->type = type; - attr->config = attr->config | ((__u64)pmu_type << PERF_PMU_TYPE_SHIFT); + attr->config = (attr->config & PERF_HW_EVENT_MASK) | + ((__u64)pmu_type << PERF_PMU_TYPE_SHIFT); } static int create_event_hybrid(__u32 config_type, int *idx, @@ -48,13 +49,25 @@ static int create_event_hybrid(__u32 config_type, int *idx, __u64 config = attr->config; config_hybrid_attr(attr, config_type, pmu->type); + + /* + * Some hybrid hardware cache events are only available on one CPU + * PMU. For example, the 'L1-dcache-load-misses' is only available + * on cpu_core, while the 'L1-icache-loads' is only available on + * cpu_atom. We need to remove "not supported" hybrid cache events. + */ + if (attr->type == PERF_TYPE_HW_CACHE + && !is_event_supported(attr->type, attr->config)) + return 0; + evsel = parse_events__add_event_hybrid(list, idx, attr, name, metric_id, pmu, config_terms); - if (evsel) + if (evsel) { evsel->pmu_name = strdup(pmu->name); - else + if (!evsel->pmu_name) + return -ENOMEM; + } else return -ENOMEM; - attr->type = type; attr->config = config; return 0; diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index f05e15acd33fe84532749dad8075ce07d1dfcdcd..5973f46c23755e7edf30e32f8b4c242854b740e8 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -28,6 +28,7 @@ #include "util/parse-events-hybrid.h" #include "util/pmu-hybrid.h" #include "tracepoint.h" +#include "thread_map.h" #define MAX_NAME_LEN 100 @@ -149,13 +150,43 @@ struct event_symbol event_symbols_sw[PERF_COUNT_SW_MAX] = { }, }; -#define __PERF_EVENT_FIELD(config, name) \ - ((config & PERF_EVENT_##name##_MASK) >> PERF_EVENT_##name##_SHIFT) +bool is_event_supported(u8 type, u64 config) +{ + bool ret = true; + int open_return; + struct evsel *evsel; + struct perf_event_attr attr = { + .type = type, + .config = config, + .disabled = 1, + }; + struct perf_thread_map *tmap = thread_map__new_by_tid(0); + + if (tmap == NULL) + return false; -#define PERF_EVENT_RAW(config) __PERF_EVENT_FIELD(config, RAW) -#define PERF_EVENT_CONFIG(config) __PERF_EVENT_FIELD(config, CONFIG) -#define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE) -#define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT) + evsel = evsel__new(&attr); + if (evsel) { + open_return = evsel__open(evsel, NULL, tmap); + ret = open_return >= 0; + + if (open_return == -EACCES) { + /* + * This happens if the paranoid value + * /proc/sys/kernel/perf_event_paranoid is set to 2 + * Re-run with exclude_kernel set; we don't do that + * by default as some ARM machines do not support it. + * + */ + evsel->core.attr.exclude_kernel = 1; + ret = evsel__open(evsel, NULL, tmap) >= 0; + } + evsel__delete(evsel); + } + + perf_thread_map__put(tmap); + return ret; +} const char *event_type(int type) { @@ -215,6 +246,9 @@ __add_event(struct list_head *list, int *idx, struct perf_cpu_map *cpus = pmu ? perf_cpu_map__get(pmu->cpus) : cpu_list ? perf_cpu_map__new(cpu_list) : NULL; + if (pmu) + perf_pmu__warn_invalid_formats(pmu); + if (pmu && attr->type == PERF_TYPE_RAW) perf_pmu__warn_invalid_config(pmu, attr->config, name); diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h index 7e6a601d9cd01bc45de9a93a8e1ef53fbd7cf187..07df7bb7b042006689e802c48d972a7518700326 100644 --- a/tools/perf/util/parse-events.h +++ b/tools/perf/util/parse-events.h @@ -19,6 +19,7 @@ struct option; struct perf_pmu; bool have_tracepoints(struct list_head *evlist); +bool is_event_supported(u8 type, u64 config); const char *event_type(int type); diff --git a/tools/perf/util/perf_event_attr_fprintf.c b/tools/perf/util/perf_event_attr_fprintf.c index 98af3fa4ea35351be6a10e303a228be6c2dfba7c..7e5e7b30510dff32c5ff1ababcd2c4fa6fa358d7 100644 --- a/tools/perf/util/perf_event_attr_fprintf.c +++ b/tools/perf/util/perf_event_attr_fprintf.c @@ -52,7 +52,7 @@ static void __p_branch_sample_type(char *buf, size_t size, u64 value) bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX), bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP), bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES), - bit_name(TYPE_SAVE), bit_name(HW_INDEX), + bit_name(TYPE_SAVE), bit_name(HW_INDEX), bit_name(PRIV_SAVE), { .name = NULL, } }; #undef bit_name @@ -64,7 +64,7 @@ static void __p_read_format(char *buf, size_t size, u64 value) #define bit_name(n) { PERF_FORMAT_##n, #n } struct bit_names bits[] = { bit_name(TOTAL_TIME_ENABLED), bit_name(TOTAL_TIME_RUNNING), - bit_name(ID), bit_name(GROUP), + bit_name(ID), bit_name(GROUP), bit_name(LOST), { .name = NULL, } }; #undef bit_name diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 89655d53117ae7eaffc33daa90e48d9e93567432..03284059175f7f43b29826f57d13790e9a3891c3 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -1005,6 +1005,23 @@ err: return NULL; } +void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu) +{ + struct perf_pmu_format *format; + + /* fake pmu doesn't have format list */ + if (pmu == &perf_pmu__fake) + return; + + list_for_each_entry(format, &pmu->format, list) + if (format->value >= PERF_PMU_FORMAT_VALUE_CONFIG_END) { + pr_warning("WARNING: '%s' format '%s' requires 'perf_event_attr::config%d'" + "which is not supported by this version of perf!\n", + pmu->name, format->name, format->value); + return; + } +} + static struct perf_pmu *pmu_find(const char *name) { struct perf_pmu *pmu; @@ -1182,7 +1199,7 @@ static char *pmu_formats_string(struct list_head *formats) struct perf_pmu_format *format; char *str = NULL; struct strbuf buf = STRBUF_INIT; - unsigned i = 0; + unsigned int i = 0; if (!formats) return NULL; diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index a7b0f9507510b0de031a0398a02e3e352eaa299a..68e15c38ae710ed5f7419ebefd89b7deb2790b50 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -17,6 +17,7 @@ enum { PERF_PMU_FORMAT_VALUE_CONFIG, PERF_PMU_FORMAT_VALUE_CONFIG1, PERF_PMU_FORMAT_VALUE_CONFIG2, + PERF_PMU_FORMAT_VALUE_CONFIG_END, }; #define PERF_PMU_FORMAT_BITS 64 @@ -139,6 +140,7 @@ int perf_pmu__caps_parse(struct perf_pmu *pmu); void perf_pmu__warn_invalid_config(struct perf_pmu *pmu, __u64 config, const char *name); +void perf_pmu__warn_invalid_formats(struct perf_pmu *pmu); bool perf_pmu__has_hybrid(void); int perf_pmu__match(char *pattern, char *name, char *tok); diff --git a/tools/perf/util/pmu.l b/tools/perf/util/pmu.l index a15d9fbd7c0ed99512b0163ffe26bdb8eaf313bf..58b4926cfaca906dab6f6f85bd7f359c943f166d 100644 --- a/tools/perf/util/pmu.l +++ b/tools/perf/util/pmu.l @@ -27,8 +27,6 @@ num_dec [0-9]+ {num_dec} { return value(10); } config { return PP_CONFIG; } -config1 { return PP_CONFIG1; } -config2 { return PP_CONFIG2; } - { return '-'; } : { return ':'; } , { return ','; } diff --git a/tools/perf/util/pmu.y b/tools/perf/util/pmu.y index bfd7e8509869b64b53fbcbcddafe89a4a73b93b3..e675d79a0274f7e8e24d65540b62443a9a4ec049 100644 --- a/tools/perf/util/pmu.y +++ b/tools/perf/util/pmu.y @@ -10,8 +10,6 @@ #include <string.h> #include "pmu.h" -extern int perf_pmu_lex (void); - #define ABORT_ON(val) \ do { \ if (val) \ @@ -20,7 +18,7 @@ do { \ %} -%token PP_CONFIG PP_CONFIG1 PP_CONFIG2 +%token PP_CONFIG %token PP_VALUE PP_ERROR %type <num> PP_VALUE %type <bits> bit_term @@ -47,18 +45,11 @@ PP_CONFIG ':' bits $3)); } | -PP_CONFIG1 ':' bits +PP_CONFIG PP_VALUE ':' bits { ABORT_ON(perf_pmu__new_format(format, name, - PERF_PMU_FORMAT_VALUE_CONFIG1, - $3)); -} -| -PP_CONFIG2 ':' bits -{ - ABORT_ON(perf_pmu__new_format(format, name, - PERF_PMU_FORMAT_VALUE_CONFIG2, - $3)); + $2, + $4)); } bits: diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c index ba1ab51346854d93339b739325dabde47accf046..c4d5d87fae2f6b5bf6aabebafa9e2db9aea420ce 100644 --- a/tools/perf/util/print-events.c +++ b/tools/perf/util/print-events.c @@ -22,7 +22,6 @@ #include "probe-file.h" #include "string2.h" #include "strlist.h" -#include "thread_map.h" #include "tracepoint.h" #include "pfm.h" #include "pmu-hybrid.h" @@ -239,44 +238,6 @@ void print_sdt_events(const char *subsys_glob, const char *event_glob, strlist__delete(sdtlist); } -static bool is_event_supported(u8 type, unsigned int config) -{ - bool ret = true; - int open_return; - struct evsel *evsel; - struct perf_event_attr attr = { - .type = type, - .config = config, - .disabled = 1, - }; - struct perf_thread_map *tmap = thread_map__new_by_tid(0); - - if (tmap == NULL) - return false; - - evsel = evsel__new(&attr); - if (evsel) { - open_return = evsel__open(evsel, NULL, tmap); - ret = open_return >= 0; - - if (open_return == -EACCES) { - /* - * This happens if the paranoid value - * /proc/sys/kernel/perf_event_paranoid is set to 2 - * Re-run with exclude_kernel set; we don't do that - * by default as some ARM machines do not support it. - * - */ - evsel->core.attr.exclude_kernel = 1; - ret = evsel__open(evsel, NULL, tmap) >= 0; - } - evsel__delete(evsel); - } - - perf_thread_map__put(tmap); - return ret; -} - int print_hwcache_events(const char *event_glob, bool name_only) { unsigned int type, op, i, evt_i = 0, evt_num = 0, npmus = 0; diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index 785246ff41790761a6a1b34b57de0cc8e8419333..0c24bc7afbca22fc2de898e2702400d1da11ce80 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -29,6 +29,7 @@ #include "color.h" #include "map.h" #include "maps.h" +#include "mutex.h" #include "symbol.h" #include <api/fs/fs.h> #include "trace-event.h" /* For __maybe_unused */ @@ -180,8 +181,10 @@ struct map *get_target_map(const char *target, struct nsinfo *nsi, bool user) map = dso__new_map(target); if (map && map->dso) { + mutex_lock(&map->dso->lock); nsinfo__put(map->dso->nsinfo); map->dso->nsinfo = nsinfo__get(nsi); + mutex_unlock(&map->dso->lock); } return map; } else { diff --git a/tools/perf/util/scripting-engines/Build b/tools/perf/util/scripting-engines/Build index c92326c2233a457c8a88834fa55d7d4bfabd18f1..0f5ba28339cf9a4dfd56c283c8e3149ebd7da68e 100644 --- a/tools/perf/util/scripting-engines/Build +++ b/tools/perf/util/scripting-engines/Build @@ -3,4 +3,4 @@ perf-$(CONFIG_LIBPYTHON) += trace-event-python.o CFLAGS_trace-event-perl.o += $(PERL_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-nested-externs -Wno-undef -Wno-switch-default -Wno-bad-function-cast -Wno-declaration-after-statement -Wno-switch-enum -CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-error=deprecated-declarations +CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-deprecated-declarations diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index 192c9274f7ade92fbfc6ffe572dea19eae1285c9..1a4f10de29ffebd29b4c6d99512d022a38bdd8b2 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -943,6 +943,11 @@ static void perf_event__cpu_map_swap(union perf_event *event, default: pr_err("cpu_map swap: unsupported long size\n"); } + break; + case PERF_CPU_MAP__RANGE_CPUS: + data->range_cpu_data.start_cpu = bswap_16(data->range_cpu_data.start_cpu); + data->range_cpu_data.end_cpu = bswap_16(data->range_cpu_data.end_cpu); + break; default: break; } @@ -1180,7 +1185,7 @@ static void branch_stack__printf(struct perf_sample *sample, bool callstack) e->flags.abort ? "A" : " ", e->flags.in_tx ? "T" : " ", (unsigned)e->flags.reserved, - e->flags.type ? branch_type_name(e->flags.type) : ""); + get_branch_type(e)); } else { if (i == 0) { printf("..... %2"PRIu64": %016" PRIx64 "\n" diff --git a/tools/perf/util/smt.c b/tools/perf/util/smt.c index 2b0a36ebf27a3aa13df0b3dd56e2cff94a372d55..994e9e4182273b9c04026bf2b3ebf9f893804ab9 100644 --- a/tools/perf/util/smt.c +++ b/tools/perf/util/smt.c @@ -1,99 +1,37 @@ -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#include <linux/bitops.h> +// SPDX-License-Identifier: GPL-2.0-only +#include <string.h> #include "api/fs/fs.h" +#include "cputopo.h" #include "smt.h" -/** - * hweight_str - Returns the number of bits set in str. Stops at first non-hex - * or ',' character. - */ -static int hweight_str(char *str) -{ - int result = 0; - - while (*str) { - switch (*str++) { - case '0': - case ',': - break; - case '1': - case '2': - case '4': - case '8': - result++; - break; - case '3': - case '5': - case '6': - case '9': - case 'a': - case 'A': - case 'c': - case 'C': - result += 2; - break; - case '7': - case 'b': - case 'B': - case 'd': - case 'D': - case 'e': - case 'E': - result += 3; - break; - case 'f': - case 'F': - result += 4; - break; - default: - goto done; - } - } -done: - return result; -} - -int smt_on(void) +bool smt_on(const struct cpu_topology *topology) { static bool cached; - static int cached_result; - int cpu; - int ncpu; + static bool cached_result; + int fs_value; if (cached) return cached_result; - if (sysfs__read_int("devices/system/cpu/smt/active", &cached_result) >= 0) { - cached = true; - return cached_result; - } - - cached_result = 0; - ncpu = sysconf(_SC_NPROCESSORS_CONF); - for (cpu = 0; cpu < ncpu; cpu++) { - unsigned long long siblings; - char *str; - size_t strlen; - char fn[256]; + if (sysfs__read_int("devices/system/cpu/smt/active", &fs_value) >= 0) + cached_result = (fs_value == 1); + else + cached_result = cpu_topology__smt_on(topology); - snprintf(fn, sizeof fn, - "devices/system/cpu/cpu%d/topology/thread_siblings", cpu); - if (sysfs__read_str(fn, &str, &strlen) < 0) { - snprintf(fn, sizeof fn, - "devices/system/cpu/cpu%d/topology/core_cpus", cpu); - if (sysfs__read_str(fn, &str, &strlen) < 0) - continue; - } - /* Entry is hex, but does not have 0x, so need custom parser */ - siblings = hweight_str(str); - free(str); - if (siblings > 1) { - cached_result = 1; - break; - } - } cached = true; return cached_result; } + +bool core_wide(bool system_wide, const char *user_requested_cpu_list, + const struct cpu_topology *topology) +{ + /* If not everything running on a core is being recorded then we can't use core_wide. */ + if (!system_wide) + return false; + + /* Cheap case that SMT is disabled and therefore we're inherently core_wide. */ + if (!smt_on(topology)) + return true; + + return cpu_topology__core_wide(topology, user_requested_cpu_list); +} diff --git a/tools/perf/util/smt.h b/tools/perf/util/smt.h index b8414b7bcbc87b45a0c38dd7c2829d74a538fe10..ae9095f2c38c663c672a257564dc59722652e708 100644 --- a/tools/perf/util/smt.h +++ b/tools/perf/util/smt.h @@ -1,6 +1,17 @@ -#ifndef SMT_H -#define SMT_H 1 +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SMT_H +#define __SMT_H 1 -int smt_on(void); +struct cpu_topology; -#endif +/* Returns true if SMT (aka hyperthreading) is enabled. */ +bool smt_on(const struct cpu_topology *topology); + +/* + * Returns true when system wide and all SMT threads for a core are in the + * user_requested_cpus map. + */ +bool core_wide(bool system_wide, const char *user_requested_cpu_list, + const struct cpu_topology *topology); + +#endif /* __SMT_H */ diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 6d5588e80935a8255131c5e287243fb8ce22ad85..2e7330867e2efd42a23a3f8bbef6b5aacb23575e 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -1948,6 +1948,43 @@ struct sort_entry sort_dso_size = { .se_width_idx = HISTC_DSO_SIZE, }; +/* --sort dso_size */ + +static int64_t +sort__addr_cmp(struct hist_entry *left, struct hist_entry *right) +{ + u64 left_ip = left->ip; + u64 right_ip = right->ip; + struct map *left_map = left->ms.map; + struct map *right_map = right->ms.map; + + if (left_map) + left_ip = left_map->unmap_ip(left_map, left_ip); + if (right_map) + right_ip = right_map->unmap_ip(right_map, right_ip); + + return _sort__addr_cmp(left_ip, right_ip); +} + +static int hist_entry__addr_snprintf(struct hist_entry *he, char *bf, + size_t size, unsigned int width) +{ + u64 ip = he->ip; + struct map *map = he->ms.map; + + if (map) + ip = map->unmap_ip(map, ip); + + return repsep_snprintf(bf, size, "%-#*llx", width, ip); +} + +struct sort_entry sort_addr = { + .se_header = "Address", + .se_cmp = sort__addr_cmp, + .se_snprintf = hist_entry__addr_snprintf, + .se_width_idx = HISTC_ADDR, +}; + struct sort_dimension { const char *name; @@ -1997,6 +2034,7 @@ static struct sort_dimension common_sort_dimensions[] = { DIM(SORT_GLOBAL_INS_LAT, "ins_lat", sort_global_ins_lat), DIM(SORT_LOCAL_PIPELINE_STAGE_CYC, "local_p_stage_cyc", sort_local_p_stage_cyc), DIM(SORT_GLOBAL_PIPELINE_STAGE_CYC, "p_stage_cyc", sort_global_p_stage_cyc), + DIM(SORT_ADDR, "addr", sort_addr), }; #undef DIM diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h index 2ddc00d1c464578689aee6c6c506b18f3340c7ca..04ff8b61a2a7c75c82be89758d4939fe914fd9d0 100644 --- a/tools/perf/util/sort.h +++ b/tools/perf/util/sort.h @@ -34,7 +34,6 @@ extern struct sort_entry sort_dso_to; extern struct sort_entry sort_sym_from; extern struct sort_entry sort_sym_to; extern struct sort_entry sort_srcline; -extern enum sort_type sort__first_dimension; extern const char default_mem_sort_order[]; struct res_sample { @@ -237,6 +236,7 @@ enum sort_type { SORT_GLOBAL_INS_LAT, SORT_LOCAL_PIPELINE_STAGE_CYC, SORT_GLOBAL_PIPELINE_STAGE_CYC, + SORT_ADDR, /* branch stack specific sort keys */ __SORT_BRANCH_STACK, @@ -295,7 +295,6 @@ struct block_hist { }; extern struct sort_entry sort_thread; -extern struct list_head hist_entry__sort_list; struct evlist; struct tep_handle; diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index b82844cb0ce77845ec1480def525e841fa65acb6..5c47ee9963a7c04ca4419d5e240531eac8778d22 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -67,7 +67,7 @@ static void print_noise(struct perf_stat_config *config, return; ps = evsel->stats; - print_noise_pct(config, stddev_stats(&ps->res_stats[0]), avg); + print_noise_pct(config, stddev_stats(&ps->res_stats), avg); } static void print_cgroup(struct perf_stat_config *config, struct evsel *evsel) @@ -168,7 +168,7 @@ static void aggr_printout(struct perf_stat_config *config, id.socket, id.die, id.core); - } else if (id.core > -1) { + } else if (id.cpu.cpu > -1) { fprintf(config->output, "\"cpu\" : \"%d\", ", id.cpu.cpu); } @@ -179,7 +179,7 @@ static void aggr_printout(struct perf_stat_config *config, id.die, config->csv_output ? 0 : -3, id.core, config->csv_sep); - } else if (id.core > -1) { + } else if (id.cpu.cpu > -1) { fprintf(config->output, "CPU%*d%s", config->csv_output ? 0 : -7, id.cpu.cpu, config->csv_sep); @@ -189,14 +189,14 @@ static void aggr_printout(struct perf_stat_config *config, case AGGR_THREAD: if (config->json_output) { fprintf(config->output, "\"thread\" : \"%s-%d\", ", - perf_thread_map__comm(evsel->core.threads, id.thread), - perf_thread_map__pid(evsel->core.threads, id.thread)); + perf_thread_map__comm(evsel->core.threads, id.thread_idx), + perf_thread_map__pid(evsel->core.threads, id.thread_idx)); } else { fprintf(config->output, "%*s-%*d%s", config->csv_output ? 0 : 16, - perf_thread_map__comm(evsel->core.threads, id.thread), + perf_thread_map__comm(evsel->core.threads, id.thread_idx), config->csv_output ? 0 : -8, - perf_thread_map__pid(evsel->core.threads, id.thread), + perf_thread_map__pid(evsel->core.threads, id.thread_idx), config->csv_sep); } break; @@ -442,7 +442,7 @@ static void print_metric_header(struct perf_stat_config *config, fprintf(os->fh, "%*s ", config->metric_only_len, unit); } -static int first_shadow_cpu_map_idx(struct perf_stat_config *config, +static int first_shadow_map_idx(struct perf_stat_config *config, struct evsel *evsel, const struct aggr_cpu_id *id) { struct perf_cpu_map *cpus = evsel__cpus(evsel); @@ -452,6 +452,9 @@ static int first_shadow_cpu_map_idx(struct perf_stat_config *config, if (config->aggr_mode == AGGR_NONE) return perf_cpu_map__idx(cpus, id->cpu); + if (config->aggr_mode == AGGR_THREAD) + return id->thread_idx; + if (!config->aggr_get_id) return 0; @@ -646,7 +649,7 @@ static void printout(struct perf_stat_config *config, struct aggr_cpu_id id, int } perf_stat__print_shadow_stats(config, counter, uval, - first_shadow_cpu_map_idx(config, counter, &id), + first_shadow_map_idx(config, counter, &id), &out, &config->metric_events, st); if (!config->csv_output && !config->metric_only && !config->json_output) { print_noise(config, counter, noise); @@ -676,7 +679,7 @@ static void aggr_update_shadow(struct perf_stat_config *config, val += perf_counts(counter->counts, idx, 0)->val; } perf_stat__update_shadow_stats(counter, val, - first_shadow_cpu_map_idx(config, counter, &id), + first_shadow_map_idx(config, counter, &id), &rt_stat); } } @@ -943,7 +946,7 @@ static struct perf_aggr_thread_value *sort_aggr_thread( buf[i].counter = counter; buf[i].id = aggr_cpu_id__empty(); - buf[i].id.thread = thread; + buf[i].id.thread_idx = thread; buf[i].uval = uval; buf[i].val = val; buf[i].run = run; @@ -979,14 +982,9 @@ static void print_aggr_thread(struct perf_stat_config *config, fprintf(output, "%s", prefix); id = buf[thread].id; - if (config->stats) - printout(config, id, 0, buf[thread].counter, buf[thread].uval, - prefix, buf[thread].run, buf[thread].ena, 1.0, - &config->stats[id.thread]); - else - printout(config, id, 0, buf[thread].counter, buf[thread].uval, - prefix, buf[thread].run, buf[thread].ena, 1.0, - &rt_stat); + printout(config, id, 0, buf[thread].counter, buf[thread].uval, + prefix, buf[thread].run, buf[thread].ena, 1.0, + &rt_stat); fputc('\n', output); } diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index 979c8cb918f724f3ff71b15b1200667a7363ba38..07b29fe272c79a823e284fe616ca95f56e197ec2 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -33,9 +33,8 @@ struct saved_value { struct evsel *evsel; enum stat_type type; int ctx; - int cpu_map_idx; + int map_idx; /* cpu or thread map index */ struct cgroup *cgrp; - struct runtime_stat *stat; struct stats stats; u64 metric_total; int metric_other; @@ -48,8 +47,8 @@ static int saved_value_cmp(struct rb_node *rb_node, const void *entry) rb_node); const struct saved_value *b = entry; - if (a->cpu_map_idx != b->cpu_map_idx) - return a->cpu_map_idx - b->cpu_map_idx; + if (a->map_idx != b->map_idx) + return a->map_idx - b->map_idx; /* * Previously the rbtree was used to link generic metrics. @@ -67,16 +66,6 @@ static int saved_value_cmp(struct rb_node *rb_node, const void *entry) if (a->cgrp != b->cgrp) return (char *)a->cgrp < (char *)b->cgrp ? -1 : +1; - if (a->evsel == NULL && b->evsel == NULL) { - if (a->stat == b->stat) - return 0; - - if ((char *)a->stat < (char *)b->stat) - return -1; - - return 1; - } - if (a->evsel == b->evsel) return 0; if ((char *)a->evsel < (char *)b->evsel) @@ -106,7 +95,7 @@ static void saved_value_delete(struct rblist *rblist __maybe_unused, } static struct saved_value *saved_value_lookup(struct evsel *evsel, - int cpu_map_idx, + int map_idx, bool create, enum stat_type type, int ctx, @@ -116,11 +105,10 @@ static struct saved_value *saved_value_lookup(struct evsel *evsel, struct rblist *rblist; struct rb_node *nd; struct saved_value dm = { - .cpu_map_idx = cpu_map_idx, + .map_idx = map_idx, .evsel = evsel, .type = type, .ctx = ctx, - .stat = st, .cgrp = cgrp, }; @@ -215,10 +203,10 @@ struct runtime_stat_data { static void update_runtime_stat(struct runtime_stat *st, enum stat_type type, - int cpu_map_idx, u64 count, + int map_idx, u64 count, struct runtime_stat_data *rsd) { - struct saved_value *v = saved_value_lookup(NULL, cpu_map_idx, true, type, + struct saved_value *v = saved_value_lookup(NULL, map_idx, true, type, rsd->ctx, st, rsd->cgrp); if (v) @@ -231,7 +219,7 @@ static void update_runtime_stat(struct runtime_stat *st, * instruction rates, etc: */ void perf_stat__update_shadow_stats(struct evsel *counter, u64 count, - int cpu_map_idx, struct runtime_stat *st) + int map_idx, struct runtime_stat *st) { u64 count_ns = count; struct saved_value *v; @@ -243,88 +231,88 @@ void perf_stat__update_shadow_stats(struct evsel *counter, u64 count, count *= counter->scale; if (evsel__is_clock(counter)) - update_runtime_stat(st, STAT_NSECS, cpu_map_idx, count_ns, &rsd); + update_runtime_stat(st, STAT_NSECS, map_idx, count_ns, &rsd); else if (evsel__match(counter, HARDWARE, HW_CPU_CYCLES)) - update_runtime_stat(st, STAT_CYCLES, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_CYCLES, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, CYCLES_IN_TX)) - update_runtime_stat(st, STAT_CYCLES_IN_TX, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_CYCLES_IN_TX, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TRANSACTION_START)) - update_runtime_stat(st, STAT_TRANSACTION, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_TRANSACTION, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, ELISION_START)) - update_runtime_stat(st, STAT_ELISION, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_ELISION, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_TOTAL_SLOTS)) update_runtime_stat(st, STAT_TOPDOWN_TOTAL_SLOTS, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_ISSUED)) update_runtime_stat(st, STAT_TOPDOWN_SLOTS_ISSUED, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_SLOTS_RETIRED)) update_runtime_stat(st, STAT_TOPDOWN_SLOTS_RETIRED, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_FETCH_BUBBLES)) update_runtime_stat(st, STAT_TOPDOWN_FETCH_BUBBLES, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_RECOVERY_BUBBLES)) update_runtime_stat(st, STAT_TOPDOWN_RECOVERY_BUBBLES, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_RETIRING)) update_runtime_stat(st, STAT_TOPDOWN_RETIRING, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_BAD_SPEC)) update_runtime_stat(st, STAT_TOPDOWN_BAD_SPEC, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_FE_BOUND)) update_runtime_stat(st, STAT_TOPDOWN_FE_BOUND, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_BE_BOUND)) update_runtime_stat(st, STAT_TOPDOWN_BE_BOUND, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_HEAVY_OPS)) update_runtime_stat(st, STAT_TOPDOWN_HEAVY_OPS, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_BR_MISPREDICT)) update_runtime_stat(st, STAT_TOPDOWN_BR_MISPREDICT, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_FETCH_LAT)) update_runtime_stat(st, STAT_TOPDOWN_FETCH_LAT, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, TOPDOWN_MEM_BOUND)) update_runtime_stat(st, STAT_TOPDOWN_MEM_BOUND, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) update_runtime_stat(st, STAT_STALLED_CYCLES_FRONT, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND)) update_runtime_stat(st, STAT_STALLED_CYCLES_BACK, - cpu_map_idx, count, &rsd); + map_idx, count, &rsd); else if (evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS)) - update_runtime_stat(st, STAT_BRANCHES, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_BRANCHES, map_idx, count, &rsd); else if (evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES)) - update_runtime_stat(st, STAT_CACHEREFS, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_CACHEREFS, map_idx, count, &rsd); else if (evsel__match(counter, HW_CACHE, HW_CACHE_L1D)) - update_runtime_stat(st, STAT_L1_DCACHE, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_L1_DCACHE, map_idx, count, &rsd); else if (evsel__match(counter, HW_CACHE, HW_CACHE_L1I)) - update_runtime_stat(st, STAT_L1_ICACHE, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_L1_ICACHE, map_idx, count, &rsd); else if (evsel__match(counter, HW_CACHE, HW_CACHE_LL)) - update_runtime_stat(st, STAT_LL_CACHE, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_LL_CACHE, map_idx, count, &rsd); else if (evsel__match(counter, HW_CACHE, HW_CACHE_DTLB)) - update_runtime_stat(st, STAT_DTLB_CACHE, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_DTLB_CACHE, map_idx, count, &rsd); else if (evsel__match(counter, HW_CACHE, HW_CACHE_ITLB)) - update_runtime_stat(st, STAT_ITLB_CACHE, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_ITLB_CACHE, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, SMI_NUM)) - update_runtime_stat(st, STAT_SMI_NUM, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_SMI_NUM, map_idx, count, &rsd); else if (perf_stat_evsel__is(counter, APERF)) - update_runtime_stat(st, STAT_APERF, cpu_map_idx, count, &rsd); + update_runtime_stat(st, STAT_APERF, map_idx, count, &rsd); if (counter->collect_stat) { - v = saved_value_lookup(counter, cpu_map_idx, true, STAT_NONE, 0, st, + v = saved_value_lookup(counter, map_idx, true, STAT_NONE, 0, st, rsd.cgrp); update_stats(&v->stats, count); if (counter->metric_leader) v->metric_total += count; } else if (counter->metric_leader) { v = saved_value_lookup(counter->metric_leader, - cpu_map_idx, true, STAT_NONE, 0, st, rsd.cgrp); + map_idx, true, STAT_NONE, 0, st, rsd.cgrp); v->metric_total += count; v->metric_other++; } @@ -466,12 +454,12 @@ void perf_stat__collect_metric_expr(struct evlist *evsel_list) } static double runtime_stat_avg(struct runtime_stat *st, - enum stat_type type, int cpu_map_idx, + enum stat_type type, int map_idx, struct runtime_stat_data *rsd) { struct saved_value *v; - v = saved_value_lookup(NULL, cpu_map_idx, false, type, rsd->ctx, st, rsd->cgrp); + v = saved_value_lookup(NULL, map_idx, false, type, rsd->ctx, st, rsd->cgrp); if (!v) return 0.0; @@ -479,12 +467,12 @@ static double runtime_stat_avg(struct runtime_stat *st, } static double runtime_stat_n(struct runtime_stat *st, - enum stat_type type, int cpu_map_idx, + enum stat_type type, int map_idx, struct runtime_stat_data *rsd) { struct saved_value *v; - v = saved_value_lookup(NULL, cpu_map_idx, false, type, rsd->ctx, st, rsd->cgrp); + v = saved_value_lookup(NULL, map_idx, false, type, rsd->ctx, st, rsd->cgrp); if (!v) return 0.0; @@ -492,7 +480,7 @@ static double runtime_stat_n(struct runtime_stat *st, } static void print_stalled_cycles_frontend(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -500,7 +488,7 @@ static void print_stalled_cycles_frontend(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -515,7 +503,7 @@ static void print_stalled_cycles_frontend(struct perf_stat_config *config, } static void print_stalled_cycles_backend(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -523,7 +511,7 @@ static void print_stalled_cycles_backend(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -534,7 +522,7 @@ static void print_stalled_cycles_backend(struct perf_stat_config *config, } static void print_branch_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -542,7 +530,7 @@ static void print_branch_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_BRANCHES, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_BRANCHES, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -553,7 +541,7 @@ static void print_branch_misses(struct perf_stat_config *config, } static void print_l1_dcache_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -561,7 +549,7 @@ static void print_l1_dcache_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_L1_DCACHE, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_L1_DCACHE, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -572,7 +560,7 @@ static void print_l1_dcache_misses(struct perf_stat_config *config, } static void print_l1_icache_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -580,7 +568,7 @@ static void print_l1_icache_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_L1_ICACHE, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_L1_ICACHE, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -590,7 +578,7 @@ static void print_l1_icache_misses(struct perf_stat_config *config, } static void print_dtlb_cache_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -598,7 +586,7 @@ static void print_dtlb_cache_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_DTLB_CACHE, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_DTLB_CACHE, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -608,7 +596,7 @@ static void print_dtlb_cache_misses(struct perf_stat_config *config, } static void print_itlb_cache_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -616,7 +604,7 @@ static void print_itlb_cache_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_ITLB_CACHE, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_ITLB_CACHE, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -626,7 +614,7 @@ static void print_itlb_cache_misses(struct perf_stat_config *config, } static void print_ll_cache_misses(struct perf_stat_config *config, - int cpu_map_idx, double avg, + int map_idx, double avg, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -634,7 +622,7 @@ static void print_ll_cache_misses(struct perf_stat_config *config, double total, ratio = 0.0; const char *color; - total = runtime_stat_avg(st, STAT_LL_CACHE, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_LL_CACHE, map_idx, rsd); if (total) ratio = avg / total * 100.0; @@ -692,61 +680,61 @@ static double sanitize_val(double x) return x; } -static double td_total_slots(int cpu_map_idx, struct runtime_stat *st, +static double td_total_slots(int map_idx, struct runtime_stat *st, struct runtime_stat_data *rsd) { - return runtime_stat_avg(st, STAT_TOPDOWN_TOTAL_SLOTS, cpu_map_idx, rsd); + return runtime_stat_avg(st, STAT_TOPDOWN_TOTAL_SLOTS, map_idx, rsd); } -static double td_bad_spec(int cpu_map_idx, struct runtime_stat *st, +static double td_bad_spec(int map_idx, struct runtime_stat *st, struct runtime_stat_data *rsd) { double bad_spec = 0; double total_slots; double total; - total = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_ISSUED, cpu_map_idx, rsd) - - runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, cpu_map_idx, rsd) + - runtime_stat_avg(st, STAT_TOPDOWN_RECOVERY_BUBBLES, cpu_map_idx, rsd); + total = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_ISSUED, map_idx, rsd) - + runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, map_idx, rsd) + + runtime_stat_avg(st, STAT_TOPDOWN_RECOVERY_BUBBLES, map_idx, rsd); - total_slots = td_total_slots(cpu_map_idx, st, rsd); + total_slots = td_total_slots(map_idx, st, rsd); if (total_slots) bad_spec = total / total_slots; return sanitize_val(bad_spec); } -static double td_retiring(int cpu_map_idx, struct runtime_stat *st, +static double td_retiring(int map_idx, struct runtime_stat *st, struct runtime_stat_data *rsd) { double retiring = 0; - double total_slots = td_total_slots(cpu_map_idx, st, rsd); + double total_slots = td_total_slots(map_idx, st, rsd); double ret_slots = runtime_stat_avg(st, STAT_TOPDOWN_SLOTS_RETIRED, - cpu_map_idx, rsd); + map_idx, rsd); if (total_slots) retiring = ret_slots / total_slots; return retiring; } -static double td_fe_bound(int cpu_map_idx, struct runtime_stat *st, +static double td_fe_bound(int map_idx, struct runtime_stat *st, struct runtime_stat_data *rsd) { double fe_bound = 0; - double total_slots = td_total_slots(cpu_map_idx, st, rsd); + double total_slots = td_total_slots(map_idx, st, rsd); double fetch_bub = runtime_stat_avg(st, STAT_TOPDOWN_FETCH_BUBBLES, - cpu_map_idx, rsd); + map_idx, rsd); if (total_slots) fe_bound = fetch_bub / total_slots; return fe_bound; } -static double td_be_bound(int cpu_map_idx, struct runtime_stat *st, +static double td_be_bound(int map_idx, struct runtime_stat *st, struct runtime_stat_data *rsd) { - double sum = (td_fe_bound(cpu_map_idx, st, rsd) + - td_bad_spec(cpu_map_idx, st, rsd) + - td_retiring(cpu_map_idx, st, rsd)); + double sum = (td_fe_bound(map_idx, st, rsd) + + td_bad_spec(map_idx, st, rsd) + + td_retiring(map_idx, st, rsd)); if (sum == 0) return 0; return sanitize_val(1.0 - sum); @@ -757,15 +745,15 @@ static double td_be_bound(int cpu_map_idx, struct runtime_stat *st, * the ratios we need to recreate the sum. */ -static double td_metric_ratio(int cpu_map_idx, enum stat_type type, +static double td_metric_ratio(int map_idx, enum stat_type type, struct runtime_stat *stat, struct runtime_stat_data *rsd) { - double sum = runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, cpu_map_idx, rsd) + - runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, cpu_map_idx, rsd) + - runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, cpu_map_idx, rsd) + - runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, cpu_map_idx, rsd); - double d = runtime_stat_avg(stat, type, cpu_map_idx, rsd); + double sum = runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, map_idx, rsd) + + runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, map_idx, rsd) + + runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, map_idx, rsd) + + runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, map_idx, rsd); + double d = runtime_stat_avg(stat, type, map_idx, rsd); if (sum) return d / sum; @@ -777,23 +765,23 @@ static double td_metric_ratio(int cpu_map_idx, enum stat_type type, * We allow two missing. */ -static bool full_td(int cpu_map_idx, struct runtime_stat *stat, +static bool full_td(int map_idx, struct runtime_stat *stat, struct runtime_stat_data *rsd) { int c = 0; - if (runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, cpu_map_idx, rsd) > 0) + if (runtime_stat_avg(stat, STAT_TOPDOWN_RETIRING, map_idx, rsd) > 0) c++; - if (runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, cpu_map_idx, rsd) > 0) + if (runtime_stat_avg(stat, STAT_TOPDOWN_BE_BOUND, map_idx, rsd) > 0) c++; - if (runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, cpu_map_idx, rsd) > 0) + if (runtime_stat_avg(stat, STAT_TOPDOWN_FE_BOUND, map_idx, rsd) > 0) c++; - if (runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, cpu_map_idx, rsd) > 0) + if (runtime_stat_avg(stat, STAT_TOPDOWN_BAD_SPEC, map_idx, rsd) > 0) c++; return c >= 2; } -static void print_smi_cost(struct perf_stat_config *config, int cpu_map_idx, +static void print_smi_cost(struct perf_stat_config *config, int map_idx, struct perf_stat_output_ctx *out, struct runtime_stat *st, struct runtime_stat_data *rsd) @@ -801,9 +789,9 @@ static void print_smi_cost(struct perf_stat_config *config, int cpu_map_idx, double smi_num, aperf, cycles, cost = 0.0; const char *color = NULL; - smi_num = runtime_stat_avg(st, STAT_SMI_NUM, cpu_map_idx, rsd); - aperf = runtime_stat_avg(st, STAT_APERF, cpu_map_idx, rsd); - cycles = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, rsd); + smi_num = runtime_stat_avg(st, STAT_SMI_NUM, map_idx, rsd); + aperf = runtime_stat_avg(st, STAT_APERF, map_idx, rsd); + cycles = runtime_stat_avg(st, STAT_CYCLES, map_idx, rsd); if ((cycles == 0) || (aperf == 0)) return; @@ -820,7 +808,7 @@ static void print_smi_cost(struct perf_stat_config *config, int cpu_map_idx, static int prepare_metric(struct evsel **metric_events, struct metric_ref *metric_refs, struct expr_parse_ctx *pctx, - int cpu_map_idx, + int map_idx, struct runtime_stat *st) { double scale; @@ -859,17 +847,22 @@ static int prepare_metric(struct evsel **metric_events, abort(); } } else { - v = saved_value_lookup(metric_events[i], cpu_map_idx, false, + v = saved_value_lookup(metric_events[i], map_idx, false, STAT_NONE, 0, st, metric_events[i]->cgrp); if (!v) break; stats = &v->stats; - scale = 1.0; + /* + * If an event was scaled during stat gathering, reverse + * the scale before computing the metric. + */ + scale = 1.0 / metric_events[i]->scale; + source_count = evsel__source_count(metric_events[i]); if (v->metric_other) - metric_total = v->metric_total; + metric_total = v->metric_total * scale; } n = strdup(evsel__metric_id(metric_events[i])); if (!n) @@ -897,7 +890,7 @@ static void generic_metric(struct perf_stat_config *config, const char *metric_name, const char *metric_unit, int runtime, - int cpu_map_idx, + int map_idx, struct perf_stat_output_ctx *out, struct runtime_stat *st) { @@ -911,8 +904,11 @@ static void generic_metric(struct perf_stat_config *config, if (!pctx) return; - pctx->runtime = runtime; - i = prepare_metric(metric_events, metric_refs, pctx, cpu_map_idx, st); + if (config->user_requested_cpu_list) + pctx->sctx.user_requested_cpu_list = strdup(config->user_requested_cpu_list); + pctx->sctx.runtime = runtime; + pctx->sctx.system_wide = config->system_wide; + i = prepare_metric(metric_events, metric_refs, pctx, map_idx, st); if (i < 0) { expr__ctx_free(pctx); return; @@ -957,7 +953,7 @@ static void generic_metric(struct perf_stat_config *config, expr__ctx_free(pctx); } -double test_generic_metric(struct metric_expr *mexp, int cpu_map_idx, struct runtime_stat *st) +double test_generic_metric(struct metric_expr *mexp, int map_idx, struct runtime_stat *st) { struct expr_parse_ctx *pctx; double ratio = 0.0; @@ -966,7 +962,7 @@ double test_generic_metric(struct metric_expr *mexp, int cpu_map_idx, struct run if (!pctx) return NAN; - if (prepare_metric(mexp->metric_events, mexp->metric_refs, pctx, cpu_map_idx, st) < 0) + if (prepare_metric(mexp->metric_events, mexp->metric_refs, pctx, map_idx, st) < 0) goto out; if (expr__parse(&ratio, pctx, mexp->metric_expr)) @@ -979,7 +975,7 @@ out: void perf_stat__print_shadow_stats(struct perf_stat_config *config, struct evsel *evsel, - double avg, int cpu_map_idx, + double avg, int map_idx, struct perf_stat_output_ctx *out, struct rblist *metric_events, struct runtime_stat *st) @@ -998,7 +994,7 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, if (config->iostat_run) { iostat_print_metric(config, evsel, out); } else if (evsel__match(evsel, HARDWARE, HW_INSTRUCTIONS)) { - total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd); if (total) { ratio = avg / total; @@ -1008,11 +1004,11 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, print_metric(config, ctxp, NULL, NULL, "insn per cycle", 0); } - total = runtime_stat_avg(st, STAT_STALLED_CYCLES_FRONT, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_STALLED_CYCLES_FRONT, map_idx, &rsd); total = max(total, runtime_stat_avg(st, STAT_STALLED_CYCLES_BACK, - cpu_map_idx, &rsd)); + map_idx, &rsd)); if (total && avg) { out->new_line(config, ctxp); @@ -1022,8 +1018,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ratio); } } else if (evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES)) { - if (runtime_stat_n(st, STAT_BRANCHES, cpu_map_idx, &rsd) != 0) - print_branch_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_BRANCHES, map_idx, &rsd) != 0) + print_branch_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all branches", 0); } else if ( @@ -1032,8 +1028,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) { - if (runtime_stat_n(st, STAT_L1_DCACHE, cpu_map_idx, &rsd) != 0) - print_l1_dcache_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_L1_DCACHE, map_idx, &rsd) != 0) + print_l1_dcache_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all L1-dcache accesses", 0); } else if ( @@ -1042,8 +1038,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) { - if (runtime_stat_n(st, STAT_L1_ICACHE, cpu_map_idx, &rsd) != 0) - print_l1_icache_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_L1_ICACHE, map_idx, &rsd) != 0) + print_l1_icache_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all L1-icache accesses", 0); } else if ( @@ -1052,8 +1048,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) { - if (runtime_stat_n(st, STAT_DTLB_CACHE, cpu_map_idx, &rsd) != 0) - print_dtlb_cache_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_DTLB_CACHE, map_idx, &rsd) != 0) + print_dtlb_cache_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all dTLB cache accesses", 0); } else if ( @@ -1062,8 +1058,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) { - if (runtime_stat_n(st, STAT_ITLB_CACHE, cpu_map_idx, &rsd) != 0) - print_itlb_cache_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_ITLB_CACHE, map_idx, &rsd) != 0) + print_itlb_cache_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all iTLB cache accesses", 0); } else if ( @@ -1072,27 +1068,27 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, ((PERF_COUNT_HW_CACHE_OP_READ) << 8) | ((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16))) { - if (runtime_stat_n(st, STAT_LL_CACHE, cpu_map_idx, &rsd) != 0) - print_ll_cache_misses(config, cpu_map_idx, avg, out, st, &rsd); + if (runtime_stat_n(st, STAT_LL_CACHE, map_idx, &rsd) != 0) + print_ll_cache_misses(config, map_idx, avg, out, st, &rsd); else print_metric(config, ctxp, NULL, NULL, "of all LL-cache accesses", 0); } else if (evsel__match(evsel, HARDWARE, HW_CACHE_MISSES)) { - total = runtime_stat_avg(st, STAT_CACHEREFS, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CACHEREFS, map_idx, &rsd); if (total) ratio = avg * 100 / total; - if (runtime_stat_n(st, STAT_CACHEREFS, cpu_map_idx, &rsd) != 0) + if (runtime_stat_n(st, STAT_CACHEREFS, map_idx, &rsd) != 0) print_metric(config, ctxp, NULL, "%8.3f %%", "of all cache refs", ratio); else print_metric(config, ctxp, NULL, NULL, "of all cache refs", 0); } else if (evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) { - print_stalled_cycles_frontend(config, cpu_map_idx, avg, out, st, &rsd); + print_stalled_cycles_frontend(config, map_idx, avg, out, st, &rsd); } else if (evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) { - print_stalled_cycles_backend(config, cpu_map_idx, avg, out, st, &rsd); + print_stalled_cycles_backend(config, map_idx, avg, out, st, &rsd); } else if (evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) { - total = runtime_stat_avg(st, STAT_NSECS, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_NSECS, map_idx, &rsd); if (total) { ratio = avg / total; @@ -1101,7 +1097,7 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, print_metric(config, ctxp, NULL, NULL, "Ghz", 0); } } else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX)) { - total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd); if (total) print_metric(config, ctxp, NULL, @@ -1111,8 +1107,8 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, print_metric(config, ctxp, NULL, NULL, "transactional cycles", 0); } else if (perf_stat_evsel__is(evsel, CYCLES_IN_TX_CP)) { - total = runtime_stat_avg(st, STAT_CYCLES, cpu_map_idx, &rsd); - total2 = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CYCLES, map_idx, &rsd); + total2 = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd); if (total2 < avg) total2 = avg; @@ -1122,19 +1118,19 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, else print_metric(config, ctxp, NULL, NULL, "aborted cycles", 0); } else if (perf_stat_evsel__is(evsel, TRANSACTION_START)) { - total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd); if (avg) ratio = total / avg; - if (runtime_stat_n(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd) != 0) + if (runtime_stat_n(st, STAT_CYCLES_IN_TX, map_idx, &rsd) != 0) print_metric(config, ctxp, NULL, "%8.0f", "cycles / transaction", ratio); else print_metric(config, ctxp, NULL, NULL, "cycles / transaction", 0); } else if (perf_stat_evsel__is(evsel, ELISION_START)) { - total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_CYCLES_IN_TX, map_idx, &rsd); if (avg) ratio = total / avg; @@ -1147,28 +1143,28 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, else print_metric(config, ctxp, NULL, NULL, "CPUs utilized", 0); } else if (perf_stat_evsel__is(evsel, TOPDOWN_FETCH_BUBBLES)) { - double fe_bound = td_fe_bound(cpu_map_idx, st, &rsd); + double fe_bound = td_fe_bound(map_idx, st, &rsd); if (fe_bound > 0.2) color = PERF_COLOR_RED; print_metric(config, ctxp, color, "%8.1f%%", "frontend bound", fe_bound * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_RETIRED)) { - double retiring = td_retiring(cpu_map_idx, st, &rsd); + double retiring = td_retiring(map_idx, st, &rsd); if (retiring > 0.7) color = PERF_COLOR_GREEN; print_metric(config, ctxp, color, "%8.1f%%", "retiring", retiring * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_RECOVERY_BUBBLES)) { - double bad_spec = td_bad_spec(cpu_map_idx, st, &rsd); + double bad_spec = td_bad_spec(map_idx, st, &rsd); if (bad_spec > 0.1) color = PERF_COLOR_RED; print_metric(config, ctxp, color, "%8.1f%%", "bad speculation", bad_spec * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_SLOTS_ISSUED)) { - double be_bound = td_be_bound(cpu_map_idx, st, &rsd); + double be_bound = td_be_bound(map_idx, st, &rsd); const char *name = "backend bound"; static int have_recovery_bubbles = -1; @@ -1181,135 +1177,136 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, if (be_bound > 0.2) color = PERF_COLOR_RED; - if (td_total_slots(cpu_map_idx, st, &rsd) > 0) + if (td_total_slots(map_idx, st, &rsd) > 0) print_metric(config, ctxp, color, "%8.1f%%", name, be_bound * 100.); else print_metric(config, ctxp, NULL, NULL, name, 0); } else if (perf_stat_evsel__is(evsel, TOPDOWN_RETIRING) && - full_td(cpu_map_idx, st, &rsd)) { - double retiring = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd)) { + double retiring = td_metric_ratio(map_idx, STAT_TOPDOWN_RETIRING, st, &rsd); if (retiring > 0.7) color = PERF_COLOR_GREEN; - print_metric(config, ctxp, color, "%8.1f%%", "retiring", + print_metric(config, ctxp, color, "%8.1f%%", "Retiring", retiring * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_FE_BOUND) && - full_td(cpu_map_idx, st, &rsd)) { - double fe_bound = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd)) { + double fe_bound = td_metric_ratio(map_idx, STAT_TOPDOWN_FE_BOUND, st, &rsd); if (fe_bound > 0.2) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "frontend bound", + print_metric(config, ctxp, color, "%8.1f%%", "Frontend Bound", fe_bound * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_BE_BOUND) && - full_td(cpu_map_idx, st, &rsd)) { - double be_bound = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd)) { + double be_bound = td_metric_ratio(map_idx, STAT_TOPDOWN_BE_BOUND, st, &rsd); if (be_bound > 0.2) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "backend bound", + print_metric(config, ctxp, color, "%8.1f%%", "Backend Bound", be_bound * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_BAD_SPEC) && - full_td(cpu_map_idx, st, &rsd)) { - double bad_spec = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd)) { + double bad_spec = td_metric_ratio(map_idx, STAT_TOPDOWN_BAD_SPEC, st, &rsd); if (bad_spec > 0.1) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "bad speculation", + print_metric(config, ctxp, color, "%8.1f%%", "Bad Speculation", bad_spec * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_HEAVY_OPS) && - full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) { - double retiring = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) { + double retiring = td_metric_ratio(map_idx, STAT_TOPDOWN_RETIRING, st, &rsd); - double heavy_ops = td_metric_ratio(cpu_map_idx, + double heavy_ops = td_metric_ratio(map_idx, STAT_TOPDOWN_HEAVY_OPS, st, &rsd); double light_ops = retiring - heavy_ops; if (retiring > 0.7 && heavy_ops > 0.1) color = PERF_COLOR_GREEN; - print_metric(config, ctxp, color, "%8.1f%%", "heavy operations", + print_metric(config, ctxp, color, "%8.1f%%", "Heavy Operations", heavy_ops * 100.); if (retiring > 0.7 && light_ops > 0.6) color = PERF_COLOR_GREEN; else color = NULL; - print_metric(config, ctxp, color, "%8.1f%%", "light operations", + print_metric(config, ctxp, color, "%8.1f%%", "Light Operations", light_ops * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_BR_MISPREDICT) && - full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) { - double bad_spec = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) { + double bad_spec = td_metric_ratio(map_idx, STAT_TOPDOWN_BAD_SPEC, st, &rsd); - double br_mis = td_metric_ratio(cpu_map_idx, + double br_mis = td_metric_ratio(map_idx, STAT_TOPDOWN_BR_MISPREDICT, st, &rsd); double m_clears = bad_spec - br_mis; if (bad_spec > 0.1 && br_mis > 0.05) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "branch mispredict", + print_metric(config, ctxp, color, "%8.1f%%", "Branch Mispredict", br_mis * 100.); if (bad_spec > 0.1 && m_clears > 0.05) color = PERF_COLOR_RED; else color = NULL; - print_metric(config, ctxp, color, "%8.1f%%", "machine clears", + print_metric(config, ctxp, color, "%8.1f%%", "Machine Clears", m_clears * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_FETCH_LAT) && - full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) { - double fe_bound = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) { + double fe_bound = td_metric_ratio(map_idx, STAT_TOPDOWN_FE_BOUND, st, &rsd); - double fetch_lat = td_metric_ratio(cpu_map_idx, + double fetch_lat = td_metric_ratio(map_idx, STAT_TOPDOWN_FETCH_LAT, st, &rsd); double fetch_bw = fe_bound - fetch_lat; if (fe_bound > 0.2 && fetch_lat > 0.15) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "fetch latency", + print_metric(config, ctxp, color, "%8.1f%%", "Fetch Latency", fetch_lat * 100.); if (fe_bound > 0.2 && fetch_bw > 0.1) color = PERF_COLOR_RED; else color = NULL; - print_metric(config, ctxp, color, "%8.1f%%", "fetch bandwidth", + print_metric(config, ctxp, color, "%8.1f%%", "Fetch Bandwidth", fetch_bw * 100.); } else if (perf_stat_evsel__is(evsel, TOPDOWN_MEM_BOUND) && - full_td(cpu_map_idx, st, &rsd) && (config->topdown_level > 1)) { - double be_bound = td_metric_ratio(cpu_map_idx, + full_td(map_idx, st, &rsd) && (config->topdown_level > 1)) { + double be_bound = td_metric_ratio(map_idx, STAT_TOPDOWN_BE_BOUND, st, &rsd); - double mem_bound = td_metric_ratio(cpu_map_idx, + double mem_bound = td_metric_ratio(map_idx, STAT_TOPDOWN_MEM_BOUND, st, &rsd); double core_bound = be_bound - mem_bound; if (be_bound > 0.2 && mem_bound > 0.2) color = PERF_COLOR_RED; - print_metric(config, ctxp, color, "%8.1f%%", "memory bound", + print_metric(config, ctxp, color, "%8.1f%%", "Memory Bound", mem_bound * 100.); if (be_bound > 0.2 && core_bound > 0.1) color = PERF_COLOR_RED; else color = NULL; - print_metric(config, ctxp, color, "%8.1f%%", "Core bound", + print_metric(config, ctxp, color, "%8.1f%%", "Core Bound", core_bound * 100.); } else if (evsel->metric_expr) { generic_metric(config, evsel->metric_expr, evsel->metric_events, NULL, - evsel->name, evsel->metric_name, NULL, 1, cpu_map_idx, out, st); - } else if (runtime_stat_n(st, STAT_NSECS, cpu_map_idx, &rsd) != 0) { + evsel->name, evsel->metric_name, NULL, 1, + map_idx, out, st); + } else if (runtime_stat_n(st, STAT_NSECS, map_idx, &rsd) != 0) { char unit = ' '; char unit_buf[10] = "/sec"; - total = runtime_stat_avg(st, STAT_NSECS, cpu_map_idx, &rsd); + total = runtime_stat_avg(st, STAT_NSECS, map_idx, &rsd); if (total) ratio = convert_unit_double(1000000000.0 * avg / total, &unit); @@ -1317,7 +1314,7 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, snprintf(unit_buf, sizeof(unit_buf), "%c/sec", unit); print_metric(config, ctxp, NULL, "%8.3f", unit_buf, ratio); } else if (perf_stat_evsel__is(evsel, SMI_NUM)) { - print_smi_cost(config, cpu_map_idx, out, st, &rsd); + print_smi_cost(config, map_idx, out, st, &rsd); } else { num = 0; } @@ -1329,8 +1326,9 @@ void perf_stat__print_shadow_stats(struct perf_stat_config *config, if (num++ > 0) out->new_line(config, ctxp); generic_metric(config, mexp->metric_expr, mexp->metric_events, - mexp->metric_refs, evsel->name, mexp->metric_name, - mexp->metric_unit, mexp->runtime, cpu_map_idx, out, st); + mexp->metric_refs, evsel->name, mexp->metric_name, + mexp->metric_unit, mexp->runtime, + map_idx, out, st); } } if (num == 0) diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 0882b4754fcf1f4e238a8ffaaf1c50d271ff8475..8ec8bb4a99129b19176cae155e0ff9e5dd6e900d 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -14,7 +14,11 @@ #include "evlist.h" #include "evsel.h" #include "thread_map.h" -#include "hashmap.h" +#ifdef HAVE_LIBBPF_SUPPORT +#include <bpf/hashmap.h> +#else +#include "util/hashmap.h" +#endif #include <linux/zalloc.h> void update_stats(struct stats *stats, u64 val) @@ -128,13 +132,9 @@ static void perf_stat_evsel_id_init(struct evsel *evsel) static void evsel__reset_stat_priv(struct evsel *evsel) { - int i; struct perf_stat_evsel *ps = evsel->stats; - for (i = 0; i < 3; i++) - init_stats(&ps->res_stats[i]); - - perf_stat_evsel_id_init(evsel); + init_stats(&ps->res_stats); } static int evsel__alloc_stat_priv(struct evsel *evsel) @@ -142,6 +142,7 @@ static int evsel__alloc_stat_priv(struct evsel *evsel) evsel->stats = zalloc(sizeof(struct perf_stat_evsel)); if (evsel->stats == NULL) return -ENOMEM; + perf_stat_evsel_id_init(evsel); evsel__reset_stat_priv(evsel); return 0; } @@ -388,12 +389,8 @@ process_counter_values(struct perf_stat_config *config, struct evsel *evsel, } if (config->aggr_mode == AGGR_THREAD) { - if (config->stats) - perf_stat__update_shadow_stats(evsel, - count->val, 0, &config->stats[thread]); - else - perf_stat__update_shadow_stats(evsel, - count->val, 0, &rt_stat); + perf_stat__update_shadow_stats(evsel, count->val, + thread, &rt_stat); } break; case AGGR_GLOBAL: @@ -416,9 +413,6 @@ static int process_counter_maps(struct perf_stat_config *config, int ncpus = evsel__nr_cpus(counter); int idx, thread; - if (counter->core.system_wide) - nthreads = 1; - for (thread = 0; thread < nthreads; thread++) { for (idx = 0; idx < ncpus; idx++) { if (process_counter_values(config, counter, idx, thread, @@ -436,7 +430,7 @@ int perf_stat_process_counter(struct perf_stat_config *config, struct perf_counts_values *aggr = &counter->counts->aggr; struct perf_stat_evsel *ps = counter->stats; u64 *count = counter->counts->aggr.values; - int i, ret; + int ret; aggr->val = aggr->ena = aggr->run = 0; @@ -454,8 +448,7 @@ int perf_stat_process_counter(struct perf_stat_config *config, evsel__compute_deltas(counter, -1, -1, aggr); perf_counts_values__scale(aggr, config->scale, &counter->counts->scaled); - for (i = 0; i < 3; i++) - update_stats(&ps->res_stats[i], count[i]); + update_stats(&ps->res_stats, *count); if (verbose > 0) { fprintf(config->output, "%s: %" PRIu64 " %" PRIu64 " %" PRIu64 "\n", diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h index 668250022f8cae90fe60fca6535eb8268be66168..b0899c6e002f50bf18368489b35258028d54c786 100644 --- a/tools/perf/util/stat.h +++ b/tools/perf/util/stat.h @@ -43,7 +43,7 @@ enum perf_stat_evsel_id { }; struct perf_stat_evsel { - struct stats res_stats[3]; + struct stats res_stats; enum perf_stat_evsel_id id; u64 *group_data; }; @@ -141,6 +141,8 @@ struct perf_stat_config { bool stop_read_counter; bool quiet; bool iostat_run; + char *user_requested_cpu_list; + bool system_wide; FILE *output; unsigned int interval; unsigned int timeout; @@ -151,8 +153,6 @@ struct perf_stat_config { int run_count; int print_free_counters_hint; int print_mixed_hw_group_error; - struct runtime_stat *stats; - int stats_num; const char *csv_sep; struct stats *walltime_nsecs_stats; struct rusage ru_data; @@ -232,7 +232,7 @@ void perf_stat__init_shadow_stats(void); void perf_stat__reset_shadow_stats(void); void perf_stat__reset_shadow_per_stat(struct runtime_stat *st); void perf_stat__update_shadow_stats(struct evsel *counter, u64 count, - int cpu_map_idx, struct runtime_stat *st); + int map_idx, struct runtime_stat *st); struct perf_stat_output_ctx { void *ctx; print_metric_t print_metric; @@ -242,7 +242,7 @@ struct perf_stat_output_ctx { void perf_stat__print_shadow_stats(struct perf_stat_config *config, struct evsel *evsel, - double avg, int cpu, + double avg, int map_idx, struct perf_stat_output_ctx *out, struct rblist *metric_events, struct runtime_stat *st); @@ -277,5 +277,5 @@ void evlist__print_counters(struct evlist *evlist, struct perf_stat_config *conf struct target *_target, struct timespec *ts, int argc, const char **argv); struct metric_expr; -double test_generic_metric(struct metric_expr *mexp, int cpu_map_idx, struct runtime_stat *st); +double test_generic_metric(struct metric_expr *mexp, int map_idx, struct runtime_stat *st); #endif diff --git a/tools/perf/util/string.c b/tools/perf/util/string.c index f6d90cdd922534fd8caba6e690e738e8f5467818..4f12a96f33cc404ca532112159eeef6a6eda0d39 100644 --- a/tools/perf/util/string.c +++ b/tools/perf/util/string.c @@ -15,7 +15,6 @@ const char *dots = "....................................................................." "....................................................................."; -#define K 1024LL /* * perf_atoll() * Parse (\d+)(b|B|kb|KB|mb|MB|gb|GB|tb|TB) (e.g. "256MB") diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 75bec32d4f571319e8577e96899dee2a24bb4381..647b7dff8ef36f6bf245df547330fc7cd408e541 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -2102,8 +2102,8 @@ static int kcore_copy__compare_file(const char *from_dir, const char *to_dir, * unusual. One significant peculiarity is that the mapping (start -> pgoff) * is not the same for the kernel map and the modules map. That happens because * the data is copied adjacently whereas the original kcore has gaps. Finally, - * kallsyms and modules files are compared with their copies to check that - * modules have not been loaded or unloaded while the copies were taking place. + * kallsyms file is compared with its copy to check that modules have not been + * loaded or unloaded while the copies were taking place. * * Return: %0 on success, %-1 on failure. */ @@ -2166,9 +2166,6 @@ int kcore_copy(const char *from_dir, const char *to_dir) goto out_extract_close; } - if (kcore_copy__compare_file(from_dir, to_dir, "modules")) - goto out_extract_close; - if (kcore_copy__compare_file(from_dir, to_dir, "kallsyms")) goto out_extract_close; diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index a4b22caa7c24c334bc0ff1af840e3d41d0c3a182..a3a165ae933adcab6580e1671b9dd2f04376c7f6 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -1791,6 +1791,7 @@ int dso__load(struct dso *dso, struct map *map) char newmapname[PATH_MAX]; const char *map_path = dso->long_name; + mutex_lock(&dso->lock); perfmap = strncmp(dso->name, "/tmp/perf-", 10) == 0; if (perfmap) { if (dso->nsinfo && (dso__find_perf_map(newmapname, @@ -1800,7 +1801,6 @@ int dso__load(struct dso *dso, struct map *map) } nsinfo__mountns_enter(dso->nsinfo, &nsc); - pthread_mutex_lock(&dso->lock); /* check again under the dso->lock */ if (dso__loaded(dso)) { @@ -1964,7 +1964,7 @@ out_free: ret = 0; out: dso__set_loaded(dso); - pthread_mutex_unlock(&dso->lock); + mutex_unlock(&dso->lock); nsinfo__mountns_exit(&nsc); return ret; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index 812424dbf2d5b95f1b19df7f704f37643a87dbd6..cccd293b531246eba4cf3150676ecbd141235594 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -364,28 +364,58 @@ static bool read_proc_maps_line(struct io *io, __u64 *start, __u64 *end, } static void perf_record_mmap2__read_build_id(struct perf_record_mmap2 *event, + struct machine *machine, bool is_kernel) { struct build_id bid; + struct nsinfo *nsi; + struct nscookie nc; + struct dso *dso = NULL; + struct dso_id id; int rc; - if (is_kernel) + if (is_kernel) { rc = sysfs__read_build_id("/sys/kernel/notes", &bid); - else - rc = filename__read_build_id(event->filename, &bid) > 0 ? 0 : -1; + goto out; + } + + id.maj = event->maj; + id.min = event->min; + id.ino = event->ino; + id.ino_generation = event->ino_generation; + + dso = dsos__findnew_id(&machine->dsos, event->filename, &id); + if (dso && dso->has_build_id) { + bid = dso->bid; + rc = 0; + goto out; + } + + nsi = nsinfo__new(event->pid); + nsinfo__mountns_enter(nsi, &nc); + + rc = filename__read_build_id(event->filename, &bid) > 0 ? 0 : -1; + nsinfo__mountns_exit(&nc); + nsinfo__put(nsi); + +out: if (rc == 0) { memcpy(event->build_id, bid.data, sizeof(bid.data)); event->build_id_size = (u8) bid.size; event->header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; event->__reserved_1 = 0; event->__reserved_2 = 0; + + if (dso && !dso->has_build_id) + dso__set_build_id(dso, &bid); } else { if (event->filename[0] == '/') { pr_debug2("Failed to read build ID for %s\n", event->filename); } } + dso__put(dso); } int perf_event__synthesize_mmap_events(struct perf_tool *tool, @@ -496,7 +526,7 @@ out: event->mmap2.tid = pid; if (symbol_conf.buildid_mmap2) - perf_record_mmap2__read_build_id(&event->mmap2, false); + perf_record_mmap2__read_build_id(&event->mmap2, machine, false); if (perf_tool__process_synth_event(tool, event, machine, process) != 0) { rc = -1; @@ -679,7 +709,7 @@ int perf_event__synthesize_modules(struct perf_tool *tool, perf_event__handler_t memcpy(event->mmap2.filename, pos->dso->long_name, pos->dso->long_name_len + 1); - perf_record_mmap2__read_build_id(&event->mmap2, false); + perf_record_mmap2__read_build_id(&event->mmap2, machine, false); } else { size = PERF_ALIGN(pos->dso->long_name_len + 1, sizeof(u64)); event->mmap.header.type = PERF_RECORD_MMAP; @@ -1115,7 +1145,7 @@ static int __perf_event__synthesize_kernel_mmap(struct perf_tool *tool, event->mmap2.len = map->end - event->mmap.start; event->mmap2.pid = machine->pid; - perf_record_mmap2__read_build_id(&event->mmap2, true); + perf_record_mmap2__read_build_id(&event->mmap2, machine, true); } else { size = snprintf(event->mmap.filename, sizeof(event->mmap.filename), "%s%s", machine->mmap_name, kmap->ref_reloc_sym->name) + 1; @@ -1184,93 +1214,97 @@ int perf_event__synthesize_thread_map2(struct perf_tool *tool, return err; } -static void synthesize_cpus(struct perf_record_cpu_map_data *data, - const struct perf_cpu_map *map) -{ - int i, map_nr = perf_cpu_map__nr(map); - - data->cpus_data.nr = map_nr; +struct synthesize_cpu_map_data { + const struct perf_cpu_map *map; + int nr; + int min_cpu; + int max_cpu; + int has_any_cpu; + int type; + size_t size; + struct perf_record_cpu_map_data *data; +}; - for (i = 0; i < map_nr; i++) - data->cpus_data.cpu[i] = perf_cpu_map__cpu(map, i).cpu; +static void synthesize_cpus(struct synthesize_cpu_map_data *data) +{ + data->data->type = PERF_CPU_MAP__CPUS; + data->data->cpus_data.nr = data->nr; + for (int i = 0; i < data->nr; i++) + data->data->cpus_data.cpu[i] = perf_cpu_map__cpu(data->map, i).cpu; } -static void synthesize_mask(struct perf_record_cpu_map_data *data, - const struct perf_cpu_map *map, int max) +static void synthesize_mask(struct synthesize_cpu_map_data *data) { int idx; struct perf_cpu cpu; /* Due to padding, the 4bytes per entry mask variant is always smaller. */ - data->mask32_data.nr = BITS_TO_U32(max); - data->mask32_data.long_size = 4; + data->data->type = PERF_CPU_MAP__MASK; + data->data->mask32_data.nr = BITS_TO_U32(data->max_cpu); + data->data->mask32_data.long_size = 4; - perf_cpu_map__for_each_cpu(cpu, idx, map) { + perf_cpu_map__for_each_cpu(cpu, idx, data->map) { int bit_word = cpu.cpu / 32; - __u32 bit_mask = 1U << (cpu.cpu & 31); + u32 bit_mask = 1U << (cpu.cpu & 31); - data->mask32_data.mask[bit_word] |= bit_mask; + data->data->mask32_data.mask[bit_word] |= bit_mask; } } -static size_t cpus_size(const struct perf_cpu_map *map) +static void synthesize_range_cpus(struct synthesize_cpu_map_data *data) { - return sizeof(struct cpu_map_entries) + perf_cpu_map__nr(map) * sizeof(u16); + data->data->type = PERF_CPU_MAP__RANGE_CPUS; + data->data->range_cpu_data.any_cpu = data->has_any_cpu; + data->data->range_cpu_data.start_cpu = data->min_cpu; + data->data->range_cpu_data.end_cpu = data->max_cpu; } -static size_t mask_size(const struct perf_cpu_map *map, int *max) -{ - *max = perf_cpu_map__max(map).cpu; - return sizeof(struct perf_record_mask_cpu_map32) + BITS_TO_U32(*max) * sizeof(__u32); -} - -static void *cpu_map_data__alloc(const struct perf_cpu_map *map, size_t *size, - u16 *type, int *max) +static void *cpu_map_data__alloc(struct synthesize_cpu_map_data *syn_data, + size_t header_size) { size_t size_cpus, size_mask; - bool is_dummy = perf_cpu_map__empty(map); - /* - * Both array and mask data have variable size based - * on the number of cpus and their actual values. - * The size of the 'struct perf_record_cpu_map_data' is: - * - * array = size of 'struct cpu_map_entries' + - * number of cpus * sizeof(u64) - * - * mask = size of 'struct perf_record_record_cpu_map' + - * maximum cpu bit converted to size of longs - * - * and finally + the size of 'struct perf_record_cpu_map_data'. - */ - size_cpus = cpus_size(map); - size_mask = mask_size(map, max); + syn_data->nr = perf_cpu_map__nr(syn_data->map); + syn_data->has_any_cpu = (perf_cpu_map__cpu(syn_data->map, 0).cpu == -1) ? 1 : 0; - if (is_dummy || (size_cpus < size_mask)) { - *size += size_cpus; - *type = PERF_CPU_MAP__CPUS; - } else { - *size += size_mask; - *type = PERF_CPU_MAP__MASK; + syn_data->min_cpu = perf_cpu_map__cpu(syn_data->map, syn_data->has_any_cpu).cpu; + syn_data->max_cpu = perf_cpu_map__max(syn_data->map).cpu; + if (syn_data->max_cpu - syn_data->min_cpu + 1 == syn_data->nr - syn_data->has_any_cpu) { + /* A consecutive range of CPUs can be encoded using a range. */ + assert(sizeof(u16) + sizeof(struct perf_record_range_cpu_map) == sizeof(u64)); + syn_data->type = PERF_CPU_MAP__RANGE_CPUS; + syn_data->size = header_size + sizeof(u64); + return zalloc(syn_data->size); } - *size += sizeof(__u16); /* For perf_record_cpu_map_data.type. */ - *size = PERF_ALIGN(*size, sizeof(u64)); - return zalloc(*size); + size_cpus = sizeof(u16) + sizeof(struct cpu_map_entries) + syn_data->nr * sizeof(u16); + /* Due to padding, the 4bytes per entry mask variant is always smaller. */ + size_mask = sizeof(u16) + sizeof(struct perf_record_mask_cpu_map32) + + BITS_TO_U32(syn_data->max_cpu) * sizeof(__u32); + if (syn_data->has_any_cpu || size_cpus < size_mask) { + /* Follow the CPU map encoding. */ + syn_data->type = PERF_CPU_MAP__CPUS; + syn_data->size = header_size + PERF_ALIGN(size_cpus, sizeof(u64)); + return zalloc(syn_data->size); + } + /* Encode using a bitmask. */ + syn_data->type = PERF_CPU_MAP__MASK; + syn_data->size = header_size + PERF_ALIGN(size_mask, sizeof(u64)); + return zalloc(syn_data->size); } -static void cpu_map_data__synthesize(struct perf_record_cpu_map_data *data, - const struct perf_cpu_map *map, - u16 type, int max) +static void cpu_map_data__synthesize(struct synthesize_cpu_map_data *data) { - data->type = type; - - switch (type) { + switch (data->type) { case PERF_CPU_MAP__CPUS: - synthesize_cpus(data, map); + synthesize_cpus(data); break; case PERF_CPU_MAP__MASK: - synthesize_mask(data, map, max); + synthesize_mask(data); + break; + case PERF_CPU_MAP__RANGE_CPUS: + synthesize_range_cpus(data); + break; default: break; } @@ -1278,23 +1312,22 @@ static void cpu_map_data__synthesize(struct perf_record_cpu_map_data *data, static struct perf_record_cpu_map *cpu_map_event__new(const struct perf_cpu_map *map) { - size_t size = sizeof(struct perf_event_header); + struct synthesize_cpu_map_data syn_data = { .map = map }; struct perf_record_cpu_map *event; - int max; - u16 type; - event = cpu_map_data__alloc(map, &size, &type, &max); + + event = cpu_map_data__alloc(&syn_data, sizeof(struct perf_event_header)); if (!event) return NULL; + syn_data.data = &event->data; event->header.type = PERF_RECORD_CPU_MAP; - event->header.size = size; - event->data.type = type; - - cpu_map_data__synthesize(&event->data, map, type, max); + event->header.size = syn_data.size; + cpu_map_data__synthesize(&syn_data); return event; } + int perf_event__synthesize_cpu_map(struct perf_tool *tool, const struct perf_cpu_map *map, perf_event__handler_t process, @@ -1944,7 +1977,7 @@ int perf_event__synthesize_event_update_unit(struct perf_tool *tool, struct evse if (ev == NULL) return -ENOMEM; - strlcpy(ev->data, evsel->unit, size + 1); + strlcpy(ev->unit, evsel->unit, size + 1); err = process(tool, (union perf_event *)ev, NULL, NULL); free(ev); return err; @@ -1961,8 +1994,7 @@ int perf_event__synthesize_event_update_scale(struct perf_tool *tool, struct evs if (ev == NULL) return -ENOMEM; - ev_data = (struct perf_record_event_update_scale *)ev->data; - ev_data->scale = evsel->scale; + ev->scale.scale = evsel->scale; err = process(tool, (union perf_event *)ev, NULL, NULL); free(ev); return err; @@ -1979,7 +2011,7 @@ int perf_event__synthesize_event_update_name(struct perf_tool *tool, struct evse if (ev == NULL) return -ENOMEM; - strlcpy(ev->data, evsel->name, len + 1); + strlcpy(ev->name, evsel->name, len + 1); err = process(tool, (union perf_event *)ev, NULL, NULL); free(ev); return err; @@ -1988,25 +2020,20 @@ int perf_event__synthesize_event_update_name(struct perf_tool *tool, struct evse int perf_event__synthesize_event_update_cpus(struct perf_tool *tool, struct evsel *evsel, perf_event__handler_t process) { - size_t size = sizeof(struct perf_record_event_update); + struct synthesize_cpu_map_data syn_data = { .map = evsel->core.own_cpus }; struct perf_record_event_update *ev; - int max, err; - u16 type; - - if (!evsel->core.own_cpus) - return 0; + int err; - ev = cpu_map_data__alloc(evsel->core.own_cpus, &size, &type, &max); + ev = cpu_map_data__alloc(&syn_data, sizeof(struct perf_event_header) + 2 * sizeof(u64)); if (!ev) return -ENOMEM; + syn_data.data = &ev->cpus.cpus; ev->header.type = PERF_RECORD_EVENT_UPDATE; - ev->header.size = (u16)size; + ev->header.size = (u16)syn_data.size; ev->type = PERF_EVENT_UPDATE__CPUS; ev->id = evsel->core.id[0]; - - cpu_map_data__synthesize((struct perf_record_cpu_map_data *)ev->data, - evsel->core.own_cpus, type, max); + cpu_map_data__synthesize(&syn_data); err = process(tool, (union perf_event *)ev, NULL, NULL); free(ev); diff --git a/tools/perf/util/top.h b/tools/perf/util/top.h index 1c2c0a838430716946dbfaf90af2e68bf20c79a6..a8b0d79bd96cfa36be55dde1995ed8e129b3d732 100644 --- a/tools/perf/util/top.h +++ b/tools/perf/util/top.h @@ -5,6 +5,7 @@ #include "tool.h" #include "evswitch.h" #include "annotate.h" +#include "mutex.h" #include "ordered-events.h" #include "record.h" #include <linux/types.h> @@ -53,8 +54,8 @@ struct perf_top { struct ordered_events *in; struct ordered_events data[2]; bool rotate; - pthread_mutex_t mutex; - pthread_cond_t cond; + struct mutex mutex; + struct cond cond; } qe; }; diff --git a/tools/power/acpi/tools/pfrut/pfrut.c b/tools/power/acpi/tools/pfrut/pfrut.c index d79c335594b201fd995e3b0b37220cf9166fbd2f..52aa0351533c315e4c7bfd4e6a318642757c5e73 100644 --- a/tools/power/acpi/tools/pfrut/pfrut.c +++ b/tools/power/acpi/tools/pfrut/pfrut.c @@ -190,7 +190,7 @@ int main(int argc, char *argv[]) void *addr_map_capsule; struct stat st; char *log_buf; - int ret = 0; + int ret; if (getuid() != 0) { printf("Please run the tool as root - Exiting.\n"); diff --git a/tools/power/x86/intel-speed-select/hfi-events.c b/tools/power/x86/intel-speed-select/hfi-events.c index f0ed697213084a8d346308b3a1905f836d2efe4e..be96e90cc2a1150a15517a833a4fa3ef4020130a 100644 --- a/tools/power/x86/intel-speed-select/hfi-events.c +++ b/tools/power/x86/intel-speed-select/hfi-events.c @@ -181,7 +181,10 @@ struct perf_cap { static void process_hfi_event(struct perf_cap *perf_cap) { - process_level_change(perf_cap->cpu); + struct isst_id id; + + set_isst_id(&id, perf_cap->cpu); + process_level_change(&id); } static int handle_event(struct nl_msg *n, void *arg) diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c index 9d35614995ee163501a56729c5479a02263ff11d..a160bad291eb7b470ef94a01d70a3a853562ddd6 100644 --- a/tools/power/x86/intel-speed-select/isst-config.c +++ b/tools/power/x86/intel-speed-select/isst-config.c @@ -15,7 +15,7 @@ struct process_cmd_struct { int arg; }; -static const char *version_str = "v1.12"; +static const char *version_str = "v1.13"; static const int supported_api_ver = 1; static struct isst_if_platform_info isst_platform_info; @@ -63,6 +63,7 @@ struct _cpu_map { unsigned short die_id; unsigned short punit_cpu; unsigned short punit_cpu_core; + unsigned short initialized; }; struct _cpu_map *cpu_map; @@ -298,10 +299,16 @@ static void store_cpu_topology(void) fclose(fp); } -int get_physical_package_id(int cpu) +static int get_physical_package_id(int cpu) { int ret; + if (cpu < 0) + return -1; + + if (cpu_map && cpu_map[cpu].initialized) + return cpu_map[cpu].pkg_id; + ret = parse_int_file(0, "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); @@ -316,10 +323,16 @@ int get_physical_package_id(int cpu) return ret; } -int get_physical_core_id(int cpu) +static int get_physical_core_id(int cpu) { int ret; + if (cpu < 0) + return -1; + + if (cpu_map && cpu_map[cpu].initialized) + return cpu_map[cpu].core_id; + ret = parse_int_file(0, "/sys/devices/system/cpu/cpu%d/topology/core_id", cpu); @@ -334,10 +347,16 @@ int get_physical_core_id(int cpu) return ret; } -int get_physical_die_id(int cpu) +static int get_physical_die_id(int cpu) { int ret; + if (cpu < 0) + return -1; + + if (cpu_map && cpu_map[cpu].initialized) + return cpu_map[cpu].die_id; + ret = parse_int_file(0, "/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); @@ -359,6 +378,31 @@ int get_physical_die_id(int cpu) return ret; } +void set_isst_id(struct isst_id *id, int cpu) +{ + id->cpu = cpu; + + id->pkg = get_physical_package_id(cpu); + if (id < 0 || id->pkg >= MAX_PACKAGE_COUNT) + id->pkg = -1; + + id->die = get_physical_die_id(cpu); + if (id < 0 || id->die >= MAX_DIE_PER_PACKAGE) + id->die = -1; +} + +int is_cpu_in_power_domain(int cpu, struct isst_id *id) +{ + struct isst_id tid; + + set_isst_id(&tid, cpu); + + if (id->pkg == tid.pkg && id->die == tid.die) + return 1; + + return 0; +} + int get_cpufreq_base_freq(int cpu) { return parse_int_file(0, "/sys/devices/system/cpu/cpu%d/cpufreq/base_frequency", cpu); @@ -410,13 +454,14 @@ static void force_all_cpus_online(void) unlink("/var/run/isst_cpu_topology.dat"); } -void for_each_online_package_in_set(void (*callback)(int, void *, void *, +void for_each_online_package_in_set(void (*callback)(struct isst_id *, void *, void *, void *, void *), void *arg1, void *arg2, void *arg3, void *arg4) { int max_packages[MAX_PACKAGE_COUNT * MAX_PACKAGE_COUNT]; int pkg_index = 0, i; + struct isst_id id; memset(max_packages, 0xff, sizeof(max_packages)); for (i = 0; i < topo_max_cpus; ++i) { @@ -450,18 +495,20 @@ void for_each_online_package_in_set(void (*callback)(int, void *, void *, } } + set_isst_id(&id, i); if (!skip && online && callback) { - callback(i, arg1, arg2, arg3, arg4); + callback(&id, arg1, arg2, arg3, arg4); max_packages[pkg_index++] = pkg_id; } } } static void for_each_online_target_cpu_in_set( - void (*callback)(int, void *, void *, void *, void *), void *arg1, + void (*callback)(struct isst_id *, void *, void *, void *, void *), void *arg1, void *arg2, void *arg3, void *arg4) { int i, found = 0; + struct isst_id id; for (i = 0; i < topo_max_cpus; ++i) { int online; @@ -475,8 +522,9 @@ static void for_each_online_target_cpu_in_set( online = 1; /* online entry for CPU 0 needs some special configs */ + set_isst_id(&id, i); if (online && callback) { - callback(i, arg1, arg2, arg3, arg4); + callback(&id, arg1, arg2, arg3, arg4); found = 1; } } @@ -536,47 +584,8 @@ void free_cpu_set(cpu_set_t *cpu_set) } static int cpu_cnt[MAX_PACKAGE_COUNT][MAX_DIE_PER_PACKAGE]; -static long long core_mask[MAX_PACKAGE_COUNT][MAX_DIE_PER_PACKAGE]; -static void set_cpu_present_cpu_mask(void) -{ - size_t size; - DIR *dir; - int i; - size = alloc_cpu_set(&present_cpumask); - present_cpumask_size = size; - for (i = 0; i < topo_max_cpus; ++i) { - char buffer[256]; - - snprintf(buffer, sizeof(buffer), - "/sys/devices/system/cpu/cpu%d", i); - dir = opendir(buffer); - if (dir) { - int pkg_id, die_id; - - CPU_SET_S(i, size, present_cpumask); - die_id = get_physical_die_id(i); - if (die_id < 0) - die_id = 0; - - pkg_id = get_physical_package_id(i); - if (pkg_id < 0) { - fprintf(stderr, "Failed to get package id, CPU %d may be offline\n", i); - continue; - } - if (pkg_id < MAX_PACKAGE_COUNT && - die_id < MAX_DIE_PER_PACKAGE) { - int core_id = get_physical_core_id(i); - - cpu_cnt[pkg_id][die_id]++; - core_mask[pkg_id][die_id] |= (1ULL << core_id); - } - } - closedir(dir); - } -} - -int get_max_punit_core_id(int pkg_id, int die_id) +int get_max_punit_core_id(struct isst_id *id) { int max_id = 0; int i; @@ -586,60 +595,74 @@ int get_max_punit_core_id(int pkg_id, int die_id) if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) continue; - if (cpu_map[i].pkg_id == pkg_id && - cpu_map[i].die_id == die_id && - cpu_map[i].punit_cpu_core > max_id) + if (is_cpu_in_power_domain(i, id) && + cpu_map[i].punit_cpu_core > max_id) max_id = cpu_map[i].punit_cpu_core; } return max_id; } -int get_cpu_count(int pkg_id, int die_id) +int get_cpu_count(struct isst_id *id) { - if (pkg_id < MAX_PACKAGE_COUNT && die_id < MAX_DIE_PER_PACKAGE) - return cpu_cnt[pkg_id][die_id]; - - return 0; -} - -static void set_cpu_target_cpu_mask(void) -{ - size_t size; - int i; - - size = alloc_cpu_set(&target_cpumask); - target_cpumask_size = size; - for (i = 0; i < max_target_cpus; ++i) { - if (!CPU_ISSET_S(target_cpus[i], present_cpumask_size, - present_cpumask)) - continue; + if (id->pkg < 0 || id->die < 0) + return 0; - CPU_SET_S(target_cpus[i], size, target_cpumask); - } + return cpu_cnt[id->pkg][id->die]; } static void create_cpu_map(void) { const char *pathname = "/dev/isst_interface"; + size_t size; + DIR *dir; int i, fd = 0; struct isst_if_cpu_maps map; - cpu_map = malloc(sizeof(*cpu_map) * topo_max_cpus); + /* Use calloc to make sure the memory is initialized to Zero */ + cpu_map = calloc(topo_max_cpus, sizeof(*cpu_map)); if (!cpu_map) err(3, "cpumap"); fd = open(pathname, O_RDWR); - if (fd < 0) + if (fd < 0 && !is_clx_n_platform()) err(-1, "%s open failed", pathname); + size = alloc_cpu_set(&present_cpumask); + present_cpumask_size = size; + for (i = 0; i < topo_max_cpus; ++i) { - if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) + char buffer[256]; + int pkg_id, die_id, core_id; + + /* check if CPU is online */ + snprintf(buffer, sizeof(buffer), + "/sys/devices/system/cpu/cpu%d", i); + dir = opendir(buffer); + if (!dir) + continue; + closedir(dir); + + CPU_SET_S(i, size, present_cpumask); + + pkg_id = get_physical_package_id(i); + die_id = get_physical_die_id(i); + core_id = get_physical_core_id(i); + + if (pkg_id < 0 || die_id < 0 || core_id < 0) continue; + cpu_map[i].pkg_id = pkg_id; + cpu_map[i].die_id = die_id; + cpu_map[i].core_id = core_id; + cpu_map[i].initialized = 1; + + cpu_cnt[pkg_id][die_id]++; + + if (fd < 0) + continue; map.cmd_count = 1; map.cpu_map[0].logical_cpu = i; - debug_printf(" map logical_cpu:%d\n", map.cpu_map[0].logical_cpu); if (ioctl(fd, ISST_IF_GET_PHY_ID, &map) == -1) { @@ -648,9 +671,6 @@ static void create_cpu_map(void) map.cpu_map[0].logical_cpu); continue; } - cpu_map[i].core_id = get_physical_core_id(i); - cpu_map[i].pkg_id = get_physical_package_id(i); - cpu_map[i].die_id = get_physical_die_id(i); cpu_map[i].punit_cpu = map.cpu_map[0].physical_cpu; cpu_map[i].punit_cpu_core = (map.cpu_map[0].physical_cpu >> 1); // shift to get core id @@ -661,35 +681,27 @@ static void create_cpu_map(void) cpu_map[i].pkg_id, cpu_map[i].punit_cpu, cpu_map[i].punit_cpu_core); } - - if (fd) + if (fd >= 0) close(fd); -} -int find_logical_cpu(int pkg_id, int die_id, int punit_core_id) -{ - int i; + size = alloc_cpu_set(&target_cpumask); + target_cpumask_size = size; + for (i = 0; i < max_target_cpus; ++i) { + if (!CPU_ISSET_S(target_cpus[i], present_cpumask_size, + present_cpumask)) + continue; - for (i = 0; i < topo_max_cpus; ++i) { - if (cpu_map[i].pkg_id == pkg_id && - cpu_map[i].die_id == die_id && - cpu_map[i].punit_cpu_core == punit_core_id) - return i; + CPU_SET_S(target_cpus[i], size, target_cpumask); } - - return -EINVAL; } -void set_cpu_mask_from_punit_coremask(int cpu, unsigned long long core_mask, +void set_cpu_mask_from_punit_coremask(struct isst_id *id, unsigned long long core_mask, size_t core_cpumask_size, cpu_set_t *core_cpumask, int *cpu_cnt) { int i, cnt = 0; - int die_id, pkg_id; *cpu_cnt = 0; - die_id = get_physical_die_id(cpu); - pkg_id = get_physical_package_id(cpu); for (i = 0; i < 64; ++i) { if (core_mask & BIT_ULL(i)) { @@ -699,8 +711,7 @@ void set_cpu_mask_from_punit_coremask(int cpu, unsigned long long core_mask, if (!CPU_ISSET_S(j, present_cpumask_size, present_cpumask)) continue; - if (cpu_map[j].pkg_id == pkg_id && - cpu_map[j].die_id == die_id && + if (is_cpu_in_power_domain(j, id) && cpu_map[j].punit_cpu_core == i) { CPU_SET_S(j, core_cpumask_size, core_cpumask); @@ -931,6 +942,7 @@ static void isst_print_extended_platform_info(void) struct isst_pkg_ctdp pkg_dev; int ret, i, j; FILE *filep; + struct isst_id id; for (i = 0; i < 256; ++i) { char path[256]; @@ -947,7 +959,8 @@ static void isst_print_extended_platform_info(void) fclose(filep); - ret = isst_get_ctdp_levels(i, &pkg_dev); + set_isst_id(&id, i); + ret = isst_get_ctdp_levels(&id, &pkg_dev); if (ret) return; @@ -964,7 +977,7 @@ static void isst_print_extended_platform_info(void) fprintf(outf, "TDP level change control is unlocked, max level: %d \n", pkg_dev.levels); for (j = 0; j <= pkg_dev.levels; ++j) { - ret = isst_get_ctdp_control(i, j, &ctdp_level); + ret = isst_get_ctdp_control(&id, j, &ctdp_level); if (ret) continue; @@ -985,7 +998,7 @@ static void isst_print_extended_platform_info(void) else fprintf(outf, "Intel(R) SST-BF (feature base-freq) is not supported\n"); - ret = isst_read_pm_config(i, &cp_state, &cp_cap); + ret = isst_read_pm_config(&id, &cp_state, &cp_cap); if (ret) { fprintf(outf, "Intel(R) SST-CP (feature core-power) status is unknown\n"); return; @@ -1007,6 +1020,10 @@ static void isst_print_platform_information(void) exit(0); } + /* Early initialization to create working cpu_map */ + set_max_cpu_num(); + create_cpu_map(); + fd = open(pathname, O_RDWR); if (fd < 0) err(-1, "%s open failed", pathname); @@ -1031,18 +1048,18 @@ static void isst_print_platform_information(void) } static char *local_str0, *local_str1; -static void exec_on_get_ctdp_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void exec_on_get_ctdp_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { - int (*fn_ptr)(int cpu, void *arg); + int (*fn_ptr)(struct isst_id *id, void *arg); int ret; fn_ptr = arg1; - ret = fn_ptr(cpu, arg2); + ret = fn_ptr(id, arg2); if (ret) isst_display_error_info_message(1, "get_tdp_* failed", 0, 0); else - isst_ctdp_display_core_info(cpu, outf, arg3, + isst_ctdp_display_core_info(id, outf, arg3, *(unsigned int *)arg4, local_str0, local_str1); } @@ -1110,9 +1127,9 @@ static int clx_n_get_base_ratio(void) return (int)(value); } -static int clx_n_config(int cpu) +static int clx_n_config(struct isst_id *id) { - int i, ret, pkg_id, die_id; + int i, ret; unsigned long cpu_bf; struct isst_pkg_ctdp_level_info *ctdp_level; struct isst_pbf_info *pbf_info; @@ -1134,15 +1151,11 @@ static int clx_n_config(int cpu) pbf_info->p1_high = 0; pbf_info->p1_low = ~0; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); - for (i = 0; i < topo_max_cpus; i++) { if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) continue; - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; CPU_SET_S(i, ctdp_level->core_cpumask_size, @@ -1179,8 +1192,7 @@ static int clx_n_config(int cpu) if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) continue; - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; cpu_bf = parse_int_file(1, @@ -1206,7 +1218,7 @@ error_ret: return ret; } -static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2, +static void dump_clx_n_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int ret; @@ -1216,7 +1228,7 @@ static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2, exit(0); } - ret = clx_n_config(cpu); + ret = clx_n_config(id); if (ret) { debug_printf("clx_n_config failed"); } else { @@ -1226,27 +1238,27 @@ static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2, ctdp_level = &clx_n_pkg_dev.ctdp_level[0]; pbf_info = &ctdp_level->pbf_info; clx_n_pkg_dev.processed = 1; - isst_ctdp_display_information(cpu, outf, tdp_level, &clx_n_pkg_dev); + isst_ctdp_display_information(id, outf, tdp_level, &clx_n_pkg_dev); free_cpu_set(ctdp_level->core_cpumask); free_cpu_set(pbf_info->core_cpumask); } } -static void dump_isst_config_for_cpu(int cpu, void *arg1, void *arg2, +static void dump_isst_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_pkg_ctdp pkg_dev; int ret; memset(&pkg_dev, 0, sizeof(pkg_dev)); - ret = isst_get_process_ctdp(cpu, tdp_level, &pkg_dev); + ret = isst_get_process_ctdp(id, tdp_level, &pkg_dev); if (ret) { - isst_display_error_info_message(1, "Failed to get perf-profile info on cpu", 1, cpu); + isst_display_error_info_message(1, "Failed to get perf-profile info on cpu", 1, id->cpu); isst_ctdp_display_information_end(outf); exit(1); } else { - isst_ctdp_display_information(cpu, outf, tdp_level, &pkg_dev); - isst_get_process_ctdp_complete(cpu, &pkg_dev); + isst_ctdp_display_information(id, outf, tdp_level, &pkg_dev); + isst_get_process_ctdp_complete(id, &pkg_dev); } } @@ -1282,23 +1294,21 @@ static void dump_isst_config(int arg) static void adjust_scaling_max_from_base_freq(int cpu); -static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void set_tdp_level_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int ret; - ret = isst_set_tdp_level(cpu, tdp_level); + ret = isst_set_tdp_level(id, tdp_level); if (ret) { isst_display_error_info_message(1, "Set TDP level failed", 0, 0); isst_ctdp_display_information_end(outf); exit(1); } else { - isst_display_result(cpu, outf, "perf-profile", "set_tdp_level", + isst_display_result(id, outf, "perf-profile", "set_tdp_level", ret); if (force_online_offline) { struct isst_pkg_ctdp_level_info ctdp_level; - int pkg_id = get_physical_package_id(cpu); - int die_id = get_physical_die_id(cpu); /* Wait for updated base frequencies */ usleep(2000); @@ -1306,7 +1316,7 @@ static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, fprintf(stderr, "Option is set to online/offline\n"); ctdp_level.core_cpumask_size = alloc_cpu_set(&ctdp_level.core_cpumask); - ret = isst_get_coremask_info(cpu, tdp_level, &ctdp_level); + ret = isst_get_coremask_info(id, tdp_level, &ctdp_level); if (ret) { isst_display_error_info_message(1, "Can't get coremask, online/offline option is ignored", 0, 0); return; @@ -1314,7 +1324,7 @@ static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, if (ctdp_level.cpu_count) { int i, max_cpus = get_topo_max_cpus(); for (i = 0; i < max_cpus; ++i) { - if (pkg_id != get_physical_package_id(i) || die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; if (CPU_ISSET_S(i, ctdp_level.core_cpumask_size, ctdp_level.core_cpumask)) { fprintf(stderr, "online cpu %d\n", i); @@ -1357,12 +1367,12 @@ static void set_tdp_level(int arg) isst_ctdp_display_information_end(outf); } -static void clx_n_dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2, +static void clx_n_dump_pbf_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int ret; - ret = clx_n_config(cpu); + ret = clx_n_config(id); if (ret) { isst_display_error_info_message(1, "clx_n_config failed", 0, 0); } else { @@ -1371,25 +1381,25 @@ static void clx_n_dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2, ctdp_level = &clx_n_pkg_dev.ctdp_level[0]; pbf_info = &ctdp_level->pbf_info; - isst_pbf_display_information(cpu, outf, tdp_level, pbf_info); + isst_pbf_display_information(id, outf, tdp_level, pbf_info); free_cpu_set(ctdp_level->core_cpumask); free_cpu_set(pbf_info->core_cpumask); } } -static void dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void dump_pbf_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_pbf_info pbf_info; int ret; - ret = isst_get_pbf_info(cpu, tdp_level, &pbf_info); + ret = isst_get_pbf_info(id, tdp_level, &pbf_info); if (ret) { isst_display_error_info_message(1, "Failed to get base-freq info at this level", 1, tdp_level); isst_ctdp_display_information_end(outf); exit(1); } else { - isst_pbf_display_information(cpu, outf, tdp_level, &pbf_info); + isst_pbf_display_information(id, outf, tdp_level, &pbf_info); isst_get_pbf_info_complete(&pbf_info); } } @@ -1426,12 +1436,12 @@ static void dump_pbf_config(int arg) isst_ctdp_display_information_end(outf); } -static int set_clos_param(int cpu, int clos, int epp, int wt, int min, int max) +static int set_clos_param(struct isst_id *id, int clos, int epp, int wt, int min, int max) { struct isst_clos_config clos_config; int ret; - ret = isst_pm_get_clos(cpu, clos, &clos_config); + ret = isst_pm_get_clos(id, clos, &clos_config); if (ret) { isst_display_error_info_message(1, "isst_pm_get_clos failed", 0, 0); return ret; @@ -1440,7 +1450,7 @@ static int set_clos_param(int cpu, int clos, int epp, int wt, int min, int max) clos_config.clos_max = max; clos_config.epp = epp; clos_config.clos_prop_prio = wt; - ret = isst_set_clos(cpu, clos, &clos_config); + ret = isst_set_clos(id, clos, &clos_config); if (ret) { isst_display_error_info_message(1, "isst_set_clos failed", 0, 0); return ret; @@ -1502,14 +1512,14 @@ static void adjust_scaling_min_from_base_freq(int cpu) set_cpufreq_scaling_min_max(cpu, 0, base_freq); } -static int set_clx_pbf_cpufreq_scaling_min_max(int cpu) +static int set_clx_pbf_cpufreq_scaling_min_max(struct isst_id *id) { struct isst_pkg_ctdp_level_info *ctdp_level; struct isst_pbf_info *pbf_info; - int i, pkg_id, die_id, freq, freq_high, freq_low; + int i, freq, freq_high, freq_low; int ret; - ret = clx_n_config(cpu); + ret = clx_n_config(id); if (ret) { debug_printf("cpufreq_scaling_min_max failed for CLX"); return ret; @@ -1520,11 +1530,8 @@ static int set_clx_pbf_cpufreq_scaling_min_max(int cpu) freq_high = pbf_info->p1_high * 100000; freq_low = pbf_info->p1_low * 100000; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); for (i = 0; i < get_topo_max_cpus(); ++i) { - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; if (CPU_ISSET_S(i, pbf_info->core_cpumask_size, @@ -1587,15 +1594,12 @@ static int set_cpufreq_scaling_min_max_from_cpuinfo(int cpu, int cpuinfo_max, in return 0; } -static void set_scaling_min_to_cpuinfo_max(int cpu) +static void set_scaling_min_to_cpuinfo_max(struct isst_id *id) { - int i, pkg_id, die_id; + int i; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); for (i = 0; i < get_topo_max_cpus(); ++i) { - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; adjust_scaling_max_from_base_freq(i); @@ -1604,15 +1608,12 @@ static void set_scaling_min_to_cpuinfo_max(int cpu) } } -static void set_scaling_min_to_cpuinfo_min(int cpu) +static void set_scaling_min_to_cpuinfo_min(struct isst_id *id) { - int i, pkg_id, die_id; + int i; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); for (i = 0; i < get_topo_max_cpus(); ++i) { - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; adjust_scaling_max_from_base_freq(i); @@ -1620,53 +1621,48 @@ static void set_scaling_min_to_cpuinfo_min(int cpu) } } -static void set_scaling_max_to_cpuinfo_max(int cpu) +static void set_scaling_max_to_cpuinfo_max(struct isst_id *id) { - int i, pkg_id, die_id; + int i; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); for (i = 0; i < get_topo_max_cpus(); ++i) { - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; set_cpufreq_scaling_min_max_from_cpuinfo(i, 1, 1); } } -static int set_core_priority_and_min(int cpu, int mask_size, +static int set_core_priority_and_min(struct isst_id *id, int mask_size, cpu_set_t *cpu_mask, int min_high, int min_low) { - int pkg_id, die_id, ret, i; + int ret, i; if (!CPU_COUNT_S(mask_size, cpu_mask)) return -1; - ret = set_clos_param(cpu, 0, 0, 0, min_high, 0xff); + ret = set_clos_param(id, 0, 0, 0, min_high, 0xff); if (ret) return ret; - ret = set_clos_param(cpu, 1, 15, 15, min_low, 0xff); + ret = set_clos_param(id, 1, 15, 15, min_low, 0xff); if (ret) return ret; - ret = set_clos_param(cpu, 2, 15, 15, min_low, 0xff); + ret = set_clos_param(id, 2, 15, 15, min_low, 0xff); if (ret) return ret; - ret = set_clos_param(cpu, 3, 15, 15, min_low, 0xff); + ret = set_clos_param(id, 3, 15, 15, min_low, 0xff); if (ret) return ret; - pkg_id = get_physical_package_id(cpu); - die_id = get_physical_die_id(cpu); for (i = 0; i < get_topo_max_cpus(); ++i) { int clos; + struct isst_id tid; - if (pkg_id != get_physical_package_id(i) || - die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; if (CPU_ISSET_S(i, mask_size, cpu_mask)) @@ -1675,7 +1671,8 @@ static int set_core_priority_and_min(int cpu, int mask_size, clos = 3; debug_printf("Associate cpu: %d clos: %d\n", i, clos); - ret = isst_clos_associate(i, clos); + set_isst_id(&tid, i); + ret = isst_clos_associate(&tid, clos); if (ret) { isst_display_error_info_message(1, "isst_clos_associate failed", 0, 0); return ret; @@ -1685,20 +1682,20 @@ static int set_core_priority_and_min(int cpu, int mask_size, return 0; } -static int set_pbf_core_power(int cpu) +static int set_pbf_core_power(struct isst_id *id) { struct isst_pbf_info pbf_info; struct isst_pkg_ctdp pkg_dev; int ret; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { debug_printf("isst_get_ctdp_levels failed"); return ret; } debug_printf("Current_level: %d\n", pkg_dev.current_level); - ret = isst_get_pbf_info(cpu, pkg_dev.current_level, &pbf_info); + ret = isst_get_pbf_info(id, pkg_dev.current_level, &pbf_info); if (ret) { debug_printf("isst_get_pbf_info failed"); return ret; @@ -1706,7 +1703,7 @@ static int set_pbf_core_power(int cpu) debug_printf("p1_high: %d p1_low: %d\n", pbf_info.p1_high, pbf_info.p1_low); - ret = set_core_priority_and_min(cpu, pbf_info.core_cpumask_size, + ret = set_core_priority_and_min(id, pbf_info.core_cpumask_size, pbf_info.core_cpumask, pbf_info.p1_high, pbf_info.p1_low); if (ret) { @@ -1714,7 +1711,7 @@ static int set_pbf_core_power(int cpu) return ret; } - ret = isst_pm_qos_config(cpu, 1, 1); + ret = isst_pm_qos_config(id, 1, 1); if (ret) { debug_printf("isst_pm_qos_config failed"); return ret; @@ -1723,7 +1720,7 @@ static int set_pbf_core_power(int cpu) return 0; } -static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void set_pbf_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_pkg_ctdp_level_info ctdp_level; @@ -1734,22 +1731,22 @@ static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, if (is_clx_n_platform()) { ret = 0; if (status) { - set_clx_pbf_cpufreq_scaling_min_max(cpu); + set_clx_pbf_cpufreq_scaling_min_max(id); } else { - set_scaling_max_to_cpuinfo_max(cpu); - set_scaling_min_to_cpuinfo_min(cpu); + set_scaling_max_to_cpuinfo_max(id); + set_scaling_min_to_cpuinfo_min(id); } goto disp_result; } - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { isst_display_error_info_message(1, "Failed to get number of levels", 0, 0); goto disp_result; } - ret = isst_get_ctdp_control(cpu, pkg_dev.current_level, &ctdp_level); + ret = isst_get_ctdp_control(id, pkg_dev.current_level, &ctdp_level); if (ret) { isst_display_error_info_message(1, "Failed to get current level", 0, 0); goto disp_result; @@ -1762,34 +1759,34 @@ static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, } if (auto_mode && status) { - ret = set_pbf_core_power(cpu); + ret = set_pbf_core_power(id); if (ret) goto disp_result; } - ret = isst_set_pbf_fact_status(cpu, 1, status); + ret = isst_set_pbf_fact_status(id, 1, status); if (ret) { debug_printf("isst_set_pbf_fact_status failed"); if (auto_mode) - isst_pm_qos_config(cpu, 0, 0); + isst_pm_qos_config(id, 0, 0); } else { if (auto_mode) { if (status) - set_scaling_min_to_cpuinfo_max(cpu); + set_scaling_min_to_cpuinfo_max(id); else - set_scaling_min_to_cpuinfo_min(cpu); + set_scaling_min_to_cpuinfo_min(id); } } if (auto_mode && !status) - isst_pm_qos_config(cpu, 0, 1); + isst_pm_qos_config(id, 0, 1); disp_result: if (status) - isst_display_result(cpu, outf, "base-freq", "enable", + isst_display_result(id, outf, "base-freq", "enable", ret); else - isst_display_result(cpu, outf, "base-freq", "disable", + isst_display_result(id, outf, "base-freq", "disable", ret); } @@ -1838,19 +1835,19 @@ static void set_pbf_enable(int arg) isst_ctdp_display_information_end(outf); } -static void dump_fact_config_for_cpu(int cpu, void *arg1, void *arg2, +static void dump_fact_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_fact_info fact_info; int ret; - ret = isst_get_fact_info(cpu, tdp_level, fact_bucket, &fact_info); + ret = isst_get_fact_info(id, tdp_level, fact_bucket, &fact_info); if (ret) { isst_display_error_info_message(1, "Failed to get turbo-freq info at this level", 1, tdp_level); isst_ctdp_display_information_end(outf); exit(1); } else { - isst_fact_display_information(cpu, outf, tdp_level, fact_bucket, + isst_fact_display_information(id, outf, tdp_level, fact_bucket, fact_avx, &fact_info); } } @@ -1884,7 +1881,7 @@ static void dump_fact_config(int arg) isst_ctdp_display_information_end(outf); } -static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void set_fact_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_pkg_ctdp_level_info ctdp_level; @@ -1898,13 +1895,13 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, goto disp_results; } - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { isst_display_error_info_message(1, "Failed to get number of levels", 0, 0); goto disp_results; } - ret = isst_get_ctdp_control(cpu, pkg_dev.current_level, &ctdp_level); + ret = isst_get_ctdp_control(id, pkg_dev.current_level, &ctdp_level); if (ret) { isst_display_error_info_message(1, "Failed to get current level", 0, 0); goto disp_results; @@ -1917,16 +1914,16 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, } if (status) { - ret = isst_pm_qos_config(cpu, 1, 1); + ret = isst_pm_qos_config(id, 1, 1); if (ret) goto disp_results; } - ret = isst_set_pbf_fact_status(cpu, 0, status); + ret = isst_set_pbf_fact_status(id, 0, status); if (ret) { debug_printf("isst_set_pbf_fact_status failed"); if (auto_mode) - isst_pm_qos_config(cpu, 0, 0); + isst_pm_qos_config(id, 0, 0); goto disp_results; } @@ -1935,31 +1932,32 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, if (status) { struct isst_pkg_ctdp pkg_dev; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (!ret) - ret = isst_set_trl(cpu, fact_trl); + ret = isst_set_trl(id, fact_trl); if (ret && auto_mode) - isst_pm_qos_config(cpu, 0, 0); + isst_pm_qos_config(id, 0, 0); } else { if (auto_mode) - isst_pm_qos_config(cpu, 0, 0); + isst_pm_qos_config(id, 0, 0); } disp_results: if (status) { - isst_display_result(cpu, outf, "turbo-freq", "enable", ret); + isst_display_result(id, outf, "turbo-freq", "enable", ret); if (ret) fact_enable_fail = ret; } else { /* Since we modified TRL during Fact enable, restore it */ - isst_set_trl_from_current_tdp(cpu, fact_trl); - isst_display_result(cpu, outf, "turbo-freq", "disable", ret); + isst_set_trl_from_current_tdp(id, fact_trl); + isst_display_result(id, outf, "turbo-freq", "disable", ret); } } static void set_fact_enable(int arg) { int i, ret, enable = arg; + struct isst_id id; if (cmd_help) { if (enable) { @@ -2033,19 +2031,20 @@ static void set_fact_enable(int arg) if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask)) continue; - ret = set_clos_param(i, 0, 0, 0, 0, 0xff); + set_isst_id(&id, i); + ret = set_clos_param(&id, 0, 0, 0, 0, 0xff); if (ret) goto error_disp; - ret = set_clos_param(i, 1, 15, 15, 0, 0xff); + ret = set_clos_param(&id, 1, 15, 15, 0, 0xff); if (ret) goto error_disp; - ret = set_clos_param(i, 2, 15, 15, 0, 0xff); + ret = set_clos_param(&id, 2, 15, 15, 0, 0xff); if (ret) goto error_disp; - ret = set_clos_param(i, 3, 15, 15, 0, 0xff); + ret = set_clos_param(&id, 3, 15, 15, 0, 0xff); if (ret) goto error_disp; @@ -2055,21 +2054,22 @@ static void set_fact_enable(int arg) clos = 3; debug_printf("Associate cpu: %d clos: %d\n", i, clos); - ret = isst_clos_associate(i, clos); + ret = isst_clos_associate(&id, clos); if (ret) goto error_disp; } - isst_display_result(-1, outf, "turbo-freq --auto", "enable", 0); + set_isst_id(&id, -1); + isst_display_result(&id, outf, "turbo-freq --auto", "enable", 0); } return; error_disp: - isst_display_result(i, outf, "turbo-freq --auto", "enable", ret); + isst_display_result(&id, outf, "turbo-freq --auto", "enable", ret); } -static void enable_clos_qos_config(int cpu, void *arg1, void *arg2, void *arg3, +static void enable_clos_qos_config(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int ret; @@ -2078,15 +2078,15 @@ static void enable_clos_qos_config(int cpu, void *arg1, void *arg2, void *arg3, if (is_skx_based_platform()) clos_priority_type = 1; - ret = isst_pm_qos_config(cpu, status, clos_priority_type); + ret = isst_pm_qos_config(id, status, clos_priority_type); if (ret) isst_display_error_info_message(1, "isst_pm_qos_config failed", 0, 0); if (status) - isst_display_result(cpu, outf, "core-power", "enable", + isst_display_result(id, outf, "core-power", "enable", ret); else - isst_display_result(cpu, outf, "core-power", "disable", + isst_display_result(id, outf, "core-power", "disable", ret); } @@ -2125,17 +2125,17 @@ static void set_clos_enable(int arg) isst_ctdp_display_information_end(outf); } -static void dump_clos_config_for_cpu(int cpu, void *arg1, void *arg2, +static void dump_clos_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_clos_config clos_config; int ret; - ret = isst_pm_get_clos(cpu, current_clos, &clos_config); + ret = isst_pm_get_clos(id, current_clos, &clos_config); if (ret) isst_display_error_info_message(1, "isst_pm_get_clos failed", 0, 0); else - isst_clos_display_information(cpu, outf, current_clos, + isst_clos_display_information(id, outf, current_clos, &clos_config); } @@ -2164,19 +2164,19 @@ static void dump_clos_config(int arg) isst_ctdp_display_information_end(outf); } -static void get_clos_info_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void get_clos_info_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int enable, ret, prio_type; - ret = isst_clos_get_clos_information(cpu, &enable, &prio_type); + ret = isst_clos_get_clos_information(id, &enable, &prio_type); if (ret) isst_display_error_info_message(1, "isst_clos_get_info failed", 0, 0); else { int cp_state, cp_cap; - isst_read_pm_config(cpu, &cp_state, &cp_cap); - isst_clos_display_clos_information(cpu, outf, enable, prio_type, + isst_read_pm_config(id, &cp_state, &cp_cap); + isst_clos_display_clos_information(id, outf, enable, prio_type, cp_state, cp_cap); } } @@ -2201,25 +2201,22 @@ static void dump_clos_info(int arg) } -static void set_clos_config_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void set_clos_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { struct isst_clos_config clos_config; int ret; - clos_config.pkg_id = get_physical_package_id(cpu); - clos_config.die_id = get_physical_die_id(cpu); - clos_config.epp = clos_epp; clos_config.clos_prop_prio = clos_prop_prio; clos_config.clos_min = clos_min; clos_config.clos_max = clos_max; clos_config.clos_desired = clos_desired; - ret = isst_set_clos(cpu, current_clos, &clos_config); + ret = isst_set_clos(id, current_clos, &clos_config); if (ret) isst_display_error_info_message(1, "isst_set_clos failed", 0, 0); else - isst_display_result(cpu, outf, "core-power", "config", ret); + isst_display_result(id, outf, "core-power", "config", ret); } static void set_clos_config(int arg) @@ -2275,16 +2272,16 @@ static void set_clos_config(int arg) isst_ctdp_display_information_end(outf); } -static void set_clos_assoc_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void set_clos_assoc_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int ret; - ret = isst_clos_associate(cpu, current_clos); + ret = isst_clos_associate(id, current_clos); if (ret) debug_printf("isst_clos_associate failed"); else - isst_display_result(cpu, outf, "core-power", "assoc", ret); + isst_display_result(id, outf, "core-power", "assoc", ret); } static void set_clos_assoc(int arg) @@ -2312,16 +2309,16 @@ static void set_clos_assoc(int arg) } } -static void get_clos_assoc_for_cpu(int cpu, void *arg1, void *arg2, void *arg3, +static void get_clos_assoc_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { int clos, ret; - ret = isst_clos_get_assoc_status(cpu, &clos); + ret = isst_clos_get_assoc_status(id, &clos); if (ret) isst_display_error_info_message(1, "isst_clos_get_assoc_status failed", 0, 0); else - isst_clos_display_assoc_information(cpu, outf, clos); + isst_clos_display_assoc_information(id, outf, clos); } static void get_clos_assoc(int arg) @@ -2343,27 +2340,28 @@ static void get_clos_assoc(int arg) isst_ctdp_display_information_end(outf); } -static void set_turbo_mode_for_cpu(int cpu, int status) +static void set_turbo_mode_for_cpu(struct isst_id *id, int status) { int base_freq; if (status) { - base_freq = get_cpufreq_base_freq(cpu); - set_cpufreq_scaling_min_max(cpu, 1, base_freq); + base_freq = get_cpufreq_base_freq(id->cpu); + set_cpufreq_scaling_min_max(id->cpu, 1, base_freq); } else { - set_scaling_max_to_cpuinfo_max(cpu); + set_scaling_max_to_cpuinfo_max(id); } if (status) { - isst_display_result(cpu, outf, "turbo-mode", "enable", 0); + isst_display_result(id, outf, "turbo-mode", "enable", 0); } else { - isst_display_result(cpu, outf, "turbo-mode", "disable", 0); + isst_display_result(id, outf, "turbo-mode", "disable", 0); } } static void set_turbo_mode(int arg) { int i, enable = arg; + struct isst_id id; if (cmd_help) { if (enable) @@ -2385,14 +2383,16 @@ static void set_turbo_mode(int arg) online = 1; /* online entry for CPU 0 needs some special configs */ - if (online) - set_turbo_mode_for_cpu(i, enable); + if (online) { + set_isst_id(&id, i); + set_turbo_mode_for_cpu(&id, enable); + } } isst_ctdp_display_information_end(outf); } -static void get_set_trl(int cpu, void *arg1, void *arg2, void *arg3, +static void get_set_trl(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { unsigned long long trl; @@ -2405,16 +2405,16 @@ static void get_set_trl(int cpu, void *arg1, void *arg2, void *arg3, } if (set) { - ret = isst_set_trl(cpu, fact_trl); - isst_display_result(cpu, outf, "turbo-mode", "set-trl", ret); + ret = isst_set_trl(id, fact_trl); + isst_display_result(id, outf, "turbo-mode", "set-trl", ret); return; } - ret = isst_get_trl(cpu, &trl); + ret = isst_get_trl(id, &trl); if (ret) - isst_display_result(cpu, outf, "turbo-mode", "get-trl", ret); + isst_display_result(id, outf, "turbo-mode", "get-trl", ret); else - isst_trl_display_information(cpu, outf, trl); + isst_trl_display_information(id, outf, trl); } static void process_trl(int arg) @@ -2754,9 +2754,6 @@ void process_command(int argc, char **argv, } } - if (!is_clx_n_platform()) - create_cpu_map(); - i = 0; while (cmds[i].feature) { if (!strcmp(cmds[i].feature, feature) && @@ -2960,11 +2957,9 @@ static void cmdline(int argc, char **argv) if (force_cpus_online) force_all_cpus_online(); store_cpu_topology(); - set_cpu_present_cpu_mask(); - set_cpu_target_cpu_mask(); + create_cpu_map(); if (oob_mode) { - create_cpu_map(); if (debug_flag) fprintf(stderr, "OOB mode is enabled in debug mode\n"); diff --git a/tools/power/x86/intel-speed-select/isst-core.c b/tools/power/x86/intel-speed-select/isst-core.c index 4431c8a0d40aee8b8014f6d4f2469db1521c1f02..f701b45c832cd39089ec3df5f679157d75075026 100644 --- a/tools/power/x86/intel-speed-select/isst-core.c +++ b/tools/power/x86/intel-speed-select/isst-core.c @@ -6,7 +6,7 @@ #include "isst.h" -int isst_write_pm_config(int cpu, int cp_state) +int isst_write_pm_config(struct isst_id *id, int cp_state) { unsigned int req, resp; int ret; @@ -16,27 +16,27 @@ int isst_write_pm_config(int cpu, int cp_state) else req = 0; - ret = isst_send_mbox_command(cpu, WRITE_PM_CONFIG, PM_FEATURE, 0, req, + ret = isst_send_mbox_command(id->cpu, WRITE_PM_CONFIG, PM_FEATURE, 0, req, &resp); if (ret) return ret; - debug_printf("cpu:%d WRITE_PM_CONFIG resp:%x\n", cpu, resp); + debug_printf("cpu:%d WRITE_PM_CONFIG resp:%x\n", id->cpu, resp); return 0; } -int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap) +int isst_read_pm_config(struct isst_id *id, int *cp_state, int *cp_cap) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, READ_PM_CONFIG, PM_FEATURE, 0, 0, + ret = isst_send_mbox_command(id->cpu, READ_PM_CONFIG, PM_FEATURE, 0, 0, &resp); if (ret) return ret; - debug_printf("cpu:%d READ_PM_CONFIG resp:%x\n", cpu, resp); + debug_printf("cpu:%d READ_PM_CONFIG resp:%x\n", id->cpu, resp); *cp_state = resp & BIT(16); *cp_cap = resp & BIT(0) ? 1 : 0; @@ -44,12 +44,12 @@ int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap) return 0; } -int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev) +int isst_get_ctdp_levels(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_LEVELS_INFO, 0, 0, &resp); if (ret) { pkg_dev->levels = 0; @@ -60,7 +60,7 @@ int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev) return 0; } - debug_printf("cpu:%d CONFIG_TDP_GET_LEVELS_INFO resp:%x\n", cpu, resp); + debug_printf("cpu:%d CONFIG_TDP_GET_LEVELS_INFO resp:%x\n", id->cpu, resp); pkg_dev->version = resp & 0xff; pkg_dev->levels = (resp >> 8) & 0xff; @@ -71,14 +71,14 @@ int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev) return 0; } -int isst_get_ctdp_control(int cpu, int config_index, +int isst_get_ctdp_control(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { int cp_state, cp_cap; unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TDP_CONTROL, 0, config_index, &resp); if (ret) @@ -89,30 +89,30 @@ int isst_get_ctdp_control(int cpu, int config_index, ctdp_level->fact_enabled = !!(resp & BIT(16)); ctdp_level->pbf_enabled = !!(resp & BIT(17)); - ret = isst_read_pm_config(cpu, &cp_state, &cp_cap); + ret = isst_read_pm_config(id, &cp_state, &cp_cap); if (ret) { - debug_printf("cpu:%d pm_config is not supported \n", cpu); + debug_printf("cpu:%d pm_config is not supported\n", id->cpu); } else { - debug_printf("cpu:%d pm_config SST-CP state:%d cap:%d \n", cpu, cp_state, cp_cap); + debug_printf("cpu:%d pm_config SST-CP state:%d cap:%d\n", id->cpu, cp_state, cp_cap); ctdp_level->sst_cp_support = cp_cap; ctdp_level->sst_cp_enabled = cp_state; } debug_printf( "cpu:%d CONFIG_TDP_GET_TDP_CONTROL resp:%x fact_support:%d pbf_support: %d fact_enabled:%d pbf_enabled:%d\n", - cpu, resp, ctdp_level->fact_support, ctdp_level->pbf_support, + id->cpu, resp, ctdp_level->fact_support, ctdp_level->pbf_support, ctdp_level->fact_enabled, ctdp_level->pbf_enabled); return 0; } -int isst_get_tdp_info(int cpu, int config_index, +int isst_get_tdp_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_TDP_INFO, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TDP_INFO, 0, config_index, &resp); if (ret) { isst_display_error_info_message(1, "Invalid level, Can't get TDP information at level", 1, config_index); @@ -124,18 +124,18 @@ int isst_get_tdp_info(int cpu, int config_index, debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_TDP_INFO resp:%x tdp_ratio:%d pkg_tdp:%d\n", - cpu, config_index, resp, ctdp_level->tdp_ratio, + id->cpu, config_index, resp, ctdp_level->tdp_ratio, ctdp_level->pkg_tdp); return 0; } -int isst_get_pwr_info(int cpu, int config_index, +int isst_get_pwr_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_PWR_INFO, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_PWR_INFO, 0, config_index, &resp); if (ret) return ret; @@ -145,18 +145,18 @@ int isst_get_pwr_info(int cpu, int config_index, debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_PWR_INFO resp:%x pkg_max_power:%d pkg_min_power:%d\n", - cpu, config_index, resp, ctdp_level->pkg_max_power, + id->cpu, config_index, resp, ctdp_level->pkg_max_power, ctdp_level->pkg_min_power); return 0; } -void isst_get_uncore_p0_p1_info(int cpu, int config_index, +void isst_get_uncore_p0_p1_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_UNCORE_P0_P1_INFO, 0, config_index, &resp); if (ret) { @@ -169,16 +169,16 @@ void isst_get_uncore_p0_p1_info(int cpu, int config_index, ctdp_level->uncore_p1 = (resp & GENMASK(15, 8)) >> 8; debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_UNCORE_P0_P1_INFO resp:%x uncore p0:%d uncore p1:%d\n", - cpu, config_index, resp, ctdp_level->uncore_p0, + id->cpu, config_index, resp, ctdp_level->uncore_p0, ctdp_level->uncore_p1); } -void isst_get_p1_info(int cpu, int config_index, +void isst_get_p1_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_P1_INFO, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_P1_INFO, 0, config_index, &resp); if (ret) { ctdp_level->sse_p1 = 0; @@ -192,17 +192,17 @@ void isst_get_p1_info(int cpu, int config_index, ctdp_level->avx512_p1 = (resp & GENMASK(23, 16)) >> 16; debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_P1_INFO resp:%x sse_p1:%d avx2_p1:%d avx512_p1:%d\n", - cpu, config_index, resp, ctdp_level->sse_p1, + id->cpu, config_index, resp, ctdp_level->sse_p1, ctdp_level->avx2_p1, ctdp_level->avx512_p1); } -void isst_get_uncore_mem_freq(int cpu, int config_index, +void isst_get_uncore_mem_freq(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_MEM_FREQ, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_MEM_FREQ, 0, config_index, &resp); if (ret) { ctdp_level->mem_freq = 0; @@ -226,16 +226,16 @@ void isst_get_uncore_mem_freq(int cpu, int config_index, } debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_MEM_FREQ resp:%x uncore mem_freq:%d\n", - cpu, config_index, resp, ctdp_level->mem_freq); + id->cpu, config_index, resp, ctdp_level->mem_freq); } -int isst_get_tjmax_info(int cpu, int config_index, +int isst_get_tjmax_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_TJMAX_INFO, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TJMAX_INFO, 0, config_index, &resp); if (ret) return ret; @@ -244,12 +244,12 @@ int isst_get_tjmax_info(int cpu, int config_index, debug_printf( "cpu:%d ctdp:%d CONFIG_TDP_GET_TJMAX_INFO resp:%x t_proc_hot:%d\n", - cpu, config_index, resp, ctdp_level->t_proc_hot); + id->cpu, config_index, resp, ctdp_level->t_proc_hot); return 0; } -int isst_get_coremask_info(int cpu, int config_index, +int isst_get_coremask_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level) { unsigned int resp; @@ -260,7 +260,7 @@ int isst_get_coremask_info(int cpu, int config_index, unsigned long long mask; int cpu_count = 0; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_CORE_MASK, 0, (i << 8) | config_index, &resp); if (ret) @@ -268,27 +268,27 @@ int isst_get_coremask_info(int cpu, int config_index, debug_printf( "cpu:%d ctdp:%d mask:%d CONFIG_TDP_GET_CORE_MASK resp:%x\n", - cpu, config_index, i, resp); + id->cpu, config_index, i, resp); mask = (unsigned long long)resp << (32 * i); - set_cpu_mask_from_punit_coremask(cpu, mask, + set_cpu_mask_from_punit_coremask(id, mask, ctdp_level->core_cpumask_size, ctdp_level->core_cpumask, &cpu_count); ctdp_level->cpu_count += cpu_count; - debug_printf("cpu:%d ctdp:%d mask:%d cpu count:%d\n", cpu, + debug_printf("cpu:%d ctdp:%d mask:%d cpu count:%d\n", id->cpu, config_index, i, ctdp_level->cpu_count); } return 0; } -int isst_get_get_trl_from_msr(int cpu, int *trl) +int isst_get_get_trl_from_msr(struct isst_id *id, int *trl) { unsigned long long msr_trl; int ret; - ret = isst_send_msr_command(cpu, 0x1AD, 0, &msr_trl); + ret = isst_send_msr_command(id->cpu, 0x1AD, 0, &msr_trl); if (ret) return ret; @@ -304,13 +304,13 @@ int isst_get_get_trl_from_msr(int cpu, int *trl) return 0; } -int isst_get_get_trl(int cpu, int level, int avx_level, int *trl) +int isst_get_get_trl(struct isst_id *id, int level, int avx_level, int *trl) { unsigned int req, resp; int ret; req = level | (avx_level << 16); - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TURBO_LIMIT_RATIOS, 0, req, &resp); if (ret) @@ -318,7 +318,7 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl) debug_printf( "cpu:%d CONFIG_TDP_GET_TURBO_LIMIT_RATIOS req:%x resp:%x\n", - cpu, req, resp); + id->cpu, req, resp); trl[0] = resp & GENMASK(7, 0); trl[1] = (resp & GENMASK(15, 8)) >> 8; @@ -326,13 +326,13 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl) trl[3] = (resp & GENMASK(31, 24)) >> 24; req = level | BIT(8) | (avx_level << 16); - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TURBO_LIMIT_RATIOS, 0, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CONFIG_TDP_GET_TURBO_LIMIT req:%x resp:%x\n", cpu, + debug_printf("cpu:%d CONFIG_TDP_GET_TURBO_LIMIT req:%x resp:%x\n", id->cpu, req, resp); trl[4] = resp & GENMASK(7, 0); @@ -343,61 +343,37 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl) return 0; } -int isst_get_trl_bucket_info(int cpu, unsigned long long *buckets_info) +int isst_get_trl_bucket_info(struct isst_id *id, unsigned long long *buckets_info) { int ret; - debug_printf("cpu:%d bucket info via MSR\n", cpu); + debug_printf("cpu:%d bucket info via MSR\n", id->cpu); *buckets_info = 0; - ret = isst_send_msr_command(cpu, 0x1ae, 0, buckets_info); + ret = isst_send_msr_command(id->cpu, 0x1ae, 0, buckets_info); if (ret) return ret; - debug_printf("cpu:%d bucket info via MSR successful 0x%llx\n", cpu, + debug_printf("cpu:%d bucket info via MSR successful 0x%llx\n", id->cpu, *buckets_info); return 0; } -int isst_set_tdp_level_msr(int cpu, int tdp_level) -{ - unsigned long long level = tdp_level; - int ret; - - debug_printf("cpu: tdp_level via MSR %d\n", cpu, tdp_level); - - if (isst_get_config_tdp_lock_status(cpu)) { - isst_display_error_info_message(1, "tdp_locked", 0, 0); - return -1; - } - - if (tdp_level > 2) - return -1; /* invalid value */ - - ret = isst_send_msr_command(cpu, 0x64b, 1, &level); - if (ret) - return ret; - - debug_printf("cpu: tdp_level via MSR successful %d\n", cpu, tdp_level); - - return 0; -} - -int isst_set_tdp_level(int cpu, int tdp_level) +int isst_set_tdp_level(struct isst_id *id, int tdp_level) { unsigned int resp; int ret; - if (isst_get_config_tdp_lock_status(cpu)) { + if (isst_get_config_tdp_lock_status(id)) { isst_display_error_info_message(1, "TDP is locked", 0, 0); return -1; } - ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_SET_LEVEL, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_SET_LEVEL, 0, tdp_level, &resp); if (ret) { isst_display_error_info_message(1, "Set TDP level failed for level", 1, tdp_level); @@ -407,14 +383,14 @@ int isst_set_tdp_level(int cpu, int tdp_level) return 0; } -int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) +int isst_get_pbf_info(struct isst_id *id, int level, struct isst_pbf_info *pbf_info) { struct isst_pkg_ctdp_level_info ctdp_level; struct isst_pkg_ctdp pkg_dev; int i, ret, max_punit_core, max_mask_index; unsigned int req, resp; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { isst_display_error_info_message(1, "Failed to get number of levels", 0, 0); return ret; @@ -425,7 +401,7 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) return -1; } - ret = isst_get_ctdp_control(cpu, level, &ctdp_level); + ret = isst_get_ctdp_control(id, level, &ctdp_level); if (ret) return ret; @@ -436,14 +412,14 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) pbf_info->core_cpumask_size = alloc_cpu_set(&pbf_info->core_cpumask); - max_punit_core = get_max_punit_core_id(get_physical_package_id(cpu), get_physical_die_id(cpu)); + max_punit_core = get_max_punit_core_id(id); max_mask_index = max_punit_core > 32 ? 2 : 1; for (i = 0; i < max_mask_index; ++i) { unsigned long long mask; int count; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_CORE_MASK_INFO, 0, (i << 8) | level, &resp); if (ret) @@ -451,23 +427,23 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) debug_printf( "cpu:%d CONFIG_TDP_PBF_GET_CORE_MASK_INFO resp:%x\n", - cpu, resp); + id->cpu, resp); mask = (unsigned long long)resp << (32 * i); - set_cpu_mask_from_punit_coremask(cpu, mask, + set_cpu_mask_from_punit_coremask(id, mask, pbf_info->core_cpumask_size, pbf_info->core_cpumask, &count); } req = level; - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO, 0, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO resp:%x\n", cpu, + debug_printf("cpu:%d CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO resp:%x\n", id->cpu, resp); pbf_info->p1_low = resp & 0xff; @@ -475,21 +451,21 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info) req = level; ret = isst_send_mbox_command( - cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TDP_INFO, 0, req, &resp); + id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TDP_INFO, 0, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TDP_INFO resp:%x\n", cpu, resp); + debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TDP_INFO resp:%x\n", id->cpu, resp); pbf_info->tdp = resp & 0xffff; req = level; ret = isst_send_mbox_command( - cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TJ_MAX_INFO, 0, req, &resp); + id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TJ_MAX_INFO, 0, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TJ_MAX_INFO resp:%x\n", cpu, + debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TJ_MAX_INFO resp:%x\n", id->cpu, resp); pbf_info->t_control = (resp >> 8) & 0xff; pbf_info->t_prochot = resp & 0xff; @@ -502,7 +478,7 @@ void isst_get_pbf_info_complete(struct isst_pbf_info *pbf_info) free_cpu_set(pbf_info->core_cpumask); } -int isst_set_pbf_fact_status(int cpu, int pbf, int enable) +int isst_set_pbf_fact_status(struct isst_id *id, int pbf, int enable) { struct isst_pkg_ctdp pkg_dev; struct isst_pkg_ctdp_level_info ctdp_level; @@ -510,13 +486,13 @@ int isst_set_pbf_fact_status(int cpu, int pbf, int enable) unsigned int req = 0, resp; int ret; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) - debug_printf("cpu:%d No support for dynamic ISST\n", cpu); + debug_printf("cpu:%d No support for dynamic ISST\n", id->cpu); current_level = pkg_dev.current_level; - ret = isst_get_ctdp_control(cpu, current_level, &ctdp_level); + ret = isst_get_ctdp_control(id, current_level, &ctdp_level); if (ret) return ret; @@ -542,18 +518,18 @@ int isst_set_pbf_fact_status(int cpu, int pbf, int enable) req &= ~BIT(16); } - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_SET_TDP_CONTROL, 0, req, &resp); if (ret) return ret; debug_printf("cpu:%d CONFIG_TDP_SET_TDP_CONTROL pbf/fact:%d req:%x\n", - cpu, pbf, req); + id->cpu, pbf, req); return 0; } -int isst_get_fact_bucket_info(int cpu, int level, +int isst_get_fact_bucket_info(struct isst_id *id, int level, struct isst_fact_bucket_info *bucket_info) { unsigned int resp; @@ -563,7 +539,7 @@ int isst_get_fact_bucket_info(int cpu, int level, int j; ret = isst_send_mbox_command( - cpu, CONFIG_TDP, + id->cpu, CONFIG_TDP, CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_NUMCORES, 0, (i << 8) | level, &resp); if (ret) @@ -571,7 +547,7 @@ int isst_get_fact_bucket_info(int cpu, int level, debug_printf( "cpu:%d CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_NUMCORES index:%d level:%d resp:%x\n", - cpu, i, level, resp); + id->cpu, i, level, resp); for (j = 0; j < 4; ++j) { bucket_info[j + (i * 4)].high_priority_cores_count = @@ -584,7 +560,7 @@ int isst_get_fact_bucket_info(int cpu, int level, int j; ret = isst_send_mbox_command( - cpu, CONFIG_TDP, + id->cpu, CONFIG_TDP, CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_RATIOS, 0, (k << 16) | (i << 8) | level, &resp); if (ret) @@ -592,7 +568,7 @@ int isst_get_fact_bucket_info(int cpu, int level, debug_printf( "cpu:%d CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_RATIOS index:%d level:%d avx:%d resp:%x\n", - cpu, i, level, k, resp); + id->cpu, i, level, k, resp); for (j = 0; j < 4; ++j) { switch (k) { @@ -618,14 +594,14 @@ int isst_get_fact_bucket_info(int cpu, int level, return 0; } -int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_info *fact_info) +int isst_get_fact_info(struct isst_id *id, int level, int fact_bucket, struct isst_fact_info *fact_info) { struct isst_pkg_ctdp_level_info ctdp_level; struct isst_pkg_ctdp pkg_dev; unsigned int resp; int j, ret, print; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { isst_display_error_info_message(1, "Failed to get number of levels", 0, 0); return ret; @@ -636,7 +612,7 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf return -1; } - ret = isst_get_ctdp_control(cpu, level, &ctdp_level); + ret = isst_get_ctdp_control(id, level, &ctdp_level); if (ret) return ret; @@ -645,20 +621,20 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf return -1; } - ret = isst_send_mbox_command(cpu, CONFIG_TDP, + ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_FACT_LP_CLIPPING_RATIO, 0, level, &resp); if (ret) return ret; debug_printf("cpu:%d CONFIG_TDP_GET_FACT_LP_CLIPPING_RATIO resp:%x\n", - cpu, resp); + id->cpu, resp); fact_info->lp_clipping_ratio_license_sse = resp & 0xff; fact_info->lp_clipping_ratio_license_avx2 = (resp >> 8) & 0xff; fact_info->lp_clipping_ratio_license_avx512 = (resp >> 16) & 0xff; - ret = isst_get_fact_bucket_info(cpu, level, fact_info->bucket_info); + ret = isst_get_fact_bucket_info(id, level, fact_info->bucket_info); if (ret) return ret; @@ -680,32 +656,32 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf return 0; } -int isst_get_trl(int cpu, unsigned long long *trl) +int isst_get_trl(struct isst_id *id, unsigned long long *trl) { int ret; - ret = isst_send_msr_command(cpu, 0x1AD, 0, trl); + ret = isst_send_msr_command(id->cpu, 0x1AD, 0, trl); if (ret) return ret; return 0; } -int isst_set_trl(int cpu, unsigned long long trl) +int isst_set_trl(struct isst_id *id, unsigned long long trl) { int ret; if (!trl) trl = 0xFFFFFFFFFFFFFFFFULL; - ret = isst_send_msr_command(cpu, 0x1AD, 1, &trl); + ret = isst_send_msr_command(id->cpu, 0x1AD, 1, &trl); if (ret) return ret; return 0; } -int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl) +int isst_set_trl_from_current_tdp(struct isst_id *id, unsigned long long trl) { unsigned long long msr_trl; int ret; @@ -717,11 +693,11 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl) int trl[8]; int i; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) return ret; - ret = isst_get_get_trl(cpu, pkg_dev.current_level, 0, trl); + ret = isst_get_get_trl(id, pkg_dev.current_level, 0, trl); if (ret) return ret; @@ -732,7 +708,7 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl) msr_trl |= (_trl << (i * 8)); } } - ret = isst_send_msr_command(cpu, 0x1AD, 1, &msr_trl); + ret = isst_send_msr_command(id->cpu, 0x1AD, 1, &msr_trl); if (ret) return ret; @@ -740,12 +716,12 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl) } /* Return 1 if locked */ -int isst_get_config_tdp_lock_status(int cpu) +int isst_get_config_tdp_lock_status(struct isst_id *id) { unsigned long long tdp_control = 0; int ret; - ret = isst_send_msr_command(cpu, 0x64b, 0, &tdp_control); + ret = isst_send_msr_command(id->cpu, 0x64b, 0, &tdp_control); if (ret) return ret; @@ -754,7 +730,7 @@ int isst_get_config_tdp_lock_status(int cpu) return ret; } -void isst_get_process_ctdp_complete(int cpu, struct isst_pkg_ctdp *pkg_dev) +void isst_get_process_ctdp_complete(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev) { int i; @@ -771,19 +747,19 @@ void isst_get_process_ctdp_complete(int cpu, struct isst_pkg_ctdp *pkg_dev) } } -int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev) +int isst_get_process_ctdp(struct isst_id *id, int tdp_level, struct isst_pkg_ctdp *pkg_dev) { int i, ret, valid = 0; if (pkg_dev->processed) return 0; - ret = isst_get_ctdp_levels(cpu, pkg_dev); + ret = isst_get_ctdp_levels(id, pkg_dev); if (ret) return ret; debug_printf("cpu: %d ctdp enable:%d current level: %d levels:%d\n", - cpu, pkg_dev->enabled, pkg_dev->current_level, + id->cpu, pkg_dev->enabled, pkg_dev->current_level, pkg_dev->levels); if (tdp_level != 0xff && tdp_level > pkg_dev->levels) { @@ -800,16 +776,16 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev) if (tdp_level != 0xff && i != tdp_level) continue; - debug_printf("cpu:%d Get Information for TDP level:%d\n", cpu, + debug_printf("cpu:%d Get Information for TDP level:%d\n", id->cpu, i); ctdp_level = &pkg_dev->ctdp_level[i]; ctdp_level->level = i; - ctdp_level->control_cpu = cpu; - ctdp_level->pkg_id = get_physical_package_id(cpu); - ctdp_level->die_id = get_physical_die_id(cpu); + ctdp_level->control_cpu = id->cpu; + ctdp_level->pkg_id = id->pkg; + ctdp_level->die_id = id->die; - ret = isst_get_ctdp_control(cpu, i, ctdp_level); + ret = isst_get_ctdp_control(id, i, ctdp_level); if (ret) continue; @@ -818,13 +794,13 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev) ctdp_level->processed = 1; if (ctdp_level->pbf_support) { - ret = isst_get_pbf_info(cpu, i, &ctdp_level->pbf_info); + ret = isst_get_pbf_info(id, i, &ctdp_level->pbf_info); if (!ret) ctdp_level->pbf_found = 1; } if (ctdp_level->fact_support) { - ret = isst_get_fact_info(cpu, i, 0xff, + ret = isst_get_fact_info(id, i, 0xff, &ctdp_level->fact_info); if (ret) return ret; @@ -833,76 +809,76 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev) if (!pkg_dev->enabled && is_skx_based_platform()) { int freq; - freq = get_cpufreq_base_freq(cpu); + freq = get_cpufreq_base_freq(id->cpu); if (freq > 0) { ctdp_level->sse_p1 = freq / 100000; ctdp_level->tdp_ratio = ctdp_level->sse_p1; } - isst_get_get_trl_from_msr(cpu, ctdp_level->trl_sse_active_cores); - isst_get_trl_bucket_info(cpu, &ctdp_level->buckets_info); + isst_get_get_trl_from_msr(id, ctdp_level->trl_sse_active_cores); + isst_get_trl_bucket_info(id, &ctdp_level->buckets_info); continue; } - ret = isst_get_tdp_info(cpu, i, ctdp_level); + ret = isst_get_tdp_info(id, i, ctdp_level); if (ret) return ret; - ret = isst_get_pwr_info(cpu, i, ctdp_level); + ret = isst_get_pwr_info(id, i, ctdp_level); if (ret) return ret; - ret = isst_get_tjmax_info(cpu, i, ctdp_level); + ret = isst_get_tjmax_info(id, i, ctdp_level); if (ret) return ret; ctdp_level->core_cpumask_size = alloc_cpu_set(&ctdp_level->core_cpumask); - ret = isst_get_coremask_info(cpu, i, ctdp_level); + ret = isst_get_coremask_info(id, i, ctdp_level); if (ret) return ret; - ret = isst_get_trl_bucket_info(cpu, &ctdp_level->buckets_info); + ret = isst_get_trl_bucket_info(id, &ctdp_level->buckets_info); if (ret) return ret; - ret = isst_get_get_trl(cpu, i, 0, + ret = isst_get_get_trl(id, i, 0, ctdp_level->trl_sse_active_cores); if (ret) return ret; - ret = isst_get_get_trl(cpu, i, 1, + ret = isst_get_get_trl(id, i, 1, ctdp_level->trl_avx_active_cores); if (ret) return ret; - ret = isst_get_get_trl(cpu, i, 2, + ret = isst_get_get_trl(id, i, 2, ctdp_level->trl_avx_512_active_cores); if (ret) return ret; - isst_get_uncore_p0_p1_info(cpu, i, ctdp_level); - isst_get_p1_info(cpu, i, ctdp_level); - isst_get_uncore_mem_freq(cpu, i, ctdp_level); + isst_get_uncore_p0_p1_info(id, i, ctdp_level); + isst_get_p1_info(id, i, ctdp_level); + isst_get_uncore_mem_freq(id, i, ctdp_level); } if (!valid) - isst_display_error_info_message(0, "Invalid level, Can't get TDP control information at specified levels on cpu", 1, cpu); + isst_display_error_info_message(0, "Invalid level, Can't get TDP control information at specified levels on cpu", 1, id->cpu); return 0; } -int isst_clos_get_clos_information(int cpu, int *enable, int *type) +int isst_clos_get_clos_information(struct isst_id *id, int *enable, int *type) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0, &resp); if (ret) return ret; - debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", cpu, resp); + debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", id->cpu, resp); if (resp & BIT(1)) *enable = 1; @@ -917,7 +893,7 @@ int isst_clos_get_clos_information(int cpu, int *enable, int *type) return 0; } -int isst_pm_qos_config(int cpu, int enable_clos, int priority_type) +int isst_pm_qos_config(struct isst_id *id, int enable_clos, int priority_type) { unsigned int req, resp; int ret; @@ -926,13 +902,13 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type) struct isst_pkg_ctdp pkg_dev; struct isst_pkg_ctdp_level_info ctdp_level; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { debug_printf("isst_get_ctdp_levels\n"); return ret; } - ret = isst_get_ctdp_control(cpu, pkg_dev.current_level, + ret = isst_get_ctdp_control(id, pkg_dev.current_level, &ctdp_level); if (ret) return ret; @@ -941,23 +917,23 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type) isst_display_error_info_message(1, "Ignoring request, turbo-freq feature is still enabled", 0, 0); return -EINVAL; } - ret = isst_write_pm_config(cpu, 0); + ret = isst_write_pm_config(id, 0); if (ret) isst_display_error_info_message(0, "WRITE_PM_CONFIG command failed, ignoring error", 0, 0); } else { - ret = isst_write_pm_config(cpu, 1); + ret = isst_write_pm_config(id, 1); if (ret) isst_display_error_info_message(0, "WRITE_PM_CONFIG command failed, ignoring error", 0, 0); } - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0, &resp); if (ret) { isst_display_error_info_message(1, "CLOS_PM_QOS_CONFIG command failed", 0, 0); return ret; } - debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", cpu, resp); + debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", id->cpu, resp); req = resp; @@ -974,30 +950,27 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type) else req = req & ~BIT(2); - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, BIT(MBOX_CMD_WRITE_BIT), req, &resp); if (ret) return ret; - debug_printf("cpu:%d CLOS_PM_QOS_CONFIG priority type:%d req:%x\n", cpu, + debug_printf("cpu:%d CLOS_PM_QOS_CONFIG priority type:%d req:%x\n", id->cpu, priority_type, req); return 0; } -int isst_pm_get_clos(int cpu, int clos, struct isst_clos_config *clos_config) +int isst_pm_get_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config) { unsigned int resp; int ret; - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_CLOS, clos, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_CLOS, clos, 0, &resp); if (ret) return ret; - clos_config->pkg_id = get_physical_package_id(cpu); - clos_config->die_id = get_physical_die_id(cpu); - clos_config->epp = resp & 0x0f; clos_config->clos_prop_prio = (resp >> 4) & 0x0f; clos_config->clos_min = (resp >> 8) & 0xff; @@ -1007,7 +980,7 @@ int isst_pm_get_clos(int cpu, int clos, struct isst_clos_config *clos_config) return 0; } -int isst_set_clos(int cpu, int clos, struct isst_clos_config *clos_config) +int isst_set_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config) { unsigned int req, resp; unsigned int param; @@ -1021,53 +994,53 @@ int isst_set_clos(int cpu, int clos, struct isst_clos_config *clos_config) param = BIT(MBOX_CMD_WRITE_BIT) | clos; - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_CLOS, param, req, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_CLOS, param, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CLOS_PM_CLOS param:%x req:%x\n", cpu, param, req); + debug_printf("cpu:%d CLOS_PM_CLOS param:%x req:%x\n", id->cpu, param, req); return 0; } -int isst_clos_get_assoc_status(int cpu, int *clos_id) +int isst_clos_get_assoc_status(struct isst_id *id, int *clos_id) { unsigned int resp; unsigned int param; int core_id, ret; - core_id = find_phy_core_num(cpu); + core_id = find_phy_core_num(id->cpu); param = core_id; - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, 0, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, 0, &resp); if (ret) return ret; - debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x resp:%x\n", cpu, param, + debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x resp:%x\n", id->cpu, param, resp); *clos_id = (resp >> 16) & 0x03; return 0; } -int isst_clos_associate(int cpu, int clos_id) +int isst_clos_associate(struct isst_id *id, int clos_id) { unsigned int req, resp; unsigned int param; int core_id, ret; req = (clos_id & 0x03) << 16; - core_id = find_phy_core_num(cpu); + core_id = find_phy_core_num(id->cpu); param = BIT(MBOX_CMD_WRITE_BIT) | core_id; - ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, + ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, req, &resp); if (ret) return ret; - debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x req:%x\n", cpu, param, + debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x req:%x\n", id->cpu, param, req); return 0; diff --git a/tools/power/x86/intel-speed-select/isst-daemon.c b/tools/power/x86/intel-speed-select/isst-daemon.c index d0400c6684ba93f82bc5e9b9c51dec3ef1568c05..0699137c090147b01b5cdedbbdaaddde00f22bd0 100644 --- a/tools/power/x86/intel-speed-select/isst-daemon.c +++ b/tools/power/x86/intel-speed-select/isst-daemon.c @@ -32,62 +32,60 @@ static void init_levels(void) per_package_levels_info[i][j] = -1; } -void process_level_change(int cpu) +void process_level_change(struct isst_id *id) { struct isst_pkg_ctdp_level_info ctdp_level; - int pkg_id = get_physical_package_id(cpu); - int die_id = get_physical_die_id(cpu); struct isst_pkg_ctdp pkg_dev; time_t tm; int ret; - if (pkg_id >= MAX_PACKAGE_COUNT || die_id >= MAX_DIE_PER_PACKAGE) { - debug_printf("Invalid package/die info for cpu:%d\n", cpu); + if (id->pkg < 0 || id->die < 0) { + debug_printf("Invalid package/die info for cpu:%d\n", id->cpu); return; } tm = time(NULL); - if (tm - per_package_levels_tm[pkg_id][die_id] < 2 ) + if (tm - per_package_levels_tm[id->pkg][id->die] < 2) return; - per_package_levels_tm[pkg_id][die_id] = tm; + per_package_levels_tm[id->pkg][id->die] = tm; - ret = isst_get_ctdp_levels(cpu, &pkg_dev); + ret = isst_get_ctdp_levels(id, &pkg_dev); if (ret) { - debug_printf("Can't get tdp levels for cpu:%d\n", cpu); + debug_printf("Can't get tdp levels for cpu:%d\n", id->cpu); return; } - debug_printf("Get Config level %d pkg:%d die:%d current_level:%d \n", cpu, - pkg_id, die_id, pkg_dev.current_level); + debug_printf("Get Config level %d pkg:%d die:%d current_level:%d\n", id->cpu, + id->pkg, id->die, pkg_dev.current_level); if (pkg_dev.locked) { debug_printf("config TDP s locked \n"); return; } - if (per_package_levels_info[pkg_id][die_id] == pkg_dev.current_level) + if (per_package_levels_info[id->pkg][id->die] == pkg_dev.current_level) return; debug_printf("**Config level change for cpu:%d pkg:%d die:%d from %d to %d\n", - cpu, pkg_id, die_id, per_package_levels_info[pkg_id][die_id], + id->cpu, id->pkg, id->die, per_package_levels_info[id->pkg][id->die], pkg_dev.current_level); - per_package_levels_info[pkg_id][die_id] = pkg_dev.current_level; + per_package_levels_info[id->pkg][id->die] = pkg_dev.current_level; ctdp_level.core_cpumask_size = alloc_cpu_set(&ctdp_level.core_cpumask); - ret = isst_get_coremask_info(cpu, pkg_dev.current_level, &ctdp_level); + ret = isst_get_coremask_info(id, pkg_dev.current_level, &ctdp_level); if (ret) { free_cpu_set(ctdp_level.core_cpumask); - debug_printf("Can't get core_mask:%d\n", cpu); + debug_printf("Can't get core_mask:%d\n", id->cpu); return; } if (ctdp_level.cpu_count) { int i, max_cpus = get_topo_max_cpus(); for (i = 0; i < max_cpus; ++i) { - if (pkg_id != get_physical_package_id(i) || die_id != get_physical_die_id(i)) + if (!is_cpu_in_power_domain(i, id)) continue; if (CPU_ISSET_S(i, ctdp_level.core_cpumask_size, ctdp_level.core_cpumask)) { fprintf(stderr, "online cpu %d\n", i); @@ -102,10 +100,10 @@ void process_level_change(int cpu) free_cpu_set(ctdp_level.core_cpumask); } -static void _poll_for_config_change(int cpu, void *arg1, void *arg2, +static void _poll_for_config_change(struct isst_id *id, void *arg1, void *arg2, void *arg3, void *arg4) { - process_level_change(cpu); + process_level_change(id); } static void poll_for_config_change(void) diff --git a/tools/power/x86/intel-speed-select/isst-display.c b/tools/power/x86/intel-speed-select/isst-display.c index f97d8859ada728b53a12f2bc55558c648ff20dfc..b19f57d30f550414bc6ef34a3dcd29e44d25cfcb 100644 --- a/tools/power/x86/intel-speed-select/isst-display.c +++ b/tools/power/x86/intel-speed-select/isst-display.c @@ -166,29 +166,27 @@ static void format_and_print(FILE *outf, int level, char *header, char *value) last_level = level; } -static int print_package_info(int cpu, FILE *outf) +static int print_package_info(struct isst_id *id, FILE *outf) { char header[256]; if (out_format_is_json()) { snprintf(header, sizeof(header), "package-%d:die-%d:cpu-%d", - get_physical_package_id(cpu), get_physical_die_id(cpu), - cpu); + id->pkg, id->die, id->cpu); format_and_print(outf, 1, header, NULL); return 1; } - snprintf(header, sizeof(header), "package-%d", - get_physical_package_id(cpu)); + snprintf(header, sizeof(header), "package-%d", id->pkg); format_and_print(outf, 1, header, NULL); - snprintf(header, sizeof(header), "die-%d", get_physical_die_id(cpu)); + snprintf(header, sizeof(header), "die-%d", id->die); format_and_print(outf, 2, header, NULL); - snprintf(header, sizeof(header), "cpu-%d", cpu); + snprintf(header, sizeof(header), "cpu-%d", id->cpu); format_and_print(outf, 3, header, NULL); return 3; } -static void _isst_pbf_display_information(int cpu, FILE *outf, int level, +static void _isst_pbf_display_information(struct isst_id *id, FILE *outf, int level, struct isst_pbf_info *pbf_info, int disp_level) { @@ -231,7 +229,7 @@ static void _isst_pbf_display_information(int cpu, FILE *outf, int level, format_and_print(outf, disp_level + 1, header, value); } -static void _isst_fact_display_information(int cpu, FILE *outf, int level, +static void _isst_fact_display_information(struct isst_id *id, FILE *outf, int level, int fact_bucket, int fact_avx, struct isst_fact_info *fact_info, int base_level) @@ -319,7 +317,7 @@ static void _isst_fact_display_information(int cpu, FILE *outf, int level, format_and_print(outf, base_level + 2, header, value); } -void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix, +void isst_ctdp_display_core_info(struct isst_id *id, FILE *outf, char *prefix, unsigned int val, char *str0, char *str1) { char header[256]; @@ -328,17 +326,14 @@ void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix, if (out_format_is_json()) { snprintf(header, sizeof(header), "package-%d:die-%d:cpu-%d", - get_physical_package_id(cpu), get_physical_die_id(cpu), - cpu); + id->pkg, id->die, id->cpu); format_and_print(outf, level++, header, NULL); } else { - snprintf(header, sizeof(header), "package-%d", - get_physical_package_id(cpu)); + snprintf(header, sizeof(header), "package-%d", id->pkg); format_and_print(outf, level++, header, NULL); - snprintf(header, sizeof(header), "die-%d", - get_physical_die_id(cpu)); + snprintf(header, sizeof(header), "die-%d", id->die); format_and_print(outf, level++, header, NULL); - snprintf(header, sizeof(header), "cpu-%d", cpu); + snprintf(header, sizeof(header), "cpu-%d", id->cpu); format_and_print(outf, level++, header, NULL); } @@ -353,7 +348,7 @@ void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix, format_and_print(outf, 1, NULL, NULL); } -void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, +void isst_ctdp_display_information(struct isst_id *id, FILE *outf, int tdp_level, struct isst_pkg_ctdp *pkg_dev) { char header[256]; @@ -362,7 +357,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, int i; if (pkg_dev->processed) - level = print_package_info(cpu, outf); + level = print_package_info(id, outf); for (i = 0; i <= pkg_dev->levels; ++i) { struct isst_pkg_ctdp_level_info *ctdp_level; @@ -377,8 +372,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, format_and_print(outf, level + 1, header, NULL); snprintf(header, sizeof(header), "cpu-count"); - j = get_cpu_count(get_physical_die_id(cpu), - get_physical_die_id(cpu)); + j = get_cpu_count(id); snprintf(value, sizeof(value), "%d", j); format_and_print(outf, level + 2, header, value); @@ -485,7 +479,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, if (is_clx_n_platform()) { if (ctdp_level->pbf_support) - _isst_pbf_display_information(cpu, outf, + _isst_pbf_display_information(id, outf, tdp_level, &ctdp_level->pbf_info, level + 2); @@ -557,11 +551,11 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, } if (ctdp_level->pbf_support) - _isst_pbf_display_information(cpu, outf, i, + _isst_pbf_display_information(id, outf, i, &ctdp_level->pbf_info, level + 2); if (ctdp_level->fact_support) - _isst_fact_display_information(cpu, outf, i, 0xff, 0xff, + _isst_fact_display_information(id, outf, i, 0xff, 0xff, &ctdp_level->fact_info, level + 2); } @@ -583,36 +577,36 @@ void isst_ctdp_display_information_end(FILE *outf) start = 0; } -void isst_pbf_display_information(int cpu, FILE *outf, int level, +void isst_pbf_display_information(struct isst_id *id, FILE *outf, int level, struct isst_pbf_info *pbf_info) { int _level; - _level = print_package_info(cpu, outf); - _isst_pbf_display_information(cpu, outf, level, pbf_info, _level + 1); + _level = print_package_info(id, outf); + _isst_pbf_display_information(id, outf, level, pbf_info, _level + 1); format_and_print(outf, 1, NULL, NULL); } -void isst_fact_display_information(int cpu, FILE *outf, int level, +void isst_fact_display_information(struct isst_id *id, FILE *outf, int level, int fact_bucket, int fact_avx, struct isst_fact_info *fact_info) { int _level; - _level = print_package_info(cpu, outf); - _isst_fact_display_information(cpu, outf, level, fact_bucket, fact_avx, + _level = print_package_info(id, outf); + _isst_fact_display_information(id, outf, level, fact_bucket, fact_avx, fact_info, _level + 1); format_and_print(outf, 1, NULL, NULL); } -void isst_clos_display_information(int cpu, FILE *outf, int clos, +void isst_clos_display_information(struct isst_id *id, FILE *outf, int clos, struct isst_clos_config *clos_config) { char header[256]; char value[256]; int level; - level = print_package_info(cpu, outf); + level = print_package_info(id, outf); snprintf(header, sizeof(header), "core-power"); format_and_print(outf, level + 1, header, NULL); @@ -647,7 +641,7 @@ void isst_clos_display_information(int cpu, FILE *outf, int clos, format_and_print(outf, level, NULL, NULL); } -void isst_clos_display_clos_information(int cpu, FILE *outf, +void isst_clos_display_clos_information(struct isst_id *id, FILE *outf, int clos_enable, int type, int state, int cap) { @@ -655,7 +649,7 @@ void isst_clos_display_clos_information(int cpu, FILE *outf, char value[256]; int level; - level = print_package_info(cpu, outf); + level = print_package_info(id, outf); snprintf(header, sizeof(header), "core-power"); format_and_print(outf, level + 1, header, NULL); @@ -691,13 +685,13 @@ void isst_clos_display_clos_information(int cpu, FILE *outf, format_and_print(outf, level, NULL, NULL); } -void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos) +void isst_clos_display_assoc_information(struct isst_id *id, FILE *outf, int clos) { char header[256]; char value[256]; int level; - level = print_package_info(cpu, outf); + level = print_package_info(id, outf); snprintf(header, sizeof(header), "get-assoc"); format_and_print(outf, level + 1, header, NULL); @@ -709,15 +703,15 @@ void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos) format_and_print(outf, level, NULL, NULL); } -void isst_display_result(int cpu, FILE *outf, char *feature, char *cmd, +void isst_display_result(struct isst_id *id, FILE *outf, char *feature, char *cmd, int result) { char header[256]; char value[256]; int level = 3; - if (cpu >= 0) - level = print_package_info(cpu, outf); + if (id->cpu >= 0) + level = print_package_info(id, outf); snprintf(header, sizeof(header), "%s", feature); format_and_print(outf, level + 1, header, NULL); @@ -772,13 +766,13 @@ void isst_display_error_info_message(int error, char *msg, int arg_valid, int ar format_and_print(outf, 0, NULL, NULL); } -void isst_trl_display_information(int cpu, FILE *outf, unsigned long long trl) +void isst_trl_display_information(struct isst_id *id, FILE *outf, unsigned long long trl) { char header[256]; char value[256]; int level; - level = print_package_info(cpu, outf); + level = print_package_info(id, outf); snprintf(header, sizeof(header), "get-trl"); format_and_print(outf, level + 1, header, NULL); diff --git a/tools/power/x86/intel-speed-select/isst.h b/tools/power/x86/intel-speed-select/isst.h index 0796d8c6a8827421625dbd71e8342b35684b6a90..409fcc9c80332b6c0e480d5bdfba9565eaa6ef2d 100644 --- a/tools/power/x86/intel-speed-select/isst.h +++ b/tools/power/x86/intel-speed-select/isst.h @@ -79,9 +79,14 @@ #define MAX_PACKAGE_COUNT 8 #define MAX_DIE_PER_PACKAGE 2 +/* Unified structure to specific a CPU or a Power Domain */ +struct isst_id { + int cpu; + int pkg; + int die; +}; + struct isst_clos_config { - int pkg_id; - int die_id; unsigned char epp; unsigned char clos_prop_prio; unsigned char clos_min; @@ -171,22 +176,20 @@ struct isst_pkg_ctdp { struct isst_pkg_ctdp_level_info ctdp_level[ISST_MAX_TDP_LEVELS]; }; +extern int is_cpu_in_power_domain(int cpu, struct isst_id *id); extern int get_topo_max_cpus(void); -extern int get_cpu_count(int pkg_id, int die_id); -extern int get_max_punit_core_id(int pkg_id, int die_id); +extern int get_cpu_count(struct isst_id *id); +extern int get_max_punit_core_id(struct isst_id *id); /* Common interfaces */ FILE *get_output_file(void); extern void debug_printf(const char *format, ...); extern int out_format_is_json(void); -extern int get_physical_package_id(int cpu); -extern int get_physical_die_id(int cpu); +extern void set_isst_id(struct isst_id *id, int cpu); extern size_t alloc_cpu_set(cpu_set_t **cpu_set); extern void free_cpu_set(cpu_set_t *cpu_set); -extern int find_logical_cpu(int pkg_id, int die_id, int phy_cpu); -extern int find_phy_cpu_num(int logical_cpu); extern int find_phy_core_num(int logical_cpu); -extern void set_cpu_mask_from_punit_coremask(int cpu, +extern void set_cpu_mask_from_punit_coremask(struct isst_id *id, unsigned long long core_mask, size_t core_cpumask_size, cpu_set_t *core_cpumask, @@ -200,77 +203,74 @@ extern int isst_send_mbox_command(unsigned int cpu, unsigned char command, extern int isst_send_msr_command(unsigned int cpu, unsigned int command, int write, unsigned long long *req_resp); -extern int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev); -extern int isst_get_ctdp_control(int cpu, int config_index, +extern int isst_get_ctdp_levels(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev); +extern int isst_get_ctdp_control(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level); -extern int isst_get_coremask_info(int cpu, int config_index, +extern int isst_get_coremask_info(struct isst_id *id, int config_index, struct isst_pkg_ctdp_level_info *ctdp_level); -extern int isst_get_process_ctdp(int cpu, int tdp_level, +extern int isst_get_process_ctdp(struct isst_id *id, int tdp_level, struct isst_pkg_ctdp *pkg_dev); -extern void isst_get_process_ctdp_complete(int cpu, +extern void isst_get_process_ctdp_complete(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev); -extern void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level, +extern void isst_ctdp_display_information(struct isst_id *id, FILE *outf, int tdp_level, struct isst_pkg_ctdp *pkg_dev); -extern void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix, +extern void isst_ctdp_display_core_info(struct isst_id *id, FILE *outf, char *prefix, unsigned int val, char *str0, char *str1); extern void isst_ctdp_display_information_start(FILE *outf); extern void isst_ctdp_display_information_end(FILE *outf); -extern void isst_pbf_display_information(int cpu, FILE *outf, int level, +extern void isst_pbf_display_information(struct isst_id *id, FILE *outf, int level, struct isst_pbf_info *info); -extern int isst_set_tdp_level(int cpu, int tdp_level); -extern int isst_set_tdp_level_msr(int cpu, int tdp_level); -extern int isst_set_pbf_fact_status(int cpu, int pbf, int enable); -extern int isst_get_pbf_info(int cpu, int level, +extern int isst_set_tdp_level(struct isst_id *id, int tdp_level); +extern int isst_set_pbf_fact_status(struct isst_id *id, int pbf, int enable); +extern int isst_get_pbf_info(struct isst_id *id, int level, struct isst_pbf_info *pbf_info); extern void isst_get_pbf_info_complete(struct isst_pbf_info *pbf_info); -extern int isst_get_fact_info(int cpu, int level, int fact_bucket, +extern int isst_get_fact_info(struct isst_id *id, int level, int fact_bucket, struct isst_fact_info *fact_info); -extern int isst_get_fact_bucket_info(int cpu, int level, +extern int isst_get_fact_bucket_info(struct isst_id *id, int level, struct isst_fact_bucket_info *bucket_info); -extern void isst_fact_display_information(int cpu, FILE *outf, int level, +extern void isst_fact_display_information(struct isst_id *id, FILE *outf, int level, int fact_bucket, int fact_avx, struct isst_fact_info *fact_info); -extern int isst_set_trl(int cpu, unsigned long long trl); -extern int isst_get_trl(int cpu, unsigned long long *trl); -extern int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl); -extern int isst_get_config_tdp_lock_status(int cpu); +extern int isst_set_trl(struct isst_id *id, unsigned long long trl); +extern int isst_get_trl(struct isst_id *id, unsigned long long *trl); +extern int isst_set_trl_from_current_tdp(struct isst_id *id, unsigned long long trl); +extern int isst_get_config_tdp_lock_status(struct isst_id *id); -extern int isst_pm_qos_config(int cpu, int enable_clos, int priority_type); -extern int isst_pm_get_clos(int cpu, int clos, +extern int isst_pm_qos_config(struct isst_id *id, int enable_clos, int priority_type); +extern int isst_pm_get_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config); -extern int isst_set_clos(int cpu, int clos, +extern int isst_set_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config); -extern int isst_clos_associate(int cpu, int clos); -extern int isst_clos_get_assoc_status(int cpu, int *clos_id); -extern void isst_clos_display_information(int cpu, FILE *outf, int clos, +extern int isst_clos_associate(struct isst_id *id, int clos); +extern int isst_clos_get_assoc_status(struct isst_id *id, int *clos_id); +extern void isst_clos_display_information(struct isst_id *id, FILE *outf, int clos, struct isst_clos_config *clos_config); -extern void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos); -extern int isst_read_reg(unsigned short reg, unsigned int *val); -extern int isst_write_reg(int reg, unsigned int val); +extern void isst_clos_display_assoc_information(struct isst_id *id, FILE *outf, int clos); -extern void isst_display_result(int cpu, FILE *outf, char *feature, char *cmd, +extern void isst_display_result(struct isst_id *id, FILE *outf, char *feature, char *cmd, int result); -extern int isst_clos_get_clos_information(int cpu, int *enable, int *type); -extern void isst_clos_display_clos_information(int cpu, FILE *outf, +extern int isst_clos_get_clos_information(struct isst_id *id, int *enable, int *type); +extern void isst_clos_display_clos_information(struct isst_id *id, FILE *outf, int clos_enable, int type, int state, int cap); extern int is_clx_n_platform(void); extern int get_cpufreq_base_freq(int cpu); -extern int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap); +extern int isst_read_pm_config(struct isst_id *id, int *cp_state, int *cp_cap); extern void isst_display_error_info_message(int error, char *msg, int arg_valid, int arg); extern int is_skx_based_platform(void); extern int is_spr_platform(void); extern int is_icx_platform(void); -extern void isst_trl_display_information(int cpu, FILE *outf, unsigned long long trl); +extern void isst_trl_display_information(struct isst_id *id, FILE *outf, unsigned long long trl); extern void set_cpu_online_offline(int cpu, int state); -extern void for_each_online_package_in_set(void (*callback)(int, void *, void *, +extern void for_each_online_package_in_set(void (*callback)(struct isst_id *, void *, void *, void *, void *), void *arg1, void *arg2, void *arg3, void *arg4); extern int isst_daemon(int debug_mode, int poll_interval, int no_daemon); -extern void process_level_change(int cpu); +extern void process_level_change(struct isst_id *id); extern int hfi_main(void); extern void hfi_exit(void); #endif diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 831dc32d45fad849161a846ac3e476e23bd4e82c..aba460410dbd1b040942806edf676b01b163da8e 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -230,6 +230,7 @@ unsigned int do_slm_cstates; unsigned int use_c1_residency_msr; unsigned int has_aperf; unsigned int has_epb; +unsigned int has_turbo; unsigned int is_hybrid; unsigned int do_irtl_snb; unsigned int do_irtl_hsw; @@ -4080,13 +4081,11 @@ static void remove_underbar(char *s) *to = 0; } -static void dump_cstate_pstate_config_info(unsigned int family, unsigned int model) +static void dump_turbo_ratio_info(unsigned int family, unsigned int model) { - if (!do_nhm_platform_info) + if (!has_turbo) return; - dump_nhm_platform_info(); - if (has_hsw_turbo_ratio_limit(family, model)) dump_hsw_turbo_ratio_limits(); @@ -4108,7 +4107,15 @@ static void dump_cstate_pstate_config_info(unsigned int family, unsigned int mod if (has_config_tdp(family, model)) dump_config_tdp(); +} + +static void dump_cstate_pstate_config_info(unsigned int family, unsigned int model) +{ + if (!do_nhm_platform_info) + return; + dump_nhm_platform_info(); + dump_turbo_ratio_info(family, model); dump_nhm_cst_cfg(); } @@ -4560,7 +4567,6 @@ static double rapl_dram_energy_units_probe(int model, double rapl_energy_units) case INTEL_FAM6_SKYLAKE_X: /* SKX */ case INTEL_FAM6_XEON_PHI_KNL: /* KNL */ case INTEL_FAM6_ICELAKE_X: /* ICX */ - case INTEL_FAM6_SAPPHIRERAPIDS_X: /* SPR */ return (rapl_dram_energy_units = 15.3 / 1000000); default: return (rapl_energy_units); @@ -5447,6 +5453,9 @@ unsigned int intel_model_duplicates(unsigned int model) case INTEL_FAM6_ALDERLAKE_N: case INTEL_FAM6_RAPTORLAKE: case INTEL_FAM6_RAPTORLAKE_P: + case INTEL_FAM6_RAPTORLAKE_S: + case INTEL_FAM6_METEORLAKE: + case INTEL_FAM6_METEORLAKE_L: return INTEL_FAM6_CANNONLAKE_L; case INTEL_FAM6_ATOM_TREMONT_L: @@ -5505,7 +5514,6 @@ void process_cpuid() { unsigned int eax, ebx, ecx, edx; unsigned int fms, family, model, stepping, ecx_flags, edx_flags; - unsigned int has_turbo; unsigned long long ucode_patch = 0; eax = ebx = ecx = edx = 0; @@ -6217,7 +6225,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2022.07.28 - Len Brown <lenb@kernel.org>\n"); + fprintf(outf, "turbostat version 2022.10.04 - Len Brown <lenb@kernel.org>\n"); } #define COMMAND_LINE_SIZE 2048 diff --git a/tools/testing/kunit/configs/all_tests_uml.config b/tools/testing/kunit/configs/all_tests.config similarity index 93% rename from tools/testing/kunit/configs/all_tests_uml.config rename to tools/testing/kunit/configs/all_tests.config index bdee36bef4a3c3cf8afcc3fc9be81ec50eb825fb..f990cbb7325003e3a0c855c36fada14470bb2f31 100644 --- a/tools/testing/kunit/configs/all_tests_uml.config +++ b/tools/testing/kunit/configs/all_tests.config @@ -16,8 +16,6 @@ CONFIG_EXT4_FS=y CONFIG_MSDOS_FS=y CONFIG_VFAT_FS=y -CONFIG_VIRTIO_UML=y -CONFIG_UML_PCI_OVER_VIRTIO=y CONFIG_PCI=y CONFIG_USB4=y diff --git a/tools/testing/kunit/configs/broken_on_uml.config b/tools/testing/kunit/configs/broken_on_uml.config deleted file mode 100644 index 690870043ac0e3a389f8ceb52530aebc62dcb7f8..0000000000000000000000000000000000000000 --- a/tools/testing/kunit/configs/broken_on_uml.config +++ /dev/null @@ -1,44 +0,0 @@ -# These are currently broken on UML and prevent allyesconfig from building -# CONFIG_STATIC_LINK is not set -# CONFIG_UML_NET_VECTOR is not set -# CONFIG_UML_NET_VDE is not set -# CONFIG_UML_NET_PCAP is not set -# CONFIG_NET_PTP_CLASSIFY is not set -# CONFIG_IP_VS is not set -# CONFIG_BRIDGE_EBT_BROUTE is not set -# CONFIG_BRIDGE_EBT_T_FILTER is not set -# CONFIG_BRIDGE_EBT_T_NAT is not set -# CONFIG_MTD_NAND_CADENCE is not set -# CONFIG_MTD_NAND_NANDSIM is not set -# CONFIG_BLK_DEV_NULL_BLK is not set -# CONFIG_BLK_DEV_RAM is not set -# CONFIG_SCSI_DEBUG is not set -# CONFIG_NET_VENDOR_XILINX is not set -# CONFIG_NULL_TTY is not set -# CONFIG_PTP_1588_CLOCK is not set -# CONFIG_PINCTRL_EQUILIBRIUM is not set -# CONFIG_DMABUF_SELFTESTS is not set -# CONFIG_COMEDI is not set -# CONFIG_XIL_AXIS_FIFO is not set -# CONFIG_EXFAT_FS is not set -# CONFIG_STM_DUMMY is not set -# CONFIG_FSI_MASTER_ASPEED is not set -# CONFIG_JFS_FS is not set -# CONFIG_UBIFS_FS is not set -# CONFIG_CRAMFS is not set -# CONFIG_CRYPTO_DEV_SAFEXCEL is not set -# CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set -# CONFIG_KCOV is not set -# CONFIG_LKDTM is not set -# CONFIG_REED_SOLOMON_TEST is not set -# CONFIG_TEST_RHASHTABLE is not set -# CONFIG_TEST_MEMINIT is not set -# CONFIG_NETWORK_PHY_TIMESTAMPING is not set -# CONFIG_DEBUG_INFO_BTF is not set -# CONFIG_PTP_1588_CLOCK_INES is not set -# CONFIG_QCOM_CPR is not set -# CONFIG_RESET_BRCMSTB_RESCAL is not set -# CONFIG_RESET_INTEL_GW is not set -# CONFIG_ADI_AXI_ADC is not set -# CONFIG_DEBUG_PAGEALLOC is not set -# CONFIG_PAGE_POISONING is not set diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index e132b065402972eaf2804b0af91c66033471980f..4d4663fb578bdafef78c08ff71feda83f69a4ef8 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -44,7 +44,6 @@ class KunitConfigRequest: @dataclass class KunitBuildRequest(KunitConfigRequest): jobs: int - alltests: bool @dataclass class KunitParseRequest: @@ -55,7 +54,6 @@ class KunitParseRequest: class KunitExecRequest(KunitParseRequest): build_dir: str timeout: int - alltests: bool filter_glob: str kernel_args: Optional[List[str]] run_isolated: Optional[str] @@ -90,8 +88,7 @@ def build_tests(linux: kunit_kernel.LinuxSourceTree, stdout.print_with_timestamp('Building KUnit Kernel ...') build_start = time.time() - success = linux.build_kernel(request.alltests, - request.jobs, + success = linux.build_kernel(request.jobs, request.build_dir, request.make_options) build_end = time.time() @@ -118,7 +115,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) args.extend(request.kernel_args) output = linux.run_kernel(args=args, - timeout=None if request.alltests else request.timeout, + timeout=request.timeout, filter_glob=request.filter_glob, build_dir=request.build_dir) lines = kunit_parser.extract_tap_lines(output) @@ -165,7 +162,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) - test_start = time.time() run_result = linux.run_kernel( args=request.kernel_args, - timeout=None if request.alltests else request.timeout, + timeout=request.timeout, filter_glob=filter_glob, build_dir=request.build_dir) @@ -206,7 +203,7 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input if request.raw_output == 'all': pass elif request.raw_output == 'kunit': - output = kunit_parser.extract_tap_lines(output) + output = kunit_parser.extract_tap_lines(output, lstrip=False) for line in output: print(line.rstrip()) @@ -288,7 +285,7 @@ def add_common_opts(parser) -> None: help='X=Y make option, can be repeated.', action='append', metavar='X=Y') parser.add_argument('--alltests', - help='Run all KUnit tests through allyesconfig', + help='Run all KUnit tests via tools/testing/kunit/configs/all_tests.config', action='store_true') parser.add_argument('--kunitconfig', help='Path to Kconfig fragment that enables KUnit tests.' @@ -381,8 +378,14 @@ def tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree for arg in cli_args.qemu_args: qemu_args.extend(shlex.split(arg)) + kunitconfigs = cli_args.kunitconfig if cli_args.kunitconfig else [] + if cli_args.alltests: + # Prepend so user-specified options take prio if we ever allow + # --kunitconfig options to have differing options. + kunitconfigs = [kunit_kernel.ALL_TESTS_CONFIG_PATH] + kunitconfigs + return kunit_kernel.LinuxSourceTree(cli_args.build_dir, - kunitconfig_paths=cli_args.kunitconfig, + kunitconfig_paths=kunitconfigs, kconfig_add=cli_args.kconfig_add, arch=cli_args.arch, cross_compile=cli_args.cross_compile, @@ -441,7 +444,6 @@ def main(argv): request = KunitRequest(build_dir=cli_args.build_dir, make_options=cli_args.make_options, jobs=cli_args.jobs, - alltests=cli_args.alltests, raw_output=cli_args.raw_output, json=cli_args.json, timeout=cli_args.timeout, @@ -469,8 +471,7 @@ def main(argv): linux = tree_from_args(cli_args) request = KunitBuildRequest(build_dir=cli_args.build_dir, make_options=cli_args.make_options, - jobs=cli_args.jobs, - alltests=cli_args.alltests) + jobs=cli_args.jobs) result = config_and_build_tests(linux, request) stdout.print_with_timestamp(( 'Elapsed time: %.3fs\n') % ( @@ -483,7 +484,6 @@ def main(argv): build_dir=cli_args.build_dir, json=cli_args.json, timeout=cli_args.timeout, - alltests=cli_args.alltests, filter_glob=cli_args.filter_glob, kernel_args=cli_args.kernel_args, run_isolated=cli_args.run_isolated) diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index f5c26ea8971486cd599d5b2a5d918aa5c05add69..53e90c3358348dbef24c2bdba395decf60b95ab2 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -25,7 +25,7 @@ KCONFIG_PATH = '.config' KUNITCONFIG_PATH = '.kunitconfig' OLD_KUNITCONFIG_PATH = 'last_used_kunitconfig' DEFAULT_KUNITCONFIG_PATH = 'tools/testing/kunit/configs/default.config' -BROKEN_ALLCONFIG_PATH = 'tools/testing/kunit/configs/broken_on_uml.config' +ALL_TESTS_CONFIG_PATH = 'tools/testing/kunit/configs/all_tests.config' UML_KCONFIG_PATH = 'tools/testing/kunit/configs/arch_uml.config' OUTFILE_PATH = 'test.log' ABS_TOOL_PATH = os.path.abspath(os.path.dirname(__file__)) @@ -57,9 +57,6 @@ class LinuxSourceTreeOperations: def make_arch_config(self, base_kunitconfig: kunit_config.Kconfig) -> kunit_config.Kconfig: return base_kunitconfig - def make_allyesconfig(self, build_dir: str, make_options) -> None: - raise ConfigError('Only the "um" arch is supported for alltests') - def make_olddefconfig(self, build_dir: str, make_options) -> None: command = ['make', 'ARCH=' + self._linux_arch, 'O=' + build_dir, 'olddefconfig'] if self._cross_compile: @@ -144,26 +141,6 @@ class LinuxSourceTreeOperationsUml(LinuxSourceTreeOperations): kconfig.merge_in_entries(base_kunitconfig) return kconfig - def make_allyesconfig(self, build_dir: str, make_options) -> None: - stdout.print_with_timestamp( - 'Enabling all CONFIGs for UML...') - command = ['make', 'ARCH=um', 'O=' + build_dir, 'allyesconfig'] - if make_options: - command.extend(make_options) - process = subprocess.Popen( - command, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT) - process.wait() - stdout.print_with_timestamp( - 'Disabling broken configs to run KUnit tests...') - - with open(get_kconfig_path(build_dir), 'a') as config: - with open(BROKEN_ALLCONFIG_PATH, 'r') as disable: - config.write(disable.read()) - stdout.print_with_timestamp( - 'Starting Kernel with all configs takes a few minutes...') - def start(self, params: List[str], build_dir: str) -> subprocess.Popen: """Runs the Linux UML binary. Must be named 'linux'.""" linux_bin = os.path.join(build_dir, 'linux') @@ -343,10 +320,8 @@ class LinuxSourceTree: os.remove(kconfig_path) return self.build_config(build_dir, make_options) - def build_kernel(self, alltests, jobs, build_dir: str, make_options) -> bool: + def build_kernel(self, jobs, build_dir: str, make_options) -> bool: try: - if alltests: - self._ops.make_allyesconfig(build_dir, make_options) self._ops.make_olddefconfig(build_dir, make_options) self._ops.make(jobs, build_dir, make_options) except (ConfigError, BuildError) as e: @@ -359,6 +334,7 @@ class LinuxSourceTree: args = [] if filter_glob: args.append('kunit.filter_glob='+filter_glob) + args.append('kunit.enable=1') process = self._ops.start(args, build_dir) assert process.stdout is not None # tell mypy it's set diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py index 12d3ec77f427c3b252568f3d92fbc712ab29d67e..1ae873e3e34159b55e6a969d52d31dae74861e06 100644 --- a/tools/testing/kunit/kunit_parser.py +++ b/tools/testing/kunit/kunit_parser.py @@ -218,7 +218,7 @@ TAP_START = re.compile(r'TAP version ([0-9]+)$') KTAP_END = re.compile('(List of all partitions:|' 'Kernel panic - not syncing: VFS:|reboot: System halted)') -def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream: +def extract_tap_lines(kernel_output: Iterable[str], lstrip=True) -> LineStream: """Extracts KTAP lines from the kernel output.""" def isolate_ktap_output(kernel_output: Iterable[str]) \ -> Iterator[Tuple[int, str]]: @@ -244,9 +244,11 @@ def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream: # stop extracting KTAP lines break elif started: - # remove prefix and any indention and yield - # line with line number - line = line[prefix_len:].lstrip() + # remove the prefix and optionally any leading + # whitespace. Our parsing logic relies on this. + line = line[prefix_len:] + if lstrip: + line = line.lstrip() yield line_num, line return LineStream(lines=isolate_ktap_output(kernel_output)) diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index 446ac432d9a409551a4594a4b484dfe4105db812..e2cd2cc2e98f6fe85015673ede28acd25a15b001 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -549,7 +549,7 @@ class KUnitMainTest(unittest.TestCase): def test_build_passes_args_pass(self): kunit.main(['build']) self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1) - self.linux_source_mock.build_kernel.assert_called_once_with(False, kunit.get_default_jobs(), '.kunit', None) + self.linux_source_mock.build_kernel.assert_called_once_with(kunit.get_default_jobs(), '.kunit', None) self.assertEqual(self.linux_source_mock.run_kernel.call_count, 0) def test_exec_passes_args_pass(self): @@ -664,7 +664,7 @@ class KUnitMainTest(unittest.TestCase): build_dir = '.kunit' jobs = kunit.get_default_jobs() kunit.main(['build', '--build_dir', build_dir]) - self.linux_source_mock.build_kernel.assert_called_once_with(False, jobs, build_dir, None) + self.linux_source_mock.build_kernel.assert_called_once_with(jobs, build_dir, None) def test_exec_builddir(self): build_dir = '.kunit' @@ -695,6 +695,18 @@ class KUnitMainTest(unittest.TestCase): qemu_config_path=None, extra_qemu_args=[]) + def test_config_alltests(self): + kunit.main(['config', '--kunitconfig=mykunitconfig', '--alltests']) + # Just verify that we parsed and initialized it correctly here. + self.mock_linux_init.assert_called_once_with('.kunit', + kunitconfig_paths=[kunit_kernel.ALL_TESTS_CONFIG_PATH, 'mykunitconfig'], + kconfig_add=None, + arch='um', + cross_compile=None, + qemu_config_path=None, + extra_qemu_args=[]) + + @mock.patch.object(kunit_kernel, 'LinuxSourceTree') def test_run_multiple_kunitconfig(self, mock_linux_init): mock_linux_init.return_value = self.linux_source_mock @@ -712,7 +724,7 @@ class KUnitMainTest(unittest.TestCase): kunit.main(['run', '--kconfig_add=CONFIG_KASAN=y', '--kconfig_add=CONFIG_KCSAN=y']) # Just verify that we parsed and initialized it correctly here. self.mock_linux_init.assert_called_once_with('.kunit', - kunitconfig_paths=None, + kunitconfig_paths=[], kconfig_add=['CONFIG_KASAN=y', 'CONFIG_KCSAN=y'], arch='um', cross_compile=None, @@ -723,7 +735,7 @@ class KUnitMainTest(unittest.TestCase): kunit.main(['run', '--arch=x86_64', '--qemu_args', '-m 2048']) # Just verify that we parsed and initialized it correctly here. self.mock_linux_init.assert_called_once_with('.kunit', - kunitconfig_paths=None, + kunitconfig_paths=[], kconfig_add=None, arch='x86_64', cross_compile=None, @@ -742,7 +754,7 @@ class KUnitMainTest(unittest.TestCase): self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want got = kunit._list_tests(self.linux_source_mock, - kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'suite')) + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'suite')) self.assertEqual(got, want) # Should respect the user's filter glob when listing tests. @@ -757,7 +769,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*.test*', None, 'suite')) + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, 'suite')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300), @@ -770,7 +782,7 @@ class KUnitMainTest(unittest.TestCase): # Should respect the user's filter glob when listing tests. mock_tests.assert_called_once_with(mock.ANY, - kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'test')) + kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'test')) self.linux_source_mock.run_kernel.assert_has_calls([ mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300), mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300), diff --git a/tools/testing/kunit/qemu_configs/riscv.py b/tools/testing/kunit/qemu_configs/riscv.py index 6207be146d26e7a62d2cc35d391dae49767a219c..12a1d525978a2a3e99c117b3696a9069b19432af 100644 --- a/tools/testing/kunit/qemu_configs/riscv.py +++ b/tools/testing/kunit/qemu_configs/riscv.py @@ -3,17 +3,13 @@ import os import os.path import sys -GITHUB_OPENSBI_URL = 'https://github.com/qemu/qemu/raw/master/pc-bios/opensbi-riscv64-generic-fw_dynamic.bin' -OPENSBI_FILE = os.path.basename(GITHUB_OPENSBI_URL) +OPENSBI_FILE = 'opensbi-riscv64-generic-fw_dynamic.bin' +OPENSBI_PATH = '/usr/share/qemu/' + OPENSBI_FILE -if not os.path.isfile(OPENSBI_FILE): - print('\n\nOpenSBI file is not in the current working directory.\n' - 'Would you like me to download it for you from:\n' + GITHUB_OPENSBI_URL + ' ?\n') - response = input('yes/[no]: ') - if response.strip() == 'yes': - os.system('wget ' + GITHUB_OPENSBI_URL) - else: - sys.exit() +if not os.path.isfile(OPENSBI_PATH): + print('\n\nOpenSBI bios was not found in "' + OPENSBI_PATH + '".\n' + 'Please ensure that qemu-system-riscv is installed, or edit the path in "qemu_configs/riscv.py"\n') + sys.exit() QEMU_ARCH = QemuArchParams(linux_arch='riscv', kconfig=''' @@ -29,4 +25,4 @@ CONFIG_SERIAL_EARLYCON_RISCV_SBI=y''', extra_qemu_params=[ '-machine', 'virt', '-cpu', 'rv64', - '-bios', 'opensbi-riscv64-generic-fw_dynamic.bin']) + '-bios', OPENSBI_PATH]) diff --git a/tools/testing/memblock/linux/mmzone.h b/tools/testing/memblock/linux/mmzone.h index 7c2eb5c9bb54de238457e02c17eebe660610a9fc..e65f89b12f1cd35992c5968d5f18a6f8d8dc22d7 100644 --- a/tools/testing/memblock/linux/mmzone.h +++ b/tools/testing/memblock/linux/mmzone.h @@ -22,6 +22,8 @@ enum zone_type { #define pageblock_order (MAX_ORDER - 1) #define pageblock_nr_pages BIT(pageblock_order) +#define pageblock_align(pfn) ALIGN((pfn), pageblock_nr_pages) +#define pageblock_start_pfn(pfn) ALIGN_DOWN((pfn), pageblock_nr_pages) struct zone { atomic_long_t managed_pages; diff --git a/tools/testing/memblock/scripts/Makefile.include b/tools/testing/memblock/scripts/Makefile.include index aa6d82d56a23a8b4a9c901ea4decb7719c34e65b..99828172359078d93e6ebec54afe98775baaddec 100644 --- a/tools/testing/memblock/scripts/Makefile.include +++ b/tools/testing/memblock/scripts/Makefile.include @@ -3,7 +3,7 @@ # Simulate CONFIG_NUMA=y ifeq ($(NUMA), 1) - CFLAGS += -D CONFIG_NUMA + CFLAGS += -D CONFIG_NUMA -D CONFIG_NODES_SHIFT=4 endif # Use 32 bit physical addresses. diff --git a/tools/testing/memblock/tests/alloc_api.c b/tools/testing/memblock/tests/alloc_api.c index a14f38eb8a89042dfe5c21f459aab5d2c01281a3..68f1a75cd72c440d7a8a5301fc100e451605f7e7 100644 --- a/tools/testing/memblock/tests/alloc_api.c +++ b/tools/testing/memblock/tests/alloc_api.c @@ -1,6 +1,22 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "alloc_api.h" +static int alloc_test_flags = TEST_F_NONE; + +static inline const char * const get_memblock_alloc_name(int flags) +{ + if (flags & TEST_F_RAW) + return "memblock_alloc_raw"; + return "memblock_alloc"; +} + +static inline void *run_memblock_alloc(phys_addr_t size, phys_addr_t align) +{ + if (alloc_test_flags & TEST_F_RAW) + return memblock_alloc_raw(size, align); + return memblock_alloc(size, align); +} + /* * A simple test that tries to allocate a small memory region. * Expect to allocate an aligned region near the end of the available memory. @@ -9,19 +25,19 @@ static int alloc_top_down_simple_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t size = SZ_2; phys_addr_t expected_start; + PREFIX_PUSH(); setup_memblock(); expected_start = memblock_end_of_DRAM() - SMP_CACHE_BYTES; - allocated_ptr = memblock_alloc(size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_test_flags); + ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, expected_start); @@ -58,15 +74,13 @@ static int alloc_top_down_disjoint_check(void) struct memblock_region *rgn2 = &memblock.reserved.regions[0]; struct region r1; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r2_size = SZ_16; /* Use custom alignment */ phys_addr_t alignment = SMP_CACHE_BYTES * 2; phys_addr_t total_size; phys_addr_t expected_start; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SZ_2; @@ -77,9 +91,11 @@ static int alloc_top_down_disjoint_check(void) memblock_reserve(r1.base, r1.size); - allocated_ptr = memblock_alloc(r2_size, alignment); + allocated_ptr = run_memblock_alloc(r2_size, alignment); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r2_size, alloc_test_flags); + ASSERT_EQ(rgn1->size, r1.size); ASSERT_EQ(rgn1->base, r1.base); @@ -108,9 +124,6 @@ static int alloc_top_down_before_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - /* * The first region ends at the aligned address to test region merging */ @@ -118,13 +131,16 @@ static int alloc_top_down_before_check(void) phys_addr_t r2_size = SZ_512; phys_addr_t total_size = r1_size + r2_size; + PREFIX_PUSH(); setup_memblock(); memblock_reserve(memblock_end_of_DRAM() - total_size, r1_size); - allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r2_size, alloc_test_flags); + ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - total_size); @@ -152,12 +168,10 @@ static int alloc_top_down_after_check(void) struct memblock_region *rgn = &memblock.reserved.regions[0]; struct region r1; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r2_size = SZ_512; phys_addr_t total_size; + PREFIX_PUSH(); setup_memblock(); /* @@ -170,9 +184,11 @@ static int alloc_top_down_after_check(void) memblock_reserve(r1.base, r1.size); - allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r2_size, alloc_test_flags); + ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, r1.base - r2_size); @@ -201,12 +217,10 @@ static int alloc_top_down_second_fit_check(void) struct memblock_region *rgn = &memblock.reserved.regions[0]; struct region r1, r2; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_1K; phys_addr_t total_size; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SZ_512; @@ -220,9 +234,11 @@ static int alloc_top_down_second_fit_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r3_size, alloc_test_flags); + ASSERT_EQ(rgn->size, r2.size + r3_size); ASSERT_EQ(rgn->base, r2.base - r3_size); @@ -250,9 +266,6 @@ static int alloc_in_between_generic_check(void) struct memblock_region *rgn = &memblock.reserved.regions[0]; struct region r1, r2; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t r3_size = SZ_64; /* @@ -261,6 +274,7 @@ static int alloc_in_between_generic_check(void) phys_addr_t rgn_size = (MEM_SIZE - (2 * gap_size + r3_size)) / 2; phys_addr_t total_size; + PREFIX_PUSH(); setup_memblock(); r1.size = rgn_size; @@ -274,9 +288,11 @@ static int alloc_in_between_generic_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r3_size, alloc_test_flags); + ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, r1.base - r2.size - r3_size); @@ -304,13 +320,11 @@ static int alloc_in_between_generic_check(void) static int alloc_small_gaps_generic_check(void) { void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t region_size = SZ_1K; phys_addr_t gap_size = SZ_256; phys_addr_t region_end; + PREFIX_PUSH(); setup_memblock(); region_end = memblock_start_of_DRAM(); @@ -320,7 +334,7 @@ static int alloc_small_gaps_generic_check(void) region_end += gap_size + region_size; } - allocated_ptr = memblock_alloc(region_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(region_size, SMP_CACHE_BYTES); ASSERT_EQ(allocated_ptr, NULL); @@ -338,13 +352,12 @@ static int alloc_all_reserved_generic_check(void) void *allocated_ptr = NULL; PREFIX_PUSH(); - setup_memblock(); /* Simulate full memory */ memblock_reserve(memblock_start_of_DRAM(), MEM_SIZE); - allocated_ptr = memblock_alloc(SZ_256, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(SZ_256, SMP_CACHE_BYTES); ASSERT_EQ(allocated_ptr, NULL); @@ -369,18 +382,16 @@ static int alloc_all_reserved_generic_check(void) static int alloc_no_space_generic_check(void) { void *allocated_ptr = NULL; + phys_addr_t available_size = SZ_256; + phys_addr_t reserved_size = MEM_SIZE - available_size; PREFIX_PUSH(); - setup_memblock(); - phys_addr_t available_size = SZ_256; - phys_addr_t reserved_size = MEM_SIZE - available_size; - /* Simulate almost-full memory */ memblock_reserve(memblock_start_of_DRAM(), reserved_size); - allocated_ptr = memblock_alloc(SZ_1K, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(SZ_1K, SMP_CACHE_BYTES); ASSERT_EQ(allocated_ptr, NULL); @@ -404,20 +415,20 @@ static int alloc_limited_space_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t available_size = SZ_256; phys_addr_t reserved_size = MEM_SIZE - available_size; + PREFIX_PUSH(); setup_memblock(); /* Simulate almost-full memory */ memblock_reserve(memblock_start_of_DRAM(), reserved_size); - allocated_ptr = memblock_alloc(available_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(available_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, available_size, alloc_test_flags); + ASSERT_EQ(rgn->size, MEM_SIZE); ASSERT_EQ(rgn->base, memblock_start_of_DRAM()); @@ -443,7 +454,40 @@ static int alloc_no_memory_generic_check(void) reset_memblock_regions(); - allocated_ptr = memblock_alloc(SZ_1K, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(SZ_1K, SMP_CACHE_BYTES); + + ASSERT_EQ(allocated_ptr, NULL); + ASSERT_EQ(rgn->size, 0); + ASSERT_EQ(rgn->base, 0); + ASSERT_EQ(memblock.reserved.total_size, 0); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a region that is larger than the total size of + * available memory (memblock.memory): + * + * +-----------------------------------+ + * | new | + * +-----------------------------------+ + * | | + * | | + * +---------------------------------+ + * + * Expect no allocation to happen. + */ +static int alloc_too_large_generic_check(void) +{ + struct memblock_region *rgn = &memblock.reserved.regions[0]; + void *allocated_ptr = NULL; + + PREFIX_PUSH(); + setup_memblock(); + + allocated_ptr = run_memblock_alloc(MEM_SIZE + SZ_2, SMP_CACHE_BYTES); ASSERT_EQ(allocated_ptr, NULL); ASSERT_EQ(rgn->size, 0); @@ -466,12 +510,13 @@ static int alloc_bottom_up_simple_check(void) void *allocated_ptr = NULL; PREFIX_PUSH(); - setup_memblock(); - allocated_ptr = memblock_alloc(SZ_2, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(SZ_2, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, SZ_2, alloc_test_flags); + ASSERT_EQ(rgn->size, SZ_2); ASSERT_EQ(rgn->base, memblock_start_of_DRAM()); @@ -506,15 +551,13 @@ static int alloc_bottom_up_disjoint_check(void) struct memblock_region *rgn2 = &memblock.reserved.regions[1]; struct region r1; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r2_size = SZ_16; /* Use custom alignment */ phys_addr_t alignment = SMP_CACHE_BYTES * 2; phys_addr_t total_size; phys_addr_t expected_start; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_start_of_DRAM() + SZ_2; @@ -525,9 +568,10 @@ static int alloc_bottom_up_disjoint_check(void) memblock_reserve(r1.base, r1.size); - allocated_ptr = memblock_alloc(r2_size, alignment); + allocated_ptr = run_memblock_alloc(r2_size, alignment); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r2_size, alloc_test_flags); ASSERT_EQ(rgn1->size, r1.size); ASSERT_EQ(rgn1->base, r1.base); @@ -557,20 +601,20 @@ static int alloc_bottom_up_before_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_512; phys_addr_t r2_size = SZ_128; phys_addr_t total_size = r1_size + r2_size; + PREFIX_PUSH(); setup_memblock(); memblock_reserve(memblock_start_of_DRAM() + r1_size, r2_size); - allocated_ptr = memblock_alloc(r1_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r1_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r1_size, alloc_test_flags); + ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, memblock_start_of_DRAM()); @@ -597,12 +641,10 @@ static int alloc_bottom_up_after_check(void) struct memblock_region *rgn = &memblock.reserved.regions[0]; struct region r1; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r2_size = SZ_512; phys_addr_t total_size; + PREFIX_PUSH(); setup_memblock(); /* @@ -615,9 +657,11 @@ static int alloc_bottom_up_after_check(void) memblock_reserve(r1.base, r1.size); - allocated_ptr = memblock_alloc(r2_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r2_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r2_size, alloc_test_flags); + ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, r1.base); @@ -647,12 +691,10 @@ static int alloc_bottom_up_second_fit_check(void) struct memblock_region *rgn = &memblock.reserved.regions[1]; struct region r1, r2; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_1K; phys_addr_t total_size; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_start_of_DRAM(); @@ -666,9 +708,11 @@ static int alloc_bottom_up_second_fit_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc(r3_size, SMP_CACHE_BYTES); + allocated_ptr = run_memblock_alloc(r3_size, SMP_CACHE_BYTES); ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, r3_size, alloc_test_flags); + ASSERT_EQ(rgn->size, r2.size + r3_size); ASSERT_EQ(rgn->base, r2.base); @@ -728,10 +772,8 @@ static int alloc_after_check(void) static int alloc_in_between_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_in_between_generic_check(); - memblock_set_bottom_up(true); - alloc_in_between_generic_check(); + run_top_down(alloc_in_between_generic_check); + run_bottom_up(alloc_in_between_generic_check); return 0; } @@ -750,10 +792,8 @@ static int alloc_second_fit_check(void) static int alloc_small_gaps_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_small_gaps_generic_check(); - memblock_set_bottom_up(true); - alloc_small_gaps_generic_check(); + run_top_down(alloc_small_gaps_generic_check); + run_bottom_up(alloc_small_gaps_generic_check); return 0; } @@ -761,10 +801,8 @@ static int alloc_small_gaps_check(void) static int alloc_all_reserved_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_all_reserved_generic_check(); - memblock_set_bottom_up(true); - alloc_all_reserved_generic_check(); + run_top_down(alloc_all_reserved_generic_check); + run_bottom_up(alloc_all_reserved_generic_check); return 0; } @@ -772,10 +810,8 @@ static int alloc_all_reserved_check(void) static int alloc_no_space_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_no_space_generic_check(); - memblock_set_bottom_up(true); - alloc_no_space_generic_check(); + run_top_down(alloc_no_space_generic_check); + run_bottom_up(alloc_no_space_generic_check); return 0; } @@ -783,10 +819,8 @@ static int alloc_no_space_check(void) static int alloc_limited_space_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_limited_space_generic_check(); - memblock_set_bottom_up(true); - alloc_limited_space_generic_check(); + run_top_down(alloc_limited_space_generic_check); + run_bottom_up(alloc_limited_space_generic_check); return 0; } @@ -794,21 +828,29 @@ static int alloc_limited_space_check(void) static int alloc_no_memory_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_no_memory_generic_check(); - memblock_set_bottom_up(true); - alloc_no_memory_generic_check(); + run_top_down(alloc_no_memory_generic_check); + run_bottom_up(alloc_no_memory_generic_check); return 0; } -int memblock_alloc_checks(void) +static int alloc_too_large_check(void) { - const char *func_testing = "memblock_alloc"; + test_print("\tRunning %s...\n", __func__); + run_top_down(alloc_too_large_generic_check); + run_bottom_up(alloc_too_large_generic_check); + return 0; +} + +static int memblock_alloc_checks_internal(int flags) +{ + const char *func = get_memblock_alloc_name(flags); + + alloc_test_flags = flags; prefix_reset(); - prefix_push(func_testing); - test_print("Running %s tests...\n", func_testing); + prefix_push(func); + test_print("Running %s tests...\n", func); reset_memblock_attributes(); dummy_physical_memory_init(); @@ -824,6 +866,7 @@ int memblock_alloc_checks(void) alloc_no_space_check(); alloc_limited_space_check(); alloc_no_memory_check(); + alloc_too_large_check(); dummy_physical_memory_cleanup(); @@ -831,3 +874,11 @@ int memblock_alloc_checks(void) return 0; } + +int memblock_alloc_checks(void) +{ + memblock_alloc_checks_internal(TEST_F_NONE); + memblock_alloc_checks_internal(TEST_F_RAW); + + return 0; +} diff --git a/tools/testing/memblock/tests/alloc_helpers_api.c b/tools/testing/memblock/tests/alloc_helpers_api.c index 1069b4bdd5fdd398d4a8d5702bfea2d019a07720..3ef9486da8a09350422634ceaafeb8b687c7757e 100644 --- a/tools/testing/memblock/tests/alloc_helpers_api.c +++ b/tools/testing/memblock/tests/alloc_helpers_api.c @@ -19,22 +19,18 @@ static int alloc_from_simple_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_16; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_end_of_DRAM() - SMP_CACHE_BYTES; allocated_ptr = memblock_alloc_from(size, SMP_CACHE_BYTES, min_addr); - b = (char *)allocated_ptr; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + ASSERT_MEM_EQ(allocated_ptr, 0, size); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, min_addr); @@ -66,23 +62,19 @@ static int alloc_from_misaligned_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_32; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); /* A misaligned address */ min_addr = memblock_end_of_DRAM() - (SMP_CACHE_BYTES * 2 - 1); allocated_ptr = memblock_alloc_from(size, SMP_CACHE_BYTES, min_addr); - b = (char *)allocated_ptr; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + ASSERT_MEM_EQ(allocated_ptr, 0, size); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - SMP_CACHE_BYTES); @@ -117,12 +109,10 @@ static int alloc_from_top_down_high_addr_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t size = SZ_32; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); /* The address is too close to the end of the memory */ @@ -162,14 +152,12 @@ static int alloc_from_top_down_no_space_above_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_64; phys_addr_t r2_size = SZ_2; phys_addr_t total_size = r1_size + r2_size; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -201,13 +189,11 @@ static int alloc_from_top_down_min_addr_cap_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_64; phys_addr_t min_addr; phys_addr_t start_addr; + PREFIX_PUSH(); setup_memblock(); start_addr = (phys_addr_t)memblock_start_of_DRAM(); @@ -249,12 +235,10 @@ static int alloc_from_bottom_up_high_addr_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t size = SZ_32; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); /* The address is too close to the end of the memory */ @@ -293,13 +277,11 @@ static int alloc_from_bottom_up_no_space_above_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_64; phys_addr_t min_addr; phys_addr_t r2_size; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SZ_128; @@ -331,13 +313,11 @@ static int alloc_from_bottom_up_min_addr_cap_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_64; phys_addr_t min_addr; phys_addr_t start_addr; + PREFIX_PUSH(); setup_memblock(); start_addr = (phys_addr_t)memblock_start_of_DRAM(); @@ -361,10 +341,8 @@ static int alloc_from_bottom_up_min_addr_cap_check(void) static int alloc_from_simple_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_from_simple_generic_check(); - memblock_set_bottom_up(true); - alloc_from_simple_generic_check(); + run_top_down(alloc_from_simple_generic_check); + run_bottom_up(alloc_from_simple_generic_check); return 0; } @@ -372,10 +350,8 @@ static int alloc_from_simple_check(void) static int alloc_from_misaligned_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_from_misaligned_generic_check(); - memblock_set_bottom_up(true); - alloc_from_misaligned_generic_check(); + run_top_down(alloc_from_misaligned_generic_check); + run_bottom_up(alloc_from_misaligned_generic_check); return 0; } diff --git a/tools/testing/memblock/tests/alloc_nid_api.c b/tools/testing/memblock/tests/alloc_nid_api.c index 255fd514e9f5de712f3a5a5abd529171422465da..2c2d60f4e3e3c3fcf10ae6166d9e2a2084f8db6b 100644 --- a/tools/testing/memblock/tests/alloc_nid_api.c +++ b/tools/testing/memblock/tests/alloc_nid_api.c @@ -1,6 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "alloc_nid_api.h" +static int alloc_nid_test_flags = TEST_F_NONE; + +/* + * contains the fraction of MEM_SIZE contained in each node in basis point + * units (one hundredth of 1% or 1/10000) + */ +static const unsigned int node_fractions[] = { + 2500, /* 1/4 */ + 625, /* 1/16 */ + 1250, /* 1/8 */ + 1250, /* 1/8 */ + 625, /* 1/16 */ + 625, /* 1/16 */ + 2500, /* 1/4 */ + 625, /* 1/16 */ +}; + +static inline const char * const get_memblock_alloc_try_nid_name(int flags) +{ + if (flags & TEST_F_RAW) + return "memblock_alloc_try_nid_raw"; + return "memblock_alloc_try_nid"; +} + +static inline void *run_memblock_alloc_try_nid(phys_addr_t size, + phys_addr_t align, + phys_addr_t min_addr, + phys_addr_t max_addr, int nid) +{ + if (alloc_nid_test_flags & TEST_F_RAW) + return memblock_alloc_try_nid_raw(size, align, min_addr, + max_addr, nid); + return memblock_alloc_try_nid(size, align, min_addr, max_addr, nid); +} + /* * A simple test that tries to allocate a memory region within min_addr and * max_addr range: @@ -13,33 +48,30 @@ * | | * min_addr max_addr * - * Expect to allocate a cleared region that ends at max_addr. + * Expect to allocate a region that ends at max_addr. */ static int alloc_try_nid_top_down_simple_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_128; phys_addr_t min_addr; phys_addr_t max_addr; phys_addr_t rgn_end; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2; max_addr = min_addr + SZ_512; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); rgn_end = rgn->base + rgn->size; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, max_addr - size); @@ -68,34 +100,31 @@ static int alloc_try_nid_top_down_simple_check(void) * Aligned address * boundary * - * Expect to allocate a cleared, aligned region that ends before max_addr. + * Expect to allocate an aligned region that ends before max_addr. */ static int alloc_try_nid_top_down_end_misaligned_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_128; phys_addr_t misalign = SZ_2; phys_addr_t min_addr; phys_addr_t max_addr; phys_addr_t rgn_end; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2; max_addr = min_addr + SZ_512 + misalign; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); rgn_end = rgn->base + rgn->size; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, max_addr - size - misalign); @@ -121,34 +150,31 @@ static int alloc_try_nid_top_down_end_misaligned_check(void) * | | * min_addr max_addr * - * Expect to allocate a cleared region that starts at min_addr and ends at + * Expect to allocate a region that starts at min_addr and ends at * max_addr, given that min_addr is aligned. */ static int alloc_try_nid_exact_address_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_1K; phys_addr_t min_addr; phys_addr_t max_addr; phys_addr_t rgn_end; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES; max_addr = min_addr + size; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); rgn_end = rgn->base + rgn->size; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, min_addr); @@ -176,32 +202,29 @@ static int alloc_try_nid_exact_address_generic_check(void) * address | * boundary min_add * - * Expect to drop the lower limit and allocate a cleared memory region which + * Expect to drop the lower limit and allocate a memory region which * ends at max_addr (if the address is aligned). */ static int alloc_try_nid_top_down_narrow_range_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_256; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SZ_512; max_addr = min_addr + SMP_CACHE_BYTES; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, max_addr - size); @@ -237,20 +260,19 @@ static int alloc_try_nid_top_down_narrow_range_check(void) static int alloc_try_nid_low_max_generic_check(void) { void *allocated_ptr = NULL; - - PREFIX_PUSH(); - phys_addr_t size = SZ_1K; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM(); max_addr = min_addr + SMP_CACHE_BYTES; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_EQ(allocated_ptr, NULL); @@ -277,10 +299,6 @@ static int alloc_try_nid_min_reserved_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_128; phys_addr_t r2_size = SZ_64; phys_addr_t total_size = r1_size + r2_size; @@ -288,6 +306,7 @@ static int alloc_try_nid_min_reserved_generic_check(void) phys_addr_t max_addr; phys_addr_t reserved_base; + PREFIX_PUSH(); setup_memblock(); max_addr = memblock_end_of_DRAM(); @@ -296,12 +315,12 @@ static int alloc_try_nid_min_reserved_generic_check(void) memblock_reserve(reserved_base, r1_size); - allocated_ptr = memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r2_size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, reserved_base); @@ -332,16 +351,13 @@ static int alloc_try_nid_max_reserved_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t r1_size = SZ_64; phys_addr_t r2_size = SZ_128; phys_addr_t total_size = r1_size + r2_size; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); max_addr = memblock_end_of_DRAM() - r1_size; @@ -349,12 +365,12 @@ static int alloc_try_nid_max_reserved_generic_check(void) memblock_reserve(max_addr, r1_size); - allocated_ptr = memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r2_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r2_size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, min_addr); @@ -389,17 +405,14 @@ static int alloc_try_nid_top_down_reserved_with_space_check(void) struct memblock_region *rgn1 = &memblock.reserved.regions[1]; struct memblock_region *rgn2 = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_64; phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t total_size; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -415,12 +428,12 @@ static int alloc_try_nid_top_down_reserved_with_space_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags); ASSERT_EQ(rgn1->size, r1.size + r3_size); ASSERT_EQ(rgn1->base, max_addr - r3_size); @@ -456,16 +469,13 @@ static int alloc_try_nid_reserved_full_merge_generic_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_64; phys_addr_t total_size; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -481,12 +491,12 @@ static int alloc_try_nid_reserved_full_merge_generic_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, total_size); ASSERT_EQ(rgn->base, r2.base); @@ -522,17 +532,14 @@ static int alloc_try_nid_top_down_reserved_no_space_check(void) struct memblock_region *rgn1 = &memblock.reserved.regions[1]; struct memblock_region *rgn2 = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_256; phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t total_size; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -548,12 +555,12 @@ static int alloc_try_nid_top_down_reserved_no_space_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags); ASSERT_EQ(rgn1->size, r1.size); ASSERT_EQ(rgn1->base, r1.base); @@ -593,14 +600,12 @@ static int alloc_try_nid_reserved_all_generic_check(void) { void *allocated_ptr = NULL; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_256; phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES; @@ -615,8 +620,9 @@ static int alloc_try_nid_reserved_all_generic_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_EQ(allocated_ptr, NULL); @@ -628,31 +634,28 @@ static int alloc_try_nid_reserved_all_generic_check(void) /* * A test that tries to allocate a memory region, where max_addr is * bigger than the end address of the available memory. Expect to allocate - * a cleared region that ends before the end of the memory. + * a region that ends before the end of the memory. */ static int alloc_try_nid_top_down_cap_max_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_256; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_end_of_DRAM() - SZ_1K; max_addr = memblock_end_of_DRAM() + SZ_256; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - size); @@ -668,31 +671,28 @@ static int alloc_try_nid_top_down_cap_max_check(void) /* * A test that tries to allocate a memory region, where min_addr is * smaller than the start address of the available memory. Expect to allocate - * a cleared region that ends before the end of the memory. + * a region that ends before the end of the memory. */ static int alloc_try_nid_top_down_cap_min_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_1K; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() - SZ_256; max_addr = memblock_end_of_DRAM(); - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, memblock_end_of_DRAM() - size); @@ -717,34 +717,30 @@ static int alloc_try_nid_top_down_cap_min_check(void) * | | * min_addr max_addr * - * Expect to allocate a cleared region that ends before max_addr. + * Expect to allocate a region that ends before max_addr. */ static int alloc_try_nid_bottom_up_simple_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_128; phys_addr_t min_addr; phys_addr_t max_addr; phys_addr_t rgn_end; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SMP_CACHE_BYTES * 2; max_addr = min_addr + SZ_512; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); rgn_end = rgn->base + rgn->size; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, min_addr); @@ -773,35 +769,31 @@ static int alloc_try_nid_bottom_up_simple_check(void) * Aligned address * boundary * - * Expect to allocate a cleared, aligned region that ends before max_addr. + * Expect to allocate an aligned region that ends before max_addr. */ static int alloc_try_nid_bottom_up_start_misaligned_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_128; phys_addr_t misalign = SZ_2; phys_addr_t min_addr; phys_addr_t max_addr; phys_addr_t rgn_end; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + misalign; max_addr = min_addr + SZ_512; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); rgn_end = rgn->base + rgn->size; ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, min_addr + (SMP_CACHE_BYTES - misalign)); @@ -829,33 +821,29 @@ static int alloc_try_nid_bottom_up_start_misaligned_check(void) * | * min_add * - * Expect to drop the lower limit and allocate a cleared memory region which + * Expect to drop the lower limit and allocate a memory region which * starts at the beginning of the available memory. */ static int alloc_try_nid_bottom_up_narrow_range_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_256; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SZ_512; max_addr = min_addr + SMP_CACHE_BYTES; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, memblock_start_of_DRAM()); @@ -890,17 +878,14 @@ static int alloc_try_nid_bottom_up_reserved_with_space_check(void) struct memblock_region *rgn1 = &memblock.reserved.regions[1]; struct memblock_region *rgn2 = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_64; phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t total_size; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -916,13 +901,12 @@ static int alloc_try_nid_bottom_up_reserved_with_space_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags); ASSERT_EQ(rgn1->size, r1.size); ASSERT_EQ(rgn1->base, max_addr); @@ -964,17 +948,14 @@ static int alloc_try_nid_bottom_up_reserved_no_space_check(void) struct memblock_region *rgn2 = &memblock.reserved.regions[1]; struct memblock_region *rgn3 = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; struct region r1, r2; - - PREFIX_PUSH(); - phys_addr_t r3_size = SZ_256; phys_addr_t gap_size = SMP_CACHE_BYTES; phys_addr_t total_size; phys_addr_t max_addr; phys_addr_t min_addr; + PREFIX_PUSH(); setup_memblock(); r1.base = memblock_end_of_DRAM() - SMP_CACHE_BYTES * 2; @@ -990,13 +971,12 @@ static int alloc_try_nid_bottom_up_reserved_no_space_check(void) memblock_reserve(r1.base, r1.size); memblock_reserve(r2.base, r2.size); - allocated_ptr = memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(r3_size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, r3_size, alloc_nid_test_flags); ASSERT_EQ(rgn3->size, r3_size); ASSERT_EQ(rgn3->base, memblock_start_of_DRAM()); @@ -1018,32 +998,28 @@ static int alloc_try_nid_bottom_up_reserved_no_space_check(void) /* * A test that tries to allocate a memory region, where max_addr is * bigger than the end address of the available memory. Expect to allocate - * a cleared region that starts at the min_addr + * a region that starts at the min_addr. */ static int alloc_try_nid_bottom_up_cap_max_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_256; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM() + SZ_1K; max_addr = memblock_end_of_DRAM() + SZ_256; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, min_addr); @@ -1059,32 +1035,28 @@ static int alloc_try_nid_bottom_up_cap_max_check(void) /* * A test that tries to allocate a memory region, where min_addr is * smaller than the start address of the available memory. Expect to allocate - * a cleared region at the beginning of the available memory. + * a region at the beginning of the available memory. */ static int alloc_try_nid_bottom_up_cap_min_check(void) { struct memblock_region *rgn = &memblock.reserved.regions[0]; void *allocated_ptr = NULL; - char *b; - - PREFIX_PUSH(); - phys_addr_t size = SZ_1K; phys_addr_t min_addr; phys_addr_t max_addr; + PREFIX_PUSH(); setup_memblock(); min_addr = memblock_start_of_DRAM(); max_addr = memblock_end_of_DRAM() - SZ_256; - allocated_ptr = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, - min_addr, max_addr, - NUMA_NO_NODE); - b = (char *)allocated_ptr; + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); ASSERT_NE(allocated_ptr, NULL); - ASSERT_EQ(*b, 0); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); ASSERT_EQ(rgn->size, size); ASSERT_EQ(rgn->base, memblock_start_of_DRAM()); @@ -1097,7 +1069,7 @@ static int alloc_try_nid_bottom_up_cap_min_check(void) return 0; } -/* Test case wrappers */ +/* Test case wrappers for range tests */ static int alloc_try_nid_simple_check(void) { test_print("\tRunning %s...\n", __func__); @@ -1178,10 +1150,8 @@ static int alloc_try_nid_cap_min_check(void) static int alloc_try_nid_min_reserved_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_min_reserved_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_min_reserved_generic_check(); + run_top_down(alloc_try_nid_min_reserved_generic_check); + run_bottom_up(alloc_try_nid_min_reserved_generic_check); return 0; } @@ -1189,10 +1159,8 @@ static int alloc_try_nid_min_reserved_check(void) static int alloc_try_nid_max_reserved_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_max_reserved_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_max_reserved_generic_check(); + run_top_down(alloc_try_nid_max_reserved_generic_check); + run_bottom_up(alloc_try_nid_max_reserved_generic_check); return 0; } @@ -1200,10 +1168,8 @@ static int alloc_try_nid_max_reserved_check(void) static int alloc_try_nid_exact_address_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_exact_address_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_exact_address_generic_check(); + run_top_down(alloc_try_nid_exact_address_generic_check); + run_bottom_up(alloc_try_nid_exact_address_generic_check); return 0; } @@ -1211,10 +1177,8 @@ static int alloc_try_nid_exact_address_check(void) static int alloc_try_nid_reserved_full_merge_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_reserved_full_merge_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_reserved_full_merge_generic_check(); + run_top_down(alloc_try_nid_reserved_full_merge_generic_check); + run_bottom_up(alloc_try_nid_reserved_full_merge_generic_check); return 0; } @@ -1222,10 +1186,8 @@ static int alloc_try_nid_reserved_full_merge_check(void) static int alloc_try_nid_reserved_all_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_reserved_all_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_reserved_all_generic_check(); + run_top_down(alloc_try_nid_reserved_all_generic_check); + run_bottom_up(alloc_try_nid_reserved_all_generic_check); return 0; } @@ -1233,24 +1195,16 @@ static int alloc_try_nid_reserved_all_check(void) static int alloc_try_nid_low_max_check(void) { test_print("\tRunning %s...\n", __func__); - memblock_set_bottom_up(false); - alloc_try_nid_low_max_generic_check(); - memblock_set_bottom_up(true); - alloc_try_nid_low_max_generic_check(); + run_top_down(alloc_try_nid_low_max_generic_check); + run_bottom_up(alloc_try_nid_low_max_generic_check); return 0; } -int memblock_alloc_nid_checks(void) +static int memblock_alloc_nid_range_checks(void) { - const char *func_testing = "memblock_alloc_try_nid"; - - prefix_reset(); - prefix_push(func_testing); - test_print("Running %s tests...\n", func_testing); - - reset_memblock_attributes(); - dummy_physical_memory_init(); + test_print("Running %s range tests...\n", + get_memblock_alloc_try_nid_name(alloc_nid_test_flags)); alloc_try_nid_simple_check(); alloc_try_nid_misaligned_check(); @@ -1267,9 +1221,1453 @@ int memblock_alloc_nid_checks(void) alloc_try_nid_reserved_all_check(); alloc_try_nid_low_max_check(); - dummy_physical_memory_cleanup(); + return 0; +} - prefix_pop(); +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * has enough memory to allocate a region of the requested size. + * Expect to allocate an aligned region at the end of the requested node. + */ +static int alloc_try_nid_top_down_numa_simple_check(void) +{ + int nid_req = 3; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_4, req_node->size); + size = req_node->size / SZ_4; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(req_node) - size); + ASSERT_LE(req_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * does not have enough memory to allocate a region of the requested size: + * + * | +-----+ +------------------+ | + * | | req | | expected | | + * +---+-----+----------+------------------+-----+ + * + * | +---------+ | + * | | rgn | | + * +-----------------------------+---------+-----+ + * + * Expect to allocate an aligned region at the end of the last node that has + * enough memory (in this case, nid = 6) after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_top_down_numa_small_node_check(void) +{ + int nid_req = 1; + int nid_exp = 6; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = SZ_2 * req_node->size; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(exp_node) - size); + ASSERT_LE(exp_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is fully reserved: + * + * | +---------+ +------------------+ | + * | |requested| | expected | | + * +--------------+---------+------------+------------------+-----+ + * + * | +---------+ +---------+ | + * | | reserved| | new | | + * +--------------+---------+---------------------+---------+-----+ + * + * Expect to allocate an aligned region at the end of the last node that is + * large enough and has enough unreserved memory (in this case, nid = 6) after + * falling back to NUMA_NO_NODE. The region count and total size get updated. + */ +static int alloc_try_nid_top_down_numa_node_reserved_check(void) +{ + int nid_req = 2; + int nid_exp = 6; + struct memblock_region *new_rgn = &memblock.reserved.regions[1]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = req_node->size; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + memblock_reserve(req_node->base, req_node->size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(exp_node) - size); + ASSERT_LE(exp_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, size + req_node->size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is partially reserved but has enough memory for the allocated region: + * + * | +---------------------------------------+ | + * | | requested | | + * +-----------+---------------------------------------+----------+ + * + * | +------------------+ +-----+ | + * | | reserved | | new | | + * +-----------+------------------+--------------+-----+----------+ + * + * Expect to allocate an aligned region at the end of the requested node. The + * region count and total size get updated. + */ +static int alloc_try_nid_top_down_numa_part_reserved_check(void) +{ + int nid_req = 4; + struct memblock_region *new_rgn = &memblock.reserved.regions[1]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + struct region r1; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_8, req_node->size); + r1.base = req_node->base; + r1.size = req_node->size / SZ_2; + size = r1.size / SZ_4; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + memblock_reserve(r1.base, r1.size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(req_node) - size); + ASSERT_LE(req_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, size + r1.size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is partially reserved and does not have enough contiguous memory for the + * allocated region: + * + * | +-----------------------+ +----------------------| + * | | requested | | expected | + * +-----------+-----------------------+---------+----------------------+ + * + * | +----------+ +-----------| + * | | reserved | | new | + * +-----------------+----------+---------------------------+-----------+ + * + * Expect to allocate an aligned region at the end of the last node that is + * large enough and has enough unreserved memory (in this case, + * nid = NUMA_NODES - 1) after falling back to NUMA_NO_NODE. The region count + * and total size get updated. + */ +static int alloc_try_nid_top_down_numa_part_reserved_fallback_check(void) +{ + int nid_req = 4; + int nid_exp = NUMA_NODES - 1; + struct memblock_region *new_rgn = &memblock.reserved.regions[1]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + struct region r1; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_4, req_node->size); + size = req_node->size / SZ_2; + r1.base = req_node->base + (size / SZ_2); + r1.size = size; + + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + memblock_reserve(r1.base, r1.size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(exp_node) - size); + ASSERT_LE(exp_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, size + r1.size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the first + * node is the requested node: + * + * min_addr + * | max_addr + * | | + * v v + * | +-----------------------+-----------+ | + * | | requested | node3 | | + * +-----------+-----------------------+-----------+--------------+ + * + + + * | +-----------+ | + * | | rgn | | + * +-----------------------+-----------+--------------------------+ + * + * Expect to drop the lower limit and allocate a memory region that ends at + * the end of the requested node. + */ +static int alloc_try_nid_top_down_numa_split_range_low_check(void) +{ + int nid_req = 2; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_512; + phys_addr_t min_addr; + phys_addr_t max_addr; + phys_addr_t req_node_end; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + req_node_end = region_end(req_node); + min_addr = req_node_end - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, req_node_end - size); + ASSERT_LE(req_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the second + * node is the requested node: + * + * min_addr + * | max_addr + * | | + * v v + * | +--------------------------+---------+ | + * | | expected |requested| | + * +------+--------------------------+---------+----------------+ + * + + + * | +---------+ | + * | | rgn | | + * +-----------------------+---------+--------------------------+ + * + * Expect to drop the lower limit and allocate a memory region that + * ends at the end of the first node that overlaps with the range. + */ +static int alloc_try_nid_top_down_numa_split_range_high_check(void) +{ + int nid_req = 3; + int nid_exp = nid_req - 1; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_512; + phys_addr_t min_addr; + phys_addr_t max_addr; + phys_addr_t exp_node_end; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + exp_node_end = region_end(exp_node); + min_addr = exp_node_end - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, exp_node_end - size); + ASSERT_LE(exp_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the requested + * node ends before min_addr: + * + * min_addr + * | max_addr + * | | + * v v + * | +---------------+ +-------------+---------+ | + * | | requested | | node1 | node2 | | + * +----+---------------+--------+-------------+---------+----------+ + * + + + * | +---------+ | + * | | rgn | | + * +----------+---------+-------------------------------------------+ + * + * Expect to drop the lower limit and allocate a memory region that ends at + * the end of the requested node. + */ +static int alloc_try_nid_top_down_numa_no_overlap_split_check(void) +{ + int nid_req = 2; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *node2 = &memblock.memory.regions[6]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = SZ_512; + min_addr = node2->base - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, region_end(req_node) - size); + ASSERT_LE(req_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_add range when + * the requested node and the range do not overlap, and requested node ends + * before min_addr. The range overlaps with multiple nodes along node + * boundaries: + * + * min_addr + * | max_addr + * | | + * v v + * |-----------+ +----------+----...----+----------+ | + * | requested | | min node | ... | max node | | + * +-----------+-----------+----------+----...----+----------+------+ + * + + + * | +-----+ | + * | | rgn | | + * +---------------------------------------------------+-----+------+ + * + * Expect to allocate a memory region at the end of the final node in + * the range after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_top_down_numa_no_overlap_low_check(void) +{ + int nid_req = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *min_node = &memblock.memory.regions[2]; + struct memblock_region *max_node = &memblock.memory.regions[5]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_64; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + min_addr = min_node->base; + max_addr = region_end(max_node); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, max_addr - size); + ASSERT_LE(max_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_add range when + * the requested node and the range do not overlap, and requested node starts + * after max_addr. The range overlaps with multiple nodes along node + * boundaries: + * + * min_addr + * | max_addr + * | | + * v v + * | +----------+----...----+----------+ +-----------+ | + * | | min node | ... | max node | | requested | | + * +-----+----------+----...----+----------+--------+-----------+---+ + * + + + * | +-----+ | + * | | rgn | | + * +---------------------------------+-----+------------------------+ + * + * Expect to allocate a memory region at the end of the final node in + * the range after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_top_down_numa_no_overlap_high_check(void) +{ + int nid_req = 7; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *min_node = &memblock.memory.regions[2]; + struct memblock_region *max_node = &memblock.memory.regions[5]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_64; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + min_addr = min_node->base; + max_addr = region_end(max_node); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, max_addr - size); + ASSERT_LE(max_node->base, new_rgn->base); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * has enough memory to allocate a region of the requested size. + * Expect to allocate an aligned region at the beginning of the requested node. + */ +static int alloc_try_nid_bottom_up_numa_simple_check(void) +{ + int nid_req = 3; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_4, req_node->size); + size = req_node->size / SZ_4; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, req_node->base); + ASSERT_LE(region_end(new_rgn), region_end(req_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * does not have enough memory to allocate a region of the requested size: + * + * |----------------------+-----+ | + * | expected | req | | + * +----------------------+-----+----------------+ + * + * |---------+ | + * | rgn | | + * +---------+-----------------------------------+ + * + * Expect to allocate an aligned region at the beginning of the first node that + * has enough memory (in this case, nid = 0) after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_bottom_up_numa_small_node_check(void) +{ + int nid_req = 1; + int nid_exp = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = SZ_2 * req_node->size; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, exp_node->base); + ASSERT_LE(region_end(new_rgn), region_end(exp_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is fully reserved: + * + * |----------------------+ +-----------+ | + * | expected | | requested | | + * +----------------------+-----+-----------+--------------------+ + * + * |-----------+ +-----------+ | + * | new | | reserved | | + * +-----------+----------------+-----------+--------------------+ + * + * Expect to allocate an aligned region at the beginning of the first node that + * is large enough and has enough unreserved memory (in this case, nid = 0) + * after falling back to NUMA_NO_NODE. The region count and total size get + * updated. + */ +static int alloc_try_nid_bottom_up_numa_node_reserved_check(void) +{ + int nid_req = 2; + int nid_exp = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = req_node->size; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + memblock_reserve(req_node->base, req_node->size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, exp_node->base); + ASSERT_LE(region_end(new_rgn), region_end(exp_node)); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, size + req_node->size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is partially reserved but has enough memory for the allocated region: + * + * | +---------------------------------------+ | + * | | requested | | + * +-----------+---------------------------------------+---------+ + * + * | +------------------+-----+ | + * | | reserved | new | | + * +-----------+------------------+-----+------------------------+ + * + * Expect to allocate an aligned region in the requested node that merges with + * the existing reserved region. The total size gets updated. + */ +static int alloc_try_nid_bottom_up_numa_part_reserved_check(void) +{ + int nid_req = 4; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + struct region r1; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + phys_addr_t total_size; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_8, req_node->size); + r1.base = req_node->base; + r1.size = req_node->size / SZ_2; + size = r1.size / SZ_4; + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + total_size = size + r1.size; + + memblock_reserve(r1.base, r1.size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, total_size); + ASSERT_EQ(new_rgn->base, req_node->base); + ASSERT_LE(region_end(new_rgn), region_end(req_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * is partially reserved and does not have enough contiguous memory for the + * allocated region: + * + * |----------------------+ +-----------------------+ | + * | expected | | requested | | + * +----------------------+-------+-----------------------+---------+ + * + * |-----------+ +----------+ | + * | new | | reserved | | + * +-----------+------------------------+----------+----------------+ + * + * Expect to allocate an aligned region at the beginning of the first + * node that is large enough and has enough unreserved memory (in this case, + * nid = 0) after falling back to NUMA_NO_NODE. The region count and total size + * get updated. + */ +static int alloc_try_nid_bottom_up_numa_part_reserved_fallback_check(void) +{ + int nid_req = 4; + int nid_exp = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + struct region r1; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + ASSERT_LE(SZ_4, req_node->size); + size = req_node->size / SZ_2; + r1.base = req_node->base + (size / SZ_2); + r1.size = size; + + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + memblock_reserve(r1.base, r1.size); + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, exp_node->base); + ASSERT_LE(region_end(new_rgn), region_end(exp_node)); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, size + r1.size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the first + * node is the requested node: + * + * min_addr + * | max_addr + * | | + * v v + * | +-----------------------+-----------+ | + * | | requested | node3 | | + * +-----------+-----------------------+-----------+--------------+ + * + + + * | +-----------+ | + * | | rgn | | + * +-----------+-----------+--------------------------------------+ + * + * Expect to drop the lower limit and allocate a memory region at the beginning + * of the requested node. + */ +static int alloc_try_nid_bottom_up_numa_split_range_low_check(void) +{ + int nid_req = 2; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_512; + phys_addr_t min_addr; + phys_addr_t max_addr; + phys_addr_t req_node_end; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + req_node_end = region_end(req_node); + min_addr = req_node_end - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, req_node->base); + ASSERT_LE(region_end(new_rgn), req_node_end); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the second + * node is the requested node: + * + * min_addr + * | max_addr + * | | + * v v + * |------------------+ +----------------------+---------+ | + * | expected | | previous |requested| | + * +------------------+--------+----------------------+---------+------+ + * + + + * |---------+ | + * | rgn | | + * +---------+---------------------------------------------------------+ + * + * Expect to drop the lower limit and allocate a memory region at the beginning + * of the first node that has enough memory. + */ +static int alloc_try_nid_bottom_up_numa_split_range_high_check(void) +{ + int nid_req = 3; + int nid_exp = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *exp_node = &memblock.memory.regions[nid_exp]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_512; + phys_addr_t min_addr; + phys_addr_t max_addr; + phys_addr_t exp_node_end; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + exp_node_end = region_end(req_node); + min_addr = req_node->base - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, exp_node->base); + ASSERT_LE(region_end(new_rgn), exp_node_end); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region that spans over the min_addr + * and max_addr range and overlaps with two different nodes, where the requested + * node ends before min_addr: + * + * min_addr + * | max_addr + * | | + * v v + * | +---------------+ +-------------+---------+ | + * | | requested | | node1 | node2 | | + * +----+---------------+--------+-------------+---------+---------+ + * + + + * | +---------+ | + * | | rgn | | + * +----+---------+------------------------------------------------+ + * + * Expect to drop the lower limit and allocate a memory region that starts at + * the beginning of the requested node. + */ +static int alloc_try_nid_bottom_up_numa_no_overlap_split_check(void) +{ + int nid_req = 2; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *node2 = &memblock.memory.regions[6]; + void *allocated_ptr = NULL; + phys_addr_t size; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + size = SZ_512; + min_addr = node2->base - SZ_256; + max_addr = min_addr + size; + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, req_node->base); + ASSERT_LE(region_end(new_rgn), region_end(req_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_add range when + * the requested node and the range do not overlap, and requested node ends + * before min_addr. The range overlaps with multiple nodes along node + * boundaries: + * + * min_addr + * | max_addr + * | | + * v v + * |-----------+ +----------+----...----+----------+ | + * | requested | | min node | ... | max node | | + * +-----------+-----------+----------+----...----+----------+------+ + * + + + * | +-----+ | + * | | rgn | | + * +-----------------------+-----+----------------------------------+ + * + * Expect to allocate a memory region at the beginning of the first node + * in the range after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_bottom_up_numa_no_overlap_low_check(void) +{ + int nid_req = 0; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *min_node = &memblock.memory.regions[2]; + struct memblock_region *max_node = &memblock.memory.regions[5]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_64; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + min_addr = min_node->base; + max_addr = region_end(max_node); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, min_addr); + ASSERT_LE(region_end(new_rgn), region_end(min_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_add range when + * the requested node and the range do not overlap, and requested node starts + * after max_addr. The range overlaps with multiple nodes along node + * boundaries: + * + * min_addr + * | max_addr + * | | + * v v + * | +----------+----...----+----------+ +---------+ | + * | | min node | ... | max node | |requested| | + * +-----+----------+----...----+----------+---------+---------+---+ + * + + + * | +-----+ | + * | | rgn | | + * +-----+-----+---------------------------------------------------+ + * + * Expect to allocate a memory region at the beginning of the first node + * in the range after falling back to NUMA_NO_NODE. + */ +static int alloc_try_nid_bottom_up_numa_no_overlap_high_check(void) +{ + int nid_req = 7; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *min_node = &memblock.memory.regions[2]; + struct memblock_region *max_node = &memblock.memory.regions[5]; + void *allocated_ptr = NULL; + phys_addr_t size = SZ_64; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + min_addr = min_node->base; + max_addr = region_end(max_node); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, size); + ASSERT_EQ(new_rgn->base, min_addr); + ASSERT_LE(region_end(new_rgn), region_end(min_node)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate a memory region in a specific NUMA node that + * does not have enough memory to allocate a region of the requested size. + * Additionally, none of the nodes have enough memory to allocate the region: + * + * +-----------------------------------+ + * | new | + * +-----------------------------------+ + * |-------+-------+-------+-------+-------+-------+-------+-------| + * | node0 | node1 | node2 | node3 | node4 | node5 | node6 | node7 | + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * + * Expect no allocation to happen. + */ +static int alloc_try_nid_numa_large_region_generic_check(void) +{ + int nid_req = 3; + void *allocated_ptr = NULL; + phys_addr_t size = MEM_SIZE / SZ_2; + phys_addr_t min_addr; + phys_addr_t max_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + min_addr = memblock_start_of_DRAM(); + max_addr = memblock_end_of_DRAM(); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + ASSERT_EQ(allocated_ptr, NULL); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_addr range when + * there are two reserved regions at the borders. The requested node starts at + * min_addr and ends at max_addr and is the same size as the region to be + * allocated: + * + * min_addr + * | max_addr + * | | + * v v + * | +-----------+-----------------------+-----------------------| + * | | node5 | requested | node7 | + * +------+-----------+-----------------------+-----------------------+ + * + + + * | +----+-----------------------+----+ | + * | | r2 | new | r1 | | + * +-------------+----+-----------------------+----+------------------+ + * + * Expect to merge all of the regions into one. The region counter and total + * size fields get updated. + */ +static int alloc_try_nid_numa_reserved_full_merge_generic_check(void) +{ + int nid_req = 6; + int nid_next = nid_req + 1; + struct memblock_region *new_rgn = &memblock.reserved.regions[0]; + struct memblock_region *req_node = &memblock.memory.regions[nid_req]; + struct memblock_region *next_node = &memblock.memory.regions[nid_next]; + void *allocated_ptr = NULL; + struct region r1, r2; + phys_addr_t size = req_node->size; + phys_addr_t total_size; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + r1.base = next_node->base; + r1.size = SZ_128; + + r2.size = SZ_128; + r2.base = r1.base - (size + r2.size); + + total_size = r1.size + r2.size + size; + min_addr = r2.base + r2.size; + max_addr = r1.base; + + memblock_reserve(r1.base, r1.size); + memblock_reserve(r2.base, r2.size); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, nid_req); + + ASSERT_NE(allocated_ptr, NULL); + assert_mem_content(allocated_ptr, size, alloc_nid_test_flags); + + ASSERT_EQ(new_rgn->size, total_size); + ASSERT_EQ(new_rgn->base, r2.base); + + ASSERT_LE(new_rgn->base, req_node->base); + ASSERT_LE(region_end(req_node), region_end(new_rgn)); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to allocate memory within min_addr and max_add range, + * where the total range can fit the region, but it is split between two nodes + * and everything else is reserved. Additionally, nid is set to NUMA_NO_NODE + * instead of requesting a specific node: + * + * +-----------+ + * | new | + * +-----------+ + * | +---------------------+-----------| + * | | prev node | next node | + * +------+---------------------+-----------+ + * + + + * |----------------------+ +-----| + * | r1 | | r2 | + * +----------------------+-----------+-----+ + * ^ ^ + * | | + * | max_addr + * | + * min_addr + * + * Expect no allocation to happen. + */ +static int alloc_try_nid_numa_split_all_reserved_generic_check(void) +{ + void *allocated_ptr = NULL; + struct memblock_region *next_node = &memblock.memory.regions[7]; + struct region r1, r2; + phys_addr_t size = SZ_256; + phys_addr_t max_addr; + phys_addr_t min_addr; + + PREFIX_PUSH(); + setup_numa_memblock(node_fractions); + + r2.base = next_node->base + SZ_128; + r2.size = memblock_end_of_DRAM() - r2.base; + + r1.size = MEM_SIZE - (r2.size + size); + r1.base = memblock_start_of_DRAM(); + + min_addr = r1.base + r1.size; + max_addr = r2.base; + + memblock_reserve(r1.base, r1.size); + memblock_reserve(r2.base, r2.size); + + allocated_ptr = run_memblock_alloc_try_nid(size, SMP_CACHE_BYTES, + min_addr, max_addr, + NUMA_NO_NODE); + + ASSERT_EQ(allocated_ptr, NULL); + + test_pass_pop(); + + return 0; +} + +/* Test case wrappers for NUMA tests */ +static int alloc_try_nid_numa_simple_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_simple_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_simple_check(); + + return 0; +} + +static int alloc_try_nid_numa_small_node_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_small_node_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_small_node_check(); + + return 0; +} + +static int alloc_try_nid_numa_node_reserved_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_node_reserved_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_node_reserved_check(); + + return 0; +} + +static int alloc_try_nid_numa_part_reserved_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_part_reserved_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_part_reserved_check(); + + return 0; +} + +static int alloc_try_nid_numa_part_reserved_fallback_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_part_reserved_fallback_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_part_reserved_fallback_check(); + + return 0; +} + +static int alloc_try_nid_numa_split_range_low_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_split_range_low_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_split_range_low_check(); + + return 0; +} + +static int alloc_try_nid_numa_split_range_high_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_split_range_high_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_split_range_high_check(); + + return 0; +} + +static int alloc_try_nid_numa_no_overlap_split_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_no_overlap_split_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_no_overlap_split_check(); + + return 0; +} + +static int alloc_try_nid_numa_no_overlap_low_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_no_overlap_low_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_no_overlap_low_check(); + + return 0; +} + +static int alloc_try_nid_numa_no_overlap_high_check(void) +{ + test_print("\tRunning %s...\n", __func__); + memblock_set_bottom_up(false); + alloc_try_nid_top_down_numa_no_overlap_high_check(); + memblock_set_bottom_up(true); + alloc_try_nid_bottom_up_numa_no_overlap_high_check(); + + return 0; +} + +static int alloc_try_nid_numa_large_region_check(void) +{ + test_print("\tRunning %s...\n", __func__); + run_top_down(alloc_try_nid_numa_large_region_generic_check); + run_bottom_up(alloc_try_nid_numa_large_region_generic_check); + + return 0; +} + +static int alloc_try_nid_numa_reserved_full_merge_check(void) +{ + test_print("\tRunning %s...\n", __func__); + run_top_down(alloc_try_nid_numa_reserved_full_merge_generic_check); + run_bottom_up(alloc_try_nid_numa_reserved_full_merge_generic_check); + + return 0; +} + +static int alloc_try_nid_numa_split_all_reserved_check(void) +{ + test_print("\tRunning %s...\n", __func__); + run_top_down(alloc_try_nid_numa_split_all_reserved_generic_check); + run_bottom_up(alloc_try_nid_numa_split_all_reserved_generic_check); + + return 0; +} + +int __memblock_alloc_nid_numa_checks(void) +{ + test_print("Running %s NUMA tests...\n", + get_memblock_alloc_try_nid_name(alloc_nid_test_flags)); + + alloc_try_nid_numa_simple_check(); + alloc_try_nid_numa_small_node_check(); + alloc_try_nid_numa_node_reserved_check(); + alloc_try_nid_numa_part_reserved_check(); + alloc_try_nid_numa_part_reserved_fallback_check(); + alloc_try_nid_numa_split_range_low_check(); + alloc_try_nid_numa_split_range_high_check(); + + alloc_try_nid_numa_no_overlap_split_check(); + alloc_try_nid_numa_no_overlap_low_check(); + alloc_try_nid_numa_no_overlap_high_check(); + alloc_try_nid_numa_large_region_check(); + alloc_try_nid_numa_reserved_full_merge_check(); + alloc_try_nid_numa_split_all_reserved_check(); + + return 0; +} + +static int memblock_alloc_nid_checks_internal(int flags) +{ + alloc_nid_test_flags = flags; + + prefix_reset(); + prefix_push(get_memblock_alloc_try_nid_name(flags)); + + reset_memblock_attributes(); + dummy_physical_memory_init(); + + memblock_alloc_nid_range_checks(); + memblock_alloc_nid_numa_checks(); + + dummy_physical_memory_cleanup(); + + prefix_pop(); + + return 0; +} + +int memblock_alloc_nid_checks(void) +{ + memblock_alloc_nid_checks_internal(TEST_F_NONE); + memblock_alloc_nid_checks_internal(TEST_F_RAW); return 0; } diff --git a/tools/testing/memblock/tests/alloc_nid_api.h b/tools/testing/memblock/tests/alloc_nid_api.h index b35cf3c3f4898134c62be9bed02d4fd5fdaee76a..92d07d230e18f3f6660b6453ffd077569d0e015d 100644 --- a/tools/testing/memblock/tests/alloc_nid_api.h +++ b/tools/testing/memblock/tests/alloc_nid_api.h @@ -5,5 +5,21 @@ #include "common.h" int memblock_alloc_nid_checks(void); +int __memblock_alloc_nid_numa_checks(void); + +#ifdef CONFIG_NUMA +static inline int memblock_alloc_nid_numa_checks(void) +{ + __memblock_alloc_nid_numa_checks(); + return 0; +} + +#else +static inline int memblock_alloc_nid_numa_checks(void) +{ + return 0; +} + +#endif /* CONFIG_NUMA */ #endif diff --git a/tools/testing/memblock/tests/basic_api.c b/tools/testing/memblock/tests/basic_api.c index 66f46f261e66835ea32af35a1436127ca71de7e9..a13a57ba0815fccf96530d9e8047758d4cab1172 100644 --- a/tools/testing/memblock/tests/basic_api.c +++ b/tools/testing/memblock/tests/basic_api.c @@ -8,6 +8,7 @@ #define FUNC_RESERVE "memblock_reserve" #define FUNC_REMOVE "memblock_remove" #define FUNC_FREE "memblock_free" +#define FUNC_TRIM "memblock_trim_memory" static int memblock_initialization_check(void) { @@ -326,6 +327,102 @@ static int memblock_add_twice_check(void) return 0; } +/* + * A test that tries to add two memory blocks that don't overlap with one + * another and then add a third memory block in the space between the first two: + * + * | +--------+--------+--------+ | + * | | r1 | r3 | r2 | | + * +--------+--------+--------+--------+--+ + * + * Expect to merge the three entries into one region that starts at r1.base + * and has size of r1.size + r2.size + r3.size. The region counter and total + * size of the available memory are updated. + */ +static int memblock_add_between_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.memory.regions[0]; + + struct region r1 = { + .base = SZ_1G, + .size = SZ_8K + }; + struct region r2 = { + .base = SZ_1G + SZ_16K, + .size = SZ_8K + }; + struct region r3 = { + .base = SZ_1G + SZ_8K, + .size = SZ_8K + }; + + PREFIX_PUSH(); + + total_size = r1.size + r2.size + r3.size; + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_add(r2.base, r2.size); + memblock_add(r3.base, r3.size); + + ASSERT_EQ(rgn->base, r1.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A simple test that tries to add a memory block r when r extends past + * PHYS_ADDR_MAX: + * + * +--------+ + * | r | + * +--------+ + * | +----+ + * | | rgn| + * +----------------------------+----+ + * + * Expect to add a memory block of size PHYS_ADDR_MAX - r.base. Expect the + * total size of available memory and the counter to be updated. + */ +static int memblock_add_near_max_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.memory.regions[0]; + + struct region r = { + .base = PHYS_ADDR_MAX - SZ_1M, + .size = SZ_2M + }; + + PREFIX_PUSH(); + + total_size = PHYS_ADDR_MAX - r.base; + + reset_memblock_regions(); + memblock_add(r.base, r.size); + + ASSERT_EQ(rgn->base, r.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.total_size, total_size); + + test_pass_pop(); + + return 0; +} + static int memblock_add_checks(void) { prefix_reset(); @@ -339,6 +436,8 @@ static int memblock_add_checks(void) memblock_add_overlap_bottom_check(); memblock_add_within_check(); memblock_add_twice_check(); + memblock_add_between_check(); + memblock_add_near_max_check(); prefix_pop(); @@ -604,6 +703,102 @@ static int memblock_reserve_twice_check(void) return 0; } +/* + * A test that tries to mark two memory blocks that don't overlap as reserved + * and then reserve a third memory block in the space between the first two: + * + * | +--------+--------+--------+ | + * | | r1 | r3 | r2 | | + * +--------+--------+--------+--------+--+ + * + * Expect to merge the three entries into one reserved region that starts at + * r1.base and has size of r1.size + r2.size + r3.size. The region counter and + * total for memblock.reserved are updated. + */ +static int memblock_reserve_between_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.reserved.regions[0]; + + struct region r1 = { + .base = SZ_1G, + .size = SZ_8K + }; + struct region r2 = { + .base = SZ_1G + SZ_16K, + .size = SZ_8K + }; + struct region r3 = { + .base = SZ_1G + SZ_8K, + .size = SZ_8K + }; + + PREFIX_PUSH(); + + total_size = r1.size + r2.size + r3.size; + + reset_memblock_regions(); + memblock_reserve(r1.base, r1.size); + memblock_reserve(r2.base, r2.size); + memblock_reserve(r3.base, r3.size); + + ASSERT_EQ(rgn->base, r1.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A simple test that tries to reserve a memory block r when r extends past + * PHYS_ADDR_MAX: + * + * +--------+ + * | r | + * +--------+ + * | +----+ + * | | rgn| + * +----------------------------+----+ + * + * Expect to reserve a memory block of size PHYS_ADDR_MAX - r.base. Expect the + * total size of reserved memory and the counter to be updated. + */ +static int memblock_reserve_near_max_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.reserved.regions[0]; + + struct region r = { + .base = PHYS_ADDR_MAX - SZ_1M, + .size = SZ_2M + }; + + PREFIX_PUSH(); + + total_size = PHYS_ADDR_MAX - r.base; + + reset_memblock_regions(); + memblock_reserve(r.base, r.size); + + ASSERT_EQ(rgn->base, r.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + static int memblock_reserve_checks(void) { prefix_reset(); @@ -616,6 +811,8 @@ static int memblock_reserve_checks(void) memblock_reserve_overlap_bottom_check(); memblock_reserve_within_check(); memblock_reserve_twice_check(); + memblock_reserve_between_check(); + memblock_reserve_near_max_check(); prefix_pop(); @@ -887,6 +1084,155 @@ static int memblock_remove_within_check(void) return 0; } +/* + * A simple test that tries to remove a region r1 from the array of + * available memory regions when r1 is the only available region. + * Expect to add a memory block r1 and then remove r1 so that a dummy + * region is added. The region counter stays the same, and the total size + * is updated. + */ +static int memblock_remove_only_region_check(void) +{ + struct memblock_region *rgn; + + rgn = &memblock.memory.regions[0]; + + struct region r1 = { + .base = SZ_2K, + .size = SZ_4K + }; + + PREFIX_PUSH(); + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_remove(r1.base, r1.size); + + ASSERT_EQ(rgn->base, 0); + ASSERT_EQ(rgn->size, 0); + + ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.total_size, 0); + + test_pass_pop(); + + return 0; +} + +/* + * A simple test that tries remove a region r2 from the array of available + * memory regions when r2 extends past PHYS_ADDR_MAX: + * + * +--------+ + * | r2 | + * +--------+ + * | +---+....+ + * | |rgn| | + * +------------------------+---+----+ + * + * Expect that only the portion between PHYS_ADDR_MAX and r2.base is removed. + * Expect the total size of available memory to be updated and the counter to + * not be updated. + */ +static int memblock_remove_near_max_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.memory.regions[0]; + + struct region r1 = { + .base = PHYS_ADDR_MAX - SZ_2M, + .size = SZ_2M + }; + + struct region r2 = { + .base = PHYS_ADDR_MAX - SZ_1M, + .size = SZ_2M + }; + + PREFIX_PUSH(); + + total_size = r1.size - (PHYS_ADDR_MAX - r2.base); + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_remove(r2.base, r2.size); + + ASSERT_EQ(rgn->base, r1.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.memory.cnt, 1); + ASSERT_EQ(memblock.memory.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to remove a region r3 that overlaps with two existing + * regions r1 and r2: + * + * +----------------+ + * | r3 | + * +----------------+ + * | +----+..... ........+--------+ + * | | |r1 : : |r2 | | + * +----+----+----+---+-------+--------+-----+ + * + * Expect that only the intersections of r1 with r3 and r2 with r3 are removed + * from the available memory pool. Expect the total size of available memory to + * be updated and the counter to not be updated. + */ +static int memblock_remove_overlap_two_check(void) +{ + struct memblock_region *rgn1, *rgn2; + phys_addr_t new_r1_size, new_r2_size, r2_end, r3_end, total_size; + + rgn1 = &memblock.memory.regions[0]; + rgn2 = &memblock.memory.regions[1]; + + struct region r1 = { + .base = SZ_16M, + .size = SZ_32M + }; + struct region r2 = { + .base = SZ_64M, + .size = SZ_64M + }; + struct region r3 = { + .base = SZ_32M, + .size = SZ_64M + }; + + PREFIX_PUSH(); + + r2_end = r2.base + r2.size; + r3_end = r3.base + r3.size; + new_r1_size = r3.base - r1.base; + new_r2_size = r2_end - r3_end; + total_size = new_r1_size + new_r2_size; + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_add(r2.base, r2.size); + memblock_remove(r3.base, r3.size); + + ASSERT_EQ(rgn1->base, r1.base); + ASSERT_EQ(rgn1->size, new_r1_size); + + ASSERT_EQ(rgn2->base, r3_end); + ASSERT_EQ(rgn2->size, new_r2_size); + + ASSERT_EQ(memblock.memory.cnt, 2); + ASSERT_EQ(memblock.memory.total_size, total_size); + + test_pass_pop(); + + return 0; +} + static int memblock_remove_checks(void) { prefix_reset(); @@ -898,6 +1244,9 @@ static int memblock_remove_checks(void) memblock_remove_overlap_top_check(); memblock_remove_overlap_bottom_check(); memblock_remove_within_check(); + memblock_remove_only_region_check(); + memblock_remove_near_max_check(); + memblock_remove_overlap_two_check(); prefix_pop(); @@ -1163,6 +1512,154 @@ static int memblock_free_within_check(void) return 0; } +/* + * A simple test that tries to free a memory block r1 that was marked + * earlier as reserved when r1 is the only available region. + * Expect to reserve a memory block r1 and then free r1 so that r1 is + * overwritten with a dummy region. The region counter stays the same, + * and the total size is updated. + */ +static int memblock_free_only_region_check(void) +{ + struct memblock_region *rgn; + + rgn = &memblock.reserved.regions[0]; + + struct region r1 = { + .base = SZ_2K, + .size = SZ_4K + }; + + PREFIX_PUSH(); + + reset_memblock_regions(); + memblock_reserve(r1.base, r1.size); + memblock_free((void *)r1.base, r1.size); + + ASSERT_EQ(rgn->base, 0); + ASSERT_EQ(rgn->size, 0); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, 0); + + test_pass_pop(); + + return 0; +} + +/* + * A simple test that tries free a region r2 when r2 extends past PHYS_ADDR_MAX: + * + * +--------+ + * | r2 | + * +--------+ + * | +---+....+ + * | |rgn| | + * +------------------------+---+----+ + * + * Expect that only the portion between PHYS_ADDR_MAX and r2.base is freed. + * Expect the total size of reserved memory to be updated and the counter to + * not be updated. + */ +static int memblock_free_near_max_check(void) +{ + struct memblock_region *rgn; + phys_addr_t total_size; + + rgn = &memblock.reserved.regions[0]; + + struct region r1 = { + .base = PHYS_ADDR_MAX - SZ_2M, + .size = SZ_2M + }; + + struct region r2 = { + .base = PHYS_ADDR_MAX - SZ_1M, + .size = SZ_2M + }; + + PREFIX_PUSH(); + + total_size = r1.size - (PHYS_ADDR_MAX - r2.base); + + reset_memblock_regions(); + memblock_reserve(r1.base, r1.size); + memblock_free((void *)r2.base, r2.size); + + ASSERT_EQ(rgn->base, r1.base); + ASSERT_EQ(rgn->size, total_size); + + ASSERT_EQ(memblock.reserved.cnt, 1); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to free a reserved region r3 that overlaps with two + * existing reserved regions r1 and r2: + * + * +----------------+ + * | r3 | + * +----------------+ + * | +----+..... ........+--------+ + * | | |r1 : : |r2 | | + * +----+----+----+---+-------+--------+-----+ + * + * Expect that only the intersections of r1 with r3 and r2 with r3 are freed + * from the collection of reserved memory. Expect the total size of reserved + * memory to be updated and the counter to not be updated. + */ +static int memblock_free_overlap_two_check(void) +{ + struct memblock_region *rgn1, *rgn2; + phys_addr_t new_r1_size, new_r2_size, r2_end, r3_end, total_size; + + rgn1 = &memblock.reserved.regions[0]; + rgn2 = &memblock.reserved.regions[1]; + + struct region r1 = { + .base = SZ_16M, + .size = SZ_32M + }; + struct region r2 = { + .base = SZ_64M, + .size = SZ_64M + }; + struct region r3 = { + .base = SZ_32M, + .size = SZ_64M + }; + + PREFIX_PUSH(); + + r2_end = r2.base + r2.size; + r3_end = r3.base + r3.size; + new_r1_size = r3.base - r1.base; + new_r2_size = r2_end - r3_end; + total_size = new_r1_size + new_r2_size; + + reset_memblock_regions(); + memblock_reserve(r1.base, r1.size); + memblock_reserve(r2.base, r2.size); + memblock_free((void *)r3.base, r3.size); + + ASSERT_EQ(rgn1->base, r1.base); + ASSERT_EQ(rgn1->size, new_r1_size); + + ASSERT_EQ(rgn2->base, r3_end); + ASSERT_EQ(rgn2->size, new_r2_size); + + ASSERT_EQ(memblock.reserved.cnt, 2); + ASSERT_EQ(memblock.reserved.total_size, total_size); + + test_pass_pop(); + + return 0; +} + static int memblock_free_checks(void) { prefix_reset(); @@ -1174,6 +1671,274 @@ static int memblock_free_checks(void) memblock_free_overlap_top_check(); memblock_free_overlap_bottom_check(); memblock_free_within_check(); + memblock_free_only_region_check(); + memblock_free_near_max_check(); + memblock_free_overlap_two_check(); + + prefix_pop(); + + return 0; +} + +static int memblock_set_bottom_up_check(void) +{ + prefix_push("memblock_set_bottom_up"); + + memblock_set_bottom_up(false); + ASSERT_EQ(memblock.bottom_up, false); + memblock_set_bottom_up(true); + ASSERT_EQ(memblock.bottom_up, true); + + reset_memblock_attributes(); + test_pass_pop(); + + return 0; +} + +static int memblock_bottom_up_check(void) +{ + prefix_push("memblock_bottom_up"); + + memblock_set_bottom_up(false); + ASSERT_EQ(memblock_bottom_up(), memblock.bottom_up); + ASSERT_EQ(memblock_bottom_up(), false); + memblock_set_bottom_up(true); + ASSERT_EQ(memblock_bottom_up(), memblock.bottom_up); + ASSERT_EQ(memblock_bottom_up(), true); + + reset_memblock_attributes(); + test_pass_pop(); + + return 0; +} + +static int memblock_bottom_up_checks(void) +{ + test_print("Running memblock_*bottom_up tests...\n"); + + prefix_reset(); + memblock_set_bottom_up_check(); + prefix_reset(); + memblock_bottom_up_check(); + + return 0; +} + +/* + * A test that tries to trim memory when both ends of the memory region are + * aligned. Expect that the memory will not be trimmed. Expect the counter to + * not be updated. + */ +static int memblock_trim_memory_aligned_check(void) +{ + struct memblock_region *rgn; + const phys_addr_t alignment = SMP_CACHE_BYTES; + + rgn = &memblock.memory.regions[0]; + + struct region r = { + .base = alignment, + .size = alignment * 4 + }; + + PREFIX_PUSH(); + + reset_memblock_regions(); + memblock_add(r.base, r.size); + memblock_trim_memory(alignment); + + ASSERT_EQ(rgn->base, r.base); + ASSERT_EQ(rgn->size, r.size); + + ASSERT_EQ(memblock.memory.cnt, 1); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to trim memory when there are two available regions, r1 and + * r2. Region r1 is aligned on both ends and region r2 is unaligned on one end + * and smaller than the alignment: + * + * alignment + * |--------| + * | +-----------------+ +------+ | + * | | r1 | | r2 | | + * +--------+-----------------+--------+------+---+ + * ^ ^ ^ ^ ^ + * |________|________|________| | + * | Unaligned address + * Aligned addresses + * + * Expect that r1 will not be trimmed and r2 will be removed. Expect the + * counter to be updated. + */ +static int memblock_trim_memory_too_small_check(void) +{ + struct memblock_region *rgn; + const phys_addr_t alignment = SMP_CACHE_BYTES; + + rgn = &memblock.memory.regions[0]; + + struct region r1 = { + .base = alignment, + .size = alignment * 2 + }; + struct region r2 = { + .base = alignment * 4, + .size = alignment - SZ_2 + }; + + PREFIX_PUSH(); + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_add(r2.base, r2.size); + memblock_trim_memory(alignment); + + ASSERT_EQ(rgn->base, r1.base); + ASSERT_EQ(rgn->size, r1.size); + + ASSERT_EQ(memblock.memory.cnt, 1); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to trim memory when there are two available regions, r1 and + * r2. Region r1 is aligned on both ends and region r2 is unaligned at the base + * and aligned at the end: + * + * Unaligned address + * | + * v + * | +-----------------+ +---------------+ | + * | | r1 | | r2 | | + * +--------+-----------------+----------+---------------+---+ + * ^ ^ ^ ^ ^ ^ + * |________|________|________|________|________| + * | + * Aligned addresses + * + * Expect that r1 will not be trimmed and r2 will be trimmed at the base. + * Expect the counter to not be updated. + */ +static int memblock_trim_memory_unaligned_base_check(void) +{ + struct memblock_region *rgn1, *rgn2; + const phys_addr_t alignment = SMP_CACHE_BYTES; + phys_addr_t offset = SZ_2; + phys_addr_t new_r2_base, new_r2_size; + + rgn1 = &memblock.memory.regions[0]; + rgn2 = &memblock.memory.regions[1]; + + struct region r1 = { + .base = alignment, + .size = alignment * 2 + }; + struct region r2 = { + .base = alignment * 4 + offset, + .size = alignment * 2 - offset + }; + + PREFIX_PUSH(); + + new_r2_base = r2.base + (alignment - offset); + new_r2_size = r2.size - (alignment - offset); + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_add(r2.base, r2.size); + memblock_trim_memory(alignment); + + ASSERT_EQ(rgn1->base, r1.base); + ASSERT_EQ(rgn1->size, r1.size); + + ASSERT_EQ(rgn2->base, new_r2_base); + ASSERT_EQ(rgn2->size, new_r2_size); + + ASSERT_EQ(memblock.memory.cnt, 2); + + test_pass_pop(); + + return 0; +} + +/* + * A test that tries to trim memory when there are two available regions, r1 and + * r2. Region r1 is aligned on both ends and region r2 is aligned at the base + * and unaligned at the end: + * + * Unaligned address + * | + * v + * | +-----------------+ +---------------+ | + * | | r1 | | r2 | | + * +--------+-----------------+--------+---------------+---+ + * ^ ^ ^ ^ ^ ^ + * |________|________|________|________|________| + * | + * Aligned addresses + * + * Expect that r1 will not be trimmed and r2 will be trimmed at the end. + * Expect the counter to not be updated. + */ +static int memblock_trim_memory_unaligned_end_check(void) +{ + struct memblock_region *rgn1, *rgn2; + const phys_addr_t alignment = SMP_CACHE_BYTES; + phys_addr_t offset = SZ_2; + phys_addr_t new_r2_size; + + rgn1 = &memblock.memory.regions[0]; + rgn2 = &memblock.memory.regions[1]; + + struct region r1 = { + .base = alignment, + .size = alignment * 2 + }; + struct region r2 = { + .base = alignment * 4, + .size = alignment * 2 - offset + }; + + PREFIX_PUSH(); + + new_r2_size = r2.size - (alignment - offset); + + reset_memblock_regions(); + memblock_add(r1.base, r1.size); + memblock_add(r2.base, r2.size); + memblock_trim_memory(alignment); + + ASSERT_EQ(rgn1->base, r1.base); + ASSERT_EQ(rgn1->size, r1.size); + + ASSERT_EQ(rgn2->base, r2.base); + ASSERT_EQ(rgn2->size, new_r2_size); + + ASSERT_EQ(memblock.memory.cnt, 2); + + test_pass_pop(); + + return 0; +} + +static int memblock_trim_memory_checks(void) +{ + prefix_reset(); + prefix_push(FUNC_TRIM); + test_print("Running %s tests...\n", FUNC_TRIM); + + memblock_trim_memory_aligned_check(); + memblock_trim_memory_too_small_check(); + memblock_trim_memory_unaligned_base_check(); + memblock_trim_memory_unaligned_end_check(); prefix_pop(); @@ -1187,6 +1952,8 @@ int memblock_basic_checks(void) memblock_reserve_checks(); memblock_remove_checks(); memblock_free_checks(); + memblock_bottom_up_checks(); + memblock_trim_memory_checks(); return 0; } diff --git a/tools/testing/memblock/tests/common.c b/tools/testing/memblock/tests/common.c index e43b2676af816c03c815a77bf537d968931775bc..3f795047bbe18c907d13d91b1e2c5fc940118480 100644 --- a/tools/testing/memblock/tests/common.c +++ b/tools/testing/memblock/tests/common.c @@ -9,19 +9,22 @@ #define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS #define PREFIXES_MAX 15 #define DELIM ": " +#define BASIS 10000 static struct test_memory memory_block; static const char __maybe_unused *prefixes[PREFIXES_MAX]; static int __maybe_unused nr_prefixes; -static const char *short_opts = "mv"; +static const char *short_opts = "hmv"; static const struct option long_opts[] = { + {"help", 0, NULL, 'h'}, {"movable-node", 0, NULL, 'm'}, {"verbose", 0, NULL, 'v'}, {NULL, 0, NULL, 0} }; static const char * const help_opts[] = { + "display this help message and exit", "disallow allocations from regions marked as hotplugged\n\t\t\t" "by simulating enabling the \"movable_node\" kernel\n\t\t\t" "parameter", @@ -58,16 +61,53 @@ void reset_memblock_attributes(void) memblock.current_limit = MEMBLOCK_ALLOC_ANYWHERE; } +static inline void fill_memblock(void) +{ + memset(memory_block.base, 1, MEM_SIZE); +} + void setup_memblock(void) { reset_memblock_regions(); memblock_add((phys_addr_t)memory_block.base, MEM_SIZE); + fill_memblock(); +} + +/** + * setup_numa_memblock: + * Set up a memory layout with multiple NUMA nodes in a previously allocated + * dummy physical memory. + * @node_fracs: an array representing the fraction of MEM_SIZE contained in + * each node in basis point units (one hundredth of 1% or 1/10000). + * For example, if node 0 should contain 1/8 of MEM_SIZE, + * node_fracs[0] = 1250. + * + * The nids will be set to 0 through NUMA_NODES - 1. + */ +void setup_numa_memblock(const unsigned int node_fracs[]) +{ + phys_addr_t base; + int flags; + + reset_memblock_regions(); + base = (phys_addr_t)memory_block.base; + flags = (movable_node_is_enabled()) ? MEMBLOCK_NONE : MEMBLOCK_HOTPLUG; + + for (int i = 0; i < NUMA_NODES; i++) { + assert(node_fracs[i] <= BASIS); + phys_addr_t size = MEM_SIZE * node_fracs[i] / BASIS; + + memblock_add_node(base, size, i, flags); + base += size; + } + fill_memblock(); } void dummy_physical_memory_init(void) { memory_block.base = malloc(MEM_SIZE); assert(memory_block.base); + fill_memblock(); } void dummy_physical_memory_cleanup(void) diff --git a/tools/testing/memblock/tests/common.h b/tools/testing/memblock/tests/common.h index 3e7f23d341d794cf7c46429cc0de7ca5a5ccc94d..d6bbbe63bfc36eb17e504ef5b0550d8841f333cb 100644 --- a/tools/testing/memblock/tests/common.h +++ b/tools/testing/memblock/tests/common.h @@ -10,13 +10,22 @@ #include <linux/printk.h> #include <../selftests/kselftest.h> -#define MEM_SIZE SZ_16K +#define MEM_SIZE SZ_16K +#define NUMA_NODES 8 + +enum test_flags { + /* No special request. */ + TEST_F_NONE = 0x0, + /* Perform raw allocations (no zeroing of memory). */ + TEST_F_RAW = 0x1, +}; /** * ASSERT_EQ(): * Check the condition * @_expected == @_seen - * If false, print failed test message (if in VERBOSE mode) and then assert + * If false, print failed test message (if running with --verbose) and then + * assert. */ #define ASSERT_EQ(_expected, _seen) do { \ if ((_expected) != (_seen)) \ @@ -28,7 +37,8 @@ * ASSERT_NE(): * Check the condition * @_expected != @_seen - * If false, print failed test message (if in VERBOSE mode) and then assert + * If false, print failed test message (if running with --verbose) and then + * assert. */ #define ASSERT_NE(_expected, _seen) do { \ if ((_expected) == (_seen)) \ @@ -40,7 +50,8 @@ * ASSERT_LT(): * Check the condition * @_expected < @_seen - * If false, print failed test message (if in VERBOSE mode) and then assert + * If false, print failed test message (if running with --verbose) and then + * assert. */ #define ASSERT_LT(_expected, _seen) do { \ if ((_expected) >= (_seen)) \ @@ -48,6 +59,43 @@ assert((_expected) < (_seen)); \ } while (0) +/** + * ASSERT_LE(): + * Check the condition + * @_expected <= @_seen + * If false, print failed test message (if running with --verbose) and then + * assert. + */ +#define ASSERT_LE(_expected, _seen) do { \ + if ((_expected) > (_seen)) \ + test_fail(); \ + assert((_expected) <= (_seen)); \ +} while (0) + +/** + * ASSERT_MEM_EQ(): + * Check that the first @_size bytes of @_seen are all equal to @_expected. + * If false, print failed test message (if running with --verbose) and then + * assert. + */ +#define ASSERT_MEM_EQ(_seen, _expected, _size) do { \ + for (int _i = 0; _i < (_size); _i++) { \ + ASSERT_EQ(((char *)_seen)[_i], (_expected)); \ + } \ +} while (0) + +/** + * ASSERT_MEM_NE(): + * Check that none of the first @_size bytes of @_seen are equal to @_expected. + * If false, print failed test message (if running with --verbose) and then + * assert. + */ +#define ASSERT_MEM_NE(_seen, _expected, _size) do { \ + for (int _i = 0; _i < (_size); _i++) { \ + ASSERT_NE(((char *)_seen)[_i], (_expected)); \ + } \ +} while (0) + #define PREFIX_PUSH() prefix_push(__func__) /* @@ -65,9 +113,15 @@ struct region { phys_addr_t size; }; +static inline phys_addr_t __maybe_unused region_end(struct memblock_region *rgn) +{ + return rgn->base + rgn->size; +} + void reset_memblock_regions(void); void reset_memblock_attributes(void); void setup_memblock(void); +void setup_numa_memblock(const unsigned int node_fracs[]); void dummy_physical_memory_init(void); void dummy_physical_memory_cleanup(void); void parse_args(int argc, char **argv); @@ -85,4 +139,28 @@ static inline void test_pass_pop(void) prefix_pop(); } +static inline void run_top_down(int (*func)()) +{ + memblock_set_bottom_up(false); + prefix_push("top-down"); + func(); + prefix_pop(); +} + +static inline void run_bottom_up(int (*func)()) +{ + memblock_set_bottom_up(true); + prefix_push("bottom-up"); + func(); + prefix_pop(); +} + +static inline void assert_mem_content(void *mem, int size, int flags) +{ + if (flags & TEST_F_RAW) + ASSERT_MEM_NE(mem, 0, size); + else + ASSERT_MEM_EQ(mem, 0, size); +} + #endif diff --git a/tools/testing/nvdimm/test/ndtest.c b/tools/testing/nvdimm/test/ndtest.c index 4d1a947367f9b66ca81474e570c036e308bfbfff..01ceb98c15a08e5bbc81444897c8cfb761f55ffd 100644 --- a/tools/testing/nvdimm/test/ndtest.c +++ b/tools/testing/nvdimm/test/ndtest.c @@ -134,39 +134,6 @@ static struct ndtest_mapping region1_mapping[] = { }, }; -static struct ndtest_mapping region2_mapping[] = { - { - .dimm = 0, - .position = 0, - .start = 0, - .size = DIMM_SIZE, - }, -}; - -static struct ndtest_mapping region3_mapping[] = { - { - .dimm = 1, - .start = 0, - .size = DIMM_SIZE, - } -}; - -static struct ndtest_mapping region4_mapping[] = { - { - .dimm = 2, - .start = 0, - .size = DIMM_SIZE, - } -}; - -static struct ndtest_mapping region5_mapping[] = { - { - .dimm = 3, - .start = 0, - .size = DIMM_SIZE, - } -}; - static struct ndtest_region bus0_regions[] = { { .type = ND_DEVICE_NAMESPACE_PMEM, @@ -182,34 +149,6 @@ static struct ndtest_region bus0_regions[] = { .size = DIMM_SIZE * 2, .range_index = 2, }, - { - .type = ND_DEVICE_NAMESPACE_BLK, - .num_mappings = ARRAY_SIZE(region2_mapping), - .mapping = region2_mapping, - .size = DIMM_SIZE, - .range_index = 3, - }, - { - .type = ND_DEVICE_NAMESPACE_BLK, - .num_mappings = ARRAY_SIZE(region3_mapping), - .mapping = region3_mapping, - .size = DIMM_SIZE, - .range_index = 4, - }, - { - .type = ND_DEVICE_NAMESPACE_BLK, - .num_mappings = ARRAY_SIZE(region4_mapping), - .mapping = region4_mapping, - .size = DIMM_SIZE, - .range_index = 5, - }, - { - .type = ND_DEVICE_NAMESPACE_BLK, - .num_mappings = ARRAY_SIZE(region5_mapping), - .mapping = region5_mapping, - .size = DIMM_SIZE, - .range_index = 6, - }, }; static struct ndtest_mapping region6_mapping[] = { @@ -501,21 +440,6 @@ static int ndtest_create_region(struct ndtest_priv *p, nd_set->altcookie = nd_set->cookie1; ndr_desc->nd_set = nd_set; - if (region->type == ND_DEVICE_NAMESPACE_BLK) { - mappings[0].start = 0; - mappings[0].size = DIMM_SIZE; - mappings[0].nvdimm = p->config->dimms[ndimm].nvdimm; - - ndr_desc->mapping = &mappings[0]; - ndr_desc->num_mappings = 1; - ndr_desc->num_lanes = 1; - ndbr_desc.enable = ndtest_blk_region_enable; - ndbr_desc.do_io = ndtest_blk_do_io; - region->region = nvdimm_blk_region_create(p->bus, ndr_desc); - - goto done; - } - for (i = 0; i < region->num_mappings; i++) { ndimm = region->mapping[i].dimm; mappings[i].start = region->mapping[i].start; @@ -527,7 +451,6 @@ static int ndtest_create_region(struct ndtest_priv *p, ndr_desc->num_mappings = region->num_mappings; region->region = nvdimm_pmem_region_create(p->bus, ndr_desc); -done: if (!region->region) { dev_err(&p->pdev.dev, "Error registering region %pR\n", ndr_desc->res); diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore index d971516401e68fb32f268c537d8e1d7f2ba93581..c901d96dd013eff47553a8bcf257dbe61b2635a3 100644 --- a/tools/testing/radix-tree/.gitignore +++ b/tools/testing/radix-tree/.gitignore @@ -6,3 +6,5 @@ main multiorder radix-tree.c xarray +maple +ma_xa_benchmark diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index c4ea4fbb0bfcd1fad58c282e43e80b099af2de9f..89d613e0505b306e8228615a92b37c01fa92038d 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -4,9 +4,9 @@ CFLAGS += -I. -I../../include -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address \ -fsanitize=undefined LDFLAGS += -fsanitize=address -fsanitize=undefined LDLIBS+= -lpthread -lurcu -TARGETS = main idr-test multiorder xarray +TARGETS = main idr-test multiorder xarray maple CORE_OFILES := xarray.o radix-tree.o idr.o linux.o test.o find_bit.o bitmap.o \ - slab.o + slab.o maple.o OFILES = main.o $(CORE_OFILES) regression1.o regression2.o regression3.o \ regression4.o tag_check.o multiorder.o idr-test.o iteration_check.o \ iteration_check_2.o benchmark.o @@ -29,6 +29,8 @@ idr-test: idr-test.o $(CORE_OFILES) xarray: $(CORE_OFILES) +maple: $(CORE_OFILES) + multiorder: multiorder.o $(CORE_OFILES) clean: @@ -40,6 +42,7 @@ $(OFILES): Makefile *.h */*.h generated/map-shift.h \ ../../include/linux/*.h \ ../../include/asm/*.h \ ../../../include/linux/xarray.h \ + ../../../include/linux/maple_tree.h \ ../../../include/linux/radix-tree.h \ ../../../include/linux/idr.h @@ -51,6 +54,8 @@ idr.c: ../../../lib/idr.c xarray.o: ../../../lib/xarray.c ../../../lib/test_xarray.c +maple.o: ../../../lib/maple_tree.c ../../../lib/test_maple_tree.c + generated/map-shift.h: @if ! grep -qws $(SHIFT) generated/map-shift.h; then \ echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h index 2218b3cc184e419d7d27e0331f3ec393f589d07b..e7da803502362baaa87ca7e77ba53c7b71d16e95 100644 --- a/tools/testing/radix-tree/generated/autoconf.h +++ b/tools/testing/radix-tree/generated/autoconf.h @@ -1 +1,2 @@ #define CONFIG_XARRAY_MULTI 1 +#define CONFIG_64BIT 1 diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c index d5c1bcba86fe00ffa379a6d5bb38095c084f3ce2..2048d12c31df36da570fe323e7fadf4b8676768b 100644 --- a/tools/testing/radix-tree/linux.c +++ b/tools/testing/radix-tree/linux.c @@ -23,15 +23,47 @@ struct kmem_cache { int nr_objs; void *objs; void (*ctor)(void *); + unsigned int non_kernel; + unsigned long nr_allocated; + unsigned long nr_tallocated; }; +void kmem_cache_set_non_kernel(struct kmem_cache *cachep, unsigned int val) +{ + cachep->non_kernel = val; +} + +unsigned long kmem_cache_get_alloc(struct kmem_cache *cachep) +{ + return cachep->size * cachep->nr_allocated; +} + +unsigned long kmem_cache_nr_allocated(struct kmem_cache *cachep) +{ + return cachep->nr_allocated; +} + +unsigned long kmem_cache_nr_tallocated(struct kmem_cache *cachep) +{ + return cachep->nr_tallocated; +} + +void kmem_cache_zero_nr_tallocated(struct kmem_cache *cachep) +{ + cachep->nr_tallocated = 0; +} + void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, int gfp) { void *p; - if (!(gfp & __GFP_DIRECT_RECLAIM)) - return NULL; + if (!(gfp & __GFP_DIRECT_RECLAIM)) { + if (!cachep->non_kernel) + return NULL; + + cachep->non_kernel--; + } pthread_mutex_lock(&cachep->lock); if (cachep->nr_objs) { @@ -53,19 +85,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, memset(p, 0, cachep->size); } + uatomic_inc(&cachep->nr_allocated); uatomic_inc(&nr_allocated); + uatomic_inc(&cachep->nr_tallocated); if (kmalloc_verbose) printf("Allocating %p from slab\n", p); return p; } -void kmem_cache_free(struct kmem_cache *cachep, void *objp) +void kmem_cache_free_locked(struct kmem_cache *cachep, void *objp) { assert(objp); uatomic_dec(&nr_allocated); + uatomic_dec(&cachep->nr_allocated); if (kmalloc_verbose) printf("Freeing %p to slab\n", objp); - pthread_mutex_lock(&cachep->lock); if (cachep->nr_objs > 10 || cachep->align) { memset(objp, POISON_FREE, cachep->size); free(objp); @@ -75,9 +109,80 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) node->parent = cachep->objs; cachep->objs = node; } +} + +void kmem_cache_free(struct kmem_cache *cachep, void *objp) +{ + pthread_mutex_lock(&cachep->lock); + kmem_cache_free_locked(cachep, objp); pthread_mutex_unlock(&cachep->lock); } +void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) +{ + if (kmalloc_verbose) + pr_debug("Bulk free %p[0-%lu]\n", list, size - 1); + + pthread_mutex_lock(&cachep->lock); + for (int i = 0; i < size; i++) + kmem_cache_free_locked(cachep, list[i]); + pthread_mutex_unlock(&cachep->lock); +} + +int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, + void **p) +{ + size_t i; + + if (kmalloc_verbose) + pr_debug("Bulk alloc %lu\n", size); + + if (!(gfp & __GFP_DIRECT_RECLAIM)) { + if (cachep->non_kernel < size) + return 0; + + cachep->non_kernel -= size; + } + + pthread_mutex_lock(&cachep->lock); + if (cachep->nr_objs >= size) { + struct radix_tree_node *node; + + for (i = 0; i < size; i++) { + node = cachep->objs; + cachep->nr_objs--; + cachep->objs = node->parent; + p[i] = node; + node->parent = NULL; + } + pthread_mutex_unlock(&cachep->lock); + } else { + pthread_mutex_unlock(&cachep->lock); + for (i = 0; i < size; i++) { + if (cachep->align) { + posix_memalign(&p[i], cachep->align, + cachep->size * size); + } else { + p[i] = malloc(cachep->size * size); + } + if (cachep->ctor) + cachep->ctor(p[i]); + else if (gfp & __GFP_ZERO) + memset(p[i], 0, cachep->size); + } + } + + for (i = 0; i < size; i++) { + uatomic_inc(&nr_allocated); + uatomic_inc(&cachep->nr_allocated); + uatomic_inc(&cachep->nr_tallocated); + if (kmalloc_verbose) + printf("Allocating %p from slab\n", p[i]); + } + + return size; +} + struct kmem_cache * kmem_cache_create(const char *name, unsigned int size, unsigned int align, unsigned int flags, void (*ctor)(void *)) @@ -88,7 +193,54 @@ kmem_cache_create(const char *name, unsigned int size, unsigned int align, ret->size = size; ret->align = align; ret->nr_objs = 0; + ret->nr_allocated = 0; + ret->nr_tallocated = 0; ret->objs = NULL; ret->ctor = ctor; + ret->non_kernel = 0; return ret; } + +/* + * Test the test infrastructure for kem_cache_alloc/free and bulk counterparts. + */ +void test_kmem_cache_bulk(void) +{ + int i; + void *list[12]; + static struct kmem_cache *test_cache, *test_cache2; + + /* + * Testing the bulk allocators without aligned kmem_cache to force the + * bulk alloc/free to reuse + */ + test_cache = kmem_cache_create("test_cache", 256, 0, SLAB_PANIC, NULL); + + for (i = 0; i < 5; i++) + list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); + + for (i = 0; i < 5; i++) + kmem_cache_free(test_cache, list[i]); + assert(test_cache->nr_objs == 5); + + kmem_cache_alloc_bulk(test_cache, __GFP_DIRECT_RECLAIM, 5, list); + kmem_cache_free_bulk(test_cache, 5, list); + + for (i = 0; i < 12 ; i++) + list[i] = kmem_cache_alloc(test_cache, __GFP_DIRECT_RECLAIM); + + for (i = 0; i < 12; i++) + kmem_cache_free(test_cache, list[i]); + + /* The last free will not be kept around */ + assert(test_cache->nr_objs == 11); + + /* Aligned caches will immediately free */ + test_cache2 = kmem_cache_create("test_cache2", 128, 128, SLAB_PANIC, NULL); + + kmem_cache_alloc_bulk(test_cache2, __GFP_DIRECT_RECLAIM, 10, list); + kmem_cache_free_bulk(test_cache2, 10, list); + assert(!test_cache2->nr_objs); + + +} diff --git a/tools/testing/radix-tree/linux/kernel.h b/tools/testing/radix-tree/linux/kernel.h index 39867fd80c8faa7db47dd070f7f71d99faa58dd0..c5c9d05f29da95e4b7d0b462675fde9dcc028924 100644 --- a/tools/testing/radix-tree/linux/kernel.h +++ b/tools/testing/radix-tree/linux/kernel.h @@ -14,6 +14,7 @@ #include "../../../include/linux/kconfig.h" #define printk printf +#define pr_err printk #define pr_info printk #define pr_debug printk #define pr_cont printk diff --git a/tools/testing/radix-tree/linux/lockdep.h b/tools/testing/radix-tree/linux/lockdep.h index 016cff473cfc483963cd1b4586243fa3e73378e7..62473ab57f99c252237d9639a2155a30735a626f 100644 --- a/tools/testing/radix-tree/linux/lockdep.h +++ b/tools/testing/radix-tree/linux/lockdep.h @@ -11,4 +11,6 @@ static inline void lockdep_set_class(spinlock_t *lock, struct lock_class_key *key) { } + +extern int lockdep_is_held(const void *); #endif /* _LINUX_LOCKDEP_H */ diff --git a/tools/testing/radix-tree/linux/maple_tree.h b/tools/testing/radix-tree/linux/maple_tree.h new file mode 100644 index 0000000000000000000000000000000000000000..7d8d1f445b8998150e7950a1a68a8d5405fddd52 --- /dev/null +++ b/tools/testing/radix-tree/linux/maple_tree.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#define atomic_t int32_t +#include "../../../../include/linux/maple_tree.h" +#define atomic_inc(x) uatomic_inc(x) +#define atomic_read(x) uatomic_read(x) +#define atomic_set(x, y) do {} while (0) +#define U8_MAX UCHAR_MAX diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c new file mode 100644 index 0000000000000000000000000000000000000000..35082671928ad5305efd743d5955f4556f3bba45 --- /dev/null +++ b/tools/testing/radix-tree/maple.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * maple_tree.c: Userspace shim for maple tree test-suite + * Copyright (c) 2018 Liam R. Howlett <Liam.Howlett@Oracle.com> + */ + +#define CONFIG_DEBUG_MAPLE_TREE +#define CONFIG_MAPLE_SEARCH +#include "test.h" + +#define module_init(x) +#define module_exit(x) +#define MODULE_AUTHOR(x) +#define MODULE_LICENSE(x) +#define dump_stack() assert(0) + +#include "../../../lib/maple_tree.c" +#undef CONFIG_DEBUG_MAPLE_TREE +#include "../../../lib/test_maple_tree.c" + +void farmer_tests(void) +{ + struct maple_node *node; + DEFINE_MTREE(tree); + + mt_dump(&tree); + + tree.ma_root = xa_mk_value(0); + mt_dump(&tree); + + node = mt_alloc_one(GFP_KERNEL); + node->parent = (void *)((unsigned long)(&tree) | 1); + node->slot[0] = xa_mk_value(0); + node->slot[1] = xa_mk_value(1); + node->mr64.pivot[0] = 0; + node->mr64.pivot[1] = 1; + node->mr64.pivot[2] = 0; + tree.ma_root = mt_mk_node(node, maple_leaf_64); + mt_dump(&tree); + + ma_free_rcu(node); +} + +void maple_tree_tests(void) +{ + farmer_tests(); + maple_tree_seed(); + maple_tree_harvest(); +} + +int __weak main(void) +{ + maple_tree_init(); + maple_tree_tests(); + rcu_barrier(); + if (nr_allocated) + printf("nr_allocated = %d\n", nr_allocated); + return 0; +} diff --git a/tools/testing/radix-tree/trace/events/maple_tree.h b/tools/testing/radix-tree/trace/events/maple_tree.h new file mode 100644 index 0000000000000000000000000000000000000000..97d0e1ddcf08eac9a09381dcac2d464e1f91fe4a --- /dev/null +++ b/tools/testing/radix-tree/trace/events/maple_tree.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#define trace_ma_op(a, b) do {} while (0) +#define trace_ma_read(a, b) do {} while (0) +#define trace_ma_write(a, b, c, d) do {} while (0) diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 10b34bb03bc1b8af48c4c056e609154ce2b14a23..0464b2c6c1e4b20f89226d682e80d569eedabee1 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 TARGETS += alsa +TARGETS += amd-pstate TARGETS += arm64 TARGETS += bpf TARGETS += breakpoints @@ -12,6 +13,8 @@ TARGETS += cpu-hotplug TARGETS += damon TARGETS += drivers/dma-buf TARGETS += drivers/s390x/uvdevice +TARGETS += drivers/net/bonding +TARGETS += drivers/net/team TARGETS += efivarfs TARGETS += exec TARGETS += filesystems diff --git a/tools/testing/selftests/amd-pstate/Makefile b/tools/testing/selftests/amd-pstate/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..199867f44b3238449f2173562b3b7bd55260308c --- /dev/null +++ b/tools/testing/selftests/amd-pstate/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Makefile for amd-pstate/ function selftests + +# No binaries, but make sure arg-less "make" doesn't trigger "run_tests" +all: + +TEST_PROGS := amd-pstate-ut.sh + +include ../lib.mk diff --git a/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh b/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh new file mode 100755 index 0000000000000000000000000000000000000000..f8e82d91ffcf80fed1f638cc237f20debd8545ba --- /dev/null +++ b/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# amd-pstate-ut is a test module for testing the amd-pstate driver. +# It can only run on x86 architectures and current cpufreq driver +# must be amd-pstate. +# (1) It can help all users to verify their processor support +# (SBIOS/Firmware or Hardware). +# (2) Kernel can have a basic function test to avoid the kernel +# regression during the update. +# (3) We can introduce more functional or performance tests to align +# the result together, it will benefit power and performance scale optimization. + +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +# amd-pstate-ut only run on x86/x86_64 AMD systems. +ARCH=$(uname -m 2>/dev/null | sed -e 's/i.86/x86/' -e 's/x86_64/x86/') +VENDOR=$(cat /proc/cpuinfo | grep -m 1 'vendor_id' | awk '{print $NF}') + +if ! echo "$ARCH" | grep -q x86; then + echo "$0 # Skipped: Test can only run on x86 architectures." + exit $ksft_skip +fi + +if ! echo "$VENDOR" | grep -iq amd; then + echo "$0 # Skipped: Test can only run on AMD CPU." + echo "$0 # Current cpu vendor is $VENDOR." + exit $ksft_skip +fi + +scaling_driver=$(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_driver) +if [ "$scaling_driver" != "amd-pstate" ]; then + echo "$0 # Skipped: Test can only run on amd-pstate driver." + echo "$0 # Please set X86_AMD_PSTATE enabled." + echo "$0 # Current cpufreq scaling drvier is $scaling_driver." + exit $ksft_skip +fi + +msg="Skip all tests:" +if [ ! -w /dev ]; then + echo $msg please run this as root >&2 + exit $ksft_skip +fi + +if ! /sbin/modprobe -q -n amd-pstate-ut; then + echo "amd-pstate-ut: module amd-pstate-ut is not found [SKIP]" + exit $ksft_skip +fi +if /sbin/modprobe -q amd-pstate-ut; then + /sbin/modprobe -q -r amd-pstate-ut + echo "amd-pstate-ut: ok" +else + echo "amd-pstate-ut: [FAIL]" + exit 1 +fi diff --git a/tools/testing/selftests/amd-pstate/config b/tools/testing/selftests/amd-pstate/config new file mode 100644 index 0000000000000000000000000000000000000000..f43103c9adc49f50a24764f25faa62159af64a4f --- /dev/null +++ b/tools/testing/selftests/amd-pstate/config @@ -0,0 +1 @@ +CONFIG_X86_AMD_PSTATE_UT=m diff --git a/tools/testing/selftests/arm64/abi/.gitignore b/tools/testing/selftests/arm64/abi/.gitignore index b9e54417250d811cba6b0183b4ab683856f5290b..44f8b80f37e36a775ffe5b3fe0b119b1b4bdf965 100644 --- a/tools/testing/selftests/arm64/abi/.gitignore +++ b/tools/testing/selftests/arm64/abi/.gitignore @@ -1,2 +1,4 @@ +hwcap +ptrace syscall-abi tpidr2 diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile index c8d7f2495eb21c3fbcc348701a3e2ec4f14b19f7..a6d30c620908424c2a49928455ec8e6173ad82a6 100644 --- a/tools/testing/selftests/arm64/abi/Makefile +++ b/tools/testing/selftests/arm64/abi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 ARM Limited -TEST_GEN_PROGS := syscall-abi tpidr2 +TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 include ../../lib.mk diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c new file mode 100644 index 0000000000000000000000000000000000000000..9f1a7b5c61933da7c458d1fd0e9c6b1864645244 --- /dev/null +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 ARM Limited. + */ + +#include <errno.h> +#include <signal.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/auxv.h> +#include <sys/prctl.h> +#include <asm/hwcap.h> +#include <asm/sigcontext.h> +#include <asm/unistd.h> + +#include "../../kselftest.h" + +#define TESTS_PER_HWCAP 2 + +/* + * Function expected to generate SIGILL when the feature is not + * supported and return when it is supported. If SIGILL is generated + * then the handler must be able to skip over the instruction safely. + * + * Note that it is expected that for many architecture extensions + * there are no specific traps due to no architecture state being + * added so we may not fault if running on a kernel which doesn't know + * to add the hwcap. + */ +typedef void (*sigill_fn)(void); + +static void rng_sigill(void) +{ + asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0"); +} + +static void sme_sigill(void) +{ + /* RDSVL x0, #0 */ + asm volatile(".inst 0x04bf5800" : : : "x0"); +} + +static void sve_sigill(void) +{ + /* RDVL x0, #0 */ + asm volatile(".inst 0x04bf5000" : : : "x0"); +} + +static void sve2_sigill(void) +{ + /* SQABS Z0.b, P0/M, Z0.B */ + asm volatile(".inst 0x4408A000" : : : "z0"); +} + +static void sveaes_sigill(void) +{ + /* AESD z0.b, z0.b, z0.b */ + asm volatile(".inst 0x4522e400" : : : "z0"); +} + +static void svepmull_sigill(void) +{ + /* PMULLB Z0.Q, Z0.D, Z0.D */ + asm volatile(".inst 0x45006800" : : : "z0"); +} + +static void svebitperm_sigill(void) +{ + /* BDEP Z0.B, Z0.B, Z0.B */ + asm volatile(".inst 0x4500b400" : : : "z0"); +} + +static void svesha3_sigill(void) +{ + /* EOR3 Z0.D, Z0.D, Z0.D, Z0.D */ + asm volatile(".inst 0x4203800" : : : "z0"); +} + +static void svesm4_sigill(void) +{ + /* SM4E Z0.S, Z0.S, Z0.S */ + asm volatile(".inst 0x4523e000" : : : "z0"); +} + +static void svei8mm_sigill(void) +{ + /* USDOT Z0.S, Z0.B, Z0.B[0] */ + asm volatile(".inst 0x44a01800" : : : "z0"); +} + +static void svef32mm_sigill(void) +{ + /* FMMLA Z0.S, Z0.S, Z0.S */ + asm volatile(".inst 0x64a0e400" : : : "z0"); +} + +static void svef64mm_sigill(void) +{ + /* FMMLA Z0.D, Z0.D, Z0.D */ + asm volatile(".inst 0x64e0e400" : : : "z0"); +} + +static void svebf16_sigill(void) +{ + /* BFCVT Z0.H, P0/M, Z0.S */ + asm volatile(".inst 0x658aa000" : : : "z0"); +} + +static const struct hwcap_data { + const char *name; + unsigned long at_hwcap; + unsigned long hwcap_bit; + const char *cpuinfo; + sigill_fn sigill_fn; + bool sigill_reliable; +} hwcaps[] = { + { + .name = "RNG", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_RNG, + .cpuinfo = "rng", + .sigill_fn = rng_sigill, + }, + { + .name = "SME", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SME, + .cpuinfo = "sme", + .sigill_fn = sme_sigill, + .sigill_reliable = true, + }, + { + .name = "SVE", + .at_hwcap = AT_HWCAP, + .hwcap_bit = HWCAP_SVE, + .cpuinfo = "sve", + .sigill_fn = sve_sigill, + .sigill_reliable = true, + }, + { + .name = "SVE 2", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVE2, + .cpuinfo = "sve2", + .sigill_fn = sve2_sigill, + }, + { + .name = "SVE AES", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEAES, + .cpuinfo = "sveaes", + .sigill_fn = sveaes_sigill, + }, + { + .name = "SVE2 PMULL", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEPMULL, + .cpuinfo = "svepmull", + .sigill_fn = svepmull_sigill, + }, + { + .name = "SVE2 BITPERM", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEBITPERM, + .cpuinfo = "svebitperm", + .sigill_fn = svebitperm_sigill, + }, + { + .name = "SVE2 SHA3", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVESHA3, + .cpuinfo = "svesha3", + .sigill_fn = svesha3_sigill, + }, + { + .name = "SVE2 SM4", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVESM4, + .cpuinfo = "svesm4", + .sigill_fn = svesm4_sigill, + }, + { + .name = "SVE2 I8MM", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEI8MM, + .cpuinfo = "svei8mm", + .sigill_fn = svei8mm_sigill, + }, + { + .name = "SVE2 F32MM", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEF32MM, + .cpuinfo = "svef32mm", + .sigill_fn = svef32mm_sigill, + }, + { + .name = "SVE2 F64MM", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEF64MM, + .cpuinfo = "svef64mm", + .sigill_fn = svef64mm_sigill, + }, + { + .name = "SVE2 BF16", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVEBF16, + .cpuinfo = "svebf16", + .sigill_fn = svebf16_sigill, + }, + { + .name = "SVE2 EBF16", + .at_hwcap = AT_HWCAP2, + .hwcap_bit = HWCAP2_SVE_EBF16, + .cpuinfo = "sveebf16", + }, +}; + +static bool seen_sigill; + +static void handle_sigill(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + seen_sigill = true; + + /* Skip over the offending instruction */ + uc->uc_mcontext.pc += 4; +} + +bool cpuinfo_present(const char *name) +{ + FILE *f; + char buf[2048], name_space[30], name_newline[30]; + char *s; + + /* + * The feature should appear with a leading space and either a + * trailing space or a newline. + */ + snprintf(name_space, sizeof(name_space), " %s ", name); + snprintf(name_newline, sizeof(name_newline), " %s\n", name); + + f = fopen("/proc/cpuinfo", "r"); + if (!f) { + ksft_print_msg("Failed to open /proc/cpuinfo\n"); + return false; + } + + while (fgets(buf, sizeof(buf), f)) { + /* Features: line? */ + if (strncmp(buf, "Features\t:", strlen("Features\t:")) != 0) + continue; + + /* All CPUs should be symmetric, don't read any more */ + fclose(f); + + s = strstr(buf, name_space); + if (s) + return true; + s = strstr(buf, name_newline); + if (s) + return true; + + return false; + } + + ksft_print_msg("Failed to find Features in /proc/cpuinfo\n"); + fclose(f); + return false; +} + +int main(void) +{ + const struct hwcap_data *hwcap; + int i, ret; + bool have_cpuinfo, have_hwcap; + struct sigaction sa; + + ksft_print_header(); + ksft_set_plan(ARRAY_SIZE(hwcaps) * TESTS_PER_HWCAP); + + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handle_sigill; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + sigemptyset(&sa.sa_mask); + ret = sigaction(SIGILL, &sa, NULL); + if (ret < 0) + ksft_exit_fail_msg("Failed to install SIGILL handler: %s (%d)\n", + strerror(errno), errno); + + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + hwcap = &hwcaps[i]; + + have_hwcap = getauxval(hwcap->at_hwcap) & hwcap->hwcap_bit; + have_cpuinfo = cpuinfo_present(hwcap->cpuinfo); + + if (have_hwcap) + ksft_print_msg("%s present\n", hwcap->name); + + ksft_test_result(have_hwcap == have_cpuinfo, + "cpuinfo_match_%s\n", hwcap->name); + + if (hwcap->sigill_fn) { + seen_sigill = false; + hwcap->sigill_fn(); + + if (have_hwcap) { + /* Should be able to use the extension */ + ksft_test_result(!seen_sigill, "sigill_%s\n", + hwcap->name); + } else if (hwcap->sigill_reliable) { + /* Guaranteed a SIGILL */ + ksft_test_result(seen_sigill, "sigill_%s\n", + hwcap->name); + } else { + /* Missing SIGILL might be fine */ + ksft_print_msg("SIGILL %sreported for %s\n", + seen_sigill ? "" : "not ", + hwcap->name); + ksft_test_result_skip("sigill_%s\n", + hwcap->name); + } + } else { + ksft_test_result_skip("sigill_%s\n", + hwcap->name); + } + } + + ksft_print_cnts(); + + return 0; +} diff --git a/tools/testing/selftests/arm64/abi/ptrace.c b/tools/testing/selftests/arm64/abi/ptrace.c new file mode 100644 index 0000000000000000000000000000000000000000..be952511af221121645177ce6868b40ccb51b644 --- /dev/null +++ b/tools/testing/selftests/arm64/abi/ptrace.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 ARM Limited. + */ +#include <errno.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/auxv.h> +#include <sys/prctl.h> +#include <sys/ptrace.h> +#include <sys/types.h> +#include <sys/uio.h> +#include <sys/wait.h> +#include <asm/sigcontext.h> +#include <asm/ptrace.h> + +#include "../../kselftest.h" + +#define EXPECTED_TESTS 7 + +#define MAX_TPIDRS 2 + +static bool have_sme(void) +{ + return getauxval(AT_HWCAP2) & HWCAP2_SME; +} + +static void test_tpidr(pid_t child) +{ + uint64_t read_val[MAX_TPIDRS]; + uint64_t write_val[MAX_TPIDRS]; + struct iovec read_iov, write_iov; + bool test_tpidr2 = false; + int ret, i; + + read_iov.iov_base = read_val; + write_iov.iov_base = write_val; + + /* Should be able to read a single TPIDR... */ + read_iov.iov_len = sizeof(uint64_t); + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov); + ksft_test_result(ret == 0, "read_tpidr_one\n"); + + /* ...write a new value.. */ + write_iov.iov_len = sizeof(uint64_t); + write_val[0] = read_val[0]++; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov); + ksft_test_result(ret == 0, "write_tpidr_one\n"); + + /* ...then read it back */ + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov); + ksft_test_result(ret == 0 && write_val[0] == read_val[0], + "verify_tpidr_one\n"); + + /* If we have TPIDR2 we should be able to read it */ + read_iov.iov_len = sizeof(read_val); + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov); + if (ret == 0) { + /* If we have SME there should be two TPIDRs */ + if (read_iov.iov_len >= sizeof(read_val)) + test_tpidr2 = true; + + if (have_sme() && test_tpidr2) { + ksft_test_result(test_tpidr2, "count_tpidrs\n"); + } else { + ksft_test_result(read_iov.iov_len % sizeof(uint64_t) == 0, + "count_tpidrs\n"); + } + } else { + ksft_test_result_fail("count_tpidrs\n"); + } + + if (test_tpidr2) { + /* Try to write new values to all known TPIDRs... */ + write_iov.iov_len = sizeof(write_val); + for (i = 0; i < MAX_TPIDRS; i++) + write_val[i] = read_val[i] + 1; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov); + + ksft_test_result(ret == 0 && + write_iov.iov_len == sizeof(write_val), + "tpidr2_write\n"); + + /* ...then read them back */ + read_iov.iov_len = sizeof(read_val); + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov); + + if (have_sme()) { + /* Should read back the written value */ + ksft_test_result(ret == 0 && + read_iov.iov_len >= sizeof(read_val) && + memcmp(read_val, write_val, + sizeof(read_val)) == 0, + "tpidr2_read\n"); + } else { + /* TPIDR2 should read as zero */ + ksft_test_result(ret == 0 && + read_iov.iov_len >= sizeof(read_val) && + read_val[0] == write_val[0] && + read_val[1] == 0, + "tpidr2_read\n"); + } + + /* Writing only TPIDR... */ + write_iov.iov_len = sizeof(uint64_t); + memcpy(write_val, read_val, sizeof(read_val)); + write_val[0] += 1; + ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov); + + if (ret == 0) { + /* ...should leave TPIDR2 untouched */ + read_iov.iov_len = sizeof(read_val); + ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, + &read_iov); + + ksft_test_result(ret == 0 && + read_iov.iov_len >= sizeof(read_val) && + memcmp(read_val, write_val, + sizeof(read_val)) == 0, + "write_tpidr_only\n"); + } else { + ksft_test_result_fail("write_tpidr_only\n"); + } + } else { + ksft_test_result_skip("tpidr2_write\n"); + ksft_test_result_skip("tpidr2_read\n"); + ksft_test_result_skip("write_tpidr_only\n"); + } +} + +static int do_child(void) +{ + if (ptrace(PTRACE_TRACEME, -1, NULL, NULL)) + ksft_exit_fail_msg("PTRACE_TRACEME", strerror(errno)); + + if (raise(SIGSTOP)) + ksft_exit_fail_msg("raise(SIGSTOP)", strerror(errno)); + + return EXIT_SUCCESS; +} + +static int do_parent(pid_t child) +{ + int ret = EXIT_FAILURE; + pid_t pid; + int status; + siginfo_t si; + + /* Attach to the child */ + while (1) { + int sig; + + pid = wait(&status); + if (pid == -1) { + perror("wait"); + goto error; + } + + /* + * This should never happen but it's hard to flag in + * the framework. + */ + if (pid != child) + continue; + + if (WIFEXITED(status) || WIFSIGNALED(status)) + ksft_exit_fail_msg("Child died unexpectedly\n"); + + if (!WIFSTOPPED(status)) + goto error; + + sig = WSTOPSIG(status); + + if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &si)) { + if (errno == ESRCH) + goto disappeared; + + if (errno == EINVAL) { + sig = 0; /* bust group-stop */ + goto cont; + } + + ksft_test_result_fail("PTRACE_GETSIGINFO: %s\n", + strerror(errno)); + goto error; + } + + if (sig == SIGSTOP && si.si_code == SI_TKILL && + si.si_pid == pid) + break; + + cont: + if (ptrace(PTRACE_CONT, pid, NULL, sig)) { + if (errno == ESRCH) + goto disappeared; + + ksft_test_result_fail("PTRACE_CONT: %s\n", + strerror(errno)); + goto error; + } + } + + ksft_print_msg("Parent is %d, child is %d\n", getpid(), child); + + test_tpidr(child); + + ret = EXIT_SUCCESS; + +error: + kill(child, SIGKILL); + +disappeared: + return ret; +} + +int main(void) +{ + int ret = EXIT_SUCCESS; + pid_t child; + + srandom(getpid()); + + ksft_print_header(); + + ksft_set_plan(EXPECTED_TESTS); + + child = fork(); + if (!child) + return do_child(); + + if (do_parent(child)) + ret = EXIT_FAILURE; + + ksft_print_cnts(); + + return ret; +} diff --git a/tools/testing/selftests/arm64/abi/syscall-abi.c b/tools/testing/selftests/arm64/abi/syscall-abi.c index b632bfe9e0227c053bc929df79b9819071ac6454..dd7ebe536d05faec4cc7053ee10c9e39bf31e094 100644 --- a/tools/testing/selftests/arm64/abi/syscall-abi.c +++ b/tools/testing/selftests/arm64/abi/syscall-abi.c @@ -112,9 +112,11 @@ static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, return errors; } +#define SVE_Z_SHARED_BYTES (128 / 8) + static uint8_t z_zero[__SVE_ZREG_SIZE(SVE_VQ_MAX)]; -uint8_t z_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; -uint8_t z_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; +uint8_t z_in[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; +uint8_t z_out[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)]; static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, uint64_t svcr) @@ -133,22 +135,39 @@ static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl, if (!sve_vl) return 0; - /* - * After a syscall the low 128 bits of the Z registers should - * be preserved and the rest be zeroed or preserved, except if - * we were in streaming mode in which case the low 128 bits may - * also be cleared by the transition out of streaming mode. - */ for (i = 0; i < SVE_NUM_ZREGS; i++) { - void *in = &z_in[reg_size * i]; - void *out = &z_out[reg_size * i]; - - if ((memcmp(in, out, SVE_VQ_BYTES) != 0) && - !((svcr & SVCR_SM_MASK) && - memcmp(z_zero, out, SVE_VQ_BYTES) == 0)) { - ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n", - cfg->name, sve_vl, i); - errors++; + uint8_t *in = &z_in[reg_size * i]; + uint8_t *out = &z_out[reg_size * i]; + + if (svcr & SVCR_SM_MASK) { + /* + * In streaming mode the whole register should + * be cleared by the transition out of + * streaming mode. + */ + if (memcmp(z_zero, out, reg_size) != 0) { + ksft_print_msg("%s SVE VL %d Z%d non-zero\n", + cfg->name, sve_vl, i); + errors++; + } + } else { + /* + * For standard SVE the low 128 bits should be + * preserved and any additional bits cleared. + */ + if (memcmp(in, out, SVE_Z_SHARED_BYTES) != 0) { + ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n", + cfg->name, sve_vl, i); + errors++; + } + + if (reg_size > SVE_Z_SHARED_BYTES && + (memcmp(z_zero, out + SVE_Z_SHARED_BYTES, + reg_size - SVE_Z_SHARED_BYTES) != 0)) { + ksft_print_msg("%s SVE VL %d Z%d high bits non-zero\n", + cfg->name, sve_vl, i); + errors++; + } } } @@ -176,9 +195,9 @@ static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl, if (!sve_vl) return 0; - /* After a syscall the P registers should be preserved or zeroed */ + /* After a syscall the P registers should be zeroed */ for (i = 0; i < SVE_NUM_PREGS * reg_size; i++) - if (p_out[i] && (p_in[i] != p_out[i])) + if (p_out[i]) errors++; if (errors) ksft_print_msg("%s SVE VL %d predicate registers non-zero\n", @@ -226,9 +245,9 @@ static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl, !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64)) return 0; - /* After a syscall the P registers should be preserved or zeroed */ + /* After a syscall FFR should be zeroed */ for (i = 0; i < reg_size; i++) - if (ffr_out[i] && (ffr_in[i] != ffr_out[i])) + if (ffr_out[i]) errors++; if (errors) ksft_print_msg("%s SVE VL %d FFR non-zero\n", diff --git a/tools/testing/selftests/arm64/fp/.gitignore b/tools/testing/selftests/arm64/fp/.gitignore index ea947af6388243c31504eb5f2468580fd1573bf8..df79d29664a1bb8e7ed39c553aa4eba7351cffc6 100644 --- a/tools/testing/selftests/arm64/fp/.gitignore +++ b/tools/testing/selftests/arm64/fp/.gitignore @@ -1,4 +1,5 @@ fp-pidbench +fp-stress fpsimd-test rdvl-sme rdvl-sve diff --git a/tools/testing/selftests/arm64/fp/Makefile b/tools/testing/selftests/arm64/fp/Makefile index a7c2286bf65ba96ac96f43792dab3c44fbb2ca57..36db61358ed5bc77fa5dc808c36df3b50d348208 100644 --- a/tools/testing/selftests/arm64/fp/Makefile +++ b/tools/testing/selftests/arm64/fp/Makefile @@ -5,7 +5,10 @@ top_srcdir = $(realpath ../../../../../) CFLAGS += -I$(top_srcdir)/usr/include/ -TEST_GEN_PROGS := sve-ptrace sve-probe-vls vec-syscfg za-fork za-ptrace +TEST_GEN_PROGS := fp-stress \ + sve-ptrace sve-probe-vls \ + vec-syscfg \ + za-fork za-ptrace TEST_GEN_PROGS_EXTENDED := fp-pidbench fpsimd-test \ rdvl-sme rdvl-sve \ sve-test \ diff --git a/tools/testing/selftests/arm64/fp/asm-offsets.h b/tools/testing/selftests/arm64/fp/asm-offsets.h index a180851496ecf2c702ae8379a69c6f32446f1fd8..757b2fd75dd78c2655ad730bdf8b9540392b56b6 100644 --- a/tools/testing/selftests/arm64/fp/asm-offsets.h +++ b/tools/testing/selftests/arm64/fp/asm-offsets.h @@ -3,6 +3,7 @@ #define sa_handler 0 #define sa_mask_sz 8 #define SIGUSR1 10 +#define SIGUSR2 12 #define SIGTERM 15 #define SIGINT 2 #define SIGABRT 6 diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c new file mode 100644 index 0000000000000000000000000000000000000000..4e62a9199f97ad446b59fc44592e9f7d65029145 --- /dev/null +++ b/tools/testing/selftests/arm64/fp/fp-stress.c @@ -0,0 +1,555 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 ARM Limited. + */ + +#define _GNU_SOURCE +#define _POSIX_C_SOURCE 199309L + +#include <errno.h> +#include <getopt.h> +#include <poll.h> +#include <signal.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <sys/auxv.h> +#include <sys/epoll.h> +#include <sys/prctl.h> +#include <sys/types.h> +#include <sys/uio.h> +#include <sys/wait.h> +#include <asm/hwcap.h> + +#include "../../kselftest.h" + +#define MAX_VLS 16 + +struct child_data { + char *name, *output; + pid_t pid; + int stdout; + bool output_seen; + bool exited; + int exit_status; +}; + +static int epoll_fd; +static struct child_data *children; +static int num_children; +static bool terminate; + +static void drain_output(bool flush); + +static int num_processors(void) +{ + long nproc = sysconf(_SC_NPROCESSORS_CONF); + if (nproc < 0) { + perror("Unable to read number of processors\n"); + exit(EXIT_FAILURE); + } + + return nproc; +} + +static void child_start(struct child_data *child, const char *program) +{ + int ret, pipefd[2], i; + struct epoll_event ev; + + ret = pipe(pipefd); + if (ret != 0) + ksft_exit_fail_msg("Failed to create stdout pipe: %s (%d)\n", + strerror(errno), errno); + + child->pid = fork(); + if (child->pid == -1) + ksft_exit_fail_msg("fork() failed: %s (%d)\n", + strerror(errno), errno); + + if (!child->pid) { + /* + * In child, replace stdout with the pipe, errors to + * stderr from here as kselftest prints to stdout. + */ + ret = dup2(pipefd[1], 1); + if (ret == -1) { + fprintf(stderr, "dup2() %d\n", errno); + exit(EXIT_FAILURE); + } + + /* + * Very dumb mechanism to clean open FDs other than + * stdio. We don't want O_CLOEXEC for the pipes... + */ + for (i = 3; i < 8192; i++) + close(i); + + ret = execl(program, program, NULL); + fprintf(stderr, "execl(%s) failed: %d (%s)\n", + program, errno, strerror(errno)); + + exit(EXIT_FAILURE); + } else { + /* + * In parent, remember the child and close our copy of the + * write side of stdout. + */ + close(pipefd[1]); + child->stdout = pipefd[0]; + child->output = NULL; + child->exited = false; + child->output_seen = false; + + ev.events = EPOLLIN | EPOLLHUP; + ev.data.ptr = child; + + ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, child->stdout, &ev); + if (ret < 0) { + ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n", + child->name, strerror(errno), errno); + } + + /* + * Keep output flowing during child startup so logs + * are more timely, can help debugging. + */ + drain_output(false); + } +} + +static bool child_output_read(struct child_data *child) +{ + char read_data[1024]; + char work[1024]; + int ret, len, cur_work, cur_read; + + ret = read(child->stdout, read_data, sizeof(read_data)); + if (ret < 0) { + if (errno == EINTR) + return true; + + ksft_print_msg("%s: read() failed: %s (%d)\n", + child->name, strerror(errno), + errno); + return false; + } + len = ret; + + child->output_seen = true; + + /* Pick up any partial read */ + if (child->output) { + strncpy(work, child->output, sizeof(work) - 1); + cur_work = strnlen(work, sizeof(work)); + free(child->output); + child->output = NULL; + } else { + cur_work = 0; + } + + cur_read = 0; + while (cur_read < len) { + work[cur_work] = read_data[cur_read++]; + + if (work[cur_work] == '\n') { + work[cur_work] = '\0'; + ksft_print_msg("%s: %s\n", child->name, work); + cur_work = 0; + } else { + cur_work++; + } + } + + if (cur_work) { + work[cur_work] = '\0'; + ret = asprintf(&child->output, "%s", work); + if (ret == -1) + ksft_exit_fail_msg("Out of memory\n"); + } + + return false; +} + +static void child_output(struct child_data *child, uint32_t events, + bool flush) +{ + bool read_more; + + if (events & EPOLLIN) { + do { + read_more = child_output_read(child); + } while (read_more); + } + + if (events & EPOLLHUP) { + close(child->stdout); + child->stdout = -1; + flush = true; + } + + if (flush && child->output) { + ksft_print_msg("%s: %s<EOF>\n", child->name, child->output); + free(child->output); + child->output = NULL; + } +} + +static void child_tickle(struct child_data *child) +{ + if (child->output_seen && !child->exited) + kill(child->pid, SIGUSR2); +} + +static void child_stop(struct child_data *child) +{ + if (!child->exited) + kill(child->pid, SIGTERM); +} + +static void child_cleanup(struct child_data *child) +{ + pid_t ret; + int status; + bool fail = false; + + if (!child->exited) { + do { + ret = waitpid(child->pid, &status, 0); + if (ret == -1 && errno == EINTR) + continue; + + if (ret == -1) { + ksft_print_msg("waitpid(%d) failed: %s (%d)\n", + child->pid, strerror(errno), + errno); + fail = true; + break; + } + } while (!WIFEXITED(status)); + child->exit_status = WEXITSTATUS(status); + } + + if (!child->output_seen) { + ksft_print_msg("%s no output seen\n", child->name); + fail = true; + } + + if (child->exit_status != 0) { + ksft_print_msg("%s exited with error code %d\n", + child->name, child->exit_status); + fail = true; + } + + ksft_test_result(!fail, "%s\n", child->name); +} + +static void handle_child_signal(int sig, siginfo_t *info, void *context) +{ + int i; + bool found = false; + + for (i = 0; i < num_children; i++) { + if (children[i].pid == info->si_pid) { + children[i].exited = true; + children[i].exit_status = info->si_status; + found = true; + break; + } + } + + if (!found) + ksft_print_msg("SIGCHLD for unknown PID %d with status %d\n", + info->si_pid, info->si_status); +} + +static void handle_exit_signal(int sig, siginfo_t *info, void *context) +{ + int i; + + /* If we're already exiting then don't signal again */ + if (terminate) + return; + + ksft_print_msg("Got signal, exiting...\n"); + + terminate = true; + + /* + * This should be redundant, the main loop should clean up + * after us, but for safety stop everything we can here. + */ + for (i = 0; i < num_children; i++) + child_stop(&children[i]); +} + +static void start_fpsimd(struct child_data *child, int cpu, int copy) +{ + int ret; + + child_start(child, "./fpsimd-test"); + + ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy); + if (ret == -1) + ksft_exit_fail_msg("asprintf() failed\n"); + + ksft_print_msg("Started %s\n", child->name); +} + +static void start_sve(struct child_data *child, int vl, int cpu) +{ + int ret; + + ret = prctl(PR_SVE_SET_VL, vl | PR_SVE_VL_INHERIT); + if (ret < 0) + ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl); + + child_start(child, "./sve-test"); + + ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu); + if (ret == -1) + ksft_exit_fail_msg("asprintf() failed\n"); + + ksft_print_msg("Started %s\n", child->name); +} + +static void start_ssve(struct child_data *child, int vl, int cpu) +{ + int ret; + + ret = prctl(PR_SME_SET_VL, vl | PR_SME_VL_INHERIT); + if (ret < 0) + ksft_exit_fail_msg("Failed to set SME VL %d\n", ret); + + child_start(child, "./ssve-test"); + + ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu); + if (ret == -1) + ksft_exit_fail_msg("asprintf() failed\n"); + + ksft_print_msg("Started %s\n", child->name); +} + +static void start_za(struct child_data *child, int vl, int cpu) +{ + int ret; + + ret = prctl(PR_SME_SET_VL, vl | PR_SVE_VL_INHERIT); + if (ret < 0) + ksft_exit_fail_msg("Failed to set SME VL %d\n", ret); + + child_start(child, "./za-test"); + + ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu); + if (ret == -1) + ksft_exit_fail_msg("asprintf() failed\n"); + + ksft_print_msg("Started %s\n", child->name); +} + +static void probe_vls(int vls[], int *vl_count, int set_vl) +{ + unsigned int vq; + int vl; + + *vl_count = 0; + + for (vq = SVE_VQ_MAX; vq > 0; --vq) { + vl = prctl(set_vl, vq * 16); + if (vl == -1) + ksft_exit_fail_msg("SET_VL failed: %s (%d)\n", + strerror(errno), errno); + + vl &= PR_SVE_VL_LEN_MASK; + + vq = sve_vq_from_vl(vl); + + vls[*vl_count] = vl; + *vl_count += 1; + } +} + +/* Handle any pending output without blocking */ +static void drain_output(bool flush) +{ + struct epoll_event ev; + int ret = 1; + + while (ret > 0) { + ret = epoll_wait(epoll_fd, &ev, 1, 0); + if (ret < 0) { + if (errno == EINTR) + continue; + ksft_print_msg("epoll_wait() failed: %s (%d)\n", + strerror(errno), errno); + } + + if (ret == 1) + child_output(ev.data.ptr, ev.events, flush); + } +} + +static const struct option options[] = { + { "timeout", required_argument, NULL, 't' }, + { } +}; + +int main(int argc, char **argv) +{ + int ret; + int timeout = 10; + int cpus, tests, i, j, c; + int sve_vl_count, sme_vl_count, fpsimd_per_cpu; + int sve_vls[MAX_VLS], sme_vls[MAX_VLS]; + struct epoll_event ev; + struct sigaction sa; + + while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) { + switch (c) { + case 't': + ret = sscanf(optarg, "%d", &timeout); + if (ret != 1) + ksft_exit_fail_msg("Failed to parse timeout %s\n", + optarg); + break; + default: + ksft_exit_fail_msg("Unknown argument\n"); + } + } + + cpus = num_processors(); + tests = 0; + + if (getauxval(AT_HWCAP) & HWCAP_SVE) { + probe_vls(sve_vls, &sve_vl_count, PR_SVE_SET_VL); + tests += sve_vl_count * cpus; + } else { + sve_vl_count = 0; + } + + if (getauxval(AT_HWCAP2) & HWCAP2_SME) { + probe_vls(sme_vls, &sme_vl_count, PR_SME_SET_VL); + tests += sme_vl_count * cpus * 2; + } else { + sme_vl_count = 0; + } + + /* Force context switching if we only have FPSIMD */ + if (!sve_vl_count && !sme_vl_count) + fpsimd_per_cpu = 2; + else + fpsimd_per_cpu = 1; + tests += cpus * fpsimd_per_cpu; + + ksft_print_header(); + ksft_set_plan(tests); + + ksft_print_msg("%d CPUs, %d SVE VLs, %d SME VLs\n", + cpus, sve_vl_count, sme_vl_count); + + if (timeout > 0) + ksft_print_msg("Will run for %ds\n", timeout); + else + ksft_print_msg("Will run until terminated\n"); + + children = calloc(sizeof(*children), tests); + if (!children) + ksft_exit_fail_msg("Unable to allocate child data\n"); + + ret = epoll_create1(EPOLL_CLOEXEC); + if (ret < 0) + ksft_exit_fail_msg("epoll_create1() failed: %s (%d)\n", + strerror(errno), ret); + epoll_fd = ret; + + /* Get signal handers ready before we start any children */ + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = handle_exit_signal; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + sigemptyset(&sa.sa_mask); + ret = sigaction(SIGINT, &sa, NULL); + if (ret < 0) + ksft_print_msg("Failed to install SIGINT handler: %s (%d)\n", + strerror(errno), errno); + ret = sigaction(SIGTERM, &sa, NULL); + if (ret < 0) + ksft_print_msg("Failed to install SIGTERM handler: %s (%d)\n", + strerror(errno), errno); + sa.sa_sigaction = handle_child_signal; + ret = sigaction(SIGCHLD, &sa, NULL); + if (ret < 0) + ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n", + strerror(errno), errno); + + for (i = 0; i < cpus; i++) { + for (j = 0; j < fpsimd_per_cpu; j++) + start_fpsimd(&children[num_children++], i, j); + + for (j = 0; j < sve_vl_count; j++) + start_sve(&children[num_children++], sve_vls[j], i); + + for (j = 0; j < sme_vl_count; j++) { + start_ssve(&children[num_children++], sme_vls[j], i); + start_za(&children[num_children++], sme_vls[j], i); + } + } + + for (;;) { + /* Did we get a signal asking us to exit? */ + if (terminate) + break; + + /* + * Timeout is counted in seconds with no output, the + * tests print during startup then are silent when + * running so this should ensure they all ran enough + * to install the signal handler, this is especially + * useful in emulation where we will both be slow and + * likely to have a large set of VLs. + */ + ret = epoll_wait(epoll_fd, &ev, 1, 1000); + if (ret < 0) { + if (errno == EINTR) + continue; + ksft_exit_fail_msg("epoll_wait() failed: %s (%d)\n", + strerror(errno), errno); + } + + /* Output? */ + if (ret == 1) { + child_output(ev.data.ptr, ev.events, false); + continue; + } + + /* Otherwise epoll_wait() timed out */ + + for (i = 0; i < num_children; i++) + child_tickle(&children[i]); + + /* Negative timeout means run indefinitely */ + if (timeout < 0) + continue; + if (--timeout == 0) + break; + } + + ksft_print_msg("Finishing up...\n"); + terminate = true; + + for (i = 0; i < tests; i++) + child_stop(&children[i]); + + drain_output(false); + + for (i = 0; i < tests; i++) + child_cleanup(&children[i]); + + drain_output(true); + + ksft_print_cnts(); + + return 0; +} diff --git a/tools/testing/selftests/arm64/fp/fpsimd-test.S b/tools/testing/selftests/arm64/fp/fpsimd-test.S index e21e8ea52c7ef6a0f48b892f0ae16f06e8cb7aae..918d04885a33f1be503ee8521c26f3eac7a2b15b 100644 --- a/tools/testing/selftests/arm64/fp/fpsimd-test.S +++ b/tools/testing/selftests/arm64/fp/fpsimd-test.S @@ -151,6 +151,15 @@ function irritator_handler ret endfunction +function tickle_handler + // Increment the signal count (x23): + ldr x0, [x2, #ucontext_regs + 8 * 23] + add x0, x0, #1 + str x0, [x2, #ucontext_regs + 8 * 23] + + ret +endfunction + function terminate_handler mov w21, w0 mov x20, x2 @@ -207,6 +216,30 @@ endfunction .globl _start function _start _start: + mov x23, #0 // signal count + + mov w0, #SIGINT + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGTERM + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGUSR1 + adr x1, irritator_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + + mov w0, #SIGUSR2 + adr x1, tickle_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + // Sanity-check and report the vector length mov x19, #128 @@ -237,24 +270,6 @@ _start: mov x0, x20 bl putdecn - mov x23, #0 // Irritation signal count - - mov w0, #SIGINT - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGTERM - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGUSR1 - adr x1, irritator_handler - mov w2, #SA_SIGINFO - orr w2, w2, #SA_NODEFER - bl setsignal - mov x22, #0 // generation number, increments per iteration .Ltest_loop: diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 589264231a2d0afd8b1184f63d6c629b882d900c..2a18cb4c528c59d634b3e5f8dc5fa070d4faa0cf 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -314,6 +314,15 @@ function irritator_handler ret endfunction +function tickle_handler + // Increment the signal count (x23): + ldr x0, [x2, #ucontext_regs + 8 * 23] + add x0, x0, #1 + str x0, [x2, #ucontext_regs + 8 * 23] + + ret +endfunction + function terminate_handler mov w21, w0 mov x20, x2 @@ -370,6 +379,30 @@ endfunction .globl _start function _start _start: + mov x23, #0 // Irritation signal count + + mov w0, #SIGINT + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGTERM + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGUSR1 + adr x1, irritator_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + + mov w0, #SIGUSR2 + adr x1, tickle_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + #ifdef SSVE puts "Streaming mode " smstart_sm @@ -405,24 +438,6 @@ _start: mov x0, x20 bl putdecn - mov x23, #0 // Irritation signal count - - mov w0, #SIGINT - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGTERM - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGUSR1 - adr x1, irritator_handler - mov w2, #SA_SIGINFO - orr w2, w2, #SA_NODEFER - bl setsignal - #ifdef SSVE smstart_sm // syscalls will have exited streaming mode #endif diff --git a/tools/testing/selftests/arm64/fp/za-test.S b/tools/testing/selftests/arm64/fp/za-test.S index 9ab6f9cd9623b3c01702920ed0dafca344b718fc..53c54af6570471d971d191c9bb9cfd29a77556ca 100644 --- a/tools/testing/selftests/arm64/fp/za-test.S +++ b/tools/testing/selftests/arm64/fp/za-test.S @@ -167,6 +167,15 @@ function irritator_handler ret endfunction +function tickle_handler + // Increment the signal count (x23): + ldr x0, [x2, #ucontext_regs + 8 * 23] + add x0, x0, #1 + str x0, [x2, #ucontext_regs + 8 * 23] + + ret +endfunction + function terminate_handler mov w21, w0 mov x20, x2 @@ -223,6 +232,30 @@ endfunction .globl _start function _start _start: + mov x23, #0 // signal count + + mov w0, #SIGINT + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGTERM + adr x1, terminate_handler + mov w2, #SA_SIGINFO + bl setsignal + + mov w0, #SIGUSR1 + adr x1, irritator_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + + mov w0, #SIGUSR2 + adr x1, tickle_handler + mov w2, #SA_SIGINFO + orr w2, w2, #SA_NODEFER + bl setsignal + puts "Streaming mode " smstart_za @@ -255,24 +288,6 @@ _start: mov x0, x20 bl putdecn - mov x23, #0 // Irritation signal count - - mov w0, #SIGINT - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGTERM - adr x1, terminate_handler - mov w2, #SA_SIGINFO - bl setsignal - - mov w0, #SIGUSR1 - adr x1, irritator_handler - mov w2, #SA_SIGINFO - orr w2, w2, #SA_NODEFER - bl setsignal - mov x22, #0 // generation number, increments per iteration .Ltest_loop: rdsvl 0, 8 @@ -287,12 +302,7 @@ _start: subs x21, x21, #1 b.ne 0b - and x8, x22, #127 // Every 128 interations... - cbz x8, 0f - mov x8, #__NR_getpid // (otherwise minimal syscall) - b 1f -0: - mov x8, #__NR_sched_yield // ...encourage preemption + mov x8, #__NR_sched_yield // encourage preemption 1: svc #0 diff --git a/tools/testing/selftests/arm64/mte/Makefile b/tools/testing/selftests/arm64/mte/Makefile index a5a0744423d8671a2b76bdedc302892139a9cd8e..037046f5784eac18aa868171c65867806752b270 100644 --- a/tools/testing/selftests/arm64/mte/Makefile +++ b/tools/testing/selftests/arm64/mte/Makefile @@ -11,11 +11,8 @@ LDFLAGS += -pthread SRCS := $(filter-out mte_common_util.c,$(wildcard *.c)) PROGS := $(patsubst %.c,%,$(SRCS)) -#Add mte compiler option -CFLAGS += -march=armv8.5-a+memtag - #check if the compiler works well -mte_cc_support := $(shell if ($(CC) $(CFLAGS) -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi) +mte_cc_support := $(shell if ($(CC) $(CFLAGS) -march=armv8.5-a+memtag -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi) ifeq ($(mte_cc_support),1) # Generated binaries to be installed by top KSFT script diff --git a/tools/testing/selftests/arm64/mte/mte_helper.S b/tools/testing/selftests/arm64/mte/mte_helper.S index a02c04cd0aac981f7171a2128186a5e389aca635..a55dbbc56ed190f779f9309d990e46ddac283512 100644 --- a/tools/testing/selftests/arm64/mte/mte_helper.S +++ b/tools/testing/selftests/arm64/mte/mte_helper.S @@ -3,6 +3,8 @@ #include "mte_def.h" +.arch armv8.5-a+memtag + #define ENTRY(name) \ .globl name ;\ .p2align 2;\ diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c index b588d10afd5b7da5db566c70f8dcb16e66485629..308e229e58abb00b469ad867c757228c87ce819f 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.c +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c @@ -165,15 +165,64 @@ static bool handle_signal_ok(struct tdescr *td, } static bool handle_signal_copyctx(struct tdescr *td, - siginfo_t *si, void *uc) + siginfo_t *si, void *uc_in) { + ucontext_t *uc = uc_in; + struct _aarch64_ctx *head; + struct extra_context *extra, *copied_extra; + size_t offset = 0; + size_t to_copy; + + ASSERT_GOOD_CONTEXT(uc); + /* Mangling PC to avoid loops on original BRK instr */ - ((ucontext_t *)uc)->uc_mcontext.pc += 4; - memcpy(td->live_uc, uc, td->live_sz); - ASSERT_GOOD_CONTEXT(td->live_uc); + uc->uc_mcontext.pc += 4; + + /* + * Check for an preserve any extra data too with fixups. + */ + head = (struct _aarch64_ctx *)uc->uc_mcontext.__reserved; + head = get_header(head, EXTRA_MAGIC, td->live_sz, &offset); + if (head) { + extra = (struct extra_context *)head; + + /* + * The extra buffer must be immediately after the + * extra_context and a 16 byte terminator. Include it + * in the copy, this was previously validated in + * ASSERT_GOOD_CONTEXT(). + */ + to_copy = offset + sizeof(struct extra_context) + 16 + + extra->size; + copied_extra = (struct extra_context *)&(td->live_uc->uc_mcontext.__reserved[offset]); + } else { + copied_extra = NULL; + to_copy = sizeof(ucontext_t); + } + + if (to_copy > td->live_sz) { + fprintf(stderr, + "Not enough space to grab context, %lu/%lu bytes\n", + td->live_sz, to_copy); + return false; + } + + memcpy(td->live_uc, uc, to_copy); + + /* + * If there was any EXTRA_CONTEXT fix up the size to be the + * struct extra_context and the following terminator record, + * this means that the rest of the code does not need to have + * special handling for the record and we don't need to fix up + * datap for the new location. + */ + if (copied_extra) + copied_extra->head.size = sizeof(*copied_extra) + 16; + td->live_uc_valid = 1; fprintf(stderr, - "GOOD CONTEXT grabbed from sig_copyctx handler\n"); + "%lu byte GOOD CONTEXT grabbed from sig_copyctx handler\n", + to_copy); return true; } diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.h b/tools/testing/selftests/arm64/signal/test_signals_utils.h index f3aa99ba67bb7e919929fbbc5c6c87a51d1fcb20..222093f51b6717ce70b406c52b8f22d8f53d7b3f 100644 --- a/tools/testing/selftests/arm64/signal/test_signals_utils.h +++ b/tools/testing/selftests/arm64/signal/test_signals_utils.h @@ -56,7 +56,8 @@ static inline bool feats_ok(struct tdescr *td) * at sizeof(ucontext_t). */ static __always_inline bool get_current_context(struct tdescr *td, - ucontext_t *dest_uc) + ucontext_t *dest_uc, + size_t dest_sz) { static volatile bool seen_already; @@ -64,7 +65,7 @@ static __always_inline bool get_current_context(struct tdescr *td, /* it's a genuine invocation..reinit */ seen_already = 0; td->live_uc_valid = 0; - td->live_sz = sizeof(*dest_uc); + td->live_sz = dest_sz; memset(dest_uc, 0x00, td->live_sz); td->live_uc = dest_uc; /* diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c index 8dc600a7d4fd1b0fe9eede11e23dee21198069fa..8c7f00ea98237020be929d75f11b9db1394a9283 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c @@ -21,7 +21,7 @@ static int fake_sigreturn_bad_magic_run(struct tdescr *td, struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head; /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; /* need at least 2*HDR_SZ space: KSFT_BAD_MAGIC + terminator. */ diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c index b3c362100666f64d8978eb7efeed7af5d764aa44..1c03f6b638e011dc02f05ebce10adf6be826ce40 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c @@ -24,7 +24,7 @@ static int fake_sigreturn_bad_size_run(struct tdescr *td, struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head; /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c index a44b88bfc81a8f81c35ac12aded5fb6582a57b1b..bc22f64b544e879648f2bcb613250bb072003d03 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c @@ -21,7 +21,7 @@ static int fake_sigreturn_bad_size_for_magic0_run(struct tdescr *td, struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head; /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; /* at least HDR_SZ for the badly sized terminator. */ diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c index afe8915f0998533106391fdc5d0920188fbda342..63e3906b631c1da9b4613db7a4e42125bde92afd 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c @@ -21,7 +21,7 @@ static int fake_sigreturn_duplicated_fpsimd_run(struct tdescr *td, struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head; /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; head = get_starting_head(shead, sizeof(struct fpsimd_context) + HDR_SZ, diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c index 1e089e66f9f3c86f20b74ce4684d74ef68cb4416..d00625ff12c238421aab6c0fad7a88af8b8e0c14 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c @@ -19,7 +19,7 @@ static int fake_sigreturn_misaligned_run(struct tdescr *td, siginfo_t *si, ucontext_t *uc) { /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; /* Forcing sigframe on misaligned SP (16 + 3) */ diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c index 08ecd8073a1abc966b63b55c4c88cbb8251b7f61..f805138cb20dd9e7f23c3bccacba0f89b57b1ac4 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c @@ -23,7 +23,7 @@ static int fake_sigreturn_missing_fpsimd_run(struct tdescr *td, struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf); /* just to fill the ucontext_t with something real */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c index 7ed762b7202f14dd5e7b55e99d98bc1f37bccd36..ebd5815b54bbaaf883ddeed306260b00180f5b0e 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c @@ -54,7 +54,7 @@ static int fake_sigreturn_ssve_change_vl(struct tdescr *td, struct sve_context *sve; /* Get a signal context with a SME ZA frame in it */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c index 915821375b0a44bc5209def4c26f8c513cb614a1..e2a452190511ff4ab7e15c084347b5bdb7b967fe 100644 --- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c +++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c @@ -56,7 +56,7 @@ static int fake_sigreturn_sve_change_vl(struct tdescr *td, struct sve_context *sve; /* Get a signal context with a SVE frame in it */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/sme_vl.c b/tools/testing/selftests/arm64/signal/testcases/sme_vl.c index 13ff3b35cbaf578e48325a36b935eb8cdc7533cf..75f387f2db81d9de5b5f59e310e7fb2300d3a3e1 100644 --- a/tools/testing/selftests/arm64/signal/testcases/sme_vl.c +++ b/tools/testing/selftests/arm64/signal/testcases/sme_vl.c @@ -34,7 +34,7 @@ static int sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc) struct za_context *za; /* Get a signal context which should have a ZA frame in it */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c b/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c index 9022a6cab4b33593ccb6cc4a969ee4986f5a5f91..d0a178945b1a82bb0097e18075ddce897d7b1c9a 100644 --- a/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c +++ b/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c @@ -13,7 +13,10 @@ #include "test_signals_utils.h" #include "testcases.h" -struct fake_sigframe sf; +static union { + ucontext_t uc; + char buf[1024 * 64]; +} context; static unsigned int vls[SVE_VQ_MAX]; unsigned int nvls = 0; @@ -55,8 +58,8 @@ static void setup_ssve_regs(void) static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, unsigned int vl) { - size_t resv_sz, offset; - struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf); + size_t offset; + struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); struct sve_context *ssve; int ret; @@ -73,11 +76,11 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, * in it. */ setup_ssve_regs(); - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &context.uc, sizeof(context))) return 1; - resv_sz = GET_SF_RESV_SIZE(sf); - head = get_header(head, SVE_MAGIC, resv_sz, &offset); + head = get_header(head, SVE_MAGIC, GET_BUF_RESV_SIZE(context), + &offset); if (!head) { fprintf(stderr, "No SVE context\n"); return 1; @@ -101,16 +104,6 @@ static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc) int i; for (i = 0; i < nvls; i++) { - /* - * TODO: the signal test helpers can't currently cope - * with signal frames bigger than struct sigcontext, - * skip VLs that will trigger that. - */ - if (vls[i] > 64) { - printf("Skipping VL %u due to stack size\n", vls[i]); - continue; - } - if (do_one_sme_vl(td, si, uc, vls[i])) return 1; } diff --git a/tools/testing/selftests/arm64/signal/testcases/sve_regs.c b/tools/testing/selftests/arm64/signal/testcases/sve_regs.c index 4b2418aa08a9e76e85282b700fb7100f5f516580..8b16eabbb7697e3cb13e3b9d747345bb12b26bb6 100644 --- a/tools/testing/selftests/arm64/signal/testcases/sve_regs.c +++ b/tools/testing/selftests/arm64/signal/testcases/sve_regs.c @@ -13,7 +13,10 @@ #include "test_signals_utils.h" #include "testcases.h" -struct fake_sigframe sf; +static union { + ucontext_t uc; + char buf[1024 * 64]; +} context; static unsigned int vls[SVE_VQ_MAX]; unsigned int nvls = 0; @@ -55,8 +58,8 @@ static void setup_sve_regs(void) static int do_one_sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, unsigned int vl) { - size_t resv_sz, offset; - struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf); + size_t offset; + struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); struct sve_context *sve; fprintf(stderr, "Testing VL %d\n", vl); @@ -71,11 +74,11 @@ static int do_one_sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, * in it. */ setup_sve_regs(); - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &context.uc, sizeof(context))) return 1; - resv_sz = GET_SF_RESV_SIZE(sf); - head = get_header(head, SVE_MAGIC, resv_sz, &offset); + head = get_header(head, SVE_MAGIC, GET_BUF_RESV_SIZE(context), + &offset); if (!head) { fprintf(stderr, "No SVE context\n"); return 1; @@ -99,14 +102,6 @@ static int sve_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc) int i; for (i = 0; i < nvls; i++) { - /* - * TODO: the signal test helpers can't currently cope - * with signal frames bigger than struct sigcontext, - * skip VLs that will trigger that. - */ - if (vls[i] > 64) - continue; - if (do_one_sve_vl(td, si, uc, vls[i])) return 1; } diff --git a/tools/testing/selftests/arm64/signal/testcases/sve_vl.c b/tools/testing/selftests/arm64/signal/testcases/sve_vl.c index 92904653add17b246780c0518f9954bff7d83ff6..aa835acec062559fea9455186d594a6c3c38d9f5 100644 --- a/tools/testing/selftests/arm64/signal/testcases/sve_vl.c +++ b/tools/testing/selftests/arm64/signal/testcases/sve_vl.c @@ -34,7 +34,7 @@ static int sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc) struct sve_context *sve; /* Get a signal context which should have a SVE frame in it */ - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &sf.uc, sizeof(sf.uc))) return 1; resv_sz = GET_SF_RESV_SIZE(sf); diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.c b/tools/testing/selftests/arm64/signal/testcases/testcases.c index 84c36bee4d82a19ef336d731fdbb14ea960e2ba3..e1c625b20ac4cb818f01f049b70e9dd417da5a08 100644 --- a/tools/testing/selftests/arm64/signal/testcases/testcases.c +++ b/tools/testing/selftests/arm64/signal/testcases/testcases.c @@ -25,7 +25,8 @@ struct _aarch64_ctx *get_header(struct _aarch64_ctx *head, uint32_t magic, return found; } -bool validate_extra_context(struct extra_context *extra, char **err) +bool validate_extra_context(struct extra_context *extra, char **err, + void **extra_data, size_t *extra_size) { struct _aarch64_ctx *term; @@ -33,7 +34,7 @@ bool validate_extra_context(struct extra_context *extra, char **err) return false; fprintf(stderr, "Validating EXTRA...\n"); - term = GET_RESV_NEXT_HEAD(extra); + term = GET_RESV_NEXT_HEAD(&extra->head); if (!term || term->magic || term->size) { *err = "Missing terminator after EXTRA context"; return false; @@ -42,11 +43,14 @@ bool validate_extra_context(struct extra_context *extra, char **err) *err = "Extra DATAP misaligned"; else if (extra->size & 0x0fUL) *err = "Extra SIZE misaligned"; - else if (extra->datap != (uint64_t)term + sizeof(*term)) + else if (extra->datap != (uint64_t)term + 0x10UL) *err = "Extra DATAP misplaced (not contiguous)"; if (*err) return false; + *extra_data = (void *)extra->datap; + *extra_size = extra->size; + return true; } @@ -105,11 +109,14 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) bool terminated = false; size_t offs = 0; int flags = 0; + int new_flags; struct extra_context *extra = NULL; struct sve_context *sve = NULL; struct za_context *za = NULL; struct _aarch64_ctx *head = (struct _aarch64_ctx *)uc->uc_mcontext.__reserved; + void *extra_data = NULL; + size_t extra_sz = 0; if (!err) return false; @@ -120,12 +127,24 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) return false; } + new_flags = 0; + switch (head->magic) { case 0: - if (head->size) + if (head->size) { *err = "Bad size for terminator"; - else + } else if (extra_data) { + /* End of main data, walking the extra data */ + head = extra_data; + resv_sz = extra_sz; + offs = 0; + + extra_data = NULL; + extra_sz = 0; + continue; + } else { terminated = true; + } break; case FPSIMD_MAGIC: if (flags & FPSIMD_CTX) @@ -133,7 +152,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) else if (head->size != sizeof(struct fpsimd_context)) *err = "Bad size for fpsimd_context"; - flags |= FPSIMD_CTX; + new_flags |= FPSIMD_CTX; break; case ESR_MAGIC: if (head->size != sizeof(struct esr_context)) @@ -144,14 +163,14 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) *err = "Multiple SVE_MAGIC"; /* Size is validated in validate_sve_context() */ sve = (struct sve_context *)head; - flags |= SVE_CTX; + new_flags |= SVE_CTX; break; case ZA_MAGIC: if (flags & ZA_CTX) *err = "Multiple ZA_MAGIC"; /* Size is validated in validate_za_context() */ za = (struct za_context *)head; - flags |= ZA_CTX; + new_flags |= ZA_CTX; break; case EXTRA_MAGIC: if (flags & EXTRA_CTX) @@ -159,7 +178,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) else if (head->size != sizeof(struct extra_context)) *err = "Bad size for extra_context"; - flags |= EXTRA_CTX; + new_flags |= EXTRA_CTX; extra = (struct extra_context *)head; break; case KSFT_BAD_MAGIC: @@ -192,16 +211,19 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err) return false; } - if (flags & EXTRA_CTX) - if (!validate_extra_context(extra, err)) + if (new_flags & EXTRA_CTX) + if (!validate_extra_context(extra, err, + &extra_data, &extra_sz)) return false; - if (flags & SVE_CTX) + if (new_flags & SVE_CTX) if (!validate_sve_context(sve, err)) return false; - if (flags & ZA_CTX) + if (new_flags & ZA_CTX) if (!validate_za_context(za, err)) return false; + flags |= new_flags; + head = GET_RESV_NEXT_HEAD(head); } diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.h b/tools/testing/selftests/arm64/signal/testcases/testcases.h index 49f1d5de7b5b72de8011120c8d6aca7b3fef5874..040afded0b76eaf8a22e2fd665a74ae4be6dc030 100644 --- a/tools/testing/selftests/arm64/signal/testcases/testcases.h +++ b/tools/testing/selftests/arm64/signal/testcases/testcases.h @@ -30,6 +30,13 @@ #define GET_SF_RESV_SIZE(sf) \ sizeof((sf).uc.uc_mcontext.__reserved) +#define GET_BUF_RESV_HEAD(buf) \ + (struct _aarch64_ctx *)(&(buf).uc.uc_mcontext.__reserved) + +#define GET_BUF_RESV_SIZE(buf) \ + (sizeof(buf) - sizeof(buf.uc) + \ + sizeof((buf).uc.uc_mcontext.__reserved)) + #define GET_UCP_RESV_SIZE(ucp) \ sizeof((ucp)->uc_mcontext.__reserved) @@ -79,8 +86,6 @@ struct fake_sigframe { bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err); -bool validate_extra_context(struct extra_context *extra, char **err); - struct _aarch64_ctx *get_header(struct _aarch64_ctx *head, uint32_t magic, size_t resv_sz, size_t *offset); diff --git a/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c b/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c new file mode 100644 index 0000000000000000000000000000000000000000..4d6f94b6178f36eb56677f0b8c4d8b7873e70832 --- /dev/null +++ b/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 ARM Limited + * + * Verify that the ZA register context in signal frames is set up as + * expected. + */ + +#include <signal.h> +#include <ucontext.h> +#include <sys/prctl.h> + +#include "test_signals_utils.h" +#include "testcases.h" + +static union { + ucontext_t uc; + char buf[1024 * 128]; +} context; +static unsigned int vls[SVE_VQ_MAX]; +unsigned int nvls = 0; + +static bool sme_get_vls(struct tdescr *td) +{ + int vq, vl; + + /* + * Enumerate up to SME_VQ_MAX vector lengths + */ + for (vq = SVE_VQ_MAX; vq > 0; --vq) { + vl = prctl(PR_SME_SET_VL, vq * 16); + if (vl == -1) + return false; + + vl &= PR_SME_VL_LEN_MASK; + + /* Skip missing VLs */ + vq = sve_vq_from_vl(vl); + + vls[nvls++] = vl; + } + + /* We need at least one VL */ + if (nvls < 1) { + fprintf(stderr, "Only %d VL supported\n", nvls); + return false; + } + + return true; +} + +static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, + unsigned int vl) +{ + size_t offset; + struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); + struct za_context *za; + + fprintf(stderr, "Testing VL %d\n", vl); + + if (prctl(PR_SME_SET_VL, vl) != vl) { + fprintf(stderr, "Failed to set VL\n"); + return 1; + } + + /* + * Get a signal context which should have a SVE frame and registers + * in it. + */ + if (!get_current_context(td, &context.uc, sizeof(context))) + return 1; + + head = get_header(head, ZA_MAGIC, GET_BUF_RESV_SIZE(context), &offset); + if (!head) { + fprintf(stderr, "No ZA context\n"); + return 1; + } + + za = (struct za_context *)head; + if (za->vl != vl) { + fprintf(stderr, "Got VL %d, expected %d\n", za->vl, vl); + return 1; + } + + if (head->size != ZA_SIG_REGS_OFFSET) { + fprintf(stderr, "Context size %u, expected %lu\n", + head->size, ZA_SIG_REGS_OFFSET); + return 1; + } + + /* The actual size validation is done in get_current_context() */ + fprintf(stderr, "Got expected size %u and VL %d\n", + head->size, za->vl); + + return 0; +} + +static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc) +{ + int i; + + for (i = 0; i < nvls; i++) { + if (do_one_sme_vl(td, si, uc, vls[i])) + return 1; + } + + td->pass = 1; + + return 0; +} + +struct tdescr tde = { + .name = "ZA registers - ZA disabled", + .descr = "Check ZA context with ZA disabled", + .feats_required = FEAT_SME, + .timeout = 3, + .init = sme_get_vls, + .run = sme_regs, +}; diff --git a/tools/testing/selftests/arm64/signal/testcases/za_regs.c b/tools/testing/selftests/arm64/signal/testcases/za_regs.c index b94e4f99fcac7085a36763bdb2ae2f8b5233cf65..ea45acb115d5b2338c31f62d37b39e5e6475a7ac 100644 --- a/tools/testing/selftests/arm64/signal/testcases/za_regs.c +++ b/tools/testing/selftests/arm64/signal/testcases/za_regs.c @@ -13,7 +13,10 @@ #include "test_signals_utils.h" #include "testcases.h" -struct fake_sigframe sf; +static union { + ucontext_t uc; + char buf[1024 * 128]; +} context; static unsigned int vls[SVE_VQ_MAX]; unsigned int nvls = 0; @@ -22,10 +25,10 @@ static bool sme_get_vls(struct tdescr *td) int vq, vl; /* - * Enumerate up to SVE_VQ_MAX vector lengths + * Enumerate up to SME_VQ_MAX vector lengths */ for (vq = SVE_VQ_MAX; vq > 0; --vq) { - vl = prctl(PR_SVE_SET_VL, vq * 16); + vl = prctl(PR_SME_SET_VL, vq * 16); if (vl == -1) return false; @@ -52,11 +55,13 @@ static void setup_za_regs(void) asm volatile(".inst 0xd503457f" : : : ); } +static char zeros[ZA_SIG_REGS_SIZE(SVE_VQ_MAX)]; + static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, unsigned int vl) { - size_t resv_sz, offset; - struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf); + size_t offset; + struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context); struct za_context *za; fprintf(stderr, "Testing VL %d\n", vl); @@ -71,11 +76,10 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, * in it. */ setup_za_regs(); - if (!get_current_context(td, &sf.uc)) + if (!get_current_context(td, &context.uc, sizeof(context))) return 1; - resv_sz = GET_SF_RESV_SIZE(sf); - head = get_header(head, ZA_MAGIC, resv_sz, &offset); + head = get_header(head, ZA_MAGIC, GET_BUF_RESV_SIZE(context), &offset); if (!head) { fprintf(stderr, "No ZA context\n"); return 1; @@ -87,10 +91,22 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc, return 1; } - /* The actual size validation is done in get_current_context() */ + if (head->size != ZA_SIG_CONTEXT_SIZE(sve_vq_from_vl(vl))) { + fprintf(stderr, "ZA context size %u, expected %lu\n", + head->size, ZA_SIG_CONTEXT_SIZE(sve_vq_from_vl(vl))); + return 1; + } + fprintf(stderr, "Got expected size %u and VL %d\n", head->size, za->vl); + /* We didn't load any data into ZA so it should be all zeros */ + if (memcmp(zeros, (char *)za + ZA_SIG_REGS_OFFSET, + ZA_SIG_REGS_SIZE(sve_vq_from_vl(za->vl))) != 0) { + fprintf(stderr, "ZA data invalid\n"); + return 1; + } + return 0; } @@ -99,16 +115,6 @@ static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc) int i; for (i = 0; i < nvls; i++) { - /* - * TODO: the signal test helpers can't currently cope - * with signal frames bigger than struct sigcontext, - * skip VLs that will trigger that. - */ - if (vls[i] > 32) { - printf("Skipping VL %u due to stack size\n", vls[i]); - continue; - } - if (do_one_sme_vl(td, si, uc, vls[i])) return 1; } diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 3a8cb2404ea6e5a5f9fc139236c7d16a412018d0..07d2d0a8c5cb4dae24dedd1e3b894d2dfa294197 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -39,6 +39,8 @@ test_cpp /tools /runqslower /bench +/veristat +/sign-file *.ko *.tmp xskxceiver diff --git a/tools/testing/selftests/bpf/DENYLIST.s390x b/tools/testing/selftests/bpf/DENYLIST.s390x index e33cab34d22fc78204d6441f33ba9050cdd1070a..17e074eb42b8afffbd44c2468c619d1184d16533 100644 --- a/tools/testing/selftests/bpf/DENYLIST.s390x +++ b/tools/testing/selftests/bpf/DENYLIST.s390x @@ -43,7 +43,7 @@ test_bpffs # bpffs test failed 255 test_bprm_opts # failed to auto-attach program 'secure_exec': -524 (trampoline) test_ima # failed to auto-attach program 'ima': -524 (trampoline) test_local_storage # failed to auto-attach program 'unlink_hook': -524 (trampoline) -test_lsm # failed to find kernel BTF type ID of '__x64_sys_setdomainname': -3 (?) +test_lsm # attach unexpected error: -524 (trampoline) test_overhead # attach_fentry unexpected error: -524 (trampoline) test_profiler # unknown func bpf_probe_read_str#45 (overlapping) timer # failed to auto-attach program 'test1': -524 (trampoline) @@ -65,3 +65,13 @@ send_signal # intermittently fails to receive signa select_reuseport # intermittently fails on new s390x setup xdp_synproxy # JIT does not support calling kernel function (kfunc) unpriv_bpf_disabled # fentry +lru_bug # prog 'printk': failed to auto-attach: -524 +setget_sockopt # attach unexpected error: -524 (trampoline) +cb_refs # expected error message unexpected error: -524 (trampoline) +cgroup_hierarchical_stats # JIT does not support calling kernel function (kfunc) +htab_update # failed to attach: ERROR: strerror_r(-524)=22 (trampoline) +tracing_struct # failed to auto-attach: -524 (trampoline) +user_ringbuf # failed to find kernel BTF type ID of '__s390x_sys_prctl': -3 (?) +lookup_key # JIT does not support calling kernel function (kfunc) +verify_pkcs7_sig # JIT does not support calling kernel function (kfunc) +kfunc_dynptr_param # JIT does not support calling kernel function (kfunc) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 8d59ec7f4c2d061803adc1a609f0ae11c14c8a34..e6cf21fad69f0c8d1d8c67163d8350871907e956 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -14,6 +14,7 @@ BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool APIDIR := $(TOOLSINCDIR)/uapi GENDIR := $(abspath ../../../../include/generated) GENHDR := $(GENDIR)/autoconf.h +HOSTPKG_CONFIG := pkg-config ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR @@ -45,7 +46,7 @@ ifneq ($(BPF_GCC),) TEST_GEN_PROGS += test_progs-bpf_gcc endif -TEST_GEN_FILES = test_lwt_ip_encap.o test_tc_edt.o +TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) # Order correspond to 'make run_tests' order @@ -75,16 +76,17 @@ TEST_PROGS := test_kmod.sh \ test_xsk.sh TEST_PROGS_EXTENDED := with_addr.sh \ - with_tunnels.sh ima_setup.sh \ + with_tunnels.sh ima_setup.sh verify_sig_setup.sh \ test_xdp_vlan.sh test_bpftool.py # Compile but not part of 'make run_tests' TEST_GEN_PROGS_EXTENDED = test_sock_addr test_skb_cgroup_id_user \ flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \ test_lirc_mode2_user xdping test_cpp runqslower bench bpf_testmod.ko \ - xskxceiver xdp_redirect_multi xdp_synproxy + xskxceiver xdp_redirect_multi xdp_synproxy veristat -TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read +TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read $(OUTPUT)/sign-file +TEST_GEN_FILES += liburandom_read.so # Emit succinct information message describing current building step # $1 - generic step name (e.g., CC, LINK, etc); @@ -189,6 +191,12 @@ $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_r -fuse-ld=$(LLD) -Wl,-znoseparate-code \ -Wl,-rpath=. -Wl,--build-id=sha1 -o $@ +$(OUTPUT)/sign-file: ../../../../scripts/sign-file.c + $(call msg,SIGN-FILE,,$@) + $(Q)$(CC) $(shell $(HOSTPKG_CONFIG)--cflags libcrypto 2> /dev/null) \ + $< -o $@ \ + $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto) + $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch]) $(call msg,MOD,,$@) $(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation @@ -323,6 +331,7 @@ $(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline $(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h +$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h # Build BPF object using Clang # $1 - input .c file @@ -350,24 +359,25 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ test_subskeleton.skel.h test_subskeleton_lib.skel.h \ test_usdt.skel.h -LSKELS := kfunc_call_test.c fentry_test.c fexit_test.c fexit_sleep.c \ +LSKELS := fentry_test.c fexit_test.c fexit_sleep.c \ test_ringbuf.c atomics.c trace_printk.c trace_vprintk.c \ map_ptr_kern.c core_kern.c core_kern_overflow.c # Generate both light skeleton and libbpf skeleton for these -LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test_subprog.c +LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ + kfunc_call_test_subprog.c SKEL_BLACKLIST += $$(LSKELS) -test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o -linked_funcs.skel.h-deps := linked_funcs1.o linked_funcs2.o -linked_vars.skel.h-deps := linked_vars1.o linked_vars2.o -linked_maps.skel.h-deps := linked_maps1.o linked_maps2.o +test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o +linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o +linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o +linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o # In the subskeleton case, we want the test_subskeleton_lib.subskel.h file # but that's created as a side-effect of the skel.h generation. -test_subskeleton.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o test_subskeleton.o -test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o -test_usdt.skel.h-deps := test_usdt.o test_usdt_multispec.o +test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o +test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o +test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o -LINKED_BPF_SRCS := $(patsubst %.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps))) +LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps))) # Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on # $eval()) and pass control to DEFINE_TEST_RUNNER_RULES. @@ -385,7 +395,7 @@ TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \ TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES)) TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c)) -TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, $$(TRUNNER_BPF_SRCS)) +TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS)) TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h, \ $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\ $$(TRUNNER_BPF_SRCS))) @@ -415,7 +425,7 @@ endif # input/output directory combination ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),) $(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y -$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o: \ +$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o: \ $(TRUNNER_BPF_PROGS_DIR)/%.c \ $(TRUNNER_BPF_PROGS_DIR)/*.h \ $$(INCLUDE_DIR)/vmlinux.h \ @@ -425,25 +435,25 @@ $(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o: \ $$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \ $(TRUNNER_BPF_CFLAGS)) -$(TRUNNER_BPF_SKELS): %.skel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT) +$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$< $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o) $(Q)$$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o) $(Q)diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o) - $(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$@ - $(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$(@:.skel.h=.subskel.h) + $(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@ + $(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h) -$(TRUNNER_BPF_LSKELS): %.lskel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT) +$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) $$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@) $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$< $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o) $(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o) $(Q)diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) - $(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.o=_lskel)) > $$@ + $(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@ $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_BPF_OBJS) $(BPFTOOL) | $(TRUNNER_OUTPUT) - $$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.o)) + $$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.bpf.o)) $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps)) $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o) $(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o) @@ -499,7 +509,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \ | $(TRUNNER_BINARY)-extras $$(call msg,BINARY,,$$@) $(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@ - $(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.o $$@ + $(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@ $(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/bootstrap/bpftool $(if $2,$2/)bpftool endef @@ -514,7 +524,8 @@ TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \ TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko \ $(OUTPUT)/liburandom_read.so \ $(OUTPUT)/xdp_synproxy \ - ima_setup.sh \ + $(OUTPUT)/sign-file \ + ima_setup.sh verify_sig_setup.sh \ $(wildcard progs/btf_dump_test_case_*.c) TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS @@ -593,6 +604,11 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(call msg,BINARY,,$@) $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ +$(OUTPUT)/veristat.o: $(BPFOBJ) +$(OUTPUT)/veristat: $(OUTPUT)/veristat.o + $(call msg,BINARY,,$@) + $(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ + EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \ prog_tests/tests.h map_tests/tests.h verifier/tests.h \ feature bpftool \ diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index eb1b7541f39dea12b7131b58043b6d13af3c2c48..d3c6b3da0bb13c5929ef1ef7426361d2bb951938 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -126,11 +126,11 @@ available in 10.0.1. The patch is available in llvm 11.0.0 trunk. __ https://reviews.llvm.org/D78466 -bpf_verif_scale/loop6.o test failure with Clang 12 -================================================== +bpf_verif_scale/loop6.bpf.o test failure with Clang 12 +====================================================== With Clang 12, the following bpf_verif_scale test failed: - * ``bpf_verif_scale/loop6.o`` + * ``bpf_verif_scale/loop6.bpf.o`` The verifier output looks like @@ -245,7 +245,7 @@ See `kernel llvm reloc`_ for more explanation and some examples. Using clang 13 to compile old libbpf which has static linker support, there will be a compilation failure:: - libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.o + libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.bpf.o Here, ``type 2`` refers to new relocation type ``R_BPF_64_ABS64``. To fix this issue, user newer libbpf. diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c index 792cb15bac40f58c23de0bf95555a1e05a31b05a..a6021d6117b5d81b87d497e4ce9bec5e0f167bf8 100644 --- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c @@ -18,6 +18,46 @@ typedef int (*func_proto_typedef_nested1)(func_proto_typedef); typedef int (*func_proto_typedef_nested2)(func_proto_typedef_nested1); DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123; +long bpf_testmod_test_struct_arg_result; + +struct bpf_testmod_struct_arg_1 { + int a; +}; +struct bpf_testmod_struct_arg_2 { + long a; + long b; +}; + +noinline int +bpf_testmod_test_struct_arg_1(struct bpf_testmod_struct_arg_2 a, int b, int c) { + bpf_testmod_test_struct_arg_result = a.a + a.b + b + c; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_2(int a, struct bpf_testmod_struct_arg_2 b, int c) { + bpf_testmod_test_struct_arg_result = a + b.a + b.b + c; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_3(int a, int b, struct bpf_testmod_struct_arg_2 c) { + bpf_testmod_test_struct_arg_result = a + b + c.a + c.b; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_4(struct bpf_testmod_struct_arg_1 a, int b, + int c, int d, struct bpf_testmod_struct_arg_2 e) { + bpf_testmod_test_struct_arg_result = a.a + b + c + d + e.a + e.b; + return bpf_testmod_test_struct_arg_result; +} + +noinline int +bpf_testmod_test_struct_arg_5(void) { + bpf_testmod_test_struct_arg_result = 1; + return bpf_testmod_test_struct_arg_result; +} noinline void bpf_testmod_test_mod_kfunc(int i) @@ -98,11 +138,19 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, .off = off, .len = len, }; + struct bpf_testmod_struct_arg_1 struct_arg1 = {10}; + struct bpf_testmod_struct_arg_2 struct_arg2 = {2, 3}; int i = 1; while (bpf_testmod_return_ptr(i)) i++; + (void)bpf_testmod_test_struct_arg_1(struct_arg2, 1, 4); + (void)bpf_testmod_test_struct_arg_2(1, struct_arg2, 4); + (void)bpf_testmod_test_struct_arg_3(1, 4, struct_arg2); + (void)bpf_testmod_test_struct_arg_4(struct_arg1, 1, 2, 3, struct_arg2); + (void)bpf_testmod_test_struct_arg_5(); + /* This is always true. Use the check to make sure the compiler * doesn't remove bpf_testmod_loop_test. */ diff --git a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h b/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h new file mode 100644 index 0000000000000000000000000000000000000000..a525d3544fd7fa779d989e5f1b602ffb79496829 --- /dev/null +++ b/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +BPF_RETVAL_HOOK(ingress, "cgroup_skb/ingress", __sk_buff, -EINVAL) +BPF_RETVAL_HOOK(egress, "cgroup_skb/egress", __sk_buff, -EINVAL) +BPF_RETVAL_HOOK(sock_create, "cgroup/sock_create", bpf_sock, 0) +BPF_RETVAL_HOOK(sock_ops, "sockops", bpf_sock_ops, -EINVAL) +BPF_RETVAL_HOOK(dev, "cgroup/dev", bpf_cgroup_dev_ctx, 0) +BPF_RETVAL_HOOK(bind4, "cgroup/bind4", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(bind6, "cgroup/bind6", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(connect4, "cgroup/connect4", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(connect6, "cgroup/connect6", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(post_bind4, "cgroup/post_bind4", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(post_bind6, "cgroup/post_bind6", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(sendmsg4, "cgroup/sendmsg4", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(sendmsg6, "cgroup/sendmsg6", bpf_sock_addr, 0) +BPF_RETVAL_HOOK(sysctl, "cgroup/sysctl", bpf_sysctl, 0) +BPF_RETVAL_HOOK(recvmsg4, "cgroup/recvmsg4", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(recvmsg6, "cgroup/recvmsg6", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(getsockopt, "cgroup/getsockopt", bpf_sockopt, 0) +BPF_RETVAL_HOOK(setsockopt, "cgroup/setsockopt", bpf_sockopt, 0) +BPF_RETVAL_HOOK(getpeername4, "cgroup/getpeername4", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(getpeername6, "cgroup/getpeername6", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(getsockname4, "cgroup/getsockname4", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(getsockname6, "cgroup/getsockname6", bpf_sock_addr, -EINVAL) +BPF_RETVAL_HOOK(sock_release, "cgroup/sock_release", bpf_sock, 0) diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c index 9d59c3990ca8d40fa7522aa1210ee05b982543b5..e914cc45b7669eae2b7f5859b90c998f61b7aec4 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.c +++ b/tools/testing/selftests/bpf/cgroup_helpers.c @@ -33,49 +33,52 @@ #define CGROUP_MOUNT_DFLT "/sys/fs/cgroup" #define NETCLS_MOUNT_PATH CGROUP_MOUNT_DFLT "/net_cls" #define CGROUP_WORK_DIR "/cgroup-test-work-dir" -#define format_cgroup_path(buf, path) \ + +#define format_cgroup_path_pid(buf, path, pid) \ snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \ - CGROUP_WORK_DIR, getpid(), path) + CGROUP_WORK_DIR, pid, path) + +#define format_cgroup_path(buf, path) \ + format_cgroup_path_pid(buf, path, getpid()) + +#define format_parent_cgroup_path(buf, path) \ + format_cgroup_path_pid(buf, path, getppid()) #define format_classid_path(buf) \ snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH, \ CGROUP_WORK_DIR) -/** - * enable_all_controllers() - Enable all available cgroup v2 controllers - * - * Enable all available cgroup v2 controllers in order to increase - * the code coverage. - * - * If successful, 0 is returned. - */ -static int enable_all_controllers(char *cgroup_path) +static int __enable_controllers(const char *cgroup_path, const char *controllers) { char path[PATH_MAX + 1]; - char buf[PATH_MAX]; + char enable[PATH_MAX + 1]; char *c, *c2; int fd, cfd; ssize_t len; - snprintf(path, sizeof(path), "%s/cgroup.controllers", cgroup_path); - fd = open(path, O_RDONLY); - if (fd < 0) { - log_err("Opening cgroup.controllers: %s", path); - return 1; - } - - len = read(fd, buf, sizeof(buf) - 1); - if (len < 0) { + /* If not controllers are passed, enable all available controllers */ + if (!controllers) { + snprintf(path, sizeof(path), "%s/cgroup.controllers", + cgroup_path); + fd = open(path, O_RDONLY); + if (fd < 0) { + log_err("Opening cgroup.controllers: %s", path); + return 1; + } + len = read(fd, enable, sizeof(enable) - 1); + if (len < 0) { + close(fd); + log_err("Reading cgroup.controllers: %s", path); + return 1; + } else if (len == 0) { /* No controllers to enable */ + close(fd); + return 0; + } + enable[len] = 0; close(fd); - log_err("Reading cgroup.controllers: %s", path); - return 1; + } else { + strncpy(enable, controllers, sizeof(enable)); } - buf[len] = 0; - close(fd); - - /* No controllers available? We're probably on cgroup v1. */ - if (len == 0) - return 0; snprintf(path, sizeof(path), "%s/cgroup.subtree_control", cgroup_path); cfd = open(path, O_RDWR); @@ -84,7 +87,7 @@ static int enable_all_controllers(char *cgroup_path) return 1; } - for (c = strtok_r(buf, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) { + for (c = strtok_r(enable, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) { if (dprintf(cfd, "+%s\n", c) <= 0) { log_err("Enabling controller %s: %s", c, path); close(cfd); @@ -95,6 +98,87 @@ static int enable_all_controllers(char *cgroup_path) return 0; } +/** + * enable_controllers() - Enable cgroup v2 controllers + * @relative_path: The cgroup path, relative to the workdir + * @controllers: List of controllers to enable in cgroup.controllers format + * + * + * Enable given cgroup v2 controllers, if @controllers is NULL, enable all + * available controllers. + * + * If successful, 0 is returned. + */ +int enable_controllers(const char *relative_path, const char *controllers) +{ + char cgroup_path[PATH_MAX + 1]; + + format_cgroup_path(cgroup_path, relative_path); + return __enable_controllers(cgroup_path, controllers); +} + +static int __write_cgroup_file(const char *cgroup_path, const char *file, + const char *buf) +{ + char file_path[PATH_MAX + 1]; + int fd; + + snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file); + fd = open(file_path, O_RDWR); + if (fd < 0) { + log_err("Opening %s", file_path); + return 1; + } + + if (dprintf(fd, "%s", buf) <= 0) { + log_err("Writing to %s", file_path); + close(fd); + return 1; + } + close(fd); + return 0; +} + +/** + * write_cgroup_file() - Write to a cgroup file + * @relative_path: The cgroup path, relative to the workdir + * @file: The name of the file in cgroupfs to write to + * @buf: Buffer to write to the file + * + * Write to a file in the given cgroup's directory. + * + * If successful, 0 is returned. + */ +int write_cgroup_file(const char *relative_path, const char *file, + const char *buf) +{ + char cgroup_path[PATH_MAX - 24]; + + format_cgroup_path(cgroup_path, relative_path); + return __write_cgroup_file(cgroup_path, file, buf); +} + +/** + * write_cgroup_file_parent() - Write to a cgroup file in the parent process + * workdir + * @relative_path: The cgroup path, relative to the parent process workdir + * @file: The name of the file in cgroupfs to write to + * @buf: Buffer to write to the file + * + * Write to a file in the given cgroup's directory under the parent process + * workdir. + * + * If successful, 0 is returned. + */ +int write_cgroup_file_parent(const char *relative_path, const char *file, + const char *buf) +{ + char cgroup_path[PATH_MAX - 24]; + + format_parent_cgroup_path(cgroup_path, relative_path); + return __write_cgroup_file(cgroup_path, file, buf); +} + /** * setup_cgroup_environment() - Setup the cgroup environment * @@ -133,7 +217,9 @@ int setup_cgroup_environment(void) return 1; } - if (enable_all_controllers(cgroup_workdir)) + /* Enable all available controllers to increase test coverage */ + if (__enable_controllers(CGROUP_MOUNT_PATH, NULL) || + __enable_controllers(cgroup_workdir, NULL)) return 1; return 0; @@ -173,7 +259,7 @@ static int join_cgroup_from_top(const char *cgroup_path) /** * join_cgroup() - Join a cgroup - * @path: The cgroup path, relative to the workdir, to join + * @relative_path: The cgroup path, relative to the workdir, to join * * This function expects a cgroup to already be created, relative to the cgroup * work dir, and it joins it. For example, passing "/my-cgroup" as the path @@ -182,11 +268,27 @@ static int join_cgroup_from_top(const char *cgroup_path) * * On success, it returns 0, otherwise on failure it returns 1. */ -int join_cgroup(const char *path) +int join_cgroup(const char *relative_path) +{ + char cgroup_path[PATH_MAX + 1]; + + format_cgroup_path(cgroup_path, relative_path); + return join_cgroup_from_top(cgroup_path); +} + +/** + * join_parent_cgroup() - Join a cgroup in the parent process workdir + * @relative_path: The cgroup path, relative to parent process workdir, to join + * + * See join_cgroup(). + * + * On success, it returns 0, otherwise on failure it returns 1. + */ +int join_parent_cgroup(const char *relative_path) { char cgroup_path[PATH_MAX + 1]; - format_cgroup_path(cgroup_path, path); + format_parent_cgroup_path(cgroup_path, relative_path); return join_cgroup_from_top(cgroup_path); } @@ -212,9 +314,27 @@ void cleanup_cgroup_environment(void) nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH | FTW_MOUNT); } +/** + * get_root_cgroup() - Get the FD of the root cgroup + * + * On success, it returns the file descriptor. On failure, it returns -1. + * If there is a failure, it prints the error to stderr. + */ +int get_root_cgroup(void) +{ + int fd; + + fd = open(CGROUP_MOUNT_PATH, O_RDONLY); + if (fd < 0) { + log_err("Opening root cgroup"); + return -1; + } + return fd; +} + /** * create_and_get_cgroup() - Create a cgroup, relative to workdir, and get the FD - * @path: The cgroup path, relative to the workdir, to join + * @relative_path: The cgroup path, relative to the workdir, to join * * This function creates a cgroup under the top level workdir and returns the * file descriptor. It is idempotent. @@ -222,14 +342,14 @@ void cleanup_cgroup_environment(void) * On success, it returns the file descriptor. On failure it returns -1. * If there is a failure, it prints the error to stderr. */ -int create_and_get_cgroup(const char *path) +int create_and_get_cgroup(const char *relative_path) { char cgroup_path[PATH_MAX + 1]; int fd; - format_cgroup_path(cgroup_path, path); + format_cgroup_path(cgroup_path, relative_path); if (mkdir(cgroup_path, 0777) && errno != EEXIST) { - log_err("mkdiring cgroup %s .. %s", path, cgroup_path); + log_err("mkdiring cgroup %s .. %s", relative_path, cgroup_path); return -1; } @@ -244,13 +364,13 @@ int create_and_get_cgroup(const char *path) /** * get_cgroup_id() - Get cgroup id for a particular cgroup path - * @path: The cgroup path, relative to the workdir, to join + * @relative_path: The cgroup path, relative to the workdir, to join * * On success, it returns the cgroup id. On failure it returns 0, * which is an invalid cgroup id. * If there is a failure, it prints the error to stderr. */ -unsigned long long get_cgroup_id(const char *path) +unsigned long long get_cgroup_id(const char *relative_path) { int dirfd, err, flags, mount_id, fhsize; union { @@ -261,7 +381,7 @@ unsigned long long get_cgroup_id(const char *path) struct file_handle *fhp, *fhp2; unsigned long long ret = 0; - format_cgroup_path(cgroup_workdir, path); + format_cgroup_path(cgroup_workdir, relative_path); dirfd = AT_FDCWD; flags = 0; diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h index fcc9cb91b21112a0b3d043c84a55172e17f2e129..3358734356ab7fd09d062bc4f11bc4f64a9ddded 100644 --- a/tools/testing/selftests/bpf/cgroup_helpers.h +++ b/tools/testing/selftests/bpf/cgroup_helpers.h @@ -10,11 +10,18 @@ __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) /* cgroupv2 related */ -int cgroup_setup_and_join(const char *path); -int create_and_get_cgroup(const char *path); -unsigned long long get_cgroup_id(const char *path); - -int join_cgroup(const char *path); +int enable_controllers(const char *relative_path, const char *controllers); +int write_cgroup_file(const char *relative_path, const char *file, + const char *buf); +int write_cgroup_file_parent(const char *relative_path, const char *file, + const char *buf); +int cgroup_setup_and_join(const char *relative_path); +int get_root_cgroup(void); +int create_and_get_cgroup(const char *relative_path); +unsigned long long get_cgroup_id(const char *relative_path); + +int join_cgroup(const char *relative_path); +int join_parent_cgroup(const char *relative_path); int setup_cgroup_environment(void); void cleanup_cgroup_environment(void); @@ -26,4 +33,4 @@ int join_classid(void); int setup_classid_environment(void); void cleanup_classid_environment(void); -#endif /* __CGROUP_HELPERS_H */ \ No newline at end of file +#endif /* __CGROUP_HELPERS_H */ diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index fabf0c0143498a92df5bc6599e205a6ca1633ffb..9213565c0311720d1446ed6c0c5e48e24f3d6ff7 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -7,9 +7,9 @@ CONFIG_BPF_LSM=y CONFIG_BPF_STREAM_PARSER=y CONFIG_BPF_SYSCALL=y CONFIG_CGROUP_BPF=y -CONFIG_CRYPTO_HMAC=m -CONFIG_CRYPTO_SHA256=m -CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_USER_API_HASH=y CONFIG_DYNAMIC_FTRACE=y CONFIG_FPROBE=y CONFIG_FTRACE_SYSCALLS=y @@ -24,41 +24,50 @@ CONFIG_IP_NF_FILTER=y CONFIG_IP_NF_RAW=y CONFIG_IP_NF_TARGET_SYNPROXY=y CONFIG_IPV6=y -CONFIG_IPV6_FOU=m -CONFIG_IPV6_FOU_TUNNEL=m +CONFIG_IPV6_FOU=y +CONFIG_IPV6_FOU_TUNNEL=y CONFIG_IPV6_GRE=y CONFIG_IPV6_SEG6_BPF=y -CONFIG_IPV6_SIT=m +CONFIG_IPV6_SIT=y CONFIG_IPV6_TUNNEL=y +CONFIG_KEYS=y CONFIG_LIRC=y CONFIG_LWTUNNEL=y +CONFIG_MODULE_SIG=y +CONFIG_MODULE_SRCVERSION_ALL=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULES=y +CONFIG_MODVERSIONS=y CONFIG_MPLS=y -CONFIG_MPLS_IPTUNNEL=m -CONFIG_MPLS_ROUTING=m +CONFIG_MPLS_IPTUNNEL=y +CONFIG_MPLS_ROUTING=y CONFIG_MPTCP=y CONFIG_NET_CLS_ACT=y CONFIG_NET_CLS_BPF=y -CONFIG_NET_CLS_FLOWER=m -CONFIG_NET_FOU=m +CONFIG_NET_CLS_FLOWER=y +CONFIG_NET_FOU=y CONFIG_NET_FOU_IP_TUNNELS=y CONFIG_NET_IPGRE=y CONFIG_NET_IPGRE_DEMUX=y CONFIG_NET_IPIP=y -CONFIG_NET_MPLS_GSO=m +CONFIG_NET_MPLS_GSO=y CONFIG_NET_SCH_INGRESS=y CONFIG_NET_SCHED=y -CONFIG_NETDEVSIM=m +CONFIG_NETDEVSIM=y CONFIG_NETFILTER=y CONFIG_NETFILTER_SYNPROXY=y +CONFIG_NETFILTER_XT_CONNMARK=y CONFIG_NETFILTER_XT_MATCH_STATE=y CONFIG_NETFILTER_XT_TARGET_CT=y CONFIG_NF_CONNTRACK=y +CONFIG_NF_CONNTRACK_MARK=y CONFIG_NF_DEFRAG_IPV4=y CONFIG_NF_DEFRAG_IPV6=y +CONFIG_NF_NAT=y CONFIG_RC_CORE=y CONFIG_SECURITY=y CONFIG_SECURITYFS=y -CONFIG_TEST_BPF=m +CONFIG_TEST_BPF=y CONFIG_USERFAULTFD=y CONFIG_VXLAN=y CONFIG_XDP_SOCKETS=y diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64 index f0859a1d37abfe2ac4df1338a794a6a3d8f5aa99..21ce5ea4304e72cc1af54cb81773750206af1de7 100644 --- a/tools/testing/selftests/bpf/config.x86_64 +++ b/tools/testing/selftests/bpf/config.x86_64 @@ -47,7 +47,7 @@ CONFIG_CPU_IDLE_GOV_LADDER=y CONFIG_CPUSETS=y CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_BLAKE2B=y -CONFIG_CRYPTO_DEV_VIRTIO=m +CONFIG_CRYPTO_DEV_VIRTIO=y CONFIG_CRYPTO_SEQIV=y CONFIG_CRYPTO_XXHASH=y CONFIG_DCB=y @@ -145,11 +145,6 @@ CONFIG_MCORE2=y CONFIG_MEMCG=y CONFIG_MEMORY_FAILURE=y CONFIG_MINIX_SUBPARTITION=y -CONFIG_MODULE_SIG=y -CONFIG_MODULE_SRCVERSION_ALL=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULES=y -CONFIG_MODVERSIONS=y CONFIG_NAMESPACES=y CONFIG_NET=y CONFIG_NET_9P=y diff --git a/tools/testing/selftests/bpf/get_cgroup_id_user.c b/tools/testing/selftests/bpf/get_cgroup_id_user.c index e021cc67dc0290ea09f09e9d540995e3f7314382..156743cf58705724658f3523a12b454ee6face38 100644 --- a/tools/testing/selftests/bpf/get_cgroup_id_user.c +++ b/tools/testing/selftests/bpf/get_cgroup_id_user.c @@ -48,7 +48,7 @@ static int bpf_find_map(const char *test, struct bpf_object *obj, int main(int argc, char **argv) { const char *probe_name = "syscalls/sys_enter_nanosleep"; - const char *file = "get_cgroup_id_kern.o"; + const char *file = "get_cgroup_id_kern.bpf.o"; int err, bytes, efd, prog_fd, pmu_fd; int cgroup_fd, cgidmap_fd, pidmap_fd; struct perf_event_attr attr = {}; diff --git a/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c index 78c76496b14ad7349e2dd5c2ff36da75a4244b12..b595556315bc3ca3792cb56bceeef092191847b4 100644 --- a/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c +++ b/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c @@ -3,6 +3,7 @@ #include <stdio.h> #include <errno.h> #include <string.h> +#include <unistd.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> @@ -137,6 +138,7 @@ static void __test_map_lookup_and_update_batch(bool is_pcpu) free(keys); free(values); free(visited); + close(map_fd); } static void array_map_batch_ops(void) diff --git a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c index f807d53fd8dd46d2fa8a02733fb72de1e68bc5bd..1230ccf901280d1cd4d4e4bfc9ebd6d0d26d2650 100644 --- a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c +++ b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c @@ -3,6 +3,7 @@ #include <stdio.h> #include <errno.h> #include <string.h> +#include <unistd.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> @@ -255,6 +256,7 @@ void __test_map_lookup_and_delete_batch(bool is_pcpu) free(visited); if (!is_pcpu) free(values); + close(map_fd); } void htab_map_batch_ops(void) diff --git a/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c index 87d07b596e1707c17c50b373e89baddb50707bdc..b66d56ddb7ef2d3daf18774fe7418c1dc475fce0 100644 --- a/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c +++ b/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c @@ -7,6 +7,7 @@ #include <errno.h> #include <string.h> #include <stdlib.h> +#include <unistd.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> @@ -150,4 +151,5 @@ void test_lpm_trie_map_batch_ops(void) free(keys); free(values); free(visited); + close(map_fd); } diff --git a/tools/testing/selftests/bpf/map_tests/task_storage_map.c b/tools/testing/selftests/bpf/map_tests/task_storage_map.c new file mode 100644 index 0000000000000000000000000000000000000000..7d050364efca1a34a25e543ddc8588d2c2d36049 --- /dev/null +++ b/tools/testing/selftests/bpf/map_tests/task_storage_map.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022. Huawei Technologies Co., Ltd */ +#define _GNU_SOURCE +#include <sched.h> +#include <unistd.h> +#include <stdlib.h> +#include <stdbool.h> +#include <errno.h> +#include <string.h> +#include <pthread.h> + +#include <bpf/bpf.h> +#include <bpf/libbpf.h> + +#include "test_maps.h" +#include "task_local_storage_helpers.h" +#include "read_bpf_task_storage_busy.skel.h" + +struct lookup_ctx { + bool start; + bool stop; + int pid_fd; + int map_fd; + int loop; +}; + +static void *lookup_fn(void *arg) +{ + struct lookup_ctx *ctx = arg; + long value; + int i = 0; + + while (!ctx->start) + usleep(1); + + while (!ctx->stop && i++ < ctx->loop) + bpf_map_lookup_elem(ctx->map_fd, &ctx->pid_fd, &value); + return NULL; +} + +static void abort_lookup(struct lookup_ctx *ctx, pthread_t *tids, unsigned int nr) +{ + unsigned int i; + + ctx->stop = true; + ctx->start = true; + for (i = 0; i < nr; i++) + pthread_join(tids[i], NULL); +} + +void test_task_storage_map_stress_lookup(void) +{ +#define MAX_NR_THREAD 4096 + unsigned int i, nr = 256, loop = 8192, cpu = 0; + struct read_bpf_task_storage_busy *skel; + pthread_t tids[MAX_NR_THREAD]; + struct lookup_ctx ctx; + cpu_set_t old, new; + const char *cfg; + int err; + + cfg = getenv("TASK_STORAGE_MAP_NR_THREAD"); + if (cfg) { + nr = atoi(cfg); + if (nr > MAX_NR_THREAD) + nr = MAX_NR_THREAD; + } + cfg = getenv("TASK_STORAGE_MAP_NR_LOOP"); + if (cfg) + loop = atoi(cfg); + cfg = getenv("TASK_STORAGE_MAP_PIN_CPU"); + if (cfg) + cpu = atoi(cfg); + + skel = read_bpf_task_storage_busy__open_and_load(); + err = libbpf_get_error(skel); + CHECK(err, "open_and_load", "error %d\n", err); + + /* Only for a fully preemptible kernel */ + if (!skel->kconfig->CONFIG_PREEMPT) { + printf("%s SKIP (no CONFIG_PREEMPT)\n", __func__); + read_bpf_task_storage_busy__destroy(skel); + skips++; + return; + } + + /* Save the old affinity setting */ + sched_getaffinity(getpid(), sizeof(old), &old); + + /* Pinned on a specific CPU */ + CPU_ZERO(&new); + CPU_SET(cpu, &new); + sched_setaffinity(getpid(), sizeof(new), &new); + + ctx.start = false; + ctx.stop = false; + ctx.pid_fd = sys_pidfd_open(getpid(), 0); + ctx.map_fd = bpf_map__fd(skel->maps.task); + ctx.loop = loop; + for (i = 0; i < nr; i++) { + err = pthread_create(&tids[i], NULL, lookup_fn, &ctx); + if (err) { + abort_lookup(&ctx, tids, i); + CHECK(err, "pthread_create", "error %d\n", err); + goto out; + } + } + + ctx.start = true; + for (i = 0; i < nr; i++) + pthread_join(tids[i], NULL); + + skel->bss->pid = getpid(); + err = read_bpf_task_storage_busy__attach(skel); + CHECK(err, "attach", "error %d\n", err); + + /* Trigger program */ + syscall(SYS_gettid); + skel->bss->pid = 0; + + CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy); +out: + read_bpf_task_storage_busy__destroy(skel); + /* Restore affinity setting */ + sched_setaffinity(getpid(), sizeof(old), &old); + printf("%s:PASS\n", __func__); +} diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index 0b899d2d8ea70fa39e3a171e77536d6ed538790f..9566d9d2f6eebecbd5eda3a3825b8edca3510665 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -6,19 +6,19 @@ volatile unsigned short uprobe_ref_ctr __attribute__((unused)) __attribute((section(".probes"))); /* uprobe attach point */ -static void trigger_func(void) +static noinline void trigger_func(void) { asm volatile (""); } /* attach point for byname uprobe */ -static void trigger_func2(void) +static noinline void trigger_func2(void) { asm volatile (""); } /* attach point for byname sleepable uprobe */ -static void trigger_func3(void) +static noinline void trigger_func3(void) { asm volatile (""); } diff --git a/tools/testing/selftests/bpf/prog_tests/autoattach.c b/tools/testing/selftests/bpf/prog_tests/autoattach.c new file mode 100644 index 0000000000000000000000000000000000000000..dc5e01d279bdb40701d8e9ec0357e4e542033935 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/autoattach.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Google */ + +#include <test_progs.h> +#include "test_autoattach.skel.h" + +void test_autoattach(void) +{ + struct test_autoattach *skel; + + skel = test_autoattach__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto cleanup; + + /* disable auto-attach for prog2 */ + bpf_program__set_autoattach(skel->progs.prog2, false); + ASSERT_TRUE(bpf_program__autoattach(skel->progs.prog1), "autoattach_prog1"); + ASSERT_FALSE(bpf_program__autoattach(skel->progs.prog2), "autoattach_prog2"); + if (!ASSERT_OK(test_autoattach__attach(skel), "skel_attach")) + goto cleanup; + + usleep(1); + + ASSERT_TRUE(skel->bss->prog1_called, "attached_prog1"); + ASSERT_FALSE(skel->bss->prog2_called, "attached_prog2"); + +cleanup: + test_autoattach__destroy(skel); +} + diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c index 2974b44f80faf17a16ab6a012dac07ad925a1f6e..2be2d61954bc63f0f77fce9a72aae7885c75c103 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c @@ -13,7 +13,7 @@ #include "kprobe_multi.skel.h" /* uprobe attach point */ -static void trigger_func(void) +static noinline void trigger_func(void) { asm volatile (""); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index e89685bd587ccfb6f8a9ef4985d45cdce33273e6..3369c5ec3a17c9eab9f9e024b804bb4fa3756607 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2020 Facebook */ #include <test_progs.h> +#include <unistd.h> +#include <sys/syscall.h> #include "bpf_iter_ipv6_route.skel.h" #include "bpf_iter_netlink.skel.h" #include "bpf_iter_bpf_map.skel.h" @@ -14,6 +16,7 @@ #include "bpf_iter_udp4.skel.h" #include "bpf_iter_udp6.skel.h" #include "bpf_iter_unix.skel.h" +#include "bpf_iter_vma_offset.skel.h" #include "bpf_iter_test_kern1.skel.h" #include "bpf_iter_test_kern2.skel.h" #include "bpf_iter_test_kern3.skel.h" @@ -43,13 +46,13 @@ static void test_btf_id_or_null(void) } } -static void do_dummy_read(struct bpf_program *prog) +static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts) { struct bpf_link *link; char buf[16] = {}; int iter_fd, len; - link = bpf_program__attach_iter(prog, NULL); + link = bpf_program__attach_iter(prog, opts); if (!ASSERT_OK_PTR(link, "attach_iter")) return; @@ -68,6 +71,11 @@ free_link: bpf_link__destroy(link); } +static void do_dummy_read(struct bpf_program *prog) +{ + do_dummy_read_opts(prog, NULL); +} + static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog, struct bpf_map *map) { @@ -167,19 +175,140 @@ static void test_bpf_map(void) bpf_iter_bpf_map__destroy(skel); } -static void test_task(void) +static int pidfd_open(pid_t pid, unsigned int flags) +{ + return syscall(SYS_pidfd_open, pid, flags); +} + +static void check_bpf_link_info(const struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + struct bpf_link_info info = {}; + struct bpf_link *link; + __u32 info_len; + int err; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.tid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(prog, &opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + return; + + info_len = sizeof(info); + err = bpf_obj_get_info_by_fd(bpf_link__fd(link), &info, &info_len); + ASSERT_OK(err, "bpf_obj_get_info_by_fd"); + ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid"); + + bpf_link__destroy(link); +} + +static pthread_mutex_t do_nothing_mutex; + +static void *do_nothing_wait(void *arg) +{ + pthread_mutex_lock(&do_nothing_mutex); + pthread_mutex_unlock(&do_nothing_mutex); + + pthread_exit(arg); +} + +static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts, + int *num_unknown, int *num_known) { struct bpf_iter_task *skel; + pthread_t thread_id; + void *ret; skel = bpf_iter_task__open_and_load(); if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load")) return; - do_dummy_read(skel->progs.dump_task); + ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock"); + + ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL), + "pthread_create"); + + skel->bss->tid = getpid(); + + do_dummy_read_opts(skel->progs.dump_task, opts); + + *num_unknown = skel->bss->num_unknown_tid; + *num_known = skel->bss->num_known_tid; + + ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock"); + ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL, + "pthread_join"); bpf_iter_task__destroy(skel); } +static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known) +{ + int num_unknown_tid, num_known_tid; + + test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid); + ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid"); + ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid"); +} + +static void test_task_tid(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + int num_unknown_tid, num_known_tid; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.tid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + test_task_common(&opts, 0, 1); + + linfo.task.tid = 0; + linfo.task.pid = getpid(); + test_task_common(&opts, 1, 1); + + test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid); + ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid"); + ASSERT_EQ(num_known_tid, 1, "check_num_known_tid"); +} + +static void test_task_pid(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.pid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + test_task_common(&opts, 1, 1); +} + +static void test_task_pidfd(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + int pidfd; + + pidfd = pidfd_open(getpid(), 0); + if (!ASSERT_GT(pidfd, 0, "pidfd_open")) + return; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.pid_fd = pidfd; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + test_task_common(&opts, 1, 1); + + close(pidfd); +} + static void test_task_sleepable(void) { struct bpf_iter_task *skel; @@ -212,14 +341,11 @@ static void test_task_stack(void) bpf_iter_task_stack__destroy(skel); } -static void *do_nothing(void *arg) -{ - pthread_exit(arg); -} - static void test_task_file(void) { + LIBBPF_OPTS(bpf_iter_attach_opts, opts); struct bpf_iter_task_file *skel; + union bpf_iter_link_info linfo; pthread_t thread_id; void *ret; @@ -229,19 +355,36 @@ static void test_task_file(void) skel->bss->tgid = getpid(); - if (!ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing, NULL), - "pthread_create")) - goto done; + ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock"); - do_dummy_read(skel->progs.dump_task_file); + ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL), + "pthread_create"); + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.tid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); - if (!ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL, - "pthread_join")) - goto done; + do_dummy_read_opts(skel->progs.dump_task_file, &opts); ASSERT_EQ(skel->bss->count, 0, "check_count"); + ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count"); + + skel->bss->last_tgid = 0; + skel->bss->count = 0; + skel->bss->unique_tgid_count = 0; + + do_dummy_read(skel->progs.dump_task_file); + + ASSERT_EQ(skel->bss->count, 0, "check_count"); + ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count"); + + check_bpf_link_info(skel->progs.dump_task_file); + + ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock"); + ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join"); + ASSERT_NULL(ret, "pthread_join"); -done: bpf_iter_task_file__destroy(skel); } @@ -1249,7 +1392,7 @@ static void str_strip_first_line(char *str) *dst = '\0'; } -static void test_task_vma(void) +static void test_task_vma_common(struct bpf_iter_attach_opts *opts) { int err, iter_fd = -1, proc_maps_fd = -1; struct bpf_iter_task_vma *skel; @@ -1261,13 +1404,14 @@ static void test_task_vma(void) return; skel->bss->pid = getpid(); + skel->bss->one_task = opts ? 1 : 0; err = bpf_iter_task_vma__load(skel); if (!ASSERT_OK(err, "bpf_iter_task_vma__load")) goto out; skel->links.proc_maps = bpf_program__attach_iter( - skel->progs.proc_maps, NULL); + skel->progs.proc_maps, opts); if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) { skel->links.proc_maps = NULL; @@ -1291,6 +1435,8 @@ static void test_task_vma(void) goto out; len += err; } + if (opts) + ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task"); /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */ snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid); @@ -1306,6 +1452,9 @@ static void test_task_vma(void) str_strip_first_line(proc_maps_output); ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output"); + + check_bpf_link_info(skel->progs.proc_maps); + out: close(proc_maps_fd); close(iter_fd); @@ -1325,8 +1474,93 @@ void test_bpf_sockmap_map_iter_fd(void) bpf_iter_sockmap__destroy(skel); } +static void test_task_vma(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.tid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + test_task_vma_common(&opts); + test_task_vma_common(NULL); +} + +/* uprobe attach point */ +static noinline int trigger_func(int arg) +{ + asm volatile (""); + return arg + 1; +} + +static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc) +{ + struct bpf_iter_vma_offset *skel; + struct bpf_link *link; + char buf[16] = {}; + int iter_fd, len; + int pgsz, shift; + + skel = bpf_iter_vma_offset__open_and_load(); + if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load")) + return; + + skel->bss->pid = getpid(); + skel->bss->address = (uintptr_t)trigger_func; + for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++) + ; + skel->bss->page_shift = shift; + + link = bpf_program__attach_iter(skel->progs.get_vma_offset, opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + return; + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GT(iter_fd, 0, "create_iter")) + goto exit; + + while ((len = read(iter_fd, buf, sizeof(buf))) > 0) + ; + buf[15] = 0; + ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp"); + + ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset"); + if (one_proc) + ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count"); + else + ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count"); + + close(iter_fd); + +exit: + bpf_link__destroy(link); +} + +static void test_task_vma_offset(void) +{ + LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + + memset(&linfo, 0, sizeof(linfo)); + linfo.task.pid = getpid(); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + test_task_vma_offset_common(&opts, true); + + linfo.task.pid = 0; + linfo.task.tid = getpid(); + test_task_vma_offset_common(&opts, true); + + test_task_vma_offset_common(NULL, false); +} + void test_bpf_iter(void) { + ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init"); + if (test__start_subtest("btf_id_or_null")) test_btf_id_or_null(); if (test__start_subtest("ipv6_route")) @@ -1335,8 +1569,12 @@ void test_bpf_iter(void) test_netlink(); if (test__start_subtest("bpf_map")) test_bpf_map(); - if (test__start_subtest("task")) - test_task(); + if (test__start_subtest("task_tid")) + test_task_tid(); + if (test__start_subtest("task_pid")) + test_task_pid(); + if (test__start_subtest("task_pidfd")) + test_task_pidfd(); if (test__start_subtest("task_sleepable")) test_task_sleepable(); if (test__start_subtest("task_stack")) @@ -1397,4 +1635,6 @@ void test_bpf_iter(void) test_ksym_iter(); if (test__start_subtest("bpf_sockmap_map_iter_fd")) test_bpf_sockmap_map_iter_fd(); + if (test__start_subtest("vma_offset")) + test_task_vma_offset(); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c index 7a74a1579076e645dcc7ed65d580f48de5d7c335..8a838ea8bdf3b2a7c51513ab0c46305fb5f2b7b8 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <network_helpers.h> +#include <linux/netfilter/nf_conntrack_common.h> #include "test_bpf_nf.skel.h" #include "test_bpf_nf_fail.skel.h" @@ -17,6 +18,7 @@ struct { { "set_status_after_insert", "kernel function bpf_ct_set_status args#0 expected pointer to STRUCT nf_conn___init but" }, { "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout args#0 expected pointer to STRUCT nf_conn but" }, { "change_status_after_alloc", "kernel function bpf_ct_change_status args#0 expected pointer to STRUCT nf_conn but" }, + { "write_not_allowlisted_field", "no write support to nf_conn at off" }, }; enum { @@ -24,10 +26,37 @@ enum { TEST_TC_BPF, }; +#define TIMEOUT_MS 3000 +#define IPS_STATUS_MASK (IPS_CONFIRMED | IPS_SEEN_REPLY | \ + IPS_SRC_NAT_DONE | IPS_DST_NAT_DONE | \ + IPS_SRC_NAT | IPS_DST_NAT) + +static int connect_to_server(int srv_fd) +{ + int fd = -1; + + fd = socket(AF_INET, SOCK_STREAM, 0); + if (!ASSERT_GE(fd, 0, "socket")) + goto out; + + if (!ASSERT_EQ(connect_fd_to_fd(fd, srv_fd, TIMEOUT_MS), 0, "connect_fd_to_fd")) { + close(fd); + fd = -1; + } +out: + return fd; +} + static void test_bpf_nf_ct(int mode) { + const char *iptables = "iptables -t raw %s PREROUTING -j CONNMARK --set-mark 42/0"; + int srv_fd = -1, client_fd = -1, srv_client_fd = -1; + struct sockaddr_in peer_addr = {}; struct test_bpf_nf *skel; int prog_fd, err; + socklen_t len; + u16 srv_port; + char cmd[64]; LIBBPF_OPTS(bpf_test_run_opts, topts, .data_in = &pkt_v4, .data_size_in = sizeof(pkt_v4), @@ -38,6 +67,32 @@ static void test_bpf_nf_ct(int mode) if (!ASSERT_OK_PTR(skel, "test_bpf_nf__open_and_load")) return; + /* Enable connection tracking */ + snprintf(cmd, sizeof(cmd), iptables, "-A"); + if (!ASSERT_OK(system(cmd), "iptables")) + goto end; + + srv_port = (mode == TEST_XDP) ? 5005 : 5006; + srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", srv_port, TIMEOUT_MS); + if (!ASSERT_GE(srv_fd, 0, "start_server")) + goto end; + + client_fd = connect_to_server(srv_fd); + if (!ASSERT_GE(client_fd, 0, "connect_to_server")) + goto end; + + len = sizeof(peer_addr); + srv_client_fd = accept(srv_fd, (struct sockaddr *)&peer_addr, &len); + if (!ASSERT_GE(srv_client_fd, 0, "accept")) + goto end; + if (!ASSERT_EQ(len, sizeof(struct sockaddr_in), "sockaddr len")) + goto end; + + skel->bss->saddr = peer_addr.sin_addr.s_addr; + skel->bss->sport = peer_addr.sin_port; + skel->bss->daddr = peer_addr.sin_addr.s_addr; + skel->bss->dport = htons(srv_port); + if (mode == TEST_XDP) prog_fd = bpf_program__fd(skel->progs.nf_xdp_ct_test); else @@ -61,9 +116,21 @@ static void test_bpf_nf_ct(int mode) /* allow some tolerance for test_delta_timeout value to avoid races. */ ASSERT_GT(skel->bss->test_delta_timeout, 8, "Test for min ct timeout update"); ASSERT_LE(skel->bss->test_delta_timeout, 10, "Test for max ct timeout update"); - /* expected status is IPS_SEEN_REPLY */ - ASSERT_EQ(skel->bss->test_status, 2, "Test for ct status update "); + ASSERT_EQ(skel->bss->test_insert_lookup_mark, 77, "Test for insert and lookup mark value"); + ASSERT_EQ(skel->bss->test_status, IPS_STATUS_MASK, "Test for ct status update "); + ASSERT_EQ(skel->data->test_exist_lookup, 0, "Test existing connection lookup"); + ASSERT_EQ(skel->bss->test_exist_lookup_mark, 43, "Test existing connection lookup ctmark"); + ASSERT_EQ(skel->data->test_snat_addr, 0, "Test for source natting"); + ASSERT_EQ(skel->data->test_dnat_addr, 0, "Test for destination natting"); end: + if (srv_client_fd != -1) + close(srv_client_fd); + if (client_fd != -1) + close(client_fd); + if (srv_fd != -1) + close(srv_fd); + snprintf(cmd, sizeof(cmd), iptables, "-D"); + system(cmd); test_bpf_nf__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c index dbe56fa8582d6782f90f8334897bb1a84779939c..e1c1e521cca2cf85af058d7ecf013f70d4899cd7 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c @@ -7,7 +7,7 @@ void serial_test_bpf_obj_id(void) { const __u64 array_magic_value = 0xfaceb00c; const __u32 array_key = 0; - const char *file = "./test_obj_id.o"; + const char *file = "./test_obj_id.bpf.o"; const char *expected_prog_name = "test_obj_id"; const char *expected_map_name = "test_map_id"; const __u64 nsec_per_sec = 1000000000; diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c index 2959a52ced06067ba7d99fbe0b288c41c45aee99..e980188d41246fac59df6dbcbb610e44b6b02a1d 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c @@ -290,6 +290,10 @@ static void test_dctcp_fallback(void) goto done; ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res"); ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res"); + /* All setsockopt(TCP_CONGESTION) in the recurred + * bpf_dctcp->init() should fail with -EBUSY. + */ + ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt"); err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len); if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)")) diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index ff6cce9fef065066be9cda13cc3665909fdfdf86..5ca252823294b6dd35c3fbb7fc977a3e015bd583 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -75,45 +75,45 @@ static void scale_test(const char *file, void test_verif_scale1() { - scale_test("test_verif_scale1.o", BPF_PROG_TYPE_SCHED_CLS, false); + scale_test("test_verif_scale1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false); } void test_verif_scale2() { - scale_test("test_verif_scale2.o", BPF_PROG_TYPE_SCHED_CLS, false); + scale_test("test_verif_scale2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false); } void test_verif_scale3() { - scale_test("test_verif_scale3.o", BPF_PROG_TYPE_SCHED_CLS, false); + scale_test("test_verif_scale3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false); } void test_verif_scale_pyperf_global() { - scale_test("pyperf_global.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf_global.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf_subprogs() { - scale_test("pyperf_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf50() { /* full unroll by llvm */ - scale_test("pyperf50.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf50.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf100() { /* full unroll by llvm */ - scale_test("pyperf100.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf100.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf180() { /* full unroll by llvm */ - scale_test("pyperf180.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf180.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf600() @@ -124,13 +124,13 @@ void test_verif_scale_pyperf600() * 16k insns in loop body. * Total of 5 such loops. Total program size ~82k insns. */ - scale_test("pyperf600.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf600.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf600_bpf_loop(void) { /* use the bpf_loop helper*/ - scale_test("pyperf600_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf600_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_pyperf600_nounroll() @@ -141,37 +141,37 @@ void test_verif_scale_pyperf600_nounroll() * ~110 insns in loop body. * Total of 5 such loops. Total program size ~1500 insns. */ - scale_test("pyperf600_nounroll.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("pyperf600_nounroll.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_loop1() { - scale_test("loop1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("loop1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_loop2() { - scale_test("loop2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("loop2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_loop3_fail() { - scale_test("loop3.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */); + scale_test("loop3.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */); } void test_verif_scale_loop4() { - scale_test("loop4.o", BPF_PROG_TYPE_SCHED_CLS, false); + scale_test("loop4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false); } void test_verif_scale_loop5() { - scale_test("loop5.o", BPF_PROG_TYPE_SCHED_CLS, false); + scale_test("loop5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false); } void test_verif_scale_loop6() { - scale_test("loop6.o", BPF_PROG_TYPE_KPROBE, false); + scale_test("loop6.bpf.o", BPF_PROG_TYPE_KPROBE, false); } void test_verif_scale_strobemeta() @@ -180,54 +180,54 @@ void test_verif_scale_strobemeta() * Total program size 20.8k insn. * ~350k processed_insns */ - scale_test("strobemeta.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("strobemeta.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_strobemeta_bpf_loop(void) { /* use the bpf_loop helper*/ - scale_test("strobemeta_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("strobemeta_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_strobemeta_nounroll1() { /* no unroll, tiny loops */ - scale_test("strobemeta_nounroll1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("strobemeta_nounroll1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_strobemeta_nounroll2() { /* no unroll, tiny loops */ - scale_test("strobemeta_nounroll2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("strobemeta_nounroll2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_strobemeta_subprogs() { /* non-inlined subprogs */ - scale_test("strobemeta_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); + scale_test("strobemeta_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false); } void test_verif_scale_sysctl_loop1() { - scale_test("test_sysctl_loop1.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false); + scale_test("test_sysctl_loop1.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false); } void test_verif_scale_sysctl_loop2() { - scale_test("test_sysctl_loop2.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false); + scale_test("test_sysctl_loop2.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false); } void test_verif_scale_xdp_loop() { - scale_test("test_xdp_loop.o", BPF_PROG_TYPE_XDP, false); + scale_test("test_xdp_loop.bpf.o", BPF_PROG_TYPE_XDP, false); } void test_verif_scale_seg6_loop() { - scale_test("test_seg6_loop.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false); + scale_test("test_seg6_loop.bpf.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false); } void test_verif_twfw() { - scale_test("twfw.o", BPF_PROG_TYPE_CGROUP_SKB, false); + scale_test("twfw.bpf.o", BPF_PROG_TYPE_CGROUP_SKB, false); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index ef6528b8084c442d5f58a8dd0889adc65d887407..127b8caa3dc1efb36f949a5109f65f1b0ad19b45 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -4651,8 +4651,8 @@ struct btf_file_test { }; static struct btf_file_test file_tests[] = { - { .file = "test_btf_newkv.o", }, - { .file = "test_btf_nokv.o", .btf_kv_notfound = true, }, + { .file = "test_btf_newkv.bpf.o", }, + { .file = "test_btf_nokv.bpf.o", .btf_kv_notfound = true, }, }; static void do_test_file(unsigned int test_num) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 5fce7008d1ff32dc0c6379de1d375e336e9f0c54..24da335482d42173525e67826e33168ad32706d6 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -52,7 +52,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) int err = 0, fd = -1; FILE *f = NULL; - snprintf(test_file, sizeof(test_file), "%s.o", t->file); + snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file); btf = btf__parse_elf(test_file, NULL); if (!ASSERT_OK_PTR(btf, "btf_parse_elf")) { @@ -764,8 +764,8 @@ static void test_btf_dump_struct_data(struct btf *btf, struct btf_dump *d, /* union with nested struct */ TEST_BTF_DUMP_DATA(btf, d, "union", str, union bpf_iter_link_info, BTF_F_COMPACT, - "(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},}", - { .map = { .map_fd = 1 }}); + "(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},.cgroup = (struct){.order = (enum bpf_cgroup_iter_order)BPF_CGROUP_ITER_SELF_ONLY,.cgroup_fd = (__u32)1,},.task = (struct){.tid = (__u32)1,.pid = (__u32)1,},}", + { .cgroup = { .order = 1, .cgroup_fd = 1, }}); /* struct skb with nested structs/unions; because type output is so * complex, we don't do a string comparison, just verify we return @@ -841,8 +841,8 @@ static void test_btf_dump_datasec_data(char *str) char license[4] = "GPL"; struct btf_dump *d; - btf = btf__parse("xdping_kern.o", NULL); - if (!ASSERT_OK_PTR(btf, "xdping_kern.o BTF not found")) + btf = btf__parse("xdping_kern.bpf.o", NULL); + if (!ASSERT_OK_PTR(btf, "xdping_kern.bpf.o BTF not found")) return; d = btf_dump__new(btf, btf_dump_snprintf, str, NULL); diff --git a/tools/testing/selftests/bpf/prog_tests/btf_endian.c b/tools/testing/selftests/bpf/prog_tests/btf_endian.c index 8afbf3d0b89a22c38bd49c7d6e4214d1a5c0e0ac..5b9f84dbeb432f46d903d5d20d7cfe14f3e3f7d6 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_endian.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_endian.c @@ -23,7 +23,7 @@ void test_btf_endian() { int var_id; /* Load BTF in native endianness */ - btf = btf__parse_elf("btf_dump_test_case_syntax.o", NULL); + btf = btf__parse_elf("btf_dump_test_case_syntax.bpf.o", NULL); if (!ASSERT_OK_PTR(btf, "parse_native_btf")) goto err_out; diff --git a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c index 664ffc0364f4f0e3a4dea2ee7a46ba5eb61cd3d2..7a277035c275b99d28e9c63f8c00a35ee5ff7384 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c @@ -22,26 +22,6 @@ static __u32 duration; #define PROG_PIN_FILE "/sys/fs/bpf/btf_skc_cls_ingress" -static int write_sysctl(const char *sysctl, const char *value) -{ - int fd, err, len; - - fd = open(sysctl, O_WRONLY); - if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n", - sysctl, strerror(errno), errno)) - return -1; - - len = strlen(value); - err = write(fd, value, len); - close(fd); - if (CHECK(err != len, "write sysctl", - "write(%s, %s, %d): err:%d %s (%d)\n", - sysctl, value, len, err, strerror(errno), errno)) - return -1; - - return 0; -} - static int prepare_netns(void) { if (CHECK(unshare(CLONE_NEWNET), "create netns", diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c new file mode 100644 index 0000000000000000000000000000000000000000..3bff680de16c392c051bcc35abedd0d2c8d09de9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "bpf/libbpf.h" +#include <test_progs.h> +#include <network_helpers.h> + +#include "cb_refs.skel.h" + +static char log_buf[1024 * 1024]; + +struct { + const char *prog_name; + const char *err_msg; +} cb_refs_tests[] = { + { "underflow_prog", "reference has not been acquired before" }, + { "leak_prog", "Unreleased reference" }, + { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */ + { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */ +}; + +void test_cb_refs(void) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf, + .kernel_log_size = sizeof(log_buf), + .kernel_log_level = 1); + struct bpf_program *prog; + struct cb_refs *skel; + int i; + + for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) { + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + skel = cb_refs__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load")) + return; + prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name); + bpf_program__set_autoload(prog, true); + if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load")) + bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts); + if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) { + fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg); + fprintf(stderr, "Verifier: %s\n", log_buf); + } + cb_refs__destroy(skel); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c index 0b47c3c000c72b8b8208cfdc957ab7dceb75d180..4d2fa99273d8edfd0ceeac577ee84b6249408d7d 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c @@ -10,6 +10,7 @@ #include "cgroup_getset_retval_setsockopt.skel.h" #include "cgroup_getset_retval_getsockopt.skel.h" +#include "cgroup_getset_retval_hooks.skel.h" #define SOL_CUSTOM 0xdeadbeef @@ -433,6 +434,50 @@ close_bpf_object: cgroup_getset_retval_getsockopt__destroy(obj); } +struct exposed_hook { + const char *name; + int expected_err; +} exposed_hooks[] = { + +#define BPF_RETVAL_HOOK(NAME, SECTION, CTX, EXPECTED_ERR) \ + { \ + .name = #NAME, \ + .expected_err = EXPECTED_ERR, \ + }, + +#include "cgroup_getset_retval_hooks.h" + +#undef BPF_RETVAL_HOOK +}; + +static void test_exposed_hooks(int cgroup_fd, int sock_fd) +{ + struct cgroup_getset_retval_hooks *skel; + struct bpf_program *prog; + int err; + int i; + + for (i = 0; i < ARRAY_SIZE(exposed_hooks); i++) { + skel = cgroup_getset_retval_hooks__open(); + if (!ASSERT_OK_PTR(skel, "cgroup_getset_retval_hooks__open")) + continue; + + prog = bpf_object__find_program_by_name(skel->obj, exposed_hooks[i].name); + if (!ASSERT_NEQ(prog, NULL, "bpf_object__find_program_by_name")) + goto close_skel; + + err = bpf_program__set_autoload(prog, true); + if (!ASSERT_OK(err, "bpf_program__set_autoload")) + goto close_skel; + + err = cgroup_getset_retval_hooks__load(skel); + ASSERT_EQ(err, exposed_hooks[i].expected_err, "expected_err"); + +close_skel: + cgroup_getset_retval_hooks__destroy(skel); + } +} + void test_cgroup_getset_retval(void) { int cgroup_fd = -1; @@ -476,6 +521,9 @@ void test_cgroup_getset_retval(void) if (test__start_subtest("getsockopt-retval_sync")) test_getsockopt_retval_sync(cgroup_fd, sock_fd); + if (test__start_subtest("exposed_hooks")) + test_exposed_hooks(cgroup_fd, sock_fd); + close_fd: close(cgroup_fd); } diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c new file mode 100644 index 0000000000000000000000000000000000000000..3bd27d2ea668c986565c45e2a5641db45df99a0e --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This test makes sure BPF stats collection using rstat works correctly. + * The test uses 3 BPF progs: + * (a) counter: This BPF prog is invoked every time we attach a process to a + * cgroup and locklessly increments a percpu counter. + * The program then calls cgroup_rstat_updated() to inform rstat + * of an update on the (cpu, cgroup) pair. + * + * (b) flusher: This BPF prog is invoked when an rstat flush is ongoing, it + * aggregates all percpu counters to a total counter, and also + * propagates the changes to the ancestor cgroups. + * + * (c) dumper: This BPF prog is a cgroup_iter. It is used to output the total + * counter of a cgroup through reading a file in userspace. + * + * The test sets up a cgroup hierarchy, and the above programs. It spawns a few + * processes in the leaf cgroups and makes sure all the counters are aggregated + * correctly. + * + * Copyright 2022 Google LLC. + */ +#include <asm-generic/errno.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/mount.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <test_progs.h> +#include <bpf/libbpf.h> +#include <bpf/bpf.h> + +#include "cgroup_helpers.h" +#include "cgroup_hierarchical_stats.skel.h" + +#define PAGE_SIZE 4096 +#define MB(x) (x << 20) + +#define PROCESSES_PER_CGROUP 3 + +#define BPFFS_ROOT "/sys/fs/bpf/" +#define BPFFS_ATTACH_COUNTERS BPFFS_ROOT "attach_counters/" + +#define CG_ROOT_NAME "root" +#define CG_ROOT_ID 1 + +#define CGROUP_PATH(p, n) {.path = p"/"n, .name = n} + +static struct { + const char *path, *name; + unsigned long long id; + int fd; +} cgroups[] = { + CGROUP_PATH("/", "test"), + CGROUP_PATH("/test", "child1"), + CGROUP_PATH("/test", "child2"), + CGROUP_PATH("/test/child1", "child1_1"), + CGROUP_PATH("/test/child1", "child1_2"), + CGROUP_PATH("/test/child2", "child2_1"), + CGROUP_PATH("/test/child2", "child2_2"), +}; + +#define N_CGROUPS ARRAY_SIZE(cgroups) +#define N_NON_LEAF_CGROUPS 3 + +static int root_cgroup_fd; +static bool mounted_bpffs; + +/* reads file at 'path' to 'buf', returns 0 on success. */ +static int read_from_file(const char *path, char *buf, size_t size) +{ + int fd, len; + + fd = open(path, O_RDONLY); + if (fd < 0) + return fd; + + len = read(fd, buf, size); + close(fd); + if (len < 0) + return len; + + buf[len] = 0; + return 0; +} + +/* mounts bpffs and mkdir for reading stats, returns 0 on success. */ +static int setup_bpffs(void) +{ + int err; + + /* Mount bpffs */ + err = mount("bpf", BPFFS_ROOT, "bpf", 0, NULL); + mounted_bpffs = !err; + if (ASSERT_FALSE(err && errno != EBUSY, "mount")) + return err; + + /* Create a directory to contain stat files in bpffs */ + err = mkdir(BPFFS_ATTACH_COUNTERS, 0755); + if (!ASSERT_OK(err, "mkdir")) + return err; + + return 0; +} + +static void cleanup_bpffs(void) +{ + /* Remove created directory in bpffs */ + ASSERT_OK(rmdir(BPFFS_ATTACH_COUNTERS), "rmdir "BPFFS_ATTACH_COUNTERS); + + /* Unmount bpffs, if it wasn't already mounted when we started */ + if (mounted_bpffs) + return; + + ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs"); +} + +/* sets up cgroups, returns 0 on success. */ +static int setup_cgroups(void) +{ + int i, fd, err; + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return err; + + root_cgroup_fd = get_root_cgroup(); + if (!ASSERT_GE(root_cgroup_fd, 0, "get_root_cgroup")) + return root_cgroup_fd; + + for (i = 0; i < N_CGROUPS; i++) { + fd = create_and_get_cgroup(cgroups[i].path); + if (!ASSERT_GE(fd, 0, "create_and_get_cgroup")) + return fd; + + cgroups[i].fd = fd; + cgroups[i].id = get_cgroup_id(cgroups[i].path); + } + return 0; +} + +static void cleanup_cgroups(void) +{ + close(root_cgroup_fd); + for (int i = 0; i < N_CGROUPS; i++) + close(cgroups[i].fd); + cleanup_cgroup_environment(); +} + +/* Sets up cgroup hiearchary, returns 0 on success. */ +static int setup_hierarchy(void) +{ + return setup_bpffs() || setup_cgroups(); +} + +static void destroy_hierarchy(void) +{ + cleanup_cgroups(); + cleanup_bpffs(); +} + +static int attach_processes(void) +{ + int i, j, status; + + /* In every leaf cgroup, attach 3 processes */ + for (i = N_NON_LEAF_CGROUPS; i < N_CGROUPS; i++) { + for (j = 0; j < PROCESSES_PER_CGROUP; j++) { + pid_t pid; + + /* Create child and attach to cgroup */ + pid = fork(); + if (pid == 0) { + if (join_parent_cgroup(cgroups[i].path)) + exit(EACCES); + exit(0); + } + + /* Cleanup child */ + waitpid(pid, &status, 0); + if (!ASSERT_TRUE(WIFEXITED(status), "child process exited")) + return 1; + if (!ASSERT_EQ(WEXITSTATUS(status), 0, + "child process exit code")) + return 1; + } + } + return 0; +} + +static unsigned long long +get_attach_counter(unsigned long long cgroup_id, const char *file_name) +{ + unsigned long long attach_counter = 0, id = 0; + static char buf[128], path[128]; + + /* For every cgroup, read the file generated by cgroup_iter */ + snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, file_name); + if (!ASSERT_OK(read_from_file(path, buf, 128), "read cgroup_iter")) + return 0; + + /* Check the output file formatting */ + ASSERT_EQ(sscanf(buf, "cg_id: %llu, attach_counter: %llu\n", + &id, &attach_counter), 2, "output format"); + + /* Check that the cgroup_id is displayed correctly */ + ASSERT_EQ(id, cgroup_id, "cgroup_id"); + /* Check that the counter is non-zero */ + ASSERT_GT(attach_counter, 0, "attach counter non-zero"); + return attach_counter; +} + +static void check_attach_counters(void) +{ + unsigned long long attach_counters[N_CGROUPS], root_attach_counter; + int i; + + for (i = 0; i < N_CGROUPS; i++) + attach_counters[i] = get_attach_counter(cgroups[i].id, + cgroups[i].name); + + /* Read stats for root too */ + root_attach_counter = get_attach_counter(CG_ROOT_ID, CG_ROOT_NAME); + + /* Check that all leafs cgroups have an attach counter of 3 */ + for (i = N_NON_LEAF_CGROUPS; i < N_CGROUPS; i++) + ASSERT_EQ(attach_counters[i], PROCESSES_PER_CGROUP, + "leaf cgroup attach counter"); + + /* Check that child1 == child1_1 + child1_2 */ + ASSERT_EQ(attach_counters[1], attach_counters[3] + attach_counters[4], + "child1_counter"); + /* Check that child2 == child2_1 + child2_2 */ + ASSERT_EQ(attach_counters[2], attach_counters[5] + attach_counters[6], + "child2_counter"); + /* Check that test == child1 + child2 */ + ASSERT_EQ(attach_counters[0], attach_counters[1] + attach_counters[2], + "test_counter"); + /* Check that root >= test */ + ASSERT_GE(root_attach_counter, attach_counters[1], "root_counter"); +} + +/* Creates iter link and pins in bpffs, returns 0 on success, -errno on failure. + */ +static int setup_cgroup_iter(struct cgroup_hierarchical_stats *obj, + int cgroup_fd, const char *file_name) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo = {}; + struct bpf_link *link; + static char path[128]; + int err; + + /* + * Create an iter link, parameterized by cgroup_fd. We only want to + * traverse one cgroup, so set the traversal order to "self". + */ + linfo.cgroup.cgroup_fd = cgroup_fd; + linfo.cgroup.order = BPF_CGROUP_ITER_SELF_ONLY; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(obj->progs.dumper, &opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + return -EFAULT; + + /* Pin the link to a bpffs file */ + snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, file_name); + err = bpf_link__pin(link, path); + ASSERT_OK(err, "pin cgroup_iter"); + + /* Remove the link, leaving only the ref held by the pinned file */ + bpf_link__destroy(link); + return err; +} + +/* Sets up programs for collecting stats, returns 0 on success. */ +static int setup_progs(struct cgroup_hierarchical_stats **skel) +{ + int i, err; + + *skel = cgroup_hierarchical_stats__open_and_load(); + if (!ASSERT_OK_PTR(*skel, "open_and_load")) + return 1; + + /* Attach cgroup_iter program that will dump the stats to cgroups */ + for (i = 0; i < N_CGROUPS; i++) { + err = setup_cgroup_iter(*skel, cgroups[i].fd, cgroups[i].name); + if (!ASSERT_OK(err, "setup_cgroup_iter")) + return err; + } + + /* Also dump stats for root */ + err = setup_cgroup_iter(*skel, root_cgroup_fd, CG_ROOT_NAME); + if (!ASSERT_OK(err, "setup_cgroup_iter")) + return err; + + bpf_program__set_autoattach((*skel)->progs.dumper, false); + err = cgroup_hierarchical_stats__attach(*skel); + if (!ASSERT_OK(err, "attach")) + return err; + + return 0; +} + +static void destroy_progs(struct cgroup_hierarchical_stats *skel) +{ + static char path[128]; + int i; + + for (i = 0; i < N_CGROUPS; i++) { + /* Delete files in bpffs that cgroup_iters are pinned in */ + snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, + cgroups[i].name); + ASSERT_OK(remove(path), "remove cgroup_iter pin"); + } + + /* Delete root file in bpffs */ + snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, CG_ROOT_NAME); + ASSERT_OK(remove(path), "remove cgroup_iter root pin"); + cgroup_hierarchical_stats__destroy(skel); +} + +void test_cgroup_hierarchical_stats(void) +{ + struct cgroup_hierarchical_stats *skel = NULL; + + if (setup_hierarchy()) + goto hierarchy_cleanup; + if (setup_progs(&skel)) + goto cleanup; + if (attach_processes()) + goto cleanup; + check_attach_counters(); +cleanup: + destroy_progs(skel); +hierarchy_cleanup: + destroy_hierarchy(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c new file mode 100644 index 0000000000000000000000000000000000000000..c4a2adb38da11effa03cffe09582cfd20ab064fe --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Google */ + +#include <test_progs.h> +#include <bpf/libbpf.h> +#include <bpf/btf.h> +#include "cgroup_iter.skel.h" +#include "cgroup_helpers.h" + +#define ROOT 0 +#define PARENT 1 +#define CHILD1 2 +#define CHILD2 3 +#define NUM_CGROUPS 4 + +#define PROLOGUE "prologue\n" +#define EPILOGUE "epilogue\n" + +static const char *cg_path[] = { + "/", "/parent", "/parent/child1", "/parent/child2" +}; + +static int cg_fd[] = {-1, -1, -1, -1}; +static unsigned long long cg_id[] = {0, 0, 0, 0}; +static char expected_output[64]; + +static int setup_cgroups(void) +{ + int fd, i = 0; + + for (i = 0; i < NUM_CGROUPS; i++) { + fd = create_and_get_cgroup(cg_path[i]); + if (fd < 0) + return fd; + + cg_fd[i] = fd; + cg_id[i] = get_cgroup_id(cg_path[i]); + } + return 0; +} + +static void cleanup_cgroups(void) +{ + int i; + + for (i = 0; i < NUM_CGROUPS; i++) + close(cg_fd[i]); +} + +static void read_from_cgroup_iter(struct bpf_program *prog, int cgroup_fd, + int order, const char *testname) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + struct bpf_link *link; + int len, iter_fd; + static char buf[128]; + size_t left; + char *p; + + memset(&linfo, 0, sizeof(linfo)); + linfo.cgroup.cgroup_fd = cgroup_fd; + linfo.cgroup.order = order; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(prog, &opts); + if (!ASSERT_OK_PTR(link, "attach_iter")) + return; + + iter_fd = bpf_iter_create(bpf_link__fd(link)); + if (iter_fd < 0) + goto free_link; + + memset(buf, 0, sizeof(buf)); + left = ARRAY_SIZE(buf); + p = buf; + while ((len = read(iter_fd, p, left)) > 0) { + p += len; + left -= len; + } + + ASSERT_STREQ(buf, expected_output, testname); + + /* read() after iter finishes should be ok. */ + if (len == 0) + ASSERT_OK(read(iter_fd, buf, sizeof(buf)), "second_read"); + + close(iter_fd); +free_link: + bpf_link__destroy(link); +} + +/* Invalid cgroup. */ +static void test_invalid_cgroup(struct cgroup_iter *skel) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + struct bpf_link *link; + + memset(&linfo, 0, sizeof(linfo)); + linfo.cgroup.cgroup_fd = (__u32)-1; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts); + ASSERT_ERR_PTR(link, "attach_iter"); + bpf_link__destroy(link); +} + +/* Specifying both cgroup_fd and cgroup_id is invalid. */ +static void test_invalid_cgroup_spec(struct cgroup_iter *skel) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + union bpf_iter_link_info linfo; + struct bpf_link *link; + + memset(&linfo, 0, sizeof(linfo)); + linfo.cgroup.cgroup_fd = (__u32)cg_fd[PARENT]; + linfo.cgroup.cgroup_id = (__u64)cg_id[PARENT]; + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + + link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts); + ASSERT_ERR_PTR(link, "attach_iter"); + bpf_link__destroy(link); +} + +/* Preorder walk prints parent and child in order. */ +static void test_walk_preorder(struct cgroup_iter *skel) +{ + snprintf(expected_output, sizeof(expected_output), + PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE, + cg_id[PARENT], cg_id[CHILD1], cg_id[CHILD2]); + + read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT], + BPF_CGROUP_ITER_DESCENDANTS_PRE, "preorder"); +} + +/* Postorder walk prints child and parent in order. */ +static void test_walk_postorder(struct cgroup_iter *skel) +{ + snprintf(expected_output, sizeof(expected_output), + PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE, + cg_id[CHILD1], cg_id[CHILD2], cg_id[PARENT]); + + read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT], + BPF_CGROUP_ITER_DESCENDANTS_POST, "postorder"); +} + +/* Walking parents prints parent and then root. */ +static void test_walk_ancestors_up(struct cgroup_iter *skel) +{ + /* terminate the walk when ROOT is met. */ + skel->bss->terminal_cgroup = cg_id[ROOT]; + + snprintf(expected_output, sizeof(expected_output), + PROLOGUE "%8llu\n%8llu\n" EPILOGUE, + cg_id[PARENT], cg_id[ROOT]); + + read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT], + BPF_CGROUP_ITER_ANCESTORS_UP, "ancestors_up"); + + skel->bss->terminal_cgroup = 0; +} + +/* Early termination prints parent only. */ +static void test_early_termination(struct cgroup_iter *skel) +{ + /* terminate the walk after the first element is processed. */ + skel->bss->terminate_early = 1; + + snprintf(expected_output, sizeof(expected_output), + PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]); + + read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT], + BPF_CGROUP_ITER_DESCENDANTS_PRE, "early_termination"); + + skel->bss->terminate_early = 0; +} + +/* Waling self prints self only. */ +static void test_walk_self_only(struct cgroup_iter *skel) +{ + snprintf(expected_output, sizeof(expected_output), + PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]); + + read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT], + BPF_CGROUP_ITER_SELF_ONLY, "self_only"); +} + +void test_cgroup_iter(void) +{ + struct cgroup_iter *skel = NULL; + + if (setup_cgroup_environment()) + return; + + if (setup_cgroups()) + goto out; + + skel = cgroup_iter__open_and_load(); + if (!ASSERT_OK_PTR(skel, "cgroup_iter__open_and_load")) + goto out; + + if (test__start_subtest("cgroup_iter__invalid_cgroup")) + test_invalid_cgroup(skel); + if (test__start_subtest("cgroup_iter__invalid_cgroup_spec")) + test_invalid_cgroup_spec(skel); + if (test__start_subtest("cgroup_iter__preorder")) + test_walk_preorder(skel); + if (test__start_subtest("cgroup_iter__postorder")) + test_walk_postorder(skel); + if (test__start_subtest("cgroup_iter__ancestors_up_walk")) + test_walk_ancestors_up(skel); + if (test__start_subtest("cgroup_iter__early_termination")) + test_early_termination(skel); + if (test__start_subtest("cgroup_iter__self_only")) + test_walk_self_only(skel); +out: + cgroup_iter__destroy(skel); + cleanup_cgroups(); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_link.c b/tools/testing/selftests/bpf/prog_tests/cgroup_link.c index 9e6e6aad347c7b5eeb1f516b2d9d7119efba87a2..15093a69510eba2f96bb5dc2f4b8bb8173d038b2 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_link.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_link.c @@ -71,10 +71,9 @@ void serial_test_cgroup_link(void) ping_and_check(cg_nr, 0); - /* query the number of effective progs and attach flags in root cg */ + /* query the number of attached progs and attach flags in root cg */ err = bpf_prog_query(cgs[0].fd, BPF_CGROUP_INET_EGRESS, - BPF_F_QUERY_EFFECTIVE, &attach_flags, NULL, - &prog_cnt); + 0, &attach_flags, NULL, &prog_cnt); CHECK_FAIL(err); CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI); if (CHECK(prog_cnt != 1, "effect_cnt", "exp %d, got %d\n", 1, prog_cnt)) @@ -85,17 +84,15 @@ void serial_test_cgroup_link(void) BPF_F_QUERY_EFFECTIVE, NULL, NULL, &prog_cnt); CHECK_FAIL(err); - CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI); if (CHECK(prog_cnt != cg_nr, "effect_cnt", "exp %d, got %d\n", cg_nr, prog_cnt)) goto cleanup; /* query the effective prog IDs in last cg */ err = bpf_prog_query(cgs[last_cg].fd, BPF_CGROUP_INET_EGRESS, - BPF_F_QUERY_EFFECTIVE, &attach_flags, - prog_ids, &prog_cnt); + BPF_F_QUERY_EFFECTIVE, NULL, prog_ids, + &prog_cnt); CHECK_FAIL(err); - CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI); if (CHECK(prog_cnt != cg_nr, "effect_cnt", "exp %d, got %d\n", cg_nr, prog_cnt)) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c index 9c4325f4aef29ea056ca1308e1e7638e5161c90f..24d553109f8dd0fd369eb3287c3b22bec8f1721e 100644 --- a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c +++ b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c @@ -53,7 +53,7 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) __u16 expected_peer_port = 60000; struct bpf_program *prog; struct bpf_object *obj; - const char *obj_file = v4 ? "connect_force_port4.o" : "connect_force_port6.o"; + const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o"; int fd, err; __u32 duration = 0; diff --git a/tools/testing/selftests/bpf/prog_tests/connect_ping.c b/tools/testing/selftests/bpf/prog_tests/connect_ping.c new file mode 100644 index 0000000000000000000000000000000000000000..289218c2216c38cfee6282c8071456c4713a8fdc --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/connect_ping.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright 2022 Google LLC. + */ + +#define _GNU_SOURCE +#include <sys/mount.h> + +#include "test_progs.h" +#include "cgroup_helpers.h" +#include "network_helpers.h" + +#include "connect_ping.skel.h" + +/* 2001:db8::1 */ +#define BINDADDR_V6 { { { 0x20,0x01,0x0d,0xb8,0,0,0,0,0,0,0,0,0,0,0,1 } } } +static const struct in6_addr bindaddr_v6 = BINDADDR_V6; + +static void subtest(int cgroup_fd, struct connect_ping *skel, + int family, int do_bind) +{ + struct sockaddr_in sa4 = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct sockaddr_in6 sa6 = { + .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT, + }; + struct sockaddr *sa; + socklen_t sa_len; + int protocol; + int sock_fd; + + switch (family) { + case AF_INET: + sa = (struct sockaddr *)&sa4; + sa_len = sizeof(sa4); + protocol = IPPROTO_ICMP; + break; + case AF_INET6: + sa = (struct sockaddr *)&sa6; + sa_len = sizeof(sa6); + protocol = IPPROTO_ICMPV6; + break; + } + + memset(skel->bss, 0, sizeof(*skel->bss)); + skel->bss->do_bind = do_bind; + + sock_fd = socket(family, SOCK_DGRAM, protocol); + if (!ASSERT_GE(sock_fd, 0, "sock-create")) + return; + + if (!ASSERT_OK(connect(sock_fd, sa, sa_len), "connect")) + goto close_sock; + + if (!ASSERT_EQ(skel->bss->invocations_v4, family == AF_INET ? 1 : 0, + "invocations_v4")) + goto close_sock; + if (!ASSERT_EQ(skel->bss->invocations_v6, family == AF_INET6 ? 1 : 0, + "invocations_v6")) + goto close_sock; + if (!ASSERT_EQ(skel->bss->has_error, 0, "has_error")) + goto close_sock; + + if (!ASSERT_OK(getsockname(sock_fd, sa, &sa_len), + "getsockname")) + goto close_sock; + + switch (family) { + case AF_INET: + if (!ASSERT_EQ(sa4.sin_family, family, "sin_family")) + goto close_sock; + if (!ASSERT_EQ(sa4.sin_addr.s_addr, + htonl(do_bind ? 0x01010101 : INADDR_LOOPBACK), + "sin_addr")) + goto close_sock; + break; + case AF_INET6: + if (!ASSERT_EQ(sa6.sin6_family, AF_INET6, "sin6_family")) + goto close_sock; + if (!ASSERT_EQ(memcmp(&sa6.sin6_addr, + do_bind ? &bindaddr_v6 : &in6addr_loopback, + sizeof(sa6.sin6_addr)), + 0, "sin6_addr")) + goto close_sock; + break; + } + +close_sock: + close(sock_fd); +} + +void test_connect_ping(void) +{ + struct connect_ping *skel; + int cgroup_fd; + + if (!ASSERT_OK(unshare(CLONE_NEWNET | CLONE_NEWNS), "unshare")) + return; + + /* overmount sysfs, and making original sysfs private so overmount + * does not propagate to other mntns. + */ + if (!ASSERT_OK(mount("none", "/sys", NULL, MS_PRIVATE, NULL), + "remount-private-sys")) + return; + if (!ASSERT_OK(mount("sysfs", "/sys", "sysfs", 0, NULL), + "mount-sys")) + return; + if (!ASSERT_OK(mount("bpffs", "/sys/fs/bpf", "bpf", 0, NULL), + "mount-bpf")) + goto clean_mount; + + if (!ASSERT_OK(system("ip link set dev lo up"), "lo-up")) + goto clean_mount; + if (!ASSERT_OK(system("ip addr add 1.1.1.1 dev lo"), "lo-addr-v4")) + goto clean_mount; + if (!ASSERT_OK(system("ip -6 addr add 2001:db8::1 dev lo"), "lo-addr-v6")) + goto clean_mount; + if (write_sysctl("/proc/sys/net/ipv4/ping_group_range", "0 0")) + goto clean_mount; + + cgroup_fd = test__join_cgroup("/connect_ping"); + if (!ASSERT_GE(cgroup_fd, 0, "cg-create")) + goto clean_mount; + + skel = connect_ping__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel-load")) + goto close_cgroup; + skel->links.connect_v4_prog = + bpf_program__attach_cgroup(skel->progs.connect_v4_prog, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.connect_v4_prog, "cg-attach-v4")) + goto skel_destroy; + skel->links.connect_v6_prog = + bpf_program__attach_cgroup(skel->progs.connect_v6_prog, cgroup_fd); + if (!ASSERT_OK_PTR(skel->links.connect_v6_prog, "cg-attach-v6")) + goto skel_destroy; + + /* Connect a v4 ping socket to localhost, assert that only v4 is called, + * and called exactly once, and that the socket's bound address is + * original loopback address. + */ + if (test__start_subtest("ipv4")) + subtest(cgroup_fd, skel, AF_INET, 0); + + /* Connect a v4 ping socket to localhost, assert that only v4 is called, + * and called exactly once, and that the socket's bound address is + * address we explicitly bound. + */ + if (test__start_subtest("ipv4-bind")) + subtest(cgroup_fd, skel, AF_INET, 1); + + /* Connect a v6 ping socket to localhost, assert that only v6 is called, + * and called exactly once, and that the socket's bound address is + * original loopback address. + */ + if (test__start_subtest("ipv6")) + subtest(cgroup_fd, skel, AF_INET6, 0); + + /* Connect a v6 ping socket to localhost, assert that only v6 is called, + * and called exactly once, and that the socket's bound address is + * address we explicitly bound. + */ + if (test__start_subtest("ipv6-bind")) + subtest(cgroup_fd, skel, AF_INET6, 1); + +skel_destroy: + connect_ping__destroy(skel); + +close_cgroup: + close(cgroup_fd); + +clean_mount: + umount2("/sys", MNT_DETACH); +} diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index c8655ba9a88f27bc1ee1e80805a7d2e933a1a17d..47f42e6801056b190aa46d1a0b1b445098bd0f57 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -13,7 +13,7 @@ static int duration = 0; #define MODULES_CASE(name, pg_name, tp_name) { \ .case_name = name, \ - .bpf_obj_file = "test_core_reloc_module.o", \ + .bpf_obj_file = "test_core_reloc_module.bpf.o", \ .btf_src_file = NULL, /* find in kernel module BTFs */ \ .input = "", \ .input_len = 0, \ @@ -43,8 +43,8 @@ static int duration = 0; #define FLAVORS_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_flavors.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_flavors.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_flavors" \ @@ -68,8 +68,8 @@ static int duration = 0; #define NESTING_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_nesting.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_nesting.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_nesting" \ @@ -96,8 +96,8 @@ static int duration = 0; #define ARRAYS_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_arrays.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_arrays.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_arrays" \ @@ -130,8 +130,8 @@ static int duration = 0; #define PRIMITIVES_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_primitives.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_primitives.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_primitives" \ @@ -150,8 +150,8 @@ static int duration = 0; #define MODS_CASE(name) { \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_mods.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_mods.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) { \ .a = 1, \ .b = 2, \ @@ -174,8 +174,8 @@ static int duration = 0; #define PTR_AS_ARR_CASE(name) { \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_ptr_as_arr.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_ptr_as_arr.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .input = (const char *)&(struct core_reloc_##name []){ \ { .a = 1 }, \ { .a = 2 }, \ @@ -203,8 +203,8 @@ static int duration = 0; #define INTS_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_ints.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_ints.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_ints" @@ -223,18 +223,18 @@ static int duration = 0; #define FIELD_EXISTS_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_existence.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_existence.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_existence" #define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \ .case_name = test_name_prefix#name, \ .bpf_obj_file = objfile, \ - .btf_src_file = "btf__core_reloc_" #name ".o" + .btf_src_file = "btf__core_reloc_" #name ".bpf.o" #define BITFIELDS_CASE(name, ...) { \ - BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o", \ "probed:", name), \ .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ .input_len = sizeof(struct core_reloc_##name), \ @@ -244,7 +244,7 @@ static int duration = 0; .raw_tp_name = "sys_enter", \ .prog_name = "test_core_bitfields", \ }, { \ - BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o", \ "direct:", name), \ .input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \ .input_len = sizeof(struct core_reloc_##name), \ @@ -256,14 +256,14 @@ static int duration = 0; #define BITFIELDS_ERR_CASE(name) { \ - BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o", \ "probed:", name), \ .fails = true, \ - .run_btfgen_fails = true, \ + .run_btfgen_fails = true, \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_bitfields", \ }, { \ - BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \ + BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o", \ "direct:", name), \ .fails = true, \ .run_btfgen_fails = true, \ @@ -272,8 +272,8 @@ static int duration = 0; #define SIZE_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_size.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_size.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_size" @@ -307,13 +307,13 @@ static int duration = 0; #define SIZE_ERR_CASE(name) { \ SIZE_CASE_COMMON(name), \ .fails = true, \ - .run_btfgen_fails = true, \ + .run_btfgen_fails = true, \ } #define TYPE_BASED_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_type_based.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_type_based.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_type_based" @@ -331,8 +331,8 @@ static int duration = 0; #define TYPE_ID_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_type_id.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_type_id.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_type_id" @@ -350,8 +350,8 @@ static int duration = 0; #define ENUMVAL_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_enumval.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_enumval.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_enumval" @@ -369,8 +369,8 @@ static int duration = 0; #define ENUM64VAL_CASE_COMMON(name) \ .case_name = #name, \ - .bpf_obj_file = "test_core_reloc_enum64val.o", \ - .btf_src_file = "btf__core_reloc_" #name ".o", \ + .bpf_obj_file = "test_core_reloc_enum64val.bpf.o", \ + .btf_src_file = "btf__core_reloc_" #name ".bpf.o", \ .raw_tp_name = "sys_enter", \ .prog_name = "test_core_enum64val" @@ -547,7 +547,7 @@ static const struct core_reloc_test_case test_cases[] = { /* validate we can find kernel image and use its BTF for relocs */ { .case_name = "kernel", - .bpf_obj_file = "test_core_reloc_kernel.o", + .bpf_obj_file = "test_core_reloc_kernel.bpf.o", .btf_src_file = NULL, /* load from /lib/modules/$(uname -r) */ .input = "", .input_len = 0, @@ -629,8 +629,8 @@ static const struct core_reloc_test_case test_cases[] = { /* validate edge cases of capturing relocations */ { .case_name = "misc", - .bpf_obj_file = "test_core_reloc_misc.o", - .btf_src_file = "btf__core_reloc_misc.o", + .bpf_obj_file = "test_core_reloc_misc.bpf.o", + .btf_src_file = "btf__core_reloc_misc.bpf.o", .input = (const char *)&(struct core_reloc_misc_extensible[]){ { .a = 1 }, { .a = 2 }, /* not read */ diff --git a/tools/testing/selftests/bpf/prog_tests/deny_namespace.c b/tools/testing/selftests/bpf/prog_tests/deny_namespace.c new file mode 100644 index 0000000000000000000000000000000000000000..1bc6241b755b1fe8be9a970ad936d6346a7bf88c --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/deny_namespace.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <test_progs.h> +#include "test_deny_namespace.skel.h" +#include <sched.h> +#include "cap_helpers.h" +#include <stdio.h> + +static int wait_for_pid(pid_t pid) +{ + int status, ret; + +again: + ret = waitpid(pid, &status, 0); + if (ret == -1) { + if (errno == EINTR) + goto again; + + return -1; + } + + if (!WIFEXITED(status)) + return -1; + + return WEXITSTATUS(status); +} + +/* negative return value -> some internal error + * positive return value -> userns creation failed + * 0 -> userns creation succeeded + */ +static int create_user_ns(void) +{ + pid_t pid; + + pid = fork(); + if (pid < 0) + return -1; + + if (pid == 0) { + if (unshare(CLONE_NEWUSER)) + _exit(EXIT_FAILURE); + _exit(EXIT_SUCCESS); + } + + return wait_for_pid(pid); +} + +static void test_userns_create_bpf(void) +{ + __u32 cap_mask = 1ULL << CAP_SYS_ADMIN; + __u64 old_caps = 0; + + cap_enable_effective(cap_mask, &old_caps); + + ASSERT_OK(create_user_ns(), "priv new user ns"); + + cap_disable_effective(cap_mask, &old_caps); + + ASSERT_EQ(create_user_ns(), EPERM, "unpriv new user ns"); + + if (cap_mask & old_caps) + cap_enable_effective(cap_mask, NULL); +} + +static void test_unpriv_userns_create_no_bpf(void) +{ + __u32 cap_mask = 1ULL << CAP_SYS_ADMIN; + __u64 old_caps = 0; + + cap_disable_effective(cap_mask, &old_caps); + + ASSERT_OK(create_user_ns(), "no-bpf unpriv new user ns"); + + if (cap_mask & old_caps) + cap_enable_effective(cap_mask, NULL); +} + +void test_deny_namespace(void) +{ + struct test_deny_namespace *skel = NULL; + int err; + + if (test__start_subtest("unpriv_userns_create_no_bpf")) + test_unpriv_userns_create_no_bpf(); + + skel = test_deny_namespace__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel load")) + goto close_prog; + + err = test_deny_namespace__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto close_prog; + + if (test__start_subtest("userns_create_bpf")) + test_userns_create_bpf(); + + test_deny_namespace__detach(skel); + +close_prog: + test_deny_namespace__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c index 3c7aa82b98e224129bd20d1b0eea8f90c8364bee..8fc4e6c02bfdafea9cc9933e3e085be0a8095ba4 100644 --- a/tools/testing/selftests/bpf/prog_tests/dynptr.c +++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c @@ -22,14 +22,15 @@ static struct { {"add_dynptr_to_map2", "invalid indirect read from stack"}, {"data_slice_out_of_bounds_ringbuf", "value is outside of the allowed memory range"}, {"data_slice_out_of_bounds_map_value", "value is outside of the allowed memory range"}, - {"data_slice_use_after_release", "invalid mem access 'scalar'"}, + {"data_slice_use_after_release1", "invalid mem access 'scalar'"}, + {"data_slice_use_after_release2", "invalid mem access 'scalar'"}, {"data_slice_missing_null_check1", "invalid mem access 'mem_or_null'"}, {"data_slice_missing_null_check2", "invalid mem access 'mem_or_null'"}, {"invalid_helper1", "invalid indirect read from stack"}, {"invalid_helper2", "Expected an initialized dynptr as arg #3"}, {"invalid_write1", "Expected an initialized dynptr as arg #1"}, {"invalid_write2", "Expected an initialized dynptr as arg #3"}, - {"invalid_write3", "Expected an initialized ringbuf dynptr as arg #1"}, + {"invalid_write3", "Expected an initialized dynptr as arg #1"}, {"invalid_write4", "arg 1 is an unacquired reference"}, {"invalid_read1", "invalid read from stack"}, {"invalid_read2", "cannot pass in dynptr at an offset"}, diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index da860b07abb5c84e48c66b500a84cdf4aad82b16..d1e32e792536c0deb4481d231766a5ab43f39c73 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -174,8 +174,8 @@ static void test_target_no_callees(void) const char *prog_name[] = { "fexit/test_pkt_md_access", }; - test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o", - "./test_pkt_md_access.o", + test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.bpf.o", + "./test_pkt_md_access.bpf.o", ARRAY_SIZE(prog_name), prog_name, true, NULL); } @@ -188,8 +188,8 @@ static void test_target_yes_callees(void) "fexit/test_pkt_access_subprog2", "fexit/test_pkt_access_subprog3", }; - test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o", - "./test_pkt_access.o", + test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o", + "./test_pkt_access.bpf.o", ARRAY_SIZE(prog_name), prog_name, true, NULL); } @@ -206,8 +206,8 @@ static void test_func_replace(void) "freplace/get_constant", "freplace/test_pkt_write_access_subprog", }; - test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o", - "./test_pkt_access.o", + test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o", + "./test_pkt_access.bpf.o", ARRAY_SIZE(prog_name), prog_name, true, NULL); } @@ -217,8 +217,8 @@ static void test_func_replace_verify(void) const char *prog_name[] = { "freplace/do_bind", }; - test_fexit_bpf2bpf_common("./freplace_connect4.o", - "./connect4_prog.o", + test_fexit_bpf2bpf_common("./freplace_connect4.bpf.o", + "./connect4_prog.bpf.o", ARRAY_SIZE(prog_name), prog_name, false, NULL); } @@ -227,7 +227,7 @@ static int test_second_attach(struct bpf_object *obj) { const char *prog_name = "security_new_get_constant"; const char *tgt_name = "get_constant"; - const char *tgt_obj_file = "./test_pkt_access.o"; + const char *tgt_obj_file = "./test_pkt_access.bpf.o"; struct bpf_program *prog = NULL; struct bpf_object *tgt_obj; struct bpf_link *link; @@ -272,8 +272,8 @@ static void test_func_replace_multi(void) const char *prog_name[] = { "freplace/get_constant", }; - test_fexit_bpf2bpf_common("./freplace_get_constant.o", - "./test_pkt_access.o", + test_fexit_bpf2bpf_common("./freplace_get_constant.bpf.o", + "./test_pkt_access.bpf.o", ARRAY_SIZE(prog_name), prog_name, true, test_second_attach); } @@ -281,10 +281,10 @@ static void test_func_replace_multi(void) static void test_fmod_ret_freplace(void) { struct bpf_object *freplace_obj = NULL, *pkt_obj, *fmod_obj = NULL; - const char *freplace_name = "./freplace_get_constant.o"; - const char *fmod_ret_name = "./fmod_ret_freplace.o"; + const char *freplace_name = "./freplace_get_constant.bpf.o"; + const char *fmod_ret_name = "./fmod_ret_freplace.bpf.o"; DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts); - const char *tgt_name = "./test_pkt_access.o"; + const char *tgt_name = "./test_pkt_access.bpf.o"; struct bpf_link *freplace_link = NULL; struct bpf_program *prog; __u32 duration = 0; @@ -339,8 +339,8 @@ static void test_func_sockmap_update(void) const char *prog_name[] = { "freplace/cls_redirect", }; - test_fexit_bpf2bpf_common("./freplace_cls_redirect.o", - "./test_cls_redirect.o", + test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o", + "./test_cls_redirect.bpf.o", ARRAY_SIZE(prog_name), prog_name, false, NULL); } @@ -385,15 +385,15 @@ close_prog: static void test_func_replace_return_code(void) { /* test invalid return code in the replaced program */ - test_obj_load_failure_common("./freplace_connect_v4_prog.o", - "./connect4_prog.o"); + test_obj_load_failure_common("./freplace_connect_v4_prog.bpf.o", + "./connect4_prog.bpf.o"); } static void test_func_map_prog_compatibility(void) { /* test with spin lock map value in the replaced program */ - test_obj_load_failure_common("./freplace_attach_probe.o", - "./test_attach_probe.o"); + test_obj_load_failure_common("./freplace_attach_probe.bpf.o", + "./test_attach_probe.bpf.o"); } static void test_func_replace_global_func(void) @@ -402,8 +402,8 @@ static void test_func_replace_global_func(void) "freplace/test_pkt_access", }; - test_fexit_bpf2bpf_common("./freplace_global_func.o", - "./test_pkt_access.o", + test_fexit_bpf2bpf_common("./freplace_global_func.bpf.o", + "./test_pkt_access.bpf.o", ARRAY_SIZE(prog_name), prog_name, false, NULL); } diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c index 0c1661ea996e0b5bd73cc7695a9d44f7de74f3d5..7acca37a3d2b56017c66283c3f332ae91c3fe7f5 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c @@ -8,6 +8,8 @@ #include "bpf_flow.skel.h" +#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */ + #ifndef IP_MF #define IP_MF 0x2000 #endif @@ -100,6 +102,7 @@ struct test { } pkt; struct bpf_flow_keys keys; __u32 flags; + __u32 retval; }; #define VLAN_HLEN 4 @@ -126,6 +129,7 @@ struct test tests[] = { .sport = 80, .dport = 8080, }, + .retval = BPF_OK, }, { .name = "ipv6", @@ -146,6 +150,7 @@ struct test tests[] = { .sport = 80, .dport = 8080, }, + .retval = BPF_OK, }, { .name = "802.1q-ipv4", @@ -168,6 +173,7 @@ struct test tests[] = { .sport = 80, .dport = 8080, }, + .retval = BPF_OK, }, { .name = "802.1ad-ipv6", @@ -191,6 +197,7 @@ struct test tests[] = { .sport = 80, .dport = 8080, }, + .retval = BPF_OK, }, { .name = "ipv4-frag", @@ -217,6 +224,7 @@ struct test tests[] = { .dport = 8080, }, .flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG, + .retval = BPF_OK, }, { .name = "ipv4-no-frag", @@ -239,6 +247,7 @@ struct test tests[] = { .is_frag = true, .is_first_frag = true, }, + .retval = BPF_OK, }, { .name = "ipv6-frag", @@ -265,6 +274,7 @@ struct test tests[] = { .dport = 8080, }, .flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG, + .retval = BPF_OK, }, { .name = "ipv6-no-frag", @@ -287,6 +297,7 @@ struct test tests[] = { .is_frag = true, .is_first_frag = true, }, + .retval = BPF_OK, }, { .name = "ipv6-flow-label", @@ -309,6 +320,7 @@ struct test tests[] = { .dport = 8080, .flow_label = __bpf_constant_htonl(0xbeeef), }, + .retval = BPF_OK, }, { .name = "ipv6-no-flow-label", @@ -331,6 +343,7 @@ struct test tests[] = { .flow_label = __bpf_constant_htonl(0xbeeef), }, .flags = BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL, + .retval = BPF_OK, }, { .name = "ipip-encap", @@ -359,6 +372,7 @@ struct test tests[] = { .sport = 80, .dport = 8080, }, + .retval = BPF_OK, }, { .name = "ipip-no-encap", @@ -386,6 +400,26 @@ struct test tests[] = { .is_encap = true, }, .flags = BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP, + .retval = BPF_OK, + }, + { + .name = "ipip-encap-dissector-continue", + .pkt.ipip = { + .eth.h_proto = __bpf_constant_htons(ETH_P_IP), + .iph.ihl = 5, + .iph.protocol = IPPROTO_IPIP, + .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES), + .iph.saddr = __bpf_constant_htonl(FLOW_CONTINUE_SADDR), + .iph_inner.ihl = 5, + .iph_inner.protocol = IPPROTO_TCP, + .iph_inner.tot_len = + __bpf_constant_htons(MAGIC_BYTES) - + sizeof(struct iphdr), + .tcp.doff = 5, + .tcp.source = 99, + .tcp.dest = 9090, + }, + .retval = BPF_FLOW_DISSECTOR_CONTINUE, }, }; @@ -503,6 +537,10 @@ static void run_tests_skb_less(int tap_fd, struct bpf_map *keys) err = tx_tap(tap_fd, &tests[i].pkt, sizeof(tests[i].pkt)); CHECK(err < 0, "tx_tap", "err %d errno %d\n", err, errno); + /* check the stored flow_keys only if BPF_OK expected */ + if (tests[i].retval != BPF_OK) + continue; + err = bpf_map_lookup_elem(keys_fd, &key, &flow_keys); ASSERT_OK(err, "bpf_map_lookup_elem"); @@ -588,7 +626,11 @@ void test_flow_dissector(void) err = bpf_prog_test_run_opts(prog_fd, &topts); ASSERT_OK(err, "test_run"); - ASSERT_EQ(topts.retval, 1, "test_run retval"); + ASSERT_EQ(topts.retval, tests[i].retval, "test_run retval"); + + /* check the resulting flow_keys only if BPF_OK returned */ + if (topts.retval != BPF_OK) + continue; ASSERT_EQ(topts.data_size_out, sizeof(flow_keys), "test_run data_size_out"); CHECK_FLOW_KEYS(tests[i].name, flow_keys, tests[i].keys); diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c index 36afb409c25f0e4131087e1ff500d5b10df57084..c7a47b57ac9159cd7a5208b30d4908fdbcfb7fd2 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c @@ -44,7 +44,7 @@ void serial_test_flow_dissector_load_bytes(void) ASSERT_OK(err, "test_run"); ASSERT_EQ(topts.data_size_out, sizeof(flow_keys), "test_run data_size_out"); - ASSERT_EQ(topts.retval, 1, "test_run retval"); + ASSERT_EQ(topts.retval, BPF_OK, "test_run retval"); if (fd >= -1) close(fd); diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c index 938dbd4d7c2ff023a464f8e2cfa9e5b898b850a2..fede8ef58b5b05c52df36c501ac01f78a3107778 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c @@ -2,7 +2,7 @@ #include <test_progs.h> #include "get_func_ip_test.skel.h" -void test_get_func_ip_test(void) +static void test_function_entry(void) { struct get_func_ip_test *skel = NULL; int err, prog_fd; @@ -12,14 +12,6 @@ void test_get_func_ip_test(void) if (!ASSERT_OK_PTR(skel, "get_func_ip_test__open")) return; - /* test6 is x86_64 specifc because of the instruction - * offset, disabling it for all other archs - */ -#ifndef __x86_64__ - bpf_program__set_autoload(skel->progs.test6, false); - bpf_program__set_autoload(skel->progs.test7, false); -#endif - err = get_func_ip_test__load(skel); if (!ASSERT_OK(err, "get_func_ip_test__load")) goto cleanup; @@ -43,11 +35,56 @@ void test_get_func_ip_test(void) ASSERT_EQ(skel->bss->test3_result, 1, "test3_result"); ASSERT_EQ(skel->bss->test4_result, 1, "test4_result"); ASSERT_EQ(skel->bss->test5_result, 1, "test5_result"); + +cleanup: + get_func_ip_test__destroy(skel); +} + +/* test6 is x86_64 specific because of the instruction + * offset, disabling it for all other archs + */ #ifdef __x86_64__ +static void test_function_body(void) +{ + struct get_func_ip_test *skel = NULL; + LIBBPF_OPTS(bpf_test_run_opts, topts); + LIBBPF_OPTS(bpf_kprobe_opts, kopts); + struct bpf_link *link6 = NULL; + int err, prog_fd; + + skel = get_func_ip_test__open(); + if (!ASSERT_OK_PTR(skel, "get_func_ip_test__open")) + return; + + bpf_program__set_autoload(skel->progs.test6, true); + + err = get_func_ip_test__load(skel); + if (!ASSERT_OK(err, "get_func_ip_test__load")) + goto cleanup; + + kopts.offset = skel->kconfig->CONFIG_X86_KERNEL_IBT ? 9 : 5; + + link6 = bpf_program__attach_kprobe_opts(skel->progs.test6, "bpf_fentry_test6", &kopts); + if (!ASSERT_OK_PTR(link6, "link6")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.test1); + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "test_run"); + ASSERT_EQ(skel->bss->test6_result, 1, "test6_result"); - ASSERT_EQ(skel->bss->test7_result, 1, "test7_result"); -#endif cleanup: + bpf_link__destroy(link6); get_func_ip_test__destroy(skel); } +#else +#define test_function_body() +#endif + +void test_get_func_ip_test(void) +{ + test_function_entry(); + test_function_body(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c index 16048978a1eff4f488d6bfd23845fc2da2e066c4..858e0575f502fdac0d38dce9a7c16841354db707 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c @@ -84,8 +84,8 @@ static void get_stack_print_output(void *ctx, int cpu, void *data, __u32 size) void test_get_stack_raw_tp(void) { - const char *file = "./test_get_stack_rawtp.o"; - const char *file_err = "./test_get_stack_rawtp_err.o"; + const char *file = "./test_get_stack_rawtp.bpf.o"; + const char *file_err = "./test_get_stack_rawtp_err.bpf.o"; const char *prog_name = "bpf_prog1"; int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP; struct perf_buffer *pb = NULL; diff --git a/tools/testing/selftests/bpf/prog_tests/global_data.c b/tools/testing/selftests/bpf/prog_tests/global_data.c index 027685858925b8e06b8ef13ffd0c6a1154958dc4..fadfb64e2a7150e8228e3f5a1de1874767703546 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data.c @@ -131,7 +131,7 @@ static void test_global_data_rdonly(struct bpf_object *obj, __u32 duration) void test_global_data(void) { - const char *file = "./test_global_data.o"; + const char *file = "./test_global_data.bpf.o"; struct bpf_object *obj; int err, prog_fd; LIBBPF_OPTS(bpf_test_run_opts, topts, diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 57331c60696461aeec83af4629faec07cc755f8a..8466332d7406f407ed8429f3e7a3ba79b11a7f3d 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -3,7 +3,7 @@ void test_global_data_init(void) { - const char *file = "./test_global_data.o"; + const char *file = "./test_global_data.bpf.o"; int err = -ENOMEM, map_fd, zero = 0; __u8 *buff = NULL, *newval = NULL; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/global_func_args.c b/tools/testing/selftests/bpf/prog_tests/global_func_args.c index 29039a36cce5d4a4b9870c58f1e54417e8df1c35..d997099f62d0fd86853159a2de5008ae609d9d6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_func_args.c +++ b/tools/testing/selftests/bpf/prog_tests/global_func_args.c @@ -39,7 +39,7 @@ static void test_global_func_args0(struct bpf_object *obj) void test_global_func_args(void) { - const char *file = "./test_global_func_args.o"; + const char *file = "./test_global_func_args.bpf.o"; struct bpf_object *obj; int err, prog_fd; LIBBPF_OPTS(bpf_test_run_opts, topts, diff --git a/tools/testing/selftests/bpf/prog_tests/htab_update.c b/tools/testing/selftests/bpf/prog_tests/htab_update.c new file mode 100644 index 0000000000000000000000000000000000000000..2bc85f4814f4f90fc5be4e5e012c14822a0f0db2 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/htab_update.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022. Huawei Technologies Co., Ltd */ +#define _GNU_SOURCE +#include <sched.h> +#include <stdbool.h> +#include <test_progs.h> +#include "htab_update.skel.h" + +struct htab_update_ctx { + int fd; + int loop; + bool stop; +}; + +static void test_reenter_update(void) +{ + struct htab_update *skel; + unsigned int key, value; + int err; + + skel = htab_update__open(); + if (!ASSERT_OK_PTR(skel, "htab_update__open")) + return; + + /* lookup_elem_raw() may be inlined and find_kernel_btf_id() will return -ESRCH */ + bpf_program__set_autoload(skel->progs.lookup_elem_raw, true); + err = htab_update__load(skel); + if (!ASSERT_TRUE(!err || err == -ESRCH, "htab_update__load") || err) + goto out; + + skel->bss->pid = getpid(); + err = htab_update__attach(skel); + if (!ASSERT_OK(err, "htab_update__attach")) + goto out; + + /* Will trigger the reentrancy of bpf_map_update_elem() */ + key = 0; + value = 0; + err = bpf_map_update_elem(bpf_map__fd(skel->maps.htab), &key, &value, 0); + if (!ASSERT_OK(err, "add element")) + goto out; + + ASSERT_EQ(skel->bss->update_err, -EBUSY, "no reentrancy"); +out: + htab_update__destroy(skel); +} + +static void *htab_update_thread(void *arg) +{ + struct htab_update_ctx *ctx = arg; + cpu_set_t cpus; + int i; + + /* Pinned on CPU 0 */ + CPU_ZERO(&cpus); + CPU_SET(0, &cpus); + pthread_setaffinity_np(pthread_self(), sizeof(cpus), &cpus); + + i = 0; + while (i++ < ctx->loop && !ctx->stop) { + unsigned int key = 0, value = 0; + int err; + + err = bpf_map_update_elem(ctx->fd, &key, &value, 0); + if (err) { + ctx->stop = true; + return (void *)(long)err; + } + } + + return NULL; +} + +static void test_concurrent_update(void) +{ + struct htab_update_ctx ctx; + struct htab_update *skel; + unsigned int i, nr; + pthread_t *tids; + int err; + + skel = htab_update__open_and_load(); + if (!ASSERT_OK_PTR(skel, "htab_update__open_and_load")) + return; + + ctx.fd = bpf_map__fd(skel->maps.htab); + ctx.loop = 1000; + ctx.stop = false; + + nr = 4; + tids = calloc(nr, sizeof(*tids)); + if (!ASSERT_NEQ(tids, NULL, "no mem")) + goto out; + + for (i = 0; i < nr; i++) { + err = pthread_create(&tids[i], NULL, htab_update_thread, &ctx); + if (!ASSERT_OK(err, "pthread_create")) { + unsigned int j; + + ctx.stop = true; + for (j = 0; j < i; j++) + pthread_join(tids[j], NULL); + goto out; + } + } + + for (i = 0; i < nr; i++) { + void *thread_err = NULL; + + pthread_join(tids[i], &thread_err); + ASSERT_EQ(thread_err, NULL, "update error"); + } + +out: + if (tids) + free(tids); + htab_update__destroy(skel); +} + +void test_htab_update(void) +{ + if (test__start_subtest("reenter_update")) + test_reenter_update(); + if (test__start_subtest("concurrent_update")) + test_concurrent_update(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c index 1cee6957285e3c3f9cfe99cb9ad7d53fe2cb788c..73579370bfbd6b75f23d0971f06156510468f219 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c +++ b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c @@ -69,7 +69,7 @@ void serial_test_kfree_skb(void) const int zero = 0; bool test_ok[2]; - err = bpf_prog_test_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("./test_pkt_access.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index c00eb974eb85eca8c2e1e022f2d7be8df963ff0f..5af1ee8f0e6ee2589e72e7f4bed15f4ea9f2857c 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -2,40 +2,229 @@ /* Copyright (c) 2021 Facebook */ #include <test_progs.h> #include <network_helpers.h> +#include "kfunc_call_fail.skel.h" +#include "kfunc_call_test.skel.h" #include "kfunc_call_test.lskel.h" #include "kfunc_call_test_subprog.skel.h" #include "kfunc_call_test_subprog.lskel.h" +#include "kfunc_call_destructive.skel.h" -static void test_main(void) +#include "cap_helpers.h" + +static size_t log_buf_sz = 1048576; /* 1 MB */ +static char obj_log_buf[1048576]; + +enum kfunc_test_type { + tc_test = 0, + syscall_test, + syscall_null_ctx_test, +}; + +struct kfunc_test_params { + const char *prog_name; + unsigned long lskel_prog_desc_offset; + int retval; + enum kfunc_test_type test_type; + const char *expected_err_msg; +}; + +#define __BPF_TEST_SUCCESS(name, __retval, type) \ + { \ + .prog_name = #name, \ + .lskel_prog_desc_offset = offsetof(struct kfunc_call_test_lskel, progs.name), \ + .retval = __retval, \ + .test_type = type, \ + .expected_err_msg = NULL, \ + } + +#define __BPF_TEST_FAIL(name, __retval, type, error_msg) \ + { \ + .prog_name = #name, \ + .lskel_prog_desc_offset = 0 /* unused when test is failing */, \ + .retval = __retval, \ + .test_type = type, \ + .expected_err_msg = error_msg, \ + } + +#define TC_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, tc_test) +#define SYSCALL_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, syscall_test) +#define SYSCALL_NULL_CTX_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, syscall_null_ctx_test) + +#define TC_FAIL(name, retval, error_msg) __BPF_TEST_FAIL(name, retval, tc_test, error_msg) +#define SYSCALL_NULL_CTX_FAIL(name, retval, error_msg) \ + __BPF_TEST_FAIL(name, retval, syscall_null_ctx_test, error_msg) + +static struct kfunc_test_params kfunc_tests[] = { + /* failure cases: + * if retval is 0 -> the program will fail to load and the error message is an error + * if retval is not 0 -> the program can be loaded but running it will gives the + * provided return value. The error message is thus the one + * from a successful load + */ + SYSCALL_NULL_CTX_FAIL(kfunc_syscall_test_fail, -EINVAL, "processed 4 insns"), + SYSCALL_NULL_CTX_FAIL(kfunc_syscall_test_null_fail, -EINVAL, "processed 4 insns"), + TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"), + TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"), + TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"), + TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), + TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), + + /* success cases */ + TC_TEST(kfunc_call_test1, 12), + TC_TEST(kfunc_call_test2, 3), + TC_TEST(kfunc_call_test_ref_btf_id, 0), + TC_TEST(kfunc_call_test_get_mem, 42), + SYSCALL_TEST(kfunc_syscall_test, 0), + SYSCALL_NULL_CTX_TEST(kfunc_syscall_test_null, 0), +}; + +struct syscall_test_args { + __u8 data[16]; + size_t size; +}; + +static void verify_success(struct kfunc_test_params *param) { - struct kfunc_call_test_lskel *skel; + struct kfunc_call_test_lskel *lskel = NULL; + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_desc *lskel_prog; + struct kfunc_call_test *skel; + struct bpf_program *prog; int prog_fd, err; - LIBBPF_OPTS(bpf_test_run_opts, topts, - .data_in = &pkt_v4, - .data_size_in = sizeof(pkt_v4), - .repeat = 1, - ); + struct syscall_test_args args = { + .size = 10, + }; + + switch (param->test_type) { + case syscall_test: + topts.ctx_in = &args; + topts.ctx_size_in = sizeof(args); + /* fallthrough */ + case syscall_null_ctx_test: + break; + case tc_test: + topts.data_in = &pkt_v4; + topts.data_size_in = sizeof(pkt_v4); + topts.repeat = 1; + break; + } - skel = kfunc_call_test_lskel__open_and_load(); + /* first test with normal libbpf */ + skel = kfunc_call_test__open_and_load(); if (!ASSERT_OK_PTR(skel, "skel")) return; - prog_fd = skel->progs.kfunc_call_test1.prog_fd; - err = bpf_prog_test_run_opts(prog_fd, &topts); - ASSERT_OK(err, "bpf_prog_test_run(test1)"); - ASSERT_EQ(topts.retval, 12, "test1-retval"); + prog = bpf_object__find_program_by_name(skel->obj, param->prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; - prog_fd = skel->progs.kfunc_call_test2.prog_fd; + prog_fd = bpf_program__fd(prog); err = bpf_prog_test_run_opts(prog_fd, &topts); - ASSERT_OK(err, "bpf_prog_test_run(test2)"); - ASSERT_EQ(topts.retval, 3, "test2-retval"); + if (!ASSERT_OK(err, param->prog_name)) + goto cleanup; + + if (!ASSERT_EQ(topts.retval, param->retval, "retval")) + goto cleanup; + + /* second test with light skeletons */ + lskel = kfunc_call_test_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "lskel")) + goto cleanup; + + lskel_prog = (struct bpf_prog_desc *)((char *)lskel + param->lskel_prog_desc_offset); - prog_fd = skel->progs.kfunc_call_test_ref_btf_id.prog_fd; + prog_fd = lskel_prog->prog_fd; err = bpf_prog_test_run_opts(prog_fd, &topts); - ASSERT_OK(err, "bpf_prog_test_run(test_ref_btf_id)"); - ASSERT_EQ(topts.retval, 0, "test_ref_btf_id-retval"); + if (!ASSERT_OK(err, param->prog_name)) + goto cleanup; - kfunc_call_test_lskel__destroy(skel); + ASSERT_EQ(topts.retval, param->retval, "retval"); + +cleanup: + kfunc_call_test__destroy(skel); + if (lskel) + kfunc_call_test_lskel__destroy(lskel); +} + +static void verify_fail(struct kfunc_test_params *param) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_program *prog; + struct kfunc_call_fail *skel; + int prog_fd, err; + struct syscall_test_args args = { + .size = 10, + }; + + opts.kernel_log_buf = obj_log_buf; + opts.kernel_log_size = log_buf_sz; + opts.kernel_log_level = 1; + + switch (param->test_type) { + case syscall_test: + topts.ctx_in = &args; + topts.ctx_size_in = sizeof(args); + /* fallthrough */ + case syscall_null_ctx_test: + break; + case tc_test: + topts.data_in = &pkt_v4; + topts.data_size_in = sizeof(pkt_v4); + break; + topts.repeat = 1; + } + + skel = kfunc_call_fail__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "kfunc_call_fail__open_opts")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, param->prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + bpf_program__set_autoload(prog, true); + + err = kfunc_call_fail__load(skel); + if (!param->retval) { + /* the verifier is supposed to complain and refuses to load */ + if (!ASSERT_ERR(err, "unexpected load success")) + goto out_err; + + } else { + /* the program is loaded but must dynamically fail */ + if (!ASSERT_OK(err, "unexpected load error")) + goto out_err; + + prog_fd = bpf_program__fd(prog); + err = bpf_prog_test_run_opts(prog_fd, &topts); + if (!ASSERT_EQ(err, param->retval, param->prog_name)) + goto out_err; + } + +out_err: + if (!ASSERT_OK_PTR(strstr(obj_log_buf, param->expected_err_msg), "expected_err_msg")) { + fprintf(stderr, "Expected err_msg: %s\n", param->expected_err_msg); + fprintf(stderr, "Verifier output: %s\n", obj_log_buf); + } + +cleanup: + kfunc_call_fail__destroy(skel); +} + +static void test_main(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kfunc_tests); i++) { + if (!test__start_subtest(kfunc_tests[i].prog_name)) + continue; + + if (!kfunc_tests[i].expected_err_msg) + verify_success(&kfunc_tests[i]); + else + verify_fail(&kfunc_tests[i]); + } } static void test_subprog(void) @@ -86,14 +275,46 @@ static void test_subprog_lskel(void) kfunc_call_test_subprog_lskel__destroy(skel); } +static int test_destructive_open_and_load(void) +{ + struct kfunc_call_destructive *skel; + int err; + + skel = kfunc_call_destructive__open(); + if (!ASSERT_OK_PTR(skel, "prog_open")) + return -1; + + err = kfunc_call_destructive__load(skel); + + kfunc_call_destructive__destroy(skel); + + return err; +} + +static void test_destructive(void) +{ + __u64 save_caps = 0; + + ASSERT_OK(test_destructive_open_and_load(), "successful_load"); + + if (!ASSERT_OK(cap_disable_effective(1ULL << CAP_SYS_BOOT, &save_caps), "drop_caps")) + return; + + ASSERT_EQ(test_destructive_open_and_load(), -13, "no_caps_failure"); + + cap_enable_effective(save_caps, NULL); +} + void test_kfunc_call(void) { - if (test__start_subtest("main")) - test_main(); + test_main(); if (test__start_subtest("subprog")) test_subprog(); if (test__start_subtest("subprog_lskel")) test_subprog_lskel(); + + if (test__start_subtest("destructive")) + test_destructive(); } diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c new file mode 100644 index 0000000000000000000000000000000000000000..c210657d4d0aaae13b1a6e38e5995028af432208 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2022 Facebook + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include <test_progs.h> +#include "test_kfunc_dynptr_param.skel.h" + +static size_t log_buf_sz = 1048576; /* 1 MB */ +static char obj_log_buf[1048576]; + +static struct { + const char *prog_name; + const char *expected_verifier_err_msg; + int expected_runtime_err; +} kfunc_dynptr_tests[] = { + {"dynptr_type_not_supp", + "arg#0 pointer type STRUCT bpf_dynptr_kern points to unsupported dynamic pointer type", 0}, + {"not_valid_dynptr", + "arg#0 pointer type STRUCT bpf_dynptr_kern must be valid and initialized", 0}, + {"not_ptr_to_stack", "arg#0 pointer type STRUCT bpf_dynptr_kern not to stack", 0}, + {"dynptr_data_null", NULL, -EBADMSG}, +}; + +static bool kfunc_not_supported; + +static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt, + va_list args) +{ + if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n")) + return 0; + + if (strcmp(va_arg(args, char *), "bpf_verify_pkcs7_signature")) + return 0; + + kfunc_not_supported = true; + return 0; +} + +static void verify_fail(const char *prog_name, const char *expected_err_msg) +{ + struct test_kfunc_dynptr_param *skel; + LIBBPF_OPTS(bpf_object_open_opts, opts); + libbpf_print_fn_t old_print_cb; + struct bpf_program *prog; + int err; + + opts.kernel_log_buf = obj_log_buf; + opts.kernel_log_size = log_buf_sz; + opts.kernel_log_level = 1; + + skel = test_kfunc_dynptr_param__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "test_kfunc_dynptr_param__open_opts")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + bpf_program__set_autoload(prog, true); + + bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize()); + + kfunc_not_supported = false; + + old_print_cb = libbpf_set_print(libbpf_print_cb); + err = test_kfunc_dynptr_param__load(skel); + libbpf_set_print(old_print_cb); + + if (err < 0 && kfunc_not_supported) { + fprintf(stderr, + "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n", + __func__); + test__skip(); + goto cleanup; + } + + if (!ASSERT_ERR(err, "unexpected load success")) + goto cleanup; + + if (!ASSERT_OK_PTR(strstr(obj_log_buf, expected_err_msg), "expected_err_msg")) { + fprintf(stderr, "Expected err_msg: %s\n", expected_err_msg); + fprintf(stderr, "Verifier output: %s\n", obj_log_buf); + } + +cleanup: + test_kfunc_dynptr_param__destroy(skel); +} + +static void verify_success(const char *prog_name, int expected_runtime_err) +{ + struct test_kfunc_dynptr_param *skel; + libbpf_print_fn_t old_print_cb; + struct bpf_program *prog; + struct bpf_link *link; + __u32 next_id; + int err; + + skel = test_kfunc_dynptr_param__open(); + if (!ASSERT_OK_PTR(skel, "test_kfunc_dynptr_param__open")) + return; + + skel->bss->pid = getpid(); + + bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize()); + + kfunc_not_supported = false; + + old_print_cb = libbpf_set_print(libbpf_print_cb); + err = test_kfunc_dynptr_param__load(skel); + libbpf_set_print(old_print_cb); + + if (err < 0 && kfunc_not_supported) { + fprintf(stderr, + "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n", + __func__); + test__skip(); + goto cleanup; + } + + if (!ASSERT_OK(err, "test_kfunc_dynptr_param__load")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + link = bpf_program__attach(prog); + if (!ASSERT_OK_PTR(link, "bpf_program__attach")) + goto cleanup; + + err = bpf_prog_get_next_id(0, &next_id); + + bpf_link__destroy(link); + + if (!ASSERT_OK(err, "bpf_prog_get_next_id")) + goto cleanup; + + ASSERT_EQ(skel->bss->err, expected_runtime_err, "err"); + +cleanup: + test_kfunc_dynptr_param__destroy(skel); +} + +void test_kfunc_dynptr_param(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kfunc_dynptr_tests); i++) { + if (!test__start_subtest(kfunc_dynptr_tests[i].prog_name)) + continue; + + if (kfunc_dynptr_tests[i].expected_verifier_err_msg) + verify_fail(kfunc_dynptr_tests[i].prog_name, + kfunc_dynptr_tests[i].expected_verifier_err_msg); + else + verify_success(kfunc_dynptr_tests[i].prog_name, + kfunc_dynptr_tests[i].expected_runtime_err); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c index 55f733ff410922042d4971ca0127712269ba9ce9..9c1a18573ffdb5b0f5078797f3fe661418bd761b 100644 --- a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c +++ b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c @@ -90,7 +90,7 @@ out: void test_l4lb_all(void) { if (test__start_subtest("l4lb_inline")) - test_l4lb("test_l4lb.o"); + test_l4lb("test_l4lb.bpf.o"); if (test__start_subtest("l4lb_noinline")) - test_l4lb("test_l4lb_noinline.o"); + test_l4lb("test_l4lb_noinline.bpf.o"); } diff --git a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c index 4e0b2ec057aad9f68c0b5cdd4cf4e8471e577896..581c0eb0a0a14ae72a164976eed2b35d7c026156 100644 --- a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c +++ b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c @@ -27,8 +27,8 @@ void test_load_bytes_relative(void) if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; - err = bpf_prog_test_load("./load_bytes_relative.o", BPF_PROG_TYPE_CGROUP_SKB, - &obj, &prog_fd); + err = bpf_prog_test_load("./load_bytes_relative.bpf.o", BPF_PROG_TYPE_CGROUP_SKB, + &obj, &prog_fd); if (CHECK_FAIL(err)) goto close_server_fd; diff --git a/tools/testing/selftests/bpf/prog_tests/lookup_key.c b/tools/testing/selftests/bpf/prog_tests/lookup_key.c new file mode 100644 index 0000000000000000000000000000000000000000..68025e88f3527be012b2b081e0b2fa9101aebbd8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lookup_key.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include <linux/keyctl.h> +#include <test_progs.h> + +#include "test_lookup_key.skel.h" + +#define KEY_LOOKUP_CREATE 0x01 +#define KEY_LOOKUP_PARTIAL 0x02 + +static bool kfunc_not_supported; + +static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt, + va_list args) +{ + char *func; + + if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n")) + return 0; + + func = va_arg(args, char *); + + if (strcmp(func, "bpf_lookup_user_key") && strcmp(func, "bpf_key_put") && + strcmp(func, "bpf_lookup_system_key")) + return 0; + + kfunc_not_supported = true; + return 0; +} + +void test_lookup_key(void) +{ + libbpf_print_fn_t old_print_cb; + struct test_lookup_key *skel; + __u32 next_id; + int ret; + + skel = test_lookup_key__open(); + if (!ASSERT_OK_PTR(skel, "test_lookup_key__open")) + return; + + old_print_cb = libbpf_set_print(libbpf_print_cb); + ret = test_lookup_key__load(skel); + libbpf_set_print(old_print_cb); + + if (ret < 0 && kfunc_not_supported) { + printf("%s:SKIP:bpf_lookup_*_key(), bpf_key_put() kfuncs not supported\n", + __func__); + test__skip(); + goto close_prog; + } + + if (!ASSERT_OK(ret, "test_lookup_key__load")) + goto close_prog; + + ret = test_lookup_key__attach(skel); + if (!ASSERT_OK(ret, "test_lookup_key__attach")) + goto close_prog; + + skel->bss->monitored_pid = getpid(); + skel->bss->key_serial = KEY_SPEC_THREAD_KEYRING; + + /* The thread-specific keyring does not exist, this test fails. */ + skel->bss->flags = 0; + + ret = bpf_prog_get_next_id(0, &next_id); + if (!ASSERT_LT(ret, 0, "bpf_prog_get_next_id")) + goto close_prog; + + /* Force creation of the thread-specific keyring, this test succeeds. */ + skel->bss->flags = KEY_LOOKUP_CREATE; + + ret = bpf_prog_get_next_id(0, &next_id); + if (!ASSERT_OK(ret, "bpf_prog_get_next_id")) + goto close_prog; + + /* Pass both lookup flags for parameter validation. */ + skel->bss->flags = KEY_LOOKUP_CREATE | KEY_LOOKUP_PARTIAL; + + ret = bpf_prog_get_next_id(0, &next_id); + if (!ASSERT_OK(ret, "bpf_prog_get_next_id")) + goto close_prog; + + /* Pass invalid flags. */ + skel->bss->flags = UINT64_MAX; + + ret = bpf_prog_get_next_id(0, &next_id); + if (!ASSERT_LT(ret, 0, "bpf_prog_get_next_id")) + goto close_prog; + + skel->bss->key_serial = 0; + skel->bss->key_id = 1; + + ret = bpf_prog_get_next_id(0, &next_id); + if (!ASSERT_OK(ret, "bpf_prog_get_next_id")) + goto close_prog; + + skel->bss->key_id = UINT32_MAX; + + ret = bpf_prog_get_next_id(0, &next_id); + ASSERT_LT(ret, 0, "bpf_prog_get_next_id"); + +close_prog: + skel->bss->monitored_pid = 0; + test_lookup_key__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_lock.c b/tools/testing/selftests/bpf/prog_tests/map_lock.c index e4e99b37df64b1bf89d258c3ccba3858684175bc..1d6726f01dd297d331dcc1914406032669c205c7 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/map_lock.c @@ -49,7 +49,7 @@ out: void test_map_lock(void) { - const char *file = "./test_map_lock.o"; + const char *file = "./test_map_lock.bpf.o"; int prog_fd, map_fd[2], vars[17] = {}; pthread_t thread_id[6]; struct bpf_object *obj = NULL; diff --git a/tools/testing/selftests/bpf/prog_tests/pinning.c b/tools/testing/selftests/bpf/prog_tests/pinning.c index 31c09ba577eb1e042b27eaa88edaa15182978d1b..d95cee5867b7715e41cce5f75cdfed45e942a90c 100644 --- a/tools/testing/selftests/bpf/prog_tests/pinning.c +++ b/tools/testing/selftests/bpf/prog_tests/pinning.c @@ -26,13 +26,13 @@ __u32 get_map_id(struct bpf_object *obj, const char *name) void test_pinning(void) { - const char *file_invalid = "./test_pinning_invalid.o"; + const char *file_invalid = "./test_pinning_invalid.bpf.o"; const char *custpinpath = "/sys/fs/bpf/custom/pinmap"; const char *nopinpath = "/sys/fs/bpf/nopinmap"; const char *nopinpath2 = "/sys/fs/bpf/nopinmap2"; const char *custpath = "/sys/fs/bpf/custom"; const char *pinpath = "/sys/fs/bpf/pinmap"; - const char *file = "./test_pinning.o"; + const char *file = "./test_pinning.bpf.o"; __u32 map_id, map_id2, duration = 0; struct stat statbuf = {}; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_access.c index 0bcccdc34fbca896f79d6a9934ebe2aab06a2b3c..682e4ff45b01025469029ceed60b58d8da39cfe8 100644 --- a/tools/testing/selftests/bpf/prog_tests/pkt_access.c +++ b/tools/testing/selftests/bpf/prog_tests/pkt_access.c @@ -4,7 +4,7 @@ void test_pkt_access(void) { - const char *file = "./test_pkt_access.o"; + const char *file = "./test_pkt_access.bpf.o"; struct bpf_object *obj; int err, prog_fd; LIBBPF_OPTS(bpf_test_run_opts, topts, diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c index 00ee1dd792aa26f6abfbf840112683e4e8faee4d..0d85e0642811f37b1ea5656f4c411c68073677ad 100644 --- a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c +++ b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c @@ -4,7 +4,7 @@ void test_pkt_md_access(void) { - const char *file = "./test_pkt_md_access.o"; + const char *file = "./test_pkt_md_access.bpf.o"; struct bpf_object *obj; int err, prog_fd; LIBBPF_OPTS(bpf_test_run_opts, topts, diff --git a/tools/testing/selftests/bpf/prog_tests/probe_user.c b/tools/testing/selftests/bpf/prog_tests/probe_user.c index 34dbd2adc1575eaeaf97c005a8a8c1fe69d8f3e1..8721671321de8d0a568a537acf618af8b9647ad8 100644 --- a/tools/testing/selftests/bpf/prog_tests/probe_user.c +++ b/tools/testing/selftests/bpf/prog_tests/probe_user.c @@ -11,7 +11,7 @@ void serial_test_probe_user(void) #endif }; enum { prog_count = ARRAY_SIZE(prog_names) }; - const char *obj_file = "./test_probe_user.o"; + const char *obj_file = "./test_probe_user.bpf.o"; DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, ); int err, results_map_fd, sock_fd, duration = 0; struct sockaddr curr, orig, tmp; diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index d2743fc10032dbda3f310737e52937136286c623..722c5f2a77768e991a938974e3455842189cbaea 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c @@ -28,9 +28,9 @@ static void test_queue_stack_map_by_type(int type) vals[i] = rand(); if (type == QUEUE) - strncpy(file, "./test_queue_map.o", sizeof(file)); + strncpy(file, "./test_queue_map.bpf.o", sizeof(file)); else if (type == STACK) - strncpy(file, "./test_stack_map.o", sizeof(file)); + strncpy(file, "./test_stack_map.bpf.o", sizeof(file)); else return; diff --git a/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c b/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c index fd5d2ddfb06276337e64fa7a592b1d434d2787bf..19e2f2526dbdc1958bd30a0d87ba934cb61864e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c +++ b/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c @@ -16,7 +16,7 @@ struct rdonly_map_subtest { void test_rdonly_maps(void) { - const char *file = "test_rdonly_maps.o"; + const char *file = "test_rdonly_maps.bpf.o"; struct rdonly_map_subtest subtests[] = { { "skip loop", "skip_loop", 0, 0 }, { "part loop", "part_loop", 3, 2 + 3 + 4 }, diff --git a/tools/testing/selftests/bpf/prog_tests/reference_tracking.c b/tools/testing/selftests/bpf/prog_tests/reference_tracking.c index 739d2ea6ca5540faf11faac8911e258abb8a74c4..d863205bbe9525ab9c409a7772115b615948b5b2 100644 --- a/tools/testing/selftests/bpf/prog_tests/reference_tracking.c +++ b/tools/testing/selftests/bpf/prog_tests/reference_tracking.c @@ -3,7 +3,7 @@ void test_reference_tracking(void) { - const char *file = "test_sk_lookup_kern.o"; + const char *file = "test_sk_lookup_kern.bpf.o"; const char *obj_name = "ref_track"; DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts, .object_name = obj_name, diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c index c197261d02e277e5a766767472a148cd48536056..f81d08d429a2433ccd24d0d7b57a1f7bcd046d5f 100644 --- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c +++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c @@ -101,7 +101,7 @@ static int resolve_symbols(void) int type_id; __u32 nr; - btf = btf__parse_elf("btf_data.o", NULL); + btf = btf__parse_elf("btf_data.bpf.o", NULL); if (CHECK(libbpf_get_error(btf), "resolve", "Failed to load BTF from btf_data.o\n")) return -1; diff --git a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c index 1cbd8cd64044a0e026b2e75db087d4694ca3ea11..64c5f5eb2994fff15da964ce7b3ada5e517fc7fd 100644 --- a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c @@ -91,9 +91,9 @@ static int prepare_bpf_obj(void) struct bpf_map *map; int err; - obj = bpf_object__open("test_select_reuseport_kern.o"); + obj = bpf_object__open("test_select_reuseport_kern.bpf.o"); err = libbpf_get_error(obj); - RET_ERR(err, "open test_select_reuseport_kern.o", + RET_ERR(err, "open test_select_reuseport_kern.bpf.o", "obj:%p PTR_ERR(obj):%d\n", obj, err); map = bpf_object__find_map_by_name(obj, "outer_map"); diff --git a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c new file mode 100644 index 0000000000000000000000000000000000000000..018611e6b248b52beb574c1f4ae19f8ad74f7088 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#define _GNU_SOURCE +#include <sched.h> +#include <linux/socket.h> +#include <net/if.h> + +#include "test_progs.h" +#include "cgroup_helpers.h" +#include "network_helpers.h" + +#include "setget_sockopt.skel.h" + +#define CG_NAME "/setget-sockopt-test" + +static const char addr4_str[] = "127.0.0.1"; +static const char addr6_str[] = "::1"; +static struct setget_sockopt *skel; +static int cg_fd; + +static int create_netns(void) +{ + if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns")) + return -1; + + if (!ASSERT_OK(system("ip link set dev lo up"), "set lo up")) + return -1; + + if (!ASSERT_OK(system("ip link add dev binddevtest1 type veth peer name binddevtest2"), + "add veth")) + return -1; + + if (!ASSERT_OK(system("ip link set dev binddevtest1 up"), + "bring veth up")) + return -1; + + return 0; +} + +static void test_tcp(int family) +{ + struct setget_sockopt__bss *bss = skel->bss; + int sfd, cfd; + + memset(bss, 0, sizeof(*bss)); + + sfd = start_server(family, SOCK_STREAM, + family == AF_INET6 ? addr6_str : addr4_str, 0, 0); + if (!ASSERT_GE(sfd, 0, "start_server")) + return; + + cfd = connect_to_fd(sfd, 0); + if (!ASSERT_GE(cfd, 0, "connect_to_fd_server")) { + close(sfd); + return; + } + close(sfd); + close(cfd); + + ASSERT_EQ(bss->nr_listen, 1, "nr_listen"); + ASSERT_EQ(bss->nr_connect, 1, "nr_connect"); + ASSERT_EQ(bss->nr_active, 1, "nr_active"); + ASSERT_EQ(bss->nr_passive, 1, "nr_passive"); + ASSERT_EQ(bss->nr_socket_post_create, 2, "nr_socket_post_create"); + ASSERT_EQ(bss->nr_binddev, 2, "nr_bind"); +} + +static void test_udp(int family) +{ + struct setget_sockopt__bss *bss = skel->bss; + int sfd; + + memset(bss, 0, sizeof(*bss)); + + sfd = start_server(family, SOCK_DGRAM, + family == AF_INET6 ? addr6_str : addr4_str, 0, 0); + if (!ASSERT_GE(sfd, 0, "start_server")) + return; + close(sfd); + + ASSERT_GE(bss->nr_socket_post_create, 1, "nr_socket_post_create"); + ASSERT_EQ(bss->nr_binddev, 1, "nr_bind"); +} + +void test_setget_sockopt(void) +{ + cg_fd = test__join_cgroup(CG_NAME); + if (cg_fd < 0) + return; + + if (create_netns()) + goto done; + + skel = setget_sockopt__open(); + if (!ASSERT_OK_PTR(skel, "open skel")) + goto done; + + strcpy(skel->rodata->veth, "binddevtest1"); + skel->rodata->veth_ifindex = if_nametoindex("binddevtest1"); + if (!ASSERT_GT(skel->rodata->veth_ifindex, 0, "if_nametoindex")) + goto done; + + if (!ASSERT_OK(setget_sockopt__load(skel), "load skel")) + goto done; + + skel->links.skops_sockopt = + bpf_program__attach_cgroup(skel->progs.skops_sockopt, cg_fd); + if (!ASSERT_OK_PTR(skel->links.skops_sockopt, "attach cgroup")) + goto done; + + skel->links.socket_post_create = + bpf_program__attach_cgroup(skel->progs.socket_post_create, cg_fd); + if (!ASSERT_OK_PTR(skel->links.socket_post_create, "attach_cgroup")) + goto done; + + test_tcp(AF_INET6); + test_tcp(AF_INET); + test_udp(AF_INET6); + test_udp(AF_INET); + +done: + setget_sockopt__destroy(skel); + close(cg_fd); +} diff --git a/tools/testing/selftests/bpf/prog_tests/sk_assign.c b/tools/testing/selftests/bpf/prog_tests/sk_assign.c index 1d272e05188eb32dd16cff4633a258b81bccaf46..3e190ed63976754d728ee32c1c398bc2fc4dfa63 100644 --- a/tools/testing/selftests/bpf/prog_tests/sk_assign.c +++ b/tools/testing/selftests/bpf/prog_tests/sk_assign.c @@ -47,7 +47,7 @@ configure_stack(void) if (CHECK_FAIL(system("tc qdisc add dev lo clsact"))) return false; sprintf(tc_cmd, "%s %s %s %s", "tc filter add dev lo ingress bpf", - "direct-action object-file ./test_sk_assign.o", + "direct-action object-file ./test_sk_assign.bpf.o", "section tc", (env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose"); if (CHECK(system(tc_cmd), "BPF load failed;", diff --git a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c index ce0e555b5e3883f3dc1c9b9714970598aefd25df..33f950e2dae3f142b301c8072fc708eaf3ff5dc6 100644 --- a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c +++ b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c @@ -31,7 +31,7 @@ void test_skb_ctx(void) struct bpf_object *obj; int err, prog_fd, i; - err = bpf_prog_test_load("./test_skb_ctx.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("./test_skb_ctx.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (!ASSERT_OK(err, "load")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c index 97dc8b14be482dd85f5a55a411c5773523771956..f7ee25f290f7c4943c043894fc65ae00280633fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c +++ b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c @@ -20,7 +20,7 @@ void test_skb_helpers(void) struct bpf_object *obj; int err, prog_fd; - err = bpf_prog_test_load("./test_skb_helpers.o", + err = bpf_prog_test_load("./test_skb_helpers.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (!ASSERT_OK(err, "load")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index cec5c08823724aafb924a056b0a63f6c687bff5d..0aa088900699bda90e7cf75d6a30336987cc430d 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -27,21 +27,21 @@ static int connected_socket_v4(void) int s, repair, err; s = socket(AF_INET, SOCK_STREAM, 0); - if (CHECK_FAIL(s == -1)) + if (!ASSERT_GE(s, 0, "socket")) goto error; repair = TCP_REPAIR_ON; err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); - if (CHECK_FAIL(err)) + if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) goto error; err = connect(s, (struct sockaddr *)&addr, len); - if (CHECK_FAIL(err)) + if (!ASSERT_OK(err, "connect")) goto error; repair = TCP_REPAIR_OFF_NO_WP; err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair)); - if (CHECK_FAIL(err)) + if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)")) goto error; return s; @@ -54,7 +54,7 @@ error: static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) { __u32 i, max_entries = bpf_map__max_entries(src); - int err, duration = 0, src_fd, dst_fd; + int err, src_fd, dst_fd; src_fd = bpf_map__fd(src); dst_fd = bpf_map__fd(dst); @@ -65,20 +65,18 @@ static void compare_cookies(struct bpf_map *src, struct bpf_map *dst) err = bpf_map_lookup_elem(src_fd, &i, &src_cookie); if (err && errno == ENOENT) { err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); - CHECK(!err, "map_lookup_elem(dst)", "element %u not deleted\n", i); - CHECK(err && errno != ENOENT, "map_lookup_elem(dst)", "%s\n", - strerror(errno)); + ASSERT_ERR(err, "map_lookup_elem(dst)"); + ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)"); continue; } - if (CHECK(err, "lookup_elem(src)", "%s\n", strerror(errno))) + if (!ASSERT_OK(err, "lookup_elem(src)")) continue; err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie); - if (CHECK(err, "lookup_elem(dst)", "%s\n", strerror(errno))) + if (!ASSERT_OK(err, "lookup_elem(dst)")) continue; - CHECK(dst_cookie != src_cookie, "cookie mismatch", - "%llu != %llu (pos %u)\n", dst_cookie, src_cookie, i); + ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch"); } } @@ -89,20 +87,16 @@ static void test_sockmap_create_update_free(enum bpf_map_type map_type) int s, map, err; s = connected_socket_v4(); - if (CHECK_FAIL(s < 0)) + if (!ASSERT_GE(s, 0, "connected_socket_v4")) return; map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); - if (CHECK_FAIL(map < 0)) { - perror("bpf_cmap_create"); + if (!ASSERT_GE(map, 0, "bpf_map_create")) goto out; - } err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST); - if (CHECK_FAIL(err)) { - perror("bpf_map_update"); + if (!ASSERT_OK(err, "bpf_map_update")) goto out; - } out: close(map); @@ -115,32 +109,26 @@ static void test_skmsg_helpers(enum bpf_map_type map_type) int err, map, verdict; skel = test_skmsg_load_helpers__open_and_load(); - if (CHECK_FAIL(!skel)) { - perror("test_skmsg_load_helpers__open_and_load"); + if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load")) return; - } verdict = bpf_program__fd(skel->progs.prog_msg_verdict); map = bpf_map__fd(skel->maps.sock_map); err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0); - if (CHECK_FAIL(err)) { - perror("bpf_prog_attach"); + if (!ASSERT_OK(err, "bpf_prog_attach")) goto out; - } err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT); - if (CHECK_FAIL(err)) { - perror("bpf_prog_detach2"); + if (!ASSERT_OK(err, "bpf_prog_detach2")) goto out; - } out: test_skmsg_load_helpers__destroy(skel); } static void test_sockmap_update(enum bpf_map_type map_type) { - int err, prog, src, duration = 0; + int err, prog, src; struct test_sockmap_update *skel; struct bpf_map *dst_map; const __u32 zero = 0; @@ -153,11 +141,11 @@ static void test_sockmap_update(enum bpf_map_type map_type) __s64 sk; sk = connected_socket_v4(); - if (CHECK(sk == -1, "connected_socket_v4", "cannot connect\n")) + if (!ASSERT_NEQ(sk, -1, "connected_socket_v4")) return; skel = test_sockmap_update__open_and_load(); - if (CHECK(!skel, "open_and_load", "cannot load skeleton\n")) + if (!ASSERT_OK_PTR(skel, "open_and_load")) goto close_sk; prog = bpf_program__fd(skel->progs.copy_sock_map); @@ -168,7 +156,7 @@ static void test_sockmap_update(enum bpf_map_type map_type) dst_map = skel->maps.dst_sock_hash; err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST); - if (CHECK(err, "update_elem(src)", "errno=%u\n", errno)) + if (!ASSERT_OK(err, "update_elem(src)")) goto out; err = bpf_prog_test_run_opts(prog, &topts); @@ -188,17 +176,16 @@ close_sk: static void test_sockmap_invalid_update(void) { struct test_sockmap_invalid_update *skel; - int duration = 0; skel = test_sockmap_invalid_update__open_and_load(); - if (CHECK(skel, "open_and_load", "verifier accepted map_update\n")) + if (!ASSERT_NULL(skel, "open_and_load")) test_sockmap_invalid_update__destroy(skel); } static void test_sockmap_copy(enum bpf_map_type map_type) { DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); - int err, len, src_fd, iter_fd, duration = 0; + int err, len, src_fd, iter_fd; union bpf_iter_link_info linfo = {}; __u32 i, num_sockets, num_elems; struct bpf_iter_sockmap *skel; @@ -208,7 +195,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type) char buf[64]; skel = bpf_iter_sockmap__open_and_load(); - if (CHECK(!skel, "bpf_iter_sockmap__open_and_load", "skeleton open_and_load failed\n")) + if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load")) return; if (map_type == BPF_MAP_TYPE_SOCKMAP) { @@ -222,7 +209,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type) } sock_fd = calloc(num_sockets, sizeof(*sock_fd)); - if (CHECK(!sock_fd, "calloc(sock_fd)", "failed to allocate\n")) + if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)")) goto out; for (i = 0; i < num_sockets; i++) @@ -232,11 +219,11 @@ static void test_sockmap_copy(enum bpf_map_type map_type) for (i = 0; i < num_sockets; i++) { sock_fd[i] = connected_socket_v4(); - if (CHECK(sock_fd[i] == -1, "connected_socket_v4", "cannot connect\n")) + if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4")) goto out; err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST); - if (CHECK(err, "map_update", "failed: %s\n", strerror(errno))) + if (!ASSERT_OK(err, "map_update")) goto out; } @@ -248,22 +235,20 @@ static void test_sockmap_copy(enum bpf_map_type map_type) goto out; iter_fd = bpf_iter_create(bpf_link__fd(link)); - if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n")) + if (!ASSERT_GE(iter_fd, 0, "create_iter")) goto free_link; /* do some tests */ while ((len = read(iter_fd, buf, sizeof(buf))) > 0) ; - if (CHECK(len < 0, "read", "failed: %s\n", strerror(errno))) + if (!ASSERT_GE(len, 0, "read")) goto close_iter; /* test results */ - if (CHECK(skel->bss->elems != num_elems, "elems", "got %u expected %u\n", - skel->bss->elems, num_elems)) + if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems")) goto close_iter; - if (CHECK(skel->bss->socks != num_sockets, "socks", "got %u expected %u\n", - skel->bss->socks, num_sockets)) + if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks")) goto close_iter; compare_cookies(src, skel->maps.dst); @@ -288,28 +273,22 @@ static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first, int err, map, verdict; skel = test_sockmap_skb_verdict_attach__open_and_load(); - if (CHECK_FAIL(!skel)) { - perror("test_sockmap_skb_verdict_attach__open_and_load"); + if (!ASSERT_OK_PTR(skel, "open_and_load")) return; - } verdict = bpf_program__fd(skel->progs.prog_skb_verdict); map = bpf_map__fd(skel->maps.sock_map); err = bpf_prog_attach(verdict, map, first, 0); - if (CHECK_FAIL(err)) { - perror("bpf_prog_attach"); + if (!ASSERT_OK(err, "bpf_prog_attach")) goto out; - } err = bpf_prog_attach(verdict, map, second, 0); ASSERT_EQ(err, -EBUSY, "prog_attach_fail"); err = bpf_prog_detach2(verdict, map, first); - if (CHECK_FAIL(err)) { - perror("bpf_prog_detach2"); + if (!ASSERT_OK(err, "bpf_prog_detach2")) goto out; - } out: test_sockmap_skb_verdict_attach__destroy(skel); } diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c index e172d89e92e19b7e854f0290ec466feabf30b584..2d0796314862ac2aafeeb130456847428e36bbf2 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c @@ -15,16 +15,12 @@ static int tcp_server(int family) int err, s; s = socket(family, SOCK_STREAM, 0); - if (CHECK_FAIL(s == -1)) { - perror("socket"); + if (!ASSERT_GE(s, 0, "socket")) return -1; - } err = listen(s, SOMAXCONN); - if (CHECK_FAIL(err)) { - perror("listen"); + if (!ASSERT_OK(err, "listen")) return -1; - } return s; } @@ -48,44 +44,31 @@ static void test_sockmap_ktls_disconnect_after_delete(int family, int map) return; err = getsockname(srv, (struct sockaddr *)&addr, &len); - if (CHECK_FAIL(err)) { - perror("getsockopt"); + if (!ASSERT_OK(err, "getsockopt")) goto close_srv; - } cli = socket(family, SOCK_STREAM, 0); - if (CHECK_FAIL(cli == -1)) { - perror("socket"); + if (!ASSERT_GE(cli, 0, "socket")) goto close_srv; - } err = connect(cli, (struct sockaddr *)&addr, len); - if (CHECK_FAIL(err)) { - perror("connect"); + if (!ASSERT_OK(err, "connect")) goto close_cli; - } err = bpf_map_update_elem(map, &zero, &cli, 0); - if (CHECK_FAIL(err)) { - perror("bpf_map_update_elem"); + if (!ASSERT_OK(err, "bpf_map_update_elem")) goto close_cli; - } err = setsockopt(cli, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls")); - if (CHECK_FAIL(err)) { - perror("setsockopt(TCP_ULP)"); + if (!ASSERT_OK(err, "setsockopt(TCP_ULP)")) goto close_cli; - } err = bpf_map_delete_elem(map, &zero); - if (CHECK_FAIL(err)) { - perror("bpf_map_delete_elem"); + if (!ASSERT_OK(err, "bpf_map_delete_elem")) goto close_cli; - } err = disconnect(cli); - if (CHECK_FAIL(err)) - perror("disconnect"); + ASSERT_OK(err, "disconnect"); close_cli: close(cli); @@ -168,10 +151,8 @@ static void run_tests(int family, enum bpf_map_type map_type) int map; map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL); - if (CHECK_FAIL(map < 0)) { - perror("bpf_map_create"); + if (!ASSERT_GE(map, 0, "bpf_map_create")) return; - } if (test__start_subtest(fmt_test_name("disconnect_after_delete", family, map_type))) test_sockmap_ktls_disconnect_after_delete(family, map); diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index cd09f4c7dd922cee08dc34f6f0f5bf89272dee0a..aa4debf62fc6d6a10e7cfe7d0a88bbfd1132c9db 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -972,12 +972,12 @@ void test_sockopt(void) int cgroup_fd, i; cgroup_fd = test__join_cgroup("/sockopt"); - if (CHECK_FAIL(cgroup_fd < 0)) + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) return; for (i = 0; i < ARRAY_SIZE(tests); i++) { test__start_subtest(tests[i].descr); - CHECK_FAIL(run_test(cgroup_fd, &tests[i])); + ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr); } close(cgroup_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c b/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c index 8ed78a9383ba31f1ec75c9f9ef51efe42bbbe4c2..60c17a8e2789982e34955ca8499513d6fc7a34f4 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c @@ -76,20 +76,16 @@ static void *server_thread(void *arg) pthread_cond_signal(&server_started); pthread_mutex_unlock(&server_started_mtx); - if (CHECK_FAIL(err < 0)) { - perror("Failed to listed on socket"); + if (!ASSERT_GE(err, 0, "listed on socket")) return NULL; - } err += verify_sockopt(fd, CUSTOM_INHERIT1, "listen", 1); err += verify_sockopt(fd, CUSTOM_INHERIT2, "listen", 1); err += verify_sockopt(fd, CUSTOM_LISTENER, "listen", 1); client_fd = accept(fd, (struct sockaddr *)&addr, &len); - if (CHECK_FAIL(client_fd < 0)) { - perror("Failed to accept client"); + if (!ASSERT_GE(client_fd, 0, "accept client")) return NULL; - } err += verify_sockopt(client_fd, CUSTOM_INHERIT1, "accept", 1); err += verify_sockopt(client_fd, CUSTOM_INHERIT2, "accept", 1); @@ -174,7 +170,7 @@ static void run_test(int cgroup_fd) pthread_t tid; int err; - obj = bpf_object__open_file("sockopt_inherit.o", NULL); + obj = bpf_object__open_file("sockopt_inherit.bpf.o", NULL); if (!ASSERT_OK_PTR(obj, "obj_open")) return; @@ -183,20 +179,20 @@ static void run_test(int cgroup_fd) goto close_bpf_object; err = prog_attach(obj, cgroup_fd, "cgroup/getsockopt", "_getsockopt"); - if (CHECK_FAIL(err)) + if (!ASSERT_OK(err, "prog_attach _getsockopt")) goto close_bpf_object; err = prog_attach(obj, cgroup_fd, "cgroup/setsockopt", "_setsockopt"); - if (CHECK_FAIL(err)) + if (!ASSERT_OK(err, "prog_attach _setsockopt")) goto close_bpf_object; server_fd = start_server(); - if (CHECK_FAIL(server_fd < 0)) + if (!ASSERT_GE(server_fd, 0, "start_server")) goto close_bpf_object; pthread_mutex_lock(&server_started_mtx); - if (CHECK_FAIL(pthread_create(&tid, NULL, server_thread, - (void *)&server_fd))) { + if (!ASSERT_OK(pthread_create(&tid, NULL, server_thread, + (void *)&server_fd), "pthread_create")) { pthread_mutex_unlock(&server_started_mtx); goto close_server_fd; } @@ -204,17 +200,17 @@ static void run_test(int cgroup_fd) pthread_mutex_unlock(&server_started_mtx); client_fd = connect_to_server(server_fd); - if (CHECK_FAIL(client_fd < 0)) + if (!ASSERT_GE(client_fd, 0, "connect_to_server")) goto close_server_fd; - CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_INHERIT1, "connect", 0)); - CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_INHERIT2, "connect", 0)); - CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_LISTENER, "connect", 0)); + ASSERT_OK(verify_sockopt(client_fd, CUSTOM_INHERIT1, "connect", 0), "verify_sockopt1"); + ASSERT_OK(verify_sockopt(client_fd, CUSTOM_INHERIT2, "connect", 0), "verify_sockopt2"); + ASSERT_OK(verify_sockopt(client_fd, CUSTOM_LISTENER, "connect", 0), "verify_sockopt ener"); pthread_join(tid, &server_err); err = (int)(long)server_err; - CHECK_FAIL(err); + ASSERT_OK(err, "pthread_join retval"); close(client_fd); @@ -229,7 +225,7 @@ void test_sockopt_inherit(void) int cgroup_fd; cgroup_fd = test__join_cgroup("/sockopt_inherit"); - if (CHECK_FAIL(cgroup_fd < 0)) + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup")) return; run_test(cgroup_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c b/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c index abce12ddcc37d621ab790d5ecec14677c10f94a6..7f5659349011baf8f7e574984bf5829d6817930e 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c @@ -303,14 +303,14 @@ void test_sockopt_multi(void) int err = -1; cg_parent = test__join_cgroup("/parent"); - if (CHECK_FAIL(cg_parent < 0)) + if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent")) goto out; cg_child = test__join_cgroup("/parent/child"); - if (CHECK_FAIL(cg_child < 0)) + if (!ASSERT_GE(cg_child, 0, "join_cgroup /parent/child")) goto out; - obj = bpf_object__open_file("sockopt_multi.o", NULL); + obj = bpf_object__open_file("sockopt_multi.bpf.o", NULL); if (!ASSERT_OK_PTR(obj, "obj_load")) goto out; @@ -319,11 +319,11 @@ void test_sockopt_multi(void) goto out; sock_fd = socket(AF_INET, SOCK_STREAM, 0); - if (CHECK_FAIL(sock_fd < 0)) + if (!ASSERT_GE(sock_fd, 0, "socket")) goto out; - CHECK_FAIL(run_getsockopt_test(obj, cg_parent, cg_child, sock_fd)); - CHECK_FAIL(run_setsockopt_test(obj, cg_parent, cg_child, sock_fd)); + ASSERT_OK(run_getsockopt_test(obj, cg_parent, cg_child, sock_fd), "getsockopt_test"); + ASSERT_OK(run_setsockopt_test(obj, cg_parent, cg_child, sock_fd), "setsockopt_test"); out: close(sock_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c index 30a99d2ed5c62c18e080d0a1f14ae8078020008d..60d952719d275bc732cecd3de355e14d0e3dbf60 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c @@ -223,7 +223,7 @@ void test_sockopt_sk(void) int cgroup_fd; cgroup_fd = test__join_cgroup("/sockopt_sk"); - if (CHECK_FAIL(cgroup_fd < 0)) + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /sockopt_sk")) return; run_test(cgroup_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/spinlock.c b/tools/testing/selftests/bpf/prog_tests/spinlock.c index 8e329eaee6d788cb9e1fa42f7cbd9eb19af67bd7..15eb1372d771fb158c96ec6689b8a329a53b58d3 100644 --- a/tools/testing/selftests/bpf/prog_tests/spinlock.c +++ b/tools/testing/selftests/bpf/prog_tests/spinlock.c @@ -19,7 +19,7 @@ static void *spin_lock_thread(void *arg) void test_spinlock(void) { - const char *file = "./test_spin_lock.o"; + const char *file = "./test_spin_lock.bpf.o"; pthread_t thread_id[4]; struct bpf_object *obj = NULL; int prog_fd; diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c index 313f0a66232efc5d54e27e6bd7e03c84476811cb..df59e4ae29510087eff80f85b2aa8d343d0fd0b1 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c @@ -6,7 +6,7 @@ void test_stacktrace_map(void) int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd; const char *prog_name = "oncpu"; int err, prog_fd, stack_trace_len; - const char *file = "./test_stacktrace_map.o"; + const char *file = "./test_stacktrace_map.bpf.o"; __u32 key, val, duration = 0; struct bpf_program *prog; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c index 1cb8dd36bd8f12fbc22d81aa3055aeb82e799d33..c6ef06f55cdb46be5d8c1b953329372b3f9bc5d0 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c @@ -5,7 +5,7 @@ void test_stacktrace_map_raw_tp(void) { const char *prog_name = "oncpu"; int control_map_fd, stackid_hmap_fd, stackmap_fd; - const char *file = "./test_stacktrace_map.o"; + const char *file = "./test_stacktrace_map.bpf.o"; __u32 key, val, duration = 0; int err, prog_fd; struct bpf_program *prog; diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index 19c70880cfb3a1c3d7f65c7d3f5849dbba1eac68..58fe2c586ed76a91fa7a3784dbcdf66d9cc059d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -20,8 +20,8 @@ static void test_tailcall_1(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj, - &prog_fd); + err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + &prog_fd); if (CHECK_FAIL(err)) return; @@ -156,8 +156,8 @@ static void test_tailcall_2(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj, - &prog_fd); + err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + &prog_fd); if (CHECK_FAIL(err)) return; @@ -299,7 +299,7 @@ out: */ static void test_tailcall_3(void) { - test_tailcall_count("tailcall3.o"); + test_tailcall_count("tailcall3.bpf.o"); } /* test_tailcall_6 checks that the count value of the tail call limit @@ -307,7 +307,7 @@ static void test_tailcall_3(void) */ static void test_tailcall_6(void) { - test_tailcall_count("tailcall6.o"); + test_tailcall_count("tailcall6.bpf.o"); } /* test_tailcall_4 checks that the kernel properly selects indirect jump @@ -329,8 +329,8 @@ static void test_tailcall_4(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj, - &prog_fd); + err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + &prog_fd); if (CHECK_FAIL(err)) return; @@ -419,8 +419,8 @@ static void test_tailcall_5(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj, - &prog_fd); + err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + &prog_fd); if (CHECK_FAIL(err)) return; @@ -507,8 +507,8 @@ static void test_tailcall_bpf2bpf_1(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS, - &obj, &prog_fd); + err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, + &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -591,8 +591,8 @@ static void test_tailcall_bpf2bpf_2(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS, - &obj, &prog_fd); + err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, + &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -671,8 +671,8 @@ static void test_tailcall_bpf2bpf_3(void) .repeat = 1, ); - err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS, - &obj, &prog_fd); + err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, + &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -766,8 +766,8 @@ static void test_tailcall_bpf2bpf_4(bool noise) .repeat = 1, ); - err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS, - &obj, &prog_fd); + err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, + &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c index 17947c9e1d660175736b7a9ea4f010eea53c727d..3d34bab01e4879c4f6f7ed397d63a8f28d1491f4 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c +++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c @@ -3,7 +3,7 @@ void test_task_fd_query_rawtp(void) { - const char *file = "./test_get_stack_rawtp.o"; + const char *file = "./test_get_stack_rawtp.bpf.o"; __u64 probe_offset, probe_addr; __u32 len, prog_id, fd_type; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c index c2a98a7a8dfc523fc6fb78a25eb4816f1768e4b0..c717741bf8b658fcf6b753345affa11d7f681597 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c @@ -4,7 +4,7 @@ static void test_task_fd_query_tp_core(const char *probe_name, const char *tp_name) { - const char *file = "./test_tracepoint.o"; + const char *file = "./test_tracepoint.bpf.o"; int err, bytes, efd, prog_fd, pmu_fd; struct perf_event_attr attr = {}; __u64 probe_offset, probe_addr; diff --git a/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c b/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c index 61935e7e056a41725a711ea00c2232f387299dab..f000734a3d1f283639192a92133ed404aa4ff125 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c +++ b/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c @@ -4,7 +4,7 @@ #include "test_task_pt_regs.skel.h" /* uprobe attach point */ -static void trigger_func(void) +static noinline void trigger_func(void) { asm volatile (""); } diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c index 11bf755be4c99efbb9c1ed0f6140c6786e2c2fa4..e070bca2b764ed1dc8aae3d6cfeaf029da5913a0 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c @@ -3,14 +3,12 @@ void test_tcp_estats(void) { - const char *file = "./test_tcp_estats.o"; + const char *file = "./test_tcp_estats.bpf.o"; int err, prog_fd; struct bpf_object *obj; - __u32 duration = 0; err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); - CHECK(err, "", "err %d errno %d\n", err, errno); - if (err) + if (!ASSERT_OK(err, "")) return; bpf_object__close(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c index 1fa772079967497386c5668829accb553afe4f91..617bbce6ef8f13de8846398b734dcffb8e8dbaaa 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c @@ -42,33 +42,10 @@ struct sk_fds { static int create_netns(void) { - if (CHECK(unshare(CLONE_NEWNET), "create netns", - "unshare(CLONE_NEWNET): %s (%d)", - strerror(errno), errno)) + if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns")) return -1; - if (CHECK(system("ip link set dev lo up"), "run ip cmd", - "failed to bring lo link up\n")) - return -1; - - return 0; -} - -static int write_sysctl(const char *sysctl, const char *value) -{ - int fd, err, len; - - fd = open(sysctl, O_WRONLY); - if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n", - sysctl, strerror(errno), errno)) - return -1; - - len = strlen(value); - err = write(fd, value, len); - close(fd); - if (CHECK(err != len, "write sysctl", - "write(%s, %s): err:%d %s (%d)\n", - sysctl, value, err, strerror(errno), errno)) + if (!ASSERT_OK(system("ip link set dev lo up"), "run ip cmd")) return -1; return 0; @@ -100,16 +77,12 @@ static int sk_fds_shutdown(struct sk_fds *sk_fds) shutdown(sk_fds->active_fd, SHUT_WR); ret = read(sk_fds->passive_fd, &abyte, sizeof(abyte)); - if (CHECK(ret != 0, "read-after-shutdown(passive_fd):", - "ret:%d %s (%d)\n", - ret, strerror(errno), errno)) + if (!ASSERT_EQ(ret, 0, "read-after-shutdown(passive_fd):")) return -1; shutdown(sk_fds->passive_fd, SHUT_WR); ret = read(sk_fds->active_fd, &abyte, sizeof(abyte)); - if (CHECK(ret != 0, "read-after-shutdown(active_fd):", - "ret:%d %s (%d)\n", - ret, strerror(errno), errno)) + if (!ASSERT_EQ(ret, 0, "read-after-shutdown(active_fd):")) return -1; return 0; @@ -122,8 +95,7 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open) socklen_t len; sk_fds->srv_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0); - if (CHECK(sk_fds->srv_fd == -1, "start_server", "%s (%d)\n", - strerror(errno), errno)) + if (!ASSERT_NEQ(sk_fds->srv_fd, -1, "start_server")) goto error; if (fast_open) @@ -132,28 +104,25 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open) else sk_fds->active_fd = connect_to_fd(sk_fds->srv_fd, 0); - if (CHECK_FAIL(sk_fds->active_fd == -1)) { + if (!ASSERT_NEQ(sk_fds->active_fd, -1, "")) { close(sk_fds->srv_fd); goto error; } len = sizeof(addr6); - if (CHECK(getsockname(sk_fds->srv_fd, (struct sockaddr *)&addr6, - &len), "getsockname(srv_fd)", "%s (%d)\n", - strerror(errno), errno)) + if (!ASSERT_OK(getsockname(sk_fds->srv_fd, (struct sockaddr *)&addr6, + &len), "getsockname(srv_fd)")) goto error_close; sk_fds->passive_lport = ntohs(addr6.sin6_port); len = sizeof(addr6); - if (CHECK(getsockname(sk_fds->active_fd, (struct sockaddr *)&addr6, - &len), "getsockname(active_fd)", "%s (%d)\n", - strerror(errno), errno)) + if (!ASSERT_OK(getsockname(sk_fds->active_fd, (struct sockaddr *)&addr6, + &len), "getsockname(active_fd)")) goto error_close; sk_fds->active_lport = ntohs(addr6.sin6_port); sk_fds->passive_fd = accept(sk_fds->srv_fd, NULL, 0); - if (CHECK(sk_fds->passive_fd == -1, "accept(srv_fd)", "%s (%d)\n", - strerror(errno), errno)) + if (!ASSERT_NEQ(sk_fds->passive_fd, -1, "accept(srv_fd)")) goto error_close; if (fast_open) { @@ -161,8 +130,7 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open) int ret; ret = read(sk_fds->passive_fd, bytes_in, sizeof(bytes_in)); - if (CHECK(ret != sizeof(fast), "read fastopen syn data", - "expected=%lu actual=%d\n", sizeof(fast), ret)) { + if (!ASSERT_EQ(ret, sizeof(fast), "read fastopen syn data")) { close(sk_fds->passive_fd); goto error_close; } @@ -183,8 +151,7 @@ static int check_hdr_opt(const struct bpf_test_option *exp, const struct bpf_test_option *act, const char *hdr_desc) { - if (CHECK(memcmp(exp, act, sizeof(*exp)), - "expected-vs-actual", "unexpected %s\n", hdr_desc)) { + if (!ASSERT_OK(memcmp(exp, act, sizeof(*exp)), hdr_desc)) { print_option(exp, "expected: "); print_option(act, " actual: "); return -1; @@ -198,13 +165,11 @@ static int check_hdr_stg(const struct hdr_stg *exp, int fd, { struct hdr_stg act; - if (CHECK(bpf_map_lookup_elem(hdr_stg_map_fd, &fd, &act), - "map_lookup(hdr_stg_map_fd)", "%s %s (%d)\n", - stg_desc, strerror(errno), errno)) + if (!ASSERT_OK(bpf_map_lookup_elem(hdr_stg_map_fd, &fd, &act), + "map_lookup(hdr_stg_map_fd)")) return -1; - if (CHECK(memcmp(exp, &act, sizeof(*exp)), - "expected-vs-actual", "unexpected %s\n", stg_desc)) { + if (!ASSERT_OK(memcmp(exp, &act, sizeof(*exp)), stg_desc)) { print_hdr_stg(exp, "expected: "); print_hdr_stg(&act, " actual: "); return -1; @@ -248,9 +213,8 @@ static void check_hdr_and_close_fds(struct sk_fds *sk_fds) if (sk_fds_shutdown(sk_fds)) goto check_linum; - if (CHECK(expected_inherit_cb_flags != skel->bss->inherit_cb_flags, - "Unexpected inherit_cb_flags", "0x%x != 0x%x\n", - skel->bss->inherit_cb_flags, expected_inherit_cb_flags)) + if (!ASSERT_EQ(expected_inherit_cb_flags, skel->bss->inherit_cb_flags, + "inherit_cb_flags")) goto check_linum; if (check_hdr_stg(&exp_passive_hdr_stg, sk_fds->passive_fd, @@ -277,7 +241,7 @@ static void check_hdr_and_close_fds(struct sk_fds *sk_fds) "active_fin_in"); check_linum: - CHECK_FAIL(check_error_linum(sk_fds)); + ASSERT_FALSE(check_error_linum(sk_fds), "check_error_linum"); sk_fds_close(sk_fds); } @@ -517,26 +481,20 @@ static void misc(void) /* MSG_EOR to ensure skb will not be combined */ ret = send(sk_fds.active_fd, send_msg, sizeof(send_msg), MSG_EOR); - if (CHECK(ret != sizeof(send_msg), "send(msg)", "ret:%d\n", - ret)) + if (!ASSERT_EQ(ret, sizeof(send_msg), "send(msg)")) goto check_linum; ret = read(sk_fds.passive_fd, recv_msg, sizeof(recv_msg)); - if (CHECK(ret != sizeof(send_msg), "read(msg)", "ret:%d\n", - ret)) + if (ASSERT_EQ(ret, sizeof(send_msg), "read(msg)")) goto check_linum; } if (sk_fds_shutdown(&sk_fds)) goto check_linum; - CHECK(misc_skel->bss->nr_syn != 1, "unexpected nr_syn", - "expected (1) != actual (%u)\n", - misc_skel->bss->nr_syn); + ASSERT_EQ(misc_skel->bss->nr_syn, 1, "unexpected nr_syn"); - CHECK(misc_skel->bss->nr_data != nr_data, "unexpected nr_data", - "expected (%u) != actual (%u)\n", - nr_data, misc_skel->bss->nr_data); + ASSERT_EQ(misc_skel->bss->nr_data, nr_data, "unexpected nr_data"); /* The last ACK may have been delayed, so it is either 1 or 2. */ CHECK(misc_skel->bss->nr_pure_ack != 1 && @@ -545,12 +503,10 @@ static void misc(void) "expected (1 or 2) != actual (%u)\n", misc_skel->bss->nr_pure_ack); - CHECK(misc_skel->bss->nr_fin != 1, "unexpected nr_fin", - "expected (1) != actual (%u)\n", - misc_skel->bss->nr_fin); + ASSERT_EQ(misc_skel->bss->nr_fin, 1, "unexpected nr_fin"); check_linum: - CHECK_FAIL(check_error_linum(&sk_fds)); + ASSERT_FALSE(check_error_linum(&sk_fds), "check_error_linum"); sk_fds_close(&sk_fds); bpf_link__destroy(link); } @@ -575,15 +531,15 @@ void test_tcp_hdr_options(void) int i; skel = test_tcp_hdr_options__open_and_load(); - if (CHECK(!skel, "open and load skel", "failed")) + if (!ASSERT_OK_PTR(skel, "open and load skel")) return; misc_skel = test_misc_tcp_hdr_options__open_and_load(); - if (CHECK(!misc_skel, "open and load misc test skel", "failed")) + if (!ASSERT_OK_PTR(misc_skel, "open and load misc test skel")) goto skel_destroy; cg_fd = test__join_cgroup(CG_NAME); - if (CHECK_FAIL(cg_fd < 0)) + if (ASSERT_GE(cg_fd, 0, "join_cgroup")) goto skel_destroy; for (i = 0; i < ARRAY_SIZE(tests); i++) { diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c b/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c index 96ff2c20af81bb5321db3c7b4c99916207988ba5..8fe84da1b9b49bb6d33ca014a809f00b6eb2ab64 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c @@ -16,8 +16,7 @@ static void send_byte(int fd) { char b = 0x55; - if (CHECK_FAIL(write(fd, &b, sizeof(b)) != 1)) - perror("Failed to send single byte"); + ASSERT_EQ(write(fd, &b, sizeof(b)), 1, "send single byte"); } static int wait_for_ack(int fd, int retries) @@ -51,10 +50,8 @@ static int verify_sk(int map_fd, int client_fd, const char *msg, __u32 invoked, int err = 0; struct tcp_rtt_storage val; - if (CHECK_FAIL(bpf_map_lookup_elem(map_fd, &client_fd, &val) < 0)) { - perror("Failed to read socket storage"); + if (!ASSERT_GE(bpf_map_lookup_elem(map_fd, &client_fd, &val), 0, "read socket storage")) return -1; - } if (val.invoked != invoked) { log_err("%s: unexpected bpf_tcp_sock.invoked %d != %d", @@ -151,14 +148,14 @@ void test_tcp_rtt(void) int server_fd, cgroup_fd; cgroup_fd = test__join_cgroup("/tcp_rtt"); - if (CHECK_FAIL(cgroup_fd < 0)) + if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /tcp_rtt")) return; server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); - if (CHECK_FAIL(server_fd < 0)) + if (!ASSERT_GE(server_fd, 0, "start_server")) goto close_cgroup_fd; - CHECK_FAIL(run_test(cgroup_fd, server_fd)); + ASSERT_OK(run_test(cgroup_fd, server_fd), "run_test"); close(server_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c b/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c index 87923d2865b7ba193bbfae9106d5b52d5ea8755b..7e8fe1bad03f513f036321cfe279dfaa077f0fdf 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c +++ b/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c @@ -8,8 +8,6 @@ #define LO_ADDR6 "::1" #define CG_NAME "/tcpbpf-user-test" -static __u32 duration; - static void verify_result(struct tcpbpf_globals *result) { __u32 expected_events = ((1 << BPF_SOCK_OPS_TIMEOUT_INIT) | @@ -22,9 +20,7 @@ static void verify_result(struct tcpbpf_globals *result) (1 << BPF_SOCK_OPS_TCP_LISTEN_CB)); /* check global map */ - CHECK(expected_events != result->event_map, "event_map", - "unexpected event_map: actual 0x%08x != expected 0x%08x\n", - result->event_map, expected_events); + ASSERT_EQ(expected_events, result->event_map, "event_map"); ASSERT_EQ(result->bytes_received, 501, "bytes_received"); ASSERT_EQ(result->bytes_acked, 1002, "bytes_acked"); @@ -56,18 +52,15 @@ static void run_test(struct tcpbpf_globals *result) int i, rv; listen_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0); - if (CHECK(listen_fd == -1, "start_server", "listen_fd:%d errno:%d\n", - listen_fd, errno)) + if (!ASSERT_NEQ(listen_fd, -1, "start_server")) goto done; cli_fd = connect_to_fd(listen_fd, 0); - if (CHECK(cli_fd == -1, "connect_to_fd(listen_fd)", - "cli_fd:%d errno:%d\n", cli_fd, errno)) + if (!ASSERT_NEQ(cli_fd, -1, "connect_to_fd(listen_fd)")) goto done; accept_fd = accept(listen_fd, NULL, NULL); - if (CHECK(accept_fd == -1, "accept(listen_fd)", - "accept_fd:%d errno:%d\n", accept_fd, errno)) + if (!ASSERT_NEQ(accept_fd, -1, "accept(listen_fd)")) goto done; /* Send 1000B of '+'s from cli_fd -> accept_fd */ @@ -75,11 +68,11 @@ static void run_test(struct tcpbpf_globals *result) buf[i] = '+'; rv = send(cli_fd, buf, 1000, 0); - if (CHECK(rv != 1000, "send(cli_fd)", "rv:%d errno:%d\n", rv, errno)) + if (!ASSERT_EQ(rv, 1000, "send(cli_fd)")) goto done; rv = recv(accept_fd, buf, 1000, 0); - if (CHECK(rv != 1000, "recv(accept_fd)", "rv:%d errno:%d\n", rv, errno)) + if (!ASSERT_EQ(rv, 1000, "recv(accept_fd)")) goto done; /* Send 500B of '.'s from accept_fd ->cli_fd */ @@ -87,11 +80,11 @@ static void run_test(struct tcpbpf_globals *result) buf[i] = '.'; rv = send(accept_fd, buf, 500, 0); - if (CHECK(rv != 500, "send(accept_fd)", "rv:%d errno:%d\n", rv, errno)) + if (!ASSERT_EQ(rv, 500, "send(accept_fd)")) goto done; rv = recv(cli_fd, buf, 500, 0); - if (CHECK(rv != 500, "recv(cli_fd)", "rv:%d errno:%d\n", rv, errno)) + if (!ASSERT_EQ(rv, 500, "recv(cli_fd)")) goto done; /* @@ -100,12 +93,12 @@ static void run_test(struct tcpbpf_globals *result) */ shutdown(accept_fd, SHUT_WR); err = recv(cli_fd, buf, 1, 0); - if (CHECK(err, "recv(cli_fd) for fin", "err:%d errno:%d\n", err, errno)) + if (!ASSERT_OK(err, "recv(cli_fd) for fin")) goto done; shutdown(cli_fd, SHUT_WR); err = recv(accept_fd, buf, 1, 0); - CHECK(err, "recv(accept_fd) for fin", "err:%d errno:%d\n", err, errno); + ASSERT_OK(err, "recv(accept_fd) for fin"); done: if (accept_fd != -1) close(accept_fd); @@ -124,12 +117,11 @@ void test_tcpbpf_user(void) int cg_fd = -1; skel = test_tcpbpf_kern__open_and_load(); - if (CHECK(!skel, "open and load skel", "failed")) + if (!ASSERT_OK_PTR(skel, "open and load skel")) return; cg_fd = test__join_cgroup(CG_NAME); - if (CHECK(cg_fd < 0, "test__join_cgroup(" CG_NAME ")", - "cg_fd:%d errno:%d", cg_fd, errno)) + if (!ASSERT_GE(cg_fd, 0, "test__join_cgroup(" CG_NAME ")")) goto err; skel->links.bpf_testcb = bpf_program__attach_cgroup(skel->progs.bpf_testcb, cg_fd); diff --git a/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c b/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c index 2559bb775762ee52b77cac3f4a4bf55a983a84b7..a0054019e6772410e2ad6ef73b9c255a6ea23129 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c +++ b/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c @@ -9,18 +9,10 @@ #include "bprm_opts.skel.h" #include "network_helpers.h" - -#ifndef __NR_pidfd_open -#define __NR_pidfd_open 434 -#endif +#include "task_local_storage_helpers.h" static const char * const bash_envp[] = { "TMPDIR=shouldnotbeset", NULL }; -static inline int sys_pidfd_open(pid_t pid, unsigned int flags) -{ - return syscall(__NR_pidfd_open, pid, flags); -} - static int update_storage(int map_fd, int secureexec) { int task_fd, ret = 0; diff --git a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c index b90ee47d31113c1b2c0b155c0a9bf914396f5b4e..7295cc60f724878ce86716ed83f69517f43e41b4 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c +++ b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c @@ -65,23 +65,23 @@ struct test_def { void test_test_global_funcs(void) { struct test_def tests[] = { - { "test_global_func1.o", "combined stack size of 4 calls is 544" }, - { "test_global_func2.o" }, - { "test_global_func3.o" , "the call stack of 8 frames" }, - { "test_global_func4.o" }, - { "test_global_func5.o" , "expected pointer to ctx, but got PTR" }, - { "test_global_func6.o" , "modified ctx ptr R2" }, - { "test_global_func7.o" , "foo() doesn't return scalar" }, - { "test_global_func8.o" }, - { "test_global_func9.o" }, - { "test_global_func10.o", "invalid indirect read from stack" }, - { "test_global_func11.o", "Caller passes invalid args into func#1" }, - { "test_global_func12.o", "invalid mem access 'mem_or_null'" }, - { "test_global_func13.o", "Caller passes invalid args into func#1" }, - { "test_global_func14.o", "reference type('FWD S') size cannot be determined" }, - { "test_global_func15.o", "At program exit the register R0 has value" }, - { "test_global_func16.o", "invalid indirect read from stack" }, - { "test_global_func17.o", "Caller passes invalid args into func#1" }, + { "test_global_func1.bpf.o", "combined stack size of 4 calls is 544" }, + { "test_global_func2.bpf.o" }, + { "test_global_func3.bpf.o", "the call stack of 8 frames" }, + { "test_global_func4.bpf.o" }, + { "test_global_func5.bpf.o", "expected pointer to ctx, but got PTR" }, + { "test_global_func6.bpf.o", "modified ctx ptr R2" }, + { "test_global_func7.bpf.o", "foo() doesn't return scalar" }, + { "test_global_func8.bpf.o" }, + { "test_global_func9.bpf.o" }, + { "test_global_func10.bpf.o", "invalid indirect read from stack" }, + { "test_global_func11.bpf.o", "Caller passes invalid args into func#1" }, + { "test_global_func12.bpf.o", "invalid mem access 'mem_or_null'" }, + { "test_global_func13.bpf.o", "Caller passes invalid args into func#1" }, + { "test_global_func14.bpf.o", "reference type('FWD S') size cannot be determined" }, + { "test_global_func15.bpf.o", "At program exit the register R0 has value" }, + { "test_global_func16.bpf.o", "invalid indirect read from stack" }, + { "test_global_func17.bpf.o", "Caller passes invalid args into func#1" }, }; libbpf_print_fn_t old_print_fn = NULL; int err, i, duration = 0; diff --git a/tools/testing/selftests/bpf/prog_tests/test_local_storage.c b/tools/testing/selftests/bpf/prog_tests/test_local_storage.c index 26ac26a880261429e74d566ce0db7a2982da875f..9c77cd6b1eafbacedfe2eac834ce4e8451a2d32f 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_local_storage.c +++ b/tools/testing/selftests/bpf/prog_tests/test_local_storage.c @@ -11,15 +11,7 @@ #include "local_storage.skel.h" #include "network_helpers.h" - -#ifndef __NR_pidfd_open -#define __NR_pidfd_open 434 -#endif - -static inline int sys_pidfd_open(pid_t pid, unsigned int flags) -{ - return syscall(__NR_pidfd_open, pid, flags); -} +#include "task_local_storage_helpers.h" static unsigned int duration; diff --git a/tools/testing/selftests/bpf/prog_tests/test_overhead.c b/tools/testing/selftests/bpf/prog_tests/test_overhead.c index 05acb376f74d8909f420459d19474745c0e3d61c..f27013e38d0373cdd5601a848b48b4bcdd4b88d1 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_overhead.c +++ b/tools/testing/selftests/bpf/prog_tests/test_overhead.c @@ -72,7 +72,7 @@ void test_test_overhead(void) if (CHECK_FAIL(prctl(PR_GET_NAME, comm, 0L, 0L, 0L))) return; - obj = bpf_object__open_file("./test_overhead.o", NULL); + obj = bpf_object__open_file("./test_overhead.bpf.o", NULL); if (!ASSERT_OK_PTR(obj, "obj_open_file")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/time_tai.c b/tools/testing/selftests/bpf/prog_tests/time_tai.c new file mode 100644 index 0000000000000000000000000000000000000000..a311198236661bed74c69a319e1f53291a46eada --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/time_tai.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022 Linutronix GmbH */ + +#include <test_progs.h> +#include <network_helpers.h> + +#include "test_time_tai.skel.h" + +#include <time.h> +#include <stdint.h> + +#define TAI_THRESHOLD 1000000000ULL /* 1s */ +#define NSEC_PER_SEC 1000000000ULL + +static __u64 ts_to_ns(const struct timespec *ts) +{ + return ts->tv_sec * NSEC_PER_SEC + ts->tv_nsec; +} + +void test_time_tai(void) +{ + struct __sk_buff skb = { + .cb[0] = 0, + .cb[1] = 0, + .tstamp = 0, + }; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .ctx_in = &skb, + .ctx_size_in = sizeof(skb), + .ctx_out = &skb, + .ctx_size_out = sizeof(skb), + ); + struct test_time_tai *skel; + struct timespec now_tai; + __u64 ts1, ts2, now; + int ret, prog_fd; + + /* Open and load */ + skel = test_time_tai__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tai_open")) + return; + + /* Run test program */ + prog_fd = bpf_program__fd(skel->progs.time_tai); + ret = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(ret, "test_run"); + + /* Retrieve generated TAI timestamps */ + ts1 = skb.tstamp; + ts2 = skb.cb[0] | ((__u64)skb.cb[1] << 32); + + /* TAI != 0 */ + ASSERT_NEQ(ts1, 0, "tai_ts1"); + ASSERT_NEQ(ts2, 0, "tai_ts2"); + + /* TAI is moving forward only */ + ASSERT_GT(ts2, ts1, "tai_forward"); + + /* Check for future */ + ret = clock_gettime(CLOCK_TAI, &now_tai); + ASSERT_EQ(ret, 0, "tai_gettime"); + now = ts_to_ns(&now_tai); + + ASSERT_TRUE(now > ts1, "tai_future_ts1"); + ASSERT_TRUE(now > ts2, "tai_future_ts2"); + + /* Check for reasonable range */ + ASSERT_TRUE(now - ts1 < TAI_THRESHOLD, "tai_range_ts1"); + ASSERT_TRUE(now - ts2 < TAI_THRESHOLD, "tai_range_ts2"); + + test_time_tai__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c index 39e79291c82b8767c0506f0663b3f79a7329445a..a479080533dbf55610587b7121db47f105085f80 100644 --- a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c +++ b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c @@ -6,7 +6,7 @@ void serial_test_tp_attach_query(void) const int num_progs = 3; int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs]; __u32 duration = 0, info_len, saved_prog_ids[num_progs]; - const char *file = "./test_tracepoint.o"; + const char *file = "./test_tracepoint.bpf.o"; struct perf_event_query_bpf *query; struct perf_event_attr attr = {}; struct bpf_object *obj[num_progs]; diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c new file mode 100644 index 0000000000000000000000000000000000000000..d5022b91d1e4c1084225edf87ecfc61a2b714881 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include <test_progs.h> +#include "tracing_struct.skel.h" + +static void test_fentry(void) +{ + struct tracing_struct *skel; + int err; + + skel = tracing_struct__open_and_load(); + if (!ASSERT_OK_PTR(skel, "tracing_struct__open_and_load")) + return; + + err = tracing_struct__attach(skel); + if (!ASSERT_OK(err, "tracing_struct__attach")) + return; + + ASSERT_OK(trigger_module_test_read(256), "trigger_read"); + + ASSERT_EQ(skel->bss->t1_a_a, 2, "t1:a.a"); + ASSERT_EQ(skel->bss->t1_a_b, 3, "t1:a.b"); + ASSERT_EQ(skel->bss->t1_b, 1, "t1:b"); + ASSERT_EQ(skel->bss->t1_c, 4, "t1:c"); + + ASSERT_EQ(skel->bss->t1_nregs, 4, "t1 nregs"); + ASSERT_EQ(skel->bss->t1_reg0, 2, "t1 reg0"); + ASSERT_EQ(skel->bss->t1_reg1, 3, "t1 reg1"); + ASSERT_EQ(skel->bss->t1_reg2, 1, "t1 reg2"); + ASSERT_EQ(skel->bss->t1_reg3, 4, "t1 reg3"); + ASSERT_EQ(skel->bss->t1_ret, 10, "t1 ret"); + + ASSERT_EQ(skel->bss->t2_a, 1, "t2:a"); + ASSERT_EQ(skel->bss->t2_b_a, 2, "t2:b.a"); + ASSERT_EQ(skel->bss->t2_b_b, 3, "t2:b.b"); + ASSERT_EQ(skel->bss->t2_c, 4, "t2:c"); + ASSERT_EQ(skel->bss->t2_ret, 10, "t2 ret"); + + ASSERT_EQ(skel->bss->t3_a, 1, "t3:a"); + ASSERT_EQ(skel->bss->t3_b, 4, "t3:b"); + ASSERT_EQ(skel->bss->t3_c_a, 2, "t3:c.a"); + ASSERT_EQ(skel->bss->t3_c_b, 3, "t3:c.b"); + ASSERT_EQ(skel->bss->t3_ret, 10, "t3 ret"); + + ASSERT_EQ(skel->bss->t4_a_a, 10, "t4:a.a"); + ASSERT_EQ(skel->bss->t4_b, 1, "t4:b"); + ASSERT_EQ(skel->bss->t4_c, 2, "t4:c"); + ASSERT_EQ(skel->bss->t4_d, 3, "t4:d"); + ASSERT_EQ(skel->bss->t4_e_a, 2, "t4:e.a"); + ASSERT_EQ(skel->bss->t4_e_b, 3, "t4:e.b"); + ASSERT_EQ(skel->bss->t4_ret, 21, "t4 ret"); + + ASSERT_EQ(skel->bss->t5_ret, 1, "t5 ret"); + + tracing_struct__detach(skel); + tracing_struct__destroy(skel); +} + +void test_tracing_struct(void) +{ + test_fentry(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c index b0acbda6dbf5e7735c0593159e77ed785708ad64..564b75bc087f39572a5916df005ce13dd75fa281 100644 --- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c +++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c @@ -35,7 +35,7 @@ static struct bpf_program *load_prog(char *file, char *name, struct inst *inst) /* TODO: use different target function to run in concurrent mode */ void serial_test_trampoline_count(void) { - char *file = "test_trampoline_count.o"; + char *file = "test_trampoline_count.bpf.o"; char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" }; struct inst inst[MAX_TRAMP_PROGS + 1] = {}; struct bpf_program *prog; diff --git a/tools/testing/selftests/bpf/prog_tests/udp_limit.c b/tools/testing/selftests/bpf/prog_tests/udp_limit.c index 56c9d6bd38a31c34144d9731a2e4872edda3d2db..2643d896ddae7e66ea85d66c1fb2693ca5a7f1cd 100644 --- a/tools/testing/selftests/bpf/prog_tests/udp_limit.c +++ b/tools/testing/selftests/bpf/prog_tests/udp_limit.c @@ -5,8 +5,6 @@ #include <sys/types.h> #include <sys/socket.h> -static int duration; - void test_udp_limit(void) { struct udp_limit *skel; @@ -14,11 +12,11 @@ void test_udp_limit(void) int cgroup_fd; cgroup_fd = test__join_cgroup("/udp_limit"); - if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno)) + if (!ASSERT_GE(cgroup_fd, 0, "cg-join")) return; skel = udp_limit__open_and_load(); - if (CHECK(!skel, "skel-load", "errno %d", errno)) + if (!ASSERT_OK_PTR(skel, "skel-load")) goto close_cgroup_fd; skel->links.sock = bpf_program__attach_cgroup(skel->progs.sock, cgroup_fd); @@ -32,11 +30,11 @@ void test_udp_limit(void) * verify that. */ fd1 = socket(AF_INET, SOCK_DGRAM, 0); - if (CHECK(fd1 < 0, "fd1", "errno %d", errno)) + if (!ASSERT_GE(fd1, 0, "socket(fd1)")) goto close_skeleton; fd2 = socket(AF_INET, SOCK_DGRAM, 0); - if (CHECK(fd2 >= 0, "fd2", "errno %d", errno)) + if (!ASSERT_LT(fd2, 0, "socket(fd2)")) goto close_skeleton; /* We can reopen again after close. */ @@ -44,7 +42,7 @@ void test_udp_limit(void) fd1 = -1; fd1 = socket(AF_INET, SOCK_DGRAM, 0); - if (CHECK(fd1 < 0, "fd1-again", "errno %d", errno)) + if (!ASSERT_GE(fd1, 0, "socket(fd1-again)")) goto close_skeleton; /* Make sure the program was invoked the expected @@ -54,13 +52,11 @@ void test_udp_limit(void) * - close fd1 - BPF_CGROUP_INET_SOCK_RELEASE * - open fd1 again - BPF_CGROUP_INET_SOCK_CREATE */ - if (CHECK(skel->bss->invocations != 4, "bss-invocations", - "invocations=%d", skel->bss->invocations)) + if (!ASSERT_EQ(skel->bss->invocations, 4, "bss-invocations")) goto close_skeleton; /* We should still have a single socket in use */ - if (CHECK(skel->bss->in_use != 1, "bss-in_use", - "in_use=%d", skel->bss->in_use)) + if (!ASSERT_EQ(skel->bss->in_use, 1, "bss-in_use")) goto close_skeleton; close_skeleton: diff --git a/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c new file mode 100644 index 0000000000000000000000000000000000000000..02b18d018b36affc4d723f1ed6bc212a08a32bb1 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#define _GNU_SOURCE +#include <linux/compiler.h> +#include <linux/ring_buffer.h> +#include <pthread.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> +#include <sys/syscall.h> +#include <sys/sysinfo.h> +#include <test_progs.h> +#include <uapi/linux/bpf.h> +#include <unistd.h> + +#include "user_ringbuf_fail.skel.h" +#include "user_ringbuf_success.skel.h" + +#include "../progs/test_user_ringbuf.h" + +static size_t log_buf_sz = 1 << 20; /* 1 MB */ +static char obj_log_buf[1048576]; +static const long c_sample_size = sizeof(struct sample) + BPF_RINGBUF_HDR_SZ; +static const long c_ringbuf_size = 1 << 12; /* 1 small page */ +static const long c_max_entries = c_ringbuf_size / c_sample_size; + +static void drain_current_samples(void) +{ + syscall(__NR_getpgid); +} + +static int write_samples(struct user_ring_buffer *ringbuf, uint32_t num_samples) +{ + int i, err = 0; + + /* Write some number of samples to the ring buffer. */ + for (i = 0; i < num_samples; i++) { + struct sample *entry; + int read; + + entry = user_ring_buffer__reserve(ringbuf, sizeof(*entry)); + if (!entry) { + err = -errno; + goto done; + } + + entry->pid = getpid(); + entry->seq = i; + entry->value = i * i; + + read = snprintf(entry->comm, sizeof(entry->comm), "%u", i); + if (read <= 0) { + /* Assert on the error path to avoid spamming logs with + * mostly success messages. + */ + ASSERT_GT(read, 0, "snprintf_comm"); + err = read; + user_ring_buffer__discard(ringbuf, entry); + goto done; + } + + user_ring_buffer__submit(ringbuf, entry); + } + +done: + drain_current_samples(); + + return err; +} + +static struct user_ringbuf_success *open_load_ringbuf_skel(void) +{ + struct user_ringbuf_success *skel; + int err; + + skel = user_ringbuf_success__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return NULL; + + err = bpf_map__set_max_entries(skel->maps.user_ringbuf, c_ringbuf_size); + if (!ASSERT_OK(err, "set_max_entries")) + goto cleanup; + + err = bpf_map__set_max_entries(skel->maps.kernel_ringbuf, c_ringbuf_size); + if (!ASSERT_OK(err, "set_max_entries")) + goto cleanup; + + err = user_ringbuf_success__load(skel); + if (!ASSERT_OK(err, "skel_load")) + goto cleanup; + + return skel; + +cleanup: + user_ringbuf_success__destroy(skel); + return NULL; +} + +static void test_user_ringbuf_mappings(void) +{ + int err, rb_fd; + int page_size = getpagesize(); + void *mmap_ptr; + struct user_ringbuf_success *skel; + + skel = open_load_ringbuf_skel(); + if (!skel) + return; + + rb_fd = bpf_map__fd(skel->maps.user_ringbuf); + /* cons_pos can be mapped R/O, can't add +X with mprotect. */ + mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, 0); + ASSERT_OK_PTR(mmap_ptr, "ro_cons_pos"); + ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_cons_pos_protect"); + ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect"); + ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "wr_prod_pos"); + err = -errno; + ASSERT_ERR(err, "wr_prod_pos_err"); + ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro_cons"); + + /* prod_pos can be mapped RW, can't add +X with mprotect. */ + mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, + rb_fd, page_size); + ASSERT_OK_PTR(mmap_ptr, "rw_prod_pos"); + ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_prod_pos_protect"); + err = -errno; + ASSERT_ERR(err, "wr_prod_pos_err"); + ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_prod"); + + /* data pages can be mapped RW, can't add +X with mprotect. */ + mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, + 2 * page_size); + ASSERT_OK_PTR(mmap_ptr, "rw_data"); + ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_data_protect"); + err = -errno; + ASSERT_ERR(err, "exec_data_err"); + ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_data"); + + user_ringbuf_success__destroy(skel); +} + +static int load_skel_create_ringbufs(struct user_ringbuf_success **skel_out, + struct ring_buffer **kern_ringbuf_out, + ring_buffer_sample_fn callback, + struct user_ring_buffer **user_ringbuf_out) +{ + struct user_ringbuf_success *skel; + struct ring_buffer *kern_ringbuf = NULL; + struct user_ring_buffer *user_ringbuf = NULL; + int err = -ENOMEM, rb_fd; + + skel = open_load_ringbuf_skel(); + if (!skel) + return err; + + /* only trigger BPF program for current process */ + skel->bss->pid = getpid(); + + if (kern_ringbuf_out) { + rb_fd = bpf_map__fd(skel->maps.kernel_ringbuf); + kern_ringbuf = ring_buffer__new(rb_fd, callback, skel, NULL); + if (!ASSERT_OK_PTR(kern_ringbuf, "kern_ringbuf_create")) + goto cleanup; + + *kern_ringbuf_out = kern_ringbuf; + } + + if (user_ringbuf_out) { + rb_fd = bpf_map__fd(skel->maps.user_ringbuf); + user_ringbuf = user_ring_buffer__new(rb_fd, NULL); + if (!ASSERT_OK_PTR(user_ringbuf, "user_ringbuf_create")) + goto cleanup; + + *user_ringbuf_out = user_ringbuf; + ASSERT_EQ(skel->bss->read, 0, "no_reads_after_load"); + } + + err = user_ringbuf_success__attach(skel); + if (!ASSERT_OK(err, "skel_attach")) + goto cleanup; + + *skel_out = skel; + return 0; + +cleanup: + if (kern_ringbuf_out) + *kern_ringbuf_out = NULL; + if (user_ringbuf_out) + *user_ringbuf_out = NULL; + ring_buffer__free(kern_ringbuf); + user_ring_buffer__free(user_ringbuf); + user_ringbuf_success__destroy(skel); + return err; +} + +static int load_skel_create_user_ringbuf(struct user_ringbuf_success **skel_out, + struct user_ring_buffer **ringbuf_out) +{ + return load_skel_create_ringbufs(skel_out, NULL, NULL, ringbuf_out); +} + +static void manually_write_test_invalid_sample(struct user_ringbuf_success *skel, + __u32 size, __u64 producer_pos, int err) +{ + void *data_ptr; + __u64 *producer_pos_ptr; + int rb_fd, page_size = getpagesize(); + + rb_fd = bpf_map__fd(skel->maps.user_ringbuf); + + ASSERT_EQ(skel->bss->read, 0, "num_samples_before_bad_sample"); + + /* Map the producer_pos as RW. */ + producer_pos_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, rb_fd, page_size); + ASSERT_OK_PTR(producer_pos_ptr, "producer_pos_ptr"); + + /* Map the data pages as RW. */ + data_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size); + ASSERT_OK_PTR(data_ptr, "rw_data"); + + memset(data_ptr, 0, BPF_RINGBUF_HDR_SZ); + *(__u32 *)data_ptr = size; + + /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in the kernel. */ + smp_store_release(producer_pos_ptr, producer_pos + BPF_RINGBUF_HDR_SZ); + + drain_current_samples(); + ASSERT_EQ(skel->bss->read, 0, "num_samples_after_bad_sample"); + ASSERT_EQ(skel->bss->err, err, "err_after_bad_sample"); + + ASSERT_OK(munmap(producer_pos_ptr, page_size), "unmap_producer_pos"); + ASSERT_OK(munmap(data_ptr, page_size), "unmap_data_ptr"); +} + +static void test_user_ringbuf_post_misaligned(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + __u32 size = (1 << 5) + 7; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "misaligned_skel")) + return; + + manually_write_test_invalid_sample(skel, size, size, -EINVAL); + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_post_producer_wrong_offset(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + __u32 size = (1 << 5); + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "wrong_offset_skel")) + return; + + manually_write_test_invalid_sample(skel, size, size - 8, -EINVAL); + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_post_larger_than_ringbuf_sz(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + __u32 size = c_ringbuf_size; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "huge_sample_skel")) + return; + + manually_write_test_invalid_sample(skel, size, size, -E2BIG); + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_basic(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "ringbuf_basic_skel")) + return; + + ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before"); + + err = write_samples(ringbuf, 2); + if (!ASSERT_OK(err, "write_samples")) + goto cleanup; + + ASSERT_EQ(skel->bss->read, 2, "num_samples_read_after"); + +cleanup: + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_sample_full_ring_buffer(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + void *sample; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "ringbuf_full_sample_skel")) + return; + + sample = user_ring_buffer__reserve(ringbuf, c_ringbuf_size - BPF_RINGBUF_HDR_SZ); + if (!ASSERT_OK_PTR(sample, "full_sample")) + goto cleanup; + + user_ring_buffer__submit(ringbuf, sample); + ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before"); + drain_current_samples(); + ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after"); + +cleanup: + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_post_alignment_autoadjust(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + struct sample *sample; + int err; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (!ASSERT_OK(err, "ringbuf_align_autoadjust_skel")) + return; + + /* libbpf should automatically round any sample up to an 8-byte alignment. */ + sample = user_ring_buffer__reserve(ringbuf, sizeof(*sample) + 1); + ASSERT_OK_PTR(sample, "reserve_autoaligned"); + user_ring_buffer__submit(ringbuf, sample); + + ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before"); + drain_current_samples(); + ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after"); + + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_overfill(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (err) + return; + + err = write_samples(ringbuf, c_max_entries * 5); + ASSERT_ERR(err, "write_samples"); + ASSERT_EQ(skel->bss->read, c_max_entries, "max_entries"); + + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_discards_properly_ignored(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err, num_discarded = 0; + __u64 *token; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (err) + return; + + ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before"); + + while (1) { + /* Write samples until the buffer is full. */ + token = user_ring_buffer__reserve(ringbuf, sizeof(*token)); + if (!token) + break; + + user_ring_buffer__discard(ringbuf, token); + num_discarded++; + } + + if (!ASSERT_GE(num_discarded, 0, "num_discarded")) + goto cleanup; + + /* Should not read any samples, as they are all discarded. */ + ASSERT_EQ(skel->bss->read, 0, "num_pre_kick"); + drain_current_samples(); + ASSERT_EQ(skel->bss->read, 0, "num_post_kick"); + + /* Now that the ring buffer has been drained, we should be able to + * reserve another token. + */ + token = user_ring_buffer__reserve(ringbuf, sizeof(*token)); + + if (!ASSERT_OK_PTR(token, "new_token")) + goto cleanup; + + user_ring_buffer__discard(ringbuf, token); +cleanup: + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void test_user_ringbuf_loop(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + uint32_t total_samples = 8192; + uint32_t remaining_samples = total_samples; + int err; + + BUILD_BUG_ON(total_samples <= c_max_entries); + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (err) + return; + + do { + uint32_t curr_samples; + + curr_samples = remaining_samples > c_max_entries + ? c_max_entries : remaining_samples; + err = write_samples(ringbuf, curr_samples); + if (err != 0) { + /* Assert inside of if statement to avoid flooding logs + * on the success path. + */ + ASSERT_OK(err, "write_samples"); + goto cleanup; + } + + remaining_samples -= curr_samples; + ASSERT_EQ(skel->bss->read, total_samples - remaining_samples, + "current_batched_entries"); + } while (remaining_samples > 0); + ASSERT_EQ(skel->bss->read, total_samples, "total_batched_entries"); + +cleanup: + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static int send_test_message(struct user_ring_buffer *ringbuf, + enum test_msg_op op, s64 operand_64, + s32 operand_32) +{ + struct test_msg *msg; + + msg = user_ring_buffer__reserve(ringbuf, sizeof(*msg)); + if (!msg) { + /* Assert on the error path to avoid spamming logs with mostly + * success messages. + */ + ASSERT_OK_PTR(msg, "reserve_msg"); + return -ENOMEM; + } + + msg->msg_op = op; + + switch (op) { + case TEST_MSG_OP_INC64: + case TEST_MSG_OP_MUL64: + msg->operand_64 = operand_64; + break; + case TEST_MSG_OP_INC32: + case TEST_MSG_OP_MUL32: + msg->operand_32 = operand_32; + break; + default: + PRINT_FAIL("Invalid operand %d\n", op); + user_ring_buffer__discard(ringbuf, msg); + return -EINVAL; + } + + user_ring_buffer__submit(ringbuf, msg); + + return 0; +} + +static void kick_kernel_read_messages(void) +{ + syscall(__NR_prctl); +} + +static int handle_kernel_msg(void *ctx, void *data, size_t len) +{ + struct user_ringbuf_success *skel = ctx; + struct test_msg *msg = data; + + switch (msg->msg_op) { + case TEST_MSG_OP_INC64: + skel->bss->user_mutated += msg->operand_64; + return 0; + case TEST_MSG_OP_INC32: + skel->bss->user_mutated += msg->operand_32; + return 0; + case TEST_MSG_OP_MUL64: + skel->bss->user_mutated *= msg->operand_64; + return 0; + case TEST_MSG_OP_MUL32: + skel->bss->user_mutated *= msg->operand_32; + return 0; + default: + fprintf(stderr, "Invalid operand %d\n", msg->msg_op); + return -EINVAL; + } +} + +static void drain_kernel_messages_buffer(struct ring_buffer *kern_ringbuf, + struct user_ringbuf_success *skel) +{ + int cnt; + + cnt = ring_buffer__consume(kern_ringbuf); + ASSERT_EQ(cnt, 8, "consume_kern_ringbuf"); + ASSERT_OK(skel->bss->err, "consume_kern_ringbuf_err"); +} + +static void test_user_ringbuf_msg_protocol(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *user_ringbuf; + struct ring_buffer *kern_ringbuf; + int err, i; + __u64 expected_kern = 0; + + err = load_skel_create_ringbufs(&skel, &kern_ringbuf, handle_kernel_msg, &user_ringbuf); + if (!ASSERT_OK(err, "create_ringbufs")) + return; + + for (i = 0; i < 64; i++) { + enum test_msg_op op = i % TEST_MSG_OP_NUM_OPS; + __u64 operand_64 = TEST_OP_64; + __u32 operand_32 = TEST_OP_32; + + err = send_test_message(user_ringbuf, op, operand_64, operand_32); + if (err) { + /* Only assert on a failure to avoid spamming success logs. */ + ASSERT_OK(err, "send_test_message"); + goto cleanup; + } + + switch (op) { + case TEST_MSG_OP_INC64: + expected_kern += operand_64; + break; + case TEST_MSG_OP_INC32: + expected_kern += operand_32; + break; + case TEST_MSG_OP_MUL64: + expected_kern *= operand_64; + break; + case TEST_MSG_OP_MUL32: + expected_kern *= operand_32; + break; + default: + PRINT_FAIL("Unexpected op %d\n", op); + goto cleanup; + } + + if (i % 8 == 0) { + kick_kernel_read_messages(); + ASSERT_EQ(skel->bss->kern_mutated, expected_kern, "expected_kern"); + ASSERT_EQ(skel->bss->err, 0, "bpf_prog_err"); + drain_kernel_messages_buffer(kern_ringbuf, skel); + } + } + +cleanup: + ring_buffer__free(kern_ringbuf); + user_ring_buffer__free(user_ringbuf); + user_ringbuf_success__destroy(skel); +} + +static void *kick_kernel_cb(void *arg) +{ + /* Kick the kernel, causing it to drain the ring buffer and then wake + * up the test thread waiting on epoll. + */ + syscall(__NR_getrlimit); + + return NULL; +} + +static int spawn_kick_thread_for_poll(void) +{ + pthread_t thread; + + return pthread_create(&thread, NULL, kick_kernel_cb, NULL); +} + +static void test_user_ringbuf_blocking_reserve(void) +{ + struct user_ringbuf_success *skel; + struct user_ring_buffer *ringbuf; + int err, num_written = 0; + __u64 *token; + + err = load_skel_create_user_ringbuf(&skel, &ringbuf); + if (err) + return; + + ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before"); + + while (1) { + /* Write samples until the buffer is full. */ + token = user_ring_buffer__reserve(ringbuf, sizeof(*token)); + if (!token) + break; + + *token = 0xdeadbeef; + + user_ring_buffer__submit(ringbuf, token); + num_written++; + } + + if (!ASSERT_GE(num_written, 0, "num_written")) + goto cleanup; + + /* Should not have read any samples until the kernel is kicked. */ + ASSERT_EQ(skel->bss->read, 0, "num_pre_kick"); + + /* We correctly time out after 1 second, without a sample. */ + token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 1000); + if (!ASSERT_EQ(token, NULL, "pre_kick_timeout_token")) + goto cleanup; + + err = spawn_kick_thread_for_poll(); + if (!ASSERT_EQ(err, 0, "deferred_kick_thread\n")) + goto cleanup; + + /* After spawning another thread that asychronously kicks the kernel to + * drain the messages, we're able to block and successfully get a + * sample once we receive an event notification. + */ + token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 10000); + + if (!ASSERT_OK_PTR(token, "block_token")) + goto cleanup; + + ASSERT_GT(skel->bss->read, 0, "num_post_kill"); + ASSERT_LE(skel->bss->read, num_written, "num_post_kill"); + ASSERT_EQ(skel->bss->err, 0, "err_post_poll"); + user_ring_buffer__discard(ringbuf, token); + +cleanup: + user_ring_buffer__free(ringbuf); + user_ringbuf_success__destroy(skel); +} + +static struct { + const char *prog_name; + const char *expected_err_msg; +} failure_tests[] = { + /* failure cases */ + {"user_ringbuf_callback_bad_access1", "negative offset dynptr_ptr ptr"}, + {"user_ringbuf_callback_bad_access2", "dereference of modified dynptr_ptr ptr"}, + {"user_ringbuf_callback_write_forbidden", "invalid mem access 'dynptr_ptr'"}, + {"user_ringbuf_callback_null_context_write", "invalid mem access 'scalar'"}, + {"user_ringbuf_callback_null_context_read", "invalid mem access 'scalar'"}, + {"user_ringbuf_callback_discard_dynptr", "arg 1 is an unacquired reference"}, + {"user_ringbuf_callback_submit_dynptr", "arg 1 is an unacquired reference"}, + {"user_ringbuf_callback_invalid_return", "At callback return the register R0 has value"}, +}; + +#define SUCCESS_TEST(_func) { _func, #_func } + +static struct { + void (*test_callback)(void); + const char *test_name; +} success_tests[] = { + SUCCESS_TEST(test_user_ringbuf_mappings), + SUCCESS_TEST(test_user_ringbuf_post_misaligned), + SUCCESS_TEST(test_user_ringbuf_post_producer_wrong_offset), + SUCCESS_TEST(test_user_ringbuf_post_larger_than_ringbuf_sz), + SUCCESS_TEST(test_user_ringbuf_basic), + SUCCESS_TEST(test_user_ringbuf_sample_full_ring_buffer), + SUCCESS_TEST(test_user_ringbuf_post_alignment_autoadjust), + SUCCESS_TEST(test_user_ringbuf_overfill), + SUCCESS_TEST(test_user_ringbuf_discards_properly_ignored), + SUCCESS_TEST(test_user_ringbuf_loop), + SUCCESS_TEST(test_user_ringbuf_msg_protocol), + SUCCESS_TEST(test_user_ringbuf_blocking_reserve), +}; + +static void verify_fail(const char *prog_name, const char *expected_err_msg) +{ + LIBBPF_OPTS(bpf_object_open_opts, opts); + struct bpf_program *prog; + struct user_ringbuf_fail *skel; + int err; + + opts.kernel_log_buf = obj_log_buf; + opts.kernel_log_size = log_buf_sz; + opts.kernel_log_level = 1; + + skel = user_ringbuf_fail__open_opts(&opts); + if (!ASSERT_OK_PTR(skel, "dynptr_fail__open_opts")) + goto cleanup; + + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name")) + goto cleanup; + + bpf_program__set_autoload(prog, true); + + bpf_map__set_max_entries(skel->maps.user_ringbuf, getpagesize()); + + err = user_ringbuf_fail__load(skel); + if (!ASSERT_ERR(err, "unexpected load success")) + goto cleanup; + + if (!ASSERT_OK_PTR(strstr(obj_log_buf, expected_err_msg), "expected_err_msg")) { + fprintf(stderr, "Expected err_msg: %s\n", expected_err_msg); + fprintf(stderr, "Verifier output: %s\n", obj_log_buf); + } + +cleanup: + user_ringbuf_fail__destroy(skel); +} + +void test_user_ringbuf(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(success_tests); i++) { + if (!test__start_subtest(success_tests[i].test_name)) + continue; + + success_tests[i].test_callback(); + } + + for (i = 0; i < ARRAY_SIZE(failure_tests); i++) { + if (!test__start_subtest(failure_tests[i].prog_name)) + continue; + + verify_fail(failure_tests[i].prog_name, failure_tests[i].expected_err_msg); + } +} diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c new file mode 100644 index 0000000000000000000000000000000000000000..579d6ee83ce01154c7817f2ae902ce84c051f448 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <unistd.h> +#include <endian.h> +#include <limits.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <sys/mman.h> +#include <linux/keyctl.h> +#include <test_progs.h> + +#include "test_verify_pkcs7_sig.skel.h" + +#define MAX_DATA_SIZE (1024 * 1024) +#define MAX_SIG_SIZE 1024 + +#define VERIFY_USE_SECONDARY_KEYRING (1UL) +#define VERIFY_USE_PLATFORM_KEYRING (2UL) + +/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */ +#define MODULE_SIG_STRING "~Module signature appended~\n" + +/* + * Module signature information block. + * + * The constituents of the signature section are, in order: + * + * - Signer's name + * - Key identifier + * - Signature data + * - Information block + */ +struct module_signature { + __u8 algo; /* Public-key crypto algorithm [0] */ + __u8 hash; /* Digest algorithm [0] */ + __u8 id_type; /* Key identifier type [PKEY_ID_PKCS7] */ + __u8 signer_len; /* Length of signer's name [0] */ + __u8 key_id_len; /* Length of key identifier [0] */ + __u8 __pad[3]; + __be32 sig_len; /* Length of signature data */ +}; + +struct data { + __u8 data[MAX_DATA_SIZE]; + __u32 data_len; + __u8 sig[MAX_SIG_SIZE]; + __u32 sig_len; +}; + +static bool kfunc_not_supported; + +static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt, + va_list args) +{ + if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n")) + return 0; + + if (strcmp(va_arg(args, char *), "bpf_verify_pkcs7_signature")) + return 0; + + kfunc_not_supported = true; + return 0; +} + +static int _run_setup_process(const char *setup_dir, const char *cmd) +{ + int child_pid, child_status; + + child_pid = fork(); + if (child_pid == 0) { + execlp("./verify_sig_setup.sh", "./verify_sig_setup.sh", cmd, + setup_dir, NULL); + exit(errno); + + } else if (child_pid > 0) { + waitpid(child_pid, &child_status, 0); + return WEXITSTATUS(child_status); + } + + return -EINVAL; +} + +static int populate_data_item_str(const char *tmp_dir, struct data *data_item) +{ + struct stat st; + char data_template[] = "/tmp/dataXXXXXX"; + char path[PATH_MAX]; + int ret, fd, child_status, child_pid; + + data_item->data_len = 4; + memcpy(data_item->data, "test", data_item->data_len); + + fd = mkstemp(data_template); + if (fd == -1) + return -errno; + + ret = write(fd, data_item->data, data_item->data_len); + + close(fd); + + if (ret != data_item->data_len) { + ret = -EIO; + goto out; + } + + child_pid = fork(); + + if (child_pid == -1) { + ret = -errno; + goto out; + } + + if (child_pid == 0) { + snprintf(path, sizeof(path), "%s/signing_key.pem", tmp_dir); + + return execlp("./sign-file", "./sign-file", "-d", "sha256", + path, path, data_template, NULL); + } + + waitpid(child_pid, &child_status, 0); + + ret = WEXITSTATUS(child_status); + if (ret) + goto out; + + snprintf(path, sizeof(path), "%s.p7s", data_template); + + ret = stat(path, &st); + if (ret == -1) { + ret = -errno; + goto out; + } + + if (st.st_size > sizeof(data_item->sig)) { + ret = -EINVAL; + goto out_sig; + } + + data_item->sig_len = st.st_size; + + fd = open(path, O_RDONLY); + if (fd == -1) { + ret = -errno; + goto out_sig; + } + + ret = read(fd, data_item->sig, data_item->sig_len); + + close(fd); + + if (ret != data_item->sig_len) { + ret = -EIO; + goto out_sig; + } + + ret = 0; +out_sig: + unlink(path); +out: + unlink(data_template); + return ret; +} + +static int populate_data_item_mod(struct data *data_item) +{ + char mod_path[PATH_MAX], *mod_path_ptr; + struct stat st; + void *mod; + FILE *fp; + struct module_signature ms; + int ret, fd, modlen, marker_len, sig_len; + + data_item->data_len = 0; + + if (stat("/lib/modules", &st) == -1) + return 0; + + /* Requires CONFIG_TCP_CONG_BIC=m. */ + fp = popen("find /lib/modules/$(uname -r) -name tcp_bic.ko", "r"); + if (!fp) + return 0; + + mod_path_ptr = fgets(mod_path, sizeof(mod_path), fp); + pclose(fp); + + if (!mod_path_ptr) + return 0; + + mod_path_ptr = strchr(mod_path, '\n'); + if (!mod_path_ptr) + return 0; + + *mod_path_ptr = '\0'; + + if (stat(mod_path, &st) == -1) + return 0; + + modlen = st.st_size; + marker_len = sizeof(MODULE_SIG_STRING) - 1; + + fd = open(mod_path, O_RDONLY); + if (fd == -1) + return -errno; + + mod = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + + close(fd); + + if (mod == MAP_FAILED) + return -errno; + + if (strncmp(mod + modlen - marker_len, MODULE_SIG_STRING, marker_len)) { + ret = -EINVAL; + goto out; + } + + modlen -= marker_len; + + memcpy(&ms, mod + (modlen - sizeof(ms)), sizeof(ms)); + + sig_len = __be32_to_cpu(ms.sig_len); + modlen -= sig_len + sizeof(ms); + + if (modlen > sizeof(data_item->data)) { + ret = -E2BIG; + goto out; + } + + memcpy(data_item->data, mod, modlen); + data_item->data_len = modlen; + + if (sig_len > sizeof(data_item->sig)) { + ret = -E2BIG; + goto out; + } + + memcpy(data_item->sig, mod + modlen, sig_len); + data_item->sig_len = sig_len; + ret = 0; +out: + munmap(mod, st.st_size); + return ret; +} + +void test_verify_pkcs7_sig(void) +{ + libbpf_print_fn_t old_print_cb; + char tmp_dir_template[] = "/tmp/verify_sigXXXXXX"; + char *tmp_dir; + struct test_verify_pkcs7_sig *skel = NULL; + struct bpf_map *map; + struct data data; + int ret, zero = 0; + + /* Trigger creation of session keyring. */ + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + + tmp_dir = mkdtemp(tmp_dir_template); + if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) + return; + + ret = _run_setup_process(tmp_dir, "setup"); + if (!ASSERT_OK(ret, "_run_setup_process")) + goto close_prog; + + skel = test_verify_pkcs7_sig__open(); + if (!ASSERT_OK_PTR(skel, "test_verify_pkcs7_sig__open")) + goto close_prog; + + old_print_cb = libbpf_set_print(libbpf_print_cb); + ret = test_verify_pkcs7_sig__load(skel); + libbpf_set_print(old_print_cb); + + if (ret < 0 && kfunc_not_supported) { + printf( + "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n", + __func__); + test__skip(); + goto close_prog; + } + + if (!ASSERT_OK(ret, "test_verify_pkcs7_sig__load")) + goto close_prog; + + ret = test_verify_pkcs7_sig__attach(skel); + if (!ASSERT_OK(ret, "test_verify_pkcs7_sig__attach")) + goto close_prog; + + map = bpf_object__find_map_by_name(skel->obj, "data_input"); + if (!ASSERT_OK_PTR(map, "data_input not found")) + goto close_prog; + + skel->bss->monitored_pid = getpid(); + + /* Test without data and signature. */ + skel->bss->user_keyring_serial = KEY_SPEC_SESSION_KEYRING; + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input")) + goto close_prog; + + /* Test successful signature verification with session keyring. */ + ret = populate_data_item_str(tmp_dir, &data); + if (!ASSERT_OK(ret, "populate_data_item_str")) + goto close_prog; + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_OK(ret, "bpf_map_update_elem data_input")) + goto close_prog; + + /* Test successful signature verification with testing keyring. */ + skel->bss->user_keyring_serial = syscall(__NR_request_key, "keyring", + "ebpf_testing_keyring", NULL, + KEY_SPEC_SESSION_KEYRING); + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_OK(ret, "bpf_map_update_elem data_input")) + goto close_prog; + + /* + * Ensure key_task_permission() is called and rejects the keyring + * (no Search permission). + */ + syscall(__NR_keyctl, KEYCTL_SETPERM, skel->bss->user_keyring_serial, + 0x37373737); + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input")) + goto close_prog; + + syscall(__NR_keyctl, KEYCTL_SETPERM, skel->bss->user_keyring_serial, + 0x3f3f3f3f); + + /* + * Ensure key_validate() is called and rejects the keyring (key expired) + */ + syscall(__NR_keyctl, KEYCTL_SET_TIMEOUT, + skel->bss->user_keyring_serial, 1); + sleep(1); + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input")) + goto close_prog; + + skel->bss->user_keyring_serial = KEY_SPEC_SESSION_KEYRING; + + /* Test with corrupted data (signature verification should fail). */ + data.data[0] = 'a'; + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY); + if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input")) + goto close_prog; + + ret = populate_data_item_mod(&data); + if (!ASSERT_OK(ret, "populate_data_item_mod")) + goto close_prog; + + /* Test signature verification with system keyrings. */ + if (data.data_len) { + skel->bss->user_keyring_serial = 0; + skel->bss->system_keyring_id = 0; + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, + BPF_ANY); + if (!ASSERT_OK(ret, "bpf_map_update_elem data_input")) + goto close_prog; + + skel->bss->system_keyring_id = VERIFY_USE_SECONDARY_KEYRING; + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, + BPF_ANY); + if (!ASSERT_OK(ret, "bpf_map_update_elem data_input")) + goto close_prog; + + skel->bss->system_keyring_id = VERIFY_USE_PLATFORM_KEYRING; + + ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, + BPF_ANY); + ASSERT_LT(ret, 0, "bpf_map_update_elem data_input"); + } + +close_prog: + _run_setup_process(tmp_dir, "cleanup"); + + if (!skel) + return; + + skel->bss->monitored_pid = 0; + test_verify_pkcs7_sig__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/xdp.c b/tools/testing/selftests/bpf/prog_tests/xdp.c index ec21c53cb1da7dde7824e8126dc6b897c94ba1bb..947863a1d536791906357b9b7ded5afcffc3106b 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp.c @@ -8,7 +8,7 @@ void test_xdp(void) struct vip key6 = {.protocol = 6, .family = AF_INET6}; struct iptnl_info value4 = {.family = AF_INET}; struct iptnl_info value6 = {.family = AF_INET6}; - const char *file = "./test_xdp.o"; + const char *file = "./test_xdp.bpf.o"; struct bpf_object *obj; char buf[128]; struct ipv6hdr iph6; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c index 2f033da4cd457a5f00eb8a5c4278bffd51e7f32a..fce203640f8cfcd7f2aa201da3055a9f7b222d76 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c @@ -4,7 +4,7 @@ static void test_xdp_update_frags(void) { - const char *file = "./test_xdp_update_frags.o"; + const char *file = "./test_xdp_update_frags.bpf.o"; int err, prog_fd, max_skb_frags, buf_size, num; struct bpf_program *prog; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c index 21ceac24e174486239c723f6d1fb25877ab89c15..9b9cf8458adf85e930be2fc33ca419e61f5a59b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c @@ -4,7 +4,7 @@ static void test_xdp_adjust_tail_shrink(void) { - const char *file = "./test_xdp_adjust_tail_shrink.o"; + const char *file = "./test_xdp_adjust_tail_shrink.bpf.o"; __u32 expect_sz; struct bpf_object *obj; int err, prog_fd; @@ -39,7 +39,7 @@ static void test_xdp_adjust_tail_shrink(void) static void test_xdp_adjust_tail_grow(void) { - const char *file = "./test_xdp_adjust_tail_grow.o"; + const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; struct bpf_object *obj; char buf[4096]; /* avoid segfault: large buf to hold grow results */ __u32 expect_sz; @@ -73,7 +73,7 @@ static void test_xdp_adjust_tail_grow(void) static void test_xdp_adjust_tail_grow2(void) { - const char *file = "./test_xdp_adjust_tail_grow.o"; + const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; char buf[4096]; /* avoid segfault: large buf to hold grow results */ int tailroom = 320; /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info))*/; struct bpf_object *obj; @@ -135,7 +135,7 @@ static void test_xdp_adjust_tail_grow2(void) static void test_xdp_adjust_frags_tail_shrink(void) { - const char *file = "./test_xdp_adjust_tail_shrink.o"; + const char *file = "./test_xdp_adjust_tail_shrink.bpf.o"; __u32 exp_size; struct bpf_program *prog; struct bpf_object *obj; @@ -202,7 +202,7 @@ out: static void test_xdp_adjust_frags_tail_grow(void) { - const char *file = "./test_xdp_adjust_tail_grow.o"; + const char *file = "./test_xdp_adjust_tail_grow.bpf.o"; __u32 exp_size; struct bpf_program *prog; struct bpf_object *obj; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c index 62aa3edda5e6270bc097e4d556ea518871e9c3c1..062fbc8c8e5e3334513e15a2e4ac9954889e572d 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c @@ -8,7 +8,7 @@ void serial_test_xdp_attach(void) { __u32 duration = 0, id1, id2, id0 = 0, len; struct bpf_object *obj1, *obj2, *obj3; - const char *file = "./test_xdp.o"; + const char *file = "./test_xdp.bpf.o"; struct bpf_prog_info info = {}; int err, fd1, fd2, fd3; LIBBPF_OPTS(bpf_xdp_attach_opts, opts); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_info.c b/tools/testing/selftests/bpf/prog_tests/xdp_info.c index 0d01ff6cb91a288b7f59f00b54fe047f8b564010..cd3aa340e65e5c01273a014a79812efe9daca32e 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_info.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_info.c @@ -7,7 +7,7 @@ void serial_test_xdp_info(void) { __u32 len = sizeof(struct bpf_prog_info), duration = 0, prog_id; - const char *file = "./xdp_dummy.o"; + const char *file = "./xdp_dummy.bpf.o"; struct bpf_prog_info info = {}; struct bpf_object *obj; int err, prog_fd; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c index f543d1bd21b8d1e53b99f8be77ecdd59dd00b630..ec5369f247cb1859c53c055bd850e541ea07cf27 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c @@ -3,7 +3,7 @@ void test_xdp_perf(void) { - const char *file = "./xdp_dummy.o"; + const char *file = "./xdp_dummy.bpf.o"; struct bpf_object *obj; char in[128], out[128]; int err, prog_fd; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c index 874a846e298ca1a68b4891563f37970417f139ab..75550a40e029d2b1ebacb5ef79393b4ded734adc 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c @@ -82,7 +82,7 @@ static void test_synproxy(bool xdp) SYS("ethtool -K tmp0 tx off"); if (xdp) /* Workaround required for veth. */ - SYS("ip link set tmp0 xdp object xdp_dummy.o section xdp 2> /dev/null"); + SYS("ip link set tmp0 xdp object xdp_dummy.bpf.o section xdp 2> /dev/null"); ns = open_netns("synproxy"); if (!ASSERT_OK_PTR(ns, "setns")) diff --git a/tools/testing/selftests/bpf/progs/bind4_prog.c b/tools/testing/selftests/bpf/progs/bind4_prog.c index 474c6a62078a12f71ada300f1601c72de55c5c9a..a487f60b73ac49ff6e54faf346e80311a176e6f8 100644 --- a/tools/testing/selftests/bpf/progs/bind4_prog.c +++ b/tools/testing/selftests/bpf/progs/bind4_prog.c @@ -6,8 +6,6 @@ #include <linux/bpf.h> #include <linux/in.h> #include <linux/in6.h> -#include <sys/socket.h> -#include <netinet/tcp.h> #include <linux/if.h> #include <errno.h> diff --git a/tools/testing/selftests/bpf/progs/bind6_prog.c b/tools/testing/selftests/bpf/progs/bind6_prog.c index c19cfa869f308b79a9bf5a5d01df1bb0999667a6..d62cd9e9cf0eafe45ef6d77de7fb6f0638f8a5f7 100644 --- a/tools/testing/selftests/bpf/progs/bind6_prog.c +++ b/tools/testing/selftests/bpf/progs/bind6_prog.c @@ -6,8 +6,6 @@ #include <linux/bpf.h> #include <linux/in.h> #include <linux/in6.h> -#include <sys/socket.h> -#include <netinet/tcp.h> #include <linux/if.h> #include <errno.h> diff --git a/tools/testing/selftests/bpf/progs/bpf_dctcp.c b/tools/testing/selftests/bpf/progs/bpf_dctcp.c index 9573be6122bef3bb5e28b713cba458a81a30c60f..460682759aed44d93634dd2196fc83f964062282 100644 --- a/tools/testing/selftests/bpf/progs/bpf_dctcp.c +++ b/tools/testing/selftests/bpf/progs/bpf_dctcp.c @@ -11,6 +11,7 @@ #include <linux/types.h> #include <linux/stddef.h> #include <linux/tcp.h> +#include <errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> #include "bpf_tcp_helpers.h" @@ -23,6 +24,7 @@ const char tcp_cdg[] = "cdg"; char cc_res[TCP_CA_NAME_MAX]; int tcp_cdg_res = 0; int stg_result = 0; +int ebusy_cnt = 0; struct { __uint(type, BPF_MAP_TYPE_SK_STORAGE); @@ -64,16 +66,23 @@ void BPF_PROG(dctcp_init, struct sock *sk) if (!(tp->ecn_flags & TCP_ECN_OK) && fallback[0]) { /* Switch to fallback */ - bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, - (void *)fallback, sizeof(fallback)); - /* Switch back to myself which the bpf trampoline - * stopped calling dctcp_init recursively. + if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, + (void *)fallback, sizeof(fallback)) == -EBUSY) + ebusy_cnt++; + + /* Switch back to myself and the recurred dctcp_init() + * will get -EBUSY for all bpf_setsockopt(TCP_CONGESTION), + * except the last "cdg" one. */ - bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, - (void *)bpf_dctcp, sizeof(bpf_dctcp)); + if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, + (void *)bpf_dctcp, sizeof(bpf_dctcp)) == -EBUSY) + ebusy_cnt++; + /* Switch back to fallback */ - bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, - (void *)fallback, sizeof(fallback)); + if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, + (void *)fallback, sizeof(fallback)) == -EBUSY) + ebusy_cnt++; + /* Expecting -ENOTSUPP for tcp_cdg_res */ tcp_cdg_res = bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION, (void *)tcp_cdg, sizeof(tcp_cdg)); diff --git a/tools/testing/selftests/bpf/progs/bpf_flow.c b/tools/testing/selftests/bpf/progs/bpf_flow.c index f266c757b3df3368f4fad8ed8ba175fe875721c4..a20c5ed5e4546cfef792ba301b8ece7efc3a2f7d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_flow.c +++ b/tools/testing/selftests/bpf/progs/bpf_flow.c @@ -22,6 +22,8 @@ #define PROG(F) PROG_(F, _##F) #define PROG_(NUM, NAME) SEC("flow_dissector") int flow_dissector_##NUM +#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */ + /* These are the identifiers of the BPF programs that will be used in tail * calls. Name is limited to 16 characters, with the terminating character and * bpf_func_ above, we have only 6 to work with, anything after will be cropped. @@ -143,6 +145,19 @@ int _dissect(struct __sk_buff *skb) { struct bpf_flow_keys *keys = skb->flow_keys; + if (keys->n_proto == bpf_htons(ETH_P_IP)) { + /* IP traffic from FLOW_CONTINUE_SADDR falls-back to + * standard dissector + */ + struct iphdr *iph, _iph; + + iph = bpf_flow_dissect_get_header(skb, sizeof(*iph), &_iph); + if (iph && iph->ihl == 5 && + iph->saddr == bpf_htonl(FLOW_CONTINUE_SADDR)) { + return BPF_FLOW_DISSECTOR_CONTINUE; + } + } + return parse_eth_proto(skb, keys->n_proto); } diff --git a/tools/testing/selftests/bpf/progs/bpf_iter.h b/tools/testing/selftests/bpf/progs/bpf_iter.h index e9846606690d7e70d9a9a149fdf3e4bf15490ab8..c41ee80533ca219a59cf5640eb008df2b6c10380 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter.h +++ b/tools/testing/selftests/bpf/progs/bpf_iter.h @@ -17,6 +17,7 @@ #define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used #define bpf_iter__sockmap bpf_iter__sockmap___not_used #define bpf_iter__bpf_link bpf_iter__bpf_link___not_used +#define bpf_iter__cgroup bpf_iter__cgroup___not_used #define btf_ptr btf_ptr___not_used #define BTF_F_COMPACT BTF_F_COMPACT___not_used #define BTF_F_NONAME BTF_F_NONAME___not_used @@ -40,6 +41,7 @@ #undef bpf_iter__bpf_sk_storage_map #undef bpf_iter__sockmap #undef bpf_iter__bpf_link +#undef bpf_iter__cgroup #undef btf_ptr #undef BTF_F_COMPACT #undef BTF_F_NONAME @@ -141,6 +143,11 @@ struct bpf_iter__bpf_link { struct bpf_link *link; }; +struct bpf_iter__cgroup { + struct bpf_iter_meta *meta; + struct cgroup *cgroup; +} __attribute__((preserve_access_index)); + struct btf_ptr { void *ptr; __u32 type_id; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task.c b/tools/testing/selftests/bpf/progs/bpf_iter_task.c index d22741272692193dcf023b259f30547acce15e01..96131b9a1caae5f4337b97f4811b3de43c5b785d 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_task.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task.c @@ -6,6 +6,10 @@ char _license[] SEC("license") = "GPL"; +uint32_t tid = 0; +int num_unknown_tid = 0; +int num_known_tid = 0; + SEC("iter/task") int dump_task(struct bpf_iter__task *ctx) { @@ -18,6 +22,11 @@ int dump_task(struct bpf_iter__task *ctx) return 0; } + if (task->pid != tid) + num_unknown_tid++; + else + num_known_tid++; + if (ctx->meta->seq_num == 0) BPF_SEQ_PRINTF(seq, " tgid gid\n"); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c index 6e7b400888fec097a4d6221dc93688d834d96f6f..b0255080662d57bfeb788ceaf2e2b570097e99e9 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c @@ -7,14 +7,16 @@ char _license[] SEC("license") = "GPL"; int count = 0; int tgid = 0; +int last_tgid = 0; +int unique_tgid_count = 0; SEC("iter/task_file") int dump_task_file(struct bpf_iter__task_file *ctx) { struct seq_file *seq = ctx->meta->seq; struct task_struct *task = ctx->task; - __u32 fd = ctx->fd; struct file *file = ctx->file; + __u32 fd = ctx->fd; if (task == (void *)0 || file == (void *)0) return 0; @@ -27,6 +29,11 @@ int dump_task_file(struct bpf_iter__task_file *ctx) if (tgid == task->tgid && task->tgid != task->pid) count++; + if (last_tgid != task->tgid) { + last_tgid = task->tgid; + unique_tgid_count++; + } + BPF_SEQ_PRINTF(seq, "%8d %8d %8d %lx\n", task->tgid, task->pid, fd, (long)file->f_op); return 0; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c index 4ea6a37d1345f059be8c52847fb0ba7e5bc4de7d..dd923dc637d5c62d187431506856265332efbad9 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c @@ -20,6 +20,8 @@ char _license[] SEC("license") = "GPL"; #define D_PATH_BUF_SIZE 1024 char d_path_buf[D_PATH_BUF_SIZE] = {}; __u32 pid = 0; +__u32 one_task = 0; +__u32 one_task_error = 0; SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx) { @@ -33,8 +35,11 @@ SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx) return 0; file = vma->vm_file; - if (task->tgid != pid) + if (task->tgid != pid) { + if (one_task) + one_task_error = 1; return 0; + } perm_str[0] = (vma->vm_flags & VM_READ) ? 'r' : '-'; perm_str[1] = (vma->vm_flags & VM_WRITE) ? 'w' : '-'; perm_str[2] = (vma->vm_flags & VM_EXEC) ? 'x' : '-'; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c b/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c new file mode 100644 index 0000000000000000000000000000000000000000..ee7455d2623aab28d307c10d09c4bd096b6de8d3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ +#include "bpf_iter.h" +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +__u32 unique_tgid_cnt = 0; +uintptr_t address = 0; +uintptr_t offset = 0; +__u32 last_tgid = 0; +__u32 pid = 0; +__u32 page_shift = 0; + +SEC("iter/task_vma") +int get_vma_offset(struct bpf_iter__task_vma *ctx) +{ + struct vm_area_struct *vma = ctx->vma; + struct seq_file *seq = ctx->meta->seq; + struct task_struct *task = ctx->task; + + if (task == NULL || vma == NULL) + return 0; + + if (last_tgid != task->tgid) + unique_tgid_cnt++; + last_tgid = task->tgid; + + if (task->tgid != pid) + return 0; + + if (vma->vm_start <= address && vma->vm_end > address) { + offset = address - vma->vm_start + (vma->vm_pgoff << page_shift); + BPF_SEQ_PRINTF(seq, "OK\n"); + } + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 98dd2c4815f0c71ba5b959a67882be1d977a40da..adb087aecc9eb4d15d15cf40d28d7537722c87aa 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -6,13 +6,41 @@ #define AF_INET6 10 #define SOL_SOCKET 1 +#define SO_REUSEADDR 2 #define SO_SNDBUF 7 -#define __SO_ACCEPTCON (1 << 16) +#define SO_RCVBUF 8 +#define SO_KEEPALIVE 9 #define SO_PRIORITY 12 +#define SO_REUSEPORT 15 +#define SO_RCVLOWAT 18 +#define SO_BINDTODEVICE 25 +#define SO_MARK 36 +#define SO_MAX_PACING_RATE 47 +#define SO_BINDTOIFINDEX 62 +#define SO_TXREHASH 74 +#define __SO_ACCEPTCON (1 << 16) + +#define IP_TOS 1 + +#define IPV6_TCLASS 67 +#define IPV6_AUTOFLOWLABEL 70 #define SOL_TCP 6 +#define TCP_NODELAY 1 +#define TCP_MAXSEG 2 +#define TCP_KEEPIDLE 4 +#define TCP_KEEPINTVL 5 +#define TCP_KEEPCNT 6 +#define TCP_SYNCNT 7 +#define TCP_WINDOW_CLAMP 10 #define TCP_CONGESTION 13 +#define TCP_THIN_LINEAR_TIMEOUTS 16 +#define TCP_USER_TIMEOUT 18 +#define TCP_NOTSENT_LOWAT 25 +#define TCP_SAVE_SYN 27 +#define TCP_SAVED_SYN 28 #define TCP_CA_NAME_MAX 16 +#define TCP_NAGLE_OFF 1 #define ICSK_TIME_RETRANS 1 #define ICSK_TIME_PROBE0 3 @@ -49,6 +77,8 @@ #define sk_state __sk_common.skc_state #define sk_v6_daddr __sk_common.skc_v6_daddr #define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr +#define sk_flags __sk_common.skc_flags +#define sk_reuse __sk_common.skc_reuse #define s6_addr32 in6_u.u6_addr32 diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c new file mode 100644 index 0000000000000000000000000000000000000000..7653df1bc7871bb069aaec98795f852a1ad3789f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cb_refs.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> + +struct map_value { + struct prog_test_ref_kfunc __kptr_ref *ptr; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value); + __uint(max_entries, 16); +} array_map SEC(".maps"); + +extern struct prog_test_ref_kfunc *bpf_kfunc_call_test_acquire(unsigned long *sp) __ksym; +extern void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym; + +static __noinline int cb1(void *map, void *key, void *value, void *ctx) +{ + void *p = *(void **)ctx; + bpf_kfunc_call_test_release(p); + /* Without the fix this would cause underflow */ + return 0; +} + +SEC("?tc") +int underflow_prog(void *ctx) +{ + struct prog_test_ref_kfunc *p; + unsigned long sl = 0; + + p = bpf_kfunc_call_test_acquire(&sl); + if (!p) + return 0; + bpf_for_each_map_elem(&array_map, cb1, &p, 0); + return 0; +} + +static __always_inline int cb2(void *map, void *key, void *value, void *ctx) +{ + unsigned long sl = 0; + + *(void **)ctx = bpf_kfunc_call_test_acquire(&sl); + /* Without the fix this would leak memory */ + return 0; +} + +SEC("?tc") +int leak_prog(void *ctx) +{ + struct prog_test_ref_kfunc *p; + struct map_value *v; + unsigned long sl; + + v = bpf_map_lookup_elem(&array_map, &(int){0}); + if (!v) + return 0; + + p = NULL; + bpf_for_each_map_elem(&array_map, cb2, &p, 0); + p = bpf_kptr_xchg(&v->ptr, p); + if (p) + bpf_kfunc_call_test_release(p); + return 0; +} + +static __always_inline int cb(void *map, void *key, void *value, void *ctx) +{ + return 0; +} + +static __always_inline int cb3(void *map, void *key, void *value, void *ctx) +{ + unsigned long sl = 0; + void *p; + + bpf_kfunc_call_test_acquire(&sl); + bpf_for_each_map_elem(&array_map, cb, &p, 0); + /* It should only complain here, not in cb. This is why we need + * callback_ref to be set to frameno. + */ + return 0; +} + +SEC("?tc") +int nested_cb(void *ctx) +{ + struct prog_test_ref_kfunc *p; + unsigned long sl = 0; + int sp = 0; + + p = bpf_kfunc_call_test_acquire(&sl); + if (!p) + return 0; + bpf_for_each_map_elem(&array_map, cb3, &sp, 0); + bpf_kfunc_call_test_release(p); + return 0; +} + +SEC("?tc") +int non_cb_transfer_ref(void *ctx) +{ + struct prog_test_ref_kfunc *p; + unsigned long sl = 0; + + p = bpf_kfunc_call_test_acquire(&sl); + if (!p) + return 0; + cb1(NULL, NULL, NULL, &p); + bpf_kfunc_call_test_acquire(&sl); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c new file mode 100644 index 0000000000000000000000000000000000000000..13dfb4bbfd2851ce06a50cb2e824dfe255335587 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +#define BPF_RETVAL_HOOK(name, section, ctx, expected_err) \ + __attribute__((__section__("?" section))) \ + int name(struct ctx *_ctx) \ + { \ + bpf_set_retval(bpf_get_retval()); \ + return 1; \ + } + +#include "cgroup_getset_retval_hooks.h" + +#undef BPF_RETVAL_HOOK diff --git a/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c new file mode 100644 index 0000000000000000000000000000000000000000..c74362854948d87da496cbc729db02b1bface2a2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2022 Google LLC. + */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +char _license[] SEC("license") = "GPL"; + +struct percpu_attach_counter { + /* Previous percpu state, to figure out if we have new updates */ + __u64 prev; + /* Current percpu state */ + __u64 state; +}; + +struct attach_counter { + /* State propagated through children, pending aggregation */ + __u64 pending; + /* Total state, including all cpus and all children */ + __u64 state; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(max_entries, 1024); + __type(key, __u64); + __type(value, struct percpu_attach_counter); +} percpu_attach_counters SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1024); + __type(key, __u64); + __type(value, struct attach_counter); +} attach_counters SEC(".maps"); + +extern void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) __ksym; +extern void cgroup_rstat_flush(struct cgroup *cgrp) __ksym; + +static uint64_t cgroup_id(struct cgroup *cgrp) +{ + return cgrp->kn->id; +} + +static int create_percpu_attach_counter(__u64 cg_id, __u64 state) +{ + struct percpu_attach_counter pcpu_init = {.state = state, .prev = 0}; + + return bpf_map_update_elem(&percpu_attach_counters, &cg_id, + &pcpu_init, BPF_NOEXIST); +} + +static int create_attach_counter(__u64 cg_id, __u64 state, __u64 pending) +{ + struct attach_counter init = {.state = state, .pending = pending}; + + return bpf_map_update_elem(&attach_counters, &cg_id, + &init, BPF_NOEXIST); +} + +SEC("fentry/cgroup_attach_task") +int BPF_PROG(counter, struct cgroup *dst_cgrp, struct task_struct *leader, + bool threadgroup) +{ + __u64 cg_id = cgroup_id(dst_cgrp); + struct percpu_attach_counter *pcpu_counter = bpf_map_lookup_elem( + &percpu_attach_counters, + &cg_id); + + if (pcpu_counter) + pcpu_counter->state += 1; + else if (create_percpu_attach_counter(cg_id, 1)) + return 0; + + cgroup_rstat_updated(dst_cgrp, bpf_get_smp_processor_id()); + return 0; +} + +SEC("fentry/bpf_rstat_flush") +int BPF_PROG(flusher, struct cgroup *cgrp, struct cgroup *parent, int cpu) +{ + struct percpu_attach_counter *pcpu_counter; + struct attach_counter *total_counter, *parent_counter; + __u64 cg_id = cgroup_id(cgrp); + __u64 parent_cg_id = parent ? cgroup_id(parent) : 0; + __u64 state; + __u64 delta = 0; + + /* Add CPU changes on this level since the last flush */ + pcpu_counter = bpf_map_lookup_percpu_elem(&percpu_attach_counters, + &cg_id, cpu); + if (pcpu_counter) { + state = pcpu_counter->state; + delta += state - pcpu_counter->prev; + pcpu_counter->prev = state; + } + + total_counter = bpf_map_lookup_elem(&attach_counters, &cg_id); + if (!total_counter) { + if (create_attach_counter(cg_id, delta, 0)) + return 0; + goto update_parent; + } + + /* Collect pending stats from subtree */ + if (total_counter->pending) { + delta += total_counter->pending; + total_counter->pending = 0; + } + + /* Propagate changes to this cgroup's total */ + total_counter->state += delta; + +update_parent: + /* Skip if there are no changes to propagate, or no parent */ + if (!delta || !parent_cg_id) + return 0; + + /* Propagate changes to cgroup's parent */ + parent_counter = bpf_map_lookup_elem(&attach_counters, + &parent_cg_id); + if (parent_counter) + parent_counter->pending += delta; + else + create_attach_counter(parent_cg_id, 0, delta); + return 0; +} + +SEC("iter.s/cgroup") +int BPF_PROG(dumper, struct bpf_iter_meta *meta, struct cgroup *cgrp) +{ + struct seq_file *seq = meta->seq; + struct attach_counter *total_counter; + __u64 cg_id = cgrp ? cgroup_id(cgrp) : 0; + + /* Do nothing for the terminal call */ + if (!cg_id) + return 1; + + /* Flush the stats to make sure we get the most updated numbers */ + cgroup_rstat_flush(cgrp); + + total_counter = bpf_map_lookup_elem(&attach_counters, &cg_id); + if (!total_counter) { + BPF_SEQ_PRINTF(seq, "cg_id: %llu, attach_counter: 0\n", + cg_id); + } else { + BPF_SEQ_PRINTF(seq, "cg_id: %llu, attach_counter: %llu\n", + cg_id, total_counter->state); + } + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/cgroup_iter.c b/tools/testing/selftests/bpf/progs/cgroup_iter.c new file mode 100644 index 0000000000000000000000000000000000000000..de03997322a793f39e4889bc6c7bc168124b8988 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/cgroup_iter.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Google */ + +#include "bpf_iter.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; +int terminate_early = 0; +u64 terminal_cgroup = 0; + +static inline u64 cgroup_id(struct cgroup *cgrp) +{ + return cgrp->kn->id; +} + +SEC("iter/cgroup") +int cgroup_id_printer(struct bpf_iter__cgroup *ctx) +{ + struct seq_file *seq = ctx->meta->seq; + struct cgroup *cgrp = ctx->cgroup; + + /* epilogue */ + if (cgrp == NULL) { + BPF_SEQ_PRINTF(seq, "epilogue\n"); + return 0; + } + + /* prologue */ + if (ctx->meta->seq_num == 0) + BPF_SEQ_PRINTF(seq, "prologue\n"); + + BPF_SEQ_PRINTF(seq, "%8llu\n", cgroup_id(cgrp)); + + if (terminal_cgroup == cgroup_id(cgrp)) + return 1; + + return terminate_early ? 1 : 0; +} diff --git a/tools/testing/selftests/bpf/progs/connect4_prog.c b/tools/testing/selftests/bpf/progs/connect4_prog.c index b241932911dbb3bb7ad44ac566a81a4d2716c695..ec25371de789db8c1f53dcb6a332d9644e2d7b87 100644 --- a/tools/testing/selftests/bpf/progs/connect4_prog.c +++ b/tools/testing/selftests/bpf/progs/connect4_prog.c @@ -7,14 +7,15 @@ #include <linux/bpf.h> #include <linux/in.h> #include <linux/in6.h> -#include <sys/socket.h> -#include <netinet/tcp.h> +#include <linux/tcp.h> #include <linux/if.h> #include <errno.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> +#include "bpf_tcp_helpers.h" + #define SRC_REWRITE_IP4 0x7f000004U #define DST_REWRITE_IP4 0x7f000001U #define DST_REWRITE_PORT4 4444 diff --git a/tools/testing/selftests/bpf/progs/connect_ping.c b/tools/testing/selftests/bpf/progs/connect_ping.c new file mode 100644 index 0000000000000000000000000000000000000000..60178192b672f1175d96aa05f334540026607d48 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/connect_ping.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright 2022 Google LLC. + */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> +#include <netinet/in.h> +#include <sys/socket.h> + +/* 2001:db8::1 */ +#define BINDADDR_V6 { { { 0x20,0x01,0x0d,0xb8,0,0,0,0,0,0,0,0,0,0,0,1 } } } + +__u32 do_bind = 0; +__u32 has_error = 0; +__u32 invocations_v4 = 0; +__u32 invocations_v6 = 0; + +SEC("cgroup/connect4") +int connect_v4_prog(struct bpf_sock_addr *ctx) +{ + struct sockaddr_in sa = { + .sin_family = AF_INET, + .sin_addr.s_addr = bpf_htonl(0x01010101), + }; + + __sync_fetch_and_add(&invocations_v4, 1); + + if (do_bind && bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa))) + has_error = 1; + + return 1; +} + +SEC("cgroup/connect6") +int connect_v6_prog(struct bpf_sock_addr *ctx) +{ + struct sockaddr_in6 sa = { + .sin6_family = AF_INET6, + .sin6_addr = BINDADDR_V6, + }; + + __sync_fetch_and_add(&invocations_v6, 1); + + if (do_bind && bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa))) + has_error = 1; + + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 0a26c243e6e9c54604f83a63334cdfcdf092b45f..b0f08ff024fb8eda284d7bfd908a0adf70ad0b38 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -65,7 +65,7 @@ static int get_map_val_dynptr(struct bpf_dynptr *ptr) /* Every bpf_ringbuf_reserve_dynptr call must have a corresponding * bpf_ringbuf_submit/discard_dynptr call */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int ringbuf_missing_release1(void *ctx) { struct bpf_dynptr ptr; @@ -77,7 +77,7 @@ int ringbuf_missing_release1(void *ctx) return 0; } -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int ringbuf_missing_release2(void *ctx) { struct bpf_dynptr ptr1, ptr2; @@ -112,7 +112,7 @@ static int missing_release_callback_fn(__u32 index, void *data) } /* Any dynptr initialized within a callback must have bpf_dynptr_put called */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int ringbuf_missing_release_callback(void *ctx) { bpf_loop(10, missing_release_callback_fn, NULL, 0); @@ -120,7 +120,7 @@ int ringbuf_missing_release_callback(void *ctx) } /* Can't call bpf_ringbuf_submit/discard_dynptr on a non-initialized dynptr */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int ringbuf_release_uninit_dynptr(void *ctx) { struct bpf_dynptr ptr; @@ -132,7 +132,7 @@ int ringbuf_release_uninit_dynptr(void *ctx) } /* A dynptr can't be used after it has been invalidated */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int use_after_invalid(void *ctx) { struct bpf_dynptr ptr; @@ -151,7 +151,7 @@ int use_after_invalid(void *ctx) } /* Can't call non-dynptr ringbuf APIs on a dynptr ringbuf sample */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int ringbuf_invalid_api(void *ctx) { struct bpf_dynptr ptr; @@ -173,7 +173,7 @@ done: } /* Can't add a dynptr to a map */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int add_dynptr_to_map1(void *ctx) { struct bpf_dynptr ptr; @@ -190,7 +190,7 @@ int add_dynptr_to_map1(void *ctx) } /* Can't add a struct with an embedded dynptr to a map */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int add_dynptr_to_map2(void *ctx) { struct test_info x; @@ -207,7 +207,7 @@ int add_dynptr_to_map2(void *ctx) } /* A data slice can't be accessed out of bounds */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int data_slice_out_of_bounds_ringbuf(void *ctx) { struct bpf_dynptr ptr; @@ -227,7 +227,7 @@ done: return 0; } -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int data_slice_out_of_bounds_map_value(void *ctx) { __u32 key = 0, map_val; @@ -247,8 +247,8 @@ int data_slice_out_of_bounds_map_value(void *ctx) } /* A data slice can't be used after it has been released */ -SEC("?raw_tp/sys_nanosleep") -int data_slice_use_after_release(void *ctx) +SEC("?raw_tp") +int data_slice_use_after_release1(void *ctx) { struct bpf_dynptr ptr; struct sample *sample; @@ -272,8 +272,44 @@ done: return 0; } +/* A data slice can't be used after it has been released. + * + * This tests the case where the data slice tracks a dynptr (ptr2) + * that is at a non-zero offset from the frame pointer (ptr1 is at fp, + * ptr2 is at fp - 16). + */ +SEC("?raw_tp") +int data_slice_use_after_release2(void *ctx) +{ + struct bpf_dynptr ptr1, ptr2; + struct sample *sample; + + bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr1); + bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr2); + + sample = bpf_dynptr_data(&ptr2, 0, sizeof(*sample)); + if (!sample) + goto done; + + sample->pid = 23; + + bpf_ringbuf_submit_dynptr(&ptr2, 0); + + /* this should fail */ + sample->pid = 23; + + bpf_ringbuf_submit_dynptr(&ptr1, 0); + + return 0; + +done: + bpf_ringbuf_discard_dynptr(&ptr2, 0); + bpf_ringbuf_discard_dynptr(&ptr1, 0); + return 0; +} + /* A data slice must be first checked for NULL */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int data_slice_missing_null_check1(void *ctx) { struct bpf_dynptr ptr; @@ -293,7 +329,7 @@ int data_slice_missing_null_check1(void *ctx) } /* A data slice can't be dereferenced if it wasn't checked for null */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int data_slice_missing_null_check2(void *ctx) { struct bpf_dynptr ptr; @@ -315,7 +351,7 @@ done: /* Can't pass in a dynptr as an arg to a helper function that doesn't take in a * dynptr argument */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_helper1(void *ctx) { struct bpf_dynptr ptr; @@ -329,7 +365,7 @@ int invalid_helper1(void *ctx) } /* A dynptr can't be passed into a helper function at a non-zero offset */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_helper2(void *ctx) { struct bpf_dynptr ptr; @@ -344,7 +380,7 @@ int invalid_helper2(void *ctx) } /* A bpf_dynptr is invalidated if it's been written into */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_write1(void *ctx) { struct bpf_dynptr ptr; @@ -365,7 +401,7 @@ int invalid_write1(void *ctx) * A bpf_dynptr can't be used as a dynptr if it has been written into at a fixed * offset */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_write2(void *ctx) { struct bpf_dynptr ptr; @@ -388,7 +424,7 @@ int invalid_write2(void *ctx) * A bpf_dynptr can't be used as a dynptr if it has been written into at a * non-const offset */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_write3(void *ctx) { struct bpf_dynptr ptr; @@ -419,7 +455,7 @@ static int invalid_write4_callback(__u32 index, void *data) /* If the dynptr is written into in a callback function, it should * be invalidated as a dynptr */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_write4(void *ctx) { struct bpf_dynptr ptr; @@ -436,7 +472,7 @@ int invalid_write4(void *ctx) /* A globally-defined bpf_dynptr can't be used (it must reside as a stack frame) */ struct bpf_dynptr global_dynptr; -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int global(void *ctx) { /* this should fail */ @@ -448,7 +484,7 @@ int global(void *ctx) } /* A direct read should fail */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_read1(void *ctx) { struct bpf_dynptr ptr; @@ -464,7 +500,7 @@ int invalid_read1(void *ctx) } /* A direct read at an offset should fail */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_read2(void *ctx) { struct bpf_dynptr ptr; @@ -479,7 +515,7 @@ int invalid_read2(void *ctx) } /* A direct read at an offset into the lower stack slot should fail */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_read3(void *ctx) { struct bpf_dynptr ptr1, ptr2; @@ -505,7 +541,7 @@ static int invalid_read4_callback(__u32 index, void *data) } /* A direct read within a callback function should fail */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_read4(void *ctx) { struct bpf_dynptr ptr; @@ -520,7 +556,7 @@ int invalid_read4(void *ctx) } /* Initializing a dynptr on an offset should fail */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int invalid_offset(void *ctx) { struct bpf_dynptr ptr; @@ -534,7 +570,7 @@ int invalid_offset(void *ctx) } /* Can't release a dynptr twice */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int release_twice(void *ctx) { struct bpf_dynptr ptr; @@ -560,7 +596,7 @@ static int release_twice_callback_fn(__u32 index, void *data) /* Test that releasing a dynptr twice, where one of the releases happens * within a calback function, fails */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int release_twice_callback(void *ctx) { struct bpf_dynptr ptr; @@ -575,7 +611,7 @@ int release_twice_callback(void *ctx) } /* Reject unsupported local mem types for dynptr_from_mem API */ -SEC("?raw_tp/sys_nanosleep") +SEC("?raw_tp") int dynptr_from_mem_invalid_api(void *ctx) { struct bpf_dynptr ptr; diff --git a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c index 48cd14b437418b4b4a88df77c0846edbe3838299..4547b059d487783205554d06ccfb5faff785f634 100644 --- a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c @@ -73,10 +73,10 @@ int test_subprog2(struct args_subprog2 *ctx) __builtin_preserve_access_index(&skb->len)); ret = ctx->ret; - /* bpf_prog_test_load() loads "test_pkt_access.o" with BPF_F_TEST_RND_HI32 - * which randomizes upper 32 bits after BPF_ALU32 insns. - * Hence after 'w0 <<= 1' upper bits of $rax are random. - * That is expected and correct. Trim them. + /* bpf_prog_test_load() loads "test_pkt_access.bpf.o" with + * BPF_F_TEST_RND_HI32 which randomizes upper 32 bits after BPF_ALU32 + * insns. Hence after 'w0 <<= 1' upper bits of $rax are random. That is + * expected and correct. Trim them. */ ret = (__u32) ret; if (len != 74 || ret != 148) diff --git a/tools/testing/selftests/bpf/progs/get_func_ip_test.c b/tools/testing/selftests/bpf/progs/get_func_ip_test.c index a587aeca5ae0848b759a2cfc95a22a0a564e87e5..8559e698b40d7ce9d103a4ba868286f33782dda8 100644 --- a/tools/testing/selftests/bpf/progs/get_func_ip_test.c +++ b/tools/testing/selftests/bpf/progs/get_func_ip_test.c @@ -2,6 +2,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> +#include <stdbool.h> char _license[] SEC("license") = "GPL"; @@ -13,6 +14,16 @@ extern const void bpf_modify_return_test __ksym; extern const void bpf_fentry_test6 __ksym; extern const void bpf_fentry_test7 __ksym; +extern bool CONFIG_X86_KERNEL_IBT __kconfig __weak; + +/* This function is here to have CONFIG_X86_KERNEL_IBT + * used and added to object BTF. + */ +int unused(void) +{ + return CONFIG_X86_KERNEL_IBT ? 0 : 1; +} + __u64 test1_result = 0; SEC("fentry/bpf_fentry_test1") int BPF_PROG(test1, int a) @@ -64,21 +75,11 @@ int BPF_PROG(test5, int a, int *b, int ret) } __u64 test6_result = 0; -SEC("kprobe/bpf_fentry_test6+0x5") +SEC("?kprobe") int test6(struct pt_regs *ctx) { __u64 addr = bpf_get_func_ip(ctx); - test6_result = (const void *) addr == &bpf_fentry_test6 + 5; - return 0; -} - -__u64 test7_result = 0; -SEC("kprobe/bpf_fentry_test7+5") -int test7(struct pt_regs *ctx) -{ - __u64 addr = bpf_get_func_ip(ctx); - - test7_result = (const void *) addr == &bpf_fentry_test7 + 5; + test6_result = (const void *) addr == 0; return 0; } diff --git a/tools/testing/selftests/bpf/progs/htab_update.c b/tools/testing/selftests/bpf/progs/htab_update.c new file mode 100644 index 0000000000000000000000000000000000000000..7481bb30b29b3f79f4f0bdea2b135a5ab4e8f7a6 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/htab_update.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022. Huawei Technologies Co., Ltd */ +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} htab SEC(".maps"); + +int pid = 0; +int update_err = 0; + +SEC("?fentry/lookup_elem_raw") +int lookup_elem_raw(void *ctx) +{ + __u32 key = 0, value = 1; + + if ((bpf_get_current_pid_tgid() >> 32) != pid) + return 0; + + update_err = bpf_map_update_elem(&htab, &key, &value, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c b/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c new file mode 100644 index 0000000000000000000000000000000000000000..767472bc5a97b5c0b49372f30ec9acf5d0a7b9e1 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +extern void bpf_kfunc_call_test_destructive(void) __ksym; + +SEC("tc") +int kfunc_destructive_test(void) +{ + bpf_kfunc_call_test_destructive(); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c new file mode 100644 index 0000000000000000000000000000000000000000..b98313d391c6616189172eda7eadda450dca06c2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include <vmlinux.h> +#include <bpf/bpf_helpers.h> + +extern struct prog_test_ref_kfunc *bpf_kfunc_call_test_acquire(unsigned long *sp) __ksym; +extern void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym; +extern void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym; +extern int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; +extern int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; +extern int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; +extern void bpf_kfunc_call_int_mem_release(int *p) __ksym; + +struct syscall_test_args { + __u8 data[16]; + size_t size; +}; + +SEC("?syscall") +int kfunc_syscall_test_fail(struct syscall_test_args *args) +{ + bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(*args) + 1); + + return 0; +} + +SEC("?syscall") +int kfunc_syscall_test_null_fail(struct syscall_test_args *args) +{ + /* Must be called with args as a NULL pointer + * we do not check for it to have the verifier consider that + * the pointer might not be null, and so we can load it. + * + * So the following can not be added: + * + * if (args) + * return -22; + */ + + bpf_kfunc_call_test_mem_len_pass1(args, sizeof(*args)); + + return 0; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_rdonly(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int)); + if (p) + p[0] = 42; /* this is a read-only buffer, so -EACCES */ + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_use_after_free(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + p = bpf_kfunc_call_test_get_rdwr_mem(pt, 2 * sizeof(int)); + if (p) { + p[0] = 42; + ret = p[1]; /* 108 */ + } else { + ret = -1; + } + + bpf_kfunc_call_test_release(pt); + } + if (p) + ret = p[0]; /* p is not valid anymore */ + + return ret; +} + +SEC("?tc") +int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int)); + if (p) + ret = p[2 * sizeof(int)]; /* oob access, so -EACCES */ + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +int not_const_size = 2 * sizeof(int); + +SEC("?tc") +int kfunc_call_test_get_mem_fail_not_const(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + p = bpf_kfunc_call_test_get_rdonly_mem(pt, not_const_size); /* non const size, -EINVAL */ + if (p) + ret = p[0]; + else + ret = -1; + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +SEC("?tc") +int kfunc_call_test_mem_acquire_fail(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + /* we are failing on this one, because we are not acquiring a PTR_TO_BTF_ID (a struct ptr) */ + p = bpf_kfunc_call_test_acq_rdonly_mem(pt, 2 * sizeof(int)); + if (p) + ret = p[0]; + else + ret = -1; + + bpf_kfunc_call_int_mem_release(p); + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c index 5aecbb9fdc68def55859da680afd8e6db8755cc6..f636e50be2599d52326844d21ff51064db9394f5 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c @@ -14,6 +14,8 @@ extern void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym; extern void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym; extern void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym; extern void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym; +extern int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym; +extern int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym; SEC("tc") int kfunc_call_test2(struct __sk_buff *skb) @@ -92,4 +94,73 @@ int kfunc_call_test_pass(struct __sk_buff *skb) return 0; } +struct syscall_test_args { + __u8 data[16]; + size_t size; +}; + +SEC("syscall") +int kfunc_syscall_test(struct syscall_test_args *args) +{ + const long size = args->size; + + if (size > sizeof(args->data)) + return -7; /* -E2BIG */ + + bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(args->data)); + bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(*args)); + bpf_kfunc_call_test_mem_len_pass1(&args->data, size); + + return 0; +} + +SEC("syscall") +int kfunc_syscall_test_null(struct syscall_test_args *args) +{ + /* Must be called with args as a NULL pointer + * we do not check for it to have the verifier consider that + * the pointer might not be null, and so we can load it. + * + * So the following can not be added: + * + * if (args) + * return -22; + */ + + bpf_kfunc_call_test_mem_len_pass1(args, 0); + + return 0; +} + +SEC("tc") +int kfunc_call_test_get_mem(struct __sk_buff *skb) +{ + struct prog_test_ref_kfunc *pt; + unsigned long s = 0; + int *p = NULL; + int ret = 0; + + pt = bpf_kfunc_call_test_acquire(&s); + if (pt) { + p = bpf_kfunc_call_test_get_rdwr_mem(pt, 2 * sizeof(int)); + if (p) { + p[0] = 42; + ret = p[1]; /* 108 */ + } else { + ret = -1; + } + + if (ret >= 0) { + p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int)); + if (p) + ret = p[0]; /* 42 */ + else + ret = -1; + } + + bpf_kfunc_call_test_release(pt); + } + return ret; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi.c b/tools/testing/selftests/bpf/progs/kprobe_multi.c index 08f95a8155d1bdeea9ce2fef2c90ca24af19954c..98c3399e15c03c1b83d09f177a9c8fa8284f2ab2 100644 --- a/tools/testing/selftests/bpf/progs/kprobe_multi.c +++ b/tools/testing/selftests/bpf/progs/kprobe_multi.c @@ -36,15 +36,13 @@ __u64 kretprobe_test6_result = 0; __u64 kretprobe_test7_result = 0; __u64 kretprobe_test8_result = 0; -extern bool CONFIG_X86_KERNEL_IBT __kconfig __weak; - static void kprobe_multi_check(void *ctx, bool is_return) { if (bpf_get_current_pid_tgid() >> 32 != pid) return; __u64 cookie = test_cookie ? bpf_get_attach_cookie(ctx) : 0; - __u64 addr = bpf_get_func_ip(ctx) - (CONFIG_X86_KERNEL_IBT ? 4 : 0); + __u64 addr = bpf_get_func_ip(ctx); #define SET(__var, __addr, __cookie) ({ \ if (((const void *) addr == __addr) && \ diff --git a/tools/testing/selftests/bpf/progs/lsm.c b/tools/testing/selftests/bpf/progs/lsm.c index 33694ef8acfab206e0011c819db789c20dc67542..d8d8af623bc2ec5b0f5dd18850f22c0b64513a0e 100644 --- a/tools/testing/selftests/bpf/progs/lsm.c +++ b/tools/testing/selftests/bpf/progs/lsm.c @@ -4,6 +4,7 @@ * Copyright 2020 Google LLC. */ +#include "bpf_misc.h" #include "vmlinux.h" #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> @@ -160,7 +161,7 @@ int BPF_PROG(test_task_free, struct task_struct *task) int copy_test = 0; -SEC("fentry.s/__x64_sys_setdomainname") +SEC("fentry.s/" SYS_PREFIX "sys_setdomainname") int BPF_PROG(test_sys_setdomainname, struct pt_regs *regs) { void *ptr = (void *)PT_REGS_PARM1(regs); diff --git a/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c b/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c new file mode 100644 index 0000000000000000000000000000000000000000..a47bb0120719928c244dfefa1a8f1a42a02336df --- /dev/null +++ b/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022. Huawei Technologies Co., Ltd */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +extern bool CONFIG_PREEMPT __kconfig __weak; +extern const int bpf_task_storage_busy __ksym; + +char _license[] SEC("license") = "GPL"; + +int pid = 0; +int busy = 0; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, long); +} task SEC(".maps"); + +SEC("raw_tp/sys_enter") +int BPF_PROG(read_bpf_task_storage_busy) +{ + int *value; + int key; + + if (!CONFIG_PREEMPT) + return 0; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + value = bpf_this_cpu_ptr(&bpf_task_storage_busy); + if (value) + busy = *value; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c new file mode 100644 index 0000000000000000000000000000000000000000..9523333b89053016877354aa5026b244aa73cb8c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include "vmlinux.h" +#include "bpf_tracing_net.h" +#include <bpf/bpf_core_read.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +extern unsigned long CONFIG_HZ __kconfig; + +const volatile char veth[IFNAMSIZ]; +const volatile int veth_ifindex; + +int nr_listen; +int nr_passive; +int nr_active; +int nr_connect; +int nr_binddev; +int nr_socket_post_create; + +struct sockopt_test { + int opt; + int new; + int restore; + int expected; + int tcp_expected; + unsigned int flip:1; +}; + +static const char not_exist_cc[] = "not_exist"; +static const char cubic_cc[] = "cubic"; +static const char reno_cc[] = "reno"; + +static const struct sockopt_test sol_socket_tests[] = { + { .opt = SO_REUSEADDR, .flip = 1, }, + { .opt = SO_SNDBUF, .new = 8123, .expected = 8123 * 2, }, + { .opt = SO_RCVBUF, .new = 8123, .expected = 8123 * 2, }, + { .opt = SO_KEEPALIVE, .flip = 1, }, + { .opt = SO_PRIORITY, .new = 0xeb9f, .expected = 0xeb9f, }, + { .opt = SO_REUSEPORT, .flip = 1, }, + { .opt = SO_RCVLOWAT, .new = 8123, .expected = 8123, }, + { .opt = SO_MARK, .new = 0xeb9f, .expected = 0xeb9f, }, + { .opt = SO_MAX_PACING_RATE, .new = 0xeb9f, .expected = 0xeb9f, }, + { .opt = SO_TXREHASH, .flip = 1, }, + { .opt = 0, }, +}; + +static const struct sockopt_test sol_tcp_tests[] = { + { .opt = TCP_NODELAY, .flip = 1, }, + { .opt = TCP_KEEPIDLE, .new = 123, .expected = 123, .restore = 321, }, + { .opt = TCP_KEEPINTVL, .new = 123, .expected = 123, .restore = 321, }, + { .opt = TCP_KEEPCNT, .new = 123, .expected = 123, .restore = 124, }, + { .opt = TCP_SYNCNT, .new = 123, .expected = 123, .restore = 124, }, + { .opt = TCP_WINDOW_CLAMP, .new = 8123, .expected = 8123, .restore = 8124, }, + { .opt = TCP_CONGESTION, }, + { .opt = TCP_THIN_LINEAR_TIMEOUTS, .flip = 1, }, + { .opt = TCP_USER_TIMEOUT, .new = 123400, .expected = 123400, }, + { .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, }, + { .opt = 0, }, +}; + +static const struct sockopt_test sol_ip_tests[] = { + { .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, + { .opt = 0, }, +}; + +static const struct sockopt_test sol_ipv6_tests[] = { + { .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, }, + { .opt = IPV6_AUTOFLOWLABEL, .flip = 1, }, + { .opt = 0, }, +}; + +struct loop_ctx { + void *ctx; + struct sock *sk; +}; + +static int bpf_test_sockopt_flip(void *ctx, struct sock *sk, + const struct sockopt_test *t, + int level) +{ + int old, tmp, new, opt = t->opt; + + opt = t->opt; + + if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old))) + return 1; + /* kernel initialized txrehash to 255 */ + if (level == SOL_SOCKET && opt == SO_TXREHASH && old != 0 && old != 1) + old = 1; + + new = !old; + if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new))) + return 1; + if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) || + tmp != new) + return 1; + + if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old))) + return 1; + + return 0; +} + +static int bpf_test_sockopt_int(void *ctx, struct sock *sk, + const struct sockopt_test *t, + int level) +{ + int old, tmp, new, expected, opt; + + opt = t->opt; + new = t->new; + if (sk->sk_type == SOCK_STREAM && t->tcp_expected) + expected = t->tcp_expected; + else + expected = t->expected; + + if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old)) || + old == new) + return 1; + + if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new))) + return 1; + if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) || + tmp != expected) + return 1; + + if (t->restore) + old = t->restore; + if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old))) + return 1; + + return 0; +} + +static int bpf_test_socket_sockopt(__u32 i, struct loop_ctx *lc) +{ + const struct sockopt_test *t; + + if (i >= ARRAY_SIZE(sol_socket_tests)) + return 1; + + t = &sol_socket_tests[i]; + if (!t->opt) + return 1; + + if (t->flip) + return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, SOL_SOCKET); + + return bpf_test_sockopt_int(lc->ctx, lc->sk, t, SOL_SOCKET); +} + +static int bpf_test_ip_sockopt(__u32 i, struct loop_ctx *lc) +{ + const struct sockopt_test *t; + + if (i >= ARRAY_SIZE(sol_ip_tests)) + return 1; + + t = &sol_ip_tests[i]; + if (!t->opt) + return 1; + + if (t->flip) + return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IP); + + return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IP); +} + +static int bpf_test_ipv6_sockopt(__u32 i, struct loop_ctx *lc) +{ + const struct sockopt_test *t; + + if (i >= ARRAY_SIZE(sol_ipv6_tests)) + return 1; + + t = &sol_ipv6_tests[i]; + if (!t->opt) + return 1; + + if (t->flip) + return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IPV6); + + return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IPV6); +} + +static int bpf_test_tcp_sockopt(__u32 i, struct loop_ctx *lc) +{ + const struct sockopt_test *t; + struct sock *sk; + void *ctx; + + if (i >= ARRAY_SIZE(sol_tcp_tests)) + return 1; + + t = &sol_tcp_tests[i]; + if (!t->opt) + return 1; + + ctx = lc->ctx; + sk = lc->sk; + + if (t->opt == TCP_CONGESTION) { + char old_cc[16], tmp_cc[16]; + const char *new_cc; + int new_cc_len; + + if (!bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, + (void *)not_exist_cc, sizeof(not_exist_cc))) + return 1; + if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc))) + return 1; + if (!bpf_strncmp(old_cc, sizeof(old_cc), cubic_cc)) { + new_cc = reno_cc; + new_cc_len = sizeof(reno_cc); + } else { + new_cc = cubic_cc; + new_cc_len = sizeof(cubic_cc); + } + if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, (void *)new_cc, + new_cc_len)) + return 1; + if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, tmp_cc, sizeof(tmp_cc))) + return 1; + if (bpf_strncmp(tmp_cc, sizeof(tmp_cc), new_cc)) + return 1; + if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc))) + return 1; + return 0; + } + + if (t->flip) + return bpf_test_sockopt_flip(ctx, sk, t, IPPROTO_TCP); + + return bpf_test_sockopt_int(ctx, sk, t, IPPROTO_TCP); +} + +static int bpf_test_sockopt(void *ctx, struct sock *sk) +{ + struct loop_ctx lc = { .ctx = ctx, .sk = sk, }; + __u16 family, proto; + int n; + + family = sk->sk_family; + proto = sk->sk_protocol; + + n = bpf_loop(ARRAY_SIZE(sol_socket_tests), bpf_test_socket_sockopt, &lc, 0); + if (n != ARRAY_SIZE(sol_socket_tests)) + return -1; + + if (proto == IPPROTO_TCP) { + n = bpf_loop(ARRAY_SIZE(sol_tcp_tests), bpf_test_tcp_sockopt, &lc, 0); + if (n != ARRAY_SIZE(sol_tcp_tests)) + return -1; + } + + if (family == AF_INET) { + n = bpf_loop(ARRAY_SIZE(sol_ip_tests), bpf_test_ip_sockopt, &lc, 0); + if (n != ARRAY_SIZE(sol_ip_tests)) + return -1; + } else { + n = bpf_loop(ARRAY_SIZE(sol_ipv6_tests), bpf_test_ipv6_sockopt, &lc, 0); + if (n != ARRAY_SIZE(sol_ipv6_tests)) + return -1; + } + + return 0; +} + +static int binddev_test(void *ctx) +{ + const char empty_ifname[] = ""; + int ifindex, zero = 0; + + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE, + (void *)veth, sizeof(veth))) + return -1; + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + &ifindex, sizeof(int)) || + ifindex != veth_ifindex) + return -1; + + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE, + (void *)empty_ifname, sizeof(empty_ifname))) + return -1; + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + &ifindex, sizeof(int)) || + ifindex != 0) + return -1; + + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + (void *)&veth_ifindex, sizeof(int))) + return -1; + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + &ifindex, sizeof(int)) || + ifindex != veth_ifindex) + return -1; + + if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + &zero, sizeof(int))) + return -1; + if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX, + &ifindex, sizeof(int)) || + ifindex != 0) + return -1; + + return 0; +} + +static int test_tcp_maxseg(void *ctx, struct sock *sk) +{ + int val = 1314, tmp; + + if (sk->sk_state != TCP_ESTABLISHED) + return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG, + &val, sizeof(val)); + + if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG, &tmp, sizeof(tmp)) || + tmp > val) + return -1; + + return 0; +} + +static int test_tcp_saved_syn(void *ctx, struct sock *sk) +{ + __u8 saved_syn[20]; + int one = 1; + + if (sk->sk_state == TCP_LISTEN) + return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_SAVE_SYN, + &one, sizeof(one)); + + return bpf_getsockopt(ctx, IPPROTO_TCP, TCP_SAVED_SYN, + saved_syn, sizeof(saved_syn)); +} + +SEC("lsm_cgroup/socket_post_create") +int BPF_PROG(socket_post_create, struct socket *sock, int family, + int type, int protocol, int kern) +{ + struct sock *sk = sock->sk; + + if (!sk) + return 1; + + nr_socket_post_create += !bpf_test_sockopt(sk, sk); + nr_binddev += !binddev_test(sk); + + return 1; +} + +SEC("sockops") +int skops_sockopt(struct bpf_sock_ops *skops) +{ + struct bpf_sock *bpf_sk = skops->sk; + struct sock *sk; + + if (!bpf_sk) + return 1; + + sk = (struct sock *)bpf_skc_to_tcp_sock(bpf_sk); + if (!sk) + return 1; + + switch (skops->op) { + case BPF_SOCK_OPS_TCP_LISTEN_CB: + nr_listen += !(bpf_test_sockopt(skops, sk) || + test_tcp_maxseg(skops, sk) || + test_tcp_saved_syn(skops, sk)); + break; + case BPF_SOCK_OPS_TCP_CONNECT_CB: + nr_connect += !(bpf_test_sockopt(skops, sk) || + test_tcp_maxseg(skops, sk)); + break; + case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: + nr_active += !(bpf_test_sockopt(skops, sk) || + test_tcp_maxseg(skops, sk)); + break; + case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: + nr_passive += !(bpf_test_sockopt(skops, sk) || + test_tcp_maxseg(skops, sk) || + test_tcp_saved_syn(skops, sk)); + break; + } + + return 1; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c index b67e8022d500920071693d3b511a6f92f7d85b06..a017d6b2f1dd070cb5696d52af0cf30fb03333f9 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c +++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c @@ -19,7 +19,7 @@ struct { int count = 0; int noise = 0; -__always_inline int subprog_noise(void) +static __always_inline int subprog_noise(void) { __u32 key = 0; diff --git a/tools/testing/selftests/bpf/progs/test_autoattach.c b/tools/testing/selftests/bpf/progs/test_autoattach.c new file mode 100644 index 0000000000000000000000000000000000000000..11a44493ebce4c2101f725b3e96661a2e3934dcc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_autoattach.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Google */ + +#include "vmlinux.h" +#include <bpf/bpf_tracing.h> + +bool prog1_called = false; +bool prog2_called = false; + +SEC("raw_tp/sys_enter") +int prog1(const void *ctx) +{ + prog1_called = true; + return 0; +} + +SEC("raw_tp/sys_exit") +int prog2(const void *ctx) +{ + prog2_called = true; + return 0; +} + diff --git a/tools/testing/selftests/bpf/progs/test_bpf_cookie.c b/tools/testing/selftests/bpf/progs/test_bpf_cookie.c index 22d0ac8709b4ab57ba4cc9da74655beda6141b5e..5a3a80f751c42e30ee78bbe220772bbd4e4d4e52 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_cookie.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_cookie.c @@ -28,14 +28,14 @@ static void update(void *ctx, __u64 *res) *res |= bpf_get_attach_cookie(ctx); } -SEC("kprobe/sys_nanosleep") +SEC("kprobe") int handle_kprobe(struct pt_regs *ctx) { update(ctx, &kprobe_res); return 0; } -SEC("kretprobe/sys_nanosleep") +SEC("kretprobe") int handle_kretprobe(struct pt_regs *ctx) { update(ctx, &kretprobe_res); diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index 196cd8dfe42a7156ba0946f19bf0cde169a8981b..227e85e85ddaf245f1cfe56b404d362822ed2d32 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <vmlinux.h> #include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> #define EAFNOSUPPORT 97 #define EPROTO 71 @@ -23,6 +24,15 @@ int test_insert_entry = -EAFNOSUPPORT; int test_succ_lookup = -ENOENT; u32 test_delta_timeout = 0; u32 test_status = 0; +u32 test_insert_lookup_mark = 0; +int test_snat_addr = -EINVAL; +int test_dnat_addr = -EINVAL; +__be32 saddr = 0; +__be16 sport = 0; +__be32 daddr = 0; +__be16 dport = 0; +int test_exist_lookup = -ENOENT; +u32 test_exist_lookup_mark = 0; struct nf_conn; @@ -47,6 +57,8 @@ void bpf_ct_set_timeout(struct nf_conn *, u32) __ksym; int bpf_ct_change_timeout(struct nf_conn *, u32) __ksym; int bpf_ct_set_status(struct nf_conn *, u32) __ksym; int bpf_ct_change_status(struct nf_conn *, u32) __ksym; +int bpf_ct_set_nat_info(struct nf_conn *, union nf_inet_addr *, + int port, enum nf_nat_manip_type) __ksym; static __always_inline void nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, @@ -134,10 +146,21 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { + __u16 sport = bpf_get_prandom_u32(); + __u16 dport = bpf_get_prandom_u32(); + union nf_inet_addr saddr = {}; + union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; bpf_ct_set_timeout(ct, 10000); - bpf_ct_set_status(ct, IPS_CONFIRMED); + ct->mark = 77; + + /* snat */ + saddr.ip = bpf_get_prandom_u32(); + bpf_ct_set_nat_info(ct, &saddr, sport, NF_NAT_MANIP_SRC); + /* dnat */ + daddr.ip = bpf_get_prandom_u32(); + bpf_ct_set_nat_info(ct, &daddr, dport, NF_NAT_MANIP_DST); ct_ins = bpf_ct_insert_entry(ct); if (ct_ins) { @@ -146,12 +169,26 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct_lk = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct_lk) { + struct nf_conntrack_tuple *tuple; + + /* check snat and dnat addresses */ + tuple = &ct_lk->tuplehash[IP_CT_DIR_REPLY].tuple; + if (tuple->dst.u3.ip == saddr.ip && + tuple->dst.u.all == bpf_htons(sport)) + test_snat_addr = 0; + if (tuple->src.u3.ip == daddr.ip && + tuple->src.u.all == bpf_htons(dport)) + test_dnat_addr = 0; + /* update ct entry timeout */ bpf_ct_change_timeout(ct_lk, 10000); test_delta_timeout = ct_lk->timeout - bpf_jiffies64(); test_delta_timeout /= CONFIG_HZ; - test_status = IPS_SEEN_REPLY; - bpf_ct_change_status(ct_lk, IPS_SEEN_REPLY); + test_insert_lookup_mark = ct_lk->mark; + bpf_ct_change_status(ct_lk, + IPS_CONFIRMED | IPS_SEEN_REPLY); + test_status = ct_lk->status; + bpf_ct_release(ct_lk); test_succ_lookup = 0; } @@ -160,6 +197,23 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, } test_alloc_entry = 0; } + + bpf_tuple.ipv4.saddr = saddr; + bpf_tuple.ipv4.daddr = daddr; + bpf_tuple.ipv4.sport = sport; + bpf_tuple.ipv4.dport = dport; + ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, + sizeof(opts_def)); + if (ct) { + test_exist_lookup = 0; + if (ct->mark == 42) { + ct->mark++; + test_exist_lookup_mark = ct->mark; + } + bpf_ct_release(ct); + } else { + test_exist_lookup = opts_def.error; + } } SEC("xdp") diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c index bf79af15c808f6fcd10ec42d12d82810b4b0ef98..0e4759ab38ff1ac92eb51c505309634b48051c67 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c @@ -69,6 +69,20 @@ int lookup_insert(struct __sk_buff *ctx) return 0; } +SEC("?tc") +int write_not_allowlisted_field(struct __sk_buff *ctx) +{ + struct bpf_ct_opts___local opts = {}; + struct bpf_sock_tuple tup = {}; + struct nf_conn *ct; + + ct = bpf_skb_ct_lookup(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts)); + if (!ct) + return 0; + ct->status = 0xF00; + return 0; +} + SEC("?tc") int set_timeout_after_insert(struct __sk_buff *ctx) { diff --git a/tools/testing/selftests/bpf/progs/test_deny_namespace.c b/tools/testing/selftests/bpf/progs/test_deny_namespace.c new file mode 100644 index 0000000000000000000000000000000000000000..09ad5a4ebd1f1c59ba6541031ce12ef0cb1ee591 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_deny_namespace.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <errno.h> +#include <linux/capability.h> + +struct kernel_cap_struct { + __u32 cap[_LINUX_CAPABILITY_U32S_3]; +} __attribute__((preserve_access_index)); + +struct cred { + struct kernel_cap_struct cap_effective; +} __attribute__((preserve_access_index)); + +char _license[] SEC("license") = "GPL"; + +SEC("lsm.s/userns_create") +int BPF_PROG(test_userns_create, const struct cred *cred, int ret) +{ + struct kernel_cap_struct caps = cred->cap_effective; + int cap_index = CAP_TO_INDEX(CAP_SYS_ADMIN); + __u32 cap_mask = CAP_TO_MASK(CAP_SYS_ADMIN); + + if (ret) + return 0; + + ret = -EPERM; + if (caps.cap[cap_index] & cap_mask) + return 0; + + return -EPERM; +} diff --git a/tools/testing/selftests/bpf/progs/test_helper_restricted.c b/tools/testing/selftests/bpf/progs/test_helper_restricted.c index 20ef9d433b973801e77a31f8ed43aa925219d408..5715c569ec0395ef1cbeee6b06c1ad45f239539a 100644 --- a/tools/testing/selftests/bpf/progs/test_helper_restricted.c +++ b/tools/testing/selftests/bpf/progs/test_helper_restricted.c @@ -72,7 +72,7 @@ int tp_timer(void *ctx) return 0; } -SEC("?kprobe/sys_nanosleep") +SEC("?kprobe") int kprobe_timer(void *ctx) { timer_work(); @@ -104,7 +104,7 @@ int tp_spin_lock(void *ctx) return 0; } -SEC("?kprobe/sys_nanosleep") +SEC("?kprobe") int kprobe_spin_lock(void *ctx) { spin_lock_work(); diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c new file mode 100644 index 0000000000000000000000000000000000000000..ce39d096bba34c9dcfe6caec52c94324b25f6c35 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; +extern void bpf_key_put(struct bpf_key *key) __ksym; +extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, + struct bpf_dynptr *sig_ptr, + struct bpf_key *trusted_keyring) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); +} ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} array_map SEC(".maps"); + +int err, pid; + +char _license[] SEC("license") = "GPL"; + +SEC("?lsm.s/bpf") +int BPF_PROG(dynptr_type_not_supp, int cmd, union bpf_attr *attr, + unsigned int size) +{ + char write_data[64] = "hello there, world!!"; + struct bpf_dynptr ptr; + + bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr); + + return bpf_verify_pkcs7_signature(&ptr, &ptr, NULL); +} + +SEC("?lsm.s/bpf") +int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size) +{ + unsigned long val; + + return bpf_verify_pkcs7_signature((struct bpf_dynptr *)&val, + (struct bpf_dynptr *)&val, NULL); +} + +SEC("?lsm.s/bpf") +int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size) +{ + unsigned long val; + + return bpf_verify_pkcs7_signature((struct bpf_dynptr *)val, + (struct bpf_dynptr *)val, NULL); +} + +SEC("lsm.s/bpf") +int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size) +{ + struct bpf_key *trusted_keyring; + struct bpf_dynptr ptr; + __u32 *value; + int ret, zero = 0; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + value = bpf_map_lookup_elem(&array_map, &zero); + if (!value) + return 0; + + /* Pass invalid flags. */ + ret = bpf_dynptr_from_mem(value, sizeof(*value), ((__u64)~0ULL), &ptr); + if (ret != -EINVAL) + return 0; + + trusted_keyring = bpf_lookup_system_key(0); + if (!trusted_keyring) + return 0; + + err = bpf_verify_pkcs7_signature(&ptr, &ptr, trusted_keyring); + + bpf_key_put(trusted_keyring); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_lookup_key.c b/tools/testing/selftests/bpf/progs/test_lookup_key.c new file mode 100644 index 0000000000000000000000000000000000000000..c73776990ae30453401fcb3bf7606150af1fcf58 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_lookup_key.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +__u32 monitored_pid; +__u32 key_serial; +__u32 key_id; +__u64 flags; + +extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym; +extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; +extern void bpf_key_put(struct bpf_key *key) __ksym; + +SEC("lsm.s/bpf") +int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size) +{ + struct bpf_key *bkey; + __u32 pid; + + pid = bpf_get_current_pid_tgid() >> 32; + if (pid != monitored_pid) + return 0; + + if (key_serial) + bkey = bpf_lookup_user_key(key_serial, flags); + else + bkey = bpf_lookup_system_key(key_id); + + if (!bkey) + return -ENOENT; + + bpf_key_put(bkey); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_tc_dtime.c b/tools/testing/selftests/bpf/progs/test_tc_dtime.c index b596479a9ebebba35129bd764db30e6666d534c9..125beec31834b7f2e860897dfa8e74574e571638 100644 --- a/tools/testing/selftests/bpf/progs/test_tc_dtime.c +++ b/tools/testing/selftests/bpf/progs/test_tc_dtime.c @@ -15,7 +15,6 @@ #include <linux/udp.h> #include <bpf/bpf_helpers.h> #include <bpf/bpf_endian.h> -#include <sys/socket.h> /* veth_src --- veth_src_fwd --- veth_det_fwd --- veth_dst * | | diff --git a/tools/testing/selftests/bpf/progs/test_time_tai.c b/tools/testing/selftests/bpf/progs/test_time_tai.c new file mode 100644 index 0000000000000000000000000000000000000000..7ea0863f3ddb81ad486a6cd9c0b52bd92d7c667e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_time_tai.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022 Linutronix GmbH */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +SEC("tc") +int time_tai(struct __sk_buff *skb) +{ + __u64 ts1, ts2; + + /* Get TAI timestamps */ + ts1 = bpf_ktime_get_tai_ns(); + ts2 = bpf_ktime_get_tai_ns(); + + /* Save TAI timestamps (Note: skb->hwtstamp is read-only) */ + skb->tstamp = ts1; + skb->cb[0] = ts2 & 0xffffffff; + skb->cb[1] = ts2 >> 32; + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c index df0673c4ecbe263237e7e64a6ccb12770aff45d1..98af55f0bcd3d169f8a92afde7a059592499a02a 100644 --- a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c +++ b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c @@ -12,6 +12,7 @@ #include <linux/bpf.h> #include <linux/if_ether.h> #include <linux/if_packet.h> +#include <linux/if_tunnel.h> #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/icmp.h> @@ -386,7 +387,8 @@ int vxlan_get_tunnel_src(struct __sk_buff *skb) __u32 orig_daddr; __u32 index = 0; - ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); + ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), + BPF_F_TUNINFO_FLAGS); if (ret < 0) { log_err(ret); return TC_ACT_SHOT; @@ -398,10 +400,13 @@ int vxlan_get_tunnel_src(struct __sk_buff *skb) return TC_ACT_SHOT; } - if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF) { - bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x\n", + if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF || + !(key.tunnel_flags & TUNNEL_KEY) || + (key.tunnel_flags & TUNNEL_CSUM)) { + bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x flags 0x%x\n", key.tunnel_id, key.local_ipv4, - key.remote_ipv4, md.gbp); + key.remote_ipv4, md.gbp, + bpf_ntohs(key.tunnel_flags)); log_err(ret); return TC_ACT_SHOT; } @@ -541,16 +546,19 @@ int ip6vxlan_get_tunnel_src(struct __sk_buff *skb) } ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), - BPF_F_TUNINFO_IPV6); + BPF_F_TUNINFO_IPV6 | BPF_F_TUNINFO_FLAGS); if (ret < 0) { log_err(ret); return TC_ACT_SHOT; } - if (bpf_ntohl(key.local_ipv6[3]) != *local_ip) { - bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x\n", + if (bpf_ntohl(key.local_ipv6[3]) != *local_ip || + !(key.tunnel_flags & TUNNEL_KEY) || + !(key.tunnel_flags & TUNNEL_CSUM)) { + bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x flags 0x%x\n", key.tunnel_id, bpf_ntohl(key.local_ipv6[3]), - bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label); + bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label, + bpf_ntohs(key.tunnel_flags)); bpf_printk("local_ip 0x%x\n", *local_ip); log_err(ret); return TC_ACT_SHOT; diff --git a/tools/testing/selftests/bpf/progs/test_user_ringbuf.h b/tools/testing/selftests/bpf/progs/test_user_ringbuf.h new file mode 100644 index 0000000000000000000000000000000000000000..1643b4d59ba77a7a116518f530da7b4e7ba3581a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_user_ringbuf.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#ifndef _TEST_USER_RINGBUF_H +#define _TEST_USER_RINGBUF_H + +#define TEST_OP_64 4 +#define TEST_OP_32 2 + +enum test_msg_op { + TEST_MSG_OP_INC64, + TEST_MSG_OP_INC32, + TEST_MSG_OP_MUL64, + TEST_MSG_OP_MUL32, + + // Must come last. + TEST_MSG_OP_NUM_OPS, +}; + +struct test_msg { + enum test_msg_op msg_op; + union { + __s64 operand_64; + __s32 operand_32; + }; +}; + +struct sample { + int pid; + int seq; + long value; + char comm[16]; +}; + +#endif /* _TEST_USER_RINGBUF_H */ diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale1.c b/tools/testing/selftests/bpf/progs/test_verif_scale1.c index d38153dab3ddb3452f7b36d57b7d88dc18c6d742..ac6135d9374c46e248225045f91ebe34de5ef89f 100644 --- a/tools/testing/selftests/bpf/progs/test_verif_scale1.c +++ b/tools/testing/selftests/bpf/progs/test_verif_scale1.c @@ -5,7 +5,7 @@ #define ATTR __attribute__((noinline)) #include "test_jhash.h" -SEC("scale90_noinline") +SEC("tc") int balancer_ingress(struct __sk_buff *ctx) { void *data_end = (void *)(long)ctx->data_end; diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale3.c b/tools/testing/selftests/bpf/progs/test_verif_scale3.c index 9beb5bf80373c64e83307cd6547f00e567cdfb50..ca33a9b711c4b1d95145f4aa8a3fa2df22ad54ca 100644 --- a/tools/testing/selftests/bpf/progs/test_verif_scale3.c +++ b/tools/testing/selftests/bpf/progs/test_verif_scale3.c @@ -5,7 +5,7 @@ #define ATTR __attribute__((noinline)) #include "test_jhash.h" -SEC("scale90_noinline32") +SEC("tc") int balancer_ingress(struct __sk_buff *ctx) { void *data_end = (void *)(long)ctx->data_end; diff --git a/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c new file mode 100644 index 0000000000000000000000000000000000000000..ce419304ff1f5e09a398041b804f39d3c8d5401d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH + * + * Author: Roberto Sassu <roberto.sassu@huawei.com> + */ + +#include "vmlinux.h" +#include <errno.h> +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define MAX_DATA_SIZE (1024 * 1024) +#define MAX_SIG_SIZE 1024 + +extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym; +extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; +extern void bpf_key_put(struct bpf_key *key) __ksym; +extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr, + struct bpf_dynptr *sig_ptr, + struct bpf_key *trusted_keyring) __ksym; + +__u32 monitored_pid; +__u32 user_keyring_serial; +__u64 system_keyring_id; + +struct data { + __u8 data[MAX_DATA_SIZE]; + __u32 data_len; + __u8 sig[MAX_SIG_SIZE]; + __u32 sig_len; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct data); +} data_input SEC(".maps"); + +char _license[] SEC("license") = "GPL"; + +SEC("lsm.s/bpf") +int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size) +{ + struct bpf_dynptr data_ptr, sig_ptr; + struct data *data_val; + struct bpf_key *trusted_keyring; + __u32 pid; + __u64 value; + int ret, zero = 0; + + pid = bpf_get_current_pid_tgid() >> 32; + if (pid != monitored_pid) + return 0; + + data_val = bpf_map_lookup_elem(&data_input, &zero); + if (!data_val) + return 0; + + bpf_probe_read(&value, sizeof(value), &attr->value); + + bpf_copy_from_user(data_val, sizeof(struct data), + (void *)(unsigned long)value); + + if (data_val->data_len > sizeof(data_val->data)) + return -EINVAL; + + bpf_dynptr_from_mem(data_val->data, data_val->data_len, 0, &data_ptr); + + if (data_val->sig_len > sizeof(data_val->sig)) + return -EINVAL; + + bpf_dynptr_from_mem(data_val->sig, data_val->sig_len, 0, &sig_ptr); + + if (user_keyring_serial) + trusted_keyring = bpf_lookup_user_key(user_keyring_serial, 0); + else + trusted_keyring = bpf_lookup_system_key(system_keyring_id); + + if (!trusted_keyring) + return -ENOENT; + + ret = bpf_verify_pkcs7_signature(&data_ptr, &sig_ptr, trusted_keyring); + + bpf_key_put(trusted_keyring); + + return ret; +} diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c index 5f5309791649137acf76b68d7785cfa0d093bd60..acda5c9cea9336e27848644b0f0b005b5647d006 100644 --- a/tools/testing/selftests/bpf/progs/timer.c +++ b/tools/testing/selftests/bpf/progs/timer.c @@ -120,7 +120,7 @@ static int timer_cb1(void *map, int *key, struct bpf_timer *timer) } SEC("fentry/bpf_fentry_test1") -int BPF_PROG(test1, int a) +int BPF_PROG2(test1, int, a) { struct bpf_timer *arr_timer, *lru_timer; struct elem init = {}; @@ -208,17 +208,6 @@ static int timer_cb2(void *map, int *key, struct hmap_elem *val) */ bpf_map_delete_elem(map, key); - /* in non-preallocated hashmap both 'key' and 'val' are RCU - * protected and still valid though this element was deleted - * from the map. Arm this timer for ~35 seconds. When callback - * finishes the call_rcu will invoke: - * htab_elem_free_rcu - * check_and_free_timer - * bpf_timer_cancel_and_free - * to cancel this 35 second sleep and delete the timer for real. - */ - if (bpf_timer_start(&val->timer, 1ull << 35, 0) != 0) - err |= 256; ok |= 4; } return 0; @@ -247,7 +236,7 @@ int bpf_timer_test(void) } SEC("fentry/bpf_fentry_test2") -int BPF_PROG(test2, int a, int b) +int BPF_PROG2(test2, int, a, int, b) { struct hmap_elem init = {}, *val; int key = HTAB, key_malloc = HTAB_MALLOC; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct.c b/tools/testing/selftests/bpf/progs/tracing_struct.c new file mode 100644 index 0000000000000000000000000000000000000000..e718f0ebee7dddc976d0534040ef5a97ebfc06b8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include <vmlinux.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_helpers.h> + +struct bpf_testmod_struct_arg_1 { + int a; +}; +struct bpf_testmod_struct_arg_2 { + long a; + long b; +}; + +long t1_a_a, t1_a_b, t1_b, t1_c, t1_ret, t1_nregs; +__u64 t1_reg0, t1_reg1, t1_reg2, t1_reg3; +long t2_a, t2_b_a, t2_b_b, t2_c, t2_ret; +long t3_a, t3_b, t3_c_a, t3_c_b, t3_ret; +long t4_a_a, t4_b, t4_c, t4_d, t4_e_a, t4_e_b, t4_ret; +long t5_ret; + +SEC("fentry/bpf_testmod_test_struct_arg_1") +int BPF_PROG2(test_struct_arg_1, struct bpf_testmod_struct_arg_2, a, int, b, int, c) +{ + t1_a_a = a.a; + t1_a_b = a.b; + t1_b = b; + t1_c = c; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_1") +int BPF_PROG2(test_struct_arg_2, struct bpf_testmod_struct_arg_2, a, int, b, int, c, int, ret) +{ + t1_nregs = bpf_get_func_arg_cnt(ctx); + /* a.a */ + bpf_get_func_arg(ctx, 0, &t1_reg0); + /* a.b */ + bpf_get_func_arg(ctx, 1, &t1_reg1); + /* b */ + bpf_get_func_arg(ctx, 2, &t1_reg2); + t1_reg2 = (int)t1_reg2; + /* c */ + bpf_get_func_arg(ctx, 3, &t1_reg3); + t1_reg3 = (int)t1_reg3; + + t1_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_2") +int BPF_PROG2(test_struct_arg_3, int, a, struct bpf_testmod_struct_arg_2, b, int, c) +{ + t2_a = a; + t2_b_a = b.a; + t2_b_b = b.b; + t2_c = c; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_2") +int BPF_PROG2(test_struct_arg_4, int, a, struct bpf_testmod_struct_arg_2, b, int, c, int, ret) +{ + t2_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_3") +int BPF_PROG2(test_struct_arg_5, int, a, int, b, struct bpf_testmod_struct_arg_2, c) +{ + t3_a = a; + t3_b = b; + t3_c_a = c.a; + t3_c_b = c.b; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_3") +int BPF_PROG2(test_struct_arg_6, int, a, int, b, struct bpf_testmod_struct_arg_2, c, int, ret) +{ + t3_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_4") +int BPF_PROG2(test_struct_arg_7, struct bpf_testmod_struct_arg_1, a, int, b, + int, c, int, d, struct bpf_testmod_struct_arg_2, e) +{ + t4_a_a = a.a; + t4_b = b; + t4_c = c; + t4_d = d; + t4_e_a = e.a; + t4_e_b = e.b; + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_4") +int BPF_PROG2(test_struct_arg_8, struct bpf_testmod_struct_arg_1, a, int, b, + int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret) +{ + t4_ret = ret; + return 0; +} + +SEC("fentry/bpf_testmod_test_struct_arg_5") +int BPF_PROG2(test_struct_arg_9) +{ + return 0; +} + +SEC("fexit/bpf_testmod_test_struct_arg_5") +int BPF_PROG2(test_struct_arg_10, int, ret) +{ + t5_ret = ret; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c new file mode 100644 index 0000000000000000000000000000000000000000..82aba4529aa977adb025c16fb20d39e27b0893e0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +struct sample { + int pid; + int seq; + long value; + char comm[16]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_USER_RINGBUF); +} user_ringbuf SEC(".maps"); + +static long +bad_access1(struct bpf_dynptr *dynptr, void *context) +{ + const struct sample *sample; + + sample = bpf_dynptr_data(dynptr - 1, 0, sizeof(*sample)); + bpf_printk("Was able to pass bad pointer %lx\n", (__u64)dynptr - 1); + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to read before the pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_bad_access1(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, bad_access1, NULL, 0); + + return 0; +} + +static long +bad_access2(struct bpf_dynptr *dynptr, void *context) +{ + const struct sample *sample; + + sample = bpf_dynptr_data(dynptr + 1, 0, sizeof(*sample)); + bpf_printk("Was able to pass bad pointer %lx\n", (__u64)dynptr + 1); + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to read past the end of the pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_bad_access2(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, bad_access2, NULL, 0); + + return 0; +} + +static long +write_forbidden(struct bpf_dynptr *dynptr, void *context) +{ + *((long *)dynptr) = 0; + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to write to that pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_write_forbidden(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, write_forbidden, NULL, 0); + + return 0; +} + +static long +null_context_write(struct bpf_dynptr *dynptr, void *context) +{ + *((__u64 *)context) = 0; + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to write to that pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_null_context_write(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, null_context_write, NULL, 0); + + return 0; +} + +static long +null_context_read(struct bpf_dynptr *dynptr, void *context) +{ + __u64 id = *((__u64 *)context); + + bpf_printk("Read id %lu\n", id); + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to write to that pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_null_context_read(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, null_context_read, NULL, 0); + + return 0; +} + +static long +try_discard_dynptr(struct bpf_dynptr *dynptr, void *context) +{ + bpf_ringbuf_discard_dynptr(dynptr, 0); + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to read past the end of the pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_discard_dynptr(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, try_discard_dynptr, NULL, 0); + + return 0; +} + +static long +try_submit_dynptr(struct bpf_dynptr *dynptr, void *context) +{ + bpf_ringbuf_submit_dynptr(dynptr, 0); + + return 0; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to read past the end of the pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_submit_dynptr(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, try_submit_dynptr, NULL, 0); + + return 0; +} + +static long +invalid_drain_callback_return(struct bpf_dynptr *dynptr, void *context) +{ + return 2; +} + +/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should + * not be able to write to that pointer. + */ +SEC("?raw_tp/sys_nanosleep") +int user_ringbuf_callback_invalid_return(void *ctx) +{ + bpf_user_ringbuf_drain(&user_ringbuf, invalid_drain_callback_return, NULL, 0); + + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_success.c b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c new file mode 100644 index 0000000000000000000000000000000000000000..099c23d9aa21b527412c6d36d1ea6d2cf2e1a27d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ + +#include <linux/bpf.h> +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" +#include "test_user_ringbuf.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_USER_RINGBUF); +} user_ringbuf SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); +} kernel_ringbuf SEC(".maps"); + +/* inputs */ +int pid, err, val; + +int read = 0; + +/* Counter used for end-to-end protocol test */ +__u64 kern_mutated = 0; +__u64 user_mutated = 0; +__u64 expected_user_mutated = 0; + +static int +is_test_process(void) +{ + int cur_pid = bpf_get_current_pid_tgid() >> 32; + + return cur_pid == pid; +} + +static long +record_sample(struct bpf_dynptr *dynptr, void *context) +{ + const struct sample *sample = NULL; + struct sample stack_sample; + int status; + static int num_calls; + + if (num_calls++ % 2 == 0) { + status = bpf_dynptr_read(&stack_sample, sizeof(stack_sample), dynptr, 0, 0); + if (status) { + bpf_printk("bpf_dynptr_read() failed: %d\n", status); + err = 1; + return 0; + } + } else { + sample = bpf_dynptr_data(dynptr, 0, sizeof(*sample)); + if (!sample) { + bpf_printk("Unexpectedly failed to get sample\n"); + err = 2; + return 0; + } + stack_sample = *sample; + } + + __sync_fetch_and_add(&read, 1); + return 0; +} + +static void +handle_sample_msg(const struct test_msg *msg) +{ + switch (msg->msg_op) { + case TEST_MSG_OP_INC64: + kern_mutated += msg->operand_64; + break; + case TEST_MSG_OP_INC32: + kern_mutated += msg->operand_32; + break; + case TEST_MSG_OP_MUL64: + kern_mutated *= msg->operand_64; + break; + case TEST_MSG_OP_MUL32: + kern_mutated *= msg->operand_32; + break; + default: + bpf_printk("Unrecognized op %d\n", msg->msg_op); + err = 2; + } +} + +static long +read_protocol_msg(struct bpf_dynptr *dynptr, void *context) +{ + const struct test_msg *msg = NULL; + + msg = bpf_dynptr_data(dynptr, 0, sizeof(*msg)); + if (!msg) { + err = 1; + bpf_printk("Unexpectedly failed to get msg\n"); + return 0; + } + + handle_sample_msg(msg); + + return 0; +} + +static int publish_next_kern_msg(__u32 index, void *context) +{ + struct test_msg *msg = NULL; + int operand_64 = TEST_OP_64; + int operand_32 = TEST_OP_32; + + msg = bpf_ringbuf_reserve(&kernel_ringbuf, sizeof(*msg), 0); + if (!msg) { + err = 4; + return 1; + } + + switch (index % TEST_MSG_OP_NUM_OPS) { + case TEST_MSG_OP_INC64: + msg->operand_64 = operand_64; + msg->msg_op = TEST_MSG_OP_INC64; + expected_user_mutated += operand_64; + break; + case TEST_MSG_OP_INC32: + msg->operand_32 = operand_32; + msg->msg_op = TEST_MSG_OP_INC32; + expected_user_mutated += operand_32; + break; + case TEST_MSG_OP_MUL64: + msg->operand_64 = operand_64; + msg->msg_op = TEST_MSG_OP_MUL64; + expected_user_mutated *= operand_64; + break; + case TEST_MSG_OP_MUL32: + msg->operand_32 = operand_32; + msg->msg_op = TEST_MSG_OP_MUL32; + expected_user_mutated *= operand_32; + break; + default: + bpf_ringbuf_discard(msg, 0); + err = 5; + return 1; + } + + bpf_ringbuf_submit(msg, 0); + + return 0; +} + +static void +publish_kern_messages(void) +{ + if (expected_user_mutated != user_mutated) { + bpf_printk("%lu != %lu\n", expected_user_mutated, user_mutated); + err = 3; + return; + } + + bpf_loop(8, publish_next_kern_msg, NULL, 0); +} + +SEC("fentry/" SYS_PREFIX "sys_prctl") +int test_user_ringbuf_protocol(void *ctx) +{ + long status = 0; + struct sample *sample = NULL; + struct bpf_dynptr ptr; + + if (!is_test_process()) + return 0; + + status = bpf_user_ringbuf_drain(&user_ringbuf, read_protocol_msg, NULL, 0); + if (status < 0) { + bpf_printk("Drain returned: %ld\n", status); + err = 1; + return 0; + } + + publish_kern_messages(); + + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getpgid") +int test_user_ringbuf(void *ctx) +{ + int status = 0; + struct sample *sample = NULL; + struct bpf_dynptr ptr; + + if (!is_test_process()) + return 0; + + err = bpf_user_ringbuf_drain(&user_ringbuf, record_sample, NULL, 0); + + return 0; +} + +static long +do_nothing_cb(struct bpf_dynptr *dynptr, void *context) +{ + __sync_fetch_and_add(&read, 1); + return 0; +} + +SEC("fentry/" SYS_PREFIX "sys_getrlimit") +int test_user_ringbuf_epoll(void *ctx) +{ + long num_samples; + + if (!is_test_process()) + return 0; + + num_samples = bpf_user_ringbuf_drain(&user_ringbuf, do_nothing_cb, NULL, 0); + if (num_samples <= 0) + err = 1; + + return 0; +} diff --git a/tools/testing/selftests/bpf/task_local_storage_helpers.h b/tools/testing/selftests/bpf/task_local_storage_helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..711d5abb7d51ca01da2efdc6b43aa8d99519d52f --- /dev/null +++ b/tools/testing/selftests/bpf/task_local_storage_helpers.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __TASK_LOCAL_STORAGE_HELPER_H +#define __TASK_LOCAL_STORAGE_HELPER_H + +#include <unistd.h> +#include <sys/syscall.h> +#include <sys/types.h> + +#ifndef __NR_pidfd_open +#define __NR_pidfd_open 434 +#endif + +static inline int sys_pidfd_open(pid_t pid, unsigned int flags) +{ + return syscall(__NR_pidfd_open, pid, flags); +} + +#endif diff --git a/tools/testing/selftests/bpf/test_dev_cgroup.c b/tools/testing/selftests/bpf/test_dev_cgroup.c index 7886265846a079a26ff38f09e1b5e214c692710f..adeaf63cb6fa3905377bc9e7c87f85bb9c74aede 100644 --- a/tools/testing/selftests/bpf/test_dev_cgroup.c +++ b/tools/testing/selftests/bpf/test_dev_cgroup.c @@ -16,7 +16,7 @@ #include "cgroup_helpers.h" #include "testing_helpers.h" -#define DEV_CGROUP_PROG "./dev_cgroup.o" +#define DEV_CGROUP_PROG "./dev_cgroup.bpf.o" #define TEST_CGROUP "/test-bpf-based-device-cgroup/" diff --git a/tools/testing/selftests/bpf/test_flow_dissector.sh b/tools/testing/selftests/bpf/test_flow_dissector.sh index dbd91221727d8f46118650bdd6d2b28c9f5c0eef..5303ce0c977bd3acf9baced98bd4f922ae13b1c8 100755 --- a/tools/testing/selftests/bpf/test_flow_dissector.sh +++ b/tools/testing/selftests/bpf/test_flow_dissector.sh @@ -115,6 +115,14 @@ tc filter add dev lo parent ffff: protocol ip pref 1337 flower ip_proto \ # Send 10 IPv4/UDP packets from port 10. Filter should not drop any. ./test_flow_dissector -i 4 -f 10 +echo "Testing IPv4 from 127.0.0.127 (fallback to generic dissector)..." +# Send 10 IPv4/UDP packets from port 8. Filter should not drop any. +./test_flow_dissector -i 4 -S 127.0.0.127 -f 8 +# Send 10 IPv4/UDP packets from port 9. Filter should drop all. +./test_flow_dissector -i 4 -S 127.0.0.127 -f 9 -F +# Send 10 IPv4/UDP packets from port 10. Filter should not drop any. +./test_flow_dissector -i 4 -S 127.0.0.127 -f 10 + echo "Testing IPIP..." # Send 10 IPv4/IPv4/UDP packets from port 8. Filter should not drop any. ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \ diff --git a/tools/testing/selftests/bpf/test_kmod.sh b/tools/testing/selftests/bpf/test_kmod.sh index 4f6444bcd53f93bb904ef22bd53744d7d32b1e84..50dca53ac536a878cb57a29ac6eb0e9ec275af7e 100755 --- a/tools/testing/selftests/bpf/test_kmod.sh +++ b/tools/testing/selftests/bpf/test_kmod.sh @@ -1,6 +1,11 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 +# Usage: +# ./test_kmod.sh [module_param]... +# Ex.: ./test_kmod.sh test_range=1,3 +# All the parameters are passed to the kernel module. + # Kselftest framework requirement - SKIP code is 4. ksft_skip=4 @@ -24,17 +29,18 @@ test_run() sysctl -w net.core.bpf_jit_harden=$2 2>&1 > /dev/null echo "[ JIT enabled:$1 hardened:$2 ]" + shift 2 dmesg -C if [ -f ${OUTPUT}/lib/test_bpf.ko ]; then - insmod ${OUTPUT}/lib/test_bpf.ko 2> /dev/null + insmod ${OUTPUT}/lib/test_bpf.ko "$@" 2> /dev/null if [ $? -ne 0 ]; then rc=1 fi else # Use modprobe dry run to check for missing test_bpf module - if ! /sbin/modprobe -q -n test_bpf; then + if ! /sbin/modprobe -q -n test_bpf "$@"; then echo "test_bpf: [SKIP]" - elif /sbin/modprobe -q test_bpf; then + elif /sbin/modprobe -q test_bpf "$@"; then echo "test_bpf: ok" else echo "test_bpf: [FAIL]" @@ -59,9 +65,9 @@ test_restore() rc=0 test_save -test_run 0 0 -test_run 1 0 -test_run 1 1 -test_run 1 2 +test_run 0 0 "$@" +test_run 1 0 "$@" +test_run 1 1 "$@" +test_run 1 2 "$@" test_restore exit $rc diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c index 2893e9f2f1e0a240a00aae65842c86164523743c..4694422aa76c36faa4afa8832e56040547dc7547 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c @@ -59,7 +59,7 @@ int main(int argc, char **argv) return 2; } - ret = bpf_prog_test_load("test_lirc_mode2_kern.o", + ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o", BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); if (ret) { printf("Failed to load bpf program\n"); diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index cbebfaa7c1e8291603343853d2eba3ed767d3d84..b73152822aa2822909130cdc6d59ffbb1adc41d6 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -30,7 +30,7 @@ #define ENOTSUPP 524 #endif -static int skips; +int skips; static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) }; @@ -264,10 +264,11 @@ static void test_hashmap_percpu(unsigned int task, void *data) close(fd); } +#define VALUE_SIZE 3 static int helper_fill_hashmap(int max_entries) { int i, fd, ret; - long long key, value; + long long key, value[VALUE_SIZE] = {}; fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value), max_entries, &map_opts); @@ -276,8 +277,8 @@ static int helper_fill_hashmap(int max_entries) "err: %s, flags: 0x%x\n", strerror(errno), map_opts.map_flags); for (i = 0; i < max_entries; i++) { - key = i; value = key; - ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST); + key = i; value[0] = key; + ret = bpf_map_update_elem(fd, &key, value, BPF_NOEXIST); CHECK(ret != 0, "can't update hashmap", "err: %s\n", strerror(ret)); @@ -288,8 +289,8 @@ static int helper_fill_hashmap(int max_entries) static void test_hashmap_walk(unsigned int task, void *data) { - int fd, i, max_entries = 1000; - long long key, value, next_key; + int fd, i, max_entries = 10000; + long long key, value[VALUE_SIZE], next_key; bool next_key_valid = true; fd = helper_fill_hashmap(max_entries); @@ -297,7 +298,7 @@ static void test_hashmap_walk(unsigned int task, void *data) for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, &next_key) == 0; i++) { key = next_key; - assert(bpf_map_lookup_elem(fd, &key, &value) == 0); + assert(bpf_map_lookup_elem(fd, &key, value) == 0); } assert(i == max_entries); @@ -305,9 +306,9 @@ static void test_hashmap_walk(unsigned int task, void *data) assert(bpf_map_get_next_key(fd, NULL, &key) == 0); for (i = 0; next_key_valid; i++) { next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0; - assert(bpf_map_lookup_elem(fd, &key, &value) == 0); - value++; - assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0); + assert(bpf_map_lookup_elem(fd, &key, value) == 0); + value[0]++; + assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0); key = next_key; } @@ -316,8 +317,8 @@ static void test_hashmap_walk(unsigned int task, void *data) for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, &next_key) == 0; i++) { key = next_key; - assert(bpf_map_lookup_elem(fd, &key, &value) == 0); - assert(value - 1 == key); + assert(bpf_map_lookup_elem(fd, &key, value) == 0); + assert(value[0] - 1 == key); } assert(i == max_entries); @@ -651,20 +652,20 @@ static void test_stackmap(unsigned int task, void *data) #include <arpa/inet.h> #include <sys/select.h> #include <linux/err.h> -#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o" -#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o" -#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o" +#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.bpf.o" +#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.bpf.o" +#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.bpf.o" static void test_sockmap(unsigned int tasks, void *data) { struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break; int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break; + struct bpf_object *parse_obj, *verdict_obj, *msg_obj; int ports[] = {50200, 50201, 50202, 50204}; int err, i, fd, udp, sfd[6] = {0xdeadbeef}; u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0}; int parse_prog, verdict_prog, msg_prog; struct sockaddr_in addr; int one = 1, s, sc, rc; - struct bpf_object *obj; struct timeval to; __u32 key, value; pid_t pid[tasks]; @@ -760,6 +761,7 @@ static void test_sockmap(unsigned int tasks, void *data) i, udp); goto out_sockmap; } + close(udp); /* Test update without programs */ for (i = 0; i < 6; i++) { @@ -822,27 +824,27 @@ static void test_sockmap(unsigned int tasks, void *data) /* Load SK_SKB program and Attach */ err = bpf_prog_test_load(SOCKMAP_PARSE_PROG, - BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog); + BPF_PROG_TYPE_SK_SKB, &parse_obj, &parse_prog); if (err) { printf("Failed to load SK_SKB parse prog\n"); goto out_sockmap; } err = bpf_prog_test_load(SOCKMAP_TCP_MSG_PROG, - BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog); + BPF_PROG_TYPE_SK_MSG, &msg_obj, &msg_prog); if (err) { printf("Failed to load SK_SKB msg prog\n"); goto out_sockmap; } err = bpf_prog_test_load(SOCKMAP_VERDICT_PROG, - BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog); + BPF_PROG_TYPE_SK_SKB, &verdict_obj, &verdict_prog); if (err) { printf("Failed to load SK_SKB verdict prog\n"); goto out_sockmap; } - bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx"); + bpf_map_rx = bpf_object__find_map_by_name(verdict_obj, "sock_map_rx"); if (!bpf_map_rx) { printf("Failed to load map rx from verdict prog\n"); goto out_sockmap; @@ -854,7 +856,7 @@ static void test_sockmap(unsigned int tasks, void *data) goto out_sockmap; } - bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx"); + bpf_map_tx = bpf_object__find_map_by_name(verdict_obj, "sock_map_tx"); if (!bpf_map_tx) { printf("Failed to load map tx from verdict prog\n"); goto out_sockmap; @@ -866,7 +868,7 @@ static void test_sockmap(unsigned int tasks, void *data) goto out_sockmap; } - bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg"); + bpf_map_msg = bpf_object__find_map_by_name(verdict_obj, "sock_map_msg"); if (!bpf_map_msg) { printf("Failed to load map msg from msg_verdict prog\n"); goto out_sockmap; @@ -878,7 +880,7 @@ static void test_sockmap(unsigned int tasks, void *data) goto out_sockmap; } - bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break"); + bpf_map_break = bpf_object__find_map_by_name(verdict_obj, "sock_map_break"); if (!bpf_map_break) { printf("Failed to load map tx from verdict prog\n"); goto out_sockmap; @@ -1124,7 +1126,9 @@ static void test_sockmap(unsigned int tasks, void *data) } close(fd); close(map_fd_rx); - bpf_object__close(obj); + bpf_object__close(parse_obj); + bpf_object__close(msg_obj); + bpf_object__close(verdict_obj); return; out: for (i = 0; i < 6; i++) @@ -1143,8 +1147,8 @@ out_sockmap: exit(1); } -#define MAPINMAP_PROG "./test_map_in_map.o" -#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.o" +#define MAPINMAP_PROG "./test_map_in_map.bpf.o" +#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.bpf.o" static void test_map_in_map(void) { struct bpf_object *obj; @@ -1282,8 +1286,11 @@ static void test_map_in_map(void) printf("Inner map mim.inner was not destroyed\n"); goto out_map_in_map; } + + close(fd); } + bpf_object__close(obj); return; out_map_in_map: @@ -1371,16 +1378,16 @@ static void __run_parallel(unsigned int tasks, static void test_map_stress(void) { + run_parallel(100, test_hashmap_walk, NULL); run_parallel(100, test_hashmap, NULL); run_parallel(100, test_hashmap_percpu, NULL); run_parallel(100, test_hashmap_sizes, NULL); - run_parallel(100, test_hashmap_walk, NULL); run_parallel(100, test_arraymap, NULL); run_parallel(100, test_arraymap_percpu, NULL); } -#define TASKS 1024 +#define TASKS 100 #define DO_UPDATE 1 #define DO_DELETE 0 @@ -1432,6 +1439,8 @@ static void test_update_delete(unsigned int fn, void *data) int fd = ((int *)data)[0]; int i, key, value, err; + if (fn & 1) + test_hashmap_walk(fn, NULL); for (i = fn; i < MAP_SIZE; i += TASKS) { key = value = i; @@ -1455,7 +1464,7 @@ static void test_update_delete(unsigned int fn, void *data) static void test_map_parallel(void) { - int i, fd, key = 0, value = 0; + int i, fd, key = 0, value = 0, j = 0; int data[2]; fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value), @@ -1466,6 +1475,7 @@ static void test_map_parallel(void) exit(1); } +again: /* Use the same fd in children to add elements to this map: * child_0 adds key=0, key=1024, key=2048, ... * child_1 adds key=1, key=1025, key=2049, ... @@ -1502,6 +1512,12 @@ static void test_map_parallel(void) key = -1; assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT); assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT); + + key = 0; + bpf_map_delete_elem(fd, &key); + if (j++ < 5) + goto again; + close(fd); } static void test_map_rdonly(void) diff --git a/tools/testing/selftests/bpf/test_maps.h b/tools/testing/selftests/bpf/test_maps.h index 77d8587ac4edb5dd1640e74771efd0f509e1cb46..f6fbca761732f9db137964fb5cefdd8308fb1bee 100644 --- a/tools/testing/selftests/bpf/test_maps.h +++ b/tools/testing/selftests/bpf/test_maps.h @@ -14,4 +14,6 @@ } \ }) +extern int skips; + #endif diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/bpf/test_offload.py index 6cd6ef9fc20b63dd070babd19361f175a2ea6377..7fc15e0d24a9f0979d8fe7b3743a904e312a8790 100755 --- a/tools/testing/selftests/bpf/test_offload.py +++ b/tools/testing/selftests/bpf/test_offload.py @@ -782,7 +782,7 @@ if out.find("/sys/kernel/debug type debugfs") == -1: cmd("mount -t debugfs none /sys/kernel/debug") # Check samples are compiled -samples = ["sample_ret0.o", "sample_map_ret0.o"] +samples = ["sample_ret0.bpf.o", "sample_map_ret0.bpf.o"] for s in samples: ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False) skip(ret != 0, "sample %s/%s not found, please compile it" % @@ -803,7 +803,7 @@ cmd("ip netns delete %s" % (ns)) netns = [] try: - obj = bpf_obj("sample_ret0.o") + obj = bpf_obj("sample_ret0.bpf.o") bytecode = bpf_bytecode("1,6 0 0 4294967295,") start_test("Test destruction of generic XDP...") @@ -1023,7 +1023,7 @@ try: sim.wait_for_flush() start_test("Test non-offload XDP attaching to HW...") - bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload") + bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/nooffload") nooffload = bpf_pinned("/sys/fs/bpf/nooffload") ret, _, err = sim.set_xdp(nooffload, "offload", fail=False, include_stderr=True) @@ -1032,7 +1032,7 @@ try: rm("/sys/fs/bpf/nooffload") start_test("Test offload XDP attaching to drv...") - bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload", + bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload", dev=sim['ifname']) offload = bpf_pinned("/sys/fs/bpf/offload") ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True) @@ -1043,7 +1043,7 @@ try: start_test("Test XDP load failure...") sim.dfs["dev/bpf_bind_verifier_accept"] = 0 - ret, _, err = bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload", + ret, _, err = bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload", dev=sim['ifname'], fail=False, include_stderr=True) fail(ret == 0, "verifier should fail on load") check_verifier_log(err, "[netdevsim] Hello from netdevsim!") @@ -1169,7 +1169,7 @@ try: simdev = NetdevSimDev() sim, = simdev.nsims - map_obj = bpf_obj("sample_map_ret0.o") + map_obj = bpf_obj("sample_map_ret0.bpf.o") start_test("Test loading program with maps...") sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON @@ -1307,10 +1307,10 @@ try: sims = (simA, simB1, simB2, simB3) simB = (simB1, simB2, simB3) - bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA", + bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA", dev=simA['ifname']) progA = bpf_pinned("/sys/fs/bpf/nsimA") - bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB", + bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB", dev=simB1['ifname']) progB = bpf_pinned("/sys/fs/bpf/nsimB") @@ -1344,14 +1344,14 @@ try: mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0] mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0] - ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_", + ret, _ = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_", dev=simB3['ifname'], maps=["idx 0 id %d" % (mapB)], fail=False) fail(ret != 0, "couldn't reuse a map on the same ASIC") rm("/sys/fs/bpf/nsimB_") - ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_", + ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA_", dev=simA['ifname'], maps=["idx 0 id %d" % (mapB)], fail=False, include_stderr=True) @@ -1359,7 +1359,7 @@ try: fail(err.count("offload device mismatch between prog and map") == 0, "error message missing for cross-ASIC map") - ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_", + ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_", dev=simB1['ifname'], maps=["idx 0 id %d" % (mapA)], fail=False, include_stderr=True) diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 3561c97701f24a194914c2ff8839b80219a238e6..0e9a47f978908d5fef7f79aa3f9ec5be53aa6b64 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -943,6 +943,23 @@ int trigger_module_test_write(int write_sz) return 0; } +int write_sysctl(const char *sysctl, const char *value) +{ + int fd, err, len; + + fd = open(sysctl, O_WRONLY); + if (!ASSERT_NEQ(fd, -1, "open sysctl")) + return -1; + + len = strlen(value); + err = write(fd, value, len); + close(fd); + if (!ASSERT_EQ(err, len, "write sysctl")) + return -1; + + return 0; +} + #define MAX_BACKTRACE_SZ 128 void crash_handler(int signum) { diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h index 5fe1365c2bb1ea5970ee3004fb59decc01dc545e..b090996daee5ca7071f7cd4466801d9e1d36e734 100644 --- a/tools/testing/selftests/bpf/test_progs.h +++ b/tools/testing/selftests/bpf/test_progs.h @@ -384,6 +384,7 @@ int extract_build_id(char *build_id, size_t size); int kern_sync_rcu(void); int trigger_module_test_read(int read_sz); int trigger_module_test_write(int write_sz); +int write_sysctl(const char *sysctl, const char *value); #ifdef __x86_64__ #define SYS_NANOSLEEP_KPROBE_NAME "__x64_sys_nanosleep" diff --git a/tools/testing/selftests/bpf/test_skb_cgroup_id.sh b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh index a9bc6f82abc163d47876d4af1f2dcbe540a6c814..515c2eafc97fc4e127520070633a17dfa445a3c2 100755 --- a/tools/testing/selftests/bpf/test_skb_cgroup_id.sh +++ b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh @@ -54,7 +54,7 @@ DIR=$(dirname $0) TEST_IF="test_cgid_1" TEST_IF_PEER="test_cgid_2" MAX_PING_TRIES=5 -BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.o" +BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.bpf.o" BPF_PROG_SECTION="cgroup_id_logger" BPF_PROG_ID=0 PROG="${DIR}/test_skb_cgroup_id_user" diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c index 458564fcfc8239bd78d90f3936f2a0776b7e246b..2c89674fc62c1581b95146ef1e9c9d4afa9a0262 100644 --- a/tools/testing/selftests/bpf/test_sock_addr.c +++ b/tools/testing/selftests/bpf/test_sock_addr.c @@ -26,14 +26,14 @@ #endif #define CG_PATH "/foo" -#define CONNECT4_PROG_PATH "./connect4_prog.o" -#define CONNECT6_PROG_PATH "./connect6_prog.o" -#define SENDMSG4_PROG_PATH "./sendmsg4_prog.o" -#define SENDMSG6_PROG_PATH "./sendmsg6_prog.o" -#define RECVMSG4_PROG_PATH "./recvmsg4_prog.o" -#define RECVMSG6_PROG_PATH "./recvmsg6_prog.o" -#define BIND4_PROG_PATH "./bind4_prog.o" -#define BIND6_PROG_PATH "./bind6_prog.o" +#define CONNECT4_PROG_PATH "./connect4_prog.bpf.o" +#define CONNECT6_PROG_PATH "./connect6_prog.bpf.o" +#define SENDMSG4_PROG_PATH "./sendmsg4_prog.bpf.o" +#define SENDMSG6_PROG_PATH "./sendmsg6_prog.bpf.o" +#define RECVMSG4_PROG_PATH "./recvmsg4_prog.bpf.o" +#define RECVMSG6_PROG_PATH "./recvmsg6_prog.bpf.o" +#define BIND4_PROG_PATH "./bind4_prog.bpf.o" +#define BIND6_PROG_PATH "./bind6_prog.bpf.o" #define SERV4_IP "192.168.1.254" #define SERV4_REWRITE_IP "127.0.0.1" diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index 0fbaccdc886126ec02ec4ba40d0728a0f1829ef8..e768181a1bd75a15cf72f82fe293a47b0cb24d35 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -52,8 +52,8 @@ static void running_handler(int a); #define S1_PORT 10000 #define S2_PORT 10001 -#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o" -#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o" +#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o" +#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o" #define CG_PATH "/sockmap" /* global sockets */ @@ -138,6 +138,7 @@ struct sockmap_options { bool data_test; bool drop_expected; bool check_recved_len; + bool tx_wait_mem; int iov_count; int iov_length; int rate; @@ -578,6 +579,10 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, sent = sendmsg(fd, &msg, flags); if (!drop && sent < 0) { + if (opt->tx_wait_mem && errno == EACCES) { + errno = 0; + goto out_errno; + } perror("sendmsg loop error"); goto out_errno; } else if (drop && sent >= 0) { @@ -644,6 +649,15 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt, goto out_errno; } + if (opt->tx_wait_mem) { + FD_ZERO(&w); + FD_SET(fd, &w); + slct = select(max_fd + 1, NULL, NULL, &w, &timeout); + errno = 0; + close(fd); + goto out_errno; + } + errno = 0; if (peek_flag) { flags |= MSG_PEEK; @@ -752,6 +766,22 @@ static int sendmsg_test(struct sockmap_options *opt) return err; } + if (opt->tx_wait_mem) { + struct timeval timeout; + int rxtx_buf_len = 1024; + + timeout.tv_sec = 3; + timeout.tv_usec = 0; + + err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval)); + err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int)); + err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int)); + if (err) { + perror("setsockopt failed()"); + return errno; + } + } + rxpid = fork(); if (rxpid == 0) { if (txmsg_pop || txmsg_start_pop) @@ -788,6 +818,9 @@ static int sendmsg_test(struct sockmap_options *opt) return errno; } + if (opt->tx_wait_mem) + close(c2); + txpid = fork(); if (txpid == 0) { if (opt->sendpage) @@ -1452,6 +1485,14 @@ static void test_txmsg_redir(int cgrp, struct sockmap_options *opt) test_send(opt, cgrp); } +static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt) +{ + txmsg_redir = 1; + opt->tx_wait_mem = true; + test_send_large(opt, cgrp); + opt->tx_wait_mem = false; +} + static void test_txmsg_drop(int cgrp, struct sockmap_options *opt) { txmsg_drop = 1; @@ -1800,6 +1841,7 @@ static int populate_progs(char *bpf_file) struct _test test[] = { {"txmsg test passthrough", test_txmsg_pass}, {"txmsg test redirect", test_txmsg_redir}, + {"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem}, {"txmsg test drop", test_txmsg_drop}, {"txmsg test ingress redirect", test_txmsg_ingress_redir}, {"txmsg test skb", test_txmsg_skb}, diff --git a/tools/testing/selftests/bpf/test_sysctl.c b/tools/testing/selftests/bpf/test_sysctl.c index 57620e7c904863081593481e589f8fb97d2d1942..bcdbd27f22f08cba4cc2785a01e00ba59e893d4c 100644 --- a/tools/testing/selftests/bpf/test_sysctl.c +++ b/tools/testing/selftests/bpf/test_sysctl.c @@ -1372,7 +1372,7 @@ static struct sysctl_test tests[] = { }, { "C prog: deny all writes", - .prog_file = "./test_sysctl_prog.o", + .prog_file = "./test_sysctl_prog.bpf.o", .attach_type = BPF_CGROUP_SYSCTL, .sysctl = "net/ipv4/tcp_mem", .open_flags = O_WRONLY, @@ -1381,7 +1381,7 @@ static struct sysctl_test tests[] = { }, { "C prog: deny access by name", - .prog_file = "./test_sysctl_prog.o", + .prog_file = "./test_sysctl_prog.bpf.o", .attach_type = BPF_CGROUP_SYSCTL, .sysctl = "net/ipv4/route/mtu_expires", .open_flags = O_RDONLY, @@ -1389,7 +1389,7 @@ static struct sysctl_test tests[] = { }, { "C prog: read tcp_mem", - .prog_file = "./test_sysctl_prog.o", + .prog_file = "./test_sysctl_prog.bpf.o", .attach_type = BPF_CGROUP_SYSCTL, .sysctl = "net/ipv4/tcp_mem", .open_flags = O_RDONLY, diff --git a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh index 102e6588e2fed4ced79560842e51c620154f7878..b42c24282c2543a61b559e70c4961011ddb8c463 100755 --- a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh +++ b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh @@ -76,7 +76,7 @@ main() DIR=$(dirname $0) TEST_IF=lo MAX_PING_TRIES=5 -BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.o" +BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.bpf.o" CLSACT_SECTION="tc" XDP_SECTION="xdp" BPF_PROG_ID=0 diff --git a/tools/testing/selftests/bpf/test_tcpnotify_user.c b/tools/testing/selftests/bpf/test_tcpnotify_user.c index 8284db8b0f1353944b1cbb7a5619a2fd25d51e88..595194453ff8f8c25e2deec801c0d8add0cfea3f 100644 --- a/tools/testing/selftests/bpf/test_tcpnotify_user.c +++ b/tools/testing/selftests/bpf/test_tcpnotify_user.c @@ -69,7 +69,7 @@ int verify_result(const struct tcpnotify_globals *result) int main(int argc, char **argv) { - const char *file = "test_tcpnotify_kern.o"; + const char *file = "test_tcpnotify_kern.bpf.o"; struct bpf_map *perf_map, *global_map; struct tcpnotify_globals g = {0}; struct perf_buffer *pb = NULL; diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index f9d553fbf68a3f451afe448494c4e10a013f4db3..2dbcbf363c181af18aa08fc064e820ff36c94dbb 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1498,7 +1498,8 @@ static void do_test_single(struct bpf_test *test, bool unpriv, opts.log_level = DEFAULT_LIBBPF_LOG_LEVEL; opts.prog_flags = pflags; - if (prog_type == BPF_PROG_TYPE_TRACING && test->kfunc) { + if ((prog_type == BPF_PROG_TYPE_TRACING || + prog_type == BPF_PROG_TYPE_LSM) && test->kfunc) { int attach_btf_id; attach_btf_id = libbpf_find_vmlinux_btf_id(test->kfunc, diff --git a/tools/testing/selftests/bpf/test_xdp_redirect.sh b/tools/testing/selftests/bpf/test_xdp_redirect.sh index 1d79f31480ad9be9e4f73d3eb8d2f3808ea2e02a..0746a4fde9d3181667c081698249a1b6dd1d7663 100755 --- a/tools/testing/selftests/bpf/test_xdp_redirect.sh +++ b/tools/testing/selftests/bpf/test_xdp_redirect.sh @@ -54,10 +54,10 @@ test_xdp_redirect() return 0 fi - ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null - ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null - ip link set dev veth1 $xdpmode obj test_xdp_redirect.o sec redirect_to_222 &> /dev/null - ip link set dev veth2 $xdpmode obj test_xdp_redirect.o sec redirect_to_111 &> /dev/null + ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null + ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null + ip link set dev veth1 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_222 &> /dev/null + ip link set dev veth2 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_111 &> /dev/null if ip netns exec ${NS1} ping -c 1 10.1.1.22 &> /dev/null && ip netns exec ${NS2} ping -c 1 10.1.1.11 &> /dev/null; then diff --git a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh index cc57cb87e65f6610353632e40898ad671e2b9a3c..4c3c3fdd2d7304cbe71abbea69f1c20601108b2d 100755 --- a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh +++ b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh @@ -94,7 +94,7 @@ setup_ns() # Add a neigh entry for IPv4 ping test ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0 ip -n ${NS[$i]} link set veth0 $mode obj \ - xdp_dummy.o sec xdp &> /dev/null || \ + xdp_dummy.bpf.o sec xdp &> /dev/null || \ { test_fail "Unable to load dummy xdp" && exit 1; } IFACES="$IFACES veth$i" veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i | awk '/link\/ether/ {print $2}') diff --git a/tools/testing/selftests/bpf/test_xdp_veth.sh b/tools/testing/selftests/bpf/test_xdp_veth.sh index 49936c4c85679da6eb9eafcccadae7d68cd00769..5211ca9a02394c7cc7dea1e8428848f248f13656 100755 --- a/tools/testing/selftests/bpf/test_xdp_veth.sh +++ b/tools/testing/selftests/bpf/test_xdp_veth.sh @@ -101,7 +101,7 @@ ip -n ${NS3} link set dev veth33 up mkdir $BPF_DIR bpftool prog loadall \ - xdp_redirect_map.o $BPF_DIR/progs type xdp \ + xdp_redirect_map.bpf.o $BPF_DIR/progs type xdp \ pinmaps $BPF_DIR/maps bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0 bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0 @@ -110,9 +110,9 @@ ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0 ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1 ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2 -ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.o sec xdp -ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.o sec xdp -ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.o sec xdp +ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.bpf.o sec xdp +ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.bpf.o sec xdp +ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.bpf.o sec xdp trap cleanup EXIT diff --git a/tools/testing/selftests/bpf/test_xsk.sh b/tools/testing/selftests/bpf/test_xsk.sh index 096a957594cdf7ce7572c9ddbc0445d70664a09c..d821fd098504913bcb4844143a78993b2ff8fac6 100755 --- a/tools/testing/selftests/bpf/test_xsk.sh +++ b/tools/testing/selftests/bpf/test_xsk.sh @@ -73,14 +73,20 @@ # # Run and dump packet contents: # sudo ./test_xsk.sh -D +# +# Run test suite for physical device in loopback mode +# sudo ./test_xsk.sh -i IFACE . xsk_prereqs.sh -while getopts "vD" flag +ETH="" + +while getopts "vDi:" flag do case "${flag}" in v) verbose=1;; D) dump_pkts=1;; + i) ETH=${OPTARG};; esac done @@ -132,18 +138,25 @@ setup_vethPairs() { ip link set ${VETH0} up } -validate_root_exec -validate_veth_support ${VETH0} -validate_ip_utility -setup_vethPairs - -retval=$? -if [ $retval -ne 0 ]; then - test_status $retval "${TEST_NAME}" - cleanup_exit ${VETH0} ${VETH1} ${NS1} - exit $retval +if [ ! -z $ETH ]; then + VETH0=${ETH} + VETH1=${ETH} + NS1="" +else + validate_root_exec + validate_veth_support ${VETH0} + validate_ip_utility + setup_vethPairs + + retval=$? + if [ $retval -ne 0 ]; then + test_status $retval "${TEST_NAME}" + cleanup_exit ${VETH0} ${VETH1} ${NS1} + exit $retval + fi fi + if [[ $verbose -eq 1 ]]; then ARGS+="-v " fi @@ -152,26 +165,33 @@ if [[ $dump_pkts -eq 1 ]]; then ARGS="-D " fi +retval=$? test_status $retval "${TEST_NAME}" ## START TESTS statusList=() -TEST_NAME="XSK_SELFTESTS_SOFTIRQ" +TEST_NAME="XSK_SELFTESTS_${VETH0}_SOFTIRQ" exec_xskxceiver -cleanup_exit ${VETH0} ${VETH1} ${NS1} -TEST_NAME="XSK_SELFTESTS_BUSY_POLL" +if [ -z $ETH ]; then + cleanup_exit ${VETH0} ${VETH1} ${NS1} +fi +TEST_NAME="XSK_SELFTESTS_${VETH0}_BUSY_POLL" busy_poll=1 -setup_vethPairs +if [ -z $ETH ]; then + setup_vethPairs +fi exec_xskxceiver ## END TESTS -cleanup_exit ${VETH0} ${VETH1} ${NS1} +if [ -z $ETH ]; then + cleanup_exit ${VETH0} ${VETH1} ${NS1} +fi failures=0 echo -e "\nSummary:" diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c index 3fb4f69b196237dbfcbf590bb154c6a5df766a69..e1a937277b54dc56e6b631c3cfc25a4c0d4d2f6c 100644 --- a/tools/testing/selftests/bpf/verifier/calls.c +++ b/tools/testing/selftests/bpf/verifier/calls.c @@ -284,7 +284,7 @@ .result = ACCEPT, }, { - "calls: not on unpriviledged", + "calls: not on unprivileged", .insns = { BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), BPF_MOV64_IMM(BPF_REG_0, 1), diff --git a/tools/testing/selftests/bpf/verifier/precise.c b/tools/testing/selftests/bpf/verifier/precise.c index 9e754423fa8b0367adabd954ba754b4bec7ddc54..6c03a7d805f9d1708e6382b31902a5087193f968 100644 --- a/tools/testing/selftests/bpf/verifier/precise.c +++ b/tools/testing/selftests/bpf/verifier/precise.c @@ -192,3 +192,28 @@ .result = VERBOSE_ACCEPT, .retval = -1, }, +{ + "precise: mark_chain_precision for ARG_CONST_ALLOC_SIZE_OR_ZERO", + .insns = { + BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_1, offsetof(struct xdp_md, ingress_ifindex)), + BPF_LD_MAP_FD(BPF_REG_6, 0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_MOV64_IMM(BPF_REG_3, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_4, 0, 1), + BPF_MOV64_IMM(BPF_REG_2, 0x1000), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_ringbuf_reserve), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_0, 42), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_ringbuf_submit), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .fixup_map_ringbuf = { 1 }, + .prog_type = BPF_PROG_TYPE_XDP, + .flags = BPF_F_TEST_STATE_FREQ, + .errstr = "invalid access to memory, mem_size=1 off=42 size=8", + .result = REJECT, +}, diff --git a/tools/testing/selftests/bpf/verifier/ref_tracking.c b/tools/testing/selftests/bpf/verifier/ref_tracking.c index 57a83d763ec1780f6c7f63116fb801ea3ee5c3b4..f18ce867271fc03cec84f4ed3732be8ae2d98925 100644 --- a/tools/testing/selftests/bpf/verifier/ref_tracking.c +++ b/tools/testing/selftests/bpf/verifier/ref_tracking.c @@ -84,6 +84,145 @@ .errstr = "Unreleased reference", .result = REJECT, }, +{ + "reference tracking: acquire/release user key reference", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, -3), + BPF_MOV64_IMM(BPF_REG_2, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .fixup_kfunc_btf_id = { + { "bpf_lookup_user_key", 2 }, + { "bpf_key_put", 5 }, + }, + .result = ACCEPT, +}, +{ + "reference tracking: acquire/release system key reference", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .fixup_kfunc_btf_id = { + { "bpf_lookup_system_key", 1 }, + { "bpf_key_put", 4 }, + }, + .result = ACCEPT, +}, +{ + "reference tracking: release user key reference without check", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, -3), + BPF_MOV64_IMM(BPF_REG_2, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar", + .fixup_kfunc_btf_id = { + { "bpf_lookup_user_key", 2 }, + { "bpf_key_put", 4 }, + }, + .result = REJECT, +}, +{ + "reference tracking: release system key reference without check", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar", + .fixup_kfunc_btf_id = { + { "bpf_lookup_system_key", 1 }, + { "bpf_key_put", 3 }, + }, + .result = REJECT, +}, +{ + "reference tracking: release with NULL key pointer", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar", + .fixup_kfunc_btf_id = { + { "bpf_key_put", 1 }, + }, + .result = REJECT, +}, +{ + "reference tracking: leak potential reference to user key", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, -3), + BPF_MOV64_IMM(BPF_REG_2, 0), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .errstr = "Unreleased reference", + .fixup_kfunc_btf_id = { + { "bpf_lookup_user_key", 2 }, + }, + .result = REJECT, +}, +{ + "reference tracking: leak potential reference to system key", + .insns = { + BPF_MOV64_IMM(BPF_REG_1, 1), + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_LSM, + .kfunc = "bpf", + .expected_attach_type = BPF_LSM_MAC, + .flags = BPF_F_SLEEPABLE, + .errstr = "Unreleased reference", + .fixup_kfunc_btf_id = { + { "bpf_lookup_system_key", 1 }, + }, + .result = REJECT, +}, { "reference tracking: release reference without check", .insns = { diff --git a/tools/testing/selftests/bpf/verifier/var_off.c b/tools/testing/selftests/bpf/verifier/var_off.c index 187c6f6e32bc1f334bf5b7ea8fcc441f9d468cfc..d37f512fad16e3b8fa2d1f46fd2974a9a6937217 100644 --- a/tools/testing/selftests/bpf/verifier/var_off.c +++ b/tools/testing/selftests/bpf/verifier/var_off.c @@ -121,7 +121,7 @@ BPF_EXIT_INSN(), }, .fixup_map_hash_8b = { 1 }, - /* The unpriviledged case is not too interesting; variable + /* The unprivileged case is not too interesting; variable * stack access is rejected. */ .errstr_unpriv = "R2 variable stack access prohibited for !root", diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh new file mode 100755 index 0000000000000000000000000000000000000000..ba08922b4a278d6dcc3a5121de4ecc63aec3bf7a --- /dev/null +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +set -e +set -u +set -o pipefail + +VERBOSE="${SELFTESTS_VERBOSE:=0}" +LOG_FILE="$(mktemp /tmp/verify_sig_setup.log.XXXXXX)" + +x509_genkey_content="\ +[ req ] +default_bits = 2048 +distinguished_name = req_distinguished_name +prompt = no +string_mask = utf8only +x509_extensions = myexts + +[ req_distinguished_name ] +CN = eBPF Signature Verification Testing Key + +[ myexts ] +basicConstraints=critical,CA:FALSE +keyUsage=digitalSignature +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid +" + +usage() +{ + echo "Usage: $0 <setup|cleanup <existing_tmp_dir>" + exit 1 +} + +setup() +{ + local tmp_dir="$1" + + echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey + + openssl req -new -nodes -utf8 -sha256 -days 36500 \ + -batch -x509 -config ${tmp_dir}/x509.genkey \ + -outform PEM -out ${tmp_dir}/signing_key.pem \ + -keyout ${tmp_dir}/signing_key.pem 2>&1 + + openssl x509 -in ${tmp_dir}/signing_key.pem -out \ + ${tmp_dir}/signing_key.der -outform der + + key_id=$(cat ${tmp_dir}/signing_key.der | keyctl padd asymmetric ebpf_testing_key @s) + + keyring_id=$(keyctl newring ebpf_testing_keyring @s) + keyctl link $key_id $keyring_id +} + +cleanup() { + local tmp_dir="$1" + + keyctl unlink $(keyctl search @s asymmetric ebpf_testing_key) @s + keyctl unlink $(keyctl search @s keyring ebpf_testing_keyring) @s + rm -rf ${tmp_dir} +} + +catch() +{ + local exit_code="$1" + local log_file="$2" + + if [[ "${exit_code}" -ne 0 ]]; then + cat "${log_file}" >&3 + fi + + rm -f "${log_file}" + exit ${exit_code} +} + +main() +{ + [[ $# -ne 2 ]] && usage + + local action="$1" + local tmp_dir="$2" + + [[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1 + + if [[ "${action}" == "setup" ]]; then + setup "${tmp_dir}" + elif [[ "${action}" == "cleanup" ]]; then + cleanup "${tmp_dir}" + else + echo "Unknown action: ${action}" + exit 1 + fi +} + +trap 'catch "$?" "${LOG_FILE}"' EXIT + +if [[ "${VERBOSE}" -eq 0 ]]; then + # Save the stderr to 3 so that we can output back to + # it incase of an error. + exec 3>&2 1>"${LOG_FILE}" 2>&1 +fi + +main "$@" +rm -f "${LOG_FILE}" diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c new file mode 100644 index 0000000000000000000000000000000000000000..b0d83a28e3486c26bf329d81b02309b724ecfd82 --- /dev/null +++ b/tools/testing/selftests/bpf/veristat.c @@ -0,0 +1,1322 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ +#define _GNU_SOURCE +#include <argp.h> +#include <string.h> +#include <stdlib.h> +#include <linux/compiler.h> +#include <sched.h> +#include <pthread.h> +#include <dirent.h> +#include <signal.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/time.h> +#include <sys/sysinfo.h> +#include <sys/stat.h> +#include <bpf/libbpf.h> +#include <libelf.h> +#include <gelf.h> + +enum stat_id { + VERDICT, + DURATION, + TOTAL_INSNS, + TOTAL_STATES, + PEAK_STATES, + MAX_STATES_PER_INSN, + MARK_READ_MAX_LEN, + + FILE_NAME, + PROG_NAME, + + ALL_STATS_CNT, + NUM_STATS_CNT = FILE_NAME - VERDICT, +}; + +struct verif_stats { + char *file_name; + char *prog_name; + + long stats[NUM_STATS_CNT]; +}; + +struct stat_specs { + int spec_cnt; + enum stat_id ids[ALL_STATS_CNT]; + bool asc[ALL_STATS_CNT]; + int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */ +}; + +enum resfmt { + RESFMT_TABLE, + RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */ + RESFMT_CSV, +}; + +struct filter { + char *file_glob; + char *prog_glob; +}; + +static struct env { + char **filenames; + int filename_cnt; + bool verbose; + bool quiet; + int log_level; + enum resfmt out_fmt; + bool comparison_mode; + + struct verif_stats *prog_stats; + int prog_stat_cnt; + + /* baseline_stats is allocated and used only in comparsion mode */ + struct verif_stats *baseline_stats; + int baseline_stat_cnt; + + struct stat_specs output_spec; + struct stat_specs sort_spec; + + struct filter *allow_filters; + struct filter *deny_filters; + int allow_filter_cnt; + int deny_filter_cnt; + + int files_processed; + int files_skipped; + int progs_processed; + int progs_skipped; +} env; + +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) +{ + if (!env.verbose) + return 0; + if (level == LIBBPF_DEBUG /* && !env.verbose */) + return 0; + return vfprintf(stderr, format, args); +} + +const char *argp_program_version = "veristat"; +const char *argp_program_bug_address = "<bpf@vger.kernel.org>"; +const char argp_program_doc[] = +"veristat BPF verifier stats collection and comparison tool.\n" +"\n" +"USAGE: veristat <obj-file> [<obj-file>...]\n" +" OR: veristat -C <baseline.csv> <comparison.csv>\n"; + +static const struct argp_option opts[] = { + { NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" }, + { "verbose", 'v', NULL, 0, "Verbose mode" }, + { "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode)" }, + { "quiet", 'q', NULL, 0, "Quiet mode" }, + { "emit", 'e', "SPEC", 0, "Specify stats to be emitted" }, + { "sort", 's', "SPEC", 0, "Specify sort order" }, + { "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." }, + { "compare", 'C', NULL, 0, "Comparison mode" }, + { "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." }, + {}, +}; + +static int parse_stats(const char *stats_str, struct stat_specs *specs); +static int append_filter(struct filter **filters, int *cnt, const char *str); +static int append_filter_file(const char *path); + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + void *tmp; + int err; + + switch (key) { + case 'h': + argp_state_help(state, stderr, ARGP_HELP_STD_HELP); + break; + case 'v': + env.verbose = true; + break; + case 'q': + env.quiet = true; + break; + case 'e': + err = parse_stats(arg, &env.output_spec); + if (err) + return err; + break; + case 's': + err = parse_stats(arg, &env.sort_spec); + if (err) + return err; + break; + case 'o': + if (strcmp(arg, "table") == 0) { + env.out_fmt = RESFMT_TABLE; + } else if (strcmp(arg, "csv") == 0) { + env.out_fmt = RESFMT_CSV; + } else { + fprintf(stderr, "Unrecognized output format '%s'\n", arg); + return -EINVAL; + } + break; + case 'l': + errno = 0; + env.log_level = strtol(arg, NULL, 10); + if (errno) { + fprintf(stderr, "invalid log level: %s\n", arg); + argp_usage(state); + } + break; + case 'C': + env.comparison_mode = true; + break; + case 'f': + if (arg[0] == '@') + err = append_filter_file(arg + 1); + else if (arg[0] == '!') + err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1); + else + err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg); + if (err) { + fprintf(stderr, "Failed to collect program filter expressions: %d\n", err); + return err; + } + break; + case ARGP_KEY_ARG: + tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames)); + if (!tmp) + return -ENOMEM; + env.filenames = tmp; + env.filenames[env.filename_cnt] = strdup(arg); + if (!env.filenames[env.filename_cnt]) + return -ENOMEM; + env.filename_cnt++; + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +static const struct argp argp = { + .options = opts, + .parser = parse_arg, + .doc = argp_program_doc, +}; + + +/* Adapted from perf/util/string.c */ +static bool glob_matches(const char *str, const char *pat) +{ + while (*str && *pat && *pat != '*') { + if (*str != *pat) + return false; + str++; + pat++; + } + /* Check wild card */ + if (*pat == '*') { + while (*pat == '*') + pat++; + if (!*pat) /* Tail wild card matches all */ + return true; + while (*str) + if (glob_matches(str++, pat)) + return true; + } + return !*str && !*pat; +} + +static bool should_process_file(const char *filename) +{ + int i; + + if (env.deny_filter_cnt > 0) { + for (i = 0; i < env.deny_filter_cnt; i++) { + if (glob_matches(filename, env.deny_filters[i].file_glob)) + return false; + } + } + + if (env.allow_filter_cnt == 0) + return true; + + for (i = 0; i < env.allow_filter_cnt; i++) { + if (glob_matches(filename, env.allow_filters[i].file_glob)) + return true; + } + + return false; +} + +static bool is_bpf_obj_file(const char *path) { + Elf64_Ehdr *ehdr; + int fd, err = -EINVAL; + Elf *elf = NULL; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return true; /* we'll fail later and propagate error */ + + /* ensure libelf is initialized */ + (void)elf_version(EV_CURRENT); + + elf = elf_begin(fd, ELF_C_READ, NULL); + if (!elf) + goto cleanup; + + if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64) + goto cleanup; + + ehdr = elf64_getehdr(elf); + /* Old LLVM set e_machine to EM_NONE */ + if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) + goto cleanup; + + err = 0; +cleanup: + if (elf) + elf_end(elf); + close(fd); + return err == 0; +} + +static bool should_process_prog(const char *path, const char *prog_name) +{ + const char *filename = basename(path); + int i; + + if (env.deny_filter_cnt > 0) { + for (i = 0; i < env.deny_filter_cnt; i++) { + if (glob_matches(filename, env.deny_filters[i].file_glob)) + return false; + if (!env.deny_filters[i].prog_glob) + continue; + if (glob_matches(prog_name, env.deny_filters[i].prog_glob)) + return false; + } + } + + if (env.allow_filter_cnt == 0) + return true; + + for (i = 0; i < env.allow_filter_cnt; i++) { + if (!glob_matches(filename, env.allow_filters[i].file_glob)) + continue; + /* if filter specifies only filename glob part, it implicitly + * allows all progs within that file + */ + if (!env.allow_filters[i].prog_glob) + return true; + if (glob_matches(prog_name, env.allow_filters[i].prog_glob)) + return true; + } + + return false; +} + +static int append_filter(struct filter **filters, int *cnt, const char *str) +{ + struct filter *f; + void *tmp; + const char *p; + + tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters)); + if (!tmp) + return -ENOMEM; + *filters = tmp; + + f = &(*filters)[*cnt]; + f->file_glob = f->prog_glob = NULL; + + /* filter can be specified either as "<obj-glob>" or "<obj-glob>/<prog-glob>" */ + p = strchr(str, '/'); + if (!p) { + f->file_glob = strdup(str); + if (!f->file_glob) + return -ENOMEM; + } else { + f->file_glob = strndup(str, p - str); + f->prog_glob = strdup(p + 1); + if (!f->file_glob || !f->prog_glob) { + free(f->file_glob); + free(f->prog_glob); + f->file_glob = f->prog_glob = NULL; + return -ENOMEM; + } + } + + *cnt = *cnt + 1; + return 0; +} + +static int append_filter_file(const char *path) +{ + char buf[1024]; + FILE *f; + int err = 0; + + f = fopen(path, "r"); + if (!f) { + err = -errno; + fprintf(stderr, "Failed to open filters in '%s': %d\n", path, err); + return err; + } + + while (fscanf(f, " %1023[^\n]\n", buf) == 1) { + /* lines starting with # are comments, skip them */ + if (buf[0] == '\0' || buf[0] == '#') + continue; + /* lines starting with ! are negative match filters */ + if (buf[0] == '!') + err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1); + else + err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf); + if (err) + goto cleanup; + } + +cleanup: + fclose(f); + return err; +} + +static const struct stat_specs default_output_spec = { + .spec_cnt = 7, + .ids = { + FILE_NAME, PROG_NAME, VERDICT, DURATION, + TOTAL_INSNS, TOTAL_STATES, PEAK_STATES, + }, +}; + +static const struct stat_specs default_sort_spec = { + .spec_cnt = 2, + .ids = { + FILE_NAME, PROG_NAME, + }, + .asc = { true, true, }, +}; + +static struct stat_def { + const char *header; + const char *names[4]; + bool asc_by_default; +} stat_defs[] = { + [FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */ }, + [PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */ }, + [VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */ }, + [DURATION] = { "Duration (us)", {"duration", "dur"}, }, + [TOTAL_INSNS] = { "Total insns", {"total_insns", "insns"}, }, + [TOTAL_STATES] = { "Total states", {"total_states", "states"}, }, + [PEAK_STATES] = { "Peak states", {"peak_states"}, }, + [MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, }, + [MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, }, +}; + +static int parse_stat(const char *stat_name, struct stat_specs *specs) +{ + int id, i; + + if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) { + fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids)); + return -E2BIG; + } + + for (id = 0; id < ARRAY_SIZE(stat_defs); id++) { + struct stat_def *def = &stat_defs[id]; + + for (i = 0; i < ARRAY_SIZE(stat_defs[id].names); i++) { + if (!def->names[i] || strcmp(def->names[i], stat_name) != 0) + continue; + + specs->ids[specs->spec_cnt] = id; + specs->asc[specs->spec_cnt] = def->asc_by_default; + specs->spec_cnt++; + + return 0; + } + } + + fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name); + return -ESRCH; +} + +static int parse_stats(const char *stats_str, struct stat_specs *specs) +{ + char *input, *state = NULL, *next; + int err; + + input = strdup(stats_str); + if (!input) + return -ENOMEM; + + while ((next = strtok_r(state ? NULL : input, ",", &state))) { + err = parse_stat(next, specs); + if (err) + return err; + } + + return 0; +} + +static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt) +{ + int i; + + if (!stats) + return; + + for (i = 0; i < stat_cnt; i++) { + free(stats[i].file_name); + free(stats[i].prog_name); + } + free(stats); +} + +static char verif_log_buf[64 * 1024]; + +#define MAX_PARSED_LOG_LINES 100 + +static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s) +{ + const char *cur; + int pos, lines; + + buf[buf_sz - 1] = '\0'; + + for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) { + /* find previous endline or otherwise take the start of log buf */ + for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) { + } + /* next time start from end of previous line (or pos goes to <0) */ + pos--; + /* if we found endline, point right after endline symbol; + * otherwise, stay at the beginning of log buf + */ + if (cur[0] == '\n') + cur++; + + if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION])) + continue; + if (6 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld", + &s->stats[TOTAL_INSNS], + &s->stats[MAX_STATES_PER_INSN], + &s->stats[TOTAL_STATES], + &s->stats[PEAK_STATES], + &s->stats[MARK_READ_MAX_LEN])) + continue; + } + + return 0; +} + +static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog) +{ + const char *prog_name = bpf_program__name(prog); + size_t buf_sz = sizeof(verif_log_buf); + char *buf = verif_log_buf; + struct verif_stats *stats; + int err = 0; + void *tmp; + + if (!should_process_prog(filename, bpf_program__name(prog))) { + env.progs_skipped++; + return 0; + } + + tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats)); + if (!tmp) + return -ENOMEM; + env.prog_stats = tmp; + stats = &env.prog_stats[env.prog_stat_cnt++]; + memset(stats, 0, sizeof(*stats)); + + if (env.verbose) { + buf_sz = 16 * 1024 * 1024; + buf = malloc(buf_sz); + if (!buf) + return -ENOMEM; + bpf_program__set_log_buf(prog, buf, buf_sz); + bpf_program__set_log_level(prog, env.log_level | 4); /* stats + log */ + } else { + bpf_program__set_log_buf(prog, buf, buf_sz); + bpf_program__set_log_level(prog, 4); /* only verifier stats */ + } + verif_log_buf[0] = '\0'; + + err = bpf_object__load(obj); + env.progs_processed++; + + stats->file_name = strdup(basename(filename)); + stats->prog_name = strdup(bpf_program__name(prog)); + stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */ + parse_verif_log(buf, buf_sz, stats); + + if (env.verbose) { + printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n", + filename, prog_name, stats->stats[DURATION], + err ? "failure" : "success", buf); + } + + if (verif_log_buf != buf) + free(buf); + + return 0; +}; + +static int process_obj(const char *filename) +{ + struct bpf_object *obj = NULL, *tobj; + struct bpf_program *prog, *tprog, *lprog; + libbpf_print_fn_t old_libbpf_print_fn; + LIBBPF_OPTS(bpf_object_open_opts, opts); + int err = 0, prog_cnt = 0; + + if (!should_process_file(basename(filename))) { + if (env.verbose) + printf("Skipping '%s' due to filters...\n", filename); + env.files_skipped++; + return 0; + } + if (!is_bpf_obj_file(filename)) { + if (env.verbose) + printf("Skipping '%s' as it's not a BPF object file...\n", filename); + env.files_skipped++; + return 0; + } + + if (!env.quiet && env.out_fmt == RESFMT_TABLE) + printf("Processing '%s'...\n", basename(filename)); + + old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn); + obj = bpf_object__open_file(filename, &opts); + if (!obj) { + /* if libbpf can't open BPF object file, it could be because + * that BPF object file is incomplete and has to be statically + * linked into a final BPF object file; instead of bailing + * out, report it into stderr, mark it as skipped, and + * proceeed + */ + fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno); + env.files_skipped++; + err = 0; + goto cleanup; + } + + env.files_processed++; + + bpf_object__for_each_program(prog, obj) { + prog_cnt++; + } + + if (prog_cnt == 1) { + prog = bpf_object__next_program(obj, NULL); + bpf_program__set_autoload(prog, true); + process_prog(filename, obj, prog); + goto cleanup; + } + + bpf_object__for_each_program(prog, obj) { + const char *prog_name = bpf_program__name(prog); + + tobj = bpf_object__open_file(filename, &opts); + if (!tobj) { + err = -errno; + fprintf(stderr, "Failed to open '%s': %d\n", filename, err); + goto cleanup; + } + + bpf_object__for_each_program(tprog, tobj) { + const char *tprog_name = bpf_program__name(tprog); + + if (strcmp(prog_name, tprog_name) == 0) { + bpf_program__set_autoload(tprog, true); + lprog = tprog; + } else { + bpf_program__set_autoload(tprog, false); + } + } + + process_prog(filename, tobj, lprog); + bpf_object__close(tobj); + } + +cleanup: + bpf_object__close(obj); + libbpf_set_print(old_libbpf_print_fn); + return err; +} + +static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2, + enum stat_id id, bool asc) +{ + int cmp = 0; + + switch (id) { + case FILE_NAME: + cmp = strcmp(s1->file_name, s2->file_name); + break; + case PROG_NAME: + cmp = strcmp(s1->prog_name, s2->prog_name); + break; + case VERDICT: + case DURATION: + case TOTAL_INSNS: + case TOTAL_STATES: + case PEAK_STATES: + case MAX_STATES_PER_INSN: + case MARK_READ_MAX_LEN: { + long v1 = s1->stats[id]; + long v2 = s2->stats[id]; + + if (v1 != v2) + cmp = v1 < v2 ? -1 : 1; + break; + } + default: + fprintf(stderr, "Unrecognized stat #%d\n", id); + exit(1); + } + + return asc ? cmp : -cmp; +} + +static int cmp_prog_stats(const void *v1, const void *v2) +{ + const struct verif_stats *s1 = v1, *s2 = v2; + int i, cmp; + + for (i = 0; i < env.sort_spec.spec_cnt; i++) { + cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], env.sort_spec.asc[i]); + if (cmp != 0) + return cmp; + } + + return 0; +} + +#define HEADER_CHAR '-' +#define COLUMN_SEP " " + +static void output_header_underlines(void) +{ + int i, j, len; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + len = env.output_spec.lens[i]; + + printf("%s", i == 0 ? "" : COLUMN_SEP); + for (j = 0; j < len; j++) + printf("%c", HEADER_CHAR); + } + printf("\n"); +} + +static void output_headers(enum resfmt fmt) +{ + int i, len; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + int id = env.output_spec.ids[i]; + int *max_len = &env.output_spec.lens[i]; + + switch (fmt) { + case RESFMT_TABLE_CALCLEN: + len = snprintf(NULL, 0, "%s", stat_defs[id].header); + if (len > *max_len) + *max_len = len; + break; + case RESFMT_TABLE: + printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, stat_defs[id].header); + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + case RESFMT_CSV: + printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]); + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + } + } + + if (fmt == RESFMT_TABLE) + output_header_underlines(); +} + +static void prepare_value(const struct verif_stats *s, enum stat_id id, + const char **str, long *val) +{ + switch (id) { + case FILE_NAME: + *str = s->file_name; + break; + case PROG_NAME: + *str = s->prog_name; + break; + case VERDICT: + *str = s->stats[VERDICT] ? "success" : "failure"; + break; + case DURATION: + case TOTAL_INSNS: + case TOTAL_STATES: + case PEAK_STATES: + case MAX_STATES_PER_INSN: + case MARK_READ_MAX_LEN: + *val = s->stats[id]; + break; + default: + fprintf(stderr, "Unrecognized stat #%d\n", id); + exit(1); + } +} + +static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last) +{ + int i; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + int id = env.output_spec.ids[i]; + int *max_len = &env.output_spec.lens[i], len; + const char *str = NULL; + long val = 0; + + prepare_value(s, id, &str, &val); + + switch (fmt) { + case RESFMT_TABLE_CALCLEN: + if (str) + len = snprintf(NULL, 0, "%s", str); + else + len = snprintf(NULL, 0, "%ld", val); + if (len > *max_len) + *max_len = len; + break; + case RESFMT_TABLE: + if (str) + printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str); + else + printf("%s%*ld", i == 0 ? "" : COLUMN_SEP, *max_len, val); + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + case RESFMT_CSV: + if (str) + printf("%s%s", i == 0 ? "" : ",", str); + else + printf("%s%ld", i == 0 ? "" : ",", val); + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + } + } + + if (last && fmt == RESFMT_TABLE) { + output_header_underlines(); + printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n", + env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped); + } +} + +static int handle_verif_mode(void) +{ + int i, err; + + if (env.filename_cnt == 0) { + fprintf(stderr, "Please provide path to BPF object file!\n"); + argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); + return -EINVAL; + } + + for (i = 0; i < env.filename_cnt; i++) { + err = process_obj(env.filenames[i]); + if (err) { + fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err); + return err; + } + } + + qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); + + if (env.out_fmt == RESFMT_TABLE) { + /* calculate column widths */ + output_headers(RESFMT_TABLE_CALCLEN); + for (i = 0; i < env.prog_stat_cnt; i++) + output_stats(&env.prog_stats[i], RESFMT_TABLE_CALCLEN, false); + } + + /* actually output the table */ + output_headers(env.out_fmt); + for (i = 0; i < env.prog_stat_cnt; i++) { + output_stats(&env.prog_stats[i], env.out_fmt, i == env.prog_stat_cnt - 1); + } + + return 0; +} + +static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st) +{ + switch (id) { + case FILE_NAME: + st->file_name = strdup(str); + if (!st->file_name) + return -ENOMEM; + break; + case PROG_NAME: + st->prog_name = strdup(str); + if (!st->prog_name) + return -ENOMEM; + break; + case VERDICT: + if (strcmp(str, "success") == 0) { + st->stats[VERDICT] = true; + } else if (strcmp(str, "failure") == 0) { + st->stats[VERDICT] = false; + } else { + fprintf(stderr, "Unrecognized verification verdict '%s'\n", str); + return -EINVAL; + } + break; + case DURATION: + case TOTAL_INSNS: + case TOTAL_STATES: + case PEAK_STATES: + case MAX_STATES_PER_INSN: + case MARK_READ_MAX_LEN: { + long val; + int err, n; + + if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) { + err = -errno; + fprintf(stderr, "Failed to parse '%s' as integer\n", str); + return err; + } + + st->stats[id] = val; + break; + } + default: + fprintf(stderr, "Unrecognized stat #%d\n", id); + return -EINVAL; + } + return 0; +} + +static int parse_stats_csv(const char *filename, struct stat_specs *specs, + struct verif_stats **statsp, int *stat_cntp) +{ + char line[4096]; + FILE *f; + int err = 0; + bool header = true; + + f = fopen(filename, "r"); + if (!f) { + err = -errno; + fprintf(stderr, "Failed to open '%s': %d\n", filename, err); + return err; + } + + *stat_cntp = 0; + + while (fgets(line, sizeof(line), f)) { + char *input = line, *state = NULL, *next; + struct verif_stats *st = NULL; + int col = 0; + + if (!header) { + void *tmp; + + tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp)); + if (!tmp) { + err = -ENOMEM; + goto cleanup; + } + *statsp = tmp; + + st = &(*statsp)[*stat_cntp]; + memset(st, 0, sizeof(*st)); + + *stat_cntp += 1; + } + + while ((next = strtok_r(state ? NULL : input, ",\n", &state))) { + if (header) { + /* for the first line, set up spec stats */ + err = parse_stat(next, specs); + if (err) + goto cleanup; + continue; + } + + /* for all other lines, parse values based on spec */ + if (col >= specs->spec_cnt) { + fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n", + col, *stat_cntp, filename); + err = -EINVAL; + goto cleanup; + } + err = parse_stat_value(next, specs->ids[col], st); + if (err) + goto cleanup; + col++; + } + + if (header) { + header = false; + continue; + } + + if (col < specs->spec_cnt) { + fprintf(stderr, "Not enough columns in row #%d in '%s'\n", + *stat_cntp, filename); + err = -EINVAL; + goto cleanup; + } + + if (!st->file_name || !st->prog_name) { + fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n", + *stat_cntp, filename); + err = -EINVAL; + goto cleanup; + } + + /* in comparison mode we can only check filters after we + * parsed entire line; if row should be ignored we pretend we + * never parsed it + */ + if (!should_process_prog(st->file_name, st->prog_name)) { + free(st->file_name); + free(st->prog_name); + *stat_cntp -= 1; + } + } + + if (!feof(f)) { + err = -errno; + fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err); + } + +cleanup: + fclose(f); + return err; +} + +/* empty/zero stats for mismatched rows */ +static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" }; + +static bool is_key_stat(enum stat_id id) +{ + return id == FILE_NAME || id == PROG_NAME; +} + +static void output_comp_header_underlines(void) +{ + int i, j, k; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + int id = env.output_spec.ids[i]; + int max_j = is_key_stat(id) ? 1 : 3; + + for (j = 0; j < max_j; j++) { + int len = env.output_spec.lens[3 * i + j]; + + printf("%s", i + j == 0 ? "" : COLUMN_SEP); + + for (k = 0; k < len; k++) + printf("%c", HEADER_CHAR); + } + } + printf("\n"); +} + +static void output_comp_headers(enum resfmt fmt) +{ + static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"}; + static const char *name_sfxs[3] = {"_base", "_comp", "_diff"}; + int i, j, len; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + int id = env.output_spec.ids[i]; + /* key stats don't have A/B/DIFF columns, they are common for both data sets */ + int max_j = is_key_stat(id) ? 1 : 3; + + for (j = 0; j < max_j; j++) { + int *max_len = &env.output_spec.lens[3 * i + j]; + bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1); + const char *sfx; + + switch (fmt) { + case RESFMT_TABLE_CALCLEN: + sfx = is_key_stat(id) ? "" : table_sfxs[j]; + len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx); + if (len > *max_len) + *max_len = len; + break; + case RESFMT_TABLE: + sfx = is_key_stat(id) ? "" : table_sfxs[j]; + printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP, + *max_len - (int)strlen(sfx), stat_defs[id].header, sfx); + if (last) + printf("\n"); + break; + case RESFMT_CSV: + sfx = is_key_stat(id) ? "" : name_sfxs[j]; + printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx); + if (last) + printf("\n"); + break; + } + } + } + + if (fmt == RESFMT_TABLE) + output_comp_header_underlines(); +} + +static void output_comp_stats(const struct verif_stats *base, const struct verif_stats *comp, + enum resfmt fmt, bool last) +{ + char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {}; + int i; + + for (i = 0; i < env.output_spec.spec_cnt; i++) { + int id = env.output_spec.ids[i], len; + int *max_len_base = &env.output_spec.lens[3 * i + 0]; + int *max_len_comp = &env.output_spec.lens[3 * i + 1]; + int *max_len_diff = &env.output_spec.lens[3 * i + 2]; + const char *base_str = NULL, *comp_str = NULL; + long base_val = 0, comp_val = 0, diff_val = 0; + + prepare_value(base, id, &base_str, &base_val); + prepare_value(comp, id, &comp_str, &comp_val); + + /* normalize all the outputs to be in string buffers for simplicity */ + if (is_key_stat(id)) { + /* key stats (file and program name) are always strings */ + if (base != &fallback_stats) + snprintf(base_buf, sizeof(base_buf), "%s", base_str); + else + snprintf(base_buf, sizeof(base_buf), "%s", comp_str); + } else if (base_str) { + snprintf(base_buf, sizeof(base_buf), "%s", base_str); + snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str); + if (strcmp(base_str, comp_str) == 0) + snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH"); + else + snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH"); + } else { + snprintf(base_buf, sizeof(base_buf), "%ld", base_val); + snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val); + + diff_val = comp_val - base_val; + if (base == &fallback_stats || comp == &fallback_stats || base_val == 0) { + snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", + diff_val, comp_val < base_val ? -100.0 : 100.0); + } else { + snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", + diff_val, diff_val * 100.0 / base_val); + } + } + + switch (fmt) { + case RESFMT_TABLE_CALCLEN: + len = strlen(base_buf); + if (len > *max_len_base) + *max_len_base = len; + if (!is_key_stat(id)) { + len = strlen(comp_buf); + if (len > *max_len_comp) + *max_len_comp = len; + len = strlen(diff_buf); + if (len > *max_len_diff) + *max_len_diff = len; + } + break; + case RESFMT_TABLE: { + /* string outputs are left-aligned, number outputs are right-aligned */ + const char *fmt = base_str ? "%s%-*s" : "%s%*s"; + + printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf); + if (!is_key_stat(id)) { + printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf); + printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf); + } + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + } + case RESFMT_CSV: + printf("%s%s", i == 0 ? "" : ",", base_buf); + if (!is_key_stat(id)) { + printf("%s%s", i == 0 ? "" : ",", comp_buf); + printf("%s%s", i == 0 ? "" : ",", diff_buf); + } + if (i == env.output_spec.spec_cnt - 1) + printf("\n"); + break; + } + } + + if (last && fmt == RESFMT_TABLE) + output_comp_header_underlines(); +} + +static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp) +{ + int r; + + r = strcmp(base->file_name, comp->file_name); + if (r != 0) + return r; + return strcmp(base->prog_name, comp->prog_name); +} + +static int handle_comparison_mode(void) +{ + struct stat_specs base_specs = {}, comp_specs = {}; + enum resfmt cur_fmt; + int err, i, j; + + if (env.filename_cnt != 2) { + fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n"); + argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); + return -EINVAL; + } + + err = parse_stats_csv(env.filenames[0], &base_specs, + &env.baseline_stats, &env.baseline_stat_cnt); + if (err) { + fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err); + return err; + } + err = parse_stats_csv(env.filenames[1], &comp_specs, + &env.prog_stats, &env.prog_stat_cnt); + if (err) { + fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err); + return err; + } + + /* To keep it simple we validate that the set and order of stats in + * both CSVs are exactly the same. This can be lifted with a bit more + * pre-processing later. + */ + if (base_specs.spec_cnt != comp_specs.spec_cnt) { + fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n", + env.filenames[0], env.filenames[1], + base_specs.spec_cnt, comp_specs.spec_cnt); + return -EINVAL; + } + for (i = 0; i < base_specs.spec_cnt; i++) { + if (base_specs.ids[i] != comp_specs.ids[i]) { + fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n", + env.filenames[0], env.filenames[1], + stat_defs[base_specs.ids[i]].names[0], + stat_defs[comp_specs.ids[i]].names[0]); + return -EINVAL; + } + } + + qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats); + qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats); + + /* for human-readable table output we need to do extra pass to + * calculate column widths, so we substitute current output format + * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE + * and do everything again. + */ + if (env.out_fmt == RESFMT_TABLE) + cur_fmt = RESFMT_TABLE_CALCLEN; + else + cur_fmt = env.out_fmt; + +one_more_time: + output_comp_headers(cur_fmt); + + /* If baseline and comparison datasets have different subset of rows + * (we match by 'object + prog' as a unique key) then assume + * empty/missing/zero value for rows that are missing in the opposite + * data set + */ + i = j = 0; + while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) { + bool last = (i == env.baseline_stat_cnt - 1) || (j == env.prog_stat_cnt - 1); + const struct verif_stats *base, *comp; + int r; + + base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats; + comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats; + + if (!base->file_name || !base->prog_name) { + fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", + i, env.filenames[0]); + return -EINVAL; + } + if (!comp->file_name || !comp->prog_name) { + fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n", + j, env.filenames[1]); + return -EINVAL; + } + + r = cmp_stats_key(base, comp); + if (r == 0) { + output_comp_stats(base, comp, cur_fmt, last); + i++; + j++; + } else if (comp == &fallback_stats || r < 0) { + output_comp_stats(base, &fallback_stats, cur_fmt, last); + i++; + } else { + output_comp_stats(&fallback_stats, comp, cur_fmt, last); + j++; + } + } + + if (cur_fmt == RESFMT_TABLE_CALCLEN) { + cur_fmt = RESFMT_TABLE; + goto one_more_time; /* ... this time with feeling */ + } + + return 0; +} + +int main(int argc, char **argv) +{ + int err = 0, i; + + if (argp_parse(&argp, argc, argv, 0, NULL, NULL)) + return 1; + + if (env.verbose && env.quiet) { + fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n"); + argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat"); + return 1; + } + if (env.verbose && env.log_level == 0) + env.log_level = 1; + + if (env.output_spec.spec_cnt == 0) + env.output_spec = default_output_spec; + if (env.sort_spec.spec_cnt == 0) + env.sort_spec = default_sort_spec; + + if (env.comparison_mode) + err = handle_comparison_mode(); + else + err = handle_verif_mode(); + + free_verif_stats(env.prog_stats, env.prog_stat_cnt); + free_verif_stats(env.baseline_stats, env.baseline_stat_cnt); + for (i = 0; i < env.filename_cnt; i++) + free(env.filenames[i]); + free(env.filenames); + for (i = 0; i < env.allow_filter_cnt; i++) { + free(env.allow_filters[i].file_glob); + free(env.allow_filters[i].prog_glob); + } + free(env.allow_filters); + for (i = 0; i < env.deny_filter_cnt; i++) { + free(env.deny_filters[i].file_glob); + free(env.deny_filters[i].prog_glob); + } + free(env.deny_filters); + return -err; +} diff --git a/tools/testing/selftests/bpf/veristat.cfg b/tools/testing/selftests/bpf/veristat.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1a385061618d849aeeb08ff39da331618c13db89 --- /dev/null +++ b/tools/testing/selftests/bpf/veristat.cfg @@ -0,0 +1,17 @@ +# pre-canned list of rather complex selftests/bpf BPF object files to monitor +# BPF verifier's performance on +bpf_flow* +bpf_loop_bench* +loop* +netif_receive_skb* +profiler* +pyperf* +strobemeta* +test_cls_redirect* +test_l4lb +test_sysctl* +test_tcp_hdr_* +test_usdt* +test_verif_scale* +test_xdp_noinline* +xdp_synproxy* diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index b86ae4a2e5c56f6c9c4af062d82d58b42d90b38d..a29aa05ebb3e98cc64129ca8bcb7fe30de17e20c 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -307,6 +307,20 @@ update_kconfig() fi } +catch() +{ + local exit_code=$1 + local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}" + # This is just a cleanup and the directory may + # have already been unmounted. So, don't let this + # clobber the error code we intend to return. + unmount_image || true + if [[ -f "${exit_status_file}" ]]; then + exit_code="$(cat ${exit_status_file})" + fi + exit ${exit_code} +} + main() { local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" @@ -319,7 +333,7 @@ main() local exit_command="poweroff -f" local debug_shell="no" - while getopts 'hskid:j:' opt; do + while getopts ':hskid:j:' opt; do case ${opt} in i) update_image="yes" @@ -353,6 +367,8 @@ main() done shift $((OPTIND -1)) + trap 'catch "$?"' EXIT + if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" else @@ -409,20 +425,4 @@ main() fi } -catch() -{ - local exit_code=$1 - local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}" - # This is just a cleanup and the directory may - # have already been unmounted. So, don't let this - # clobber the error code we intend to return. - unmount_image || true - if [[ -f "${exit_status_file}" ]]; then - exit_code="$(cat ${exit_status_file})" - fi - exit ${exit_code} -} - -trap 'catch "$?"' EXIT - main "$@" diff --git a/tools/testing/selftests/bpf/xdp_redirect_multi.c b/tools/testing/selftests/bpf/xdp_redirect_multi.c index c03b3a75991f8753cc8e34657c67b10a8a2aea34..c1fc44c87c300c72df65a2fb00f9293c3b4f2ffc 100644 --- a/tools/testing/selftests/bpf/xdp_redirect_multi.c +++ b/tools/testing/selftests/bpf/xdp_redirect_multi.c @@ -142,7 +142,7 @@ int main(int argc, char **argv) } printf("\n"); - snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]); obj = bpf_object__open_file(filename, NULL); err = libbpf_get_error(obj); if (err) diff --git a/tools/testing/selftests/bpf/xdp_synproxy.c b/tools/testing/selftests/bpf/xdp_synproxy.c index d874ddfb39c46c942a22c446b7dc523b3a010101..ff35320d2be9720293ddbb2fff6bb7dbfabdbe5d 100644 --- a/tools/testing/selftests/bpf/xdp_synproxy.c +++ b/tools/testing/selftests/bpf/xdp_synproxy.c @@ -193,7 +193,7 @@ static int syncookie_attach(const char *argv0, unsigned int ifindex, bool tc) int prog_fd; int err; - snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv0); + snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.bpf.o", argv0); obj = bpf_object__open_file(xdp_filename, NULL); err = libbpf_get_error(obj); if (err < 0) { diff --git a/tools/testing/selftests/bpf/xdping.c b/tools/testing/selftests/bpf/xdping.c index 5b6f977870f8f58502f9c573f56e825c5a35fda6..1503a1d2faa090e13c88bb7cec2fc9d44226b2c6 100644 --- a/tools/testing/selftests/bpf/xdping.c +++ b/tools/testing/selftests/bpf/xdping.c @@ -168,7 +168,7 @@ int main(int argc, char **argv) /* Use libbpf 1.0 API mode */ libbpf_set_strict_mode(LIBBPF_STRICT_ALL); - snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]); if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) { fprintf(stderr, "load of %s failed\n", filename); diff --git a/tools/testing/selftests/bpf/xsk.c b/tools/testing/selftests/bpf/xsk.c index f2721a4ae7c505f942f869342300b51c95004746..0b3ff49c740d455c5101f15d3ec2ea25e3096bae 100644 --- a/tools/testing/selftests/bpf/xsk.c +++ b/tools/testing/selftests/bpf/xsk.c @@ -1237,15 +1237,15 @@ void xsk_socket__delete(struct xsk_socket *xsk) ctx = xsk->ctx; umem = ctx->umem; - xsk_put_ctx(ctx, true); - - if (!ctx->refcount) { + if (ctx->refcount == 1) { xsk_delete_bpf_maps(xsk); close(ctx->prog_fd); if (ctx->has_bpf_link) close(ctx->link_fd); } + xsk_put_ctx(ctx, true); + err = xsk_get_mmap_offsets(xsk->fd, &off); if (!err) { if (xsk->rx) { diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index 74d56d971bafc1e34a5e4ca846668334694a2099..681a5db80dae0c2ee202033ca94adb6c28d5588a 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -99,6 +99,8 @@ #include <stdatomic.h> #include "xsk.h" #include "xskxceiver.h" +#include <bpf/bpf.h> +#include <linux/filter.h> #include "../kselftest.h" /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf. @@ -122,9 +124,20 @@ static void __exit_with_error(int error, const char *file, const char *func, int } #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) - -#define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV" #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" +static char *mode_string(struct test_spec *test) +{ + switch (test->mode) { + case TEST_MODE_SKB: + return "SKB"; + case TEST_MODE_DRV: + return "DRV"; + case TEST_MODE_ZC: + return "ZC"; + default: + return "BOGUS"; + } +} static void report_failure(struct test_spec *test) { @@ -244,6 +257,11 @@ static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject, memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE); } +static bool is_umem_valid(struct ifobject *ifobj) +{ + return !!ifobj->umem->umem; +} + static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr) { udp_hdr->check = 0; @@ -294,8 +312,8 @@ static void enable_busy_poll(struct xsk_socket_info *xsk) exit_with_error(errno); } -static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, - struct ifobject *ifobject, bool shared) +static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, + struct ifobject *ifobject, bool shared) { struct xsk_socket_config cfg = {}; struct xsk_ring_cons *rxr; @@ -315,6 +333,51 @@ static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_inf return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg); } +static bool ifobj_zc_avail(struct ifobject *ifobject) +{ + size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; + struct xsk_socket_info *xsk; + struct xsk_umem_info *umem; + bool zc_avail = false; + void *bufs; + int ret; + + bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); + if (bufs == MAP_FAILED) + exit_with_error(errno); + + umem = calloc(1, sizeof(struct xsk_umem_info)); + if (!umem) { + munmap(bufs, umem_sz); + exit_with_error(-ENOMEM); + } + umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; + ret = xsk_configure_umem(umem, bufs, umem_sz); + if (ret) + exit_with_error(-ret); + + xsk = calloc(1, sizeof(struct xsk_socket_info)); + if (!xsk) + goto out; + ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; + ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE; + ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; + ifobject->rx_on = true; + xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; + ret = __xsk_configure_socket(xsk, umem, ifobject, false); + if (!ret) + zc_avail = true; + + xsk_socket__delete(xsk->xsk); + free(xsk); +out: + munmap(umem->buffer, umem_sz); + xsk_umem__delete(umem->umem); + free(umem); + return zc_avail; +} + static struct option long_options[] = { {"interface", required_argument, 0, 'i'}, {"busy-poll", no_argument, 0, 'b'}, @@ -426,20 +489,24 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, ifobj->use_poll = false; ifobj->use_fill_ring = true; ifobj->release_rx = true; - ifobj->pkt_stream = test->pkt_stream_default; ifobj->validation_func = NULL; if (i == 0) { ifobj->rx_on = false; ifobj->tx_on = true; + ifobj->pkt_stream = test->tx_pkt_stream_default; } else { ifobj->rx_on = true; ifobj->tx_on = false; + ifobj->pkt_stream = test->rx_pkt_stream_default; } memset(ifobj->umem, 0, sizeof(*ifobj->umem)); ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; + if (ifobj->shared_umem && ifobj->rx_on) + ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS * + XSK_UMEM__DEFAULT_FRAME_SIZE; for (j = 0; j < MAX_SOCKETS; j++) { memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); @@ -458,12 +525,15 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, enum test_mode mode) { - struct pkt_stream *pkt_stream; + struct pkt_stream *tx_pkt_stream; + struct pkt_stream *rx_pkt_stream; u32 i; - pkt_stream = test->pkt_stream_default; + tx_pkt_stream = test->tx_pkt_stream_default; + rx_pkt_stream = test->rx_pkt_stream_default; memset(test, 0, sizeof(*test)); - test->pkt_stream_default = pkt_stream; + test->tx_pkt_stream_default = tx_pkt_stream; + test->rx_pkt_stream_default = rx_pkt_stream; for (i = 0; i < MAX_INTERFACES; i++) { struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; @@ -474,9 +544,14 @@ static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, else ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE; - ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY; + ifobj->bind_flags = XDP_USE_NEED_WAKEUP; + if (mode == TEST_MODE_ZC) + ifobj->bind_flags |= XDP_ZEROCOPY; + else + ifobj->bind_flags |= XDP_COPY; } + test->mode = mode; __test_spec_init(test, ifobj_tx, ifobj_rx); } @@ -524,16 +599,17 @@ static void pkt_stream_delete(struct pkt_stream *pkt_stream) static void pkt_stream_restore_default(struct test_spec *test) { struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream; + struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream; - if (tx_pkt_stream != test->pkt_stream_default) { + if (tx_pkt_stream != test->tx_pkt_stream_default) { pkt_stream_delete(test->ifobj_tx->pkt_stream); - test->ifobj_tx->pkt_stream = test->pkt_stream_default; + test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default; } - if (test->ifobj_rx->pkt_stream != test->pkt_stream_default && - test->ifobj_rx->pkt_stream != tx_pkt_stream) + if (rx_pkt_stream != test->rx_pkt_stream_default) { pkt_stream_delete(test->ifobj_rx->pkt_stream); - test->ifobj_rx->pkt_stream = test->pkt_stream_default; + test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default; + } } static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) @@ -556,7 +632,7 @@ static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len) { - pkt->addr = addr; + pkt->addr = addr + umem->base_addr; pkt->len = len; if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom) pkt->valid = false; @@ -595,22 +671,29 @@ static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len); test->ifobj_tx->pkt_stream = pkt_stream; + pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len); test->ifobj_rx->pkt_stream = pkt_stream; } -static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) +static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, + int offset) { - struct xsk_umem_info *umem = test->ifobj_tx->umem; + struct xsk_umem_info *umem = ifobj->umem; struct pkt_stream *pkt_stream; u32 i; - pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default); - for (i = 1; i < test->pkt_stream_default->nb_pkts; i += 2) + pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream); + for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2) pkt_set(umem, &pkt_stream->pkts[i], (i % umem->num_frames) * umem->frame_size + offset, pkt_len); - test->ifobj_tx->pkt_stream = pkt_stream; - test->ifobj_rx->pkt_stream = pkt_stream; + ifobj->pkt_stream = pkt_stream; +} + +static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) +{ + __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); + __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); } static void pkt_stream_receive_half(struct test_spec *test) @@ -652,7 +735,8 @@ static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) return pkt; } -static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) +static void __pkt_stream_generate_custom(struct ifobject *ifobj, + struct pkt *pkts, u32 nb_pkts) { struct pkt_stream *pkt_stream; u32 i; @@ -661,15 +745,20 @@ static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, if (!pkt_stream) exit_with_error(ENOMEM); - test->ifobj_tx->pkt_stream = pkt_stream; - test->ifobj_rx->pkt_stream = pkt_stream; - for (i = 0; i < nb_pkts; i++) { - pkt_stream->pkts[i].addr = pkts[i].addr; + pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr; pkt_stream->pkts[i].len = pkts[i].len; pkt_stream->pkts[i].payload = i; pkt_stream->pkts[i].valid = pkts[i].valid; } + + ifobj->pkt_stream = pkt_stream; +} + +static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) +{ + __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts); + __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts); } static void pkt_dump(void *pkt, u32 len) @@ -817,12 +906,13 @@ static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) return TEST_PASS; } -static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds) +static int receive_pkts(struct test_spec *test, struct pollfd *fds) { - struct timeval tv_end, tv_now, tv_timeout = {RECV_TMOUT, 0}; + struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; + struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream; u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0; - struct pkt_stream *pkt_stream = ifobj->pkt_stream; - struct xsk_socket_info *xsk = ifobj->xsk; + struct xsk_socket_info *xsk = test->ifobj_rx->xsk; + struct ifobject *ifobj = test->ifobj_rx; struct xsk_umem_info *umem = xsk->umem; struct pkt *pkt; int ret; @@ -843,17 +933,28 @@ static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds) } kick_rx(xsk); + if (ifobj->use_poll) { + ret = poll(fds, 1, POLL_TMOUT); + if (ret < 0) + exit_with_error(-ret); + + if (!ret) { + if (!is_umem_valid(test->ifobj_tx)) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); + return TEST_FAILURE; - rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); - if (!rcvd) { - if (xsk_ring_prod__needs_wakeup(&umem->fq)) { - ret = poll(fds, 1, POLL_TMOUT); - if (ret < 0) - exit_with_error(-ret); } - continue; + + if (!(fds->revents & POLLIN)) + continue; } + rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); + if (!rcvd) + continue; + if (ifobj->use_fill_ring) { ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); while (ret != rcvd) { @@ -900,13 +1001,35 @@ static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds) return TEST_PASS; } -static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb) +static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds, + bool timeout) { struct xsk_socket_info *xsk = ifobject->xsk; - u32 i, idx, valid_pkts = 0; + bool use_poll = ifobject->use_poll; + u32 i, idx = 0, ret, valid_pkts = 0; + + while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) { + if (use_poll) { + ret = poll(fds, 1, POLL_TMOUT); + if (timeout) { + if (ret < 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, ret); + return TEST_FAILURE; + } + if (ret == 0) + return TEST_PASS; + break; + } + if (ret <= 0) { + ksft_print_msg("ERROR: [%s] Poll error %d\n", + __func__, ret); + return TEST_FAILURE; + } + } - while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) complete_pkts(xsk, BATCH_SIZE); + } for (i = 0; i < BATCH_SIZE; i++) { struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); @@ -933,11 +1056,27 @@ static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb) xsk_ring_prod__submit(&xsk->tx, i); xsk->outstanding_tx += valid_pkts; - if (complete_pkts(xsk, i)) - return TEST_FAILURE; - usleep(10); - return TEST_PASS; + if (use_poll) { + ret = poll(fds, 1, POLL_TMOUT); + if (ret <= 0) { + if (ret == 0 && timeout) + return TEST_PASS; + + ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); + return TEST_FAILURE; + } + } + + if (!timeout) { + if (complete_pkts(xsk, i)) + return TEST_FAILURE; + + usleep(10); + return TEST_PASS; + } + + return TEST_CONTINUE; } static void wait_for_tx_completion(struct xsk_socket_info *xsk) @@ -948,29 +1087,19 @@ static void wait_for_tx_completion(struct xsk_socket_info *xsk) static int send_pkts(struct test_spec *test, struct ifobject *ifobject) { + bool timeout = !is_umem_valid(test->ifobj_rx); struct pollfd fds = { }; - u32 pkt_cnt = 0; + u32 pkt_cnt = 0, ret; fds.fd = xsk_socket__fd(ifobject->xsk->xsk); fds.events = POLLOUT; while (pkt_cnt < ifobject->pkt_stream->nb_pkts) { - int err; - - if (ifobject->use_poll) { - int ret; - - ret = poll(&fds, 1, POLL_TMOUT); - if (ret <= 0) - continue; - - if (!(fds.revents & POLLOUT)) - continue; - } - - err = __send_pkts(ifobject, &pkt_cnt); - if (err || test->fail) + ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout); + if ((ret || test->fail) && !timeout) return TEST_FAILURE; + else if (ret == TEST_PASS && timeout) + return ret; } wait_for_tx_completion(ifobject->xsk); @@ -1081,6 +1210,70 @@ static int validate_tx_invalid_descs(struct ifobject *ifobject) return TEST_PASS; } +static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, + struct xsk_umem_info *umem, bool tx) +{ + int i, ret; + + for (i = 0; i < test->nb_sockets; i++) { + bool shared = (ifobject->shared_umem && tx) ? true : !!i; + u32 ctr = 0; + + while (ctr++ < SOCK_RECONF_CTR) { + ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, + ifobject, shared); + if (!ret) + break; + + /* Retry if it fails as xsk_socket__create() is asynchronous */ + if (ctr >= SOCK_RECONF_CTR) + exit_with_error(-ret); + usleep(USLEEP_MAX); + } + if (ifobject->busy_poll) + enable_busy_poll(&ifobject->xsk_arr[i]); + } +} + +static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) +{ + xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); + ifobject->xsk = &ifobject->xsk_arr[0]; + ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd; + memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); +} + +static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream) +{ + u32 idx = 0, i, buffers_to_fill; + int ret; + + if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) + buffers_to_fill = umem->num_frames; + else + buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; + + ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); + if (ret != buffers_to_fill) + exit_with_error(ENOSPC); + for (i = 0; i < buffers_to_fill; i++) { + u64 addr; + + if (pkt_stream->use_addr_for_fill) { + struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i); + + if (!pkt) + break; + addr = pkt->addr; + } else { + addr = i * umem->frame_size; + } + + *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; + } + xsk_ring_prod__submit(&umem->fq, buffers_to_fill); +} + static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) { u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; @@ -1088,13 +1281,15 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) LIBBPF_OPTS(bpf_xdp_query_opts, opts); int ret, ifindex; void *bufs; - u32 i; ifobject->ns_fd = switch_namespace(ifobject->nsname); if (ifobject->umem->unaligned_mode) mmap_flags |= MAP_HUGETLB; + if (ifobject->shared_umem) + umem_sz *= 2; + bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); if (bufs == MAP_FAILED) exit_with_error(errno); @@ -1103,24 +1298,9 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) if (ret) exit_with_error(-ret); - for (i = 0; i < test->nb_sockets; i++) { - u32 ctr = 0; - - while (ctr++ < SOCK_RECONF_CTR) { - ret = xsk_configure_socket(&ifobject->xsk_arr[i], ifobject->umem, - ifobject, !!i); - if (!ret) - break; + xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream); - /* Retry if it fails as xsk_socket__create() is asynchronous */ - if (ctr >= SOCK_RECONF_CTR) - exit_with_error(-ret); - usleep(USLEEP_MAX); - } - - if (ifobject->busy_poll) - enable_busy_poll(&ifobject->xsk_arr[i]); - } + xsk_configure_socket(test, ifobject, ifobject->umem, false); ifobject->xsk = &ifobject->xsk_arr[0]; @@ -1156,22 +1336,18 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) exit_with_error(-ret); } -static void testapp_cleanup_xsk_res(struct ifobject *ifobj) -{ - print_verbose("Destroying socket\n"); - xsk_socket__delete(ifobj->xsk->xsk); - munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size); - xsk_umem__delete(ifobj->umem->umem); -} - static void *worker_testapp_validate_tx(void *arg) { struct test_spec *test = (struct test_spec *)arg; struct ifobject *ifobject = test->ifobj_tx; int err; - if (test->current_step == 1) - thread_common_ops(test, ifobject); + if (test->current_step == 1) { + if (!ifobject->shared_umem) + thread_common_ops(test, ifobject); + else + thread_common_ops_tx(test, ifobject); + } print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts, ifobject->ifname); @@ -1182,60 +1358,30 @@ static void *worker_testapp_validate_tx(void *arg) if (err) report_failure(test); - if (test->total_steps == test->current_step || err) - testapp_cleanup_xsk_res(ifobject); pthread_exit(NULL); } -static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream) -{ - u32 idx = 0, i, buffers_to_fill; - int ret; - - if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) - buffers_to_fill = umem->num_frames; - else - buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS; - - ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); - if (ret != buffers_to_fill) - exit_with_error(ENOSPC); - for (i = 0; i < buffers_to_fill; i++) { - u64 addr; - - if (pkt_stream->use_addr_for_fill) { - struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i); - - if (!pkt) - break; - addr = pkt->addr; - } else { - addr = i * umem->frame_size; - } - - *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; - } - xsk_ring_prod__submit(&umem->fq, buffers_to_fill); -} - static void *worker_testapp_validate_rx(void *arg) { struct test_spec *test = (struct test_spec *)arg; struct ifobject *ifobject = test->ifobj_rx; struct pollfd fds = { }; + int id = 0; int err; - if (test->current_step == 1) + if (test->current_step == 1) { thread_common_ops(test, ifobject); - - xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream); + } else { + bpf_map_delete_elem(ifobject->xsk_map_fd, &id); + xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd); + } fds.fd = xsk_socket__fd(ifobject->xsk->xsk); fds.events = POLLIN; pthread_barrier_wait(&barr); - err = receive_pkts(ifobject, &fds); + err = receive_pkts(test, &fds); if (!err && ifobject->validation_func) err = ifobject->validation_func(ifobject); @@ -1246,11 +1392,66 @@ static void *worker_testapp_validate_rx(void *arg) pthread_mutex_unlock(&pacing_mutex); } - if (test->total_steps == test->current_step || err) - testapp_cleanup_xsk_res(ifobject); pthread_exit(NULL); } +static void testapp_clean_xsk_umem(struct ifobject *ifobj) +{ + u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; + + if (ifobj->shared_umem) + umem_sz *= 2; + + xsk_umem__delete(ifobj->umem->umem); + munmap(ifobj->umem->buffer, umem_sz); +} + +static void handler(int signum) +{ + pthread_exit(NULL); +} + +static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj, + enum test_type type) +{ + bool old_shared_umem = ifobj->shared_umem; + pthread_t t0; + + if (pthread_barrier_init(&barr, NULL, 2)) + exit_with_error(errno); + + test->current_step++; + if (type == TEST_TYPE_POLL_RXQ_TMOUT) + pkt_stream_reset(ifobj->pkt_stream); + pkts_in_flight = 0; + + test->ifobj_rx->shared_umem = false; + test->ifobj_tx->shared_umem = false; + + signal(SIGUSR1, handler); + /* Spawn thread */ + pthread_create(&t0, NULL, ifobj->func_ptr, test); + + if (type != TEST_TYPE_POLL_TXQ_TMOUT) + pthread_barrier_wait(&barr); + + if (pthread_barrier_destroy(&barr)) + exit_with_error(errno); + + pthread_kill(t0, SIGUSR1); + pthread_join(t0, NULL); + + if (test->total_steps == test->current_step || test->fail) { + xsk_socket__delete(ifobj->xsk->xsk); + testapp_clean_xsk_umem(ifobj); + } + + test->ifobj_rx->shared_umem = old_shared_umem; + test->ifobj_tx->shared_umem = old_shared_umem; + + return !!test->fail; +} + static int testapp_validate_traffic(struct test_spec *test) { struct ifobject *ifobj_tx = test->ifobj_tx; @@ -1277,6 +1478,14 @@ static int testapp_validate_traffic(struct test_spec *test) pthread_join(t1, NULL); pthread_join(t0, NULL); + if (test->total_steps == test->current_step || test->fail) { + xsk_socket__delete(ifobj_tx->xsk->xsk); + xsk_socket__delete(ifobj_rx->xsk->xsk); + testapp_clean_xsk_umem(ifobj_rx); + if (!ifobj_tx->shared_umem) + testapp_clean_xsk_umem(ifobj_tx); + } + return !!test->fail; } @@ -1356,9 +1565,9 @@ static void testapp_headroom(struct test_spec *test) static void testapp_stats_rx_dropped(struct test_spec *test) { test_spec_set_name(test, "STAT_RX_DROPPED"); + pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; - pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); pkt_stream_receive_half(test); test->ifobj_rx->validation_func = validate_rx_dropped; testapp_validate_traffic(test); @@ -1481,6 +1690,11 @@ static void testapp_invalid_desc(struct test_spec *test) pkts[7].valid = false; } + if (test->ifobj_tx->shared_umem) { + pkts[4].addr += UMEM_SIZE; + pkts[5].addr += UMEM_SIZE; + } + pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); testapp_validate_traffic(test); pkt_stream_restore_default(test); @@ -1511,6 +1725,10 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ { switch (type) { case TEST_TYPE_STATS_RX_DROPPED: + if (mode == TEST_MODE_ZC) { + ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); + return; + } testapp_stats_rx_dropped(test); break; case TEST_TYPE_STATS_TX_INVALID_DESCS: @@ -1548,12 +1766,30 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ pkt_stream_restore_default(test); break; - case TEST_TYPE_POLL: - test->ifobj_tx->use_poll = true; + case TEST_TYPE_RX_POLL: test->ifobj_rx->use_poll = true; - test_spec_set_name(test, "POLL"); + test_spec_set_name(test, "POLL_RX"); testapp_validate_traffic(test); break; + case TEST_TYPE_TX_POLL: + test->ifobj_tx->use_poll = true; + test_spec_set_name(test, "POLL_TX"); + testapp_validate_traffic(test); + break; + case TEST_TYPE_POLL_TXQ_TMOUT: + test_spec_set_name(test, "POLL_TXQ_FULL"); + test->ifobj_tx->use_poll = true; + /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ + test->ifobj_tx->umem->frame_size = 2048; + pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); + testapp_validate_traffic_single_thread(test, test->ifobj_tx, type); + pkt_stream_restore_default(test); + break; + case TEST_TYPE_POLL_RXQ_TMOUT: + test_spec_set_name(test, "POLL_RXQ_EMPTY"); + test->ifobj_rx->use_poll = true; + testapp_validate_traffic_single_thread(test, test->ifobj_rx, type); + break; case TEST_TYPE_ALIGNED_INV_DESC: test_spec_set_name(test, "ALIGNED_INV_DESC"); testapp_invalid_desc(test); @@ -1606,6 +1842,8 @@ static struct ifobject *ifobject_create(void) if (!ifobj->umem) goto out_umem; + ifobj->ns_fd = -1; + return ifobj; out_umem: @@ -1617,17 +1855,51 @@ out_xsk_arr: static void ifobject_delete(struct ifobject *ifobj) { + if (ifobj->ns_fd != -1) + close(ifobj->ns_fd); free(ifobj->umem); free(ifobj->xsk_arr); free(ifobj); } +static bool is_xdp_supported(struct ifobject *ifobject) +{ + int flags = XDP_FLAGS_DRV_MODE; + + LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); + struct bpf_insn insns[2] = { + BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), + BPF_EXIT_INSN() + }; + int ifindex = if_nametoindex(ifobject->ifname); + int prog_fd, insn_cnt = ARRAY_SIZE(insns); + int err; + + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); + if (prog_fd < 0) + return false; + + err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); + if (err) { + close(prog_fd); + return false; + } + + bpf_xdp_detach(ifindex, flags, NULL); + close(prog_fd); + + return true; +} + int main(int argc, char **argv) { - struct pkt_stream *pkt_stream_default; + struct pkt_stream *rx_pkt_stream_default; + struct pkt_stream *tx_pkt_stream_default; struct ifobject *ifobj_tx, *ifobj_rx; + int modes = TEST_MODE_SKB + 1; u32 i, j, failed_tests = 0; struct test_spec test; + bool shared_umem; /* Use libbpf 1.0 API mode */ libbpf_set_strict_mode(LIBBPF_STRICT_ALL); @@ -1642,6 +1914,10 @@ int main(int argc, char **argv) setlocale(LC_ALL, ""); parse_command_line(ifobj_tx, ifobj_rx, argc, argv); + shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname); + + ifobj_tx->shared_umem = shared_umem; + ifobj_rx->shared_umem = shared_umem; if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) { usage(basename(argv[0])); @@ -1653,15 +1929,23 @@ int main(int argc, char **argv) init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, worker_testapp_validate_rx); + if (is_xdp_supported(ifobj_tx)) { + modes++; + if (ifobj_zc_avail(ifobj_tx)) + modes++; + } + test_spec_init(&test, ifobj_tx, ifobj_rx, 0); - pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE); - if (!pkt_stream_default) + tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE); + rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE); + if (!tx_pkt_stream_default || !rx_pkt_stream_default) exit_with_error(ENOMEM); - test.pkt_stream_default = pkt_stream_default; + test.tx_pkt_stream_default = tx_pkt_stream_default; + test.rx_pkt_stream_default = rx_pkt_stream_default; - ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX); + ksft_set_plan(modes * TEST_TYPE_MAX); - for (i = 0; i < TEST_MODE_MAX; i++) + for (i = 0; i < modes; i++) for (j = 0; j < TEST_TYPE_MAX; j++) { test_spec_init(&test, ifobj_tx, ifobj_rx, i); run_pkt_test(&test, i, j); @@ -1671,7 +1955,8 @@ int main(int argc, char **argv) failed_tests++; } - pkt_stream_delete(pkt_stream_default); + pkt_stream_delete(tx_pkt_stream_default); + pkt_stream_delete(rx_pkt_stream_default); ifobject_delete(ifobj_tx); ifobject_delete(ifobj_rx); diff --git a/tools/testing/selftests/bpf/xskxceiver.h b/tools/testing/selftests/bpf/xskxceiver.h index 3d17053f98e53ccb5296a08423d44257e6496976..edb76d2def9fe554007e08ea37b2480a5539bced 100644 --- a/tools/testing/selftests/bpf/xskxceiver.h +++ b/tools/testing/selftests/bpf/xskxceiver.h @@ -27,9 +27,10 @@ #define TEST_PASS 0 #define TEST_FAILURE -1 +#define TEST_CONTINUE 1 #define MAX_INTERFACES 2 -#define MAX_INTERFACE_NAME_CHARS 7 -#define MAX_INTERFACES_NAMESPACE_CHARS 10 +#define MAX_INTERFACE_NAME_CHARS 16 +#define MAX_INTERFACES_NAMESPACE_CHARS 16 #define MAX_SOCKETS 2 #define MAX_TEST_NAME_SIZE 32 #define MAX_TEARDOWN_ITER 10 @@ -48,7 +49,7 @@ #define SOCK_RECONF_CTR 10 #define BATCH_SIZE 64 #define POLL_TMOUT 1000 -#define RECV_TMOUT 3 +#define THREAD_TMOUT 3 #define DEFAULT_PKT_CNT (4 * 1024) #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) #define UMEM_SIZE (DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE) @@ -61,6 +62,7 @@ enum test_mode { TEST_MODE_SKB, TEST_MODE_DRV, + TEST_MODE_ZC, TEST_MODE_MAX }; @@ -68,7 +70,10 @@ enum test_type { TEST_TYPE_RUN_TO_COMPLETION, TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME, TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT, - TEST_TYPE_POLL, + TEST_TYPE_RX_POLL, + TEST_TYPE_TX_POLL, + TEST_TYPE_POLL_RXQ_TMOUT, + TEST_TYPE_POLL_TXQ_TMOUT, TEST_TYPE_UNALIGNED, TEST_TYPE_ALIGNED_INV_DESC, TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME, @@ -95,6 +100,7 @@ struct xsk_umem_info { u32 frame_headroom; void *buffer; u32 frame_size; + u32 base_addr; bool unaligned_mode; }; @@ -148,6 +154,7 @@ struct ifobject { bool busy_poll; bool use_fill_ring; bool release_rx; + bool shared_umem; u8 dst_mac[ETH_ALEN]; u8 src_mac[ETH_ALEN]; }; @@ -155,11 +162,13 @@ struct ifobject { struct test_spec { struct ifobject *ifobj_tx; struct ifobject *ifobj_rx; - struct pkt_stream *pkt_stream_default; + struct pkt_stream *tx_pkt_stream_default; + struct pkt_stream *rx_pkt_stream_default; u16 total_steps; u16 current_step; u16 nb_sockets; bool fail; + enum test_mode mode; char name[MAX_TEST_NAME_SIZE]; }; diff --git a/tools/testing/selftests/cgroup/.gitignore b/tools/testing/selftests/cgroup/.gitignore index 306ee1b01e72f4abb2af8a501c830fb4c87dd9e9..c4a57e69f749e32f3856aca526f306ff2b9920fb 100644 --- a/tools/testing/selftests/cgroup/.gitignore +++ b/tools/testing/selftests/cgroup/.gitignore @@ -5,3 +5,4 @@ test_freezer test_kmem test_kill test_cpu +wait_inotify diff --git a/tools/testing/selftests/cgroup/Makefile b/tools/testing/selftests/cgroup/Makefile index 478217cc1371469302ebb049cf6efdb01869cc6a..3d263747d2ad0b4d9de0d81aa2f0438e77e3dd5f 100644 --- a/tools/testing/selftests/cgroup/Makefile +++ b/tools/testing/selftests/cgroup/Makefile @@ -1,10 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 CFLAGS += -Wall -pthread -all: +all: ${HELPER_PROGS} TEST_FILES := with_stress.sh -TEST_PROGS := test_stress.sh +TEST_PROGS := test_stress.sh test_cpuset_prs.sh +TEST_GEN_FILES := wait_inotify TEST_GEN_PROGS = test_memcontrol TEST_GEN_PROGS += test_kmem TEST_GEN_PROGS += test_core diff --git a/tools/testing/selftests/cgroup/config b/tools/testing/selftests/cgroup/config index 84fe884fad8674eb650bdfb66c8306312e2f9a93..97d549ee894fa197687e0da293b2c007941f39ae 100644 --- a/tools/testing/selftests/cgroup/config +++ b/tools/testing/selftests/cgroup/config @@ -4,5 +4,4 @@ CONFIG_CGROUP_FREEZER=y CONFIG_CGROUP_SCHED=y CONFIG_MEMCG=y CONFIG_MEMCG_KMEM=y -CONFIG_MEMCG_SWAP=y CONFIG_PAGE_COUNTER=y diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh new file mode 100755 index 0000000000000000000000000000000000000000..526d2c42d8706bc2fb283e757a23cd1d5b97b0cf --- /dev/null +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -0,0 +1,674 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test for cpuset v2 partition root state (PRS) +# +# The sched verbose flag is set, if available, so that the console log +# can be examined for the correct setting of scheduling domain. +# + +skip_test() { + echo "$1" + echo "Test SKIPPED" + exit 0 +} + +[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!" + +# Set sched verbose flag, if available +[[ -d /sys/kernel/debug/sched ]] && echo Y > /sys/kernel/debug/sched/verbose + +# Get wait_inotify location +WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify + +# Find cgroup v2 mount point +CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}') +[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!" + +CPUS=$(lscpu | grep "^CPU(s)" | sed -e "s/.*:[[:space:]]*//") +[[ $CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!" + +# Set verbose flag and delay factor +PROG=$1 +VERBOSE= +DELAY_FACTOR=1 +while [[ "$1" = -* ]] +do + case "$1" in + -v) VERBOSE=1 + break + ;; + -d) DELAY_FACTOR=$2 + shift + break + ;; + *) echo "Usage: $PROG [-v] [-d <delay-factor>" + exit + ;; + esac + shift +done + +cd $CGROUP2 +echo +cpuset > cgroup.subtree_control +[[ -d test ]] || mkdir test +cd test + +# Pause in ms +pause() +{ + DELAY=$1 + LOOP=0 + while [[ $LOOP -lt $DELAY_FACTOR ]] + do + sleep $DELAY + ((LOOP++)) + done + return 0 +} + +console_msg() +{ + MSG=$1 + echo "$MSG" + echo "" > /dev/console + echo "$MSG" > /dev/console + pause 0.01 +} + +test_partition() +{ + EXPECTED_VAL=$1 + echo $EXPECTED_VAL > cpuset.cpus.partition + [[ $? -eq 0 ]] || exit 1 + ACTUAL_VAL=$(cat cpuset.cpus.partition) + [[ $ACTUAL_VAL != $EXPECTED_VAL ]] && { + echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $EXPECTED_VAL" + echo "Test FAILED" + exit 1 + } +} + +test_effective_cpus() +{ + EXPECTED_VAL=$1 + ACTUAL_VAL=$(cat cpuset.cpus.effective) + [[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && { + echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$EXPECTED_VAL'" + echo "Test FAILED" + exit 1 + } +} + +# Adding current process to cgroup.procs as a test +test_add_proc() +{ + OUTSTR="$1" + ERRMSG=$((echo $$ > cgroup.procs) |& cat) + echo $ERRMSG | grep -q "$OUTSTR" + [[ $? -ne 0 ]] && { + echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'" + echo "Test FAILED" + exit 1 + } + echo $$ > $CGROUP2/cgroup.procs # Move out the task +} + +# +# Testing the new "isolated" partition root type +# +test_isolated() +{ + echo 2-3 > cpuset.cpus + TYPE=$(cat cpuset.cpus.partition) + [[ $TYPE = member ]] || echo member > cpuset.cpus.partition + + console_msg "Change from member to root" + test_partition root + + console_msg "Change from root to isolated" + test_partition isolated + + console_msg "Change from isolated to member" + test_partition member + + console_msg "Change from member to isolated" + test_partition isolated + + console_msg "Change from isolated to root" + test_partition root + + console_msg "Change from root to member" + test_partition member + + # + # Testing partition root with no cpu + # + console_msg "Distribute all cpus to child partition" + echo +cpuset > cgroup.subtree_control + test_partition root + + mkdir A1 + cd A1 + echo 2-3 > cpuset.cpus + test_partition root + test_effective_cpus 2-3 + cd .. + test_effective_cpus "" + + console_msg "Moving task to partition test" + test_add_proc "No space left" + cd A1 + test_add_proc "" + cd .. + + console_msg "Shrink and expand child partition" + cd A1 + echo 2 > cpuset.cpus + cd .. + test_effective_cpus 3 + cd A1 + echo 2-3 > cpuset.cpus + cd .. + test_effective_cpus "" + + # Cleaning up + console_msg "Cleaning up" + echo $$ > $CGROUP2/cgroup.procs + [[ -d A1 ]] && rmdir A1 +} + +# +# Cpuset controller state transition test matrix. +# +# Cgroup test hierarchy +# +# test -- A1 -- A2 -- A3 +# \- B1 +# +# P<v> = set cpus.partition (0:member, 1:root, 2:isolated, -1:root invalid) +# C<l> = add cpu-list +# S<p> = use prefix in subtree_control +# T = put a task into cgroup +# O<c>-<v> = Write <v> to CPU online file of <c> +# +SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1" +TEST_MATRIX=( + # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate + # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ + " S+ C0-1 . . C2-3 S+ C4-5 . . 0 A2:0-1" + " S+ C0-1 . . C2-3 P1 . . . 0 " + " S+ C0-1 . . C2-3 P1:S+ C0-1:P1 . . 0 " + " S+ C0-1 . . C2-3 P1:S+ C1:P1 . . 0 " + " S+ C0-1:S+ . . C2-3 . . . P1 0 " + " S+ C0-1:P1 . . C2-3 S+ C1 . . 0 " + " S+ C0-1:P1 . . C2-3 S+ C1:P1 . . 0 " + " S+ C0-1:P1 . . C2-3 S+ C1:P1 . P1 0 " + " S+ C0-1:P1 . . C2-3 C4-5 . . . 0 A1:4-5" + " S+ C0-1:P1 . . C2-3 S+:C4-5 . . . 0 A1:4-5" + " S+ C0-1 . . C2-3:P1 . . . C2 0 " + " S+ C0-1 . . C2-3:P1 . . . C4-5 0 B1:4-5" + " S+ C0-3:P1:S+ C2-3:P1 . . . . . . 0 A1:0-1,A2:2-3" + " S+ C0-3:P1:S+ C2-3:P1 . . C1-3 . . . 0 A1:1,A2:2-3" + " S+ C2-3:P1:S+ C3:P1 . . C3 . . . 0 A1:,A2:3 A1:P1,A2:P1" + " S+ C2-3:P1:S+ C3:P1 . . C3 P0 . . 0 A1:3,A2:3 A1:P1,A2:P0" + " S+ C2-3:P1:S+ C2:P1 . . C2-4 . . . 0 A1:3-4,A2:2" + " S+ C2-3:P1:S+ C3:P1 . . C3 . . C0-2 0 A1:,B1:0-2 A1:P1,A2:P1" + " S+ $SETUP_A123_PARTITIONS . C2-3 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + + # CPU offlining cases: + " S+ C0-1 . . C2-3 S+ C4-5 . O2-0 0 A1:0-1,B1:3" + " S+ C0-3:P1:S+ C2-3:P1 . . O2-0 . . . 0 A1:0-1,A2:3" + " S+ C0-3:P1:S+ C2-3:P1 . . O2-0 O2-1 . . 0 A1:0-1,A2:2-3" + " S+ C0-3:P1:S+ C2-3:P1 . . O1-0 . . . 0 A1:0,A2:2-3" + " S+ C0-3:P1:S+ C2-3:P1 . . O1-0 O1-1 . . 0 A1:0-1,A2:2-3" + " S+ C2-3:P1:S+ C3:P1 . . O3-0 O3-1 . . 0 A1:2,A2:3 A1:P1,A2:P1" + " S+ C2-3:P1:S+ C3:P2 . . O3-0 O3-1 . . 0 A1:2,A2:3 A1:P1,A2:P2" + " S+ C2-3:P1:S+ C3:P1 . . O2-0 O2-1 . . 0 A1:2,A2:3 A1:P1,A2:P1" + " S+ C2-3:P1:S+ C3:P2 . . O2-0 O2-1 . . 0 A1:2,A2:3 A1:P1,A2:P2" + " S+ C2-3:P1:S+ C3:P1 . . O2-0 . . . 0 A1:,A2:3 A1:P1,A2:P1" + " S+ C2-3:P1:S+ C3:P1 . . O3-0 . . . 0 A1:2,A2: A1:P1,A2:P1" + " S+ C2-3:P1:S+ C3:P1 . . T:O2-0 . . . 0 A1:3,A2:3 A1:P1,A2:P-1" + " S+ C2-3:P1:S+ C3:P1 . . . T:O3-0 . . 0 A1:2,A2:2 A1:P1,A2:P-1" + " S+ $SETUP_A123_PARTITIONS . O1-0 . . . 0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . O2-0 . . . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . O3-0 . . . 0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . T:O1-0 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" + " S+ $SETUP_A123_PARTITIONS . . T:O2-0 . . 0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1" + " S+ $SETUP_A123_PARTITIONS . . . T:O3-0 . 0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1" + " S+ $SETUP_A123_PARTITIONS . T:O1-0 O1-1 . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . . T:O2-0 O2-1 . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . . . T:O3-0 O3-1 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . T:O1-0 O2-0 O1-1 . 0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1" + " S+ $SETUP_A123_PARTITIONS . T:O1-0 O2-0 O2-1 . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" + + # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate + # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ + # + # Incorrect change to cpuset.cpus invalidates partition root + # + # Adding CPUs to partition root that are not in parent's + # cpuset.cpus is allowed, but those extra CPUs are ignored. + " S+ C2-3:P1:S+ C3:P1 . . . C2-4 . . 0 A1:,A2:2-3 A1:P1,A2:P1" + + # Taking away all CPUs from parent or itself if there are tasks + # will make the partition invalid. + " S+ C2-3:P1:S+ C3:P1 . . T C2-3 . . 0 A1:2-3,A2:2-3 A1:P1,A2:P-1" + " S+ $SETUP_A123_PARTITIONS . T:C2-3 . . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1" + " S+ $SETUP_A123_PARTITIONS . T:C2-3:C1-3 . . . 0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1" + + # Changing a partition root to member makes child partitions invalid + " S+ C2-3:P1:S+ C3:P1 . . P0 . . . 0 A1:2-3,A2:3 A1:P0,A2:P-1" + " S+ $SETUP_A123_PARTITIONS . C2-3 P0 . . 0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1" + + # cpuset.cpus can contains cpus not in parent's cpuset.cpus as long + # as they overlap. + " S+ C2-3:P1:S+ . . . . C3-4:P1 . . 0 A1:2,A2:3 A1:P1,A2:P1" + + # Deletion of CPUs distributed to child cgroup is allowed. + " S+ C0-1:P1:S+ C1 . C2-3 C4-5 . . . 0 A1:4-5,A2:4-5" + + # To become a valid partition root, cpuset.cpus must overlap parent's + # cpuset.cpus. + " S+ C0-1:P1 . . C2-3 S+ C4-5:P1 . . 0 A1:0-1,A2:0-1 A1:P1,A2:P-1" + + # Enabling partition with child cpusets is allowed + " S+ C0-1:S+ C1 . C2-3 P1 . . . 0 A1:0-1,A2:1 A1:P1" + + # A partition root with non-partition root parent is invalid, but it + # can be made valid if its parent becomes a partition root too. + " S+ C0-1:S+ C1 . C2-3 . P2 . . 0 A1:0-1,A2:1 A1:P0,A2:P-2" + " S+ C0-1:S+ C1:P2 . C2-3 P1 . . . 0 A1:0,A2:1 A1:P1,A2:P2" + + # A non-exclusive cpuset.cpus change will invalidate partition and its siblings + " S+ C0-1:P1 . . C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P0" + " S+ C0-1:P1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P-1,B1:P-1" + " S+ C0-1 . . P1:C2-3 C0-2 . . . 0 A1:0-2,B1:2-3 A1:P0,B1:P-1" + + # test old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate + # ---- ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ + # Failure cases: + + # A task cannot be added to a partition with no cpu + " S+ C2-3:P1:S+ C3:P1 . . O2-0:T . . . 1 A1:,A2:3 A1:P1,A2:P1" +) + +# +# Write to the cpu online file +# $1 - <c>-<v> where <c> = cpu number, <v> value to be written +# +write_cpu_online() +{ + CPU=${1%-*} + VAL=${1#*-} + CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online + if [[ $VAL -eq 0 ]] + then + OFFLINE_CPUS="$OFFLINE_CPUS $CPU" + else + [[ -n "$OFFLINE_CPUS" ]] && { + OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\ + sort | uniq -u) + } + fi + echo $VAL > $CPUFILE + pause 0.01 +} + +# +# Set controller state +# $1 - cgroup directory +# $2 - state +# $3 - showerr +# +# The presence of ":" in state means transition from one to the next. +# +set_ctrl_state() +{ + TMPMSG=/tmp/.msg_$$ + CGRP=$1 + STATE=$2 + SHOWERR=${3}${VERBOSE} + CTRL=${CTRL:=$CONTROLLER} + HASERR=0 + REDIRECT="2> $TMPMSG" + [[ -z "$STATE" || "$STATE" = '.' ]] && return 0 + + rm -f $TMPMSG + for CMD in $(echo $STATE | sed -e "s/:/ /g") + do + TFILE=$CGRP/cgroup.procs + SFILE=$CGRP/cgroup.subtree_control + PFILE=$CGRP/cpuset.cpus.partition + CFILE=$CGRP/cpuset.cpus + S=$(expr substr $CMD 1 1) + if [[ $S = S ]] + then + PREFIX=${CMD#?} + COMM="echo ${PREFIX}${CTRL} > $SFILE" + eval $COMM $REDIRECT + elif [[ $S = C ]] + then + CPUS=${CMD#?} + COMM="echo $CPUS > $CFILE" + eval $COMM $REDIRECT + elif [[ $S = P ]] + then + VAL=${CMD#?} + case $VAL in + 0) VAL=member + ;; + 1) VAL=root + ;; + 2) VAL=isolated + ;; + *) + echo "Invalid partition state - $VAL" + exit 1 + ;; + esac + COMM="echo $VAL > $PFILE" + eval $COMM $REDIRECT + elif [[ $S = O ]] + then + VAL=${CMD#?} + write_cpu_online $VAL + elif [[ $S = T ]] + then + COMM="echo 0 > $TFILE" + eval $COMM $REDIRECT + fi + RET=$? + [[ $RET -ne 0 ]] && { + [[ -n "$SHOWERR" ]] && { + echo "$COMM" + cat $TMPMSG + } + HASERR=1 + } + pause 0.01 + rm -f $TMPMSG + done + return $HASERR +} + +set_ctrl_state_noerr() +{ + CGRP=$1 + STATE=$2 + [[ -d $CGRP ]] || mkdir $CGRP + set_ctrl_state $CGRP $STATE 1 + [[ $? -ne 0 ]] && { + echo "ERROR: Failed to set $2 to cgroup $1!" + exit 1 + } +} + +online_cpus() +{ + [[ -n "OFFLINE_CPUS" ]] && { + for C in $OFFLINE_CPUS + do + write_cpu_online ${C}-1 + done + } +} + +# +# Return 1 if the list of effective cpus isn't the same as the initial list. +# +reset_cgroup_states() +{ + echo 0 > $CGROUP2/cgroup.procs + online_cpus + rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1 + set_ctrl_state . S- + pause 0.01 +} + +dump_states() +{ + for DIR in A1 A1/A2 A1/A2/A3 B1 + do + ECPUS=$DIR/cpuset.cpus.effective + PRS=$DIR/cpuset.cpus.partition + [[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)" + [[ -e $PRS ]] && echo "$PRS: $(cat $PRS)" + done +} + +# +# Check effective cpus +# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]* +# +check_effective_cpus() +{ + CHK_STR=$1 + for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") + do + set -- $(echo $CHK | sed -e "s/:/ /g") + CGRP=$1 + CPUS=$2 + [[ $CGRP = A2 ]] && CGRP=A1/A2 + [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 + FILE=$CGRP/cpuset.cpus.effective + [[ -e $FILE ]] || return 1 + [[ $CPUS = $(cat $FILE) ]] || return 1 + done +} + +# +# Check cgroup states +# $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]* +# +check_cgroup_states() +{ + CHK_STR=$1 + for CHK in $(echo $CHK_STR | sed -e "s/,/ /g") + do + set -- $(echo $CHK | sed -e "s/:/ /g") + CGRP=$1 + STATE=$2 + FILE= + EVAL=$(expr substr $STATE 2 2) + [[ $CGRP = A2 ]] && CGRP=A1/A2 + [[ $CGRP = A3 ]] && CGRP=A1/A2/A3 + + case $STATE in + P*) FILE=$CGRP/cpuset.cpus.partition + ;; + *) echo "Unknown state: $STATE!" + exit 1 + ;; + esac + VAL=$(cat $FILE) + + case "$VAL" in + member) VAL=0 + ;; + root) VAL=1 + ;; + isolated) + VAL=2 + ;; + "root invalid"*) + VAL=-1 + ;; + "isolated invalid"*) + VAL=-2 + ;; + esac + [[ $EVAL != $VAL ]] && return 1 + done + return 0 +} + +# +# Run cpuset state transition test +# $1 - test matrix name +# +# This test is somewhat fragile as delays (sleep x) are added in various +# places to make sure state changes are fully propagated before the next +# action. These delays may need to be adjusted if running in a slower machine. +# +run_state_test() +{ + TEST=$1 + CONTROLLER=cpuset + CPULIST=0-6 + I=0 + eval CNT="\${#$TEST[@]}" + + reset_cgroup_states + echo $CPULIST > cpuset.cpus + echo root > cpuset.cpus.partition + console_msg "Running state transition test ..." + + while [[ $I -lt $CNT ]] + do + echo "Running test $I ..." > /dev/console + eval set -- "\${$TEST[$I]}" + ROOT=$1 + OLD_A1=$2 + OLD_A2=$3 + OLD_A3=$4 + OLD_B1=$5 + NEW_A1=$6 + NEW_A2=$7 + NEW_A3=$8 + NEW_B1=$9 + RESULT=${10} + ECPUS=${11} + STATES=${12} + + set_ctrl_state_noerr . $ROOT + set_ctrl_state_noerr A1 $OLD_A1 + set_ctrl_state_noerr A1/A2 $OLD_A2 + set_ctrl_state_noerr A1/A2/A3 $OLD_A3 + set_ctrl_state_noerr B1 $OLD_B1 + RETVAL=0 + set_ctrl_state A1 $NEW_A1; ((RETVAL += $?)) + set_ctrl_state A1/A2 $NEW_A2; ((RETVAL += $?)) + set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?)) + set_ctrl_state B1 $NEW_B1; ((RETVAL += $?)) + + [[ $RETVAL -ne $RESULT ]] && { + echo "Test $TEST[$I] failed result check!" + eval echo \"\${$TEST[$I]}\" + dump_states + online_cpus + exit 1 + } + + [[ -n "$ECPUS" && "$ECPUS" != . ]] && { + check_effective_cpus $ECPUS + [[ $? -ne 0 ]] && { + echo "Test $TEST[$I] failed effective CPU check!" + eval echo \"\${$TEST[$I]}\" + echo + dump_states + online_cpus + exit 1 + } + } + + [[ -n "$STATES" ]] && { + check_cgroup_states $STATES + [[ $? -ne 0 ]] && { + echo "FAILED: Test $TEST[$I] failed states check!" + eval echo \"\${$TEST[$I]}\" + echo + dump_states + online_cpus + exit 1 + } + } + + reset_cgroup_states + # + # Check to see if effective cpu list changes + # + pause 0.05 + NEWLIST=$(cat cpuset.cpus.effective) + [[ $NEWLIST != $CPULIST ]] && { + echo "Effective cpus changed to $NEWLIST after test $I!" + exit 1 + } + [[ -n "$VERBOSE" ]] && echo "Test $I done." + ((I++)) + done + echo "All $I tests of $TEST PASSED." + + echo member > cpuset.cpus.partition +} + +# +# Wait for inotify event for the given file and read it +# $1: cgroup file to wait for +# $2: file to store the read result +# +wait_inotify() +{ + CGROUP_FILE=$1 + OUTPUT_FILE=$2 + + $WAIT_INOTIFY $CGROUP_FILE + cat $CGROUP_FILE > $OUTPUT_FILE +} + +# +# Test if inotify events are properly generated when going into and out of +# invalid partition state. +# +test_inotify() +{ + ERR=0 + PRS=/tmp/.prs_$$ + [[ -f $WAIT_INOTIFY ]] || { + echo "wait_inotify not found, inotify test SKIPPED." + return + } + + pause 0.01 + echo 1 > cpuset.cpus + echo 0 > cgroup.procs + echo root > cpuset.cpus.partition + pause 0.01 + rm -f $PRS + wait_inotify $PWD/cpuset.cpus.partition $PRS & + pause 0.01 + set_ctrl_state . "O1-0" + pause 0.01 + check_cgroup_states ".:P-1" + if [[ $? -ne 0 ]] + then + echo "FAILED: Inotify test - partition not invalid" + ERR=1 + elif [[ ! -f $PRS ]] + then + echo "FAILED: Inotify test - event not generated" + ERR=1 + kill %1 + elif [[ $(cat $PRS) != "root invalid"* ]] + then + echo "FAILED: Inotify test - incorrect state" + cat $PRS + ERR=1 + fi + online_cpus + echo member > cpuset.cpus.partition + echo 0 > ../cgroup.procs + if [[ $ERR -ne 0 ]] + then + exit 1 + else + echo "Inotify test PASSED" + fi +} + +run_state_test TEST_MATRIX +test_isolated +test_inotify +echo "All tests PASSED." +cd .. +rmdir test diff --git a/tools/testing/selftests/cgroup/wait_inotify.c b/tools/testing/selftests/cgroup/wait_inotify.c new file mode 100644 index 0000000000000000000000000000000000000000..e11b431e1b620818054a183d0af63e98d28b0647 --- /dev/null +++ b/tools/testing/selftests/cgroup/wait_inotify.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Wait until an inotify event on the given cgroup file. + */ +#include <linux/limits.h> +#include <sys/inotify.h> +#include <sys/mman.h> +#include <sys/ptrace.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +static const char usage[] = "Usage: %s [-v] <cgroup_file>\n"; +static char *file; +static int verbose; + +static inline void fail_message(char *msg) +{ + fprintf(stderr, msg, file); + exit(1); +} + +int main(int argc, char *argv[]) +{ + char *cmd = argv[0]; + int c, fd; + struct pollfd fds = { .events = POLLIN, }; + + while ((c = getopt(argc, argv, "v")) != -1) { + switch (c) { + case 'v': + verbose++; + break; + } + argv++, argc--; + } + + if (argc != 2) { + fprintf(stderr, usage, cmd); + return -1; + } + file = argv[1]; + fd = open(file, O_RDONLY); + if (fd < 0) + fail_message("Cgroup file %s not found!\n"); + close(fd); + + fd = inotify_init(); + if (fd < 0) + fail_message("inotify_init() fails on %s!\n"); + if (inotify_add_watch(fd, file, IN_MODIFY) < 0) + fail_message("inotify_add_watch() fails on %s!\n"); + fds.fd = fd; + + /* + * poll waiting loop + */ + for (;;) { + int ret = poll(&fds, 1, 10000); + + if (ret < 0) { + if (errno == EINTR) + continue; + perror("poll"); + exit(1); + } + if ((ret > 0) && (fds.revents & POLLIN)) + break; + } + if (verbose) { + struct inotify_event events[10]; + long len; + + usleep(1000); + len = read(fd, events, sizeof(events)); + printf("Number of events read = %ld\n", + len/sizeof(struct inotify_event)); + } + close(fd); + return 0; +} diff --git a/tools/testing/selftests/cpu-hotplug/Makefile b/tools/testing/selftests/cpu-hotplug/Makefile index d8be047ee5b6aabb637c87a31f1dc2df4e8543bc..8b66c473834404be7d52e41475ac4d71ba1aea75 100644 --- a/tools/testing/selftests/cpu-hotplug/Makefile +++ b/tools/testing/selftests/cpu-hotplug/Makefile @@ -6,6 +6,6 @@ TEST_PROGS := cpu-on-off-test.sh include ../lib.mk run_full_test: - @/bin/bash ./cpu-on-off-test.sh -a || echo "cpu-hotplug selftests: [FAIL]" + @/bin/bash ./cpu-on-off-test.sh -a && echo "cpu-hotplug selftests: [PASS]" || echo "cpu-hotplug selftests: [FAIL]" clean: diff --git a/tools/testing/selftests/cpu-hotplug/config b/tools/testing/selftests/cpu-hotplug/config deleted file mode 100644 index d4aca2ad5069bf5db35f7f32b9643cce34b88e43..0000000000000000000000000000000000000000 --- a/tools/testing/selftests/cpu-hotplug/config +++ /dev/null @@ -1 +0,0 @@ -CONFIG_NOTIFIER_ERROR_INJECTION=y diff --git a/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh b/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh index 0d26b5e3f9667bf94b3acf6610f16f6a422277b0..d5dc7e0dc726351a739bc487ee72bae9bd6cf090 100755 --- a/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh +++ b/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh @@ -4,6 +4,7 @@ SYSFS= # Kselftest framework requirement - SKIP code is 4. ksft_skip=4 +retval=0 prerequisite() { @@ -102,10 +103,10 @@ online_cpu_expect_success() if ! online_cpu $cpu; then echo $FUNCNAME $cpu: unexpected fail >&2 - exit 1 + retval=1 elif ! cpu_is_online $cpu; then echo $FUNCNAME $cpu: unexpected offline >&2 - exit 1 + retval=1 fi } @@ -115,10 +116,10 @@ online_cpu_expect_fail() if online_cpu $cpu 2> /dev/null; then echo $FUNCNAME $cpu: unexpected success >&2 - exit 1 + retval=1 elif ! cpu_is_offline $cpu; then echo $FUNCNAME $cpu: unexpected online >&2 - exit 1 + retval=1 fi } @@ -128,10 +129,10 @@ offline_cpu_expect_success() if ! offline_cpu $cpu; then echo $FUNCNAME $cpu: unexpected fail >&2 - exit 1 + retval=1 elif ! cpu_is_offline $cpu; then echo $FUNCNAME $cpu: unexpected offline >&2 - exit 1 + retval=1 fi } @@ -141,16 +142,33 @@ offline_cpu_expect_fail() if offline_cpu $cpu 2> /dev/null; then echo $FUNCNAME $cpu: unexpected success >&2 - exit 1 + retval=1 elif ! cpu_is_online $cpu; then echo $FUNCNAME $cpu: unexpected offline >&2 - exit 1 + retval=1 fi } -error=-12 +online_all_hot_pluggable_cpus() +{ + for cpu in `hotplaggable_offline_cpus`; do + online_cpu_expect_success $cpu + done +} + +offline_all_hot_pluggable_cpus() +{ + local reserve_cpu=$online_max + for cpu in `hotpluggable_online_cpus`; do + # Reserve one cpu oneline at least. + if [ $cpu -eq $reserve_cpu ];then + continue + fi + offline_cpu_expect_success $cpu + done +} + allcpus=0 -priority=0 online_cpus=0 online_max=0 offline_cpus=0 @@ -158,31 +176,20 @@ offline_max=0 present_cpus=0 present_max=0 -while getopts e:ahp: opt; do +while getopts ah opt; do case $opt in - e) - error=$OPTARG - ;; a) allcpus=1 ;; h) - echo "Usage $0 [ -a ] [ -e errno ] [ -p notifier-priority ]" + echo "Usage $0 [ -a ]" echo -e "\t default offline one cpu" echo -e "\t run with -a option to offline all cpus" exit ;; - p) - priority=$OPTARG - ;; esac done -if ! [ "$error" -ge -4095 -a "$error" -lt 0 ]; then - echo "error code must be -4095 <= errno < 0" >&2 - exit 1 -fi - prerequisite # @@ -196,12 +203,12 @@ if [ $allcpus -eq 0 ]; then online_cpu_expect_success $online_max if [[ $offline_cpus -gt 0 ]]; then - echo -e "\t offline to online to offline: cpu $present_max" + echo -e "\t online to offline to online: cpu $present_max" online_cpu_expect_success $present_max offline_cpu_expect_success $present_max online_cpu $present_max fi - exit 0 + exit $retval else echo "Full scope test: all hotplug cpus" echo -e "\t online all offline cpus" @@ -209,85 +216,10 @@ else echo -e "\t online all offline cpus" fi -# -# Online all hot-pluggable CPUs -# -for cpu in `hotplaggable_offline_cpus`; do - online_cpu_expect_success $cpu -done - -# -# Offline all hot-pluggable CPUs -# -for cpu in `hotpluggable_online_cpus`; do - offline_cpu_expect_success $cpu -done - -# -# Online all hot-pluggable CPUs again -# -for cpu in `hotplaggable_offline_cpus`; do - online_cpu_expect_success $cpu -done - -# -# Test with cpu notifier error injection -# +online_all_hot_pluggable_cpus -DEBUGFS=`mount -t debugfs | head -1 | awk '{ print $3 }'` -NOTIFIER_ERR_INJECT_DIR=$DEBUGFS/notifier-error-inject/cpu +offline_all_hot_pluggable_cpus -prerequisite_extra() -{ - msg="skip extra tests:" - - /sbin/modprobe -q -r cpu-notifier-error-inject - /sbin/modprobe -q cpu-notifier-error-inject priority=$priority - - if [ ! -d "$DEBUGFS" ]; then - echo $msg debugfs is not mounted >&2 - exit $ksft_skip - fi - - if [ ! -d $NOTIFIER_ERR_INJECT_DIR ]; then - echo $msg cpu-notifier-error-inject module is not available >&2 - exit $ksft_skip - fi -} - -prerequisite_extra - -# -# Offline all hot-pluggable CPUs -# -echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error -for cpu in `hotpluggable_online_cpus`; do - offline_cpu_expect_success $cpu -done - -# -# Test CPU hot-add error handling (offline => online) -# -echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_UP_PREPARE/error -for cpu in `hotplaggable_offline_cpus`; do - online_cpu_expect_fail $cpu -done - -# -# Online all hot-pluggable CPUs -# -echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_UP_PREPARE/error -for cpu in `hotplaggable_offline_cpus`; do - online_cpu_expect_success $cpu -done - -# -# Test CPU hot-remove error handling (online => offline) -# -echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error -for cpu in `hotpluggable_online_cpus`; do - offline_cpu_expect_fail $cpu -done +online_all_hot_pluggable_cpus -echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error -/sbin/modprobe -q -r cpu-notifier-error-inject +exit $retval diff --git a/tools/testing/selftests/damon/Makefile b/tools/testing/selftests/damon/Makefile index 0470c5f3e69068084b7778cb83e36da0df2cd52e..a1fa2eff8192fd14414c027bc951dc65fa693d7c 100644 --- a/tools/testing/selftests/damon/Makefile +++ b/tools/testing/selftests/damon/Makefile @@ -6,6 +6,7 @@ TEST_GEN_FILES += huge_count_read_write TEST_FILES = _chk_dependency.sh _debugfs_common.sh TEST_PROGS = debugfs_attrs.sh debugfs_schemes.sh debugfs_target_ids.sh TEST_PROGS += debugfs_empty_targets.sh debugfs_huge_count_read_write.sh +TEST_PROGS += debugfs_duplicate_context_creation.sh TEST_PROGS += sysfs.sh include ../lib.mk diff --git a/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh new file mode 100644 index 0000000000000000000000000000000000000000..4a76e37ef16b1f2c67ce393b7cbfc5934397c71b --- /dev/null +++ b/tools/testing/selftests/damon/debugfs_duplicate_context_creation.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source _debugfs_common.sh + +# Test duplicated context creation +# ================================ + +if ! echo foo > "$DBGFS/mk_contexts" +then + echo "context creation failed" + exit 1 +fi + +if echo foo > "$DBGFS/mk_contexts" +then + echo "duplicate context creation success" + exit 1 +fi + +if ! echo foo > "$DBGFS/rm_contexts" +then + echo "context deletion failed" + exit 1 +fi + +exit 0 diff --git a/tools/testing/selftests/drivers/net/bonding/Makefile b/tools/testing/selftests/drivers/net/bonding/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e9dab5f9d77380e1eacb5eb025b93c589a7f5a88 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for net selftests + +TEST_PROGS := \ + bond-arp-interval-causes-panic.sh \ + bond-break-lacpdu-tx.sh \ + bond-lladdr-target.sh \ + dev_addr_lists.sh + +TEST_FILES := lag_lib.sh + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/net/bonding/bond-arp-interval-causes-panic.sh b/tools/testing/selftests/drivers/net/bonding/bond-arp-interval-causes-panic.sh new file mode 100755 index 0000000000000000000000000000000000000000..71c00bfafbc99d6ac8cd60f53eb3010032bc0353 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/bond-arp-interval-causes-panic.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# cause kernel oops in bond_rr_gen_slave_id +DEBUG=${DEBUG:-0} + +set -e +test ${DEBUG} -ne 0 && set -x + +finish() +{ + ip netns delete server || true + ip netns delete client || true + ip link del link1_1 || true +} + +trap finish EXIT + +client_ip4=192.168.1.198 +server_ip4=192.168.1.254 + +# setup kernel so it reboots after causing the panic +echo 180 >/proc/sys/kernel/panic + +# build namespaces +ip link add dev link1_1 type veth peer name link1_2 + +ip netns add "server" +ip link set dev link1_2 netns server up name eth0 +ip netns exec server ip addr add ${server_ip4}/24 dev eth0 + +ip netns add "client" +ip link set dev link1_1 netns client down name eth0 +ip netns exec client ip link add dev bond0 down type bond mode 1 \ + miimon 100 all_slaves_active 1 +ip netns exec client ip link set dev eth0 down master bond0 +ip netns exec client ip link set dev bond0 up +ip netns exec client ip addr add ${client_ip4}/24 dev bond0 +ip netns exec client ping -c 5 $server_ip4 >/dev/null + +ip netns exec client ip link set dev eth0 down nomaster +ip netns exec client ip link set dev bond0 down +ip netns exec client ip link set dev bond0 type bond mode 0 \ + arp_interval 1000 arp_ip_target "+${server_ip4}" +ip netns exec client ip link set dev eth0 down master bond0 +ip netns exec client ip link set dev bond0 up +ip netns exec client ping -c 5 $server_ip4 >/dev/null + +exit 0 diff --git a/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh b/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh new file mode 100755 index 0000000000000000000000000000000000000000..47ab90596acb2096f2b70863b0835a109c524b8b --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/bond-break-lacpdu-tx.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# Regression Test: +# Verify LACPDUs get transmitted after setting the MAC address of +# the bond. +# +# https://bugzilla.redhat.com/show_bug.cgi?id=2020773 +# +# +---------+ +# | fab-br0 | +# +---------+ +# | +# +---------+ +# | fbond | +# +---------+ +# | | +# +------+ +------+ +# |veth1 | |veth2 | +# +------+ +------+ +# +# We use veths instead of physical interfaces + +set -e +tmp=$(mktemp -q dump.XXXXXX) +cleanup() { + ip link del fab-br0 >/dev/null 2>&1 || : + ip link del fbond >/dev/null 2>&1 || : + ip link del veth1-bond >/dev/null 2>&1 || : + ip link del veth2-bond >/dev/null 2>&1 || : + modprobe -r bonding >/dev/null 2>&1 || : + rm -f -- ${tmp} +} + +trap cleanup 0 1 2 +cleanup +sleep 1 + +# create the bridge +ip link add fab-br0 address 52:54:00:3B:7C:A6 mtu 1500 type bridge \ + forward_delay 15 + +# create the bond +ip link add fbond type bond mode 4 miimon 200 xmit_hash_policy 1 \ + ad_actor_sys_prio 65535 lacp_rate fast + +# set bond address +ip link set fbond address 52:54:00:3B:7C:A6 +ip link set fbond up + +# set again bond sysfs parameters +ip link set fbond type bond ad_actor_sys_prio 65535 + +# create veths +ip link add name veth1-bond type veth peer name veth1-end +ip link add name veth2-bond type veth peer name veth2-end + +# add ports +ip link set fbond master fab-br0 +ip link set veth1-bond down master fbond +ip link set veth2-bond down master fbond + +# bring up +ip link set veth1-end up +ip link set veth2-end up +ip link set fab-br0 up +ip link set fbond up +ip addr add dev fab-br0 10.0.0.3 + +tcpdump -n -i veth1-end -e ether proto 0x8809 >${tmp} 2>&1 & +sleep 15 +pkill tcpdump >/dev/null 2>&1 +rc=0 +num=$(grep "packets captured" ${tmp} | awk '{print $1}') +if test "$num" -gt 0; then + echo "PASS, captured ${num}" +else + echo "FAIL" + rc=1 +fi +exit $rc diff --git a/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh b/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh new file mode 100755 index 0000000000000000000000000000000000000000..89af402fabbea88f5eb2fe517d64bf4f79bdc78a --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Regression Test: +# Verify bond interface could up when set IPv6 link local address target. +# +# +----------------+ +# | br0 | +# | | | sw +# | veth0 veth1 | +# +---+-------+----+ +# | | +# +---+-------+----+ +# | veth0 veth1 | +# | | | host +# | bond0 | +# +----------------+ +# +# We use veths instead of physical interfaces +sw="sw-$(mktemp -u XXXXXX)" +host="ns-$(mktemp -u XXXXXX)" + +cleanup() +{ + ip netns del $sw + ip netns del $host +} + +trap cleanup 0 1 2 + +ip netns add $sw +ip netns add $host + +ip -n $host link add veth0 type veth peer name veth0 netns $sw +ip -n $host link add veth1 type veth peer name veth1 netns $sw + +ip -n $sw link add br0 type bridge +ip -n $sw link set br0 up +sw_lladdr=$(ip -n $sw addr show br0 | awk '/fe80/{print $2}' | cut -d'/' -f1) +# sleep some time to make sure bridge lladdr pass DAD +sleep 2 + +ip -n $host link add bond0 type bond mode 1 ns_ip6_target ${sw_lladdr} \ + arp_validate 3 arp_interval 1000 +# add a lladdr for bond to make sure there is a route to target +ip -n $host addr add fe80::beef/64 dev bond0 +ip -n $host link set bond0 up +ip -n $host link set veth0 master bond0 +ip -n $host link set veth1 master bond0 + +ip -n $sw link set veth0 master br0 +ip -n $sw link set veth1 master br0 +ip -n $sw link set veth0 up +ip -n $sw link set veth1 up + +sleep 5 + +rc=0 +if ip -n $host link show bond0 | grep -q LOWER_UP; then + echo "PASS" +else + echo "FAIL" + rc=1 +fi +exit $rc diff --git a/tools/testing/selftests/drivers/net/bonding/config b/tools/testing/selftests/drivers/net/bonding/config new file mode 100644 index 0000000000000000000000000000000000000000..70638fa50b2cc872747bd9cdd34a1a111251dc97 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/config @@ -0,0 +1,2 @@ +CONFIG_BONDING=y +CONFIG_MACVLAN=y diff --git a/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh b/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh new file mode 100755 index 0000000000000000000000000000000000000000..e6fa24eded5b82004c5be39fb658f92582c8e1c7 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test bond device handling of addr lists (dev->uc, mc) +# + +ALL_TESTS=" + bond_cleanup_mode1 + bond_cleanup_mode4 + bond_listen_lacpdu_multicast_case_down + bond_listen_lacpdu_multicast_case_up +" + +REQUIRE_MZ=no +NUM_NETIFS=0 +lib_dir=$(dirname "$0") +source "$lib_dir"/../../../net/forwarding/lib.sh + +source "$lib_dir"/lag_lib.sh + + +destroy() +{ + local ifnames=(dummy1 dummy2 bond1 mv0) + local ifname + + for ifname in "${ifnames[@]}"; do + ip link del "$ifname" &>/dev/null + done +} + +cleanup() +{ + pre_cleanup + + destroy +} + + +# bond driver control paths vary between modes that have a primary slave +# (bond_uses_primary()) and others. Test both kinds of modes. + +bond_cleanup_mode1() +{ + RET=0 + + test_LAG_cleanup "bonding" "active-backup" +} + +bond_cleanup_mode4() { + RET=0 + + test_LAG_cleanup "bonding" "802.3ad" +} + +bond_listen_lacpdu_multicast() +{ + # Initial state of bond device, up | down + local init_state=$1 + local lacpdu_mc="01:80:c2:00:00:02" + + ip link add dummy1 type dummy + ip link add bond1 "$init_state" type bond mode 802.3ad + ip link set dev dummy1 master bond1 + if [ "$init_state" = "down" ]; then + ip link set dev bond1 up + fi + + grep_bridge_fdb "$lacpdu_mc" bridge fdb show brport dummy1 >/dev/null + check_err $? "LACPDU multicast address not present on slave (1)" + + ip link set dev bond1 down + + not grep_bridge_fdb "$lacpdu_mc" bridge fdb show brport dummy1 >/dev/null + check_err $? "LACPDU multicast address still present on slave" + + ip link set dev bond1 up + + grep_bridge_fdb "$lacpdu_mc" bridge fdb show brport dummy1 >/dev/null + check_err $? "LACPDU multicast address not present on slave (2)" + + cleanup + + log_test "bonding LACPDU multicast address to slave (from bond $init_state)" +} + +# The LACPDU mc addr is added by different paths depending on the initial state +# of the bond when enslaving a device. Test both cases. + +bond_listen_lacpdu_multicast_case_down() +{ + RET=0 + + bond_listen_lacpdu_multicast "down" +} + +bond_listen_lacpdu_multicast_case_up() +{ + RET=0 + + bond_listen_lacpdu_multicast "up" +} + + +trap cleanup EXIT + +tests_run + +exit "$EXIT_STATUS" diff --git a/tools/testing/selftests/drivers/net/bonding/lag_lib.sh b/tools/testing/selftests/drivers/net/bonding/lag_lib.sh new file mode 100644 index 0000000000000000000000000000000000000000..16c7fb858ac103d5e85c5192676f0c453a316a42 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/lag_lib.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test that a link aggregation device (bonding, team) removes the hardware +# addresses that it adds on its underlying devices. +test_LAG_cleanup() +{ + local driver=$1 + local mode=$2 + local ucaddr="02:00:00:12:34:56" + local addr6="fe80::78:9abc/64" + local mcaddr="33:33:ff:78:9a:bc" + local name + + ip link add dummy1 type dummy + ip link add dummy2 type dummy + if [ "$driver" = "bonding" ]; then + name="bond1" + ip link add "$name" up type bond mode "$mode" + ip link set dev dummy1 master "$name" + ip link set dev dummy2 master "$name" + elif [ "$driver" = "team" ]; then + name="team0" + teamd -d -c ' + { + "device": "'"$name"'", + "runner": { + "name": "'"$mode"'" + }, + "ports": { + "dummy1": + {}, + "dummy2": + {} + } + } + ' + ip link set dev "$name" up + else + check_err 1 + log_test test_LAG_cleanup ": unknown driver \"$driver\"" + return + fi + + # Used to test dev->uc handling + ip link add mv0 link "$name" up address "$ucaddr" type macvlan + # Used to test dev->mc handling + ip address add "$addr6" dev "$name" + ip link set dev "$name" down + ip link del "$name" + + not grep_bridge_fdb "$ucaddr" bridge fdb show >/dev/null + check_err $? "macvlan unicast address still present on a slave" + + not grep_bridge_fdb "$mcaddr" bridge fdb show >/dev/null + check_err $? "IPv6 solicited-node multicast mac address still present on a slave" + + cleanup + + log_test "$driver cleanup mode $mode" +} diff --git a/tools/testing/selftests/drivers/net/bonding/settings b/tools/testing/selftests/drivers/net/bonding/settings new file mode 100644 index 0000000000000000000000000000000000000000..867e118223cd652fc12bd01c6abdcb67b0d6eca1 --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/settings @@ -0,0 +1 @@ +timeout=60 diff --git a/tools/testing/selftests/drivers/net/dsa/Makefile b/tools/testing/selftests/drivers/net/dsa/Makefile index 2a731d5c6d859a7703a2f86a33f4ee9f475f573d..c393e7b738051ed37c152ebd007792544c800759 100644 --- a/tools/testing/selftests/drivers/net/dsa/Makefile +++ b/tools/testing/selftests/drivers/net/dsa/Makefile @@ -8,9 +8,10 @@ TEST_PROGS = bridge_locked_port.sh \ bridge_vlan_unaware.sh \ local_termination.sh \ no_forwarding.sh \ + tc_actions.sh \ test_bridge_fdb_stress.sh -TEST_PROGS_EXTENDED := lib.sh +TEST_PROGS_EXTENDED := lib.sh tc_common.sh TEST_FILES := forwarding.config diff --git a/tools/testing/selftests/drivers/net/dsa/tc_actions.sh b/tools/testing/selftests/drivers/net/dsa/tc_actions.sh new file mode 120000 index 0000000000000000000000000000000000000000..306213d9430e0b664f16d8ad2eb7c2c9049a05e9 --- /dev/null +++ b/tools/testing/selftests/drivers/net/dsa/tc_actions.sh @@ -0,0 +1 @@ +../../../net/forwarding/tc_actions.sh \ No newline at end of file diff --git a/tools/testing/selftests/drivers/net/dsa/tc_common.sh b/tools/testing/selftests/drivers/net/dsa/tc_common.sh new file mode 120000 index 0000000000000000000000000000000000000000..bc3465bdc36bf2e07ad41f957d8af55e7838552e --- /dev/null +++ b/tools/testing/selftests/drivers/net/dsa/tc_common.sh @@ -0,0 +1 @@ +../../../net/forwarding/tc_common.sh \ No newline at end of file diff --git a/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh b/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh new file mode 100755 index 0000000000000000000000000000000000000000..0cf9e47e32092f44a07970605399a13f2b286e49 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test VLAN classification after routing and verify that the order of +# configuration does not impact switch behavior. Verify that {RIF, Port}->VID +# mapping is added correctly for existing {Port, VID}->FID mapping and that +# {RIF, Port}->VID mapping is added correctly for new {Port, VID}->FID mapping. + +# +-------------------+ +--------------------+ +# | H1 | | H2 | +# | | | | +# | $h1.10 + | | + $h2.10 | +# | 192.0.2.1/28 | | | | 192.0.2.3/28 | +# | | | | | | +# | $h1 + | | + $h2 | +# +----------------|--+ +--|-----------------+ +# | | +# +----------------|-------------------------|-----------------+ +# | SW | | | +# | +--------------|-------------------------|---------------+ | +# | | $swp1 + + $swp2 | | +# | | | | | | +# | | $swp1.10 + + $swp2.10 | | +# | | | | +# | | br0 | | +# | | 192.0.2.2/28 | | +# | +--------------------------------------------------------+ | +# | | +# | $swp3.20 + | +# | 192.0.2.17/28 | | +# | | | +# | $swp3 + | +# +---------------|--------------------------------------------+ +# | +# +---------------|--+ +# | $h3 + | +# | | | +# | $h3.20 + | +# | 192.0.2.18/28 | +# | | +# | H3 | +# +------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + port_vid_map_rif + rif_port_vid_map +" + +NUM_NETIFS=6 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 10 v$h1 192.0.2.1/28 + + ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 +} + +h1_destroy() +{ + ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 + + vlan_destroy $h1 10 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + vlan_create $h2 10 v$h2 192.0.2.3/28 +} + +h2_destroy() +{ + vlan_destroy $h2 10 + simple_if_fini $h2 +} + +h3_create() +{ + simple_if_init $h3 + vlan_create $h3 20 v$h3 192.0.2.18/28 + + ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 +} + +h3_destroy() +{ + ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 + + vlan_destroy $h3 20 + simple_if_fini $h3 +} + +switch_create() +{ + ip link set dev $swp1 up + tc qdisc add dev $swp1 clsact + + ip link add dev br0 type bridge mcast_snooping 0 + + # By default, a link-local address is generated when netdevice becomes + # up. Adding an address to the bridge will cause creating a RIF for it. + # Prevent generating link-local address to be able to control when the + # RIF is added. + sysctl_set net.ipv6.conf.br0.addr_gen_mode 1 + ip link set dev br0 up + + ip link set dev $swp2 up + vlan_create $swp2 10 + ip link set dev $swp2.10 master br0 + + ip link set dev $swp3 up + vlan_create $swp3 20 "" 192.0.2.17/28 + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev $swp3.20 192.0.2.18 lladdr $(mac_get $h3.20) +} + +switch_destroy() +{ + vlan_destroy $swp3 20 + ip link set dev $swp3 down + + ip link set dev $swp2.10 nomaster + vlan_destroy $swp2 10 + ip link set dev $swp2 down + + ip link set dev br0 down + sysctl_restore net.ipv6.conf.br0.addr_gen_mode + ip link del dev br0 + + tc qdisc del dev $swp1 clsact + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + h3=${NETIFS[p6]} + + vrf_prepare + forwarding_enable + + h1_create + h2_create + h3_create + + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + + h3_destroy + h2_destroy + h1_destroy + + forwarding_restore + vrf_cleanup +} + +bridge_rif_add() +{ + rifs_occ_t0=$(devlink_resource_occ_get rifs) + __addr_add_del br0 add 192.0.2.2/28 + rifs_occ_t1=$(devlink_resource_occ_get rifs) + + expected_rifs=$((rifs_occ_t0 + 1)) + + [[ $expected_rifs -eq $rifs_occ_t1 ]] + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + sleep 1 +} + +bridge_rif_del() +{ + __addr_add_del br0 del 192.0.2.2/28 +} + +port_vid_map_rif() +{ + RET=0 + + # First add {port, VID}->FID for swp1.10, then add a RIF and verify that + # packets get the correct VID after routing. + vlan_create $swp1 10 + ip link set dev $swp1.10 master br0 + bridge_rif_add + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev br0 192.0.2.1 lladdr $(mac_get $h1.10) + + # The hardware matches on the first ethertype which is not VLAN, + # so the protocol should be IP. + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.1 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp1 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add RIF for existing {port, VID}->FID mapping" + + tc filter del dev $swp1 egress + + bridge_rif_del + ip link set dev $swp1.10 nomaster + vlan_destroy $swp1 10 +} + +rif_port_vid_map() +{ + RET=0 + + # First add an address to the bridge, which will create a RIF on top of + # it, then add a new {port, VID}->FID mapping and verify that packets + # get the correct VID after routing. + bridge_rif_add + vlan_create $swp1 10 + ip link set dev $swp1.10 master br0 + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev br0 192.0.2.1 lladdr $(mac_get $h1.10) + + # The hardware matches on the first ethertype which is not VLAN, + # so the protocol should be IP. + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.1 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp1 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add {port, VID}->FID mapping for FID with a RIF" + + tc filter del dev $swp1 egress + + ip link set dev $swp1.10 nomaster + vlan_destroy $swp1 10 + bridge_rif_del +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh new file mode 100755 index 0000000000000000000000000000000000000000..df2b099668862c684a9d8460fb04c6be28fdd9f3 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test routing over bridge and verify that the order of configuration does not +# impact switch behavior. Verify that RIF is added correctly for existing +# mappings and that new mappings use the correct RIF. + +# +-------------------+ +--------------------+ +# | H1 | | H2 | +# | | | | +# | $h1.10 + | | + $h2.10 | +# | 192.0.2.1/28 | | | | 192.0.2.3/28 | +# | | | | | | +# | $h1 + | | + $h2 | +# +----------------|--+ +--|-----------------+ +# | | +# +----------------|-------------------------|-----------------+ +# | SW | | | +# | +--------------|-------------------------|---------------+ | +# | | $swp1 + + $swp2 | | +# | | | | | | +# | | $swp1.10 + + $swp2.10 | | +# | | | | +# | | br0 | | +# | | 192.0.2.2/28 | | +# | +--------------------------------------------------------+ | +# | | +# | $swp3.10 + | +# | 192.0.2.17/28 | | +# | | | +# | $swp3 + | +# +---------------|--------------------------------------------+ +# | +# +---------------|--+ +# | $h3 + | +# | | | +# | $h3.10 + | +# | 192.0.2.18/28 | +# | | +# | H3 | +# +------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + port_vid_map_rif + rif_port_vid_map +" + +NUM_NETIFS=6 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 10 v$h1 192.0.2.1/28 + + ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 +} + +h1_destroy() +{ + ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 + + vlan_destroy $h1 10 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + vlan_create $h2 10 v$h2 192.0.2.3/28 +} + +h2_destroy() +{ + vlan_destroy $h2 10 + simple_if_fini $h2 +} + +h3_create() +{ + simple_if_init $h3 + vlan_create $h3 10 v$h3 192.0.2.18/28 + + ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 +} + +h3_destroy() +{ + ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 + + vlan_destroy $h3 10 + simple_if_fini $h3 +} + +switch_create() +{ + ip link set dev $swp1 up + + ip link add dev br0 type bridge mcast_snooping 0 + + # By default, a link-local address is generated when netdevice becomes + # up. Adding an address to the bridge will cause creating a RIF for it. + # Prevent generating link-local address to be able to control when the + # RIF is added. + sysctl_set net.ipv6.conf.br0.addr_gen_mode 1 + ip link set dev br0 up + + ip link set dev $swp2 up + vlan_create $swp2 10 + ip link set dev $swp2.10 master br0 + + ip link set dev $swp3 up + vlan_create $swp3 10 "" 192.0.2.17/28 + tc qdisc add dev $swp3 clsact + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev $swp3.10 192.0.2.18 lladdr $(mac_get $h3.10) +} + +switch_destroy() +{ + tc qdisc del dev $swp3 clsact + vlan_destroy $swp3 10 + ip link set dev $swp3 down + + ip link set dev $swp2.10 nomaster + vlan_destroy $swp2 10 + ip link set dev $swp2 down + + ip link set dev br0 down + sysctl_restore net.ipv6.conf.br0.addr_gen_mode + ip link del dev br0 + + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + h3=${NETIFS[p6]} + + vrf_prepare + forwarding_enable + + h1_create + h2_create + h3_create + + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + + h3_destroy + h2_destroy + h1_destroy + + forwarding_restore + vrf_cleanup +} + +bridge_rif_add() +{ + rifs_occ_t0=$(devlink_resource_occ_get rifs) + __addr_add_del br0 add 192.0.2.2/28 + rifs_occ_t1=$(devlink_resource_occ_get rifs) + + expected_rifs=$((rifs_occ_t0 + 1)) + + [[ $expected_rifs -eq $rifs_occ_t1 ]] + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + sleep 1 +} + +bridge_rif_del() +{ + __addr_add_del br0 del 192.0.2.2/28 +} + +port_vid_map_rif() +{ + RET=0 + + # First add {port, VID}->FID for $swp1.10, then add a RIF and verify + # that packets can be routed via the existing mapping. + vlan_create $swp1 10 + ip link set dev $swp1.10 master br0 + bridge_rif_add + + # The hardware matches on the first ethertype which is not VLAN, + # so the protocol should be IP. + tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.18 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp3 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add RIF for existing {port, VID}->FID mapping" + + tc filter del dev $swp3 egress + + bridge_rif_del + ip link set dev $swp1.10 nomaster + vlan_destroy $swp1 10 +} + +rif_port_vid_map() +{ + RET=0 + + # First add an address to the bridge, which will create a RIF on top of + # it, then add a new {port, VID}->FID mapping and verify that packets + # can be routed via the new mapping. + bridge_rif_add + vlan_create $swp1 10 + ip link set dev $swp1.10 master br0 + + # The hardware matches on the first ethertype which is not VLAN, + # so the protocol should be IP. + tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.18 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp3 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add {port, VID}->FID mapping for FID with a RIF" + + tc filter del dev $swp3 egress + + ip link set dev $swp1.10 nomaster + vlan_destroy $swp1 10 + bridge_rif_del +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh new file mode 100755 index 0000000000000000000000000000000000000000..577293bab88bc20580fd84fac7a0821fe1ffb725 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test routing over bridge and verify that the order of configuration does not +# impact switch behavior. Verify that RIF is added correctly for existing +# mapping and that packets can be routed via port which is added after the FID +# already has a RIF. + +# +-------------------+ +--------------------+ +# | H1 | | H2 | +# | | | | +# | $h1.10 + | | + $h2.10 | +# | 192.0.2.1/28 | | | | 192.0.2.3/28 | +# | | | | | | +# | $h1 + | | + $h2 | +# +----------------|--+ +--|-----------------+ +# | | +# +----------------|-------------------------|-----------------+ +# | SW | | | +# | +--------------|-------------------------|---------------+ | +# | | $swp1 + + $swp2 | | +# | | | | +# | | br0 | | +# | +--------------------------------------------------------+ | +# | | | +# | br0.10 | +# | 192.0.2.2/28 | +# | | +# | | +# | $swp3 + | +# | 192.0.2.17/28 | | +# +----------------|-------------------------------------------+ +# | +# +----------------|--+ +# | $h3 + | +# | 192.0.2.18/28 | +# | | +# | H3 | +# +-------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + vid_map_rif + rif_vid_map +" + +NUM_NETIFS=6 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +h1_create() +{ + simple_if_init $h1 + vlan_create $h1 10 v$h1 192.0.2.1/28 + + ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 +} + +h1_destroy() +{ + ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2 + + vlan_destroy $h1 10 + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + vlan_create $h2 10 v$h2 192.0.2.3/28 +} + +h2_destroy() +{ + vlan_destroy $h2 10 + simple_if_fini $h2 +} + +h3_create() +{ + simple_if_init $h3 192.0.2.18/28 + ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 +} + +h3_destroy() +{ + ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17 + simple_if_fini $h3 192.0.2.18/28 +} + +switch_create() +{ + ip link set dev $swp1 up + + ip link add dev br0 type bridge vlan_filtering 1 mcast_snooping 0 + + # By default, a link-local address is generated when netdevice becomes + # up. Adding an address to the bridge will cause creating a RIF for it. + # Prevent generating link-local address to be able to control when the + # RIF is added. + sysctl_set net.ipv6.conf.br0.addr_gen_mode 1 + ip link set dev br0 up + + ip link set dev $swp2 up + ip link set dev $swp2 master br0 + bridge vlan add vid 10 dev $swp2 + + ip link set dev $swp3 up + __addr_add_del $swp3 add 192.0.2.17/28 + tc qdisc add dev $swp3 clsact + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev $swp3 192.0.2.18 lladdr $(mac_get $h3) +} + +switch_destroy() +{ + tc qdisc del dev $swp3 clsact + __addr_add_del $swp3 del 192.0.2.17/28 + ip link set dev $swp3 down + + bridge vlan del vid 10 dev $swp2 + ip link set dev $swp2 nomaster + ip link set dev $swp2 down + + ip link set dev br0 down + sysctl_restore net.ipv6.conf.br0.addr_gen_mode + ip link del dev br0 + + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + swp3=${NETIFS[p5]} + h3=${NETIFS[p6]} + + vrf_prepare + forwarding_enable + + h1_create + h2_create + h3_create + + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + + h3_destroy + h2_destroy + h1_destroy + + forwarding_restore + vrf_cleanup +} + +bridge_rif_add() +{ + rifs_occ_t0=$(devlink_resource_occ_get rifs) + vlan_create br0 10 "" 192.0.2.2/28 + rifs_occ_t1=$(devlink_resource_occ_get rifs) + + expected_rifs=$((rifs_occ_t0 + 1)) + + [[ $expected_rifs -eq $rifs_occ_t1 ]] + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" + + sleep 1 +} + +bridge_rif_del() +{ + vlan_destroy br0 10 +} + +vid_map_rif() +{ + RET=0 + + # First add VID->FID for vlan 10, then add a RIF and verify that + # packets can be routed via the existing mapping. + bridge vlan add vid 10 dev br0 self + ip link set dev $swp1 master br0 + bridge vlan add vid 10 dev $swp1 + + bridge_rif_add + + tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.18 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp3 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add RIF for existing VID->FID mapping" + + tc filter del dev $swp3 egress + + bridge_rif_del + + bridge vlan del vid 10 dev $swp1 + ip link set dev $swp1 nomaster + bridge vlan del vid 10 dev br0 self +} + +rif_vid_map() +{ + RET=0 + + # Using 802.1Q, there is only one VID->FID map for each VID. That means + # that we cannot really check adding a new map for existing FID with a + # RIF. Verify that packets can be routed via port which is added after + # the FID already has a RIF, although in practice there is no new + # mapping in the hardware. + bridge vlan add vid 10 dev br0 self + bridge_rif_add + + ip link set dev $swp1 master br0 + bridge vlan add vid 10 dev $swp1 + + tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.18 action pass + + ping_do $h1.10 192.0.2.18 + check_err $? "Ping failed" + + tc_check_at_least_x_packets "dev $swp3 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add port to VID->FID mapping for FID with a RIF" + + tc filter del dev $swp3 egress + + bridge vlan del vid 10 dev $swp1 + ip link set dev $swp1 nomaster + + bridge_rif_del + bridge vlan del vid 10 dev br0 self +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh new file mode 100755 index 0000000000000000000000000000000000000000..90450216a10d042e82a09a4ff70a257d097ebad4 --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh @@ -0,0 +1,311 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# Test routing after VXLAN decapsulation and verify that the order of +# configuration does not impact switch behavior. Verify that RIF is added +# correctly for existing mapping and that new mapping uses the correct RIF. + +# +---------------------------+ +# | H1 | +# | + $h1 | +# | | 192.0.2.1/28 | +# +----|----------------------+ +# | +# +----|----------------------------------------------------------------------+ +# | SW | | +# | +--|--------------------------------------------------------------------+ | +# | | + $swp1 br1 | | +# | | vid 10 pvid untagged | | +# | | | | +# | | | | +# | | + vx4001 | | +# | | local 192.0.2.17 | | +# | | remote 192.0.2.18 | | +# | | id 104001 | | +# | | dstport $VXPORT | | +# | | vid 4001 pvid untagged | | +# | | | | +# | +----------------------------------+------------------------------------+ | +# | | | +# | +----------------------------------|------------------------------------+ | +# | | | | | +# | | +-------------------------------+---------------------------------+ | | +# | | | | | | +# | | + vlan10 vlan4001 + | | +# | | 192.0.2.2/28 | | +# | | | | +# | | vrf-green | | +# | +-----------------------------------------------------------------------+ | +# | | +# | + $rp1 +lo | +# | | 198.51.100.1/24 192.0.2.17/32 | +# +----|----------------------------------------------------------------------+ +# | +# +----|--------------------------------------------------------+ +# | | v$rp2 | +# | + $rp2 | +# | 198.51.100.2/24 | +# | | +# +-------------------------------------------------------------+ + +lib_dir=$(dirname $0)/../../../net/forwarding + +ALL_TESTS=" + vni_fid_map_rif + rif_vni_fid_map +" + +NUM_NETIFS=4 +source $lib_dir/lib.sh +source $lib_dir/tc_common.sh +source $lib_dir/devlink_lib.sh + +: ${VXPORT:=4789} +export VXPORT + +h1_create() +{ + simple_if_init $h1 192.0.2.1/28 +} + +h1_destroy() +{ + simple_if_fini $h1 192.0.2.1/28 +} + +switch_create() +{ + ip link add name br1 type bridge vlan_filtering 1 vlan_default_pvid 0 \ + mcast_snooping 0 + # Make sure the bridge uses the MAC address of the local port and not + # that of the VxLAN's device. + ip link set dev br1 address $(mac_get $swp1) + ip link set dev br1 up + + ip link set dev $rp1 up + ip address add dev $rp1 198.51.100.1/24 + + ip link set dev $swp1 master br1 + ip link set dev $swp1 up + bridge vlan add vid 10 dev $swp1 pvid untagged + + tc qdisc add dev $swp1 clsact + + ip link add name vx4001 type vxlan id 104001 \ + local 192.0.2.17 dstport $VXPORT \ + nolearning noudpcsum tos inherit ttl 100 + ip link set dev vx4001 up + + ip link set dev vx4001 master br1 + + ip address add 192.0.2.17/32 dev lo + + # Create SVIs. + vrf_create "vrf-green" + ip link set dev vrf-green up + + ip link add link br1 name vlan10 up master vrf-green type vlan id 10 + + # Replace neighbor to avoid 1 packet which is forwarded in software due + # to "unresolved neigh". + ip neigh replace dev vlan10 192.0.2.1 lladdr $(mac_get $h1) + + ip address add 192.0.2.2/28 dev vlan10 + + bridge vlan add vid 10 dev br1 self + bridge vlan add vid 4001 dev br1 self + + sysctl_set net.ipv4.conf.all.rp_filter 0 +} + +switch_destroy() +{ + sysctl_restore net.ipv4.conf.all.rp_filter + + bridge vlan del vid 4001 dev br1 self + bridge vlan del vid 10 dev br1 self + + ip link del dev vlan10 + + vrf_destroy "vrf-green" + + ip address del 192.0.2.17/32 dev lo + + tc qdisc del dev $swp1 clsact + + bridge vlan del vid 10 dev $swp1 + ip link set dev $swp1 down + ip link set dev $swp1 nomaster + + ip link set dev vx4001 nomaster + + ip link set dev vx4001 down + ip link del dev vx4001 + + ip address del dev $rp1 198.51.100.1/24 + ip link set dev $rp1 down + + ip link set dev br1 down + ip link del dev br1 +} + +vrp2_create() +{ + simple_if_init $rp2 198.51.100.2/24 + + ip route add 192.0.2.17/32 vrf v$rp2 nexthop via 198.51.100.1 +} + +vrp2_destroy() +{ + ip route del 192.0.2.17/32 vrf v$rp2 nexthop via 198.51.100.1 + + simple_if_fini $rp2 198.51.100.2/24 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + rp1=${NETIFS[p3]} + rp2=${NETIFS[p4]} + + vrf_prepare + forwarding_enable + + h1_create + switch_create + + vrp2_create +} + +cleanup() +{ + pre_cleanup + + vrp2_destroy + + switch_destroy + h1_destroy + + forwarding_restore + vrf_cleanup +} + +payload_get() +{ + local dest_mac=$(mac_get vlan4001) + local src_mac=$(mac_get $rp1) + + p=$(: + )"08:"$( : VXLAN flags + )"00:00:00:"$( : VXLAN reserved + )"01:96:41:"$( : VXLAN VNI : 104001 + )"00:"$( : VXLAN reserved + )"$dest_mac:"$( : ETH daddr + )"$src_mac:"$( : ETH saddr + )"08:00:"$( : ETH type + )"45:"$( : IP version + IHL + )"00:"$( : IP TOS + )"00:54:"$( : IP total length + )"3f:49:"$( : IP identification + )"00:00:"$( : IP flags + frag off + )"3f:"$( : IP TTL + )"01:"$( : IP proto + )"50:21:"$( : IP header csum + )"c6:33:64:0a:"$( : IP saddr: 198.51.100.10 + )"c0:00:02:01:"$( : IP daddr: 192.0.2.1 + ) + echo $p +} + +vlan_rif_add() +{ + rifs_occ_t0=$(devlink_resource_occ_get rifs) + + ip link add link br1 name vlan4001 up master vrf-green \ + type vlan id 4001 + + rifs_occ_t1=$(devlink_resource_occ_get rifs) + expected_rifs=$((rifs_occ_t0 + 1)) + + [[ $expected_rifs -eq $rifs_occ_t1 ]] + check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used" +} + +vlan_rif_del() +{ + ip link del dev vlan4001 +} + +vni_fid_map_rif() +{ + local rp1_mac=$(mac_get $rp1) + + RET=0 + + # First add VNI->FID mapping to the FID of VLAN 4001 + bridge vlan add vid 4001 dev vx4001 pvid untagged + + # Add a RIF to the FID with VNI->FID mapping + vlan_rif_add + + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.1 action pass + + payload=$(payload_get) + ip vrf exec v$rp2 $MZ $rp2 -c 10 -d 1msec -b $rp1_mac \ + -B 192.0.2.17 -A 192.0.2.18 \ + -t udp sp=12345,dp=$VXPORT,p=$payload -q + + tc_check_at_least_x_packets "dev $swp1 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add RIF for existing VNI->FID mapping" + + tc filter del dev $swp1 egress + + bridge vlan del vid 4001 dev vx4001 pvid untagged + vlan_rif_del +} + +rif_vni_fid_map() +{ + local rp1_mac=$(mac_get $rp1) + + RET=0 + + # First add a RIF to the FID of VLAN 4001 + vlan_rif_add + + # Add VNI->FID mapping to FID with a RIF + bridge vlan add vid 4001 dev vx4001 pvid untagged + + tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \ + flower skip_sw dst_ip 192.0.2.1 action pass + + payload=$(payload_get) + ip vrf exec v$rp2 $MZ $rp2 -c 10 -d 1msec -b $rp1_mac \ + -B 192.0.2.17 -A 192.0.2.18 \ + -t udp sp=12345,dp=$VXPORT,p=$payload -q + + tc_check_at_least_x_packets "dev $swp1 egress" 101 10 + check_err $? "Packets were not routed in hardware" + + log_test "Add VNI->FID mapping for FID with a RIF" + + tc filter del dev $swp1 egress + + bridge vlan del vid 4001 dev vx4001 pvid untagged + vlan_rif_del +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh b/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh index a95856aafd2aba07086482ae51f560491b8d8d36..6369927e9c378b015842fb600e88e8da621f7d1f 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh @@ -61,3 +61,17 @@ mlxsw_only_on_spectrum() return 1 } + +mlxsw_max_descriptors_get() +{ + local spectrum_rev=$MLXSW_SPECTRUM_REV + + case $spectrum_rev in + 1) echo 81920 ;; + 2) echo 136960 ;; + 3) echo 204800 ;; + 4) echo 220000 ;; + *) echo "Unknown max descriptors for chip revision." > /dev/stderr + return 1 ;; + esac +} diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh deleted file mode 100755 index 82a47b903f928f0b75d223afb5aa76eda8ca8ed4..0000000000000000000000000000000000000000 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh +++ /dev/null @@ -1,480 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 -# -# This test sends 1Gbps of traffic through the switch, into which it then -# injects a burst of traffic and tests that there are no drops. -# -# The 1Gbps stream is created by sending >1Gbps stream from H1. This stream -# ingresses through $swp1, and is forwarded thtrough a small temporary pool to a -# 1Gbps $swp3. -# -# Thus a 1Gbps stream enters $swp4, and is forwarded through a large pool to -# $swp2, and eventually to H2. Since $swp2 is a 1Gbps port as well, no backlog -# is generated. -# -# At this point, a burst of traffic is forwarded from H3. This enters $swp5, is -# forwarded to $swp2, which is fully subscribed by the 1Gbps stream. The -# expectation is that the burst is wholly absorbed by the large pool and no -# drops are caused. After the burst, there should be a backlog that is hard to -# get rid of, because $sw2 is fully subscribed. But because each individual -# packet is scheduled soon after getting enqueued, SLL and HLL do not impact the -# test. -# -# +-----------------------+ +-----------------------+ -# | H1 | | H3 | -# | + $h1.111 | | $h3.111 + | -# | | 192.0.2.33/28 | | 192.0.2.35/28 | | -# | | | | | | -# | + $h1 | | $h3 + | -# +---|-------------------+ +--------------------+ +------------------|----+ -# | | | | -# +---|----------------------|--------------------|----------------------|----+ -# | + $swp1 $swp3 + + $swp4 $swp5 | | -# | | iPOOL1 iPOOL0 | | iPOOL2 iPOOL2 | | -# | | ePOOL4 ePOOL5 | | ePOOL4 ePOOL4 | | -# | | 1Gbps | | 1Gbps | | -# | +-|----------------------|-+ +-|----------------------|-+ | -# | | + $swp1.111 $swp3.111 + | | + $swp4.111 $swp5.111 + | | -# | | | | | | -# | | BR1 | | BR2 | | -# | | | | | | -# | | | | + $swp2.111 | | -# | +--------------------------+ +---------|----------------+ | -# | | | -# | iPOOL0: 500KB dynamic | | -# | iPOOL1: 500KB dynamic | | -# | iPOOL2: 10MB dynamic + $swp2 | -# | ePOOL4: 500KB dynamic | iPOOL0 | -# | ePOOL5: 500KB dnamic | ePOOL6 | -# | ePOOL6: 10MB dynamic | 1Gbps | -# +-------------------------------------------------------|-------------------+ -# | -# +---|-------------------+ -# | + $h2 H2 | -# | | 1Gbps | -# | | | -# | + $h2.111 | -# | 192.0.2.34/28 | -# +-----------------------+ -# -# iPOOL0+ePOOL4 are helper pools for control traffic etc. -# iPOOL1+ePOOL5 are helper pools for modeling the 1Gbps stream -# iPOOL2+ePOOL6 are pools for soaking the burst traffic - -ALL_TESTS=" - ping_ipv4 - test_8K - test_800 -" - -lib_dir=$(dirname $0)/../../../net/forwarding - -NUM_NETIFS=8 -source $lib_dir/lib.sh -source $lib_dir/devlink_lib.sh -source qos_lib.sh -source mlxsw_lib.sh - -_1KB=1000 -_500KB=$((500 * _1KB)) -_1MB=$((1000 * _1KB)) - -# The failure mode that this specifically tests is exhaustion of descriptor -# buffer. The point is to produce a burst that shared buffer should be able -# to accommodate, but produce it with small enough packets that the machine -# runs out of the descriptor buffer space with default configuration. -# -# The machine therefore needs to be able to produce line rate with as small -# packets as possible, and at the same time have large enough buffer that -# when filled with these small packets, it runs out of descriptors. -# Spectrum-2 is very close, but cannot perform this test. Therefore use -# Spectrum-3 as a minimum, and permit larger burst size, and therefore -# larger packets, to reduce spurious failures. -# -mlxsw_only_on_spectrum 3+ || exit - -BURST_SIZE=$((50000000)) -POOL_SIZE=$BURST_SIZE - -h1_create() -{ - simple_if_init $h1 - mtu_set $h1 10000 - - vlan_create $h1 111 v$h1 192.0.2.33/28 - ip link set dev $h1.111 type vlan egress-qos-map 0:1 -} - -h1_destroy() -{ - vlan_destroy $h1 111 - - mtu_restore $h1 - simple_if_fini $h1 -} - -h2_create() -{ - simple_if_init $h2 - mtu_set $h2 10000 - ethtool -s $h2 speed 1000 autoneg off - - vlan_create $h2 111 v$h2 192.0.2.34/28 -} - -h2_destroy() -{ - vlan_destroy $h2 111 - - ethtool -s $h2 autoneg on - mtu_restore $h2 - simple_if_fini $h2 -} - -h3_create() -{ - simple_if_init $h3 - mtu_set $h3 10000 - - vlan_create $h3 111 v$h3 192.0.2.35/28 -} - -h3_destroy() -{ - vlan_destroy $h3 111 - - mtu_restore $h3 - simple_if_fini $h3 -} - -switch_create() -{ - # pools - # ----- - - devlink_pool_size_thtype_save 0 - devlink_pool_size_thtype_save 4 - devlink_pool_size_thtype_save 1 - devlink_pool_size_thtype_save 5 - devlink_pool_size_thtype_save 2 - devlink_pool_size_thtype_save 6 - - devlink_port_pool_th_save $swp1 1 - devlink_port_pool_th_save $swp2 6 - devlink_port_pool_th_save $swp3 5 - devlink_port_pool_th_save $swp4 2 - devlink_port_pool_th_save $swp5 2 - - devlink_tc_bind_pool_th_save $swp1 1 ingress - devlink_tc_bind_pool_th_save $swp2 1 egress - devlink_tc_bind_pool_th_save $swp3 1 egress - devlink_tc_bind_pool_th_save $swp4 1 ingress - devlink_tc_bind_pool_th_save $swp5 1 ingress - - # Control traffic pools. Just reduce the size. - devlink_pool_size_thtype_set 0 dynamic $_500KB - devlink_pool_size_thtype_set 4 dynamic $_500KB - - # Stream modeling pools. - devlink_pool_size_thtype_set 1 dynamic $_500KB - devlink_pool_size_thtype_set 5 dynamic $_500KB - - # Burst soak pools. - devlink_pool_size_thtype_set 2 static $POOL_SIZE - devlink_pool_size_thtype_set 6 static $POOL_SIZE - - # $swp1 - # ----- - - ip link set dev $swp1 up - mtu_set $swp1 10000 - vlan_create $swp1 111 - ip link set dev $swp1.111 type vlan ingress-qos-map 0:0 1:1 - - devlink_port_pool_th_set $swp1 1 16 - devlink_tc_bind_pool_th_set $swp1 1 ingress 1 16 - - # Configure qdisc... - tc qdisc replace dev $swp1 root handle 1: \ - ets bands 8 strict 8 priomap 7 6 - # ... so that we can assign prio1 traffic to PG1. - dcb buffer set dev $swp1 prio-buffer all:0 1:1 - - # $swp2 - # ----- - - ip link set dev $swp2 up - mtu_set $swp2 10000 - ethtool -s $swp2 speed 1000 autoneg off - vlan_create $swp2 111 - ip link set dev $swp2.111 type vlan egress-qos-map 0:0 1:1 - - devlink_port_pool_th_set $swp2 6 $POOL_SIZE - devlink_tc_bind_pool_th_set $swp2 1 egress 6 $POOL_SIZE - - # prio 0->TC0 (band 7), 1->TC1 (band 6) - tc qdisc replace dev $swp2 root handle 1: \ - ets bands 8 strict 8 priomap 7 6 - - # $swp3 - # ----- - - ip link set dev $swp3 up - mtu_set $swp3 10000 - ethtool -s $swp3 speed 1000 autoneg off - vlan_create $swp3 111 - ip link set dev $swp3.111 type vlan egress-qos-map 0:0 1:1 - - devlink_port_pool_th_set $swp3 5 16 - devlink_tc_bind_pool_th_set $swp3 1 egress 5 16 - - # prio 0->TC0 (band 7), 1->TC1 (band 6) - tc qdisc replace dev $swp3 root handle 1: \ - ets bands 8 strict 8 priomap 7 6 - - # $swp4 - # ----- - - ip link set dev $swp4 up - mtu_set $swp4 10000 - ethtool -s $swp4 speed 1000 autoneg off - vlan_create $swp4 111 - ip link set dev $swp4.111 type vlan ingress-qos-map 0:0 1:1 - - devlink_port_pool_th_set $swp4 2 $POOL_SIZE - devlink_tc_bind_pool_th_set $swp4 1 ingress 2 $POOL_SIZE - - # Configure qdisc... - tc qdisc replace dev $swp4 root handle 1: \ - ets bands 8 strict 8 priomap 7 6 - # ... so that we can assign prio1 traffic to PG1. - dcb buffer set dev $swp4 prio-buffer all:0 1:1 - - # $swp5 - # ----- - - ip link set dev $swp5 up - mtu_set $swp5 10000 - vlan_create $swp5 111 - ip link set dev $swp5.111 type vlan ingress-qos-map 0:0 1:1 - - devlink_port_pool_th_set $swp5 2 $POOL_SIZE - devlink_tc_bind_pool_th_set $swp5 1 ingress 2 $POOL_SIZE - - # Configure qdisc... - tc qdisc replace dev $swp5 root handle 1: \ - ets bands 8 strict 8 priomap 7 6 - # ... so that we can assign prio1 traffic to PG1. - dcb buffer set dev $swp5 prio-buffer all:0 1:1 - - # bridges - # ------- - - ip link add name br1 type bridge vlan_filtering 0 - ip link set dev $swp1.111 master br1 - ip link set dev $swp3.111 master br1 - ip link set dev br1 up - - ip link add name br2 type bridge vlan_filtering 0 - ip link set dev $swp2.111 master br2 - ip link set dev $swp4.111 master br2 - ip link set dev $swp5.111 master br2 - ip link set dev br2 up -} - -switch_destroy() -{ - # Do this first so that we can reset the limits to values that are only - # valid for the original static / dynamic setting. - devlink_pool_size_thtype_restore 6 - devlink_pool_size_thtype_restore 5 - devlink_pool_size_thtype_restore 4 - devlink_pool_size_thtype_restore 2 - devlink_pool_size_thtype_restore 1 - devlink_pool_size_thtype_restore 0 - - # bridges - # ------- - - ip link set dev br2 down - ip link set dev $swp5.111 nomaster - ip link set dev $swp4.111 nomaster - ip link set dev $swp2.111 nomaster - ip link del dev br2 - - ip link set dev br1 down - ip link set dev $swp3.111 nomaster - ip link set dev $swp1.111 nomaster - ip link del dev br1 - - # $swp5 - # ----- - - dcb buffer set dev $swp5 prio-buffer all:0 - tc qdisc del dev $swp5 root - - devlink_tc_bind_pool_th_restore $swp5 1 ingress - devlink_port_pool_th_restore $swp5 2 - - vlan_destroy $swp5 111 - mtu_restore $swp5 - ip link set dev $swp5 down - - # $swp4 - # ----- - - dcb buffer set dev $swp4 prio-buffer all:0 - tc qdisc del dev $swp4 root - - devlink_tc_bind_pool_th_restore $swp4 1 ingress - devlink_port_pool_th_restore $swp4 2 - - vlan_destroy $swp4 111 - ethtool -s $swp4 autoneg on - mtu_restore $swp4 - ip link set dev $swp4 down - - # $swp3 - # ----- - - tc qdisc del dev $swp3 root - - devlink_tc_bind_pool_th_restore $swp3 1 egress - devlink_port_pool_th_restore $swp3 5 - - vlan_destroy $swp3 111 - ethtool -s $swp3 autoneg on - mtu_restore $swp3 - ip link set dev $swp3 down - - # $swp2 - # ----- - - tc qdisc del dev $swp2 root - - devlink_tc_bind_pool_th_restore $swp2 1 egress - devlink_port_pool_th_restore $swp2 6 - - vlan_destroy $swp2 111 - ethtool -s $swp2 autoneg on - mtu_restore $swp2 - ip link set dev $swp2 down - - # $swp1 - # ----- - - dcb buffer set dev $swp1 prio-buffer all:0 - tc qdisc del dev $swp1 root - - devlink_tc_bind_pool_th_restore $swp1 1 ingress - devlink_port_pool_th_restore $swp1 1 - - vlan_destroy $swp1 111 - mtu_restore $swp1 - ip link set dev $swp1 down -} - -setup_prepare() -{ - h1=${NETIFS[p1]} - swp1=${NETIFS[p2]} - - swp2=${NETIFS[p3]} - h2=${NETIFS[p4]} - - swp3=${NETIFS[p5]} - swp4=${NETIFS[p6]} - - swp5=${NETIFS[p7]} - h3=${NETIFS[p8]} - - h2mac=$(mac_get $h2) - - vrf_prepare - - h1_create - h2_create - h3_create - switch_create -} - -cleanup() -{ - pre_cleanup - - switch_destroy - h3_destroy - h2_destroy - h1_destroy - - vrf_cleanup -} - -ping_ipv4() -{ - ping_test $h1 192.0.2.34 " h1->h2" - ping_test $h3 192.0.2.34 " h3->h2" -} - -__test_qos_burst() -{ - local pktsize=$1; shift - - RET=0 - - start_traffic_pktsize $pktsize $h1.111 192.0.2.33 192.0.2.34 $h2mac - sleep 1 - - local q0=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1) - ((q0 == 0)) - check_err $? "Transmit queue non-zero?" - - local d0=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1) - - local cell_size=$(devlink_cell_size_get) - local cells=$((BURST_SIZE / cell_size)) - # Each packet is $pktsize of payload + headers. - local pkt_cells=$(((pktsize + 50 + cell_size - 1) / cell_size)) - # How many packets can we admit: - local pkts=$((cells / pkt_cells)) - - $MZ $h3 -p $pktsize -Q 1:111 -A 192.0.2.35 -B 192.0.2.34 \ - -a own -b $h2mac -c $pkts -t udp -q - sleep 1 - - local d1=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1) - ((d1 == d0)) - check_err $? "Drops seen on egress port: $d0 -> $d1 ($((d1 - d0)))" - - # Check that the queue is somewhat close to the burst size This - # makes sure that the lack of drops above was not due to port - # undersubscribtion. - local q0=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1) - local qe=$((90 * BURST_SIZE / 100)) - ((q0 > qe)) - check_err $? "Queue size expected >$qe, got $q0" - - stop_traffic - sleep 2 - - log_test "Burst: absorb $pkts ${pktsize}-B packets" -} - -test_8K() -{ - __test_qos_burst 8000 -} - -test_800() -{ - __test_qos_burst 800 -} - -bail_on_lldpad - -trap cleanup EXIT -setup_prepare -setup_wait -tests_run - -exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh index e9f8718af979aa84a302419d134d8747ed29d2c1..690d8daa71b4946f8feb5d9d24a0ae9c14923ef0 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh @@ -130,7 +130,8 @@ switch_create() ip link set dev $swp3 up mtu_set $swp3 10000 - ethtool -s $swp3 speed 1000 autoneg off + tc qdisc replace dev $swp3 root handle 101: tbf rate 1gbit \ + burst 128K limit 1G vlan_create $swp1 111 vlan_create $swp2 222 @@ -193,7 +194,7 @@ switch_destroy() vlan_destroy $swp2 222 vlan_destroy $swp1 111 - ethtool -s $swp3 autoneg on + tc qdisc del dev $swp3 root handle 101: mtu_restore $swp3 ip link set dev $swp3 down lldptool -T -i $swp3 -V ETS-CFG up2tc=0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0 diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh new file mode 100755 index 0000000000000000000000000000000000000000..5ac4f795e333b49229b5bc42c618057e635424da --- /dev/null +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# This test sends many small packets (size is less than cell size) through the +# switch. A shaper is used in $swp2, so the traffic is limited there. Packets +# are queued till they will be sent. +# +# The idea is to verify that the switch can handle at least 85% of maximum +# supported descrpitors by hardware. Then, we verify that the driver configures +# firmware to allow infinite size of egress descriptor pool, and does not use a +# lower limitation. Increase the size of the relevant pools such that the pool's +# size does not limit the traffic. + +# +-----------------------+ +# | H1 | +# | + $h1.111 | +# | | 192.0.2.33/28 | +# | | | +# | + $h1 | +# +---|-------------------+ +# | +# +---|-----------------------------+ +# | + $swp1 | +# | | iPOOL1 | +# | | | +# | +-|------------------------+ | +# | | + $swp1.111 | | +# | | | | +# | | BR1 | | +# | | | | +# | | + $swp2.111 | | +# | +-|------------------------+ | +# | | | +# | + $swp2 | +# | | ePOOL6 | +# | | 1mbit | +# +---+-----------------------------+ +# | +# +---|-------------------+ +# | + $h2 H2 | +# | | | +# | + $h2.111 | +# | 192.0.2.34/28 | +# +-----------------------+ +# + +ALL_TESTS=" + ping_ipv4 + max_descriptors +" + +lib_dir=$(dirname $0)/../../../net/forwarding + +NUM_NETIFS=4 +source $lib_dir/lib.sh +source $lib_dir/devlink_lib.sh +source mlxsw_lib.sh + +MAX_POOL_SIZE=$(devlink_pool_size_get) +SHAPER_RATE=1mbit + +# The current TBF qdisc interface does not allow us to configure the shaper to +# flat zero. The ASIC shaper is guaranteed to work with a granularity of +# 200Mbps. On Spectrum-2, writing a value close to zero instead of zero works +# well, but the performance on Spectrum-1 is unpredictable. Thus, do not run the +# test on Spectrum-1. +mlxsw_only_on_spectrum 2+ || exit + +h1_create() +{ + simple_if_init $h1 + + vlan_create $h1 111 v$h1 192.0.2.33/28 + ip link set dev $h1.111 type vlan egress-qos-map 0:1 +} + +h1_destroy() +{ + vlan_destroy $h1 111 + + simple_if_fini $h1 +} + +h2_create() +{ + simple_if_init $h2 + + vlan_create $h2 111 v$h2 192.0.2.34/28 +} + +h2_destroy() +{ + vlan_destroy $h2 111 + + simple_if_fini $h2 +} + +switch_create() +{ + # pools + # ----- + + devlink_pool_size_thtype_save 1 + devlink_pool_size_thtype_save 6 + + devlink_port_pool_th_save $swp1 1 + devlink_port_pool_th_save $swp2 6 + + devlink_tc_bind_pool_th_save $swp1 1 ingress + devlink_tc_bind_pool_th_save $swp2 1 egress + + devlink_pool_size_thtype_set 1 dynamic $MAX_POOL_SIZE + devlink_pool_size_thtype_set 6 static $MAX_POOL_SIZE + + # $swp1 + # ----- + + ip link set dev $swp1 up + vlan_create $swp1 111 + ip link set dev $swp1.111 type vlan ingress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp1 1 16 + devlink_tc_bind_pool_th_set $swp1 1 ingress 1 16 + + tc qdisc replace dev $swp1 root handle 1: \ + ets bands 8 strict 8 priomap 7 6 + dcb buffer set dev $swp1 prio-buffer all:0 1:1 + + # $swp2 + # ----- + + ip link set dev $swp2 up + vlan_create $swp2 111 + ip link set dev $swp2.111 type vlan egress-qos-map 0:0 1:1 + + devlink_port_pool_th_set $swp2 6 $MAX_POOL_SIZE + devlink_tc_bind_pool_th_set $swp2 1 egress 6 $MAX_POOL_SIZE + + tc qdisc replace dev $swp2 root handle 1: tbf rate $SHAPER_RATE \ + burst 128K limit 500M + tc qdisc replace dev $swp2 parent 1:1 handle 11: \ + ets bands 8 strict 8 priomap 7 6 + + # bridge + # ------ + + ip link add name br1 type bridge vlan_filtering 0 + ip link set dev $swp1.111 master br1 + ip link set dev br1 up + + ip link set dev $swp2.111 master br1 +} + +switch_destroy() +{ + # Do this first so that we can reset the limits to values that are only + # valid for the original static / dynamic setting. + devlink_pool_size_thtype_restore 6 + devlink_pool_size_thtype_restore 1 + + # bridge + # ------ + + ip link set dev $swp2.111 nomaster + + ip link set dev br1 down + ip link set dev $swp1.111 nomaster + ip link del dev br1 + + # $swp2 + # ----- + + tc qdisc del dev $swp2 parent 1:1 handle 11: + tc qdisc del dev $swp2 root + + devlink_tc_bind_pool_th_restore $swp2 1 egress + devlink_port_pool_th_restore $swp2 6 + + vlan_destroy $swp2 111 + ip link set dev $swp2 down + + # $swp1 + # ----- + + dcb buffer set dev $swp1 prio-buffer all:0 + tc qdisc del dev $swp1 root + + devlink_tc_bind_pool_th_restore $swp1 1 ingress + devlink_port_pool_th_restore $swp1 1 + + vlan_destroy $swp1 111 + ip link set dev $swp1 down +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + h2mac=$(mac_get $h2) + + vrf_prepare + + h1_create + h2_create + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + h2_destroy + h1_destroy + + vrf_cleanup +} + +ping_ipv4() +{ + ping_test $h1 192.0.2.34 " h1->h2" +} + +percentage_used() +{ + local num_packets=$1; shift + local max_packets=$1; shift + + bc <<< " + scale=2 + 100 * $num_packets / $max_packets + " +} + +max_descriptors() +{ + local cell_size=$(devlink_cell_size_get) + local exp_perc_used=85 + local max_descriptors + local pktsize=30 + + RET=0 + + max_descriptors=$(mlxsw_max_descriptors_get) || exit 1 + + local d0=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1) + + log_info "Send many small packets, packet size = $pktsize bytes" + start_traffic_pktsize $pktsize $h1.111 192.0.2.33 192.0.2.34 $h2mac + + # Sleep to wait for congestion. + sleep 5 + + local d1=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1) + ((d1 == d0)) + check_err $? "Drops seen on egress port: $d0 -> $d1 ($((d1 - d0)))" + + # Check how many packets the switch can handle, the limitation is + # maximum descriptors. + local pkts_bytes=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1) + local pkts_num=$((pkts_bytes / cell_size)) + local perc_used=$(percentage_used $pkts_num $max_descriptors) + + check_err $(bc <<< "$perc_used < $exp_perc_used") \ + "Expected > $exp_perc_used% of descriptors, handle $perc_used%" + + stop_traffic + sleep 1 + + log_test "Maximum descriptors usage. The percentage used is $perc_used%" +} + +trap cleanup EXIT +setup_prepare +setup_wait +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh index 8f164c80e2154ec4ea5ac24d1fae6e20e72a18a2..c8e55fa916609b3b3ec7d12c5549b2487bb425d7 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh @@ -129,9 +129,10 @@ switch_create() vlan_create $swp2 111 vlan_create $swp3 111 - ethtool -s $swp3 speed 1000 autoneg off - tc qdisc replace dev $swp3 root handle 3: \ - prio bands 8 priomap 7 7 7 7 7 7 7 7 + tc qdisc replace dev $swp3 root handle 3: tbf rate 1gbit \ + burst 128K limit 1G + tc qdisc replace dev $swp3 parent 3:3 handle 33: \ + prio bands 8 priomap 7 7 7 7 7 7 7 7 ip link add name br1 type bridge vlan_filtering 0 ip link set dev br1 up @@ -172,8 +173,8 @@ switch_destroy() ip link del dev br111 ip link del dev br1 + tc qdisc del dev $swp3 parent 3:3 handle 33: tc qdisc del dev $swp3 root handle 3: - ethtool -s $swp3 autoneg on vlan_destroy $swp3 111 vlan_destroy $swp2 111 diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh index af64bc9ea8ab7d20f3ef7e19fed735f36f2d94d7..ceaa76b17a43c18e31ef13698f1ad3abbfddac10 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh @@ -15,13 +15,15 @@ ALL_TESTS=" ets_test_dwrr " +PARENT="parent 3:3" + switch_create() { - ets_switch_create - # Create a bottleneck so that the DWRR process can kick in. - ethtool -s $h2 speed 1000 autoneg off - ethtool -s $swp2 speed 1000 autoneg off + tc qdisc replace dev $swp2 root handle 3: tbf rate 1gbit \ + burst 128K limit 1G + + ets_switch_create # Set the ingress quota high and use the three egress TCs to limit the # amount of traffic that is admitted to the shared buffers. This makes @@ -55,10 +57,9 @@ switch_destroy() devlink_tc_bind_pool_th_restore $swp1 0 ingress devlink_port_pool_th_restore $swp1 0 - ethtool -s $swp2 autoneg on - ethtool -s $h2 autoneg on - ets_switch_destroy + + tc qdisc del dev $swp2 root handle 3: } # Callback from sch_ets_tests.sh diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh index f260f01db0e8075128aa74c9e3f689a909bcca48..45b41b8f323224c211aeb26e24f4ce079409b851 100644 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh @@ -135,14 +135,16 @@ h2_create() # cause packets to fail to queue up at $swp3 due to shared buffer # quotas, and the test to spuriously fail. # - # Prevent this by setting the speed of $h2 to 1Gbps. + # Prevent this by adding a shaper which limits the traffic in $h2 to + # 1Gbps. - ethtool -s $h2 speed 1000 autoneg off + tc qdisc replace dev $h2 root handle 10: tbf rate 1gbit \ + burst 128K limit 1G } h2_destroy() { - ethtool -s $h2 autoneg on + tc qdisc del dev $h2 root handle 10: tc qdisc del dev $h2 clsact host_destroy $h2 } @@ -150,12 +152,10 @@ h2_destroy() h3_create() { host_create $h3 3 - ethtool -s $h3 speed 1000 autoneg off } h3_destroy() { - ethtool -s $h3 autoneg on host_destroy $h3 } @@ -199,8 +199,9 @@ switch_create() done done - for intf in $swp2 $swp3 $swp4 $swp5; do - ethtool -s $intf speed 1000 autoneg off + for intf in $swp3 $swp4; do + tc qdisc replace dev $intf root handle 1: tbf rate 1gbit \ + burst 128K limit 1G done ip link set dev br1_10 up @@ -220,15 +221,13 @@ switch_destroy() devlink_port_pool_th_restore $swp3 8 - tc qdisc del dev $swp3 root 2>/dev/null - ip link set dev br2_11 down ip link set dev br2_10 down ip link set dev br1_11 down ip link set dev br1_10 down - for intf in $swp5 $swp4 $swp3 $swp2; do - ethtool -s $intf autoneg on + for intf in $swp4 $swp3; do + tc qdisc del dev $intf root handle 1: done for intf in $swp5 $swp3 $swp2 $swp4 $swp1; do @@ -536,7 +535,7 @@ do_red_test() check_err $? "backlog $backlog / $limit Got $pct% marked packets, expected == 0." local diff=$((limit - backlog)) pct=$((100 * diff / limit)) - ((0 <= pct && pct <= 10)) + ((-10 <= pct && pct <= 10)) check_err $? "backlog $backlog / $limit expected <= 10% distance" log_test "TC $((vlan - 10)): RED backlog > limit" diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh index 7a73057206cd090af039462be7c8e43bcc37a317..0d01c7cd82a182ca73c09329404ce54d048279e2 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh @@ -25,7 +25,7 @@ BACKLOG2=500000 install_root_qdisc() { - tc qdisc add dev $swp3 root handle 10: $QDISC \ + tc qdisc add dev $swp3 parent 1: handle 10: $QDISC \ bands 8 priomap 7 6 5 4 3 2 1 0 } @@ -67,7 +67,7 @@ uninstall_qdisc_tc1() uninstall_root_qdisc() { - tc qdisc del dev $swp3 root + tc qdisc del dev $swp3 parent 1: } uninstall_qdisc() diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh index 501d192529ac0fa2f44708a876214d34904c7840..860205338e6fb6ad940488a75eddb7e32c3e08f0 100755 --- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh +++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh @@ -18,7 +18,7 @@ install_qdisc() { local -a args=("$@") - tc qdisc add dev $swp3 root handle 108: red \ + tc qdisc add dev $swp3 parent 1: handle 108: red \ limit 1000000 min $BACKLOG max $((BACKLOG + 1)) \ probability 1.0 avpkt 8000 burst 38 "${args[@]}" sleep 1 @@ -26,7 +26,7 @@ install_qdisc() uninstall_qdisc() { - tc qdisc del dev $swp3 root + tc qdisc del dev $swp3 parent 1: } ecn_test() diff --git a/tools/testing/selftests/drivers/net/ocelot/psfp.sh b/tools/testing/selftests/drivers/net/ocelot/psfp.sh index 5a5cee92c665a935755eff3803cedbac3fdd1f96..bed748dde4b066daeb60eaba7d13907a1b682f1f 100755 --- a/tools/testing/selftests/drivers/net/ocelot/psfp.sh +++ b/tools/testing/selftests/drivers/net/ocelot/psfp.sh @@ -181,7 +181,7 @@ setup_prepare() # Set up swp1 as a master PHC for h1, synchronized to the local # CLOCK_REALTIME. - phc2sys_start ${swp1} ${UDS_ADDRESS_SWP1} + phc2sys_start ${UDS_ADDRESS_SWP1} # Assumption true for LS1028A: h1 and h2 use the same PHC. So by # synchronizing h1 to swp1 via PTP, h2 is also implicitly synchronized diff --git a/tools/testing/selftests/drivers/net/team/Makefile b/tools/testing/selftests/drivers/net/team/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..642d8df1c137b4a6fce03ccb5881cc0e30aa54bc --- /dev/null +++ b/tools/testing/selftests/drivers/net/team/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for net selftests + +TEST_PROGS := dev_addr_lists.sh + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/net/team/config b/tools/testing/selftests/drivers/net/team/config new file mode 100644 index 0000000000000000000000000000000000000000..265b6882cc21ed0c285ae9f37f9282bfb2e440d1 --- /dev/null +++ b/tools/testing/selftests/drivers/net/team/config @@ -0,0 +1,3 @@ +CONFIG_NET_TEAM=y +CONFIG_NET_TEAM_MODE_LOADBALANCE=y +CONFIG_MACVLAN=y diff --git a/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh new file mode 100755 index 0000000000000000000000000000000000000000..debda72629564b02d0313435bfbdcef334a3489a --- /dev/null +++ b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test team device handling of addr lists (dev->uc, mc) +# + +ALL_TESTS=" + team_cleanup +" + +REQUIRE_MZ=no +NUM_NETIFS=0 +lib_dir=$(dirname "$0") +source "$lib_dir"/../../../net/forwarding/lib.sh + +source "$lib_dir"/../bonding/lag_lib.sh + + +destroy() +{ + local ifnames=(dummy0 dummy1 team0 mv0) + local ifname + + for ifname in "${ifnames[@]}"; do + ip link del "$ifname" &>/dev/null + done +} + +cleanup() +{ + pre_cleanup + + destroy +} + + +team_cleanup() +{ + RET=0 + + test_LAG_cleanup "team" "lacp" +} + + +require_command teamd + +trap cleanup EXIT + +tests_run + +exit "$EXIT_STATUS" diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc new file mode 100644 index 0000000000000000000000000000000000000000..fc1daac7f066826d4381f0e76149fcc75e1a1c2e --- /dev/null +++ b/tools/testing/selftests/ftrace/test.d/dynevent/eprobes_syntax_errors.tc @@ -0,0 +1,27 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# description: Event probe event parser error log check +# requires: dynamic_events events/syscalls/sys_enter_openat "<attached-group>.<attached-event> [<args>]":README error_log + +check_error() { # command-with-error-pos-by-^ + ftrace_errlog_check 'event_probe' "$1" 'dynamic_events' +} + +check_error 'e ^a.' # NO_EVENT_INFO +check_error 'e ^.b' # NO_EVENT_INFO +check_error 'e ^a.b' # BAD_ATTACH_EVENT +check_error 'e syscalls/sys_enter_openat ^foo' # BAD_ATTACH_ARG +check_error 'e:^/bar syscalls/sys_enter_openat' # NO_GROUP_NAME +check_error 'e:^12345678901234567890123456789012345678901234567890123456789012345/bar syscalls/sys_enter_openat' # GROUP_TOO_LONG + +check_error 'e:^foo.1/bar syscalls/sys_enter_openat' # BAD_GROUP_NAME +check_error 'e:^ syscalls/sys_enter_openat' # NO_EVENT_NAME +check_error 'e:foo/^12345678901234567890123456789012345678901234567890123456789012345 syscalls/sys_enter_openat' # EVENT_TOO_LONG +check_error 'e:foo/^bar.1 syscalls/sys_enter_openat' # BAD_EVENT_NAME + +check_error 'e:foo/bar syscalls/sys_enter_openat arg=^dfd' # BAD_FETCH_ARG +check_error 'e:foo/bar syscalls/sys_enter_openat ^arg=$foo' # BAD_ATTACH_ARG + +check_error 'e:foo/bar syscalls/sys_enter_openat if ^' # NO_EP_FILTER + +exit 0 diff --git a/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc b/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc index 3145b0f1835c37f6e0ce55b33e933ac08bc7cdfd..8d26d5505808b6d116a4618b420f7ff00d03f5c9 100644 --- a/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc +++ b/tools/testing/selftests/ftrace/test.d/ftrace/func_event_triggers.tc @@ -85,7 +85,7 @@ run_enable_disable() { echo $check_disable > $EVENT_ENABLE done sleep $SLEEP_TIME - echo " make sure it's still works" + echo " make sure it still works" test_event_enabled $check_enable_star reset_ftrace_filter diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c index f8c43ce8fe663ed1a3839e20d1630bac85648eda..c6b8f32990c87587551b14577b200e62c56af223 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) /* * If res is non-zero, we either requeued the waiter or hit an * error, break out and handle it. If it is zero, then the - * signal may have hit before the the waiter was blocked on f1. + * signal may have hit before the waiter was blocked on f1. * Try again. */ if (res > 0) { diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index d625a3f837806a3f03d76671f03d87d0c8e5c76c..2f0d705db9dba5ad0d9923658908d934f4024a93 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +/aarch64/aarch32_id_regs /aarch64/arch_timer /aarch64/debug-exceptions /aarch64/get-reg-list @@ -28,6 +29,7 @@ /x86_64/max_vcpuid_cap_test /x86_64/mmio_warning_test /x86_64/monitor_mwait_test +/x86_64/nested_exceptions_test /x86_64/nx_huge_pages_test /x86_64/platform_info_test /x86_64/pmu_event_filter_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 4c122f1b17378011beca374acdc7035bbe76e5ab..0172eb6cb6eee228cd2b3bea864abdb561f4346c 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -48,6 +48,8 @@ LIBKVM += lib/rbtree.c LIBKVM += lib/sparsebit.c LIBKVM += lib/test_util.c +LIBKVM_STRING += lib/string_override.c + LIBKVM_x86_64 += lib/x86_64/apic.c LIBKVM_x86_64 += lib/x86_64/handlers.S LIBKVM_x86_64 += lib/x86_64/perf_test_util.c @@ -89,6 +91,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/kvm_clock_test TEST_GEN_PROGS_x86_64 += x86_64/kvm_pv_test TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test TEST_GEN_PROGS_x86_64 += x86_64/monitor_mwait_test +TEST_GEN_PROGS_x86_64 += x86_64/nested_exceptions_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test TEST_GEN_PROGS_x86_64 += x86_64/pmu_event_filter_test TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id @@ -144,6 +147,7 @@ TEST_GEN_PROGS_x86_64 += system_counter_offset_test # Compiled outputs used by test targets TEST_GEN_PROGS_EXTENDED_x86_64 += x86_64/nx_huge_pages_test +TEST_GEN_PROGS_aarch64 += aarch64/aarch32_id_regs TEST_GEN_PROGS_aarch64 += aarch64/arch_timer TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list @@ -220,7 +224,8 @@ LIBKVM_C := $(filter %.c,$(LIBKVM)) LIBKVM_S := $(filter %.S,$(LIBKVM)) LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C)) LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S)) -LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) +LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING)) +LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) EXTRA_CLEAN += $(LIBKVM_OBJS) cscope.* @@ -231,6 +236,12 @@ $(LIBKVM_C_OBJ): $(OUTPUT)/%.o: %.c $(LIBKVM_S_OBJ): $(OUTPUT)/%.o: %.S $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@ +# Compile the string overrides as freestanding to prevent the compiler from +# generating self-referential code, e.g. without "freestanding" the compiler may +# "optimize" memcmp() by invoking memcmp(), thus causing infinite recursion. +$(LIBKVM_STRING_OBJ): $(OUTPUT)/%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -ffreestanding $< -o $@ + x := $(shell mkdir -p $(sort $(dir $(TEST_GEN_PROGS)))) $(TEST_GEN_PROGS): $(LIBKVM_OBJS) $(TEST_GEN_PROGS_EXTENDED): $(LIBKVM_OBJS) diff --git a/tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c b/tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c new file mode 100644 index 0000000000000000000000000000000000000000..6f9c1f19c7f6469536d7bd5131612186b8d20e06 --- /dev/null +++ b/tools/testing/selftests/kvm/aarch64/aarch32_id_regs.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * aarch32_id_regs - Test for ID register behavior on AArch64-only systems + * + * Copyright (c) 2022 Google LLC. + * + * Test that KVM handles the AArch64 views of the AArch32 ID registers as RAZ + * and WI from userspace. + */ + +#include <stdint.h> + +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" + +#define BAD_ID_REG_VAL 0x1badc0deul + +#define GUEST_ASSERT_REG_RAZ(reg) GUEST_ASSERT_EQ(read_sysreg_s(reg), 0) + +static void guest_main(void) +{ + GUEST_ASSERT_REG_RAZ(SYS_ID_PFR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_PFR1_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_DFR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_AFR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR1_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR2_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR3_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR1_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR2_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR3_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR4_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR5_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR4_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_ISAR6_EL1); + GUEST_ASSERT_REG_RAZ(SYS_MVFR0_EL1); + GUEST_ASSERT_REG_RAZ(SYS_MVFR1_EL1); + GUEST_ASSERT_REG_RAZ(SYS_MVFR2_EL1); + GUEST_ASSERT_REG_RAZ(sys_reg(3, 0, 0, 3, 3)); + GUEST_ASSERT_REG_RAZ(SYS_ID_PFR2_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_DFR1_EL1); + GUEST_ASSERT_REG_RAZ(SYS_ID_MMFR5_EL1); + GUEST_ASSERT_REG_RAZ(sys_reg(3, 0, 0, 3, 7)); + + GUEST_DONE(); +} + +static void test_guest_raz(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + vcpu_run(vcpu); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } +} + +static uint64_t raz_wi_reg_ids[] = { + KVM_ARM64_SYS_REG(SYS_ID_PFR0_EL1), + KVM_ARM64_SYS_REG(SYS_ID_PFR1_EL1), + KVM_ARM64_SYS_REG(SYS_ID_DFR0_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR0_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR1_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR2_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR3_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR0_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR1_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR2_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR3_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR4_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR5_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR4_EL1), + KVM_ARM64_SYS_REG(SYS_ID_ISAR6_EL1), + KVM_ARM64_SYS_REG(SYS_MVFR0_EL1), + KVM_ARM64_SYS_REG(SYS_MVFR1_EL1), + KVM_ARM64_SYS_REG(SYS_MVFR2_EL1), + KVM_ARM64_SYS_REG(SYS_ID_PFR2_EL1), + KVM_ARM64_SYS_REG(SYS_ID_MMFR5_EL1), +}; + +static void test_user_raz_wi(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(raz_wi_reg_ids); i++) { + uint64_t reg_id = raz_wi_reg_ids[i]; + uint64_t val; + + vcpu_get_reg(vcpu, reg_id, &val); + ASSERT_EQ(val, 0); + + /* + * Expect the ioctl to succeed with no effect on the register + * value. + */ + vcpu_set_reg(vcpu, reg_id, BAD_ID_REG_VAL); + + vcpu_get_reg(vcpu, reg_id, &val); + ASSERT_EQ(val, 0); + } +} + +static uint64_t raz_invariant_reg_ids[] = { + KVM_ARM64_SYS_REG(SYS_ID_AFR0_EL1), + KVM_ARM64_SYS_REG(sys_reg(3, 0, 0, 3, 3)), + KVM_ARM64_SYS_REG(SYS_ID_DFR1_EL1), + KVM_ARM64_SYS_REG(sys_reg(3, 0, 0, 3, 7)), +}; + +static void test_user_raz_invariant(struct kvm_vcpu *vcpu) +{ + int i, r; + + for (i = 0; i < ARRAY_SIZE(raz_invariant_reg_ids); i++) { + uint64_t reg_id = raz_invariant_reg_ids[i]; + uint64_t val; + + vcpu_get_reg(vcpu, reg_id, &val); + ASSERT_EQ(val, 0); + + r = __vcpu_set_reg(vcpu, reg_id, BAD_ID_REG_VAL); + TEST_ASSERT(r < 0 && errno == EINVAL, + "unexpected KVM_SET_ONE_REG error: r=%d, errno=%d", r, errno); + + vcpu_get_reg(vcpu, reg_id, &val); + ASSERT_EQ(val, 0); + } +} + + + +static bool vcpu_aarch64_only(struct kvm_vcpu *vcpu) +{ + uint64_t val, el0; + + vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1), &val); + + el0 = (val & ARM64_FEATURE_MASK(ID_AA64PFR0_EL0)) >> ID_AA64PFR0_EL0_SHIFT; + return el0 == ID_AA64PFR0_ELx_64BIT_ONLY; +} + +int main(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + vm = vm_create_with_one_vcpu(&vcpu, guest_main); + + TEST_REQUIRE(vcpu_aarch64_only(vcpu)); + + ucall_init(vm, NULL); + + test_user_raz_wi(vcpu); + test_user_raz_invariant(vcpu); + test_guest_raz(vcpu); + + ucall_uninit(vm); + kvm_vm_free(vm); +} diff --git a/tools/testing/selftests/kvm/aarch64/debug-exceptions.c b/tools/testing/selftests/kvm/aarch64/debug-exceptions.c index 2ee35cf9801e1398e62caddeea3f9127fd3d3970..947bd201435ce27adedbd8216b54524ba82dac11 100644 --- a/tools/testing/selftests/kvm/aarch64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/aarch64/debug-exceptions.c @@ -22,6 +22,7 @@ #define SPSR_SS (1 << 21) extern unsigned char sw_bp, sw_bp2, hw_bp, hw_bp2, bp_svc, bp_brk, hw_wp, ss_start; +extern unsigned char iter_ss_begin, iter_ss_end; static volatile uint64_t sw_bp_addr, hw_bp_addr; static volatile uint64_t wp_addr, wp_data_addr; static volatile uint64_t svc_addr; @@ -238,6 +239,46 @@ static void guest_svc_handler(struct ex_regs *regs) svc_addr = regs->pc; } +enum single_step_op { + SINGLE_STEP_ENABLE = 0, + SINGLE_STEP_DISABLE = 1, +}; + +static void guest_code_ss(int test_cnt) +{ + uint64_t i; + uint64_t bvr, wvr, w_bvr, w_wvr; + + for (i = 0; i < test_cnt; i++) { + /* Bits [1:0] of dbg{b,w}vr are RES0 */ + w_bvr = i << 2; + w_wvr = i << 2; + + /* Enable Single Step execution */ + GUEST_SYNC(SINGLE_STEP_ENABLE); + + /* + * The userspace will veriry that the pc is as expected during + * single step execution between iter_ss_begin and iter_ss_end. + */ + asm volatile("iter_ss_begin:nop\n"); + + write_sysreg(w_bvr, dbgbvr0_el1); + write_sysreg(w_wvr, dbgwvr0_el1); + bvr = read_sysreg(dbgbvr0_el1); + wvr = read_sysreg(dbgwvr0_el1); + + asm volatile("iter_ss_end:\n"); + + /* Disable Single Step execution */ + GUEST_SYNC(SINGLE_STEP_DISABLE); + + GUEST_ASSERT(bvr == w_bvr); + GUEST_ASSERT(wvr == w_wvr); + } + GUEST_DONE(); +} + static int debug_version(struct kvm_vcpu *vcpu) { uint64_t id_aa64dfr0; @@ -246,7 +287,7 @@ static int debug_version(struct kvm_vcpu *vcpu) return id_aa64dfr0 & 0xf; } -int main(int argc, char *argv[]) +static void test_guest_debug_exceptions(void) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; @@ -259,9 +300,6 @@ int main(int argc, char *argv[]) vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(vcpu); - __TEST_REQUIRE(debug_version(vcpu) >= 6, - "Armv8 debug architecture not supported."); - vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_EC_BRK_INS, guest_sw_bp_handler); vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, @@ -294,5 +332,108 @@ int main(int argc, char *argv[]) done: kvm_vm_free(vm); +} + +void test_single_step_from_userspace(int test_cnt) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + struct kvm_run *run; + uint64_t pc, cmd; + uint64_t test_pc = 0; + bool ss_enable = false; + struct kvm_guest_debug debug = {}; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_ss); + ucall_init(vm, NULL); + run = vcpu->run; + vcpu_args_set(vcpu, 1, test_cnt); + + while (1) { + vcpu_run(vcpu); + if (run->exit_reason != KVM_EXIT_DEBUG) { + cmd = get_ucall(vcpu, &uc); + if (cmd == UCALL_ABORT) { + REPORT_GUEST_ASSERT(uc); + /* NOT REACHED */ + } else if (cmd == UCALL_DONE) { + break; + } + + TEST_ASSERT(cmd == UCALL_SYNC, + "Unexpected ucall cmd 0x%lx", cmd); + + if (uc.args[1] == SINGLE_STEP_ENABLE) { + debug.control = KVM_GUESTDBG_ENABLE | + KVM_GUESTDBG_SINGLESTEP; + ss_enable = true; + } else { + debug.control = SINGLE_STEP_DISABLE; + ss_enable = false; + } + + vcpu_guest_debug_set(vcpu, &debug); + continue; + } + + TEST_ASSERT(ss_enable, "Unexpected KVM_EXIT_DEBUG"); + + /* Check if the current pc is expected. */ + vcpu_get_reg(vcpu, ARM64_CORE_REG(regs.pc), &pc); + TEST_ASSERT(!test_pc || pc == test_pc, + "Unexpected pc 0x%lx (expected 0x%lx)", + pc, test_pc); + + /* + * If the current pc is between iter_ss_bgin and + * iter_ss_end, the pc for the next KVM_EXIT_DEBUG should + * be the current pc + 4. + */ + if ((pc >= (uint64_t)&iter_ss_begin) && + (pc < (uint64_t)&iter_ss_end)) + test_pc = pc + 4; + else + test_pc = 0; + } + + kvm_vm_free(vm); +} + +static void help(char *name) +{ + puts(""); + printf("Usage: %s [-h] [-i iterations of the single step test]\n", name); + puts(""); + exit(0); +} + +int main(int argc, char *argv[]) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int opt; + int ss_iteration = 10000; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code); + __TEST_REQUIRE(debug_version(vcpu) >= 6, + "Armv8 debug architecture not supported."); + kvm_vm_free(vm); + + while ((opt = getopt(argc, argv, "i:")) != -1) { + switch (opt) { + case 'i': + ss_iteration = atoi(optarg); + break; + case 'h': + default: + help(argv[0]); + break; + } + } + + test_guest_debug_exceptions(); + test_single_step_from_userspace(ss_iteration); + return 0; } diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index f7621f6e938e461a6c2dc1645be11c215b58e68b..e0b9e81a3e0918210868b352f6f3038ebf493368 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * psci_cpu_on_test - Test that the observable state of a vCPU targeted by the - * CPU_ON PSCI call matches what the caller requested. + * psci_test - Tests relating to KVM's PSCI implementation. * * Copyright (c) 2021 Google LLC. * - * This is a regression test for a race between KVM servicing the PSCI call and - * userspace reading the vCPUs registers. + * This test includes: + * - A regression test for a race between KVM servicing the PSCI CPU_ON call + * and userspace reading the targeted vCPU's registers. + * - A test for KVM's handling of PSCI SYSTEM_SUSPEND and the associated + * KVM_SYSTEM_EVENT_SUSPEND UAPI. */ #define _GNU_SOURCE diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c index 1c2749b1481ac2f8a3e94f80ff67a3a753a5717d..76c583a07ea2211e9a2e66bd08c986a040d6e7cf 100644 --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c @@ -31,8 +31,9 @@ * These limitations are worked around in this test by using a large enough * region of memory for each vCPU such that the number of translations cached in * the TLB and the number of pages held in pagevecs are a small fraction of the - * overall workload. And if either of those conditions are not true this test - * will fail rather than silently passing. + * overall workload. And if either of those conditions are not true (for example + * in nesting, where TLB size is unlimited) this test will print a warning + * rather than silently passing. */ #include <inttypes.h> #include <limits.h> @@ -172,17 +173,23 @@ static void mark_vcpu_memory_idle(struct kvm_vm *vm, vcpu_idx, no_pfn, pages); /* - * Test that at least 90% of memory has been marked idle (the rest might - * not be marked idle because the pages have not yet made it to an LRU - * list or the translations are still cached in the TLB). 90% is + * Check that at least 90% of memory has been marked idle (the rest + * might not be marked idle because the pages have not yet made it to an + * LRU list or the translations are still cached in the TLB). 90% is * arbitrary; high enough that we ensure most memory access went through * access tracking but low enough as to not make the test too brittle * over time and across architectures. + * + * Note that when run in nested virtualization, this check will trigger + * much more frequently because TLB size is unlimited and since no flush + * happens, much more pages are cached there and guest won't see the + * "idle" bit cleared. */ - TEST_ASSERT(still_idle < pages / 10, - "vCPU%d: Too many pages still idle (%"PRIu64 " out of %" - PRIu64 ").\n", - vcpu_idx, still_idle, pages); + if (still_idle < pages / 10) + printf("WARNING: vCPU%d: Too many pages still idle (%" PRIu64 + "out of %" PRIu64 "), this will affect performance results" + ".\n", + vcpu_idx, still_idle, pages); close(page_idle_fd); close(pagemap_fd); diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 9c883c94d478f33f4f21d4938aeac97dcfea69b9..b5234d6efbe1509a62ece9670123d22c9eb1ec3e 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -17,6 +17,7 @@ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/atomic.h> +#include <asm/barrier.h> #include "kvm_util.h" #include "test_util.h" @@ -264,7 +265,8 @@ static void default_after_vcpu_run(struct kvm_vcpu *vcpu, int ret, int err) static bool dirty_ring_supported(void) { - return kvm_has_cap(KVM_CAP_DIRTY_LOG_RING); + return (kvm_has_cap(KVM_CAP_DIRTY_LOG_RING) || + kvm_has_cap(KVM_CAP_DIRTY_LOG_RING_ACQ_REL)); } static void dirty_ring_create_vm_done(struct kvm_vm *vm) @@ -279,12 +281,12 @@ static void dirty_ring_create_vm_done(struct kvm_vm *vm) static inline bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn) { - return gfn->flags == KVM_DIRTY_GFN_F_DIRTY; + return smp_load_acquire(&gfn->flags) == KVM_DIRTY_GFN_F_DIRTY; } static inline void dirty_gfn_set_collected(struct kvm_dirty_gfn *gfn) { - gfn->flags = KVM_DIRTY_GFN_F_RESET; + smp_store_release(&gfn->flags, KVM_DIRTY_GFN_F_RESET); } static uint32_t dirty_ring_collect_one(struct kvm_dirty_gfn *dirty_gfns, diff --git a/tools/testing/selftests/kvm/include/kvm_util_base.h b/tools/testing/selftests/kvm/include/kvm_util_base.h index 24fde97f612110daaa5628b6580c7ba4c2b844ab..e42a09cd24a04bb87d54be0505358460d09ca115 100644 --- a/tools/testing/selftests/kvm/include/kvm_util_base.h +++ b/tools/testing/selftests/kvm/include/kvm_util_base.h @@ -175,6 +175,10 @@ extern const struct vm_guest_mode_params vm_guest_mode_params[]; int open_path_or_exit(const char *path, int flags); int open_kvm_dev_path_or_exit(void); + +bool get_kvm_intel_param_bool(const char *param); +bool get_kvm_amd_param_bool(const char *param); + unsigned int kvm_check_cap(long cap); static inline bool kvm_has_cap(long cap) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 5c5a88180b6c39bda1ebb02cb1be8fd848031a40..befc754ce9b3b712d5dde489bc7bf358f6809e64 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -63,8 +63,10 @@ void test_assert(bool exp, const char *exp_str, #a, #b, #a, (unsigned long) __a, #b, (unsigned long) __b); \ } while (0) -#define TEST_FAIL(fmt, ...) \ - TEST_ASSERT(false, fmt, ##__VA_ARGS__) +#define TEST_FAIL(fmt, ...) do { \ + TEST_ASSERT(false, fmt, ##__VA_ARGS__); \ + __builtin_unreachable(); \ +} while (0) size_t parse_size(const char *size); diff --git a/tools/testing/selftests/kvm/include/x86_64/evmcs.h b/tools/testing/selftests/kvm/include/x86_64/evmcs.h index 3c9260f8e11677814a88f310024de595510a1c2b..58db74f68af25d7eb36f3ca07cef4a847a084aa0 100644 --- a/tools/testing/selftests/kvm/include/x86_64/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86_64/evmcs.h @@ -203,14 +203,25 @@ struct hv_enlightened_vmcs { u32 reserved:30; } hv_enlightenments_control; u32 hv_vp_id; - + u32 padding32_2; u64 hv_vm_id; u64 partition_assist_page; u64 padding64_4[4]; u64 guest_bndcfgs; - u64 padding64_5[7]; + u64 guest_ia32_perf_global_ctrl; + u64 guest_ia32_s_cet; + u64 guest_ssp; + u64 guest_ia32_int_ssp_table_addr; + u64 guest_ia32_lbr_ctl; + u64 padding64_5[2]; u64 xss_exit_bitmap; - u64 padding64_6[7]; + u64 encls_exiting_bitmap; + u64 host_ia32_perf_global_ctrl; + u64 tsc_multiplier; + u64 host_ia32_s_cet; + u64 host_ssp; + u64 host_ia32_int_ssp_table_addr; + u64 padding64_6; }; #define HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE 0 @@ -656,6 +667,18 @@ static inline int evmcs_vmread(uint64_t encoding, uint64_t *value) case VIRTUAL_PROCESSOR_ID: *value = current_evmcs->virtual_processor_id; break; + case HOST_IA32_PERF_GLOBAL_CTRL: + *value = current_evmcs->host_ia32_perf_global_ctrl; + break; + case GUEST_IA32_PERF_GLOBAL_CTRL: + *value = current_evmcs->guest_ia32_perf_global_ctrl; + break; + case ENCLS_EXITING_BITMAP: + *value = current_evmcs->encls_exiting_bitmap; + break; + case TSC_MULTIPLIER: + *value = current_evmcs->tsc_multiplier; + break; default: return 1; } @@ -1169,6 +1192,22 @@ static inline int evmcs_vmwrite(uint64_t encoding, uint64_t value) current_evmcs->virtual_processor_id = value; current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT; break; + case HOST_IA32_PERF_GLOBAL_CTRL: + current_evmcs->host_ia32_perf_global_ctrl = value; + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; + break; + case GUEST_IA32_PERF_GLOBAL_CTRL: + current_evmcs->guest_ia32_perf_global_ctrl = value; + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1; + break; + case ENCLS_EXITING_BITMAP: + current_evmcs->encls_exiting_bitmap = value; + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2; + break; + case TSC_MULTIPLIER: + current_evmcs->tsc_multiplier = value; + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2; + break; default: return 1; } diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index 45edf45821d05391e2bf597bc77b0813fee36fd7..e8ca0d8a6a7e0a06d190bbceba030ffa95dae420 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -754,7 +754,7 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector, void (*handler)(struct ex_regs *)); /* If a toddler were to say "abracadabra". */ -#define KVM_EXCEPTION_MAGIC 0xabacadabaull +#define KVM_EXCEPTION_MAGIC 0xabacadabaULL /* * KVM selftest exception fixup uses registers to coordinate with the exception @@ -786,7 +786,7 @@ void vm_install_exception_handler(struct kvm_vm *vm, int vector, "lea 1f(%%rip), %%r10\n\t" \ "lea 2f(%%rip), %%r11\n\t" \ "1: " insn "\n\t" \ - "mov $0, %[vector]\n\t" \ + "movb $0, %[vector]\n\t" \ "jmp 3f\n\t" \ "2:\n\t" \ "mov %%r9b, %[vector]\n\t" \ @@ -825,6 +825,8 @@ static inline uint8_t wrmsr_safe(uint32_t msr, uint64_t val) return kvm_asm_safe("wrmsr", "a"(val & -1u), "d"(val >> 32), "c"(msr)); } +bool kvm_is_tdp_enabled(void); + uint64_t vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu, uint64_t vaddr); void vm_set_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu, @@ -855,6 +857,8 @@ enum pg_level { #define PG_SIZE_1G PG_LEVEL_SIZE(PG_LEVEL_1G) void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, int level); +void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, + uint64_t nr_bytes, int level); /* * Basic CPU control in CR0 diff --git a/tools/testing/selftests/kvm/include/x86_64/svm_util.h b/tools/testing/selftests/kvm/include/x86_64/svm_util.h index a339b537a57581f6d7364e81bd6e8bd2e4c787af..7aee6244ab6ac9e39b9343643fed683a44740179 100644 --- a/tools/testing/selftests/kvm/include/x86_64/svm_util.h +++ b/tools/testing/selftests/kvm/include/x86_64/svm_util.h @@ -9,15 +9,12 @@ #ifndef SELFTEST_KVM_SVM_UTILS_H #define SELFTEST_KVM_SVM_UTILS_H +#include <asm/svm.h> + #include <stdint.h> #include "svm.h" #include "processor.h" -#define SVM_EXIT_EXCP_BASE 0x040 -#define SVM_EXIT_HLT 0x078 -#define SVM_EXIT_MSR 0x07c -#define SVM_EXIT_VMMCALL 0x081 - struct svm_test_data { /* VMCB */ struct vmcb *vmcb; /* gva */ diff --git a/tools/testing/selftests/kvm/include/x86_64/vmx.h b/tools/testing/selftests/kvm/include/x86_64/vmx.h index 99fa1410964cac2d9d0bca6a05fbec1f279cbc8c..71b290b6469d6ba9f4230c016e3d10156c034648 100644 --- a/tools/testing/selftests/kvm/include/x86_64/vmx.h +++ b/tools/testing/selftests/kvm/include/x86_64/vmx.h @@ -8,6 +8,8 @@ #ifndef SELFTEST_KVM_VMX_H #define SELFTEST_KVM_VMX_H +#include <asm/vmx.h> + #include <stdint.h> #include "processor.h" #include "apic.h" @@ -100,55 +102,6 @@ #define VMX_EPT_VPID_CAP_AD_BITS 0x00200000 #define EXIT_REASON_FAILED_VMENTRY 0x80000000 -#define EXIT_REASON_EXCEPTION_NMI 0 -#define EXIT_REASON_EXTERNAL_INTERRUPT 1 -#define EXIT_REASON_TRIPLE_FAULT 2 -#define EXIT_REASON_INTERRUPT_WINDOW 7 -#define EXIT_REASON_NMI_WINDOW 8 -#define EXIT_REASON_TASK_SWITCH 9 -#define EXIT_REASON_CPUID 10 -#define EXIT_REASON_HLT 12 -#define EXIT_REASON_INVD 13 -#define EXIT_REASON_INVLPG 14 -#define EXIT_REASON_RDPMC 15 -#define EXIT_REASON_RDTSC 16 -#define EXIT_REASON_VMCALL 18 -#define EXIT_REASON_VMCLEAR 19 -#define EXIT_REASON_VMLAUNCH 20 -#define EXIT_REASON_VMPTRLD 21 -#define EXIT_REASON_VMPTRST 22 -#define EXIT_REASON_VMREAD 23 -#define EXIT_REASON_VMRESUME 24 -#define EXIT_REASON_VMWRITE 25 -#define EXIT_REASON_VMOFF 26 -#define EXIT_REASON_VMON 27 -#define EXIT_REASON_CR_ACCESS 28 -#define EXIT_REASON_DR_ACCESS 29 -#define EXIT_REASON_IO_INSTRUCTION 30 -#define EXIT_REASON_MSR_READ 31 -#define EXIT_REASON_MSR_WRITE 32 -#define EXIT_REASON_INVALID_STATE 33 -#define EXIT_REASON_MWAIT_INSTRUCTION 36 -#define EXIT_REASON_MONITOR_INSTRUCTION 39 -#define EXIT_REASON_PAUSE_INSTRUCTION 40 -#define EXIT_REASON_MCE_DURING_VMENTRY 41 -#define EXIT_REASON_TPR_BELOW_THRESHOLD 43 -#define EXIT_REASON_APIC_ACCESS 44 -#define EXIT_REASON_EOI_INDUCED 45 -#define EXIT_REASON_EPT_VIOLATION 48 -#define EXIT_REASON_EPT_MISCONFIG 49 -#define EXIT_REASON_INVEPT 50 -#define EXIT_REASON_RDTSCP 51 -#define EXIT_REASON_PREEMPTION_TIMER 52 -#define EXIT_REASON_INVVPID 53 -#define EXIT_REASON_WBINVD 54 -#define EXIT_REASON_XSETBV 55 -#define EXIT_REASON_APIC_WRITE 56 -#define EXIT_REASON_INVPCID 58 -#define EXIT_REASON_PML_FULL 62 -#define EXIT_REASON_XSAVES 63 -#define EXIT_REASON_XRSTORS 64 -#define LAST_EXIT_REASON 64 enum vmcs_field { VIRTUAL_PROCESSOR_ID = 0x00000000, @@ -208,6 +161,8 @@ enum vmcs_field { VMWRITE_BITMAP_HIGH = 0x00002029, XSS_EXIT_BITMAP = 0x0000202C, XSS_EXIT_BITMAP_HIGH = 0x0000202D, + ENCLS_EXITING_BITMAP = 0x0000202E, + ENCLS_EXITING_BITMAP_HIGH = 0x0000202F, TSC_MULTIPLIER = 0x00002032, TSC_MULTIPLIER_HIGH = 0x00002033, GUEST_PHYSICAL_ADDRESS = 0x00002400, @@ -617,6 +572,7 @@ void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm, uint32_t memslot); void nested_identity_map_1g(struct vmx_pages *vmx, struct kvm_vm *vm, uint64_t addr, uint64_t size); +bool kvm_vm_has_ept(struct kvm_vm *vm); void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm, uint32_t eptp_memslot); void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm); diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c index 71ade6100fd397ba1da8f7d02f1b8b2c2d37e60d..2bd25b191d150c99caf011661b1d201b27da3ce6 100644 --- a/tools/testing/selftests/kvm/lib/assert.c +++ b/tools/testing/selftests/kvm/lib/assert.c @@ -22,7 +22,7 @@ static void test_dump_stack(void) * Build and run this command: * * addr2line -s -e /proc/$PPID/exe -fpai {backtrace addresses} | \ - * grep -v test_dump_stack | cat -n 1>&2 + * cat -n 1>&2 * * Note that the spacing is different and there's no newline. */ @@ -36,18 +36,24 @@ static void test_dump_stack(void) n * (((sizeof(void *)) * 2) + 1) + /* Null terminator: */ 1]; - char *c; + char *c = cmd; n = backtrace(stack, n); - c = &cmd[0]; - c += sprintf(c, "%s", addr2line); /* - * Skip the first 3 frames: backtrace, test_dump_stack, and - * test_assert. We hope that backtrace isn't inlined and the other two - * we've declared noinline. + * Skip the first 2 frames, which should be test_dump_stack() and + * test_assert(); both of which are declared noinline. Bail if the + * resulting stack trace would be empty. Otherwise, addr2line will block + * waiting for addresses to be passed in via stdin. */ + if (n <= 2) { + fputs(" (stack trace empty)\n", stderr); + return; + } + + c += sprintf(c, "%s", addr2line); for (i = 2; i < n; i++) c += sprintf(c, " %lx", ((unsigned long) stack[i]) - 1); + c += sprintf(c, "%s", pipeline); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-result" diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9889fe0d8919c95bebc7b3e7b7774e12e7b25e39..f1cb1627161fde1ba5c3501b353fb765459347df 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -50,6 +50,45 @@ int open_kvm_dev_path_or_exit(void) return _open_kvm_dev_path_or_exit(O_RDONLY); } +static bool get_module_param_bool(const char *module_name, const char *param) +{ + const int path_size = 128; + char path[path_size]; + char value; + ssize_t r; + int fd; + + r = snprintf(path, path_size, "/sys/module/%s/parameters/%s", + module_name, param); + TEST_ASSERT(r < path_size, + "Failed to construct sysfs path in %d bytes.", path_size); + + fd = open_path_or_exit(path, O_RDONLY); + + r = read(fd, &value, 1); + TEST_ASSERT(r == 1, "read(%s) failed", path); + + r = close(fd); + TEST_ASSERT(!r, "close(%s) failed", path); + + if (value == 'Y') + return true; + else if (value == 'N') + return false; + + TEST_FAIL("Unrecognized value '%c' for boolean module param", value); +} + +bool get_kvm_intel_param_bool(const char *param) +{ + return get_module_param_bool("kvm_intel", param); +} + +bool get_kvm_amd_param_bool(const char *param) +{ + return get_module_param_bool("kvm_amd", param); +} + /* * Capability * @@ -82,7 +121,10 @@ unsigned int kvm_check_cap(long cap) void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size) { - vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size); + if (vm_check_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL)) + vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING_ACQ_REL, ring_size); + else + vm_enable_cap(vm, KVM_CAP_DIRTY_LOG_RING, ring_size); vm->dirty_ring_size = ring_size; } diff --git a/tools/testing/selftests/kvm/lib/string_override.c b/tools/testing/selftests/kvm/lib/string_override.c new file mode 100644 index 0000000000000000000000000000000000000000..632398adc2295c356991d0cc59d203f7b58e4240 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/string_override.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <stddef.h> + +/* + * Override the "basic" built-in string helpers so that they can be used in + * guest code. KVM selftests don't support dynamic loading in guest code and + * will jump into the weeds if the compiler decides to insert an out-of-line + * call via the PLT. + */ +int memcmp(const void *cs, const void *ct, size_t count) +{ + const unsigned char *su1, *su2; + int res = 0; + + for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) { + if ((res = *su1 - *su2) != 0) + break; + } + return res; +} + +void *memcpy(void *dest, const void *src, size_t count) +{ + char *tmp = dest; + const char *s = src; + + while (count--) + *tmp++ = *s++; + return dest; +} + +void *memset(void *s, int c, size_t count) +{ + char *xs = s; + + while (count--) + *xs++ = c; + return s; +} diff --git a/tools/testing/selftests/kvm/lib/x86_64/processor.c b/tools/testing/selftests/kvm/lib/x86_64/processor.c index 2e6e61bbe81b39646ea63047792c2c172badb4bf..39c4409ef56a6a95a2c16dfab2fbaa003a9e2f2b 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/processor.c +++ b/tools/testing/selftests/kvm/lib/x86_64/processor.c @@ -111,6 +111,14 @@ static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, uint8_t indent) } } +bool kvm_is_tdp_enabled(void) +{ + if (is_intel_cpu()) + return get_kvm_intel_param_bool("ept"); + else + return get_kvm_amd_param_bool("npt"); +} + void virt_arch_pgd_alloc(struct kvm_vm *vm) { TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " @@ -214,6 +222,25 @@ void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) __virt_pg_map(vm, vaddr, paddr, PG_LEVEL_4K); } +void virt_map_level(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, + uint64_t nr_bytes, int level) +{ + uint64_t pg_size = PG_LEVEL_SIZE(level); + uint64_t nr_pages = nr_bytes / pg_size; + int i; + + TEST_ASSERT(nr_bytes % pg_size == 0, + "Region size not aligned: nr_bytes: 0x%lx, page size: 0x%lx", + nr_bytes, pg_size); + + for (i = 0; i < nr_pages; i++) { + __virt_pg_map(vm, vaddr, paddr, level); + + vaddr += pg_size; + paddr += pg_size; + } +} + static uint64_t *_vm_get_page_table_entry(struct kvm_vm *vm, struct kvm_vcpu *vcpu, uint64_t vaddr) @@ -1294,20 +1321,9 @@ done: /* Returns true if kvm_intel was loaded with unrestricted_guest=1. */ bool vm_is_unrestricted_guest(struct kvm_vm *vm) { - char val = 'N'; - size_t count; - FILE *f; - /* Ensure that a KVM vendor-specific module is loaded. */ if (vm == NULL) close(open_kvm_dev_path_or_exit()); - f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r"); - if (f) { - count = fread(&val, sizeof(char), 1, f); - TEST_ASSERT(count == 1, "Unable to read from param file."); - fclose(f); - } - - return val == 'Y'; + return get_kvm_intel_param_bool("unrestricted_guest"); } diff --git a/tools/testing/selftests/kvm/lib/x86_64/svm.c b/tools/testing/selftests/kvm/lib/x86_64/svm.c index 6d445886e16c58a5771a8ba4ac0af2e62baf3731..5495a92dfd5a47ed2154da6b24e21e42698cc67d 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/svm.c +++ b/tools/testing/selftests/kvm/lib/x86_64/svm.c @@ -60,18 +60,6 @@ static void vmcb_set_seg(struct vmcb_seg *seg, u16 selector, seg->base = base; } -/* - * Avoid using memset to clear the vmcb, since libc may not be - * available in L1 (and, even if it is, features that libc memset may - * want to use, like AVX, may not be enabled). - */ -static void clear_vmcb(struct vmcb *vmcb) -{ - int n = sizeof(*vmcb) / sizeof(u32); - - asm volatile ("rep stosl" : "+c"(n), "+D"(vmcb) : "a"(0) : "memory"); -} - void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp) { struct vmcb *vmcb = svm->vmcb; @@ -88,7 +76,7 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r wrmsr(MSR_EFER, efer | EFER_SVME); wrmsr(MSR_VM_HSAVE_PA, svm->save_area_gpa); - clear_vmcb(vmcb); + memset(vmcb, 0, sizeof(*vmcb)); asm volatile ("vmsave %0\n\t" : : "a" (vmcb_gpa) : "memory"); vmcb_set_seg(&save->es, get_es(), 0, -1U, data_seg_attr); vmcb_set_seg(&save->cs, get_cs(), 0, -1U, code_seg_attr); diff --git a/tools/testing/selftests/kvm/lib/x86_64/vmx.c b/tools/testing/selftests/kvm/lib/x86_64/vmx.c index 80a568c439b832ebc05cd77d7cec580b16687c5a..d21049c38fc5637b9804b539a08e69fdfcddb38b 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86_64/vmx.c @@ -5,6 +5,8 @@ * Copyright (C) 2018, Google LLC. */ +#include <asm/msr-index.h> + #include "test_util.h" #include "kvm_util.h" #include "processor.h" @@ -542,9 +544,27 @@ void nested_identity_map_1g(struct vmx_pages *vmx, struct kvm_vm *vm, __nested_map(vmx, vm, addr, addr, size, PG_LEVEL_1G); } +bool kvm_vm_has_ept(struct kvm_vm *vm) +{ + struct kvm_vcpu *vcpu; + uint64_t ctrl; + + vcpu = list_first_entry(&vm->vcpus, struct kvm_vcpu, list); + TEST_ASSERT(vcpu, "Cannot determine EPT support without vCPUs.\n"); + + ctrl = vcpu_get_msr(vcpu, MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; + if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + return false; + + ctrl = vcpu_get_msr(vcpu, MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; + return ctrl & SECONDARY_EXEC_ENABLE_EPT; +} + void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm, uint32_t eptp_memslot) { + TEST_REQUIRE(kvm_vm_has_ept(vm)); + vmx->eptp = (void *)vm_vaddr_alloc_page(vm); vmx->eptp_hva = addr_gva2hva(vm, (uintptr_t)vmx->eptp); vmx->eptp_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->eptp); diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c index fac248a436667a38f2d43dc2bb6e63bb9cfea482..6f88da7e60be63844a4c0f18f4ee0f84c5d1c0da 100644 --- a/tools/testing/selftests/kvm/rseq_test.c +++ b/tools/testing/selftests/kvm/rseq_test.c @@ -227,7 +227,7 @@ int main(int argc, char *argv[]) ucall_init(vm, NULL); pthread_create(&migration_thread, NULL, migration_worker, - (void *)(unsigned long)gettid()); + (void *)(unsigned long)syscall(SYS_gettid)); for (i = 0; !done; i++) { vcpu_run(vcpu); diff --git a/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c index b1905d280ef55891a90a957b5e96e2f3fec5014a..32f7e09ef67cb5af17e87ccd0613ffb86b819668 100644 --- a/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c +++ b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c @@ -14,89 +14,73 @@ #include "kvm_util.h" #include "processor.h" -static bool ud_expected; +/* VMCALL and VMMCALL are both 3-byte opcodes. */ +#define HYPERCALL_INSN_SIZE 3 + +static bool quirk_disabled; static void guest_ud_handler(struct ex_regs *regs) { - GUEST_ASSERT(ud_expected); - GUEST_DONE(); + regs->rax = -EFAULT; + regs->rip += HYPERCALL_INSN_SIZE; } -extern unsigned char svm_hypercall_insn; -static uint64_t svm_do_sched_yield(uint8_t apic_id) -{ - uint64_t ret; - - asm volatile("mov %1, %%rax\n\t" - "mov %2, %%rbx\n\t" - "svm_hypercall_insn:\n\t" - "vmmcall\n\t" - "mov %%rax, %0\n\t" - : "=r"(ret) - : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id) - : "rax", "rbx", "memory"); +static const uint8_t vmx_vmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xc1 }; +static const uint8_t svm_vmmcall[HYPERCALL_INSN_SIZE] = { 0x0f, 0x01, 0xd9 }; - return ret; -} - -extern unsigned char vmx_hypercall_insn; -static uint64_t vmx_do_sched_yield(uint8_t apic_id) +extern uint8_t hypercall_insn[HYPERCALL_INSN_SIZE]; +static uint64_t do_sched_yield(uint8_t apic_id) { uint64_t ret; - asm volatile("mov %1, %%rax\n\t" - "mov %2, %%rbx\n\t" - "vmx_hypercall_insn:\n\t" - "vmcall\n\t" - "mov %%rax, %0\n\t" - : "=r"(ret) - : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id) - : "rax", "rbx", "memory"); + asm volatile("hypercall_insn:\n\t" + ".byte 0xcc,0xcc,0xcc\n\t" + : "=a"(ret) + : "a"((uint64_t)KVM_HC_SCHED_YIELD), "b"((uint64_t)apic_id) + : "memory"); return ret; } -static void assert_hypercall_insn(unsigned char *exp_insn, unsigned char *obs_insn) -{ - uint32_t exp = 0, obs = 0; - - memcpy(&exp, exp_insn, sizeof(exp)); - memcpy(&obs, obs_insn, sizeof(obs)); - - GUEST_ASSERT_EQ(exp, obs); -} - static void guest_main(void) { - unsigned char *native_hypercall_insn, *hypercall_insn; - uint8_t apic_id; - - apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID)); + const uint8_t *native_hypercall_insn; + const uint8_t *other_hypercall_insn; + uint64_t ret; if (is_intel_cpu()) { - native_hypercall_insn = &vmx_hypercall_insn; - hypercall_insn = &svm_hypercall_insn; - svm_do_sched_yield(apic_id); + native_hypercall_insn = vmx_vmcall; + other_hypercall_insn = svm_vmmcall; } else if (is_amd_cpu()) { - native_hypercall_insn = &svm_hypercall_insn; - hypercall_insn = &vmx_hypercall_insn; - vmx_do_sched_yield(apic_id); + native_hypercall_insn = svm_vmmcall; + other_hypercall_insn = vmx_vmcall; } else { GUEST_ASSERT(0); /* unreachable */ return; } - GUEST_ASSERT(!ud_expected); - assert_hypercall_insn(native_hypercall_insn, hypercall_insn); - GUEST_DONE(); -} + memcpy(hypercall_insn, other_hypercall_insn, HYPERCALL_INSN_SIZE); -static void setup_ud_vector(struct kvm_vcpu *vcpu) -{ - vm_init_descriptor_tables(vcpu->vm); - vcpu_init_descriptor_tables(vcpu); - vm_install_exception_handler(vcpu->vm, UD_VECTOR, guest_ud_handler); + ret = do_sched_yield(GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID))); + + /* + * If the quirk is disabled, verify that guest_ud_handler() "returned" + * -EFAULT and that KVM did NOT patch the hypercall. If the quirk is + * enabled, verify that the hypercall succeeded and that KVM patched in + * the "right" hypercall. + */ + if (quirk_disabled) { + GUEST_ASSERT(ret == (uint64_t)-EFAULT); + GUEST_ASSERT(!memcmp(other_hypercall_insn, hypercall_insn, + HYPERCALL_INSN_SIZE)); + } else { + GUEST_ASSERT(!ret); + GUEST_ASSERT(!memcmp(native_hypercall_insn, hypercall_insn, + HYPERCALL_INSN_SIZE)); + } + + GUEST_DONE(); } static void enter_guest(struct kvm_vcpu *vcpu) @@ -119,35 +103,23 @@ static void enter_guest(struct kvm_vcpu *vcpu) } } -static void test_fix_hypercall(void) +static void test_fix_hypercall(bool disable_quirk) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; vm = vm_create_with_one_vcpu(&vcpu, guest_main); - setup_ud_vector(vcpu); - - ud_expected = false; - sync_global_to_guest(vm, ud_expected); - - virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); - - enter_guest(vcpu); -} - -static void test_fix_hypercall_disabled(void) -{ - struct kvm_vcpu *vcpu; - struct kvm_vm *vm; - vm = vm_create_with_one_vcpu(&vcpu, guest_main); - setup_ud_vector(vcpu); + vm_init_descriptor_tables(vcpu->vm); + vcpu_init_descriptor_tables(vcpu); + vm_install_exception_handler(vcpu->vm, UD_VECTOR, guest_ud_handler); - vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2, - KVM_X86_QUIRK_FIX_HYPERCALL_INSN); + if (disable_quirk) + vm_enable_cap(vm, KVM_CAP_DISABLE_QUIRKS2, + KVM_X86_QUIRK_FIX_HYPERCALL_INSN); - ud_expected = true; - sync_global_to_guest(vm, ud_expected); + quirk_disabled = disable_quirk; + sync_global_to_guest(vm, quirk_disabled); virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); @@ -158,6 +130,6 @@ int main(void) { TEST_REQUIRE(kvm_check_cap(KVM_CAP_DISABLE_QUIRKS2) & KVM_X86_QUIRK_FIX_HYPERCALL_INSN); - test_fix_hypercall(); - test_fix_hypercall_disabled(); + test_fix_hypercall(false); + test_fix_hypercall(true); } diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_features.c b/tools/testing/selftests/kvm/x86_64/hyperv_features.c index 79ab0152d2810e31d81b334e101bd7781c0c98bd..05b32e550a8025b727dc8cf05424505de5eaf0b6 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_features.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_features.c @@ -26,7 +26,8 @@ static inline uint8_t hypercall(u64 control, vm_vaddr_t input_address, : "=a" (*hv_status), "+c" (control), "+d" (input_address), KVM_ASM_SAFE_OUTPUTS(vector) - : [output_address] "r"(output_address) + : [output_address] "r"(output_address), + "a" (-EFAULT) : "cc", "memory", "r8", KVM_ASM_SAFE_CLOBBERS); return vector; } @@ -81,13 +82,13 @@ static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall) } vector = hypercall(hcall->control, input, output, &res); - if (hcall->ud_expected) + if (hcall->ud_expected) { GUEST_ASSERT_2(vector == UD_VECTOR, hcall->control, vector); - else + } else { GUEST_ASSERT_2(!vector, hcall->control, vector); + GUEST_ASSERT_2(res == hcall->expect, hcall->expect, res); + } - GUEST_ASSERT_2(!hcall->ud_expected || res == hcall->expect, - hcall->expect, res); GUEST_DONE(); } @@ -507,7 +508,7 @@ static void guest_test_hcalls_access(void) switch (stage) { case 0: feat->eax |= HV_MSR_HYPERCALL_AVAILABLE; - hcall->control = 0xdeadbeef; + hcall->control = 0xbeef; hcall->expect = HV_STATUS_INVALID_HYPERCALL_CODE; break; diff --git a/tools/testing/selftests/kvm/x86_64/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86_64/nested_exceptions_test.c new file mode 100644 index 0000000000000000000000000000000000000000..ac33835f78f45afa2a1bc2d387bf03c18025efc5 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/nested_exceptions_test.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE /* for program_invocation_short_name */ + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" +#include "svm_util.h" + +#define L2_GUEST_STACK_SIZE 256 + +/* + * Arbitrary, never shoved into KVM/hardware, just need to avoid conflict with + * the "real" exceptions used, #SS/#GP/#DF (12/13/8). + */ +#define FAKE_TRIPLE_FAULT_VECTOR 0xaa + +/* Arbitrary 32-bit error code injected by this test. */ +#define SS_ERROR_CODE 0xdeadbeef + +/* + * Bit '0' is set on Intel if the exception occurs while delivering a previous + * event/exception. AMD's wording is ambiguous, but presumably the bit is set + * if the exception occurs while delivering an external event, e.g. NMI or INTR, + * but not for exceptions that occur when delivering other exceptions or + * software interrupts. + * + * Note, Intel's name for it, "External event", is misleading and much more + * aligned with AMD's behavior, but the SDM is quite clear on its behavior. + */ +#define ERROR_CODE_EXT_FLAG BIT(0) + +/* + * Bit '1' is set if the fault occurred when looking up a descriptor in the + * IDT, which is the case here as the IDT is empty/NULL. + */ +#define ERROR_CODE_IDT_FLAG BIT(1) + +/* + * The #GP that occurs when vectoring #SS should show the index into the IDT + * for #SS, plus have the "IDT flag" set. + */ +#define GP_ERROR_CODE_AMD ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) +#define GP_ERROR_CODE_INTEL ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG | ERROR_CODE_EXT_FLAG) + +/* + * Intel and AMD both shove '0' into the error code on #DF, regardless of what + * led to the double fault. + */ +#define DF_ERROR_CODE 0 + +#define INTERCEPT_SS (BIT_ULL(SS_VECTOR)) +#define INTERCEPT_SS_DF (INTERCEPT_SS | BIT_ULL(DF_VECTOR)) +#define INTERCEPT_SS_GP_DF (INTERCEPT_SS_DF | BIT_ULL(GP_VECTOR)) + +static void l2_ss_pending_test(void) +{ + GUEST_SYNC(SS_VECTOR); +} + +static void l2_ss_injected_gp_test(void) +{ + GUEST_SYNC(GP_VECTOR); +} + +static void l2_ss_injected_df_test(void) +{ + GUEST_SYNC(DF_VECTOR); +} + +static void l2_ss_injected_tf_test(void) +{ + GUEST_SYNC(FAKE_TRIPLE_FAULT_VECTOR); +} + +static void svm_run_l2(struct svm_test_data *svm, void *l2_code, int vector, + uint32_t error_code) +{ + struct vmcb *vmcb = svm->vmcb; + struct vmcb_control_area *ctrl = &vmcb->control; + + vmcb->save.rip = (u64)l2_code; + run_guest(vmcb, svm->vmcb_gpa); + + if (vector == FAKE_TRIPLE_FAULT_VECTOR) + return; + + GUEST_ASSERT_EQ(ctrl->exit_code, (SVM_EXIT_EXCP_BASE + vector)); + GUEST_ASSERT_EQ(ctrl->exit_info_1, error_code); +} + +static void l1_svm_code(struct svm_test_data *svm) +{ + struct vmcb_control_area *ctrl = &svm->vmcb->control; + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + + generic_svm_setup(svm, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + svm->vmcb->save.idtr.limit = 0; + ctrl->intercept |= BIT_ULL(INTERCEPT_SHUTDOWN); + + ctrl->intercept_exceptions = INTERCEPT_SS_GP_DF; + svm_run_l2(svm, l2_ss_pending_test, SS_VECTOR, SS_ERROR_CODE); + svm_run_l2(svm, l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_AMD); + + ctrl->intercept_exceptions = INTERCEPT_SS_DF; + svm_run_l2(svm, l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); + + ctrl->intercept_exceptions = INTERCEPT_SS; + svm_run_l2(svm, l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); + GUEST_ASSERT_EQ(ctrl->exit_code, SVM_EXIT_SHUTDOWN); + + GUEST_DONE(); +} + +static void vmx_run_l2(void *l2_code, int vector, uint32_t error_code) +{ + GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code)); + + GUEST_ASSERT_EQ(vector == SS_VECTOR ? vmlaunch() : vmresume(), 0); + + if (vector == FAKE_TRIPLE_FAULT_VECTOR) + return; + + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), vector); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_ERROR_CODE), error_code); +} + +static void l1_vmx_code(struct vmx_pages *vmx) +{ + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + + GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + + GUEST_ASSERT_EQ(load_vmcs(vmx), true); + + prepare_vmcs(vmx, NULL, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); + + /* + * VMX disallows injecting an exception with error_code[31:16] != 0, + * and hardware will never generate a VM-Exit with bits 31:16 set. + * KVM should likewise truncate the "bad" userspace value. + */ + GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF), 0); + vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE); + vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL); + + GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF), 0); + vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); + + GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS), 0); + vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); + GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); + + GUEST_DONE(); +} + +static void __attribute__((__flatten__)) l1_guest_code(void *test_data) +{ + if (this_cpu_has(X86_FEATURE_SVM)) + l1_svm_code(test_data); + else + l1_vmx_code(test_data); +} + +static void assert_ucall_vector(struct kvm_vcpu *vcpu, int vector) +{ + struct kvm_run *run = vcpu->run; + struct ucall uc; + + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Unexpected exit reason: %u (%s),\n", + run->exit_reason, exit_reason_str(run->exit_reason)); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + TEST_ASSERT(vector == uc.args[1], + "Expected L2 to ask for %d, got %ld", vector, uc.args[1]); + break; + case UCALL_DONE: + TEST_ASSERT(vector == -1, + "Expected L2 to ask for %d, L2 says it's done", vector); + break; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld (0x%lx != 0x%lx)", + (const char *)uc.args[0], __FILE__, uc.args[1], + uc.args[2], uc.args[3]); + break; + default: + TEST_FAIL("Expected L2 to ask for %d, got unexpected ucall %lu", vector, uc.cmd); + } +} + +static void queue_ss_exception(struct kvm_vcpu *vcpu, bool inject) +{ + struct kvm_vcpu_events events; + + vcpu_events_get(vcpu, &events); + + TEST_ASSERT(!events.exception.pending, + "Vector %d unexpectedlt pending", events.exception.nr); + TEST_ASSERT(!events.exception.injected, + "Vector %d unexpectedly injected", events.exception.nr); + + events.flags = KVM_VCPUEVENT_VALID_PAYLOAD; + events.exception.pending = !inject; + events.exception.injected = inject; + events.exception.nr = SS_VECTOR; + events.exception.has_error_code = true; + events.exception.error_code = SS_ERROR_CODE; + vcpu_events_set(vcpu, &events); +} + +/* + * Verify KVM_{G,S}ET_EVENTS play nice with pending vs. injected exceptions + * when an exception is being queued for L2. Specifically, verify that KVM + * honors L1 exception intercept controls when a #SS is pending/injected, + * triggers a #GP on vectoring the #SS, morphs to #DF if #GP isn't intercepted + * by L1, and finally causes (nested) SHUTDOWN if #DF isn't intercepted by L1. + */ +int main(int argc, char *argv[]) +{ + vm_vaddr_t nested_test_data_gva; + struct kvm_vcpu_events events; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + TEST_REQUIRE(kvm_has_cap(KVM_CAP_EXCEPTION_PAYLOAD)); + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM) || kvm_cpu_has(X86_FEATURE_VMX)); + + vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + vm_enable_cap(vm, KVM_CAP_EXCEPTION_PAYLOAD, -2ul); + + if (kvm_cpu_has(X86_FEATURE_SVM)) + vcpu_alloc_svm(vm, &nested_test_data_gva); + else + vcpu_alloc_vmx(vm, &nested_test_data_gva); + + vcpu_args_set(vcpu, 1, nested_test_data_gva); + + /* Run L1 => L2. L2 should sync and request #SS. */ + vcpu_run(vcpu); + assert_ucall_vector(vcpu, SS_VECTOR); + + /* Pend #SS and request immediate exit. #SS should still be pending. */ + queue_ss_exception(vcpu, false); + vcpu->run->immediate_exit = true; + vcpu_run_complete_io(vcpu); + + /* Verify the pending events comes back out the same as it went in. */ + vcpu_events_get(vcpu, &events); + ASSERT_EQ(events.flags & KVM_VCPUEVENT_VALID_PAYLOAD, + KVM_VCPUEVENT_VALID_PAYLOAD); + ASSERT_EQ(events.exception.pending, true); + ASSERT_EQ(events.exception.nr, SS_VECTOR); + ASSERT_EQ(events.exception.has_error_code, true); + ASSERT_EQ(events.exception.error_code, SS_ERROR_CODE); + + /* + * Run for real with the pending #SS, L1 should get a VM-Exit due to + * #SS interception and re-enter L2 to request #GP (via injected #SS). + */ + vcpu->run->immediate_exit = false; + vcpu_run(vcpu); + assert_ucall_vector(vcpu, GP_VECTOR); + + /* + * Inject #SS, the #SS should bypass interception and cause #GP, which + * L1 should intercept before KVM morphs it to #DF. L1 should then + * disable #GP interception and run L2 to request #DF (via #SS => #GP). + */ + queue_ss_exception(vcpu, true); + vcpu_run(vcpu); + assert_ucall_vector(vcpu, DF_VECTOR); + + /* + * Inject #SS, the #SS should bypass interception and cause #GP, which + * L1 is no longer interception, and so should see a #DF VM-Exit. L1 + * should then signal that is done. + */ + queue_ss_exception(vcpu, true); + vcpu_run(vcpu); + assert_ucall_vector(vcpu, FAKE_TRIPLE_FAULT_VECTOR); + + /* + * Inject #SS yet again. L1 is not intercepting #GP or #DF, and so + * should see nested TRIPLE_FAULT / SHUTDOWN. + */ + queue_ss_exception(vcpu, true); + vcpu_run(vcpu); + assert_ucall_vector(vcpu, -1); + + kvm_vm_free(vm); +} diff --git a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c index cc64217164005f24bca8a23f8a1ba75ea37e660a..59ffe7fd354f8919c1ff7f4fe2e9bb9a95fc6ed8 100644 --- a/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86_64/nx_huge_pages_test.c @@ -112,19 +112,13 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, { struct kvm_vcpu *vcpu; struct kvm_vm *vm; + uint64_t nr_bytes; void *hva; int r; vm = vm_create(1); if (disable_nx_huge_pages) { - /* - * Cannot run the test without NX huge pages if the kernel - * does not support it. - */ - if (!kvm_check_cap(KVM_CAP_VM_DISABLE_NX_HUGE_PAGES)) - return; - r = __vm_disable_nx_huge_pages(vm); if (reboot_permissions) { TEST_ASSERT(!r, "Disabling NX huge pages should succeed if process has reboot permissions"); @@ -141,10 +135,24 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, HPAGE_GPA, HPAGE_SLOT, HPAGE_SLOT_NPAGES, 0); - virt_map(vm, HPAGE_GVA, HPAGE_GPA, HPAGE_SLOT_NPAGES); + nr_bytes = HPAGE_SLOT_NPAGES * vm->page_size; + + /* + * Ensure that KVM can map HPAGE_SLOT with huge pages by mapping the + * region into the guest with 2MiB pages whenever TDP is disabled (i.e. + * whenever KVM is shadowing the guest page tables). + * + * When TDP is enabled, KVM should be able to map HPAGE_SLOT with huge + * pages irrespective of the guest page size, so map with 4KiB pages + * to test that that is the case. + */ + if (kvm_is_tdp_enabled()) + virt_map_level(vm, HPAGE_GVA, HPAGE_GPA, nr_bytes, PG_LEVEL_4K); + else + virt_map_level(vm, HPAGE_GVA, HPAGE_GPA, nr_bytes, PG_LEVEL_2M); hva = addr_gpa2hva(vm, HPAGE_GPA); - memset(hva, RETURN_OPCODE, HPAGE_SLOT_NPAGES * PAGE_SIZE); + memset(hva, RETURN_OPCODE, nr_bytes); check_2m_page_count(vm, 0); check_split_count(vm, 0); @@ -248,18 +256,13 @@ int main(int argc, char **argv) } } - if (token != MAGIC_TOKEN) { - print_skip("This test must be run with the magic token %d.\n" - "This is done by nx_huge_pages_test.sh, which\n" - "also handles environment setup for the test.", - MAGIC_TOKEN); - exit(KSFT_SKIP); - } + TEST_REQUIRE(kvm_has_cap(KVM_CAP_VM_DISABLE_NX_HUGE_PAGES)); + TEST_REQUIRE(reclaim_period_ms > 0); - if (!reclaim_period_ms) { - print_skip("The NX reclaim period must be specified and non-zero"); - exit(KSFT_SKIP); - } + __TEST_REQUIRE(token == MAGIC_TOKEN, + "This test must be run with the magic token %d.\n" + "This is done by nx_huge_pages_test.sh, which\n" + "also handles environment setup for the test."); run_test(reclaim_period_ms, false, reboot_permissions); run_test(reclaim_period_ms, true, reboot_permissions); diff --git a/tools/testing/selftests/landlock/Makefile b/tools/testing/selftests/landlock/Makefile index 02868ac3bc717f341029489ef9ef67eda5ff417c..6632bfff486b8a187e8e42a7f4b8d8f835e115ca 100644 --- a/tools/testing/selftests/landlock/Makefile +++ b/tools/testing/selftests/landlock/Makefile @@ -1,6 +1,11 @@ # SPDX-License-Identifier: GPL-2.0 +# +# First run: make -C ../../../.. headers_install CFLAGS += -Wall -O2 $(KHDR_INCLUDES) +LDLIBS += -lcap + +LOCAL_HDRS += common.h src_test := $(wildcard *_test.c) @@ -8,14 +13,10 @@ TEST_GEN_PROGS := $(src_test:.c=) TEST_GEN_PROGS_EXTENDED := true -OVERRIDE_TARGETS := 1 -top_srcdir := ../../../.. -include ../lib.mk - -khdr_dir = $(top_srcdir)/usr/include +# Static linking for short targets: +$(TEST_GEN_PROGS_EXTENDED): LDFLAGS += -static -$(OUTPUT)/true: true.c - $(LINK.c) $< $(LDLIBS) -o $@ -static +include ../lib.mk -$(OUTPUT)/%_test: %_test.c $(khdr_dir)/linux/landlock.h ../kselftest_harness.h common.h - $(LINK.c) $< $(LDLIBS) -o $@ -lcap -I$(khdr_dir) +# Static linking for targets with $(OUTPUT)/ prefix: +$(TEST_GEN_PROGS_EXTENDED): LDFLAGS += -static diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index 21a2ce8fa739d8a11d9a656fee2c6e816ed4e0ed..45de42a027c549c0e3deb61197d6a6b23241467a 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -4,7 +4,7 @@ * * Copyright © 2017-2020 Mickaël Salaün <mic@digikod.net> * Copyright © 2020 ANSSI - * Copyright © 2020-2021 Microsoft Corporation + * Copyright © 2020-2022 Microsoft Corporation */ #define _GNU_SOURCE @@ -371,6 +371,13 @@ TEST_F_FORK(layout1, inval) ASSERT_EQ(EINVAL, errno); path_beneath.allowed_access &= ~LANDLOCK_ACCESS_FS_EXECUTE; + /* Tests with denied-by-default access right. */ + path_beneath.allowed_access |= LANDLOCK_ACCESS_FS_REFER; + ASSERT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &path_beneath, 0)); + ASSERT_EQ(EINVAL, errno); + path_beneath.allowed_access &= ~LANDLOCK_ACCESS_FS_REFER; + /* Test with unknown (64-bits) value. */ path_beneath.allowed_access |= (1ULL << 60); ASSERT_EQ(-1, landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, @@ -1826,6 +1833,20 @@ TEST_F_FORK(layout1, link) ASSERT_EQ(0, link(file1_s1d3, file2_s1d3)); } +static int test_rename(const char *const oldpath, const char *const newpath) +{ + if (rename(oldpath, newpath)) + return errno; + return 0; +} + +static int test_exchange(const char *const oldpath, const char *const newpath) +{ + if (renameat2(AT_FDCWD, oldpath, AT_FDCWD, newpath, RENAME_EXCHANGE)) + return errno; + return 0; +} + TEST_F_FORK(layout1, rename_file) { const struct rule rules[] = { @@ -1867,10 +1888,10 @@ TEST_F_FORK(layout1, rename_file) * to a different directory (which allows file removal). */ ASSERT_EQ(-1, rename(file1_s2d1, file1_s1d3)); - ASSERT_EQ(EXDEV, errno); + ASSERT_EQ(EACCES, errno); ASSERT_EQ(-1, renameat2(AT_FDCWD, file1_s2d1, AT_FDCWD, file1_s1d3, RENAME_EXCHANGE)); - ASSERT_EQ(EXDEV, errno); + ASSERT_EQ(EACCES, errno); ASSERT_EQ(-1, renameat2(AT_FDCWD, dir_s2d2, AT_FDCWD, file1_s1d3, RENAME_EXCHANGE)); ASSERT_EQ(EXDEV, errno); @@ -1894,7 +1915,7 @@ TEST_F_FORK(layout1, rename_file) ASSERT_EQ(EXDEV, errno); ASSERT_EQ(0, unlink(file1_s1d3)); ASSERT_EQ(-1, rename(file1_s2d1, file1_s1d3)); - ASSERT_EQ(EXDEV, errno); + ASSERT_EQ(EACCES, errno); /* Exchanges and renames files with same parent. */ ASSERT_EQ(0, renameat2(AT_FDCWD, file2_s2d3, AT_FDCWD, file1_s2d3, @@ -2014,6 +2035,115 @@ TEST_F_FORK(layout1, reparent_refer) ASSERT_EQ(0, rename(dir_s1d3, dir_s2d3)); } +/* Checks renames beneath dir_s1d1. */ +static void refer_denied_by_default(struct __test_metadata *const _metadata, + const struct rule layer1[], + const int layer1_err, + const struct rule layer2[]) +{ + int ruleset_fd; + + ASSERT_EQ(0, unlink(file1_s1d2)); + + ruleset_fd = create_ruleset(_metadata, layer1[0].access, layer1); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* + * If the first layer handles LANDLOCK_ACCESS_FS_REFER (according to + * layer1_err), then it allows some different-parent renames and links. + */ + ASSERT_EQ(layer1_err, test_rename(file1_s1d1, file1_s1d2)); + if (layer1_err == 0) + ASSERT_EQ(layer1_err, test_rename(file1_s1d2, file1_s1d1)); + ASSERT_EQ(layer1_err, test_exchange(file2_s1d1, file2_s1d2)); + ASSERT_EQ(layer1_err, test_exchange(file2_s1d2, file2_s1d1)); + + ruleset_fd = create_ruleset(_metadata, layer2[0].access, layer2); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + ASSERT_EQ(0, close(ruleset_fd)); + + /* + * Now, either the first or the second layer does not handle + * LANDLOCK_ACCESS_FS_REFER, which means that any different-parent + * renames and links are denied, thus making the layer handling + * LANDLOCK_ACCESS_FS_REFER null and void. + */ + ASSERT_EQ(EXDEV, test_rename(file1_s1d1, file1_s1d2)); + ASSERT_EQ(EXDEV, test_exchange(file2_s1d1, file2_s1d2)); + ASSERT_EQ(EXDEV, test_exchange(file2_s1d2, file2_s1d1)); +} + +const struct rule layer_dir_s1d1_refer[] = { + { + .path = dir_s1d1, + .access = LANDLOCK_ACCESS_FS_REFER, + }, + {}, +}; + +const struct rule layer_dir_s1d1_execute[] = { + { + /* Matches a parent directory. */ + .path = dir_s1d1, + .access = LANDLOCK_ACCESS_FS_EXECUTE, + }, + {}, +}; + +const struct rule layer_dir_s2d1_execute[] = { + { + /* Does not match a parent directory. */ + .path = dir_s2d1, + .access = LANDLOCK_ACCESS_FS_EXECUTE, + }, + {}, +}; + +/* + * Tests precedence over renames: denied by default for different parent + * directories, *with* a rule matching a parent directory, but not directly + * denying access (with MAKE_REG nor REMOVE). + */ +TEST_F_FORK(layout1, refer_denied_by_default1) +{ + refer_denied_by_default(_metadata, layer_dir_s1d1_refer, 0, + layer_dir_s1d1_execute); +} + +/* + * Same test but this time turning around the ABI version order: the first + * layer does not handle LANDLOCK_ACCESS_FS_REFER. + */ +TEST_F_FORK(layout1, refer_denied_by_default2) +{ + refer_denied_by_default(_metadata, layer_dir_s1d1_execute, EXDEV, + layer_dir_s1d1_refer); +} + +/* + * Tests precedence over renames: denied by default for different parent + * directories, *without* a rule matching a parent directory, but not directly + * denying access (with MAKE_REG nor REMOVE). + */ +TEST_F_FORK(layout1, refer_denied_by_default3) +{ + refer_denied_by_default(_metadata, layer_dir_s1d1_refer, 0, + layer_dir_s2d1_execute); +} + +/* + * Same test but this time turning around the ABI version order: the first + * layer does not handle LANDLOCK_ACCESS_FS_REFER. + */ +TEST_F_FORK(layout1, refer_denied_by_default4) +{ + refer_denied_by_default(_metadata, layer_dir_s2d1_execute, EXDEV, + layer_dir_s1d1_refer); +} + TEST_F_FORK(layout1, reparent_link) { const struct rule layer1[] = { @@ -2336,11 +2466,12 @@ TEST_F_FORK(layout1, reparent_exdev_layers_rename1) ASSERT_EQ(EXDEV, errno); /* - * However, moving the file2_s1d3 file below dir_s2d3 is allowed - * because it cannot inherit MAKE_REG nor MAKE_DIR rights (which are - * dedicated to directories). + * Moving the file2_s1d3 file below dir_s2d3 is denied because the + * second layer does not handle REFER, which is always denied by + * default. */ - ASSERT_EQ(0, rename(file2_s1d3, file1_s2d3)); + ASSERT_EQ(-1, rename(file2_s1d3, file1_s2d3)); + ASSERT_EQ(EXDEV, errno); } TEST_F_FORK(layout1, reparent_exdev_layers_rename2) @@ -2373,8 +2504,12 @@ TEST_F_FORK(layout1, reparent_exdev_layers_rename2) ASSERT_EQ(EACCES, errno); ASSERT_EQ(-1, rename(file1_s1d1, file1_s2d3)); ASSERT_EQ(EXDEV, errno); - /* Modify layout! */ - ASSERT_EQ(0, rename(file2_s1d2, file1_s2d3)); + /* + * Modifying the layout is now denied because the second layer does not + * handle REFER, which is always denied by default. + */ + ASSERT_EQ(-1, rename(file2_s1d2, file1_s2d3)); + ASSERT_EQ(EXDEV, errno); /* Without REFER source, EACCES wins over EXDEV. */ ASSERT_EQ(-1, rename(dir_s1d1, file1_s2d2)); diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 947fc72413e9ae46ea95a1e8f1a8faa0a6ffcdd5..9d4cb94cf43741013242cd28908047f97b9775ac 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -40,6 +40,11 @@ ifeq (0,$(MAKELEVEL)) endif endif selfdir = $(realpath $(dir $(filter %/lib.mk,$(MAKEFILE_LIST)))) +top_srcdir = $(selfdir)/../../.. + +ifeq ($(KHDR_INCLUDES),) +KHDR_INCLUDES := -isystem $(top_srcdir)/usr/include +endif # The following are built by lib.mk common compile rules. # TEST_CUSTOM_PROGS should be used by tests that require diff --git a/tools/testing/selftests/livepatch/Makefile b/tools/testing/selftests/livepatch/Makefile index 1acc9e1fa3fbca78db5fca858f01c042a2fe62ff..02fadc9d55e0f576b22b0053189739d51ed6fba5 100644 --- a/tools/testing/selftests/livepatch/Makefile +++ b/tools/testing/selftests/livepatch/Makefile @@ -6,7 +6,8 @@ TEST_PROGS := \ test-callbacks.sh \ test-shadow-vars.sh \ test-state.sh \ - test-ftrace.sh + test-ftrace.sh \ + test-sysfs.sh TEST_FILES := settings diff --git a/tools/testing/selftests/livepatch/functions.sh b/tools/testing/selftests/livepatch/functions.sh index 9230b869371d951f33d2286690708889a7e86f92..c8416c54b4637b1380810f0c9f71bc99e1710b8e 100644 --- a/tools/testing/selftests/livepatch/functions.sh +++ b/tools/testing/selftests/livepatch/functions.sh @@ -6,6 +6,7 @@ MAX_RETRIES=600 RETRY_INTERVAL=".1" # seconds +KLP_SYSFS_DIR="/sys/kernel/livepatch" # Kselftest framework requirement - SKIP code is 4 ksft_skip=4 @@ -86,7 +87,7 @@ function set_ftrace_enabled() { if [[ "$result" != "$1" ]] ; then if [[ $can_fail -eq 1 ]] ; then - echo "livepatch: $err" > /dev/kmsg + echo "livepatch: $err" | sed 's#/proc/sys/kernel/#kernel.#' > /dev/kmsg return fi @@ -308,3 +309,36 @@ function check_result { cleanup_dmesg_file } + +# check_sysfs_rights(modname, rel_path, expected_rights) - check sysfs +# path permissions +# modname - livepatch module creating the sysfs interface +# rel_path - relative path of the sysfs interface +# expected_rights - expected access rights +function check_sysfs_rights() { + local mod="$1"; shift + local rel_path="$1"; shift + local expected_rights="$1"; shift + + local path="$KLP_SYSFS_DIR/$mod/$rel_path" + local rights=$(/bin/stat --format '%A' "$path") + if test "$rights" != "$expected_rights" ; then + die "Unexpected access rights of $path: $expected_rights vs. $rights" + fi +} + +# check_sysfs_value(modname, rel_path, expected_value) - check sysfs value +# modname - livepatch module creating the sysfs interface +# rel_path - relative path of the sysfs interface +# expected_value - expected value read from the file +function check_sysfs_value() { + local mod="$1"; shift + local rel_path="$1"; shift + local expected_value="$1"; shift + + local path="$KLP_SYSFS_DIR/$mod/$rel_path" + local value=`cat $path` + if test "$value" != "$expected_value" ; then + die "Unexpected value in $path: $expected_value vs. $value" + fi +} diff --git a/tools/testing/selftests/livepatch/test-sysfs.sh b/tools/testing/selftests/livepatch/test-sysfs.sh new file mode 100755 index 0000000000000000000000000000000000000000..7f76f280189a0c341517e641612387ae1ec63143 --- /dev/null +++ b/tools/testing/selftests/livepatch/test-sysfs.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2022 Song Liu <song@kernel.org> + +. $(dirname $0)/functions.sh + +MOD_LIVEPATCH=test_klp_livepatch + +setup_config + +# - load a livepatch and verifies the sysfs entries work as expected + +start_test "sysfs test" + +load_lp $MOD_LIVEPATCH + +check_sysfs_rights "$MOD_LIVEPATCH" "" "drwxr-xr-x" +check_sysfs_rights "$MOD_LIVEPATCH" "enabled" "-rw-r--r--" +check_sysfs_value "$MOD_LIVEPATCH" "enabled" "1" +check_sysfs_rights "$MOD_LIVEPATCH" "force" "--w-------" +check_sysfs_rights "$MOD_LIVEPATCH" "transition" "-r--r--r--" +check_sysfs_value "$MOD_LIVEPATCH" "transition" "0" +check_sysfs_rights "$MOD_LIVEPATCH" "vmlinux/patched" "-r--r--r--" +check_sysfs_value "$MOD_LIVEPATCH" "vmlinux/patched" "1" + +disable_lp $MOD_LIVEPATCH + +unload_lp $MOD_LIVEPATCH + +check_result "% modprobe $MOD_LIVEPATCH +livepatch: enabling patch '$MOD_LIVEPATCH' +livepatch: '$MOD_LIVEPATCH': initializing patching transition +livepatch: '$MOD_LIVEPATCH': starting patching transition +livepatch: '$MOD_LIVEPATCH': completing patching transition +livepatch: '$MOD_LIVEPATCH': patching complete +% echo 0 > /sys/kernel/livepatch/$MOD_LIVEPATCH/enabled +livepatch: '$MOD_LIVEPATCH': initializing unpatching transition +livepatch: '$MOD_LIVEPATCH': starting unpatching transition +livepatch: '$MOD_LIVEPATCH': completing unpatching transition +livepatch: '$MOD_LIVEPATCH': unpatching complete +% rmmod $MOD_LIVEPATCH" + +start_test "sysfs test object/patched" + +MOD_LIVEPATCH=test_klp_callbacks_demo +MOD_TARGET=test_klp_callbacks_mod +load_lp $MOD_LIVEPATCH + +# check the "patch" file changes as target module loads/unloads +check_sysfs_value "$MOD_LIVEPATCH" "$MOD_TARGET/patched" "0" +load_mod $MOD_TARGET +check_sysfs_value "$MOD_LIVEPATCH" "$MOD_TARGET/patched" "1" +unload_mod $MOD_TARGET +check_sysfs_value "$MOD_LIVEPATCH" "$MOD_TARGET/patched" "0" + +disable_lp $MOD_LIVEPATCH +unload_lp $MOD_LIVEPATCH + +check_result "% modprobe test_klp_callbacks_demo +livepatch: enabling patch 'test_klp_callbacks_demo' +livepatch: 'test_klp_callbacks_demo': initializing patching transition +test_klp_callbacks_demo: pre_patch_callback: vmlinux +livepatch: 'test_klp_callbacks_demo': starting patching transition +livepatch: 'test_klp_callbacks_demo': completing patching transition +test_klp_callbacks_demo: post_patch_callback: vmlinux +livepatch: 'test_klp_callbacks_demo': patching complete +% modprobe test_klp_callbacks_mod +livepatch: applying patch 'test_klp_callbacks_demo' to loading module 'test_klp_callbacks_mod' +test_klp_callbacks_demo: pre_patch_callback: test_klp_callbacks_mod -> [MODULE_STATE_COMING] Full formed, running module_init +test_klp_callbacks_demo: post_patch_callback: test_klp_callbacks_mod -> [MODULE_STATE_COMING] Full formed, running module_init +test_klp_callbacks_mod: test_klp_callbacks_mod_init +% rmmod test_klp_callbacks_mod +test_klp_callbacks_mod: test_klp_callbacks_mod_exit +test_klp_callbacks_demo: pre_unpatch_callback: test_klp_callbacks_mod -> [MODULE_STATE_GOING] Going away +livepatch: reverting patch 'test_klp_callbacks_demo' on unloading module 'test_klp_callbacks_mod' +test_klp_callbacks_demo: post_unpatch_callback: test_klp_callbacks_mod -> [MODULE_STATE_GOING] Going away +% echo 0 > /sys/kernel/livepatch/test_klp_callbacks_demo/enabled +livepatch: 'test_klp_callbacks_demo': initializing unpatching transition +test_klp_callbacks_demo: pre_unpatch_callback: vmlinux +livepatch: 'test_klp_callbacks_demo': starting unpatching transition +livepatch: 'test_klp_callbacks_demo': completing unpatching transition +test_klp_callbacks_demo: post_unpatch_callback: vmlinux +livepatch: 'test_klp_callbacks_demo': unpatching complete +% rmmod test_klp_callbacks_demo" + +exit 0 diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt index 65e53eb0840ba89ea9d3d1695570c3edfc6f93af..607b8d7e3ea34dc055789d493149c636f7040c5d 100644 --- a/tools/testing/selftests/lkdtm/tests.txt +++ b/tools/testing/selftests/lkdtm/tests.txt @@ -75,7 +75,9 @@ USERCOPY_KERNEL STACKLEAK_ERASING OK: the rest of the thread stack is properly erased CFI_FORWARD_PROTO CFI_BACKWARD call trace:|ok: control flow unchanged -FORTIFIED_STRSCPY -FORTIFIED_OBJECT -FORTIFIED_SUBOBJECT +FORTIFY_STRSCPY detected buffer overflow +FORTIFY_STR_OBJECT detected buffer overflow +FORTIFY_STR_MEMBER detected buffer overflow +FORTIFY_MEM_OBJECT detected buffer overflow +FORTIFY_MEM_MEMBER detected field-spanning write PPC_SLB_MULTIHIT Recovered diff --git a/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh b/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh index 46a97f318f58e4447a30063cdc50f98a69c215e3..74ee5067a8ce286fbb12c3d93456ab66834d8adb 100755 --- a/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh +++ b/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh @@ -134,6 +134,16 @@ offline_memory_expect_fail() return 0 } +online_all_offline_memory() +{ + for memory in `hotpluggable_offline_memory`; do + if ! online_memory_expect_success $memory; then + echo "$FUNCNAME $memory: unexpected fail" >&2 + retval=1 + fi + done +} + error=-12 priority=0 # Run with default of ratio=2 for Kselftest run @@ -197,8 +207,11 @@ echo -e "\t trying to offline $target out of $hotpluggable_num memory block(s):" for memory in `hotpluggable_online_memory`; do if [ "$target" -gt 0 ]; then echo "online->offline memory$memory" - if offline_memory_expect_success $memory; then + if offline_memory_expect_success $memory &>/dev/null; then target=$(($target - 1)) + echo "-> Success" + else + echo "-> Failure" fi fi done @@ -257,7 +270,7 @@ prerequisite_extra echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error for memory in `hotpluggable_online_memory`; do if [ $((RANDOM % 100)) -lt $ratio ]; then - offline_memory_expect_success $memory + offline_memory_expect_success $memory &>/dev/null fi done @@ -266,16 +279,16 @@ done # echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_ONLINE/error for memory in `hotpluggable_offline_memory`; do - online_memory_expect_fail $memory + if ! online_memory_expect_fail $memory; then + retval=1 + fi done # # Online all hot-pluggable memory # echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_ONLINE/error -for memory in `hotpluggable_offline_memory`; do - online_memory_expect_success $memory -done +online_all_offline_memory # # Test memory hot-remove error handling (online => offline) @@ -283,11 +296,18 @@ done echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error for memory in `hotpluggable_online_memory`; do if [ $((RANDOM % 100)) -lt $ratio ]; then - offline_memory_expect_fail $memory + if ! offline_memory_expect_fail $memory; then + retval=1 + fi fi done echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/MEM_GOING_OFFLINE/error /sbin/modprobe -q -r memory-notifier-error-inject +# +# Restore memory before exit +# +online_all_offline_memory + exit $retval diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index 0e5751af6247f1ff97b65afef4a3e3231ce0c86b..3d7adee7a3e6fa3b8b5f0343b4e79220250c1f00 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -1,42 +1,45 @@ # SPDX-License-Identifier: GPL-2.0-only +bind_bhash +cmsg_sender +fin_ack_lat +gro +hwtstamp_config +ioam6_parser +ip_defrag ipsec +ipv6_flowlabel +ipv6_flowlabel_mgr msg_zerocopy -socket +nettest psock_fanout psock_snd psock_tpacket -stress_reuseport_listen +reuseaddr_conflict +reuseaddr_ports_exhausted reuseport_addr_any reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa reuseport_dualstack -reuseaddr_conflict -tcp_mmap -udpgso -udpgso_bench_rx -udpgso_bench_tx -tcp_inq -tls -txring_overwrite -ip_defrag -ipv6_flowlabel -ipv6_flowlabel_mgr -so_txtime -tcp_fastopen_backup_key -nettest -fin_ack_lat -reuseaddr_ports_exhausted -hwtstamp_config rxtimestamp -timestamping -txtimestamp +sk_bind_sendto_listen +sk_connect_zero_addr +socket so_netns_cookie +so_txtime +stress_reuseport_listen +tap +tcp_fastopen_backup_key +tcp_inq +tcp_mmap test_unix_oob -gro -ioam6_parser +timestamping +tls toeplitz tun -cmsg_sender +txring_overwrite +txtimestamp +udpgso +udpgso_bench_rx +udpgso_bench_tx unix_connect -tap \ No newline at end of file diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index c0ee2955fe5424ec3976f681c46bf708844dfa24..2a6b0bc648c4ffb06fc688660801f8046dee17eb 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -37,11 +37,14 @@ TEST_PROGS += srv6_end_dt4_l3vpn_test.sh TEST_PROGS += srv6_end_dt6_l3vpn_test.sh TEST_PROGS += srv6_hencap_red_l3vpn_test.sh TEST_PROGS += srv6_hl2encap_red_l2vpn_test.sh +TEST_PROGS += srv6_end_next_csid_l3vpn_test.sh TEST_PROGS += vrf_strict_mode_test.sh TEST_PROGS += arp_ndisc_evict_nocarrier.sh TEST_PROGS += ndisc_unsolicited_na_test.sh TEST_PROGS += arp_ndisc_untracked_subnets.sh TEST_PROGS += stress_reuseport_listen.sh +TEST_PROGS += l2_tos_ttl_inherit.sh +TEST_PROGS += bind_bhash.sh TEST_PROGS_EXTENDED := in_netns.sh setup_loopback.sh setup_veth.sh TEST_PROGS_EXTENDED += toeplitz_client.sh toeplitz.sh TEST_GEN_FILES = socket nettest @@ -63,6 +66,10 @@ TEST_GEN_FILES += cmsg_sender TEST_GEN_FILES += stress_reuseport_listen TEST_PROGS += test_vxlan_vnifiltering.sh TEST_GEN_FILES += io_uring_zerocopy_tx +TEST_PROGS += io_uring_zerocopy_tx.sh +TEST_GEN_FILES += bind_bhash +TEST_GEN_PROGS += sk_bind_sendto_listen +TEST_GEN_PROGS += sk_connect_zero_addr TEST_FILES := settings @@ -73,3 +80,4 @@ include bpf/Makefile $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread $(OUTPUT)/tcp_inq: LDLIBS += -lpthread +$(OUTPUT)/bind_bhash: LDLIBS += -lpthread diff --git a/tools/testing/selftests/net/bind_bhash.c b/tools/testing/selftests/net/bind_bhash.c new file mode 100644 index 0000000000000000000000000000000000000000..57ff67a3751eb3bcb880244b0eb407a667d31d76 --- /dev/null +++ b/tools/testing/selftests/net/bind_bhash.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This times how long it takes to bind to a port when the port already + * has multiple sockets in its bhash table. + * + * In the setup(), we populate the port's bhash table with + * MAX_THREADS * MAX_CONNECTIONS number of entries. + */ + +#include <unistd.h> +#include <stdio.h> +#include <netdb.h> +#include <pthread.h> +#include <string.h> +#include <stdbool.h> + +#define MAX_THREADS 600 +#define MAX_CONNECTIONS 40 + +static const char *setup_addr_v6 = "::1"; +static const char *setup_addr_v4 = "127.0.0.1"; +static const char *setup_addr; +static const char *bind_addr; +static const char *port; +bool use_v6; +int ret; + +static int fd_array[MAX_THREADS][MAX_CONNECTIONS]; + +static int bind_socket(int opt, const char *addr) +{ + struct addrinfo *res, hint = {}; + int sock_fd, reuse = 1, err; + int domain = use_v6 ? AF_INET6 : AF_INET; + + sock_fd = socket(domain, SOCK_STREAM, 0); + if (sock_fd < 0) { + perror("socket fd err"); + return sock_fd; + } + + hint.ai_family = domain; + hint.ai_socktype = SOCK_STREAM; + + err = getaddrinfo(addr, port, &hint, &res); + if (err) { + perror("getaddrinfo failed"); + goto cleanup; + } + + if (opt) { + err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse)); + if (err) { + perror("setsockopt failed"); + goto cleanup; + } + } + + err = bind(sock_fd, res->ai_addr, res->ai_addrlen); + if (err) { + perror("failed to bind to port"); + goto cleanup; + } + + return sock_fd; + +cleanup: + close(sock_fd); + return err; +} + +static void *setup(void *arg) +{ + int sock_fd, i; + int *array = (int *)arg; + + for (i = 0; i < MAX_CONNECTIONS; i++) { + sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr); + if (sock_fd < 0) { + ret = sock_fd; + pthread_exit(&ret); + } + array[i] = sock_fd; + } + + return NULL; +} + +int main(int argc, const char *argv[]) +{ + int listener_fd, sock_fd, i, j; + pthread_t tid[MAX_THREADS]; + clock_t begin, end; + + if (argc != 4) { + printf("Usage: listener <port> <ipv6 | ipv4> <bind-addr>\n"); + return -1; + } + + port = argv[1]; + use_v6 = strcmp(argv[2], "ipv6") == 0; + bind_addr = argv[3]; + + setup_addr = use_v6 ? setup_addr_v6 : setup_addr_v4; + + listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr); + if (listen(listener_fd, 100) < 0) { + perror("listen failed"); + return -1; + } + + /* Set up threads to populate the bhash table entry for the port */ + for (i = 0; i < MAX_THREADS; i++) + pthread_create(&tid[i], NULL, setup, fd_array[i]); + + for (i = 0; i < MAX_THREADS; i++) + pthread_join(tid[i], NULL); + + if (ret) + goto done; + + begin = clock(); + + /* Bind to the same port on a different address */ + sock_fd = bind_socket(0, bind_addr); + if (sock_fd < 0) + goto done; + + end = clock(); + + printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC); + + /* clean up */ + close(sock_fd); + +done: + close(listener_fd); + for (i = 0; i < MAX_THREADS; i++) { + for (j = 0; i < MAX_THREADS; i++) + close(fd_array[i][j]); + } + + return 0; +} diff --git a/tools/testing/selftests/net/bind_bhash.sh b/tools/testing/selftests/net/bind_bhash.sh new file mode 100755 index 0000000000000000000000000000000000000000..ca0292d4b441996f3c07fef4492af4aa12d714de --- /dev/null +++ b/tools/testing/selftests/net/bind_bhash.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +NR_FILES=32768 +SAVED_NR_FILES=$(ulimit -n) + +# default values +port=443 +addr_v6="2001:0db8:0:f101::1" +addr_v4="10.8.8.8" +use_v6=true +addr="" + +usage() { + echo "Usage: $0 [-6 | -4] [-p port] [-a address]" + echo -e "\t6: use ipv6" + echo -e "\t4: use ipv4" + echo -e "\tport: Port number" + echo -e "\taddress: ip address" +} + +while getopts "ha:p:64" opt; do + case ${opt} in + h) + usage $0 + exit 0 + ;; + a) addr=$OPTARG;; + p) + port=$OPTARG;; + 6) + use_v6=true;; + 4) + use_v6=false;; + esac +done + +setup() { + if [[ "$use_v6" == true ]]; then + ip addr add $addr_v6 nodad dev eth0 + else + ip addr add $addr_v4 dev lo + fi + ulimit -n $NR_FILES +} + +cleanup() { + if [[ "$use_v6" == true ]]; then + ip addr del $addr_v6 dev eth0 + else + ip addr del $addr_v4/32 dev lo + fi + ulimit -n $SAVED_NR_FILES +} + +if [[ "$addr" != "" ]]; then + addr_v4=$addr; + addr_v6=$addr; +fi +setup +if [[ "$use_v6" == true ]] ; then + ./bind_bhash $port "ipv6" $addr_v6 +else + ./bind_bhash $port "ipv4" $addr_v4 +fi +cleanup diff --git a/tools/testing/selftests/net/fcnal-test.sh b/tools/testing/selftests/net/fcnal-test.sh index 03b586760164a481917517012f0a8da2137fc683..31c3b6ebd388bf31e5ee33c81faa805d54cd0512 100755 --- a/tools/testing/selftests/net/fcnal-test.sh +++ b/tools/testing/selftests/net/fcnal-test.sh @@ -1466,6 +1466,13 @@ ipv4_udp_novrf() run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP} log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF" + log_start + run_cmd_nsb nettest -D -s & + sleep 1 + run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP} -U + log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF, with connect()" + + log_start show_hint "Should fail 'Connection refused'" run_cmd nettest -D -r ${a} @@ -1525,6 +1532,13 @@ ipv4_udp_novrf() run_cmd nettest -D -d ${NSA_DEV} -S -r ${a} log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection" + log_start + run_cmd nettest -s -D & + sleep 1 + run_cmd nettest -D -d ${NSA_DEV} -S -r ${a} -U + log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" + + # IPv4 with device bind has really weird behavior - it overrides the # fib lookup, generates an rtable and tries to send the packet. This # causes failures for local traffic at different places @@ -1550,6 +1564,15 @@ ipv4_udp_novrf() sleep 1 run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection" + + log_start + show_hint "Should fail since addresses on loopback are out of device scope" + run_cmd nettest -D -s & + sleep 1 + run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -U + log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" + + done a=${NSA_IP} @@ -3157,6 +3180,13 @@ ipv6_udp_novrf() sleep 1 run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection" + + log_start + show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope" + run_cmd nettest -6 -D -s & + sleep 1 + run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S -U + log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()" done a=${NSA_IP6} diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index d5a0dd548989b919875a4ab3aac7461187df3b22..ee5e98204d3d2edbb2533744c7a829b45d132d6f 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -1223,6 +1223,11 @@ ipv4_fcnal() log_test $rc 0 "Delete nexthop route warning" run_cmd "$IP route delete 172.16.101.1/32 nhid 12" run_cmd "$IP nexthop del id 12" + + run_cmd "$IP nexthop add id 21 via 172.16.1.6 dev veth1" + run_cmd "$IP ro add 172.16.101.0/24 nhid 21" + run_cmd "$IP ro del 172.16.101.0/24 nexthop via 172.16.1.7 dev veth1 nexthop via 172.16.1.8 dev veth1" + log_test $? 2 "Delete multipath route with only nh id based entry" } ipv4_grp_fcnal() diff --git a/tools/testing/selftests/net/forwarding/devlink_lib.sh b/tools/testing/selftests/net/forwarding/devlink_lib.sh index de9944d42027c4366045b452acb8c462c0b718a2..601990c6881bf3d9649bc1e8268b446b1dcd0cb5 100644 --- a/tools/testing/selftests/net/forwarding/devlink_lib.sh +++ b/tools/testing/selftests/net/forwarding/devlink_lib.sh @@ -584,3 +584,8 @@ devlink_cell_size_get() devlink sb pool show "$DEVLINK_DEV" pool 0 -j \ | jq '.pool[][].cell_size' } + +devlink_pool_size_get() +{ + devlink sb show "$DEVLINK_DEV" -j | jq '.[][][]["size"]' +} diff --git a/tools/testing/selftests/net/forwarding/router_multicast.sh b/tools/testing/selftests/net/forwarding/router_multicast.sh index 57e90c873a2cf4e6e326ed57b6410414a9dcb597..5a58b1ec8aefb573c944834502084e3d46c4aaf5 100755 --- a/tools/testing/selftests/net/forwarding/router_multicast.sh +++ b/tools/testing/selftests/net/forwarding/router_multicast.sh @@ -28,7 +28,7 @@ # +------------------+ +------------------+ # -ALL_TESTS="mcast_v4 mcast_v6 rpf_v4 rpf_v6" +ALL_TESTS="mcast_v4 mcast_v6 rpf_v4 rpf_v6 unres_v4 unres_v6" NUM_NETIFS=6 source lib.sh source tc_common.sh @@ -406,6 +406,96 @@ rpf_v6() log_test "RPF IPv6" } +unres_v4() +{ + # Send a multicast packet not corresponding to an installed route, + # causing the kernel to queue the packet for resolution and emit an + # IGMPMSG_NOCACHE notification. smcrouted will react to this + # notification by consulting its (*, G) list and installing an (S, G) + # route, which will be used to forward the queued packet. + + RET=0 + + tc filter add dev $h2 ingress protocol ip pref 1 handle 1 flower \ + dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action drop + tc filter add dev $h3 ingress protocol ip pref 1 handle 1 flower \ + dst_ip 225.1.2.3 ip_proto udp dst_port 12345 action drop + + # Forwarding should fail before installing a matching (*, G). + $MZ $h1 -c 1 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \ + -a 00:11:22:33:44:55 -b 01:00:5e:01:02:03 \ + -A 198.51.100.2 -B 225.1.2.3 -q + + tc_check_packets "dev $h2 ingress" 1 0 + check_err $? "Multicast received on first host when should not" + tc_check_packets "dev $h3 ingress" 1 0 + check_err $? "Multicast received on second host when should not" + + # Create (*, G). Will not be installed in the kernel. + create_mcast_sg $rp1 0.0.0.0 225.1.2.3 $rp2 $rp3 + + $MZ $h1 -c 1 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \ + -a 00:11:22:33:44:55 -b 01:00:5e:01:02:03 \ + -A 198.51.100.2 -B 225.1.2.3 -q + + tc_check_packets "dev $h2 ingress" 1 1 + check_err $? "Multicast not received on first host" + tc_check_packets "dev $h3 ingress" 1 1 + check_err $? "Multicast not received on second host" + + delete_mcast_sg $rp1 0.0.0.0 225.1.2.3 $rp2 $rp3 + + tc filter del dev $h3 ingress protocol ip pref 1 handle 1 flower + tc filter del dev $h2 ingress protocol ip pref 1 handle 1 flower + + log_test "Unresolved queue IPv4" +} + +unres_v6() +{ + # Send a multicast packet not corresponding to an installed route, + # causing the kernel to queue the packet for resolution and emit an + # MRT6MSG_NOCACHE notification. smcrouted will react to this + # notification by consulting its (*, G) list and installing an (S, G) + # route, which will be used to forward the queued packet. + + RET=0 + + tc filter add dev $h2 ingress protocol ipv6 pref 1 handle 1 flower \ + dst_ip ff0e::3 ip_proto udp dst_port 12345 action drop + tc filter add dev $h3 ingress protocol ipv6 pref 1 handle 1 flower \ + dst_ip ff0e::3 ip_proto udp dst_port 12345 action drop + + # Forwarding should fail before installing a matching (*, G). + $MZ $h1 -6 -c 1 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \ + -a 00:11:22:33:44:55 -b 33:33:00:00:00:03 \ + -A 2001:db8:1::2 -B ff0e::3 -q + + tc_check_packets "dev $h2 ingress" 1 0 + check_err $? "Multicast received on first host when should not" + tc_check_packets "dev $h3 ingress" 1 0 + check_err $? "Multicast received on second host when should not" + + # Create (*, G). Will not be installed in the kernel. + create_mcast_sg $rp1 :: ff0e::3 $rp2 $rp3 + + $MZ $h1 -6 -c 1 -p 128 -t udp "ttl=10,sp=54321,dp=12345" \ + -a 00:11:22:33:44:55 -b 33:33:00:00:00:03 \ + -A 2001:db8:1::2 -B ff0e::3 -q + + tc_check_packets "dev $h2 ingress" 1 1 + check_err $? "Multicast not received on first host" + tc_check_packets "dev $h3 ingress" 1 1 + check_err $? "Multicast not received on second host" + + delete_mcast_sg $rp1 :: ff0e::3 $rp2 $rp3 + + tc filter del dev $h3 ingress protocol ipv6 pref 1 handle 1 flower + tc filter del dev $h2 ingress protocol ipv6 pref 1 handle 1 flower + + log_test "Unresolved queue IPv6" +} + trap cleanup EXIT setup_prepare diff --git a/tools/testing/selftests/net/forwarding/sch_red.sh b/tools/testing/selftests/net/forwarding/sch_red.sh index e714bae473fb4b424a900f8a94141c6c62eb2c3d..81f31179ac8878d3728755a4a814fae866468799 100755 --- a/tools/testing/selftests/net/forwarding/sch_red.sh +++ b/tools/testing/selftests/net/forwarding/sch_red.sh @@ -1,3 +1,4 @@ +#!/bin/bash # SPDX-License-Identifier: GPL-2.0 # This test sends one stream of traffic from H1 through a TBF shaper, to a RED diff --git a/tools/testing/selftests/net/forwarding/tsn_lib.sh b/tools/testing/selftests/net/forwarding/tsn_lib.sh index 60a1423e81167c77c0658479c7f6b0c56c29a3e4..b91bcd8008a9930529e6235c3a5d45e04740cf6b 100644 --- a/tools/testing/selftests/net/forwarding/tsn_lib.sh +++ b/tools/testing/selftests/net/forwarding/tsn_lib.sh @@ -22,8 +22,7 @@ fi phc2sys_start() { - local if_name=$1 - local uds_address=$2 + local uds_address=$1 local extra_args="" if ! [ -z "${uds_address}" ]; then @@ -33,9 +32,7 @@ phc2sys_start() phc2sys_log="$(mktemp)" chrt -f 10 phc2sys -m \ - -c ${if_name} \ - -s CLOCK_REALTIME \ - -O ${UTC_TAI_OFFSET} \ + -a -rr \ --step_threshold 0.00002 \ --first_step_threshold 0.00002 \ ${extra_args} \ @@ -53,15 +50,27 @@ phc2sys_stop() rm "${phc2sys_log}" 2> /dev/null } +# Replace space separators from interface list with underscores +if_names_to_label() +{ + local if_name_list="$1" + + echo "${if_name_list/ /_}" +} + ptp4l_start() { - local if_name=$1 + local if_names="$1" local slave_only=$2 local uds_address=$3 - local log="ptp4l_log_${if_name}" - local pid="ptp4l_pid_${if_name}" + local log="ptp4l_log_$(if_names_to_label ${if_names})" + local pid="ptp4l_pid_$(if_names_to_label ${if_names})" local extra_args="" + for if_name in ${if_names}; do + extra_args="${extra_args} -i ${if_name}" + done + if [ "${slave_only}" = true ]; then extra_args="${extra_args} -s" fi @@ -71,7 +80,6 @@ ptp4l_start() declare -g "${log}=$(mktemp)" chrt -f 10 ptp4l -m -2 -P \ - -i ${if_name} \ --step_threshold 0.00002 \ --first_step_threshold 0.00002 \ --tx_timestamp_timeout 100 \ @@ -80,16 +88,16 @@ ptp4l_start() > "${!log}" 2>&1 & declare -g "${pid}=$!" - echo "ptp4l for interface ${if_name} logs to ${!log} and has pid ${!pid}" + echo "ptp4l for interfaces ${if_names} logs to ${!log} and has pid ${!pid}" sleep 1 } ptp4l_stop() { - local if_name=$1 - local log="ptp4l_log_${if_name}" - local pid="ptp4l_pid_${if_name}" + local if_names="$1" + local log="ptp4l_log_$(if_names_to_label ${if_names})" + local pid="ptp4l_pid_$(if_names_to_label ${if_names})" { kill ${!pid} && wait ${!pid}; } 2> /dev/null rm "${!log}" 2> /dev/null @@ -136,10 +144,12 @@ isochron_recv_start() { local if_name=$1 local uds=$2 - local extra_args=$3 + local stats_port=$3 + local extra_args=$4 + local pid="isochron_pid_${stats_port}" if ! [ -z "${uds}" ]; then - extra_args="--unix-domain-socket ${uds}" + extra_args="${extra_args} --unix-domain-socket ${uds}" fi isochron rcv \ @@ -147,16 +157,20 @@ isochron_recv_start() --sched-priority 98 \ --sched-fifo \ --utc-tai-offset ${UTC_TAI_OFFSET} \ + --stats-port ${stats_port} \ --quiet \ ${extra_args} & \ - isochron_pid=$! + declare -g "${pid}=$!" sleep 1 } isochron_recv_stop() { - { kill ${isochron_pid} && wait ${isochron_pid}; } 2> /dev/null + local stats_port=$1 + local pid="isochron_pid_${stats_port}" + + { kill ${!pid} && wait ${!pid}; } 2> /dev/null } isochron_do() @@ -208,7 +222,7 @@ isochron_do() cpufreq_max ${ISOCHRON_CPU} - isochron_recv_start "${h2}" "${receiver_uds}" "${receiver_extra_args}" + isochron_recv_start "${h2}" "${receiver_uds}" 5000 "${receiver_extra_args}" isochron send \ --interface ${sender_if_name} \ @@ -229,7 +243,7 @@ isochron_do() ${extra_args} \ --quiet - isochron_recv_stop + isochron_recv_stop 5000 cpufreq_restore ${ISOCHRON_CPU} } diff --git a/tools/testing/selftests/net/io_uring_zerocopy_tx.c b/tools/testing/selftests/net/io_uring_zerocopy_tx.c index 9d64c560a2d61bd8c136df2de528357243486c31..154287740172617f0f6eb914429335b7e9a0cc89 100644 --- a/tools/testing/selftests/net/io_uring_zerocopy_tx.c +++ b/tools/testing/selftests/net/io_uring_zerocopy_tx.c @@ -47,7 +47,6 @@ enum { MODE_MIXED = 3, }; -static bool cfg_flush = false; static bool cfg_cork = false; static int cfg_mode = MODE_ZC_FIXED; static int cfg_nr_reqs = 8; @@ -166,21 +165,6 @@ static int io_uring_register_buffers(struct io_uring *ring, return (ret < 0) ? -errno : ret; } -static int io_uring_register_notifications(struct io_uring *ring, - unsigned nr, - struct io_uring_notification_slot *slots) -{ - int ret; - struct io_uring_notification_register r = { - .nr_slots = nr, - .data = (unsigned long)slots, - }; - - ret = syscall(__NR_io_uring_register, ring->ring_fd, - IORING_REGISTER_NOTIFIERS, &r, sizeof(r)); - return (ret < 0) ? -errno : ret; -} - static int io_uring_mmap(int fd, struct io_uring_params *p, struct io_uring_sq *sq, struct io_uring_cq *cq) { @@ -297,11 +281,10 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd, const void *buf, size_t len, int flags, - unsigned slot_idx, unsigned zc_flags) + unsigned zc_flags) { io_uring_prep_send(sqe, sockfd, buf, len, flags); - sqe->opcode = (__u8) IORING_OP_SENDZC_NOTIF; - sqe->notification_idx = slot_idx; + sqe->opcode = (__u8) IORING_OP_SEND_ZC; sqe->ioprio = zc_flags; } @@ -374,7 +357,6 @@ static int do_setup_tx(int domain, int type, int protocol) static void do_tx(int domain, int type, int protocol) { - struct io_uring_notification_slot b[1] = {{.tag = NOTIF_TAG}}; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; unsigned long packets = 0, bytes = 0; @@ -390,10 +372,6 @@ static void do_tx(int domain, int type, int protocol) if (ret) error(1, ret, "io_uring: queue init"); - ret = io_uring_register_notifications(&ring, 1, b); - if (ret) - error(1, ret, "io_uring: tx ctx registration"); - iov.iov_base = payload; iov.iov_len = cfg_payload_len; @@ -409,9 +387,8 @@ static void do_tx(int domain, int type, int protocol) for (i = 0; i < cfg_nr_reqs; i++) { unsigned zc_flags = 0; unsigned buf_idx = 0; - unsigned slot_idx = 0; unsigned mode = cfg_mode; - unsigned msg_flags = 0; + unsigned msg_flags = MSG_WAITALL; if (cfg_mode == MODE_MIXED) mode = rand() % 3; @@ -423,13 +400,9 @@ static void do_tx(int domain, int type, int protocol) cfg_payload_len, msg_flags); sqe->user_data = NONZC_TAG; } else { - if (cfg_flush) { - zc_flags |= IORING_RECVSEND_NOTIF_FLUSH; - compl_cqes++; - } io_uring_prep_sendzc(sqe, fd, payload, cfg_payload_len, - msg_flags, slot_idx, zc_flags); + msg_flags, zc_flags); if (mode == MODE_ZC_FIXED) { sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; sqe->buf_index = buf_idx; @@ -442,51 +415,62 @@ static void do_tx(int domain, int type, int protocol) if (ret != cfg_nr_reqs) error(1, ret, "submit"); + if (cfg_cork) + do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0); for (i = 0; i < cfg_nr_reqs; i++) { ret = io_uring_wait_cqe(&ring, &cqe); if (ret) error(1, ret, "wait cqe"); - if (cqe->user_data == NOTIF_TAG) { + if (cqe->user_data != NONZC_TAG && + cqe->user_data != ZC_TAG) + error(1, -EINVAL, "invalid cqe->user_data"); + + if (cqe->flags & IORING_CQE_F_NOTIF) { + if (cqe->flags & IORING_CQE_F_MORE) + error(1, -EINVAL, "invalid notif flags"); + if (compl_cqes <= 0) + error(1, -EINVAL, "notification mismatch"); compl_cqes--; i--; - } else if (cqe->user_data != NONZC_TAG && - cqe->user_data != ZC_TAG) { - error(1, cqe->res, "invalid user_data"); - } else if (cqe->res <= 0 && cqe->res != -EAGAIN) { + io_uring_cqe_seen(&ring); + continue; + } + if (cqe->flags & IORING_CQE_F_MORE) { + if (cqe->user_data != ZC_TAG) + error(1, cqe->res, "unexpected F_MORE"); + compl_cqes++; + } + if (cqe->res >= 0) { + packets++; + bytes += cqe->res; + } else if (cqe->res != -EAGAIN) { error(1, cqe->res, "send failed"); - } else { - if (cqe->res > 0) { - packets++; - bytes += cqe->res; - } - /* failed requests don't flush */ - if (cfg_flush && - cqe->res <= 0 && - cqe->user_data == ZC_TAG) - compl_cqes--; } io_uring_cqe_seen(&ring); } - if (cfg_cork) - do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0); } while (gettimeofday_ms() < tstop); - if (close(fd)) - error(1, errno, "close"); - - fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n", - packets, bytes >> 20, - packets / (cfg_runtime_ms / 1000), - (bytes >> 20) / (cfg_runtime_ms / 1000)); - while (compl_cqes) { ret = io_uring_wait_cqe(&ring, &cqe); if (ret) error(1, ret, "wait cqe"); + if (cqe->flags & IORING_CQE_F_MORE) + error(1, -EINVAL, "invalid notif flags"); + if (!(cqe->flags & IORING_CQE_F_NOTIF)) + error(1, -EINVAL, "missing notif flag"); + io_uring_cqe_seen(&ring); compl_cqes--; } + + fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n", + packets, bytes >> 20, + packets / (cfg_runtime_ms / 1000), + (bytes >> 20) / (cfg_runtime_ms / 1000)); + + if (close(fd)) + error(1, errno, "close"); } static void do_test(int domain, int type, int protocol) @@ -500,8 +484,8 @@ static void do_test(int domain, int type, int protocol) static void usage(const char *filepath) { - error(1, 0, "Usage: %s [-f] [-n<N>] [-z0] [-s<payload size>] " - "(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath); + error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] " + "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath); } static void parse_opts(int argc, char **argv) @@ -519,7 +503,7 @@ static void parse_opts(int argc, char **argv) usage(argv[0]); cfg_payload_len = max_payload_len; - while ((c = getopt(argc, argv, "46D:p:s:t:n:fc:m:")) != -1) { + while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) { switch (c) { case '4': if (cfg_family != PF_UNSPEC) @@ -548,9 +532,6 @@ static void parse_opts(int argc, char **argv) case 'n': cfg_nr_reqs = strtoul(optarg, NULL, 0); break; - case 'f': - cfg_flush = 1; - break; case 'c': cfg_cork = strtol(optarg, NULL, 0); break; @@ -583,8 +564,6 @@ static void parse_opts(int argc, char **argv) if (cfg_payload_len > max_payload_len) error(1, 0, "-s: payload exceeds max (%d)", max_payload_len); - if (cfg_mode == MODE_NONZC && cfg_flush) - error(1, 0, "-f: only zerocopy modes support notifications"); if (optind != argc - 1) usage(argv[0]); } diff --git a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh index 6a65e44376408b09c3c634d6d35c680549ac3331..32aa6e9dacc26d943480114a02903007f506326e 100755 --- a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh +++ b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh @@ -25,15 +25,11 @@ readonly path_sysctl_mem="net.core.optmem_max" # No arguments: automated test if [[ "$#" -eq "0" ]]; then IPs=( "4" "6" ) - protocols=( "tcp" "udp" ) for IP in "${IPs[@]}"; do - for proto in "${protocols[@]}"; do - for mode in $(seq 1 3); do - $0 "$IP" "$proto" -m "$mode" -t 1 -n 32 - $0 "$IP" "$proto" -m "$mode" -t 1 -n 32 -f - $0 "$IP" "$proto" -m "$mode" -t 1 -n 32 -c -f - done + for mode in $(seq 1 3); do + $0 "$IP" udp -m "$mode" -t 1 -n 32 + $0 "$IP" tcp -m "$mode" -t 1 -n 32 done done diff --git a/tools/testing/selftests/net/ipsec.c b/tools/testing/selftests/net/ipsec.c index cc10c10c5ed9670b3c23532ce4d220e5e8006fb5..9a8229abfa026a73bc20d586b70bdd1ce5cc4079 100644 --- a/tools/testing/selftests/net/ipsec.c +++ b/tools/testing/selftests/net/ipsec.c @@ -58,6 +58,8 @@ #define VETH_FMT "ktst-%d" #define VETH_LEN 12 +#define XFRM_ALGO_NR_KEYS 29 + static int nsfd_parent = -1; static int nsfd_childa = -1; static int nsfd_childb = -1; @@ -75,6 +77,43 @@ const unsigned int ping_timeout = 300; const unsigned int ping_count = 100; const unsigned int ping_success = 80; +struct xfrm_key_entry { + char algo_name[35]; + int key_len; +}; + +struct xfrm_key_entry xfrm_key_entries[] = { + {"digest_null", 0}, + {"ecb(cipher_null)", 0}, + {"cbc(des)", 64}, + {"hmac(md5)", 128}, + {"cmac(aes)", 128}, + {"xcbc(aes)", 128}, + {"cbc(cast5)", 128}, + {"cbc(serpent)", 128}, + {"hmac(sha1)", 160}, + {"hmac(rmd160)", 160}, + {"cbc(des3_ede)", 192}, + {"hmac(sha256)", 256}, + {"cbc(aes)", 256}, + {"cbc(camellia)", 256}, + {"cbc(twofish)", 256}, + {"rfc3686(ctr(aes))", 288}, + {"hmac(sha384)", 384}, + {"cbc(blowfish)", 448}, + {"hmac(sha512)", 512}, + {"rfc4106(gcm(aes))-128", 160}, + {"rfc4543(gcm(aes))-128", 160}, + {"rfc4309(ccm(aes))-128", 152}, + {"rfc4106(gcm(aes))-192", 224}, + {"rfc4543(gcm(aes))-192", 224}, + {"rfc4309(ccm(aes))-192", 216}, + {"rfc4106(gcm(aes))-256", 288}, + {"rfc4543(gcm(aes))-256", 288}, + {"rfc4309(ccm(aes))-256", 280}, + {"rfc7539(chacha20,poly1305)-128", 0} +}; + static void randomize_buffer(void *buf, size_t buflen) { int *p = (int *)buf; @@ -767,65 +806,12 @@ static int do_ping(int cmd_fd, char *buf, size_t buf_len, struct in_addr from, static int xfrm_fill_key(char *name, char *buf, size_t buf_len, unsigned int *key_len) { - /* TODO: use set/map instead */ - if (strncmp(name, "digest_null", ALGO_LEN) == 0) - *key_len = 0; - else if (strncmp(name, "ecb(cipher_null)", ALGO_LEN) == 0) - *key_len = 0; - else if (strncmp(name, "cbc(des)", ALGO_LEN) == 0) - *key_len = 64; - else if (strncmp(name, "hmac(md5)", ALGO_LEN) == 0) - *key_len = 128; - else if (strncmp(name, "cmac(aes)", ALGO_LEN) == 0) - *key_len = 128; - else if (strncmp(name, "xcbc(aes)", ALGO_LEN) == 0) - *key_len = 128; - else if (strncmp(name, "cbc(cast5)", ALGO_LEN) == 0) - *key_len = 128; - else if (strncmp(name, "cbc(serpent)", ALGO_LEN) == 0) - *key_len = 128; - else if (strncmp(name, "hmac(sha1)", ALGO_LEN) == 0) - *key_len = 160; - else if (strncmp(name, "hmac(rmd160)", ALGO_LEN) == 0) - *key_len = 160; - else if (strncmp(name, "cbc(des3_ede)", ALGO_LEN) == 0) - *key_len = 192; - else if (strncmp(name, "hmac(sha256)", ALGO_LEN) == 0) - *key_len = 256; - else if (strncmp(name, "cbc(aes)", ALGO_LEN) == 0) - *key_len = 256; - else if (strncmp(name, "cbc(camellia)", ALGO_LEN) == 0) - *key_len = 256; - else if (strncmp(name, "cbc(twofish)", ALGO_LEN) == 0) - *key_len = 256; - else if (strncmp(name, "rfc3686(ctr(aes))", ALGO_LEN) == 0) - *key_len = 288; - else if (strncmp(name, "hmac(sha384)", ALGO_LEN) == 0) - *key_len = 384; - else if (strncmp(name, "cbc(blowfish)", ALGO_LEN) == 0) - *key_len = 448; - else if (strncmp(name, "hmac(sha512)", ALGO_LEN) == 0) - *key_len = 512; - else if (strncmp(name, "rfc4106(gcm(aes))-128", ALGO_LEN) == 0) - *key_len = 160; - else if (strncmp(name, "rfc4543(gcm(aes))-128", ALGO_LEN) == 0) - *key_len = 160; - else if (strncmp(name, "rfc4309(ccm(aes))-128", ALGO_LEN) == 0) - *key_len = 152; - else if (strncmp(name, "rfc4106(gcm(aes))-192", ALGO_LEN) == 0) - *key_len = 224; - else if (strncmp(name, "rfc4543(gcm(aes))-192", ALGO_LEN) == 0) - *key_len = 224; - else if (strncmp(name, "rfc4309(ccm(aes))-192", ALGO_LEN) == 0) - *key_len = 216; - else if (strncmp(name, "rfc4106(gcm(aes))-256", ALGO_LEN) == 0) - *key_len = 288; - else if (strncmp(name, "rfc4543(gcm(aes))-256", ALGO_LEN) == 0) - *key_len = 288; - else if (strncmp(name, "rfc4309(ccm(aes))-256", ALGO_LEN) == 0) - *key_len = 280; - else if (strncmp(name, "rfc7539(chacha20,poly1305)-128", ALGO_LEN) == 0) - *key_len = 0; + int i; + + for (i = 0; i < XFRM_ALGO_NR_KEYS; i++) { + if (strncmp(name, xfrm_key_entries[i].algo_name, ALGO_LEN) == 0) + *key_len = xfrm_key_entries[i].key_len; + } if (*key_len > buf_len) { printk("Can't pack a key - too big for buffer"); diff --git a/tools/testing/selftests/net/l2_tos_ttl_inherit.sh b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh new file mode 100755 index 0000000000000000000000000000000000000000..dca1e6f777a8979d0b0fedf5e7f2fb05c9515fc6 --- /dev/null +++ b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh @@ -0,0 +1,390 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +# Author: Matthias May <matthias.may@westermo.com> +# +# This script evaluates ip tunnels that are capable of carrying L2 traffic +# if they inherit or set the inheritable fields. +# Namely these tunnels are: 'gretap', 'vxlan' and 'geneve'. +# Checked inheritable fields are: TOS and TTL. +# The outer tunnel protocol of 'IPv4' or 'IPv6' is verified- +# As payload frames of type 'IPv4', 'IPv6' and 'other'(ARP) are verified. +# In addition this script also checks if forcing a specific field in the +# outer header is working. + +if [ "$(id -u)" != "0" ]; then + echo "Please run as root." + exit 0 +fi +if ! which tcpdump > /dev/null 2>&1; then + echo "No tcpdump found. Required for this test." + exit 0 +fi + +expected_tos="0x00" +expected_ttl="0" +failed=false + +get_random_tos() { + # Get a random hex tos value between 0x00 and 0xfc, a multiple of 4 + echo "0x$(tr -dc '0-9a-f' < /dev/urandom | head -c 1)\ +$(tr -dc '048c' < /dev/urandom | head -c 1)" +} +get_random_ttl() { + # Get a random dec value between 0 and 255 + printf "%d" "0x$(tr -dc '0-9a-f' < /dev/urandom | head -c 2)" +} +get_field() { + # Expects to get the 'head -n 1' of a captured frame by tcpdump. + # Parses this first line and returns the specified field. + local field="$1" + local input="$2" + local found=false + input="$(echo "$input" | tr -d '(),')" + for input_field in $input; do + if $found; then + echo "$input_field" + return + fi + # The next field that we iterate over is the looked for value + if [ "$input_field" = "$field" ]; then + found=true + fi + done + echo "0" +} +setup() { + local type="$1" + local outer="$2" + local inner="$3" + local tos_ttl="$4" + local vlan="$5" + local test_tos="0x00" + local test_ttl="0" + local ns="ip netns exec testing" + + # We don't want a test-tos of 0x00, + # because this is the value that we get when no tos is set. + expected_tos="$(get_random_tos)" + while [ "$expected_tos" = "0x00" ]; do + expected_tos="$(get_random_tos)" + done + if [ "$tos_ttl" = "random" ]; then + test_tos="$expected_tos" + tos="fixed $test_tos" + elif [ "$tos_ttl" = "inherit" ]; then + test_tos="$tos_ttl" + tos="inherit $expected_tos" + fi + + # We don't want a test-ttl of 64 or 0, + # because 64 is when no ttl is set and 0 is not a valid ttl. + expected_ttl="$(get_random_ttl)" + while [ "$expected_ttl" = "64" ] || [ "$expected_ttl" = "0" ]; do + expected_ttl="$(get_random_ttl)" + done + + if [ "$tos_ttl" = "random" ]; then + test_ttl="$expected_ttl" + ttl="fixed $test_ttl" + elif [ "$tos_ttl" = "inherit" ]; then + test_ttl="$tos_ttl" + ttl="inherit $expected_ttl" + fi + printf "│%7s │%6s │%6s │%13s │%13s │%6s │" \ + "$type" "$outer" "$inner" "$tos" "$ttl" "$vlan" + + # Create 'testing' netns, veth pair and connect main ns with testing ns + ip netns add testing + ip link add type veth + ip link set veth1 netns testing + ip link set veth0 up + $ns ip link set veth1 up + ip addr flush dev veth0 + $ns ip addr flush dev veth1 + + local local_addr1="" + local local_addr2="" + if [ "$type" = "gre" ] || [ "$type" = "vxlan" ]; then + if [ "$outer" = "4" ]; then + local_addr1="local 198.18.0.1" + local_addr2="local 198.18.0.2" + elif [ "$outer" = "6" ]; then + local_addr1="local fdd1:ced0:5d88:3fce::1" + local_addr2="local fdd1:ced0:5d88:3fce::2" + fi + fi + local vxlan="" + if [ "$type" = "vxlan" ]; then + vxlan="vni 100 dstport 4789" + fi + local geneve="" + if [ "$type" = "geneve" ]; then + geneve="vni 100" + fi + # Create tunnel and assign outer IPv4/IPv6 addresses + if [ "$outer" = "4" ]; then + if [ "$type" = "gre" ]; then + type="gretap" + fi + ip addr add 198.18.0.1/24 dev veth0 + $ns ip addr add 198.18.0.2/24 dev veth1 + ip link add name tep0 type $type $local_addr1 remote \ + 198.18.0.2 tos $test_tos ttl $test_ttl $vxlan $geneve + $ns ip link add name tep1 type $type $local_addr2 remote \ + 198.18.0.1 tos $test_tos ttl $test_ttl $vxlan $geneve + elif [ "$outer" = "6" ]; then + if [ "$type" = "gre" ]; then + type="ip6gretap" + fi + ip addr add fdd1:ced0:5d88:3fce::1/64 dev veth0 + $ns ip addr add fdd1:ced0:5d88:3fce::2/64 dev veth1 + ip link add name tep0 type $type $local_addr1 \ + remote fdd1:ced0:5d88:3fce::2 tos $test_tos ttl $test_ttl \ + $vxlan $geneve + $ns ip link add name tep1 type $type $local_addr2 \ + remote fdd1:ced0:5d88:3fce::1 tos $test_tos ttl $test_ttl \ + $vxlan $geneve + fi + + # Bring L2-tunnel link up and create VLAN on top + ip link set tep0 up + $ns ip link set tep1 up + ip addr flush dev tep0 + $ns ip addr flush dev tep1 + local parent + if $vlan; then + parent="vlan99-" + ip link add link tep0 name ${parent}0 type vlan id 99 + $ns ip link add link tep1 name ${parent}1 type vlan id 99 + ip link set ${parent}0 up + $ns ip link set ${parent}1 up + ip addr flush dev ${parent}0 + $ns ip addr flush dev ${parent}1 + else + parent="tep" + fi + + # Assign inner IPv4/IPv6 addresses + if [ "$inner" = "4" ] || [ "$inner" = "other" ]; then + ip addr add 198.19.0.1/24 brd + dev ${parent}0 + $ns ip addr add 198.19.0.2/24 brd + dev ${parent}1 + elif [ "$inner" = "6" ]; then + ip addr add fdd4:96cf:4eae:443b::1/64 dev ${parent}0 + $ns ip addr add fdd4:96cf:4eae:443b::2/64 dev ${parent}1 + fi +} + +verify() { + local outer="$1" + local inner="$2" + local tos_ttl="$3" + local vlan="$4" + + local ping_pid out captured_tos captured_ttl result + + local ping_dst + if [ "$inner" = "4" ]; then + ping_dst="198.19.0.2" + elif [ "$inner" = "6" ]; then + ping_dst="fdd4:96cf:4eae:443b::2" + elif [ "$inner" = "other" ]; then + ping_dst="198.19.0.3" # Generates ARPs which are not IPv4/IPv6 + fi + if [ "$tos_ttl" = "inherit" ]; then + ping -i 0.1 $ping_dst -Q "$expected_tos" -t "$expected_ttl" \ + 2>/dev/null 1>&2 & ping_pid="$!" + else + ping -i 0.1 $ping_dst 2>/dev/null 1>&2 & ping_pid="$!" + fi + local tunnel_type_offset tunnel_type_proto req_proto_offset req_offset + if [ "$type" = "gre" ]; then + tunnel_type_proto="0x2f" + elif [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + tunnel_type_proto="0x11" + fi + if [ "$outer" = "4" ]; then + tunnel_type_offset="9" + if [ "$inner" = "4" ]; then + req_proto_offset="47" + req_offset="58" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 12))" + req_offset="$((req_offset + 12))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x01 and \ + ip[$req_offset] = 0x08 2>/dev/null | head -n 1)" + elif [ "$inner" = "6" ]; then + req_proto_offset="44" + req_offset="78" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 12))" + req_offset="$((req_offset + 12))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x3a and \ + ip[$req_offset] = 0x80 2>/dev/null | head -n 1)" + elif [ "$inner" = "other" ]; then + req_proto_offset="36" + req_offset="45" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 12))" + req_offset="$((req_offset + 12))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + if [ "$tos_ttl" = "inherit" ]; then + expected_tos="0x00" + expected_ttl="64" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip[$tunnel_type_offset] = $tunnel_type_proto and \ + ip[$req_proto_offset] = 0x08 and \ + ip[$((req_proto_offset + 1))] = 0x06 and \ + ip[$req_offset] = 0x01 2>/dev/null | head -n 1)" + fi + elif [ "$outer" = "6" ]; then + if [ "$type" = "gre" ]; then + tunnel_type_offset="40" + elif [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + tunnel_type_offset="6" + fi + if [ "$inner" = "4" ]; then + local req_proto_offset="75" + local req_offset="86" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x01 and \ + ip6[$req_offset] = 0x08 2>/dev/null | head -n 1)" + elif [ "$inner" = "6" ]; then + local req_proto_offset="72" + local req_offset="106" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x3a and \ + ip6[$req_offset] = 0x80 2>/dev/null | head -n 1)" + elif [ "$inner" = "other" ]; then + local req_proto_offset="64" + local req_offset="73" + if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + if $vlan; then + req_proto_offset="$((req_proto_offset + 4))" + req_offset="$((req_offset + 4))" + fi + if [ "$tos_ttl" = "inherit" ]; then + expected_tos="0x00" + expected_ttl="64" + fi + out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \ + ip6[$tunnel_type_offset] = $tunnel_type_proto and \ + ip6[$req_proto_offset] = 0x08 and \ + ip6[$((req_proto_offset + 1))] = 0x06 and \ + ip6[$req_offset] = 0x01 2>/dev/null | head -n 1)" + fi + fi + kill -9 $ping_pid + wait $ping_pid 2>/dev/null + result="FAIL" + if [ "$outer" = "4" ]; then + captured_ttl="$(get_field "ttl" "$out")" + captured_tos="$(printf "0x%02x" "$(get_field "tos" "$out")")" + if [ "$captured_tos" = "$expected_tos" ] && + [ "$captured_ttl" = "$expected_ttl" ]; then + result="OK" + fi + elif [ "$outer" = "6" ]; then + captured_ttl="$(get_field "hlim" "$out")" + captured_tos="$(printf "0x%02x" "$(get_field "class" "$out")")" + if [ "$captured_tos" = "$expected_tos" ] && + [ "$captured_ttl" = "$expected_ttl" ]; then + result="OK" + fi + fi + + printf "%7s │\n" "$result" + if [ "$result" = "FAIL" ]; then + failed=true + if [ "$captured_tos" != "$expected_tos" ]; then + printf "│%43s%27s │\n" \ + "Expected TOS value: $expected_tos" \ + "Captured TOS value: $captured_tos" + fi + if [ "$captured_ttl" != "$expected_ttl" ]; then + printf "│%43s%27s │\n" \ + "Expected TTL value: $expected_ttl" \ + "Captured TTL value: $captured_ttl" + fi + printf "│%71s│\n" " " + fi +} + +cleanup() { + ip link del veth0 2>/dev/null + ip netns del testing 2>/dev/null + ip link del tep0 2>/dev/null +} + +printf "┌────────┬───────┬───────┬──────────────┬" +printf "──────────────┬───────┬────────┐\n" +for type in gre vxlan geneve; do + if ! $(modprobe "$type" 2>/dev/null); then + continue + fi + for outer in 4 6; do + printf "├────────┼───────┼───────┼──────────────┼" + printf "──────────────┼───────┼────────┤\n" + printf "│ Type │ outer | inner │ tos │" + printf " ttl │ vlan │ result │\n" + for inner in 4 6 other; do + printf "├────────┼───────┼───────┼──────────────┼" + printf "──────────────┼───────┼────────┤\n" + for tos_ttl in inherit random; do + for vlan in false true; do + setup "$type" "$outer" "$inner" \ + "$tos_ttl" "$vlan" + verify "$outer" "$inner" "$tos_ttl" \ + "$vlan" + cleanup + done + done + done + done +done +printf "└────────┴───────┴───────┴──────────────┴" +printf "──────────────┴───────┴────────┘\n" + +if $failed; then + exit 1 +fi diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index 24d4e9cb617e42a42e53abc3a1997656aa4d6b25..e54653ea2ed4f8c0862bd99c158ca1583c6ddd3d 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -72,6 +72,8 @@ static int cfg_wait; static uint32_t cfg_mark; static char *cfg_input; static int cfg_repeat = 1; +static int cfg_truncate; +static int cfg_rcv_trunc; struct cfg_cmsg_types { unsigned int cmsg_enabled:1; @@ -95,11 +97,15 @@ static struct cfg_sockopt_types cfg_sockopt_types; static void die_usage(void) { - fprintf(stderr, "Usage: mptcp_connect [-6] [-c cmsg] [-i file] [-I num] [-j] [-l] " + fprintf(stderr, "Usage: mptcp_connect [-6] [-c cmsg] [-f offset] [-i file] [-I num] [-j] [-l] " "[-m mode] [-M mark] [-o option] [-p port] [-P mode] [-j] [-l] [-r num] " "[-s MPTCP|TCP] [-S num] [-r num] [-t num] [-T num] [-u] [-w sec] connect_address\n"); fprintf(stderr, "\t-6 use ipv6\n"); fprintf(stderr, "\t-c cmsg -- test cmsg type <cmsg>\n"); + fprintf(stderr, "\t-f offset -- stop the I/O after receiving and sending the specified amount " + "of bytes. If there are unread bytes in the receive queue, that will cause a MPTCP " + "fastclose at close/shutdown. If offset is negative, expect the peer to close before " + "all the local data as been sent, thus toleration errors on write and EPIPE signals\n"); fprintf(stderr, "\t-i file -- read the data to send from the given file instead of stdin"); fprintf(stderr, "\t-I num -- repeat the transfer 'num' times. In listen mode accepts num " "incoming connections, in client mode, disconnect and reconnect to the server\n"); @@ -382,7 +388,7 @@ static size_t do_rnd_write(const int fd, char *buf, const size_t len) bw = write(fd, buf, do_w); if (bw < 0) - perror("write"); + return bw; /* let the join handshake complete, before going on */ if (cfg_join && first) { @@ -571,7 +577,7 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after .fd = peerfd, .events = POLLIN | POLLOUT, }; - unsigned int woff = 0, wlen = 0; + unsigned int woff = 0, wlen = 0, total_wlen = 0, total_rlen = 0; char wbuf[8192]; set_nonblock(peerfd, true); @@ -597,7 +603,16 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after } if (fds.revents & POLLIN) { - len = do_rnd_read(peerfd, rbuf, sizeof(rbuf)); + ssize_t rb = sizeof(rbuf); + + /* limit the total amount of read data to the trunc value*/ + if (cfg_truncate > 0) { + if (rb + total_rlen > cfg_truncate) + rb = cfg_truncate - total_rlen; + len = read(peerfd, rbuf, rb); + } else { + len = do_rnd_read(peerfd, rbuf, sizeof(rbuf)); + } if (len == 0) { /* no more data to receive: * peer has closed its write side @@ -612,10 +627,13 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after /* Else, still have data to transmit */ } else if (len < 0) { + if (cfg_rcv_trunc) + return 0; perror("read"); return 3; } + total_rlen += len; do_write(outfd, rbuf, len); } @@ -628,12 +646,21 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after if (wlen > 0) { ssize_t bw; + /* limit the total amount of written data to the trunc value */ + if (cfg_truncate > 0 && wlen + total_wlen > cfg_truncate) + wlen = cfg_truncate - total_wlen; + bw = do_rnd_write(peerfd, wbuf + woff, wlen); - if (bw < 0) + if (bw < 0) { + if (cfg_rcv_trunc) + return 0; + perror("write"); return 111; + } woff += bw; wlen -= bw; + total_wlen += bw; } else if (wlen == 0) { /* We have no more data to send. */ fds.events &= ~POLLOUT; @@ -652,10 +679,16 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after } if (fds.revents & (POLLERR | POLLNVAL)) { + if (cfg_rcv_trunc) + return 0; fprintf(stderr, "Unexpected revents: " "POLLERR/POLLNVAL(%x)\n", fds.revents); return 5; } + + if (cfg_truncate > 0 && total_wlen >= cfg_truncate && + total_rlen >= cfg_truncate) + break; } /* leave some time for late join/announce */ @@ -1160,11 +1193,13 @@ again: } /* close the client socket open only if we are not going to reconnect */ - ret = copyfd_io(fd_in, fd, 1, cfg_repeat == 1); + ret = copyfd_io(fd_in, fd, 1, 0); if (ret) return ret; - if (--cfg_repeat > 0) { + if (cfg_truncate > 0) { + xdisconnect(fd, peer->ai_addrlen); + } else if (--cfg_repeat > 0) { xdisconnect(fd, peer->ai_addrlen); /* the socket could be unblocking at this point, we need the @@ -1176,7 +1211,10 @@ again: if (cfg_input) close(fd_in); goto again; + } else { + close(fd); } + return 0; } @@ -1262,8 +1300,19 @@ static void parse_opts(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "6c:hi:I:jlm:M:o:p:P:r:R:s:S:t:T:w:")) != -1) { + while ((c = getopt(argc, argv, "6c:f:hi:I:jlm:M:o:p:P:r:R:s:S:t:T:w:")) != -1) { switch (c) { + case 'f': + cfg_truncate = atoi(optarg); + + /* when receiving a fastclose, ignore PIPE signals and + * all the I/O errors later in the code + */ + if (cfg_truncate < 0) { + cfg_rcv_trunc = true; + signal(SIGPIPE, handle_signal); + } + break; case 'j': cfg_join = true; cfg_mode = CFG_MODE_POLL; diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index ff83ef426df5265bfd86816f997d4c75974cd800..f3dd5f2a0272cbba80226ac915cbcd9e1927a9de 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -346,10 +346,21 @@ check_transfer() local in=$1 local out=$2 local what=$3 + local bytes=$4 local i a b local line - cmp -l "$in" "$out" | while read -r i a b; do + if [ -n "$bytes" ]; then + # when truncating we must check the size explicitly + local out_size=$(wc -c $out | awk '{print $1}') + if [ $out_size -ne $bytes ]; then + echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)" + fail_test + return 1 + fi + bytes="--bytes=${bytes}" + fi + cmp -l "$in" "$out" ${bytes} | while read -r i a b; do local sum=$((0${a} + 0${b})) if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then echo "[ FAIL ] $what does not match (in, out):" @@ -706,13 +717,39 @@ do_transfer() addr_nr_ns1=${addr_nr_ns1:10} fi + local flags="subflow" + local extra_cl_args="" + local extra_srv_args="" + local trunc_size="" if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then + if [ ${test_link_fail} -le 1 ]; then + echo "fastclose tests need test_link_fail argument" + fail_test + return 1 + fi + # disconnect - extra_args="$extra_args -I ${addr_nr_ns2:10}" + trunc_size=${test_link_fail} + local side=${addr_nr_ns2:10} + + if [ ${side} = "client" ]; then + extra_cl_args="-f ${test_link_fail}" + extra_srv_args="-f -1" + elif [ ${side} = "server" ]; then + extra_srv_args="-f ${test_link_fail}" + extra_cl_args="-f -1" + else + echo "wrong/unknown fastclose spec ${side}" + fail_test + return 1 + fi addr_nr_ns2=0 elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then userspace_pm=1 addr_nr_ns2=${addr_nr_ns2:10} + elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then + flags="${flags},fullmesh" + addr_nr_ns2=${addr_nr_ns2:9} fi if [ $userspace_pm -eq 1 ]; then @@ -733,39 +770,41 @@ do_transfer() local_addr="0.0.0.0" fi + extra_srv_args="$extra_args $extra_srv_args" if [ "$test_link_fail" -gt 1 ];then timeout ${timeout_test} \ ip netns exec ${listener_ns} \ ./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \ - $extra_args ${local_addr} < "$sinfail" > "$sout" & + $extra_srv_args ${local_addr} < "$sinfail" > "$sout" & else timeout ${timeout_test} \ ip netns exec ${listener_ns} \ ./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \ - $extra_args ${local_addr} < "$sin" > "$sout" & + $extra_srv_args ${local_addr} < "$sin" > "$sout" & fi local spid=$! wait_local_port_listen "${listener_ns}" "${port}" + extra_cl_args="$extra_args $extra_cl_args" if [ "$test_link_fail" -eq 0 ];then timeout ${timeout_test} \ ip netns exec ${connector_ns} \ ./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ - $extra_args $connect_addr < "$cin" > "$cout" & + $extra_cl_args $connect_addr < "$cin" > "$cout" & elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then ( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \ tee "$cinsent" | \ timeout ${timeout_test} \ ip netns exec ${connector_ns} \ ./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ - $extra_args $connect_addr > "$cout" & + $extra_cl_args $connect_addr > "$cout" & else tee "$cinsent" < "$cinfail" | \ timeout ${timeout_test} \ ip netns exec ${connector_ns} \ ./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \ - $extra_args $connect_addr > "$cout" & + $extra_cl_args $connect_addr > "$cout" & fi local cpid=$! @@ -832,12 +871,6 @@ do_transfer() fi fi - local flags="subflow" - if [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then - flags="${flags},fullmesh" - addr_nr_ns2=${addr_nr_ns2:9} - fi - # if newly added endpoints must be deleted, give the background msk # some time to created them [ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1 @@ -973,15 +1006,15 @@ do_transfer() fi if [ "$test_link_fail" -gt 1 ];then - check_transfer $sinfail $cout "file received by client" + check_transfer $sinfail $cout "file received by client" $trunc_size else - check_transfer $sin $cout "file received by client" + check_transfer $sin $cout "file received by client" $trunc_size fi retc=$? if [ "$test_link_fail" -eq 0 ];then - check_transfer $cin $sout "file received by server" + check_transfer $cin $sout "file received by server" $trunc_size else - check_transfer $cinsent $sout "file received by server" + check_transfer $cinsent $sout "file received by server" $trunc_size fi rets=$? @@ -1190,12 +1223,23 @@ chk_fclose_nr() { local fclose_tx=$1 local fclose_rx=$2 + local ns_invert=$3 local count local dump_stats + local ns_tx=$ns2 + local ns_rx=$ns1 + local extra_msg=" " + + if [[ $ns_invert = "invert" ]]; then + ns_tx=$ns1 + ns_rx=$ns2 + extra_msg=${extra_msg}"invert" + fi printf "%-${nr_blank}s %s" " " "ctx" - count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}') + count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}') [ -z "$count" ] && count=0 + [ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count" if [ "$count" != "$fclose_tx" ]; then echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx" fail_test @@ -1205,17 +1249,20 @@ chk_fclose_nr() fi echo -n " - fclzrx" - count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}') + count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}') [ -z "$count" ] && count=0 + [ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count" if [ "$count" != "$fclose_rx" ]; then echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx" fail_test dump_stats=1 else - echo "[ ok ]" + echo -n "[ ok ]" fi [ "${dump_stats}" = 1 ] && dump_stats + + echo "$extra_msg" } chk_rst_nr() @@ -1238,7 +1285,7 @@ chk_rst_nr() printf "%-${nr_blank}s %s" " " "rtx" count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}') [ -z "$count" ] && count=0 - if [ "$count" != "$rst_tx" ]; then + if [ $count -lt $rst_tx ]; then echo "[fail] got $count MP_RST[s] TX expected $rst_tx" fail_test dump_stats=1 @@ -1249,7 +1296,7 @@ chk_rst_nr() echo -n " - rstrx " count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}') [ -z "$count" ] && count=0 - if [ "$count" != "$rst_rx" ]; then + if [ "$count" -lt "$rst_rx" ]; then echo "[fail] got $count MP_RST[s] RX expected $rst_rx" fail_test dump_stats=1 @@ -2803,11 +2850,18 @@ fullmesh_tests() fastclose_tests() { if reset "fastclose test"; then - run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_2 + run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client chk_join_nr 0 0 0 chk_fclose_nr 1 1 chk_rst_nr 1 1 invert fi + + if reset "fastclose server test"; then + run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server + chk_join_nr 0 0 0 + chk_fclose_nr 1 1 invert + chk_rst_nr 1 1 + fi } pedit_action_pkts() diff --git a/tools/testing/selftests/net/nettest.c b/tools/testing/selftests/net/nettest.c index d9a6fd2cd9d318dd2c0377b43cf0896df118b62d..7900fa98eccb15abc28e97eb15d68cd40abca70d 100644 --- a/tools/testing/selftests/net/nettest.c +++ b/tools/testing/selftests/net/nettest.c @@ -127,6 +127,9 @@ struct sock_args { /* ESP in UDP encap test */ int use_xfrm; + + /* use send() and connect() instead of sendto */ + int datagram_connect; }; static int server_mode; @@ -979,6 +982,11 @@ static int send_msg(int sd, void *addr, socklen_t alen, struct sock_args *args) log_err_errno("write failed sending msg to peer"); return 1; } + } else if (args->datagram_connect) { + if (send(sd, msg, msglen, 0) < 0) { + log_err_errno("send failed sending msg to peer"); + return 1; + } } else if (args->ifindex && args->use_cmsg) { if (send_msg_cmsg(sd, addr, alen, args->ifindex, args->version)) return 1; @@ -1659,7 +1667,7 @@ static int connectsock(void *addr, socklen_t alen, struct sock_args *args) if (args->has_local_ip && bind_socket(sd, args)) goto err; - if (args->type != SOCK_STREAM) + if (args->type != SOCK_STREAM && !args->datagram_connect) goto out; if (args->password && tcp_md5sig(sd, addr, alen, args)) @@ -1854,7 +1862,7 @@ static int ipc_parent(int cpid, int fd, struct sock_args *args) return client_status; } -#define GETOPT_STR "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SCi6xL:0:1:2:3:Fbqf" +#define GETOPT_STR "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SUCi6xL:0:1:2:3:Fbqf" #define OPT_FORCE_BIND_KEY_IFINDEX 1001 #define OPT_NO_BIND_KEY_IFINDEX 1002 @@ -1891,6 +1899,7 @@ static void print_usage(char *prog) " -I dev bind socket to given device name - server mode\n" " -S use setsockopt (IP_UNICAST_IF or IP_MULTICAST_IF)\n" " to set device binding\n" + " -U Use connect() and send() for datagram sockets\n" " -f bind socket with the IP[V6]_FREEBIND option\n" " -C use cmsg and IP_PKTINFO to specify device binding\n" "\n" @@ -2074,6 +2083,9 @@ int main(int argc, char *argv[]) case 'x': args.use_xfrm = 1; break; + case 'U': + args.datagram_connect = 1; + break; default: print_usage(argv[0]); return 1; diff --git a/tools/testing/selftests/net/reuseport_bpf.c b/tools/testing/selftests/net/reuseport_bpf.c index 072d709c96b482e57bca41aa96165faeb4ecd4e8..65aea27d761cacca392ebb3696e360abb1389935 100644 --- a/tools/testing/selftests/net/reuseport_bpf.c +++ b/tools/testing/selftests/net/reuseport_bpf.c @@ -328,7 +328,7 @@ static void test_extra_filter(const struct test_params p) if (bind(fd1, addr, sockaddr_size())) error(1, errno, "failed to bind recv socket 1"); - if (!bind(fd2, addr, sockaddr_size()) && errno != EADDRINUSE) + if (!bind(fd2, addr, sockaddr_size()) || errno != EADDRINUSE) error(1, errno, "bind socket 2 should fail with EADDRINUSE"); free(addr); diff --git a/tools/testing/selftests/net/sk_bind_sendto_listen.c b/tools/testing/selftests/net/sk_bind_sendto_listen.c new file mode 100644 index 0000000000000000000000000000000000000000..b420d830f72c53b45b490d928cb670f6cf60f350 --- /dev/null +++ b/tools/testing/selftests/net/sk_bind_sendto_listen.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <arpa/inet.h> +#include <error.h> +#include <errno.h> +#include <unistd.h> + +int main(void) +{ + int fd1, fd2, one = 1; + struct sockaddr_in6 bind_addr = { + .sin6_family = AF_INET6, + .sin6_port = htons(20000), + .sin6_flowinfo = htonl(0), + .sin6_addr = {}, + .sin6_scope_id = 0, + }; + + inet_pton(AF_INET6, "::", &bind_addr.sin6_addr); + + fd1 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP); + if (fd1 < 0) { + error(1, errno, "socket fd1"); + return -1; + } + + if (setsockopt(fd1, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) { + error(1, errno, "setsockopt(SO_REUSEADDR) fd1"); + goto out_err1; + } + + if (bind(fd1, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) { + error(1, errno, "bind fd1"); + goto out_err1; + } + + if (sendto(fd1, NULL, 0, MSG_FASTOPEN, (struct sockaddr *)&bind_addr, + sizeof(bind_addr))) { + error(1, errno, "sendto fd1"); + goto out_err1; + } + + fd2 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP); + if (fd2 < 0) { + error(1, errno, "socket fd2"); + goto out_err1; + } + + if (setsockopt(fd2, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) { + error(1, errno, "setsockopt(SO_REUSEADDR) fd2"); + goto out_err2; + } + + if (bind(fd2, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) { + error(1, errno, "bind fd2"); + goto out_err2; + } + + if (sendto(fd2, NULL, 0, MSG_FASTOPEN, (struct sockaddr *)&bind_addr, + sizeof(bind_addr)) != -1) { + error(1, errno, "sendto fd2"); + goto out_err2; + } + + if (listen(fd2, 0)) { + error(1, errno, "listen"); + goto out_err2; + } + + close(fd2); + close(fd1); + return 0; + +out_err2: + close(fd2); + +out_err1: + close(fd1); + return -1; +} diff --git a/tools/testing/selftests/net/sk_connect_zero_addr.c b/tools/testing/selftests/net/sk_connect_zero_addr.c new file mode 100644 index 0000000000000000000000000000000000000000..4be418aefd9f78a50e2b9daa96976beec0f5b355 --- /dev/null +++ b/tools/testing/selftests/net/sk_connect_zero_addr.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <arpa/inet.h> +#include <error.h> +#include <errno.h> +#include <unistd.h> + +int main(void) +{ + int fd1, fd2, one = 1; + struct sockaddr_in6 bind_addr = { + .sin6_family = AF_INET6, + .sin6_port = htons(20000), + .sin6_flowinfo = htonl(0), + .sin6_addr = {}, + .sin6_scope_id = 0, + }; + + inet_pton(AF_INET6, "::", &bind_addr.sin6_addr); + + fd1 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP); + if (fd1 < 0) { + error(1, errno, "socket fd1"); + return -1; + } + + if (setsockopt(fd1, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) { + error(1, errno, "setsockopt(SO_REUSEADDR) fd1"); + goto out_err1; + } + + if (bind(fd1, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) { + error(1, errno, "bind fd1"); + goto out_err1; + } + + if (listen(fd1, 0)) { + error(1, errno, "listen"); + goto out_err1; + } + + fd2 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP); + if (fd2 < 0) { + error(1, errno, "socket fd2"); + goto out_err1; + } + + if (connect(fd2, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) { + error(1, errno, "bind fd2"); + goto out_err2; + } + + close(fd2); + close(fd1); + return 0; + +out_err2: + close(fd2); +out_err1: + close(fd1); + return -1; +} diff --git a/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh b/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh new file mode 100755 index 0000000000000000000000000000000000000000..87e414cc417c498bb8942ba2151d01a368dfa06a --- /dev/null +++ b/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh @@ -0,0 +1,1145 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# author: Andrea Mayer <andrea.mayer@uniroma2.it> +# +# This script is designed for testing the support of NEXT-C-SID flavor for SRv6 +# End behavior. +# A basic knowledge of SRv6 architecture [1] and of the compressed SID approach +# [2] is assumed for the reader. +# +# The network topology used in the selftest is depicted hereafter, composed by +# two hosts and four routers. Hosts hs-1 and hs-2 are connected through an +# IPv4/IPv6 L3 VPN service, offered by routers rt-1, rt-2, rt-3 and rt-4 using +# the NEXT-C-SID flavor. The key components for such VPNs are: +# +# i) The SRv6 H.Encaps/H.Encaps.Red behaviors [1] apply SRv6 Policies on +# traffic received by connected hosts, initiating the VPN tunnel; +# +# ii) The SRv6 End behavior [1] advances the active SID in the SID List +# carried by the SRH; +# +# iii) The NEXT-C-SID mechanism [2] offers the possibility of encoding several +# SRv6 segments within a single 128-bit SID address, referred to as a +# Compressed SID (C-SID) container. In this way, the length of the SID +# List can be drastically reduced. +# The NEXT-C-SID is provided as a "flavor" of the SRv6 End behavior +# which advances the current C-SID (i.e. the Locator-Node Function defined +# in [2]) with the next one carried in the Argument, if available. +# When no more C-SIDs are available in the Argument, the SRv6 End behavior +# will apply the End function selecting the next SID in the SID List. +# +# iv) The SRv6 End.DT46 behavior [1] is used for removing the SRv6 Policy and, +# thus, it terminates the VPN tunnel. Such a behavior is capable of +# handling, at the same time, both tunneled IPv4 and IPv6 traffic. +# +# [1] https://datatracker.ietf.org/doc/html/rfc8986 +# [2] https://datatracker.ietf.org/doc/html/draft-ietf-spring-srv6-srh-compression +# +# +# cafe::1 cafe::2 +# 10.0.0.1 10.0.0.2 +# +--------+ +--------+ +# | | | | +# | hs-1 | | hs-2 | +# | | | | +# +---+----+ +----+---+ +# cafe::/64 | | cafe::/64 +# 10.0.0.0/24 | | 10.0.0.0/24 +# +---+----+ +----+---+ +# | | fcf0:0:1:2::/64 | | +# | rt-1 +-------------------+ rt-2 | +# | | | | +# +---+----+ +----+---+ +# | . . | +# | fcf0:0:1:3::/64 . | +# | . . | +# | . . | +# fcf0:0:1:4::/64 | . | fcf0:0:2:3::/64 +# | . . | +# | . . | +# | fcf0:0:2:4::/64 . | +# | . . | +# +---+----+ +----+---+ +# | | | | +# | rt-4 +-------------------+ rt-3 | +# | | fcf0:0:3:4::/64 | | +# +---+----+ +----+---+ +# +# Every fcf0:0:x:y::/64 network interconnects the SRv6 routers rt-x with rt-y in +# the selftest network. +# +# Local SID/C-SID table +# ===================== +# +# Each SRv6 router is configured with a Local SID/C-SID table in which +# SIDs/C-SIDs are stored. Considering an SRv6 router rt-x, SIDs/C-SIDs are +# configured in the Local SID/C-SIDs table as follows: +# +# Local SID/C-SID table for SRv6 router rt-x +# +-----------------------------------------------------------+ +# |fcff:x::d46 is associated with the non-compressed SRv6 | +# | End.DT46 behavior | +# +-----------------------------------------------------------+ +# |fcbb:0:0x00::/48 is associated with the NEXT-C-SID flavor | +# | of SRv6 End behavior | +# +-----------------------------------------------------------+ +# |fcbb:0:0x00:d46::/64 is associated with the SRv6 End.DT46 | +# | behavior when NEXT-C-SID compression is turned on | +# +-----------------------------------------------------------+ +# +# The fcff::/16 prefix is reserved for implementing SRv6 services with regular +# (non compressed) SIDs. Reachability of SIDs is ensured by proper configuration +# of the IPv6 routing tables in the routers. +# Similarly, the fcbb:0::/32 prefix is reserved for implementing SRv6 VPN +# services leveraging the NEXT-C-SID compression mechanism. Indeed, the +# fcbb:0::/32 is used for encoding the Locator-Block while the Locator-Node +# Function is encoded with 16 bits. +# +# Incoming traffic classification and application of SRv6 Policies +# ================================================================ +# +# An SRv6 ingress router applies different SRv6 Policies to the traffic received +# from a connected host, considering the IPv4 or IPv6 destination address. +# SRv6 policy enforcement consists of encapsulating the received traffic into a +# new IPv6 packet with a given SID List contained in the SRH. +# When the SID List contains only one SID, the SRH could be omitted completely +# and that SID is stored directly in the IPv6 Destination Address (DA) (this is +# called "reduced" encapsulation). +# +# Test cases for NEXT-C-SID +# ========================= +# +# We consider two test cases for NEXT-C-SID: i) single SID and ii) double SID. +# +# In the single SID test case we have a number of segments that are all +# contained in a single Compressed SID (C-SID) container. Therefore the +# resulting SID List has only one SID. Using the reduced encapsulation format +# this will result in a packet with no SRH. +# +# In the double SID test case we have one segment carried in a Compressed SID +# (C-SID) container, followed by a regular (non compressed) SID. The resulting +# SID List has two segments and it is possible to test the advance to the next +# SID when all the C-SIDs in a C-SID container have been processed. Using the +# reduced encapsulation format this will result in a packet with an SRH +# containing 1 segment. +# +# For the single SID test case, we use the IPv4 addresses of hs-1 and hs-2, for +# the double SID test case, we use their IPv6 addresses. This is only done to +# simplify the test setup and avoid adding other hosts or multiple addresses on +# the same interface of a host. +# +# Traffic from hs-1 to hs-2 +# ------------------------- +# +# Packets generated from hs-1 and directed towards hs-2 are handled by rt-1 +# which applies the SRv6 Policies as follows: +# +# i) IPv6 DA=cafe::2, H.Encaps.Red with SID List=fcbb:0:0400:0300:0200:d46:: +# ii) IPv4 DA=10.0.0.2, H.Encaps.Red with SID List=fcbb:0:0300::,fcff:2::d46 +# +# ### i) single SID +# +# The router rt-1 is configured to enforce the given Policy through the SRv6 +# H.Encaps.Red behavior which avoids the presence of the SRH at all, since it +# pushes the single SID directly in the IPv6 DA. Such a SID encodes a whole +# C-SID container carrying several C-SIDs (e.g. 0400, 0300, etc). +# +# As the packet reaches the router rt-4, the enabled NEXT-C-SID SRv6 End +# behavior (associated with fcbb:0:0400::/48) is triggered. This behavior +# analyzes the IPv6 DA and checks whether the Argument of the C-SID container +# is zero or not. In this case, the Argument is *NOT* zero and the IPv6 DA is +# updated as follows: +# +# +---------------------------------------------------------------+ +# | Before applying the rt-4 enabled NEXT-C-SID SRv6 End behavior | +# +---------------------------------------------------------------+ +# | +---------- Argument | +# | vvvvvvvvvvvvvvvv | +# | IPv6 DA fcbb:0:0400:0300:0200:d46:: | +# | ^^^^ <-- shifting | +# | | | +# | Locator-Node Function | +# +---------------------------------------------------------------+ +# | After applying the rt-4 enabled NEXT-C-SID SRv6 End behavior | +# +---------------------------------------------------------------+ +# | +---------- Argument | +# | vvvvvvvvvvvv | +# | IPv6 DA fcbb:0:0300:0200:d46:: | +# | ^^^^ | +# | | | +# | Locator-Node Function | +# +---------------------------------------------------------------+ +# +# After having applied the enabled NEXT-C-SID SRv6 End behavior, the packet is +# sent to the next node, i.e. rt-3. +# +# The enabled NEXT-C-SID SRv6 End behavior on rt-3 is executed as the packet is +# received. This behavior processes the packet and updates the IPv6 DA with +# fcbb:0:0200:d46::, since the Argument is *NOT* zero. Then, the packet is sent +# to the router rt-2. +# +# The router rt-2 is configured for decapsulating the inner IPv6 packet and, +# for this reason, it applies the SRv6 End.DT46 behavior on the received +# packet. It is worth noting that the SRv6 End.DT46 behavior does not require +# the presence of the SRH: it is fully capable to operate properly on +# IPv4/IPv6-in-IPv6 encapsulations. +# At the end of the decap operation, the packet is sent to the +# host hs-2. +# +# ### ii) double SID +# +# The router rt-1 is configured to enforce the given Policy through the SRv6 +# H.Encaps.Red. As a result, the first SID fcbb:0:0300:: is stored into the +# IPv6 DA, while the SRH pushed into the packet is made of only one SID, i.e. +# fcff:2::d46. Hence, the packet sent by hs-1 to hs-2 is encapsulated in an +# outer IPv6 header plus the SRH. +# +# As the packet reaches the node rt-3, the router applies the enabled NEXT-C-SID +# SRv6 End behavior. +# +# +---------------------------------------------------------------+ +# | Before applying the rt-3 enabled NEXT-C-SID SRv6 End behavior | +# +---------------------------------------------------------------+ +# | +---------- Argument | +# | vvvv (Argument is all filled with zeros) | +# | IPv6 DA fcbb:0:0300:: | +# | ^^^^ | +# | | | +# | Locator-Node Function | +# +---------------------------------------------------------------+ +# | After applying the rt-3 enabled NEXT-C-SID SRv6 End behavior | +# +---------------------------------------------------------------+ +# | | +# | IPv6 DA fcff:2::d46 | +# | ^^^^^^^^^^^ | +# | | | +# | SID copied from the SID List contained in the SRH | +# +---------------------------------------------------------------+ +# +# Since the Argument of the C-SID container is zero, the behavior can not +# update the Locator-Node function with the next C-SID carried in the Argument +# itself. Thus, the enabled NEXT-C-SID SRv6 End behavior operates as the +# traditional End behavior: it updates the IPv6 DA by copying the next +# available SID in the SID List carried by the SRH. After that, the packet is +# sent to the node rt-2. +# +# Once the packet is received by rt-2, the router decapsulates the inner IPv6 +# packet using the SRv6 End.DT46 behavior (associated with the SID fcff:2::d46) +# and sends it to the host hs-2. +# +# Traffic from hs-2 to hs-1 +# ------------------------- +# +# Packets generated from hs-2 and directed towards hs-1 are handled by rt-2 +# which applies the SRv6 Policies as follows: +# +# i) IPv6 DA=cafe::1, SID List=fcbb:0:0300:0400:0100:d46:: +# ii) IPv4 DA=10.0.0.1, SID List=fcbb:0:0300::,fcff:1::d46 +# +# For simplicity, such SRv6 Policies were chosen so that, in both use cases (i) +# and (ii), the network paths crossed by traffic from hs-2 to hs-1 are the same +# as those taken by traffic from hs-1 to hs-2. +# In this way, traffic from hs-2 to hs-1 is processed similarly to traffic from +# hs-1 to hs-2. So, the traffic processing scheme turns out to be the same as +# that adopted in the use cases already examined (of course, it is necessary to +# consider the different SIDs/C-SIDs). + +# Kselftest framework requirement - SKIP code is 4. +readonly ksft_skip=4 + +readonly RDMSUFF="$(mktemp -u XXXXXXXX)" +readonly DUMMY_DEVNAME="dum0" +readonly VRF_TID=100 +readonly VRF_DEVNAME="vrf-${VRF_TID}" +readonly RT2HS_DEVNAME="veth-t${VRF_TID}" +readonly LOCALSID_TABLE_ID=90 +readonly IPv6_RT_NETWORK=fcf0:0 +readonly IPv6_HS_NETWORK=cafe +readonly IPv4_HS_NETWORK=10.0.0 +readonly VPN_LOCATOR_SERVICE=fcff +readonly DT46_FUNC=0d46 +readonly HEADEND_ENCAP="encap.red" + +# do not add ':' as separator +readonly LCBLOCK_ADDR=fcbb0000 +readonly LCBLOCK_BLEN=32 +# do not add ':' as separator +readonly LCNODEFUNC_FMT="0%d00" +readonly LCNODEFUNC_BLEN=16 + +readonly LCBLOCK_NODEFUNC_BLEN=$((LCBLOCK_BLEN + LCNODEFUNC_BLEN)) + +readonly CSID_CNTR_PREFIX="dead:beaf::/32" +# ID of the router used for testing the C-SID container cfgs +readonly CSID_CNTR_RT_ID_TEST=1 +# Routing table used for testing the C-SID container cfgs +readonly CSID_CNTR_RT_TABLE=91 + +# C-SID container configurations to be tested +# +# An entry of the array is defined as "a,b,c" where: +# - 'a' and 'b' elements represent respectively the Locator-Block length +# (lblen) in bits and the Locator-Node Function length (nflen) in bits. +# 'a' and 'b' can be set to default values using the placeholder "d" which +# indicates the default kernel values (32 for lblen and 16 for nflen); +# otherwise, any numeric value is accepted; +# - 'c' indicates whether the C-SID configuration provided by the values 'a' +# and 'b' should be considered valid ("y") or invalid ("n"). +declare -ra CSID_CONTAINER_CFGS=( + "d,d,y" + "d,16,y" + "16,d,y" + "16,32,y" + "32,16,y" + "48,8,y" + "8,48,y" + "d,0,n" + "0,d,n" + "32,0,n" + "0,32,n" + "17,d,n" + "d,17,n" + "120,16,n" + "16,120,n" + "0,128,n" + "128,0,n" + "130,0,n" + "0,130,n" + "0,0,n" +) + +PING_TIMEOUT_SEC=4 +PAUSE_ON_FAIL=${PAUSE_ON_FAIL:=no} + +# IDs of routers and hosts are initialized during the setup of the testing +# network +ROUTERS='' +HOSTS='' + +SETUP_ERR=1 + +ret=${ksft_skip} +nsuccess=0 +nfail=0 + +log_test() +{ + local rc="$1" + local expected="$2" + local msg="$3" + + if [ "${rc}" -eq "${expected}" ]; then + nsuccess=$((nsuccess+1)) + printf "\n TEST: %-60s [ OK ]\n" "${msg}" + else + ret=1 + nfail=$((nfail+1)) + printf "\n TEST: %-60s [FAIL]\n" "${msg}" + if [ "${PAUSE_ON_FAIL}" = "yes" ]; then + echo + echo "hit enter to continue, 'q' to quit" + read a + [ "$a" = "q" ] && exit 1 + fi + fi +} + +print_log_test_results() +{ + printf "\nTests passed: %3d\n" "${nsuccess}" + printf "Tests failed: %3d\n" "${nfail}" + + # when a test fails, the value of 'ret' is set to 1 (error code). + # Conversely, when all tests are passed successfully, the 'ret' value + # is set to 0 (success code). + if [ "${ret}" -ne 1 ]; then + ret=0 + fi +} + +log_section() +{ + echo + echo "################################################################################" + echo "TEST SECTION: $*" + echo "################################################################################" +} + +test_command_or_ksft_skip() +{ + local cmd="$1" + + if [ ! -x "$(command -v "${cmd}")" ]; then + echo "SKIP: Could not run test without \"${cmd}\" tool"; + exit "${ksft_skip}" + fi +} + +get_nodename() +{ + local name="$1" + + echo "${name}-${RDMSUFF}" +} + +get_rtname() +{ + local rtid="$1" + + get_nodename "rt-${rtid}" +} + +get_hsname() +{ + local hsid="$1" + + get_nodename "hs-${hsid}" +} + +__create_namespace() +{ + local name="$1" + + ip netns add "${name}" +} + +create_router() +{ + local rtid="$1" + local nsname + + nsname="$(get_rtname "${rtid}")" + + __create_namespace "${nsname}" +} + +create_host() +{ + local hsid="$1" + local nsname + + nsname="$(get_hsname "${hsid}")" + + __create_namespace "${nsname}" +} + +cleanup() +{ + local nsname + local i + + # destroy routers + for i in ${ROUTERS}; do + nsname="$(get_rtname "${i}")" + + ip netns del "${nsname}" &>/dev/null || true + done + + # destroy hosts + for i in ${HOSTS}; do + nsname="$(get_hsname "${i}")" + + ip netns del "${nsname}" &>/dev/null || true + done + + # check whether the setup phase was completed successfully or not. In + # case of an error during the setup phase of the testing environment, + # the selftest is considered as "skipped". + if [ "${SETUP_ERR}" -ne 0 ]; then + echo "SKIP: Setting up the testing environment failed" + exit "${ksft_skip}" + fi + + exit "${ret}" +} + +add_link_rt_pairs() +{ + local rt="$1" + local rt_neighs="$2" + local neigh + local nsname + local neigh_nsname + + nsname="$(get_rtname "${rt}")" + + for neigh in ${rt_neighs}; do + neigh_nsname="$(get_rtname "${neigh}")" + + ip link add "veth-rt-${rt}-${neigh}" netns "${nsname}" \ + type veth peer name "veth-rt-${neigh}-${rt}" \ + netns "${neigh_nsname}" + done +} + +get_network_prefix() +{ + local rt="$1" + local neigh="$2" + local p="${rt}" + local q="${neigh}" + + if [ "${p}" -gt "${q}" ]; then + p="${q}"; q="${rt}" + fi + + echo "${IPv6_RT_NETWORK}:${p}:${q}" +} + +# Setup the basic networking for the routers +setup_rt_networking() +{ + local rt="$1" + local rt_neighs="$2" + local nsname + local net_prefix + local devname + local neigh + + nsname="$(get_rtname "${rt}")" + + for neigh in ${rt_neighs}; do + devname="veth-rt-${rt}-${neigh}" + + net_prefix="$(get_network_prefix "${rt}" "${neigh}")" + + ip -netns "${nsname}" addr \ + add "${net_prefix}::${rt}/64" dev "${devname}" nodad + + ip -netns "${nsname}" link set "${devname}" up + done + + ip -netns "${nsname}" link add "${DUMMY_DEVNAME}" type dummy + + ip -netns "${nsname}" link set "${DUMMY_DEVNAME}" up + ip -netns "${nsname}" link set lo up + + ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.all.accept_dad=0 + ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.default.accept_dad=0 + ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.all.forwarding=1 + + ip netns exec "${nsname}" sysctl -wq net.ipv4.conf.all.rp_filter=0 + ip netns exec "${nsname}" sysctl -wq net.ipv4.conf.default.rp_filter=0 + ip netns exec "${nsname}" sysctl -wq net.ipv4.ip_forward=1 +} + +# build an ipv6 prefix/address based on the input string +# Note that the input string does not contain ':' and '::' which are considered +# to be implicit. +# e.g.: +# - input: fbcc00000400300 +# - output: fbcc:0000:0400:0300:0000:0000:0000:0000 +# ^^^^^^^^^^^^^^^^^^^ +# fill the address with 0s +build_ipv6_addr() +{ + local addr="$1" + local out="" + local strlen="${#addr}" + local padn + local i + + # add ":" every 4 digits (16 bits) + for (( i = 0; i < strlen; i++ )); do + if (( i > 0 && i < 32 && (i % 4) == 0 )); then + out="${out}:" + fi + + out="${out}${addr:$i:1}" + done + + # fill the remaining bits of the address with 0s + padn=$((32 - strlen)) + for (( i = padn; i > 0; i-- )); do + if (( i > 0 && i < 32 && (i % 4) == 0 )); then + out="${out}:" + fi + + out="${out}0" + done + + printf "${out}" +} + +build_csid() +{ + local nodeid="$1" + + printf "${LCNODEFUNC_FMT}" "${nodeid}" +} + +build_lcnode_func_prefix() +{ + local nodeid="$1" + local lcnodefunc + local prefix + local out + + lcnodefunc="$(build_csid "${nodeid}")" + prefix="$(build_ipv6_addr "${LCBLOCK_ADDR}${lcnodefunc}")" + + out="${prefix}/${LCBLOCK_NODEFUNC_BLEN}" + + echo "${out}" +} + +# Setup local SIDs for an SRv6 router +setup_rt_local_sids() +{ + local rt="$1" + local rt_neighs="$2" + local net_prefix + local devname + local nsname + local neigh + local lcnode_func_prefix + local lcblock_prefix + + nsname="$(get_rtname "${rt}")" + + for neigh in ${rt_neighs}; do + devname="veth-rt-${rt}-${neigh}" + + net_prefix="$(get_network_prefix "${rt}" "${neigh}")" + + # set underlay network routes for SIDs reachability + ip -netns "${nsname}" -6 route \ + add "${VPN_LOCATOR_SERVICE}:${neigh}::/32" \ + table "${LOCALSID_TABLE_ID}" \ + via "${net_prefix}::${neigh}" dev "${devname}" + + # set the underlay network for C-SIDs reachability + lcnode_func_prefix="$(build_lcnode_func_prefix "${neigh}")" + + ip -netns "${nsname}" -6 route \ + add "${lcnode_func_prefix}" \ + table "${LOCALSID_TABLE_ID}" \ + via "${net_prefix}::${neigh}" dev "${devname}" + done + + lcnode_func_prefix="$(build_lcnode_func_prefix "${rt}")" + + # enabled NEXT-C-SID SRv6 End behavior (note that "dev" is the dummy + # dum0 device chosen for the sake of simplicity). + ip -netns "${nsname}" -6 route \ + add "${lcnode_func_prefix}" \ + table "${LOCALSID_TABLE_ID}" \ + encap seg6local action End flavors next-csid \ + lblen "${LCBLOCK_BLEN}" nflen "${LCNODEFUNC_BLEN}" \ + dev "${DUMMY_DEVNAME}" + + # all SIDs for VPNs start with a common locator. Routes and SRv6 + # Endpoint behavior instaces are grouped together in the 'localsid' + # table. + ip -netns "${nsname}" -6 rule \ + add to "${VPN_LOCATOR_SERVICE}::/16" \ + lookup "${LOCALSID_TABLE_ID}" prio 999 + + # common locator block for NEXT-C-SIDS compression mechanism. + lcblock_prefix="$(build_ipv6_addr "${LCBLOCK_ADDR}")" + ip -netns "${nsname}" -6 rule \ + add to "${lcblock_prefix}/${LCBLOCK_BLEN}" \ + lookup "${LOCALSID_TABLE_ID}" prio 999 +} + +# build and install the SRv6 policy into the ingress SRv6 router as well as the +# decap SID in the egress one. +# args: +# $1 - src host (evaluate automatically the ingress router) +# $2 - dst host (evaluate automatically the egress router) +# $3 - SRv6 routers configured for steering traffic (End behaviors) +# $4 - single SID or double SID +# $5 - traffic type (IPv6 or IPv4) +__setup_l3vpn() +{ + local src="$1" + local dst="$2" + local end_rts="$3" + local mode="$4" + local traffic="$5" + local nsname + local policy + local container + local decapsid + local lcnfunc + local dt + local n + local rtsrc_nsname + local rtdst_nsname + + rtsrc_nsname="$(get_rtname "${src}")" + rtdst_nsname="$(get_rtname "${dst}")" + + container="${LCBLOCK_ADDR}" + + # build first SID (C-SID container) + for n in ${end_rts}; do + lcnfunc="$(build_csid "${n}")" + + container="${container}${lcnfunc}" + done + + if [ "${mode}" -eq 1 ]; then + # single SID policy + dt="$(build_csid "${dst}")${DT46_FUNC}" + container="${container}${dt}" + # build the full ipv6 address for the container + policy="$(build_ipv6_addr "${container}")" + + # build the decap SID used in the decap node + container="${LCBLOCK_ADDR}${dt}" + decapsid="$(build_ipv6_addr "${container}")" + else + # double SID policy + decapsid="${VPN_LOCATOR_SERVICE}:${dst}::${DT46_FUNC}" + + policy="$(build_ipv6_addr "${container}"),${decapsid}" + fi + + # apply encap policy + if [ "${traffic}" -eq 6 ]; then + ip -netns "${rtsrc_nsname}" -6 route \ + add "${IPv6_HS_NETWORK}::${dst}" vrf "${VRF_DEVNAME}" \ + encap seg6 mode "${HEADEND_ENCAP}" segs "${policy}" \ + dev "${VRF_DEVNAME}" + + ip -netns "${rtsrc_nsname}" -6 neigh \ + add proxy "${IPv6_HS_NETWORK}::${dst}" \ + dev "${RT2HS_DEVNAME}" + else + # "dev" must be different from the one where the packet is + # received, otherwise the proxy arp does not work. + ip -netns "${rtsrc_nsname}" -4 route \ + add "${IPv4_HS_NETWORK}.${dst}" vrf "${VRF_DEVNAME}" \ + encap seg6 mode "${HEADEND_ENCAP}" segs "${policy}" \ + dev "${VRF_DEVNAME}" + fi + + # apply decap + # Local End.DT46 behavior (decap) + ip -netns "${rtdst_nsname}" -6 route \ + add "${decapsid}" \ + table "${LOCALSID_TABLE_ID}" \ + encap seg6local action End.DT46 vrftable "${VRF_TID}" \ + dev "${VRF_DEVNAME}" +} + +# see __setup_l3vpn() +setup_ipv4_vpn_2sids() +{ + __setup_l3vpn "$1" "$2" "$3" 2 4 +} + +# see __setup_l3vpn() +setup_ipv6_vpn_1sid() +{ + __setup_l3vpn "$1" "$2" "$3" 1 6 +} + +setup_hs() +{ + local hs="$1" + local rt="$2" + local hsname + local rtname + + hsname="$(get_hsname "${hs}")" + rtname="$(get_rtname "${rt}")" + + ip netns exec "${hsname}" sysctl -wq net.ipv6.conf.all.accept_dad=0 + ip netns exec "${hsname}" sysctl -wq net.ipv6.conf.default.accept_dad=0 + + ip -netns "${hsname}" link add veth0 type veth \ + peer name "${RT2HS_DEVNAME}" netns "${rtname}" + + ip -netns "${hsname}" addr \ + add "${IPv6_HS_NETWORK}::${hs}/64" dev veth0 nodad + ip -netns "${hsname}" addr add "${IPv4_HS_NETWORK}.${hs}/24" dev veth0 + + ip -netns "${hsname}" link set veth0 up + ip -netns "${hsname}" link set lo up + + # configure the VRF on the router which is directly connected to the + # source host. + ip -netns "${rtname}" link \ + add "${VRF_DEVNAME}" type vrf table "${VRF_TID}" + ip -netns "${rtname}" link set "${VRF_DEVNAME}" up + + # enslave the veth interface connecting the router with the host to the + # VRF in the access router + ip -netns "${rtname}" link \ + set "${RT2HS_DEVNAME}" master "${VRF_DEVNAME}" + + # set default routes to unreachable for both ipv6 and ipv4 + ip -netns "${rtname}" -6 route \ + add unreachable default metric 4278198272 \ + vrf "${VRF_DEVNAME}" + ip -netns "${rtname}" -4 route \ + add unreachable default metric 4278198272 \ + vrf "${VRF_DEVNAME}" + + ip -netns "${rtname}" addr \ + add "${IPv6_HS_NETWORK}::254/64" dev "${RT2HS_DEVNAME}" nodad + ip -netns "${rtname}" addr \ + add "${IPv4_HS_NETWORK}.254/24" dev "${RT2HS_DEVNAME}" + + ip -netns "${rtname}" link set "${RT2HS_DEVNAME}" up + + ip netns exec "${rtname}" \ + sysctl -wq net.ipv6.conf."${RT2HS_DEVNAME}".proxy_ndp=1 + ip netns exec "${rtname}" \ + sysctl -wq net.ipv4.conf."${RT2HS_DEVNAME}".proxy_arp=1 + + # disable the rp_filter otherwise the kernel gets confused about how + # to route decap ipv4 packets. + ip netns exec "${rtname}" \ + sysctl -wq net.ipv4.conf."${RT2HS_DEVNAME}".rp_filter=0 + + ip netns exec "${rtname}" sh -c "echo 1 > /proc/sys/net/vrf/strict_mode" +} + +setup() +{ + local i + + # create routers + ROUTERS="1 2 3 4"; readonly ROUTERS + for i in ${ROUTERS}; do + create_router "${i}" + done + + # create hosts + HOSTS="1 2"; readonly HOSTS + for i in ${HOSTS}; do + create_host "${i}" + done + + # set up the links for connecting routers + add_link_rt_pairs 1 "2 3 4" + add_link_rt_pairs 2 "3 4" + add_link_rt_pairs 3 "4" + + # set up the basic connectivity of routers and routes required for + # reachability of SIDs. + setup_rt_networking 1 "2 3 4" + setup_rt_networking 2 "1 3 4" + setup_rt_networking 3 "1 2 4" + setup_rt_networking 4 "1 2 3" + + # set up the hosts connected to routers + setup_hs 1 1 + setup_hs 2 2 + + # set up default SRv6 Endpoints (i.e. SRv6 End and SRv6 End.DT46) + setup_rt_local_sids 1 "2 3 4" + setup_rt_local_sids 2 "1 3 4" + setup_rt_local_sids 3 "1 2 4" + setup_rt_local_sids 4 "1 2 3" + + # set up SRv6 Policies + + # create an IPv6 VPN between hosts hs-1 and hs-2. + # + # Direction hs-1 -> hs-2 + # - rt-1 encap (H.Encaps.Red) + # - rt-4 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-3 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-2 SRv6 End.DT46 behavior + setup_ipv6_vpn_1sid 1 2 "4 3" + + # Direction hs2 -> hs-1 + # - rt-2 encap (H.Encaps.Red) + # - rt-3 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-4 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-1 SRv6 End.DT46 behavior + setup_ipv6_vpn_1sid 2 1 "3 4" + + # create an IPv4 VPN between hosts hs-1 and hs-2 + # + # Direction hs-1 -> hs-2 + # - rt-1 encap (H.Encaps.Red) + # - rt-3 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-2 SRv6 End.DT46 behavior + setup_ipv4_vpn_2sids 1 2 "3" + + # Direction hs-2 -> hs-1 + # - rt-2 encap (H.Encaps.Red) + # - rt-3 SRv6 End behavior (NEXT-C-SID flavor) + # - rt-1 SRv6 End.DT46 behavior + setup_ipv4_vpn_2sids 2 1 "3" + + # testing environment was set up successfully + SETUP_ERR=0 +} + +check_rt_connectivity() +{ + local rtsrc="$1" + local rtdst="$2" + local prefix + local rtsrc_nsname + + rtsrc_nsname="$(get_rtname "${rtsrc}")" + + prefix="$(get_network_prefix "${rtsrc}" "${rtdst}")" + + ip netns exec "${rtsrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \ + "${prefix}::${rtdst}" >/dev/null 2>&1 +} + +check_and_log_rt_connectivity() +{ + local rtsrc="$1" + local rtdst="$2" + + check_rt_connectivity "${rtsrc}" "${rtdst}" + log_test $? 0 "Routers connectivity: rt-${rtsrc} -> rt-${rtdst}" +} + +check_hs_ipv6_connectivity() +{ + local hssrc="$1" + local hsdst="$2" + local hssrc_nsname + + hssrc_nsname="$(get_hsname "${hssrc}")" + + ip netns exec "${hssrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \ + "${IPv6_HS_NETWORK}::${hsdst}" >/dev/null 2>&1 +} + +check_hs_ipv4_connectivity() +{ + local hssrc="$1" + local hsdst="$2" + local hssrc_nsname + + hssrc_nsname="$(get_hsname "${hssrc}")" + + ip netns exec "${hssrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \ + "${IPv4_HS_NETWORK}.${hsdst}" >/dev/null 2>&1 +} + +check_and_log_hs2gw_connectivity() +{ + local hssrc="$1" + + check_hs_ipv6_connectivity "${hssrc}" 254 + log_test $? 0 "IPv6 Hosts connectivity: hs-${hssrc} -> gw" + + check_hs_ipv4_connectivity "${hssrc}" 254 + log_test $? 0 "IPv4 Hosts connectivity: hs-${hssrc} -> gw" +} + +check_and_log_hs_ipv6_connectivity() +{ + local hssrc="$1" + local hsdst="$2" + + check_hs_ipv6_connectivity "${hssrc}" "${hsdst}" + log_test $? 0 "IPv6 Hosts connectivity: hs-${hssrc} -> hs-${hsdst}" +} + +check_and_log_hs_ipv4_connectivity() +{ + local hssrc="$1" + local hsdst="$2" + + check_hs_ipv4_connectivity "${hssrc}" "${hsdst}" + log_test $? 0 "IPv4 Hosts connectivity: hs-${hssrc} -> hs-${hsdst}" +} + +router_tests() +{ + local i + local j + + log_section "IPv6 routers connectivity test" + + for i in ${ROUTERS}; do + for j in ${ROUTERS}; do + if [ "${i}" -eq "${j}" ]; then + continue + fi + + check_and_log_rt_connectivity "${i}" "${j}" + done + done +} + +host2gateway_tests() +{ + local hs + + log_section "IPv4/IPv6 connectivity test among hosts and gateways" + + for hs in ${HOSTS}; do + check_and_log_hs2gw_connectivity "${hs}" + done +} + +host_vpn_tests() +{ + log_section "SRv6 VPN connectivity test hosts (h1 <-> h2, IPv6)" + + check_and_log_hs_ipv6_connectivity 1 2 + check_and_log_hs_ipv6_connectivity 2 1 + + log_section "SRv6 VPN connectivity test hosts (h1 <-> h2, IPv4)" + + check_and_log_hs_ipv4_connectivity 1 2 + check_and_log_hs_ipv4_connectivity 2 1 +} + +__nextcsid_end_behavior_test() +{ + local nsname="$1" + local cmd="$2" + local blen="$3" + local flen="$4" + local layout="" + + if [ "${blen}" != "d" ]; then + layout="${layout} lblen ${blen}" + fi + + if [ "${flen}" != "d" ]; then + layout="${layout} nflen ${flen}" + fi + + ip -netns "${nsname}" -6 route \ + "${cmd}" "${CSID_CNTR_PREFIX}" \ + table "${CSID_CNTR_RT_TABLE}" \ + encap seg6local action End flavors next-csid ${layout} \ + dev "${DUMMY_DEVNAME}" &>/dev/null + + return "$?" +} + +rt_x_nextcsid_end_behavior_test() +{ + local rt="$1" + local blen="$2" + local flen="$3" + local nsname + local ret + + nsname="$(get_rtname "${rt}")" + + __nextcsid_end_behavior_test "${nsname}" "add" "${blen}" "${flen}" + ret="$?" + __nextcsid_end_behavior_test "${nsname}" "del" "${blen}" "${flen}" + + return "${ret}" +} + +__parse_csid_container_cfg() +{ + local cfg="$1" + local index="$2" + local out + + echo "${cfg}" | cut -d',' -f"${index}" +} + +csid_container_cfg_tests() +{ + local valid + local blen + local flen + local cfg + local ret + + log_section "C-SID Container config tests (legend: d='kernel default')" + + for cfg in "${CSID_CONTAINER_CFGS[@]}"; do + blen="$(__parse_csid_container_cfg "${cfg}" 1)" + flen="$(__parse_csid_container_cfg "${cfg}" 2)" + valid="$(__parse_csid_container_cfg "${cfg}" 3)" + + rt_x_nextcsid_end_behavior_test \ + "${CSID_CNTR_RT_ID_TEST}" \ + "${blen}" \ + "${flen}" + ret="$?" + + if [ "${valid}" == "y" ]; then + log_test "${ret}" 0 \ + "Accept valid C-SID container cfg (lblen=${blen}, nflen=${flen})" + else + log_test "${ret}" 2 \ + "Reject invalid C-SID container cfg (lblen=${blen}, nflen=${flen})" + fi + done +} + +test_iproute2_supp_or_ksft_skip() +{ + if ! ip route help 2>&1 | grep -qo "next-csid"; then + echo "SKIP: Missing SRv6 NEXT-C-SID flavor support in iproute2" + exit "${ksft_skip}" + fi +} + +test_dummy_dev_or_ksft_skip() +{ + local test_netns + + test_netns="dummy-$(mktemp -u XXXXXXXX)" + + if ! ip netns add "${test_netns}"; then + echo "SKIP: Cannot set up netns for testing dummy dev support" + exit "${ksft_skip}" + fi + + modprobe dummy &>/dev/null || true + if ! ip -netns "${test_netns}" link \ + add "${DUMMY_DEVNAME}" type dummy; then + echo "SKIP: dummy dev not supported" + + ip netns del "${test_netns}" + exit "${ksft_skip}" + fi + + ip netns del "${test_netns}" +} + +test_vrf_or_ksft_skip() +{ + modprobe vrf &>/dev/null || true + if [ ! -e /proc/sys/net/vrf/strict_mode ]; then + echo "SKIP: vrf sysctl does not exist" + exit "${ksft_skip}" + fi +} + +if [ "$(id -u)" -ne 0 ]; then + echo "SKIP: Need root privileges" + exit "${ksft_skip}" +fi + +# required programs to carry out this selftest +test_command_or_ksft_skip ip +test_command_or_ksft_skip ping +test_command_or_ksft_skip sysctl +test_command_or_ksft_skip grep +test_command_or_ksft_skip cut + +test_iproute2_supp_or_ksft_skip +test_dummy_dev_or_ksft_skip +test_vrf_or_ksft_skip + +set -e +trap cleanup EXIT + +setup +set +e + +csid_container_cfg_tests + +router_tests +host2gateway_tests +host_vpn_tests + +print_log_test_results diff --git a/tools/testing/selftests/netfilter/Makefile b/tools/testing/selftests/netfilter/Makefile index 600e3a19d5e28d0f85e601e39ea8b9e0ebc5bafd..4504ee07be08dad866087acdf835fd5a0341bc25 100644 --- a/tools/testing/selftests/netfilter/Makefile +++ b/tools/testing/selftests/netfilter/Makefile @@ -6,7 +6,7 @@ TEST_PROGS := nft_trans_stress.sh nft_fib.sh nft_nat.sh bridge_brouter.sh \ nft_concat_range.sh nft_conntrack_helper.sh \ nft_queue.sh nft_meta.sh nf_nat_edemux.sh \ ipip-conntrack-mtu.sh conntrack_tcp_unreplied.sh \ - conntrack_vrf.sh nft_synproxy.sh + conntrack_vrf.sh nft_synproxy.sh rpath.sh CFLAGS += $(shell pkg-config --cflags libmnl 2>/dev/null || echo "-I/usr/include/libmnl") LDLIBS = -lmnl diff --git a/tools/testing/selftests/netfilter/nft_concat_range.sh b/tools/testing/selftests/netfilter/nft_concat_range.sh index a6991877e50cdc83546502a8b4dafb5f0f63c5f8..e908009576c74d41823f6ef666b1b75e4ff995d7 100755 --- a/tools/testing/selftests/netfilter/nft_concat_range.sh +++ b/tools/testing/selftests/netfilter/nft_concat_range.sh @@ -91,7 +91,7 @@ src start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 3 @@ -116,7 +116,7 @@ src start 10 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp6 race_repeat 3 @@ -141,7 +141,7 @@ src start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 0 @@ -163,7 +163,7 @@ src mac start 10 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp6 race_repeat 0 @@ -185,7 +185,7 @@ src mac proto start 10 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp6 race_repeat 0 @@ -207,7 +207,7 @@ src addr4 start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 3 @@ -227,7 +227,7 @@ src addr6 port start 10 count 5 src_delta 2000 -tools sendip nc +tools sendip socat nc proto udp6 race_repeat 3 @@ -247,7 +247,7 @@ src mac proto addr4 start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 0 @@ -264,7 +264,7 @@ src mac start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 0 @@ -286,7 +286,7 @@ src mac addr4 start 1 count 5 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 0 @@ -337,7 +337,7 @@ src addr4 start 1 count 5 src_delta 2000 -tools sendip nc +tools sendip socat nc proto udp race_repeat 3 @@ -363,7 +363,7 @@ src mac start 1 count 1 src_delta 2000 -tools sendip nc bash +tools sendip socat nc bash proto udp race_repeat 0 @@ -541,6 +541,24 @@ setup_send_udp() { dst_port= src_addr4= } + elif command -v socat -v >/dev/null; then + send_udp() { + if [ -n "${src_addr4}" ]; then + B ip addr add "${src_addr4}" dev veth_b + __socatbind=",bind=${src_addr4}" + if [ -n "${src_port}" ];then + __socatbind="${__socatbind}:${src_port}" + fi + fi + + ip addr add "${dst_addr4}" dev veth_a 2>/dev/null + [ -z "${dst_port}" ] && dst_port=12345 + + echo "test4" | B socat -t 0.01 STDIN UDP4-DATAGRAM:${dst_addr4}:${dst_port}"${__socatbind}" + + src_addr4= + src_port= + } elif command -v nc >/dev/null; then if nc -u -w0 1.1.1.1 1 2>/dev/null; then # OpenBSD netcat @@ -606,6 +624,29 @@ setup_send_udp6() { dst_port= src_addr6= } + elif command -v socat -v >/dev/null; then + send_udp6() { + ip -6 addr add "${dst_addr6}" dev veth_a nodad \ + 2>/dev/null + + __socatbind6= + + if [ -n "${src_addr6}" ]; then + if [ -n "${src_addr6} != "${src_addr6_added} ]; then + B ip addr add "${src_addr6}" dev veth_b nodad + + src_addr6_added=${src_addr6} + fi + + __socatbind6=",bind=[${src_addr6}]" + + if [ -n "${src_port}" ] ;then + __socatbind6="${__socatbind6}:${src_port}" + fi + fi + + echo "test6" | B socat -t 0.01 STDIN UDP6-DATAGRAM:[${dst_addr6}]:${dst_port}"${__socatbind6}" + } elif command -v nc >/dev/null && nc -u -w0 1.1.1.1 1 2>/dev/null; then # GNU netcat might not work with IPv6, try next tool send_udp6() { diff --git a/tools/testing/selftests/netfilter/nft_conntrack_helper.sh b/tools/testing/selftests/netfilter/nft_conntrack_helper.sh index bf6b9626c7dd2749547cb13062188d4bb834c5b6..faa7778d7bd15e97b6662d665cb7c1dedaf7c31c 100755 --- a/tools/testing/selftests/netfilter/nft_conntrack_helper.sh +++ b/tools/testing/selftests/netfilter/nft_conntrack_helper.sh @@ -102,26 +102,42 @@ check_for_helper() ip netns exec ${netns} conntrack -L -f $family -p tcp --dport $port 2> /dev/null |grep -q 'helper=ftp' if [ $? -ne 0 ] ; then - echo "FAIL: ${netns} did not show attached helper $message" 1>&2 - ret=1 + if [ $autoassign -eq 0 ] ;then + echo "FAIL: ${netns} did not show attached helper $message" 1>&2 + ret=1 + else + echo "PASS: ${netns} did not show attached helper $message" 1>&2 + fi + else + if [ $autoassign -eq 0 ] ;then + echo "PASS: ${netns} connection on port $port has ftp helper attached" 1>&2 + else + echo "FAIL: ${netns} connection on port $port has ftp helper attached" 1>&2 + ret=1 + fi fi - echo "PASS: ${netns} connection on port $port has ftp helper attached" 1>&2 return 0 } test_helper() { local port=$1 - local msg=$2 + local autoassign=$2 + + if [ $autoassign -eq 0 ] ;then + msg="set via ruleset" + else + msg="auto-assign" + fi sleep 3 | ip netns exec ${ns2} nc -w 2 -l -p $port > /dev/null & sleep 1 | ip netns exec ${ns1} nc -w 2 10.0.1.2 $port > /dev/null & sleep 1 - check_for_helper "$ns1" "ip $msg" $port - check_for_helper "$ns2" "ip $msg" $port + check_for_helper "$ns1" "ip $msg" $port $autoassign + check_for_helper "$ns2" "ip $msg" $port $autoassign wait @@ -173,9 +189,9 @@ if [ $? -ne 0 ];then fi fi -test_helper 2121 "set via ruleset" -ip netns exec ${ns1} sysctl -q 'net.netfilter.nf_conntrack_helper=1' -ip netns exec ${ns2} sysctl -q 'net.netfilter.nf_conntrack_helper=1' -test_helper 21 "auto-assign" +test_helper 2121 0 +ip netns exec ${ns1} sysctl -qe 'net.netfilter.nf_conntrack_helper=1' +ip netns exec ${ns2} sysctl -qe 'net.netfilter.nf_conntrack_helper=1' +test_helper 21 1 exit $ret diff --git a/tools/testing/selftests/netfilter/nft_fib.sh b/tools/testing/selftests/netfilter/nft_fib.sh index fd76b69635a44c878b44810972b29df7d4644d89..dff476e45e772701be6a3f54c78eb0681c3baddb 100755 --- a/tools/testing/selftests/netfilter/nft_fib.sh +++ b/tools/testing/selftests/netfilter/nft_fib.sh @@ -188,6 +188,7 @@ test_ping() { ip netns exec ${nsrouter} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.forwarding=1 > /dev/null ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth1.forwarding=1 > /dev/null +ip netns exec ${nsrouter} sysctl net.ipv4.conf.all.rp_filter=0 > /dev/null ip netns exec ${nsrouter} sysctl net.ipv4.conf.veth0.rp_filter=0 > /dev/null sleep 3 diff --git a/tools/testing/selftests/netfilter/rpath.sh b/tools/testing/selftests/netfilter/rpath.sh new file mode 100755 index 0000000000000000000000000000000000000000..2d8da7bd8ab7472af75eafe315e9843abc3199a6 --- /dev/null +++ b/tools/testing/selftests/netfilter/rpath.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +# return code to signal skipped test +ksft_skip=4 + +# search for legacy iptables (it uses the xtables extensions +if iptables-legacy --version >/dev/null 2>&1; then + iptables='iptables-legacy' +elif iptables --version >/dev/null 2>&1; then + iptables='iptables' +else + iptables='' +fi + +if ip6tables-legacy --version >/dev/null 2>&1; then + ip6tables='ip6tables-legacy' +elif ! ip6tables --version >/dev/null 2>&1; then + ip6tables='ip6tables' +else + ip6tables='' +fi + +if nft --version >/dev/null 2>&1; then + nft='nft' +else + nft='' +fi + +if [ -z "$iptables$ip6tables$nft" ]; then + echo "SKIP: Test needs iptables, ip6tables or nft" + exit $ksft_skip +fi + +sfx=$(mktemp -u "XXXXXXXX") +ns1="ns1-$sfx" +ns2="ns2-$sfx" +trap "ip netns del $ns1; ip netns del $ns2" EXIT + +# create two netns, disable rp_filter in ns2 and +# keep IPv6 address when moving into VRF +ip netns add "$ns1" +ip netns add "$ns2" +ip netns exec "$ns2" sysctl -q net.ipv4.conf.all.rp_filter=0 +ip netns exec "$ns2" sysctl -q net.ipv4.conf.default.rp_filter=0 +ip netns exec "$ns2" sysctl -q net.ipv6.conf.all.keep_addr_on_down=1 + +# a standard connection between the netns, should not trigger rp filter +ip -net "$ns1" link add v0 type veth peer name v0 netns "$ns2" +ip -net "$ns1" link set v0 up; ip -net "$ns2" link set v0 up +ip -net "$ns1" a a 192.168.23.2/24 dev v0 +ip -net "$ns2" a a 192.168.23.1/24 dev v0 +ip -net "$ns1" a a fec0:23::2/64 dev v0 nodad +ip -net "$ns2" a a fec0:23::1/64 dev v0 nodad + +# rp filter testing: ns1 sends packets via v0 which ns2 would route back via d0 +ip -net "$ns2" link add d0 type dummy +ip -net "$ns2" link set d0 up +ip -net "$ns1" a a 192.168.42.2/24 dev v0 +ip -net "$ns2" a a 192.168.42.1/24 dev d0 +ip -net "$ns1" a a fec0:42::2/64 dev v0 nodad +ip -net "$ns2" a a fec0:42::1/64 dev d0 nodad + +# firewall matches to test +ip netns exec "$ns2" "$iptables" -t raw -A PREROUTING -s 192.168.0.0/16 -m rpfilter +ip netns exec "$ns2" "$ip6tables" -t raw -A PREROUTING -s fec0::/16 -m rpfilter +ip netns exec "$ns2" nft -f - <<EOF +table inet t { + chain c { + type filter hook prerouting priority raw; + ip saddr 192.168.0.0/16 fib saddr . iif oif exists counter + ip6 saddr fec0::/16 fib saddr . iif oif exists counter + } +} +EOF + +die() { + echo "FAIL: $*" + #ip netns exec "$ns2" "$iptables" -t raw -vS + #ip netns exec "$ns2" "$ip6tables" -t raw -vS + #ip netns exec "$ns2" nft list ruleset + exit 1 +} + +# check rule counters, return true if rule did not match +ipt_zero_rule() { # (command) + [ -n "$1" ] || return 0 + ip netns exec "$ns2" "$1" -t raw -vS | grep -q -- "-m rpfilter -c 0 0" +} +nft_zero_rule() { # (family) + [ -n "$nft" ] || return 0 + ip netns exec "$ns2" "$nft" list chain inet t c | \ + grep -q "$1 saddr .* counter packets 0 bytes 0" +} + +netns_ping() { # (netns, args...) + local netns="$1" + shift + ip netns exec "$netns" ping -q -c 1 -W 1 "$@" >/dev/null +} + +testrun() { + # clear counters first + [ -n "$iptables" ] && ip netns exec "$ns2" "$iptables" -t raw -Z + [ -n "$ip6tables" ] && ip netns exec "$ns2" "$ip6tables" -t raw -Z + if [ -n "$nft" ]; then + ( + echo "delete table inet t"; + ip netns exec "$ns2" nft -s list table inet t; + ) | ip netns exec "$ns2" nft -f - + fi + + # test 1: martian traffic should fail rpfilter matches + netns_ping "$ns1" -I v0 192.168.42.1 && \ + die "martian ping 192.168.42.1 succeeded" + netns_ping "$ns1" -I v0 fec0:42::1 && \ + die "martian ping fec0:42::1 succeeded" + + ipt_zero_rule "$iptables" || die "iptables matched martian" + ipt_zero_rule "$ip6tables" || die "ip6tables matched martian" + nft_zero_rule ip || die "nft IPv4 matched martian" + nft_zero_rule ip6 || die "nft IPv6 matched martian" + + # test 2: rpfilter match should pass for regular traffic + netns_ping "$ns1" 192.168.23.1 || \ + die "regular ping 192.168.23.1 failed" + netns_ping "$ns1" fec0:23::1 || \ + die "regular ping fec0:23::1 failed" + + ipt_zero_rule "$iptables" && die "iptables match not effective" + ipt_zero_rule "$ip6tables" && die "ip6tables match not effective" + nft_zero_rule ip && die "nft IPv4 match not effective" + nft_zero_rule ip6 && die "nft IPv6 match not effective" + +} + +testrun + +# repeat test with vrf device in $ns2 +ip -net "$ns2" link add vrf0 type vrf table 10 +ip -net "$ns2" link set vrf0 up +ip -net "$ns2" link set v0 master vrf0 + +testrun + +echo "PASS: netfilter reverse path match works as intended" +exit 0 diff --git a/tools/testing/selftests/nolibc/.gitignore b/tools/testing/selftests/nolibc/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4696df589d68e54e3bed875566a7d0598a0b3a34 --- /dev/null +++ b/tools/testing/selftests/nolibc/.gitignore @@ -0,0 +1,4 @@ +/initramfs/ +/nolibc-test +/run.out +/sysroot/ diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..69ea659caca98d27a3530c674cd81c3596d27699 --- /dev/null +++ b/tools/testing/selftests/nolibc/Makefile @@ -0,0 +1,135 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for nolibc tests +include ../../../scripts/Makefile.include + +# we're in ".../tools/testing/selftests/nolibc" +ifeq ($(srctree),) +srctree := $(patsubst %/tools/testing/selftests/,%,$(dir $(CURDIR))) +endif + +ifeq ($(ARCH),) +include $(srctree)/scripts/subarch.include +ARCH = $(SUBARCH) +endif + +# kernel image names by architecture +IMAGE_i386 = arch/x86/boot/bzImage +IMAGE_x86 = arch/x86/boot/bzImage +IMAGE_arm64 = arch/arm64/boot/Image +IMAGE_arm = arch/arm/boot/zImage +IMAGE_mips = vmlinuz +IMAGE_riscv = arch/riscv/boot/Image +IMAGE = $(IMAGE_$(ARCH)) +IMAGE_NAME = $(notdir $(IMAGE)) + +# default kernel configurations that appear to be usable +DEFCONFIG_i386 = defconfig +DEFCONFIG_x86 = defconfig +DEFCONFIG_arm64 = defconfig +DEFCONFIG_arm = multi_v7_defconfig +DEFCONFIG_mips = malta_defconfig +DEFCONFIG_riscv = defconfig +DEFCONFIG = $(DEFCONFIG_$(ARCH)) + +# optional tests to run (default = all) +TEST = + +# QEMU_ARCH: arch names used by qemu +QEMU_ARCH_i386 = i386 +QEMU_ARCH_x86 = x86_64 +QEMU_ARCH_arm64 = aarch64 +QEMU_ARCH_arm = arm +QEMU_ARCH_mips = mipsel # works with malta_defconfig +QEMU_ARCH_riscv = riscv64 +QEMU_ARCH = $(QEMU_ARCH_$(ARCH)) + +# QEMU_ARGS : some arch-specific args to pass to qemu +QEMU_ARGS_i386 = -M pc -append "console=ttyS0,9600 i8042.noaux panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS_x86 = -M pc -append "console=ttyS0,9600 i8042.noaux panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS_arm64 = -M virt -cpu cortex-a53 -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS_arm = -M virt -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS_mips = -M malta -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS_riscv = -M virt -append "console=ttyS0 panic=-1 $(TEST:%=NOLIBC_TEST=%)" +QEMU_ARGS = $(QEMU_ARGS_$(ARCH)) + +# OUTPUT is only set when run from the main makefile, otherwise +# it defaults to this nolibc directory. +OUTPUT ?= $(CURDIR)/ + +ifeq ($(V),1) +Q= +else +Q=@ +endif + +CFLAGS ?= -Os -fno-ident -fno-asynchronous-unwind-tables +LDFLAGS := -s + +help: + @echo "Supported targets under selftests/nolibc:" + @echo " all call the \"run\" target below" + @echo " help this help" + @echo " sysroot create the nolibc sysroot here (uses \$$ARCH)" + @echo " nolibc-test build the executable (uses \$$CC and \$$CROSS_COMPILE)" + @echo " initramfs prepare the initramfs with nolibc-test" + @echo " defconfig create a fresh new default config (uses \$$ARCH)" + @echo " kernel (re)build the kernel with the initramfs (uses \$$ARCH)" + @echo " run runs the kernel in QEMU after building it (uses \$$ARCH, \$$TEST)" + @echo " rerun runs a previously prebuilt kernel in QEMU (uses \$$ARCH, \$$TEST)" + @echo " clean clean the sysroot, initramfs, build and output files" + @echo "" + @echo "The output file is \"run.out\". Test ranges may be passed using \$$TEST." + @echo "" + @echo "Currently using the following variables:" + @echo " ARCH = $(ARCH)" + @echo " CROSS_COMPILE = $(CROSS_COMPILE)" + @echo " CC = $(CC)" + @echo " OUTPUT = $(OUTPUT)" + @echo " TEST = $(TEST)" + @echo " QEMU_ARCH = $(if $(QEMU_ARCH),$(QEMU_ARCH),UNKNOWN_ARCH) [determined from \$$ARCH]" + @echo " IMAGE_NAME = $(if $(IMAGE_NAME),$(IMAGE_NAME),UNKNOWN_ARCH) [determined from \$$ARCH]" + @echo "" + +all: run + +sysroot: sysroot/$(ARCH)/include + +sysroot/$(ARCH)/include: + $(QUIET_MKDIR)mkdir -p sysroot + $(Q)$(MAKE) -C ../../../include/nolibc ARCH=$(ARCH) OUTPUT=$(CURDIR)/sysroot/ headers_standalone + $(Q)mv sysroot/sysroot sysroot/$(ARCH) + +nolibc-test: nolibc-test.c sysroot/$(ARCH)/include + $(QUIET_CC)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ \ + -nostdlib -static -Isysroot/$(ARCH)/include $< -lgcc + +initramfs: nolibc-test + $(QUIET_MKDIR)mkdir -p initramfs + $(call QUIET_INSTALL, initramfs/init) + $(Q)cp nolibc-test initramfs/init + +defconfig: + $(Q)$(MAKE) -C $(srctree) ARCH=$(ARCH) CC=$(CC) CROSS_COMPILE=$(CROSS_COMPILE) mrproper $(DEFCONFIG) prepare + +kernel: initramfs + $(Q)$(MAKE) -C $(srctree) ARCH=$(ARCH) CC=$(CC) CROSS_COMPILE=$(CROSS_COMPILE) $(IMAGE_NAME) CONFIG_INITRAMFS_SOURCE=$(CURDIR)/initramfs + +# run the tests after building the kernel +run: kernel + $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" + $(Q)grep -w FAIL "$(CURDIR)/run.out" && echo "See all results in $(CURDIR)/run.out" || echo "$$(grep -c ^[0-9].*OK $(CURDIR)/run.out) test(s) passed." + +# re-run the tests from an existing kernel +rerun: + $(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out" + $(Q)grep -w FAIL "$(CURDIR)/run.out" && echo "See all results in $(CURDIR)/run.out" || echo "$$(grep -c ^[0-9].*OK $(CURDIR)/run.out) test(s) passed." + +clean: + $(call QUIET_CLEAN, sysroot) + $(Q)rm -rf sysroot + $(call QUIET_CLEAN, nolibc-test) + $(Q)rm -f nolibc-test + $(call QUIET_CLEAN, initramfs) + $(Q)rm -rf initramfs + $(call QUIET_CLEAN, run.out) + $(Q)rm -rf run.out diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c new file mode 100644 index 0000000000000000000000000000000000000000..78bced95ac63095e92be83de94b052b454f8d822 --- /dev/null +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -0,0 +1,757 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE + +/* platform-specific include files coming from the compiler */ +#include <limits.h> + +/* libc-specific include files + * The program may be built in 3 ways: + * $(CC) -nostdlib -include /path/to/nolibc.h => NOLIBC already defined + * $(CC) -nostdlib -I/path/to/nolibc/sysroot => _NOLIBC_* guards are present + * $(CC) with default libc => NOLIBC* never defined + */ +#ifndef NOLIBC +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#ifndef _NOLIBC_STDIO_H +/* standard libcs need more includes */ +#include <linux/reboot.h> +#include <sys/io.h> +#include <sys/ioctl.h> +#include <sys/mount.h> +#include <sys/reboot.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <sys/sysmacros.h> +#include <sys/time.h> +#include <sys/wait.h> +#include <dirent.h> +#include <errno.h> +#include <fcntl.h> +#include <poll.h> +#include <sched.h> +#include <signal.h> +#include <stdarg.h> +#include <unistd.h> +#endif +#endif + +/* will be used by nolibc by getenv() */ +char **environ; + +/* definition of a series of tests */ +struct test { + const char *name; // test name + int (*func)(int min, int max); // handler +}; + +#ifndef _NOLIBC_STDLIB_H +char *itoa(int i) +{ + static char buf[12]; + int ret; + + ret = snprintf(buf, sizeof(buf), "%d", i); + return (ret >= 0 && ret < sizeof(buf)) ? buf : "#err"; +} +#endif + +#define CASE_ERR(err) \ + case err: return #err + +/* returns the error name (e.g. "ENOENT") for common errors, "SUCCESS" for 0, + * or the decimal value for less common ones. + */ +const char *errorname(int err) +{ + switch (err) { + case 0: return "SUCCESS"; + CASE_ERR(EPERM); + CASE_ERR(ENOENT); + CASE_ERR(ESRCH); + CASE_ERR(EINTR); + CASE_ERR(EIO); + CASE_ERR(ENXIO); + CASE_ERR(E2BIG); + CASE_ERR(ENOEXEC); + CASE_ERR(EBADF); + CASE_ERR(ECHILD); + CASE_ERR(EAGAIN); + CASE_ERR(ENOMEM); + CASE_ERR(EACCES); + CASE_ERR(EFAULT); + CASE_ERR(ENOTBLK); + CASE_ERR(EBUSY); + CASE_ERR(EEXIST); + CASE_ERR(EXDEV); + CASE_ERR(ENODEV); + CASE_ERR(ENOTDIR); + CASE_ERR(EISDIR); + CASE_ERR(EINVAL); + CASE_ERR(ENFILE); + CASE_ERR(EMFILE); + CASE_ERR(ENOTTY); + CASE_ERR(ETXTBSY); + CASE_ERR(EFBIG); + CASE_ERR(ENOSPC); + CASE_ERR(ESPIPE); + CASE_ERR(EROFS); + CASE_ERR(EMLINK); + CASE_ERR(EPIPE); + CASE_ERR(EDOM); + CASE_ERR(ERANGE); + CASE_ERR(ENOSYS); + default: + return itoa(err); + } +} + +static int pad_spc(int llen, int cnt, const char *fmt, ...) +{ + va_list args; + int len; + int ret; + + for (len = 0; len < cnt - llen; len++) + putchar(' '); + + va_start(args, fmt); + ret = vfprintf(stdout, fmt, args); + va_end(args); + return ret < 0 ? ret : ret + len; +} + +/* The tests below are intended to be used by the macroes, which evaluate + * expression <expr>, print the status to stdout, and update the "ret" + * variable to count failures. The functions themselves return the number + * of failures, thus either 0 or 1. + */ + +#define EXPECT_ZR(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_zr(expr, llen); } while (0) + +static int expect_zr(int expr, int llen) +{ + int ret = !(expr == 0); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_NZ(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_nz(expr, llen; } while (0) + +static int expect_nz(int expr, int llen) +{ + int ret = !(expr != 0); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_EQ(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_eq(expr, llen, val); } while (0) + +static int expect_eq(int expr, int llen, int val) +{ + int ret = !(expr == val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_NE(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ne(expr, llen, val); } while (0) + +static int expect_ne(int expr, int llen, int val) +{ + int ret = !(expr != val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_GE(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ge(expr, llen, val); } while (0) + +static int expect_ge(int expr, int llen, int val) +{ + int ret = !(expr >= val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_GT(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_gt(expr, llen, val); } while (0) + +static int expect_gt(int expr, int llen, int val) +{ + int ret = !(expr > val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_LE(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_le(expr, llen, val); } while (0) + +static int expect_le(int expr, int llen, int val) +{ + int ret = !(expr <= val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_LT(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_lt(expr, llen, val); } while (0) + +static int expect_lt(int expr, int llen, int val) +{ + int ret = !(expr < val); + + llen += printf(" = %d ", expr); + pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n"); + return ret; +} + + +#define EXPECT_SYSZR(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syszr(expr, llen); } while (0) + +static int expect_syszr(int expr, int llen) +{ + int ret = 0; + + if (expr) { + ret = 1; + llen += printf(" = %d %s ", expr, errorname(errno)); + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += printf(" = %d ", expr); + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_SYSEQ(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syseq(expr, llen, val); } while (0) + +static int expect_syseq(int expr, int llen, int val) +{ + int ret = 0; + + if (expr != val) { + ret = 1; + llen += printf(" = %d %s ", expr, errorname(errno)); + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += printf(" = %d ", expr); + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_SYSNE(cond, expr, val) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_sysne(expr, llen, val); } while (0) + +static int expect_sysne(int expr, int llen, int val) +{ + int ret = 0; + + if (expr == val) { + ret = 1; + llen += printf(" = %d %s ", expr, errorname(errno)); + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += printf(" = %d ", expr); + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_SYSER(cond, expr, expret, experr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syserr(expr, expret, experr, llen); } while (0) + +static int expect_syserr(int expr, int expret, int experr, int llen) +{ + int ret = 0; + int _errno = errno; + + llen += printf(" = %d %s ", expr, errorname(_errno)); + if (expr != expret || _errno != experr) { + ret = 1; + llen += printf(" != (%d %s) ", expret, errorname(experr)); + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_PTRZR(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ptrzr(expr, llen); } while (0) + +static int expect_ptrzr(const void *expr, int llen) +{ + int ret = 0; + + llen += printf(" = <%p> ", expr); + if (expr) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_PTRNZ(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ptrnz(expr, llen); } while (0) + +static int expect_ptrnz(const void *expr, int llen) +{ + int ret = 0; + + llen += printf(" = <%p> ", expr); + if (!expr) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_STRZR(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strzr(expr, llen); } while (0) + +static int expect_strzr(const char *expr, int llen) +{ + int ret = 0; + + llen += printf(" = <%s> ", expr); + if (expr) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_STRNZ(cond, expr) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strnz(expr, llen); } while (0) + +static int expect_strnz(const char *expr, int llen) +{ + int ret = 0; + + llen += printf(" = <%s> ", expr); + if (!expr) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_STREQ(cond, expr, cmp) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_streq(expr, llen, cmp); } while (0) + +static int expect_streq(const char *expr, int llen, const char *cmp) +{ + int ret = 0; + + llen += printf(" = <%s> ", expr); + if (strcmp(expr, cmp) != 0) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +#define EXPECT_STRNE(cond, expr, cmp) \ + do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strne(expr, llen, cmp); } while (0) + +static int expect_strne(const char *expr, int llen, const char *cmp) +{ + int ret = 0; + + llen += printf(" = <%s> ", expr); + if (strcmp(expr, cmp) == 0) { + ret = 1; + llen += pad_spc(llen, 40, "[FAIL]\n"); + } else { + llen += pad_spc(llen, 40, " [OK]\n"); + } + return ret; +} + + +/* declare tests based on line numbers. There must be exactly one test per line. */ +#define CASE_TEST(name) \ + case __LINE__: llen += printf("%d %s", test, #name); + + +/* used by some syscall tests below */ +int test_getdents64(const char *dir) +{ + char buffer[4096]; + int fd, ret; + int err; + + ret = fd = open(dir, O_RDONLY | O_DIRECTORY, 0); + if (ret < 0) + return ret; + + ret = getdents64(fd, (void *)buffer, sizeof(buffer)); + err = errno; + close(fd); + + errno = err; + return ret; +} + +/* Run syscall tests between IDs <min> and <max>. + * Return 0 on success, non-zero on failure. + */ +int run_syscall(int min, int max) +{ + struct stat stat_buf; + int proc; + int test; + int tmp; + int ret = 0; + void *p1, *p2; + + /* <proc> indicates whether or not /proc is mounted */ + proc = stat("/proc", &stat_buf) == 0; + + for (test = min; test >= 0 && test <= max; test++) { + int llen = 0; // line length + + /* avoid leaving empty lines below, this will insert holes into + * test numbers. + */ + switch (test + __LINE__ + 1) { + CASE_TEST(getpid); EXPECT_SYSNE(1, getpid(), -1); break; + CASE_TEST(getppid); EXPECT_SYSNE(1, getppid(), -1); break; +#ifdef NOLIBC + CASE_TEST(gettid); EXPECT_SYSNE(1, gettid(), -1); break; +#endif + CASE_TEST(getpgid_self); EXPECT_SYSNE(1, getpgid(0), -1); break; + CASE_TEST(getpgid_bad); EXPECT_SYSER(1, getpgid(-1), -1, ESRCH); break; + CASE_TEST(kill_0); EXPECT_SYSZR(1, kill(getpid(), 0)); break; + CASE_TEST(kill_CONT); EXPECT_SYSZR(1, kill(getpid(), 0)); break; + CASE_TEST(kill_BADPID); EXPECT_SYSER(1, kill(INT_MAX, 0), -1, ESRCH); break; + CASE_TEST(sbrk); if ((p1 = p2 = sbrk(4096)) != (void *)-1) p2 = sbrk(-4096); EXPECT_SYSZR(1, (p2 == (void *)-1) || p2 == p1); break; + CASE_TEST(brk); EXPECT_SYSZR(1, brk(sbrk(0))); break; + CASE_TEST(chdir_root); EXPECT_SYSZR(1, chdir("/")); break; + CASE_TEST(chdir_dot); EXPECT_SYSZR(1, chdir(".")); break; + CASE_TEST(chdir_blah); EXPECT_SYSER(1, chdir("/blah"), -1, ENOENT); break; + CASE_TEST(chmod_net); EXPECT_SYSZR(proc, chmod("/proc/self/net", 0555)); break; + CASE_TEST(chmod_self); EXPECT_SYSER(proc, chmod("/proc/self", 0555), -1, EPERM); break; + CASE_TEST(chown_self); EXPECT_SYSER(proc, chown("/proc/self", 0, 0), -1, EPERM); break; + CASE_TEST(chroot_root); EXPECT_SYSZR(1, chroot("/")); break; + CASE_TEST(chroot_blah); EXPECT_SYSER(1, chroot("/proc/self/blah"), -1, ENOENT); break; + CASE_TEST(chroot_exe); EXPECT_SYSER(proc, chroot("/proc/self/exe"), -1, ENOTDIR); break; + CASE_TEST(close_m1); EXPECT_SYSER(1, close(-1), -1, EBADF); break; + CASE_TEST(close_dup); EXPECT_SYSZR(1, close(dup(0))); break; + CASE_TEST(dup_0); tmp = dup(0); EXPECT_SYSNE(1, tmp, -1); close(tmp); break; + CASE_TEST(dup_m1); tmp = dup(-1); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break; + CASE_TEST(dup2_0); tmp = dup2(0, 100); EXPECT_SYSNE(1, tmp, -1); close(tmp); break; + CASE_TEST(dup2_m1); tmp = dup2(-1, 100); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break; + CASE_TEST(dup3_0); tmp = dup3(0, 100, 0); EXPECT_SYSNE(1, tmp, -1); close(tmp); break; + CASE_TEST(dup3_m1); tmp = dup3(-1, 100, 0); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break; + CASE_TEST(execve_root); EXPECT_SYSER(1, execve("/", (char*[]){ [0] = "/", [1] = NULL }, NULL), -1, EACCES); break; + CASE_TEST(getdents64_root); EXPECT_SYSNE(1, test_getdents64("/"), -1); break; + CASE_TEST(getdents64_null); EXPECT_SYSER(1, test_getdents64("/dev/null"), -1, ENOTDIR); break; + CASE_TEST(gettimeofday_null); EXPECT_SYSZR(1, gettimeofday(NULL, NULL)); break; +#ifdef NOLIBC + CASE_TEST(gettimeofday_bad1); EXPECT_SYSER(1, gettimeofday((void *)1, NULL), -1, EFAULT); break; + CASE_TEST(gettimeofday_bad2); EXPECT_SYSER(1, gettimeofday(NULL, (void *)1), -1, EFAULT); break; + CASE_TEST(gettimeofday_bad2); EXPECT_SYSER(1, gettimeofday(NULL, (void *)1), -1, EFAULT); break; +#endif + CASE_TEST(ioctl_tiocinq); EXPECT_SYSZR(1, ioctl(0, TIOCINQ, &tmp)); break; + CASE_TEST(ioctl_tiocinq); EXPECT_SYSZR(1, ioctl(0, TIOCINQ, &tmp)); break; + CASE_TEST(link_root1); EXPECT_SYSER(1, link("/", "/"), -1, EEXIST); break; + CASE_TEST(link_blah); EXPECT_SYSER(1, link("/proc/self/blah", "/blah"), -1, ENOENT); break; + CASE_TEST(link_dir); EXPECT_SYSER(1, link("/", "/blah"), -1, EPERM); break; + CASE_TEST(link_cross); EXPECT_SYSER(proc, link("/proc/self/net", "/blah"), -1, EXDEV); break; + CASE_TEST(lseek_m1); EXPECT_SYSER(1, lseek(-1, 0, SEEK_SET), -1, EBADF); break; + CASE_TEST(lseek_0); EXPECT_SYSER(1, lseek(0, 0, SEEK_SET), -1, ESPIPE); break; + CASE_TEST(mkdir_root); EXPECT_SYSER(1, mkdir("/", 0755), -1, EEXIST); break; + CASE_TEST(open_tty); EXPECT_SYSNE(1, tmp = open("/dev/null", 0), -1); if (tmp != -1) close(tmp); break; + CASE_TEST(open_blah); EXPECT_SYSER(1, tmp = open("/proc/self/blah", 0), -1, ENOENT); if (tmp != -1) close(tmp); break; + CASE_TEST(poll_null); EXPECT_SYSZR(1, poll(NULL, 0, 0)); break; + CASE_TEST(poll_stdout); EXPECT_SYSNE(1, ({ struct pollfd fds = { 1, POLLOUT, 0}; poll(&fds, 1, 0); }), -1); break; + CASE_TEST(poll_fault); EXPECT_SYSER(1, poll((void *)1, 1, 0), -1, EFAULT); break; + CASE_TEST(read_badf); EXPECT_SYSER(1, read(-1, &tmp, 1), -1, EBADF); break; + CASE_TEST(sched_yield); EXPECT_SYSZR(1, sched_yield()); break; + CASE_TEST(select_null); EXPECT_SYSZR(1, ({ struct timeval tv = { 0 }; select(0, NULL, NULL, NULL, &tv); })); break; + CASE_TEST(select_stdout); EXPECT_SYSNE(1, ({ fd_set fds; FD_ZERO(&fds); FD_SET(1, &fds); select(2, NULL, &fds, NULL, NULL); }), -1); break; + CASE_TEST(select_fault); EXPECT_SYSER(1, select(1, (void *)1, NULL, NULL, 0), -1, EFAULT); break; + CASE_TEST(stat_blah); EXPECT_SYSER(1, stat("/proc/self/blah", &stat_buf), -1, ENOENT); break; + CASE_TEST(stat_fault); EXPECT_SYSER(1, stat(NULL, &stat_buf), -1, EFAULT); break; + CASE_TEST(symlink_root); EXPECT_SYSER(1, symlink("/", "/"), -1, EEXIST); break; + CASE_TEST(unlink_root); EXPECT_SYSER(1, unlink("/"), -1, EISDIR); break; + CASE_TEST(unlink_blah); EXPECT_SYSER(1, unlink("/proc/self/blah"), -1, ENOENT); break; + CASE_TEST(wait_child); EXPECT_SYSER(1, wait(&tmp), -1, ECHILD); break; + CASE_TEST(waitpid_min); EXPECT_SYSER(1, waitpid(INT_MIN, &tmp, WNOHANG), -1, ESRCH); break; + CASE_TEST(waitpid_child); EXPECT_SYSER(1, waitpid(getpid(), &tmp, WNOHANG), -1, ECHILD); break; + CASE_TEST(write_badf); EXPECT_SYSER(1, write(-1, &tmp, 1), -1, EBADF); break; + CASE_TEST(write_zero); EXPECT_SYSZR(1, write(1, &tmp, 0)); break; + case __LINE__: + return ret; /* must be last */ + /* note: do not set any defaults so as to permit holes above */ + } + } + return ret; +} + +int run_stdlib(int min, int max) +{ + int test; + int tmp; + int ret = 0; + void *p1, *p2; + + for (test = min; test >= 0 && test <= max; test++) { + int llen = 0; // line length + + /* avoid leaving empty lines below, this will insert holes into + * test numbers. + */ + switch (test + __LINE__ + 1) { + CASE_TEST(getenv_TERM); EXPECT_STRNZ(1, getenv("TERM")); break; + CASE_TEST(getenv_blah); EXPECT_STRZR(1, getenv("blah")); break; + CASE_TEST(setcmp_blah_blah); EXPECT_EQ(1, strcmp("blah", "blah"), 0); break; + CASE_TEST(setcmp_blah_blah2); EXPECT_NE(1, strcmp("blah", "blah2"), 0); break; + CASE_TEST(setncmp_blah_blah); EXPECT_EQ(1, strncmp("blah", "blah", 10), 0); break; + CASE_TEST(setncmp_blah_blah4); EXPECT_EQ(1, strncmp("blah", "blah4", 4), 0); break; + CASE_TEST(setncmp_blah_blah5); EXPECT_NE(1, strncmp("blah", "blah5", 5), 0); break; + CASE_TEST(setncmp_blah_blah6); EXPECT_NE(1, strncmp("blah", "blah6", 6), 0); break; + CASE_TEST(strchr_foobar_o); EXPECT_STREQ(1, strchr("foobar", 'o'), "oobar"); break; + CASE_TEST(strchr_foobar_z); EXPECT_STRZR(1, strchr("foobar", 'z')); break; + CASE_TEST(strrchr_foobar_o); EXPECT_STREQ(1, strrchr("foobar", 'o'), "obar"); break; + CASE_TEST(strrchr_foobar_z); EXPECT_STRZR(1, strrchr("foobar", 'z')); break; + case __LINE__: + return ret; /* must be last */ + /* note: do not set any defaults so as to permit holes above */ + } + } + return ret; +} + +/* prepare what needs to be prepared for pid 1 (stdio, /dev, /proc, etc) */ +int prepare(void) +{ + struct stat stat_buf; + + /* It's possible that /dev doesn't even exist or was not mounted, so + * we'll try to create it, mount it, or create minimal entries into it. + * We want at least /dev/null and /dev/console. + */ + if (stat("/dev/.", &stat_buf) == 0 || mkdir("/dev", 0755) == 0) { + if (stat("/dev/console", &stat_buf) != 0 || + stat("/dev/null", &stat_buf) != 0) { + /* try devtmpfs first, otherwise fall back to manual creation */ + if (mount("/dev", "/dev", "devtmpfs", 0, 0) != 0) { + mknod("/dev/console", 0600 | S_IFCHR, makedev(5, 1)); + mknod("/dev/null", 0666 | S_IFCHR, makedev(1, 3)); + } + } + } + + /* If no /dev/console was found before calling init, stdio is closed so + * we need to reopen it from /dev/console. If it failed above, it will + * still fail here and we cannot emit a message anyway. + */ + if (close(dup(1)) == -1) { + int fd = open("/dev/console", O_RDWR); + + if (fd >= 0) { + if (fd != 0) + dup2(fd, 0); + if (fd != 1) + dup2(fd, 1); + if (fd != 2) + dup2(fd, 2); + if (fd > 2) + close(fd); + puts("\nSuccessfully reopened /dev/console."); + } + } + + /* try to mount /proc if not mounted. Silently fail otherwise */ + if (stat("/proc/.", &stat_buf) == 0 || mkdir("/proc", 0755) == 0) { + if (stat("/proc/self", &stat_buf) != 0) + mount("/proc", "/proc", "proc", 0, 0); + } + + return 0; +} + +/* This is the definition of known test names, with their functions */ +static struct test test_names[] = { + /* add new tests here */ + { .name = "syscall", .func = run_syscall }, + { .name = "stdlib", .func = run_stdlib }, + { 0 } +}; + +int main(int argc, char **argv, char **envp) +{ + int min = 0; + int max = __INT_MAX__; + int ret = 0; + int err; + int idx; + char *test; + + environ = envp; + + /* when called as init, it's possible that no console was opened, for + * example if no /dev file system was provided. We'll check that fd#1 + * was opened, and if not we'll attempt to create and open /dev/console + * and /dev/null that we'll use for later tests. + */ + if (getpid() == 1) + prepare(); + + /* the definition of a series of tests comes from either argv[1] or the + * "NOLIBC_TEST" environment variable. It's made of a comma-delimited + * series of test names and optional ranges: + * syscall:5-15[:.*],stdlib:8-10 + */ + test = argv[1]; + if (!test) + test = getenv("NOLIBC_TEST"); + + if (test) { + char *comma, *colon, *dash, *value; + + do { + comma = strchr(test, ','); + if (comma) + *(comma++) = '\0'; + + colon = strchr(test, ':'); + if (colon) + *(colon++) = '\0'; + + for (idx = 0; test_names[idx].name; idx++) { + if (strcmp(test, test_names[idx].name) == 0) + break; + } + + if (test_names[idx].name) { + /* The test was named, it will be called at least + * once. We may have an optional range at <colon> + * here, which defaults to the full range. + */ + do { + min = 0; max = __INT_MAX__; + value = colon; + if (value && *value) { + colon = strchr(value, ':'); + if (colon) + *(colon++) = '\0'; + + dash = strchr(value, '-'); + if (dash) + *(dash++) = '\0'; + + /* support :val: :min-max: :min-: :-max: */ + if (*value) + min = atoi(value); + if (!dash) + max = min; + else if (*dash) + max = atoi(dash); + + value = colon; + } + + /* now's time to call the test */ + printf("Running test '%s'\n", test_names[idx].name); + err = test_names[idx].func(min, max); + ret += err; + printf("Errors during this test: %d\n\n", err); + } while (colon && *colon); + } else + printf("Ignoring unknown test name '%s'\n", test); + + test = comma; + } while (test && *test); + } else { + /* no test mentioned, run everything */ + for (idx = 0; test_names[idx].name; idx++) { + printf("Running test '%s'\n", test_names[idx].name); + err = test_names[idx].func(min, max); + ret += err; + printf("Errors during this test: %d\n\n", err); + } + } + + printf("Total number of errors: %d\n", ret); + + if (getpid() == 1) { + /* we're running as init, there's no other process on the + * system, thus likely started from a VM for a quick check. + * Exiting will provoke a kernel panic that may be reported + * as an error by Qemu or the hypervisor, while stopping + * cleanly will often be reported as a success. This allows + * to use the output of this program for bisecting kernels. + */ + printf("Leaving init with final status: %d\n", !!ret); + if (ret == 0) + reboot(LINUX_REBOOT_CMD_POWER_OFF); +#if defined(__x86_64__) + /* QEMU started with "-device isa-debug-exit -no-reboot" will + * exit with status code 2N+1 when N is written to 0x501. We + * hard-code the syscall here as it's arch-dependent. + */ +#if defined(_NOLIBC_SYS_H) + else if (my_syscall3(__NR_ioperm, 0x501, 1, 1) == 0) +#else + else if (ioperm(0x501, 1, 1) == 0) +#endif + asm volatile ("outb %%al, %%dx" :: "d"(0x501), "a"(0)); + /* if it does nothing, fall back to the regular panic */ +#endif + } + + printf("Exiting with status %d\n", !!ret); + return !!ret; +} diff --git a/tools/testing/selftests/powerpc/benchmarks/gettimeofday.c b/tools/testing/selftests/powerpc/benchmarks/gettimeofday.c index 6b415683357b5ed148c7bc33e651260be502f293..580fcac0a09f31bf284d46c97f25148c96909386 100644 --- a/tools/testing/selftests/powerpc/benchmarks/gettimeofday.c +++ b/tools/testing/selftests/powerpc/benchmarks/gettimeofday.c @@ -12,7 +12,7 @@ static int test_gettimeofday(void) { int i; - struct timeval tv_start, tv_end; + struct timeval tv_start, tv_end, tv_diff; gettimeofday(&tv_start, NULL); @@ -20,7 +20,9 @@ static int test_gettimeofday(void) gettimeofday(&tv_end, NULL); } - printf("time = %.6f\n", tv_end.tv_sec - tv_start.tv_sec + (tv_end.tv_usec - tv_start.tv_usec) * 1e-6); + timersub(&tv_start, &tv_end, &tv_diff); + + printf("time = %.6f\n", tv_diff.tv_sec + (tv_diff.tv_usec) * 1e-6); return 0; } diff --git a/tools/testing/selftests/powerpc/mm/Makefile b/tools/testing/selftests/powerpc/mm/Makefile index 27dc09d0bfeed287cd6e1aa0f6b73a5a402f6912..19dd0b2ea397405f7d40a1f761c0992a64cdd83b 100644 --- a/tools/testing/selftests/powerpc/mm/Makefile +++ b/tools/testing/selftests/powerpc/mm/Makefile @@ -3,7 +3,7 @@ noarg: $(MAKE) -C ../ TEST_GEN_PROGS := hugetlb_vs_thp_test subpage_prot prot_sao segv_errors wild_bctr \ - large_vm_fork_separation bad_accesses pkey_exec_prot \ + large_vm_fork_separation bad_accesses exec_prot pkey_exec_prot \ pkey_siginfo stack_expansion_signal stack_expansion_ldst \ large_vm_gpr_corruption TEST_PROGS := stress_code_patching.sh @@ -22,6 +22,7 @@ $(OUTPUT)/wild_bctr: CFLAGS += -m64 $(OUTPUT)/large_vm_fork_separation: CFLAGS += -m64 $(OUTPUT)/large_vm_gpr_corruption: CFLAGS += -m64 $(OUTPUT)/bad_accesses: CFLAGS += -m64 +$(OUTPUT)/exec_prot: CFLAGS += -m64 $(OUTPUT)/pkey_exec_prot: CFLAGS += -m64 $(OUTPUT)/pkey_siginfo: CFLAGS += -m64 diff --git a/tools/testing/selftests/powerpc/mm/exec_prot.c b/tools/testing/selftests/powerpc/mm/exec_prot.c new file mode 100644 index 0000000000000000000000000000000000000000..db75b2225de19cbae77f7376fc5070eecd853e48 --- /dev/null +++ b/tools/testing/selftests/powerpc/mm/exec_prot.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright 2022, Nicholas Miehlbradt, IBM Corporation + * based on pkey_exec_prot.c + * + * Test if applying execute protection on pages works as expected. + */ + +#define _GNU_SOURCE +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> + +#include <unistd.h> +#include <sys/mman.h> + +#include "pkeys.h" + + +#define PPC_INST_NOP 0x60000000 +#define PPC_INST_TRAP 0x7fe00008 +#define PPC_INST_BLR 0x4e800020 + +static volatile sig_atomic_t fault_code; +static volatile sig_atomic_t remaining_faults; +static volatile unsigned int *fault_addr; +static unsigned long pgsize, numinsns; +static unsigned int *insns; +static bool pkeys_supported; + +static bool is_fault_expected(int fault_code) +{ + if (fault_code == SEGV_ACCERR) + return true; + + /* Assume any pkey error is fine since pkey_exec_prot test covers them */ + if (fault_code == SEGV_PKUERR && pkeys_supported) + return true; + + return false; +} + +static void trap_handler(int signum, siginfo_t *sinfo, void *ctx) +{ + /* Check if this fault originated from the expected address */ + if (sinfo->si_addr != (void *)fault_addr) + sigsafe_err("got a fault for an unexpected address\n"); + + _exit(1); +} + +static void segv_handler(int signum, siginfo_t *sinfo, void *ctx) +{ + fault_code = sinfo->si_code; + + /* Check if this fault originated from the expected address */ + if (sinfo->si_addr != (void *)fault_addr) { + sigsafe_err("got a fault for an unexpected address\n"); + _exit(1); + } + + /* Check if too many faults have occurred for a single test case */ + if (!remaining_faults) { + sigsafe_err("got too many faults for the same address\n"); + _exit(1); + } + + + /* Restore permissions in order to continue */ + if (is_fault_expected(fault_code)) { + if (mprotect(insns, pgsize, PROT_READ | PROT_WRITE | PROT_EXEC)) { + sigsafe_err("failed to set access permissions\n"); + _exit(1); + } + } else { + sigsafe_err("got a fault with an unexpected code\n"); + _exit(1); + } + + remaining_faults--; +} + +static int check_exec_fault(int rights) +{ + /* + * Jump to the executable region. + * + * The first iteration also checks if the overwrite of the + * first instruction word from a trap to a no-op succeeded. + */ + fault_code = -1; + remaining_faults = 0; + if (!(rights & PROT_EXEC)) + remaining_faults = 1; + + FAIL_IF(mprotect(insns, pgsize, rights) != 0); + asm volatile("mtctr %0; bctrl" : : "r"(insns)); + + FAIL_IF(remaining_faults != 0); + if (!(rights & PROT_EXEC)) + FAIL_IF(!is_fault_expected(fault_code)); + + return 0; +} + +static int test(void) +{ + struct sigaction segv_act, trap_act; + int i; + + /* Skip the test if the CPU doesn't support Radix */ + SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00)); + + /* Check if pkeys are supported */ + pkeys_supported = pkeys_unsupported() == 0; + + /* Setup SIGSEGV handler */ + segv_act.sa_handler = 0; + segv_act.sa_sigaction = segv_handler; + FAIL_IF(sigprocmask(SIG_SETMASK, 0, &segv_act.sa_mask) != 0); + segv_act.sa_flags = SA_SIGINFO; + segv_act.sa_restorer = 0; + FAIL_IF(sigaction(SIGSEGV, &segv_act, NULL) != 0); + + /* Setup SIGTRAP handler */ + trap_act.sa_handler = 0; + trap_act.sa_sigaction = trap_handler; + FAIL_IF(sigprocmask(SIG_SETMASK, 0, &trap_act.sa_mask) != 0); + trap_act.sa_flags = SA_SIGINFO; + trap_act.sa_restorer = 0; + FAIL_IF(sigaction(SIGTRAP, &trap_act, NULL) != 0); + + /* Setup executable region */ + pgsize = getpagesize(); + numinsns = pgsize / sizeof(unsigned int); + insns = (unsigned int *)mmap(NULL, pgsize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + FAIL_IF(insns == MAP_FAILED); + + /* Write the instruction words */ + for (i = 1; i < numinsns - 1; i++) + insns[i] = PPC_INST_NOP; + + /* + * Set the first instruction as an unconditional trap. If + * the last write to this address succeeds, this should + * get overwritten by a no-op. + */ + insns[0] = PPC_INST_TRAP; + + /* + * Later, to jump to the executable region, we use a branch + * and link instruction (bctrl) which sets the return address + * automatically in LR. Use that to return back. + */ + insns[numinsns - 1] = PPC_INST_BLR; + + /* + * Pick the first instruction's address from the executable + * region. + */ + fault_addr = insns; + + /* + * Read an instruction word from the address when the page + * is execute only. This should generate an access fault. + */ + fault_code = -1; + remaining_faults = 1; + printf("Testing read on --x, should fault..."); + FAIL_IF(mprotect(insns, pgsize, PROT_EXEC) != 0); + i = *fault_addr; + FAIL_IF(remaining_faults != 0 || !is_fault_expected(fault_code)); + printf("ok!\n"); + + /* + * Write an instruction word to the address when the page + * execute only. This should also generate an access fault. + */ + fault_code = -1; + remaining_faults = 1; + printf("Testing write on --x, should fault..."); + FAIL_IF(mprotect(insns, pgsize, PROT_EXEC) != 0); + *fault_addr = PPC_INST_NOP; + FAIL_IF(remaining_faults != 0 || !is_fault_expected(fault_code)); + printf("ok!\n"); + + printf("Testing exec on ---, should fault..."); + FAIL_IF(check_exec_fault(PROT_NONE)); + printf("ok!\n"); + + printf("Testing exec on r--, should fault..."); + FAIL_IF(check_exec_fault(PROT_READ)); + printf("ok!\n"); + + printf("Testing exec on -w-, should fault..."); + FAIL_IF(check_exec_fault(PROT_WRITE)); + printf("ok!\n"); + + printf("Testing exec on rw-, should fault..."); + FAIL_IF(check_exec_fault(PROT_READ | PROT_WRITE)); + printf("ok!\n"); + + printf("Testing exec on --x, should succeed..."); + FAIL_IF(check_exec_fault(PROT_EXEC)); + printf("ok!\n"); + + printf("Testing exec on r-x, should succeed..."); + FAIL_IF(check_exec_fault(PROT_READ | PROT_EXEC)); + printf("ok!\n"); + + printf("Testing exec on -wx, should succeed..."); + FAIL_IF(check_exec_fault(PROT_WRITE | PROT_EXEC)); + printf("ok!\n"); + + printf("Testing exec on rwx, should succeed..."); + FAIL_IF(check_exec_fault(PROT_READ | PROT_WRITE | PROT_EXEC)); + printf("ok!\n"); + + /* Cleanup */ + FAIL_IF(munmap((void *)insns, pgsize)); + + return 0; +} + +int main(void) +{ + return test_harness(test, "exec_prot"); +} diff --git a/tools/testing/selftests/powerpc/mm/large_vm_gpr_corruption.c b/tools/testing/selftests/powerpc/mm/large_vm_gpr_corruption.c index 927bfae99ed90b8b8c8936ab65eb20348ce6202c..7da515f1da723c2d9f82357edf0892ceb66d7cb1 100644 --- a/tools/testing/selftests/powerpc/mm/large_vm_gpr_corruption.c +++ b/tools/testing/selftests/powerpc/mm/large_vm_gpr_corruption.c @@ -112,6 +112,8 @@ static int test(void) // This tests a hash MMU specific bug. FAIL_IF(using_hash_mmu(&hash_mmu)); SKIP_IF(!hash_mmu); + // 4K kernels don't support 4PB address space + SKIP_IF(sysconf(_SC_PAGESIZE) < 65536); page_size = sysconf(_SC_PAGESIZE); diff --git a/tools/testing/selftests/powerpc/pmu/sampling_tests/bhrb_filter_map_test.c b/tools/testing/selftests/powerpc/pmu/sampling_tests/bhrb_filter_map_test.c index 8182647c63c8705df05b9aa9d4a54855d33b316c..3f43c315c66641f12d034e41745ca10e2fa241af 100644 --- a/tools/testing/selftests/powerpc/pmu/sampling_tests/bhrb_filter_map_test.c +++ b/tools/testing/selftests/powerpc/pmu/sampling_tests/bhrb_filter_map_test.c @@ -96,6 +96,15 @@ static int bhrb_filter_map_test(void) } } + /* + * Combine filter maps which includes a valid branch filter and an invalid branch + * filter. Example: any ( PERF_SAMPLE_BRANCH_ANY) and any_call + * (PERF_SAMPLE_BRANCH_ANY_CALL). + * The perf_event_open should fail in this case. + */ + event.attr.branch_sample_type = PERF_SAMPLE_BRANCH_ANY | PERF_SAMPLE_BRANCH_ANY_CALL; + FAIL_IF(!event_open(&event)); + return 0; } diff --git a/tools/testing/selftests/proc/.gitignore b/tools/testing/selftests/proc/.gitignore index c4e6a34f9657bac2e5939409719443906f549fb5..a156ac5dd2c6a65622b7d5471e1bebd216a86dec 100644 --- a/tools/testing/selftests/proc/.gitignore +++ b/tools/testing/selftests/proc/.gitignore @@ -5,6 +5,7 @@ /proc-fsconfig-hidepid /proc-loadavg-001 /proc-multiple-procfs +/proc-empty-vm /proc-pid-vm /proc-self-map-files-001 /proc-self-map-files-002 diff --git a/tools/testing/selftests/proc/Makefile b/tools/testing/selftests/proc/Makefile index 219fc61138473b09841561f1ef0d81354035b4a6..cd95369254c088234b6279392f8f619193049c0e 100644 --- a/tools/testing/selftests/proc/Makefile +++ b/tools/testing/selftests/proc/Makefile @@ -8,6 +8,7 @@ TEST_GEN_PROGS += fd-001-lookup TEST_GEN_PROGS += fd-002-posix-eq TEST_GEN_PROGS += fd-003-kthread TEST_GEN_PROGS += proc-loadavg-001 +TEST_GEN_PROGS += proc-empty-vm TEST_GEN_PROGS += proc-pid-vm TEST_GEN_PROGS += proc-self-map-files-001 TEST_GEN_PROGS += proc-self-map-files-002 diff --git a/tools/testing/selftests/proc/proc-empty-vm.c b/tools/testing/selftests/proc/proc-empty-vm.c new file mode 100644 index 0000000000000000000000000000000000000000..d95b1cb43d9d08cb6c41eaef267b992082705910 --- /dev/null +++ b/tools/testing/selftests/proc/proc-empty-vm.c @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2022 Alexey Dobriyan <adobriyan@gmail.com> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +/* + * Create a process without mappings by unmapping everything at once and + * holding it with ptrace(2). See what happens to + * + * /proc/${pid}/maps + * /proc/${pid}/numa_maps + * /proc/${pid}/smaps + * /proc/${pid}/smaps_rollup + */ +#undef NDEBUG +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/ptrace.h> +#include <sys/resource.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +/* + * 0: vsyscall VMA doesn't exist vsyscall=none + * 1: vsyscall VMA is --xp vsyscall=xonly + * 2: vsyscall VMA is r-xp vsyscall=emulate + */ +static int g_vsyscall; +static const char *g_proc_pid_maps_vsyscall; +static const char *g_proc_pid_smaps_vsyscall; + +static const char proc_pid_maps_vsyscall_0[] = ""; +static const char proc_pid_maps_vsyscall_1[] = +"ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n"; +static const char proc_pid_maps_vsyscall_2[] = +"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; + +static const char proc_pid_smaps_vsyscall_0[] = ""; + +static const char proc_pid_smaps_vsyscall_1[] = +"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n" +"Size: 4 kB\n" +"KernelPageSize: 4 kB\n" +"MMUPageSize: 4 kB\n" +"Rss: 0 kB\n" +"Pss: 0 kB\n" +"Pss_Dirty: 0 kB\n" +"Shared_Clean: 0 kB\n" +"Shared_Dirty: 0 kB\n" +"Private_Clean: 0 kB\n" +"Private_Dirty: 0 kB\n" +"Referenced: 0 kB\n" +"Anonymous: 0 kB\n" +"LazyFree: 0 kB\n" +"AnonHugePages: 0 kB\n" +"ShmemPmdMapped: 0 kB\n" +"FilePmdMapped: 0 kB\n" +"Shared_Hugetlb: 0 kB\n" +"Private_Hugetlb: 0 kB\n" +"Swap: 0 kB\n" +"SwapPss: 0 kB\n" +"Locked: 0 kB\n" +"THPeligible: 0\n" +/* + * "ProtectionKey:" field is conditional. It is possible to check it as well, + * but I don't have such machine. + */ +; + +static const char proc_pid_smaps_vsyscall_2[] = +"ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n" +"Size: 4 kB\n" +"KernelPageSize: 4 kB\n" +"MMUPageSize: 4 kB\n" +"Rss: 0 kB\n" +"Pss: 0 kB\n" +"Pss_Dirty: 0 kB\n" +"Shared_Clean: 0 kB\n" +"Shared_Dirty: 0 kB\n" +"Private_Clean: 0 kB\n" +"Private_Dirty: 0 kB\n" +"Referenced: 0 kB\n" +"Anonymous: 0 kB\n" +"LazyFree: 0 kB\n" +"AnonHugePages: 0 kB\n" +"ShmemPmdMapped: 0 kB\n" +"FilePmdMapped: 0 kB\n" +"Shared_Hugetlb: 0 kB\n" +"Private_Hugetlb: 0 kB\n" +"Swap: 0 kB\n" +"SwapPss: 0 kB\n" +"Locked: 0 kB\n" +"THPeligible: 0\n" +/* + * "ProtectionKey:" field is conditional. It is possible to check it as well, + * but I'm too tired. + */ +; + +static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___) +{ + _exit(EXIT_FAILURE); +} + +static void sigaction_SIGSEGV_vsyscall(int _, siginfo_t *__, void *___) +{ + _exit(g_vsyscall); +} + +/* + * vsyscall page can't be unmapped, probe it directly. + */ +static void vsyscall(void) +{ + pid_t pid; + int wstatus; + + pid = fork(); + if (pid < 0) { + fprintf(stderr, "fork, errno %d\n", errno); + exit(1); + } + if (pid == 0) { + setrlimit(RLIMIT_CORE, &(struct rlimit){}); + + /* Hide "segfault at ffffffffff600000" messages. */ + struct sigaction act = {}; + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = sigaction_SIGSEGV_vsyscall; + sigaction(SIGSEGV, &act, NULL); + + g_vsyscall = 0; + /* gettimeofday(NULL, NULL); */ + asm volatile ( + "call %P0" + : + : "i" (0xffffffffff600000), "D" (NULL), "S" (NULL) + : "rax", "rcx", "r11" + ); + + g_vsyscall = 1; + *(volatile int *)0xffffffffff600000UL; + + g_vsyscall = 2; + exit(g_vsyscall); + } + waitpid(pid, &wstatus, 0); + if (WIFEXITED(wstatus)) { + g_vsyscall = WEXITSTATUS(wstatus); + } else { + fprintf(stderr, "error: vsyscall wstatus %08x\n", wstatus); + exit(1); + } +} + +static int test_proc_pid_maps(pid_t pid) +{ + char buf[4096]; + snprintf(buf, sizeof(buf), "/proc/%u/maps", pid); + int fd = open(buf, O_RDONLY); + if (fd == -1) { + perror("open /proc/${pid}/maps"); + return EXIT_FAILURE; + } else { + ssize_t rv = read(fd, buf, sizeof(buf)); + close(fd); + if (g_vsyscall == 0) { + assert(rv == 0); + } else { + size_t len = strlen(g_proc_pid_maps_vsyscall); + assert(rv == len); + assert(memcmp(buf, g_proc_pid_maps_vsyscall, len) == 0); + } + return EXIT_SUCCESS; + } +} + +static int test_proc_pid_numa_maps(pid_t pid) +{ + char buf[4096]; + snprintf(buf, sizeof(buf), "/proc/%u/numa_maps", pid); + int fd = open(buf, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) { + /* + * /proc/${pid}/numa_maps is under CONFIG_NUMA, + * it doesn't necessarily exist. + */ + return EXIT_SUCCESS; + } + perror("open /proc/${pid}/numa_maps"); + return EXIT_FAILURE; + } else { + ssize_t rv = read(fd, buf, sizeof(buf)); + close(fd); + assert(rv == 0); + return EXIT_SUCCESS; + } +} + +static int test_proc_pid_smaps(pid_t pid) +{ + char buf[4096]; + snprintf(buf, sizeof(buf), "/proc/%u/smaps", pid); + int fd = open(buf, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) { + /* + * /proc/${pid}/smaps is under CONFIG_PROC_PAGE_MONITOR, + * it doesn't necessarily exist. + */ + return EXIT_SUCCESS; + } + perror("open /proc/${pid}/smaps"); + return EXIT_FAILURE; + } else { + ssize_t rv = read(fd, buf, sizeof(buf)); + close(fd); + if (g_vsyscall == 0) { + assert(rv == 0); + } else { + size_t len = strlen(g_proc_pid_maps_vsyscall); + /* TODO "ProtectionKey:" */ + assert(rv > len); + assert(memcmp(buf, g_proc_pid_maps_vsyscall, len) == 0); + } + return EXIT_SUCCESS; + } +} + +static const char g_smaps_rollup[] = +"00000000-00000000 ---p 00000000 00:00 0 [rollup]\n" +"Rss: 0 kB\n" +"Pss: 0 kB\n" +"Pss_Dirty: 0 kB\n" +"Pss_Anon: 0 kB\n" +"Pss_File: 0 kB\n" +"Pss_Shmem: 0 kB\n" +"Shared_Clean: 0 kB\n" +"Shared_Dirty: 0 kB\n" +"Private_Clean: 0 kB\n" +"Private_Dirty: 0 kB\n" +"Referenced: 0 kB\n" +"Anonymous: 0 kB\n" +"LazyFree: 0 kB\n" +"AnonHugePages: 0 kB\n" +"ShmemPmdMapped: 0 kB\n" +"FilePmdMapped: 0 kB\n" +"Shared_Hugetlb: 0 kB\n" +"Private_Hugetlb: 0 kB\n" +"Swap: 0 kB\n" +"SwapPss: 0 kB\n" +"Locked: 0 kB\n" +; + +static int test_proc_pid_smaps_rollup(pid_t pid) +{ + char buf[4096]; + snprintf(buf, sizeof(buf), "/proc/%u/smaps_rollup", pid); + int fd = open(buf, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) { + /* + * /proc/${pid}/smaps_rollup is under CONFIG_PROC_PAGE_MONITOR, + * it doesn't necessarily exist. + */ + return EXIT_SUCCESS; + } + perror("open /proc/${pid}/smaps_rollup"); + return EXIT_FAILURE; + } else { + ssize_t rv = read(fd, buf, sizeof(buf)); + close(fd); + assert(rv == sizeof(g_smaps_rollup) - 1); + assert(memcmp(buf, g_smaps_rollup, sizeof(g_smaps_rollup) - 1) == 0); + return EXIT_SUCCESS; + } +} + +int main(void) +{ + int rv = EXIT_SUCCESS; + + vsyscall(); + + switch (g_vsyscall) { + case 0: + g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_0; + g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_0; + break; + case 1: + g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_1; + g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_1; + break; + case 2: + g_proc_pid_maps_vsyscall = proc_pid_maps_vsyscall_2; + g_proc_pid_smaps_vsyscall = proc_pid_smaps_vsyscall_2; + break; + default: + abort(); + } + + pid_t pid = fork(); + if (pid == -1) { + perror("fork"); + return EXIT_FAILURE; + } else if (pid == 0) { + rv = ptrace(PTRACE_TRACEME, 0, NULL, NULL); + if (rv != 0) { + if (errno == EPERM) { + fprintf(stderr, +"Did you know? ptrace(PTRACE_TRACEME) doesn't work under strace.\n" + ); + kill(getppid(), SIGTERM); + return EXIT_FAILURE; + } + perror("ptrace PTRACE_TRACEME"); + return EXIT_FAILURE; + } + + /* + * Hide "segfault at ..." messages. Signal handler won't run. + */ + struct sigaction act = {}; + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = sigaction_SIGSEGV; + sigaction(SIGSEGV, &act, NULL); + +#ifdef __amd64__ + munmap(NULL, ((size_t)1 << 47) - 4096); +#else +#error "implement 'unmap everything'" +#endif + return EXIT_FAILURE; + } else { + /* + * TODO find reliable way to signal parent that munmap(2) completed. + * Child can't do it directly because it effectively doesn't exist + * anymore. Looking at child's VM files isn't 100% reliable either: + * due to a bug they may not become empty or empty-like. + */ + sleep(1); + + if (rv == EXIT_SUCCESS) { + rv = test_proc_pid_maps(pid); + } + if (rv == EXIT_SUCCESS) { + rv = test_proc_pid_numa_maps(pid); + } + if (rv == EXIT_SUCCESS) { + rv = test_proc_pid_smaps(pid); + } + if (rv == EXIT_SUCCESS) { + rv = test_proc_pid_smaps_rollup(pid); + } + /* + * TODO test /proc/${pid}/statm, task_statm() + * ->start_code, ->end_code aren't updated by munmap(). + * Output can be "0 0 0 2 0 0 0\n" where "2" can be anything. + */ + + /* Cut the rope. */ + int wstatus; + waitpid(pid, &wstatus, 0); + assert(WIFSTOPPED(wstatus)); + assert(WSTOPSIG(wstatus) == SIGSEGV); + } + + return rv; +} diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c index e5962f4794f5668e602a2e23df6d0efc9e4dc750..69551bfa215c4e8321a1971f405e602a67940148 100644 --- a/tools/testing/selftests/proc/proc-pid-vm.c +++ b/tools/testing/selftests/proc/proc-pid-vm.c @@ -213,22 +213,22 @@ static int make_exe(const uint8_t *payload, size_t len) /* * 0: vsyscall VMA doesn't exist vsyscall=none - * 1: vsyscall VMA is r-xp vsyscall=emulate - * 2: vsyscall VMA is --xp vsyscall=xonly + * 1: vsyscall VMA is --xp vsyscall=xonly + * 2: vsyscall VMA is r-xp vsyscall=emulate */ -static int g_vsyscall; +static volatile int g_vsyscall; static const char *str_vsyscall; static const char str_vsyscall_0[] = ""; static const char str_vsyscall_1[] = -"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; -static const char str_vsyscall_2[] = "ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall]\n"; +static const char str_vsyscall_2[] = +"ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]\n"; #ifdef __x86_64__ static void sigaction_SIGSEGV(int _, siginfo_t *__, void *___) { - _exit(1); + _exit(g_vsyscall); } /* @@ -255,6 +255,7 @@ static void vsyscall(void) act.sa_sigaction = sigaction_SIGSEGV; (void)sigaction(SIGSEGV, &act, NULL); + g_vsyscall = 0; /* gettimeofday(NULL, NULL); */ asm volatile ( "call %P0" @@ -262,45 +263,20 @@ static void vsyscall(void) : "i" (0xffffffffff600000), "D" (NULL), "S" (NULL) : "rax", "rcx", "r11" ); - exit(0); - } - waitpid(pid, &wstatus, 0); - if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0) { - /* vsyscall page exists and is executable. */ - } else { - /* vsyscall page doesn't exist. */ - g_vsyscall = 0; - return; - } - - pid = fork(); - if (pid < 0) { - fprintf(stderr, "fork, errno %d\n", errno); - exit(1); - } - if (pid == 0) { - struct rlimit rlim = {0, 0}; - (void)setrlimit(RLIMIT_CORE, &rlim); - - /* Hide "segfault at ffffffffff600000" messages. */ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = sigaction_SIGSEGV; - (void)sigaction(SIGSEGV, &act, NULL); + g_vsyscall = 1; *(volatile int *)0xffffffffff600000UL; - exit(0); + + g_vsyscall = 2; + exit(g_vsyscall); } waitpid(pid, &wstatus, 0); - if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0) { - /* vsyscall page is readable and executable. */ - g_vsyscall = 1; - return; + if (WIFEXITED(wstatus)) { + g_vsyscall = WEXITSTATUS(wstatus); + } else { + fprintf(stderr, "error: wstatus %08x\n", wstatus); + exit(1); } - - /* vsyscall page is executable but unreadable. */ - g_vsyscall = 2; } int main(void) diff --git a/tools/testing/selftests/sgx/sigstruct.c b/tools/testing/selftests/sgx/sigstruct.c index 50c5ab1aa6fa1a3918546e84df23682c9f846153..a07896a463643d3bd8e6e29c1dfc7f6b44e5f49e 100644 --- a/tools/testing/selftests/sgx/sigstruct.c +++ b/tools/testing/selftests/sgx/sigstruct.c @@ -17,6 +17,12 @@ #include "defines.h" #include "main.h" +/* + * FIXME: OpenSSL 3.0 has deprecated some functions. For now just ignore + * the warnings. + */ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + struct q1q2_ctx { BN_CTX *bn_ctx; BIGNUM *m; diff --git a/tools/testing/selftests/tc-testing/config b/tools/testing/selftests/tc-testing/config index a3239d5e40c79e9683b0626f7666d25e0f169260..4638c63a339ff51dd742335b89ff7f66dce70b00 100644 --- a/tools/testing/selftests/tc-testing/config +++ b/tools/testing/selftests/tc-testing/config @@ -12,8 +12,35 @@ CONFIG_NET_SCHED=y # # Queueing/Scheduling # -CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_ATM=m +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_CBS=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_ETF=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_HTB=m CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_SKBPRIO=m +CONFIG_NET_SCH_TAPRIO=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_TEQL=m # # Classification @@ -23,6 +50,15 @@ CONFIG_NET_CLS_FW=m CONFIG_NET_CLS_U32=m CONFIG_CLS_U32_PERF=y CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_TCINDEX=m CONFIG_NET_EMATCH=y CONFIG_NET_EMATCH_STACK=32 CONFIG_NET_EMATCH_CMP=m @@ -54,6 +90,7 @@ CONFIG_NET_ACT_IFE=m CONFIG_NET_ACT_TUNNEL_KEY=m CONFIG_NET_ACT_CT=m CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_GATE=m CONFIG_NET_IFE_SKBMARK=m CONFIG_NET_IFE_SKBPRIO=m CONFIG_NET_IFE_SKBTCINDEX=m @@ -67,3 +104,4 @@ CONFIG_NETDEVSIM=m ## Network testing # CONFIG_CAN=m +CONFIG_ATM=y diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json b/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json index cadde8f41fcd3db8cdaa21afd117cda720802b05..0de2f79ea329301957e69b68978926b5b215c0d6 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json @@ -312,5 +312,55 @@ "teardown": [ "$TC actions flush action connmark" ] + }, + { + "id": "6571", + "name": "Delete connmark action with valid index", + "category": [ + "actions", + "connmark" + ], + "setup": [ + [ + "$TC actions flush action connmark", + 0, + 1, + 255 + ], + "$TC actions add action connmark pass index 20" + ], + "cmdUnderTest": "$TC actions del action connmark index 20", + "expExitCode": "0", + "verifyCmd": "$TC actions get action connmark index 20", + "matchPattern": "action order [0-9]+: connmark zone 0 pass.*index 20 ref", + "matchCount": "0", + "teardown": [ + "$TC actions flush action connmark" + ] + }, + { + "id": "3426", + "name": "Delete connmark action with invalid index", + "category": [ + "actions", + "connmark" + ], + "setup": [ + [ + "$TC actions flush action connmark", + 0, + 1, + 255 + ], + "$TC actions add action connmark pass index 20" + ], + "cmdUnderTest": "$TC actions del action connmark index 1", + "expExitCode": "255", + "verifyCmd": "$TC actions get action connmark index 20", + "matchPattern": "action order [0-9]+: connmark zone 0 pass.*index 20 ref", + "matchCount": "1", + "teardown": [ + "$TC actions flush action connmark" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json new file mode 100644 index 0000000000000000000000000000000000000000..d9710c067eb73cadc79df72cff4ae37b78b2515b --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json @@ -0,0 +1,316 @@ +[ + { + "id": "c826", + "name": "Add ctinfo action with default setting", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC action flush action ctinfo", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action ctinfo index 10", + "expExitCode": "0", + "verifyCmd": "$TC action get action ctinfo index 10", + "matchPattern": "action order [0-9]*: ctinfo zone 0 pipe.*index 10 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "0286", + "name": "Add ctinfo action with dscp", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action ctinfo dscp 0xfc000000 0x01000000 index 100", + "expExitCode": "0", + "verifyCmd": "$TC action ls action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo zone 0 pipe.*index 100 ref.*dscp 0xfc000000 0x01000000", + "matchCount": "1", + "teardown": [ + "$TC actions flush action ctinfo" + ] + }, + { + "id": "4938", + "name": "Add ctinfo action with valid cpmark and zone", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC action flush action ctinfo", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action ctinfo cpmark 0x01000000 zone 1 index 1", + "expExitCode": "0", + "verifyCmd": "$TC action get action ctinfo index 1", + "matchPattern": "action order [0-9]*: ctinfo zone 1 pipe.*index 1 ref.*cpmark 0x01000000", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "7593", + "name": "Add ctinfo action with drop control", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC action flush action ctinfo", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action ctinfo drop index 1000", + "expExitCode": "0", + "verifyCmd": "$TC action ls action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo zone 0 drop.*index 1000 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "2961", + "name": "Replace ctinfo action zone and action control", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + [ + "$TC action add action ctinfo zone 1 drop index 1", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action replace action ctinfo zone 200 pass index 1", + "expExitCode": "0", + "verifyCmd": "$TC action get action ctinfo index 1", + "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "e567", + "name": "Delete ctinfo action with valid index", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + [ + "$TC action add action ctinfo zone 200 pass index 1", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action ctinfo index 1", + "expExitCode": "0", + "verifyCmd": "$TC action get action ctinfo index 1", + "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref", + "matchCount": "0", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "6a91", + "name": "Delete ctinfo action with invalid index", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + [ + "$TC action add action ctinfo zone 200 pass index 1", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action ctinfo index 333", + "expExitCode": "255", + "verifyCmd": "$TC action get action ctinfo index 1", + "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "5232", + "name": "List ctinfo actions", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC action flush action ctinfo", + 0, + 1, + 255 + ], + "$TC action add action ctinfo zone 20 pass index 101", + "$TC action add action ctinfo cpmark 0x02000000 drop index 102", + "$TC action add action ctinfo continue index 103" + ], + "cmdUnderTest": "$TC action list action ctinfo", + "expExitCode": "0", + "verifyCmd": "$TC action list action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo", + "matchCount": "3", + "teardown": [ + "$TC actions flush action ctinfo" + ] + }, + { + "id": "7702", + "name": "Flush ctinfo actions", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + "$TC action add action ctinfo zone 20 pass index 101", + "$TC action add action ctinfo cpmark 0x02000000 drop index 102", + "$TC action add action ctinfo continue index 103" + ], + "cmdUnderTest": "$TC action flush action ctinfo", + "expExitCode": "0", + "verifyCmd": "$TC action list action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo", + "matchCount": "0", + "teardown": [ + "$TC actions flush action ctinfo" + ] + }, + { + "id": "3201", + "name": "Add ctinfo action with duplicate index", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + "$TC action add action ctinfo zone 20 pass index 101" + ], + "cmdUnderTest": "$TC action add action ctinfo cpmark 0x02000000 drop index 101", + "expExitCode": "255", + "verifyCmd": "$TC action get action ctinfo index 101", + "matchPattern": "action order [0-9]*: ctinfo zone 20 pass.*index 101", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "8295", + "name": "Add ctinfo action with invalid index", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action ctinfo zone 20 index 4294967296", + "expExitCode": "255", + "verifyCmd": "$TC action ls action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo", + "matchCount": "0", + "teardown": [ + "$TC action flush action ctinfo" + ] + }, + { + "id": "3964", + "name": "Replace ctinfo action with invalid goto_chain control", + "category": [ + "actions", + "ctinfo" + ], + "setup": [ + [ + "$TC actions flush action ctinfo", + 0, + 1, + 255 + ], + "$TC action add action ctinfo pass index 90" + ], + "cmdUnderTest": "$TC action replace action ctinfo goto chain 42 index 90", + "expExitCode": "255", + "verifyCmd": "$TC action list action ctinfo", + "matchPattern": "action order [0-9]*: ctinfo.*pass.*index 90", + "matchCount": "1", + "teardown": [ + "$TC action flush action ctinfo" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json b/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json new file mode 100644 index 0000000000000000000000000000000000000000..e16a4963fdd23662b29e81bfb98be64df7c23e19 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json @@ -0,0 +1,315 @@ +[ + { + "id": "5153", + "name": "Add gate action with priority and sched-entry", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC action flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate priority 1 sched-entry close 100000000ns index 100", + "expExitCode": "0", + "verifyCmd": "$TC action get action gate index 100", + "matchPattern": "action order [0-9]*: .*priority 1.*index 100 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "7189", + "name": "Add gate action with base-time", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate base-time 200000000000ns sched-entry close 100000000ns index 10", + "expExitCode": "0", + "verifyCmd": "$TC action ls action gate", + "matchPattern": "action order [0-9]*: .*base-time 200s.*index 10 ref", + "matchCount": "1", + "teardown": [ + "$TC actions flush action gate" + ] + }, + { + "id": "a721", + "name": "Add gate action with cycle-time", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC action flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate cycle-time 200000000000ns sched-entry close 100000000ns index 1000", + "expExitCode": "0", + "verifyCmd": "$TC action ls action gate", + "matchPattern": "action order [0-9]*: .*cycle-time 200s.*index 1000 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "c029", + "name": "Add gate action with cycle-time-ext", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC action flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate cycle-time-ext 20000000000ns sched-entry close 100000000ns index 1000", + "expExitCode": "0", + "verifyCmd": "$TC action get action gate index 1000", + "matchPattern": "action order [0-9]*: .*cycle-time-ext 20s.*index 1000 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "3719", + "name": "Replace gate base-time action", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ], + [ + "$TC action add action gate base-time 200000000000ns sched-entry open 200000000ns -1 8000000b index 20", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action replace action gate base-time 400000000000ns index 20", + "expExitCode": "0", + "verifyCmd": "$TC action get action gate index 20", + "matchPattern": "action order [0-9]*: .*base-time 400s.*index 20 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "d821", + "name": "Delete gate action with valid index", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ], + [ + "$TC action add action gate base-time 200000000000ns sched-entry open 200000000ns -1 8000000b index 302", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action gate index 302", + "expExitCode": "0", + "verifyCmd": "$TC action get action bpf index 302", + "matchPattern": "action order [0-9]*: .*base-time 200s.*index 302 ref", + "matchCount": "0", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "3128", + "name": "Delete gate action with invalid index", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ], + [ + "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 999", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action gate index 333", + "expExitCode": "255", + "verifyCmd": "$TC action get action gate index 999", + "matchPattern": "action order [0-9]*: .*base-time 600s.*index 999 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "7837", + "name": "List gate actions", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC action flush action gate", + 0, + 1, + 255 + ], + "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 101", + "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 102", + "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 103" + ], + "cmdUnderTest": "$TC action list action gate", + "expExitCode": "0", + "verifyCmd": "$TC action list action gate", + "matchPattern": "action order [0-9]*:", + "matchCount": "3", + "teardown": [ + "$TC actions flush action gate" + ] + }, + { + "id": "9273", + "name": "Flush gate actions", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ], + "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 101", + "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 102", + "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 103" + ], + "cmdUnderTest": "$TC action flush action gate", + "expExitCode": "0", + "verifyCmd": "$TC action list action gate", + "matchPattern": "action order [0-9]*: .*priority", + "matchCount": "0", + "teardown": [ + "$TC actions flush action gate" + ] + }, + { + "id": "c829", + "name": "Add gate action with duplicate index", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ], + "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 4294967295" + ], + "cmdUnderTest": "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 4294967295", + "expExitCode": "255", + "verifyCmd": "$TC action get action gate index 4294967295", + "matchPattern": "action order [0-9]*: .*index 4294967295", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "3043", + "name": "Add gate action with invalid index", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 4294967296", + "expExitCode": "255", + "verifyCmd": "$TC action ls action gate", + "matchPattern": "action order [0-9]*:", + "matchCount": "0", + "teardown": [ + "$TC action flush action gate" + ] + }, + { + "id": "2930", + "name": "Add gate action with cookie", + "category": [ + "actions", + "gate" + ], + "setup": [ + [ + "$TC actions flush action gate", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 4294 cookie d0d0d0d0d0d0d0d0", + "expExitCode": "0", + "verifyCmd": "$TC action list action gate", + "matchPattern": "action order [0-9]*: .*cookie d0d0d0d0d0d0d0", + "matchCount": "1", + "teardown": [ + "$TC action flush action gate" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json index c13a68b98fc775086d2087205ccefeb5304a4e7a..459bcad35810f15805db74e1437ef5797ff9fd5e 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json @@ -1085,5 +1085,55 @@ "teardown": [ "$TC actions flush action ife" ] + }, + { + "id": "a972", + "name": "Delete ife encode action with valid index", + "category": [ + "actions", + "ife" + ], + "setup": [ + [ + "$TC actions flush action ife", + 0, + 1, + 255 + ], + "$TC actions add action ife encode allow mark pass index 20" + ], + "cmdUnderTest": "$TC actions del action ife index 20", + "expExitCode": "0", + "verifyCmd": "$TC actions ls action ife index 20", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 20", + "matchCount": "0", + "teardown": [ + "$TC actions flush action ife" + ] + }, + { + "id": "1272", + "name": "Delete ife encode action with invalid index", + "category": [ + "actions", + "ife" + ], + "setup": [ + [ + "$TC actions flush action ife", + 0, + 1, + 255 + ], + "$TC actions add action ife encode allow mark pass index 20" + ], + "cmdUnderTest": "$TC actions del action ife index 10", + "expExitCode": "255", + "verifyCmd": "$TC actions ls action ife index 20", + "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 20", + "matchCount": "1", + "teardown": [ + "$TC actions flush action ife" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json b/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json index bc12c1ccad30e9660c5ab19abfc8064aece57f05..0a3c491edbc59457f9a21d86683f9e0bc4ff5d6d 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json @@ -614,5 +614,55 @@ "teardown": [ "$TC actions flush action nat" ] + }, + { + "id": "b811", + "name": "Delete nat action with valid index", + "category": [ + "actions", + "nat" + ], + "setup": [ + [ + "$TC actions flush action nat", + 0, + 1, + 255 + ], + "$TC actions add action nat ingress 1.1.1.1 2.2.2.2 drop index 20" + ], + "cmdUnderTest": "$TC actions del action nat index 20", + "expExitCode": "0", + "verifyCmd": "$TC actions ls action nat index 20", + "matchPattern": "action order [0-9]+: nat ingress 1.1.1.1/32 2.2.2.2 drop.*index 20 ref", + "matchCount": "0", + "teardown": [ + "$TC actions flush action nat" + ] + }, + { + "id": "a521", + "name": "Delete nat action with invalid index", + "category": [ + "actions", + "nat" + ], + "setup": [ + [ + "$TC actions flush action nat", + 0, + 1, + 255 + ], + "$TC actions add action nat ingress 1.1.1.1 2.2.2.2 drop index 20" + ], + "cmdUnderTest": "$TC actions del action nat index 10", + "expExitCode": "255", + "verifyCmd": "$TC actions ls action nat index 20", + "matchPattern": "action order [0-9]+: nat ingress 1.1.1.1/32 2.2.2.2 drop.*index 20 ref", + "matchCount": "1", + "teardown": [ + "$TC actions flush action nat" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json b/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json index ddabb160a11bacc151b9891f14f7da62846b8ed1..148d8bcb8606ebab879883a5e858cef5b3c61711 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json @@ -633,5 +633,55 @@ "teardown": [ "$TC actions flush action sample" ] + }, + { + "id": "3872", + "name": "Delete sample action with valid index", + "category": [ + "actions", + "sample" + ], + "setup": [ + [ + "$TC actions flush action sample", + 0, + 1, + 255 + ], + "$TC actions add action sample rate 10 group 1 index 20" + ], + "cmdUnderTest": "$TC actions del action sample index 20", + "expExitCode": "0", + "verifyCmd": "$TC actions get action sample index 20", + "matchPattern": "action order [0-9]+: sample rate 1/10 group 1.*index 20 ref", + "matchCount": "0", + "teardown": [ + "$TC actions flush action sample" + ] + }, + { + "id": "a394", + "name": "Delete sample action with invalid index", + "category": [ + "actions", + "sample" + ], + "setup": [ + [ + "$TC actions flush action sample", + 0, + 1, + 255 + ], + "$TC actions add action sample rate 10 group 1 index 20" + ], + "cmdUnderTest": "$TC actions del action sample index 10", + "expExitCode": "255", + "verifyCmd": "$TC actions get action sample index 20", + "matchPattern": "action order [0-9]+: sample rate 1/10 group 1.*index 20 ref", + "matchCount": "1", + "teardown": [ + "$TC actions flush action sample" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json index d06346968bcbdbccfb82094423ca340bc62a717f..b40ee602918a6f8d72ea860809b849aabb30b864 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json @@ -933,5 +933,55 @@ "teardown": [ "$TC actions flush action tunnel_key" ] + }, + { + "id": "3671", + "name": "Delete tunnel_key set action with valid index", + "category": [ + "actions", + "tunnel_key" + ], + "setup": [ + [ + "$TC actions flush action tunnel_key", + 0, + 1, + 255 + ], + "$TC actions add action tunnel_key set src_ip 1.1.1.1 dst_ip 2.2.2.2 index 1" + ], + "cmdUnderTest": "$TC actions del action tunnel_key index 1", + "expExitCode": "0", + "verifyCmd": "$TC actions list action tunnel_key", + "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 1.1.1.1.*dst_ip 2.2.2.2.*index 1", + "matchCount": "0", + "teardown": [ + "$TC actions flush action tunnel_key" + ] + }, + { + "id": "8597", + "name": "Delete tunnel_key set action with invalid index", + "category": [ + "actions", + "tunnel_key" + ], + "setup": [ + [ + "$TC actions flush action tunnel_key", + 0, + 1, + 255 + ], + "$TC actions add action tunnel_key set src_ip 1.1.1.1 dst_ip 2.2.2.2 index 1" + ], + "cmdUnderTest": "$TC actions del action tunnel_key index 10", + "expExitCode": "255", + "verifyCmd": "$TC actions list action tunnel_key", + "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 1.1.1.1.*dst_ip 2.2.2.2.*index 1", + "matchCount": "1", + "teardown": [ + "$TC actions flush action tunnel_key" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json b/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json new file mode 100644 index 0000000000000000000000000000000000000000..c9f002aea6d44cd8318bee4a002e85c940176ffe --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json @@ -0,0 +1,219 @@ +[ + { + "id": "2029", + "name": "Add xt action with log-prefix", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix PONG index 100", + "expExitCode": "0", + "verifyCmd": "$TC action ls action xt", + "matchPattern": "action order [0-9]*:.*target LOG level warning prefix \"PONG\".*index 100 ref", + "matchCount": "1", + "teardown": [ + "$TC actions flush action xt" + ] + }, + { + "id": "3562", + "name": "Replace xt action log-prefix", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ], + [ + "$TC action add action xt -j LOG --log-prefix PONG index 1", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action replace action xt -j LOG --log-prefix WIN index 1", + "expExitCode": "0", + "verifyCmd": "$TC action get action xt index 1", + "matchPattern": "action order [0-9]*:.*target LOG level warning prefix \"WIN\".*index 1 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action xt" + ] + }, + { + "id": "8291", + "name": "Delete xt action with valid index", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ], + [ + "$TC action add action xt -j LOG --log-prefix PONG index 1000", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action xt index 1000", + "expExitCode": "0", + "verifyCmd": "$TC action get action xt index 1000", + "matchPattern": "action order [0-9]*:.*target LOG level warning prefix \"PONG\".*index 1000 ref", + "matchCount": "0", + "teardown": [ + "$TC action flush action xt" + ] + }, + { + "id": "5169", + "name": "Delete xt action with invalid index", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ], + [ + "$TC action add action xt -j LOG --log-prefix PONG index 1000", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action delete action xt index 333", + "expExitCode": "255", + "verifyCmd": "$TC action get action xt index 1000", + "matchPattern": "action order [0-9]*:.*target LOG level warning prefix \"PONG\".*index 1000 ref", + "matchCount": "1", + "teardown": [ + "$TC action flush action xt" + ] + }, + { + "id": "7284", + "name": "List xt actions", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC action flush action xt", + 0, + 1, + 255 + ], + "$TC action add action xt -j LOG --log-prefix PONG index 1001", + "$TC action add action xt -j LOG --log-prefix WIN index 1002", + "$TC action add action xt -j LOG --log-prefix LOSE index 1003" + ], + "cmdUnderTest": "$TC action list action xt", + "expExitCode": "0", + "verifyCmd": "$TC action list action xt", + "matchPattern": "action order [0-9]*: tablename:", + "matchCount": "3", + "teardown": [ + "$TC actions flush action xt" + ] + }, + { + "id": "5010", + "name": "Flush xt actions", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ], + "$TC action add action xt -j LOG --log-prefix PONG index 1001", + "$TC action add action xt -j LOG --log-prefix WIN index 1002", + "$TC action add action xt -j LOG --log-prefix LOSE index 1003" + ], + "cmdUnderTest": "$TC action flush action xt", + "expExitCode": "0", + "verifyCmd": "$TC action list action xt", + "matchPattern": "action order [0-9]*: tablename:", + "matchCount": "0", + "teardown": [ + "$TC actions flush action xt" + ] + }, + { + "id": "8437", + "name": "Add xt action with duplicate index", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ], + "$TC action add action xt -j LOG --log-prefix PONG index 101" + ], + "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix WIN index 101", + "expExitCode": "255", + "verifyCmd": "$TC action get action xt index 101", + "matchPattern": "action order [0-9]*:.*target LOG level warning prefix \"PONG\".*index 101", + "matchCount": "1", + "teardown": [ + "$TC action flush action xt" + ] + }, + { + "id": "2837", + "name": "Add xt action with invalid index", + "category": [ + "actions", + "xt" + ], + "setup": [ + [ + "$TC actions flush action xt", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix WIN index 4294967296", + "expExitCode": "255", + "verifyCmd": "$TC action ls action xt", + "matchPattern": "action order [0-9]*:*target LOG level warning prefix \"WIN\"", + "matchCount": "0", + "teardown": [ + "$TC action flush action xt" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json b/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json index e788c114a484140d2cbd599b5214cdfbbed21726..d1278de8ebc3a3acbc12c782e292d7a0557131ac 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json @@ -1274,5 +1274,52 @@ "teardown": [ "$TC qdisc del dev $DEV1 ingress" ] + }, + { + "id": "0811", + "name": "Add multiple basic filter with cmp ematch u8/link layer and default action and dump them", + "category": [ + "filter", + "basic" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 basic", + "matchCount": "3", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5129", + "name": "List basic filters", + "category": [ + "filter", + "basic" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1", + "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json b/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json new file mode 100644 index 0000000000000000000000000000000000000000..1f0cae474db2b42203f3e5d4f55099e928121102 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json @@ -0,0 +1,171 @@ +[ + { + "id": "23c3", + "name": "Add cBPF filter with valid bytecode", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1563", + "name": "Add cBPF filter with invalid bytecode", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,31 0 1 2048,6 0 0 262144,6 0 0 0'", + "expExitCode": "2", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2334", + "name": "Add eBPF filter with valid object-file", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "buildebpfPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf object-file $EBPFDIR/action.o section action-ok", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1 action.o:\\[action-ok\\].*tag [0-9a-f]{16}( jited)?", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2373", + "name": "Add eBPF filter with invalid object-file", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "buildebpfPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf object-file $EBPFDIR/action.o section action-ko", + "expExitCode": "1", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1 action.o:\\[action-ko\\].*tag [0-9a-f]{16}( jited)?", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4423", + "name": "Replace cBPF bytecode", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + [ + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5122", + "name": "Delete cBPF filter", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + [ + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + 0, + 1, + 255 + ] + ], + "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf", + "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "e0a9", + "name": "List cBPF filters", + "category": [ + "filter", + "bpf-filter" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'", + "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'", + "$TC filter add dev $DEV1 parent ffff: handle 100 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 33024,6 0 0 262144,6 0 0 0'" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref 100 bpf chain [0-9]+ handle", + "matchCount": "3", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json b/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json new file mode 100644 index 0000000000000000000000000000000000000000..03723cf843793384c82453a15549b0807b3fa607 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json @@ -0,0 +1,1236 @@ +[ + { + "id": "6273", + "name": "Add cgroup filter with cmp ematch u8/link layer and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4721", + "name": "Add cgroup filter with cmp ematch u8/link layer with trans flag and pass action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff trans gt 10)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff trans gt 10\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "d392", + "name": "Add cgroup filter with cmp ematch u16/link layer and pipe action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u16 at 0 layer 0 mask 0xff00 lt 3)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u16 at 0 layer 0 mask 0xff00 lt 3\\).*action pipe", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "0234", + "name": "Add cgroup filter with cmp ematch u32/link layer and miltiple actions", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u32 at 4 layer link mask 0xff00ff00 eq 3)' action skbedit mark 7 pipe action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u32 at 4 layer 0 mask 0xff00ff00 eq 3\\).*action.*skbedit.*mark 7 pipe.*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8499", + "name": "Add cgroup filter with cmp ematch u8/network layer and pass action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0xab protocol ip prio 11 cgroup match 'cmp(u8 at 0 layer 1 mask 0xff gt 10)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 11 cgroup.*handle 0xab.*cmp\\(u8 at 0 layer 1 mask 0xff gt 10\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "b273", + "name": "Add cgroup filter with cmp ematch u8/network layer with trans flag and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0xab protocol ip prio 11 cgroup match 'cmp(u8 at 0 layer 1 mask 0xff trans gt 10)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 11 cgroup.*handle 0xab.*cmp\\(u8 at 0 layer 1 mask 0xff trans gt 10\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1934", + "name": "Add cgroup filter with cmp ematch u16/network layer and pipe action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0x100 protocol ip prio 100 cgroup match 'cmp(u16 at 0 layer network mask 0xff00 lt 3)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref 100 cgroup.*handle 0x100..*cmp\\(u16 at 0 layer 1 mask 0xff00 lt 3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2733", + "name": "Add cgroup filter with cmp ematch u32/network layer and miltiple actions", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0x112233 protocol ip prio 7 cgroup match 'cmp(u32 at 4 layer network mask 0xff00ff00 eq 3)' action skbedit mark 7 pipe action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 7 cgroup.*handle 0x112233.*cmp\\(u32 at 4 layer 1 mask 0xff00ff00 eq 3\\).*action.*skbedit.*mark 7 pipe.*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3271", + "name": "Add cgroup filter with NOT cmp ematch rule and pass action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'not cmp(u8 at 0 layer link mask 0xff eq 3)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*NOT cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2362", + "name": "Add cgroup filter with two ANDed cmp ematch rules and single action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7)' action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "9993", + "name": "Add cgroup filter with two ORed cmp ematch rules and single action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) or cmp(u16 at 8 layer link mask 0x00ff gt 7)' action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*OR cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2331", + "name": "Add cgroup filter with two ANDed cmp ematch rules and one ORed ematch rule and single action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7) or cmp(u32 at 4 layer network mask 0xa0a0 lt 3)' action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*OR cmp\\(u32 at 4 layer 1 mask 0xa0a0 lt 3\\).*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3645", + "name": "Add cgroup filter with two ANDed cmp ematch rules and one NOT ORed ematch rule and single action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7) or not cmp(u32 at 4 layer network mask 0xa0a0 lt 3)' action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*OR NOT cmp\\(u32 at 4 layer 1 mask 0xa0a0 lt 3\\).*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "b124", + "name": "Add cgroup filter with u32 ematch u8/zero offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x11 0x0f at 0)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(01000000/0f000000 at 0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7381", + "name": "Add cgroup filter with u32 ematch u8/zero offset and invalid value >0xFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'u32(u8 0x1122 0x0f at 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/0f000000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2231", + "name": "Add cgroup filter with u32 ematch u8/positive offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0x1f at 12)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(17000000/1f000000 at 12\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1882", + "name": "Add cgroup filter with u32 ematch u8/invalid mask >0xFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff00 at 12)' action drop", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000/ff000000 at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1237", + "name": "Add cgroup filter with u32 ematch u8/missing offset", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff at)' action pipe", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000 at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3812", + "name": "Add cgroup filter with u32 ematch u8/missing AT keyword", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000 at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1112", + "name": "Add cgroup filter with u32 ematch u8/missing value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 at 12)' action drop", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3241", + "name": "Add cgroup filter with u32 ematch u8/non-numeric value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 zero 0xff at 0)' action pipe", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1 flowid 1:1.*u32\\(00000000/ff000000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "e231", + "name": "Add cgroup filter with u32 ematch u8/non-numeric mask", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x11 mask at 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11000000/00000000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4652", + "name": "Add cgroup filter with u32 ematch u8/negative offset and pass action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0xaa 0xf0 at -14)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(0000a000/0000f000 at -16\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7566", + "name": "Add cgroup filter with u32 ematch u8/nexthdr+ offset and drop action", + "category": [ + "filter", + "drop" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0xaa 0xf0 at nexthdr+0)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(a0000000/f0000000 at nexthdr\\+0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1331", + "name": "Add cgroup filter with u32 ematch u16/zero offset and pipe action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x1122 0xffff at 0)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/ffff0000 at 0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "e354", + "name": "Add cgroup filter with u32 ematch u16/zero offset and invalid value >0xFFFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x112233 0xffff at 0)'", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11223300/ffff0000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3538", + "name": "Add cgroup filter with u32 ematch u16/positive offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0x1fff at 12)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(17880000/1fff0000 at 12\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4576", + "name": "Add cgroup filter with u32 ematch u16/invalid mask >0xFFFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffffffff at 12)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000/ffffffff at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "b842", + "name": "Add cgroup filter with u32 ematch u16/missing offset", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffff at)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000 at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "c924", + "name": "Add cgroup filter with u32 ematch u16/missing AT keyword", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffff 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000/ffff0000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "cc93", + "name": "Add cgroup filter with u32 ematch u16/missing value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 at 12)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "123c", + "name": "Add cgroup filter with u32 ematch u16/non-numeric value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 zero 0xffff at 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(00000000/ffff0000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3675", + "name": "Add cgroup filter with u32 ematch u16/non-numeric mask", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x1122 mask at 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/00000000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1123", + "name": "Add cgroup filter with u32 ematch u16/negative offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0xaabb 0xffff at -12)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabb0000/ffff0000 at -12\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4234", + "name": "Add cgroup filter with u32 ematch u16/nexthdr+ offset and pass action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0xaabb 0xf0f0 at nexthdr+0)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(a0b00000/f0f00000 at nexthdr\\+0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "e912", + "name": "Add cgroup filter with u32 ematch u32/zero offset and pipe action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xffffffff at 0)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabbccdd/ffffffff at 0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1435", + "name": "Add cgroup filter with u32 ematch u32/positive offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11227788 0x1ffff0f0 at 12)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11227080/1ffff0f0 at 12\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1282", + "name": "Add cgroup filter with u32 ematch u32/missing offset", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11227788 0xffffffff at)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11227788/ffffffff at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "6456", + "name": "Add cgroup filter with u32 ematch u32/missing AT keyword", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x77889900 0xfffff0f0 0)' action pipe", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77889900/fffff0f0 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4231", + "name": "Add cgroup filter with u32 ematch u32/missing value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 at 12)' action pipe", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2131", + "name": "Add cgroup filter with u32 ematch u32/non-numeric value", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 zero 0xffff at 0)' action pipe", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(00000000/ffff0000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "f125", + "name": "Add cgroup filter with u32 ematch u32/non-numeric mask", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11223344 mask at 0)' action pass", + "expExitCode": "1", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11223344/00000000 at 0\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4316", + "name": "Add cgroup filter with u32 ematch u32/negative offset and drop action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xff00ff00 at -12)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aa00cc00/ff00ff00 at -12\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "23ae", + "name": "Add cgroup filter with u32 ematch u32/nexthdr+ offset and pipe action", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xffffffff at nexthdr+0)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabbccdd/ffffffff at nexthdr\\+0\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "23a1", + "name": "Add cgroup filter with canid ematch and single SFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 1)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x1\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "324f", + "name": "Add cgroup filter with canid ematch and single SFF with mask", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0xaabb:0x00ff)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x2BB:0xFF\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2576", + "name": "Add cgroup filter with canid ematch and multiple SFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 1 sff 2 sff 3)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x1 sff 0x2 sff 0x3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4839", + "name": "Add cgroup filter with canid ematch and multiple SFF with masks", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0xaa:0x01 sff 0xbb:0x02 sff 0xcc:0x03)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0xAA:0x1 sff 0xBB:0x2 sff 0xCC:0x3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "6713", + "name": "Add cgroup filter with canid ematch and single EFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 1)' action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x1\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4572", + "name": "Add cgroup filter with canid ematch and single EFF with mask", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 0xaabb:0xf1f1)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0xAABB:0xF1F1\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8031", + "name": "Add cgroup filter with canid ematch and multiple EFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 1 eff 2 eff 3)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x1 eff 0x2 eff 0x3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "ab9d", + "name": "Add cgroup filter with canid ematch and multiple EFF with masks", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 0xaa:0x01 eff 0xbb:0x02 eff 0xcc:0x03)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0xAA:0x1 eff 0xBB:0x2 eff 0xCC:0x3\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5349", + "name": "Add cgroup filter with canid ematch and a combination of SFF/EFF", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0x01 eff 0x02)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x2 sff 0x1\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "c934", + "name": "Add cgroup filter with canid ematch and a combination of SFF/EFF with masks", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0x01:0xf eff 0x02:0xf)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x2:0xF sff 0x1:0xF\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4319", + "name": "Replace cgroup filter with diffferent match", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass" + ], + "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 8)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 8\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4636", + "name": "Delete cgroup filter", + "category": [ + "filter", + "cgroup" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass" + ], + "cmdUnderTest": "$TC filter delete dev $DEV1 parent ffff: protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 8\\)", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json b/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json new file mode 100644 index 0000000000000000000000000000000000000000..58189327f6444adfb20be819cc30d0630a7c374f --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json @@ -0,0 +1,623 @@ +[ + { + "id": "5294", + "name": "Add flow filter with map key and ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst and 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst and 0x000000ff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3514", + "name": "Add flow filter with map key or ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst or 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst.*or 0x000000ff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7534", + "name": "Add flow filter with map key xor ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst xor 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst xor 0x000000ff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4524", + "name": "Add flow filter with map key rshift ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst rshift 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst rshift 255 baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "0230", + "name": "Add flow filter with map key addend ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2344", + "name": "Add flow filter with src map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key src addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys src addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "9304", + "name": "Add flow filter with proto map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "9038", + "name": "Add flow filter with proto-src map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto-src addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto-src addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2a03", + "name": "Add flow filter with proto-dst map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto-dst addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto-dst addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "a073", + "name": "Add flow filter with iif map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key iif addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys iif addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3b20", + "name": "Add flow filter with priority map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key priority addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys priority addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8945", + "name": "Add flow filter with mark map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key mark addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys mark addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "c034", + "name": "Add flow filter with nfct map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "0205", + "name": "Add flow filter with nfct-src map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-dst addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-dst addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5315", + "name": "Add flow filter with nfct-src map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-src addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-src addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7849", + "name": "Add flow filter with nfct-proto-src map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-proto-src addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-proto-src addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "9902", + "name": "Add flow filter with nfct-proto-dst map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-proto-dst addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-proto-dst addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "6742", + "name": "Add flow filter with rt-classid map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rt-classid addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rt-classid addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5432", + "name": "Add flow filter with sk-uid map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key sk-uid addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys sk-uid addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4134", + "name": "Add flow filter with sk-gid map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key sk-gid addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys sk-gid addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4522", + "name": "Add flow filter with vlan-tag map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key vlan-tag addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys vlan-tag addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4253", + "name": "Add flow filter with rxhash map key", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0xff baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4452", + "name": "Add flow filter with hash key list", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow hash keys src", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 hash keys src baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4341", + "name": "Add flow filter with muliple ops", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow hash keys src divisor 1024 baseclass 1:1 match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 hash keys src divisor 1024 baseclass 1:1.*cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4392", + "name": "List flow filters", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff", + "$TC filter add dev $DEV1 parent ffff: handle 2 prio 1 protocol ip flow map key rxhash or 0xff" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref 1 flow chain 0 handle 0x[0-9]+ map keys rxhash", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4322", + "name": "Change flow filter with map key num", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff" + ], + "cmdUnderTest": "$TC filter change dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0x22", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x22 baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2320", + "name": "Replace flow filter with map key num", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff" + ], + "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0x88", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x88 baseclass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "3213", + "name": "Delete flow filter with map key num", + "category": [ + "filter", + "flow" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff" + ], + "cmdUnderTest": "$TC filter delete dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow", + "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x88 baseclass", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/route.json b/tools/testing/selftests/tc-testing/tc-tests/filters/route.json new file mode 100644 index 0000000000000000000000000000000000000000..1f6f19f0299799097257ff367f59d24a679c118c --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/route.json @@ -0,0 +1,181 @@ +[ + { + "id": "e122", + "name": "Add route filter with from and to tag", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 100 route from 1 to 10 classid 1:10", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "flowid 1:10 to 10 from 1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "6573", + "name": "Add route filter with fromif and to tag", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 100 route fromif $DEV1 to 10 classid 1:10", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "flowid 1:10 to 10 fromif", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1362", + "name": "Add route filter with to flag and reclassify action", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route to 10 classid 1:20 action reclassify", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:20 to 10.*action order [0-9]+: gact action reclassify", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4720", + "name": "Add route filter with from flag and continue actions", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 classid 1:100 action continue", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:100 from 10.*action continue", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2812", + "name": "Add route filter with form tag and pipe action", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action pipe", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7994", + "name": "Add route filter with miltiple actions", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action skbedit mark 7 pipe action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action order [0-9]+: skbedit mark 7 pipe.*action order [0-9]+: gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "4312", + "name": "List route filters", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 20 to 1 classid 1:20 action pipe" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "action order [0-9]+: gact action pipe", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2634", + "name": "Delete route filter with pipe action", + "category": [ + "filter", + "route" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe" + ], + "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action pipe", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json b/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json new file mode 100644 index 0000000000000000000000000000000000000000..bdcbaa4c5663d80158bf3a37353b3a031c650817 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json @@ -0,0 +1,203 @@ +[ + { + "id": "2141", + "name": "Add rsvp filter with tcp proto and specific IP address", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto tcp session 198.168.10.64", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*session 198.168.10.64 ipproto tcp", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5267", + "name": "Add rsvp filter with udp proto and specific IP address", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*session 1.1.1.1 ipproto udp", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2819", + "name": "Add rsvp filter with src ip and src port", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 sender 2.2.2.2/5021 classid 1:1", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp sender 2.2.2.2/5021", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "c967", + "name": "Add rsvp filter with tunnelid and continue action", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnelid 2 classid 1:1 action continue", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp tunnelid 2.*action order [0-9]+: gact action continue", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "5463", + "name": "Add rsvp filter with tunnel and pipe action", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnel 2 skip 1 action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*tunnel 2 skip 1 session 1.1.1.1 ipproto udp.*action order [0-9]+: gact action pipe", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2332", + "name": "Add rsvp filter with miltiple actions", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 7 rsvp ipproto udp session 1.1.1.1 classid 1:1 action skbedit mark 7 pipe action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp.*action order [0-9]+: skbedit mark 7 pipe.*action order [0-9]+: gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8879", + "name": "Add rsvp filter with tunnel and skp flag", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnel 2 skip 1 action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*tunnel 2 skip 1 session 1.1.1.1 ipproto udp.*action order [0-9]+: gact action pipe", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8261", + "name": "List rsvp filters", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 classid 1:1", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto tcp session 2.2.2.2/1234 classid 2:1" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "8989", + "name": "Delete rsvp filter", + "category": [ + "filter", + "rsvp" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 tunnelid 9 classid 2:1" + ], + "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 tunnelid 9 classid 2:1", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 2:1 session 1.1.1.1/1234 ipproto udp tunnelid 9", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json b/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json new file mode 100644 index 0000000000000000000000000000000000000000..44901db70376425b8e4ab15672789e75b6dd33af --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json @@ -0,0 +1,227 @@ +[ + { + "id": "8293", + "name": "Add tcindex filter with default action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref 1 tcindex chain 0 handle 0x0001 classid 1:1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7281", + "name": "Add tcindex filter with hash size and pass action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 fall_through classid 1:1 action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "b294", + "name": "Add tcindex filter with mask shift and reclassify action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action reclassify", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action reclassify", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "0532", + "name": "Add tcindex filter with pass_on and continue actions", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 pass_on classid 1:1 action continue", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action continue", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "d473", + "name": "Add tcindex filter with pipe action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action pipe", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action pipe", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2940", + "name": "Add tcindex filter with miltiple actions", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress" + ], + "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 7 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action skbedit mark 7 pipe action gact drop", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 7 protocol ip tcindex", + "matchPattern": "^filter parent ffff: protocol ip pref 7 tcindex.*handle 0x0001.*action.*skbedit.*mark 7 pipe.*action.*gact action drop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "1893", + "name": "List tcindex filters", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1", + "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 tcindex classid 1:1" + ], + "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:", + "expExitCode": "0", + "verifyCmd": "$TC filter show dev $DEV1 parent ffff:", + "matchPattern": "handle 0x000[0-9]+ classid 1:1", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "2041", + "name": "Change tcindex filter with pass action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop" + ], + "cmdUnderTest": "$TC filter change dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "9203", + "name": "Replace tcindex filter with pass action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop" + ], + "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action pass", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + }, + { + "id": "7957", + "name": "Delete tcindex filter with drop action", + "category": [ + "filter", + "tcindex" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DEV1 ingress", + "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop" + ], + "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop", + "expExitCode": "0", + "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex", + "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action drop", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DEV1 ingress" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json new file mode 100644 index 0000000000000000000000000000000000000000..f5bc8670a67d1d3eaa36be8009746bf32101b362 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json @@ -0,0 +1,94 @@ +[ + { + "id": "7628", + "name": "Create ATM with default setting", + "category": [ + "qdisc", + "atm" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc atm 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "390a", + "name": "Delete ATM with valid handle", + "category": [ + "qdisc", + "atm" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root atm" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc atm 1: root refcnt", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "32a0", + "name": "Show ATM class", + "category": [ + "qdisc", + "atm" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class atm 1: parent 1:", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6310", + "name": "Dump ATM stats", + "category": [ + "qdisc", + "atm" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm", + "expExitCode": "0", + "verifyCmd": "$TC -s qdisc show dev $DUMMY", + "matchPattern": "qdisc atm 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json new file mode 100644 index 0000000000000000000000000000000000000000..1134b72d281d4c71d5d5902e8619ce3339868266 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json @@ -0,0 +1,487 @@ +[ + { + "id": "1212", + "name": "Create CAKE with default setting", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3281", + "name": "Create CAKE with bandwidth limit", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake bandwidth 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth 1Kbit diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c940", + "name": "Create CAKE with autorate-ingress flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake autorate-ingress", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited autorate-ingress diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2310", + "name": "Create CAKE with rtt time", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake rtt 200", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 200us raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2385", + "name": "Create CAKE with besteffort flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake besteffort", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited besteffort triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a032", + "name": "Create CAKE with diffserv8 flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake diffserv8", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv8 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2349", + "name": "Create CAKE with diffserv4 flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake diffserv4", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv4 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8472", + "name": "Create CAKE with flowblind flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake flowblind", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 flowblind nonat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2341", + "name": "Create CAKE with dsthost and nat flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dsthost nat", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dsthost nat nowash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5134", + "name": "Create CAKE with wash flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake hosts wash", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 hosts nonat wash no-ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2302", + "name": "Create CAKE with flowblind and no-split-gso flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake flowblind no-split-gso", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 flowblind nonat nowash no-ack-filter no-split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0768", + "name": "Create CAKE with dual-srchost and ack-filter flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dual-srchost ack-filter", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dual-srchost nonat nowash ack-filter split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0238", + "name": "Create CAKE with dual-dsthost and ack-filter-aggressive flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dual-dsthost ack-filter-aggressive", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dual-dsthost nonat nowash ack-filter-aggressive split-gso rtt 100ms raw overhead", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6572", + "name": "Create CAKE with memlimit and ptm flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake memlimit 10000 ptm", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw ptm overhead 0 memlimit 10000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2436", + "name": "Create CAKE with fwmark and atm flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake fwmark 8 atm", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw atm overhead 0 fwmark 0x8", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3984", + "name": "Create CAKE with overhead and mpu", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 256", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5421", + "name": "Create CAKE with conservative and ingress flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake conservative ingress", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash ingress no-ack-filter split-gso rtt 100ms atm overhead 48", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6854", + "name": "Delete CAKE with conservative and ingress flag", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cake conservative ingress" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash ingress no-ack-filter split-gso rtt 100ms atm overhead 48", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2342", + "name": "Replace CAKE with mpu", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root cake mpu 128", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 128", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2313", + "name": "Change CAKE with mpu", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root cake mpu 128", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 128", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4365", + "name": "Show CAKE class", + "category": [ + "qdisc", + "cake" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class cake", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json new file mode 100644 index 0000000000000000000000000000000000000000..1ab21c83a1223863459876560cfc2e485725de54 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json @@ -0,0 +1,184 @@ +[ + { + "id": "3460", + "name": "Create CBQ with default setting", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0592", + "name": "Create CBQ with mpu", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 mpu 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4684", + "name": "Create CBQ with valid cell num", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 cell 128", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4345", + "name": "Create CBQ with invalid cell num", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 cell 100", + "expExitCode": "1", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4525", + "name": "Create CBQ with valid ewma", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 ewma 16", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6784", + "name": "Create CBQ with invalid ewma", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 ewma 128", + "expExitCode": "1", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5468", + "name": "Delete CBQ with handle", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "492a", + "name": "Show CBQ class", + "category": [ + "qdisc", + "cbq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class cbq 1: root rate 10Kbit \\(bounded,isolated\\) prio no-transmit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json new file mode 100644 index 0000000000000000000000000000000000000000..a46bf5ff827782450d1e770c57888ba08410e770 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json @@ -0,0 +1,234 @@ +[ + { + "id": "1820", + "name": "Create CBS with default setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "1532", + "name": "Create CBS with hicredit setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs hicredit 64", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 64 locredit 0 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2078", + "name": "Create CBS with locredit setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs locredit 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 10 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9271", + "name": "Create CBS with sendslope setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs sendslope 888", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 888 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0482", + "name": "Create CBS with idleslope setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 666 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "e8f3", + "name": "Create CBS with multiple setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs hicredit 10 locredit 75 sendslope 2 idleslope 666", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 10 locredit 75 sendslope 2 idleslope 666 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "23c9", + "name": "Replace CBS with sendslope setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root cbs sendslope 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 10 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a07a", + "name": "Change CBS with idleslope setting", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root cbs idleslope 1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 1 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "43b3", + "name": "Delete CBS with handle", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 1 offload 0.*qdisc pfifo 0: parent 1: limit 1000p", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9472", + "name": "Show CBS class", + "category": [ + "qdisc", + "cbs" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class cbs 1:[0-9]+ parent 1:", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json new file mode 100644 index 0000000000000000000000000000000000000000..31b7775d25fcdaf6a2a5260abd7d0748ab88ff08 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json @@ -0,0 +1,188 @@ +[ + { + "id": "8937", + "name": "Create CHOKE with default setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "48c0", + "name": "Create CHOKE with min packet setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "38c1", + "name": "Create CHOKE with max packet setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 max 900", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min.*max 900p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "234a", + "name": "Create CHOKE with ecn setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 ecn", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p ecn", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4380", + "name": "Create CHOKE with burst setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 burst 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "48c7", + "name": "Delete CHOKE with valid handle", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4398", + "name": "Replace CHOKE with min setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0301", + "name": "Change CHOKE with limit setting", + "category": [ + "qdisc", + "choke" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json new file mode 100644 index 0000000000000000000000000000000000000000..ea38099d48e5bdac02ca5bb55d1e8eaf85285138 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json @@ -0,0 +1,211 @@ +[ + { + "id": "983a", + "name": "Create CODEL with default setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "38aa", + "name": "Create CODEL with limit packet setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel limit 1500", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1500p target 5ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9178", + "name": "Create CODEL with target setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel target 100ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 100ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "78d1", + "name": "Create CODEL with interval setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel interval 20ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 20ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "238a", + "name": "Create CODEL with ecn setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel ecn", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms ecn", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "939c", + "name": "Create CODEL with ce_threshold setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel ce_threshold 20ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms ce_threshold 20ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8380", + "name": "Delete CODEL with valid handle", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root codel" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "289c", + "name": "Replace CODEL with limit setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root codel" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root codel limit 5000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 5000p target 5ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0648", + "name": "Change CODEL with limit setting", + "category": [ + "qdisc", + "codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root codel" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root codel limit 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 100p target 5ms interval 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json new file mode 100644 index 0000000000000000000000000000000000000000..486a425b3c1cb1da35b09d36c1ebcf1b05a3399a --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json @@ -0,0 +1,71 @@ +[ + { + "id": "0385", + "name": "Create DRR with default setting", + "category": [ + "qdisc", + "drr" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root drr", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc drr 1: root refcnt [0-9]+", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2375", + "name": "Delete DRR with handle", + "category": [ + "qdisc", + "drr" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root drr" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc drr 1: root refcnt [0-9]+", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3092", + "name": "Show DRR class", + "category": [ + "qdisc", + "drr" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root drr", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class drr 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json new file mode 100644 index 0000000000000000000000000000000000000000..c030795f9c37d4405904d3875b187c912f1e4e47 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json @@ -0,0 +1,140 @@ +[ + { + "id": "6345", + "name": "Create DSMARK with default setting", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3462", + "name": "Create DSMARK with default_index setting", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 512", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 default_index 0x0200", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "ca95", + "name": "Create DSMARK with set_tc_index flag", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 set_tc_index", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 set_tc_index", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a950", + "name": "Create DSMARK with multiple setting", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 1024 set_tc_index", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 default_index 0x0400 set_tc_index", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4092", + "name": "Delete DSMARK with handle", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 1024" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5930", + "name": "Show DSMARK class", + "category": [ + "qdisc", + "dsmark" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class dsmark 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json new file mode 100644 index 0000000000000000000000000000000000000000..0046d44bcd93324847bcb79e30d3ce18d578af05 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json @@ -0,0 +1,117 @@ +[ + { + "id": "34ba", + "name": "Create ETF with default setting", + "category": [ + "qdisc", + "etf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check off", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "438f", + "name": "Create ETF with delta nanos setting", + "category": [ + "qdisc", + "etf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf delta 100 clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 100 offload off deadline_mode off skip_sock_check off", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9041", + "name": "Create ETF with deadline_mode setting", + "category": [ + "qdisc", + "etf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI deadline_mode", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode on skip_sock_check off", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9a0c", + "name": "Create ETF with skip_sock_check setting", + "category": [ + "qdisc", + "etf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI skip_sock_check", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check on", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2093", + "name": "Delete ETF with valid handle", + "category": [ + "qdisc", + "etf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check off", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json new file mode 100644 index 0000000000000000000000000000000000000000..8acb904d14193274ccb5326fc532a8391481c238 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json @@ -0,0 +1,395 @@ +[ + { + "id": "983b", + "name": "Create FQ with default setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "38a1", + "name": "Create FQ with limit packet setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq limit 3000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 3000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0a18", + "name": "Create FQ with flow_limit setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq flow_limit 300", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 300p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2390", + "name": "Create FQ with quantum setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq quantum 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*orphan_mask 1023 quantum 9000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "845b", + "name": "Create FQ with initial_quantum setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq initial_quantum 900000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*initial_quantum 900000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9398", + "name": "Create FQ with maxrate setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq maxrate 100000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*maxrate 100Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "342c", + "name": "Create FQ with nopacing setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq nopacing", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*nopacing", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6391", + "name": "Create FQ with refill_delay setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq refill_delay 100ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*refill_delay 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "238b", + "name": "Create FQ with low_rate_threshold setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq low_rate_threshold 10000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*low_rate_threshold 10Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7582", + "name": "Create FQ with orphan_mask setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq orphan_mask 255", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*orphan_mask 255", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4894", + "name": "Create FQ with timer_slack setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq timer_slack 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*timer_slack 100ns", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "324c", + "name": "Create FQ with ce_threshold setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq ce_threshold 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "424a", + "name": "Create FQ with horizon time setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq horizon 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*horizon 100us", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "89e1", + "name": "Create FQ with horizon_cap setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq horizon_cap", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*horizon_cap", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "32e1", + "name": "Delete FQ with valid handle", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "49b0", + "name": "Replace FQ with limit setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root fq limit 5000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 5000p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9478", + "name": "Change FQ with limit setting", + "category": [ + "qdisc", + "fq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root fq limit 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 100p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json new file mode 100644 index 0000000000000000000000000000000000000000..a65266357a9a38c4cb5db18dafc34df2fc7f83ae --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json @@ -0,0 +1,326 @@ +[ + { + "id": "4957", + "name": "Create FQ_CODEL with default setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7621", + "name": "Create FQ_CODEL with limit setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6871", + "name": "Create FQ_CODEL with memory_limit setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel memory_limit 100000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 100000b ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5636", + "name": "Create FQ_CODEL with target setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel target 2000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 2ms interval 100ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "630a", + "name": "Create FQ_CODEL with interval setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel interval 5000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 5ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4324", + "name": "Create FQ_CODEL with quantum setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel quantum 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum 9000 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "b190", + "name": "Create FQ_CODEL with noecn flag", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel noecn", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5381", + "name": "Create FQ_CODEL with ce_threshold setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel ce_threshold 1024000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms ce_threshold 1.02s interval 100ms memory_limit 32Mb ecn drop_batch 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c9d2", + "name": "Create FQ_CODEL with drop_batch setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel drop_batch 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "523b", + "name": "Create FQ_CODEL with multiple setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9283", + "name": "Replace FQ_CODEL with noecn setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root fq_codel noecn", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb drop_batch 100", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3459", + "name": "Change FQ_CODEL with limit setting", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root fq_codel limit 2000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 2000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0128", + "name": "Delete FQ_CODEL with handle", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb noecn drop_batch 100", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0435", + "name": "Show FQ_CODEL class", + "category": [ + "qdisc", + "fq_codel" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class fq_codel 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json new file mode 100644 index 0000000000000000000000000000000000000000..013c8ee037a4bce2f25826dc96a55b20b598be85 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json @@ -0,0 +1,164 @@ +[ + { + "id": "8942", + "name": "Create GRED with default setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5783", + "name": "Create GRED with grio setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1 grio", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1.*grio", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8a09", + "name": "Create GRED with limit setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1 limit 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1 limit 1000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "48ca", + "name": "Create GRED with ecn setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 2 ecn", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 2.*ecn", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "48cb", + "name": "Create GRED with harddrop setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 2 harddrop", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 2.*harddrop", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "763a", + "name": "Change GRED setting", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root gred limit 60KB min 15K max 25K burst 64 avpkt 1500 bandwidth 10Mbit DP 1 probability 0.1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1 limit.*vq 1 prio [0-9]+ limit 60Kb min 15Kb max 25Kb", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8309", + "name": "Show GRED class", + "category": [ + "qdisc", + "gred" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class gred 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json new file mode 100644 index 0000000000000000000000000000000000000000..af27b2c20e17f758d964c98791c462045e2fd8b4 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json @@ -0,0 +1,167 @@ +[ + { + "id": "3254", + "name": "Create HFSC with default setting", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hfsc", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hfsc 1: root refcnt [0-9]+", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0289", + "name": "Create HFSC with class sc and ul rate setting", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc sc rate 20000 ul rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hfsc 1:1 parent 1: sc m1 0bit d 0us m2 20Kbit ul m1 0bit d 0us m2 10Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "846a", + "name": "Create HFSC with class sc umax and dmax setting", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc sc umax 1540 dmax 5ms rate 10000 ul rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hfsc 1:1 parent 1: sc m1 2464Kbit d 5ms m2 10Kbit ul m1 0bit d 0us m2 10Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5413", + "name": "Create HFSC with class rt and ls rate setting", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc rt rate 20000 ls rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hfsc 1:1 parent 1: rt m1 0bit d 0us m2 20Kbit ls m1 0bit d 0us m2 10Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9312", + "name": "Create HFSC with class rt umax and dmax setting", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc rt umax 1540 dmax 5ms rate 10000 ls rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hfsc 1:1 parent 1: rt m1 2464Kbit d 5ms m2 10Kbit ls m1 0bit d 0us m2 10Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6931", + "name": "Delete HFSC with handle", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hfsc 1: root refcnt [0-9]+", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8436", + "name": "Show HFSC class", + "category": [ + "qdisc", + "hfsc" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hfsc", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hfsc 1: root", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json new file mode 100644 index 0000000000000000000000000000000000000000..949f6e5de902cd7da08ba74c1db1b8a9baa642cc --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json @@ -0,0 +1,210 @@ +[ + { + "id": "4812", + "name": "Create HHF with default setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8a92", + "name": "Create HHF with limit setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf limit 1500", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+ limit 1500p.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3491", + "name": "Create HHF with quantum setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf quantum 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*quantum 9000b hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "ba04", + "name": "Create HHF with reset_timeout setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf reset_timeout 100ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 100ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4238", + "name": "Create HHF with admit_bytes setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf admit_bytes 100000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 100000b evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "839f", + "name": "Create HHF with evict_timeout setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf evict_timeout 0.5s", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 500ms non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a044", + "name": "Create HHF with non_hh_weight setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf non_hh_weight 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 10", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "32f9", + "name": "Change HHF with limit setting", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root hhf" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root hhf limit 1500", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc hhf 1: root refcnt [0-9]+ limit 1500p.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "385e", + "name": "Show HHF class", + "category": [ + "qdisc", + "hhf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class hhf 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json new file mode 100644 index 0000000000000000000000000000000000000000..9529899482e046a80ad55332a5d3c33a241aac1c --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json @@ -0,0 +1,285 @@ +[ + { + "id": "0904", + "name": "Create HTB with default setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0 direct_packets_stat.*direct_qlen", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3906", + "name": "Create HTB with default-N setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb default 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0x10 direct_packets_stat.* direct_qlen", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8492", + "name": "Create HTB with r2q setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb r2q 5", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 5 default 0 direct_packets_stat.*direct_qlen", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9502", + "name": "Create HTB with direct_qlen setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb direct_qlen 1024", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0 direct_packets_stat.*direct_qlen 1024", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "b924", + "name": "Create HTB with class rate and burst setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20kbit burst 1000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1000b cburst 1600b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4359", + "name": "Create HTB with class mpu setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit mpu 64", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9048", + "name": "Create HTB with class prio setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit prio 1", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 1 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4994", + "name": "Create HTB with class ceil setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit ceil 10Kbit", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 10Kbit burst 1600b cburst 1600b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9523", + "name": "Create HTB with class cburst setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit cburst 2000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 2000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5353", + "name": "Create HTB with class mtu setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit mtu 2048", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 2Kb cburst 2Kb", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "346a", + "name": "Create HTB with class quantum setting", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit quantum 2048", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "303a", + "name": "Delete HTB with handle", + "category": [ + "qdisc", + "htb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root htb r2q 5" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc htb 1: root refcnt [0-9]+", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json index d99dba6e2b1a0ad1932d634c6ab9eae26eac055f..11d33362408c0323b8a2b004acfe5a3facd02815 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json @@ -98,5 +98,25 @@ "teardown": [ "$IP link del dev $DUMMY type dummy" ] + }, + { + "id": "0521", + "name": "Show ingress class", + "category": [ + "qdisc", + "ingress" + ], + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY ingress", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class ingress", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY ingress", + "$IP link del dev $DUMMY type dummy" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json index c6046096d9db8a660b0dec6733d25a7223a80538..44fbfc6caec7b384153428df360a01cd9cef2498 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json @@ -133,5 +133,27 @@ "teardown": [ "echo \"1\" > /sys/bus/netdevsim/del_device" ] - } + }, + { + "id": "1023", + "name": "Show mq class", + "category": [ + "qdisc", + "mq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mq", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $ETH", + "matchPattern": "class mq 1:[0-9]+ root", + "matchCount": "4", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json new file mode 100644 index 0000000000000000000000000000000000000000..6e1973f731e9bb3e4245641beea1dc7d9f69d95e --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json @@ -0,0 +1,114 @@ +[ + { + "id": "9903", + "name": "Add mqprio Qdisc to multi-queue device (8 queues)", + "category": [ + "qdisc", + "mqprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc mqprio 1: root tc 8 map 0 1 2 3 4 5 6 7 0 0 0 0 0 0 0 0.*queues:\\(0:0\\) \\(1:1\\) \\(2:2\\) \\(3:3\\) \\(4:4\\) \\(5:5\\) \\(6:6\\) \\(7:7\\)", + "matchCount": "1", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "453a", + "name": "Delete nonexistent mqprio Qdisc", + "category": [ + "qdisc", + "mqprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1: mqprio", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc mqprio 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "5292", + "name": "Delete mqprio Qdisc twice", + "category": [ + "qdisc", + "mqprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device", + "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0", + "$TC qdisc del dev $ETH root handle 1:" + ], + "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc mqprio 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "45a9", + "name": "Add mqprio Qdisc to single-queue device", + "category": [ + "qdisc", + "mqprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc mqprio 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "2ba9", + "name": "Show mqprio class", + "category": [ + "qdisc", + "mqprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $ETH", + "matchPattern": "class mqprio 1:", + "matchCount": "16", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json new file mode 100644 index 0000000000000000000000000000000000000000..12c0af7a145dfd61aabbc528db16f0166c1551f1 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json @@ -0,0 +1,114 @@ +[ + { + "id": "20ba", + "name": "Add multiq Qdisc to multi-queue device (8 queues)", + "category": [ + "qdisc", + "multiq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc multiq 1: root refcnt [0-9]+ bands 8", + "matchCount": "1", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "4301", + "name": "List multiq Class", + "category": [ + "qdisc", + "multiq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $ETH", + "matchPattern": "class multiq 1:[0-9]+ parent 1:", + "matchCount": "8", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "7832", + "name": "Delete nonexistent multiq Qdisc", + "category": [ + "qdisc", + "multiq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1: multiq", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc multiq 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "2891", + "name": "Delete multiq Qdisc twice", + "category": [ + "qdisc", + "multiq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device", + "$TC qdisc add dev $ETH root handle 1: multiq", + "$TC qdisc del dev $ETH root handle 1:" + ], + "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc mqprio 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "1329", + "name": "Add multiq Qdisc to single-queue device", + "category": [ + "qdisc", + "multiq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc multiq 1: root", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json new file mode 100644 index 0000000000000000000000000000000000000000..7e41f548f8e8b0f40d9904f821b8f9c2fb4ce793 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json @@ -0,0 +1,372 @@ +[ + { + "id": "cb28", + "name": "Create NETEM with default setting", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ limit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a089", + "name": "Create NETEM with limit flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem limit 200", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ limit 200", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3449", + "name": "Create NETEM with delay time", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3782", + "name": "Create NETEM with distribution and corrupt flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal corrupt 1%", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms 10ms corrupt 1%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2b82", + "name": "Create NETEM with distribution and duplicate flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal duplicate 1%", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms 10ms duplicate 1%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a932", + "name": "Create NETEM with distribution and loss flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution pareto loss 1%", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms 10ms loss 1%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "e01a", + "name": "Create NETEM with distribution and loss state flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution paretonormal loss state 1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms 10ms loss state p13 1% p31 99% p32 0% p23 100% p14 0%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "ba29", + "name": "Create NETEM with loss gemodel flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem loss gemodel 1%", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*loss gemodel p 1%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0492", + "name": "Create NETEM with reorder flag", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms reorder 2% gap 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*reorder 2%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7862", + "name": "Create NETEM with rate limit", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem rate 20000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*rate 20Kbit", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7235", + "name": "Create NETEM with multiple slot rate", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem slot 10 200 packets 2000 bytes 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*slot 10ns 200ns packets 2000 bytes 9000", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5439", + "name": "Create NETEM with multiple slot setting", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem slot distribution pareto 1ms 0.1ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*slot distribution 1ms 100us", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5029", + "name": "Change NETEM with loss state", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 1%" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 2%", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*loss 2%", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3785", + "name": "Replace NETEM with delay time", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 1%" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root netem delay 200ms 10ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 200ms 10ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4502", + "name": "Delete NETEM with handle", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms 10ms", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0785", + "name": "Show NETEM class", + "category": [ + "qdisc", + "netem" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class netem 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json new file mode 100644 index 0000000000000000000000000000000000000000..ab53238f4c5a5a71899f2f1cf7d53a8e9adf35e5 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json @@ -0,0 +1,119 @@ +[ + { + "id": "900c", + "name": "Create pfifo_fast with default setting", + "category": [ + "qdisc", + "pfifo_fast" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7470", + "name": "Dump pfifo_fast stats", + "category": [ + "qdisc", + "pfifo_fast" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast", + "expExitCode": "0", + "verifyCmd": "$TC -s qdisc show dev $DUMMY", + "matchPattern": "Sent.*bytes.*pkt \\(dropped.*overlimits.*requeues .*\\)", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "b974", + "name": "Replace pfifo_fast with different handle", + "category": [ + "qdisc", + "pfifo_fast" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 2: root pfifo_fast", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc pfifo_fast 2: root refcnt [0-9]+ bands 3 priomap", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 2: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3240", + "name": "Delete pfifo_fast with valid handle", + "category": [ + "qdisc", + "pfifo_fast" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4385", + "name": "Delete pfifo_fast with invalid handle", + "category": [ + "qdisc", + "pfifo_fast" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 2: root", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json new file mode 100644 index 0000000000000000000000000000000000000000..6454518af178311c008b9c633a1274de243ef3cf --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json @@ -0,0 +1,188 @@ +[ + { + "id": "3289", + "name": "Create PLUG with default setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0917", + "name": "Create PLUG with block setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug block", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "483b", + "name": "Create PLUG with release setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug release", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4995", + "name": "Create PLUG with release_indefinite setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug release_indefinite", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "389c", + "name": "Create PLUG with limit setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug limit 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "384a", + "name": "Delete PLUG with valid handle", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root plug" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "439a", + "name": "Replace PLUG with limit setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root plug" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root plug limit 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9831", + "name": "Change PLUG with limit setting", + "category": [ + "qdisc", + "plug" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root plug" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root plug limit 1000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc plug 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json index 3076c02d08d61bce13f4551d2840f352c65c73ed..8186de2f0dcff992f98c45a403d0128c3e30f361 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json @@ -272,5 +272,25 @@ "teardown": [ "$IP link del dev $DUMMY type dummy" ] + }, + { + "id": "2410", + "name": "Show prio class", + "category": [ + "qdisc", + "prio" + ], + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root prio", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class prio 1:[0-9]+ parent 1:", + "matchCount": "3", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root prio", + "$IP link del dev $DUMMY type dummy" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json new file mode 100644 index 0000000000000000000000000000000000000000..330f1a25e0ab685b2d99b725a797a3e0d54e8422 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json @@ -0,0 +1,145 @@ +[ + { + "id": "0582", + "name": "Create QFQ with default setting", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root qfq", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc qfq 1: root refcnt [0-9]+", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c9a3", + "name": "Create QFQ with class weight setting", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root qfq" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class qfq 1:1 root weight 100 maxpkt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8452", + "name": "Create QFQ with class maxpkt setting", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root qfq" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 qfq maxpkt 2000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class qfq 1:1 root weight 1 maxpkt 2000", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "d920", + "name": "Create QFQ with multiple class setting", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root qfq", + "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100" + ], + "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:2 qfq weight 200", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class qfq 1:[0-9]+ root weight [0-9]+00 maxpkt", + "matchCount": "2", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0548", + "name": "Delete QFQ with handle", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root qfq", + "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "qdisc qfq 1: root refcnt [0-9]+", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5901", + "name": "Show QFQ class", + "category": [ + "qdisc", + "qfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root qfq", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class qfq 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json index 0703a2a255eb7d218158939ac25cc0b6c5e549f8..4b3e449857f2ea80eb4e1918cbf9efadfdad50d5 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json @@ -181,5 +181,28 @@ "$TC qdisc del dev $DUMMY handle 1: root", "$IP link del dev $DUMMY type dummy" ] + }, + { + "id": "290a", + "name": "Show RED class", + "category": [ + "qdisc", + "red" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root red limit 1M avpkt 1500 min 100K max 300K", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class red 1:[0-9]+ parent 1:", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] } ] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json new file mode 100644 index 0000000000000000000000000000000000000000..ba2f5e79cdbfec9288a88631b11e3aa019fb30b3 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json @@ -0,0 +1,279 @@ +[ + { + "id": "3294", + "name": "Create SFB with default setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 60s", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "430a", + "name": "Create SFB with rehash setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb rehash 60", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 60ms db 60s", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3410", + "name": "Create SFB with db setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb db 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 10ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "49a0", + "name": "Create SFB with limit setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb limit 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 60s limit 100p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "1241", + "name": "Create SFB with max setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb max 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*max 100p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3249", + "name": "Create SFB with target setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb target 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*target 100p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "30a9", + "name": "Create SFB with increment setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb increment 0.1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*increment 0.1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "239a", + "name": "Create SFB with decrement setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb decrement 0.1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*decrement 0.1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "9301", + "name": "Create SFB with penalty_rate setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_rate 4000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*penalty_rate 4000pps", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2a01", + "name": "Create SFB with penalty_burst setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_burst 64", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*penalty_burst 64p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3209", + "name": "Change SFB with rehash setting", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_burst 64" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root sfb rehash 100", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 100ms db 60s", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "5447", + "name": "Show SFB class", + "category": [ + "qdisc", + "sfb" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class sfb 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json new file mode 100644 index 0000000000000000000000000000000000000000..b6be718a174aac07241067341d183a19d6253f1e --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json @@ -0,0 +1,232 @@ +[ + { + "id": "7482", + "name": "Create SFQ with default setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum.*depth 127 divisor 1024", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c186", + "name": "Create SFQ with limit setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq limit 8", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 8p", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "ae23", + "name": "Create SFQ with perturb setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq perturb 10", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "depth 127 divisor 1024 perturb 10sec", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "a430", + "name": "Create SFQ with quantum setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq quantum 9000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 9000b depth 127 divisor 1024", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "4539", + "name": "Create SFQ with divisor setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq divisor 512", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 divisor 512", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "b089", + "name": "Create SFQ with flows setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq flows 20", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "99a0", + "name": "Create SFQ with depth setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq depth 64", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 64 divisor 1024", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7389", + "name": "Create SFQ with headdrop setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq headdrop", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 headdrop divisor 1024", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6472", + "name": "Create SFQ with redflowlimit setting", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq redflowlimit 100000 min 8000 max 60000 probability 0.20 ecn headdrop", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 headdrop divisor 1024 ewma 6 min 8000b max 60000b probability 0.2 ecn", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "8929", + "name": "Show SFQ class", + "category": [ + "qdisc", + "sfq" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class sfq 1:", + "matchCount": "0", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json new file mode 100644 index 0000000000000000000000000000000000000000..5766045c9d33bf65ef331d76baa5cc017668fb4f --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json @@ -0,0 +1,95 @@ +[ + { + "id": "283e", + "name": "Create skbprio with default setting", + "category": [ + "qdisc", + "skbprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 64", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c086", + "name": "Create skbprio with limit setting", + "category": [ + "qdisc", + "skbprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio limit 1", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 1", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6733", + "name": "Change skbprio with limit setting", + "category": [ + "qdisc", + "skbprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root skbprio" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root skbprio limit 32", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 32", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2958", + "name": "Show skbprio class", + "category": [ + "qdisc", + "skbprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class skbprio 1:", + "matchCount": "64", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json new file mode 100644 index 0000000000000000000000000000000000000000..a44455372646ad9ee6c0a64a053a79e271125c8e --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json @@ -0,0 +1,135 @@ +[ + { + "id": "ba39", + "name": "Add taprio Qdisc to multi-queue device (8 queues)", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc taprio 1: root refcnt [0-9]+ tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2", + "matchCount": "1", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "9462", + "name": "Add taprio Qdisc with multiple sched-entry", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 sched-entry S 03 300000 sched-entry S 04 400000 flags 0x1 clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "index [0-9]+ cmd S gatemask 0x[0-9]+ interval [0-9]+00000", + "matchCount": "3", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "8d92", + "name": "Add taprio Qdisc with txtime-delay", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 txtime-delay 500000 clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "clockid TAI flags 0x1 txtime delay 500000", + "matchCount": "1", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "d092", + "name": "Delete taprio Qdisc with valid handle", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device", + "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI" + ], + "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc taprio 1: root refcnt", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "8471", + "name": "Show taprio class", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1 8\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $ETH", + "matchPattern": "class taprio 1:[0-9]+ root leaf 1:", + "matchCount": "8", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + }, + { + "id": "0a85", + "name": "Add taprio Qdisc to single-queue device", + "category": [ + "qdisc", + "taprio" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "echo \"1 1\" > /sys/bus/netdevsim/new_device" + ], + "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI", + "expExitCode": "2", + "verifyCmd": "$TC qdisc show dev $ETH", + "matchPattern": "qdisc taprio 1: root refcnt", + "matchCount": "0", + "teardown": [ + "echo \"1\" > /sys/bus/netdevsim/del_device" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json new file mode 100644 index 0000000000000000000000000000000000000000..a4b3dfe51ff5d25ae881c8acc4c95f6696f7a373 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json @@ -0,0 +1,211 @@ +[ + { + "id": "6430", + "name": "Create TBF with default setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 10Kbit burst 1500b limit 1000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "0518", + "name": "Create TBF with mtu setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 mtu 2048", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b limit 1000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "320a", + "name": "Create TBF with peakrate setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 mtu 1510 peakrate 30000", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b peakrate 30Kbit minburst.*limit 1000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "239b", + "name": "Create TBF with latency setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 100ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b lat 100ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "c975", + "name": "Create TBF with overhead setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 overhead 300", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1800b limit 1000b overhead 300", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "948c", + "name": "Create TBF with linklayer setting", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer atm", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1696b limit 1000b linklayer atm", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "3549", + "name": "Replace TBF with mtu", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer atm" + ], + "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer ethernet", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b limit 1000b", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "f948", + "name": "Change TBF with latency time", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 10ms" + ], + "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 200ms", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b lat 200ms", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "2348", + "name": "Show TBF class", + "category": [ + "qdisc", + "tbf" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 10000", + "expExitCode": "0", + "verifyCmd": "$TC class show dev $DUMMY", + "matchPattern": "class tbf.*parent 1:", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json new file mode 100644 index 0000000000000000000000000000000000000000..0082be0e93acb9b62baaa5c4c3f356f8c127f801 --- /dev/null +++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json @@ -0,0 +1,97 @@ +[ + { + "id": "84a0", + "name": "Create TEQL with default setting", + "category": [ + "qdisc", + "teql" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc teql0 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "7734", + "name": "Create TEQL with multiple device", + "category": [ + "qdisc", + "teql" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "echo \"1 1 4\" > /sys/bus/netdevsim/new_device", + "$TC qdisc add dev $ETH root handle 1: teql0" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc teql0 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "echo \"1\" > /sys/bus/netdevsim/del_device", + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "34a9", + "name": "Delete TEQL with valid handle", + "category": [ + "qdisc", + "teql" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true", + "$TC qdisc add dev $DUMMY handle 1: root teql0" + ], + "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root", + "expExitCode": "0", + "verifyCmd": "$TC qdisc show dev $DUMMY", + "matchPattern": "qdisc teql0 1: root refcnt", + "matchCount": "0", + "teardown": [ + "$IP link del dev $DUMMY type dummy" + ] + }, + { + "id": "6289", + "name": "Show TEQL stats", + "category": [ + "qdisc", + "teql" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$IP link add dev $DUMMY type dummy || /bin/true" + ], + "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0", + "expExitCode": "0", + "verifyCmd": "$TC -s qdisc show dev $DUMMY", + "matchPattern": "qdisc teql0 1: root refcnt", + "matchCount": "1", + "teardown": [ + "$TC qdisc del dev $DUMMY handle 1: root", + "$IP link del dev $DUMMY type dummy" + ] + } +] diff --git a/tools/testing/selftests/timens/Makefile b/tools/testing/selftests/timens/Makefile index f0d51d4d2c879cffaea6143793096e19a9162ec5..3a5936cc10abc29347f1e03d49e7bf4b1600000d 100644 --- a/tools/testing/selftests/timens/Makefile +++ b/tools/testing/selftests/timens/Makefile @@ -1,4 +1,4 @@ -TEST_GEN_PROGS := timens timerfd timer clock_nanosleep procfs exec futex vfork_exec +TEST_GEN_PROGS := timens timerfd timer clock_nanosleep procfs exec futex TEST_GEN_PROGS_EXTENDED := gettime_perf CFLAGS := -Wall -Werror -pthread diff --git a/tools/testing/selftests/timens/vfork_exec.c b/tools/testing/selftests/timens/vfork_exec.c deleted file mode 100644 index e6ccd900f30a445e4ae6ed9b49e3c4e08b2ac723..0000000000000000000000000000000000000000 --- a/tools/testing/selftests/timens/vfork_exec.c +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define _GNU_SOURCE -#include <errno.h> -#include <fcntl.h> -#include <sched.h> -#include <stdio.h> -#include <stdbool.h> -#include <sys/stat.h> -#include <sys/syscall.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <time.h> -#include <unistd.h> -#include <string.h> - -#include "log.h" -#include "timens.h" - -#define OFFSET (36000) - -int main(int argc, char *argv[]) -{ - struct timespec now, tst; - int status, i; - pid_t pid; - - if (argc > 1) { - if (sscanf(argv[1], "%ld", &now.tv_sec) != 1) - return pr_perror("sscanf"); - - for (i = 0; i < 2; i++) { - _gettime(CLOCK_MONOTONIC, &tst, i); - if (abs(tst.tv_sec - now.tv_sec) > 5) - return pr_fail("%ld %ld\n", now.tv_sec, tst.tv_sec); - } - return 0; - } - - nscheck(); - - ksft_set_plan(1); - - clock_gettime(CLOCK_MONOTONIC, &now); - - if (unshare_timens()) - return 1; - - if (_settime(CLOCK_MONOTONIC, OFFSET)) - return 1; - - for (i = 0; i < 2; i++) { - _gettime(CLOCK_MONOTONIC, &tst, i); - if (abs(tst.tv_sec - now.tv_sec) > 5) - return pr_fail("%ld %ld\n", - now.tv_sec, tst.tv_sec); - } - - pid = vfork(); - if (pid < 0) - return pr_perror("fork"); - - if (pid == 0) { - char now_str[64]; - char *cargv[] = {"exec", now_str, NULL}; - char *cenv[] = {NULL}; - - // Check that we are still in the source timens. - for (i = 0; i < 2; i++) { - _gettime(CLOCK_MONOTONIC, &tst, i); - if (abs(tst.tv_sec - now.tv_sec) > 5) - return pr_fail("%ld %ld\n", - now.tv_sec, tst.tv_sec); - } - - /* Check for proper vvar offsets after execve. */ - snprintf(now_str, sizeof(now_str), "%ld", now.tv_sec + OFFSET); - execve("/proc/self/exe", cargv, cenv); - return pr_perror("execve"); - } - - if (waitpid(pid, &status, 0) != pid) - return pr_perror("waitpid"); - - if (status) - ksft_exit_fail(); - - ksft_test_result_pass("exec\n"); - ksft_exit_pass(); - return 0; -} diff --git a/tools/testing/selftests/tpm2/tpm2.py b/tools/testing/selftests/tpm2/tpm2.py index 057a4f49c79d963e0657088a10d09ec1baf31bd5..c7363c6764fc63fbcf8284cda88591b25f0010d1 100644 --- a/tools/testing/selftests/tpm2/tpm2.py +++ b/tools/testing/selftests/tpm2/tpm2.py @@ -371,6 +371,10 @@ class Client: fcntl.fcntl(self.tpm, fcntl.F_SETFL, flags) self.tpm_poll = select.poll() + def __del__(self): + if self.tpm: + self.tpm.close() + def close(self): self.tpm.close() diff --git a/tools/testing/selftests/user_events/ftrace_test.c b/tools/testing/selftests/user_events/ftrace_test.c index a80fb5ef61d504946adfbbd74f18eec59bebe32d..404a2713dcae8c0ec4b7351c4028006aee5b591d 100644 --- a/tools/testing/selftests/user_events/ftrace_test.c +++ b/tools/testing/selftests/user_events/ftrace_test.c @@ -22,6 +22,11 @@ const char *enable_file = "/sys/kernel/debug/tracing/events/user_events/__test_e const char *trace_file = "/sys/kernel/debug/tracing/trace"; const char *fmt_file = "/sys/kernel/debug/tracing/events/user_events/__test_event/format"; +static inline int status_check(char *status_page, int status_bit) +{ + return status_page[status_bit >> 3] & (1 << (status_bit & 7)); +} + static int trace_bytes(void) { int fd = open(trace_file, O_RDONLY); @@ -197,12 +202,12 @@ TEST_F(user, register_events) { /* Register should work */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); + ASSERT_NE(0, reg.status_bit); /* Multiple registers should result in same index */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); + ASSERT_NE(0, reg.status_bit); /* Ensure disabled */ self->enable_fd = open(enable_file, O_RDWR); @@ -212,15 +217,15 @@ TEST_F(user, register_events) { /* MMAP should work and be zero'd */ ASSERT_NE(MAP_FAILED, status_page); ASSERT_NE(NULL, status_page); - ASSERT_EQ(0, status_page[reg.status_index]); + ASSERT_EQ(0, status_check(status_page, reg.status_bit)); /* Enable event and ensure bits updated in status */ ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1"))) - ASSERT_EQ(EVENT_STATUS_FTRACE, status_page[reg.status_index]); + ASSERT_NE(0, status_check(status_page, reg.status_bit)); /* Disable event and ensure bits updated in status */ ASSERT_NE(-1, write(self->enable_fd, "0", sizeof("0"))) - ASSERT_EQ(0, status_page[reg.status_index]); + ASSERT_EQ(0, status_check(status_page, reg.status_bit)); /* File still open should return -EBUSY for delete */ ASSERT_EQ(-1, ioctl(self->data_fd, DIAG_IOCSDEL, "__test_event")); @@ -240,6 +245,8 @@ TEST_F(user, write_events) { struct iovec io[3]; __u32 field1, field2; int before = 0, after = 0; + int page_size = sysconf(_SC_PAGESIZE); + char *status_page; reg.size = sizeof(reg); reg.name_args = (__u64)"__test_event u32 field1; u32 field2"; @@ -254,10 +261,18 @@ TEST_F(user, write_events) { io[2].iov_base = &field2; io[2].iov_len = sizeof(field2); + status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED, + self->status_fd, 0); + /* Register should work */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); + ASSERT_NE(0, reg.status_bit); + + /* MMAP should work and be zero'd */ + ASSERT_NE(MAP_FAILED, status_page); + ASSERT_NE(NULL, status_page); + ASSERT_EQ(0, status_check(status_page, reg.status_bit)); /* Write should fail on invalid slot with ENOENT */ io[0].iov_base = &field2; @@ -271,6 +286,9 @@ TEST_F(user, write_events) { self->enable_fd = open(enable_file, O_RDWR); ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1"))) + /* Event should now be enabled */ + ASSERT_NE(0, status_check(status_page, reg.status_bit)); + /* Write should make it out to ftrace buffers */ before = trace_bytes(); ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 3)); @@ -298,7 +316,7 @@ TEST_F(user, write_fault) { /* Register should work */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); + ASSERT_NE(0, reg.status_bit); /* Write should work normally */ ASSERT_NE(-1, writev(self->data_fd, (const struct iovec *)io, 2)); @@ -315,6 +333,11 @@ TEST_F(user, write_validator) { int loc, bytes; char data[8]; int before = 0, after = 0; + int page_size = sysconf(_SC_PAGESIZE); + char *status_page; + + status_page = mmap(NULL, page_size, PROT_READ, MAP_SHARED, + self->status_fd, 0); reg.size = sizeof(reg); reg.name_args = (__u64)"__test_event __rel_loc char[] data"; @@ -322,7 +345,12 @@ TEST_F(user, write_validator) { /* Register should work */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); + ASSERT_NE(0, reg.status_bit); + + /* MMAP should work and be zero'd */ + ASSERT_NE(MAP_FAILED, status_page); + ASSERT_NE(NULL, status_page); + ASSERT_EQ(0, status_check(status_page, reg.status_bit)); io[0].iov_base = ®.write_index; io[0].iov_len = sizeof(reg.write_index); @@ -340,6 +368,9 @@ TEST_F(user, write_validator) { self->enable_fd = open(enable_file, O_RDWR); ASSERT_NE(-1, write(self->enable_fd, "1", sizeof("1"))) + /* Event should now be enabled */ + ASSERT_NE(0, status_check(status_page, reg.status_bit)); + /* Full in-bounds write should work */ before = trace_bytes(); loc = DYN_LOC(0, bytes); diff --git a/tools/testing/selftests/user_events/perf_test.c b/tools/testing/selftests/user_events/perf_test.c index 26851d51d6bbe4f0ce1588dfd02da73659b94fae..8b4c7879d5a729c62394baefa0bfe56670866285 100644 --- a/tools/testing/selftests/user_events/perf_test.c +++ b/tools/testing/selftests/user_events/perf_test.c @@ -35,6 +35,11 @@ static long perf_event_open(struct perf_event_attr *pe, pid_t pid, return syscall(__NR_perf_event_open, pe, pid, cpu, group_fd, flags); } +static inline int status_check(char *status_page, int status_bit) +{ + return status_page[status_bit >> 3] & (1 << (status_bit & 7)); +} + static int get_id(void) { FILE *fp = fopen(id_file, "r"); @@ -120,8 +125,8 @@ TEST_F(user, perf_write) { /* Register should work */ ASSERT_EQ(0, ioctl(self->data_fd, DIAG_IOCSREG, ®)); ASSERT_EQ(0, reg.write_index); - ASSERT_NE(0, reg.status_index); - ASSERT_EQ(0, status_page[reg.status_index]); + ASSERT_NE(0, reg.status_bit); + ASSERT_EQ(0, status_check(status_page, reg.status_bit)); /* Id should be there */ id = get_id(); @@ -144,7 +149,7 @@ TEST_F(user, perf_write) { ASSERT_NE(MAP_FAILED, perf_page); /* Status should be updated */ - ASSERT_EQ(EVENT_STATUS_PERF, status_page[reg.status_index]); + ASSERT_NE(0, status_check(status_page, reg.status_bit)); event.index = reg.write_index; event.field1 = 0xc001; diff --git a/tools/testing/selftests/vm/.gitignore b/tools/testing/selftests/vm/.gitignore index 31e5eea2a9b902acbe6609ce862764f5167425d1..7b9dc2426f189166656af5f9d8a50ba876f7a8a8 100644 --- a/tools/testing/selftests/vm/.gitignore +++ b/tools/testing/selftests/vm/.gitignore @@ -30,7 +30,6 @@ map_fixed_noreplace write_to_hugetlbfs hmm-tests memfd_secret -local_config.* soft-dirty split_huge_page_test ksm_tests diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile index d9fa6a9ea5844f738ec58546c41062179bfbc028..163c2fde3cb3ae831e4eb72ed31988aba298f725 100644 --- a/tools/testing/selftests/vm/Makefile +++ b/tools/testing/selftests/vm/Makefile @@ -1,9 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # Makefile for vm selftests -LOCAL_HDRS += $(selfdir)/vm/local_config.h $(top_srcdir)/mm/gup_test.h - -include local_config.mk +LOCAL_HDRS += $(top_srcdir)/mm/gup_test.h uname_M := $(shell uname -m 2>/dev/null || echo not) MACHINE ?= $(shell echo $(uname_M) | sed -e 's/aarch64.*/arm64/' -e 's/ppc64.*/ppc64/') @@ -25,7 +23,7 @@ MACHINE ?= $(shell echo $(uname_M) | sed -e 's/aarch64.*/arm64/' -e 's/ppc64.*/p # LDLIBS. MAKEFLAGS += --no-builtin-rules -CFLAGS = -Wall -I ../../../../usr/include $(EXTRA_CFLAGS) $(KHDR_INCLUDES) +CFLAGS = -Wall -I $(top_srcdir) -I $(top_srcdir)/usr/include $(EXTRA_CFLAGS) $(KHDR_INCLUDES) LDLIBS = -lrt -lpthread TEST_GEN_FILES = compaction_test TEST_GEN_FILES += gup_test @@ -97,9 +95,11 @@ TEST_FILES += va_128TBswitch.sh include ../lib.mk +$(OUTPUT)/khugepaged: vm_util.c $(OUTPUT)/madv_populate: vm_util.c $(OUTPUT)/soft-dirty: vm_util.c $(OUTPUT)/split_huge_page_test: vm_util.c +$(OUTPUT)/userfaultfd: vm_util.c ifeq ($(MACHINE),x86_64) BINARIES_32 := $(patsubst %,$(OUTPUT)/%,$(BINARIES_32)) @@ -152,23 +152,6 @@ endif $(OUTPUT)/mlock-random-test $(OUTPUT)/memfd_secret: LDLIBS += -lcap -# HMM_EXTRA_LIBS may get set in local_config.mk, or it may be left empty. -$(OUTPUT)/hmm-tests: LDLIBS += $(HMM_EXTRA_LIBS) - $(OUTPUT)/ksm_tests: LDLIBS += -lnuma $(OUTPUT)/migration: LDLIBS += -lnuma - -local_config.mk local_config.h: check_config.sh - /bin/sh ./check_config.sh $(CC) - -EXTRA_CLEAN += local_config.mk local_config.h - -ifeq ($(HMM_EXTRA_LIBS),) -all: warn_missing_hugelibs - -warn_missing_hugelibs: - @echo ; \ - echo "Warning: missing libhugetlbfs support. Some HMM tests will be skipped." ; \ - echo -endif diff --git a/tools/testing/selftests/vm/check_config.sh b/tools/testing/selftests/vm/check_config.sh deleted file mode 100644 index 079c8a40b85df26ec97199d0c424c2a03017f410..0000000000000000000000000000000000000000 --- a/tools/testing/selftests/vm/check_config.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# -# Probe for libraries and create header files to record the results. Both C -# header files and Makefile include fragments are created. - -OUTPUT_H_FILE=local_config.h -OUTPUT_MKFILE=local_config.mk - -# libhugetlbfs -tmpname=$(mktemp) -tmpfile_c=${tmpname}.c -tmpfile_o=${tmpname}.o - -echo "#include <sys/types.h>" > $tmpfile_c -echo "#include <hugetlbfs.h>" >> $tmpfile_c -echo "int func(void) { return 0; }" >> $tmpfile_c - -CC=${1:?"Usage: $0 <compiler> # example compiler: gcc"} -$CC -c $tmpfile_c -o $tmpfile_o >/dev/null 2>&1 - -if [ -f $tmpfile_o ]; then - echo "#define LOCAL_CONFIG_HAVE_LIBHUGETLBFS 1" > $OUTPUT_H_FILE - echo "HMM_EXTRA_LIBS = -lhugetlbfs" > $OUTPUT_MKFILE -else - echo "// No libhugetlbfs support found" > $OUTPUT_H_FILE - echo "# No libhugetlbfs support found, so:" > $OUTPUT_MKFILE - echo "HMM_EXTRA_LIBS = " >> $OUTPUT_MKFILE -fi - -rm ${tmpname}.* diff --git a/tools/testing/selftests/vm/gup_test.c b/tools/testing/selftests/vm/gup_test.c index a309876d832fb778597a43bb76100e2677037fb6..e43879291dacda10841467883d52fdc19c163262 100644 --- a/tools/testing/selftests/vm/gup_test.c +++ b/tools/testing/selftests/vm/gup_test.c @@ -10,7 +10,7 @@ #include <sys/types.h> #include <pthread.h> #include <assert.h> -#include "../../../../mm/gup_test.h" +#include <mm/gup_test.h> #include "../kselftest.h" #include "util.h" diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c index 529f53b402963354e247ac7d89b3aa1291d82679..4adaad1b822f0bd4c8b7b7ab87fd5cd9c61c67ba 100644 --- a/tools/testing/selftests/vm/hmm-tests.c +++ b/tools/testing/selftests/vm/hmm-tests.c @@ -26,17 +26,13 @@ #include <sys/mman.h> #include <sys/ioctl.h> -#include "./local_config.h" -#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS -#include <hugetlbfs.h> -#endif /* * This is a private UAPI to the kernel test module so it isn't exported * in the usual include/uapi/... directory. */ -#include "../../../../lib/test_hmm_uapi.h" -#include "../../../../mm/gup_test.h" +#include <lib/test_hmm_uapi.h> +#include <mm/gup_test.h> struct hmm_buffer { void *ptr; @@ -733,7 +729,54 @@ TEST_F(hmm, anon_write_huge) hmm_buffer_free(buffer); } -#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS +/* + * Read numeric data from raw and tagged kernel status files. Used to read + * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag). + */ +static long file_read_ulong(char *file, const char *tag) +{ + int fd; + char buf[2048]; + int len; + char *p, *q; + long val; + + fd = open(file, O_RDONLY); + if (fd < 0) { + /* Error opening the file */ + return -1; + } + + len = read(fd, buf, sizeof(buf)); + close(fd); + if (len < 0) { + /* Error in reading the file */ + return -1; + } + if (len == sizeof(buf)) { + /* Error file is too large */ + return -1; + } + buf[len] = '\0'; + + /* Search for a tag if provided */ + if (tag) { + p = strstr(buf, tag); + if (!p) + return -1; /* looks like the line we want isn't there */ + p += strlen(tag); + } else + p = buf; + + val = strtol(p, &q, 0); + if (*q != ' ') { + /* Error parsing the file */ + return -1; + } + + return val; +} + /* * Write huge TLBFS page. */ @@ -742,29 +785,27 @@ TEST_F(hmm, anon_write_hugetlbfs) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; + unsigned long default_hsize; unsigned long i; int *ptr; int ret; - long pagesizes[4]; - int n, idx; - /* Skip test if we can't allocate a hugetlbfs page. */ - - n = gethugepagesizes(pagesizes, 4); - if (n <= 0) + default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); + if (default_hsize < 0 || default_hsize*1024 < default_hsize) SKIP(return, "Huge page size could not be determined"); - for (idx = 0; --n > 0; ) { - if (pagesizes[n] < pagesizes[idx]) - idx = n; - } - size = ALIGN(TWOMEG, pagesizes[idx]); + default_hsize = default_hsize*1024; /* KB to B */ + + size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); - buffer->ptr = get_hugepage_region(size, GHR_STRICT); - if (buffer->ptr == NULL) { + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, + -1, 0); + if (buffer->ptr == MAP_FAILED) { free(buffer); SKIP(return, "Huge page could not be allocated"); } @@ -788,11 +829,10 @@ TEST_F(hmm, anon_write_hugetlbfs) for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); - free_hugepage_region(buffer->ptr); + munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; hmm_buffer_free(buffer); } -#endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */ /* * Read mmap'ed file memory. @@ -1014,6 +1054,55 @@ TEST_F(hmm, migrate_fault) hmm_buffer_free(buffer); } +TEST_F(hmm, migrate_release) +{ + struct hmm_buffer *buffer; + unsigned long npages; + unsigned long size; + unsigned long i; + int *ptr; + int ret; + + npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; + ASSERT_NE(npages, 0); + size = npages << self->page_shift; + + buffer = malloc(sizeof(*buffer)); + ASSERT_NE(buffer, NULL); + + buffer->fd = -1; + buffer->size = size; + buffer->mirror = malloc(size); + ASSERT_NE(buffer->mirror, NULL); + + buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); + ASSERT_NE(buffer->ptr, MAP_FAILED); + + /* Initialize buffer in system memory. */ + for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) + ptr[i] = i; + + /* Migrate memory to device. */ + ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); + ASSERT_EQ(ret, 0); + ASSERT_EQ(buffer->cpages, npages); + + /* Check what the device read. */ + for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) + ASSERT_EQ(ptr[i], i); + + /* Release device memory. */ + ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); + ASSERT_EQ(ret, 0); + + /* Fault pages back to system memory and check them. */ + for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) + ASSERT_EQ(ptr[i], i); + + hmm_buffer_free(buffer); +} + /* * Migrate anonymous shared memory to device private memory. */ @@ -1467,7 +1556,6 @@ TEST_F(hmm2, snapshot) hmm_buffer_free(buffer); } -#ifdef LOCAL_CONFIG_HAVE_LIBHUGETLBFS /* * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that * should be mapped by a large page table entry. @@ -1477,30 +1565,30 @@ TEST_F(hmm, compound) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; + unsigned long default_hsize; int *ptr; unsigned char *m; int ret; - long pagesizes[4]; - int n, idx; unsigned long i; /* Skip test if we can't allocate a hugetlbfs page. */ - n = gethugepagesizes(pagesizes, 4); - if (n <= 0) - return; - for (idx = 0; --n > 0; ) { - if (pagesizes[n] < pagesizes[idx]) - idx = n; - } - size = ALIGN(TWOMEG, pagesizes[idx]); + default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); + if (default_hsize < 0 || default_hsize*1024 < default_hsize) + SKIP(return, "Huge page size could not be determined"); + default_hsize = default_hsize*1024; /* KB to B */ + + size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); - buffer->ptr = get_hugepage_region(size, GHR_STRICT); - if (buffer->ptr == NULL) { + buffer->ptr = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, + -1, 0); + if (buffer->ptr == MAP_FAILED) { free(buffer); return; } @@ -1539,11 +1627,10 @@ TEST_F(hmm, compound) ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | HMM_DMIRROR_PROT_PMD); - free_hugepage_region(buffer->ptr); + munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; hmm_buffer_free(buffer); } -#endif /* LOCAL_CONFIG_HAVE_LIBHUGETLBFS */ /* * Test two devices reading the same memory (double mapped). diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c index 155120b67a165d8497e9ca73b41bc819f18bd3e8..64126c8cd561298627b9ed748074068abc0812c5 100644 --- a/tools/testing/selftests/vm/khugepaged.c +++ b/tools/testing/selftests/vm/khugepaged.c @@ -1,6 +1,9 @@ #define _GNU_SOURCE +#include <ctype.h> +#include <errno.h> #include <fcntl.h> #include <limits.h> +#include <dirent.h> #include <signal.h> #include <stdio.h> #include <stdlib.h> @@ -10,10 +13,24 @@ #include <sys/mman.h> #include <sys/wait.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/sysmacros.h> +#include <sys/vfs.h> + +#include "linux/magic.h" + +#include "vm_util.h" #ifndef MADV_PAGEOUT #define MADV_PAGEOUT 21 #endif +#ifndef MADV_POPULATE_READ +#define MADV_POPULATE_READ 22 +#endif +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif #define BASE_ADDR ((void *)(1UL << 30)) static unsigned long hpage_pmd_size; @@ -22,6 +39,47 @@ static int hpage_pmd_nr; #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/" #define PID_SMAPS "/proc/self/smaps" +#define TEST_FILE "collapse_test_file" + +#define MAX_LINE_LENGTH 500 + +enum vma_type { + VMA_ANON, + VMA_FILE, + VMA_SHMEM, +}; + +struct mem_ops { + void *(*setup_area)(int nr_hpages); + void (*cleanup_area)(void *p, unsigned long size); + void (*fault)(void *p, unsigned long start, unsigned long end); + bool (*check_huge)(void *addr, int nr_hpages); + const char *name; +}; + +static struct mem_ops *file_ops; +static struct mem_ops *anon_ops; +static struct mem_ops *shmem_ops; + +struct collapse_context { + void (*collapse)(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops, bool expect); + bool enforce_pte_scan_limits; + const char *name; +}; + +static struct collapse_context *khugepaged_context; +static struct collapse_context *madvise_context; + +struct file_info { + const char *dir; + char path[PATH_MAX]; + enum vma_type type; + int fd; + char dev_queue_read_ahead_path[PATH_MAX]; +}; + +static struct file_info finfo; enum thp_enabled { THP_ALWAYS, @@ -88,18 +146,7 @@ struct settings { enum shmem_enabled shmem_enabled; bool use_zero_page; struct khugepaged_settings khugepaged; -}; - -static struct settings default_settings = { - .thp_enabled = THP_MADVISE, - .thp_defrag = THP_DEFRAG_ALWAYS, - .shmem_enabled = SHMEM_NEVER, - .use_zero_page = 0, - .khugepaged = { - .defrag = 1, - .alloc_sleep_millisecs = 10, - .scan_sleep_millisecs = 10, - }, + unsigned long read_ahead_kb; }; static struct settings saved_settings; @@ -118,6 +165,11 @@ static void fail(const char *msg) exit_status++; } +static void skip(const char *msg) +{ + printf(" \e[33m%s\e[0m\n", msg); +} + static int read_file(const char *path, char *buf, size_t buflen) { int fd; @@ -145,13 +197,19 @@ static int write_file(const char *path, const char *buf, size_t buflen) ssize_t numwritten; fd = open(path, O_WRONLY); - if (fd == -1) + if (fd == -1) { + printf("open(%s)\n", path); + exit(EXIT_FAILURE); return 0; + } numwritten = write(fd, buf, buflen - 1); close(fd); - if (numwritten < 1) + if (numwritten < 1) { + printf("write(%s)\n", buf); + exit(EXIT_FAILURE); return 0; + } return (unsigned int) numwritten; } @@ -218,20 +276,11 @@ static void write_string(const char *name, const char *val) } } -static const unsigned long read_num(const char *name) +static const unsigned long _read_num(const char *path) { - char path[PATH_MAX]; char buf[21]; - int ret; - - ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); - if (ret >= PATH_MAX) { - printf("%s: Pathname is too long\n", __func__); - exit(EXIT_FAILURE); - } - ret = read_file(path, buf, sizeof(buf)); - if (ret < 0) { + if (read_file(path, buf, sizeof(buf)) < 0) { perror("read_file(read_num)"); exit(EXIT_FAILURE); } @@ -239,10 +288,9 @@ static const unsigned long read_num(const char *name) return strtoul(buf, NULL, 10); } -static void write_num(const char *name, unsigned long num) +static const unsigned long read_num(const char *name) { char path[PATH_MAX]; - char buf[21]; int ret; ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); @@ -250,6 +298,12 @@ static void write_num(const char *name, unsigned long num) printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } + return _read_num(path); +} + +static void _write_num(const char *path, unsigned long num) +{ + char buf[21]; sprintf(buf, "%ld", num); if (!write_file(path, buf, strlen(buf) + 1)) { @@ -258,6 +312,19 @@ static void write_num(const char *name, unsigned long num) } } +static void write_num(const char *name, unsigned long num) +{ + char path[PATH_MAX]; + int ret; + + ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name); + if (ret >= PATH_MAX) { + printf("%s: Pathname is too long\n", __func__); + exit(EXIT_FAILURE); + } + _write_num(path, num); +} + static void write_settings(struct settings *settings) { struct khugepaged_settings *khugepaged = &settings->khugepaged; @@ -277,6 +344,43 @@ static void write_settings(struct settings *settings) write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap); write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared); write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan); + + if (file_ops && finfo.type == VMA_FILE) + _write_num(finfo.dev_queue_read_ahead_path, + settings->read_ahead_kb); +} + +#define MAX_SETTINGS_DEPTH 4 +static struct settings settings_stack[MAX_SETTINGS_DEPTH]; +static int settings_index; + +static struct settings *current_settings(void) +{ + if (!settings_index) { + printf("Fail: No settings set"); + exit(EXIT_FAILURE); + } + return settings_stack + settings_index - 1; +} + +static void push_settings(struct settings *settings) +{ + if (settings_index >= MAX_SETTINGS_DEPTH) { + printf("Fail: Settings stack exceeded"); + exit(EXIT_FAILURE); + } + settings_stack[settings_index++] = *settings; + write_settings(current_settings()); +} + +static void pop_settings(void) +{ + if (settings_index <= 0) { + printf("Fail: Settings stack empty"); + exit(EXIT_FAILURE); + } + --settings_index; + write_settings(current_settings()); } static void restore_settings(int sig) @@ -314,6 +418,10 @@ static void save_settings(void) .max_ptes_shared = read_num("khugepaged/max_ptes_shared"), .pages_to_scan = read_num("khugepaged/pages_to_scan"), }; + if (file_ops && finfo.type == VMA_FILE) + saved_settings.read_ahead_kb = + _read_num(finfo.dev_queue_read_ahead_path); + success("OK"); signal(SIGTERM, restore_settings); @@ -322,72 +430,90 @@ static void save_settings(void) signal(SIGQUIT, restore_settings); } -static void adjust_settings(void) +static void get_finfo(const char *dir) { + struct stat path_stat; + struct statfs fs; + char buf[1 << 10]; + char path[PATH_MAX]; + char *str, *end; - printf("Adjust settings..."); - write_settings(&default_settings); - success("OK"); -} - -#define MAX_LINE_LENGTH 500 - -static bool check_for_pattern(FILE *fp, char *pattern, char *buf) -{ - while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) { - if (!strncmp(buf, pattern, strlen(pattern))) - return true; + finfo.dir = dir; + stat(finfo.dir, &path_stat); + if (!S_ISDIR(path_stat.st_mode)) { + printf("%s: Not a directory (%s)\n", __func__, finfo.dir); + exit(EXIT_FAILURE); } - return false; -} - -static bool check_huge(void *addr) -{ - bool thp = false; - int ret; - FILE *fp; - char buffer[MAX_LINE_LENGTH]; - char addr_pattern[MAX_LINE_LENGTH]; - - ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-", - (unsigned long) addr); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); + if (snprintf(finfo.path, sizeof(finfo.path), "%s/" TEST_FILE, + finfo.dir) >= sizeof(finfo.path)) { + printf("%s: Pathname is too long\n", __func__); exit(EXIT_FAILURE); } - - - fp = fopen(PID_SMAPS, "r"); - if (!fp) { - printf("%s: Failed to open file %s\n", __func__, PID_SMAPS); + if (statfs(finfo.dir, &fs)) { + perror("statfs()"); exit(EXIT_FAILURE); } - if (!check_for_pattern(fp, addr_pattern, buffer)) - goto err_out; - - ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB", - hpage_pmd_size >> 10); - if (ret >= MAX_LINE_LENGTH) { - printf("%s: Pattern is too long\n", __func__); + finfo.type = fs.f_type == TMPFS_MAGIC ? VMA_SHMEM : VMA_FILE; + if (finfo.type == VMA_SHMEM) + return; + + /* Find owning device's queue/read_ahead_kb control */ + if (snprintf(path, sizeof(path), "/sys/dev/block/%d:%d/uevent", + major(path_stat.st_dev), minor(path_stat.st_dev)) + >= sizeof(path)) { + printf("%s: Pathname is too long\n", __func__); + exit(EXIT_FAILURE); + } + if (read_file(path, buf, sizeof(buf)) < 0) { + perror("read_file(read_num)"); + exit(EXIT_FAILURE); + } + if (strstr(buf, "DEVTYPE=disk")) { + /* Found it */ + if (snprintf(finfo.dev_queue_read_ahead_path, + sizeof(finfo.dev_queue_read_ahead_path), + "/sys/dev/block/%d:%d/queue/read_ahead_kb", + major(path_stat.st_dev), minor(path_stat.st_dev)) + >= sizeof(finfo.dev_queue_read_ahead_path)) { + printf("%s: Pathname is too long\n", __func__); + exit(EXIT_FAILURE); + } + return; + } + if (!strstr(buf, "DEVTYPE=partition")) { + printf("%s: Unknown device type: %s\n", __func__, path); exit(EXIT_FAILURE); } /* - * Fetch the AnonHugePages: in the same block and check whether it got - * the expected number of hugeepages next. + * Partition of block device - need to find actual device. + * Using naming convention that devnameN is partition of + * device devname. */ - if (!check_for_pattern(fp, "AnonHugePages:", buffer)) - goto err_out; - - if (strncmp(buffer, addr_pattern, strlen(addr_pattern))) - goto err_out; - - thp = true; -err_out: - fclose(fp); - return thp; + str = strstr(buf, "DEVNAME="); + if (!str) { + printf("%s: Could not read: %s", __func__, path); + exit(EXIT_FAILURE); + } + str += 8; + end = str; + while (*end) { + if (isdigit(*end)) { + *end = '\0'; + if (snprintf(finfo.dev_queue_read_ahead_path, + sizeof(finfo.dev_queue_read_ahead_path), + "/sys/block/%s/queue/read_ahead_kb", + str) >= sizeof(finfo.dev_queue_read_ahead_path)) { + printf("%s: Pathname is too long\n", __func__); + exit(EXIT_FAILURE); + } + return; + } + ++end; + } + printf("%s: Could not read: %s\n", __func__, path); + exit(EXIT_FAILURE); } - static bool check_swap(void *addr, unsigned long size) { bool swap = false; @@ -409,7 +535,7 @@ static bool check_swap(void *addr, unsigned long size) printf("%s: Failed to open file %s\n", __func__, PID_SMAPS); exit(EXIT_FAILURE); } - if (!check_for_pattern(fp, addr_pattern, buffer)) + if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) goto err_out; ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB", @@ -422,7 +548,7 @@ static bool check_swap(void *addr, unsigned long size) * Fetch the Swap: in the same block and check whether it got * the expected number of hugeepages next. */ - if (!check_for_pattern(fp, "Swap:", buffer)) + if (!check_for_pattern(fp, "Swap:", buffer, sizeof(buffer))) goto err_out; if (strncmp(buffer, addr_pattern, strlen(addr_pattern))) @@ -434,12 +560,12 @@ err_out: return swap; } -static void *alloc_mapping(void) +static void *alloc_mapping(int nr) { void *p; - p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + p = mmap(BASE_ADDR, nr * hpage_pmd_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); if (p != BASE_ADDR) { printf("Failed to allocate VMA at %p\n", BASE_ADDR); exit(EXIT_FAILURE); @@ -456,6 +582,60 @@ static void fill_memory(int *p, unsigned long start, unsigned long end) p[i * page_size / sizeof(*p)] = i + 0xdead0000; } +/* + * MADV_COLLAPSE is a best-effort request and may fail if an internal + * resource is temporarily unavailable, in which case it will set errno to + * EAGAIN. In such a case, immediately reattempt the operation one more + * time. + */ +static int madvise_collapse_retry(void *p, unsigned long size) +{ + bool retry = true; + int ret; + +retry: + ret = madvise(p, size, MADV_COLLAPSE); + if (ret && errno == EAGAIN && retry) { + retry = false; + goto retry; + } + return ret; +} + +/* + * Returns pmd-mapped hugepage in VMA marked VM_HUGEPAGE, filled with + * validate_memory()'able contents. + */ +static void *alloc_hpage(struct mem_ops *ops) +{ + void *p = ops->setup_area(1); + + ops->fault(p, 0, hpage_pmd_size); + + /* + * VMA should be neither VM_HUGEPAGE nor VM_NOHUGEPAGE. + * The latter is ineligible for collapse by MADV_COLLAPSE + * while the former might cause MADV_COLLAPSE to race with + * khugepaged on low-load system (like a test machine), which + * would cause MADV_COLLAPSE to fail with EAGAIN. + */ + printf("Allocate huge page..."); + if (madvise_collapse_retry(p, hpage_pmd_size)) { + perror("madvise(MADV_COLLAPSE)"); + exit(EXIT_FAILURE); + } + if (!ops->check_huge(p, 1)) { + perror("madvise(MADV_COLLAPSE)"); + exit(EXIT_FAILURE); + } + if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE)) { + perror("madvise(MADV_HUGEPAGE)"); + exit(EXIT_FAILURE); + } + success("OK"); + return p; +} + static void validate_memory(int *p, unsigned long start, unsigned long end) { int i; @@ -469,26 +649,216 @@ static void validate_memory(int *p, unsigned long start, unsigned long end) } } +static void *anon_setup_area(int nr_hpages) +{ + return alloc_mapping(nr_hpages); +} + +static void anon_cleanup_area(void *p, unsigned long size) +{ + munmap(p, size); +} + +static void anon_fault(void *p, unsigned long start, unsigned long end) +{ + fill_memory(p, start, end); +} + +static bool anon_check_huge(void *addr, int nr_hpages) +{ + return check_huge_anon(addr, nr_hpages, hpage_pmd_size); +} + +static void *file_setup_area(int nr_hpages) +{ + int fd; + void *p; + unsigned long size; + + unlink(finfo.path); /* Cleanup from previous failed tests */ + printf("Creating %s for collapse%s...", finfo.path, + finfo.type == VMA_SHMEM ? " (tmpfs)" : ""); + fd = open(finfo.path, O_DSYNC | O_CREAT | O_RDWR | O_TRUNC | O_EXCL, + 777); + if (fd < 0) { + perror("open()"); + exit(EXIT_FAILURE); + } + + size = nr_hpages * hpage_pmd_size; + p = alloc_mapping(nr_hpages); + fill_memory(p, 0, size); + write(fd, p, size); + close(fd); + munmap(p, size); + success("OK"); + + printf("Opening %s read only for collapse...", finfo.path); + finfo.fd = open(finfo.path, O_RDONLY, 777); + if (finfo.fd < 0) { + perror("open()"); + exit(EXIT_FAILURE); + } + p = mmap(BASE_ADDR, size, PROT_READ | PROT_EXEC, + MAP_PRIVATE, finfo.fd, 0); + if (p == MAP_FAILED || p != BASE_ADDR) { + perror("mmap()"); + exit(EXIT_FAILURE); + } + + /* Drop page cache */ + write_file("/proc/sys/vm/drop_caches", "3", 2); + success("OK"); + return p; +} + +static void file_cleanup_area(void *p, unsigned long size) +{ + munmap(p, size); + close(finfo.fd); + unlink(finfo.path); +} + +static void file_fault(void *p, unsigned long start, unsigned long end) +{ + if (madvise(((char *)p) + start, end - start, MADV_POPULATE_READ)) { + perror("madvise(MADV_POPULATE_READ"); + exit(EXIT_FAILURE); + } +} + +static bool file_check_huge(void *addr, int nr_hpages) +{ + switch (finfo.type) { + case VMA_FILE: + return check_huge_file(addr, nr_hpages, hpage_pmd_size); + case VMA_SHMEM: + return check_huge_shmem(addr, nr_hpages, hpage_pmd_size); + default: + exit(EXIT_FAILURE); + return false; + } +} + +static void *shmem_setup_area(int nr_hpages) +{ + void *p; + unsigned long size = nr_hpages * hpage_pmd_size; + + finfo.fd = memfd_create("khugepaged-selftest-collapse-shmem", 0); + if (finfo.fd < 0) { + perror("memfd_create()"); + exit(EXIT_FAILURE); + } + if (ftruncate(finfo.fd, size)) { + perror("ftruncate()"); + exit(EXIT_FAILURE); + } + p = mmap(BASE_ADDR, size, PROT_READ | PROT_WRITE, MAP_SHARED, finfo.fd, + 0); + if (p != BASE_ADDR) { + perror("mmap()"); + exit(EXIT_FAILURE); + } + return p; +} + +static void shmem_cleanup_area(void *p, unsigned long size) +{ + munmap(p, size); + close(finfo.fd); +} + +static bool shmem_check_huge(void *addr, int nr_hpages) +{ + return check_huge_shmem(addr, nr_hpages, hpage_pmd_size); +} + +static struct mem_ops __anon_ops = { + .setup_area = &anon_setup_area, + .cleanup_area = &anon_cleanup_area, + .fault = &anon_fault, + .check_huge = &anon_check_huge, + .name = "anon", +}; + +static struct mem_ops __file_ops = { + .setup_area = &file_setup_area, + .cleanup_area = &file_cleanup_area, + .fault = &file_fault, + .check_huge = &file_check_huge, + .name = "file", +}; + +static struct mem_ops __shmem_ops = { + .setup_area = &shmem_setup_area, + .cleanup_area = &shmem_cleanup_area, + .fault = &anon_fault, + .check_huge = &shmem_check_huge, + .name = "shmem", +}; + +static void __madvise_collapse(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops, bool expect) +{ + int ret; + struct settings settings = *current_settings(); + + printf("%s...", msg); + + /* + * Prevent khugepaged interference and tests that MADV_COLLAPSE + * ignores /sys/kernel/mm/transparent_hugepage/enabled + */ + settings.thp_enabled = THP_NEVER; + settings.shmem_enabled = SHMEM_NEVER; + push_settings(&settings); + + /* Clear VM_NOHUGEPAGE */ + madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); + ret = madvise_collapse_retry(p, nr_hpages * hpage_pmd_size); + if (((bool)ret) == expect) + fail("Fail: Bad return value"); + else if (!ops->check_huge(p, expect ? nr_hpages : 0)) + fail("Fail: check_huge()"); + else + success("OK"); + + pop_settings(); +} + +static void madvise_collapse(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops, bool expect) +{ + /* Sanity check */ + if (!ops->check_huge(p, 0)) { + printf("Unexpected huge page\n"); + exit(EXIT_FAILURE); + } + __madvise_collapse(msg, p, nr_hpages, ops, expect); +} + #define TICK 500000 -static bool wait_for_scan(const char *msg, char *p) +static bool wait_for_scan(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops) { int full_scans; int timeout = 6; /* 3 seconds */ /* Sanity check */ - if (check_huge(p)) { + if (!ops->check_huge(p, 0)) { printf("Unexpected huge page\n"); exit(EXIT_FAILURE); } - madvise(p, hpage_pmd_size, MADV_HUGEPAGE); + madvise(p, nr_hpages * hpage_pmd_size, MADV_HUGEPAGE); /* Wait until the second full_scan completed */ full_scans = read_num("khugepaged/full_scans") + 2; printf("%s...", msg); while (timeout--) { - if (check_huge(p)) + if (ops->check_huge(p, nr_hpages)) break; if (read_num("khugepaged/full_scans") >= full_scans) break; @@ -496,122 +866,155 @@ static bool wait_for_scan(const char *msg, char *p) usleep(TICK); } - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); + madvise(p, nr_hpages * hpage_pmd_size, MADV_NOHUGEPAGE); return timeout == -1; } +static void khugepaged_collapse(const char *msg, char *p, int nr_hpages, + struct mem_ops *ops, bool expect) +{ + if (wait_for_scan(msg, p, nr_hpages, ops)) { + if (expect) + fail("Timeout"); + else + success("OK"); + return; + } + + /* + * For file and shmem memory, khugepaged only retracts pte entries after + * putting the new hugepage in the page cache. The hugepage must be + * subsequently refaulted to install the pmd mapping for the mm. + */ + if (ops != &__anon_ops) + ops->fault(p, 0, nr_hpages * hpage_pmd_size); + + if (ops->check_huge(p, expect ? nr_hpages : 0)) + success("OK"); + else + fail("Fail"); +} + +static struct collapse_context __khugepaged_context = { + .collapse = &khugepaged_collapse, + .enforce_pte_scan_limits = true, + .name = "khugepaged", +}; + +static struct collapse_context __madvise_context = { + .collapse = &madvise_collapse, + .enforce_pte_scan_limits = false, + .name = "madvise", +}; + +static bool is_tmpfs(struct mem_ops *ops) +{ + return ops == &__file_ops && finfo.type == VMA_SHMEM; +} + static void alloc_at_fault(void) { - struct settings settings = default_settings; + struct settings settings = *current_settings(); char *p; settings.thp_enabled = THP_ALWAYS; - write_settings(&settings); + push_settings(&settings); - p = alloc_mapping(); + p = alloc_mapping(1); *p = 1; printf("Allocate huge page on fault..."); - if (check_huge(p)) + if (check_huge_anon(p, 1, hpage_pmd_size)) success("OK"); else fail("Fail"); - write_settings(&default_settings); + pop_settings(); madvise(p, page_size, MADV_DONTNEED); printf("Split huge PMD on MADV_DONTNEED..."); - if (!check_huge(p)) + if (check_huge_anon(p, 0, hpage_pmd_size)) success("OK"); else fail("Fail"); munmap(p, hpage_pmd_size); } -static void collapse_full(void) +static void collapse_full(struct collapse_context *c, struct mem_ops *ops) { void *p; - - p = alloc_mapping(); - fill_memory(p, 0, hpage_pmd_size); - if (wait_for_scan("Collapse fully populated PTE table", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); - validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + int nr_hpages = 4; + unsigned long size = nr_hpages * hpage_pmd_size; + + p = ops->setup_area(nr_hpages); + ops->fault(p, 0, size); + c->collapse("Collapse multiple fully populated PTE table", p, nr_hpages, + ops, true); + validate_memory(p, 0, size); + ops->cleanup_area(p, size); } -static void collapse_empty(void) +static void collapse_empty(struct collapse_context *c, struct mem_ops *ops) { void *p; - p = alloc_mapping(); - if (wait_for_scan("Do not collapse empty PTE table", p)) - fail("Timeout"); - else if (check_huge(p)) - fail("Fail"); - else - success("OK"); - munmap(p, hpage_pmd_size); + p = ops->setup_area(1); + c->collapse("Do not collapse empty PTE table", p, 1, ops, false); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_single_pte_entry(void) +static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops) { void *p; - p = alloc_mapping(); - fill_memory(p, 0, page_size); - if (wait_for_scan("Collapse PTE table with single PTE entry present", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); - validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); + p = ops->setup_area(1); + ops->fault(p, 0, page_size); + c->collapse("Collapse PTE table with single PTE entry present", p, + 1, ops, true); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_max_ptes_none(void) +static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops) { int max_ptes_none = hpage_pmd_nr / 2; - struct settings settings = default_settings; + struct settings settings = *current_settings(); void *p; settings.khugepaged.max_ptes_none = max_ptes_none; - write_settings(&settings); + push_settings(&settings); - p = alloc_mapping(); + p = ops->setup_area(1); - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); - if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p)) - fail("Timeout"); - else if (check_huge(p)) - fail("Fail"); - else - success("OK"); - validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); + if (is_tmpfs(ops)) { + /* shmem pages always in the page cache */ + printf("tmpfs..."); + skip("Skip"); + goto skip; + } - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); - if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); - validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); + c->collapse("Maybe collapse with max_ptes_none exceeded", p, 1, + ops, !c->enforce_pte_scan_limits); + validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size); - munmap(p, hpage_pmd_size); - write_settings(&default_settings); + if (c->enforce_pte_scan_limits) { + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size); + c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops, + true); + validate_memory(p, 0, + (hpage_pmd_nr - max_ptes_none) * page_size); + } +skip: + ops->cleanup_area(p, hpage_pmd_size); + pop_settings(); } -static void collapse_swapin_single_pte(void) +static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops) { void *p; - p = alloc_mapping(); - fill_memory(p, 0, hpage_pmd_size); + + p = ops->setup_area(1); + ops->fault(p, 0, hpage_pmd_size); printf("Swapout one page..."); if (madvise(p, page_size, MADV_PAGEOUT)) { @@ -625,25 +1028,21 @@ static void collapse_swapin_single_pte(void) goto out; } - if (wait_for_scan("Collapse with swapping in single PTE entry", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Collapse with swapping in single PTE entry", p, 1, ops, + true); validate_memory(p, 0, hpage_pmd_size); out: - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_max_ptes_swap(void) +static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops) { int max_ptes_swap = read_num("khugepaged/max_ptes_swap"); void *p; - p = alloc_mapping(); + p = ops->setup_area(1); + ops->fault(p, 0, hpage_pmd_size); - fill_memory(p, 0, hpage_pmd_size); printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr); if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) { perror("madvise(MADV_PAGEOUT)"); @@ -656,115 +1055,93 @@ static void collapse_max_ptes_swap(void) goto out; } - if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p)) - fail("Timeout"); - else if (check_huge(p)) - fail("Fail"); - else - success("OK"); + c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops, + !c->enforce_pte_scan_limits); validate_memory(p, 0, hpage_pmd_size); - fill_memory(p, 0, hpage_pmd_size); - printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr); - if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { - perror("madvise(MADV_PAGEOUT)"); - exit(EXIT_FAILURE); - } - if (check_swap(p, max_ptes_swap * page_size)) { - success("OK"); - } else { - fail("Fail"); - goto out; - } + if (c->enforce_pte_scan_limits) { + ops->fault(p, 0, hpage_pmd_size); + printf("Swapout %d of %d pages...", max_ptes_swap, + hpage_pmd_nr); + if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) { + perror("madvise(MADV_PAGEOUT)"); + exit(EXIT_FAILURE); + } + if (check_swap(p, max_ptes_swap * page_size)) { + success("OK"); + } else { + fail("Fail"); + goto out; + } - if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); - validate_memory(p, 0, hpage_pmd_size); + c->collapse("Collapse with max_ptes_swap pages swapped out", p, + 1, ops, true); + validate_memory(p, 0, hpage_pmd_size); + } out: - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_single_pte_entry_compound(void) +static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops) { void *p; - p = alloc_mapping(); + p = alloc_hpage(ops); - printf("Allocate huge page..."); - madvise(p, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(p, 0, hpage_pmd_size); - if (check_huge(p)) - success("OK"); - else - fail("Fail"); - madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); + if (is_tmpfs(ops)) { + /* MADV_DONTNEED won't evict tmpfs pages */ + printf("tmpfs..."); + skip("Skip"); + goto skip; + } + madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); printf("Split huge page leaving single PTE mapping compound page..."); madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED); - if (!check_huge(p)) + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Collapse PTE table with single PTE mapping compound page", + p, 1, ops, true); validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); +skip: + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_full_of_compound(void) +static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops) { void *p; - p = alloc_mapping(); - - printf("Allocate huge page..."); - madvise(p, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(p, 0, hpage_pmd_size); - if (check_huge(p)) - success("OK"); - else - fail("Fail"); - + p = alloc_hpage(ops); printf("Split huge page leaving single PTE page table full of compound pages..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (!check_huge(p)) + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - if (wait_for_scan("Collapse PTE table full of compound pages", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Collapse PTE table full of compound pages", p, 1, ops, + true); validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_compound_extreme(void) +static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops) { void *p; int i; - p = alloc_mapping(); + p = ops->setup_area(1); for (i = 0; i < hpage_pmd_nr; i++) { printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...", i + 1, hpage_pmd_nr); madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(BASE_ADDR, 0, hpage_pmd_size); - if (!check_huge(BASE_ADDR)) { + ops->fault(BASE_ADDR, 0, hpage_pmd_size); + if (!ops->check_huge(BASE_ADDR, 1)) { printf("Failed to allocate huge page\n"); exit(EXIT_FAILURE); } @@ -791,34 +1168,30 @@ static void collapse_compound_extreme(void) } } - munmap(BASE_ADDR, hpage_pmd_size); - fill_memory(p, 0, hpage_pmd_size); - if (!check_huge(p)) + ops->cleanup_area(BASE_ADDR, hpage_pmd_size); + ops->fault(p, 0, hpage_pmd_size); + if (!ops->check_huge(p, 1)) success("OK"); else fail("Fail"); - if (wait_for_scan("Collapse PTE table full of different compound pages", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Collapse PTE table full of different compound pages", p, 1, + ops, true); validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_fork(void) +static void collapse_fork(struct collapse_context *c, struct mem_ops *ops) { int wstatus; void *p; - p = alloc_mapping(); + p = ops->setup_area(1); printf("Allocate small page..."); - fill_memory(p, 0, page_size); - if (!check_huge(p)) + ops->fault(p, 0, page_size); + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); @@ -829,22 +1202,17 @@ static void collapse_fork(void) skip_settings_restore = true; exit_status = 0; - if (!check_huge(p)) + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - fill_memory(p, page_size, 2 * page_size); - - if (wait_for_scan("Collapse PTE table with single page shared with parent process", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + ops->fault(p, page_size, 2 * page_size); + c->collapse("Collapse PTE table with single page shared with parent process", + p, 1, ops, true); validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); exit(exit_status); } @@ -852,36 +1220,27 @@ static void collapse_fork(void) exit_status += WEXITSTATUS(wstatus); printf("Check if parent still has small page..."); - if (!check_huge(p)) + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); validate_memory(p, 0, page_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_fork_compound(void) +static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops) { int wstatus; void *p; - p = alloc_mapping(); - - printf("Allocate huge page..."); - madvise(p, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(p, 0, hpage_pmd_size); - if (check_huge(p)) - success("OK"); - else - fail("Fail"); - + p = alloc_hpage(ops); printf("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ skip_settings_restore = true; exit_status = 0; - if (check_huge(p)) + if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); @@ -889,24 +1248,20 @@ static void collapse_fork_compound(void) printf("Split huge page PMD in child process..."); madvise(p, page_size, MADV_NOHUGEPAGE); madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - if (!check_huge(p)) + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - fill_memory(p, 0, page_size); + ops->fault(p, 0, page_size); write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1); - if (wait_for_scan("Collapse PTE table full of compound pages in child", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Collapse PTE table full of compound pages in child", + p, 1, ops, true); write_num("khugepaged/max_ptes_shared", - default_settings.khugepaged.max_ptes_shared); + current_settings()->khugepaged.max_ptes_shared); validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); exit(exit_status); } @@ -914,74 +1269,59 @@ static void collapse_fork_compound(void) exit_status += WEXITSTATUS(wstatus); printf("Check if parent still has huge page..."); - if (check_huge(p)) + if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -static void collapse_max_ptes_shared() +static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops) { int max_ptes_shared = read_num("khugepaged/max_ptes_shared"); int wstatus; void *p; - p = alloc_mapping(); - - printf("Allocate huge page..."); - madvise(p, hpage_pmd_size, MADV_HUGEPAGE); - fill_memory(p, 0, hpage_pmd_size); - if (check_huge(p)) - success("OK"); - else - fail("Fail"); - + p = alloc_hpage(ops); printf("Share huge page over fork()..."); if (!fork()) { /* Do not touch settings on child exit */ skip_settings_restore = true; exit_status = 0; - if (check_huge(p)) + if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); printf("Trigger CoW on page %d of %d...", hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr); - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); - if (!check_huge(p)) - success("OK"); - else - fail("Fail"); - - if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p)) - fail("Timeout"); - else if (!check_huge(p)) - success("OK"); - else - fail("Fail"); - - printf("Trigger CoW on page %d of %d...", - hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); - fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size); - if (!check_huge(p)) + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size); + if (ops->check_huge(p, 0)) success("OK"); else fail("Fail"); - - if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p)) - fail("Timeout"); - else if (check_huge(p)) - success("OK"); - else - fail("Fail"); + c->collapse("Maybe collapse with max_ptes_shared exceeded", p, + 1, ops, !c->enforce_pte_scan_limits); + + if (c->enforce_pte_scan_limits) { + printf("Trigger CoW on page %d of %d...", + hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr); + ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) * + page_size); + if (ops->check_huge(p, 0)) + success("OK"); + else + fail("Fail"); + + c->collapse("Collapse with max_ptes_shared PTEs shared", + p, 1, ops, true); + } validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); exit(exit_status); } @@ -989,20 +1329,153 @@ static void collapse_max_ptes_shared() exit_status += WEXITSTATUS(wstatus); printf("Check if parent still has huge page..."); - if (check_huge(p)) + if (ops->check_huge(p, 1)) success("OK"); else fail("Fail"); validate_memory(p, 0, hpage_pmd_size); - munmap(p, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); } -int main(void) +static void madvise_collapse_existing_thps(struct collapse_context *c, + struct mem_ops *ops) { + void *p; + + p = ops->setup_area(1); + ops->fault(p, 0, hpage_pmd_size); + c->collapse("Collapse fully populated PTE table...", p, 1, ops, true); + validate_memory(p, 0, hpage_pmd_size); + + /* c->collapse() will find a hugepage and complain - call directly. */ + __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true); + validate_memory(p, 0, hpage_pmd_size); + ops->cleanup_area(p, hpage_pmd_size); +} + +/* + * Test race with khugepaged where page tables have been retracted and + * pmd cleared. + */ +static void madvise_retracted_page_tables(struct collapse_context *c, + struct mem_ops *ops) +{ + void *p; + int nr_hpages = 1; + unsigned long size = nr_hpages * hpage_pmd_size; + + p = ops->setup_area(nr_hpages); + ops->fault(p, 0, size); + + /* Let khugepaged collapse and leave pmd cleared */ + if (wait_for_scan("Collapse and leave PMD cleared", p, nr_hpages, + ops)) { + fail("Timeout"); + return; + } + success("OK"); + c->collapse("Install huge PMD from page cache", p, nr_hpages, ops, + true); + validate_memory(p, 0, size); + ops->cleanup_area(p, size); +} + +static void usage(void) +{ + fprintf(stderr, "\nUsage: ./khugepaged <test type> [dir]\n\n"); + fprintf(stderr, "\t<test type>\t: <context>:<mem_type>\n"); + fprintf(stderr, "\t<context>\t: [all|khugepaged|madvise]\n"); + fprintf(stderr, "\t<mem_type>\t: [all|anon|file|shmem]\n"); + fprintf(stderr, "\n\t\"file,all\" mem_type requires [dir] argument\n"); + fprintf(stderr, "\n\t\"file,all\" mem_type requires kernel built with\n"); + fprintf(stderr, "\tCONFIG_READ_ONLY_THP_FOR_FS=y\n"); + fprintf(stderr, "\n\tif [dir] is a (sub)directory of a tmpfs mount, tmpfs must be\n"); + fprintf(stderr, "\tmounted with huge=madvise option for khugepaged tests to work\n"); + exit(1); +} + +static void parse_test_type(int argc, const char **argv) +{ + char *buf; + const char *token; + + if (argc == 1) { + /* Backwards compatibility */ + khugepaged_context = &__khugepaged_context; + madvise_context = &__madvise_context; + anon_ops = &__anon_ops; + return; + } + + buf = strdup(argv[1]); + token = strsep(&buf, ":"); + + if (!strcmp(token, "all")) { + khugepaged_context = &__khugepaged_context; + madvise_context = &__madvise_context; + } else if (!strcmp(token, "khugepaged")) { + khugepaged_context = &__khugepaged_context; + } else if (!strcmp(token, "madvise")) { + madvise_context = &__madvise_context; + } else { + usage(); + } + + if (!buf) + usage(); + + if (!strcmp(buf, "all")) { + file_ops = &__file_ops; + anon_ops = &__anon_ops; + shmem_ops = &__shmem_ops; + } else if (!strcmp(buf, "anon")) { + anon_ops = &__anon_ops; + } else if (!strcmp(buf, "file")) { + file_ops = &__file_ops; + } else if (!strcmp(buf, "shmem")) { + shmem_ops = &__shmem_ops; + } else { + usage(); + } + + if (!file_ops) + return; + + if (argc != 3) + usage(); +} + +int main(int argc, const char **argv) +{ + struct settings default_settings = { + .thp_enabled = THP_MADVISE, + .thp_defrag = THP_DEFRAG_ALWAYS, + .shmem_enabled = SHMEM_ADVISE, + .use_zero_page = 0, + .khugepaged = { + .defrag = 1, + .alloc_sleep_millisecs = 10, + .scan_sleep_millisecs = 10, + }, + /* + * When testing file-backed memory, the collapse path + * looks at how many pages are found in the page cache, not + * what pages are mapped. Disable read ahead optimization so + * pages don't find their way into the page cache unless + * we mem_ops->fault() them in. + */ + .read_ahead_kb = 0, + }; + + parse_test_type(argc, argv); + + if (file_ops) + get_finfo(argv[2]); + setbuf(stdout, NULL); page_size = getpagesize(); - hpage_pmd_size = read_num("hpage_pmd_size"); + hpage_pmd_size = read_pmd_pagesize(); hpage_pmd_nr = hpage_pmd_size / page_size; default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1; @@ -1011,21 +1484,75 @@ int main(void) default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8; save_settings(); - adjust_settings(); + push_settings(&default_settings); alloc_at_fault(); - collapse_full(); - collapse_empty(); - collapse_single_pte_entry(); - collapse_max_ptes_none(); - collapse_swapin_single_pte(); - collapse_max_ptes_swap(); - collapse_single_pte_entry_compound(); - collapse_full_of_compound(); - collapse_compound_extreme(); - collapse_fork(); - collapse_fork_compound(); - collapse_max_ptes_shared(); + +#define TEST(t, c, o) do { \ + if (c && o) { \ + printf("\nRun test: " #t " (%s:%s)\n", c->name, o->name); \ + t(c, o); \ + } \ + } while (0) + + TEST(collapse_full, khugepaged_context, anon_ops); + TEST(collapse_full, khugepaged_context, file_ops); + TEST(collapse_full, khugepaged_context, shmem_ops); + TEST(collapse_full, madvise_context, anon_ops); + TEST(collapse_full, madvise_context, file_ops); + TEST(collapse_full, madvise_context, shmem_ops); + + TEST(collapse_empty, khugepaged_context, anon_ops); + TEST(collapse_empty, madvise_context, anon_ops); + + TEST(collapse_single_pte_entry, khugepaged_context, anon_ops); + TEST(collapse_single_pte_entry, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry, khugepaged_context, shmem_ops); + TEST(collapse_single_pte_entry, madvise_context, anon_ops); + TEST(collapse_single_pte_entry, madvise_context, file_ops); + TEST(collapse_single_pte_entry, madvise_context, shmem_ops); + + TEST(collapse_max_ptes_none, khugepaged_context, anon_ops); + TEST(collapse_max_ptes_none, khugepaged_context, file_ops); + TEST(collapse_max_ptes_none, madvise_context, anon_ops); + TEST(collapse_max_ptes_none, madvise_context, file_ops); + + TEST(collapse_single_pte_entry_compound, khugepaged_context, anon_ops); + TEST(collapse_single_pte_entry_compound, khugepaged_context, file_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, anon_ops); + TEST(collapse_single_pte_entry_compound, madvise_context, file_ops); + + TEST(collapse_full_of_compound, khugepaged_context, anon_ops); + TEST(collapse_full_of_compound, khugepaged_context, file_ops); + TEST(collapse_full_of_compound, khugepaged_context, shmem_ops); + TEST(collapse_full_of_compound, madvise_context, anon_ops); + TEST(collapse_full_of_compound, madvise_context, file_ops); + TEST(collapse_full_of_compound, madvise_context, shmem_ops); + + TEST(collapse_compound_extreme, khugepaged_context, anon_ops); + TEST(collapse_compound_extreme, madvise_context, anon_ops); + + TEST(collapse_swapin_single_pte, khugepaged_context, anon_ops); + TEST(collapse_swapin_single_pte, madvise_context, anon_ops); + + TEST(collapse_max_ptes_swap, khugepaged_context, anon_ops); + TEST(collapse_max_ptes_swap, madvise_context, anon_ops); + + TEST(collapse_fork, khugepaged_context, anon_ops); + TEST(collapse_fork, madvise_context, anon_ops); + + TEST(collapse_fork_compound, khugepaged_context, anon_ops); + TEST(collapse_fork_compound, madvise_context, anon_ops); + + TEST(collapse_max_ptes_shared, khugepaged_context, anon_ops); + TEST(collapse_max_ptes_shared, madvise_context, anon_ops); + + TEST(madvise_collapse_existing_thps, madvise_context, anon_ops); + TEST(madvise_collapse_existing_thps, madvise_context, file_ops); + TEST(madvise_collapse_existing_thps, madvise_context, shmem_ops); + + TEST(madvise_retracted_page_tables, madvise_context, file_ops); + TEST(madvise_retracted_page_tables, madvise_context, shmem_ops); restore_settings(0); } diff --git a/tools/testing/selftests/vm/ksm_tests.c b/tools/testing/selftests/vm/ksm_tests.c index f5e4e0bbd081534b8e17bc3e936e198970e78c16..0d85be2350fa36a6e982d14edf6d2c0740413a10 100644 --- a/tools/testing/selftests/vm/ksm_tests.c +++ b/tools/testing/selftests/vm/ksm_tests.c @@ -11,7 +11,7 @@ #include <err.h> #include "../kselftest.h" -#include "../../../../include/vdso/time64.h" +#include <include/vdso/time64.h> #include "util.h" #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" diff --git a/tools/testing/selftests/vm/mremap_test.c b/tools/testing/selftests/vm/mremap_test.c index db0270127aeb041ab4e53f5f4dcdf7c372a22422..9496346973d44a6debc46ffcb783c9131aace9de 100644 --- a/tools/testing/selftests/vm/mremap_test.c +++ b/tools/testing/selftests/vm/mremap_test.c @@ -118,6 +118,50 @@ static unsigned long long get_mmap_min_addr(void) return addr; } +/* + * This test validates that merge is called when expanding a mapping. + * Mapping containing three pages is created, middle page is unmapped + * and then the mapping containing the first page is expanded so that + * it fills the created hole. The two parts should merge creating + * single mapping with three pages. + */ +static void mremap_expand_merge(unsigned long page_size) +{ + char *test_name = "mremap expand merge"; + FILE *fp; + char *line = NULL; + size_t len = 0; + bool success = false; + char *start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + munmap(start + page_size, page_size); + mremap(start, page_size, 2 * page_size, 0); + + fp = fopen("/proc/self/maps", "r"); + if (fp == NULL) { + ksft_test_result_fail("%s\n", test_name); + return; + } + + while (getline(&line, &len, fp) != -1) { + char *first = strtok(line, "- "); + void *first_val = (void *)strtol(first, NULL, 16); + char *second = strtok(NULL, "- "); + void *second_val = (void *) strtol(second, NULL, 16); + + if (first_val == start && second_val == start + 3 * page_size) { + success = true; + break; + } + } + if (success) + ksft_test_result_pass("%s\n", test_name); + else + ksft_test_result_fail("%s\n", test_name); + fclose(fp); +} + /* * Returns the start address of the mapping on success, else returns * NULL on failure. @@ -336,6 +380,7 @@ int main(int argc, char **argv) int i, run_perf_tests; unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; unsigned int pattern_seed; + int num_expand_tests = 1; struct test test_cases[MAX_TEST]; struct test perf_test_cases[MAX_PERF_TEST]; int page_size; @@ -407,12 +452,14 @@ int main(int argc, char **argv) (threshold_mb * _1MB >= _1GB); ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? - ARRAY_SIZE(perf_test_cases) : 0)); + ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests); for (i = 0; i < ARRAY_SIZE(test_cases); i++) run_mremap_test_case(test_cases[i], &failures, threshold_mb, pattern_seed); + mremap_expand_merge(page_size); + if (run_perf_tests) { ksft_print_msg("\n%s\n", "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); diff --git a/tools/testing/selftests/vm/run_vmtests.sh b/tools/testing/selftests/vm/run_vmtests.sh index de86983b8a0f35839cbff9e529cff6863fb8aecc..e780e76c26b874f4b92f0ea0099270a487026e37 100755 --- a/tools/testing/selftests/vm/run_vmtests.sh +++ b/tools/testing/selftests/vm/run_vmtests.sh @@ -120,11 +120,16 @@ run_test ./gup_test -a # Dump pages 0, 19, and 4096, using pin_user_pages: run_test ./gup_test -ct -F 0x1 0 19 0x1000 -run_test ./userfaultfd anon 20 16 -# Test requires source and destination huge pages. Size of source -# (half_ufd_size_MB) is passed as argument to test. -run_test ./userfaultfd hugetlb "$half_ufd_size_MB" 32 -run_test ./userfaultfd shmem 20 16 +uffd_mods=("" ":dev") +for mod in "${uffd_mods[@]}"; do + run_test ./userfaultfd anon${mod} 20 16 + # Hugetlb tests require source and destination huge pages. Pass in half + # the size ($half_ufd_size_MB), which is used for *each*. + run_test ./userfaultfd hugetlb${mod} "$half_ufd_size_MB" 32 + run_test ./userfaultfd hugetlb_shared${mod} "$half_ufd_size_MB" 32 "$mnt"/uffd-test + rm -f "$mnt"/uffd-test + run_test ./userfaultfd shmem${mod} 20 16 +done #cleanup umount "$mnt" diff --git a/tools/testing/selftests/vm/soft-dirty.c b/tools/testing/selftests/vm/soft-dirty.c index e3a43f5d4fa2bb8f3916161a0efb0eb39ac657f8..21d8830c5f243aaf32a0b5eeae181775225056de 100644 --- a/tools/testing/selftests/vm/soft-dirty.c +++ b/tools/testing/selftests/vm/soft-dirty.c @@ -91,7 +91,7 @@ static void test_hugepage(int pagemap_fd, int pagesize) for (i = 0; i < hpage_len; i++) map[i] = (char)i; - if (check_huge(map)) { + if (check_huge_anon(map, 1, hpage_len)) { ksft_test_result_pass("Test %s huge page allocation\n", __func__); clear_softdirty(); diff --git a/tools/testing/selftests/vm/split_huge_page_test.c b/tools/testing/selftests/vm/split_huge_page_test.c index 6aa2b8253aeda158aed9d07747962cfccfc324a1..76e1c36dd9e5777b43ba5958bfdac46a6a5a0a71 100644 --- a/tools/testing/selftests/vm/split_huge_page_test.c +++ b/tools/testing/selftests/vm/split_huge_page_test.c @@ -92,7 +92,6 @@ void split_pmd_thp(void) { char *one_page; size_t len = 4 * pmd_pagesize; - uint64_t thp_size; size_t i; one_page = memalign(pmd_pagesize, len); @@ -107,8 +106,7 @@ void split_pmd_thp(void) for (i = 0; i < len; i++) one_page[i] = (char)i; - thp_size = check_huge(one_page); - if (!thp_size) { + if (!check_huge_anon(one_page, 1, pmd_pagesize)) { printf("No THP is allocated\n"); exit(EXIT_FAILURE); } @@ -124,9 +122,8 @@ void split_pmd_thp(void) } - thp_size = check_huge(one_page); - if (thp_size) { - printf("Still %ld kB AnonHugePages not split\n", thp_size); + if (check_huge_anon(one_page, 0, pmd_pagesize)) { + printf("Still AnonHugePages not split\n"); exit(EXIT_FAILURE); } @@ -172,8 +169,7 @@ void split_pte_mapped_thp(void) for (i = 0; i < len; i++) one_page[i] = (char)i; - thp_size = check_huge(one_page); - if (!thp_size) { + if (!check_huge_anon(one_page, 1, pmd_pagesize)) { printf("No THP is allocated\n"); exit(EXIT_FAILURE); } diff --git a/tools/testing/selftests/vm/test_hmm.sh b/tools/testing/selftests/vm/test_hmm.sh index 539c9371e592a127beb55404b8a56be9d9eecc3a..46e19b5d648d61b04ecc9739b3f8e907c5652634 100755 --- a/tools/testing/selftests/vm/test_hmm.sh +++ b/tools/testing/selftests/vm/test_hmm.sh @@ -52,21 +52,11 @@ load_driver() usage fi fi - if [ $? == 0 ]; then - major=$(awk "\$2==\"HMM_DMIRROR\" {print \$1}" /proc/devices) - mknod /dev/hmm_dmirror0 c $major 0 - mknod /dev/hmm_dmirror1 c $major 1 - if [ $# -eq 2 ]; then - mknod /dev/hmm_dmirror2 c $major 2 - mknod /dev/hmm_dmirror3 c $major 3 - fi - fi } unload_driver() { modprobe -r $DRIVER > /dev/null 2>&1 - rm -f /dev/hmm_dmirror? } run_smoke() diff --git a/tools/testing/selftests/vm/userfaultfd.c b/tools/testing/selftests/vm/userfaultfd.c index 7c3f1b0ab468a02506e00cdb92116ce506e3863c..297f250c1d9564023a67e4fa72a0de02993088a3 100644 --- a/tools/testing/selftests/vm/userfaultfd.c +++ b/tools/testing/selftests/vm/userfaultfd.c @@ -61,10 +61,11 @@ #include <sys/random.h> #include "../kselftest.h" +#include "vm_util.h" #ifdef __NR_userfaultfd -static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size; +static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size, hpage_size; #define BOUNCE_RANDOM (1<<0) #define BOUNCE_RACINGFAULTS (1<<1) @@ -77,6 +78,13 @@ static int bounces; #define TEST_SHMEM 3 static int test_type; +#define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) + +#define BASE_PMD_ADDR ((void *)(1UL << 30)) + +/* test using /dev/userfaultfd, instead of userfaultfd(2) */ +static bool test_dev_userfaultfd; + /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */ #define ALARM_INTERVAL_SECS 10 static volatile bool test_uffdio_copy_eexist = true; @@ -92,9 +100,10 @@ static int huge_fd; static unsigned long long *count_verify; static int uffd = -1; static int uffd_flags, finished, *pipefd; -static char *area_src, *area_src_alias, *area_dst, *area_dst_alias; +static char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; static char *zeropage; pthread_attr_t attr; +static bool test_collapse; /* Userfaultfd test statistics */ struct uffd_stats { @@ -122,9 +131,13 @@ struct uffd_stats { #define swap(a, b) \ do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0) +#define factor_of_2(x) ((x) ^ ((x) & ((x) - 1))) + const char *examples = "# Run anonymous memory test on 100MiB region with 99999 bounces:\n" "./userfaultfd anon 100 99999\n\n" + "# Run the same anonymous memory test, but using /dev/userfaultfd:\n" + "./userfaultfd anon:dev 100 99999\n\n" "# Run share memory test on 1GiB region with 99 bounces:\n" "./userfaultfd shmem 1000 99\n\n" "# Run hugetlb memory test on 256MiB region with 50 bounces:\n" @@ -141,6 +154,16 @@ static void usage(void) "[hugetlbfs_file]\n\n"); fprintf(stderr, "Supported <test type>: anon, hugetlb, " "hugetlb_shared, shmem\n\n"); + fprintf(stderr, "'Test mods' can be joined to the test type string with a ':'. " + "Supported mods:\n"); + fprintf(stderr, "\tsyscall - Use userfaultfd(2) (default)\n"); + fprintf(stderr, "\tdev - Use /dev/userfaultfd instead of userfaultfd(2)\n"); + fprintf(stderr, "\tcollapse - Test MADV_COLLAPSE of UFFDIO_REGISTER_MODE_MINOR\n" + "memory\n"); + fprintf(stderr, "\nExample test mod usage:\n"); + fprintf(stderr, "# Run anonymous memory test with /dev/userfaultfd:\n"); + fprintf(stderr, "./userfaultfd anon:dev 100 99999\n\n"); + fprintf(stderr, "Examples:\n\n"); fprintf(stderr, "%s", examples); exit(1); @@ -154,12 +177,14 @@ static void usage(void) ret, __LINE__); \ } while (0) -#define err(fmt, ...) \ +#define errexit(exitcode, fmt, ...) \ do { \ _err(fmt, ##__VA_ARGS__); \ - exit(1); \ + exit(exitcode); \ } while (0) +#define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__) + static void uffd_stats_reset(struct uffd_stats *uffd_stats, unsigned long n_cpus) { @@ -212,12 +237,10 @@ static void anon_release_pages(char *rel_area) err("madvise(MADV_DONTNEED) failed"); } -static void anon_allocate_area(void **alloc_area) +static void anon_allocate_area(void **alloc_area, bool is_src) { *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (*alloc_area == MAP_FAILED) - err("mmap of anonymous memory failed"); } static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset) @@ -235,7 +258,7 @@ static void hugetlb_release_pages(char *rel_area) } } -static void hugetlb_allocate_area(void **alloc_area) +static void hugetlb_allocate_area(void **alloc_area, bool is_src) { void *area_alias = NULL; char **alloc_area_alias; @@ -245,7 +268,7 @@ static void hugetlb_allocate_area(void **alloc_area) nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | - (*alloc_area == area_src ? 0 : MAP_NORESERVE), + (is_src ? 0 : MAP_NORESERVE), -1, 0); else @@ -253,9 +276,9 @@ static void hugetlb_allocate_area(void **alloc_area) nr_pages * page_size, PROT_READ | PROT_WRITE, MAP_SHARED | - (*alloc_area == area_src ? 0 : MAP_NORESERVE), + (is_src ? 0 : MAP_NORESERVE), huge_fd, - *alloc_area == area_src ? 0 : nr_pages * page_size); + is_src ? 0 : nr_pages * page_size); if (*alloc_area == MAP_FAILED) err("mmap of hugetlbfs file failed"); @@ -265,12 +288,12 @@ static void hugetlb_allocate_area(void **alloc_area) PROT_READ | PROT_WRITE, MAP_SHARED, huge_fd, - *alloc_area == area_src ? 0 : nr_pages * page_size); + is_src ? 0 : nr_pages * page_size); if (area_alias == MAP_FAILED) err("mmap of hugetlb file alias failed"); } - if (*alloc_area == area_src) { + if (is_src) { alloc_area_alias = &area_src_alias; } else { alloc_area_alias = &area_dst_alias; @@ -293,21 +316,36 @@ static void shmem_release_pages(char *rel_area) err("madvise(MADV_REMOVE) failed"); } -static void shmem_allocate_area(void **alloc_area) +static void shmem_allocate_area(void **alloc_area, bool is_src) { void *area_alias = NULL; - bool is_src = alloc_area == (void **)&area_src; - unsigned long offset = is_src ? 0 : nr_pages * page_size; + size_t bytes = nr_pages * page_size; + unsigned long offset = is_src ? 0 : bytes; + char *p = NULL, *p_alias = NULL; + + if (test_collapse) { + p = BASE_PMD_ADDR; + if (!is_src) + /* src map + alias + interleaved hpages */ + p += 2 * (bytes + hpage_size); + p_alias = p; + p_alias += bytes; + p_alias += hpage_size; /* Prevent src/dst VMA merge */ + } - *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, - MAP_SHARED, shm_fd, offset); + *alloc_area = mmap(p, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, + shm_fd, offset); if (*alloc_area == MAP_FAILED) err("mmap of memfd failed"); + if (test_collapse && *alloc_area != p) + err("mmap of memfd failed at %p", p); - area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE, - MAP_SHARED, shm_fd, offset); + area_alias = mmap(p_alias, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, + shm_fd, offset); if (area_alias == MAP_FAILED) err("mmap of memfd alias failed"); + if (test_collapse && area_alias != p_alias) + err("mmap of anonymous memory failed at %p", p_alias); if (is_src) area_src_alias = area_alias; @@ -320,28 +358,39 @@ static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset) *start = (unsigned long)area_dst_alias + offset; } +static void shmem_check_pmd_mapping(void *p, int expect_nr_hpages) +{ + if (!check_huge_shmem(area_dst_alias, expect_nr_hpages, hpage_size)) + err("Did not find expected %d number of hugepages", + expect_nr_hpages); +} + struct uffd_test_ops { - void (*allocate_area)(void **alloc_area); + void (*allocate_area)(void **alloc_area, bool is_src); void (*release_pages)(char *rel_area); void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset); + void (*check_pmd_mapping)(void *p, int expect_nr_hpages); }; static struct uffd_test_ops anon_uffd_test_ops = { .allocate_area = anon_allocate_area, .release_pages = anon_release_pages, .alias_mapping = noop_alias_mapping, + .check_pmd_mapping = NULL, }; static struct uffd_test_ops shmem_uffd_test_ops = { .allocate_area = shmem_allocate_area, .release_pages = shmem_release_pages, .alias_mapping = shmem_alias_mapping, + .check_pmd_mapping = shmem_check_pmd_mapping, }; static struct uffd_test_ops hugetlb_uffd_test_ops = { .allocate_area = hugetlb_allocate_area, .release_pages = hugetlb_release_pages, .alias_mapping = hugetlb_alias_mapping, + .check_pmd_mapping = NULL, }; static struct uffd_test_ops *uffd_test_ops; @@ -383,13 +432,34 @@ static void assert_expected_ioctls_present(uint64_t mode, uint64_t ioctls) } } +static int __userfaultfd_open_dev(void) +{ + int fd, _uffd; + + fd = open("/dev/userfaultfd", O_RDWR | O_CLOEXEC); + if (fd < 0) + errexit(KSFT_SKIP, "opening /dev/userfaultfd failed"); + + _uffd = ioctl(fd, USERFAULTFD_IOC_NEW, UFFD_FLAGS); + if (_uffd < 0) + errexit(errno == ENOTTY ? KSFT_SKIP : 1, + "creating userfaultfd failed"); + close(fd); + return _uffd; +} + static void userfaultfd_open(uint64_t *features) { struct uffdio_api uffdio_api; - uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); - if (uffd < 0) - err("userfaultfd syscall not available in this kernel"); + if (test_dev_userfaultfd) + uffd = __userfaultfd_open_dev(); + else { + uffd = syscall(__NR_userfaultfd, UFFD_FLAGS); + if (uffd < 0) + errexit(errno == ENOSYS ? KSFT_SKIP : 1, + "creating userfaultfd failed"); + } uffd_flags = fcntl(uffd, F_GETFD, NULL); uffdio_api.api = UFFD_API; @@ -440,6 +510,7 @@ static void uffd_test_ctx_clear(void) munmap_area((void **)&area_src_alias); munmap_area((void **)&area_dst); munmap_area((void **)&area_dst_alias); + munmap_area((void **)&area_remap); } static void uffd_test_ctx_init(uint64_t features) @@ -448,8 +519,8 @@ static void uffd_test_ctx_init(uint64_t features) uffd_test_ctx_clear(); - uffd_test_ops->allocate_area((void **)&area_src); - uffd_test_ops->allocate_area((void **)&area_dst); + uffd_test_ops->allocate_area((void **)&area_src, true); + uffd_test_ops->allocate_area((void **)&area_dst, false); userfaultfd_open(&features); @@ -703,7 +774,27 @@ static void uffd_handle_page_fault(struct uffd_msg *msg, continue_range(uffd, msg->arg.pagefault.address, page_size); stats->minor_faults++; } else { - /* Missing page faults */ + /* + * Missing page faults. + * + * Here we force a write check for each of the missing mode + * faults. It's guaranteed because the only threads that + * will trigger uffd faults are the locking threads, and + * their first instruction to touch the missing page will + * always be pthread_mutex_lock(). + * + * Note that here we relied on an NPTL glibc impl detail to + * always read the lock type at the entry of the lock op + * (pthread_mutex_t.__data.__type, offset 0x10) before + * doing any locking operations to guarantee that. It's + * actually not good to rely on this impl detail because + * logically a pthread-compatible lib can implement the + * locks without types and we can fail when linking with + * them. However since we used to find bugs with this + * strict check we still keep it around. Hopefully this + * could be a good hint when it fails again. If one day + * it'll break on some other impl of glibc we'll revisit. + */ if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE) err("unexpected write fault"); @@ -766,6 +857,7 @@ static void *uffd_poll_thread(void *arg) err("remove failure"); break; case UFFD_EVENT_REMAP: + area_remap = area_dst; /* save for later unmap */ area_dst = (char *)(unsigned long)msg.arg.remap.to; break; } @@ -1218,13 +1310,30 @@ static int userfaultfd_sig_test(void) return userfaults != 0; } +void check_memory_contents(char *p) +{ + unsigned long i; + uint8_t expected_byte; + void *expected_page; + + if (posix_memalign(&expected_page, page_size, page_size)) + err("out of memory"); + + for (i = 0; i < nr_pages; ++i) { + expected_byte = ~((uint8_t)(i % ((uint8_t)-1))); + memset(expected_page, expected_byte, page_size); + if (my_bcmp(expected_page, p + (i * page_size), page_size)) + err("unexpected page contents after minor fault"); + } + + free(expected_page); +} + static int userfaultfd_minor_test(void) { - struct uffdio_register uffdio_register; unsigned long p; + struct uffdio_register uffdio_register; pthread_t uffd_mon; - uint8_t expected_byte; - void *expected_page; char c; struct uffd_stats stats = { 0 }; @@ -1263,17 +1372,7 @@ static int userfaultfd_minor_test(void) * fault. uffd_poll_thread will resolve the fault by bit-flipping the * page's contents, and then issuing a CONTINUE ioctl. */ - - if (posix_memalign(&expected_page, page_size, page_size)) - err("out of memory"); - - for (p = 0; p < nr_pages; ++p) { - expected_byte = ~((uint8_t)(p % ((uint8_t)-1))); - memset(expected_page, expected_byte, page_size); - if (my_bcmp(expected_page, area_dst_alias + (p * page_size), - page_size)) - err("unexpected page contents after minor fault"); - } + check_memory_contents(area_dst_alias); if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) err("pipe write"); @@ -1282,6 +1381,23 @@ static int userfaultfd_minor_test(void) uffd_stats_report(&stats, 1); + if (test_collapse) { + printf("testing collapse of uffd memory into PMD-mapped THPs:"); + if (madvise(area_dst_alias, nr_pages * page_size, + MADV_COLLAPSE)) + err("madvise(MADV_COLLAPSE)"); + + uffd_test_ops->check_pmd_mapping(area_dst, + nr_pages * page_size / + hpage_size); + /* + * This won't cause uffd-fault - it purely just makes sure there + * was no corruption. + */ + check_memory_contents(area_dst_alias); + printf(" done.\n"); + } + return stats.missing_faults != 0 || stats.minor_faults != nr_pages; } @@ -1584,8 +1700,6 @@ unsigned long default_huge_page_size(void) static void set_test_type(const char *type) { - uint64_t features = UFFD_API_FEATURES; - if (!strcmp(type, "anon")) { test_type = TEST_ANON; uffd_test_ops = &anon_uffd_test_ops; @@ -1603,12 +1717,37 @@ static void set_test_type(const char *type) test_type = TEST_SHMEM; uffd_test_ops = &shmem_uffd_test_ops; test_uffdio_minor = true; - } else { - err("Unknown test type: %s", type); } +} + +static void parse_test_type_arg(const char *raw_type) +{ + char *buf = strdup(raw_type); + uint64_t features = UFFD_API_FEATURES; + + while (buf) { + const char *token = strsep(&buf, ":"); + + if (!test_type) + set_test_type(token); + else if (!strcmp(token, "dev")) + test_dev_userfaultfd = true; + else if (!strcmp(token, "syscall")) + test_dev_userfaultfd = false; + else if (!strcmp(token, "collapse")) + test_collapse = true; + else + err("unrecognized test mod '%s'", token); + } + + if (!test_type) + err("failed to parse test type argument: '%s'", raw_type); + + if (test_collapse && test_type != TEST_SHMEM) + err("Unsupported test: %s", raw_type); if (test_type == TEST_HUGETLB) - page_size = default_huge_page_size(); + page_size = hpage_size; else page_size = sysconf(_SC_PAGE_SIZE); @@ -1646,6 +1785,8 @@ static void sigalrm(int sig) int main(int argc, char **argv) { + size_t bytes; + if (argc < 4) usage(); @@ -1653,11 +1794,41 @@ int main(int argc, char **argv) err("failed to arm SIGALRM"); alarm(ALARM_INTERVAL_SECS); - set_test_type(argv[1]); + hpage_size = default_huge_page_size(); + parse_test_type_arg(argv[1]); + bytes = atol(argv[2]) * 1024 * 1024; + + if (test_collapse && bytes & (hpage_size - 1)) + err("MiB must be multiple of %lu if :collapse mod set", + hpage_size >> 20); nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); - nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size / - nr_cpus; + + if (test_collapse) { + /* nr_cpus must divide (bytes / page_size), otherwise, + * area allocations of (nr_pages * paze_size) won't be a + * multiple of hpage_size, even if bytes is a multiple of + * hpage_size. + * + * This means that nr_cpus must divide (N * (2 << (H-P)) + * where: + * bytes = hpage_size * N + * hpage_size = 2 << H + * page_size = 2 << P + * + * And we want to chose nr_cpus to be the largest value + * satisfying this constraint, not larger than the number + * of online CPUs. Unfortunately, prime factorization of + * N and nr_cpus may be arbitrary, so have to search for it. + * Instead, just use the highest power of 2 dividing both + * nr_cpus and (bytes / page_size). + */ + int x = factor_of_2(nr_cpus); + int y = factor_of_2(bytes / page_size); + + nr_cpus = x < y ? x : y; + } + nr_pages_per_cpu = bytes / page_size / nr_cpus; if (!nr_pages_per_cpu) { _err("invalid MiB"); usage(); diff --git a/tools/testing/selftests/vm/vm_util.c b/tools/testing/selftests/vm/vm_util.c index b58ab11a7a302a96b2ce297f9aa23c1e0e9bfa7e..f11f8adda521867f52097aba2babca730941affa 100644 --- a/tools/testing/selftests/vm/vm_util.c +++ b/tools/testing/selftests/vm/vm_util.c @@ -42,9 +42,9 @@ void clear_softdirty(void) ksft_exit_fail_msg("writing clear_refs failed\n"); } -static bool check_for_pattern(FILE *fp, const char *pattern, char *buf) +bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len) { - while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) { + while (fgets(buf, len, fp)) { if (!strncmp(buf, pattern, strlen(pattern))) return true; } @@ -72,9 +72,10 @@ uint64_t read_pmd_pagesize(void) return strtoul(buf, NULL, 10); } -uint64_t check_huge(void *addr) +bool __check_huge(void *addr, char *pattern, int nr_hpages, + uint64_t hpage_size) { - uint64_t thp = 0; + uint64_t thp = -1; int ret; FILE *fp; char buffer[MAX_LINE_LENGTH]; @@ -89,20 +90,37 @@ uint64_t check_huge(void *addr) if (!fp) ksft_exit_fail_msg("%s: Failed to open file %s\n", __func__, SMAP_FILE_PATH); - if (!check_for_pattern(fp, addr_pattern, buffer)) + if (!check_for_pattern(fp, addr_pattern, buffer, sizeof(buffer))) goto err_out; /* - * Fetch the AnonHugePages: in the same block and check the number of + * Fetch the pattern in the same block and check the number of * hugepages. */ - if (!check_for_pattern(fp, "AnonHugePages:", buffer)) + if (!check_for_pattern(fp, pattern, buffer, sizeof(buffer))) goto err_out; - if (sscanf(buffer, "AnonHugePages:%10ld kB", &thp) != 1) + snprintf(addr_pattern, MAX_LINE_LENGTH, "%s%%9ld kB", pattern); + + if (sscanf(buffer, addr_pattern, &thp) != 1) ksft_exit_fail_msg("Reading smap error\n"); err_out: fclose(fp); - return thp; + return thp == (nr_hpages * (hpage_size >> 10)); +} + +bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size) +{ + return __check_huge(addr, "AnonHugePages: ", nr_hpages, hpage_size); +} + +bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size) +{ + return __check_huge(addr, "FilePmdMapped:", nr_hpages, hpage_size); +} + +bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size) +{ + return __check_huge(addr, "ShmemPmdMapped:", nr_hpages, hpage_size); } diff --git a/tools/testing/selftests/vm/vm_util.h b/tools/testing/selftests/vm/vm_util.h index 2e512bd57ae141e40f3494f3b82124aac1a1bd00..5c35de454e08f33c22faa089f593d79fc3a4d0d2 100644 --- a/tools/testing/selftests/vm/vm_util.h +++ b/tools/testing/selftests/vm/vm_util.h @@ -5,5 +5,8 @@ uint64_t pagemap_get_entry(int fd, char *start); bool pagemap_is_softdirty(int fd, char *start); void clear_softdirty(void); +bool check_for_pattern(FILE *fp, const char *pattern, char *buf, size_t len); uint64_t read_pmd_pagesize(void); -uint64_t check_huge(void *addr); +bool check_huge_anon(void *addr, int nr_hpages, uint64_t hpage_size); +bool check_huge_file(void *addr, int nr_hpages, uint64_t hpage_size); +bool check_huge_shmem(void *addr, int nr_hpages, uint64_t hpage_size); diff --git a/tools/testing/selftests/wireguard/qemu/Makefile b/tools/testing/selftests/wireguard/qemu/Makefile index fda76282d34b8381d4ca8d1827c78d9ccae98970..e95bd56b332f757321db82ead8181179fce4376b 100644 --- a/tools/testing/selftests/wireguard/qemu/Makefile +++ b/tools/testing/selftests/wireguard/qemu/Makefile @@ -343,8 +343,10 @@ $(KERNEL_BZIMAGE): $(TOOLCHAIN_PATH)/.installed $(KERNEL_BUILD_PATH)/.config $(B .PHONY: $(KERNEL_BZIMAGE) $(TOOLCHAIN_PATH)/$(CHOST)/include/linux/.installed: | $(KERNEL_BUILD_PATH)/.config $(TOOLCHAIN_PATH)/.installed +ifneq ($(ARCH),um) rm -rf $(TOOLCHAIN_PATH)/$(CHOST)/include/linux $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) INSTALL_HDR_PATH=$(TOOLCHAIN_PATH)/$(CHOST) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) headers_install +endif touch $@ $(TOOLCHAIN_PATH)/.installed: $(TOOLCHAIN_TAR) diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c index dc577461afc20546e44f700ddd49b83b78ececb4..bb6d691cb30d0d2fccdd1ea53c824f83ebde11c7 100644 --- a/tools/testing/vsock/vsock_test.c +++ b/tools/testing/vsock/vsock_test.c @@ -18,6 +18,7 @@ #include <sys/socket.h> #include <time.h> #include <sys/mman.h> +#include <poll.h> #include "timeout.h" #include "control.h" @@ -596,6 +597,108 @@ static void test_seqpacket_invalid_rec_buffer_server(const struct test_opts *opt close(fd); } +#define RCVLOWAT_BUF_SIZE 128 + +static void test_stream_poll_rcvlowat_server(const struct test_opts *opts) +{ + int fd; + int i; + + fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); + if (fd < 0) { + perror("accept"); + exit(EXIT_FAILURE); + } + + /* Send 1 byte. */ + send_byte(fd, 1, 0); + + control_writeln("SRVSENT"); + + /* Wait until client is ready to receive rest of data. */ + control_expectln("CLNSENT"); + + for (i = 0; i < RCVLOWAT_BUF_SIZE - 1; i++) + send_byte(fd, 1, 0); + + /* Keep socket in active state. */ + control_expectln("POLLDONE"); + + close(fd); +} + +static void test_stream_poll_rcvlowat_client(const struct test_opts *opts) +{ + unsigned long lowat_val = RCVLOWAT_BUF_SIZE; + char buf[RCVLOWAT_BUF_SIZE]; + struct pollfd fds; + ssize_t read_res; + short poll_flags; + int fd; + + fd = vsock_stream_connect(opts->peer_cid, 1234); + if (fd < 0) { + perror("connect"); + exit(EXIT_FAILURE); + } + + if (setsockopt(fd, SOL_SOCKET, SO_RCVLOWAT, + &lowat_val, sizeof(lowat_val))) { + perror("setsockopt"); + exit(EXIT_FAILURE); + } + + control_expectln("SRVSENT"); + + /* At this point, server sent 1 byte. */ + fds.fd = fd; + poll_flags = POLLIN | POLLRDNORM; + fds.events = poll_flags; + + /* Try to wait for 1 sec. */ + if (poll(&fds, 1, 1000) < 0) { + perror("poll"); + exit(EXIT_FAILURE); + } + + /* poll() must return nothing. */ + if (fds.revents) { + fprintf(stderr, "Unexpected poll result %hx\n", + fds.revents); + exit(EXIT_FAILURE); + } + + /* Tell server to send rest of data. */ + control_writeln("CLNSENT"); + + /* Poll for data. */ + if (poll(&fds, 1, 10000) < 0) { + perror("poll"); + exit(EXIT_FAILURE); + } + + /* Only these two bits are expected. */ + if (fds.revents != poll_flags) { + fprintf(stderr, "Unexpected poll result %hx\n", + fds.revents); + exit(EXIT_FAILURE); + } + + /* Use MSG_DONTWAIT, if call is going to wait, EAGAIN + * will be returned. + */ + read_res = recv(fd, buf, sizeof(buf), MSG_DONTWAIT); + if (read_res != RCVLOWAT_BUF_SIZE) { + fprintf(stderr, "Unexpected recv result %zi\n", + read_res); + exit(EXIT_FAILURE); + } + + control_writeln("POLLDONE"); + + close(fd); +} + static struct test_case test_cases[] = { { .name = "SOCK_STREAM connection reset", @@ -646,6 +749,11 @@ static struct test_case test_cases[] = { .run_client = test_seqpacket_invalid_rec_buffer_client, .run_server = test_seqpacket_invalid_rec_buffer_server, }, + { + .name = "SOCK_STREAM poll() + SO_RCVLOWAT", + .run_client = test_stream_poll_rcvlowat_client, + .run_server = test_stream_poll_rcvlowat_server, + }, {}, }; diff --git a/tools/verification/dot2/dot2k_templates/main_global.c b/tools/verification/dot2/dot2k_templates/main_global.c index f4b712dbc92e07d95fd18bc5d3ae169c29e69e5a..a5658bfb904455c84a72fc2723315b1754412c97 100644 --- a/tools/verification/dot2/dot2k_templates/main_global.c +++ b/tools/verification/dot2/dot2k_templates/main_global.c @@ -27,7 +27,7 @@ * * The rv monitor reference is needed for the monitor declaration. */ -struct rv_monitor rv_MODEL_NAME; +static struct rv_monitor rv_MODEL_NAME; DECLARE_DA_MON_GLOBAL(MODEL_NAME, MIN_TYPE); /* @@ -63,7 +63,7 @@ TRACEPOINT_DETACH /* * This is the monitor register section. */ -struct rv_monitor rv_MODEL_NAME = { +static struct rv_monitor rv_MODEL_NAME = { .name = "MODEL_NAME", .description = "auto-generated MODEL_NAME", .enable = enable_MODEL_NAME, @@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { .enabled = 0, }; -static int register_MODEL_NAME(void) +static int __init register_MODEL_NAME(void) { rv_register_monitor(&rv_MODEL_NAME); return 0; } -static void unregister_MODEL_NAME(void) +static void __exit unregister_MODEL_NAME(void) { rv_unregister_monitor(&rv_MODEL_NAME); } diff --git a/tools/verification/dot2/dot2k_templates/main_per_cpu.c b/tools/verification/dot2/dot2k_templates/main_per_cpu.c index 4080d1ca3354a7c836467531dde611b065e06021..03539a97633f144555de3c06789b026624851e88 100644 --- a/tools/verification/dot2/dot2k_templates/main_per_cpu.c +++ b/tools/verification/dot2/dot2k_templates/main_per_cpu.c @@ -27,7 +27,7 @@ * * The rv monitor reference is needed for the monitor declaration. */ -struct rv_monitor rv_MODEL_NAME; +static struct rv_monitor rv_MODEL_NAME; DECLARE_DA_MON_PER_CPU(MODEL_NAME, MIN_TYPE); /* @@ -63,7 +63,7 @@ TRACEPOINT_DETACH /* * This is the monitor register section. */ -struct rv_monitor rv_MODEL_NAME = { +static struct rv_monitor rv_MODEL_NAME = { .name = "MODEL_NAME", .description = "auto-generated MODEL_NAME", .enable = enable_MODEL_NAME, @@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { .enabled = 0, }; -static int register_MODEL_NAME(void) +static int __init register_MODEL_NAME(void) { rv_register_monitor(&rv_MODEL_NAME); return 0; } -static void unregister_MODEL_NAME(void) +static void __exit unregister_MODEL_NAME(void) { rv_unregister_monitor(&rv_MODEL_NAME); } diff --git a/tools/verification/dot2/dot2k_templates/main_per_task.c b/tools/verification/dot2/dot2k_templates/main_per_task.c index 89197175384f7e1b18f35c90f0e8af89a6f182e4..ffd92af87a86945f56f0c5138d71f1f498a45ae7 100644 --- a/tools/verification/dot2/dot2k_templates/main_per_task.c +++ b/tools/verification/dot2/dot2k_templates/main_per_task.c @@ -27,7 +27,7 @@ * * The rv monitor reference is needed for the monitor declaration. */ -struct rv_monitor rv_MODEL_NAME; +static struct rv_monitor rv_MODEL_NAME; DECLARE_DA_MON_PER_TASK(MODEL_NAME, MIN_TYPE); /* @@ -63,7 +63,7 @@ TRACEPOINT_DETACH /* * This is the monitor register section. */ -struct rv_monitor rv_MODEL_NAME = { +static struct rv_monitor rv_MODEL_NAME = { .name = "MODEL_NAME", .description = "auto-generated MODEL_NAME", .enable = enable_MODEL_NAME, @@ -72,13 +72,13 @@ struct rv_monitor rv_MODEL_NAME = { .enabled = 0, }; -static int register_MODEL_NAME(void) +static int __init register_MODEL_NAME(void) { rv_register_monitor(&rv_MODEL_NAME); return 0; } -static void unregister_MODEL_NAME(void) +static void __exit unregister_MODEL_NAME(void) { rv_unregister_monitor(&rv_MODEL_NAME); } diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h index 363b982283011542ef05cd495987a6305bd77599..5d3440f474dd3770fc7bd58bd44d9f3a645d7f3e 100644 --- a/tools/virtio/linux/virtio.h +++ b/tools/virtio/linux/virtio.h @@ -14,6 +14,7 @@ struct virtio_device { u64 features; struct list_head vqs; spinlock_t vqs_list_lock; + const struct virtio_config_ops *config; }; struct virtqueue { @@ -23,7 +24,9 @@ struct virtqueue { struct virtio_device *vdev; unsigned int index; unsigned int num_free; + unsigned int num_max; void *priv; + bool reset; }; /* Interfaces exported by virtio_ring. */ diff --git a/tools/virtio/linux/virtio_config.h b/tools/virtio/linux/virtio_config.h index f2640e505c4e794363d3df16bed27904dd1f9b60..2a8a70e2a950e6d0ab657752951b5ca567241d4c 100644 --- a/tools/virtio/linux/virtio_config.h +++ b/tools/virtio/linux/virtio_config.h @@ -3,6 +3,11 @@ #include <linux/virtio.h> #include <uapi/linux/virtio_config.h> +struct virtio_config_ops { + int (*disable_vq_and_reset)(struct virtqueue *vq); + int (*enable_vq_after_reset)(struct virtqueue *vq); +}; + /* * __virtio_test_bit - helper to test feature bits. For use by transports. * Devices should normally use virtio_has_feature, diff --git a/tools/vm/page_owner_sort.c b/tools/vm/page_owner_sort.c index ec2e67c85b84995fa911f979bf9c862a16e99577..ce860ab941629778428cb6c8cca3b7e69bb6373f 100644 --- a/tools/vm/page_owner_sort.c +++ b/tools/vm/page_owner_sort.c @@ -470,7 +470,12 @@ static bool match_str_list(const char *str, char **list, int list_size) static bool is_need(char *buf) { - if ((filter & FILTER_UNRELEASE) && get_free_ts_nsec(buf) != 0) + __u64 ts_nsec, free_ts_nsec; + + ts_nsec = get_ts_nsec(buf); + free_ts_nsec = get_free_ts_nsec(buf); + + if ((filter & FILTER_UNRELEASE) && free_ts_nsec != 0 && ts_nsec < free_ts_nsec) return false; if ((filter & FILTER_PID) && !match_num_list(get_pid(buf), fc.pids, fc.pids_size)) return false; diff --git a/usr/gen_init_cpio.c b/usr/gen_init_cpio.c index dc838e26a5b9aa796fdcf867a2563e90d573cb58..ee01e40e8bc65e06ba3b74e3f50d5aaa485bcea1 100644 --- a/usr/gen_init_cpio.c +++ b/usr/gen_init_cpio.c @@ -326,7 +326,7 @@ static int cpio_mkfile(const char *name, const char *location, char s[256]; struct stat buf; unsigned long size; - int file = -1; + int file; int retval; int rc = -1; int namesize; diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index a8c5c9f06b3cf7f3c74158efd3b7d3ec89116eea..800f9470e36b1ca6d522c65c9a1d572ba7575280 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -19,6 +19,20 @@ config HAVE_KVM_IRQ_ROUTING config HAVE_KVM_DIRTY_RING bool +# Only strongly ordered architectures can select this, as it doesn't +# put any explicit constraint on userspace ordering. They can also +# select the _ACQ_REL version. +config HAVE_KVM_DIRTY_RING_TSO + bool + select HAVE_KVM_DIRTY_RING + depends on X86 + +# Weakly ordered architectures can only select this, advertising +# to userspace the additional ordering requirements. +config HAVE_KVM_DIRTY_RING_ACQ_REL + bool + select HAVE_KVM_DIRTY_RING + config HAVE_KVM_EVENTFD bool select EVENTFD diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c index f4c2a6eb1666b99f2e1bf98be6edb874e956ba95..d6fabf238032a6bae3fb3211191874d2ae73213e 100644 --- a/virt/kvm/dirty_ring.c +++ b/virt/kvm/dirty_ring.c @@ -74,7 +74,7 @@ int kvm_dirty_ring_alloc(struct kvm_dirty_ring *ring, int index, u32 size) static inline void kvm_dirty_gfn_set_invalid(struct kvm_dirty_gfn *gfn) { - gfn->flags = 0; + smp_store_release(&gfn->flags, 0); } static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn) @@ -84,7 +84,7 @@ static inline void kvm_dirty_gfn_set_dirtied(struct kvm_dirty_gfn *gfn) static inline bool kvm_dirty_gfn_harvested(struct kvm_dirty_gfn *gfn) { - return gfn->flags & KVM_DIRTY_GFN_F_RESET; + return smp_load_acquire(&gfn->flags) & KVM_DIRTY_GFN_F_RESET; } int kvm_dirty_ring_reset(struct kvm *kvm, struct kvm_dirty_ring *ring) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 584a5bab3af395e392b4f4d83f37c75c25859bdb..e30f1b4ecfa5d1c3110bdbd36108aa417de008a9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3409,10 +3409,8 @@ static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu) int ret = -EINTR; int idx = srcu_read_lock(&vcpu->kvm->srcu); - if (kvm_arch_vcpu_runnable(vcpu)) { - kvm_make_request(KVM_REQ_UNHALT, vcpu); + if (kvm_arch_vcpu_runnable(vcpu)) goto out; - } if (kvm_cpu_has_pending_timer(vcpu)) goto out; if (signal_pending(current)) @@ -4475,7 +4473,13 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_NR_MEMSLOTS: return KVM_USER_MEM_SLOTS; case KVM_CAP_DIRTY_LOG_RING: -#ifdef CONFIG_HAVE_KVM_DIRTY_RING +#ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO + return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn); +#else + return 0; +#endif + case KVM_CAP_DIRTY_LOG_RING_ACQ_REL: +#ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn); #else return 0; @@ -4580,6 +4584,7 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, return 0; } case KVM_CAP_DIRTY_LOG_RING: + case KVM_CAP_DIRTY_LOG_RING_ACQ_REL: return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]); default: return kvm_vm_ioctl_enable_cap(kvm, cap); @@ -5881,7 +5886,7 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, r = kvm_async_pf_init(); if (r) - goto out_free_5; + goto out_free_4; kvm_chardev_ops.owner = module; @@ -5905,10 +5910,9 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, out_unreg: kvm_async_pf_deinit(); -out_free_5: +out_free_4: for_each_possible_cpu(cpu) free_cpumask_var(per_cpu(cpu_kick_mask, cpu)); -out_free_4: kmem_cache_destroy(kvm_vcpu_cache); out_free_3: unregister_reboot_notifier(&kvm_reboot_notifier); diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index ce1b01d02c51976b28f6d4d0a643b007c7916207..495ceabffe88bb7270083e404d6576c7258634a0 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -24,6 +24,9 @@ struct kvm_vfio_group { struct list_head node; struct file *file; +#ifdef CONFIG_SPAPR_TCE_IOMMU + struct iommu_group *iommu_group; +#endif }; struct kvm_vfio { @@ -61,6 +64,23 @@ static bool kvm_vfio_file_enforced_coherent(struct file *file) return ret; } +static bool kvm_vfio_file_is_group(struct file *file) +{ + bool (*fn)(struct file *file); + bool ret; + + fn = symbol_get(vfio_file_is_group); + if (!fn) + return false; + + ret = fn(file); + + symbol_put(vfio_file_is_group); + + return ret; +} + +#ifdef CONFIG_SPAPR_TCE_IOMMU static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file) { struct iommu_group *(*fn)(struct file *file); @@ -77,16 +97,15 @@ static struct iommu_group *kvm_vfio_file_iommu_group(struct file *file) return ret; } -#ifdef CONFIG_SPAPR_TCE_IOMMU static void kvm_spapr_tce_release_vfio_group(struct kvm *kvm, struct kvm_vfio_group *kvg) { - struct iommu_group *grp = kvm_vfio_file_iommu_group(kvg->file); - - if (WARN_ON_ONCE(!grp)) + if (WARN_ON_ONCE(!kvg->iommu_group)) return; - kvm_spapr_tce_release_iommu_group(kvm, grp); + kvm_spapr_tce_release_iommu_group(kvm, kvg->iommu_group); + iommu_group_put(kvg->iommu_group); + kvg->iommu_group = NULL; } #endif @@ -136,7 +155,7 @@ static int kvm_vfio_group_add(struct kvm_device *dev, unsigned int fd) return -EBADF; /* Ensure the FD is a vfio group FD.*/ - if (!kvm_vfio_file_iommu_group(filp)) { + if (!kvm_vfio_file_is_group(filp)) { ret = -EINVAL; goto err_fput; } @@ -236,19 +255,19 @@ static int kvm_vfio_group_set_spapr_tce(struct kvm_device *dev, mutex_lock(&kv->lock); list_for_each_entry(kvg, &kv->group_list, node) { - struct iommu_group *grp; - if (kvg->file != f.file) continue; - grp = kvm_vfio_file_iommu_group(kvg->file); - if (WARN_ON_ONCE(!grp)) { - ret = -EIO; - goto err_fdput; + if (!kvg->iommu_group) { + kvg->iommu_group = kvm_vfio_file_iommu_group(kvg->file); + if (WARN_ON_ONCE(!kvg->iommu_group)) { + ret = -EIO; + goto err_fdput; + } } ret = kvm_spapr_tce_attach_iommu_group(dev->kvm, param.tablefd, - grp); + kvg->iommu_group); break; }